2023-09-23T19:51:30.874 INFO:root:teuthology version: 0.0.1.dev194+g54e62bc 2023-09-23T19:51:30.874 DEBUG:teuthology.run:Teuthology command: teuthology --owner scheduled_teuthology@teuthology --verbose --description upgrade:quincy-x/stress-split/{0-distro/rhel_8.6_container_tools_rhel8 0-roles 1-start 2-first-half-tasks/rbd-import-export 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/classic} --archive /home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:quincy-x-reef-distro-default-smithi/7401824 --name teuthology-2023-09-23_14:23:10-upgrade:quincy-x-reef-distro-default-smithi -- /home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:quincy-x-reef-distro-default-smithi/7401824/orig.config.yaml 2023-09-23T19:51:30.932 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2023-09-23T19:51:31.021 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:quincy-x-reef-distro-default-smithi/7401824 branch: reef description: upgrade:quincy-x/stress-split/{0-distro/rhel_8.6_container_tools_rhel8 0-roles 1-start 2-first-half-tasks/rbd-import-export 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/classic} email: ceph-qa@ceph.io first-half-sequence: - cephadm.shell: env: - sha1 mon.a: - ceph config set mgr mgr/cephadm/daemon_cache_timeout 60 - ceph config set global log_to_journald false --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - ceph orch ps - echo wait for minority of mons to upgrade - while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for majority of mons to upgrade - 'while ! ceph mon versions | grep $sha1 | egrep '': [23]'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for all mons to upgrade - 'while ! ceph mon versions | grep $sha1 | grep '': 3'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for half of osds to upgrade - 'while ! ceph osd versions | grep $sha1 | egrep '': [45678]''; do sleep 2 ; done' - ceph orch upgrade pause - ceph orch ps - ceph orch ps - ceph versions first-half-tasks: - workunit: branch: quincy clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' first_in_suite: false job_id: '7401824' kernel: kdb: true sha1: distro last_in_suite: false machine_type: smithi meta: - desc: 'run basic import/export cli tests for rbd ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' - desc: 'run basic cls tests for rbd ' - desc: 'run basic import/export cli tests for rbd ' - desc: 'librbd C and C++ api tests ' - desc: 'randomized correctness test for rados operations on a replicated pool, using only reads, writes, and deletes ' - desc: 'randomized correctness test for rados operations on a replicated pool with snapshot operations ' - desc: 'run basic import/export cli tests for rbd ' name: teuthology-2023-09-23_14:23:10-upgrade:quincy-x-reef-distro-default-smithi no_nested_subset: false nuke-on-error: true openstack: - volumes: count: 4 size: 10 os_type: rhel os_version: '8.6' overrides: admin_socket: branch: reef ceph: conf: global: mon election default strategy: 1 mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true create_rbd_pool: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 selinux: whitelist: - scontext=system_u:system_r:logrotate_t:s0 workunit: branch: reef sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 owner: scheduled_teuthology@teuthology priority: 100 repo: https://github.com/ceph/ceph.git roles: - - mon.a - mon.c - mgr.y - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - node-exporter.a - alertmanager.a - - mon.b - mgr.x - osd.4 - osd.5 - osd.6 - osd.7 - client.1 - prometheus.a - grafana.a - node-exporter.b second-half-sequence: sequential: - cephadm.shell: env: - sha1 mon.a: - ceph orch upgrade resume - sleep 60 - echo wait for upgrade to complete - while ceph orch upgrade status | jq '.in_progress' | grep true ; do ceph orch ps ; ceph versions ; sleep 30 ; done - echo upgrade complete - ceph orch ps - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 second-half-tasks: - workunit: branch: quincy clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' seed: 4997 sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 sleep_before_teardown: 0 stress-tasks: - thrashosds: aggressive_pg_num_changes: false chance_force_recovery: 0 chance_pgnum_grow: 1 chance_pgpnum_fix: 1 chance_thrash_cluster_full: 0 chance_thrash_pg_upmap: 0 chance_thrash_pg_upmap_items: 0 disable_objectstore_tool_tests: true timeout: 1200 - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' - workunit: branch: quincy clients: client.0: - cls/test_cls_rbd.sh - print: '**** done cls/test_cls_rbd.sh 5-workload' - workunit: branch: quincy clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' - workunit: branch: quincy clients: client.0: - rbd/test_librbd.sh env: RBD_FEATURES: '61' - print: '**** done rbd/test_librbd.sh 7-workload' - full_sequential: - rados: clients: - client.0 objects: 500 op_weights: delete: 10 read: 45 write: 45 ops: 4000 write_append_excl: false - print: '**** done rados/readwrite 5-workload' - full_sequential: - rados: clients: - client.0 objects: 50 op_weights: delete: 50 read: 100 rollback: 50 snap_create: 50 snap_remove: 50 write: 100 ops: 4000 write_append_excl: false - print: '**** done rados/snaps-few-objects 5-workload' suite: upgrade:quincy-x suite_branch: reef suite_path: /home/teuthworker/src/github.com_ceph_ceph_bb0daeff97f020f686596fca6eba20c310e26fb7/qa suite_relpath: qa suite_repo: https://github.com/ceph/ceph.git suite_sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 targets: smithi018.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLVvN3+LmzPZ4ixiwyCkS3KZLZ9R6MOPm+kPtaJbl8fExpzbk8TudXCcJWHBL7DGPWFvye2jT6sei9eiX0EUQYs= smithi149.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLVvN3+LmzPZ4ixiwyCkS3KZLZ9R6MOPm+kPtaJbl8fExpzbk8TudXCcJWHBL7DGPWFvye2jT6sei9eiX0EUQYs= tasks: - pexec: all: - sudo cp /etc/containers/registries.conf /etc/containers/registries.conf.backup - sudo dnf -y module reset container-tools - sudo dnf -y module install container-tools:rhel8 --allowerasing --nobest - sudo cp /etc/containers/registries.conf.backup /etc/containers/registries.conf - install: branch: quincy exclude_packages: - ceph-volume - cephadm: cephadm_branch: quincy cephadm_git_url: https://github.com/ceph/ceph conf: osd: osd_class_default_list: '*' osd_class_load_list: '*' image: quay.ceph.io/ceph-ci/ceph:quincy - cephadm.shell: mon.a: - ceph fs volume create foo - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph.healthy: null - print: '**** upgrading first half of cluster, with stress ****' - parallel: - first-half-tasks - first-half-sequence - print: '**** done upgrading first half of cluster ****' - ceph.healthy: null - print: '**** applying stress + thrashing to mixed-version cluster ****' - parallel: - stress-tasks - ceph.healthy: null - print: '**** finishing upgrade ****' - parallel: - second-half-tasks - second-half-sequence - ceph.healthy: null teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 54e62bcbac4e53d9685e08328b790d3b20d71cae timestamp: 2023-09-23_14:23:10 tube: smithi user: teuthology verbose: true worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2830 2023-09-23T19:51:31.022 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/github.com_ceph_ceph_bb0daeff97f020f686596fca6eba20c310e26fb7/qa; will attempt to use it 2023-09-23T19:51:31.023 INFO:teuthology.run:Found tasks at /home/teuthworker/src/github.com_ceph_ceph_bb0daeff97f020f686596fca6eba20c310e26fb7/qa/tasks 2023-09-23T19:51:31.023 INFO:teuthology.run_tasks:Running task internal.check_packages... 2023-09-23T19:51:31.024 INFO:teuthology.task.internal:Checking packages... 2023-09-23T19:51:31.047 INFO:teuthology.task.internal:Checking packages for os_type 'rhel', flavor 'default' and ceph hash 'bb0daeff97f020f686596fca6eba20c310e26fb7' 2023-09-23T19:51:31.048 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-09-23T19:51:31.048 INFO:teuthology.packaging:ref: None 2023-09-23T19:51:31.048 INFO:teuthology.packaging:tag: None 2023-09-23T19:51:31.048 INFO:teuthology.packaging:branch: reef 2023-09-23T19:51:31.048 INFO:teuthology.packaging:sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T19:51:31.048 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=reef 2023-09-23T19:51:31.365 INFO:teuthology.task.internal:Found packages for ceph version 18.2.0-460.gbb0daeff 2023-09-23T19:51:31.366 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2023-09-23T19:51:31.372 INFO:teuthology.task.internal:no buildpackages task found 2023-09-23T19:51:31.372 INFO:teuthology.run_tasks:Running task internal.save_config... 2023-09-23T19:51:31.384 INFO:teuthology.task.internal:Saving configuration 2023-09-23T19:51:31.406 INFO:teuthology.run_tasks:Running task internal.check_lock... 2023-09-23T19:51:31.411 INFO:teuthology.task.internal.check_lock:Checking locks... 2023-09-23T19:51:31.438 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi018.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:quincy-x-reef-distro-default-smithi/7401824', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-09-23 19:46:16.661480', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLVvN3+LmzPZ4ixiwyCkS3KZLZ9R6MOPm+kPtaJbl8fExpzbk8TudXCcJWHBL7DGPWFvye2jT6sei9eiX0EUQYs='} 2023-09-23T19:51:31.460 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi149.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:quincy-x-reef-distro-default-smithi/7401824', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-09-23 19:46:16.660141', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLVvN3+LmzPZ4ixiwyCkS3KZLZ9R6MOPm+kPtaJbl8fExpzbk8TudXCcJWHBL7DGPWFvye2jT6sei9eiX0EUQYs='} 2023-09-23T19:51:31.460 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2023-09-23T19:51:31.466 INFO:teuthology.task.internal:roles: ubuntu@smithi018.front.sepia.ceph.com - ['mon.a', 'mon.c', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a', 'alertmanager.a'] 2023-09-23T19:51:31.466 INFO:teuthology.task.internal:roles: ubuntu@smithi149.front.sepia.ceph.com - ['mon.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'grafana.a', 'node-exporter.b'] 2023-09-23T19:51:31.466 INFO:teuthology.run_tasks:Running task console_log... 2023-09-23T19:51:31.539 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7ff4e1adaaf0>, signals=[15]) 2023-09-23T19:51:31.539 INFO:teuthology.run_tasks:Running task internal.connect... 2023-09-23T19:51:31.545 INFO:teuthology.task.internal:Opening connections... 2023-09-23T19:51:31.545 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi018.front.sepia.ceph.com 2023-09-23T19:51:31.547 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi018.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-09-23T19:51:31.614 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi149.front.sepia.ceph.com 2023-09-23T19:51:31.615 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi149.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-09-23T19:51:31.681 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2023-09-23T19:51:31.687 DEBUG:teuthology.orchestra.run.smithi018:> uname -m 2023-09-23T19:51:31.709 INFO:teuthology.orchestra.run.smithi018.stdout:x86_64 2023-09-23T19:51:31.709 DEBUG:teuthology.orchestra.run.smithi018:> cat /etc/os-release 2023-09-23T19:51:31.770 INFO:teuthology.orchestra.run.smithi018.stdout:NAME="Red Hat Enterprise Linux" 2023-09-23T19:51:31.771 INFO:teuthology.orchestra.run.smithi018.stdout:VERSION="8.6 (Ootpa)" 2023-09-23T19:51:31.771 INFO:teuthology.orchestra.run.smithi018.stdout:ID="rhel" 2023-09-23T19:51:31.771 INFO:teuthology.orchestra.run.smithi018.stdout:ID_LIKE="fedora" 2023-09-23T19:51:31.771 INFO:teuthology.orchestra.run.smithi018.stdout:VERSION_ID="8.6" 2023-09-23T19:51:31.771 INFO:teuthology.orchestra.run.smithi018.stdout:PLATFORM_ID="platform:el8" 2023-09-23T19:51:31.772 INFO:teuthology.orchestra.run.smithi018.stdout:PRETTY_NAME="Red Hat Enterprise Linux 8.6 (Ootpa)" 2023-09-23T19:51:31.772 INFO:teuthology.orchestra.run.smithi018.stdout:ANSI_COLOR="0;31" 2023-09-23T19:51:31.772 INFO:teuthology.orchestra.run.smithi018.stdout:CPE_NAME="cpe:/o:redhat:enterprise_linux:8::baseos" 2023-09-23T19:51:31.772 INFO:teuthology.orchestra.run.smithi018.stdout:HOME_URL="https://www.redhat.com/" 2023-09-23T19:51:31.772 INFO:teuthology.orchestra.run.smithi018.stdout:DOCUMENTATION_URL="https://access.redhat.com/documentation/red_hat_enterprise_linux/8/" 2023-09-23T19:51:31.772 INFO:teuthology.orchestra.run.smithi018.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2023-09-23T19:51:31.772 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T19:51:31.773 INFO:teuthology.orchestra.run.smithi018.stdout:REDHAT_BUGZILLA_PRODUCT="Red Hat Enterprise Linux 8" 2023-09-23T19:51:31.773 INFO:teuthology.orchestra.run.smithi018.stdout:REDHAT_BUGZILLA_PRODUCT_VERSION=8.6 2023-09-23T19:51:31.773 INFO:teuthology.orchestra.run.smithi018.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux" 2023-09-23T19:51:31.773 INFO:teuthology.orchestra.run.smithi018.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="8.6" 2023-09-23T19:51:31.774 INFO:teuthology.lock.ops:Updating smithi018.front.sepia.ceph.com on lock server 2023-09-23T19:51:31.799 DEBUG:teuthology.orchestra.run.smithi149:> uname -m 2023-09-23T19:51:31.819 INFO:teuthology.orchestra.run.smithi149.stdout:x86_64 2023-09-23T19:51:31.820 DEBUG:teuthology.orchestra.run.smithi149:> cat /etc/os-release 2023-09-23T19:51:31.879 INFO:teuthology.orchestra.run.smithi149.stdout:NAME="Red Hat Enterprise Linux" 2023-09-23T19:51:31.879 INFO:teuthology.orchestra.run.smithi149.stdout:VERSION="8.6 (Ootpa)" 2023-09-23T19:51:31.879 INFO:teuthology.orchestra.run.smithi149.stdout:ID="rhel" 2023-09-23T19:51:31.879 INFO:teuthology.orchestra.run.smithi149.stdout:ID_LIKE="fedora" 2023-09-23T19:51:31.879 INFO:teuthology.orchestra.run.smithi149.stdout:VERSION_ID="8.6" 2023-09-23T19:51:31.879 INFO:teuthology.orchestra.run.smithi149.stdout:PLATFORM_ID="platform:el8" 2023-09-23T19:51:31.879 INFO:teuthology.orchestra.run.smithi149.stdout:PRETTY_NAME="Red Hat Enterprise Linux 8.6 (Ootpa)" 2023-09-23T19:51:31.879 INFO:teuthology.orchestra.run.smithi149.stdout:ANSI_COLOR="0;31" 2023-09-23T19:51:31.879 INFO:teuthology.orchestra.run.smithi149.stdout:CPE_NAME="cpe:/o:redhat:enterprise_linux:8::baseos" 2023-09-23T19:51:31.880 INFO:teuthology.orchestra.run.smithi149.stdout:HOME_URL="https://www.redhat.com/" 2023-09-23T19:51:31.880 INFO:teuthology.orchestra.run.smithi149.stdout:DOCUMENTATION_URL="https://access.redhat.com/documentation/red_hat_enterprise_linux/8/" 2023-09-23T19:51:31.880 INFO:teuthology.orchestra.run.smithi149.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2023-09-23T19:51:31.880 INFO:teuthology.orchestra.run.smithi149.stdout: 2023-09-23T19:51:31.880 INFO:teuthology.orchestra.run.smithi149.stdout:REDHAT_BUGZILLA_PRODUCT="Red Hat Enterprise Linux 8" 2023-09-23T19:51:31.880 INFO:teuthology.orchestra.run.smithi149.stdout:REDHAT_BUGZILLA_PRODUCT_VERSION=8.6 2023-09-23T19:51:31.880 INFO:teuthology.orchestra.run.smithi149.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux" 2023-09-23T19:51:31.880 INFO:teuthology.orchestra.run.smithi149.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="8.6" 2023-09-23T19:51:31.880 INFO:teuthology.lock.ops:Updating smithi149.front.sepia.ceph.com on lock server 2023-09-23T19:51:31.899 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2023-09-23T19:51:31.908 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2023-09-23T19:51:31.913 INFO:teuthology.task.internal:Checking for old test directory... 2023-09-23T19:51:31.913 DEBUG:teuthology.orchestra.run.smithi018:> test '!' -e /home/ubuntu/cephtest 2023-09-23T19:51:31.915 DEBUG:teuthology.orchestra.run.smithi149:> test '!' -e /home/ubuntu/cephtest 2023-09-23T19:51:31.938 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2023-09-23T19:51:31.944 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2023-09-23T19:51:31.944 DEBUG:teuthology.orchestra.run.smithi018:> test -z $(ls -A /var/lib/ceph) 2023-09-23T19:51:31.975 DEBUG:teuthology.orchestra.run.smithi149:> test -z $(ls -A /var/lib/ceph) 2023-09-23T19:51:32.007 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2023-09-23T19:51:32.064 INFO:teuthology.run_tasks:Running task kernel... 2023-09-23T19:51:32.077 INFO:teuthology.task.kernel:normalize config orig: {'kdb': True, 'sha1': 'distro'} 2023-09-23T19:51:32.078 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': True, 'sha1': 'distro'}, 'mon.b': {'kdb': True, 'sha1': 'distro'}}, timeout 300 2023-09-23T19:51:32.078 DEBUG:teuthology.orchestra.run.smithi018:> test -f /run/.containerenv -o -f /.dockerenv 2023-09-23T19:51:32.078 DEBUG:teuthology.orchestra.run.smithi149:> test -f /run/.containerenv -o -f /.dockerenv 2023-09-23T19:51:32.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:32.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:32.098 DEBUG:teuthology.orchestra.run.smithi018:> uname -r 2023-09-23T19:51:32.098 DEBUG:teuthology.orchestra.run.smithi149:> uname -r 2023-09-23T19:51:32.157 INFO:teuthology.orchestra.run.smithi149.stdout:4.18.0-372.9.1.el8.x86_64 2023-09-23T19:51:32.158 INFO:teuthology.task.kernel:Running kernel on smithi149: 4.18.0-372.9.1.el8.x86_64 2023-09-23T19:51:32.158 DEBUG:teuthology.orchestra.run.smithi149:> sudo yum install -y kernel 2023-09-23T19:51:32.159 INFO:teuthology.orchestra.run.smithi018.stdout:4.18.0-372.9.1.el8.x86_64 2023-09-23T19:51:32.159 INFO:teuthology.task.kernel:Running kernel on smithi018: 4.18.0-372.9.1.el8.x86_64 2023-09-23T19:51:32.159 DEBUG:teuthology.orchestra.run.smithi018:> sudo yum install -y kernel 2023-09-23T19:51:33.295 INFO:teuthology.orchestra.run.smithi149.stdout:Updating Subscription Management repositories. 2023-09-23T19:51:33.295 INFO:teuthology.orchestra.run.smithi149.stdout:Unable to read consumer identity 2023-09-23T19:51:33.445 INFO:teuthology.orchestra.run.smithi018.stdout:Updating Subscription Management repositories. 2023-09-23T19:51:33.446 INFO:teuthology.orchestra.run.smithi018.stdout:Unable to read consumer identity 2023-09-23T19:51:34.760 INFO:teuthology.orchestra.run.smithi149.stdout:Copr repo for python3-asyncssh owned by ceph 15 kB/s | 3.5 kB 00:00 2023-09-23T19:51:34.895 INFO:teuthology.orchestra.run.smithi018.stdout:Copr repo for python3-asyncssh owned by ceph 22 kB/s | 3.5 kB 00:00 2023-09-23T19:51:35.510 INFO:teuthology.orchestra.run.smithi149.stdout:Extra Packages for Enterprise Linux 22 MB/s | 16 MB 00:00 2023-09-23T19:51:35.971 INFO:teuthology.orchestra.run.smithi018.stdout:Extra Packages for Enterprise Linux 15 MB/s | 16 MB 00:01 2023-09-23T19:51:39.394 INFO:teuthology.orchestra.run.smithi149.stdout:lab-extras 66 kB/s | 24 kB 00:00 2023-09-23T19:51:39.697 INFO:teuthology.orchestra.run.smithi018.stdout:lab-extras 422 kB/s | 24 kB 00:00 2023-09-23T19:51:40.097 INFO:teuthology.orchestra.run.smithi149.stdout:Package kernel-4.18.0-372.9.1.el8.x86_64 is already installed. 2023-09-23T19:51:40.147 INFO:teuthology.orchestra.run.smithi149.stdout:Dependencies resolved. 2023-09-23T19:51:40.148 INFO:teuthology.orchestra.run.smithi149.stdout:Nothing to do. 2023-09-23T19:51:40.148 INFO:teuthology.orchestra.run.smithi149.stdout:Complete! 2023-09-23T19:51:40.285 DEBUG:teuthology.orchestra.run.smithi149:> echo no | sudo yum reinstall kernel || true 2023-09-23T19:51:40.432 INFO:teuthology.orchestra.run.smithi018.stdout:Package kernel-4.18.0-372.9.1.el8.x86_64 is already installed. 2023-09-23T19:51:40.494 INFO:teuthology.orchestra.run.smithi018.stdout:Dependencies resolved. 2023-09-23T19:51:40.494 INFO:teuthology.orchestra.run.smithi018.stdout:Nothing to do. 2023-09-23T19:51:40.494 INFO:teuthology.orchestra.run.smithi018.stdout:Complete! 2023-09-23T19:51:40.617 INFO:teuthology.orchestra.run.smithi149.stdout:Updating Subscription Management repositories. 2023-09-23T19:51:40.617 INFO:teuthology.orchestra.run.smithi149.stdout:Unable to read consumer identity 2023-09-23T19:51:40.619 DEBUG:teuthology.orchestra.run.smithi018:> echo no | sudo yum reinstall kernel || true 2023-09-23T19:51:40.889 INFO:teuthology.orchestra.run.smithi149.stdout:Last metadata expiration check: 0:00:01 ago on Sat 23 Sep 2023 07:51:39 PM UTC. 2023-09-23T19:51:40.930 INFO:teuthology.orchestra.run.smithi149.stdout:Installed package kernel-4.18.0-372.9.1.el8.x86_64 (from anaconda) not available. 2023-09-23T19:51:40.984 INFO:teuthology.orchestra.run.smithi149.stderr:Error: No packages marked for reinstall. 2023-09-23T19:51:41.010 INFO:teuthology.orchestra.run.smithi018.stdout:Updating Subscription Management repositories. 2023-09-23T19:51:41.010 INFO:teuthology.orchestra.run.smithi018.stdout:Unable to read consumer identity 2023-09-23T19:51:41.033 DEBUG:teuthology.orchestra.run.smithi149:> sudo yum reinstall -y kernel || true 2023-09-23T19:51:41.284 INFO:teuthology.orchestra.run.smithi018.stdout:Last metadata expiration check: 0:00:02 ago on Sat 23 Sep 2023 07:51:39 PM UTC. 2023-09-23T19:51:41.328 INFO:teuthology.orchestra.run.smithi018.stdout:Installed package kernel-4.18.0-372.9.1.el8.x86_64 (from anaconda) not available. 2023-09-23T19:51:41.361 INFO:teuthology.orchestra.run.smithi149.stdout:Updating Subscription Management repositories. 2023-09-23T19:51:41.362 INFO:teuthology.orchestra.run.smithi149.stdout:Unable to read consumer identity 2023-09-23T19:51:41.367 INFO:teuthology.orchestra.run.smithi018.stderr:Error: No packages marked for reinstall. 2023-09-23T19:51:41.415 DEBUG:teuthology.orchestra.run.smithi018:> sudo yum reinstall -y kernel || true 2023-09-23T19:51:41.624 INFO:teuthology.orchestra.run.smithi149.stdout:Last metadata expiration check: 0:00:02 ago on Sat 23 Sep 2023 07:51:39 PM UTC. 2023-09-23T19:51:41.667 INFO:teuthology.orchestra.run.smithi149.stdout:Installed package kernel-4.18.0-372.9.1.el8.x86_64 (from anaconda) not available. 2023-09-23T19:51:41.706 INFO:teuthology.orchestra.run.smithi149.stderr:Error: No packages marked for reinstall. 2023-09-23T19:51:41.755 DEBUG:teuthology.orchestra.run.smithi149:> rpm -q kernel | sort -rV | head -n 1 2023-09-23T19:51:41.765 INFO:teuthology.orchestra.run.smithi018.stdout:Updating Subscription Management repositories. 2023-09-23T19:51:41.765 INFO:teuthology.orchestra.run.smithi018.stdout:Unable to read consumer identity 2023-09-23T19:51:41.835 INFO:teuthology.orchestra.run.smithi149.stdout:kernel-4.18.0-372.9.1.el8.x86_64 2023-09-23T19:51:41.836 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-372.9.1.el8.x86_64 2023-09-23T19:51:41.836 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2023-09-23T19:51:41.836 INFO:teuthology.task.kernel:Enabling kdb on mon.b... 2023-09-23T19:51:41.836 DEBUG:teuthology.orchestra.run.smithi149:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2023-09-23T19:51:41.869 INFO:teuthology.orchestra.run.smithi149.stdout:ttyS1 2023-09-23T19:51:41.881 DEBUG:teuthology.parallel:result is None 2023-09-23T19:51:42.040 INFO:teuthology.orchestra.run.smithi018.stdout:Last metadata expiration check: 0:00:03 ago on Sat 23 Sep 2023 07:51:39 PM UTC. 2023-09-23T19:51:42.083 INFO:teuthology.orchestra.run.smithi018.stdout:Installed package kernel-4.18.0-372.9.1.el8.x86_64 (from anaconda) not available. 2023-09-23T19:51:42.107 INFO:teuthology.orchestra.run.smithi018.stderr:Error: No packages marked for reinstall. 2023-09-23T19:51:42.160 DEBUG:teuthology.orchestra.run.smithi018:> rpm -q kernel | sort -rV | head -n 1 2023-09-23T19:51:42.242 INFO:teuthology.orchestra.run.smithi018.stdout:kernel-4.18.0-372.9.1.el8.x86_64 2023-09-23T19:51:42.243 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-372.9.1.el8.x86_64 2023-09-23T19:51:42.243 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2023-09-23T19:51:42.243 INFO:teuthology.task.kernel:Enabling kdb on mon.a... 2023-09-23T19:51:42.243 DEBUG:teuthology.orchestra.run.smithi018:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2023-09-23T19:51:42.274 INFO:teuthology.orchestra.run.smithi018.stdout:ttyS1 2023-09-23T19:51:42.287 DEBUG:teuthology.parallel:result is None 2023-09-23T19:51:42.287 INFO:teuthology.run_tasks:Running task internal.base... 2023-09-23T19:51:42.294 INFO:teuthology.task.internal:Creating test directory... 2023-09-23T19:51:42.294 DEBUG:teuthology.orchestra.run.smithi018:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2023-09-23T19:51:42.329 DEBUG:teuthology.orchestra.run.smithi149:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2023-09-23T19:51:42.346 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2023-09-23T19:51:42.352 INFO:teuthology.run_tasks:Running task internal.archive... 2023-09-23T19:51:42.362 INFO:teuthology.task.internal:Creating archive directory... 2023-09-23T19:51:42.362 DEBUG:teuthology.orchestra.run.smithi018:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2023-09-23T19:51:42.386 DEBUG:teuthology.orchestra.run.smithi149:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2023-09-23T19:51:42.408 INFO:teuthology.run_tasks:Running task internal.coredump... 2023-09-23T19:51:42.414 INFO:teuthology.task.internal:Enabling coredump saving... 2023-09-23T19:51:42.415 DEBUG:teuthology.orchestra.run.smithi018:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2023-09-23T19:51:42.444 DEBUG:teuthology.orchestra.run.smithi149:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2023-09-23T19:51:42.474 INFO:teuthology.orchestra.run.smithi018.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-09-23T19:51:42.485 INFO:teuthology.orchestra.run.smithi018.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-09-23T19:51:42.498 INFO:teuthology.orchestra.run.smithi149.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-09-23T19:51:42.509 INFO:teuthology.orchestra.run.smithi149.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-09-23T19:51:42.511 INFO:teuthology.run_tasks:Running task internal.sudo... 2023-09-23T19:51:42.518 INFO:teuthology.task.internal:Configuring sudo... 2023-09-23T19:51:42.518 DEBUG:teuthology.orchestra.run.smithi018:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2023-09-23T19:51:42.529 DEBUG:teuthology.orchestra.run.smithi149:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2023-09-23T19:51:42.555 INFO:teuthology.run_tasks:Running task internal.syslog... 2023-09-23T19:51:42.564 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2023-09-23T19:51:42.564 DEBUG:teuthology.orchestra.run.smithi018:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2023-09-23T19:51:42.597 DEBUG:teuthology.orchestra.run.smithi149:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2023-09-23T19:51:42.614 DEBUG:teuthology.orchestra.run.smithi018:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2023-09-23T19:51:42.695 DEBUG:teuthology.orchestra.run.smithi018:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2023-09-23T19:51:42.751 DEBUG:teuthology.orchestra.run.smithi018:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2023-09-23T19:51:42.835 DEBUG:teuthology.orchestra.run.smithi018:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2023-09-23T19:51:42.877 DEBUG:teuthology.orchestra.run.smithi018:> set -ex 2023-09-23T19:51:42.877 DEBUG:teuthology.orchestra.run.smithi018:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2023-09-23T19:51:42.967 DEBUG:teuthology.orchestra.run.smithi149:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2023-09-23T19:51:43.012 DEBUG:teuthology.orchestra.run.smithi149:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2023-09-23T19:51:43.059 DEBUG:teuthology.orchestra.run.smithi149:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2023-09-23T19:51:43.143 DEBUG:teuthology.orchestra.run.smithi149:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2023-09-23T19:51:43.183 DEBUG:teuthology.orchestra.run.smithi149:> set -ex 2023-09-23T19:51:43.183 DEBUG:teuthology.orchestra.run.smithi149:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2023-09-23T19:51:43.250 DEBUG:teuthology.orchestra.run.smithi018:> sudo service rsyslog restart 2023-09-23T19:51:43.252 DEBUG:teuthology.orchestra.run.smithi149:> sudo service rsyslog restart 2023-09-23T19:51:43.312 INFO:teuthology.orchestra.run.smithi018.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2023-09-23T19:51:43.384 INFO:teuthology.orchestra.run.smithi149.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2023-09-23T19:51:43.727 INFO:teuthology.run_tasks:Running task internal.timer... 2023-09-23T19:51:43.735 INFO:teuthology.task.internal:Starting timer... 2023-09-23T19:51:43.735 INFO:teuthology.run_tasks:Running task pcp... 2023-09-23T19:51:43.744 INFO:teuthology.run_tasks:Running task selinux... 2023-09-23T19:51:43.750 DEBUG:teuthology.task:Applying overrides for task selinux: {'whitelist': ['scontext=system_u:system_r:logrotate_t:s0']} 2023-09-23T19:51:43.751 DEBUG:teuthology.orchestra.run.smithi018:> sudo service auditd rotate 2023-09-23T19:51:43.817 INFO:teuthology.orchestra.run.smithi018.stdout:Rotating logs: 2023-09-23T19:51:43.819 DEBUG:teuthology.orchestra.run.smithi149:> sudo service auditd rotate 2023-09-23T19:51:43.888 INFO:teuthology.orchestra.run.smithi149.stdout:Rotating logs: 2023-09-23T19:51:43.890 DEBUG:teuthology.task.selinux:Getting current SELinux state 2023-09-23T19:51:43.890 DEBUG:teuthology.orchestra.run.smithi018:> /usr/sbin/getenforce 2023-09-23T19:51:43.913 INFO:teuthology.orchestra.run.smithi018.stdout:Permissive 2023-09-23T19:51:43.913 DEBUG:teuthology.orchestra.run.smithi149:> /usr/sbin/getenforce 2023-09-23T19:51:43.936 INFO:teuthology.orchestra.run.smithi149.stdout:Permissive 2023-09-23T19:51:43.936 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi018.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi149.front.sepia.ceph.com': 'permissive'} 2023-09-23T19:51:43.936 DEBUG:teuthology.orchestra.run.smithi018:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-09-23T19:51:43.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:43.982 DEBUG:teuthology.orchestra.run.smithi149:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-09-23T19:51:44.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T19:51:44.010 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2023-09-23T19:51:44.010 DEBUG:teuthology.orchestra.run.smithi018:> sudo /usr/sbin/setenforce permissive 2023-09-23T19:51:44.049 DEBUG:teuthology.orchestra.run.smithi149:> sudo /usr/sbin/setenforce permissive 2023-09-23T19:51:44.093 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2023-09-23T19:51:44.102 DEBUG:teuthology.repo_utils:Setting repo remote to https://git.ceph.com/ceph-cm-ansible.git 2023-09-23T19:51:44.107 INFO:teuthology.repo_utils:Fetching main from origin 2023-09-23T19:51:44.243 INFO:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2023-09-23T19:51:44.258 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2023-09-23T19:51:44.259 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi018.front.sepia.ceph.com,smithi149.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2023-09-23T20:00:30.704 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi018.front.sepia.ceph.com'), Remote(name='ubuntu@smithi149.front.sepia.ceph.com')] 2023-09-23T20:00:30.705 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi018.front.sepia.ceph.com' 2023-09-23T20:00:30.705 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi018.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-09-23T20:00:30.775 DEBUG:teuthology.orchestra.run.smithi018:> true 2023-09-23T20:00:30.844 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi018.front.sepia.ceph.com' 2023-09-23T20:00:30.845 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi149.front.sepia.ceph.com' 2023-09-23T20:00:30.845 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi149.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-09-23T20:00:30.912 DEBUG:teuthology.orchestra.run.smithi149:> true 2023-09-23T20:00:30.979 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi149.front.sepia.ceph.com' 2023-09-23T20:00:30.979 INFO:teuthology.run_tasks:Running task clock... 2023-09-23T20:00:30.992 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2023-09-23T20:00:30.992 INFO:teuthology.orchestra.run:Running command with timeout 360 2023-09-23T20:00:30.993 DEBUG:teuthology.orchestra.run.smithi018:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-09-23T20:00:30.995 INFO:teuthology.orchestra.run:Running command with timeout 360 2023-09-23T20:00:30.996 DEBUG:teuthology.orchestra.run.smithi149:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-09-23T20:00:31.026 INFO:teuthology.orchestra.run.smithi018.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2023-09-23T20:00:31.041 INFO:teuthology.orchestra.run.smithi018.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2023-09-23T20:00:31.050 INFO:teuthology.orchestra.run.smithi149.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2023-09-23T20:00:31.064 INFO:teuthology.orchestra.run.smithi149.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2023-09-23T20:00:31.103 INFO:teuthology.orchestra.run.smithi018.stderr:sudo: ntpd: command not found 2023-09-23T20:00:31.115 INFO:teuthology.orchestra.run.smithi018.stdout:506 Cannot talk to daemon 2023-09-23T20:00:31.125 INFO:teuthology.orchestra.run.smithi149.stderr:sudo: ntpd: command not found 2023-09-23T20:00:31.130 INFO:teuthology.orchestra.run.smithi018.stderr:Failed to start ntp.service: Unit ntp.service not found. 2023-09-23T20:00:31.137 INFO:teuthology.orchestra.run.smithi149.stdout:506 Cannot talk to daemon 2023-09-23T20:00:31.144 INFO:teuthology.orchestra.run.smithi018.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2023-09-23T20:00:31.151 INFO:teuthology.orchestra.run.smithi149.stderr:Failed to start ntp.service: Unit ntp.service not found. 2023-09-23T20:00:31.165 INFO:teuthology.orchestra.run.smithi149.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2023-09-23T20:00:31.224 INFO:teuthology.orchestra.run.smithi018.stderr:bash: ntpq: command not found 2023-09-23T20:00:31.228 INFO:teuthology.orchestra.run.smithi018.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2023-09-23T20:00:31.228 INFO:teuthology.orchestra.run.smithi018.stdout:=============================================================================== 2023-09-23T20:00:31.228 INFO:teuthology.orchestra.run.smithi018.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-09-23T20:00:31.228 INFO:teuthology.orchestra.run.smithi018.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-09-23T20:00:31.228 INFO:teuthology.orchestra.run.smithi018.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-09-23T20:00:31.228 INFO:teuthology.orchestra.run.smithi018.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-09-23T20:00:31.232 INFO:teuthology.orchestra.run.smithi149.stderr:bash: ntpq: command not found 2023-09-23T20:00:31.235 INFO:teuthology.orchestra.run.smithi149.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2023-09-23T20:00:31.235 INFO:teuthology.orchestra.run.smithi149.stdout:=============================================================================== 2023-09-23T20:00:31.236 INFO:teuthology.orchestra.run.smithi149.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-09-23T20:00:31.236 INFO:teuthology.orchestra.run.smithi149.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-09-23T20:00:31.236 INFO:teuthology.orchestra.run.smithi149.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-09-23T20:00:31.236 INFO:teuthology.orchestra.run.smithi149.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-09-23T20:00:31.237 INFO:teuthology.run_tasks:Running task pexec... 2023-09-23T20:00:31.246 INFO:teuthology.task.pexec:Executing custom commands... 2023-09-23T20:00:31.246 INFO:teuthology.task.pexec:Running commands on host ubuntu@smithi018.front.sepia.ceph.com 2023-09-23T20:00:31.247 DEBUG:teuthology.orchestra.run.smithi018:> TESTDIR=/home/ubuntu/cephtest bash -s 2023-09-23T20:00:31.247 INFO:teuthology.task.pexec:Running commands on host ubuntu@smithi149.front.sepia.ceph.com 2023-09-23T20:00:31.247 DEBUG:teuthology.orchestra.run.smithi149:> TESTDIR=/home/ubuntu/cephtest bash -s 2023-09-23T20:00:31.647 INFO:teuthology.orchestra.run.smithi149.stdout:Updating Subscription Management repositories. 2023-09-23T20:00:31.661 INFO:teuthology.orchestra.run.smithi018.stdout:Updating Subscription Management repositories. 2023-09-23T20:00:33.249 INFO:teuthology.orchestra.run.smithi149.stdout:Last metadata expiration check: 0:01:38 ago on Sat 23 Sep 2023 07:58:54 PM UTC. 2023-09-23T20:00:33.259 INFO:teuthology.orchestra.run.smithi018.stdout:Last metadata expiration check: 0:01:28 ago on Sat 23 Sep 2023 07:59:04 PM UTC. 2023-09-23T20:00:35.057 INFO:teuthology.orchestra.run.smithi149.stdout:Dependencies resolved. 2023-09-23T20:00:35.058 INFO:teuthology.orchestra.run.smithi149.stdout:================================================================================ 2023-09-23T20:00:35.058 INFO:teuthology.orchestra.run.smithi149.stdout: Package Architecture Version Repository Size 2023-09-23T20:00:35.058 INFO:teuthology.orchestra.run.smithi149.stdout:================================================================================ 2023-09-23T20:00:35.058 INFO:teuthology.orchestra.run.smithi149.stdout:Resetting modules: 2023-09-23T20:00:35.059 INFO:teuthology.orchestra.run.smithi149.stdout: container-tools 2023-09-23T20:00:35.059 INFO:teuthology.orchestra.run.smithi149.stdout: 2023-09-23T20:00:35.059 INFO:teuthology.orchestra.run.smithi149.stdout:Transaction Summary 2023-09-23T20:00:35.059 INFO:teuthology.orchestra.run.smithi149.stdout:================================================================================ 2023-09-23T20:00:35.059 INFO:teuthology.orchestra.run.smithi149.stdout: 2023-09-23T20:00:35.238 INFO:teuthology.orchestra.run.smithi018.stdout:Dependencies resolved. 2023-09-23T20:00:35.239 INFO:teuthology.orchestra.run.smithi018.stdout:================================================================================ 2023-09-23T20:00:35.239 INFO:teuthology.orchestra.run.smithi018.stdout: Package Architecture Version Repository Size 2023-09-23T20:00:35.239 INFO:teuthology.orchestra.run.smithi018.stdout:================================================================================ 2023-09-23T20:00:35.239 INFO:teuthology.orchestra.run.smithi018.stdout:Resetting modules: 2023-09-23T20:00:35.239 INFO:teuthology.orchestra.run.smithi018.stdout: container-tools 2023-09-23T20:00:35.239 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:00:35.239 INFO:teuthology.orchestra.run.smithi018.stdout:Transaction Summary 2023-09-23T20:00:35.240 INFO:teuthology.orchestra.run.smithi018.stdout:================================================================================ 2023-09-23T20:00:35.240 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:00:36.037 INFO:teuthology.orchestra.run.smithi149.stdout:Last metadata expiration check: 0:01:41 ago on Sat 23 Sep 2023 07:58:54 PM UTC. 2023-09-23T20:00:36.244 INFO:teuthology.orchestra.run.smithi018.stdout:Last metadata expiration check: 0:01:31 ago on Sat 23 Sep 2023 07:59:04 PM UTC. 2023-09-23T20:00:38.786 INFO:teuthology.orchestra.run.smithi149.stdout:Complete! 2023-09-23T20:00:39.156 INFO:teuthology.orchestra.run.smithi018.stdout:Complete! 2023-09-23T20:00:39.383 INFO:teuthology.orchestra.run.smithi149.stdout:Updating Subscription Management repositories. 2023-09-23T20:00:39.623 INFO:teuthology.orchestra.run.smithi018.stdout:Updating Subscription Management repositories. 2023-09-23T20:00:40.286 INFO:teuthology.orchestra.run.smithi149.stdout:Last metadata expiration check: 0:01:45 ago on Sat 23 Sep 2023 07:58:54 PM UTC. 2023-09-23T20:00:40.581 INFO:teuthology.orchestra.run.smithi018.stdout:Last metadata expiration check: 0:01:36 ago on Sat 23 Sep 2023 07:59:04 PM UTC. 2023-09-23T20:00:41.995 INFO:teuthology.orchestra.run.smithi149.stdout:Dependencies resolved. 2023-09-23T20:00:42.089 INFO:teuthology.orchestra.run.smithi149.stdout:============================================================================================================ 2023-09-23T20:00:42.089 INFO:teuthology.orchestra.run.smithi149.stdout: Package Arch Version Repository Size 2023-09-23T20:00:42.089 INFO:teuthology.orchestra.run.smithi149.stdout:============================================================================================================ 2023-09-23T20:00:42.090 INFO:teuthology.orchestra.run.smithi149.stdout:Installing group/module packages: 2023-09-23T20:00:42.090 INFO:teuthology.orchestra.run.smithi149.stdout: buildah x86_64 1:1.26.2-1.module+el8.6.0+16771+28dfca77 rhel-8-for-x86_64-appstream-rpms 8.0 M 2023-09-23T20:00:42.090 INFO:teuthology.orchestra.run.smithi149.stdout: cockpit-podman noarch 49.1-1.module+el8.6.0+16771+28dfca77 rhel-8-for-x86_64-appstream-rpms 524 k 2023-09-23T20:00:42.090 INFO:teuthology.orchestra.run.smithi149.stdout: crun x86_64 1.5-1.module+el8.6.0+16771+28dfca77 rhel-8-for-x86_64-appstream-rpms 213 k 2023-09-23T20:00:42.090 INFO:teuthology.orchestra.run.smithi149.stdout: python3-podman noarch 4.0.1-1.module+el8.6.0+16837+85b82211 rhel-8-for-x86_64-appstream-rpms 149 k 2023-09-23T20:00:42.090 INFO:teuthology.orchestra.run.smithi149.stdout: skopeo x86_64 2:1.9.1-1.module+el8.6.0+16771+28dfca77 rhel-8-for-x86_64-appstream-rpms 7.0 M 2023-09-23T20:00:42.090 INFO:teuthology.orchestra.run.smithi149.stdout: toolbox x86_64 0.0.99.3-0.6.module+el8.6.0+16771+28dfca77 rhel-8-for-x86_64-appstream-rpms 2.2 M 2023-09-23T20:00:42.090 INFO:teuthology.orchestra.run.smithi149.stdout: udica noarch 0.2.6-4.module+el8.6.0+16771+28dfca77 rhel-8-for-x86_64-appstream-rpms 49 k 2023-09-23T20:00:42.090 INFO:teuthology.orchestra.run.smithi149.stdout:Installing dependencies: 2023-09-23T20:00:42.090 INFO:teuthology.orchestra.run.smithi149.stdout: python3-pytoml noarch 0.1.14-5.git7dea353.el8 rhel-8-for-x86_64-appstream-rpms 25 k 2023-09-23T20:00:42.090 INFO:teuthology.orchestra.run.smithi149.stdout: python3-pyxdg noarch 0.25-16.el8 rhel-8-for-x86_64-appstream-rpms 94 k 2023-09-23T20:00:42.091 INFO:teuthology.orchestra.run.smithi149.stdout: yajl x86_64 2.1.0-10.el8 rhel-8-for-x86_64-appstream-rpms 41 k 2023-09-23T20:00:42.091 INFO:teuthology.orchestra.run.smithi149.stdout:Installing module profiles: 2023-09-23T20:00:42.091 INFO:teuthology.orchestra.run.smithi149.stdout: container-tools/common 2023-09-23T20:00:42.091 INFO:teuthology.orchestra.run.smithi149.stdout: 2023-09-23T20:00:42.091 INFO:teuthology.orchestra.run.smithi149.stdout:Enabling module streams: 2023-09-23T20:00:42.091 INFO:teuthology.orchestra.run.smithi149.stdout: container-tools rhel8 2023-09-23T20:00:42.091 INFO:teuthology.orchestra.run.smithi149.stdout: 2023-09-23T20:00:42.091 INFO:teuthology.orchestra.run.smithi149.stdout:Transaction Summary 2023-09-23T20:00:42.091 INFO:teuthology.orchestra.run.smithi149.stdout:============================================================================================================ 2023-09-23T20:00:42.091 INFO:teuthology.orchestra.run.smithi149.stdout:Install 10 Packages 2023-09-23T20:00:42.091 INFO:teuthology.orchestra.run.smithi149.stdout: 2023-09-23T20:00:42.092 INFO:teuthology.orchestra.run.smithi149.stdout:Total download size: 18 M 2023-09-23T20:00:42.092 INFO:teuthology.orchestra.run.smithi149.stdout:Installed size: 65 M 2023-09-23T20:00:42.092 INFO:teuthology.orchestra.run.smithi149.stdout:Downloading Packages: 2023-09-23T20:00:42.419 INFO:teuthology.orchestra.run.smithi149.stdout:(1/10): yajl-2.1.0-10.el8.x86_64.rpm 125 kB/s | 41 kB 00:00 2023-09-23T20:00:42.444 INFO:teuthology.orchestra.run.smithi149.stdout:(2/10): python3-pytoml-0.1.14-5.git7dea353.el8. 70 kB/s | 25 kB 00:00 2023-09-23T20:00:42.469 INFO:teuthology.orchestra.run.smithi149.stdout:(3/10): python3-pyxdg-0.25-16.el8.noarch.rpm 249 kB/s | 94 kB 00:00 2023-09-23T20:00:42.495 INFO:teuthology.orchestra.run.smithi149.stdout:(4/10): udica-0.2.6-4.module+el8.6.0+16771+28df 655 kB/s | 49 kB 00:00 2023-09-23T20:00:42.545 INFO:teuthology.orchestra.run.smithi149.stdout:(5/10): cockpit-podman-49.1-1.module+el8.6.0+16 5.1 MB/s | 524 kB 00:00 2023-09-23T20:00:42.556 INFO:teuthology.orchestra.run.smithi018.stdout:Dependencies resolved. 2023-09-23T20:00:42.603 INFO:teuthology.orchestra.run.smithi149.stdout:(6/10): python3-podman-4.0.1-1.module+el8.6.0+1 1.3 MB/s | 149 kB 00:00 2023-09-23T20:00:42.628 INFO:teuthology.orchestra.run.smithi149.stdout:(7/10): crun-1.5-1.module+el8.6.0+16771+28dfca7 2.5 MB/s | 213 kB 00:00 2023-09-23T20:00:42.659 INFO:teuthology.orchestra.run.smithi018.stdout:============================================================================================================ 2023-09-23T20:00:42.660 INFO:teuthology.orchestra.run.smithi018.stdout: Package Arch Version Repository Size 2023-09-23T20:00:42.660 INFO:teuthology.orchestra.run.smithi018.stdout:============================================================================================================ 2023-09-23T20:00:42.660 INFO:teuthology.orchestra.run.smithi018.stdout:Installing group/module packages: 2023-09-23T20:00:42.660 INFO:teuthology.orchestra.run.smithi018.stdout: buildah x86_64 1:1.26.2-1.module+el8.6.0+16771+28dfca77 rhel-8-for-x86_64-appstream-rpms 8.0 M 2023-09-23T20:00:42.660 INFO:teuthology.orchestra.run.smithi018.stdout: cockpit-podman noarch 49.1-1.module+el8.6.0+16771+28dfca77 rhel-8-for-x86_64-appstream-rpms 524 k 2023-09-23T20:00:42.660 INFO:teuthology.orchestra.run.smithi018.stdout: crun x86_64 1.5-1.module+el8.6.0+16771+28dfca77 rhel-8-for-x86_64-appstream-rpms 213 k 2023-09-23T20:00:42.660 INFO:teuthology.orchestra.run.smithi018.stdout: python3-podman noarch 4.0.1-1.module+el8.6.0+16837+85b82211 rhel-8-for-x86_64-appstream-rpms 149 k 2023-09-23T20:00:42.660 INFO:teuthology.orchestra.run.smithi018.stdout: skopeo x86_64 2:1.9.1-1.module+el8.6.0+16771+28dfca77 rhel-8-for-x86_64-appstream-rpms 7.0 M 2023-09-23T20:00:42.660 INFO:teuthology.orchestra.run.smithi018.stdout: toolbox x86_64 0.0.99.3-0.6.module+el8.6.0+16771+28dfca77 rhel-8-for-x86_64-appstream-rpms 2.2 M 2023-09-23T20:00:42.660 INFO:teuthology.orchestra.run.smithi018.stdout: udica noarch 0.2.6-4.module+el8.6.0+16771+28dfca77 rhel-8-for-x86_64-appstream-rpms 49 k 2023-09-23T20:00:42.661 INFO:teuthology.orchestra.run.smithi018.stdout:Installing dependencies: 2023-09-23T20:00:42.661 INFO:teuthology.orchestra.run.smithi018.stdout: python3-pytoml noarch 0.1.14-5.git7dea353.el8 rhel-8-for-x86_64-appstream-rpms 25 k 2023-09-23T20:00:42.661 INFO:teuthology.orchestra.run.smithi018.stdout: python3-pyxdg noarch 0.25-16.el8 rhel-8-for-x86_64-appstream-rpms 94 k 2023-09-23T20:00:42.661 INFO:teuthology.orchestra.run.smithi018.stdout: yajl x86_64 2.1.0-10.el8 rhel-8-for-x86_64-appstream-rpms 41 k 2023-09-23T20:00:42.661 INFO:teuthology.orchestra.run.smithi018.stdout:Installing module profiles: 2023-09-23T20:00:42.661 INFO:teuthology.orchestra.run.smithi018.stdout: container-tools/common 2023-09-23T20:00:42.661 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:00:42.661 INFO:teuthology.orchestra.run.smithi018.stdout:Enabling module streams: 2023-09-23T20:00:42.661 INFO:teuthology.orchestra.run.smithi018.stdout: container-tools rhel8 2023-09-23T20:00:42.661 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:00:42.661 INFO:teuthology.orchestra.run.smithi018.stdout:Transaction Summary 2023-09-23T20:00:42.662 INFO:teuthology.orchestra.run.smithi018.stdout:============================================================================================================ 2023-09-23T20:00:42.662 INFO:teuthology.orchestra.run.smithi018.stdout:Install 10 Packages 2023-09-23T20:00:42.662 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:00:42.662 INFO:teuthology.orchestra.run.smithi018.stdout:Total download size: 18 M 2023-09-23T20:00:42.662 INFO:teuthology.orchestra.run.smithi018.stdout:Installed size: 65 M 2023-09-23T20:00:42.663 INFO:teuthology.orchestra.run.smithi018.stdout:Downloading Packages: 2023-09-23T20:00:42.759 INFO:teuthology.orchestra.run.smithi018.stdout:(1/10): python3-pyxdg-0.25-16.el8.noarch.rpm 972 kB/s | 94 kB 00:00 2023-09-23T20:00:42.771 INFO:teuthology.orchestra.run.smithi149.stdout:(8/10): buildah-1.26.2-1.module+el8.6.0+16771+2 26 MB/s | 8.0 MB 00:00 2023-09-23T20:00:42.784 INFO:teuthology.orchestra.run.smithi018.stdout:(2/10): python3-pytoml-0.1.14-5.git7dea353.el8. 204 kB/s | 25 kB 00:00 2023-09-23T20:00:42.810 INFO:teuthology.orchestra.run.smithi018.stdout:(3/10): yajl-2.1.0-10.el8.x86_64.rpm 277 kB/s | 41 kB 00:00 2023-09-23T20:00:42.835 INFO:teuthology.orchestra.run.smithi018.stdout:(4/10): udica-0.2.6-4.module+el8.6.0+16771+28df 655 kB/s | 49 kB 00:00 2023-09-23T20:00:42.837 INFO:teuthology.orchestra.run.smithi149.stdout:(9/10): toolbox-0.0.99.3-0.6.module+el8.6.0+167 11 MB/s | 2.2 MB 00:00 2023-09-23T20:00:42.877 INFO:teuthology.orchestra.run.smithi018.stdout:(5/10): cockpit-podman-49.1-1.module+el8.6.0+16 5.5 MB/s | 524 kB 00:00 2023-09-23T20:00:42.902 INFO:teuthology.orchestra.run.smithi018.stdout:(6/10): python3-podman-4.0.1-1.module+el8.6.0+1 2.2 MB/s | 149 kB 00:00 2023-09-23T20:00:42.927 INFO:teuthology.orchestra.run.smithi018.stdout:(7/10): crun-1.5-1.module+el8.6.0+16771+28dfca7 4.2 MB/s | 213 kB 00:00 2023-09-23T20:00:42.929 INFO:teuthology.orchestra.run.smithi149.stdout:(10/10): skopeo-1.9.1-1.module+el8.6.0+16771+28 22 MB/s | 7.0 MB 00:00 2023-09-23T20:00:42.930 INFO:teuthology.orchestra.run.smithi149.stdout:-------------------------------------------------------------------------------- 2023-09-23T20:00:42.930 INFO:teuthology.orchestra.run.smithi149.stdout:Total 22 MB/s | 18 MB 00:00 2023-09-23T20:00:43.036 INFO:teuthology.orchestra.run.smithi018.stdout:(8/10): toolbox-0.0.99.3-0.6.module+el8.6.0+167 20 MB/s | 2.2 MB 00:00 2023-09-23T20:00:43.133 INFO:teuthology.orchestra.run.smithi149.stdout:Running transaction check 2023-09-23T20:00:43.153 INFO:teuthology.orchestra.run.smithi018.stdout:(9/10): buildah-1.26.2-1.module+el8.6.0+16771+2 23 MB/s | 8.0 MB 00:00 2023-09-23T20:00:43.153 INFO:teuthology.orchestra.run.smithi149.stdout:Transaction check succeeded. 2023-09-23T20:00:43.153 INFO:teuthology.orchestra.run.smithi149.stdout:Running transaction test 2023-09-23T20:00:43.253 INFO:teuthology.orchestra.run.smithi018.stdout:(10/10): skopeo-1.9.1-1.module+el8.6.0+16771+28 20 MB/s | 7.0 MB 00:00 2023-09-23T20:00:43.253 INFO:teuthology.orchestra.run.smithi018.stdout:-------------------------------------------------------------------------------- 2023-09-23T20:00:43.253 INFO:teuthology.orchestra.run.smithi018.stdout:Total 31 MB/s | 18 MB 00:00 2023-09-23T20:00:43.335 INFO:teuthology.orchestra.run.smithi149.stdout:Transaction test succeeded. 2023-09-23T20:00:43.339 INFO:teuthology.orchestra.run.smithi149.stdout:Running transaction 2023-09-23T20:00:43.459 INFO:teuthology.orchestra.run.smithi018.stdout:Running transaction check 2023-09-23T20:00:43.479 INFO:teuthology.orchestra.run.smithi018.stdout:Transaction check succeeded. 2023-09-23T20:00:43.479 INFO:teuthology.orchestra.run.smithi018.stdout:Running transaction test 2023-09-23T20:00:43.672 INFO:teuthology.orchestra.run.smithi018.stdout:Transaction test succeeded. 2023-09-23T20:00:43.676 INFO:teuthology.orchestra.run.smithi018.stdout:Running transaction 2023-09-23T20:00:43.868 INFO:teuthology.orchestra.run.smithi149.stdout: Preparing : 1/1 2023-09-23T20:00:44.038 INFO:teuthology.orchestra.run.smithi149.stdout: Installing : python3-pytoml-0.1.14-5.git7dea353.el8.noarch 1/10 2023-09-23T20:00:44.185 INFO:teuthology.orchestra.run.smithi018.stdout: Preparing : 1/1 2023-09-23T20:00:44.195 INFO:teuthology.orchestra.run.smithi149.stdout: Installing : yajl-2.1.0-10.el8.x86_64 2/10 2023-09-23T20:00:44.373 INFO:teuthology.orchestra.run.smithi149.stdout: Installing : python3-pyxdg-0.25-16.el8.noarch 3/10 2023-09-23T20:00:44.487 INFO:teuthology.orchestra.run.smithi018.stdout: Installing : python3-pytoml-0.1.14-5.git7dea353.el8.noarch 1/10 2023-09-23T20:00:44.568 INFO:teuthology.orchestra.run.smithi149.stdout: Installing : python3-podman-4.0.1-1.module+el8.6.0+16837+85b822 4/10 2023-09-23T20:00:44.648 INFO:teuthology.orchestra.run.smithi018.stdout: Installing : yajl-2.1.0-10.el8.x86_64 2/10 2023-09-23T20:00:44.862 INFO:teuthology.orchestra.run.smithi018.stdout: Installing : python3-pyxdg-0.25-16.el8.noarch 3/10 2023-09-23T20:00:44.950 INFO:teuthology.orchestra.run.smithi149.stdout: Installing : crun-1.5-1.module+el8.6.0+16771+28dfca77.x86_64 5/10 2023-09-23T20:00:45.100 INFO:teuthology.orchestra.run.smithi018.stdout: Installing : python3-podman-4.0.1-1.module+el8.6.0+16837+85b822 4/10 2023-09-23T20:00:45.447 INFO:teuthology.orchestra.run.smithi018.stdout: Installing : crun-1.5-1.module+el8.6.0+16771+28dfca77.x86_64 5/10 2023-09-23T20:00:45.690 INFO:teuthology.orchestra.run.smithi149.stdout: Installing : toolbox-0.0.99.3-0.6.module+el8.6.0+16771+28dfca77 6/10 2023-09-23T20:00:46.245 INFO:teuthology.orchestra.run.smithi018.stdout: Installing : toolbox-0.0.99.3-0.6.module+el8.6.0+16771+28dfca77 6/10 2023-09-23T20:00:46.597 INFO:teuthology.orchestra.run.smithi149.stdout: Installing : skopeo-2:1.9.1-1.module+el8.6.0+16771+28dfca77.x86 7/10 2023-09-23T20:00:46.813 INFO:teuthology.orchestra.run.smithi149.stdout: Installing : buildah-1:1.26.2-1.module+el8.6.0+16771+28dfca77.x 8/10 2023-09-23T20:00:46.978 INFO:teuthology.orchestra.run.smithi149.stdout: Installing : cockpit-podman-49.1-1.module+el8.6.0+16771+28dfca7 9/10 2023-09-23T20:00:47.171 INFO:teuthology.orchestra.run.smithi149.stdout: Installing : udica-0.2.6-4.module+el8.6.0+16771+28dfca77.noarch 10/10 2023-09-23T20:00:47.183 INFO:teuthology.orchestra.run.smithi018.stdout: Installing : skopeo-2:1.9.1-1.module+el8.6.0+16771+28dfca77.x86 7/10 2023-09-23T20:00:47.395 INFO:teuthology.orchestra.run.smithi018.stdout: Installing : buildah-1:1.26.2-1.module+el8.6.0+16771+28dfca77.x 8/10 2023-09-23T20:00:47.450 INFO:teuthology.orchestra.run.smithi149.stdout: Running scriptlet: udica-0.2.6-4.module+el8.6.0+16771+28dfca77.noarch 10/10 2023-09-23T20:00:47.450 INFO:teuthology.orchestra.run.smithi149.stdout: Verifying : python3-pyxdg-0.25-16.el8.noarch 1/10 2023-09-23T20:00:47.450 INFO:teuthology.orchestra.run.smithi149.stdout: Verifying : yajl-2.1.0-10.el8.x86_64 2/10 2023-09-23T20:00:47.450 INFO:teuthology.orchestra.run.smithi149.stdout: Verifying : python3-pytoml-0.1.14-5.git7dea353.el8.noarch 3/10 2023-09-23T20:00:47.450 INFO:teuthology.orchestra.run.smithi149.stdout: Verifying : udica-0.2.6-4.module+el8.6.0+16771+28dfca77.noarch 4/10 2023-09-23T20:00:47.450 INFO:teuthology.orchestra.run.smithi149.stdout: Verifying : cockpit-podman-49.1-1.module+el8.6.0+16771+28dfca7 5/10 2023-09-23T20:00:47.450 INFO:teuthology.orchestra.run.smithi149.stdout: Verifying : buildah-1:1.26.2-1.module+el8.6.0+16771+28dfca77.x 6/10 2023-09-23T20:00:47.450 INFO:teuthology.orchestra.run.smithi149.stdout: Verifying : python3-podman-4.0.1-1.module+el8.6.0+16837+85b822 7/10 2023-09-23T20:00:47.450 INFO:teuthology.orchestra.run.smithi149.stdout: Verifying : crun-1.5-1.module+el8.6.0+16771+28dfca77.x86_64 8/10 2023-09-23T20:00:47.451 INFO:teuthology.orchestra.run.smithi149.stdout: Verifying : skopeo-2:1.9.1-1.module+el8.6.0+16771+28dfca77.x86 9/10 2023-09-23T20:00:47.554 INFO:teuthology.orchestra.run.smithi018.stdout: Installing : cockpit-podman-49.1-1.module+el8.6.0+16771+28dfca7 9/10 2023-09-23T20:00:47.747 INFO:teuthology.orchestra.run.smithi018.stdout: Installing : udica-0.2.6-4.module+el8.6.0+16771+28dfca77.noarch 10/10 2023-09-23T20:00:48.025 INFO:teuthology.orchestra.run.smithi018.stdout: Running scriptlet: udica-0.2.6-4.module+el8.6.0+16771+28dfca77.noarch 10/10 2023-09-23T20:00:48.025 INFO:teuthology.orchestra.run.smithi018.stdout: Verifying : python3-pyxdg-0.25-16.el8.noarch 1/10 2023-09-23T20:00:48.025 INFO:teuthology.orchestra.run.smithi018.stdout: Verifying : yajl-2.1.0-10.el8.x86_64 2/10 2023-09-23T20:00:48.025 INFO:teuthology.orchestra.run.smithi018.stdout: Verifying : python3-pytoml-0.1.14-5.git7dea353.el8.noarch 3/10 2023-09-23T20:00:48.025 INFO:teuthology.orchestra.run.smithi018.stdout: Verifying : udica-0.2.6-4.module+el8.6.0+16771+28dfca77.noarch 4/10 2023-09-23T20:00:48.025 INFO:teuthology.orchestra.run.smithi018.stdout: Verifying : cockpit-podman-49.1-1.module+el8.6.0+16771+28dfca7 5/10 2023-09-23T20:00:48.025 INFO:teuthology.orchestra.run.smithi018.stdout: Verifying : buildah-1:1.26.2-1.module+el8.6.0+16771+28dfca77.x 6/10 2023-09-23T20:00:48.026 INFO:teuthology.orchestra.run.smithi018.stdout: Verifying : python3-podman-4.0.1-1.module+el8.6.0+16837+85b822 7/10 2023-09-23T20:00:48.026 INFO:teuthology.orchestra.run.smithi018.stdout: Verifying : crun-1.5-1.module+el8.6.0+16771+28dfca77.x86_64 8/10 2023-09-23T20:00:48.026 INFO:teuthology.orchestra.run.smithi018.stdout: Verifying : skopeo-2:1.9.1-1.module+el8.6.0+16771+28dfca77.x86 9/10 2023-09-23T20:00:48.612 INFO:teuthology.orchestra.run.smithi149.stdout: Verifying : toolbox-0.0.99.3-0.6.module+el8.6.0+16771+28dfca77 10/10 2023-09-23T20:00:48.612 INFO:teuthology.orchestra.run.smithi149.stdout:Last metadata expiration check: 0:01:54 ago on Sat 23 Sep 2023 07:58:54 PM UTC. 2023-09-23T20:00:49.149 INFO:teuthology.orchestra.run.smithi018.stdout: Verifying : toolbox-0.0.99.3-0.6.module+el8.6.0+16771+28dfca77 10/10 2023-09-23T20:00:49.149 INFO:teuthology.orchestra.run.smithi018.stdout:Last metadata expiration check: 0:01:44 ago on Sat 23 Sep 2023 07:59:04 PM UTC. 2023-09-23T20:00:58.075 INFO:teuthology.orchestra.run.smithi149.stdout:Installed products updated. 2023-09-23T20:00:58.267 INFO:teuthology.orchestra.run.smithi149.stdout: 2023-09-23T20:00:58.268 INFO:teuthology.orchestra.run.smithi149.stdout:Installed: 2023-09-23T20:00:58.268 INFO:teuthology.orchestra.run.smithi149.stdout: buildah-1:1.26.2-1.module+el8.6.0+16771+28dfca77.x86_64 2023-09-23T20:00:58.268 INFO:teuthology.orchestra.run.smithi149.stdout: cockpit-podman-49.1-1.module+el8.6.0+16771+28dfca77.noarch 2023-09-23T20:00:58.268 INFO:teuthology.orchestra.run.smithi149.stdout: crun-1.5-1.module+el8.6.0+16771+28dfca77.x86_64 2023-09-23T20:00:58.268 INFO:teuthology.orchestra.run.smithi149.stdout: python3-podman-4.0.1-1.module+el8.6.0+16837+85b82211.noarch 2023-09-23T20:00:58.268 INFO:teuthology.orchestra.run.smithi149.stdout: python3-pytoml-0.1.14-5.git7dea353.el8.noarch 2023-09-23T20:00:58.268 INFO:teuthology.orchestra.run.smithi149.stdout: python3-pyxdg-0.25-16.el8.noarch 2023-09-23T20:00:58.268 INFO:teuthology.orchestra.run.smithi149.stdout: skopeo-2:1.9.1-1.module+el8.6.0+16771+28dfca77.x86_64 2023-09-23T20:00:58.268 INFO:teuthology.orchestra.run.smithi149.stdout: toolbox-0.0.99.3-0.6.module+el8.6.0+16771+28dfca77.x86_64 2023-09-23T20:00:58.268 INFO:teuthology.orchestra.run.smithi149.stdout: udica-0.2.6-4.module+el8.6.0+16771+28dfca77.noarch 2023-09-23T20:00:58.268 INFO:teuthology.orchestra.run.smithi149.stdout: yajl-2.1.0-10.el8.x86_64 2023-09-23T20:00:58.269 INFO:teuthology.orchestra.run.smithi149.stdout: 2023-09-23T20:00:58.269 INFO:teuthology.orchestra.run.smithi149.stdout:Complete! 2023-09-23T20:00:58.339 INFO:teuthology.orchestra.run.smithi018.stdout:Installed products updated. 2023-09-23T20:00:58.417 DEBUG:teuthology.parallel:result is None 2023-09-23T20:00:58.551 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:00:58.551 INFO:teuthology.orchestra.run.smithi018.stdout:Installed: 2023-09-23T20:00:58.551 INFO:teuthology.orchestra.run.smithi018.stdout: buildah-1:1.26.2-1.module+el8.6.0+16771+28dfca77.x86_64 2023-09-23T20:00:58.552 INFO:teuthology.orchestra.run.smithi018.stdout: cockpit-podman-49.1-1.module+el8.6.0+16771+28dfca77.noarch 2023-09-23T20:00:58.552 INFO:teuthology.orchestra.run.smithi018.stdout: crun-1.5-1.module+el8.6.0+16771+28dfca77.x86_64 2023-09-23T20:00:58.552 INFO:teuthology.orchestra.run.smithi018.stdout: python3-podman-4.0.1-1.module+el8.6.0+16837+85b82211.noarch 2023-09-23T20:00:58.552 INFO:teuthology.orchestra.run.smithi018.stdout: python3-pytoml-0.1.14-5.git7dea353.el8.noarch 2023-09-23T20:00:58.552 INFO:teuthology.orchestra.run.smithi018.stdout: python3-pyxdg-0.25-16.el8.noarch 2023-09-23T20:00:58.552 INFO:teuthology.orchestra.run.smithi018.stdout: skopeo-2:1.9.1-1.module+el8.6.0+16771+28dfca77.x86_64 2023-09-23T20:00:58.552 INFO:teuthology.orchestra.run.smithi018.stdout: toolbox-0.0.99.3-0.6.module+el8.6.0+16771+28dfca77.x86_64 2023-09-23T20:00:58.552 INFO:teuthology.orchestra.run.smithi018.stdout: udica-0.2.6-4.module+el8.6.0+16771+28dfca77.noarch 2023-09-23T20:00:58.552 INFO:teuthology.orchestra.run.smithi018.stdout: yajl-2.1.0-10.el8.x86_64 2023-09-23T20:00:58.552 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:00:58.552 INFO:teuthology.orchestra.run.smithi018.stdout:Complete! 2023-09-23T20:00:58.691 DEBUG:teuthology.parallel:result is None 2023-09-23T20:00:58.692 INFO:teuthology.run_tasks:Running task install... 2023-09-23T20:00:58.704 DEBUG:teuthology.task.install:project ceph 2023-09-23T20:00:58.704 DEBUG:teuthology.task.install:INSTALL overrides: {'ceph': {'flavor': 'default', 'sha1': 'bb0daeff97f020f686596fca6eba20c310e26fb7'}} 2023-09-23T20:00:58.705 DEBUG:teuthology.task.install:config {'branch': 'quincy', 'exclude_packages': ['ceph-volume'], 'flavor': 'default', 'sha1': 'bb0daeff97f020f686596fca6eba20c310e26fb7'} 2023-09-23T20:00:58.705 INFO:teuthology.task.install:Using flavor: default 2023-09-23T20:00:58.718 DEBUG:teuthology.task.install:Package list is: {'deb': ['ceph', 'cephadm', 'ceph-mds', 'ceph-mgr', 'ceph-common', 'ceph-fuse', 'ceph-test', 'radosgw', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'libcephfs2', 'libcephfs-dev', 'librados2', 'librbd1', 'rbd-fuse'], 'rpm': ['ceph-radosgw', 'ceph-test', 'ceph', 'ceph-base', 'cephadm', 'ceph-immutable-object-cache', 'ceph-mgr', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'ceph-fuse', 'librados-devel', 'libcephfs2', 'libcephfs-devel', 'librados2', 'librbd1', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'rbd-fuse', 'rbd-mirror', 'rbd-nbd']} 2023-09-23T20:00:58.718 INFO:teuthology.task.install:extra packages: [] 2023-09-23T20:00:58.718 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': 'quincy', 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': ['ceph-volume'], 'extra_packages': [], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': 'bb0daeff97f020f686596fca6eba20c310e26fb7', 'tag': None, 'wait_for_package': False} 2023-09-23T20:00:58.719 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-09-23T20:00:58.719 INFO:teuthology.packaging:ref: None 2023-09-23T20:00:58.719 INFO:teuthology.packaging:tag: None 2023-09-23T20:00:58.719 INFO:teuthology.packaging:branch: quincy 2023-09-23T20:00:58.720 INFO:teuthology.packaging:sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T20:00:58.720 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=quincy 2023-09-23T20:00:58.723 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': 'quincy', 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': ['ceph-volume'], 'extra_packages': [], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': 'bb0daeff97f020f686596fca6eba20c310e26fb7', 'tag': None, 'wait_for_package': False} 2023-09-23T20:00:58.723 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-09-23T20:00:58.724 INFO:teuthology.packaging:ref: None 2023-09-23T20:00:58.724 INFO:teuthology.packaging:tag: None 2023-09-23T20:00:58.724 INFO:teuthology.packaging:branch: quincy 2023-09-23T20:00:58.724 INFO:teuthology.packaging:sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T20:00:58.725 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=quincy 2023-09-23T20:00:58.948 INFO:teuthology.task.install.rpm:Pulling from https://3.chacra.ceph.com/r/ceph/quincy/61431f6ba2752649b506867ec856b4a6236f90dd/centos/8/flavors/default/ 2023-09-23T20:00:58.949 INFO:teuthology.task.install.rpm:Package version is 17.2.6-1123.g61431f6b 2023-09-23T20:00:59.055 INFO:teuthology.task.install.rpm:Pulling from https://3.chacra.ceph.com/r/ceph/quincy/61431f6ba2752649b506867ec856b4a6236f90dd/centos/8/flavors/default/ 2023-09-23T20:00:59.056 INFO:teuthology.task.install.rpm:Package version is 17.2.6-1123.g61431f6b 2023-09-23T20:00:59.060 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://3.chacra.ceph.com/r/ceph/quincy/61431f6ba2752649b506867ec856b4a6236f90dd/centos/8/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://3.chacra.ceph.com/r/ceph/quincy/61431f6ba2752649b506867ec856b4a6236f90dd/centos/8/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://3.chacra.ceph.com/r/ceph/quincy/61431f6ba2752649b506867ec856b4a6236f90dd/centos/8/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2023-09-23T20:00:59.060 DEBUG:teuthology.orchestra.run.smithi149:> set -ex 2023-09-23T20:00:59.060 DEBUG:teuthology.orchestra.run.smithi149:> sudo dd of=/etc/yum.repos.d/ceph.repo 2023-09-23T20:00:59.089 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd on remote rpm x86_64 2023-09-23T20:00:59.089 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-09-23T20:00:59.089 INFO:teuthology.packaging:ref: None 2023-09-23T20:00:59.090 INFO:teuthology.packaging:tag: None 2023-09-23T20:00:59.090 INFO:teuthology.packaging:branch: quincy 2023-09-23T20:00:59.090 INFO:teuthology.packaging:sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T20:00:59.090 DEBUG:teuthology.orchestra.run.smithi149:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;ref/quincy/;g' /etc/yum.repos.d/ceph.repo ; fi 2023-09-23T20:00:59.158 DEBUG:teuthology.orchestra.run.smithi149:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2023-09-23T20:00:59.223 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://3.chacra.ceph.com/r/ceph/quincy/61431f6ba2752649b506867ec856b4a6236f90dd/centos/8/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://3.chacra.ceph.com/r/ceph/quincy/61431f6ba2752649b506867ec856b4a6236f90dd/centos/8/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://3.chacra.ceph.com/r/ceph/quincy/61431f6ba2752649b506867ec856b4a6236f90dd/centos/8/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2023-09-23T20:00:59.224 DEBUG:teuthology.orchestra.run.smithi018:> set -ex 2023-09-23T20:00:59.224 DEBUG:teuthology.orchestra.run.smithi018:> sudo dd of=/etc/yum.repos.d/ceph.repo 2023-09-23T20:00:59.236 DEBUG:teuthology.orchestra.run.smithi149:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2023-09-23T20:00:59.253 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd on remote rpm x86_64 2023-09-23T20:00:59.253 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-09-23T20:00:59.253 INFO:teuthology.packaging:ref: None 2023-09-23T20:00:59.254 INFO:teuthology.packaging:tag: None 2023-09-23T20:00:59.254 INFO:teuthology.packaging:branch: quincy 2023-09-23T20:00:59.254 INFO:teuthology.packaging:sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T20:00:59.254 DEBUG:teuthology.orchestra.run.smithi018:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;ref/quincy/;g' /etc/yum.repos.d/ceph.repo ; fi 2023-09-23T20:00:59.303 INFO:teuthology.orchestra.run.smithi149.stdout:check_obsoletes = 1 2023-09-23T20:00:59.304 DEBUG:teuthology.orchestra.run.smithi149:> sudo yum clean all 2023-09-23T20:00:59.324 DEBUG:teuthology.orchestra.run.smithi018:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2023-09-23T20:00:59.404 DEBUG:teuthology.orchestra.run.smithi018:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2023-09-23T20:00:59.472 INFO:teuthology.orchestra.run.smithi018.stdout:check_obsoletes = 1 2023-09-23T20:00:59.473 DEBUG:teuthology.orchestra.run.smithi018:> sudo yum clean all 2023-09-23T20:00:59.677 INFO:teuthology.orchestra.run.smithi149.stdout:Updating Subscription Management repositories. 2023-09-23T20:00:59.866 INFO:teuthology.orchestra.run.smithi018.stdout:Updating Subscription Management repositories. 2023-09-23T20:01:00.614 INFO:teuthology.orchestra.run.smithi149.stdout:44 files removed 2023-09-23T20:01:00.667 DEBUG:teuthology.orchestra.run.smithi149:> sudo yum -y install ceph-radosgw 2023-09-23T20:01:00.684 INFO:teuthology.orchestra.run.smithi018.stdout:44 files removed 2023-09-23T20:01:00.736 DEBUG:teuthology.orchestra.run.smithi018:> sudo yum -y install ceph-radosgw 2023-09-23T20:01:00.999 INFO:teuthology.orchestra.run.smithi149.stdout:Updating Subscription Management repositories. 2023-09-23T20:01:01.091 INFO:teuthology.orchestra.run.smithi018.stdout:Updating Subscription Management repositories. 2023-09-23T20:01:02.451 INFO:teuthology.orchestra.run.smithi018.stdout:ceph packages for x86_64 244 kB/s | 78 kB 00:00 2023-09-23T20:01:02.531 INFO:teuthology.orchestra.run.smithi149.stdout:ceph packages for x86_64 182 kB/s | 78 kB 00:00 2023-09-23T20:01:02.806 INFO:teuthology.orchestra.run.smithi018.stdout:ceph noarch packages 28 kB/s | 9.1 kB 00:00 2023-09-23T20:01:02.832 INFO:teuthology.orchestra.run.smithi149.stdout:ceph noarch packages 34 kB/s | 9.1 kB 00:00 2023-09-23T20:01:03.057 INFO:teuthology.orchestra.run.smithi018.stdout:ceph source packages 6.4 kB/s | 1.5 kB 00:00 2023-09-23T20:01:03.183 INFO:teuthology.orchestra.run.smithi149.stdout:ceph source packages 4.4 kB/s | 1.5 kB 00:00 2023-09-23T20:01:03.191 INFO:teuthology.orchestra.run.smithi018.stdout:Copr repo for python3-asyncssh owned by ceph 31 kB/s | 3.5 kB 00:00 2023-09-23T20:01:03.317 INFO:teuthology.orchestra.run.smithi149.stdout:Copr repo for python3-asyncssh owned by ceph 30 kB/s | 3.5 kB 00:00 2023-09-23T20:01:08.868 INFO:teuthology.orchestra.run.smithi018.stdout:Extra Packages for Enterprise Linux 2.8 MB/s | 16 MB 00:05 2023-09-23T20:01:12.557 INFO:teuthology.orchestra.run.smithi018.stdout:lab-extras 418 kB/s | 24 kB 00:00 2023-09-23T20:01:13.834 INFO:teuthology.orchestra.run.smithi018.stdout:Red Hat Enterprise Linux 8 for x86_64 - BaseOS 44 MB/s | 53 MB 00:01 2023-09-23T20:01:22.935 INFO:teuthology.orchestra.run.smithi018.stdout:Red Hat Enterprise Linux 8 for x86_64 - AppStre 36 MB/s | 47 MB 00:01 2023-09-23T20:01:34.747 INFO:teuthology.orchestra.run.smithi018.stdout:Dependencies resolved. 2023-09-23T20:01:34.749 INFO:teuthology.orchestra.run.smithi018.stdout:==================================================================================================== 2023-09-23T20:01:34.750 INFO:teuthology.orchestra.run.smithi018.stdout: Package Arch Version Repository Size 2023-09-23T20:01:34.750 INFO:teuthology.orchestra.run.smithi018.stdout:==================================================================================================== 2023-09-23T20:01:34.750 INFO:teuthology.orchestra.run.smithi018.stdout:Installing: 2023-09-23T20:01:34.750 INFO:teuthology.orchestra.run.smithi018.stdout: ceph-radosgw x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 17 M 2023-09-23T20:01:34.750 INFO:teuthology.orchestra.run.smithi018.stdout:Upgrading: 2023-09-23T20:01:34.750 INFO:teuthology.orchestra.run.smithi018.stdout: librados2 x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 3.5 M 2023-09-23T20:01:34.750 INFO:teuthology.orchestra.run.smithi018.stdout: librbd1 x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 3.6 M 2023-09-23T20:01:34.750 INFO:teuthology.orchestra.run.smithi018.stdout:Installing dependencies: 2023-09-23T20:01:34.750 INFO:teuthology.orchestra.run.smithi018.stdout: ceph-base x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 5.2 M 2023-09-23T20:01:34.750 INFO:teuthology.orchestra.run.smithi018.stdout: ceph-common x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 23 M 2023-09-23T20:01:34.750 INFO:teuthology.orchestra.run.smithi018.stdout: ceph-selinux x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 24 k 2023-09-23T20:01:34.751 INFO:teuthology.orchestra.run.smithi018.stdout: gperftools-libs x86_64 1:2.7-9.el8 epel 306 k 2023-09-23T20:01:34.751 INFO:teuthology.orchestra.run.smithi018.stdout: libcephfs2 x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 796 k 2023-09-23T20:01:34.751 INFO:teuthology.orchestra.run.smithi018.stdout: liboath x86_64 2.6.2-3.el8 epel 59 k 2023-09-23T20:01:34.751 INFO:teuthology.orchestra.run.smithi018.stdout: librabbitmq x86_64 0.9.0-3.el8 rhel-8-for-x86_64-baseos-rpms 47 k 2023-09-23T20:01:34.751 INFO:teuthology.orchestra.run.smithi018.stdout: libradosstriper1 x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 506 k 2023-09-23T20:01:34.751 INFO:teuthology.orchestra.run.smithi018.stdout: librdkafka x86_64 0.11.4-3.el8 rhel-8-for-x86_64-appstream-rpms 354 k 2023-09-23T20:01:34.751 INFO:teuthology.orchestra.run.smithi018.stdout: librgw2 x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 5.5 M 2023-09-23T20:01:34.751 INFO:teuthology.orchestra.run.smithi018.stdout: libunwind x86_64 1.3.1-3.el8 epel 75 k 2023-09-23T20:01:34.751 INFO:teuthology.orchestra.run.smithi018.stdout: lttng-ust x86_64 2.8.1-11.el8 rhel-8-for-x86_64-appstream-rpms 259 k 2023-09-23T20:01:34.751 INFO:teuthology.orchestra.run.smithi018.stdout: python3-ceph-argparse x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 45 k 2023-09-23T20:01:34.751 INFO:teuthology.orchestra.run.smithi018.stdout: python3-ceph-common x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 115 k 2023-09-23T20:01:34.752 INFO:teuthology.orchestra.run.smithi018.stdout: python3-cephfs x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 190 k 2023-09-23T20:01:34.752 INFO:teuthology.orchestra.run.smithi018.stdout: python3-prettytable noarch 0.7.2-14.el8 rhel-8-for-x86_64-appstream-rpms 44 k 2023-09-23T20:01:34.752 INFO:teuthology.orchestra.run.smithi018.stdout: python3-rados x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 371 k 2023-09-23T20:01:34.752 INFO:teuthology.orchestra.run.smithi018.stdout: python3-rbd x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 358 k 2023-09-23T20:01:34.752 INFO:teuthology.orchestra.run.smithi018.stdout: python3-rgw x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 106 k 2023-09-23T20:01:34.752 INFO:teuthology.orchestra.run.smithi018.stdout: thrift x86_64 0.13.0-2.el8 epel 1.7 M 2023-09-23T20:01:34.752 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:01:34.752 INFO:teuthology.orchestra.run.smithi018.stdout:Transaction Summary 2023-09-23T20:01:34.752 INFO:teuthology.orchestra.run.smithi018.stdout:==================================================================================================== 2023-09-23T20:01:34.752 INFO:teuthology.orchestra.run.smithi018.stdout:Install 21 Packages 2023-09-23T20:01:34.752 INFO:teuthology.orchestra.run.smithi018.stdout:Upgrade 2 Packages 2023-09-23T20:01:34.753 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:01:34.753 INFO:teuthology.orchestra.run.smithi018.stdout:Total download size: 63 M 2023-09-23T20:01:34.753 INFO:teuthology.orchestra.run.smithi018.stdout:Downloading Packages: 2023-09-23T20:01:35.444 INFO:teuthology.orchestra.run.smithi018.stdout:(1/23): ceph-base-17.2.6-1123.g61431f6b.el8.x86 8.1 MB/s | 5.2 MB 00:00 2023-09-23T20:01:35.510 INFO:teuthology.orchestra.run.smithi018.stdout:(2/23): ceph-selinux-17.2.6-1123.g61431f6b.el8. 366 kB/s | 24 kB 00:00 2023-09-23T20:01:35.635 INFO:teuthology.orchestra.run.smithi018.stdout:(3/23): libcephfs2-17.2.6-1123.g61431f6b.el8.x8 6.2 MB/s | 796 kB 00:00 2023-09-23T20:01:35.727 INFO:teuthology.orchestra.run.smithi018.stdout:(4/23): libradosstriper1-17.2.6-1123.g61431f6b. 5.4 MB/s | 506 kB 00:00 2023-09-23T20:01:36.386 INFO:teuthology.orchestra.run.smithi018.stdout:(5/23): librgw2-17.2.6-1123.g61431f6b.el8.x86_6 8.4 MB/s | 5.5 MB 00:00 2023-09-23T20:01:36.620 INFO:teuthology.orchestra.run.smithi018.stdout:(6/23): ceph-common-17.2.6-1123.g61431f6b.el8.x 13 MB/s | 23 MB 00:01 2023-09-23T20:01:36.645 INFO:teuthology.orchestra.run.smithi018.stdout:(7/23): python3-ceph-argparse-17.2.6-1123.g6143 172 kB/s | 45 kB 00:00 2023-09-23T20:01:36.670 INFO:teuthology.orchestra.run.smithi018.stdout:(8/23): python3-ceph-common-17.2.6-1123.g61431f 2.2 MB/s | 115 kB 00:00 2023-09-23T20:01:36.704 INFO:teuthology.orchestra.run.smithi018.stdout:(9/23): python3-cephfs-17.2.6-1123.g61431f6b.el 3.2 MB/s | 190 kB 00:00 2023-09-23T20:01:36.729 INFO:teuthology.orchestra.run.smithi018.stdout:(10/23): python3-rados-17.2.6-1123.g61431f6b.el 6.2 MB/s | 371 kB 00:00 2023-09-23T20:01:36.763 INFO:teuthology.orchestra.run.smithi018.stdout:(11/23): python3-rbd-17.2.6-1123.g61431f6b.el8. 6.0 MB/s | 358 kB 00:00 2023-09-23T20:01:36.788 INFO:teuthology.orchestra.run.smithi018.stdout:(12/23): python3-rgw-17.2.6-1123.g61431f6b.el8. 1.8 MB/s | 106 kB 00:00 2023-09-23T20:01:37.189 INFO:teuthology.orchestra.run.smithi018.stdout:(13/23): liboath-2.6.2-3.el8.x86_64.rpm 147 kB/s | 59 kB 00:00 2023-09-23T20:01:37.339 INFO:teuthology.orchestra.run.smithi018.stdout:(14/23): gperftools-libs-2.7-9.el8.x86_64.rpm 532 kB/s | 306 kB 00:00 2023-09-23T20:01:37.364 INFO:teuthology.orchestra.run.smithi018.stdout:(15/23): libunwind-1.3.1-3.el8.x86_64.rpm 428 kB/s | 75 kB 00:00 2023-09-23T20:01:37.415 INFO:teuthology.orchestra.run.smithi018.stdout:(16/23): librabbitmq-0.9.0-3.el8.x86_64.rpm 940 kB/s | 47 kB 00:00 2023-09-23T20:01:37.473 INFO:teuthology.orchestra.run.smithi018.stdout:(17/23): lttng-ust-2.8.1-11.el8.x86_64.rpm 4.3 MB/s | 259 kB 00:00 2023-09-23T20:01:37.498 INFO:teuthology.orchestra.run.smithi018.stdout:(18/23): python3-prettytable-0.7.2-14.el8.noarc 1.7 MB/s | 44 kB 00:00 2023-09-23T20:01:37.540 INFO:teuthology.orchestra.run.smithi018.stdout:(19/23): librdkafka-0.11.4-3.el8.x86_64.rpm 8.3 MB/s | 354 kB 00:00 2023-09-23T20:01:37.699 INFO:teuthology.orchestra.run.smithi018.stdout:(20/23): thrift-0.13.0-2.el8.x86_64.rpm 4.8 MB/s | 1.7 MB 00:00 2023-09-23T20:01:37.825 INFO:teuthology.orchestra.run.smithi018.stdout:(21/23): librados2-17.2.6-1123.g61431f6b.el8.x8 12 MB/s | 3.5 MB 00:00 2023-09-23T20:01:38.041 INFO:teuthology.orchestra.run.smithi018.stdout:(22/23): librbd1-17.2.6-1123.g61431f6b.el8.x86_ 10 MB/s | 3.6 MB 00:00 2023-09-23T20:01:38.408 INFO:teuthology.orchestra.run.smithi018.stdout:(23/23): ceph-radosgw-17.2.6-1123.g61431f6b.el8 4.7 MB/s | 17 MB 00:03 2023-09-23T20:01:38.411 INFO:teuthology.orchestra.run.smithi018.stdout:-------------------------------------------------------------------------------- 2023-09-23T20:01:38.411 INFO:teuthology.orchestra.run.smithi018.stdout:Total 17 MB/s | 63 MB 00:03 2023-09-23T20:01:38.458 INFO:teuthology.orchestra.run.smithi018.stdout:Running transaction check 2023-09-23T20:01:38.507 INFO:teuthology.orchestra.run.smithi018.stdout:Transaction check succeeded. 2023-09-23T20:01:38.507 INFO:teuthology.orchestra.run.smithi018.stdout:Running transaction test 2023-09-23T20:01:38.875 INFO:teuthology.orchestra.run.smithi018.stdout:Transaction test succeeded. 2023-09-23T20:01:38.879 INFO:teuthology.orchestra.run.smithi018.stdout:Running transaction 2023-09-23T20:01:39.971 INFO:teuthology.orchestra.run.smithi018.stdout: Preparing : 1/1 2023-09-23T20:01:40.157 INFO:teuthology.orchestra.run.smithi018.stdout: Installing : lttng-ust-2.8.1-11.el8.x86_64 1/25 2023-09-23T20:01:40.559 INFO:teuthology.orchestra.run.smithi018.stdout: Running scriptlet: lttng-ust-2.8.1-11.el8.x86_64 1/25 2023-09-23T20:01:40.749 INFO:teuthology.orchestra.run.smithi018.stdout: Upgrading : librados2-2:17.2.6-1123.g61431f6b.el8.x86_64 2/25 2023-09-23T20:01:40.860 INFO:teuthology.orchestra.run.smithi018.stdout: Running scriptlet: librados2-2:17.2.6-1123.g61431f6b.el8.x86_64 2/25 2023-09-23T20:01:41.046 INFO:teuthology.orchestra.run.smithi018.stdout: Installing : python3-rados-2:17.2.6-1123.g61431f6b.el8.x86_64 3/25 2023-09-23T20:01:41.311 INFO:teuthology.orchestra.run.smithi018.stdout: Installing : liboath-2.6.2-3.el8.x86_64 4/25 2023-09-23T20:01:41.640 INFO:teuthology.orchestra.run.smithi018.stdout: Installing : libcephfs2-2:17.2.6-1123.g61431f6b.el8.x86_64 5/25 2023-09-23T20:01:42.066 INFO:teuthology.orchestra.run.smithi018.stdout: Running scriptlet: libcephfs2-2:17.2.6-1123.g61431f6b.el8.x86_64 5/25 2023-09-23T20:01:42.282 INFO:teuthology.orchestra.run.smithi018.stdout: Upgrading : librbd1-2:17.2.6-1123.g61431f6b.el8.x86_64 6/25 2023-09-23T20:01:42.379 INFO:teuthology.orchestra.run.smithi018.stdout: Running scriptlet: librbd1-2:17.2.6-1123.g61431f6b.el8.x86_64 6/25 2023-09-23T20:01:42.582 INFO:teuthology.orchestra.run.smithi018.stdout: Installing : librdkafka-0.11.4-3.el8.x86_64 7/25 2023-09-23T20:01:42.682 INFO:teuthology.orchestra.run.smithi018.stdout: Running scriptlet: librdkafka-0.11.4-3.el8.x86_64 7/25 2023-09-23T20:01:43.071 INFO:teuthology.orchestra.run.smithi018.stdout: Installing : librabbitmq-0.9.0-3.el8.x86_64 8/25 2023-09-23T20:01:43.763 INFO:teuthology.orchestra.run.smithi018.stdout: Installing : thrift-0.13.0-2.el8.x86_64 9/25 2023-09-23T20:01:44.023 INFO:teuthology.orchestra.run.smithi018.stdout: Installing : librgw2-2:17.2.6-1123.g61431f6b.el8.x86_64 10/25 2023-09-23T20:01:44.087 INFO:teuthology.orchestra.run.smithi018.stdout: Running scriptlet: librgw2-2:17.2.6-1123.g61431f6b.el8.x86_64 10/25 2023-09-23T20:01:44.218 INFO:teuthology.orchestra.run.smithi018.stdout: Installing : python3-ceph-argparse-2:17.2.6-1123.g61431f6b.el8. 11/25 2023-09-23T20:01:44.368 INFO:teuthology.orchestra.run.smithi018.stdout: Installing : python3-cephfs-2:17.2.6-1123.g61431f6b.el8.x86_64 12/25 2023-09-23T20:01:44.569 INFO:teuthology.orchestra.run.smithi018.stdout: Installing : python3-rgw-2:17.2.6-1123.g61431f6b.el8.x86_64 13/25 2023-09-23T20:01:44.771 INFO:teuthology.orchestra.run.smithi018.stdout: Installing : python3-rbd-2:17.2.6-1123.g61431f6b.el8.x86_64 14/25 2023-09-23T20:01:44.923 INFO:teuthology.orchestra.run.smithi018.stdout: Installing : libradosstriper1-2:17.2.6-1123.g61431f6b.el8.x86_6 15/25 2023-09-23T20:01:44.975 INFO:teuthology.orchestra.run.smithi018.stdout: Running scriptlet: libradosstriper1-2:17.2.6-1123.g61431f6b.el8.x86_6 15/25 2023-09-23T20:01:45.125 INFO:teuthology.orchestra.run.smithi018.stdout: Installing : python3-prettytable-0.7.2-14.el8.noarch 16/25 2023-09-23T20:01:45.299 INFO:teuthology.orchestra.run.smithi018.stdout: Installing : libunwind-1.3.1-3.el8.x86_64 17/25 2023-09-23T20:01:45.531 INFO:teuthology.orchestra.run.smithi018.stdout: Installing : gperftools-libs-1:2.7-9.el8.x86_64 18/25 2023-09-23T20:01:45.715 INFO:teuthology.orchestra.run.smithi018.stdout: Installing : python3-ceph-common-2:17.2.6-1123.g61431f6b.el8.x8 19/25 2023-09-23T20:01:49.715 INFO:teuthology.orchestra.run.smithi018.stdout: Running scriptlet: ceph-common-2:17.2.6-1123.g61431f6b.el8.x86_64 20/25 2023-09-23T20:01:49.947 INFO:teuthology.orchestra.run.smithi018.stdout: Installing : ceph-common-2:17.2.6-1123.g61431f6b.el8.x86_64 20/25 2023-09-23T20:01:50.567 INFO:teuthology.orchestra.run.smithi018.stdout: Running scriptlet: ceph-common-2:17.2.6-1123.g61431f6b.el8.x86_64 20/25 2023-09-23T20:01:50.571 INFO:teuthology.orchestra.run.smithi149.stdout:Extra Packages for Enterprise Linux 344 kB/s | 16 MB 00:47 2023-09-23T20:01:50.808 INFO:teuthology.orchestra.run.smithi018.stdout: Installing : ceph-base-2:17.2.6-1123.g61431f6b.el8.x86_64 21/25 2023-09-23T20:01:50.890 INFO:teuthology.orchestra.run.smithi018.stdout: Running scriptlet: ceph-base-2:17.2.6-1123.g61431f6b.el8.x86_64 21/25 2023-09-23T20:01:51.047 INFO:teuthology.orchestra.run.smithi018.stdout: Installing : ceph-selinux-2:17.2.6-1123.g61431f6b.el8.x86_64 22/25 2023-09-23T20:01:54.087 INFO:teuthology.orchestra.run.smithi149.stdout:lab-extras 356 kB/s | 24 kB 00:00 2023-09-23T20:01:55.014 INFO:teuthology.orchestra.run.smithi149.stdout:Red Hat Enterprise Linux 8 for x86_64 - BaseOS 62 MB/s | 53 MB 00:00 2023-09-23T20:02:03.456 INFO:teuthology.orchestra.run.smithi149.stdout:Red Hat Enterprise Linux 8 for x86_64 - AppStre 45 MB/s | 47 MB 00:01 2023-09-23T20:02:10.329 INFO:teuthology.orchestra.run.smithi018.stdout: Running scriptlet: ceph-selinux-2:17.2.6-1123.g61431f6b.el8.x86_64 22/25 2023-09-23T20:02:10.329 INFO:teuthology.orchestra.run.smithi018.stdout:skipping the directory /sys 2023-09-23T20:02:10.329 INFO:teuthology.orchestra.run.smithi018.stdout:skipping the directory /proc 2023-09-23T20:02:10.330 INFO:teuthology.orchestra.run.smithi018.stdout:skipping the directory /dev 2023-09-23T20:02:10.330 INFO:teuthology.orchestra.run.smithi018.stdout:skipping the directory /run 2023-09-23T20:02:10.330 INFO:teuthology.orchestra.run.smithi018.stdout:skipping the directory /mnt 2023-09-23T20:02:10.330 INFO:teuthology.orchestra.run.smithi018.stdout:skipping the directory /var/tmp 2023-09-23T20:02:10.330 INFO:teuthology.orchestra.run.smithi018.stdout:skipping the directory /home 2023-09-23T20:02:10.330 INFO:teuthology.orchestra.run.smithi018.stdout:skipping the directory /tmp 2023-09-23T20:02:10.331 INFO:teuthology.orchestra.run.smithi018.stdout:skipping the directory /dev 2023-09-23T20:02:10.331 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:02:12.087 INFO:teuthology.orchestra.run.smithi018.stdout: Installing : ceph-radosgw-2:17.2.6-1123.g61431f6b.el8.x86_64 23/25 2023-09-23T20:02:12.165 INFO:teuthology.orchestra.run.smithi018.stdout: Running scriptlet: ceph-radosgw-2:17.2.6-1123.g61431f6b.el8.x86_64 23/25 2023-09-23T20:02:12.165 INFO:teuthology.orchestra.run.smithi018.stdout: Cleanup : librbd1-1:12.2.7-9.el8.x86_64 24/25 2023-09-23T20:02:12.346 INFO:teuthology.orchestra.run.smithi018.stdout: Running scriptlet: librbd1-1:12.2.7-9.el8.x86_64 24/25 2023-09-23T20:02:12.346 INFO:teuthology.orchestra.run.smithi018.stdout: Cleanup : librados2-1:12.2.7-9.el8.x86_64 25/25 2023-09-23T20:02:12.939 INFO:teuthology.orchestra.run.smithi018.stdout: Running scriptlet: librados2-1:12.2.7-9.el8.x86_64 25/25 2023-09-23T20:02:12.939 INFO:teuthology.orchestra.run.smithi018.stdout: Verifying : ceph-base-2:17.2.6-1123.g61431f6b.el8.x86_64 1/25 2023-09-23T20:02:12.939 INFO:teuthology.orchestra.run.smithi018.stdout: Verifying : ceph-common-2:17.2.6-1123.g61431f6b.el8.x86_64 2/25 2023-09-23T20:02:12.939 INFO:teuthology.orchestra.run.smithi018.stdout: Verifying : ceph-radosgw-2:17.2.6-1123.g61431f6b.el8.x86_64 3/25 2023-09-23T20:02:12.939 INFO:teuthology.orchestra.run.smithi018.stdout: Verifying : ceph-selinux-2:17.2.6-1123.g61431f6b.el8.x86_64 4/25 2023-09-23T20:02:12.939 INFO:teuthology.orchestra.run.smithi018.stdout: Verifying : libcephfs2-2:17.2.6-1123.g61431f6b.el8.x86_64 5/25 2023-09-23T20:02:12.939 INFO:teuthology.orchestra.run.smithi018.stdout: Verifying : libradosstriper1-2:17.2.6-1123.g61431f6b.el8.x86_6 6/25 2023-09-23T20:02:12.939 INFO:teuthology.orchestra.run.smithi018.stdout: Verifying : librgw2-2:17.2.6-1123.g61431f6b.el8.x86_64 7/25 2023-09-23T20:02:12.939 INFO:teuthology.orchestra.run.smithi018.stdout: Verifying : python3-ceph-argparse-2:17.2.6-1123.g61431f6b.el8. 8/25 2023-09-23T20:02:12.940 INFO:teuthology.orchestra.run.smithi018.stdout: Verifying : python3-ceph-common-2:17.2.6-1123.g61431f6b.el8.x8 9/25 2023-09-23T20:02:12.940 INFO:teuthology.orchestra.run.smithi018.stdout: Verifying : python3-cephfs-2:17.2.6-1123.g61431f6b.el8.x86_64 10/25 2023-09-23T20:02:12.940 INFO:teuthology.orchestra.run.smithi018.stdout: Verifying : python3-rados-2:17.2.6-1123.g61431f6b.el8.x86_64 11/25 2023-09-23T20:02:12.940 INFO:teuthology.orchestra.run.smithi018.stdout: Verifying : python3-rbd-2:17.2.6-1123.g61431f6b.el8.x86_64 12/25 2023-09-23T20:02:12.940 INFO:teuthology.orchestra.run.smithi018.stdout: Verifying : python3-rgw-2:17.2.6-1123.g61431f6b.el8.x86_64 13/25 2023-09-23T20:02:12.940 INFO:teuthology.orchestra.run.smithi018.stdout: Verifying : gperftools-libs-1:2.7-9.el8.x86_64 14/25 2023-09-23T20:02:12.940 INFO:teuthology.orchestra.run.smithi018.stdout: Verifying : liboath-2.6.2-3.el8.x86_64 15/25 2023-09-23T20:02:12.940 INFO:teuthology.orchestra.run.smithi018.stdout: Verifying : libunwind-1.3.1-3.el8.x86_64 16/25 2023-09-23T20:02:12.940 INFO:teuthology.orchestra.run.smithi018.stdout: Verifying : thrift-0.13.0-2.el8.x86_64 17/25 2023-09-23T20:02:12.940 INFO:teuthology.orchestra.run.smithi018.stdout: Verifying : librabbitmq-0.9.0-3.el8.x86_64 18/25 2023-09-23T20:02:12.940 INFO:teuthology.orchestra.run.smithi018.stdout: Verifying : lttng-ust-2.8.1-11.el8.x86_64 19/25 2023-09-23T20:02:12.940 INFO:teuthology.orchestra.run.smithi018.stdout: Verifying : python3-prettytable-0.7.2-14.el8.noarch 20/25 2023-09-23T20:02:12.940 INFO:teuthology.orchestra.run.smithi018.stdout: Verifying : librdkafka-0.11.4-3.el8.x86_64 21/25 2023-09-23T20:02:12.941 INFO:teuthology.orchestra.run.smithi018.stdout: Verifying : librados2-2:17.2.6-1123.g61431f6b.el8.x86_64 22/25 2023-09-23T20:02:12.941 INFO:teuthology.orchestra.run.smithi018.stdout: Verifying : librados2-1:12.2.7-9.el8.x86_64 23/25 2023-09-23T20:02:12.941 INFO:teuthology.orchestra.run.smithi018.stdout: Verifying : librbd1-2:17.2.6-1123.g61431f6b.el8.x86_64 24/25 2023-09-23T20:02:14.263 INFO:teuthology.orchestra.run.smithi018.stdout: Verifying : librbd1-1:12.2.7-9.el8.x86_64 25/25 2023-09-23T20:02:14.263 INFO:teuthology.orchestra.run.smithi018.stdout:Last metadata expiration check: 0:00:52 ago on Sat 23 Sep 2023 08:01:22 PM UTC. 2023-09-23T20:02:14.747 INFO:teuthology.orchestra.run.smithi149.stdout:Dependencies resolved. 2023-09-23T20:02:14.749 INFO:teuthology.orchestra.run.smithi149.stdout:==================================================================================================== 2023-09-23T20:02:14.749 INFO:teuthology.orchestra.run.smithi149.stdout: Package Arch Version Repository Size 2023-09-23T20:02:14.750 INFO:teuthology.orchestra.run.smithi149.stdout:==================================================================================================== 2023-09-23T20:02:14.750 INFO:teuthology.orchestra.run.smithi149.stdout:Installing: 2023-09-23T20:02:14.750 INFO:teuthology.orchestra.run.smithi149.stdout: ceph-radosgw x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 17 M 2023-09-23T20:02:14.750 INFO:teuthology.orchestra.run.smithi149.stdout:Upgrading: 2023-09-23T20:02:14.750 INFO:teuthology.orchestra.run.smithi149.stdout: librados2 x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 3.5 M 2023-09-23T20:02:14.750 INFO:teuthology.orchestra.run.smithi149.stdout: librbd1 x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 3.6 M 2023-09-23T20:02:14.750 INFO:teuthology.orchestra.run.smithi149.stdout:Installing dependencies: 2023-09-23T20:02:14.750 INFO:teuthology.orchestra.run.smithi149.stdout: ceph-base x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 5.2 M 2023-09-23T20:02:14.750 INFO:teuthology.orchestra.run.smithi149.stdout: ceph-common x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 23 M 2023-09-23T20:02:14.751 INFO:teuthology.orchestra.run.smithi149.stdout: ceph-selinux x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 24 k 2023-09-23T20:02:14.751 INFO:teuthology.orchestra.run.smithi149.stdout: gperftools-libs x86_64 1:2.7-9.el8 epel 306 k 2023-09-23T20:02:14.751 INFO:teuthology.orchestra.run.smithi149.stdout: libcephfs2 x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 796 k 2023-09-23T20:02:14.751 INFO:teuthology.orchestra.run.smithi149.stdout: liboath x86_64 2.6.2-3.el8 epel 59 k 2023-09-23T20:02:14.751 INFO:teuthology.orchestra.run.smithi149.stdout: librabbitmq x86_64 0.9.0-3.el8 rhel-8-for-x86_64-baseos-rpms 47 k 2023-09-23T20:02:14.751 INFO:teuthology.orchestra.run.smithi149.stdout: libradosstriper1 x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 506 k 2023-09-23T20:02:14.751 INFO:teuthology.orchestra.run.smithi149.stdout: librdkafka x86_64 0.11.4-3.el8 rhel-8-for-x86_64-appstream-rpms 354 k 2023-09-23T20:02:14.751 INFO:teuthology.orchestra.run.smithi149.stdout: librgw2 x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 5.5 M 2023-09-23T20:02:14.751 INFO:teuthology.orchestra.run.smithi149.stdout: libunwind x86_64 1.3.1-3.el8 epel 75 k 2023-09-23T20:02:14.751 INFO:teuthology.orchestra.run.smithi149.stdout: lttng-ust x86_64 2.8.1-11.el8 rhel-8-for-x86_64-appstream-rpms 259 k 2023-09-23T20:02:14.751 INFO:teuthology.orchestra.run.smithi149.stdout: python3-ceph-argparse x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 45 k 2023-09-23T20:02:14.751 INFO:teuthology.orchestra.run.smithi149.stdout: python3-ceph-common x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 115 k 2023-09-23T20:02:14.752 INFO:teuthology.orchestra.run.smithi149.stdout: python3-cephfs x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 190 k 2023-09-23T20:02:14.752 INFO:teuthology.orchestra.run.smithi149.stdout: python3-prettytable noarch 0.7.2-14.el8 rhel-8-for-x86_64-appstream-rpms 44 k 2023-09-23T20:02:14.752 INFO:teuthology.orchestra.run.smithi149.stdout: python3-rados x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 371 k 2023-09-23T20:02:14.752 INFO:teuthology.orchestra.run.smithi149.stdout: python3-rbd x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 358 k 2023-09-23T20:02:14.752 INFO:teuthology.orchestra.run.smithi149.stdout: python3-rgw x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 106 k 2023-09-23T20:02:14.752 INFO:teuthology.orchestra.run.smithi149.stdout: thrift x86_64 0.13.0-2.el8 epel 1.7 M 2023-09-23T20:02:14.752 INFO:teuthology.orchestra.run.smithi149.stdout: 2023-09-23T20:02:14.752 INFO:teuthology.orchestra.run.smithi149.stdout:Transaction Summary 2023-09-23T20:02:14.752 INFO:teuthology.orchestra.run.smithi149.stdout:==================================================================================================== 2023-09-23T20:02:14.752 INFO:teuthology.orchestra.run.smithi149.stdout:Install 21 Packages 2023-09-23T20:02:14.752 INFO:teuthology.orchestra.run.smithi149.stdout:Upgrade 2 Packages 2023-09-23T20:02:14.752 INFO:teuthology.orchestra.run.smithi149.stdout: 2023-09-23T20:02:14.753 INFO:teuthology.orchestra.run.smithi149.stdout:Total download size: 63 M 2023-09-23T20:02:14.753 INFO:teuthology.orchestra.run.smithi149.stdout:Downloading Packages: 2023-09-23T20:02:15.306 INFO:teuthology.orchestra.run.smithi149.stdout:(1/23): ceph-base-17.2.6-1123.g61431f6b.el8.x86 11 MB/s | 5.2 MB 00:00 2023-09-23T20:02:15.339 INFO:teuthology.orchestra.run.smithi149.stdout:(2/23): ceph-selinux-17.2.6-1123.g61431f6b.el8. 731 kB/s | 24 kB 00:00 2023-09-23T20:02:15.423 INFO:teuthology.orchestra.run.smithi149.stdout:(3/23): libcephfs2-17.2.6-1123.g61431f6b.el8.x8 9.3 MB/s | 796 kB 00:00 2023-09-23T20:02:15.473 INFO:teuthology.orchestra.run.smithi149.stdout:(4/23): libradosstriper1-17.2.6-1123.g61431f6b. 9.9 MB/s | 506 kB 00:00 2023-09-23T20:02:15.816 INFO:teuthology.orchestra.run.smithi149.stdout:(5/23): librgw2-17.2.6-1123.g61431f6b.el8.x86_6 16 MB/s | 5.5 MB 00:00 2023-09-23T20:02:15.868 INFO:teuthology.orchestra.run.smithi149.stdout:(6/23): python3-ceph-argparse-17.2.6-1123.g6143 897 kB/s | 45 kB 00:00 2023-09-23T20:02:15.925 INFO:teuthology.orchestra.run.smithi149.stdout:(7/23): python3-ceph-common-17.2.6-1123.g61431f 1.9 MB/s | 115 kB 00:00 2023-09-23T20:02:15.991 INFO:teuthology.orchestra.run.smithi149.stdout:(8/23): python3-cephfs-17.2.6-1123.g61431f6b.el 2.8 MB/s | 190 kB 00:00 2023-09-23T20:02:16.050 INFO:teuthology.orchestra.run.smithi149.stdout:(9/23): python3-rados-17.2.6-1123.g61431f6b.el8 6.2 MB/s | 371 kB 00:00 2023-09-23T20:02:16.117 INFO:teuthology.orchestra.run.smithi149.stdout:(10/23): python3-rbd-17.2.6-1123.g61431f6b.el8. 5.2 MB/s | 358 kB 00:00 2023-09-23T20:02:16.168 INFO:teuthology.orchestra.run.smithi149.stdout:(11/23): python3-rgw-17.2.6-1123.g61431f6b.el8. 2.1 MB/s | 106 kB 00:00 2023-09-23T20:02:16.767 INFO:teuthology.orchestra.run.smithi149.stdout:(12/23): gperftools-libs-2.7-9.el8.x86_64.rpm 510 kB/s | 306 kB 00:00 2023-09-23T20:02:16.782 INFO:teuthology.orchestra.run.smithi018.stdout:Installed products updated. 2023-09-23T20:02:16.894 INFO:teuthology.orchestra.run.smithi149.stdout:(13/23): liboath-2.6.2-3.el8.x86_64.rpm 471 kB/s | 59 kB 00:00 2023-09-23T20:02:16.989 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:02:16.990 INFO:teuthology.orchestra.run.smithi018.stdout:Upgraded: 2023-09-23T20:02:16.990 INFO:teuthology.orchestra.run.smithi018.stdout: librados2-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:02:16.990 INFO:teuthology.orchestra.run.smithi018.stdout: librbd1-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:02:16.990 INFO:teuthology.orchestra.run.smithi018.stdout:Installed: 2023-09-23T20:02:16.990 INFO:teuthology.orchestra.run.smithi018.stdout: ceph-base-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:02:16.990 INFO:teuthology.orchestra.run.smithi018.stdout: ceph-common-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:02:16.990 INFO:teuthology.orchestra.run.smithi018.stdout: ceph-radosgw-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:02:16.990 INFO:teuthology.orchestra.run.smithi018.stdout: ceph-selinux-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:02:16.990 INFO:teuthology.orchestra.run.smithi018.stdout: gperftools-libs-1:2.7-9.el8.x86_64 2023-09-23T20:02:16.990 INFO:teuthology.orchestra.run.smithi018.stdout: libcephfs2-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:02:16.991 INFO:teuthology.orchestra.run.smithi018.stdout: liboath-2.6.2-3.el8.x86_64 2023-09-23T20:02:16.991 INFO:teuthology.orchestra.run.smithi018.stdout: librabbitmq-0.9.0-3.el8.x86_64 2023-09-23T20:02:16.991 INFO:teuthology.orchestra.run.smithi018.stdout: libradosstriper1-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:02:16.991 INFO:teuthology.orchestra.run.smithi018.stdout: librdkafka-0.11.4-3.el8.x86_64 2023-09-23T20:02:16.991 INFO:teuthology.orchestra.run.smithi018.stdout: librgw2-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:02:16.991 INFO:teuthology.orchestra.run.smithi018.stdout: libunwind-1.3.1-3.el8.x86_64 2023-09-23T20:02:16.991 INFO:teuthology.orchestra.run.smithi018.stdout: lttng-ust-2.8.1-11.el8.x86_64 2023-09-23T20:02:16.991 INFO:teuthology.orchestra.run.smithi018.stdout: python3-ceph-argparse-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:02:16.991 INFO:teuthology.orchestra.run.smithi018.stdout: python3-ceph-common-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:02:16.991 INFO:teuthology.orchestra.run.smithi018.stdout: python3-cephfs-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:02:16.991 INFO:teuthology.orchestra.run.smithi018.stdout: python3-prettytable-0.7.2-14.el8.noarch 2023-09-23T20:02:16.991 INFO:teuthology.orchestra.run.smithi018.stdout: python3-rados-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:02:16.992 INFO:teuthology.orchestra.run.smithi018.stdout: python3-rbd-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:02:16.992 INFO:teuthology.orchestra.run.smithi018.stdout: python3-rgw-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:02:16.992 INFO:teuthology.orchestra.run.smithi018.stdout: thrift-0.13.0-2.el8.x86_64 2023-09-23T20:02:16.992 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:02:16.992 INFO:teuthology.orchestra.run.smithi018.stdout:Complete! 2023-09-23T20:02:17.060 INFO:teuthology.orchestra.run.smithi149.stdout:(14/23): ceph-radosgw-17.2.6-1123.g61431f6b.el8 7.5 MB/s | 17 MB 00:02 2023-09-23T20:02:17.093 INFO:teuthology.orchestra.run.smithi149.stdout:(15/23): libunwind-1.3.1-3.el8.x86_64.rpm 374 kB/s | 75 kB 00:00 2023-09-23T20:02:17.144 INFO:teuthology.orchestra.run.smithi149.stdout:(16/23): librabbitmq-0.9.0-3.el8.x86_64.rpm 939 kB/s | 47 kB 00:00 2023-09-23T20:02:17.173 DEBUG:teuthology.orchestra.run.smithi018:> sudo yum -y install ceph-test 2023-09-23T20:02:17.185 INFO:teuthology.orchestra.run.smithi149.stdout:(17/23): lttng-ust-2.8.1-11.el8.x86_64.rpm 6.8 MB/s | 259 kB 00:00 2023-09-23T20:02:17.219 INFO:teuthology.orchestra.run.smithi149.stdout:(18/23): python3-prettytable-0.7.2-14.el8.noarc 1.3 MB/s | 44 kB 00:00 2023-09-23T20:02:17.461 INFO:teuthology.orchestra.run.smithi149.stdout:(19/23): ceph-common-17.2.6-1123.g61431f6b.el8. 8.9 MB/s | 23 MB 00:02 2023-09-23T20:02:17.495 INFO:teuthology.orchestra.run.smithi149.stdout:(20/23): librdkafka-0.11.4-3.el8.x86_64.rpm 1.3 MB/s | 354 kB 00:00 2023-09-23T20:02:17.542 INFO:teuthology.orchestra.run.smithi018.stdout:Updating Subscription Management repositories. 2023-09-23T20:02:17.820 INFO:teuthology.orchestra.run.smithi149.stdout:(21/23): librados2-17.2.6-1123.g61431f6b.el8.x8 9.7 MB/s | 3.5 MB 00:00 2023-09-23T20:02:17.870 INFO:teuthology.orchestra.run.smithi149.stdout:(22/23): librbd1-17.2.6-1123.g61431f6b.el8.x86_ 9.6 MB/s | 3.6 MB 00:00 2023-09-23T20:02:17.912 INFO:teuthology.orchestra.run.smithi149.stdout:(23/23): thrift-0.13.0-2.el8.x86_64.rpm 2.0 MB/s | 1.7 MB 00:00 2023-09-23T20:02:17.915 INFO:teuthology.orchestra.run.smithi149.stdout:-------------------------------------------------------------------------------- 2023-09-23T20:02:17.915 INFO:teuthology.orchestra.run.smithi149.stdout:Total 20 MB/s | 63 MB 00:03 2023-09-23T20:02:17.957 INFO:teuthology.orchestra.run.smithi149.stdout:Running transaction check 2023-09-23T20:02:18.000 INFO:teuthology.orchestra.run.smithi149.stdout:Transaction check succeeded. 2023-09-23T20:02:18.001 INFO:teuthology.orchestra.run.smithi149.stdout:Running transaction test 2023-09-23T20:02:18.345 INFO:teuthology.orchestra.run.smithi149.stdout:Transaction test succeeded. 2023-09-23T20:02:18.349 INFO:teuthology.orchestra.run.smithi149.stdout:Running transaction 2023-09-23T20:02:18.369 INFO:teuthology.orchestra.run.smithi018.stdout:Last metadata expiration check: 0:00:56 ago on Sat 23 Sep 2023 08:01:22 PM UTC. 2023-09-23T20:02:19.402 INFO:teuthology.orchestra.run.smithi149.stdout: Preparing : 1/1 2023-09-23T20:02:19.594 INFO:teuthology.orchestra.run.smithi149.stdout: Installing : lttng-ust-2.8.1-11.el8.x86_64 1/25 2023-09-23T20:02:19.921 INFO:teuthology.orchestra.run.smithi018.stdout:Dependencies resolved. 2023-09-23T20:02:19.922 INFO:teuthology.orchestra.run.smithi018.stdout:============================================================================================ 2023-09-23T20:02:19.922 INFO:teuthology.orchestra.run.smithi018.stdout: Package Arch Version Repository Size 2023-09-23T20:02:19.922 INFO:teuthology.orchestra.run.smithi018.stdout:============================================================================================ 2023-09-23T20:02:19.922 INFO:teuthology.orchestra.run.smithi018.stdout:Installing: 2023-09-23T20:02:19.922 INFO:teuthology.orchestra.run.smithi018.stdout: ceph-test x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 52 M 2023-09-23T20:02:19.922 INFO:teuthology.orchestra.run.smithi018.stdout:Installing dependencies: 2023-09-23T20:02:19.922 INFO:teuthology.orchestra.run.smithi018.stdout: jq x86_64 1.6-3.el8 rhel-8-for-x86_64-appstream-rpms 202 k 2023-09-23T20:02:19.922 INFO:teuthology.orchestra.run.smithi018.stdout: libcephsqlite x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 174 k 2023-09-23T20:02:19.923 INFO:teuthology.orchestra.run.smithi018.stdout: oniguruma x86_64 6.8.2-2.el8 rhel-8-for-x86_64-appstream-rpms 187 k 2023-09-23T20:02:19.923 INFO:teuthology.orchestra.run.smithi018.stdout: socat x86_64 1.7.4.1-1.el8 rhel-8-for-x86_64-appstream-rpms 323 k 2023-09-23T20:02:19.923 INFO:teuthology.orchestra.run.smithi018.stdout: xmlstarlet x86_64 1.6.1-20.el8 lab-extras 70 k 2023-09-23T20:02:19.923 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:02:19.923 INFO:teuthology.orchestra.run.smithi018.stdout:Transaction Summary 2023-09-23T20:02:19.923 INFO:teuthology.orchestra.run.smithi018.stdout:============================================================================================ 2023-09-23T20:02:19.923 INFO:teuthology.orchestra.run.smithi018.stdout:Install 6 Packages 2023-09-23T20:02:19.923 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:02:19.924 INFO:teuthology.orchestra.run.smithi018.stdout:Total download size: 53 M 2023-09-23T20:02:19.924 INFO:teuthology.orchestra.run.smithi018.stdout:Installed size: 233 M 2023-09-23T20:02:19.924 INFO:teuthology.orchestra.run.smithi018.stdout:Downloading Packages: 2023-09-23T20:02:19.968 INFO:teuthology.orchestra.run.smithi149.stdout: Running scriptlet: lttng-ust-2.8.1-11.el8.x86_64 1/25 2023-09-23T20:02:19.982 INFO:teuthology.orchestra.run.smithi018.stdout:(1/6): xmlstarlet-1.6.1-20.el8.x86_64.rpm 1.2 MB/s | 70 kB 00:00 2023-09-23T20:02:20.049 INFO:teuthology.orchestra.run.smithi018.stdout:(2/6): socat-1.7.4.1-1.el8.x86_64.rpm 5.5 MB/s | 323 kB 00:00 2023-09-23T20:02:20.091 INFO:teuthology.orchestra.run.smithi018.stdout:(3/6): oniguruma-6.8.2-2.el8.x86_64.rpm 4.5 MB/s | 187 kB 00:00 2023-09-23T20:02:20.133 INFO:teuthology.orchestra.run.smithi018.stdout:(4/6): jq-1.6-3.el8.x86_64.rpm 4.8 MB/s | 202 kB 00:00 2023-09-23T20:02:20.194 INFO:teuthology.orchestra.run.smithi149.stdout: Upgrading : librados2-2:17.2.6-1123.g61431f6b.el8.x86_64 2/25 2023-09-23T20:02:20.278 INFO:teuthology.orchestra.run.smithi149.stdout: Running scriptlet: librados2-2:17.2.6-1123.g61431f6b.el8.x86_64 2/25 2023-09-23T20:02:20.283 INFO:teuthology.orchestra.run.smithi018.stdout:(5/6): libcephsqlite-17.2.6-1123.g61431f6b.el8. 486 kB/s | 174 kB 00:00 2023-09-23T20:02:20.482 INFO:teuthology.orchestra.run.smithi149.stdout: Installing : python3-rados-2:17.2.6-1123.g61431f6b.el8.x86_64 3/25 2023-09-23T20:02:20.717 INFO:teuthology.orchestra.run.smithi149.stdout: Installing : liboath-2.6.2-3.el8.x86_64 4/25 2023-09-23T20:02:20.926 INFO:teuthology.orchestra.run.smithi149.stdout: Installing : libcephfs2-2:17.2.6-1123.g61431f6b.el8.x86_64 5/25 2023-09-23T20:02:21.331 INFO:teuthology.orchestra.run.smithi149.stdout: Running scriptlet: libcephfs2-2:17.2.6-1123.g61431f6b.el8.x86_64 5/25 2023-09-23T20:02:21.551 INFO:teuthology.orchestra.run.smithi149.stdout: Upgrading : librbd1-2:17.2.6-1123.g61431f6b.el8.x86_64 6/25 2023-09-23T20:02:21.649 INFO:teuthology.orchestra.run.smithi149.stdout: Running scriptlet: librbd1-2:17.2.6-1123.g61431f6b.el8.x86_64 6/25 2023-09-23T20:02:21.834 INFO:teuthology.orchestra.run.smithi149.stdout: Installing : librdkafka-0.11.4-3.el8.x86_64 7/25 2023-09-23T20:02:21.894 INFO:teuthology.orchestra.run.smithi149.stdout: Running scriptlet: librdkafka-0.11.4-3.el8.x86_64 7/25 2023-09-23T20:02:22.211 INFO:teuthology.orchestra.run.smithi149.stdout: Installing : librabbitmq-0.9.0-3.el8.x86_64 8/25 2023-09-23T20:02:22.417 INFO:teuthology.orchestra.run.smithi018.stdout:(6/6): ceph-test-17.2.6-1123.g61431f6b.el8.x86_ 21 MB/s | 52 MB 00:02 2023-09-23T20:02:22.419 INFO:teuthology.orchestra.run.smithi018.stdout:-------------------------------------------------------------------------------- 2023-09-23T20:02:22.419 INFO:teuthology.orchestra.run.smithi018.stdout:Total 21 MB/s | 53 MB 00:02 2023-09-23T20:02:22.459 INFO:teuthology.orchestra.run.smithi018.stdout:Running transaction check 2023-09-23T20:02:22.481 INFO:teuthology.orchestra.run.smithi018.stdout:Transaction check succeeded. 2023-09-23T20:02:22.482 INFO:teuthology.orchestra.run.smithi018.stdout:Running transaction test 2023-09-23T20:02:22.773 INFO:teuthology.orchestra.run.smithi018.stdout:Transaction test succeeded. 2023-09-23T20:02:22.777 INFO:teuthology.orchestra.run.smithi018.stdout:Running transaction 2023-09-23T20:02:22.893 INFO:teuthology.orchestra.run.smithi149.stdout: Installing : thrift-0.13.0-2.el8.x86_64 9/25 2023-09-23T20:02:23.059 INFO:teuthology.orchestra.run.smithi149.stdout: Installing : librgw2-2:17.2.6-1123.g61431f6b.el8.x86_64 10/25 2023-09-23T20:02:23.123 INFO:teuthology.orchestra.run.smithi149.stdout: Running scriptlet: librgw2-2:17.2.6-1123.g61431f6b.el8.x86_64 10/25 2023-09-23T20:02:23.246 INFO:teuthology.orchestra.run.smithi018.stdout: Preparing : 1/1 2023-09-23T20:02:23.275 INFO:teuthology.orchestra.run.smithi149.stdout: Installing : python3-ceph-argparse-2:17.2.6-1123.g61431f6b.el8. 11/25 2023-09-23T20:02:23.358 INFO:teuthology.orchestra.run.smithi018.stdout: Installing : oniguruma-6.8.2-2.el8.x86_64 1/6 2023-09-23T20:02:23.429 INFO:teuthology.orchestra.run.smithi018.stdout: Running scriptlet: oniguruma-6.8.2-2.el8.x86_64 1/6 2023-09-23T20:02:23.436 INFO:teuthology.orchestra.run.smithi149.stdout: Installing : python3-cephfs-2:17.2.6-1123.g61431f6b.el8.x86_64 12/25 2023-09-23T20:02:23.589 INFO:teuthology.orchestra.run.smithi018.stdout: Installing : jq-1.6-3.el8.x86_64 2/6 2023-09-23T20:02:23.694 INFO:teuthology.orchestra.run.smithi149.stdout: Installing : python3-rgw-2:17.2.6-1123.g61431f6b.el8.x86_64 13/25 2023-09-23T20:02:23.751 INFO:teuthology.orchestra.run.smithi018.stdout: Installing : socat-1.7.4.1-1.el8.x86_64 3/6 2023-09-23T20:02:23.875 INFO:teuthology.orchestra.run.smithi018.stdout: Installing : xmlstarlet-1.6.1-20.el8.x86_64 4/6 2023-09-23T20:02:23.879 INFO:teuthology.orchestra.run.smithi149.stdout: Installing : python3-rbd-2:17.2.6-1123.g61431f6b.el8.x86_64 14/25 2023-09-23T20:02:24.034 INFO:teuthology.orchestra.run.smithi149.stdout: Installing : libradosstriper1-2:17.2.6-1123.g61431f6b.el8.x86_6 15/25 2023-09-23T20:02:24.058 INFO:teuthology.orchestra.run.smithi018.stdout: Installing : libcephsqlite-2:17.2.6-1123.g61431f6b.el8.x86_64 5/6 2023-09-23T20:02:24.103 INFO:teuthology.orchestra.run.smithi149.stdout: Running scriptlet: libradosstriper1-2:17.2.6-1123.g61431f6b.el8.x86_6 15/25 2023-09-23T20:02:24.226 INFO:teuthology.orchestra.run.smithi149.stdout: Installing : python3-prettytable-0.7.2-14.el8.noarch 16/25 2023-09-23T20:02:24.399 INFO:teuthology.orchestra.run.smithi149.stdout: Installing : libunwind-1.3.1-3.el8.x86_64 17/25 2023-09-23T20:02:24.904 INFO:teuthology.orchestra.run.smithi149.stdout: Installing : gperftools-libs-1:2.7-9.el8.x86_64 18/25 2023-09-23T20:02:25.068 INFO:teuthology.orchestra.run.smithi149.stdout: Installing : python3-ceph-common-2:17.2.6-1123.g61431f6b.el8.x8 19/25 2023-09-23T20:02:28.879 INFO:teuthology.orchestra.run.smithi149.stdout: Running scriptlet: ceph-common-2:17.2.6-1123.g61431f6b.el8.x86_64 20/25 2023-09-23T20:02:29.117 INFO:teuthology.orchestra.run.smithi149.stdout: Installing : ceph-common-2:17.2.6-1123.g61431f6b.el8.x86_64 20/25 2023-09-23T20:02:29.326 INFO:teuthology.orchestra.run.smithi018.stdout: Running scriptlet: libcephsqlite-2:17.2.6-1123.g61431f6b.el8.x86_64 5/6 2023-09-23T20:02:29.529 INFO:teuthology.orchestra.run.smithi018.stdout: Installing : ceph-test-2:17.2.6-1123.g61431f6b.el8.x86_64 6/6 2023-09-23T20:02:29.686 INFO:teuthology.orchestra.run.smithi149.stdout: Running scriptlet: ceph-common-2:17.2.6-1123.g61431f6b.el8.x86_64 20/25 2023-09-23T20:02:29.881 INFO:teuthology.orchestra.run.smithi018.stdout: Running scriptlet: ceph-test-2:17.2.6-1123.g61431f6b.el8.x86_64 6/6 2023-09-23T20:02:29.881 INFO:teuthology.orchestra.run.smithi018.stdout: Verifying : ceph-test-2:17.2.6-1123.g61431f6b.el8.x86_64 1/6 2023-09-23T20:02:29.881 INFO:teuthology.orchestra.run.smithi018.stdout: Verifying : libcephsqlite-2:17.2.6-1123.g61431f6b.el8.x86_64 2/6 2023-09-23T20:02:29.881 INFO:teuthology.orchestra.run.smithi018.stdout: Verifying : xmlstarlet-1.6.1-20.el8.x86_64 3/6 2023-09-23T20:02:29.881 INFO:teuthology.orchestra.run.smithi018.stdout: Verifying : socat-1.7.4.1-1.el8.x86_64 4/6 2023-09-23T20:02:29.881 INFO:teuthology.orchestra.run.smithi018.stdout: Verifying : oniguruma-6.8.2-2.el8.x86_64 5/6 2023-09-23T20:02:29.926 INFO:teuthology.orchestra.run.smithi149.stdout: Installing : ceph-base-2:17.2.6-1123.g61431f6b.el8.x86_64 21/25 2023-09-23T20:02:30.004 INFO:teuthology.orchestra.run.smithi149.stdout: Running scriptlet: ceph-base-2:17.2.6-1123.g61431f6b.el8.x86_64 21/25 2023-09-23T20:02:30.158 INFO:teuthology.orchestra.run.smithi149.stdout: Installing : ceph-selinux-2:17.2.6-1123.g61431f6b.el8.x86_64 22/25 2023-09-23T20:02:31.002 INFO:teuthology.orchestra.run.smithi018.stdout: Verifying : jq-1.6-3.el8.x86_64 6/6 2023-09-23T20:02:31.002 INFO:teuthology.orchestra.run.smithi018.stdout:Last metadata expiration check: 0:01:09 ago on Sat 23 Sep 2023 08:01:22 PM UTC. 2023-09-23T20:02:33.575 INFO:teuthology.orchestra.run.smithi018.stdout:Installed products updated. 2023-09-23T20:02:33.778 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:02:33.778 INFO:teuthology.orchestra.run.smithi018.stdout:Installed: 2023-09-23T20:02:33.778 INFO:teuthology.orchestra.run.smithi018.stdout: ceph-test-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:02:33.779 INFO:teuthology.orchestra.run.smithi018.stdout: jq-1.6-3.el8.x86_64 2023-09-23T20:02:33.779 INFO:teuthology.orchestra.run.smithi018.stdout: libcephsqlite-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:02:33.779 INFO:teuthology.orchestra.run.smithi018.stdout: oniguruma-6.8.2-2.el8.x86_64 2023-09-23T20:02:33.779 INFO:teuthology.orchestra.run.smithi018.stdout: socat-1.7.4.1-1.el8.x86_64 2023-09-23T20:02:33.779 INFO:teuthology.orchestra.run.smithi018.stdout: xmlstarlet-1.6.1-20.el8.x86_64 2023-09-23T20:02:33.779 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:02:33.779 INFO:teuthology.orchestra.run.smithi018.stdout:Complete! 2023-09-23T20:02:33.911 DEBUG:teuthology.orchestra.run.smithi018:> sudo yum -y install ceph 2023-09-23T20:02:34.266 INFO:teuthology.orchestra.run.smithi018.stdout:Updating Subscription Management repositories. 2023-09-23T20:02:35.112 INFO:teuthology.orchestra.run.smithi018.stdout:Last metadata expiration check: 0:01:13 ago on Sat 23 Sep 2023 08:01:22 PM UTC. 2023-09-23T20:02:36.676 INFO:teuthology.orchestra.run.smithi018.stdout:Dependencies resolved. 2023-09-23T20:02:36.680 INFO:teuthology.orchestra.run.smithi018.stdout:======================================================================================================= 2023-09-23T20:02:36.680 INFO:teuthology.orchestra.run.smithi018.stdout: Package Arch Version Repository Size 2023-09-23T20:02:36.680 INFO:teuthology.orchestra.run.smithi018.stdout:======================================================================================================= 2023-09-23T20:02:36.680 INFO:teuthology.orchestra.run.smithi018.stdout:Installing: 2023-09-23T20:02:36.680 INFO:teuthology.orchestra.run.smithi018.stdout: ceph x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 6.4 k 2023-09-23T20:02:36.681 INFO:teuthology.orchestra.run.smithi018.stdout:Installing dependencies: 2023-09-23T20:02:36.681 INFO:teuthology.orchestra.run.smithi018.stdout: ceph-mds x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 2.3 M 2023-09-23T20:02:36.681 INFO:teuthology.orchestra.run.smithi018.stdout: ceph-mgr x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 1.6 M 2023-09-23T20:02:36.681 INFO:teuthology.orchestra.run.smithi018.stdout: ceph-mgr-modules-core noarch 2:17.2.6-1123.g61431f6b.el8 ceph-noarch 238 k 2023-09-23T20:02:36.681 INFO:teuthology.orchestra.run.smithi018.stdout: ceph-mon x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 3.9 M 2023-09-23T20:02:36.681 INFO:teuthology.orchestra.run.smithi018.stdout: ceph-osd x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 17 M 2023-09-23T20:02:36.681 INFO:teuthology.orchestra.run.smithi018.stdout: python3-bcrypt x86_64 3.1.6-2.el8.1 epel 44 k 2023-09-23T20:02:36.681 INFO:teuthology.orchestra.run.smithi018.stdout: python3-beautifulsoup4 noarch 4.6.3-2.el8.1 epel 185 k 2023-09-23T20:02:36.681 INFO:teuthology.orchestra.run.smithi018.stdout: python3-cffi x86_64 1.11.5-5.el8 rhel-8-for-x86_64-baseos-rpms 238 k 2023-09-23T20:02:36.682 INFO:teuthology.orchestra.run.smithi018.stdout: python3-cheroot noarch 8.5.2-1.el8 epel 173 k 2023-09-23T20:02:36.682 INFO:teuthology.orchestra.run.smithi018.stdout: python3-cherrypy noarch 18.4.0-1.el8 epel 384 k 2023-09-23T20:02:36.682 INFO:teuthology.orchestra.run.smithi018.stdout: python3-cryptography x86_64 3.2.1-5.el8 rhel-8-for-x86_64-baseos-rpms 559 k 2023-09-23T20:02:36.682 INFO:teuthology.orchestra.run.smithi018.stdout: python3-jaraco noarch 6.2-6.el8 epel 11 k 2023-09-23T20:02:36.682 INFO:teuthology.orchestra.run.smithi018.stdout: python3-jaraco-functools noarch 2.0-4.el8 epel 18 k 2023-09-23T20:02:36.682 INFO:teuthology.orchestra.run.smithi018.stdout: python3-logutils noarch 0.3.5-11.el8 epel 49 k 2023-09-23T20:02:36.682 INFO:teuthology.orchestra.run.smithi018.stdout: python3-mako noarch 1.0.6-13.el8 rhel-8-for-x86_64-appstream-rpms 157 k 2023-09-23T20:02:36.682 INFO:teuthology.orchestra.run.smithi018.stdout: python3-markupsafe x86_64 0.23-19.el8 rhel-8-for-x86_64-appstream-rpms 39 k 2023-09-23T20:02:36.682 INFO:teuthology.orchestra.run.smithi018.stdout: python3-more-itertools noarch 7.2.0-3.el8 epel 59 k 2023-09-23T20:02:36.682 INFO:teuthology.orchestra.run.smithi018.stdout: python3-pecan noarch 1.3.2-9.el8 epel 283 k 2023-09-23T20:02:36.683 INFO:teuthology.orchestra.run.smithi018.stdout: python3-portend noarch 2.6-1.el8 epel 16 k 2023-09-23T20:02:36.683 INFO:teuthology.orchestra.run.smithi018.stdout: python3-pyOpenSSL noarch 19.0.0-1.el8 rhel-8-for-x86_64-appstream-rpms 103 k 2023-09-23T20:02:36.683 INFO:teuthology.orchestra.run.smithi018.stdout: python3-pycparser noarch 2.14-14.el8 rhel-8-for-x86_64-baseos-rpms 109 k 2023-09-23T20:02:36.683 INFO:teuthology.orchestra.run.smithi018.stdout: python3-pytz noarch 2017.2-9.el8 rhel-8-for-x86_64-appstream-rpms 54 k 2023-09-23T20:02:36.683 INFO:teuthology.orchestra.run.smithi018.stdout: python3-simplegeneric noarch 0.8.1-17.el8 epel 19 k 2023-09-23T20:02:36.683 INFO:teuthology.orchestra.run.smithi018.stdout: python3-singledispatch noarch 3.4.0.3-18.el8 epel 24 k 2023-09-23T20:02:36.683 INFO:teuthology.orchestra.run.smithi018.stdout: python3-tempora noarch 1.14.1-5.el8 epel 29 k 2023-09-23T20:02:36.683 INFO:teuthology.orchestra.run.smithi018.stdout: python3-trustme noarch 0.6.0-4.el8 epel 27 k 2023-09-23T20:02:36.683 INFO:teuthology.orchestra.run.smithi018.stdout: python3-waitress noarch 1.4.3-1.el8 epel 255 k 2023-09-23T20:02:36.683 INFO:teuthology.orchestra.run.smithi018.stdout: python3-webob noarch 1.8.5-1.el8.1 epel 251 k 2023-09-23T20:02:36.684 INFO:teuthology.orchestra.run.smithi018.stdout: python3-webtest noarch 2.0.33-1.el8 epel 85 k 2023-09-23T20:02:36.684 INFO:teuthology.orchestra.run.smithi018.stdout: python3-werkzeug noarch 0.12.2-4.el8 rhel-8-for-x86_64-appstream-rpms 457 k 2023-09-23T20:02:36.684 INFO:teuthology.orchestra.run.smithi018.stdout: python3-zc-lockfile noarch 2.0-2.el8 epel 23 k 2023-09-23T20:02:36.684 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:02:36.684 INFO:teuthology.orchestra.run.smithi018.stdout:Transaction Summary 2023-09-23T20:02:36.684 INFO:teuthology.orchestra.run.smithi018.stdout:======================================================================================================= 2023-09-23T20:02:36.684 INFO:teuthology.orchestra.run.smithi018.stdout:Install 32 Packages 2023-09-23T20:02:36.684 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:02:36.685 INFO:teuthology.orchestra.run.smithi018.stdout:Total download size: 28 M 2023-09-23T20:02:36.685 INFO:teuthology.orchestra.run.smithi018.stdout:Installed size: 110 M 2023-09-23T20:02:36.685 INFO:teuthology.orchestra.run.smithi018.stdout:Downloading Packages: 2023-09-23T20:02:37.041 INFO:teuthology.orchestra.run.smithi018.stdout:(1/32): ceph-17.2.6-1123.g61431f6b.el8.x86_64.r 34 kB/s | 6.4 kB 00:00 2023-09-23T20:02:37.367 INFO:teuthology.orchestra.run.smithi018.stdout:(2/32): ceph-mds-17.2.6-1123.g61431f6b.el8.x86_ 4.5 MB/s | 2.3 MB 00:00 2023-09-23T20:02:37.408 INFO:teuthology.orchestra.run.smithi018.stdout:(3/32): ceph-mgr-17.2.6-1123.g61431f6b.el8.x86_ 2.8 MB/s | 1.6 MB 00:00 2023-09-23T20:02:37.517 INFO:teuthology.orchestra.run.smithi018.stdout:(4/32): ceph-mgr-modules-core-17.2.6-1123.g6143 2.1 MB/s | 238 kB 00:00 2023-09-23T20:02:37.618 INFO:teuthology.orchestra.run.smithi018.stdout:(5/32): ceph-mon-17.2.6-1123.g61431f6b.el8.x86_ 6.8 MB/s | 3.9 MB 00:00 2023-09-23T20:02:37.693 INFO:teuthology.orchestra.run.smithi018.stdout:(6/32): python3-bcrypt-3.1.6-2.el8.1.x86_64.rpm 249 kB/s | 44 kB 00:00 2023-09-23T20:02:37.768 INFO:teuthology.orchestra.run.smithi018.stdout:(7/32): python3-cheroot-8.5.2-1.el8.noarch.rpm 2.3 MB/s | 173 kB 00:00 2023-09-23T20:02:37.794 INFO:teuthology.orchestra.run.smithi018.stdout:(8/32): python3-beautifulsoup4-4.6.3-2.el8.1.no 1.0 MB/s | 185 kB 00:00 2023-09-23T20:02:37.835 INFO:teuthology.orchestra.run.smithi018.stdout:(9/32): python3-jaraco-6.2-6.el8.noarch.rpm 268 kB/s | 11 kB 00:00 2023-09-23T20:02:37.869 INFO:teuthology.orchestra.run.smithi018.stdout:(10/32): python3-jaraco-functools-2.0-4.el8.noa 527 kB/s | 18 kB 00:00 2023-09-23T20:02:37.894 INFO:teuthology.orchestra.run.smithi018.stdout:(11/32): python3-cherrypy-18.4.0-1.el8.noarch.r 3.0 MB/s | 384 kB 00:00 2023-09-23T20:02:37.911 INFO:teuthology.orchestra.run.smithi018.stdout:(12/32): python3-logutils-0.3.5-11.el8.noarch.r 1.1 MB/s | 49 kB 00:00 2023-09-23T20:02:37.928 INFO:teuthology.orchestra.run.smithi018.stdout:(13/32): python3-more-itertools-7.2.0-3.el8.noa 1.7 MB/s | 59 kB 00:00 2023-09-23T20:02:37.970 INFO:teuthology.orchestra.run.smithi018.stdout:(14/32): python3-portend-2.6-1.el8.noarch.rpm 377 kB/s | 16 kB 00:00 2023-09-23T20:02:38.004 INFO:teuthology.orchestra.run.smithi018.stdout:(15/32): python3-pecan-1.3.2-9.el8.noarch.rpm 3.0 MB/s | 283 kB 00:00 2023-09-23T20:02:38.021 INFO:teuthology.orchestra.run.smithi018.stdout:(16/32): python3-simplegeneric-0.8.1-17.el8.noa 372 kB/s | 19 kB 00:00 2023-09-23T20:02:38.038 INFO:teuthology.orchestra.run.smithi018.stdout:(17/32): python3-singledispatch-3.4.0.3-18.el8. 714 kB/s | 24 kB 00:00 2023-09-23T20:02:38.055 INFO:teuthology.orchestra.run.smithi018.stdout:(18/32): python3-tempora-1.14.1-5.el8.noarch.rp 869 kB/s | 29 kB 00:00 2023-09-23T20:02:38.072 INFO:teuthology.orchestra.run.smithi018.stdout:(19/32): python3-trustme-0.6.0-4.el8.noarch.rpm 813 kB/s | 27 kB 00:00 2023-09-23T20:02:38.248 INFO:teuthology.orchestra.run.smithi018.stdout:(20/32): ceph-osd-17.2.6-1123.g61431f6b.el8.x86 19 MB/s | 17 MB 00:00 2023-09-23T20:02:38.273 INFO:teuthology.orchestra.run.smithi018.stdout:(21/32): python3-waitress-1.4.3-1.el8.noarch.rp 1.1 MB/s | 255 kB 00:00 2023-09-23T20:02:38.330 INFO:teuthology.orchestra.run.smithi018.stdout:(22/32): python3-webob-1.8.5-1.el8.1.noarch.rpm 980 kB/s | 251 kB 00:00 2023-09-23T20:02:38.357 INFO:teuthology.orchestra.run.smithi018.stdout:(23/32): python3-zc-lockfile-2.0-2.el8.noarch.r 291 kB/s | 23 kB 00:00 2023-09-23T20:02:38.415 INFO:teuthology.orchestra.run.smithi018.stdout:(24/32): python3-pycparser-2.14-14.el8.noarch.r 1.3 MB/s | 109 kB 00:00 2023-09-23T20:02:38.441 INFO:teuthology.orchestra.run.smithi018.stdout:(25/32): python3-webtest-2.0.33-1.el8.noarch.rp 443 kB/s | 85 kB 00:00 2023-09-23T20:02:38.474 INFO:teuthology.orchestra.run.smithi018.stdout:(26/32): python3-cffi-1.11.5-5.el8.x86_64.rpm 4.0 MB/s | 238 kB 00:00 2023-09-23T20:02:38.516 INFO:teuthology.orchestra.run.smithi018.stdout:(27/32): python3-cryptography-3.2.1-5.el8.x86_6 3.4 MB/s | 559 kB 00:00 2023-09-23T20:02:38.541 INFO:teuthology.orchestra.run.smithi018.stdout:(28/32): python3-markupsafe-0.23-19.el8.x86_64. 662 kB/s | 39 kB 00:00 2023-09-23T20:02:38.567 INFO:teuthology.orchestra.run.smithi018.stdout:(29/32): python3-pytz-2017.2-9.el8.noarch.rpm 430 kB/s | 54 kB 00:00 2023-09-23T20:02:38.592 INFO:teuthology.orchestra.run.smithi018.stdout:(30/32): python3-mako-1.0.6-13.el8.noarch.rpm 2.0 MB/s | 157 kB 00:00 2023-09-23T20:02:38.617 INFO:teuthology.orchestra.run.smithi018.stdout:(31/32): python3-pyOpenSSL-19.0.0-1.el8.noarch. 2.0 MB/s | 103 kB 00:00 2023-09-23T20:02:38.642 INFO:teuthology.orchestra.run.smithi018.stdout:(32/32): python3-werkzeug-0.12.2-4.el8.noarch.r 4.5 MB/s | 457 kB 00:00 2023-09-23T20:02:38.646 INFO:teuthology.orchestra.run.smithi018.stdout:-------------------------------------------------------------------------------- 2023-09-23T20:02:38.646 INFO:teuthology.orchestra.run.smithi018.stdout:Total 14 MB/s | 28 MB 00:01 2023-09-23T20:02:38.743 INFO:teuthology.orchestra.run.smithi018.stdout:Running transaction check 2023-09-23T20:02:38.782 INFO:teuthology.orchestra.run.smithi018.stdout:Transaction check succeeded. 2023-09-23T20:02:38.782 INFO:teuthology.orchestra.run.smithi018.stdout:Running transaction test 2023-09-23T20:02:39.003 INFO:teuthology.orchestra.run.smithi018.stdout:Transaction test succeeded. 2023-09-23T20:02:39.007 INFO:teuthology.orchestra.run.smithi018.stdout:Running transaction 2023-09-23T20:02:40.109 INFO:teuthology.orchestra.run.smithi018.stdout: Preparing : 1/1 2023-09-23T20:02:40.390 INFO:teuthology.orchestra.run.smithi018.stdout: Installing : python3-more-itertools-7.2.0-3.el8.noarch 1/32 2023-09-23T20:02:40.654 INFO:teuthology.orchestra.run.smithi018.stdout: Installing : python3-webob-1.8.5-1.el8.1.noarch 2/32 2023-09-23T20:02:40.860 INFO:teuthology.orchestra.run.smithi018.stdout: Installing : python3-werkzeug-0.12.2-4.el8.noarch 3/32 2023-09-23T20:02:41.070 INFO:teuthology.orchestra.run.smithi018.stdout: Installing : python3-markupsafe-0.23-19.el8.x86_64 4/32 2023-09-23T20:02:41.297 INFO:teuthology.orchestra.run.smithi018.stdout: Installing : python3-mako-1.0.6-13.el8.noarch 5/32 2023-09-23T20:02:41.520 INFO:teuthology.orchestra.run.smithi018.stdout: Installing : python3-pytz-2017.2-9.el8.noarch 6/32 2023-09-23T20:02:41.761 INFO:teuthology.orchestra.run.smithi018.stdout: Installing : python3-pycparser-2.14-14.el8.noarch 7/32 2023-09-23T20:02:42.118 INFO:teuthology.orchestra.run.smithi018.stdout: Installing : python3-cffi-1.11.5-5.el8.x86_64 8/32 2023-09-23T20:02:42.376 INFO:teuthology.orchestra.run.smithi018.stdout: Installing : python3-cryptography-3.2.1-5.el8.x86_64 9/32 2023-09-23T20:02:42.573 INFO:teuthology.orchestra.run.smithi018.stdout: Installing : python3-pyOpenSSL-19.0.0-1.el8.noarch 10/32 2023-09-23T20:02:42.744 INFO:teuthology.orchestra.run.smithi018.stdout: Installing : python3-trustme-0.6.0-4.el8.noarch 11/32 2023-09-23T20:02:42.965 INFO:teuthology.orchestra.run.smithi018.stdout: Installing : python3-bcrypt-3.1.6-2.el8.1.x86_64 12/32 2023-09-23T20:02:43.199 INFO:teuthology.orchestra.run.smithi018.stdout: Installing : python3-zc-lockfile-2.0-2.el8.noarch 13/32 2023-09-23T20:02:43.379 INFO:teuthology.orchestra.run.smithi018.stdout: Installing : python3-waitress-1.4.3-1.el8.noarch 14/32 2023-09-23T20:02:43.543 INFO:teuthology.orchestra.run.smithi018.stdout: Installing : python3-singledispatch-3.4.0.3-18.el8.noarch 15/32 2023-09-23T20:02:43.710 INFO:teuthology.orchestra.run.smithi018.stdout: Installing : python3-simplegeneric-0.8.1-17.el8.noarch 16/32 2023-09-23T20:02:43.866 INFO:teuthology.orchestra.run.smithi018.stdout: Installing : python3-logutils-0.3.5-11.el8.noarch 17/32 2023-09-23T20:02:43.986 INFO:teuthology.orchestra.run.smithi018.stdout: Installing : python3-jaraco-6.2-6.el8.noarch 18/32 2023-09-23T20:02:44.171 INFO:teuthology.orchestra.run.smithi018.stdout: Installing : python3-jaraco-functools-2.0-4.el8.noarch 19/32 2023-09-23T20:02:44.385 INFO:teuthology.orchestra.run.smithi018.stdout: Installing : python3-cheroot-8.5.2-1.el8.noarch 20/32 2023-09-23T20:02:44.577 INFO:teuthology.orchestra.run.smithi018.stdout: Installing : python3-tempora-1.14.1-5.el8.noarch 21/32 2023-09-23T20:02:44.849 INFO:teuthology.orchestra.run.smithi018.stdout: Installing : python3-portend-2.6-1.el8.noarch 22/32 2023-09-23T20:02:45.071 INFO:teuthology.orchestra.run.smithi018.stdout: Installing : python3-cherrypy-18.4.0-1.el8.noarch 23/32 2023-09-23T20:02:45.249 INFO:teuthology.orchestra.run.smithi018.stdout: Installing : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 24/32 2023-09-23T20:02:46.175 INFO:teuthology.orchestra.run.smithi018.stdout: Installing : python3-webtest-2.0.33-1.el8.noarch 25/32 2023-09-23T20:02:46.492 INFO:teuthology.orchestra.run.smithi018.stdout: Installing : python3-pecan-1.3.2-9.el8.noarch 26/32 2023-09-23T20:02:46.853 INFO:teuthology.orchestra.run.smithi018.stdout: Installing : ceph-mgr-modules-core-2:17.2.6-1123.g61431f6b.el8. 27/32 2023-09-23T20:02:46.989 INFO:teuthology.orchestra.run.smithi018.stdout: Installing : ceph-mgr-2:17.2.6-1123.g61431f6b.el8.x86_64 28/32 2023-09-23T20:02:48.572 INFO:teuthology.orchestra.run.smithi018.stdout: Running scriptlet: ceph-mgr-2:17.2.6-1123.g61431f6b.el8.x86_64 28/32 2023-09-23T20:02:48.781 INFO:teuthology.orchestra.run.smithi018.stdout: Installing : ceph-osd-2:17.2.6-1123.g61431f6b.el8.x86_64 29/32 2023-09-23T20:02:48.893 INFO:teuthology.orchestra.run.smithi149.stdout: Running scriptlet: ceph-selinux-2:17.2.6-1123.g61431f6b.el8.x86_64 22/25 2023-09-23T20:02:48.893 INFO:teuthology.orchestra.run.smithi149.stdout:skipping the directory /sys 2023-09-23T20:02:48.893 INFO:teuthology.orchestra.run.smithi149.stdout:skipping the directory /proc 2023-09-23T20:02:48.893 INFO:teuthology.orchestra.run.smithi149.stdout:skipping the directory /dev 2023-09-23T20:02:48.893 INFO:teuthology.orchestra.run.smithi149.stdout:skipping the directory /run 2023-09-23T20:02:48.893 INFO:teuthology.orchestra.run.smithi149.stdout:skipping the directory /mnt 2023-09-23T20:02:48.894 INFO:teuthology.orchestra.run.smithi149.stdout:skipping the directory /var/tmp 2023-09-23T20:02:48.894 INFO:teuthology.orchestra.run.smithi149.stdout:skipping the directory /home 2023-09-23T20:02:48.894 INFO:teuthology.orchestra.run.smithi149.stdout:skipping the directory /tmp 2023-09-23T20:02:48.894 INFO:teuthology.orchestra.run.smithi149.stdout:skipping the directory /dev 2023-09-23T20:02:48.894 INFO:teuthology.orchestra.run.smithi149.stdout: 2023-09-23T20:02:49.200 INFO:teuthology.orchestra.run.smithi018.stdout: Running scriptlet: ceph-osd-2:17.2.6-1123.g61431f6b.el8.x86_64 29/32 2023-09-23T20:02:49.337 INFO:teuthology.orchestra.run.smithi018.stdout: Installing : ceph-mon-2:17.2.6-1123.g61431f6b.el8.x86_64 30/32 2023-09-23T20:02:49.567 INFO:teuthology.orchestra.run.smithi018.stdout: Running scriptlet: ceph-mon-2:17.2.6-1123.g61431f6b.el8.x86_64 30/32 2023-09-23T20:02:49.729 INFO:teuthology.orchestra.run.smithi018.stdout: Installing : ceph-mds-2:17.2.6-1123.g61431f6b.el8.x86_64 31/32 2023-09-23T20:02:49.748 INFO:teuthology.orchestra.run.smithi018.stdout: Running scriptlet: ceph-mds-2:17.2.6-1123.g61431f6b.el8.x86_64 31/32 2023-09-23T20:02:49.894 INFO:teuthology.orchestra.run.smithi018.stdout: Installing : ceph-2:17.2.6-1123.g61431f6b.el8.x86_64 32/32 2023-09-23T20:02:50.267 INFO:teuthology.orchestra.run.smithi018.stdout: Running scriptlet: ceph-2:17.2.6-1123.g61431f6b.el8.x86_64 32/32 2023-09-23T20:02:50.267 INFO:teuthology.orchestra.run.smithi018.stdout: Verifying : ceph-2:17.2.6-1123.g61431f6b.el8.x86_64 1/32 2023-09-23T20:02:50.267 INFO:teuthology.orchestra.run.smithi018.stdout: Verifying : ceph-mds-2:17.2.6-1123.g61431f6b.el8.x86_64 2/32 2023-09-23T20:02:50.267 INFO:teuthology.orchestra.run.smithi018.stdout: Verifying : ceph-mgr-2:17.2.6-1123.g61431f6b.el8.x86_64 3/32 2023-09-23T20:02:50.267 INFO:teuthology.orchestra.run.smithi018.stdout: Verifying : ceph-mon-2:17.2.6-1123.g61431f6b.el8.x86_64 4/32 2023-09-23T20:02:50.268 INFO:teuthology.orchestra.run.smithi018.stdout: Verifying : ceph-osd-2:17.2.6-1123.g61431f6b.el8.x86_64 5/32 2023-09-23T20:02:50.268 INFO:teuthology.orchestra.run.smithi018.stdout: Verifying : ceph-mgr-modules-core-2:17.2.6-1123.g61431f6b.el8. 6/32 2023-09-23T20:02:50.268 INFO:teuthology.orchestra.run.smithi018.stdout: Verifying : python3-bcrypt-3.1.6-2.el8.1.x86_64 7/32 2023-09-23T20:02:50.268 INFO:teuthology.orchestra.run.smithi018.stdout: Verifying : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 8/32 2023-09-23T20:02:50.268 INFO:teuthology.orchestra.run.smithi018.stdout: Verifying : python3-cheroot-8.5.2-1.el8.noarch 9/32 2023-09-23T20:02:50.268 INFO:teuthology.orchestra.run.smithi018.stdout: Verifying : python3-cherrypy-18.4.0-1.el8.noarch 10/32 2023-09-23T20:02:50.268 INFO:teuthology.orchestra.run.smithi018.stdout: Verifying : python3-jaraco-6.2-6.el8.noarch 11/32 2023-09-23T20:02:50.268 INFO:teuthology.orchestra.run.smithi018.stdout: Verifying : python3-jaraco-functools-2.0-4.el8.noarch 12/32 2023-09-23T20:02:50.268 INFO:teuthology.orchestra.run.smithi018.stdout: Verifying : python3-logutils-0.3.5-11.el8.noarch 13/32 2023-09-23T20:02:50.268 INFO:teuthology.orchestra.run.smithi018.stdout: Verifying : python3-more-itertools-7.2.0-3.el8.noarch 14/32 2023-09-23T20:02:50.268 INFO:teuthology.orchestra.run.smithi018.stdout: Verifying : python3-pecan-1.3.2-9.el8.noarch 15/32 2023-09-23T20:02:50.268 INFO:teuthology.orchestra.run.smithi018.stdout: Verifying : python3-portend-2.6-1.el8.noarch 16/32 2023-09-23T20:02:50.269 INFO:teuthology.orchestra.run.smithi018.stdout: Verifying : python3-simplegeneric-0.8.1-17.el8.noarch 17/32 2023-09-23T20:02:50.269 INFO:teuthology.orchestra.run.smithi018.stdout: Verifying : python3-singledispatch-3.4.0.3-18.el8.noarch 18/32 2023-09-23T20:02:50.269 INFO:teuthology.orchestra.run.smithi018.stdout: Verifying : python3-tempora-1.14.1-5.el8.noarch 19/32 2023-09-23T20:02:50.269 INFO:teuthology.orchestra.run.smithi018.stdout: Verifying : python3-trustme-0.6.0-4.el8.noarch 20/32 2023-09-23T20:02:50.269 INFO:teuthology.orchestra.run.smithi018.stdout: Verifying : python3-waitress-1.4.3-1.el8.noarch 21/32 2023-09-23T20:02:50.269 INFO:teuthology.orchestra.run.smithi018.stdout: Verifying : python3-webob-1.8.5-1.el8.1.noarch 22/32 2023-09-23T20:02:50.269 INFO:teuthology.orchestra.run.smithi018.stdout: Verifying : python3-webtest-2.0.33-1.el8.noarch 23/32 2023-09-23T20:02:50.269 INFO:teuthology.orchestra.run.smithi018.stdout: Verifying : python3-zc-lockfile-2.0-2.el8.noarch 24/32 2023-09-23T20:02:50.269 INFO:teuthology.orchestra.run.smithi018.stdout: Verifying : python3-pycparser-2.14-14.el8.noarch 25/32 2023-09-23T20:02:50.269 INFO:teuthology.orchestra.run.smithi018.stdout: Verifying : python3-cryptography-3.2.1-5.el8.x86_64 26/32 2023-09-23T20:02:50.269 INFO:teuthology.orchestra.run.smithi018.stdout: Verifying : python3-cffi-1.11.5-5.el8.x86_64 27/32 2023-09-23T20:02:50.270 INFO:teuthology.orchestra.run.smithi018.stdout: Verifying : python3-pytz-2017.2-9.el8.noarch 28/32 2023-09-23T20:02:50.270 INFO:teuthology.orchestra.run.smithi018.stdout: Verifying : python3-markupsafe-0.23-19.el8.x86_64 29/32 2023-09-23T20:02:50.270 INFO:teuthology.orchestra.run.smithi018.stdout: Verifying : python3-mako-1.0.6-13.el8.noarch 30/32 2023-09-23T20:02:50.270 INFO:teuthology.orchestra.run.smithi018.stdout: Verifying : python3-werkzeug-0.12.2-4.el8.noarch 31/32 2023-09-23T20:02:50.556 INFO:teuthology.orchestra.run.smithi149.stdout: Installing : ceph-radosgw-2:17.2.6-1123.g61431f6b.el8.x86_64 23/25 2023-09-23T20:02:50.634 INFO:teuthology.orchestra.run.smithi149.stdout: Running scriptlet: ceph-radosgw-2:17.2.6-1123.g61431f6b.el8.x86_64 23/25 2023-09-23T20:02:50.634 INFO:teuthology.orchestra.run.smithi149.stdout: Cleanup : librbd1-1:12.2.7-9.el8.x86_64 24/25 2023-09-23T20:02:50.832 INFO:teuthology.orchestra.run.smithi149.stdout: Running scriptlet: librbd1-1:12.2.7-9.el8.x86_64 24/25 2023-09-23T20:02:50.832 INFO:teuthology.orchestra.run.smithi149.stdout: Cleanup : librados2-1:12.2.7-9.el8.x86_64 25/25 2023-09-23T20:02:51.432 INFO:teuthology.orchestra.run.smithi149.stdout: Running scriptlet: librados2-1:12.2.7-9.el8.x86_64 25/25 2023-09-23T20:02:51.432 INFO:teuthology.orchestra.run.smithi149.stdout: Verifying : ceph-base-2:17.2.6-1123.g61431f6b.el8.x86_64 1/25 2023-09-23T20:02:51.432 INFO:teuthology.orchestra.run.smithi149.stdout: Verifying : ceph-common-2:17.2.6-1123.g61431f6b.el8.x86_64 2/25 2023-09-23T20:02:51.432 INFO:teuthology.orchestra.run.smithi149.stdout: Verifying : ceph-radosgw-2:17.2.6-1123.g61431f6b.el8.x86_64 3/25 2023-09-23T20:02:51.432 INFO:teuthology.orchestra.run.smithi149.stdout: Verifying : ceph-selinux-2:17.2.6-1123.g61431f6b.el8.x86_64 4/25 2023-09-23T20:02:51.432 INFO:teuthology.orchestra.run.smithi149.stdout: Verifying : libcephfs2-2:17.2.6-1123.g61431f6b.el8.x86_64 5/25 2023-09-23T20:02:51.432 INFO:teuthology.orchestra.run.smithi149.stdout: Verifying : libradosstriper1-2:17.2.6-1123.g61431f6b.el8.x86_6 6/25 2023-09-23T20:02:51.432 INFO:teuthology.orchestra.run.smithi149.stdout: Verifying : librgw2-2:17.2.6-1123.g61431f6b.el8.x86_64 7/25 2023-09-23T20:02:51.433 INFO:teuthology.orchestra.run.smithi149.stdout: Verifying : python3-ceph-argparse-2:17.2.6-1123.g61431f6b.el8. 8/25 2023-09-23T20:02:51.433 INFO:teuthology.orchestra.run.smithi149.stdout: Verifying : python3-ceph-common-2:17.2.6-1123.g61431f6b.el8.x8 9/25 2023-09-23T20:02:51.433 INFO:teuthology.orchestra.run.smithi149.stdout: Verifying : python3-cephfs-2:17.2.6-1123.g61431f6b.el8.x86_64 10/25 2023-09-23T20:02:51.433 INFO:teuthology.orchestra.run.smithi149.stdout: Verifying : python3-rados-2:17.2.6-1123.g61431f6b.el8.x86_64 11/25 2023-09-23T20:02:51.433 INFO:teuthology.orchestra.run.smithi149.stdout: Verifying : python3-rbd-2:17.2.6-1123.g61431f6b.el8.x86_64 12/25 2023-09-23T20:02:51.433 INFO:teuthology.orchestra.run.smithi149.stdout: Verifying : python3-rgw-2:17.2.6-1123.g61431f6b.el8.x86_64 13/25 2023-09-23T20:02:51.433 INFO:teuthology.orchestra.run.smithi149.stdout: Verifying : gperftools-libs-1:2.7-9.el8.x86_64 14/25 2023-09-23T20:02:51.433 INFO:teuthology.orchestra.run.smithi149.stdout: Verifying : liboath-2.6.2-3.el8.x86_64 15/25 2023-09-23T20:02:51.433 INFO:teuthology.orchestra.run.smithi149.stdout: Verifying : libunwind-1.3.1-3.el8.x86_64 16/25 2023-09-23T20:02:51.433 INFO:teuthology.orchestra.run.smithi149.stdout: Verifying : thrift-0.13.0-2.el8.x86_64 17/25 2023-09-23T20:02:51.433 INFO:teuthology.orchestra.run.smithi149.stdout: Verifying : librabbitmq-0.9.0-3.el8.x86_64 18/25 2023-09-23T20:02:51.433 INFO:teuthology.orchestra.run.smithi149.stdout: Verifying : lttng-ust-2.8.1-11.el8.x86_64 19/25 2023-09-23T20:02:51.433 INFO:teuthology.orchestra.run.smithi149.stdout: Verifying : python3-prettytable-0.7.2-14.el8.noarch 20/25 2023-09-23T20:02:51.434 INFO:teuthology.orchestra.run.smithi149.stdout: Verifying : librdkafka-0.11.4-3.el8.x86_64 21/25 2023-09-23T20:02:51.434 INFO:teuthology.orchestra.run.smithi149.stdout: Verifying : librados2-2:17.2.6-1123.g61431f6b.el8.x86_64 22/25 2023-09-23T20:02:51.434 INFO:teuthology.orchestra.run.smithi149.stdout: Verifying : librados2-1:12.2.7-9.el8.x86_64 23/25 2023-09-23T20:02:51.434 INFO:teuthology.orchestra.run.smithi149.stdout: Verifying : librbd1-2:17.2.6-1123.g61431f6b.el8.x86_64 24/25 2023-09-23T20:02:51.698 INFO:teuthology.orchestra.run.smithi018.stdout: Verifying : python3-pyOpenSSL-19.0.0-1.el8.noarch 32/32 2023-09-23T20:02:51.698 INFO:teuthology.orchestra.run.smithi018.stdout:Last metadata expiration check: 0:01:29 ago on Sat 23 Sep 2023 08:01:22 PM UTC. 2023-09-23T20:02:52.796 INFO:teuthology.orchestra.run.smithi149.stdout: Verifying : librbd1-1:12.2.7-9.el8.x86_64 25/25 2023-09-23T20:02:52.796 INFO:teuthology.orchestra.run.smithi149.stdout:Last metadata expiration check: 0:00:50 ago on Sat 23 Sep 2023 08:02:02 PM UTC. 2023-09-23T20:02:54.317 INFO:teuthology.orchestra.run.smithi018.stdout:Installed products updated. 2023-09-23T20:02:54.525 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:02:54.525 INFO:teuthology.orchestra.run.smithi018.stdout:Installed: 2023-09-23T20:02:54.526 INFO:teuthology.orchestra.run.smithi018.stdout: ceph-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:02:54.526 INFO:teuthology.orchestra.run.smithi018.stdout: ceph-mds-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:02:54.526 INFO:teuthology.orchestra.run.smithi018.stdout: ceph-mgr-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:02:54.526 INFO:teuthology.orchestra.run.smithi018.stdout: ceph-mgr-modules-core-2:17.2.6-1123.g61431f6b.el8.noarch 2023-09-23T20:02:54.526 INFO:teuthology.orchestra.run.smithi018.stdout: ceph-mon-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:02:54.526 INFO:teuthology.orchestra.run.smithi018.stdout: ceph-osd-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:02:54.526 INFO:teuthology.orchestra.run.smithi018.stdout: python3-bcrypt-3.1.6-2.el8.1.x86_64 2023-09-23T20:02:54.526 INFO:teuthology.orchestra.run.smithi018.stdout: python3-beautifulsoup4-4.6.3-2.el8.1.noarch 2023-09-23T20:02:54.526 INFO:teuthology.orchestra.run.smithi018.stdout: python3-cffi-1.11.5-5.el8.x86_64 2023-09-23T20:02:54.526 INFO:teuthology.orchestra.run.smithi018.stdout: python3-cheroot-8.5.2-1.el8.noarch 2023-09-23T20:02:54.527 INFO:teuthology.orchestra.run.smithi018.stdout: python3-cherrypy-18.4.0-1.el8.noarch 2023-09-23T20:02:54.527 INFO:teuthology.orchestra.run.smithi018.stdout: python3-cryptography-3.2.1-5.el8.x86_64 2023-09-23T20:02:54.527 INFO:teuthology.orchestra.run.smithi018.stdout: python3-jaraco-6.2-6.el8.noarch 2023-09-23T20:02:54.527 INFO:teuthology.orchestra.run.smithi018.stdout: python3-jaraco-functools-2.0-4.el8.noarch 2023-09-23T20:02:54.527 INFO:teuthology.orchestra.run.smithi018.stdout: python3-logutils-0.3.5-11.el8.noarch 2023-09-23T20:02:54.527 INFO:teuthology.orchestra.run.smithi018.stdout: python3-mako-1.0.6-13.el8.noarch 2023-09-23T20:02:54.527 INFO:teuthology.orchestra.run.smithi018.stdout: python3-markupsafe-0.23-19.el8.x86_64 2023-09-23T20:02:54.527 INFO:teuthology.orchestra.run.smithi018.stdout: python3-more-itertools-7.2.0-3.el8.noarch 2023-09-23T20:02:54.527 INFO:teuthology.orchestra.run.smithi018.stdout: python3-pecan-1.3.2-9.el8.noarch 2023-09-23T20:02:54.527 INFO:teuthology.orchestra.run.smithi018.stdout: python3-portend-2.6-1.el8.noarch 2023-09-23T20:02:54.527 INFO:teuthology.orchestra.run.smithi018.stdout: python3-pyOpenSSL-19.0.0-1.el8.noarch 2023-09-23T20:02:54.528 INFO:teuthology.orchestra.run.smithi018.stdout: python3-pycparser-2.14-14.el8.noarch 2023-09-23T20:02:54.528 INFO:teuthology.orchestra.run.smithi018.stdout: python3-pytz-2017.2-9.el8.noarch 2023-09-23T20:02:54.528 INFO:teuthology.orchestra.run.smithi018.stdout: python3-simplegeneric-0.8.1-17.el8.noarch 2023-09-23T20:02:54.528 INFO:teuthology.orchestra.run.smithi018.stdout: python3-singledispatch-3.4.0.3-18.el8.noarch 2023-09-23T20:02:54.528 INFO:teuthology.orchestra.run.smithi018.stdout: python3-tempora-1.14.1-5.el8.noarch 2023-09-23T20:02:54.528 INFO:teuthology.orchestra.run.smithi018.stdout: python3-trustme-0.6.0-4.el8.noarch 2023-09-23T20:02:54.528 INFO:teuthology.orchestra.run.smithi018.stdout: python3-waitress-1.4.3-1.el8.noarch 2023-09-23T20:02:54.528 INFO:teuthology.orchestra.run.smithi018.stdout: python3-webob-1.8.5-1.el8.1.noarch 2023-09-23T20:02:54.528 INFO:teuthology.orchestra.run.smithi018.stdout: python3-webtest-2.0.33-1.el8.noarch 2023-09-23T20:02:54.528 INFO:teuthology.orchestra.run.smithi018.stdout: python3-werkzeug-0.12.2-4.el8.noarch 2023-09-23T20:02:54.528 INFO:teuthology.orchestra.run.smithi018.stdout: python3-zc-lockfile-2.0-2.el8.noarch 2023-09-23T20:02:54.528 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:02:54.529 INFO:teuthology.orchestra.run.smithi018.stdout:Complete! 2023-09-23T20:02:54.680 DEBUG:teuthology.orchestra.run.smithi018:> sudo yum -y install ceph-base 2023-09-23T20:02:55.047 INFO:teuthology.orchestra.run.smithi018.stdout:Updating Subscription Management repositories. 2023-09-23T20:02:55.076 INFO:teuthology.orchestra.run.smithi149.stdout:Installed products updated. 2023-09-23T20:02:55.266 INFO:teuthology.orchestra.run.smithi149.stdout: 2023-09-23T20:02:55.267 INFO:teuthology.orchestra.run.smithi149.stdout:Upgraded: 2023-09-23T20:02:55.267 INFO:teuthology.orchestra.run.smithi149.stdout: librados2-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:02:55.267 INFO:teuthology.orchestra.run.smithi149.stdout: librbd1-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:02:55.267 INFO:teuthology.orchestra.run.smithi149.stdout:Installed: 2023-09-23T20:02:55.267 INFO:teuthology.orchestra.run.smithi149.stdout: ceph-base-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:02:55.267 INFO:teuthology.orchestra.run.smithi149.stdout: ceph-common-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:02:55.267 INFO:teuthology.orchestra.run.smithi149.stdout: ceph-radosgw-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:02:55.267 INFO:teuthology.orchestra.run.smithi149.stdout: ceph-selinux-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:02:55.267 INFO:teuthology.orchestra.run.smithi149.stdout: gperftools-libs-1:2.7-9.el8.x86_64 2023-09-23T20:02:55.267 INFO:teuthology.orchestra.run.smithi149.stdout: libcephfs2-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:02:55.267 INFO:teuthology.orchestra.run.smithi149.stdout: liboath-2.6.2-3.el8.x86_64 2023-09-23T20:02:55.268 INFO:teuthology.orchestra.run.smithi149.stdout: librabbitmq-0.9.0-3.el8.x86_64 2023-09-23T20:02:55.268 INFO:teuthology.orchestra.run.smithi149.stdout: libradosstriper1-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:02:55.268 INFO:teuthology.orchestra.run.smithi149.stdout: librdkafka-0.11.4-3.el8.x86_64 2023-09-23T20:02:55.268 INFO:teuthology.orchestra.run.smithi149.stdout: librgw2-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:02:55.268 INFO:teuthology.orchestra.run.smithi149.stdout: libunwind-1.3.1-3.el8.x86_64 2023-09-23T20:02:55.268 INFO:teuthology.orchestra.run.smithi149.stdout: lttng-ust-2.8.1-11.el8.x86_64 2023-09-23T20:02:55.268 INFO:teuthology.orchestra.run.smithi149.stdout: python3-ceph-argparse-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:02:55.268 INFO:teuthology.orchestra.run.smithi149.stdout: python3-ceph-common-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:02:55.268 INFO:teuthology.orchestra.run.smithi149.stdout: python3-cephfs-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:02:55.268 INFO:teuthology.orchestra.run.smithi149.stdout: python3-prettytable-0.7.2-14.el8.noarch 2023-09-23T20:02:55.268 INFO:teuthology.orchestra.run.smithi149.stdout: python3-rados-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:02:55.268 INFO:teuthology.orchestra.run.smithi149.stdout: python3-rbd-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:02:55.269 INFO:teuthology.orchestra.run.smithi149.stdout: python3-rgw-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:02:55.269 INFO:teuthology.orchestra.run.smithi149.stdout: thrift-0.13.0-2.el8.x86_64 2023-09-23T20:02:55.269 INFO:teuthology.orchestra.run.smithi149.stdout: 2023-09-23T20:02:55.269 INFO:teuthology.orchestra.run.smithi149.stdout:Complete! 2023-09-23T20:02:55.448 DEBUG:teuthology.orchestra.run.smithi149:> sudo yum -y install ceph-test 2023-09-23T20:02:55.796 INFO:teuthology.orchestra.run.smithi149.stdout:Updating Subscription Management repositories. 2023-09-23T20:02:55.913 INFO:teuthology.orchestra.run.smithi018.stdout:Last metadata expiration check: 0:01:33 ago on Sat 23 Sep 2023 08:01:22 PM UTC. 2023-09-23T20:02:56.666 INFO:teuthology.orchestra.run.smithi149.stdout:Last metadata expiration check: 0:00:54 ago on Sat 23 Sep 2023 08:02:02 PM UTC. 2023-09-23T20:02:57.347 INFO:teuthology.orchestra.run.smithi018.stdout:Package ceph-base-2:17.2.6-1123.g61431f6b.el8.x86_64 is already installed. 2023-09-23T20:02:57.466 INFO:teuthology.orchestra.run.smithi018.stdout:Dependencies resolved. 2023-09-23T20:02:57.467 INFO:teuthology.orchestra.run.smithi018.stdout:Nothing to do. 2023-09-23T20:02:57.467 INFO:teuthology.orchestra.run.smithi018.stdout:Complete! 2023-09-23T20:02:57.986 DEBUG:teuthology.orchestra.run.smithi018:> sudo yum -y install cephadm 2023-09-23T20:02:58.046 INFO:teuthology.orchestra.run.smithi149.stdout:Dependencies resolved. 2023-09-23T20:02:58.047 INFO:teuthology.orchestra.run.smithi149.stdout:============================================================================================ 2023-09-23T20:02:58.047 INFO:teuthology.orchestra.run.smithi149.stdout: Package Arch Version Repository Size 2023-09-23T20:02:58.047 INFO:teuthology.orchestra.run.smithi149.stdout:============================================================================================ 2023-09-23T20:02:58.047 INFO:teuthology.orchestra.run.smithi149.stdout:Installing: 2023-09-23T20:02:58.047 INFO:teuthology.orchestra.run.smithi149.stdout: ceph-test x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 52 M 2023-09-23T20:02:58.047 INFO:teuthology.orchestra.run.smithi149.stdout:Installing dependencies: 2023-09-23T20:02:58.047 INFO:teuthology.orchestra.run.smithi149.stdout: jq x86_64 1.6-3.el8 rhel-8-for-x86_64-appstream-rpms 202 k 2023-09-23T20:02:58.048 INFO:teuthology.orchestra.run.smithi149.stdout: libcephsqlite x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 174 k 2023-09-23T20:02:58.048 INFO:teuthology.orchestra.run.smithi149.stdout: oniguruma x86_64 6.8.2-2.el8 rhel-8-for-x86_64-appstream-rpms 187 k 2023-09-23T20:02:58.048 INFO:teuthology.orchestra.run.smithi149.stdout: socat x86_64 1.7.4.1-1.el8 rhel-8-for-x86_64-appstream-rpms 323 k 2023-09-23T20:02:58.048 INFO:teuthology.orchestra.run.smithi149.stdout: xmlstarlet x86_64 1.6.1-20.el8 lab-extras 70 k 2023-09-23T20:02:58.048 INFO:teuthology.orchestra.run.smithi149.stdout: 2023-09-23T20:02:58.048 INFO:teuthology.orchestra.run.smithi149.stdout:Transaction Summary 2023-09-23T20:02:58.048 INFO:teuthology.orchestra.run.smithi149.stdout:============================================================================================ 2023-09-23T20:02:58.048 INFO:teuthology.orchestra.run.smithi149.stdout:Install 6 Packages 2023-09-23T20:02:58.048 INFO:teuthology.orchestra.run.smithi149.stdout: 2023-09-23T20:02:58.049 INFO:teuthology.orchestra.run.smithi149.stdout:Total download size: 53 M 2023-09-23T20:02:58.049 INFO:teuthology.orchestra.run.smithi149.stdout:Installed size: 233 M 2023-09-23T20:02:58.049 INFO:teuthology.orchestra.run.smithi149.stdout:Downloading Packages: 2023-09-23T20:02:58.092 INFO:teuthology.orchestra.run.smithi149.stdout:(1/6): xmlstarlet-1.6.1-20.el8.x86_64.rpm 1.6 MB/s | 70 kB 00:00 2023-09-23T20:02:58.160 INFO:teuthology.orchestra.run.smithi149.stdout:(2/6): socat-1.7.4.1-1.el8.x86_64.rpm 5.4 MB/s | 323 kB 00:00 2023-09-23T20:02:58.193 INFO:teuthology.orchestra.run.smithi149.stdout:(3/6): oniguruma-6.8.2-2.el8.x86_64.rpm 5.5 MB/s | 187 kB 00:00 2023-09-23T20:02:58.235 INFO:teuthology.orchestra.run.smithi149.stdout:(4/6): jq-1.6-3.el8.x86_64.rpm 4.7 MB/s | 202 kB 00:00 2023-09-23T20:02:58.277 INFO:teuthology.orchestra.run.smithi149.stdout:(5/6): libcephsqlite-17.2.6-1123.g61431f6b.el8. 767 kB/s | 174 kB 00:00 2023-09-23T20:02:58.351 INFO:teuthology.orchestra.run.smithi018.stdout:Updating Subscription Management repositories. 2023-09-23T20:02:59.195 INFO:teuthology.orchestra.run.smithi018.stdout:Last metadata expiration check: 0:01:37 ago on Sat 23 Sep 2023 08:01:22 PM UTC. 2023-09-23T20:03:00.668 INFO:teuthology.orchestra.run.smithi149.stdout:(6/6): ceph-test-17.2.6-1123.g61431f6b.el8.x86_ 20 MB/s | 52 MB 00:02 2023-09-23T20:03:00.670 INFO:teuthology.orchestra.run.smithi149.stdout:-------------------------------------------------------------------------------- 2023-09-23T20:03:00.671 INFO:teuthology.orchestra.run.smithi149.stdout:Total 20 MB/s | 53 MB 00:02 2023-09-23T20:03:00.706 INFO:teuthology.orchestra.run.smithi149.stdout:Running transaction check 2023-09-23T20:03:00.727 INFO:teuthology.orchestra.run.smithi149.stdout:Transaction check succeeded. 2023-09-23T20:03:00.727 INFO:teuthology.orchestra.run.smithi149.stdout:Running transaction test 2023-09-23T20:03:00.764 INFO:teuthology.orchestra.run.smithi018.stdout:Dependencies resolved. 2023-09-23T20:03:00.765 INFO:teuthology.orchestra.run.smithi018.stdout:================================================================================ 2023-09-23T20:03:00.765 INFO:teuthology.orchestra.run.smithi018.stdout: Package Arch Version Repository Size 2023-09-23T20:03:00.765 INFO:teuthology.orchestra.run.smithi018.stdout:================================================================================ 2023-09-23T20:03:00.765 INFO:teuthology.orchestra.run.smithi018.stdout:Installing: 2023-09-23T20:03:00.765 INFO:teuthology.orchestra.run.smithi018.stdout: cephadm noarch 2:17.2.6-1123.g61431f6b.el8 ceph-noarch 80 k 2023-09-23T20:03:00.765 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:03:00.766 INFO:teuthology.orchestra.run.smithi018.stdout:Transaction Summary 2023-09-23T20:03:00.766 INFO:teuthology.orchestra.run.smithi018.stdout:================================================================================ 2023-09-23T20:03:00.766 INFO:teuthology.orchestra.run.smithi018.stdout:Install 1 Package 2023-09-23T20:03:00.766 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:03:00.767 INFO:teuthology.orchestra.run.smithi018.stdout:Total download size: 80 k 2023-09-23T20:03:00.767 INFO:teuthology.orchestra.run.smithi018.stdout:Installed size: 363 k 2023-09-23T20:03:00.767 INFO:teuthology.orchestra.run.smithi018.stdout:Downloading Packages: 2023-09-23T20:03:01.001 INFO:teuthology.orchestra.run.smithi149.stdout:Transaction test succeeded. 2023-09-23T20:03:01.005 INFO:teuthology.orchestra.run.smithi149.stdout:Running transaction 2023-09-23T20:03:01.016 INFO:teuthology.orchestra.run.smithi018.stdout:cephadm-17.2.6-1123.g61431f6b.el8.noarch.rpm 320 kB/s | 80 kB 00:00 2023-09-23T20:03:01.017 INFO:teuthology.orchestra.run.smithi018.stdout:-------------------------------------------------------------------------------- 2023-09-23T20:03:01.017 INFO:teuthology.orchestra.run.smithi018.stdout:Total 319 kB/s | 80 kB 00:00 2023-09-23T20:03:01.018 INFO:teuthology.orchestra.run.smithi018.stdout:Running transaction check 2023-09-23T20:03:01.026 INFO:teuthology.orchestra.run.smithi018.stdout:Transaction check succeeded. 2023-09-23T20:03:01.026 INFO:teuthology.orchestra.run.smithi018.stdout:Running transaction test 2023-09-23T20:03:01.028 INFO:teuthology.orchestra.run.smithi018.stdout:Transaction test succeeded. 2023-09-23T20:03:01.033 INFO:teuthology.orchestra.run.smithi018.stdout:Running transaction 2023-09-23T20:03:01.077 INFO:teuthology.orchestra.run.smithi018.stdout: Preparing : 1/1 2023-09-23T20:03:01.466 INFO:teuthology.orchestra.run.smithi149.stdout: Preparing : 1/1 2023-09-23T20:03:01.611 INFO:teuthology.orchestra.run.smithi149.stdout: Installing : oniguruma-6.8.2-2.el8.x86_64 1/6 2023-09-23T20:03:01.696 INFO:teuthology.orchestra.run.smithi149.stdout: Running scriptlet: oniguruma-6.8.2-2.el8.x86_64 1/6 2023-09-23T20:03:01.898 INFO:teuthology.orchestra.run.smithi149.stdout: Installing : jq-1.6-3.el8.x86_64 2/6 2023-09-23T20:03:02.054 INFO:teuthology.orchestra.run.smithi149.stdout: Installing : socat-1.7.4.1-1.el8.x86_64 3/6 2023-09-23T20:03:02.219 INFO:teuthology.orchestra.run.smithi149.stdout: Installing : xmlstarlet-1.6.1-20.el8.x86_64 4/6 2023-09-23T20:03:02.369 INFO:teuthology.orchestra.run.smithi149.stdout: Installing : libcephsqlite-2:17.2.6-1123.g61431f6b.el8.x86_64 5/6 2023-09-23T20:03:02.927 INFO:teuthology.orchestra.run.smithi018.stdout: Running scriptlet: cephadm-2:17.2.6-1123.g61431f6b.el8.noarch 1/1 2023-09-23T20:03:03.083 INFO:teuthology.orchestra.run.smithi018.stdout: Installing : cephadm-2:17.2.6-1123.g61431f6b.el8.noarch 1/1 2023-09-23T20:03:03.258 INFO:teuthology.orchestra.run.smithi018.stdout: Running scriptlet: cephadm-2:17.2.6-1123.g61431f6b.el8.noarch 1/1 2023-09-23T20:03:04.585 INFO:teuthology.orchestra.run.smithi018.stdout: Verifying : cephadm-2:17.2.6-1123.g61431f6b.el8.noarch 1/1 2023-09-23T20:03:04.585 INFO:teuthology.orchestra.run.smithi018.stdout:Last metadata expiration check: 0:01:42 ago on Sat 23 Sep 2023 08:01:22 PM UTC. 2023-09-23T20:03:07.233 INFO:teuthology.orchestra.run.smithi018.stdout:Installed products updated. 2023-09-23T20:03:07.324 INFO:teuthology.orchestra.run.smithi149.stdout: Running scriptlet: libcephsqlite-2:17.2.6-1123.g61431f6b.el8.x86_64 5/6 2023-09-23T20:03:07.431 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:03:07.431 INFO:teuthology.orchestra.run.smithi018.stdout:Installed: 2023-09-23T20:03:07.431 INFO:teuthology.orchestra.run.smithi018.stdout: cephadm-2:17.2.6-1123.g61431f6b.el8.noarch 2023-09-23T20:03:07.431 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:03:07.431 INFO:teuthology.orchestra.run.smithi018.stdout:Complete! 2023-09-23T20:03:07.558 DEBUG:teuthology.orchestra.run.smithi018:> sudo yum -y install ceph-immutable-object-cache 2023-09-23T20:03:07.582 INFO:teuthology.orchestra.run.smithi149.stdout: Installing : ceph-test-2:17.2.6-1123.g61431f6b.el8.x86_64 6/6 2023-09-23T20:03:07.822 INFO:teuthology.orchestra.run.smithi149.stdout: Running scriptlet: ceph-test-2:17.2.6-1123.g61431f6b.el8.x86_64 6/6 2023-09-23T20:03:07.823 INFO:teuthology.orchestra.run.smithi149.stdout: Verifying : ceph-test-2:17.2.6-1123.g61431f6b.el8.x86_64 1/6 2023-09-23T20:03:07.823 INFO:teuthology.orchestra.run.smithi149.stdout: Verifying : libcephsqlite-2:17.2.6-1123.g61431f6b.el8.x86_64 2/6 2023-09-23T20:03:07.823 INFO:teuthology.orchestra.run.smithi149.stdout: Verifying : xmlstarlet-1.6.1-20.el8.x86_64 3/6 2023-09-23T20:03:07.823 INFO:teuthology.orchestra.run.smithi149.stdout: Verifying : socat-1.7.4.1-1.el8.x86_64 4/6 2023-09-23T20:03:07.823 INFO:teuthology.orchestra.run.smithi149.stdout: Verifying : oniguruma-6.8.2-2.el8.x86_64 5/6 2023-09-23T20:03:07.938 INFO:teuthology.orchestra.run.smithi018.stdout:Updating Subscription Management repositories. 2023-09-23T20:03:08.788 INFO:teuthology.orchestra.run.smithi018.stdout:Last metadata expiration check: 0:01:46 ago on Sat 23 Sep 2023 08:01:22 PM UTC. 2023-09-23T20:03:08.929 INFO:teuthology.orchestra.run.smithi149.stdout: Verifying : jq-1.6-3.el8.x86_64 6/6 2023-09-23T20:03:08.929 INFO:teuthology.orchestra.run.smithi149.stdout:Last metadata expiration check: 0:01:06 ago on Sat 23 Sep 2023 08:02:02 PM UTC. 2023-09-23T20:03:10.321 INFO:teuthology.orchestra.run.smithi018.stdout:Dependencies resolved. 2023-09-23T20:03:10.322 INFO:teuthology.orchestra.run.smithi018.stdout:================================================================================ 2023-09-23T20:03:10.322 INFO:teuthology.orchestra.run.smithi018.stdout: Package Arch Version Repo Size 2023-09-23T20:03:10.322 INFO:teuthology.orchestra.run.smithi018.stdout:================================================================================ 2023-09-23T20:03:10.322 INFO:teuthology.orchestra.run.smithi018.stdout:Installing: 2023-09-23T20:03:10.322 INFO:teuthology.orchestra.run.smithi018.stdout: ceph-immutable-object-cache x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 161 k 2023-09-23T20:03:10.322 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:03:10.322 INFO:teuthology.orchestra.run.smithi018.stdout:Transaction Summary 2023-09-23T20:03:10.323 INFO:teuthology.orchestra.run.smithi018.stdout:================================================================================ 2023-09-23T20:03:10.323 INFO:teuthology.orchestra.run.smithi018.stdout:Install 1 Package 2023-09-23T20:03:10.323 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:03:10.323 INFO:teuthology.orchestra.run.smithi018.stdout:Total download size: 161 k 2023-09-23T20:03:10.323 INFO:teuthology.orchestra.run.smithi018.stdout:Installed size: 490 k 2023-09-23T20:03:10.323 INFO:teuthology.orchestra.run.smithi018.stdout:Downloading Packages: 2023-09-23T20:03:10.505 INFO:teuthology.orchestra.run.smithi018.stdout:ceph-immutable-object-cache-17.2.6-1123.g61431f 885 kB/s | 161 kB 00:00 2023-09-23T20:03:10.506 INFO:teuthology.orchestra.run.smithi018.stdout:-------------------------------------------------------------------------------- 2023-09-23T20:03:10.506 INFO:teuthology.orchestra.run.smithi018.stdout:Total 880 kB/s | 161 kB 00:00 2023-09-23T20:03:10.506 INFO:teuthology.orchestra.run.smithi018.stdout:Running transaction check 2023-09-23T20:03:10.518 INFO:teuthology.orchestra.run.smithi018.stdout:Transaction check succeeded. 2023-09-23T20:03:10.518 INFO:teuthology.orchestra.run.smithi018.stdout:Running transaction test 2023-09-23T20:03:10.575 INFO:teuthology.orchestra.run.smithi018.stdout:Transaction test succeeded. 2023-09-23T20:03:10.579 INFO:teuthology.orchestra.run.smithi018.stdout:Running transaction 2023-09-23T20:03:10.699 INFO:teuthology.orchestra.run.smithi018.stdout: Preparing : 1/1 2023-09-23T20:03:10.850 INFO:teuthology.orchestra.run.smithi018.stdout: Installing : ceph-immutable-object-cache-2:17.2.6-1123.g61431f6b. 1/1 2023-09-23T20:03:11.222 INFO:teuthology.orchestra.run.smithi018.stdout: Running scriptlet: ceph-immutable-object-cache-2:17.2.6-1123.g61431f6b. 1/1 2023-09-23T20:03:11.422 INFO:teuthology.orchestra.run.smithi149.stdout:Installed products updated. 2023-09-23T20:03:11.602 INFO:teuthology.orchestra.run.smithi149.stdout: 2023-09-23T20:03:11.603 INFO:teuthology.orchestra.run.smithi149.stdout:Installed: 2023-09-23T20:03:11.603 INFO:teuthology.orchestra.run.smithi149.stdout: ceph-test-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:03:11.603 INFO:teuthology.orchestra.run.smithi149.stdout: jq-1.6-3.el8.x86_64 2023-09-23T20:03:11.603 INFO:teuthology.orchestra.run.smithi149.stdout: libcephsqlite-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:03:11.603 INFO:teuthology.orchestra.run.smithi149.stdout: oniguruma-6.8.2-2.el8.x86_64 2023-09-23T20:03:11.603 INFO:teuthology.orchestra.run.smithi149.stdout: socat-1.7.4.1-1.el8.x86_64 2023-09-23T20:03:11.604 INFO:teuthology.orchestra.run.smithi149.stdout: xmlstarlet-1.6.1-20.el8.x86_64 2023-09-23T20:03:11.604 INFO:teuthology.orchestra.run.smithi149.stdout: 2023-09-23T20:03:11.604 INFO:teuthology.orchestra.run.smithi149.stdout:Complete! 2023-09-23T20:03:11.751 DEBUG:teuthology.orchestra.run.smithi149:> sudo yum -y install ceph 2023-09-23T20:03:12.093 INFO:teuthology.orchestra.run.smithi149.stdout:Updating Subscription Management repositories. 2023-09-23T20:03:12.301 INFO:teuthology.orchestra.run.smithi018.stdout: Verifying : ceph-immutable-object-cache-2:17.2.6-1123.g61431f6b. 1/1 2023-09-23T20:03:12.301 INFO:teuthology.orchestra.run.smithi018.stdout:Last metadata expiration check: 0:01:50 ago on Sat 23 Sep 2023 08:01:22 PM UTC. 2023-09-23T20:03:12.919 INFO:teuthology.orchestra.run.smithi149.stdout:Last metadata expiration check: 0:01:10 ago on Sat 23 Sep 2023 08:02:02 PM UTC. 2023-09-23T20:03:14.314 INFO:teuthology.orchestra.run.smithi149.stdout:Dependencies resolved. 2023-09-23T20:03:14.317 INFO:teuthology.orchestra.run.smithi149.stdout:======================================================================================================= 2023-09-23T20:03:14.317 INFO:teuthology.orchestra.run.smithi149.stdout: Package Arch Version Repository Size 2023-09-23T20:03:14.317 INFO:teuthology.orchestra.run.smithi149.stdout:======================================================================================================= 2023-09-23T20:03:14.317 INFO:teuthology.orchestra.run.smithi149.stdout:Installing: 2023-09-23T20:03:14.317 INFO:teuthology.orchestra.run.smithi149.stdout: ceph x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 6.4 k 2023-09-23T20:03:14.318 INFO:teuthology.orchestra.run.smithi149.stdout:Installing dependencies: 2023-09-23T20:03:14.318 INFO:teuthology.orchestra.run.smithi149.stdout: ceph-mds x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 2.3 M 2023-09-23T20:03:14.318 INFO:teuthology.orchestra.run.smithi149.stdout: ceph-mgr x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 1.6 M 2023-09-23T20:03:14.318 INFO:teuthology.orchestra.run.smithi149.stdout: ceph-mgr-modules-core noarch 2:17.2.6-1123.g61431f6b.el8 ceph-noarch 238 k 2023-09-23T20:03:14.318 INFO:teuthology.orchestra.run.smithi149.stdout: ceph-mon x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 3.9 M 2023-09-23T20:03:14.318 INFO:teuthology.orchestra.run.smithi149.stdout: ceph-osd x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 17 M 2023-09-23T20:03:14.318 INFO:teuthology.orchestra.run.smithi149.stdout: python3-bcrypt x86_64 3.1.6-2.el8.1 epel 44 k 2023-09-23T20:03:14.318 INFO:teuthology.orchestra.run.smithi149.stdout: python3-beautifulsoup4 noarch 4.6.3-2.el8.1 epel 185 k 2023-09-23T20:03:14.318 INFO:teuthology.orchestra.run.smithi149.stdout: python3-cffi x86_64 1.11.5-5.el8 rhel-8-for-x86_64-baseos-rpms 238 k 2023-09-23T20:03:14.318 INFO:teuthology.orchestra.run.smithi149.stdout: python3-cheroot noarch 8.5.2-1.el8 epel 173 k 2023-09-23T20:03:14.318 INFO:teuthology.orchestra.run.smithi149.stdout: python3-cherrypy noarch 18.4.0-1.el8 epel 384 k 2023-09-23T20:03:14.319 INFO:teuthology.orchestra.run.smithi149.stdout: python3-cryptography x86_64 3.2.1-5.el8 rhel-8-for-x86_64-baseos-rpms 559 k 2023-09-23T20:03:14.319 INFO:teuthology.orchestra.run.smithi149.stdout: python3-jaraco noarch 6.2-6.el8 epel 11 k 2023-09-23T20:03:14.319 INFO:teuthology.orchestra.run.smithi149.stdout: python3-jaraco-functools noarch 2.0-4.el8 epel 18 k 2023-09-23T20:03:14.319 INFO:teuthology.orchestra.run.smithi149.stdout: python3-logutils noarch 0.3.5-11.el8 epel 49 k 2023-09-23T20:03:14.319 INFO:teuthology.orchestra.run.smithi149.stdout: python3-mako noarch 1.0.6-13.el8 rhel-8-for-x86_64-appstream-rpms 157 k 2023-09-23T20:03:14.319 INFO:teuthology.orchestra.run.smithi149.stdout: python3-markupsafe x86_64 0.23-19.el8 rhel-8-for-x86_64-appstream-rpms 39 k 2023-09-23T20:03:14.319 INFO:teuthology.orchestra.run.smithi149.stdout: python3-more-itertools noarch 7.2.0-3.el8 epel 59 k 2023-09-23T20:03:14.319 INFO:teuthology.orchestra.run.smithi149.stdout: python3-pecan noarch 1.3.2-9.el8 epel 283 k 2023-09-23T20:03:14.319 INFO:teuthology.orchestra.run.smithi149.stdout: python3-portend noarch 2.6-1.el8 epel 16 k 2023-09-23T20:03:14.319 INFO:teuthology.orchestra.run.smithi149.stdout: python3-pyOpenSSL noarch 19.0.0-1.el8 rhel-8-for-x86_64-appstream-rpms 103 k 2023-09-23T20:03:14.319 INFO:teuthology.orchestra.run.smithi149.stdout: python3-pycparser noarch 2.14-14.el8 rhel-8-for-x86_64-baseos-rpms 109 k 2023-09-23T20:03:14.319 INFO:teuthology.orchestra.run.smithi149.stdout: python3-pytz noarch 2017.2-9.el8 rhel-8-for-x86_64-appstream-rpms 54 k 2023-09-23T20:03:14.320 INFO:teuthology.orchestra.run.smithi149.stdout: python3-simplegeneric noarch 0.8.1-17.el8 epel 19 k 2023-09-23T20:03:14.320 INFO:teuthology.orchestra.run.smithi149.stdout: python3-singledispatch noarch 3.4.0.3-18.el8 epel 24 k 2023-09-23T20:03:14.320 INFO:teuthology.orchestra.run.smithi149.stdout: python3-tempora noarch 1.14.1-5.el8 epel 29 k 2023-09-23T20:03:14.320 INFO:teuthology.orchestra.run.smithi149.stdout: python3-trustme noarch 0.6.0-4.el8 epel 27 k 2023-09-23T20:03:14.320 INFO:teuthology.orchestra.run.smithi149.stdout: python3-waitress noarch 1.4.3-1.el8 epel 255 k 2023-09-23T20:03:14.320 INFO:teuthology.orchestra.run.smithi149.stdout: python3-webob noarch 1.8.5-1.el8.1 epel 251 k 2023-09-23T20:03:14.320 INFO:teuthology.orchestra.run.smithi149.stdout: python3-webtest noarch 2.0.33-1.el8 epel 85 k 2023-09-23T20:03:14.320 INFO:teuthology.orchestra.run.smithi149.stdout: python3-werkzeug noarch 0.12.2-4.el8 rhel-8-for-x86_64-appstream-rpms 457 k 2023-09-23T20:03:14.320 INFO:teuthology.orchestra.run.smithi149.stdout: python3-zc-lockfile noarch 2.0-2.el8 epel 23 k 2023-09-23T20:03:14.320 INFO:teuthology.orchestra.run.smithi149.stdout: 2023-09-23T20:03:14.320 INFO:teuthology.orchestra.run.smithi149.stdout:Transaction Summary 2023-09-23T20:03:14.320 INFO:teuthology.orchestra.run.smithi149.stdout:======================================================================================================= 2023-09-23T20:03:14.321 INFO:teuthology.orchestra.run.smithi149.stdout:Install 32 Packages 2023-09-23T20:03:14.321 INFO:teuthology.orchestra.run.smithi149.stdout: 2023-09-23T20:03:14.321 INFO:teuthology.orchestra.run.smithi149.stdout:Total download size: 28 M 2023-09-23T20:03:14.321 INFO:teuthology.orchestra.run.smithi149.stdout:Installed size: 110 M 2023-09-23T20:03:14.321 INFO:teuthology.orchestra.run.smithi149.stdout:Downloading Packages: 2023-09-23T20:03:14.668 INFO:teuthology.orchestra.run.smithi149.stdout:(1/32): ceph-17.2.6-1123.g61431f6b.el8.x86_64.r 36 kB/s | 6.4 kB 00:00 2023-09-23T20:03:14.967 INFO:teuthology.orchestra.run.smithi018.stdout:Installed products updated. 2023-09-23T20:03:14.993 INFO:teuthology.orchestra.run.smithi149.stdout:(2/32): ceph-mds-17.2.6-1123.g61431f6b.el8.x86_ 4.6 MB/s | 2.3 MB 00:00 2023-09-23T20:03:15.035 INFO:teuthology.orchestra.run.smithi149.stdout:(3/32): ceph-mgr-17.2.6-1123.g61431f6b.el8.x86_ 2.9 MB/s | 1.6 MB 00:00 2023-09-23T20:03:15.166 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:03:15.166 INFO:teuthology.orchestra.run.smithi018.stdout:Installed: 2023-09-23T20:03:15.166 INFO:teuthology.orchestra.run.smithi018.stdout: ceph-immutable-object-cache-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:03:15.166 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:03:15.166 INFO:teuthology.orchestra.run.smithi018.stdout:Complete! 2023-09-23T20:03:15.193 INFO:teuthology.orchestra.run.smithi149.stdout:(4/32): ceph-mgr-modules-core-17.2.6-1123.g6143 1.5 MB/s | 238 kB 00:00 2023-09-23T20:03:15.294 INFO:teuthology.orchestra.run.smithi149.stdout:(5/32): python3-bcrypt-3.1.6-2.el8.1.x86_64.rpm 436 kB/s | 44 kB 00:00 2023-09-23T20:03:15.309 DEBUG:teuthology.orchestra.run.smithi018:> sudo yum -y install ceph-mgr 2023-09-23T20:03:15.352 INFO:teuthology.orchestra.run.smithi149.stdout:(6/32): ceph-mon-17.2.6-1123.g61431f6b.el8.x86_ 5.7 MB/s | 3.9 MB 00:00 2023-09-23T20:03:15.403 INFO:teuthology.orchestra.run.smithi149.stdout:(7/32): python3-beautifulsoup4-4.6.3-2.el8.1.no 1.7 MB/s | 185 kB 00:00 2023-09-23T20:03:15.470 INFO:teuthology.orchestra.run.smithi149.stdout:(8/32): python3-cheroot-8.5.2-1.el8.noarch.rpm 1.4 MB/s | 173 kB 00:00 2023-09-23T20:03:15.497 INFO:teuthology.orchestra.run.smithi149.stdout:(9/32): python3-cherrypy-18.4.0-1.el8.noarch.rp 4.0 MB/s | 384 kB 00:00 2023-09-23T20:03:15.514 INFO:teuthology.orchestra.run.smithi149.stdout:(10/32): python3-jaraco-6.2-6.el8.noarch.rpm 253 kB/s | 11 kB 00:00 2023-09-23T20:03:15.539 INFO:teuthology.orchestra.run.smithi149.stdout:(11/32): python3-jaraco-functools-2.0-4.el8.noa 417 kB/s | 18 kB 00:00 2023-09-23T20:03:15.556 INFO:teuthology.orchestra.run.smithi149.stdout:(12/32): python3-logutils-0.3.5-11.el8.noarch.r 1.1 MB/s | 49 kB 00:00 2023-09-23T20:03:15.573 INFO:teuthology.orchestra.run.smithi149.stdout:(13/32): python3-more-itertools-7.2.0-3.el8.noa 1.7 MB/s | 59 kB 00:00 2023-09-23T20:03:15.640 INFO:teuthology.orchestra.run.smithi149.stdout:(14/32): python3-portend-2.6-1.el8.noarch.rpm 234 kB/s | 16 kB 00:00 2023-09-23T20:03:15.674 INFO:teuthology.orchestra.run.smithi149.stdout:(15/32): python3-pecan-1.3.2-9.el8.noarch.rpm 2.4 MB/s | 283 kB 00:00 2023-09-23T20:03:15.677 INFO:teuthology.orchestra.run.smithi018.stdout:Updating Subscription Management repositories. 2023-09-23T20:03:15.699 INFO:teuthology.orchestra.run.smithi149.stdout:(16/32): python3-simplegeneric-0.8.1-17.el8.noa 320 kB/s | 19 kB 00:00 2023-09-23T20:03:15.724 INFO:teuthology.orchestra.run.smithi149.stdout:(17/32): python3-singledispatch-3.4.0.3-18.el8. 475 kB/s | 24 kB 00:00 2023-09-23T20:03:15.749 INFO:teuthology.orchestra.run.smithi149.stdout:(18/32): python3-tempora-1.14.1-5.el8.noarch.rp 581 kB/s | 29 kB 00:00 2023-09-23T20:03:15.775 INFO:teuthology.orchestra.run.smithi149.stdout:(19/32): python3-trustme-0.6.0-4.el8.noarch.rpm 547 kB/s | 27 kB 00:00 2023-09-23T20:03:15.800 INFO:teuthology.orchestra.run.smithi149.stdout:(20/32): python3-waitress-1.4.3-1.el8.noarch.rp 5.0 MB/s | 255 kB 00:00 2023-09-23T20:03:15.825 INFO:teuthology.orchestra.run.smithi149.stdout:(21/32): python3-webob-1.8.5-1.el8.1.noarch.rpm 4.9 MB/s | 251 kB 00:00 2023-09-23T20:03:15.842 INFO:teuthology.orchestra.run.smithi149.stdout:(22/32): python3-webtest-2.0.33-1.el8.noarch.rp 2.0 MB/s | 85 kB 00:00 2023-09-23T20:03:15.867 INFO:teuthology.orchestra.run.smithi149.stdout:(23/32): python3-zc-lockfile-2.0-2.el8.noarch.r 546 kB/s | 23 kB 00:00 2023-09-23T20:03:16.051 INFO:teuthology.orchestra.run.smithi149.stdout:(24/32): ceph-osd-17.2.6-1123.g61431f6b.el8.x86 16 MB/s | 17 MB 00:01 2023-09-23T20:03:16.085 INFO:teuthology.orchestra.run.smithi149.stdout:(25/32): python3-pycparser-2.14-14.el8.noarch.r 450 kB/s | 109 kB 00:00 2023-09-23T20:03:16.119 INFO:teuthology.orchestra.run.smithi149.stdout:(26/32): python3-cryptography-3.2.1-5.el8.x86_6 2.2 MB/s | 559 kB 00:00 2023-09-23T20:03:16.144 INFO:teuthology.orchestra.run.smithi149.stdout:(27/32): python3-pytz-2017.2-9.el8.noarch.rpm 1.0 MB/s | 54 kB 00:00 2023-09-23T20:03:16.169 INFO:teuthology.orchestra.run.smithi149.stdout:(28/32): python3-markupsafe-0.23-19.el8.x86_64. 771 kB/s | 39 kB 00:00 2023-09-23T20:03:16.194 INFO:teuthology.orchestra.run.smithi149.stdout:(29/32): python3-cffi-1.11.5-5.el8.x86_64.rpm 1.7 MB/s | 238 kB 00:00 2023-09-23T20:03:16.220 INFO:teuthology.orchestra.run.smithi149.stdout:(30/32): python3-mako-1.0.6-13.el8.noarch.rpm 2.0 MB/s | 157 kB 00:00 2023-09-23T20:03:16.245 INFO:teuthology.orchestra.run.smithi149.stdout:(31/32): python3-pyOpenSSL-19.0.0-1.el8.noarch. 2.0 MB/s | 103 kB 00:00 2023-09-23T20:03:16.270 INFO:teuthology.orchestra.run.smithi149.stdout:(32/32): python3-werkzeug-0.12.2-4.el8.noarch.r 4.4 MB/s | 457 kB 00:00 2023-09-23T20:03:16.272 INFO:teuthology.orchestra.run.smithi149.stdout:-------------------------------------------------------------------------------- 2023-09-23T20:03:16.272 INFO:teuthology.orchestra.run.smithi149.stdout:Total 15 MB/s | 28 MB 00:01 2023-09-23T20:03:16.358 INFO:teuthology.orchestra.run.smithi149.stdout:Running transaction check 2023-09-23T20:03:16.393 INFO:teuthology.orchestra.run.smithi149.stdout:Transaction check succeeded. 2023-09-23T20:03:16.393 INFO:teuthology.orchestra.run.smithi149.stdout:Running transaction test 2023-09-23T20:03:16.533 INFO:teuthology.orchestra.run.smithi018.stdout:Last metadata expiration check: 0:01:54 ago on Sat 23 Sep 2023 08:01:22 PM UTC. 2023-09-23T20:03:16.598 INFO:teuthology.orchestra.run.smithi149.stdout:Transaction test succeeded. 2023-09-23T20:03:16.602 INFO:teuthology.orchestra.run.smithi149.stdout:Running transaction 2023-09-23T20:03:17.673 INFO:teuthology.orchestra.run.smithi149.stdout: Preparing : 1/1 2023-09-23T20:03:17.872 INFO:teuthology.orchestra.run.smithi149.stdout: Installing : python3-more-itertools-7.2.0-3.el8.noarch 1/32 2023-09-23T20:03:17.957 INFO:teuthology.orchestra.run.smithi018.stdout:Package ceph-mgr-2:17.2.6-1123.g61431f6b.el8.x86_64 is already installed. 2023-09-23T20:03:18.079 INFO:teuthology.orchestra.run.smithi018.stdout:Dependencies resolved. 2023-09-23T20:03:18.080 INFO:teuthology.orchestra.run.smithi018.stdout:Nothing to do. 2023-09-23T20:03:18.080 INFO:teuthology.orchestra.run.smithi018.stdout:Complete! 2023-09-23T20:03:18.183 INFO:teuthology.orchestra.run.smithi149.stdout: Installing : python3-webob-1.8.5-1.el8.1.noarch 2/32 2023-09-23T20:03:18.194 DEBUG:teuthology.orchestra.run.smithi018:> sudo yum -y install ceph-mgr-dashboard 2023-09-23T20:03:18.404 INFO:teuthology.orchestra.run.smithi149.stdout: Installing : python3-werkzeug-0.12.2-4.el8.noarch 3/32 2023-09-23T20:03:18.561 INFO:teuthology.orchestra.run.smithi018.stdout:Updating Subscription Management repositories. 2023-09-23T20:03:18.618 INFO:teuthology.orchestra.run.smithi149.stdout: Installing : python3-markupsafe-0.23-19.el8.x86_64 4/32 2023-09-23T20:03:18.882 INFO:teuthology.orchestra.run.smithi149.stdout: Installing : python3-mako-1.0.6-13.el8.noarch 5/32 2023-09-23T20:03:19.054 INFO:teuthology.orchestra.run.smithi149.stdout: Installing : python3-pytz-2017.2-9.el8.noarch 6/32 2023-09-23T20:03:19.310 INFO:teuthology.orchestra.run.smithi149.stdout: Installing : python3-pycparser-2.14-14.el8.noarch 7/32 2023-09-23T20:03:19.437 INFO:teuthology.orchestra.run.smithi018.stdout:Last metadata expiration check: 0:01:57 ago on Sat 23 Sep 2023 08:01:22 PM UTC. 2023-09-23T20:03:19.640 INFO:teuthology.orchestra.run.smithi149.stdout: Installing : python3-cffi-1.11.5-5.el8.x86_64 8/32 2023-09-23T20:03:19.903 INFO:teuthology.orchestra.run.smithi149.stdout: Installing : python3-cryptography-3.2.1-5.el8.x86_64 9/32 2023-09-23T20:03:20.066 INFO:teuthology.orchestra.run.smithi149.stdout: Installing : python3-pyOpenSSL-19.0.0-1.el8.noarch 10/32 2023-09-23T20:03:20.257 INFO:teuthology.orchestra.run.smithi149.stdout: Installing : python3-trustme-0.6.0-4.el8.noarch 11/32 2023-09-23T20:03:20.475 INFO:teuthology.orchestra.run.smithi149.stdout: Installing : python3-bcrypt-3.1.6-2.el8.1.x86_64 12/32 2023-09-23T20:03:20.684 INFO:teuthology.orchestra.run.smithi149.stdout: Installing : python3-zc-lockfile-2.0-2.el8.noarch 13/32 2023-09-23T20:03:20.882 INFO:teuthology.orchestra.run.smithi149.stdout: Installing : python3-waitress-1.4.3-1.el8.noarch 14/32 2023-09-23T20:03:20.990 INFO:teuthology.orchestra.run.smithi018.stdout:Dependencies resolved. 2023-09-23T20:03:20.991 INFO:teuthology.orchestra.run.smithi018.stdout:=================================================================================================== 2023-09-23T20:03:20.991 INFO:teuthology.orchestra.run.smithi018.stdout: Package Arch Version Repository Size 2023-09-23T20:03:20.992 INFO:teuthology.orchestra.run.smithi018.stdout:=================================================================================================== 2023-09-23T20:03:20.992 INFO:teuthology.orchestra.run.smithi018.stdout:Installing: 2023-09-23T20:03:20.992 INFO:teuthology.orchestra.run.smithi018.stdout: ceph-mgr-dashboard noarch 2:17.2.6-1123.g61431f6b.el8 ceph-noarch 1.7 M 2023-09-23T20:03:20.992 INFO:teuthology.orchestra.run.smithi018.stdout:Installing dependencies: 2023-09-23T20:03:20.992 INFO:teuthology.orchestra.run.smithi018.stdout: ceph-grafana-dashboards noarch 2:17.2.6-1123.g61431f6b.el8 ceph-noarch 23 k 2023-09-23T20:03:20.992 INFO:teuthology.orchestra.run.smithi018.stdout: ceph-prometheus-alerts noarch 2:17.2.6-1123.g61431f6b.el8 ceph-noarch 14 k 2023-09-23T20:03:20.992 INFO:teuthology.orchestra.run.smithi018.stdout: python3-jwt noarch 1.6.1-2.el8 rhel-8-for-x86_64-baseos-rpms 43 k 2023-09-23T20:03:20.993 INFO:teuthology.orchestra.run.smithi018.stdout: python3-repoze-lru noarch 0.7-6.el8 epel 34 k 2023-09-23T20:03:20.993 INFO:teuthology.orchestra.run.smithi018.stdout: python3-routes noarch 2.4.1-12.el8 epel 196 k 2023-09-23T20:03:20.993 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:03:20.993 INFO:teuthology.orchestra.run.smithi018.stdout:Transaction Summary 2023-09-23T20:03:20.993 INFO:teuthology.orchestra.run.smithi018.stdout:=================================================================================================== 2023-09-23T20:03:20.993 INFO:teuthology.orchestra.run.smithi018.stdout:Install 6 Packages 2023-09-23T20:03:20.993 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:03:20.994 INFO:teuthology.orchestra.run.smithi018.stdout:Total download size: 2.0 M 2023-09-23T20:03:20.994 INFO:teuthology.orchestra.run.smithi018.stdout:Installed size: 8.0 M 2023-09-23T20:03:20.994 INFO:teuthology.orchestra.run.smithi018.stdout:Downloading Packages: 2023-09-23T20:03:21.021 INFO:teuthology.orchestra.run.smithi149.stdout: Installing : python3-singledispatch-3.4.0.3-18.el8.noarch 15/32 2023-09-23T20:03:21.201 INFO:teuthology.orchestra.run.smithi149.stdout: Installing : python3-simplegeneric-0.8.1-17.el8.noarch 16/32 2023-09-23T20:03:21.319 INFO:teuthology.orchestra.run.smithi018.stdout:(1/6): ceph-grafana-dashboards-17.2.6-1123.g614 162 kB/s | 23 kB 00:00 2023-09-23T20:03:21.344 INFO:teuthology.orchestra.run.smithi018.stdout:(2/6): ceph-prometheus-alerts-17.2.6-1123.g6143 87 kB/s | 14 kB 00:00 2023-09-23T20:03:21.394 INFO:teuthology.orchestra.run.smithi149.stdout: Installing : python3-logutils-0.3.5-11.el8.noarch 17/32 2023-09-23T20:03:21.436 INFO:teuthology.orchestra.run.smithi018.stdout:(3/6): python3-repoze-lru-0.7-6.el8.noarch.rpm 288 kB/s | 34 kB 00:00 2023-09-23T20:03:21.495 INFO:teuthology.orchestra.run.smithi018.stdout:(4/6): ceph-mgr-dashboard-17.2.6-1123.g61431f6b 5.3 MB/s | 1.7 MB 00:00 2023-09-23T20:03:21.520 INFO:teuthology.orchestra.run.smithi018.stdout:(5/6): python3-routes-2.4.1-12.el8.noarch.rpm 1.1 MB/s | 196 kB 00:00 2023-09-23T20:03:21.522 INFO:teuthology.orchestra.run.smithi149.stdout: Installing : python3-jaraco-6.2-6.el8.noarch 18/32 2023-09-23T20:03:21.545 INFO:teuthology.orchestra.run.smithi018.stdout:(6/6): python3-jwt-1.6.1-2.el8.noarch.rpm 392 kB/s | 43 kB 00:00 2023-09-23T20:03:21.548 INFO:teuthology.orchestra.run.smithi018.stdout:-------------------------------------------------------------------------------- 2023-09-23T20:03:21.548 INFO:teuthology.orchestra.run.smithi018.stdout:Total 3.6 MB/s | 2.0 MB 00:00 2023-09-23T20:03:21.560 INFO:teuthology.orchestra.run.smithi018.stdout:Running transaction check 2023-09-23T20:03:21.570 INFO:teuthology.orchestra.run.smithi018.stdout:Transaction check succeeded. 2023-09-23T20:03:21.570 INFO:teuthology.orchestra.run.smithi018.stdout:Running transaction test 2023-09-23T20:03:21.630 INFO:teuthology.orchestra.run.smithi018.stdout:Transaction test succeeded. 2023-09-23T20:03:21.634 INFO:teuthology.orchestra.run.smithi018.stdout:Running transaction 2023-09-23T20:03:21.710 INFO:teuthology.orchestra.run.smithi149.stdout: Installing : python3-jaraco-functools-2.0-4.el8.noarch 19/32 2023-09-23T20:03:21.888 INFO:teuthology.orchestra.run.smithi018.stdout: Preparing : 1/1 2023-09-23T20:03:21.911 INFO:teuthology.orchestra.run.smithi149.stdout: Installing : python3-cheroot-8.5.2-1.el8.noarch 20/32 2023-09-23T20:03:22.041 INFO:teuthology.orchestra.run.smithi018.stdout: Installing : python3-jwt-1.6.1-2.el8.noarch 1/6 2023-09-23T20:03:22.089 INFO:teuthology.orchestra.run.smithi149.stdout: Installing : python3-tempora-1.14.1-5.el8.noarch 21/32 2023-09-23T20:03:22.223 INFO:teuthology.orchestra.run.smithi018.stdout: Installing : python3-repoze-lru-0.7-6.el8.noarch 2/6 2023-09-23T20:03:22.361 INFO:teuthology.orchestra.run.smithi149.stdout: Installing : python3-portend-2.6-1.el8.noarch 22/32 2023-09-23T20:03:22.384 INFO:teuthology.orchestra.run.smithi018.stdout: Installing : python3-routes-2.4.1-12.el8.noarch 3/6 2023-09-23T20:03:22.514 INFO:teuthology.orchestra.run.smithi018.stdout: Installing : ceph-prometheus-alerts-2:17.2.6-1123.g61431f6b.el8.n 4/6 2023-09-23T20:03:22.621 INFO:teuthology.orchestra.run.smithi149.stdout: Installing : python3-cherrypy-18.4.0-1.el8.noarch 23/32 2023-09-23T20:03:22.791 INFO:teuthology.orchestra.run.smithi149.stdout: Installing : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 24/32 2023-09-23T20:03:22.893 INFO:teuthology.orchestra.run.smithi018.stdout: Installing : ceph-grafana-dashboards-2:17.2.6-1123.g61431f6b.el8. 5/6 2023-09-23T20:03:23.058 INFO:teuthology.orchestra.run.smithi149.stdout: Installing : python3-webtest-2.0.33-1.el8.noarch 25/32 2023-09-23T20:03:23.063 INFO:teuthology.orchestra.run.smithi018.stdout: Installing : ceph-mgr-dashboard-2:17.2.6-1123.g61431f6b.el8.noarc 6/6 2023-09-23T20:03:23.312 INFO:teuthology.orchestra.run.smithi018.stdout: Running scriptlet: ceph-mgr-dashboard-2:17.2.6-1123.g61431f6b.el8.noarc 6/6 2023-09-23T20:03:23.312 INFO:teuthology.orchestra.run.smithi018.stdout: Verifying : ceph-grafana-dashboards-2:17.2.6-1123.g61431f6b.el8. 1/6 2023-09-23T20:03:23.312 INFO:teuthology.orchestra.run.smithi018.stdout: Verifying : ceph-mgr-dashboard-2:17.2.6-1123.g61431f6b.el8.noarc 2/6 2023-09-23T20:03:23.312 INFO:teuthology.orchestra.run.smithi018.stdout: Verifying : ceph-prometheus-alerts-2:17.2.6-1123.g61431f6b.el8.n 3/6 2023-09-23T20:03:23.312 INFO:teuthology.orchestra.run.smithi018.stdout: Verifying : python3-repoze-lru-0.7-6.el8.noarch 4/6 2023-09-23T20:03:23.312 INFO:teuthology.orchestra.run.smithi018.stdout: Verifying : python3-routes-2.4.1-12.el8.noarch 5/6 2023-09-23T20:03:23.363 INFO:teuthology.orchestra.run.smithi149.stdout: Installing : python3-pecan-1.3.2-9.el8.noarch 26/32 2023-09-23T20:03:24.378 INFO:teuthology.orchestra.run.smithi018.stdout: Verifying : python3-jwt-1.6.1-2.el8.noarch 6/6 2023-09-23T20:03:24.378 INFO:teuthology.orchestra.run.smithi018.stdout:Last metadata expiration check: 0:02:02 ago on Sat 23 Sep 2023 08:01:22 PM UTC. 2023-09-23T20:03:24.392 INFO:teuthology.orchestra.run.smithi149.stdout: Installing : ceph-mgr-modules-core-2:17.2.6-1123.g61431f6b.el8. 27/32 2023-09-23T20:03:24.543 INFO:teuthology.orchestra.run.smithi149.stdout: Installing : ceph-mgr-2:17.2.6-1123.g61431f6b.el8.x86_64 28/32 2023-09-23T20:03:26.018 INFO:teuthology.orchestra.run.smithi149.stdout: Running scriptlet: ceph-mgr-2:17.2.6-1123.g61431f6b.el8.x86_64 28/32 2023-09-23T20:03:26.242 INFO:teuthology.orchestra.run.smithi149.stdout: Installing : ceph-osd-2:17.2.6-1123.g61431f6b.el8.x86_64 29/32 2023-09-23T20:03:26.635 INFO:teuthology.orchestra.run.smithi149.stdout: Running scriptlet: ceph-osd-2:17.2.6-1123.g61431f6b.el8.x86_64 29/32 2023-09-23T20:03:26.774 INFO:teuthology.orchestra.run.smithi149.stdout: Installing : ceph-mon-2:17.2.6-1123.g61431f6b.el8.x86_64 30/32 2023-09-23T20:03:26.852 INFO:teuthology.orchestra.run.smithi018.stdout:Installed products updated. 2023-09-23T20:03:26.990 INFO:teuthology.orchestra.run.smithi149.stdout: Running scriptlet: ceph-mon-2:17.2.6-1123.g61431f6b.el8.x86_64 30/32 2023-09-23T20:03:27.052 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:03:27.053 INFO:teuthology.orchestra.run.smithi018.stdout:Installed: 2023-09-23T20:03:27.053 INFO:teuthology.orchestra.run.smithi018.stdout: ceph-grafana-dashboards-2:17.2.6-1123.g61431f6b.el8.noarch 2023-09-23T20:03:27.053 INFO:teuthology.orchestra.run.smithi018.stdout: ceph-mgr-dashboard-2:17.2.6-1123.g61431f6b.el8.noarch 2023-09-23T20:03:27.053 INFO:teuthology.orchestra.run.smithi018.stdout: ceph-prometheus-alerts-2:17.2.6-1123.g61431f6b.el8.noarch 2023-09-23T20:03:27.053 INFO:teuthology.orchestra.run.smithi018.stdout: python3-jwt-1.6.1-2.el8.noarch 2023-09-23T20:03:27.053 INFO:teuthology.orchestra.run.smithi018.stdout: python3-repoze-lru-0.7-6.el8.noarch 2023-09-23T20:03:27.053 INFO:teuthology.orchestra.run.smithi018.stdout: python3-routes-2.4.1-12.el8.noarch 2023-09-23T20:03:27.053 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:03:27.053 INFO:teuthology.orchestra.run.smithi018.stdout:Complete! 2023-09-23T20:03:27.174 INFO:teuthology.orchestra.run.smithi149.stdout: Installing : ceph-mds-2:17.2.6-1123.g61431f6b.el8.x86_64 31/32 2023-09-23T20:03:27.192 INFO:teuthology.orchestra.run.smithi149.stdout: Running scriptlet: ceph-mds-2:17.2.6-1123.g61431f6b.el8.x86_64 31/32 2023-09-23T20:03:27.230 DEBUG:teuthology.orchestra.run.smithi018:> sudo yum -y install ceph-mgr-diskprediction-local 2023-09-23T20:03:27.322 INFO:teuthology.orchestra.run.smithi149.stdout: Installing : ceph-2:17.2.6-1123.g61431f6b.el8.x86_64 32/32 2023-09-23T20:03:27.599 INFO:teuthology.orchestra.run.smithi018.stdout:Updating Subscription Management repositories. 2023-09-23T20:03:27.686 INFO:teuthology.orchestra.run.smithi149.stdout: Running scriptlet: ceph-2:17.2.6-1123.g61431f6b.el8.x86_64 32/32 2023-09-23T20:03:27.686 INFO:teuthology.orchestra.run.smithi149.stdout: Verifying : ceph-2:17.2.6-1123.g61431f6b.el8.x86_64 1/32 2023-09-23T20:03:27.686 INFO:teuthology.orchestra.run.smithi149.stdout: Verifying : ceph-mds-2:17.2.6-1123.g61431f6b.el8.x86_64 2/32 2023-09-23T20:03:27.687 INFO:teuthology.orchestra.run.smithi149.stdout: Verifying : ceph-mgr-2:17.2.6-1123.g61431f6b.el8.x86_64 3/32 2023-09-23T20:03:27.687 INFO:teuthology.orchestra.run.smithi149.stdout: Verifying : ceph-mon-2:17.2.6-1123.g61431f6b.el8.x86_64 4/32 2023-09-23T20:03:27.687 INFO:teuthology.orchestra.run.smithi149.stdout: Verifying : ceph-osd-2:17.2.6-1123.g61431f6b.el8.x86_64 5/32 2023-09-23T20:03:27.687 INFO:teuthology.orchestra.run.smithi149.stdout: Verifying : ceph-mgr-modules-core-2:17.2.6-1123.g61431f6b.el8. 6/32 2023-09-23T20:03:27.687 INFO:teuthology.orchestra.run.smithi149.stdout: Verifying : python3-bcrypt-3.1.6-2.el8.1.x86_64 7/32 2023-09-23T20:03:27.687 INFO:teuthology.orchestra.run.smithi149.stdout: Verifying : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 8/32 2023-09-23T20:03:27.687 INFO:teuthology.orchestra.run.smithi149.stdout: Verifying : python3-cheroot-8.5.2-1.el8.noarch 9/32 2023-09-23T20:03:27.687 INFO:teuthology.orchestra.run.smithi149.stdout: Verifying : python3-cherrypy-18.4.0-1.el8.noarch 10/32 2023-09-23T20:03:27.687 INFO:teuthology.orchestra.run.smithi149.stdout: Verifying : python3-jaraco-6.2-6.el8.noarch 11/32 2023-09-23T20:03:27.688 INFO:teuthology.orchestra.run.smithi149.stdout: Verifying : python3-jaraco-functools-2.0-4.el8.noarch 12/32 2023-09-23T20:03:27.688 INFO:teuthology.orchestra.run.smithi149.stdout: Verifying : python3-logutils-0.3.5-11.el8.noarch 13/32 2023-09-23T20:03:27.688 INFO:teuthology.orchestra.run.smithi149.stdout: Verifying : python3-more-itertools-7.2.0-3.el8.noarch 14/32 2023-09-23T20:03:27.688 INFO:teuthology.orchestra.run.smithi149.stdout: Verifying : python3-pecan-1.3.2-9.el8.noarch 15/32 2023-09-23T20:03:27.688 INFO:teuthology.orchestra.run.smithi149.stdout: Verifying : python3-portend-2.6-1.el8.noarch 16/32 2023-09-23T20:03:27.688 INFO:teuthology.orchestra.run.smithi149.stdout: Verifying : python3-simplegeneric-0.8.1-17.el8.noarch 17/32 2023-09-23T20:03:27.688 INFO:teuthology.orchestra.run.smithi149.stdout: Verifying : python3-singledispatch-3.4.0.3-18.el8.noarch 18/32 2023-09-23T20:03:27.688 INFO:teuthology.orchestra.run.smithi149.stdout: Verifying : python3-tempora-1.14.1-5.el8.noarch 19/32 2023-09-23T20:03:27.689 INFO:teuthology.orchestra.run.smithi149.stdout: Verifying : python3-trustme-0.6.0-4.el8.noarch 20/32 2023-09-23T20:03:27.689 INFO:teuthology.orchestra.run.smithi149.stdout: Verifying : python3-waitress-1.4.3-1.el8.noarch 21/32 2023-09-23T20:03:27.689 INFO:teuthology.orchestra.run.smithi149.stdout: Verifying : python3-webob-1.8.5-1.el8.1.noarch 22/32 2023-09-23T20:03:27.689 INFO:teuthology.orchestra.run.smithi149.stdout: Verifying : python3-webtest-2.0.33-1.el8.noarch 23/32 2023-09-23T20:03:27.689 INFO:teuthology.orchestra.run.smithi149.stdout: Verifying : python3-zc-lockfile-2.0-2.el8.noarch 24/32 2023-09-23T20:03:27.689 INFO:teuthology.orchestra.run.smithi149.stdout: Verifying : python3-pycparser-2.14-14.el8.noarch 25/32 2023-09-23T20:03:27.689 INFO:teuthology.orchestra.run.smithi149.stdout: Verifying : python3-cryptography-3.2.1-5.el8.x86_64 26/32 2023-09-23T20:03:27.689 INFO:teuthology.orchestra.run.smithi149.stdout: Verifying : python3-cffi-1.11.5-5.el8.x86_64 27/32 2023-09-23T20:03:27.689 INFO:teuthology.orchestra.run.smithi149.stdout: Verifying : python3-pytz-2017.2-9.el8.noarch 28/32 2023-09-23T20:03:27.690 INFO:teuthology.orchestra.run.smithi149.stdout: Verifying : python3-markupsafe-0.23-19.el8.x86_64 29/32 2023-09-23T20:03:27.690 INFO:teuthology.orchestra.run.smithi149.stdout: Verifying : python3-mako-1.0.6-13.el8.noarch 30/32 2023-09-23T20:03:27.690 INFO:teuthology.orchestra.run.smithi149.stdout: Verifying : python3-werkzeug-0.12.2-4.el8.noarch 31/32 2023-09-23T20:03:28.435 INFO:teuthology.orchestra.run.smithi018.stdout:Last metadata expiration check: 0:02:06 ago on Sat 23 Sep 2023 08:01:22 PM UTC. 2023-09-23T20:03:29.163 INFO:teuthology.orchestra.run.smithi149.stdout: Verifying : python3-pyOpenSSL-19.0.0-1.el8.noarch 32/32 2023-09-23T20:03:29.163 INFO:teuthology.orchestra.run.smithi149.stdout:Last metadata expiration check: 0:01:27 ago on Sat 23 Sep 2023 08:02:02 PM UTC. 2023-09-23T20:03:29.993 INFO:teuthology.orchestra.run.smithi018.stdout:Dependencies resolved. 2023-09-23T20:03:29.994 INFO:teuthology.orchestra.run.smithi018.stdout:======================================================================================================================= 2023-09-23T20:03:29.994 INFO:teuthology.orchestra.run.smithi018.stdout: Package Arch Version Repository Size 2023-09-23T20:03:29.994 INFO:teuthology.orchestra.run.smithi018.stdout:======================================================================================================================= 2023-09-23T20:03:29.994 INFO:teuthology.orchestra.run.smithi018.stdout:Installing: 2023-09-23T20:03:29.994 INFO:teuthology.orchestra.run.smithi018.stdout: ceph-mgr-diskprediction-local noarch 2:17.2.6-1123.g61431f6b.el8 ceph-noarch 7.4 M 2023-09-23T20:03:29.994 INFO:teuthology.orchestra.run.smithi018.stdout:Installing dependencies: 2023-09-23T20:03:29.995 INFO:teuthology.orchestra.run.smithi018.stdout: libgfortran x86_64 8.5.0-10.1.el8_6 rhel-8-for-x86_64-baseos-rpms 644 k 2023-09-23T20:03:29.995 INFO:teuthology.orchestra.run.smithi018.stdout: libquadmath x86_64 8.5.0-10.1.el8_6 rhel-8-for-x86_64-baseos-rpms 171 k 2023-09-23T20:03:29.995 INFO:teuthology.orchestra.run.smithi018.stdout: openblas x86_64 0.3.15-3.el8 rhel-8-for-x86_64-appstream-rpms 4.7 M 2023-09-23T20:03:29.995 INFO:teuthology.orchestra.run.smithi018.stdout: openblas-threads x86_64 0.3.15-3.el8 rhel-8-for-x86_64-appstream-rpms 4.9 M 2023-09-23T20:03:29.995 INFO:teuthology.orchestra.run.smithi018.stdout: python3-numpy x86_64 1:1.14.3-10.el8 rhel-8-for-x86_64-appstream-rpms 3.7 M 2023-09-23T20:03:29.995 INFO:teuthology.orchestra.run.smithi018.stdout: python3-numpy-f2py x86_64 1:1.14.3-10.el8 rhel-8-for-x86_64-appstream-rpms 225 k 2023-09-23T20:03:29.995 INFO:teuthology.orchestra.run.smithi018.stdout: python3-scipy x86_64 1.0.0-21.module+el8.5.0+10916+41bd434d rhel-8-for-x86_64-appstream-rpms 14 M 2023-09-23T20:03:29.995 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:03:29.995 INFO:teuthology.orchestra.run.smithi018.stdout:Transaction Summary 2023-09-23T20:03:29.995 INFO:teuthology.orchestra.run.smithi018.stdout:======================================================================================================================= 2023-09-23T20:03:29.995 INFO:teuthology.orchestra.run.smithi018.stdout:Install 8 Packages 2023-09-23T20:03:29.996 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:03:29.996 INFO:teuthology.orchestra.run.smithi018.stdout:Total download size: 35 M 2023-09-23T20:03:29.996 INFO:teuthology.orchestra.run.smithi018.stdout:Installed size: 210 M 2023-09-23T20:03:29.996 INFO:teuthology.orchestra.run.smithi018.stdout:Downloading Packages: 2023-09-23T20:03:30.059 INFO:teuthology.orchestra.run.smithi018.stdout:(1/8): libquadmath-8.5.0-10.1.el8_6.x86_64.rpm 2.7 MB/s | 171 kB 00:00 2023-09-23T20:03:30.093 INFO:teuthology.orchestra.run.smithi018.stdout:(2/8): libgfortran-8.5.0-10.1.el8_6.x86_64.rpm 6.6 MB/s | 644 kB 00:00 2023-09-23T20:03:30.210 INFO:teuthology.orchestra.run.smithi018.stdout:(3/8): python3-numpy-1.14.3-10.el8.x86_64.rpm 32 MB/s | 3.7 MB 00:00 2023-09-23T20:03:30.252 INFO:teuthology.orchestra.run.smithi018.stdout:(4/8): python3-numpy-f2py-1.14.3-10.el8.x86_64. 5.3 MB/s | 225 kB 00:00 2023-09-23T20:03:30.485 INFO:teuthology.orchestra.run.smithi018.stdout:(5/8): python3-scipy-1.0.0-21.module+el8.5.0+10 32 MB/s | 14 MB 00:00 2023-09-23T20:03:30.552 INFO:teuthology.orchestra.run.smithi018.stdout:(6/8): openblas-threads-0.3.15-3.el8.x86_64.rpm 16 MB/s | 4.9 MB 00:00 2023-09-23T20:03:30.653 INFO:teuthology.orchestra.run.smithi018.stdout:(7/8): openblas-0.3.15-3.el8.x86_64.rpm 28 MB/s | 4.7 MB 00:00 2023-09-23T20:03:30.886 INFO:teuthology.orchestra.run.smithi018.stdout:(8/8): ceph-mgr-diskprediction-local-17.2.6-112 8.3 MB/s | 7.4 MB 00:00 2023-09-23T20:03:30.887 INFO:teuthology.orchestra.run.smithi018.stdout:-------------------------------------------------------------------------------- 2023-09-23T20:03:30.887 INFO:teuthology.orchestra.run.smithi018.stdout:Total 40 MB/s | 35 MB 00:00 2023-09-23T20:03:31.133 INFO:teuthology.orchestra.run.smithi018.stdout:Running transaction check 2023-09-23T20:03:31.152 INFO:teuthology.orchestra.run.smithi018.stdout:Transaction check succeeded. 2023-09-23T20:03:31.152 INFO:teuthology.orchestra.run.smithi018.stdout:Running transaction test 2023-09-23T20:03:31.463 INFO:teuthology.orchestra.run.smithi018.stdout:Transaction test succeeded. 2023-09-23T20:03:31.467 INFO:teuthology.orchestra.run.smithi018.stdout:Running transaction 2023-09-23T20:03:31.751 INFO:teuthology.orchestra.run.smithi149.stdout:Installed products updated. 2023-09-23T20:03:31.939 INFO:teuthology.orchestra.run.smithi149.stdout: 2023-09-23T20:03:31.939 INFO:teuthology.orchestra.run.smithi149.stdout:Installed: 2023-09-23T20:03:31.939 INFO:teuthology.orchestra.run.smithi149.stdout: ceph-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:03:31.939 INFO:teuthology.orchestra.run.smithi149.stdout: ceph-mds-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:03:31.939 INFO:teuthology.orchestra.run.smithi149.stdout: ceph-mgr-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:03:31.939 INFO:teuthology.orchestra.run.smithi149.stdout: ceph-mgr-modules-core-2:17.2.6-1123.g61431f6b.el8.noarch 2023-09-23T20:03:31.939 INFO:teuthology.orchestra.run.smithi149.stdout: ceph-mon-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:03:31.940 INFO:teuthology.orchestra.run.smithi149.stdout: ceph-osd-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:03:31.940 INFO:teuthology.orchestra.run.smithi149.stdout: python3-bcrypt-3.1.6-2.el8.1.x86_64 2023-09-23T20:03:31.940 INFO:teuthology.orchestra.run.smithi149.stdout: python3-beautifulsoup4-4.6.3-2.el8.1.noarch 2023-09-23T20:03:31.940 INFO:teuthology.orchestra.run.smithi149.stdout: python3-cffi-1.11.5-5.el8.x86_64 2023-09-23T20:03:31.940 INFO:teuthology.orchestra.run.smithi149.stdout: python3-cheroot-8.5.2-1.el8.noarch 2023-09-23T20:03:31.940 INFO:teuthology.orchestra.run.smithi149.stdout: python3-cherrypy-18.4.0-1.el8.noarch 2023-09-23T20:03:31.940 INFO:teuthology.orchestra.run.smithi149.stdout: python3-cryptography-3.2.1-5.el8.x86_64 2023-09-23T20:03:31.940 INFO:teuthology.orchestra.run.smithi149.stdout: python3-jaraco-6.2-6.el8.noarch 2023-09-23T20:03:31.940 INFO:teuthology.orchestra.run.smithi149.stdout: python3-jaraco-functools-2.0-4.el8.noarch 2023-09-23T20:03:31.940 INFO:teuthology.orchestra.run.smithi149.stdout: python3-logutils-0.3.5-11.el8.noarch 2023-09-23T20:03:31.941 INFO:teuthology.orchestra.run.smithi149.stdout: python3-mako-1.0.6-13.el8.noarch 2023-09-23T20:03:31.941 INFO:teuthology.orchestra.run.smithi149.stdout: python3-markupsafe-0.23-19.el8.x86_64 2023-09-23T20:03:31.941 INFO:teuthology.orchestra.run.smithi149.stdout: python3-more-itertools-7.2.0-3.el8.noarch 2023-09-23T20:03:31.941 INFO:teuthology.orchestra.run.smithi149.stdout: python3-pecan-1.3.2-9.el8.noarch 2023-09-23T20:03:31.941 INFO:teuthology.orchestra.run.smithi149.stdout: python3-portend-2.6-1.el8.noarch 2023-09-23T20:03:31.941 INFO:teuthology.orchestra.run.smithi149.stdout: python3-pyOpenSSL-19.0.0-1.el8.noarch 2023-09-23T20:03:31.941 INFO:teuthology.orchestra.run.smithi149.stdout: python3-pycparser-2.14-14.el8.noarch 2023-09-23T20:03:31.941 INFO:teuthology.orchestra.run.smithi149.stdout: python3-pytz-2017.2-9.el8.noarch 2023-09-23T20:03:31.941 INFO:teuthology.orchestra.run.smithi149.stdout: python3-simplegeneric-0.8.1-17.el8.noarch 2023-09-23T20:03:31.942 INFO:teuthology.orchestra.run.smithi149.stdout: python3-singledispatch-3.4.0.3-18.el8.noarch 2023-09-23T20:03:31.942 INFO:teuthology.orchestra.run.smithi149.stdout: python3-tempora-1.14.1-5.el8.noarch 2023-09-23T20:03:31.942 INFO:teuthology.orchestra.run.smithi149.stdout: python3-trustme-0.6.0-4.el8.noarch 2023-09-23T20:03:31.942 INFO:teuthology.orchestra.run.smithi149.stdout: python3-waitress-1.4.3-1.el8.noarch 2023-09-23T20:03:31.942 INFO:teuthology.orchestra.run.smithi149.stdout: python3-webob-1.8.5-1.el8.1.noarch 2023-09-23T20:03:31.942 INFO:teuthology.orchestra.run.smithi149.stdout: python3-webtest-2.0.33-1.el8.noarch 2023-09-23T20:03:31.942 INFO:teuthology.orchestra.run.smithi149.stdout: python3-werkzeug-0.12.2-4.el8.noarch 2023-09-23T20:03:31.942 INFO:teuthology.orchestra.run.smithi149.stdout: python3-zc-lockfile-2.0-2.el8.noarch 2023-09-23T20:03:31.942 INFO:teuthology.orchestra.run.smithi149.stdout: 2023-09-23T20:03:31.943 INFO:teuthology.orchestra.run.smithi149.stdout:Complete! 2023-09-23T20:03:32.014 INFO:teuthology.orchestra.run.smithi018.stdout: Preparing : 1/1 2023-09-23T20:03:32.091 DEBUG:teuthology.orchestra.run.smithi149:> sudo yum -y install ceph-base 2023-09-23T20:03:32.146 INFO:teuthology.orchestra.run.smithi018.stdout: Installing : libquadmath-8.5.0-10.1.el8_6.x86_64 1/8 2023-09-23T20:03:32.289 INFO:teuthology.orchestra.run.smithi018.stdout: Running scriptlet: libquadmath-8.5.0-10.1.el8_6.x86_64 1/8 2023-09-23T20:03:32.421 INFO:teuthology.orchestra.run.smithi018.stdout: Installing : libgfortran-8.5.0-10.1.el8_6.x86_64 2/8 2023-09-23T20:03:32.441 INFO:teuthology.orchestra.run.smithi149.stdout:Updating Subscription Management repositories. 2023-09-23T20:03:33.077 INFO:teuthology.orchestra.run.smithi018.stdout: Running scriptlet: libgfortran-8.5.0-10.1.el8_6.x86_64 2/8 2023-09-23T20:03:33.238 INFO:teuthology.orchestra.run.smithi018.stdout: Installing : openblas-threads-0.3.15-3.el8.x86_64 3/8 2023-09-23T20:03:33.271 INFO:teuthology.orchestra.run.smithi149.stdout:Last metadata expiration check: 0:01:31 ago on Sat 23 Sep 2023 08:02:02 PM UTC. 2023-09-23T20:03:33.875 INFO:teuthology.orchestra.run.smithi018.stdout: Running scriptlet: openblas-threads-0.3.15-3.el8.x86_64 3/8 2023-09-23T20:03:34.054 INFO:teuthology.orchestra.run.smithi018.stdout: Installing : openblas-0.3.15-3.el8.x86_64 4/8 2023-09-23T20:03:34.568 INFO:teuthology.orchestra.run.smithi149.stdout:Package ceph-base-2:17.2.6-1123.g61431f6b.el8.x86_64 is already installed. 2023-09-23T20:03:34.678 INFO:teuthology.orchestra.run.smithi149.stdout:Dependencies resolved. 2023-09-23T20:03:34.679 INFO:teuthology.orchestra.run.smithi149.stdout:Nothing to do. 2023-09-23T20:03:34.680 INFO:teuthology.orchestra.run.smithi149.stdout:Complete! 2023-09-23T20:03:34.788 INFO:teuthology.orchestra.run.smithi018.stdout: Running scriptlet: openblas-0.3.15-3.el8.x86_64 4/8 2023-09-23T20:03:35.051 DEBUG:teuthology.orchestra.run.smithi149:> sudo yum -y install cephadm 2023-09-23T20:03:35.096 INFO:teuthology.orchestra.run.smithi018.stdout: Installing : python3-numpy-1:1.14.3-10.el8.x86_64 5/8 2023-09-23T20:03:35.312 INFO:teuthology.orchestra.run.smithi018.stdout: Installing : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-09-23T20:03:35.402 INFO:teuthology.orchestra.run.smithi149.stdout:Updating Subscription Management repositories. 2023-09-23T20:03:36.255 INFO:teuthology.orchestra.run.smithi149.stdout:Last metadata expiration check: 0:01:34 ago on Sat 23 Sep 2023 08:02:02 PM UTC. 2023-09-23T20:03:37.271 INFO:teuthology.orchestra.run.smithi018.stdout: Running scriptlet: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-09-23T20:03:37.713 INFO:teuthology.orchestra.run.smithi149.stdout:Dependencies resolved. 2023-09-23T20:03:37.714 INFO:teuthology.orchestra.run.smithi149.stdout:================================================================================ 2023-09-23T20:03:37.714 INFO:teuthology.orchestra.run.smithi149.stdout: Package Arch Version Repository Size 2023-09-23T20:03:37.714 INFO:teuthology.orchestra.run.smithi149.stdout:================================================================================ 2023-09-23T20:03:37.714 INFO:teuthology.orchestra.run.smithi149.stdout:Installing: 2023-09-23T20:03:37.714 INFO:teuthology.orchestra.run.smithi149.stdout: cephadm noarch 2:17.2.6-1123.g61431f6b.el8 ceph-noarch 80 k 2023-09-23T20:03:37.714 INFO:teuthology.orchestra.run.smithi149.stdout: 2023-09-23T20:03:37.714 INFO:teuthology.orchestra.run.smithi149.stdout:Transaction Summary 2023-09-23T20:03:37.714 INFO:teuthology.orchestra.run.smithi149.stdout:================================================================================ 2023-09-23T20:03:37.715 INFO:teuthology.orchestra.run.smithi149.stdout:Install 1 Package 2023-09-23T20:03:37.715 INFO:teuthology.orchestra.run.smithi149.stdout: 2023-09-23T20:03:37.715 INFO:teuthology.orchestra.run.smithi149.stdout:Total download size: 80 k 2023-09-23T20:03:37.715 INFO:teuthology.orchestra.run.smithi149.stdout:Installed size: 363 k 2023-09-23T20:03:37.715 INFO:teuthology.orchestra.run.smithi149.stdout:Downloading Packages: 2023-09-23T20:03:37.875 INFO:teuthology.orchestra.run.smithi149.stdout:cephadm-17.2.6-1123.g61431f6b.el8.noarch.rpm 502 kB/s | 80 kB 00:00 2023-09-23T20:03:37.876 INFO:teuthology.orchestra.run.smithi149.stdout:-------------------------------------------------------------------------------- 2023-09-23T20:03:37.876 INFO:teuthology.orchestra.run.smithi149.stdout:Total 498 kB/s | 80 kB 00:00 2023-09-23T20:03:37.876 INFO:teuthology.orchestra.run.smithi149.stdout:Running transaction check 2023-09-23T20:03:37.884 INFO:teuthology.orchestra.run.smithi149.stdout:Transaction check succeeded. 2023-09-23T20:03:37.885 INFO:teuthology.orchestra.run.smithi149.stdout:Running transaction test 2023-09-23T20:03:37.886 INFO:teuthology.orchestra.run.smithi149.stdout:Transaction test succeeded. 2023-09-23T20:03:37.890 INFO:teuthology.orchestra.run.smithi149.stdout:Running transaction 2023-09-23T20:03:37.935 INFO:teuthology.orchestra.run.smithi149.stdout: Preparing : 1/1 2023-09-23T20:03:38.581 INFO:teuthology.orchestra.run.smithi018.stdout: Installing : python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd434d 7/8 2023-09-23T20:03:38.728 INFO:teuthology.orchestra.run.smithi018.stdout: Installing : ceph-mgr-diskprediction-local-2:17.2.6-1123.g61431f6 8/8 2023-09-23T20:03:38.995 INFO:teuthology.orchestra.run.smithi018.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:17.2.6-1123.g61431f6 8/8 2023-09-23T20:03:38.996 INFO:teuthology.orchestra.run.smithi018.stdout: Verifying : ceph-mgr-diskprediction-local-2:17.2.6-1123.g61431f6 1/8 2023-09-23T20:03:38.996 INFO:teuthology.orchestra.run.smithi018.stdout: Verifying : libgfortran-8.5.0-10.1.el8_6.x86_64 2/8 2023-09-23T20:03:38.996 INFO:teuthology.orchestra.run.smithi018.stdout: Verifying : libquadmath-8.5.0-10.1.el8_6.x86_64 3/8 2023-09-23T20:03:38.996 INFO:teuthology.orchestra.run.smithi018.stdout: Verifying : python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd434d 4/8 2023-09-23T20:03:38.996 INFO:teuthology.orchestra.run.smithi018.stdout: Verifying : python3-numpy-1:1.14.3-10.el8.x86_64 5/8 2023-09-23T20:03:38.996 INFO:teuthology.orchestra.run.smithi018.stdout: Verifying : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-09-23T20:03:38.996 INFO:teuthology.orchestra.run.smithi018.stdout: Verifying : openblas-threads-0.3.15-3.el8.x86_64 7/8 2023-09-23T20:03:39.610 INFO:teuthology.orchestra.run.smithi149.stdout: Running scriptlet: cephadm-2:17.2.6-1123.g61431f6b.el8.noarch 1/1 2023-09-23T20:03:39.767 INFO:teuthology.orchestra.run.smithi149.stdout: Installing : cephadm-2:17.2.6-1123.g61431f6b.el8.noarch 1/1 2023-09-23T20:03:39.940 INFO:teuthology.orchestra.run.smithi149.stdout: Running scriptlet: cephadm-2:17.2.6-1123.g61431f6b.el8.noarch 1/1 2023-09-23T20:03:40.135 INFO:teuthology.orchestra.run.smithi018.stdout: Verifying : openblas-0.3.15-3.el8.x86_64 8/8 2023-09-23T20:03:40.135 INFO:teuthology.orchestra.run.smithi018.stdout:Last metadata expiration check: 0:02:18 ago on Sat 23 Sep 2023 08:01:22 PM UTC. 2023-09-23T20:03:41.015 INFO:teuthology.orchestra.run.smithi149.stdout: Verifying : cephadm-2:17.2.6-1123.g61431f6b.el8.noarch 1/1 2023-09-23T20:03:41.015 INFO:teuthology.orchestra.run.smithi149.stdout:Last metadata expiration check: 0:01:39 ago on Sat 23 Sep 2023 08:02:02 PM UTC. 2023-09-23T20:03:42.907 INFO:teuthology.orchestra.run.smithi018.stdout:Installed products updated. 2023-09-23T20:03:43.116 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:03:43.116 INFO:teuthology.orchestra.run.smithi018.stdout:Installed: 2023-09-23T20:03:43.116 INFO:teuthology.orchestra.run.smithi018.stdout: ceph-mgr-diskprediction-local-2:17.2.6-1123.g61431f6b.el8.noarch 2023-09-23T20:03:43.116 INFO:teuthology.orchestra.run.smithi018.stdout: libgfortran-8.5.0-10.1.el8_6.x86_64 2023-09-23T20:03:43.116 INFO:teuthology.orchestra.run.smithi018.stdout: libquadmath-8.5.0-10.1.el8_6.x86_64 2023-09-23T20:03:43.116 INFO:teuthology.orchestra.run.smithi018.stdout: openblas-0.3.15-3.el8.x86_64 2023-09-23T20:03:43.117 INFO:teuthology.orchestra.run.smithi018.stdout: openblas-threads-0.3.15-3.el8.x86_64 2023-09-23T20:03:43.117 INFO:teuthology.orchestra.run.smithi018.stdout: python3-numpy-1:1.14.3-10.el8.x86_64 2023-09-23T20:03:43.117 INFO:teuthology.orchestra.run.smithi018.stdout: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 2023-09-23T20:03:43.117 INFO:teuthology.orchestra.run.smithi018.stdout: python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd434d.x86_64 2023-09-23T20:03:43.117 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:03:43.117 INFO:teuthology.orchestra.run.smithi018.stdout:Complete! 2023-09-23T20:03:43.274 DEBUG:teuthology.orchestra.run.smithi018:> sudo yum -y install ceph-mgr-rook 2023-09-23T20:03:43.549 INFO:teuthology.orchestra.run.smithi149.stdout:Installed products updated. 2023-09-23T20:03:43.642 INFO:teuthology.orchestra.run.smithi018.stdout:Updating Subscription Management repositories. 2023-09-23T20:03:43.730 INFO:teuthology.orchestra.run.smithi149.stdout: 2023-09-23T20:03:43.730 INFO:teuthology.orchestra.run.smithi149.stdout:Installed: 2023-09-23T20:03:43.730 INFO:teuthology.orchestra.run.smithi149.stdout: cephadm-2:17.2.6-1123.g61431f6b.el8.noarch 2023-09-23T20:03:43.730 INFO:teuthology.orchestra.run.smithi149.stdout: 2023-09-23T20:03:43.730 INFO:teuthology.orchestra.run.smithi149.stdout:Complete! 2023-09-23T20:03:43.878 DEBUG:teuthology.orchestra.run.smithi149:> sudo yum -y install ceph-immutable-object-cache 2023-09-23T20:03:44.238 INFO:teuthology.orchestra.run.smithi149.stdout:Updating Subscription Management repositories. 2023-09-23T20:03:44.541 INFO:teuthology.orchestra.run.smithi018.stdout:Last metadata expiration check: 0:02:22 ago on Sat 23 Sep 2023 08:01:22 PM UTC. 2023-09-23T20:03:45.077 INFO:teuthology.orchestra.run.smithi149.stdout:Last metadata expiration check: 0:01:43 ago on Sat 23 Sep 2023 08:02:02 PM UTC. 2023-09-23T20:03:46.095 INFO:teuthology.orchestra.run.smithi018.stdout:Dependencies resolved. 2023-09-23T20:03:46.097 INFO:teuthology.orchestra.run.smithi018.stdout:======================================================================================================== 2023-09-23T20:03:46.097 INFO:teuthology.orchestra.run.smithi018.stdout: Package Arch Version Repository Size 2023-09-23T20:03:46.097 INFO:teuthology.orchestra.run.smithi018.stdout:======================================================================================================== 2023-09-23T20:03:46.098 INFO:teuthology.orchestra.run.smithi018.stdout:Installing: 2023-09-23T20:03:46.098 INFO:teuthology.orchestra.run.smithi018.stdout: ceph-mgr-rook noarch 2:17.2.6-1123.g61431f6b.el8 ceph-noarch 47 k 2023-09-23T20:03:46.098 INFO:teuthology.orchestra.run.smithi018.stdout:Installing dependencies: 2023-09-23T20:03:46.098 INFO:teuthology.orchestra.run.smithi018.stdout: python3-cachetools noarch 3.1.1-4.el8 epel 33 k 2023-09-23T20:03:46.098 INFO:teuthology.orchestra.run.smithi018.stdout: python3-certifi noarch 2018.10.15-7.el8 epel 16 k 2023-09-23T20:03:46.098 INFO:teuthology.orchestra.run.smithi018.stdout: python3-google-auth noarch 1:1.1.1-10.el8 epel 81 k 2023-09-23T20:03:46.098 INFO:teuthology.orchestra.run.smithi018.stdout: python3-jsonpatch noarch 1.21-2.el8 rhel-8-for-x86_64-appstream-rpms 27 k 2023-09-23T20:03:46.098 INFO:teuthology.orchestra.run.smithi018.stdout: python3-jsonpointer noarch 1.10-11.el8 rhel-8-for-x86_64-appstream-rpms 20 k 2023-09-23T20:03:46.098 INFO:teuthology.orchestra.run.smithi018.stdout: python3-kubernetes noarch 1:11.0.0-6.el8 epel 1.2 M 2023-09-23T20:03:46.098 INFO:teuthology.orchestra.run.smithi018.stdout: python3-oauthlib noarch 2.1.0-1.el8 rhel-8-for-x86_64-baseos-rpms 155 k 2023-09-23T20:03:46.098 INFO:teuthology.orchestra.run.smithi018.stdout: python3-pyasn1 noarch 0.3.7-6.el8 rhel-8-for-x86_64-appstream-rpms 126 k 2023-09-23T20:03:46.098 INFO:teuthology.orchestra.run.smithi018.stdout: python3-pyasn1-modules noarch 0.3.7-6.el8 rhel-8-for-x86_64-appstream-rpms 110 k 2023-09-23T20:03:46.099 INFO:teuthology.orchestra.run.smithi018.stdout: python3-requests-oauthlib noarch 1.0.0-1.el8 rhel-8-for-x86_64-baseos-rpms 43 k 2023-09-23T20:03:46.099 INFO:teuthology.orchestra.run.smithi018.stdout: python3-rsa noarch 4.9-2.el8 epel 64 k 2023-09-23T20:03:46.099 INFO:teuthology.orchestra.run.smithi018.stdout: python3-websocket-client noarch 0.56.0-5.el8 epel 61 k 2023-09-23T20:03:46.099 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:03:46.099 INFO:teuthology.orchestra.run.smithi018.stdout:Transaction Summary 2023-09-23T20:03:46.099 INFO:teuthology.orchestra.run.smithi018.stdout:======================================================================================================== 2023-09-23T20:03:46.099 INFO:teuthology.orchestra.run.smithi018.stdout:Install 13 Packages 2023-09-23T20:03:46.099 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:03:46.100 INFO:teuthology.orchestra.run.smithi018.stdout:Total download size: 1.9 M 2023-09-23T20:03:46.100 INFO:teuthology.orchestra.run.smithi018.stdout:Installed size: 25 M 2023-09-23T20:03:46.100 INFO:teuthology.orchestra.run.smithi018.stdout:Downloading Packages: 2023-09-23T20:03:46.375 INFO:teuthology.orchestra.run.smithi018.stdout:(1/13): python3-certifi-2018.10.15-7.el8.noarch 178 kB/s | 16 kB 00:00 2023-09-23T20:03:46.392 INFO:teuthology.orchestra.run.smithi018.stdout:(2/13): python3-cachetools-3.1.1-4.el8.noarch.r 304 kB/s | 33 kB 00:00 2023-09-23T20:03:46.426 INFO:teuthology.orchestra.run.smithi018.stdout:(3/13): python3-google-auth-1.1.1-10.el8.noarch 1.6 MB/s | 81 kB 00:00 2023-09-23T20:03:46.451 INFO:teuthology.orchestra.run.smithi018.stdout:(4/13): ceph-mgr-rook-17.2.6-1123.g61431f6b.el8 283 kB/s | 47 kB 00:00 2023-09-23T20:03:46.467 INFO:teuthology.orchestra.run.smithi149.stdout:Dependencies resolved. 2023-09-23T20:03:46.468 INFO:teuthology.orchestra.run.smithi149.stdout:================================================================================ 2023-09-23T20:03:46.468 INFO:teuthology.orchestra.run.smithi149.stdout: Package Arch Version Repo Size 2023-09-23T20:03:46.468 INFO:teuthology.orchestra.run.smithi149.stdout:================================================================================ 2023-09-23T20:03:46.468 INFO:teuthology.orchestra.run.smithi149.stdout:Installing: 2023-09-23T20:03:46.468 INFO:teuthology.orchestra.run.smithi149.stdout: ceph-immutable-object-cache x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 161 k 2023-09-23T20:03:46.468 INFO:teuthology.orchestra.run.smithi149.stdout: 2023-09-23T20:03:46.468 INFO:teuthology.orchestra.run.smithi149.stdout:Transaction Summary 2023-09-23T20:03:46.468 INFO:teuthology.orchestra.run.smithi149.stdout:================================================================================ 2023-09-23T20:03:46.468 INFO:teuthology.orchestra.run.smithi149.stdout:Install 1 Package 2023-09-23T20:03:46.468 INFO:teuthology.orchestra.run.smithi149.stdout: 2023-09-23T20:03:46.469 INFO:teuthology.orchestra.run.smithi149.stdout:Total download size: 161 k 2023-09-23T20:03:46.469 INFO:teuthology.orchestra.run.smithi149.stdout:Installed size: 490 k 2023-09-23T20:03:46.469 INFO:teuthology.orchestra.run.smithi149.stdout:Downloading Packages: 2023-09-23T20:03:46.476 INFO:teuthology.orchestra.run.smithi018.stdout:(5/13): python3-rsa-4.9-2.el8.noarch.rpm 1.2 MB/s | 64 kB 00:00 2023-09-23T20:03:46.519 INFO:teuthology.orchestra.run.smithi018.stdout:(6/13): python3-kubernetes-11.0.0-6.el8.noarch. 9.3 MB/s | 1.2 MB 00:00 2023-09-23T20:03:46.569 INFO:teuthology.orchestra.run.smithi018.stdout:(7/13): python3-websocket-client-0.56.0-5.el8.n 521 kB/s | 61 kB 00:00 2023-09-23T20:03:46.655 INFO:teuthology.orchestra.run.smithi149.stdout:ceph-immutable-object-cache-17.2.6-1123.g61431f 864 kB/s | 161 kB 00:00 2023-09-23T20:03:46.656 INFO:teuthology.orchestra.run.smithi149.stdout:-------------------------------------------------------------------------------- 2023-09-23T20:03:46.656 INFO:teuthology.orchestra.run.smithi149.stdout:Total 858 kB/s | 161 kB 00:00 2023-09-23T20:03:46.656 INFO:teuthology.orchestra.run.smithi149.stdout:Running transaction check 2023-09-23T20:03:46.667 INFO:teuthology.orchestra.run.smithi149.stdout:Transaction check succeeded. 2023-09-23T20:03:46.667 INFO:teuthology.orchestra.run.smithi149.stdout:Running transaction test 2023-09-23T20:03:46.669 INFO:teuthology.orchestra.run.smithi018.stdout:(8/13): python3-requests-oauthlib-1.0.0-1.el8.n 227 kB/s | 43 kB 00:00 2023-09-23T20:03:46.694 INFO:teuthology.orchestra.run.smithi018.stdout:(9/13): python3-oauthlib-2.1.0-1.el8.noarch.rpm 930 kB/s | 155 kB 00:00 2023-09-23T20:03:46.718 INFO:teuthology.orchestra.run.smithi149.stdout:Transaction test succeeded. 2023-09-23T20:03:46.719 INFO:teuthology.orchestra.run.smithi018.stdout:(10/13): python3-pyasn1-modules-0.3.7-6.el8.noa 774 kB/s | 110 kB 00:00 2023-09-23T20:03:46.721 INFO:teuthology.orchestra.run.smithi149.stdout:Running transaction 2023-09-23T20:03:46.803 INFO:teuthology.orchestra.run.smithi018.stdout:(11/13): python3-jsonpatch-1.21-2.el8.noarch.rp 198 kB/s | 27 kB 00:00 2023-09-23T20:03:46.835 INFO:teuthology.orchestra.run.smithi149.stdout: Preparing : 1/1 2023-09-23T20:03:46.853 INFO:teuthology.orchestra.run.smithi018.stdout:(12/13): python3-pyasn1-0.3.7-6.el8.noarch.rpm 796 kB/s | 126 kB 00:00 2023-09-23T20:03:46.904 INFO:teuthology.orchestra.run.smithi018.stdout:(13/13): python3-jsonpointer-1.10-11.el8.noarch 109 kB/s | 20 kB 00:00 2023-09-23T20:03:46.906 INFO:teuthology.orchestra.run.smithi018.stdout:-------------------------------------------------------------------------------- 2023-09-23T20:03:46.907 INFO:teuthology.orchestra.run.smithi018.stdout:Total 2.4 MB/s | 1.9 MB 00:00 2023-09-23T20:03:46.973 INFO:teuthology.orchestra.run.smithi018.stdout:Running transaction check 2023-09-23T20:03:46.988 INFO:teuthology.orchestra.run.smithi018.stdout:Transaction check succeeded. 2023-09-23T20:03:46.988 INFO:teuthology.orchestra.run.smithi018.stdout:Running transaction test 2023-09-23T20:03:46.996 INFO:teuthology.orchestra.run.smithi149.stdout: Installing : ceph-immutable-object-cache-2:17.2.6-1123.g61431f6b. 1/1 2023-09-23T20:03:47.064 INFO:teuthology.orchestra.run.smithi018.stdout:Transaction test succeeded. 2023-09-23T20:03:47.069 INFO:teuthology.orchestra.run.smithi018.stdout:Running transaction 2023-09-23T20:03:47.336 INFO:teuthology.orchestra.run.smithi149.stdout: Running scriptlet: ceph-immutable-object-cache-2:17.2.6-1123.g61431f6b. 1/1 2023-09-23T20:03:47.612 INFO:teuthology.orchestra.run.smithi018.stdout: Preparing : 1/1 2023-09-23T20:03:47.804 INFO:teuthology.orchestra.run.smithi018.stdout: Installing : python3-pyasn1-0.3.7-6.el8.noarch 1/13 2023-09-23T20:03:48.015 INFO:teuthology.orchestra.run.smithi018.stdout: Installing : python3-rsa-4.9-2.el8.noarch 2/13 2023-09-23T20:03:48.183 INFO:teuthology.orchestra.run.smithi018.stdout: Installing : python3-pyasn1-modules-0.3.7-6.el8.noarch 3/13 2023-09-23T20:03:48.376 INFO:teuthology.orchestra.run.smithi018.stdout: Installing : python3-jsonpointer-1.10-11.el8.noarch 4/13 2023-09-23T20:03:48.386 INFO:teuthology.orchestra.run.smithi149.stdout: Verifying : ceph-immutable-object-cache-2:17.2.6-1123.g61431f6b. 1/1 2023-09-23T20:03:48.386 INFO:teuthology.orchestra.run.smithi149.stdout:Last metadata expiration check: 0:01:46 ago on Sat 23 Sep 2023 08:02:02 PM UTC. 2023-09-23T20:03:48.578 INFO:teuthology.orchestra.run.smithi018.stdout: Installing : python3-jsonpatch-1.21-2.el8.noarch 5/13 2023-09-23T20:03:48.792 INFO:teuthology.orchestra.run.smithi018.stdout: Installing : python3-oauthlib-2.1.0-1.el8.noarch 6/13 2023-09-23T20:03:48.974 INFO:teuthology.orchestra.run.smithi018.stdout: Installing : python3-requests-oauthlib-1.0.0-1.el8.noarch 7/13 2023-09-23T20:03:49.177 INFO:teuthology.orchestra.run.smithi018.stdout: Installing : python3-websocket-client-0.56.0-5.el8.noarch 8/13 2023-09-23T20:03:49.394 INFO:teuthology.orchestra.run.smithi018.stdout: Installing : python3-certifi-2018.10.15-7.el8.noarch 9/13 2023-09-23T20:03:49.612 INFO:teuthology.orchestra.run.smithi018.stdout: Installing : python3-cachetools-3.1.1-4.el8.noarch 10/13 2023-09-23T20:03:50.752 INFO:teuthology.orchestra.run.smithi018.stdout: Installing : python3-google-auth-1:1.1.1-10.el8.noarch 11/13 2023-09-23T20:03:51.003 INFO:teuthology.orchestra.run.smithi018.stdout: Installing : python3-kubernetes-1:11.0.0-6.el8.noarch 12/13 2023-09-23T20:03:51.160 INFO:teuthology.orchestra.run.smithi018.stdout: Installing : ceph-mgr-rook-2:17.2.6-1123.g61431f6b.el8.noarch 13/13 2023-09-23T20:03:51.193 INFO:teuthology.orchestra.run.smithi149.stdout:Installed products updated. 2023-09-23T20:03:51.376 INFO:teuthology.orchestra.run.smithi149.stdout: 2023-09-23T20:03:51.377 INFO:teuthology.orchestra.run.smithi149.stdout:Installed: 2023-09-23T20:03:51.377 INFO:teuthology.orchestra.run.smithi149.stdout: ceph-immutable-object-cache-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:03:51.377 INFO:teuthology.orchestra.run.smithi149.stdout: 2023-09-23T20:03:51.377 INFO:teuthology.orchestra.run.smithi149.stdout:Complete! 2023-09-23T20:03:51.410 INFO:teuthology.orchestra.run.smithi018.stdout: Running scriptlet: ceph-mgr-rook-2:17.2.6-1123.g61431f6b.el8.noarch 13/13 2023-09-23T20:03:51.410 INFO:teuthology.orchestra.run.smithi018.stdout: Verifying : ceph-mgr-rook-2:17.2.6-1123.g61431f6b.el8.noarch 1/13 2023-09-23T20:03:51.410 INFO:teuthology.orchestra.run.smithi018.stdout: Verifying : python3-cachetools-3.1.1-4.el8.noarch 2/13 2023-09-23T20:03:51.410 INFO:teuthology.orchestra.run.smithi018.stdout: Verifying : python3-certifi-2018.10.15-7.el8.noarch 3/13 2023-09-23T20:03:51.411 INFO:teuthology.orchestra.run.smithi018.stdout: Verifying : python3-google-auth-1:1.1.1-10.el8.noarch 4/13 2023-09-23T20:03:51.411 INFO:teuthology.orchestra.run.smithi018.stdout: Verifying : python3-kubernetes-1:11.0.0-6.el8.noarch 5/13 2023-09-23T20:03:51.411 INFO:teuthology.orchestra.run.smithi018.stdout: Verifying : python3-rsa-4.9-2.el8.noarch 6/13 2023-09-23T20:03:51.411 INFO:teuthology.orchestra.run.smithi018.stdout: Verifying : python3-websocket-client-0.56.0-5.el8.noarch 7/13 2023-09-23T20:03:51.411 INFO:teuthology.orchestra.run.smithi018.stdout: Verifying : python3-requests-oauthlib-1.0.0-1.el8.noarch 8/13 2023-09-23T20:03:51.411 INFO:teuthology.orchestra.run.smithi018.stdout: Verifying : python3-oauthlib-2.1.0-1.el8.noarch 9/13 2023-09-23T20:03:51.411 INFO:teuthology.orchestra.run.smithi018.stdout: Verifying : python3-pyasn1-modules-0.3.7-6.el8.noarch 10/13 2023-09-23T20:03:51.411 INFO:teuthology.orchestra.run.smithi018.stdout: Verifying : python3-jsonpatch-1.21-2.el8.noarch 11/13 2023-09-23T20:03:51.411 INFO:teuthology.orchestra.run.smithi018.stdout: Verifying : python3-pyasn1-0.3.7-6.el8.noarch 12/13 2023-09-23T20:03:51.527 DEBUG:teuthology.orchestra.run.smithi149:> sudo yum -y install ceph-mgr 2023-09-23T20:03:51.875 INFO:teuthology.orchestra.run.smithi149.stdout:Updating Subscription Management repositories. 2023-09-23T20:03:52.578 INFO:teuthology.orchestra.run.smithi018.stdout: Verifying : python3-jsonpointer-1.10-11.el8.noarch 13/13 2023-09-23T20:03:52.578 INFO:teuthology.orchestra.run.smithi018.stdout:Last metadata expiration check: 0:02:30 ago on Sat 23 Sep 2023 08:01:22 PM UTC. 2023-09-23T20:03:53.373 INFO:teuthology.orchestra.run.smithi149.stdout:Last metadata expiration check: 0:01:51 ago on Sat 23 Sep 2023 08:02:02 PM UTC. 2023-09-23T20:03:54.658 INFO:teuthology.orchestra.run.smithi149.stdout:Package ceph-mgr-2:17.2.6-1123.g61431f6b.el8.x86_64 is already installed. 2023-09-23T20:03:54.770 INFO:teuthology.orchestra.run.smithi149.stdout:Dependencies resolved. 2023-09-23T20:03:54.771 INFO:teuthology.orchestra.run.smithi149.stdout:Nothing to do. 2023-09-23T20:03:54.771 INFO:teuthology.orchestra.run.smithi149.stdout:Complete! 2023-09-23T20:03:54.923 DEBUG:teuthology.orchestra.run.smithi149:> sudo yum -y install ceph-mgr-dashboard 2023-09-23T20:03:55.270 INFO:teuthology.orchestra.run.smithi149.stdout:Updating Subscription Management repositories. 2023-09-23T20:03:56.642 INFO:teuthology.orchestra.run.smithi149.stdout:Last metadata expiration check: 0:01:54 ago on Sat 23 Sep 2023 08:02:02 PM UTC. 2023-09-23T20:03:56.713 INFO:teuthology.orchestra.run.smithi018.stdout:Installed products updated. 2023-09-23T20:03:56.917 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:03:56.917 INFO:teuthology.orchestra.run.smithi018.stdout:Installed: 2023-09-23T20:03:56.917 INFO:teuthology.orchestra.run.smithi018.stdout: ceph-mgr-rook-2:17.2.6-1123.g61431f6b.el8.noarch 2023-09-23T20:03:56.917 INFO:teuthology.orchestra.run.smithi018.stdout: python3-cachetools-3.1.1-4.el8.noarch 2023-09-23T20:03:56.917 INFO:teuthology.orchestra.run.smithi018.stdout: python3-certifi-2018.10.15-7.el8.noarch 2023-09-23T20:03:56.917 INFO:teuthology.orchestra.run.smithi018.stdout: python3-google-auth-1:1.1.1-10.el8.noarch 2023-09-23T20:03:56.918 INFO:teuthology.orchestra.run.smithi018.stdout: python3-jsonpatch-1.21-2.el8.noarch 2023-09-23T20:03:56.918 INFO:teuthology.orchestra.run.smithi018.stdout: python3-jsonpointer-1.10-11.el8.noarch 2023-09-23T20:03:56.918 INFO:teuthology.orchestra.run.smithi018.stdout: python3-kubernetes-1:11.0.0-6.el8.noarch 2023-09-23T20:03:56.918 INFO:teuthology.orchestra.run.smithi018.stdout: python3-oauthlib-2.1.0-1.el8.noarch 2023-09-23T20:03:56.918 INFO:teuthology.orchestra.run.smithi018.stdout: python3-pyasn1-0.3.7-6.el8.noarch 2023-09-23T20:03:56.918 INFO:teuthology.orchestra.run.smithi018.stdout: python3-pyasn1-modules-0.3.7-6.el8.noarch 2023-09-23T20:03:56.918 INFO:teuthology.orchestra.run.smithi018.stdout: python3-requests-oauthlib-1.0.0-1.el8.noarch 2023-09-23T20:03:56.918 INFO:teuthology.orchestra.run.smithi018.stdout: python3-rsa-4.9-2.el8.noarch 2023-09-23T20:03:56.918 INFO:teuthology.orchestra.run.smithi018.stdout: python3-websocket-client-0.56.0-5.el8.noarch 2023-09-23T20:03:56.918 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:03:56.918 INFO:teuthology.orchestra.run.smithi018.stdout:Complete! 2023-09-23T20:03:57.053 DEBUG:teuthology.orchestra.run.smithi018:> sudo yum -y install ceph-mgr-cephadm 2023-09-23T20:03:57.428 INFO:teuthology.orchestra.run.smithi018.stdout:Updating Subscription Management repositories. 2023-09-23T20:03:58.108 INFO:teuthology.orchestra.run.smithi149.stdout:Dependencies resolved. 2023-09-23T20:03:58.109 INFO:teuthology.orchestra.run.smithi149.stdout:=================================================================================================== 2023-09-23T20:03:58.109 INFO:teuthology.orchestra.run.smithi149.stdout: Package Arch Version Repository Size 2023-09-23T20:03:58.109 INFO:teuthology.orchestra.run.smithi149.stdout:=================================================================================================== 2023-09-23T20:03:58.109 INFO:teuthology.orchestra.run.smithi149.stdout:Installing: 2023-09-23T20:03:58.109 INFO:teuthology.orchestra.run.smithi149.stdout: ceph-mgr-dashboard noarch 2:17.2.6-1123.g61431f6b.el8 ceph-noarch 1.7 M 2023-09-23T20:03:58.109 INFO:teuthology.orchestra.run.smithi149.stdout:Installing dependencies: 2023-09-23T20:03:58.109 INFO:teuthology.orchestra.run.smithi149.stdout: ceph-grafana-dashboards noarch 2:17.2.6-1123.g61431f6b.el8 ceph-noarch 23 k 2023-09-23T20:03:58.109 INFO:teuthology.orchestra.run.smithi149.stdout: ceph-prometheus-alerts noarch 2:17.2.6-1123.g61431f6b.el8 ceph-noarch 14 k 2023-09-23T20:03:58.109 INFO:teuthology.orchestra.run.smithi149.stdout: python3-jwt noarch 1.6.1-2.el8 rhel-8-for-x86_64-baseos-rpms 43 k 2023-09-23T20:03:58.110 INFO:teuthology.orchestra.run.smithi149.stdout: python3-repoze-lru noarch 0.7-6.el8 epel 34 k 2023-09-23T20:03:58.110 INFO:teuthology.orchestra.run.smithi149.stdout: python3-routes noarch 2.4.1-12.el8 epel 196 k 2023-09-23T20:03:58.110 INFO:teuthology.orchestra.run.smithi149.stdout: 2023-09-23T20:03:58.110 INFO:teuthology.orchestra.run.smithi149.stdout:Transaction Summary 2023-09-23T20:03:58.110 INFO:teuthology.orchestra.run.smithi149.stdout:=================================================================================================== 2023-09-23T20:03:58.110 INFO:teuthology.orchestra.run.smithi149.stdout:Install 6 Packages 2023-09-23T20:03:58.110 INFO:teuthology.orchestra.run.smithi149.stdout: 2023-09-23T20:03:58.111 INFO:teuthology.orchestra.run.smithi149.stdout:Total download size: 2.0 M 2023-09-23T20:03:58.111 INFO:teuthology.orchestra.run.smithi149.stdout:Installed size: 8.0 M 2023-09-23T20:03:58.111 INFO:teuthology.orchestra.run.smithi149.stdout:Downloading Packages: 2023-09-23T20:03:58.611 INFO:teuthology.orchestra.run.smithi149.stdout:(1/6): ceph-grafana-dashboards-17.2.6-1123.g614 68 kB/s | 23 kB 00:00 2023-09-23T20:03:58.628 INFO:teuthology.orchestra.run.smithi149.stdout:(2/6): ceph-prometheus-alerts-17.2.6-1123.g6143 41 kB/s | 14 kB 00:00 2023-09-23T20:03:58.811 INFO:teuthology.orchestra.run.smithi149.stdout:(3/6): ceph-mgr-dashboard-17.2.6-1123.g61431f6b 3.2 MB/s | 1.7 MB 00:00 2023-09-23T20:03:58.814 INFO:teuthology.orchestra.run.smithi018.stdout:Last metadata expiration check: 0:02:36 ago on Sat 23 Sep 2023 08:01:22 PM UTC. 2023-09-23T20:03:58.928 INFO:teuthology.orchestra.run.smithi149.stdout:(4/6): python3-jwt-1.6.1-2.el8.noarch.rpm 365 kB/s | 43 kB 00:00 2023-09-23T20:03:59.437 INFO:teuthology.orchestra.run.smithi149.stdout:(5/6): python3-repoze-lru-0.7-6.el8.noarch.rpm 41 kB/s | 34 kB 00:00 2023-09-23T20:03:59.620 INFO:teuthology.orchestra.run.smithi149.stdout:(6/6): python3-routes-2.4.1-12.el8.noarch.rpm 198 kB/s | 196 kB 00:00 2023-09-23T20:03:59.627 INFO:teuthology.orchestra.run.smithi149.stdout:-------------------------------------------------------------------------------- 2023-09-23T20:03:59.627 INFO:teuthology.orchestra.run.smithi149.stdout:Total 1.3 MB/s | 2.0 MB 00:01 2023-09-23T20:03:59.637 INFO:teuthology.orchestra.run.smithi149.stdout:Running transaction check 2023-09-23T20:03:59.647 INFO:teuthology.orchestra.run.smithi149.stdout:Transaction check succeeded. 2023-09-23T20:03:59.647 INFO:teuthology.orchestra.run.smithi149.stdout:Running transaction test 2023-09-23T20:03:59.709 INFO:teuthology.orchestra.run.smithi149.stdout:Transaction test succeeded. 2023-09-23T20:03:59.713 INFO:teuthology.orchestra.run.smithi149.stdout:Running transaction 2023-09-23T20:03:59.967 INFO:teuthology.orchestra.run.smithi149.stdout: Preparing : 1/1 2023-09-23T20:04:00.136 INFO:teuthology.orchestra.run.smithi149.stdout: Installing : python3-jwt-1.6.1-2.el8.noarch 1/6 2023-09-23T20:04:00.333 INFO:teuthology.orchestra.run.smithi149.stdout: Installing : python3-repoze-lru-0.7-6.el8.noarch 2/6 2023-09-23T20:04:00.383 INFO:teuthology.orchestra.run.smithi018.stdout:Dependencies resolved. 2023-09-23T20:04:00.384 INFO:teuthology.orchestra.run.smithi018.stdout:=============================================================================================== 2023-09-23T20:04:00.384 INFO:teuthology.orchestra.run.smithi018.stdout: Package Arch Version Repository Size 2023-09-23T20:04:00.384 INFO:teuthology.orchestra.run.smithi018.stdout:=============================================================================================== 2023-09-23T20:04:00.384 INFO:teuthology.orchestra.run.smithi018.stdout:Installing: 2023-09-23T20:04:00.384 INFO:teuthology.orchestra.run.smithi018.stdout: ceph-mgr-cephadm noarch 2:17.2.6-1123.g61431f6b.el8 ceph-noarch 121 k 2023-09-23T20:04:00.385 INFO:teuthology.orchestra.run.smithi018.stdout:Installing dependencies: 2023-09-23T20:04:00.385 INFO:teuthology.orchestra.run.smithi018.stdout: python3-asyncssh noarch 2.7.0-2.el8 epel 476 k 2023-09-23T20:04:00.385 INFO:teuthology.orchestra.run.smithi018.stdout: python3-babel noarch 2.5.1-7.el8 rhel-8-for-x86_64-appstream-rpms 4.8 M 2023-09-23T20:04:00.385 INFO:teuthology.orchestra.run.smithi018.stdout: python3-jinja2 noarch 2.10.1-3.el8 rhel-8-for-x86_64-appstream-rpms 538 k 2023-09-23T20:04:00.385 INFO:teuthology.orchestra.run.smithi018.stdout: python3-natsort noarch 7.1.1-2.el8 epel 60 k 2023-09-23T20:04:00.385 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:04:00.385 INFO:teuthology.orchestra.run.smithi018.stdout:Transaction Summary 2023-09-23T20:04:00.385 INFO:teuthology.orchestra.run.smithi018.stdout:=============================================================================================== 2023-09-23T20:04:00.385 INFO:teuthology.orchestra.run.smithi018.stdout:Install 5 Packages 2023-09-23T20:04:00.385 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:04:00.386 INFO:teuthology.orchestra.run.smithi018.stdout:Total download size: 6.0 M 2023-09-23T20:04:00.386 INFO:teuthology.orchestra.run.smithi018.stdout:Installed size: 26 M 2023-09-23T20:04:00.386 INFO:teuthology.orchestra.run.smithi018.stdout:Downloading Packages: 2023-09-23T20:04:00.513 INFO:teuthology.orchestra.run.smithi149.stdout: Installing : python3-routes-2.4.1-12.el8.noarch 3/6 2023-09-23T20:04:00.658 INFO:teuthology.orchestra.run.smithi149.stdout: Installing : ceph-prometheus-alerts-2:17.2.6-1123.g61431f6b.el8.n 4/6 2023-09-23T20:04:00.886 INFO:teuthology.orchestra.run.smithi018.stdout:(1/5): python3-natsort-7.1.1-2.el8.noarch.rpm 418 kB/s | 60 kB 00:00 2023-09-23T20:04:00.987 INFO:teuthology.orchestra.run.smithi018.stdout:(2/5): python3-jinja2-2.10.1-3.el8.noarch.rpm 5.3 MB/s | 538 kB 00:00 2023-09-23T20:04:00.987 INFO:teuthology.orchestra.run.smithi149.stdout: Installing : ceph-grafana-dashboards-2:17.2.6-1123.g61431f6b.el8. 5/6 2023-09-23T20:04:01.012 INFO:teuthology.orchestra.run.smithi018.stdout:(3/5): python3-asyncssh-2.7.0-2.el8.noarch.rpm 1.7 MB/s | 476 kB 00:00 2023-09-23T20:04:01.037 INFO:teuthology.orchestra.run.smithi018.stdout:(4/5): ceph-mgr-cephadm-17.2.6-1123.g61431f6b.e 411 kB/s | 121 kB 00:00 2023-09-23T20:04:01.181 INFO:teuthology.orchestra.run.smithi149.stdout: Installing : ceph-mgr-dashboard-2:17.2.6-1123.g61431f6b.el8.noarc 6/6 2023-09-23T20:04:01.287 INFO:teuthology.orchestra.run.smithi018.stdout:(5/5): python3-babel-2.5.1-7.el8.noarch.rpm 16 MB/s | 4.8 MB 00:00 2023-09-23T20:04:01.290 INFO:teuthology.orchestra.run.smithi018.stdout:-------------------------------------------------------------------------------- 2023-09-23T20:04:01.290 INFO:teuthology.orchestra.run.smithi018.stdout:Total 6.6 MB/s | 6.0 MB 00:00 2023-09-23T20:04:01.348 INFO:teuthology.orchestra.run.smithi018.stdout:Running transaction check 2023-09-23T20:04:01.361 INFO:teuthology.orchestra.run.smithi018.stdout:Transaction check succeeded. 2023-09-23T20:04:01.361 INFO:teuthology.orchestra.run.smithi018.stdout:Running transaction test 2023-09-23T20:04:01.416 INFO:teuthology.orchestra.run.smithi149.stdout: Running scriptlet: ceph-mgr-dashboard-2:17.2.6-1123.g61431f6b.el8.noarc 6/6 2023-09-23T20:04:01.417 INFO:teuthology.orchestra.run.smithi149.stdout: Verifying : ceph-grafana-dashboards-2:17.2.6-1123.g61431f6b.el8. 1/6 2023-09-23T20:04:01.417 INFO:teuthology.orchestra.run.smithi149.stdout: Verifying : ceph-mgr-dashboard-2:17.2.6-1123.g61431f6b.el8.noarc 2/6 2023-09-23T20:04:01.417 INFO:teuthology.orchestra.run.smithi149.stdout: Verifying : ceph-prometheus-alerts-2:17.2.6-1123.g61431f6b.el8.n 3/6 2023-09-23T20:04:01.417 INFO:teuthology.orchestra.run.smithi149.stdout: Verifying : python3-repoze-lru-0.7-6.el8.noarch 4/6 2023-09-23T20:04:01.417 INFO:teuthology.orchestra.run.smithi149.stdout: Verifying : python3-routes-2.4.1-12.el8.noarch 5/6 2023-09-23T20:04:01.447 INFO:teuthology.orchestra.run.smithi018.stdout:Transaction test succeeded. 2023-09-23T20:04:01.452 INFO:teuthology.orchestra.run.smithi018.stdout:Running transaction 2023-09-23T20:04:02.468 INFO:teuthology.orchestra.run.smithi018.stdout: Preparing : 1/1 2023-09-23T20:04:02.504 INFO:teuthology.orchestra.run.smithi149.stdout: Verifying : python3-jwt-1.6.1-2.el8.noarch 6/6 2023-09-23T20:04:02.504 INFO:teuthology.orchestra.run.smithi149.stdout:Last metadata expiration check: 0:02:00 ago on Sat 23 Sep 2023 08:02:02 PM UTC. 2023-09-23T20:04:02.828 INFO:teuthology.orchestra.run.smithi018.stdout: Installing : python3-babel-2.5.1-7.el8.noarch 1/5 2023-09-23T20:04:03.061 INFO:teuthology.orchestra.run.smithi018.stdout: Installing : python3-jinja2-2.10.1-3.el8.noarch 2/5 2023-09-23T20:04:03.349 INFO:teuthology.orchestra.run.smithi018.stdout: Installing : python3-natsort-7.1.1-2.el8.noarch 3/5 2023-09-23T20:04:03.611 INFO:teuthology.orchestra.run.smithi018.stdout: Installing : python3-asyncssh-2.7.0-2.el8.noarch 4/5 2023-09-23T20:04:03.799 INFO:teuthology.orchestra.run.smithi018.stdout: Installing : ceph-mgr-cephadm-2:17.2.6-1123.g61431f6b.el8.noarch 5/5 2023-09-23T20:04:04.040 INFO:teuthology.orchestra.run.smithi018.stdout: Running scriptlet: ceph-mgr-cephadm-2:17.2.6-1123.g61431f6b.el8.noarch 5/5 2023-09-23T20:04:04.041 INFO:teuthology.orchestra.run.smithi018.stdout: Verifying : ceph-mgr-cephadm-2:17.2.6-1123.g61431f6b.el8.noarch 1/5 2023-09-23T20:04:04.041 INFO:teuthology.orchestra.run.smithi018.stdout: Verifying : python3-asyncssh-2.7.0-2.el8.noarch 2/5 2023-09-23T20:04:04.041 INFO:teuthology.orchestra.run.smithi018.stdout: Verifying : python3-natsort-7.1.1-2.el8.noarch 3/5 2023-09-23T20:04:04.041 INFO:teuthology.orchestra.run.smithi018.stdout: Verifying : python3-jinja2-2.10.1-3.el8.noarch 4/5 2023-09-23T20:04:05.102 INFO:teuthology.orchestra.run.smithi018.stdout: Verifying : python3-babel-2.5.1-7.el8.noarch 5/5 2023-09-23T20:04:05.102 INFO:teuthology.orchestra.run.smithi018.stdout:Last metadata expiration check: 0:02:43 ago on Sat 23 Sep 2023 08:01:22 PM UTC. 2023-09-23T20:04:05.487 INFO:teuthology.orchestra.run.smithi149.stdout:Installed products updated. 2023-09-23T20:04:05.672 INFO:teuthology.orchestra.run.smithi149.stdout: 2023-09-23T20:04:05.672 INFO:teuthology.orchestra.run.smithi149.stdout:Installed: 2023-09-23T20:04:05.672 INFO:teuthology.orchestra.run.smithi149.stdout: ceph-grafana-dashboards-2:17.2.6-1123.g61431f6b.el8.noarch 2023-09-23T20:04:05.672 INFO:teuthology.orchestra.run.smithi149.stdout: ceph-mgr-dashboard-2:17.2.6-1123.g61431f6b.el8.noarch 2023-09-23T20:04:05.672 INFO:teuthology.orchestra.run.smithi149.stdout: ceph-prometheus-alerts-2:17.2.6-1123.g61431f6b.el8.noarch 2023-09-23T20:04:05.672 INFO:teuthology.orchestra.run.smithi149.stdout: python3-jwt-1.6.1-2.el8.noarch 2023-09-23T20:04:05.673 INFO:teuthology.orchestra.run.smithi149.stdout: python3-repoze-lru-0.7-6.el8.noarch 2023-09-23T20:04:05.673 INFO:teuthology.orchestra.run.smithi149.stdout: python3-routes-2.4.1-12.el8.noarch 2023-09-23T20:04:05.673 INFO:teuthology.orchestra.run.smithi149.stdout: 2023-09-23T20:04:05.673 INFO:teuthology.orchestra.run.smithi149.stdout:Complete! 2023-09-23T20:04:05.813 DEBUG:teuthology.orchestra.run.smithi149:> sudo yum -y install ceph-mgr-diskprediction-local 2023-09-23T20:04:06.164 INFO:teuthology.orchestra.run.smithi149.stdout:Updating Subscription Management repositories. 2023-09-23T20:04:06.975 INFO:teuthology.orchestra.run.smithi149.stdout:Last metadata expiration check: 0:02:04 ago on Sat 23 Sep 2023 08:02:02 PM UTC. 2023-09-23T20:04:08.163 INFO:teuthology.orchestra.run.smithi018.stdout:Installed products updated. 2023-09-23T20:04:08.367 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:04:08.367 INFO:teuthology.orchestra.run.smithi018.stdout:Installed: 2023-09-23T20:04:08.367 INFO:teuthology.orchestra.run.smithi018.stdout: ceph-mgr-cephadm-2:17.2.6-1123.g61431f6b.el8.noarch 2023-09-23T20:04:08.367 INFO:teuthology.orchestra.run.smithi018.stdout: python3-asyncssh-2.7.0-2.el8.noarch 2023-09-23T20:04:08.367 INFO:teuthology.orchestra.run.smithi018.stdout: python3-babel-2.5.1-7.el8.noarch 2023-09-23T20:04:08.367 INFO:teuthology.orchestra.run.smithi018.stdout: python3-jinja2-2.10.1-3.el8.noarch 2023-09-23T20:04:08.367 INFO:teuthology.orchestra.run.smithi018.stdout: python3-natsort-7.1.1-2.el8.noarch 2023-09-23T20:04:08.368 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:04:08.368 INFO:teuthology.orchestra.run.smithi018.stdout:Complete! 2023-09-23T20:04:08.375 INFO:teuthology.orchestra.run.smithi149.stdout:Dependencies resolved. 2023-09-23T20:04:08.377 INFO:teuthology.orchestra.run.smithi149.stdout:======================================================================================================================= 2023-09-23T20:04:08.377 INFO:teuthology.orchestra.run.smithi149.stdout: Package Arch Version Repository Size 2023-09-23T20:04:08.377 INFO:teuthology.orchestra.run.smithi149.stdout:======================================================================================================================= 2023-09-23T20:04:08.377 INFO:teuthology.orchestra.run.smithi149.stdout:Installing: 2023-09-23T20:04:08.377 INFO:teuthology.orchestra.run.smithi149.stdout: ceph-mgr-diskprediction-local noarch 2:17.2.6-1123.g61431f6b.el8 ceph-noarch 7.4 M 2023-09-23T20:04:08.377 INFO:teuthology.orchestra.run.smithi149.stdout:Installing dependencies: 2023-09-23T20:04:08.377 INFO:teuthology.orchestra.run.smithi149.stdout: libgfortran x86_64 8.5.0-10.1.el8_6 rhel-8-for-x86_64-baseos-rpms 644 k 2023-09-23T20:04:08.377 INFO:teuthology.orchestra.run.smithi149.stdout: libquadmath x86_64 8.5.0-10.1.el8_6 rhel-8-for-x86_64-baseos-rpms 171 k 2023-09-23T20:04:08.378 INFO:teuthology.orchestra.run.smithi149.stdout: openblas x86_64 0.3.15-3.el8 rhel-8-for-x86_64-appstream-rpms 4.7 M 2023-09-23T20:04:08.378 INFO:teuthology.orchestra.run.smithi149.stdout: openblas-threads x86_64 0.3.15-3.el8 rhel-8-for-x86_64-appstream-rpms 4.9 M 2023-09-23T20:04:08.378 INFO:teuthology.orchestra.run.smithi149.stdout: python3-numpy x86_64 1:1.14.3-10.el8 rhel-8-for-x86_64-appstream-rpms 3.7 M 2023-09-23T20:04:08.378 INFO:teuthology.orchestra.run.smithi149.stdout: python3-numpy-f2py x86_64 1:1.14.3-10.el8 rhel-8-for-x86_64-appstream-rpms 225 k 2023-09-23T20:04:08.378 INFO:teuthology.orchestra.run.smithi149.stdout: python3-scipy x86_64 1.0.0-21.module+el8.5.0+10916+41bd434d rhel-8-for-x86_64-appstream-rpms 14 M 2023-09-23T20:04:08.378 INFO:teuthology.orchestra.run.smithi149.stdout: 2023-09-23T20:04:08.378 INFO:teuthology.orchestra.run.smithi149.stdout:Transaction Summary 2023-09-23T20:04:08.378 INFO:teuthology.orchestra.run.smithi149.stdout:======================================================================================================================= 2023-09-23T20:04:08.378 INFO:teuthology.orchestra.run.smithi149.stdout:Install 8 Packages 2023-09-23T20:04:08.378 INFO:teuthology.orchestra.run.smithi149.stdout: 2023-09-23T20:04:08.379 INFO:teuthology.orchestra.run.smithi149.stdout:Total download size: 35 M 2023-09-23T20:04:08.379 INFO:teuthology.orchestra.run.smithi149.stdout:Installed size: 210 M 2023-09-23T20:04:08.379 INFO:teuthology.orchestra.run.smithi149.stdout:Downloading Packages: 2023-09-23T20:04:08.459 INFO:teuthology.orchestra.run.smithi149.stdout:(1/8): libgfortran-8.5.0-10.1.el8_6.x86_64.rpm 7.9 MB/s | 644 kB 00:00 2023-09-23T20:04:08.484 INFO:teuthology.orchestra.run.smithi149.stdout:(2/8): libquadmath-8.5.0-10.1.el8_6.x86_64.rpm 1.6 MB/s | 171 kB 00:00 2023-09-23T20:04:08.618 INFO:teuthology.orchestra.run.smithi149.stdout:(3/8): python3-numpy-1.14.3-10.el8.x86_64.rpm 28 MB/s | 3.7 MB 00:00 2023-09-23T20:04:08.669 INFO:teuthology.orchestra.run.smithi149.stdout:(4/8): python3-numpy-f2py-1.14.3-10.el8.x86_64. 4.4 MB/s | 225 kB 00:00 2023-09-23T20:04:08.869 INFO:teuthology.orchestra.run.smithi149.stdout:(5/8): python3-scipy-1.0.0-21.module+el8.5.0+10 33 MB/s | 14 MB 00:00 2023-09-23T20:04:08.936 INFO:teuthology.orchestra.run.smithi149.stdout:(6/8): openblas-threads-0.3.15-3.el8.x86_64.rpm 18 MB/s | 4.9 MB 00:00 2023-09-23T20:04:09.214 INFO:teuthology.orchestra.run.smithi149.stdout:(7/8): ceph-mgr-diskprediction-local-17.2.6-112 8.9 MB/s | 7.4 MB 00:00 2023-09-23T20:04:09.260 DEBUG:teuthology.orchestra.run.smithi018:> sudo yum -y install ceph-fuse 2023-09-23T20:04:09.270 INFO:teuthology.orchestra.run.smithi149.stdout:(8/8): openblas-0.3.15-3.el8.x86_64.rpm 12 MB/s | 4.7 MB 00:00 2023-09-23T20:04:09.271 INFO:teuthology.orchestra.run.smithi149.stdout:-------------------------------------------------------------------------------- 2023-09-23T20:04:09.271 INFO:teuthology.orchestra.run.smithi149.stdout:Total 40 MB/s | 35 MB 00:00 2023-09-23T20:04:09.509 INFO:teuthology.orchestra.run.smithi149.stdout:Running transaction check 2023-09-23T20:04:09.528 INFO:teuthology.orchestra.run.smithi149.stdout:Transaction check succeeded. 2023-09-23T20:04:09.528 INFO:teuthology.orchestra.run.smithi149.stdout:Running transaction test 2023-09-23T20:04:09.641 INFO:teuthology.orchestra.run.smithi018.stdout:Updating Subscription Management repositories. 2023-09-23T20:04:09.822 INFO:teuthology.orchestra.run.smithi149.stdout:Transaction test succeeded. 2023-09-23T20:04:09.826 INFO:teuthology.orchestra.run.smithi149.stdout:Running transaction 2023-09-23T20:04:10.370 INFO:teuthology.orchestra.run.smithi149.stdout: Preparing : 1/1 2023-09-23T20:04:10.525 INFO:teuthology.orchestra.run.smithi149.stdout: Installing : libquadmath-8.5.0-10.1.el8_6.x86_64 1/8 2023-09-23T20:04:10.529 INFO:teuthology.orchestra.run.smithi018.stdout:Last metadata expiration check: 0:02:48 ago on Sat 23 Sep 2023 08:01:22 PM UTC. 2023-09-23T20:04:10.658 INFO:teuthology.orchestra.run.smithi149.stdout: Running scriptlet: libquadmath-8.5.0-10.1.el8_6.x86_64 1/8 2023-09-23T20:04:10.800 INFO:teuthology.orchestra.run.smithi149.stdout: Installing : libgfortran-8.5.0-10.1.el8_6.x86_64 2/8 2023-09-23T20:04:11.420 INFO:teuthology.orchestra.run.smithi149.stdout: Running scriptlet: libgfortran-8.5.0-10.1.el8_6.x86_64 2/8 2023-09-23T20:04:11.584 INFO:teuthology.orchestra.run.smithi149.stdout: Installing : openblas-threads-0.3.15-3.el8.x86_64 3/8 2023-09-23T20:04:12.083 INFO:teuthology.orchestra.run.smithi018.stdout:Dependencies resolved. 2023-09-23T20:04:12.084 INFO:teuthology.orchestra.run.smithi018.stdout:================================================================================ 2023-09-23T20:04:12.084 INFO:teuthology.orchestra.run.smithi018.stdout: Package Arch Version Repository Size 2023-09-23T20:04:12.084 INFO:teuthology.orchestra.run.smithi018.stdout:================================================================================ 2023-09-23T20:04:12.084 INFO:teuthology.orchestra.run.smithi018.stdout:Installing: 2023-09-23T20:04:12.084 INFO:teuthology.orchestra.run.smithi018.stdout: ceph-fuse 2023-09-23T20:04:12.084 INFO:teuthology.orchestra.run.smithi018.stdout: x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 874 k 2023-09-23T20:04:12.084 INFO:teuthology.orchestra.run.smithi018.stdout:Installing dependencies: 2023-09-23T20:04:12.084 INFO:teuthology.orchestra.run.smithi018.stdout: fuse x86_64 2.9.7-15.el8 rhel-8-for-x86_64-baseos-rpms 83 k 2023-09-23T20:04:12.085 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:04:12.085 INFO:teuthology.orchestra.run.smithi018.stdout:Transaction Summary 2023-09-23T20:04:12.085 INFO:teuthology.orchestra.run.smithi018.stdout:================================================================================ 2023-09-23T20:04:12.085 INFO:teuthology.orchestra.run.smithi018.stdout:Install 2 Packages 2023-09-23T20:04:12.085 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:04:12.085 INFO:teuthology.orchestra.run.smithi018.stdout:Total download size: 957 k 2023-09-23T20:04:12.086 INFO:teuthology.orchestra.run.smithi018.stdout:Installed size: 3.0 M 2023-09-23T20:04:12.086 INFO:teuthology.orchestra.run.smithi018.stdout:Downloading Packages: 2023-09-23T20:04:12.165 INFO:teuthology.orchestra.run.smithi018.stdout:(1/2): fuse-2.9.7-15.el8.x86_64.rpm 1.0 MB/s | 83 kB 00:00 2023-09-23T20:04:12.186 INFO:teuthology.orchestra.run.smithi149.stdout: Running scriptlet: openblas-threads-0.3.15-3.el8.x86_64 3/8 2023-09-23T20:04:12.350 INFO:teuthology.orchestra.run.smithi149.stdout: Installing : openblas-0.3.15-3.el8.x86_64 4/8 2023-09-23T20:04:12.491 INFO:teuthology.orchestra.run.smithi018.stdout:(2/2): ceph-fuse-17.2.6-1123.g61431f6b.el8.x86_ 2.1 MB/s | 874 kB 00:00 2023-09-23T20:04:12.491 INFO:teuthology.orchestra.run.smithi018.stdout:-------------------------------------------------------------------------------- 2023-09-23T20:04:12.491 INFO:teuthology.orchestra.run.smithi018.stdout:Total 2.3 MB/s | 957 kB 00:00 2023-09-23T20:04:12.503 INFO:teuthology.orchestra.run.smithi018.stdout:Running transaction check 2023-09-23T20:04:12.525 INFO:teuthology.orchestra.run.smithi018.stdout:Transaction check succeeded. 2023-09-23T20:04:12.525 INFO:teuthology.orchestra.run.smithi018.stdout:Running transaction test 2023-09-23T20:04:12.603 INFO:teuthology.orchestra.run.smithi018.stdout:Transaction test succeeded. 2023-09-23T20:04:12.607 INFO:teuthology.orchestra.run.smithi018.stdout:Running transaction 2023-09-23T20:04:12.767 INFO:teuthology.orchestra.run.smithi018.stdout: Preparing : 1/1 2023-09-23T20:04:13.020 INFO:teuthology.orchestra.run.smithi018.stdout: Installing : fuse-2.9.7-15.el8.x86_64 1/2 2023-09-23T20:04:13.028 INFO:teuthology.orchestra.run.smithi149.stdout: Running scriptlet: openblas-0.3.15-3.el8.x86_64 4/8 2023-09-23T20:04:13.196 INFO:teuthology.orchestra.run.smithi018.stdout: Installing : ceph-fuse-2:17.2.6-1123.g61431f6b.el8.x86_64 2/2 2023-09-23T20:04:13.354 INFO:teuthology.orchestra.run.smithi149.stdout: Installing : python3-numpy-1:1.14.3-10.el8.x86_64 5/8 2023-09-23T20:04:13.544 INFO:teuthology.orchestra.run.smithi018.stdout: Running scriptlet: ceph-fuse-2:17.2.6-1123.g61431f6b.el8.x86_64 2/2 2023-09-23T20:04:13.545 INFO:teuthology.orchestra.run.smithi018.stdout: Verifying : ceph-fuse-2:17.2.6-1123.g61431f6b.el8.x86_64 1/2 2023-09-23T20:04:13.575 INFO:teuthology.orchestra.run.smithi149.stdout: Installing : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-09-23T20:04:14.633 INFO:teuthology.orchestra.run.smithi018.stdout: Verifying : fuse-2.9.7-15.el8.x86_64 2/2 2023-09-23T20:04:14.633 INFO:teuthology.orchestra.run.smithi018.stdout:Last metadata expiration check: 0:02:52 ago on Sat 23 Sep 2023 08:01:22 PM UTC. 2023-09-23T20:04:15.343 INFO:teuthology.orchestra.run.smithi149.stdout: Running scriptlet: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-09-23T20:04:16.602 INFO:teuthology.orchestra.run.smithi149.stdout: Installing : python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd434d 7/8 2023-09-23T20:04:16.816 INFO:teuthology.orchestra.run.smithi149.stdout: Installing : ceph-mgr-diskprediction-local-2:17.2.6-1123.g61431f6 8/8 2023-09-23T20:04:17.088 INFO:teuthology.orchestra.run.smithi149.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:17.2.6-1123.g61431f6 8/8 2023-09-23T20:04:17.088 INFO:teuthology.orchestra.run.smithi149.stdout: Verifying : ceph-mgr-diskprediction-local-2:17.2.6-1123.g61431f6 1/8 2023-09-23T20:04:17.089 INFO:teuthology.orchestra.run.smithi149.stdout: Verifying : libgfortran-8.5.0-10.1.el8_6.x86_64 2/8 2023-09-23T20:04:17.089 INFO:teuthology.orchestra.run.smithi149.stdout: Verifying : libquadmath-8.5.0-10.1.el8_6.x86_64 3/8 2023-09-23T20:04:17.089 INFO:teuthology.orchestra.run.smithi149.stdout: Verifying : python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd434d 4/8 2023-09-23T20:04:17.089 INFO:teuthology.orchestra.run.smithi149.stdout: Verifying : python3-numpy-1:1.14.3-10.el8.x86_64 5/8 2023-09-23T20:04:17.089 INFO:teuthology.orchestra.run.smithi149.stdout: Verifying : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-09-23T20:04:17.089 INFO:teuthology.orchestra.run.smithi149.stdout: Verifying : openblas-threads-0.3.15-3.el8.x86_64 7/8 2023-09-23T20:04:17.604 INFO:teuthology.orchestra.run.smithi018.stdout:Installed products updated. 2023-09-23T20:04:17.808 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:04:17.808 INFO:teuthology.orchestra.run.smithi018.stdout:Installed: 2023-09-23T20:04:17.808 INFO:teuthology.orchestra.run.smithi018.stdout: ceph-fuse-2:17.2.6-1123.g61431f6b.el8.x86_64 fuse-2.9.7-15.el8.x86_64 2023-09-23T20:04:17.808 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:04:17.809 INFO:teuthology.orchestra.run.smithi018.stdout:Complete! 2023-09-23T20:04:17.945 DEBUG:teuthology.orchestra.run.smithi018:> sudo yum -y install librados-devel 2023-09-23T20:04:18.164 INFO:teuthology.orchestra.run.smithi149.stdout: Verifying : openblas-0.3.15-3.el8.x86_64 8/8 2023-09-23T20:04:18.164 INFO:teuthology.orchestra.run.smithi149.stdout:Last metadata expiration check: 0:02:16 ago on Sat 23 Sep 2023 08:02:02 PM UTC. 2023-09-23T20:04:18.322 INFO:teuthology.orchestra.run.smithi018.stdout:Updating Subscription Management repositories. 2023-09-23T20:04:19.244 INFO:teuthology.orchestra.run.smithi018.stdout:Last metadata expiration check: 0:02:57 ago on Sat 23 Sep 2023 08:01:22 PM UTC. 2023-09-23T20:04:20.799 INFO:teuthology.orchestra.run.smithi018.stdout:Dependencies resolved. 2023-09-23T20:04:20.800 INFO:teuthology.orchestra.run.smithi018.stdout:================================================================================ 2023-09-23T20:04:20.800 INFO:teuthology.orchestra.run.smithi018.stdout: Package Arch Version Repo Size 2023-09-23T20:04:20.800 INFO:teuthology.orchestra.run.smithi018.stdout:================================================================================ 2023-09-23T20:04:20.800 INFO:teuthology.orchestra.run.smithi018.stdout:Installing: 2023-09-23T20:04:20.800 INFO:teuthology.orchestra.run.smithi018.stdout: librados-devel x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 130 k 2023-09-23T20:04:20.800 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:04:20.801 INFO:teuthology.orchestra.run.smithi018.stdout:Transaction Summary 2023-09-23T20:04:20.801 INFO:teuthology.orchestra.run.smithi018.stdout:================================================================================ 2023-09-23T20:04:20.801 INFO:teuthology.orchestra.run.smithi018.stdout:Install 1 Package 2023-09-23T20:04:20.801 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:04:20.802 INFO:teuthology.orchestra.run.smithi018.stdout:Total download size: 130 k 2023-09-23T20:04:20.802 INFO:teuthology.orchestra.run.smithi018.stdout:Installed size: 464 k 2023-09-23T20:04:20.802 INFO:teuthology.orchestra.run.smithi018.stdout:Downloading Packages: 2023-09-23T20:04:21.095 INFO:teuthology.orchestra.run.smithi018.stdout:librados-devel-17.2.6-1123.g61431f6b.el8.x86_64 442 kB/s | 130 kB 00:00 2023-09-23T20:04:21.096 INFO:teuthology.orchestra.run.smithi018.stdout:-------------------------------------------------------------------------------- 2023-09-23T20:04:21.096 INFO:teuthology.orchestra.run.smithi018.stdout:Total 440 kB/s | 130 kB 00:00 2023-09-23T20:04:21.096 INFO:teuthology.orchestra.run.smithi018.stdout:Running transaction check 2023-09-23T20:04:21.102 INFO:teuthology.orchestra.run.smithi018.stdout:Transaction check succeeded. 2023-09-23T20:04:21.102 INFO:teuthology.orchestra.run.smithi018.stdout:Running transaction test 2023-09-23T20:04:21.161 INFO:teuthology.orchestra.run.smithi018.stdout:Transaction test succeeded. 2023-09-23T20:04:21.165 INFO:teuthology.orchestra.run.smithi018.stdout:Running transaction 2023-09-23T20:04:21.295 INFO:teuthology.orchestra.run.smithi018.stdout: Preparing : 1/1 2023-09-23T20:04:21.444 INFO:teuthology.orchestra.run.smithi018.stdout: Installing : librados-devel-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T20:04:21.675 INFO:teuthology.orchestra.run.smithi018.stdout: Running scriptlet: librados-devel-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T20:04:21.947 INFO:teuthology.orchestra.run.smithi149.stdout:Installed products updated. 2023-09-23T20:04:22.133 INFO:teuthology.orchestra.run.smithi149.stdout: 2023-09-23T20:04:22.134 INFO:teuthology.orchestra.run.smithi149.stdout:Installed: 2023-09-23T20:04:22.134 INFO:teuthology.orchestra.run.smithi149.stdout: ceph-mgr-diskprediction-local-2:17.2.6-1123.g61431f6b.el8.noarch 2023-09-23T20:04:22.134 INFO:teuthology.orchestra.run.smithi149.stdout: libgfortran-8.5.0-10.1.el8_6.x86_64 2023-09-23T20:04:22.134 INFO:teuthology.orchestra.run.smithi149.stdout: libquadmath-8.5.0-10.1.el8_6.x86_64 2023-09-23T20:04:22.134 INFO:teuthology.orchestra.run.smithi149.stdout: openblas-0.3.15-3.el8.x86_64 2023-09-23T20:04:22.134 INFO:teuthology.orchestra.run.smithi149.stdout: openblas-threads-0.3.15-3.el8.x86_64 2023-09-23T20:04:22.134 INFO:teuthology.orchestra.run.smithi149.stdout: python3-numpy-1:1.14.3-10.el8.x86_64 2023-09-23T20:04:22.134 INFO:teuthology.orchestra.run.smithi149.stdout: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 2023-09-23T20:04:22.134 INFO:teuthology.orchestra.run.smithi149.stdout: python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd434d.x86_64 2023-09-23T20:04:22.135 INFO:teuthology.orchestra.run.smithi149.stdout: 2023-09-23T20:04:22.135 INFO:teuthology.orchestra.run.smithi149.stdout:Complete! 2023-09-23T20:04:22.301 DEBUG:teuthology.orchestra.run.smithi149:> sudo yum -y install ceph-mgr-rook 2023-09-23T20:04:22.652 INFO:teuthology.orchestra.run.smithi149.stdout:Updating Subscription Management repositories. 2023-09-23T20:04:22.777 INFO:teuthology.orchestra.run.smithi018.stdout: Verifying : librados-devel-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T20:04:22.778 INFO:teuthology.orchestra.run.smithi018.stdout:Last metadata expiration check: 0:03:00 ago on Sat 23 Sep 2023 08:01:22 PM UTC. 2023-09-23T20:04:23.831 INFO:teuthology.orchestra.run.smithi149.stdout:Last metadata expiration check: 0:02:21 ago on Sat 23 Sep 2023 08:02:02 PM UTC. 2023-09-23T20:04:25.209 INFO:teuthology.orchestra.run.smithi149.stdout:Dependencies resolved. 2023-09-23T20:04:25.210 INFO:teuthology.orchestra.run.smithi149.stdout:======================================================================================================== 2023-09-23T20:04:25.210 INFO:teuthology.orchestra.run.smithi149.stdout: Package Arch Version Repository Size 2023-09-23T20:04:25.210 INFO:teuthology.orchestra.run.smithi149.stdout:======================================================================================================== 2023-09-23T20:04:25.210 INFO:teuthology.orchestra.run.smithi149.stdout:Installing: 2023-09-23T20:04:25.211 INFO:teuthology.orchestra.run.smithi149.stdout: ceph-mgr-rook noarch 2:17.2.6-1123.g61431f6b.el8 ceph-noarch 47 k 2023-09-23T20:04:25.211 INFO:teuthology.orchestra.run.smithi149.stdout:Installing dependencies: 2023-09-23T20:04:25.211 INFO:teuthology.orchestra.run.smithi149.stdout: python3-cachetools noarch 3.1.1-4.el8 epel 33 k 2023-09-23T20:04:25.211 INFO:teuthology.orchestra.run.smithi149.stdout: python3-certifi noarch 2018.10.15-7.el8 epel 16 k 2023-09-23T20:04:25.211 INFO:teuthology.orchestra.run.smithi149.stdout: python3-google-auth noarch 1:1.1.1-10.el8 epel 81 k 2023-09-23T20:04:25.211 INFO:teuthology.orchestra.run.smithi149.stdout: python3-jsonpatch noarch 1.21-2.el8 rhel-8-for-x86_64-appstream-rpms 27 k 2023-09-23T20:04:25.211 INFO:teuthology.orchestra.run.smithi149.stdout: python3-jsonpointer noarch 1.10-11.el8 rhel-8-for-x86_64-appstream-rpms 20 k 2023-09-23T20:04:25.211 INFO:teuthology.orchestra.run.smithi149.stdout: python3-kubernetes noarch 1:11.0.0-6.el8 epel 1.2 M 2023-09-23T20:04:25.212 INFO:teuthology.orchestra.run.smithi149.stdout: python3-oauthlib noarch 2.1.0-1.el8 rhel-8-for-x86_64-baseos-rpms 155 k 2023-09-23T20:04:25.212 INFO:teuthology.orchestra.run.smithi149.stdout: python3-pyasn1 noarch 0.3.7-6.el8 rhel-8-for-x86_64-appstream-rpms 126 k 2023-09-23T20:04:25.212 INFO:teuthology.orchestra.run.smithi149.stdout: python3-pyasn1-modules noarch 0.3.7-6.el8 rhel-8-for-x86_64-appstream-rpms 110 k 2023-09-23T20:04:25.212 INFO:teuthology.orchestra.run.smithi149.stdout: python3-requests-oauthlib noarch 1.0.0-1.el8 rhel-8-for-x86_64-baseos-rpms 43 k 2023-09-23T20:04:25.212 INFO:teuthology.orchestra.run.smithi149.stdout: python3-rsa noarch 4.9-2.el8 epel 64 k 2023-09-23T20:04:25.212 INFO:teuthology.orchestra.run.smithi149.stdout: python3-websocket-client noarch 0.56.0-5.el8 epel 61 k 2023-09-23T20:04:25.212 INFO:teuthology.orchestra.run.smithi149.stdout: 2023-09-23T20:04:25.212 INFO:teuthology.orchestra.run.smithi149.stdout:Transaction Summary 2023-09-23T20:04:25.212 INFO:teuthology.orchestra.run.smithi149.stdout:======================================================================================================== 2023-09-23T20:04:25.212 INFO:teuthology.orchestra.run.smithi149.stdout:Install 13 Packages 2023-09-23T20:04:25.212 INFO:teuthology.orchestra.run.smithi149.stdout: 2023-09-23T20:04:25.213 INFO:teuthology.orchestra.run.smithi149.stdout:Total download size: 1.9 M 2023-09-23T20:04:25.213 INFO:teuthology.orchestra.run.smithi149.stdout:Installed size: 25 M 2023-09-23T20:04:25.213 INFO:teuthology.orchestra.run.smithi149.stdout:Downloading Packages: 2023-09-23T20:04:25.683 INFO:teuthology.orchestra.run.smithi149.stdout:(1/13): python3-certifi-2018.10.15-7.el8.noarch 130 kB/s | 16 kB 00:00 2023-09-23T20:04:25.708 INFO:teuthology.orchestra.run.smithi149.stdout:(2/13): python3-cachetools-3.1.1-4.el8.noarch.r 219 kB/s | 33 kB 00:00 2023-09-23T20:04:25.767 INFO:teuthology.orchestra.run.smithi149.stdout:(3/13): python3-google-auth-1.1.1-10.el8.noarch 972 kB/s | 81 kB 00:00 2023-09-23T20:04:25.817 INFO:teuthology.orchestra.run.smithi149.stdout:(4/13): ceph-mgr-rook-17.2.6-1123.g61431f6b.el8 182 kB/s | 47 kB 00:00 2023-09-23T20:04:25.842 INFO:teuthology.orchestra.run.smithi149.stdout:(5/13): python3-rsa-4.9-2.el8.noarch.rpm 854 kB/s | 64 kB 00:00 2023-09-23T20:04:25.951 INFO:teuthology.orchestra.run.smithi149.stdout:(6/13): python3-websocket-client-0.56.0-5.el8.n 458 kB/s | 61 kB 00:00 2023-09-23T20:04:25.976 INFO:teuthology.orchestra.run.smithi149.stdout:(7/13): python3-requests-oauthlib-1.0.0-1.el8.n 319 kB/s | 43 kB 00:00 2023-09-23T20:04:26.010 INFO:teuthology.orchestra.run.smithi149.stdout:(8/13): python3-kubernetes-11.0.0-6.el8.noarch. 3.9 MB/s | 1.2 MB 00:00 2023-09-23T20:04:26.110 INFO:teuthology.orchestra.run.smithi149.stdout:(9/13): python3-oauthlib-2.1.0-1.el8.noarch.rpm 975 kB/s | 155 kB 00:00 2023-09-23T20:04:26.127 INFO:teuthology.orchestra.run.smithi149.stdout:(10/13): python3-pyasn1-modules-0.3.7-6.el8.noa 759 kB/s | 110 kB 00:00 2023-09-23T20:04:26.185 INFO:teuthology.orchestra.run.smithi149.stdout:(11/13): python3-pyasn1-0.3.7-6.el8.noarch.rpm 1.6 MB/s | 126 kB 00:00 2023-09-23T20:04:26.202 INFO:teuthology.orchestra.run.smithi149.stdout:(12/13): python3-jsonpointer-1.10-11.el8.noarch 266 kB/s | 20 kB 00:00 2023-09-23T20:04:26.219 INFO:teuthology.orchestra.run.smithi149.stdout:(13/13): python3-jsonpatch-1.21-2.el8.noarch.rp 127 kB/s | 27 kB 00:00 2023-09-23T20:04:26.220 INFO:teuthology.orchestra.run.smithi149.stdout:-------------------------------------------------------------------------------- 2023-09-23T20:04:26.220 INFO:teuthology.orchestra.run.smithi149.stdout:Total 1.9 MB/s | 1.9 MB 00:01 2023-09-23T20:04:26.280 INFO:teuthology.orchestra.run.smithi149.stdout:Running transaction check 2023-09-23T20:04:26.295 INFO:teuthology.orchestra.run.smithi149.stdout:Transaction check succeeded. 2023-09-23T20:04:26.295 INFO:teuthology.orchestra.run.smithi149.stdout:Running transaction test 2023-09-23T20:04:26.366 INFO:teuthology.orchestra.run.smithi149.stdout:Transaction test succeeded. 2023-09-23T20:04:26.370 INFO:teuthology.orchestra.run.smithi149.stdout:Running transaction 2023-09-23T20:04:26.884 INFO:teuthology.orchestra.run.smithi149.stdout: Preparing : 1/1 2023-09-23T20:04:27.089 INFO:teuthology.orchestra.run.smithi149.stdout: Installing : python3-pyasn1-0.3.7-6.el8.noarch 1/13 2023-09-23T20:04:27.317 INFO:teuthology.orchestra.run.smithi149.stdout: Installing : python3-rsa-4.9-2.el8.noarch 2/13 2023-09-23T20:04:27.487 INFO:teuthology.orchestra.run.smithi149.stdout: Installing : python3-pyasn1-modules-0.3.7-6.el8.noarch 3/13 2023-09-23T20:04:27.655 INFO:teuthology.orchestra.run.smithi149.stdout: Installing : python3-jsonpointer-1.10-11.el8.noarch 4/13 2023-09-23T20:04:27.868 INFO:teuthology.orchestra.run.smithi149.stdout: Installing : python3-jsonpatch-1.21-2.el8.noarch 5/13 2023-09-23T20:04:27.942 INFO:teuthology.orchestra.run.smithi018.stdout:Installed products updated. 2023-09-23T20:04:28.094 INFO:teuthology.orchestra.run.smithi149.stdout: Installing : python3-oauthlib-2.1.0-1.el8.noarch 6/13 2023-09-23T20:04:28.144 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:04:28.144 INFO:teuthology.orchestra.run.smithi018.stdout:Installed: 2023-09-23T20:04:28.144 INFO:teuthology.orchestra.run.smithi018.stdout: librados-devel-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:04:28.145 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:04:28.145 INFO:teuthology.orchestra.run.smithi018.stdout:Complete! 2023-09-23T20:04:28.266 DEBUG:teuthology.orchestra.run.smithi018:> sudo yum -y install libcephfs2 2023-09-23T20:04:28.292 INFO:teuthology.orchestra.run.smithi149.stdout: Installing : python3-requests-oauthlib-1.0.0-1.el8.noarch 7/13 2023-09-23T20:04:28.480 INFO:teuthology.orchestra.run.smithi149.stdout: Installing : python3-websocket-client-0.56.0-5.el8.noarch 8/13 2023-09-23T20:04:28.637 INFO:teuthology.orchestra.run.smithi149.stdout: Installing : python3-certifi-2018.10.15-7.el8.noarch 9/13 2023-09-23T20:04:28.644 INFO:teuthology.orchestra.run.smithi018.stdout:Updating Subscription Management repositories. 2023-09-23T20:04:28.853 INFO:teuthology.orchestra.run.smithi149.stdout: Installing : python3-cachetools-3.1.1-4.el8.noarch 10/13 2023-09-23T20:04:29.929 INFO:teuthology.orchestra.run.smithi149.stdout: Installing : python3-google-auth-1:1.1.1-10.el8.noarch 11/13 2023-09-23T20:04:29.957 INFO:teuthology.orchestra.run.smithi018.stdout:Last metadata expiration check: 0:03:07 ago on Sat 23 Sep 2023 08:01:22 PM UTC. 2023-09-23T20:04:30.172 INFO:teuthology.orchestra.run.smithi149.stdout: Installing : python3-kubernetes-1:11.0.0-6.el8.noarch 12/13 2023-09-23T20:04:30.339 INFO:teuthology.orchestra.run.smithi149.stdout: Installing : ceph-mgr-rook-2:17.2.6-1123.g61431f6b.el8.noarch 13/13 2023-09-23T20:04:30.574 INFO:teuthology.orchestra.run.smithi149.stdout: Running scriptlet: ceph-mgr-rook-2:17.2.6-1123.g61431f6b.el8.noarch 13/13 2023-09-23T20:04:30.574 INFO:teuthology.orchestra.run.smithi149.stdout: Verifying : ceph-mgr-rook-2:17.2.6-1123.g61431f6b.el8.noarch 1/13 2023-09-23T20:04:30.574 INFO:teuthology.orchestra.run.smithi149.stdout: Verifying : python3-cachetools-3.1.1-4.el8.noarch 2/13 2023-09-23T20:04:30.574 INFO:teuthology.orchestra.run.smithi149.stdout: Verifying : python3-certifi-2018.10.15-7.el8.noarch 3/13 2023-09-23T20:04:30.575 INFO:teuthology.orchestra.run.smithi149.stdout: Verifying : python3-google-auth-1:1.1.1-10.el8.noarch 4/13 2023-09-23T20:04:30.575 INFO:teuthology.orchestra.run.smithi149.stdout: Verifying : python3-kubernetes-1:11.0.0-6.el8.noarch 5/13 2023-09-23T20:04:30.575 INFO:teuthology.orchestra.run.smithi149.stdout: Verifying : python3-rsa-4.9-2.el8.noarch 6/13 2023-09-23T20:04:30.575 INFO:teuthology.orchestra.run.smithi149.stdout: Verifying : python3-websocket-client-0.56.0-5.el8.noarch 7/13 2023-09-23T20:04:30.575 INFO:teuthology.orchestra.run.smithi149.stdout: Verifying : python3-requests-oauthlib-1.0.0-1.el8.noarch 8/13 2023-09-23T20:04:30.575 INFO:teuthology.orchestra.run.smithi149.stdout: Verifying : python3-oauthlib-2.1.0-1.el8.noarch 9/13 2023-09-23T20:04:30.575 INFO:teuthology.orchestra.run.smithi149.stdout: Verifying : python3-pyasn1-modules-0.3.7-6.el8.noarch 10/13 2023-09-23T20:04:30.575 INFO:teuthology.orchestra.run.smithi149.stdout: Verifying : python3-jsonpatch-1.21-2.el8.noarch 11/13 2023-09-23T20:04:30.575 INFO:teuthology.orchestra.run.smithi149.stdout: Verifying : python3-pyasn1-0.3.7-6.el8.noarch 12/13 2023-09-23T20:04:31.364 INFO:teuthology.orchestra.run.smithi018.stdout:Package libcephfs2-2:17.2.6-1123.g61431f6b.el8.x86_64 is already installed. 2023-09-23T20:04:31.485 INFO:teuthology.orchestra.run.smithi018.stdout:Dependencies resolved. 2023-09-23T20:04:31.486 INFO:teuthology.orchestra.run.smithi018.stdout:Nothing to do. 2023-09-23T20:04:31.486 INFO:teuthology.orchestra.run.smithi018.stdout:Complete! 2023-09-23T20:04:31.600 DEBUG:teuthology.orchestra.run.smithi018:> sudo yum -y install libcephfs-devel 2023-09-23T20:04:31.685 INFO:teuthology.orchestra.run.smithi149.stdout: Verifying : python3-jsonpointer-1.10-11.el8.noarch 13/13 2023-09-23T20:04:31.686 INFO:teuthology.orchestra.run.smithi149.stdout:Last metadata expiration check: 0:02:29 ago on Sat 23 Sep 2023 08:02:02 PM UTC. 2023-09-23T20:04:31.978 INFO:teuthology.orchestra.run.smithi018.stdout:Updating Subscription Management repositories. 2023-09-23T20:04:33.513 INFO:teuthology.orchestra.run.smithi018.stdout:Last metadata expiration check: 0:03:11 ago on Sat 23 Sep 2023 08:01:22 PM UTC. 2023-09-23T20:04:34.998 INFO:teuthology.orchestra.run.smithi018.stdout:Dependencies resolved. 2023-09-23T20:04:34.999 INFO:teuthology.orchestra.run.smithi018.stdout:================================================================================ 2023-09-23T20:04:34.999 INFO:teuthology.orchestra.run.smithi018.stdout: Package Arch Version Repo Size 2023-09-23T20:04:34.999 INFO:teuthology.orchestra.run.smithi018.stdout:================================================================================ 2023-09-23T20:04:34.999 INFO:teuthology.orchestra.run.smithi018.stdout:Installing: 2023-09-23T20:04:34.999 INFO:teuthology.orchestra.run.smithi018.stdout: libcephfs-devel x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 25 k 2023-09-23T20:04:34.999 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:04:34.999 INFO:teuthology.orchestra.run.smithi018.stdout:Transaction Summary 2023-09-23T20:04:34.999 INFO:teuthology.orchestra.run.smithi018.stdout:================================================================================ 2023-09-23T20:04:34.999 INFO:teuthology.orchestra.run.smithi018.stdout:Install 1 Package 2023-09-23T20:04:34.999 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:04:35.000 INFO:teuthology.orchestra.run.smithi018.stdout:Total download size: 25 k 2023-09-23T20:04:35.000 INFO:teuthology.orchestra.run.smithi018.stdout:Installed size: 109 k 2023-09-23T20:04:35.000 INFO:teuthology.orchestra.run.smithi018.stdout:Downloading Packages: 2023-09-23T20:04:35.142 INFO:teuthology.orchestra.run.smithi018.stdout:libcephfs-devel-17.2.6-1123.g61431f6b.el8.x86_6 176 kB/s | 25 kB 00:00 2023-09-23T20:04:35.142 INFO:teuthology.orchestra.run.smithi018.stdout:-------------------------------------------------------------------------------- 2023-09-23T20:04:35.142 INFO:teuthology.orchestra.run.smithi018.stdout:Total 174 kB/s | 25 kB 00:00 2023-09-23T20:04:35.142 INFO:teuthology.orchestra.run.smithi018.stdout:Running transaction check 2023-09-23T20:04:35.146 INFO:teuthology.orchestra.run.smithi018.stdout:Transaction check succeeded. 2023-09-23T20:04:35.146 INFO:teuthology.orchestra.run.smithi018.stdout:Running transaction test 2023-09-23T20:04:35.148 INFO:teuthology.orchestra.run.smithi018.stdout:Transaction test succeeded. 2023-09-23T20:04:35.152 INFO:teuthology.orchestra.run.smithi018.stdout:Running transaction 2023-09-23T20:04:35.206 INFO:teuthology.orchestra.run.smithi018.stdout: Preparing : 1/1 2023-09-23T20:04:35.350 INFO:teuthology.orchestra.run.smithi018.stdout: Installing : libcephfs-devel-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T20:04:35.640 INFO:teuthology.orchestra.run.smithi018.stdout: Running scriptlet: libcephfs-devel-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T20:04:36.230 INFO:teuthology.orchestra.run.smithi149.stdout:Installed products updated. 2023-09-23T20:04:36.416 INFO:teuthology.orchestra.run.smithi149.stdout: 2023-09-23T20:04:36.416 INFO:teuthology.orchestra.run.smithi149.stdout:Installed: 2023-09-23T20:04:36.416 INFO:teuthology.orchestra.run.smithi149.stdout: ceph-mgr-rook-2:17.2.6-1123.g61431f6b.el8.noarch 2023-09-23T20:04:36.416 INFO:teuthology.orchestra.run.smithi149.stdout: python3-cachetools-3.1.1-4.el8.noarch 2023-09-23T20:04:36.417 INFO:teuthology.orchestra.run.smithi149.stdout: python3-certifi-2018.10.15-7.el8.noarch 2023-09-23T20:04:36.417 INFO:teuthology.orchestra.run.smithi149.stdout: python3-google-auth-1:1.1.1-10.el8.noarch 2023-09-23T20:04:36.417 INFO:teuthology.orchestra.run.smithi149.stdout: python3-jsonpatch-1.21-2.el8.noarch 2023-09-23T20:04:36.417 INFO:teuthology.orchestra.run.smithi149.stdout: python3-jsonpointer-1.10-11.el8.noarch 2023-09-23T20:04:36.417 INFO:teuthology.orchestra.run.smithi149.stdout: python3-kubernetes-1:11.0.0-6.el8.noarch 2023-09-23T20:04:36.417 INFO:teuthology.orchestra.run.smithi149.stdout: python3-oauthlib-2.1.0-1.el8.noarch 2023-09-23T20:04:36.417 INFO:teuthology.orchestra.run.smithi149.stdout: python3-pyasn1-0.3.7-6.el8.noarch 2023-09-23T20:04:36.417 INFO:teuthology.orchestra.run.smithi149.stdout: python3-pyasn1-modules-0.3.7-6.el8.noarch 2023-09-23T20:04:36.417 INFO:teuthology.orchestra.run.smithi149.stdout: python3-requests-oauthlib-1.0.0-1.el8.noarch 2023-09-23T20:04:36.417 INFO:teuthology.orchestra.run.smithi149.stdout: python3-rsa-4.9-2.el8.noarch 2023-09-23T20:04:36.417 INFO:teuthology.orchestra.run.smithi149.stdout: python3-websocket-client-0.56.0-5.el8.noarch 2023-09-23T20:04:36.418 INFO:teuthology.orchestra.run.smithi149.stdout: 2023-09-23T20:04:36.418 INFO:teuthology.orchestra.run.smithi149.stdout:Complete! 2023-09-23T20:04:36.569 DEBUG:teuthology.orchestra.run.smithi149:> sudo yum -y install ceph-mgr-cephadm 2023-09-23T20:04:36.728 INFO:teuthology.orchestra.run.smithi018.stdout: Verifying : libcephfs-devel-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T20:04:36.729 INFO:teuthology.orchestra.run.smithi018.stdout:Last metadata expiration check: 0:03:14 ago on Sat 23 Sep 2023 08:01:22 PM UTC. 2023-09-23T20:04:36.924 INFO:teuthology.orchestra.run.smithi149.stdout:Updating Subscription Management repositories. 2023-09-23T20:04:38.230 INFO:teuthology.orchestra.run.smithi149.stdout:Last metadata expiration check: 0:02:36 ago on Sat 23 Sep 2023 08:02:02 PM UTC. 2023-09-23T20:04:39.620 INFO:teuthology.orchestra.run.smithi149.stdout:Dependencies resolved. 2023-09-23T20:04:39.621 INFO:teuthology.orchestra.run.smithi149.stdout:=============================================================================================== 2023-09-23T20:04:39.621 INFO:teuthology.orchestra.run.smithi149.stdout: Package Arch Version Repository Size 2023-09-23T20:04:39.621 INFO:teuthology.orchestra.run.smithi149.stdout:=============================================================================================== 2023-09-23T20:04:39.621 INFO:teuthology.orchestra.run.smithi149.stdout:Installing: 2023-09-23T20:04:39.621 INFO:teuthology.orchestra.run.smithi149.stdout: ceph-mgr-cephadm noarch 2:17.2.6-1123.g61431f6b.el8 ceph-noarch 121 k 2023-09-23T20:04:39.621 INFO:teuthology.orchestra.run.smithi149.stdout:Installing dependencies: 2023-09-23T20:04:39.621 INFO:teuthology.orchestra.run.smithi149.stdout: python3-asyncssh noarch 2.7.0-2.el8 epel 476 k 2023-09-23T20:04:39.621 INFO:teuthology.orchestra.run.smithi149.stdout: python3-babel noarch 2.5.1-7.el8 rhel-8-for-x86_64-appstream-rpms 4.8 M 2023-09-23T20:04:39.622 INFO:teuthology.orchestra.run.smithi149.stdout: python3-jinja2 noarch 2.10.1-3.el8 rhel-8-for-x86_64-appstream-rpms 538 k 2023-09-23T20:04:39.622 INFO:teuthology.orchestra.run.smithi149.stdout: python3-natsort noarch 7.1.1-2.el8 epel 60 k 2023-09-23T20:04:39.622 INFO:teuthology.orchestra.run.smithi149.stdout: 2023-09-23T20:04:39.622 INFO:teuthology.orchestra.run.smithi149.stdout:Transaction Summary 2023-09-23T20:04:39.622 INFO:teuthology.orchestra.run.smithi149.stdout:=============================================================================================== 2023-09-23T20:04:39.622 INFO:teuthology.orchestra.run.smithi149.stdout:Install 5 Packages 2023-09-23T20:04:39.622 INFO:teuthology.orchestra.run.smithi149.stdout: 2023-09-23T20:04:39.623 INFO:teuthology.orchestra.run.smithi149.stdout:Total download size: 6.0 M 2023-09-23T20:04:39.623 INFO:teuthology.orchestra.run.smithi149.stdout:Installed size: 26 M 2023-09-23T20:04:39.623 INFO:teuthology.orchestra.run.smithi149.stdout:Downloading Packages: 2023-09-23T20:04:39.885 INFO:teuthology.orchestra.run.smithi149.stdout:(1/5): python3-natsort-7.1.1-2.el8.noarch.rpm 686 kB/s | 60 kB 00:00 2023-09-23T20:04:39.935 INFO:teuthology.orchestra.run.smithi149.stdout:(2/5): python3-asyncssh-2.7.0-2.el8.noarch.rpm 3.4 MB/s | 476 kB 00:00 2023-09-23T20:04:40.110 INFO:teuthology.orchestra.run.smithi149.stdout:(3/5): python3-jinja2-2.10.1-3.el8.noarch.rpm 2.3 MB/s | 538 kB 00:00 2023-09-23T20:04:40.187 INFO:teuthology.orchestra.run.smithi149.stdout:(4/5): ceph-mgr-cephadm-17.2.6-1123.g61431f6b.e 358 kB/s | 121 kB 00:00 2023-09-23T20:04:40.319 INFO:teuthology.orchestra.run.smithi149.stdout:(5/5): python3-babel-2.5.1-7.el8.noarch.rpm 13 MB/s | 4.8 MB 00:00 2023-09-23T20:04:40.319 INFO:teuthology.orchestra.run.smithi149.stdout:-------------------------------------------------------------------------------- 2023-09-23T20:04:40.320 INFO:teuthology.orchestra.run.smithi149.stdout:Total 8.5 MB/s | 6.0 MB 00:00 2023-09-23T20:04:40.372 INFO:teuthology.orchestra.run.smithi149.stdout:Running transaction check 2023-09-23T20:04:40.383 INFO:teuthology.orchestra.run.smithi149.stdout:Transaction check succeeded. 2023-09-23T20:04:40.383 INFO:teuthology.orchestra.run.smithi149.stdout:Running transaction test 2023-09-23T20:04:40.464 INFO:teuthology.orchestra.run.smithi149.stdout:Transaction test succeeded. 2023-09-23T20:04:40.467 INFO:teuthology.orchestra.run.smithi149.stdout:Running transaction 2023-09-23T20:04:41.419 INFO:teuthology.orchestra.run.smithi149.stdout: Preparing : 1/1 2023-09-23T20:04:41.749 INFO:teuthology.orchestra.run.smithi149.stdout: Installing : python3-babel-2.5.1-7.el8.noarch 1/5 2023-09-23T20:04:41.862 INFO:teuthology.orchestra.run.smithi018.stdout:Installed products updated. 2023-09-23T20:04:41.964 INFO:teuthology.orchestra.run.smithi149.stdout: Installing : python3-jinja2-2.10.1-3.el8.noarch 2/5 2023-09-23T20:04:42.064 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:04:42.064 INFO:teuthology.orchestra.run.smithi018.stdout:Installed: 2023-09-23T20:04:42.064 INFO:teuthology.orchestra.run.smithi018.stdout: libcephfs-devel-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:04:42.064 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:04:42.064 INFO:teuthology.orchestra.run.smithi018.stdout:Complete! 2023-09-23T20:04:42.195 DEBUG:teuthology.orchestra.run.smithi018:> sudo yum -y install librados2 2023-09-23T20:04:42.284 INFO:teuthology.orchestra.run.smithi149.stdout: Installing : python3-natsort-7.1.1-2.el8.noarch 3/5 2023-09-23T20:04:42.547 INFO:teuthology.orchestra.run.smithi149.stdout: Installing : python3-asyncssh-2.7.0-2.el8.noarch 4/5 2023-09-23T20:04:42.573 INFO:teuthology.orchestra.run.smithi018.stdout:Updating Subscription Management repositories. 2023-09-23T20:04:42.737 INFO:teuthology.orchestra.run.smithi149.stdout: Installing : ceph-mgr-cephadm-2:17.2.6-1123.g61431f6b.el8.noarch 5/5 2023-09-23T20:04:42.978 INFO:teuthology.orchestra.run.smithi149.stdout: Running scriptlet: ceph-mgr-cephadm-2:17.2.6-1123.g61431f6b.el8.noarch 5/5 2023-09-23T20:04:42.979 INFO:teuthology.orchestra.run.smithi149.stdout: Verifying : ceph-mgr-cephadm-2:17.2.6-1123.g61431f6b.el8.noarch 1/5 2023-09-23T20:04:42.979 INFO:teuthology.orchestra.run.smithi149.stdout: Verifying : python3-asyncssh-2.7.0-2.el8.noarch 2/5 2023-09-23T20:04:42.979 INFO:teuthology.orchestra.run.smithi149.stdout: Verifying : python3-natsort-7.1.1-2.el8.noarch 3/5 2023-09-23T20:04:42.979 INFO:teuthology.orchestra.run.smithi149.stdout: Verifying : python3-jinja2-2.10.1-3.el8.noarch 4/5 2023-09-23T20:04:44.016 INFO:teuthology.orchestra.run.smithi149.stdout: Verifying : python3-babel-2.5.1-7.el8.noarch 5/5 2023-09-23T20:04:44.017 INFO:teuthology.orchestra.run.smithi149.stdout:Last metadata expiration check: 0:02:42 ago on Sat 23 Sep 2023 08:02:02 PM UTC. 2023-09-23T20:04:44.122 INFO:teuthology.orchestra.run.smithi018.stdout:Last metadata expiration check: 0:03:22 ago on Sat 23 Sep 2023 08:01:22 PM UTC. 2023-09-23T20:04:45.553 INFO:teuthology.orchestra.run.smithi018.stdout:Package librados2-2:17.2.6-1123.g61431f6b.el8.x86_64 is already installed. 2023-09-23T20:04:45.674 INFO:teuthology.orchestra.run.smithi018.stdout:Dependencies resolved. 2023-09-23T20:04:45.675 INFO:teuthology.orchestra.run.smithi018.stdout:Nothing to do. 2023-09-23T20:04:45.675 INFO:teuthology.orchestra.run.smithi018.stdout:Complete! 2023-09-23T20:04:45.795 DEBUG:teuthology.orchestra.run.smithi018:> sudo yum -y install librbd1 2023-09-23T20:04:46.174 INFO:teuthology.orchestra.run.smithi018.stdout:Updating Subscription Management repositories. 2023-09-23T20:04:47.691 INFO:teuthology.orchestra.run.smithi018.stdout:Last metadata expiration check: 0:03:25 ago on Sat 23 Sep 2023 08:01:22 PM UTC. 2023-09-23T20:04:48.605 INFO:teuthology.orchestra.run.smithi149.stdout:Installed products updated. 2023-09-23T20:04:48.796 INFO:teuthology.orchestra.run.smithi149.stdout: 2023-09-23T20:04:48.797 INFO:teuthology.orchestra.run.smithi149.stdout:Installed: 2023-09-23T20:04:48.797 INFO:teuthology.orchestra.run.smithi149.stdout: ceph-mgr-cephadm-2:17.2.6-1123.g61431f6b.el8.noarch 2023-09-23T20:04:48.797 INFO:teuthology.orchestra.run.smithi149.stdout: python3-asyncssh-2.7.0-2.el8.noarch 2023-09-23T20:04:48.797 INFO:teuthology.orchestra.run.smithi149.stdout: python3-babel-2.5.1-7.el8.noarch 2023-09-23T20:04:48.797 INFO:teuthology.orchestra.run.smithi149.stdout: python3-jinja2-2.10.1-3.el8.noarch 2023-09-23T20:04:48.797 INFO:teuthology.orchestra.run.smithi149.stdout: python3-natsort-7.1.1-2.el8.noarch 2023-09-23T20:04:48.797 INFO:teuthology.orchestra.run.smithi149.stdout: 2023-09-23T20:04:48.797 INFO:teuthology.orchestra.run.smithi149.stdout:Complete! 2023-09-23T20:04:48.944 DEBUG:teuthology.orchestra.run.smithi149:> sudo yum -y install ceph-fuse 2023-09-23T20:04:49.087 INFO:teuthology.orchestra.run.smithi018.stdout:Package librbd1-2:17.2.6-1123.g61431f6b.el8.x86_64 is already installed. 2023-09-23T20:04:49.211 INFO:teuthology.orchestra.run.smithi018.stdout:Dependencies resolved. 2023-09-23T20:04:49.213 INFO:teuthology.orchestra.run.smithi018.stdout:Nothing to do. 2023-09-23T20:04:49.213 INFO:teuthology.orchestra.run.smithi018.stdout:Complete! 2023-09-23T20:04:49.300 INFO:teuthology.orchestra.run.smithi149.stdout:Updating Subscription Management repositories. 2023-09-23T20:04:49.326 DEBUG:teuthology.orchestra.run.smithi018:> sudo yum -y install python3-rados 2023-09-23T20:04:49.704 INFO:teuthology.orchestra.run.smithi018.stdout:Updating Subscription Management repositories. 2023-09-23T20:04:50.157 INFO:teuthology.orchestra.run.smithi149.stdout:Last metadata expiration check: 0:02:48 ago on Sat 23 Sep 2023 08:02:02 PM UTC. 2023-09-23T20:04:50.566 INFO:teuthology.orchestra.run.smithi018.stdout:Last metadata expiration check: 0:03:28 ago on Sat 23 Sep 2023 08:01:22 PM UTC. 2023-09-23T20:04:51.552 INFO:teuthology.orchestra.run.smithi149.stdout:Dependencies resolved. 2023-09-23T20:04:51.552 INFO:teuthology.orchestra.run.smithi149.stdout:================================================================================ 2023-09-23T20:04:51.552 INFO:teuthology.orchestra.run.smithi149.stdout: Package Arch Version Repository Size 2023-09-23T20:04:51.553 INFO:teuthology.orchestra.run.smithi149.stdout:================================================================================ 2023-09-23T20:04:51.553 INFO:teuthology.orchestra.run.smithi149.stdout:Installing: 2023-09-23T20:04:51.553 INFO:teuthology.orchestra.run.smithi149.stdout: ceph-fuse 2023-09-23T20:04:51.553 INFO:teuthology.orchestra.run.smithi149.stdout: x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 874 k 2023-09-23T20:04:51.553 INFO:teuthology.orchestra.run.smithi149.stdout:Installing dependencies: 2023-09-23T20:04:51.553 INFO:teuthology.orchestra.run.smithi149.stdout: fuse x86_64 2.9.7-15.el8 rhel-8-for-x86_64-baseos-rpms 83 k 2023-09-23T20:04:51.553 INFO:teuthology.orchestra.run.smithi149.stdout: 2023-09-23T20:04:51.553 INFO:teuthology.orchestra.run.smithi149.stdout:Transaction Summary 2023-09-23T20:04:51.553 INFO:teuthology.orchestra.run.smithi149.stdout:================================================================================ 2023-09-23T20:04:51.553 INFO:teuthology.orchestra.run.smithi149.stdout:Install 2 Packages 2023-09-23T20:04:51.553 INFO:teuthology.orchestra.run.smithi149.stdout: 2023-09-23T20:04:51.554 INFO:teuthology.orchestra.run.smithi149.stdout:Total download size: 957 k 2023-09-23T20:04:51.554 INFO:teuthology.orchestra.run.smithi149.stdout:Installed size: 3.0 M 2023-09-23T20:04:51.554 INFO:teuthology.orchestra.run.smithi149.stdout:Downloading Packages: 2023-09-23T20:04:51.764 INFO:teuthology.orchestra.run.smithi149.stdout:(1/2): fuse-2.9.7-15.el8.x86_64.rpm 396 kB/s | 83 kB 00:00 2023-09-23T20:04:51.955 INFO:teuthology.orchestra.run.smithi149.stdout:(2/2): ceph-fuse-17.2.6-1123.g61431f6b.el8.x86_ 2.1 MB/s | 874 kB 00:00 2023-09-23T20:04:51.956 INFO:teuthology.orchestra.run.smithi149.stdout:-------------------------------------------------------------------------------- 2023-09-23T20:04:51.956 INFO:teuthology.orchestra.run.smithi149.stdout:Total 2.3 MB/s | 957 kB 00:00 2023-09-23T20:04:51.967 INFO:teuthology.orchestra.run.smithi149.stdout:Running transaction check 2023-09-23T20:04:51.987 INFO:teuthology.orchestra.run.smithi149.stdout:Transaction check succeeded. 2023-09-23T20:04:51.987 INFO:teuthology.orchestra.run.smithi149.stdout:Running transaction test 2023-09-23T20:04:52.004 INFO:teuthology.orchestra.run.smithi018.stdout:Package python3-rados-2:17.2.6-1123.g61431f6b.el8.x86_64 is already installed. 2023-09-23T20:04:52.062 INFO:teuthology.orchestra.run.smithi149.stdout:Transaction test succeeded. 2023-09-23T20:04:52.066 INFO:teuthology.orchestra.run.smithi149.stdout:Running transaction 2023-09-23T20:04:52.126 INFO:teuthology.orchestra.run.smithi018.stdout:Dependencies resolved. 2023-09-23T20:04:52.127 INFO:teuthology.orchestra.run.smithi018.stdout:Nothing to do. 2023-09-23T20:04:52.127 INFO:teuthology.orchestra.run.smithi018.stdout:Complete! 2023-09-23T20:04:52.221 INFO:teuthology.orchestra.run.smithi149.stdout: Preparing : 1/1 2023-09-23T20:04:52.244 DEBUG:teuthology.orchestra.run.smithi018:> sudo yum -y install python3-rgw 2023-09-23T20:04:52.520 INFO:teuthology.orchestra.run.smithi149.stdout: Installing : fuse-2.9.7-15.el8.x86_64 1/2 2023-09-23T20:04:52.623 INFO:teuthology.orchestra.run.smithi018.stdout:Updating Subscription Management repositories. 2023-09-23T20:04:52.659 INFO:teuthology.orchestra.run.smithi149.stdout: Installing : ceph-fuse-2:17.2.6-1123.g61431f6b.el8.x86_64 2/2 2023-09-23T20:04:53.011 INFO:teuthology.orchestra.run.smithi149.stdout: Running scriptlet: ceph-fuse-2:17.2.6-1123.g61431f6b.el8.x86_64 2/2 2023-09-23T20:04:53.012 INFO:teuthology.orchestra.run.smithi149.stdout: Verifying : ceph-fuse-2:17.2.6-1123.g61431f6b.el8.x86_64 1/2 2023-09-23T20:04:54.079 INFO:teuthology.orchestra.run.smithi149.stdout: Verifying : fuse-2.9.7-15.el8.x86_64 2/2 2023-09-23T20:04:54.079 INFO:teuthology.orchestra.run.smithi149.stdout:Last metadata expiration check: 0:02:52 ago on Sat 23 Sep 2023 08:02:02 PM UTC. 2023-09-23T20:04:54.239 INFO:teuthology.orchestra.run.smithi018.stdout:Last metadata expiration check: 0:03:32 ago on Sat 23 Sep 2023 08:01:22 PM UTC. 2023-09-23T20:04:55.671 INFO:teuthology.orchestra.run.smithi018.stdout:Package python3-rgw-2:17.2.6-1123.g61431f6b.el8.x86_64 is already installed. 2023-09-23T20:04:55.793 INFO:teuthology.orchestra.run.smithi018.stdout:Dependencies resolved. 2023-09-23T20:04:55.794 INFO:teuthology.orchestra.run.smithi018.stdout:Nothing to do. 2023-09-23T20:04:55.794 INFO:teuthology.orchestra.run.smithi018.stdout:Complete! 2023-09-23T20:04:55.911 DEBUG:teuthology.orchestra.run.smithi018:> sudo yum -y install python3-cephfs 2023-09-23T20:04:56.289 INFO:teuthology.orchestra.run.smithi018.stdout:Updating Subscription Management repositories. 2023-09-23T20:04:57.828 INFO:teuthology.orchestra.run.smithi018.stdout:Last metadata expiration check: 0:03:35 ago on Sat 23 Sep 2023 08:01:22 PM UTC. 2023-09-23T20:04:58.900 INFO:teuthology.orchestra.run.smithi149.stdout:Installed products updated. 2023-09-23T20:04:59.087 INFO:teuthology.orchestra.run.smithi149.stdout: 2023-09-23T20:04:59.088 INFO:teuthology.orchestra.run.smithi149.stdout:Installed: 2023-09-23T20:04:59.088 INFO:teuthology.orchestra.run.smithi149.stdout: ceph-fuse-2:17.2.6-1123.g61431f6b.el8.x86_64 fuse-2.9.7-15.el8.x86_64 2023-09-23T20:04:59.088 INFO:teuthology.orchestra.run.smithi149.stdout: 2023-09-23T20:04:59.088 INFO:teuthology.orchestra.run.smithi149.stdout:Complete! 2023-09-23T20:04:59.233 DEBUG:teuthology.orchestra.run.smithi149:> sudo yum -y install librados-devel 2023-09-23T20:04:59.258 INFO:teuthology.orchestra.run.smithi018.stdout:Package python3-cephfs-2:17.2.6-1123.g61431f6b.el8.x86_64 is already installed. 2023-09-23T20:04:59.378 INFO:teuthology.orchestra.run.smithi018.stdout:Dependencies resolved. 2023-09-23T20:04:59.379 INFO:teuthology.orchestra.run.smithi018.stdout:Nothing to do. 2023-09-23T20:04:59.379 INFO:teuthology.orchestra.run.smithi018.stdout:Complete! 2023-09-23T20:04:59.501 DEBUG:teuthology.orchestra.run.smithi018:> sudo yum -y install python3-rbd 2023-09-23T20:04:59.591 INFO:teuthology.orchestra.run.smithi149.stdout:Updating Subscription Management repositories. 2023-09-23T20:04:59.877 INFO:teuthology.orchestra.run.smithi018.stdout:Updating Subscription Management repositories. 2023-09-23T20:05:00.502 INFO:teuthology.orchestra.run.smithi149.stdout:Last metadata expiration check: 0:02:58 ago on Sat 23 Sep 2023 08:02:02 PM UTC. 2023-09-23T20:05:00.791 INFO:teuthology.orchestra.run.smithi018.stdout:Last metadata expiration check: 0:03:38 ago on Sat 23 Sep 2023 08:01:22 PM UTC. 2023-09-23T20:05:01.894 INFO:teuthology.orchestra.run.smithi149.stdout:Dependencies resolved. 2023-09-23T20:05:01.895 INFO:teuthology.orchestra.run.smithi149.stdout:================================================================================ 2023-09-23T20:05:01.895 INFO:teuthology.orchestra.run.smithi149.stdout: Package Arch Version Repo Size 2023-09-23T20:05:01.895 INFO:teuthology.orchestra.run.smithi149.stdout:================================================================================ 2023-09-23T20:05:01.895 INFO:teuthology.orchestra.run.smithi149.stdout:Installing: 2023-09-23T20:05:01.895 INFO:teuthology.orchestra.run.smithi149.stdout: librados-devel x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 130 k 2023-09-23T20:05:01.896 INFO:teuthology.orchestra.run.smithi149.stdout: 2023-09-23T20:05:01.896 INFO:teuthology.orchestra.run.smithi149.stdout:Transaction Summary 2023-09-23T20:05:01.896 INFO:teuthology.orchestra.run.smithi149.stdout:================================================================================ 2023-09-23T20:05:01.896 INFO:teuthology.orchestra.run.smithi149.stdout:Install 1 Package 2023-09-23T20:05:01.896 INFO:teuthology.orchestra.run.smithi149.stdout: 2023-09-23T20:05:01.896 INFO:teuthology.orchestra.run.smithi149.stdout:Total download size: 130 k 2023-09-23T20:05:01.896 INFO:teuthology.orchestra.run.smithi149.stdout:Installed size: 464 k 2023-09-23T20:05:01.896 INFO:teuthology.orchestra.run.smithi149.stdout:Downloading Packages: 2023-09-23T20:05:02.192 INFO:teuthology.orchestra.run.smithi149.stdout:librados-devel-17.2.6-1123.g61431f6b.el8.x86_64 438 kB/s | 130 kB 00:00 2023-09-23T20:05:02.193 INFO:teuthology.orchestra.run.smithi149.stdout:-------------------------------------------------------------------------------- 2023-09-23T20:05:02.193 INFO:teuthology.orchestra.run.smithi149.stdout:Total 437 kB/s | 130 kB 00:00 2023-09-23T20:05:02.193 INFO:teuthology.orchestra.run.smithi149.stdout:Running transaction check 2023-09-23T20:05:02.199 INFO:teuthology.orchestra.run.smithi149.stdout:Transaction check succeeded. 2023-09-23T20:05:02.199 INFO:teuthology.orchestra.run.smithi149.stdout:Running transaction test 2023-09-23T20:05:02.241 INFO:teuthology.orchestra.run.smithi018.stdout:Package python3-rbd-2:17.2.6-1123.g61431f6b.el8.x86_64 is already installed. 2023-09-23T20:05:02.252 INFO:teuthology.orchestra.run.smithi149.stdout:Transaction test succeeded. 2023-09-23T20:05:02.256 INFO:teuthology.orchestra.run.smithi149.stdout:Running transaction 2023-09-23T20:05:02.360 INFO:teuthology.orchestra.run.smithi018.stdout:Dependencies resolved. 2023-09-23T20:05:02.361 INFO:teuthology.orchestra.run.smithi018.stdout:Nothing to do. 2023-09-23T20:05:02.361 INFO:teuthology.orchestra.run.smithi018.stdout:Complete! 2023-09-23T20:05:02.377 INFO:teuthology.orchestra.run.smithi149.stdout: Preparing : 1/1 2023-09-23T20:05:02.479 DEBUG:teuthology.orchestra.run.smithi018:> sudo yum -y install rbd-fuse 2023-09-23T20:05:02.532 INFO:teuthology.orchestra.run.smithi149.stdout: Installing : librados-devel-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T20:05:02.778 INFO:teuthology.orchestra.run.smithi149.stdout: Running scriptlet: librados-devel-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T20:05:02.854 INFO:teuthology.orchestra.run.smithi018.stdout:Updating Subscription Management repositories. 2023-09-23T20:05:04.194 INFO:teuthology.orchestra.run.smithi149.stdout: Verifying : librados-devel-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T20:05:04.194 INFO:teuthology.orchestra.run.smithi149.stdout:Last metadata expiration check: 0:03:02 ago on Sat 23 Sep 2023 08:02:02 PM UTC. 2023-09-23T20:05:04.364 INFO:teuthology.orchestra.run.smithi018.stdout:Last metadata expiration check: 0:03:42 ago on Sat 23 Sep 2023 08:01:22 PM UTC. 2023-09-23T20:05:05.908 INFO:teuthology.orchestra.run.smithi018.stdout:Dependencies resolved. 2023-09-23T20:05:05.908 INFO:teuthology.orchestra.run.smithi018.stdout:================================================================================ 2023-09-23T20:05:05.908 INFO:teuthology.orchestra.run.smithi018.stdout: Package Architecture Version Repository Size 2023-09-23T20:05:05.908 INFO:teuthology.orchestra.run.smithi018.stdout:================================================================================ 2023-09-23T20:05:05.909 INFO:teuthology.orchestra.run.smithi018.stdout:Installing: 2023-09-23T20:05:05.909 INFO:teuthology.orchestra.run.smithi018.stdout: rbd-fuse x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 90 k 2023-09-23T20:05:05.909 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:05:05.909 INFO:teuthology.orchestra.run.smithi018.stdout:Transaction Summary 2023-09-23T20:05:05.909 INFO:teuthology.orchestra.run.smithi018.stdout:================================================================================ 2023-09-23T20:05:05.909 INFO:teuthology.orchestra.run.smithi018.stdout:Install 1 Package 2023-09-23T20:05:05.909 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:05:05.910 INFO:teuthology.orchestra.run.smithi018.stdout:Total download size: 90 k 2023-09-23T20:05:05.910 INFO:teuthology.orchestra.run.smithi018.stdout:Installed size: 264 k 2023-09-23T20:05:05.910 INFO:teuthology.orchestra.run.smithi018.stdout:Downloading Packages: 2023-09-23T20:05:06.094 INFO:teuthology.orchestra.run.smithi018.stdout:rbd-fuse-17.2.6-1123.g61431f6b.el8.x86_64.rpm 490 kB/s | 90 kB 00:00 2023-09-23T20:05:06.094 INFO:teuthology.orchestra.run.smithi018.stdout:-------------------------------------------------------------------------------- 2023-09-23T20:05:06.094 INFO:teuthology.orchestra.run.smithi018.stdout:Total 487 kB/s | 90 kB 00:00 2023-09-23T20:05:06.095 INFO:teuthology.orchestra.run.smithi018.stdout:Running transaction check 2023-09-23T20:05:06.105 INFO:teuthology.orchestra.run.smithi018.stdout:Transaction check succeeded. 2023-09-23T20:05:06.106 INFO:teuthology.orchestra.run.smithi018.stdout:Running transaction test 2023-09-23T20:05:06.164 INFO:teuthology.orchestra.run.smithi018.stdout:Transaction test succeeded. 2023-09-23T20:05:06.168 INFO:teuthology.orchestra.run.smithi018.stdout:Running transaction 2023-09-23T20:05:06.278 INFO:teuthology.orchestra.run.smithi018.stdout: Preparing : 1/1 2023-09-23T20:05:06.586 INFO:teuthology.orchestra.run.smithi018.stdout: Installing : rbd-fuse-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T20:05:06.887 INFO:teuthology.orchestra.run.smithi018.stdout: Running scriptlet: rbd-fuse-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T20:05:07.576 INFO:teuthology.orchestra.run.smithi149.stdout:Installed products updated. 2023-09-23T20:05:07.761 INFO:teuthology.orchestra.run.smithi149.stdout: 2023-09-23T20:05:07.762 INFO:teuthology.orchestra.run.smithi149.stdout:Installed: 2023-09-23T20:05:07.762 INFO:teuthology.orchestra.run.smithi149.stdout: librados-devel-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:05:07.762 INFO:teuthology.orchestra.run.smithi149.stdout: 2023-09-23T20:05:07.762 INFO:teuthology.orchestra.run.smithi149.stdout:Complete! 2023-09-23T20:05:07.906 DEBUG:teuthology.orchestra.run.smithi149:> sudo yum -y install libcephfs2 2023-09-23T20:05:07.964 INFO:teuthology.orchestra.run.smithi018.stdout: Verifying : rbd-fuse-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T20:05:07.964 INFO:teuthology.orchestra.run.smithi018.stdout:Last metadata expiration check: 0:03:45 ago on Sat 23 Sep 2023 08:01:22 PM UTC. 2023-09-23T20:05:08.265 INFO:teuthology.orchestra.run.smithi149.stdout:Updating Subscription Management repositories. 2023-09-23T20:05:09.610 INFO:teuthology.orchestra.run.smithi149.stdout:Last metadata expiration check: 0:03:07 ago on Sat 23 Sep 2023 08:02:02 PM UTC. 2023-09-23T20:05:10.890 INFO:teuthology.orchestra.run.smithi149.stdout:Package libcephfs2-2:17.2.6-1123.g61431f6b.el8.x86_64 is already installed. 2023-09-23T20:05:10.999 INFO:teuthology.orchestra.run.smithi149.stdout:Dependencies resolved. 2023-09-23T20:05:11.001 INFO:teuthology.orchestra.run.smithi149.stdout:Nothing to do. 2023-09-23T20:05:11.001 INFO:teuthology.orchestra.run.smithi149.stdout:Complete! 2023-09-23T20:05:11.130 DEBUG:teuthology.orchestra.run.smithi149:> sudo yum -y install libcephfs-devel 2023-09-23T20:05:11.484 INFO:teuthology.orchestra.run.smithi149.stdout:Updating Subscription Management repositories. 2023-09-23T20:05:12.401 INFO:teuthology.orchestra.run.smithi018.stdout:Installed products updated. 2023-09-23T20:05:12.602 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:05:12.602 INFO:teuthology.orchestra.run.smithi018.stdout:Installed: 2023-09-23T20:05:12.602 INFO:teuthology.orchestra.run.smithi018.stdout: rbd-fuse-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:05:12.603 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:05:12.603 INFO:teuthology.orchestra.run.smithi018.stdout:Complete! 2023-09-23T20:05:12.742 DEBUG:teuthology.orchestra.run.smithi018:> sudo yum -y install rbd-mirror 2023-09-23T20:05:13.032 INFO:teuthology.orchestra.run.smithi149.stdout:Last metadata expiration check: 0:03:11 ago on Sat 23 Sep 2023 08:02:02 PM UTC. 2023-09-23T20:05:13.122 INFO:teuthology.orchestra.run.smithi018.stdout:Updating Subscription Management repositories. 2023-09-23T20:05:14.420 INFO:teuthology.orchestra.run.smithi149.stdout:Dependencies resolved. 2023-09-23T20:05:14.421 INFO:teuthology.orchestra.run.smithi149.stdout:================================================================================ 2023-09-23T20:05:14.421 INFO:teuthology.orchestra.run.smithi149.stdout: Package Arch Version Repo Size 2023-09-23T20:05:14.421 INFO:teuthology.orchestra.run.smithi149.stdout:================================================================================ 2023-09-23T20:05:14.421 INFO:teuthology.orchestra.run.smithi149.stdout:Installing: 2023-09-23T20:05:14.421 INFO:teuthology.orchestra.run.smithi149.stdout: libcephfs-devel x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 25 k 2023-09-23T20:05:14.421 INFO:teuthology.orchestra.run.smithi149.stdout: 2023-09-23T20:05:14.421 INFO:teuthology.orchestra.run.smithi149.stdout:Transaction Summary 2023-09-23T20:05:14.421 INFO:teuthology.orchestra.run.smithi149.stdout:================================================================================ 2023-09-23T20:05:14.421 INFO:teuthology.orchestra.run.smithi149.stdout:Install 1 Package 2023-09-23T20:05:14.422 INFO:teuthology.orchestra.run.smithi149.stdout: 2023-09-23T20:05:14.422 INFO:teuthology.orchestra.run.smithi149.stdout:Total download size: 25 k 2023-09-23T20:05:14.422 INFO:teuthology.orchestra.run.smithi149.stdout:Installed size: 109 k 2023-09-23T20:05:14.422 INFO:teuthology.orchestra.run.smithi149.stdout:Downloading Packages: 2023-09-23T20:05:14.613 INFO:teuthology.orchestra.run.smithi018.stdout:Last metadata expiration check: 0:03:52 ago on Sat 23 Sep 2023 08:01:22 PM UTC. 2023-09-23T20:05:14.650 INFO:teuthology.orchestra.run.smithi149.stdout:libcephfs-devel-17.2.6-1123.g61431f6b.el8.x86_6 109 kB/s | 25 kB 00:00 2023-09-23T20:05:14.650 INFO:teuthology.orchestra.run.smithi149.stdout:-------------------------------------------------------------------------------- 2023-09-23T20:05:14.650 INFO:teuthology.orchestra.run.smithi149.stdout:Total 108 kB/s | 25 kB 00:00 2023-09-23T20:05:14.651 INFO:teuthology.orchestra.run.smithi149.stdout:Running transaction check 2023-09-23T20:05:14.654 INFO:teuthology.orchestra.run.smithi149.stdout:Transaction check succeeded. 2023-09-23T20:05:14.654 INFO:teuthology.orchestra.run.smithi149.stdout:Running transaction test 2023-09-23T20:05:14.656 INFO:teuthology.orchestra.run.smithi149.stdout:Transaction test succeeded. 2023-09-23T20:05:14.660 INFO:teuthology.orchestra.run.smithi149.stdout:Running transaction 2023-09-23T20:05:14.706 INFO:teuthology.orchestra.run.smithi149.stdout: Preparing : 1/1 2023-09-23T20:05:14.896 INFO:teuthology.orchestra.run.smithi149.stdout: Installing : libcephfs-devel-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T20:05:15.115 INFO:teuthology.orchestra.run.smithi149.stdout: Running scriptlet: libcephfs-devel-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T20:05:16.085 INFO:teuthology.orchestra.run.smithi018.stdout:Dependencies resolved. 2023-09-23T20:05:16.086 INFO:teuthology.orchestra.run.smithi018.stdout:================================================================================ 2023-09-23T20:05:16.086 INFO:teuthology.orchestra.run.smithi018.stdout: Package Arch Version Repo Size 2023-09-23T20:05:16.086 INFO:teuthology.orchestra.run.smithi018.stdout:================================================================================ 2023-09-23T20:05:16.086 INFO:teuthology.orchestra.run.smithi018.stdout:Installing: 2023-09-23T20:05:16.086 INFO:teuthology.orchestra.run.smithi018.stdout: rbd-mirror x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 3.5 M 2023-09-23T20:05:16.086 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:05:16.086 INFO:teuthology.orchestra.run.smithi018.stdout:Transaction Summary 2023-09-23T20:05:16.086 INFO:teuthology.orchestra.run.smithi018.stdout:================================================================================ 2023-09-23T20:05:16.087 INFO:teuthology.orchestra.run.smithi018.stdout:Install 1 Package 2023-09-23T20:05:16.087 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:05:16.087 INFO:teuthology.orchestra.run.smithi018.stdout:Total download size: 3.5 M 2023-09-23T20:05:16.087 INFO:teuthology.orchestra.run.smithi018.stdout:Installed size: 16 M 2023-09-23T20:05:16.087 INFO:teuthology.orchestra.run.smithi018.stdout:Downloading Packages: 2023-09-23T20:05:16.126 INFO:teuthology.orchestra.run.smithi149.stdout: Verifying : libcephfs-devel-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T20:05:16.126 INFO:teuthology.orchestra.run.smithi149.stdout:Last metadata expiration check: 0:03:14 ago on Sat 23 Sep 2023 08:02:02 PM UTC. 2023-09-23T20:05:16.599 INFO:teuthology.orchestra.run.smithi018.stdout:rbd-mirror-17.2.6-1123.g61431f6b.el8.x86_64.rpm 6.8 MB/s | 3.5 MB 00:00 2023-09-23T20:05:16.599 INFO:teuthology.orchestra.run.smithi018.stdout:-------------------------------------------------------------------------------- 2023-09-23T20:05:16.600 INFO:teuthology.orchestra.run.smithi018.stdout:Total 6.7 MB/s | 3.5 MB 00:00 2023-09-23T20:05:16.600 INFO:teuthology.orchestra.run.smithi018.stdout:Running transaction check 2023-09-23T20:05:16.613 INFO:teuthology.orchestra.run.smithi018.stdout:Transaction check succeeded. 2023-09-23T20:05:16.613 INFO:teuthology.orchestra.run.smithi018.stdout:Running transaction test 2023-09-23T20:05:16.685 INFO:teuthology.orchestra.run.smithi018.stdout:Transaction test succeeded. 2023-09-23T20:05:16.689 INFO:teuthology.orchestra.run.smithi018.stdout:Running transaction 2023-09-23T20:05:17.181 INFO:teuthology.orchestra.run.smithi018.stdout: Preparing : 1/1 2023-09-23T20:05:17.353 INFO:teuthology.orchestra.run.smithi018.stdout: Installing : rbd-mirror-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T20:05:17.727 INFO:teuthology.orchestra.run.smithi018.stdout: Running scriptlet: rbd-mirror-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T20:05:18.846 INFO:teuthology.orchestra.run.smithi018.stdout: Verifying : rbd-mirror-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T20:05:18.846 INFO:teuthology.orchestra.run.smithi018.stdout:Last metadata expiration check: 0:03:56 ago on Sat 23 Sep 2023 08:01:22 PM UTC. 2023-09-23T20:05:20.501 INFO:teuthology.orchestra.run.smithi149.stdout:Installed products updated. 2023-09-23T20:05:20.687 INFO:teuthology.orchestra.run.smithi149.stdout: 2023-09-23T20:05:20.688 INFO:teuthology.orchestra.run.smithi149.stdout:Installed: 2023-09-23T20:05:20.688 INFO:teuthology.orchestra.run.smithi149.stdout: libcephfs-devel-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:05:20.688 INFO:teuthology.orchestra.run.smithi149.stdout: 2023-09-23T20:05:20.688 INFO:teuthology.orchestra.run.smithi149.stdout:Complete! 2023-09-23T20:05:20.855 DEBUG:teuthology.orchestra.run.smithi149:> sudo yum -y install librados2 2023-09-23T20:05:21.210 INFO:teuthology.orchestra.run.smithi149.stdout:Updating Subscription Management repositories. 2023-09-23T20:05:22.212 INFO:teuthology.orchestra.run.smithi149.stdout:Last metadata expiration check: 0:03:20 ago on Sat 23 Sep 2023 08:02:02 PM UTC. 2023-09-23T20:05:22.500 INFO:teuthology.orchestra.run.smithi018.stdout:Installed products updated. 2023-09-23T20:05:22.703 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:05:22.703 INFO:teuthology.orchestra.run.smithi018.stdout:Installed: 2023-09-23T20:05:22.704 INFO:teuthology.orchestra.run.smithi018.stdout: rbd-mirror-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:05:22.704 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:05:22.704 INFO:teuthology.orchestra.run.smithi018.stdout:Complete! 2023-09-23T20:05:22.833 DEBUG:teuthology.orchestra.run.smithi018:> sudo yum -y install rbd-nbd 2023-09-23T20:05:23.210 INFO:teuthology.orchestra.run.smithi018.stdout:Updating Subscription Management repositories. 2023-09-23T20:05:23.560 INFO:teuthology.orchestra.run.smithi149.stdout:Package librados2-2:17.2.6-1123.g61431f6b.el8.x86_64 is already installed. 2023-09-23T20:05:23.669 INFO:teuthology.orchestra.run.smithi149.stdout:Dependencies resolved. 2023-09-23T20:05:23.671 INFO:teuthology.orchestra.run.smithi149.stdout:Nothing to do. 2023-09-23T20:05:23.671 INFO:teuthology.orchestra.run.smithi149.stdout:Complete! 2023-09-23T20:05:23.809 DEBUG:teuthology.orchestra.run.smithi149:> sudo yum -y install librbd1 2023-09-23T20:05:24.165 INFO:teuthology.orchestra.run.smithi149.stdout:Updating Subscription Management repositories. 2023-09-23T20:05:24.529 INFO:teuthology.orchestra.run.smithi018.stdout:Last metadata expiration check: 0:04:02 ago on Sat 23 Sep 2023 08:01:22 PM UTC. 2023-09-23T20:05:25.451 INFO:teuthology.orchestra.run.smithi149.stdout:Last metadata expiration check: 0:03:23 ago on Sat 23 Sep 2023 08:02:02 PM UTC. 2023-09-23T20:05:26.081 INFO:teuthology.orchestra.run.smithi018.stdout:Dependencies resolved. 2023-09-23T20:05:26.082 INFO:teuthology.orchestra.run.smithi018.stdout:================================================================================ 2023-09-23T20:05:26.082 INFO:teuthology.orchestra.run.smithi018.stdout: Package Architecture Version Repository Size 2023-09-23T20:05:26.082 INFO:teuthology.orchestra.run.smithi018.stdout:================================================================================ 2023-09-23T20:05:26.082 INFO:teuthology.orchestra.run.smithi018.stdout:Installing: 2023-09-23T20:05:26.082 INFO:teuthology.orchestra.run.smithi018.stdout: rbd-nbd x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 173 k 2023-09-23T20:05:26.082 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:05:26.083 INFO:teuthology.orchestra.run.smithi018.stdout:Transaction Summary 2023-09-23T20:05:26.083 INFO:teuthology.orchestra.run.smithi018.stdout:================================================================================ 2023-09-23T20:05:26.083 INFO:teuthology.orchestra.run.smithi018.stdout:Install 1 Package 2023-09-23T20:05:26.083 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:05:26.083 INFO:teuthology.orchestra.run.smithi018.stdout:Total download size: 173 k 2023-09-23T20:05:26.084 INFO:teuthology.orchestra.run.smithi018.stdout:Installed size: 552 k 2023-09-23T20:05:26.084 INFO:teuthology.orchestra.run.smithi018.stdout:Downloading Packages: 2023-09-23T20:05:26.278 INFO:teuthology.orchestra.run.smithi018.stdout:rbd-nbd-17.2.6-1123.g61431f6b.el8.x86_64.rpm 887 kB/s | 173 kB 00:00 2023-09-23T20:05:26.279 INFO:teuthology.orchestra.run.smithi018.stdout:-------------------------------------------------------------------------------- 2023-09-23T20:05:26.279 INFO:teuthology.orchestra.run.smithi018.stdout:Total 882 kB/s | 173 kB 00:00 2023-09-23T20:05:26.279 INFO:teuthology.orchestra.run.smithi018.stdout:Running transaction check 2023-09-23T20:05:26.291 INFO:teuthology.orchestra.run.smithi018.stdout:Transaction check succeeded. 2023-09-23T20:05:26.291 INFO:teuthology.orchestra.run.smithi018.stdout:Running transaction test 2023-09-23T20:05:26.350 INFO:teuthology.orchestra.run.smithi018.stdout:Transaction test succeeded. 2023-09-23T20:05:26.354 INFO:teuthology.orchestra.run.smithi018.stdout:Running transaction 2023-09-23T20:05:26.477 INFO:teuthology.orchestra.run.smithi018.stdout: Preparing : 1/1 2023-09-23T20:05:26.658 INFO:teuthology.orchestra.run.smithi018.stdout: Installing : rbd-nbd-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T20:05:26.818 INFO:teuthology.orchestra.run.smithi149.stdout:Package librbd1-2:17.2.6-1123.g61431f6b.el8.x86_64 is already installed. 2023-09-23T20:05:26.883 INFO:teuthology.orchestra.run.smithi018.stdout: Running scriptlet: rbd-nbd-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T20:05:26.927 INFO:teuthology.orchestra.run.smithi149.stdout:Dependencies resolved. 2023-09-23T20:05:26.929 INFO:teuthology.orchestra.run.smithi149.stdout:Nothing to do. 2023-09-23T20:05:26.929 INFO:teuthology.orchestra.run.smithi149.stdout:Complete! 2023-09-23T20:05:27.087 DEBUG:teuthology.orchestra.run.smithi149:> sudo yum -y install python3-rados 2023-09-23T20:05:27.446 INFO:teuthology.orchestra.run.smithi149.stdout:Updating Subscription Management repositories. 2023-09-23T20:05:27.982 INFO:teuthology.orchestra.run.smithi018.stdout: Verifying : rbd-nbd-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T20:05:27.982 INFO:teuthology.orchestra.run.smithi018.stdout:Last metadata expiration check: 0:04:05 ago on Sat 23 Sep 2023 08:01:22 PM UTC. 2023-09-23T20:05:28.992 INFO:teuthology.orchestra.run.smithi149.stdout:Last metadata expiration check: 0:03:26 ago on Sat 23 Sep 2023 08:02:02 PM UTC. 2023-09-23T20:05:30.273 INFO:teuthology.orchestra.run.smithi149.stdout:Package python3-rados-2:17.2.6-1123.g61431f6b.el8.x86_64 is already installed. 2023-09-23T20:05:30.383 INFO:teuthology.orchestra.run.smithi149.stdout:Dependencies resolved. 2023-09-23T20:05:30.384 INFO:teuthology.orchestra.run.smithi149.stdout:Nothing to do. 2023-09-23T20:05:30.384 INFO:teuthology.orchestra.run.smithi149.stdout:Complete! 2023-09-23T20:05:30.536 DEBUG:teuthology.orchestra.run.smithi149:> sudo yum -y install python3-rgw 2023-09-23T20:05:30.890 INFO:teuthology.orchestra.run.smithi149.stdout:Updating Subscription Management repositories. 2023-09-23T20:05:32.422 INFO:teuthology.orchestra.run.smithi149.stdout:Last metadata expiration check: 0:03:30 ago on Sat 23 Sep 2023 08:02:02 PM UTC. 2023-09-23T20:05:32.869 INFO:teuthology.orchestra.run.smithi018.stdout:Installed products updated. 2023-09-23T20:05:33.075 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:05:33.075 INFO:teuthology.orchestra.run.smithi018.stdout:Installed: 2023-09-23T20:05:33.075 INFO:teuthology.orchestra.run.smithi018.stdout: rbd-nbd-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:05:33.076 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:05:33.076 INFO:teuthology.orchestra.run.smithi018.stdout:Complete! 2023-09-23T20:05:33.230 DEBUG:teuthology.parallel:result is None 2023-09-23T20:05:33.772 INFO:teuthology.orchestra.run.smithi149.stdout:Package python3-rgw-2:17.2.6-1123.g61431f6b.el8.x86_64 is already installed. 2023-09-23T20:05:33.884 INFO:teuthology.orchestra.run.smithi149.stdout:Dependencies resolved. 2023-09-23T20:05:33.885 INFO:teuthology.orchestra.run.smithi149.stdout:Nothing to do. 2023-09-23T20:05:33.885 INFO:teuthology.orchestra.run.smithi149.stdout:Complete! 2023-09-23T20:05:34.025 DEBUG:teuthology.orchestra.run.smithi149:> sudo yum -y install python3-cephfs 2023-09-23T20:05:34.381 INFO:teuthology.orchestra.run.smithi149.stdout:Updating Subscription Management repositories. 2023-09-23T20:05:35.573 INFO:teuthology.orchestra.run.smithi149.stdout:Last metadata expiration check: 0:03:33 ago on Sat 23 Sep 2023 08:02:02 PM UTC. 2023-09-23T20:05:36.905 INFO:teuthology.orchestra.run.smithi149.stdout:Package python3-cephfs-2:17.2.6-1123.g61431f6b.el8.x86_64 is already installed. 2023-09-23T20:05:37.016 INFO:teuthology.orchestra.run.smithi149.stdout:Dependencies resolved. 2023-09-23T20:05:37.017 INFO:teuthology.orchestra.run.smithi149.stdout:Nothing to do. 2023-09-23T20:05:37.017 INFO:teuthology.orchestra.run.smithi149.stdout:Complete! 2023-09-23T20:05:37.166 DEBUG:teuthology.orchestra.run.smithi149:> sudo yum -y install python3-rbd 2023-09-23T20:05:37.528 INFO:teuthology.orchestra.run.smithi149.stdout:Updating Subscription Management repositories. 2023-09-23T20:05:39.150 INFO:teuthology.orchestra.run.smithi149.stdout:Last metadata expiration check: 0:03:37 ago on Sat 23 Sep 2023 08:02:02 PM UTC. 2023-09-23T20:05:40.431 INFO:teuthology.orchestra.run.smithi149.stdout:Package python3-rbd-2:17.2.6-1123.g61431f6b.el8.x86_64 is already installed. 2023-09-23T20:05:40.545 INFO:teuthology.orchestra.run.smithi149.stdout:Dependencies resolved. 2023-09-23T20:05:40.546 INFO:teuthology.orchestra.run.smithi149.stdout:Nothing to do. 2023-09-23T20:05:40.546 INFO:teuthology.orchestra.run.smithi149.stdout:Complete! 2023-09-23T20:05:40.688 DEBUG:teuthology.orchestra.run.smithi149:> sudo yum -y install rbd-fuse 2023-09-23T20:05:41.047 INFO:teuthology.orchestra.run.smithi149.stdout:Updating Subscription Management repositories. 2023-09-23T20:05:42.511 INFO:teuthology.orchestra.run.smithi149.stdout:Last metadata expiration check: 0:03:40 ago on Sat 23 Sep 2023 08:02:02 PM UTC. 2023-09-23T20:05:43.965 INFO:teuthology.orchestra.run.smithi149.stdout:Dependencies resolved. 2023-09-23T20:05:43.965 INFO:teuthology.orchestra.run.smithi149.stdout:================================================================================ 2023-09-23T20:05:43.966 INFO:teuthology.orchestra.run.smithi149.stdout: Package Architecture Version Repository Size 2023-09-23T20:05:43.966 INFO:teuthology.orchestra.run.smithi149.stdout:================================================================================ 2023-09-23T20:05:43.966 INFO:teuthology.orchestra.run.smithi149.stdout:Installing: 2023-09-23T20:05:43.966 INFO:teuthology.orchestra.run.smithi149.stdout: rbd-fuse x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 90 k 2023-09-23T20:05:43.966 INFO:teuthology.orchestra.run.smithi149.stdout: 2023-09-23T20:05:43.966 INFO:teuthology.orchestra.run.smithi149.stdout:Transaction Summary 2023-09-23T20:05:43.966 INFO:teuthology.orchestra.run.smithi149.stdout:================================================================================ 2023-09-23T20:05:43.966 INFO:teuthology.orchestra.run.smithi149.stdout:Install 1 Package 2023-09-23T20:05:43.966 INFO:teuthology.orchestra.run.smithi149.stdout: 2023-09-23T20:05:43.967 INFO:teuthology.orchestra.run.smithi149.stdout:Total download size: 90 k 2023-09-23T20:05:43.967 INFO:teuthology.orchestra.run.smithi149.stdout:Installed size: 264 k 2023-09-23T20:05:43.967 INFO:teuthology.orchestra.run.smithi149.stdout:Downloading Packages: 2023-09-23T20:05:44.136 INFO:teuthology.orchestra.run.smithi149.stdout:rbd-fuse-17.2.6-1123.g61431f6b.el8.x86_64.rpm 534 kB/s | 90 kB 00:00 2023-09-23T20:05:44.136 INFO:teuthology.orchestra.run.smithi149.stdout:-------------------------------------------------------------------------------- 2023-09-23T20:05:44.137 INFO:teuthology.orchestra.run.smithi149.stdout:Total 530 kB/s | 90 kB 00:00 2023-09-23T20:05:44.137 INFO:teuthology.orchestra.run.smithi149.stdout:Running transaction check 2023-09-23T20:05:44.147 INFO:teuthology.orchestra.run.smithi149.stdout:Transaction check succeeded. 2023-09-23T20:05:44.147 INFO:teuthology.orchestra.run.smithi149.stdout:Running transaction test 2023-09-23T20:05:44.200 INFO:teuthology.orchestra.run.smithi149.stdout:Transaction test succeeded. 2023-09-23T20:05:44.204 INFO:teuthology.orchestra.run.smithi149.stdout:Running transaction 2023-09-23T20:05:44.334 INFO:teuthology.orchestra.run.smithi149.stdout: Preparing : 1/1 2023-09-23T20:05:44.483 INFO:teuthology.orchestra.run.smithi149.stdout: Installing : rbd-fuse-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T20:05:44.723 INFO:teuthology.orchestra.run.smithi149.stdout: Running scriptlet: rbd-fuse-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T20:05:45.800 INFO:teuthology.orchestra.run.smithi149.stdout: Verifying : rbd-fuse-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T20:05:45.800 INFO:teuthology.orchestra.run.smithi149.stdout:Last metadata expiration check: 0:03:43 ago on Sat 23 Sep 2023 08:02:02 PM UTC. 2023-09-23T20:05:50.438 INFO:teuthology.orchestra.run.smithi149.stdout:Installed products updated. 2023-09-23T20:05:50.625 INFO:teuthology.orchestra.run.smithi149.stdout: 2023-09-23T20:05:50.625 INFO:teuthology.orchestra.run.smithi149.stdout:Installed: 2023-09-23T20:05:50.625 INFO:teuthology.orchestra.run.smithi149.stdout: rbd-fuse-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:05:50.625 INFO:teuthology.orchestra.run.smithi149.stdout: 2023-09-23T20:05:50.625 INFO:teuthology.orchestra.run.smithi149.stdout:Complete! 2023-09-23T20:05:50.780 DEBUG:teuthology.orchestra.run.smithi149:> sudo yum -y install rbd-mirror 2023-09-23T20:05:51.138 INFO:teuthology.orchestra.run.smithi149.stdout:Updating Subscription Management repositories. 2023-09-23T20:05:52.635 INFO:teuthology.orchestra.run.smithi149.stdout:Last metadata expiration check: 0:03:50 ago on Sat 23 Sep 2023 08:02:02 PM UTC. 2023-09-23T20:05:54.038 INFO:teuthology.orchestra.run.smithi149.stdout:Dependencies resolved. 2023-09-23T20:05:54.039 INFO:teuthology.orchestra.run.smithi149.stdout:================================================================================ 2023-09-23T20:05:54.039 INFO:teuthology.orchestra.run.smithi149.stdout: Package Arch Version Repo Size 2023-09-23T20:05:54.039 INFO:teuthology.orchestra.run.smithi149.stdout:================================================================================ 2023-09-23T20:05:54.039 INFO:teuthology.orchestra.run.smithi149.stdout:Installing: 2023-09-23T20:05:54.039 INFO:teuthology.orchestra.run.smithi149.stdout: rbd-mirror x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 3.5 M 2023-09-23T20:05:54.039 INFO:teuthology.orchestra.run.smithi149.stdout: 2023-09-23T20:05:54.039 INFO:teuthology.orchestra.run.smithi149.stdout:Transaction Summary 2023-09-23T20:05:54.040 INFO:teuthology.orchestra.run.smithi149.stdout:================================================================================ 2023-09-23T20:05:54.040 INFO:teuthology.orchestra.run.smithi149.stdout:Install 1 Package 2023-09-23T20:05:54.040 INFO:teuthology.orchestra.run.smithi149.stdout: 2023-09-23T20:05:54.040 INFO:teuthology.orchestra.run.smithi149.stdout:Total download size: 3.5 M 2023-09-23T20:05:54.040 INFO:teuthology.orchestra.run.smithi149.stdout:Installed size: 16 M 2023-09-23T20:05:54.040 INFO:teuthology.orchestra.run.smithi149.stdout:Downloading Packages: 2023-09-23T20:05:54.406 INFO:teuthology.orchestra.run.smithi149.stdout:rbd-mirror-17.2.6-1123.g61431f6b.el8.x86_64.rpm 9.4 MB/s | 3.5 MB 00:00 2023-09-23T20:05:54.407 INFO:teuthology.orchestra.run.smithi149.stdout:-------------------------------------------------------------------------------- 2023-09-23T20:05:54.407 INFO:teuthology.orchestra.run.smithi149.stdout:Total 9.4 MB/s | 3.5 MB 00:00 2023-09-23T20:05:54.408 INFO:teuthology.orchestra.run.smithi149.stdout:Running transaction check 2023-09-23T20:05:54.419 INFO:teuthology.orchestra.run.smithi149.stdout:Transaction check succeeded. 2023-09-23T20:05:54.419 INFO:teuthology.orchestra.run.smithi149.stdout:Running transaction test 2023-09-23T20:05:54.485 INFO:teuthology.orchestra.run.smithi149.stdout:Transaction test succeeded. 2023-09-23T20:05:54.489 INFO:teuthology.orchestra.run.smithi149.stdout:Running transaction 2023-09-23T20:05:54.931 INFO:teuthology.orchestra.run.smithi149.stdout: Preparing : 1/1 2023-09-23T20:05:55.101 INFO:teuthology.orchestra.run.smithi149.stdout: Installing : rbd-mirror-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T20:05:55.476 INFO:teuthology.orchestra.run.smithi149.stdout: Running scriptlet: rbd-mirror-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T20:05:56.541 INFO:teuthology.orchestra.run.smithi149.stdout: Verifying : rbd-mirror-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T20:05:56.541 INFO:teuthology.orchestra.run.smithi149.stdout:Last metadata expiration check: 0:03:54 ago on Sat 23 Sep 2023 08:02:02 PM UTC. 2023-09-23T20:06:00.894 INFO:teuthology.orchestra.run.smithi149.stdout:Installed products updated. 2023-09-23T20:06:01.082 INFO:teuthology.orchestra.run.smithi149.stdout: 2023-09-23T20:06:01.082 INFO:teuthology.orchestra.run.smithi149.stdout:Installed: 2023-09-23T20:06:01.082 INFO:teuthology.orchestra.run.smithi149.stdout: rbd-mirror-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:06:01.082 INFO:teuthology.orchestra.run.smithi149.stdout: 2023-09-23T20:06:01.082 INFO:teuthology.orchestra.run.smithi149.stdout:Complete! 2023-09-23T20:06:01.238 DEBUG:teuthology.orchestra.run.smithi149:> sudo yum -y install rbd-nbd 2023-09-23T20:06:01.597 INFO:teuthology.orchestra.run.smithi149.stdout:Updating Subscription Management repositories. 2023-09-23T20:06:02.498 INFO:teuthology.orchestra.run.smithi149.stdout:Last metadata expiration check: 0:04:00 ago on Sat 23 Sep 2023 08:02:02 PM UTC. 2023-09-23T20:06:03.897 INFO:teuthology.orchestra.run.smithi149.stdout:Dependencies resolved. 2023-09-23T20:06:03.897 INFO:teuthology.orchestra.run.smithi149.stdout:================================================================================ 2023-09-23T20:06:03.897 INFO:teuthology.orchestra.run.smithi149.stdout: Package Architecture Version Repository Size 2023-09-23T20:06:03.897 INFO:teuthology.orchestra.run.smithi149.stdout:================================================================================ 2023-09-23T20:06:03.897 INFO:teuthology.orchestra.run.smithi149.stdout:Installing: 2023-09-23T20:06:03.898 INFO:teuthology.orchestra.run.smithi149.stdout: rbd-nbd x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 173 k 2023-09-23T20:06:03.898 INFO:teuthology.orchestra.run.smithi149.stdout: 2023-09-23T20:06:03.898 INFO:teuthology.orchestra.run.smithi149.stdout:Transaction Summary 2023-09-23T20:06:03.898 INFO:teuthology.orchestra.run.smithi149.stdout:================================================================================ 2023-09-23T20:06:03.898 INFO:teuthology.orchestra.run.smithi149.stdout:Install 1 Package 2023-09-23T20:06:03.898 INFO:teuthology.orchestra.run.smithi149.stdout: 2023-09-23T20:06:03.899 INFO:teuthology.orchestra.run.smithi149.stdout:Total download size: 173 k 2023-09-23T20:06:03.899 INFO:teuthology.orchestra.run.smithi149.stdout:Installed size: 552 k 2023-09-23T20:06:03.899 INFO:teuthology.orchestra.run.smithi149.stdout:Downloading Packages: 2023-09-23T20:06:04.377 INFO:teuthology.orchestra.run.smithi149.stdout:rbd-nbd-17.2.6-1123.g61431f6b.el8.x86_64.rpm 363 kB/s | 173 kB 00:00 2023-09-23T20:06:04.377 INFO:teuthology.orchestra.run.smithi149.stdout:-------------------------------------------------------------------------------- 2023-09-23T20:06:04.377 INFO:teuthology.orchestra.run.smithi149.stdout:Total 362 kB/s | 173 kB 00:00 2023-09-23T20:06:04.377 INFO:teuthology.orchestra.run.smithi149.stdout:Running transaction check 2023-09-23T20:06:04.388 INFO:teuthology.orchestra.run.smithi149.stdout:Transaction check succeeded. 2023-09-23T20:06:04.388 INFO:teuthology.orchestra.run.smithi149.stdout:Running transaction test 2023-09-23T20:06:04.444 INFO:teuthology.orchestra.run.smithi149.stdout:Transaction test succeeded. 2023-09-23T20:06:04.447 INFO:teuthology.orchestra.run.smithi149.stdout:Running transaction 2023-09-23T20:06:04.573 INFO:teuthology.orchestra.run.smithi149.stdout: Preparing : 1/1 2023-09-23T20:06:04.747 INFO:teuthology.orchestra.run.smithi149.stdout: Installing : rbd-nbd-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T20:06:04.991 INFO:teuthology.orchestra.run.smithi149.stdout: Running scriptlet: rbd-nbd-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T20:06:06.310 INFO:teuthology.orchestra.run.smithi149.stdout: Verifying : rbd-nbd-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T20:06:06.310 INFO:teuthology.orchestra.run.smithi149.stdout:Last metadata expiration check: 0:04:04 ago on Sat 23 Sep 2023 08:02:02 PM UTC. 2023-09-23T20:06:08.866 INFO:teuthology.orchestra.run.smithi149.stdout:Installed products updated. 2023-09-23T20:06:09.048 INFO:teuthology.orchestra.run.smithi149.stdout: 2023-09-23T20:06:09.048 INFO:teuthology.orchestra.run.smithi149.stdout:Installed: 2023-09-23T20:06:09.048 INFO:teuthology.orchestra.run.smithi149.stdout: rbd-nbd-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T20:06:09.048 INFO:teuthology.orchestra.run.smithi149.stdout: 2023-09-23T20:06:09.048 INFO:teuthology.orchestra.run.smithi149.stdout:Complete! 2023-09-23T20:06:09.204 DEBUG:teuthology.parallel:result is None 2023-09-23T20:06:09.204 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-09-23T20:06:09.205 INFO:teuthology.packaging:ref: None 2023-09-23T20:06:09.205 INFO:teuthology.packaging:tag: None 2023-09-23T20:06:09.205 INFO:teuthology.packaging:branch: quincy 2023-09-23T20:06:09.205 INFO:teuthology.packaging:sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T20:06:09.205 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=quincy 2023-09-23T20:06:09.544 DEBUG:teuthology.orchestra.run.smithi018:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2023-09-23T20:06:09.572 INFO:teuthology.orchestra.run.smithi018.stdout:17.2.6-1123.g61431f6b.el8 2023-09-23T20:06:09.572 INFO:teuthology.packaging:The installed version of ceph is 17.2.6-1123.g61431f6b.el8 2023-09-23T20:06:09.572 INFO:teuthology.task.install:The correct ceph version 17.2.6-1123.g61431f6b is installed. 2023-09-23T20:06:09.573 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-09-23T20:06:09.573 INFO:teuthology.packaging:ref: None 2023-09-23T20:06:09.573 INFO:teuthology.packaging:tag: None 2023-09-23T20:06:09.573 INFO:teuthology.packaging:branch: quincy 2023-09-23T20:06:09.573 INFO:teuthology.packaging:sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T20:06:09.573 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=quincy 2023-09-23T20:06:09.797 DEBUG:teuthology.orchestra.run.smithi149:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2023-09-23T20:06:09.821 INFO:teuthology.orchestra.run.smithi149.stdout:17.2.6-1123.g61431f6b.el8 2023-09-23T20:06:09.821 INFO:teuthology.packaging:The installed version of ceph is 17.2.6-1123.g61431f6b.el8 2023-09-23T20:06:09.821 INFO:teuthology.task.install:The correct ceph version 17.2.6-1123.g61431f6b is installed. 2023-09-23T20:06:09.821 INFO:teuthology.task.install.util:Shipping valgrind.supp... 2023-09-23T20:06:09.822 DEBUG:teuthology.orchestra.run.smithi018:> set -ex 2023-09-23T20:06:09.822 DEBUG:teuthology.orchestra.run.smithi018:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2023-09-23T20:06:09.856 DEBUG:teuthology.orchestra.run.smithi149:> set -ex 2023-09-23T20:06:09.857 DEBUG:teuthology.orchestra.run.smithi149:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2023-09-23T20:06:09.889 INFO:teuthology.task.install.util:Shipping 'daemon-helper'... 2023-09-23T20:06:09.890 DEBUG:teuthology.orchestra.run.smithi018:> set -ex 2023-09-23T20:06:09.890 DEBUG:teuthology.orchestra.run.smithi018:> sudo dd of=/usr/bin/daemon-helper 2023-09-23T20:06:09.929 DEBUG:teuthology.orchestra.run.smithi018:> sudo chmod a=rx -- /usr/bin/daemon-helper 2023-09-23T20:06:10.000 DEBUG:teuthology.orchestra.run.smithi149:> set -ex 2023-09-23T20:06:10.000 DEBUG:teuthology.orchestra.run.smithi149:> sudo dd of=/usr/bin/daemon-helper 2023-09-23T20:06:10.027 DEBUG:teuthology.orchestra.run.smithi149:> sudo chmod a=rx -- /usr/bin/daemon-helper 2023-09-23T20:06:10.094 INFO:teuthology.task.install.util:Shipping 'adjust-ulimits'... 2023-09-23T20:06:10.094 DEBUG:teuthology.orchestra.run.smithi018:> set -ex 2023-09-23T20:06:10.094 DEBUG:teuthology.orchestra.run.smithi018:> sudo dd of=/usr/bin/adjust-ulimits 2023-09-23T20:06:10.126 DEBUG:teuthology.orchestra.run.smithi018:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2023-09-23T20:06:10.198 DEBUG:teuthology.orchestra.run.smithi149:> set -ex 2023-09-23T20:06:10.198 DEBUG:teuthology.orchestra.run.smithi149:> sudo dd of=/usr/bin/adjust-ulimits 2023-09-23T20:06:10.224 DEBUG:teuthology.orchestra.run.smithi149:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2023-09-23T20:06:10.290 INFO:teuthology.task.install.util:Shipping 'stdin-killer'... 2023-09-23T20:06:10.290 DEBUG:teuthology.orchestra.run.smithi018:> set -ex 2023-09-23T20:06:10.290 DEBUG:teuthology.orchestra.run.smithi018:> sudo dd of=/usr/bin/stdin-killer 2023-09-23T20:06:10.321 DEBUG:teuthology.orchestra.run.smithi018:> sudo chmod a=rx -- /usr/bin/stdin-killer 2023-09-23T20:06:10.390 DEBUG:teuthology.orchestra.run.smithi149:> set -ex 2023-09-23T20:06:10.390 DEBUG:teuthology.orchestra.run.smithi149:> sudo dd of=/usr/bin/stdin-killer 2023-09-23T20:06:10.417 DEBUG:teuthology.orchestra.run.smithi149:> sudo chmod a=rx -- /usr/bin/stdin-killer 2023-09-23T20:06:10.483 INFO:teuthology.run_tasks:Running task cephadm... 2023-09-23T20:06:10.553 INFO:tasks.cephadm:Config: {'cephadm_branch': 'quincy', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'conf': {'osd': {'osd_class_default_list': '*', 'osd_class_load_list': '*', 'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}, 'global': {'mon election default strategy': 1}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}}, 'image': 'quay.ceph.io/ceph-ci/ceph:quincy', 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'sha1': 'bb0daeff97f020f686596fca6eba20c310e26fb7'} 2023-09-23T20:06:10.553 INFO:tasks.cephadm:Cluster image is quay.ceph.io/ceph-ci/ceph:quincy 2023-09-23T20:06:10.554 INFO:tasks.cephadm:Cluster fsid is a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 2023-09-23T20:06:10.554 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2023-09-23T20:06:10.554 INFO:tasks.cephadm:Monitor IPs: {'mon.a': '172.21.15.18', 'mon.c': '[v2:172.21.15.18:3301,v1:172.21.15.18:6790]', 'mon.b': '172.21.15.149'} 2023-09-23T20:06:10.554 INFO:tasks.cephadm:First mon is mon.a on smithi018 2023-09-23T20:06:10.554 INFO:tasks.cephadm:First mgr is y 2023-09-23T20:06:10.554 INFO:tasks.cephadm:Normalizing hostnames... 2023-09-23T20:06:10.555 DEBUG:teuthology.orchestra.run.smithi018:> sudo hostname $(hostname -s) 2023-09-23T20:06:10.587 DEBUG:teuthology.orchestra.run.smithi149:> sudo hostname $(hostname -s) 2023-09-23T20:06:10.613 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref quincy)... 2023-09-23T20:06:10.613 DEBUG:teuthology.orchestra.run.smithi018:> curl --silent https://raw.githubusercontent.com/ceph/ceph/quincy/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2023-09-23T20:06:10.761 INFO:teuthology.orchestra.run.smithi018.stdout:-rw-rw-r--. 1 ubuntu ubuntu 366215 Sep 23 20:06 /home/ubuntu/cephtest/cephadm 2023-09-23T20:06:10.761 DEBUG:teuthology.orchestra.run.smithi149:> curl --silent https://raw.githubusercontent.com/ceph/ceph/quincy/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2023-09-23T20:06:10.897 INFO:teuthology.orchestra.run.smithi149.stdout:-rw-rw-r--. 1 ubuntu ubuntu 366215 Sep 23 20:06 /home/ubuntu/cephtest/cephadm 2023-09-23T20:06:10.897 DEBUG:teuthology.orchestra.run.smithi018:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2023-09-23T20:06:10.919 DEBUG:teuthology.orchestra.run.smithi149:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2023-09-23T20:06:10.947 INFO:tasks.cephadm:Pulling image quay.ceph.io/ceph-ci/ceph:quincy on all hosts... 2023-09-23T20:06:10.947 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy pull 2023-09-23T20:06:10.961 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy pull 2023-09-23T20:06:12.376 INFO:teuthology.orchestra.run.smithi018.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2023-09-23T20:06:12.479 INFO:teuthology.orchestra.run.smithi149.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2023-09-23T20:06:37.861 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2023-09-23T20:06:37.861 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph_version": "ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)", 2023-09-23T20:06:37.861 INFO:teuthology.orchestra.run.smithi018.stdout: "image_id": "0db14aa010946f310e3ede6ffd96448192b98ccc76c262396fe9cf91887d0d64", 2023-09-23T20:06:37.861 INFO:teuthology.orchestra.run.smithi018.stdout: "repo_digests": [ 2023-09-23T20:06:37.861 INFO:teuthology.orchestra.run.smithi018.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:1d807b57f44e899861bfb3500d17f81ef0ce7a664575560e306bb83d8fe31c44" 2023-09-23T20:06:37.862 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:06:37.862 INFO:teuthology.orchestra.run.smithi018.stdout:} 2023-09-23T20:06:43.486 INFO:teuthology.orchestra.run.smithi149.stdout:{ 2023-09-23T20:06:43.486 INFO:teuthology.orchestra.run.smithi149.stdout: "ceph_version": "ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)", 2023-09-23T20:06:43.486 INFO:teuthology.orchestra.run.smithi149.stdout: "image_id": "0db14aa010946f310e3ede6ffd96448192b98ccc76c262396fe9cf91887d0d64", 2023-09-23T20:06:43.486 INFO:teuthology.orchestra.run.smithi149.stdout: "repo_digests": [ 2023-09-23T20:06:43.486 INFO:teuthology.orchestra.run.smithi149.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:1d807b57f44e899861bfb3500d17f81ef0ce7a664575560e306bb83d8fe31c44" 2023-09-23T20:06:43.486 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:06:43.486 INFO:teuthology.orchestra.run.smithi149.stdout:} 2023-09-23T20:06:43.505 DEBUG:teuthology.orchestra.run.smithi018:> sudo mkdir -p /etc/ceph 2023-09-23T20:06:43.542 DEBUG:teuthology.orchestra.run.smithi149:> sudo mkdir -p /etc/ceph 2023-09-23T20:06:43.573 DEBUG:teuthology.orchestra.run.smithi018:> sudo chmod 777 /etc/ceph 2023-09-23T20:06:43.614 DEBUG:teuthology.orchestra.run.smithi149:> sudo chmod 777 /etc/ceph 2023-09-23T20:06:43.640 INFO:tasks.cephadm:Writing seed config... 2023-09-23T20:06:43.641 INFO:tasks.cephadm: override: [osd] osd_class_default_list = * 2023-09-23T20:06:43.641 INFO:tasks.cephadm: override: [osd] osd_class_load_list = * 2023-09-23T20:06:43.641 INFO:tasks.cephadm: override: [osd] debug ms = 1 2023-09-23T20:06:43.641 INFO:tasks.cephadm: override: [osd] debug osd = 20 2023-09-23T20:06:43.641 INFO:tasks.cephadm: override: [osd] osd shutdown pgref assert = True 2023-09-23T20:06:43.641 INFO:tasks.cephadm: override: [global] mon election default strategy = 1 2023-09-23T20:06:43.642 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2023-09-23T20:06:43.642 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2023-09-23T20:06:43.642 INFO:tasks.cephadm: override: [mon] debug mon = 20 2023-09-23T20:06:43.642 INFO:tasks.cephadm: override: [mon] debug ms = 1 2023-09-23T20:06:43.642 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2023-09-23T20:06:43.642 DEBUG:teuthology.orchestra.run.smithi018:> set -ex 2023-09-23T20:06:43.643 DEBUG:teuthology.orchestra.run.smithi018:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2023-09-23T20:06:43.674 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 mon election default strategy = 1 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = True bdev debug aio = true osd sloppy crc = true osd_class_default_list = * osd_class_load_list = * debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2023-09-23T20:06:43.674 DEBUG:teuthology.orchestra.run.smithi018:mon.a> sudo journalctl -f -n 0 -u ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7@mon.a.service 2023-09-23T20:06:43.716 DEBUG:teuthology.orchestra.run.smithi018:mgr.y> sudo journalctl -f -n 0 -u ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7@mgr.y.service 2023-09-23T20:06:43.747 INFO:journalctl@ceph.mon.a.smithi018.stdout:-- Logs begin at Sat 2023-09-23 19:50:10 UTC. -- 2023-09-23T20:06:43.748 INFO:tasks.cephadm:Bootstrapping... 2023-09-23T20:06:43.748 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy -v bootstrap --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-id a --mgr-id y --orphan-initial-daemons --skip-monitoring-stack --mon-ip 172.21.15.18 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2023-09-23T20:06:43.777 INFO:journalctl@ceph.mgr.y.smithi018.stdout:-- Logs begin at Sat 2023-09-23 19:50:10 UTC. -- 2023-09-23T20:06:43.962 INFO:teuthology.orchestra.run.smithi018.stdout:-------------------------------------------------------------------------------- 2023-09-23T20:06:43.962 INFO:teuthology.orchestra.run.smithi018.stdout:cephadm ['--image', 'quay.ceph.io/ceph-ci/ceph:quincy', '-v', 'bootstrap', '--fsid', 'a3b3b17e-5a4c-11ee-8db4-212e2dc638e7', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-id', 'a', '--mgr-id', 'y', '--orphan-initial-daemons', '--skip-monitoring-stack', '--mon-ip', '172.21.15.18', '--skip-admin-label'] 2023-09-23T20:06:44.109 INFO:teuthology.orchestra.run.smithi018.stdout:/bin/podman: stdout 4.1.1 2023-09-23T20:06:44.109 INFO:teuthology.orchestra.run.smithi018.stderr:Specifying an fsid for your cluster offers no advantages and may increase the likelihood of fsid conflicts. 2023-09-23T20:06:44.110 INFO:teuthology.orchestra.run.smithi018.stdout:Verifying podman|docker is present... 2023-09-23T20:06:44.252 INFO:teuthology.orchestra.run.smithi018.stdout:/bin/podman: stdout 4.1.1 2023-09-23T20:06:44.252 INFO:teuthology.orchestra.run.smithi018.stdout:Verifying lvm2 is present... 2023-09-23T20:06:44.252 INFO:teuthology.orchestra.run.smithi018.stdout:Verifying time synchronization is in place... 2023-09-23T20:06:44.259 INFO:teuthology.orchestra.run.smithi018.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2023-09-23T20:06:44.259 INFO:teuthology.orchestra.run.smithi018.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2023-09-23T20:06:44.265 INFO:teuthology.orchestra.run.smithi018.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2023-09-23T20:06:44.265 INFO:teuthology.orchestra.run.smithi018.stdout:systemctl: stdout inactive 2023-09-23T20:06:44.271 INFO:teuthology.orchestra.run.smithi018.stdout:systemctl: stdout enabled 2023-09-23T20:06:44.277 INFO:teuthology.orchestra.run.smithi018.stdout:systemctl: stdout active 2023-09-23T20:06:44.277 INFO:teuthology.orchestra.run.smithi018.stdout:Unit chronyd.service is enabled and running 2023-09-23T20:06:44.278 INFO:teuthology.orchestra.run.smithi018.stdout:Repeating the final host check... 2023-09-23T20:06:44.421 INFO:teuthology.orchestra.run.smithi018.stdout:/bin/podman: stdout 4.1.1 2023-09-23T20:06:44.421 INFO:teuthology.orchestra.run.smithi018.stdout:podman (/bin/podman) version 4.1.1 is present 2023-09-23T20:06:44.421 INFO:teuthology.orchestra.run.smithi018.stdout:systemctl is present 2023-09-23T20:06:44.421 INFO:teuthology.orchestra.run.smithi018.stdout:lvcreate is present 2023-09-23T20:06:44.427 INFO:teuthology.orchestra.run.smithi018.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2023-09-23T20:06:44.427 INFO:teuthology.orchestra.run.smithi018.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2023-09-23T20:06:44.433 INFO:teuthology.orchestra.run.smithi018.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2023-09-23T20:06:44.433 INFO:teuthology.orchestra.run.smithi018.stdout:systemctl: stdout inactive 2023-09-23T20:06:44.439 INFO:teuthology.orchestra.run.smithi018.stdout:systemctl: stdout enabled 2023-09-23T20:06:44.445 INFO:teuthology.orchestra.run.smithi018.stdout:systemctl: stdout active 2023-09-23T20:06:44.446 INFO:teuthology.orchestra.run.smithi018.stdout:Unit chronyd.service is enabled and running 2023-09-23T20:06:44.446 INFO:teuthology.orchestra.run.smithi018.stdout:Host looks OK 2023-09-23T20:06:44.446 INFO:teuthology.orchestra.run.smithi018.stdout:Cluster fsid: a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 2023-09-23T20:06:44.446 INFO:teuthology.orchestra.run.smithi018.stdout:Acquiring lock 139685521336808 on /run/cephadm/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7.lock 2023-09-23T20:06:44.446 INFO:teuthology.orchestra.run.smithi018.stdout:Lock 139685521336808 acquired on /run/cephadm/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7.lock 2023-09-23T20:06:44.446 INFO:teuthology.orchestra.run.smithi018.stdout:Verifying IP 172.21.15.18 port 3300 ... 2023-09-23T20:06:44.447 INFO:teuthology.orchestra.run.smithi018.stdout:Verifying IP 172.21.15.18 port 6789 ... 2023-09-23T20:06:44.447 INFO:teuthology.orchestra.run.smithi018.stdout:Base mon IP(s) is [172.21.15.18:3300, 172.21.15.18:6789], mon addrv is [v2:172.21.15.18:3300,v1:172.21.15.18:6789] 2023-09-23T20:06:44.450 INFO:teuthology.orchestra.run.smithi018.stdout:/sbin/ip: stdout default via 172.21.15.254 dev ens1f0 proto dhcp metric 100 2023-09-23T20:06:44.450 INFO:teuthology.orchestra.run.smithi018.stdout:/sbin/ip: stdout 172.21.0.0/20 dev ens1f0 proto kernel scope link src 172.21.15.18 metric 100 2023-09-23T20:06:44.453 INFO:teuthology.orchestra.run.smithi018.stdout:/sbin/ip: stdout ::1 dev lo proto kernel metric 256 pref medium 2023-09-23T20:06:44.453 INFO:teuthology.orchestra.run.smithi018.stdout:/sbin/ip: stdout fe80::/64 dev ens1f0 proto kernel metric 256 pref medium 2023-09-23T20:06:44.453 INFO:teuthology.orchestra.run.smithi018.stdout:/sbin/ip: stdout default via fe80::327c:5e00:6487:71e0 dev ens1f0 proto ra metric 1024 expires 1794sec hoplimit 64 pref medium 2023-09-23T20:06:44.456 INFO:teuthology.orchestra.run.smithi018.stdout:/sbin/ip: stdout 1: lo: mtu 65536 state UNKNOWN qlen 1000 2023-09-23T20:06:44.456 INFO:teuthology.orchestra.run.smithi018.stdout:/sbin/ip: stdout inet6 ::1/128 scope host 2023-09-23T20:06:44.456 INFO:teuthology.orchestra.run.smithi018.stdout:/sbin/ip: stdout valid_lft forever preferred_lft forever 2023-09-23T20:06:44.456 INFO:teuthology.orchestra.run.smithi018.stdout:/sbin/ip: stdout 4: ens1f0: mtu 1500 state UP qlen 1000 2023-09-23T20:06:44.456 INFO:teuthology.orchestra.run.smithi018.stdout:/sbin/ip: stdout inet6 fe80::ec4:7aff:febd:1362/64 scope link 2023-09-23T20:06:44.456 INFO:teuthology.orchestra.run.smithi018.stdout:/sbin/ip: stdout valid_lft forever preferred_lft forever 2023-09-23T20:06:44.457 INFO:teuthology.orchestra.run.smithi018.stdout:Mon IP `172.21.15.18` is in CIDR network `172.21.0.0/20` 2023-09-23T20:06:44.457 INFO:teuthology.orchestra.run.smithi018.stdout:Mon IP `172.21.15.18` is in CIDR network `172.21.0.0/20` 2023-09-23T20:06:44.457 INFO:teuthology.orchestra.run.smithi018.stdout:Inferred mon public CIDR from local network configuration ['172.21.0.0/20', '172.21.0.0/20'] 2023-09-23T20:06:44.457 INFO:teuthology.orchestra.run.smithi018.stdout:Internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2023-09-23T20:06:44.458 INFO:teuthology.orchestra.run.smithi018.stdout:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2023-09-23T20:06:45.207 INFO:teuthology.orchestra.run.smithi018.stdout:/bin/podman: stdout 0db14aa010946f310e3ede6ffd96448192b98ccc76c262396fe9cf91887d0d64 2023-09-23T20:06:45.207 INFO:teuthology.orchestra.run.smithi018.stdout:/bin/podman: stderr Trying to pull quay.ceph.io/ceph-ci/ceph:quincy... 2023-09-23T20:06:45.207 INFO:teuthology.orchestra.run.smithi018.stdout:/bin/podman: stderr Getting image source signatures 2023-09-23T20:06:45.207 INFO:teuthology.orchestra.run.smithi018.stdout:/bin/podman: stderr Copying blob sha256:af7d028f70e220a466755c6119effe51eebe359d9c00eb6b4883db3fabc6becd 2023-09-23T20:06:45.208 INFO:teuthology.orchestra.run.smithi018.stdout:/bin/podman: stderr Copying blob sha256:3f4fd5c3c800c6da9ef94691df203a05a5bbbf32c483d09d620ec06eef42ea2b 2023-09-23T20:06:45.208 INFO:teuthology.orchestra.run.smithi018.stdout:/bin/podman: stderr Copying config sha256:0db14aa010946f310e3ede6ffd96448192b98ccc76c262396fe9cf91887d0d64 2023-09-23T20:06:45.208 INFO:teuthology.orchestra.run.smithi018.stdout:/bin/podman: stderr Writing manifest to image destination 2023-09-23T20:06:45.208 INFO:teuthology.orchestra.run.smithi018.stdout:/bin/podman: stderr Storing signatures 2023-09-23T20:06:46.171 INFO:teuthology.orchestra.run.smithi018.stdout:ceph: stdout ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable) 2023-09-23T20:06:46.171 INFO:teuthology.orchestra.run.smithi018.stdout:Ceph version: ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable) 2023-09-23T20:06:46.171 INFO:teuthology.orchestra.run.smithi018.stdout:Extracting ceph user uid/gid from container image... 2023-09-23T20:06:47.022 INFO:teuthology.orchestra.run.smithi018.stdout:stat: stdout 167 167 2023-09-23T20:06:47.023 INFO:teuthology.orchestra.run.smithi018.stdout:Creating initial keys... 2023-09-23T20:06:47.874 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph-authtool: stdout AQBXRQ9l82qUHRAA+jmlR1L7/vbI+ossdNeS7Q== 2023-09-23T20:06:50.331 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph-authtool: stdout AQBYRQ9lXl7pIBAAV4IzJ/A3JGEFKMsJl1QwXQ== 2023-09-23T20:06:51.786 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph-authtool: stdout AQBbRQ9lnXodFxAAkzphSaqVmgv6zmqTfmexkw== 2023-09-23T20:06:51.787 INFO:teuthology.orchestra.run.smithi018.stdout:Creating initial monmap... 2023-09-23T20:06:52.541 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: monmap file /tmp/monmap 2023-09-23T20:06:52.542 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/monmaptool: stdout setting min_mon_release = octopus 2023-09-23T20:06:52.542 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: set fsid to a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 2023-09-23T20:06:52.542 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2023-09-23T20:06:52.542 INFO:teuthology.orchestra.run.smithi018.stdout:monmaptool for a [v2:172.21.15.18:3300,v1:172.21.15.18:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2023-09-23T20:06:52.542 INFO:teuthology.orchestra.run.smithi018.stdout:setting min_mon_release = octopus 2023-09-23T20:06:52.542 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/monmaptool: set fsid to a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 2023-09-23T20:06:52.542 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2023-09-23T20:06:52.542 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:06:52.542 INFO:teuthology.orchestra.run.smithi018.stdout:Creating mon... 2023-09-23T20:06:53.297 INFO:teuthology.orchestra.run.smithi018.stdout:create mon.a on 2023-09-23T20:06:53.397 INFO:teuthology.orchestra.run.smithi018.stdout:systemctl: stderr Removed /etc/systemd/system/multi-user.target.wants/ceph.target. 2023-09-23T20:06:53.489 INFO:teuthology.orchestra.run.smithi018.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2023-09-23T20:06:53.588 INFO:teuthology.orchestra.run.smithi018.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7.target → /etc/systemd/system/ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7.target. 2023-09-23T20:06:53.588 INFO:teuthology.orchestra.run.smithi018.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph.target.wants/ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7.target → /etc/systemd/system/ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7.target. 2023-09-23T20:06:53.716 INFO:teuthology.orchestra.run.smithi018.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7@mon.a 2023-09-23T20:06:53.716 INFO:teuthology.orchestra.run.smithi018.stdout:systemctl: stderr Failed to reset failed state of unit ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7@mon.a.service: Unit ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7@mon.a.service not loaded. 2023-09-23T20:06:53.804 INFO:teuthology.orchestra.run.smithi018.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7.target.wants/ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7@mon.a.service → /etc/systemd/system/ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7@.service. 2023-09-23T20:06:54.112 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:53 smithi018 systemd[1]: Starting Ceph mon.a for a3b3b17e-5a4c-11ee-8db4-212e2dc638e7... 2023-09-23T20:06:54.362 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:54 smithi018 podman[95345]: 2023-09-23T20:06:54.362 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:54 smithi018 bash[95345]: 258bd092da6f3d4e65855619b3473ab281b459b3e9c77f588b49fbb1e5e2d096 2023-09-23T20:06:54.362 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:54 smithi018 ceph-mon[95386]: mkfs a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 2023-09-23T20:06:54.362 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:54 smithi018 ceph-mon[95386]: mon.a is new leader, mons a in quorum (ranks 0) 2023-09-23T20:06:54.414 INFO:teuthology.orchestra.run.smithi018.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2023-09-23T20:06:54.414 INFO:teuthology.orchestra.run.smithi018.stdout:systemctl: stdout disabled 2023-09-23T20:06:54.422 INFO:teuthology.orchestra.run.smithi018.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2023-09-23T20:06:54.423 INFO:teuthology.orchestra.run.smithi018.stdout:systemctl: stdout inactive 2023-09-23T20:06:54.423 INFO:teuthology.orchestra.run.smithi018.stdout:firewalld.service is not enabled 2023-09-23T20:06:54.423 INFO:teuthology.orchestra.run.smithi018.stdout:Not possible to enable service . firewalld.service is not available 2023-09-23T20:06:54.423 INFO:teuthology.orchestra.run.smithi018.stdout:Waiting for mon to start... 2023-09-23T20:06:54.423 INFO:teuthology.orchestra.run.smithi018.stdout:Waiting for mon... 2023-09-23T20:06:54.621 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:54 smithi018 systemd[1]: Started Ceph mon.a for a3b3b17e-5a4c-11ee-8db4-212e2dc638e7. 2023-09-23T20:06:55.370 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout cluster: 2023-09-23T20:06:55.370 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout id: a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 2023-09-23T20:06:55.370 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout health: HEALTH_OK 2023-09-23T20:06:55.370 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout 2023-09-23T20:06:55.370 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout services: 2023-09-23T20:06:55.370 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout mon: 1 daemons, quorum a (age 0.661882s) 2023-09-23T20:06:55.370 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout mgr: no daemons active 2023-09-23T20:06:55.371 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout osd: 0 osds: 0 up, 0 in 2023-09-23T20:06:55.371 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout 2023-09-23T20:06:55.371 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout data: 2023-09-23T20:06:55.371 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout pools: 0 pools, 0 pgs 2023-09-23T20:06:55.371 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout objects: 0 objects, 0 B 2023-09-23T20:06:55.371 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout usage: 0 B used, 0 B / 0 B avail 2023-09-23T20:06:55.371 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout pgs: 2023-09-23T20:06:55.371 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout 2023-09-23T20:06:55.371 INFO:teuthology.orchestra.run.smithi018.stdout:mon is available 2023-09-23T20:06:55.371 INFO:teuthology.orchestra.run.smithi018.stdout:Assimilating anything we can from ceph.conf... 2023-09-23T20:06:55.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:55 smithi018 ceph-mon[95386]: mon.a is new leader, mons a in quorum (ranks 0) 2023-09-23T20:06:55.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:55 smithi018 ceph-mon[95386]: monmap e1: 1 mons at {a=[v2:172.21.15.18:3300/0,v1:172.21.15.18:6789/0]} removed_ranks: {} 2023-09-23T20:06:55.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:55 smithi018 ceph-mon[95386]: fsmap 2023-09-23T20:06:55.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:55 smithi018 ceph-mon[95386]: osdmap e1: 0 total, 0 up, 0 in 2023-09-23T20:06:55.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:55 smithi018 ceph-mon[95386]: mgrmap e1: no daemons active 2023-09-23T20:06:55.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:55 smithi018 ceph-mon[95386]: from='client.? 172.21.15.18:0/2566420506' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2023-09-23T20:06:56.313 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout 2023-09-23T20:06:56.313 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout [global] 2023-09-23T20:06:56.314 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout fsid = a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 2023-09-23T20:06:56.314 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout mon_host = [v2:172.21.15.18:3300,v1:172.21.15.18:6789] 2023-09-23T20:06:56.314 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2023-09-23T20:06:56.314 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2023-09-23T20:06:56.314 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2023-09-23T20:06:56.314 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2023-09-23T20:06:56.314 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout 2023-09-23T20:06:56.314 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout [mgr] 2023-09-23T20:06:56.314 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2023-09-23T20:06:56.314 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout 2023-09-23T20:06:56.314 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout [osd] 2023-09-23T20:06:56.315 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2023-09-23T20:06:56.315 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2023-09-23T20:06:56.315 INFO:teuthology.orchestra.run.smithi018.stdout:Generating new minimal ceph.conf... 2023-09-23T20:06:56.558 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:56 smithi018 ceph-mon[95386]: from='client.? 172.21.15.18:0/3694479' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2023-09-23T20:06:56.558 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:56 smithi018 ceph-mon[95386]: from='client.? 172.21.15.18:0/3694479' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2023-09-23T20:06:57.254 INFO:teuthology.orchestra.run.smithi018.stdout:Restarting the monitor... 2023-09-23T20:06:57.598 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:57 smithi018 systemd[1]: Stopping Ceph mon.a for a3b3b17e-5a4c-11ee-8db4-212e2dc638e7... 2023-09-23T20:06:57.598 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:57 smithi018 ceph-mon[95386]: from='client.? 172.21.15.18:0/480740316' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:06:57.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:57 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mon-a[95361]: 2023-09-23T20:06:57.596+0000 7f80863f2700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T20:06:57.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:57 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mon-a[95361]: 2023-09-23T20:06:57.596+0000 7f80863f2700 -1 mon.a@0(leader) e1 *** Got Signal Terminated *** 2023-09-23T20:06:58.159 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 bash[95721]: ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mon-a 2023-09-23T20:06:58.603 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 systemd[1]: ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7@mon.a.service: Succeeded. 2023-09-23T20:06:58.603 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 systemd[1]: Stopped Ceph mon.a for a3b3b17e-5a4c-11ee-8db4-212e2dc638e7. 2023-09-23T20:06:58.603 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 systemd[1]: Starting Ceph mon.a for a3b3b17e-5a4c-11ee-8db4-212e2dc638e7... 2023-09-23T20:06:58.827 INFO:teuthology.orchestra.run.smithi018.stdout:Setting mon public_network to 172.21.0.0/20 2023-09-23T20:06:58.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 podman[95849]: 2023-09-23T20:06:58.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: set uid:gid to 167:167 (ceph:ceph) 2023-09-23T20:06:58.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable), process ceph-mon, pid 7 2023-09-23T20:06:58.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: pidfile_write: ignore empty --pid-file 2023-09-23T20:06:58.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: load: jerasure load: lrc 2023-09-23T20:06:58.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: RocksDB version: 6.15.5 2023-09-23T20:06:58.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2023-09-23T20:06:58.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Compile date Sep 22 2023 2023-09-23T20:06:58.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: DB SUMMARY 2023-09-23T20:06:58.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: DB Session ID: B42EVPEW1JXQ3971DC8F 2023-09-23T20:06:58.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: CURRENT file: CURRENT 2023-09-23T20:06:58.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: IDENTITY file: IDENTITY 2023-09-23T20:06:58.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: MANIFEST file: MANIFEST-000009 size: 131 Bytes 2023-09-23T20:06:58.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000008.sst 2023-09-23T20:06:58.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000010.log size: 83561 ; 2023-09-23T20:06:58.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.error_if_exists: 0 2023-09-23T20:06:58.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.create_if_missing: 0 2023-09-23T20:06:58.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.paranoid_checks: 1 2023-09-23T20:06:58.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-09-23T20:06:58.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.env: 0x55f817251a80 2023-09-23T20:06:58.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.fs: Posix File System 2023-09-23T20:06:58.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.info_log: 0x55f818a01d80 2023-09-23T20:06:58.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.max_file_opening_threads: 16 2023-09-23T20:06:58.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.statistics: (nil) 2023-09-23T20:06:58.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.use_fsync: 0 2023-09-23T20:06:58.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.max_log_file_size: 0 2023-09-23T20:06:58.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.max_manifest_file_size: 1073741824 2023-09-23T20:06:58.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.log_file_time_to_roll: 0 2023-09-23T20:06:58.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.keep_log_file_num: 1000 2023-09-23T20:06:58.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.recycle_log_file_num: 0 2023-09-23T20:06:58.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.allow_fallocate: 1 2023-09-23T20:06:58.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.allow_mmap_reads: 0 2023-09-23T20:06:58.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.allow_mmap_writes: 0 2023-09-23T20:06:58.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.use_direct_reads: 0 2023-09-23T20:06:58.895 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-09-23T20:06:58.895 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.create_missing_column_families: 0 2023-09-23T20:06:58.895 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.db_log_dir: 2023-09-23T20:06:58.895 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-a/store.db 2023-09-23T20:06:58.895 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.table_cache_numshardbits: 6 2023-09-23T20:06:58.895 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.WAL_ttl_seconds: 0 2023-09-23T20:06:58.895 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.WAL_size_limit_MB: 0 2023-09-23T20:06:58.895 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-09-23T20:06:58.895 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.manifest_preallocation_size: 4194304 2023-09-23T20:06:58.895 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.is_fd_close_on_exec: 1 2023-09-23T20:06:58.896 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.advise_random_on_open: 1 2023-09-23T20:06:58.896 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.db_write_buffer_size: 0 2023-09-23T20:06:58.896 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.write_buffer_manager: 0x55f8196f4960 2023-09-23T20:06:58.896 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.access_hint_on_compaction_start: 1 2023-09-23T20:06:58.896 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2023-09-23T20:06:58.896 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.random_access_max_buffer_size: 1048576 2023-09-23T20:06:58.896 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.use_adaptive_mutex: 0 2023-09-23T20:06:58.896 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.rate_limiter: (nil) 2023-09-23T20:06:58.896 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-09-23T20:06:58.896 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.wal_recovery_mode: 2 2023-09-23T20:06:58.897 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.enable_thread_tracking: 0 2023-09-23T20:06:58.897 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 bash[95849]: 45b1b543db196f617572ef4bb86e32930a08e32d2e0a118e6a333deacd3b7436 2023-09-23T20:06:58.897 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.enable_pipelined_write: 0 2023-09-23T20:06:58.897 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.unordered_write: 0 2023-09-23T20:06:58.897 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.allow_concurrent_memtable_write: 1 2023-09-23T20:06:58.897 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-09-23T20:06:58.897 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.write_thread_max_yield_usec: 100 2023-09-23T20:06:58.897 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.write_thread_slow_yield_usec: 3 2023-09-23T20:06:58.897 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.row_cache: None 2023-09-23T20:06:58.897 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.wal_filter: None 2023-09-23T20:06:58.898 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.avoid_flush_during_recovery: 0 2023-09-23T20:06:58.898 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.allow_ingest_behind: 0 2023-09-23T20:06:58.898 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.preserve_deletes: 0 2023-09-23T20:06:58.898 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.two_write_queues: 0 2023-09-23T20:06:58.898 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.manual_wal_flush: 0 2023-09-23T20:06:58.898 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.atomic_flush: 0 2023-09-23T20:06:58.898 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-09-23T20:06:58.898 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.persist_stats_to_disk: 0 2023-09-23T20:06:58.898 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.write_dbid_to_manifest: 0 2023-09-23T20:06:58.898 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.log_readahead_size: 0 2023-09-23T20:06:58.899 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.file_checksum_gen_factory: Unknown 2023-09-23T20:06:58.899 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.best_efforts_recovery: 0 2023-09-23T20:06:58.899 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-09-23T20:06:58.899 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-09-23T20:06:58.899 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.allow_data_in_errors: 0 2023-09-23T20:06:58.899 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.db_host_id: __hostname__ 2023-09-23T20:06:58.900 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.max_background_jobs: 2 2023-09-23T20:06:58.900 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.max_background_compactions: -1 2023-09-23T20:06:58.900 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.max_subcompactions: 1 2023-09-23T20:06:58.901 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.avoid_flush_during_shutdown: 0 2023-09-23T20:06:58.901 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-09-23T20:06:58.901 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.delayed_write_rate : 16777216 2023-09-23T20:06:58.901 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.max_total_wal_size: 0 2023-09-23T20:06:58.901 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-09-23T20:06:58.901 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.stats_dump_period_sec: 600 2023-09-23T20:06:58.901 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.stats_persist_period_sec: 600 2023-09-23T20:06:58.901 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.stats_history_buffer_size: 1048576 2023-09-23T20:06:58.901 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.max_open_files: -1 2023-09-23T20:06:58.901 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.bytes_per_sync: 0 2023-09-23T20:06:58.902 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.wal_bytes_per_sync: 0 2023-09-23T20:06:58.902 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.strict_bytes_per_sync: 0 2023-09-23T20:06:58.902 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.compaction_readahead_size: 0 2023-09-23T20:06:58.902 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.max_background_flushes: -1 2023-09-23T20:06:58.902 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Compression algorithms supported: 2023-09-23T20:06:58.902 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: kZSTDNotFinalCompression supported: 0 2023-09-23T20:06:58.902 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: kZSTD supported: 0 2023-09-23T20:06:58.902 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: kXpressCompression supported: 0 2023-09-23T20:06:58.902 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: kLZ4HCCompression supported: 1 2023-09-23T20:06:58.902 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: kZlibCompression supported: 1 2023-09-23T20:06:58.903 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: kSnappyCompression supported: 1 2023-09-23T20:06:58.903 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: kLZ4Compression supported: 1 2023-09-23T20:06:58.903 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: kBZip2Compression supported: 0 2023-09-23T20:06:58.903 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Fast CRC32 supported: Supported on x86 2023-09-23T20:06:58.903 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 2023-09-23T20:06:58.903 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2023-09-23T20:06:58.903 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-09-23T20:06:58.903 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.merge_operator: 2023-09-23T20:06:58.903 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.compaction_filter: None 2023-09-23T20:06:58.903 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.compaction_filter_factory: None 2023-09-23T20:06:58.904 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.sst_partitioner_factory: None 2023-09-23T20:06:58.904 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.memtable_factory: SkipListFactory 2023-09-23T20:06:58.904 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.table_factory: BlockBasedTable 2023-09-23T20:06:58.904 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55f8189d4fa0) 2023-09-23T20:06:58.904 INFO:journalctl@ceph.mon.a.smithi018.stdout: cache_index_and_filter_blocks: 1 2023-09-23T20:06:58.904 INFO:journalctl@ceph.mon.a.smithi018.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2023-09-23T20:06:58.904 INFO:journalctl@ceph.mon.a.smithi018.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2023-09-23T20:06:58.904 INFO:journalctl@ceph.mon.a.smithi018.stdout: pin_top_level_index_and_filter: 1 2023-09-23T20:06:58.904 INFO:journalctl@ceph.mon.a.smithi018.stdout: index_type: 0 2023-09-23T20:06:58.904 INFO:journalctl@ceph.mon.a.smithi018.stdout: data_block_index_type: 0 2023-09-23T20:06:58.905 INFO:journalctl@ceph.mon.a.smithi018.stdout: index_shortening: 1 2023-09-23T20:06:58.905 INFO:journalctl@ceph.mon.a.smithi018.stdout: data_block_hash_table_util_ratio: 0.750000 2023-09-23T20:06:58.905 INFO:journalctl@ceph.mon.a.smithi018.stdout: hash_index_allow_collision: 1 2023-09-23T20:06:58.905 INFO:journalctl@ceph.mon.a.smithi018.stdout: checksum: 1 2023-09-23T20:06:58.905 INFO:journalctl@ceph.mon.a.smithi018.stdout: no_block_cache: 0 2023-09-23T20:06:58.905 INFO:journalctl@ceph.mon.a.smithi018.stdout: block_cache: 0x55f818a07090 2023-09-23T20:06:58.905 INFO:journalctl@ceph.mon.a.smithi018.stdout: block_cache_name: BinnedLRUCache 2023-09-23T20:06:58.905 INFO:journalctl@ceph.mon.a.smithi018.stdout: block_cache_options: 2023-09-23T20:06:58.905 INFO:journalctl@ceph.mon.a.smithi018.stdout: capacity : 536870912 2023-09-23T20:06:58.905 INFO:journalctl@ceph.mon.a.smithi018.stdout: num_shard_bits : 4 2023-09-23T20:06:58.906 INFO:journalctl@ceph.mon.a.smithi018.stdout: strict_capacity_limit : 0 2023-09-23T20:06:58.906 INFO:journalctl@ceph.mon.a.smithi018.stdout: high_pri_pool_ratio: 0.000 2023-09-23T20:06:58.906 INFO:journalctl@ceph.mon.a.smithi018.stdout: block_cache_compressed: (nil) 2023-09-23T20:06:58.906 INFO:journalctl@ceph.mon.a.smithi018.stdout: persistent_cache: (nil) 2023-09-23T20:06:58.906 INFO:journalctl@ceph.mon.a.smithi018.stdout: block_size: 4096 2023-09-23T20:06:58.906 INFO:journalctl@ceph.mon.a.smithi018.stdout: block_size_deviation: 10 2023-09-23T20:06:58.906 INFO:journalctl@ceph.mon.a.smithi018.stdout: block_restart_interval: 16 2023-09-23T20:06:58.906 INFO:journalctl@ceph.mon.a.smithi018.stdout: index_block_restart_interval: 1 2023-09-23T20:06:58.906 INFO:journalctl@ceph.mon.a.smithi018.stdout: metadata_block_size: 4096 2023-09-23T20:06:58.906 INFO:journalctl@ceph.mon.a.smithi018.stdout: partition_filters: 0 2023-09-23T20:06:58.907 INFO:journalctl@ceph.mon.a.smithi018.stdout: use_delta_encoding: 1 2023-09-23T20:06:58.907 INFO:journalctl@ceph.mon.a.smithi018.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2023-09-23T20:06:58.907 INFO:journalctl@ceph.mon.a.smithi018.stdout: whole_key_filtering: 1 2023-09-23T20:06:58.907 INFO:journalctl@ceph.mon.a.smithi018.stdout: verify_compression: 0 2023-09-23T20:06:58.907 INFO:journalctl@ceph.mon.a.smithi018.stdout: read_amp_bytes_per_bit: 0 2023-09-23T20:06:58.907 INFO:journalctl@ceph.mon.a.smithi018.stdout: format_version: 4 2023-09-23T20:06:58.907 INFO:journalctl@ceph.mon.a.smithi018.stdout: enable_index_compression: 1 2023-09-23T20:06:58.907 INFO:journalctl@ceph.mon.a.smithi018.stdout: block_align: 0 2023-09-23T20:06:58.907 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.write_buffer_size: 33554432 2023-09-23T20:06:58.907 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.max_write_buffer_number: 2 2023-09-23T20:06:58.908 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.compression: NoCompression 2023-09-23T20:06:58.908 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.bottommost_compression: Disabled 2023-09-23T20:06:58.908 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.prefix_extractor: nullptr 2023-09-23T20:06:58.908 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-09-23T20:06:58.908 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.num_levels: 7 2023-09-23T20:06:58.908 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-09-23T20:06:58.908 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-09-23T20:06:58.908 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-09-23T20:06:58.908 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-09-23T20:06:58.908 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.bottommost_compression_opts.level: 32767 2023-09-23T20:06:58.908 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-09-23T20:06:58.909 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-09-23T20:06:58.909 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-09-23T20:06:58.909 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-09-23T20:06:58.909 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.bottommost_compression_opts.enabled: false 2023-09-23T20:06:58.909 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.compression_opts.window_bits: -14 2023-09-23T20:06:58.909 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.compression_opts.level: 32767 2023-09-23T20:06:58.909 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.compression_opts.strategy: 0 2023-09-23T20:06:58.909 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-09-23T20:06:58.909 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-09-23T20:06:58.910 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.compression_opts.parallel_threads: 1 2023-09-23T20:06:58.910 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.compression_opts.enabled: false 2023-09-23T20:06:58.910 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-09-23T20:06:58.910 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-09-23T20:06:58.910 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.level0_stop_writes_trigger: 36 2023-09-23T20:06:58.910 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.target_file_size_base: 67108864 2023-09-23T20:06:58.910 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.target_file_size_multiplier: 1 2023-09-23T20:06:58.910 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.max_bytes_for_level_base: 268435456 2023-09-23T20:06:58.910 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-09-23T20:06:58.910 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-09-23T20:06:58.911 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-09-23T20:06:58.911 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-09-23T20:06:58.911 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-09-23T20:06:58.911 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-09-23T20:06:58.911 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-09-23T20:06:58.911 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-09-23T20:06:58.911 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-09-23T20:06:58.911 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-09-23T20:06:58.911 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.max_compaction_bytes: 1677721600 2023-09-23T20:06:58.911 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.arena_block_size: 4194304 2023-09-23T20:06:58.912 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-09-23T20:06:58.912 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-09-23T20:06:58.912 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2023-09-23T20:06:58.912 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.disable_auto_compactions: 0 2023-09-23T20:06:58.912 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-09-23T20:06:58.912 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-09-23T20:06:58.912 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-09-23T20:06:58.912 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-09-23T20:06:58.912 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-09-23T20:06:58.913 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-09-23T20:06:58.913 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-09-23T20:06:58.913 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-09-23T20:06:58.913 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-09-23T20:06:58.913 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-09-23T20:06:58.913 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.table_properties_collectors: 2023-09-23T20:06:58.913 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.inplace_update_support: 0 2023-09-23T20:06:58.913 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.inplace_update_num_locks: 10000 2023-09-23T20:06:58.913 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-09-23T20:06:58.914 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.memtable_whole_key_filtering: 0 2023-09-23T20:06:58.914 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.memtable_huge_page_size: 0 2023-09-23T20:06:58.914 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.bloom_locality: 0 2023-09-23T20:06:58.914 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.max_successive_merges: 0 2023-09-23T20:06:58.914 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.optimize_filters_for_hits: 0 2023-09-23T20:06:58.914 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.paranoid_file_checks: 0 2023-09-23T20:06:58.914 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.force_consistency_checks: 1 2023-09-23T20:06:58.914 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.report_bg_io_stats: 0 2023-09-23T20:06:58.914 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.ttl: 2592000 2023-09-23T20:06:58.914 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.periodic_compaction_seconds: 0 2023-09-23T20:06:58.915 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.enable_blob_files: false 2023-09-23T20:06:58.915 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.min_blob_size: 0 2023-09-23T20:06:58.915 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.blob_file_size: 268435456 2023-09-23T20:06:58.915 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.blob_compression_type: NoCompression 2023-09-23T20:06:58.915 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.enable_blob_garbage_collection: false 2023-09-23T20:06:58.915 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-09-23T20:06:58.915 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 11, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-09-23T20:06:58.915 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 5 2023-09-23T20:06:58.915 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: [db/version_set.cc:4082] Creating manifest 13 2023-09-23T20:06:58.915 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695499618779004, "job": 1, "event": "recovery_started", "wal_files": [10]} 2023-09-23T20:06:58.916 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: [db/db_impl/db_impl_open.cc:845] Recovering log #10 mode 2 2023-09-23T20:06:58.916 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: [table/block_based/filter_policy.cc:991] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2023-09-23T20:06:58.916 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695499618779758, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 14, "file_size": 80108, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 78376, "index_size": 220, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 10136, "raw_average_key_size": 47, "raw_value_size": 72584, "raw_average_value_size": 340, "num_data_blocks": 10, "num_entries": 213, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1695499618, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "fe7cf128-cbad-4385-8069-6200b44b8e3a", "db_session_id": "B42EVPEW1JXQ3971DC8F"}} 2023-09-23T20:06:58.916 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: [db/version_set.cc:4082] Creating manifest 15 2023-09-23T20:06:58.916 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695499618780107, "job": 1, "event": "recovery_finished"} 2023-09-23T20:06:58.916 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: [file/delete_scheduler.cc:69] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000010.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-09-23T20:06:58.916 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x55f81899b180 2023-09-23T20:06:58.916 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: DB pointer 0x55f818a70000 2023-09-23T20:06:58.916 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2023-09-23T20:06:58.916 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: rocksdb: [db/db_impl/db_impl.cc:903] 2023-09-23T20:06:58.917 INFO:journalctl@ceph.mon.a.smithi018.stdout: ** DB Stats ** 2023-09-23T20:06:58.917 INFO:journalctl@ceph.mon.a.smithi018.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T20:06:58.917 INFO:journalctl@ceph.mon.a.smithi018.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-09-23T20:06:58.917 INFO:journalctl@ceph.mon.a.smithi018.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-09-23T20:06:58.917 INFO:journalctl@ceph.mon.a.smithi018.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T20:06:58.917 INFO:journalctl@ceph.mon.a.smithi018.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-09-23T20:06:58.917 INFO:journalctl@ceph.mon.a.smithi018.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2023-09-23T20:06:58.917 INFO:journalctl@ceph.mon.a.smithi018.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T20:06:58.917 INFO:journalctl@ceph.mon.a.smithi018.stdout: 2023-09-23T20:06:58.917 INFO:journalctl@ceph.mon.a.smithi018.stdout: ** Compaction Stats [default] ** 2023-09-23T20:06:58.918 INFO:journalctl@ceph.mon.a.smithi018.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-23T20:06:58.918 INFO:journalctl@ceph.mon.a.smithi018.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T20:06:58.918 INFO:journalctl@ceph.mon.a.smithi018.stdout: L0 2/0 79.99 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 156.2 0.00 0.00 1 0.000 0 0 2023-09-23T20:06:58.918 INFO:journalctl@ceph.mon.a.smithi018.stdout: Sum 2/0 79.99 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 156.2 0.00 0.00 1 0.000 0 0 2023-09-23T20:06:58.918 INFO:journalctl@ceph.mon.a.smithi018.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 156.2 0.00 0.00 1 0.000 0 0 2023-09-23T20:06:58.918 INFO:journalctl@ceph.mon.a.smithi018.stdout: 2023-09-23T20:06:58.918 INFO:journalctl@ceph.mon.a.smithi018.stdout: ** Compaction Stats [default] ** 2023-09-23T20:06:58.918 INFO:journalctl@ceph.mon.a.smithi018.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-23T20:06:58.918 INFO:journalctl@ceph.mon.a.smithi018.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T20:06:58.918 INFO:journalctl@ceph.mon.a.smithi018.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 156.2 0.00 0.00 1 0.000 0 0 2023-09-23T20:06:58.919 INFO:journalctl@ceph.mon.a.smithi018.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T20:06:58.919 INFO:journalctl@ceph.mon.a.smithi018.stdout: Flush(GB): cumulative 0.000, interval 0.000 2023-09-23T20:06:58.919 INFO:journalctl@ceph.mon.a.smithi018.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-09-23T20:06:58.919 INFO:journalctl@ceph.mon.a.smithi018.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-09-23T20:06:58.919 INFO:journalctl@ceph.mon.a.smithi018.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-09-23T20:06:58.919 INFO:journalctl@ceph.mon.a.smithi018.stdout: AddFile(Keys): cumulative 0, interval 0 2023-09-23T20:06:58.919 INFO:journalctl@ceph.mon.a.smithi018.stdout: Cumulative compaction: 0.00 GB write, 25.55 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T20:06:58.919 INFO:journalctl@ceph.mon.a.smithi018.stdout: Interval compaction: 0.00 GB write, 25.55 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T20:06:58.919 INFO:journalctl@ceph.mon.a.smithi018.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-09-23T20:06:58.919 INFO:journalctl@ceph.mon.a.smithi018.stdout: 2023-09-23T20:06:58.920 INFO:journalctl@ceph.mon.a.smithi018.stdout: ** File Read Latency Histogram By Level [default] ** 2023-09-23T20:06:58.920 INFO:journalctl@ceph.mon.a.smithi018.stdout: 2023-09-23T20:06:58.920 INFO:journalctl@ceph.mon.a.smithi018.stdout: ** Compaction Stats [default] ** 2023-09-23T20:06:58.920 INFO:journalctl@ceph.mon.a.smithi018.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-23T20:06:58.920 INFO:journalctl@ceph.mon.a.smithi018.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T20:06:58.920 INFO:journalctl@ceph.mon.a.smithi018.stdout: L0 2/0 79.99 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 156.2 0.00 0.00 1 0.000 0 0 2023-09-23T20:06:58.920 INFO:journalctl@ceph.mon.a.smithi018.stdout: Sum 2/0 79.99 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 156.2 0.00 0.00 1 0.000 0 0 2023-09-23T20:06:58.920 INFO:journalctl@ceph.mon.a.smithi018.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-09-23T20:06:58.920 INFO:journalctl@ceph.mon.a.smithi018.stdout: 2023-09-23T20:06:58.920 INFO:journalctl@ceph.mon.a.smithi018.stdout: ** Compaction Stats [default] ** 2023-09-23T20:06:58.921 INFO:journalctl@ceph.mon.a.smithi018.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-23T20:06:58.921 INFO:journalctl@ceph.mon.a.smithi018.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T20:06:58.921 INFO:journalctl@ceph.mon.a.smithi018.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 156.2 0.00 0.00 1 0.000 0 0 2023-09-23T20:06:58.921 INFO:journalctl@ceph.mon.a.smithi018.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T20:06:58.921 INFO:journalctl@ceph.mon.a.smithi018.stdout: Flush(GB): cumulative 0.000, interval 0.000 2023-09-23T20:06:58.921 INFO:journalctl@ceph.mon.a.smithi018.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-09-23T20:06:58.921 INFO:journalctl@ceph.mon.a.smithi018.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-09-23T20:06:58.921 INFO:journalctl@ceph.mon.a.smithi018.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-09-23T20:06:58.921 INFO:journalctl@ceph.mon.a.smithi018.stdout: AddFile(Keys): cumulative 0, interval 0 2023-09-23T20:06:58.921 INFO:journalctl@ceph.mon.a.smithi018.stdout: Cumulative compaction: 0.00 GB write, 25.30 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T20:06:58.922 INFO:journalctl@ceph.mon.a.smithi018.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T20:06:58.922 INFO:journalctl@ceph.mon.a.smithi018.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-09-23T20:06:58.922 INFO:journalctl@ceph.mon.a.smithi018.stdout: 2023-09-23T20:06:58.922 INFO:journalctl@ceph.mon.a.smithi018.stdout: ** File Read Latency Histogram By Level [default] ** 2023-09-23T20:06:58.922 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: starting mon.a rank 0 at public addrs [v2:172.21.15.18:3300/0,v1:172.21.15.18:6789/0] at bind addrs [v2:172.21.15.18:3300/0,v1:172.21.15.18:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 2023-09-23T20:06:58.922 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: mon.a@-1(???) e1 preinit fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 2023-09-23T20:06:58.922 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: mon.a@-1(???).mds e1 new map 2023-09-23T20:06:58.922 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: mon.a@-1(???).mds e1 print_map 2023-09-23T20:06:58.922 INFO:journalctl@ceph.mon.a.smithi018.stdout: e1 2023-09-23T20:06:58.922 INFO:journalctl@ceph.mon.a.smithi018.stdout: enable_multiple, ever_enabled_multiple: 1,1 2023-09-23T20:06:58.923 INFO:journalctl@ceph.mon.a.smithi018.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-09-23T20:06:58.923 INFO:journalctl@ceph.mon.a.smithi018.stdout: legacy client fscid: -1 2023-09-23T20:06:58.923 INFO:journalctl@ceph.mon.a.smithi018.stdout: 2023-09-23T20:06:58.923 INFO:journalctl@ceph.mon.a.smithi018.stdout: No filesystems configured 2023-09-23T20:06:58.923 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: mon.a@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2023-09-23T20:06:58.923 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2023-09-23T20:06:58.923 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2023-09-23T20:06:58.923 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2023-09-23T20:06:58.923 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: mon.a@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2023-09-23T20:06:58.923 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 systemd[1]: Started Ceph mon.a for a3b3b17e-5a4c-11ee-8db4-212e2dc638e7. 2023-09-23T20:06:58.924 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: mon.a is new leader, mons a in quorum (ranks 0) 2023-09-23T20:06:58.924 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: monmap e1: 1 mons at {a=[v2:172.21.15.18:3300/0,v1:172.21.15.18:6789/0]} removed_ranks: {} 2023-09-23T20:06:58.924 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: fsmap 2023-09-23T20:06:58.924 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: osdmap e1: 0 total, 0 up, 0 in 2023-09-23T20:06:58.924 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:06:58 smithi018 ceph-mon[95889]: mgrmap e1: no daemons active 2023-09-23T20:06:59.874 INFO:teuthology.orchestra.run.smithi018.stdout:Wrote config to /etc/ceph/ceph.conf 2023-09-23T20:06:59.875 INFO:teuthology.orchestra.run.smithi018.stdout:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2023-09-23T20:06:59.875 INFO:teuthology.orchestra.run.smithi018.stdout:Creating mgr... 2023-09-23T20:06:59.875 INFO:teuthology.orchestra.run.smithi018.stdout:Verifying port 9283 ... 2023-09-23T20:07:00.016 INFO:teuthology.orchestra.run.smithi018.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7@mgr.y 2023-09-23T20:07:00.016 INFO:teuthology.orchestra.run.smithi018.stdout:systemctl: stderr Failed to reset failed state of unit ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7@mgr.y.service: Unit ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7@mgr.y.service not loaded. 2023-09-23T20:07:00.109 INFO:teuthology.orchestra.run.smithi018.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7.target.wants/ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7@mgr.y.service → /etc/systemd/system/ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7@.service. 2023-09-23T20:07:00.391 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:07:00 smithi018 systemd[1]: Starting Ceph mgr.y for a3b3b17e-5a4c-11ee-8db4-212e2dc638e7... 2023-09-23T20:07:00.701 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:07:00 smithi018 ceph-mon[95889]: from='client.? 172.21.15.18:0/3055568858' entity='client.admin' 2023-09-23T20:07:00.702 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:07:00 smithi018 podman[96117]: 2023-09-23T20:07:00.702 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:07:00 smithi018 bash[96117]: e62744cd423b579126c06ee7e00f0daeccf7697447a841136fe07925afed4dc7 2023-09-23T20:07:00.713 INFO:teuthology.orchestra.run.smithi018.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2023-09-23T20:07:00.714 INFO:teuthology.orchestra.run.smithi018.stdout:systemctl: stdout disabled 2023-09-23T20:07:00.721 INFO:teuthology.orchestra.run.smithi018.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2023-09-23T20:07:00.722 INFO:teuthology.orchestra.run.smithi018.stdout:systemctl: stdout inactive 2023-09-23T20:07:00.722 INFO:teuthology.orchestra.run.smithi018.stdout:firewalld.service is not enabled 2023-09-23T20:07:00.722 INFO:teuthology.orchestra.run.smithi018.stdout:Not possible to enable service . firewalld.service is not available 2023-09-23T20:07:00.731 INFO:teuthology.orchestra.run.smithi018.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2023-09-23T20:07:00.731 INFO:teuthology.orchestra.run.smithi018.stdout:systemctl: stdout disabled 2023-09-23T20:07:00.739 INFO:teuthology.orchestra.run.smithi018.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2023-09-23T20:07:00.739 INFO:teuthology.orchestra.run.smithi018.stdout:systemctl: stdout inactive 2023-09-23T20:07:00.739 INFO:teuthology.orchestra.run.smithi018.stdout:firewalld.service is not enabled 2023-09-23T20:07:00.739 INFO:teuthology.orchestra.run.smithi018.stdout:Not possible to open ports <[9283]>. firewalld.service is not available 2023-09-23T20:07:00.739 INFO:teuthology.orchestra.run.smithi018.stdout:Waiting for mgr to start... 2023-09-23T20:07:00.739 INFO:teuthology.orchestra.run.smithi018.stdout:Waiting for mgr... 2023-09-23T20:07:00.954 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:07:00 smithi018 systemd[1]: Started Ceph mgr.y for a3b3b17e-5a4c-11ee-8db4-212e2dc638e7. 2023-09-23T20:07:00.955 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:07:00 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:07:00.794+0000 7f2cecf121c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-23T20:07:00.955 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:07:00 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:07:00.888+0000 7f2cecf121c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T20:07:01.315 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:07:00 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:07:00.998+0000 7f2cecf121c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T20:07:01.315 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:07:01 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:07:01.108+0000 7f2cecf121c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T20:07:01.640 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:07:01 smithi018 ceph-mon[95889]: from='client.? 172.21.15.18:0/3642343355' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-09-23T20:07:01.641 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:07:01 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:07:01.314+0000 7f2cecf121c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T20:07:01.973 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:07:01 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:07:01.714+0000 7f2cecf121c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T20:07:01.973 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:07:01 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:07:01.824+0000 7f2cecf121c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T20:07:01.973 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:07:01 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:07:01.973+0000 7f2cecf121c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T20:07:02.307 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:07:02 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:07:02.086+0000 7f2cecf121c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T20:07:02.891 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:07:02 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:07:02.596+0000 7f2cecf121c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T20:07:03.496 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout 2023-09-23T20:07:03.496 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout { 2023-09-23T20:07:03.496 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "fsid": "a3b3b17e-5a4c-11ee-8db4-212e2dc638e7", 2023-09-23T20:07:03.496 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "health": { 2023-09-23T20:07:03.496 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2023-09-23T20:07:03.496 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "checks": {}, 2023-09-23T20:07:03.496 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "mutes": [] 2023-09-23T20:07:03.497 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout }, 2023-09-23T20:07:03.497 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2023-09-23T20:07:03.497 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "quorum": [ 2023-09-23T20:07:03.497 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout 0 2023-09-23T20:07:03.497 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout ], 2023-09-23T20:07:03.497 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2023-09-23T20:07:03.497 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "a" 2023-09-23T20:07:03.497 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout ], 2023-09-23T20:07:03.497 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "quorum_age": 2, 2023-09-23T20:07:03.497 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "monmap": { 2023-09-23T20:07:03.497 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-23T20:07:03.497 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2023-09-23T20:07:03.498 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "num_mons": 1 2023-09-23T20:07:03.498 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout }, 2023-09-23T20:07:03.498 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "osdmap": { 2023-09-23T20:07:03.498 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-23T20:07:03.498 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2023-09-23T20:07:03.498 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2023-09-23T20:07:03.498 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2023-09-23T20:07:03.498 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2023-09-23T20:07:03.498 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2023-09-23T20:07:03.498 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2023-09-23T20:07:03.498 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout }, 2023-09-23T20:07:03.498 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "pgmap": { 2023-09-23T20:07:03.499 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2023-09-23T20:07:03.499 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2023-09-23T20:07:03.499 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2023-09-23T20:07:03.499 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2023-09-23T20:07:03.499 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2023-09-23T20:07:03.499 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2023-09-23T20:07:03.499 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2023-09-23T20:07:03.499 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2023-09-23T20:07:03.499 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout }, 2023-09-23T20:07:03.499 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "fsmap": { 2023-09-23T20:07:03.499 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-23T20:07:03.499 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "by_rank": [], 2023-09-23T20:07:03.500 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "up:standby": 0 2023-09-23T20:07:03.500 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout }, 2023-09-23T20:07:03.500 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "mgrmap": { 2023-09-23T20:07:03.500 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "available": false, 2023-09-23T20:07:03.500 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2023-09-23T20:07:03.500 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "modules": [ 2023-09-23T20:07:03.500 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "iostat", 2023-09-23T20:07:03.500 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "nfs", 2023-09-23T20:07:03.500 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "restful" 2023-09-23T20:07:03.500 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout ], 2023-09-23T20:07:03.500 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "services": {} 2023-09-23T20:07:03.500 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout }, 2023-09-23T20:07:03.500 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "servicemap": { 2023-09-23T20:07:03.501 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-23T20:07:03.501 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "modified": "2023-09-23T20:06:54.359506+0000", 2023-09-23T20:07:03.501 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "services": {} 2023-09-23T20:07:03.501 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout }, 2023-09-23T20:07:03.501 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "progress_events": {} 2023-09-23T20:07:03.501 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout } 2023-09-23T20:07:03.501 INFO:teuthology.orchestra.run.smithi018.stdout:mgr not available, waiting (1/15)... 2023-09-23T20:07:03.640 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:07:03 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:07:03.289+0000 7f2cecf121c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T20:07:03.641 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:07:03 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:07:03.482+0000 7f2cecf121c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T20:07:04.768 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:07:04 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:07:04.445+0000 7f2cecf121c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T20:07:04.768 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:07:04 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:07:04.553+0000 7f2cecf121c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-23T20:07:04.768 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:07:04 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:07:04.767+0000 7f2cecf121c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T20:07:05.390 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:07:05 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:07:05.111+0000 7f2cecf121c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-23T20:07:05.391 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:07:05 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:07:05.214+0000 7f2cecf121c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T20:07:05.868 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:07:05 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:07:05.549+0000 7f2cecf121c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T20:07:06.140 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:07:05 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:07:05.868+0000 7f2cecf121c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T20:07:07.083 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:07:06 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:07:06.689+0000 7f2cecf121c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T20:07:07.083 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:07:06 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:07:06.845+0000 7f2cecf121c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T20:07:07.390 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:07:07 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:07:07.083+0000 7f2cecf121c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T20:07:07.390 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:07:07 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:07:07.221+0000 7f2cecf121c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T20:07:07.890 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:07:07 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:07:07.491+0000 7f2cecf121c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T20:07:08.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:07:08 smithi018 ceph-mon[95889]: from='client.? 172.21.15.18:0/2925493663' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-09-23T20:07:08.891 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:07:08 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:07:08.535+0000 7f2cecf121c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T20:07:09.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:07:09 smithi018 ceph-mon[95889]: Activating manager daemon y 2023-09-23T20:07:09.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:07:09 smithi018 ceph-mon[95889]: mgrmap e2: y(active, starting, since 0.00283436s) 2023-09-23T20:07:09.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:07:09 smithi018 ceph-mon[95889]: from='mgr.14100 172.21.15.18:0/4231738002' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T20:07:09.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:07:09 smithi018 ceph-mon[95889]: from='mgr.14100 172.21.15.18:0/4231738002' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T20:07:09.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:07:09 smithi018 ceph-mon[95889]: from='mgr.14100 172.21.15.18:0/4231738002' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T20:07:09.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:07:09 smithi018 ceph-mon[95889]: from='mgr.14100 172.21.15.18:0/4231738002' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T20:07:09.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:07:09 smithi018 ceph-mon[95889]: from='mgr.14100 172.21.15.18:0/4231738002' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T20:07:09.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:07:09 smithi018 ceph-mon[95889]: Manager daemon y is now available 2023-09-23T20:07:09.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:07:09 smithi018 ceph-mon[95889]: from='mgr.14100 172.21.15.18:0/4231738002' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T20:07:09.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:07:09 smithi018 ceph-mon[95889]: from='mgr.14100 172.21.15.18:0/4231738002' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T20:07:09.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:07:09 smithi018 ceph-mon[95889]: from='mgr.14100 172.21.15.18:0/4231738002' entity='mgr.y' 2023-09-23T20:07:09.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:07:09 smithi018 ceph-mon[95889]: from='mgr.14100 172.21.15.18:0/4231738002' entity='mgr.y' 2023-09-23T20:07:09.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:07:09 smithi018 ceph-mon[95889]: from='mgr.14100 172.21.15.18:0/4231738002' entity='mgr.y' 2023-09-23T20:07:10.551 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout 2023-09-23T20:07:10.551 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout { 2023-09-23T20:07:10.551 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "fsid": "a3b3b17e-5a4c-11ee-8db4-212e2dc638e7", 2023-09-23T20:07:10.551 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "health": { 2023-09-23T20:07:10.551 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2023-09-23T20:07:10.551 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "checks": {}, 2023-09-23T20:07:10.551 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "mutes": [] 2023-09-23T20:07:10.551 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout }, 2023-09-23T20:07:10.552 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2023-09-23T20:07:10.552 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "quorum": [ 2023-09-23T20:07:10.552 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout 0 2023-09-23T20:07:10.552 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout ], 2023-09-23T20:07:10.552 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2023-09-23T20:07:10.552 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "a" 2023-09-23T20:07:10.552 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout ], 2023-09-23T20:07:10.552 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "quorum_age": 9, 2023-09-23T20:07:10.552 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "monmap": { 2023-09-23T20:07:10.552 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-23T20:07:10.553 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2023-09-23T20:07:10.554 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "num_mons": 1 2023-09-23T20:07:10.554 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout }, 2023-09-23T20:07:10.554 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "osdmap": { 2023-09-23T20:07:10.554 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-23T20:07:10.554 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2023-09-23T20:07:10.554 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2023-09-23T20:07:10.554 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2023-09-23T20:07:10.554 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2023-09-23T20:07:10.554 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2023-09-23T20:07:10.554 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2023-09-23T20:07:10.557 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout }, 2023-09-23T20:07:10.557 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "pgmap": { 2023-09-23T20:07:10.557 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2023-09-23T20:07:10.558 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2023-09-23T20:07:10.558 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2023-09-23T20:07:10.558 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2023-09-23T20:07:10.558 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2023-09-23T20:07:10.558 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2023-09-23T20:07:10.558 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2023-09-23T20:07:10.558 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2023-09-23T20:07:10.558 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout }, 2023-09-23T20:07:10.558 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "fsmap": { 2023-09-23T20:07:10.558 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-23T20:07:10.558 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "by_rank": [], 2023-09-23T20:07:10.559 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "up:standby": 0 2023-09-23T20:07:10.559 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout }, 2023-09-23T20:07:10.559 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "mgrmap": { 2023-09-23T20:07:10.559 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "available": false, 2023-09-23T20:07:10.559 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2023-09-23T20:07:10.559 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "modules": [ 2023-09-23T20:07:10.559 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "iostat", 2023-09-23T20:07:10.559 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "nfs", 2023-09-23T20:07:10.559 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "restful" 2023-09-23T20:07:10.559 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout ], 2023-09-23T20:07:10.559 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "services": {} 2023-09-23T20:07:10.559 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout }, 2023-09-23T20:07:10.560 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "servicemap": { 2023-09-23T20:07:10.560 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-23T20:07:10.560 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "modified": "2023-09-23T20:06:54.359506+0000", 2023-09-23T20:07:10.560 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "services": {} 2023-09-23T20:07:10.560 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout }, 2023-09-23T20:07:10.560 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "progress_events": {} 2023-09-23T20:07:10.560 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout } 2023-09-23T20:07:10.560 INFO:teuthology.orchestra.run.smithi018.stdout:mgr not available, waiting (2/15)... 2023-09-23T20:07:10.890 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:07:10 smithi018 ceph-mon[95889]: mgrmap e3: y(active, since 1.00689s) 2023-09-23T20:07:12.832 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:07:12 smithi018 ceph-mon[95889]: mgrmap e4: y(active, since 3s) 2023-09-23T20:07:13.890 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:07:13 smithi018 ceph-mon[95889]: from='client.? 172.21.15.18:0/3334456656' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-09-23T20:07:13.905 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout 2023-09-23T20:07:13.908 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout { 2023-09-23T20:07:13.909 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "fsid": "a3b3b17e-5a4c-11ee-8db4-212e2dc638e7", 2023-09-23T20:07:13.909 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "health": { 2023-09-23T20:07:13.909 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2023-09-23T20:07:13.909 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "checks": {}, 2023-09-23T20:07:13.909 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "mutes": [] 2023-09-23T20:07:13.909 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout }, 2023-09-23T20:07:13.909 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2023-09-23T20:07:13.910 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "quorum": [ 2023-09-23T20:07:13.910 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout 0 2023-09-23T20:07:13.910 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout ], 2023-09-23T20:07:13.910 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2023-09-23T20:07:13.910 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "a" 2023-09-23T20:07:13.910 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout ], 2023-09-23T20:07:13.910 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "quorum_age": 14, 2023-09-23T20:07:13.910 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "monmap": { 2023-09-23T20:07:13.910 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-23T20:07:13.911 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2023-09-23T20:07:13.911 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "num_mons": 1 2023-09-23T20:07:13.911 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout }, 2023-09-23T20:07:13.911 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "osdmap": { 2023-09-23T20:07:13.911 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-23T20:07:13.911 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2023-09-23T20:07:13.911 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2023-09-23T20:07:13.911 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2023-09-23T20:07:13.911 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2023-09-23T20:07:13.911 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2023-09-23T20:07:13.912 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2023-09-23T20:07:13.915 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout }, 2023-09-23T20:07:13.915 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "pgmap": { 2023-09-23T20:07:13.915 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2023-09-23T20:07:13.915 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2023-09-23T20:07:13.915 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2023-09-23T20:07:13.916 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2023-09-23T20:07:13.916 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2023-09-23T20:07:13.916 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2023-09-23T20:07:13.916 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2023-09-23T20:07:13.916 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2023-09-23T20:07:13.916 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout }, 2023-09-23T20:07:13.916 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "fsmap": { 2023-09-23T20:07:13.916 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-23T20:07:13.916 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "by_rank": [], 2023-09-23T20:07:13.916 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "up:standby": 0 2023-09-23T20:07:13.917 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout }, 2023-09-23T20:07:13.917 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "mgrmap": { 2023-09-23T20:07:13.917 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "available": true, 2023-09-23T20:07:13.917 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2023-09-23T20:07:13.917 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "modules": [ 2023-09-23T20:07:13.917 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "iostat", 2023-09-23T20:07:13.917 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "nfs", 2023-09-23T20:07:13.917 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "restful" 2023-09-23T20:07:13.917 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout ], 2023-09-23T20:07:13.917 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "services": {} 2023-09-23T20:07:13.918 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout }, 2023-09-23T20:07:13.918 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "servicemap": { 2023-09-23T20:07:13.918 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-23T20:07:13.918 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "modified": "2023-09-23T20:06:54.359506+0000", 2023-09-23T20:07:13.918 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "services": {} 2023-09-23T20:07:13.918 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout }, 2023-09-23T20:07:13.918 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "progress_events": {} 2023-09-23T20:07:13.918 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout } 2023-09-23T20:07:13.918 INFO:teuthology.orchestra.run.smithi018.stdout:mgr is available 2023-09-23T20:07:15.052 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout 2023-09-23T20:07:15.053 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout [global] 2023-09-23T20:07:15.053 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout fsid = a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 2023-09-23T20:07:15.053 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2023-09-23T20:07:15.053 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2023-09-23T20:07:15.053 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2023-09-23T20:07:15.054 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2023-09-23T20:07:15.054 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout 2023-09-23T20:07:15.054 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout [mgr] 2023-09-23T20:07:15.054 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2023-09-23T20:07:15.054 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout 2023-09-23T20:07:15.054 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout [osd] 2023-09-23T20:07:15.054 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2023-09-23T20:07:15.055 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2023-09-23T20:07:15.055 INFO:teuthology.orchestra.run.smithi018.stdout:Enabling cephadm module... 2023-09-23T20:07:15.069 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:07:14 smithi018 ceph-mon[95889]: from='client.? 172.21.15.18:0/404075581' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2023-09-23T20:07:16.298 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:07:16 smithi018 ceph-mon[95889]: from='client.? 172.21.15.18:0/1249600209' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2023-09-23T20:07:16.298 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:07:16 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: ignoring --setuser ceph since I am not root 2023-09-23T20:07:16.298 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:07:16 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: ignoring --setgroup ceph since I am not root 2023-09-23T20:07:16.298 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:07:16 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:07:16.204+0000 7f6714c591c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-23T20:07:16.591 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:07:16 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:07:16.297+0000 7f6714c591c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T20:07:16.591 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:07:16 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:07:16.407+0000 7f6714c591c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T20:07:16.591 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:07:16 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:07:16.522+0000 7f6714c591c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T20:07:16.890 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:07:16 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:07:16.734+0000 7f6714c591c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T20:07:17.390 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:07:17 smithi018 ceph-mon[95889]: from='client.? 172.21.15.18:0/1249600209' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2023-09-23T20:07:17.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:07:17 smithi018 ceph-mon[95889]: mgrmap e5: y(active, since 7s) 2023-09-23T20:07:17.391 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:07:17 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:07:17.135+0000 7f6714c591c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T20:07:17.391 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:07:17 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:07:17.241+0000 7f6714c591c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T20:07:17.640 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:07:17 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:07:17.392+0000 7f6714c591c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T20:07:17.641 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:07:17 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:07:17.509+0000 7f6714c591c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T20:07:17.739 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout { 2023-09-23T20:07:17.740 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "epoch": 5, 2023-09-23T20:07:17.740 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "available": true, 2023-09-23T20:07:17.740 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "active_name": "y", 2023-09-23T20:07:17.740 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "num_standby": 0 2023-09-23T20:07:17.740 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout } 2023-09-23T20:07:17.740 INFO:teuthology.orchestra.run.smithi018.stdout:Waiting for the mgr to restart... 2023-09-23T20:07:17.740 INFO:teuthology.orchestra.run.smithi018.stdout:Waiting for mgr epoch 5... 2023-09-23T20:07:18.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:07:18 smithi018 ceph-mon[95889]: from='client.? 172.21.15.18:0/1026033816' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2023-09-23T20:07:18.391 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:07:18 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:07:18.028+0000 7f6714c591c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T20:07:19.140 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:07:18 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:07:18.712+0000 7f6714c591c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T20:07:19.141 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:07:18 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:07:18.902+0000 7f6714c591c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T20:07:20.140 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:07:19 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:07:19.853+0000 7f6714c591c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T20:07:20.141 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:07:19 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:07:19.961+0000 7f6714c591c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-23T20:07:20.515 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:07:20 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:07:20.174+0000 7f6714c591c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T20:07:20.890 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:07:20 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:07:20.514+0000 7f6714c591c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-23T20:07:20.891 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:07:20 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:07:20.617+0000 7f6714c591c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T20:07:21.265 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:07:20 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:07:20.946+0000 7f6714c591c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T20:07:21.640 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:07:21 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:07:21.265+0000 7f6714c591c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T20:07:22.390 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:07:22 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:07:22.086+0000 7f6714c591c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T20:07:22.391 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:07:22 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:07:22.243+0000 7f6714c591c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T20:07:22.878 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:07:22 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:07:22.478+0000 7f6714c591c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T20:07:22.878 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:07:22 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:07:22.613+0000 7f6714c591c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T20:07:23.140 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:07:22 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:07:22.878+0000 7f6714c591c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T20:07:24.390 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:07:23 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:07:23.906+0000 7f6714c591c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T20:07:25.059 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:07:24 smithi018 ceph-mon[95889]: Active manager daemon y restarted 2023-09-23T20:07:25.059 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:07:24 smithi018 ceph-mon[95889]: Activating manager daemon y 2023-09-23T20:07:25.059 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:07:24 smithi018 ceph-mon[95889]: osdmap e2: 0 total, 0 up, 0 in 2023-09-23T20:07:25.060 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:07:24 smithi018 ceph-mon[95889]: mgrmap e6: y(active, starting, since 0.00344349s) 2023-09-23T20:07:25.060 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:07:24 smithi018 ceph-mon[95889]: from='mgr.14118 172.21.15.18:0/2062881632' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T20:07:25.060 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:07:24 smithi018 ceph-mon[95889]: from='mgr.14118 172.21.15.18:0/2062881632' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T20:07:25.060 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:07:24 smithi018 ceph-mon[95889]: from='mgr.14118 172.21.15.18:0/2062881632' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T20:07:25.060 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:07:24 smithi018 ceph-mon[95889]: from='mgr.14118 172.21.15.18:0/2062881632' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T20:07:25.060 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:07:24 smithi018 ceph-mon[95889]: from='mgr.14118 172.21.15.18:0/2062881632' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T20:07:25.060 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:07:24 smithi018 ceph-mon[95889]: Manager daemon y is now available 2023-09-23T20:07:25.060 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:07:24 smithi018 ceph-mon[95889]: Found migration_current of "None". Setting to last migration. 2023-09-23T20:07:25.060 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:07:24 smithi018 ceph-mon[95889]: from='mgr.14118 172.21.15.18:0/2062881632' entity='mgr.y' 2023-09-23T20:07:25.061 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:07:24 smithi018 ceph-mon[95889]: from='mgr.14118 172.21.15.18:0/2062881632' entity='mgr.y' 2023-09-23T20:07:25.061 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:07:24 smithi018 ceph-mon[95889]: from='mgr.14118 172.21.15.18:0/2062881632' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:07:25.061 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:07:24 smithi018 ceph-mon[95889]: from='mgr.14118 172.21.15.18:0/2062881632' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:07:25.061 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:07:24 smithi018 ceph-mon[95889]: from='mgr.14118 172.21.15.18:0/2062881632' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:07:25.061 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:07:24 smithi018 ceph-mon[95889]: from='mgr.14118 172.21.15.18:0/2062881632' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T20:07:25.061 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:07:24 smithi018 ceph-mon[95889]: from='mgr.14118 172.21.15.18:0/2062881632' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T20:07:25.296 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout { 2023-09-23T20:07:25.296 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 7, 2023-09-23T20:07:25.296 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "initialized": true 2023-09-23T20:07:25.296 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout } 2023-09-23T20:07:25.297 INFO:teuthology.orchestra.run.smithi018.stdout:mgr epoch 5 is available 2023-09-23T20:07:25.297 INFO:teuthology.orchestra.run.smithi018.stdout:Setting orchestrator backend to cephadm... 2023-09-23T20:07:26.172 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:07:25 smithi018 ceph-mon[95889]: from='client.14122 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2023-09-23T20:07:26.172 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:07:25 smithi018 ceph-mon[95889]: mgrmap e7: y(active, since 1.00513s) 2023-09-23T20:07:26.173 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:07:25 smithi018 ceph-mon[95889]: from='client.14122 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2023-09-23T20:07:26.173 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:07:25 smithi018 ceph-mon[95889]: [23/Sep/2023:20:07:25] ENGINE Bus STARTING 2023-09-23T20:07:26.173 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:07:25 smithi018 ceph-mon[95889]: from='mgr.14118 172.21.15.18:0/2062881632' entity='mgr.y' 2023-09-23T20:07:26.173 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:07:25 smithi018 ceph-mon[95889]: from='mgr.14118 172.21.15.18:0/2062881632' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:07:27.140 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:07:26 smithi018 ceph-mon[95889]: [23/Sep/2023:20:07:25] ENGINE Serving on https://172.21.15.18:7150 2023-09-23T20:07:27.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:07:26 smithi018 ceph-mon[95889]: [23/Sep/2023:20:07:25] ENGINE Bus STARTED 2023-09-23T20:07:27.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:07:26 smithi018 ceph-mon[95889]: from='mgr.14118 172.21.15.18:0/2062881632' entity='mgr.y' 2023-09-23T20:07:27.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:07:26 smithi018 ceph-mon[95889]: from='mgr.14118 172.21.15.18:0/2062881632' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:07:27.780 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout value unchanged 2023-09-23T20:07:27.780 INFO:teuthology.orchestra.run.smithi018.stdout:Generating ssh key... 2023-09-23T20:07:28.017 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:07:27 smithi018 ceph-mon[95889]: from='client.14130 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:07:28.017 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:07:27 smithi018 ceph-mon[95889]: mgrmap e8: y(active, since 3s) 2023-09-23T20:07:29.140 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:07:28 smithi018 ceph-mon[95889]: from='client.14132 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:07:29.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:07:28 smithi018 ceph-mon[95889]: from='mgr.14118 172.21.15.18:0/2062881632' entity='mgr.y' 2023-09-23T20:07:29.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:07:28 smithi018 ceph-mon[95889]: from='mgr.14118 172.21.15.18:0/2062881632' entity='mgr.y' 2023-09-23T20:07:29.141 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:07:28 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: Generating public/private rsa key pair. 2023-09-23T20:07:29.141 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:07:28 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: Your identification has been saved in /tmp/tmpm3dnkutw/key. 2023-09-23T20:07:29.141 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:07:28 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: Your public key has been saved in /tmp/tmpm3dnkutw/key.pub. 2023-09-23T20:07:29.141 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:07:28 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: The key fingerprint is: 2023-09-23T20:07:29.141 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:07:28 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: SHA256:wqKsh2595loxKNe5Wx0xwBcpkPrNdBqUHTod5BhqXIg ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 2023-09-23T20:07:29.141 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:07:28 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: The key's randomart image is: 2023-09-23T20:07:29.142 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:07:28 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: +---[RSA 3072]----+ 2023-09-23T20:07:29.142 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:07:28 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: | ..=++=+ | 2023-09-23T20:07:29.142 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:07:28 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: | E.oo=B+. | 2023-09-23T20:07:29.142 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:07:28 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: | .+.+o= | 2023-09-23T20:07:29.142 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:07:28 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: | .+..o..o | 2023-09-23T20:07:29.142 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:07:28 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: | . oo=* S. | 2023-09-23T20:07:29.142 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:07:28 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: | .o. o+=. . | 2023-09-23T20:07:29.142 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:07:28 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: | .+ o . . | 2023-09-23T20:07:29.142 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:07:28 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: |.o...oo | 2023-09-23T20:07:29.142 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:07:28 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: |+o .=o | 2023-09-23T20:07:29.143 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:07:28 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: +----[SHA256]-----+ 2023-09-23T20:07:30.199 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:07:29 smithi018 ceph-mon[95889]: from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:07:30.199 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:07:29 smithi018 ceph-mon[95889]: Generating ssh key... 2023-09-23T20:07:30.462 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC+wE4KNWwjHrBS2IVTtvmoMWh3jFGBcXmP+MQVqHr/XgSkEwskVANTX+YHciPQGIu1MUem0lmxVeXc6+TNyzSBe22gO+TYPRWzArY0c1+wvNzMofhHMe2ewRzXzIp1x+HPaRGV8+yqvrqid3F64gSYcgd51s0oTUeDjbnVRXoL1Sa3P+1N379vyUu2AfoYewmX7+svnzvwQLXnjAY/yC+94GqNobpa9XZGV7ji913SBsP8X1d1pI+Wgh9HtiV4i7Mh8biutLs8eS8HaUONNFYxYSIcI4qlo1loN4/hrRYfRu44uQve71q2EY/O7nEhyrTMuwHlIKHZ1r6AVGiLZbawxsdQwHyHxwUnPiS3F06Fg+tXQb0WHk/xvSLT396A1b/EsbxuYotTW1I5GpxbGtwpgCnQkfP289W3BI1GExs2nxcc+iw3z17LT1FLR0BFewdFayJfGTmoKfAisIc+J4+xUSGvH5VANGvVPUe/rdlNxu+i6sJN7Ka5tR4tI/aBYGM= ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 2023-09-23T20:07:30.463 INFO:teuthology.orchestra.run.smithi018.stdout:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2023-09-23T20:07:30.463 INFO:teuthology.orchestra.run.smithi018.stdout:Adding key to root@localhost authorized_keys... 2023-09-23T20:07:30.464 INFO:teuthology.orchestra.run.smithi018.stdout:Adding host smithi018... 2023-09-23T20:07:31.348 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:07:30 smithi018 ceph-mon[95889]: from='client.14136 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:07:32.140 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:07:32 smithi018 ceph-mon[95889]: from='client.14138 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi018", "addr": "172.21.15.18", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:07:32.140 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:07:32 smithi018 ceph-mon[95889]: Deploying cephadm binary to smithi018 2023-09-23T20:07:33.711 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout Added host 'smithi018' with addr '172.21.15.18' 2023-09-23T20:07:33.712 INFO:teuthology.orchestra.run.smithi018.stdout:Deploying unmanaged mon service... 2023-09-23T20:07:34.640 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:07:34 smithi018 ceph-mon[95889]: from='mgr.14118 172.21.15.18:0/2062881632' entity='mgr.y' 2023-09-23T20:07:34.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:07:34 smithi018 ceph-mon[95889]: Added host smithi018 2023-09-23T20:07:34.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:07:34 smithi018 ceph-mon[95889]: from='mgr.14118 172.21.15.18:0/2062881632' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:07:35.459 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout Scheduled mon update... 2023-09-23T20:07:35.459 INFO:teuthology.orchestra.run.smithi018.stdout:Deploying unmanaged mgr service... 2023-09-23T20:07:35.890 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:07:35 smithi018 ceph-mon[95889]: from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:07:35.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:07:35 smithi018 ceph-mon[95889]: Saving service mon spec with placement count:5 2023-09-23T20:07:35.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:07:35 smithi018 ceph-mon[95889]: from='mgr.14118 172.21.15.18:0/2062881632' entity='mgr.y' 2023-09-23T20:07:35.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:07:35 smithi018 ceph-mon[95889]: from='mgr.14118 172.21.15.18:0/2062881632' entity='mgr.y' 2023-09-23T20:07:36.702 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout Scheduled mgr update... 2023-09-23T20:07:36.962 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:07:36 smithi018 ceph-mon[95889]: from='mgr.14118 172.21.15.18:0/2062881632' entity='mgr.y' 2023-09-23T20:07:36.962 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:07:36 smithi018 ceph-mon[95889]: from='mgr.14118 172.21.15.18:0/2062881632' entity='mgr.y' 2023-09-23T20:07:38.124 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:07:37 smithi018 ceph-mon[95889]: from='client.14142 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:07:38.124 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:07:37 smithi018 ceph-mon[95889]: Saving service mgr spec with placement count:2 2023-09-23T20:07:38.124 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:07:37 smithi018 ceph-mon[95889]: from='client.? 172.21.15.18:0/2725688053' entity='client.admin' 2023-09-23T20:07:38.125 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:07:37 smithi018 ceph-mon[95889]: from='mgr.14118 172.21.15.18:0/2062881632' entity='mgr.y' 2023-09-23T20:07:39.092 INFO:teuthology.orchestra.run.smithi018.stdout:Enabling the dashboard module... 2023-09-23T20:07:40.041 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:07:39 smithi018 ceph-mon[95889]: from='mgr.14118 172.21.15.18:0/2062881632' entity='mgr.y' 2023-09-23T20:07:40.041 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:07:39 smithi018 ceph-mon[95889]: from='client.? 172.21.15.18:0/4067842578' entity='client.admin' 2023-09-23T20:07:41.017 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:07:40 smithi018 ceph-mon[95889]: from='client.? 172.21.15.18:0/234036976' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2023-09-23T20:07:41.017 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:07:40 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: ignoring --setuser ceph since I am not root 2023-09-23T20:07:41.017 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:07:40 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: ignoring --setgroup ceph since I am not root 2023-09-23T20:07:41.017 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:07:40 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:07:40.924+0000 7fd3ae5ff1c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-23T20:07:41.390 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:07:41 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:07:41.016+0000 7fd3ae5ff1c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T20:07:41.390 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:07:41 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:07:41.122+0000 7fd3ae5ff1c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T20:07:41.391 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:07:41 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:07:41.245+0000 7fd3ae5ff1c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T20:07:41.722 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:07:41 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:07:41.447+0000 7fd3ae5ff1c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T20:07:42.105 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:07:41 smithi018 ceph-mon[95889]: from='client.? 172.21.15.18:0/234036976' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2023-09-23T20:07:42.105 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:07:41 smithi018 ceph-mon[95889]: mgrmap e9: y(active, since 16s) 2023-09-23T20:07:42.105 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:07:41 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:07:41.846+0000 7fd3ae5ff1c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T20:07:42.106 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:07:41 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:07:41.956+0000 7fd3ae5ff1c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T20:07:42.390 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:07:42 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:07:42.105+0000 7fd3ae5ff1c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T20:07:42.390 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:07:42 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:07:42.218+0000 7fd3ae5ff1c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T20:07:42.690 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout { 2023-09-23T20:07:42.690 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "epoch": 9, 2023-09-23T20:07:42.690 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "available": true, 2023-09-23T20:07:42.690 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "active_name": "y", 2023-09-23T20:07:42.690 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "num_standby": 0 2023-09-23T20:07:42.691 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout } 2023-09-23T20:07:42.691 INFO:teuthology.orchestra.run.smithi018.stdout:Waiting for the mgr to restart... 2023-09-23T20:07:42.691 INFO:teuthology.orchestra.run.smithi018.stdout:Waiting for mgr epoch 9... 2023-09-23T20:07:42.733 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:07:42 smithi018 ceph-mon[95889]: from='client.? 172.21.15.18:0/817336037' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2023-09-23T20:07:43.140 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:07:42 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:07:42.733+0000 7fd3ae5ff1c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T20:07:43.890 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:07:43 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:07:43.417+0000 7fd3ae5ff1c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T20:07:43.891 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:07:43 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:07:43.607+0000 7fd3ae5ff1c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T20:07:44.879 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:07:44 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:07:44.557+0000 7fd3ae5ff1c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T20:07:44.879 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:07:44 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:07:44.665+0000 7fd3ae5ff1c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-23T20:07:45.140 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:07:44 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:07:44.879+0000 7fd3ae5ff1c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T20:07:45.640 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:07:45 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:07:45.219+0000 7fd3ae5ff1c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-23T20:07:45.641 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:07:45 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:07:45.322+0000 7fd3ae5ff1c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T20:07:45.971 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:07:45 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:07:45.650+0000 7fd3ae5ff1c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T20:07:46.391 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:07:45 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:07:45.971+0000 7fd3ae5ff1c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T20:07:47.140 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:07:46 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:07:46.802+0000 7fd3ae5ff1c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T20:07:47.140 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:07:46 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:07:46.958+0000 7fd3ae5ff1c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T20:07:47.598 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:07:47 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:07:47.197+0000 7fd3ae5ff1c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T20:07:47.598 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:07:47 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:07:47.332+0000 7fd3ae5ff1c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T20:07:47.890 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:07:47 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:07:47.598+0000 7fd3ae5ff1c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T20:07:48.890 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:07:48 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:07:48.622+0000 7fd3ae5ff1c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T20:07:48.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:07:48 smithi018 ceph-mon[95889]: Active manager daemon y restarted 2023-09-23T20:07:48.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:07:48 smithi018 ceph-mon[95889]: Activating manager daemon y 2023-09-23T20:07:48.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:07:48 smithi018 ceph-mon[95889]: osdmap e3: 0 total, 0 up, 0 in 2023-09-23T20:07:48.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:07:48 smithi018 ceph-mon[95889]: mgrmap e10: y(active, starting, since 0.00313807s) 2023-09-23T20:07:48.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:07:48 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T20:07:48.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:07:48 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T20:07:48.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:07:48 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T20:07:48.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:07:48 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T20:07:48.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:07:48 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T20:07:48.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:07:48 smithi018 ceph-mon[95889]: Manager daemon y is now available 2023-09-23T20:07:48.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:07:48 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:07:48.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:07:48 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:07:50.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:07:49 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T20:07:50.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:07:49 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T20:07:50.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:07:49 smithi018 ceph-mon[95889]: mgrmap e11: y(active, since 1.00667s) 2023-09-23T20:07:50.257 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout { 2023-09-23T20:07:50.257 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 11, 2023-09-23T20:07:50.257 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout "initialized": true 2023-09-23T20:07:50.257 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout } 2023-09-23T20:07:50.257 INFO:teuthology.orchestra.run.smithi018.stdout:mgr epoch 9 is available 2023-09-23T20:07:50.257 INFO:teuthology.orchestra.run.smithi018.stdout:Generating a dashboard self-signed certificate... 2023-09-23T20:07:51.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:07:50 smithi018 ceph-mon[95889]: from='client.14154 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2023-09-23T20:07:51.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:07:50 smithi018 ceph-mon[95889]: from='client.14154 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2023-09-23T20:07:51.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:07:50 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:07:51.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:07:50 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:07:51.598 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout Self-signed certificate created 2023-09-23T20:07:51.598 INFO:teuthology.orchestra.run.smithi018.stdout:Creating initial admin user... 2023-09-23T20:07:51.846 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:07:51 smithi018 ceph-mon[95889]: [23/Sep/2023:20:07:50] ENGINE Bus STARTING 2023-09-23T20:07:51.846 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:07:51 smithi018 ceph-mon[95889]: [23/Sep/2023:20:07:50] ENGINE Serving on https://172.21.15.18:7150 2023-09-23T20:07:51.846 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:07:51 smithi018 ceph-mon[95889]: [23/Sep/2023:20:07:50] ENGINE Bus STARTED 2023-09-23T20:07:51.846 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:07:51 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:07:51.846 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:07:51 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:07:51.846 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:07:51 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:07:51.846 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:07:51 smithi018 ceph-mon[95889]: mgrmap e12: y(active, since 2s) 2023-09-23T20:07:53.041 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout {"username": "admin", "password": "$2b$12$HaaKje.XCoP/.N.9JSPR.uMj8cQARnl/G8mxK8fVOm0IbMO6jLftq", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1695499672, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2023-09-23T20:07:53.041 INFO:teuthology.orchestra.run.smithi018.stdout:Fetching dashboard port number... 2023-09-23T20:07:53.073 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:07:52 smithi018 ceph-mon[95889]: from='client.14162 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:07:53.890 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:07:53 smithi018 ceph-mon[95889]: from='client.14164 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:07:53.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:07:53 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:07:54.286 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stdout 8443 2023-09-23T20:07:54.297 INFO:teuthology.orchestra.run.smithi018.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2023-09-23T20:07:54.297 INFO:teuthology.orchestra.run.smithi018.stdout:systemctl: stdout disabled 2023-09-23T20:07:54.304 INFO:teuthology.orchestra.run.smithi018.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2023-09-23T20:07:54.304 INFO:teuthology.orchestra.run.smithi018.stdout:systemctl: stdout inactive 2023-09-23T20:07:54.305 INFO:teuthology.orchestra.run.smithi018.stdout:firewalld.service is not enabled 2023-09-23T20:07:54.305 INFO:teuthology.orchestra.run.smithi018.stdout:Not possible to open ports <[8443]>. firewalld.service is not available 2023-09-23T20:07:54.306 INFO:teuthology.orchestra.run.smithi018.stdout:Ceph Dashboard is now available at: 2023-09-23T20:07:54.307 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:07:54.307 INFO:teuthology.orchestra.run.smithi018.stdout: URL: https://smithi018:8443/ 2023-09-23T20:07:54.307 INFO:teuthology.orchestra.run.smithi018.stdout: User: admin 2023-09-23T20:07:54.307 INFO:teuthology.orchestra.run.smithi018.stdout: Password: vtpgojkc4x 2023-09-23T20:07:54.307 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:07:54.307 INFO:teuthology.orchestra.run.smithi018.stdout:Saving cluster configuration to /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/config directory 2023-09-23T20:07:54.307 INFO:teuthology.orchestra.run.smithi018.stdout:Enabling autotune for osd_memory_target 2023-09-23T20:07:55.140 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:07:54 smithi018 ceph-mon[95889]: from='client.? 172.21.15.18:0/4150526900' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2023-09-23T20:07:56.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:07:55 smithi018 ceph-mon[95889]: mgrmap e13: y(active, since 6s) 2023-09-23T20:07:56.990 INFO:teuthology.orchestra.run.smithi018.stdout:/usr/bin/ceph: stderr set mgr/dashboard/cluster/status 2023-09-23T20:07:56.990 INFO:teuthology.orchestra.run.smithi018.stdout:You can access the Ceph CLI as following in case of multi-cluster or non-default config: 2023-09-23T20:07:56.990 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:07:56.990 INFO:teuthology.orchestra.run.smithi018.stdout: sudo /home/ubuntu/cephtest/cephadm shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2023-09-23T20:07:56.991 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:07:56.991 INFO:teuthology.orchestra.run.smithi018.stdout:Or, if you are only running a single cluster on this host: 2023-09-23T20:07:56.991 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:07:56.991 INFO:teuthology.orchestra.run.smithi018.stdout: sudo /home/ubuntu/cephtest/cephadm shell 2023-09-23T20:07:56.991 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:07:56.991 INFO:teuthology.orchestra.run.smithi018.stdout:Please consider enabling telemetry to help improve Ceph: 2023-09-23T20:07:56.991 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:07:56.991 INFO:teuthology.orchestra.run.smithi018.stdout: ceph telemetry on 2023-09-23T20:07:56.991 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:07:56.991 INFO:teuthology.orchestra.run.smithi018.stdout:For more information see: 2023-09-23T20:07:56.991 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:07:56.991 INFO:teuthology.orchestra.run.smithi018.stdout: https://docs.ceph.com/docs/master/mgr/telemetry/ 2023-09-23T20:07:56.991 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:07:56.992 INFO:teuthology.orchestra.run.smithi018.stdout:Bootstrap complete. 2023-09-23T20:07:57.027 INFO:tasks.cephadm:Fetching config... 2023-09-23T20:07:57.027 DEBUG:teuthology.orchestra.run.smithi018:> set -ex 2023-09-23T20:07:57.027 DEBUG:teuthology.orchestra.run.smithi018:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2023-09-23T20:07:57.045 INFO:tasks.cephadm:Fetching client.admin keyring... 2023-09-23T20:07:57.046 DEBUG:teuthology.orchestra.run.smithi018:> set -ex 2023-09-23T20:07:57.046 DEBUG:teuthology.orchestra.run.smithi018:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2023-09-23T20:07:57.103 INFO:tasks.cephadm:Fetching mon keyring... 2023-09-23T20:07:57.103 DEBUG:teuthology.orchestra.run.smithi018:> set -ex 2023-09-23T20:07:57.104 DEBUG:teuthology.orchestra.run.smithi018:> sudo dd if=/var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/keyring of=/dev/stdout 2023-09-23T20:07:57.172 INFO:tasks.cephadm:Fetching pub ssh key... 2023-09-23T20:07:57.172 DEBUG:teuthology.orchestra.run.smithi018:> set -ex 2023-09-23T20:07:57.172 DEBUG:teuthology.orchestra.run.smithi018:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2023-09-23T20:07:57.229 INFO:tasks.cephadm:Installing pub ssh key for root users... 2023-09-23T20:07:57.229 DEBUG:teuthology.orchestra.run.smithi018:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC+wE4KNWwjHrBS2IVTtvmoMWh3jFGBcXmP+MQVqHr/XgSkEwskVANTX+YHciPQGIu1MUem0lmxVeXc6+TNyzSBe22gO+TYPRWzArY0c1+wvNzMofhHMe2ewRzXzIp1x+HPaRGV8+yqvrqid3F64gSYcgd51s0oTUeDjbnVRXoL1Sa3P+1N379vyUu2AfoYewmX7+svnzvwQLXnjAY/yC+94GqNobpa9XZGV7ji913SBsP8X1d1pI+Wgh9HtiV4i7Mh8biutLs8eS8HaUONNFYxYSIcI4qlo1loN4/hrRYfRu44uQve71q2EY/O7nEhyrTMuwHlIKHZ1r6AVGiLZbawxsdQwHyHxwUnPiS3F06Fg+tXQb0WHk/xvSLT396A1b/EsbxuYotTW1I5GpxbGtwpgCnQkfP289W3BI1GExs2nxcc+iw3z17LT1FLR0BFewdFayJfGTmoKfAisIc+J4+xUSGvH5VANGvVPUe/rdlNxu+i6sJN7Ka5tR4tI/aBYGM= ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2023-09-23T20:07:57.305 INFO:teuthology.orchestra.run.smithi018.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC+wE4KNWwjHrBS2IVTtvmoMWh3jFGBcXmP+MQVqHr/XgSkEwskVANTX+YHciPQGIu1MUem0lmxVeXc6+TNyzSBe22gO+TYPRWzArY0c1+wvNzMofhHMe2ewRzXzIp1x+HPaRGV8+yqvrqid3F64gSYcgd51s0oTUeDjbnVRXoL1Sa3P+1N379vyUu2AfoYewmX7+svnzvwQLXnjAY/yC+94GqNobpa9XZGV7ji913SBsP8X1d1pI+Wgh9HtiV4i7Mh8biutLs8eS8HaUONNFYxYSIcI4qlo1loN4/hrRYfRu44uQve71q2EY/O7nEhyrTMuwHlIKHZ1r6AVGiLZbawxsdQwHyHxwUnPiS3F06Fg+tXQb0WHk/xvSLT396A1b/EsbxuYotTW1I5GpxbGtwpgCnQkfP289W3BI1GExs2nxcc+iw3z17LT1FLR0BFewdFayJfGTmoKfAisIc+J4+xUSGvH5VANGvVPUe/rdlNxu+i6sJN7Ka5tR4tI/aBYGM= ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 2023-09-23T20:07:57.317 DEBUG:teuthology.orchestra.run.smithi149:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC+wE4KNWwjHrBS2IVTtvmoMWh3jFGBcXmP+MQVqHr/XgSkEwskVANTX+YHciPQGIu1MUem0lmxVeXc6+TNyzSBe22gO+TYPRWzArY0c1+wvNzMofhHMe2ewRzXzIp1x+HPaRGV8+yqvrqid3F64gSYcgd51s0oTUeDjbnVRXoL1Sa3P+1N379vyUu2AfoYewmX7+svnzvwQLXnjAY/yC+94GqNobpa9XZGV7ji913SBsP8X1d1pI+Wgh9HtiV4i7Mh8biutLs8eS8HaUONNFYxYSIcI4qlo1loN4/hrRYfRu44uQve71q2EY/O7nEhyrTMuwHlIKHZ1r6AVGiLZbawxsdQwHyHxwUnPiS3F06Fg+tXQb0WHk/xvSLT396A1b/EsbxuYotTW1I5GpxbGtwpgCnQkfP289W3BI1GExs2nxcc+iw3z17LT1FLR0BFewdFayJfGTmoKfAisIc+J4+xUSGvH5VANGvVPUe/rdlNxu+i6sJN7Ka5tR4tI/aBYGM= ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2023-09-23T20:07:57.362 INFO:teuthology.orchestra.run.smithi149.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC+wE4KNWwjHrBS2IVTtvmoMWh3jFGBcXmP+MQVqHr/XgSkEwskVANTX+YHciPQGIu1MUem0lmxVeXc6+TNyzSBe22gO+TYPRWzArY0c1+wvNzMofhHMe2ewRzXzIp1x+HPaRGV8+yqvrqid3F64gSYcgd51s0oTUeDjbnVRXoL1Sa3P+1N379vyUu2AfoYewmX7+svnzvwQLXnjAY/yC+94GqNobpa9XZGV7ji913SBsP8X1d1pI+Wgh9HtiV4i7Mh8biutLs8eS8HaUONNFYxYSIcI4qlo1loN4/hrRYfRu44uQve71q2EY/O7nEhyrTMuwHlIKHZ1r6AVGiLZbawxsdQwHyHxwUnPiS3F06Fg+tXQb0WHk/xvSLT396A1b/EsbxuYotTW1I5GpxbGtwpgCnQkfP289W3BI1GExs2nxcc+iw3z17LT1FLR0BFewdFayJfGTmoKfAisIc+J4+xUSGvH5VANGvVPUe/rdlNxu+i6sJN7Ka5tR4tI/aBYGM= ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 2023-09-23T20:07:57.375 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2023-09-23T20:07:57.890 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:07:57 smithi018 ceph-mon[95889]: from='client.? 172.21.15.18:0/2235186447' entity='client.admin' 2023-09-23T20:07:58.083 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:08:00.335 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2023-09-23T20:08:00.335 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph orch client-keyring set client.admin '*' --mode 0755 2023-09-23T20:08:00.942 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:08:01.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:00 smithi018 ceph-mon[95889]: from='client.? 172.21.15.18:0/3107241137' entity='client.admin' 2023-09-23T20:08:01.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:00 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:01.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:00 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:01.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:00 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi018", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:08:01.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:00 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:01.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:00 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:08:01.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:00 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:02.802 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi149 2023-09-23T20:08:02.802 DEBUG:teuthology.orchestra.run.smithi149:> set -ex 2023-09-23T20:08:02.802 DEBUG:teuthology.orchestra.run.smithi149:> dd of=/etc/ceph/ceph.conf 2023-09-23T20:08:02.822 DEBUG:teuthology.orchestra.run.smithi149:> set -ex 2023-09-23T20:08:02.822 DEBUG:teuthology.orchestra.run.smithi149:> dd of=/etc/ceph/ceph.client.admin.keyring 2023-09-23T20:08:02.884 INFO:tasks.cephadm:Adding host smithi149 to orchestrator... 2023-09-23T20:08:02.884 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph orch host add smithi149 2023-09-23T20:08:03.497 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:08:03.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:03 smithi018 ceph-mon[95889]: from='client.14174 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:08:03.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:03 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:03.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:03 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:08:03.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:03 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:08:03.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:03 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:08:03.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:03 smithi018 ceph-mon[95889]: Updating smithi018:/etc/ceph/ceph.conf 2023-09-23T20:08:03.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:03 smithi018 ceph-mon[95889]: Updating smithi018:/var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T20:08:03.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:03 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:03.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:03 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:03.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:03 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:04.519 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:04 smithi018 ceph-mon[95889]: Updating smithi018:/etc/ceph/ceph.client.admin.keyring 2023-09-23T20:08:04.520 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:04 smithi018 ceph-mon[95889]: Updating smithi018:/var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T20:08:06.140 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:05 smithi018 ceph-mon[95889]: from='client.14176 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi149", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:08:06.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:05 smithi018 ceph-mon[95889]: Deploying cephadm binary to smithi149 2023-09-23T20:08:06.864 INFO:teuthology.orchestra.run.smithi018.stdout:Added host 'smithi149' with addr '172.21.15.149' 2023-09-23T20:08:07.248 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph orch host ls --format=json 2023-09-23T20:08:07.849 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:08:08.113 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:07 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:08.113 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:07 smithi018 ceph-mon[95889]: Added host smithi149 2023-09-23T20:08:08.113 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:07 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:08:08.113 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:07 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:09.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:09 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:09.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:09 smithi018 ceph-mon[95889]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:08:09.248 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:08:09.248 INFO:teuthology.orchestra.run.smithi018.stdout:[{"addr": "172.21.15.18", "hostname": "smithi018", "labels": [], "status": ""}, {"addr": "172.21.15.149", "hostname": "smithi149", "labels": [], "status": ""}] 2023-09-23T20:08:09.610 INFO:tasks.cephadm:Setting crush tunables to default 2023-09-23T20:08:09.611 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd crush tunables default 2023-09-23T20:08:10.223 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:08:10.627 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:10 smithi018 ceph-mon[95889]: from='client.14178 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:08:10.627 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:10 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:11.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:11 smithi018 ceph-mon[95889]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:08:12.323 INFO:teuthology.orchestra.run.smithi018.stderr:adjusted tunables profile to default 2023-09-23T20:08:12.640 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:12 smithi018 ceph-mon[95889]: from='client.? 172.21.15.18:0/360296344' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2023-09-23T20:08:12.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:12 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:12.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:12 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:12.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:12 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:12.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:12 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:12.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:12 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi149", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:08:12.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:12 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:08:12.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:12 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:08:12.787 INFO:tasks.cephadm:Adding mon.a on smithi018 2023-09-23T20:08:12.787 INFO:tasks.cephadm:Adding mon.c on smithi018 2023-09-23T20:08:12.787 INFO:tasks.cephadm:Adding mon.b on smithi149 2023-09-23T20:08:12.787 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph orch apply mon '3;smithi018:172.21.15.18=a;smithi018:[v2:172.21.15.18:3301,v1:172.21.15.18:6790]=c;smithi149:172.21.15.149=b' 2023-09-23T20:08:13.417 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T20:08:13.640 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:13 smithi018 ceph-mon[95889]: Updating smithi149:/etc/ceph/ceph.conf 2023-09-23T20:08:13.640 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:13 smithi018 ceph-mon[95889]: from='client.? 172.21.15.18:0/360296344' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2023-09-23T20:08:13.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:13 smithi018 ceph-mon[95889]: osdmap e4: 0 total, 0 up, 0 in 2023-09-23T20:08:13.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:13 smithi018 ceph-mon[95889]: Updating smithi149:/var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T20:08:13.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:13 smithi018 ceph-mon[95889]: Updating smithi149:/etc/ceph/ceph.client.admin.keyring 2023-09-23T20:08:13.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:13 smithi018 ceph-mon[95889]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:08:13.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:13 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:13.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:13 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:13.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:13 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:14.640 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:14 smithi018 ceph-mon[95889]: Updating smithi149:/var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T20:08:14.756 INFO:teuthology.orchestra.run.smithi149.stdout:Scheduled mon update... 2023-09-23T20:08:15.077 DEBUG:teuthology.orchestra.run.smithi018:mon.c> sudo journalctl -f -n 0 -u ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7@mon.c.service 2023-09-23T20:08:15.080 DEBUG:teuthology.orchestra.run.smithi149:mon.b> sudo journalctl -f -n 0 -u ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7@mon.b.service 2023-09-23T20:08:15.082 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2023-09-23T20:08:15.083 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph mon dump -f json 2023-09-23T20:08:15.111 INFO:journalctl@ceph.mon.b.smithi149.stdout:-- Logs begin at Sat 2023-09-23 19:50:38 UTC. -- 2023-09-23T20:08:15.117 INFO:journalctl@ceph.mon.c.smithi018.stdout:-- Logs begin at Sat 2023-09-23 19:50:10 UTC. -- 2023-09-23T20:08:15.801 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T20:08:16.140 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:15 smithi018 ceph-mon[95889]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:08:16.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:15 smithi018 ceph-mon[95889]: from='client.14182 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi018:172.21.15.18=a;smithi018:[v2:172.21.15.18:3301,v1:172.21.15.18:6790]=c;smithi149:172.21.15.149=b", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:08:16.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:15 smithi018 ceph-mon[95889]: Saving service mon spec with placement smithi018:172.21.15.18=a;smithi018:[v2:172.21.15.18:3301,v1:172.21.15.18:6790]=c;smithi149:172.21.15.149=b;count:3 2023-09-23T20:08:16.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:15 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:16.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:15 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:08:16.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:15 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:08:16.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:15 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:08:16.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:15 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:16.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:15 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T20:08:16.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:15 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:08:16.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:15 smithi018 ceph-mon[95889]: Deploying daemon mon.b on smithi149 2023-09-23T20:08:17.792 INFO:teuthology.orchestra.run.smithi149.stdout: 2023-09-23T20:08:17.792 INFO:teuthology.orchestra.run.smithi149.stdout:{"epoch":1,"fsid":"a3b3b17e-5a4c-11ee-8db4-212e2dc638e7","modified":"2023-09-23T20:06:52.152880Z","created":"2023-09-23T20:06:52.152880Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:3300","nonce":0},{"type":"v1","addr":"172.21.15.18:6789","nonce":0}]},"addr":"172.21.15.18:6789/0","public_addr":"172.21.15.18:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-09-23T20:08:17.793 INFO:teuthology.orchestra.run.smithi149.stderr:dumped monmap epoch 1 2023-09-23T20:08:18.140 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:17 smithi018 ceph-mon[95889]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:08:18.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:17 smithi149 podman[100699]: 2023-09-23T20:08:18.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: set uid:gid to 167:167 (ceph:ceph) 2023-09-23T20:08:18.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable), process ceph-mon, pid 7 2023-09-23T20:08:18.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: pidfile_write: ignore empty --pid-file 2023-09-23T20:08:18.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: load: jerasure load: lrc 2023-09-23T20:08:18.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: RocksDB version: 6.15.5 2023-09-23T20:08:18.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2023-09-23T20:08:18.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Compile date Sep 22 2023 2023-09-23T20:08:18.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: DB SUMMARY 2023-09-23T20:08:18.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: DB Session ID: YIPZV6E99ZBW47V3SSB3 2023-09-23T20:08:18.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: CURRENT file: CURRENT 2023-09-23T20:08:18.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: IDENTITY file: IDENTITY 2023-09-23T20:08:18.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: MANIFEST file: MANIFEST-000003 size: 57 Bytes 2023-09-23T20:08:18.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 0, files: 2023-09-23T20:08:18.718 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000004.log size: 511 ; 2023-09-23T20:08:18.718 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.error_if_exists: 0 2023-09-23T20:08:18.718 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.create_if_missing: 0 2023-09-23T20:08:18.718 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.paranoid_checks: 1 2023-09-23T20:08:18.718 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-09-23T20:08:18.718 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.env: 0x5621ccf95a80 2023-09-23T20:08:18.718 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.fs: Posix File System 2023-09-23T20:08:18.718 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.info_log: 0x5621cf15fd80 2023-09-23T20:08:18.718 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.max_file_opening_threads: 16 2023-09-23T20:08:18.718 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.statistics: (nil) 2023-09-23T20:08:18.718 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.use_fsync: 0 2023-09-23T20:08:18.718 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.max_log_file_size: 0 2023-09-23T20:08:18.719 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.max_manifest_file_size: 1073741824 2023-09-23T20:08:18.719 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.log_file_time_to_roll: 0 2023-09-23T20:08:18.719 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.keep_log_file_num: 1000 2023-09-23T20:08:18.719 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.recycle_log_file_num: 0 2023-09-23T20:08:18.719 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.allow_fallocate: 1 2023-09-23T20:08:18.719 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.allow_mmap_reads: 0 2023-09-23T20:08:18.719 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.allow_mmap_writes: 0 2023-09-23T20:08:18.719 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.use_direct_reads: 0 2023-09-23T20:08:18.719 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-09-23T20:08:18.719 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.create_missing_column_families: 0 2023-09-23T20:08:18.719 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.db_log_dir: 2023-09-23T20:08:18.719 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-b/store.db 2023-09-23T20:08:18.719 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.table_cache_numshardbits: 6 2023-09-23T20:08:18.720 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 bash[100699]: e9fa4ded74294044a89a2c2eb37bae66ed193a35c97149eb31ce687199407fe9 2023-09-23T20:08:18.720 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.WAL_ttl_seconds: 0 2023-09-23T20:08:18.720 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.WAL_size_limit_MB: 0 2023-09-23T20:08:18.720 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-09-23T20:08:18.720 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.manifest_preallocation_size: 4194304 2023-09-23T20:08:18.720 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.is_fd_close_on_exec: 1 2023-09-23T20:08:18.720 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.advise_random_on_open: 1 2023-09-23T20:08:18.720 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.db_write_buffer_size: 0 2023-09-23T20:08:18.720 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.write_buffer_manager: 0x5621cfe52960 2023-09-23T20:08:18.720 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.access_hint_on_compaction_start: 1 2023-09-23T20:08:18.720 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2023-09-23T20:08:18.720 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.random_access_max_buffer_size: 1048576 2023-09-23T20:08:18.721 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.use_adaptive_mutex: 0 2023-09-23T20:08:18.721 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.rate_limiter: (nil) 2023-09-23T20:08:18.721 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-09-23T20:08:18.721 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.wal_recovery_mode: 2 2023-09-23T20:08:18.721 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.enable_thread_tracking: 0 2023-09-23T20:08:18.721 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.enable_pipelined_write: 0 2023-09-23T20:08:18.721 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.unordered_write: 0 2023-09-23T20:08:18.721 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.allow_concurrent_memtable_write: 1 2023-09-23T20:08:18.721 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-09-23T20:08:18.721 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.write_thread_max_yield_usec: 100 2023-09-23T20:08:18.721 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.write_thread_slow_yield_usec: 3 2023-09-23T20:08:18.721 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.row_cache: None 2023-09-23T20:08:18.721 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.wal_filter: None 2023-09-23T20:08:18.722 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.avoid_flush_during_recovery: 0 2023-09-23T20:08:18.722 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.allow_ingest_behind: 0 2023-09-23T20:08:18.722 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.preserve_deletes: 0 2023-09-23T20:08:18.722 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.two_write_queues: 0 2023-09-23T20:08:18.722 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.manual_wal_flush: 0 2023-09-23T20:08:18.722 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.atomic_flush: 0 2023-09-23T20:08:18.722 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-09-23T20:08:18.722 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.persist_stats_to_disk: 0 2023-09-23T20:08:18.722 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.write_dbid_to_manifest: 0 2023-09-23T20:08:18.722 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.log_readahead_size: 0 2023-09-23T20:08:18.722 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.file_checksum_gen_factory: Unknown 2023-09-23T20:08:18.722 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.best_efforts_recovery: 0 2023-09-23T20:08:18.722 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-09-23T20:08:18.723 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-09-23T20:08:18.723 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.allow_data_in_errors: 0 2023-09-23T20:08:18.723 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.db_host_id: __hostname__ 2023-09-23T20:08:18.723 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.max_background_jobs: 2 2023-09-23T20:08:18.723 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.max_background_compactions: -1 2023-09-23T20:08:18.723 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.max_subcompactions: 1 2023-09-23T20:08:18.723 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.avoid_flush_during_shutdown: 0 2023-09-23T20:08:18.723 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-09-23T20:08:18.723 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.delayed_write_rate : 16777216 2023-09-23T20:08:18.723 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.max_total_wal_size: 0 2023-09-23T20:08:18.723 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-09-23T20:08:18.723 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.stats_dump_period_sec: 600 2023-09-23T20:08:18.723 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.stats_persist_period_sec: 600 2023-09-23T20:08:18.724 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.stats_history_buffer_size: 1048576 2023-09-23T20:08:18.724 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.max_open_files: -1 2023-09-23T20:08:18.724 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.bytes_per_sync: 0 2023-09-23T20:08:18.724 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.wal_bytes_per_sync: 0 2023-09-23T20:08:18.724 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.strict_bytes_per_sync: 0 2023-09-23T20:08:18.724 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.compaction_readahead_size: 0 2023-09-23T20:08:18.724 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.max_background_flushes: -1 2023-09-23T20:08:18.724 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Compression algorithms supported: 2023-09-23T20:08:18.724 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: kZSTDNotFinalCompression supported: 0 2023-09-23T20:08:18.724 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: kZSTD supported: 0 2023-09-23T20:08:18.724 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: kXpressCompression supported: 0 2023-09-23T20:08:18.724 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: kLZ4HCCompression supported: 1 2023-09-23T20:08:18.724 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: kZlibCompression supported: 1 2023-09-23T20:08:18.725 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: kSnappyCompression supported: 1 2023-09-23T20:08:18.725 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: kLZ4Compression supported: 1 2023-09-23T20:08:18.725 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: kBZip2Compression supported: 0 2023-09-23T20:08:18.725 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Fast CRC32 supported: Supported on x86 2023-09-23T20:08:18.725 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000003 2023-09-23T20:08:18.725 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2023-09-23T20:08:18.725 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-09-23T20:08:18.725 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.merge_operator: 2023-09-23T20:08:18.725 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.compaction_filter: None 2023-09-23T20:08:18.725 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.compaction_filter_factory: None 2023-09-23T20:08:18.725 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.sst_partitioner_factory: None 2023-09-23T20:08:18.725 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.memtable_factory: SkipListFactory 2023-09-23T20:08:18.725 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.table_factory: BlockBasedTable 2023-09-23T20:08:18.726 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x5621cf132fa0) 2023-09-23T20:08:18.726 INFO:journalctl@ceph.mon.b.smithi149.stdout: cache_index_and_filter_blocks: 1 2023-09-23T20:08:18.726 INFO:journalctl@ceph.mon.b.smithi149.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2023-09-23T20:08:18.726 INFO:journalctl@ceph.mon.b.smithi149.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2023-09-23T20:08:18.726 INFO:journalctl@ceph.mon.b.smithi149.stdout: pin_top_level_index_and_filter: 1 2023-09-23T20:08:18.726 INFO:journalctl@ceph.mon.b.smithi149.stdout: index_type: 0 2023-09-23T20:08:18.726 INFO:journalctl@ceph.mon.b.smithi149.stdout: data_block_index_type: 0 2023-09-23T20:08:18.726 INFO:journalctl@ceph.mon.b.smithi149.stdout: index_shortening: 1 2023-09-23T20:08:18.726 INFO:journalctl@ceph.mon.b.smithi149.stdout: data_block_hash_table_util_ratio: 0.750000 2023-09-23T20:08:18.726 INFO:journalctl@ceph.mon.b.smithi149.stdout: hash_index_allow_collision: 1 2023-09-23T20:08:18.726 INFO:journalctl@ceph.mon.b.smithi149.stdout: checksum: 1 2023-09-23T20:08:18.726 INFO:journalctl@ceph.mon.b.smithi149.stdout: no_block_cache: 0 2023-09-23T20:08:18.727 INFO:journalctl@ceph.mon.b.smithi149.stdout: block_cache: 0x5621cf165090 2023-09-23T20:08:18.727 INFO:journalctl@ceph.mon.b.smithi149.stdout: block_cache_name: BinnedLRUCache 2023-09-23T20:08:18.727 INFO:journalctl@ceph.mon.b.smithi149.stdout: block_cache_options: 2023-09-23T20:08:18.727 INFO:journalctl@ceph.mon.b.smithi149.stdout: capacity : 536870912 2023-09-23T20:08:18.727 INFO:journalctl@ceph.mon.b.smithi149.stdout: num_shard_bits : 4 2023-09-23T20:08:18.727 INFO:journalctl@ceph.mon.b.smithi149.stdout: strict_capacity_limit : 0 2023-09-23T20:08:18.727 INFO:journalctl@ceph.mon.b.smithi149.stdout: high_pri_pool_ratio: 0.000 2023-09-23T20:08:18.727 INFO:journalctl@ceph.mon.b.smithi149.stdout: block_cache_compressed: (nil) 2023-09-23T20:08:18.727 INFO:journalctl@ceph.mon.b.smithi149.stdout: persistent_cache: (nil) 2023-09-23T20:08:18.727 INFO:journalctl@ceph.mon.b.smithi149.stdout: block_size: 4096 2023-09-23T20:08:18.727 INFO:journalctl@ceph.mon.b.smithi149.stdout: block_size_deviation: 10 2023-09-23T20:08:18.727 INFO:journalctl@ceph.mon.b.smithi149.stdout: block_restart_interval: 16 2023-09-23T20:08:18.727 INFO:journalctl@ceph.mon.b.smithi149.stdout: index_block_restart_interval: 1 2023-09-23T20:08:18.728 INFO:journalctl@ceph.mon.b.smithi149.stdout: metadata_block_size: 4096 2023-09-23T20:08:18.728 INFO:journalctl@ceph.mon.b.smithi149.stdout: partition_filters: 0 2023-09-23T20:08:18.728 INFO:journalctl@ceph.mon.b.smithi149.stdout: use_delta_encoding: 1 2023-09-23T20:08:18.728 INFO:journalctl@ceph.mon.b.smithi149.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2023-09-23T20:08:18.728 INFO:journalctl@ceph.mon.b.smithi149.stdout: whole_key_filtering: 1 2023-09-23T20:08:18.728 INFO:journalctl@ceph.mon.b.smithi149.stdout: verify_compression: 0 2023-09-23T20:08:18.728 INFO:journalctl@ceph.mon.b.smithi149.stdout: read_amp_bytes_per_bit: 0 2023-09-23T20:08:18.728 INFO:journalctl@ceph.mon.b.smithi149.stdout: format_version: 4 2023-09-23T20:08:18.728 INFO:journalctl@ceph.mon.b.smithi149.stdout: enable_index_compression: 1 2023-09-23T20:08:18.728 INFO:journalctl@ceph.mon.b.smithi149.stdout: block_align: 0 2023-09-23T20:08:18.728 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.write_buffer_size: 33554432 2023-09-23T20:08:18.728 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.max_write_buffer_number: 2 2023-09-23T20:08:18.728 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.compression: NoCompression 2023-09-23T20:08:18.728 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.bottommost_compression: Disabled 2023-09-23T20:08:18.729 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.prefix_extractor: nullptr 2023-09-23T20:08:18.729 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-09-23T20:08:18.729 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.num_levels: 7 2023-09-23T20:08:18.729 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-09-23T20:08:18.729 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-09-23T20:08:18.729 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-09-23T20:08:18.729 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-09-23T20:08:18.729 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.bottommost_compression_opts.level: 32767 2023-09-23T20:08:18.729 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-09-23T20:08:18.729 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-09-23T20:08:18.729 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-09-23T20:08:18.729 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-09-23T20:08:18.729 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.bottommost_compression_opts.enabled: false 2023-09-23T20:08:18.730 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.compression_opts.window_bits: -14 2023-09-23T20:08:18.730 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.compression_opts.level: 32767 2023-09-23T20:08:18.731 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.compression_opts.strategy: 0 2023-09-23T20:08:18.731 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-09-23T20:08:18.731 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-09-23T20:08:18.731 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.compression_opts.parallel_threads: 1 2023-09-23T20:08:18.731 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.compression_opts.enabled: false 2023-09-23T20:08:18.731 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-09-23T20:08:18.731 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-09-23T20:08:18.731 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.level0_stop_writes_trigger: 36 2023-09-23T20:08:18.731 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.target_file_size_base: 67108864 2023-09-23T20:08:18.731 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.target_file_size_multiplier: 1 2023-09-23T20:08:18.731 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.max_bytes_for_level_base: 268435456 2023-09-23T20:08:18.731 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-09-23T20:08:18.731 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-09-23T20:08:18.732 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-09-23T20:08:18.732 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-09-23T20:08:18.732 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-09-23T20:08:18.732 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-09-23T20:08:18.732 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-09-23T20:08:18.732 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-09-23T20:08:18.732 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-09-23T20:08:18.732 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-09-23T20:08:18.732 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.max_compaction_bytes: 1677721600 2023-09-23T20:08:18.732 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.arena_block_size: 4194304 2023-09-23T20:08:18.732 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-09-23T20:08:18.732 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-09-23T20:08:18.733 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2023-09-23T20:08:18.733 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.disable_auto_compactions: 0 2023-09-23T20:08:18.733 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-09-23T20:08:18.733 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-09-23T20:08:18.733 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-09-23T20:08:18.733 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-09-23T20:08:18.733 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-09-23T20:08:18.733 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-09-23T20:08:18.733 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-09-23T20:08:18.733 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-09-23T20:08:18.733 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-09-23T20:08:18.733 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-09-23T20:08:18.733 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.table_properties_collectors: 2023-09-23T20:08:18.733 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.inplace_update_support: 0 2023-09-23T20:08:18.734 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.inplace_update_num_locks: 10000 2023-09-23T20:08:18.734 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-09-23T20:08:18.734 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.memtable_whole_key_filtering: 0 2023-09-23T20:08:18.734 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.memtable_huge_page_size: 0 2023-09-23T20:08:18.734 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.bloom_locality: 0 2023-09-23T20:08:18.734 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.max_successive_merges: 0 2023-09-23T20:08:18.734 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.optimize_filters_for_hits: 0 2023-09-23T20:08:18.734 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.paranoid_file_checks: 0 2023-09-23T20:08:18.734 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.force_consistency_checks: 1 2023-09-23T20:08:18.734 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.report_bg_io_stats: 0 2023-09-23T20:08:18.734 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.ttl: 2592000 2023-09-23T20:08:18.734 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.periodic_compaction_seconds: 0 2023-09-23T20:08:18.735 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.enable_blob_files: false 2023-09-23T20:08:18.735 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.min_blob_size: 0 2023-09-23T20:08:18.735 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.blob_file_size: 268435456 2023-09-23T20:08:18.735 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.blob_compression_type: NoCompression 2023-09-23T20:08:18.735 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.enable_blob_garbage_collection: false 2023-09-23T20:08:18.735 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-09-23T20:08:18.735 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000003 succeeded,manifest_file_number is 3, next_file_number is 5, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-09-23T20:08:18.735 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 0 2023-09-23T20:08:18.735 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: [db/version_set.cc:4082] Creating manifest 7 2023-09-23T20:08:18.735 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695499698303404, "job": 1, "event": "recovery_started", "wal_files": [4]} 2023-09-23T20:08:18.735 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: [db/db_impl/db_impl_open.cc:845] Recovering log #4 mode 2 2023-09-23T20:08:18.735 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: [table/block_based/filter_policy.cc:991] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2023-09-23T20:08:18.735 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695499698303795, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1545, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 523, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 401, "raw_average_value_size": 80, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1695499698, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "b2baa7e8-8c2d-43a6-81ee-62d771b8beb7", "db_session_id": "YIPZV6E99ZBW47V3SSB3"}} 2023-09-23T20:08:18.736 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: [db/version_set.cc:4082] Creating manifest 9 2023-09-23T20:08:18.736 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695499698304235, "job": 1, "event": "recovery_finished"} 2023-09-23T20:08:18.736 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: [file/delete_scheduler.cc:69] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-09-23T20:08:18.736 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x5621cf0f9180 2023-09-23T20:08:18.736 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: DB pointer 0x5621cf1ce000 2023-09-23T20:08:18.736 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2023-09-23T20:08:18.736 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: rocksdb: [db/db_impl/db_impl.cc:903] 2023-09-23T20:08:18.736 INFO:journalctl@ceph.mon.b.smithi149.stdout: ** DB Stats ** 2023-09-23T20:08:18.736 INFO:journalctl@ceph.mon.b.smithi149.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T20:08:18.736 INFO:journalctl@ceph.mon.b.smithi149.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-09-23T20:08:18.736 INFO:journalctl@ceph.mon.b.smithi149.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-09-23T20:08:18.736 INFO:journalctl@ceph.mon.b.smithi149.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T20:08:18.737 INFO:journalctl@ceph.mon.b.smithi149.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-09-23T20:08:18.737 INFO:journalctl@ceph.mon.b.smithi149.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2023-09-23T20:08:18.737 INFO:journalctl@ceph.mon.b.smithi149.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T20:08:18.737 INFO:journalctl@ceph.mon.b.smithi149.stdout: 2023-09-23T20:08:18.737 INFO:journalctl@ceph.mon.b.smithi149.stdout: ** Compaction Stats [default] ** 2023-09-23T20:08:18.737 INFO:journalctl@ceph.mon.b.smithi149.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-23T20:08:18.737 INFO:journalctl@ceph.mon.b.smithi149.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T20:08:18.737 INFO:journalctl@ceph.mon.b.smithi149.stdout: L0 1/0 1.51 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.0 0.00 0.00 1 0.000 0 0 2023-09-23T20:08:18.737 INFO:journalctl@ceph.mon.b.smithi149.stdout: Sum 1/0 1.51 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.0 0.00 0.00 1 0.000 0 0 2023-09-23T20:08:18.737 INFO:journalctl@ceph.mon.b.smithi149.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.0 0.00 0.00 1 0.000 0 0 2023-09-23T20:08:18.737 INFO:journalctl@ceph.mon.b.smithi149.stdout: 2023-09-23T20:08:18.737 INFO:journalctl@ceph.mon.b.smithi149.stdout: ** Compaction Stats [default] ** 2023-09-23T20:08:18.737 INFO:journalctl@ceph.mon.b.smithi149.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-23T20:08:18.738 INFO:journalctl@ceph.mon.b.smithi149.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T20:08:18.738 INFO:journalctl@ceph.mon.b.smithi149.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 4.0 0.00 0.00 1 0.000 0 0 2023-09-23T20:08:18.738 INFO:journalctl@ceph.mon.b.smithi149.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T20:08:18.738 INFO:journalctl@ceph.mon.b.smithi149.stdout: Flush(GB): cumulative 0.000, interval 0.000 2023-09-23T20:08:18.738 INFO:journalctl@ceph.mon.b.smithi149.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-09-23T20:08:18.738 INFO:journalctl@ceph.mon.b.smithi149.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-09-23T20:08:18.738 INFO:journalctl@ceph.mon.b.smithi149.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-09-23T20:08:18.738 INFO:journalctl@ceph.mon.b.smithi149.stdout: AddFile(Keys): cumulative 0, interval 0 2023-09-23T20:08:18.738 INFO:journalctl@ceph.mon.b.smithi149.stdout: Cumulative compaction: 0.00 GB write, 0.43 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T20:08:18.738 INFO:journalctl@ceph.mon.b.smithi149.stdout: Interval compaction: 0.00 GB write, 0.43 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T20:08:18.738 INFO:journalctl@ceph.mon.b.smithi149.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-09-23T20:08:18.738 INFO:journalctl@ceph.mon.b.smithi149.stdout: 2023-09-23T20:08:18.738 INFO:journalctl@ceph.mon.b.smithi149.stdout: ** File Read Latency Histogram By Level [default] ** 2023-09-23T20:08:18.739 INFO:journalctl@ceph.mon.b.smithi149.stdout: 2023-09-23T20:08:18.739 INFO:journalctl@ceph.mon.b.smithi149.stdout: ** Compaction Stats [default] ** 2023-09-23T20:08:18.739 INFO:journalctl@ceph.mon.b.smithi149.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-23T20:08:18.739 INFO:journalctl@ceph.mon.b.smithi149.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T20:08:18.739 INFO:journalctl@ceph.mon.b.smithi149.stdout: L0 1/0 1.51 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.0 0.00 0.00 1 0.000 0 0 2023-09-23T20:08:18.739 INFO:journalctl@ceph.mon.b.smithi149.stdout: Sum 1/0 1.51 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.0 0.00 0.00 1 0.000 0 0 2023-09-23T20:08:18.739 INFO:journalctl@ceph.mon.b.smithi149.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-09-23T20:08:18.739 INFO:journalctl@ceph.mon.b.smithi149.stdout: 2023-09-23T20:08:18.739 INFO:journalctl@ceph.mon.b.smithi149.stdout: ** Compaction Stats [default] ** 2023-09-23T20:08:18.739 INFO:journalctl@ceph.mon.b.smithi149.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-23T20:08:18.739 INFO:journalctl@ceph.mon.b.smithi149.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T20:08:18.739 INFO:journalctl@ceph.mon.b.smithi149.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 4.0 0.00 0.00 1 0.000 0 0 2023-09-23T20:08:18.739 INFO:journalctl@ceph.mon.b.smithi149.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T20:08:18.740 INFO:journalctl@ceph.mon.b.smithi149.stdout: Flush(GB): cumulative 0.000, interval 0.000 2023-09-23T20:08:18.740 INFO:journalctl@ceph.mon.b.smithi149.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-09-23T20:08:18.740 INFO:journalctl@ceph.mon.b.smithi149.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-09-23T20:08:18.740 INFO:journalctl@ceph.mon.b.smithi149.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-09-23T20:08:18.740 INFO:journalctl@ceph.mon.b.smithi149.stdout: AddFile(Keys): cumulative 0, interval 0 2023-09-23T20:08:18.740 INFO:journalctl@ceph.mon.b.smithi149.stdout: Cumulative compaction: 0.00 GB write, 0.43 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T20:08:18.740 INFO:journalctl@ceph.mon.b.smithi149.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T20:08:18.740 INFO:journalctl@ceph.mon.b.smithi149.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-09-23T20:08:18.740 INFO:journalctl@ceph.mon.b.smithi149.stdout: 2023-09-23T20:08:18.740 INFO:journalctl@ceph.mon.b.smithi149.stdout: ** File Read Latency Histogram By Level [default] ** 2023-09-23T20:08:18.740 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: mon.b does not exist in monmap, will attempt to join an existing cluster 2023-09-23T20:08:18.740 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: using public_addr v2:172.21.15.149:0/0 -> [v2:172.21.15.149:3300/0,v1:172.21.15.149:6789/0] 2023-09-23T20:08:18.740 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: starting mon.b rank -1 at public addrs [v2:172.21.15.149:3300/0,v1:172.21.15.149:6789/0] at bind addrs [v2:172.21.15.149:3300/0,v1:172.21.15.149:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 2023-09-23T20:08:18.740 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: mon.b@-1(???) e0 preinit fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 2023-09-23T20:08:18.741 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: mon.b@-1(synchronizing).mds e1 new map 2023-09-23T20:08:18.741 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: mon.b@-1(synchronizing).mds e1 print_map 2023-09-23T20:08:18.741 INFO:journalctl@ceph.mon.b.smithi149.stdout: e1 2023-09-23T20:08:18.741 INFO:journalctl@ceph.mon.b.smithi149.stdout: enable_multiple, ever_enabled_multiple: 1,1 2023-09-23T20:08:18.741 INFO:journalctl@ceph.mon.b.smithi149.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-09-23T20:08:18.741 INFO:journalctl@ceph.mon.b.smithi149.stdout: legacy client fscid: -1 2023-09-23T20:08:18.741 INFO:journalctl@ceph.mon.b.smithi149.stdout: 2023-09-23T20:08:18.741 INFO:journalctl@ceph.mon.b.smithi149.stdout: No filesystems configured 2023-09-23T20:08:18.741 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: mon.b@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2023-09-23T20:08:18.741 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: mon.b@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2023-09-23T20:08:18.741 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: mon.b@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2023-09-23T20:08:18.741 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: mon.b@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2023-09-23T20:08:18.741 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: mon.b@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2023-09-23T20:08:18.742 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: mon.b@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2023-09-23T20:08:18.742 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: mon.b@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2023-09-23T20:08:18.742 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-09-23T20:08:18.742 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-09-23T20:08:18.742 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-09-23T20:08:18.742 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: Updating smithi149:/etc/ceph/ceph.conf 2023-09-23T20:08:18.742 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: from='client.? 172.21.15.18:0/360296344' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2023-09-23T20:08:18.742 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: osdmap e4: 0 total, 0 up, 0 in 2023-09-23T20:08:18.742 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: Updating smithi149:/var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T20:08:18.742 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: Updating smithi149:/etc/ceph/ceph.client.admin.keyring 2023-09-23T20:08:18.742 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:08:18.742 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:18.742 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:18.743 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:18.743 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: Updating smithi149:/var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T20:08:18.743 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:08:18.743 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: from='client.14182 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi018:172.21.15.18=a;smithi018:[v2:172.21.15.18:3301,v1:172.21.15.18:6790]=c;smithi149:172.21.15.149=b", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:08:18.743 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: Saving service mon spec with placement smithi018:172.21.15.18=a;smithi018:[v2:172.21.15.18:3301,v1:172.21.15.18:6790]=c;smithi149:172.21.15.149=b;count:3 2023-09-23T20:08:18.743 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:18.743 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:08:18.743 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:08:18.743 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:08:18.743 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:18.743 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T20:08:18.743 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:08:18.743 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: Deploying daemon mon.b on smithi149 2023-09-23T20:08:18.744 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:08:18.744 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: mon.b@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2023-09-23T20:08:18.744 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 ceph-mon[100764]: mon.b@-1(synchronizing) e1 handle_conf_change mon_allow_pool_delete 2023-09-23T20:08:18.744 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:18 smithi149 systemd[1]: Started Ceph mon.b for a3b3b17e-5a4c-11ee-8db4-212e2dc638e7. 2023-09-23T20:08:19.208 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2023-09-23T20:08:19.209 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph mon dump -f json 2023-09-23T20:08:19.809 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:08:23.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:23 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T20:08:23.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:23 smithi149 ceph-mon[100764]: mon.a calling monitor election 2023-09-23T20:08:23.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:23 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:08:23.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:23 smithi149 ceph-mon[100764]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:08:23.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:23 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:08:23.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:23 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:08:23.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:23 smithi149 ceph-mon[100764]: mon.b calling monitor election 2023-09-23T20:08:23.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:23 smithi149 ceph-mon[100764]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:08:23.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:23 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:08:23.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:23 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:08:23.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:23 smithi149 ceph-mon[100764]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:08:23.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:23 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:08:23.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:23 smithi149 ceph-mon[100764]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-09-23T20:08:23.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:23 smithi149 ceph-mon[100764]: monmap e2: 2 mons at {a=[v2:172.21.15.18:3300/0,v1:172.21.15.18:6789/0],b=[v2:172.21.15.149:3300/0,v1:172.21.15.149:6789/0]} removed_ranks: {} 2023-09-23T20:08:23.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:23 smithi149 ceph-mon[100764]: fsmap 2023-09-23T20:08:23.718 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:23 smithi149 ceph-mon[100764]: osdmap e4: 0 total, 0 up, 0 in 2023-09-23T20:08:23.718 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:23 smithi149 ceph-mon[100764]: mgrmap e13: y(active, since 34s) 2023-09-23T20:08:23.718 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:23 smithi149 ceph-mon[100764]: overall HEALTH_OK 2023-09-23T20:08:23.718 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:23 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:23.718 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:23 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:23.718 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:23 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:23.718 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:23 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T20:08:23.718 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:23 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:08:23.890 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:23 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T20:08:23.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:23 smithi018 ceph-mon[95889]: mon.a calling monitor election 2023-09-23T20:08:23.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:23 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:08:23.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:23 smithi018 ceph-mon[95889]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:08:23.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:23 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:08:23.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:23 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:08:23.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:23 smithi018 ceph-mon[95889]: mon.b calling monitor election 2023-09-23T20:08:23.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:23 smithi018 ceph-mon[95889]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:08:23.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:23 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:08:23.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:23 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:08:23.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:23 smithi018 ceph-mon[95889]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:08:23.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:23 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:08:23.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:23 smithi018 ceph-mon[95889]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-09-23T20:08:23.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:23 smithi018 ceph-mon[95889]: monmap e2: 2 mons at {a=[v2:172.21.15.18:3300/0,v1:172.21.15.18:6789/0],b=[v2:172.21.15.149:3300/0,v1:172.21.15.149:6789/0]} removed_ranks: {} 2023-09-23T20:08:23.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:23 smithi018 ceph-mon[95889]: fsmap 2023-09-23T20:08:23.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:23 smithi018 ceph-mon[95889]: osdmap e4: 0 total, 0 up, 0 in 2023-09-23T20:08:23.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:23 smithi018 ceph-mon[95889]: mgrmap e13: y(active, since 34s) 2023-09-23T20:08:23.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:23 smithi018 ceph-mon[95889]: overall HEALTH_OK 2023-09-23T20:08:23.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:23 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:23.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:23 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:23.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:23 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:23.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:23 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T20:08:23.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:23 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:08:24.231 INFO:teuthology.orchestra.run.smithi149.stdout: 2023-09-23T20:08:24.231 INFO:teuthology.orchestra.run.smithi149.stdout:{"epoch":2,"fsid":"a3b3b17e-5a4c-11ee-8db4-212e2dc638e7","modified":"2023-09-23T20:08:18.327121Z","created":"2023-09-23T20:06:52.152880Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:3300","nonce":0},{"type":"v1","addr":"172.21.15.18:6789","nonce":0}]},"addr":"172.21.15.18:6789/0","public_addr":"172.21.15.18:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:3300","nonce":0},{"type":"v1","addr":"172.21.15.149:6789","nonce":0}]},"addr":"172.21.15.149:6789/0","public_addr":"172.21.15.149:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2023-09-23T20:08:24.234 INFO:teuthology.orchestra.run.smithi149.stderr:dumped monmap epoch 2 2023-09-23T20:08:24.563 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:24 smithi149 ceph-mon[100764]: Deploying daemon mon.c on smithi018 2023-09-23T20:08:24.563 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:24 smithi149 ceph-mon[100764]: from='client.? 172.21.15.149:0/3896725470' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T20:08:24.563 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:24 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:08:24.661 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:24 smithi018 ceph-mon[95889]: Deploying daemon mon.c on smithi018 2023-09-23T20:08:24.661 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:24 smithi018 ceph-mon[95889]: from='client.? 172.21.15.149:0/3896725470' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T20:08:24.661 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:24 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:08:25.621 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:25 smithi018 ceph-mon[95889]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:08:25.658 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2023-09-23T20:08:25.659 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph mon dump -f json 2023-09-23T20:08:25.692 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:25 smithi149 ceph-mon[100764]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:08:25.890 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:25 smithi018 systemd[1]: Starting Ceph mon.c for a3b3b17e-5a4c-11ee-8db4-212e2dc638e7... 2023-09-23T20:08:26.213 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:25 smithi018 podman[109098]: 2023-09-23T20:08:26.213 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: set uid:gid to 167:167 (ceph:ceph) 2023-09-23T20:08:26.213 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable), process ceph-mon, pid 7 2023-09-23T20:08:26.213 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: pidfile_write: ignore empty --pid-file 2023-09-23T20:08:26.213 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: load: jerasure load: lrc 2023-09-23T20:08:26.213 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: RocksDB version: 6.15.5 2023-09-23T20:08:26.213 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2023-09-23T20:08:26.213 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Compile date Sep 22 2023 2023-09-23T20:08:26.213 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: DB SUMMARY 2023-09-23T20:08:26.213 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: DB Session ID: SDHENWEHFIO6QXKPNG9F 2023-09-23T20:08:26.213 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: CURRENT file: CURRENT 2023-09-23T20:08:26.213 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: IDENTITY file: IDENTITY 2023-09-23T20:08:26.214 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: MANIFEST file: MANIFEST-000003 size: 57 Bytes 2023-09-23T20:08:26.214 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 0, files: 2023-09-23T20:08:26.214 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000004.log size: 636 ; 2023-09-23T20:08:26.214 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.error_if_exists: 0 2023-09-23T20:08:26.214 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.create_if_missing: 0 2023-09-23T20:08:26.214 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.paranoid_checks: 1 2023-09-23T20:08:26.214 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-09-23T20:08:26.214 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.env: 0x5561cebe6a80 2023-09-23T20:08:26.214 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.fs: Posix File System 2023-09-23T20:08:26.214 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.info_log: 0x5561d06c3d80 2023-09-23T20:08:26.214 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.max_file_opening_threads: 16 2023-09-23T20:08:26.214 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.statistics: (nil) 2023-09-23T20:08:26.214 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.use_fsync: 0 2023-09-23T20:08:26.215 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.max_log_file_size: 0 2023-09-23T20:08:26.215 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.max_manifest_file_size: 1073741824 2023-09-23T20:08:26.215 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.log_file_time_to_roll: 0 2023-09-23T20:08:26.215 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.keep_log_file_num: 1000 2023-09-23T20:08:26.215 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.recycle_log_file_num: 0 2023-09-23T20:08:26.215 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.allow_fallocate: 1 2023-09-23T20:08:26.215 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.allow_mmap_reads: 0 2023-09-23T20:08:26.215 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.allow_mmap_writes: 0 2023-09-23T20:08:26.215 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.use_direct_reads: 0 2023-09-23T20:08:26.215 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-09-23T20:08:26.215 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.create_missing_column_families: 0 2023-09-23T20:08:26.215 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.db_log_dir: 2023-09-23T20:08:26.216 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-c/store.db 2023-09-23T20:08:26.216 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.table_cache_numshardbits: 6 2023-09-23T20:08:26.216 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.WAL_ttl_seconds: 0 2023-09-23T20:08:26.216 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.WAL_size_limit_MB: 0 2023-09-23T20:08:26.216 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-09-23T20:08:26.216 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.manifest_preallocation_size: 4194304 2023-09-23T20:08:26.216 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.is_fd_close_on_exec: 1 2023-09-23T20:08:26.216 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.advise_random_on_open: 1 2023-09-23T20:08:26.216 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.db_write_buffer_size: 0 2023-09-23T20:08:26.216 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.write_buffer_manager: 0x5561d13b6930 2023-09-23T20:08:26.216 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.access_hint_on_compaction_start: 1 2023-09-23T20:08:26.216 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2023-09-23T20:08:26.217 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.random_access_max_buffer_size: 1048576 2023-09-23T20:08:26.217 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.use_adaptive_mutex: 0 2023-09-23T20:08:26.217 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.rate_limiter: (nil) 2023-09-23T20:08:26.217 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-09-23T20:08:26.217 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.wal_recovery_mode: 2 2023-09-23T20:08:26.217 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.enable_thread_tracking: 0 2023-09-23T20:08:26.217 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.enable_pipelined_write: 0 2023-09-23T20:08:26.217 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.unordered_write: 0 2023-09-23T20:08:26.217 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.allow_concurrent_memtable_write: 1 2023-09-23T20:08:26.217 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-09-23T20:08:26.217 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.write_thread_max_yield_usec: 100 2023-09-23T20:08:26.217 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.write_thread_slow_yield_usec: 3 2023-09-23T20:08:26.218 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.row_cache: None 2023-09-23T20:08:26.218 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.wal_filter: None 2023-09-23T20:08:26.218 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.avoid_flush_during_recovery: 0 2023-09-23T20:08:26.218 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.allow_ingest_behind: 0 2023-09-23T20:08:26.218 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.preserve_deletes: 0 2023-09-23T20:08:26.218 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.two_write_queues: 0 2023-09-23T20:08:26.218 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.manual_wal_flush: 0 2023-09-23T20:08:26.218 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.atomic_flush: 0 2023-09-23T20:08:26.218 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-09-23T20:08:26.218 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.persist_stats_to_disk: 0 2023-09-23T20:08:26.218 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.write_dbid_to_manifest: 0 2023-09-23T20:08:26.218 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.log_readahead_size: 0 2023-09-23T20:08:26.218 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.file_checksum_gen_factory: Unknown 2023-09-23T20:08:26.219 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.best_efforts_recovery: 0 2023-09-23T20:08:26.219 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-09-23T20:08:26.219 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-09-23T20:08:26.219 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.allow_data_in_errors: 0 2023-09-23T20:08:26.219 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.db_host_id: __hostname__ 2023-09-23T20:08:26.220 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.max_background_jobs: 2 2023-09-23T20:08:26.220 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.max_background_compactions: -1 2023-09-23T20:08:26.220 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.max_subcompactions: 1 2023-09-23T20:08:26.220 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.avoid_flush_during_shutdown: 0 2023-09-23T20:08:26.220 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-09-23T20:08:26.220 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.delayed_write_rate : 16777216 2023-09-23T20:08:26.220 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.max_total_wal_size: 0 2023-09-23T20:08:26.221 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-09-23T20:08:26.221 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.stats_dump_period_sec: 600 2023-09-23T20:08:26.221 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.stats_persist_period_sec: 600 2023-09-23T20:08:26.221 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.stats_history_buffer_size: 1048576 2023-09-23T20:08:26.221 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.max_open_files: -1 2023-09-23T20:08:26.221 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.bytes_per_sync: 0 2023-09-23T20:08:26.221 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.wal_bytes_per_sync: 0 2023-09-23T20:08:26.221 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.strict_bytes_per_sync: 0 2023-09-23T20:08:26.221 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.compaction_readahead_size: 0 2023-09-23T20:08:26.221 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.max_background_flushes: -1 2023-09-23T20:08:26.221 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Compression algorithms supported: 2023-09-23T20:08:26.221 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: kZSTDNotFinalCompression supported: 0 2023-09-23T20:08:26.221 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: kZSTD supported: 0 2023-09-23T20:08:26.222 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: kXpressCompression supported: 0 2023-09-23T20:08:26.222 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: kLZ4HCCompression supported: 1 2023-09-23T20:08:26.222 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: kZlibCompression supported: 1 2023-09-23T20:08:26.222 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: kSnappyCompression supported: 1 2023-09-23T20:08:26.222 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: kLZ4Compression supported: 1 2023-09-23T20:08:26.222 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: kBZip2Compression supported: 0 2023-09-23T20:08:26.222 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Fast CRC32 supported: Supported on x86 2023-09-23T20:08:26.222 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000003 2023-09-23T20:08:26.222 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2023-09-23T20:08:26.222 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-09-23T20:08:26.222 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.merge_operator: 2023-09-23T20:08:26.222 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.compaction_filter: None 2023-09-23T20:08:26.223 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.compaction_filter_factory: None 2023-09-23T20:08:26.223 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.sst_partitioner_factory: None 2023-09-23T20:08:26.223 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.memtable_factory: SkipListFactory 2023-09-23T20:08:26.223 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.table_factory: BlockBasedTable 2023-09-23T20:08:26.223 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x5561d0696fa0) 2023-09-23T20:08:26.223 INFO:journalctl@ceph.mon.c.smithi018.stdout: cache_index_and_filter_blocks: 1 2023-09-23T20:08:26.223 INFO:journalctl@ceph.mon.c.smithi018.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2023-09-23T20:08:26.223 INFO:journalctl@ceph.mon.c.smithi018.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2023-09-23T20:08:26.223 INFO:journalctl@ceph.mon.c.smithi018.stdout: pin_top_level_index_and_filter: 1 2023-09-23T20:08:26.223 INFO:journalctl@ceph.mon.c.smithi018.stdout: index_type: 0 2023-09-23T20:08:26.223 INFO:journalctl@ceph.mon.c.smithi018.stdout: data_block_index_type: 0 2023-09-23T20:08:26.223 INFO:journalctl@ceph.mon.c.smithi018.stdout: index_shortening: 1 2023-09-23T20:08:26.223 INFO:journalctl@ceph.mon.c.smithi018.stdout: data_block_hash_table_util_ratio: 0.750000 2023-09-23T20:08:26.224 INFO:journalctl@ceph.mon.c.smithi018.stdout: hash_index_allow_collision: 1 2023-09-23T20:08:26.224 INFO:journalctl@ceph.mon.c.smithi018.stdout: checksum: 1 2023-09-23T20:08:26.224 INFO:journalctl@ceph.mon.c.smithi018.stdout: no_block_cache: 0 2023-09-23T20:08:26.224 INFO:journalctl@ceph.mon.c.smithi018.stdout: block_cache: 0x5561d06c9090 2023-09-23T20:08:26.224 INFO:journalctl@ceph.mon.c.smithi018.stdout: block_cache_name: BinnedLRUCache 2023-09-23T20:08:26.224 INFO:journalctl@ceph.mon.c.smithi018.stdout: block_cache_options: 2023-09-23T20:08:26.224 INFO:journalctl@ceph.mon.c.smithi018.stdout: capacity : 536870912 2023-09-23T20:08:26.224 INFO:journalctl@ceph.mon.c.smithi018.stdout: num_shard_bits : 4 2023-09-23T20:08:26.224 INFO:journalctl@ceph.mon.c.smithi018.stdout: strict_capacity_limit : 0 2023-09-23T20:08:26.224 INFO:journalctl@ceph.mon.c.smithi018.stdout: high_pri_pool_ratio: 0.000 2023-09-23T20:08:26.224 INFO:journalctl@ceph.mon.c.smithi018.stdout: block_cache_compressed: (nil) 2023-09-23T20:08:26.224 INFO:journalctl@ceph.mon.c.smithi018.stdout: persistent_cache: (nil) 2023-09-23T20:08:26.224 INFO:journalctl@ceph.mon.c.smithi018.stdout: block_size: 4096 2023-09-23T20:08:26.225 INFO:journalctl@ceph.mon.c.smithi018.stdout: block_size_deviation: 10 2023-09-23T20:08:26.225 INFO:journalctl@ceph.mon.c.smithi018.stdout: block_restart_interval: 16 2023-09-23T20:08:26.225 INFO:journalctl@ceph.mon.c.smithi018.stdout: index_block_restart_interval: 1 2023-09-23T20:08:26.225 INFO:journalctl@ceph.mon.c.smithi018.stdout: metadata_block_size: 4096 2023-09-23T20:08:26.225 INFO:journalctl@ceph.mon.c.smithi018.stdout: partition_filters: 0 2023-09-23T20:08:26.225 INFO:journalctl@ceph.mon.c.smithi018.stdout: use_delta_encoding: 1 2023-09-23T20:08:26.225 INFO:journalctl@ceph.mon.c.smithi018.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2023-09-23T20:08:26.225 INFO:journalctl@ceph.mon.c.smithi018.stdout: whole_key_filtering: 1 2023-09-23T20:08:26.225 INFO:journalctl@ceph.mon.c.smithi018.stdout: verify_compression: 0 2023-09-23T20:08:26.225 INFO:journalctl@ceph.mon.c.smithi018.stdout: read_amp_bytes_per_bit: 0 2023-09-23T20:08:26.225 INFO:journalctl@ceph.mon.c.smithi018.stdout: format_version: 4 2023-09-23T20:08:26.225 INFO:journalctl@ceph.mon.c.smithi018.stdout: enable_index_compression: 1 2023-09-23T20:08:26.226 INFO:journalctl@ceph.mon.c.smithi018.stdout: block_align: 0 2023-09-23T20:08:26.226 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.write_buffer_size: 33554432 2023-09-23T20:08:26.226 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.max_write_buffer_number: 2 2023-09-23T20:08:26.226 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.compression: NoCompression 2023-09-23T20:08:26.226 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.bottommost_compression: Disabled 2023-09-23T20:08:26.226 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.prefix_extractor: nullptr 2023-09-23T20:08:26.226 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-09-23T20:08:26.226 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.num_levels: 7 2023-09-23T20:08:26.226 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-09-23T20:08:26.226 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-09-23T20:08:26.226 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-09-23T20:08:26.226 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-09-23T20:08:26.226 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.bottommost_compression_opts.level: 32767 2023-09-23T20:08:26.227 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-09-23T20:08:26.227 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-09-23T20:08:26.227 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-09-23T20:08:26.227 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-09-23T20:08:26.227 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.bottommost_compression_opts.enabled: false 2023-09-23T20:08:26.227 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.compression_opts.window_bits: -14 2023-09-23T20:08:26.227 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.compression_opts.level: 32767 2023-09-23T20:08:26.227 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.compression_opts.strategy: 0 2023-09-23T20:08:26.227 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-09-23T20:08:26.227 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-09-23T20:08:26.227 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.compression_opts.parallel_threads: 1 2023-09-23T20:08:26.227 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.compression_opts.enabled: false 2023-09-23T20:08:26.227 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-09-23T20:08:26.228 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-09-23T20:08:26.228 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.level0_stop_writes_trigger: 36 2023-09-23T20:08:26.228 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.target_file_size_base: 67108864 2023-09-23T20:08:26.228 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.target_file_size_multiplier: 1 2023-09-23T20:08:26.228 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.max_bytes_for_level_base: 268435456 2023-09-23T20:08:26.228 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-09-23T20:08:26.228 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-09-23T20:08:26.228 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-09-23T20:08:26.228 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-09-23T20:08:26.228 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-09-23T20:08:26.228 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-09-23T20:08:26.228 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-09-23T20:08:26.228 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-09-23T20:08:26.229 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-09-23T20:08:26.229 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-09-23T20:08:26.229 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.max_compaction_bytes: 1677721600 2023-09-23T20:08:26.229 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.arena_block_size: 4194304 2023-09-23T20:08:26.229 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-09-23T20:08:26.229 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-09-23T20:08:26.229 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2023-09-23T20:08:26.229 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.disable_auto_compactions: 0 2023-09-23T20:08:26.229 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-09-23T20:08:26.229 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-09-23T20:08:26.229 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-09-23T20:08:26.229 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-09-23T20:08:26.229 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-09-23T20:08:26.229 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-09-23T20:08:26.230 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-09-23T20:08:26.230 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-09-23T20:08:26.230 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-09-23T20:08:26.230 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-09-23T20:08:26.230 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.table_properties_collectors: 2023-09-23T20:08:26.230 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.inplace_update_support: 0 2023-09-23T20:08:26.230 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.inplace_update_num_locks: 10000 2023-09-23T20:08:26.230 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-09-23T20:08:26.230 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.memtable_whole_key_filtering: 0 2023-09-23T20:08:26.230 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.memtable_huge_page_size: 0 2023-09-23T20:08:26.230 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.bloom_locality: 0 2023-09-23T20:08:26.230 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.max_successive_merges: 0 2023-09-23T20:08:26.231 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.optimize_filters_for_hits: 0 2023-09-23T20:08:26.231 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.paranoid_file_checks: 0 2023-09-23T20:08:26.231 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.force_consistency_checks: 1 2023-09-23T20:08:26.231 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.report_bg_io_stats: 0 2023-09-23T20:08:26.231 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.ttl: 2592000 2023-09-23T20:08:26.231 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.periodic_compaction_seconds: 0 2023-09-23T20:08:26.231 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.enable_blob_files: false 2023-09-23T20:08:26.231 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.min_blob_size: 0 2023-09-23T20:08:26.231 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.blob_file_size: 268435456 2023-09-23T20:08:26.231 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.blob_compression_type: NoCompression 2023-09-23T20:08:26.231 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.enable_blob_garbage_collection: false 2023-09-23T20:08:26.231 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-09-23T20:08:26.231 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000003 succeeded,manifest_file_number is 3, next_file_number is 5, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-09-23T20:08:26.232 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 0 2023-09-23T20:08:26.232 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: [db/version_set.cc:4082] Creating manifest 7 2023-09-23T20:08:26.232 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695499706110288, "job": 1, "event": "recovery_started", "wal_files": [4]} 2023-09-23T20:08:26.232 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: [db/db_impl/db_impl_open.cc:845] Recovering log #4 mode 2 2023-09-23T20:08:26.232 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: [table/block_based/filter_policy.cc:991] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2023-09-23T20:08:26.232 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695499706110561, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1670, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 648, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 526, "raw_average_value_size": 105, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1695499706, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "40cdb6c8-0a11-44b1-b6ed-8b6d9e54d197", "db_session_id": "SDHENWEHFIO6QXKPNG9F"}} 2023-09-23T20:08:26.232 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: [db/version_set.cc:4082] Creating manifest 9 2023-09-23T20:08:26.232 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695499706110907, "job": 1, "event": "recovery_finished"} 2023-09-23T20:08:26.232 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: [file/delete_scheduler.cc:69] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-09-23T20:08:26.232 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x5561d065d180 2023-09-23T20:08:26.232 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: DB pointer 0x5561d0732000 2023-09-23T20:08:26.232 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2023-09-23T20:08:26.232 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: rocksdb: [db/db_impl/db_impl.cc:903] 2023-09-23T20:08:26.233 INFO:journalctl@ceph.mon.c.smithi018.stdout: ** DB Stats ** 2023-09-23T20:08:26.233 INFO:journalctl@ceph.mon.c.smithi018.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T20:08:26.233 INFO:journalctl@ceph.mon.c.smithi018.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-09-23T20:08:26.233 INFO:journalctl@ceph.mon.c.smithi018.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-09-23T20:08:26.233 INFO:journalctl@ceph.mon.c.smithi018.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T20:08:26.233 INFO:journalctl@ceph.mon.c.smithi018.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-09-23T20:08:26.233 INFO:journalctl@ceph.mon.c.smithi018.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2023-09-23T20:08:26.233 INFO:journalctl@ceph.mon.c.smithi018.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T20:08:26.233 INFO:journalctl@ceph.mon.c.smithi018.stdout: 2023-09-23T20:08:26.233 INFO:journalctl@ceph.mon.c.smithi018.stdout: ** Compaction Stats [default] ** 2023-09-23T20:08:26.233 INFO:journalctl@ceph.mon.c.smithi018.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-23T20:08:26.233 INFO:journalctl@ceph.mon.c.smithi018.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T20:08:26.233 INFO:journalctl@ceph.mon.c.smithi018.stdout: L0 1/0 1.63 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 6.2 0.00 0.00 1 0.000 0 0 2023-09-23T20:08:26.234 INFO:journalctl@ceph.mon.c.smithi018.stdout: Sum 1/0 1.63 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 6.2 0.00 0.00 1 0.000 0 0 2023-09-23T20:08:26.234 INFO:journalctl@ceph.mon.c.smithi018.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 6.2 0.00 0.00 1 0.000 0 0 2023-09-23T20:08:26.234 INFO:journalctl@ceph.mon.c.smithi018.stdout: 2023-09-23T20:08:26.234 INFO:journalctl@ceph.mon.c.smithi018.stdout: ** Compaction Stats [default] ** 2023-09-23T20:08:26.234 INFO:journalctl@ceph.mon.c.smithi018.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-23T20:08:26.234 INFO:journalctl@ceph.mon.c.smithi018.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T20:08:26.234 INFO:journalctl@ceph.mon.c.smithi018.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 6.2 0.00 0.00 1 0.000 0 0 2023-09-23T20:08:26.234 INFO:journalctl@ceph.mon.c.smithi018.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T20:08:26.234 INFO:journalctl@ceph.mon.c.smithi018.stdout: Flush(GB): cumulative 0.000, interval 0.000 2023-09-23T20:08:26.234 INFO:journalctl@ceph.mon.c.smithi018.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-09-23T20:08:26.234 INFO:journalctl@ceph.mon.c.smithi018.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-09-23T20:08:26.234 INFO:journalctl@ceph.mon.c.smithi018.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-09-23T20:08:26.235 INFO:journalctl@ceph.mon.c.smithi018.stdout: AddFile(Keys): cumulative 0, interval 0 2023-09-23T20:08:26.235 INFO:journalctl@ceph.mon.c.smithi018.stdout: Cumulative compaction: 0.00 GB write, 0.61 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T20:08:26.235 INFO:journalctl@ceph.mon.c.smithi018.stdout: Interval compaction: 0.00 GB write, 0.61 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T20:08:26.235 INFO:journalctl@ceph.mon.c.smithi018.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-09-23T20:08:26.235 INFO:journalctl@ceph.mon.c.smithi018.stdout: 2023-09-23T20:08:26.235 INFO:journalctl@ceph.mon.c.smithi018.stdout: ** File Read Latency Histogram By Level [default] ** 2023-09-23T20:08:26.235 INFO:journalctl@ceph.mon.c.smithi018.stdout: 2023-09-23T20:08:26.235 INFO:journalctl@ceph.mon.c.smithi018.stdout: ** Compaction Stats [default] ** 2023-09-23T20:08:26.235 INFO:journalctl@ceph.mon.c.smithi018.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-23T20:08:26.235 INFO:journalctl@ceph.mon.c.smithi018.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T20:08:26.235 INFO:journalctl@ceph.mon.c.smithi018.stdout: L0 1/0 1.63 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 6.2 0.00 0.00 1 0.000 0 0 2023-09-23T20:08:26.235 INFO:journalctl@ceph.mon.c.smithi018.stdout: Sum 1/0 1.63 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 6.2 0.00 0.00 1 0.000 0 0 2023-09-23T20:08:26.236 INFO:journalctl@ceph.mon.c.smithi018.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-09-23T20:08:26.236 INFO:journalctl@ceph.mon.c.smithi018.stdout: 2023-09-23T20:08:26.236 INFO:journalctl@ceph.mon.c.smithi018.stdout: ** Compaction Stats [default] ** 2023-09-23T20:08:26.236 INFO:journalctl@ceph.mon.c.smithi018.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-23T20:08:26.236 INFO:journalctl@ceph.mon.c.smithi018.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T20:08:26.236 INFO:journalctl@ceph.mon.c.smithi018.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 6.2 0.00 0.00 1 0.000 0 0 2023-09-23T20:08:26.236 INFO:journalctl@ceph.mon.c.smithi018.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T20:08:26.236 INFO:journalctl@ceph.mon.c.smithi018.stdout: Flush(GB): cumulative 0.000, interval 0.000 2023-09-23T20:08:26.236 INFO:journalctl@ceph.mon.c.smithi018.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-09-23T20:08:26.236 INFO:journalctl@ceph.mon.c.smithi018.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-09-23T20:08:26.236 INFO:journalctl@ceph.mon.c.smithi018.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-09-23T20:08:26.236 INFO:journalctl@ceph.mon.c.smithi018.stdout: AddFile(Keys): cumulative 0, interval 0 2023-09-23T20:08:26.236 INFO:journalctl@ceph.mon.c.smithi018.stdout: Cumulative compaction: 0.00 GB write, 0.60 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T20:08:26.237 INFO:journalctl@ceph.mon.c.smithi018.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T20:08:26.237 INFO:journalctl@ceph.mon.c.smithi018.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-09-23T20:08:26.237 INFO:journalctl@ceph.mon.c.smithi018.stdout: 2023-09-23T20:08:26.237 INFO:journalctl@ceph.mon.c.smithi018.stdout: ** File Read Latency Histogram By Level [default] ** 2023-09-23T20:08:26.237 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: mon.c does not exist in monmap, will attempt to join an existing cluster 2023-09-23T20:08:26.237 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: using public_addrv [v2:172.21.15.18:3301/0,v1:172.21.15.18:6790/0] 2023-09-23T20:08:26.237 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: starting mon.c rank -1 at public addrs [v2:172.21.15.18:3301/0,v1:172.21.15.18:6790/0] at bind addrs [v2:172.21.15.18:3301/0,v1:172.21.15.18:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 2023-09-23T20:08:26.237 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: mon.c@-1(???) e0 preinit fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 2023-09-23T20:08:26.237 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 bash[109098]: d35a46c484d5c725b584b634d9defdfd6a0e8475c495f6b5e320bdef36f1125f 2023-09-23T20:08:26.237 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: mon.c@-1(synchronizing).mds e1 new map 2023-09-23T20:08:26.237 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: mon.c@-1(synchronizing).mds e1 print_map 2023-09-23T20:08:26.237 INFO:journalctl@ceph.mon.c.smithi018.stdout: e1 2023-09-23T20:08:26.237 INFO:journalctl@ceph.mon.c.smithi018.stdout: enable_multiple, ever_enabled_multiple: 1,1 2023-09-23T20:08:26.238 INFO:journalctl@ceph.mon.c.smithi018.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-09-23T20:08:26.238 INFO:journalctl@ceph.mon.c.smithi018.stdout: legacy client fscid: -1 2023-09-23T20:08:26.238 INFO:journalctl@ceph.mon.c.smithi018.stdout: 2023-09-23T20:08:26.238 INFO:journalctl@ceph.mon.c.smithi018.stdout: No filesystems configured 2023-09-23T20:08:26.238 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: mon.c@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2023-09-23T20:08:26.238 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: mon.c@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2023-09-23T20:08:26.238 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: mon.c@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2023-09-23T20:08:26.238 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: mon.c@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2023-09-23T20:08:26.238 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: mon.c@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2023-09-23T20:08:26.238 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: mon.c@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2023-09-23T20:08:26.238 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: mon.c@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2023-09-23T20:08:26.238 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-09-23T20:08:26.239 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-09-23T20:08:26.239 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-09-23T20:08:26.239 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: Updating smithi149:/etc/ceph/ceph.conf 2023-09-23T20:08:26.239 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: from='client.? 172.21.15.18:0/360296344' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2023-09-23T20:08:26.239 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: osdmap e4: 0 total, 0 up, 0 in 2023-09-23T20:08:26.239 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: Updating smithi149:/var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T20:08:26.239 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: Updating smithi149:/etc/ceph/ceph.client.admin.keyring 2023-09-23T20:08:26.239 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:08:26.239 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:26.239 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:26.239 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:26.239 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: Updating smithi149:/var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T20:08:26.240 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:08:26.240 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: from='client.14182 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi018:172.21.15.18=a;smithi018:[v2:172.21.15.18:3301,v1:172.21.15.18:6790]=c;smithi149:172.21.15.149=b", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:08:26.240 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: Saving service mon spec with placement smithi018:172.21.15.18=a;smithi018:[v2:172.21.15.18:3301,v1:172.21.15.18:6790]=c;smithi149:172.21.15.149=b;count:3 2023-09-23T20:08:26.240 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:26.240 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:08:26.240 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:08:26.240 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:08:26.240 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:26.240 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T20:08:26.240 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:08:26.240 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: Deploying daemon mon.b on smithi149 2023-09-23T20:08:26.240 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:08:26.240 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T20:08:26.241 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: mon.a calling monitor election 2023-09-23T20:08:26.241 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:08:26.241 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:08:26.241 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:08:26.241 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:08:26.241 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: mon.b calling monitor election 2023-09-23T20:08:26.241 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:08:26.241 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:08:26.241 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:08:26.241 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:08:26.241 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:08:26.241 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-09-23T20:08:26.241 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: monmap e2: 2 mons at {a=[v2:172.21.15.18:3300/0,v1:172.21.15.18:6789/0],b=[v2:172.21.15.149:3300/0,v1:172.21.15.149:6789/0]} removed_ranks: {} 2023-09-23T20:08:26.242 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: fsmap 2023-09-23T20:08:26.242 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: osdmap e4: 0 total, 0 up, 0 in 2023-09-23T20:08:26.242 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: mgrmap e13: y(active, since 34s) 2023-09-23T20:08:26.242 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: overall HEALTH_OK 2023-09-23T20:08:26.242 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:26.242 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:26.242 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:26.242 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T20:08:26.242 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:08:26.242 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: Deploying daemon mon.c on smithi018 2023-09-23T20:08:26.242 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: from='client.? 172.21.15.149:0/3896725470' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T20:08:26.242 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:08:26.243 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:08:26.243 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: mon.c@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2023-09-23T20:08:26.243 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 ceph-mon[109137]: mon.c@-1(synchronizing) e2 handle_conf_change mon_allow_pool_delete 2023-09-23T20:08:26.243 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:26 smithi018 systemd[1]: Started Ceph mon.c for a3b3b17e-5a4c-11ee-8db4-212e2dc638e7. 2023-09-23T20:08:26.262 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:08:31.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:31 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T20:08:31.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:31 smithi018 ceph-mon[95889]: mon.a calling monitor election 2023-09-23T20:08:31.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:31 smithi018 ceph-mon[95889]: mon.b calling monitor election 2023-09-23T20:08:31.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:31 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:08:31.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:31 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:08:31.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:31 smithi018 ceph-mon[95889]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:08:31.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:31 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:08:31.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:31 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:08:31.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:31 smithi018 ceph-mon[95889]: mon.c calling monitor election 2023-09-23T20:08:31.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:31 smithi018 ceph-mon[95889]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:08:31.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:31 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:08:31.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:31 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:08:31.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:31 smithi018 ceph-mon[95889]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:08:31.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:31 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:08:31.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:31 smithi018 ceph-mon[95889]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T20:08:31.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:31 smithi018 ceph-mon[95889]: monmap e3: 3 mons at {a=[v2:172.21.15.18:3300/0,v1:172.21.15.18:6789/0],b=[v2:172.21.15.149:3300/0,v1:172.21.15.149:6789/0],c=[v2:172.21.15.18:3301/0,v1:172.21.15.18:6790/0]} removed_ranks: {} 2023-09-23T20:08:31.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:31 smithi018 ceph-mon[95889]: fsmap 2023-09-23T20:08:31.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:31 smithi018 ceph-mon[95889]: osdmap e4: 0 total, 0 up, 0 in 2023-09-23T20:08:31.644 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:31 smithi018 ceph-mon[95889]: mgrmap e13: y(active, since 42s) 2023-09-23T20:08:31.644 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:31 smithi018 ceph-mon[95889]: overall HEALTH_OK 2023-09-23T20:08:31.644 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:31 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:31.644 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:31 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:31.644 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:31 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:31.644 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:31 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:31.645 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:31 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T20:08:31.645 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:31 smithi018 ceph-mon[109137]: mon.a calling monitor election 2023-09-23T20:08:31.645 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:31 smithi018 ceph-mon[109137]: mon.b calling monitor election 2023-09-23T20:08:31.645 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:31 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:08:31.645 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:31 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:08:31.645 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:31 smithi018 ceph-mon[109137]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:08:31.645 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:31 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:08:31.645 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:31 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:08:31.646 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:31 smithi018 ceph-mon[109137]: mon.c calling monitor election 2023-09-23T20:08:31.646 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:31 smithi018 ceph-mon[109137]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:08:31.646 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:31 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:08:31.646 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:31 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:08:31.646 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:31 smithi018 ceph-mon[109137]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:08:31.646 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:31 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:08:31.646 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:31 smithi018 ceph-mon[109137]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T20:08:31.646 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:31 smithi018 ceph-mon[109137]: monmap e3: 3 mons at {a=[v2:172.21.15.18:3300/0,v1:172.21.15.18:6789/0],b=[v2:172.21.15.149:3300/0,v1:172.21.15.149:6789/0],c=[v2:172.21.15.18:3301/0,v1:172.21.15.18:6790/0]} removed_ranks: {} 2023-09-23T20:08:31.647 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:31 smithi018 ceph-mon[109137]: fsmap 2023-09-23T20:08:31.647 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:31 smithi018 ceph-mon[109137]: osdmap e4: 0 total, 0 up, 0 in 2023-09-23T20:08:31.647 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:31 smithi018 ceph-mon[109137]: mgrmap e13: y(active, since 42s) 2023-09-23T20:08:31.647 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:31 smithi018 ceph-mon[109137]: overall HEALTH_OK 2023-09-23T20:08:31.647 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:31 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:31.647 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:31 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:31.647 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:31 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:31.647 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:31 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:31.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:31 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T20:08:31.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:31 smithi149 ceph-mon[100764]: mon.a calling monitor election 2023-09-23T20:08:31.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:31 smithi149 ceph-mon[100764]: mon.b calling monitor election 2023-09-23T20:08:31.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:31 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:08:31.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:31 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:08:31.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:31 smithi149 ceph-mon[100764]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:08:31.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:31 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:08:31.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:31 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:08:31.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:31 smithi149 ceph-mon[100764]: mon.c calling monitor election 2023-09-23T20:08:31.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:31 smithi149 ceph-mon[100764]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:08:31.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:31 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:08:31.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:31 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:08:31.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:31 smithi149 ceph-mon[100764]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:08:31.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:31 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:08:31.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:31 smithi149 ceph-mon[100764]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T20:08:31.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:31 smithi149 ceph-mon[100764]: monmap e3: 3 mons at {a=[v2:172.21.15.18:3300/0,v1:172.21.15.18:6789/0],b=[v2:172.21.15.149:3300/0,v1:172.21.15.149:6789/0],c=[v2:172.21.15.18:3301/0,v1:172.21.15.18:6790/0]} removed_ranks: {} 2023-09-23T20:08:31.718 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:31 smithi149 ceph-mon[100764]: fsmap 2023-09-23T20:08:31.718 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:31 smithi149 ceph-mon[100764]: osdmap e4: 0 total, 0 up, 0 in 2023-09-23T20:08:31.718 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:31 smithi149 ceph-mon[100764]: mgrmap e13: y(active, since 42s) 2023-09-23T20:08:31.718 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:31 smithi149 ceph-mon[100764]: overall HEALTH_OK 2023-09-23T20:08:31.718 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:31 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:31.718 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:31 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:31.718 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:31 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:31.718 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:31 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:31.976 INFO:teuthology.orchestra.run.smithi149.stdout: 2023-09-23T20:08:31.976 INFO:teuthology.orchestra.run.smithi149.stdout:{"epoch":3,"fsid":"a3b3b17e-5a4c-11ee-8db4-212e2dc638e7","modified":"2023-09-23T20:08:26.130565Z","created":"2023-09-23T20:06:52.152880Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:3300","nonce":0},{"type":"v1","addr":"172.21.15.18:6789","nonce":0}]},"addr":"172.21.15.18:6789/0","public_addr":"172.21.15.18:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:3300","nonce":0},{"type":"v1","addr":"172.21.15.149:6789","nonce":0}]},"addr":"172.21.15.149:6789/0","public_addr":"172.21.15.149:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":2,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:3301","nonce":0},{"type":"v1","addr":"172.21.15.18:6790","nonce":0}]},"addr":"172.21.15.18:6790/0","public_addr":"172.21.15.18:6790/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1,2]} 2023-09-23T20:08:31.979 INFO:teuthology.orchestra.run.smithi149.stderr:dumped monmap epoch 3 2023-09-23T20:08:32.295 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:32 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:08:32.296 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:32 smithi149 ceph-mon[100764]: from='client.? 172.21.15.149:0/695057181' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T20:08:32.296 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:32 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:08:32.316 INFO:tasks.cephadm:Generating final ceph.conf file... 2023-09-23T20:08:32.316 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph config generate-minimal-conf 2023-09-23T20:08:32.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:32 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:08:32.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:32 smithi018 ceph-mon[95889]: from='client.? 172.21.15.149:0/695057181' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T20:08:32.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:32 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:08:32.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:32 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:08:32.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:32 smithi018 ceph-mon[109137]: from='client.? 172.21.15.149:0/695057181' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T20:08:32.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:32 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:08:32.923 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:08:34.010 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:33 smithi018 ceph-mon[109137]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:08:34.011 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:33 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:34.011 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:33 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:34.011 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:33 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:34.011 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:33 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:34.011 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:33 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:08:34.011 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:33 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:08:34.011 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:33 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:34.011 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:33 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:34.011 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:33 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:34.012 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:33 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:34.012 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:33 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:34.012 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:33 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:34.012 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:33 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T20:08:34.012 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:33 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T20:08:34.012 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:33 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:08:34.012 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:33 smithi018 ceph-mon[95889]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:08:34.012 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:33 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:34.013 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:33 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:34.013 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:33 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:34.013 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:33 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:34.013 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:33 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:08:34.013 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:33 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:08:34.013 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:33 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:34.013 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:33 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:34.013 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:33 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:34.013 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:33 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:34.014 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:33 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:34.014 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:33 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:34.014 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:33 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T20:08:34.014 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:33 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T20:08:34.014 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:33 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:08:34.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:33 smithi149 ceph-mon[100764]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:08:34.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:33 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:34.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:33 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:34.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:33 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:34.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:33 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:34.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:33 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:08:34.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:33 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:08:34.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:33 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:34.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:33 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:34.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:33 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:34.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:33 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:34.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:33 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:34.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:33 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:34.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:33 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T20:08:34.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:33 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T20:08:34.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:33 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:08:34.401 INFO:teuthology.orchestra.run.smithi018.stdout:# minimal ceph.conf for a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 2023-09-23T20:08:34.401 INFO:teuthology.orchestra.run.smithi018.stdout:[global] 2023-09-23T20:08:34.402 INFO:teuthology.orchestra.run.smithi018.stdout: fsid = a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 2023-09-23T20:08:34.402 INFO:teuthology.orchestra.run.smithi018.stdout: mon_host = [v2:172.21.15.18:3300/0,v1:172.21.15.18:6789/0] [v2:172.21.15.149:3300/0,v1:172.21.15.149:6789/0] [v2:172.21.15.18:3301/0,v1:172.21.15.18:6790/0] 2023-09-23T20:08:35.121 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:34 smithi018 ceph-mon[109137]: Updating smithi018:/etc/ceph/ceph.conf 2023-09-23T20:08:35.122 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:34 smithi018 ceph-mon[109137]: Updating smithi149:/etc/ceph/ceph.conf 2023-09-23T20:08:35.122 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:34 smithi018 ceph-mon[109137]: Updating smithi018:/var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T20:08:35.122 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:34 smithi018 ceph-mon[109137]: Updating smithi149:/var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T20:08:35.122 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:34 smithi018 ceph-mon[109137]: Reconfiguring mon.a (unknown last config time)... 2023-09-23T20:08:35.122 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:34 smithi018 ceph-mon[109137]: Reconfiguring daemon mon.a on smithi018 2023-09-23T20:08:35.122 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:34 smithi018 ceph-mon[109137]: from='client.? 172.21.15.18:0/2780143209' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:08:35.122 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:34 smithi018 ceph-mon[95889]: Updating smithi018:/etc/ceph/ceph.conf 2023-09-23T20:08:35.122 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:34 smithi018 ceph-mon[95889]: Updating smithi149:/etc/ceph/ceph.conf 2023-09-23T20:08:35.122 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:34 smithi018 ceph-mon[95889]: Updating smithi018:/var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T20:08:35.123 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:34 smithi018 ceph-mon[95889]: Updating smithi149:/var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T20:08:35.123 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:34 smithi018 ceph-mon[95889]: Reconfiguring mon.a (unknown last config time)... 2023-09-23T20:08:35.123 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:34 smithi018 ceph-mon[95889]: Reconfiguring daemon mon.a on smithi018 2023-09-23T20:08:35.123 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:34 smithi018 ceph-mon[95889]: from='client.? 172.21.15.18:0/2780143209' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:08:35.185 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2023-09-23T20:08:35.185 DEBUG:teuthology.orchestra.run.smithi018:> set -ex 2023-09-23T20:08:35.185 DEBUG:teuthology.orchestra.run.smithi018:> sudo dd of=/etc/ceph/ceph.conf 2023-09-23T20:08:35.215 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:34 smithi149 ceph-mon[100764]: Updating smithi018:/etc/ceph/ceph.conf 2023-09-23T20:08:35.215 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:34 smithi149 ceph-mon[100764]: Updating smithi149:/etc/ceph/ceph.conf 2023-09-23T20:08:35.215 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:34 smithi149 ceph-mon[100764]: Updating smithi018:/var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T20:08:35.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:34 smithi149 ceph-mon[100764]: Updating smithi149:/var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T20:08:35.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:34 smithi149 ceph-mon[100764]: Reconfiguring mon.a (unknown last config time)... 2023-09-23T20:08:35.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:34 smithi149 ceph-mon[100764]: Reconfiguring daemon mon.a on smithi018 2023-09-23T20:08:35.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:34 smithi149 ceph-mon[100764]: from='client.? 172.21.15.18:0/2780143209' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:08:35.217 DEBUG:teuthology.orchestra.run.smithi018:> set -ex 2023-09-23T20:08:35.217 DEBUG:teuthology.orchestra.run.smithi018:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2023-09-23T20:08:35.285 DEBUG:teuthology.orchestra.run.smithi149:> set -ex 2023-09-23T20:08:35.285 DEBUG:teuthology.orchestra.run.smithi149:> sudo dd of=/etc/ceph/ceph.conf 2023-09-23T20:08:35.322 DEBUG:teuthology.orchestra.run.smithi149:> set -ex 2023-09-23T20:08:35.322 DEBUG:teuthology.orchestra.run.smithi149:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2023-09-23T20:08:35.395 INFO:tasks.cephadm:Adding mgr.y on smithi018 2023-09-23T20:08:35.395 INFO:tasks.cephadm:Adding mgr.x on smithi149 2023-09-23T20:08:35.395 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph orch apply mgr '2;smithi018=y;smithi149=x' 2023-09-23T20:08:36.030 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:08:36.047 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:35 smithi018 ceph-mon[95889]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:08:36.047 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:35 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:36.047 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:35 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:36.047 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:35 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T20:08:36.047 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:35 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T20:08:36.047 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:35 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:08:36.048 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:35 smithi018 ceph-mon[109137]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:08:36.048 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:35 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:36.048 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:35 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:36.048 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:35 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T20:08:36.048 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:35 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T20:08:36.048 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:35 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:08:36.199 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:35 smithi149 ceph-mon[100764]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:08:36.199 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:35 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:36.199 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:35 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:36.199 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:35 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T20:08:36.199 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:35 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T20:08:36.199 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:35 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:08:37.014 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:36 smithi149 ceph-mon[100764]: Reconfiguring mon.c (monmap changed)... 2023-09-23T20:08:37.014 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:36 smithi149 ceph-mon[100764]: Reconfiguring daemon mon.c on smithi018 2023-09-23T20:08:37.014 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:36 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:37.014 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:36 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:37.014 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:36 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T20:08:37.014 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:36 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T20:08:37.014 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:36 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:08:37.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:36 smithi018 ceph-mon[95889]: Reconfiguring mon.c (monmap changed)... 2023-09-23T20:08:37.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:36 smithi018 ceph-mon[95889]: Reconfiguring daemon mon.c on smithi018 2023-09-23T20:08:37.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:36 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:37.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:36 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:37.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:36 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T20:08:37.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:36 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T20:08:37.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:36 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:08:37.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:36 smithi018 ceph-mon[109137]: Reconfiguring mon.c (monmap changed)... 2023-09-23T20:08:37.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:36 smithi018 ceph-mon[109137]: Reconfiguring daemon mon.c on smithi018 2023-09-23T20:08:37.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:36 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:37.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:36 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:37.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:36 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T20:08:37.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:36 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T20:08:37.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:36 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:08:37.410 INFO:teuthology.orchestra.run.smithi149.stdout:Scheduled mgr update... 2023-09-23T20:08:37.875 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:37 smithi149 ceph-mon[100764]: Reconfiguring mon.b (monmap changed)... 2023-09-23T20:08:37.875 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:37 smithi149 ceph-mon[100764]: Reconfiguring daemon mon.b on smithi149 2023-09-23T20:08:37.875 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:37 smithi149 ceph-mon[100764]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:08:37.875 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:37 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:37.876 DEBUG:teuthology.orchestra.run.smithi149:mgr.x> sudo journalctl -f -n 0 -u ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7@mgr.x.service 2023-09-23T20:08:37.878 INFO:tasks.cephadm:Deploying OSDs... 2023-09-23T20:08:37.878 DEBUG:teuthology.orchestra.run.smithi018:> set -ex 2023-09-23T20:08:37.878 DEBUG:teuthology.orchestra.run.smithi018:> dd if=/scratch_devs of=/dev/stdout 2023-09-23T20:08:37.901 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2023-09-23T20:08:37.902 DEBUG:teuthology.orchestra.run.smithi018:> stat /dev/vg_nvme/lv_1 2023-09-23T20:08:37.907 INFO:journalctl@ceph.mgr.x.smithi149.stdout:-- Logs begin at Sat 2023-09-23 19:50:38 UTC. -- 2023-09-23T20:08:37.962 INFO:teuthology.orchestra.run.smithi018.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2023-09-23T20:08:37.962 INFO:teuthology.orchestra.run.smithi018.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-09-23T20:08:37.962 INFO:teuthology.orchestra.run.smithi018.stdout:Device: 6h/6d Inode: 386467 Links: 1 2023-09-23T20:08:37.963 INFO:teuthology.orchestra.run.smithi018.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-09-23T20:08:37.963 INFO:teuthology.orchestra.run.smithi018.stdout:Context: system_u:object_r:device_t:s0 2023-09-23T20:08:37.963 INFO:teuthology.orchestra.run.smithi018.stdout:Access: 2023-09-23 20:08:25.572785062 +0000 2023-09-23T20:08:37.963 INFO:teuthology.orchestra.run.smithi018.stdout:Modify: 2023-09-23 20:00:01.336724205 +0000 2023-09-23T20:08:37.963 INFO:teuthology.orchestra.run.smithi018.stdout:Change: 2023-09-23 20:00:01.336724205 +0000 2023-09-23T20:08:37.963 INFO:teuthology.orchestra.run.smithi018.stdout: Birth: - 2023-09-23T20:08:37.963 DEBUG:teuthology.orchestra.run.smithi018:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2023-09-23T20:08:38.029 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:37 smithi018 ceph-mon[95889]: Reconfiguring mon.b (monmap changed)... 2023-09-23T20:08:38.030 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:37 smithi018 ceph-mon[95889]: Reconfiguring daemon mon.b on smithi149 2023-09-23T20:08:38.030 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:37 smithi018 ceph-mon[95889]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:08:38.030 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:37 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:38.030 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:37 smithi018 ceph-mon[109137]: Reconfiguring mon.b (monmap changed)... 2023-09-23T20:08:38.030 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:37 smithi018 ceph-mon[109137]: Reconfiguring daemon mon.b on smithi149 2023-09-23T20:08:38.030 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:37 smithi018 ceph-mon[109137]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:08:38.030 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:37 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:38.033 INFO:teuthology.orchestra.run.smithi018.stderr:1+0 records in 2023-09-23T20:08:38.033 INFO:teuthology.orchestra.run.smithi018.stderr:1+0 records out 2023-09-23T20:08:38.033 INFO:teuthology.orchestra.run.smithi018.stderr:512 bytes copied, 0.000121823 s, 4.2 MB/s 2023-09-23T20:08:38.034 DEBUG:teuthology.orchestra.run.smithi018:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2023-09-23T20:08:38.094 DEBUG:teuthology.orchestra.run.smithi018:> stat /dev/vg_nvme/lv_2 2023-09-23T20:08:38.154 INFO:teuthology.orchestra.run.smithi018.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2023-09-23T20:08:38.154 INFO:teuthology.orchestra.run.smithi018.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-09-23T20:08:38.154 INFO:teuthology.orchestra.run.smithi018.stdout:Device: 6h/6d Inode: 386529 Links: 1 2023-09-23T20:08:38.154 INFO:teuthology.orchestra.run.smithi018.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-09-23T20:08:38.154 INFO:teuthology.orchestra.run.smithi018.stdout:Context: system_u:object_r:device_t:s0 2023-09-23T20:08:38.154 INFO:teuthology.orchestra.run.smithi018.stdout:Access: 2023-09-23 20:08:25.572785062 +0000 2023-09-23T20:08:38.154 INFO:teuthology.orchestra.run.smithi018.stdout:Modify: 2023-09-23 20:00:01.847718090 +0000 2023-09-23T20:08:38.154 INFO:teuthology.orchestra.run.smithi018.stdout:Change: 2023-09-23 20:00:01.847718090 +0000 2023-09-23T20:08:38.154 INFO:teuthology.orchestra.run.smithi018.stdout: Birth: - 2023-09-23T20:08:38.155 DEBUG:teuthology.orchestra.run.smithi018:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2023-09-23T20:08:38.221 INFO:teuthology.orchestra.run.smithi018.stderr:1+0 records in 2023-09-23T20:08:38.221 INFO:teuthology.orchestra.run.smithi018.stderr:1+0 records out 2023-09-23T20:08:38.221 INFO:teuthology.orchestra.run.smithi018.stderr:512 bytes copied, 0.00011826 s, 4.3 MB/s 2023-09-23T20:08:38.222 DEBUG:teuthology.orchestra.run.smithi018:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2023-09-23T20:08:38.282 DEBUG:teuthology.orchestra.run.smithi018:> stat /dev/vg_nvme/lv_3 2023-09-23T20:08:38.340 INFO:teuthology.orchestra.run.smithi018.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2023-09-23T20:08:38.341 INFO:teuthology.orchestra.run.smithi018.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-09-23T20:08:38.341 INFO:teuthology.orchestra.run.smithi018.stdout:Device: 6h/6d Inode: 388064 Links: 1 2023-09-23T20:08:38.341 INFO:teuthology.orchestra.run.smithi018.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-09-23T20:08:38.341 INFO:teuthology.orchestra.run.smithi018.stdout:Context: system_u:object_r:device_t:s0 2023-09-23T20:08:38.341 INFO:teuthology.orchestra.run.smithi018.stdout:Access: 2023-09-23 20:08:25.572785062 +0000 2023-09-23T20:08:38.341 INFO:teuthology.orchestra.run.smithi018.stdout:Modify: 2023-09-23 20:00:02.368711855 +0000 2023-09-23T20:08:38.341 INFO:teuthology.orchestra.run.smithi018.stdout:Change: 2023-09-23 20:00:02.368711855 +0000 2023-09-23T20:08:38.341 INFO:teuthology.orchestra.run.smithi018.stdout: Birth: - 2023-09-23T20:08:38.341 DEBUG:teuthology.orchestra.run.smithi018:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2023-09-23T20:08:38.408 INFO:teuthology.orchestra.run.smithi018.stderr:1+0 records in 2023-09-23T20:08:38.408 INFO:teuthology.orchestra.run.smithi018.stderr:1+0 records out 2023-09-23T20:08:38.408 INFO:teuthology.orchestra.run.smithi018.stderr:512 bytes copied, 0.000113497 s, 4.5 MB/s 2023-09-23T20:08:38.409 DEBUG:teuthology.orchestra.run.smithi018:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2023-09-23T20:08:38.468 DEBUG:teuthology.orchestra.run.smithi018:> stat /dev/vg_nvme/lv_4 2023-09-23T20:08:38.526 INFO:teuthology.orchestra.run.smithi018.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2023-09-23T20:08:38.526 INFO:teuthology.orchestra.run.smithi018.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-09-23T20:08:38.526 INFO:teuthology.orchestra.run.smithi018.stdout:Device: 6h/6d Inode: 394368 Links: 1 2023-09-23T20:08:38.526 INFO:teuthology.orchestra.run.smithi018.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-09-23T20:08:38.526 INFO:teuthology.orchestra.run.smithi018.stdout:Context: system_u:object_r:device_t:s0 2023-09-23T20:08:38.526 INFO:teuthology.orchestra.run.smithi018.stdout:Access: 2023-09-23 20:08:25.572785062 +0000 2023-09-23T20:08:38.526 INFO:teuthology.orchestra.run.smithi018.stdout:Modify: 2023-09-23 20:00:02.870705847 +0000 2023-09-23T20:08:38.526 INFO:teuthology.orchestra.run.smithi018.stdout:Change: 2023-09-23 20:00:02.870705847 +0000 2023-09-23T20:08:38.527 INFO:teuthology.orchestra.run.smithi018.stdout: Birth: - 2023-09-23T20:08:38.527 DEBUG:teuthology.orchestra.run.smithi018:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2023-09-23T20:08:38.593 INFO:teuthology.orchestra.run.smithi018.stderr:1+0 records in 2023-09-23T20:08:38.593 INFO:teuthology.orchestra.run.smithi018.stderr:1+0 records out 2023-09-23T20:08:38.593 INFO:teuthology.orchestra.run.smithi018.stderr:512 bytes copied, 9.8526e-05 s, 5.2 MB/s 2023-09-23T20:08:38.594 DEBUG:teuthology.orchestra.run.smithi018:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2023-09-23T20:08:38.652 DEBUG:teuthology.orchestra.run.smithi149:> set -ex 2023-09-23T20:08:38.652 DEBUG:teuthology.orchestra.run.smithi149:> dd if=/scratch_devs of=/dev/stdout 2023-09-23T20:08:38.669 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2023-09-23T20:08:38.669 DEBUG:teuthology.orchestra.run.smithi149:> stat /dev/vg_nvme/lv_1 2023-09-23T20:08:38.726 INFO:teuthology.orchestra.run.smithi149.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2023-09-23T20:08:38.726 INFO:teuthology.orchestra.run.smithi149.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-09-23T20:08:38.726 INFO:teuthology.orchestra.run.smithi149.stdout:Device: 6h/6d Inode: 388791 Links: 1 2023-09-23T20:08:38.727 INFO:teuthology.orchestra.run.smithi149.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-09-23T20:08:38.727 INFO:teuthology.orchestra.run.smithi149.stdout:Context: system_u:object_r:device_t:s0 2023-09-23T20:08:38.727 INFO:teuthology.orchestra.run.smithi149.stdout:Access: 2023-09-23 20:08:17.543690945 +0000 2023-09-23T20:08:38.727 INFO:teuthology.orchestra.run.smithi149.stdout:Modify: 2023-09-23 19:59:51.688467115 +0000 2023-09-23T20:08:38.727 INFO:teuthology.orchestra.run.smithi149.stdout:Change: 2023-09-23 19:59:51.688467115 +0000 2023-09-23T20:08:38.727 INFO:teuthology.orchestra.run.smithi149.stdout: Birth: - 2023-09-23T20:08:38.727 DEBUG:teuthology.orchestra.run.smithi149:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2023-09-23T20:08:38.795 INFO:teuthology.orchestra.run.smithi149.stderr:1+0 records in 2023-09-23T20:08:38.795 INFO:teuthology.orchestra.run.smithi149.stderr:1+0 records out 2023-09-23T20:08:38.795 INFO:teuthology.orchestra.run.smithi149.stderr:512 bytes copied, 0.000223326 s, 2.3 MB/s 2023-09-23T20:08:38.796 DEBUG:teuthology.orchestra.run.smithi149:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2023-09-23T20:08:38.850 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:38 smithi149 ceph-mon[100764]: from='client.24103 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi018=y;smithi149=x", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:08:38.850 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:38 smithi149 ceph-mon[100764]: Saving service mgr spec with placement smithi018=y;smithi149=x;count:2 2023-09-23T20:08:38.850 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:38 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:38.851 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:38 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:38.851 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:38 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:08:38.851 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:38 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:08:38.851 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:38 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:08:38.851 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:38 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:38.851 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:38 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T20:08:38.851 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:38 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2023-09-23T20:08:38.852 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:38 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T20:08:38.852 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:38 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:08:38.854 DEBUG:teuthology.orchestra.run.smithi149:> stat /dev/vg_nvme/lv_2 2023-09-23T20:08:38.911 INFO:teuthology.orchestra.run.smithi149.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2023-09-23T20:08:38.911 INFO:teuthology.orchestra.run.smithi149.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-09-23T20:08:38.911 INFO:teuthology.orchestra.run.smithi149.stdout:Device: 6h/6d Inode: 391180 Links: 1 2023-09-23T20:08:38.911 INFO:teuthology.orchestra.run.smithi149.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-09-23T20:08:38.911 INFO:teuthology.orchestra.run.smithi149.stdout:Context: system_u:object_r:device_t:s0 2023-09-23T20:08:38.911 INFO:teuthology.orchestra.run.smithi149.stdout:Access: 2023-09-23 20:08:17.543690945 +0000 2023-09-23T20:08:38.911 INFO:teuthology.orchestra.run.smithi149.stdout:Modify: 2023-09-23 19:59:52.198456901 +0000 2023-09-23T20:08:38.912 INFO:teuthology.orchestra.run.smithi149.stdout:Change: 2023-09-23 19:59:52.198456901 +0000 2023-09-23T20:08:38.912 INFO:teuthology.orchestra.run.smithi149.stdout: Birth: - 2023-09-23T20:08:38.912 DEBUG:teuthology.orchestra.run.smithi149:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2023-09-23T20:08:38.977 INFO:teuthology.orchestra.run.smithi149.stderr:1+0 records in 2023-09-23T20:08:38.977 INFO:teuthology.orchestra.run.smithi149.stderr:1+0 records out 2023-09-23T20:08:38.978 INFO:teuthology.orchestra.run.smithi149.stderr:512 bytes copied, 0.000224667 s, 2.3 MB/s 2023-09-23T20:08:38.979 DEBUG:teuthology.orchestra.run.smithi149:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2023-09-23T20:08:39.037 DEBUG:teuthology.orchestra.run.smithi149:> stat /dev/vg_nvme/lv_3 2023-09-23T20:08:39.094 INFO:teuthology.orchestra.run.smithi149.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2023-09-23T20:08:39.095 INFO:teuthology.orchestra.run.smithi149.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-09-23T20:08:39.095 INFO:teuthology.orchestra.run.smithi149.stdout:Device: 6h/6d Inode: 389396 Links: 1 2023-09-23T20:08:39.095 INFO:teuthology.orchestra.run.smithi149.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-09-23T20:08:39.095 INFO:teuthology.orchestra.run.smithi149.stdout:Context: system_u:object_r:device_t:s0 2023-09-23T20:08:39.095 INFO:teuthology.orchestra.run.smithi149.stdout:Access: 2023-09-23 20:08:17.543690945 +0000 2023-09-23T20:08:39.095 INFO:teuthology.orchestra.run.smithi149.stdout:Modify: 2023-09-23 19:59:52.710446646 +0000 2023-09-23T20:08:39.095 INFO:teuthology.orchestra.run.smithi149.stdout:Change: 2023-09-23 19:59:52.710446646 +0000 2023-09-23T20:08:39.095 INFO:teuthology.orchestra.run.smithi149.stdout: Birth: - 2023-09-23T20:08:39.096 DEBUG:teuthology.orchestra.run.smithi149:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2023-09-23T20:08:39.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:38 smithi018 ceph-mon[95889]: from='client.24103 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi018=y;smithi149=x", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:08:39.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:38 smithi018 ceph-mon[95889]: Saving service mgr spec with placement smithi018=y;smithi149=x;count:2 2023-09-23T20:08:39.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:38 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:39.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:38 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:39.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:38 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:08:39.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:38 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:08:39.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:38 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:08:39.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:38 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:39.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:38 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T20:08:39.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:38 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2023-09-23T20:08:39.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:38 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T20:08:39.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:38 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:08:39.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:38 smithi018 ceph-mon[109137]: from='client.24103 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi018=y;smithi149=x", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:08:39.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:38 smithi018 ceph-mon[109137]: Saving service mgr spec with placement smithi018=y;smithi149=x;count:2 2023-09-23T20:08:39.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:38 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:39.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:38 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:39.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:38 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:08:39.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:38 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:08:39.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:38 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:08:39.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:38 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:39.144 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:38 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T20:08:39.144 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:38 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2023-09-23T20:08:39.144 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:38 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T20:08:39.144 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:38 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:08:39.213 INFO:teuthology.orchestra.run.smithi149.stderr:1+0 records in 2023-09-23T20:08:39.213 INFO:teuthology.orchestra.run.smithi149.stderr:1+0 records out 2023-09-23T20:08:39.213 INFO:teuthology.orchestra.run.smithi149.stderr:512 bytes copied, 0.000266119 s, 1.9 MB/s 2023-09-23T20:08:39.214 DEBUG:teuthology.orchestra.run.smithi149:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2023-09-23T20:08:39.231 DEBUG:teuthology.orchestra.run.smithi149:> stat /dev/vg_nvme/lv_4 2023-09-23T20:08:39.289 INFO:teuthology.orchestra.run.smithi149.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2023-09-23T20:08:39.289 INFO:teuthology.orchestra.run.smithi149.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-09-23T20:08:39.289 INFO:teuthology.orchestra.run.smithi149.stdout:Device: 6h/6d Inode: 389062 Links: 1 2023-09-23T20:08:39.289 INFO:teuthology.orchestra.run.smithi149.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-09-23T20:08:39.289 INFO:teuthology.orchestra.run.smithi149.stdout:Context: system_u:object_r:device_t:s0 2023-09-23T20:08:39.289 INFO:teuthology.orchestra.run.smithi149.stdout:Access: 2023-09-23 20:08:39.165266323 +0000 2023-09-23T20:08:39.290 INFO:teuthology.orchestra.run.smithi149.stdout:Modify: 2023-09-23 19:59:53.227436291 +0000 2023-09-23T20:08:39.290 INFO:teuthology.orchestra.run.smithi149.stdout:Change: 2023-09-23 19:59:53.227436291 +0000 2023-09-23T20:08:39.290 INFO:teuthology.orchestra.run.smithi149.stdout: Birth: - 2023-09-23T20:08:39.290 DEBUG:teuthology.orchestra.run.smithi149:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2023-09-23T20:08:39.356 INFO:teuthology.orchestra.run.smithi149.stderr:1+0 records in 2023-09-23T20:08:39.356 INFO:teuthology.orchestra.run.smithi149.stderr:1+0 records out 2023-09-23T20:08:39.356 INFO:teuthology.orchestra.run.smithi149.stderr:512 bytes copied, 0.000177875 s, 2.9 MB/s 2023-09-23T20:08:39.357 DEBUG:teuthology.orchestra.run.smithi149:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2023-09-23T20:08:39.417 INFO:tasks.cephadm:Deploying osd.0 on smithi018 with /dev/vg_nvme/lv_4... 2023-09-23T20:08:39.417 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2023-09-23T20:08:39.465 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:08:39 smithi149 systemd[1]: Starting Ceph mgr.x for a3b3b17e-5a4c-11ee-8db4-212e2dc638e7... 2023-09-23T20:08:39.934 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:39 smithi149 ceph-mon[100764]: Deploying daemon mgr.x on smithi149 2023-09-23T20:08:39.934 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:39 smithi149 ceph-mon[100764]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:08:39.934 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:08:39 smithi149 podman[102864]: 2023-09-23T20:08:39.934 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:08:39 smithi149 ceph-mgr[102904]: -- 172.21.15.149:0/1579246018 <== mon.1 v2:172.21.15.149:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55886a3d49c0 con 0x55886a3aa800 2023-09-23T20:08:39.934 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:08:39 smithi149 bash[102864]: fe4e079c3943af7e2af8cb487963ddded339594221ea6e5d90108460962105ff 2023-09-23T20:08:40.029 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:08:40.140 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:39 smithi018 ceph-mon[95889]: Deploying daemon mgr.x on smithi149 2023-09-23T20:08:40.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:39 smithi018 ceph-mon[95889]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:08:40.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:39 smithi018 ceph-mon[109137]: Deploying daemon mgr.x on smithi149 2023-09-23T20:08:40.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:39 smithi018 ceph-mon[109137]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:08:40.215 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:08:39 smithi149 systemd[1]: Started Ceph mgr.x for a3b3b17e-5a4c-11ee-8db4-212e2dc638e7. 2023-09-23T20:08:40.217 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:08:40 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[102880]: 2023-09-23T20:08:40.002+0000 7f16073161c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-23T20:08:40.217 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:08:40 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[102880]: 2023-09-23T20:08:40.079+0000 7f16073161c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T20:08:40.217 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:08:40 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[102880]: 2023-09-23T20:08:40.168+0000 7f16073161c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T20:08:40.716 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:08:40 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[102880]: 2023-09-23T20:08:40.268+0000 7f16073161c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T20:08:40.716 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:08:40 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[102880]: 2023-09-23T20:08:40.455+0000 7f16073161c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T20:08:41.113 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:08:40 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[102880]: 2023-09-23T20:08:40.780+0000 7f16073161c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T20:08:41.113 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:08:40 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[102880]: 2023-09-23T20:08:40.885+0000 7f16073161c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T20:08:41.113 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:08:41 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[102880]: 2023-09-23T20:08:41.019+0000 7f16073161c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T20:08:41.113 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:41 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:41.113 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:41 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:41.113 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:41 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:41.113 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:41 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:41.113 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:41 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:08:41.113 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:41 smithi149 ceph-mon[100764]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:08:41.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:41 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:41.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:41 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:41.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:41 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:41.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:41 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:41.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:41 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:08:41.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:41 smithi018 ceph-mon[95889]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:08:41.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:41 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:41.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:41 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:41.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:41 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:41.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:41 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:41.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:41 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:08:41.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:41 smithi018 ceph-mon[109137]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:08:41.466 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:08:41 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[102880]: 2023-09-23T20:08:41.113+0000 7f16073161c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T20:08:41.647 INFO:teuthology.orchestra.run.smithi018.stderr:--> Zapping: /dev/vg_nvme/lv_4 2023-09-23T20:08:41.647 INFO:teuthology.orchestra.run.smithi018.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2023-09-23T20:08:41.647 INFO:teuthology.orchestra.run.smithi018.stderr: stderr: 10+0 records in 2023-09-23T20:08:41.647 INFO:teuthology.orchestra.run.smithi018.stderr:10+0 records out 2023-09-23T20:08:41.647 INFO:teuthology.orchestra.run.smithi018.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.00835743 s, 1.3 GB/s 2023-09-23T20:08:41.647 INFO:teuthology.orchestra.run.smithi018.stderr:--> Zapping successful for: 2023-09-23T20:08:41.965 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:08:41 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[102880]: 2023-09-23T20:08:41.564+0000 7f16073161c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T20:08:41.996 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph orch daemon add osd smithi018:vg_nvme/lv_4 2023-09-23T20:08:42.465 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:08:42 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[102880]: 2023-09-23T20:08:42.140+0000 7f16073161c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T20:08:42.466 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:08:42 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[102880]: 2023-09-23T20:08:42.311+0000 7f16073161c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T20:08:42.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:42 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:42.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:42 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:42.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:42 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:08:42.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:42 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:08:42.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:42 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:42.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:42 smithi018 ceph-mon[95889]: Reconfiguring mgr.y (unknown last config time)... 2023-09-23T20:08:42.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:42 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T20:08:42.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:42 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T20:08:42.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:42 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:08:42.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:42 smithi018 ceph-mon[95889]: Reconfiguring daemon mgr.y on smithi018 2023-09-23T20:08:42.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:42 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:42.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:42 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:42.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:42 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:08:42.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:42 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:08:42.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:42 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:42.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:42 smithi018 ceph-mon[109137]: Reconfiguring mgr.y (unknown last config time)... 2023-09-23T20:08:42.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:42 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T20:08:42.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:42 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T20:08:42.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:42 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:08:42.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:42 smithi018 ceph-mon[109137]: Reconfiguring daemon mgr.y on smithi018 2023-09-23T20:08:42.921 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:08:42.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:42 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:42.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:42 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:42.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:42 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:08:42.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:42 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:08:42.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:42 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:42.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:42 smithi149 ceph-mon[100764]: Reconfiguring mgr.y (unknown last config time)... 2023-09-23T20:08:42.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:42 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T20:08:42.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:42 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T20:08:42.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:42 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:08:42.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:42 smithi149 ceph-mon[100764]: Reconfiguring daemon mgr.y on smithi018 2023-09-23T20:08:43.327 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:08:43 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[102880]: 2023-09-23T20:08:43.072+0000 7f16073161c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T20:08:43.327 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:08:43 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[102880]: 2023-09-23T20:08:43.162+0000 7f16073161c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-23T20:08:43.595 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:08:43 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[102880]: 2023-09-23T20:08:43.327+0000 7f16073161c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T20:08:43.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:43 smithi018 ceph-mon[109137]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:08:43.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:43 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:43.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:43 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:43.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:43 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:08:43.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:43 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:08:43.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:43 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:08:43.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:43 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:43.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:43 smithi018 ceph-mon[95889]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:08:43.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:43 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:43.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:43 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:43.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:43 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:08:43.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:43 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:08:43.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:43 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:08:43.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:43 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:43.966 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:08:43 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[102880]: 2023-09-23T20:08:43.595+0000 7f16073161c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-23T20:08:43.966 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:08:43 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[102880]: 2023-09-23T20:08:43.682+0000 7f16073161c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T20:08:43.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:43 smithi149 ceph-mon[100764]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:08:43.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:43 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:43.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:43 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:43.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:43 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:08:43.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:43 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:08:43.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:43 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:08:43.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:43 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:44.228 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:08:43 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[102880]: 2023-09-23T20:08:43.973+0000 7f16073161c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T20:08:44.702 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:08:44 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[102880]: 2023-09-23T20:08:44.228+0000 7f16073161c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T20:08:44.966 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:08:44 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[102880]: 2023-09-23T20:08:44.932+0000 7f16073161c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T20:08:44.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:44 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:44.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:44 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T20:08:44.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:44 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T20:08:44.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:44 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:08:45.140 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:44 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:45.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:44 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T20:08:45.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:44 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T20:08:45.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:44 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:08:45.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:44 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:45.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:44 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T20:08:45.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:44 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T20:08:45.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:44 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:08:45.359 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:08:45 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[102880]: 2023-09-23T20:08:45.062+0000 7f16073161c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T20:08:45.359 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:08:45 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[102880]: 2023-09-23T20:08:45.245+0000 7f16073161c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T20:08:45.703 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:08:45 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[102880]: 2023-09-23T20:08:45.359+0000 7f16073161c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T20:08:45.704 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:08:45 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[102880]: 2023-09-23T20:08:45.569+0000 7f16073161c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T20:08:45.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:45 smithi018 ceph-mon[95889]: from='client.14217 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi018:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:08:45.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:45 smithi018 ceph-mon[95889]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:08:45.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:45 smithi018 ceph-mon[109137]: from='client.14217 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi018:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:08:45.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:45 smithi018 ceph-mon[109137]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:08:45.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:45 smithi149 ceph-mon[100764]: from='client.14217 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi018:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:08:45.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:45 smithi149 ceph-mon[100764]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:08:46.705 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:08:46 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[102880]: 2023-09-23T20:08:46.450+0000 7f16073161c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T20:08:46.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:46 smithi149 ceph-mon[100764]: Standby manager daemon x started 2023-09-23T20:08:46.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:46 smithi149 ceph-mon[100764]: from='mgr.? 172.21.15.149:0/1599666152' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T20:08:46.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:46 smithi149 ceph-mon[100764]: from='mgr.? 172.21.15.149:0/1599666152' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T20:08:46.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:46 smithi149 ceph-mon[100764]: from='mgr.? 172.21.15.149:0/1599666152' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T20:08:46.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:46 smithi149 ceph-mon[100764]: from='mgr.? 172.21.15.149:0/1599666152' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T20:08:47.140 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:46 smithi018 ceph-mon[95889]: Standby manager daemon x started 2023-09-23T20:08:47.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:46 smithi018 ceph-mon[95889]: from='mgr.? 172.21.15.149:0/1599666152' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T20:08:47.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:46 smithi018 ceph-mon[95889]: from='mgr.? 172.21.15.149:0/1599666152' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T20:08:47.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:46 smithi018 ceph-mon[95889]: from='mgr.? 172.21.15.149:0/1599666152' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T20:08:47.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:46 smithi018 ceph-mon[95889]: from='mgr.? 172.21.15.149:0/1599666152' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T20:08:47.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:46 smithi018 ceph-mon[109137]: Standby manager daemon x started 2023-09-23T20:08:47.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:46 smithi018 ceph-mon[109137]: from='mgr.? 172.21.15.149:0/1599666152' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T20:08:47.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:46 smithi018 ceph-mon[109137]: from='mgr.? 172.21.15.149:0/1599666152' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T20:08:47.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:46 smithi018 ceph-mon[109137]: from='mgr.? 172.21.15.149:0/1599666152' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T20:08:47.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:46 smithi018 ceph-mon[109137]: from='mgr.? 172.21.15.149:0/1599666152' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T20:08:47.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:47 smithi149 ceph-mon[100764]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:08:47.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:47 smithi149 ceph-mon[100764]: mgrmap e14: y(active, since 58s), standbys: x 2023-09-23T20:08:47.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:47 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T20:08:47.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:47 smithi149 ceph-mon[100764]: from='client.? 172.21.15.18:0/3806566863' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "44c8aaa3-bbd1-4330-8d34-73f7a08b0d4d"}]: dispatch 2023-09-23T20:08:47.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:47 smithi149 ceph-mon[100764]: from='client.? 172.21.15.18:0/3806566863' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "44c8aaa3-bbd1-4330-8d34-73f7a08b0d4d"}]': finished 2023-09-23T20:08:47.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:47 smithi149 ceph-mon[100764]: osdmap e5: 1 total, 0 up, 1 in 2023-09-23T20:08:47.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:47 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T20:08:47.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:47 smithi149 ceph-mon[100764]: from='client.? 172.21.15.18:0/2844074010' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T20:08:48.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:47 smithi018 ceph-mon[95889]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:08:48.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:47 smithi018 ceph-mon[95889]: mgrmap e14: y(active, since 58s), standbys: x 2023-09-23T20:08:48.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:47 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T20:08:48.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:47 smithi018 ceph-mon[95889]: from='client.? 172.21.15.18:0/3806566863' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "44c8aaa3-bbd1-4330-8d34-73f7a08b0d4d"}]: dispatch 2023-09-23T20:08:48.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:47 smithi018 ceph-mon[95889]: from='client.? 172.21.15.18:0/3806566863' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "44c8aaa3-bbd1-4330-8d34-73f7a08b0d4d"}]': finished 2023-09-23T20:08:48.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:47 smithi018 ceph-mon[95889]: osdmap e5: 1 total, 0 up, 1 in 2023-09-23T20:08:48.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:47 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T20:08:48.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:47 smithi018 ceph-mon[95889]: from='client.? 172.21.15.18:0/2844074010' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T20:08:48.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:47 smithi018 ceph-mon[109137]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:08:48.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:47 smithi018 ceph-mon[109137]: mgrmap e14: y(active, since 58s), standbys: x 2023-09-23T20:08:48.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:47 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T20:08:48.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:47 smithi018 ceph-mon[109137]: from='client.? 172.21.15.18:0/3806566863' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "44c8aaa3-bbd1-4330-8d34-73f7a08b0d4d"}]: dispatch 2023-09-23T20:08:48.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:47 smithi018 ceph-mon[109137]: from='client.? 172.21.15.18:0/3806566863' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "44c8aaa3-bbd1-4330-8d34-73f7a08b0d4d"}]': finished 2023-09-23T20:08:48.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:47 smithi018 ceph-mon[109137]: osdmap e5: 1 total, 0 up, 1 in 2023-09-23T20:08:48.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:47 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T20:08:48.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:47 smithi018 ceph-mon[109137]: from='client.? 172.21.15.18:0/2844074010' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T20:08:49.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:49 smithi149 ceph-mon[100764]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:08:50.028 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:49 smithi018 ceph-mon[95889]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:08:50.029 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:49 smithi018 ceph-mon[109137]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:08:51.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:51 smithi149 ceph-mon[100764]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:08:52.140 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:51 smithi018 ceph-mon[95889]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:08:52.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:51 smithi018 ceph-mon[109137]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:08:53.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:52 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-09-23T20:08:53.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:52 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:08:53.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:52 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-09-23T20:08:53.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:52 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:08:53.215 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:52 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-09-23T20:08:53.215 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:52 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:08:53.890 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:53 smithi018 ceph-mon[95889]: Deploying daemon osd.0 on smithi018 2023-09-23T20:08:53.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:53 smithi018 ceph-mon[95889]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:08:53.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:53 smithi018 ceph-mon[109137]: Deploying daemon osd.0 on smithi018 2023-09-23T20:08:53.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:53 smithi018 ceph-mon[109137]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:08:54.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:53 smithi149 ceph-mon[100764]: Deploying daemon osd.0 on smithi018 2023-09-23T20:08:54.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:53 smithi149 ceph-mon[100764]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:08:55.095 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:54 smithi018 ceph-mon[95889]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:08:55.095 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:54 smithi018 ceph-mon[109137]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:08:55.215 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:54 smithi149 ceph-mon[100764]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:08:57.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:57 smithi149 ceph-mon[100764]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:08:57.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:57 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:57.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:57 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:57.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:57 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:08:57.985 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:57 smithi018 ceph-mon[109137]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:08:57.985 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:57 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:57.985 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:57 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:57.985 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:57 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:08:57.985 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:57 smithi018 ceph-mon[95889]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:08:57.986 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:57 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:57.986 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:57 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:57.986 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:57 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:08:59.547 INFO:teuthology.orchestra.run.smithi018.stdout:Created osd(s) 0 on host 'smithi018' 2023-09-23T20:08:59.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:59 smithi018 ceph-mon[109137]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:08:59.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:59 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:59.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:59 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:59.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:59 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:59.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:08:59 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:59.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:59 smithi018 ceph-mon[95889]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:08:59.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:59 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:59.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:59 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:59.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:59 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:59.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:08:59 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:59.925 DEBUG:teuthology.orchestra.run.smithi018:osd.0> sudo journalctl -f -n 0 -u ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7@osd.0.service 2023-09-23T20:08:59.927 INFO:tasks.cephadm:Deploying osd.1 on smithi018 with /dev/vg_nvme/lv_3... 2023-09-23T20:08:59.927 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2023-09-23T20:08:59.958 INFO:journalctl@ceph.osd.0.smithi018.stdout:-- Logs begin at Sat 2023-09-23 19:50:10 UTC. -- 2023-09-23T20:08:59.965 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:59 smithi149 ceph-mon[100764]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:08:59.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:59 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:59.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:59 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:59.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:59 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:08:59.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:08:59 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:09:00.391 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 23 20:08:59 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-0[113592]: 2023-09-23T20:08:59.997+0000 7f07ae761540 -1 osd.0 0 log_to_monitors true 2023-09-23T20:09:00.767 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:09:00.859 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:00 smithi018 ceph-mon[95889]: from='osd.0 [v2:172.21.15.18:6802/2209550685,v1:172.21.15.18:6803/2209550685]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-23T20:09:00.860 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:00 smithi018 ceph-mon[109137]: from='osd.0 [v2:172.21.15.18:6802/2209550685,v1:172.21.15.18:6803/2209550685]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-23T20:09:00.965 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:00 smithi149 ceph-mon[100764]: from='osd.0 [v2:172.21.15.18:6802/2209550685,v1:172.21.15.18:6803/2209550685]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-23T20:09:01.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:01 smithi149 ceph-mon[100764]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:09:01.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:01 smithi149 ceph-mon[100764]: from='osd.0 [v2:172.21.15.18:6802/2209550685,v1:172.21.15.18:6803/2209550685]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-09-23T20:09:01.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:01 smithi149 ceph-mon[100764]: osdmap e6: 1 total, 0 up, 1 in 2023-09-23T20:09:01.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:01 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T20:09:01.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:01 smithi149 ceph-mon[100764]: from='osd.0 [v2:172.21.15.18:6802/2209550685,v1:172.21.15.18:6803/2209550685]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi018", "root=default"]}]: dispatch 2023-09-23T20:09:02.134 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:01 smithi018 ceph-mon[109137]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:09:02.135 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:01 smithi018 ceph-mon[109137]: from='osd.0 [v2:172.21.15.18:6802/2209550685,v1:172.21.15.18:6803/2209550685]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-09-23T20:09:02.135 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:01 smithi018 ceph-mon[109137]: osdmap e6: 1 total, 0 up, 1 in 2023-09-23T20:09:02.135 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:01 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T20:09:02.135 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:01 smithi018 ceph-mon[109137]: from='osd.0 [v2:172.21.15.18:6802/2209550685,v1:172.21.15.18:6803/2209550685]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi018", "root=default"]}]: dispatch 2023-09-23T20:09:02.135 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 23 20:09:01 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-0[113592]: 2023-09-23T20:09:01.715+0000 7f07a4f47700 -1 osd.0 0 waiting for initial osdmap 2023-09-23T20:09:02.135 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 23 20:09:01 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-0[113592]: 2023-09-23T20:09:01.719+0000 7f079f541700 -1 osd.0 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T20:09:02.135 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:01 smithi018 ceph-mon[95889]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:09:02.136 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:01 smithi018 ceph-mon[95889]: from='osd.0 [v2:172.21.15.18:6802/2209550685,v1:172.21.15.18:6803/2209550685]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-09-23T20:09:02.136 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:01 smithi018 ceph-mon[95889]: osdmap e6: 1 total, 0 up, 1 in 2023-09-23T20:09:02.136 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:01 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T20:09:02.136 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:01 smithi018 ceph-mon[95889]: from='osd.0 [v2:172.21.15.18:6802/2209550685,v1:172.21.15.18:6803/2209550685]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi018", "root=default"]}]: dispatch 2023-09-23T20:09:02.656 INFO:teuthology.orchestra.run.smithi018.stderr:--> Zapping: /dev/vg_nvme/lv_3 2023-09-23T20:09:02.656 INFO:teuthology.orchestra.run.smithi018.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2023-09-23T20:09:02.656 INFO:teuthology.orchestra.run.smithi018.stderr: stderr: 10+0 records in 2023-09-23T20:09:02.656 INFO:teuthology.orchestra.run.smithi018.stderr:10+0 records out 2023-09-23T20:09:02.656 INFO:teuthology.orchestra.run.smithi018.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.00833341 s, 1.3 GB/s 2023-09-23T20:09:02.657 INFO:teuthology.orchestra.run.smithi018.stderr:--> Zapping successful for: 2023-09-23T20:09:02.867 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:02 smithi018 ceph-mon[109137]: from='osd.0 [v2:172.21.15.18:6802/2209550685,v1:172.21.15.18:6803/2209550685]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi018", "root=default"]}]': finished 2023-09-23T20:09:02.868 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:02 smithi018 ceph-mon[109137]: osdmap e7: 1 total, 0 up, 1 in 2023-09-23T20:09:02.868 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:02 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T20:09:02.868 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:02 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T20:09:02.868 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:02 smithi018 ceph-mon[95889]: from='osd.0 [v2:172.21.15.18:6802/2209550685,v1:172.21.15.18:6803/2209550685]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi018", "root=default"]}]': finished 2023-09-23T20:09:02.868 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:02 smithi018 ceph-mon[95889]: osdmap e7: 1 total, 0 up, 1 in 2023-09-23T20:09:02.868 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:02 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T20:09:02.869 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:02 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T20:09:03.182 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph orch daemon add osd smithi018:vg_nvme/lv_3 2023-09-23T20:09:03.215 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:02 smithi149 ceph-mon[100764]: from='osd.0 [v2:172.21.15.18:6802/2209550685,v1:172.21.15.18:6803/2209550685]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi018", "root=default"]}]': finished 2023-09-23T20:09:03.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:02 smithi149 ceph-mon[100764]: osdmap e7: 1 total, 0 up, 1 in 2023-09-23T20:09:03.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:02 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T20:09:03.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:02 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T20:09:03.786 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:09:04.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:03 smithi018 ceph-mon[95889]: purged_snaps scrub starts 2023-09-23T20:09:04.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:03 smithi018 ceph-mon[95889]: purged_snaps scrub ok 2023-09-23T20:09:04.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:03 smithi018 ceph-mon[95889]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:09:04.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:03 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T20:09:04.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:03 smithi018 ceph-mon[95889]: osd.0 [v2:172.21.15.18:6802/2209550685,v1:172.21.15.18:6803/2209550685] boot 2023-09-23T20:09:04.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:03 smithi018 ceph-mon[95889]: osdmap e8: 1 total, 1 up, 1 in 2023-09-23T20:09:04.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:03 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T20:09:04.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:03 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:09:04.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:03 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:09:04.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:03 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:09:04.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:03 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:09:04.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:03 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:09:04.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:03 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:09:04.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:03 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:09:04.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:03 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:09:04.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:03 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:09:04.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:03 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:09:04.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:03 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:09:04.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:03 smithi018 ceph-mon[109137]: purged_snaps scrub starts 2023-09-23T20:09:04.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:03 smithi018 ceph-mon[109137]: purged_snaps scrub ok 2023-09-23T20:09:04.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:03 smithi018 ceph-mon[109137]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:09:04.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:03 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T20:09:04.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:03 smithi018 ceph-mon[109137]: osd.0 [v2:172.21.15.18:6802/2209550685,v1:172.21.15.18:6803/2209550685] boot 2023-09-23T20:09:04.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:03 smithi018 ceph-mon[109137]: osdmap e8: 1 total, 1 up, 1 in 2023-09-23T20:09:04.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:03 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T20:09:04.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:03 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:09:04.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:03 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:09:04.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:03 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:09:04.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:03 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:09:04.144 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:03 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:09:04.144 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:03 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:09:04.144 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:03 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:09:04.144 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:03 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:09:04.144 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:03 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:09:04.144 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:03 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:09:04.144 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:03 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:09:04.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:03 smithi149 ceph-mon[100764]: purged_snaps scrub starts 2023-09-23T20:09:04.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:03 smithi149 ceph-mon[100764]: purged_snaps scrub ok 2023-09-23T20:09:04.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:03 smithi149 ceph-mon[100764]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T20:09:04.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:03 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T20:09:04.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:03 smithi149 ceph-mon[100764]: osd.0 [v2:172.21.15.18:6802/2209550685,v1:172.21.15.18:6803/2209550685] boot 2023-09-23T20:09:04.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:03 smithi149 ceph-mon[100764]: osdmap e8: 1 total, 1 up, 1 in 2023-09-23T20:09:04.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:03 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T20:09:04.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:03 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:09:04.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:03 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:09:04.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:03 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:09:04.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:03 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:09:04.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:03 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:09:04.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:03 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:09:04.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:03 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:09:04.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:03 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:09:04.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:03 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:09:04.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:03 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:09:04.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:03 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:09:05.140 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:04 smithi018 ceph-mon[95889]: Detected new or changed devices on smithi018 2023-09-23T20:09:05.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:04 smithi018 ceph-mon[95889]: Adjusting osd_memory_target on smithi018 to 16149M 2023-09-23T20:09:05.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:04 smithi018 ceph-mon[95889]: osdmap e9: 1 total, 1 up, 1 in 2023-09-23T20:09:05.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:04 smithi018 ceph-mon[109137]: Detected new or changed devices on smithi018 2023-09-23T20:09:05.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:04 smithi018 ceph-mon[109137]: Adjusting osd_memory_target on smithi018 to 16149M 2023-09-23T20:09:05.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:04 smithi018 ceph-mon[109137]: osdmap e9: 1 total, 1 up, 1 in 2023-09-23T20:09:05.215 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:04 smithi149 ceph-mon[100764]: Detected new or changed devices on smithi018 2023-09-23T20:09:05.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:04 smithi149 ceph-mon[100764]: Adjusting osd_memory_target on smithi018 to 16149M 2023-09-23T20:09:05.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:04 smithi149 ceph-mon[100764]: osdmap e9: 1 total, 1 up, 1 in 2023-09-23T20:09:06.139 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:05 smithi018 ceph-mon[109137]: pgmap v37: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T20:09:06.139 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:05 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T20:09:06.139 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:05 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T20:09:06.140 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:05 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:09:06.140 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:05 smithi018 ceph-mon[95889]: pgmap v37: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T20:09:06.140 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:05 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T20:09:06.140 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:05 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T20:09:06.140 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:05 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:09:06.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:05 smithi149 ceph-mon[100764]: pgmap v37: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T20:09:06.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:05 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T20:09:06.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:05 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T20:09:06.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:05 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:09:06.726 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:06 smithi018 ceph-mon[95889]: from='client.14244 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi018:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:09:06.983 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:06 smithi018 ceph-mon[109137]: from='client.14244 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi018:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:09:07.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:06 smithi149 ceph-mon[100764]: from='client.14244 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi018:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:09:08.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:07 smithi018 ceph-mon[95889]: pgmap v38: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T20:09:08.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:07 smithi018 ceph-mon[95889]: from='client.? 172.21.15.18:0/3046633832' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c0534324-49c8-4804-bced-c4d13510e74a"}]: dispatch 2023-09-23T20:09:08.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:07 smithi018 ceph-mon[95889]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c0534324-49c8-4804-bced-c4d13510e74a"}]: dispatch 2023-09-23T20:09:08.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:07 smithi018 ceph-mon[95889]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "c0534324-49c8-4804-bced-c4d13510e74a"}]': finished 2023-09-23T20:09:08.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:07 smithi018 ceph-mon[95889]: osdmap e10: 2 total, 1 up, 2 in 2023-09-23T20:09:08.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:07 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T20:09:08.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:07 smithi018 ceph-mon[109137]: pgmap v38: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T20:09:08.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:07 smithi018 ceph-mon[109137]: from='client.? 172.21.15.18:0/3046633832' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c0534324-49c8-4804-bced-c4d13510e74a"}]: dispatch 2023-09-23T20:09:08.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:07 smithi018 ceph-mon[109137]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c0534324-49c8-4804-bced-c4d13510e74a"}]: dispatch 2023-09-23T20:09:08.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:07 smithi018 ceph-mon[109137]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "c0534324-49c8-4804-bced-c4d13510e74a"}]': finished 2023-09-23T20:09:08.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:07 smithi018 ceph-mon[109137]: osdmap e10: 2 total, 1 up, 2 in 2023-09-23T20:09:08.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:07 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T20:09:08.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:07 smithi149 ceph-mon[100764]: pgmap v38: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T20:09:08.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:07 smithi149 ceph-mon[100764]: from='client.? 172.21.15.18:0/3046633832' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c0534324-49c8-4804-bced-c4d13510e74a"}]: dispatch 2023-09-23T20:09:08.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:07 smithi149 ceph-mon[100764]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c0534324-49c8-4804-bced-c4d13510e74a"}]: dispatch 2023-09-23T20:09:08.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:07 smithi149 ceph-mon[100764]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "c0534324-49c8-4804-bced-c4d13510e74a"}]': finished 2023-09-23T20:09:08.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:07 smithi149 ceph-mon[100764]: osdmap e10: 2 total, 1 up, 2 in 2023-09-23T20:09:08.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:07 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T20:09:09.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:08 smithi018 ceph-mon[95889]: from='client.? 172.21.15.18:0/38876017' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T20:09:09.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:08 smithi018 ceph-mon[109137]: from='client.? 172.21.15.18:0/38876017' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T20:09:09.215 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:08 smithi149 ceph-mon[100764]: from='client.? 172.21.15.18:0/38876017' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T20:09:10.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:09 smithi018 ceph-mon[95889]: pgmap v40: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T20:09:10.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:09 smithi018 ceph-mon[109137]: pgmap v40: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T20:09:10.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:09 smithi149 ceph-mon[100764]: pgmap v40: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T20:09:12.024 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:11 smithi018 ceph-mon[95889]: pgmap v41: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T20:09:12.025 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:11 smithi018 ceph-mon[109137]: pgmap v41: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T20:09:12.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:11 smithi149 ceph-mon[100764]: pgmap v41: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T20:09:14.095 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:13 smithi018 ceph-mon[109137]: pgmap v42: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T20:09:14.096 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:13 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-09-23T20:09:14.096 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:13 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:09:14.096 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:13 smithi018 ceph-mon[95889]: pgmap v42: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T20:09:14.096 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:13 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-09-23T20:09:14.096 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:13 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:09:14.215 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:13 smithi149 ceph-mon[100764]: pgmap v42: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T20:09:14.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:13 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-09-23T20:09:14.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:13 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:09:15.112 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:14 smithi018 ceph-mon[95889]: Deploying daemon osd.1 on smithi018 2023-09-23T20:09:15.112 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:14 smithi018 ceph-mon[109137]: Deploying daemon osd.1 on smithi018 2023-09-23T20:09:15.215 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:14 smithi149 ceph-mon[100764]: Deploying daemon osd.1 on smithi018 2023-09-23T20:09:16.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:15 smithi018 ceph-mon[95889]: pgmap v43: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T20:09:16.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:15 smithi018 ceph-mon[109137]: pgmap v43: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T20:09:16.215 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:15 smithi149 ceph-mon[100764]: pgmap v43: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T20:09:17.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:17 smithi018 ceph-mon[95889]: pgmap v44: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T20:09:17.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:17 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:09:17.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:17 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:09:17.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:17 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:09:17.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:17 smithi018 ceph-mon[109137]: pgmap v44: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T20:09:17.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:17 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:09:17.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:17 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:09:17.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:17 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:09:18.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:17 smithi149 ceph-mon[100764]: pgmap v44: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T20:09:18.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:17 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:09:18.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:17 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:09:18.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:17 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:09:19.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:19 smithi018 ceph-mon[109137]: pgmap v45: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T20:09:19.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:19 smithi018 ceph-mon[95889]: pgmap v45: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T20:09:20.215 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:19 smithi149 ceph-mon[100764]: pgmap v45: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T20:09:20.431 INFO:teuthology.orchestra.run.smithi018.stdout:Created osd(s) 1 on host 'smithi018' 2023-09-23T20:09:20.871 DEBUG:teuthology.orchestra.run.smithi018:osd.1> sudo journalctl -f -n 0 -u ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7@osd.1.service 2023-09-23T20:09:20.873 INFO:tasks.cephadm:Deploying osd.2 on smithi018 with /dev/vg_nvme/lv_2... 2023-09-23T20:09:20.873 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2023-09-23T20:09:20.902 INFO:journalctl@ceph.osd.1.smithi018.stdout:-- Logs begin at Sat 2023-09-23 19:50:10 UTC. -- 2023-09-23T20:09:21.670 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:21 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:09:21.671 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:21 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:09:21.671 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:21 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:09:21.671 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:21 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:09:21.671 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:21 smithi018 ceph-mon[95889]: pgmap v46: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T20:09:21.671 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:21 smithi018 ceph-mon[95889]: from='osd.1 [v2:172.21.15.18:6810/314354451,v1:172.21.15.18:6811/314354451]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T20:09:21.671 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:21 smithi018 ceph-mon[95889]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T20:09:21.671 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:21 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:09:21.671 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:21 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:09:21.672 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:21 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:09:21.672 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:21 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:09:21.672 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:21 smithi018 ceph-mon[109137]: pgmap v46: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T20:09:21.672 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:21 smithi018 ceph-mon[109137]: from='osd.1 [v2:172.21.15.18:6810/314354451,v1:172.21.15.18:6811/314354451]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T20:09:21.672 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:21 smithi018 ceph-mon[109137]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T20:09:21.673 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:09:21.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:21 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:09:21.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:21 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:09:21.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:21 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:09:21.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:21 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:09:21.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:21 smithi149 ceph-mon[100764]: pgmap v46: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T20:09:21.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:21 smithi149 ceph-mon[100764]: from='osd.1 [v2:172.21.15.18:6810/314354451,v1:172.21.15.18:6811/314354451]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T20:09:21.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:21 smithi149 ceph-mon[100764]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T20:09:22.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:22 smithi018 ceph-mon[95889]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-09-23T20:09:22.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:22 smithi018 ceph-mon[95889]: osdmap e11: 2 total, 1 up, 2 in 2023-09-23T20:09:22.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:22 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T20:09:22.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:22 smithi018 ceph-mon[95889]: from='osd.1 [v2:172.21.15.18:6810/314354451,v1:172.21.15.18:6811/314354451]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi018", "root=default"]}]: dispatch 2023-09-23T20:09:22.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:22 smithi018 ceph-mon[95889]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi018", "root=default"]}]: dispatch 2023-09-23T20:09:22.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:22 smithi018 ceph-mon[109137]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-09-23T20:09:22.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:22 smithi018 ceph-mon[109137]: osdmap e11: 2 total, 1 up, 2 in 2023-09-23T20:09:22.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:22 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T20:09:22.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:22 smithi018 ceph-mon[109137]: from='osd.1 [v2:172.21.15.18:6810/314354451,v1:172.21.15.18:6811/314354451]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi018", "root=default"]}]: dispatch 2023-09-23T20:09:22.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:22 smithi018 ceph-mon[109137]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi018", "root=default"]}]: dispatch 2023-09-23T20:09:22.642 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 23 20:09:22 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-1[121464]: 2023-09-23T20:09:22.618+0000 7f63cb589700 -1 osd.1 0 waiting for initial osdmap 2023-09-23T20:09:22.642 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 23 20:09:22 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-1[121464]: 2023-09-23T20:09:22.620+0000 7f63c4380700 -1 osd.1 12 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T20:09:22.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:22 smithi149 ceph-mon[100764]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-09-23T20:09:22.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:22 smithi149 ceph-mon[100764]: osdmap e11: 2 total, 1 up, 2 in 2023-09-23T20:09:22.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:22 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T20:09:22.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:22 smithi149 ceph-mon[100764]: from='osd.1 [v2:172.21.15.18:6810/314354451,v1:172.21.15.18:6811/314354451]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi018", "root=default"]}]: dispatch 2023-09-23T20:09:22.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:22 smithi149 ceph-mon[100764]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi018", "root=default"]}]: dispatch 2023-09-23T20:09:23.571 INFO:teuthology.orchestra.run.smithi018.stderr:--> Zapping: /dev/vg_nvme/lv_2 2023-09-23T20:09:23.571 INFO:teuthology.orchestra.run.smithi018.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2023-09-23T20:09:23.571 INFO:teuthology.orchestra.run.smithi018.stderr: stderr: 10+0 records in 2023-09-23T20:09:23.571 INFO:teuthology.orchestra.run.smithi018.stderr:10+0 records out 2023-09-23T20:09:23.571 INFO:teuthology.orchestra.run.smithi018.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.00796766 s, 1.3 GB/s 2023-09-23T20:09:23.572 INFO:teuthology.orchestra.run.smithi018.stderr:--> Zapping successful for: 2023-09-23T20:09:23.823 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:23 smithi018 ceph-mon[95889]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi018", "root=default"]}]': finished 2023-09-23T20:09:23.823 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:23 smithi018 ceph-mon[95889]: osdmap e12: 2 total, 1 up, 2 in 2023-09-23T20:09:23.823 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:23 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T20:09:23.823 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:23 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T20:09:23.823 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:23 smithi018 ceph-mon[95889]: pgmap v49: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T20:09:23.824 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:23 smithi018 ceph-mon[109137]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi018", "root=default"]}]': finished 2023-09-23T20:09:23.824 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:23 smithi018 ceph-mon[109137]: osdmap e12: 2 total, 1 up, 2 in 2023-09-23T20:09:23.824 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:23 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T20:09:23.824 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:23 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T20:09:23.824 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:23 smithi018 ceph-mon[109137]: pgmap v49: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T20:09:23.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:23 smithi149 ceph-mon[100764]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi018", "root=default"]}]': finished 2023-09-23T20:09:23.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:23 smithi149 ceph-mon[100764]: osdmap e12: 2 total, 1 up, 2 in 2023-09-23T20:09:23.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:23 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T20:09:23.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:23 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T20:09:23.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:23 smithi149 ceph-mon[100764]: pgmap v49: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T20:09:24.137 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph orch daemon add osd smithi018:vg_nvme/lv_2 2023-09-23T20:09:24.746 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:09:24.890 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:24 smithi018 ceph-mon[109137]: purged_snaps scrub starts 2023-09-23T20:09:24.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:24 smithi018 ceph-mon[109137]: purged_snaps scrub ok 2023-09-23T20:09:24.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:24 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T20:09:24.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:24 smithi018 ceph-mon[109137]: osd.1 [v2:172.21.15.18:6810/314354451,v1:172.21.15.18:6811/314354451] boot 2023-09-23T20:09:24.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:24 smithi018 ceph-mon[109137]: osdmap e13: 2 total, 2 up, 2 in 2023-09-23T20:09:24.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:24 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T20:09:24.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:24 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:09:24.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:24 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:09:24.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:24 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:09:24.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:24 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:09:24.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:24 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:09:24.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:24 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:09:24.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:24 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:09:24.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:24 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:09:24.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:24 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:09:24.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:24 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:09:24.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:24 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:09:24.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:24 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:09:24.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:24 smithi018 ceph-mon[95889]: purged_snaps scrub starts 2023-09-23T20:09:24.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:24 smithi018 ceph-mon[95889]: purged_snaps scrub ok 2023-09-23T20:09:24.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:24 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T20:09:24.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:24 smithi018 ceph-mon[95889]: osd.1 [v2:172.21.15.18:6810/314354451,v1:172.21.15.18:6811/314354451] boot 2023-09-23T20:09:24.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:24 smithi018 ceph-mon[95889]: osdmap e13: 2 total, 2 up, 2 in 2023-09-23T20:09:24.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:24 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T20:09:24.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:24 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:09:24.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:24 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:09:24.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:24 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:09:24.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:24 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:09:24.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:24 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:09:24.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:24 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:09:24.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:24 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:09:24.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:24 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:09:24.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:24 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:09:24.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:24 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:09:24.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:24 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:09:24.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:24 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:09:24.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:24 smithi149 ceph-mon[100764]: purged_snaps scrub starts 2023-09-23T20:09:24.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:24 smithi149 ceph-mon[100764]: purged_snaps scrub ok 2023-09-23T20:09:24.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:24 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T20:09:24.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:24 smithi149 ceph-mon[100764]: osd.1 [v2:172.21.15.18:6810/314354451,v1:172.21.15.18:6811/314354451] boot 2023-09-23T20:09:24.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:24 smithi149 ceph-mon[100764]: osdmap e13: 2 total, 2 up, 2 in 2023-09-23T20:09:24.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:24 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T20:09:24.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:24 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:09:24.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:24 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:09:24.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:24 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:09:24.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:24 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:09:24.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:24 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:09:24.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:24 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:09:24.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:24 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:09:24.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:24 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:09:24.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:24 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:09:24.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:24 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:09:24.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:24 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:09:24.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:24 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:09:25.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:25 smithi018 ceph-mon[95889]: Detected new or changed devices on smithi018 2023-09-23T20:09:25.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:25 smithi018 ceph-mon[95889]: Adjusting osd_memory_target on smithi018 to 8074M 2023-09-23T20:09:25.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:25 smithi018 ceph-mon[95889]: osdmap e14: 2 total, 2 up, 2 in 2023-09-23T20:09:25.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:25 smithi018 ceph-mon[95889]: pgmap v52: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-09-23T20:09:25.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:25 smithi018 ceph-mon[109137]: Detected new or changed devices on smithi018 2023-09-23T20:09:25.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:25 smithi018 ceph-mon[109137]: Adjusting osd_memory_target on smithi018 to 8074M 2023-09-23T20:09:25.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:25 smithi018 ceph-mon[109137]: osdmap e14: 2 total, 2 up, 2 in 2023-09-23T20:09:25.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:25 smithi018 ceph-mon[109137]: pgmap v52: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-09-23T20:09:25.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:25 smithi149 ceph-mon[100764]: Detected new or changed devices on smithi018 2023-09-23T20:09:25.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:25 smithi149 ceph-mon[100764]: Adjusting osd_memory_target on smithi018 to 8074M 2023-09-23T20:09:25.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:25 smithi149 ceph-mon[100764]: osdmap e14: 2 total, 2 up, 2 in 2023-09-23T20:09:25.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:25 smithi149 ceph-mon[100764]: pgmap v52: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-09-23T20:09:26.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:26 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T20:09:26.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:26 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T20:09:26.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:26 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:09:26.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:26 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T20:09:26.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:26 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T20:09:26.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:26 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:09:26.965 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:26 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T20:09:26.965 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:26 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T20:09:26.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:26 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:09:27.830 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:27 smithi018 ceph-mon[95889]: from='client.24170 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi018:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:09:27.831 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:27 smithi018 ceph-mon[95889]: pgmap v53: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-09-23T20:09:27.831 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:27 smithi018 ceph-mon[109137]: from='client.24170 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi018:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:09:27.831 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:27 smithi018 ceph-mon[109137]: pgmap v53: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-09-23T20:09:27.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:27 smithi149 ceph-mon[100764]: from='client.24170 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi018:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:09:27.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:27 smithi149 ceph-mon[100764]: pgmap v53: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-09-23T20:09:28.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:28 smithi018 ceph-mon[95889]: from='client.? 172.21.15.18:0/514882446' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f3b2116d-a3d5-4279-91dc-eb47458d3239"}]: dispatch 2023-09-23T20:09:28.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:28 smithi018 ceph-mon[95889]: from='client.? 172.21.15.18:0/514882446' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f3b2116d-a3d5-4279-91dc-eb47458d3239"}]': finished 2023-09-23T20:09:28.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:28 smithi018 ceph-mon[95889]: osdmap e15: 3 total, 2 up, 3 in 2023-09-23T20:09:28.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:28 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T20:09:28.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:28 smithi018 ceph-mon[109137]: from='client.? 172.21.15.18:0/514882446' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f3b2116d-a3d5-4279-91dc-eb47458d3239"}]: dispatch 2023-09-23T20:09:28.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:28 smithi018 ceph-mon[109137]: from='client.? 172.21.15.18:0/514882446' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f3b2116d-a3d5-4279-91dc-eb47458d3239"}]': finished 2023-09-23T20:09:28.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:28 smithi018 ceph-mon[109137]: osdmap e15: 3 total, 2 up, 3 in 2023-09-23T20:09:28.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:28 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T20:09:28.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:28 smithi149 ceph-mon[100764]: from='client.? 172.21.15.18:0/514882446' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f3b2116d-a3d5-4279-91dc-eb47458d3239"}]: dispatch 2023-09-23T20:09:28.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:28 smithi149 ceph-mon[100764]: from='client.? 172.21.15.18:0/514882446' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f3b2116d-a3d5-4279-91dc-eb47458d3239"}]': finished 2023-09-23T20:09:28.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:28 smithi149 ceph-mon[100764]: osdmap e15: 3 total, 2 up, 3 in 2023-09-23T20:09:28.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:28 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T20:09:29.890 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:29 smithi018 ceph-mon[95889]: pgmap v55: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-09-23T20:09:29.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:29 smithi018 ceph-mon[95889]: from='client.? 172.21.15.18:0/1926963138' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T20:09:29.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:29 smithi018 ceph-mon[109137]: pgmap v55: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-09-23T20:09:29.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:29 smithi018 ceph-mon[109137]: from='client.? 172.21.15.18:0/1926963138' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T20:09:29.965 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:29 smithi149 ceph-mon[100764]: pgmap v55: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-09-23T20:09:29.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:29 smithi149 ceph-mon[100764]: from='client.? 172.21.15.18:0/1926963138' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T20:09:31.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:31 smithi018 ceph-mon[95889]: pgmap v56: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-09-23T20:09:31.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:31 smithi018 ceph-mon[109137]: pgmap v56: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-09-23T20:09:31.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:31 smithi149 ceph-mon[100764]: pgmap v56: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-09-23T20:09:33.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:33 smithi018 ceph-mon[95889]: pgmap v57: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-09-23T20:09:33.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:33 smithi018 ceph-mon[109137]: pgmap v57: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-09-23T20:09:33.965 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:33 smithi149 ceph-mon[100764]: pgmap v57: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-09-23T20:09:35.072 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:34 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-09-23T20:09:35.072 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:34 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:09:35.072 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:34 smithi018 ceph-mon[95889]: Deploying daemon osd.2 on smithi018 2023-09-23T20:09:35.073 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:34 smithi018 ceph-mon[95889]: pgmap v58: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T20:09:35.073 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:34 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-09-23T20:09:35.073 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:34 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:09:35.073 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:34 smithi018 ceph-mon[109137]: Deploying daemon osd.2 on smithi018 2023-09-23T20:09:35.073 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:34 smithi018 ceph-mon[109137]: pgmap v58: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T20:09:35.215 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:34 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-09-23T20:09:35.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:34 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:09:35.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:34 smithi149 ceph-mon[100764]: Deploying daemon osd.2 on smithi018 2023-09-23T20:09:35.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:34 smithi149 ceph-mon[100764]: pgmap v58: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T20:09:37.969 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:37 smithi149 ceph-mon[100764]: pgmap v59: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T20:09:38.015 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:37 smithi018 ceph-mon[95889]: pgmap v59: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T20:09:38.015 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:37 smithi018 ceph-mon[109137]: pgmap v59: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T20:09:39.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:39 smithi149 ceph-mon[100764]: pgmap v60: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T20:09:39.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:39 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:09:39.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:39 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:09:39.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:39 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:09:40.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:39 smithi018 ceph-mon[109137]: pgmap v60: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T20:09:40.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:39 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:09:40.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:39 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:09:40.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:39 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:09:40.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:39 smithi018 ceph-mon[95889]: pgmap v60: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T20:09:40.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:39 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:09:40.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:39 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:09:40.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:39 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:09:41.465 INFO:teuthology.orchestra.run.smithi018.stdout:Created osd(s) 2 on host 'smithi018' 2023-09-23T20:09:41.858 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:41 smithi018 ceph-mon[109137]: pgmap v61: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T20:09:41.858 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:41 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:09:41.858 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:41 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:09:41.858 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:41 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:09:41.858 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:41 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:09:41.858 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:41 smithi018 ceph-mon[95889]: pgmap v61: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T20:09:41.858 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:41 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:09:41.859 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:41 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:09:41.859 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:41 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:09:41.859 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:41 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:09:41.950 DEBUG:teuthology.orchestra.run.smithi018:osd.2> sudo journalctl -f -n 0 -u ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7@osd.2.service 2023-09-23T20:09:41.952 INFO:tasks.cephadm:Deploying osd.3 on smithi018 with /dev/vg_nvme/lv_1... 2023-09-23T20:09:41.952 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2023-09-23T20:09:41.965 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:41 smithi149 ceph-mon[100764]: pgmap v61: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T20:09:41.965 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:41 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:09:41.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:41 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:09:41.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:41 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:09:41.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:41 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:09:41.984 INFO:journalctl@ceph.osd.2.smithi018.stdout:-- Logs begin at Sat 2023-09-23 19:50:10 UTC. -- 2023-09-23T20:09:42.140 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 23 20:09:41 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-2[129269]: 2023-09-23T20:09:41.986+0000 7f668a10b540 -1 osd.2 0 log_to_monitors true 2023-09-23T20:09:42.745 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:09:42.827 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:42 smithi018 ceph-mon[95889]: from='osd.2 [v2:172.21.15.18:6818/1413681698,v1:172.21.15.18:6819/1413681698]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T20:09:42.827 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:42 smithi018 ceph-mon[95889]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T20:09:42.828 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:42 smithi018 ceph-mon[109137]: from='osd.2 [v2:172.21.15.18:6818/1413681698,v1:172.21.15.18:6819/1413681698]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T20:09:42.828 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:42 smithi018 ceph-mon[109137]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T20:09:43.215 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:42 smithi149 ceph-mon[100764]: from='osd.2 [v2:172.21.15.18:6818/1413681698,v1:172.21.15.18:6819/1413681698]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T20:09:43.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:42 smithi149 ceph-mon[100764]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T20:09:43.831 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 23 20:09:43 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-2[129269]: 2023-09-23T20:09:43.723+0000 7f66808f1700 -1 osd.2 0 waiting for initial osdmap 2023-09-23T20:09:43.831 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 23 20:09:43 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-2[129269]: 2023-09-23T20:09:43.726+0000 7f667d6f0700 -1 osd.2 17 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T20:09:43.832 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:43 smithi018 ceph-mon[109137]: pgmap v62: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T20:09:43.832 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:43 smithi018 ceph-mon[109137]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-09-23T20:09:43.832 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:43 smithi018 ceph-mon[109137]: osdmap e16: 3 total, 2 up, 3 in 2023-09-23T20:09:43.832 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:43 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T20:09:43.832 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:43 smithi018 ceph-mon[109137]: from='osd.2 [v2:172.21.15.18:6818/1413681698,v1:172.21.15.18:6819/1413681698]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi018", "root=default"]}]: dispatch 2023-09-23T20:09:43.832 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:43 smithi018 ceph-mon[109137]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi018", "root=default"]}]: dispatch 2023-09-23T20:09:43.832 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:43 smithi018 ceph-mon[95889]: pgmap v62: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T20:09:43.833 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:43 smithi018 ceph-mon[95889]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-09-23T20:09:43.833 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:43 smithi018 ceph-mon[95889]: osdmap e16: 3 total, 2 up, 3 in 2023-09-23T20:09:43.833 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:43 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T20:09:43.833 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:43 smithi018 ceph-mon[95889]: from='osd.2 [v2:172.21.15.18:6818/1413681698,v1:172.21.15.18:6819/1413681698]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi018", "root=default"]}]: dispatch 2023-09-23T20:09:43.833 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:43 smithi018 ceph-mon[95889]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi018", "root=default"]}]: dispatch 2023-09-23T20:09:44.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:43 smithi149 ceph-mon[100764]: pgmap v62: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T20:09:44.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:43 smithi149 ceph-mon[100764]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-09-23T20:09:44.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:43 smithi149 ceph-mon[100764]: osdmap e16: 3 total, 2 up, 3 in 2023-09-23T20:09:44.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:43 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T20:09:44.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:43 smithi149 ceph-mon[100764]: from='osd.2 [v2:172.21.15.18:6818/1413681698,v1:172.21.15.18:6819/1413681698]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi018", "root=default"]}]: dispatch 2023-09-23T20:09:44.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:43 smithi149 ceph-mon[100764]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi018", "root=default"]}]: dispatch 2023-09-23T20:09:44.725 INFO:teuthology.orchestra.run.smithi018.stderr:--> Zapping: /dev/vg_nvme/lv_1 2023-09-23T20:09:44.725 INFO:teuthology.orchestra.run.smithi018.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2023-09-23T20:09:44.725 INFO:teuthology.orchestra.run.smithi018.stderr: stderr: 10+0 records in 2023-09-23T20:09:44.726 INFO:teuthology.orchestra.run.smithi018.stderr:10+0 records out 2023-09-23T20:09:44.726 INFO:teuthology.orchestra.run.smithi018.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.00835195 s, 1.3 GB/s 2023-09-23T20:09:44.726 INFO:teuthology.orchestra.run.smithi018.stderr:--> Zapping successful for: 2023-09-23T20:09:44.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:44 smithi018 ceph-mon[95889]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi018", "root=default"]}]': finished 2023-09-23T20:09:44.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:44 smithi018 ceph-mon[95889]: osdmap e17: 3 total, 2 up, 3 in 2023-09-23T20:09:44.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:44 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T20:09:44.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:44 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T20:09:44.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:44 smithi018 ceph-mon[109137]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi018", "root=default"]}]': finished 2023-09-23T20:09:44.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:44 smithi018 ceph-mon[109137]: osdmap e17: 3 total, 2 up, 3 in 2023-09-23T20:09:44.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:44 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T20:09:44.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:44 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T20:09:45.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:44 smithi149 ceph-mon[100764]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi018", "root=default"]}]': finished 2023-09-23T20:09:45.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:44 smithi149 ceph-mon[100764]: osdmap e17: 3 total, 2 up, 3 in 2023-09-23T20:09:45.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:44 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T20:09:45.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:44 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T20:09:45.320 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph orch daemon add osd smithi018:vg_nvme/lv_1 2023-09-23T20:09:45.932 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:09:46.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:45 smithi018 ceph-mon[95889]: purged_snaps scrub starts 2023-09-23T20:09:46.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:45 smithi018 ceph-mon[95889]: purged_snaps scrub ok 2023-09-23T20:09:46.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:45 smithi018 ceph-mon[95889]: pgmap v65: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T20:09:46.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:45 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T20:09:46.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:45 smithi018 ceph-mon[95889]: osd.2 [v2:172.21.15.18:6818/1413681698,v1:172.21.15.18:6819/1413681698] boot 2023-09-23T20:09:46.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:45 smithi018 ceph-mon[95889]: osdmap e18: 3 total, 3 up, 3 in 2023-09-23T20:09:46.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:45 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T20:09:46.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:45 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:09:46.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:45 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:09:46.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:45 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:09:46.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:45 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:09:46.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:45 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:09:46.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:45 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:09:46.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:45 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:09:46.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:45 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:09:46.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:45 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:09:46.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:45 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:09:46.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:45 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:09:46.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:45 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:09:46.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:45 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:09:46.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:45 smithi018 ceph-mon[109137]: purged_snaps scrub starts 2023-09-23T20:09:46.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:45 smithi018 ceph-mon[109137]: purged_snaps scrub ok 2023-09-23T20:09:46.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:45 smithi018 ceph-mon[109137]: pgmap v65: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T20:09:46.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:45 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T20:09:46.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:45 smithi018 ceph-mon[109137]: osd.2 [v2:172.21.15.18:6818/1413681698,v1:172.21.15.18:6819/1413681698] boot 2023-09-23T20:09:46.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:45 smithi018 ceph-mon[109137]: osdmap e18: 3 total, 3 up, 3 in 2023-09-23T20:09:46.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:45 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T20:09:46.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:45 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:09:46.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:45 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:09:46.144 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:45 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:09:46.144 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:45 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:09:46.144 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:45 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:09:46.144 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:45 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:09:46.144 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:45 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:09:46.144 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:45 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:09:46.144 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:45 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:09:46.144 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:45 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:09:46.144 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:45 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:09:46.144 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:45 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:09:46.144 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:45 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:09:46.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:45 smithi149 ceph-mon[100764]: purged_snaps scrub starts 2023-09-23T20:09:46.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:45 smithi149 ceph-mon[100764]: purged_snaps scrub ok 2023-09-23T20:09:46.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:45 smithi149 ceph-mon[100764]: pgmap v65: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T20:09:46.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:45 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T20:09:46.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:45 smithi149 ceph-mon[100764]: osd.2 [v2:172.21.15.18:6818/1413681698,v1:172.21.15.18:6819/1413681698] boot 2023-09-23T20:09:46.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:45 smithi149 ceph-mon[100764]: osdmap e18: 3 total, 3 up, 3 in 2023-09-23T20:09:46.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:45 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T20:09:46.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:45 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:09:46.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:45 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:09:46.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:45 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:09:46.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:45 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:09:46.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:45 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:09:46.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:45 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:09:46.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:45 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:09:46.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:45 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:09:46.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:45 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:09:46.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:45 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:09:46.218 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:45 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:09:46.218 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:45 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:09:46.218 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:45 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:09:47.465 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:47 smithi149 ceph-mon[100764]: Detected new or changed devices on smithi018 2023-09-23T20:09:47.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:47 smithi149 ceph-mon[100764]: Adjusting osd_memory_target on smithi018 to 5383M 2023-09-23T20:09:47.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:47 smithi149 ceph-mon[100764]: osdmap e19: 3 total, 3 up, 3 in 2023-09-23T20:09:47.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:47 smithi018 ceph-mon[95889]: Detected new or changed devices on smithi018 2023-09-23T20:09:47.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:47 smithi018 ceph-mon[95889]: Adjusting osd_memory_target on smithi018 to 5383M 2023-09-23T20:09:47.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:47 smithi018 ceph-mon[95889]: osdmap e19: 3 total, 3 up, 3 in 2023-09-23T20:09:47.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:47 smithi018 ceph-mon[109137]: Detected new or changed devices on smithi018 2023-09-23T20:09:47.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:47 smithi018 ceph-mon[109137]: Adjusting osd_memory_target on smithi018 to 5383M 2023-09-23T20:09:47.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:47 smithi018 ceph-mon[109137]: osdmap e19: 3 total, 3 up, 3 in 2023-09-23T20:09:48.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:48 smithi149 ceph-mon[100764]: pgmap v68: 0 pgs: ; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-09-23T20:09:48.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:48 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-09-23T20:09:48.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:48 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2023-09-23T20:09:48.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:48 smithi149 ceph-mon[100764]: osdmap e20: 3 total, 3 up, 3 in 2023-09-23T20:09:48.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:48 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T20:09:48.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:48 smithi149 ceph-mon[100764]: from='client.24197 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi018:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:09:48.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:48 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T20:09:48.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:48 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T20:09:48.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:48 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:09:48.486 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:48 smithi018 ceph-mon[109137]: pgmap v68: 0 pgs: ; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-09-23T20:09:48.486 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:48 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-09-23T20:09:48.486 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:48 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2023-09-23T20:09:48.486 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:48 smithi018 ceph-mon[109137]: osdmap e20: 3 total, 3 up, 3 in 2023-09-23T20:09:48.486 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:48 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T20:09:48.486 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:48 smithi018 ceph-mon[109137]: from='client.24197 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi018:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:09:48.486 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:48 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T20:09:48.486 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:48 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T20:09:48.487 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:48 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:09:48.487 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:48 smithi018 ceph-mon[95889]: pgmap v68: 0 pgs: ; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-09-23T20:09:48.487 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:48 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-09-23T20:09:48.487 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:48 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2023-09-23T20:09:48.487 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:48 smithi018 ceph-mon[95889]: osdmap e20: 3 total, 3 up, 3 in 2023-09-23T20:09:48.487 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:48 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T20:09:48.487 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:48 smithi018 ceph-mon[95889]: from='client.24197 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi018:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:09:48.487 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:48 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T20:09:48.488 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:48 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T20:09:48.488 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:48 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:09:49.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:49 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2023-09-23T20:09:49.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:49 smithi018 ceph-mon[95889]: osdmap e21: 3 total, 3 up, 3 in 2023-09-23T20:09:49.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:49 smithi018 ceph-mon[95889]: pgmap v71: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-09-23T20:09:49.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:49 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2023-09-23T20:09:49.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:49 smithi018 ceph-mon[109137]: osdmap e21: 3 total, 3 up, 3 in 2023-09-23T20:09:49.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:49 smithi018 ceph-mon[109137]: pgmap v71: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-09-23T20:09:49.715 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:49 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2023-09-23T20:09:49.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:49 smithi149 ceph-mon[100764]: osdmap e21: 3 total, 3 up, 3 in 2023-09-23T20:09:49.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:49 smithi149 ceph-mon[100764]: pgmap v71: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-09-23T20:09:49.890 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 23 20:09:49 smithi018 sudo[134499]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2023-09-23T20:09:49.891 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 23 20:09:49 smithi018 sudo[134499]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-09-23T20:09:49.891 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 23 20:09:49 smithi018 sudo[134499]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-09-23T20:09:49.891 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 23 20:09:49 smithi018 sudo[134499]: pam_unix(sudo:session): session closed for user root 2023-09-23T20:09:50.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:50 smithi018 ceph-mon[109137]: from='client.? 172.21.15.18:0/3122581490' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8edccb7f-c4c9-47f3-a6f5-38a1a38cb099"}]: dispatch 2023-09-23T20:09:50.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:50 smithi018 ceph-mon[109137]: from='client.? 172.21.15.18:0/3122581490' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "8edccb7f-c4c9-47f3-a6f5-38a1a38cb099"}]': finished 2023-09-23T20:09:50.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:50 smithi018 ceph-mon[109137]: osdmap e22: 4 total, 3 up, 4 in 2023-09-23T20:09:50.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:50 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T20:09:50.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:50 smithi018 ceph-mon[109137]: mgrmap e15: y(active, since 2m), standbys: x 2023-09-23T20:09:50.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:50 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:09:50.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:50 smithi018 ceph-mon[95889]: from='client.? 172.21.15.18:0/3122581490' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8edccb7f-c4c9-47f3-a6f5-38a1a38cb099"}]: dispatch 2023-09-23T20:09:50.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:50 smithi018 ceph-mon[95889]: from='client.? 172.21.15.18:0/3122581490' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "8edccb7f-c4c9-47f3-a6f5-38a1a38cb099"}]': finished 2023-09-23T20:09:50.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:50 smithi018 ceph-mon[95889]: osdmap e22: 4 total, 3 up, 4 in 2023-09-23T20:09:50.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:50 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T20:09:50.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:50 smithi018 ceph-mon[95889]: mgrmap e15: y(active, since 2m), standbys: x 2023-09-23T20:09:50.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:50 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:09:50.393 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 23 20:09:49 smithi018 sudo[134528]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2023-09-23T20:09:50.393 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 23 20:09:49 smithi018 sudo[134528]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-09-23T20:09:50.393 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 23 20:09:49 smithi018 sudo[134528]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-09-23T20:09:50.393 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 23 20:09:49 smithi018 sudo[134528]: pam_unix(sudo:session): session closed for user root 2023-09-23T20:09:50.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:50 smithi149 ceph-mon[100764]: from='client.? 172.21.15.18:0/3122581490' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "8edccb7f-c4c9-47f3-a6f5-38a1a38cb099"}]: dispatch 2023-09-23T20:09:50.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:50 smithi149 ceph-mon[100764]: from='client.? 172.21.15.18:0/3122581490' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "8edccb7f-c4c9-47f3-a6f5-38a1a38cb099"}]': finished 2023-09-23T20:09:50.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:50 smithi149 ceph-mon[100764]: osdmap e22: 4 total, 3 up, 4 in 2023-09-23T20:09:50.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:50 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T20:09:50.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:50 smithi149 ceph-mon[100764]: mgrmap e15: y(active, since 2m), standbys: x 2023-09-23T20:09:50.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:50 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:09:50.890 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 23 20:09:50 smithi018 sudo[134560]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2023-09-23T20:09:50.891 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 23 20:09:50 smithi018 sudo[134560]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-09-23T20:09:50.891 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 23 20:09:50 smithi018 sudo[134560]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-09-23T20:09:50.891 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 23 20:09:50 smithi018 sudo[134560]: pam_unix(sudo:session): session closed for user root 2023-09-23T20:09:51.224 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 23 20:09:50 smithi018 sudo[134615]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2023-09-23T20:09:51.224 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 23 20:09:50 smithi018 sudo[134615]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-09-23T20:09:51.225 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 23 20:09:50 smithi018 sudo[134615]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-09-23T20:09:51.225 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 23 20:09:50 smithi018 sudo[134615]: pam_unix(sudo:session): session closed for user root 2023-09-23T20:09:51.640 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:51 smithi018 ceph-mon[95889]: from='client.? 172.21.15.18:0/3497873556' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T20:09:51.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:51 smithi018 ceph-mon[95889]: pgmap v73: 1 pgs: 1 active+clean; 449 KiB data, 16 MiB used, 268 GiB / 268 GiB avail 2023-09-23T20:09:51.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:51 smithi018 ceph-mon[109137]: from='client.? 172.21.15.18:0/3497873556' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T20:09:51.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:51 smithi018 ceph-mon[109137]: pgmap v73: 1 pgs: 1 active+clean; 449 KiB data, 16 MiB used, 268 GiB / 268 GiB avail 2023-09-23T20:09:51.641 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 23 20:09:51 smithi018 sudo[134989]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2023-09-23T20:09:51.641 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 23 20:09:51 smithi018 sudo[134989]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-09-23T20:09:51.641 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 23 20:09:51 smithi018 sudo[134989]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-09-23T20:09:51.642 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 23 20:09:51 smithi018 sudo[134989]: pam_unix(sudo:session): session closed for user root 2023-09-23T20:09:51.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:51 smithi149 ceph-mon[100764]: from='client.? 172.21.15.18:0/3497873556' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T20:09:51.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:51 smithi149 ceph-mon[100764]: pgmap v73: 1 pgs: 1 active+clean; 449 KiB data, 16 MiB used, 268 GiB / 268 GiB avail 2023-09-23T20:09:52.140 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 23 20:09:51 smithi018 sudo[134994]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2023-09-23T20:09:52.141 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 23 20:09:51 smithi018 sudo[134994]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-09-23T20:09:52.141 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 23 20:09:51 smithi018 sudo[134994]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-09-23T20:09:52.141 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 23 20:09:51 smithi018 sudo[134994]: pam_unix(sudo:session): session closed for user root 2023-09-23T20:09:52.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:52 smithi018 sudo[135376]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2023-09-23T20:09:52.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:52 smithi018 sudo[135376]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-09-23T20:09:52.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:52 smithi018 sudo[135376]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-09-23T20:09:52.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:52 smithi018 sudo[135376]: pam_unix(sudo:session): session closed for user root 2023-09-23T20:09:53.112 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:52 smithi018 sudo[135477]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2023-09-23T20:09:53.112 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:52 smithi018 sudo[135477]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-09-23T20:09:53.112 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:52 smithi018 sudo[135477]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-09-23T20:09:53.112 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:52 smithi018 sudo[135477]: pam_unix(sudo:session): session closed for user root 2023-09-23T20:09:53.181 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:52 smithi149 sudo[103605]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2023-09-23T20:09:53.181 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:52 smithi149 sudo[103605]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-09-23T20:09:53.181 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:52 smithi149 sudo[103605]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-09-23T20:09:53.181 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:52 smithi149 sudo[103605]: pam_unix(sudo:session): session closed for user root 2023-09-23T20:09:53.390 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:53 smithi018 ceph-mon[95889]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-09-23T20:09:53.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:53 smithi018 ceph-mon[95889]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-09-23T20:09:53.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:53 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T20:09:53.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:53 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:09:53.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:53 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:09:53.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:53 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T20:09:53.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:53 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:09:53.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:53 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:09:53.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:53 smithi018 ceph-mon[109137]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-09-23T20:09:53.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:53 smithi018 ceph-mon[109137]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-09-23T20:09:53.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:53 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T20:09:53.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:53 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:09:53.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:53 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:09:53.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:53 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T20:09:53.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:53 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:09:53.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:53 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:09:53.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:53 smithi149 sudo[103608]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2023-09-23T20:09:53.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:53 smithi149 sudo[103608]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-09-23T20:09:53.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:53 smithi149 sudo[103608]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-09-23T20:09:53.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:53 smithi149 sudo[103608]: pam_unix(sudo:session): session closed for user root 2023-09-23T20:09:53.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:53 smithi149 ceph-mon[100764]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-09-23T20:09:53.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:53 smithi149 ceph-mon[100764]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-09-23T20:09:53.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:53 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T20:09:53.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:53 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:09:53.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:53 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:09:53.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:53 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T20:09:53.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:53 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:09:53.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:53 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:09:53.991 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:53 smithi018 sudo[135624]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2023-09-23T20:09:53.991 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:53 smithi018 sudo[135624]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-09-23T20:09:53.991 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:53 smithi018 sudo[135624]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-09-23T20:09:53.991 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:53 smithi018 sudo[135624]: pam_unix(sudo:session): session closed for user root 2023-09-23T20:09:54.351 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:54 smithi018 ceph-mon[95889]: pgmap v74: 1 pgs: 1 active+clean; 449 KiB data, 17 MiB used, 268 GiB / 268 GiB avail 2023-09-23T20:09:54.351 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:54 smithi018 ceph-mon[95889]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-09-23T20:09:54.351 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:54 smithi018 ceph-mon[95889]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-09-23T20:09:54.351 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:54 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:09:54.351 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:54 smithi018 ceph-mon[95889]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-09-23T20:09:54.352 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:54 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T20:09:54.352 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:54 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:09:54.352 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:54 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:09:54.352 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:54 smithi018 ceph-mon[95889]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-09-23T20:09:54.352 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:54 smithi018 sudo[135676]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2023-09-23T20:09:54.352 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:54 smithi018 sudo[135676]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-09-23T20:09:54.352 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:54 smithi018 sudo[135676]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-09-23T20:09:54.352 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:54 smithi018 sudo[135676]: pam_unix(sudo:session): session closed for user root 2023-09-23T20:09:54.352 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:54 smithi018 ceph-mon[109137]: pgmap v74: 1 pgs: 1 active+clean; 449 KiB data, 17 MiB used, 268 GiB / 268 GiB avail 2023-09-23T20:09:54.352 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:54 smithi018 ceph-mon[109137]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-09-23T20:09:54.353 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:54 smithi018 ceph-mon[109137]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-09-23T20:09:54.353 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:54 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:09:54.353 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:54 smithi018 ceph-mon[109137]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-09-23T20:09:54.353 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:54 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T20:09:54.353 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:54 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:09:54.353 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:54 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:09:54.353 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:54 smithi018 ceph-mon[109137]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-09-23T20:09:54.465 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:54 smithi149 ceph-mon[100764]: pgmap v74: 1 pgs: 1 active+clean; 449 KiB data, 17 MiB used, 268 GiB / 268 GiB avail 2023-09-23T20:09:54.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:54 smithi149 ceph-mon[100764]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-09-23T20:09:54.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:54 smithi149 ceph-mon[100764]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-09-23T20:09:54.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:54 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:09:54.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:54 smithi149 ceph-mon[100764]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-09-23T20:09:54.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:54 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T20:09:54.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:54 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:09:54.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:54 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:09:54.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:54 smithi149 ceph-mon[100764]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-09-23T20:09:55.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:55 smithi149 ceph-mon[100764]: pgmap v75: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 5.5 KiB/s rd, 268 KiB/s wr, 18 op/s 2023-09-23T20:09:55.586 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:55 smithi018 ceph-mon[109137]: pgmap v75: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 5.5 KiB/s rd, 268 KiB/s wr, 18 op/s 2023-09-23T20:09:55.586 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:55 smithi018 ceph-mon[95889]: pgmap v75: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 5.5 KiB/s rd, 268 KiB/s wr, 18 op/s 2023-09-23T20:09:56.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:56 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-09-23T20:09:56.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:56 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:09:56.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:56 smithi149 ceph-mon[100764]: Deploying daemon osd.3 on smithi018 2023-09-23T20:09:56.482 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:56 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-09-23T20:09:56.482 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:56 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:09:56.482 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:56 smithi018 ceph-mon[109137]: Deploying daemon osd.3 on smithi018 2023-09-23T20:09:56.482 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:56 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-09-23T20:09:56.482 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:56 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:09:56.482 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:56 smithi018 ceph-mon[95889]: Deploying daemon osd.3 on smithi018 2023-09-23T20:09:57.465 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:57 smithi149 ceph-mon[100764]: pgmap v76: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 3.9 KiB/s rd, 190 KiB/s wr, 13 op/s 2023-09-23T20:09:57.513 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:57 smithi018 ceph-mon[109137]: pgmap v76: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 3.9 KiB/s rd, 190 KiB/s wr, 13 op/s 2023-09-23T20:09:57.513 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:57 smithi018 ceph-mon[95889]: pgmap v76: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 3.9 KiB/s rd, 190 KiB/s wr, 13 op/s 2023-09-23T20:09:59.890 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:09:59 smithi018 ceph-mon[109137]: pgmap v77: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 3.0 KiB/s rd, 147 KiB/s wr, 10 op/s 2023-09-23T20:09:59.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:09:59 smithi018 ceph-mon[95889]: pgmap v77: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 3.0 KiB/s rd, 147 KiB/s wr, 10 op/s 2023-09-23T20:09:59.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:09:59 smithi149 ceph-mon[100764]: pgmap v77: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 3.0 KiB/s rd, 147 KiB/s wr, 10 op/s 2023-09-23T20:10:00.965 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:00 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:10:00.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:00 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:10:00.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:00 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:10:00.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:00 smithi149 ceph-mon[100764]: overall HEALTH_OK 2023-09-23T20:10:01.016 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:00 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:10:01.016 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:00 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:10:01.016 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:00 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:10:01.016 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:00 smithi018 ceph-mon[109137]: overall HEALTH_OK 2023-09-23T20:10:01.016 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:00 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:10:01.017 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:00 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:10:01.017 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:00 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:10:01.017 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:00 smithi018 ceph-mon[95889]: overall HEALTH_OK 2023-09-23T20:10:01.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:01 smithi149 ceph-mon[100764]: pgmap v78: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 2.5 KiB/s rd, 120 KiB/s wr, 8 op/s 2023-09-23T20:10:02.100 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:01 smithi018 ceph-mon[109137]: pgmap v78: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 2.5 KiB/s rd, 120 KiB/s wr, 8 op/s 2023-09-23T20:10:02.101 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:01 smithi018 ceph-mon[95889]: pgmap v78: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 2.5 KiB/s rd, 120 KiB/s wr, 8 op/s 2023-09-23T20:10:02.395 INFO:teuthology.orchestra.run.smithi018.stdout:Created osd(s) 3 on host 'smithi018' 2023-09-23T20:10:02.831 DEBUG:teuthology.orchestra.run.smithi018:osd.3> sudo journalctl -f -n 0 -u ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7@osd.3.service 2023-09-23T20:10:02.833 INFO:tasks.cephadm:Deploying osd.4 on smithi149 with /dev/vg_nvme/lv_4... 2023-09-23T20:10:02.833 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2023-09-23T20:10:02.868 INFO:journalctl@ceph.osd.3.smithi018.stdout:-- Logs begin at Sat 2023-09-23 19:50:10 UTC. -- 2023-09-23T20:10:03.077 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 23 20:10:03 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-3[136408]: 2023-09-23T20:10:03.073+0000 7f4f576d8540 -1 osd.3 0 log_to_monitors true 2023-09-23T20:10:03.388 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:03 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:10:03.454 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:10:03.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:03 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:10:03.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:03 smithi018 ceph-mon[95889]: pgmap v79: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 2.2 KiB/s rd, 109 KiB/s wr, 7 op/s 2023-09-23T20:10:03.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:03 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:10:03.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:03 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:10:03.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:03 smithi018 ceph-mon[95889]: from='osd.3 [v2:172.21.15.18:6826/423024182,v1:172.21.15.18:6827/423024182]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-09-23T20:10:03.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:03 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:10:03.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:03 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:10:03.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:03 smithi018 ceph-mon[109137]: pgmap v79: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 2.2 KiB/s rd, 109 KiB/s wr, 7 op/s 2023-09-23T20:10:03.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:03 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:10:03.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:03 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:10:03.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:03 smithi018 ceph-mon[109137]: from='osd.3 [v2:172.21.15.18:6826/423024182,v1:172.21.15.18:6827/423024182]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-09-23T20:10:03.685 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:03 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:10:03.685 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:03 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:10:03.685 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:03 smithi149 ceph-mon[100764]: pgmap v79: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 2.2 KiB/s rd, 109 KiB/s wr, 7 op/s 2023-09-23T20:10:03.685 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:03 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:10:03.686 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:03 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:10:03.686 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:03 smithi149 ceph-mon[100764]: from='osd.3 [v2:172.21.15.18:6826/423024182,v1:172.21.15.18:6827/423024182]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-09-23T20:10:04.878 INFO:teuthology.orchestra.run.smithi149.stderr:--> Zapping: /dev/vg_nvme/lv_4 2023-09-23T20:10:04.879 INFO:teuthology.orchestra.run.smithi149.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2023-09-23T20:10:04.879 INFO:teuthology.orchestra.run.smithi149.stderr: stderr: 10+0 records in 2023-09-23T20:10:04.879 INFO:teuthology.orchestra.run.smithi149.stderr:10+0 records out 2023-09-23T20:10:04.879 INFO:teuthology.orchestra.run.smithi149.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0111035 s, 944 MB/s 2023-09-23T20:10:04.879 INFO:teuthology.orchestra.run.smithi149.stderr:--> Zapping successful for: 2023-09-23T20:10:05.198 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:05 smithi149 ceph-mon[100764]: from='osd.3 [v2:172.21.15.18:6826/423024182,v1:172.21.15.18:6827/423024182]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-09-23T20:10:05.198 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:05 smithi149 ceph-mon[100764]: osdmap e23: 4 total, 3 up, 4 in 2023-09-23T20:10:05.198 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:05 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T20:10:05.198 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:05 smithi149 ceph-mon[100764]: from='osd.3 [v2:172.21.15.18:6826/423024182,v1:172.21.15.18:6827/423024182]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi018", "root=default"]}]: dispatch 2023-09-23T20:10:05.198 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:05 smithi149 ceph-mon[100764]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2023-09-23T20:10:05.210 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph orch daemon add osd smithi149:vg_nvme/lv_4 2023-09-23T20:10:05.391 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 23 20:10:05 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-3[136408]: 2023-09-23T20:10:05.028+0000 7f4f4debe700 -1 osd.3 0 waiting for initial osdmap 2023-09-23T20:10:05.391 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 23 20:10:05 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-3[136408]: 2023-09-23T20:10:05.033+0000 7f4f4a4bc700 -1 osd.3 24 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T20:10:05.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:05 smithi018 ceph-mon[95889]: from='osd.3 [v2:172.21.15.18:6826/423024182,v1:172.21.15.18:6827/423024182]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-09-23T20:10:05.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:05 smithi018 ceph-mon[95889]: osdmap e23: 4 total, 3 up, 4 in 2023-09-23T20:10:05.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:05 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T20:10:05.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:05 smithi018 ceph-mon[95889]: from='osd.3 [v2:172.21.15.18:6826/423024182,v1:172.21.15.18:6827/423024182]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi018", "root=default"]}]: dispatch 2023-09-23T20:10:05.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:05 smithi018 ceph-mon[95889]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2023-09-23T20:10:05.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:05 smithi018 ceph-mon[109137]: from='osd.3 [v2:172.21.15.18:6826/423024182,v1:172.21.15.18:6827/423024182]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-09-23T20:10:05.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:05 smithi018 ceph-mon[109137]: osdmap e23: 4 total, 3 up, 4 in 2023-09-23T20:10:05.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:05 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T20:10:05.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:05 smithi018 ceph-mon[109137]: from='osd.3 [v2:172.21.15.18:6826/423024182,v1:172.21.15.18:6827/423024182]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi018", "root=default"]}]: dispatch 2023-09-23T20:10:05.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:05 smithi018 ceph-mon[109137]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2023-09-23T20:10:05.805 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:10:06.312 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:06 smithi149 ceph-mon[100764]: from='osd.3 [v2:172.21.15.18:6826/423024182,v1:172.21.15.18:6827/423024182]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi018", "root=default"]}]': finished 2023-09-23T20:10:06.312 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:06 smithi149 ceph-mon[100764]: osdmap e24: 4 total, 3 up, 4 in 2023-09-23T20:10:06.312 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:06 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T20:10:06.312 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:06 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T20:10:06.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:06 smithi018 ceph-mon[109137]: from='osd.3 [v2:172.21.15.18:6826/423024182,v1:172.21.15.18:6827/423024182]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi018", "root=default"]}]': finished 2023-09-23T20:10:06.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:06 smithi018 ceph-mon[109137]: osdmap e24: 4 total, 3 up, 4 in 2023-09-23T20:10:06.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:06 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T20:10:06.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:06 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T20:10:06.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:06 smithi018 ceph-mon[95889]: from='osd.3 [v2:172.21.15.18:6826/423024182,v1:172.21.15.18:6827/423024182]' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi018", "root=default"]}]': finished 2023-09-23T20:10:06.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:06 smithi018 ceph-mon[95889]: osdmap e24: 4 total, 3 up, 4 in 2023-09-23T20:10:06.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:06 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T20:10:06.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:06 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T20:10:07.030 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:07 smithi018 ceph-mon[95889]: purged_snaps scrub starts 2023-09-23T20:10:07.030 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:07 smithi018 ceph-mon[95889]: purged_snaps scrub ok 2023-09-23T20:10:07.030 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:07 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T20:10:07.030 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:07 smithi018 ceph-mon[95889]: osd.3 [v2:172.21.15.18:6826/423024182,v1:172.21.15.18:6827/423024182] boot 2023-09-23T20:10:07.030 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:07 smithi018 ceph-mon[95889]: osdmap e25: 4 total, 4 up, 4 in 2023-09-23T20:10:07.030 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:07 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T20:10:07.030 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:07 smithi018 ceph-mon[95889]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2023-09-23T20:10:07.030 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:07 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:10:07.030 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:07 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:10:07.031 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:07 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:10:07.031 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:07 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:10:07.031 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:07 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:10:07.031 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:07 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:10:07.031 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:07 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:10:07.031 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:07 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:10:07.031 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:07 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:10:07.031 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:07 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:10:07.031 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:07 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:10:07.031 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:07 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:10:07.031 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:07 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:10:07.031 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:07 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:10:07.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:07 smithi149 ceph-mon[100764]: purged_snaps scrub starts 2023-09-23T20:10:07.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:07 smithi149 ceph-mon[100764]: purged_snaps scrub ok 2023-09-23T20:10:07.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:07 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T20:10:07.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:07 smithi149 ceph-mon[100764]: osd.3 [v2:172.21.15.18:6826/423024182,v1:172.21.15.18:6827/423024182] boot 2023-09-23T20:10:07.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:07 smithi149 ceph-mon[100764]: osdmap e25: 4 total, 4 up, 4 in 2023-09-23T20:10:07.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:07 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T20:10:07.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:07 smithi149 ceph-mon[100764]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2023-09-23T20:10:07.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:07 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:10:07.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:07 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:10:07.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:07 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:10:07.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:07 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:10:07.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:07 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:10:07.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:07 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:10:07.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:07 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:10:07.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:07 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:10:07.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:07 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:10:07.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:07 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:10:07.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:07 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:10:07.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:07 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:10:07.218 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:07 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:10:07.218 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:07 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:10:07.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:07 smithi018 ceph-mon[109137]: purged_snaps scrub starts 2023-09-23T20:10:07.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:07 smithi018 ceph-mon[109137]: purged_snaps scrub ok 2023-09-23T20:10:07.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:07 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T20:10:07.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:07 smithi018 ceph-mon[109137]: osd.3 [v2:172.21.15.18:6826/423024182,v1:172.21.15.18:6827/423024182] boot 2023-09-23T20:10:07.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:07 smithi018 ceph-mon[109137]: osdmap e25: 4 total, 4 up, 4 in 2023-09-23T20:10:07.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:07 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T20:10:07.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:07 smithi018 ceph-mon[109137]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2023-09-23T20:10:07.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:07 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:10:07.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:07 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:10:07.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:07 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:10:07.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:07 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:10:07.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:07 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:10:07.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:07 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:10:07.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:07 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:10:07.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:07 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:10:07.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:07 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:10:07.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:07 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:10:07.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:07 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:10:07.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:07 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:10:07.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:07 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:10:07.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:07 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:10:08.287 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:08 smithi149 ceph-mon[100764]: Detected new or changed devices on smithi018 2023-09-23T20:10:08.288 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:08 smithi149 ceph-mon[100764]: Adjusting osd_memory_target on smithi018 to 4037M 2023-09-23T20:10:08.288 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:08 smithi149 ceph-mon[100764]: osdmap e26: 4 total, 4 up, 4 in 2023-09-23T20:10:08.288 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:08 smithi149 ceph-mon[100764]: from='client.24220 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi149:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:10:08.288 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:08 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T20:10:08.288 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:08 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T20:10:08.288 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:08 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:10:08.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:08 smithi018 ceph-mon[109137]: Detected new or changed devices on smithi018 2023-09-23T20:10:08.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:08 smithi018 ceph-mon[109137]: Adjusting osd_memory_target on smithi018 to 4037M 2023-09-23T20:10:08.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:08 smithi018 ceph-mon[109137]: osdmap e26: 4 total, 4 up, 4 in 2023-09-23T20:10:08.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:08 smithi018 ceph-mon[109137]: from='client.24220 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi149:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:10:08.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:08 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T20:10:08.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:08 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T20:10:08.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:08 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:10:08.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:08 smithi018 ceph-mon[95889]: Detected new or changed devices on smithi018 2023-09-23T20:10:08.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:08 smithi018 ceph-mon[95889]: Adjusting osd_memory_target on smithi018 to 4037M 2023-09-23T20:10:08.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:08 smithi018 ceph-mon[95889]: osdmap e26: 4 total, 4 up, 4 in 2023-09-23T20:10:08.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:08 smithi018 ceph-mon[95889]: from='client.24220 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi149:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:10:08.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:08 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T20:10:08.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:08 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T20:10:08.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:08 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:10:09.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:09 smithi018 ceph-mon[109137]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T20:10:09.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:09 smithi018 ceph-mon[95889]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T20:10:09.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:09 smithi149 ceph-mon[100764]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T20:10:10.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:10 smithi018 ceph-mon[109137]: from='client.? 172.21.15.149:0/2903799120' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1d5a64ce-fe42-4355-991a-dfb20cbe5423"}]: dispatch 2023-09-23T20:10:10.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:10 smithi018 ceph-mon[109137]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1d5a64ce-fe42-4355-991a-dfb20cbe5423"}]: dispatch 2023-09-23T20:10:10.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:10 smithi018 ceph-mon[109137]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "1d5a64ce-fe42-4355-991a-dfb20cbe5423"}]': finished 2023-09-23T20:10:10.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:10 smithi018 ceph-mon[109137]: osdmap e27: 5 total, 4 up, 5 in 2023-09-23T20:10:10.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:10 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T20:10:10.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:10 smithi018 ceph-mon[95889]: from='client.? 172.21.15.149:0/2903799120' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1d5a64ce-fe42-4355-991a-dfb20cbe5423"}]: dispatch 2023-09-23T20:10:10.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:10 smithi018 ceph-mon[95889]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1d5a64ce-fe42-4355-991a-dfb20cbe5423"}]: dispatch 2023-09-23T20:10:10.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:10 smithi018 ceph-mon[95889]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "1d5a64ce-fe42-4355-991a-dfb20cbe5423"}]': finished 2023-09-23T20:10:10.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:10 smithi018 ceph-mon[95889]: osdmap e27: 5 total, 4 up, 5 in 2023-09-23T20:10:10.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:10 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T20:10:10.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:10 smithi149 ceph-mon[100764]: from='client.? 172.21.15.149:0/2903799120' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1d5a64ce-fe42-4355-991a-dfb20cbe5423"}]: dispatch 2023-09-23T20:10:10.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:10 smithi149 ceph-mon[100764]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1d5a64ce-fe42-4355-991a-dfb20cbe5423"}]: dispatch 2023-09-23T20:10:10.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:10 smithi149 ceph-mon[100764]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "1d5a64ce-fe42-4355-991a-dfb20cbe5423"}]': finished 2023-09-23T20:10:10.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:10 smithi149 ceph-mon[100764]: osdmap e27: 5 total, 4 up, 5 in 2023-09-23T20:10:10.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:10 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T20:10:11.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:11 smithi018 ceph-mon[95889]: from='client.? 172.21.15.149:0/1657407612' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T20:10:11.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:11 smithi018 ceph-mon[95889]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2023-09-23T20:10:11.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:11 smithi018 ceph-mon[109137]: from='client.? 172.21.15.149:0/1657407612' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T20:10:11.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:11 smithi018 ceph-mon[109137]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2023-09-23T20:10:11.465 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:11 smithi149 ceph-mon[100764]: from='client.? 172.21.15.149:0/1657407612' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T20:10:11.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:11 smithi149 ceph-mon[100764]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2023-09-23T20:10:13.802 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:13 smithi149 ceph-mon[100764]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T20:10:14.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:13 smithi018 ceph-mon[95889]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T20:10:14.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:13 smithi018 ceph-mon[109137]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T20:10:14.954 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:14 smithi149 ceph-mon[100764]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T20:10:15.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:14 smithi018 ceph-mon[95889]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T20:10:15.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:14 smithi018 ceph-mon[109137]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T20:10:15.965 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:15 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-09-23T20:10:15.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:15 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:10:15.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:15 smithi149 ceph-mon[100764]: Deploying daemon osd.4 on smithi149 2023-09-23T20:10:16.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:15 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-09-23T20:10:16.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:15 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:10:16.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:15 smithi018 ceph-mon[95889]: Deploying daemon osd.4 on smithi149 2023-09-23T20:10:16.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:15 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-09-23T20:10:16.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:15 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:10:16.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:15 smithi018 ceph-mon[109137]: Deploying daemon osd.4 on smithi149 2023-09-23T20:10:17.131 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:16 smithi149 ceph-mon[100764]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T20:10:17.140 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:16 smithi018 ceph-mon[109137]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T20:10:17.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:16 smithi018 ceph-mon[95889]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T20:10:19.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:19 smithi149 ceph-mon[100764]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T20:10:19.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:19 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:10:19.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:19 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:10:19.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:19 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:10:20.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:19 smithi018 ceph-mon[95889]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T20:10:20.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:19 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:10:20.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:19 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:10:20.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:19 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:10:20.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:19 smithi018 ceph-mon[109137]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T20:10:20.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:19 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:10:20.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:19 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:10:20.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:19 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:10:21.741 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:21 smithi149 ceph-mon[100764]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T20:10:21.742 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:21 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:10:21.742 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:21 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:10:21.742 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:21 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:10:21.742 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:21 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:10:21.742 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:21 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:10:21.798 INFO:teuthology.orchestra.run.smithi149.stdout:Created osd(s) 4 on host 'smithi149' 2023-09-23T20:10:22.140 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:21 smithi018 ceph-mon[95889]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T20:10:22.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:21 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:10:22.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:21 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:10:22.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:21 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:10:22.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:21 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:10:22.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:21 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:10:22.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:21 smithi018 ceph-mon[109137]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T20:10:22.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:21 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:10:22.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:21 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:10:22.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:21 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:10:22.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:21 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:10:22.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:21 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:10:22.181 DEBUG:teuthology.orchestra.run.smithi149:osd.4> sudo journalctl -f -n 0 -u ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7@osd.4.service 2023-09-23T20:10:22.183 INFO:tasks.cephadm:Deploying osd.5 on smithi149 with /dev/vg_nvme/lv_3... 2023-09-23T20:10:22.183 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2023-09-23T20:10:22.212 INFO:journalctl@ceph.osd.4.smithi149.stdout:-- Logs begin at Sat 2023-09-23 19:50:38 UTC. -- 2023-09-23T20:10:22.465 INFO:journalctl@ceph.osd.4.smithi149.stdout:Sep 23 20:10:22 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-4[106331]: 2023-09-23T20:10:22.277+0000 7fbc2abef540 -1 osd.4 0 log_to_monitors true 2023-09-23T20:10:22.917 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:22 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:10:22.917 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:22 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:10:22.917 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:22 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:10:22.917 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:22 smithi149 ceph-mon[100764]: from='osd.4 [v2:172.21.15.149:6800/3640432477,v1:172.21.15.149:6801/3640432477]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-09-23T20:10:22.917 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:22 smithi149 ceph-mon[100764]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-09-23T20:10:23.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:22 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:10:23.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:22 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:10:23.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:22 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:10:23.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:22 smithi018 ceph-mon[95889]: from='osd.4 [v2:172.21.15.149:6800/3640432477,v1:172.21.15.149:6801/3640432477]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-09-23T20:10:23.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:22 smithi018 ceph-mon[95889]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-09-23T20:10:23.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:22 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:10:23.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:22 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:10:23.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:22 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:10:23.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:22 smithi018 ceph-mon[109137]: from='osd.4 [v2:172.21.15.149:6800/3640432477,v1:172.21.15.149:6801/3640432477]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-09-23T20:10:23.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:22 smithi018 ceph-mon[109137]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-09-23T20:10:23.194 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:10:23.966 INFO:journalctl@ceph.osd.4.smithi149.stdout:Sep 23 20:10:23 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-4[106331]: 2023-09-23T20:10:23.775+0000 7fbc22bd8700 -1 osd.4 0 waiting for initial osdmap 2023-09-23T20:10:23.966 INFO:journalctl@ceph.osd.4.smithi149.stdout:Sep 23 20:10:23 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-4[106331]: 2023-09-23T20:10:23.780+0000 7fbc1c1d0700 -1 osd.4 29 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T20:10:23.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:23 smithi149 ceph-mon[100764]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T20:10:23.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:23 smithi149 ceph-mon[100764]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-09-23T20:10:23.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:23 smithi149 ceph-mon[100764]: osdmap e28: 5 total, 4 up, 5 in 2023-09-23T20:10:23.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:23 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T20:10:23.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:23 smithi149 ceph-mon[100764]: from='osd.4 [v2:172.21.15.149:6800/3640432477,v1:172.21.15.149:6801/3640432477]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi149", "root=default"]}]: dispatch 2023-09-23T20:10:23.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:23 smithi149 ceph-mon[100764]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi149", "root=default"]}]: dispatch 2023-09-23T20:10:24.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:23 smithi018 ceph-mon[95889]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T20:10:24.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:23 smithi018 ceph-mon[95889]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-09-23T20:10:24.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:23 smithi018 ceph-mon[95889]: osdmap e28: 5 total, 4 up, 5 in 2023-09-23T20:10:24.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:23 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T20:10:24.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:23 smithi018 ceph-mon[95889]: from='osd.4 [v2:172.21.15.149:6800/3640432477,v1:172.21.15.149:6801/3640432477]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi149", "root=default"]}]: dispatch 2023-09-23T20:10:24.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:23 smithi018 ceph-mon[95889]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi149", "root=default"]}]: dispatch 2023-09-23T20:10:24.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:23 smithi018 ceph-mon[109137]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T20:10:24.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:23 smithi018 ceph-mon[109137]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-09-23T20:10:24.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:23 smithi018 ceph-mon[109137]: osdmap e28: 5 total, 4 up, 5 in 2023-09-23T20:10:24.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:23 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T20:10:24.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:23 smithi018 ceph-mon[109137]: from='osd.4 [v2:172.21.15.149:6800/3640432477,v1:172.21.15.149:6801/3640432477]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi149", "root=default"]}]: dispatch 2023-09-23T20:10:24.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:23 smithi018 ceph-mon[109137]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi149", "root=default"]}]: dispatch 2023-09-23T20:10:24.875 INFO:teuthology.orchestra.run.smithi149.stderr:--> Zapping: /dev/vg_nvme/lv_3 2023-09-23T20:10:24.875 INFO:teuthology.orchestra.run.smithi149.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2023-09-23T20:10:24.875 INFO:teuthology.orchestra.run.smithi149.stderr: stderr: 10+0 records in 2023-09-23T20:10:24.875 INFO:teuthology.orchestra.run.smithi149.stderr:10+0 records out 2023-09-23T20:10:24.875 INFO:teuthology.orchestra.run.smithi149.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0110612 s, 948 MB/s 2023-09-23T20:10:24.875 INFO:teuthology.orchestra.run.smithi149.stderr: stderr: 2023-09-23T20:10:24.876 INFO:teuthology.orchestra.run.smithi149.stderr:--> Zapping successful for: 2023-09-23T20:10:24.965 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:24 smithi149 ceph-mon[100764]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi149", "root=default"]}]': finished 2023-09-23T20:10:24.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:24 smithi149 ceph-mon[100764]: osdmap e29: 5 total, 4 up, 5 in 2023-09-23T20:10:24.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:24 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T20:10:24.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:24 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T20:10:24.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:24 smithi149 ceph-mon[100764]: osd.4 [v2:172.21.15.149:6800/3640432477,v1:172.21.15.149:6801/3640432477] boot 2023-09-23T20:10:24.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:24 smithi149 ceph-mon[100764]: osdmap e30: 5 total, 5 up, 5 in 2023-09-23T20:10:24.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:24 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T20:10:25.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:24 smithi018 ceph-mon[109137]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi149", "root=default"]}]': finished 2023-09-23T20:10:25.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:24 smithi018 ceph-mon[109137]: osdmap e29: 5 total, 4 up, 5 in 2023-09-23T20:10:25.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:24 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T20:10:25.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:24 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T20:10:25.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:24 smithi018 ceph-mon[109137]: osd.4 [v2:172.21.15.149:6800/3640432477,v1:172.21.15.149:6801/3640432477] boot 2023-09-23T20:10:25.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:24 smithi018 ceph-mon[109137]: osdmap e30: 5 total, 5 up, 5 in 2023-09-23T20:10:25.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:24 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T20:10:25.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:24 smithi018 ceph-mon[95889]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi149", "root=default"]}]': finished 2023-09-23T20:10:25.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:24 smithi018 ceph-mon[95889]: osdmap e29: 5 total, 4 up, 5 in 2023-09-23T20:10:25.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:24 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T20:10:25.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:24 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T20:10:25.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:24 smithi018 ceph-mon[95889]: osd.4 [v2:172.21.15.149:6800/3640432477,v1:172.21.15.149:6801/3640432477] boot 2023-09-23T20:10:25.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:24 smithi018 ceph-mon[95889]: osdmap e30: 5 total, 5 up, 5 in 2023-09-23T20:10:25.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:24 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T20:10:25.327 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph orch daemon add osd smithi149:vg_nvme/lv_3 2023-09-23T20:10:25.923 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:10:26.106 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:25 smithi149 ceph-mon[100764]: purged_snaps scrub starts 2023-09-23T20:10:26.106 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:25 smithi149 ceph-mon[100764]: purged_snaps scrub ok 2023-09-23T20:10:26.106 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:25 smithi149 ceph-mon[100764]: pgmap v98: 1 pgs: 1 remapped+peering; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T20:10:26.107 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:25 smithi149 ceph-mon[100764]: Detected new or changed devices on smithi149 2023-09-23T20:10:26.107 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:25 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:10:26.107 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:25 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:10:26.107 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:25 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:10:26.107 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:25 smithi149 ceph-mon[100764]: Adjusting osd_memory_target on smithi149 to 5879M 2023-09-23T20:10:26.107 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:25 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:10:26.107 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:25 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:10:26.107 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:25 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:10:26.107 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:25 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:10:26.107 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:25 smithi149 ceph-mon[100764]: osdmap e31: 5 total, 5 up, 5 in 2023-09-23T20:10:26.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:25 smithi018 ceph-mon[95889]: purged_snaps scrub starts 2023-09-23T20:10:26.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:25 smithi018 ceph-mon[95889]: purged_snaps scrub ok 2023-09-23T20:10:26.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:25 smithi018 ceph-mon[95889]: pgmap v98: 1 pgs: 1 remapped+peering; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T20:10:26.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:25 smithi018 ceph-mon[95889]: Detected new or changed devices on smithi149 2023-09-23T20:10:26.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:25 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:10:26.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:25 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:10:26.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:25 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:10:26.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:25 smithi018 ceph-mon[95889]: Adjusting osd_memory_target on smithi149 to 5879M 2023-09-23T20:10:26.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:25 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:10:26.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:25 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:10:26.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:25 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:10:26.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:25 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:10:26.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:25 smithi018 ceph-mon[95889]: osdmap e31: 5 total, 5 up, 5 in 2023-09-23T20:10:26.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:25 smithi018 ceph-mon[109137]: purged_snaps scrub starts 2023-09-23T20:10:26.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:25 smithi018 ceph-mon[109137]: purged_snaps scrub ok 2023-09-23T20:10:26.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:25 smithi018 ceph-mon[109137]: pgmap v98: 1 pgs: 1 remapped+peering; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T20:10:26.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:25 smithi018 ceph-mon[109137]: Detected new or changed devices on smithi149 2023-09-23T20:10:26.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:25 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:10:26.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:25 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:10:26.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:25 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:10:26.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:25 smithi018 ceph-mon[109137]: Adjusting osd_memory_target on smithi149 to 5879M 2023-09-23T20:10:26.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:25 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:10:26.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:25 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:10:26.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:25 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:10:26.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:25 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:10:26.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:25 smithi018 ceph-mon[109137]: osdmap e31: 5 total, 5 up, 5 in 2023-09-23T20:10:27.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:26 smithi018 ceph-mon[95889]: osdmap e32: 5 total, 5 up, 5 in 2023-09-23T20:10:27.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:26 smithi018 ceph-mon[95889]: pgmap v101: 1 pgs: 1 remapped+peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T20:10:27.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:26 smithi018 ceph-mon[109137]: osdmap e32: 5 total, 5 up, 5 in 2023-09-23T20:10:27.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:26 smithi018 ceph-mon[109137]: pgmap v101: 1 pgs: 1 remapped+peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T20:10:27.215 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:26 smithi149 ceph-mon[100764]: osdmap e32: 5 total, 5 up, 5 in 2023-09-23T20:10:27.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:26 smithi149 ceph-mon[100764]: pgmap v101: 1 pgs: 1 remapped+peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T20:10:28.099 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:27 smithi149 ceph-mon[100764]: osdmap e33: 5 total, 5 up, 5 in 2023-09-23T20:10:28.099 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:27 smithi149 ceph-mon[100764]: from='client.24247 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi149:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:10:28.099 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:27 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T20:10:28.099 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:27 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T20:10:28.099 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:27 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:10:28.140 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:27 smithi018 ceph-mon[95889]: osdmap e33: 5 total, 5 up, 5 in 2023-09-23T20:10:28.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:27 smithi018 ceph-mon[95889]: from='client.24247 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi149:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:10:28.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:27 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T20:10:28.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:27 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T20:10:28.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:27 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:10:28.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:27 smithi018 ceph-mon[109137]: osdmap e33: 5 total, 5 up, 5 in 2023-09-23T20:10:28.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:27 smithi018 ceph-mon[109137]: from='client.24247 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi149:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:10:28.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:27 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T20:10:28.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:27 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T20:10:28.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:27 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:10:29.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:28 smithi018 ceph-mon[95889]: osdmap e34: 5 total, 5 up, 5 in 2023-09-23T20:10:29.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:28 smithi018 ceph-mon[95889]: pgmap v104: 1 pgs: 1 remapped+peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T20:10:29.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:28 smithi018 ceph-mon[109137]: osdmap e34: 5 total, 5 up, 5 in 2023-09-23T20:10:29.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:28 smithi018 ceph-mon[109137]: pgmap v104: 1 pgs: 1 remapped+peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T20:10:29.215 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:28 smithi149 ceph-mon[100764]: osdmap e34: 5 total, 5 up, 5 in 2023-09-23T20:10:29.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:28 smithi149 ceph-mon[100764]: pgmap v104: 1 pgs: 1 remapped+peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T20:10:30.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:29 smithi018 ceph-mon[95889]: from='client.? 172.21.15.149:0/1777713457' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "105ae3b2-b2fc-477d-9134-3d517bc9427e"}]: dispatch 2023-09-23T20:10:30.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:29 smithi018 ceph-mon[95889]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "105ae3b2-b2fc-477d-9134-3d517bc9427e"}]: dispatch 2023-09-23T20:10:30.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:29 smithi018 ceph-mon[95889]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "105ae3b2-b2fc-477d-9134-3d517bc9427e"}]': finished 2023-09-23T20:10:30.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:29 smithi018 ceph-mon[95889]: osdmap e35: 6 total, 5 up, 6 in 2023-09-23T20:10:30.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:29 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T20:10:30.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:29 smithi018 ceph-mon[109137]: from='client.? 172.21.15.149:0/1777713457' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "105ae3b2-b2fc-477d-9134-3d517bc9427e"}]: dispatch 2023-09-23T20:10:30.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:29 smithi018 ceph-mon[109137]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "105ae3b2-b2fc-477d-9134-3d517bc9427e"}]: dispatch 2023-09-23T20:10:30.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:29 smithi018 ceph-mon[109137]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "105ae3b2-b2fc-477d-9134-3d517bc9427e"}]': finished 2023-09-23T20:10:30.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:29 smithi018 ceph-mon[109137]: osdmap e35: 6 total, 5 up, 6 in 2023-09-23T20:10:30.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:29 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T20:10:30.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:29 smithi149 ceph-mon[100764]: from='client.? 172.21.15.149:0/1777713457' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "105ae3b2-b2fc-477d-9134-3d517bc9427e"}]: dispatch 2023-09-23T20:10:30.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:29 smithi149 ceph-mon[100764]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "105ae3b2-b2fc-477d-9134-3d517bc9427e"}]: dispatch 2023-09-23T20:10:30.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:29 smithi149 ceph-mon[100764]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "105ae3b2-b2fc-477d-9134-3d517bc9427e"}]': finished 2023-09-23T20:10:30.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:29 smithi149 ceph-mon[100764]: osdmap e35: 6 total, 5 up, 6 in 2023-09-23T20:10:30.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:29 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T20:10:31.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:30 smithi018 ceph-mon[95889]: from='client.? 172.21.15.149:0/3408532155' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T20:10:31.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:30 smithi018 ceph-mon[95889]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 30 MiB used, 447 GiB / 447 GiB avail; 119 KiB/s, 0 objects/s recovering 2023-09-23T20:10:31.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:30 smithi018 ceph-mon[109137]: from='client.? 172.21.15.149:0/3408532155' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T20:10:31.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:30 smithi018 ceph-mon[109137]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 30 MiB used, 447 GiB / 447 GiB avail; 119 KiB/s, 0 objects/s recovering 2023-09-23T20:10:31.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:30 smithi149 ceph-mon[100764]: from='client.? 172.21.15.149:0/3408532155' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T20:10:31.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:30 smithi149 ceph-mon[100764]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 30 MiB used, 447 GiB / 447 GiB avail; 119 KiB/s, 0 objects/s recovering 2023-09-23T20:10:33.915 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:33 smithi149 ceph-mon[100764]: pgmap v107: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 96 KiB/s, 0 objects/s recovering 2023-09-23T20:10:34.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:33 smithi018 ceph-mon[95889]: pgmap v107: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 96 KiB/s, 0 objects/s recovering 2023-09-23T20:10:34.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:33 smithi018 ceph-mon[109137]: pgmap v107: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 96 KiB/s, 0 objects/s recovering 2023-09-23T20:10:35.032 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:34 smithi149 ceph-mon[100764]: pgmap v108: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 74 KiB/s, 0 objects/s recovering 2023-09-23T20:10:35.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:34 smithi018 ceph-mon[109137]: pgmap v108: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 74 KiB/s, 0 objects/s recovering 2023-09-23T20:10:35.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:34 smithi018 ceph-mon[95889]: pgmap v108: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 74 KiB/s, 0 objects/s recovering 2023-09-23T20:10:36.007 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:35 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-09-23T20:10:36.007 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:35 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:10:36.007 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:35 smithi149 ceph-mon[100764]: Deploying daemon osd.5 on smithi149 2023-09-23T20:10:36.140 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:35 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-09-23T20:10:36.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:35 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:10:36.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:35 smithi018 ceph-mon[95889]: Deploying daemon osd.5 on smithi149 2023-09-23T20:10:36.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:35 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-09-23T20:10:36.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:35 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:10:36.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:35 smithi018 ceph-mon[109137]: Deploying daemon osd.5 on smithi149 2023-09-23T20:10:37.140 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:36 smithi149 ceph-mon[100764]: pgmap v109: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 65 KiB/s, 0 objects/s recovering 2023-09-23T20:10:37.140 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:36 smithi018 ceph-mon[95889]: pgmap v109: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 65 KiB/s, 0 objects/s recovering 2023-09-23T20:10:37.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:36 smithi018 ceph-mon[109137]: pgmap v109: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 65 KiB/s, 0 objects/s recovering 2023-09-23T20:10:39.965 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:39 smithi149 ceph-mon[100764]: pgmap v110: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-09-23T20:10:39.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:39 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:10:39.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:39 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:10:39.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:39 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:10:40.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:39 smithi018 ceph-mon[109137]: pgmap v110: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-09-23T20:10:40.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:39 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:10:40.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:39 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:10:40.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:39 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:10:40.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:39 smithi018 ceph-mon[95889]: pgmap v110: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-09-23T20:10:40.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:39 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:10:40.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:39 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:10:40.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:39 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:10:41.806 INFO:teuthology.orchestra.run.smithi149.stdout:Created osd(s) 5 on host 'smithi149' 2023-09-23T20:10:42.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:41 smithi018 ceph-mon[95889]: pgmap v111: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 52 KiB/s, 0 objects/s recovering 2023-09-23T20:10:42.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:41 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:10:42.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:41 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:10:42.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:41 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:10:42.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:41 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:10:42.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:41 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:10:42.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:41 smithi018 ceph-mon[109137]: pgmap v111: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 52 KiB/s, 0 objects/s recovering 2023-09-23T20:10:42.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:41 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:10:42.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:41 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:10:42.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:41 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:10:42.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:41 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:10:42.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:41 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:10:42.151 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:41 smithi149 ceph-mon[100764]: pgmap v111: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 52 KiB/s, 0 objects/s recovering 2023-09-23T20:10:42.151 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:41 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:10:42.151 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:41 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:10:42.151 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:41 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:10:42.152 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:41 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:10:42.152 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:41 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:10:42.205 DEBUG:teuthology.orchestra.run.smithi149:osd.5> sudo journalctl -f -n 0 -u ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7@osd.5.service 2023-09-23T20:10:42.207 INFO:tasks.cephadm:Deploying osd.6 on smithi149 with /dev/vg_nvme/lv_2... 2023-09-23T20:10:42.207 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2023-09-23T20:10:42.235 INFO:journalctl@ceph.osd.5.smithi149.stdout:-- Logs begin at Sat 2023-09-23 19:50:38 UTC. -- 2023-09-23T20:10:42.465 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 23 20:10:42 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-5[114166]: 2023-09-23T20:10:42.315+0000 7f117b262540 -1 osd.5 0 log_to_monitors true 2023-09-23T20:10:42.909 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:10:42.965 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:42 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:10:42.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:42 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:10:42.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:42 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:10:42.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:42 smithi149 ceph-mon[100764]: from='osd.5 [v2:172.21.15.149:6808/2969709367,v1:172.21.15.149:6809/2969709367]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-09-23T20:10:42.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:42 smithi149 ceph-mon[100764]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-09-23T20:10:43.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:42 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:10:43.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:42 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:10:43.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:42 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:10:43.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:42 smithi018 ceph-mon[95889]: from='osd.5 [v2:172.21.15.149:6808/2969709367,v1:172.21.15.149:6809/2969709367]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-09-23T20:10:43.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:42 smithi018 ceph-mon[95889]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-09-23T20:10:43.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:42 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:10:43.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:42 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:10:43.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:42 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:10:43.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:42 smithi018 ceph-mon[109137]: from='osd.5 [v2:172.21.15.149:6808/2969709367,v1:172.21.15.149:6809/2969709367]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-09-23T20:10:43.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:42 smithi018 ceph-mon[109137]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-09-23T20:10:43.850 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:43 smithi149 ceph-mon[100764]: pgmap v112: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T20:10:43.850 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:43 smithi149 ceph-mon[100764]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-09-23T20:10:43.850 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:43 smithi149 ceph-mon[100764]: osdmap e36: 6 total, 5 up, 6 in 2023-09-23T20:10:43.850 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:43 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T20:10:43.850 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:43 smithi149 ceph-mon[100764]: from='osd.5 [v2:172.21.15.149:6808/2969709367,v1:172.21.15.149:6809/2969709367]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi149", "root=default"]}]: dispatch 2023-09-23T20:10:43.850 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:43 smithi149 ceph-mon[100764]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi149", "root=default"]}]: dispatch 2023-09-23T20:10:43.851 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 23 20:10:43 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-5[114166]: 2023-09-23T20:10:43.815+0000 7f1171a48700 -1 osd.5 0 waiting for initial osdmap 2023-09-23T20:10:43.851 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 23 20:10:43 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-5[114166]: 2023-09-23T20:10:43.819+0000 7f116c843700 -1 osd.5 37 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T20:10:44.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:43 smithi018 ceph-mon[109137]: pgmap v112: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T20:10:44.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:43 smithi018 ceph-mon[109137]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-09-23T20:10:44.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:43 smithi018 ceph-mon[109137]: osdmap e36: 6 total, 5 up, 6 in 2023-09-23T20:10:44.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:43 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T20:10:44.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:43 smithi018 ceph-mon[109137]: from='osd.5 [v2:172.21.15.149:6808/2969709367,v1:172.21.15.149:6809/2969709367]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi149", "root=default"]}]: dispatch 2023-09-23T20:10:44.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:43 smithi018 ceph-mon[109137]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi149", "root=default"]}]: dispatch 2023-09-23T20:10:44.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:43 smithi018 ceph-mon[95889]: pgmap v112: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T20:10:44.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:43 smithi018 ceph-mon[95889]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-09-23T20:10:44.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:43 smithi018 ceph-mon[95889]: osdmap e36: 6 total, 5 up, 6 in 2023-09-23T20:10:44.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:43 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T20:10:44.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:43 smithi018 ceph-mon[95889]: from='osd.5 [v2:172.21.15.149:6808/2969709367,v1:172.21.15.149:6809/2969709367]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi149", "root=default"]}]: dispatch 2023-09-23T20:10:44.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:43 smithi018 ceph-mon[95889]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi149", "root=default"]}]: dispatch 2023-09-23T20:10:44.595 INFO:teuthology.orchestra.run.smithi149.stderr:--> Zapping: /dev/vg_nvme/lv_2 2023-09-23T20:10:44.595 INFO:teuthology.orchestra.run.smithi149.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2023-09-23T20:10:44.595 INFO:teuthology.orchestra.run.smithi149.stderr: stderr: 10+0 records in 2023-09-23T20:10:44.595 INFO:teuthology.orchestra.run.smithi149.stderr:10+0 records out 2023-09-23T20:10:44.595 INFO:teuthology.orchestra.run.smithi149.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0112846 s, 929 MB/s 2023-09-23T20:10:44.596 INFO:teuthology.orchestra.run.smithi149.stderr:--> Zapping successful for: 2023-09-23T20:10:45.057 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:44 smithi149 ceph-mon[100764]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi149", "root=default"]}]': finished 2023-09-23T20:10:45.057 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:44 smithi149 ceph-mon[100764]: osdmap e37: 6 total, 5 up, 6 in 2023-09-23T20:10:45.058 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:44 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T20:10:45.058 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:44 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T20:10:45.058 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:44 smithi149 ceph-mon[100764]: osdmap e38: 6 total, 5 up, 6 in 2023-09-23T20:10:45.058 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:44 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T20:10:45.058 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:44 smithi149 ceph-mon[100764]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T20:10:45.064 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph orch daemon add osd smithi149:vg_nvme/lv_2 2023-09-23T20:10:45.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:44 smithi018 ceph-mon[95889]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi149", "root=default"]}]': finished 2023-09-23T20:10:45.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:44 smithi018 ceph-mon[95889]: osdmap e37: 6 total, 5 up, 6 in 2023-09-23T20:10:45.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:44 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T20:10:45.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:44 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T20:10:45.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:44 smithi018 ceph-mon[95889]: osdmap e38: 6 total, 5 up, 6 in 2023-09-23T20:10:45.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:44 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T20:10:45.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:44 smithi018 ceph-mon[95889]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T20:10:45.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:44 smithi018 ceph-mon[109137]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi149", "root=default"]}]': finished 2023-09-23T20:10:45.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:44 smithi018 ceph-mon[109137]: osdmap e37: 6 total, 5 up, 6 in 2023-09-23T20:10:45.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:44 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T20:10:45.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:44 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T20:10:45.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:44 smithi018 ceph-mon[109137]: osdmap e38: 6 total, 5 up, 6 in 2023-09-23T20:10:45.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:44 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T20:10:45.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:44 smithi018 ceph-mon[109137]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T20:10:45.666 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:10:46.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:45 smithi018 ceph-mon[95889]: purged_snaps scrub starts 2023-09-23T20:10:46.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:45 smithi018 ceph-mon[95889]: purged_snaps scrub ok 2023-09-23T20:10:46.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:45 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T20:10:46.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:45 smithi018 ceph-mon[95889]: osd.5 [v2:172.21.15.149:6808/2969709367,v1:172.21.15.149:6809/2969709367] boot 2023-09-23T20:10:46.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:45 smithi018 ceph-mon[95889]: osdmap e39: 6 total, 6 up, 6 in 2023-09-23T20:10:46.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:45 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T20:10:46.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:45 smithi018 ceph-mon[95889]: Detected new or changed devices on smithi149 2023-09-23T20:10:46.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:45 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:10:46.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:45 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:10:46.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:45 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:10:46.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:45 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:10:46.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:45 smithi018 ceph-mon[95889]: Adjusting osd_memory_target on smithi149 to 2939M 2023-09-23T20:10:46.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:45 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:10:46.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:45 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:10:46.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:45 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:10:46.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:45 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:10:46.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:45 smithi018 ceph-mon[109137]: purged_snaps scrub starts 2023-09-23T20:10:46.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:45 smithi018 ceph-mon[109137]: purged_snaps scrub ok 2023-09-23T20:10:46.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:45 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T20:10:46.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:45 smithi018 ceph-mon[109137]: osd.5 [v2:172.21.15.149:6808/2969709367,v1:172.21.15.149:6809/2969709367] boot 2023-09-23T20:10:46.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:45 smithi018 ceph-mon[109137]: osdmap e39: 6 total, 6 up, 6 in 2023-09-23T20:10:46.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:45 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T20:10:46.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:45 smithi018 ceph-mon[109137]: Detected new or changed devices on smithi149 2023-09-23T20:10:46.144 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:45 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:10:46.144 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:45 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:10:46.144 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:45 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:10:46.144 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:45 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:10:46.144 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:45 smithi018 ceph-mon[109137]: Adjusting osd_memory_target on smithi149 to 2939M 2023-09-23T20:10:46.144 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:45 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:10:46.144 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:45 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:10:46.144 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:45 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:10:46.144 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:45 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:10:46.164 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:45 smithi149 ceph-mon[100764]: purged_snaps scrub starts 2023-09-23T20:10:46.164 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:45 smithi149 ceph-mon[100764]: purged_snaps scrub ok 2023-09-23T20:10:46.164 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:45 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T20:10:46.164 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:45 smithi149 ceph-mon[100764]: osd.5 [v2:172.21.15.149:6808/2969709367,v1:172.21.15.149:6809/2969709367] boot 2023-09-23T20:10:46.164 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:45 smithi149 ceph-mon[100764]: osdmap e39: 6 total, 6 up, 6 in 2023-09-23T20:10:46.164 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:45 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T20:10:46.164 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:45 smithi149 ceph-mon[100764]: Detected new or changed devices on smithi149 2023-09-23T20:10:46.164 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:45 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:10:46.164 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:45 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:10:46.165 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:45 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:10:46.165 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:45 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:10:46.165 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:45 smithi149 ceph-mon[100764]: Adjusting osd_memory_target on smithi149 to 2939M 2023-09-23T20:10:46.165 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:45 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:10:46.165 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:45 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:10:46.165 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:45 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:10:46.165 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:45 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:10:46.965 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:46 smithi149 ceph-mon[100764]: osdmap e40: 6 total, 6 up, 6 in 2023-09-23T20:10:46.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:46 smithi149 ceph-mon[100764]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T20:10:47.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:46 smithi018 ceph-mon[95889]: osdmap e40: 6 total, 6 up, 6 in 2023-09-23T20:10:47.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:46 smithi018 ceph-mon[95889]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T20:10:47.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:46 smithi018 ceph-mon[109137]: osdmap e40: 6 total, 6 up, 6 in 2023-09-23T20:10:47.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:46 smithi018 ceph-mon[109137]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T20:10:48.130 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:47 smithi149 ceph-mon[100764]: osdmap e41: 6 total, 6 up, 6 in 2023-09-23T20:10:48.131 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:47 smithi149 ceph-mon[100764]: from='client.24274 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi149:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:10:48.131 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:47 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T20:10:48.131 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:47 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T20:10:48.131 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:47 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:10:48.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:47 smithi018 ceph-mon[109137]: osdmap e41: 6 total, 6 up, 6 in 2023-09-23T20:10:48.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:47 smithi018 ceph-mon[109137]: from='client.24274 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi149:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:10:48.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:47 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T20:10:48.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:47 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T20:10:48.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:47 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:10:48.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:47 smithi018 ceph-mon[95889]: osdmap e41: 6 total, 6 up, 6 in 2023-09-23T20:10:48.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:47 smithi018 ceph-mon[95889]: from='client.24274 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi149:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:10:48.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:47 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T20:10:48.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:47 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T20:10:48.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:47 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:10:49.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:48 smithi018 ceph-mon[95889]: osdmap e42: 6 total, 6 up, 6 in 2023-09-23T20:10:49.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:48 smithi018 ceph-mon[95889]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T20:10:49.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:48 smithi018 ceph-mon[109137]: osdmap e42: 6 total, 6 up, 6 in 2023-09-23T20:10:49.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:48 smithi018 ceph-mon[109137]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T20:10:49.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:48 smithi149 ceph-mon[100764]: osdmap e42: 6 total, 6 up, 6 in 2023-09-23T20:10:49.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:48 smithi149 ceph-mon[100764]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T20:10:50.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:49 smithi018 ceph-mon[109137]: osdmap e43: 6 total, 6 up, 6 in 2023-09-23T20:10:50.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:49 smithi018 ceph-mon[109137]: from='client.? 172.21.15.149:0/551840162' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "27d2d2a8-e8c6-4a05-a85e-bd54376cf4c1"}]: dispatch 2023-09-23T20:10:50.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:49 smithi018 ceph-mon[109137]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "27d2d2a8-e8c6-4a05-a85e-bd54376cf4c1"}]: dispatch 2023-09-23T20:10:50.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:49 smithi018 ceph-mon[109137]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "27d2d2a8-e8c6-4a05-a85e-bd54376cf4c1"}]': finished 2023-09-23T20:10:50.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:49 smithi018 ceph-mon[109137]: osdmap e44: 7 total, 6 up, 7 in 2023-09-23T20:10:50.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:49 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T20:10:50.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:49 smithi018 ceph-mon[95889]: osdmap e43: 6 total, 6 up, 6 in 2023-09-23T20:10:50.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:49 smithi018 ceph-mon[95889]: from='client.? 172.21.15.149:0/551840162' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "27d2d2a8-e8c6-4a05-a85e-bd54376cf4c1"}]: dispatch 2023-09-23T20:10:50.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:49 smithi018 ceph-mon[95889]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "27d2d2a8-e8c6-4a05-a85e-bd54376cf4c1"}]: dispatch 2023-09-23T20:10:50.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:49 smithi018 ceph-mon[95889]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "27d2d2a8-e8c6-4a05-a85e-bd54376cf4c1"}]': finished 2023-09-23T20:10:50.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:49 smithi018 ceph-mon[95889]: osdmap e44: 7 total, 6 up, 7 in 2023-09-23T20:10:50.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:49 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T20:10:50.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:49 smithi149 ceph-mon[100764]: osdmap e43: 6 total, 6 up, 6 in 2023-09-23T20:10:50.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:49 smithi149 ceph-mon[100764]: from='client.? 172.21.15.149:0/551840162' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "27d2d2a8-e8c6-4a05-a85e-bd54376cf4c1"}]: dispatch 2023-09-23T20:10:50.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:49 smithi149 ceph-mon[100764]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "27d2d2a8-e8c6-4a05-a85e-bd54376cf4c1"}]: dispatch 2023-09-23T20:10:50.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:49 smithi149 ceph-mon[100764]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "27d2d2a8-e8c6-4a05-a85e-bd54376cf4c1"}]': finished 2023-09-23T20:10:50.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:49 smithi149 ceph-mon[100764]: osdmap e44: 7 total, 6 up, 7 in 2023-09-23T20:10:50.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:49 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T20:10:51.140 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:50 smithi018 ceph-mon[95889]: from='client.? 172.21.15.149:0/3499574336' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T20:10:51.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:50 smithi018 ceph-mon[95889]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T20:10:51.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:50 smithi018 ceph-mon[109137]: from='client.? 172.21.15.149:0/3499574336' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T20:10:51.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:50 smithi018 ceph-mon[109137]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T20:10:51.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:50 smithi149 ceph-mon[100764]: from='client.? 172.21.15.149:0/3499574336' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T20:10:51.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:50 smithi149 ceph-mon[100764]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T20:10:53.965 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:53 smithi149 ceph-mon[100764]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T20:10:54.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:53 smithi018 ceph-mon[109137]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T20:10:54.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:53 smithi018 ceph-mon[95889]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T20:10:55.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:54 smithi018 ceph-mon[95889]: pgmap v127: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T20:10:55.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:54 smithi018 ceph-mon[109137]: pgmap v127: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T20:10:55.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:54 smithi149 ceph-mon[100764]: pgmap v127: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T20:10:56.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:55 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-09-23T20:10:56.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:55 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:10:56.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:55 smithi018 ceph-mon[95889]: Deploying daemon osd.6 on smithi149 2023-09-23T20:10:56.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:55 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-09-23T20:10:56.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:55 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:10:56.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:55 smithi018 ceph-mon[109137]: Deploying daemon osd.6 on smithi149 2023-09-23T20:10:56.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:55 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-09-23T20:10:56.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:55 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:10:56.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:55 smithi149 ceph-mon[100764]: Deploying daemon osd.6 on smithi149 2023-09-23T20:10:56.923 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:56 smithi149 ceph-mon[100764]: pgmap v128: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T20:10:57.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:56 smithi018 ceph-mon[95889]: pgmap v128: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T20:10:57.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:56 smithi018 ceph-mon[109137]: pgmap v128: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T20:11:00.095 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:59 smithi149 ceph-mon[100764]: pgmap v129: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T20:11:00.095 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:59 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:11:00.096 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:59 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:11:00.096 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:10:59 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:11:00.140 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:59 smithi018 ceph-mon[109137]: pgmap v129: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T20:11:00.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:59 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:11:00.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:59 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:11:00.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:10:59 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:11:00.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:59 smithi018 ceph-mon[95889]: pgmap v129: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T20:11:00.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:59 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:11:00.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:59 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:11:00.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:10:59 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:11:01.485 INFO:teuthology.orchestra.run.smithi149.stdout:Created osd(s) 6 on host 'smithi149' 2023-09-23T20:11:01.949 DEBUG:teuthology.orchestra.run.smithi149:osd.6> sudo journalctl -f -n 0 -u ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7@osd.6.service 2023-09-23T20:11:01.951 INFO:tasks.cephadm:Deploying osd.7 on smithi149 with /dev/vg_nvme/lv_1... 2023-09-23T20:11:01.952 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2023-09-23T20:11:01.977 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:01 smithi149 ceph-mon[100764]: pgmap v130: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T20:11:01.977 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:01 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:11:01.977 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:01 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:11:01.977 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:01 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:11:01.977 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:01 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:11:01.980 INFO:journalctl@ceph.osd.6.smithi149.stdout:-- Logs begin at Sat 2023-09-23 19:50:38 UTC. -- 2023-09-23T20:11:02.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:01 smithi018 ceph-mon[95889]: pgmap v130: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T20:11:02.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:01 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:11:02.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:01 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:11:02.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:01 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:11:02.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:01 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:11:02.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:01 smithi018 ceph-mon[109137]: pgmap v130: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T20:11:02.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:01 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:11:02.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:01 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:11:02.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:01 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:11:02.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:01 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:11:02.396 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 23 20:11:02 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-6[121170]: 2023-09-23T20:11:02.124+0000 7f82d53b1540 -1 osd.6 0 log_to_monitors true 2023-09-23T20:11:02.899 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:11:02.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:02 smithi149 ceph-mon[100764]: from='osd.6 [v2:172.21.15.149:6816/1291453593,v1:172.21.15.149:6817/1291453593]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-09-23T20:11:02.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:02 smithi149 ceph-mon[100764]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-09-23T20:11:03.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:02 smithi018 ceph-mon[95889]: from='osd.6 [v2:172.21.15.149:6816/1291453593,v1:172.21.15.149:6817/1291453593]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-09-23T20:11:03.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:02 smithi018 ceph-mon[95889]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-09-23T20:11:03.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:02 smithi018 ceph-mon[109137]: from='osd.6 [v2:172.21.15.149:6816/1291453593,v1:172.21.15.149:6817/1291453593]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-09-23T20:11:03.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:02 smithi018 ceph-mon[109137]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-09-23T20:11:03.931 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:03 smithi149 ceph-mon[100764]: pgmap v131: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T20:11:03.931 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:03 smithi149 ceph-mon[100764]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-09-23T20:11:03.931 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:03 smithi149 ceph-mon[100764]: osdmap e45: 7 total, 6 up, 7 in 2023-09-23T20:11:03.932 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:03 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T20:11:03.932 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:03 smithi149 ceph-mon[100764]: from='osd.6 [v2:172.21.15.149:6816/1291453593,v1:172.21.15.149:6817/1291453593]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi149", "root=default"]}]: dispatch 2023-09-23T20:11:03.932 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:03 smithi149 ceph-mon[100764]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi149", "root=default"]}]: dispatch 2023-09-23T20:11:03.932 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 23 20:11:03 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-6[121170]: 2023-09-23T20:11:03.740+0000 7f82cbb97700 -1 osd.6 0 waiting for initial osdmap 2023-09-23T20:11:03.932 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 23 20:11:03 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-6[121170]: 2023-09-23T20:11:03.747+0000 7f82c498e700 -1 osd.6 46 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T20:11:04.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:03 smithi018 ceph-mon[95889]: pgmap v131: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T20:11:04.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:03 smithi018 ceph-mon[95889]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-09-23T20:11:04.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:03 smithi018 ceph-mon[95889]: osdmap e45: 7 total, 6 up, 7 in 2023-09-23T20:11:04.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:03 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T20:11:04.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:03 smithi018 ceph-mon[95889]: from='osd.6 [v2:172.21.15.149:6816/1291453593,v1:172.21.15.149:6817/1291453593]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi149", "root=default"]}]: dispatch 2023-09-23T20:11:04.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:03 smithi018 ceph-mon[95889]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi149", "root=default"]}]: dispatch 2023-09-23T20:11:04.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:03 smithi018 ceph-mon[109137]: pgmap v131: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T20:11:04.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:03 smithi018 ceph-mon[109137]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-09-23T20:11:04.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:03 smithi018 ceph-mon[109137]: osdmap e45: 7 total, 6 up, 7 in 2023-09-23T20:11:04.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:03 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T20:11:04.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:03 smithi018 ceph-mon[109137]: from='osd.6 [v2:172.21.15.149:6816/1291453593,v1:172.21.15.149:6817/1291453593]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi149", "root=default"]}]: dispatch 2023-09-23T20:11:04.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:03 smithi018 ceph-mon[109137]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi149", "root=default"]}]: dispatch 2023-09-23T20:11:04.405 INFO:teuthology.orchestra.run.smithi149.stderr:--> Zapping: /dev/vg_nvme/lv_1 2023-09-23T20:11:04.406 INFO:teuthology.orchestra.run.smithi149.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2023-09-23T20:11:04.406 INFO:teuthology.orchestra.run.smithi149.stderr: stderr: 10+0 records in 2023-09-23T20:11:04.406 INFO:teuthology.orchestra.run.smithi149.stderr:10+0 records out 2023-09-23T20:11:04.406 INFO:teuthology.orchestra.run.smithi149.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0137094 s, 765 MB/s 2023-09-23T20:11:04.406 INFO:teuthology.orchestra.run.smithi149.stderr: stderr: 2023-09-23T20:11:04.406 INFO:teuthology.orchestra.run.smithi149.stderr:--> Zapping successful for: 2023-09-23T20:11:04.866 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:04 smithi149 ceph-mon[100764]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi149", "root=default"]}]': finished 2023-09-23T20:11:04.866 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:04 smithi149 ceph-mon[100764]: osdmap e46: 7 total, 6 up, 7 in 2023-09-23T20:11:04.867 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:04 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T20:11:04.867 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:04 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T20:11:04.867 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:04 smithi149 ceph-mon[100764]: osd.6 [v2:172.21.15.149:6816/1291453593,v1:172.21.15.149:6817/1291453593] boot 2023-09-23T20:11:04.867 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:04 smithi149 ceph-mon[100764]: osdmap e47: 7 total, 7 up, 7 in 2023-09-23T20:11:04.867 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:04 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T20:11:04.868 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph orch daemon add osd smithi149:vg_nvme/lv_1 2023-09-23T20:11:05.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:04 smithi018 ceph-mon[109137]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi149", "root=default"]}]': finished 2023-09-23T20:11:05.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:04 smithi018 ceph-mon[109137]: osdmap e46: 7 total, 6 up, 7 in 2023-09-23T20:11:05.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:04 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T20:11:05.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:04 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T20:11:05.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:04 smithi018 ceph-mon[109137]: osd.6 [v2:172.21.15.149:6816/1291453593,v1:172.21.15.149:6817/1291453593] boot 2023-09-23T20:11:05.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:04 smithi018 ceph-mon[109137]: osdmap e47: 7 total, 7 up, 7 in 2023-09-23T20:11:05.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:04 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T20:11:05.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:04 smithi018 ceph-mon[95889]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi149", "root=default"]}]': finished 2023-09-23T20:11:05.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:04 smithi018 ceph-mon[95889]: osdmap e46: 7 total, 6 up, 7 in 2023-09-23T20:11:05.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:04 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T20:11:05.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:04 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T20:11:05.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:04 smithi018 ceph-mon[95889]: osd.6 [v2:172.21.15.149:6816/1291453593,v1:172.21.15.149:6817/1291453593] boot 2023-09-23T20:11:05.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:04 smithi018 ceph-mon[95889]: osdmap e47: 7 total, 7 up, 7 in 2023-09-23T20:11:05.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:04 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T20:11:05.509 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:11:06.044 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:05 smithi149 ceph-mon[100764]: purged_snaps scrub starts 2023-09-23T20:11:06.044 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:05 smithi149 ceph-mon[100764]: purged_snaps scrub ok 2023-09-23T20:11:06.044 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:05 smithi149 ceph-mon[100764]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 36 MiB used, 536 GiB / 536 GiB avail 2023-09-23T20:11:06.044 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:05 smithi149 ceph-mon[100764]: osdmap e48: 7 total, 7 up, 7 in 2023-09-23T20:11:06.044 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:05 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:11:06.044 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:05 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:11:06.044 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:05 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:11:06.045 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:05 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:11:06.045 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:05 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:11:06.045 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:05 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:11:06.045 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:05 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:11:06.045 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:05 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:11:06.045 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:05 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:11:06.045 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:05 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:11:06.045 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:05 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:11:06.045 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:05 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:11:06.045 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:05 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:11:06.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:05 smithi018 ceph-mon[95889]: purged_snaps scrub starts 2023-09-23T20:11:06.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:05 smithi018 ceph-mon[95889]: purged_snaps scrub ok 2023-09-23T20:11:06.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:05 smithi018 ceph-mon[95889]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 36 MiB used, 536 GiB / 536 GiB avail 2023-09-23T20:11:06.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:05 smithi018 ceph-mon[95889]: osdmap e48: 7 total, 7 up, 7 in 2023-09-23T20:11:06.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:05 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:11:06.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:05 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:11:06.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:05 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:11:06.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:05 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:11:06.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:05 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:11:06.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:05 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:11:06.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:05 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:11:06.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:05 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:11:06.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:05 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:11:06.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:05 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:11:06.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:05 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:11:06.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:05 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:11:06.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:05 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:11:06.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:05 smithi018 ceph-mon[109137]: purged_snaps scrub starts 2023-09-23T20:11:06.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:05 smithi018 ceph-mon[109137]: purged_snaps scrub ok 2023-09-23T20:11:06.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:05 smithi018 ceph-mon[109137]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 36 MiB used, 536 GiB / 536 GiB avail 2023-09-23T20:11:06.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:05 smithi018 ceph-mon[109137]: osdmap e48: 7 total, 7 up, 7 in 2023-09-23T20:11:06.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:05 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:11:06.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:05 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:11:06.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:05 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:11:06.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:05 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:11:06.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:05 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:11:06.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:05 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:11:06.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:05 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:11:06.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:05 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:11:06.144 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:05 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:11:06.144 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:05 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:11:06.144 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:05 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:11:06.144 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:05 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:11:06.144 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:05 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:11:06.965 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:06 smithi149 ceph-mon[100764]: Detected new or changed devices on smithi149 2023-09-23T20:11:06.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:06 smithi149 ceph-mon[100764]: Adjusting osd_memory_target on smithi149 to 1959M 2023-09-23T20:11:06.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:06 smithi149 ceph-mon[100764]: osdmap e49: 7 total, 7 up, 7 in 2023-09-23T20:11:07.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:06 smithi018 ceph-mon[95889]: Detected new or changed devices on smithi149 2023-09-23T20:11:07.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:06 smithi018 ceph-mon[95889]: Adjusting osd_memory_target on smithi149 to 1959M 2023-09-23T20:11:07.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:06 smithi018 ceph-mon[95889]: osdmap e49: 7 total, 7 up, 7 in 2023-09-23T20:11:07.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:06 smithi018 ceph-mon[109137]: Detected new or changed devices on smithi149 2023-09-23T20:11:07.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:06 smithi018 ceph-mon[109137]: Adjusting osd_memory_target on smithi149 to 1959M 2023-09-23T20:11:07.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:06 smithi018 ceph-mon[109137]: osdmap e49: 7 total, 7 up, 7 in 2023-09-23T20:11:08.010 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:07 smithi149 ceph-mon[100764]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2023-09-23T20:11:08.011 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:07 smithi149 ceph-mon[100764]: osdmap e50: 7 total, 7 up, 7 in 2023-09-23T20:11:08.011 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:07 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T20:11:08.011 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:07 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T20:11:08.011 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:07 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:11:08.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:07 smithi018 ceph-mon[95889]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2023-09-23T20:11:08.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:07 smithi018 ceph-mon[95889]: osdmap e50: 7 total, 7 up, 7 in 2023-09-23T20:11:08.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:07 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T20:11:08.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:07 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T20:11:08.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:07 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:11:08.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:07 smithi018 ceph-mon[109137]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2023-09-23T20:11:08.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:07 smithi018 ceph-mon[109137]: osdmap e50: 7 total, 7 up, 7 in 2023-09-23T20:11:08.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:07 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T20:11:08.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:07 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T20:11:08.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:07 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:11:09.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:08 smithi018 ceph-mon[109137]: from='client.24301 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi149:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:11:09.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:08 smithi018 ceph-mon[109137]: osdmap e51: 7 total, 7 up, 7 in 2023-09-23T20:11:09.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:08 smithi018 ceph-mon[95889]: from='client.24301 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi149:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:11:09.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:08 smithi018 ceph-mon[95889]: osdmap e51: 7 total, 7 up, 7 in 2023-09-23T20:11:09.215 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:08 smithi149 ceph-mon[100764]: from='client.24301 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi149:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:11:09.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:08 smithi149 ceph-mon[100764]: osdmap e51: 7 total, 7 up, 7 in 2023-09-23T20:11:10.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:09 smithi018 ceph-mon[95889]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2023-09-23T20:11:10.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:09 smithi018 ceph-mon[95889]: from='client.? 172.21.15.149:0/1703877464' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9da636f7-995c-4f29-9264-70d09c6f66c6"}]: dispatch 2023-09-23T20:11:10.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:09 smithi018 ceph-mon[95889]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9da636f7-995c-4f29-9264-70d09c6f66c6"}]: dispatch 2023-09-23T20:11:10.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:09 smithi018 ceph-mon[95889]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "9da636f7-995c-4f29-9264-70d09c6f66c6"}]': finished 2023-09-23T20:11:10.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:09 smithi018 ceph-mon[95889]: osdmap e52: 8 total, 7 up, 8 in 2023-09-23T20:11:10.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:09 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T20:11:10.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:09 smithi018 ceph-mon[109137]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2023-09-23T20:11:10.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:09 smithi018 ceph-mon[109137]: from='client.? 172.21.15.149:0/1703877464' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9da636f7-995c-4f29-9264-70d09c6f66c6"}]: dispatch 2023-09-23T20:11:10.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:09 smithi018 ceph-mon[109137]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9da636f7-995c-4f29-9264-70d09c6f66c6"}]: dispatch 2023-09-23T20:11:10.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:09 smithi018 ceph-mon[109137]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "9da636f7-995c-4f29-9264-70d09c6f66c6"}]': finished 2023-09-23T20:11:10.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:09 smithi018 ceph-mon[109137]: osdmap e52: 8 total, 7 up, 8 in 2023-09-23T20:11:10.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:09 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T20:11:10.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:09 smithi149 ceph-mon[100764]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2023-09-23T20:11:10.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:09 smithi149 ceph-mon[100764]: from='client.? 172.21.15.149:0/1703877464' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9da636f7-995c-4f29-9264-70d09c6f66c6"}]: dispatch 2023-09-23T20:11:10.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:09 smithi149 ceph-mon[100764]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9da636f7-995c-4f29-9264-70d09c6f66c6"}]: dispatch 2023-09-23T20:11:10.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:09 smithi149 ceph-mon[100764]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "9da636f7-995c-4f29-9264-70d09c6f66c6"}]': finished 2023-09-23T20:11:10.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:09 smithi149 ceph-mon[100764]: osdmap e52: 8 total, 7 up, 8 in 2023-09-23T20:11:10.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:09 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T20:11:11.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:10 smithi018 ceph-mon[109137]: from='client.? 172.21.15.149:0/3115645027' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T20:11:11.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:10 smithi018 ceph-mon[95889]: from='client.? 172.21.15.149:0/3115645027' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T20:11:11.215 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:10 smithi149 ceph-mon[100764]: from='client.? 172.21.15.149:0/3115645027' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T20:11:12.120 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:11 smithi149 ceph-mon[100764]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 119 KiB/s, 0 objects/s recovering 2023-09-23T20:11:12.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:11 smithi018 ceph-mon[109137]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 119 KiB/s, 0 objects/s recovering 2023-09-23T20:11:12.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:11 smithi018 ceph-mon[95889]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 119 KiB/s, 0 objects/s recovering 2023-09-23T20:11:13.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:13 smithi149 ceph-mon[100764]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 96 KiB/s, 0 objects/s recovering 2023-09-23T20:11:14.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:13 smithi018 ceph-mon[109137]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 96 KiB/s, 0 objects/s recovering 2023-09-23T20:11:14.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:13 smithi018 ceph-mon[95889]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 96 KiB/s, 0 objects/s recovering 2023-09-23T20:11:15.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:14 smithi018 ceph-mon[95889]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 74 KiB/s, 0 objects/s recovering 2023-09-23T20:11:15.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:14 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-09-23T20:11:15.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:14 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:11:15.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:14 smithi018 ceph-mon[95889]: Deploying daemon osd.7 on smithi149 2023-09-23T20:11:15.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:14 smithi018 ceph-mon[109137]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 74 KiB/s, 0 objects/s recovering 2023-09-23T20:11:15.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:14 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-09-23T20:11:15.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:14 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:11:15.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:14 smithi018 ceph-mon[109137]: Deploying daemon osd.7 on smithi149 2023-09-23T20:11:15.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:14 smithi149 ceph-mon[100764]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 74 KiB/s, 0 objects/s recovering 2023-09-23T20:11:15.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:14 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-09-23T20:11:15.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:14 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:11:15.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:14 smithi149 ceph-mon[100764]: Deploying daemon osd.7 on smithi149 2023-09-23T20:11:17.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:17 smithi149 ceph-mon[100764]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 65 KiB/s, 0 objects/s recovering 2023-09-23T20:11:18.140 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:17 smithi018 ceph-mon[95889]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 65 KiB/s, 0 objects/s recovering 2023-09-23T20:11:18.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:17 smithi018 ceph-mon[109137]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 65 KiB/s, 0 objects/s recovering 2023-09-23T20:11:19.965 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:19 smithi149 ceph-mon[100764]: pgmap v147: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-09-23T20:11:19.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:19 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:11:19.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:19 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:11:19.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:19 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:11:20.140 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:19 smithi018 ceph-mon[95889]: pgmap v147: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-09-23T20:11:20.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:19 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:11:20.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:19 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:11:20.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:19 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:11:20.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:19 smithi018 ceph-mon[109137]: pgmap v147: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-09-23T20:11:20.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:19 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:11:20.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:19 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:11:20.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:19 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:11:21.405 INFO:teuthology.orchestra.run.smithi149.stdout:Created osd(s) 7 on host 'smithi149' 2023-09-23T20:11:21.800 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:21 smithi149 ceph-mon[100764]: pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 50 KiB/s, 0 objects/s recovering 2023-09-23T20:11:21.800 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:21 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:11:21.800 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:21 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:11:21.800 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:21 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:11:21.800 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:21 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:11:21.802 DEBUG:teuthology.orchestra.run.smithi149:osd.7> sudo journalctl -f -n 0 -u ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7@osd.7.service 2023-09-23T20:11:21.804 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2023-09-23T20:11:21.804 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd stat -f json 2023-09-23T20:11:21.835 INFO:journalctl@ceph.osd.7.smithi149.stdout:-- Logs begin at Sat 2023-09-23 19:50:38 UTC. -- 2023-09-23T20:11:22.140 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:21 smithi018 ceph-mon[95889]: pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 50 KiB/s, 0 objects/s recovering 2023-09-23T20:11:22.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:21 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:11:22.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:21 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:11:22.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:21 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:11:22.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:21 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:11:22.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:21 smithi018 ceph-mon[109137]: pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 50 KiB/s, 0 objects/s recovering 2023-09-23T20:11:22.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:21 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:11:22.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:21 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:11:22.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:21 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:11:22.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:21 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:11:22.215 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 23 20:11:22 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-7[128937]: 2023-09-23T20:11:22.048+0000 7f14f7c86540 -1 osd.7 0 log_to_monitors true 2023-09-23T20:11:22.421 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:11:22.890 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:22 smithi018 ceph-mon[95889]: from='osd.7 [v2:172.21.15.149:6824/3117985178,v1:172.21.15.149:6825/3117985178]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-09-23T20:11:22.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:22 smithi018 ceph-mon[95889]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-09-23T20:11:22.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:22 smithi018 ceph-mon[109137]: from='osd.7 [v2:172.21.15.149:6824/3117985178,v1:172.21.15.149:6825/3117985178]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-09-23T20:11:22.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:22 smithi018 ceph-mon[109137]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-09-23T20:11:23.006 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:22 smithi149 ceph-mon[100764]: from='osd.7 [v2:172.21.15.149:6824/3117985178,v1:172.21.15.149:6825/3117985178]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-09-23T20:11:23.006 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:22 smithi149 ceph-mon[100764]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-09-23T20:11:23.745 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:23 smithi018 ceph-mon[109137]: pgmap v149: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2023-09-23T20:11:23.745 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:23 smithi018 ceph-mon[109137]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-09-23T20:11:23.745 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:23 smithi018 ceph-mon[109137]: osdmap e53: 8 total, 7 up, 8 in 2023-09-23T20:11:23.745 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:23 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T20:11:23.745 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:23 smithi018 ceph-mon[109137]: from='osd.7 [v2:172.21.15.149:6824/3117985178,v1:172.21.15.149:6825/3117985178]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi149", "root=default"]}]: dispatch 2023-09-23T20:11:23.746 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:23 smithi018 ceph-mon[109137]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi149", "root=default"]}]: dispatch 2023-09-23T20:11:23.746 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:23 smithi018 ceph-mon[95889]: pgmap v149: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2023-09-23T20:11:23.746 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:23 smithi018 ceph-mon[95889]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-09-23T20:11:23.746 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:23 smithi018 ceph-mon[95889]: osdmap e53: 8 total, 7 up, 8 in 2023-09-23T20:11:23.746 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:23 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T20:11:23.746 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:23 smithi018 ceph-mon[95889]: from='osd.7 [v2:172.21.15.149:6824/3117985178,v1:172.21.15.149:6825/3117985178]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi149", "root=default"]}]: dispatch 2023-09-23T20:11:23.746 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:23 smithi018 ceph-mon[95889]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi149", "root=default"]}]: dispatch 2023-09-23T20:11:23.860 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:11:24.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:23 smithi149 ceph-mon[100764]: pgmap v149: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2023-09-23T20:11:24.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:23 smithi149 ceph-mon[100764]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-09-23T20:11:24.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:23 smithi149 ceph-mon[100764]: osdmap e53: 8 total, 7 up, 8 in 2023-09-23T20:11:24.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:23 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T20:11:24.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:23 smithi149 ceph-mon[100764]: from='osd.7 [v2:172.21.15.149:6824/3117985178,v1:172.21.15.149:6825/3117985178]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi149", "root=default"]}]: dispatch 2023-09-23T20:11:24.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:23 smithi149 ceph-mon[100764]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi149", "root=default"]}]: dispatch 2023-09-23T20:11:24.217 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 23 20:11:23 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-7[128937]: 2023-09-23T20:11:23.746+0000 7f14efc6f700 -1 osd.7 0 waiting for initial osdmap 2023-09-23T20:11:24.217 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 23 20:11:23 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-7[128937]: 2023-09-23T20:11:23.754+0000 7f14e9267700 -1 osd.7 54 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T20:11:24.278 INFO:teuthology.orchestra.run.smithi018.stdout:{"epoch":55,"num_osds":8,"num_up_osds":8,"osd_up_since":1695499883,"num_in_osds":8,"osd_in_since":1695499869,"num_remapped_pgs":1} 2023-09-23T20:11:24.279 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd dump --format=json 2023-09-23T20:11:24.883 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:11:24.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:24 smithi149 ceph-mon[100764]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi149", "root=default"]}]': finished 2023-09-23T20:11:24.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:24 smithi149 ceph-mon[100764]: osdmap e54: 8 total, 7 up, 8 in 2023-09-23T20:11:24.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:24 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T20:11:24.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:24 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T20:11:24.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:24 smithi149 ceph-mon[100764]: osd.7 [v2:172.21.15.149:6824/3117985178,v1:172.21.15.149:6825/3117985178] boot 2023-09-23T20:11:24.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:24 smithi149 ceph-mon[100764]: osdmap e55: 8 total, 8 up, 8 in 2023-09-23T20:11:24.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:24 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T20:11:24.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:24 smithi149 ceph-mon[100764]: from='client.? 172.21.15.18:0/935137142' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-09-23T20:11:25.091 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:24 smithi018 ceph-mon[109137]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi149", "root=default"]}]': finished 2023-09-23T20:11:25.091 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:24 smithi018 ceph-mon[109137]: osdmap e54: 8 total, 7 up, 8 in 2023-09-23T20:11:25.091 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:24 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T20:11:25.091 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:24 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T20:11:25.091 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:24 smithi018 ceph-mon[109137]: osd.7 [v2:172.21.15.149:6824/3117985178,v1:172.21.15.149:6825/3117985178] boot 2023-09-23T20:11:25.091 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:24 smithi018 ceph-mon[109137]: osdmap e55: 8 total, 8 up, 8 in 2023-09-23T20:11:25.092 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:24 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T20:11:25.092 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:24 smithi018 ceph-mon[109137]: from='client.? 172.21.15.18:0/935137142' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-09-23T20:11:25.092 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:24 smithi018 ceph-mon[95889]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi149", "root=default"]}]': finished 2023-09-23T20:11:25.092 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:24 smithi018 ceph-mon[95889]: osdmap e54: 8 total, 7 up, 8 in 2023-09-23T20:11:25.092 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:24 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T20:11:25.092 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:24 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T20:11:25.092 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:24 smithi018 ceph-mon[95889]: osd.7 [v2:172.21.15.149:6824/3117985178,v1:172.21.15.149:6825/3117985178] boot 2023-09-23T20:11:25.092 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:24 smithi018 ceph-mon[95889]: osdmap e55: 8 total, 8 up, 8 in 2023-09-23T20:11:25.093 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:24 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T20:11:25.093 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:24 smithi018 ceph-mon[95889]: from='client.? 172.21.15.18:0/935137142' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-09-23T20:11:25.917 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:25 smithi018 ceph-mon[95889]: purged_snaps scrub starts 2023-09-23T20:11:25.917 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:25 smithi018 ceph-mon[95889]: purged_snaps scrub ok 2023-09-23T20:11:25.917 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:25 smithi018 ceph-mon[95889]: pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2023-09-23T20:11:25.918 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:25 smithi018 ceph-mon[95889]: osdmap e56: 8 total, 8 up, 8 in 2023-09-23T20:11:25.918 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:25 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:11:25.918 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:25 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:11:25.918 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:25 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:11:25.918 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:25 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:11:25.918 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:25 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:11:25.918 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:25 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:11:25.918 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:25 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:11:25.918 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:25 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:11:25.918 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:25 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:11:25.919 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:25 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:11:25.919 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:25 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:11:25.919 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:25 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:11:25.919 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:25 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:11:25.919 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:25 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:11:25.919 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:25 smithi018 ceph-mon[109137]: purged_snaps scrub starts 2023-09-23T20:11:25.919 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:25 smithi018 ceph-mon[109137]: purged_snaps scrub ok 2023-09-23T20:11:25.919 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:25 smithi018 ceph-mon[109137]: pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2023-09-23T20:11:25.919 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:25 smithi018 ceph-mon[109137]: osdmap e56: 8 total, 8 up, 8 in 2023-09-23T20:11:25.919 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:25 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:11:25.920 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:25 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:11:25.920 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:25 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:11:25.920 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:25 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:11:25.920 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:25 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:11:25.920 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:25 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:11:25.920 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:25 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:11:25.920 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:25 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:11:25.920 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:25 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:11:25.920 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:25 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:11:25.920 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:25 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:11:25.920 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:25 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:11:25.921 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:25 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:11:25.921 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:25 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:11:26.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:25 smithi149 ceph-mon[100764]: purged_snaps scrub starts 2023-09-23T20:11:26.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:25 smithi149 ceph-mon[100764]: purged_snaps scrub ok 2023-09-23T20:11:26.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:25 smithi149 ceph-mon[100764]: pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2023-09-23T20:11:26.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:25 smithi149 ceph-mon[100764]: osdmap e56: 8 total, 8 up, 8 in 2023-09-23T20:11:26.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:25 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:11:26.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:25 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:11:26.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:25 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:11:26.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:25 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:11:26.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:25 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:11:26.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:25 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:11:26.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:25 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:11:26.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:25 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:11:26.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:25 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:11:26.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:25 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:11:26.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:25 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:11:26.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:25 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:11:26.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:25 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:11:26.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:25 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:11:26.270 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:11:26.270 INFO:teuthology.orchestra.run.smithi018.stdout:{"epoch":57,"fsid":"a3b3b17e-5a4c-11ee-8db4-212e2dc638e7","created":"2023-09-23T20:06:54.359034+0000","modified":"2023-09-23T20:11:25.835667+0000","last_up_change":"2023-09-23T20:11:23.829606+0000","last_in_change":"2023-09-23T20:11:09.202130+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-09-23T20:09:47.162973+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"44c8aaa3-bbd1-4330-8d34-73f7a08b0d4d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6802","nonce":2209550685},{"type":"v1","addr":"172.21.15.18:6803","nonce":2209550685}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6804","nonce":2209550685},{"type":"v1","addr":"172.21.15.18:6805","nonce":2209550685}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6808","nonce":2209550685},{"type":"v1","addr":"172.21.15.18:6809","nonce":2209550685}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6806","nonce":2209550685},{"type":"v1","addr":"172.21.15.18:6807","nonce":2209550685}]},"public_addr":"172.21.15.18:6803/2209550685","cluster_addr":"172.21.15.18:6805/2209550685","heartbeat_back_addr":"172.21.15.18:6809/2209550685","heartbeat_front_addr":"172.21.15.18:6807/2209550685","state":["exists","up"]},{"osd":1,"uuid":"c0534324-49c8-4804-bced-c4d13510e74a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6810","nonce":314354451},{"type":"v1","addr":"172.21.15.18:6811","nonce":314354451}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6812","nonce":314354451},{"type":"v1","addr":"172.21.15.18:6813","nonce":314354451}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6816","nonce":314354451},{"type":"v1","addr":"172.21.15.18:6817","nonce":314354451}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6814","nonce":314354451},{"type":"v1","addr":"172.21.15.18:6815","nonce":314354451}]},"public_addr":"172.21.15.18:6811/314354451","cluster_addr":"172.21.15.18:6813/314354451","heartbeat_back_addr":"172.21.15.18:6817/314354451","heartbeat_front_addr":"172.21.15.18:6815/314354451","state":["exists","up"]},{"osd":2,"uuid":"f3b2116d-a3d5-4279-91dc-eb47458d3239","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6818","nonce":1413681698},{"type":"v1","addr":"172.21.15.18:6819","nonce":1413681698}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6820","nonce":1413681698},{"type":"v1","addr":"172.21.15.18:6821","nonce":1413681698}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6824","nonce":1413681698},{"type":"v1","addr":"172.21.15.18:6825","nonce":1413681698}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6822","nonce":1413681698},{"type":"v1","addr":"172.21.15.18:6823","nonce":1413681698}]},"public_addr":"172.21.15.18:6819/1413681698","cluster_addr":"172.21.15.18:6821/1413681698","heartbeat_back_addr":"172.21.15.18:6825/1413681698","heartbeat_front_addr":"172.21.15.18:6823/1413681698","state":["exists","up"]},{"osd":3,"uuid":"8edccb7f-c4c9-47f3-a6f5-38a1a38cb099","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6826","nonce":423024182},{"type":"v1","addr":"172.21.15.18:6827","nonce":423024182}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6828","nonce":423024182},{"type":"v1","addr":"172.21.15.18:6829","nonce":423024182}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6832","nonce":423024182},{"type":"v1","addr":"172.21.15.18:6833","nonce":423024182}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6830","nonce":423024182},{"type":"v1","addr":"172.21.15.18:6831","nonce":423024182}]},"public_addr":"172.21.15.18:6827/423024182","cluster_addr":"172.21.15.18:6829/423024182","heartbeat_back_addr":"172.21.15.18:6833/423024182","heartbeat_front_addr":"172.21.15.18:6831/423024182","state":["exists","up"]},{"osd":4,"uuid":"1d5a64ce-fe42-4355-991a-dfb20cbe5423","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6800","nonce":3640432477},{"type":"v1","addr":"172.21.15.149:6801","nonce":3640432477}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6802","nonce":3640432477},{"type":"v1","addr":"172.21.15.149:6803","nonce":3640432477}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6806","nonce":3640432477},{"type":"v1","addr":"172.21.15.149:6807","nonce":3640432477}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6804","nonce":3640432477},{"type":"v1","addr":"172.21.15.149:6805","nonce":3640432477}]},"public_addr":"172.21.15.149:6801/3640432477","cluster_addr":"172.21.15.149:6803/3640432477","heartbeat_back_addr":"172.21.15.149:6807/3640432477","heartbeat_front_addr":"172.21.15.149:6805/3640432477","state":["exists","up"]},{"osd":5,"uuid":"105ae3b2-b2fc-477d-9134-3d517bc9427e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6808","nonce":2969709367},{"type":"v1","addr":"172.21.15.149:6809","nonce":2969709367}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6810","nonce":2969709367},{"type":"v1","addr":"172.21.15.149:6811","nonce":2969709367}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6814","nonce":2969709367},{"type":"v1","addr":"172.21.15.149:6815","nonce":2969709367}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6812","nonce":2969709367},{"type":"v1","addr":"172.21.15.149:6813","nonce":2969709367}]},"public_addr":"172.21.15.149:6809/2969709367","cluster_addr":"172.21.15.149:6811/2969709367","heartbeat_back_addr":"172.21.15.149:6815/2969709367","heartbeat_front_addr":"172.21.15.149:6813/2969709367","state":["exists","up"]},{"osd":6,"uuid":"27d2d2a8-e8c6-4a05-a85e-bd54376cf4c1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6816","nonce":1291453593},{"type":"v1","addr":"172.21.15.149:6817","nonce":1291453593}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6818","nonce":1291453593},{"type":"v1","addr":"172.21.15.149:6819","nonce":1291453593}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6822","nonce":1291453593},{"type":"v1","addr":"172.21.15.149:6823","nonce":1291453593}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6820","nonce":1291453593},{"type":"v1","addr":"172.21.15.149:6821","nonce":1291453593}]},"public_addr":"172.21.15.149:6817/1291453593","cluster_addr":"172.21.15.149:6819/1291453593","heartbeat_back_addr":"172.21.15.149:6823/1291453593","heartbeat_front_addr":"172.21.15.149:6821/1291453593","state":["exists","up"]},{"osd":7,"uuid":"9da636f7-995c-4f29-9264-70d09c6f66c6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6824","nonce":3117985178},{"type":"v1","addr":"172.21.15.149:6825","nonce":3117985178}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6826","nonce":3117985178},{"type":"v1","addr":"172.21.15.149:6827","nonce":3117985178}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6830","nonce":3117985178},{"type":"v1","addr":"172.21.15.149:6831","nonce":3117985178}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6828","nonce":3117985178},{"type":"v1","addr":"172.21.15.149:6829","nonce":3117985178}]},"public_addr":"172.21.15.149:6825/3117985178","cluster_addr":"172.21.15.149:6827/3117985178","heartbeat_back_addr":"172.21.15.149:6831/3117985178","heartbeat_front_addr":"172.21.15.149:6829/3117985178","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:09:01.034520+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:09:21.795238+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:09:42.987835+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:10:04.092458+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:10:23.309112+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:10:43.348231+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:11:03.128458+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:11:23.084228+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[{"pgid":"1.0","osds":[0,6]}],"primary_temp":[],"blocklist":{"172.21.15.18:6801/380813540":"2023-09-24T20:07:48.635784+0000","172.21.15.18:6800/380813540":"2023-09-24T20:07:48.635784+0000","172.21.15.18:0/3831267686":"2023-09-24T20:07:48.635784+0000","172.21.15.18:0/950924496":"2023-09-24T20:07:23.919815+0000","172.21.15.18:0/1082665289":"2023-09-24T20:07:48.635784+0000","172.21.15.18:0/2993111051":"2023-09-24T20:07:23.919815+0000","172.21.15.18:0/69122222":"2023-09-24T20:07:23.919815+0000","172.21.15.18:0/1645667747":"2023-09-24T20:07:48.635784+0000","172.21.15.18:6800/1930027051":"2023-09-24T20:07:23.919815+0000","172.21.15.18:6801/1930027051":"2023-09-24T20:07:23.919815+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-23T20:11:26.630 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2023-09-23T20:09:47.162973+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '22', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}}] 2023-09-23T20:11:26.630 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd pool get .mgr pg_num 2023-09-23T20:11:26.890 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:26 smithi018 ceph-mon[95889]: Detected new or changed devices on smithi149 2023-09-23T20:11:26.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:26 smithi018 ceph-mon[95889]: Adjusting osd_memory_target on smithi149 to 1469M 2023-09-23T20:11:26.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:26 smithi018 ceph-mon[95889]: osdmap e57: 8 total, 8 up, 8 in 2023-09-23T20:11:26.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:26 smithi018 ceph-mon[95889]: from='client.? 172.21.15.18:0/2955115411' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T20:11:26.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:26 smithi018 ceph-mon[109137]: Detected new or changed devices on smithi149 2023-09-23T20:11:26.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:26 smithi018 ceph-mon[109137]: Adjusting osd_memory_target on smithi149 to 1469M 2023-09-23T20:11:26.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:26 smithi018 ceph-mon[109137]: osdmap e57: 8 total, 8 up, 8 in 2023-09-23T20:11:26.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:26 smithi018 ceph-mon[109137]: from='client.? 172.21.15.18:0/2955115411' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T20:11:27.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:26 smithi149 ceph-mon[100764]: Detected new or changed devices on smithi149 2023-09-23T20:11:27.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:26 smithi149 ceph-mon[100764]: Adjusting osd_memory_target on smithi149 to 1469M 2023-09-23T20:11:27.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:26 smithi149 ceph-mon[100764]: osdmap e57: 8 total, 8 up, 8 in 2023-09-23T20:11:27.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:26 smithi149 ceph-mon[100764]: from='client.? 172.21.15.18:0/2955115411' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T20:11:27.235 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:11:28.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:27 smithi018 ceph-mon[95889]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:11:28.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:27 smithi018 ceph-mon[95889]: osdmap e58: 8 total, 8 up, 8 in 2023-09-23T20:11:28.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:27 smithi018 ceph-mon[109137]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:11:28.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:27 smithi018 ceph-mon[109137]: osdmap e58: 8 total, 8 up, 8 in 2023-09-23T20:11:28.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:27 smithi149 ceph-mon[100764]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:11:28.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:27 smithi149 ceph-mon[100764]: osdmap e58: 8 total, 8 up, 8 in 2023-09-23T20:11:28.626 INFO:teuthology.orchestra.run.smithi018.stdout:pg_num: 1 2023-09-23T20:11:28.982 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:28 smithi018 ceph-mon[109137]: osdmap e59: 8 total, 8 up, 8 in 2023-09-23T20:11:28.983 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:28 smithi018 ceph-mon[109137]: from='client.? 172.21.15.18:0/461092029' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2023-09-23T20:11:28.983 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:28 smithi018 ceph-mon[109137]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:11:28.983 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:28 smithi018 ceph-mon[95889]: osdmap e59: 8 total, 8 up, 8 in 2023-09-23T20:11:28.983 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:28 smithi018 ceph-mon[95889]: from='client.? 172.21.15.18:0/461092029' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2023-09-23T20:11:28.983 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:28 smithi018 ceph-mon[95889]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:11:28.994 INFO:tasks.cephadm:Adding prometheus.a on smithi149 2023-09-23T20:11:28.994 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph orch apply prometheus '1;smithi149=a' 2023-09-23T20:11:29.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:28 smithi149 ceph-mon[100764]: osdmap e59: 8 total, 8 up, 8 in 2023-09-23T20:11:29.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:28 smithi149 ceph-mon[100764]: from='client.? 172.21.15.18:0/461092029' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2023-09-23T20:11:29.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:28 smithi149 ceph-mon[100764]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:11:29.601 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:11:30.970 INFO:teuthology.orchestra.run.smithi149.stdout:Scheduled prometheus update... 2023-09-23T20:11:31.344 DEBUG:teuthology.orchestra.run.smithi149:prometheus.a> sudo journalctl -f -n 0 -u ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7@prometheus.a.service 2023-09-23T20:11:31.347 INFO:tasks.cephadm:Adding node-exporter.a on smithi018 2023-09-23T20:11:31.347 INFO:tasks.cephadm:Adding node-exporter.b on smithi149 2023-09-23T20:11:31.347 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph orch apply node-exporter '2;smithi018=a;smithi149=b' 2023-09-23T20:11:31.376 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:-- Logs begin at Sat 2023-09-23 19:50:38 UTC. -- 2023-09-23T20:11:31.984 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:11:32.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:31 smithi018 ceph-mon[95889]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 99 KiB/s, 0 objects/s recovering 2023-09-23T20:11:32.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:31 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:11:32.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:31 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:11:32.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:31 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:11:32.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:31 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:11:32.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:31 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:11:32.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:31 smithi018 ceph-mon[109137]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 99 KiB/s, 0 objects/s recovering 2023-09-23T20:11:32.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:31 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:11:32.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:31 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:11:32.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:31 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:11:32.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:31 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:11:32.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:31 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:11:32.179 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:31 smithi149 ceph-mon[100764]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 99 KiB/s, 0 objects/s recovering 2023-09-23T20:11:32.179 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:31 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:11:32.179 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:31 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:11:32.180 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:31 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:11:32.180 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:31 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:11:32.180 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:31 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:11:32.965 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:32 smithi149 ceph-mon[100764]: from='client.24340 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi149=a", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:11:32.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:32 smithi149 ceph-mon[100764]: Saving service prometheus spec with placement smithi149=a;count:1 2023-09-23T20:11:32.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:32 smithi149 ceph-mon[100764]: Deploying daemon prometheus.a on smithi149 2023-09-23T20:11:33.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:32 smithi018 ceph-mon[95889]: from='client.24340 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi149=a", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:11:33.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:32 smithi018 ceph-mon[95889]: Saving service prometheus spec with placement smithi149=a;count:1 2023-09-23T20:11:33.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:32 smithi018 ceph-mon[95889]: Deploying daemon prometheus.a on smithi149 2023-09-23T20:11:33.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:32 smithi018 ceph-mon[109137]: from='client.24340 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi149=a", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:11:33.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:32 smithi018 ceph-mon[109137]: Saving service prometheus spec with placement smithi149=a;count:1 2023-09-23T20:11:33.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:32 smithi018 ceph-mon[109137]: Deploying daemon prometheus.a on smithi149 2023-09-23T20:11:33.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:33 smithi149 ceph-mon[100764]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 84 KiB/s, 0 objects/s recovering 2023-09-23T20:11:34.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:33 smithi018 ceph-mon[95889]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 84 KiB/s, 0 objects/s recovering 2023-09-23T20:11:34.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:33 smithi018 ceph-mon[109137]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 84 KiB/s, 0 objects/s recovering 2023-09-23T20:11:34.163 INFO:teuthology.orchestra.run.smithi149.stdout:Scheduled node-exporter update... 2023-09-23T20:11:34.516 DEBUG:teuthology.orchestra.run.smithi018:node-exporter.a> sudo journalctl -f -n 0 -u ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7@node-exporter.a.service 2023-09-23T20:11:34.519 DEBUG:teuthology.orchestra.run.smithi149:node-exporter.b> sudo journalctl -f -n 0 -u ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7@node-exporter.b.service 2023-09-23T20:11:34.521 INFO:tasks.cephadm:Adding alertmanager.a on smithi018 2023-09-23T20:11:34.521 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph orch apply alertmanager '1;smithi018=a' 2023-09-23T20:11:34.553 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:-- Logs begin at Sat 2023-09-23 19:50:38 UTC. -- 2023-09-23T20:11:34.557 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:-- Logs begin at Sat 2023-09-23 19:50:10 UTC. -- 2023-09-23T20:11:35.465 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:35 smithi149 ceph-mon[100764]: from='client.24346 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi018=a;smithi149=b", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:11:35.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:35 smithi149 ceph-mon[100764]: Saving service node-exporter spec with placement smithi018=a;smithi149=b;count:2 2023-09-23T20:11:35.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:35 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:11:35.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:35 smithi149 ceph-mon[100764]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-09-23T20:11:35.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:35 smithi018 ceph-mon[109137]: from='client.24346 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi018=a;smithi149=b", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:11:35.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:35 smithi018 ceph-mon[109137]: Saving service node-exporter spec with placement smithi018=a;smithi149=b;count:2 2023-09-23T20:11:35.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:35 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:11:35.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:35 smithi018 ceph-mon[109137]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-09-23T20:11:35.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:35 smithi018 ceph-mon[95889]: from='client.24346 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi018=a;smithi149=b", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:11:35.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:35 smithi018 ceph-mon[95889]: Saving service node-exporter spec with placement smithi018=a;smithi149=b;count:2 2023-09-23T20:11:35.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:35 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:11:35.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:35 smithi018 ceph-mon[95889]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-09-23T20:11:36.618 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:11:37.897 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:37 smithi149 ceph-mon[100764]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 59 KiB/s, 0 objects/s recovering 2023-09-23T20:11:38.140 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:37 smithi018 ceph-mon[109137]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 59 KiB/s, 0 objects/s recovering 2023-09-23T20:11:38.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:37 smithi018 ceph-mon[95889]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 59 KiB/s, 0 objects/s recovering 2023-09-23T20:11:39.336 INFO:teuthology.orchestra.run.smithi149.stdout:Scheduled alertmanager update... 2023-09-23T20:11:39.797 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:39 smithi149 ceph-mon[100764]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 53 KiB/s, 0 objects/s recovering 2023-09-23T20:11:39.797 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:39 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:11:40.008 DEBUG:teuthology.orchestra.run.smithi018:alertmanager.a> sudo journalctl -f -n 0 -u ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7@alertmanager.a.service 2023-09-23T20:11:40.011 INFO:tasks.cephadm:Adding grafana.a on smithi149 2023-09-23T20:11:40.012 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph orch apply grafana '1;smithi149=a' 2023-09-23T20:11:40.039 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:39 smithi018 ceph-mon[109137]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 53 KiB/s, 0 objects/s recovering 2023-09-23T20:11:40.039 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:39 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:11:40.039 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:39 smithi018 ceph-mon[95889]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 53 KiB/s, 0 objects/s recovering 2023-09-23T20:11:40.039 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:39 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:11:40.045 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:-- Logs begin at Sat 2023-09-23 19:50:10 UTC. -- 2023-09-23T20:11:40.336 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:11:40 smithi149 systemd[1]: Starting Ceph prometheus.a for a3b3b17e-5a4c-11ee-8db4-212e2dc638e7... 2023-09-23T20:11:40.629 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:11:40 smithi149 podman[135301]: 2023-09-23T20:11:40.805 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:11:40.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:40 smithi149 ceph-mon[100764]: from='client.24352 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi018=a", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:11:40.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:40 smithi149 ceph-mon[100764]: Saving service alertmanager spec with placement smithi018=a;count:1 2023-09-23T20:11:40.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:40 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:11:40.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:40 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:11:40.966 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:11:40 smithi149 bash[135301]: c7cfe1bd899ebc86a6cf987088539b8112a62526c379d50d6a8acfdc2222b754 2023-09-23T20:11:40.967 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:11:40 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[135333]: ts=2023-09-23T20:11:40.630Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2023-09-23T20:11:40.967 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:11:40 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[135333]: ts=2023-09-23T20:11:40.630Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2023-09-23T20:11:40.967 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:11:40 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[135333]: ts=2023-09-23T20:11:40.630Z caller=main.go:570 level=info host_details="(Linux 4.18.0-372.9.1.el8.x86_64 #1 SMP Fri Apr 15 22:12:19 EDT 2022 x86_64 smithi149 (none))" 2023-09-23T20:11:40.967 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:11:40 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[135333]: ts=2023-09-23T20:11:40.630Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2023-09-23T20:11:40.967 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:11:40 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[135333]: ts=2023-09-23T20:11:40.630Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2023-09-23T20:11:40.967 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:11:40 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[135333]: ts=2023-09-23T20:11:40.632Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2023-09-23T20:11:40.967 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:11:40 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[135333]: ts=2023-09-23T20:11:40.633Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2023-09-23T20:11:40.967 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:11:40 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[135333]: ts=2023-09-23T20:11:40.635Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2023-09-23T20:11:40.967 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:11:40 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[135333]: ts=2023-09-23T20:11:40.635Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2023-09-23T20:11:40.967 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:11:40 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[135333]: ts=2023-09-23T20:11:40.637Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2023-09-23T20:11:40.968 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:11:40 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[135333]: ts=2023-09-23T20:11:40.637Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.972µs 2023-09-23T20:11:40.968 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:11:40 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[135333]: ts=2023-09-23T20:11:40.637Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2023-09-23T20:11:40.968 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:11:40 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[135333]: ts=2023-09-23T20:11:40.637Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=0 2023-09-23T20:11:40.968 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:11:40 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[135333]: ts=2023-09-23T20:11:40.637Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=18.966µs wal_replay_duration=173.334µs wbl_replay_duration=111ns total_replay_duration=210.961µs 2023-09-23T20:11:40.968 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:11:40 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[135333]: ts=2023-09-23T20:11:40.638Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2023-09-23T20:11:40.968 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:11:40 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[135333]: ts=2023-09-23T20:11:40.638Z caller=main.go:1029 level=info msg="TSDB started" 2023-09-23T20:11:40.968 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:11:40 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[135333]: ts=2023-09-23T20:11:40.638Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2023-09-23T20:11:40.968 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:11:40 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[135333]: ts=2023-09-23T20:11:40.653Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=14.735803ms db_storage=751ns remote_storage=1.178µs web_handler=322ns query_engine=590ns scrape=477.103µs scrape_sd=17.332µs notify=656ns notify_sd=1.381µs rules=14.094896ms tracing=5.833µs 2023-09-23T20:11:40.968 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:11:40 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[135333]: ts=2023-09-23T20:11:40.653Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2023-09-23T20:11:40.968 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:11:40 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[135333]: ts=2023-09-23T20:11:40.653Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2023-09-23T20:11:40.968 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:11:40 smithi149 systemd[1]: Started Ceph prometheus.a for a3b3b17e-5a4c-11ee-8db4-212e2dc638e7. 2023-09-23T20:11:41.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:40 smithi018 ceph-mon[95889]: from='client.24352 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi018=a", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:11:41.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:40 smithi018 ceph-mon[95889]: Saving service alertmanager spec with placement smithi018=a;count:1 2023-09-23T20:11:41.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:40 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:11:41.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:40 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:11:41.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:40 smithi018 ceph-mon[109137]: from='client.24352 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi018=a", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:11:41.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:40 smithi018 ceph-mon[109137]: Saving service alertmanager spec with placement smithi018=a;count:1 2023-09-23T20:11:41.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:40 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:11:41.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:40 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:11:41.915 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:41 smithi149 ceph-mon[100764]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2023-09-23T20:11:41.915 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:41 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:11:41.916 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:41 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2023-09-23T20:11:41.916 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:11:41 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[102880]: ignoring --setuser ceph since I am not root 2023-09-23T20:11:41.916 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:11:41 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[102880]: ignoring --setgroup ceph since I am not root 2023-09-23T20:11:41.916 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:11:41 smithi149 ceph-mgr[102904]: -- 172.21.15.149:0/220520591 <== mon.2 v2:172.21.15.18:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5559b933e9c0 con 0x5559b9314c00 2023-09-23T20:11:41.916 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:11:41 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[102880]: 2023-09-23T20:11:41.914+0000 7fa603dc31c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-23T20:11:42.031 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:41 smithi018 ceph-mon[109137]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2023-09-23T20:11:42.031 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:41 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:11:42.031 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:41 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2023-09-23T20:11:42.032 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:41 smithi018 ceph-mon[95889]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2023-09-23T20:11:42.032 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:41 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' 2023-09-23T20:11:42.032 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:41 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2023-09-23T20:11:42.032 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:11:41 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: ignoring --setuser ceph since I am not root 2023-09-23T20:11:42.032 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:11:41 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: ignoring --setgroup ceph since I am not root 2023-09-23T20:11:42.032 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:11:41 smithi018 ceph-mgr[96157]: -- 172.21.15.18:0/1778643403 <== mon.2 v2:172.21.15.18:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55b6cca2e9c0 con 0x55b6cca04c00 2023-09-23T20:11:42.033 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:11:41 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:11:41.937+0000 7faf133b21c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-23T20:11:42.033 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:11:42 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:11:42.031+0000 7faf133b21c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T20:11:42.190 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:11:41 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[102880]: 2023-09-23T20:11:41.997+0000 7fa603dc31c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T20:11:42.190 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:11:42 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[102880]: 2023-09-23T20:11:42.090+0000 7fa603dc31c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T20:11:42.190 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:11:42 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[102880]: 2023-09-23T20:11:42.189+0000 7fa603dc31c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T20:11:42.391 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:11:42 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:11:42.137+0000 7faf133b21c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T20:11:42.391 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:11:42 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:11:42.247+0000 7faf133b21c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T20:11:42.465 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:11:42 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[102880]: 2023-09-23T20:11:42.360+0000 7fa603dc31c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T20:11:42.753 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:11:42 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:11:42.450+0000 7faf133b21c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T20:11:42.965 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:42 smithi149 ceph-mon[100764]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2023-09-23T20:11:42.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:42 smithi149 ceph-mon[100764]: mgrmap e16: y(active, since 3m), standbys: x 2023-09-23T20:11:42.966 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:11:42 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[102880]: 2023-09-23T20:11:42.666+0000 7fa603dc31c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T20:11:42.966 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:11:42 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[102880]: 2023-09-23T20:11:42.753+0000 7fa603dc31c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T20:11:42.966 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:11:42 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[102880]: 2023-09-23T20:11:42.886+0000 7fa603dc31c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T20:11:43.100 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:11:42 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:11:42.845+0000 7faf133b21c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T20:11:43.100 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:11:42 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:11:42.950+0000 7faf133b21c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T20:11:43.100 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:42 smithi018 ceph-mon[109137]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2023-09-23T20:11:43.100 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:42 smithi018 ceph-mon[109137]: mgrmap e16: y(active, since 3m), standbys: x 2023-09-23T20:11:43.101 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:42 smithi018 ceph-mon[95889]: from='mgr.14150 172.21.15.18:0/1543182197' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2023-09-23T20:11:43.101 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:42 smithi018 ceph-mon[95889]: mgrmap e16: y(active, since 3m), standbys: x 2023-09-23T20:11:43.390 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:11:43 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:11:43.099+0000 7faf133b21c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T20:11:43.390 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:11:43 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:11:43.213+0000 7faf133b21c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T20:11:43.396 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:11:42 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[102880]: 2023-09-23T20:11:42.978+0000 7fa603dc31c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T20:11:43.716 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:11:43 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[102880]: 2023-09-23T20:11:43.396+0000 7fa603dc31c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T20:11:44.140 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:11:43 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:11:43.718+0000 7faf133b21c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T20:11:44.216 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:11:43 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[102880]: 2023-09-23T20:11:43.956+0000 7fa603dc31c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T20:11:44.216 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:11:44 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[102880]: 2023-09-23T20:11:44.128+0000 7fa603dc31c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T20:11:44.890 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:11:44 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:11:44.395+0000 7faf133b21c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T20:11:44.891 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:11:44 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:11:44.586+0000 7faf133b21c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T20:11:45.142 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:11:44 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[102880]: 2023-09-23T20:11:44.887+0000 7fa603dc31c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T20:11:45.143 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:11:44 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[102880]: 2023-09-23T20:11:44.975+0000 7fa603dc31c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-23T20:11:45.417 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:11:45 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[102880]: 2023-09-23T20:11:45.142+0000 7fa603dc31c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T20:11:45.715 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:11:45 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[102880]: 2023-09-23T20:11:45.416+0000 7fa603dc31c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-23T20:11:45.716 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:11:45 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[102880]: 2023-09-23T20:11:45.502+0000 7fa603dc31c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T20:11:45.868 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:11:45 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:11:45.542+0000 7faf133b21c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T20:11:45.868 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:11:45 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:11:45.651+0000 7faf133b21c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-23T20:11:46.049 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:11:45 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[102880]: 2023-09-23T20:11:45.793+0000 7fa603dc31c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T20:11:46.140 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:11:45 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:11:45.868+0000 7faf133b21c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T20:11:46.465 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:11:46 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[102880]: 2023-09-23T20:11:46.048+0000 7fa603dc31c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T20:11:46.640 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:11:46 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:11:46.212+0000 7faf133b21c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-23T20:11:46.641 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:11:46 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:11:46.316+0000 7faf133b21c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T20:11:46.968 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:11:46 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:11:46.644+0000 7faf133b21c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T20:11:47.070 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:11:46 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[102880]: 2023-09-23T20:11:46.753+0000 7fa603dc31c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T20:11:47.070 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:11:46 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[102880]: 2023-09-23T20:11:46.883+0000 7fa603dc31c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T20:11:47.390 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:11:46 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:11:46.968+0000 7faf133b21c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T20:11:47.394 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:11:47 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[102880]: 2023-09-23T20:11:47.069+0000 7fa603dc31c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T20:11:47.394 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:11:47 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[102880]: 2023-09-23T20:11:47.182+0000 7fa603dc31c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T20:11:47.716 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:11:47 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[102880]: 2023-09-23T20:11:47.393+0000 7fa603dc31c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T20:11:48.140 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:11:47 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:11:47.791+0000 7faf133b21c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T20:11:48.141 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:11:47 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:11:47.949+0000 7faf133b21c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T20:11:48.603 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:11:48 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:11:48.189+0000 7faf133b21c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T20:11:48.604 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:11:48 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:11:48.331+0000 7faf133b21c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T20:11:48.604 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:48 smithi018 ceph-mon[109137]: Standby manager daemon x restarted 2023-09-23T20:11:48.604 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:48 smithi018 ceph-mon[109137]: Standby manager daemon x started 2023-09-23T20:11:48.604 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:48 smithi018 ceph-mon[109137]: from='mgr.? 172.21.15.149:0/2940153980' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T20:11:48.604 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:48 smithi018 ceph-mon[109137]: from='mgr.? 172.21.15.149:0/2940153980' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T20:11:48.604 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:48 smithi018 ceph-mon[109137]: from='mgr.? 172.21.15.149:0/2940153980' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T20:11:48.605 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:48 smithi018 ceph-mon[109137]: from='mgr.? 172.21.15.149:0/2940153980' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T20:11:48.605 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:48 smithi018 ceph-mon[95889]: Standby manager daemon x restarted 2023-09-23T20:11:48.605 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:48 smithi018 ceph-mon[95889]: Standby manager daemon x started 2023-09-23T20:11:48.605 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:48 smithi018 ceph-mon[95889]: from='mgr.? 172.21.15.149:0/2940153980' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T20:11:48.605 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:48 smithi018 ceph-mon[95889]: from='mgr.? 172.21.15.149:0/2940153980' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T20:11:48.605 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:48 smithi018 ceph-mon[95889]: from='mgr.? 172.21.15.149:0/2940153980' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T20:11:48.605 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:48 smithi018 ceph-mon[95889]: from='mgr.? 172.21.15.149:0/2940153980' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T20:11:48.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:48 smithi149 ceph-mon[100764]: Standby manager daemon x restarted 2023-09-23T20:11:48.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:48 smithi149 ceph-mon[100764]: Standby manager daemon x started 2023-09-23T20:11:48.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:48 smithi149 ceph-mon[100764]: from='mgr.? 172.21.15.149:0/2940153980' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T20:11:48.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:48 smithi149 ceph-mon[100764]: from='mgr.? 172.21.15.149:0/2940153980' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T20:11:48.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:48 smithi149 ceph-mon[100764]: from='mgr.? 172.21.15.149:0/2940153980' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T20:11:48.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:48 smithi149 ceph-mon[100764]: from='mgr.? 172.21.15.149:0/2940153980' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T20:11:48.717 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:11:48 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[102880]: 2023-09-23T20:11:48.247+0000 7fa603dc31c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T20:11:48.717 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:11:48 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[102880]: [23/Sep/2023:20:11:48] ENGINE Bus STARTING 2023-09-23T20:11:48.717 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:11:48 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[102880]: CherryPy Checker: 2023-09-23T20:11:48.717 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:11:48 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[102880]: The Application mounted at '' has an empty config. 2023-09-23T20:11:48.717 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:11:48 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[102880]: 2023-09-23T20:11:48.717 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:11:48 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[102880]: [23/Sep/2023:20:11:48] ENGINE Serving on http://:::9283 2023-09-23T20:11:48.717 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:11:48 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[102880]: [23/Sep/2023:20:11:48] ENGINE Bus STARTED 2023-09-23T20:11:48.890 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:11:48 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:11:48.603+0000 7faf133b21c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T20:11:49.639 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:49 smithi018 ceph-mon[109137]: mgrmap e17: y(active, since 3m), standbys: x 2023-09-23T20:11:49.639 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:49 smithi018 ceph-mon[95889]: mgrmap e17: y(active, since 3m), standbys: x 2023-09-23T20:11:49.715 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:49 smithi149 ceph-mon[100764]: mgrmap e17: y(active, since 3m), standbys: x 2023-09-23T20:11:49.891 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:11:49 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:11:49.639+0000 7faf133b21c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T20:11:49.891 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:11:49 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: [23/Sep/2023:20:11:49] ENGINE Bus STARTING 2023-09-23T20:11:49.891 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:11:49 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: CherryPy Checker: 2023-09-23T20:11:49.891 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:11:49 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: The Application mounted at '' has an empty config. 2023-09-23T20:11:49.891 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:11:49 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:11:49.891 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:11:49 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: [23/Sep/2023:20:11:49] ENGINE Serving on http://:::9283 2023-09-23T20:11:49.891 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:11:49 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: [23/Sep/2023:20:11:49] ENGINE Bus STARTED 2023-09-23T20:11:50.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:50 smithi018 ceph-mon[109137]: Active manager daemon y restarted 2023-09-23T20:11:50.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:50 smithi018 ceph-mon[109137]: Activating manager daemon y 2023-09-23T20:11:50.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:50 smithi018 ceph-mon[109137]: osdmap e60: 8 total, 8 up, 8 in 2023-09-23T20:11:50.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:50 smithi018 ceph-mon[109137]: mgrmap e18: y(active, starting, since 0.00680665s), standbys: x 2023-09-23T20:11:50.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:50 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T20:11:50.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:50 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:11:50.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:50 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:11:50.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:50 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T20:11:50.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:50 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T20:11:50.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:50 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T20:11:50.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:50 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T20:11:50.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:50 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T20:11:50.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:50 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T20:11:50.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:50 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T20:11:50.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:50 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T20:11:50.643 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:50 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T20:11:50.643 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:50 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T20:11:50.643 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:50 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T20:11:50.643 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:50 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T20:11:50.643 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:50 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T20:11:50.643 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:50 smithi018 ceph-mon[109137]: Manager daemon y is now available 2023-09-23T20:11:50.643 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:50 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:11:50.643 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:50 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:11:50.643 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:50 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:11:50.643 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:50 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:11:50.643 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:50 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T20:11:50.644 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:50 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T20:11:50.644 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:50 smithi018 ceph-mon[95889]: Active manager daemon y restarted 2023-09-23T20:11:50.644 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:50 smithi018 ceph-mon[95889]: Activating manager daemon y 2023-09-23T20:11:50.644 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:50 smithi018 ceph-mon[95889]: osdmap e60: 8 total, 8 up, 8 in 2023-09-23T20:11:50.644 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:50 smithi018 ceph-mon[95889]: mgrmap e18: y(active, starting, since 0.00680665s), standbys: x 2023-09-23T20:11:50.644 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:50 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T20:11:50.645 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:50 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:11:50.645 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:50 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:11:50.645 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:50 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T20:11:50.645 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:50 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T20:11:50.645 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:50 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T20:11:50.645 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:50 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T20:11:50.645 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:50 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T20:11:50.645 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:50 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T20:11:50.645 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:50 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T20:11:50.645 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:50 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T20:11:50.645 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:50 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T20:11:50.646 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:50 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T20:11:50.646 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:50 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T20:11:50.646 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:50 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T20:11:50.646 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:50 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T20:11:50.646 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:50 smithi018 ceph-mon[95889]: Manager daemon y is now available 2023-09-23T20:11:50.646 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:50 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:11:50.646 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:50 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:11:50.646 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:50 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:11:50.646 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:50 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:11:50.646 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:50 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T20:11:50.646 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:50 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T20:11:50.676 INFO:teuthology.orchestra.run.smithi149.stdout:Scheduled grafana update... 2023-09-23T20:11:50.702 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:50 smithi149 ceph-mon[100764]: Active manager daemon y restarted 2023-09-23T20:11:50.703 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:50 smithi149 ceph-mon[100764]: Activating manager daemon y 2023-09-23T20:11:50.703 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:50 smithi149 ceph-mon[100764]: osdmap e60: 8 total, 8 up, 8 in 2023-09-23T20:11:50.703 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:50 smithi149 ceph-mon[100764]: mgrmap e18: y(active, starting, since 0.00680665s), standbys: x 2023-09-23T20:11:50.703 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:50 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T20:11:50.703 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:50 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:11:50.703 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:50 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:11:50.703 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:50 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T20:11:50.703 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:50 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T20:11:50.703 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:50 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T20:11:50.704 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:50 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T20:11:50.704 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:50 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T20:11:50.704 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:50 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T20:11:50.704 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:50 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T20:11:50.704 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:50 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T20:11:50.704 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:50 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T20:11:50.704 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:50 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T20:11:50.704 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:50 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T20:11:50.704 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:50 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T20:11:50.704 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:50 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T20:11:50.704 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:50 smithi149 ceph-mon[100764]: Manager daemon y is now available 2023-09-23T20:11:50.705 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:50 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:11:50.705 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:50 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:11:50.705 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:50 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:11:50.705 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:50 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:11:50.705 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:50 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T20:11:50.705 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:50 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T20:11:51.072 DEBUG:teuthology.orchestra.run.smithi149:grafana.a> sudo journalctl -f -n 0 -u ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7@grafana.a.service 2023-09-23T20:11:51.074 INFO:tasks.cephadm:Setting up client nodes... 2023-09-23T20:11:51.075 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2023-09-23T20:11:51.107 INFO:journalctl@ceph.grafana.a.smithi149.stdout:-- Logs begin at Sat 2023-09-23 19:50:38 UTC. -- 2023-09-23T20:11:51.687 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:11:51.927 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:51 smithi018 ceph-mon[109137]: mgrmap e19: y(active, since 1.01126s), standbys: x 2023-09-23T20:11:51.928 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:51 smithi018 ceph-mon[109137]: Saving service grafana spec with placement smithi149=a;count:1 2023-09-23T20:11:51.928 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:51 smithi018 ceph-mon[109137]: pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:11:51.928 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:51 smithi018 ceph-mon[109137]: [23/Sep/2023:20:11:50] ENGINE Bus STARTING 2023-09-23T20:11:51.928 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:51 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:11:51.928 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:51 smithi018 ceph-mon[109137]: [23/Sep/2023:20:11:50] ENGINE Serving on https://172.21.15.18:7150 2023-09-23T20:11:51.928 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:51 smithi018 ceph-mon[109137]: [23/Sep/2023:20:11:50] ENGINE Bus STARTED 2023-09-23T20:11:51.928 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:51 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:11:51.928 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:51 smithi018 ceph-mon[95889]: mgrmap e19: y(active, since 1.01126s), standbys: x 2023-09-23T20:11:51.929 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:51 smithi018 ceph-mon[95889]: Saving service grafana spec with placement smithi149=a;count:1 2023-09-23T20:11:51.929 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:51 smithi018 ceph-mon[95889]: pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:11:51.929 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:51 smithi018 ceph-mon[95889]: [23/Sep/2023:20:11:50] ENGINE Bus STARTING 2023-09-23T20:11:51.929 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:51 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:11:51.929 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:51 smithi018 ceph-mon[95889]: [23/Sep/2023:20:11:50] ENGINE Serving on https://172.21.15.18:7150 2023-09-23T20:11:51.929 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:51 smithi018 ceph-mon[95889]: [23/Sep/2023:20:11:50] ENGINE Bus STARTED 2023-09-23T20:11:51.929 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:51 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:11:51.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:51 smithi149 ceph-mon[100764]: mgrmap e19: y(active, since 1.01126s), standbys: x 2023-09-23T20:11:51.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:51 smithi149 ceph-mon[100764]: Saving service grafana spec with placement smithi149=a;count:1 2023-09-23T20:11:51.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:51 smithi149 ceph-mon[100764]: pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:11:51.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:51 smithi149 ceph-mon[100764]: [23/Sep/2023:20:11:50] ENGINE Bus STARTING 2023-09-23T20:11:51.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:51 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:11:51.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:51 smithi149 ceph-mon[100764]: [23/Sep/2023:20:11:50] ENGINE Serving on https://172.21.15.18:7150 2023-09-23T20:11:51.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:51 smithi149 ceph-mon[100764]: [23/Sep/2023:20:11:50] ENGINE Bus STARTED 2023-09-23T20:11:51.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:51 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:11:52.713 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:52 smithi018 ceph-mon[109137]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:11:52.713 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:52 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:11:52.713 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:52 smithi018 ceph-mon[95889]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:11:52.713 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:52 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:11:52.965 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:52 smithi149 ceph-mon[100764]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:11:52.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:52 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:11:53.206 INFO:teuthology.orchestra.run.smithi018.stdout:[client.0] 2023-09-23T20:11:53.206 INFO:teuthology.orchestra.run.smithi018.stdout: key = AQCJRg9lA1oTDBAAknGxTxMV6XYUBOklnJEwCQ== 2023-09-23T20:11:53.633 DEBUG:teuthology.orchestra.run.smithi018:> set -ex 2023-09-23T20:11:53.634 DEBUG:teuthology.orchestra.run.smithi018:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2023-09-23T20:11:53.634 DEBUG:teuthology.orchestra.run.smithi018:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2023-09-23T20:11:53.678 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph auth get-or-create client.1 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2023-09-23T20:11:53.890 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:53 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:11:53.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:53 smithi018 ceph-mon[109137]: mgrmap e20: y(active, since 3s), standbys: x 2023-09-23T20:11:53.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:53 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:11:53.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:53 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:11:53.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:53 smithi018 ceph-mon[109137]: from='client.? 172.21.15.18:0/1928457013' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-23T20:11:53.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:53 smithi018 ceph-mon[109137]: from='client.? 172.21.15.18:0/1928457013' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-09-23T20:11:53.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:53 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:11:53.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:53 smithi018 ceph-mon[95889]: mgrmap e20: y(active, since 3s), standbys: x 2023-09-23T20:11:53.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:53 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:11:53.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:53 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:11:53.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:53 smithi018 ceph-mon[95889]: from='client.? 172.21.15.18:0/1928457013' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-23T20:11:53.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:53 smithi018 ceph-mon[95889]: from='client.? 172.21.15.18:0/1928457013' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-09-23T20:11:53.965 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:11:53 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[102880]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:11:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T20:11:53.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:53 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:11:53.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:53 smithi149 ceph-mon[100764]: mgrmap e20: y(active, since 3s), standbys: x 2023-09-23T20:11:53.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:53 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:11:53.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:53 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:11:53.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:53 smithi149 ceph-mon[100764]: from='client.? 172.21.15.18:0/1928457013' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-23T20:11:53.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:53 smithi149 ceph-mon[100764]: from='client.? 172.21.15.18:0/1928457013' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-09-23T20:11:54.278 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:11:54.795 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:54 smithi149 ceph-mon[100764]: pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:11:54.795 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:54 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:11:54.795 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:54 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:11:54.795 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:54 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:11:54.795 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:54 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:11:54.796 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:54 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:11:54.796 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:54 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:11:54.796 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:54 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:11:54.796 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:54 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:11:54.796 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:54 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:11:54.796 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:54 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:11:54.796 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:54 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:11:55.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:54 smithi018 ceph-mon[109137]: pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:11:55.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:54 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:11:55.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:54 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:11:55.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:54 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:11:55.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:54 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:11:55.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:54 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:11:55.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:54 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:11:55.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:54 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:11:55.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:54 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:11:55.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:54 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:11:55.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:54 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:11:55.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:54 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:11:55.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:54 smithi018 ceph-mon[95889]: pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:11:55.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:54 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:11:55.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:54 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:11:55.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:54 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:11:55.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:54 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:11:55.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:54 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:11:55.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:54 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:11:55.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:54 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:11:55.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:54 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:11:55.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:54 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:11:55.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:54 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:11:55.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:54 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:11:55.789 INFO:teuthology.orchestra.run.smithi149.stdout:[client.1] 2023-09-23T20:11:55.789 INFO:teuthology.orchestra.run.smithi149.stdout: key = AQCLRg9lmWy9LhAAsgvOJgfexgYNd6oWDhNb5w== 2023-09-23T20:11:55.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:55 smithi149 ceph-mon[100764]: Adjusting osd_memory_target on smithi149 to 1213M 2023-09-23T20:11:55.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:55 smithi149 ceph-mon[100764]: Updating smithi018:/etc/ceph/ceph.conf 2023-09-23T20:11:55.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:55 smithi149 ceph-mon[100764]: Updating smithi149:/etc/ceph/ceph.conf 2023-09-23T20:11:55.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:55 smithi149 ceph-mon[100764]: Updating smithi018:/var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T20:11:55.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:55 smithi149 ceph-mon[100764]: Updating smithi149:/var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T20:11:55.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:55 smithi149 ceph-mon[100764]: Updating smithi149:/etc/ceph/ceph.client.admin.keyring 2023-09-23T20:11:55.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:55 smithi149 ceph-mon[100764]: Updating smithi018:/etc/ceph/ceph.client.admin.keyring 2023-09-23T20:11:55.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:55 smithi149 ceph-mon[100764]: Updating smithi018:/var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T20:11:55.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:55 smithi149 ceph-mon[100764]: Updating smithi149:/var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T20:11:55.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:55 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:11:55.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:55 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:11:55.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:55 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:11:55.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:55 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:11:55.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:55 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:11:56.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:55 smithi018 ceph-mon[109137]: Adjusting osd_memory_target on smithi149 to 1213M 2023-09-23T20:11:56.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:55 smithi018 ceph-mon[109137]: Updating smithi018:/etc/ceph/ceph.conf 2023-09-23T20:11:56.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:55 smithi018 ceph-mon[109137]: Updating smithi149:/etc/ceph/ceph.conf 2023-09-23T20:11:56.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:55 smithi018 ceph-mon[109137]: Updating smithi018:/var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T20:11:56.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:55 smithi018 ceph-mon[109137]: Updating smithi149:/var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T20:11:56.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:55 smithi018 ceph-mon[109137]: Updating smithi149:/etc/ceph/ceph.client.admin.keyring 2023-09-23T20:11:56.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:55 smithi018 ceph-mon[109137]: Updating smithi018:/etc/ceph/ceph.client.admin.keyring 2023-09-23T20:11:56.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:55 smithi018 ceph-mon[109137]: Updating smithi018:/var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T20:11:56.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:55 smithi018 ceph-mon[109137]: Updating smithi149:/var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T20:11:56.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:55 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:11:56.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:55 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:11:56.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:55 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:11:56.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:55 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:11:56.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:55 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:11:56.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:55 smithi018 ceph-mon[95889]: Adjusting osd_memory_target on smithi149 to 1213M 2023-09-23T20:11:56.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:55 smithi018 ceph-mon[95889]: Updating smithi018:/etc/ceph/ceph.conf 2023-09-23T20:11:56.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:55 smithi018 ceph-mon[95889]: Updating smithi149:/etc/ceph/ceph.conf 2023-09-23T20:11:56.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:55 smithi018 ceph-mon[95889]: Updating smithi018:/var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T20:11:56.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:55 smithi018 ceph-mon[95889]: Updating smithi149:/var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T20:11:56.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:55 smithi018 ceph-mon[95889]: Updating smithi149:/etc/ceph/ceph.client.admin.keyring 2023-09-23T20:11:56.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:55 smithi018 ceph-mon[95889]: Updating smithi018:/etc/ceph/ceph.client.admin.keyring 2023-09-23T20:11:56.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:55 smithi018 ceph-mon[95889]: Updating smithi018:/var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T20:11:56.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:55 smithi018 ceph-mon[95889]: Updating smithi149:/var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T20:11:56.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:55 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:11:56.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:55 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:11:56.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:55 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:11:56.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:55 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:11:56.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:55 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:11:56.222 DEBUG:teuthology.orchestra.run.smithi149:> set -ex 2023-09-23T20:11:56.222 DEBUG:teuthology.orchestra.run.smithi149:> sudo dd of=/etc/ceph/ceph.client.1.keyring 2023-09-23T20:11:56.222 DEBUG:teuthology.orchestra.run.smithi149:> sudo chmod 0644 /etc/ceph/ceph.client.1.keyring 2023-09-23T20:11:56.263 INFO:tasks.cephadm:Waiting for OSDs to come up 2023-09-23T20:11:56.263 DEBUG:teuthology.orchestra.run.smithi018:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster ceph osd dump --format=json 2023-09-23T20:11:56.609 INFO:teuthology.misc.health.smithi018.stdout: 2023-09-23T20:11:56.609 INFO:teuthology.misc.health.smithi018.stdout:{"epoch":60,"fsid":"a3b3b17e-5a4c-11ee-8db4-212e2dc638e7","created":"2023-09-23T20:06:54.359034+0000","modified":"2023-09-23T20:11:49.652840+0000","last_up_change":"2023-09-23T20:11:23.829606+0000","last_in_change":"2023-09-23T20:11:09.202130+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-09-23T20:09:47.162973+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"44c8aaa3-bbd1-4330-8d34-73f7a08b0d4d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6802","nonce":2209550685},{"type":"v1","addr":"172.21.15.18:6803","nonce":2209550685}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6804","nonce":2209550685},{"type":"v1","addr":"172.21.15.18:6805","nonce":2209550685}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6808","nonce":2209550685},{"type":"v1","addr":"172.21.15.18:6809","nonce":2209550685}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6806","nonce":2209550685},{"type":"v1","addr":"172.21.15.18:6807","nonce":2209550685}]},"public_addr":"172.21.15.18:6803/2209550685","cluster_addr":"172.21.15.18:6805/2209550685","heartbeat_back_addr":"172.21.15.18:6809/2209550685","heartbeat_front_addr":"172.21.15.18:6807/2209550685","state":["exists","up"]},{"osd":1,"uuid":"c0534324-49c8-4804-bced-c4d13510e74a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6810","nonce":314354451},{"type":"v1","addr":"172.21.15.18:6811","nonce":314354451}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6812","nonce":314354451},{"type":"v1","addr":"172.21.15.18:6813","nonce":314354451}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6816","nonce":314354451},{"type":"v1","addr":"172.21.15.18:6817","nonce":314354451}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6814","nonce":314354451},{"type":"v1","addr":"172.21.15.18:6815","nonce":314354451}]},"public_addr":"172.21.15.18:6811/314354451","cluster_addr":"172.21.15.18:6813/314354451","heartbeat_back_addr":"172.21.15.18:6817/314354451","heartbeat_front_addr":"172.21.15.18:6815/314354451","state":["exists","up"]},{"osd":2,"uuid":"f3b2116d-a3d5-4279-91dc-eb47458d3239","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6818","nonce":1413681698},{"type":"v1","addr":"172.21.15.18:6819","nonce":1413681698}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6820","nonce":1413681698},{"type":"v1","addr":"172.21.15.18:6821","nonce":1413681698}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6824","nonce":1413681698},{"type":"v1","addr":"172.21.15.18:6825","nonce":1413681698}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6822","nonce":1413681698},{"type":"v1","addr":"172.21.15.18:6823","nonce":1413681698}]},"public_addr":"172.21.15.18:6819/1413681698","cluster_addr":"172.21.15.18:6821/1413681698","heartbeat_back_addr":"172.21.15.18:6825/1413681698","heartbeat_front_addr":"172.21.15.18:6823/1413681698","state":["exists","up"]},{"osd":3,"uuid":"8edccb7f-c4c9-47f3-a6f5-38a1a38cb099","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6826","nonce":423024182},{"type":"v1","addr":"172.21.15.18:6827","nonce":423024182}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6828","nonce":423024182},{"type":"v1","addr":"172.21.15.18:6829","nonce":423024182}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6832","nonce":423024182},{"type":"v1","addr":"172.21.15.18:6833","nonce":423024182}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6830","nonce":423024182},{"type":"v1","addr":"172.21.15.18:6831","nonce":423024182}]},"public_addr":"172.21.15.18:6827/423024182","cluster_addr":"172.21.15.18:6829/423024182","heartbeat_back_addr":"172.21.15.18:6833/423024182","heartbeat_front_addr":"172.21.15.18:6831/423024182","state":["exists","up"]},{"osd":4,"uuid":"1d5a64ce-fe42-4355-991a-dfb20cbe5423","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6800","nonce":3640432477},{"type":"v1","addr":"172.21.15.149:6801","nonce":3640432477}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6802","nonce":3640432477},{"type":"v1","addr":"172.21.15.149:6803","nonce":3640432477}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6806","nonce":3640432477},{"type":"v1","addr":"172.21.15.149:6807","nonce":3640432477}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6804","nonce":3640432477},{"type":"v1","addr":"172.21.15.149:6805","nonce":3640432477}]},"public_addr":"172.21.15.149:6801/3640432477","cluster_addr":"172.21.15.149:6803/3640432477","heartbeat_back_addr":"172.21.15.149:6807/3640432477","heartbeat_front_addr":"172.21.15.149:6805/3640432477","state":["exists","up"]},{"osd":5,"uuid":"105ae3b2-b2fc-477d-9134-3d517bc9427e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6808","nonce":2969709367},{"type":"v1","addr":"172.21.15.149:6809","nonce":2969709367}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6810","nonce":2969709367},{"type":"v1","addr":"172.21.15.149:6811","nonce":2969709367}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6814","nonce":2969709367},{"type":"v1","addr":"172.21.15.149:6815","nonce":2969709367}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6812","nonce":2969709367},{"type":"v1","addr":"172.21.15.149:6813","nonce":2969709367}]},"public_addr":"172.21.15.149:6809/2969709367","cluster_addr":"172.21.15.149:6811/2969709367","heartbeat_back_addr":"172.21.15.149:6815/2969709367","heartbeat_front_addr":"172.21.15.149:6813/2969709367","state":["exists","up"]},{"osd":6,"uuid":"27d2d2a8-e8c6-4a05-a85e-bd54376cf4c1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6816","nonce":1291453593},{"type":"v1","addr":"172.21.15.149:6817","nonce":1291453593}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6818","nonce":1291453593},{"type":"v1","addr":"172.21.15.149:6819","nonce":1291453593}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6822","nonce":1291453593},{"type":"v1","addr":"172.21.15.149:6823","nonce":1291453593}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6820","nonce":1291453593},{"type":"v1","addr":"172.21.15.149:6821","nonce":1291453593}]},"public_addr":"172.21.15.149:6817/1291453593","cluster_addr":"172.21.15.149:6819/1291453593","heartbeat_back_addr":"172.21.15.149:6823/1291453593","heartbeat_front_addr":"172.21.15.149:6821/1291453593","state":["exists","up"]},{"osd":7,"uuid":"9da636f7-995c-4f29-9264-70d09c6f66c6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6824","nonce":3117985178},{"type":"v1","addr":"172.21.15.149:6825","nonce":3117985178}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6826","nonce":3117985178},{"type":"v1","addr":"172.21.15.149:6827","nonce":3117985178}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6830","nonce":3117985178},{"type":"v1","addr":"172.21.15.149:6831","nonce":3117985178}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6828","nonce":3117985178},{"type":"v1","addr":"172.21.15.149:6829","nonce":3117985178}]},"public_addr":"172.21.15.149:6825/3117985178","cluster_addr":"172.21.15.149:6827/3117985178","heartbeat_back_addr":"172.21.15.149:6831/3117985178","heartbeat_front_addr":"172.21.15.149:6829/3117985178","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:09:01.034520+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:09:21.795238+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:09:42.987835+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:10:04.092458+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:10:23.309112+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:10:43.348231+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:11:03.128458+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:11:23.084228+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.18:0/3912151186":"2023-09-24T20:11:49.652762+0000","172.21.15.18:0/3863388488":"2023-09-24T20:11:49.652762+0000","172.21.15.18:6801/2611527229":"2023-09-24T20:11:49.652762+0000","172.21.15.18:6800/2611527229":"2023-09-24T20:11:49.652762+0000","172.21.15.18:0/1698512393":"2023-09-24T20:11:49.652762+0000","172.21.15.18:6801/380813540":"2023-09-24T20:07:48.635784+0000","172.21.15.18:6800/380813540":"2023-09-24T20:07:48.635784+0000","172.21.15.18:0/3831267686":"2023-09-24T20:07:48.635784+0000","172.21.15.18:0/950924496":"2023-09-24T20:07:23.919815+0000","172.21.15.18:0/1988472148":"2023-09-24T20:11:49.652762+0000","172.21.15.18:0/1082665289":"2023-09-24T20:07:48.635784+0000","172.21.15.18:0/2993111051":"2023-09-24T20:07:23.919815+0000","172.21.15.18:0/69122222":"2023-09-24T20:07:23.919815+0000","172.21.15.18:0/1645667747":"2023-09-24T20:07:48.635784+0000","172.21.15.18:6800/1930027051":"2023-09-24T20:07:23.919815+0000","172.21.15.18:6801/1930027051":"2023-09-24T20:07:23.919815+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-23T20:11:56.622 DEBUG:teuthology.misc:8 of 8 OSDs are up 2023-09-23T20:11:56.622 INFO:tasks.cephadm:Creating RBD pool 2023-09-23T20:11:56.622 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- sudo ceph --cluster ceph osd pool create rbd 8 2023-09-23T20:11:56.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:56 smithi149 ceph-mon[100764]: Deploying daemon alertmanager.a on smithi018 2023-09-23T20:11:56.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:56 smithi149 ceph-mon[100764]: pgmap v6: 1 pgs: 1 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:11:56.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:56 smithi149 ceph-mon[100764]: from='client.? 172.21.15.149:0/3465038008' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-23T20:11:56.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:56 smithi149 ceph-mon[100764]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-23T20:11:56.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:56 smithi149 ceph-mon[100764]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-09-23T20:11:56.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:56 smithi149 ceph-mon[100764]: from='client.? 172.21.15.18:0/1920937858' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T20:11:57.140 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:56 smithi018 ceph-mon[109137]: Deploying daemon alertmanager.a on smithi018 2023-09-23T20:11:57.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:56 smithi018 ceph-mon[109137]: pgmap v6: 1 pgs: 1 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:11:57.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:56 smithi018 ceph-mon[109137]: from='client.? 172.21.15.149:0/3465038008' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-23T20:11:57.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:56 smithi018 ceph-mon[109137]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-23T20:11:57.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:56 smithi018 ceph-mon[109137]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-09-23T20:11:57.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:56 smithi018 ceph-mon[109137]: from='client.? 172.21.15.18:0/1920937858' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T20:11:57.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:56 smithi018 ceph-mon[95889]: Deploying daemon alertmanager.a on smithi018 2023-09-23T20:11:57.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:56 smithi018 ceph-mon[95889]: pgmap v6: 1 pgs: 1 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:11:57.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:56 smithi018 ceph-mon[95889]: from='client.? 172.21.15.149:0/3465038008' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-23T20:11:57.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:56 smithi018 ceph-mon[95889]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-23T20:11:57.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:56 smithi018 ceph-mon[95889]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-09-23T20:11:57.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:56 smithi018 ceph-mon[95889]: from='client.? 172.21.15.18:0/1920937858' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T20:11:57.240 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:11:58.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:58 smithi018 ceph-mon[95889]: pgmap v7: 1 pgs: 1 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:11:58.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:58 smithi018 ceph-mon[109137]: pgmap v7: 1 pgs: 1 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:11:58.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:58 smithi149 ceph-mon[100764]: pgmap v7: 1 pgs: 1 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:11:59.848 INFO:teuthology.orchestra.run.smithi018.stderr:pool 'rbd' created 2023-09-23T20:12:00.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:59 smithi018 ceph-mon[109137]: from='client.? 172.21.15.18:0/496009648' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-09-23T20:12:00.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:11:59 smithi018 ceph-mon[109137]: pgmap v8: 1 pgs: 1 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:12:00.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:59 smithi018 ceph-mon[95889]: from='client.? 172.21.15.18:0/496009648' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-09-23T20:12:00.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:11:59 smithi018 ceph-mon[95889]: pgmap v8: 1 pgs: 1 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:12:00.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:59 smithi149 ceph-mon[100764]: from='client.? 172.21.15.18:0/496009648' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-09-23T20:12:00.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:11:59 smithi149 ceph-mon[100764]: pgmap v8: 1 pgs: 1 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:12:00.597 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- sudo ceph --cluster ceph osd pool application enable rbd rbd --yes-i-really-mean-it 2023-09-23T20:12:00.850 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:00 smithi018 ceph-mon[95889]: from='client.? 172.21.15.18:0/496009648' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2023-09-23T20:12:00.850 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:00 smithi018 ceph-mon[95889]: osdmap e61: 8 total, 8 up, 8 in 2023-09-23T20:12:01.140 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:00 smithi018 ceph-mon[109137]: from='client.? 172.21.15.18:0/496009648' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2023-09-23T20:12:01.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:00 smithi018 ceph-mon[109137]: osdmap e61: 8 total, 8 up, 8 in 2023-09-23T20:12:01.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:00 smithi149 ceph-mon[100764]: from='client.? 172.21.15.18:0/496009648' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2023-09-23T20:12:01.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:00 smithi149 ceph-mon[100764]: osdmap e61: 8 total, 8 up, 8 in 2023-09-23T20:12:01.313 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:12:02.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:01 smithi018 ceph-mon[109137]: osdmap e62: 8 total, 8 up, 8 in 2023-09-23T20:12:02.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:01 smithi018 ceph-mon[109137]: pgmap v11: 9 pgs: 8 creating+peering, 1 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:12:02.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:01 smithi018 ceph-mon[95889]: osdmap e62: 8 total, 8 up, 8 in 2023-09-23T20:12:02.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:01 smithi018 ceph-mon[95889]: pgmap v11: 9 pgs: 8 creating+peering, 1 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:12:02.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:01 smithi149 ceph-mon[100764]: osdmap e62: 8 total, 8 up, 8 in 2023-09-23T20:12:02.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:01 smithi149 ceph-mon[100764]: pgmap v11: 9 pgs: 8 creating+peering, 1 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:12:03.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:02 smithi018 ceph-mon[109137]: osdmap e63: 8 total, 8 up, 8 in 2023-09-23T20:12:03.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:02 smithi018 ceph-mon[95889]: osdmap e63: 8 total, 8 up, 8 in 2023-09-23T20:12:03.215 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:02 smithi149 ceph-mon[100764]: osdmap e63: 8 total, 8 up, 8 in 2023-09-23T20:12:03.528 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 23 20:12:03 smithi018 systemd[1]: Starting Ceph alertmanager.a for a3b3b17e-5a4c-11ee-8db4-212e2dc638e7... 2023-09-23T20:12:03.528 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 23 20:12:03 smithi018 podman[145407]: 2023-09-23T20:12:03.789 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 23 20:12:03 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-alertmanager-a[145423]: ts=2023-09-23T20:12:03.759Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2023-09-23T20:12:03.789 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 23 20:12:03 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-alertmanager-a[145423]: ts=2023-09-23T20:12:03.759Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2023-09-23T20:12:03.789 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 23 20:12:03 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-alertmanager-a[145423]: ts=2023-09-23T20:12:03.761Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.18 port=9094 2023-09-23T20:12:03.790 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 23 20:12:03 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-alertmanager-a[145423]: ts=2023-09-23T20:12:03.762Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2023-09-23T20:12:03.790 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 23 20:12:03 smithi018 bash[145407]: b39e964ccebca0f702db823a17c86f13cbfe661fcf7fbbc868decd57be71e93e 2023-09-23T20:12:03.790 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 23 20:12:03 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-alertmanager-a[145423]: ts=2023-09-23T20:12:03.789Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2023-09-23T20:12:03.868 INFO:teuthology.orchestra.run.smithi018.stderr:enabled application 'rbd' on pool 'rbd' 2023-09-23T20:12:04.140 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:03 smithi018 ceph-mon[109137]: from='client.? 172.21.15.18:0/1877527282' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T20:12:04.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:03 smithi018 ceph-mon[109137]: pgmap v13: 9 pgs: 8 creating+peering, 1 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:12:04.141 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 23 20:12:03 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-alertmanager-a[145423]: ts=2023-09-23T20:12:03.790Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2023-09-23T20:12:04.141 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 23 20:12:03 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-alertmanager-a[145423]: ts=2023-09-23T20:12:03.793Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2023-09-23T20:12:04.141 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 23 20:12:03 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-alertmanager-a[145423]: ts=2023-09-23T20:12:03.793Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2023-09-23T20:12:04.141 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 23 20:12:03 smithi018 systemd[1]: Started Ceph alertmanager.a for a3b3b17e-5a4c-11ee-8db4-212e2dc638e7. 2023-09-23T20:12:04.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:03 smithi018 ceph-mon[95889]: from='client.? 172.21.15.18:0/1877527282' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T20:12:04.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:03 smithi018 ceph-mon[95889]: pgmap v13: 9 pgs: 8 creating+peering, 1 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:12:04.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:03 smithi149 ceph-mon[100764]: from='client.? 172.21.15.18:0/1877527282' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T20:12:04.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:03 smithi149 ceph-mon[100764]: pgmap v13: 9 pgs: 8 creating+peering, 1 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:12:04.216 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:12:03 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[102880]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:12:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T20:12:04.264 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2023-09-23T20:12:04.264 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2023-09-23T20:12:04.264 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph mgr dump --format=json 2023-09-23T20:12:04.862 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:12:04 smithi018 systemd[1]: Starting Ceph node-exporter.a for a3b3b17e-5a4c-11ee-8db4-212e2dc638e7... 2023-09-23T20:12:04.866 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:04 smithi018 ceph-mon[95889]: from='client.? 172.21.15.18:0/1877527282' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2023-09-23T20:12:04.866 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:04 smithi018 ceph-mon[95889]: osdmap e64: 8 total, 8 up, 8 in 2023-09-23T20:12:04.866 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:04 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:04.866 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:04 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:04.866 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:04 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:04.866 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:04 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:04.867 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:04 smithi018 ceph-mon[95889]: Deploying daemon node-exporter.a on smithi018 2023-09-23T20:12:04.867 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:04 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:04.867 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:04 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:12:04.962 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:12:05.140 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:12:05 smithi018 bash[145761]: Trying to pull quay.io/prometheus/node-exporter:v1.5.0... 2023-09-23T20:12:05.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:04 smithi018 ceph-mon[109137]: from='client.? 172.21.15.18:0/1877527282' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2023-09-23T20:12:05.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:04 smithi018 ceph-mon[109137]: osdmap e64: 8 total, 8 up, 8 in 2023-09-23T20:12:05.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:04 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:05.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:04 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:05.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:04 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:05.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:04 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:05.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:04 smithi018 ceph-mon[109137]: Deploying daemon node-exporter.a on smithi018 2023-09-23T20:12:05.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:04 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:05.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:04 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:12:05.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:04 smithi149 ceph-mon[100764]: from='client.? 172.21.15.18:0/1877527282' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2023-09-23T20:12:05.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:04 smithi149 ceph-mon[100764]: osdmap e64: 8 total, 8 up, 8 in 2023-09-23T20:12:05.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:04 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:05.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:04 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:05.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:04 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:05.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:04 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:05.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:04 smithi149 ceph-mon[100764]: Deploying daemon node-exporter.a on smithi018 2023-09-23T20:12:05.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:04 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:05.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:04 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:12:06.026 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:05 smithi018 ceph-mon[95889]: pgmap v15: 9 pgs: 8 creating+peering, 1 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:12:06.026 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:05 smithi018 ceph-mon[109137]: pgmap v15: 9 pgs: 8 creating+peering, 1 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:12:06.027 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:12:05 smithi018 bash[145761]: Getting image source signatures 2023-09-23T20:12:06.027 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:12:05 smithi018 bash[145761]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2023-09-23T20:12:06.027 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:12:05 smithi018 bash[145761]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2023-09-23T20:12:06.027 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:12:05 smithi018 bash[145761]: Copying blob sha256:c0e87333d3805a7b7e1c25783dd25f27da0fe70a8e262eee7ed190c8dc843fa7 2023-09-23T20:12:06.027 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:12:05 smithi018 bash[145761]: Copying blob sha256:c0e87333d3805a7b7e1c25783dd25f27da0fe70a8e262eee7ed190c8dc843fa7 2023-09-23T20:12:06.027 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:12:05 smithi018 bash[145761]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2023-09-23T20:12:06.027 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:12:05 smithi018 bash[145761]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2023-09-23T20:12:06.027 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 23 20:12:05 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-alertmanager-a[145423]: ts=2023-09-23T20:12:05.762Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000864688s 2023-09-23T20:12:06.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:05 smithi149 ceph-mon[100764]: pgmap v15: 9 pgs: 8 creating+peering, 1 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:12:06.527 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:12:06.867 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:06 smithi018 ceph-mon[95889]: from='client.? 172.21.15.18:0/1949000616' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-09-23T20:12:07.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:06 smithi018 ceph-mon[109137]: from='client.? 172.21.15.18:0/1949000616' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-09-23T20:12:07.141 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:12:07 smithi018 bash[145761]: Copying config sha256:0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 2023-09-23T20:12:07.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:06 smithi149 ceph-mon[100764]: from='client.? 172.21.15.18:0/1949000616' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-09-23T20:12:07.316 INFO:teuthology.orchestra.run.smithi018.stdout:{"epoch":20,"active_gid":14454,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6800","nonce":115279545},{"type":"v1","addr":"172.21.15.18:6801","nonce":115279545}]},"active_addr":"172.21.15.18:6801/115279545","active_change":"2023-09-23T20:11:49.652957+0000","active_mgr_features":4540138320759226367,"available":true,"standbys":[{"gid":24361,"name":"x","mgr_features":4540138320759226367,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.18:8443/","prometheus":"http://172.21.15.18:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":60,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.18:0","nonce":3356195690}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.18:0","nonce":2809002822}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.18:0","nonce":3990640673}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.18:0","nonce":2543728509}]}]} 2023-09-23T20:12:07.321 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2023-09-23T20:12:07.321 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2023-09-23T20:12:07.321 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd dump --format=json 2023-09-23T20:12:07.641 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:12:07 smithi018 bash[145761]: Writing manifest to image destination 2023-09-23T20:12:07.641 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:12:07 smithi018 bash[145761]: Storing signatures 2023-09-23T20:12:07.964 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:07 smithi018 ceph-mon[109137]: pgmap v16: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:12:07.964 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:07 smithi018 ceph-mon[95889]: pgmap v16: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:12:07.965 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:12:07 smithi018 podman[145761]: 2023-09-23T20:12:07.965 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:12:07 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-a[146018]: ts=2023-09-23T20:12:07.892Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2023-09-23T20:12:07.965 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:12:07 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-a[146018]: ts=2023-09-23T20:12:07.892Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2023-09-23T20:12:07.965 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:12:07 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-a[146018]: ts=2023-09-23T20:12:07.893Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2023-09-23T20:12:07.965 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:12:07 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-a[146018]: ts=2023-09-23T20:12:07.893Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2023-09-23T20:12:07.965 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:12:07 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-a[146018]: ts=2023-09-23T20:12:07.893Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2023-09-23T20:12:07.965 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:12:07 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-a[146018]: ts=2023-09-23T20:12:07.893Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2023-09-23T20:12:07.966 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:12:07 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-a[146018]: ts=2023-09-23T20:12:07.893Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2023-09-23T20:12:07.966 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:12:07 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-a[146018]: ts=2023-09-23T20:12:07.893Z caller=node_exporter.go:117 level=info collector=arp 2023-09-23T20:12:07.966 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:12:07 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-a[146018]: ts=2023-09-23T20:12:07.893Z caller=node_exporter.go:117 level=info collector=bcache 2023-09-23T20:12:07.966 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:12:07 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-a[146018]: ts=2023-09-23T20:12:07.893Z caller=node_exporter.go:117 level=info collector=bonding 2023-09-23T20:12:07.966 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:12:07 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-a[146018]: ts=2023-09-23T20:12:07.893Z caller=node_exporter.go:117 level=info collector=btrfs 2023-09-23T20:12:07.966 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:12:07 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-a[146018]: ts=2023-09-23T20:12:07.893Z caller=node_exporter.go:117 level=info collector=conntrack 2023-09-23T20:12:07.966 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:12:07 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-a[146018]: ts=2023-09-23T20:12:07.893Z caller=node_exporter.go:117 level=info collector=cpu 2023-09-23T20:12:07.966 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:12:07 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-a[146018]: ts=2023-09-23T20:12:07.893Z caller=node_exporter.go:117 level=info collector=cpufreq 2023-09-23T20:12:07.966 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:12:07 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-a[146018]: ts=2023-09-23T20:12:07.893Z caller=node_exporter.go:117 level=info collector=diskstats 2023-09-23T20:12:07.966 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:12:07 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-a[146018]: ts=2023-09-23T20:12:07.893Z caller=node_exporter.go:117 level=info collector=dmi 2023-09-23T20:12:07.966 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:12:07 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-a[146018]: ts=2023-09-23T20:12:07.893Z caller=node_exporter.go:117 level=info collector=edac 2023-09-23T20:12:07.967 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:12:07 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-a[146018]: ts=2023-09-23T20:12:07.893Z caller=node_exporter.go:117 level=info collector=entropy 2023-09-23T20:12:07.967 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:12:07 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-a[146018]: ts=2023-09-23T20:12:07.893Z caller=node_exporter.go:117 level=info collector=fibrechannel 2023-09-23T20:12:07.967 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:12:07 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-a[146018]: ts=2023-09-23T20:12:07.894Z caller=node_exporter.go:117 level=info collector=filefd 2023-09-23T20:12:07.967 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:12:07 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-a[146018]: ts=2023-09-23T20:12:07.894Z caller=node_exporter.go:117 level=info collector=filesystem 2023-09-23T20:12:07.967 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:12:07 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-a[146018]: ts=2023-09-23T20:12:07.894Z caller=node_exporter.go:117 level=info collector=hwmon 2023-09-23T20:12:07.967 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:12:07 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-a[146018]: ts=2023-09-23T20:12:07.894Z caller=node_exporter.go:117 level=info collector=infiniband 2023-09-23T20:12:07.967 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:12:07 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-a[146018]: ts=2023-09-23T20:12:07.894Z caller=node_exporter.go:117 level=info collector=ipvs 2023-09-23T20:12:07.967 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:12:07 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-a[146018]: ts=2023-09-23T20:12:07.894Z caller=node_exporter.go:117 level=info collector=loadavg 2023-09-23T20:12:07.967 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:12:07 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-a[146018]: ts=2023-09-23T20:12:07.894Z caller=node_exporter.go:117 level=info collector=mdadm 2023-09-23T20:12:07.967 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:12:07 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-a[146018]: ts=2023-09-23T20:12:07.894Z caller=node_exporter.go:117 level=info collector=meminfo 2023-09-23T20:12:07.967 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:12:07 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-a[146018]: ts=2023-09-23T20:12:07.894Z caller=node_exporter.go:117 level=info collector=netclass 2023-09-23T20:12:07.967 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:12:07 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-a[146018]: ts=2023-09-23T20:12:07.894Z caller=node_exporter.go:117 level=info collector=netdev 2023-09-23T20:12:07.968 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:12:07 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-a[146018]: ts=2023-09-23T20:12:07.894Z caller=node_exporter.go:117 level=info collector=netstat 2023-09-23T20:12:07.968 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:12:07 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-a[146018]: ts=2023-09-23T20:12:07.894Z caller=node_exporter.go:117 level=info collector=nfs 2023-09-23T20:12:07.968 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:12:07 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-a[146018]: ts=2023-09-23T20:12:07.894Z caller=node_exporter.go:117 level=info collector=nfsd 2023-09-23T20:12:07.968 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:12:07 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-a[146018]: ts=2023-09-23T20:12:07.894Z caller=node_exporter.go:117 level=info collector=nvme 2023-09-23T20:12:07.968 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:12:07 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-a[146018]: ts=2023-09-23T20:12:07.894Z caller=node_exporter.go:117 level=info collector=os 2023-09-23T20:12:07.968 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:12:07 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-a[146018]: ts=2023-09-23T20:12:07.894Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2023-09-23T20:12:07.968 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:12:07 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-a[146018]: ts=2023-09-23T20:12:07.894Z caller=node_exporter.go:117 level=info collector=pressure 2023-09-23T20:12:07.968 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:12:07 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-a[146018]: ts=2023-09-23T20:12:07.894Z caller=node_exporter.go:117 level=info collector=rapl 2023-09-23T20:12:07.968 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:12:07 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-a[146018]: ts=2023-09-23T20:12:07.894Z caller=node_exporter.go:117 level=info collector=schedstat 2023-09-23T20:12:07.968 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:12:07 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-a[146018]: ts=2023-09-23T20:12:07.894Z caller=node_exporter.go:117 level=info collector=selinux 2023-09-23T20:12:07.968 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:12:07 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-a[146018]: ts=2023-09-23T20:12:07.894Z caller=node_exporter.go:117 level=info collector=sockstat 2023-09-23T20:12:07.968 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:12:07 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-a[146018]: ts=2023-09-23T20:12:07.894Z caller=node_exporter.go:117 level=info collector=softnet 2023-09-23T20:12:07.969 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:12:07 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-a[146018]: ts=2023-09-23T20:12:07.894Z caller=node_exporter.go:117 level=info collector=stat 2023-09-23T20:12:07.969 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:12:07 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-a[146018]: ts=2023-09-23T20:12:07.894Z caller=node_exporter.go:117 level=info collector=tapestats 2023-09-23T20:12:07.969 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:12:07 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-a[146018]: ts=2023-09-23T20:12:07.894Z caller=node_exporter.go:117 level=info collector=textfile 2023-09-23T20:12:07.969 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:12:07 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-a[146018]: ts=2023-09-23T20:12:07.894Z caller=node_exporter.go:117 level=info collector=thermal_zone 2023-09-23T20:12:07.969 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:12:07 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-a[146018]: ts=2023-09-23T20:12:07.894Z caller=node_exporter.go:117 level=info collector=time 2023-09-23T20:12:07.969 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:12:07 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-a[146018]: ts=2023-09-23T20:12:07.894Z caller=node_exporter.go:117 level=info collector=udp_queues 2023-09-23T20:12:07.969 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:12:07 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-a[146018]: ts=2023-09-23T20:12:07.894Z caller=node_exporter.go:117 level=info collector=uname 2023-09-23T20:12:07.969 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:12:07 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-a[146018]: ts=2023-09-23T20:12:07.894Z caller=node_exporter.go:117 level=info collector=vmstat 2023-09-23T20:12:07.969 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:12:07 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-a[146018]: ts=2023-09-23T20:12:07.894Z caller=node_exporter.go:117 level=info collector=xfs 2023-09-23T20:12:07.969 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:12:07 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-a[146018]: ts=2023-09-23T20:12:07.894Z caller=node_exporter.go:117 level=info collector=zfs 2023-09-23T20:12:07.969 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:12:07 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-a[146018]: ts=2023-09-23T20:12:07.894Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2023-09-23T20:12:07.969 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:12:07 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-a[146018]: ts=2023-09-23T20:12:07.894Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2023-09-23T20:12:07.970 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:12:07 smithi018 bash[145761]: 46e682cc69120327f7bafd37011371911a51fcd8b6580fc563c50c04bb3b206b 2023-09-23T20:12:08.103 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:12:08.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:07 smithi149 ceph-mon[100764]: pgmap v16: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:12:08.284 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:12:07 smithi018 systemd[1]: Started Ceph node-exporter.a for a3b3b17e-5a4c-11ee-8db4-212e2dc638e7. 2023-09-23T20:12:08.966 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:12:08 smithi149 systemd[1]: Starting Ceph node-exporter.b for a3b3b17e-5a4c-11ee-8db4-212e2dc638e7... 2023-09-23T20:12:09.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:09 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:09.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:09 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:09.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:09 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:09.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:09 smithi018 ceph-mon[109137]: Deploying daemon node-exporter.b on smithi149 2023-09-23T20:12:09.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:09 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:09.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:09 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:09.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:09 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:09.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:09 smithi018 ceph-mon[95889]: Deploying daemon node-exporter.b on smithi149 2023-09-23T20:12:09.417 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:12:09.417 INFO:teuthology.orchestra.run.smithi018.stdout:{"epoch":64,"fsid":"a3b3b17e-5a4c-11ee-8db4-212e2dc638e7","created":"2023-09-23T20:06:54.359034+0000","modified":"2023-09-23T20:12:03.854680+0000","last_up_change":"2023-09-23T20:11:23.829606+0000","last_in_change":"2023-09-23T20:11:09.202130+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-09-23T20:09:47.162973+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-09-23T20:11:59.431837+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"64","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"44c8aaa3-bbd1-4330-8d34-73f7a08b0d4d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6802","nonce":2209550685},{"type":"v1","addr":"172.21.15.18:6803","nonce":2209550685}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6804","nonce":2209550685},{"type":"v1","addr":"172.21.15.18:6805","nonce":2209550685}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6808","nonce":2209550685},{"type":"v1","addr":"172.21.15.18:6809","nonce":2209550685}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6806","nonce":2209550685},{"type":"v1","addr":"172.21.15.18:6807","nonce":2209550685}]},"public_addr":"172.21.15.18:6803/2209550685","cluster_addr":"172.21.15.18:6805/2209550685","heartbeat_back_addr":"172.21.15.18:6809/2209550685","heartbeat_front_addr":"172.21.15.18:6807/2209550685","state":["exists","up"]},{"osd":1,"uuid":"c0534324-49c8-4804-bced-c4d13510e74a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6810","nonce":314354451},{"type":"v1","addr":"172.21.15.18:6811","nonce":314354451}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6812","nonce":314354451},{"type":"v1","addr":"172.21.15.18:6813","nonce":314354451}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6816","nonce":314354451},{"type":"v1","addr":"172.21.15.18:6817","nonce":314354451}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6814","nonce":314354451},{"type":"v1","addr":"172.21.15.18:6815","nonce":314354451}]},"public_addr":"172.21.15.18:6811/314354451","cluster_addr":"172.21.15.18:6813/314354451","heartbeat_back_addr":"172.21.15.18:6817/314354451","heartbeat_front_addr":"172.21.15.18:6815/314354451","state":["exists","up"]},{"osd":2,"uuid":"f3b2116d-a3d5-4279-91dc-eb47458d3239","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6818","nonce":1413681698},{"type":"v1","addr":"172.21.15.18:6819","nonce":1413681698}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6820","nonce":1413681698},{"type":"v1","addr":"172.21.15.18:6821","nonce":1413681698}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6824","nonce":1413681698},{"type":"v1","addr":"172.21.15.18:6825","nonce":1413681698}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6822","nonce":1413681698},{"type":"v1","addr":"172.21.15.18:6823","nonce":1413681698}]},"public_addr":"172.21.15.18:6819/1413681698","cluster_addr":"172.21.15.18:6821/1413681698","heartbeat_back_addr":"172.21.15.18:6825/1413681698","heartbeat_front_addr":"172.21.15.18:6823/1413681698","state":["exists","up"]},{"osd":3,"uuid":"8edccb7f-c4c9-47f3-a6f5-38a1a38cb099","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6826","nonce":423024182},{"type":"v1","addr":"172.21.15.18:6827","nonce":423024182}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6828","nonce":423024182},{"type":"v1","addr":"172.21.15.18:6829","nonce":423024182}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6832","nonce":423024182},{"type":"v1","addr":"172.21.15.18:6833","nonce":423024182}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6830","nonce":423024182},{"type":"v1","addr":"172.21.15.18:6831","nonce":423024182}]},"public_addr":"172.21.15.18:6827/423024182","cluster_addr":"172.21.15.18:6829/423024182","heartbeat_back_addr":"172.21.15.18:6833/423024182","heartbeat_front_addr":"172.21.15.18:6831/423024182","state":["exists","up"]},{"osd":4,"uuid":"1d5a64ce-fe42-4355-991a-dfb20cbe5423","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6800","nonce":3640432477},{"type":"v1","addr":"172.21.15.149:6801","nonce":3640432477}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6802","nonce":3640432477},{"type":"v1","addr":"172.21.15.149:6803","nonce":3640432477}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6806","nonce":3640432477},{"type":"v1","addr":"172.21.15.149:6807","nonce":3640432477}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6804","nonce":3640432477},{"type":"v1","addr":"172.21.15.149:6805","nonce":3640432477}]},"public_addr":"172.21.15.149:6801/3640432477","cluster_addr":"172.21.15.149:6803/3640432477","heartbeat_back_addr":"172.21.15.149:6807/3640432477","heartbeat_front_addr":"172.21.15.149:6805/3640432477","state":["exists","up"]},{"osd":5,"uuid":"105ae3b2-b2fc-477d-9134-3d517bc9427e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6808","nonce":2969709367},{"type":"v1","addr":"172.21.15.149:6809","nonce":2969709367}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6810","nonce":2969709367},{"type":"v1","addr":"172.21.15.149:6811","nonce":2969709367}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6814","nonce":2969709367},{"type":"v1","addr":"172.21.15.149:6815","nonce":2969709367}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6812","nonce":2969709367},{"type":"v1","addr":"172.21.15.149:6813","nonce":2969709367}]},"public_addr":"172.21.15.149:6809/2969709367","cluster_addr":"172.21.15.149:6811/2969709367","heartbeat_back_addr":"172.21.15.149:6815/2969709367","heartbeat_front_addr":"172.21.15.149:6813/2969709367","state":["exists","up"]},{"osd":6,"uuid":"27d2d2a8-e8c6-4a05-a85e-bd54376cf4c1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6816","nonce":1291453593},{"type":"v1","addr":"172.21.15.149:6817","nonce":1291453593}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6818","nonce":1291453593},{"type":"v1","addr":"172.21.15.149:6819","nonce":1291453593}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6822","nonce":1291453593},{"type":"v1","addr":"172.21.15.149:6823","nonce":1291453593}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6820","nonce":1291453593},{"type":"v1","addr":"172.21.15.149:6821","nonce":1291453593}]},"public_addr":"172.21.15.149:6817/1291453593","cluster_addr":"172.21.15.149:6819/1291453593","heartbeat_back_addr":"172.21.15.149:6823/1291453593","heartbeat_front_addr":"172.21.15.149:6821/1291453593","state":["exists","up"]},{"osd":7,"uuid":"9da636f7-995c-4f29-9264-70d09c6f66c6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6824","nonce":3117985178},{"type":"v1","addr":"172.21.15.149:6825","nonce":3117985178}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6826","nonce":3117985178},{"type":"v1","addr":"172.21.15.149:6827","nonce":3117985178}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6830","nonce":3117985178},{"type":"v1","addr":"172.21.15.149:6831","nonce":3117985178}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6828","nonce":3117985178},{"type":"v1","addr":"172.21.15.149:6829","nonce":3117985178}]},"public_addr":"172.21.15.149:6825/3117985178","cluster_addr":"172.21.15.149:6827/3117985178","heartbeat_back_addr":"172.21.15.149:6831/3117985178","heartbeat_front_addr":"172.21.15.149:6829/3117985178","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:09:01.034520+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:09:21.795238+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:09:42.987835+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:10:04.092458+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:10:23.309112+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:10:43.348231+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:11:03.128458+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:11:23.084228+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.18:0/3912151186":"2023-09-24T20:11:49.652762+0000","172.21.15.18:0/3863388488":"2023-09-24T20:11:49.652762+0000","172.21.15.18:6801/2611527229":"2023-09-24T20:11:49.652762+0000","172.21.15.18:6800/2611527229":"2023-09-24T20:11:49.652762+0000","172.21.15.18:0/1698512393":"2023-09-24T20:11:49.652762+0000","172.21.15.18:6801/380813540":"2023-09-24T20:07:48.635784+0000","172.21.15.18:6800/380813540":"2023-09-24T20:07:48.635784+0000","172.21.15.18:0/3831267686":"2023-09-24T20:07:48.635784+0000","172.21.15.18:0/950924496":"2023-09-24T20:07:23.919815+0000","172.21.15.18:0/1988472148":"2023-09-24T20:11:49.652762+0000","172.21.15.18:0/1082665289":"2023-09-24T20:07:48.635784+0000","172.21.15.18:0/2993111051":"2023-09-24T20:07:23.919815+0000","172.21.15.18:0/69122222":"2023-09-24T20:07:23.919815+0000","172.21.15.18:0/1645667747":"2023-09-24T20:07:48.635784+0000","172.21.15.18:6800/1930027051":"2023-09-24T20:07:23.919815+0000","172.21.15.18:6801/1930027051":"2023-09-24T20:07:23.919815+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-23T20:12:09.466 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:12:09 smithi149 bash[137508]: Trying to pull quay.io/prometheus/node-exporter:v1.5.0... 2023-09-23T20:12:09.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:09 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:09.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:09 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:09.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:09 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:09.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:09 smithi149 ceph-mon[100764]: Deploying daemon node-exporter.b on smithi149 2023-09-23T20:12:09.852 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2023-09-23T20:12:09.853 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd dump --format=json 2023-09-23T20:12:10.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:10 smithi149 ceph-mon[100764]: from='client.? 172.21.15.18:0/2127230387' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T20:12:10.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:10 smithi149 ceph-mon[100764]: pgmap v17: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:12:10.216 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:12:09 smithi149 bash[137508]: Getting image source signatures 2023-09-23T20:12:10.216 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:12:09 smithi149 bash[137508]: Copying blob sha256:c0e87333d3805a7b7e1c25783dd25f27da0fe70a8e262eee7ed190c8dc843fa7 2023-09-23T20:12:10.216 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:12:09 smithi149 bash[137508]: Copying blob sha256:c0e87333d3805a7b7e1c25783dd25f27da0fe70a8e262eee7ed190c8dc843fa7 2023-09-23T20:12:10.216 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:12:09 smithi149 bash[137508]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2023-09-23T20:12:10.216 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:12:09 smithi149 bash[137508]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2023-09-23T20:12:10.216 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:12:09 smithi149 bash[137508]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2023-09-23T20:12:10.217 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:12:09 smithi149 bash[137508]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2023-09-23T20:12:10.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:10 smithi018 ceph-mon[109137]: from='client.? 172.21.15.18:0/2127230387' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T20:12:10.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:10 smithi018 ceph-mon[109137]: pgmap v17: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:12:10.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:10 smithi018 ceph-mon[95889]: from='client.? 172.21.15.18:0/2127230387' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T20:12:10.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:10 smithi018 ceph-mon[95889]: pgmap v17: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:12:10.461 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:12:11.871 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:12:11.871 INFO:teuthology.orchestra.run.smithi018.stdout:{"epoch":64,"fsid":"a3b3b17e-5a4c-11ee-8db4-212e2dc638e7","created":"2023-09-23T20:06:54.359034+0000","modified":"2023-09-23T20:12:03.854680+0000","last_up_change":"2023-09-23T20:11:23.829606+0000","last_in_change":"2023-09-23T20:11:09.202130+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-09-23T20:09:47.162973+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-09-23T20:11:59.431837+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"64","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"44c8aaa3-bbd1-4330-8d34-73f7a08b0d4d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6802","nonce":2209550685},{"type":"v1","addr":"172.21.15.18:6803","nonce":2209550685}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6804","nonce":2209550685},{"type":"v1","addr":"172.21.15.18:6805","nonce":2209550685}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6808","nonce":2209550685},{"type":"v1","addr":"172.21.15.18:6809","nonce":2209550685}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6806","nonce":2209550685},{"type":"v1","addr":"172.21.15.18:6807","nonce":2209550685}]},"public_addr":"172.21.15.18:6803/2209550685","cluster_addr":"172.21.15.18:6805/2209550685","heartbeat_back_addr":"172.21.15.18:6809/2209550685","heartbeat_front_addr":"172.21.15.18:6807/2209550685","state":["exists","up"]},{"osd":1,"uuid":"c0534324-49c8-4804-bced-c4d13510e74a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6810","nonce":314354451},{"type":"v1","addr":"172.21.15.18:6811","nonce":314354451}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6812","nonce":314354451},{"type":"v1","addr":"172.21.15.18:6813","nonce":314354451}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6816","nonce":314354451},{"type":"v1","addr":"172.21.15.18:6817","nonce":314354451}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6814","nonce":314354451},{"type":"v1","addr":"172.21.15.18:6815","nonce":314354451}]},"public_addr":"172.21.15.18:6811/314354451","cluster_addr":"172.21.15.18:6813/314354451","heartbeat_back_addr":"172.21.15.18:6817/314354451","heartbeat_front_addr":"172.21.15.18:6815/314354451","state":["exists","up"]},{"osd":2,"uuid":"f3b2116d-a3d5-4279-91dc-eb47458d3239","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6818","nonce":1413681698},{"type":"v1","addr":"172.21.15.18:6819","nonce":1413681698}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6820","nonce":1413681698},{"type":"v1","addr":"172.21.15.18:6821","nonce":1413681698}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6824","nonce":1413681698},{"type":"v1","addr":"172.21.15.18:6825","nonce":1413681698}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6822","nonce":1413681698},{"type":"v1","addr":"172.21.15.18:6823","nonce":1413681698}]},"public_addr":"172.21.15.18:6819/1413681698","cluster_addr":"172.21.15.18:6821/1413681698","heartbeat_back_addr":"172.21.15.18:6825/1413681698","heartbeat_front_addr":"172.21.15.18:6823/1413681698","state":["exists","up"]},{"osd":3,"uuid":"8edccb7f-c4c9-47f3-a6f5-38a1a38cb099","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6826","nonce":423024182},{"type":"v1","addr":"172.21.15.18:6827","nonce":423024182}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6828","nonce":423024182},{"type":"v1","addr":"172.21.15.18:6829","nonce":423024182}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6832","nonce":423024182},{"type":"v1","addr":"172.21.15.18:6833","nonce":423024182}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6830","nonce":423024182},{"type":"v1","addr":"172.21.15.18:6831","nonce":423024182}]},"public_addr":"172.21.15.18:6827/423024182","cluster_addr":"172.21.15.18:6829/423024182","heartbeat_back_addr":"172.21.15.18:6833/423024182","heartbeat_front_addr":"172.21.15.18:6831/423024182","state":["exists","up"]},{"osd":4,"uuid":"1d5a64ce-fe42-4355-991a-dfb20cbe5423","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6800","nonce":3640432477},{"type":"v1","addr":"172.21.15.149:6801","nonce":3640432477}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6802","nonce":3640432477},{"type":"v1","addr":"172.21.15.149:6803","nonce":3640432477}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6806","nonce":3640432477},{"type":"v1","addr":"172.21.15.149:6807","nonce":3640432477}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6804","nonce":3640432477},{"type":"v1","addr":"172.21.15.149:6805","nonce":3640432477}]},"public_addr":"172.21.15.149:6801/3640432477","cluster_addr":"172.21.15.149:6803/3640432477","heartbeat_back_addr":"172.21.15.149:6807/3640432477","heartbeat_front_addr":"172.21.15.149:6805/3640432477","state":["exists","up"]},{"osd":5,"uuid":"105ae3b2-b2fc-477d-9134-3d517bc9427e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6808","nonce":2969709367},{"type":"v1","addr":"172.21.15.149:6809","nonce":2969709367}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6810","nonce":2969709367},{"type":"v1","addr":"172.21.15.149:6811","nonce":2969709367}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6814","nonce":2969709367},{"type":"v1","addr":"172.21.15.149:6815","nonce":2969709367}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6812","nonce":2969709367},{"type":"v1","addr":"172.21.15.149:6813","nonce":2969709367}]},"public_addr":"172.21.15.149:6809/2969709367","cluster_addr":"172.21.15.149:6811/2969709367","heartbeat_back_addr":"172.21.15.149:6815/2969709367","heartbeat_front_addr":"172.21.15.149:6813/2969709367","state":["exists","up"]},{"osd":6,"uuid":"27d2d2a8-e8c6-4a05-a85e-bd54376cf4c1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6816","nonce":1291453593},{"type":"v1","addr":"172.21.15.149:6817","nonce":1291453593}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6818","nonce":1291453593},{"type":"v1","addr":"172.21.15.149:6819","nonce":1291453593}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6822","nonce":1291453593},{"type":"v1","addr":"172.21.15.149:6823","nonce":1291453593}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6820","nonce":1291453593},{"type":"v1","addr":"172.21.15.149:6821","nonce":1291453593}]},"public_addr":"172.21.15.149:6817/1291453593","cluster_addr":"172.21.15.149:6819/1291453593","heartbeat_back_addr":"172.21.15.149:6823/1291453593","heartbeat_front_addr":"172.21.15.149:6821/1291453593","state":["exists","up"]},{"osd":7,"uuid":"9da636f7-995c-4f29-9264-70d09c6f66c6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6824","nonce":3117985178},{"type":"v1","addr":"172.21.15.149:6825","nonce":3117985178}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6826","nonce":3117985178},{"type":"v1","addr":"172.21.15.149:6827","nonce":3117985178}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6830","nonce":3117985178},{"type":"v1","addr":"172.21.15.149:6831","nonce":3117985178}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6828","nonce":3117985178},{"type":"v1","addr":"172.21.15.149:6829","nonce":3117985178}]},"public_addr":"172.21.15.149:6825/3117985178","cluster_addr":"172.21.15.149:6827/3117985178","heartbeat_back_addr":"172.21.15.149:6831/3117985178","heartbeat_front_addr":"172.21.15.149:6829/3117985178","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:09:01.034520+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:09:21.795238+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:09:42.987835+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:10:04.092458+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:10:23.309112+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:10:43.348231+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:11:03.128458+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:11:23.084228+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.18:0/3912151186":"2023-09-24T20:11:49.652762+0000","172.21.15.18:0/3863388488":"2023-09-24T20:11:49.652762+0000","172.21.15.18:6801/2611527229":"2023-09-24T20:11:49.652762+0000","172.21.15.18:6800/2611527229":"2023-09-24T20:11:49.652762+0000","172.21.15.18:0/1698512393":"2023-09-24T20:11:49.652762+0000","172.21.15.18:6801/380813540":"2023-09-24T20:07:48.635784+0000","172.21.15.18:6800/380813540":"2023-09-24T20:07:48.635784+0000","172.21.15.18:0/3831267686":"2023-09-24T20:07:48.635784+0000","172.21.15.18:0/950924496":"2023-09-24T20:07:23.919815+0000","172.21.15.18:0/1988472148":"2023-09-24T20:11:49.652762+0000","172.21.15.18:0/1082665289":"2023-09-24T20:07:48.635784+0000","172.21.15.18:0/2993111051":"2023-09-24T20:07:23.919815+0000","172.21.15.18:0/69122222":"2023-09-24T20:07:23.919815+0000","172.21.15.18:0/1645667747":"2023-09-24T20:07:48.635784+0000","172.21.15.18:6800/1930027051":"2023-09-24T20:07:23.919815+0000","172.21.15.18:6801/1930027051":"2023-09-24T20:07:23.919815+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-23T20:12:12.207 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph tell osd.0 flush_pg_stats 2023-09-23T20:12:12.207 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph tell osd.1 flush_pg_stats 2023-09-23T20:12:12.207 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph tell osd.2 flush_pg_stats 2023-09-23T20:12:12.208 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph tell osd.3 flush_pg_stats 2023-09-23T20:12:12.208 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph tell osd.4 flush_pg_stats 2023-09-23T20:12:12.208 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph tell osd.5 flush_pg_stats 2023-09-23T20:12:12.209 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph tell osd.6 flush_pg_stats 2023-09-23T20:12:12.209 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph tell osd.7 flush_pg_stats 2023-09-23T20:12:12.716 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:12:12 smithi149 bash[137508]: Copying config sha256:0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 2023-09-23T20:12:12.716 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:12:12 smithi149 bash[137508]: Writing manifest to image destination 2023-09-23T20:12:12.716 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:12:12 smithi149 bash[137508]: Storing signatures 2023-09-23T20:12:13.022 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:12 smithi149 ceph-mon[100764]: pgmap v18: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:12:13.023 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:12 smithi149 ceph-mon[100764]: from='client.? 172.21.15.18:0/3299574728' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T20:12:13.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:12 smithi018 ceph-mon[109137]: pgmap v18: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:12:13.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:12 smithi018 ceph-mon[109137]: from='client.? 172.21.15.18:0/3299574728' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T20:12:13.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:12 smithi018 ceph-mon[95889]: pgmap v18: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:12:13.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:12 smithi018 ceph-mon[95889]: from='client.? 172.21.15.18:0/3299574728' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T20:12:13.207 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:12:13.292 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:12:13 smithi149 podman[137508]: 2023-09-23T20:12:13.292 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:12:13 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-b[137558]: ts=2023-09-23T20:12:13.244Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2023-09-23T20:12:13.292 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:12:13 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-b[137558]: ts=2023-09-23T20:12:13.244Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2023-09-23T20:12:13.292 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:12:13 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-b[137558]: ts=2023-09-23T20:12:13.245Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2023-09-23T20:12:13.292 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:12:13 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-b[137558]: ts=2023-09-23T20:12:13.245Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2023-09-23T20:12:13.292 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:12:13 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-b[137558]: ts=2023-09-23T20:12:13.245Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2023-09-23T20:12:13.292 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:12:13 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-b[137558]: ts=2023-09-23T20:12:13.245Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2023-09-23T20:12:13.292 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:12:13 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-b[137558]: ts=2023-09-23T20:12:13.245Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2023-09-23T20:12:13.292 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:12:13 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-b[137558]: ts=2023-09-23T20:12:13.245Z caller=node_exporter.go:117 level=info collector=arp 2023-09-23T20:12:13.292 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:12:13 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-b[137558]: ts=2023-09-23T20:12:13.245Z caller=node_exporter.go:117 level=info collector=bcache 2023-09-23T20:12:13.293 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:12:13 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-b[137558]: ts=2023-09-23T20:12:13.245Z caller=node_exporter.go:117 level=info collector=bonding 2023-09-23T20:12:13.293 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:12:13 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-b[137558]: ts=2023-09-23T20:12:13.245Z caller=node_exporter.go:117 level=info collector=btrfs 2023-09-23T20:12:13.293 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:12:13 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-b[137558]: ts=2023-09-23T20:12:13.245Z caller=node_exporter.go:117 level=info collector=conntrack 2023-09-23T20:12:13.293 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:12:13 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-b[137558]: ts=2023-09-23T20:12:13.245Z caller=node_exporter.go:117 level=info collector=cpu 2023-09-23T20:12:13.293 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:12:13 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-b[137558]: ts=2023-09-23T20:12:13.245Z caller=node_exporter.go:117 level=info collector=cpufreq 2023-09-23T20:12:13.293 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:12:13 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-b[137558]: ts=2023-09-23T20:12:13.245Z caller=node_exporter.go:117 level=info collector=diskstats 2023-09-23T20:12:13.293 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:12:13 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-b[137558]: ts=2023-09-23T20:12:13.245Z caller=node_exporter.go:117 level=info collector=dmi 2023-09-23T20:12:13.293 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:12:13 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-b[137558]: ts=2023-09-23T20:12:13.245Z caller=node_exporter.go:117 level=info collector=edac 2023-09-23T20:12:13.293 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:12:13 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-b[137558]: ts=2023-09-23T20:12:13.245Z caller=node_exporter.go:117 level=info collector=entropy 2023-09-23T20:12:13.293 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:12:13 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-b[137558]: ts=2023-09-23T20:12:13.245Z caller=node_exporter.go:117 level=info collector=fibrechannel 2023-09-23T20:12:13.293 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:12:13 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-b[137558]: ts=2023-09-23T20:12:13.245Z caller=node_exporter.go:117 level=info collector=filefd 2023-09-23T20:12:13.293 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:12:13 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-b[137558]: ts=2023-09-23T20:12:13.245Z caller=node_exporter.go:117 level=info collector=filesystem 2023-09-23T20:12:13.293 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:12:13 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-b[137558]: ts=2023-09-23T20:12:13.245Z caller=node_exporter.go:117 level=info collector=hwmon 2023-09-23T20:12:13.294 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:12:13 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-b[137558]: ts=2023-09-23T20:12:13.245Z caller=node_exporter.go:117 level=info collector=infiniband 2023-09-23T20:12:13.294 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:12:13 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-b[137558]: ts=2023-09-23T20:12:13.245Z caller=node_exporter.go:117 level=info collector=ipvs 2023-09-23T20:12:13.294 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:12:13 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-b[137558]: ts=2023-09-23T20:12:13.245Z caller=node_exporter.go:117 level=info collector=loadavg 2023-09-23T20:12:13.294 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:12:13 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-b[137558]: ts=2023-09-23T20:12:13.245Z caller=node_exporter.go:117 level=info collector=mdadm 2023-09-23T20:12:13.294 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:12:13 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-b[137558]: ts=2023-09-23T20:12:13.245Z caller=node_exporter.go:117 level=info collector=meminfo 2023-09-23T20:12:13.294 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:12:13 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-b[137558]: ts=2023-09-23T20:12:13.245Z caller=node_exporter.go:117 level=info collector=netclass 2023-09-23T20:12:13.294 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:12:13 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-b[137558]: ts=2023-09-23T20:12:13.245Z caller=node_exporter.go:117 level=info collector=netdev 2023-09-23T20:12:13.294 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:12:13 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-b[137558]: ts=2023-09-23T20:12:13.245Z caller=node_exporter.go:117 level=info collector=netstat 2023-09-23T20:12:13.294 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:12:13 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-b[137558]: ts=2023-09-23T20:12:13.245Z caller=node_exporter.go:117 level=info collector=nfs 2023-09-23T20:12:13.294 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:12:13 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-b[137558]: ts=2023-09-23T20:12:13.245Z caller=node_exporter.go:117 level=info collector=nfsd 2023-09-23T20:12:13.294 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:12:13 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-b[137558]: ts=2023-09-23T20:12:13.245Z caller=node_exporter.go:117 level=info collector=nvme 2023-09-23T20:12:13.295 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:12:13 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-b[137558]: ts=2023-09-23T20:12:13.245Z caller=node_exporter.go:117 level=info collector=os 2023-09-23T20:12:13.295 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:12:13 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-b[137558]: ts=2023-09-23T20:12:13.245Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2023-09-23T20:12:13.295 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:12:13 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-b[137558]: ts=2023-09-23T20:12:13.245Z caller=node_exporter.go:117 level=info collector=pressure 2023-09-23T20:12:13.295 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:12:13 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-b[137558]: ts=2023-09-23T20:12:13.245Z caller=node_exporter.go:117 level=info collector=rapl 2023-09-23T20:12:13.295 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:12:13 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-b[137558]: ts=2023-09-23T20:12:13.245Z caller=node_exporter.go:117 level=info collector=schedstat 2023-09-23T20:12:13.295 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:12:13 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-b[137558]: ts=2023-09-23T20:12:13.245Z caller=node_exporter.go:117 level=info collector=selinux 2023-09-23T20:12:13.295 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:12:13 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-b[137558]: ts=2023-09-23T20:12:13.245Z caller=node_exporter.go:117 level=info collector=sockstat 2023-09-23T20:12:13.295 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:12:13 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-b[137558]: ts=2023-09-23T20:12:13.245Z caller=node_exporter.go:117 level=info collector=softnet 2023-09-23T20:12:13.295 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:12:13 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-b[137558]: ts=2023-09-23T20:12:13.245Z caller=node_exporter.go:117 level=info collector=stat 2023-09-23T20:12:13.295 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:12:13 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-b[137558]: ts=2023-09-23T20:12:13.245Z caller=node_exporter.go:117 level=info collector=tapestats 2023-09-23T20:12:13.295 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:12:13 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-b[137558]: ts=2023-09-23T20:12:13.245Z caller=node_exporter.go:117 level=info collector=textfile 2023-09-23T20:12:13.296 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:12:13 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-b[137558]: ts=2023-09-23T20:12:13.245Z caller=node_exporter.go:117 level=info collector=thermal_zone 2023-09-23T20:12:13.296 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:12:13 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-b[137558]: ts=2023-09-23T20:12:13.245Z caller=node_exporter.go:117 level=info collector=time 2023-09-23T20:12:13.296 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:12:13 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-b[137558]: ts=2023-09-23T20:12:13.245Z caller=node_exporter.go:117 level=info collector=udp_queues 2023-09-23T20:12:13.296 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:12:13 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-b[137558]: ts=2023-09-23T20:12:13.245Z caller=node_exporter.go:117 level=info collector=uname 2023-09-23T20:12:13.296 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:12:13 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-b[137558]: ts=2023-09-23T20:12:13.245Z caller=node_exporter.go:117 level=info collector=vmstat 2023-09-23T20:12:13.296 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:12:13 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-b[137558]: ts=2023-09-23T20:12:13.245Z caller=node_exporter.go:117 level=info collector=xfs 2023-09-23T20:12:13.296 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:12:13 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-b[137558]: ts=2023-09-23T20:12:13.245Z caller=node_exporter.go:117 level=info collector=zfs 2023-09-23T20:12:13.296 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:12:13 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-b[137558]: ts=2023-09-23T20:12:13.246Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2023-09-23T20:12:13.296 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:12:13 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-b[137558]: ts=2023-09-23T20:12:13.246Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2023-09-23T20:12:13.296 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:12:13 smithi149 bash[137508]: f0943dfd81ce1e01d9161b687d3caa18da05f98f74eda80aa03955567e9218c6 2023-09-23T20:12:13.361 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:12:13.565 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:12:13.565 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:12:13.620 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:12:13.700 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:12:13 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: [23/Sep/2023:20:12:13] ENGINE Bus STOPPING 2023-09-23T20:12:13.700 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:12:13 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: [23/Sep/2023:20:12:13] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T20:12:13.700 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:12:13 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: [23/Sep/2023:20:12:13] ENGINE Bus STOPPED 2023-09-23T20:12:13.700 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:12:13 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: [23/Sep/2023:20:12:13] ENGINE Bus STARTING 2023-09-23T20:12:13.701 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:12:13 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: [23/Sep/2023:20:12:13] ENGINE Serving on http://:::9283 2023-09-23T20:12:13.701 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:12:13 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: [23/Sep/2023:20:12:13] ENGINE Bus STARTED 2023-09-23T20:12:13.716 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:12:13 smithi149 systemd[1]: Started Ceph node-exporter.b for a3b3b17e-5a4c-11ee-8db4-212e2dc638e7. 2023-09-23T20:12:13.982 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:12:14.006 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 23 20:12:13 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-alertmanager-a[145423]: ts=2023-09-23T20:12:13.765Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.003691257s 2023-09-23T20:12:14.216 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:12:13 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[102880]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:12:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T20:12:14.609 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:14 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:14.609 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:14 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:14.609 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:14 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:14.609 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:14 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:14.609 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:14 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:14.610 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:14 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:14.610 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:14 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:14.610 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:14 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-09-23T20:12:14.610 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:14 smithi018 ceph-mon[95889]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-09-23T20:12:14.610 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:14 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:14.610 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:14 smithi018 ceph-mon[95889]: Deploying daemon grafana.a on smithi149 2023-09-23T20:12:14.610 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:14 smithi018 ceph-mon[95889]: pgmap v19: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:12:14.611 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:14 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:14.611 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:14 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:14.611 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:14 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:14.611 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:14 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:14.611 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:14 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:14.611 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:14 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:14.611 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:14 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:14.611 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:14 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-09-23T20:12:14.611 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:14 smithi018 ceph-mon[109137]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-09-23T20:12:14.611 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:14 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:14.612 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:14 smithi018 ceph-mon[109137]: Deploying daemon grafana.a on smithi149 2023-09-23T20:12:14.612 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:14 smithi018 ceph-mon[109137]: pgmap v19: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:12:14.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:14 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:14.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:14 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:14.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:14 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:14.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:14 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:14.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:14 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:14.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:14 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:14.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:14 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:14.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:14 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-09-23T20:12:14.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:14 smithi149 ceph-mon[100764]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-09-23T20:12:14.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:14 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:14.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:14 smithi149 ceph-mon[100764]: Deploying daemon grafana.a on smithi149 2023-09-23T20:12:14.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:14 smithi149 ceph-mon[100764]: pgmap v19: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:12:14.737 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:12:15.346 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:12:16.103 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:15 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:16.103 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:15 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:16.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:15 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:17.101 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:16 smithi018 ceph-mon[109137]: pgmap v20: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:12:17.101 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:16 smithi018 ceph-mon[95889]: pgmap v20: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:12:17.215 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:16 smithi149 ceph-mon[100764]: pgmap v20: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:12:18.026 INFO:teuthology.orchestra.run.smithi018.stdout:55834574885 2023-09-23T20:12:18.026 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.1 2023-09-23T20:12:18.926 INFO:teuthology.orchestra.run.smithi018.stdout:34359738408 2023-09-23T20:12:18.926 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.0 2023-09-23T20:12:19.107 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:18 smithi018 ceph-mon[109137]: pgmap v21: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:12:19.108 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:18 smithi018 ceph-mon[95889]: pgmap v21: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:12:19.215 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:18 smithi149 ceph-mon[100764]: pgmap v21: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:12:19.754 INFO:teuthology.orchestra.run.smithi018.stdout:236223201292 2023-09-23T20:12:19.755 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.7 2023-09-23T20:12:19.934 INFO:teuthology.orchestra.run.smithi018.stdout:107374182427 2023-09-23T20:12:19.934 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.3 2023-09-23T20:12:20.140 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:19 smithi018 ceph-mon[109137]: pgmap v22: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:12:20.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:19 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:12:20.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:19 smithi018 ceph-mon[95889]: pgmap v22: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:12:20.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:19 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:12:20.215 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:19 smithi149 ceph-mon[100764]: pgmap v22: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:12:20.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:19 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:12:20.651 INFO:teuthology.orchestra.run.smithi018.stdout:77309411361 2023-09-23T20:12:20.652 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.2 2023-09-23T20:12:20.886 INFO:teuthology.orchestra.run.smithi018.stdout:128849018904 2023-09-23T20:12:20.887 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.4 2023-09-23T20:12:21.039 INFO:teuthology.orchestra.run.smithi018.stdout:167503724565 2023-09-23T20:12:21.039 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.5 2023-09-23T20:12:21.279 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:12:21.338 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:12:21.633 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:12:21.977 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:12:22.120 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:12:22.170 INFO:teuthology.orchestra.run.smithi018.stdout:201863462929 2023-09-23T20:12:22.170 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.6 2023-09-23T20:12:22.580 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:12:22.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:22 smithi018 ceph-mon[109137]: pgmap v23: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:12:22.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:22 smithi018 ceph-mon[95889]: pgmap v23: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:12:23.215 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:22 smithi149 ceph-mon[100764]: pgmap v23: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:12:24.102 INFO:teuthology.orchestra.run.smithi018.stdout:236223201293 2023-09-23T20:12:24.216 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:12:23 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[102880]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:12:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T20:12:24.806 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:12:24.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:24 smithi149 ceph-mon[100764]: pgmap v24: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:12:24.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:24 smithi149 ceph-mon[100764]: from='client.? 172.21.15.18:0/2004628861' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T20:12:24.990 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:24 smithi018 ceph-mon[109137]: pgmap v24: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:12:24.990 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:24 smithi018 ceph-mon[109137]: from='client.? 172.21.15.18:0/2004628861' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T20:12:24.991 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:24 smithi018 ceph-mon[95889]: pgmap v24: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:12:24.991 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:24 smithi018 ceph-mon[95889]: from='client.? 172.21.15.18:0/2004628861' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T20:12:25.376 INFO:tasks.cephadm.ceph_manager.ceph:need seq 236223201292 got 236223201293 for osd.7 2023-09-23T20:12:25.377 DEBUG:teuthology.parallel:result is None 2023-09-23T20:12:25.745 INFO:teuthology.orchestra.run.smithi018.stdout:34359738409 2023-09-23T20:12:25.925 INFO:teuthology.orchestra.run.smithi018.stdout:107374182429 2023-09-23T20:12:26.001 INFO:teuthology.orchestra.run.smithi018.stdout:55834574886 2023-09-23T20:12:26.216 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 systemd[1]: Starting Ceph grafana.a for a3b3b17e-5a4c-11ee-8db4-212e2dc638e7... 2023-09-23T20:12:26.663 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 podman[137947]: 2023-09-23T20:12:26.777 INFO:tasks.cephadm.ceph_manager.ceph:need seq 107374182427 got 107374182429 for osd.3 2023-09-23T20:12:26.777 DEBUG:teuthology.parallel:result is None 2023-09-23T20:12:26.913 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-mon[100764]: pgmap v25: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:12:26.914 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-mon[100764]: from='client.? 172.21.15.18:0/601965911' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T20:12:26.914 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-mon[100764]: from='client.? 172.21.15.18:0/2596734430' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T20:12:26.914 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-mon[100764]: from='client.? 172.21.15.18:0/1700894474' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T20:12:26.916 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 bash[137947]: 6aef9de808da0dfb5e0aff8aa14561955799fa7634fefb9b7333b72465d58046 2023-09-23T20:12:26.916 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 systemd[1]: Started Ceph grafana.a for a3b3b17e-5a4c-11ee-8db4-212e2dc638e7. 2023-09-23T20:12:26.917 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=settings t=2023-09-23T20:12:26.808808587Z level=info msg="Starting Grafana" version=9.4.7 commit=4add91f03d branch=HEAD compiled=2023-03-16T23:56:52Z 2023-09-23T20:12:26.917 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=settings t=2023-09-23T20:12:26.808984894Z level=warn msg="\"sentry\" frontend logging provider is deprecated and will be removed in the next major version. Use \"grafana\" provider instead." 2023-09-23T20:12:26.917 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=settings t=2023-09-23T20:12:26.809003025Z level=info msg="Config loaded from" file=/usr/share/grafana/conf/defaults.ini 2023-09-23T20:12:26.917 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=settings t=2023-09-23T20:12:26.809009551Z level=info msg="Config loaded from" file=/etc/grafana/grafana.ini 2023-09-23T20:12:26.917 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=settings t=2023-09-23T20:12:26.809015605Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_DATA=/var/lib/grafana" 2023-09-23T20:12:26.917 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=settings t=2023-09-23T20:12:26.809021424Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_LOGS=/var/log/grafana" 2023-09-23T20:12:26.918 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=settings t=2023-09-23T20:12:26.809027121Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2023-09-23T20:12:26.918 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=settings t=2023-09-23T20:12:26.809032876Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2023-09-23T20:12:26.918 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=settings t=2023-09-23T20:12:26.809038679Z level=info msg="Path Home" path=/usr/share/grafana 2023-09-23T20:12:26.918 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=settings t=2023-09-23T20:12:26.809045122Z level=info msg="Path Data" path=/var/lib/grafana 2023-09-23T20:12:26.918 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=settings t=2023-09-23T20:12:26.809050596Z level=info msg="Path Logs" path=/var/log/grafana 2023-09-23T20:12:26.918 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=settings t=2023-09-23T20:12:26.809056073Z level=info msg="Path Plugins" path=/var/lib/grafana/plugins 2023-09-23T20:12:26.919 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=settings t=2023-09-23T20:12:26.809061783Z level=info msg="Path Provisioning" path=/etc/grafana/provisioning 2023-09-23T20:12:26.919 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=settings t=2023-09-23T20:12:26.80906746Z level=info msg="App mode production" 2023-09-23T20:12:26.919 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=sqlstore t=2023-09-23T20:12:26.809131853Z level=info msg="Connecting to DB" dbtype=sqlite3 2023-09-23T20:12:26.919 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=sqlstore t=2023-09-23T20:12:26.80914984Z level=warn msg="SQLite database file has broader permissions than it should" path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2023-09-23T20:12:26.919 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.809973258Z level=info msg="Starting DB migrations" 2023-09-23T20:12:26.920 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.810048867Z level=info msg="Executing migration" id="create migration_log table" 2023-09-23T20:12:26.920 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.810622099Z level=info msg="Executing migration" id="create user table" 2023-09-23T20:12:26.920 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.811034663Z level=info msg="Executing migration" id="add unique index user.login" 2023-09-23T20:12:26.920 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.81144926Z level=info msg="Executing migration" id="add unique index user.email" 2023-09-23T20:12:26.920 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.811882037Z level=info msg="Executing migration" id="drop index UQE_user_login - v1" 2023-09-23T20:12:26.920 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.812275632Z level=info msg="Executing migration" id="drop index UQE_user_email - v1" 2023-09-23T20:12:26.920 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.812677217Z level=info msg="Executing migration" id="Rename table user to user_v1 - v1" 2023-09-23T20:12:26.920 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.813367294Z level=info msg="Executing migration" id="create user table v2" 2023-09-23T20:12:26.921 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.813762922Z level=info msg="Executing migration" id="create index UQE_user_login - v2" 2023-09-23T20:12:26.921 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.81412018Z level=info msg="Executing migration" id="create index UQE_user_email - v2" 2023-09-23T20:12:26.921 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.814529191Z level=info msg="Executing migration" id="copy data_source v1 to v2" 2023-09-23T20:12:26.921 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.814861405Z level=info msg="Executing migration" id="Drop old table user_v1" 2023-09-23T20:12:26.921 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.815180606Z level=info msg="Executing migration" id="Add column help_flags1 to user table" 2023-09-23T20:12:26.921 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.815600612Z level=info msg="Executing migration" id="Update user table charset" 2023-09-23T20:12:26.921 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.815892062Z level=info msg="Executing migration" id="Add last_seen_at column to user" 2023-09-23T20:12:26.921 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.816270519Z level=info msg="Executing migration" id="Add missing user data" 2023-09-23T20:12:26.922 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.816622723Z level=info msg="Executing migration" id="Add is_disabled column to user" 2023-09-23T20:12:26.922 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.817005243Z level=info msg="Executing migration" id="Add index user.login/user.email" 2023-09-23T20:12:26.922 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.817388859Z level=info msg="Executing migration" id="Add is_service_account column to user" 2023-09-23T20:12:26.922 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.817801529Z level=info msg="Executing migration" id="Update is_service_account column to nullable" 2023-09-23T20:12:26.922 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.818962186Z level=info msg="Executing migration" id="create temp user table v1-7" 2023-09-23T20:12:26.922 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.819392921Z level=info msg="Executing migration" id="create index IDX_temp_user_email - v1-7" 2023-09-23T20:12:26.922 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.819815012Z level=info msg="Executing migration" id="create index IDX_temp_user_org_id - v1-7" 2023-09-23T20:12:26.923 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.820194647Z level=info msg="Executing migration" id="create index IDX_temp_user_code - v1-7" 2023-09-23T20:12:26.923 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.820588929Z level=info msg="Executing migration" id="create index IDX_temp_user_status - v1-7" 2023-09-23T20:12:26.923 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.820974015Z level=info msg="Executing migration" id="Update temp_user table charset" 2023-09-23T20:12:26.923 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.821234822Z level=info msg="Executing migration" id="drop index IDX_temp_user_email - v1" 2023-09-23T20:12:26.923 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.821615129Z level=info msg="Executing migration" id="drop index IDX_temp_user_org_id - v1" 2023-09-23T20:12:26.923 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.821996143Z level=info msg="Executing migration" id="drop index IDX_temp_user_code - v1" 2023-09-23T20:12:26.923 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.822349022Z level=info msg="Executing migration" id="drop index IDX_temp_user_status - v1" 2023-09-23T20:12:26.923 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.822725504Z level=info msg="Executing migration" id="Rename table temp_user to temp_user_tmp_qwerty - v1" 2023-09-23T20:12:26.924 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.823377694Z level=info msg="Executing migration" id="create temp_user v2" 2023-09-23T20:12:26.924 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.823791505Z level=info msg="Executing migration" id="create index IDX_temp_user_email - v2" 2023-09-23T20:12:26.924 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.824166057Z level=info msg="Executing migration" id="create index IDX_temp_user_org_id - v2" 2023-09-23T20:12:26.924 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.824574422Z level=info msg="Executing migration" id="create index IDX_temp_user_code - v2" 2023-09-23T20:12:26.924 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.824971314Z level=info msg="Executing migration" id="create index IDX_temp_user_status - v2" 2023-09-23T20:12:26.924 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.825379098Z level=info msg="Executing migration" id="copy temp_user v1 to v2" 2023-09-23T20:12:26.925 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.825717199Z level=info msg="Executing migration" id="drop temp_user_tmp_qwerty" 2023-09-23T20:12:26.925 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.826038432Z level=info msg="Executing migration" id="Set created for temp users that will otherwise prematurely expire" 2023-09-23T20:12:26.925 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.826401335Z level=info msg="Executing migration" id="create star table" 2023-09-23T20:12:26.925 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.82678494Z level=info msg="Executing migration" id="add unique index star.user_id_dashboard_id" 2023-09-23T20:12:26.925 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.827185644Z level=info msg="Executing migration" id="create org table v1" 2023-09-23T20:12:26.925 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.827583714Z level=info msg="Executing migration" id="create index UQE_org_name - v1" 2023-09-23T20:12:26.925 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.827974907Z level=info msg="Executing migration" id="create org_user table v1" 2023-09-23T20:12:26.925 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.828350222Z level=info msg="Executing migration" id="create index IDX_org_user_org_id - v1" 2023-09-23T20:12:26.926 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.828749314Z level=info msg="Executing migration" id="create index UQE_org_user_org_id_user_id - v1" 2023-09-23T20:12:26.926 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.829160803Z level=info msg="Executing migration" id="create index IDX_org_user_user_id - v1" 2023-09-23T20:12:26.926 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.829550557Z level=info msg="Executing migration" id="Update org table charset" 2023-09-23T20:12:26.926 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.829836213Z level=info msg="Executing migration" id="Update org_user table charset" 2023-09-23T20:12:26.926 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.83009669Z level=info msg="Executing migration" id="Migrate all Read Only Viewers to Viewers" 2023-09-23T20:12:26.926 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.830385163Z level=info msg="Executing migration" id="create dashboard table" 2023-09-23T20:12:26.926 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.830753243Z level=info msg="Executing migration" id="add index dashboard.account_id" 2023-09-23T20:12:26.926 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.831139227Z level=info msg="Executing migration" id="add unique index dashboard_account_id_slug" 2023-09-23T20:12:26.927 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.831557839Z level=info msg="Executing migration" id="create dashboard_tag table" 2023-09-23T20:12:26.927 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.831931276Z level=info msg="Executing migration" id="add unique index dashboard_tag.dasboard_id_term" 2023-09-23T20:12:26.927 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.832311646Z level=info msg="Executing migration" id="drop index UQE_dashboard_tag_dashboard_id_term - v1" 2023-09-23T20:12:26.927 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.832705282Z level=info msg="Executing migration" id="Rename table dashboard to dashboard_v1 - v1" 2023-09-23T20:12:26.927 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.833544234Z level=info msg="Executing migration" id="create dashboard v2" 2023-09-23T20:12:26.927 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.835227519Z level=info msg="Executing migration" id="create index IDX_dashboard_org_id - v2" 2023-09-23T20:12:26.927 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.837906011Z level=info msg="Executing migration" id="create index UQE_dashboard_org_id_slug - v2" 2023-09-23T20:12:26.927 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.838365542Z level=info msg="Executing migration" id="copy dashboard v1 to v2" 2023-09-23T20:12:26.928 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.83870195Z level=info msg="Executing migration" id="drop table dashboard_v1" 2023-09-23T20:12:26.928 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.839057888Z level=info msg="Executing migration" id="alter dashboard.data to mediumtext v1" 2023-09-23T20:12:26.928 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.839315472Z level=info msg="Executing migration" id="Add column updated_by in dashboard - v2" 2023-09-23T20:12:26.928 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.839811103Z level=info msg="Executing migration" id="Add column created_by in dashboard - v2" 2023-09-23T20:12:26.928 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.840246406Z level=info msg="Executing migration" id="Add column gnetId in dashboard" 2023-09-23T20:12:26.928 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.840718689Z level=info msg="Executing migration" id="Add index for gnetId in dashboard" 2023-09-23T20:12:26.928 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.841120468Z level=info msg="Executing migration" id="Add column plugin_id in dashboard" 2023-09-23T20:12:26.928 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.84162207Z level=info msg="Executing migration" id="Add index for plugin_id in dashboard" 2023-09-23T20:12:26.929 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.842042691Z level=info msg="Executing migration" id="Add index for dashboard_id in dashboard_tag" 2023-09-23T20:12:26.929 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.8424504Z level=info msg="Executing migration" id="Update dashboard table charset" 2023-09-23T20:12:26.929 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.84274171Z level=info msg="Executing migration" id="Update dashboard_tag table charset" 2023-09-23T20:12:26.929 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.843013568Z level=info msg="Executing migration" id="Add column folder_id in dashboard" 2023-09-23T20:12:26.929 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.843527553Z level=info msg="Executing migration" id="Add column isFolder in dashboard" 2023-09-23T20:12:26.929 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.844024816Z level=info msg="Executing migration" id="Add column has_acl in dashboard" 2023-09-23T20:12:26.929 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.84693097Z level=info msg="Executing migration" id="Add column uid in dashboard" 2023-09-23T20:12:26.929 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.84974887Z level=info msg="Executing migration" id="Update uid column values in dashboard" 2023-09-23T20:12:26.930 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.852527648Z level=info msg="Executing migration" id="Add unique index dashboard_org_id_uid" 2023-09-23T20:12:26.930 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.855557469Z level=info msg="Executing migration" id="Remove unique index org_id_slug" 2023-09-23T20:12:26.930 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.85825444Z level=info msg="Executing migration" id="Update dashboard title length" 2023-09-23T20:12:26.930 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.861111068Z level=info msg="Executing migration" id="Add unique index for dashboard_org_id_title_folder_id" 2023-09-23T20:12:26.930 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.863831214Z level=info msg="Executing migration" id="create dashboard_provisioning" 2023-09-23T20:12:26.930 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.864246109Z level=info msg="Executing migration" id="Rename table dashboard_provisioning to dashboard_provisioning_tmp_qwerty - v1" 2023-09-23T20:12:26.930 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.867212675Z level=info msg="Executing migration" id="create dashboard_provisioning v2" 2023-09-23T20:12:26.931 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.870268716Z level=info msg="Executing migration" id="create index IDX_dashboard_provisioning_dashboard_id - v2" 2023-09-23T20:12:26.931 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.872786559Z level=info msg="Executing migration" id="create index IDX_dashboard_provisioning_dashboard_id_name - v2" 2023-09-23T20:12:26.931 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.873252575Z level=info msg="Executing migration" id="copy dashboard_provisioning v1 to v2" 2023-09-23T20:12:26.931 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.876053368Z level=info msg="Executing migration" id="drop dashboard_provisioning_tmp_qwerty" 2023-09-23T20:12:26.931 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.87850618Z level=info msg="Executing migration" id="Add check_sum column" 2023-09-23T20:12:26.931 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.881507452Z level=info msg="Executing migration" id="Add index for dashboard_title" 2023-09-23T20:12:26.931 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.881959055Z level=info msg="Executing migration" id="delete tags for deleted dashboards" 2023-09-23T20:12:26.931 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.884555221Z level=info msg="Executing migration" id="delete stars for deleted dashboards" 2023-09-23T20:12:26.932 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.884970798Z level=info msg="Executing migration" id="Add index for dashboard_is_folder" 2023-09-23T20:12:26.932 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.887337418Z level=info msg="Executing migration" id="Add isPublic for dashboard" 2023-09-23T20:12:26.932 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.889929754Z level=info msg="Executing migration" id="create data_source table" 2023-09-23T20:12:26.932 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.890548787Z level=info msg="Executing migration" id="add index data_source.account_id" 2023-09-23T20:12:26.932 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.893179641Z level=info msg="Executing migration" id="add unique index data_source.account_id_name" 2023-09-23T20:12:26.932 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.893635495Z level=info msg="Executing migration" id="drop index IDX_data_source_account_id - v1" 2023-09-23T20:12:26.932 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.896430352Z level=info msg="Executing migration" id="drop index UQE_data_source_account_id_name - v1" 2023-09-23T20:12:26.933 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.896833206Z level=info msg="Executing migration" id="Rename table data_source to data_source_v1 - v1" 2023-09-23T20:12:26.933 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.899600067Z level=info msg="Executing migration" id="create data_source table v2" 2023-09-23T20:12:26.933 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.902221637Z level=info msg="Executing migration" id="create index IDX_data_source_org_id - v2" 2023-09-23T20:12:26.933 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.904787827Z level=info msg="Executing migration" id="create index UQE_data_source_org_id_name - v2" 2023-09-23T20:12:26.933 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.905218492Z level=info msg="Executing migration" id="Drop old table data_source_v1 #2" 2023-09-23T20:12:26.933 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.906132727Z level=info msg="Executing migration" id="Add column with_credentials" 2023-09-23T20:12:26.933 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.906905792Z level=info msg="Executing migration" id="Add secure json data column" 2023-09-23T20:12:26.934 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.907917671Z level=info msg="Executing migration" id="Update data_source table charset" 2023-09-23T20:12:26.934 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.908283183Z level=info msg="Executing migration" id="Update initial version to 1" 2023-09-23T20:12:26.934 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.909086541Z level=info msg="Executing migration" id="Add read_only data column" 2023-09-23T20:12:26.934 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.909977188Z level=info msg="Executing migration" id="Migrate logging ds to loki ds" 2023-09-23T20:12:26.934 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.910813108Z level=info msg="Executing migration" id="Update json_data with nulls" 2023-09-23T20:12:26.934 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.911150593Z level=info msg="Executing migration" id="Add uid column" 2023-09-23T20:12:26.934 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.911972992Z level=info msg="Executing migration" id="Update uid value" 2023-09-23T20:12:26.934 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.91278712Z level=info msg="Executing migration" id="Add unique index datasource_org_id_uid" 2023-09-23T20:12:26.935 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.913629073Z level=info msg="Executing migration" id="add unique index datasource_org_id_is_default" 2023-09-23T20:12:26.935 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.914455498Z level=info msg="Executing migration" id="create api_key table" 2023-09-23T20:12:26.935 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.915524156Z level=info msg="Executing migration" id="add index api_key.account_id" 2023-09-23T20:12:26.935 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.916265423Z level=info msg="Executing migration" id="add index api_key.key" 2023-09-23T20:12:27.016 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:26 smithi018 ceph-mon[109137]: pgmap v25: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:12:27.017 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:26 smithi018 ceph-mon[109137]: from='client.? 172.21.15.18:0/601965911' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T20:12:27.017 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:26 smithi018 ceph-mon[109137]: from='client.? 172.21.15.18:0/2596734430' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T20:12:27.017 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:26 smithi018 ceph-mon[109137]: from='client.? 172.21.15.18:0/1700894474' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T20:12:27.017 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:26 smithi018 ceph-mon[95889]: pgmap v25: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:12:27.017 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:26 smithi018 ceph-mon[95889]: from='client.? 172.21.15.18:0/601965911' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T20:12:27.017 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:26 smithi018 ceph-mon[95889]: from='client.? 172.21.15.18:0/2596734430' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T20:12:27.018 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:26 smithi018 ceph-mon[95889]: from='client.? 172.21.15.18:0/1700894474' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T20:12:27.166 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.916985381Z level=info msg="Executing migration" id="add index api_key.account_id_name" 2023-09-23T20:12:27.166 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.917472663Z level=info msg="Executing migration" id="drop index IDX_api_key_account_id - v1" 2023-09-23T20:12:27.166 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.918705539Z level=info msg="Executing migration" id="drop index UQE_api_key_key - v1" 2023-09-23T20:12:27.166 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.91918296Z level=info msg="Executing migration" id="drop index UQE_api_key_account_id_name - v1" 2023-09-23T20:12:27.166 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.920083941Z level=info msg="Executing migration" id="Rename table api_key to api_key_v1 - v1" 2023-09-23T20:12:27.166 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.922489215Z level=info msg="Executing migration" id="create api_key table v2" 2023-09-23T20:12:27.166 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.925101928Z level=info msg="Executing migration" id="create index IDX_api_key_org_id - v2" 2023-09-23T20:12:27.166 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.927751419Z level=info msg="Executing migration" id="create index UQE_api_key_key - v2" 2023-09-23T20:12:27.167 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.930437899Z level=info msg="Executing migration" id="create index UQE_api_key_org_id_name - v2" 2023-09-23T20:12:27.167 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.93315556Z level=info msg="Executing migration" id="copy api_key v1 to v2" 2023-09-23T20:12:27.167 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.93397885Z level=info msg="Executing migration" id="Drop old table api_key_v1" 2023-09-23T20:12:27.167 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.934803807Z level=info msg="Executing migration" id="Update api_key table charset" 2023-09-23T20:12:27.167 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.935560765Z level=info msg="Executing migration" id="Add expires to api_key table" 2023-09-23T20:12:27.167 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.936417417Z level=info msg="Executing migration" id="Add service account foreign key" 2023-09-23T20:12:27.167 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.937182611Z level=info msg="Executing migration" id="set service account foreign key to nil if 0" 2023-09-23T20:12:27.167 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.937992181Z level=info msg="Executing migration" id="Add last_used_at to api_key table" 2023-09-23T20:12:27.167 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.938749822Z level=info msg="Executing migration" id="Add is_revoked column to api_key table" 2023-09-23T20:12:27.167 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.939532437Z level=info msg="Executing migration" id="create dashboard_snapshot table v4" 2023-09-23T20:12:27.167 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.940354693Z level=info msg="Executing migration" id="drop table dashboard_snapshot_v4 #1" 2023-09-23T20:12:27.167 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.941134692Z level=info msg="Executing migration" id="create dashboard_snapshot table v5 #2" 2023-09-23T20:12:27.168 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.94196044Z level=info msg="Executing migration" id="create index UQE_dashboard_snapshot_key - v5" 2023-09-23T20:12:27.168 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.942737467Z level=info msg="Executing migration" id="create index UQE_dashboard_snapshot_delete_key - v5" 2023-09-23T20:12:27.168 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.943582189Z level=info msg="Executing migration" id="create index IDX_dashboard_snapshot_user_id - v5" 2023-09-23T20:12:27.168 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.944348016Z level=info msg="Executing migration" id="alter dashboard_snapshot to mediumtext v2" 2023-09-23T20:12:27.168 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.945165676Z level=info msg="Executing migration" id="Update dashboard_snapshot table charset" 2023-09-23T20:12:27.168 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.945973386Z level=info msg="Executing migration" id="Add column external_delete_url to dashboard_snapshots table" 2023-09-23T20:12:27.168 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.946762244Z level=info msg="Executing migration" id="Add encrypted dashboard json column" 2023-09-23T20:12:27.168 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.947548671Z level=info msg="Executing migration" id="Change dashboard_encrypted column to MEDIUMBLOB" 2023-09-23T20:12:27.168 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.948429205Z level=info msg="Executing migration" id="create quota table v1" 2023-09-23T20:12:27.168 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.94926176Z level=info msg="Executing migration" id="create index UQE_quota_org_id_user_id_target - v1" 2023-09-23T20:12:27.168 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.950036522Z level=info msg="Executing migration" id="Update quota table charset" 2023-09-23T20:12:27.168 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.950850134Z level=info msg="Executing migration" id="create plugin_setting table" 2023-09-23T20:12:27.169 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.951348298Z level=info msg="Executing migration" id="create index UQE_plugin_setting_org_id_plugin_id - v1" 2023-09-23T20:12:27.169 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.951837331Z level=info msg="Executing migration" id="Add column plugin_version to plugin_settings" 2023-09-23T20:12:27.169 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.952655586Z level=info msg="Executing migration" id="Update plugin_setting table charset" 2023-09-23T20:12:27.169 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.952987844Z level=info msg="Executing migration" id="create session table" 2023-09-23T20:12:27.169 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.953502248Z level=info msg="Executing migration" id="Drop old table playlist table" 2023-09-23T20:12:27.169 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.954324919Z level=info msg="Executing migration" id="Drop old table playlist_item table" 2023-09-23T20:12:27.169 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.955164006Z level=info msg="Executing migration" id="create playlist table v2" 2023-09-23T20:12:27.169 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.956220345Z level=info msg="Executing migration" id="create playlist item table v2" 2023-09-23T20:12:27.169 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.956822866Z level=info msg="Executing migration" id="Update playlist table charset" 2023-09-23T20:12:27.169 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.957628481Z level=info msg="Executing migration" id="Update playlist_item table charset" 2023-09-23T20:12:27.169 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.958583159Z level=info msg="Executing migration" id="drop preferences table v2" 2023-09-23T20:12:27.170 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.959274728Z level=info msg="Executing migration" id="drop preferences table v3" 2023-09-23T20:12:27.170 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.960191192Z level=info msg="Executing migration" id="create preferences table v3" 2023-09-23T20:12:27.170 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.962975662Z level=info msg="Executing migration" id="Update preferences table charset" 2023-09-23T20:12:27.170 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.965678881Z level=info msg="Executing migration" id="Add column team_id in preferences" 2023-09-23T20:12:27.170 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.968335419Z level=info msg="Executing migration" id="Update team_id column values in preferences" 2023-09-23T20:12:27.170 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.971069234Z level=info msg="Executing migration" id="Add column week_start in preferences" 2023-09-23T20:12:27.170 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.973571139Z level=info msg="Executing migration" id="Add column preferences.json_data" 2023-09-23T20:12:27.170 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.976384249Z level=info msg="Executing migration" id="alter preferences.json_data to mediumtext v1" 2023-09-23T20:12:27.170 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.979029048Z level=info msg="Executing migration" id="Add preferences index org_id" 2023-09-23T20:12:27.170 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.981770043Z level=info msg="Executing migration" id="Add preferences index user_id" 2023-09-23T20:12:27.170 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.991527896Z level=info msg="Executing migration" id="create alert table v1" 2023-09-23T20:12:27.170 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.992054773Z level=info msg="Executing migration" id="add index alert org_id & id " 2023-09-23T20:12:27.171 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.994646832Z level=info msg="Executing migration" id="add index alert state" 2023-09-23T20:12:27.171 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:26.997334705Z level=info msg="Executing migration" id="add index alert dashboard_id" 2023-09-23T20:12:27.171 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.000052944Z level=info msg="Executing migration" id="Create alert_rule_tag table v1" 2023-09-23T20:12:27.171 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.002628648Z level=info msg="Executing migration" id="Add unique index alert_rule_tag.alert_id_tag_id" 2023-09-23T20:12:27.171 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.005449117Z level=info msg="Executing migration" id="drop index UQE_alert_rule_tag_alert_id_tag_id - v1" 2023-09-23T20:12:27.172 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.008206129Z level=info msg="Executing migration" id="Rename table alert_rule_tag to alert_rule_tag_v1 - v1" 2023-09-23T20:12:27.172 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.011028416Z level=info msg="Executing migration" id="Create alert_rule_tag table v2" 2023-09-23T20:12:27.172 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.013762055Z level=info msg="Executing migration" id="create index UQE_alert_rule_tag_alert_id_tag_id - Add unique index alert_rule_tag.alert_id_tag_id V2" 2023-09-23T20:12:27.172 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.0164516Z level=info msg="Executing migration" id="copy alert_rule_tag v1 to v2" 2023-09-23T20:12:27.172 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.019020329Z level=info msg="Executing migration" id="drop table alert_rule_tag_v1" 2023-09-23T20:12:27.172 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.021689538Z level=info msg="Executing migration" id="create alert_notification table v1" 2023-09-23T20:12:27.172 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.02439246Z level=info msg="Executing migration" id="Add column is_default" 2023-09-23T20:12:27.172 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.026841375Z level=info msg="Executing migration" id="Add column frequency" 2023-09-23T20:12:27.172 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.02945481Z level=info msg="Executing migration" id="Add column send_reminder" 2023-09-23T20:12:27.172 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.032108524Z level=info msg="Executing migration" id="Add column disable_resolve_message" 2023-09-23T20:12:27.172 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.034613728Z level=info msg="Executing migration" id="add index alert_notification org_id & name" 2023-09-23T20:12:27.173 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.037273236Z level=info msg="Executing migration" id="Update alert table charset" 2023-09-23T20:12:27.173 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.040077874Z level=info msg="Executing migration" id="Update alert_notification table charset" 2023-09-23T20:12:27.173 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.042773114Z level=info msg="Executing migration" id="create notification_journal table v1" 2023-09-23T20:12:27.173 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.045391285Z level=info msg="Executing migration" id="add index notification_journal org_id & alert_id & notifier_id" 2023-09-23T20:12:27.173 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.048077773Z level=info msg="Executing migration" id="drop alert_notification_journal" 2023-09-23T20:12:27.173 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.050804024Z level=info msg="Executing migration" id="create alert_notification_state table v1" 2023-09-23T20:12:27.173 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.053349509Z level=info msg="Executing migration" id="add index alert_notification_state org_id & alert_id & notifier_id" 2023-09-23T20:12:27.173 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.053775318Z level=info msg="Executing migration" id="Add for to alert table" 2023-09-23T20:12:27.173 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.056286168Z level=info msg="Executing migration" id="Add column uid in alert_notification" 2023-09-23T20:12:27.173 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.056967736Z level=info msg="Executing migration" id="Update uid column values in alert_notification" 2023-09-23T20:12:27.173 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.057264883Z level=info msg="Executing migration" id="Add unique index alert_notification_org_id_uid" 2023-09-23T20:12:27.173 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.057715833Z level=info msg="Executing migration" id="Remove unique index org_id_name" 2023-09-23T20:12:27.174 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.058097217Z level=info msg="Executing migration" id="Add column secure_settings in alert_notification" 2023-09-23T20:12:27.174 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.058727361Z level=info msg="Executing migration" id="alter alert.settings to mediumtext" 2023-09-23T20:12:27.174 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.059035161Z level=info msg="Executing migration" id="Add non-unique index alert_notification_state_alert_id" 2023-09-23T20:12:27.174 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.059845937Z level=info msg="Executing migration" id="Add non-unique index alert_rule_tag_alert_id" 2023-09-23T20:12:27.174 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.060273359Z level=info msg="Executing migration" id="Drop old annotation table v4" 2023-09-23T20:12:27.174 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.06063489Z level=info msg="Executing migration" id="create annotation table v5" 2023-09-23T20:12:27.174 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.061417186Z level=info msg="Executing migration" id="add index annotation 0 v3" 2023-09-23T20:12:27.174 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.061820957Z level=info msg="Executing migration" id="add index annotation 1 v3" 2023-09-23T20:12:27.174 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.062611625Z level=info msg="Executing migration" id="add index annotation 2 v3" 2023-09-23T20:12:27.174 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.063413219Z level=info msg="Executing migration" id="add index annotation 3 v3" 2023-09-23T20:12:27.174 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.064187869Z level=info msg="Executing migration" id="add index annotation 4 v3" 2023-09-23T20:12:27.174 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.064639713Z level=info msg="Executing migration" id="Update annotation table charset" 2023-09-23T20:12:27.175 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.065443381Z level=info msg="Executing migration" id="Add column region_id to annotation table" 2023-09-23T20:12:27.175 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.066258901Z level=info msg="Executing migration" id="Drop category_id index" 2023-09-23T20:12:27.175 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.067082582Z level=info msg="Executing migration" id="Add column tags to annotation table" 2023-09-23T20:12:27.175 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.067751843Z level=info msg="Executing migration" id="Create annotation_tag table v2" 2023-09-23T20:12:27.175 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.068596439Z level=info msg="Executing migration" id="Add unique index annotation_tag.annotation_id_tag_id" 2023-09-23T20:12:27.175 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.069438701Z level=info msg="Executing migration" id="drop index UQE_annotation_tag_annotation_id_tag_id - v2" 2023-09-23T20:12:27.175 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.07023882Z level=info msg="Executing migration" id="Rename table annotation_tag to annotation_tag_v2 - v2" 2023-09-23T20:12:27.175 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.071853399Z level=info msg="Executing migration" id="Create annotation_tag table v3" 2023-09-23T20:12:27.175 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.072645616Z level=info msg="Executing migration" id="create index UQE_annotation_tag_annotation_id_tag_id - Add unique index annotation_tag.annotation_id_tag_id V3" 2023-09-23T20:12:27.175 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.073497731Z level=info msg="Executing migration" id="copy annotation_tag v2 to v3" 2023-09-23T20:12:27.175 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.07434699Z level=info msg="Executing migration" id="drop table annotation_tag_v2" 2023-09-23T20:12:27.175 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.075151644Z level=info msg="Executing migration" id="Update alert annotations and set TEXT to empty" 2023-09-23T20:12:27.176 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.076006257Z level=info msg="Executing migration" id="Add created time to annotation table" 2023-09-23T20:12:27.176 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.076798866Z level=info msg="Executing migration" id="Add updated time to annotation table" 2023-09-23T20:12:27.176 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.077561132Z level=info msg="Executing migration" id="Add index for created in annotation table" 2023-09-23T20:12:27.176 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.077999444Z level=info msg="Executing migration" id="Add index for updated in annotation table" 2023-09-23T20:12:27.176 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.078845345Z level=info msg="Executing migration" id="Convert existing annotations from seconds to milliseconds" 2023-09-23T20:12:27.176 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.079582331Z level=info msg="Executing migration" id="Add epoch_end column" 2023-09-23T20:12:27.176 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.08044623Z level=info msg="Executing migration" id="Add index for epoch_end" 2023-09-23T20:12:27.176 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.081100236Z level=info msg="Executing migration" id="Make epoch_end the same as epoch" 2023-09-23T20:12:27.176 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.083696799Z level=info msg="Executing migration" id="Move region to single row" 2023-09-23T20:12:27.176 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.086465762Z level=info msg="Executing migration" id="Remove index org_id_epoch from annotation table" 2023-09-23T20:12:27.176 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.089122913Z level=info msg="Executing migration" id="Remove index org_id_dashboard_id_panel_id_epoch from annotation table" 2023-09-23T20:12:27.176 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.08977851Z level=info msg="Executing migration" id="Add index for org_id_dashboard_id_epoch_end_epoch on annotation table" 2023-09-23T20:12:27.177 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.092631539Z level=info msg="Executing migration" id="Add index for org_id_epoch_end_epoch on annotation table" 2023-09-23T20:12:27.177 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.095262232Z level=info msg="Executing migration" id="Remove index org_id_epoch_epoch_end from annotation table" 2023-09-23T20:12:27.177 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.097943598Z level=info msg="Executing migration" id="Add index for alert_id on annotation table" 2023-09-23T20:12:27.177 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.098574639Z level=info msg="Executing migration" id="Increase tags column to length 4096" 2023-09-23T20:12:27.177 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.101353683Z level=info msg="Executing migration" id="create test_data table" 2023-09-23T20:12:27.177 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.103961418Z level=info msg="Executing migration" id="create dashboard_version table v1" 2023-09-23T20:12:27.177 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.106793622Z level=info msg="Executing migration" id="add index dashboard_version.dashboard_id" 2023-09-23T20:12:27.177 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.107468532Z level=info msg="Executing migration" id="add unique index dashboard_version.dashboard_id and dashboard_version.version" 2023-09-23T20:12:27.177 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.110239809Z level=info msg="Executing migration" id="Set dashboard version to 1 where 0" 2023-09-23T20:12:27.177 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.112886252Z level=info msg="Executing migration" id="save existing dashboard data in dashboard_version table v1" 2023-09-23T20:12:27.177 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.115680611Z level=info msg="Executing migration" id="alter dashboard_version.data to mediumtext v1" 2023-09-23T20:12:27.178 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.116146072Z level=info msg="Executing migration" id="create team table" 2023-09-23T20:12:27.178 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.118883238Z level=info msg="Executing migration" id="add index team.org_id" 2023-09-23T20:12:27.178 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.121451703Z level=info msg="Executing migration" id="add unique index team_org_id_name" 2023-09-23T20:12:27.178 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.124024934Z level=info msg="Executing migration" id="create team member table" 2023-09-23T20:12:27.178 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.124451224Z level=info msg="Executing migration" id="add index team_member.org_id" 2023-09-23T20:12:27.178 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.127133705Z level=info msg="Executing migration" id="add unique index team_member_org_id_team_id_user_id" 2023-09-23T20:12:27.178 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.129803224Z level=info msg="Executing migration" id="add index team_member.team_id" 2023-09-23T20:12:27.178 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.132543845Z level=info msg="Executing migration" id="Add column email to team table" 2023-09-23T20:12:27.178 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.133298674Z level=info msg="Executing migration" id="Add column external to team_member table" 2023-09-23T20:12:27.178 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.134160462Z level=info msg="Executing migration" id="Add column permission to team_member table" 2023-09-23T20:12:27.178 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.135014548Z level=info msg="Executing migration" id="create dashboard acl table" 2023-09-23T20:12:27.178 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.135900325Z level=info msg="Executing migration" id="add index dashboard_acl_dashboard_id" 2023-09-23T20:12:27.179 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.136402752Z level=info msg="Executing migration" id="add unique index dashboard_acl_dashboard_id_user_id" 2023-09-23T20:12:27.179 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.137195547Z level=info msg="Executing migration" id="add unique index dashboard_acl_dashboard_id_team_id" 2023-09-23T20:12:27.179 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.137625446Z level=info msg="Executing migration" id="add index dashboard_acl_user_id" 2023-09-23T20:12:27.179 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.138474099Z level=info msg="Executing migration" id="add index dashboard_acl_team_id" 2023-09-23T20:12:27.179 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.13890564Z level=info msg="Executing migration" id="add index dashboard_acl_org_id_role" 2023-09-23T20:12:27.179 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.139790973Z level=info msg="Executing migration" id="add index dashboard_permission" 2023-09-23T20:12:27.179 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.140634661Z level=info msg="Executing migration" id="save default acl rules in dashboard_acl table" 2023-09-23T20:12:27.179 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.141474613Z level=info msg="Executing migration" id="delete acl rules for deleted dashboards and folders" 2023-09-23T20:12:27.179 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.142329809Z level=info msg="Executing migration" id="create tag table" 2023-09-23T20:12:27.179 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.14322287Z level=info msg="Executing migration" id="add index tag.key_value" 2023-09-23T20:12:27.179 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.144195726Z level=info msg="Executing migration" id="create login attempt table" 2023-09-23T20:12:27.180 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.144938527Z level=info msg="Executing migration" id="add index login_attempt.username" 2023-09-23T20:12:27.180 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.145837051Z level=info msg="Executing migration" id="drop index IDX_login_attempt_username - v1" 2023-09-23T20:12:27.180 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.146761376Z level=info msg="Executing migration" id="Rename table login_attempt to login_attempt_tmp_qwerty - v1" 2023-09-23T20:12:27.180 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.148886473Z level=info msg="Executing migration" id="create login_attempt v2" 2023-09-23T20:12:27.180 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.152012743Z level=info msg="Executing migration" id="create index IDX_login_attempt_username - v2" 2023-09-23T20:12:27.180 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.154704268Z level=info msg="Executing migration" id="copy login_attempt v1 to v2" 2023-09-23T20:12:27.180 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.15763053Z level=info msg="Executing migration" id="drop login_attempt_tmp_qwerty" 2023-09-23T20:12:27.180 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.160503665Z level=info msg="Executing migration" id="create user auth table" 2023-09-23T20:12:27.180 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.163293259Z level=info msg="Executing migration" id="create index IDX_user_auth_auth_module_auth_id - v1" 2023-09-23T20:12:27.180 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.166191641Z level=info msg="Executing migration" id="alter user_auth.auth_id to length 190" 2023-09-23T20:12:27.219 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574885 got 55834574886 for osd.1 2023-09-23T20:12:27.220 DEBUG:teuthology.parallel:result is None 2023-09-23T20:12:27.362 INFO:teuthology.orchestra.run.smithi018.stdout:77309411362 2023-09-23T20:12:27.417 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.169117698Z level=info msg="Executing migration" id="Add OAuth access token to user_auth" 2023-09-23T20:12:27.417 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.172233178Z level=info msg="Executing migration" id="Add OAuth refresh token to user_auth" 2023-09-23T20:12:27.417 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.174624382Z level=info msg="Executing migration" id="Add OAuth token type to user_auth" 2023-09-23T20:12:27.417 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.177403988Z level=info msg="Executing migration" id="Add OAuth expiry to user_auth" 2023-09-23T20:12:27.417 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.180212658Z level=info msg="Executing migration" id="Add index to user_id column in user_auth" 2023-09-23T20:12:27.417 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.183024029Z level=info msg="Executing migration" id="Add OAuth ID token to user_auth" 2023-09-23T20:12:27.417 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.185938041Z level=info msg="Executing migration" id="create server_lock table" 2023-09-23T20:12:27.418 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.18849545Z level=info msg="Executing migration" id="add index server_lock.operation_uid" 2023-09-23T20:12:27.418 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.191391047Z level=info msg="Executing migration" id="create user auth token table" 2023-09-23T20:12:27.418 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.194196795Z level=info msg="Executing migration" id="add unique index user_auth_token.auth_token" 2023-09-23T20:12:27.418 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.197113925Z level=info msg="Executing migration" id="add unique index user_auth_token.prev_auth_token" 2023-09-23T20:12:27.418 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.200080656Z level=info msg="Executing migration" id="add index user_auth_token.user_id" 2023-09-23T20:12:27.418 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.202659368Z level=info msg="Executing migration" id="Add revoked_at to the user auth token" 2023-09-23T20:12:27.418 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.205274362Z level=info msg="Executing migration" id="create cache_data table" 2023-09-23T20:12:27.418 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.207539525Z level=info msg="Executing migration" id="add unique index cache_data.cache_key" 2023-09-23T20:12:27.418 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.210424972Z level=info msg="Executing migration" id="create short_url table v1" 2023-09-23T20:12:27.418 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.212871541Z level=info msg="Executing migration" id="add index short_url.org_id-uid" 2023-09-23T20:12:27.418 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.215439449Z level=info msg="Executing migration" id="alter table short_url alter column created_by type to bigint" 2023-09-23T20:12:27.419 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.218124755Z level=info msg="Executing migration" id="delete alert_definition table" 2023-09-23T20:12:27.419 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.220733452Z level=info msg="Executing migration" id="recreate alert_definition table" 2023-09-23T20:12:27.419 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.223296308Z level=info msg="Executing migration" id="add index in alert_definition on org_id and title columns" 2023-09-23T20:12:27.419 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.225781495Z level=info msg="Executing migration" id="add index in alert_definition on org_id and uid columns" 2023-09-23T20:12:27.419 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.228446033Z level=info msg="Executing migration" id="alter alert_definition table data column to mediumtext in mysql" 2023-09-23T20:12:27.419 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.231160217Z level=info msg="Executing migration" id="drop index in alert_definition on org_id and title columns" 2023-09-23T20:12:27.419 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.233747438Z level=info msg="Executing migration" id="drop index in alert_definition on org_id and uid columns" 2023-09-23T20:12:27.419 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.236335713Z level=info msg="Executing migration" id="add unique index in alert_definition on org_id and title columns" 2023-09-23T20:12:27.419 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.236826074Z level=info msg="Executing migration" id="add unique index in alert_definition on org_id and uid columns" 2023-09-23T20:12:27.419 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.237301712Z level=info msg="Executing migration" id="Add column paused in alert_definition" 2023-09-23T20:12:27.419 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.238121094Z level=info msg="Executing migration" id="drop alert_definition table" 2023-09-23T20:12:27.419 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.238573221Z level=info msg="Executing migration" id="delete alert_definition_version table" 2023-09-23T20:12:27.420 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.238881628Z level=info msg="Executing migration" id="recreate alert_definition_version table" 2023-09-23T20:12:27.420 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.239282793Z level=info msg="Executing migration" id="add index in alert_definition_version table on alert_definition_id and version columns" 2023-09-23T20:12:27.420 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.239788678Z level=info msg="Executing migration" id="add index in alert_definition_version table on alert_definition_uid and version columns" 2023-09-23T20:12:27.420 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.240209057Z level=info msg="Executing migration" id="alter alert_definition_version table data column to mediumtext in mysql" 2023-09-23T20:12:27.420 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.240544877Z level=info msg="Executing migration" id="drop alert_definition_version table" 2023-09-23T20:12:27.420 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.241011052Z level=info msg="Executing migration" id="create alert_instance table" 2023-09-23T20:12:27.420 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.241401779Z level=info msg="Executing migration" id="add index in alert_instance table on def_org_id, def_uid and current_state columns" 2023-09-23T20:12:27.420 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.241862309Z level=info msg="Executing migration" id="add index in alert_instance table on def_org_id, current_state columns" 2023-09-23T20:12:27.420 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.242339754Z level=info msg="Executing migration" id="add column current_state_end to alert_instance" 2023-09-23T20:12:27.420 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.243182446Z level=info msg="Executing migration" id="remove index def_org_id, def_uid, current_state on alert_instance" 2023-09-23T20:12:27.420 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.243622231Z level=info msg="Executing migration" id="remove index def_org_id, current_state on alert_instance" 2023-09-23T20:12:27.420 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.244070122Z level=info msg="Executing migration" id="rename def_org_id to rule_org_id in alert_instance" 2023-09-23T20:12:27.421 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.247422947Z level=info msg="Executing migration" id="rename def_uid to rule_uid in alert_instance" 2023-09-23T20:12:27.421 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.250747117Z level=info msg="Executing migration" id="add index rule_org_id, rule_uid, current_state on alert_instance" 2023-09-23T20:12:27.421 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.251214211Z level=info msg="Executing migration" id="add index rule_org_id, current_state on alert_instance" 2023-09-23T20:12:27.421 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.251693025Z level=info msg="Executing migration" id="add current_reason column related to current_state" 2023-09-23T20:12:27.421 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.252539983Z level=info msg="Executing migration" id="create alert_rule table" 2023-09-23T20:12:27.421 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.252981808Z level=info msg="Executing migration" id="add index in alert_rule on org_id and title columns" 2023-09-23T20:12:27.421 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.253434886Z level=info msg="Executing migration" id="add index in alert_rule on org_id and uid columns" 2023-09-23T20:12:27.421 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.253894661Z level=info msg="Executing migration" id="add index in alert_rule on org_id, namespace_uid, group_uid columns" 2023-09-23T20:12:27.421 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.254358065Z level=info msg="Executing migration" id="alter alert_rule table data column to mediumtext in mysql" 2023-09-23T20:12:27.422 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.254674404Z level=info msg="Executing migration" id="add column for to alert_rule" 2023-09-23T20:12:27.422 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.255502486Z level=info msg="Executing migration" id="add column annotations to alert_rule" 2023-09-23T20:12:27.422 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.256353075Z level=info msg="Executing migration" id="add column labels to alert_rule" 2023-09-23T20:12:27.422 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.257199726Z level=info msg="Executing migration" id="remove unique index from alert_rule on org_id, title columns" 2023-09-23T20:12:27.422 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.257654729Z level=info msg="Executing migration" id="add index in alert_rule on org_id, namespase_uid and title columns" 2023-09-23T20:12:27.422 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.258128571Z level=info msg="Executing migration" id="add dashboard_uid column to alert_rule" 2023-09-23T20:12:27.422 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.258979089Z level=info msg="Executing migration" id="add panel_id column to alert_rule" 2023-09-23T20:12:27.422 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.259842067Z level=info msg="Executing migration" id="add index in alert_rule on org_id, dashboard_uid and panel_id columns" 2023-09-23T20:12:27.422 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.26032859Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule" 2023-09-23T20:12:27.422 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.261162764Z level=info msg="Executing migration" id="add is_paused column to alert_rule table" 2023-09-23T20:12:27.422 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.262027347Z level=info msg="Executing migration" id="fix is_paused column for alert_rule table" 2023-09-23T20:12:27.423 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.262361119Z level=info msg="Executing migration" id="create alert_rule_version table" 2023-09-23T20:12:27.423 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.262853101Z level=info msg="Executing migration" id="add index in alert_rule_version table on rule_org_id, rule_uid and version columns" 2023-09-23T20:12:27.423 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.263342758Z level=info msg="Executing migration" id="add index in alert_rule_version table on rule_org_id, rule_namespace_uid and rule_group columns" 2023-09-23T20:12:27.423 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.263817671Z level=info msg="Executing migration" id="alter alert_rule_version table data column to mediumtext in mysql" 2023-09-23T20:12:27.423 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.264096488Z level=info msg="Executing migration" id="add column for to alert_rule_version" 2023-09-23T20:12:27.423 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.264992436Z level=info msg="Executing migration" id="add column annotations to alert_rule_version" 2023-09-23T20:12:27.424 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.267634029Z level=info msg="Executing migration" id="add column labels to alert_rule_version" 2023-09-23T20:12:27.424 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.26850594Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule_version" 2023-09-23T20:12:27.424 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.269358814Z level=info msg="Executing migration" id="add is_paused column to alert_rule_versions table" 2023-09-23T20:12:27.424 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.270222694Z level=info msg="Executing migration" id="fix is_paused column for alert_rule_version table" 2023-09-23T20:12:27.425 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.270545537Z level=info msg="Executing migration" id=create_alert_configuration_table 2023-09-23T20:12:27.425 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.270983616Z level=info msg="Executing migration" id="Add column default in alert_configuration" 2023-09-23T20:12:27.425 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.271881214Z level=info msg="Executing migration" id="alert alert_configuration alertmanager_configuration column from TEXT to MEDIUMTEXT if mysql" 2023-09-23T20:12:27.425 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.272202857Z level=info msg="Executing migration" id="add column org_id in alert_configuration" 2023-09-23T20:12:27.425 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.273108238Z level=info msg="Executing migration" id="add index in alert_configuration table on org_id column" 2023-09-23T20:12:27.426 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.275730741Z level=info msg="Executing migration" id="add configuration_hash column to alert_configuration" 2023-09-23T20:12:27.426 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.27662255Z level=info msg="Executing migration" id=create_ngalert_configuration_table 2023-09-23T20:12:27.426 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.277023025Z level=info msg="Executing migration" id="add index in ngalert_configuration on org_id column" 2023-09-23T20:12:27.426 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.277489644Z level=info msg="Executing migration" id="add column send_alerts_to in ngalert_configuration" 2023-09-23T20:12:27.426 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.278355327Z level=info msg="Executing migration" id="create provenance_type table" 2023-09-23T20:12:27.426 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.278800119Z level=info msg="Executing migration" id="add index to uniquify (record_key, record_type, org_id) columns" 2023-09-23T20:12:27.426 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.279292658Z level=info msg="Executing migration" id="create alert_image table" 2023-09-23T20:12:27.426 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.279743789Z level=info msg="Executing migration" id="add unique index on token to alert_image table" 2023-09-23T20:12:27.426 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.280186217Z level=info msg="Executing migration" id="support longer URLs in alert_image table" 2023-09-23T20:12:27.427 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.28051161Z level=info msg="Executing migration" id=create_alert_configuration_history_table 2023-09-23T20:12:27.427 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.280945641Z level=info msg="Executing migration" id="drop non-unique orgID index on alert_configuration" 2023-09-23T20:12:27.427 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.281395245Z level=info msg="Executing migration" id="drop unique orgID index on alert_configuration if exists" 2023-09-23T20:12:27.427 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.281490939Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop unique orgID index on alert_configuration if exists" 2023-09-23T20:12:27.427 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.281792177Z level=info msg="Executing migration" id="extract alertmanager configuration history to separate table" 2023-09-23T20:12:27.427 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.282189581Z level=info msg="Executing migration" id="add unique index on orgID to alert_configuration" 2023-09-23T20:12:27.427 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.282689138Z level=info msg="Executing migration" id="clear migration entry \"remove unified alerting data\"" 2023-09-23T20:12:27.427 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.282821454Z level=info msg="Executing migration" id="move dashboard alerts to unified alerting" 2023-09-23T20:12:27.427 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.282945726Z level=info msg="alerts found to migrate" alerts=0 2023-09-23T20:12:27.427 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.283381436Z level=info msg="Executing migration" id="create library_element table v1" 2023-09-23T20:12:27.428 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.284226838Z level=info msg="Executing migration" id="add index library_element org_id-folder_id-name-kind" 2023-09-23T20:12:27.428 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.28503832Z level=info msg="Executing migration" id="create library_element_connection table v1" 2023-09-23T20:12:27.428 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.285909905Z level=info msg="Executing migration" id="add index library_element_connection element_id-kind-connection_id" 2023-09-23T20:12:27.428 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.286708887Z level=info msg="Executing migration" id="add unique index library_element org_id_uid" 2023-09-23T20:12:27.428 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.287535299Z level=info msg="Executing migration" id="increase max description length to 2048" 2023-09-23T20:12:27.428 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.288337279Z level=info msg="Executing migration" id="clone move dashboard alerts to unified alerting" 2023-09-23T20:12:27.428 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.289176873Z level=info msg="Executing migration" id="create data_keys table" 2023-09-23T20:12:27.428 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.290055301Z level=info msg="Executing migration" id="create secrets table" 2023-09-23T20:12:27.428 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.29094179Z level=info msg="Executing migration" id="rename data_keys name column to id" 2023-09-23T20:12:27.428 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.295498654Z level=info msg="Executing migration" id="add name column into data_keys" 2023-09-23T20:12:27.429 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.296470197Z level=info msg="Executing migration" id="copy data_keys id column values into name" 2023-09-23T20:12:27.429 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.297313323Z level=info msg="Executing migration" id="rename data_keys name column to label" 2023-09-23T20:12:27.429 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.301823921Z level=info msg="Executing migration" id="rename data_keys id column back to name" 2023-09-23T20:12:27.429 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.306834548Z level=info msg="Executing migration" id="create kv_store table v1" 2023-09-23T20:12:27.429 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.307813449Z level=info msg="Executing migration" id="add index kv_store.org_id-namespace-key" 2023-09-23T20:12:27.429 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.308561544Z level=info msg="Executing migration" id="update dashboard_uid and panel_id from existing annotations" 2023-09-23T20:12:27.429 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.311119286Z level=info msg="Executing migration" id="create permission table" 2023-09-23T20:12:27.429 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.313701069Z level=info msg="Executing migration" id="add unique index permission.role_id" 2023-09-23T20:12:27.429 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.316364645Z level=info msg="Executing migration" id="add unique index role_id_action_scope" 2023-09-23T20:12:27.430 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.318988481Z level=info msg="Executing migration" id="create role table" 2023-09-23T20:12:27.430 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.321586077Z level=info msg="Executing migration" id="add column display_name" 2023-09-23T20:12:27.430 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.324288302Z level=info msg="Executing migration" id="add column group_name" 2023-09-23T20:12:27.430 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.326925315Z level=info msg="Executing migration" id="add index role.org_id" 2023-09-23T20:12:27.430 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.329673377Z level=info msg="Executing migration" id="add unique index role_org_id_name" 2023-09-23T20:12:27.430 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.332210434Z level=info msg="Executing migration" id="add index role_org_id_uid" 2023-09-23T20:12:27.430 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.335003664Z level=info msg="Executing migration" id="create team role table" 2023-09-23T20:12:27.430 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.337411895Z level=info msg="Executing migration" id="add index team_role.org_id" 2023-09-23T20:12:27.430 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.340003052Z level=info msg="Executing migration" id="add unique index team_role_org_id_team_id_role_id" 2023-09-23T20:12:27.430 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.342586408Z level=info msg="Executing migration" id="add index team_role.team_id" 2023-09-23T20:12:27.431 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.345264932Z level=info msg="Executing migration" id="create user role table" 2023-09-23T20:12:27.431 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.347909987Z level=info msg="Executing migration" id="add index user_role.org_id" 2023-09-23T20:12:27.431 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.350555684Z level=info msg="Executing migration" id="add unique index user_role_org_id_user_id_role_id" 2023-09-23T20:12:27.431 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.35336095Z level=info msg="Executing migration" id="add index user_role.user_id" 2023-09-23T20:12:27.431 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.356125866Z level=info msg="Executing migration" id="create builtin role table" 2023-09-23T20:12:27.431 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.356960579Z level=info msg="Executing migration" id="add index builtin_role.role_id" 2023-09-23T20:12:27.431 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.3576343Z level=info msg="Executing migration" id="add index builtin_role.name" 2023-09-23T20:12:27.431 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.358429448Z level=info msg="Executing migration" id="Add column org_id to builtin_role table" 2023-09-23T20:12:27.431 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.359425092Z level=info msg="Executing migration" id="add index builtin_role.org_id" 2023-09-23T20:12:27.432 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.360268072Z level=info msg="Executing migration" id="add unique index builtin_role_org_id_role_id_role" 2023-09-23T20:12:27.432 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.361033562Z level=info msg="Executing migration" id="Remove unique index role_org_id_uid" 2023-09-23T20:12:27.432 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.361814165Z level=info msg="Executing migration" id="add unique index role.uid" 2023-09-23T20:12:27.432 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.362693126Z level=info msg="Executing migration" id="create seed assignment table" 2023-09-23T20:12:27.432 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.363393464Z level=info msg="Executing migration" id="add unique index builtin_role_role_name" 2023-09-23T20:12:27.432 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.364405437Z level=info msg="Executing migration" id="add column hidden to role table" 2023-09-23T20:12:27.432 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.36545551Z level=info msg="Executing migration" id="create query_history table v1" 2023-09-23T20:12:27.432 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.366261901Z level=info msg="Executing migration" id="add index query_history.org_id-created_by-datasource_uid" 2023-09-23T20:12:27.432 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.367207261Z level=info msg="Executing migration" id="alter table query_history alter column created_by type to bigint" 2023-09-23T20:12:27.432 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.367831702Z level=info msg="Executing migration" id="teams permissions migration" 2023-09-23T20:12:27.433 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.368643995Z level=info msg="Executing migration" id="dashboard permissions" 2023-09-23T20:12:27.433 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.369157667Z level=info msg="Executing migration" id="dashboard permissions uid scopes" 2023-09-23T20:12:27.433 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.369571796Z level=info msg="Executing migration" id="drop managed folder create actions" 2023-09-23T20:12:27.433 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.370389207Z level=info msg="Executing migration" id="alerting notification permissions" 2023-09-23T20:12:27.433 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.371399759Z level=info msg="Executing migration" id="create query_history_star table v1" 2023-09-23T20:12:27.433 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.372108663Z level=info msg="Executing migration" id="add index query_history.user_id-query_uid" 2023-09-23T20:12:27.433 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.372908713Z level=info msg="Executing migration" id="add column org_id in query_history_star" 2023-09-23T20:12:27.433 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.373914159Z level=info msg="Executing migration" id="alter table query_history_star_mig column user_id type to bigint" 2023-09-23T20:12:27.433 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.374747876Z level=info msg="Executing migration" id="create correlation table v1" 2023-09-23T20:12:27.433 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.375537119Z level=info msg="Executing migration" id="add index correlations.uid" 2023-09-23T20:12:27.434 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.376590641Z level=info msg="Executing migration" id="add index correlations.source_uid" 2023-09-23T20:12:27.434 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.377165945Z level=info msg="Executing migration" id="add correlation config column" 2023-09-23T20:12:27.434 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.378117683Z level=info msg="Executing migration" id="create entity_events table" 2023-09-23T20:12:27.434 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.379001351Z level=info msg="Executing migration" id="create dashboard public config v1" 2023-09-23T20:12:27.434 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.379865996Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v1" 2023-09-23T20:12:27.434 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.379952194Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index UQE_dashboard_public_config_uid - v1" 2023-09-23T20:12:27.434 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.380798573Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2023-09-23T20:12:27.434 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.380896006Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2023-09-23T20:12:27.434 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.381461599Z level=info msg="Executing migration" id="Drop old dashboard public config table" 2023-09-23T20:12:27.434 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.382295412Z level=info msg="Executing migration" id="recreate dashboard public config v1" 2023-09-23T20:12:27.435 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.383193136Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v1" 2023-09-23T20:12:27.435 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.38397601Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2023-09-23T20:12:27.435 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.384939255Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v2" 2023-09-23T20:12:27.435 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.387392656Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2023-09-23T20:12:27.435 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.3902242Z level=info msg="Executing migration" id="Drop public config table" 2023-09-23T20:12:27.435 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.39272692Z level=info msg="Executing migration" id="Recreate dashboard public config v2" 2023-09-23T20:12:27.435 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.395383345Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v2" 2023-09-23T20:12:27.435 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.398013444Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2023-09-23T20:12:27.435 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.400591666Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_access_token - v2" 2023-09-23T20:12:27.435 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.403288227Z level=info msg="Executing migration" id="Rename table dashboard_public_config to dashboard_public - v2" 2023-09-23T20:12:27.435 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.40625704Z level=info msg="Executing migration" id="add annotations_enabled column" 2023-09-23T20:12:27.435 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.409081908Z level=info msg="Executing migration" id="add time_selection_enabled column" 2023-09-23T20:12:27.436 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.411614311Z level=info msg="Executing migration" id="delete orphaned public dashboards" 2023-09-23T20:12:27.436 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.414335172Z level=info msg="Executing migration" id="add share column" 2023-09-23T20:12:27.436 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.416798084Z level=info msg="Executing migration" id="create default alerting folders" 2023-09-23T20:12:27.436 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.419426332Z level=info msg="Executing migration" id="create file table" 2023-09-23T20:12:27.641 INFO:teuthology.orchestra.run.smithi018.stdout:128849018906 2023-09-23T20:12:27.690 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:12:27.716 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.422059729Z level=info msg="Executing migration" id="file table idx: path natural pk" 2023-09-23T20:12:27.716 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.424728508Z level=info msg="Executing migration" id="file table idx: parent_folder_path_hash fast folder retrieval" 2023-09-23T20:12:27.716 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.427578663Z level=info msg="Executing migration" id="create file_meta table" 2023-09-23T20:12:27.716 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.430305814Z level=info msg="Executing migration" id="file table idx: path key" 2023-09-23T20:12:27.716 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.432870092Z level=info msg="Executing migration" id="set path collation in file table" 2023-09-23T20:12:27.717 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.435561571Z level=info msg="Executing migration" id="managed permissions migration" 2023-09-23T20:12:27.717 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.438119907Z level=info msg="Executing migration" id="managed folder permissions alert actions migration" 2023-09-23T20:12:27.717 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.440763823Z level=info msg="Executing migration" id="RBAC action name migrator" 2023-09-23T20:12:27.717 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.443314135Z level=info msg="Executing migration" id="Add UID column to playlist" 2023-09-23T20:12:27.717 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.446103183Z level=info msg="Executing migration" id="Update uid column values in playlist" 2023-09-23T20:12:27.717 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.448761278Z level=info msg="Executing migration" id="Add index for uid in playlist" 2023-09-23T20:12:27.717 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.451385284Z level=info msg="Executing migration" id="update group index for alert rules" 2023-09-23T20:12:27.717 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.454049297Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated migration" 2023-09-23T20:12:27.717 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.456607416Z level=info msg="Executing migration" id="admin only folder/dashboard permission" 2023-09-23T20:12:27.717 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.459403765Z level=info msg="Executing migration" id="add action column to seed_assignment" 2023-09-23T20:12:27.718 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.461950085Z level=info msg="Executing migration" id="add scope column to seed_assignment" 2023-09-23T20:12:27.718 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.464478688Z level=info msg="Executing migration" id="remove unique index builtin_role_role_name before nullable update" 2023-09-23T20:12:27.718 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.466961545Z level=info msg="Executing migration" id="update seed_assignment role_name column to nullable" 2023-09-23T20:12:27.718 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.47684094Z level=info msg="Executing migration" id="add unique index builtin_role_name back" 2023-09-23T20:12:27.718 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.479621967Z level=info msg="Executing migration" id="add unique index builtin_role_action_scope" 2023-09-23T20:12:27.718 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.482217587Z level=info msg="Executing migration" id="add primary key to seed_assigment" 2023-09-23T20:12:27.718 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.485674215Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated fixed migration" 2023-09-23T20:12:27.718 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.486440772Z level=info msg="Executing migration" id="migrate external alertmanagers to datsourcse" 2023-09-23T20:12:27.718 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.487410502Z level=info msg="Executing migration" id="create folder table" 2023-09-23T20:12:27.718 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.488184424Z level=info msg="Executing migration" id="Add index for parent_uid" 2023-09-23T20:12:27.718 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.488853482Z level=info msg="Executing migration" id="Add unique index for folder.uid and folder.org_id" 2023-09-23T20:12:27.718 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.48971096Z level=info msg="Executing migration" id="Update folder title length" 2023-09-23T20:12:27.719 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.490575399Z level=info msg="Executing migration" id="Add unique index for folder.title and folder.parent_uid" 2023-09-23T20:12:27.719 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=migrator t=2023-09-23T20:12:27.491247817Z level=info msg="migrations completed" performed=485 skipped=0 duration=681.219709ms 2023-09-23T20:12:27.719 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=sqlstore t=2023-09-23T20:12:27.491648674Z level=info msg="Created default organization" 2023-09-23T20:12:27.719 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=plugin.loader t=2023-09-23T20:12:27.54946196Z level=info msg="Plugin registered" pluginID=input 2023-09-23T20:12:27.719 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=plugin.loader t=2023-09-23T20:12:27.57765198Z level=info msg="Plugin registered" pluginID=grafana-piechart-panel 2023-09-23T20:12:27.719 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=plugin.loader t=2023-09-23T20:12:27.577689243Z level=info msg="Plugin registered" pluginID=vonage-status-panel 2023-09-23T20:12:27.719 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=secrets t=2023-09-23T20:12:27.577738146Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1 2023-09-23T20:12:27.719 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=query_data t=2023-09-23T20:12:27.579691669Z level=info msg="Query Service initialization" 2023-09-23T20:12:27.719 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=live.push_http t=2023-09-23T20:12:27.582738115Z level=info msg="Live Push Gateway initialization" 2023-09-23T20:12:27.724 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738408 got 34359738409 for osd.0 2023-09-23T20:12:27.725 DEBUG:teuthology.parallel:result is None 2023-09-23T20:12:27.906 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:27 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:27.906 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:27 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:27.906 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:27 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:27.906 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:27 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:27.906 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:27 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:12:27.907 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:27 smithi018 ceph-mon[95889]: from='client.? 172.21.15.18:0/3408599938' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T20:12:27.907 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:27 smithi018 ceph-mon[95889]: from='client.? 172.21.15.18:0/2376579283' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T20:12:27.907 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:27 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:27.907 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:27 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:27.907 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:27 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:27.907 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:27 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:27.907 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:27 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:12:27.908 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:27 smithi018 ceph-mon[109137]: from='client.? 172.21.15.18:0/3408599938' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T20:12:27.908 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:27 smithi018 ceph-mon[109137]: from='client.? 172.21.15.18:0/2376579283' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T20:12:28.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:28.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:28.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:28.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:28.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:12:28.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-mon[100764]: from='client.? 172.21.15.18:0/3408599938' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T20:12:28.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:27 smithi149 ceph-mon[100764]: from='client.? 172.21.15.18:0/2376579283' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T20:12:28.716 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:28 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=infra.usagestats.collector t=2023-09-23T20:12:28.365164341Z level=info msg="registering usage stat providers" usageStatsProvidersLen=2 2023-09-23T20:12:28.716 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:28 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=provisioning.datasources t=2023-09-23T20:12:28.365517014Z level=warn msg="[Deprecated] the datasource provisioning config is outdated. please upgrade" filename=/etc/grafana/provisioning/datasources/ceph-dashboard.yml 2023-09-23T20:12:28.716 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:28 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=provisioning.datasources t=2023-09-23T20:12:28.397646184Z level=info msg="inserting datasource from configuration " name=Dashboard1 uid=P43CA22E17D0F9596 2023-09-23T20:12:28.716 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:28 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=provisioning.datasources t=2023-09-23T20:12:28.42951344Z level=info msg="inserting datasource from configuration " name=Loki uid=P8E80F9AEF21F6940 2023-09-23T20:12:28.716 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:28 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=provisioning.alerting t=2023-09-23T20:12:28.451064864Z level=info msg="starting to provision alerting" 2023-09-23T20:12:28.716 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:28 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=provisioning.alerting t=2023-09-23T20:12:28.45108505Z level=info msg="finished to provision alerting" 2023-09-23T20:12:28.716 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:28 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=ngalert.state.manager t=2023-09-23T20:12:28.451252228Z level=info msg="Warming state cache for startup" 2023-09-23T20:12:28.716 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:28 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=grafanaStorageLogger t=2023-09-23T20:12:28.451276415Z level=info msg="storage starting" 2023-09-23T20:12:28.717 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:28 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=http.server t=2023-09-23T20:12:28.453386561Z level=info msg="HTTP Server Listen" address=[::]:3000 protocol=https subUrl= socket= 2023-09-23T20:12:28.717 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:28 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=ngalert.state.manager t=2023-09-23T20:12:28.461567799Z level=info msg="State cache has been initialized" states=0 duration=10.314826ms 2023-09-23T20:12:28.717 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:28 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=ticker t=2023-09-23T20:12:28.461618138Z level=info msg=starting first_tick=2023-09-23T20:12:30Z 2023-09-23T20:12:28.717 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:28 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=ngalert.multiorg.alertmanager t=2023-09-23T20:12:28.461633086Z level=info msg="starting MultiOrg Alertmanager" 2023-09-23T20:12:28.717 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:28 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=sqlstore.transactions t=2023-09-23T20:12:28.461643938Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2023-09-23T20:12:28.717 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:28 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=sqlstore.transactions t=2023-09-23T20:12:28.495607385Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2023-09-23T20:12:28.717 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:28 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=sqlstore.transactions t=2023-09-23T20:12:28.536396885Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2023-09-23T20:12:28.717 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:12:28 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[137962]: logger=sqlstore.transactions t=2023-09-23T20:12:28.547035314Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=1 code="database is locked" 2023-09-23T20:12:28.800 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849018904 got 128849018906 for osd.4 2023-09-23T20:12:28.800 DEBUG:teuthology.parallel:result is None 2023-09-23T20:12:28.828 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411361 got 77309411362 for osd.2 2023-09-23T20:12:28.828 DEBUG:teuthology.parallel:result is None 2023-09-23T20:12:29.131 INFO:teuthology.orchestra.run.smithi018.stdout:167503724567 2023-09-23T20:12:29.140 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:28 smithi018 ceph-mon[109137]: pgmap v26: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:12:29.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:28 smithi018 ceph-mon[95889]: pgmap v26: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:12:29.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:28 smithi149 ceph-mon[100764]: pgmap v26: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:12:29.792 INFO:teuthology.orchestra.run.smithi018.stdout:201863462930 2023-09-23T20:12:30.140 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:29 smithi018 ceph-mon[109137]: from='client.? 172.21.15.18:0/397331363' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T20:12:30.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:29 smithi018 ceph-mon[109137]: pgmap v27: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:12:30.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:29 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:30.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:29 smithi018 ceph-mon[109137]: from='client.? 172.21.15.18:0/3371295565' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T20:12:30.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:29 smithi018 ceph-mon[95889]: from='client.? 172.21.15.18:0/397331363' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T20:12:30.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:29 smithi018 ceph-mon[95889]: pgmap v27: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:12:30.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:29 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:30.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:29 smithi018 ceph-mon[95889]: from='client.? 172.21.15.18:0/3371295565' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T20:12:30.150 INFO:tasks.cephadm.ceph_manager.ceph:need seq 167503724565 got 167503724567 for osd.5 2023-09-23T20:12:30.150 DEBUG:teuthology.parallel:result is None 2023-09-23T20:12:30.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:29 smithi149 ceph-mon[100764]: from='client.? 172.21.15.18:0/397331363' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T20:12:30.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:29 smithi149 ceph-mon[100764]: pgmap v27: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:12:30.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:29 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:30.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:29 smithi149 ceph-mon[100764]: from='client.? 172.21.15.18:0/3371295565' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T20:12:30.667 INFO:tasks.cephadm.ceph_manager.ceph:need seq 201863462929 got 201863462930 for osd.6 2023-09-23T20:12:30.667 DEBUG:teuthology.parallel:result is None 2023-09-23T20:12:30.667 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2023-09-23T20:12:30.668 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:12:31.276 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:12:32.083 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:32 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:32.083 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:32 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:32.083 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:32 smithi018 ceph-mon[95889]: pgmap v28: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:12:32.084 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:32 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:32.084 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:32 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:32.084 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:32 smithi018 ceph-mon[109137]: pgmap v28: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:12:32.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:32 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:32.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:32 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:32.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:32 smithi149 ceph-mon[100764]: pgmap v28: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:12:32.955 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:12:32.957 INFO:teuthology.orchestra.run.smithi018.stderr:dumped all 2023-09-23T20:12:33.494 INFO:teuthology.orchestra.run.smithi018.stdout:{"pg_ready":true,"pg_map":{"version":28,"stamp":"2023-09-23T20:12:31.672049+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":50472,"kb_used_data":6760,"kb_used_omap":0,"kb_used_meta":43648,"kb_avail":749910744,"statfs":{"total":767960285184,"available":767908601856,"internally_reserved":0,"allocated":6922240,"data_stored":3908453,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":44695552},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002220"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":10,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-09-23T20:12:00.851079+0000","last_change":"2023-09-23T20:12:00.850675+0000","last_active":"2023-09-23T20:12:00.851079+0000","last_peered":"2023-09-23T20:12:00.851079+0000","last_clean":"2023-09-23T20:12:00.851079+0000","last_became_active":"2023-09-23T20:12:00.850315+0000","last_became_peered":"2023-09-23T20:12:00.850315+0000","last_unstale":"2023-09-23T20:12:00.851079+0000","last_undegraded":"2023-09-23T20:12:00.851079+0000","last_fullsized":"2023-09-23T20:12:00.851079+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:03:33.446629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2023-09-23T20:12:01.854874+0000","last_change":"2023-09-23T20:12:01.062560+0000","last_active":"2023-09-23T20:12:01.854874+0000","last_peered":"2023-09-23T20:12:01.854874+0000","last_clean":"2023-09-23T20:12:01.854874+0000","last_became_active":"2023-09-23T20:12:01.062216+0000","last_became_peered":"2023-09-23T20:12:01.062216+0000","last_unstale":"2023-09-23T20:12:01.854874+0000","last_undegraded":"2023-09-23T20:12:01.854874+0000","last_fullsized":"2023-09-23T20:12:01.854874+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:01:52.795311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":10,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-09-23T20:12:01.062834+0000","last_change":"2023-09-23T20:12:01.062442+0000","last_active":"2023-09-23T20:12:01.062834+0000","last_peered":"2023-09-23T20:12:01.062834+0000","last_clean":"2023-09-23T20:12:01.062834+0000","last_became_active":"2023-09-23T20:12:01.062047+0000","last_became_peered":"2023-09-23T20:12:01.062047+0000","last_unstale":"2023-09-23T20:12:01.062834+0000","last_undegraded":"2023-09-23T20:12:01.062834+0000","last_fullsized":"2023-09-23T20:12:01.062834+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:58:49.032117+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":10,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-09-23T20:12:00.849374+0000","last_change":"2023-09-23T20:12:00.848951+0000","last_active":"2023-09-23T20:12:00.849374+0000","last_peered":"2023-09-23T20:12:00.849374+0000","last_clean":"2023-09-23T20:12:00.849374+0000","last_became_active":"2023-09-23T20:12:00.848720+0000","last_became_peered":"2023-09-23T20:12:00.848720+0000","last_unstale":"2023-09-23T20:12:00.849374+0000","last_undegraded":"2023-09-23T20:12:00.849374+0000","last_fullsized":"2023-09-23T20:12:00.849374+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:45:24.720986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"60'128","reported_seq":272,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-09-23T20:12:00.846751+0000","last_change":"2023-09-23T20:11:27.849030+0000","last_active":"2023-09-23T20:12:00.846751+0000","last_peered":"2023-09-23T20:12:00.846751+0000","last_clean":"2023-09-23T20:12:00.846751+0000","last_became_active":"2023-09-23T20:11:27.848678+0000","last_became_peered":"2023-09-23T20:11:27.848678+0000","last_unstale":"2023-09-23T20:12:00.846751+0000","last_undegraded":"2023-09-23T20:12:00.846751+0000","last_fullsized":"2023-09-23T20:12:00.846751+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:09:47.213130+0000","objects_scrubbed":0,"log_size":128,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:33:53.621689+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":10,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-09-23T20:12:00.852221+0000","last_change":"2023-09-23T20:12:00.851792+0000","last_active":"2023-09-23T20:12:00.852221+0000","last_peered":"2023-09-23T20:12:00.852221+0000","last_clean":"2023-09-23T20:12:00.852221+0000","last_became_active":"2023-09-23T20:12:00.851408+0000","last_became_peered":"2023-09-23T20:12:00.851408+0000","last_unstale":"2023-09-23T20:12:00.852221+0000","last_undegraded":"2023-09-23T20:12:00.852221+0000","last_fullsized":"2023-09-23T20:12:00.852221+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:19:26.564015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":10,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-09-23T20:12:01.011265+0000","last_change":"2023-09-23T20:12:01.010922+0000","last_active":"2023-09-23T20:12:01.011265+0000","last_peered":"2023-09-23T20:12:01.011265+0000","last_clean":"2023-09-23T20:12:01.011265+0000","last_became_active":"2023-09-23T20:12:01.010591+0000","last_became_peered":"2023-09-23T20:12:01.010591+0000","last_unstale":"2023-09-23T20:12:01.011265+0000","last_undegraded":"2023-09-23T20:12:01.011265+0000","last_fullsized":"2023-09-23T20:12:01.011265+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:11:25.451578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":10,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-09-23T20:12:00.852173+0000","last_change":"2023-09-23T20:12:00.851594+0000","last_active":"2023-09-23T20:12:00.852173+0000","last_peered":"2023-09-23T20:12:00.852173+0000","last_clean":"2023-09-23T20:12:00.852173+0000","last_became_active":"2023-09-23T20:12:00.851256+0000","last_became_peered":"2023-09-23T20:12:00.851256+0000","last_unstale":"2023-09-23T20:12:00.852173+0000","last_undegraded":"2023-09-23T20:12:00.852173+0000","last_fullsized":"2023-09-23T20:12:00.852173+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:12:00.121999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2023-09-23T20:12:01.854802+0000","last_change":"2023-09-23T20:12:00.847882+0000","last_active":"2023-09-23T20:12:01.854802+0000","last_peered":"2023-09-23T20:12:01.854802+0000","last_clean":"2023-09-23T20:12:01.854802+0000","last_became_active":"2023-09-23T20:12:00.847511+0000","last_became_peered":"2023-09-23T20:12:00.847511+0000","last_unstale":"2023-09-23T20:12:01.854802+0000","last_undegraded":"2023-09-23T20:12:01.854802+0000","last_fullsized":"2023-09-23T20:12:01.854802+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:52:12.888646+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201295,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6336,"kb_used_data":1208,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93738816,"statfs":{"total":95995035648,"available":95988547584,"internally_reserved":0,"allocated":1236992,"data_stored":857577,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:12:25 2023","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":0.52300000000000002,"15min":0.52300000000000002},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.74299999999999999,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.61599999999999999},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":1.004,"5min":1.004,"15min":1.004},"last":0.65800000000000003}]},{"osd":1,"last update":"Sat Sep 23 20:12:25 2023","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.86799999999999999,"5min":0.86799999999999999,"15min":0.86799999999999999},"last":0.71899999999999997},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":1.0309999999999999,"5min":1.0309999999999999,"15min":1.0309999999999999},"last":0.75700000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:12:25 2023","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.98999999999999999,"5min":0.98999999999999999,"15min":0.98999999999999999},"last":0.74299999999999999},{"interface":"front","average":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.92400000000000004,"5min":0.92400000000000004,"15min":0.92400000000000004},"last":0.67600000000000005}]},{"osd":3,"last update":"Sat Sep 23 20:12:25 2023","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.88800000000000001,"5min":0.88800000000000001,"15min":0.88800000000000001},"last":0.77900000000000003},{"interface":"front","average":{"1min":0.61899999999999999,"5min":0.61899999999999999,"15min":0.61899999999999999},"min":{"1min":0.45300000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"max":{"1min":0.97899999999999998,"5min":0.97899999999999998,"15min":0.97899999999999998},"last":0.76800000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:12:25 2023","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":0.52400000000000002,"15min":0.52400000000000002},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.94099999999999995,"5min":0.94099999999999995,"15min":0.94099999999999995},"last":0.60499999999999998},{"interface":"front","average":{"1min":0.54200000000000004,"5min":0.54200000000000004,"15min":0.54200000000000004},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.80700000000000005,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.63}]},{"osd":5,"last update":"Sat Sep 23 20:12:25 2023","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.85899999999999999,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.56599999999999995},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":0.97699999999999998,"5min":0.97699999999999998,"15min":0.97699999999999998},"last":0.73199999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:12:25 2023","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":1.0569999999999999,"5min":1.0569999999999999,"15min":1.0569999999999999},"last":0.68899999999999995},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.879,"5min":0.879,"15min":0.879},"last":0.69799999999999995}]}]},{"osd":6,"up_from":47,"seq":201863462931,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6400,"kb_used_data":1208,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93738752,"statfs":{"total":95995035648,"available":95988482048,"internally_reserved":0,"allocated":1236992,"data_stored":857577,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:12:06 2023","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.77300000000000002,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.42199999999999999},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.78400000000000003,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.59199999999999997}]},{"osd":1,"last update":"Sat Sep 23 20:12:06 2023","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.73999999999999999,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.57299999999999995},{"interface":"front","average":{"1min":0.53700000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.72499999999999998,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.50600000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:12:06 2023","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.55100000000000005,"15min":0.55100000000000005},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.70699999999999996,"5min":0.70699999999999996,"15min":0.70699999999999996},"last":0.48399999999999999},{"interface":"front","average":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.68999999999999995,"5min":0.68999999999999995,"15min":0.68999999999999995},"last":0.55900000000000005}]},{"osd":3,"last update":"Sat Sep 23 20:12:06 2023","interfaces":[{"interface":"back","average":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.77200000000000002,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.54200000000000004},{"interface":"front","average":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.81000000000000005,"5min":0.81000000000000005,"15min":0.81000000000000005},"last":0.39400000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:12:06 2023","interfaces":[{"interface":"back","average":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.72299999999999998,"5min":0.72299999999999998,"15min":0.72299999999999998},"last":0.49299999999999999},{"interface":"front","average":{"1min":0.49199999999999999,"5min":0.49199999999999999,"15min":0.49199999999999999},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.73099999999999998,"5min":0.73099999999999998,"15min":0.73099999999999998},"last":0.46999999999999997}]},{"osd":5,"last update":"Sat Sep 23 20:12:06 2023","interfaces":[{"interface":"back","average":{"1min":0.48499999999999999,"5min":0.48499999999999999,"15min":0.48499999999999999},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.69399999999999995,"5min":0.69399999999999995,"15min":0.69399999999999995},"last":0.45000000000000001},{"interface":"front","average":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.68400000000000005,"5min":0.68400000000000005,"15min":0.68400000000000005},"last":0.51300000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:12:26 2023","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.74399999999999999,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.53100000000000003},{"interface":"front","average":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.75,"5min":0.75,"15min":0.75},"last":0.60399999999999998}]}]},{"osd":1,"up_from":13,"seq":55834574888,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6524,"kb_used_data":628,"kb_used_omap":0,"kb_used_meta":5888,"kb_avail":93738628,"statfs":{"total":95995035648,"available":95988355072,"internally_reserved":0,"allocated":643072,"data_stored":267209,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6029312},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:12:27 2023","interfaces":[{"interface":"back","average":{"1min":1.3380000000000001,"5min":0.55900000000000005,"15min":0.41699999999999998},"min":{"1min":0.23499999999999999,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":17.280000000000001,"5min":17.280000000000001,"15min":17.280000000000001},"last":0.28499999999999998},{"interface":"front","average":{"1min":0.41099999999999998,"5min":0.39300000000000002,"15min":0.36699999999999999},"min":{"1min":0.28100000000000003,"5min":0.247,"15min":0.247},"max":{"1min":0.98999999999999999,"5min":0.98999999999999999,"15min":0.98999999999999999},"last":0.308}]},{"osd":2,"last update":"Sat Sep 23 20:11:53 2023","interfaces":[{"interface":"back","average":{"1min":0.45800000000000002,"5min":0.44400000000000001,"15min":0.442},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":0.67100000000000004,"5min":0.67100000000000004,"15min":0.67100000000000004},"last":0.35299999999999998},{"interface":"front","average":{"1min":0.42499999999999999,"5min":0.42099999999999999,"15min":0.41999999999999998},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.68799999999999994,"5min":0.68799999999999994,"15min":0.68799999999999994},"last":0.32400000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:12:09 2023","interfaces":[{"interface":"back","average":{"1min":0.46500000000000002,"5min":0.50900000000000001,"15min":0.51600000000000001},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.67100000000000004,"5min":0.67100000000000004,"15min":0.67100000000000004},"last":0.38100000000000001},{"interface":"front","average":{"1min":0.44800000000000001,"5min":0.496,"15min":0.504},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.625,"5min":0.65400000000000003,"15min":0.65400000000000003},"last":0.36699999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:12:27 2023","interfaces":[{"interface":"back","average":{"1min":0.58499999999999996,"5min":0.67300000000000004,"15min":0.68799999999999994},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":0.79700000000000004,"5min":2.226,"15min":2.226},"last":0.41199999999999998},{"interface":"front","average":{"1min":1.288,"5min":0.74299999999999999,"15min":0.65200000000000002},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":12.914999999999999,"5min":12.914999999999999,"15min":12.914999999999999},"last":0.46100000000000002}]},{"osd":5,"last update":"Sat Sep 23 20:11:48 2023","interfaces":[{"interface":"back","average":{"1min":0.63400000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"min":{"1min":0.45300000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"max":{"1min":0.75600000000000001,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.47399999999999998},{"interface":"front","average":{"1min":0.61699999999999999,"5min":0.61699999999999999,"15min":0.61699999999999999},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":0.89200000000000002,"5min":0.89200000000000002,"15min":0.89200000000000002},"last":0.441}]},{"osd":6,"last update":"Sat Sep 23 20:12:04 2023","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"min":{"1min":0.48299999999999998,"5min":0.48299999999999998,"15min":0.48299999999999998},"max":{"1min":0.82999999999999996,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.495},{"interface":"front","average":{"1min":0.73199999999999998,"5min":0.73199999999999998,"15min":0.73199999999999998},"min":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":2.2069999999999999,"5min":2.2069999999999999,"15min":2.2069999999999999},"last":0.40400000000000003}]},{"osd":7,"last update":"Sat Sep 23 20:12:27 2023","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"min":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":0.90700000000000003,"5min":0.90700000000000003,"15min":0.90700000000000003},"last":0.48299999999999998},{"interface":"front","average":{"1min":1.29,"5min":1.29,"15min":1.29},"min":{"1min":0.51000000000000001,"5min":0.51000000000000001,"15min":0.51000000000000001},"max":{"1min":12.878,"5min":12.878,"15min":12.878},"last":0.42499999999999999}]}]},{"osd":0,"up_from":8,"seq":34359738411,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":7168,"kb_used_data":1208,"kb_used_omap":0,"kb_used_meta":5952,"kb_avail":93737984,"statfs":{"total":95995035648,"available":95987695616,"internally_reserved":0,"allocated":1236992,"data_stored":857577,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6094848},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:11:28 2023","interfaces":[{"interface":"back","average":{"1min":0.39300000000000002,"5min":0.34100000000000003,"15min":0.33300000000000002},"min":{"1min":0.26600000000000001,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":0.67400000000000004,"5min":0.67400000000000004,"15min":0.67400000000000004},"last":0.29499999999999998},{"interface":"front","average":{"1min":0.42699999999999999,"5min":0.35899999999999999,"15min":0.34699999999999998},"min":{"1min":0.29999999999999999,"5min":0.217,"15min":0.217},"max":{"1min":0.56899999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"last":0.36099999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:11:49 2023","interfaces":[{"interface":"back","average":{"1min":0.439,"5min":0.42699999999999999,"15min":0.42499999999999999},"min":{"1min":0.32600000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"last":0.312},{"interface":"front","average":{"1min":0.45300000000000001,"5min":0.44500000000000001,"15min":0.443},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.76400000000000001,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.34000000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:12:11 2023","interfaces":[{"interface":"back","average":{"1min":0.40100000000000002,"5min":0.441,"15min":0.44800000000000001},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.55700000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"last":0.33000000000000002},{"interface":"front","average":{"1min":0.44700000000000001,"5min":0.48599999999999999,"15min":0.49199999999999999},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.58099999999999996,"5min":0.60299999999999998,"15min":0.60299999999999998},"last":0.39900000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:11:28 2023","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":0.85299999999999998,"5min":0.85299999999999998,"15min":0.85299999999999998},"last":0.57199999999999995},{"interface":"front","average":{"1min":0.59899999999999998,"5min":0.59899999999999998,"15min":0.59899999999999998},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.80300000000000005,"5min":0.80300000000000005,"15min":0.80300000000000005},"last":0.438}]},{"osd":5,"last update":"Sat Sep 23 20:11:49 2023","interfaces":[{"interface":"back","average":{"1min":0.63100000000000001,"5min":0.63100000000000001,"15min":0.63100000000000001},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.88400000000000001,"5min":0.88400000000000001,"15min":0.88400000000000001},"last":0.49199999999999999},{"interface":"front","average":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.93300000000000005,"5min":0.93300000000000005,"15min":0.93300000000000005},"last":0.503}]},{"osd":6,"last update":"Sat Sep 23 20:12:06 2023","interfaces":[{"interface":"back","average":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"min":{"1min":0.45300000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"max":{"1min":0.80800000000000005,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.51200000000000001},{"interface":"front","average":{"1min":0.61299999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.85699999999999998,"5min":0.85699999999999998,"15min":0.85699999999999998},"last":0.58199999999999996}]},{"osd":7,"last update":"Sat Sep 23 20:12:28 2023","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.89100000000000001,"5min":0.89100000000000001,"15min":0.89100000000000001},"last":0.59199999999999997},{"interface":"front","average":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.91700000000000004,"5min":0.91700000000000004,"15min":0.91700000000000004},"last":0.53400000000000003}]}]},{"osd":2,"up_from":18,"seq":77309411364,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6396,"kb_used_data":628,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738756,"statfs":{"total":95995035648,"available":95988486144,"internally_reserved":0,"allocated":643072,"data_stored":267209,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:11:47 2023","interfaces":[{"interface":"back","average":{"1min":0.41599999999999998,"5min":0.36699999999999999,"15min":0.35899999999999999},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.628,"5min":0.628,"15min":0.628},"last":0.372},{"interface":"front","average":{"1min":0.42999999999999999,"5min":0.38400000000000001,"15min":0.376},"min":{"1min":0.29699999999999999,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"last":0.39000000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:11:47 2023","interfaces":[{"interface":"back","average":{"1min":0.434,"5min":0.40999999999999998,"15min":0.40600000000000003},"min":{"1min":0.312,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"last":0.41599999999999998},{"interface":"front","average":{"1min":0.48799999999999999,"5min":0.42399999999999999,"15min":0.41299999999999998},"min":{"1min":0.35799999999999998,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.75800000000000001,"5min":0.75800000000000001,"15min":0.75800000000000001},"last":0.40699999999999997}]},{"osd":3,"last update":"Sat Sep 23 20:12:09 2023","interfaces":[{"interface":"back","average":{"1min":0.46500000000000002,"5min":0.496,"15min":0.501},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.66600000000000004,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":0.49299999999999999},{"interface":"front","average":{"1min":0.48899999999999999,"5min":0.51700000000000002,"15min":0.52200000000000002},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":0.71299999999999997,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.435}]},{"osd":4,"last update":"Sat Sep 23 20:11:26 2023","interfaces":[{"interface":"back","average":{"1min":0.60499999999999998,"5min":0.60499999999999998,"15min":0.60499999999999998},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.84099999999999997,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.47399999999999998},{"interface":"front","average":{"1min":0.65100000000000002,"5min":0.65100000000000002,"15min":0.65100000000000002},"min":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"max":{"1min":0.93200000000000005,"5min":0.93200000000000005,"15min":0.93200000000000005},"last":0.59299999999999997}]},{"osd":5,"last update":"Sat Sep 23 20:11:47 2023","interfaces":[{"interface":"back","average":{"1min":0.66700000000000004,"5min":0.66700000000000004,"15min":0.66700000000000004},"min":{"1min":0.47299999999999998,"5min":0.47299999999999998,"15min":0.47299999999999998},"max":{"1min":0.98599999999999999,"5min":0.98599999999999999,"15min":0.98599999999999999},"last":0.65700000000000003},{"interface":"front","average":{"1min":0.68300000000000005,"5min":0.68300000000000005,"15min":0.68300000000000005},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":1.0409999999999999,"5min":1.0409999999999999,"15min":1.0409999999999999},"last":0.68500000000000005}]},{"osd":6,"last update":"Sat Sep 23 20:12:04 2023","interfaces":[{"interface":"back","average":{"1min":0.67400000000000004,"5min":0.67400000000000004,"15min":0.67400000000000004},"min":{"1min":0.47099999999999997,"5min":0.47099999999999997,"15min":0.47099999999999997},"max":{"1min":0.96199999999999997,"5min":0.96199999999999997,"15min":0.96199999999999997},"last":0.50600000000000001},{"interface":"front","average":{"1min":0.72199999999999998,"5min":0.72199999999999998,"15min":0.72199999999999998},"min":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":0.91700000000000004,"5min":0.91700000000000004,"15min":0.91700000000000004},"last":0.61499999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:12:25 2023","interfaces":[{"interface":"back","average":{"1min":0.68799999999999994,"5min":0.68799999999999994,"15min":0.68799999999999994},"min":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"max":{"1min":0.97399999999999998,"5min":0.97399999999999998,"15min":0.97399999999999998},"last":0.624},{"interface":"front","average":{"1min":0.66700000000000004,"5min":0.66700000000000004,"15min":0.66700000000000004},"min":{"1min":0.46200000000000002,"5min":0.46200000000000002,"15min":0.46200000000000002},"max":{"1min":0.99199999999999999,"5min":0.99199999999999999,"15min":0.99199999999999999},"last":0.52900000000000003}]}]},{"osd":3,"up_from":25,"seq":107374182430,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5884,"kb_used_data":628,"kb_used_omap":0,"kb_used_meta":5248,"kb_avail":93739268,"statfs":{"total":95995035648,"available":95989010432,"internally_reserved":0,"allocated":643072,"data_stored":267209,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5373952},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:12:06 2023","interfaces":[{"interface":"back","average":{"1min":0.39900000000000002,"5min":0.38300000000000001,"15min":0.38},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"last":0.26200000000000001},{"interface":"front","average":{"1min":0.37,"5min":0.40400000000000003,"15min":0.40999999999999998},"min":{"1min":0.26700000000000002,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.52700000000000002,"5min":0.63100000000000001,"15min":0.63100000000000001},"last":0.31900000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:12:06 2023","interfaces":[{"interface":"back","average":{"1min":0.441,"5min":0.437,"15min":0.437},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.97399999999999998,"5min":0.97399999999999998,"15min":0.97399999999999998},"last":0.29399999999999998},{"interface":"front","average":{"1min":0.495,"5min":0.48099999999999998,"15min":0.47899999999999998},"min":{"1min":0.315,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.995,"5min":0.995,"15min":0.995},"last":0.33500000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:12:06 2023","interfaces":[{"interface":"back","average":{"1min":0.42299999999999999,"5min":0.45300000000000001,"15min":0.45800000000000002},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.64300000000000002,"5min":0.64300000000000002,"15min":0.64300000000000002},"last":0.308},{"interface":"front","average":{"1min":0.434,"5min":0.47099999999999997,"15min":0.47699999999999998},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.63,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":0.28000000000000003}]},{"osd":4,"last update":"Sat Sep 23 20:12:25 2023","interfaces":[{"interface":"back","average":{"1min":1.01,"5min":0.71799999999999997,"15min":0.66900000000000004},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":8.2949999999999999,"5min":8.2949999999999999,"15min":8.2949999999999999},"last":0.63200000000000001},{"interface":"front","average":{"1min":1.073,"5min":0.73299999999999998,"15min":0.67700000000000005},"min":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"max":{"1min":9.8539999999999992,"5min":9.8539999999999992,"15min":9.8539999999999992},"last":0.67000000000000004}]},{"osd":5,"last update":"Sat Sep 23 20:11:47 2023","interfaces":[{"interface":"back","average":{"1min":0.61099999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.93600000000000005,"5min":0.93600000000000005,"15min":0.93600000000000005},"last":0.58199999999999996},{"interface":"front","average":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.86399999999999999,"5min":0.86399999999999999,"15min":0.86399999999999999},"last":0.54500000000000004}]},{"osd":6,"last update":"Sat Sep 23 20:12:06 2023","interfaces":[{"interface":"back","average":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.92300000000000004,"5min":0.92300000000000004,"15min":0.92300000000000004},"last":0.69599999999999995},{"interface":"front","average":{"1min":0.63800000000000001,"5min":0.63800000000000001,"15min":0.63800000000000001},"min":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":1.0049999999999999,"5min":1.0049999999999999,"15min":1.0049999999999999},"last":0.65800000000000003}]},{"osd":7,"last update":"Sat Sep 23 20:12:25 2023","interfaces":[{"interface":"back","average":{"1min":1.105,"5min":1.105,"15min":1.105},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":10.846,"5min":10.846,"15min":10.846},"last":0.55700000000000005},{"interface":"front","average":{"1min":1.0049999999999999,"5min":1.0049999999999999,"15min":1.0049999999999999},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":8.2170000000000005,"5min":8.2170000000000005,"15min":8.2170000000000005},"last":0.68200000000000005}]}]},{"osd":4,"up_from":30,"seq":128849018907,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5944,"kb_used_data":624,"kb_used_omap":0,"kb_used_meta":5312,"kb_avail":93739208,"statfs":{"total":95995035648,"available":95988948992,"internally_reserved":0,"allocated":638976,"data_stored":266886,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5439488},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:12:26 2023","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.54000000000000004,"15min":0.54100000000000004},"min":{"1min":0.34599999999999997,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.68300000000000005,"5min":0.83599999999999997,"15min":0.83599999999999997},"last":0.61599999999999999},{"interface":"front","average":{"1min":0.51600000000000001,"5min":0.54600000000000004,"15min":0.55100000000000005},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.78700000000000003,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.56599999999999995}]},{"osd":1,"last update":"Sat Sep 23 20:12:26 2023","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.56200000000000006,"15min":0.56999999999999995},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.72999999999999998,"5min":0.86499999999999999,"15min":0.86499999999999999},"last":0.504},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.55300000000000005,"15min":0.55700000000000005},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.70999999999999996,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.57599999999999996}]},{"osd":2,"last update":"Sat Sep 23 20:12:26 2023","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.56999999999999995,"15min":0.57699999999999996},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.77200000000000002,"5min":0.81599999999999995,"15min":0.81599999999999995},"last":0.51500000000000001},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.57899999999999996,"15min":0.58099999999999996},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.86199999999999999,"5min":0.91500000000000004,"15min":0.91500000000000004},"last":0.60199999999999998}]},{"osd":3,"last update":"Sat Sep 23 20:12:26 2023","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.56399999999999995,"15min":0.56999999999999995},"min":{"1min":0.36099999999999999,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.69599999999999995,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.495},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.55000000000000004,"15min":0.54700000000000004},"min":{"1min":0.40200000000000002,"5min":0.376,"15min":0.376},"max":{"1min":0.77100000000000002,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.59099999999999997}]},{"osd":5,"last update":"Sat Sep 23 20:11:45 2023","interfaces":[{"interface":"back","average":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"last":0.42499999999999999},{"interface":"front","average":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":0.67300000000000004,"5min":0.67300000000000004,"15min":0.67300000000000004},"last":0.41099999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:12:07 2023","interfaces":[{"interface":"back","average":{"1min":0.434,"5min":0.434,"15min":0.434},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.44800000000000001},{"interface":"front","average":{"1min":0.45500000000000002,"5min":0.45500000000000002,"15min":0.45500000000000002},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.69899999999999995,"5min":0.69899999999999995,"15min":0.69899999999999995},"last":0.46600000000000003}]},{"osd":7,"last update":"Sat Sep 23 20:12:26 2023","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.90400000000000003,"5min":0.90400000000000003,"15min":0.90400000000000003},"last":0.47599999999999998},{"interface":"front","average":{"1min":0.52400000000000002,"5min":0.52400000000000002,"15min":0.52400000000000002},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.77400000000000002,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.55400000000000005}]}]},{"osd":5,"up_from":39,"seq":167503724568,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5820,"kb_used_data":628,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93739332,"statfs":{"total":95995035648,"available":95989075968,"internally_reserved":0,"allocated":643072,"data_stored":267209,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:11:45 2023","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":1.0640000000000001,"5min":1.0640000000000001,"15min":1.0640000000000001},"last":0.502},{"interface":"front","average":{"1min":0.51300000000000001,"5min":0.51300000000000001,"15min":0.51300000000000001},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.73699999999999999,"5min":0.73699999999999999,"15min":0.73699999999999999},"last":0.39200000000000002}]},{"osd":1,"last update":"Sat Sep 23 20:11:45 2023","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.69199999999999995,"5min":0.69199999999999995,"15min":0.69199999999999995},"last":0.41799999999999998},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":1.1519999999999999,"5min":1.1519999999999999,"15min":1.1519999999999999},"last":0.42999999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:11:45 2023","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":1.1719999999999999,"5min":1.1719999999999999,"15min":1.1719999999999999},"last":0.49299999999999999},{"interface":"front","average":{"1min":0.54300000000000004,"5min":0.54300000000000004,"15min":0.54300000000000004},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.375}]},{"osd":3,"last update":"Sat Sep 23 20:11:45 2023","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":1.2130000000000001,"5min":1.2130000000000001,"15min":1.2130000000000001},"last":0.443},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.52300000000000002,"15min":0.52300000000000002},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.752,"5min":0.752,"15min":0.752},"last":0.48199999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:11:45 2023","interfaces":[{"interface":"back","average":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.52900000000000003,"5min":0.52900000000000003,"15min":0.52900000000000003},"last":0.33000000000000002},{"interface":"front","average":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":1.4450000000000001,"5min":1.4450000000000001,"15min":1.4450000000000001},"last":0.40500000000000003}]},{"osd":6,"last update":"Sat Sep 23 20:12:05 2023","interfaces":[{"interface":"back","average":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.627,"5min":0.627,"15min":0.627},"last":0.46400000000000002},{"interface":"front","average":{"1min":0.52700000000000002,"5min":0.52700000000000002,"15min":0.52700000000000002},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.73699999999999999,"5min":0.73699999999999999,"15min":0.73699999999999999},"last":0.47299999999999998}]},{"osd":7,"last update":"Sat Sep 23 20:12:25 2023","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.68899999999999995,"5min":0.68899999999999995,"15min":0.68899999999999995},"last":0.35799999999999998},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.71699999999999997,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.45300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:12:33.497 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:12:34.108 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:12:34.216 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:12:33 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[102880]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:12:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T20:12:34.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:34 smithi018 ceph-mon[95889]: from='client.14628 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:12:34.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:34 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:34.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:34 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:34.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:34 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:12:34.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:34 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:12:34.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:34 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:34.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:34 smithi018 ceph-mon[95889]: pgmap v29: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:12:34.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:34 smithi018 ceph-mon[95889]: Reconfiguring alertmanager.a (dependencies changed)... 2023-09-23T20:12:34.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:34 smithi018 ceph-mon[95889]: Reconfiguring daemon alertmanager.a on smithi018 2023-09-23T20:12:34.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:34 smithi018 ceph-mon[109137]: from='client.14628 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:12:34.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:34 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:34.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:34 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:34.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:34 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:12:34.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:34 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:12:34.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:34 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:34.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:34 smithi018 ceph-mon[109137]: pgmap v29: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:12:34.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:34 smithi018 ceph-mon[109137]: Reconfiguring alertmanager.a (dependencies changed)... 2023-09-23T20:12:34.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:34 smithi018 ceph-mon[109137]: Reconfiguring daemon alertmanager.a on smithi018 2023-09-23T20:12:34.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:34 smithi149 ceph-mon[100764]: from='client.14628 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:12:34.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:34 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:34.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:34 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:34.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:34 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:12:34.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:34 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:12:34.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:34 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:34.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:34 smithi149 ceph-mon[100764]: pgmap v29: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:12:34.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:34 smithi149 ceph-mon[100764]: Reconfiguring alertmanager.a (dependencies changed)... 2023-09-23T20:12:34.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:34 smithi149 ceph-mon[100764]: Reconfiguring daemon alertmanager.a on smithi018 2023-09-23T20:12:35.838 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:35 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:12:35.839 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:35 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:12:35.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:35 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:12:36.240 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:12:36.242 INFO:teuthology.orchestra.run.smithi018.stderr:dumped all 2023-09-23T20:12:36.541 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 23 20:12:36 smithi018 systemd[1]: Stopping Ceph alertmanager.a for a3b3b17e-5a4c-11ee-8db4-212e2dc638e7... 2023-09-23T20:12:36.675 INFO:teuthology.orchestra.run.smithi018.stdout:{"pg_ready":true,"pg_map":{"version":30,"stamp":"2023-09-23T20:12:35.672732+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":50472,"kb_used_data":6760,"kb_used_omap":0,"kb_used_meta":43648,"kb_avail":749910744,"statfs":{"total":767960285184,"available":767908601856,"internally_reserved":0,"allocated":6922240,"data_stored":3908453,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":44695552},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001994"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":10,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-09-23T20:12:00.851079+0000","last_change":"2023-09-23T20:12:00.850675+0000","last_active":"2023-09-23T20:12:00.851079+0000","last_peered":"2023-09-23T20:12:00.851079+0000","last_clean":"2023-09-23T20:12:00.851079+0000","last_became_active":"2023-09-23T20:12:00.850315+0000","last_became_peered":"2023-09-23T20:12:00.850315+0000","last_unstale":"2023-09-23T20:12:00.851079+0000","last_undegraded":"2023-09-23T20:12:00.851079+0000","last_fullsized":"2023-09-23T20:12:00.851079+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:03:33.446629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2023-09-23T20:12:01.854874+0000","last_change":"2023-09-23T20:12:01.062560+0000","last_active":"2023-09-23T20:12:01.854874+0000","last_peered":"2023-09-23T20:12:01.854874+0000","last_clean":"2023-09-23T20:12:01.854874+0000","last_became_active":"2023-09-23T20:12:01.062216+0000","last_became_peered":"2023-09-23T20:12:01.062216+0000","last_unstale":"2023-09-23T20:12:01.854874+0000","last_undegraded":"2023-09-23T20:12:01.854874+0000","last_fullsized":"2023-09-23T20:12:01.854874+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:01:52.795311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":10,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-09-23T20:12:01.062834+0000","last_change":"2023-09-23T20:12:01.062442+0000","last_active":"2023-09-23T20:12:01.062834+0000","last_peered":"2023-09-23T20:12:01.062834+0000","last_clean":"2023-09-23T20:12:01.062834+0000","last_became_active":"2023-09-23T20:12:01.062047+0000","last_became_peered":"2023-09-23T20:12:01.062047+0000","last_unstale":"2023-09-23T20:12:01.062834+0000","last_undegraded":"2023-09-23T20:12:01.062834+0000","last_fullsized":"2023-09-23T20:12:01.062834+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:58:49.032117+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":10,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-09-23T20:12:00.849374+0000","last_change":"2023-09-23T20:12:00.848951+0000","last_active":"2023-09-23T20:12:00.849374+0000","last_peered":"2023-09-23T20:12:00.849374+0000","last_clean":"2023-09-23T20:12:00.849374+0000","last_became_active":"2023-09-23T20:12:00.848720+0000","last_became_peered":"2023-09-23T20:12:00.848720+0000","last_unstale":"2023-09-23T20:12:00.849374+0000","last_undegraded":"2023-09-23T20:12:00.849374+0000","last_fullsized":"2023-09-23T20:12:00.849374+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:45:24.720986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"60'128","reported_seq":272,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-09-23T20:12:00.846751+0000","last_change":"2023-09-23T20:11:27.849030+0000","last_active":"2023-09-23T20:12:00.846751+0000","last_peered":"2023-09-23T20:12:00.846751+0000","last_clean":"2023-09-23T20:12:00.846751+0000","last_became_active":"2023-09-23T20:11:27.848678+0000","last_became_peered":"2023-09-23T20:11:27.848678+0000","last_unstale":"2023-09-23T20:12:00.846751+0000","last_undegraded":"2023-09-23T20:12:00.846751+0000","last_fullsized":"2023-09-23T20:12:00.846751+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:09:47.213130+0000","objects_scrubbed":0,"log_size":128,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:33:53.621689+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":10,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-09-23T20:12:00.852221+0000","last_change":"2023-09-23T20:12:00.851792+0000","last_active":"2023-09-23T20:12:00.852221+0000","last_peered":"2023-09-23T20:12:00.852221+0000","last_clean":"2023-09-23T20:12:00.852221+0000","last_became_active":"2023-09-23T20:12:00.851408+0000","last_became_peered":"2023-09-23T20:12:00.851408+0000","last_unstale":"2023-09-23T20:12:00.852221+0000","last_undegraded":"2023-09-23T20:12:00.852221+0000","last_fullsized":"2023-09-23T20:12:00.852221+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:19:26.564015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":10,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-09-23T20:12:01.011265+0000","last_change":"2023-09-23T20:12:01.010922+0000","last_active":"2023-09-23T20:12:01.011265+0000","last_peered":"2023-09-23T20:12:01.011265+0000","last_clean":"2023-09-23T20:12:01.011265+0000","last_became_active":"2023-09-23T20:12:01.010591+0000","last_became_peered":"2023-09-23T20:12:01.010591+0000","last_unstale":"2023-09-23T20:12:01.011265+0000","last_undegraded":"2023-09-23T20:12:01.011265+0000","last_fullsized":"2023-09-23T20:12:01.011265+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:11:25.451578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":10,"reported_epoch":62,"state":"active+clean","last_fresh":"2023-09-23T20:12:00.852173+0000","last_change":"2023-09-23T20:12:00.851594+0000","last_active":"2023-09-23T20:12:00.852173+0000","last_peered":"2023-09-23T20:12:00.852173+0000","last_clean":"2023-09-23T20:12:00.852173+0000","last_became_active":"2023-09-23T20:12:00.851256+0000","last_became_peered":"2023-09-23T20:12:00.851256+0000","last_unstale":"2023-09-23T20:12:00.852173+0000","last_undegraded":"2023-09-23T20:12:00.852173+0000","last_fullsized":"2023-09-23T20:12:00.852173+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:12:00.121999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2023-09-23T20:12:01.854802+0000","last_change":"2023-09-23T20:12:00.847882+0000","last_active":"2023-09-23T20:12:01.854802+0000","last_peered":"2023-09-23T20:12:01.854802+0000","last_clean":"2023-09-23T20:12:01.854802+0000","last_became_active":"2023-09-23T20:12:00.847511+0000","last_became_peered":"2023-09-23T20:12:00.847511+0000","last_unstale":"2023-09-23T20:12:01.854802+0000","last_undegraded":"2023-09-23T20:12:01.854802+0000","last_fullsized":"2023-09-23T20:12:01.854802+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:52:12.888646+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201296,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6336,"kb_used_data":1208,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93738816,"statfs":{"total":95995035648,"available":95988547584,"internally_reserved":0,"allocated":1236992,"data_stored":857577,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:12:25 2023","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":0.52300000000000002,"15min":0.52300000000000002},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.74299999999999999,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.35399999999999998},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":1.004,"5min":1.004,"15min":1.004},"last":0.42299999999999999}]},{"osd":1,"last update":"Sat Sep 23 20:12:25 2023","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.86799999999999999,"5min":0.86799999999999999,"15min":0.86799999999999999},"last":0.39800000000000002},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":1.0309999999999999,"5min":1.0309999999999999,"15min":1.0309999999999999},"last":0.48599999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:12:25 2023","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.98999999999999999,"5min":0.98999999999999999,"15min":0.98999999999999999},"last":0.44900000000000001},{"interface":"front","average":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.92400000000000004,"5min":0.92400000000000004,"15min":0.92400000000000004},"last":0.51400000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:12:25 2023","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.88800000000000001,"5min":0.88800000000000001,"15min":0.88800000000000001},"last":0.53100000000000003},{"interface":"front","average":{"1min":0.61899999999999999,"5min":0.61899999999999999,"15min":0.61899999999999999},"min":{"1min":0.45300000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"max":{"1min":0.97899999999999998,"5min":0.97899999999999998,"15min":0.97899999999999998},"last":0.56899999999999995}]},{"osd":4,"last update":"Sat Sep 23 20:12:25 2023","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":0.52400000000000002,"15min":0.52400000000000002},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.94099999999999995,"5min":0.94099999999999995,"15min":0.94099999999999995},"last":0.54100000000000004},{"interface":"front","average":{"1min":0.54200000000000004,"5min":0.54200000000000004,"15min":0.54200000000000004},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.80700000000000005,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.504}]},{"osd":5,"last update":"Sat Sep 23 20:12:25 2023","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.85899999999999999,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.46700000000000003},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":0.97699999999999998,"5min":0.97699999999999998,"15min":0.97699999999999998},"last":0.59999999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:12:25 2023","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":1.0569999999999999,"5min":1.0569999999999999,"15min":1.0569999999999999},"last":0.63200000000000001},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.879,"5min":0.879,"15min":0.879},"last":0.55700000000000005}]}]},{"osd":6,"up_from":47,"seq":201863462932,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6400,"kb_used_data":1208,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93738752,"statfs":{"total":95995035648,"available":95988482048,"internally_reserved":0,"allocated":1236992,"data_stored":857577,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:12:06 2023","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.77300000000000002,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.55000000000000004},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.78400000000000003,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.49299999999999999}]},{"osd":1,"last update":"Sat Sep 23 20:12:06 2023","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.73999999999999999,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.52700000000000002},{"interface":"front","average":{"1min":0.53700000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.72499999999999998,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.51200000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:12:06 2023","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.55100000000000005,"15min":0.55100000000000005},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.70699999999999996,"5min":0.70699999999999996,"15min":0.70699999999999996},"last":0.60799999999999998},{"interface":"front","average":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.68999999999999995,"5min":0.68999999999999995,"15min":0.68999999999999995},"last":0.44500000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:12:06 2023","interfaces":[{"interface":"back","average":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.77200000000000002,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.58299999999999996},{"interface":"front","average":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.81000000000000005,"5min":0.81000000000000005,"15min":0.81000000000000005},"last":0.47399999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:12:06 2023","interfaces":[{"interface":"back","average":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.72299999999999998,"5min":0.72299999999999998,"15min":0.72299999999999998},"last":0.52100000000000002},{"interface":"front","average":{"1min":0.49199999999999999,"5min":0.49199999999999999,"15min":0.49199999999999999},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.73099999999999998,"5min":0.73099999999999998,"15min":0.73099999999999998},"last":0.623}]},{"osd":5,"last update":"Sat Sep 23 20:12:06 2023","interfaces":[{"interface":"back","average":{"1min":0.48499999999999999,"5min":0.48499999999999999,"15min":0.48499999999999999},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.69399999999999995,"5min":0.69399999999999995,"15min":0.69399999999999995},"last":0.57099999999999995},{"interface":"front","average":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.68400000000000005,"5min":0.68400000000000005,"15min":0.68400000000000005},"last":0.65000000000000002}]},{"osd":7,"last update":"Sat Sep 23 20:12:26 2023","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.74399999999999999,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.59699999999999998},{"interface":"front","average":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.75,"5min":0.75,"15min":0.75},"last":0.78400000000000003}]}]},{"osd":1,"up_from":13,"seq":55834574889,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6524,"kb_used_data":628,"kb_used_omap":0,"kb_used_meta":5888,"kb_avail":93738628,"statfs":{"total":95995035648,"available":95988355072,"internally_reserved":0,"allocated":643072,"data_stored":267209,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6029312},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:12:27 2023","interfaces":[{"interface":"back","average":{"1min":1.3380000000000001,"5min":0.55900000000000005,"15min":0.41699999999999998},"min":{"1min":0.23499999999999999,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":17.280000000000001,"5min":17.280000000000001,"15min":17.280000000000001},"last":0.30199999999999999},{"interface":"front","average":{"1min":0.41099999999999998,"5min":0.39300000000000002,"15min":0.36699999999999999},"min":{"1min":0.28100000000000003,"5min":0.247,"15min":0.247},"max":{"1min":0.98999999999999999,"5min":0.98999999999999999,"15min":0.98999999999999999},"last":0.41899999999999998}]},{"osd":2,"last update":"Sat Sep 23 20:11:53 2023","interfaces":[{"interface":"back","average":{"1min":0.45800000000000002,"5min":0.44400000000000001,"15min":0.442},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":0.67100000000000004,"5min":0.67100000000000004,"15min":0.67100000000000004},"last":0.34499999999999997},{"interface":"front","average":{"1min":0.42499999999999999,"5min":0.42099999999999999,"15min":0.41999999999999998},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.68799999999999994,"5min":0.68799999999999994,"15min":0.68799999999999994},"last":0.45400000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:12:09 2023","interfaces":[{"interface":"back","average":{"1min":0.46500000000000002,"5min":0.50900000000000001,"15min":0.51600000000000001},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.67100000000000004,"5min":0.67100000000000004,"15min":0.67100000000000004},"last":0.375},{"interface":"front","average":{"1min":0.44800000000000001,"5min":0.496,"15min":0.504},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.625,"5min":0.65400000000000003,"15min":0.65400000000000003},"last":0.501}]},{"osd":4,"last update":"Sat Sep 23 20:12:27 2023","interfaces":[{"interface":"back","average":{"1min":0.58499999999999996,"5min":0.67300000000000004,"15min":0.68799999999999994},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":0.79700000000000004,"5min":2.226,"15min":2.226},"last":0.71899999999999997},{"interface":"front","average":{"1min":1.288,"5min":0.74299999999999999,"15min":0.65200000000000002},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":12.914999999999999,"5min":12.914999999999999,"15min":12.914999999999999},"last":0.57399999999999995}]},{"osd":5,"last update":"Sat Sep 23 20:11:48 2023","interfaces":[{"interface":"back","average":{"1min":0.63400000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"min":{"1min":0.45300000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"max":{"1min":0.75600000000000001,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.626},{"interface":"front","average":{"1min":0.61699999999999999,"5min":0.61699999999999999,"15min":0.61699999999999999},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":0.89200000000000002,"5min":0.89200000000000002,"15min":0.89200000000000002},"last":0.755}]},{"osd":6,"last update":"Sat Sep 23 20:12:04 2023","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"min":{"1min":0.48299999999999998,"5min":0.48299999999999998,"15min":0.48299999999999998},"max":{"1min":0.82999999999999996,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.77000000000000002},{"interface":"front","average":{"1min":0.73199999999999998,"5min":0.73199999999999998,"15min":0.73199999999999998},"min":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":2.2069999999999999,"5min":2.2069999999999999,"15min":2.2069999999999999},"last":0.90900000000000003}]},{"osd":7,"last update":"Sat Sep 23 20:12:27 2023","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"min":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":0.90700000000000003,"5min":0.90700000000000003,"15min":0.90700000000000003},"last":0.89100000000000001},{"interface":"front","average":{"1min":1.29,"5min":1.29,"15min":1.29},"min":{"1min":0.51000000000000001,"5min":0.51000000000000001,"15min":0.51000000000000001},"max":{"1min":12.878,"5min":12.878,"15min":12.878},"last":0.86899999999999999}]}]},{"osd":0,"up_from":8,"seq":34359738411,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":7168,"kb_used_data":1208,"kb_used_omap":0,"kb_used_meta":5952,"kb_avail":93737984,"statfs":{"total":95995035648,"available":95987695616,"internally_reserved":0,"allocated":1236992,"data_stored":857577,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6094848},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:11:28 2023","interfaces":[{"interface":"back","average":{"1min":0.39300000000000002,"5min":0.34100000000000003,"15min":0.33300000000000002},"min":{"1min":0.26600000000000001,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":0.67400000000000004,"5min":0.67400000000000004,"15min":0.67400000000000004},"last":0.29499999999999998},{"interface":"front","average":{"1min":0.42699999999999999,"5min":0.35899999999999999,"15min":0.34699999999999998},"min":{"1min":0.29999999999999999,"5min":0.217,"15min":0.217},"max":{"1min":0.56899999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"last":0.36099999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:11:49 2023","interfaces":[{"interface":"back","average":{"1min":0.439,"5min":0.42699999999999999,"15min":0.42499999999999999},"min":{"1min":0.32600000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"last":0.312},{"interface":"front","average":{"1min":0.45300000000000001,"5min":0.44500000000000001,"15min":0.443},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.76400000000000001,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.34000000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:12:11 2023","interfaces":[{"interface":"back","average":{"1min":0.40100000000000002,"5min":0.441,"15min":0.44800000000000001},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.55700000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"last":0.33000000000000002},{"interface":"front","average":{"1min":0.44700000000000001,"5min":0.48599999999999999,"15min":0.49199999999999999},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.58099999999999996,"5min":0.60299999999999998,"15min":0.60299999999999998},"last":0.39900000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:11:28 2023","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":0.85299999999999998,"5min":0.85299999999999998,"15min":0.85299999999999998},"last":0.57199999999999995},{"interface":"front","average":{"1min":0.59899999999999998,"5min":0.59899999999999998,"15min":0.59899999999999998},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.80300000000000005,"5min":0.80300000000000005,"15min":0.80300000000000005},"last":0.438}]},{"osd":5,"last update":"Sat Sep 23 20:11:49 2023","interfaces":[{"interface":"back","average":{"1min":0.63100000000000001,"5min":0.63100000000000001,"15min":0.63100000000000001},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.88400000000000001,"5min":0.88400000000000001,"15min":0.88400000000000001},"last":0.49199999999999999},{"interface":"front","average":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.93300000000000005,"5min":0.93300000000000005,"15min":0.93300000000000005},"last":0.503}]},{"osd":6,"last update":"Sat Sep 23 20:12:06 2023","interfaces":[{"interface":"back","average":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"min":{"1min":0.45300000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"max":{"1min":0.80800000000000005,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.51200000000000001},{"interface":"front","average":{"1min":0.61299999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.85699999999999998,"5min":0.85699999999999998,"15min":0.85699999999999998},"last":0.58199999999999996}]},{"osd":7,"last update":"Sat Sep 23 20:12:28 2023","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.89100000000000001,"5min":0.89100000000000001,"15min":0.89100000000000001},"last":0.59199999999999997},{"interface":"front","average":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.91700000000000004,"5min":0.91700000000000004,"15min":0.91700000000000004},"last":0.53400000000000003}]}]},{"osd":2,"up_from":18,"seq":77309411364,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6396,"kb_used_data":628,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738756,"statfs":{"total":95995035648,"available":95988486144,"internally_reserved":0,"allocated":643072,"data_stored":267209,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:11:47 2023","interfaces":[{"interface":"back","average":{"1min":0.41599999999999998,"5min":0.36699999999999999,"15min":0.35899999999999999},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.628,"5min":0.628,"15min":0.628},"last":0.372},{"interface":"front","average":{"1min":0.42999999999999999,"5min":0.38400000000000001,"15min":0.376},"min":{"1min":0.29699999999999999,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"last":0.39000000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:11:47 2023","interfaces":[{"interface":"back","average":{"1min":0.434,"5min":0.40999999999999998,"15min":0.40600000000000003},"min":{"1min":0.312,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"last":0.41599999999999998},{"interface":"front","average":{"1min":0.48799999999999999,"5min":0.42399999999999999,"15min":0.41299999999999998},"min":{"1min":0.35799999999999998,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.75800000000000001,"5min":0.75800000000000001,"15min":0.75800000000000001},"last":0.40699999999999997}]},{"osd":3,"last update":"Sat Sep 23 20:12:09 2023","interfaces":[{"interface":"back","average":{"1min":0.46500000000000002,"5min":0.496,"15min":0.501},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.66600000000000004,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":0.49299999999999999},{"interface":"front","average":{"1min":0.48899999999999999,"5min":0.51700000000000002,"15min":0.52200000000000002},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":0.71299999999999997,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.435}]},{"osd":4,"last update":"Sat Sep 23 20:11:26 2023","interfaces":[{"interface":"back","average":{"1min":0.60499999999999998,"5min":0.60499999999999998,"15min":0.60499999999999998},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.84099999999999997,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.47399999999999998},{"interface":"front","average":{"1min":0.65100000000000002,"5min":0.65100000000000002,"15min":0.65100000000000002},"min":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"max":{"1min":0.93200000000000005,"5min":0.93200000000000005,"15min":0.93200000000000005},"last":0.59299999999999997}]},{"osd":5,"last update":"Sat Sep 23 20:11:47 2023","interfaces":[{"interface":"back","average":{"1min":0.66700000000000004,"5min":0.66700000000000004,"15min":0.66700000000000004},"min":{"1min":0.47299999999999998,"5min":0.47299999999999998,"15min":0.47299999999999998},"max":{"1min":0.98599999999999999,"5min":0.98599999999999999,"15min":0.98599999999999999},"last":0.65700000000000003},{"interface":"front","average":{"1min":0.68300000000000005,"5min":0.68300000000000005,"15min":0.68300000000000005},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":1.0409999999999999,"5min":1.0409999999999999,"15min":1.0409999999999999},"last":0.68500000000000005}]},{"osd":6,"last update":"Sat Sep 23 20:12:04 2023","interfaces":[{"interface":"back","average":{"1min":0.67400000000000004,"5min":0.67400000000000004,"15min":0.67400000000000004},"min":{"1min":0.47099999999999997,"5min":0.47099999999999997,"15min":0.47099999999999997},"max":{"1min":0.96199999999999997,"5min":0.96199999999999997,"15min":0.96199999999999997},"last":0.50600000000000001},{"interface":"front","average":{"1min":0.72199999999999998,"5min":0.72199999999999998,"15min":0.72199999999999998},"min":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":0.91700000000000004,"5min":0.91700000000000004,"15min":0.91700000000000004},"last":0.61499999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:12:25 2023","interfaces":[{"interface":"back","average":{"1min":0.68799999999999994,"5min":0.68799999999999994,"15min":0.68799999999999994},"min":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"max":{"1min":0.97399999999999998,"5min":0.97399999999999998,"15min":0.97399999999999998},"last":0.624},{"interface":"front","average":{"1min":0.66700000000000004,"5min":0.66700000000000004,"15min":0.66700000000000004},"min":{"1min":0.46200000000000002,"5min":0.46200000000000002,"15min":0.46200000000000002},"max":{"1min":0.99199999999999999,"5min":0.99199999999999999,"15min":0.99199999999999999},"last":0.52900000000000003}]}]},{"osd":3,"up_from":25,"seq":107374182431,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5884,"kb_used_data":628,"kb_used_omap":0,"kb_used_meta":5248,"kb_avail":93739268,"statfs":{"total":95995035648,"available":95989010432,"internally_reserved":0,"allocated":643072,"data_stored":267209,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5373952},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:12:06 2023","interfaces":[{"interface":"back","average":{"1min":0.39900000000000002,"5min":0.38300000000000001,"15min":0.38},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"last":0.29399999999999998},{"interface":"front","average":{"1min":0.37,"5min":0.40400000000000003,"15min":0.40999999999999998},"min":{"1min":0.26700000000000002,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.52700000000000002,"5min":0.63100000000000001,"15min":0.63100000000000001},"last":0.40600000000000003}]},{"osd":1,"last update":"Sat Sep 23 20:12:06 2023","interfaces":[{"interface":"back","average":{"1min":0.441,"5min":0.437,"15min":0.437},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.97399999999999998,"5min":0.97399999999999998,"15min":0.97399999999999998},"last":0.37},{"interface":"front","average":{"1min":0.495,"5min":0.48099999999999998,"15min":0.47899999999999998},"min":{"1min":0.315,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.995,"5min":0.995,"15min":0.995},"last":0.42399999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:12:06 2023","interfaces":[{"interface":"back","average":{"1min":0.42299999999999999,"5min":0.45300000000000001,"15min":0.45800000000000002},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.64300000000000002,"5min":0.64300000000000002,"15min":0.64300000000000002},"last":0.34999999999999998},{"interface":"front","average":{"1min":0.434,"5min":0.47099999999999997,"15min":0.47699999999999998},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.63,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":0.33000000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:12:25 2023","interfaces":[{"interface":"back","average":{"1min":1.01,"5min":0.71799999999999997,"15min":0.66900000000000004},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":8.2949999999999999,"5min":8.2949999999999999,"15min":8.2949999999999999},"last":0.61099999999999999},{"interface":"front","average":{"1min":1.073,"5min":0.73299999999999998,"15min":0.67700000000000005},"min":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"max":{"1min":9.8539999999999992,"5min":9.8539999999999992,"15min":9.8539999999999992},"last":0.623}]},{"osd":5,"last update":"Sat Sep 23 20:11:47 2023","interfaces":[{"interface":"back","average":{"1min":0.61099999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.93600000000000005,"5min":0.93600000000000005,"15min":0.93600000000000005},"last":0.55600000000000005},{"interface":"front","average":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.86399999999999999,"5min":0.86399999999999999,"15min":0.86399999999999999},"last":0.442}]},{"osd":6,"last update":"Sat Sep 23 20:12:06 2023","interfaces":[{"interface":"back","average":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.92300000000000004,"5min":0.92300000000000004,"15min":0.92300000000000004},"last":0.46500000000000002},{"interface":"front","average":{"1min":0.63800000000000001,"5min":0.63800000000000001,"15min":0.63800000000000001},"min":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":1.0049999999999999,"5min":1.0049999999999999,"15min":1.0049999999999999},"last":0.52700000000000002}]},{"osd":7,"last update":"Sat Sep 23 20:12:25 2023","interfaces":[{"interface":"back","average":{"1min":1.105,"5min":1.105,"15min":1.105},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":10.846,"5min":10.846,"15min":10.846},"last":0.59599999999999997},{"interface":"front","average":{"1min":1.0049999999999999,"5min":1.0049999999999999,"15min":1.0049999999999999},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":8.2170000000000005,"5min":8.2170000000000005,"15min":8.2170000000000005},"last":0.47299999999999998}]}]},{"osd":4,"up_from":30,"seq":128849018908,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5944,"kb_used_data":624,"kb_used_omap":0,"kb_used_meta":5312,"kb_avail":93739208,"statfs":{"total":95995035648,"available":95988948992,"internally_reserved":0,"allocated":638976,"data_stored":266886,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5439488},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:12:26 2023","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.54000000000000004,"15min":0.54100000000000004},"min":{"1min":0.34599999999999997,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.68300000000000005,"5min":0.83599999999999997,"15min":0.83599999999999997},"last":0.438},{"interface":"front","average":{"1min":0.51600000000000001,"5min":0.54600000000000004,"15min":0.55100000000000005},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.78700000000000003,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.59199999999999997}]},{"osd":1,"last update":"Sat Sep 23 20:12:26 2023","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.56200000000000006,"15min":0.56999999999999995},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.72999999999999998,"5min":0.86499999999999999,"15min":0.86499999999999999},"last":0.56999999999999995},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.55300000000000005,"15min":0.55700000000000005},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.70999999999999996,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.55400000000000005}]},{"osd":2,"last update":"Sat Sep 23 20:12:26 2023","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.56999999999999995,"15min":0.57699999999999996},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.77200000000000002,"5min":0.81599999999999995,"15min":0.81599999999999995},"last":0.46700000000000003},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.57899999999999996,"15min":0.58099999999999996},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.86199999999999999,"5min":0.91500000000000004,"15min":0.91500000000000004},"last":0.64800000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:12:26 2023","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.56399999999999995,"15min":0.56999999999999995},"min":{"1min":0.36099999999999999,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.69599999999999995,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.53700000000000003},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.55000000000000004,"15min":0.54700000000000004},"min":{"1min":0.40200000000000002,"5min":0.376,"15min":0.376},"max":{"1min":0.77100000000000002,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.69299999999999995}]},{"osd":5,"last update":"Sat Sep 23 20:11:45 2023","interfaces":[{"interface":"back","average":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"last":0.626},{"interface":"front","average":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":0.67300000000000004,"5min":0.67300000000000004,"15min":0.67300000000000004},"last":0.36299999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:12:07 2023","interfaces":[{"interface":"back","average":{"1min":0.434,"5min":0.434,"15min":0.434},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.44900000000000001},{"interface":"front","average":{"1min":0.45500000000000002,"5min":0.45500000000000002,"15min":0.45500000000000002},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.69899999999999995,"5min":0.69899999999999995,"15min":0.69899999999999995},"last":0.66400000000000003}]},{"osd":7,"last update":"Sat Sep 23 20:12:26 2023","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.90400000000000003,"5min":0.90400000000000003,"15min":0.90400000000000003},"last":0.495},{"interface":"front","average":{"1min":0.52400000000000002,"5min":0.52400000000000002,"15min":0.52400000000000002},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.77400000000000002,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.71599999999999997}]}]},{"osd":5,"up_from":39,"seq":167503724569,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5820,"kb_used_data":628,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93739332,"statfs":{"total":95995035648,"available":95989075968,"internally_reserved":0,"allocated":643072,"data_stored":267209,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:11:45 2023","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":1.0640000000000001,"5min":1.0640000000000001,"15min":1.0640000000000001},"last":0.39400000000000002},{"interface":"front","average":{"1min":0.51300000000000001,"5min":0.51300000000000001,"15min":0.51300000000000001},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.73699999999999999,"5min":0.73699999999999999,"15min":0.73699999999999999},"last":0.53800000000000003}]},{"osd":1,"last update":"Sat Sep 23 20:11:45 2023","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.69199999999999995,"5min":0.69199999999999995,"15min":0.69199999999999995},"last":0.45200000000000001},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":1.1519999999999999,"5min":1.1519999999999999,"15min":1.1519999999999999},"last":0.438}]},{"osd":2,"last update":"Sat Sep 23 20:11:45 2023","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":1.1719999999999999,"5min":1.1719999999999999,"15min":1.1719999999999999},"last":0.47699999999999998},{"interface":"front","average":{"1min":0.54300000000000004,"5min":0.54300000000000004,"15min":0.54300000000000004},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.56599999999999995}]},{"osd":3,"last update":"Sat Sep 23 20:11:45 2023","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":1.2130000000000001,"5min":1.2130000000000001,"15min":1.2130000000000001},"last":0.55600000000000005},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.52300000000000002,"15min":0.52300000000000002},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.752,"5min":0.752,"15min":0.752},"last":0.60899999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:11:45 2023","interfaces":[{"interface":"back","average":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.52900000000000003,"5min":0.52900000000000003,"15min":0.52900000000000003},"last":0.48899999999999999},{"interface":"front","average":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":1.4450000000000001,"5min":1.4450000000000001,"15min":1.4450000000000001},"last":0.41399999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:12:05 2023","interfaces":[{"interface":"back","average":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.627,"5min":0.627,"15min":0.627},"last":0.57999999999999996},{"interface":"front","average":{"1min":0.52700000000000002,"5min":0.52700000000000002,"15min":0.52700000000000002},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.73699999999999999,"5min":0.73699999999999999,"15min":0.73699999999999999},"last":0.59199999999999997}]},{"osd":7,"last update":"Sat Sep 23 20:12:25 2023","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.68899999999999995,"5min":0.68899999999999995,"15min":0.68899999999999995},"last":0.505},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.71699999999999997,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.51600000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:12:36.677 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2023-09-23T20:12:36.677 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2023-09-23T20:12:36.677 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2023-09-23T20:12:36.678 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph health --format=json 2023-09-23T20:12:36.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:36 smithi018 ceph-mon[109137]: pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:12:36.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:36 smithi018 ceph-mon[95889]: pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:12:36.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:36 smithi149 ceph-mon[100764]: pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:12:37.213 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 23 20:12:36 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-alertmanager-a[145423]: ts=2023-09-23T20:12:36.895Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2023-09-23T20:12:37.481 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:12:37.554 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 23 20:12:37 smithi018 bash[151175]: ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-alertmanager-a 2023-09-23T20:12:37.554 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 23 20:12:37 smithi018 systemd[1]: ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7@alertmanager.a.service: Succeeded. 2023-09-23T20:12:37.554 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 23 20:12:37 smithi018 systemd[1]: Stopped Ceph alertmanager.a for a3b3b17e-5a4c-11ee-8db4-212e2dc638e7. 2023-09-23T20:12:37.554 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 23 20:12:37 smithi018 systemd[1]: Starting Ceph alertmanager.a for a3b3b17e-5a4c-11ee-8db4-212e2dc638e7... 2023-09-23T20:12:37.875 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:37 smithi018 ceph-mon[109137]: from='client.14634 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:12:37.875 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:37 smithi018 ceph-mon[95889]: from='client.14634 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:12:37.965 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:37 smithi149 ceph-mon[100764]: from='client.14634 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:12:38.141 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 23 20:12:37 smithi018 podman[151372]: 2023-09-23T20:12:38.560 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 23 20:12:38 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-alertmanager-a[151406]: ts=2023-09-23T20:12:38.309Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2023-09-23T20:12:38.560 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 23 20:12:38 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-alertmanager-a[151406]: ts=2023-09-23T20:12:38.309Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2023-09-23T20:12:38.560 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 23 20:12:38 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-alertmanager-a[151406]: ts=2023-09-23T20:12:38.311Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.18 port=9094 2023-09-23T20:12:38.560 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 23 20:12:38 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-alertmanager-a[151406]: ts=2023-09-23T20:12:38.312Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2023-09-23T20:12:38.560 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 23 20:12:38 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-alertmanager-a[151406]: ts=2023-09-23T20:12:38.340Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2023-09-23T20:12:38.560 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 23 20:12:38 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-alertmanager-a[151406]: ts=2023-09-23T20:12:38.341Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2023-09-23T20:12:38.560 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 23 20:12:38 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-alertmanager-a[151406]: ts=2023-09-23T20:12:38.344Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2023-09-23T20:12:38.560 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 23 20:12:38 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-alertmanager-a[151406]: ts=2023-09-23T20:12:38.344Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2023-09-23T20:12:38.561 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 23 20:12:38 smithi018 bash[151372]: a755b5c2ba2fb1ba8b22d99da158fac35ffc2ca0d64e5d0bbdae266e36f302a9 2023-09-23T20:12:38.561 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 23 20:12:38 smithi018 systemd[1]: Started Ceph alertmanager.a for a3b3b17e-5a4c-11ee-8db4-212e2dc638e7. 2023-09-23T20:12:38.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:38 smithi018 ceph-mon[109137]: pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:12:38.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:38 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:38.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:38 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:38.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:38 smithi018 ceph-mon[95889]: pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:12:38.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:38 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:38.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:38 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:38.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:38 smithi149 ceph-mon[100764]: pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:12:38.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:38 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:38.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:38 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:39.110 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:12:39.110 INFO:teuthology.orchestra.run.smithi018.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2023-09-23T20:12:39.549 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2023-09-23T20:12:39.549 INFO:tasks.cephadm:Setup complete, yielding 2023-09-23T20:12:39.549 INFO:teuthology.run_tasks:Running task cephadm.shell... 2023-09-23T20:12:39.559 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi018.front.sepia.ceph.com 2023-09-23T20:12:39.559 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- bash -c 'ceph fs volume create foo' 2023-09-23T20:12:39.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:39 smithi149 ceph-mon[100764]: Reconfiguring prometheus.a (dependencies changed)... 2023-09-23T20:12:39.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:39 smithi149 ceph-mon[100764]: Reconfiguring daemon prometheus.a on smithi149 2023-09-23T20:12:39.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:39 smithi149 ceph-mon[100764]: from='client.? 172.21.15.18:0/2737330918' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-09-23T20:12:39.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:39 smithi018 ceph-mon[109137]: Reconfiguring prometheus.a (dependencies changed)... 2023-09-23T20:12:39.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:39 smithi018 ceph-mon[109137]: Reconfiguring daemon prometheus.a on smithi149 2023-09-23T20:12:39.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:39 smithi018 ceph-mon[109137]: from='client.? 172.21.15.18:0/2737330918' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-09-23T20:12:39.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:39 smithi018 ceph-mon[95889]: Reconfiguring prometheus.a (dependencies changed)... 2023-09-23T20:12:39.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:39 smithi018 ceph-mon[95889]: Reconfiguring daemon prometheus.a on smithi149 2023-09-23T20:12:39.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:39 smithi018 ceph-mon[95889]: from='client.? 172.21.15.18:0/2737330918' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-09-23T20:12:40.165 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:12:40.564 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 23 20:12:40 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-alertmanager-a[151406]: ts=2023-09-23T20:12:40.312Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000722919s 2023-09-23T20:12:40.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:40 smithi149 ceph-mon[100764]: pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:12:40.716 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:12:40 smithi149 systemd[1]: Stopping Ceph prometheus.a for a3b3b17e-5a4c-11ee-8db4-212e2dc638e7... 2023-09-23T20:12:40.872 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:40 smithi018 ceph-mon[109137]: pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:12:40.872 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:40 smithi018 ceph-mon[95889]: pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:12:41.036 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:12:40 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[135333]: ts=2023-09-23T20:12:40.768Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2023-09-23T20:12:41.037 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:12:40 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[135333]: ts=2023-09-23T20:12:40.768Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2023-09-23T20:12:41.037 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:12:40 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[135333]: ts=2023-09-23T20:12:40.768Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2023-09-23T20:12:41.037 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:12:40 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[135333]: ts=2023-09-23T20:12:40.768Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2023-09-23T20:12:41.037 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:12:40 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[135333]: ts=2023-09-23T20:12:40.768Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2023-09-23T20:12:41.037 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:12:40 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[135333]: ts=2023-09-23T20:12:40.768Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2023-09-23T20:12:41.037 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:12:40 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[135333]: ts=2023-09-23T20:12:40.768Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2023-09-23T20:12:41.037 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:12:40 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[135333]: ts=2023-09-23T20:12:40.768Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2023-09-23T20:12:41.037 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:12:40 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[135333]: ts=2023-09-23T20:12:40.769Z caller=main.go:907 level=info msg="Scrape manager stopped" 2023-09-23T20:12:41.037 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:12:40 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[135333]: ts=2023-09-23T20:12:40.769Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2023-09-23T20:12:41.037 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:12:40 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[135333]: ts=2023-09-23T20:12:40.769Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2023-09-23T20:12:41.038 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:12:40 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[135333]: ts=2023-09-23T20:12:40.769Z caller=main.go:1147 level=info msg="See you next time!" 2023-09-23T20:12:41.038 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:12:41 smithi149 bash[138859]: ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a 2023-09-23T20:12:41.506 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:12:41 smithi149 systemd[1]: ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7@prometheus.a.service: Succeeded. 2023-09-23T20:12:41.506 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:12:41 smithi149 systemd[1]: Stopped Ceph prometheus.a for a3b3b17e-5a4c-11ee-8db4-212e2dc638e7. 2023-09-23T20:12:41.506 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:12:41 smithi149 systemd[1]: Starting Ceph prometheus.a for a3b3b17e-5a4c-11ee-8db4-212e2dc638e7... 2023-09-23T20:12:41.770 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:12:41 smithi149 podman[138983]: 2023-09-23T20:12:42.216 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:12:41 smithi149 bash[138983]: 4a99f58b6591500707d7b7f573df2d4f56feeb74574611e233f306889e6eb13f 2023-09-23T20:12:42.216 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:12:41 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[138998]: ts=2023-09-23T20:12:41.776Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2023-09-23T20:12:42.217 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:12:41 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[138998]: ts=2023-09-23T20:12:41.776Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2023-09-23T20:12:42.217 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:12:41 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[138998]: ts=2023-09-23T20:12:41.776Z caller=main.go:570 level=info host_details="(Linux 4.18.0-372.9.1.el8.x86_64 #1 SMP Fri Apr 15 22:12:19 EDT 2022 x86_64 smithi149 (none))" 2023-09-23T20:12:42.217 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:12:41 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[138998]: ts=2023-09-23T20:12:41.776Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2023-09-23T20:12:42.217 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:12:41 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[138998]: ts=2023-09-23T20:12:41.776Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2023-09-23T20:12:42.217 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:12:41 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[138998]: ts=2023-09-23T20:12:41.777Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2023-09-23T20:12:42.217 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:12:41 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[138998]: ts=2023-09-23T20:12:41.778Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2023-09-23T20:12:42.217 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:12:41 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[138998]: ts=2023-09-23T20:12:41.779Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2023-09-23T20:12:42.217 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:12:41 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[138998]: ts=2023-09-23T20:12:41.779Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2023-09-23T20:12:42.217 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:12:41 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[138998]: ts=2023-09-23T20:12:41.781Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2023-09-23T20:12:42.217 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:12:41 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[138998]: ts=2023-09-23T20:12:41.781Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.656µs 2023-09-23T20:12:42.217 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:12:41 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[138998]: ts=2023-09-23T20:12:41.781Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2023-09-23T20:12:42.218 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:12:41 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[138998]: ts=2023-09-23T20:12:41.781Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=1 2023-09-23T20:12:42.218 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:12:41 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[138998]: ts=2023-09-23T20:12:41.782Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=1 2023-09-23T20:12:42.218 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:12:41 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[138998]: ts=2023-09-23T20:12:41.782Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=20.681µs wal_replay_duration=451.624µs wbl_replay_duration=94ns total_replay_duration=504.897µs 2023-09-23T20:12:42.218 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:12:41 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[138998]: ts=2023-09-23T20:12:41.782Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2023-09-23T20:12:42.218 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:12:41 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[138998]: ts=2023-09-23T20:12:41.782Z caller=main.go:1029 level=info msg="TSDB started" 2023-09-23T20:12:42.218 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:12:41 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[138998]: ts=2023-09-23T20:12:41.782Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2023-09-23T20:12:42.218 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:12:41 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[138998]: ts=2023-09-23T20:12:41.798Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=15.6097ms db_storage=820ns remote_storage=1.288µs web_handler=300ns query_engine=470ns scrape=399.763µs scrape_sd=22.97µs notify=25.678µs notify_sd=6.299µs rules=14.952269ms tracing=6.421µs 2023-09-23T20:12:42.218 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:12:41 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[138998]: ts=2023-09-23T20:12:41.798Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2023-09-23T20:12:42.218 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:12:41 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[138998]: ts=2023-09-23T20:12:41.798Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2023-09-23T20:12:42.218 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:12:41 smithi149 systemd[1]: Started Ceph prometheus.a for a3b3b17e-5a4c-11ee-8db4-212e2dc638e7. 2023-09-23T20:12:42.641 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:12:42 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: [23/Sep/2023:20:12:42] ENGINE Bus STOPPING 2023-09-23T20:12:42.641 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:12:42 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: [23/Sep/2023:20:12:42] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T20:12:42.641 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:12:42 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: [23/Sep/2023:20:12:42] ENGINE Bus STOPPED 2023-09-23T20:12:42.641 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:12:42 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: [23/Sep/2023:20:12:42] ENGINE Bus STARTING 2023-09-23T20:12:42.642 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:12:42 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: [23/Sep/2023:20:12:42] ENGINE Serving on http://:::9283 2023-09-23T20:12:42.642 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:12:42 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: [23/Sep/2023:20:12:42] ENGINE Bus STARTED 2023-09-23T20:12:42.642 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:12:42 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: [23/Sep/2023:20:12:42] ENGINE Bus STOPPING 2023-09-23T20:12:42.642 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:12:42 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: [23/Sep/2023:20:12:42] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T20:12:42.642 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:12:42 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: [23/Sep/2023:20:12:42] ENGINE Bus STOPPED 2023-09-23T20:12:42.642 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:12:42 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: [23/Sep/2023:20:12:42] ENGINE Bus STARTING 2023-09-23T20:12:42.642 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:12:42 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: [23/Sep/2023:20:12:42] ENGINE Serving on http://:::9283 2023-09-23T20:12:42.642 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:12:42 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: [23/Sep/2023:20:12:42] ENGINE Bus STARTED 2023-09-23T20:12:42.642 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:12:42 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: [23/Sep/2023:20:12:42] ENGINE Bus STOPPING 2023-09-23T20:12:42.642 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:12:42 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: [23/Sep/2023:20:12:42] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T20:12:42.642 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:12:42 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: [23/Sep/2023:20:12:42] ENGINE Bus STOPPED 2023-09-23T20:12:42.643 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:12:42 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: [23/Sep/2023:20:12:42] ENGINE Bus STARTING 2023-09-23T20:12:43.140 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:12:42 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: [23/Sep/2023:20:12:42] ENGINE Serving on http://:::9283 2023-09-23T20:12:43.141 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:12:42 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: [23/Sep/2023:20:12:42] ENGINE Bus STARTED 2023-09-23T20:12:43.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:42 smithi018 ceph-mon[95889]: pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:12:43.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:42 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-09-23T20:12:43.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:42 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:43.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:42 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:43.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:42 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T20:12:43.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:42 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi018:9093"}]: dispatch 2023-09-23T20:12:43.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:42 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:43.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:42 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T20:12:43.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:42 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi149.front.sepia.ceph.com:9095"}]: dispatch 2023-09-23T20:12:43.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:42 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:43.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:42 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T20:12:43.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:42 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi149.front.sepia.ceph.com:3000"}]: dispatch 2023-09-23T20:12:43.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:42 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:43.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:42 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:12:43.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:42 smithi018 ceph-mon[109137]: pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:12:43.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:42 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-09-23T20:12:43.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:42 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:43.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:42 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:43.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:42 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T20:12:43.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:42 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi018:9093"}]: dispatch 2023-09-23T20:12:43.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:42 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:43.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:42 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T20:12:43.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:42 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi149.front.sepia.ceph.com:9095"}]: dispatch 2023-09-23T20:12:43.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:42 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:43.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:42 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T20:12:43.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:42 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi149.front.sepia.ceph.com:3000"}]: dispatch 2023-09-23T20:12:43.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:42 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:43.144 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:42 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:12:43.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:42 smithi149 ceph-mon[100764]: pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:12:43.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:42 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-09-23T20:12:43.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:42 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:43.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:42 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:43.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:42 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T20:12:43.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:42 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi018:9093"}]: dispatch 2023-09-23T20:12:43.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:42 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:43.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:42 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T20:12:43.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:42 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi149.front.sepia.ceph.com:9095"}]: dispatch 2023-09-23T20:12:43.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:42 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:43.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:42 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T20:12:43.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:42 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi149.front.sepia.ceph.com:3000"}]: dispatch 2023-09-23T20:12:43.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:42 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:43.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:42 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:12:44.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:43 smithi018 ceph-mon[109137]: from='client.24554 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:12:44.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:43 smithi018 ceph-mon[109137]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T20:12:44.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:43 smithi018 ceph-mon[109137]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi018:9093"}]: dispatch 2023-09-23T20:12:44.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:43 smithi018 ceph-mon[109137]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T20:12:44.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:43 smithi018 ceph-mon[109137]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi149.front.sepia.ceph.com:9095"}]: dispatch 2023-09-23T20:12:44.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:43 smithi018 ceph-mon[109137]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T20:12:44.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:43 smithi018 ceph-mon[109137]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi149.front.sepia.ceph.com:3000"}]: dispatch 2023-09-23T20:12:44.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:43 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2023-09-23T20:12:44.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:43 smithi018 ceph-mon[109137]: osdmap e65: 8 total, 8 up, 8 in 2023-09-23T20:12:44.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:43 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-09-23T20:12:44.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:43 smithi018 ceph-mon[95889]: from='client.24554 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:12:44.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:43 smithi018 ceph-mon[95889]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T20:12:44.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:43 smithi018 ceph-mon[95889]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi018:9093"}]: dispatch 2023-09-23T20:12:44.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:43 smithi018 ceph-mon[95889]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T20:12:44.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:43 smithi018 ceph-mon[95889]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi149.front.sepia.ceph.com:9095"}]: dispatch 2023-09-23T20:12:44.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:43 smithi018 ceph-mon[95889]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T20:12:44.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:43 smithi018 ceph-mon[95889]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi149.front.sepia.ceph.com:3000"}]: dispatch 2023-09-23T20:12:44.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:43 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2023-09-23T20:12:44.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:43 smithi018 ceph-mon[95889]: osdmap e65: 8 total, 8 up, 8 in 2023-09-23T20:12:44.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:43 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-09-23T20:12:44.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:43 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mon-a[95865]: 2023-09-23T20:12:43.744+0000 7fd62e3b0700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-09-23T20:12:44.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:43 smithi149 ceph-mon[100764]: from='client.24554 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:12:44.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:43 smithi149 ceph-mon[100764]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T20:12:44.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:43 smithi149 ceph-mon[100764]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi018:9093"}]: dispatch 2023-09-23T20:12:44.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:43 smithi149 ceph-mon[100764]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T20:12:44.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:43 smithi149 ceph-mon[100764]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi149.front.sepia.ceph.com:9095"}]: dispatch 2023-09-23T20:12:44.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:43 smithi149 ceph-mon[100764]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T20:12:44.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:43 smithi149 ceph-mon[100764]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi149.front.sepia.ceph.com:3000"}]: dispatch 2023-09-23T20:12:44.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:43 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2023-09-23T20:12:44.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:43 smithi149 ceph-mon[100764]: osdmap e65: 8 total, 8 up, 8 in 2023-09-23T20:12:44.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:43 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-09-23T20:12:44.225 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2023-09-23T20:12:44.833 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:12:45.019 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:44 smithi018 ceph-mon[109137]: pgmap v35: 41 pgs: 32 unknown, 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:12:45.019 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:44 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2023-09-23T20:12:45.019 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:44 smithi018 ceph-mon[109137]: osdmap e66: 8 total, 8 up, 8 in 2023-09-23T20:12:45.019 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:44 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-09-23T20:12:45.019 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:44 smithi018 ceph-mon[109137]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-09-23T20:12:45.019 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:44 smithi018 ceph-mon[109137]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2023-09-23T20:12:45.020 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:44 smithi018 ceph-mon[109137]: osdmap e67: 8 total, 8 up, 8 in 2023-09-23T20:12:45.020 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:44 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2023-09-23T20:12:45.020 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:44 smithi018 ceph-mon[109137]: fsmap foo:0 2023-09-23T20:12:45.020 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:44 smithi018 ceph-mon[109137]: Saving service mds.foo spec with placement count:2 2023-09-23T20:12:45.020 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:44 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:45.020 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:44 smithi018 ceph-mon[95889]: pgmap v35: 41 pgs: 32 unknown, 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:12:45.020 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:44 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2023-09-23T20:12:45.020 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:44 smithi018 ceph-mon[95889]: osdmap e66: 8 total, 8 up, 8 in 2023-09-23T20:12:45.020 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:44 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-09-23T20:12:45.020 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:44 smithi018 ceph-mon[95889]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-09-23T20:12:45.021 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:44 smithi018 ceph-mon[95889]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2023-09-23T20:12:45.021 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:44 smithi018 ceph-mon[95889]: osdmap e67: 8 total, 8 up, 8 in 2023-09-23T20:12:45.021 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:44 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2023-09-23T20:12:45.021 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:44 smithi018 ceph-mon[95889]: fsmap foo:0 2023-09-23T20:12:45.021 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:44 smithi018 ceph-mon[95889]: Saving service mds.foo spec with placement count:2 2023-09-23T20:12:45.021 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:44 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:45.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:44 smithi149 ceph-mon[100764]: pgmap v35: 41 pgs: 32 unknown, 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:12:45.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:44 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2023-09-23T20:12:45.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:44 smithi149 ceph-mon[100764]: osdmap e66: 8 total, 8 up, 8 in 2023-09-23T20:12:45.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:44 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-09-23T20:12:45.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:44 smithi149 ceph-mon[100764]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-09-23T20:12:45.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:44 smithi149 ceph-mon[100764]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2023-09-23T20:12:45.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:44 smithi149 ceph-mon[100764]: osdmap e67: 8 total, 8 up, 8 in 2023-09-23T20:12:45.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:44 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2023-09-23T20:12:45.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:44 smithi149 ceph-mon[100764]: fsmap foo:0 2023-09-23T20:12:45.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:44 smithi149 ceph-mon[100764]: Saving service mds.foo spec with placement count:2 2023-09-23T20:12:45.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:44 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:46.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:45 smithi018 ceph-mon[109137]: osdmap e68: 8 total, 8 up, 8 in 2023-09-23T20:12:46.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:45 smithi018 ceph-mon[95889]: osdmap e68: 8 total, 8 up, 8 in 2023-09-23T20:12:46.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:45 smithi149 ceph-mon[100764]: osdmap e68: 8 total, 8 up, 8 in 2023-09-23T20:12:46.988 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:46 smithi018 ceph-mon[109137]: pgmap v39: 73 pgs: 73 active+clean; 577 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:12:46.988 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:46 smithi018 ceph-mon[109137]: osdmap e69: 8 total, 8 up, 8 in 2023-09-23T20:12:46.989 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:46 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:46.989 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:46 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:46.989 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:46 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:46.989 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:46 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:46.989 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:46 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:12:46.989 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:46 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:12:46.989 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:46 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:46.989 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:46 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi018.hduiol", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T20:12:46.989 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:46 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi018.hduiol", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-09-23T20:12:46.989 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:46 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:12:46.989 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:46 smithi018 ceph-mon[95889]: pgmap v39: 73 pgs: 73 active+clean; 577 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:12:46.990 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:46 smithi018 ceph-mon[95889]: osdmap e69: 8 total, 8 up, 8 in 2023-09-23T20:12:46.990 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:46 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:46.990 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:46 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:46.990 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:46 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:46.990 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:46 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:46.990 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:46 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:12:46.990 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:46 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:12:46.990 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:46 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:46.990 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:46 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi018.hduiol", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T20:12:46.990 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:46 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi018.hduiol", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-09-23T20:12:46.990 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:46 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:12:47.000 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2023-09-23T20:12:47.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:46 smithi149 ceph-mon[100764]: pgmap v39: 73 pgs: 73 active+clean; 577 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:12:47.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:46 smithi149 ceph-mon[100764]: osdmap e69: 8 total, 8 up, 8 in 2023-09-23T20:12:47.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:46 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:47.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:46 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:47.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:46 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:47.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:46 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:47.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:46 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:12:47.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:46 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:12:47.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:46 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:47.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:46 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi018.hduiol", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T20:12:47.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:46 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi018.hduiol", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-09-23T20:12:47.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:46 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:12:47.761 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:12:47.768 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:47 smithi018 ceph-mon[109137]: Deploying daemon mds.foo.smithi018.hduiol on smithi018 2023-09-23T20:12:48.025 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:47 smithi018 ceph-mon[95889]: Deploying daemon mds.foo.smithi018.hduiol on smithi018 2023-09-23T20:12:48.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:47 smithi149 ceph-mon[100764]: Deploying daemon mds.foo.smithi018.hduiol on smithi018 2023-09-23T20:12:48.641 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 23 20:12:48 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-alertmanager-a[151406]: ts=2023-09-23T20:12:48.316Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.004709868s 2023-09-23T20:12:48.966 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:48 smithi018 ceph-mon[109137]: pgmap v41: 73 pgs: 73 active+clean; 577 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:12:48.966 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:48 smithi018 ceph-mon[95889]: pgmap v41: 73 pgs: 73 active+clean; 577 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:12:49.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:48 smithi149 ceph-mon[100764]: pgmap v41: 73 pgs: 73 active+clean; 577 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:12:49.231 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:12:49 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:12:49] "GET /metrics HTTP/1.1" 200 203100 "" "Prometheus/2.43.0" 2023-09-23T20:12:50.075 INFO:teuthology.run_tasks:Running task ceph.healthy... 2023-09-23T20:12:50.084 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2023-09-23T20:12:50.084 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2023-09-23T20:12:50.084 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph mgr dump --format=json 2023-09-23T20:12:50.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:50 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:50.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:50 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:50.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:50 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:50.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:50 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi149.urtezf", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T20:12:50.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:50 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi149.urtezf", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-09-23T20:12:50.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:50 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:12:50.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:50 smithi018 ceph-mon[95889]: Deploying daemon mds.foo.smithi149.urtezf on smithi149 2023-09-23T20:12:50.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:50 smithi018 ceph-mon[95889]: pgmap v42: 73 pgs: 73 active+clean; 577 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:12:50.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:50 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2023-09-23T20:12:50.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:50 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2023-09-23T20:12:50.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:50 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2023-09-23T20:12:50.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:50 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:50.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:50 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:12:50.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:50 smithi018 ceph-mon[95889]: mds.? [v2:172.21.15.18:6834/3661910111,v1:172.21.15.18:6835/3661910111] up:boot 2023-09-23T20:12:50.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:50 smithi018 ceph-mon[95889]: daemon mds.foo.smithi018.hduiol assigned to filesystem foo as rank 0 (now has 1 ranks) 2023-09-23T20:12:50.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:50 smithi018 ceph-mon[95889]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2023-09-23T20:12:50.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:50 smithi018 ceph-mon[95889]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2023-09-23T20:12:50.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:50 smithi018 ceph-mon[95889]: Cluster is now healthy 2023-09-23T20:12:50.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:50 smithi018 ceph-mon[95889]: fsmap foo:0 1 up:standby 2023-09-23T20:12:50.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:50 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi018.hduiol"}]: dispatch 2023-09-23T20:12:50.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:50 smithi018 ceph-mon[95889]: fsmap foo:1 {0=foo.smithi018.hduiol=up:creating} 2023-09-23T20:12:50.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:50 smithi018 ceph-mon[95889]: daemon mds.foo.smithi018.hduiol is now active in filesystem foo as rank 0 2023-09-23T20:12:50.643 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:50 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:50.643 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:50 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:50.644 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:50 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:50.644 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:50 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi149.urtezf", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T20:12:50.644 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:50 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi149.urtezf", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-09-23T20:12:50.644 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:50 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:12:50.644 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:50 smithi018 ceph-mon[109137]: Deploying daemon mds.foo.smithi149.urtezf on smithi149 2023-09-23T20:12:50.644 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:50 smithi018 ceph-mon[109137]: pgmap v42: 73 pgs: 73 active+clean; 577 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:12:50.644 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:50 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2023-09-23T20:12:50.644 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:50 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2023-09-23T20:12:50.644 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:50 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2023-09-23T20:12:50.644 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:50 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:50.645 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:50 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:12:50.645 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:50 smithi018 ceph-mon[109137]: mds.? [v2:172.21.15.18:6834/3661910111,v1:172.21.15.18:6835/3661910111] up:boot 2023-09-23T20:12:50.645 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:50 smithi018 ceph-mon[109137]: daemon mds.foo.smithi018.hduiol assigned to filesystem foo as rank 0 (now has 1 ranks) 2023-09-23T20:12:50.645 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:50 smithi018 ceph-mon[109137]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2023-09-23T20:12:50.645 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:50 smithi018 ceph-mon[109137]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2023-09-23T20:12:50.645 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:50 smithi018 ceph-mon[109137]: Cluster is now healthy 2023-09-23T20:12:50.645 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:50 smithi018 ceph-mon[109137]: fsmap foo:0 1 up:standby 2023-09-23T20:12:50.645 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:50 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi018.hduiol"}]: dispatch 2023-09-23T20:12:50.645 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:50 smithi018 ceph-mon[109137]: fsmap foo:1 {0=foo.smithi018.hduiol=up:creating} 2023-09-23T20:12:50.645 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:50 smithi018 ceph-mon[109137]: daemon mds.foo.smithi018.hduiol is now active in filesystem foo as rank 0 2023-09-23T20:12:50.690 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:12:50.702 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:50 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:50.702 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:50 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:50.703 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:50 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:50.703 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:50 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi149.urtezf", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T20:12:50.703 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:50 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi149.urtezf", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-09-23T20:12:50.703 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:50 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:12:50.703 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:50 smithi149 ceph-mon[100764]: Deploying daemon mds.foo.smithi149.urtezf on smithi149 2023-09-23T20:12:50.703 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:50 smithi149 ceph-mon[100764]: pgmap v42: 73 pgs: 73 active+clean; 577 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:12:50.703 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:50 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2023-09-23T20:12:50.703 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:50 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2023-09-23T20:12:50.703 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:50 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2023-09-23T20:12:50.703 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:50 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:50.703 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:50 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:12:50.704 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:50 smithi149 ceph-mon[100764]: mds.? [v2:172.21.15.18:6834/3661910111,v1:172.21.15.18:6835/3661910111] up:boot 2023-09-23T20:12:50.704 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:50 smithi149 ceph-mon[100764]: daemon mds.foo.smithi018.hduiol assigned to filesystem foo as rank 0 (now has 1 ranks) 2023-09-23T20:12:50.704 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:50 smithi149 ceph-mon[100764]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2023-09-23T20:12:50.704 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:50 smithi149 ceph-mon[100764]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2023-09-23T20:12:50.704 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:50 smithi149 ceph-mon[100764]: Cluster is now healthy 2023-09-23T20:12:50.704 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:50 smithi149 ceph-mon[100764]: fsmap foo:0 1 up:standby 2023-09-23T20:12:50.704 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:50 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi018.hduiol"}]: dispatch 2023-09-23T20:12:50.704 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:50 smithi149 ceph-mon[100764]: fsmap foo:1 {0=foo.smithi018.hduiol=up:creating} 2023-09-23T20:12:50.704 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:50 smithi149 ceph-mon[100764]: daemon mds.foo.smithi018.hduiol is now active in filesystem foo as rank 0 2023-09-23T20:12:51.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:51 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]': finished 2023-09-23T20:12:51.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:51 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]': finished 2023-09-23T20:12:51.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:51 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]': finished 2023-09-23T20:12:51.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:51 smithi018 ceph-mon[109137]: osdmap e70: 8 total, 8 up, 8 in 2023-09-23T20:12:51.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:51 smithi018 ceph-mon[109137]: mds.? [v2:172.21.15.18:6834/3661910111,v1:172.21.15.18:6835/3661910111] up:active 2023-09-23T20:12:51.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:51 smithi018 ceph-mon[109137]: fsmap foo:1 {0=foo.smithi018.hduiol=up:active} 2023-09-23T20:12:51.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:51 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]': finished 2023-09-23T20:12:51.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:51 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]': finished 2023-09-23T20:12:51.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:51 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]': finished 2023-09-23T20:12:51.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:51 smithi018 ceph-mon[95889]: osdmap e70: 8 total, 8 up, 8 in 2023-09-23T20:12:51.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:51 smithi018 ceph-mon[95889]: mds.? [v2:172.21.15.18:6834/3661910111,v1:172.21.15.18:6835/3661910111] up:active 2023-09-23T20:12:51.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:51 smithi018 ceph-mon[95889]: fsmap foo:1 {0=foo.smithi018.hduiol=up:active} 2023-09-23T20:12:51.715 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:51 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]': finished 2023-09-23T20:12:51.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:51 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]': finished 2023-09-23T20:12:51.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:51 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]': finished 2023-09-23T20:12:51.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:51 smithi149 ceph-mon[100764]: osdmap e70: 8 total, 8 up, 8 in 2023-09-23T20:12:51.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:51 smithi149 ceph-mon[100764]: mds.? [v2:172.21.15.18:6834/3661910111,v1:172.21.15.18:6835/3661910111] up:active 2023-09-23T20:12:51.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:51 smithi149 ceph-mon[100764]: fsmap foo:1 {0=foo.smithi018.hduiol=up:active} 2023-09-23T20:12:52.297 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:12:52.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:52 smithi018 ceph-mon[109137]: osdmap e71: 8 total, 8 up, 8 in 2023-09-23T20:12:52.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:52 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:52.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:52 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:52.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:52 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:52.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:52 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:52.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:52 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:52.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:52 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:12:52.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:52 smithi018 ceph-mon[109137]: pgmap v45: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 2.2 KiB/s wr, 7 op/s 2023-09-23T20:12:52.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:52 smithi018 ceph-mon[109137]: mds.? [v2:172.21.15.149:6832/3529449999,v1:172.21.15.149:6833/3529449999] up:boot 2023-09-23T20:12:52.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:52 smithi018 ceph-mon[109137]: fsmap foo:1 {0=foo.smithi018.hduiol=up:active} 1 up:standby 2023-09-23T20:12:52.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:52 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi149.urtezf"}]: dispatch 2023-09-23T20:12:52.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:52 smithi018 ceph-mon[109137]: fsmap foo:1 {0=foo.smithi018.hduiol=up:active} 1 up:standby 2023-09-23T20:12:52.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:52 smithi018 ceph-mon[109137]: from='client.? 172.21.15.18:0/2425760930' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-09-23T20:12:52.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:52 smithi018 ceph-mon[95889]: osdmap e71: 8 total, 8 up, 8 in 2023-09-23T20:12:52.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:52 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:52.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:52 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:52.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:52 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:52.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:52 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:52.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:52 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:52.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:52 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:12:52.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:52 smithi018 ceph-mon[95889]: pgmap v45: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 2.2 KiB/s wr, 7 op/s 2023-09-23T20:12:52.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:52 smithi018 ceph-mon[95889]: mds.? [v2:172.21.15.149:6832/3529449999,v1:172.21.15.149:6833/3529449999] up:boot 2023-09-23T20:12:52.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:52 smithi018 ceph-mon[95889]: fsmap foo:1 {0=foo.smithi018.hduiol=up:active} 1 up:standby 2023-09-23T20:12:52.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:52 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi149.urtezf"}]: dispatch 2023-09-23T20:12:52.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:52 smithi018 ceph-mon[95889]: fsmap foo:1 {0=foo.smithi018.hduiol=up:active} 1 up:standby 2023-09-23T20:12:52.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:52 smithi018 ceph-mon[95889]: from='client.? 172.21.15.18:0/2425760930' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-09-23T20:12:52.649 INFO:teuthology.orchestra.run.smithi018.stdout:{"epoch":20,"active_gid":14454,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6800","nonce":115279545},{"type":"v1","addr":"172.21.15.18:6801","nonce":115279545}]},"active_addr":"172.21.15.18:6801/115279545","active_change":"2023-09-23T20:11:49.652957+0000","active_mgr_features":4540138320759226367,"available":true,"standbys":[{"gid":24361,"name":"x","mgr_features":4540138320759226367,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.18:8443/","prometheus":"http://172.21.15.18:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":60,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.18:0","nonce":3356195690}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.18:0","nonce":2809002822}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.18:0","nonce":3990640673}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.18:0","nonce":2543728509}]}]} 2023-09-23T20:12:52.653 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2023-09-23T20:12:52.654 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2023-09-23T20:12:52.654 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd dump --format=json 2023-09-23T20:12:52.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:52 smithi149 ceph-mon[100764]: osdmap e71: 8 total, 8 up, 8 in 2023-09-23T20:12:52.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:52 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:52.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:52 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:52.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:52 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:52.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:52 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:52.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:52 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:52.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:52 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:12:52.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:52 smithi149 ceph-mon[100764]: pgmap v45: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 2.2 KiB/s wr, 7 op/s 2023-09-23T20:12:52.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:52 smithi149 ceph-mon[100764]: mds.? [v2:172.21.15.149:6832/3529449999,v1:172.21.15.149:6833/3529449999] up:boot 2023-09-23T20:12:52.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:52 smithi149 ceph-mon[100764]: fsmap foo:1 {0=foo.smithi018.hduiol=up:active} 1 up:standby 2023-09-23T20:12:52.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:52 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi149.urtezf"}]: dispatch 2023-09-23T20:12:52.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:52 smithi149 ceph-mon[100764]: fsmap foo:1 {0=foo.smithi018.hduiol=up:active} 1 up:standby 2023-09-23T20:12:52.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:52 smithi149 ceph-mon[100764]: from='client.? 172.21.15.18:0/2425760930' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-09-23T20:12:53.364 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:12:54.216 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:12:53 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[102880]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:12:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T20:12:54.730 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:54 smithi018 ceph-mon[109137]: pgmap v46: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s wr, 5 op/s 2023-09-23T20:12:54.730 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:54 smithi018 ceph-mon[95889]: pgmap v46: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s wr, 5 op/s 2023-09-23T20:12:54.863 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:12:54.863 INFO:teuthology.orchestra.run.smithi018.stdout:{"epoch":71,"fsid":"a3b3b17e-5a4c-11ee-8db4-212e2dc638e7","created":"2023-09-23T20:06:54.359034+0000","modified":"2023-09-23T20:12:51.367725+0000","last_up_change":"2023-09-23T20:11:23.829606+0000","last_in_change":"2023-09-23T20:11:09.202130+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-09-23T20:09:47.162973+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-09-23T20:11:59.431837+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"64","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-09-23T20:12:42.254901+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-09-23T20:12:43.191578+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"69","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}}}],"osds":[{"osd":0,"uuid":"44c8aaa3-bbd1-4330-8d34-73f7a08b0d4d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6802","nonce":2209550685},{"type":"v1","addr":"172.21.15.18:6803","nonce":2209550685}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6804","nonce":2209550685},{"type":"v1","addr":"172.21.15.18:6805","nonce":2209550685}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6808","nonce":2209550685},{"type":"v1","addr":"172.21.15.18:6809","nonce":2209550685}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6806","nonce":2209550685},{"type":"v1","addr":"172.21.15.18:6807","nonce":2209550685}]},"public_addr":"172.21.15.18:6803/2209550685","cluster_addr":"172.21.15.18:6805/2209550685","heartbeat_back_addr":"172.21.15.18:6809/2209550685","heartbeat_front_addr":"172.21.15.18:6807/2209550685","state":["exists","up"]},{"osd":1,"uuid":"c0534324-49c8-4804-bced-c4d13510e74a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6810","nonce":314354451},{"type":"v1","addr":"172.21.15.18:6811","nonce":314354451}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6812","nonce":314354451},{"type":"v1","addr":"172.21.15.18:6813","nonce":314354451}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6816","nonce":314354451},{"type":"v1","addr":"172.21.15.18:6817","nonce":314354451}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6814","nonce":314354451},{"type":"v1","addr":"172.21.15.18:6815","nonce":314354451}]},"public_addr":"172.21.15.18:6811/314354451","cluster_addr":"172.21.15.18:6813/314354451","heartbeat_back_addr":"172.21.15.18:6817/314354451","heartbeat_front_addr":"172.21.15.18:6815/314354451","state":["exists","up"]},{"osd":2,"uuid":"f3b2116d-a3d5-4279-91dc-eb47458d3239","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":70,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6818","nonce":1413681698},{"type":"v1","addr":"172.21.15.18:6819","nonce":1413681698}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6820","nonce":1413681698},{"type":"v1","addr":"172.21.15.18:6821","nonce":1413681698}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6824","nonce":1413681698},{"type":"v1","addr":"172.21.15.18:6825","nonce":1413681698}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6822","nonce":1413681698},{"type":"v1","addr":"172.21.15.18:6823","nonce":1413681698}]},"public_addr":"172.21.15.18:6819/1413681698","cluster_addr":"172.21.15.18:6821/1413681698","heartbeat_back_addr":"172.21.15.18:6825/1413681698","heartbeat_front_addr":"172.21.15.18:6823/1413681698","state":["exists","up"]},{"osd":3,"uuid":"8edccb7f-c4c9-47f3-a6f5-38a1a38cb099","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6826","nonce":423024182},{"type":"v1","addr":"172.21.15.18:6827","nonce":423024182}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6828","nonce":423024182},{"type":"v1","addr":"172.21.15.18:6829","nonce":423024182}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6832","nonce":423024182},{"type":"v1","addr":"172.21.15.18:6833","nonce":423024182}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6830","nonce":423024182},{"type":"v1","addr":"172.21.15.18:6831","nonce":423024182}]},"public_addr":"172.21.15.18:6827/423024182","cluster_addr":"172.21.15.18:6829/423024182","heartbeat_back_addr":"172.21.15.18:6833/423024182","heartbeat_front_addr":"172.21.15.18:6831/423024182","state":["exists","up"]},{"osd":4,"uuid":"1d5a64ce-fe42-4355-991a-dfb20cbe5423","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":70,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6800","nonce":3640432477},{"type":"v1","addr":"172.21.15.149:6801","nonce":3640432477}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6802","nonce":3640432477},{"type":"v1","addr":"172.21.15.149:6803","nonce":3640432477}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6806","nonce":3640432477},{"type":"v1","addr":"172.21.15.149:6807","nonce":3640432477}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6804","nonce":3640432477},{"type":"v1","addr":"172.21.15.149:6805","nonce":3640432477}]},"public_addr":"172.21.15.149:6801/3640432477","cluster_addr":"172.21.15.149:6803/3640432477","heartbeat_back_addr":"172.21.15.149:6807/3640432477","heartbeat_front_addr":"172.21.15.149:6805/3640432477","state":["exists","up"]},{"osd":5,"uuid":"105ae3b2-b2fc-477d-9134-3d517bc9427e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6808","nonce":2969709367},{"type":"v1","addr":"172.21.15.149:6809","nonce":2969709367}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6810","nonce":2969709367},{"type":"v1","addr":"172.21.15.149:6811","nonce":2969709367}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6814","nonce":2969709367},{"type":"v1","addr":"172.21.15.149:6815","nonce":2969709367}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6812","nonce":2969709367},{"type":"v1","addr":"172.21.15.149:6813","nonce":2969709367}]},"public_addr":"172.21.15.149:6809/2969709367","cluster_addr":"172.21.15.149:6811/2969709367","heartbeat_back_addr":"172.21.15.149:6815/2969709367","heartbeat_front_addr":"172.21.15.149:6813/2969709367","state":["exists","up"]},{"osd":6,"uuid":"27d2d2a8-e8c6-4a05-a85e-bd54376cf4c1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6816","nonce":1291453593},{"type":"v1","addr":"172.21.15.149:6817","nonce":1291453593}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6818","nonce":1291453593},{"type":"v1","addr":"172.21.15.149:6819","nonce":1291453593}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6822","nonce":1291453593},{"type":"v1","addr":"172.21.15.149:6823","nonce":1291453593}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6820","nonce":1291453593},{"type":"v1","addr":"172.21.15.149:6821","nonce":1291453593}]},"public_addr":"172.21.15.149:6817/1291453593","cluster_addr":"172.21.15.149:6819/1291453593","heartbeat_back_addr":"172.21.15.149:6823/1291453593","heartbeat_front_addr":"172.21.15.149:6821/1291453593","state":["exists","up"]},{"osd":7,"uuid":"9da636f7-995c-4f29-9264-70d09c6f66c6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6824","nonce":3117985178},{"type":"v1","addr":"172.21.15.149:6825","nonce":3117985178}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6826","nonce":3117985178},{"type":"v1","addr":"172.21.15.149:6827","nonce":3117985178}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6830","nonce":3117985178},{"type":"v1","addr":"172.21.15.149:6831","nonce":3117985178}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6828","nonce":3117985178},{"type":"v1","addr":"172.21.15.149:6829","nonce":3117985178}]},"public_addr":"172.21.15.149:6825/3117985178","cluster_addr":"172.21.15.149:6827/3117985178","heartbeat_back_addr":"172.21.15.149:6831/3117985178","heartbeat_front_addr":"172.21.15.149:6829/3117985178","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:09:01.034520+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:09:21.795238+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:09:42.987835+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:10:04.092458+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:10:23.309112+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:10:43.348231+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:11:03.128458+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:11:23.084228+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.d","mappings":[{"from":1,"to":5}]},{"pgid":"4.f","mappings":[{"from":1,"to":2}]},{"pgid":"4.18","mappings":[{"from":1,"to":2}]}],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.18:0/3912151186":"2023-09-24T20:11:49.652762+0000","172.21.15.18:0/3863388488":"2023-09-24T20:11:49.652762+0000","172.21.15.18:6801/2611527229":"2023-09-24T20:11:49.652762+0000","172.21.15.18:6800/2611527229":"2023-09-24T20:11:49.652762+0000","172.21.15.18:0/1698512393":"2023-09-24T20:11:49.652762+0000","172.21.15.18:6801/380813540":"2023-09-24T20:07:48.635784+0000","172.21.15.18:6800/380813540":"2023-09-24T20:07:48.635784+0000","172.21.15.18:0/3831267686":"2023-09-24T20:07:48.635784+0000","172.21.15.18:0/950924496":"2023-09-24T20:07:23.919815+0000","172.21.15.18:0/1988472148":"2023-09-24T20:11:49.652762+0000","172.21.15.18:0/1082665289":"2023-09-24T20:07:48.635784+0000","172.21.15.18:0/2993111051":"2023-09-24T20:07:23.919815+0000","172.21.15.18:0/69122222":"2023-09-24T20:07:23.919815+0000","172.21.15.18:0/1645667747":"2023-09-24T20:07:48.635784+0000","172.21.15.18:6800/1930027051":"2023-09-24T20:07:23.919815+0000","172.21.15.18:6801/1930027051":"2023-09-24T20:07:23.919815+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-23T20:12:54.965 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:54 smithi149 ceph-mon[100764]: pgmap v46: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s wr, 5 op/s 2023-09-23T20:12:55.530 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2023-09-23T20:12:55.530 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd dump --format=json 2023-09-23T20:12:55.754 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:55 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:55.754 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:55 smithi018 ceph-mon[109137]: mds.? [v2:172.21.15.18:6834/3661910111,v1:172.21.15.18:6835/3661910111] up:active 2023-09-23T20:12:55.754 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:55 smithi018 ceph-mon[109137]: fsmap foo:1 {0=foo.smithi018.hduiol=up:active} 1 up:standby 2023-09-23T20:12:55.754 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:55 smithi018 ceph-mon[109137]: from='client.? 172.21.15.18:0/2373956746' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T20:12:56.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:55 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:56.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:55 smithi018 ceph-mon[95889]: mds.? [v2:172.21.15.18:6834/3661910111,v1:172.21.15.18:6835/3661910111] up:active 2023-09-23T20:12:56.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:55 smithi018 ceph-mon[95889]: fsmap foo:1 {0=foo.smithi018.hduiol=up:active} 1 up:standby 2023-09-23T20:12:56.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:55 smithi018 ceph-mon[95889]: from='client.? 172.21.15.18:0/2373956746' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T20:12:56.150 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:12:56.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:55 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:56.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:55 smithi149 ceph-mon[100764]: mds.? [v2:172.21.15.18:6834/3661910111,v1:172.21.15.18:6835/3661910111] up:active 2023-09-23T20:12:56.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:55 smithi149 ceph-mon[100764]: fsmap foo:1 {0=foo.smithi018.hduiol=up:active} 1 up:standby 2023-09-23T20:12:56.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:55 smithi149 ceph-mon[100764]: from='client.? 172.21.15.18:0/2373956746' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T20:12:57.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:56 smithi018 ceph-mon[109137]: pgmap v47: 73 pgs: 73 active+clean; 579 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s wr, 5 op/s 2023-09-23T20:12:57.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:56 smithi018 ceph-mon[109137]: mds.? [v2:172.21.15.149:6832/3529449999,v1:172.21.15.149:6833/3529449999] up:standby 2023-09-23T20:12:57.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:56 smithi018 ceph-mon[109137]: fsmap foo:1 {0=foo.smithi018.hduiol=up:active} 1 up:standby 2023-09-23T20:12:57.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:56 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:57.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:56 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:57.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:56 smithi018 ceph-mon[95889]: pgmap v47: 73 pgs: 73 active+clean; 579 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s wr, 5 op/s 2023-09-23T20:12:57.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:56 smithi018 ceph-mon[95889]: mds.? [v2:172.21.15.149:6832/3529449999,v1:172.21.15.149:6833/3529449999] up:standby 2023-09-23T20:12:57.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:56 smithi018 ceph-mon[95889]: fsmap foo:1 {0=foo.smithi018.hduiol=up:active} 1 up:standby 2023-09-23T20:12:57.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:56 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:57.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:56 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:57.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:56 smithi149 ceph-mon[100764]: pgmap v47: 73 pgs: 73 active+clean; 579 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s wr, 5 op/s 2023-09-23T20:12:57.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:56 smithi149 ceph-mon[100764]: mds.? [v2:172.21.15.149:6832/3529449999,v1:172.21.15.149:6833/3529449999] up:standby 2023-09-23T20:12:57.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:56 smithi149 ceph-mon[100764]: fsmap foo:1 {0=foo.smithi018.hduiol=up:active} 1 up:standby 2023-09-23T20:12:57.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:56 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:57.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:56 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:12:57.835 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:12:57.835 INFO:teuthology.orchestra.run.smithi018.stdout:{"epoch":71,"fsid":"a3b3b17e-5a4c-11ee-8db4-212e2dc638e7","created":"2023-09-23T20:06:54.359034+0000","modified":"2023-09-23T20:12:51.367725+0000","last_up_change":"2023-09-23T20:11:23.829606+0000","last_in_change":"2023-09-23T20:11:09.202130+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-09-23T20:09:47.162973+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-09-23T20:11:59.431837+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"64","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-09-23T20:12:42.254901+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-09-23T20:12:43.191578+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"69","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}}}],"osds":[{"osd":0,"uuid":"44c8aaa3-bbd1-4330-8d34-73f7a08b0d4d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6802","nonce":2209550685},{"type":"v1","addr":"172.21.15.18:6803","nonce":2209550685}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6804","nonce":2209550685},{"type":"v1","addr":"172.21.15.18:6805","nonce":2209550685}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6808","nonce":2209550685},{"type":"v1","addr":"172.21.15.18:6809","nonce":2209550685}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6806","nonce":2209550685},{"type":"v1","addr":"172.21.15.18:6807","nonce":2209550685}]},"public_addr":"172.21.15.18:6803/2209550685","cluster_addr":"172.21.15.18:6805/2209550685","heartbeat_back_addr":"172.21.15.18:6809/2209550685","heartbeat_front_addr":"172.21.15.18:6807/2209550685","state":["exists","up"]},{"osd":1,"uuid":"c0534324-49c8-4804-bced-c4d13510e74a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6810","nonce":314354451},{"type":"v1","addr":"172.21.15.18:6811","nonce":314354451}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6812","nonce":314354451},{"type":"v1","addr":"172.21.15.18:6813","nonce":314354451}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6816","nonce":314354451},{"type":"v1","addr":"172.21.15.18:6817","nonce":314354451}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6814","nonce":314354451},{"type":"v1","addr":"172.21.15.18:6815","nonce":314354451}]},"public_addr":"172.21.15.18:6811/314354451","cluster_addr":"172.21.15.18:6813/314354451","heartbeat_back_addr":"172.21.15.18:6817/314354451","heartbeat_front_addr":"172.21.15.18:6815/314354451","state":["exists","up"]},{"osd":2,"uuid":"f3b2116d-a3d5-4279-91dc-eb47458d3239","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":70,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6818","nonce":1413681698},{"type":"v1","addr":"172.21.15.18:6819","nonce":1413681698}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6820","nonce":1413681698},{"type":"v1","addr":"172.21.15.18:6821","nonce":1413681698}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6824","nonce":1413681698},{"type":"v1","addr":"172.21.15.18:6825","nonce":1413681698}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6822","nonce":1413681698},{"type":"v1","addr":"172.21.15.18:6823","nonce":1413681698}]},"public_addr":"172.21.15.18:6819/1413681698","cluster_addr":"172.21.15.18:6821/1413681698","heartbeat_back_addr":"172.21.15.18:6825/1413681698","heartbeat_front_addr":"172.21.15.18:6823/1413681698","state":["exists","up"]},{"osd":3,"uuid":"8edccb7f-c4c9-47f3-a6f5-38a1a38cb099","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6826","nonce":423024182},{"type":"v1","addr":"172.21.15.18:6827","nonce":423024182}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6828","nonce":423024182},{"type":"v1","addr":"172.21.15.18:6829","nonce":423024182}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6832","nonce":423024182},{"type":"v1","addr":"172.21.15.18:6833","nonce":423024182}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6830","nonce":423024182},{"type":"v1","addr":"172.21.15.18:6831","nonce":423024182}]},"public_addr":"172.21.15.18:6827/423024182","cluster_addr":"172.21.15.18:6829/423024182","heartbeat_back_addr":"172.21.15.18:6833/423024182","heartbeat_front_addr":"172.21.15.18:6831/423024182","state":["exists","up"]},{"osd":4,"uuid":"1d5a64ce-fe42-4355-991a-dfb20cbe5423","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":70,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6800","nonce":3640432477},{"type":"v1","addr":"172.21.15.149:6801","nonce":3640432477}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6802","nonce":3640432477},{"type":"v1","addr":"172.21.15.149:6803","nonce":3640432477}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6806","nonce":3640432477},{"type":"v1","addr":"172.21.15.149:6807","nonce":3640432477}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6804","nonce":3640432477},{"type":"v1","addr":"172.21.15.149:6805","nonce":3640432477}]},"public_addr":"172.21.15.149:6801/3640432477","cluster_addr":"172.21.15.149:6803/3640432477","heartbeat_back_addr":"172.21.15.149:6807/3640432477","heartbeat_front_addr":"172.21.15.149:6805/3640432477","state":["exists","up"]},{"osd":5,"uuid":"105ae3b2-b2fc-477d-9134-3d517bc9427e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6808","nonce":2969709367},{"type":"v1","addr":"172.21.15.149:6809","nonce":2969709367}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6810","nonce":2969709367},{"type":"v1","addr":"172.21.15.149:6811","nonce":2969709367}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6814","nonce":2969709367},{"type":"v1","addr":"172.21.15.149:6815","nonce":2969709367}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6812","nonce":2969709367},{"type":"v1","addr":"172.21.15.149:6813","nonce":2969709367}]},"public_addr":"172.21.15.149:6809/2969709367","cluster_addr":"172.21.15.149:6811/2969709367","heartbeat_back_addr":"172.21.15.149:6815/2969709367","heartbeat_front_addr":"172.21.15.149:6813/2969709367","state":["exists","up"]},{"osd":6,"uuid":"27d2d2a8-e8c6-4a05-a85e-bd54376cf4c1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6816","nonce":1291453593},{"type":"v1","addr":"172.21.15.149:6817","nonce":1291453593}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6818","nonce":1291453593},{"type":"v1","addr":"172.21.15.149:6819","nonce":1291453593}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6822","nonce":1291453593},{"type":"v1","addr":"172.21.15.149:6823","nonce":1291453593}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6820","nonce":1291453593},{"type":"v1","addr":"172.21.15.149:6821","nonce":1291453593}]},"public_addr":"172.21.15.149:6817/1291453593","cluster_addr":"172.21.15.149:6819/1291453593","heartbeat_back_addr":"172.21.15.149:6823/1291453593","heartbeat_front_addr":"172.21.15.149:6821/1291453593","state":["exists","up"]},{"osd":7,"uuid":"9da636f7-995c-4f29-9264-70d09c6f66c6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6824","nonce":3117985178},{"type":"v1","addr":"172.21.15.149:6825","nonce":3117985178}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6826","nonce":3117985178},{"type":"v1","addr":"172.21.15.149:6827","nonce":3117985178}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6830","nonce":3117985178},{"type":"v1","addr":"172.21.15.149:6831","nonce":3117985178}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6828","nonce":3117985178},{"type":"v1","addr":"172.21.15.149:6829","nonce":3117985178}]},"public_addr":"172.21.15.149:6825/3117985178","cluster_addr":"172.21.15.149:6827/3117985178","heartbeat_back_addr":"172.21.15.149:6831/3117985178","heartbeat_front_addr":"172.21.15.149:6829/3117985178","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:09:01.034520+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:09:21.795238+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:09:42.987835+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:10:04.092458+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:10:23.309112+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:10:43.348231+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:11:03.128458+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:11:23.084228+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.d","mappings":[{"from":1,"to":5}]},{"pgid":"4.f","mappings":[{"from":1,"to":2}]},{"pgid":"4.18","mappings":[{"from":1,"to":2}]}],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.18:0/3912151186":"2023-09-24T20:11:49.652762+0000","172.21.15.18:0/3863388488":"2023-09-24T20:11:49.652762+0000","172.21.15.18:6801/2611527229":"2023-09-24T20:11:49.652762+0000","172.21.15.18:6800/2611527229":"2023-09-24T20:11:49.652762+0000","172.21.15.18:0/1698512393":"2023-09-24T20:11:49.652762+0000","172.21.15.18:6801/380813540":"2023-09-24T20:07:48.635784+0000","172.21.15.18:6800/380813540":"2023-09-24T20:07:48.635784+0000","172.21.15.18:0/3831267686":"2023-09-24T20:07:48.635784+0000","172.21.15.18:0/950924496":"2023-09-24T20:07:23.919815+0000","172.21.15.18:0/1988472148":"2023-09-24T20:11:49.652762+0000","172.21.15.18:0/1082665289":"2023-09-24T20:07:48.635784+0000","172.21.15.18:0/2993111051":"2023-09-24T20:07:23.919815+0000","172.21.15.18:0/69122222":"2023-09-24T20:07:23.919815+0000","172.21.15.18:0/1645667747":"2023-09-24T20:07:48.635784+0000","172.21.15.18:6800/1930027051":"2023-09-24T20:07:23.919815+0000","172.21.15.18:6801/1930027051":"2023-09-24T20:07:23.919815+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-23T20:12:58.294 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph tell osd.0 flush_pg_stats 2023-09-23T20:12:58.295 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph tell osd.1 flush_pg_stats 2023-09-23T20:12:58.295 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph tell osd.2 flush_pg_stats 2023-09-23T20:12:58.296 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph tell osd.3 flush_pg_stats 2023-09-23T20:12:58.296 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph tell osd.4 flush_pg_stats 2023-09-23T20:12:58.296 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph tell osd.5 flush_pg_stats 2023-09-23T20:12:58.297 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph tell osd.6 flush_pg_stats 2023-09-23T20:12:58.297 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph tell osd.7 flush_pg_stats 2023-09-23T20:12:59.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:58 smithi018 ceph-mon[109137]: pgmap v48: 73 pgs: 73 active+clean; 579 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s wr, 5 op/s 2023-09-23T20:12:59.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:12:58 smithi018 ceph-mon[109137]: from='client.? 172.21.15.18:0/3107448954' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T20:12:59.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:58 smithi018 ceph-mon[95889]: pgmap v48: 73 pgs: 73 active+clean; 579 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s wr, 5 op/s 2023-09-23T20:12:59.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:12:58 smithi018 ceph-mon[95889]: from='client.? 172.21.15.18:0/3107448954' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T20:12:59.209 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:12:59.215 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:58 smithi149 ceph-mon[100764]: pgmap v48: 73 pgs: 73 active+clean; 579 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 1.6 KiB/s wr, 5 op/s 2023-09-23T20:12:59.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:12:58 smithi149 ceph-mon[100764]: from='client.? 172.21.15.18:0/3107448954' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T20:12:59.256 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:12:59.472 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:12:59.520 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:12:59 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:12:59] "GET /metrics HTTP/1.1" 200 208026 "" "Prometheus/2.43.0" 2023-09-23T20:12:59.522 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:12:59.710 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:12:59.741 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:13:00.499 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:13:00.499 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:13:00.640 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:13:00 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:13:00.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:13:00 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:13:00.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:13:00 smithi018 ceph-mon[95889]: pgmap v49: 73 pgs: 73 active+clean; 579 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s wr, 4 op/s 2023-09-23T20:13:00.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:13:00 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:13:00.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:13:00 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:13:00.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:13:00 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:13:00.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:13:00 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:13:00.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:13:00 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:13:00.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:13:00 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:13:00.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:13:00 smithi018 ceph-mon[109137]: pgmap v49: 73 pgs: 73 active+clean; 579 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s wr, 4 op/s 2023-09-23T20:13:00.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:13:00 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:13:00.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:13:00 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:13:00.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:13:00 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:13:00.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:13:00 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:13:00.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:13:00 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:13:00.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:13:00 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:13:00.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:13:00 smithi149 ceph-mon[100764]: pgmap v49: 73 pgs: 73 active+clean; 579 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s wr, 4 op/s 2023-09-23T20:13:00.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:13:00 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:13:00.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:13:00 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:13:00.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:13:00 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:13:00.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:13:00 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:13:03.011 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:13:02 smithi018 ceph-mon[95889]: pgmap v50: 73 pgs: 73 active+clean; 579 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-09-23T20:13:03.012 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:13:02 smithi018 ceph-mon[109137]: pgmap v50: 73 pgs: 73 active+clean; 579 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-09-23T20:13:03.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:13:02 smithi149 ceph-mon[100764]: pgmap v50: 73 pgs: 73 active+clean; 579 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-09-23T20:13:04.216 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:13:03 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[102880]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:13:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T20:13:04.408 INFO:teuthology.orchestra.run.smithi018.stdout:77309411371 2023-09-23T20:13:04.409 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.2 2023-09-23T20:13:04.597 INFO:teuthology.orchestra.run.smithi018.stdout:34359738418 2023-09-23T20:13:04.598 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.0 2023-09-23T20:13:04.991 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:13:04 smithi018 ceph-mon[109137]: pgmap v51: 73 pgs: 73 active+clean; 579 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-09-23T20:13:04.991 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:13:04 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:13:04.991 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:13:04 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:13:04.991 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:13:04 smithi018 ceph-mon[95889]: pgmap v51: 73 pgs: 73 active+clean; 579 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-09-23T20:13:04.992 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:13:04 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:13:04.992 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:13:04 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:13:05.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:13:04 smithi149 ceph-mon[100764]: pgmap v51: 73 pgs: 73 active+clean; 579 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-09-23T20:13:05.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:13:04 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:13:05.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:13:04 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:13:05.340 INFO:teuthology.orchestra.run.smithi018.stdout:167503724575 2023-09-23T20:13:05.341 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.5 2023-09-23T20:13:05.657 INFO:teuthology.orchestra.run.smithi018.stdout:128849018914 2023-09-23T20:13:05.657 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.4 2023-09-23T20:13:05.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:13:05 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:13:05.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:13:05 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:13:05.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:13:05 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:13:05.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:13:05 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:13:06.105 INFO:teuthology.orchestra.run.smithi018.stdout:201863462938 2023-09-23T20:13:06.105 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.6 2023-09-23T20:13:06.215 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:13:05 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:13:06.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:13:05 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:13:06.789 INFO:teuthology.orchestra.run.smithi018.stdout:107374182437 2023-09-23T20:13:06.789 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.3 2023-09-23T20:13:07.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:13:06 smithi018 ceph-mon[95889]: pgmap v52: 73 pgs: 73 active+clean; 579 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-09-23T20:13:07.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:13:06 smithi018 ceph-mon[109137]: pgmap v52: 73 pgs: 73 active+clean; 579 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-09-23T20:13:07.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:13:06 smithi149 ceph-mon[100764]: pgmap v52: 73 pgs: 73 active+clean; 579 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-09-23T20:13:07.435 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:13:07.500 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:13:07.525 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:13:07.543 INFO:teuthology.orchestra.run.smithi018.stdout:55834574896 2023-09-23T20:13:07.544 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.1 2023-09-23T20:13:07.755 INFO:teuthology.orchestra.run.smithi018.stdout:236223201303 2023-09-23T20:13:07.756 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.7 2023-09-23T20:13:07.762 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:13:08.044 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:13:08.717 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:13:08.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:13:08 smithi018 ceph-mon[109137]: pgmap v53: 73 pgs: 73 active+clean; 579 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:13:08.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:13:08 smithi018 ceph-mon[95889]: pgmap v53: 73 pgs: 73 active+clean; 579 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:13:09.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:13:08 smithi149 ceph-mon[100764]: pgmap v53: 73 pgs: 73 active+clean; 579 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:13:09.601 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:13:09 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:13:09] "GET /metrics HTTP/1.1" 200 238782 "" "Prometheus/2.43.0" 2023-09-23T20:13:10.121 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:13:09 smithi018 ceph-mon[109137]: pgmap v54: 73 pgs: 73 active+clean; 579 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:13:10.121 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:13:09 smithi018 ceph-mon[95889]: pgmap v54: 73 pgs: 73 active+clean; 579 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:13:10.161 INFO:teuthology.orchestra.run.smithi018.stdout:34359738419 2023-09-23T20:13:10.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:13:09 smithi149 ceph-mon[100764]: pgmap v54: 73 pgs: 73 active+clean; 579 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:13:11.032 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738418 got 34359738419 for osd.0 2023-09-23T20:13:11.032 DEBUG:teuthology.parallel:result is None 2023-09-23T20:13:11.045 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:13:10 smithi018 ceph-mon[95889]: from='client.? 172.21.15.18:0/109603822' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T20:13:11.046 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:13:10 smithi018 ceph-mon[109137]: from='client.? 172.21.15.18:0/109603822' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T20:13:11.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:13:10 smithi149 ceph-mon[100764]: from='client.? 172.21.15.18:0/109603822' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T20:13:11.586 INFO:teuthology.orchestra.run.smithi018.stdout:77309411372 2023-09-23T20:13:11.606 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:13:12.131 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:13:11 smithi018 ceph-mon[95889]: from='client.? 172.21.15.18:0/2556527585' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T20:13:12.131 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:13:11 smithi018 ceph-mon[95889]: pgmap v55: 73 pgs: 73 active+clean; 579 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:13:12.131 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:13:11 smithi018 ceph-mon[109137]: from='client.? 172.21.15.18:0/2556527585' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T20:13:12.131 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:13:11 smithi018 ceph-mon[109137]: pgmap v55: 73 pgs: 73 active+clean; 579 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:13:12.151 INFO:teuthology.orchestra.run.smithi018.stdout:128849018916 2023-09-23T20:13:12.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:13:11 smithi149 ceph-mon[100764]: from='client.? 172.21.15.18:0/2556527585' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T20:13:12.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:13:11 smithi149 ceph-mon[100764]: pgmap v55: 73 pgs: 73 active+clean; 579 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:13:12.479 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411371 got 77309411372 for osd.2 2023-09-23T20:13:12.479 DEBUG:teuthology.parallel:result is None 2023-09-23T20:13:12.568 INFO:teuthology.orchestra.run.smithi018.stdout:107374182439 2023-09-23T20:13:12.581 INFO:teuthology.orchestra.run.smithi018.stdout:167503724577 2023-09-23T20:13:12.719 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:13:12.747 INFO:teuthology.orchestra.run.smithi018.stdout:201863462940 2023-09-23T20:13:13.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:13:12 smithi018 ceph-mon[109137]: from='client.? 172.21.15.18:0/3326245793' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T20:13:13.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:13:12 smithi018 ceph-mon[109137]: from='client.? 172.21.15.18:0/3115810785' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T20:13:13.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:13:12 smithi018 ceph-mon[109137]: from='client.? 172.21.15.18:0/2650819719' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T20:13:13.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:13:12 smithi018 ceph-mon[109137]: from='client.? 172.21.15.18:0/2108083135' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T20:13:13.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:13:12 smithi018 ceph-mon[95889]: from='client.? 172.21.15.18:0/3326245793' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T20:13:13.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:13:12 smithi018 ceph-mon[95889]: from='client.? 172.21.15.18:0/3115810785' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T20:13:13.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:13:12 smithi018 ceph-mon[95889]: from='client.? 172.21.15.18:0/2650819719' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T20:13:13.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:13:12 smithi018 ceph-mon[95889]: from='client.? 172.21.15.18:0/2108083135' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T20:13:13.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:13:12 smithi149 ceph-mon[100764]: from='client.? 172.21.15.18:0/3326245793' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T20:13:13.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:13:12 smithi149 ceph-mon[100764]: from='client.? 172.21.15.18:0/3115810785' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T20:13:13.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:13:12 smithi149 ceph-mon[100764]: from='client.? 172.21.15.18:0/2650819719' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T20:13:13.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:13:12 smithi149 ceph-mon[100764]: from='client.? 172.21.15.18:0/2108083135' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T20:13:14.017 INFO:tasks.cephadm.ceph_manager.ceph:need seq 107374182437 got 107374182439 for osd.3 2023-09-23T20:13:14.018 DEBUG:teuthology.parallel:result is None 2023-09-23T20:13:14.047 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849018914 got 128849018916 for osd.4 2023-09-23T20:13:14.047 DEBUG:teuthology.parallel:result is None 2023-09-23T20:13:14.130 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:13:13 smithi018 ceph-mon[109137]: pgmap v56: 73 pgs: 73 active+clean; 579 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:13:14.131 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:13:13 smithi018 ceph-mon[95889]: pgmap v56: 73 pgs: 73 active+clean; 579 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:13:14.161 INFO:tasks.cephadm.ceph_manager.ceph:need seq 201863462938 got 201863462940 for osd.6 2023-09-23T20:13:14.161 DEBUG:teuthology.parallel:result is None 2023-09-23T20:13:14.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:13:13 smithi149 ceph-mon[100764]: pgmap v56: 73 pgs: 73 active+clean; 579 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:13:14.216 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:13:13 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[102880]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:13:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T20:13:14.321 INFO:tasks.cephadm.ceph_manager.ceph:need seq 167503724575 got 167503724577 for osd.5 2023-09-23T20:13:14.322 DEBUG:teuthology.parallel:result is None 2023-09-23T20:13:15.285 INFO:teuthology.orchestra.run.smithi018.stdout:55834574897 2023-09-23T20:13:15.403 INFO:teuthology.orchestra.run.smithi018.stdout:236223201304 2023-09-23T20:13:15.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:13:15 smithi018 ceph-mon[109137]: from='client.? 172.21.15.18:0/2754358851' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T20:13:15.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:13:15 smithi018 ceph-mon[95889]: from='client.? 172.21.15.18:0/2754358851' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T20:13:15.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:13:15 smithi149 ceph-mon[100764]: from='client.? 172.21.15.18:0/2754358851' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T20:13:15.872 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574896 got 55834574897 for osd.1 2023-09-23T20:13:15.872 DEBUG:teuthology.parallel:result is None 2023-09-23T20:13:16.076 INFO:tasks.cephadm.ceph_manager.ceph:need seq 236223201303 got 236223201304 for osd.7 2023-09-23T20:13:16.076 DEBUG:teuthology.parallel:result is None 2023-09-23T20:13:16.076 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2023-09-23T20:13:16.076 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:13:16.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:13:16 smithi018 ceph-mon[95889]: from='client.? 172.21.15.18:0/1041603218' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T20:13:16.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:13:16 smithi018 ceph-mon[95889]: pgmap v57: 73 pgs: 73 active+clean; 579 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:13:16.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:13:16 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:13:16.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:13:16 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:13:16.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:13:16 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:13:16.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:13:16 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:13:16.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:13:16 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:13:16.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:13:16 smithi018 ceph-mon[109137]: from='client.? 172.21.15.18:0/1041603218' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T20:13:16.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:13:16 smithi018 ceph-mon[109137]: pgmap v57: 73 pgs: 73 active+clean; 579 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:13:16.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:13:16 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:13:16.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:13:16 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:13:16.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:13:16 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:13:16.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:13:16 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:13:16.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:13:16 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:13:16.705 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:13:16.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:13:16 smithi149 ceph-mon[100764]: from='client.? 172.21.15.18:0/1041603218' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T20:13:16.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:13:16 smithi149 ceph-mon[100764]: pgmap v57: 73 pgs: 73 active+clean; 579 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:13:16.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:13:16 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:13:16.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:13:16 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:13:16.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:13:16 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:13:16.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:13:16 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:13:16.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:13:16 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:13:18.100 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:13:18.101 INFO:teuthology.orchestra.run.smithi018.stderr:dumped all 2023-09-23T20:13:18.454 INFO:teuthology.orchestra.run.smithi018.stdout:{"pg_ready":true,"pg_map":{"version":58,"stamp":"2023-09-23T20:13:17.683152+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592674,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":224,"num_write_kb":2429,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":167,"ondisk_log_size":167,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":52840,"kb_used_data":7528,"kb_used_omap":0,"kb_used_meta":45248,"kb_avail":749908376,"statfs":{"total":767960285184,"available":767906177024,"internally_reserved":0,"allocated":7708672,"data_stored":4318723,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":46333952},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002789"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":23,"reported_epoch":71,"state":"active+clean","last_fresh":"2023-09-23T20:12:51.376081+0000","last_change":"2023-09-23T20:12:51.376081+0000","last_active":"2023-09-23T20:12:51.376081+0000","last_peered":"2023-09-23T20:12:51.376081+0000","last_clean":"2023-09-23T20:12:51.376081+0000","last_became_active":"2023-09-23T20:12:51.375807+0000","last_became_peered":"2023-09-23T20:12:51.375807+0000","last_unstale":"2023-09-23T20:12:51.376081+0000","last_undegraded":"2023-09-23T20:12:51.376081+0000","last_fullsized":"2023-09-23T20:12:51.376081+0000","mapping_epoch":70,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:29:58.345734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.369615+0000","last_change":"2023-09-23T20:12:43.749326+0000","last_active":"2023-09-23T20:12:50.369615+0000","last_peered":"2023-09-23T20:12:50.369615+0000","last_clean":"2023-09-23T20:12:50.369615+0000","last_became_active":"2023-09-23T20:12:43.749020+0000","last_became_peered":"2023-09-23T20:12:43.749020+0000","last_unstale":"2023-09-23T20:12:50.369615+0000","last_undegraded":"2023-09-23T20:12:50.369615+0000","last_fullsized":"2023-09-23T20:12:50.369615+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":4,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:36:42.363869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.374420+0000","last_change":"2023-09-23T20:12:44.770095+0000","last_active":"2023-09-23T20:12:50.374420+0000","last_peered":"2023-09-23T20:12:50.374420+0000","last_clean":"2023-09-23T20:12:50.374420+0000","last_became_active":"2023-09-23T20:12:44.769778+0000","last_became_peered":"2023-09-23T20:12:44.769778+0000","last_unstale":"2023-09-23T20:12:50.374420+0000","last_undegraded":"2023-09-23T20:12:50.374420+0000","last_fullsized":"2023-09-23T20:12:50.374420+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:31:22.309973+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.374743+0000","last_change":"2023-09-23T20:12:43.755660+0000","last_active":"2023-09-23T20:12:50.374743+0000","last_peered":"2023-09-23T20:12:50.374743+0000","last_clean":"2023-09-23T20:12:50.374743+0000","last_became_active":"2023-09-23T20:12:43.755312+0000","last_became_peered":"2023-09-23T20:12:43.755312+0000","last_unstale":"2023-09-23T20:12:50.374743+0000","last_undegraded":"2023-09-23T20:12:50.374743+0000","last_fullsized":"2023-09-23T20:12:50.374743+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:48:38.650179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.371355+0000","last_change":"2023-09-23T20:12:43.903122+0000","last_active":"2023-09-23T20:12:50.371355+0000","last_peered":"2023-09-23T20:12:50.371355+0000","last_clean":"2023-09-23T20:12:50.371355+0000","last_became_active":"2023-09-23T20:12:43.902816+0000","last_became_peered":"2023-09-23T20:12:43.902816+0000","last_unstale":"2023-09-23T20:12:50.371355+0000","last_undegraded":"2023-09-23T20:12:50.371355+0000","last_fullsized":"2023-09-23T20:12:50.371355+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:31:11.122636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":16,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.374209+0000","last_change":"2023-09-23T20:12:43.753383+0000","last_active":"2023-09-23T20:12:50.374209+0000","last_peered":"2023-09-23T20:12:50.374209+0000","last_clean":"2023-09-23T20:12:50.374209+0000","last_became_active":"2023-09-23T20:12:43.752921+0000","last_became_peered":"2023-09-23T20:12:43.752921+0000","last_unstale":"2023-09-23T20:12:50.374209+0000","last_undegraded":"2023-09-23T20:12:50.374209+0000","last_fullsized":"2023-09-23T20:12:50.374209+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:28:26.982998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.371877+0000","last_change":"2023-09-23T20:12:44.041322+0000","last_active":"2023-09-23T20:12:50.371877+0000","last_peered":"2023-09-23T20:12:50.371877+0000","last_clean":"2023-09-23T20:12:50.371877+0000","last_became_active":"2023-09-23T20:12:44.041028+0000","last_became_peered":"2023-09-23T20:12:44.041028+0000","last_unstale":"2023-09-23T20:12:50.371877+0000","last_undegraded":"2023-09-23T20:12:50.371877+0000","last_fullsized":"2023-09-23T20:12:50.371877+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:56:27.407493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":16,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.374018+0000","last_change":"2023-09-23T20:12:43.753359+0000","last_active":"2023-09-23T20:12:50.374018+0000","last_peered":"2023-09-23T20:12:50.374018+0000","last_clean":"2023-09-23T20:12:50.374018+0000","last_became_active":"2023-09-23T20:12:43.753064+0000","last_became_peered":"2023-09-23T20:12:43.753064+0000","last_unstale":"2023-09-23T20:12:50.374018+0000","last_undegraded":"2023-09-23T20:12:50.374018+0000","last_fullsized":"2023-09-23T20:12:50.374018+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:30:02.892722+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.369369+0000","last_change":"2023-09-23T20:12:44.770501+0000","last_active":"2023-09-23T20:12:50.369369+0000","last_peered":"2023-09-23T20:12:50.369369+0000","last_clean":"2023-09-23T20:12:50.369369+0000","last_became_active":"2023-09-23T20:12:44.770244+0000","last_became_peered":"2023-09-23T20:12:44.770244+0000","last_unstale":"2023-09-23T20:12:50.369369+0000","last_undegraded":"2023-09-23T20:12:50.369369+0000","last_fullsized":"2023-09-23T20:12:50.369369+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:32:03.174269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.369091+0000","last_change":"2023-09-23T20:12:43.750365+0000","last_active":"2023-09-23T20:12:50.369091+0000","last_peered":"2023-09-23T20:12:50.369091+0000","last_clean":"2023-09-23T20:12:50.369091+0000","last_became_active":"2023-09-23T20:12:43.750100+0000","last_became_peered":"2023-09-23T20:12:43.750100+0000","last_unstale":"2023-09-23T20:12:50.369091+0000","last_undegraded":"2023-09-23T20:12:50.369091+0000","last_fullsized":"2023-09-23T20:12:50.369091+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:34:57.785225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.371346+0000","last_change":"2023-09-23T20:12:43.765908+0000","last_active":"2023-09-23T20:12:50.371346+0000","last_peered":"2023-09-23T20:12:50.371346+0000","last_clean":"2023-09-23T20:12:50.371346+0000","last_became_active":"2023-09-23T20:12:43.765558+0000","last_became_peered":"2023-09-23T20:12:43.765558+0000","last_unstale":"2023-09-23T20:12:50.371346+0000","last_undegraded":"2023-09-23T20:12:50.371346+0000","last_fullsized":"2023-09-23T20:12:50.371346+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:35:43.809898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2023-09-23T20:12:51.517376+0000","last_change":"2023-09-23T20:12:43.751685+0000","last_active":"2023-09-23T20:12:51.517376+0000","last_peered":"2023-09-23T20:12:51.517376+0000","last_clean":"2023-09-23T20:12:51.517376+0000","last_became_active":"2023-09-23T20:12:43.751413+0000","last_became_peered":"2023-09-23T20:12:43.751413+0000","last_unstale":"2023-09-23T20:12:51.517376+0000","last_undegraded":"2023-09-23T20:12:51.517376+0000","last_fullsized":"2023-09-23T20:12:51.517376+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:05:57.982527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":17,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.373501+0000","last_change":"2023-09-23T20:12:00.850675+0000","last_active":"2023-09-23T20:12:50.373501+0000","last_peered":"2023-09-23T20:12:50.373501+0000","last_clean":"2023-09-23T20:12:50.373501+0000","last_became_active":"2023-09-23T20:12:00.850315+0000","last_became_peered":"2023-09-23T20:12:00.850315+0000","last_unstale":"2023-09-23T20:12:50.373501+0000","last_undegraded":"2023-09-23T20:12:50.373501+0000","last_fullsized":"2023-09-23T20:12:50.373501+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:03:33.446629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.374215+0000","last_change":"2023-09-23T20:12:45.180160+0000","last_active":"2023-09-23T20:12:50.374215+0000","last_peered":"2023-09-23T20:12:50.374215+0000","last_clean":"2023-09-23T20:12:50.374215+0000","last_became_active":"2023-09-23T20:12:45.179718+0000","last_became_peered":"2023-09-23T20:12:45.179718+0000","last_unstale":"2023-09-23T20:12:50.374215+0000","last_undegraded":"2023-09-23T20:12:50.374215+0000","last_fullsized":"2023-09-23T20:12:50.374215+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:40:30.530253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.373735+0000","last_change":"2023-09-23T20:12:01.062560+0000","last_active":"2023-09-23T20:12:50.373735+0000","last_peered":"2023-09-23T20:12:50.373735+0000","last_clean":"2023-09-23T20:12:50.373735+0000","last_became_active":"2023-09-23T20:12:01.062216+0000","last_became_peered":"2023-09-23T20:12:01.062216+0000","last_unstale":"2023-09-23T20:12:50.373735+0000","last_undegraded":"2023-09-23T20:12:50.373735+0000","last_fullsized":"2023-09-23T20:12:50.373735+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:01:52.795311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":16,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.374853+0000","last_change":"2023-09-23T20:12:43.760973+0000","last_active":"2023-09-23T20:12:50.374853+0000","last_peered":"2023-09-23T20:12:50.374853+0000","last_clean":"2023-09-23T20:12:50.374853+0000","last_became_active":"2023-09-23T20:12:43.760442+0000","last_became_peered":"2023-09-23T20:12:43.760442+0000","last_unstale":"2023-09-23T20:12:50.374853+0000","last_undegraded":"2023-09-23T20:12:50.374853+0000","last_fullsized":"2023-09-23T20:12:50.374853+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:47:10.459217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.369002+0000","last_change":"2023-09-23T20:12:45.178571+0000","last_active":"2023-09-23T20:12:50.369002+0000","last_peered":"2023-09-23T20:12:50.369002+0000","last_clean":"2023-09-23T20:12:50.369002+0000","last_became_active":"2023-09-23T20:12:45.178269+0000","last_became_peered":"2023-09-23T20:12:45.178269+0000","last_unstale":"2023-09-23T20:12:50.369002+0000","last_undegraded":"2023-09-23T20:12:50.369002+0000","last_fullsized":"2023-09-23T20:12:50.369002+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:09:15.260586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.373597+0000","last_change":"2023-09-23T20:12:43.750138+0000","last_active":"2023-09-23T20:12:50.373597+0000","last_peered":"2023-09-23T20:12:50.373597+0000","last_clean":"2023-09-23T20:12:50.373597+0000","last_became_active":"2023-09-23T20:12:43.749812+0000","last_became_peered":"2023-09-23T20:12:43.749812+0000","last_unstale":"2023-09-23T20:12:50.373597+0000","last_undegraded":"2023-09-23T20:12:50.373597+0000","last_fullsized":"2023-09-23T20:12:50.373597+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:54:32.020961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":17,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.370668+0000","last_change":"2023-09-23T20:12:01.062442+0000","last_active":"2023-09-23T20:12:50.370668+0000","last_peered":"2023-09-23T20:12:50.370668+0000","last_clean":"2023-09-23T20:12:50.370668+0000","last_became_active":"2023-09-23T20:12:01.062047+0000","last_became_peered":"2023-09-23T20:12:01.062047+0000","last_unstale":"2023-09-23T20:12:50.370668+0000","last_undegraded":"2023-09-23T20:12:50.370668+0000","last_fullsized":"2023-09-23T20:12:50.370668+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:58:49.032117+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.369247+0000","last_change":"2023-09-23T20:12:44.772724+0000","last_active":"2023-09-23T20:12:50.369247+0000","last_peered":"2023-09-23T20:12:50.369247+0000","last_clean":"2023-09-23T20:12:50.369247+0000","last_became_active":"2023-09-23T20:12:44.772404+0000","last_became_peered":"2023-09-23T20:12:44.772404+0000","last_unstale":"2023-09-23T20:12:50.369247+0000","last_undegraded":"2023-09-23T20:12:50.369247+0000","last_fullsized":"2023-09-23T20:12:50.369247+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:12:45.284142+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.368740+0000","last_change":"2023-09-23T20:12:43.756939+0000","last_active":"2023-09-23T20:12:50.368740+0000","last_peered":"2023-09-23T20:12:50.368740+0000","last_clean":"2023-09-23T20:12:50.368740+0000","last_became_active":"2023-09-23T20:12:43.756632+0000","last_became_peered":"2023-09-23T20:12:43.756632+0000","last_unstale":"2023-09-23T20:12:50.368740+0000","last_undegraded":"2023-09-23T20:12:50.368740+0000","last_fullsized":"2023-09-23T20:12:50.368740+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:13:19.514304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":17,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.370283+0000","last_change":"2023-09-23T20:12:00.848951+0000","last_active":"2023-09-23T20:12:50.370283+0000","last_peered":"2023-09-23T20:12:50.370283+0000","last_clean":"2023-09-23T20:12:50.370283+0000","last_became_active":"2023-09-23T20:12:00.848720+0000","last_became_peered":"2023-09-23T20:12:00.848720+0000","last_unstale":"2023-09-23T20:12:50.370283+0000","last_undegraded":"2023-09-23T20:12:50.370283+0000","last_fullsized":"2023-09-23T20:12:50.370283+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:45:24.720986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.373396+0000","last_change":"2023-09-23T20:12:45.609829+0000","last_active":"2023-09-23T20:12:50.373396+0000","last_peered":"2023-09-23T20:12:50.373396+0000","last_clean":"2023-09-23T20:12:50.373396+0000","last_became_active":"2023-09-23T20:12:45.609467+0000","last_became_peered":"2023-09-23T20:12:45.609467+0000","last_unstale":"2023-09-23T20:12:50.373396+0000","last_undegraded":"2023-09-23T20:12:50.373396+0000","last_fullsized":"2023-09-23T20:12:50.373396+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:27:31.490555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.374536+0000","last_change":"2023-09-23T20:12:43.759474+0000","last_active":"2023-09-23T20:12:50.374536+0000","last_peered":"2023-09-23T20:12:50.374536+0000","last_clean":"2023-09-23T20:12:50.374536+0000","last_became_active":"2023-09-23T20:12:43.759191+0000","last_became_peered":"2023-09-23T20:12:43.759191+0000","last_unstale":"2023-09-23T20:12:50.374536+0000","last_undegraded":"2023-09-23T20:12:50.374536+0000","last_fullsized":"2023-09-23T20:12:50.374536+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:54:48.352278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"60'128","reported_seq":279,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.370262+0000","last_change":"2023-09-23T20:11:27.849030+0000","last_active":"2023-09-23T20:12:50.370262+0000","last_peered":"2023-09-23T20:12:50.370262+0000","last_clean":"2023-09-23T20:12:50.370262+0000","last_became_active":"2023-09-23T20:11:27.848678+0000","last_became_peered":"2023-09-23T20:11:27.848678+0000","last_unstale":"2023-09-23T20:12:50.370262+0000","last_undegraded":"2023-09-23T20:12:50.370262+0000","last_fullsized":"2023-09-23T20:12:50.370262+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:09:47.213130+0000","objects_scrubbed":0,"log_size":128,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:33:53.621689+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":17,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.373844+0000","last_change":"2023-09-23T20:12:00.851792+0000","last_active":"2023-09-23T20:12:50.373844+0000","last_peered":"2023-09-23T20:12:50.373844+0000","last_clean":"2023-09-23T20:12:50.373844+0000","last_became_active":"2023-09-23T20:12:00.851408+0000","last_became_peered":"2023-09-23T20:12:00.851408+0000","last_unstale":"2023-09-23T20:12:50.373844+0000","last_undegraded":"2023-09-23T20:12:50.373844+0000","last_fullsized":"2023-09-23T20:12:50.373844+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:19:26.564015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.373631+0000","last_change":"2023-09-23T20:12:44.767464+0000","last_active":"2023-09-23T20:12:50.373631+0000","last_peered":"2023-09-23T20:12:50.373631+0000","last_clean":"2023-09-23T20:12:50.373631+0000","last_became_active":"2023-09-23T20:12:44.767179+0000","last_became_peered":"2023-09-23T20:12:44.767179+0000","last_unstale":"2023-09-23T20:12:50.373631+0000","last_undegraded":"2023-09-23T20:12:50.373631+0000","last_fullsized":"2023-09-23T20:12:50.373631+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:58:39.324206+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.373819+0000","last_change":"2023-09-23T20:12:43.751903+0000","last_active":"2023-09-23T20:12:50.373819+0000","last_peered":"2023-09-23T20:12:50.373819+0000","last_clean":"2023-09-23T20:12:50.373819+0000","last_became_active":"2023-09-23T20:12:43.751617+0000","last_became_peered":"2023-09-23T20:12:43.751617+0000","last_unstale":"2023-09-23T20:12:50.373819+0000","last_undegraded":"2023-09-23T20:12:50.373819+0000","last_fullsized":"2023-09-23T20:12:50.373819+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:55:05.695074+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":17,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.369252+0000","last_change":"2023-09-23T20:12:01.010922+0000","last_active":"2023-09-23T20:12:50.369252+0000","last_peered":"2023-09-23T20:12:50.369252+0000","last_clean":"2023-09-23T20:12:50.369252+0000","last_became_active":"2023-09-23T20:12:01.010591+0000","last_became_peered":"2023-09-23T20:12:01.010591+0000","last_unstale":"2023-09-23T20:12:50.369252+0000","last_undegraded":"2023-09-23T20:12:50.369252+0000","last_fullsized":"2023-09-23T20:12:50.369252+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:11:25.451578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.373485+0000","last_change":"2023-09-23T20:12:44.769014+0000","last_active":"2023-09-23T20:12:50.373485+0000","last_peered":"2023-09-23T20:12:50.373485+0000","last_clean":"2023-09-23T20:12:50.373485+0000","last_became_active":"2023-09-23T20:12:44.768697+0000","last_became_peered":"2023-09-23T20:12:44.768697+0000","last_unstale":"2023-09-23T20:12:50.373485+0000","last_undegraded":"2023-09-23T20:12:50.373485+0000","last_fullsized":"2023-09-23T20:12:50.373485+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:15:05.830365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.371771+0000","last_change":"2023-09-23T20:12:43.756068+0000","last_active":"2023-09-23T20:12:50.371771+0000","last_peered":"2023-09-23T20:12:50.371771+0000","last_clean":"2023-09-23T20:12:50.371771+0000","last_became_active":"2023-09-23T20:12:43.755372+0000","last_became_peered":"2023-09-23T20:12:43.755372+0000","last_unstale":"2023-09-23T20:12:50.371771+0000","last_undegraded":"2023-09-23T20:12:50.371771+0000","last_fullsized":"2023-09-23T20:12:50.371771+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:26:33.645748+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":17,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.373753+0000","last_change":"2023-09-23T20:12:00.851594+0000","last_active":"2023-09-23T20:12:50.373753+0000","last_peered":"2023-09-23T20:12:50.373753+0000","last_clean":"2023-09-23T20:12:50.373753+0000","last_became_active":"2023-09-23T20:12:00.851256+0000","last_became_peered":"2023-09-23T20:12:00.851256+0000","last_unstale":"2023-09-23T20:12:50.373753+0000","last_undegraded":"2023-09-23T20:12:50.373753+0000","last_fullsized":"2023-09-23T20:12:50.373753+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:12:00.121999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.373774+0000","last_change":"2023-09-23T20:12:45.610141+0000","last_active":"2023-09-23T20:12:50.373774+0000","last_peered":"2023-09-23T20:12:50.373774+0000","last_clean":"2023-09-23T20:12:50.373774+0000","last_became_active":"2023-09-23T20:12:45.609753+0000","last_became_peered":"2023-09-23T20:12:45.609753+0000","last_unstale":"2023-09-23T20:12:50.373774+0000","last_undegraded":"2023-09-23T20:12:50.373774+0000","last_fullsized":"2023-09-23T20:12:50.373774+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:19:21.663951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.374143+0000","last_change":"2023-09-23T20:12:43.758600+0000","last_active":"2023-09-23T20:12:50.374143+0000","last_peered":"2023-09-23T20:12:50.374143+0000","last_clean":"2023-09-23T20:12:50.374143+0000","last_became_active":"2023-09-23T20:12:43.758205+0000","last_became_peered":"2023-09-23T20:12:43.758205+0000","last_unstale":"2023-09-23T20:12:50.374143+0000","last_undegraded":"2023-09-23T20:12:50.374143+0000","last_fullsized":"2023-09-23T20:12:50.374143+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:10:08.802544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.373386+0000","last_change":"2023-09-23T20:12:00.847882+0000","last_active":"2023-09-23T20:12:50.373386+0000","last_peered":"2023-09-23T20:12:50.373386+0000","last_clean":"2023-09-23T20:12:50.373386+0000","last_became_active":"2023-09-23T20:12:00.847511+0000","last_became_peered":"2023-09-23T20:12:00.847511+0000","last_unstale":"2023-09-23T20:12:50.373386+0000","last_undegraded":"2023-09-23T20:12:50.373386+0000","last_fullsized":"2023-09-23T20:12:50.373386+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:52:12.888646+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":10,"reported_epoch":71,"state":"active+clean","last_fresh":"2023-09-23T20:12:51.376014+0000","last_change":"2023-09-23T20:12:51.376014+0000","last_active":"2023-09-23T20:12:51.376014+0000","last_peered":"2023-09-23T20:12:51.376014+0000","last_clean":"2023-09-23T20:12:51.376014+0000","last_became_active":"2023-09-23T20:12:51.375741+0000","last_became_peered":"2023-09-23T20:12:51.375741+0000","last_unstale":"2023-09-23T20:12:51.376014+0000","last_undegraded":"2023-09-23T20:12:51.376014+0000","last_fullsized":"2023-09-23T20:12:51.376014+0000","mapping_epoch":70,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:44:48.074944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.374306+0000","last_change":"2023-09-23T20:12:43.762681+0000","last_active":"2023-09-23T20:12:50.374306+0000","last_peered":"2023-09-23T20:12:50.374306+0000","last_clean":"2023-09-23T20:12:50.374306+0000","last_became_active":"2023-09-23T20:12:43.761957+0000","last_became_peered":"2023-09-23T20:12:43.761957+0000","last_unstale":"2023-09-23T20:12:50.374306+0000","last_undegraded":"2023-09-23T20:12:50.374306+0000","last_fullsized":"2023-09-23T20:12:50.374306+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:42:47.159570+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.371787+0000","last_change":"2023-09-23T20:12:44.041342+0000","last_active":"2023-09-23T20:12:50.371787+0000","last_peered":"2023-09-23T20:12:50.371787+0000","last_clean":"2023-09-23T20:12:50.371787+0000","last_became_active":"2023-09-23T20:12:44.041038+0000","last_became_peered":"2023-09-23T20:12:44.041038+0000","last_unstale":"2023-09-23T20:12:50.371787+0000","last_undegraded":"2023-09-23T20:12:50.371787+0000","last_fullsized":"2023-09-23T20:12:50.371787+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:44:11.913129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.371442+0000","last_change":"2023-09-23T20:12:43.751731+0000","last_active":"2023-09-23T20:12:50.371442+0000","last_peered":"2023-09-23T20:12:50.371442+0000","last_clean":"2023-09-23T20:12:50.371442+0000","last_became_active":"2023-09-23T20:12:43.751461+0000","last_became_peered":"2023-09-23T20:12:43.751461+0000","last_unstale":"2023-09-23T20:12:50.371442+0000","last_undegraded":"2023-09-23T20:12:50.371442+0000","last_fullsized":"2023-09-23T20:12:50.371442+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:14:48.827695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":23,"reported_epoch":71,"state":"active+clean","last_fresh":"2023-09-23T20:12:51.517785+0000","last_change":"2023-09-23T20:12:51.517785+0000","last_active":"2023-09-23T20:12:51.517785+0000","last_peered":"2023-09-23T20:12:51.517785+0000","last_clean":"2023-09-23T20:12:51.517785+0000","last_became_active":"2023-09-23T20:12:51.517427+0000","last_became_peered":"2023-09-23T20:12:51.517427+0000","last_unstale":"2023-09-23T20:12:51.517785+0000","last_undegraded":"2023-09-23T20:12:51.517785+0000","last_fullsized":"2023-09-23T20:12:51.517785+0000","mapping_epoch":70,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:37:50.497067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.373372+0000","last_change":"2023-09-23T20:12:43.750514+0000","last_active":"2023-09-23T20:12:50.373372+0000","last_peered":"2023-09-23T20:12:50.373372+0000","last_clean":"2023-09-23T20:12:50.373372+0000","last_became_active":"2023-09-23T20:12:43.750016+0000","last_became_peered":"2023-09-23T20:12:43.750016+0000","last_unstale":"2023-09-23T20:12:50.373372+0000","last_undegraded":"2023-09-23T20:12:50.373372+0000","last_fullsized":"2023-09-23T20:12:50.373372+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:19:44.437641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.371440+0000","last_change":"2023-09-23T20:12:43.768568+0000","last_active":"2023-09-23T20:12:50.371440+0000","last_peered":"2023-09-23T20:12:50.371440+0000","last_clean":"2023-09-23T20:12:50.371440+0000","last_became_active":"2023-09-23T20:12:43.768298+0000","last_became_peered":"2023-09-23T20:12:43.768298+0000","last_unstale":"2023-09-23T20:12:50.371440+0000","last_undegraded":"2023-09-23T20:12:50.371440+0000","last_fullsized":"2023-09-23T20:12:50.371440+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:54:55.895563+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":16,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.374597+0000","last_change":"2023-09-23T20:12:43.759951+0000","last_active":"2023-09-23T20:12:50.374597+0000","last_peered":"2023-09-23T20:12:50.374597+0000","last_clean":"2023-09-23T20:12:50.374597+0000","last_became_active":"2023-09-23T20:12:43.759674+0000","last_became_peered":"2023-09-23T20:12:43.759674+0000","last_unstale":"2023-09-23T20:12:50.374597+0000","last_undegraded":"2023-09-23T20:12:50.374597+0000","last_fullsized":"2023-09-23T20:12:50.374597+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:20:36.648505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.368919+0000","last_change":"2023-09-23T20:12:45.610079+0000","last_active":"2023-09-23T20:12:50.368919+0000","last_peered":"2023-09-23T20:12:50.368919+0000","last_clean":"2023-09-23T20:12:50.368919+0000","last_became_active":"2023-09-23T20:12:45.609739+0000","last_became_peered":"2023-09-23T20:12:45.609739+0000","last_unstale":"2023-09-23T20:12:50.368919+0000","last_undegraded":"2023-09-23T20:12:50.368919+0000","last_fullsized":"2023-09-23T20:12:50.368919+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:20:47.579236+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.373973+0000","last_change":"2023-09-23T20:12:43.758487+0000","last_active":"2023-09-23T20:12:50.373973+0000","last_peered":"2023-09-23T20:12:50.373973+0000","last_clean":"2023-09-23T20:12:50.373973+0000","last_became_active":"2023-09-23T20:12:43.758195+0000","last_became_peered":"2023-09-23T20:12:43.758195+0000","last_unstale":"2023-09-23T20:12:50.373973+0000","last_undegraded":"2023-09-23T20:12:50.373973+0000","last_fullsized":"2023-09-23T20:12:50.373973+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:21:45.873977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.373239+0000","last_change":"2023-09-23T20:12:45.178940+0000","last_active":"2023-09-23T20:12:50.373239+0000","last_peered":"2023-09-23T20:12:50.373239+0000","last_clean":"2023-09-23T20:12:50.373239+0000","last_became_active":"2023-09-23T20:12:45.178660+0000","last_became_peered":"2023-09-23T20:12:45.178660+0000","last_unstale":"2023-09-23T20:12:50.373239+0000","last_undegraded":"2023-09-23T20:12:50.373239+0000","last_fullsized":"2023-09-23T20:12:50.373239+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:36:30.877060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.370711+0000","last_change":"2023-09-23T20:12:43.751523+0000","last_active":"2023-09-23T20:12:50.370711+0000","last_peered":"2023-09-23T20:12:50.370711+0000","last_clean":"2023-09-23T20:12:50.370711+0000","last_became_active":"2023-09-23T20:12:43.751257+0000","last_became_peered":"2023-09-23T20:12:43.751257+0000","last_unstale":"2023-09-23T20:12:50.370711+0000","last_undegraded":"2023-09-23T20:12:50.370711+0000","last_fullsized":"2023-09-23T20:12:50.370711+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:50:27.040225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.371244+0000","last_change":"2023-09-23T20:12:44.040128+0000","last_active":"2023-09-23T20:12:50.371244+0000","last_peered":"2023-09-23T20:12:50.371244+0000","last_clean":"2023-09-23T20:12:50.371244+0000","last_became_active":"2023-09-23T20:12:44.039778+0000","last_became_peered":"2023-09-23T20:12:44.039778+0000","last_unstale":"2023-09-23T20:12:50.371244+0000","last_undegraded":"2023-09-23T20:12:50.371244+0000","last_fullsized":"2023-09-23T20:12:50.371244+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:34:22.994439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":16,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.370511+0000","last_change":"2023-09-23T20:12:43.752550+0000","last_active":"2023-09-23T20:12:50.370511+0000","last_peered":"2023-09-23T20:12:50.370511+0000","last_clean":"2023-09-23T20:12:50.370511+0000","last_became_active":"2023-09-23T20:12:43.752154+0000","last_became_peered":"2023-09-23T20:12:43.752154+0000","last_unstale":"2023-09-23T20:12:50.370511+0000","last_undegraded":"2023-09-23T20:12:50.370511+0000","last_fullsized":"2023-09-23T20:12:50.370511+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:32:56.806990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.373095+0000","last_change":"2023-09-23T20:12:45.179211+0000","last_active":"2023-09-23T20:12:50.373095+0000","last_peered":"2023-09-23T20:12:50.373095+0000","last_clean":"2023-09-23T20:12:50.373095+0000","last_became_active":"2023-09-23T20:12:45.178966+0000","last_became_peered":"2023-09-23T20:12:45.178966+0000","last_unstale":"2023-09-23T20:12:50.373095+0000","last_undegraded":"2023-09-23T20:12:50.373095+0000","last_fullsized":"2023-09-23T20:12:50.373095+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:58.902140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.370850+0000","last_change":"2023-09-23T20:12:43.755378+0000","last_active":"2023-09-23T20:12:50.370850+0000","last_peered":"2023-09-23T20:12:50.370850+0000","last_clean":"2023-09-23T20:12:50.370850+0000","last_became_active":"2023-09-23T20:12:43.755139+0000","last_became_peered":"2023-09-23T20:12:43.755139+0000","last_unstale":"2023-09-23T20:12:50.370850+0000","last_undegraded":"2023-09-23T20:12:50.370850+0000","last_fullsized":"2023-09-23T20:12:50.370850+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:16:44.085179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.374734+0000","last_change":"2023-09-23T20:12:45.178536+0000","last_active":"2023-09-23T20:12:50.374734+0000","last_peered":"2023-09-23T20:12:50.374734+0000","last_clean":"2023-09-23T20:12:50.374734+0000","last_became_active":"2023-09-23T20:12:45.178183+0000","last_became_peered":"2023-09-23T20:12:45.178183+0000","last_unstale":"2023-09-23T20:12:50.374734+0000","last_undegraded":"2023-09-23T20:12:50.374734+0000","last_fullsized":"2023-09-23T20:12:50.374734+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:04:26.406019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":22,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.373904+0000","last_change":"2023-09-23T20:12:43.749116+0000","last_active":"2023-09-23T20:12:50.373904+0000","last_peered":"2023-09-23T20:12:50.373904+0000","last_clean":"2023-09-23T20:12:50.373904+0000","last_became_active":"2023-09-23T20:12:43.748611+0000","last_became_peered":"2023-09-23T20:12:43.748611+0000","last_unstale":"2023-09-23T20:12:50.373904+0000","last_undegraded":"2023-09-23T20:12:50.373904+0000","last_fullsized":"2023-09-23T20:12:50.373904+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":8,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:49:08.648511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.368865+0000","last_change":"2023-09-23T20:12:45.179306+0000","last_active":"2023-09-23T20:12:50.368865+0000","last_peered":"2023-09-23T20:12:50.368865+0000","last_clean":"2023-09-23T20:12:50.368865+0000","last_became_active":"2023-09-23T20:12:45.178943+0000","last_became_peered":"2023-09-23T20:12:45.178943+0000","last_unstale":"2023-09-23T20:12:50.368865+0000","last_undegraded":"2023-09-23T20:12:50.368865+0000","last_fullsized":"2023-09-23T20:12:50.368865+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:32:28.793104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.370842+0000","last_change":"2023-09-23T20:12:43.749139+0000","last_active":"2023-09-23T20:12:50.370842+0000","last_peered":"2023-09-23T20:12:50.370842+0000","last_clean":"2023-09-23T20:12:50.370842+0000","last_became_active":"2023-09-23T20:12:43.748854+0000","last_became_peered":"2023-09-23T20:12:43.748854+0000","last_unstale":"2023-09-23T20:12:50.370842+0000","last_undegraded":"2023-09-23T20:12:50.370842+0000","last_fullsized":"2023-09-23T20:12:50.370842+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:57:11.630712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.374118+0000","last_change":"2023-09-23T20:12:45.179288+0000","last_active":"2023-09-23T20:12:50.374118+0000","last_peered":"2023-09-23T20:12:50.374118+0000","last_clean":"2023-09-23T20:12:50.374118+0000","last_became_active":"2023-09-23T20:12:45.178967+0000","last_became_peered":"2023-09-23T20:12:45.178967+0000","last_unstale":"2023-09-23T20:12:50.374118+0000","last_undegraded":"2023-09-23T20:12:50.374118+0000","last_fullsized":"2023-09-23T20:12:50.374118+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:44:04.068111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.368938+0000","last_change":"2023-09-23T20:12:43.761680+0000","last_active":"2023-09-23T20:12:50.368938+0000","last_peered":"2023-09-23T20:12:50.368938+0000","last_clean":"2023-09-23T20:12:50.368938+0000","last_became_active":"2023-09-23T20:12:43.761396+0000","last_became_peered":"2023-09-23T20:12:43.761396+0000","last_unstale":"2023-09-23T20:12:50.368938+0000","last_undegraded":"2023-09-23T20:12:50.368938+0000","last_fullsized":"2023-09-23T20:12:50.368938+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:03:17.664405+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.374688+0000","last_change":"2023-09-23T20:12:45.178538+0000","last_active":"2023-09-23T20:12:50.374688+0000","last_peered":"2023-09-23T20:12:50.374688+0000","last_clean":"2023-09-23T20:12:50.374688+0000","last_became_active":"2023-09-23T20:12:45.178249+0000","last_became_peered":"2023-09-23T20:12:45.178249+0000","last_unstale":"2023-09-23T20:12:50.374688+0000","last_undegraded":"2023-09-23T20:12:50.374688+0000","last_fullsized":"2023-09-23T20:12:50.374688+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:16:10.055478+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":16,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.370515+0000","last_change":"2023-09-23T20:12:43.754736+0000","last_active":"2023-09-23T20:12:50.370515+0000","last_peered":"2023-09-23T20:12:50.370515+0000","last_clean":"2023-09-23T20:12:50.370515+0000","last_became_active":"2023-09-23T20:12:43.754466+0000","last_became_peered":"2023-09-23T20:12:43.754466+0000","last_unstale":"2023-09-23T20:12:50.370515+0000","last_undegraded":"2023-09-23T20:12:50.370515+0000","last_fullsized":"2023-09-23T20:12:50.370515+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:55:00.435148+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.371796+0000","last_change":"2023-09-23T20:12:44.041449+0000","last_active":"2023-09-23T20:12:50.371796+0000","last_peered":"2023-09-23T20:12:50.371796+0000","last_clean":"2023-09-23T20:12:50.371796+0000","last_became_active":"2023-09-23T20:12:44.041126+0000","last_became_peered":"2023-09-23T20:12:44.041126+0000","last_unstale":"2023-09-23T20:12:50.371796+0000","last_undegraded":"2023-09-23T20:12:50.371796+0000","last_fullsized":"2023-09-23T20:12:50.371796+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:18:44.388244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"69'6","reported_seq":20,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.371532+0000","last_change":"2023-09-23T20:12:43.750217+0000","last_active":"2023-09-23T20:12:50.371532+0000","last_peered":"2023-09-23T20:12:50.371532+0000","last_clean":"2023-09-23T20:12:50.371532+0000","last_became_active":"2023-09-23T20:12:43.749929+0000","last_became_peered":"2023-09-23T20:12:43.749929+0000","last_unstale":"2023-09-23T20:12:50.371532+0000","last_undegraded":"2023-09-23T20:12:50.371532+0000","last_fullsized":"2023-09-23T20:12:50.371532+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":6,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:50:28.470211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1536,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.373706+0000","last_change":"2023-09-23T20:12:44.769615+0000","last_active":"2023-09-23T20:12:50.373706+0000","last_peered":"2023-09-23T20:12:50.373706+0000","last_clean":"2023-09-23T20:12:50.373706+0000","last_became_active":"2023-09-23T20:12:44.769342+0000","last_became_peered":"2023-09-23T20:12:44.769342+0000","last_unstale":"2023-09-23T20:12:50.373706+0000","last_undegraded":"2023-09-23T20:12:50.373706+0000","last_fullsized":"2023-09-23T20:12:50.373706+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:59:12.723748+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":17,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.370627+0000","last_change":"2023-09-23T20:12:43.759472+0000","last_active":"2023-09-23T20:12:50.370627+0000","last_peered":"2023-09-23T20:12:50.370627+0000","last_clean":"2023-09-23T20:12:50.370627+0000","last_became_active":"2023-09-23T20:12:43.759261+0000","last_became_peered":"2023-09-23T20:12:43.759261+0000","last_unstale":"2023-09-23T20:12:50.370627+0000","last_undegraded":"2023-09-23T20:12:50.370627+0000","last_fullsized":"2023-09-23T20:12:50.370627+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":3,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:23:34.964302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.374441+0000","last_change":"2023-09-23T20:12:45.179990+0000","last_active":"2023-09-23T20:12:50.374441+0000","last_peered":"2023-09-23T20:12:50.374441+0000","last_clean":"2023-09-23T20:12:50.374441+0000","last_became_active":"2023-09-23T20:12:45.179728+0000","last_became_peered":"2023-09-23T20:12:45.179728+0000","last_unstale":"2023-09-23T20:12:50.374441+0000","last_undegraded":"2023-09-23T20:12:50.374441+0000","last_fullsized":"2023-09-23T20:12:50.374441+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:09:11.955320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.374205+0000","last_change":"2023-09-23T20:12:43.753000+0000","last_active":"2023-09-23T20:12:50.374205+0000","last_peered":"2023-09-23T20:12:50.374205+0000","last_clean":"2023-09-23T20:12:50.374205+0000","last_became_active":"2023-09-23T20:12:43.752714+0000","last_became_peered":"2023-09-23T20:12:43.752714+0000","last_unstale":"2023-09-23T20:12:50.374205+0000","last_undegraded":"2023-09-23T20:12:50.374205+0000","last_fullsized":"2023-09-23T20:12:50.374205+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:56:09.052225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.374337+0000","last_change":"2023-09-23T20:12:44.767475+0000","last_active":"2023-09-23T20:12:50.374337+0000","last_peered":"2023-09-23T20:12:50.374337+0000","last_clean":"2023-09-23T20:12:50.374337+0000","last_became_active":"2023-09-23T20:12:44.767199+0000","last_became_peered":"2023-09-23T20:12:44.767199+0000","last_unstale":"2023-09-23T20:12:50.374337+0000","last_undegraded":"2023-09-23T20:12:50.374337+0000","last_fullsized":"2023-09-23T20:12:50.374337+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:58:12.782887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.369529+0000","last_change":"2023-09-23T20:12:43.759944+0000","last_active":"2023-09-23T20:12:50.369529+0000","last_peered":"2023-09-23T20:12:50.369529+0000","last_clean":"2023-09-23T20:12:50.369529+0000","last_became_active":"2023-09-23T20:12:43.759617+0000","last_became_peered":"2023-09-23T20:12:43.759617+0000","last_unstale":"2023-09-23T20:12:50.369529+0000","last_undegraded":"2023-09-23T20:12:50.369529+0000","last_fullsized":"2023-09-23T20:12:50.369529+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:22:38.687584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.373466+0000","last_change":"2023-09-23T20:12:44.769490+0000","last_active":"2023-09-23T20:12:50.373466+0000","last_peered":"2023-09-23T20:12:50.373466+0000","last_clean":"2023-09-23T20:12:50.373466+0000","last_became_active":"2023-09-23T20:12:44.769284+0000","last_became_peered":"2023-09-23T20:12:44.769284+0000","last_unstale":"2023-09-23T20:12:50.373466+0000","last_undegraded":"2023-09-23T20:12:50.373466+0000","last_fullsized":"2023-09-23T20:12:50.373466+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:44:28.639674+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.374393+0000","last_change":"2023-09-23T20:12:43.762873+0000","last_active":"2023-09-23T20:12:50.374393+0000","last_peered":"2023-09-23T20:12:50.374393+0000","last_clean":"2023-09-23T20:12:50.374393+0000","last_became_active":"2023-09-23T20:12:43.762384+0000","last_became_peered":"2023-09-23T20:12:43.762384+0000","last_unstale":"2023-09-23T20:12:50.374393+0000","last_undegraded":"2023-09-23T20:12:50.374393+0000","last_fullsized":"2023-09-23T20:12:50.374393+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:51:30.037644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.368938+0000","last_change":"2023-09-23T20:12:45.179234+0000","last_active":"2023-09-23T20:12:50.368938+0000","last_peered":"2023-09-23T20:12:50.368938+0000","last_clean":"2023-09-23T20:12:50.368938+0000","last_became_active":"2023-09-23T20:12:45.178892+0000","last_became_peered":"2023-09-23T20:12:45.178892+0000","last_unstale":"2023-09-23T20:12:50.368938+0000","last_undegraded":"2023-09-23T20:12:50.368938+0000","last_fullsized":"2023-09-23T20:12:50.368938+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:59:51.830813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.373051+0000","last_change":"2023-09-23T20:12:43.760050+0000","last_active":"2023-09-23T20:12:50.373051+0000","last_peered":"2023-09-23T20:12:50.373051+0000","last_clean":"2023-09-23T20:12:50.373051+0000","last_became_active":"2023-09-23T20:12:43.759738+0000","last_became_peered":"2023-09-23T20:12:43.759738+0000","last_unstale":"2023-09-23T20:12:50.373051+0000","last_undegraded":"2023-09-23T20:12:50.373051+0000","last_fullsized":"2023-09-23T20:12:50.373051+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:59:37.640763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.374840+0000","last_change":"2023-09-23T20:12:45.609879+0000","last_active":"2023-09-23T20:12:50.374840+0000","last_peered":"2023-09-23T20:12:50.374840+0000","last_clean":"2023-09-23T20:12:50.374840+0000","last_became_active":"2023-09-23T20:12:45.609533+0000","last_became_peered":"2023-09-23T20:12:45.609533+0000","last_unstale":"2023-09-23T20:12:50.374840+0000","last_undegraded":"2023-09-23T20:12:50.374840+0000","last_fullsized":"2023-09-23T20:12:50.374840+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:54:21.264486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"69'1","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.371277+0000","last_change":"2023-09-23T20:12:43.751650+0000","last_active":"2023-09-23T20:12:50.371277+0000","last_peered":"2023-09-23T20:12:50.371277+0000","last_clean":"2023-09-23T20:12:50.371277+0000","last_became_active":"2023-09-23T20:12:43.751383+0000","last_became_peered":"2023-09-23T20:12:43.751383+0000","last_unstale":"2023-09-23T20:12:50.371277+0000","last_undegraded":"2023-09-23T20:12:50.371277+0000","last_fullsized":"2023-09-23T20:12:50.371277+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:00:20.810162+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2306,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":6918,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201305,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6640,"kb_used_data":1320,"kb_used_omap":0,"kb_used_meta":5312,"kb_avail":93738512,"statfs":{"total":95995035648,"available":95988236288,"internally_reserved":0,"allocated":1351680,"data_stored":909748,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5439488},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:12:25 2023","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":0.52300000000000002,"15min":0.52300000000000002},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.74299999999999999,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.434},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":1.004,"5min":1.004,"15min":1.004},"last":0.51700000000000002}]},{"osd":1,"last update":"Sat Sep 23 20:12:25 2023","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.86799999999999999,"5min":0.86799999999999999,"15min":0.86799999999999999},"last":0.48299999999999998},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":1.0309999999999999,"5min":1.0309999999999999,"15min":1.0309999999999999},"last":0.71199999999999997}]},{"osd":2,"last update":"Sat Sep 23 20:12:25 2023","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.98999999999999999,"5min":0.98999999999999999,"15min":0.98999999999999999},"last":0.65500000000000003},{"interface":"front","average":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.92400000000000004,"5min":0.92400000000000004,"15min":0.92400000000000004},"last":0.56000000000000005}]},{"osd":3,"last update":"Sat Sep 23 20:12:25 2023","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.88800000000000001,"5min":0.88800000000000001,"15min":0.88800000000000001},"last":0.61099999999999999},{"interface":"front","average":{"1min":0.61899999999999999,"5min":0.61899999999999999,"15min":0.61899999999999999},"min":{"1min":0.45300000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"max":{"1min":0.97899999999999998,"5min":0.97899999999999998,"15min":0.97899999999999998},"last":0.60299999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:12:25 2023","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":0.52400000000000002,"15min":0.52400000000000002},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.94099999999999995,"5min":0.94099999999999995,"15min":0.94099999999999995},"last":0.57899999999999996},{"interface":"front","average":{"1min":0.54200000000000004,"5min":0.54200000000000004,"15min":0.54200000000000004},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.80700000000000005,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.73899999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:12:25 2023","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.85899999999999999,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.628},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":0.97699999999999998,"5min":0.97699999999999998,"15min":0.97699999999999998},"last":0.76400000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:12:25 2023","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":1.0569999999999999,"5min":1.0569999999999999,"15min":1.0569999999999999},"last":0.79300000000000004},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.879,"5min":0.879,"15min":0.879},"last":0.69799999999999995}]}]},{"osd":6,"up_from":47,"seq":201863462941,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6692,"kb_used_data":1308,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":93738460,"statfs":{"total":95995035648,"available":95988183040,"internally_reserved":0,"allocated":1339392,"data_stored":909602,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:13:08 2023","interfaces":[{"interface":"back","average":{"1min":0.46300000000000002,"5min":0.53900000000000003,"15min":0.55200000000000005},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.63500000000000001,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.54800000000000004},{"interface":"front","average":{"1min":0.52400000000000002,"5min":0.57099999999999995,"15min":0.57899999999999996},"min":{"1min":0.439,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.64700000000000002,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.623}]},{"osd":1,"last update":"Sat Sep 23 20:13:08 2023","interfaces":[{"interface":"back","average":{"1min":0.48599999999999999,"5min":0.55300000000000005,"15min":0.56399999999999995},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.629,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.64000000000000001},{"interface":"front","average":{"1min":0.50600000000000001,"5min":0.53000000000000003,"15min":0.53400000000000003},"min":{"1min":0.38,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.63300000000000001,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.64900000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:13:08 2023","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":0.54300000000000004,"15min":0.54800000000000004},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.625,"5min":0.70699999999999996,"15min":0.70699999999999996},"last":0.61199999999999999},{"interface":"front","average":{"1min":0.63400000000000001,"5min":0.57799999999999996,"15min":0.56799999999999995},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":2.7090000000000001,"5min":2.7090000000000001,"15min":2.7090000000000001},"last":0.67100000000000004}]},{"osd":3,"last update":"Sat Sep 23 20:13:08 2023","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.54300000000000004,"15min":0.54500000000000004},"min":{"1min":0.42099999999999999,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.69799999999999995,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.50700000000000001},{"interface":"front","average":{"1min":0.63200000000000001,"5min":0.60099999999999998,"15min":0.59599999999999997},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":2.0670000000000002,"5min":2.0670000000000002,"15min":2.0670000000000002},"last":0.69699999999999995}]},{"osd":4,"last update":"Sat Sep 23 20:13:08 2023","interfaces":[{"interface":"back","average":{"1min":0.45400000000000001,"5min":0.46200000000000002,"15min":0.46300000000000002},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.63600000000000001,"5min":0.72299999999999998,"15min":0.72299999999999998},"last":0.57499999999999996},{"interface":"front","average":{"1min":0.47799999999999998,"5min":0.48899999999999999,"15min":0.49099999999999999},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.67400000000000004,"5min":0.73099999999999998,"15min":0.73099999999999998},"last":0.68600000000000005}]},{"osd":5,"last update":"Sat Sep 23 20:13:08 2023","interfaces":[{"interface":"back","average":{"1min":0.46899999999999997,"5min":0.48099999999999998,"15min":0.48299999999999998},"min":{"1min":0.35499999999999998,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.59399999999999997,"5min":0.69399999999999995,"15min":0.69399999999999995},"last":0.747},{"interface":"front","average":{"1min":0.51000000000000001,"5min":0.51100000000000001,"15min":0.51100000000000001},"min":{"1min":0.39000000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.72099999999999997,"5min":0.72099999999999997,"15min":0.72099999999999997},"last":0.59599999999999997}]},{"osd":7,"last update":"Sat Sep 23 20:12:26 2023","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.74399999999999999,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.71499999999999997},{"interface":"front","average":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.75,"5min":0.75,"15min":0.75},"last":0.73299999999999998}]}]},{"osd":1,"up_from":13,"seq":55834574898,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6864,"kb_used_data":712,"kb_used_omap":0,"kb_used_meta":6144,"kb_avail":93738288,"statfs":{"total":95995035648,"available":95988006912,"internally_reserved":0,"allocated":729088,"data_stored":317628,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6291456},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:12:27 2023","interfaces":[{"interface":"back","average":{"1min":1.3380000000000001,"5min":0.55900000000000005,"15min":0.41699999999999998},"min":{"1min":0.23499999999999999,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":17.280000000000001,"5min":17.280000000000001,"15min":17.280000000000001},"last":0.40000000000000002},{"interface":"front","average":{"1min":0.41099999999999998,"5min":0.39300000000000002,"15min":0.36699999999999999},"min":{"1min":0.28100000000000003,"5min":0.247,"15min":0.247},"max":{"1min":0.98999999999999999,"5min":0.98999999999999999,"15min":0.98999999999999999},"last":0.26600000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:12:54 2023","interfaces":[{"interface":"back","average":{"1min":0.442,"5min":0.44400000000000001,"15min":0.442},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":1.3740000000000001,"5min":1.3740000000000001,"15min":1.3740000000000001},"last":0.371},{"interface":"front","average":{"1min":0.40500000000000003,"5min":0.41799999999999998,"15min":0.41899999999999998},"min":{"1min":0.26700000000000002,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":1.302,"5min":1.302,"15min":1.302},"last":0.28399999999999997}]},{"osd":3,"last update":"Sat Sep 23 20:13:12 2023","interfaces":[{"interface":"back","average":{"1min":0.94699999999999995,"5min":0.59399999999999997,"15min":0.54400000000000004},"min":{"1min":0.224,"5min":0.224,"15min":0.224},"max":{"1min":7.3949999999999996,"5min":7.3949999999999996,"15min":7.3949999999999996},"last":0.33000000000000002},{"interface":"front","average":{"1min":0.47099999999999997,"5min":0.48899999999999999,"15min":0.502},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":1.762,"5min":1.762,"15min":1.762},"last":0.35899999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:12:27 2023","interfaces":[{"interface":"back","average":{"1min":0.58499999999999996,"5min":0.67300000000000004,"15min":0.68799999999999994},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":0.79700000000000004,"5min":2.226,"15min":2.226},"last":0.63200000000000001},{"interface":"front","average":{"1min":1.288,"5min":0.74299999999999999,"15min":0.65200000000000002},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":12.914999999999999,"5min":12.914999999999999,"15min":12.914999999999999},"last":0.52000000000000002}]},{"osd":5,"last update":"Sat Sep 23 20:12:48 2023","interfaces":[{"interface":"back","average":{"1min":0.61399999999999999,"5min":0.63,"15min":0.63200000000000001},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.91400000000000003,"5min":0.91400000000000003,"15min":0.91400000000000003},"last":0.52800000000000002},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.60699999999999998,"15min":0.61299999999999999},"min":{"1min":0.438,"5min":0.435,"15min":0.435},"max":{"1min":0.755,"5min":0.89200000000000002,"15min":0.89200000000000002},"last":0.67200000000000004}]},{"osd":6,"last update":"Sat Sep 23 20:13:04 2023","interfaces":[{"interface":"back","average":{"1min":1.2470000000000001,"5min":0.74099999999999999,"15min":0.65700000000000003},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":12.824,"5min":12.824,"15min":12.824},"last":0.56799999999999995},{"interface":"front","average":{"1min":0.76900000000000002,"5min":0.73899999999999999,"15min":0.73399999999999999},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":3.5390000000000001,"5min":3.5390000000000001,"15min":3.5390000000000001},"last":0.57499999999999996}]},{"osd":7,"last update":"Sat Sep 23 20:12:27 2023","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"min":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":0.90700000000000003,"5min":0.90700000000000003,"15min":0.90700000000000003},"last":0.70599999999999996},{"interface":"front","average":{"1min":1.29,"5min":1.29,"15min":1.29},"min":{"1min":0.51000000000000001,"5min":0.51000000000000001,"15min":0.51000000000000001},"max":{"1min":12.878,"5min":12.878,"15min":12.878},"last":0.68400000000000005}]}]},{"osd":0,"up_from":8,"seq":34359738421,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":7452,"kb_used_data":1300,"kb_used_omap":0,"kb_used_meta":6144,"kb_avail":93737700,"statfs":{"total":95995035648,"available":95987404800,"internally_reserved":0,"allocated":1331200,"data_stored":908572,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6291456},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:12:31 2023","interfaces":[{"interface":"back","average":{"1min":0.39200000000000002,"5min":0.35399999999999998,"15min":0.33700000000000002},"min":{"1min":0.25800000000000001,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":1.0860000000000001,"5min":1.0860000000000001,"15min":1.0860000000000001},"last":0.29899999999999999},{"interface":"front","average":{"1min":0.39300000000000002,"5min":0.36899999999999999,"15min":0.35099999999999998},"min":{"1min":0.24299999999999999,"5min":0.217,"15min":0.217},"max":{"1min":1.0309999999999999,"5min":1.0309999999999999,"15min":1.0309999999999999},"last":0.38}]},{"osd":2,"last update":"Sat Sep 23 20:12:52 2023","interfaces":[{"interface":"back","average":{"1min":0.41199999999999998,"5min":0.42399999999999999,"15min":0.42399999999999999},"min":{"1min":0.28699999999999998,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":1.0640000000000001,"5min":1.0640000000000001,"15min":1.0640000000000001},"last":0.32900000000000001},{"interface":"front","average":{"1min":0.42599999999999999,"5min":0.441,"15min":0.442},"min":{"1min":0.31,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":1.016,"5min":1.016,"15min":1.016},"last":0.36199999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:12:11 2023","interfaces":[{"interface":"back","average":{"1min":0.40100000000000002,"5min":0.441,"15min":0.44800000000000001},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.55700000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"last":0.39600000000000002},{"interface":"front","average":{"1min":0.44700000000000001,"5min":0.48599999999999999,"15min":0.49199999999999999},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.58099999999999996,"5min":0.60299999999999998,"15min":0.60299999999999998},"last":0.40999999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:12:31 2023","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.58299999999999996,"15min":0.58899999999999997},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.93200000000000005,"5min":0.93200000000000005,"15min":0.93200000000000005},"last":0.52100000000000002},{"interface":"front","average":{"1min":0.57199999999999995,"5min":0.59299999999999997,"15min":0.59699999999999998},"min":{"1min":0.42499999999999999,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.873,"5min":0.873,"15min":0.873},"last":0.53300000000000003}]},{"osd":5,"last update":"Sat Sep 23 20:12:52 2023","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.61799999999999999,"15min":0.626},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.85799999999999998,"5min":0.88400000000000001,"15min":0.88400000000000001},"last":0.58699999999999997},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.63400000000000001,"15min":0.64300000000000002},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.90300000000000002,"5min":0.93300000000000005,"15min":0.93300000000000005},"last":0.57499999999999996}]},{"osd":6,"last update":"Sat Sep 23 20:13:10 2023","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":0.58999999999999997,"15min":0.59799999999999998},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.79100000000000004,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.56100000000000005},{"interface":"front","average":{"1min":0.56999999999999995,"5min":0.60399999999999998,"15min":0.60999999999999999},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.88800000000000001,"5min":0.88800000000000001,"15min":0.88800000000000001},"last":0.54700000000000004}]},{"osd":7,"last update":"Sat Sep 23 20:12:28 2023","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.89100000000000001,"5min":0.89100000000000001,"15min":0.89100000000000001},"last":0.61199999999999999},{"interface":"front","average":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.91700000000000004,"5min":0.91700000000000004,"15min":0.91700000000000004},"last":0.50800000000000001}]}]},{"osd":2,"up_from":18,"seq":77309411374,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6680,"kb_used_data":720,"kb_used_omap":0,"kb_used_meta":5952,"kb_avail":93738472,"statfs":{"total":95995035648,"available":95988195328,"internally_reserved":0,"allocated":737280,"data_stored":318216,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6094848},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:12:49 2023","interfaces":[{"interface":"back","average":{"1min":0.39100000000000001,"5min":0.374,"15min":0.36099999999999999},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.80700000000000005,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.26000000000000001},{"interface":"front","average":{"1min":0.46999999999999997,"5min":0.40300000000000002,"15min":0.38300000000000001},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":1.423,"5min":1.423,"15min":1.423},"last":0.28000000000000003}]},{"osd":1,"last update":"Sat Sep 23 20:12:49 2023","interfaces":[{"interface":"back","average":{"1min":0.5,"5min":0.42899999999999999,"15min":0.41299999999999998},"min":{"1min":0.223,"5min":0.223,"15min":0.223},"max":{"1min":2.387,"5min":2.387,"15min":2.387},"last":0.307},{"interface":"front","average":{"1min":0.41599999999999998,"5min":0.42499999999999999,"15min":0.41299999999999998},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.70399999999999996,"5min":0.75800000000000001,"15min":0.75800000000000001},"last":0.32800000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:13:10 2023","interfaces":[{"interface":"back","average":{"1min":0.439,"5min":0.48299999999999998,"15min":0.497},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.88600000000000001,"5min":0.88600000000000001,"15min":0.88600000000000001},"last":0.34799999999999998},{"interface":"front","average":{"1min":0.46800000000000003,"5min":0.50600000000000001,"15min":0.51800000000000002},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":1.0189999999999999,"5min":1.0189999999999999,"15min":1.0189999999999999},"last":0.36199999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:12:31 2023","interfaces":[{"interface":"back","average":{"1min":0.61699999999999999,"5min":0.60699999999999998,"15min":0.60499999999999998},"min":{"1min":0.42499999999999999,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.95399999999999996,"5min":0.95399999999999996,"15min":0.95399999999999996},"last":0.442},{"interface":"front","average":{"1min":0.67500000000000004,"5min":0.65500000000000003,"15min":0.65200000000000002},"min":{"1min":0.499,"5min":0.47199999999999998,"15min":0.47199999999999998},"max":{"1min":0.93700000000000006,"5min":0.93700000000000006,"15min":0.93700000000000006},"last":0.622}]},{"osd":5,"last update":"Sat Sep 23 20:12:49 2023","interfaces":[{"interface":"back","average":{"1min":0.63800000000000001,"5min":0.66100000000000003,"15min":0.66500000000000004},"min":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":0.82899999999999996,"5min":0.98599999999999999,"15min":0.98599999999999999},"last":0.59799999999999998},{"interface":"front","average":{"1min":0.64400000000000002,"5min":0.67500000000000004,"15min":0.68000000000000005},"min":{"1min":0.44,"5min":0.435,"15min":0.435},"max":{"1min":0.84399999999999997,"5min":1.0409999999999999,"15min":1.0409999999999999},"last":0.47199999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:13:05 2023","interfaces":[{"interface":"back","average":{"1min":0.63500000000000001,"5min":0.66600000000000004,"15min":0.67100000000000004},"min":{"1min":0.48399999999999999,"5min":0.47099999999999997,"15min":0.47099999999999997},"max":{"1min":0.81599999999999995,"5min":0.96199999999999997,"15min":0.96199999999999997},"last":0.48399999999999999},{"interface":"front","average":{"1min":0.64000000000000001,"5min":0.70499999999999996,"15min":0.71599999999999997},"min":{"1min":0.46500000000000002,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":0.85499999999999998,"5min":0.91700000000000004,"15min":0.91700000000000004},"last":0.57099999999999995}]},{"osd":7,"last update":"Sat Sep 23 20:12:25 2023","interfaces":[{"interface":"back","average":{"1min":0.68799999999999994,"5min":0.68799999999999994,"15min":0.68799999999999994},"min":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"max":{"1min":0.97399999999999998,"5min":0.97399999999999998,"15min":0.97399999999999998},"last":0.54100000000000004},{"interface":"front","average":{"1min":0.66700000000000004,"5min":0.66700000000000004,"15min":0.66700000000000004},"min":{"1min":0.46200000000000002,"5min":0.46200000000000002,"15min":0.46200000000000002},"max":{"1min":0.99199999999999999,"5min":0.99199999999999999,"15min":0.99199999999999999},"last":0.58399999999999996}]}]},{"osd":3,"up_from":25,"seq":107374182440,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6164,"kb_used_data":716,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":93738988,"statfs":{"total":95995035648,"available":95988723712,"internally_reserved":0,"allocated":733184,"data_stored":317718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:13:07 2023","interfaces":[{"interface":"back","average":{"1min":0.94299999999999995,"5min":0.495,"15min":0.41699999999999998},"min":{"1min":0.26200000000000001,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":9.9580000000000002,"5min":9.9580000000000002,"15min":9.9580000000000002},"last":0.32300000000000001},{"interface":"front","average":{"1min":0.96299999999999997,"5min":0.51400000000000001,"15min":0.44600000000000001},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":11.028,"5min":11.028,"15min":11.028},"last":0.40000000000000002}]},{"osd":1,"last update":"Sat Sep 23 20:13:07 2023","interfaces":[{"interface":"back","average":{"1min":0.83199999999999996,"5min":0.51600000000000001,"15min":0.46300000000000002},"min":{"1min":0.27800000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":8.3239999999999998,"5min":8.3239999999999998,"15min":8.3239999999999998},"last":0.26800000000000002},{"interface":"front","average":{"1min":0.995,"5min":0.58399999999999996,"15min":0.51300000000000001},"min":{"1min":0.33500000000000002,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":9.8930000000000007,"5min":9.8930000000000007,"15min":9.8930000000000007},"last":0.36399999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:13:07 2023","interfaces":[{"interface":"back","average":{"1min":0.96499999999999997,"5min":0.55400000000000005,"15min":0.49199999999999999},"min":{"1min":0.308,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":9.9260000000000002,"5min":9.9260000000000002,"15min":9.9260000000000002},"last":0.42299999999999999},{"interface":"front","average":{"1min":0.38400000000000001,"5min":0.45200000000000001,"15min":0.47099999999999997},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.60499999999999998,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":0.45100000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:12:25 2023","interfaces":[{"interface":"back","average":{"1min":1.01,"5min":0.71799999999999997,"15min":0.66900000000000004},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":8.2949999999999999,"5min":8.2949999999999999,"15min":8.2949999999999999},"last":0.50700000000000001},{"interface":"front","average":{"1min":1.073,"5min":0.73299999999999998,"15min":0.67700000000000005},"min":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"max":{"1min":9.8539999999999992,"5min":9.8539999999999992,"15min":9.8539999999999992},"last":0.59499999999999997}]},{"osd":5,"last update":"Sat Sep 23 20:12:48 2023","interfaces":[{"interface":"back","average":{"1min":1.1279999999999999,"5min":0.71399999999999997,"15min":0.64500000000000002},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":10.917,"5min":10.917,"15min":10.917},"last":0.60499999999999998},{"interface":"front","average":{"1min":0.97599999999999998,"5min":0.68400000000000005,"15min":0.63600000000000001},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":8.2609999999999992,"5min":8.2609999999999992,"15min":8.2609999999999992},"last":0.61799999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:13:07 2023","interfaces":[{"interface":"back","average":{"1min":1.1519999999999999,"5min":0.748,"15min":0.68100000000000005},"min":{"1min":0.46300000000000002,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":9.8119999999999994,"5min":9.8119999999999994,"15min":9.8119999999999994},"last":0.54300000000000004},{"interface":"front","average":{"1min":1.2,"5min":0.75,"15min":0.67500000000000004},"min":{"1min":0.439,"5min":0.439,"15min":0.439},"max":{"1min":10.885,"5min":10.885,"15min":10.885},"last":0.64500000000000002}]},{"osd":7,"last update":"Sat Sep 23 20:12:25 2023","interfaces":[{"interface":"back","average":{"1min":1.105,"5min":1.105,"15min":1.105},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":10.846,"5min":10.846,"15min":10.846},"last":0.68600000000000005},{"interface":"front","average":{"1min":1.0049999999999999,"5min":1.0049999999999999,"15min":1.0049999999999999},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":8.2170000000000005,"5min":8.2170000000000005,"15min":8.2170000000000005},"last":0.55600000000000005}]}]},{"osd":4,"up_from":30,"seq":128849018917,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6244,"kb_used_data":732,"kb_used_omap":0,"kb_used_meta":5504,"kb_avail":93738908,"statfs":{"total":95995035648,"available":95988641792,"internally_reserved":0,"allocated":749568,"data_stored":318987,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5636096},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:12:26 2023","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.54000000000000004,"15min":0.54100000000000004},"min":{"1min":0.34599999999999997,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.68300000000000005,"5min":0.83599999999999997,"15min":0.83599999999999997},"last":0.57499999999999996},{"interface":"front","average":{"1min":0.51600000000000001,"5min":0.54600000000000004,"15min":0.55100000000000005},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.78700000000000003,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.59699999999999998}]},{"osd":1,"last update":"Sat Sep 23 20:12:26 2023","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.56200000000000006,"15min":0.56999999999999995},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.72999999999999998,"5min":0.86499999999999999,"15min":0.86499999999999999},"last":0.65700000000000003},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.55300000000000005,"15min":0.55700000000000005},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.70999999999999996,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.499}]},{"osd":2,"last update":"Sat Sep 23 20:12:26 2023","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.56999999999999995,"15min":0.57699999999999996},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.77200000000000002,"5min":0.81599999999999995,"15min":0.81599999999999995},"last":0.61099999999999999},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.57899999999999996,"15min":0.58099999999999996},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.86199999999999999,"5min":0.91500000000000004,"15min":0.91500000000000004},"last":0.63900000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:12:26 2023","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.56399999999999995,"15min":0.56999999999999995},"min":{"1min":0.36099999999999999,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.69599999999999995,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.56200000000000006},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.55000000000000004,"15min":0.54700000000000004},"min":{"1min":0.40200000000000002,"5min":0.376,"15min":0.376},"max":{"1min":0.77100000000000002,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.623}]},{"osd":5,"last update":"Sat Sep 23 20:12:45 2023","interfaces":[{"interface":"back","average":{"1min":0.48899999999999999,"5min":0.41699999999999998,"15min":0.40500000000000003},"min":{"1min":0.28100000000000003,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.83999999999999997,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.373},{"interface":"front","average":{"1min":0.48299999999999998,"5min":0.41099999999999998,"15min":0.39900000000000002},"min":{"1min":0.36299999999999999,"5min":0.248,"15min":0.248},"max":{"1min":0.73999999999999999,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.40400000000000003}]},{"osd":6,"last update":"Sat Sep 23 20:13:07 2023","interfaces":[{"interface":"back","average":{"1min":0.48699999999999999,"5min":0.44400000000000001,"15min":0.437},"min":{"1min":0.28999999999999998,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.65500000000000003,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.69299999999999995},{"interface":"front","average":{"1min":0.51300000000000001,"5min":0.46600000000000003,"15min":0.45800000000000002},"min":{"1min":0.33500000000000002,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.745,"5min":0.745,"15min":0.745},"last":0.53800000000000003}]},{"osd":7,"last update":"Sat Sep 23 20:12:26 2023","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.90400000000000003,"5min":0.90400000000000003,"15min":0.90400000000000003},"last":0.66700000000000004},{"interface":"front","average":{"1min":0.52400000000000002,"5min":0.52400000000000002,"15min":0.52400000000000002},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.77400000000000002,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.58699999999999997}]}]},{"osd":5,"up_from":39,"seq":167503724578,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6104,"kb_used_data":720,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":93739048,"statfs":{"total":95995035648,"available":95988785152,"internally_reserved":0,"allocated":737280,"data_stored":318252,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:12:49 2023","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":0.55800000000000005,"15min":0.56599999999999995},"min":{"1min":0.39400000000000002,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.65800000000000003,"5min":1.0640000000000001,"15min":1.0640000000000001},"last":0.65900000000000003},{"interface":"front","average":{"1min":0.51900000000000002,"5min":0.51400000000000001,"15min":0.51300000000000001},"min":{"1min":0.378,"5min":0.312,"15min":0.312},"max":{"1min":0.76100000000000001,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.64900000000000002}]},{"osd":1,"last update":"Sat Sep 23 20:12:49 2023","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.53200000000000003,"15min":0.53000000000000003},"min":{"1min":0.35199999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.78400000000000003,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.70099999999999996},{"interface":"front","average":{"1min":0.52400000000000002,"5min":0.54600000000000004,"15min":0.55000000000000004},"min":{"1min":0.34699999999999998,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.72399999999999998,"5min":1.1519999999999999,"15min":1.1519999999999999},"last":0.67200000000000004}]},{"osd":2,"last update":"Sat Sep 23 20:12:49 2023","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.55900000000000005,"15min":0.56299999999999994},"min":{"1min":0.42099999999999999,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.70199999999999996,"5min":1.1719999999999999,"15min":1.1719999999999999},"last":0.72299999999999998},{"interface":"front","average":{"1min":0.53300000000000003,"5min":0.54100000000000004,"15min":0.54200000000000004},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.76000000000000001,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.59299999999999997}]},{"osd":3,"last update":"Sat Sep 23 20:12:49 2023","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":0.57999999999999996,"15min":0.58599999999999997},"min":{"1min":0.374,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.71699999999999997,"5min":1.2130000000000001,"15min":1.2130000000000001},"last":0.68500000000000005},{"interface":"front","average":{"1min":0.67200000000000004,"5min":0.55200000000000005,"15min":0.53200000000000003},"min":{"1min":0.46100000000000002,"5min":0.371,"15min":0.371},"max":{"1min":2.2599999999999998,"5min":2.2599999999999998,"15min":2.2599999999999998},"last":0.73999999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:12:49 2023","interfaces":[{"interface":"back","average":{"1min":0.46899999999999997,"5min":0.40400000000000003,"15min":0.39300000000000002},"min":{"1min":0.28799999999999998,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.72399999999999998,"5min":0.72399999999999998,"15min":0.72399999999999998},"last":0.52200000000000002},{"interface":"front","average":{"1min":0.47899999999999998,"5min":0.45100000000000001,"15min":0.44600000000000001},"min":{"1min":0.30399999999999999,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.65300000000000002,"5min":1.4450000000000001,"15min":1.4450000000000001},"last":0.46000000000000002}]},{"osd":6,"last update":"Sat Sep 23 20:13:09 2023","interfaces":[{"interface":"back","average":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.69899999999999995,"5min":0.69899999999999995,"15min":0.69899999999999995},"last":0.61599999999999999},{"interface":"front","average":{"1min":0.52500000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"min":{"1min":0.38900000000000001,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.71799999999999997,"5min":0.73699999999999999,"15min":0.73699999999999999},"last":0.63}]},{"osd":7,"last update":"Sat Sep 23 20:12:25 2023","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.68899999999999995,"5min":0.68899999999999995,"15min":0.68899999999999995},"last":0.55200000000000005},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.71699999999999997,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.48699999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1682,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1606,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:13:18.461 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:13:19.065 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:13:19.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:13:18 smithi018 ceph-mon[109137]: pgmap v58: 73 pgs: 73 active+clean; 579 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:13:19.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:13:18 smithi018 ceph-mon[95889]: pgmap v58: 73 pgs: 73 active+clean; 579 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:13:19.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:13:18 smithi149 ceph-mon[100764]: pgmap v58: 73 pgs: 73 active+clean; 579 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:13:19.471 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:13:19 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:13:19] "GET /metrics HTTP/1.1" 200 238782 "" "Prometheus/2.43.0" 2023-09-23T20:13:19.738 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:13:19 smithi018 ceph-mon[95889]: from='client.24695 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:13:20.064 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:13:19 smithi018 ceph-mon[109137]: from='client.24695 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:13:20.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:13:19 smithi149 ceph-mon[100764]: from='client.24695 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:13:20.444 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:13:20.446 INFO:teuthology.orchestra.run.smithi018.stderr:dumped all 2023-09-23T20:13:20.814 INFO:teuthology.orchestra.run.smithi018.stdout:{"pg_ready":true,"pg_map":{"version":59,"stamp":"2023-09-23T20:13:19.683452+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592674,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":224,"num_write_kb":2429,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":167,"ondisk_log_size":167,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":52840,"kb_used_data":7528,"kb_used_omap":0,"kb_used_meta":45248,"kb_avail":749908376,"statfs":{"total":767960285184,"available":767906177024,"internally_reserved":0,"allocated":7708672,"data_stored":4318723,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":46333952},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002482"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":23,"reported_epoch":71,"state":"active+clean","last_fresh":"2023-09-23T20:12:51.376081+0000","last_change":"2023-09-23T20:12:51.376081+0000","last_active":"2023-09-23T20:12:51.376081+0000","last_peered":"2023-09-23T20:12:51.376081+0000","last_clean":"2023-09-23T20:12:51.376081+0000","last_became_active":"2023-09-23T20:12:51.375807+0000","last_became_peered":"2023-09-23T20:12:51.375807+0000","last_unstale":"2023-09-23T20:12:51.376081+0000","last_undegraded":"2023-09-23T20:12:51.376081+0000","last_fullsized":"2023-09-23T20:12:51.376081+0000","mapping_epoch":70,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:29:58.345734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.369615+0000","last_change":"2023-09-23T20:12:43.749326+0000","last_active":"2023-09-23T20:12:50.369615+0000","last_peered":"2023-09-23T20:12:50.369615+0000","last_clean":"2023-09-23T20:12:50.369615+0000","last_became_active":"2023-09-23T20:12:43.749020+0000","last_became_peered":"2023-09-23T20:12:43.749020+0000","last_unstale":"2023-09-23T20:12:50.369615+0000","last_undegraded":"2023-09-23T20:12:50.369615+0000","last_fullsized":"2023-09-23T20:12:50.369615+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":4,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:36:42.363869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.374420+0000","last_change":"2023-09-23T20:12:44.770095+0000","last_active":"2023-09-23T20:12:50.374420+0000","last_peered":"2023-09-23T20:12:50.374420+0000","last_clean":"2023-09-23T20:12:50.374420+0000","last_became_active":"2023-09-23T20:12:44.769778+0000","last_became_peered":"2023-09-23T20:12:44.769778+0000","last_unstale":"2023-09-23T20:12:50.374420+0000","last_undegraded":"2023-09-23T20:12:50.374420+0000","last_fullsized":"2023-09-23T20:12:50.374420+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:31:22.309973+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.374743+0000","last_change":"2023-09-23T20:12:43.755660+0000","last_active":"2023-09-23T20:12:50.374743+0000","last_peered":"2023-09-23T20:12:50.374743+0000","last_clean":"2023-09-23T20:12:50.374743+0000","last_became_active":"2023-09-23T20:12:43.755312+0000","last_became_peered":"2023-09-23T20:12:43.755312+0000","last_unstale":"2023-09-23T20:12:50.374743+0000","last_undegraded":"2023-09-23T20:12:50.374743+0000","last_fullsized":"2023-09-23T20:12:50.374743+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:48:38.650179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.371355+0000","last_change":"2023-09-23T20:12:43.903122+0000","last_active":"2023-09-23T20:12:50.371355+0000","last_peered":"2023-09-23T20:12:50.371355+0000","last_clean":"2023-09-23T20:12:50.371355+0000","last_became_active":"2023-09-23T20:12:43.902816+0000","last_became_peered":"2023-09-23T20:12:43.902816+0000","last_unstale":"2023-09-23T20:12:50.371355+0000","last_undegraded":"2023-09-23T20:12:50.371355+0000","last_fullsized":"2023-09-23T20:12:50.371355+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:31:11.122636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":16,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.374209+0000","last_change":"2023-09-23T20:12:43.753383+0000","last_active":"2023-09-23T20:12:50.374209+0000","last_peered":"2023-09-23T20:12:50.374209+0000","last_clean":"2023-09-23T20:12:50.374209+0000","last_became_active":"2023-09-23T20:12:43.752921+0000","last_became_peered":"2023-09-23T20:12:43.752921+0000","last_unstale":"2023-09-23T20:12:50.374209+0000","last_undegraded":"2023-09-23T20:12:50.374209+0000","last_fullsized":"2023-09-23T20:12:50.374209+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:28:26.982998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.371877+0000","last_change":"2023-09-23T20:12:44.041322+0000","last_active":"2023-09-23T20:12:50.371877+0000","last_peered":"2023-09-23T20:12:50.371877+0000","last_clean":"2023-09-23T20:12:50.371877+0000","last_became_active":"2023-09-23T20:12:44.041028+0000","last_became_peered":"2023-09-23T20:12:44.041028+0000","last_unstale":"2023-09-23T20:12:50.371877+0000","last_undegraded":"2023-09-23T20:12:50.371877+0000","last_fullsized":"2023-09-23T20:12:50.371877+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:56:27.407493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":16,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.374018+0000","last_change":"2023-09-23T20:12:43.753359+0000","last_active":"2023-09-23T20:12:50.374018+0000","last_peered":"2023-09-23T20:12:50.374018+0000","last_clean":"2023-09-23T20:12:50.374018+0000","last_became_active":"2023-09-23T20:12:43.753064+0000","last_became_peered":"2023-09-23T20:12:43.753064+0000","last_unstale":"2023-09-23T20:12:50.374018+0000","last_undegraded":"2023-09-23T20:12:50.374018+0000","last_fullsized":"2023-09-23T20:12:50.374018+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:30:02.892722+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.369369+0000","last_change":"2023-09-23T20:12:44.770501+0000","last_active":"2023-09-23T20:12:50.369369+0000","last_peered":"2023-09-23T20:12:50.369369+0000","last_clean":"2023-09-23T20:12:50.369369+0000","last_became_active":"2023-09-23T20:12:44.770244+0000","last_became_peered":"2023-09-23T20:12:44.770244+0000","last_unstale":"2023-09-23T20:12:50.369369+0000","last_undegraded":"2023-09-23T20:12:50.369369+0000","last_fullsized":"2023-09-23T20:12:50.369369+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:32:03.174269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.369091+0000","last_change":"2023-09-23T20:12:43.750365+0000","last_active":"2023-09-23T20:12:50.369091+0000","last_peered":"2023-09-23T20:12:50.369091+0000","last_clean":"2023-09-23T20:12:50.369091+0000","last_became_active":"2023-09-23T20:12:43.750100+0000","last_became_peered":"2023-09-23T20:12:43.750100+0000","last_unstale":"2023-09-23T20:12:50.369091+0000","last_undegraded":"2023-09-23T20:12:50.369091+0000","last_fullsized":"2023-09-23T20:12:50.369091+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:34:57.785225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.371346+0000","last_change":"2023-09-23T20:12:43.765908+0000","last_active":"2023-09-23T20:12:50.371346+0000","last_peered":"2023-09-23T20:12:50.371346+0000","last_clean":"2023-09-23T20:12:50.371346+0000","last_became_active":"2023-09-23T20:12:43.765558+0000","last_became_peered":"2023-09-23T20:12:43.765558+0000","last_unstale":"2023-09-23T20:12:50.371346+0000","last_undegraded":"2023-09-23T20:12:50.371346+0000","last_fullsized":"2023-09-23T20:12:50.371346+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:35:43.809898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2023-09-23T20:12:51.517376+0000","last_change":"2023-09-23T20:12:43.751685+0000","last_active":"2023-09-23T20:12:51.517376+0000","last_peered":"2023-09-23T20:12:51.517376+0000","last_clean":"2023-09-23T20:12:51.517376+0000","last_became_active":"2023-09-23T20:12:43.751413+0000","last_became_peered":"2023-09-23T20:12:43.751413+0000","last_unstale":"2023-09-23T20:12:51.517376+0000","last_undegraded":"2023-09-23T20:12:51.517376+0000","last_fullsized":"2023-09-23T20:12:51.517376+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:05:57.982527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":17,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.373501+0000","last_change":"2023-09-23T20:12:00.850675+0000","last_active":"2023-09-23T20:12:50.373501+0000","last_peered":"2023-09-23T20:12:50.373501+0000","last_clean":"2023-09-23T20:12:50.373501+0000","last_became_active":"2023-09-23T20:12:00.850315+0000","last_became_peered":"2023-09-23T20:12:00.850315+0000","last_unstale":"2023-09-23T20:12:50.373501+0000","last_undegraded":"2023-09-23T20:12:50.373501+0000","last_fullsized":"2023-09-23T20:12:50.373501+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:03:33.446629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.374215+0000","last_change":"2023-09-23T20:12:45.180160+0000","last_active":"2023-09-23T20:12:50.374215+0000","last_peered":"2023-09-23T20:12:50.374215+0000","last_clean":"2023-09-23T20:12:50.374215+0000","last_became_active":"2023-09-23T20:12:45.179718+0000","last_became_peered":"2023-09-23T20:12:45.179718+0000","last_unstale":"2023-09-23T20:12:50.374215+0000","last_undegraded":"2023-09-23T20:12:50.374215+0000","last_fullsized":"2023-09-23T20:12:50.374215+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:40:30.530253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.373735+0000","last_change":"2023-09-23T20:12:01.062560+0000","last_active":"2023-09-23T20:12:50.373735+0000","last_peered":"2023-09-23T20:12:50.373735+0000","last_clean":"2023-09-23T20:12:50.373735+0000","last_became_active":"2023-09-23T20:12:01.062216+0000","last_became_peered":"2023-09-23T20:12:01.062216+0000","last_unstale":"2023-09-23T20:12:50.373735+0000","last_undegraded":"2023-09-23T20:12:50.373735+0000","last_fullsized":"2023-09-23T20:12:50.373735+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:01:52.795311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":16,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.374853+0000","last_change":"2023-09-23T20:12:43.760973+0000","last_active":"2023-09-23T20:12:50.374853+0000","last_peered":"2023-09-23T20:12:50.374853+0000","last_clean":"2023-09-23T20:12:50.374853+0000","last_became_active":"2023-09-23T20:12:43.760442+0000","last_became_peered":"2023-09-23T20:12:43.760442+0000","last_unstale":"2023-09-23T20:12:50.374853+0000","last_undegraded":"2023-09-23T20:12:50.374853+0000","last_fullsized":"2023-09-23T20:12:50.374853+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:47:10.459217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.369002+0000","last_change":"2023-09-23T20:12:45.178571+0000","last_active":"2023-09-23T20:12:50.369002+0000","last_peered":"2023-09-23T20:12:50.369002+0000","last_clean":"2023-09-23T20:12:50.369002+0000","last_became_active":"2023-09-23T20:12:45.178269+0000","last_became_peered":"2023-09-23T20:12:45.178269+0000","last_unstale":"2023-09-23T20:12:50.369002+0000","last_undegraded":"2023-09-23T20:12:50.369002+0000","last_fullsized":"2023-09-23T20:12:50.369002+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:09:15.260586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.373597+0000","last_change":"2023-09-23T20:12:43.750138+0000","last_active":"2023-09-23T20:12:50.373597+0000","last_peered":"2023-09-23T20:12:50.373597+0000","last_clean":"2023-09-23T20:12:50.373597+0000","last_became_active":"2023-09-23T20:12:43.749812+0000","last_became_peered":"2023-09-23T20:12:43.749812+0000","last_unstale":"2023-09-23T20:12:50.373597+0000","last_undegraded":"2023-09-23T20:12:50.373597+0000","last_fullsized":"2023-09-23T20:12:50.373597+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:54:32.020961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":17,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.370668+0000","last_change":"2023-09-23T20:12:01.062442+0000","last_active":"2023-09-23T20:12:50.370668+0000","last_peered":"2023-09-23T20:12:50.370668+0000","last_clean":"2023-09-23T20:12:50.370668+0000","last_became_active":"2023-09-23T20:12:01.062047+0000","last_became_peered":"2023-09-23T20:12:01.062047+0000","last_unstale":"2023-09-23T20:12:50.370668+0000","last_undegraded":"2023-09-23T20:12:50.370668+0000","last_fullsized":"2023-09-23T20:12:50.370668+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:58:49.032117+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.369247+0000","last_change":"2023-09-23T20:12:44.772724+0000","last_active":"2023-09-23T20:12:50.369247+0000","last_peered":"2023-09-23T20:12:50.369247+0000","last_clean":"2023-09-23T20:12:50.369247+0000","last_became_active":"2023-09-23T20:12:44.772404+0000","last_became_peered":"2023-09-23T20:12:44.772404+0000","last_unstale":"2023-09-23T20:12:50.369247+0000","last_undegraded":"2023-09-23T20:12:50.369247+0000","last_fullsized":"2023-09-23T20:12:50.369247+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:12:45.284142+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.368740+0000","last_change":"2023-09-23T20:12:43.756939+0000","last_active":"2023-09-23T20:12:50.368740+0000","last_peered":"2023-09-23T20:12:50.368740+0000","last_clean":"2023-09-23T20:12:50.368740+0000","last_became_active":"2023-09-23T20:12:43.756632+0000","last_became_peered":"2023-09-23T20:12:43.756632+0000","last_unstale":"2023-09-23T20:12:50.368740+0000","last_undegraded":"2023-09-23T20:12:50.368740+0000","last_fullsized":"2023-09-23T20:12:50.368740+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:13:19.514304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":17,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.370283+0000","last_change":"2023-09-23T20:12:00.848951+0000","last_active":"2023-09-23T20:12:50.370283+0000","last_peered":"2023-09-23T20:12:50.370283+0000","last_clean":"2023-09-23T20:12:50.370283+0000","last_became_active":"2023-09-23T20:12:00.848720+0000","last_became_peered":"2023-09-23T20:12:00.848720+0000","last_unstale":"2023-09-23T20:12:50.370283+0000","last_undegraded":"2023-09-23T20:12:50.370283+0000","last_fullsized":"2023-09-23T20:12:50.370283+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:45:24.720986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.373396+0000","last_change":"2023-09-23T20:12:45.609829+0000","last_active":"2023-09-23T20:12:50.373396+0000","last_peered":"2023-09-23T20:12:50.373396+0000","last_clean":"2023-09-23T20:12:50.373396+0000","last_became_active":"2023-09-23T20:12:45.609467+0000","last_became_peered":"2023-09-23T20:12:45.609467+0000","last_unstale":"2023-09-23T20:12:50.373396+0000","last_undegraded":"2023-09-23T20:12:50.373396+0000","last_fullsized":"2023-09-23T20:12:50.373396+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:27:31.490555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.374536+0000","last_change":"2023-09-23T20:12:43.759474+0000","last_active":"2023-09-23T20:12:50.374536+0000","last_peered":"2023-09-23T20:12:50.374536+0000","last_clean":"2023-09-23T20:12:50.374536+0000","last_became_active":"2023-09-23T20:12:43.759191+0000","last_became_peered":"2023-09-23T20:12:43.759191+0000","last_unstale":"2023-09-23T20:12:50.374536+0000","last_undegraded":"2023-09-23T20:12:50.374536+0000","last_fullsized":"2023-09-23T20:12:50.374536+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:54:48.352278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"60'128","reported_seq":279,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.370262+0000","last_change":"2023-09-23T20:11:27.849030+0000","last_active":"2023-09-23T20:12:50.370262+0000","last_peered":"2023-09-23T20:12:50.370262+0000","last_clean":"2023-09-23T20:12:50.370262+0000","last_became_active":"2023-09-23T20:11:27.848678+0000","last_became_peered":"2023-09-23T20:11:27.848678+0000","last_unstale":"2023-09-23T20:12:50.370262+0000","last_undegraded":"2023-09-23T20:12:50.370262+0000","last_fullsized":"2023-09-23T20:12:50.370262+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:09:47.213130+0000","objects_scrubbed":0,"log_size":128,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:33:53.621689+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":17,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.373844+0000","last_change":"2023-09-23T20:12:00.851792+0000","last_active":"2023-09-23T20:12:50.373844+0000","last_peered":"2023-09-23T20:12:50.373844+0000","last_clean":"2023-09-23T20:12:50.373844+0000","last_became_active":"2023-09-23T20:12:00.851408+0000","last_became_peered":"2023-09-23T20:12:00.851408+0000","last_unstale":"2023-09-23T20:12:50.373844+0000","last_undegraded":"2023-09-23T20:12:50.373844+0000","last_fullsized":"2023-09-23T20:12:50.373844+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:19:26.564015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.373631+0000","last_change":"2023-09-23T20:12:44.767464+0000","last_active":"2023-09-23T20:12:50.373631+0000","last_peered":"2023-09-23T20:12:50.373631+0000","last_clean":"2023-09-23T20:12:50.373631+0000","last_became_active":"2023-09-23T20:12:44.767179+0000","last_became_peered":"2023-09-23T20:12:44.767179+0000","last_unstale":"2023-09-23T20:12:50.373631+0000","last_undegraded":"2023-09-23T20:12:50.373631+0000","last_fullsized":"2023-09-23T20:12:50.373631+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:58:39.324206+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.373819+0000","last_change":"2023-09-23T20:12:43.751903+0000","last_active":"2023-09-23T20:12:50.373819+0000","last_peered":"2023-09-23T20:12:50.373819+0000","last_clean":"2023-09-23T20:12:50.373819+0000","last_became_active":"2023-09-23T20:12:43.751617+0000","last_became_peered":"2023-09-23T20:12:43.751617+0000","last_unstale":"2023-09-23T20:12:50.373819+0000","last_undegraded":"2023-09-23T20:12:50.373819+0000","last_fullsized":"2023-09-23T20:12:50.373819+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:55:05.695074+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":17,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.369252+0000","last_change":"2023-09-23T20:12:01.010922+0000","last_active":"2023-09-23T20:12:50.369252+0000","last_peered":"2023-09-23T20:12:50.369252+0000","last_clean":"2023-09-23T20:12:50.369252+0000","last_became_active":"2023-09-23T20:12:01.010591+0000","last_became_peered":"2023-09-23T20:12:01.010591+0000","last_unstale":"2023-09-23T20:12:50.369252+0000","last_undegraded":"2023-09-23T20:12:50.369252+0000","last_fullsized":"2023-09-23T20:12:50.369252+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:11:25.451578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.373485+0000","last_change":"2023-09-23T20:12:44.769014+0000","last_active":"2023-09-23T20:12:50.373485+0000","last_peered":"2023-09-23T20:12:50.373485+0000","last_clean":"2023-09-23T20:12:50.373485+0000","last_became_active":"2023-09-23T20:12:44.768697+0000","last_became_peered":"2023-09-23T20:12:44.768697+0000","last_unstale":"2023-09-23T20:12:50.373485+0000","last_undegraded":"2023-09-23T20:12:50.373485+0000","last_fullsized":"2023-09-23T20:12:50.373485+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:15:05.830365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.371771+0000","last_change":"2023-09-23T20:12:43.756068+0000","last_active":"2023-09-23T20:12:50.371771+0000","last_peered":"2023-09-23T20:12:50.371771+0000","last_clean":"2023-09-23T20:12:50.371771+0000","last_became_active":"2023-09-23T20:12:43.755372+0000","last_became_peered":"2023-09-23T20:12:43.755372+0000","last_unstale":"2023-09-23T20:12:50.371771+0000","last_undegraded":"2023-09-23T20:12:50.371771+0000","last_fullsized":"2023-09-23T20:12:50.371771+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:26:33.645748+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":17,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.373753+0000","last_change":"2023-09-23T20:12:00.851594+0000","last_active":"2023-09-23T20:12:50.373753+0000","last_peered":"2023-09-23T20:12:50.373753+0000","last_clean":"2023-09-23T20:12:50.373753+0000","last_became_active":"2023-09-23T20:12:00.851256+0000","last_became_peered":"2023-09-23T20:12:00.851256+0000","last_unstale":"2023-09-23T20:12:50.373753+0000","last_undegraded":"2023-09-23T20:12:50.373753+0000","last_fullsized":"2023-09-23T20:12:50.373753+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:12:00.121999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.373774+0000","last_change":"2023-09-23T20:12:45.610141+0000","last_active":"2023-09-23T20:12:50.373774+0000","last_peered":"2023-09-23T20:12:50.373774+0000","last_clean":"2023-09-23T20:12:50.373774+0000","last_became_active":"2023-09-23T20:12:45.609753+0000","last_became_peered":"2023-09-23T20:12:45.609753+0000","last_unstale":"2023-09-23T20:12:50.373774+0000","last_undegraded":"2023-09-23T20:12:50.373774+0000","last_fullsized":"2023-09-23T20:12:50.373774+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:19:21.663951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.374143+0000","last_change":"2023-09-23T20:12:43.758600+0000","last_active":"2023-09-23T20:12:50.374143+0000","last_peered":"2023-09-23T20:12:50.374143+0000","last_clean":"2023-09-23T20:12:50.374143+0000","last_became_active":"2023-09-23T20:12:43.758205+0000","last_became_peered":"2023-09-23T20:12:43.758205+0000","last_unstale":"2023-09-23T20:12:50.374143+0000","last_undegraded":"2023-09-23T20:12:50.374143+0000","last_fullsized":"2023-09-23T20:12:50.374143+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:10:08.802544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.373386+0000","last_change":"2023-09-23T20:12:00.847882+0000","last_active":"2023-09-23T20:12:50.373386+0000","last_peered":"2023-09-23T20:12:50.373386+0000","last_clean":"2023-09-23T20:12:50.373386+0000","last_became_active":"2023-09-23T20:12:00.847511+0000","last_became_peered":"2023-09-23T20:12:00.847511+0000","last_unstale":"2023-09-23T20:12:50.373386+0000","last_undegraded":"2023-09-23T20:12:50.373386+0000","last_fullsized":"2023-09-23T20:12:50.373386+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:52:12.888646+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":10,"reported_epoch":71,"state":"active+clean","last_fresh":"2023-09-23T20:12:51.376014+0000","last_change":"2023-09-23T20:12:51.376014+0000","last_active":"2023-09-23T20:12:51.376014+0000","last_peered":"2023-09-23T20:12:51.376014+0000","last_clean":"2023-09-23T20:12:51.376014+0000","last_became_active":"2023-09-23T20:12:51.375741+0000","last_became_peered":"2023-09-23T20:12:51.375741+0000","last_unstale":"2023-09-23T20:12:51.376014+0000","last_undegraded":"2023-09-23T20:12:51.376014+0000","last_fullsized":"2023-09-23T20:12:51.376014+0000","mapping_epoch":70,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:44:48.074944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.374306+0000","last_change":"2023-09-23T20:12:43.762681+0000","last_active":"2023-09-23T20:12:50.374306+0000","last_peered":"2023-09-23T20:12:50.374306+0000","last_clean":"2023-09-23T20:12:50.374306+0000","last_became_active":"2023-09-23T20:12:43.761957+0000","last_became_peered":"2023-09-23T20:12:43.761957+0000","last_unstale":"2023-09-23T20:12:50.374306+0000","last_undegraded":"2023-09-23T20:12:50.374306+0000","last_fullsized":"2023-09-23T20:12:50.374306+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:42:47.159570+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.371787+0000","last_change":"2023-09-23T20:12:44.041342+0000","last_active":"2023-09-23T20:12:50.371787+0000","last_peered":"2023-09-23T20:12:50.371787+0000","last_clean":"2023-09-23T20:12:50.371787+0000","last_became_active":"2023-09-23T20:12:44.041038+0000","last_became_peered":"2023-09-23T20:12:44.041038+0000","last_unstale":"2023-09-23T20:12:50.371787+0000","last_undegraded":"2023-09-23T20:12:50.371787+0000","last_fullsized":"2023-09-23T20:12:50.371787+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:44:11.913129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.371442+0000","last_change":"2023-09-23T20:12:43.751731+0000","last_active":"2023-09-23T20:12:50.371442+0000","last_peered":"2023-09-23T20:12:50.371442+0000","last_clean":"2023-09-23T20:12:50.371442+0000","last_became_active":"2023-09-23T20:12:43.751461+0000","last_became_peered":"2023-09-23T20:12:43.751461+0000","last_unstale":"2023-09-23T20:12:50.371442+0000","last_undegraded":"2023-09-23T20:12:50.371442+0000","last_fullsized":"2023-09-23T20:12:50.371442+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:14:48.827695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":23,"reported_epoch":71,"state":"active+clean","last_fresh":"2023-09-23T20:12:51.517785+0000","last_change":"2023-09-23T20:12:51.517785+0000","last_active":"2023-09-23T20:12:51.517785+0000","last_peered":"2023-09-23T20:12:51.517785+0000","last_clean":"2023-09-23T20:12:51.517785+0000","last_became_active":"2023-09-23T20:12:51.517427+0000","last_became_peered":"2023-09-23T20:12:51.517427+0000","last_unstale":"2023-09-23T20:12:51.517785+0000","last_undegraded":"2023-09-23T20:12:51.517785+0000","last_fullsized":"2023-09-23T20:12:51.517785+0000","mapping_epoch":70,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:37:50.497067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.373372+0000","last_change":"2023-09-23T20:12:43.750514+0000","last_active":"2023-09-23T20:12:50.373372+0000","last_peered":"2023-09-23T20:12:50.373372+0000","last_clean":"2023-09-23T20:12:50.373372+0000","last_became_active":"2023-09-23T20:12:43.750016+0000","last_became_peered":"2023-09-23T20:12:43.750016+0000","last_unstale":"2023-09-23T20:12:50.373372+0000","last_undegraded":"2023-09-23T20:12:50.373372+0000","last_fullsized":"2023-09-23T20:12:50.373372+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:19:44.437641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.371440+0000","last_change":"2023-09-23T20:12:43.768568+0000","last_active":"2023-09-23T20:12:50.371440+0000","last_peered":"2023-09-23T20:12:50.371440+0000","last_clean":"2023-09-23T20:12:50.371440+0000","last_became_active":"2023-09-23T20:12:43.768298+0000","last_became_peered":"2023-09-23T20:12:43.768298+0000","last_unstale":"2023-09-23T20:12:50.371440+0000","last_undegraded":"2023-09-23T20:12:50.371440+0000","last_fullsized":"2023-09-23T20:12:50.371440+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:54:55.895563+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":16,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.374597+0000","last_change":"2023-09-23T20:12:43.759951+0000","last_active":"2023-09-23T20:12:50.374597+0000","last_peered":"2023-09-23T20:12:50.374597+0000","last_clean":"2023-09-23T20:12:50.374597+0000","last_became_active":"2023-09-23T20:12:43.759674+0000","last_became_peered":"2023-09-23T20:12:43.759674+0000","last_unstale":"2023-09-23T20:12:50.374597+0000","last_undegraded":"2023-09-23T20:12:50.374597+0000","last_fullsized":"2023-09-23T20:12:50.374597+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:20:36.648505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.368919+0000","last_change":"2023-09-23T20:12:45.610079+0000","last_active":"2023-09-23T20:12:50.368919+0000","last_peered":"2023-09-23T20:12:50.368919+0000","last_clean":"2023-09-23T20:12:50.368919+0000","last_became_active":"2023-09-23T20:12:45.609739+0000","last_became_peered":"2023-09-23T20:12:45.609739+0000","last_unstale":"2023-09-23T20:12:50.368919+0000","last_undegraded":"2023-09-23T20:12:50.368919+0000","last_fullsized":"2023-09-23T20:12:50.368919+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:20:47.579236+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.373973+0000","last_change":"2023-09-23T20:12:43.758487+0000","last_active":"2023-09-23T20:12:50.373973+0000","last_peered":"2023-09-23T20:12:50.373973+0000","last_clean":"2023-09-23T20:12:50.373973+0000","last_became_active":"2023-09-23T20:12:43.758195+0000","last_became_peered":"2023-09-23T20:12:43.758195+0000","last_unstale":"2023-09-23T20:12:50.373973+0000","last_undegraded":"2023-09-23T20:12:50.373973+0000","last_fullsized":"2023-09-23T20:12:50.373973+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:21:45.873977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.373239+0000","last_change":"2023-09-23T20:12:45.178940+0000","last_active":"2023-09-23T20:12:50.373239+0000","last_peered":"2023-09-23T20:12:50.373239+0000","last_clean":"2023-09-23T20:12:50.373239+0000","last_became_active":"2023-09-23T20:12:45.178660+0000","last_became_peered":"2023-09-23T20:12:45.178660+0000","last_unstale":"2023-09-23T20:12:50.373239+0000","last_undegraded":"2023-09-23T20:12:50.373239+0000","last_fullsized":"2023-09-23T20:12:50.373239+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:36:30.877060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.370711+0000","last_change":"2023-09-23T20:12:43.751523+0000","last_active":"2023-09-23T20:12:50.370711+0000","last_peered":"2023-09-23T20:12:50.370711+0000","last_clean":"2023-09-23T20:12:50.370711+0000","last_became_active":"2023-09-23T20:12:43.751257+0000","last_became_peered":"2023-09-23T20:12:43.751257+0000","last_unstale":"2023-09-23T20:12:50.370711+0000","last_undegraded":"2023-09-23T20:12:50.370711+0000","last_fullsized":"2023-09-23T20:12:50.370711+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:50:27.040225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.371244+0000","last_change":"2023-09-23T20:12:44.040128+0000","last_active":"2023-09-23T20:12:50.371244+0000","last_peered":"2023-09-23T20:12:50.371244+0000","last_clean":"2023-09-23T20:12:50.371244+0000","last_became_active":"2023-09-23T20:12:44.039778+0000","last_became_peered":"2023-09-23T20:12:44.039778+0000","last_unstale":"2023-09-23T20:12:50.371244+0000","last_undegraded":"2023-09-23T20:12:50.371244+0000","last_fullsized":"2023-09-23T20:12:50.371244+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:34:22.994439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":16,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.370511+0000","last_change":"2023-09-23T20:12:43.752550+0000","last_active":"2023-09-23T20:12:50.370511+0000","last_peered":"2023-09-23T20:12:50.370511+0000","last_clean":"2023-09-23T20:12:50.370511+0000","last_became_active":"2023-09-23T20:12:43.752154+0000","last_became_peered":"2023-09-23T20:12:43.752154+0000","last_unstale":"2023-09-23T20:12:50.370511+0000","last_undegraded":"2023-09-23T20:12:50.370511+0000","last_fullsized":"2023-09-23T20:12:50.370511+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:32:56.806990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.373095+0000","last_change":"2023-09-23T20:12:45.179211+0000","last_active":"2023-09-23T20:12:50.373095+0000","last_peered":"2023-09-23T20:12:50.373095+0000","last_clean":"2023-09-23T20:12:50.373095+0000","last_became_active":"2023-09-23T20:12:45.178966+0000","last_became_peered":"2023-09-23T20:12:45.178966+0000","last_unstale":"2023-09-23T20:12:50.373095+0000","last_undegraded":"2023-09-23T20:12:50.373095+0000","last_fullsized":"2023-09-23T20:12:50.373095+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:58.902140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.370850+0000","last_change":"2023-09-23T20:12:43.755378+0000","last_active":"2023-09-23T20:12:50.370850+0000","last_peered":"2023-09-23T20:12:50.370850+0000","last_clean":"2023-09-23T20:12:50.370850+0000","last_became_active":"2023-09-23T20:12:43.755139+0000","last_became_peered":"2023-09-23T20:12:43.755139+0000","last_unstale":"2023-09-23T20:12:50.370850+0000","last_undegraded":"2023-09-23T20:12:50.370850+0000","last_fullsized":"2023-09-23T20:12:50.370850+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:16:44.085179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.374734+0000","last_change":"2023-09-23T20:12:45.178536+0000","last_active":"2023-09-23T20:12:50.374734+0000","last_peered":"2023-09-23T20:12:50.374734+0000","last_clean":"2023-09-23T20:12:50.374734+0000","last_became_active":"2023-09-23T20:12:45.178183+0000","last_became_peered":"2023-09-23T20:12:45.178183+0000","last_unstale":"2023-09-23T20:12:50.374734+0000","last_undegraded":"2023-09-23T20:12:50.374734+0000","last_fullsized":"2023-09-23T20:12:50.374734+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:04:26.406019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":22,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.373904+0000","last_change":"2023-09-23T20:12:43.749116+0000","last_active":"2023-09-23T20:12:50.373904+0000","last_peered":"2023-09-23T20:12:50.373904+0000","last_clean":"2023-09-23T20:12:50.373904+0000","last_became_active":"2023-09-23T20:12:43.748611+0000","last_became_peered":"2023-09-23T20:12:43.748611+0000","last_unstale":"2023-09-23T20:12:50.373904+0000","last_undegraded":"2023-09-23T20:12:50.373904+0000","last_fullsized":"2023-09-23T20:12:50.373904+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":8,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:49:08.648511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.368865+0000","last_change":"2023-09-23T20:12:45.179306+0000","last_active":"2023-09-23T20:12:50.368865+0000","last_peered":"2023-09-23T20:12:50.368865+0000","last_clean":"2023-09-23T20:12:50.368865+0000","last_became_active":"2023-09-23T20:12:45.178943+0000","last_became_peered":"2023-09-23T20:12:45.178943+0000","last_unstale":"2023-09-23T20:12:50.368865+0000","last_undegraded":"2023-09-23T20:12:50.368865+0000","last_fullsized":"2023-09-23T20:12:50.368865+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:32:28.793104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.370842+0000","last_change":"2023-09-23T20:12:43.749139+0000","last_active":"2023-09-23T20:12:50.370842+0000","last_peered":"2023-09-23T20:12:50.370842+0000","last_clean":"2023-09-23T20:12:50.370842+0000","last_became_active":"2023-09-23T20:12:43.748854+0000","last_became_peered":"2023-09-23T20:12:43.748854+0000","last_unstale":"2023-09-23T20:12:50.370842+0000","last_undegraded":"2023-09-23T20:12:50.370842+0000","last_fullsized":"2023-09-23T20:12:50.370842+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:57:11.630712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.374118+0000","last_change":"2023-09-23T20:12:45.179288+0000","last_active":"2023-09-23T20:12:50.374118+0000","last_peered":"2023-09-23T20:12:50.374118+0000","last_clean":"2023-09-23T20:12:50.374118+0000","last_became_active":"2023-09-23T20:12:45.178967+0000","last_became_peered":"2023-09-23T20:12:45.178967+0000","last_unstale":"2023-09-23T20:12:50.374118+0000","last_undegraded":"2023-09-23T20:12:50.374118+0000","last_fullsized":"2023-09-23T20:12:50.374118+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:44:04.068111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.368938+0000","last_change":"2023-09-23T20:12:43.761680+0000","last_active":"2023-09-23T20:12:50.368938+0000","last_peered":"2023-09-23T20:12:50.368938+0000","last_clean":"2023-09-23T20:12:50.368938+0000","last_became_active":"2023-09-23T20:12:43.761396+0000","last_became_peered":"2023-09-23T20:12:43.761396+0000","last_unstale":"2023-09-23T20:12:50.368938+0000","last_undegraded":"2023-09-23T20:12:50.368938+0000","last_fullsized":"2023-09-23T20:12:50.368938+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:03:17.664405+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.374688+0000","last_change":"2023-09-23T20:12:45.178538+0000","last_active":"2023-09-23T20:12:50.374688+0000","last_peered":"2023-09-23T20:12:50.374688+0000","last_clean":"2023-09-23T20:12:50.374688+0000","last_became_active":"2023-09-23T20:12:45.178249+0000","last_became_peered":"2023-09-23T20:12:45.178249+0000","last_unstale":"2023-09-23T20:12:50.374688+0000","last_undegraded":"2023-09-23T20:12:50.374688+0000","last_fullsized":"2023-09-23T20:12:50.374688+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:16:10.055478+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":16,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.370515+0000","last_change":"2023-09-23T20:12:43.754736+0000","last_active":"2023-09-23T20:12:50.370515+0000","last_peered":"2023-09-23T20:12:50.370515+0000","last_clean":"2023-09-23T20:12:50.370515+0000","last_became_active":"2023-09-23T20:12:43.754466+0000","last_became_peered":"2023-09-23T20:12:43.754466+0000","last_unstale":"2023-09-23T20:12:50.370515+0000","last_undegraded":"2023-09-23T20:12:50.370515+0000","last_fullsized":"2023-09-23T20:12:50.370515+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:55:00.435148+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.371796+0000","last_change":"2023-09-23T20:12:44.041449+0000","last_active":"2023-09-23T20:12:50.371796+0000","last_peered":"2023-09-23T20:12:50.371796+0000","last_clean":"2023-09-23T20:12:50.371796+0000","last_became_active":"2023-09-23T20:12:44.041126+0000","last_became_peered":"2023-09-23T20:12:44.041126+0000","last_unstale":"2023-09-23T20:12:50.371796+0000","last_undegraded":"2023-09-23T20:12:50.371796+0000","last_fullsized":"2023-09-23T20:12:50.371796+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:18:44.388244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"69'6","reported_seq":20,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.371532+0000","last_change":"2023-09-23T20:12:43.750217+0000","last_active":"2023-09-23T20:12:50.371532+0000","last_peered":"2023-09-23T20:12:50.371532+0000","last_clean":"2023-09-23T20:12:50.371532+0000","last_became_active":"2023-09-23T20:12:43.749929+0000","last_became_peered":"2023-09-23T20:12:43.749929+0000","last_unstale":"2023-09-23T20:12:50.371532+0000","last_undegraded":"2023-09-23T20:12:50.371532+0000","last_fullsized":"2023-09-23T20:12:50.371532+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":6,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:50:28.470211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1536,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.373706+0000","last_change":"2023-09-23T20:12:44.769615+0000","last_active":"2023-09-23T20:12:50.373706+0000","last_peered":"2023-09-23T20:12:50.373706+0000","last_clean":"2023-09-23T20:12:50.373706+0000","last_became_active":"2023-09-23T20:12:44.769342+0000","last_became_peered":"2023-09-23T20:12:44.769342+0000","last_unstale":"2023-09-23T20:12:50.373706+0000","last_undegraded":"2023-09-23T20:12:50.373706+0000","last_fullsized":"2023-09-23T20:12:50.373706+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:59:12.723748+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":17,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.370627+0000","last_change":"2023-09-23T20:12:43.759472+0000","last_active":"2023-09-23T20:12:50.370627+0000","last_peered":"2023-09-23T20:12:50.370627+0000","last_clean":"2023-09-23T20:12:50.370627+0000","last_became_active":"2023-09-23T20:12:43.759261+0000","last_became_peered":"2023-09-23T20:12:43.759261+0000","last_unstale":"2023-09-23T20:12:50.370627+0000","last_undegraded":"2023-09-23T20:12:50.370627+0000","last_fullsized":"2023-09-23T20:12:50.370627+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":3,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:23:34.964302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.374441+0000","last_change":"2023-09-23T20:12:45.179990+0000","last_active":"2023-09-23T20:12:50.374441+0000","last_peered":"2023-09-23T20:12:50.374441+0000","last_clean":"2023-09-23T20:12:50.374441+0000","last_became_active":"2023-09-23T20:12:45.179728+0000","last_became_peered":"2023-09-23T20:12:45.179728+0000","last_unstale":"2023-09-23T20:12:50.374441+0000","last_undegraded":"2023-09-23T20:12:50.374441+0000","last_fullsized":"2023-09-23T20:12:50.374441+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:09:11.955320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.374205+0000","last_change":"2023-09-23T20:12:43.753000+0000","last_active":"2023-09-23T20:12:50.374205+0000","last_peered":"2023-09-23T20:12:50.374205+0000","last_clean":"2023-09-23T20:12:50.374205+0000","last_became_active":"2023-09-23T20:12:43.752714+0000","last_became_peered":"2023-09-23T20:12:43.752714+0000","last_unstale":"2023-09-23T20:12:50.374205+0000","last_undegraded":"2023-09-23T20:12:50.374205+0000","last_fullsized":"2023-09-23T20:12:50.374205+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:56:09.052225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.374337+0000","last_change":"2023-09-23T20:12:44.767475+0000","last_active":"2023-09-23T20:12:50.374337+0000","last_peered":"2023-09-23T20:12:50.374337+0000","last_clean":"2023-09-23T20:12:50.374337+0000","last_became_active":"2023-09-23T20:12:44.767199+0000","last_became_peered":"2023-09-23T20:12:44.767199+0000","last_unstale":"2023-09-23T20:12:50.374337+0000","last_undegraded":"2023-09-23T20:12:50.374337+0000","last_fullsized":"2023-09-23T20:12:50.374337+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:58:12.782887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.369529+0000","last_change":"2023-09-23T20:12:43.759944+0000","last_active":"2023-09-23T20:12:50.369529+0000","last_peered":"2023-09-23T20:12:50.369529+0000","last_clean":"2023-09-23T20:12:50.369529+0000","last_became_active":"2023-09-23T20:12:43.759617+0000","last_became_peered":"2023-09-23T20:12:43.759617+0000","last_unstale":"2023-09-23T20:12:50.369529+0000","last_undegraded":"2023-09-23T20:12:50.369529+0000","last_fullsized":"2023-09-23T20:12:50.369529+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:22:38.687584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.373466+0000","last_change":"2023-09-23T20:12:44.769490+0000","last_active":"2023-09-23T20:12:50.373466+0000","last_peered":"2023-09-23T20:12:50.373466+0000","last_clean":"2023-09-23T20:12:50.373466+0000","last_became_active":"2023-09-23T20:12:44.769284+0000","last_became_peered":"2023-09-23T20:12:44.769284+0000","last_unstale":"2023-09-23T20:12:50.373466+0000","last_undegraded":"2023-09-23T20:12:50.373466+0000","last_fullsized":"2023-09-23T20:12:50.373466+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:44:28.639674+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.374393+0000","last_change":"2023-09-23T20:12:43.762873+0000","last_active":"2023-09-23T20:12:50.374393+0000","last_peered":"2023-09-23T20:12:50.374393+0000","last_clean":"2023-09-23T20:12:50.374393+0000","last_became_active":"2023-09-23T20:12:43.762384+0000","last_became_peered":"2023-09-23T20:12:43.762384+0000","last_unstale":"2023-09-23T20:12:50.374393+0000","last_undegraded":"2023-09-23T20:12:50.374393+0000","last_fullsized":"2023-09-23T20:12:50.374393+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:51:30.037644+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.368938+0000","last_change":"2023-09-23T20:12:45.179234+0000","last_active":"2023-09-23T20:12:50.368938+0000","last_peered":"2023-09-23T20:12:50.368938+0000","last_clean":"2023-09-23T20:12:50.368938+0000","last_became_active":"2023-09-23T20:12:45.178892+0000","last_became_peered":"2023-09-23T20:12:45.178892+0000","last_unstale":"2023-09-23T20:12:50.368938+0000","last_undegraded":"2023-09-23T20:12:50.368938+0000","last_fullsized":"2023-09-23T20:12:50.368938+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:59:51.830813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.373051+0000","last_change":"2023-09-23T20:12:43.760050+0000","last_active":"2023-09-23T20:12:50.373051+0000","last_peered":"2023-09-23T20:12:50.373051+0000","last_clean":"2023-09-23T20:12:50.373051+0000","last_became_active":"2023-09-23T20:12:43.759738+0000","last_became_peered":"2023-09-23T20:12:43.759738+0000","last_unstale":"2023-09-23T20:12:50.373051+0000","last_undegraded":"2023-09-23T20:12:50.373051+0000","last_fullsized":"2023-09-23T20:12:50.373051+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:59:37.640763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.374840+0000","last_change":"2023-09-23T20:12:45.609879+0000","last_active":"2023-09-23T20:12:50.374840+0000","last_peered":"2023-09-23T20:12:50.374840+0000","last_clean":"2023-09-23T20:12:50.374840+0000","last_became_active":"2023-09-23T20:12:45.609533+0000","last_became_peered":"2023-09-23T20:12:45.609533+0000","last_unstale":"2023-09-23T20:12:50.374840+0000","last_undegraded":"2023-09-23T20:12:50.374840+0000","last_fullsized":"2023-09-23T20:12:50.374840+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:54:21.264486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"69'1","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2023-09-23T20:12:50.371277+0000","last_change":"2023-09-23T20:12:43.751650+0000","last_active":"2023-09-23T20:12:50.371277+0000","last_peered":"2023-09-23T20:12:50.371277+0000","last_clean":"2023-09-23T20:12:50.371277+0000","last_became_active":"2023-09-23T20:12:43.751383+0000","last_became_peered":"2023-09-23T20:12:43.751383+0000","last_unstale":"2023-09-23T20:12:50.371277+0000","last_undegraded":"2023-09-23T20:12:50.371277+0000","last_fullsized":"2023-09-23T20:12:50.371277+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:00:20.810162+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2306,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":6918,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201305,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6640,"kb_used_data":1320,"kb_used_omap":0,"kb_used_meta":5312,"kb_avail":93738512,"statfs":{"total":95995035648,"available":95988236288,"internally_reserved":0,"allocated":1351680,"data_stored":909748,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5439488},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:12:25 2023","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":0.52300000000000002,"15min":0.52300000000000002},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.74299999999999999,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.434},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":1.004,"5min":1.004,"15min":1.004},"last":0.51700000000000002}]},{"osd":1,"last update":"Sat Sep 23 20:12:25 2023","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.86799999999999999,"5min":0.86799999999999999,"15min":0.86799999999999999},"last":0.48299999999999998},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":1.0309999999999999,"5min":1.0309999999999999,"15min":1.0309999999999999},"last":0.71199999999999997}]},{"osd":2,"last update":"Sat Sep 23 20:12:25 2023","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.98999999999999999,"5min":0.98999999999999999,"15min":0.98999999999999999},"last":0.65500000000000003},{"interface":"front","average":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.92400000000000004,"5min":0.92400000000000004,"15min":0.92400000000000004},"last":0.56000000000000005}]},{"osd":3,"last update":"Sat Sep 23 20:12:25 2023","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":0.88800000000000001,"5min":0.88800000000000001,"15min":0.88800000000000001},"last":0.61099999999999999},{"interface":"front","average":{"1min":0.61899999999999999,"5min":0.61899999999999999,"15min":0.61899999999999999},"min":{"1min":0.45300000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"max":{"1min":0.97899999999999998,"5min":0.97899999999999998,"15min":0.97899999999999998},"last":0.60299999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:12:25 2023","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":0.52400000000000002,"15min":0.52400000000000002},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.94099999999999995,"5min":0.94099999999999995,"15min":0.94099999999999995},"last":0.57899999999999996},{"interface":"front","average":{"1min":0.54200000000000004,"5min":0.54200000000000004,"15min":0.54200000000000004},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.80700000000000005,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.73899999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:12:25 2023","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.85899999999999999,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.628},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":0.97699999999999998,"5min":0.97699999999999998,"15min":0.97699999999999998},"last":0.76400000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:12:25 2023","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":1.0569999999999999,"5min":1.0569999999999999,"15min":1.0569999999999999},"last":0.79300000000000004},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.879,"5min":0.879,"15min":0.879},"last":0.69799999999999995}]}]},{"osd":6,"up_from":47,"seq":201863462941,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6692,"kb_used_data":1308,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":93738460,"statfs":{"total":95995035648,"available":95988183040,"internally_reserved":0,"allocated":1339392,"data_stored":909602,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:13:08 2023","interfaces":[{"interface":"back","average":{"1min":0.46300000000000002,"5min":0.53900000000000003,"15min":0.55200000000000005},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.63500000000000001,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.54800000000000004},{"interface":"front","average":{"1min":0.52400000000000002,"5min":0.57099999999999995,"15min":0.57899999999999996},"min":{"1min":0.439,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.64700000000000002,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.623}]},{"osd":1,"last update":"Sat Sep 23 20:13:08 2023","interfaces":[{"interface":"back","average":{"1min":0.48599999999999999,"5min":0.55300000000000005,"15min":0.56399999999999995},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.629,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.64000000000000001},{"interface":"front","average":{"1min":0.50600000000000001,"5min":0.53000000000000003,"15min":0.53400000000000003},"min":{"1min":0.38,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.63300000000000001,"5min":0.72499999999999998,"15min":0.72499999999999998},"last":0.64900000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:13:08 2023","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":0.54300000000000004,"15min":0.54800000000000004},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.625,"5min":0.70699999999999996,"15min":0.70699999999999996},"last":0.61199999999999999},{"interface":"front","average":{"1min":0.63400000000000001,"5min":0.57799999999999996,"15min":0.56799999999999995},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":2.7090000000000001,"5min":2.7090000000000001,"15min":2.7090000000000001},"last":0.67100000000000004}]},{"osd":3,"last update":"Sat Sep 23 20:13:08 2023","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.54300000000000004,"15min":0.54500000000000004},"min":{"1min":0.42099999999999999,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.69799999999999995,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.50700000000000001},{"interface":"front","average":{"1min":0.63200000000000001,"5min":0.60099999999999998,"15min":0.59599999999999997},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":2.0670000000000002,"5min":2.0670000000000002,"15min":2.0670000000000002},"last":0.69699999999999995}]},{"osd":4,"last update":"Sat Sep 23 20:13:08 2023","interfaces":[{"interface":"back","average":{"1min":0.45400000000000001,"5min":0.46200000000000002,"15min":0.46300000000000002},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.63600000000000001,"5min":0.72299999999999998,"15min":0.72299999999999998},"last":0.57499999999999996},{"interface":"front","average":{"1min":0.47799999999999998,"5min":0.48899999999999999,"15min":0.49099999999999999},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.67400000000000004,"5min":0.73099999999999998,"15min":0.73099999999999998},"last":0.68600000000000005}]},{"osd":5,"last update":"Sat Sep 23 20:13:08 2023","interfaces":[{"interface":"back","average":{"1min":0.46899999999999997,"5min":0.48099999999999998,"15min":0.48299999999999998},"min":{"1min":0.35499999999999998,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.59399999999999997,"5min":0.69399999999999995,"15min":0.69399999999999995},"last":0.747},{"interface":"front","average":{"1min":0.51000000000000001,"5min":0.51100000000000001,"15min":0.51100000000000001},"min":{"1min":0.39000000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.72099999999999997,"5min":0.72099999999999997,"15min":0.72099999999999997},"last":0.59599999999999997}]},{"osd":7,"last update":"Sat Sep 23 20:12:26 2023","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.74399999999999999,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.71499999999999997},{"interface":"front","average":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.75,"5min":0.75,"15min":0.75},"last":0.73299999999999998}]}]},{"osd":1,"up_from":13,"seq":55834574898,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6864,"kb_used_data":712,"kb_used_omap":0,"kb_used_meta":6144,"kb_avail":93738288,"statfs":{"total":95995035648,"available":95988006912,"internally_reserved":0,"allocated":729088,"data_stored":317628,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6291456},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:12:27 2023","interfaces":[{"interface":"back","average":{"1min":1.3380000000000001,"5min":0.55900000000000005,"15min":0.41699999999999998},"min":{"1min":0.23499999999999999,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":17.280000000000001,"5min":17.280000000000001,"15min":17.280000000000001},"last":0.40000000000000002},{"interface":"front","average":{"1min":0.41099999999999998,"5min":0.39300000000000002,"15min":0.36699999999999999},"min":{"1min":0.28100000000000003,"5min":0.247,"15min":0.247},"max":{"1min":0.98999999999999999,"5min":0.98999999999999999,"15min":0.98999999999999999},"last":0.26600000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:12:54 2023","interfaces":[{"interface":"back","average":{"1min":0.442,"5min":0.44400000000000001,"15min":0.442},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":1.3740000000000001,"5min":1.3740000000000001,"15min":1.3740000000000001},"last":0.371},{"interface":"front","average":{"1min":0.40500000000000003,"5min":0.41799999999999998,"15min":0.41899999999999998},"min":{"1min":0.26700000000000002,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":1.302,"5min":1.302,"15min":1.302},"last":0.28399999999999997}]},{"osd":3,"last update":"Sat Sep 23 20:13:12 2023","interfaces":[{"interface":"back","average":{"1min":0.94699999999999995,"5min":0.59399999999999997,"15min":0.54400000000000004},"min":{"1min":0.224,"5min":0.224,"15min":0.224},"max":{"1min":7.3949999999999996,"5min":7.3949999999999996,"15min":7.3949999999999996},"last":0.33000000000000002},{"interface":"front","average":{"1min":0.47099999999999997,"5min":0.48899999999999999,"15min":0.502},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":1.762,"5min":1.762,"15min":1.762},"last":0.35899999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:12:27 2023","interfaces":[{"interface":"back","average":{"1min":0.58499999999999996,"5min":0.67300000000000004,"15min":0.68799999999999994},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":0.79700000000000004,"5min":2.226,"15min":2.226},"last":0.63200000000000001},{"interface":"front","average":{"1min":1.288,"5min":0.74299999999999999,"15min":0.65200000000000002},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":12.914999999999999,"5min":12.914999999999999,"15min":12.914999999999999},"last":0.52000000000000002}]},{"osd":5,"last update":"Sat Sep 23 20:12:48 2023","interfaces":[{"interface":"back","average":{"1min":0.61399999999999999,"5min":0.63,"15min":0.63200000000000001},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.91400000000000003,"5min":0.91400000000000003,"15min":0.91400000000000003},"last":0.52800000000000002},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.60699999999999998,"15min":0.61299999999999999},"min":{"1min":0.438,"5min":0.435,"15min":0.435},"max":{"1min":0.755,"5min":0.89200000000000002,"15min":0.89200000000000002},"last":0.67200000000000004}]},{"osd":6,"last update":"Sat Sep 23 20:13:04 2023","interfaces":[{"interface":"back","average":{"1min":1.2470000000000001,"5min":0.74099999999999999,"15min":0.65700000000000003},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":12.824,"5min":12.824,"15min":12.824},"last":0.56799999999999995},{"interface":"front","average":{"1min":0.76900000000000002,"5min":0.73899999999999999,"15min":0.73399999999999999},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":3.5390000000000001,"5min":3.5390000000000001,"15min":3.5390000000000001},"last":0.57499999999999996}]},{"osd":7,"last update":"Sat Sep 23 20:12:27 2023","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"min":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":0.90700000000000003,"5min":0.90700000000000003,"15min":0.90700000000000003},"last":0.70599999999999996},{"interface":"front","average":{"1min":1.29,"5min":1.29,"15min":1.29},"min":{"1min":0.51000000000000001,"5min":0.51000000000000001,"15min":0.51000000000000001},"max":{"1min":12.878,"5min":12.878,"15min":12.878},"last":0.68400000000000005}]}]},{"osd":0,"up_from":8,"seq":34359738421,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":7452,"kb_used_data":1300,"kb_used_omap":0,"kb_used_meta":6144,"kb_avail":93737700,"statfs":{"total":95995035648,"available":95987404800,"internally_reserved":0,"allocated":1331200,"data_stored":908572,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6291456},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:12:31 2023","interfaces":[{"interface":"back","average":{"1min":0.39200000000000002,"5min":0.35399999999999998,"15min":0.33700000000000002},"min":{"1min":0.25800000000000001,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":1.0860000000000001,"5min":1.0860000000000001,"15min":1.0860000000000001},"last":0.29899999999999999},{"interface":"front","average":{"1min":0.39300000000000002,"5min":0.36899999999999999,"15min":0.35099999999999998},"min":{"1min":0.24299999999999999,"5min":0.217,"15min":0.217},"max":{"1min":1.0309999999999999,"5min":1.0309999999999999,"15min":1.0309999999999999},"last":0.38}]},{"osd":2,"last update":"Sat Sep 23 20:12:52 2023","interfaces":[{"interface":"back","average":{"1min":0.41199999999999998,"5min":0.42399999999999999,"15min":0.42399999999999999},"min":{"1min":0.28699999999999998,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":1.0640000000000001,"5min":1.0640000000000001,"15min":1.0640000000000001},"last":0.32900000000000001},{"interface":"front","average":{"1min":0.42599999999999999,"5min":0.441,"15min":0.442},"min":{"1min":0.31,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":1.016,"5min":1.016,"15min":1.016},"last":0.36199999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:12:11 2023","interfaces":[{"interface":"back","average":{"1min":0.40100000000000002,"5min":0.441,"15min":0.44800000000000001},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.55700000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"last":0.39600000000000002},{"interface":"front","average":{"1min":0.44700000000000001,"5min":0.48599999999999999,"15min":0.49199999999999999},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.58099999999999996,"5min":0.60299999999999998,"15min":0.60299999999999998},"last":0.40999999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:12:31 2023","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.58299999999999996,"15min":0.58899999999999997},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.93200000000000005,"5min":0.93200000000000005,"15min":0.93200000000000005},"last":0.52100000000000002},{"interface":"front","average":{"1min":0.57199999999999995,"5min":0.59299999999999997,"15min":0.59699999999999998},"min":{"1min":0.42499999999999999,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.873,"5min":0.873,"15min":0.873},"last":0.53300000000000003}]},{"osd":5,"last update":"Sat Sep 23 20:12:52 2023","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.61799999999999999,"15min":0.626},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.85799999999999998,"5min":0.88400000000000001,"15min":0.88400000000000001},"last":0.58699999999999997},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.63400000000000001,"15min":0.64300000000000002},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.90300000000000002,"5min":0.93300000000000005,"15min":0.93300000000000005},"last":0.57499999999999996}]},{"osd":6,"last update":"Sat Sep 23 20:13:10 2023","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":0.58999999999999997,"15min":0.59799999999999998},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.79100000000000004,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.56100000000000005},{"interface":"front","average":{"1min":0.56999999999999995,"5min":0.60399999999999998,"15min":0.60999999999999999},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.88800000000000001,"5min":0.88800000000000001,"15min":0.88800000000000001},"last":0.54700000000000004}]},{"osd":7,"last update":"Sat Sep 23 20:12:28 2023","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.89100000000000001,"5min":0.89100000000000001,"15min":0.89100000000000001},"last":0.61199999999999999},{"interface":"front","average":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.91700000000000004,"5min":0.91700000000000004,"15min":0.91700000000000004},"last":0.50800000000000001}]}]},{"osd":2,"up_from":18,"seq":77309411374,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6680,"kb_used_data":720,"kb_used_omap":0,"kb_used_meta":5952,"kb_avail":93738472,"statfs":{"total":95995035648,"available":95988195328,"internally_reserved":0,"allocated":737280,"data_stored":318216,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6094848},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:12:49 2023","interfaces":[{"interface":"back","average":{"1min":0.39100000000000001,"5min":0.374,"15min":0.36099999999999999},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.80700000000000005,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.26000000000000001},{"interface":"front","average":{"1min":0.46999999999999997,"5min":0.40300000000000002,"15min":0.38300000000000001},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":1.423,"5min":1.423,"15min":1.423},"last":0.28000000000000003}]},{"osd":1,"last update":"Sat Sep 23 20:12:49 2023","interfaces":[{"interface":"back","average":{"1min":0.5,"5min":0.42899999999999999,"15min":0.41299999999999998},"min":{"1min":0.223,"5min":0.223,"15min":0.223},"max":{"1min":2.387,"5min":2.387,"15min":2.387},"last":0.307},{"interface":"front","average":{"1min":0.41599999999999998,"5min":0.42499999999999999,"15min":0.41299999999999998},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.70399999999999996,"5min":0.75800000000000001,"15min":0.75800000000000001},"last":0.32800000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:13:10 2023","interfaces":[{"interface":"back","average":{"1min":0.439,"5min":0.48299999999999998,"15min":0.497},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.88600000000000001,"5min":0.88600000000000001,"15min":0.88600000000000001},"last":0.34799999999999998},{"interface":"front","average":{"1min":0.46800000000000003,"5min":0.50600000000000001,"15min":0.51800000000000002},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":1.0189999999999999,"5min":1.0189999999999999,"15min":1.0189999999999999},"last":0.36199999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:12:31 2023","interfaces":[{"interface":"back","average":{"1min":0.61699999999999999,"5min":0.60699999999999998,"15min":0.60499999999999998},"min":{"1min":0.42499999999999999,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.95399999999999996,"5min":0.95399999999999996,"15min":0.95399999999999996},"last":0.442},{"interface":"front","average":{"1min":0.67500000000000004,"5min":0.65500000000000003,"15min":0.65200000000000002},"min":{"1min":0.499,"5min":0.47199999999999998,"15min":0.47199999999999998},"max":{"1min":0.93700000000000006,"5min":0.93700000000000006,"15min":0.93700000000000006},"last":0.622}]},{"osd":5,"last update":"Sat Sep 23 20:12:49 2023","interfaces":[{"interface":"back","average":{"1min":0.63800000000000001,"5min":0.66100000000000003,"15min":0.66500000000000004},"min":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":0.82899999999999996,"5min":0.98599999999999999,"15min":0.98599999999999999},"last":0.59799999999999998},{"interface":"front","average":{"1min":0.64400000000000002,"5min":0.67500000000000004,"15min":0.68000000000000005},"min":{"1min":0.44,"5min":0.435,"15min":0.435},"max":{"1min":0.84399999999999997,"5min":1.0409999999999999,"15min":1.0409999999999999},"last":0.47199999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:13:05 2023","interfaces":[{"interface":"back","average":{"1min":0.63500000000000001,"5min":0.66600000000000004,"15min":0.67100000000000004},"min":{"1min":0.48399999999999999,"5min":0.47099999999999997,"15min":0.47099999999999997},"max":{"1min":0.81599999999999995,"5min":0.96199999999999997,"15min":0.96199999999999997},"last":0.48399999999999999},{"interface":"front","average":{"1min":0.64000000000000001,"5min":0.70499999999999996,"15min":0.71599999999999997},"min":{"1min":0.46500000000000002,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":0.85499999999999998,"5min":0.91700000000000004,"15min":0.91700000000000004},"last":0.57099999999999995}]},{"osd":7,"last update":"Sat Sep 23 20:12:25 2023","interfaces":[{"interface":"back","average":{"1min":0.68799999999999994,"5min":0.68799999999999994,"15min":0.68799999999999994},"min":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"max":{"1min":0.97399999999999998,"5min":0.97399999999999998,"15min":0.97399999999999998},"last":0.54100000000000004},{"interface":"front","average":{"1min":0.66700000000000004,"5min":0.66700000000000004,"15min":0.66700000000000004},"min":{"1min":0.46200000000000002,"5min":0.46200000000000002,"15min":0.46200000000000002},"max":{"1min":0.99199999999999999,"5min":0.99199999999999999,"15min":0.99199999999999999},"last":0.58399999999999996}]}]},{"osd":3,"up_from":25,"seq":107374182440,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6164,"kb_used_data":716,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":93738988,"statfs":{"total":95995035648,"available":95988723712,"internally_reserved":0,"allocated":733184,"data_stored":317718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:13:07 2023","interfaces":[{"interface":"back","average":{"1min":0.94299999999999995,"5min":0.495,"15min":0.41699999999999998},"min":{"1min":0.26200000000000001,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":9.9580000000000002,"5min":9.9580000000000002,"15min":9.9580000000000002},"last":0.32300000000000001},{"interface":"front","average":{"1min":0.96299999999999997,"5min":0.51400000000000001,"15min":0.44600000000000001},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":11.028,"5min":11.028,"15min":11.028},"last":0.40000000000000002}]},{"osd":1,"last update":"Sat Sep 23 20:13:07 2023","interfaces":[{"interface":"back","average":{"1min":0.83199999999999996,"5min":0.51600000000000001,"15min":0.46300000000000002},"min":{"1min":0.27800000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":8.3239999999999998,"5min":8.3239999999999998,"15min":8.3239999999999998},"last":0.26800000000000002},{"interface":"front","average":{"1min":0.995,"5min":0.58399999999999996,"15min":0.51300000000000001},"min":{"1min":0.33500000000000002,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":9.8930000000000007,"5min":9.8930000000000007,"15min":9.8930000000000007},"last":0.36399999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:13:07 2023","interfaces":[{"interface":"back","average":{"1min":0.96499999999999997,"5min":0.55400000000000005,"15min":0.49199999999999999},"min":{"1min":0.308,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":9.9260000000000002,"5min":9.9260000000000002,"15min":9.9260000000000002},"last":0.42299999999999999},{"interface":"front","average":{"1min":0.38400000000000001,"5min":0.45200000000000001,"15min":0.47099999999999997},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.60499999999999998,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":0.45100000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:12:25 2023","interfaces":[{"interface":"back","average":{"1min":1.01,"5min":0.71799999999999997,"15min":0.66900000000000004},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":8.2949999999999999,"5min":8.2949999999999999,"15min":8.2949999999999999},"last":0.50700000000000001},{"interface":"front","average":{"1min":1.073,"5min":0.73299999999999998,"15min":0.67700000000000005},"min":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"max":{"1min":9.8539999999999992,"5min":9.8539999999999992,"15min":9.8539999999999992},"last":0.59499999999999997}]},{"osd":5,"last update":"Sat Sep 23 20:12:48 2023","interfaces":[{"interface":"back","average":{"1min":1.1279999999999999,"5min":0.71399999999999997,"15min":0.64500000000000002},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":10.917,"5min":10.917,"15min":10.917},"last":0.60499999999999998},{"interface":"front","average":{"1min":0.97599999999999998,"5min":0.68400000000000005,"15min":0.63600000000000001},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":8.2609999999999992,"5min":8.2609999999999992,"15min":8.2609999999999992},"last":0.61799999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:13:07 2023","interfaces":[{"interface":"back","average":{"1min":1.1519999999999999,"5min":0.748,"15min":0.68100000000000005},"min":{"1min":0.46300000000000002,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":9.8119999999999994,"5min":9.8119999999999994,"15min":9.8119999999999994},"last":0.54300000000000004},{"interface":"front","average":{"1min":1.2,"5min":0.75,"15min":0.67500000000000004},"min":{"1min":0.439,"5min":0.439,"15min":0.439},"max":{"1min":10.885,"5min":10.885,"15min":10.885},"last":0.64500000000000002}]},{"osd":7,"last update":"Sat Sep 23 20:12:25 2023","interfaces":[{"interface":"back","average":{"1min":1.105,"5min":1.105,"15min":1.105},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":10.846,"5min":10.846,"15min":10.846},"last":0.68600000000000005},{"interface":"front","average":{"1min":1.0049999999999999,"5min":1.0049999999999999,"15min":1.0049999999999999},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":8.2170000000000005,"5min":8.2170000000000005,"15min":8.2170000000000005},"last":0.55600000000000005}]}]},{"osd":4,"up_from":30,"seq":128849018917,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6244,"kb_used_data":732,"kb_used_omap":0,"kb_used_meta":5504,"kb_avail":93738908,"statfs":{"total":95995035648,"available":95988641792,"internally_reserved":0,"allocated":749568,"data_stored":318987,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5636096},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:12:26 2023","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.54000000000000004,"15min":0.54100000000000004},"min":{"1min":0.34599999999999997,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.68300000000000005,"5min":0.83599999999999997,"15min":0.83599999999999997},"last":0.57499999999999996},{"interface":"front","average":{"1min":0.51600000000000001,"5min":0.54600000000000004,"15min":0.55100000000000005},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.78700000000000003,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.59699999999999998}]},{"osd":1,"last update":"Sat Sep 23 20:12:26 2023","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.56200000000000006,"15min":0.56999999999999995},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.72999999999999998,"5min":0.86499999999999999,"15min":0.86499999999999999},"last":0.65700000000000003},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.55300000000000005,"15min":0.55700000000000005},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.70999999999999996,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.499}]},{"osd":2,"last update":"Sat Sep 23 20:12:26 2023","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.56999999999999995,"15min":0.57699999999999996},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.77200000000000002,"5min":0.81599999999999995,"15min":0.81599999999999995},"last":0.61099999999999999},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.57899999999999996,"15min":0.58099999999999996},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.86199999999999999,"5min":0.91500000000000004,"15min":0.91500000000000004},"last":0.63900000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:12:26 2023","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.56399999999999995,"15min":0.56999999999999995},"min":{"1min":0.36099999999999999,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.69599999999999995,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.56200000000000006},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.55000000000000004,"15min":0.54700000000000004},"min":{"1min":0.40200000000000002,"5min":0.376,"15min":0.376},"max":{"1min":0.77100000000000002,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.623}]},{"osd":5,"last update":"Sat Sep 23 20:12:45 2023","interfaces":[{"interface":"back","average":{"1min":0.48899999999999999,"5min":0.41699999999999998,"15min":0.40500000000000003},"min":{"1min":0.28100000000000003,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.83999999999999997,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.373},{"interface":"front","average":{"1min":0.48299999999999998,"5min":0.41099999999999998,"15min":0.39900000000000002},"min":{"1min":0.36299999999999999,"5min":0.248,"15min":0.248},"max":{"1min":0.73999999999999999,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.40400000000000003}]},{"osd":6,"last update":"Sat Sep 23 20:13:07 2023","interfaces":[{"interface":"back","average":{"1min":0.48699999999999999,"5min":0.44400000000000001,"15min":0.437},"min":{"1min":0.28999999999999998,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.65500000000000003,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.69299999999999995},{"interface":"front","average":{"1min":0.51300000000000001,"5min":0.46600000000000003,"15min":0.45800000000000002},"min":{"1min":0.33500000000000002,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.745,"5min":0.745,"15min":0.745},"last":0.53800000000000003}]},{"osd":7,"last update":"Sat Sep 23 20:12:26 2023","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.90400000000000003,"5min":0.90400000000000003,"15min":0.90400000000000003},"last":0.66700000000000004},{"interface":"front","average":{"1min":0.52400000000000002,"5min":0.52400000000000002,"15min":0.52400000000000002},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.77400000000000002,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.58699999999999997}]}]},{"osd":5,"up_from":39,"seq":167503724578,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6104,"kb_used_data":720,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":93739048,"statfs":{"total":95995035648,"available":95988785152,"internally_reserved":0,"allocated":737280,"data_stored":318252,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:12:49 2023","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":0.55800000000000005,"15min":0.56599999999999995},"min":{"1min":0.39400000000000002,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.65800000000000003,"5min":1.0640000000000001,"15min":1.0640000000000001},"last":0.65900000000000003},{"interface":"front","average":{"1min":0.51900000000000002,"5min":0.51400000000000001,"15min":0.51300000000000001},"min":{"1min":0.378,"5min":0.312,"15min":0.312},"max":{"1min":0.76100000000000001,"5min":0.76100000000000001,"15min":0.76100000000000001},"last":0.64900000000000002}]},{"osd":1,"last update":"Sat Sep 23 20:12:49 2023","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.53200000000000003,"15min":0.53000000000000003},"min":{"1min":0.35199999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.78400000000000003,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.70099999999999996},{"interface":"front","average":{"1min":0.52400000000000002,"5min":0.54600000000000004,"15min":0.55000000000000004},"min":{"1min":0.34699999999999998,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.72399999999999998,"5min":1.1519999999999999,"15min":1.1519999999999999},"last":0.67200000000000004}]},{"osd":2,"last update":"Sat Sep 23 20:12:49 2023","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.55900000000000005,"15min":0.56299999999999994},"min":{"1min":0.42099999999999999,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.70199999999999996,"5min":1.1719999999999999,"15min":1.1719999999999999},"last":0.72299999999999998},{"interface":"front","average":{"1min":0.53300000000000003,"5min":0.54100000000000004,"15min":0.54200000000000004},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.76000000000000001,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.59299999999999997}]},{"osd":3,"last update":"Sat Sep 23 20:12:49 2023","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":0.57999999999999996,"15min":0.58599999999999997},"min":{"1min":0.374,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.71699999999999997,"5min":1.2130000000000001,"15min":1.2130000000000001},"last":0.68500000000000005},{"interface":"front","average":{"1min":0.67200000000000004,"5min":0.55200000000000005,"15min":0.53200000000000003},"min":{"1min":0.46100000000000002,"5min":0.371,"15min":0.371},"max":{"1min":2.2599999999999998,"5min":2.2599999999999998,"15min":2.2599999999999998},"last":0.73999999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:12:49 2023","interfaces":[{"interface":"back","average":{"1min":0.46899999999999997,"5min":0.40400000000000003,"15min":0.39300000000000002},"min":{"1min":0.28799999999999998,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.72399999999999998,"5min":0.72399999999999998,"15min":0.72399999999999998},"last":0.52200000000000002},{"interface":"front","average":{"1min":0.47899999999999998,"5min":0.45100000000000001,"15min":0.44600000000000001},"min":{"1min":0.30399999999999999,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.65300000000000002,"5min":1.4450000000000001,"15min":1.4450000000000001},"last":0.46000000000000002}]},{"osd":6,"last update":"Sat Sep 23 20:13:09 2023","interfaces":[{"interface":"back","average":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.69899999999999995,"5min":0.69899999999999995,"15min":0.69899999999999995},"last":0.61599999999999999},{"interface":"front","average":{"1min":0.52500000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"min":{"1min":0.38900000000000001,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.71799999999999997,"5min":0.73699999999999999,"15min":0.73699999999999999},"last":0.63}]},{"osd":7,"last update":"Sat Sep 23 20:12:25 2023","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.68899999999999995,"5min":0.68899999999999995,"15min":0.68899999999999995},"last":0.55200000000000005},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.71699999999999997,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.48699999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1682,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1606,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:13:20.820 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2023-09-23T20:13:20.820 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2023-09-23T20:13:20.820 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2023-09-23T20:13:20.820 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph health --format=json 2023-09-23T20:13:21.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:13:20 smithi018 ceph-mon[109137]: pgmap v59: 73 pgs: 73 active+clean; 579 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:13:21.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:13:20 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:13:21.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:13:20 smithi018 ceph-mon[95889]: pgmap v59: 73 pgs: 73 active+clean; 579 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:13:21.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:13:20 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:13:21.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:13:20 smithi149 ceph-mon[100764]: pgmap v59: 73 pgs: 73 active+clean; 579 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:13:21.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:13:20 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:13:21.425 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:13:21.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:13:21 smithi018 ceph-mon[109137]: from='client.24701 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:13:21.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:13:21 smithi018 ceph-mon[95889]: from='client.24701 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:13:22.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:13:21 smithi149 ceph-mon[100764]: from='client.24701 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:13:22.902 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:13:22.902 INFO:teuthology.orchestra.run.smithi018.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2023-09-23T20:13:23.015 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:13:22 smithi018 ceph-mon[109137]: pgmap v60: 73 pgs: 73 active+clean; 579 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:13:23.015 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:13:22 smithi018 ceph-mon[95889]: pgmap v60: 73 pgs: 73 active+clean; 579 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:13:23.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:13:22 smithi149 ceph-mon[100764]: pgmap v60: 73 pgs: 73 active+clean; 579 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:13:23.274 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2023-09-23T20:13:23.274 INFO:teuthology.run_tasks:Running task print... 2023-09-23T20:13:23.283 INFO:teuthology.task.print:**** upgrading first half of cluster, with stress **** 2023-09-23T20:13:23.283 INFO:teuthology.run_tasks:Running task parallel... 2023-09-23T20:13:23.292 INFO:teuthology.task.parallel:starting parallel... 2023-09-23T20:13:23.292 INFO:teuthology.task.parallel:In parallel, running task sequential... 2023-09-23T20:13:23.293 INFO:teuthology.task.sequential:In sequential, running task workunit... 2023-09-23T20:13:23.296 INFO:tasks.workunit:Pulling workunits from ref quincy 2023-09-23T20:13:23.297 INFO:tasks.workunit:Making a separate scratch dir for every client... 2023-09-23T20:13:23.297 DEBUG:teuthology.orchestra.run.smithi018:> stat -- /home/ubuntu/cephtest/mnt.0 2023-09-23T20:13:23.297 INFO:teuthology.task.parallel:In parallel, running task sequential... 2023-09-23T20:13:23.297 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2023-09-23T20:13:23.298 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi018.front.sepia.ceph.com 2023-09-23T20:13:23.298 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph config set mgr mgr/cephadm/daemon_cache_timeout 60' 2023-09-23T20:13:23.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:13:23.317 INFO:teuthology.orchestra.run.smithi018.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2023-09-23T20:13:23.317 DEBUG:teuthology.orchestra.run.smithi018:> mkdir -- /home/ubuntu/cephtest/mnt.0 2023-09-23T20:13:23.374 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2023-09-23T20:13:23.374 DEBUG:teuthology.orchestra.run.smithi018:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2023-09-23T20:13:23.431 INFO:tasks.workunit:timeout=3h 2023-09-23T20:13:23.431 INFO:tasks.workunit:cleanup=True 2023-09-23T20:13:23.431 DEBUG:teuthology.orchestra.run.smithi018:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone --depth 1 --branch quincy https://github.com/ceph/ceph.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 2023-09-23T20:13:23.488 INFO:tasks.workunit.client.0.smithi018.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2023-09-23T20:13:23.910 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:13:24.113 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:13:23 smithi018 ceph-mon[109137]: from='client.? 172.21.15.18:0/1325894548' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-09-23T20:13:24.113 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:13:23 smithi018 ceph-mon[95889]: from='client.? 172.21.15.18:0/1325894548' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-09-23T20:13:24.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:13:23 smithi149 ceph-mon[100764]: from='client.? 172.21.15.18:0/1325894548' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-09-23T20:13:24.216 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:13:23 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[102880]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:13:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T20:13:25.140 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:13:24 smithi018 ceph-mon[109137]: pgmap v61: 73 pgs: 73 active+clean; 579 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:13:25.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:13:24 smithi018 ceph-mon[95889]: pgmap v61: 73 pgs: 73 active+clean; 579 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:13:25.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:13:24 smithi149 ceph-mon[100764]: pgmap v61: 73 pgs: 73 active+clean; 579 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:13:25.640 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:13:25 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: [23/Sep/2023:20:13:25] ENGINE Bus STOPPING 2023-09-23T20:13:25.641 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:13:25 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: [23/Sep/2023:20:13:25] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T20:13:25.641 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:13:25 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: [23/Sep/2023:20:13:25] ENGINE Bus STOPPED 2023-09-23T20:13:25.641 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:13:25 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: [23/Sep/2023:20:13:25] ENGINE Bus STARTING 2023-09-23T20:13:25.641 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:13:25 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: [23/Sep/2023:20:13:25] ENGINE Serving on http://:::9283 2023-09-23T20:13:25.641 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:13:25 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: [23/Sep/2023:20:13:25] ENGINE Bus STARTED 2023-09-23T20:13:25.759 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph config set global log_to_journald false --force' 2023-09-23T20:13:26.360 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:13:26 smithi018 ceph-mon[95889]: from='client.? ' entity='client.admin' 2023-09-23T20:13:26.360 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:13:26 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:13:26.360 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:13:26 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:13:26.360 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:13:26 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:13:26.360 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:13:26 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:13:26.360 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:13:26 smithi018 ceph-mon[95889]: pgmap v62: 73 pgs: 73 active+clean; 579 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:13:26.411 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:13:26.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:13:26 smithi018 ceph-mon[109137]: from='client.? ' entity='client.admin' 2023-09-23T20:13:26.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:13:26 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:13:26.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:13:26 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:13:26.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:13:26 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:13:26.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:13:26 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:13:26.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:13:26 smithi018 ceph-mon[109137]: pgmap v62: 73 pgs: 73 active+clean; 579 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:13:26.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:13:26 smithi149 ceph-mon[100764]: from='client.? ' entity='client.admin' 2023-09-23T20:13:26.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:13:26 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:13:26.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:13:26 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:13:26.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:13:26 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:13:26.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:13:26 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:13:26.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:13:26 smithi149 ceph-mon[100764]: pgmap v62: 73 pgs: 73 active+clean; 579 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:13:27.768 DEBUG:teuthology.orchestra.run.smithi018:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2023-09-23T20:13:27.794 INFO:tasks.workunit.client.0.smithi018.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2023-09-23T20:13:27.796 INFO:tasks.workunit.client.0.smithi018.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-09-23T20:13:27.796 INFO:tasks.workunit.client.0.smithi018.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2023-09-23T20:13:28.884 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:13:28 smithi018 ceph-mon[95889]: pgmap v63: 73 pgs: 73 active+clean; 579 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:13:28.884 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:13:28 smithi018 ceph-mon[109137]: pgmap v63: 73 pgs: 73 active+clean; 579 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:13:28.885 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1' 2023-09-23T20:13:28.972 INFO:tasks.workunit.client.0.smithi018.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2023-09-23T20:13:29.040 INFO:tasks.workunit.client.0.smithi018.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2023-09-23T20:13:29.079 INFO:tasks.workunit.client.0.smithi018.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-09-23T20:13:29.080 INFO:tasks.workunit.client.0.smithi018.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-09-23T20:13:29.080 INFO:tasks.workunit.client.0.smithi018.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2023-09-23T20:13:29.121 INFO:tasks.workunit.client.0.smithi018.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-09-23T20:13:29.125 DEBUG:teuthology.orchestra.run.smithi018:> set -ex 2023-09-23T20:13:29.125 DEBUG:teuthology.orchestra.run.smithi018:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2023-09-23T20:13:29.143 INFO:tasks.workunit:Running workunits matching rbd/import_export.sh on client.0... 2023-09-23T20:13:29.144 INFO:tasks.workunit:Running workunit rbd/import_export.sh... 2023-09-23T20:13:29.145 DEBUG:teuthology.orchestra.run.smithi018:workunit test rbd/import_export.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 RBD_CREATE_ARGS=--new-format adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/rbd/import_export.sh 2023-09-23T20:13:29.206 INFO:tasks.workunit.client.0.smithi018.stderr:+ export RBD_FORCE_ALLOW_V1=1 2023-09-23T20:13:29.206 INFO:tasks.workunit.client.0.smithi018.stderr:+ RBD_FORCE_ALLOW_V1=1 2023-09-23T20:13:29.206 INFO:tasks.workunit.client.0.smithi018.stderr:+ TMPDIR=/tmp/rbd_import_export_159798 2023-09-23T20:13:29.206 INFO:tasks.workunit.client.0.smithi018.stderr:+ rm -rf /tmp/rbd_import_export_159798 2023-09-23T20:13:29.207 INFO:tasks.workunit.client.0.smithi018.stderr:+ mkdir /tmp/rbd_import_export_159798 2023-09-23T20:13:29.208 INFO:tasks.workunit.client.0.smithi018.stderr:+ trap 'rm -rf /tmp/rbd_import_export_159798' INT TERM EXIT 2023-09-23T20:13:29.208 INFO:tasks.workunit.client.0.smithi018.stderr:+ mkdir foo.159798 2023-09-23T20:13:29.208 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd import foo.159798 foo.dir 2023-09-23T20:13:29.215 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:13:28 smithi149 ceph-mon[100764]: pgmap v63: 73 pgs: 73 active+clean; 579 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:13:29.250 INFO:tasks.workunit.client.0.smithi018.stderr:rbd: cannot import a directory 2023-09-23T20:13:29.250 INFO:tasks.workunit.client.0.smithi018.stderr: Importing image: 0% complete...failed. 2023-09-23T20:13:29.251 INFO:tasks.workunit.client.0.smithi018.stderr:rbd: import failed: (21) Is a directory 2023-09-23T20:13:29.253 INFO:tasks.workunit.client.0.smithi018.stderr:+ true 2023-09-23T20:13:29.253 INFO:tasks.workunit.client.0.smithi018.stderr:+ rmdir foo.159798 2023-09-23T20:13:29.262 INFO:tasks.workunit.client.0.smithi018.stderr:+ dd if=/bin/sh of=/tmp/rbd_import_export_159798/img bs=1k count=1 seek=10 2023-09-23T20:13:29.262 INFO:tasks.workunit.client.0.smithi018.stderr:1+0 records in 2023-09-23T20:13:29.262 INFO:tasks.workunit.client.0.smithi018.stderr:1+0 records out 2023-09-23T20:13:29.263 INFO:tasks.workunit.client.0.smithi018.stderr:1024 bytes (1.0 kB, 1.0 KiB) copied, 4.9431e-05 s, 20.7 MB/s 2023-09-23T20:13:29.263 INFO:tasks.workunit.client.0.smithi018.stderr:+ dd if=/bin/dd of=/tmp/rbd_import_export_159798/img bs=1k count=10 seek=100 2023-09-23T20:13:29.263 INFO:tasks.workunit.client.0.smithi018.stderr:10+0 records in 2023-09-23T20:13:29.263 INFO:tasks.workunit.client.0.smithi018.stderr:10+0 records out 2023-09-23T20:13:29.263 INFO:tasks.workunit.client.0.smithi018.stderr:10240 bytes (10 kB, 10 KiB) copied, 6.0875e-05 s, 168 MB/s 2023-09-23T20:13:29.263 INFO:tasks.workunit.client.0.smithi018.stderr:+ dd if=/bin/rm of=/tmp/rbd_import_export_159798/img bs=1k count=100 seek=1000 2023-09-23T20:13:29.264 INFO:tasks.workunit.client.0.smithi018.stderr:70+1 records in 2023-09-23T20:13:29.264 INFO:tasks.workunit.client.0.smithi018.stderr:70+1 records out 2023-09-23T20:13:29.265 INFO:tasks.workunit.client.0.smithi018.stderr:72048 bytes (72 kB, 70 KiB) copied, 0.000207178 s, 348 MB/s 2023-09-23T20:13:29.265 INFO:tasks.workunit.client.0.smithi018.stderr:+ dd if=/bin/ls of=/tmp/rbd_import_export_159798/img bs=1k seek=10000 2023-09-23T20:13:29.265 INFO:tasks.workunit.client.0.smithi018.stderr:139+1 records in 2023-09-23T20:13:29.265 INFO:tasks.workunit.client.0.smithi018.stderr:139+1 records out 2023-09-23T20:13:29.265 INFO:tasks.workunit.client.0.smithi018.stderr:143224 bytes (143 kB, 140 KiB) copied, 0.000366333 s, 391 MB/s 2023-09-23T20:13:29.266 INFO:tasks.workunit.client.0.smithi018.stderr:+ dd if=/bin/ln of=/tmp/rbd_import_export_159798/img bs=1k seek=100000 2023-09-23T20:13:29.270 INFO:tasks.workunit.client.0.smithi018.stderr:70+1 records in 2023-09-23T20:13:29.270 INFO:tasks.workunit.client.0.smithi018.stderr:70+1 records out 2023-09-23T20:13:29.270 INFO:tasks.workunit.client.0.smithi018.stderr:72168 bytes (72 kB, 70 KiB) copied, 0.00358302 s, 20.1 MB/s 2023-09-23T20:13:29.270 INFO:tasks.workunit.client.0.smithi018.stderr:+ dd if=/bin/grep of=/tmp/rbd_import_export_159798/img bs=1k seek=1000000 2023-09-23T20:13:29.271 INFO:tasks.workunit.client.0.smithi018.stderr:189+1 records in 2023-09-23T20:13:29.271 INFO:tasks.workunit.client.0.smithi018.stderr:189+1 records out 2023-09-23T20:13:29.271 INFO:tasks.workunit.client.0.smithi018.stderr:193768 bytes (194 kB, 189 KiB) copied, 0.000475916 s, 407 MB/s 2023-09-23T20:13:29.271 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd rm testimg 2023-09-23T20:13:29.313 INFO:tasks.workunit.client.0.smithi018.stderr: Removing image: 0% complete...failed. 2023-09-23T20:13:29.313 INFO:tasks.workunit.client.0.smithi018.stderr:rbd: delete error: (2) No such file or directory 2023-09-23T20:13:29.316 INFO:tasks.workunit.client.0.smithi018.stderr:+ true 2023-09-23T20:13:29.316 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd import --new-format /tmp/rbd_import_export_159798/img testimg 2023-09-23T20:13:29.537 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:13:29.641 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:13:29 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:13:29] "GET /metrics HTTP/1.1" 200 238782 "" "Prometheus/2.43.0" 2023-09-23T20:13:30.030 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:13:29 smithi018 ceph-mon[109137]: pgmap v64: 73 pgs: 73 active+clean; 579 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:13:30.031 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:13:29 smithi018 ceph-mon[95889]: pgmap v64: 73 pgs: 73 active+clean; 579 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:13:30.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:13:29 smithi149 ceph-mon[100764]: pgmap v64: 73 pgs: 73 active+clean; 579 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:13:30.866 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:13:30 smithi018 ceph-mon[95889]: osdmap e72: 8 total, 8 up, 8 in 2023-09-23T20:13:30.949 INFO:teuthology.orchestra.run.smithi018.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T20:13:31.140 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:13:30 smithi018 ceph-mon[109137]: osdmap e72: 8 total, 8 up, 8 in 2023-09-23T20:13:31.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:13:30 smithi149 ceph-mon[100764]: osdmap e72: 8 total, 8 up, 8 in 2023-09-23T20:13:31.309 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch ps' 2023-09-23T20:13:31.405 INFO:tasks.workunit.client.0.smithi018.stderr: Importing image: 1% complete... Importing image: 2% complete... Importing image: 3% complete... Importing image: 4% complete... Importing image: 5% complete... Importing image: 6% complete... Importing image: 7% complete... Importing image: 8% complete... Importing image: 9% complete... Importing image: 10% complete... Importing image: 11% complete... Importing image: 12% complete... Importing image: 13% complete... Importing image: 14% complete... Importing image: 15% complete... Importing image: 16% complete... Importing image: 17% complete... Importing image: 18% complete... Importing image: 19% complete... Importing image: 20% complete... Importing image: 21% complete... Importing image: 22% complete... Importing image: 23% complete... Importing image: 24% complete... Importing image: 25% complete... Importing image: 26% complete... Importing image: 27% complete... Importing image: 28% complete... Importing image: 29% complete... Importing image: 30% complete... Importing image: 31% complete... Importing image: 32% complete... Importing image: 33% complete... Importing image: 34% complete... Importing image: 35% complete... Importing image: 36% complete... Importing image: 37% complete... Importing image: 38% complete... Importing image: 39% complete... Importing image: 40% complete... Importing image: 41% complete... Importing image: 42% complete... Importing image: 43% complete... Importing image: 44% complete... Importing image: 45% complete... Importing image: 46% complete... Importing image: 47% complete... Importing image: 48% complete... Importing image: 49% complete... Importing image: 50% complete... Importing image: 51% complete... Importing image: 52% complete... Importing image: 53% complete... Importing image: 54% complete... Importing image: 55% complete... Importing image: 56% complete... Importing image: 57% complete... Importing image: 58% complete... Importing image: 59% complete... Importing image: 60% complete... Importing image: 61% complete... Importing image: 62% complete... Importing image: 63% complete... Importing image: 64% complete... Importing image: 65% complete... Importing image: 66% complete... Importing image: 67% complete... Importing image: 68% complete... Importing image: 69% complete... Importing image: 70% complete... Importing image: 71% complete... Importing image: 72% complete... Importing image: 73% complete... Importing image: 74% complete... Importing image: 75% complete... Importing image: 76% complete... Importing image: 77% complete... Importing image: 78% complete... Importing image: 79% complete... Importing image: 80% complete... Importing image: 81% complete... Importing image: 82% complete... Importing image: 83% complete... Importing image: 84% complete... Importing image: 85% complete... Importing image: 86% complete... Importing image: 87% complete... Importing image: 88% complete... Importing image: 89% complete... Importing image: 90% complete... Importing image: 91% complete... Importing image: 92% complete... Importing image: 93% complete... Importing image: 94% complete... Importing image: 95% complete... Importing image: 96% complete... Importing image: 97% complete... Importing image: 98% complete... Importing image: 99% complete... Importing image: 100% complete...done. 2023-09-23T20:13:31.406 INFO:tasks.workunit.client.0.smithi018.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T20:13:31.408 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd export testimg /tmp/rbd_import_export_159798/img2 2023-09-23T20:13:31.711 INFO:tasks.workunit.client.0.smithi018.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-09-23T20:13:31.715 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd export testimg - 2023-09-23T20:13:31.966 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:13:32.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:13:31 smithi018 ceph-mon[109137]: osdmap e73: 8 total, 8 up, 8 in 2023-09-23T20:13:32.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:13:31 smithi018 ceph-mon[109137]: from='client.14838 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:bb0daeff97f020f686596fca6eba20c310e26fb7", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:13:32.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:13:31 smithi018 ceph-mon[109137]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T20:13:32.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:13:31 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:13:32.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:13:31 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:13:32.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:13:31 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:13:32.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:13:31 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:13:32.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:13:31 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:13:32.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:13:31 smithi018 ceph-mon[109137]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T20:13:32.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:13:31 smithi018 ceph-mon[109137]: pgmap v67: 73 pgs: 73 active+clean; 579 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 127 B/s wr, 0 op/s 2023-09-23T20:13:32.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:13:31 smithi018 ceph-mon[95889]: osdmap e73: 8 total, 8 up, 8 in 2023-09-23T20:13:32.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:13:31 smithi018 ceph-mon[95889]: from='client.14838 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:bb0daeff97f020f686596fca6eba20c310e26fb7", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:13:32.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:13:31 smithi018 ceph-mon[95889]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T20:13:32.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:13:31 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:13:32.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:13:31 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:13:32.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:13:31 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:13:32.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:13:31 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:13:32.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:13:31 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:13:32.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:13:31 smithi018 ceph-mon[95889]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T20:13:32.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:13:31 smithi018 ceph-mon[95889]: pgmap v67: 73 pgs: 73 active+clean; 579 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 127 B/s wr, 0 op/s 2023-09-23T20:13:32.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:13:31 smithi149 ceph-mon[100764]: osdmap e73: 8 total, 8 up, 8 in 2023-09-23T20:13:32.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:13:31 smithi149 ceph-mon[100764]: from='client.14838 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:bb0daeff97f020f686596fca6eba20c310e26fb7", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:13:32.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:13:31 smithi149 ceph-mon[100764]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T20:13:32.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:13:31 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:13:32.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:13:31 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:13:32.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:13:31 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:13:32.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:13:31 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:13:32.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:13:31 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:13:32.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:13:31 smithi149 ceph-mon[100764]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T20:13:32.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:13:31 smithi149 ceph-mon[100764]: pgmap v67: 73 pgs: 73 active+clean; 579 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 127 B/s wr, 0 op/s 2023-09-23T20:13:32.535 INFO:tasks.workunit.client.0.smithi018.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-09-23T20:13:32.539 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd rm testimg 2023-09-23T20:13:32.641 INFO:tasks.workunit.client.0.smithi018.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2023-09-23T20:13:32.644 INFO:tasks.workunit.client.0.smithi018.stderr:+ cmp /tmp/rbd_import_export_159798/img /tmp/rbd_import_export_159798/img2 2023-09-23T20:13:33.415 INFO:tasks.workunit.client.0.smithi018.stderr:+ cmp /tmp/rbd_import_export_159798/img /tmp/rbd_import_export_159798/img3 2023-09-23T20:13:33.679 INFO:teuthology.orchestra.run.smithi018.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-09-23T20:13:33.679 INFO:teuthology.orchestra.run.smithi018.stdout:alertmanager.a smithi018 *:9093,9094 running (55s) 17s ago 89s 17.2M - 0.25.0 c8568f914cd2 a755b5c2ba2f 2023-09-23T20:13:33.679 INFO:teuthology.orchestra.run.smithi018.stdout:grafana.a smithi149 *:3000 running (67s) 29s ago 66s 87.0M - 9.4.7 2c41d148cca3 6aef9de808da 2023-09-23T20:13:33.679 INFO:teuthology.orchestra.run.smithi018.stdout:mds.foo.smithi018.hduiol smithi018 running (44s) 17s ago 44s 15.0M - 17.2.6-1123-g61431f6b 0db14aa01094 7b6a15d633a4 2023-09-23T20:13:33.679 INFO:teuthology.orchestra.run.smithi018.stdout:mds.foo.smithi149.urtezf smithi149 running (42s) 29s ago 42s 15.7M - 17.2.6-1123-g61431f6b 0db14aa01094 6ee4df523f8c 2023-09-23T20:13:33.679 INFO:teuthology.orchestra.run.smithi018.stdout:mgr.x smithi149 *:8443 running (4m) 29s ago 4m 439M - 17.2.6-1123-g61431f6b 0db14aa01094 fe4e079c3943 2023-09-23T20:13:33.679 INFO:teuthology.orchestra.run.smithi018.stdout:mgr.y smithi018 *:9283 running (6m) 17s ago 6m 494M - 17.2.6-1123-g61431f6b 0db14aa01094 e62744cd423b 2023-09-23T20:13:33.679 INFO:teuthology.orchestra.run.smithi018.stdout:mon.a smithi018 running (6m) 17s ago 6m 80.5M 2048M 17.2.6-1123-g61431f6b 0db14aa01094 45b1b543db19 2023-09-23T20:13:33.679 INFO:teuthology.orchestra.run.smithi018.stdout:mon.b smithi149 running (5m) 29s ago 5m 52.0M 2048M 17.2.6-1123-g61431f6b 0db14aa01094 e9fa4ded7429 2023-09-23T20:13:33.680 INFO:teuthology.orchestra.run.smithi018.stdout:mon.c smithi018 running (5m) 17s ago 5m 54.7M 2048M 17.2.6-1123-g61431f6b 0db14aa01094 d35a46c484d5 2023-09-23T20:13:33.680 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter.a smithi018 *:9100 running (86s) 17s ago 85s 17.4M - 1.5.0 0da6a335fe13 46e682cc6912 2023-09-23T20:13:33.680 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter.b smithi149 *:9100 running (80s) 29s ago 80s 14.8M - 1.5.0 0da6a335fe13 f0943dfd81ce 2023-09-23T20:13:33.680 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0 smithi018 running (4m) 17s ago 4m 55.5M 4037M 17.2.6-1123-g61431f6b 0db14aa01094 6749d0af6e50 2023-09-23T20:13:33.680 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1 smithi018 running (4m) 17s ago 4m 59.7M 4037M 17.2.6-1123-g61431f6b 0db14aa01094 5d199baeab0e 2023-09-23T20:13:33.680 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2 smithi018 running (3m) 17s ago 3m 53.4M 4037M 17.2.6-1123-g61431f6b 0db14aa01094 2effc63f4610 2023-09-23T20:13:33.680 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3 smithi018 running (3m) 17s ago 3m 48.6M 4037M 17.2.6-1123-g61431f6b 0db14aa01094 3301c3e11bd5 2023-09-23T20:13:33.680 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4 smithi149 running (3m) 29s ago 3m 53.1M 1213M 17.2.6-1123-g61431f6b 0db14aa01094 b4b39c43de75 2023-09-23T20:13:33.680 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5 smithi149 running (2m) 29s ago 2m 49.4M 1213M 17.2.6-1123-g61431f6b 0db14aa01094 cec31ff38f4c 2023-09-23T20:13:33.680 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6 smithi149 running (2m) 29s ago 2m 48.6M 1213M 17.2.6-1123-g61431f6b 0db14aa01094 ce687413759d 2023-09-23T20:13:33.680 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7 smithi149 running (2m) 29s ago 2m 49.7M 1213M 17.2.6-1123-g61431f6b 0db14aa01094 d3145e28f8c3 2023-09-23T20:13:33.680 INFO:teuthology.orchestra.run.smithi018.stdout:prometheus.a smithi149 *:9095 running (52s) 29s ago 113s 38.6M - 2.43.0 a07b618ecd1d 4a99f58b6591 2023-09-23T20:13:33.963 INFO:tasks.workunit.client.0.smithi018.stderr:+ rm /tmp/rbd_import_export_159798/img2 /tmp/rbd_import_export_159798/img3 2023-09-23T20:13:34.044 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'echo wait for minority of mons to upgrade' 2023-09-23T20:13:34.185 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd import --new-format - testimg 2023-09-23T20:13:34.216 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:13:33 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[102880]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:13:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T20:13:34.628 INFO:tasks.workunit.client.0.smithi018.stderr: Importing image: 100% complete...done. 2023-09-23T20:13:34.628 INFO:tasks.workunit.client.0.smithi018.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T20:13:34.631 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd export testimg /tmp/rbd_import_export_159798/img2 2023-09-23T20:13:34.741 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:13:34.950 INFO:tasks.workunit.client.0.smithi018.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-09-23T20:13:34.954 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd export testimg - 2023-09-23T20:13:34.999 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:13:34 smithi018 ceph-mon[109137]: from='client.24767 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:13:34.999 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:13:34 smithi018 ceph-mon[109137]: pgmap v68: 73 pgs: 73 active+clean; 579 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 127 B/s wr, 0 op/s 2023-09-23T20:13:35.000 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:13:34 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:13:35.000 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:13:34 smithi018 ceph-mon[95889]: from='client.24767 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:13:35.000 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:13:34 smithi018 ceph-mon[95889]: pgmap v68: 73 pgs: 73 active+clean; 579 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 127 B/s wr, 0 op/s 2023-09-23T20:13:35.000 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:13:34 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:13:35.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:13:34 smithi149 ceph-mon[100764]: from='client.24767 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:13:35.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:13:34 smithi149 ceph-mon[100764]: pgmap v68: 73 pgs: 73 active+clean; 579 KiB data, 52 MiB used, 715 GiB / 715 GiB avail; 127 B/s wr, 0 op/s 2023-09-23T20:13:35.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:13:34 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:13:35.760 INFO:tasks.workunit.client.0.smithi018.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-09-23T20:13:35.764 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd rm testimg 2023-09-23T20:13:35.873 INFO:tasks.workunit.client.0.smithi018.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2023-09-23T20:13:35.876 INFO:tasks.workunit.client.0.smithi018.stderr:+ cmp /tmp/rbd_import_export_159798/img /tmp/rbd_import_export_159798/img2 2023-09-23T20:13:36.119 INFO:teuthology.orchestra.run.smithi018.stdout:wait for minority of mons to upgrade 2023-09-23T20:13:36.451 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done' 2023-09-23T20:13:36.607 INFO:tasks.workunit.client.0.smithi018.stderr:+ cmp /tmp/rbd_import_export_159798/img /tmp/rbd_import_export_159798/img3 2023-09-23T20:13:36.744 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:13:36 smithi018 ceph-mon[95889]: pgmap v69: 73 pgs: 73 active+clean; 6.0 MiB data, 54 MiB used, 715 GiB / 715 GiB avail; 278 KiB/s rd, 129 KiB/s wr, 81 op/s 2023-09-23T20:13:37.108 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:13:37.137 INFO:tasks.workunit.client.0.smithi018.stderr:+ rm /tmp/rbd_import_export_159798/img /tmp/rbd_import_export_159798/img2 /tmp/rbd_import_export_159798/img3 2023-09-23T20:13:37.140 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:13:36 smithi018 ceph-mon[109137]: pgmap v69: 73 pgs: 73 active+clean; 6.0 MiB data, 54 MiB used, 715 GiB / 715 GiB avail; 278 KiB/s rd, 129 KiB/s wr, 81 op/s 2023-09-23T20:13:37.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:13:36 smithi149 ceph-mon[100764]: pgmap v69: 73 pgs: 73 active+clean; 6.0 MiB data, 54 MiB used, 715 GiB / 715 GiB avail; 278 KiB/s rd, 129 KiB/s wr, 81 op/s 2023-09-23T20:13:37.425 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd help export 2023-09-23T20:13:37.426 INFO:tasks.workunit.client.0.smithi018.stderr:+ grep -q export-format 2023-09-23T20:13:37.447 INFO:tasks.workunit.client.0.smithi018.stderr:+ dd if=/bin/dd of=/tmp/rbd_import_export_159798/img bs=1k count=10 seek=100 2023-09-23T20:13:37.448 INFO:tasks.workunit.client.0.smithi018.stderr:10+0 records in 2023-09-23T20:13:37.448 INFO:tasks.workunit.client.0.smithi018.stderr:10+0 records out 2023-09-23T20:13:37.448 INFO:tasks.workunit.client.0.smithi018.stderr:10240 bytes (10 kB, 10 KiB) copied, 6.6267e-05 s, 155 MB/s 2023-09-23T20:13:37.449 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd import --new-format /tmp/rbd_import_export_159798/img testimg 2023-09-23T20:13:37.511 INFO:tasks.workunit.client.0.smithi018.stderr: Importing image: 100% complete...done. 2023-09-23T20:13:37.513 INFO:tasks.workunit.client.0.smithi018.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T20:13:37.515 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd snap create testimg@snap 2023-09-23T20:13:37.761 INFO:tasks.workunit.client.0.smithi018.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2023-09-23T20:13:37.769 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd image-meta set testimg key1 value1 2023-09-23T20:13:37.822 INFO:tasks.workunit.client.0.smithi018.stderr:++ rbd image-meta list testimg 2023-09-23T20:13:37.863 INFO:tasks.workunit.client.0.smithi018.stderr:+ IMAGEMETA_BEFORE='There is 1 metadatum on this image: 2023-09-23T20:13:37.863 INFO:tasks.workunit.client.0.smithi018.stderr: 2023-09-23T20:13:37.863 INFO:tasks.workunit.client.0.smithi018.stderr:Key Value 2023-09-23T20:13:37.863 INFO:tasks.workunit.client.0.smithi018.stderr:key1 value1' 2023-09-23T20:13:37.863 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_159798/img_v2 2023-09-23T20:13:37.904 INFO:tasks.workunit.client.0.smithi018.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-09-23T20:13:37.909 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_159798/img_v2 testimg_import 2023-09-23T20:13:38.775 INFO:tasks.workunit.client.0.smithi018.stderr: Importing image: 50% complete... Importing image: 100% complete... Importing image: 100% complete...done. 2023-09-23T20:13:38.778 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd info testimg_import 2023-09-23T20:13:38.815 INFO:tasks.workunit.client.0.smithi018.stdout:rbd image 'testimg_import': 2023-09-23T20:13:38.815 INFO:tasks.workunit.client.0.smithi018.stdout: size 110 KiB in 1 objects 2023-09-23T20:13:38.815 INFO:tasks.workunit.client.0.smithi018.stdout: order 22 (4 MiB objects) 2023-09-23T20:13:38.815 INFO:tasks.workunit.client.0.smithi018.stdout: snapshot_count: 1 2023-09-23T20:13:38.815 INFO:tasks.workunit.client.0.smithi018.stdout: id: 60f83db31002 2023-09-23T20:13:38.815 INFO:tasks.workunit.client.0.smithi018.stdout: block_name_prefix: rbd_data.60f83db31002 2023-09-23T20:13:38.815 INFO:tasks.workunit.client.0.smithi018.stdout: format: 2 2023-09-23T20:13:38.815 INFO:tasks.workunit.client.0.smithi018.stdout: features: layering, exclusive-lock, object-map, fast-diff, deep-flatten 2023-09-23T20:13:38.816 INFO:tasks.workunit.client.0.smithi018.stdout: op_features: 2023-09-23T20:13:38.816 INFO:tasks.workunit.client.0.smithi018.stdout: flags: 2023-09-23T20:13:38.816 INFO:tasks.workunit.client.0.smithi018.stdout: create_timestamp: Sat Sep 23 20:13:37 2023 2023-09-23T20:13:38.816 INFO:tasks.workunit.client.0.smithi018.stdout: access_timestamp: Sat Sep 23 20:13:37 2023 2023-09-23T20:13:38.816 INFO:tasks.workunit.client.0.smithi018.stdout: modify_timestamp: Sat Sep 23 20:13:37 2023 2023-09-23T20:13:38.818 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd info testimg_import@snap 2023-09-23T20:13:38.854 INFO:tasks.workunit.client.0.smithi018.stdout:rbd image 'testimg_import': 2023-09-23T20:13:38.854 INFO:tasks.workunit.client.0.smithi018.stdout: size 110 KiB in 1 objects 2023-09-23T20:13:38.854 INFO:tasks.workunit.client.0.smithi018.stdout: order 22 (4 MiB objects) 2023-09-23T20:13:38.854 INFO:tasks.workunit.client.0.smithi018.stdout: snapshot_count: 1 2023-09-23T20:13:38.854 INFO:tasks.workunit.client.0.smithi018.stdout: id: 60f83db31002 2023-09-23T20:13:38.855 INFO:tasks.workunit.client.0.smithi018.stdout: block_name_prefix: rbd_data.60f83db31002 2023-09-23T20:13:38.855 INFO:tasks.workunit.client.0.smithi018.stdout: format: 2 2023-09-23T20:13:38.855 INFO:tasks.workunit.client.0.smithi018.stdout: features: layering, exclusive-lock, object-map, fast-diff, deep-flatten 2023-09-23T20:13:38.855 INFO:tasks.workunit.client.0.smithi018.stdout: op_features: 2023-09-23T20:13:38.855 INFO:tasks.workunit.client.0.smithi018.stdout: flags: 2023-09-23T20:13:38.856 INFO:tasks.workunit.client.0.smithi018.stdout: create_timestamp: Sat Sep 23 20:13:37 2023 2023-09-23T20:13:38.856 INFO:tasks.workunit.client.0.smithi018.stdout: access_timestamp: Sat Sep 23 20:13:37 2023 2023-09-23T20:13:38.856 INFO:tasks.workunit.client.0.smithi018.stdout: modify_timestamp: Sat Sep 23 20:13:37 2023 2023-09-23T20:13:38.856 INFO:tasks.workunit.client.0.smithi018.stdout: protected: False 2023-09-23T20:13:38.857 INFO:tasks.workunit.client.0.smithi018.stderr:++ rbd image-meta list testimg_import 2023-09-23T20:13:38.903 INFO:tasks.workunit.client.0.smithi018.stderr:+ IMAGEMETA_AFTER='There is 1 metadatum on this image: 2023-09-23T20:13:38.903 INFO:tasks.workunit.client.0.smithi018.stderr: 2023-09-23T20:13:38.903 INFO:tasks.workunit.client.0.smithi018.stderr:Key Value 2023-09-23T20:13:38.903 INFO:tasks.workunit.client.0.smithi018.stderr:key1 value1' 2023-09-23T20:13:38.903 INFO:tasks.workunit.client.0.smithi018.stderr:+ '[' 'There is 1 metadatum on this image: 2023-09-23T20:13:38.903 INFO:tasks.workunit.client.0.smithi018.stderr: 2023-09-23T20:13:38.903 INFO:tasks.workunit.client.0.smithi018.stderr:Key Value 2023-09-23T20:13:38.903 INFO:tasks.workunit.client.0.smithi018.stderr:key1 value1' = 'There is 1 metadatum on this image: 2023-09-23T20:13:38.903 INFO:tasks.workunit.client.0.smithi018.stderr: 2023-09-23T20:13:38.904 INFO:tasks.workunit.client.0.smithi018.stderr:Key Value 2023-09-23T20:13:38.904 INFO:tasks.workunit.client.0.smithi018.stderr:key1 value1' ']' 2023-09-23T20:13:38.904 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd export testimg_import /tmp/rbd_import_export_159798/img_import 2023-09-23T20:13:38.941 INFO:tasks.workunit.client.0.smithi018.stderr: Exporting image: 100% complete...done. 2023-09-23T20:13:38.943 INFO:tasks.workunit.client.0.smithi018.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_159798/img /tmp/rbd_import_export_159798/img_import 2023-09-23T20:13:38.943 INFO:tasks.workunit.client.0.smithi018.stderr:+ cmp -l /tmp/rbd_import_export_159798/img /tmp/rbd_import_export_159798/img_import 2023-09-23T20:13:38.945 INFO:tasks.workunit.client.0.smithi018.stderr:++ stat /tmp/rbd_import_export_159798/img --format %b 2023-09-23T20:13:38.946 INFO:tasks.workunit.client.0.smithi018.stderr:+ origsize=24 2023-09-23T20:13:38.946 INFO:tasks.workunit.client.0.smithi018.stderr:++ stat /tmp/rbd_import_export_159798/img_import --format %b 2023-09-23T20:13:38.947 INFO:tasks.workunit.client.0.smithi018.stderr:+ exportsize=224 2023-09-23T20:13:38.948 INFO:tasks.workunit.client.0.smithi018.stderr:+ difference=200 2023-09-23T20:13:38.948 INFO:tasks.workunit.client.0.smithi018.stderr:+ difference=200 2023-09-23T20:13:38.948 INFO:tasks.workunit.client.0.smithi018.stderr:+ test 200 -ge 0 -a 200 -lt 4096 2023-09-23T20:13:38.948 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd export testimg@snap /tmp/rbd_import_export_159798/img_snap 2023-09-23T20:13:38.985 INFO:tasks.workunit.client.0.smithi018.stderr: Exporting image: 100% complete...done. 2023-09-23T20:13:38.989 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd export testimg_import@snap /tmp/rbd_import_export_159798/img_snap_import 2023-09-23T20:13:39.028 INFO:tasks.workunit.client.0.smithi018.stderr: Exporting image: 100% complete...done. 2023-09-23T20:13:39.032 INFO:tasks.workunit.client.0.smithi018.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_159798/img_snap /tmp/rbd_import_export_159798/img_snap_import 2023-09-23T20:13:39.032 INFO:tasks.workunit.client.0.smithi018.stderr:+ cmp -l /tmp/rbd_import_export_159798/img_snap /tmp/rbd_import_export_159798/img_snap_import 2023-09-23T20:13:39.033 INFO:tasks.workunit.client.0.smithi018.stderr:++ stat /tmp/rbd_import_export_159798/img_snap --format %b 2023-09-23T20:13:39.034 INFO:tasks.workunit.client.0.smithi018.stderr:+ origsize=224 2023-09-23T20:13:39.034 INFO:tasks.workunit.client.0.smithi018.stderr:++ stat /tmp/rbd_import_export_159798/img_snap_import --format %b 2023-09-23T20:13:39.035 INFO:tasks.workunit.client.0.smithi018.stderr:+ exportsize=224 2023-09-23T20:13:39.035 INFO:tasks.workunit.client.0.smithi018.stderr:+ difference=0 2023-09-23T20:13:39.035 INFO:tasks.workunit.client.0.smithi018.stderr:+ difference=0 2023-09-23T20:13:39.035 INFO:tasks.workunit.client.0.smithi018.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-09-23T20:13:39.035 INFO:tasks.workunit.client.0.smithi018.stderr:+ rm /tmp/rbd_import_export_159798/img_v2 2023-09-23T20:13:39.036 INFO:tasks.workunit.client.0.smithi018.stderr:+ rm /tmp/rbd_import_export_159798/img_import 2023-09-23T20:13:39.037 INFO:tasks.workunit.client.0.smithi018.stderr:+ rm /tmp/rbd_import_export_159798/img_snap 2023-09-23T20:13:39.038 INFO:tasks.workunit.client.0.smithi018.stderr:+ rm /tmp/rbd_import_export_159798/img_snap_import 2023-09-23T20:13:39.038 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd snap rm testimg_import@snap 2023-09-23T20:13:39.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:13:38 smithi018 ceph-mon[95889]: pgmap v70: 73 pgs: 73 active+clean; 6.0 MiB data, 54 MiB used, 715 GiB / 715 GiB avail; 278 KiB/s rd, 129 KiB/s wr, 81 op/s 2023-09-23T20:13:39.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:13:38 smithi018 ceph-mon[95889]: osdmap e74: 8 total, 8 up, 8 in 2023-09-23T20:13:39.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:13:38 smithi018 ceph-mon[95889]: from='client.? 172.21.15.18:0/1542808989' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:13:39.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:13:38 smithi018 ceph-mon[109137]: pgmap v70: 73 pgs: 73 active+clean; 6.0 MiB data, 54 MiB used, 715 GiB / 715 GiB avail; 278 KiB/s rd, 129 KiB/s wr, 81 op/s 2023-09-23T20:13:39.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:13:38 smithi018 ceph-mon[109137]: osdmap e74: 8 total, 8 up, 8 in 2023-09-23T20:13:39.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:13:38 smithi018 ceph-mon[109137]: from='client.? 172.21.15.18:0/1542808989' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:13:39.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:13:38 smithi149 ceph-mon[100764]: pgmap v70: 73 pgs: 73 active+clean; 6.0 MiB data, 54 MiB used, 715 GiB / 715 GiB avail; 278 KiB/s rd, 129 KiB/s wr, 81 op/s 2023-09-23T20:13:39.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:13:38 smithi149 ceph-mon[100764]: osdmap e74: 8 total, 8 up, 8 in 2023-09-23T20:13:39.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:13:38 smithi149 ceph-mon[100764]: from='client.? 172.21.15.18:0/1542808989' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:13:39.640 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:13:39 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:13:39] "GET /metrics HTTP/1.1" 200 238806 "" "Prometheus/2.43.0" 2023-09-23T20:13:39.877 INFO:tasks.workunit.client.0.smithi018.stderr: Removing snap: 100% complete...done. 2023-09-23T20:13:39.885 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd remove testimg_import 2023-09-23T20:13:39.974 INFO:tasks.workunit.client.0.smithi018.stderr: Removing image: 100% complete...done. 2023-09-23T20:13:39.977 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd snap rm testimg@snap 2023-09-23T20:13:40.140 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:13:39 smithi018 ceph-mon[95889]: osdmap e75: 8 total, 8 up, 8 in 2023-09-23T20:13:40.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:13:39 smithi018 ceph-mon[95889]: osdmap e76: 8 total, 8 up, 8 in 2023-09-23T20:13:40.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:13:39 smithi018 ceph-mon[109137]: osdmap e75: 8 total, 8 up, 8 in 2023-09-23T20:13:40.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:13:39 smithi018 ceph-mon[109137]: osdmap e76: 8 total, 8 up, 8 in 2023-09-23T20:13:40.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:13:39 smithi149 ceph-mon[100764]: osdmap e75: 8 total, 8 up, 8 in 2023-09-23T20:13:40.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:13:39 smithi149 ceph-mon[100764]: osdmap e76: 8 total, 8 up, 8 in 2023-09-23T20:13:40.876 INFO:tasks.workunit.client.0.smithi018.stderr: Removing snap: 100% complete...done. 2023-09-23T20:13:40.888 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd rm testimg 2023-09-23T20:13:40.983 INFO:tasks.workunit.client.0.smithi018.stderr: Removing image: 100% complete...done. 2023-09-23T20:13:40.986 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd import --order 20 /tmp/rbd_import_export_159798/img testimg 2023-09-23T20:13:41.046 INFO:tasks.workunit.client.0.smithi018.stderr: Importing image: 100% complete...done. 2023-09-23T20:13:41.047 INFO:tasks.workunit.client.0.smithi018.stderr:rbd: --order is deprecated, use --object-size 2023-09-23T20:13:41.050 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_159798/img_v2 2023-09-23T20:13:41.093 INFO:tasks.workunit.client.0.smithi018.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-09-23T20:13:41.096 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_159798/img_v2 testimg_import 2023-09-23T20:13:41.140 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:13:40 smithi018 ceph-mon[109137]: pgmap v74: 73 pgs: 73 active+clean; 6.0 MiB data, 54 MiB used, 715 GiB / 715 GiB avail; 371 KiB/s rd, 172 KiB/s wr, 108 op/s 2023-09-23T20:13:41.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:13:40 smithi018 ceph-mon[109137]: osdmap e77: 8 total, 8 up, 8 in 2023-09-23T20:13:41.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:13:40 smithi018 ceph-mon[95889]: pgmap v74: 73 pgs: 73 active+clean; 6.0 MiB data, 54 MiB used, 715 GiB / 715 GiB avail; 371 KiB/s rd, 172 KiB/s wr, 108 op/s 2023-09-23T20:13:41.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:13:40 smithi018 ceph-mon[95889]: osdmap e77: 8 total, 8 up, 8 in 2023-09-23T20:13:41.159 INFO:tasks.workunit.client.0.smithi018.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2023-09-23T20:13:41.164 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd info testimg_import 2023-09-23T20:13:41.164 INFO:tasks.workunit.client.0.smithi018.stderr:+ grep order 2023-09-23T20:13:41.164 INFO:tasks.workunit.client.0.smithi018.stderr:+ awk '{print $2}' 2023-09-23T20:13:41.164 INFO:tasks.workunit.client.0.smithi018.stderr:+ grep 20 2023-09-23T20:13:41.203 INFO:tasks.workunit.client.0.smithi018.stdout:20 2023-09-23T20:13:41.203 INFO:tasks.workunit.client.0.smithi018.stderr:+ rm /tmp/rbd_import_export_159798/img_v2 2023-09-23T20:13:41.204 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd remove testimg_import 2023-09-23T20:13:41.215 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:13:40 smithi149 ceph-mon[100764]: pgmap v74: 73 pgs: 73 active+clean; 6.0 MiB data, 54 MiB used, 715 GiB / 715 GiB avail; 371 KiB/s rd, 172 KiB/s wr, 108 op/s 2023-09-23T20:13:41.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:13:40 smithi149 ceph-mon[100764]: osdmap e77: 8 total, 8 up, 8 in 2023-09-23T20:13:41.295 INFO:tasks.workunit.client.0.smithi018.stderr: Removing image: 100% complete...done. 2023-09-23T20:13:41.298 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd remove testimg 2023-09-23T20:13:41.388 INFO:tasks.workunit.client.0.smithi018.stderr: Removing image: 100% complete...done. 2023-09-23T20:13:41.391 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd import --image-feature layering /tmp/rbd_import_export_159798/img testimg 2023-09-23T20:13:41.447 INFO:tasks.workunit.client.0.smithi018.stderr: Importing image: 100% complete...done. 2023-09-23T20:13:41.451 INFO:tasks.workunit.client.0.smithi018.stderr:++ rbd info testimg 2023-09-23T20:13:41.452 INFO:tasks.workunit.client.0.smithi018.stderr:++ grep features 2023-09-23T20:13:41.490 INFO:tasks.workunit.client.0.smithi018.stderr:+ FEATURES_BEFORE=' features: layering 2023-09-23T20:13:41.490 INFO:tasks.workunit.client.0.smithi018.stderr: op_features: ' 2023-09-23T20:13:41.490 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_159798/img_v2 2023-09-23T20:13:41.528 INFO:tasks.workunit.client.0.smithi018.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-09-23T20:13:41.531 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_159798/img_v2 testimg_import 2023-09-23T20:13:41.585 INFO:tasks.workunit.client.0.smithi018.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2023-09-23T20:13:41.589 INFO:tasks.workunit.client.0.smithi018.stderr:++ rbd info testimg_import 2023-09-23T20:13:41.589 INFO:tasks.workunit.client.0.smithi018.stderr:++ grep features 2023-09-23T20:13:41.628 INFO:tasks.workunit.client.0.smithi018.stderr:+ FEATURES_AFTER=' features: layering 2023-09-23T20:13:41.628 INFO:tasks.workunit.client.0.smithi018.stderr: op_features: ' 2023-09-23T20:13:41.628 INFO:tasks.workunit.client.0.smithi018.stderr:+ '[' ' features: layering 2023-09-23T20:13:41.628 INFO:tasks.workunit.client.0.smithi018.stderr: op_features: ' '!=' ' features: layering 2023-09-23T20:13:41.628 INFO:tasks.workunit.client.0.smithi018.stderr: op_features: ' ']' 2023-09-23T20:13:41.628 INFO:tasks.workunit.client.0.smithi018.stderr:+ rm /tmp/rbd_import_export_159798/img_v2 2023-09-23T20:13:41.629 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd remove testimg_import 2023-09-23T20:13:41.699 INFO:tasks.workunit.client.0.smithi018.stderr: Removing image: 100% complete...done. 2023-09-23T20:13:41.702 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd remove testimg 2023-09-23T20:13:41.772 INFO:tasks.workunit.client.0.smithi018.stderr: Removing image: 100% complete...done. 2023-09-23T20:13:41.775 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd import --stripe-count 1000 --stripe-unit 4096 /tmp/rbd_import_export_159798/img testimg 2023-09-23T20:13:41.839 INFO:tasks.workunit.client.0.smithi018.stderr: Importing image: 100% complete...done. 2023-09-23T20:13:41.843 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_159798/img_v2 2023-09-23T20:13:41.886 INFO:tasks.workunit.client.0.smithi018.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-09-23T20:13:41.889 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_159798/img_v2 testimg_import 2023-09-23T20:13:41.952 INFO:tasks.workunit.client.0.smithi018.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2023-09-23T20:13:41.955 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd info testimg_import 2023-09-23T20:13:41.955 INFO:tasks.workunit.client.0.smithi018.stderr:+ grep 'stripe unit' 2023-09-23T20:13:41.955 INFO:tasks.workunit.client.0.smithi018.stderr:+ grep -Ei '(4 KiB|4096)' 2023-09-23T20:13:41.993 INFO:tasks.workunit.client.0.smithi018.stdout: stripe unit: 4 KiB 2023-09-23T20:13:41.993 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd info testimg_import 2023-09-23T20:13:41.994 INFO:tasks.workunit.client.0.smithi018.stderr:+ grep 'stripe count' 2023-09-23T20:13:41.994 INFO:tasks.workunit.client.0.smithi018.stderr:+ awk '{print $3}' 2023-09-23T20:13:41.994 INFO:tasks.workunit.client.0.smithi018.stderr:+ grep 1000 2023-09-23T20:13:42.032 INFO:tasks.workunit.client.0.smithi018.stdout:1000 2023-09-23T20:13:42.033 INFO:tasks.workunit.client.0.smithi018.stderr:+ rm /tmp/rbd_import_export_159798/img_v2 2023-09-23T20:13:42.033 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd remove testimg_import 2023-09-23T20:13:42.122 INFO:tasks.workunit.client.0.smithi018.stderr: Removing image: 3% complete... Removing image: 7% complete... Removing image: 10% complete... Removing image: 14% complete... Removing image: 17% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 32% complete... Removing image: 35% complete... Removing image: 39% complete... Removing image: 42% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 57% complete... Removing image: 60% complete... Removing image: 64% complete... Removing image: 67% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 82% complete... Removing image: 85% complete... Removing image: 89% complete... Removing image: 92% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2023-09-23T20:13:42.125 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd remove testimg 2023-09-23T20:13:42.140 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:13:41 smithi018 ceph-mon[109137]: osdmap e78: 8 total, 8 up, 8 in 2023-09-23T20:13:42.140 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:13:41 smithi018 ceph-mon[109137]: from='client.? 172.21.15.18:0/3900110229' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:13:42.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:13:41 smithi018 ceph-mon[109137]: pgmap v77: 73 pgs: 73 active+clean; 689 KiB data, 54 MiB used, 715 GiB / 715 GiB avail; 251 KiB/s rd, 17 KiB/s wr, 260 op/s 2023-09-23T20:13:42.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:13:41 smithi018 ceph-mon[95889]: osdmap e78: 8 total, 8 up, 8 in 2023-09-23T20:13:42.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:13:41 smithi018 ceph-mon[95889]: from='client.? 172.21.15.18:0/3900110229' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:13:42.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:13:41 smithi018 ceph-mon[95889]: pgmap v77: 73 pgs: 73 active+clean; 689 KiB data, 54 MiB used, 715 GiB / 715 GiB avail; 251 KiB/s rd, 17 KiB/s wr, 260 op/s 2023-09-23T20:13:42.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:13:41 smithi149 ceph-mon[100764]: osdmap e78: 8 total, 8 up, 8 in 2023-09-23T20:13:42.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:13:41 smithi149 ceph-mon[100764]: from='client.? 172.21.15.18:0/3900110229' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:13:42.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:13:41 smithi149 ceph-mon[100764]: pgmap v77: 73 pgs: 73 active+clean; 689 KiB data, 54 MiB used, 715 GiB / 715 GiB avail; 251 KiB/s rd, 17 KiB/s wr, 260 op/s 2023-09-23T20:13:42.222 INFO:tasks.workunit.client.0.smithi018.stderr: Removing image: 3% complete... Removing image: 7% complete... Removing image: 10% complete... Removing image: 14% complete... Removing image: 17% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 32% complete... Removing image: 35% complete... Removing image: 39% complete... Removing image: 42% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 57% complete... Removing image: 60% complete... Removing image: 64% complete... Removing image: 67% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 82% complete... Removing image: 85% complete... Removing image: 89% complete... Removing image: 92% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2023-09-23T20:13:42.225 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd import --image-format=2 /tmp/rbd_import_export_159798/img testimg 2023-09-23T20:13:42.286 INFO:tasks.workunit.client.0.smithi018.stderr: Importing image: 100% complete...done. 2023-09-23T20:13:42.289 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd snap create testimg@snap1 2023-09-23T20:13:42.888 INFO:tasks.workunit.client.0.smithi018.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2023-09-23T20:13:42.894 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd snap create testimg@snap2 2023-09-23T20:13:43.870 INFO:tasks.workunit.client.0.smithi018.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2023-09-23T20:13:43.878 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd snap protect testimg@snap2 2023-09-23T20:13:43.924 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_159798/snap_protect 2023-09-23T20:13:43.965 INFO:tasks.workunit.client.0.smithi018.stderr: Exporting image: 33% complete... Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-09-23T20:13:43.968 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_159798/snap_protect testimg_import 2023-09-23T20:13:44.140 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:13:43 smithi018 ceph-mon[109137]: osdmap e79: 8 total, 8 up, 8 in 2023-09-23T20:13:44.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:13:43 smithi018 ceph-mon[109137]: pgmap v79: 73 pgs: 73 active+clean; 689 KiB data, 54 MiB used, 715 GiB / 715 GiB avail; 205 KiB/s rd, 14 KiB/s wr, 212 op/s 2023-09-23T20:13:44.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:13:43 smithi018 ceph-mon[109137]: from='client.? 172.21.15.18:0/2694122231' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:13:44.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:13:43 smithi018 ceph-mon[109137]: osdmap e80: 8 total, 8 up, 8 in 2023-09-23T20:13:44.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:13:43 smithi018 ceph-mon[95889]: osdmap e79: 8 total, 8 up, 8 in 2023-09-23T20:13:44.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:13:43 smithi018 ceph-mon[95889]: pgmap v79: 73 pgs: 73 active+clean; 689 KiB data, 54 MiB used, 715 GiB / 715 GiB avail; 205 KiB/s rd, 14 KiB/s wr, 212 op/s 2023-09-23T20:13:44.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:13:43 smithi018 ceph-mon[95889]: from='client.? 172.21.15.18:0/2694122231' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:13:44.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:13:43 smithi018 ceph-mon[95889]: osdmap e80: 8 total, 8 up, 8 in 2023-09-23T20:13:44.216 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:13:43 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[102880]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:13:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T20:13:44.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:13:43 smithi149 ceph-mon[100764]: osdmap e79: 8 total, 8 up, 8 in 2023-09-23T20:13:44.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:13:43 smithi149 ceph-mon[100764]: pgmap v79: 73 pgs: 73 active+clean; 689 KiB data, 54 MiB used, 715 GiB / 715 GiB avail; 205 KiB/s rd, 14 KiB/s wr, 212 op/s 2023-09-23T20:13:44.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:13:43 smithi149 ceph-mon[100764]: from='client.? 172.21.15.18:0/2694122231' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:13:44.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:13:43 smithi149 ceph-mon[100764]: osdmap e80: 8 total, 8 up, 8 in 2023-09-23T20:13:45.911 INFO:tasks.workunit.client.0.smithi018.stderr: Importing image: 33% complete... Importing image: 66% complete... Importing image: 100% complete... Importing image: 100% complete...done. 2023-09-23T20:13:45.915 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd info testimg_import@snap1 2023-09-23T20:13:45.915 INFO:tasks.workunit.client.0.smithi018.stderr:+ grep 'protected: False' 2023-09-23T20:13:45.954 INFO:tasks.workunit.client.0.smithi018.stdout: protected: False 2023-09-23T20:13:45.954 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd info testimg_import@snap2 2023-09-23T20:13:45.955 INFO:tasks.workunit.client.0.smithi018.stderr:+ grep 'protected: True' 2023-09-23T20:13:45.996 INFO:tasks.workunit.client.0.smithi018.stdout: protected: True 2023-09-23T20:13:45.997 INFO:tasks.workunit.client.0.smithi018.stderr:+ rm /tmp/rbd_import_export_159798/snap_protect 2023-09-23T20:13:45.998 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd snap unprotect testimg@snap2 2023-09-23T20:13:46.044 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd snap unprotect testimg_import@snap2 2023-09-23T20:13:46.089 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd snap purge testimg 2023-09-23T20:13:46.140 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:13:45 smithi018 ceph-mon[109137]: osdmap e81: 8 total, 8 up, 8 in 2023-09-23T20:13:46.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:13:45 smithi018 ceph-mon[109137]: pgmap v82: 73 pgs: 73 active+clean; 799 KiB data, 56 MiB used, 715 GiB / 715 GiB avail; 320 KiB/s rd, 40 KiB/s wr, 429 op/s 2023-09-23T20:13:46.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:13:45 smithi018 ceph-mon[95889]: osdmap e81: 8 total, 8 up, 8 in 2023-09-23T20:13:46.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:13:45 smithi018 ceph-mon[95889]: pgmap v82: 73 pgs: 73 active+clean; 799 KiB data, 56 MiB used, 715 GiB / 715 GiB avail; 320 KiB/s rd, 40 KiB/s wr, 429 op/s 2023-09-23T20:13:46.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:13:45 smithi149 ceph-mon[100764]: osdmap e81: 8 total, 8 up, 8 in 2023-09-23T20:13:46.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:13:45 smithi149 ceph-mon[100764]: pgmap v82: 73 pgs: 73 active+clean; 799 KiB data, 56 MiB used, 715 GiB / 715 GiB avail; 320 KiB/s rd, 40 KiB/s wr, 429 op/s 2023-09-23T20:13:47.140 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:13:46 smithi018 ceph-mon[95889]: osdmap e82: 8 total, 8 up, 8 in 2023-09-23T20:13:47.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:13:46 smithi018 ceph-mon[95889]: from='client.? 172.21.15.18:0/1770310662' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:13:47.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:13:46 smithi018 ceph-mon[109137]: osdmap e82: 8 total, 8 up, 8 in 2023-09-23T20:13:47.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:13:46 smithi018 ceph-mon[109137]: from='client.? 172.21.15.18:0/1770310662' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:13:47.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:13:46 smithi149 ceph-mon[100764]: osdmap e82: 8 total, 8 up, 8 in 2023-09-23T20:13:47.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:13:46 smithi149 ceph-mon[100764]: from='client.? 172.21.15.18:0/1770310662' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:13:47.900 INFO:tasks.workunit.client.0.smithi018.stderr: Removing all snapshots: 50% complete... Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2023-09-23T20:13:47.909 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd snap purge testimg_import 2023-09-23T20:13:48.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:13:47 smithi149 ceph-mon[100764]: osdmap e83: 8 total, 8 up, 8 in 2023-09-23T20:13:48.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:13:47 smithi149 ceph-mon[100764]: pgmap v85: 73 pgs: 73 active+clean; 799 KiB data, 56 MiB used, 715 GiB / 715 GiB avail; 386 KiB/s rd, 49 KiB/s wr, 517 op/s 2023-09-23T20:13:48.390 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:13:47 smithi018 ceph-mon[109137]: osdmap e83: 8 total, 8 up, 8 in 2023-09-23T20:13:48.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:13:47 smithi018 ceph-mon[109137]: pgmap v85: 73 pgs: 73 active+clean; 799 KiB data, 56 MiB used, 715 GiB / 715 GiB avail; 386 KiB/s rd, 49 KiB/s wr, 517 op/s 2023-09-23T20:13:48.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:13:47 smithi018 ceph-mon[95889]: osdmap e83: 8 total, 8 up, 8 in 2023-09-23T20:13:48.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:13:47 smithi018 ceph-mon[95889]: pgmap v85: 73 pgs: 73 active+clean; 799 KiB data, 56 MiB used, 715 GiB / 715 GiB avail; 386 KiB/s rd, 49 KiB/s wr, 517 op/s 2023-09-23T20:13:49.188 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:13:48 smithi018 ceph-mon[109137]: osdmap e84: 8 total, 8 up, 8 in 2023-09-23T20:13:49.189 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:13:48 smithi018 ceph-mon[109137]: from='client.? 172.21.15.18:0/1617397576' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:13:49.189 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:13:48 smithi018 ceph-mon[109137]: osdmap e85: 8 total, 8 up, 8 in 2023-09-23T20:13:49.189 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:13:48 smithi018 ceph-mon[95889]: osdmap e84: 8 total, 8 up, 8 in 2023-09-23T20:13:49.189 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:13:48 smithi018 ceph-mon[95889]: from='client.? 172.21.15.18:0/1617397576' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:13:49.189 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:13:48 smithi018 ceph-mon[95889]: osdmap e85: 8 total, 8 up, 8 in 2023-09-23T20:13:49.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:13:48 smithi149 ceph-mon[100764]: osdmap e84: 8 total, 8 up, 8 in 2023-09-23T20:13:49.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:13:48 smithi149 ceph-mon[100764]: from='client.? 172.21.15.18:0/1617397576' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:13:49.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:13:48 smithi149 ceph-mon[100764]: osdmap e85: 8 total, 8 up, 8 in 2023-09-23T20:13:49.640 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:13:49 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:13:49] "GET /metrics HTTP/1.1" 200 238806 "" "Prometheus/2.43.0" 2023-09-23T20:13:49.876 INFO:tasks.workunit.client.0.smithi018.stderr: Removing all snapshots: 50% complete... Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2023-09-23T20:13:49.886 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd remove testimg 2023-09-23T20:13:49.989 INFO:tasks.workunit.client.0.smithi018.stderr: Removing image: 100% complete...done. 2023-09-23T20:13:49.993 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd remove testimg_import 2023-09-23T20:13:50.104 INFO:tasks.workunit.client.0.smithi018.stderr: Removing image: 100% complete...done. 2023-09-23T20:13:50.107 INFO:tasks.workunit.client.0.smithi018.stderr:+ tiered=0 2023-09-23T20:13:50.107 INFO:tasks.workunit.client.0.smithi018.stderr:+ ceph osd dump 2023-09-23T20:13:50.108 INFO:tasks.workunit.client.0.smithi018.stderr:+ grep '^pool' 2023-09-23T20:13:50.108 INFO:tasks.workunit.client.0.smithi018.stderr:+ grep ''\''rbd'\''' 2023-09-23T20:13:50.108 INFO:tasks.workunit.client.0.smithi018.stderr:+ grep tier 2023-09-23T20:13:50.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:13:49 smithi149 ceph-mon[100764]: pgmap v88: 73 pgs: 73 active+clean; 799 KiB data, 56 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:13:50.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:13:49 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:13:50.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:13:49 smithi149 ceph-mon[100764]: osdmap e86: 8 total, 8 up, 8 in 2023-09-23T20:13:50.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:13:49 smithi018 ceph-mon[109137]: pgmap v88: 73 pgs: 73 active+clean; 799 KiB data, 56 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:13:50.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:13:49 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:13:50.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:13:49 smithi018 ceph-mon[109137]: osdmap e86: 8 total, 8 up, 8 in 2023-09-23T20:13:50.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:13:49 smithi018 ceph-mon[95889]: pgmap v88: 73 pgs: 73 active+clean; 799 KiB data, 56 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:13:50.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:13:49 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:13:50.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:13:49 smithi018 ceph-mon[95889]: osdmap e86: 8 total, 8 up, 8 in 2023-09-23T20:13:50.685 INFO:tasks.workunit.client.0.smithi018.stderr:+ dd if=/dev/urandom bs=1M seek=1 count=1 of=/tmp/rbd_import_export_159798/sparse1 2023-09-23T20:13:50.692 INFO:tasks.workunit.client.0.smithi018.stderr:1+0 records in 2023-09-23T20:13:50.692 INFO:tasks.workunit.client.0.smithi018.stderr:1+0 records out 2023-09-23T20:13:50.692 INFO:tasks.workunit.client.0.smithi018.stderr:1048576 bytes (1.0 MB, 1.0 MiB) copied, 0.00610109 s, 172 MB/s 2023-09-23T20:13:50.693 INFO:tasks.workunit.client.0.smithi018.stderr:+ dd if=/dev/urandom bs=1M count=1 of=/tmp/rbd_import_export_159798/sparse2 2023-09-23T20:13:50.699 INFO:tasks.workunit.client.0.smithi018.stderr:1+0 records in 2023-09-23T20:13:50.699 INFO:tasks.workunit.client.0.smithi018.stderr:1+0 records out 2023-09-23T20:13:50.699 INFO:tasks.workunit.client.0.smithi018.stderr:1048576 bytes (1.0 MB, 1.0 MiB) copied, 0.00558395 s, 188 MB/s 2023-09-23T20:13:50.699 INFO:tasks.workunit.client.0.smithi018.stderr:+ truncate /tmp/rbd_import_export_159798/sparse2 -s 2M 2023-09-23T20:13:50.715 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd rm sparse1 2023-09-23T20:13:50.758 INFO:tasks.workunit.client.0.smithi018.stderr: Removing image: 0% complete...failed. 2023-09-23T20:13:50.758 INFO:tasks.workunit.client.0.smithi018.stderr:rbd: delete error: (2) No such file or directory 2023-09-23T20:13:50.762 INFO:tasks.workunit.client.0.smithi018.stderr:+ true 2023-09-23T20:13:50.762 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_159798/sparse1 2023-09-23T20:13:50.832 INFO:tasks.workunit.client.0.smithi018.stderr: Importing image: 50% complete... Importing image: 100% complete...done. 2023-09-23T20:13:50.832 INFO:tasks.workunit.client.0.smithi018.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T20:13:50.832 INFO:tasks.workunit.client.0.smithi018.stderr:rbd: --order is deprecated, use --object-size 2023-09-23T20:13:50.835 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd ls -l 2023-09-23T20:13:50.835 INFO:tasks.workunit.client.0.smithi018.stderr:+ grep sparse1 2023-09-23T20:13:50.835 INFO:tasks.workunit.client.0.smithi018.stderr:+ grep -Ei '(2 MiB|2048k)' 2023-09-23T20:13:50.875 INFO:tasks.workunit.client.0.smithi018.stdout:sparse1 2 MiB 2 2023-09-23T20:13:50.876 INFO:tasks.workunit.client.0.smithi018.stderr:++ objects sparse1 2023-09-23T20:13:50.876 INFO:tasks.workunit.client.0.smithi018.stderr:++ image=sparse1 2023-09-23T20:13:50.877 INFO:tasks.workunit.client.0.smithi018.stderr:+++ rbd info sparse1 2023-09-23T20:13:50.877 INFO:tasks.workunit.client.0.smithi018.stderr:+++ grep block_name_prefix 2023-09-23T20:13:50.877 INFO:tasks.workunit.client.0.smithi018.stderr:+++ awk '{print $NF;}' 2023-09-23T20:13:50.920 INFO:tasks.workunit.client.0.smithi018.stderr:++ prefix=rbd_data.3b768beaa6a8 2023-09-23T20:13:50.921 INFO:tasks.workunit.client.0.smithi018.stderr:+++ grep rbd_data.3b768beaa6a8 2023-09-23T20:13:50.921 INFO:tasks.workunit.client.0.smithi018.stderr:++++ get_image_data_pool sparse1 2023-09-23T20:13:50.921 INFO:tasks.workunit.client.0.smithi018.stderr:+++ sed -e 's/rbd_data.3b768beaa6a8\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-09-23T20:13:50.921 INFO:tasks.workunit.client.0.smithi018.stderr:++++ image=sparse1 2023-09-23T20:13:50.922 INFO:tasks.workunit.client.0.smithi018.stderr:+++ sort -u 2023-09-23T20:13:50.924 INFO:tasks.workunit.client.0.smithi018.stderr:+++++ rbd info sparse1 2023-09-23T20:13:50.924 INFO:tasks.workunit.client.0.smithi018.stderr:+++++ grep 'data_pool: ' 2023-09-23T20:13:50.924 INFO:tasks.workunit.client.0.smithi018.stderr:+++++ awk -F: '{ print $NF }' 2023-09-23T20:13:50.963 INFO:tasks.workunit.client.0.smithi018.stderr:++++ data_pool= 2023-09-23T20:13:50.964 INFO:tasks.workunit.client.0.smithi018.stderr:++++ '[' -z ']' 2023-09-23T20:13:50.964 INFO:tasks.workunit.client.0.smithi018.stderr:++++ data_pool=rbd 2023-09-23T20:13:50.964 INFO:tasks.workunit.client.0.smithi018.stderr:++++ echo rbd 2023-09-23T20:13:50.964 INFO:tasks.workunit.client.0.smithi018.stderr:+++ rados ls -p rbd 2023-09-23T20:13:51.001 INFO:tasks.workunit.client.0.smithi018.stderr:++ objects=1 2023-09-23T20:13:51.002 INFO:tasks.workunit.client.0.smithi018.stderr:++ echo 1 2023-09-23T20:13:51.002 INFO:tasks.workunit.client.0.smithi018.stderr:+ '[' 0 -eq 1 -o 1 = 1 ']' 2023-09-23T20:13:51.002 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd export sparse1 /tmp/rbd_import_export_159798/sparse1.out 2023-09-23T20:13:51.042 INFO:tasks.workunit.client.0.smithi018.stderr: Exporting image: 50% complete... Exporting image: 100% complete...done. 2023-09-23T20:13:51.046 INFO:tasks.workunit.client.0.smithi018.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_159798/sparse1 /tmp/rbd_import_export_159798/sparse1.out 2023-09-23T20:13:51.046 INFO:tasks.workunit.client.0.smithi018.stderr:+ cmp -l /tmp/rbd_import_export_159798/sparse1 /tmp/rbd_import_export_159798/sparse1.out 2023-09-23T20:13:51.048 INFO:tasks.workunit.client.0.smithi018.stderr:++ stat /tmp/rbd_import_export_159798/sparse1 --format %b 2023-09-23T20:13:51.049 INFO:tasks.workunit.client.0.smithi018.stderr:+ origsize=2048 2023-09-23T20:13:51.050 INFO:tasks.workunit.client.0.smithi018.stderr:++ stat /tmp/rbd_import_export_159798/sparse1.out --format %b 2023-09-23T20:13:51.051 INFO:tasks.workunit.client.0.smithi018.stderr:+ exportsize=2048 2023-09-23T20:13:51.051 INFO:tasks.workunit.client.0.smithi018.stderr:+ difference=0 2023-09-23T20:13:51.051 INFO:tasks.workunit.client.0.smithi018.stderr:+ difference=0 2023-09-23T20:13:51.051 INFO:tasks.workunit.client.0.smithi018.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-09-23T20:13:51.051 INFO:tasks.workunit.client.0.smithi018.stderr:+ rm /tmp/rbd_import_export_159798/sparse1.out 2023-09-23T20:13:51.052 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd rm sparse1 2023-09-23T20:13:51.142 INFO:tasks.workunit.client.0.smithi018.stderr: Removing image: 50% complete... Removing image: 100% complete...done. 2023-09-23T20:13:51.146 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd rm sparse2 2023-09-23T20:13:51.185 INFO:tasks.workunit.client.0.smithi018.stderr: Removing image: 0% complete...failed. 2023-09-23T20:13:51.185 INFO:tasks.workunit.client.0.smithi018.stderr:rbd: delete error: (2) No such file or directory 2023-09-23T20:13:51.188 INFO:tasks.workunit.client.0.smithi018.stderr:+ true 2023-09-23T20:13:51.188 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_159798/sparse2 2023-09-23T20:13:51.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:13:50 smithi149 ceph-mon[100764]: from='client.? 172.21.15.18:0/3852415096' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T20:13:51.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:13:50 smithi149 ceph-mon[100764]: from='client.? 172.21.15.18:0/3852415096' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T20:13:51.248 INFO:tasks.workunit.client.0.smithi018.stderr: Importing image: 50% complete... Importing image: 100% complete...done. 2023-09-23T20:13:51.249 INFO:tasks.workunit.client.0.smithi018.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T20:13:51.249 INFO:tasks.workunit.client.0.smithi018.stderr:rbd: --order is deprecated, use --object-size 2023-09-23T20:13:51.252 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd ls -l 2023-09-23T20:13:51.252 INFO:tasks.workunit.client.0.smithi018.stderr:+ grep sparse2 2023-09-23T20:13:51.252 INFO:tasks.workunit.client.0.smithi018.stderr:+ grep -Ei '(2 MiB|2048k)' 2023-09-23T20:13:51.290 INFO:tasks.workunit.client.0.smithi018.stdout:sparse2 2 MiB 2 2023-09-23T20:13:51.291 INFO:tasks.workunit.client.0.smithi018.stderr:++ objects sparse2 2023-09-23T20:13:51.291 INFO:tasks.workunit.client.0.smithi018.stderr:++ image=sparse2 2023-09-23T20:13:51.291 INFO:tasks.workunit.client.0.smithi018.stderr:+++ rbd info sparse2 2023-09-23T20:13:51.291 INFO:tasks.workunit.client.0.smithi018.stderr:+++ grep block_name_prefix 2023-09-23T20:13:51.291 INFO:tasks.workunit.client.0.smithi018.stderr:+++ awk '{print $NF;}' 2023-09-23T20:13:51.335 INFO:tasks.workunit.client.0.smithi018.stderr:++ prefix=rbd_data.3ba9456184ad 2023-09-23T20:13:51.335 INFO:tasks.workunit.client.0.smithi018.stderr:+++ grep rbd_data.3ba9456184ad 2023-09-23T20:13:51.335 INFO:tasks.workunit.client.0.smithi018.stderr:++++ get_image_data_pool sparse2 2023-09-23T20:13:51.336 INFO:tasks.workunit.client.0.smithi018.stderr:++++ image=sparse2 2023-09-23T20:13:51.336 INFO:tasks.workunit.client.0.smithi018.stderr:+++ sed -e 's/rbd_data.3ba9456184ad\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-09-23T20:13:51.336 INFO:tasks.workunit.client.0.smithi018.stderr:+++ sort -u 2023-09-23T20:13:51.339 INFO:tasks.workunit.client.0.smithi018.stderr:+++++ rbd info sparse2 2023-09-23T20:13:51.340 INFO:tasks.workunit.client.0.smithi018.stderr:+++++ grep 'data_pool: ' 2023-09-23T20:13:51.340 INFO:tasks.workunit.client.0.smithi018.stderr:+++++ awk -F: '{ print $NF }' 2023-09-23T20:13:51.376 INFO:tasks.workunit.client.0.smithi018.stderr:++++ data_pool= 2023-09-23T20:13:51.376 INFO:tasks.workunit.client.0.smithi018.stderr:++++ '[' -z ']' 2023-09-23T20:13:51.377 INFO:tasks.workunit.client.0.smithi018.stderr:++++ data_pool=rbd 2023-09-23T20:13:51.377 INFO:tasks.workunit.client.0.smithi018.stderr:++++ echo rbd 2023-09-23T20:13:51.377 INFO:tasks.workunit.client.0.smithi018.stderr:+++ rados ls -p rbd 2023-09-23T20:13:51.390 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:13:50 smithi018 ceph-mon[109137]: from='client.? 172.21.15.18:0/3852415096' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T20:13:51.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:13:50 smithi018 ceph-mon[109137]: from='client.? 172.21.15.18:0/3852415096' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T20:13:51.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:13:50 smithi018 ceph-mon[95889]: from='client.? 172.21.15.18:0/3852415096' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T20:13:51.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:13:50 smithi018 ceph-mon[95889]: from='client.? 172.21.15.18:0/3852415096' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T20:13:51.410 INFO:tasks.workunit.client.0.smithi018.stderr:++ objects=0 2023-09-23T20:13:51.410 INFO:tasks.workunit.client.0.smithi018.stderr:++ echo 0 2023-09-23T20:13:51.410 INFO:tasks.workunit.client.0.smithi018.stderr:+ '[' 0 -eq 1 -o 0 = 0 ']' 2023-09-23T20:13:51.410 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd export sparse2 /tmp/rbd_import_export_159798/sparse2.out 2023-09-23T20:13:51.449 INFO:tasks.workunit.client.0.smithi018.stderr: Exporting image: 50% complete... Exporting image: 100% complete...done. 2023-09-23T20:13:51.452 INFO:tasks.workunit.client.0.smithi018.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_159798/sparse2 /tmp/rbd_import_export_159798/sparse2.out 2023-09-23T20:13:51.452 INFO:tasks.workunit.client.0.smithi018.stderr:+ cmp -l /tmp/rbd_import_export_159798/sparse2 /tmp/rbd_import_export_159798/sparse2.out 2023-09-23T20:13:51.455 INFO:tasks.workunit.client.0.smithi018.stderr:++ stat /tmp/rbd_import_export_159798/sparse2 --format %b 2023-09-23T20:13:51.455 INFO:tasks.workunit.client.0.smithi018.stderr:+ origsize=2048 2023-09-23T20:13:51.456 INFO:tasks.workunit.client.0.smithi018.stderr:++ stat /tmp/rbd_import_export_159798/sparse2.out --format %b 2023-09-23T20:13:51.457 INFO:tasks.workunit.client.0.smithi018.stderr:+ exportsize=2048 2023-09-23T20:13:51.457 INFO:tasks.workunit.client.0.smithi018.stderr:+ difference=0 2023-09-23T20:13:51.457 INFO:tasks.workunit.client.0.smithi018.stderr:+ difference=0 2023-09-23T20:13:51.457 INFO:tasks.workunit.client.0.smithi018.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-09-23T20:13:51.457 INFO:tasks.workunit.client.0.smithi018.stderr:+ rm /tmp/rbd_import_export_159798/sparse2.out 2023-09-23T20:13:51.458 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd rm sparse2 2023-09-23T20:13:51.551 INFO:tasks.workunit.client.0.smithi018.stderr: Removing image: 50% complete... Removing image: 100% complete...done. 2023-09-23T20:13:51.555 INFO:tasks.workunit.client.0.smithi018.stderr:+ truncate /tmp/rbd_import_export_159798/sparse1 -s 10M 2023-09-23T20:13:51.555 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd import --new-format --order 20 - sparse1 2023-09-23T20:13:51.644 INFO:tasks.workunit.client.0.smithi018.stderr: Importing image: 100% complete...done. 2023-09-23T20:13:51.645 INFO:tasks.workunit.client.0.smithi018.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T20:13:51.645 INFO:tasks.workunit.client.0.smithi018.stderr:rbd: --order is deprecated, use --object-size 2023-09-23T20:13:51.648 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd ls -l 2023-09-23T20:13:51.648 INFO:tasks.workunit.client.0.smithi018.stderr:+ grep sparse1 2023-09-23T20:13:51.648 INFO:tasks.workunit.client.0.smithi018.stderr:+ grep -Ei '(10 MiB|10240k)' 2023-09-23T20:13:51.686 INFO:tasks.workunit.client.0.smithi018.stdout:sparse1 10 MiB 2 2023-09-23T20:13:51.687 INFO:tasks.workunit.client.0.smithi018.stderr:++ objects sparse1 2023-09-23T20:13:51.687 INFO:tasks.workunit.client.0.smithi018.stderr:++ image=sparse1 2023-09-23T20:13:51.687 INFO:tasks.workunit.client.0.smithi018.stderr:+++ rbd info sparse1 2023-09-23T20:13:51.687 INFO:tasks.workunit.client.0.smithi018.stderr:+++ grep block_name_prefix 2023-09-23T20:13:51.687 INFO:tasks.workunit.client.0.smithi018.stderr:+++ awk '{print $NF;}' 2023-09-23T20:13:51.726 INFO:tasks.workunit.client.0.smithi018.stderr:++ prefix=rbd_data.3bd64441c7c2 2023-09-23T20:13:51.727 INFO:tasks.workunit.client.0.smithi018.stderr:+++ grep rbd_data.3bd64441c7c2 2023-09-23T20:13:51.727 INFO:tasks.workunit.client.0.smithi018.stderr:++++ get_image_data_pool sparse1 2023-09-23T20:13:51.727 INFO:tasks.workunit.client.0.smithi018.stderr:+++ sed -e 's/rbd_data.3bd64441c7c2\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-09-23T20:13:51.727 INFO:tasks.workunit.client.0.smithi018.stderr:++++ image=sparse1 2023-09-23T20:13:51.727 INFO:tasks.workunit.client.0.smithi018.stderr:+++ sort -u 2023-09-23T20:13:51.728 INFO:tasks.workunit.client.0.smithi018.stderr:+++++ rbd info sparse1 2023-09-23T20:13:51.728 INFO:tasks.workunit.client.0.smithi018.stderr:+++++ grep 'data_pool: ' 2023-09-23T20:13:51.729 INFO:tasks.workunit.client.0.smithi018.stderr:+++++ awk -F: '{ print $NF }' 2023-09-23T20:13:51.766 INFO:tasks.workunit.client.0.smithi018.stderr:++++ data_pool= 2023-09-23T20:13:51.766 INFO:tasks.workunit.client.0.smithi018.stderr:++++ '[' -z ']' 2023-09-23T20:13:51.766 INFO:tasks.workunit.client.0.smithi018.stderr:++++ data_pool=rbd 2023-09-23T20:13:51.767 INFO:tasks.workunit.client.0.smithi018.stderr:++++ echo rbd 2023-09-23T20:13:51.767 INFO:tasks.workunit.client.0.smithi018.stderr:+++ rados ls -p rbd 2023-09-23T20:13:51.803 INFO:tasks.workunit.client.0.smithi018.stderr:++ objects=1 2023-09-23T20:13:51.803 INFO:tasks.workunit.client.0.smithi018.stderr:++ echo 1 2023-09-23T20:13:51.803 INFO:tasks.workunit.client.0.smithi018.stderr:+ '[' 0 -eq 1 -o 1 = 1 ']' 2023-09-23T20:13:51.804 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd export sparse1 /tmp/rbd_import_export_159798/sparse1.out 2023-09-23T20:13:51.841 INFO:tasks.workunit.client.0.smithi018.stderr: Exporting image: 10% complete... Exporting image: 20% complete... Exporting image: 30% complete... Exporting image: 40% complete... Exporting image: 50% complete... Exporting image: 60% complete... Exporting image: 70% complete... Exporting image: 80% complete... Exporting image: 90% complete... Exporting image: 100% complete...done. 2023-09-23T20:13:51.845 INFO:tasks.workunit.client.0.smithi018.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_159798/sparse1 /tmp/rbd_import_export_159798/sparse1.out 2023-09-23T20:13:51.845 INFO:tasks.workunit.client.0.smithi018.stderr:+ cmp -l /tmp/rbd_import_export_159798/sparse1 /tmp/rbd_import_export_159798/sparse1.out 2023-09-23T20:13:51.853 INFO:tasks.workunit.client.0.smithi018.stderr:++ stat /tmp/rbd_import_export_159798/sparse1 --format %b 2023-09-23T20:13:51.853 INFO:tasks.workunit.client.0.smithi018.stderr:+ origsize=2048 2023-09-23T20:13:51.854 INFO:tasks.workunit.client.0.smithi018.stderr:++ stat /tmp/rbd_import_export_159798/sparse1.out --format %b 2023-09-23T20:13:51.855 INFO:tasks.workunit.client.0.smithi018.stderr:+ exportsize=2048 2023-09-23T20:13:51.855 INFO:tasks.workunit.client.0.smithi018.stderr:+ difference=0 2023-09-23T20:13:51.855 INFO:tasks.workunit.client.0.smithi018.stderr:+ difference=0 2023-09-23T20:13:51.855 INFO:tasks.workunit.client.0.smithi018.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-09-23T20:13:51.856 INFO:tasks.workunit.client.0.smithi018.stderr:+ rm /tmp/rbd_import_export_159798/sparse1.out 2023-09-23T20:13:51.856 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd rm sparse1 2023-09-23T20:13:51.945 INFO:tasks.workunit.client.0.smithi018.stderr: Removing image: 10% complete... Removing image: 20% complete... Removing image: 30% complete... Removing image: 40% complete... Removing image: 50% complete... Removing image: 60% complete... Removing image: 70% complete... Removing image: 80% complete... Removing image: 90% complete... Removing image: 100% complete...done. 2023-09-23T20:13:51.948 INFO:tasks.workunit.client.0.smithi018.stderr:+ dd if=/dev/urandom bs=2M count=1 of=/tmp/rbd_import_export_159798/sparse2 oflag=append conv=notrunc 2023-09-23T20:13:51.959 INFO:tasks.workunit.client.0.smithi018.stderr:1+0 records in 2023-09-23T20:13:51.960 INFO:tasks.workunit.client.0.smithi018.stderr:1+0 records out 2023-09-23T20:13:51.960 INFO:tasks.workunit.client.0.smithi018.stderr:2097152 bytes (2.1 MB, 2.0 MiB) copied, 0.0102249 s, 205 MB/s 2023-09-23T20:13:51.960 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd import --new-format --order 20 - sparse2 2023-09-23T20:13:52.039 INFO:tasks.workunit.client.0.smithi018.stderr: Importing image: 100% complete...done. 2023-09-23T20:13:52.039 INFO:tasks.workunit.client.0.smithi018.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T20:13:52.039 INFO:tasks.workunit.client.0.smithi018.stderr:rbd: --order is deprecated, use --object-size 2023-09-23T20:13:52.042 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd ls -l 2023-09-23T20:13:52.042 INFO:tasks.workunit.client.0.smithi018.stderr:+ grep sparse2 2023-09-23T20:13:52.042 INFO:tasks.workunit.client.0.smithi018.stderr:+ grep -Ei '(4 MiB|4096k)' 2023-09-23T20:13:52.080 INFO:tasks.workunit.client.0.smithi018.stdout:sparse2 4 MiB 2 2023-09-23T20:13:52.081 INFO:tasks.workunit.client.0.smithi018.stderr:++ objects sparse2 2023-09-23T20:13:52.081 INFO:tasks.workunit.client.0.smithi018.stderr:++ image=sparse2 2023-09-23T20:13:52.082 INFO:tasks.workunit.client.0.smithi018.stderr:+++ rbd info sparse2 2023-09-23T20:13:52.082 INFO:tasks.workunit.client.0.smithi018.stderr:+++ grep block_name_prefix 2023-09-23T20:13:52.082 INFO:tasks.workunit.client.0.smithi018.stderr:+++ awk '{print $NF;}' 2023-09-23T20:13:52.119 INFO:tasks.workunit.client.0.smithi018.stderr:++ prefix=rbd_data.3c00b54fc162 2023-09-23T20:13:52.121 INFO:tasks.workunit.client.0.smithi018.stderr:+++ grep rbd_data.3c00b54fc162 2023-09-23T20:13:52.121 INFO:tasks.workunit.client.0.smithi018.stderr:++++ get_image_data_pool sparse2 2023-09-23T20:13:52.121 INFO:tasks.workunit.client.0.smithi018.stderr:++++ image=sparse2 2023-09-23T20:13:52.121 INFO:tasks.workunit.client.0.smithi018.stderr:+++ sed -e 's/rbd_data.3c00b54fc162\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-09-23T20:13:52.121 INFO:tasks.workunit.client.0.smithi018.stderr:+++ sort -u 2023-09-23T20:13:52.124 INFO:tasks.workunit.client.0.smithi018.stderr:+++++ rbd info sparse2 2023-09-23T20:13:52.125 INFO:tasks.workunit.client.0.smithi018.stderr:+++++ grep 'data_pool: ' 2023-09-23T20:13:52.125 INFO:tasks.workunit.client.0.smithi018.stderr:+++++ awk -F: '{ print $NF }' 2023-09-23T20:13:52.159 INFO:tasks.workunit.client.0.smithi018.stderr:++++ data_pool= 2023-09-23T20:13:52.159 INFO:tasks.workunit.client.0.smithi018.stderr:++++ '[' -z ']' 2023-09-23T20:13:52.159 INFO:tasks.workunit.client.0.smithi018.stderr:++++ data_pool=rbd 2023-09-23T20:13:52.160 INFO:tasks.workunit.client.0.smithi018.stderr:++++ echo rbd 2023-09-23T20:13:52.160 INFO:tasks.workunit.client.0.smithi018.stderr:+++ rados ls -p rbd 2023-09-23T20:13:52.195 INFO:tasks.workunit.client.0.smithi018.stderr:++ objects='0 2023-09-23T20:13:52.195 INFO:tasks.workunit.client.0.smithi018.stderr:2 2023-09-23T20:13:52.195 INFO:tasks.workunit.client.0.smithi018.stderr:3' 2023-09-23T20:13:52.195 INFO:tasks.workunit.client.0.smithi018.stderr:++ echo 0 2 3 2023-09-23T20:13:52.195 INFO:tasks.workunit.client.0.smithi018.stderr:+ '[' 0 -eq 1 -o '0 2 3' = '0 2 3' ']' 2023-09-23T20:13:52.195 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd export sparse2 /tmp/rbd_import_export_159798/sparse2.out 2023-09-23T20:13:52.215 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:13:51 smithi149 ceph-mon[100764]: from='client.? 172.21.15.18:0/3766449124' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:13:52.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:13:51 smithi149 ceph-mon[100764]: pgmap v90: 73 pgs: 73 active+clean; 579 KiB data, 58 MiB used, 715 GiB / 715 GiB avail; 125 KiB/s rd, 8.8 KiB/s wr, 169 op/s 2023-09-23T20:13:52.235 INFO:tasks.workunit.client.0.smithi018.stderr: Exporting image: 25% complete... Exporting image: 50% complete... Exporting image: 75% complete... Exporting image: 100% complete...done. 2023-09-23T20:13:52.238 INFO:tasks.workunit.client.0.smithi018.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_159798/sparse2 /tmp/rbd_import_export_159798/sparse2.out 2023-09-23T20:13:52.238 INFO:tasks.workunit.client.0.smithi018.stderr:+ cmp -l /tmp/rbd_import_export_159798/sparse2 /tmp/rbd_import_export_159798/sparse2.out 2023-09-23T20:13:52.241 INFO:tasks.workunit.client.0.smithi018.stderr:++ stat /tmp/rbd_import_export_159798/sparse2 --format %b 2023-09-23T20:13:52.242 INFO:tasks.workunit.client.0.smithi018.stderr:+ origsize=6144 2023-09-23T20:13:52.243 INFO:tasks.workunit.client.0.smithi018.stderr:++ stat /tmp/rbd_import_export_159798/sparse2.out --format %b 2023-09-23T20:13:52.245 INFO:tasks.workunit.client.0.smithi018.stderr:+ exportsize=6144 2023-09-23T20:13:52.245 INFO:tasks.workunit.client.0.smithi018.stderr:+ difference=0 2023-09-23T20:13:52.245 INFO:tasks.workunit.client.0.smithi018.stderr:+ difference=0 2023-09-23T20:13:52.246 INFO:tasks.workunit.client.0.smithi018.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-09-23T20:13:52.246 INFO:tasks.workunit.client.0.smithi018.stderr:+ rm /tmp/rbd_import_export_159798/sparse2.out 2023-09-23T20:13:52.246 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd rm sparse2 2023-09-23T20:13:52.336 INFO:tasks.workunit.client.0.smithi018.stderr: Removing image: 25% complete... Removing image: 50% complete... Removing image: 75% complete... Removing image: 100% complete...done. 2023-09-23T20:13:52.339 INFO:tasks.workunit.client.0.smithi018.stderr:+ echo 'partially-sparse file imports to partially-sparse image' 2023-09-23T20:13:52.339 INFO:tasks.workunit.client.0.smithi018.stdout:partially-sparse file imports to partially-sparse image 2023-09-23T20:13:52.340 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_159798/sparse1 sparse 2023-09-23T20:13:52.390 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:13:51 smithi018 ceph-mon[95889]: from='client.? 172.21.15.18:0/3766449124' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:13:52.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:13:51 smithi018 ceph-mon[95889]: pgmap v90: 73 pgs: 73 active+clean; 579 KiB data, 58 MiB used, 715 GiB / 715 GiB avail; 125 KiB/s rd, 8.8 KiB/s wr, 169 op/s 2023-09-23T20:13:52.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:13:51 smithi018 ceph-mon[109137]: from='client.? 172.21.15.18:0/3766449124' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:13:52.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:13:51 smithi018 ceph-mon[109137]: pgmap v90: 73 pgs: 73 active+clean; 579 KiB data, 58 MiB used, 715 GiB / 715 GiB avail; 125 KiB/s rd, 8.8 KiB/s wr, 169 op/s 2023-09-23T20:13:52.404 INFO:tasks.workunit.client.0.smithi018.stderr: Importing image: 10% complete... Importing image: 20% complete... Importing image: 30% complete... Importing image: 40% complete... Importing image: 50% complete... Importing image: 60% complete... Importing image: 70% complete... Importing image: 80% complete... Importing image: 90% complete... Importing image: 100% complete...done. 2023-09-23T20:13:52.405 INFO:tasks.workunit.client.0.smithi018.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T20:13:52.405 INFO:tasks.workunit.client.0.smithi018.stderr:rbd: --order is deprecated, use --object-size 2023-09-23T20:13:52.408 INFO:tasks.workunit.client.0.smithi018.stderr:++ objects sparse 2023-09-23T20:13:52.408 INFO:tasks.workunit.client.0.smithi018.stderr:++ image=sparse 2023-09-23T20:13:52.408 INFO:tasks.workunit.client.0.smithi018.stderr:+++ rbd info sparse 2023-09-23T20:13:52.408 INFO:tasks.workunit.client.0.smithi018.stderr:+++ grep block_name_prefix 2023-09-23T20:13:52.408 INFO:tasks.workunit.client.0.smithi018.stderr:+++ awk '{print $NF;}' 2023-09-23T20:13:52.446 INFO:tasks.workunit.client.0.smithi018.stderr:++ prefix=rbd_data.3c2d9f962f28 2023-09-23T20:13:52.447 INFO:tasks.workunit.client.0.smithi018.stderr:+++ grep rbd_data.3c2d9f962f28 2023-09-23T20:13:52.447 INFO:tasks.workunit.client.0.smithi018.stderr:++++ get_image_data_pool sparse 2023-09-23T20:13:52.447 INFO:tasks.workunit.client.0.smithi018.stderr:+++ sed -e 's/rbd_data.3c2d9f962f28\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-09-23T20:13:52.447 INFO:tasks.workunit.client.0.smithi018.stderr:++++ image=sparse 2023-09-23T20:13:52.447 INFO:tasks.workunit.client.0.smithi018.stderr:+++ sort -u 2023-09-23T20:13:52.449 INFO:tasks.workunit.client.0.smithi018.stderr:+++++ rbd info sparse 2023-09-23T20:13:52.450 INFO:tasks.workunit.client.0.smithi018.stderr:+++++ grep 'data_pool: ' 2023-09-23T20:13:52.450 INFO:tasks.workunit.client.0.smithi018.stderr:+++++ awk -F: '{ print $NF }' 2023-09-23T20:13:52.485 INFO:tasks.workunit.client.0.smithi018.stderr:++++ data_pool= 2023-09-23T20:13:52.486 INFO:tasks.workunit.client.0.smithi018.stderr:++++ '[' -z ']' 2023-09-23T20:13:52.486 INFO:tasks.workunit.client.0.smithi018.stderr:++++ data_pool=rbd 2023-09-23T20:13:52.486 INFO:tasks.workunit.client.0.smithi018.stderr:++++ echo rbd 2023-09-23T20:13:52.486 INFO:tasks.workunit.client.0.smithi018.stderr:+++ rados ls -p rbd 2023-09-23T20:13:52.524 INFO:tasks.workunit.client.0.smithi018.stderr:++ objects=1 2023-09-23T20:13:52.524 INFO:tasks.workunit.client.0.smithi018.stderr:++ echo 1 2023-09-23T20:13:52.524 INFO:tasks.workunit.client.0.smithi018.stderr:+ '[' 0 -eq 1 -o 1 = 1 ']' 2023-09-23T20:13:52.525 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd rm sparse 2023-09-23T20:13:52.623 INFO:tasks.workunit.client.0.smithi018.stderr: Removing image: 10% complete... Removing image: 20% complete... Removing image: 30% complete... Removing image: 40% complete... Removing image: 50% complete... Removing image: 60% complete... Removing image: 70% complete... Removing image: 80% complete... Removing image: 90% complete... Removing image: 100% complete...done. 2023-09-23T20:13:52.626 INFO:tasks.workunit.client.0.smithi018.stdout:zeros import through stdin to sparse image 2023-09-23T20:13:52.627 INFO:tasks.workunit.client.0.smithi018.stderr:+ echo 'zeros import through stdin to sparse image' 2023-09-23T20:13:52.627 INFO:tasks.workunit.client.0.smithi018.stderr:+ dd if=/dev/zero bs=1M count=4 2023-09-23T20:13:52.627 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd import --new-format - sparse 2023-09-23T20:13:52.687 INFO:tasks.workunit.client.0.smithi018.stderr:4+0 records in 2023-09-23T20:13:52.687 INFO:tasks.workunit.client.0.smithi018.stderr:4+0 records out 2023-09-23T20:13:52.688 INFO:tasks.workunit.client.0.smithi018.stderr:4194304 bytes (4.2 MB, 4.0 MiB) copied, 0.0598116 s, 70.1 MB/s 2023-09-23T20:13:52.697 INFO:tasks.workunit.client.0.smithi018.stderr: Importing image: 100% complete...done. 2023-09-23T20:13:52.698 INFO:tasks.workunit.client.0.smithi018.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T20:13:52.700 INFO:tasks.workunit.client.0.smithi018.stderr:++ objects sparse 2023-09-23T20:13:52.701 INFO:tasks.workunit.client.0.smithi018.stderr:++ image=sparse 2023-09-23T20:13:52.701 INFO:tasks.workunit.client.0.smithi018.stderr:+++ rbd info sparse 2023-09-23T20:13:52.701 INFO:tasks.workunit.client.0.smithi018.stderr:+++ grep block_name_prefix 2023-09-23T20:13:52.701 INFO:tasks.workunit.client.0.smithi018.stderr:+++ awk '{print $NF;}' 2023-09-23T20:13:52.742 INFO:tasks.workunit.client.0.smithi018.stderr:++ prefix=rbd_data.3c4b89de8b9a 2023-09-23T20:13:52.743 INFO:tasks.workunit.client.0.smithi018.stderr:+++ grep rbd_data.3c4b89de8b9a 2023-09-23T20:13:52.743 INFO:tasks.workunit.client.0.smithi018.stderr:+++ sed -e 's/rbd_data.3c4b89de8b9a\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-09-23T20:13:52.743 INFO:tasks.workunit.client.0.smithi018.stderr:++++ get_image_data_pool sparse 2023-09-23T20:13:52.743 INFO:tasks.workunit.client.0.smithi018.stderr:++++ image=sparse 2023-09-23T20:13:52.743 INFO:tasks.workunit.client.0.smithi018.stderr:+++ sort -u 2023-09-23T20:13:52.744 INFO:tasks.workunit.client.0.smithi018.stderr:+++++ rbd info sparse 2023-09-23T20:13:52.744 INFO:tasks.workunit.client.0.smithi018.stderr:+++++ awk -F: '{ print $NF }' 2023-09-23T20:13:52.746 INFO:tasks.workunit.client.0.smithi018.stderr:+++++ grep 'data_pool: ' 2023-09-23T20:13:52.787 INFO:tasks.workunit.client.0.smithi018.stderr:++++ data_pool= 2023-09-23T20:13:52.788 INFO:tasks.workunit.client.0.smithi018.stderr:++++ '[' -z ']' 2023-09-23T20:13:52.788 INFO:tasks.workunit.client.0.smithi018.stderr:++++ data_pool=rbd 2023-09-23T20:13:52.788 INFO:tasks.workunit.client.0.smithi018.stderr:++++ echo rbd 2023-09-23T20:13:52.788 INFO:tasks.workunit.client.0.smithi018.stderr:+++ rados ls -p rbd 2023-09-23T20:13:52.832 INFO:tasks.workunit.client.0.smithi018.stderr:++ objects= 2023-09-23T20:13:52.833 INFO:tasks.workunit.client.0.smithi018.stderr:++ echo 2023-09-23T20:13:52.833 INFO:tasks.workunit.client.0.smithi018.stderr:+ '[' 0 -eq 1 -o '' = '' ']' 2023-09-23T20:13:52.833 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd rm sparse 2023-09-23T20:13:52.932 INFO:tasks.workunit.client.0.smithi018.stderr: Removing image: 100% complete...done. 2023-09-23T20:13:52.934 INFO:tasks.workunit.client.0.smithi018.stdout:zeros export to sparse file 2023-09-23T20:13:52.935 INFO:tasks.workunit.client.0.smithi018.stderr:+ echo 'zeros export to sparse file' 2023-09-23T20:13:52.935 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd create --new-format sparse --size 4 2023-09-23T20:13:52.976 INFO:tasks.workunit.client.0.smithi018.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T20:13:52.979 INFO:tasks.workunit.client.0.smithi018.stderr:++ rbd info sparse 2023-09-23T20:13:52.980 INFO:tasks.workunit.client.0.smithi018.stderr:++ grep block_name_prefix 2023-09-23T20:13:52.980 INFO:tasks.workunit.client.0.smithi018.stderr:++ awk '{print $NF;}' 2023-09-23T20:13:53.019 INFO:tasks.workunit.client.0.smithi018.stderr:+ prefix=rbd_data.631a4a8c7a00 2023-09-23T20:13:53.019 INFO:tasks.workunit.client.0.smithi018.stderr:+ dd if=/dev/zero bs=4M count=1 2023-09-23T20:13:53.020 INFO:tasks.workunit.client.0.smithi018.stderr:++ get_image_data_pool sparse 2023-09-23T20:13:53.020 INFO:tasks.workunit.client.0.smithi018.stderr:++ image=sparse 2023-09-23T20:13:53.020 INFO:tasks.workunit.client.0.smithi018.stderr:+++ rbd info sparse 2023-09-23T20:13:53.020 INFO:tasks.workunit.client.0.smithi018.stderr:+++ grep 'data_pool: ' 2023-09-23T20:13:53.020 INFO:tasks.workunit.client.0.smithi018.stderr:+++ awk -F: '{ print $NF }' 2023-09-23T20:13:53.062 INFO:tasks.workunit.client.0.smithi018.stderr:++ data_pool= 2023-09-23T20:13:53.062 INFO:tasks.workunit.client.0.smithi018.stderr:++ '[' -z ']' 2023-09-23T20:13:53.062 INFO:tasks.workunit.client.0.smithi018.stderr:++ data_pool=rbd 2023-09-23T20:13:53.062 INFO:tasks.workunit.client.0.smithi018.stderr:++ echo rbd 2023-09-23T20:13:53.062 INFO:tasks.workunit.client.0.smithi018.stderr:+ rados -p rbd put rbd_data.631a4a8c7a00.000000000000 - 2023-09-23T20:13:53.099 INFO:tasks.workunit.client.0.smithi018.stderr:1+0 records in 2023-09-23T20:13:53.100 INFO:tasks.workunit.client.0.smithi018.stderr:1+0 records out 2023-09-23T20:13:53.100 INFO:tasks.workunit.client.0.smithi018.stderr:4194304 bytes (4.2 MB, 4.0 MiB) copied, 0.0797081 s, 52.6 MB/s 2023-09-23T20:13:53.124 INFO:tasks.workunit.client.0.smithi018.stderr:++ objects sparse 2023-09-23T20:13:53.124 INFO:tasks.workunit.client.0.smithi018.stderr:++ image=sparse 2023-09-23T20:13:53.124 INFO:tasks.workunit.client.0.smithi018.stderr:+++ rbd info sparse 2023-09-23T20:13:53.124 INFO:tasks.workunit.client.0.smithi018.stderr:+++ grep block_name_prefix 2023-09-23T20:13:53.125 INFO:tasks.workunit.client.0.smithi018.stderr:+++ awk '{print $NF;}' 2023-09-23T20:13:53.166 INFO:tasks.workunit.client.0.smithi018.stderr:++ prefix=rbd_data.631a4a8c7a00 2023-09-23T20:13:53.167 INFO:tasks.workunit.client.0.smithi018.stderr:+++ grep rbd_data.631a4a8c7a00 2023-09-23T20:13:53.167 INFO:tasks.workunit.client.0.smithi018.stderr:++++ get_image_data_pool sparse 2023-09-23T20:13:53.167 INFO:tasks.workunit.client.0.smithi018.stderr:++++ image=sparse 2023-09-23T20:13:53.167 INFO:tasks.workunit.client.0.smithi018.stderr:+++ sed -e 's/rbd_data.631a4a8c7a00\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-09-23T20:13:53.167 INFO:tasks.workunit.client.0.smithi018.stderr:+++ sort -u 2023-09-23T20:13:53.168 INFO:tasks.workunit.client.0.smithi018.stderr:+++++ grep 'data_pool: ' 2023-09-23T20:13:53.168 INFO:tasks.workunit.client.0.smithi018.stderr:+++++ awk -F: '{ print $NF }' 2023-09-23T20:13:53.169 INFO:tasks.workunit.client.0.smithi018.stderr:+++++ rbd info sparse 2023-09-23T20:13:53.210 INFO:tasks.workunit.client.0.smithi018.stderr:++++ data_pool= 2023-09-23T20:13:53.210 INFO:tasks.workunit.client.0.smithi018.stderr:++++ '[' -z ']' 2023-09-23T20:13:53.210 INFO:tasks.workunit.client.0.smithi018.stderr:++++ data_pool=rbd 2023-09-23T20:13:53.210 INFO:tasks.workunit.client.0.smithi018.stderr:++++ echo rbd 2023-09-23T20:13:53.210 INFO:tasks.workunit.client.0.smithi018.stderr:+++ rados ls -p rbd 2023-09-23T20:13:53.258 INFO:tasks.workunit.client.0.smithi018.stderr:++ objects=0 2023-09-23T20:13:53.258 INFO:tasks.workunit.client.0.smithi018.stderr:++ echo 0 2023-09-23T20:13:53.258 INFO:tasks.workunit.client.0.smithi018.stderr:+ '[' 0 -eq 1 -o 0 = 0 ']' 2023-09-23T20:13:53.258 INFO:tasks.workunit.client.0.smithi018.stderr:+ rm /tmp/rbd_import_export_159798/sparse 2023-09-23T20:13:53.259 INFO:tasks.workunit.client.0.smithi018.stderr:rm: cannot remove '/tmp/rbd_import_export_159798/sparse': No such file or directory 2023-09-23T20:13:53.259 INFO:tasks.workunit.client.0.smithi018.stderr:+ true 2023-09-23T20:13:53.259 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd export sparse /tmp/rbd_import_export_159798/sparse 2023-09-23T20:13:53.299 INFO:tasks.workunit.client.0.smithi018.stderr: Exporting image: 100% complete...done. 2023-09-23T20:13:53.304 INFO:tasks.workunit.client.0.smithi018.stderr:++ stat /tmp/rbd_import_export_159798/sparse --format=%b 2023-09-23T20:13:53.305 INFO:tasks.workunit.client.0.smithi018.stderr:+ '[' 0 = 0 ']' 2023-09-23T20:13:53.305 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd rm sparse 2023-09-23T20:13:53.401 INFO:tasks.workunit.client.0.smithi018.stderr: Removing image: 100% complete...done. 2023-09-23T20:13:53.405 INFO:tasks.workunit.client.0.smithi018.stderr:+ rm /tmp/rbd_import_export_159798/sparse /tmp/rbd_import_export_159798/sparse1 /tmp/rbd_import_export_159798/sparse2 /tmp/rbd_import_export_159798/sparse3 2023-09-23T20:13:53.407 INFO:tasks.workunit.client.0.smithi018.stderr:rm: cannot remove '/tmp/rbd_import_export_159798/sparse3': No such file or directory 2023-09-23T20:13:53.407 INFO:tasks.workunit.client.0.smithi018.stderr:+ true 2023-09-23T20:13:53.407 INFO:tasks.workunit.client.0.smithi018.stderr:+ echo OK 2023-09-23T20:13:53.407 INFO:tasks.workunit.client.0.smithi018.stdout:OK 2023-09-23T20:13:53.408 INFO:tasks.workunit.client.0.smithi018.stderr:+ rm -rf /tmp/rbd_import_export_159798 2023-09-23T20:13:53.408 INFO:teuthology.orchestra.run:Running command with timeout 3600 2023-09-23T20:13:53.408 DEBUG:teuthology.orchestra.run.smithi018:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2023-09-23T20:13:53.443 INFO:tasks.workunit:Stopping ['rbd/import_export.sh'] on client.0... 2023-09-23T20:13:53.443 DEBUG:teuthology.orchestra.run.smithi018:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2023-09-23T20:13:53.686 DEBUG:teuthology.parallel:result is None 2023-09-23T20:13:53.687 DEBUG:teuthology.orchestra.run.smithi018:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0 2023-09-23T20:13:53.711 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:13:53 smithi018 ceph-mon[95889]: from='client.? 172.21.15.18:0/3430673084' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:13:53.712 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:13:53 smithi018 ceph-mon[109137]: from='client.? 172.21.15.18:0/3430673084' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:13:53.715 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.0/client.0 2023-09-23T20:13:53.715 DEBUG:teuthology.orchestra.run.smithi018:> rmdir -- /home/ubuntu/cephtest/mnt.0 2023-09-23T20:13:53.772 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.0/client.0 2023-09-23T20:13:53.772 INFO:teuthology.task.sequential:In sequential, running task print... 2023-09-23T20:13:53.772 INFO:teuthology.task.print:**** done rbd/import_export.sh 5-workload 2023-09-23T20:13:53.772 DEBUG:teuthology.parallel:result is None 2023-09-23T20:13:53.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:13:53 smithi149 ceph-mon[100764]: from='client.? 172.21.15.18:0/3430673084' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:13:53.966 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:13:53 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[102880]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:13:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T20:13:54.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:13:54 smithi149 ceph-mon[100764]: pgmap v91: 73 pgs: 73 active+clean; 579 KiB data, 58 MiB used, 715 GiB / 715 GiB avail; 100 KiB/s rd, 7.0 KiB/s wr, 135 op/s 2023-09-23T20:13:54.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:13:54 smithi149 ceph-mon[100764]: osdmap e87: 8 total, 8 up, 8 in 2023-09-23T20:13:55.140 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:13:54 smithi018 ceph-mon[109137]: pgmap v91: 73 pgs: 73 active+clean; 579 KiB data, 58 MiB used, 715 GiB / 715 GiB avail; 100 KiB/s rd, 7.0 KiB/s wr, 135 op/s 2023-09-23T20:13:55.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:13:54 smithi018 ceph-mon[109137]: osdmap e87: 8 total, 8 up, 8 in 2023-09-23T20:13:55.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:13:54 smithi018 ceph-mon[95889]: pgmap v91: 73 pgs: 73 active+clean; 579 KiB data, 58 MiB used, 715 GiB / 715 GiB avail; 100 KiB/s rd, 7.0 KiB/s wr, 135 op/s 2023-09-23T20:13:55.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:13:54 smithi018 ceph-mon[95889]: osdmap e87: 8 total, 8 up, 8 in 2023-09-23T20:13:57.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:13:56 smithi018 ceph-mon[109137]: pgmap v93: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 1.2 MiB/s rd, 1.6 MiB/s wr, 452 op/s 2023-09-23T20:13:57.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:13:56 smithi018 ceph-mon[109137]: from='client.? 172.21.15.18:0/1275149549' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:13:57.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:13:56 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:13:57.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:13:56 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:13:57.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:13:56 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:13:57.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:13:56 smithi018 ceph-mon[95889]: pgmap v93: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 1.2 MiB/s rd, 1.6 MiB/s wr, 452 op/s 2023-09-23T20:13:57.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:13:56 smithi018 ceph-mon[95889]: from='client.? 172.21.15.18:0/1275149549' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:13:57.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:13:56 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:13:57.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:13:56 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:13:57.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:13:56 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:13:57.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:13:56 smithi149 ceph-mon[100764]: pgmap v93: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 1.2 MiB/s rd, 1.6 MiB/s wr, 452 op/s 2023-09-23T20:13:57.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:13:56 smithi149 ceph-mon[100764]: from='client.? 172.21.15.18:0/1275149549' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:13:57.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:13:56 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:13:57.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:13:56 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:13:57.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:13:56 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:13:58.140 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:13:57 smithi018 ceph-mon[109137]: Upgrade: Target is version 18.2.0-460-gbb0daeff (unknown) 2023-09-23T20:13:58.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:13:57 smithi018 ceph-mon[109137]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d, digests ['quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d'] 2023-09-23T20:13:58.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:13:57 smithi018 ceph-mon[109137]: Upgrade: Need to upgrade myself (mgr.y) 2023-09-23T20:13:58.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:13:57 smithi018 ceph-mon[95889]: Upgrade: Target is version 18.2.0-460-gbb0daeff (unknown) 2023-09-23T20:13:58.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:13:57 smithi018 ceph-mon[95889]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d, digests ['quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d'] 2023-09-23T20:13:58.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:13:57 smithi018 ceph-mon[95889]: Upgrade: Need to upgrade myself (mgr.y) 2023-09-23T20:13:58.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:13:57 smithi149 ceph-mon[100764]: Upgrade: Target is version 18.2.0-460-gbb0daeff (unknown) 2023-09-23T20:13:58.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:13:57 smithi149 ceph-mon[100764]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d, digests ['quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d'] 2023-09-23T20:13:58.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:13:57 smithi149 ceph-mon[100764]: Upgrade: Need to upgrade myself (mgr.y) 2023-09-23T20:13:59.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:13:58 smithi018 ceph-mon[95889]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d on smithi149 2023-09-23T20:13:59.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:13:58 smithi018 ceph-mon[95889]: pgmap v94: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 1.0 MiB/s rd, 1.4 MiB/s wr, 386 op/s 2023-09-23T20:13:59.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:13:58 smithi018 ceph-mon[95889]: from='client.? 172.21.15.18:0/175249510' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:13:59.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:13:58 smithi018 ceph-mon[109137]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d on smithi149 2023-09-23T20:13:59.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:13:58 smithi018 ceph-mon[109137]: pgmap v94: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 1.0 MiB/s rd, 1.4 MiB/s wr, 386 op/s 2023-09-23T20:13:59.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:13:58 smithi018 ceph-mon[109137]: from='client.? 172.21.15.18:0/175249510' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:13:59.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:13:58 smithi149 ceph-mon[100764]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d on smithi149 2023-09-23T20:13:59.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:13:58 smithi149 ceph-mon[100764]: pgmap v94: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 1.0 MiB/s rd, 1.4 MiB/s wr, 386 op/s 2023-09-23T20:13:59.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:13:58 smithi149 ceph-mon[100764]: from='client.? 172.21.15.18:0/175249510' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:13:59.640 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:13:59 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:13:59] "GET /metrics HTTP/1.1" 200 239530 "" "Prometheus/2.43.0" 2023-09-23T20:14:00.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:13:59 smithi018 ceph-mon[95889]: pgmap v95: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 860 KiB/s rd, 1.1 MiB/s wr, 314 op/s 2023-09-23T20:14:00.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:13:59 smithi018 ceph-mon[109137]: pgmap v95: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 860 KiB/s rd, 1.1 MiB/s wr, 314 op/s 2023-09-23T20:14:00.215 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:13:59 smithi149 ceph-mon[100764]: pgmap v95: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 860 KiB/s rd, 1.1 MiB/s wr, 314 op/s 2023-09-23T20:14:01.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:00 smithi018 ceph-mon[109137]: from='client.? 172.21.15.18:0/1042944687' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:14:01.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:00 smithi018 ceph-mon[95889]: from='client.? 172.21.15.18:0/1042944687' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:14:01.215 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:00 smithi149 ceph-mon[100764]: from='client.? 172.21.15.18:0/1042944687' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:14:02.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:01 smithi018 ceph-mon[95889]: pgmap v96: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 785 KiB/s rd, 1.1 MiB/s wr, 228 op/s 2023-09-23T20:14:02.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:01 smithi018 ceph-mon[109137]: pgmap v96: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 785 KiB/s rd, 1.1 MiB/s wr, 228 op/s 2023-09-23T20:14:02.215 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:01 smithi149 ceph-mon[100764]: pgmap v96: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 785 KiB/s rd, 1.1 MiB/s wr, 228 op/s 2023-09-23T20:14:03.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:03 smithi018 ceph-mon[95889]: from='client.? 172.21.15.18:0/834228484' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:14:03.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:03 smithi018 ceph-mon[109137]: from='client.? 172.21.15.18:0/834228484' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:14:03.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:03 smithi149 ceph-mon[100764]: from='client.? 172.21.15.18:0/834228484' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:14:04.216 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:14:03 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[102880]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:14:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T20:14:04.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:04 smithi018 ceph-mon[109137]: pgmap v97: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 785 KiB/s rd, 1.1 MiB/s wr, 228 op/s 2023-09-23T20:14:04.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:04 smithi018 ceph-mon[95889]: pgmap v97: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 785 KiB/s rd, 1.1 MiB/s wr, 228 op/s 2023-09-23T20:14:04.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:04 smithi149 ceph-mon[100764]: pgmap v97: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 785 KiB/s rd, 1.1 MiB/s wr, 228 op/s 2023-09-23T20:14:05.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:05 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:14:05.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:05 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:14:05.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:05 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:14:06.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:06 smithi018 ceph-mon[109137]: from='client.? 172.21.15.18:0/3246905038' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:14:06.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:06 smithi018 ceph-mon[109137]: pgmap v98: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 664 KiB/s rd, 960 KiB/s wr, 192 op/s 2023-09-23T20:14:06.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:06 smithi018 ceph-mon[95889]: from='client.? 172.21.15.18:0/3246905038' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:14:06.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:06 smithi018 ceph-mon[95889]: pgmap v98: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 664 KiB/s rd, 960 KiB/s wr, 192 op/s 2023-09-23T20:14:06.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:06 smithi149 ceph-mon[100764]: from='client.? 172.21.15.18:0/3246905038' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:14:06.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:06 smithi149 ceph-mon[100764]: pgmap v98: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 664 KiB/s rd, 960 KiB/s wr, 192 op/s 2023-09-23T20:14:09.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:08 smithi018 ceph-mon[109137]: pgmap v99: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:14:09.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:08 smithi018 ceph-mon[109137]: from='client.? 172.21.15.18:0/3314944106' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:14:09.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:08 smithi018 ceph-mon[95889]: pgmap v99: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:14:09.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:08 smithi018 ceph-mon[95889]: from='client.? 172.21.15.18:0/3314944106' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:14:09.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:08 smithi149 ceph-mon[100764]: pgmap v99: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:14:09.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:08 smithi149 ceph-mon[100764]: from='client.? 172.21.15.18:0/3314944106' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:14:09.641 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:14:09 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:14:09] "GET /metrics HTTP/1.1" 200 239628 "" "Prometheus/2.43.0" 2023-09-23T20:14:10.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:09 smithi018 ceph-mon[109137]: pgmap v100: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:14:10.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:09 smithi018 ceph-mon[95889]: pgmap v100: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:14:10.215 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:09 smithi149 ceph-mon[100764]: pgmap v100: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:14:11.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:10 smithi018 ceph-mon[109137]: from='client.? 172.21.15.18:0/3399291786' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:14:11.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:10 smithi018 ceph-mon[95889]: from='client.? 172.21.15.18:0/3399291786' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:14:11.215 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:10 smithi149 ceph-mon[100764]: from='client.? 172.21.15.18:0/3399291786' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:14:12.140 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:11 smithi018 ceph-mon[109137]: pgmap v101: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:14:12.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:11 smithi018 ceph-mon[95889]: pgmap v101: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:14:12.215 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:11 smithi149 ceph-mon[100764]: pgmap v101: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:14:13.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:12 smithi018 ceph-mon[109137]: from='client.? 172.21.15.18:0/4158416177' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:14:13.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:12 smithi018 ceph-mon[95889]: from='client.? 172.21.15.18:0/4158416177' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:14:13.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:12 smithi149 ceph-mon[100764]: from='client.? 172.21.15.18:0/4158416177' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:14:14.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:13 smithi018 ceph-mon[109137]: pgmap v102: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:14:14.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:13 smithi018 ceph-mon[95889]: pgmap v102: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:14:14.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:13 smithi149 ceph-mon[100764]: pgmap v102: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:14:14.216 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:14:13 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[102880]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:14:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T20:14:15.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:15 smithi018 ceph-mon[109137]: from='client.? 172.21.15.18:0/1043209031' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:14:15.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:15 smithi018 ceph-mon[95889]: from='client.? 172.21.15.18:0/1043209031' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:14:15.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:15 smithi149 ceph-mon[100764]: from='client.? 172.21.15.18:0/1043209031' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:14:16.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:16 smithi018 ceph-mon[109137]: pgmap v103: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:14:16.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:16 smithi018 ceph-mon[95889]: pgmap v103: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:14:16.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:16 smithi149 ceph-mon[100764]: pgmap v103: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:14:17.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:17 smithi149 ceph-mon[100764]: from='client.? 172.21.15.18:0/2693964623' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:14:18.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:17 smithi018 ceph-mon[109137]: from='client.? 172.21.15.18:0/2693964623' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:14:18.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:17 smithi018 ceph-mon[95889]: from='client.? 172.21.15.18:0/2693964623' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:14:18.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:18 smithi149 ceph-mon[100764]: pgmap v104: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:14:19.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:18 smithi018 ceph-mon[109137]: pgmap v104: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:14:19.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:18 smithi018 ceph-mon[95889]: pgmap v104: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:14:19.641 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:14:19 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:14:19] "GET /metrics HTTP/1.1" 200 239628 "" "Prometheus/2.43.0" 2023-09-23T20:14:20.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:19 smithi018 ceph-mon[109137]: pgmap v105: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:14:20.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:19 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:14:20.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:19 smithi018 ceph-mon[95889]: pgmap v105: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:14:20.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:19 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:14:20.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:19 smithi149 ceph-mon[100764]: pgmap v105: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:14:20.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:19 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:14:21.140 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:20 smithi018 ceph-mon[109137]: from='client.? 172.21.15.18:0/3798782627' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:14:21.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:20 smithi018 ceph-mon[95889]: from='client.? 172.21.15.18:0/3798782627' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:14:21.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:20 smithi149 ceph-mon[100764]: from='client.? 172.21.15.18:0/3798782627' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:14:22.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:21 smithi018 ceph-mon[109137]: pgmap v106: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:14:22.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:21 smithi018 ceph-mon[95889]: pgmap v106: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:14:22.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:21 smithi149 ceph-mon[100764]: pgmap v106: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:14:23.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:22 smithi018 ceph-mon[109137]: from='client.? 172.21.15.18:0/3935855699' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:14:23.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:22 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:14:23.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:22 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T20:14:23.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:22 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T20:14:23.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:22 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:14:23.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:22 smithi018 ceph-mon[95889]: from='client.? 172.21.15.18:0/3935855699' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:14:23.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:22 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:14:23.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:22 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T20:14:23.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:22 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T20:14:23.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:22 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:14:23.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:22 smithi149 ceph-mon[100764]: from='client.? 172.21.15.18:0/3935855699' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:14:23.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:22 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:14:23.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:22 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T20:14:23.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:22 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T20:14:23.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:22 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:14:24.104 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:23 smithi149 ceph-mon[100764]: Upgrade: Updating mgr.x 2023-09-23T20:14:24.104 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:23 smithi149 ceph-mon[100764]: Deploying daemon mgr.x on smithi149 2023-09-23T20:14:24.104 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:23 smithi149 ceph-mon[100764]: pgmap v107: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:14:24.105 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:14:23 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[102880]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:14:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T20:14:24.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:23 smithi018 ceph-mon[109137]: Upgrade: Updating mgr.x 2023-09-23T20:14:24.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:23 smithi018 ceph-mon[109137]: Deploying daemon mgr.x on smithi149 2023-09-23T20:14:24.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:23 smithi018 ceph-mon[109137]: pgmap v107: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:14:24.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:23 smithi018 ceph-mon[95889]: Upgrade: Updating mgr.x 2023-09-23T20:14:24.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:23 smithi018 ceph-mon[95889]: Deploying daemon mgr.x on smithi149 2023-09-23T20:14:24.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:23 smithi018 ceph-mon[95889]: pgmap v107: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:14:24.466 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:14:24 smithi149 systemd[1]: Stopping Ceph mgr.x for a3b3b17e-5a4c-11ee-8db4-212e2dc638e7... 2023-09-23T20:14:24.846 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:14:24 smithi149 bash[141787]: ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x 2023-09-23T20:14:24.847 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:14:24 smithi149 systemd[1]: ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7@mgr.x.service: Main process exited, code=exited, status=143/n/a 2023-09-23T20:14:25.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:24 smithi018 ceph-mon[109137]: from='client.? 172.21.15.18:0/3827281700' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:14:25.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:24 smithi018 ceph-mon[95889]: from='client.? 172.21.15.18:0/3827281700' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:14:25.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:24 smithi149 ceph-mon[100764]: from='client.? 172.21.15.18:0/3827281700' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:14:25.217 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:14:24 smithi149 systemd[1]: ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7@mgr.x.service: Failed with result 'exit-code'. 2023-09-23T20:14:25.217 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:14:24 smithi149 systemd[1]: Stopped Ceph mgr.x for a3b3b17e-5a4c-11ee-8db4-212e2dc638e7. 2023-09-23T20:14:25.217 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:14:24 smithi149 systemd[1]: Starting Ceph mgr.x for a3b3b17e-5a4c-11ee-8db4-212e2dc638e7... 2023-09-23T20:14:25.560 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:14:25 smithi149 podman[141931]: 2023-09-23T20:14:25.561 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:14:25 smithi149 bash[141931]: 65a82a8811ab57164d550bf5d4aa2a2dade5575c692aac93aebeaf06b7c1b4e3 2023-09-23T20:14:25.561 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:14:25 smithi149 ceph-mgr[141971]: -- 172.21.15.149:0/336141148 <== mon.1 v2:172.21.15.149:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x559115d3f1e0 con 0x559115d15000 2023-09-23T20:14:25.910 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:14:25 smithi149 systemd[1]: Started Ceph mgr.x for a3b3b17e-5a4c-11ee-8db4-212e2dc638e7. 2023-09-23T20:14:25.910 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:14:25 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[141947]: 2023-09-23T20:14:25.627+0000 7f34bb218200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-23T20:14:25.910 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:14:25 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[141947]: 2023-09-23T20:14:25.712+0000 7f34bb218200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T20:14:25.910 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:14:25 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[141947]: 2023-09-23T20:14:25.806+0000 7f34bb218200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T20:14:25.910 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:14:25 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[141947]: 2023-09-23T20:14:25.909+0000 7f34bb218200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T20:14:26.216 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:14:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[141947]: 2023-09-23T20:14:26.087+0000 7f34bb218200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T20:14:26.828 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:26 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:14:26.828 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:26 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:14:26.828 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:26 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:14:26.828 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:26 smithi149 ceph-mon[100764]: pgmap v108: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:14:26.828 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:14:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[141947]: 2023-09-23T20:14:26.498+0000 7f34bb218200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T20:14:26.828 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:14:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[141947]: 2023-09-23T20:14:26.641+0000 7f34bb218200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T20:14:26.829 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:14:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[141947]: 2023-09-23T20:14:26.731+0000 7f34bb218200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T20:14:26.829 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:14:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[141947]: 2023-09-23T20:14:26.827+0000 7f34bb218200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T20:14:26.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:26 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:14:26.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:26 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:14:26.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:26 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:14:26.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:26 smithi018 ceph-mon[109137]: pgmap v108: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:14:26.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:26 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:14:26.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:26 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:14:26.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:26 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:14:26.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:26 smithi018 ceph-mon[95889]: pgmap v108: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:14:27.612 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:14:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[141947]: 2023-09-23T20:14:27.260+0000 7f34bb218200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T20:14:27.890 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:27 smithi018 ceph-mon[109137]: from='client.? 172.21.15.18:0/4083056545' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:14:27.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:27 smithi018 ceph-mon[95889]: from='client.? 172.21.15.18:0/4083056545' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:14:27.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:27 smithi149 ceph-mon[100764]: from='client.? 172.21.15.18:0/4083056545' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:14:28.466 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:14:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[141947]: 2023-09-23T20:14:27.974+0000 7f34bb218200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T20:14:28.466 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:14:28 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[141947]: 2023-09-23T20:14:28.058+0000 7f34bb218200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T20:14:28.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:28 smithi018 ceph-mon[109137]: pgmap v109: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:14:28.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:28 smithi018 ceph-mon[95889]: pgmap v109: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:14:28.893 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:28 smithi149 ceph-mon[100764]: pgmap v109: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:14:29.161 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:14:28 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[141947]: 2023-09-23T20:14:28.893+0000 7f34bb218200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T20:14:29.161 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:14:28 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[141947]: 2023-09-23T20:14:28.986+0000 7f34bb218200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-23T20:14:29.448 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:14:29 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[141947]: 2023-09-23T20:14:29.160+0000 7f34bb218200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T20:14:29.640 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:14:29 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:14:29] "GET /metrics HTTP/1.1" 200 239576 "" "Prometheus/2.43.0" 2023-09-23T20:14:29.716 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:14:29 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[141947]: 2023-09-23T20:14:29.447+0000 7f34bb218200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-23T20:14:29.716 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:14:29 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[141947]: 2023-09-23T20:14:29.684+0000 7f34bb218200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T20:14:30.116 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:29 smithi149 ceph-mon[100764]: from='client.? 172.21.15.18:0/2824562937' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:14:30.116 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:29 smithi149 ceph-mon[100764]: pgmap v110: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:14:30.116 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:14:29 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[141947]: 2023-09-23T20:14:29.836+0000 7f34bb218200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T20:14:30.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:29 smithi018 ceph-mon[109137]: from='client.? 172.21.15.18:0/2824562937' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:14:30.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:29 smithi018 ceph-mon[109137]: pgmap v110: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:14:30.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:29 smithi018 ceph-mon[95889]: from='client.? 172.21.15.18:0/2824562937' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:14:30.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:29 smithi018 ceph-mon[95889]: pgmap v110: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:14:30.466 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:14:30 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[141947]: 2023-09-23T20:14:30.114+0000 7f34bb218200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T20:14:31.183 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:14:30 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[141947]: 2023-09-23T20:14:30.840+0000 7f34bb218200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T20:14:31.184 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:14:30 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[141947]: 2023-09-23T20:14:30.972+0000 7f34bb218200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T20:14:31.184 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:14:31 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[141947]: 2023-09-23T20:14:31.183+0000 7f34bb218200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T20:14:31.466 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:14:31 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[141947]: 2023-09-23T20:14:31.311+0000 7f34bb218200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T20:14:31.966 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:14:31 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[141947]: 2023-09-23T20:14:31.557+0000 7f34bb218200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T20:14:32.582 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:32 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:14:32.582 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:32 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:14:32.582 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:32 smithi149 ceph-mon[100764]: pgmap v111: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:14:32.582 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:32 smithi149 ceph-mon[100764]: from='client.? 172.21.15.18:0/1068815958' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:14:32.582 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:14:32 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[141947]: 2023-09-23T20:14:32.474+0000 7f34bb218200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T20:14:32.582 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:14:32 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[141947]: [23/Sep/2023:20:14:32] ENGINE Bus STARTING 2023-09-23T20:14:32.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:32 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:14:32.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:32 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:14:32.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:32 smithi018 ceph-mon[109137]: pgmap v111: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:14:32.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:32 smithi018 ceph-mon[109137]: from='client.? 172.21.15.18:0/1068815958' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:14:32.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:32 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:14:32.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:32 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:14:32.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:32 smithi018 ceph-mon[95889]: pgmap v111: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:14:32.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:32 smithi018 ceph-mon[95889]: from='client.? 172.21.15.18:0/1068815958' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:14:32.966 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:14:32 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[141947]: CherryPy Checker: 2023-09-23T20:14:32.966 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:14:32 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[141947]: The Application mounted at '' has an empty config. 2023-09-23T20:14:32.966 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:14:32 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[141947]: 2023-09-23T20:14:32.966 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:14:32 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[141947]: [23/Sep/2023:20:14:32] ENGINE Serving on http://:::9283 2023-09-23T20:14:32.966 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:14:32 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[141947]: [23/Sep/2023:20:14:32] ENGINE Bus STARTED 2023-09-23T20:14:33.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:33 smithi018 ceph-mon[109137]: Standby manager daemon x restarted 2023-09-23T20:14:33.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:33 smithi018 ceph-mon[109137]: Standby manager daemon x started 2023-09-23T20:14:33.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:33 smithi018 ceph-mon[109137]: from='mgr.? 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T20:14:33.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:33 smithi018 ceph-mon[109137]: from='mgr.? 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T20:14:33.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:33 smithi018 ceph-mon[109137]: from='mgr.? 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T20:14:33.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:33 smithi018 ceph-mon[109137]: from='mgr.? 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T20:14:33.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:33 smithi018 ceph-mon[95889]: Standby manager daemon x restarted 2023-09-23T20:14:33.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:33 smithi018 ceph-mon[95889]: Standby manager daemon x started 2023-09-23T20:14:33.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:33 smithi018 ceph-mon[95889]: from='mgr.? 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T20:14:33.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:33 smithi018 ceph-mon[95889]: from='mgr.? 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T20:14:33.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:33 smithi018 ceph-mon[95889]: from='mgr.? 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T20:14:33.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:33 smithi018 ceph-mon[95889]: from='mgr.? 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T20:14:33.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:33 smithi149 ceph-mon[100764]: Standby manager daemon x restarted 2023-09-23T20:14:33.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:33 smithi149 ceph-mon[100764]: Standby manager daemon x started 2023-09-23T20:14:33.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:33 smithi149 ceph-mon[100764]: from='mgr.? 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T20:14:33.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:33 smithi149 ceph-mon[100764]: from='mgr.? 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T20:14:33.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:33 smithi149 ceph-mon[100764]: from='mgr.? 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T20:14:33.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:33 smithi149 ceph-mon[100764]: from='mgr.? 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T20:14:34.216 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:14:33 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[141947]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:14:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T20:14:34.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:34 smithi018 ceph-mon[109137]: mgrmap e21: y(active, since 2m), standbys: x 2023-09-23T20:14:34.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:34 smithi018 ceph-mon[109137]: pgmap v112: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:14:34.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:34 smithi018 ceph-mon[95889]: mgrmap e21: y(active, since 2m), standbys: x 2023-09-23T20:14:34.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:34 smithi018 ceph-mon[95889]: pgmap v112: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:14:34.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:34 smithi149 ceph-mon[100764]: mgrmap e21: y(active, since 2m), standbys: x 2023-09-23T20:14:34.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:34 smithi149 ceph-mon[100764]: pgmap v112: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:14:35.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:35 smithi018 ceph-mon[109137]: from='client.? 172.21.15.18:0/2437765497' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:14:35.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:35 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:14:35.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:35 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:14:35.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:35 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:14:35.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:35 smithi018 ceph-mon[95889]: from='client.? 172.21.15.18:0/2437765497' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:14:35.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:35 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:14:35.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:35 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:14:35.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:35 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:14:35.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:35 smithi149 ceph-mon[100764]: from='client.? 172.21.15.18:0/2437765497' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:14:35.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:35 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:14:35.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:35 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:14:35.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:35 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:14:36.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:36 smithi018 ceph-mon[109137]: pgmap v113: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:14:36.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:36 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:14:36.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:36 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:14:36.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:36 smithi018 ceph-mon[95889]: pgmap v113: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:14:36.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:36 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:14:36.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:36 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:14:36.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:36 smithi149 ceph-mon[100764]: pgmap v113: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:14:36.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:36 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:14:36.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:36 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:14:37.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:37 smithi018 ceph-mon[109137]: from='client.? 172.21.15.18:0/2945193698' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:14:37.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:37 smithi018 ceph-mon[95889]: from='client.? 172.21.15.18:0/2945193698' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:14:37.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:37 smithi149 ceph-mon[100764]: from='client.? 172.21.15.18:0/2945193698' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:14:38.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:38 smithi018 ceph-mon[109137]: pgmap v114: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:14:38.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:38 smithi018 ceph-mon[95889]: pgmap v114: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:14:38.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:38 smithi149 ceph-mon[100764]: pgmap v114: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:14:39.142 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:14:38 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[141947]: [23/Sep/2023:20:14:38] ENGINE Bus STOPPING 2023-09-23T20:14:39.142 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:14:39 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[141947]: [23/Sep/2023:20:14:39] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T20:14:39.142 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:14:39 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[141947]: [23/Sep/2023:20:14:39] ENGINE Bus STOPPED 2023-09-23T20:14:39.142 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:14:39 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[141947]: [23/Sep/2023:20:14:39] ENGINE Bus STARTING 2023-09-23T20:14:39.275 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:14:38 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:14:38.979+0000 7faf088c0700 -1 mgr handle_mgr_map I was active but no longer am 2023-09-23T20:14:39.275 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:14:39 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: ignoring --setuser ceph since I am not root 2023-09-23T20:14:39.275 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:14:39 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: ignoring --setgroup ceph since I am not root 2023-09-23T20:14:39.275 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:14:39 smithi018 ceph-mgr[96157]: -- 172.21.15.18:0/2253454444 <== mon.2 v2:172.21.15.18:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55cb9a8aa9c0 con 0x55cb9a880c00 2023-09-23T20:14:39.275 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:14:39 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:14:39.176+0000 7f7a5ed051c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-23T20:14:39.276 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:14:39 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:14:39.273+0000 7f7a5ed051c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T20:14:39.466 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:14:39 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[141947]: 2023-09-23T20:14:39.144+0000 7f3415644700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T20:14:39.466 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:14:39 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[141947]: 2023-09-23T20:14:39.144+0000 7f3415644700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T20:14:39.466 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:14:39 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[141947]: 2023-09-23T20:14:39.144+0000 7f3415644700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T20:14:39.466 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:14:39 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[141947]: 2023-09-23T20:14:39.144+0000 7f3415644700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T20:14:39.467 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:14:39 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[141947]: 2023-09-23T20:14:39.144+0000 7f3415644700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T20:14:39.467 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:14:39 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[141947]: 2023-09-23T20:14:39.145+0000 7f341864a700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T20:14:39.467 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:14:39 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[141947]: 2023-09-23T20:14:39.145+0000 7f341864a700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T20:14:39.467 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:14:39 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[141947]: 2023-09-23T20:14:39.145+0000 7f341864a700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T20:14:39.467 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:14:39 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[141947]: 2023-09-23T20:14:39.145+0000 7f341864a700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T20:14:39.467 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:14:39 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[141947]: 2023-09-23T20:14:39.145+0000 7f341864a700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T20:14:39.467 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:14:39 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[141947]: [23/Sep/2023:20:14:39] ENGINE Serving on http://:::9283 2023-09-23T20:14:39.467 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:14:39 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[141947]: [23/Sep/2023:20:14:39] ENGINE Bus STARTED 2023-09-23T20:14:39.641 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:14:39 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:14:39.384+0000 7f7a5ed051c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T20:14:39.641 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:14:39 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:14:39.497+0000 7f7a5ed051c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T20:14:40.106 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:14:39 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:14:39.703+0000 7f7a5ed051c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T20:14:40.106 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:39 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:14:40.106 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:39 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:14:40.106 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:39 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:14:40.106 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:39 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:14:40.106 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:39 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:14:40.107 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:39 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:14:40.107 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:39 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-09-23T20:14:40.107 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:39 smithi018 ceph-mon[95889]: osdmap e88: 8 total, 8 up, 8 in 2023-09-23T20:14:40.107 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:39 smithi018 ceph-mon[95889]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2023-09-23T20:14:40.107 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:39 smithi018 ceph-mon[95889]: mgrmap e22: x(active, starting, since 0.00666562s) 2023-09-23T20:14:40.107 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:39 smithi018 ceph-mon[95889]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T20:14:40.107 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:39 smithi018 ceph-mon[95889]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:14:40.107 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:39 smithi018 ceph-mon[95889]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:14:40.107 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:39 smithi018 ceph-mon[95889]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi149.urtezf"}]: dispatch 2023-09-23T20:14:40.107 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:39 smithi018 ceph-mon[95889]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi018.hduiol"}]: dispatch 2023-09-23T20:14:40.107 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:39 smithi018 ceph-mon[95889]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T20:14:40.107 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:39 smithi018 ceph-mon[95889]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T20:14:40.108 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:39 smithi018 ceph-mon[95889]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T20:14:40.108 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:39 smithi018 ceph-mon[95889]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T20:14:40.108 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:39 smithi018 ceph-mon[95889]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T20:14:40.108 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:39 smithi018 ceph-mon[95889]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T20:14:40.108 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:39 smithi018 ceph-mon[95889]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T20:14:40.108 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:39 smithi018 ceph-mon[95889]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T20:14:40.108 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:39 smithi018 ceph-mon[95889]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T20:14:40.108 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:39 smithi018 ceph-mon[95889]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T20:14:40.108 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:39 smithi018 ceph-mon[95889]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T20:14:40.108 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:39 smithi018 ceph-mon[95889]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T20:14:40.108 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:39 smithi018 ceph-mon[95889]: Manager daemon x is now available 2023-09-23T20:14:40.108 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:39 smithi018 ceph-mon[95889]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:14:40.109 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:39 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:14:40.109 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:39 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:14:40.109 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:39 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:14:40.109 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:39 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:14:40.109 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:39 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:14:40.109 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:39 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:14:40.109 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:39 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-09-23T20:14:40.109 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:39 smithi018 ceph-mon[109137]: osdmap e88: 8 total, 8 up, 8 in 2023-09-23T20:14:40.109 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:39 smithi018 ceph-mon[109137]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2023-09-23T20:14:40.110 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:39 smithi018 ceph-mon[109137]: mgrmap e22: x(active, starting, since 0.00666562s) 2023-09-23T20:14:40.110 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:39 smithi018 ceph-mon[109137]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T20:14:40.110 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:39 smithi018 ceph-mon[109137]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:14:40.110 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:39 smithi018 ceph-mon[109137]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:14:40.110 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:39 smithi018 ceph-mon[109137]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi149.urtezf"}]: dispatch 2023-09-23T20:14:40.110 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:39 smithi018 ceph-mon[109137]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi018.hduiol"}]: dispatch 2023-09-23T20:14:40.110 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:39 smithi018 ceph-mon[109137]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T20:14:40.110 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:39 smithi018 ceph-mon[109137]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T20:14:40.110 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:39 smithi018 ceph-mon[109137]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T20:14:40.110 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:39 smithi018 ceph-mon[109137]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T20:14:40.110 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:39 smithi018 ceph-mon[109137]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T20:14:40.111 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:39 smithi018 ceph-mon[109137]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T20:14:40.111 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:39 smithi018 ceph-mon[109137]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T20:14:40.111 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:39 smithi018 ceph-mon[109137]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T20:14:40.111 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:39 smithi018 ceph-mon[109137]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T20:14:40.111 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:39 smithi018 ceph-mon[109137]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T20:14:40.111 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:39 smithi018 ceph-mon[109137]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T20:14:40.111 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:39 smithi018 ceph-mon[109137]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T20:14:40.111 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:39 smithi018 ceph-mon[109137]: Manager daemon x is now available 2023-09-23T20:14:40.112 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:39 smithi018 ceph-mon[95889]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:14:40.112 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:39 smithi018 ceph-mon[95889]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:14:40.112 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:39 smithi018 ceph-mon[95889]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:14:40.112 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:39 smithi018 ceph-mon[95889]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-09-23T20:14:40.112 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:39 smithi018 ceph-mon[95889]: from='mgr.25414 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-09-23T20:14:40.112 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:39 smithi018 ceph-mon[95889]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-09-23T20:14:40.112 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:39 smithi018 ceph-mon[95889]: from='mgr.25414 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-09-23T20:14:40.112 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:39 smithi018 ceph-mon[95889]: from='client.? 172.21.15.18:0/1082131447' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:14:40.112 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:39 smithi018 ceph-mon[95889]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:14:40.112 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:39 smithi018 ceph-mon[95889]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:14:40.113 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:39 smithi018 ceph-mon[95889]: Deploying cephadm binary to smithi018 2023-09-23T20:14:40.113 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:39 smithi018 ceph-mon[95889]: Deploying cephadm binary to smithi149 2023-09-23T20:14:40.113 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:39 smithi018 ceph-mon[109137]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:14:40.113 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:39 smithi018 ceph-mon[109137]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:14:40.113 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:39 smithi018 ceph-mon[109137]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:14:40.113 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:39 smithi018 ceph-mon[109137]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:14:40.114 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:39 smithi018 ceph-mon[109137]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-09-23T20:14:40.114 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:39 smithi018 ceph-mon[109137]: from='mgr.25414 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-09-23T20:14:40.114 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:39 smithi018 ceph-mon[109137]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-09-23T20:14:40.114 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:39 smithi018 ceph-mon[109137]: from='mgr.25414 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-09-23T20:14:40.114 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:39 smithi018 ceph-mon[109137]: from='client.? 172.21.15.18:0/1082131447' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:14:40.114 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:39 smithi018 ceph-mon[109137]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:14:40.114 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:39 smithi018 ceph-mon[109137]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:14:40.114 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:39 smithi018 ceph-mon[109137]: Deploying cephadm binary to smithi018 2023-09-23T20:14:40.114 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:39 smithi018 ceph-mon[109137]: Deploying cephadm binary to smithi149 2023-09-23T20:14:40.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:39 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:14:40.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:39 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:14:40.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:39 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:14:40.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:39 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:14:40.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:39 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' 2023-09-23T20:14:40.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:39 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:14:40.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:39 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-09-23T20:14:40.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:39 smithi149 ceph-mon[100764]: osdmap e88: 8 total, 8 up, 8 in 2023-09-23T20:14:40.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:39 smithi149 ceph-mon[100764]: from='mgr.14454 172.21.15.18:0/2228727492' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2023-09-23T20:14:40.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:39 smithi149 ceph-mon[100764]: mgrmap e22: x(active, starting, since 0.00666562s) 2023-09-23T20:14:40.218 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:39 smithi149 ceph-mon[100764]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T20:14:40.218 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:39 smithi149 ceph-mon[100764]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:14:40.218 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:39 smithi149 ceph-mon[100764]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:14:40.218 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:39 smithi149 ceph-mon[100764]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi149.urtezf"}]: dispatch 2023-09-23T20:14:40.218 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:39 smithi149 ceph-mon[100764]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi018.hduiol"}]: dispatch 2023-09-23T20:14:40.218 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:39 smithi149 ceph-mon[100764]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T20:14:40.218 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:39 smithi149 ceph-mon[100764]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T20:14:40.219 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:39 smithi149 ceph-mon[100764]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T20:14:40.219 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:39 smithi149 ceph-mon[100764]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T20:14:40.219 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:39 smithi149 ceph-mon[100764]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T20:14:40.219 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:39 smithi149 ceph-mon[100764]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T20:14:40.219 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:39 smithi149 ceph-mon[100764]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T20:14:40.219 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:39 smithi149 ceph-mon[100764]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T20:14:40.219 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:39 smithi149 ceph-mon[100764]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T20:14:40.220 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:39 smithi149 ceph-mon[100764]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T20:14:40.220 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:39 smithi149 ceph-mon[100764]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T20:14:40.220 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:39 smithi149 ceph-mon[100764]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T20:14:40.220 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:39 smithi149 ceph-mon[100764]: Manager daemon x is now available 2023-09-23T20:14:40.220 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:39 smithi149 ceph-mon[100764]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:14:40.220 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:39 smithi149 ceph-mon[100764]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:14:40.220 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:39 smithi149 ceph-mon[100764]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:14:40.220 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:39 smithi149 ceph-mon[100764]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:14:40.221 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:39 smithi149 ceph-mon[100764]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-09-23T20:14:40.221 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:39 smithi149 ceph-mon[100764]: from='mgr.25414 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-09-23T20:14:40.221 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:39 smithi149 ceph-mon[100764]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-09-23T20:14:40.221 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:39 smithi149 ceph-mon[100764]: from='mgr.25414 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-09-23T20:14:40.221 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:39 smithi149 ceph-mon[100764]: from='client.? 172.21.15.18:0/1082131447' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:14:40.221 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:39 smithi149 ceph-mon[100764]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:14:40.221 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:39 smithi149 ceph-mon[100764]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:14:40.222 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:39 smithi149 ceph-mon[100764]: Deploying cephadm binary to smithi018 2023-09-23T20:14:40.222 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:39 smithi149 ceph-mon[100764]: Deploying cephadm binary to smithi149 2023-09-23T20:14:40.363 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:14:40 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:14:40.105+0000 7f7a5ed051c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T20:14:40.363 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:14:40 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:14:40.212+0000 7f7a5ed051c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T20:14:40.640 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:14:40 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:14:40.362+0000 7f7a5ed051c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T20:14:40.641 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:14:40 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:14:40.475+0000 7f7a5ed051c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T20:14:41.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:40 smithi018 ceph-mon[109137]: [23/Sep/2023:20:14:39] ENGINE Bus STARTING 2023-09-23T20:14:41.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:40 smithi018 ceph-mon[109137]: mgrmap e23: x(active, since 1.00773s) 2023-09-23T20:14:41.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:40 smithi018 ceph-mon[109137]: pgmap v3: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:14:41.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:40 smithi018 ceph-mon[109137]: [23/Sep/2023:20:14:40] ENGINE Serving on http://172.21.15.149:8765 2023-09-23T20:14:41.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:40 smithi018 ceph-mon[109137]: [23/Sep/2023:20:14:40] ENGINE Serving on https://172.21.15.149:7150 2023-09-23T20:14:41.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:40 smithi018 ceph-mon[109137]: [23/Sep/2023:20:14:40] ENGINE Bus STARTED 2023-09-23T20:14:41.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:40 smithi018 ceph-mon[95889]: [23/Sep/2023:20:14:39] ENGINE Bus STARTING 2023-09-23T20:14:41.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:40 smithi018 ceph-mon[95889]: mgrmap e23: x(active, since 1.00773s) 2023-09-23T20:14:41.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:40 smithi018 ceph-mon[95889]: pgmap v3: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:14:41.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:40 smithi018 ceph-mon[95889]: [23/Sep/2023:20:14:40] ENGINE Serving on http://172.21.15.149:8765 2023-09-23T20:14:41.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:40 smithi018 ceph-mon[95889]: [23/Sep/2023:20:14:40] ENGINE Serving on https://172.21.15.149:7150 2023-09-23T20:14:41.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:40 smithi018 ceph-mon[95889]: [23/Sep/2023:20:14:40] ENGINE Bus STARTED 2023-09-23T20:14:41.392 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:14:40 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:14:40.986+0000 7f7a5ed051c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T20:14:41.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:40 smithi149 ceph-mon[100764]: [23/Sep/2023:20:14:39] ENGINE Bus STARTING 2023-09-23T20:14:41.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:40 smithi149 ceph-mon[100764]: mgrmap e23: x(active, since 1.00773s) 2023-09-23T20:14:41.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:40 smithi149 ceph-mon[100764]: pgmap v3: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:14:41.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:40 smithi149 ceph-mon[100764]: [23/Sep/2023:20:14:40] ENGINE Serving on http://172.21.15.149:8765 2023-09-23T20:14:41.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:40 smithi149 ceph-mon[100764]: [23/Sep/2023:20:14:40] ENGINE Serving on https://172.21.15.149:7150 2023-09-23T20:14:41.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:40 smithi149 ceph-mon[100764]: [23/Sep/2023:20:14:40] ENGINE Bus STARTED 2023-09-23T20:14:41.985 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:14:41 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:14:41.712+0000 7f7a5ed051c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T20:14:41.985 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:14:41 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:14:41.917+0000 7f7a5ed051c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T20:14:41.985 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:41 smithi018 ceph-mon[95889]: mgrmap e24: x(active, since 2s) 2023-09-23T20:14:41.985 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:41 smithi018 ceph-mon[95889]: pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:14:41.986 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:41 smithi018 ceph-mon[95889]: from='client.? 172.21.15.18:0/1086856550' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:14:42.390 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:41 smithi018 ceph-mon[109137]: mgrmap e24: x(active, since 2s) 2023-09-23T20:14:42.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:41 smithi018 ceph-mon[109137]: pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:14:42.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:41 smithi018 ceph-mon[109137]: from='client.? 172.21.15.18:0/1086856550' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:14:42.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:41 smithi149 ceph-mon[100764]: mgrmap e24: x(active, since 2s) 2023-09-23T20:14:42.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:41 smithi149 ceph-mon[100764]: pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:14:42.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:41 smithi149 ceph-mon[100764]: from='client.? 172.21.15.18:0/1086856550' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:14:43.266 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:14:42 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:14:42.916+0000 7f7a5ed051c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T20:14:43.266 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:14:43 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:14:43.039+0000 7f7a5ed051c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-23T20:14:43.266 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:14:43 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:14:43.265+0000 7f7a5ed051c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T20:14:43.891 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:14:43 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:14:43.629+0000 7f7a5ed051c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-23T20:14:43.891 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:14:43 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:14:43.736+0000 7f7a5ed051c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T20:14:44.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:44 smithi149 ceph-mon[100764]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:14:44.216 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:14:43 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[141947]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:14:43] "GET /metrics HTTP/1.1" 200 31015 "" "Prometheus/2.43.0" 2023-09-23T20:14:44.390 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:14:44 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:14:44.085+0000 7f7a5ed051c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T20:14:44.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:44 smithi018 ceph-mon[109137]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:14:44.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:44 smithi018 ceph-mon[95889]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:14:44.891 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:14:44 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:14:44.432+0000 7f7a5ed051c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T20:14:45.296 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:45 smithi018 ceph-mon[109137]: from='client.? 172.21.15.18:0/173242987' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:14:45.296 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:45 smithi018 ceph-mon[109137]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:14:45.297 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:45 smithi018 ceph-mon[109137]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:14:45.297 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:45 smithi018 ceph-mon[95889]: from='client.? 172.21.15.18:0/173242987' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:14:45.297 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:45 smithi018 ceph-mon[95889]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:14:45.297 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:45 smithi018 ceph-mon[95889]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:14:45.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:45 smithi149 ceph-mon[100764]: from='client.? 172.21.15.18:0/173242987' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:14:45.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:45 smithi149 ceph-mon[100764]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:14:45.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:45 smithi149 ceph-mon[100764]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:14:45.641 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:14:45 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:14:45.296+0000 7f7a5ed051c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T20:14:45.641 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:14:45 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:14:45.459+0000 7f7a5ed051c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T20:14:46.041 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:14:45 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:14:45.710+0000 7f7a5ed051c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T20:14:46.042 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:14:45 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:14:45.850+0000 7f7a5ed051c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T20:14:46.043 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:46 smithi018 ceph-mon[95889]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:14:46.043 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:46 smithi018 ceph-mon[95889]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:14:46.043 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:46 smithi018 ceph-mon[95889]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:14:46.043 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:46 smithi018 ceph-mon[95889]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:14:46.043 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:46 smithi018 ceph-mon[95889]: from='mgr.25414 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:14:46.043 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:46 smithi018 ceph-mon[95889]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:14:46.044 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:46 smithi018 ceph-mon[95889]: from='mgr.25414 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:14:46.044 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:46 smithi018 ceph-mon[95889]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:14:46.044 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:46 smithi018 ceph-mon[95889]: from='mgr.25414 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:14:46.044 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:46 smithi018 ceph-mon[95889]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:14:46.044 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:46 smithi018 ceph-mon[95889]: from='mgr.25414 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:14:46.044 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:46 smithi018 ceph-mon[95889]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:14:46.044 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:46 smithi018 ceph-mon[95889]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:14:46.044 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:46 smithi018 ceph-mon[95889]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:14:46.391 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:14:46 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:14:46.126+0000 7f7a5ed051c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T20:14:46.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:46 smithi018 ceph-mon[109137]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:14:46.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:46 smithi018 ceph-mon[109137]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:14:46.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:46 smithi018 ceph-mon[109137]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:14:46.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:46 smithi018 ceph-mon[109137]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:14:46.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:46 smithi018 ceph-mon[109137]: from='mgr.25414 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:14:46.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:46 smithi018 ceph-mon[109137]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:14:46.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:46 smithi018 ceph-mon[109137]: from='mgr.25414 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:14:46.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:46 smithi018 ceph-mon[109137]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:14:46.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:46 smithi018 ceph-mon[109137]: from='mgr.25414 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:14:46.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:46 smithi018 ceph-mon[109137]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:14:46.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:46 smithi018 ceph-mon[109137]: from='mgr.25414 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:14:46.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:46 smithi018 ceph-mon[109137]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:14:46.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:46 smithi018 ceph-mon[109137]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:14:46.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:46 smithi018 ceph-mon[109137]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:14:46.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:46 smithi149 ceph-mon[100764]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:14:46.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:46 smithi149 ceph-mon[100764]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:14:46.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:46 smithi149 ceph-mon[100764]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:14:46.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:46 smithi149 ceph-mon[100764]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:14:46.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:46 smithi149 ceph-mon[100764]: from='mgr.25414 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:14:46.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:46 smithi149 ceph-mon[100764]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:14:46.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:46 smithi149 ceph-mon[100764]: from='mgr.25414 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:14:46.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:46 smithi149 ceph-mon[100764]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:14:46.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:46 smithi149 ceph-mon[100764]: from='mgr.25414 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:14:46.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:46 smithi149 ceph-mon[100764]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:14:46.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:46 smithi149 ceph-mon[100764]: from='mgr.25414 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:14:46.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:46 smithi149 ceph-mon[100764]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:14:46.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:46 smithi149 ceph-mon[100764]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:14:46.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:46 smithi149 ceph-mon[100764]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:14:47.311 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:14:47 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:14:47.190+0000 7f7a5ed051c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T20:14:47.312 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:14:47 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: [23/Sep/2023:20:14:47] ENGINE Bus STARTING 2023-09-23T20:14:47.312 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:47 smithi018 ceph-mon[95889]: Adjusting osd_memory_target on smithi018 to 2501M 2023-09-23T20:14:47.312 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:47 smithi018 ceph-mon[95889]: from='client.? 172.21.15.18:0/4147592509' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:14:47.312 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:47 smithi018 ceph-mon[95889]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:14:47.312 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:47 smithi018 ceph-mon[95889]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:14:47.312 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:47 smithi018 ceph-mon[109137]: Adjusting osd_memory_target on smithi018 to 2501M 2023-09-23T20:14:47.313 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:47 smithi018 ceph-mon[109137]: from='client.? 172.21.15.18:0/4147592509' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:14:47.313 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:47 smithi018 ceph-mon[109137]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:14:47.313 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:47 smithi018 ceph-mon[109137]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:14:47.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:47 smithi149 ceph-mon[100764]: Adjusting osd_memory_target on smithi018 to 2501M 2023-09-23T20:14:47.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:47 smithi149 ceph-mon[100764]: from='client.? 172.21.15.18:0/4147592509' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:14:47.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:47 smithi149 ceph-mon[100764]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:14:47.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:47 smithi149 ceph-mon[100764]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:14:47.641 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:14:47 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: CherryPy Checker: 2023-09-23T20:14:47.641 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:14:47 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: The Application mounted at '' has an empty config. 2023-09-23T20:14:47.641 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:14:47 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: 2023-09-23T20:14:47.641 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:14:47 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: [23/Sep/2023:20:14:47] ENGINE Serving on http://:::9283 2023-09-23T20:14:47.641 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:14:47 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: [23/Sep/2023:20:14:47] ENGINE Bus STARTED 2023-09-23T20:14:48.079 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:48 smithi149 ceph-mon[100764]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:14:48.079 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:48 smithi149 ceph-mon[100764]: Standby manager daemon y started 2023-09-23T20:14:48.079 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:48 smithi149 ceph-mon[100764]: from='mgr.? 172.21.15.18:0/1940500973' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-09-23T20:14:48.079 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:48 smithi149 ceph-mon[100764]: from='mgr.? 172.21.15.18:0/1940500973' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T20:14:48.079 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:48 smithi149 ceph-mon[100764]: from='mgr.? 172.21.15.18:0/1940500973' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-09-23T20:14:48.079 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:48 smithi149 ceph-mon[100764]: from='mgr.? 172.21.15.18:0/1940500973' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T20:14:48.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:48 smithi018 ceph-mon[95889]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:14:48.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:48 smithi018 ceph-mon[95889]: Standby manager daemon y started 2023-09-23T20:14:48.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:48 smithi018 ceph-mon[95889]: from='mgr.? 172.21.15.18:0/1940500973' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-09-23T20:14:48.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:48 smithi018 ceph-mon[95889]: from='mgr.? 172.21.15.18:0/1940500973' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T20:14:48.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:48 smithi018 ceph-mon[95889]: from='mgr.? 172.21.15.18:0/1940500973' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-09-23T20:14:48.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:48 smithi018 ceph-mon[95889]: from='mgr.? 172.21.15.18:0/1940500973' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T20:14:48.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:48 smithi018 ceph-mon[109137]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:14:48.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:48 smithi018 ceph-mon[109137]: Standby manager daemon y started 2023-09-23T20:14:48.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:48 smithi018 ceph-mon[109137]: from='mgr.? 172.21.15.18:0/1940500973' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-09-23T20:14:48.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:48 smithi018 ceph-mon[109137]: from='mgr.? 172.21.15.18:0/1940500973' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T20:14:48.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:48 smithi018 ceph-mon[109137]: from='mgr.? 172.21.15.18:0/1940500973' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-09-23T20:14:48.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:48 smithi018 ceph-mon[109137]: from='mgr.? 172.21.15.18:0/1940500973' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T20:14:49.364 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:49 smithi149 ceph-mon[100764]: mgrmap e25: x(active, since 9s), standbys: y 2023-09-23T20:14:49.364 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:49 smithi149 ceph-mon[100764]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T20:14:49.391 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:14:49 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[96133]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:14:49] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T20:14:49.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:49 smithi018 ceph-mon[95889]: mgrmap e25: x(active, since 9s), standbys: y 2023-09-23T20:14:49.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:49 smithi018 ceph-mon[95889]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T20:14:49.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:49 smithi018 ceph-mon[109137]: mgrmap e25: x(active, since 9s), standbys: y 2023-09-23T20:14:49.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:49 smithi018 ceph-mon[109137]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T20:14:50.390 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:50 smithi018 ceph-mon[109137]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:14:50.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:50 smithi018 ceph-mon[109137]: from='client.? 172.21.15.18:0/2612930810' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:14:50.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:50 smithi018 ceph-mon[109137]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:14:50.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:50 smithi018 ceph-mon[109137]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:14:50.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:50 smithi018 ceph-mon[109137]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd/host:smithi149", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:14:50.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:50 smithi018 ceph-mon[109137]: from='mgr.25414 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd/host:smithi149", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:14:50.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:50 smithi018 ceph-mon[109137]: from='mgr.25414 ' entity='mgr.x' cmd='[{"prefix": "config rm", "who": "osd/host:smithi149", "name": "osd_memory_target"}]': finished 2023-09-23T20:14:50.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:50 smithi018 ceph-mon[109137]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:14:50.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:50 smithi018 ceph-mon[109137]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:14:50.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:50 smithi018 ceph-mon[95889]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:14:50.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:50 smithi018 ceph-mon[95889]: from='client.? 172.21.15.18:0/2612930810' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:14:50.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:50 smithi018 ceph-mon[95889]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:14:50.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:50 smithi018 ceph-mon[95889]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:14:50.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:50 smithi018 ceph-mon[95889]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd/host:smithi149", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:14:50.394 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:50 smithi018 ceph-mon[95889]: from='mgr.25414 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd/host:smithi149", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:14:50.394 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:50 smithi018 ceph-mon[95889]: from='mgr.25414 ' entity='mgr.x' cmd='[{"prefix": "config rm", "who": "osd/host:smithi149", "name": "osd_memory_target"}]': finished 2023-09-23T20:14:50.394 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:50 smithi018 ceph-mon[95889]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:14:50.394 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:50 smithi018 ceph-mon[95889]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:14:50.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:50 smithi149 ceph-mon[100764]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:14:50.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:50 smithi149 ceph-mon[100764]: from='client.? 172.21.15.18:0/2612930810' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:14:50.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:50 smithi149 ceph-mon[100764]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:14:50.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:50 smithi149 ceph-mon[100764]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:14:50.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:50 smithi149 ceph-mon[100764]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd/host:smithi149", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:14:50.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:50 smithi149 ceph-mon[100764]: from='mgr.25414 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd/host:smithi149", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:14:50.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:50 smithi149 ceph-mon[100764]: from='mgr.25414 ' entity='mgr.x' cmd='[{"prefix": "config rm", "who": "osd/host:smithi149", "name": "osd_memory_target"}]': finished 2023-09-23T20:14:50.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:50 smithi149 ceph-mon[100764]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:14:50.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:50 smithi149 ceph-mon[100764]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:14:51.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:51 smithi018 ceph-mon[109137]: Updating smithi018:/etc/ceph/ceph.conf 2023-09-23T20:14:51.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:51 smithi018 ceph-mon[109137]: Updating smithi149:/etc/ceph/ceph.conf 2023-09-23T20:14:51.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:51 smithi018 ceph-mon[109137]: Updating smithi149:/var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T20:14:51.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:51 smithi018 ceph-mon[109137]: Updating smithi018:/var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T20:14:51.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:51 smithi018 ceph-mon[109137]: Updating smithi149:/etc/ceph/ceph.client.admin.keyring 2023-09-23T20:14:51.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:51 smithi018 ceph-mon[109137]: Updating smithi018:/etc/ceph/ceph.client.admin.keyring 2023-09-23T20:14:51.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:51 smithi018 ceph-mon[109137]: Updating smithi149:/var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T20:14:51.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:51 smithi018 ceph-mon[109137]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:14:51.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:51 smithi018 ceph-mon[109137]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:14:51.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:51 smithi018 ceph-mon[109137]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:14:51.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:51 smithi018 ceph-mon[109137]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:14:51.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:51 smithi018 ceph-mon[109137]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:14:51.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:51 smithi018 ceph-mon[109137]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:14:51.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:51 smithi018 ceph-mon[109137]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:14:51.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:51 smithi018 ceph-mon[109137]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:14:51.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:51 smithi018 ceph-mon[109137]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:14:51.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:51 smithi018 ceph-mon[95889]: Updating smithi018:/etc/ceph/ceph.conf 2023-09-23T20:14:51.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:51 smithi018 ceph-mon[95889]: Updating smithi149:/etc/ceph/ceph.conf 2023-09-23T20:14:51.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:51 smithi018 ceph-mon[95889]: Updating smithi149:/var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T20:14:51.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:51 smithi018 ceph-mon[95889]: Updating smithi018:/var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T20:14:51.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:51 smithi018 ceph-mon[95889]: Updating smithi149:/etc/ceph/ceph.client.admin.keyring 2023-09-23T20:14:51.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:51 smithi018 ceph-mon[95889]: Updating smithi018:/etc/ceph/ceph.client.admin.keyring 2023-09-23T20:14:51.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:51 smithi018 ceph-mon[95889]: Updating smithi149:/var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T20:14:51.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:51 smithi018 ceph-mon[95889]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:14:51.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:51 smithi018 ceph-mon[95889]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:14:51.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:51 smithi018 ceph-mon[95889]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:14:51.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:51 smithi018 ceph-mon[95889]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:14:51.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:51 smithi018 ceph-mon[95889]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:14:51.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:51 smithi018 ceph-mon[95889]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:14:51.394 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:51 smithi018 ceph-mon[95889]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:14:51.394 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:51 smithi018 ceph-mon[95889]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:14:51.394 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:51 smithi018 ceph-mon[95889]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:14:51.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:51 smithi149 ceph-mon[100764]: Updating smithi018:/etc/ceph/ceph.conf 2023-09-23T20:14:51.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:51 smithi149 ceph-mon[100764]: Updating smithi149:/etc/ceph/ceph.conf 2023-09-23T20:14:51.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:51 smithi149 ceph-mon[100764]: Updating smithi149:/var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T20:14:51.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:51 smithi149 ceph-mon[100764]: Updating smithi018:/var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T20:14:51.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:51 smithi149 ceph-mon[100764]: Updating smithi149:/etc/ceph/ceph.client.admin.keyring 2023-09-23T20:14:51.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:51 smithi149 ceph-mon[100764]: Updating smithi018:/etc/ceph/ceph.client.admin.keyring 2023-09-23T20:14:51.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:51 smithi149 ceph-mon[100764]: Updating smithi149:/var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T20:14:51.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:51 smithi149 ceph-mon[100764]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:14:51.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:51 smithi149 ceph-mon[100764]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:14:51.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:51 smithi149 ceph-mon[100764]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:14:51.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:51 smithi149 ceph-mon[100764]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:14:51.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:51 smithi149 ceph-mon[100764]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:14:51.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:51 smithi149 ceph-mon[100764]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:14:51.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:51 smithi149 ceph-mon[100764]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:14:51.468 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:51 smithi149 ceph-mon[100764]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:14:51.468 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:51 smithi149 ceph-mon[100764]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:14:52.307 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:52 smithi018 ceph-mon[109137]: Updating smithi018:/var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T20:14:52.308 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:52 smithi018 ceph-mon[109137]: Reconfiguring alertmanager.a (dependencies changed)... 2023-09-23T20:14:52.308 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:52 smithi018 ceph-mon[109137]: Reconfiguring daemon alertmanager.a on smithi018 2023-09-23T20:14:52.308 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:52 smithi018 ceph-mon[109137]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:14:52.308 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:52 smithi018 ceph-mon[109137]: from='client.? 172.21.15.18:0/3876468890' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:14:52.308 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:52 smithi018 ceph-mon[95889]: Updating smithi018:/var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T20:14:52.308 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:52 smithi018 ceph-mon[95889]: Reconfiguring alertmanager.a (dependencies changed)... 2023-09-23T20:14:52.308 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:52 smithi018 ceph-mon[95889]: Reconfiguring daemon alertmanager.a on smithi018 2023-09-23T20:14:52.308 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:52 smithi018 ceph-mon[95889]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:14:52.309 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:52 smithi018 ceph-mon[95889]: from='client.? 172.21.15.18:0/3876468890' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:14:52.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:52 smithi149 ceph-mon[100764]: Updating smithi018:/var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T20:14:52.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:52 smithi149 ceph-mon[100764]: Reconfiguring alertmanager.a (dependencies changed)... 2023-09-23T20:14:52.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:52 smithi149 ceph-mon[100764]: Reconfiguring daemon alertmanager.a on smithi018 2023-09-23T20:14:52.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:52 smithi149 ceph-mon[100764]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:14:52.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:52 smithi149 ceph-mon[100764]: from='client.? 172.21.15.18:0/3876468890' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:14:53.081 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 23 20:14:52 smithi018 systemd[1]: Stopping Ceph alertmanager.a for a3b3b17e-5a4c-11ee-8db4-212e2dc638e7... 2023-09-23T20:14:53.331 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 23 20:14:53 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-alertmanager-a[151406]: ts=2023-09-23T20:14:53.080Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2023-09-23T20:14:53.332 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 23 20:14:53 smithi018 bash[166604]: ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-alertmanager-a 2023-09-23T20:14:53.981 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 23 20:14:53 smithi018 systemd[1]: ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7@alertmanager.a.service: Succeeded. 2023-09-23T20:14:53.981 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 23 20:14:53 smithi018 systemd[1]: Stopped Ceph alertmanager.a for a3b3b17e-5a4c-11ee-8db4-212e2dc638e7. 2023-09-23T20:14:53.981 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 23 20:14:53 smithi018 systemd[1]: Starting Ceph alertmanager.a for a3b3b17e-5a4c-11ee-8db4-212e2dc638e7... 2023-09-23T20:14:54.056 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:14:53 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[141947]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:14:53] "GET /metrics HTTP/1.1" 200 31015 "" "Prometheus/2.43.0" 2023-09-23T20:14:54.232 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:54 smithi018 ceph-mon[109137]: pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:14:54.232 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:54 smithi018 ceph-mon[109137]: from='client.? 172.21.15.18:0/3078525219' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:14:54.232 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 23 20:14:53 smithi018 podman[166761]: 2023-09-23T20:14:54.232 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 23 20:14:54 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-alertmanager-a[166780]: ts=2023-09-23T20:14:54.230Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2023-09-23T20:14:54.232 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 23 20:14:54 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-alertmanager-a[166780]: ts=2023-09-23T20:14:54.230Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2023-09-23T20:14:54.232 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 23 20:14:54 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-alertmanager-a[166780]: ts=2023-09-23T20:14:54.231Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.18 port=9094 2023-09-23T20:14:54.232 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 23 20:14:54 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-alertmanager-a[166780]: ts=2023-09-23T20:14:54.232Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2023-09-23T20:14:54.233 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:54 smithi018 ceph-mon[95889]: pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:14:54.233 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:54 smithi018 ceph-mon[95889]: from='client.? 172.21.15.18:0/3078525219' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:14:54.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:54 smithi149 ceph-mon[100764]: pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:14:54.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:54 smithi149 ceph-mon[100764]: from='client.? 172.21.15.18:0/3078525219' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:14:54.641 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 23 20:14:54 smithi018 bash[166761]: 57241c9344be655c7093a7a4c4684c5c42802fb74b4d03a6413a61125270a77c 2023-09-23T20:14:54.641 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 23 20:14:54 smithi018 systemd[1]: Started Ceph alertmanager.a for a3b3b17e-5a4c-11ee-8db4-212e2dc638e7. 2023-09-23T20:14:54.641 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 23 20:14:54 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-alertmanager-a[166780]: ts=2023-09-23T20:14:54.261Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2023-09-23T20:14:54.641 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 23 20:14:54 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-alertmanager-a[166780]: ts=2023-09-23T20:14:54.261Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2023-09-23T20:14:54.641 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 23 20:14:54 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-alertmanager-a[166780]: ts=2023-09-23T20:14:54.264Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2023-09-23T20:14:54.641 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 23 20:14:54 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-alertmanager-a[166780]: ts=2023-09-23T20:14:54.264Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2023-09-23T20:14:55.058 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:14:54 smithi018 systemd[1]: Stopping Ceph node-exporter.a for a3b3b17e-5a4c-11ee-8db4-212e2dc638e7... 2023-09-23T20:14:55.361 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:55 smithi018 ceph-mon[109137]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:14:55.361 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:55 smithi018 ceph-mon[109137]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:14:55.362 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:55 smithi018 ceph-mon[109137]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:14:55.362 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:55 smithi018 ceph-mon[109137]: Reconfiguring node-exporter.a (dependencies changed)... 2023-09-23T20:14:55.362 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:55 smithi018 ceph-mon[109137]: Reconfiguring daemon node-exporter.a on smithi018 2023-09-23T20:14:55.362 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:14:55 smithi018 bash[166939]: ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-a 2023-09-23T20:14:55.362 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:14:55 smithi018 systemd[1]: ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2023-09-23T20:14:55.362 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:55 smithi018 ceph-mon[95889]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:14:55.362 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:55 smithi018 ceph-mon[95889]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:14:55.363 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:55 smithi018 ceph-mon[95889]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:14:55.363 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:55 smithi018 ceph-mon[95889]: Reconfiguring node-exporter.a (dependencies changed)... 2023-09-23T20:14:55.363 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:55 smithi018 ceph-mon[95889]: Reconfiguring daemon node-exporter.a on smithi018 2023-09-23T20:14:55.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:55 smithi149 ceph-mon[100764]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:14:55.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:55 smithi149 ceph-mon[100764]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:14:55.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:55 smithi149 ceph-mon[100764]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:14:55.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:55 smithi149 ceph-mon[100764]: Reconfiguring node-exporter.a (dependencies changed)... 2023-09-23T20:14:55.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:55 smithi149 ceph-mon[100764]: Reconfiguring daemon node-exporter.a on smithi018 2023-09-23T20:14:55.641 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:14:55 smithi018 systemd[1]: ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7@node-exporter.a.service: Failed with result 'exit-code'. 2023-09-23T20:14:55.641 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:14:55 smithi018 systemd[1]: Stopped Ceph node-exporter.a for a3b3b17e-5a4c-11ee-8db4-212e2dc638e7. 2023-09-23T20:14:55.641 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:14:55 smithi018 systemd[1]: Starting Ceph node-exporter.a for a3b3b17e-5a4c-11ee-8db4-212e2dc638e7... 2023-09-23T20:14:56.141 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:14:55 smithi018 podman[167061]: 2023-09-23T20:14:56.141 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:14:56 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-a[167076]: ts=2023-09-23T20:14:56.079Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2023-09-23T20:14:56.141 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:14:56 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-a[167076]: ts=2023-09-23T20:14:56.079Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2023-09-23T20:14:56.141 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:14:56 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-a[167076]: ts=2023-09-23T20:14:56.080Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2023-09-23T20:14:56.142 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:14:56 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-a[167076]: ts=2023-09-23T20:14:56.080Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2023-09-23T20:14:56.142 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:14:56 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-a[167076]: ts=2023-09-23T20:14:56.080Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2023-09-23T20:14:56.142 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:14:56 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-a[167076]: ts=2023-09-23T20:14:56.080Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2023-09-23T20:14:56.142 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:14:56 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-a[167076]: ts=2023-09-23T20:14:56.081Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2023-09-23T20:14:56.142 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:14:56 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-a[167076]: ts=2023-09-23T20:14:56.081Z caller=node_exporter.go:117 level=info collector=arp 2023-09-23T20:14:56.142 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:14:56 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-a[167076]: ts=2023-09-23T20:14:56.081Z caller=node_exporter.go:117 level=info collector=bcache 2023-09-23T20:14:56.142 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:14:56 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-a[167076]: ts=2023-09-23T20:14:56.081Z caller=node_exporter.go:117 level=info collector=bonding 2023-09-23T20:14:56.142 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:14:56 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-a[167076]: ts=2023-09-23T20:14:56.081Z caller=node_exporter.go:117 level=info collector=btrfs 2023-09-23T20:14:56.142 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:14:56 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-a[167076]: ts=2023-09-23T20:14:56.081Z caller=node_exporter.go:117 level=info collector=conntrack 2023-09-23T20:14:56.142 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:14:56 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-a[167076]: ts=2023-09-23T20:14:56.081Z caller=node_exporter.go:117 level=info collector=cpu 2023-09-23T20:14:56.143 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:14:56 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-a[167076]: ts=2023-09-23T20:14:56.081Z caller=node_exporter.go:117 level=info collector=cpufreq 2023-09-23T20:14:56.143 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:14:56 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-a[167076]: ts=2023-09-23T20:14:56.081Z caller=node_exporter.go:117 level=info collector=diskstats 2023-09-23T20:14:56.143 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:14:56 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-a[167076]: ts=2023-09-23T20:14:56.081Z caller=node_exporter.go:117 level=info collector=dmi 2023-09-23T20:14:56.143 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:14:56 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-a[167076]: ts=2023-09-23T20:14:56.081Z caller=node_exporter.go:117 level=info collector=edac 2023-09-23T20:14:56.143 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:14:56 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-a[167076]: ts=2023-09-23T20:14:56.081Z caller=node_exporter.go:117 level=info collector=entropy 2023-09-23T20:14:56.143 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:14:56 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-a[167076]: ts=2023-09-23T20:14:56.081Z caller=node_exporter.go:117 level=info collector=fibrechannel 2023-09-23T20:14:56.143 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:14:56 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-a[167076]: ts=2023-09-23T20:14:56.081Z caller=node_exporter.go:117 level=info collector=filefd 2023-09-23T20:14:56.143 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:14:56 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-a[167076]: ts=2023-09-23T20:14:56.081Z caller=node_exporter.go:117 level=info collector=filesystem 2023-09-23T20:14:56.143 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:14:56 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-a[167076]: ts=2023-09-23T20:14:56.081Z caller=node_exporter.go:117 level=info collector=hwmon 2023-09-23T20:14:56.144 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:14:56 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-a[167076]: ts=2023-09-23T20:14:56.081Z caller=node_exporter.go:117 level=info collector=infiniband 2023-09-23T20:14:56.144 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:14:56 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-a[167076]: ts=2023-09-23T20:14:56.081Z caller=node_exporter.go:117 level=info collector=ipvs 2023-09-23T20:14:56.144 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:14:56 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-a[167076]: ts=2023-09-23T20:14:56.081Z caller=node_exporter.go:117 level=info collector=loadavg 2023-09-23T20:14:56.144 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:14:56 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-a[167076]: ts=2023-09-23T20:14:56.081Z caller=node_exporter.go:117 level=info collector=mdadm 2023-09-23T20:14:56.144 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:14:56 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-a[167076]: ts=2023-09-23T20:14:56.081Z caller=node_exporter.go:117 level=info collector=meminfo 2023-09-23T20:14:56.144 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:14:56 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-a[167076]: ts=2023-09-23T20:14:56.081Z caller=node_exporter.go:117 level=info collector=netclass 2023-09-23T20:14:56.144 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:14:56 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-a[167076]: ts=2023-09-23T20:14:56.081Z caller=node_exporter.go:117 level=info collector=netdev 2023-09-23T20:14:56.144 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:14:56 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-a[167076]: ts=2023-09-23T20:14:56.081Z caller=node_exporter.go:117 level=info collector=netstat 2023-09-23T20:14:56.144 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:14:56 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-a[167076]: ts=2023-09-23T20:14:56.081Z caller=node_exporter.go:117 level=info collector=nfs 2023-09-23T20:14:56.144 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:14:56 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-a[167076]: ts=2023-09-23T20:14:56.081Z caller=node_exporter.go:117 level=info collector=nfsd 2023-09-23T20:14:56.144 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:14:56 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-a[167076]: ts=2023-09-23T20:14:56.081Z caller=node_exporter.go:117 level=info collector=nvme 2023-09-23T20:14:56.144 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:14:56 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-a[167076]: ts=2023-09-23T20:14:56.081Z caller=node_exporter.go:117 level=info collector=os 2023-09-23T20:14:56.145 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:14:56 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-a[167076]: ts=2023-09-23T20:14:56.081Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2023-09-23T20:14:56.145 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:14:56 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-a[167076]: ts=2023-09-23T20:14:56.081Z caller=node_exporter.go:117 level=info collector=pressure 2023-09-23T20:14:56.145 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:14:56 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-a[167076]: ts=2023-09-23T20:14:56.081Z caller=node_exporter.go:117 level=info collector=rapl 2023-09-23T20:14:56.145 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:14:56 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-a[167076]: ts=2023-09-23T20:14:56.081Z caller=node_exporter.go:117 level=info collector=schedstat 2023-09-23T20:14:56.145 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:14:56 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-a[167076]: ts=2023-09-23T20:14:56.081Z caller=node_exporter.go:117 level=info collector=selinux 2023-09-23T20:14:56.145 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:14:56 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-a[167076]: ts=2023-09-23T20:14:56.081Z caller=node_exporter.go:117 level=info collector=sockstat 2023-09-23T20:14:56.145 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:14:56 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-a[167076]: ts=2023-09-23T20:14:56.081Z caller=node_exporter.go:117 level=info collector=softnet 2023-09-23T20:14:56.145 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:14:56 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-a[167076]: ts=2023-09-23T20:14:56.081Z caller=node_exporter.go:117 level=info collector=stat 2023-09-23T20:14:56.145 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:14:56 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-a[167076]: ts=2023-09-23T20:14:56.081Z caller=node_exporter.go:117 level=info collector=tapestats 2023-09-23T20:14:56.145 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:14:56 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-a[167076]: ts=2023-09-23T20:14:56.081Z caller=node_exporter.go:117 level=info collector=textfile 2023-09-23T20:14:56.146 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:14:56 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-a[167076]: ts=2023-09-23T20:14:56.081Z caller=node_exporter.go:117 level=info collector=thermal_zone 2023-09-23T20:14:56.146 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:14:56 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-a[167076]: ts=2023-09-23T20:14:56.081Z caller=node_exporter.go:117 level=info collector=time 2023-09-23T20:14:56.146 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:14:56 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-a[167076]: ts=2023-09-23T20:14:56.081Z caller=node_exporter.go:117 level=info collector=udp_queues 2023-09-23T20:14:56.146 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:14:56 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-a[167076]: ts=2023-09-23T20:14:56.081Z caller=node_exporter.go:117 level=info collector=uname 2023-09-23T20:14:56.146 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:14:56 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-a[167076]: ts=2023-09-23T20:14:56.081Z caller=node_exporter.go:117 level=info collector=vmstat 2023-09-23T20:14:56.146 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:14:56 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-a[167076]: ts=2023-09-23T20:14:56.081Z caller=node_exporter.go:117 level=info collector=xfs 2023-09-23T20:14:56.146 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:14:56 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-a[167076]: ts=2023-09-23T20:14:56.081Z caller=node_exporter.go:117 level=info collector=zfs 2023-09-23T20:14:56.146 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:14:56 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-a[167076]: ts=2023-09-23T20:14:56.082Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2023-09-23T20:14:56.146 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:14:56 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-a[167076]: ts=2023-09-23T20:14:56.082Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2023-09-23T20:14:56.146 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:14:56 smithi018 bash[167061]: 500f76739e6a50f0f52b22defc8dd9795faff765f8946b3f3e44e11b5c238a2a 2023-09-23T20:14:56.641 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:14:56 smithi018 systemd[1]: Started Ceph node-exporter.a for a3b3b17e-5a4c-11ee-8db4-212e2dc638e7. 2023-09-23T20:14:56.641 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 23 20:14:56 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-alertmanager-a[166780]: ts=2023-09-23T20:14:56.232Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.00016655s 2023-09-23T20:14:56.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:56 smithi018 ceph-mon[95889]: pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:14:56.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:56 smithi018 ceph-mon[95889]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:14:56.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:56 smithi018 ceph-mon[95889]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:14:56.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:56 smithi018 ceph-mon[109137]: pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:14:56.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:56 smithi018 ceph-mon[109137]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:14:56.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:56 smithi018 ceph-mon[109137]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:14:56.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:56 smithi149 ceph-mon[100764]: pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:14:56.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:56 smithi149 ceph-mon[100764]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:14:56.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:56 smithi149 ceph-mon[100764]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:14:57.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:57 smithi018 ceph-mon[95889]: Reconfiguring prometheus.a (dependencies changed)... 2023-09-23T20:14:57.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:57 smithi018 ceph-mon[95889]: from='client.? 172.21.15.18:0/3779561052' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:14:57.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:57 smithi018 ceph-mon[95889]: Reconfiguring daemon prometheus.a on smithi149 2023-09-23T20:14:57.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:57 smithi018 ceph-mon[109137]: Reconfiguring prometheus.a (dependencies changed)... 2023-09-23T20:14:57.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:57 smithi018 ceph-mon[109137]: from='client.? 172.21.15.18:0/3779561052' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:14:57.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:57 smithi018 ceph-mon[109137]: Reconfiguring daemon prometheus.a on smithi149 2023-09-23T20:14:57.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:57 smithi149 ceph-mon[100764]: Reconfiguring prometheus.a (dependencies changed)... 2023-09-23T20:14:57.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:57 smithi149 ceph-mon[100764]: from='client.? 172.21.15.18:0/3779561052' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:14:57.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:57 smithi149 ceph-mon[100764]: Reconfiguring daemon prometheus.a on smithi149 2023-09-23T20:14:58.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:58 smithi149 ceph-mon[100764]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:14:58.466 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:14:58 smithi149 systemd[1]: Stopping Ceph prometheus.a for a3b3b17e-5a4c-11ee-8db4-212e2dc638e7... 2023-09-23T20:14:58.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:58 smithi018 ceph-mon[109137]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:14:58.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:58 smithi018 ceph-mon[95889]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:14:58.966 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:14:58 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[138998]: ts=2023-09-23T20:14:58.563Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2023-09-23T20:14:58.966 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:14:58 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[138998]: ts=2023-09-23T20:14:58.564Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2023-09-23T20:14:58.966 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:14:58 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[138998]: ts=2023-09-23T20:14:58.564Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2023-09-23T20:14:58.966 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:14:58 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[138998]: ts=2023-09-23T20:14:58.564Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2023-09-23T20:14:58.966 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:14:58 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[138998]: ts=2023-09-23T20:14:58.564Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2023-09-23T20:14:58.966 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:14:58 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[138998]: ts=2023-09-23T20:14:58.564Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2023-09-23T20:14:58.966 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:14:58 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[138998]: ts=2023-09-23T20:14:58.564Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2023-09-23T20:14:58.967 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:14:58 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[138998]: ts=2023-09-23T20:14:58.564Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2023-09-23T20:14:58.967 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:14:58 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[138998]: ts=2023-09-23T20:14:58.564Z caller=main.go:907 level=info msg="Scrape manager stopped" 2023-09-23T20:14:58.967 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:14:58 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[138998]: ts=2023-09-23T20:14:58.564Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2023-09-23T20:14:58.967 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:14:58 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[138998]: ts=2023-09-23T20:14:58.564Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2023-09-23T20:14:58.967 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:14:58 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[138998]: ts=2023-09-23T20:14:58.564Z caller=main.go:1147 level=info msg="See you next time!" 2023-09-23T20:14:59.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:14:59 smithi149 ceph-mon[100764]: from='client.? 172.21.15.18:0/2760096749' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:14:59.466 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:14:59 smithi149 bash[152803]: ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a 2023-09-23T20:14:59.466 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:14:59 smithi149 systemd[1]: ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7@prometheus.a.service: Succeeded. 2023-09-23T20:14:59.466 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:14:59 smithi149 systemd[1]: Stopped Ceph prometheus.a for a3b3b17e-5a4c-11ee-8db4-212e2dc638e7. 2023-09-23T20:14:59.466 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:14:59 smithi149 systemd[1]: Starting Ceph prometheus.a for a3b3b17e-5a4c-11ee-8db4-212e2dc638e7... 2023-09-23T20:14:59.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:14:59 smithi018 ceph-mon[109137]: from='client.? 172.21.15.18:0/2760096749' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:14:59.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:14:59 smithi018 ceph-mon[95889]: from='client.? 172.21.15.18:0/2760096749' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:14:59.981 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:14:59 smithi149 podman[152922]: 2023-09-23T20:15:00.294 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:14:59 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[152938]: ts=2023-09-23T20:14:59.981Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2023-09-23T20:15:00.294 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:14:59 smithi149 bash[152922]: 7b55398a40546c6845931c435aae2276c133cbd9babe8d301761ed17218b678a 2023-09-23T20:15:00.294 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:14:59 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[152938]: ts=2023-09-23T20:14:59.981Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2023-09-23T20:15:00.294 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:14:59 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[152938]: ts=2023-09-23T20:14:59.981Z caller=main.go:570 level=info host_details="(Linux 4.18.0-372.9.1.el8.x86_64 #1 SMP Fri Apr 15 22:12:19 EDT 2022 x86_64 smithi149 (none))" 2023-09-23T20:15:00.294 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:14:59 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[152938]: ts=2023-09-23T20:14:59.982Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2023-09-23T20:15:00.294 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:14:59 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[152938]: ts=2023-09-23T20:14:59.982Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2023-09-23T20:15:00.294 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:14:59 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[152938]: ts=2023-09-23T20:14:59.983Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2023-09-23T20:15:00.295 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:14:59 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[152938]: ts=2023-09-23T20:14:59.984Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2023-09-23T20:15:00.295 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:14:59 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[152938]: ts=2023-09-23T20:14:59.985Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2023-09-23T20:15:00.295 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:14:59 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[152938]: ts=2023-09-23T20:14:59.985Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2023-09-23T20:15:00.295 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:14:59 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[152938]: ts=2023-09-23T20:14:59.987Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2023-09-23T20:15:00.295 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:14:59 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[152938]: ts=2023-09-23T20:14:59.987Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.76µs 2023-09-23T20:15:00.295 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:14:59 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[152938]: ts=2023-09-23T20:14:59.987Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2023-09-23T20:15:00.295 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:14:59 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[152938]: ts=2023-09-23T20:14:59.988Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=2 2023-09-23T20:15:00.295 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:15:00 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[152938]: ts=2023-09-23T20:15:00.000Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=2 2023-09-23T20:15:00.295 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:15:00 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[152938]: ts=2023-09-23T20:15:00.001Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=2 2023-09-23T20:15:00.296 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:15:00 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[152938]: ts=2023-09-23T20:15:00.001Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=20.914µs wal_replay_duration=13.257368ms wbl_replay_duration=122ns total_replay_duration=13.296617ms 2023-09-23T20:15:00.296 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:15:00 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[152938]: ts=2023-09-23T20:15:00.003Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2023-09-23T20:15:00.296 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:15:00 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[152938]: ts=2023-09-23T20:15:00.003Z caller=main.go:1029 level=info msg="TSDB started" 2023-09-23T20:15:00.296 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:15:00 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[152938]: ts=2023-09-23T20:15:00.003Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2023-09-23T20:15:00.296 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:15:00 smithi149 systemd[1]: Started Ceph prometheus.a for a3b3b17e-5a4c-11ee-8db4-212e2dc638e7. 2023-09-23T20:15:00.296 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:15:00 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[152938]: ts=2023-09-23T20:15:00.020Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=17.052497ms db_storage=828ns remote_storage=1.254µs web_handler=400ns query_engine=582ns scrape=434.115µs scrape_sd=77.091µs notify=14.228µs notify_sd=16.559µs rules=16.270891ms tracing=7.327µs 2023-09-23T20:15:00.296 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:15:00 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[152938]: ts=2023-09-23T20:15:00.020Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2023-09-23T20:15:00.296 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:15:00 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[152938]: ts=2023-09-23T20:15:00.020Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2023-09-23T20:15:00.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:00 smithi018 ceph-mon[95889]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:15:00.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:00 smithi018 ceph-mon[95889]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:15:00.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:00 smithi018 ceph-mon[95889]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:15:00.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:00 smithi018 ceph-mon[109137]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:15:00.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:00 smithi018 ceph-mon[109137]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:15:00.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:00 smithi018 ceph-mon[109137]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:15:00.716 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:15:00 smithi149 systemd[1]: Stopping Ceph node-exporter.b for a3b3b17e-5a4c-11ee-8db4-212e2dc638e7... 2023-09-23T20:15:00.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:00 smithi149 ceph-mon[100764]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:15:00.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:00 smithi149 ceph-mon[100764]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:15:00.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:00 smithi149 ceph-mon[100764]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:15:01.207 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:15:00 smithi149 bash[153100]: ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-b 2023-09-23T20:15:01.207 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:15:01 smithi149 systemd[1]: ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2023-09-23T20:15:01.466 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:15:01 smithi149 systemd[1]: ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7@node-exporter.b.service: Failed with result 'exit-code'. 2023-09-23T20:15:01.466 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:15:01 smithi149 systemd[1]: Stopped Ceph node-exporter.b for a3b3b17e-5a4c-11ee-8db4-212e2dc638e7. 2023-09-23T20:15:01.466 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:15:01 smithi149 systemd[1]: Starting Ceph node-exporter.b for a3b3b17e-5a4c-11ee-8db4-212e2dc638e7... 2023-09-23T20:15:01.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:01 smithi149 ceph-mon[100764]: Reconfiguring node-exporter.b (dependencies changed)... 2023-09-23T20:15:01.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:01 smithi149 ceph-mon[100764]: Reconfiguring daemon node-exporter.b on smithi149 2023-09-23T20:15:01.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:01 smithi149 ceph-mon[100764]: from='client.? 172.21.15.18:0/1432714455' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:15:01.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:01 smithi018 ceph-mon[109137]: Reconfiguring node-exporter.b (dependencies changed)... 2023-09-23T20:15:01.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:01 smithi018 ceph-mon[109137]: Reconfiguring daemon node-exporter.b on smithi149 2023-09-23T20:15:01.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:01 smithi018 ceph-mon[109137]: from='client.? 172.21.15.18:0/1432714455' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:15:01.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:01 smithi018 ceph-mon[95889]: Reconfiguring node-exporter.b (dependencies changed)... 2023-09-23T20:15:01.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:01 smithi018 ceph-mon[95889]: Reconfiguring daemon node-exporter.b on smithi149 2023-09-23T20:15:01.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:01 smithi018 ceph-mon[95889]: from='client.? 172.21.15.18:0/1432714455' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:15:01.796 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:15:01 smithi149 podman[153220]: 2023-09-23T20:15:01.796 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:15:01 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-b[153236]: ts=2023-09-23T20:15:01.728Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2023-09-23T20:15:01.796 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:15:01 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-b[153236]: ts=2023-09-23T20:15:01.728Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2023-09-23T20:15:01.797 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:15:01 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-b[153236]: ts=2023-09-23T20:15:01.728Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2023-09-23T20:15:01.797 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:15:01 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-b[153236]: ts=2023-09-23T20:15:01.728Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2023-09-23T20:15:01.797 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:15:01 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-b[153236]: ts=2023-09-23T20:15:01.729Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2023-09-23T20:15:01.797 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:15:01 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-b[153236]: ts=2023-09-23T20:15:01.729Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2023-09-23T20:15:01.797 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:15:01 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-b[153236]: ts=2023-09-23T20:15:01.729Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2023-09-23T20:15:01.797 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:15:01 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-b[153236]: ts=2023-09-23T20:15:01.729Z caller=node_exporter.go:117 level=info collector=arp 2023-09-23T20:15:01.797 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:15:01 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-b[153236]: ts=2023-09-23T20:15:01.729Z caller=node_exporter.go:117 level=info collector=bcache 2023-09-23T20:15:01.797 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:15:01 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-b[153236]: ts=2023-09-23T20:15:01.729Z caller=node_exporter.go:117 level=info collector=bonding 2023-09-23T20:15:01.797 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:15:01 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-b[153236]: ts=2023-09-23T20:15:01.729Z caller=node_exporter.go:117 level=info collector=btrfs 2023-09-23T20:15:01.797 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:15:01 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-b[153236]: ts=2023-09-23T20:15:01.729Z caller=node_exporter.go:117 level=info collector=conntrack 2023-09-23T20:15:01.798 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:15:01 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-b[153236]: ts=2023-09-23T20:15:01.729Z caller=node_exporter.go:117 level=info collector=cpu 2023-09-23T20:15:01.798 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:15:01 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-b[153236]: ts=2023-09-23T20:15:01.729Z caller=node_exporter.go:117 level=info collector=cpufreq 2023-09-23T20:15:01.798 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:15:01 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-b[153236]: ts=2023-09-23T20:15:01.729Z caller=node_exporter.go:117 level=info collector=diskstats 2023-09-23T20:15:01.798 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:15:01 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-b[153236]: ts=2023-09-23T20:15:01.729Z caller=node_exporter.go:117 level=info collector=dmi 2023-09-23T20:15:01.798 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:15:01 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-b[153236]: ts=2023-09-23T20:15:01.729Z caller=node_exporter.go:117 level=info collector=edac 2023-09-23T20:15:01.798 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:15:01 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-b[153236]: ts=2023-09-23T20:15:01.729Z caller=node_exporter.go:117 level=info collector=entropy 2023-09-23T20:15:01.798 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:15:01 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-b[153236]: ts=2023-09-23T20:15:01.729Z caller=node_exporter.go:117 level=info collector=fibrechannel 2023-09-23T20:15:01.798 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:15:01 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-b[153236]: ts=2023-09-23T20:15:01.729Z caller=node_exporter.go:117 level=info collector=filefd 2023-09-23T20:15:01.798 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:15:01 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-b[153236]: ts=2023-09-23T20:15:01.729Z caller=node_exporter.go:117 level=info collector=filesystem 2023-09-23T20:15:01.798 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:15:01 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-b[153236]: ts=2023-09-23T20:15:01.729Z caller=node_exporter.go:117 level=info collector=hwmon 2023-09-23T20:15:01.798 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:15:01 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-b[153236]: ts=2023-09-23T20:15:01.729Z caller=node_exporter.go:117 level=info collector=infiniband 2023-09-23T20:15:01.799 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:15:01 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-b[153236]: ts=2023-09-23T20:15:01.729Z caller=node_exporter.go:117 level=info collector=ipvs 2023-09-23T20:15:01.799 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:15:01 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-b[153236]: ts=2023-09-23T20:15:01.729Z caller=node_exporter.go:117 level=info collector=loadavg 2023-09-23T20:15:01.799 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:15:01 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-b[153236]: ts=2023-09-23T20:15:01.729Z caller=node_exporter.go:117 level=info collector=mdadm 2023-09-23T20:15:01.799 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:15:01 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-b[153236]: ts=2023-09-23T20:15:01.729Z caller=node_exporter.go:117 level=info collector=meminfo 2023-09-23T20:15:01.799 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:15:01 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-b[153236]: ts=2023-09-23T20:15:01.729Z caller=node_exporter.go:117 level=info collector=netclass 2023-09-23T20:15:01.799 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:15:01 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-b[153236]: ts=2023-09-23T20:15:01.729Z caller=node_exporter.go:117 level=info collector=netdev 2023-09-23T20:15:01.799 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:15:01 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-b[153236]: ts=2023-09-23T20:15:01.729Z caller=node_exporter.go:117 level=info collector=netstat 2023-09-23T20:15:01.799 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:15:01 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-b[153236]: ts=2023-09-23T20:15:01.729Z caller=node_exporter.go:117 level=info collector=nfs 2023-09-23T20:15:01.799 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:15:01 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-b[153236]: ts=2023-09-23T20:15:01.729Z caller=node_exporter.go:117 level=info collector=nfsd 2023-09-23T20:15:01.799 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:15:01 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-b[153236]: ts=2023-09-23T20:15:01.729Z caller=node_exporter.go:117 level=info collector=nvme 2023-09-23T20:15:01.799 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:15:01 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-b[153236]: ts=2023-09-23T20:15:01.729Z caller=node_exporter.go:117 level=info collector=os 2023-09-23T20:15:01.799 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:15:01 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-b[153236]: ts=2023-09-23T20:15:01.729Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2023-09-23T20:15:01.800 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:15:01 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-b[153236]: ts=2023-09-23T20:15:01.729Z caller=node_exporter.go:117 level=info collector=pressure 2023-09-23T20:15:01.800 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:15:01 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-b[153236]: ts=2023-09-23T20:15:01.729Z caller=node_exporter.go:117 level=info collector=rapl 2023-09-23T20:15:01.800 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:15:01 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-b[153236]: ts=2023-09-23T20:15:01.729Z caller=node_exporter.go:117 level=info collector=schedstat 2023-09-23T20:15:01.800 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:15:01 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-b[153236]: ts=2023-09-23T20:15:01.729Z caller=node_exporter.go:117 level=info collector=selinux 2023-09-23T20:15:01.800 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:15:01 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-b[153236]: ts=2023-09-23T20:15:01.729Z caller=node_exporter.go:117 level=info collector=sockstat 2023-09-23T20:15:01.800 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:15:01 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-b[153236]: ts=2023-09-23T20:15:01.729Z caller=node_exporter.go:117 level=info collector=softnet 2023-09-23T20:15:01.800 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:15:01 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-b[153236]: ts=2023-09-23T20:15:01.729Z caller=node_exporter.go:117 level=info collector=stat 2023-09-23T20:15:01.800 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:15:01 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-b[153236]: ts=2023-09-23T20:15:01.729Z caller=node_exporter.go:117 level=info collector=tapestats 2023-09-23T20:15:01.800 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:15:01 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-b[153236]: ts=2023-09-23T20:15:01.729Z caller=node_exporter.go:117 level=info collector=textfile 2023-09-23T20:15:01.800 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:15:01 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-b[153236]: ts=2023-09-23T20:15:01.729Z caller=node_exporter.go:117 level=info collector=thermal_zone 2023-09-23T20:15:01.800 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:15:01 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-b[153236]: ts=2023-09-23T20:15:01.729Z caller=node_exporter.go:117 level=info collector=time 2023-09-23T20:15:01.801 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:15:01 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-b[153236]: ts=2023-09-23T20:15:01.729Z caller=node_exporter.go:117 level=info collector=udp_queues 2023-09-23T20:15:01.801 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:15:01 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-b[153236]: ts=2023-09-23T20:15:01.729Z caller=node_exporter.go:117 level=info collector=uname 2023-09-23T20:15:01.801 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:15:01 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-b[153236]: ts=2023-09-23T20:15:01.729Z caller=node_exporter.go:117 level=info collector=vmstat 2023-09-23T20:15:01.801 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:15:01 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-b[153236]: ts=2023-09-23T20:15:01.729Z caller=node_exporter.go:117 level=info collector=xfs 2023-09-23T20:15:01.801 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:15:01 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-b[153236]: ts=2023-09-23T20:15:01.729Z caller=node_exporter.go:117 level=info collector=zfs 2023-09-23T20:15:01.801 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:15:01 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-b[153236]: ts=2023-09-23T20:15:01.729Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2023-09-23T20:15:01.801 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:15:01 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-b[153236]: ts=2023-09-23T20:15:01.729Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2023-09-23T20:15:01.801 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:15:01 smithi149 bash[153220]: f1789a46736bc9b20b5cdd837d70b646107a281bb2ecf47881fc92e15dbf3016 2023-09-23T20:15:01.801 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:15:01 smithi149 systemd[1]: Started Ceph node-exporter.b for a3b3b17e-5a4c-11ee-8db4-212e2dc638e7. 2023-09-23T20:15:02.586 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:02 smithi149 ceph-mon[100764]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:15:02.586 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:02 smithi149 ceph-mon[100764]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:15:02.586 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:02 smithi149 ceph-mon[100764]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:15:02.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:02 smithi018 ceph-mon[109137]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:15:02.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:02 smithi018 ceph-mon[109137]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:15:02.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:02 smithi018 ceph-mon[109137]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:15:02.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:02 smithi018 ceph-mon[95889]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:15:02.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:02 smithi018 ceph-mon[95889]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:15:02.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:02 smithi018 ceph-mon[95889]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:15:03.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:03 smithi149 ceph-mon[100764]: Reconfiguring grafana.a (dependencies changed)... 2023-09-23T20:15:03.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:03 smithi149 ceph-mon[100764]: Reconfiguring daemon grafana.a on smithi149 2023-09-23T20:15:03.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:03 smithi018 ceph-mon[109137]: Reconfiguring grafana.a (dependencies changed)... 2023-09-23T20:15:03.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:03 smithi018 ceph-mon[109137]: Reconfiguring daemon grafana.a on smithi149 2023-09-23T20:15:03.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:03 smithi018 ceph-mon[95889]: Reconfiguring grafana.a (dependencies changed)... 2023-09-23T20:15:03.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:03 smithi018 ceph-mon[95889]: Reconfiguring daemon grafana.a on smithi149 2023-09-23T20:15:03.882 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:15:03 smithi149 systemd[1]: Stopping Ceph grafana.a for a3b3b17e-5a4c-11ee-8db4-212e2dc638e7... 2023-09-23T20:15:04.179 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:15:04 smithi149 bash[153518]: ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a 2023-09-23T20:15:04.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:04 smithi149 ceph-mon[100764]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:15:04.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:04 smithi149 ceph-mon[100764]: from='client.? 172.21.15.18:0/1156152657' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:15:04.466 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:15:04 smithi149 systemd[1]: ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7@grafana.a.service: Main process exited, code=exited, status=143/n/a 2023-09-23T20:15:04.466 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:15:04 smithi149 systemd[1]: ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7@grafana.a.service: Failed with result 'exit-code'. 2023-09-23T20:15:04.466 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:15:04 smithi149 systemd[1]: Stopped Ceph grafana.a for a3b3b17e-5a4c-11ee-8db4-212e2dc638e7. 2023-09-23T20:15:04.466 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:15:04 smithi149 systemd[1]: Starting Ceph grafana.a for a3b3b17e-5a4c-11ee-8db4-212e2dc638e7... 2023-09-23T20:15:04.641 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 23 20:15:04 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-alertmanager-a[166780]: ts=2023-09-23T20:15:04.235Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.002505306s 2023-09-23T20:15:04.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:04 smithi018 ceph-mon[95889]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:15:04.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:04 smithi018 ceph-mon[95889]: from='client.? 172.21.15.18:0/1156152657' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:15:04.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:04 smithi018 ceph-mon[109137]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:15:04.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:04 smithi018 ceph-mon[109137]: from='client.? 172.21.15.18:0/1156152657' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:15:06.216 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:05 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[141947]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:15:05] "GET /metrics HTTP/1.1" 200 33916 "" "Prometheus/2.43.0" 2023-09-23T20:15:06.216 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:15:05 smithi149 podman[153634]: 2023-09-23T20:15:06.587 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:06 smithi149 ceph-mon[100764]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:15:06.587 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:06 smithi149 ceph-mon[100764]: from='client.? 172.21.15.18:0/1045187776' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:15:06.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:06 smithi018 ceph-mon[109137]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:15:06.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:06 smithi018 ceph-mon[109137]: from='client.? 172.21.15.18:0/1045187776' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:15:06.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:06 smithi018 ceph-mon[95889]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:15:06.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:06 smithi018 ceph-mon[95889]: from='client.? 172.21.15.18:0/1045187776' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:15:07.005 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:06 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[141947]: [23/Sep/2023:20:15:06] ENGINE Bus STOPPING 2023-09-23T20:15:07.005 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:06 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[141947]: [23/Sep/2023:20:15:06] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T20:15:07.005 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:06 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[141947]: [23/Sep/2023:20:15:06] ENGINE Bus STOPPED 2023-09-23T20:15:07.005 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:06 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[141947]: [23/Sep/2023:20:15:06] ENGINE Bus STARTING 2023-09-23T20:15:07.005 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:06 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[141947]: [23/Sep/2023:20:15:06] ENGINE Serving on http://:::9283 2023-09-23T20:15:07.006 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:06 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[141947]: [23/Sep/2023:20:15:06] ENGINE Bus STARTED 2023-09-23T20:15:07.006 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:06 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[141947]: [23/Sep/2023:20:15:06] ENGINE Bus STOPPING 2023-09-23T20:15:07.006 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:06 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[141947]: [23/Sep/2023:20:15:06] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T20:15:07.006 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:06 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[141947]: [23/Sep/2023:20:15:06] ENGINE Bus STOPPED 2023-09-23T20:15:07.006 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:06 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[141947]: [23/Sep/2023:20:15:06] ENGINE Bus STARTING 2023-09-23T20:15:07.006 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:15:06 smithi149 bash[153634]: 62150314e898e91374c2fe048ad64d05316c9b7e309ddbe7670c268bf52c9b81 2023-09-23T20:15:07.006 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:15:06 smithi149 systemd[1]: Started Ceph grafana.a for a3b3b17e-5a4c-11ee-8db4-212e2dc638e7. 2023-09-23T20:15:07.006 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:15:06 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[153650]: logger=settings t=2023-09-23T20:15:06.89527531Z level=info msg="Starting Grafana" version=9.4.7 commit=4add91f03d branch=HEAD compiled=2023-03-16T23:56:52Z 2023-09-23T20:15:07.007 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:15:06 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[153650]: logger=settings t=2023-09-23T20:15:06.895491208Z level=warn msg="\"sentry\" frontend logging provider is deprecated and will be removed in the next major version. Use \"grafana\" provider instead." 2023-09-23T20:15:07.007 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:15:06 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[153650]: logger=settings t=2023-09-23T20:15:06.895523318Z level=info msg="Config loaded from" file=/usr/share/grafana/conf/defaults.ini 2023-09-23T20:15:07.007 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:15:06 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[153650]: logger=settings t=2023-09-23T20:15:06.895536442Z level=info msg="Config loaded from" file=/etc/grafana/grafana.ini 2023-09-23T20:15:07.007 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:15:06 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[153650]: logger=settings t=2023-09-23T20:15:06.895547512Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_DATA=/var/lib/grafana" 2023-09-23T20:15:07.007 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:15:06 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[153650]: logger=settings t=2023-09-23T20:15:06.895558446Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_LOGS=/var/log/grafana" 2023-09-23T20:15:07.007 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:15:06 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[153650]: logger=settings t=2023-09-23T20:15:06.895568957Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2023-09-23T20:15:07.007 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:15:06 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[153650]: logger=settings t=2023-09-23T20:15:06.89557955Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2023-09-23T20:15:07.007 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:15:06 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[153650]: logger=settings t=2023-09-23T20:15:06.89558977Z level=info msg="Path Home" path=/usr/share/grafana 2023-09-23T20:15:07.007 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:15:06 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[153650]: logger=settings t=2023-09-23T20:15:06.895599581Z level=info msg="Path Data" path=/var/lib/grafana 2023-09-23T20:15:07.007 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:15:06 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[153650]: logger=settings t=2023-09-23T20:15:06.895610092Z level=info msg="Path Logs" path=/var/log/grafana 2023-09-23T20:15:07.007 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:15:06 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[153650]: logger=settings t=2023-09-23T20:15:06.895620253Z level=info msg="Path Plugins" path=/var/lib/grafana/plugins 2023-09-23T20:15:07.007 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:15:06 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[153650]: logger=settings t=2023-09-23T20:15:06.895631189Z level=info msg="Path Provisioning" path=/etc/grafana/provisioning 2023-09-23T20:15:07.008 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:15:06 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[153650]: logger=settings t=2023-09-23T20:15:06.895641592Z level=info msg="App mode production" 2023-09-23T20:15:07.008 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:15:06 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[153650]: logger=sqlstore t=2023-09-23T20:15:06.895710103Z level=info msg="Connecting to DB" dbtype=sqlite3 2023-09-23T20:15:07.008 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:15:06 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[153650]: logger=sqlstore t=2023-09-23T20:15:06.895737737Z level=warn msg="SQLite database file has broader permissions than it should" path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2023-09-23T20:15:07.008 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:15:06 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[153650]: logger=migrator t=2023-09-23T20:15:06.914772391Z level=info msg="Starting DB migrations" 2023-09-23T20:15:07.008 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:15:06 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[153650]: logger=migrator t=2023-09-23T20:15:06.919678909Z level=info msg="migrations completed" performed=0 skipped=481 duration=518.371µs 2023-09-23T20:15:07.008 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:15:06 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[153650]: logger=sqlstore t=2023-09-23T20:15:06.920037588Z level=info msg="Created default organization" 2023-09-23T20:15:07.008 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:15:06 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[153650]: logger=plugin.loader t=2023-09-23T20:15:06.976459957Z level=info msg="Plugin registered" pluginID=input 2023-09-23T20:15:07.008 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:15:07 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[153650]: logger=plugin.loader t=2023-09-23T20:15:07.005581731Z level=info msg="Plugin registered" pluginID=grafana-piechart-panel 2023-09-23T20:15:07.008 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:15:07 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[153650]: logger=plugin.loader t=2023-09-23T20:15:07.005631362Z level=info msg="Plugin registered" pluginID=vonage-status-panel 2023-09-23T20:15:07.008 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:15:07 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[153650]: logger=secrets t=2023-09-23T20:15:07.00568184Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1 2023-09-23T20:15:07.466 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:15:07 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[153650]: logger=query_data t=2023-09-23T20:15:07.007601138Z level=info msg="Query Service initialization" 2023-09-23T20:15:07.466 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:15:07 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[153650]: logger=live.push_http t=2023-09-23T20:15:07.009958617Z level=info msg="Live Push Gateway initialization" 2023-09-23T20:15:07.466 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:07 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[141947]: [23/Sep/2023:20:15:07] ENGINE Serving on http://:::9283 2023-09-23T20:15:07.466 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:07 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[141947]: [23/Sep/2023:20:15:07] ENGINE Bus STARTED 2023-09-23T20:15:07.466 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:07 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[141947]: [23/Sep/2023:20:15:07] ENGINE Bus STOPPING 2023-09-23T20:15:07.466 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:07 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[141947]: [23/Sep/2023:20:15:07] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T20:15:07.467 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:07 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[141947]: [23/Sep/2023:20:15:07] ENGINE Bus STOPPED 2023-09-23T20:15:07.467 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:07 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[141947]: [23/Sep/2023:20:15:07] ENGINE Bus STARTING 2023-09-23T20:15:07.467 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:07 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[141947]: [23/Sep/2023:20:15:07] ENGINE Serving on http://:::9283 2023-09-23T20:15:07.467 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:07 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[141947]: [23/Sep/2023:20:15:07] ENGINE Bus STARTED 2023-09-23T20:15:07.858 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:07 smithi018 ceph-mon[109137]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:15:07.859 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:07 smithi018 ceph-mon[109137]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:15:07.859 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:07 smithi018 ceph-mon[109137]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T20:15:07.859 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:07 smithi018 ceph-mon[109137]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T20:15:07.859 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:07 smithi018 ceph-mon[109137]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi018.front.sepia.ceph.com:9093"}]: dispatch 2023-09-23T20:15:07.859 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:07 smithi018 ceph-mon[109137]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi018.front.sepia.ceph.com:9093"}]: dispatch 2023-09-23T20:15:07.859 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:07 smithi018 ceph-mon[109137]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:15:07.859 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:07 smithi018 ceph-mon[109137]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T20:15:07.859 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:07 smithi018 ceph-mon[109137]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T20:15:07.859 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:07 smithi018 ceph-mon[109137]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi149:9095"}]: dispatch 2023-09-23T20:15:07.859 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:07 smithi018 ceph-mon[109137]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi149:9095"}]: dispatch 2023-09-23T20:15:07.860 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:07 smithi018 ceph-mon[109137]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:15:07.860 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:07 smithi018 ceph-mon[109137]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T20:15:07.860 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:07 smithi018 ceph-mon[109137]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T20:15:07.860 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:07 smithi018 ceph-mon[109137]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi149:3000"}]: dispatch 2023-09-23T20:15:07.860 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:07 smithi018 ceph-mon[109137]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi149:3000"}]: dispatch 2023-09-23T20:15:07.860 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:07 smithi018 ceph-mon[109137]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:15:07.860 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:07 smithi018 ceph-mon[109137]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:15:07.860 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:07 smithi018 ceph-mon[109137]: Upgrade: Need to upgrade myself (mgr.x) 2023-09-23T20:15:07.860 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:07 smithi018 ceph-mon[109137]: Upgrade: Need to upgrade myself (mgr.x) 2023-09-23T20:15:07.860 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:07 smithi018 ceph-mon[109137]: pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:15:07.861 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:07 smithi018 ceph-mon[95889]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:15:07.861 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:07 smithi018 ceph-mon[95889]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:15:07.861 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:07 smithi018 ceph-mon[95889]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T20:15:07.861 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:07 smithi018 ceph-mon[95889]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T20:15:07.861 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:07 smithi018 ceph-mon[95889]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi018.front.sepia.ceph.com:9093"}]: dispatch 2023-09-23T20:15:07.861 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:07 smithi018 ceph-mon[95889]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi018.front.sepia.ceph.com:9093"}]: dispatch 2023-09-23T20:15:07.861 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:07 smithi018 ceph-mon[95889]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:15:07.861 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:07 smithi018 ceph-mon[95889]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T20:15:07.861 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:07 smithi018 ceph-mon[95889]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T20:15:07.861 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:07 smithi018 ceph-mon[95889]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi149:9095"}]: dispatch 2023-09-23T20:15:07.861 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:07 smithi018 ceph-mon[95889]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi149:9095"}]: dispatch 2023-09-23T20:15:07.862 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:07 smithi018 ceph-mon[95889]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:15:07.862 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:07 smithi018 ceph-mon[95889]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T20:15:07.862 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:07 smithi018 ceph-mon[95889]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T20:15:07.862 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:07 smithi018 ceph-mon[95889]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi149:3000"}]: dispatch 2023-09-23T20:15:07.862 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:07 smithi018 ceph-mon[95889]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi149:3000"}]: dispatch 2023-09-23T20:15:07.862 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:07 smithi018 ceph-mon[95889]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:15:07.862 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:07 smithi018 ceph-mon[95889]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:15:07.862 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:07 smithi018 ceph-mon[95889]: Upgrade: Need to upgrade myself (mgr.x) 2023-09-23T20:15:07.862 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:07 smithi018 ceph-mon[95889]: Upgrade: Need to upgrade myself (mgr.x) 2023-09-23T20:15:07.862 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:07 smithi018 ceph-mon[95889]: pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:15:08.216 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:15:07 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[153650]: logger=infra.usagestats.collector t=2023-09-23T20:15:07.735378148Z level=info msg="registering usage stat providers" usageStatsProvidersLen=2 2023-09-23T20:15:08.216 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:15:07 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[153650]: logger=provisioning.datasources t=2023-09-23T20:15:07.82258194Z level=info msg="deleted correlations based on configuration" ds_name=Dashboard1 2023-09-23T20:15:08.216 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:15:07 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[153650]: logger=provisioning.datasources t=2023-09-23T20:15:07.822614899Z level=info msg="deleted datasource based on configuration" name=Dashboard1 2023-09-23T20:15:08.217 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:15:07 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[153650]: logger=provisioning.datasources t=2023-09-23T20:15:07.832841969Z level=info msg="inserting datasource from configuration " name=Dashboard1 uid=P43CA22E17D0F9596 2023-09-23T20:15:08.217 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:15:07 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[153650]: logger=provisioning.alerting t=2023-09-23T20:15:07.913166133Z level=info msg="starting to provision alerting" 2023-09-23T20:15:08.217 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:15:07 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[153650]: logger=provisioning.alerting t=2023-09-23T20:15:07.913194203Z level=info msg="finished to provision alerting" 2023-09-23T20:15:08.217 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:15:07 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[153650]: logger=grafanaStorageLogger t=2023-09-23T20:15:07.91329933Z level=info msg="storage starting" 2023-09-23T20:15:08.217 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:15:07 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[153650]: logger=ngalert.state.manager t=2023-09-23T20:15:07.913514425Z level=info msg="Warming state cache for startup" 2023-09-23T20:15:08.217 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:15:07 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[153650]: logger=http.server t=2023-09-23T20:15:07.918313782Z level=info msg="HTTP Server Listen" address=[::]:3000 protocol=https subUrl= socket= 2023-09-23T20:15:08.217 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:15:07 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[153650]: logger=ngalert.state.manager t=2023-09-23T20:15:07.924619494Z level=info msg="State cache has been initialized" states=0 duration=11.105052ms 2023-09-23T20:15:08.217 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:15:07 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[153650]: logger=ngalert.multiorg.alertmanager t=2023-09-23T20:15:07.924681614Z level=info msg="starting MultiOrg Alertmanager" 2023-09-23T20:15:08.217 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:15:07 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[153650]: logger=ticker t=2023-09-23T20:15:07.924697552Z level=info msg=starting first_tick=2023-09-23T20:15:10Z 2023-09-23T20:15:08.218 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:07 smithi149 ceph-mon[100764]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:15:08.218 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:07 smithi149 ceph-mon[100764]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:15:08.218 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:07 smithi149 ceph-mon[100764]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T20:15:08.218 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:07 smithi149 ceph-mon[100764]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T20:15:08.218 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:07 smithi149 ceph-mon[100764]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi018.front.sepia.ceph.com:9093"}]: dispatch 2023-09-23T20:15:08.218 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:07 smithi149 ceph-mon[100764]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi018.front.sepia.ceph.com:9093"}]: dispatch 2023-09-23T20:15:08.218 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:07 smithi149 ceph-mon[100764]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:15:08.218 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:07 smithi149 ceph-mon[100764]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T20:15:08.218 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:07 smithi149 ceph-mon[100764]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T20:15:08.218 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:07 smithi149 ceph-mon[100764]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi149:9095"}]: dispatch 2023-09-23T20:15:08.219 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:07 smithi149 ceph-mon[100764]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi149:9095"}]: dispatch 2023-09-23T20:15:08.219 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:07 smithi149 ceph-mon[100764]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:15:08.219 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:07 smithi149 ceph-mon[100764]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T20:15:08.219 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:07 smithi149 ceph-mon[100764]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T20:15:08.219 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:07 smithi149 ceph-mon[100764]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi149:3000"}]: dispatch 2023-09-23T20:15:08.219 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:07 smithi149 ceph-mon[100764]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi149:3000"}]: dispatch 2023-09-23T20:15:08.219 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:07 smithi149 ceph-mon[100764]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:15:08.219 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:07 smithi149 ceph-mon[100764]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:15:08.219 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:07 smithi149 ceph-mon[100764]: Upgrade: Need to upgrade myself (mgr.x) 2023-09-23T20:15:08.219 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:07 smithi149 ceph-mon[100764]: Upgrade: Need to upgrade myself (mgr.x) 2023-09-23T20:15:08.220 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:07 smithi149 ceph-mon[100764]: pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:15:08.966 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:15:08 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[152938]: ts=2023-09-23T20:15:08.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.18\", device_class=\"ssd\", hostname=\"smithi018\", instance=\"smithi149:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.18\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.18\", device_class=\"ssd\", hostname=\"smithi018\", instance=\"smithi149.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.18\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T20:15:09.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:09 smithi018 ceph-mon[109137]: Upgrade: Updating mgr.y 2023-09-23T20:15:09.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:09 smithi018 ceph-mon[109137]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:15:09.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:09 smithi018 ceph-mon[109137]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T20:15:09.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:09 smithi018 ceph-mon[109137]: from='mgr.25414 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T20:15:09.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:09 smithi018 ceph-mon[109137]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T20:15:09.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:09 smithi018 ceph-mon[109137]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:15:09.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:09 smithi018 ceph-mon[109137]: Deploying daemon mgr.y on smithi018 2023-09-23T20:15:09.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:09 smithi018 ceph-mon[109137]: from='client.? 172.21.15.18:0/316660711' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:15:09.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:09 smithi018 ceph-mon[109137]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:15:09.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:09 smithi018 ceph-mon[95889]: Upgrade: Updating mgr.y 2023-09-23T20:15:09.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:09 smithi018 ceph-mon[95889]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:15:09.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:09 smithi018 ceph-mon[95889]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T20:15:09.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:09 smithi018 ceph-mon[95889]: from='mgr.25414 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T20:15:09.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:09 smithi018 ceph-mon[95889]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T20:15:09.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:09 smithi018 ceph-mon[95889]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:15:09.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:09 smithi018 ceph-mon[95889]: Deploying daemon mgr.y on smithi018 2023-09-23T20:15:09.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:09 smithi018 ceph-mon[95889]: from='client.? 172.21.15.18:0/316660711' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:15:09.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:09 smithi018 ceph-mon[95889]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:15:09.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:09 smithi149 ceph-mon[100764]: Upgrade: Updating mgr.y 2023-09-23T20:15:09.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:09 smithi149 ceph-mon[100764]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:15:09.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:09 smithi149 ceph-mon[100764]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T20:15:09.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:09 smithi149 ceph-mon[100764]: from='mgr.25414 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T20:15:09.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:09 smithi149 ceph-mon[100764]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T20:15:09.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:09 smithi149 ceph-mon[100764]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:15:09.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:09 smithi149 ceph-mon[100764]: Deploying daemon mgr.y on smithi018 2023-09-23T20:15:09.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:09 smithi149 ceph-mon[100764]: from='client.? 172.21.15.18:0/316660711' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:15:09.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:09 smithi149 ceph-mon[100764]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:15:09.891 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:15:09 smithi018 systemd[1]: Stopping Ceph mgr.y for a3b3b17e-5a4c-11ee-8db4-212e2dc638e7... 2023-09-23T20:15:10.281 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:15:10 smithi018 bash[167612]: ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y 2023-09-23T20:15:10.281 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:15:10 smithi018 systemd[1]: ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7@mgr.y.service: Main process exited, code=exited, status=143/n/a 2023-09-23T20:15:10.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:10 smithi018 ceph-mon[109137]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:15:10.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:10 smithi018 ceph-mon[95889]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:15:10.642 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:15:10 smithi018 systemd[1]: ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7@mgr.y.service: Failed with result 'exit-code'. 2023-09-23T20:15:10.642 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:15:10 smithi018 systemd[1]: Stopped Ceph mgr.y for a3b3b17e-5a4c-11ee-8db4-212e2dc638e7. 2023-09-23T20:15:10.642 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:15:10 smithi018 systemd[1]: Starting Ceph mgr.y for a3b3b17e-5a4c-11ee-8db4-212e2dc638e7... 2023-09-23T20:15:10.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:10 smithi149 ceph-mon[100764]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:15:11.029 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:15:10 smithi018 podman[167778]: 2023-09-23T20:15:11.282 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:15:11 smithi018 bash[167778]: 5aa74f13c0a1d2b84d3bc68ae00baceac35340dcd3838988c5d26a2791d3fa69 2023-09-23T20:15:11.282 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:15:11 smithi018 ceph-mgr[167821]: -- 172.21.15.18:0/538781552 <== mon.2 v2:172.21.15.18:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55f9bb83f1e0 con 0x55f9bb814c00 2023-09-23T20:15:11.282 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:15:11 smithi018 systemd[1]: Started Ceph mgr.y for a3b3b17e-5a4c-11ee-8db4-212e2dc638e7. 2023-09-23T20:15:11.283 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:15:11 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: 2023-09-23T20:15:11.177+0000 7f6818c3a200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-23T20:15:11.283 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:15:11 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: 2023-09-23T20:15:11.280+0000 7f6818c3a200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T20:15:11.641 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:15:11 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: 2023-09-23T20:15:11.395+0000 7f6818c3a200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T20:15:11.641 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:15:11 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: 2023-09-23T20:15:11.517+0000 7f6818c3a200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T20:15:11.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:11 smithi018 ceph-mon[109137]: from='client.? 172.21.15.18:0/1370675472' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:15:11.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:11 smithi018 ceph-mon[109137]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:15:11.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:11 smithi018 ceph-mon[109137]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:15:11.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:11 smithi018 ceph-mon[109137]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:15:11.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:11 smithi018 ceph-mon[95889]: from='client.? 172.21.15.18:0/1370675472' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:15:11.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:11 smithi018 ceph-mon[95889]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:15:11.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:11 smithi018 ceph-mon[95889]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:15:11.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:11 smithi018 ceph-mon[95889]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:15:11.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:11 smithi149 ceph-mon[100764]: from='client.? 172.21.15.18:0/1370675472' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:15:11.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:11 smithi149 ceph-mon[100764]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:15:11.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:11 smithi149 ceph-mon[100764]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:15:11.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:11 smithi149 ceph-mon[100764]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:15:12.140 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:15:11 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: 2023-09-23T20:15:11.740+0000 7f6818c3a200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T20:15:12.521 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:12 smithi018 ceph-mon[109137]: pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:15:12.522 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:15:12 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: 2023-09-23T20:15:12.246+0000 7f6818c3a200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T20:15:12.522 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:15:12 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: 2023-09-23T20:15:12.408+0000 7f6818c3a200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T20:15:12.522 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:15:12 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: 2023-09-23T20:15:12.518+0000 7f6818c3a200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T20:15:12.523 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:12 smithi018 ceph-mon[95889]: pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:15:12.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:12 smithi149 ceph-mon[100764]: pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:15:12.890 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:15:12 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: 2023-09-23T20:15:12.650+0000 7f6818c3a200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T20:15:13.641 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:15:13 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: 2023-09-23T20:15:13.206+0000 7f6818c3a200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T20:15:13.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:13 smithi018 ceph-mon[109137]: from='client.? 172.21.15.18:0/66646784' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:15:13.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:13 smithi018 ceph-mon[95889]: from='client.? 172.21.15.18:0/66646784' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:15:13.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:13 smithi149 ceph-mon[100764]: from='client.? 172.21.15.18:0/66646784' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:15:14.390 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:14 smithi018 ceph-mon[109137]: pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:15:14.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:14 smithi018 ceph-mon[95889]: pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:15:14.391 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:15:14 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: 2023-09-23T20:15:14.043+0000 7f6818c3a200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T20:15:14.391 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:15:14 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: 2023-09-23T20:15:14.153+0000 7f6818c3a200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T20:15:14.715 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:14 smithi149 ceph-mon[100764]: pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:15:15.549 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:15:15 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: 2023-09-23T20:15:15.185+0000 7f6818c3a200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T20:15:15.549 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:15:15 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: 2023-09-23T20:15:15.309+0000 7f6818c3a200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-23T20:15:15.549 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:15:15 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: 2023-09-23T20:15:15.548+0000 7f6818c3a200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T20:15:16.196 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:15:15 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: 2023-09-23T20:15:15.932+0000 7f6818c3a200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-23T20:15:16.216 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:15 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[141947]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:15:15] "GET /metrics HTTP/1.1" 200 33912 "" "Prometheus/2.43.0" 2023-09-23T20:15:16.641 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:15:16 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: 2023-09-23T20:15:16.194+0000 7f6818c3a200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T20:15:16.641 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:15:16 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: 2023-09-23T20:15:16.386+0000 7f6818c3a200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T20:15:16.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:16 smithi018 ceph-mon[109137]: pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:15:16.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:16 smithi018 ceph-mon[109137]: from='client.? 172.21.15.18:0/1044971373' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:15:16.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:16 smithi018 ceph-mon[109137]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:15:16.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:16 smithi018 ceph-mon[109137]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:15:16.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:16 smithi018 ceph-mon[109137]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:15:16.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:16 smithi018 ceph-mon[109137]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:15:16.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:16 smithi018 ceph-mon[95889]: pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:15:16.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:16 smithi018 ceph-mon[95889]: from='client.? 172.21.15.18:0/1044971373' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:15:16.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:16 smithi018 ceph-mon[95889]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:15:16.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:16 smithi018 ceph-mon[95889]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:15:16.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:16 smithi018 ceph-mon[95889]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:15:16.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:16 smithi018 ceph-mon[95889]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:15:16.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:16 smithi149 ceph-mon[100764]: pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:15:16.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:16 smithi149 ceph-mon[100764]: from='client.? 172.21.15.18:0/1044971373' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:15:16.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:16 smithi149 ceph-mon[100764]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:15:16.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:16 smithi149 ceph-mon[100764]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:15:16.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:16 smithi149 ceph-mon[100764]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:15:16.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:16 smithi149 ceph-mon[100764]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:15:17.141 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:15:16 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: 2023-09-23T20:15:16.740+0000 7f6818c3a200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T20:15:17.771 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:17 smithi018 ceph-mon[109137]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:15:17.771 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:17 smithi018 ceph-mon[109137]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:15:17.771 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:17 smithi018 ceph-mon[109137]: pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:15:17.772 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:17 smithi018 ceph-mon[95889]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:15:17.772 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:17 smithi018 ceph-mon[95889]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:15:17.772 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:17 smithi018 ceph-mon[95889]: pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:15:17.772 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:15:17 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: 2023-09-23T20:15:17.618+0000 7f6818c3a200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T20:15:17.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:17 smithi149 ceph-mon[100764]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:15:17.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:17 smithi149 ceph-mon[100764]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:15:17.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:17 smithi149 ceph-mon[100764]: pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:15:18.071 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:15:17 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: 2023-09-23T20:15:17.790+0000 7f6818c3a200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T20:15:18.391 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:15:18 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: 2023-09-23T20:15:18.069+0000 7f6818c3a200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T20:15:18.391 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:15:18 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: 2023-09-23T20:15:18.217+0000 7f6818c3a200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T20:15:18.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:18 smithi018 ceph-mon[95889]: from='client.? 172.21.15.18:0/3304769789' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:15:18.892 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:15:18 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: 2023-09-23T20:15:18.509+0000 7f6818c3a200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T20:15:18.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:18 smithi018 ceph-mon[109137]: from='client.? 172.21.15.18:0/3304769789' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:15:18.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:18 smithi149 ceph-mon[100764]: from='client.? 172.21.15.18:0/3304769789' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:15:18.966 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:15:18 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[152938]: ts=2023-09-23T20:15:18.664Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.18\", device_class=\"ssd\", hostname=\"smithi018\", instance=\"smithi149:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.18\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.18\", device_class=\"ssd\", hostname=\"smithi018\", instance=\"smithi149.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.18\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T20:15:19.890 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:15:19 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: 2023-09-23T20:15:19.601+0000 7f6818c3a200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T20:15:19.890 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:15:19 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: [23/Sep/2023:20:15:19] ENGINE Bus STARTING 2023-09-23T20:15:19.890 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:15:19 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: CherryPy Checker: 2023-09-23T20:15:19.890 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:15:19 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: The Application mounted at '' has an empty config. 2023-09-23T20:15:19.891 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:15:19 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: 2023-09-23T20:15:19.891 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:15:19 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: [23/Sep/2023:20:15:19] ENGINE Serving on http://:::9283 2023-09-23T20:15:19.891 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:15:19 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: [23/Sep/2023:20:15:19] ENGINE Bus STARTED 2023-09-23T20:15:20.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:19 smithi018 ceph-mon[95889]: pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:15:20.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:19 smithi018 ceph-mon[95889]: Standby manager daemon y restarted 2023-09-23T20:15:20.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:19 smithi018 ceph-mon[95889]: Standby manager daemon y started 2023-09-23T20:15:20.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:19 smithi018 ceph-mon[95889]: from='mgr.? 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-09-23T20:15:20.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:19 smithi018 ceph-mon[95889]: from='mgr.? 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T20:15:20.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:19 smithi018 ceph-mon[95889]: from='mgr.? 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-09-23T20:15:20.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:19 smithi018 ceph-mon[95889]: from='mgr.? 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T20:15:20.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:19 smithi018 ceph-mon[95889]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:15:20.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:19 smithi018 ceph-mon[95889]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:15:20.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:19 smithi018 ceph-mon[95889]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:15:20.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:19 smithi018 ceph-mon[95889]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:15:20.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:19 smithi018 ceph-mon[95889]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:15:20.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:19 smithi018 ceph-mon[95889]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:15:20.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:19 smithi018 ceph-mon[95889]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-09-23T20:15:20.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:19 smithi018 ceph-mon[95889]: from='mgr.25414 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-09-23T20:15:20.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:19 smithi018 ceph-mon[109137]: pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:15:20.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:19 smithi018 ceph-mon[109137]: Standby manager daemon y restarted 2023-09-23T20:15:20.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:19 smithi018 ceph-mon[109137]: Standby manager daemon y started 2023-09-23T20:15:20.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:19 smithi018 ceph-mon[109137]: from='mgr.? 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-09-23T20:15:20.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:19 smithi018 ceph-mon[109137]: from='mgr.? 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T20:15:20.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:19 smithi018 ceph-mon[109137]: from='mgr.? 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-09-23T20:15:20.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:19 smithi018 ceph-mon[109137]: from='mgr.? 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T20:15:20.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:19 smithi018 ceph-mon[109137]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:15:20.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:19 smithi018 ceph-mon[109137]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:15:20.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:19 smithi018 ceph-mon[109137]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:15:20.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:19 smithi018 ceph-mon[109137]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:15:20.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:19 smithi018 ceph-mon[109137]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:15:20.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:19 smithi018 ceph-mon[109137]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:15:20.144 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:19 smithi018 ceph-mon[109137]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-09-23T20:15:20.144 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:19 smithi018 ceph-mon[109137]: from='mgr.25414 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-09-23T20:15:20.144 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:15:19 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: [23/Sep/2023:20:15:19] ENGINE Bus STOPPING 2023-09-23T20:15:20.144 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:15:19 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: [23/Sep/2023:20:15:19] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T20:15:20.144 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:15:19 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: [23/Sep/2023:20:15:19] ENGINE Bus STOPPED 2023-09-23T20:15:20.144 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:15:19 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: [23/Sep/2023:20:15:19] ENGINE Bus STARTING 2023-09-23T20:15:20.144 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:15:20 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: 2023-09-23T20:15:20.034+0000 7f67750aa700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T20:15:20.144 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:15:20 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: 2023-09-23T20:15:20.034+0000 7f67750aa700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T20:15:20.144 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:15:20 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: 2023-09-23T20:15:20.034+0000 7f67750aa700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T20:15:20.144 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:15:20 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: 2023-09-23T20:15:20.034+0000 7f67750aa700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T20:15:20.145 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:15:20 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: 2023-09-23T20:15:20.034+0000 7f67750aa700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T20:15:20.145 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:15:20 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: 2023-09-23T20:15:20.035+0000 7f67718a3700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T20:15:20.145 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:15:20 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: 2023-09-23T20:15:20.035+0000 7f67718a3700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T20:15:20.145 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:15:20 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: 2023-09-23T20:15:20.035+0000 7f67718a3700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T20:15:20.145 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:15:20 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: 2023-09-23T20:15:20.035+0000 7f67718a3700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T20:15:20.145 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:15:20 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: 2023-09-23T20:15:20.035+0000 7f67718a3700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T20:15:20.145 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:15:20 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: [23/Sep/2023:20:15:20] ENGINE Serving on http://:::9283 2023-09-23T20:15:20.145 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:15:20 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: [23/Sep/2023:20:15:20] ENGINE Bus STARTED 2023-09-23T20:15:20.145 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:19 smithi149 ceph-mon[100764]: pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:15:20.146 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:19 smithi149 ceph-mon[100764]: Standby manager daemon y restarted 2023-09-23T20:15:20.146 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:19 smithi149 ceph-mon[100764]: Standby manager daemon y started 2023-09-23T20:15:20.146 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:19 smithi149 ceph-mon[100764]: from='mgr.? 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-09-23T20:15:20.146 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:19 smithi149 ceph-mon[100764]: from='mgr.? 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T20:15:20.146 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:19 smithi149 ceph-mon[100764]: from='mgr.? 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-09-23T20:15:20.146 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:19 smithi149 ceph-mon[100764]: from='mgr.? 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T20:15:20.146 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:19 smithi149 ceph-mon[100764]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:15:20.146 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:19 smithi149 ceph-mon[100764]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:15:20.146 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:19 smithi149 ceph-mon[100764]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:15:20.146 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:19 smithi149 ceph-mon[100764]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:15:20.146 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:19 smithi149 ceph-mon[100764]: from='mgr.25414 ' entity='mgr.x' 2023-09-23T20:15:20.147 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:19 smithi149 ceph-mon[100764]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:15:20.147 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:19 smithi149 ceph-mon[100764]: from='mgr.25414 172.21.15.149:0/3528169943' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-09-23T20:15:20.147 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:19 smithi149 ceph-mon[100764]: from='mgr.25414 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-09-23T20:15:20.147 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:19 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[141947]: ignoring --setuser ceph since I am not root 2023-09-23T20:15:20.147 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:19 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[141947]: ignoring --setgroup ceph since I am not root 2023-09-23T20:15:20.147 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:19 smithi149 ceph-mgr[141971]: -- 172.21.15.149:0/1166523697 <== mon.1 v2:172.21.15.149:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55c48d85f1e0 con 0x55c48d834c00 2023-09-23T20:15:20.147 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:20 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[141947]: 2023-09-23T20:15:20.059+0000 7fb8bec8a200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-23T20:15:20.147 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:20 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[141947]: 2023-09-23T20:15:20.142+0000 7fb8bec8a200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T20:15:20.466 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:20 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[141947]: 2023-09-23T20:15:20.232+0000 7fb8bec8a200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T20:15:20.466 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:20 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[141947]: 2023-09-23T20:15:20.329+0000 7fb8bec8a200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T20:15:20.892 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:20 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[141947]: 2023-09-23T20:15:20.507+0000 7fb8bec8a200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T20:15:21.169 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:20 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[141947]: 2023-09-23T20:15:20.914+0000 7fb8bec8a200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T20:15:21.169 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:21 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[141947]: 2023-09-23T20:15:21.064+0000 7fb8bec8a200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T20:15:21.169 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:21 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[141947]: 2023-09-23T20:15:21.169+0000 7fb8bec8a200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T20:15:21.170 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:20 smithi149 ceph-mon[100764]: osdmap e89: 8 total, 8 up, 8 in 2023-09-23T20:15:21.170 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:20 smithi149 ceph-mon[100764]: from='mgr.25414 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2023-09-23T20:15:21.170 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:20 smithi149 ceph-mon[100764]: mgrmap e26: y(active, starting, since 0.00533162s) 2023-09-23T20:15:21.170 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:20 smithi149 ceph-mon[100764]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T20:15:21.170 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:20 smithi149 ceph-mon[100764]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:15:21.170 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:20 smithi149 ceph-mon[100764]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:15:21.170 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:20 smithi149 ceph-mon[100764]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi149.urtezf"}]: dispatch 2023-09-23T20:15:21.170 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:20 smithi149 ceph-mon[100764]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi018.hduiol"}]: dispatch 2023-09-23T20:15:21.170 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:20 smithi149 ceph-mon[100764]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T20:15:21.171 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:20 smithi149 ceph-mon[100764]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T20:15:21.171 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:20 smithi149 ceph-mon[100764]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T20:15:21.171 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:20 smithi149 ceph-mon[100764]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T20:15:21.171 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:20 smithi149 ceph-mon[100764]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T20:15:21.171 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:20 smithi149 ceph-mon[100764]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T20:15:21.171 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:20 smithi149 ceph-mon[100764]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T20:15:21.171 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:20 smithi149 ceph-mon[100764]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T20:15:21.171 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:20 smithi149 ceph-mon[100764]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T20:15:21.171 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:20 smithi149 ceph-mon[100764]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T20:15:21.171 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:20 smithi149 ceph-mon[100764]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T20:15:21.171 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:20 smithi149 ceph-mon[100764]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T20:15:21.171 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:20 smithi149 ceph-mon[100764]: Manager daemon y is now available 2023-09-23T20:15:21.171 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:20 smithi149 ceph-mon[100764]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:15:21.172 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:20 smithi149 ceph-mon[100764]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:15:21.172 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:20 smithi149 ceph-mon[100764]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:15:21.172 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:20 smithi149 ceph-mon[100764]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T20:15:21.172 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:20 smithi149 ceph-mon[100764]: from='mgr.15762 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T20:15:21.172 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:20 smithi149 ceph-mon[100764]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T20:15:21.172 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:20 smithi149 ceph-mon[100764]: from='mgr.15762 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T20:15:21.172 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:20 smithi149 ceph-mon[100764]: from='client.? 172.21.15.18:0/642913254' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:15:21.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:20 smithi018 ceph-mon[95889]: osdmap e89: 8 total, 8 up, 8 in 2023-09-23T20:15:21.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:20 smithi018 ceph-mon[95889]: from='mgr.25414 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2023-09-23T20:15:21.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:20 smithi018 ceph-mon[95889]: mgrmap e26: y(active, starting, since 0.00533162s) 2023-09-23T20:15:21.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:20 smithi018 ceph-mon[95889]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T20:15:21.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:20 smithi018 ceph-mon[95889]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:15:21.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:20 smithi018 ceph-mon[95889]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:15:21.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:20 smithi018 ceph-mon[95889]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi149.urtezf"}]: dispatch 2023-09-23T20:15:21.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:20 smithi018 ceph-mon[95889]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi018.hduiol"}]: dispatch 2023-09-23T20:15:21.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:20 smithi018 ceph-mon[95889]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T20:15:21.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:20 smithi018 ceph-mon[95889]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T20:15:21.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:20 smithi018 ceph-mon[95889]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T20:15:21.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:20 smithi018 ceph-mon[95889]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T20:15:21.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:20 smithi018 ceph-mon[95889]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T20:15:21.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:20 smithi018 ceph-mon[95889]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T20:15:21.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:20 smithi018 ceph-mon[95889]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T20:15:21.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:20 smithi018 ceph-mon[95889]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T20:15:21.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:20 smithi018 ceph-mon[95889]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T20:15:21.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:20 smithi018 ceph-mon[95889]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T20:15:21.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:20 smithi018 ceph-mon[95889]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T20:15:21.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:20 smithi018 ceph-mon[95889]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T20:15:21.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:20 smithi018 ceph-mon[95889]: Manager daemon y is now available 2023-09-23T20:15:21.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:20 smithi018 ceph-mon[95889]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:15:21.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:20 smithi018 ceph-mon[95889]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:15:21.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:20 smithi018 ceph-mon[95889]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:15:21.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:20 smithi018 ceph-mon[95889]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T20:15:21.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:20 smithi018 ceph-mon[95889]: from='mgr.15762 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T20:15:21.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:20 smithi018 ceph-mon[95889]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T20:15:21.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:20 smithi018 ceph-mon[95889]: from='mgr.15762 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T20:15:21.394 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:20 smithi018 ceph-mon[95889]: from='client.? 172.21.15.18:0/642913254' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:15:21.394 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:20 smithi018 ceph-mon[109137]: osdmap e89: 8 total, 8 up, 8 in 2023-09-23T20:15:21.394 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:20 smithi018 ceph-mon[109137]: from='mgr.25414 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2023-09-23T20:15:21.394 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:20 smithi018 ceph-mon[109137]: mgrmap e26: y(active, starting, since 0.00533162s) 2023-09-23T20:15:21.394 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:20 smithi018 ceph-mon[109137]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T20:15:21.394 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:20 smithi018 ceph-mon[109137]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:15:21.394 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:20 smithi018 ceph-mon[109137]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:15:21.394 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:20 smithi018 ceph-mon[109137]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi149.urtezf"}]: dispatch 2023-09-23T20:15:21.394 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:20 smithi018 ceph-mon[109137]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi018.hduiol"}]: dispatch 2023-09-23T20:15:21.395 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:20 smithi018 ceph-mon[109137]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T20:15:21.395 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:20 smithi018 ceph-mon[109137]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T20:15:21.395 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:20 smithi018 ceph-mon[109137]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T20:15:21.395 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:20 smithi018 ceph-mon[109137]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T20:15:21.395 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:20 smithi018 ceph-mon[109137]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T20:15:21.395 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:20 smithi018 ceph-mon[109137]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T20:15:21.395 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:20 smithi018 ceph-mon[109137]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T20:15:21.395 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:20 smithi018 ceph-mon[109137]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T20:15:21.395 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:20 smithi018 ceph-mon[109137]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T20:15:21.395 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:20 smithi018 ceph-mon[109137]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T20:15:21.395 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:20 smithi018 ceph-mon[109137]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T20:15:21.396 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:20 smithi018 ceph-mon[109137]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T20:15:21.396 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:20 smithi018 ceph-mon[109137]: Manager daemon y is now available 2023-09-23T20:15:21.396 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:20 smithi018 ceph-mon[109137]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:15:21.396 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:20 smithi018 ceph-mon[109137]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:15:21.396 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:20 smithi018 ceph-mon[109137]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:15:21.396 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:20 smithi018 ceph-mon[109137]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T20:15:21.396 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:20 smithi018 ceph-mon[109137]: from='mgr.15762 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T20:15:21.396 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:20 smithi018 ceph-mon[109137]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T20:15:21.396 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:20 smithi018 ceph-mon[109137]: from='mgr.15762 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T20:15:21.396 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:20 smithi018 ceph-mon[109137]: from='client.? 172.21.15.18:0/642913254' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:15:21.466 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:21 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[141947]: 2023-09-23T20:15:21.268+0000 7fb8bec8a200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T20:15:22.216 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:21 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[141947]: 2023-09-23T20:15:21.735+0000 7fb8bec8a200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T20:15:22.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:21 smithi149 ceph-mon[100764]: [23/Sep/2023:20:15:20] ENGINE Bus STARTING 2023-09-23T20:15:22.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:21 smithi149 ceph-mon[100764]: mgrmap e27: y(active, since 1.02123s) 2023-09-23T20:15:22.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:21 smithi149 ceph-mon[100764]: pgmap v3: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:15:22.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:21 smithi149 ceph-mon[100764]: [23/Sep/2023:20:15:20] ENGINE Serving on http://172.21.15.18:8765 2023-09-23T20:15:22.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:21 smithi018 ceph-mon[109137]: [23/Sep/2023:20:15:20] ENGINE Bus STARTING 2023-09-23T20:15:22.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:21 smithi018 ceph-mon[109137]: mgrmap e27: y(active, since 1.02123s) 2023-09-23T20:15:22.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:21 smithi018 ceph-mon[109137]: pgmap v3: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:15:22.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:21 smithi018 ceph-mon[109137]: [23/Sep/2023:20:15:20] ENGINE Serving on http://172.21.15.18:8765 2023-09-23T20:15:22.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:21 smithi018 ceph-mon[95889]: [23/Sep/2023:20:15:20] ENGINE Bus STARTING 2023-09-23T20:15:22.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:21 smithi018 ceph-mon[95889]: mgrmap e27: y(active, since 1.02123s) 2023-09-23T20:15:22.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:21 smithi018 ceph-mon[95889]: pgmap v3: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:15:22.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:21 smithi018 ceph-mon[95889]: [23/Sep/2023:20:15:20] ENGINE Serving on http://172.21.15.18:8765 2023-09-23T20:15:22.716 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:22 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[141947]: 2023-09-23T20:15:22.456+0000 7fb8bec8a200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T20:15:22.716 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:22 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[141947]: 2023-09-23T20:15:22.546+0000 7fb8bec8a200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T20:15:23.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:22 smithi149 ceph-mon[100764]: [23/Sep/2023:20:15:21] ENGINE Serving on https://172.21.15.18:7150 2023-09-23T20:15:23.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:22 smithi149 ceph-mon[100764]: [23/Sep/2023:20:15:21] ENGINE Bus STARTED 2023-09-23T20:15:23.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:22 smithi149 ceph-mon[100764]: pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:15:23.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:22 smithi149 ceph-mon[100764]: mgrmap e28: y(active, since 2s) 2023-09-23T20:15:23.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:22 smithi018 ceph-mon[109137]: [23/Sep/2023:20:15:21] ENGINE Serving on https://172.21.15.18:7150 2023-09-23T20:15:23.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:22 smithi018 ceph-mon[109137]: [23/Sep/2023:20:15:21] ENGINE Bus STARTED 2023-09-23T20:15:23.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:22 smithi018 ceph-mon[109137]: pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:15:23.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:22 smithi018 ceph-mon[109137]: mgrmap e28: y(active, since 2s) 2023-09-23T20:15:23.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:22 smithi018 ceph-mon[95889]: [23/Sep/2023:20:15:21] ENGINE Serving on https://172.21.15.18:7150 2023-09-23T20:15:23.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:22 smithi018 ceph-mon[95889]: [23/Sep/2023:20:15:21] ENGINE Bus STARTED 2023-09-23T20:15:23.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:22 smithi018 ceph-mon[95889]: pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:15:23.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:22 smithi018 ceph-mon[95889]: mgrmap e28: y(active, since 2s) 2023-09-23T20:15:23.687 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:23 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[141947]: 2023-09-23T20:15:23.399+0000 7fb8bec8a200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T20:15:23.688 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:23 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[141947]: 2023-09-23T20:15:23.504+0000 7fb8bec8a200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-23T20:15:23.688 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:23 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[141947]: 2023-09-23T20:15:23.688+0000 7fb8bec8a200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T20:15:23.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:23 smithi149 ceph-mon[100764]: from='client.? 172.21.15.18:0/2411627707' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:15:24.258 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:23 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[141947]: 2023-09-23T20:15:23.998+0000 7fb8bec8a200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-23T20:15:24.258 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:24 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[141947]: 2023-09-23T20:15:24.258+0000 7fb8bec8a200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T20:15:24.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:23 smithi018 ceph-mon[109137]: from='client.? 172.21.15.18:0/2411627707' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:15:24.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:23 smithi018 ceph-mon[95889]: from='client.? 172.21.15.18:0/2411627707' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:15:24.705 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:24 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[141947]: 2023-09-23T20:15:24.419+0000 7fb8bec8a200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T20:15:24.966 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:24 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[141947]: 2023-09-23T20:15:24.703+0000 7fb8bec8a200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T20:15:24.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:24 smithi149 ceph-mon[100764]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:15:24.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:24 smithi149 ceph-mon[100764]: from='mgr.15762 ' entity='mgr.y' 2023-09-23T20:15:24.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:24 smithi149 ceph-mon[100764]: from='mgr.15762 ' entity='mgr.y' 2023-09-23T20:15:25.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:24 smithi018 ceph-mon[109137]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:15:25.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:24 smithi018 ceph-mon[109137]: from='mgr.15762 ' entity='mgr.y' 2023-09-23T20:15:25.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:24 smithi018 ceph-mon[109137]: from='mgr.15762 ' entity='mgr.y' 2023-09-23T20:15:25.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:24 smithi018 ceph-mon[95889]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:15:25.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:24 smithi018 ceph-mon[95889]: from='mgr.15762 ' entity='mgr.y' 2023-09-23T20:15:25.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:24 smithi018 ceph-mon[95889]: from='mgr.15762 ' entity='mgr.y' 2023-09-23T20:15:25.840 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:25 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[141947]: 2023-09-23T20:15:25.494+0000 7fb8bec8a200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T20:15:25.840 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:25 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[141947]: 2023-09-23T20:15:25.630+0000 7fb8bec8a200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T20:15:26.195 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:25 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[141947]: 2023-09-23T20:15:25.840+0000 7fb8bec8a200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T20:15:26.195 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:25 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[141947]: 2023-09-23T20:15:25.963+0000 7fb8bec8a200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T20:15:26.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:26 smithi018 ceph-mon[109137]: from='mgr.15762 ' entity='mgr.y' 2023-09-23T20:15:26.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:26 smithi018 ceph-mon[109137]: from='mgr.15762 ' entity='mgr.y' 2023-09-23T20:15:26.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:26 smithi018 ceph-mon[109137]: from='mgr.15762 ' entity='mgr.y' 2023-09-23T20:15:26.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:26 smithi018 ceph-mon[109137]: from='mgr.15762 ' entity='mgr.y' 2023-09-23T20:15:26.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:26 smithi018 ceph-mon[109137]: from='client.? 172.21.15.18:0/67514910' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:15:26.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:26 smithi018 ceph-mon[109137]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:15:26.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:26 smithi018 ceph-mon[109137]: from='mgr.15762 ' entity='mgr.y' 2023-09-23T20:15:26.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:26 smithi018 ceph-mon[109137]: from='mgr.15762 ' entity='mgr.y' 2023-09-23T20:15:26.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:26 smithi018 ceph-mon[109137]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi149", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:15:26.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:26 smithi018 ceph-mon[109137]: from='mgr.15762 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi149", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:15:26.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:26 smithi018 ceph-mon[95889]: from='mgr.15762 ' entity='mgr.y' 2023-09-23T20:15:26.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:26 smithi018 ceph-mon[95889]: from='mgr.15762 ' entity='mgr.y' 2023-09-23T20:15:26.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:26 smithi018 ceph-mon[95889]: from='mgr.15762 ' entity='mgr.y' 2023-09-23T20:15:26.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:26 smithi018 ceph-mon[95889]: from='mgr.15762 ' entity='mgr.y' 2023-09-23T20:15:26.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:26 smithi018 ceph-mon[95889]: from='client.? 172.21.15.18:0/67514910' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:15:26.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:26 smithi018 ceph-mon[95889]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:15:26.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:26 smithi018 ceph-mon[95889]: from='mgr.15762 ' entity='mgr.y' 2023-09-23T20:15:26.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:26 smithi018 ceph-mon[95889]: from='mgr.15762 ' entity='mgr.y' 2023-09-23T20:15:26.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:26 smithi018 ceph-mon[95889]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi149", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:15:26.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:26 smithi018 ceph-mon[95889]: from='mgr.15762 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi149", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:15:26.466 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[141947]: 2023-09-23T20:15:26.195+0000 7fb8bec8a200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T20:15:26.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:26 smithi149 ceph-mon[100764]: from='mgr.15762 ' entity='mgr.y' 2023-09-23T20:15:26.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:26 smithi149 ceph-mon[100764]: from='mgr.15762 ' entity='mgr.y' 2023-09-23T20:15:26.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:26 smithi149 ceph-mon[100764]: from='mgr.15762 ' entity='mgr.y' 2023-09-23T20:15:26.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:26 smithi149 ceph-mon[100764]: from='mgr.15762 ' entity='mgr.y' 2023-09-23T20:15:26.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:26 smithi149 ceph-mon[100764]: from='client.? 172.21.15.18:0/67514910' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:15:26.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:26 smithi149 ceph-mon[100764]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:15:26.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:26 smithi149 ceph-mon[100764]: from='mgr.15762 ' entity='mgr.y' 2023-09-23T20:15:26.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:26 smithi149 ceph-mon[100764]: from='mgr.15762 ' entity='mgr.y' 2023-09-23T20:15:26.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:26 smithi149 ceph-mon[100764]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi149", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:15:26.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:26 smithi149 ceph-mon[100764]: from='mgr.15762 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi149", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:15:27.355 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[141947]: 2023-09-23T20:15:27.073+0000 7fb8bec8a200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T20:15:27.355 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[141947]: [23/Sep/2023:20:15:27] ENGINE Bus STARTING 2023-09-23T20:15:27.355 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[141947]: CherryPy Checker: 2023-09-23T20:15:27.355 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[141947]: The Application mounted at '' has an empty config. 2023-09-23T20:15:27.355 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[141947]: 2023-09-23T20:15:27.355 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[141947]: [23/Sep/2023:20:15:27] ENGINE Serving on http://:::9283 2023-09-23T20:15:27.356 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[141947]: [23/Sep/2023:20:15:27] ENGINE Bus STARTED 2023-09-23T20:15:27.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:27 smithi018 ceph-mon[95889]: Standby manager daemon x started 2023-09-23T20:15:27.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:27 smithi018 ceph-mon[95889]: from='mgr.? 172.21.15.149:0/829346127' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T20:15:27.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:27 smithi018 ceph-mon[95889]: from='mgr.? 172.21.15.149:0/829346127' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T20:15:27.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:27 smithi018 ceph-mon[95889]: from='mgr.? 172.21.15.149:0/829346127' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T20:15:27.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:27 smithi018 ceph-mon[95889]: from='mgr.? 172.21.15.149:0/829346127' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T20:15:27.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:27 smithi018 ceph-mon[109137]: Standby manager daemon x started 2023-09-23T20:15:27.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:27 smithi018 ceph-mon[109137]: from='mgr.? 172.21.15.149:0/829346127' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T20:15:27.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:27 smithi018 ceph-mon[109137]: from='mgr.? 172.21.15.149:0/829346127' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T20:15:27.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:27 smithi018 ceph-mon[109137]: from='mgr.? 172.21.15.149:0/829346127' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T20:15:27.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:27 smithi018 ceph-mon[109137]: from='mgr.? 172.21.15.149:0/829346127' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T20:15:27.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:27 smithi149 ceph-mon[100764]: Standby manager daemon x started 2023-09-23T20:15:27.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:27 smithi149 ceph-mon[100764]: from='mgr.? 172.21.15.149:0/829346127' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T20:15:27.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:27 smithi149 ceph-mon[100764]: from='mgr.? 172.21.15.149:0/829346127' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T20:15:27.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:27 smithi149 ceph-mon[100764]: from='mgr.? 172.21.15.149:0/829346127' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T20:15:27.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:27 smithi149 ceph-mon[100764]: from='mgr.? 172.21.15.149:0/829346127' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T20:15:28.640 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:28 smithi018 ceph-mon[95889]: mgrmap e29: y(active, since 7s), standbys: x 2023-09-23T20:15:28.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:28 smithi018 ceph-mon[95889]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T20:15:28.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:28 smithi018 ceph-mon[95889]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:15:28.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:28 smithi018 ceph-mon[95889]: from='client.? 172.21.15.18:0/89265027' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:15:28.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:28 smithi018 ceph-mon[109137]: mgrmap e29: y(active, since 7s), standbys: x 2023-09-23T20:15:28.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:28 smithi018 ceph-mon[109137]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T20:15:28.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:28 smithi018 ceph-mon[109137]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:15:28.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:28 smithi018 ceph-mon[109137]: from='client.? 172.21.15.18:0/89265027' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:15:28.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:28 smithi149 ceph-mon[100764]: mgrmap e29: y(active, since 7s), standbys: x 2023-09-23T20:15:28.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:28 smithi149 ceph-mon[100764]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T20:15:28.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:28 smithi149 ceph-mon[100764]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:15:28.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:28 smithi149 ceph-mon[100764]: from='client.? 172.21.15.18:0/89265027' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:15:29.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:29 smithi149 ceph-mon[100764]: from='mgr.15762 ' entity='mgr.y' 2023-09-23T20:15:29.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:29 smithi149 ceph-mon[100764]: from='mgr.15762 ' entity='mgr.y' 2023-09-23T20:15:29.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:29 smithi149 ceph-mon[100764]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:15:29.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:29 smithi149 ceph-mon[100764]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:15:29.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:29 smithi149 ceph-mon[100764]: Updating smithi018:/etc/ceph/ceph.conf 2023-09-23T20:15:29.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:29 smithi149 ceph-mon[100764]: Updating smithi149:/etc/ceph/ceph.conf 2023-09-23T20:15:29.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:29 smithi149 ceph-mon[100764]: Updating smithi149:/var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T20:15:29.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:29 smithi149 ceph-mon[100764]: Updating smithi018:/var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T20:15:29.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:29 smithi149 ceph-mon[100764]: Updating smithi149:/etc/ceph/ceph.client.admin.keyring 2023-09-23T20:15:29.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:29 smithi149 ceph-mon[100764]: Updating smithi018:/etc/ceph/ceph.client.admin.keyring 2023-09-23T20:15:29.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:29 smithi018 ceph-mon[109137]: from='mgr.15762 ' entity='mgr.y' 2023-09-23T20:15:29.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:29 smithi018 ceph-mon[109137]: from='mgr.15762 ' entity='mgr.y' 2023-09-23T20:15:29.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:29 smithi018 ceph-mon[109137]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:15:29.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:29 smithi018 ceph-mon[109137]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:15:29.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:29 smithi018 ceph-mon[109137]: Updating smithi018:/etc/ceph/ceph.conf 2023-09-23T20:15:29.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:29 smithi018 ceph-mon[109137]: Updating smithi149:/etc/ceph/ceph.conf 2023-09-23T20:15:29.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:29 smithi018 ceph-mon[109137]: Updating smithi149:/var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T20:15:29.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:29 smithi018 ceph-mon[109137]: Updating smithi018:/var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T20:15:29.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:29 smithi018 ceph-mon[109137]: Updating smithi149:/etc/ceph/ceph.client.admin.keyring 2023-09-23T20:15:29.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:29 smithi018 ceph-mon[109137]: Updating smithi018:/etc/ceph/ceph.client.admin.keyring 2023-09-23T20:15:29.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:29 smithi018 ceph-mon[95889]: from='mgr.15762 ' entity='mgr.y' 2023-09-23T20:15:29.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:29 smithi018 ceph-mon[95889]: from='mgr.15762 ' entity='mgr.y' 2023-09-23T20:15:29.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:29 smithi018 ceph-mon[95889]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:15:29.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:29 smithi018 ceph-mon[95889]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:15:29.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:29 smithi018 ceph-mon[95889]: Updating smithi018:/etc/ceph/ceph.conf 2023-09-23T20:15:29.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:29 smithi018 ceph-mon[95889]: Updating smithi149:/etc/ceph/ceph.conf 2023-09-23T20:15:29.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:29 smithi018 ceph-mon[95889]: Updating smithi149:/var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T20:15:29.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:29 smithi018 ceph-mon[95889]: Updating smithi018:/var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T20:15:29.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:29 smithi018 ceph-mon[95889]: Updating smithi149:/etc/ceph/ceph.client.admin.keyring 2023-09-23T20:15:29.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:29 smithi018 ceph-mon[95889]: Updating smithi018:/etc/ceph/ceph.client.admin.keyring 2023-09-23T20:15:30.803 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:30 smithi149 ceph-mon[100764]: Updating smithi018:/var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T20:15:30.803 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:30 smithi149 ceph-mon[100764]: Updating smithi149:/var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T20:15:30.803 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:30 smithi149 ceph-mon[100764]: from='mgr.15762 ' entity='mgr.y' 2023-09-23T20:15:30.803 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:30 smithi149 ceph-mon[100764]: from='mgr.15762 ' entity='mgr.y' 2023-09-23T20:15:30.803 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:30 smithi149 ceph-mon[100764]: from='mgr.15762 ' entity='mgr.y' 2023-09-23T20:15:30.804 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:30 smithi149 ceph-mon[100764]: from='mgr.15762 ' entity='mgr.y' 2023-09-23T20:15:30.804 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:30 smithi149 ceph-mon[100764]: from='mgr.15762 ' entity='mgr.y' 2023-09-23T20:15:30.804 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:30 smithi149 ceph-mon[100764]: Reconfiguring prometheus.a (dependencies changed)... 2023-09-23T20:15:30.804 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:30 smithi149 ceph-mon[100764]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:15:30.804 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:30 smithi149 ceph-mon[100764]: Reconfiguring daemon prometheus.a on smithi149 2023-09-23T20:15:30.804 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:30 smithi149 ceph-mon[100764]: from='client.? 172.21.15.18:0/1112660672' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:15:30.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:30 smithi018 ceph-mon[109137]: Updating smithi018:/var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T20:15:30.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:30 smithi018 ceph-mon[109137]: Updating smithi149:/var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T20:15:30.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:30 smithi018 ceph-mon[109137]: from='mgr.15762 ' entity='mgr.y' 2023-09-23T20:15:30.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:30 smithi018 ceph-mon[109137]: from='mgr.15762 ' entity='mgr.y' 2023-09-23T20:15:30.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:30 smithi018 ceph-mon[109137]: from='mgr.15762 ' entity='mgr.y' 2023-09-23T20:15:30.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:30 smithi018 ceph-mon[109137]: from='mgr.15762 ' entity='mgr.y' 2023-09-23T20:15:30.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:30 smithi018 ceph-mon[109137]: from='mgr.15762 ' entity='mgr.y' 2023-09-23T20:15:30.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:30 smithi018 ceph-mon[109137]: Reconfiguring prometheus.a (dependencies changed)... 2023-09-23T20:15:30.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:30 smithi018 ceph-mon[109137]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:15:30.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:30 smithi018 ceph-mon[109137]: Reconfiguring daemon prometheus.a on smithi149 2023-09-23T20:15:30.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:30 smithi018 ceph-mon[109137]: from='client.? 172.21.15.18:0/1112660672' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:15:30.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:30 smithi018 ceph-mon[95889]: Updating smithi018:/var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T20:15:30.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:30 smithi018 ceph-mon[95889]: Updating smithi149:/var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T20:15:30.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:30 smithi018 ceph-mon[95889]: from='mgr.15762 ' entity='mgr.y' 2023-09-23T20:15:30.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:30 smithi018 ceph-mon[95889]: from='mgr.15762 ' entity='mgr.y' 2023-09-23T20:15:30.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:30 smithi018 ceph-mon[95889]: from='mgr.15762 ' entity='mgr.y' 2023-09-23T20:15:30.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:30 smithi018 ceph-mon[95889]: from='mgr.15762 ' entity='mgr.y' 2023-09-23T20:15:30.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:30 smithi018 ceph-mon[95889]: from='mgr.15762 ' entity='mgr.y' 2023-09-23T20:15:30.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:30 smithi018 ceph-mon[95889]: Reconfiguring prometheus.a (dependencies changed)... 2023-09-23T20:15:30.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:30 smithi018 ceph-mon[95889]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:15:30.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:30 smithi018 ceph-mon[95889]: Reconfiguring daemon prometheus.a on smithi149 2023-09-23T20:15:30.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:30 smithi018 ceph-mon[95889]: from='client.? 172.21.15.18:0/1112660672' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:15:32.167 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:15:31 smithi149 systemd[1]: Stopping Ceph prometheus.a for a3b3b17e-5a4c-11ee-8db4-212e2dc638e7... 2023-09-23T20:15:32.433 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:15:32 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[152938]: ts=2023-09-23T20:15:32.166Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2023-09-23T20:15:32.433 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:15:32 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[152938]: ts=2023-09-23T20:15:32.167Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2023-09-23T20:15:32.433 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:15:32 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[152938]: ts=2023-09-23T20:15:32.167Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2023-09-23T20:15:32.433 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:15:32 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[152938]: ts=2023-09-23T20:15:32.167Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2023-09-23T20:15:32.433 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:15:32 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[152938]: ts=2023-09-23T20:15:32.167Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2023-09-23T20:15:32.433 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:15:32 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[152938]: ts=2023-09-23T20:15:32.167Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2023-09-23T20:15:32.433 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:15:32 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[152938]: ts=2023-09-23T20:15:32.167Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2023-09-23T20:15:32.433 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:15:32 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[152938]: ts=2023-09-23T20:15:32.167Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2023-09-23T20:15:32.434 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:15:32 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[152938]: ts=2023-09-23T20:15:32.167Z caller=main.go:907 level=info msg="Scrape manager stopped" 2023-09-23T20:15:32.434 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:15:32 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[152938]: ts=2023-09-23T20:15:32.167Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2023-09-23T20:15:32.434 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:15:32 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[152938]: ts=2023-09-23T20:15:32.167Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2023-09-23T20:15:32.434 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:15:32 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[152938]: ts=2023-09-23T20:15:32.168Z caller=main.go:1147 level=info msg="See you next time!" 2023-09-23T20:15:32.434 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:15:32 smithi149 bash[156368]: ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a 2023-09-23T20:15:32.913 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:15:32 smithi149 systemd[1]: ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7@prometheus.a.service: Succeeded. 2023-09-23T20:15:32.913 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:15:32 smithi149 systemd[1]: Stopped Ceph prometheus.a for a3b3b17e-5a4c-11ee-8db4-212e2dc638e7. 2023-09-23T20:15:32.913 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:15:32 smithi149 systemd[1]: Starting Ceph prometheus.a for a3b3b17e-5a4c-11ee-8db4-212e2dc638e7... 2023-09-23T20:15:33.201 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:32 smithi149 ceph-mon[100764]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:15:33.201 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:32 smithi149 ceph-mon[100764]: from='client.? 172.21.15.18:0/3521373153' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:15:33.202 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:15:32 smithi149 podman[156491]: 2023-09-23T20:15:33.353 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:32 smithi018 ceph-mon[109137]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:15:33.353 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:32 smithi018 ceph-mon[109137]: from='client.? 172.21.15.18:0/3521373153' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:15:33.354 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:15:33 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: [23/Sep/2023:20:15:33] ENGINE Bus STOPPING 2023-09-23T20:15:33.354 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:32 smithi018 ceph-mon[95889]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:15:33.354 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:32 smithi018 ceph-mon[95889]: from='client.? 172.21.15.18:0/3521373153' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:15:33.466 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:15:33 smithi149 bash[156491]: bb2162e75dc72140a2b876b88b0c707f25fd952f59f454608aad662b97483cf9 2023-09-23T20:15:33.466 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:15:33 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[156506]: ts=2023-09-23T20:15:33.202Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2023-09-23T20:15:33.466 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:15:33 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[156506]: ts=2023-09-23T20:15:33.202Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2023-09-23T20:15:33.466 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:15:33 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[156506]: ts=2023-09-23T20:15:33.202Z caller=main.go:570 level=info host_details="(Linux 4.18.0-372.9.1.el8.x86_64 #1 SMP Fri Apr 15 22:12:19 EDT 2022 x86_64 smithi149 (none))" 2023-09-23T20:15:33.466 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:15:33 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[156506]: ts=2023-09-23T20:15:33.202Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2023-09-23T20:15:33.467 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:15:33 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[156506]: ts=2023-09-23T20:15:33.202Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2023-09-23T20:15:33.467 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:15:33 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[156506]: ts=2023-09-23T20:15:33.205Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2023-09-23T20:15:33.467 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:15:33 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[156506]: ts=2023-09-23T20:15:33.205Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2023-09-23T20:15:33.467 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:15:33 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[156506]: ts=2023-09-23T20:15:33.207Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2023-09-23T20:15:33.467 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:15:33 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[156506]: ts=2023-09-23T20:15:33.207Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2023-09-23T20:15:33.467 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:15:33 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[156506]: ts=2023-09-23T20:15:33.208Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2023-09-23T20:15:33.467 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:15:33 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[156506]: ts=2023-09-23T20:15:33.208Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.315µs 2023-09-23T20:15:33.467 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:15:33 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[156506]: ts=2023-09-23T20:15:33.208Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2023-09-23T20:15:33.467 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:15:33 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[156506]: ts=2023-09-23T20:15:33.208Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=3 2023-09-23T20:15:33.468 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:15:33 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[156506]: ts=2023-09-23T20:15:33.224Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=3 2023-09-23T20:15:33.468 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:15:33 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[156506]: ts=2023-09-23T20:15:33.226Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=3 2023-09-23T20:15:33.468 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:15:33 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[156506]: ts=2023-09-23T20:15:33.226Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=3 2023-09-23T20:15:33.468 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:15:33 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[156506]: ts=2023-09-23T20:15:33.226Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=21.139µs wal_replay_duration=18.34684ms wbl_replay_duration=107ns total_replay_duration=18.389883ms 2023-09-23T20:15:33.468 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:15:33 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[156506]: ts=2023-09-23T20:15:33.229Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2023-09-23T20:15:33.468 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:15:33 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[156506]: ts=2023-09-23T20:15:33.229Z caller=main.go:1029 level=info msg="TSDB started" 2023-09-23T20:15:33.468 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:15:33 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[156506]: ts=2023-09-23T20:15:33.229Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2023-09-23T20:15:33.468 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:15:33 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[156506]: ts=2023-09-23T20:15:33.243Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=14.23147ms db_storage=809ns remote_storage=1.343µs web_handler=416ns query_engine=718ns scrape=487.719µs scrape_sd=70.513µs notify=10.831µs notify_sd=7.988µs rules=13.435307ms tracing=5.659µs 2023-09-23T20:15:33.469 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:15:33 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[156506]: ts=2023-09-23T20:15:33.243Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2023-09-23T20:15:33.469 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:15:33 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[156506]: ts=2023-09-23T20:15:33.243Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2023-09-23T20:15:33.469 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:15:33 smithi149 systemd[1]: Started Ceph prometheus.a for a3b3b17e-5a4c-11ee-8db4-212e2dc638e7. 2023-09-23T20:15:33.641 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:15:33 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: [23/Sep/2023:20:15:33] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T20:15:33.641 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:15:33 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: [23/Sep/2023:20:15:33] ENGINE Bus STOPPED 2023-09-23T20:15:33.641 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:15:33 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: [23/Sep/2023:20:15:33] ENGINE Bus STARTING 2023-09-23T20:15:33.641 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:15:33 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: [23/Sep/2023:20:15:33] ENGINE Serving on http://:::9283 2023-09-23T20:15:33.641 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:15:33 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: [23/Sep/2023:20:15:33] ENGINE Bus STARTED 2023-09-23T20:15:34.590 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:34 smithi149 ceph-mon[100764]: from='mgr.15762 ' entity='mgr.y' 2023-09-23T20:15:34.591 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:34 smithi149 ceph-mon[100764]: from='mgr.15762 ' entity='mgr.y' 2023-09-23T20:15:34.591 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:34 smithi149 ceph-mon[100764]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T20:15:34.591 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:34 smithi149 ceph-mon[100764]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T20:15:34.591 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:34 smithi149 ceph-mon[100764]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi149.front.sepia.ceph.com:9095"}]: dispatch 2023-09-23T20:15:34.591 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:34 smithi149 ceph-mon[100764]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi149.front.sepia.ceph.com:9095"}]: dispatch 2023-09-23T20:15:34.591 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:34 smithi149 ceph-mon[100764]: from='mgr.15762 ' entity='mgr.y' 2023-09-23T20:15:34.591 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:34 smithi149 ceph-mon[100764]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:15:34.591 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:34 smithi149 ceph-mon[100764]: pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:15:34.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:34 smithi018 ceph-mon[109137]: from='mgr.15762 ' entity='mgr.y' 2023-09-23T20:15:34.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:34 smithi018 ceph-mon[109137]: from='mgr.15762 ' entity='mgr.y' 2023-09-23T20:15:34.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:34 smithi018 ceph-mon[109137]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T20:15:34.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:34 smithi018 ceph-mon[109137]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T20:15:34.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:34 smithi018 ceph-mon[109137]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi149.front.sepia.ceph.com:9095"}]: dispatch 2023-09-23T20:15:34.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:34 smithi018 ceph-mon[109137]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi149.front.sepia.ceph.com:9095"}]: dispatch 2023-09-23T20:15:34.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:34 smithi018 ceph-mon[109137]: from='mgr.15762 ' entity='mgr.y' 2023-09-23T20:15:34.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:34 smithi018 ceph-mon[109137]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:15:34.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:34 smithi018 ceph-mon[109137]: pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:15:34.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:34 smithi018 ceph-mon[95889]: from='mgr.15762 ' entity='mgr.y' 2023-09-23T20:15:34.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:34 smithi018 ceph-mon[95889]: from='mgr.15762 ' entity='mgr.y' 2023-09-23T20:15:34.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:34 smithi018 ceph-mon[95889]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T20:15:34.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:34 smithi018 ceph-mon[95889]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T20:15:34.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:34 smithi018 ceph-mon[95889]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi149.front.sepia.ceph.com:9095"}]: dispatch 2023-09-23T20:15:34.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:34 smithi018 ceph-mon[95889]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi149.front.sepia.ceph.com:9095"}]: dispatch 2023-09-23T20:15:34.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:34 smithi018 ceph-mon[95889]: from='mgr.15762 ' entity='mgr.y' 2023-09-23T20:15:34.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:34 smithi018 ceph-mon[95889]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:15:34.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:34 smithi018 ceph-mon[95889]: pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:15:35.419 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:35 smithi149 ceph-mon[100764]: Upgrade: Updating mgr.x 2023-09-23T20:15:35.419 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:35 smithi149 ceph-mon[100764]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T20:15:35.419 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:35 smithi149 ceph-mon[100764]: from='mgr.15762 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T20:15:35.419 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:35 smithi149 ceph-mon[100764]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T20:15:35.419 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:35 smithi149 ceph-mon[100764]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:15:35.419 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:35 smithi149 ceph-mon[100764]: Deploying daemon mgr.x on smithi149 2023-09-23T20:15:35.419 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:35 smithi149 ceph-mon[100764]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:15:35.419 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:35 smithi149 ceph-mon[100764]: from='client.? 172.21.15.18:0/2877356774' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:15:35.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:35 smithi018 ceph-mon[109137]: Upgrade: Updating mgr.x 2023-09-23T20:15:35.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:35 smithi018 ceph-mon[109137]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T20:15:35.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:35 smithi018 ceph-mon[109137]: from='mgr.15762 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T20:15:35.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:35 smithi018 ceph-mon[109137]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T20:15:35.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:35 smithi018 ceph-mon[109137]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:15:35.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:35 smithi018 ceph-mon[109137]: Deploying daemon mgr.x on smithi149 2023-09-23T20:15:35.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:35 smithi018 ceph-mon[109137]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:15:35.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:35 smithi018 ceph-mon[109137]: from='client.? 172.21.15.18:0/2877356774' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:15:35.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:35 smithi018 ceph-mon[95889]: Upgrade: Updating mgr.x 2023-09-23T20:15:35.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:35 smithi018 ceph-mon[95889]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T20:15:35.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:35 smithi018 ceph-mon[95889]: from='mgr.15762 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T20:15:35.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:35 smithi018 ceph-mon[95889]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T20:15:35.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:35 smithi018 ceph-mon[95889]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:15:35.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:35 smithi018 ceph-mon[95889]: Deploying daemon mgr.x on smithi149 2023-09-23T20:15:35.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:35 smithi018 ceph-mon[95889]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:15:35.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:35 smithi018 ceph-mon[95889]: from='client.? 172.21.15.18:0/2877356774' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:15:35.966 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:35 smithi149 systemd[1]: Stopping Ceph mgr.x for a3b3b17e-5a4c-11ee-8db4-212e2dc638e7... 2023-09-23T20:15:36.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:36 smithi149 ceph-mon[100764]: pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:15:36.466 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:36 smithi149 bash[156899]: ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x 2023-09-23T20:15:36.466 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:36 smithi149 systemd[1]: ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7@mgr.x.service: Main process exited, code=exited, status=143/n/a 2023-09-23T20:15:36.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:36 smithi018 ceph-mon[109137]: pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:15:36.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:36 smithi018 ceph-mon[95889]: pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:15:36.966 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:36 smithi149 systemd[1]: ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7@mgr.x.service: Failed with result 'exit-code'. 2023-09-23T20:15:36.966 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:36 smithi149 systemd[1]: Stopped Ceph mgr.x for a3b3b17e-5a4c-11ee-8db4-212e2dc638e7. 2023-09-23T20:15:36.966 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:36 smithi149 systemd[1]: Starting Ceph mgr.x for a3b3b17e-5a4c-11ee-8db4-212e2dc638e7... 2023-09-23T20:15:37.361 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:37 smithi149 podman[157047]: 2023-09-23T20:15:37.361 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:37 smithi149 bash[157047]: 9513b5c2a2f7a50f20df5ec69950948190174ffbaa65d1c3e68c9ad8875be785 2023-09-23T20:15:37.361 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:37 smithi149 ceph-mgr[157087]: -- 172.21.15.149:0/3421967298 <== mon.1 v2:172.21.15.149:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55e6487eb1e0 con 0x55e6487c0c00 2023-09-23T20:15:37.616 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:37 smithi149 systemd[1]: Started Ceph mgr.x for a3b3b17e-5a4c-11ee-8db4-212e2dc638e7. 2023-09-23T20:15:37.616 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:37 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[157063]: 2023-09-23T20:15:37.437+0000 7f92e78aa200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-23T20:15:37.616 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:37 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[157063]: 2023-09-23T20:15:37.521+0000 7f92e78aa200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T20:15:37.616 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:37 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[157063]: 2023-09-23T20:15:37.615+0000 7f92e78aa200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T20:15:37.893 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:37 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[157063]: 2023-09-23T20:15:37.716+0000 7f92e78aa200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T20:15:38.216 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:37 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[157063]: 2023-09-23T20:15:37.892+0000 7f92e78aa200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T20:15:38.642 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:38 smithi149 ceph-mon[100764]: from='mgr.15762 ' entity='mgr.y' 2023-09-23T20:15:38.642 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:38 smithi149 ceph-mon[100764]: from='mgr.15762 ' entity='mgr.y' 2023-09-23T20:15:38.643 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:38 smithi149 ceph-mon[100764]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:15:38.643 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:38 smithi149 ceph-mon[100764]: from='client.? 172.21.15.18:0/2799256446' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:15:38.643 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:38 smithi149 ceph-mon[100764]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:15:38.643 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:38 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[157063]: 2023-09-23T20:15:38.289+0000 7f92e78aa200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T20:15:38.643 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:38 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[157063]: 2023-09-23T20:15:38.426+0000 7f92e78aa200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T20:15:38.643 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:38 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[157063]: 2023-09-23T20:15:38.536+0000 7f92e78aa200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T20:15:38.643 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:38 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[157063]: 2023-09-23T20:15:38.641+0000 7f92e78aa200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T20:15:38.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:38 smithi018 ceph-mon[109137]: from='mgr.15762 ' entity='mgr.y' 2023-09-23T20:15:38.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:38 smithi018 ceph-mon[109137]: from='mgr.15762 ' entity='mgr.y' 2023-09-23T20:15:38.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:38 smithi018 ceph-mon[109137]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:15:38.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:38 smithi018 ceph-mon[109137]: from='client.? 172.21.15.18:0/2799256446' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:15:38.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:38 smithi018 ceph-mon[109137]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:15:38.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:38 smithi018 ceph-mon[95889]: from='mgr.15762 ' entity='mgr.y' 2023-09-23T20:15:38.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:38 smithi018 ceph-mon[95889]: from='mgr.15762 ' entity='mgr.y' 2023-09-23T20:15:38.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:38 smithi018 ceph-mon[95889]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:15:38.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:38 smithi018 ceph-mon[95889]: from='client.? 172.21.15.18:0/2799256446' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:15:38.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:38 smithi018 ceph-mon[95889]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:15:39.466 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:39 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[157063]: 2023-09-23T20:15:39.098+0000 7f92e78aa200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T20:15:40.216 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:39 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[157063]: 2023-09-23T20:15:39.829+0000 7f92e78aa200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T20:15:40.216 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:39 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[157063]: 2023-09-23T20:15:39.915+0000 7f92e78aa200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T20:15:41.025 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:40 smithi149 ceph-mon[100764]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:15:41.025 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:40 smithi149 ceph-mon[100764]: from='client.? 172.21.15.18:0/839271606' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:15:41.026 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:40 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[157063]: 2023-09-23T20:15:40.742+0000 7f92e78aa200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T20:15:41.026 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:40 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[157063]: 2023-09-23T20:15:40.844+0000 7f92e78aa200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-23T20:15:41.026 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:41 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[157063]: 2023-09-23T20:15:41.024+0000 7f92e78aa200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T20:15:41.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:40 smithi018 ceph-mon[109137]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:15:41.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:40 smithi018 ceph-mon[109137]: from='client.? 172.21.15.18:0/839271606' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:15:41.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:40 smithi018 ceph-mon[95889]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:15:41.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:40 smithi018 ceph-mon[95889]: from='client.? 172.21.15.18:0/839271606' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:15:41.716 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:41 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[157063]: 2023-09-23T20:15:41.326+0000 7f92e78aa200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-23T20:15:41.716 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:41 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[157063]: 2023-09-23T20:15:41.574+0000 7f92e78aa200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T20:15:42.039 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:41 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[157063]: 2023-09-23T20:15:41.734+0000 7f92e78aa200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T20:15:42.466 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:42 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[157063]: 2023-09-23T20:15:42.038+0000 7f92e78aa200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T20:15:43.163 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:42 smithi149 ceph-mon[100764]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:15:43.163 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:42 smithi149 ceph-mon[100764]: from='client.? 172.21.15.18:0/3133968054' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:15:43.164 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:42 smithi149 ceph-mon[100764]: from='mgr.15762 ' entity='mgr.y' 2023-09-23T20:15:43.164 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:42 smithi149 ceph-mon[100764]: from='mgr.15762 ' entity='mgr.y' 2023-09-23T20:15:43.164 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:42 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[157063]: 2023-09-23T20:15:42.808+0000 7f92e78aa200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T20:15:43.164 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:42 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[157063]: 2023-09-23T20:15:42.952+0000 7f92e78aa200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T20:15:43.394 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:42 smithi018 ceph-mon[109137]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:15:43.395 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:42 smithi018 ceph-mon[109137]: from='client.? 172.21.15.18:0/3133968054' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:15:43.395 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:42 smithi018 ceph-mon[109137]: from='mgr.15762 ' entity='mgr.y' 2023-09-23T20:15:43.395 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:42 smithi018 ceph-mon[109137]: from='mgr.15762 ' entity='mgr.y' 2023-09-23T20:15:43.395 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:42 smithi018 ceph-mon[95889]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:15:43.395 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:42 smithi018 ceph-mon[95889]: from='client.? 172.21.15.18:0/3133968054' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:15:43.396 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:42 smithi018 ceph-mon[95889]: from='mgr.15762 ' entity='mgr.y' 2023-09-23T20:15:43.396 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:42 smithi018 ceph-mon[95889]: from='mgr.15762 ' entity='mgr.y' 2023-09-23T20:15:43.468 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:43 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[157063]: 2023-09-23T20:15:43.161+0000 7f92e78aa200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T20:15:43.468 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:43 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[157063]: 2023-09-23T20:15:43.286+0000 7f92e78aa200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T20:15:43.966 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:43 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[157063]: 2023-09-23T20:15:43.526+0000 7f92e78aa200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T20:15:44.543 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:44 smithi149 ceph-mon[100764]: from='mgr.15762 ' entity='mgr.y' 2023-09-23T20:15:44.543 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:44 smithi149 ceph-mon[100764]: from='mgr.15762 ' entity='mgr.y' 2023-09-23T20:15:44.543 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:44 smithi149 ceph-mon[100764]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:15:44.543 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:44 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[157063]: 2023-09-23T20:15:44.466+0000 7f92e78aa200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T20:15:44.543 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:44 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[157063]: [23/Sep/2023:20:15:44] ENGINE Bus STARTING 2023-09-23T20:15:44.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:44 smithi018 ceph-mon[95889]: from='mgr.15762 ' entity='mgr.y' 2023-09-23T20:15:44.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:44 smithi018 ceph-mon[95889]: from='mgr.15762 ' entity='mgr.y' 2023-09-23T20:15:44.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:44 smithi018 ceph-mon[95889]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:15:44.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:44 smithi018 ceph-mon[109137]: from='mgr.15762 ' entity='mgr.y' 2023-09-23T20:15:44.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:44 smithi018 ceph-mon[109137]: from='mgr.15762 ' entity='mgr.y' 2023-09-23T20:15:44.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:44 smithi018 ceph-mon[109137]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:15:44.966 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:44 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[157063]: CherryPy Checker: 2023-09-23T20:15:44.967 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:44 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[157063]: The Application mounted at '' has an empty config. 2023-09-23T20:15:44.967 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:44 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[157063]: 2023-09-23T20:15:44.967 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:44 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[157063]: [23/Sep/2023:20:15:44] ENGINE Serving on http://:::9283 2023-09-23T20:15:44.967 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:44 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[157063]: [23/Sep/2023:20:15:44] ENGINE Bus STARTED 2023-09-23T20:15:45.391 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:15:44 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:15:44] "GET /metrics HTTP/1.1" 200 33918 "" "Prometheus/2.43.0" 2023-09-23T20:15:45.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:45 smithi149 ceph-mon[100764]: Standby manager daemon x restarted 2023-09-23T20:15:45.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:45 smithi149 ceph-mon[100764]: Standby manager daemon x started 2023-09-23T20:15:45.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:45 smithi149 ceph-mon[100764]: from='mgr.? 172.21.15.149:0/3453384181' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T20:15:45.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:45 smithi149 ceph-mon[100764]: from='mgr.? 172.21.15.149:0/3453384181' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T20:15:45.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:45 smithi149 ceph-mon[100764]: from='mgr.? 172.21.15.149:0/3453384181' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T20:15:45.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:45 smithi149 ceph-mon[100764]: from='mgr.? 172.21.15.149:0/3453384181' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T20:15:45.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:45 smithi149 ceph-mon[100764]: from='client.? 172.21.15.18:0/1780727548' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:15:45.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:45 smithi018 ceph-mon[109137]: Standby manager daemon x restarted 2023-09-23T20:15:45.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:45 smithi018 ceph-mon[109137]: Standby manager daemon x started 2023-09-23T20:15:45.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:45 smithi018 ceph-mon[109137]: from='mgr.? 172.21.15.149:0/3453384181' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T20:15:45.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:45 smithi018 ceph-mon[109137]: from='mgr.? 172.21.15.149:0/3453384181' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T20:15:45.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:45 smithi018 ceph-mon[109137]: from='mgr.? 172.21.15.149:0/3453384181' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T20:15:45.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:45 smithi018 ceph-mon[109137]: from='mgr.? 172.21.15.149:0/3453384181' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T20:15:45.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:45 smithi018 ceph-mon[109137]: from='client.? 172.21.15.18:0/1780727548' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:15:45.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:45 smithi018 ceph-mon[95889]: Standby manager daemon x restarted 2023-09-23T20:15:45.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:45 smithi018 ceph-mon[95889]: Standby manager daemon x started 2023-09-23T20:15:45.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:45 smithi018 ceph-mon[95889]: from='mgr.? 172.21.15.149:0/3453384181' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T20:15:45.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:45 smithi018 ceph-mon[95889]: from='mgr.? 172.21.15.149:0/3453384181' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T20:15:45.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:45 smithi018 ceph-mon[95889]: from='mgr.? 172.21.15.149:0/3453384181' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T20:15:45.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:45 smithi018 ceph-mon[95889]: from='mgr.? 172.21.15.149:0/3453384181' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T20:15:45.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:45 smithi018 ceph-mon[95889]: from='client.? 172.21.15.18:0/1780727548' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:15:46.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:46 smithi149 ceph-mon[100764]: mgrmap e30: y(active, since 25s), standbys: x 2023-09-23T20:15:46.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:46 smithi149 ceph-mon[100764]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:15:46.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:46 smithi149 ceph-mon[100764]: from='mgr.15762 ' entity='mgr.y' 2023-09-23T20:15:46.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:46 smithi149 ceph-mon[100764]: from='mgr.15762 ' entity='mgr.y' 2023-09-23T20:15:46.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:46 smithi149 ceph-mon[100764]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:15:46.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:46 smithi149 ceph-mon[100764]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:15:46.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:46 smithi149 ceph-mon[100764]: from='mgr.15762 ' entity='mgr.y' 2023-09-23T20:15:46.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:46 smithi018 ceph-mon[109137]: mgrmap e30: y(active, since 25s), standbys: x 2023-09-23T20:15:46.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:46 smithi018 ceph-mon[109137]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:15:46.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:46 smithi018 ceph-mon[109137]: from='mgr.15762 ' entity='mgr.y' 2023-09-23T20:15:46.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:46 smithi018 ceph-mon[109137]: from='mgr.15762 ' entity='mgr.y' 2023-09-23T20:15:46.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:46 smithi018 ceph-mon[109137]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:15:46.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:46 smithi018 ceph-mon[109137]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:15:46.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:46 smithi018 ceph-mon[109137]: from='mgr.15762 ' entity='mgr.y' 2023-09-23T20:15:46.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:46 smithi018 ceph-mon[95889]: mgrmap e30: y(active, since 25s), standbys: x 2023-09-23T20:15:46.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:46 smithi018 ceph-mon[95889]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:15:46.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:46 smithi018 ceph-mon[95889]: from='mgr.15762 ' entity='mgr.y' 2023-09-23T20:15:46.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:46 smithi018 ceph-mon[95889]: from='mgr.15762 ' entity='mgr.y' 2023-09-23T20:15:46.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:46 smithi018 ceph-mon[95889]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:15:46.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:46 smithi018 ceph-mon[95889]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:15:46.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:46 smithi018 ceph-mon[95889]: from='mgr.15762 ' entity='mgr.y' 2023-09-23T20:15:47.550 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:47 smithi018 ceph-mon[109137]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:15:47.551 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:47 smithi018 ceph-mon[109137]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:15:47.551 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:47 smithi018 ceph-mon[109137]: Upgrade: Setting container_image for all mgr 2023-09-23T20:15:47.551 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:47 smithi018 ceph-mon[109137]: from='mgr.15762 ' entity='mgr.y' 2023-09-23T20:15:47.551 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:47 smithi018 ceph-mon[109137]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-09-23T20:15:47.551 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:47 smithi018 ceph-mon[109137]: from='mgr.15762 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-09-23T20:15:47.551 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:47 smithi018 ceph-mon[109137]: from='mgr.15762 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2023-09-23T20:15:47.551 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:47 smithi018 ceph-mon[109137]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-09-23T20:15:47.551 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:47 smithi018 ceph-mon[109137]: from='mgr.15762 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-09-23T20:15:47.551 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:47 smithi018 ceph-mon[109137]: from='mgr.15762 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2023-09-23T20:15:47.551 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:47 smithi018 ceph-mon[109137]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-09-23T20:15:47.552 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:47 smithi018 ceph-mon[109137]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2023-09-23T20:15:47.552 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:47 smithi018 ceph-mon[109137]: Upgrade: It appears safe to stop mon.a 2023-09-23T20:15:47.552 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:47 smithi018 ceph-mon[109137]: from='client.? 172.21.15.18:0/1285864501' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:15:47.552 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:47 smithi018 ceph-mon[95889]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:15:47.552 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:47 smithi018 ceph-mon[95889]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:15:47.552 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:47 smithi018 ceph-mon[95889]: Upgrade: Setting container_image for all mgr 2023-09-23T20:15:47.552 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:47 smithi018 ceph-mon[95889]: from='mgr.15762 ' entity='mgr.y' 2023-09-23T20:15:47.552 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:47 smithi018 ceph-mon[95889]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-09-23T20:15:47.552 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:47 smithi018 ceph-mon[95889]: from='mgr.15762 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-09-23T20:15:47.552 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:47 smithi018 ceph-mon[95889]: from='mgr.15762 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2023-09-23T20:15:47.552 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:47 smithi018 ceph-mon[95889]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-09-23T20:15:47.553 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:47 smithi018 ceph-mon[95889]: from='mgr.15762 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-09-23T20:15:47.553 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:47 smithi018 ceph-mon[95889]: from='mgr.15762 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2023-09-23T20:15:47.553 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:47 smithi018 ceph-mon[95889]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-09-23T20:15:47.553 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:47 smithi018 ceph-mon[95889]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2023-09-23T20:15:47.553 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:47 smithi018 ceph-mon[95889]: Upgrade: It appears safe to stop mon.a 2023-09-23T20:15:47.553 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:47 smithi018 ceph-mon[95889]: from='client.? 172.21.15.18:0/1285864501' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:15:47.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:47 smithi149 ceph-mon[100764]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:15:47.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:47 smithi149 ceph-mon[100764]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:15:47.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:47 smithi149 ceph-mon[100764]: Upgrade: Setting container_image for all mgr 2023-09-23T20:15:47.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:47 smithi149 ceph-mon[100764]: from='mgr.15762 ' entity='mgr.y' 2023-09-23T20:15:47.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:47 smithi149 ceph-mon[100764]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-09-23T20:15:47.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:47 smithi149 ceph-mon[100764]: from='mgr.15762 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-09-23T20:15:47.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:47 smithi149 ceph-mon[100764]: from='mgr.15762 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2023-09-23T20:15:47.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:47 smithi149 ceph-mon[100764]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-09-23T20:15:47.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:47 smithi149 ceph-mon[100764]: from='mgr.15762 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-09-23T20:15:47.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:47 smithi149 ceph-mon[100764]: from='mgr.15762 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2023-09-23T20:15:47.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:47 smithi149 ceph-mon[100764]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-09-23T20:15:47.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:47 smithi149 ceph-mon[100764]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2023-09-23T20:15:47.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:47 smithi149 ceph-mon[100764]: Upgrade: It appears safe to stop mon.a 2023-09-23T20:15:47.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:47 smithi149 ceph-mon[100764]: from='client.? 172.21.15.18:0/1285864501' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:15:48.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:48 smithi149 ceph-mon[100764]: Upgrade: Updating mon.a 2023-09-23T20:15:48.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:48 smithi149 ceph-mon[100764]: from='mgr.15762 ' entity='mgr.y' 2023-09-23T20:15:48.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:48 smithi149 ceph-mon[100764]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T20:15:48.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:48 smithi149 ceph-mon[100764]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T20:15:48.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:48 smithi149 ceph-mon[100764]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:15:48.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:48 smithi149 ceph-mon[100764]: Deploying daemon mon.a on smithi018 2023-09-23T20:15:48.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:48 smithi149 ceph-mon[100764]: pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:15:48.967 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:15:48 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[156506]: ts=2023-09-23T20:15:48.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.18\", device_class=\"ssd\", hostname=\"smithi018\", instance=\"smithi149.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.18\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.18\", device_class=\"ssd\", hostname=\"smithi018\", instance=\"smithi018:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.18\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T20:15:49.034 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:48 smithi018 ceph-mon[109137]: Upgrade: Updating mon.a 2023-09-23T20:15:49.034 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:48 smithi018 ceph-mon[109137]: from='mgr.15762 ' entity='mgr.y' 2023-09-23T20:15:49.034 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:48 smithi018 ceph-mon[109137]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T20:15:49.034 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:48 smithi018 ceph-mon[109137]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T20:15:49.035 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:48 smithi018 ceph-mon[109137]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:15:49.035 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:48 smithi018 ceph-mon[109137]: Deploying daemon mon.a on smithi018 2023-09-23T20:15:49.035 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:48 smithi018 ceph-mon[109137]: pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:15:49.035 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:48 smithi018 ceph-mon[95889]: Upgrade: Updating mon.a 2023-09-23T20:15:49.035 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:48 smithi018 ceph-mon[95889]: from='mgr.15762 ' entity='mgr.y' 2023-09-23T20:15:49.035 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:48 smithi018 ceph-mon[95889]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T20:15:49.035 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:48 smithi018 ceph-mon[95889]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T20:15:49.035 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:48 smithi018 ceph-mon[95889]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:15:49.035 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:48 smithi018 ceph-mon[95889]: Deploying daemon mon.a on smithi018 2023-09-23T20:15:49.036 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:48 smithi018 ceph-mon[95889]: pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:15:49.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:49 smithi018 systemd[1]: Stopping Ceph mon.a for a3b3b17e-5a4c-11ee-8db4-212e2dc638e7... 2023-09-23T20:15:49.847 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:49 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mon-a[95865]: 2023-09-23T20:15:49.473+0000 7fd633bbb700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T20:15:49.847 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:49 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mon-a[95865]: 2023-09-23T20:15:49.473+0000 7fd633bbb700 -1 mon.a@0(leader) e3 *** Got Signal Terminated *** 2023-09-23T20:15:50.117 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:49 smithi018 bash[179459]: ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mon-a 2023-09-23T20:15:50.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 systemd[1]: ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7@mon.a.service: Succeeded. 2023-09-23T20:15:50.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 systemd[1]: Stopped Ceph mon.a for a3b3b17e-5a4c-11ee-8db4-212e2dc638e7. 2023-09-23T20:15:50.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 systemd[1]: Starting Ceph mon.a for a3b3b17e-5a4c-11ee-8db4-212e2dc638e7... 2023-09-23T20:15:50.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 podman[179607]: 2023-09-23T20:15:50.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 bash[179607]: 425294bbcbd129babcc5bda3a3a25ce26dcb3ac1fc7fecec8b634141fb4c83c9 2023-09-23T20:15:50.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: set uid:gid to 167:167 (ceph:ceph) 2023-09-23T20:15:50.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable), process ceph-mon, pid 7 2023-09-23T20:15:50.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: pidfile_write: ignore empty --pid-file 2023-09-23T20:15:50.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: load: jerasure load: lrc 2023-09-23T20:15:50.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: RocksDB version: 7.9.2 2023-09-23T20:15:50.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Git sha 0 2023-09-23T20:15:50.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Compile date 2023-09-22 15:19:43 2023-09-23T20:15:50.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: DB SUMMARY 2023-09-23T20:15:50.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: DB Session ID: 22EYDE2FLLZOUG1WWGJ8 2023-09-23T20:15:50.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: CURRENT file: CURRENT 2023-09-23T20:15:50.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: IDENTITY file: IDENTITY 2023-09-23T20:15:50.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: MANIFEST file: MANIFEST-000015 size: 779 Bytes 2023-09-23T20:15:50.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000027.sst 2023-09-23T20:15:50.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000025.log size: 2422116 ; 2023-09-23T20:15:50.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.error_if_exists: 0 2023-09-23T20:15:50.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.create_if_missing: 0 2023-09-23T20:15:50.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.paranoid_checks: 1 2023-09-23T20:15:50.895 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.flush_verify_memtable_count: 1 2023-09-23T20:15:50.895 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-09-23T20:15:50.895 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2023-09-23T20:15:50.895 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.env: 0x564313bdf720 2023-09-23T20:15:50.895 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.fs: PosixFileSystem 2023-09-23T20:15:50.895 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.info_log: 0x564315c99420 2023-09-23T20:15:50.895 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.max_file_opening_threads: 16 2023-09-23T20:15:50.895 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.statistics: (nil) 2023-09-23T20:15:50.896 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.use_fsync: 0 2023-09-23T20:15:50.896 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.max_log_file_size: 0 2023-09-23T20:15:50.896 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.max_manifest_file_size: 1073741824 2023-09-23T20:15:50.896 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.log_file_time_to_roll: 0 2023-09-23T20:15:50.896 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.keep_log_file_num: 1000 2023-09-23T20:15:50.896 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.recycle_log_file_num: 0 2023-09-23T20:15:50.896 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.allow_fallocate: 1 2023-09-23T20:15:50.896 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.allow_mmap_reads: 0 2023-09-23T20:15:50.896 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.allow_mmap_writes: 0 2023-09-23T20:15:50.896 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.use_direct_reads: 0 2023-09-23T20:15:50.897 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-09-23T20:15:50.897 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.create_missing_column_families: 0 2023-09-23T20:15:50.897 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.db_log_dir: 2023-09-23T20:15:50.897 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.wal_dir: 2023-09-23T20:15:50.897 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.table_cache_numshardbits: 6 2023-09-23T20:15:50.897 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.WAL_ttl_seconds: 0 2023-09-23T20:15:50.897 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.WAL_size_limit_MB: 0 2023-09-23T20:15:50.897 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-09-23T20:15:50.897 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.manifest_preallocation_size: 4194304 2023-09-23T20:15:50.897 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.is_fd_close_on_exec: 1 2023-09-23T20:15:50.898 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.advise_random_on_open: 1 2023-09-23T20:15:50.898 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.db_write_buffer_size: 0 2023-09-23T20:15:50.898 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.write_buffer_manager: 0x564314f2ac80 2023-09-23T20:15:50.898 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.access_hint_on_compaction_start: 1 2023-09-23T20:15:50.898 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.random_access_max_buffer_size: 1048576 2023-09-23T20:15:50.898 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.use_adaptive_mutex: 0 2023-09-23T20:15:50.898 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.rate_limiter: (nil) 2023-09-23T20:15:50.899 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-09-23T20:15:50.899 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.wal_recovery_mode: 2 2023-09-23T20:15:50.899 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.enable_thread_tracking: 0 2023-09-23T20:15:50.899 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.enable_pipelined_write: 0 2023-09-23T20:15:50.899 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.unordered_write: 0 2023-09-23T20:15:50.899 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.allow_concurrent_memtable_write: 1 2023-09-23T20:15:50.899 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-09-23T20:15:50.899 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.write_thread_max_yield_usec: 100 2023-09-23T20:15:50.899 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.write_thread_slow_yield_usec: 3 2023-09-23T20:15:50.900 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.row_cache: None 2023-09-23T20:15:50.900 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.wal_filter: None 2023-09-23T20:15:50.900 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.avoid_flush_during_recovery: 0 2023-09-23T20:15:50.900 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.allow_ingest_behind: 0 2023-09-23T20:15:50.900 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.two_write_queues: 0 2023-09-23T20:15:50.900 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.manual_wal_flush: 0 2023-09-23T20:15:50.900 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.wal_compression: 0 2023-09-23T20:15:50.900 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.atomic_flush: 0 2023-09-23T20:15:50.901 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-09-23T20:15:50.901 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.persist_stats_to_disk: 0 2023-09-23T20:15:50.901 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.write_dbid_to_manifest: 0 2023-09-23T20:15:50.901 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.log_readahead_size: 0 2023-09-23T20:15:50.901 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.file_checksum_gen_factory: Unknown 2023-09-23T20:15:50.901 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.best_efforts_recovery: 0 2023-09-23T20:15:50.901 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-09-23T20:15:50.901 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-09-23T20:15:50.901 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.allow_data_in_errors: 0 2023-09-23T20:15:50.901 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.db_host_id: __hostname__ 2023-09-23T20:15:50.901 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.enforce_single_del_contracts: true 2023-09-23T20:15:50.902 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.max_background_jobs: 2 2023-09-23T20:15:50.902 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.max_background_compactions: -1 2023-09-23T20:15:50.902 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.max_subcompactions: 1 2023-09-23T20:15:50.902 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.avoid_flush_during_shutdown: 0 2023-09-23T20:15:50.902 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-09-23T20:15:50.902 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.delayed_write_rate : 16777216 2023-09-23T20:15:50.902 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.max_total_wal_size: 0 2023-09-23T20:15:50.902 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-09-23T20:15:50.902 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.stats_dump_period_sec: 600 2023-09-23T20:15:50.902 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.stats_persist_period_sec: 600 2023-09-23T20:15:50.902 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.stats_history_buffer_size: 1048576 2023-09-23T20:15:50.902 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.max_open_files: -1 2023-09-23T20:15:50.903 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.bytes_per_sync: 0 2023-09-23T20:15:50.903 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.wal_bytes_per_sync: 0 2023-09-23T20:15:50.903 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.strict_bytes_per_sync: 0 2023-09-23T20:15:50.903 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.compaction_readahead_size: 0 2023-09-23T20:15:50.903 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.max_background_flushes: -1 2023-09-23T20:15:50.903 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Compression algorithms supported: 2023-09-23T20:15:50.903 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: kZSTDNotFinalCompression supported: 0 2023-09-23T20:15:50.903 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: kZSTD supported: 0 2023-09-23T20:15:50.903 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: kXpressCompression supported: 0 2023-09-23T20:15:50.903 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: kLZ4HCCompression supported: 1 2023-09-23T20:15:50.903 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: kZlibCompression supported: 1 2023-09-23T20:15:50.903 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: kSnappyCompression supported: 1 2023-09-23T20:15:50.904 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: kLZ4Compression supported: 1 2023-09-23T20:15:50.904 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: kBZip2Compression supported: 0 2023-09-23T20:15:50.904 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Fast CRC32 supported: Supported on x86 2023-09-23T20:15:50.904 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: DMutex implementation: pthread_mutex_t 2023-09-23T20:15:50.904 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 2023-09-23T20:15:50.904 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2023-09-23T20:15:50.904 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-09-23T20:15:50.904 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.merge_operator: 2023-09-23T20:15:50.904 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.compaction_filter: None 2023-09-23T20:15:50.904 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.compaction_filter_factory: None 2023-09-23T20:15:50.904 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.sst_partitioner_factory: None 2023-09-23T20:15:50.904 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.memtable_factory: SkipListFactory 2023-09-23T20:15:50.905 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.table_factory: BlockBasedTable 2023-09-23T20:15:50.905 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x564315c99520) 2023-09-23T20:15:50.905 INFO:journalctl@ceph.mon.a.smithi018.stdout: cache_index_and_filter_blocks: 1 2023-09-23T20:15:50.905 INFO:journalctl@ceph.mon.a.smithi018.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2023-09-23T20:15:50.905 INFO:journalctl@ceph.mon.a.smithi018.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2023-09-23T20:15:50.905 INFO:journalctl@ceph.mon.a.smithi018.stdout: pin_top_level_index_and_filter: 1 2023-09-23T20:15:50.905 INFO:journalctl@ceph.mon.a.smithi018.stdout: index_type: 0 2023-09-23T20:15:50.905 INFO:journalctl@ceph.mon.a.smithi018.stdout: data_block_index_type: 0 2023-09-23T20:15:50.905 INFO:journalctl@ceph.mon.a.smithi018.stdout: index_shortening: 1 2023-09-23T20:15:50.905 INFO:journalctl@ceph.mon.a.smithi018.stdout: data_block_hash_table_util_ratio: 0.750000 2023-09-23T20:15:50.905 INFO:journalctl@ceph.mon.a.smithi018.stdout: checksum: 4 2023-09-23T20:15:50.905 INFO:journalctl@ceph.mon.a.smithi018.stdout: no_block_cache: 0 2023-09-23T20:15:50.905 INFO:journalctl@ceph.mon.a.smithi018.stdout: block_cache: 0x564314fab350 2023-09-23T20:15:50.906 INFO:journalctl@ceph.mon.a.smithi018.stdout: block_cache_name: BinnedLRUCache 2023-09-23T20:15:50.906 INFO:journalctl@ceph.mon.a.smithi018.stdout: block_cache_options: 2023-09-23T20:15:50.906 INFO:journalctl@ceph.mon.a.smithi018.stdout: capacity : 536870912 2023-09-23T20:15:50.906 INFO:journalctl@ceph.mon.a.smithi018.stdout: num_shard_bits : 4 2023-09-23T20:15:50.906 INFO:journalctl@ceph.mon.a.smithi018.stdout: strict_capacity_limit : 0 2023-09-23T20:15:50.906 INFO:journalctl@ceph.mon.a.smithi018.stdout: high_pri_pool_ratio: 0.000 2023-09-23T20:15:50.906 INFO:journalctl@ceph.mon.a.smithi018.stdout: block_cache_compressed: (nil) 2023-09-23T20:15:50.906 INFO:journalctl@ceph.mon.a.smithi018.stdout: persistent_cache: (nil) 2023-09-23T20:15:50.906 INFO:journalctl@ceph.mon.a.smithi018.stdout: block_size: 4096 2023-09-23T20:15:50.906 INFO:journalctl@ceph.mon.a.smithi018.stdout: block_size_deviation: 10 2023-09-23T20:15:50.906 INFO:journalctl@ceph.mon.a.smithi018.stdout: block_restart_interval: 16 2023-09-23T20:15:50.907 INFO:journalctl@ceph.mon.a.smithi018.stdout: index_block_restart_interval: 1 2023-09-23T20:15:50.907 INFO:journalctl@ceph.mon.a.smithi018.stdout: metadata_block_size: 4096 2023-09-23T20:15:50.907 INFO:journalctl@ceph.mon.a.smithi018.stdout: partition_filters: 0 2023-09-23T20:15:50.907 INFO:journalctl@ceph.mon.a.smithi018.stdout: use_delta_encoding: 1 2023-09-23T20:15:50.907 INFO:journalctl@ceph.mon.a.smithi018.stdout: filter_policy: bloomfilter 2023-09-23T20:15:50.907 INFO:journalctl@ceph.mon.a.smithi018.stdout: whole_key_filtering: 1 2023-09-23T20:15:50.907 INFO:journalctl@ceph.mon.a.smithi018.stdout: verify_compression: 0 2023-09-23T20:15:50.907 INFO:journalctl@ceph.mon.a.smithi018.stdout: read_amp_bytes_per_bit: 0 2023-09-23T20:15:50.907 INFO:journalctl@ceph.mon.a.smithi018.stdout: format_version: 5 2023-09-23T20:15:50.907 INFO:journalctl@ceph.mon.a.smithi018.stdout: enable_index_compression: 1 2023-09-23T20:15:50.907 INFO:journalctl@ceph.mon.a.smithi018.stdout: block_align: 0 2023-09-23T20:15:50.907 INFO:journalctl@ceph.mon.a.smithi018.stdout: max_auto_readahead_size: 262144 2023-09-23T20:15:50.908 INFO:journalctl@ceph.mon.a.smithi018.stdout: prepopulate_block_cache: 0 2023-09-23T20:15:50.908 INFO:journalctl@ceph.mon.a.smithi018.stdout: initial_auto_readahead_size: 8192 2023-09-23T20:15:50.908 INFO:journalctl@ceph.mon.a.smithi018.stdout: num_file_reads_for_auto_readahead: 2 2023-09-23T20:15:50.908 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.write_buffer_size: 33554432 2023-09-23T20:15:50.908 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.max_write_buffer_number: 2 2023-09-23T20:15:50.908 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.compression: NoCompression 2023-09-23T20:15:50.908 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.bottommost_compression: Disabled 2023-09-23T20:15:50.908 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.prefix_extractor: nullptr 2023-09-23T20:15:50.908 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-09-23T20:15:50.908 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.num_levels: 7 2023-09-23T20:15:50.908 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-09-23T20:15:50.908 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-09-23T20:15:50.909 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-09-23T20:15:50.909 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-09-23T20:15:50.909 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.bottommost_compression_opts.level: 32767 2023-09-23T20:15:50.909 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-09-23T20:15:50.909 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-09-23T20:15:50.909 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-09-23T20:15:50.909 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-09-23T20:15:50.909 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.bottommost_compression_opts.enabled: false 2023-09-23T20:15:50.909 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2023-09-23T20:15:50.909 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2023-09-23T20:15:50.909 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.compression_opts.window_bits: -14 2023-09-23T20:15:50.909 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.compression_opts.level: 32767 2023-09-23T20:15:50.910 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.compression_opts.strategy: 0 2023-09-23T20:15:50.910 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-09-23T20:15:50.910 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-09-23T20:15:50.910 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2023-09-23T20:15:50.910 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.compression_opts.parallel_threads: 1 2023-09-23T20:15:50.910 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.compression_opts.enabled: false 2023-09-23T20:15:50.910 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2023-09-23T20:15:50.910 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-09-23T20:15:50.910 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-09-23T20:15:50.910 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.level0_stop_writes_trigger: 36 2023-09-23T20:15:50.910 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.target_file_size_base: 67108864 2023-09-23T20:15:50.910 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.target_file_size_multiplier: 1 2023-09-23T20:15:50.911 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.max_bytes_for_level_base: 268435456 2023-09-23T20:15:50.911 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-09-23T20:15:50.911 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-09-23T20:15:50.911 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-09-23T20:15:50.911 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-09-23T20:15:50.911 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-09-23T20:15:50.911 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-09-23T20:15:50.911 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-09-23T20:15:50.911 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-09-23T20:15:50.911 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-09-23T20:15:50.911 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-09-23T20:15:50.911 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.max_compaction_bytes: 1677721600 2023-09-23T20:15:50.912 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2023-09-23T20:15:50.912 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.arena_block_size: 1048576 2023-09-23T20:15:50.912 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-09-23T20:15:50.912 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-09-23T20:15:50.912 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.disable_auto_compactions: 0 2023-09-23T20:15:50.912 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-09-23T20:15:50.912 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-09-23T20:15:50.912 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-09-23T20:15:50.912 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-09-23T20:15:50.913 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-09-23T20:15:50.913 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-09-23T20:15:50.913 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-09-23T20:15:50.913 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-09-23T20:15:50.913 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-09-23T20:15:50.913 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-09-23T20:15:50.913 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.table_properties_collectors: 2023-09-23T20:15:50.913 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.inplace_update_support: 0 2023-09-23T20:15:50.913 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.inplace_update_num_locks: 10000 2023-09-23T20:15:50.913 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-09-23T20:15:50.913 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.memtable_whole_key_filtering: 0 2023-09-23T20:15:50.913 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.memtable_huge_page_size: 0 2023-09-23T20:15:50.914 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.bloom_locality: 0 2023-09-23T20:15:50.914 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.max_successive_merges: 0 2023-09-23T20:15:50.914 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.optimize_filters_for_hits: 0 2023-09-23T20:15:50.914 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.paranoid_file_checks: 0 2023-09-23T20:15:50.914 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.force_consistency_checks: 1 2023-09-23T20:15:50.914 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.report_bg_io_stats: 0 2023-09-23T20:15:50.914 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.ttl: 2592000 2023-09-23T20:15:50.914 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.periodic_compaction_seconds: 0 2023-09-23T20:15:50.914 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.preclude_last_level_data_seconds: 0 2023-09-23T20:15:50.914 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.preserve_internal_time_seconds: 0 2023-09-23T20:15:50.914 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.enable_blob_files: false 2023-09-23T20:15:50.914 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.min_blob_size: 0 2023-09-23T20:15:50.915 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.blob_file_size: 268435456 2023-09-23T20:15:50.915 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.blob_compression_type: NoCompression 2023-09-23T20:15:50.915 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.enable_blob_garbage_collection: false 2023-09-23T20:15:50.915 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-09-23T20:15:50.915 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2023-09-23T20:15:50.915 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.blob_compaction_readahead_size: 0 2023-09-23T20:15:50.915 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.blob_file_starting_level: 0 2023-09-23T20:15:50.915 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2023-09-23T20:15:50.915 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 27.sst 2023-09-23T20:15:50.915 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-09-23T20:15:50.916 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 succeeded,manifest_file_number is 15, next_file_number is 29, last_sequence is 10228, log_number is 25,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-09-23T20:15:50.916 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 25 2023-09-23T20:15:50.916 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: fe7cf128-cbad-4385-8069-6200b44b8e3a 2023-09-23T20:15:50.916 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695500150729719, "job": 1, "event": "recovery_started", "wal_files": [25]} 2023-09-23T20:15:50.916 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #25 mode 2 2023-09-23T20:15:50.916 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695500150736982, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 30, "file_size": 2176524, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 10229, "largest_seqno": 11516, "table_properties": {"data_size": 2170333, "index_size": 3700, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 1477, "raw_key_size": 14126, "raw_average_key_size": 24, "raw_value_size": 2157659, "raw_average_value_size": 3688, "num_data_blocks": 171, "num_entries": 585, "num_filter_entries": 585, "num_deletions": 2, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1695500150, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "fe7cf128-cbad-4385-8069-6200b44b8e3a", "db_session_id": "22EYDE2FLLZOUG1WWGJ8", "orig_file_number": 30, "seqno_to_time_mapping": "N/A"}} 2023-09-23T20:15:50.916 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695500150737082, "job": 1, "event": "recovery_finished"} 2023-09-23T20:15:50.916 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: [db/version_set.cc:5047] Creating manifest 32 2023-09-23T20:15:50.916 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-09-23T20:15:50.916 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000025.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-09-23T20:15:50.917 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x564315048000 2023-09-23T20:15:50.917 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: DB pointer 0x564315034000 2023-09-23T20:15:50.917 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2023-09-23T20:15:50.917 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: rocksdb: [db/db_impl/db_impl.cc:1111] 2023-09-23T20:15:50.917 INFO:journalctl@ceph.mon.a.smithi018.stdout: ** DB Stats ** 2023-09-23T20:15:50.917 INFO:journalctl@ceph.mon.a.smithi018.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T20:15:50.917 INFO:journalctl@ceph.mon.a.smithi018.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-09-23T20:15:50.917 INFO:journalctl@ceph.mon.a.smithi018.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-09-23T20:15:50.917 INFO:journalctl@ceph.mon.a.smithi018.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T20:15:50.917 INFO:journalctl@ceph.mon.a.smithi018.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-09-23T20:15:50.917 INFO:journalctl@ceph.mon.a.smithi018.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-09-23T20:15:50.918 INFO:journalctl@ceph.mon.a.smithi018.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T20:15:50.918 INFO:journalctl@ceph.mon.a.smithi018.stdout: 2023-09-23T20:15:50.918 INFO:journalctl@ceph.mon.a.smithi018.stdout: ** Compaction Stats [default] ** 2023-09-23T20:15:50.918 INFO:journalctl@ceph.mon.a.smithi018.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-09-23T20:15:50.918 INFO:journalctl@ceph.mon.a.smithi018.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2023-09-23T20:15:50.918 INFO:journalctl@ceph.mon.a.smithi018.stdout: L0 1/0 2.08 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 577.2 0.00 0.00 1 0.004 0 0 0.0 0.0 2023-09-23T20:15:50.918 INFO:journalctl@ceph.mon.a.smithi018.stdout: L6 1/0 6.58 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2023-09-23T20:15:50.918 INFO:journalctl@ceph.mon.a.smithi018.stdout: Sum 2/0 8.65 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 577.2 0.00 0.00 1 0.004 0 0 0.0 0.0 2023-09-23T20:15:50.918 INFO:journalctl@ceph.mon.a.smithi018.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 577.2 0.00 0.00 1 0.004 0 0 0.0 0.0 2023-09-23T20:15:50.918 INFO:journalctl@ceph.mon.a.smithi018.stdout: 2023-09-23T20:15:50.918 INFO:journalctl@ceph.mon.a.smithi018.stdout: ** Compaction Stats [default] ** 2023-09-23T20:15:50.918 INFO:journalctl@ceph.mon.a.smithi018.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-09-23T20:15:50.919 INFO:journalctl@ceph.mon.a.smithi018.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T20:15:50.919 INFO:journalctl@ceph.mon.a.smithi018.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 577.2 0.00 0.00 1 0.004 0 0 0.0 0.0 2023-09-23T20:15:50.919 INFO:journalctl@ceph.mon.a.smithi018.stdout: 2023-09-23T20:15:50.919 INFO:journalctl@ceph.mon.a.smithi018.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2023-09-23T20:15:50.919 INFO:journalctl@ceph.mon.a.smithi018.stdout: 2023-09-23T20:15:50.919 INFO:journalctl@ceph.mon.a.smithi018.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T20:15:50.919 INFO:journalctl@ceph.mon.a.smithi018.stdout: Flush(GB): cumulative 0.002, interval 0.002 2023-09-23T20:15:50.919 INFO:journalctl@ceph.mon.a.smithi018.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-09-23T20:15:50.919 INFO:journalctl@ceph.mon.a.smithi018.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-09-23T20:15:50.919 INFO:journalctl@ceph.mon.a.smithi018.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-09-23T20:15:50.919 INFO:journalctl@ceph.mon.a.smithi018.stdout: AddFile(Keys): cumulative 0, interval 0 2023-09-23T20:15:50.919 INFO:journalctl@ceph.mon.a.smithi018.stdout: Cumulative compaction: 0.00 GB write, 208.72 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T20:15:50.920 INFO:journalctl@ceph.mon.a.smithi018.stdout: Interval compaction: 0.00 GB write, 208.72 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T20:15:50.920 INFO:journalctl@ceph.mon.a.smithi018.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-09-23T20:15:50.920 INFO:journalctl@ceph.mon.a.smithi018.stdout: Block cache BinnedLRUCache@0x564314fab350#7 capacity: 512.00 MB usage: 5.61 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.1e-05 secs_since: 0 2023-09-23T20:15:50.920 INFO:journalctl@ceph.mon.a.smithi018.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,1.53 KB,0.000292063%) IndexBlock(1,4.08 KB,0.000777841%) Misc(1,0.00 KB,0%) 2023-09-23T20:15:50.920 INFO:journalctl@ceph.mon.a.smithi018.stdout: 2023-09-23T20:15:50.920 INFO:journalctl@ceph.mon.a.smithi018.stdout: ** File Read Latency Histogram By Level [default] ** 2023-09-23T20:15:50.920 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: starting mon.a rank 0 at public addrs [v2:172.21.15.18:3300/0,v1:172.21.15.18:6789/0] at bind addrs [v2:172.21.15.18:3300/0,v1:172.21.15.18:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 2023-09-23T20:15:50.920 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: mon.a@-1(???) e3 preinit fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 2023-09-23T20:15:50.921 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: mon.a@-1(???).mds e9 new map 2023-09-23T20:15:50.921 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: mon.a@-1(???).mds e9 print_map 2023-09-23T20:15:50.921 INFO:journalctl@ceph.mon.a.smithi018.stdout: e9 2023-09-23T20:15:50.921 INFO:journalctl@ceph.mon.a.smithi018.stdout: enable_multiple, ever_enabled_multiple: 1,1 2023-09-23T20:15:50.921 INFO:journalctl@ceph.mon.a.smithi018.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-09-23T20:15:50.921 INFO:journalctl@ceph.mon.a.smithi018.stdout: legacy client fscid: 1 2023-09-23T20:15:50.921 INFO:journalctl@ceph.mon.a.smithi018.stdout: 2023-09-23T20:15:50.921 INFO:journalctl@ceph.mon.a.smithi018.stdout: Filesystem 'foo' (1) 2023-09-23T20:15:50.921 INFO:journalctl@ceph.mon.a.smithi018.stdout: fs_name foo 2023-09-23T20:15:50.921 INFO:journalctl@ceph.mon.a.smithi018.stdout: epoch 8 2023-09-23T20:15:50.922 INFO:journalctl@ceph.mon.a.smithi018.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2023-09-23T20:15:50.922 INFO:journalctl@ceph.mon.a.smithi018.stdout: created 2023-09-23T20:12:43.745113+0000 2023-09-23T20:15:50.922 INFO:journalctl@ceph.mon.a.smithi018.stdout: modified 2023-09-23T20:12:54.848699+0000 2023-09-23T20:15:50.922 INFO:journalctl@ceph.mon.a.smithi018.stdout: tableserver 0 2023-09-23T20:15:50.922 INFO:journalctl@ceph.mon.a.smithi018.stdout: root 0 2023-09-23T20:15:50.922 INFO:journalctl@ceph.mon.a.smithi018.stdout: session_timeout 60 2023-09-23T20:15:50.922 INFO:journalctl@ceph.mon.a.smithi018.stdout: session_autoclose 300 2023-09-23T20:15:50.922 INFO:journalctl@ceph.mon.a.smithi018.stdout: max_file_size 1099511627776 2023-09-23T20:15:50.922 INFO:journalctl@ceph.mon.a.smithi018.stdout: required_client_features {} 2023-09-23T20:15:50.922 INFO:journalctl@ceph.mon.a.smithi018.stdout: last_failure 0 2023-09-23T20:15:50.922 INFO:journalctl@ceph.mon.a.smithi018.stdout: last_failure_osd_epoch 0 2023-09-23T20:15:50.923 INFO:journalctl@ceph.mon.a.smithi018.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-09-23T20:15:50.923 INFO:journalctl@ceph.mon.a.smithi018.stdout: max_mds 1 2023-09-23T20:15:50.923 INFO:journalctl@ceph.mon.a.smithi018.stdout: in 0 2023-09-23T20:15:50.923 INFO:journalctl@ceph.mon.a.smithi018.stdout: up {0=24566} 2023-09-23T20:15:50.923 INFO:journalctl@ceph.mon.a.smithi018.stdout: failed 2023-09-23T20:15:50.923 INFO:journalctl@ceph.mon.a.smithi018.stdout: damaged 2023-09-23T20:15:50.923 INFO:journalctl@ceph.mon.a.smithi018.stdout: stopped 2023-09-23T20:15:50.923 INFO:journalctl@ceph.mon.a.smithi018.stdout: data_pools [4] 2023-09-23T20:15:50.924 INFO:journalctl@ceph.mon.a.smithi018.stdout: metadata_pool 3 2023-09-23T20:15:50.924 INFO:journalctl@ceph.mon.a.smithi018.stdout: inline_data disabled 2023-09-23T20:15:50.924 INFO:journalctl@ceph.mon.a.smithi018.stdout: balancer 2023-09-23T20:15:50.924 INFO:journalctl@ceph.mon.a.smithi018.stdout: bal_rank_mask -1 2023-09-23T20:15:50.924 INFO:journalctl@ceph.mon.a.smithi018.stdout: standby_count_wanted 1 2023-09-23T20:15:50.924 INFO:journalctl@ceph.mon.a.smithi018.stdout: [mds.foo.smithi018.hduiol{0:24566} state up:active seq 3 join_fscid=1 addr [v2:172.21.15.18:6834/3661910111,v1:172.21.15.18:6835/3661910111] compat {c=[1],r=[1],i=[7ff]}] 2023-09-23T20:15:50.924 INFO:journalctl@ceph.mon.a.smithi018.stdout: 2023-09-23T20:15:50.924 INFO:journalctl@ceph.mon.a.smithi018.stdout: 2023-09-23T20:15:50.924 INFO:journalctl@ceph.mon.a.smithi018.stdout: Standby daemons: 2023-09-23T20:15:50.924 INFO:journalctl@ceph.mon.a.smithi018.stdout: 2023-09-23T20:15:50.924 INFO:journalctl@ceph.mon.a.smithi018.stdout: [mds.foo.smithi149.urtezf{-1:24559} state up:standby seq 2 join_fscid=1 addr [v2:172.21.15.149:6832/3529449999,v1:172.21.15.149:6833/3529449999] compat {c=[1],r=[1],i=[7ff]}] 2023-09-23T20:15:50.924 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: mon.a@-1(???).osd e89 crush map has features 3314933000854323200, adjusting msgr requires 2023-09-23T20:15:50.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: mon.a@-1(???).osd e89 crush map has features 432629239337189376, adjusting msgr requires 2023-09-23T20:15:50.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: mon.a@-1(???).osd e89 crush map has features 432629239337189376, adjusting msgr requires 2023-09-23T20:15:50.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: mon.a@-1(???).osd e89 crush map has features 432629239337189376, adjusting msgr requires 2023-09-23T20:15:50.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 ceph-mon[179647]: mon.a@-1(???).paxosservice(auth 1..18) refresh upgraded, format 0 -> 3 2023-09-23T20:15:50.925 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:50 smithi018 systemd[1]: Started Ceph mon.a for a3b3b17e-5a4c-11ee-8db4-212e2dc638e7. 2023-09-23T20:15:51.273 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:15:51 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ignoring --setuser ceph since I am not root 2023-09-23T20:15:51.273 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:15:51 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ignoring --setgroup ceph since I am not root 2023-09-23T20:15:51.273 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:15:51 smithi018 ceph-mgr[167821]: -- 172.21.15.18:0/1614113476 <== mon.2 v2:172.21.15.18:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55fe0df311e0 con 0x55fe0df06800 2023-09-23T20:15:51.273 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:15:51 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: 2023-09-23T20:15:51.176+0000 7f958270a200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-23T20:15:51.273 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:51 smithi018 ceph-mon[109137]: from='client.? 172.21.15.18:0/3501006880' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:15:51.274 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:51 smithi018 ceph-mon[109137]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:15:51.274 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:51 smithi018 ceph-mon[109137]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:15:51.274 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:51 smithi018 ceph-mon[109137]: mon.a calling monitor election 2023-09-23T20:15:51.274 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:51 smithi018 ceph-mon[109137]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T20:15:51.274 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:51 smithi018 ceph-mon[109137]: monmap e3: 3 mons at {a=[v2:172.21.15.18:3300/0,v1:172.21.15.18:6789/0],b=[v2:172.21.15.149:3300/0,v1:172.21.15.149:6789/0],c=[v2:172.21.15.18:3301/0,v1:172.21.15.18:6790/0]} removed_ranks: {} 2023-09-23T20:15:51.274 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:51 smithi018 ceph-mon[109137]: fsmap foo:1 {0=foo.smithi018.hduiol=up:active} 1 up:standby 2023-09-23T20:15:51.275 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:51 smithi018 ceph-mon[109137]: osdmap e89: 8 total, 8 up, 8 in 2023-09-23T20:15:51.275 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:51 smithi018 ceph-mon[109137]: mgrmap e30: y(active, since 31s), standbys: x 2023-09-23T20:15:51.275 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:51 smithi018 ceph-mon[109137]: mgrmap e31: y(active, since 31s), standbys: x 2023-09-23T20:15:51.275 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:51 smithi018 ceph-mon[109137]: overall HEALTH_OK 2023-09-23T20:15:51.275 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:51 smithi018 ceph-mon[109137]: from='mgr.15762 ' entity='' 2023-09-23T20:15:51.275 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:51 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3501006880' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:15:51.275 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:51 smithi018 ceph-mon[179647]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:15:51.275 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:51 smithi018 ceph-mon[179647]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:15:51.276 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:51 smithi018 ceph-mon[179647]: mon.a calling monitor election 2023-09-23T20:15:51.276 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:51 smithi018 ceph-mon[179647]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T20:15:51.276 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:51 smithi018 ceph-mon[179647]: monmap e3: 3 mons at {a=[v2:172.21.15.18:3300/0,v1:172.21.15.18:6789/0],b=[v2:172.21.15.149:3300/0,v1:172.21.15.149:6789/0],c=[v2:172.21.15.18:3301/0,v1:172.21.15.18:6790/0]} removed_ranks: {} 2023-09-23T20:15:51.276 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:51 smithi018 ceph-mon[179647]: fsmap foo:1 {0=foo.smithi018.hduiol=up:active} 1 up:standby 2023-09-23T20:15:51.276 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:51 smithi018 ceph-mon[179647]: osdmap e89: 8 total, 8 up, 8 in 2023-09-23T20:15:51.276 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:51 smithi018 ceph-mon[179647]: mgrmap e30: y(active, since 31s), standbys: x 2023-09-23T20:15:51.276 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:51 smithi018 ceph-mon[179647]: mgrmap e31: y(active, since 31s), standbys: x 2023-09-23T20:15:51.276 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:51 smithi018 ceph-mon[179647]: overall HEALTH_OK 2023-09-23T20:15:51.276 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:51 smithi018 ceph-mon[179647]: from='mgr.15762 ' entity='' 2023-09-23T20:15:51.313 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:51 smithi149 ceph-mon[100764]: from='client.? 172.21.15.18:0/3501006880' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:15:51.313 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:51 smithi149 ceph-mon[100764]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:15:51.313 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:51 smithi149 ceph-mon[100764]: from='mgr.15762 172.21.15.18:0/1461339875' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:15:51.313 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:51 smithi149 ceph-mon[100764]: mon.a calling monitor election 2023-09-23T20:15:51.313 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:51 smithi149 ceph-mon[100764]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T20:15:51.314 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:51 smithi149 ceph-mon[100764]: monmap e3: 3 mons at {a=[v2:172.21.15.18:3300/0,v1:172.21.15.18:6789/0],b=[v2:172.21.15.149:3300/0,v1:172.21.15.149:6789/0],c=[v2:172.21.15.18:3301/0,v1:172.21.15.18:6790/0]} removed_ranks: {} 2023-09-23T20:15:51.314 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:51 smithi149 ceph-mon[100764]: fsmap foo:1 {0=foo.smithi018.hduiol=up:active} 1 up:standby 2023-09-23T20:15:51.314 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:51 smithi149 ceph-mon[100764]: osdmap e89: 8 total, 8 up, 8 in 2023-09-23T20:15:51.314 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:51 smithi149 ceph-mon[100764]: mgrmap e30: y(active, since 31s), standbys: x 2023-09-23T20:15:51.314 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:51 smithi149 ceph-mon[100764]: mgrmap e31: y(active, since 31s), standbys: x 2023-09-23T20:15:51.314 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:51 smithi149 ceph-mon[100764]: overall HEALTH_OK 2023-09-23T20:15:51.314 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:51 smithi149 ceph-mon[100764]: from='mgr.15762 ' entity='' 2023-09-23T20:15:51.314 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:51 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[157063]: ignoring --setuser ceph since I am not root 2023-09-23T20:15:51.314 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:51 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[157063]: ignoring --setgroup ceph since I am not root 2023-09-23T20:15:51.314 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:51 smithi149 ceph-mgr[157087]: -- 172.21.15.149:0/4012687755 <== mon.1 v2:172.21.15.149:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x56554057f1e0 con 0x565540554c00 2023-09-23T20:15:51.315 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:51 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[157063]: 2023-09-23T20:15:51.144+0000 7f769e2cc200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-23T20:15:51.315 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:51 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[157063]: 2023-09-23T20:15:51.223+0000 7f769e2cc200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T20:15:51.315 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:51 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[157063]: 2023-09-23T20:15:51.312+0000 7f769e2cc200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T20:15:51.575 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:51 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[157063]: 2023-09-23T20:15:51.406+0000 7f769e2cc200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T20:15:51.641 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:15:51 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: 2023-09-23T20:15:51.272+0000 7f958270a200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T20:15:51.641 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:15:51 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: 2023-09-23T20:15:51.381+0000 7f958270a200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T20:15:51.641 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:15:51 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: 2023-09-23T20:15:51.494+0000 7f958270a200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T20:15:51.937 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:51 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[157063]: 2023-09-23T20:15:51.574+0000 7f769e2cc200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T20:15:52.079 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable)": 1 2023-09-23T20:15:52.090 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:15:51 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: 2023-09-23T20:15:51.702+0000 7f958270a200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T20:15:52.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:52 smithi149 ceph-mon[100764]: from='client.? 172.21.15.18:0/2816646057' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:15:52.216 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:51 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[157063]: 2023-09-23T20:15:51.936+0000 7f769e2cc200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T20:15:52.217 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:52 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[157063]: 2023-09-23T20:15:52.068+0000 7f769e2cc200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T20:15:52.217 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:52 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[157063]: 2023-09-23T20:15:52.157+0000 7f769e2cc200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T20:15:52.361 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:52 smithi018 ceph-mon[109137]: from='client.? 172.21.15.18:0/2816646057' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:15:52.361 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:52 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2816646057' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:15:52.362 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:15:52 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: 2023-09-23T20:15:52.206+0000 7f958270a200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T20:15:52.362 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:15:52 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: 2023-09-23T20:15:52.360+0000 7f958270a200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T20:15:52.378 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch ps' 2023-09-23T20:15:52.641 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:15:52 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: 2023-09-23T20:15:52.475+0000 7f958270a200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T20:15:52.641 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:15:52 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: 2023-09-23T20:15:52.596+0000 7f958270a200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T20:15:52.673 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:52 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[157063]: 2023-09-23T20:15:52.251+0000 7f769e2cc200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T20:15:52.966 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:52 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[157063]: 2023-09-23T20:15:52.672+0000 7f769e2cc200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T20:15:52.989 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:15:53.391 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:15:53 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: 2023-09-23T20:15:53.124+0000 7f958270a200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T20:15:53.716 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:53 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[157063]: 2023-09-23T20:15:53.331+0000 7f769e2cc200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T20:15:53.716 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:53 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[157063]: 2023-09-23T20:15:53.414+0000 7f769e2cc200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T20:15:54.047 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:15:53 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: 2023-09-23T20:15:53.944+0000 7f958270a200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T20:15:54.047 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:15:54 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: 2023-09-23T20:15:54.047+0000 7f958270a200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T20:15:54.457 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:54 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[157063]: 2023-09-23T20:15:54.191+0000 7f769e2cc200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T20:15:54.457 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:54 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[157063]: 2023-09-23T20:15:54.283+0000 7f769e2cc200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-23T20:15:54.716 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:54 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[157063]: 2023-09-23T20:15:54.456+0000 7f769e2cc200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T20:15:55.120 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:54 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[157063]: 2023-09-23T20:15:54.739+0000 7f769e2cc200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-23T20:15:55.120 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:54 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[157063]: 2023-09-23T20:15:54.970+0000 7f769e2cc200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T20:15:55.387 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:15:55 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: 2023-09-23T20:15:55.046+0000 7f958270a200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T20:15:55.387 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:15:55 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: 2023-09-23T20:15:55.161+0000 7f958270a200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-23T20:15:55.387 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:15:55 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: 2023-09-23T20:15:55.386+0000 7f958270a200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T20:15:55.388 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:55 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[157063]: 2023-09-23T20:15:55.119+0000 7f769e2cc200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T20:15:55.716 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:55 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[157063]: 2023-09-23T20:15:55.387+0000 7f769e2cc200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T20:15:56.006 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:15:55 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: 2023-09-23T20:15:55.745+0000 7f958270a200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-23T20:15:56.391 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:15:56 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: 2023-09-23T20:15:56.005+0000 7f958270a200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T20:15:56.391 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:15:56 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: 2023-09-23T20:15:56.191+0000 7f958270a200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T20:15:56.435 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:56 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[157063]: 2023-09-23T20:15:56.104+0000 7f769e2cc200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T20:15:56.435 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:56 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[157063]: 2023-09-23T20:15:56.239+0000 7f769e2cc200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T20:15:56.435 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:56 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[157063]: 2023-09-23T20:15:56.434+0000 7f769e2cc200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T20:15:56.716 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:56 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[157063]: 2023-09-23T20:15:56.548+0000 7f769e2cc200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T20:15:56.891 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:15:56 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: 2023-09-23T20:15:56.530+0000 7f958270a200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T20:15:57.216 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:56 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[157063]: 2023-09-23T20:15:56.767+0000 7f769e2cc200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T20:15:57.641 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:15:57 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: 2023-09-23T20:15:57.372+0000 7f958270a200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T20:15:57.641 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:15:57 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: 2023-09-23T20:15:57.534+0000 7f958270a200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T20:15:57.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:57 smithi149 ceph-mon[100764]: Standby manager daemon x restarted 2023-09-23T20:15:57.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:57 smithi149 ceph-mon[100764]: Standby manager daemon x started 2023-09-23T20:15:57.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:57 smithi149 ceph-mon[100764]: from='mgr.? 172.21.15.149:0/4214261145' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T20:15:57.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:57 smithi149 ceph-mon[100764]: from='mgr.? 172.21.15.149:0/4214261145' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T20:15:57.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:57 smithi149 ceph-mon[100764]: from='mgr.? 172.21.15.149:0/4214261145' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T20:15:57.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:57 smithi149 ceph-mon[100764]: from='mgr.? 172.21.15.149:0/4214261145' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T20:15:57.967 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:57 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[157063]: 2023-09-23T20:15:57.637+0000 7f769e2cc200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T20:15:57.967 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:57 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[157063]: [23/Sep/2023:20:15:57] ENGINE Bus STARTING 2023-09-23T20:15:57.967 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:57 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[157063]: CherryPy Checker: 2023-09-23T20:15:57.967 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:57 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[157063]: The Application mounted at '' has an empty config. 2023-09-23T20:15:57.967 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:57 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[157063]: 2023-09-23T20:15:57.967 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:57 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[157063]: [23/Sep/2023:20:15:57] ENGINE Serving on http://:::9283 2023-09-23T20:15:57.967 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:15:57 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x[157063]: [23/Sep/2023:20:15:57] ENGINE Bus STARTED 2023-09-23T20:15:58.141 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:15:57 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: 2023-09-23T20:15:57.790+0000 7f958270a200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T20:15:58.141 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:15:57 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: 2023-09-23T20:15:57.932+0000 7f958270a200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T20:15:58.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:57 smithi018 ceph-mon[179647]: Standby manager daemon x restarted 2023-09-23T20:15:58.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:57 smithi018 ceph-mon[179647]: Standby manager daemon x started 2023-09-23T20:15:58.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:57 smithi018 ceph-mon[179647]: from='mgr.? 172.21.15.149:0/4214261145' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T20:15:58.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:57 smithi018 ceph-mon[179647]: from='mgr.? 172.21.15.149:0/4214261145' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T20:15:58.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:57 smithi018 ceph-mon[179647]: from='mgr.? 172.21.15.149:0/4214261145' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T20:15:58.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:57 smithi018 ceph-mon[179647]: from='mgr.? 172.21.15.149:0/4214261145' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T20:15:58.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:57 smithi018 ceph-mon[109137]: Standby manager daemon x restarted 2023-09-23T20:15:58.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:57 smithi018 ceph-mon[109137]: Standby manager daemon x started 2023-09-23T20:15:58.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:57 smithi018 ceph-mon[109137]: from='mgr.? 172.21.15.149:0/4214261145' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T20:15:58.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:57 smithi018 ceph-mon[109137]: from='mgr.? 172.21.15.149:0/4214261145' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T20:15:58.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:57 smithi018 ceph-mon[109137]: from='mgr.? 172.21.15.149:0/4214261145' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T20:15:58.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:57 smithi018 ceph-mon[109137]: from='mgr.? 172.21.15.149:0/4214261145' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T20:15:58.641 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:15:58 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: 2023-09-23T20:15:58.215+0000 7f958270a200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T20:15:58.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:58 smithi149 ceph-mon[100764]: mgrmap e32: y(active, since 37s), standbys: x 2023-09-23T20:15:59.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:58 smithi018 ceph-mon[179647]: mgrmap e32: y(active, since 37s), standbys: x 2023-09-23T20:15:59.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:58 smithi018 ceph-mon[109137]: mgrmap e32: y(active, since 37s), standbys: x 2023-09-23T20:15:59.547 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:59 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mon-c[109113]: 2023-09-23T20:15:59.312+0000 7fecc2504700 -1 mon.c@2(peon).osd e90 update_from_paxos full map CRC mismatch, resetting to canonical 2023-09-23T20:15:59.548 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:15:59 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: 2023-09-23T20:15:59.296+0000 7f958270a200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T20:15:59.548 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:15:59 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: [23/Sep/2023:20:15:59] ENGINE Bus STARTING 2023-09-23T20:15:59.548 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:15:59 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: 2023-09-23T20:15:59.477+0000 7f94e023d700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T20:15:59.548 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:15:59 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: 2023-09-23T20:15:59.477+0000 7f94e023d700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T20:15:59.548 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:15:59 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: 2023-09-23T20:15:59.477+0000 7f94e023d700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T20:15:59.548 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:15:59 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: 2023-09-23T20:15:59.477+0000 7f94dd237700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T20:15:59.548 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:15:59 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: 2023-09-23T20:15:59.477+0000 7f94dd237700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T20:15:59.549 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:15:59 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: 2023-09-23T20:15:59.477+0000 7f94dd237700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T20:15:59.549 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:15:59 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: 2023-09-23T20:15:59.477+0000 7f94dd237700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T20:15:59.549 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:15:59 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: 2023-09-23T20:15:59.477+0000 7f94dd237700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T20:15:59.549 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:15:59 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: 2023-09-23T20:15:59.477+0000 7f94e023d700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T20:15:59.549 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:15:59 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: 2023-09-23T20:15:59.477+0000 7f94e023d700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T20:15:59.712 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:59 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mon-b[100740]: 2023-09-23T20:15:59.312+0000 7f40ff977700 -1 mon.b@1(peon).osd e90 update_from_paxos full map CRC mismatch, resetting to canonical 2023-09-23T20:15:59.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:59 smithi018 ceph-mon[179647]: Active manager daemon y restarted 2023-09-23T20:15:59.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:59 smithi018 ceph-mon[179647]: Activating manager daemon y 2023-09-23T20:15:59.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:59 smithi018 ceph-mon[179647]: osdmap e90: 8 total, 8 up, 8 in 2023-09-23T20:15:59.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:59 smithi018 ceph-mon[179647]: mgrmap e33: y(active, starting, since 0.00607579s), standbys: x 2023-09-23T20:15:59.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:59 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T20:15:59.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:59 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:15:59.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:59 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:15:59.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:59 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi149.urtezf"}]: dispatch 2023-09-23T20:15:59.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:59 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi018.hduiol"}]: dispatch 2023-09-23T20:15:59.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:59 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T20:15:59.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:59 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T20:15:59.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:59 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T20:15:59.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:59 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T20:15:59.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:59 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T20:15:59.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:59 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T20:15:59.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:59 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T20:15:59.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:59 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T20:15:59.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:59 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T20:15:59.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:59 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T20:15:59.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:59 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T20:15:59.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:59 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T20:15:59.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:59 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T20:15:59.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:59 smithi018 ceph-mon[179647]: Manager daemon y is now available 2023-09-23T20:15:59.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:59 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:15:59.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:59 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:15:59.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:59 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:15:59.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:59 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T20:15:59.895 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:59 smithi018 ceph-mon[179647]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T20:15:59.895 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:59 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T20:15:59.895 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:15:59 smithi018 ceph-mon[179647]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T20:15:59.895 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:15:59 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: CherryPy Checker: 2023-09-23T20:15:59.895 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:15:59 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: The Application mounted at '' has an empty config. 2023-09-23T20:15:59.895 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:15:59 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: 2023-09-23T20:15:59.895 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:15:59 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: [23/Sep/2023:20:15:59] ENGINE Serving on http://:::9283 2023-09-23T20:15:59.895 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:15:59 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: [23/Sep/2023:20:15:59] ENGINE Bus STARTED 2023-09-23T20:15:59.896 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:59 smithi018 ceph-mon[109137]: Active manager daemon y restarted 2023-09-23T20:15:59.896 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:59 smithi018 ceph-mon[109137]: Activating manager daemon y 2023-09-23T20:15:59.896 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:59 smithi018 ceph-mon[109137]: osdmap e90: 8 total, 8 up, 8 in 2023-09-23T20:15:59.896 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:59 smithi018 ceph-mon[109137]: mgrmap e33: y(active, starting, since 0.00607579s), standbys: x 2023-09-23T20:15:59.896 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:59 smithi018 ceph-mon[109137]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T20:15:59.896 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:59 smithi018 ceph-mon[109137]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:15:59.896 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:59 smithi018 ceph-mon[109137]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:15:59.896 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:59 smithi018 ceph-mon[109137]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi149.urtezf"}]: dispatch 2023-09-23T20:15:59.896 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:59 smithi018 ceph-mon[109137]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi018.hduiol"}]: dispatch 2023-09-23T20:15:59.896 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:59 smithi018 ceph-mon[109137]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T20:15:59.896 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:59 smithi018 ceph-mon[109137]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T20:15:59.897 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:59 smithi018 ceph-mon[109137]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T20:15:59.897 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:59 smithi018 ceph-mon[109137]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T20:15:59.897 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:59 smithi018 ceph-mon[109137]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T20:15:59.897 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:59 smithi018 ceph-mon[109137]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T20:15:59.897 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:59 smithi018 ceph-mon[109137]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T20:15:59.897 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:59 smithi018 ceph-mon[109137]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T20:15:59.897 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:59 smithi018 ceph-mon[109137]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T20:15:59.897 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:59 smithi018 ceph-mon[109137]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T20:15:59.897 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:59 smithi018 ceph-mon[109137]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T20:15:59.897 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:59 smithi018 ceph-mon[109137]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T20:15:59.897 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:59 smithi018 ceph-mon[109137]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T20:15:59.897 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:59 smithi018 ceph-mon[109137]: Manager daemon y is now available 2023-09-23T20:15:59.897 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:59 smithi018 ceph-mon[109137]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:15:59.898 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:59 smithi018 ceph-mon[109137]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:15:59.898 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:59 smithi018 ceph-mon[109137]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:15:59.898 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:59 smithi018 ceph-mon[109137]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T20:15:59.898 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:59 smithi018 ceph-mon[109137]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T20:15:59.898 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:59 smithi018 ceph-mon[109137]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T20:15:59.898 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:15:59 smithi018 ceph-mon[109137]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T20:15:59.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:59 smithi149 ceph-mon[100764]: Active manager daemon y restarted 2023-09-23T20:15:59.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:59 smithi149 ceph-mon[100764]: Activating manager daemon y 2023-09-23T20:15:59.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:59 smithi149 ceph-mon[100764]: osdmap e90: 8 total, 8 up, 8 in 2023-09-23T20:15:59.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:59 smithi149 ceph-mon[100764]: mgrmap e33: y(active, starting, since 0.00607579s), standbys: x 2023-09-23T20:15:59.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:59 smithi149 ceph-mon[100764]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T20:15:59.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:59 smithi149 ceph-mon[100764]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:15:59.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:59 smithi149 ceph-mon[100764]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:15:59.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:59 smithi149 ceph-mon[100764]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi149.urtezf"}]: dispatch 2023-09-23T20:15:59.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:59 smithi149 ceph-mon[100764]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi018.hduiol"}]: dispatch 2023-09-23T20:15:59.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:59 smithi149 ceph-mon[100764]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T20:15:59.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:59 smithi149 ceph-mon[100764]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T20:15:59.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:59 smithi149 ceph-mon[100764]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T20:15:59.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:59 smithi149 ceph-mon[100764]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T20:15:59.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:59 smithi149 ceph-mon[100764]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T20:15:59.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:59 smithi149 ceph-mon[100764]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T20:15:59.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:59 smithi149 ceph-mon[100764]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T20:15:59.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:59 smithi149 ceph-mon[100764]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T20:15:59.968 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:59 smithi149 ceph-mon[100764]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T20:15:59.968 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:59 smithi149 ceph-mon[100764]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T20:15:59.968 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:59 smithi149 ceph-mon[100764]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T20:15:59.968 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:59 smithi149 ceph-mon[100764]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T20:15:59.968 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:59 smithi149 ceph-mon[100764]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T20:15:59.968 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:59 smithi149 ceph-mon[100764]: Manager daemon y is now available 2023-09-23T20:15:59.968 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:59 smithi149 ceph-mon[100764]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:15:59.968 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:59 smithi149 ceph-mon[100764]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:15:59.968 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:59 smithi149 ceph-mon[100764]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:15:59.968 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:59 smithi149 ceph-mon[100764]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T20:15:59.968 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:59 smithi149 ceph-mon[100764]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T20:15:59.968 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:59 smithi149 ceph-mon[100764]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T20:15:59.969 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:15:59 smithi149 ceph-mon[100764]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T20:16:00.340 INFO:teuthology.orchestra.run.smithi018.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-09-23T20:16:00.340 INFO:teuthology.orchestra.run.smithi018.stdout:alertmanager.a smithi018 *:9093,9094 running (66s) - 3m 17.6M - 0.25.0 c8568f914cd2 57241c9344be 2023-09-23T20:16:00.340 INFO:teuthology.orchestra.run.smithi018.stdout:grafana.a smithi149 *:3000 running (55s) - 3m 82.7M - 9.4.7 2c41d148cca3 62150314e898 2023-09-23T20:16:00.340 INFO:teuthology.orchestra.run.smithi018.stdout:mds.foo.smithi018.hduiol smithi018 running (3m) - 3m 16.4M - 17.2.6-1123-g61431f6b 0db14aa01094 7b6a15d633a4 2023-09-23T20:16:00.340 INFO:teuthology.orchestra.run.smithi018.stdout:mds.foo.smithi149.urtezf smithi149 running (3m) - 3m 17.0M - 17.2.6-1123-g61431f6b 0db14aa01094 6ee4df523f8c 2023-09-23T20:16:00.340 INFO:teuthology.orchestra.run.smithi018.stdout:mgr.x smithi149 *:8443,9283,8765 running (23s) - 7m 59.3M - 18.2.0-460-gbb0daeff e32c2a64c126 9513b5c2a2f7 2023-09-23T20:16:00.340 INFO:teuthology.orchestra.run.smithi018.stdout:mgr.y smithi018 *:8443,9283,8765 running (49s) - 8m 474M - 18.2.0-460-gbb0daeff e32c2a64c126 5aa74f13c0a1 2023-09-23T20:16:00.340 INFO:teuthology.orchestra.run.smithi018.stdout:mon.a smithi018 running (9m) - 9m 100M 2048M 17.2.6-1123-g61431f6b 0db14aa01094 45b1b543db19 2023-09-23T20:16:00.341 INFO:teuthology.orchestra.run.smithi018.stdout:mon.b smithi149 running (7m) - 7m 67.8M 2048M 17.2.6-1123-g61431f6b 0db14aa01094 e9fa4ded7429 2023-09-23T20:16:00.341 INFO:teuthology.orchestra.run.smithi018.stdout:mon.c smithi018 running (7m) - 7m 65.1M 2048M 17.2.6-1123-g61431f6b 0db14aa01094 d35a46c484d5 2023-09-23T20:16:00.341 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter.a smithi018 *:9100 running (64s) - 3m 8363k - 1.5.0 0da6a335fe13 500f76739e6a 2023-09-23T20:16:00.341 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter.b smithi149 *:9100 running (58s) - 3m 14.5M - 1.5.0 0da6a335fe13 f1789a46736b 2023-09-23T20:16:00.341 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0 smithi018 running (7m) - 7m 73.0M 2501M 17.2.6-1123-g61431f6b 0db14aa01094 6749d0af6e50 2023-09-23T20:16:00.341 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1 smithi018 running (6m) - 6m 91.5M 2501M 17.2.6-1123-g61431f6b 0db14aa01094 5d199baeab0e 2023-09-23T20:16:00.341 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2 smithi018 running (6m) - 6m 75.4M 2501M 17.2.6-1123-g61431f6b 0db14aa01094 2effc63f4610 2023-09-23T20:16:00.341 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3 smithi018 running (6m) - 6m 61.1M 2501M 17.2.6-1123-g61431f6b 0db14aa01094 3301c3e11bd5 2023-09-23T20:16:00.341 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4 smithi149 running (5m) - 5m 70.1M 4096M 17.2.6-1123-g61431f6b 0db14aa01094 b4b39c43de75 2023-09-23T20:16:00.341 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5 smithi149 running (5m) - 5m 84.0M 4096M 17.2.6-1123-g61431f6b 0db14aa01094 cec31ff38f4c 2023-09-23T20:16:00.341 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6 smithi149 running (5m) - 5m 74.8M 4096M 17.2.6-1123-g61431f6b 0db14aa01094 ce687413759d 2023-09-23T20:16:00.341 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7 smithi149 running (4m) - 4m 86.3M 4096M 17.2.6-1123-g61431f6b 0db14aa01094 d3145e28f8c3 2023-09-23T20:16:00.341 INFO:teuthology.orchestra.run.smithi018.stdout:prometheus.a smithi149 *:9095 running (27s) - 4m 41.4M - 2.43.0 a07b618ecd1d bb2162e75dc7 2023-09-23T20:16:00.641 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:16:00 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: 2023-09-23T20:16:00.327+0000 7f953be6a700 -1 mgr.server handle_report got status from non-daemon mon.a 2023-09-23T20:16:00.758 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch upgrade pause' 2023-09-23T20:16:00.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:16:00 smithi149 ceph-mon[100764]: failed to encode map e90 with expected crc 2023-09-23T20:16:00.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:16:00 smithi149 ceph-mon[100764]: failed to encode map e90 with expected crc 2023-09-23T20:16:00.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:16:00 smithi149 ceph-mon[100764]: failed to encode map e90 with expected crc 2023-09-23T20:16:00.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:16:00 smithi149 ceph-mon[100764]: failed to encode map e90 with expected crc 2023-09-23T20:16:00.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:16:00 smithi149 ceph-mon[100764]: failed to encode map e90 with expected crc 2023-09-23T20:16:00.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:16:00 smithi149 ceph-mon[100764]: failed to encode map e90 with expected crc 2023-09-23T20:16:00.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:16:00 smithi149 ceph-mon[100764]: [23/Sep/2023:20:15:59] ENGINE Bus STARTING 2023-09-23T20:16:00.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:16:00 smithi149 ceph-mon[100764]: [23/Sep/2023:20:16:00] ENGINE Serving on http://172.21.15.18:8765 2023-09-23T20:16:00.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:16:00 smithi149 ceph-mon[100764]: mgrmap e34: y(active, since 1.01051s), standbys: x 2023-09-23T20:16:01.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:16:00 smithi018 ceph-mon[179647]: failed to encode map e90 with expected crc 2023-09-23T20:16:01.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:16:00 smithi018 ceph-mon[179647]: failed to encode map e90 with expected crc 2023-09-23T20:16:01.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:16:00 smithi018 ceph-mon[179647]: failed to encode map e90 with expected crc 2023-09-23T20:16:01.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:16:00 smithi018 ceph-mon[179647]: failed to encode map e90 with expected crc 2023-09-23T20:16:01.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:16:00 smithi018 ceph-mon[179647]: failed to encode map e90 with expected crc 2023-09-23T20:16:01.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:16:00 smithi018 ceph-mon[179647]: failed to encode map e90 with expected crc 2023-09-23T20:16:01.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:16:00 smithi018 ceph-mon[179647]: [23/Sep/2023:20:15:59] ENGINE Bus STARTING 2023-09-23T20:16:01.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:16:00 smithi018 ceph-mon[179647]: [23/Sep/2023:20:16:00] ENGINE Serving on http://172.21.15.18:8765 2023-09-23T20:16:01.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:16:00 smithi018 ceph-mon[179647]: mgrmap e34: y(active, since 1.01051s), standbys: x 2023-09-23T20:16:01.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:16:00 smithi018 ceph-mon[109137]: failed to encode map e90 with expected crc 2023-09-23T20:16:01.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:16:00 smithi018 ceph-mon[109137]: failed to encode map e90 with expected crc 2023-09-23T20:16:01.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:16:00 smithi018 ceph-mon[109137]: failed to encode map e90 with expected crc 2023-09-23T20:16:01.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:16:00 smithi018 ceph-mon[109137]: failed to encode map e90 with expected crc 2023-09-23T20:16:01.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:16:00 smithi018 ceph-mon[109137]: failed to encode map e90 with expected crc 2023-09-23T20:16:01.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:16:00 smithi018 ceph-mon[109137]: failed to encode map e90 with expected crc 2023-09-23T20:16:01.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:16:00 smithi018 ceph-mon[109137]: [23/Sep/2023:20:15:59] ENGINE Bus STARTING 2023-09-23T20:16:01.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:16:00 smithi018 ceph-mon[109137]: [23/Sep/2023:20:16:00] ENGINE Serving on http://172.21.15.18:8765 2023-09-23T20:16:01.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:16:00 smithi018 ceph-mon[109137]: mgrmap e34: y(active, since 1.01051s), standbys: x 2023-09-23T20:16:01.473 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:16:02.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:16:01 smithi018 ceph-mon[179647]: [23/Sep/2023:20:16:00] ENGINE Serving on https://172.21.15.18:7150 2023-09-23T20:16:02.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:16:01 smithi018 ceph-mon[179647]: [23/Sep/2023:20:16:00] ENGINE Bus STARTED 2023-09-23T20:16:02.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:16:01 smithi018 ceph-mon[179647]: from='client.25826 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:16:02.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:16:01 smithi018 ceph-mon[179647]: pgmap v3: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:16:02.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:16:01 smithi018 ceph-mon[109137]: [23/Sep/2023:20:16:00] ENGINE Serving on https://172.21.15.18:7150 2023-09-23T20:16:02.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:16:01 smithi018 ceph-mon[109137]: [23/Sep/2023:20:16:00] ENGINE Bus STARTED 2023-09-23T20:16:02.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:16:01 smithi018 ceph-mon[109137]: from='client.25826 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:16:02.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:16:01 smithi018 ceph-mon[109137]: pgmap v3: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:16:02.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:16:01 smithi149 ceph-mon[100764]: [23/Sep/2023:20:16:00] ENGINE Serving on https://172.21.15.18:7150 2023-09-23T20:16:02.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:16:01 smithi149 ceph-mon[100764]: [23/Sep/2023:20:16:00] ENGINE Bus STARTED 2023-09-23T20:16:02.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:16:01 smithi149 ceph-mon[100764]: from='client.25826 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:16:02.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:16:01 smithi149 ceph-mon[100764]: pgmap v3: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:16:02.805 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:16:02 smithi018 ceph-mon[109137]: pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:16:02.806 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:16:02 smithi018 ceph-mon[109137]: mgrmap e35: y(active, since 3s), standbys: x 2023-09-23T20:16:02.806 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:16:02 smithi018 ceph-mon[179647]: pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:16:02.806 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:16:02 smithi018 ceph-mon[179647]: mgrmap e35: y(active, since 3s), standbys: x 2023-09-23T20:16:03.215 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:16:02 smithi149 ceph-mon[100764]: pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:16:03.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:16:02 smithi149 ceph-mon[100764]: mgrmap e35: y(active, since 3s), standbys: x 2023-09-23T20:16:03.257 INFO:teuthology.orchestra.run.smithi018.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T20:16:03.842 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'sleep 60' 2023-09-23T20:16:04.500 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:16:04.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:16:04 smithi018 ceph-mon[109137]: from='client.25862 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:16:04.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:16:04 smithi018 ceph-mon[109137]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T20:16:04.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:16:04 smithi018 ceph-mon[109137]: from='mgr.34107 ' entity='mgr.y' 2023-09-23T20:16:04.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:16:04 smithi018 ceph-mon[109137]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:16:04.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:16:04 smithi018 ceph-mon[179647]: from='client.25862 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:16:04.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:16:04 smithi018 ceph-mon[179647]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T20:16:04.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:16:04 smithi018 ceph-mon[179647]: from='mgr.34107 ' entity='mgr.y' 2023-09-23T20:16:04.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:16:04 smithi018 ceph-mon[179647]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:16:04.715 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:16:04 smithi149 ceph-mon[100764]: from='client.25862 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:16:04.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:16:04 smithi149 ceph-mon[100764]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T20:16:04.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:16:04 smithi149 ceph-mon[100764]: from='mgr.34107 ' entity='mgr.y' 2023-09-23T20:16:04.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:16:04 smithi149 ceph-mon[100764]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:16:05.391 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:16:04 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:16:04] "GET /metrics HTTP/1.1" 200 31228 "" "Prometheus/2.43.0" 2023-09-23T20:16:05.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:16:05 smithi018 ceph-mon[109137]: from='mgr.34107 ' entity='mgr.y' 2023-09-23T20:16:05.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:16:05 smithi018 ceph-mon[109137]: from='mgr.34107 ' entity='mgr.y' 2023-09-23T20:16:05.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:16:05 smithi018 ceph-mon[109137]: from='mgr.34107 ' entity='mgr.y' 2023-09-23T20:16:05.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:16:05 smithi018 ceph-mon[109137]: from='mgr.34107 ' entity='mgr.y' 2023-09-23T20:16:05.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:16:05 smithi018 ceph-mon[109137]: from='mgr.34107 ' entity='mgr.y' 2023-09-23T20:16:05.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:16:05 smithi018 ceph-mon[109137]: from='mgr.34107 ' entity='mgr.y' 2023-09-23T20:16:05.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:16:05 smithi018 ceph-mon[109137]: from='mgr.34107 ' entity='mgr.y' 2023-09-23T20:16:05.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:16:05 smithi018 ceph-mon[109137]: from='mgr.34107 ' entity='mgr.y' 2023-09-23T20:16:05.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:16:05 smithi018 ceph-mon[179647]: from='mgr.34107 ' entity='mgr.y' 2023-09-23T20:16:05.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:16:05 smithi018 ceph-mon[179647]: from='mgr.34107 ' entity='mgr.y' 2023-09-23T20:16:05.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:16:05 smithi018 ceph-mon[179647]: from='mgr.34107 ' entity='mgr.y' 2023-09-23T20:16:05.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:16:05 smithi018 ceph-mon[179647]: from='mgr.34107 ' entity='mgr.y' 2023-09-23T20:16:05.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:16:05 smithi018 ceph-mon[179647]: from='mgr.34107 ' entity='mgr.y' 2023-09-23T20:16:05.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:16:05 smithi018 ceph-mon[179647]: from='mgr.34107 ' entity='mgr.y' 2023-09-23T20:16:05.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:16:05 smithi018 ceph-mon[179647]: from='mgr.34107 ' entity='mgr.y' 2023-09-23T20:16:05.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:16:05 smithi018 ceph-mon[179647]: from='mgr.34107 ' entity='mgr.y' 2023-09-23T20:16:05.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:16:05 smithi149 ceph-mon[100764]: from='mgr.34107 ' entity='mgr.y' 2023-09-23T20:16:05.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:16:05 smithi149 ceph-mon[100764]: from='mgr.34107 ' entity='mgr.y' 2023-09-23T20:16:05.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:16:05 smithi149 ceph-mon[100764]: from='mgr.34107 ' entity='mgr.y' 2023-09-23T20:16:05.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:16:05 smithi149 ceph-mon[100764]: from='mgr.34107 ' entity='mgr.y' 2023-09-23T20:16:05.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:16:05 smithi149 ceph-mon[100764]: from='mgr.34107 ' entity='mgr.y' 2023-09-23T20:16:05.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:16:05 smithi149 ceph-mon[100764]: from='mgr.34107 ' entity='mgr.y' 2023-09-23T20:16:05.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:16:05 smithi149 ceph-mon[100764]: from='mgr.34107 ' entity='mgr.y' 2023-09-23T20:16:05.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:16:05 smithi149 ceph-mon[100764]: from='mgr.34107 ' entity='mgr.y' 2023-09-23T20:16:07.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:16:06 smithi018 ceph-mon[109137]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:16:07.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:16:06 smithi018 ceph-mon[179647]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:16:07.206 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:16:06 smithi149 ceph-mon[100764]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:16:08.966 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:16:08 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[156506]: ts=2023-09-23T20:16:08.658Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.18\", device_class=\"ssd\", hostname=\"smithi018\", instance=\"smithi149.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.18\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.18\", device_class=\"ssd\", hostname=\"smithi018\", instance=\"smithi018:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.18\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T20:16:08.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:16:08 smithi149 ceph-mon[100764]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:16:08.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:16:08 smithi149 ceph-mon[100764]: from='mgr.34107 ' entity='mgr.y' 2023-09-23T20:16:08.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:16:08 smithi149 ceph-mon[100764]: from='mgr.34107 ' entity='mgr.y' 2023-09-23T20:16:08.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:16:08 smithi149 ceph-mon[100764]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi149", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:16:08.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:16:08 smithi149 ceph-mon[100764]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi149", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:16:08.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:16:08 smithi149 ceph-mon[100764]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:16:08.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:16:08 smithi149 ceph-mon[100764]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:16:09.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:16:08 smithi018 ceph-mon[109137]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:16:09.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:16:08 smithi018 ceph-mon[109137]: from='mgr.34107 ' entity='mgr.y' 2023-09-23T20:16:09.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:16:08 smithi018 ceph-mon[109137]: from='mgr.34107 ' entity='mgr.y' 2023-09-23T20:16:09.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:16:08 smithi018 ceph-mon[109137]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi149", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:16:09.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:16:08 smithi018 ceph-mon[109137]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi149", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:16:09.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:16:08 smithi018 ceph-mon[109137]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:16:09.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:16:08 smithi018 ceph-mon[109137]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:16:09.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:16:08 smithi018 ceph-mon[179647]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:16:09.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:16:08 smithi018 ceph-mon[179647]: from='mgr.34107 ' entity='mgr.y' 2023-09-23T20:16:09.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:16:08 smithi018 ceph-mon[179647]: from='mgr.34107 ' entity='mgr.y' 2023-09-23T20:16:09.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:16:08 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi149", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:16:09.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:16:08 smithi018 ceph-mon[179647]: from='mgr.34107 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi149", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:16:09.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:16:08 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:16:09.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:16:08 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:16:10.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:16:09 smithi018 ceph-mon[109137]: Updating smithi018:/etc/ceph/ceph.conf 2023-09-23T20:16:10.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:16:09 smithi018 ceph-mon[109137]: Updating smithi149:/etc/ceph/ceph.conf 2023-09-23T20:16:10.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:16:09 smithi018 ceph-mon[109137]: Updating smithi018:/var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T20:16:10.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:16:09 smithi018 ceph-mon[109137]: Updating smithi149:/var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T20:16:10.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:16:09 smithi018 ceph-mon[109137]: from='mgr.34107 ' entity='mgr.y' 2023-09-23T20:16:10.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:16:09 smithi018 ceph-mon[109137]: from='mgr.34107 ' entity='mgr.y' 2023-09-23T20:16:10.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:16:09 smithi018 ceph-mon[109137]: from='mgr.34107 ' entity='mgr.y' 2023-09-23T20:16:10.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:16:09 smithi018 ceph-mon[109137]: from='mgr.34107 ' entity='mgr.y' 2023-09-23T20:16:10.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:16:09 smithi018 ceph-mon[109137]: from='mgr.34107 ' entity='mgr.y' 2023-09-23T20:16:10.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:16:09 smithi018 ceph-mon[179647]: Updating smithi018:/etc/ceph/ceph.conf 2023-09-23T20:16:10.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:16:09 smithi018 ceph-mon[179647]: Updating smithi149:/etc/ceph/ceph.conf 2023-09-23T20:16:10.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:16:09 smithi018 ceph-mon[179647]: Updating smithi018:/var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T20:16:10.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:16:09 smithi018 ceph-mon[179647]: Updating smithi149:/var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T20:16:10.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:16:09 smithi018 ceph-mon[179647]: from='mgr.34107 ' entity='mgr.y' 2023-09-23T20:16:10.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:16:09 smithi018 ceph-mon[179647]: from='mgr.34107 ' entity='mgr.y' 2023-09-23T20:16:10.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:16:09 smithi018 ceph-mon[179647]: from='mgr.34107 ' entity='mgr.y' 2023-09-23T20:16:10.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:16:09 smithi018 ceph-mon[179647]: from='mgr.34107 ' entity='mgr.y' 2023-09-23T20:16:10.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:16:09 smithi018 ceph-mon[179647]: from='mgr.34107 ' entity='mgr.y' 2023-09-23T20:16:10.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:16:09 smithi149 ceph-mon[100764]: Updating smithi018:/etc/ceph/ceph.conf 2023-09-23T20:16:10.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:16:09 smithi149 ceph-mon[100764]: Updating smithi149:/etc/ceph/ceph.conf 2023-09-23T20:16:10.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:16:09 smithi149 ceph-mon[100764]: Updating smithi018:/var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T20:16:10.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:16:09 smithi149 ceph-mon[100764]: Updating smithi149:/var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T20:16:10.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:16:09 smithi149 ceph-mon[100764]: from='mgr.34107 ' entity='mgr.y' 2023-09-23T20:16:10.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:16:09 smithi149 ceph-mon[100764]: from='mgr.34107 ' entity='mgr.y' 2023-09-23T20:16:10.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:16:09 smithi149 ceph-mon[100764]: from='mgr.34107 ' entity='mgr.y' 2023-09-23T20:16:10.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:16:09 smithi149 ceph-mon[100764]: from='mgr.34107 ' entity='mgr.y' 2023-09-23T20:16:10.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:16:09 smithi149 ceph-mon[100764]: from='mgr.34107 ' entity='mgr.y' 2023-09-23T20:16:11.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:16:10 smithi018 ceph-mon[179647]: Updating smithi018:/etc/ceph/ceph.client.admin.keyring 2023-09-23T20:16:11.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:16:10 smithi018 ceph-mon[179647]: Updating smithi149:/etc/ceph/ceph.client.admin.keyring 2023-09-23T20:16:11.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:16:10 smithi018 ceph-mon[179647]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:16:11.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:16:10 smithi018 ceph-mon[179647]: Updating smithi149:/var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T20:16:11.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:16:10 smithi018 ceph-mon[179647]: Updating smithi018:/var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T20:16:11.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:16:10 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:16:11.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:16:10 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:16:11.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:16:10 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:16:11.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:16:10 smithi018 ceph-mon[179647]: from='mgr.34107 ' entity='mgr.y' 2023-09-23T20:16:11.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:16:10 smithi018 ceph-mon[109137]: Updating smithi018:/etc/ceph/ceph.client.admin.keyring 2023-09-23T20:16:11.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:16:10 smithi018 ceph-mon[109137]: Updating smithi149:/etc/ceph/ceph.client.admin.keyring 2023-09-23T20:16:11.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:16:10 smithi018 ceph-mon[109137]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:16:11.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:16:10 smithi018 ceph-mon[109137]: Updating smithi149:/var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T20:16:11.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:16:10 smithi018 ceph-mon[109137]: Updating smithi018:/var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T20:16:11.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:16:10 smithi018 ceph-mon[109137]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:16:11.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:16:10 smithi018 ceph-mon[109137]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:16:11.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:16:10 smithi018 ceph-mon[109137]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:16:11.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:16:10 smithi018 ceph-mon[109137]: from='mgr.34107 ' entity='mgr.y' 2023-09-23T20:16:11.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:16:10 smithi149 ceph-mon[100764]: Updating smithi018:/etc/ceph/ceph.client.admin.keyring 2023-09-23T20:16:11.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:16:10 smithi149 ceph-mon[100764]: Updating smithi149:/etc/ceph/ceph.client.admin.keyring 2023-09-23T20:16:11.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:16:10 smithi149 ceph-mon[100764]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:16:11.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:16:10 smithi149 ceph-mon[100764]: Updating smithi149:/var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T20:16:11.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:16:10 smithi149 ceph-mon[100764]: Updating smithi018:/var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T20:16:11.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:16:10 smithi149 ceph-mon[100764]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:16:11.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:16:10 smithi149 ceph-mon[100764]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:16:11.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:16:10 smithi149 ceph-mon[100764]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:16:11.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:16:10 smithi149 ceph-mon[100764]: from='mgr.34107 ' entity='mgr.y' 2023-09-23T20:16:13.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:16:12 smithi018 ceph-mon[109137]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:16:13.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:16:12 smithi018 ceph-mon[179647]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:16:13.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:16:12 smithi149 ceph-mon[100764]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:16:15.141 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:16:14 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:16:14] "GET /metrics HTTP/1.1" 200 33909 "" "Prometheus/2.43.0" 2023-09-23T20:16:15.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:16:14 smithi018 ceph-mon[179647]: pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:16:15.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:16:14 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:16:15.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:16:14 smithi018 ceph-mon[109137]: pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:16:15.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:16:14 smithi018 ceph-mon[109137]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:16:15.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:16:14 smithi149 ceph-mon[100764]: pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:16:15.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:16:14 smithi149 ceph-mon[100764]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:16:17.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:16:16 smithi018 ceph-mon[109137]: pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:16:17.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:16:16 smithi018 ceph-mon[179647]: pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:16:17.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:16:16 smithi149 ceph-mon[100764]: pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:16:18.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:16:18 smithi149 ceph-mon[100764]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:16:18.967 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:16:18 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[156506]: ts=2023-09-23T20:16:18.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.18\", device_class=\"ssd\", hostname=\"smithi018\", instance=\"smithi149.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.18\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.18\", device_class=\"ssd\", hostname=\"smithi018\", instance=\"smithi018:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.18\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T20:16:19.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:16:18 smithi018 ceph-mon[109137]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:16:19.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:16:18 smithi018 ceph-mon[179647]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:16:21.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:16:20 smithi018 ceph-mon[109137]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:16:21.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:16:20 smithi018 ceph-mon[179647]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:16:21.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:16:20 smithi149 ceph-mon[100764]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:16:23.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:16:22 smithi018 ceph-mon[109137]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:16:23.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:16:22 smithi018 ceph-mon[179647]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:16:23.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:16:22 smithi149 ceph-mon[100764]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:16:25.141 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:16:24 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:16:24] "GET /metrics HTTP/1.1" 200 33909 "" "Prometheus/2.43.0" 2023-09-23T20:16:25.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:16:24 smithi018 ceph-mon[109137]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:16:25.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:16:24 smithi018 ceph-mon[179647]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:16:25.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:16:24 smithi149 ceph-mon[100764]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:16:27.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:16:26 smithi018 ceph-mon[109137]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:16:27.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:16:26 smithi018 ceph-mon[179647]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:16:27.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:16:26 smithi149 ceph-mon[100764]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:16:28.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:16:28 smithi149 ceph-mon[100764]: pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:16:28.967 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:16:28 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[156506]: ts=2023-09-23T20:16:28.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.18\", device_class=\"ssd\", hostname=\"smithi018\", instance=\"smithi149.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.18\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.18\", device_class=\"ssd\", hostname=\"smithi018\", instance=\"smithi018:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.18\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T20:16:29.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:16:28 smithi018 ceph-mon[109137]: pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:16:29.144 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:16:28 smithi018 ceph-mon[179647]: pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:16:30.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:16:29 smithi018 ceph-mon[109137]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:16:30.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:16:29 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:16:30.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:16:29 smithi149 ceph-mon[100764]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:16:31.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:16:30 smithi018 ceph-mon[109137]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:16:31.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:16:30 smithi018 ceph-mon[179647]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:16:31.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:16:30 smithi149 ceph-mon[100764]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:16:33.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:16:32 smithi018 ceph-mon[109137]: pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:16:33.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:16:32 smithi018 ceph-mon[179647]: pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:16:33.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:16:32 smithi149 ceph-mon[100764]: pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:16:35.141 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:16:34 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:16:34] "GET /metrics HTTP/1.1" 200 33909 "" "Prometheus/2.43.0" 2023-09-23T20:16:35.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:16:34 smithi018 ceph-mon[109137]: pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:16:35.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:16:34 smithi018 ceph-mon[179647]: pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:16:35.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:16:34 smithi149 ceph-mon[100764]: pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:16:37.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:16:36 smithi018 ceph-mon[179647]: pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:16:37.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:16:36 smithi018 ceph-mon[109137]: pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:16:37.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:16:36 smithi149 ceph-mon[100764]: pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:16:38.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:16:38 smithi149 ceph-mon[100764]: pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:16:38.966 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:16:38 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[156506]: ts=2023-09-23T20:16:38.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.18\", device_class=\"ssd\", hostname=\"smithi018\", instance=\"smithi149.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.18\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.18\", device_class=\"ssd\", hostname=\"smithi018\", instance=\"smithi018:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.18\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T20:16:39.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:16:38 smithi018 ceph-mon[179647]: pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:16:39.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:16:38 smithi018 ceph-mon[109137]: pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:16:41.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:16:40 smithi018 ceph-mon[109137]: pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:16:41.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:16:40 smithi018 ceph-mon[179647]: pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:16:41.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:16:40 smithi149 ceph-mon[100764]: pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:16:43.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:16:42 smithi018 ceph-mon[179647]: pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:16:43.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:16:42 smithi018 ceph-mon[109137]: pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:16:43.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:16:42 smithi149 ceph-mon[100764]: pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:16:45.141 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:16:44 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:16:44] "GET /metrics HTTP/1.1" 200 33908 "" "Prometheus/2.43.0" 2023-09-23T20:16:45.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:16:44 smithi018 ceph-mon[179647]: pgmap v25: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:16:45.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:16:44 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:16:45.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:16:44 smithi018 ceph-mon[109137]: pgmap v25: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:16:45.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:16:44 smithi018 ceph-mon[109137]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:16:45.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:16:44 smithi149 ceph-mon[100764]: pgmap v25: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:16:45.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:16:44 smithi149 ceph-mon[100764]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:16:47.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:16:46 smithi018 ceph-mon[109137]: pgmap v26: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:16:47.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:16:46 smithi018 ceph-mon[179647]: pgmap v26: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:16:47.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:16:46 smithi149 ceph-mon[100764]: pgmap v26: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:16:48.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:16:48 smithi149 ceph-mon[100764]: pgmap v27: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:16:48.966 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:16:48 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[156506]: ts=2023-09-23T20:16:48.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.18\", device_class=\"ssd\", hostname=\"smithi018\", instance=\"smithi149.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.18\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.18\", device_class=\"ssd\", hostname=\"smithi018\", instance=\"smithi018:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.18\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T20:16:49.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:16:48 smithi018 ceph-mon[179647]: pgmap v27: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:16:49.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:16:48 smithi018 ceph-mon[109137]: pgmap v27: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:16:51.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:16:50 smithi018 ceph-mon[179647]: pgmap v28: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:16:51.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:16:50 smithi018 ceph-mon[109137]: pgmap v28: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:16:51.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:16:50 smithi149 ceph-mon[100764]: pgmap v28: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:16:53.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:16:52 smithi018 ceph-mon[109137]: pgmap v29: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:16:53.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:16:52 smithi018 ceph-mon[179647]: pgmap v29: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:16:53.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:16:52 smithi149 ceph-mon[100764]: pgmap v29: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:16:55.141 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:16:54 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:16:54] "GET /metrics HTTP/1.1" 200 33908 "" "Prometheus/2.43.0" 2023-09-23T20:16:55.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:16:54 smithi018 ceph-mon[109137]: pgmap v30: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:16:55.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:16:54 smithi018 ceph-mon[179647]: pgmap v30: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:16:55.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:16:54 smithi149 ceph-mon[100764]: pgmap v30: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:16:57.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:16:56 smithi018 ceph-mon[109137]: pgmap v31: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:16:57.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:16:56 smithi018 ceph-mon[179647]: pgmap v31: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:16:57.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:16:56 smithi149 ceph-mon[100764]: pgmap v31: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:16:58.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:16:58 smithi149 ceph-mon[100764]: pgmap v32: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:16:58.967 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:16:58 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[156506]: ts=2023-09-23T20:16:58.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.18\", device_class=\"ssd\", hostname=\"smithi018\", instance=\"smithi149.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.18\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.18\", device_class=\"ssd\", hostname=\"smithi018\", instance=\"smithi018:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.18\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T20:16:59.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:16:58 smithi018 ceph-mon[109137]: pgmap v32: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:16:59.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:16:58 smithi018 ceph-mon[179647]: pgmap v32: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:17:00.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:16:59 smithi018 ceph-mon[109137]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:17:00.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:16:59 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:17:00.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:16:59 smithi149 ceph-mon[100764]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:17:01.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:00 smithi018 ceph-mon[109137]: pgmap v33: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:17:01.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:17:00 smithi018 ceph-mon[179647]: pgmap v33: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:17:01.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:17:00 smithi149 ceph-mon[100764]: pgmap v33: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:17:03.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:02 smithi018 ceph-mon[109137]: pgmap v34: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:17:03.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:17:02 smithi018 ceph-mon[179647]: pgmap v34: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:17:03.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:17:02 smithi149 ceph-mon[100764]: pgmap v34: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:17:05.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:04 smithi018 ceph-mon[109137]: pgmap v35: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:17:05.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:17:04 smithi018 ceph-mon[179647]: pgmap v35: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:17:05.142 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:17:04 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:17:04] "GET /metrics HTTP/1.1" 200 33907 "" "Prometheus/2.43.0" 2023-09-23T20:17:05.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:17:04 smithi149 ceph-mon[100764]: pgmap v35: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:17:05.910 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch upgrade resume' 2023-09-23T20:17:06.524 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:17:06.968 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:06 smithi018 ceph-mon[109137]: pgmap v36: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:17:06.969 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:17:06 smithi018 ceph-mon[179647]: pgmap v36: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:17:07.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:17:06 smithi149 ceph-mon[100764]: pgmap v36: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:17:07.826 INFO:teuthology.orchestra.run.smithi018.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T20:17:08.274 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'echo wait for majority of mons to upgrade' 2023-09-23T20:17:08.877 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:17:08.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:17:08 smithi149 ceph-mon[100764]: pgmap v37: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:17:08.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:17:08 smithi149 ceph-mon[100764]: from='client.34164 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:17:08.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:17:08 smithi149 ceph-mon[100764]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T20:17:08.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:17:08 smithi149 ceph-mon[100764]: from='mgr.34107 ' entity='mgr.y' 2023-09-23T20:17:08.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:17:08 smithi149 ceph-mon[100764]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:17:08.967 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:17:08 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[156506]: ts=2023-09-23T20:17:08.658Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.18\", device_class=\"ssd\", hostname=\"smithi018\", instance=\"smithi149.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.18\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.18\", device_class=\"ssd\", hostname=\"smithi018\", instance=\"smithi018:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.18\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T20:17:09.100 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:08 smithi018 ceph-mon[109137]: pgmap v37: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:17:09.100 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:08 smithi018 ceph-mon[109137]: from='client.34164 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:17:09.100 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:08 smithi018 ceph-mon[109137]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T20:17:09.100 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:08 smithi018 ceph-mon[109137]: from='mgr.34107 ' entity='mgr.y' 2023-09-23T20:17:09.100 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:08 smithi018 ceph-mon[109137]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:17:09.100 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:17:08 smithi018 ceph-mon[179647]: pgmap v37: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:17:09.101 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:17:08 smithi018 ceph-mon[179647]: from='client.34164 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:17:09.101 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:17:08 smithi018 ceph-mon[179647]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T20:17:09.101 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:17:08 smithi018 ceph-mon[179647]: from='mgr.34107 ' entity='mgr.y' 2023-09-23T20:17:09.101 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:17:08 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:17:10.493 INFO:teuthology.orchestra.run.smithi018.stdout:wait for majority of mons to upgrade 2023-09-23T20:17:10.785 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'while ! ceph mon versions | grep $sha1 | egrep '"'"': [23]'"'"' ; do sleep 2 ; done' 2023-09-23T20:17:11.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:10 smithi018 ceph-mon[109137]: pgmap v38: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:17:11.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:17:10 smithi018 ceph-mon[179647]: pgmap v38: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:17:11.215 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:17:10 smithi149 ceph-mon[100764]: pgmap v38: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:17:11.401 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:17:13.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:12 smithi018 ceph-mon[109137]: pgmap v39: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:17:13.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:12 smithi018 ceph-mon[109137]: from='mgr.34107 ' entity='mgr.y' 2023-09-23T20:17:13.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:12 smithi018 ceph-mon[109137]: from='mgr.34107 ' entity='mgr.y' 2023-09-23T20:17:13.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:17:12 smithi018 ceph-mon[179647]: pgmap v39: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:17:13.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:17:12 smithi018 ceph-mon[179647]: from='mgr.34107 ' entity='mgr.y' 2023-09-23T20:17:13.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:17:12 smithi018 ceph-mon[179647]: from='mgr.34107 ' entity='mgr.y' 2023-09-23T20:17:13.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:17:12 smithi149 ceph-mon[100764]: pgmap v39: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:17:13.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:17:12 smithi149 ceph-mon[100764]: from='mgr.34107 ' entity='mgr.y' 2023-09-23T20:17:13.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:17:12 smithi149 ceph-mon[100764]: from='mgr.34107 ' entity='mgr.y' 2023-09-23T20:17:14.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:13 smithi018 ceph-mon[109137]: from='mgr.34107 ' entity='mgr.y' 2023-09-23T20:17:14.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:13 smithi018 ceph-mon[109137]: from='mgr.34107 ' entity='mgr.y' 2023-09-23T20:17:14.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:13 smithi018 ceph-mon[109137]: from='client.? 172.21.15.18:0/2198923867' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:17:14.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:13 smithi018 ceph-mon[109137]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:17:14.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:13 smithi018 ceph-mon[109137]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:17:14.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:13 smithi018 ceph-mon[109137]: from='mgr.34107 ' entity='mgr.y' 2023-09-23T20:17:14.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:13 smithi018 ceph-mon[109137]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:17:14.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:13 smithi018 ceph-mon[109137]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:17:14.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:13 smithi018 ceph-mon[109137]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-09-23T20:17:14.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:13 smithi018 ceph-mon[109137]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2023-09-23T20:17:14.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:17:13 smithi018 ceph-mon[179647]: from='mgr.34107 ' entity='mgr.y' 2023-09-23T20:17:14.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:17:13 smithi018 ceph-mon[179647]: from='mgr.34107 ' entity='mgr.y' 2023-09-23T20:17:14.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:17:13 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2198923867' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:17:14.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:17:13 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:17:14.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:17:13 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:17:14.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:17:13 smithi018 ceph-mon[179647]: from='mgr.34107 ' entity='mgr.y' 2023-09-23T20:17:14.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:17:13 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:17:14.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:17:13 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:17:14.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:17:13 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-09-23T20:17:14.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:17:13 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2023-09-23T20:17:14.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:17:13 smithi149 ceph-mon[100764]: from='mgr.34107 ' entity='mgr.y' 2023-09-23T20:17:14.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:17:13 smithi149 ceph-mon[100764]: from='mgr.34107 ' entity='mgr.y' 2023-09-23T20:17:14.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:17:13 smithi149 ceph-mon[100764]: from='client.? 172.21.15.18:0/2198923867' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:17:14.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:17:13 smithi149 ceph-mon[100764]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:17:14.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:17:13 smithi149 ceph-mon[100764]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:17:14.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:17:13 smithi149 ceph-mon[100764]: from='mgr.34107 ' entity='mgr.y' 2023-09-23T20:17:14.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:17:13 smithi149 ceph-mon[100764]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:17:14.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:17:13 smithi149 ceph-mon[100764]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:17:14.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:17:13 smithi149 ceph-mon[100764]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-09-23T20:17:14.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:17:13 smithi149 ceph-mon[100764]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2023-09-23T20:17:14.994 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:14 smithi018 ceph-mon[109137]: pgmap v40: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:17:14.994 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:14 smithi018 ceph-mon[109137]: Upgrade: It appears safe to stop mon.c 2023-09-23T20:17:14.994 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:14 smithi018 ceph-mon[109137]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:17:14.994 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:14 smithi018 ceph-mon[109137]: from='mgr.34107 ' entity='mgr.y' 2023-09-23T20:17:14.994 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:14 smithi018 ceph-mon[109137]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T20:17:14.994 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:14 smithi018 ceph-mon[109137]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T20:17:14.994 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:14 smithi018 ceph-mon[109137]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:17:14.995 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:17:14 smithi018 ceph-mon[179647]: pgmap v40: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:17:14.995 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:17:14 smithi018 ceph-mon[179647]: Upgrade: It appears safe to stop mon.c 2023-09-23T20:17:14.995 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:17:14 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:17:14.995 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:17:14 smithi018 ceph-mon[179647]: from='mgr.34107 ' entity='mgr.y' 2023-09-23T20:17:14.995 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:17:14 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T20:17:14.995 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:17:14 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T20:17:14.995 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:17:14 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:17:15.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:17:14 smithi149 ceph-mon[100764]: pgmap v40: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:17:15.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:17:14 smithi149 ceph-mon[100764]: Upgrade: It appears safe to stop mon.c 2023-09-23T20:17:15.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:17:14 smithi149 ceph-mon[100764]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:17:15.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:17:14 smithi149 ceph-mon[100764]: from='mgr.34107 ' entity='mgr.y' 2023-09-23T20:17:15.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:17:14 smithi149 ceph-mon[100764]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T20:17:15.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:17:14 smithi149 ceph-mon[100764]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T20:17:15.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:17:14 smithi149 ceph-mon[100764]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:17:15.266 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:17:14 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:17:14] "GET /metrics HTTP/1.1" 200 33909 "" "Prometheus/2.43.0" 2023-09-23T20:17:15.854 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:17:15 smithi018 ceph-mon[179647]: Upgrade: Updating mon.c 2023-09-23T20:17:15.854 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:17:15 smithi018 ceph-mon[179647]: Deploying daemon mon.c on smithi018 2023-09-23T20:17:15.854 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:17:15 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1041116911' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:17:16.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:15 smithi018 ceph-mon[109137]: Upgrade: Updating mon.c 2023-09-23T20:17:16.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:15 smithi018 ceph-mon[109137]: Deploying daemon mon.c on smithi018 2023-09-23T20:17:16.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:15 smithi018 ceph-mon[109137]: from='client.? 172.21.15.18:0/1041116911' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:17:16.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:16 smithi018 systemd[1]: Stopping Ceph mon.c for a3b3b17e-5a4c-11ee-8db4-212e2dc638e7... 2023-09-23T20:17:16.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:17:15 smithi149 ceph-mon[100764]: Upgrade: Updating mon.c 2023-09-23T20:17:16.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:17:15 smithi149 ceph-mon[100764]: Deploying daemon mon.c on smithi018 2023-09-23T20:17:16.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:17:15 smithi149 ceph-mon[100764]: from='client.? 172.21.15.18:0/1041116911' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:17:16.595 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:16 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mon-c[109113]: 2023-09-23T20:17:16.339+0000 7fecc7d0f700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T20:17:16.595 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:16 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mon-c[109113]: 2023-09-23T20:17:16.339+0000 7fecc7d0f700 -1 mon.c@2(peon) e3 *** Got Signal Terminated *** 2023-09-23T20:17:16.595 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:17:16 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: [23/Sep/2023:20:17:16] ENGINE Bus STOPPING 2023-09-23T20:17:16.596 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:17:16 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: [23/Sep/2023:20:17:16] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T20:17:16.596 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:17:16 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: [23/Sep/2023:20:17:16] ENGINE Bus STOPPED 2023-09-23T20:17:16.596 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:17:16 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: [23/Sep/2023:20:17:16] ENGINE Bus STARTING 2023-09-23T20:17:16.596 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:17:16 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: [23/Sep/2023:20:17:16] ENGINE Serving on http://:::9283 2023-09-23T20:17:16.596 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:17:16 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: [23/Sep/2023:20:17:16] ENGINE Bus STARTED 2023-09-23T20:17:16.890 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:16 smithi018 bash[183924]: ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mon-c 2023-09-23T20:17:17.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:16 smithi018 systemd[1]: ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7@mon.c.service: Succeeded. 2023-09-23T20:17:17.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:16 smithi018 systemd[1]: Stopped Ceph mon.c for a3b3b17e-5a4c-11ee-8db4-212e2dc638e7. 2023-09-23T20:17:17.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 systemd[1]: Starting Ceph mon.c for a3b3b17e-5a4c-11ee-8db4-212e2dc638e7... 2023-09-23T20:17:17.821 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 podman[184081]: 2023-09-23T20:17:17.821 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: set uid:gid to 167:167 (ceph:ceph) 2023-09-23T20:17:17.821 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable), process ceph-mon, pid 7 2023-09-23T20:17:17.821 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: pidfile_write: ignore empty --pid-file 2023-09-23T20:17:17.821 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 bash[184081]: e2701e7c5d359b7ca33b63a25d2bdbefcdb86f5822851aca652bce25013799f1 2023-09-23T20:17:17.821 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: load: jerasure load: lrc 2023-09-23T20:17:17.822 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: RocksDB version: 7.9.2 2023-09-23T20:17:17.822 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Git sha 0 2023-09-23T20:17:17.822 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Compile date 2023-09-22 15:19:43 2023-09-23T20:17:17.822 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: DB SUMMARY 2023-09-23T20:17:17.822 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: DB Session ID: J122X4W4NI017QS66CZI 2023-09-23T20:17:17.822 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: CURRENT file: CURRENT 2023-09-23T20:17:17.822 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: IDENTITY file: IDENTITY 2023-09-23T20:17:17.822 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: MANIFEST file: MANIFEST-000009 size: 1083 Bytes 2023-09-23T20:17:17.822 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 1, files: 000027.sst 2023-09-23T20:17:17.822 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000025.log size: 339503 ; 2023-09-23T20:17:17.822 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.error_if_exists: 0 2023-09-23T20:17:17.822 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.create_if_missing: 0 2023-09-23T20:17:17.823 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.paranoid_checks: 1 2023-09-23T20:17:17.823 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.flush_verify_memtable_count: 1 2023-09-23T20:17:17.823 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-09-23T20:17:17.823 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2023-09-23T20:17:17.823 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.env: 0x563598d07720 2023-09-23T20:17:17.823 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.fs: PosixFileSystem 2023-09-23T20:17:17.823 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.info_log: 0x56359b67d420 2023-09-23T20:17:17.823 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.max_file_opening_threads: 16 2023-09-23T20:17:17.823 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.statistics: (nil) 2023-09-23T20:17:17.823 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.use_fsync: 0 2023-09-23T20:17:17.823 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.max_log_file_size: 0 2023-09-23T20:17:17.823 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.max_manifest_file_size: 1073741824 2023-09-23T20:17:17.824 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.log_file_time_to_roll: 0 2023-09-23T20:17:17.824 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.keep_log_file_num: 1000 2023-09-23T20:17:17.824 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.recycle_log_file_num: 0 2023-09-23T20:17:17.824 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.allow_fallocate: 1 2023-09-23T20:17:17.824 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.allow_mmap_reads: 0 2023-09-23T20:17:17.824 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.allow_mmap_writes: 0 2023-09-23T20:17:17.824 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.use_direct_reads: 0 2023-09-23T20:17:17.824 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-09-23T20:17:17.824 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.create_missing_column_families: 0 2023-09-23T20:17:17.824 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.db_log_dir: 2023-09-23T20:17:17.824 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.wal_dir: 2023-09-23T20:17:17.824 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.table_cache_numshardbits: 6 2023-09-23T20:17:17.824 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.WAL_ttl_seconds: 0 2023-09-23T20:17:17.825 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.WAL_size_limit_MB: 0 2023-09-23T20:17:17.825 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-09-23T20:17:17.825 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.manifest_preallocation_size: 4194304 2023-09-23T20:17:17.825 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.is_fd_close_on_exec: 1 2023-09-23T20:17:17.825 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.advise_random_on_open: 1 2023-09-23T20:17:17.825 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.db_write_buffer_size: 0 2023-09-23T20:17:17.825 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.write_buffer_manager: 0x56359a90ec80 2023-09-23T20:17:17.825 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.access_hint_on_compaction_start: 1 2023-09-23T20:17:17.825 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.random_access_max_buffer_size: 1048576 2023-09-23T20:17:17.825 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.use_adaptive_mutex: 0 2023-09-23T20:17:17.825 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.rate_limiter: (nil) 2023-09-23T20:17:17.825 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-09-23T20:17:17.826 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.wal_recovery_mode: 2 2023-09-23T20:17:17.826 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.enable_thread_tracking: 0 2023-09-23T20:17:17.826 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.enable_pipelined_write: 0 2023-09-23T20:17:17.826 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.unordered_write: 0 2023-09-23T20:17:17.826 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.allow_concurrent_memtable_write: 1 2023-09-23T20:17:17.826 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-09-23T20:17:17.826 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.write_thread_max_yield_usec: 100 2023-09-23T20:17:17.826 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.write_thread_slow_yield_usec: 3 2023-09-23T20:17:17.826 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.row_cache: None 2023-09-23T20:17:17.826 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.wal_filter: None 2023-09-23T20:17:17.827 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.avoid_flush_during_recovery: 0 2023-09-23T20:17:17.827 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.allow_ingest_behind: 0 2023-09-23T20:17:17.827 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.two_write_queues: 0 2023-09-23T20:17:17.827 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.manual_wal_flush: 0 2023-09-23T20:17:17.827 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.wal_compression: 0 2023-09-23T20:17:17.827 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.atomic_flush: 0 2023-09-23T20:17:17.827 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-09-23T20:17:17.827 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.persist_stats_to_disk: 0 2023-09-23T20:17:17.827 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.write_dbid_to_manifest: 0 2023-09-23T20:17:17.827 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.log_readahead_size: 0 2023-09-23T20:17:17.828 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.file_checksum_gen_factory: Unknown 2023-09-23T20:17:17.828 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.best_efforts_recovery: 0 2023-09-23T20:17:17.828 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-09-23T20:17:17.828 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-09-23T20:17:17.829 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.allow_data_in_errors: 0 2023-09-23T20:17:17.829 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.db_host_id: __hostname__ 2023-09-23T20:17:17.829 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.enforce_single_del_contracts: true 2023-09-23T20:17:17.829 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.max_background_jobs: 2 2023-09-23T20:17:17.829 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.max_background_compactions: -1 2023-09-23T20:17:17.829 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.max_subcompactions: 1 2023-09-23T20:17:17.829 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.avoid_flush_during_shutdown: 0 2023-09-23T20:17:17.830 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-09-23T20:17:17.830 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.delayed_write_rate : 16777216 2023-09-23T20:17:17.830 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.max_total_wal_size: 0 2023-09-23T20:17:17.830 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-09-23T20:17:17.830 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.stats_dump_period_sec: 600 2023-09-23T20:17:17.830 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.stats_persist_period_sec: 600 2023-09-23T20:17:17.830 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.stats_history_buffer_size: 1048576 2023-09-23T20:17:17.830 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.max_open_files: -1 2023-09-23T20:17:17.830 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.bytes_per_sync: 0 2023-09-23T20:17:17.830 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.wal_bytes_per_sync: 0 2023-09-23T20:17:17.830 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.strict_bytes_per_sync: 0 2023-09-23T20:17:17.831 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.compaction_readahead_size: 0 2023-09-23T20:17:17.831 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.max_background_flushes: -1 2023-09-23T20:17:17.831 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Compression algorithms supported: 2023-09-23T20:17:17.831 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: kZSTDNotFinalCompression supported: 0 2023-09-23T20:17:17.831 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: kZSTD supported: 0 2023-09-23T20:17:17.831 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: kXpressCompression supported: 0 2023-09-23T20:17:17.831 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: kLZ4HCCompression supported: 1 2023-09-23T20:17:17.831 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: kZlibCompression supported: 1 2023-09-23T20:17:17.831 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: kSnappyCompression supported: 1 2023-09-23T20:17:17.831 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: kLZ4Compression supported: 1 2023-09-23T20:17:17.831 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: kBZip2Compression supported: 0 2023-09-23T20:17:17.831 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Fast CRC32 supported: Supported on x86 2023-09-23T20:17:17.832 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: DMutex implementation: pthread_mutex_t 2023-09-23T20:17:17.832 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000009 2023-09-23T20:17:17.832 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2023-09-23T20:17:17.832 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-09-23T20:17:17.832 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.merge_operator: 2023-09-23T20:17:17.832 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.compaction_filter: None 2023-09-23T20:17:17.832 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.compaction_filter_factory: None 2023-09-23T20:17:17.832 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.sst_partitioner_factory: None 2023-09-23T20:17:17.832 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.memtable_factory: SkipListFactory 2023-09-23T20:17:17.832 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.table_factory: BlockBasedTable 2023-09-23T20:17:17.832 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x56359b67d520) 2023-09-23T20:17:17.832 INFO:journalctl@ceph.mon.c.smithi018.stdout: cache_index_and_filter_blocks: 1 2023-09-23T20:17:17.833 INFO:journalctl@ceph.mon.c.smithi018.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2023-09-23T20:17:17.833 INFO:journalctl@ceph.mon.c.smithi018.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2023-09-23T20:17:17.833 INFO:journalctl@ceph.mon.c.smithi018.stdout: pin_top_level_index_and_filter: 1 2023-09-23T20:17:17.833 INFO:journalctl@ceph.mon.c.smithi018.stdout: index_type: 0 2023-09-23T20:17:17.833 INFO:journalctl@ceph.mon.c.smithi018.stdout: data_block_index_type: 0 2023-09-23T20:17:17.833 INFO:journalctl@ceph.mon.c.smithi018.stdout: index_shortening: 1 2023-09-23T20:17:17.833 INFO:journalctl@ceph.mon.c.smithi018.stdout: data_block_hash_table_util_ratio: 0.750000 2023-09-23T20:17:17.833 INFO:journalctl@ceph.mon.c.smithi018.stdout: checksum: 4 2023-09-23T20:17:17.833 INFO:journalctl@ceph.mon.c.smithi018.stdout: no_block_cache: 0 2023-09-23T20:17:17.833 INFO:journalctl@ceph.mon.c.smithi018.stdout: block_cache: 0x56359a98f350 2023-09-23T20:17:17.833 INFO:journalctl@ceph.mon.c.smithi018.stdout: block_cache_name: BinnedLRUCache 2023-09-23T20:17:17.833 INFO:journalctl@ceph.mon.c.smithi018.stdout: block_cache_options: 2023-09-23T20:17:17.834 INFO:journalctl@ceph.mon.c.smithi018.stdout: capacity : 536870912 2023-09-23T20:17:17.834 INFO:journalctl@ceph.mon.c.smithi018.stdout: num_shard_bits : 4 2023-09-23T20:17:17.834 INFO:journalctl@ceph.mon.c.smithi018.stdout: strict_capacity_limit : 0 2023-09-23T20:17:17.834 INFO:journalctl@ceph.mon.c.smithi018.stdout: high_pri_pool_ratio: 0.000 2023-09-23T20:17:17.834 INFO:journalctl@ceph.mon.c.smithi018.stdout: block_cache_compressed: (nil) 2023-09-23T20:17:17.834 INFO:journalctl@ceph.mon.c.smithi018.stdout: persistent_cache: (nil) 2023-09-23T20:17:17.834 INFO:journalctl@ceph.mon.c.smithi018.stdout: block_size: 4096 2023-09-23T20:17:17.834 INFO:journalctl@ceph.mon.c.smithi018.stdout: block_size_deviation: 10 2023-09-23T20:17:17.834 INFO:journalctl@ceph.mon.c.smithi018.stdout: block_restart_interval: 16 2023-09-23T20:17:17.834 INFO:journalctl@ceph.mon.c.smithi018.stdout: index_block_restart_interval: 1 2023-09-23T20:17:17.834 INFO:journalctl@ceph.mon.c.smithi018.stdout: metadata_block_size: 4096 2023-09-23T20:17:17.834 INFO:journalctl@ceph.mon.c.smithi018.stdout: partition_filters: 0 2023-09-23T20:17:17.835 INFO:journalctl@ceph.mon.c.smithi018.stdout: use_delta_encoding: 1 2023-09-23T20:17:17.835 INFO:journalctl@ceph.mon.c.smithi018.stdout: filter_policy: bloomfilter 2023-09-23T20:17:17.835 INFO:journalctl@ceph.mon.c.smithi018.stdout: whole_key_filtering: 1 2023-09-23T20:17:17.835 INFO:journalctl@ceph.mon.c.smithi018.stdout: verify_compression: 0 2023-09-23T20:17:17.835 INFO:journalctl@ceph.mon.c.smithi018.stdout: read_amp_bytes_per_bit: 0 2023-09-23T20:17:17.835 INFO:journalctl@ceph.mon.c.smithi018.stdout: format_version: 5 2023-09-23T20:17:17.835 INFO:journalctl@ceph.mon.c.smithi018.stdout: enable_index_compression: 1 2023-09-23T20:17:17.835 INFO:journalctl@ceph.mon.c.smithi018.stdout: block_align: 0 2023-09-23T20:17:17.835 INFO:journalctl@ceph.mon.c.smithi018.stdout: max_auto_readahead_size: 262144 2023-09-23T20:17:17.835 INFO:journalctl@ceph.mon.c.smithi018.stdout: prepopulate_block_cache: 0 2023-09-23T20:17:17.835 INFO:journalctl@ceph.mon.c.smithi018.stdout: initial_auto_readahead_size: 8192 2023-09-23T20:17:17.835 INFO:journalctl@ceph.mon.c.smithi018.stdout: num_file_reads_for_auto_readahead: 2 2023-09-23T20:17:17.836 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.write_buffer_size: 33554432 2023-09-23T20:17:17.836 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.max_write_buffer_number: 2 2023-09-23T20:17:17.836 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.compression: NoCompression 2023-09-23T20:17:17.836 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.bottommost_compression: Disabled 2023-09-23T20:17:17.836 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.prefix_extractor: nullptr 2023-09-23T20:17:17.836 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-09-23T20:17:17.836 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.num_levels: 7 2023-09-23T20:17:17.836 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-09-23T20:17:17.836 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-09-23T20:17:17.836 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-09-23T20:17:17.836 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-09-23T20:17:17.836 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.bottommost_compression_opts.level: 32767 2023-09-23T20:17:17.837 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-09-23T20:17:17.837 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-09-23T20:17:17.837 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-09-23T20:17:17.837 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-09-23T20:17:17.837 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.bottommost_compression_opts.enabled: false 2023-09-23T20:17:17.837 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2023-09-23T20:17:17.837 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2023-09-23T20:17:17.837 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.compression_opts.window_bits: -14 2023-09-23T20:17:17.837 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.compression_opts.level: 32767 2023-09-23T20:17:17.837 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.compression_opts.strategy: 0 2023-09-23T20:17:17.837 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-09-23T20:17:17.837 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-09-23T20:17:17.838 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2023-09-23T20:17:17.838 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.compression_opts.parallel_threads: 1 2023-09-23T20:17:17.838 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.compression_opts.enabled: false 2023-09-23T20:17:17.838 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2023-09-23T20:17:17.838 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-09-23T20:17:17.838 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-09-23T20:17:17.838 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.level0_stop_writes_trigger: 36 2023-09-23T20:17:17.838 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.target_file_size_base: 67108864 2023-09-23T20:17:17.838 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.target_file_size_multiplier: 1 2023-09-23T20:17:17.838 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.max_bytes_for_level_base: 268435456 2023-09-23T20:17:17.838 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-09-23T20:17:17.839 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-09-23T20:17:17.839 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-09-23T20:17:17.839 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-09-23T20:17:17.839 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-09-23T20:17:17.839 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-09-23T20:17:17.839 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-09-23T20:17:17.839 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-09-23T20:17:17.839 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-09-23T20:17:17.839 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-09-23T20:17:17.839 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.max_compaction_bytes: 1677721600 2023-09-23T20:17:17.839 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2023-09-23T20:17:17.839 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.arena_block_size: 1048576 2023-09-23T20:17:17.840 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-09-23T20:17:17.840 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-09-23T20:17:17.840 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.disable_auto_compactions: 0 2023-09-23T20:17:17.840 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-09-23T20:17:17.840 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-09-23T20:17:17.840 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-09-23T20:17:17.840 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-09-23T20:17:17.840 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-09-23T20:17:17.840 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-09-23T20:17:17.840 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-09-23T20:17:17.840 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-09-23T20:17:17.840 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-09-23T20:17:17.840 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-09-23T20:17:17.841 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.table_properties_collectors: 2023-09-23T20:17:17.841 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.inplace_update_support: 0 2023-09-23T20:17:17.841 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.inplace_update_num_locks: 10000 2023-09-23T20:17:17.841 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-09-23T20:17:17.841 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.memtable_whole_key_filtering: 0 2023-09-23T20:17:17.841 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.memtable_huge_page_size: 0 2023-09-23T20:17:17.841 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.bloom_locality: 0 2023-09-23T20:17:17.841 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.max_successive_merges: 0 2023-09-23T20:17:17.841 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.optimize_filters_for_hits: 0 2023-09-23T20:17:17.841 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.paranoid_file_checks: 0 2023-09-23T20:17:17.841 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.force_consistency_checks: 1 2023-09-23T20:17:17.841 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.report_bg_io_stats: 0 2023-09-23T20:17:17.842 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.ttl: 2592000 2023-09-23T20:17:17.842 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.periodic_compaction_seconds: 0 2023-09-23T20:17:17.842 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.preclude_last_level_data_seconds: 0 2023-09-23T20:17:17.842 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.preserve_internal_time_seconds: 0 2023-09-23T20:17:17.842 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.enable_blob_files: false 2023-09-23T20:17:17.842 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.min_blob_size: 0 2023-09-23T20:17:17.842 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.blob_file_size: 268435456 2023-09-23T20:17:17.842 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.blob_compression_type: NoCompression 2023-09-23T20:17:17.842 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.enable_blob_garbage_collection: false 2023-09-23T20:17:17.842 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-09-23T20:17:17.842 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2023-09-23T20:17:17.842 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.blob_compaction_readahead_size: 0 2023-09-23T20:17:17.843 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.blob_file_starting_level: 0 2023-09-23T20:17:17.843 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2023-09-23T20:17:17.843 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 27.sst 2023-09-23T20:17:17.843 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-09-23T20:17:17.843 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 29, last_sequence is 13569, log_number is 25,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-09-23T20:17:17.843 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 25 2023-09-23T20:17:17.843 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 40cdb6c8-0a11-44b1-b6ed-8b6d9e54d197 2023-09-23T20:17:17.843 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695500237728462, "job": 1, "event": "recovery_started", "wal_files": [25]} 2023-09-23T20:17:17.843 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #25 mode 2 2023-09-23T20:17:17.843 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695500237730890, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 30, "file_size": 224083, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 13564, "largest_seqno": 13826, "table_properties": {"data_size": 222033, "index_size": 718, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 325, "raw_key_size": 2672, "raw_average_key_size": 24, "raw_value_size": 219510, "raw_average_value_size": 1995, "num_data_blocks": 30, "num_entries": 110, "num_filter_entries": 110, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1695500237, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "40cdb6c8-0a11-44b1-b6ed-8b6d9e54d197", "db_session_id": "J122X4W4NI017QS66CZI", "orig_file_number": 30, "seqno_to_time_mapping": "N/A"}} 2023-09-23T20:17:17.843 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695500237730997, "job": 1, "event": "recovery_finished"} 2023-09-23T20:17:17.843 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: [db/version_set.cc:5047] Creating manifest 32 2023-09-23T20:17:17.844 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-09-23T20:17:17.844 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000025.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-09-23T20:17:17.844 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x56359aa2c000 2023-09-23T20:17:17.844 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: DB pointer 0x56359aa18000 2023-09-23T20:17:17.844 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2023-09-23T20:17:17.844 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: rocksdb: [db/db_impl/db_impl.cc:1111] 2023-09-23T20:17:17.844 INFO:journalctl@ceph.mon.c.smithi018.stdout: ** DB Stats ** 2023-09-23T20:17:17.844 INFO:journalctl@ceph.mon.c.smithi018.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T20:17:17.844 INFO:journalctl@ceph.mon.c.smithi018.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-09-23T20:17:17.844 INFO:journalctl@ceph.mon.c.smithi018.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-09-23T20:17:17.844 INFO:journalctl@ceph.mon.c.smithi018.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T20:17:17.844 INFO:journalctl@ceph.mon.c.smithi018.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-09-23T20:17:17.845 INFO:journalctl@ceph.mon.c.smithi018.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-09-23T20:17:17.845 INFO:journalctl@ceph.mon.c.smithi018.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T20:17:17.845 INFO:journalctl@ceph.mon.c.smithi018.stdout: 2023-09-23T20:17:17.845 INFO:journalctl@ceph.mon.c.smithi018.stdout: ** Compaction Stats [default] ** 2023-09-23T20:17:17.845 INFO:journalctl@ceph.mon.c.smithi018.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-09-23T20:17:17.845 INFO:journalctl@ceph.mon.c.smithi018.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2023-09-23T20:17:17.845 INFO:journalctl@ceph.mon.c.smithi018.stdout: L0 1/0 218.83 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 162.9 0.00 0.00 1 0.001 0 0 0.0 0.0 2023-09-23T20:17:17.845 INFO:journalctl@ceph.mon.c.smithi018.stdout: L6 1/0 7.84 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2023-09-23T20:17:17.845 INFO:journalctl@ceph.mon.c.smithi018.stdout: Sum 2/0 8.05 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 162.9 0.00 0.00 1 0.001 0 0 0.0 0.0 2023-09-23T20:17:17.845 INFO:journalctl@ceph.mon.c.smithi018.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 162.9 0.00 0.00 1 0.001 0 0 0.0 0.0 2023-09-23T20:17:17.845 INFO:journalctl@ceph.mon.c.smithi018.stdout: 2023-09-23T20:17:17.846 INFO:journalctl@ceph.mon.c.smithi018.stdout: ** Compaction Stats [default] ** 2023-09-23T20:17:17.846 INFO:journalctl@ceph.mon.c.smithi018.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-09-23T20:17:17.846 INFO:journalctl@ceph.mon.c.smithi018.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T20:17:17.846 INFO:journalctl@ceph.mon.c.smithi018.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 162.9 0.00 0.00 1 0.001 0 0 0.0 0.0 2023-09-23T20:17:17.846 INFO:journalctl@ceph.mon.c.smithi018.stdout: 2023-09-23T20:17:17.846 INFO:journalctl@ceph.mon.c.smithi018.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2023-09-23T20:17:17.846 INFO:journalctl@ceph.mon.c.smithi018.stdout: 2023-09-23T20:17:17.846 INFO:journalctl@ceph.mon.c.smithi018.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T20:17:17.846 INFO:journalctl@ceph.mon.c.smithi018.stdout: Flush(GB): cumulative 0.000, interval 0.000 2023-09-23T20:17:17.846 INFO:journalctl@ceph.mon.c.smithi018.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-09-23T20:17:17.846 INFO:journalctl@ceph.mon.c.smithi018.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-09-23T20:17:17.846 INFO:journalctl@ceph.mon.c.smithi018.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-09-23T20:17:17.847 INFO:journalctl@ceph.mon.c.smithi018.stdout: AddFile(Keys): cumulative 0, interval 0 2023-09-23T20:17:17.847 INFO:journalctl@ceph.mon.c.smithi018.stdout: Cumulative compaction: 0.00 GB write, 37.77 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T20:17:17.847 INFO:journalctl@ceph.mon.c.smithi018.stdout: Interval compaction: 0.00 GB write, 37.77 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T20:17:17.847 INFO:journalctl@ceph.mon.c.smithi018.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-09-23T20:17:17.847 INFO:journalctl@ceph.mon.c.smithi018.stdout: Block cache BinnedLRUCache@0x56359a98f350#7 capacity: 512.00 MB usage: 1.20 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 9e-06 secs_since: 0 2023-09-23T20:17:17.847 INFO:journalctl@ceph.mon.c.smithi018.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,0.38 KB,7.15256e-05%) IndexBlock(1,0.83 KB,0.000157952%) Misc(1,0.00 KB,0%) 2023-09-23T20:17:17.847 INFO:journalctl@ceph.mon.c.smithi018.stdout: 2023-09-23T20:17:17.847 INFO:journalctl@ceph.mon.c.smithi018.stdout: ** File Read Latency Histogram By Level [default] ** 2023-09-23T20:17:17.847 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: starting mon.c rank 2 at public addrs [v2:172.21.15.18:3301/0,v1:172.21.15.18:6790/0] at bind addrs [v2:172.21.15.18:3301/0,v1:172.21.15.18:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 2023-09-23T20:17:17.847 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: mon.c@-1(???) e3 preinit fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 2023-09-23T20:17:17.847 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: mon.c@-1(???).mds e9 new map 2023-09-23T20:17:17.848 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: mon.c@-1(???).mds e9 print_map 2023-09-23T20:17:17.848 INFO:journalctl@ceph.mon.c.smithi018.stdout: e9 2023-09-23T20:17:17.848 INFO:journalctl@ceph.mon.c.smithi018.stdout: enable_multiple, ever_enabled_multiple: 1,1 2023-09-23T20:17:17.848 INFO:journalctl@ceph.mon.c.smithi018.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-09-23T20:17:17.848 INFO:journalctl@ceph.mon.c.smithi018.stdout: legacy client fscid: 1 2023-09-23T20:17:17.848 INFO:journalctl@ceph.mon.c.smithi018.stdout: 2023-09-23T20:17:17.848 INFO:journalctl@ceph.mon.c.smithi018.stdout: Filesystem 'foo' (1) 2023-09-23T20:17:17.848 INFO:journalctl@ceph.mon.c.smithi018.stdout: fs_name foo 2023-09-23T20:17:17.848 INFO:journalctl@ceph.mon.c.smithi018.stdout: epoch 8 2023-09-23T20:17:17.848 INFO:journalctl@ceph.mon.c.smithi018.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2023-09-23T20:17:17.848 INFO:journalctl@ceph.mon.c.smithi018.stdout: created 2023-09-23T20:12:43.745113+0000 2023-09-23T20:17:17.848 INFO:journalctl@ceph.mon.c.smithi018.stdout: modified 2023-09-23T20:12:54.848699+0000 2023-09-23T20:17:17.849 INFO:journalctl@ceph.mon.c.smithi018.stdout: tableserver 0 2023-09-23T20:17:17.849 INFO:journalctl@ceph.mon.c.smithi018.stdout: root 0 2023-09-23T20:17:17.849 INFO:journalctl@ceph.mon.c.smithi018.stdout: session_timeout 60 2023-09-23T20:17:17.849 INFO:journalctl@ceph.mon.c.smithi018.stdout: session_autoclose 300 2023-09-23T20:17:17.849 INFO:journalctl@ceph.mon.c.smithi018.stdout: max_file_size 1099511627776 2023-09-23T20:17:17.849 INFO:journalctl@ceph.mon.c.smithi018.stdout: required_client_features {} 2023-09-23T20:17:17.849 INFO:journalctl@ceph.mon.c.smithi018.stdout: last_failure 0 2023-09-23T20:17:17.849 INFO:journalctl@ceph.mon.c.smithi018.stdout: last_failure_osd_epoch 0 2023-09-23T20:17:17.849 INFO:journalctl@ceph.mon.c.smithi018.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-09-23T20:17:17.849 INFO:journalctl@ceph.mon.c.smithi018.stdout: max_mds 1 2023-09-23T20:17:17.849 INFO:journalctl@ceph.mon.c.smithi018.stdout: in 0 2023-09-23T20:17:17.849 INFO:journalctl@ceph.mon.c.smithi018.stdout: up {0=24566} 2023-09-23T20:17:17.849 INFO:journalctl@ceph.mon.c.smithi018.stdout: failed 2023-09-23T20:17:17.850 INFO:journalctl@ceph.mon.c.smithi018.stdout: damaged 2023-09-23T20:17:17.850 INFO:journalctl@ceph.mon.c.smithi018.stdout: stopped 2023-09-23T20:17:17.850 INFO:journalctl@ceph.mon.c.smithi018.stdout: data_pools [4] 2023-09-23T20:17:17.850 INFO:journalctl@ceph.mon.c.smithi018.stdout: metadata_pool 3 2023-09-23T20:17:17.850 INFO:journalctl@ceph.mon.c.smithi018.stdout: inline_data disabled 2023-09-23T20:17:17.850 INFO:journalctl@ceph.mon.c.smithi018.stdout: balancer 2023-09-23T20:17:17.850 INFO:journalctl@ceph.mon.c.smithi018.stdout: bal_rank_mask -1 2023-09-23T20:17:17.850 INFO:journalctl@ceph.mon.c.smithi018.stdout: standby_count_wanted 1 2023-09-23T20:17:17.850 INFO:journalctl@ceph.mon.c.smithi018.stdout: [mds.foo.smithi018.hduiol{0:24566} state up:active seq 3 join_fscid=1 addr [v2:172.21.15.18:6834/3661910111,v1:172.21.15.18:6835/3661910111] compat {c=[1],r=[1],i=[7ff]}] 2023-09-23T20:17:17.850 INFO:journalctl@ceph.mon.c.smithi018.stdout: 2023-09-23T20:17:17.850 INFO:journalctl@ceph.mon.c.smithi018.stdout: 2023-09-23T20:17:17.850 INFO:journalctl@ceph.mon.c.smithi018.stdout: Standby daemons: 2023-09-23T20:17:17.851 INFO:journalctl@ceph.mon.c.smithi018.stdout: 2023-09-23T20:17:17.851 INFO:journalctl@ceph.mon.c.smithi018.stdout: [mds.foo.smithi149.urtezf{-1:24559} state up:standby seq 2 join_fscid=1 addr [v2:172.21.15.149:6832/3529449999,v1:172.21.15.149:6833/3529449999] compat {c=[1],r=[1],i=[7ff]}] 2023-09-23T20:17:17.851 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: mon.c@-1(???).osd e90 crush map has features 3314933000854323200, adjusting msgr requires 2023-09-23T20:17:17.851 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: mon.c@-1(???).osd e90 crush map has features 432629239337189376, adjusting msgr requires 2023-09-23T20:17:17.851 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: mon.c@-1(???).osd e90 crush map has features 432629239337189376, adjusting msgr requires 2023-09-23T20:17:17.851 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: mon.c@-1(???).osd e90 crush map has features 432629239337189376, adjusting msgr requires 2023-09-23T20:17:17.851 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: mon.c@-1(???).paxosservice(auth 1..19) refresh upgraded, format 0 -> 3 2023-09-23T20:17:17.851 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: expand_channel_meta expand map: {default=false} 2023-09-23T20:17:17.851 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: expand_channel_meta from 'false' to 'false' 2023-09-23T20:17:17.851 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: expand_channel_meta expanded map: {default=false} 2023-09-23T20:17:17.851 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: expand_channel_meta expand map: {default=info} 2023-09-23T20:17:17.851 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: expand_channel_meta from 'info' to 'info' 2023-09-23T20:17:17.852 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: expand_channel_meta expanded map: {default=info} 2023-09-23T20:17:17.852 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: expand_channel_meta expand map: {default=daemon} 2023-09-23T20:17:17.852 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: expand_channel_meta from 'daemon' to 'daemon' 2023-09-23T20:17:17.852 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: expand_channel_meta expanded map: {default=daemon} 2023-09-23T20:17:17.852 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: expand_channel_meta expand map: {default=debug} 2023-09-23T20:17:17.852 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: expand_channel_meta from 'debug' to 'debug' 2023-09-23T20:17:17.852 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 ceph-mon[184144]: expand_channel_meta expanded map: {default=debug} 2023-09-23T20:17:17.852 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:17 smithi018 systemd[1]: Started Ceph mon.c for a3b3b17e-5a4c-11ee-8db4-212e2dc638e7. 2023-09-23T20:17:18.966 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:17:18 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[156506]: ts=2023-09-23T20:17:18.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.18\", device_class=\"ssd\", hostname=\"smithi018\", instance=\"smithi149.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.18\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.18\", device_class=\"ssd\", hostname=\"smithi018\", instance=\"smithi018:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.18\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T20:17:23.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:17:22 smithi149 ceph-mon[100764]: pgmap v41: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:17:23.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:17:22 smithi149 ceph-mon[100764]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T20:17:23.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:17:22 smithi149 ceph-mon[100764]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:17:23.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:17:22 smithi149 ceph-mon[100764]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:17:23.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:17:22 smithi018 ceph-mon[179647]: pgmap v41: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:17:23.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:17:22 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T20:17:23.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:17:22 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:17:23.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:17:22 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:17:24.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:17:23 smithi149 ceph-mon[100764]: pgmap v42: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:17:24.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:17:23 smithi149 ceph-mon[100764]: mon.b calling monitor election 2023-09-23T20:17:24.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:17:23 smithi149 ceph-mon[100764]: mon.a calling monitor election 2023-09-23T20:17:24.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:17:23 smithi149 ceph-mon[100764]: from='client.? 172.21.15.18:0/2751831046' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:17:24.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:17:23 smithi149 ceph-mon[100764]: pgmap v43: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:17:24.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:17:23 smithi149 ceph-mon[100764]: pgmap v44: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:17:24.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:17:23 smithi149 ceph-mon[100764]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-09-23T20:17:24.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:17:23 smithi149 ceph-mon[100764]: monmap e3: 3 mons at {a=[v2:172.21.15.18:3300/0,v1:172.21.15.18:6789/0],b=[v2:172.21.15.149:3300/0,v1:172.21.15.149:6789/0],c=[v2:172.21.15.18:3301/0,v1:172.21.15.18:6790/0]} removed_ranks: {} 2023-09-23T20:17:24.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:17:23 smithi149 ceph-mon[100764]: fsmap foo:1 {0=foo.smithi018.hduiol=up:active} 1 up:standby 2023-09-23T20:17:24.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:17:23 smithi149 ceph-mon[100764]: osdmap e90: 8 total, 8 up, 8 in 2023-09-23T20:17:24.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:17:23 smithi149 ceph-mon[100764]: mgrmap e35: y(active, since 83s), standbys: x 2023-09-23T20:17:24.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:17:23 smithi149 ceph-mon[100764]: Health check failed: 1/3 mons down, quorum a,b (MON_DOWN) 2023-09-23T20:17:24.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:17:23 smithi149 ceph-mon[100764]: Health detail: HEALTH_WARN 1/3 mons down, quorum a,b 2023-09-23T20:17:24.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:17:23 smithi149 ceph-mon[100764]: [WRN] MON_DOWN: 1/3 mons down, quorum a,b 2023-09-23T20:17:24.218 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:17:23 smithi149 ceph-mon[100764]: mon.c (rank 2) addr [v2:172.21.15.18:3301/0,v1:172.21.15.18:6790/0] is down (out of quorum) 2023-09-23T20:17:24.218 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:17:23 smithi149 ceph-mon[100764]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:17:24.218 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:17:23 smithi149 ceph-mon[100764]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:17:24.218 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:17:23 smithi149 ceph-mon[100764]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:17:24.218 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:17:23 smithi149 ceph-mon[100764]: from='client.? 172.21.15.18:0/3068878422' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:17:24.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:17:23 smithi018 ceph-mon[179647]: pgmap v42: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:17:24.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:17:23 smithi018 ceph-mon[179647]: mon.b calling monitor election 2023-09-23T20:17:24.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:17:23 smithi018 ceph-mon[179647]: mon.a calling monitor election 2023-09-23T20:17:24.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:17:23 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2751831046' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:17:24.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:17:23 smithi018 ceph-mon[179647]: pgmap v43: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:17:24.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:17:23 smithi018 ceph-mon[179647]: pgmap v44: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:17:24.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:17:23 smithi018 ceph-mon[179647]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-09-23T20:17:24.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:17:23 smithi018 ceph-mon[179647]: monmap e3: 3 mons at {a=[v2:172.21.15.18:3300/0,v1:172.21.15.18:6789/0],b=[v2:172.21.15.149:3300/0,v1:172.21.15.149:6789/0],c=[v2:172.21.15.18:3301/0,v1:172.21.15.18:6790/0]} removed_ranks: {} 2023-09-23T20:17:24.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:17:23 smithi018 ceph-mon[179647]: fsmap foo:1 {0=foo.smithi018.hduiol=up:active} 1 up:standby 2023-09-23T20:17:24.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:17:23 smithi018 ceph-mon[179647]: osdmap e90: 8 total, 8 up, 8 in 2023-09-23T20:17:24.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:17:23 smithi018 ceph-mon[179647]: mgrmap e35: y(active, since 83s), standbys: x 2023-09-23T20:17:24.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:17:23 smithi018 ceph-mon[179647]: Health check failed: 1/3 mons down, quorum a,b (MON_DOWN) 2023-09-23T20:17:24.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:17:23 smithi018 ceph-mon[179647]: Health detail: HEALTH_WARN 1/3 mons down, quorum a,b 2023-09-23T20:17:24.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:17:23 smithi018 ceph-mon[179647]: [WRN] MON_DOWN: 1/3 mons down, quorum a,b 2023-09-23T20:17:24.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:17:23 smithi018 ceph-mon[179647]: mon.c (rank 2) addr [v2:172.21.15.18:3301/0,v1:172.21.15.18:6790/0] is down (out of quorum) 2023-09-23T20:17:24.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:17:23 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:17:24.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:17:23 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:17:24.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:17:23 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:17:24.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:17:23 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3068878422' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:17:24.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:23 smithi018 ceph-mon[184144]: pgmap v41: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:17:24.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:23 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T20:17:24.394 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:23 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:17:24.394 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:23 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:17:24.394 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:23 smithi018 ceph-mon[184144]: pgmap v42: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:17:24.394 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:23 smithi018 ceph-mon[184144]: mon.b calling monitor election 2023-09-23T20:17:24.394 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:23 smithi018 ceph-mon[184144]: mon.a calling monitor election 2023-09-23T20:17:24.394 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:23 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2751831046' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:17:24.394 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:23 smithi018 ceph-mon[184144]: pgmap v43: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:17:24.394 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:23 smithi018 ceph-mon[184144]: pgmap v44: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:17:24.394 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:23 smithi018 ceph-mon[184144]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-09-23T20:17:24.395 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:23 smithi018 ceph-mon[184144]: monmap e3: 3 mons at {a=[v2:172.21.15.18:3300/0,v1:172.21.15.18:6789/0],b=[v2:172.21.15.149:3300/0,v1:172.21.15.149:6789/0],c=[v2:172.21.15.18:3301/0,v1:172.21.15.18:6790/0]} removed_ranks: {} 2023-09-23T20:17:24.395 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:23 smithi018 ceph-mon[184144]: fsmap foo:1 {0=foo.smithi018.hduiol=up:active} 1 up:standby 2023-09-23T20:17:24.395 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:23 smithi018 ceph-mon[184144]: osdmap e90: 8 total, 8 up, 8 in 2023-09-23T20:17:24.395 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:23 smithi018 ceph-mon[184144]: mgrmap e35: y(active, since 83s), standbys: x 2023-09-23T20:17:24.395 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:23 smithi018 ceph-mon[184144]: Health check failed: 1/3 mons down, quorum a,b (MON_DOWN) 2023-09-23T20:17:24.395 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:23 smithi018 ceph-mon[184144]: Health detail: HEALTH_WARN 1/3 mons down, quorum a,b 2023-09-23T20:17:24.395 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:23 smithi018 ceph-mon[184144]: [WRN] MON_DOWN: 1/3 mons down, quorum a,b 2023-09-23T20:17:24.395 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:23 smithi018 ceph-mon[184144]: mon.c (rank 2) addr [v2:172.21.15.18:3301/0,v1:172.21.15.18:6790/0] is down (out of quorum) 2023-09-23T20:17:24.395 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:23 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:17:24.395 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:23 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:17:24.395 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:23 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:17:24.395 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:23 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3068878422' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:17:25.391 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:17:24 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:17:24] "GET /metrics HTTP/1.1" 200 33909 "" "Prometheus/2.43.0" 2023-09-23T20:17:25.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:24 smithi018 ceph-mon[184144]: mon.c calling monitor election 2023-09-23T20:17:25.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:24 smithi018 ceph-mon[184144]: pgmap v45: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:17:25.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:24 smithi018 ceph-mon[184144]: mon.b calling monitor election 2023-09-23T20:17:25.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:24 smithi018 ceph-mon[184144]: mon.a calling monitor election 2023-09-23T20:17:25.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:24 smithi018 ceph-mon[184144]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T20:17:25.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:24 smithi018 ceph-mon[184144]: monmap e3: 3 mons at {a=[v2:172.21.15.18:3300/0,v1:172.21.15.18:6789/0],b=[v2:172.21.15.149:3300/0,v1:172.21.15.149:6789/0],c=[v2:172.21.15.18:3301/0,v1:172.21.15.18:6790/0]} removed_ranks: {} 2023-09-23T20:17:25.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:24 smithi018 ceph-mon[184144]: fsmap foo:1 {0=foo.smithi018.hduiol=up:active} 1 up:standby 2023-09-23T20:17:25.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:24 smithi018 ceph-mon[184144]: osdmap e90: 8 total, 8 up, 8 in 2023-09-23T20:17:25.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:24 smithi018 ceph-mon[184144]: mgrmap e35: y(active, since 84s), standbys: x 2023-09-23T20:17:25.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:24 smithi018 ceph-mon[184144]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2023-09-23T20:17:25.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:24 smithi018 ceph-mon[184144]: Cluster is now healthy 2023-09-23T20:17:25.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:24 smithi018 ceph-mon[184144]: overall HEALTH_OK 2023-09-23T20:17:25.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:17:24 smithi018 ceph-mon[179647]: mon.c calling monitor election 2023-09-23T20:17:25.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:17:24 smithi018 ceph-mon[179647]: pgmap v45: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:17:25.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:17:24 smithi018 ceph-mon[179647]: mon.b calling monitor election 2023-09-23T20:17:25.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:17:24 smithi018 ceph-mon[179647]: mon.a calling monitor election 2023-09-23T20:17:25.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:17:24 smithi018 ceph-mon[179647]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T20:17:25.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:17:24 smithi018 ceph-mon[179647]: monmap e3: 3 mons at {a=[v2:172.21.15.18:3300/0,v1:172.21.15.18:6789/0],b=[v2:172.21.15.149:3300/0,v1:172.21.15.149:6789/0],c=[v2:172.21.15.18:3301/0,v1:172.21.15.18:6790/0]} removed_ranks: {} 2023-09-23T20:17:25.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:17:24 smithi018 ceph-mon[179647]: fsmap foo:1 {0=foo.smithi018.hduiol=up:active} 1 up:standby 2023-09-23T20:17:25.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:17:24 smithi018 ceph-mon[179647]: osdmap e90: 8 total, 8 up, 8 in 2023-09-23T20:17:25.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:17:24 smithi018 ceph-mon[179647]: mgrmap e35: y(active, since 84s), standbys: x 2023-09-23T20:17:25.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:17:24 smithi018 ceph-mon[179647]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2023-09-23T20:17:25.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:17:24 smithi018 ceph-mon[179647]: Cluster is now healthy 2023-09-23T20:17:25.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:17:24 smithi018 ceph-mon[179647]: overall HEALTH_OK 2023-09-23T20:17:25.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:17:24 smithi149 ceph-mon[100764]: mon.c calling monitor election 2023-09-23T20:17:25.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:17:24 smithi149 ceph-mon[100764]: pgmap v45: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:17:25.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:17:24 smithi149 ceph-mon[100764]: mon.b calling monitor election 2023-09-23T20:17:25.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:17:24 smithi149 ceph-mon[100764]: mon.a calling monitor election 2023-09-23T20:17:25.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:17:24 smithi149 ceph-mon[100764]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T20:17:25.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:17:24 smithi149 ceph-mon[100764]: monmap e3: 3 mons at {a=[v2:172.21.15.18:3300/0,v1:172.21.15.18:6789/0],b=[v2:172.21.15.149:3300/0,v1:172.21.15.149:6789/0],c=[v2:172.21.15.18:3301/0,v1:172.21.15.18:6790/0]} removed_ranks: {} 2023-09-23T20:17:25.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:17:24 smithi149 ceph-mon[100764]: fsmap foo:1 {0=foo.smithi018.hduiol=up:active} 1 up:standby 2023-09-23T20:17:25.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:17:24 smithi149 ceph-mon[100764]: osdmap e90: 8 total, 8 up, 8 in 2023-09-23T20:17:25.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:17:24 smithi149 ceph-mon[100764]: mgrmap e35: y(active, since 84s), standbys: x 2023-09-23T20:17:25.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:17:24 smithi149 ceph-mon[100764]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2023-09-23T20:17:25.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:17:24 smithi149 ceph-mon[100764]: Cluster is now healthy 2023-09-23T20:17:25.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:17:24 smithi149 ceph-mon[100764]: overall HEALTH_OK 2023-09-23T20:17:25.855 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable)": 2 2023-09-23T20:17:26.245 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch ps' 2023-09-23T20:17:26.865 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:17:27.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:26 smithi018 ceph-mon[184144]: pgmap v46: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:17:27.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:26 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3226564595' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:17:27.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:17:26 smithi018 ceph-mon[179647]: pgmap v46: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:17:27.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:17:26 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3226564595' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:17:27.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:17:26 smithi149 ceph-mon[100764]: pgmap v46: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:17:27.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:17:26 smithi149 ceph-mon[100764]: from='client.? 172.21.15.18:0/3226564595' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:17:27.891 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:17:27 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: 2023-09-23T20:17:27.745+0000 7f953be6a700 -1 mgr.server handle_report got status from non-daemon mon.c 2023-09-23T20:17:28.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:28 smithi018 ceph-mon[184144]: pgmap v47: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:17:28.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:28 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:17:28.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:28 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:17:28.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:17:28 smithi018 ceph-mon[179647]: pgmap v47: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:17:28.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:17:28 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:17:28.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:17:28 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:17:28.700 INFO:teuthology.orchestra.run.smithi018.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-09-23T20:17:28.700 INFO:teuthology.orchestra.run.smithi018.stdout:alertmanager.a smithi018 *:9093,9094 running (2m) - 5m 28.7M - 0.25.0 c8568f914cd2 57241c9344be 2023-09-23T20:17:28.700 INFO:teuthology.orchestra.run.smithi018.stdout:grafana.a smithi149 *:3000 running (2m) - 5m 84.2M - 9.4.7 2c41d148cca3 62150314e898 2023-09-23T20:17:28.700 INFO:teuthology.orchestra.run.smithi018.stdout:mds.foo.smithi018.hduiol smithi018 running (4m) - 4m 17.8M - 17.2.6-1123-g61431f6b 0db14aa01094 7b6a15d633a4 2023-09-23T20:17:28.700 INFO:teuthology.orchestra.run.smithi018.stdout:mds.foo.smithi149.urtezf smithi149 running (4m) - 4m 17.5M - 17.2.6-1123-g61431f6b 0db14aa01094 6ee4df523f8c 2023-09-23T20:17:28.700 INFO:teuthology.orchestra.run.smithi018.stdout:mgr.x smithi149 *:8443,9283,8765 running (111s) - 8m 448M - 18.2.0-460-gbb0daeff e32c2a64c126 9513b5c2a2f7 2023-09-23T20:17:28.700 INFO:teuthology.orchestra.run.smithi018.stdout:mgr.y smithi018 *:8443,9283,8765 running (2m) - 10m 502M - 18.2.0-460-gbb0daeff e32c2a64c126 5aa74f13c0a1 2023-09-23T20:17:28.700 INFO:teuthology.orchestra.run.smithi018.stdout:mon.a smithi018 running (98s) - 10m 56.0M 2048M 18.2.0-460-gbb0daeff e32c2a64c126 425294bbcbd1 2023-09-23T20:17:28.700 INFO:teuthology.orchestra.run.smithi018.stdout:mon.b smithi149 running (9m) - 9m 70.9M 2048M 17.2.6-1123-g61431f6b 0db14aa01094 e9fa4ded7429 2023-09-23T20:17:28.701 INFO:teuthology.orchestra.run.smithi018.stdout:mon.c smithi018 running (11s) - 9m 17.9M 2048M 18.2.0-460-gbb0daeff e32c2a64c126 e2701e7c5d35 2023-09-23T20:17:28.701 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter.a smithi018 *:9100 running (2m) - 5m 17.4M - 1.5.0 0da6a335fe13 500f76739e6a 2023-09-23T20:17:28.701 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter.b smithi149 *:9100 running (2m) - 5m 17.5M - 1.5.0 0da6a335fe13 f1789a46736b 2023-09-23T20:17:28.701 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0 smithi018 running (8m) - 8m 78.9M 2501M 17.2.6-1123-g61431f6b 0db14aa01094 6749d0af6e50 2023-09-23T20:17:28.701 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1 smithi018 running (8m) - 8m 97.1M 2501M 17.2.6-1123-g61431f6b 0db14aa01094 5d199baeab0e 2023-09-23T20:17:28.701 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2 smithi018 running (7m) - 7m 79.4M 2501M 17.2.6-1123-g61431f6b 0db14aa01094 2effc63f4610 2023-09-23T20:17:28.701 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3 smithi018 running (7m) - 7m 66.8M 2501M 17.2.6-1123-g61431f6b 0db14aa01094 3301c3e11bd5 2023-09-23T20:17:28.701 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4 smithi149 running (7m) - 7m 75.1M 4096M 17.2.6-1123-g61431f6b 0db14aa01094 b4b39c43de75 2023-09-23T20:17:28.701 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5 smithi149 running (6m) - 6m 87.7M 4096M 17.2.6-1123-g61431f6b 0db14aa01094 cec31ff38f4c 2023-09-23T20:17:28.701 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6 smithi149 running (6m) - 6m 78.4M 4096M 17.2.6-1123-g61431f6b 0db14aa01094 ce687413759d 2023-09-23T20:17:28.701 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7 smithi149 running (6m) - 6m 91.2M 4096M 17.2.6-1123-g61431f6b 0db14aa01094 d3145e28f8c3 2023-09-23T20:17:28.701 INFO:teuthology.orchestra.run.smithi018.stdout:prometheus.a smithi149 *:9095 running (115s) - 5m 50.0M - 2.43.0 a07b618ecd1d bb2162e75dc7 2023-09-23T20:17:28.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:17:28 smithi149 ceph-mon[100764]: pgmap v47: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:17:28.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:17:28 smithi149 ceph-mon[100764]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:17:28.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:17:28 smithi149 ceph-mon[100764]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:17:28.716 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:17:28 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[156506]: ts=2023-09-23T20:17:28.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.18\", device_class=\"ssd\", hostname=\"smithi018\", instance=\"smithi149.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.18\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.18\", device_class=\"ssd\", hostname=\"smithi018\", instance=\"smithi018:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.18\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T20:17:29.291 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch upgrade pause' 2023-09-23T20:17:29.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:17:29 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:17:29.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:17:29 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:17:29.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:17:29 smithi018 ceph-mon[179647]: from='client.34200 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:17:29.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:17:29 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:17:29.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:29 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:17:29.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:29 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:17:29.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:29 smithi018 ceph-mon[184144]: from='client.34200 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:17:29.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:29 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:17:29.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:17:29 smithi149 ceph-mon[100764]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:17:29.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:17:29 smithi149 ceph-mon[100764]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:17:29.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:17:29 smithi149 ceph-mon[100764]: from='client.34200 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:17:29.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:17:29 smithi149 ceph-mon[100764]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:17:30.152 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:17:30.631 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:30 smithi018 ceph-mon[184144]: pgmap v48: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:17:30.631 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:17:30 smithi018 ceph-mon[179647]: pgmap v48: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:17:30.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:17:30 smithi149 ceph-mon[100764]: pgmap v48: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:17:31.630 INFO:teuthology.orchestra.run.smithi018.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T20:17:31.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:31 smithi018 ceph-mon[184144]: mgrmap e36: y(active, since 91s), standbys: x 2023-09-23T20:17:31.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:17:31 smithi018 ceph-mon[179647]: mgrmap e36: y(active, since 91s), standbys: x 2023-09-23T20:17:31.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:17:31 smithi149 ceph-mon[100764]: mgrmap e36: y(active, since 91s), standbys: x 2023-09-23T20:17:32.204 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'sleep 60' 2023-09-23T20:17:32.823 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:17:32.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:32 smithi018 ceph-mon[184144]: pgmap v49: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T20:17:32.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:32 smithi018 ceph-mon[184144]: from='client.44108 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:17:32.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:32 smithi018 ceph-mon[184144]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T20:17:32.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:32 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:17:32.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:32 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:17:32.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:32 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:17:32.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:32 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:17:32.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:32 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:17:32.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:32 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:17:32.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:32 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:17:32.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:32 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:17:32.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:32 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:17:32.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:32 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:17:32.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:17:32 smithi018 ceph-mon[179647]: pgmap v49: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T20:17:32.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:17:32 smithi018 ceph-mon[179647]: from='client.44108 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:17:32.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:17:32 smithi018 ceph-mon[179647]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T20:17:32.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:17:32 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:17:32.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:17:32 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:17:32.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:17:32 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:17:32.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:17:32 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:17:32.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:17:32 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:17:32.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:17:32 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:17:32.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:17:32 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:17:32.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:17:32 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:17:32.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:17:32 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:17:32.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:17:32 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:17:32.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:17:32 smithi149 ceph-mon[100764]: pgmap v49: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T20:17:32.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:17:32 smithi149 ceph-mon[100764]: from='client.44108 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:17:32.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:17:32 smithi149 ceph-mon[100764]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T20:17:32.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:17:32 smithi149 ceph-mon[100764]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:17:32.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:17:32 smithi149 ceph-mon[100764]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:17:32.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:17:32 smithi149 ceph-mon[100764]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:17:32.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:17:32 smithi149 ceph-mon[100764]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:17:32.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:17:32 smithi149 ceph-mon[100764]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:17:32.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:17:32 smithi149 ceph-mon[100764]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:17:32.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:17:32 smithi149 ceph-mon[100764]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:17:32.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:17:32 smithi149 ceph-mon[100764]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:17:32.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:17:32 smithi149 ceph-mon[100764]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:17:32.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:17:32 smithi149 ceph-mon[100764]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:17:34.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:17:34 smithi018 ceph-mon[179647]: pgmap v50: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T20:17:34.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:34 smithi018 ceph-mon[184144]: pgmap v50: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T20:17:34.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:17:34 smithi149 ceph-mon[100764]: pgmap v50: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T20:17:35.393 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:17:34 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:17:34] "GET /metrics HTTP/1.1" 200 33898 "" "Prometheus/2.43.0" 2023-09-23T20:17:37.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:36 smithi018 ceph-mon[184144]: pgmap v51: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T20:17:37.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:17:36 smithi018 ceph-mon[179647]: pgmap v51: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T20:17:37.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:17:36 smithi149 ceph-mon[100764]: pgmap v51: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T20:17:38.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:17:38 smithi149 ceph-mon[100764]: pgmap v52: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T20:17:38.966 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:17:38 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[156506]: ts=2023-09-23T20:17:38.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.18\", device_class=\"ssd\", hostname=\"smithi018\", instance=\"smithi149.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.18\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.18\", device_class=\"ssd\", hostname=\"smithi018\", instance=\"smithi018:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.18\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T20:17:39.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:38 smithi018 ceph-mon[184144]: pgmap v52: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T20:17:39.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:17:38 smithi018 ceph-mon[179647]: pgmap v52: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T20:17:41.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:40 smithi018 ceph-mon[184144]: pgmap v53: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T20:17:41.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:17:40 smithi018 ceph-mon[179647]: pgmap v53: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T20:17:41.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:17:40 smithi149 ceph-mon[100764]: pgmap v53: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T20:17:43.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:42 smithi018 ceph-mon[184144]: pgmap v54: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T20:17:43.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:17:42 smithi018 ceph-mon[179647]: pgmap v54: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T20:17:43.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:17:42 smithi149 ceph-mon[100764]: pgmap v54: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T20:17:45.141 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:17:44 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:17:44] "GET /metrics HTTP/1.1" 200 33902 "" "Prometheus/2.43.0" 2023-09-23T20:17:45.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:44 smithi018 ceph-mon[184144]: pgmap v55: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:17:45.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:44 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:17:45.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:17:44 smithi018 ceph-mon[179647]: pgmap v55: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:17:45.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:17:44 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:17:45.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:17:44 smithi149 ceph-mon[100764]: pgmap v55: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:17:45.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:17:44 smithi149 ceph-mon[100764]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:17:47.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:46 smithi018 ceph-mon[184144]: pgmap v56: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:17:47.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:17:46 smithi018 ceph-mon[179647]: pgmap v56: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:17:47.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:17:46 smithi149 ceph-mon[100764]: pgmap v56: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:17:48.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:17:48 smithi149 ceph-mon[100764]: pgmap v57: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:17:48.966 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:17:48 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[156506]: ts=2023-09-23T20:17:48.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.18\", device_class=\"ssd\", hostname=\"smithi018\", instance=\"smithi149.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.18\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.18\", device_class=\"ssd\", hostname=\"smithi018\", instance=\"smithi018:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.18\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T20:17:49.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:48 smithi018 ceph-mon[184144]: pgmap v57: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:17:49.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:17:48 smithi018 ceph-mon[179647]: pgmap v57: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:17:51.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:50 smithi018 ceph-mon[184144]: pgmap v58: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:17:51.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:17:50 smithi018 ceph-mon[179647]: pgmap v58: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:17:51.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:17:50 smithi149 ceph-mon[100764]: pgmap v58: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:17:53.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:52 smithi018 ceph-mon[184144]: pgmap v59: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:17:53.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:17:52 smithi018 ceph-mon[179647]: pgmap v59: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:17:53.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:17:52 smithi149 ceph-mon[100764]: pgmap v59: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:17:55.141 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:17:54 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:17:54] "GET /metrics HTTP/1.1" 200 33902 "" "Prometheus/2.43.0" 2023-09-23T20:17:55.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:54 smithi018 ceph-mon[184144]: pgmap v60: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:17:55.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:17:54 smithi018 ceph-mon[179647]: pgmap v60: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:17:55.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:17:54 smithi149 ceph-mon[100764]: pgmap v60: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:17:57.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:56 smithi018 ceph-mon[184144]: pgmap v61: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:17:57.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:17:56 smithi018 ceph-mon[179647]: pgmap v61: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:17:57.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:17:56 smithi149 ceph-mon[100764]: pgmap v61: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:17:58.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:17:58 smithi149 ceph-mon[100764]: pgmap v62: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:17:58.966 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:17:58 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[156506]: ts=2023-09-23T20:17:58.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.18\", device_class=\"ssd\", hostname=\"smithi018\", instance=\"smithi149.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.18\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.18\", device_class=\"ssd\", hostname=\"smithi018\", instance=\"smithi018:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.18\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T20:17:59.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:58 smithi018 ceph-mon[184144]: pgmap v62: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:17:59.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:17:58 smithi018 ceph-mon[179647]: pgmap v62: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:18:00.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:17:59 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:18:00.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:17:59 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:18:00.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:17:59 smithi149 ceph-mon[100764]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:18:01.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:18:00 smithi018 ceph-mon[184144]: pgmap v63: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:18:01.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:18:00 smithi018 ceph-mon[179647]: pgmap v63: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:18:01.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:00 smithi149 ceph-mon[100764]: pgmap v63: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:18:03.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:18:02 smithi018 ceph-mon[184144]: pgmap v64: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:18:03.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:18:02 smithi018 ceph-mon[179647]: pgmap v64: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:18:03.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:02 smithi149 ceph-mon[100764]: pgmap v64: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:18:05.141 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:18:04 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:18:04] "GET /metrics HTTP/1.1" 200 33903 "" "Prometheus/2.43.0" 2023-09-23T20:18:05.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:18:04 smithi018 ceph-mon[184144]: pgmap v65: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:18:05.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:18:04 smithi018 ceph-mon[179647]: pgmap v65: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:18:05.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:04 smithi149 ceph-mon[100764]: pgmap v65: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:18:07.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:18:06 smithi018 ceph-mon[184144]: pgmap v66: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:18:07.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:18:06 smithi018 ceph-mon[179647]: pgmap v66: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:18:07.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:06 smithi149 ceph-mon[100764]: pgmap v66: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:18:08.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:08 smithi149 ceph-mon[100764]: pgmap v67: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:18:08.966 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:18:08 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[156506]: ts=2023-09-23T20:18:08.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.18\", device_class=\"ssd\", hostname=\"smithi018\", instance=\"smithi149.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.18\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.18\", device_class=\"ssd\", hostname=\"smithi018\", instance=\"smithi018:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.18\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T20:18:09.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:18:08 smithi018 ceph-mon[179647]: pgmap v67: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:18:09.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:18:08 smithi018 ceph-mon[184144]: pgmap v67: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:18:11.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:18:10 smithi018 ceph-mon[184144]: pgmap v68: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:18:11.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:18:10 smithi018 ceph-mon[179647]: pgmap v68: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:18:11.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:10 smithi149 ceph-mon[100764]: pgmap v68: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:18:13.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:18:12 smithi018 ceph-mon[184144]: pgmap v69: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:18:13.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:18:12 smithi018 ceph-mon[179647]: pgmap v69: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:18:13.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:12 smithi149 ceph-mon[100764]: pgmap v69: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:18:15.141 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:18:14 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:18:14] "GET /metrics HTTP/1.1" 200 33913 "" "Prometheus/2.43.0" 2023-09-23T20:18:15.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:18:14 smithi018 ceph-mon[184144]: pgmap v70: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:18:15.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:18:14 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:18:15.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:18:14 smithi018 ceph-mon[179647]: pgmap v70: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:18:15.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:18:14 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:18:15.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:14 smithi149 ceph-mon[100764]: pgmap v70: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:18:15.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:14 smithi149 ceph-mon[100764]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:18:17.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:18:16 smithi018 ceph-mon[184144]: pgmap v71: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:18:17.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:18:16 smithi018 ceph-mon[179647]: pgmap v71: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:18:17.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:16 smithi149 ceph-mon[100764]: pgmap v71: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:18:18.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:18 smithi149 ceph-mon[100764]: pgmap v72: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:18:18.966 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:18:18 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[156506]: ts=2023-09-23T20:18:18.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.18\", device_class=\"ssd\", hostname=\"smithi018\", instance=\"smithi149.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.18\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.18\", device_class=\"ssd\", hostname=\"smithi018\", instance=\"smithi018:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.18\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T20:18:19.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:18:18 smithi018 ceph-mon[179647]: pgmap v72: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:18:19.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:18:18 smithi018 ceph-mon[184144]: pgmap v72: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:18:21.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:18:20 smithi018 ceph-mon[179647]: pgmap v73: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:18:21.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:18:20 smithi018 ceph-mon[184144]: pgmap v73: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:18:21.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:20 smithi149 ceph-mon[100764]: pgmap v73: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:18:23.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:18:22 smithi018 ceph-mon[184144]: pgmap v74: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:18:23.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:18:22 smithi018 ceph-mon[179647]: pgmap v74: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:18:23.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:22 smithi149 ceph-mon[100764]: pgmap v74: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:18:25.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:18:24 smithi018 ceph-mon[179647]: pgmap v75: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:18:25.141 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:18:24 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:18:24] "GET /metrics HTTP/1.1" 200 33913 "" "Prometheus/2.43.0" 2023-09-23T20:18:25.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:18:24 smithi018 ceph-mon[184144]: pgmap v75: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:18:25.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:24 smithi149 ceph-mon[100764]: pgmap v75: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:18:27.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:18:26 smithi018 ceph-mon[184144]: pgmap v76: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:18:27.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:18:26 smithi018 ceph-mon[179647]: pgmap v76: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:18:27.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:26 smithi149 ceph-mon[100764]: pgmap v76: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:18:28.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:28 smithi149 ceph-mon[100764]: pgmap v77: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:18:28.966 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:18:28 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[156506]: ts=2023-09-23T20:18:28.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.18\", device_class=\"ssd\", hostname=\"smithi018\", instance=\"smithi149.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.18\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.18\", device_class=\"ssd\", hostname=\"smithi018\", instance=\"smithi018:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.18\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T20:18:29.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:18:28 smithi018 ceph-mon[179647]: pgmap v77: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:18:29.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:18:28 smithi018 ceph-mon[184144]: pgmap v77: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:18:30.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:18:29 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:18:30.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:18:29 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:18:30.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:29 smithi149 ceph-mon[100764]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:18:31.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:18:30 smithi018 ceph-mon[179647]: pgmap v78: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:18:31.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:18:30 smithi018 ceph-mon[184144]: pgmap v78: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:18:31.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:30 smithi149 ceph-mon[100764]: pgmap v78: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:18:33.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:18:32 smithi018 ceph-mon[184144]: pgmap v79: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:18:33.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:18:32 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:18:33.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:18:32 smithi018 ceph-mon[179647]: pgmap v79: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:18:33.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:18:32 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:18:33.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:32 smithi149 ceph-mon[100764]: pgmap v79: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:18:33.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:32 smithi149 ceph-mon[100764]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:18:34.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:18:34 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:18:34.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:18:34 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:18:34.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:18:34 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:18:34.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:18:34 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:18:34.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:18:34 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:18:34.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:18:34 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:18:34.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:18:34 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:18:34.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:18:34 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:18:34.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:34 smithi149 ceph-mon[100764]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:18:34.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:34 smithi149 ceph-mon[100764]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:18:34.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:34 smithi149 ceph-mon[100764]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:18:34.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:34 smithi149 ceph-mon[100764]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:18:34.542 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch upgrade resume' 2023-09-23T20:18:35.173 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:18:35.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:18:35 smithi018 ceph-mon[184144]: pgmap v80: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:18:35.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:18:35 smithi018 ceph-mon[179647]: pgmap v80: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:18:35.391 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:18:34 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:18:34] "GET /metrics HTTP/1.1" 200 33912 "" "Prometheus/2.43.0" 2023-09-23T20:18:35.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:35 smithi149 ceph-mon[100764]: pgmap v80: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:18:36.954 INFO:teuthology.orchestra.run.smithi018.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T20:18:37.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:18:37 smithi018 ceph-mon[184144]: pgmap v81: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:18:37.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:18:37 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:18:37.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:18:37 smithi018 ceph-mon[179647]: pgmap v81: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:18:37.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:18:37 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:18:37.442 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'echo wait for all mons to upgrade' 2023-09-23T20:18:37.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:37 smithi149 ceph-mon[100764]: pgmap v81: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:18:37.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:37 smithi149 ceph-mon[100764]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:18:38.197 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:18:38.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:18:38 smithi018 ceph-mon[184144]: from='client.34212 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:18:38.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:18:38 smithi018 ceph-mon[184144]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T20:18:38.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:18:38 smithi018 ceph-mon[179647]: from='client.34212 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:18:38.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:18:38 smithi018 ceph-mon[179647]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T20:18:38.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:38 smithi149 ceph-mon[100764]: from='client.34212 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:18:38.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:38 smithi149 ceph-mon[100764]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T20:18:38.966 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:18:38 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[156506]: ts=2023-09-23T20:18:38.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.18\", device_class=\"ssd\", hostname=\"smithi018\", instance=\"smithi149.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.18\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.18\", device_class=\"ssd\", hostname=\"smithi018\", instance=\"smithi018:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.18\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T20:18:39.302 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:18:39 smithi018 ceph-mon[184144]: pgmap v82: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:18:39.302 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:18:39 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:18:39.302 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:18:39 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:18:39.302 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:18:39 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:18:39.302 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:18:39 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:18:39.302 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:18:39 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:18:39.302 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:18:39 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:18:39.302 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:18:39 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:18:39.303 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:18:39 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:18:39.303 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:18:39 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:18:39.303 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:18:39 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-09-23T20:18:39.303 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:18:39 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2023-09-23T20:18:39.303 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:18:39 smithi018 ceph-mon[179647]: pgmap v82: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:18:39.303 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:18:39 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:18:39.303 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:18:39 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:18:39.303 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:18:39 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:18:39.303 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:18:39 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:18:39.303 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:18:39 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:18:39.304 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:18:39 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:18:39.304 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:18:39 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:18:39.304 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:18:39 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:18:39.304 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:18:39 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:18:39.304 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:18:39 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-09-23T20:18:39.304 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:18:39 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2023-09-23T20:18:39.406 INFO:teuthology.orchestra.run.smithi018.stdout:wait for all mons to upgrade 2023-09-23T20:18:39.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:39 smithi149 ceph-mon[100764]: pgmap v82: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:18:39.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:39 smithi149 ceph-mon[100764]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:18:39.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:39 smithi149 ceph-mon[100764]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:18:39.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:39 smithi149 ceph-mon[100764]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:18:39.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:39 smithi149 ceph-mon[100764]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:18:39.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:39 smithi149 ceph-mon[100764]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:18:39.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:39 smithi149 ceph-mon[100764]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:18:39.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:39 smithi149 ceph-mon[100764]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:18:39.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:39 smithi149 ceph-mon[100764]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:18:39.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:39 smithi149 ceph-mon[100764]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:18:39.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:39 smithi149 ceph-mon[100764]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-09-23T20:18:39.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:39 smithi149 ceph-mon[100764]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2023-09-23T20:18:39.852 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'while ! ceph mon versions | grep $sha1 | grep '"'"': 3'"'"' ; do sleep 2 ; done' 2023-09-23T20:18:40.171 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:40 smithi149 ceph-mon[100764]: Upgrade: It appears safe to stop mon.b 2023-09-23T20:18:40.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:18:40 smithi018 ceph-mon[184144]: Upgrade: It appears safe to stop mon.b 2023-09-23T20:18:40.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:18:40 smithi018 ceph-mon[179647]: Upgrade: It appears safe to stop mon.b 2023-09-23T20:18:40.459 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:18:41.300 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:41 smithi149 ceph-mon[100764]: pgmap v83: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:18:41.300 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:41 smithi149 ceph-mon[100764]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:18:41.300 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:41 smithi149 ceph-mon[100764]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T20:18:41.301 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:41 smithi149 ceph-mon[100764]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T20:18:41.301 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:41 smithi149 ceph-mon[100764]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:18:41.339 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:18:41 smithi018 ceph-mon[184144]: pgmap v83: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:18:41.339 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:18:41 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:18:41.340 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:18:41 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T20:18:41.340 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:18:41 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T20:18:41.340 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:18:41 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:18:41.340 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:18:41 smithi018 ceph-mon[179647]: pgmap v83: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:18:41.340 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:18:41 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:18:41.340 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:18:41 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T20:18:41.340 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:18:41 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T20:18:41.340 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:18:41 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:18:41.712 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:41 smithi149 systemd[1]: Stopping Ceph mon.b for a3b3b17e-5a4c-11ee-8db4-212e2dc638e7... 2023-09-23T20:18:41.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:41 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mon-b[100740]: 2023-09-23T20:18:41.712+0000 7f4105182700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T20:18:41.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:41 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mon-b[100740]: 2023-09-23T20:18:41.712+0000 7f4105182700 -1 mon.b@1(peon) e3 *** Got Signal Terminated *** 2023-09-23T20:18:42.380 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 bash[170204]: ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mon-b 2023-09-23T20:18:42.380 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 systemd[1]: ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7@mon.b.service: Succeeded. 2023-09-23T20:18:42.380 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 systemd[1]: Stopped Ceph mon.b for a3b3b17e-5a4c-11ee-8db4-212e2dc638e7. 2023-09-23T20:18:42.640 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 systemd[1]: Starting Ceph mon.b for a3b3b17e-5a4c-11ee-8db4-212e2dc638e7... 2023-09-23T20:18:42.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 podman[170354]: 2023-09-23T20:18:42.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 bash[170354]: b1930bb5f3e19ef9cd7733fa584623b6420363c6f967a9b7ab63099dd2e4a0aa 2023-09-23T20:18:42.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: set uid:gid to 167:167 (ceph:ceph) 2023-09-23T20:18:42.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable), process ceph-mon, pid 7 2023-09-23T20:18:42.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: pidfile_write: ignore empty --pid-file 2023-09-23T20:18:42.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: load: jerasure load: lrc 2023-09-23T20:18:42.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: RocksDB version: 7.9.2 2023-09-23T20:18:42.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Git sha 0 2023-09-23T20:18:42.968 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Compile date 2023-09-22 15:19:43 2023-09-23T20:18:42.968 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: DB SUMMARY 2023-09-23T20:18:42.968 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: DB Session ID: 321FLHWHF7PRYNMM5F1C 2023-09-23T20:18:42.968 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: CURRENT file: CURRENT 2023-09-23T20:18:42.968 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: IDENTITY file: IDENTITY 2023-09-23T20:18:42.968 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: MANIFEST file: MANIFEST-000009 size: 1083 Bytes 2023-09-23T20:18:42.968 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 1, files: 000027.sst 2023-09-23T20:18:42.968 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000025.log size: 2104570 ; 2023-09-23T20:18:42.968 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.error_if_exists: 0 2023-09-23T20:18:42.968 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.create_if_missing: 0 2023-09-23T20:18:42.968 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.paranoid_checks: 1 2023-09-23T20:18:42.968 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.flush_verify_memtable_count: 1 2023-09-23T20:18:42.969 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-09-23T20:18:42.969 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2023-09-23T20:18:42.969 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.env: 0x5645c5339720 2023-09-23T20:18:42.969 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.fs: PosixFileSystem 2023-09-23T20:18:42.969 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.info_log: 0x5645c82a5420 2023-09-23T20:18:42.969 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.max_file_opening_threads: 16 2023-09-23T20:18:42.969 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.statistics: (nil) 2023-09-23T20:18:42.969 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.use_fsync: 0 2023-09-23T20:18:42.969 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.max_log_file_size: 0 2023-09-23T20:18:42.969 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.max_manifest_file_size: 1073741824 2023-09-23T20:18:42.969 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.log_file_time_to_roll: 0 2023-09-23T20:18:42.969 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.keep_log_file_num: 1000 2023-09-23T20:18:42.970 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.recycle_log_file_num: 0 2023-09-23T20:18:42.970 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.allow_fallocate: 1 2023-09-23T20:18:42.970 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.allow_mmap_reads: 0 2023-09-23T20:18:42.970 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.allow_mmap_writes: 0 2023-09-23T20:18:42.970 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.use_direct_reads: 0 2023-09-23T20:18:42.970 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-09-23T20:18:42.970 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.create_missing_column_families: 0 2023-09-23T20:18:42.970 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.db_log_dir: 2023-09-23T20:18:42.970 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.wal_dir: 2023-09-23T20:18:42.970 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.table_cache_numshardbits: 6 2023-09-23T20:18:42.970 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.WAL_ttl_seconds: 0 2023-09-23T20:18:42.970 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.WAL_size_limit_MB: 0 2023-09-23T20:18:42.971 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-09-23T20:18:42.971 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.manifest_preallocation_size: 4194304 2023-09-23T20:18:42.971 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.is_fd_close_on_exec: 1 2023-09-23T20:18:42.971 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.advise_random_on_open: 1 2023-09-23T20:18:42.971 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.db_write_buffer_size: 0 2023-09-23T20:18:42.971 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.write_buffer_manager: 0x5645c7536c80 2023-09-23T20:18:42.971 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.access_hint_on_compaction_start: 1 2023-09-23T20:18:42.971 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.random_access_max_buffer_size: 1048576 2023-09-23T20:18:42.971 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.use_adaptive_mutex: 0 2023-09-23T20:18:42.971 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.rate_limiter: (nil) 2023-09-23T20:18:42.971 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-09-23T20:18:42.971 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.wal_recovery_mode: 2 2023-09-23T20:18:42.972 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.enable_thread_tracking: 0 2023-09-23T20:18:42.972 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.enable_pipelined_write: 0 2023-09-23T20:18:42.972 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.unordered_write: 0 2023-09-23T20:18:42.972 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.allow_concurrent_memtable_write: 1 2023-09-23T20:18:42.972 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-09-23T20:18:42.972 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.write_thread_max_yield_usec: 100 2023-09-23T20:18:42.972 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.write_thread_slow_yield_usec: 3 2023-09-23T20:18:42.972 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.row_cache: None 2023-09-23T20:18:42.972 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.wal_filter: None 2023-09-23T20:18:42.972 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.avoid_flush_during_recovery: 0 2023-09-23T20:18:42.972 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.allow_ingest_behind: 0 2023-09-23T20:18:42.972 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.two_write_queues: 0 2023-09-23T20:18:42.973 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.manual_wal_flush: 0 2023-09-23T20:18:42.973 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.wal_compression: 0 2023-09-23T20:18:42.973 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.atomic_flush: 0 2023-09-23T20:18:42.973 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-09-23T20:18:42.973 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.persist_stats_to_disk: 0 2023-09-23T20:18:42.973 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.write_dbid_to_manifest: 0 2023-09-23T20:18:42.973 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.log_readahead_size: 0 2023-09-23T20:18:42.973 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.file_checksum_gen_factory: Unknown 2023-09-23T20:18:42.973 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.best_efforts_recovery: 0 2023-09-23T20:18:42.973 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-09-23T20:18:42.973 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-09-23T20:18:42.973 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.allow_data_in_errors: 0 2023-09-23T20:18:42.974 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.db_host_id: __hostname__ 2023-09-23T20:18:42.974 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.enforce_single_del_contracts: true 2023-09-23T20:18:42.974 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.max_background_jobs: 2 2023-09-23T20:18:42.974 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.max_background_compactions: -1 2023-09-23T20:18:42.974 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.max_subcompactions: 1 2023-09-23T20:18:42.974 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.avoid_flush_during_shutdown: 0 2023-09-23T20:18:42.974 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-09-23T20:18:42.974 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.delayed_write_rate : 16777216 2023-09-23T20:18:42.974 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.max_total_wal_size: 0 2023-09-23T20:18:42.974 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-09-23T20:18:42.974 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.stats_dump_period_sec: 600 2023-09-23T20:18:42.975 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.stats_persist_period_sec: 600 2023-09-23T20:18:42.975 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.stats_history_buffer_size: 1048576 2023-09-23T20:18:42.975 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.max_open_files: -1 2023-09-23T20:18:42.975 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.bytes_per_sync: 0 2023-09-23T20:18:42.975 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.wal_bytes_per_sync: 0 2023-09-23T20:18:42.975 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.strict_bytes_per_sync: 0 2023-09-23T20:18:42.975 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.compaction_readahead_size: 0 2023-09-23T20:18:42.975 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.max_background_flushes: -1 2023-09-23T20:18:42.975 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Compression algorithms supported: 2023-09-23T20:18:42.975 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: kZSTDNotFinalCompression supported: 0 2023-09-23T20:18:42.975 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: kZSTD supported: 0 2023-09-23T20:18:42.975 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: kXpressCompression supported: 0 2023-09-23T20:18:42.976 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: kLZ4HCCompression supported: 1 2023-09-23T20:18:42.976 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: kZlibCompression supported: 1 2023-09-23T20:18:42.976 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: kSnappyCompression supported: 1 2023-09-23T20:18:42.976 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: kLZ4Compression supported: 1 2023-09-23T20:18:42.976 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: kBZip2Compression supported: 0 2023-09-23T20:18:42.976 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Fast CRC32 supported: Supported on x86 2023-09-23T20:18:42.976 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: DMutex implementation: pthread_mutex_t 2023-09-23T20:18:42.976 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000009 2023-09-23T20:18:42.976 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2023-09-23T20:18:42.977 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-09-23T20:18:42.977 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.merge_operator: 2023-09-23T20:18:42.977 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.compaction_filter: None 2023-09-23T20:18:42.977 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.compaction_filter_factory: None 2023-09-23T20:18:42.977 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.sst_partitioner_factory: None 2023-09-23T20:18:42.977 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.memtable_factory: SkipListFactory 2023-09-23T20:18:42.977 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.table_factory: BlockBasedTable 2023-09-23T20:18:42.977 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x5645c82a5520) 2023-09-23T20:18:42.977 INFO:journalctl@ceph.mon.b.smithi149.stdout: cache_index_and_filter_blocks: 1 2023-09-23T20:18:42.977 INFO:journalctl@ceph.mon.b.smithi149.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2023-09-23T20:18:42.977 INFO:journalctl@ceph.mon.b.smithi149.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2023-09-23T20:18:42.977 INFO:journalctl@ceph.mon.b.smithi149.stdout: pin_top_level_index_and_filter: 1 2023-09-23T20:18:42.977 INFO:journalctl@ceph.mon.b.smithi149.stdout: index_type: 0 2023-09-23T20:18:42.978 INFO:journalctl@ceph.mon.b.smithi149.stdout: data_block_index_type: 0 2023-09-23T20:18:42.978 INFO:journalctl@ceph.mon.b.smithi149.stdout: index_shortening: 1 2023-09-23T20:18:42.978 INFO:journalctl@ceph.mon.b.smithi149.stdout: data_block_hash_table_util_ratio: 0.750000 2023-09-23T20:18:42.978 INFO:journalctl@ceph.mon.b.smithi149.stdout: checksum: 4 2023-09-23T20:18:42.978 INFO:journalctl@ceph.mon.b.smithi149.stdout: no_block_cache: 0 2023-09-23T20:18:42.978 INFO:journalctl@ceph.mon.b.smithi149.stdout: block_cache: 0x5645c75b71f0 2023-09-23T20:18:42.978 INFO:journalctl@ceph.mon.b.smithi149.stdout: block_cache_name: BinnedLRUCache 2023-09-23T20:18:42.978 INFO:journalctl@ceph.mon.b.smithi149.stdout: block_cache_options: 2023-09-23T20:18:42.978 INFO:journalctl@ceph.mon.b.smithi149.stdout: capacity : 536870912 2023-09-23T20:18:42.978 INFO:journalctl@ceph.mon.b.smithi149.stdout: num_shard_bits : 4 2023-09-23T20:18:42.979 INFO:journalctl@ceph.mon.b.smithi149.stdout: strict_capacity_limit : 0 2023-09-23T20:18:42.979 INFO:journalctl@ceph.mon.b.smithi149.stdout: high_pri_pool_ratio: 0.000 2023-09-23T20:18:42.979 INFO:journalctl@ceph.mon.b.smithi149.stdout: block_cache_compressed: (nil) 2023-09-23T20:18:42.979 INFO:journalctl@ceph.mon.b.smithi149.stdout: persistent_cache: (nil) 2023-09-23T20:18:42.979 INFO:journalctl@ceph.mon.b.smithi149.stdout: block_size: 4096 2023-09-23T20:18:42.979 INFO:journalctl@ceph.mon.b.smithi149.stdout: block_size_deviation: 10 2023-09-23T20:18:42.979 INFO:journalctl@ceph.mon.b.smithi149.stdout: block_restart_interval: 16 2023-09-23T20:18:42.979 INFO:journalctl@ceph.mon.b.smithi149.stdout: index_block_restart_interval: 1 2023-09-23T20:18:42.979 INFO:journalctl@ceph.mon.b.smithi149.stdout: metadata_block_size: 4096 2023-09-23T20:18:42.979 INFO:journalctl@ceph.mon.b.smithi149.stdout: partition_filters: 0 2023-09-23T20:18:42.979 INFO:journalctl@ceph.mon.b.smithi149.stdout: use_delta_encoding: 1 2023-09-23T20:18:42.979 INFO:journalctl@ceph.mon.b.smithi149.stdout: filter_policy: bloomfilter 2023-09-23T20:18:42.980 INFO:journalctl@ceph.mon.b.smithi149.stdout: whole_key_filtering: 1 2023-09-23T20:18:42.980 INFO:journalctl@ceph.mon.b.smithi149.stdout: verify_compression: 0 2023-09-23T20:18:42.980 INFO:journalctl@ceph.mon.b.smithi149.stdout: read_amp_bytes_per_bit: 0 2023-09-23T20:18:42.980 INFO:journalctl@ceph.mon.b.smithi149.stdout: format_version: 5 2023-09-23T20:18:42.980 INFO:journalctl@ceph.mon.b.smithi149.stdout: enable_index_compression: 1 2023-09-23T20:18:42.980 INFO:journalctl@ceph.mon.b.smithi149.stdout: block_align: 0 2023-09-23T20:18:42.980 INFO:journalctl@ceph.mon.b.smithi149.stdout: max_auto_readahead_size: 262144 2023-09-23T20:18:42.980 INFO:journalctl@ceph.mon.b.smithi149.stdout: prepopulate_block_cache: 0 2023-09-23T20:18:42.980 INFO:journalctl@ceph.mon.b.smithi149.stdout: initial_auto_readahead_size: 8192 2023-09-23T20:18:42.980 INFO:journalctl@ceph.mon.b.smithi149.stdout: num_file_reads_for_auto_readahead: 2 2023-09-23T20:18:42.980 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.write_buffer_size: 33554432 2023-09-23T20:18:42.980 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.max_write_buffer_number: 2 2023-09-23T20:18:42.981 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.compression: NoCompression 2023-09-23T20:18:42.981 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.bottommost_compression: Disabled 2023-09-23T20:18:42.981 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.prefix_extractor: nullptr 2023-09-23T20:18:42.981 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-09-23T20:18:42.981 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.num_levels: 7 2023-09-23T20:18:42.981 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-09-23T20:18:42.981 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-09-23T20:18:42.981 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-09-23T20:18:42.981 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-09-23T20:18:42.981 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.bottommost_compression_opts.level: 32767 2023-09-23T20:18:42.981 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-09-23T20:18:42.981 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-09-23T20:18:42.982 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-09-23T20:18:42.982 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-09-23T20:18:42.982 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.bottommost_compression_opts.enabled: false 2023-09-23T20:18:42.982 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2023-09-23T20:18:42.982 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2023-09-23T20:18:42.982 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.compression_opts.window_bits: -14 2023-09-23T20:18:42.982 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.compression_opts.level: 32767 2023-09-23T20:18:42.982 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.compression_opts.strategy: 0 2023-09-23T20:18:42.982 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-09-23T20:18:42.982 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-09-23T20:18:42.982 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2023-09-23T20:18:42.982 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.compression_opts.parallel_threads: 1 2023-09-23T20:18:42.983 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.compression_opts.enabled: false 2023-09-23T20:18:42.983 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2023-09-23T20:18:42.983 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-09-23T20:18:42.983 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-09-23T20:18:42.983 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.level0_stop_writes_trigger: 36 2023-09-23T20:18:42.983 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.target_file_size_base: 67108864 2023-09-23T20:18:42.983 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.target_file_size_multiplier: 1 2023-09-23T20:18:42.983 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.max_bytes_for_level_base: 268435456 2023-09-23T20:18:42.983 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-09-23T20:18:42.983 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-09-23T20:18:42.983 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-09-23T20:18:42.983 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-09-23T20:18:42.984 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-09-23T20:18:42.984 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-09-23T20:18:42.984 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-09-23T20:18:42.984 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-09-23T20:18:42.984 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-09-23T20:18:42.984 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-09-23T20:18:42.984 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.max_compaction_bytes: 1677721600 2023-09-23T20:18:42.984 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2023-09-23T20:18:42.984 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.arena_block_size: 1048576 2023-09-23T20:18:42.984 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-09-23T20:18:42.984 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-09-23T20:18:42.984 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.disable_auto_compactions: 0 2023-09-23T20:18:42.985 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-09-23T20:18:42.985 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-09-23T20:18:42.985 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-09-23T20:18:42.986 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-09-23T20:18:42.986 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-09-23T20:18:42.986 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-09-23T20:18:42.986 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-09-23T20:18:42.986 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-09-23T20:18:42.986 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-09-23T20:18:42.986 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-09-23T20:18:42.986 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.table_properties_collectors: 2023-09-23T20:18:42.986 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.inplace_update_support: 0 2023-09-23T20:18:42.986 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.inplace_update_num_locks: 10000 2023-09-23T20:18:42.986 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-09-23T20:18:42.986 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.memtable_whole_key_filtering: 0 2023-09-23T20:18:42.987 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.memtable_huge_page_size: 0 2023-09-23T20:18:42.987 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.bloom_locality: 0 2023-09-23T20:18:42.987 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.max_successive_merges: 0 2023-09-23T20:18:42.987 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.optimize_filters_for_hits: 0 2023-09-23T20:18:42.987 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.paranoid_file_checks: 0 2023-09-23T20:18:42.987 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.force_consistency_checks: 1 2023-09-23T20:18:42.987 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.report_bg_io_stats: 0 2023-09-23T20:18:42.987 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.ttl: 2592000 2023-09-23T20:18:42.987 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.periodic_compaction_seconds: 0 2023-09-23T20:18:42.987 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.preclude_last_level_data_seconds: 0 2023-09-23T20:18:42.987 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.preserve_internal_time_seconds: 0 2023-09-23T20:18:42.987 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.enable_blob_files: false 2023-09-23T20:18:42.988 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.min_blob_size: 0 2023-09-23T20:18:42.988 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.blob_file_size: 268435456 2023-09-23T20:18:42.988 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.blob_compression_type: NoCompression 2023-09-23T20:18:42.988 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.enable_blob_garbage_collection: false 2023-09-23T20:18:42.988 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-09-23T20:18:42.988 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2023-09-23T20:18:42.988 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.blob_compaction_readahead_size: 0 2023-09-23T20:18:42.988 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.blob_file_starting_level: 0 2023-09-23T20:18:42.988 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2023-09-23T20:18:42.988 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 27.sst 2023-09-23T20:18:42.988 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-09-23T20:18:42.988 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 29, last_sequence is 13646, log_number is 25,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-09-23T20:18:42.989 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 25 2023-09-23T20:18:42.989 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: b2baa7e8-8c2d-43a6-81ee-62d771b8beb7 2023-09-23T20:18:42.989 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695500322853303, "job": 1, "event": "recovery_started", "wal_files": [25]} 2023-09-23T20:18:42.989 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #25 mode 2 2023-09-23T20:18:42.989 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695500322858999, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 30, "file_size": 1279699, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 13641, "largest_seqno": 15088, "table_properties": {"data_size": 1274120, "index_size": 3348, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 1221, "raw_key_size": 10683, "raw_average_key_size": 22, "raw_value_size": 1263625, "raw_average_value_size": 2605, "num_data_blocks": 154, "num_entries": 485, "num_filter_entries": 485, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1695500322, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "b2baa7e8-8c2d-43a6-81ee-62d771b8beb7", "db_session_id": "321FLHWHF7PRYNMM5F1C", "orig_file_number": 30, "seqno_to_time_mapping": "N/A"}} 2023-09-23T20:18:42.989 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695500322859095, "job": 1, "event": "recovery_finished"} 2023-09-23T20:18:42.989 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: [db/version_set.cc:5047] Creating manifest 32 2023-09-23T20:18:42.989 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-09-23T20:18:42.989 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000025.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-09-23T20:18:42.989 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x5645c7654000 2023-09-23T20:18:42.989 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: DB pointer 0x5645c7640000 2023-09-23T20:18:42.989 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2023-09-23T20:18:42.989 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: rocksdb: [db/db_impl/db_impl.cc:1111] 2023-09-23T20:18:42.990 INFO:journalctl@ceph.mon.b.smithi149.stdout: ** DB Stats ** 2023-09-23T20:18:42.990 INFO:journalctl@ceph.mon.b.smithi149.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T20:18:42.990 INFO:journalctl@ceph.mon.b.smithi149.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-09-23T20:18:42.990 INFO:journalctl@ceph.mon.b.smithi149.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-09-23T20:18:42.990 INFO:journalctl@ceph.mon.b.smithi149.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T20:18:42.990 INFO:journalctl@ceph.mon.b.smithi149.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-09-23T20:18:42.990 INFO:journalctl@ceph.mon.b.smithi149.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-09-23T20:18:42.990 INFO:journalctl@ceph.mon.b.smithi149.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T20:18:42.990 INFO:journalctl@ceph.mon.b.smithi149.stdout: 2023-09-23T20:18:42.990 INFO:journalctl@ceph.mon.b.smithi149.stdout: ** Compaction Stats [default] ** 2023-09-23T20:18:42.990 INFO:journalctl@ceph.mon.b.smithi149.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-09-23T20:18:42.991 INFO:journalctl@ceph.mon.b.smithi149.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2023-09-23T20:18:42.991 INFO:journalctl@ceph.mon.b.smithi149.stdout: L0 1/0 1.22 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 526.3 0.00 0.00 1 0.002 0 0 0.0 0.0 2023-09-23T20:18:42.991 INFO:journalctl@ceph.mon.b.smithi149.stdout: L6 1/0 7.84 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2023-09-23T20:18:42.991 INFO:journalctl@ceph.mon.b.smithi149.stdout: Sum 2/0 9.06 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 526.3 0.00 0.00 1 0.002 0 0 0.0 0.0 2023-09-23T20:18:42.991 INFO:journalctl@ceph.mon.b.smithi149.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 526.3 0.00 0.00 1 0.002 0 0 0.0 0.0 2023-09-23T20:18:42.991 INFO:journalctl@ceph.mon.b.smithi149.stdout: 2023-09-23T20:18:42.991 INFO:journalctl@ceph.mon.b.smithi149.stdout: ** Compaction Stats [default] ** 2023-09-23T20:18:42.991 INFO:journalctl@ceph.mon.b.smithi149.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-09-23T20:18:42.991 INFO:journalctl@ceph.mon.b.smithi149.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T20:18:42.991 INFO:journalctl@ceph.mon.b.smithi149.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 526.3 0.00 0.00 1 0.002 0 0 0.0 0.0 2023-09-23T20:18:42.991 INFO:journalctl@ceph.mon.b.smithi149.stdout: 2023-09-23T20:18:42.991 INFO:journalctl@ceph.mon.b.smithi149.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2023-09-23T20:18:42.992 INFO:journalctl@ceph.mon.b.smithi149.stdout: 2023-09-23T20:18:42.992 INFO:journalctl@ceph.mon.b.smithi149.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T20:18:42.992 INFO:journalctl@ceph.mon.b.smithi149.stdout: Flush(GB): cumulative 0.001, interval 0.001 2023-09-23T20:18:42.992 INFO:journalctl@ceph.mon.b.smithi149.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-09-23T20:18:42.992 INFO:journalctl@ceph.mon.b.smithi149.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-09-23T20:18:42.992 INFO:journalctl@ceph.mon.b.smithi149.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-09-23T20:18:42.992 INFO:journalctl@ceph.mon.b.smithi149.stdout: AddFile(Keys): cumulative 0, interval 0 2023-09-23T20:18:42.992 INFO:journalctl@ceph.mon.b.smithi149.stdout: Cumulative compaction: 0.00 GB write, 118.75 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T20:18:42.992 INFO:journalctl@ceph.mon.b.smithi149.stdout: Interval compaction: 0.00 GB write, 118.75 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T20:18:42.992 INFO:journalctl@ceph.mon.b.smithi149.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-09-23T20:18:42.992 INFO:journalctl@ceph.mon.b.smithi149.stdout: Block cache BinnedLRUCache@0x5645c75b71f0#7 capacity: 512.00 MB usage: 5.36 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.3e-05 secs_since: 0 2023-09-23T20:18:42.992 INFO:journalctl@ceph.mon.b.smithi149.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,1.28 KB,0.000244379%) IndexBlock(1,4.08 KB,0.000777841%) Misc(1,0.00 KB,0%) 2023-09-23T20:18:42.993 INFO:journalctl@ceph.mon.b.smithi149.stdout: 2023-09-23T20:18:42.993 INFO:journalctl@ceph.mon.b.smithi149.stdout: ** File Read Latency Histogram By Level [default] ** 2023-09-23T20:18:42.993 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: starting mon.b rank 1 at public addrs [v2:172.21.15.149:3300/0,v1:172.21.15.149:6789/0] at bind addrs [v2:172.21.15.149:3300/0,v1:172.21.15.149:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 2023-09-23T20:18:42.993 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: mon.b@-1(???) e3 preinit fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 2023-09-23T20:18:42.993 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: mon.b@-1(???).mds e9 new map 2023-09-23T20:18:42.993 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: mon.b@-1(???).mds e9 print_map 2023-09-23T20:18:42.993 INFO:journalctl@ceph.mon.b.smithi149.stdout: e9 2023-09-23T20:18:42.993 INFO:journalctl@ceph.mon.b.smithi149.stdout: enable_multiple, ever_enabled_multiple: 1,1 2023-09-23T20:18:42.993 INFO:journalctl@ceph.mon.b.smithi149.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-09-23T20:18:42.993 INFO:journalctl@ceph.mon.b.smithi149.stdout: legacy client fscid: 1 2023-09-23T20:18:42.993 INFO:journalctl@ceph.mon.b.smithi149.stdout: 2023-09-23T20:18:42.993 INFO:journalctl@ceph.mon.b.smithi149.stdout: Filesystem 'foo' (1) 2023-09-23T20:18:42.994 INFO:journalctl@ceph.mon.b.smithi149.stdout: fs_name foo 2023-09-23T20:18:42.994 INFO:journalctl@ceph.mon.b.smithi149.stdout: epoch 8 2023-09-23T20:18:42.994 INFO:journalctl@ceph.mon.b.smithi149.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2023-09-23T20:18:42.994 INFO:journalctl@ceph.mon.b.smithi149.stdout: created 2023-09-23T20:12:43.745113+0000 2023-09-23T20:18:42.994 INFO:journalctl@ceph.mon.b.smithi149.stdout: modified 2023-09-23T20:12:54.848699+0000 2023-09-23T20:18:42.994 INFO:journalctl@ceph.mon.b.smithi149.stdout: tableserver 0 2023-09-23T20:18:42.994 INFO:journalctl@ceph.mon.b.smithi149.stdout: root 0 2023-09-23T20:18:42.994 INFO:journalctl@ceph.mon.b.smithi149.stdout: session_timeout 60 2023-09-23T20:18:42.994 INFO:journalctl@ceph.mon.b.smithi149.stdout: session_autoclose 300 2023-09-23T20:18:42.994 INFO:journalctl@ceph.mon.b.smithi149.stdout: max_file_size 1099511627776 2023-09-23T20:18:42.994 INFO:journalctl@ceph.mon.b.smithi149.stdout: required_client_features {} 2023-09-23T20:18:42.994 INFO:journalctl@ceph.mon.b.smithi149.stdout: last_failure 0 2023-09-23T20:18:42.995 INFO:journalctl@ceph.mon.b.smithi149.stdout: last_failure_osd_epoch 0 2023-09-23T20:18:42.995 INFO:journalctl@ceph.mon.b.smithi149.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-09-23T20:18:42.995 INFO:journalctl@ceph.mon.b.smithi149.stdout: max_mds 1 2023-09-23T20:18:42.995 INFO:journalctl@ceph.mon.b.smithi149.stdout: in 0 2023-09-23T20:18:42.995 INFO:journalctl@ceph.mon.b.smithi149.stdout: up {0=24566} 2023-09-23T20:18:42.995 INFO:journalctl@ceph.mon.b.smithi149.stdout: failed 2023-09-23T20:18:42.995 INFO:journalctl@ceph.mon.b.smithi149.stdout: damaged 2023-09-23T20:18:42.995 INFO:journalctl@ceph.mon.b.smithi149.stdout: stopped 2023-09-23T20:18:42.995 INFO:journalctl@ceph.mon.b.smithi149.stdout: data_pools [4] 2023-09-23T20:18:42.995 INFO:journalctl@ceph.mon.b.smithi149.stdout: metadata_pool 3 2023-09-23T20:18:42.995 INFO:journalctl@ceph.mon.b.smithi149.stdout: inline_data disabled 2023-09-23T20:18:42.995 INFO:journalctl@ceph.mon.b.smithi149.stdout: balancer 2023-09-23T20:18:42.995 INFO:journalctl@ceph.mon.b.smithi149.stdout: bal_rank_mask -1 2023-09-23T20:18:42.996 INFO:journalctl@ceph.mon.b.smithi149.stdout: standby_count_wanted 1 2023-09-23T20:18:42.996 INFO:journalctl@ceph.mon.b.smithi149.stdout: [mds.foo.smithi018.hduiol{0:24566} state up:active seq 3 join_fscid=1 addr [v2:172.21.15.18:6834/3661910111,v1:172.21.15.18:6835/3661910111] compat {c=[1],r=[1],i=[7ff]}] 2023-09-23T20:18:42.996 INFO:journalctl@ceph.mon.b.smithi149.stdout: 2023-09-23T20:18:42.996 INFO:journalctl@ceph.mon.b.smithi149.stdout: 2023-09-23T20:18:42.996 INFO:journalctl@ceph.mon.b.smithi149.stdout: Standby daemons: 2023-09-23T20:18:42.996 INFO:journalctl@ceph.mon.b.smithi149.stdout: 2023-09-23T20:18:42.996 INFO:journalctl@ceph.mon.b.smithi149.stdout: [mds.foo.smithi149.urtezf{-1:24559} state up:standby seq 2 join_fscid=1 addr [v2:172.21.15.149:6832/3529449999,v1:172.21.15.149:6833/3529449999] compat {c=[1],r=[1],i=[7ff]}] 2023-09-23T20:18:42.996 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: mon.b@-1(???).osd e90 crush map has features 3314933000854323200, adjusting msgr requires 2023-09-23T20:18:42.996 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: mon.b@-1(???).osd e90 crush map has features 432629239337189376, adjusting msgr requires 2023-09-23T20:18:42.996 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: mon.b@-1(???).osd e90 crush map has features 432629239337189376, adjusting msgr requires 2023-09-23T20:18:42.996 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: mon.b@-1(???).osd e90 crush map has features 432629239337189376, adjusting msgr requires 2023-09-23T20:18:42.997 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: mon.b@-1(???).paxosservice(auth 1..20) refresh upgraded, format 0 -> 3 2023-09-23T20:18:42.997 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 systemd[1]: Started Ceph mon.b for a3b3b17e-5a4c-11ee-8db4-212e2dc638e7. 2023-09-23T20:18:42.997 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: expand_channel_meta expand map: {default=false} 2023-09-23T20:18:42.997 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: expand_channel_meta from 'false' to 'false' 2023-09-23T20:18:42.997 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: expand_channel_meta expanded map: {default=false} 2023-09-23T20:18:42.997 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: expand_channel_meta expand map: {default=info} 2023-09-23T20:18:42.997 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: expand_channel_meta from 'info' to 'info' 2023-09-23T20:18:42.997 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: expand_channel_meta expanded map: {default=info} 2023-09-23T20:18:42.997 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: expand_channel_meta expand map: {default=daemon} 2023-09-23T20:18:42.997 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: expand_channel_meta from 'daemon' to 'daemon' 2023-09-23T20:18:42.997 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: expand_channel_meta expanded map: {default=daemon} 2023-09-23T20:18:42.997 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: expand_channel_meta expand map: {default=debug} 2023-09-23T20:18:42.997 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: expand_channel_meta from 'debug' to 'debug' 2023-09-23T20:18:42.998 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:42 smithi149 ceph-mon[170394]: expand_channel_meta expanded map: {default=debug} 2023-09-23T20:18:44.599 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable)": 3 2023-09-23T20:18:44.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:44 smithi149 ceph-mon[170394]: pgmap v85: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:18:44.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:44 smithi149 ceph-mon[170394]: overall HEALTH_OK 2023-09-23T20:18:44.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:44 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T20:18:44.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:44 smithi149 ceph-mon[170394]: mon.b calling monitor election 2023-09-23T20:18:44.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:44 smithi149 ceph-mon[170394]: mon.a calling monitor election 2023-09-23T20:18:44.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:44 smithi149 ceph-mon[170394]: mon.c calling monitor election 2023-09-23T20:18:44.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:44 smithi149 ceph-mon[170394]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T20:18:44.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:44 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:18:44.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:44 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:18:44.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:44 smithi149 ceph-mon[170394]: monmap e4: 3 mons at {a=[v2:172.21.15.18:3300/0,v1:172.21.15.18:6789/0],b=[v2:172.21.15.149:3300/0,v1:172.21.15.149:6789/0],c=[v2:172.21.15.18:3301/0,v1:172.21.15.18:6790/0]} removed_ranks: {} 2023-09-23T20:18:44.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:44 smithi149 ceph-mon[170394]: fsmap foo:1 {0=foo.smithi018.hduiol=up:active} 1 up:standby 2023-09-23T20:18:44.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:44 smithi149 ceph-mon[170394]: osdmap e90: 8 total, 8 up, 8 in 2023-09-23T20:18:44.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:44 smithi149 ceph-mon[170394]: mgrmap e36: y(active, since 2m), standbys: x 2023-09-23T20:18:44.718 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:44 smithi149 ceph-mon[170394]: overall HEALTH_OK 2023-09-23T20:18:44.718 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:44 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:18:44.718 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:44 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:18:44.718 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:44 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:18:44.735 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:18:44 smithi018 ceph-mon[184144]: pgmap v85: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:18:44.735 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:18:44 smithi018 ceph-mon[184144]: overall HEALTH_OK 2023-09-23T20:18:44.735 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:18:44 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T20:18:44.736 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:18:44 smithi018 ceph-mon[184144]: mon.b calling monitor election 2023-09-23T20:18:44.736 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:18:44 smithi018 ceph-mon[184144]: mon.a calling monitor election 2023-09-23T20:18:44.736 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:18:44 smithi018 ceph-mon[184144]: mon.c calling monitor election 2023-09-23T20:18:44.736 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:18:44 smithi018 ceph-mon[184144]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T20:18:44.736 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:18:44 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:18:44.736 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:18:44 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:18:44.736 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:18:44 smithi018 ceph-mon[184144]: monmap e4: 3 mons at {a=[v2:172.21.15.18:3300/0,v1:172.21.15.18:6789/0],b=[v2:172.21.15.149:3300/0,v1:172.21.15.149:6789/0],c=[v2:172.21.15.18:3301/0,v1:172.21.15.18:6790/0]} removed_ranks: {} 2023-09-23T20:18:44.736 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:18:44 smithi018 ceph-mon[184144]: fsmap foo:1 {0=foo.smithi018.hduiol=up:active} 1 up:standby 2023-09-23T20:18:44.736 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:18:44 smithi018 ceph-mon[184144]: osdmap e90: 8 total, 8 up, 8 in 2023-09-23T20:18:44.736 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:18:44 smithi018 ceph-mon[184144]: mgrmap e36: y(active, since 2m), standbys: x 2023-09-23T20:18:44.736 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:18:44 smithi018 ceph-mon[184144]: overall HEALTH_OK 2023-09-23T20:18:44.736 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:18:44 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:18:44.737 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:18:44 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:18:44.737 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:18:44 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:18:44.737 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:18:44 smithi018 ceph-mon[179647]: pgmap v85: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:18:44.737 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:18:44 smithi018 ceph-mon[179647]: overall HEALTH_OK 2023-09-23T20:18:44.737 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:18:44 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T20:18:44.737 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:18:44 smithi018 ceph-mon[179647]: mon.b calling monitor election 2023-09-23T20:18:44.737 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:18:44 smithi018 ceph-mon[179647]: mon.a calling monitor election 2023-09-23T20:18:44.737 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:18:44 smithi018 ceph-mon[179647]: mon.c calling monitor election 2023-09-23T20:18:44.737 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:18:44 smithi018 ceph-mon[179647]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T20:18:44.737 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:18:44 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T20:18:44.738 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:18:44 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T20:18:44.738 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:18:44 smithi018 ceph-mon[179647]: monmap e4: 3 mons at {a=[v2:172.21.15.18:3300/0,v1:172.21.15.18:6789/0],b=[v2:172.21.15.149:3300/0,v1:172.21.15.149:6789/0],c=[v2:172.21.15.18:3301/0,v1:172.21.15.18:6790/0]} removed_ranks: {} 2023-09-23T20:18:44.738 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:18:44 smithi018 ceph-mon[179647]: fsmap foo:1 {0=foo.smithi018.hduiol=up:active} 1 up:standby 2023-09-23T20:18:44.738 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:18:44 smithi018 ceph-mon[179647]: osdmap e90: 8 total, 8 up, 8 in 2023-09-23T20:18:44.738 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:18:44 smithi018 ceph-mon[179647]: mgrmap e36: y(active, since 2m), standbys: x 2023-09-23T20:18:44.738 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:18:44 smithi018 ceph-mon[179647]: overall HEALTH_OK 2023-09-23T20:18:44.738 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:18:44 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:18:44.738 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:18:44 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:18:44.738 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:18:44 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:18:45.024 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch ps' 2023-09-23T20:18:45.391 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:18:44 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:18:44] "GET /metrics HTTP/1.1" 200 33840 "" "Prometheus/2.43.0" 2023-09-23T20:18:45.630 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:18:45.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:45 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:18:45.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:45 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3672032740' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:18:45.825 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:18:45 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:18:45.825 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:18:45 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3672032740' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:18:45.825 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:18:45 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:18:45.825 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:18:45 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3672032740' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T20:18:46.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:18:46 smithi018 ceph-mon[184144]: pgmap v86: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:18:46.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:18:46 smithi018 ceph-mon[179647]: pgmap v86: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:18:47.021 INFO:teuthology.orchestra.run.smithi018.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-09-23T20:18:47.022 INFO:teuthology.orchestra.run.smithi018.stdout:alertmanager.a smithi018 *:9093,9094 running (3m) - 6m 28.7M - 0.25.0 c8568f914cd2 57241c9344be 2023-09-23T20:18:47.022 INFO:teuthology.orchestra.run.smithi018.stdout:grafana.a smithi149 *:3000 running (3m) - 6m 93.3M - 9.4.7 2c41d148cca3 62150314e898 2023-09-23T20:18:47.022 INFO:teuthology.orchestra.run.smithi018.stdout:mds.foo.smithi018.hduiol smithi018 running (5m) - 5m 18.2M - 17.2.6-1123-g61431f6b 0db14aa01094 7b6a15d633a4 2023-09-23T20:18:47.022 INFO:teuthology.orchestra.run.smithi018.stdout:mds.foo.smithi149.urtezf smithi149 running (5m) - 5m 17.7M - 17.2.6-1123-g61431f6b 0db14aa01094 6ee4df523f8c 2023-09-23T20:18:47.022 INFO:teuthology.orchestra.run.smithi018.stdout:mgr.x smithi149 *:8443,9283,8765 running (3m) - 10m 448M - 18.2.0-460-gbb0daeff e32c2a64c126 9513b5c2a2f7 2023-09-23T20:18:47.022 INFO:teuthology.orchestra.run.smithi018.stdout:mgr.y smithi018 *:8443,9283,8765 running (3m) - 11m 504M - 18.2.0-460-gbb0daeff e32c2a64c126 5aa74f13c0a1 2023-09-23T20:18:47.022 INFO:teuthology.orchestra.run.smithi018.stdout:mon.a smithi018 running (2m) - 11m 61.2M 2048M 18.2.0-460-gbb0daeff e32c2a64c126 425294bbcbd1 2023-09-23T20:18:47.022 INFO:teuthology.orchestra.run.smithi018.stdout:mon.b smithi149 starting - - - 2048M 2023-09-23T20:18:47.022 INFO:teuthology.orchestra.run.smithi018.stdout:mon.c smithi018 running (89s) - 10m 35.0M 2048M 18.2.0-460-gbb0daeff e32c2a64c126 e2701e7c5d35 2023-09-23T20:18:47.022 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter.a smithi018 *:9100 running (3m) - 6m 17.3M - 1.5.0 0da6a335fe13 500f76739e6a 2023-09-23T20:18:47.023 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter.b smithi149 *:9100 running (3m) - 6m 17.8M - 1.5.0 0da6a335fe13 f1789a46736b 2023-09-23T20:18:47.023 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0 smithi018 running (9m) - 9m 82.0M 2501M 17.2.6-1123-g61431f6b 0db14aa01094 6749d0af6e50 2023-09-23T20:18:47.023 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1 smithi018 running (9m) - 9m 100M 2501M 17.2.6-1123-g61431f6b 0db14aa01094 5d199baeab0e 2023-09-23T20:18:47.023 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2 smithi018 running (9m) - 9m 81.4M 2501M 17.2.6-1123-g61431f6b 0db14aa01094 2effc63f4610 2023-09-23T20:18:47.023 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3 smithi018 running (8m) - 8m 69.9M 2501M 17.2.6-1123-g61431f6b 0db14aa01094 3301c3e11bd5 2023-09-23T20:18:47.023 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4 smithi149 running (8m) - 8m 79.0M 4096M 17.2.6-1123-g61431f6b 0db14aa01094 b4b39c43de75 2023-09-23T20:18:47.023 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5 smithi149 running (8m) - 8m 90.5M 4096M 17.2.6-1123-g61431f6b 0db14aa01094 cec31ff38f4c 2023-09-23T20:18:47.023 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6 smithi149 running (7m) - 7m 81.7M 4096M 17.2.6-1123-g61431f6b 0db14aa01094 ce687413759d 2023-09-23T20:18:47.023 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7 smithi149 running (7m) - 7m 94.5M 4096M 17.2.6-1123-g61431f6b 0db14aa01094 d3145e28f8c3 2023-09-23T20:18:47.023 INFO:teuthology.orchestra.run.smithi018.stdout:prometheus.a smithi149 *:9095 running (3m) - 7m 58.3M - 2.43.0 a07b618ecd1d bb2162e75dc7 2023-09-23T20:18:47.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:46 smithi149 ceph-mon[170394]: pgmap v86: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:18:47.477 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch upgrade pause' 2023-09-23T20:18:48.083 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:18:48.141 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:18:47 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: 2023-09-23T20:18:47.870+0000 7f953be6a700 -1 mgr.server handle_report got status from non-daemon mon.b 2023-09-23T20:18:48.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:18:47 smithi018 ceph-mon[184144]: from='client.34230 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:18:48.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:18:47 smithi018 ceph-mon[179647]: from='client.34230 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:18:48.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:47 smithi149 ceph-mon[170394]: from='client.34230 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:18:48.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:48 smithi149 ceph-mon[170394]: pgmap v87: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:18:48.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:48 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:18:48.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:48 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:18:48.966 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:18:48 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[156506]: ts=2023-09-23T20:18:48.658Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.18\", device_class=\"ssd\", hostname=\"smithi018\", instance=\"smithi149.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.18\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.18\", device_class=\"ssd\", hostname=\"smithi018\", instance=\"smithi018:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.18\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T20:18:49.132 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:18:48 smithi018 ceph-mon[184144]: pgmap v87: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:18:49.132 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:18:48 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:18:49.132 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:18:48 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:18:49.132 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:18:48 smithi018 ceph-mon[179647]: pgmap v87: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:18:49.133 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:18:48 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:18:49.133 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:18:48 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:18:49.539 INFO:teuthology.orchestra.run.smithi018.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T20:18:49.943 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'sleep 60' 2023-09-23T20:18:50.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:50 smithi149 ceph-mon[170394]: pgmap v88: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:18:50.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:50 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:18:50.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:50 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:18:50.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:50 smithi149 ceph-mon[170394]: from='client.44138 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:18:50.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:50 smithi149 ceph-mon[170394]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T20:18:50.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:50 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:18:50.546 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:18:50.757 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:18:50 smithi018 ceph-mon[184144]: pgmap v88: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:18:50.757 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:18:50 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:18:50.758 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:18:50 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:18:50.758 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:18:50 smithi018 ceph-mon[184144]: from='client.44138 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:18:50.758 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:18:50 smithi018 ceph-mon[184144]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T20:18:50.758 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:18:50 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:18:50.758 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:18:50 smithi018 ceph-mon[179647]: pgmap v88: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:18:50.758 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:18:50 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:18:50.758 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:18:50 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:18:50.758 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:18:50 smithi018 ceph-mon[179647]: from='client.44138 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:18:50.758 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:18:50 smithi018 ceph-mon[179647]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T20:18:50.758 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:18:50 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:18:53.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:18:52 smithi018 ceph-mon[179647]: pgmap v89: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:18:53.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:18:52 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:18:53.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:18:52 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:18:53.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:18:52 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:18:53.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:18:52 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:18:53.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:18:52 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:18:53.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:18:52 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T20:18:53.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:18:52 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T20:18:53.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:18:52 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:18:53.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:18:52 smithi018 ceph-mon[184144]: pgmap v89: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:18:53.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:18:52 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:18:53.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:18:52 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:18:53.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:18:52 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:18:53.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:18:52 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:18:53.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:18:52 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:18:53.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:18:52 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T20:18:53.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:18:52 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T20:18:53.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:18:52 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:18:53.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:52 smithi149 ceph-mon[170394]: pgmap v89: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:18:53.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:52 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:18:53.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:52 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:18:53.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:52 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:18:53.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:52 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:18:53.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:52 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:18:53.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:52 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T20:18:53.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:52 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T20:18:53.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:52 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:18:53.791 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:18:53 smithi018 ceph-mon[179647]: Reconfiguring mon.a (monmap changed)... 2023-09-23T20:18:53.791 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:18:53 smithi018 ceph-mon[179647]: Reconfiguring daemon mon.a on smithi018 2023-09-23T20:18:53.791 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:18:53 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:18:53.791 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:18:53 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:18:53.791 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:18:53 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T20:18:53.792 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:18:53 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T20:18:53.792 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:18:53 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:18:54.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:18:53 smithi018 ceph-mon[184144]: Reconfiguring mon.a (monmap changed)... 2023-09-23T20:18:54.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:18:53 smithi018 ceph-mon[184144]: Reconfiguring daemon mon.a on smithi018 2023-09-23T20:18:54.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:18:53 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:18:54.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:18:53 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:18:54.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:18:53 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T20:18:54.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:18:53 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T20:18:54.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:18:53 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:18:54.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:53 smithi149 ceph-mon[170394]: Reconfiguring mon.a (monmap changed)... 2023-09-23T20:18:54.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:53 smithi149 ceph-mon[170394]: Reconfiguring daemon mon.a on smithi018 2023-09-23T20:18:54.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:53 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:18:54.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:53 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:18:54.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:53 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T20:18:54.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:53 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T20:18:54.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:53 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:18:54.792 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:18:54 smithi018 ceph-mon[179647]: pgmap v90: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:18:54.792 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:18:54 smithi018 ceph-mon[179647]: Reconfiguring mgr.y (monmap changed)... 2023-09-23T20:18:54.792 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:18:54 smithi018 ceph-mon[179647]: Reconfiguring daemon mgr.y on smithi018 2023-09-23T20:18:55.141 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:18:54 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:18:54] "GET /metrics HTTP/1.1" 200 33840 "" "Prometheus/2.43.0" 2023-09-23T20:18:55.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:18:54 smithi018 ceph-mon[184144]: pgmap v90: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:18:55.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:18:54 smithi018 ceph-mon[184144]: Reconfiguring mgr.y (monmap changed)... 2023-09-23T20:18:55.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:18:54 smithi018 ceph-mon[184144]: Reconfiguring daemon mgr.y on smithi018 2023-09-23T20:18:55.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:18:54 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:18:55.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:18:54 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:18:55.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:18:54 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T20:18:55.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:18:54 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T20:18:55.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:18:54 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:18:55.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:18:54 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:18:55.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:18:54 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:18:55.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:18:54 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T20:18:55.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:18:54 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T20:18:55.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:18:54 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:18:55.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:54 smithi149 ceph-mon[170394]: pgmap v90: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:18:55.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:54 smithi149 ceph-mon[170394]: Reconfiguring mgr.y (monmap changed)... 2023-09-23T20:18:55.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:54 smithi149 ceph-mon[170394]: Reconfiguring daemon mgr.y on smithi018 2023-09-23T20:18:55.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:54 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:18:55.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:54 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:18:55.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:54 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T20:18:55.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:54 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T20:18:55.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:54 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:18:55.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:18:55 smithi018 ceph-mon[179647]: Reconfiguring mon.c (monmap changed)... 2023-09-23T20:18:55.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:18:55 smithi018 ceph-mon[179647]: Reconfiguring daemon mon.c on smithi018 2023-09-23T20:18:55.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:18:55 smithi018 ceph-mon[184144]: Reconfiguring mon.c (monmap changed)... 2023-09-23T20:18:55.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:18:55 smithi018 ceph-mon[184144]: Reconfiguring daemon mon.c on smithi018 2023-09-23T20:18:56.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:55 smithi149 ceph-mon[170394]: Reconfiguring mon.c (monmap changed)... 2023-09-23T20:18:56.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:55 smithi149 ceph-mon[170394]: Reconfiguring daemon mon.c on smithi018 2023-09-23T20:18:57.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:18:56 smithi018 ceph-mon[179647]: pgmap v91: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:18:57.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:18:56 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:18:57.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:18:56 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:18:57.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:18:56 smithi018 ceph-mon[179647]: Reconfiguring osd.0 (monmap changed)... 2023-09-23T20:18:57.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:18:56 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-09-23T20:18:57.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:18:56 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:18:57.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:18:56 smithi018 ceph-mon[179647]: Reconfiguring daemon osd.0 on smithi018 2023-09-23T20:18:57.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:18:56 smithi018 ceph-mon[184144]: pgmap v91: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:18:57.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:18:56 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:18:57.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:18:56 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:18:57.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:18:56 smithi018 ceph-mon[184144]: Reconfiguring osd.0 (monmap changed)... 2023-09-23T20:18:57.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:18:56 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-09-23T20:18:57.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:18:56 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:18:57.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:18:56 smithi018 ceph-mon[184144]: Reconfiguring daemon osd.0 on smithi018 2023-09-23T20:18:57.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:56 smithi149 ceph-mon[170394]: pgmap v91: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:18:57.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:56 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:18:57.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:56 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:18:57.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:56 smithi149 ceph-mon[170394]: Reconfiguring osd.0 (monmap changed)... 2023-09-23T20:18:57.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:56 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-09-23T20:18:57.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:56 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:18:57.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:56 smithi149 ceph-mon[170394]: Reconfiguring daemon osd.0 on smithi018 2023-09-23T20:18:58.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:18:58 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:18:58.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:18:58 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:18:58.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:18:58 smithi018 ceph-mon[179647]: Reconfiguring osd.1 (monmap changed)... 2023-09-23T20:18:58.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:18:58 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-09-23T20:18:58.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:18:58 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:18:58.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:18:58 smithi018 ceph-mon[179647]: Reconfiguring daemon osd.1 on smithi018 2023-09-23T20:18:58.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:18:58 smithi018 ceph-mon[179647]: pgmap v92: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:18:58.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:18:58 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:18:58.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:18:58 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:18:58.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:18:58 smithi018 ceph-mon[184144]: Reconfiguring osd.1 (monmap changed)... 2023-09-23T20:18:58.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:18:58 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-09-23T20:18:58.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:18:58 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:18:58.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:18:58 smithi018 ceph-mon[184144]: Reconfiguring daemon osd.1 on smithi018 2023-09-23T20:18:58.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:18:58 smithi018 ceph-mon[184144]: pgmap v92: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:18:58.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:58 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:18:58.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:58 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:18:58.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:58 smithi149 ceph-mon[170394]: Reconfiguring osd.1 (monmap changed)... 2023-09-23T20:18:58.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:58 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-09-23T20:18:58.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:58 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:18:58.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:58 smithi149 ceph-mon[170394]: Reconfiguring daemon osd.1 on smithi018 2023-09-23T20:18:58.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:58 smithi149 ceph-mon[170394]: pgmap v92: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:18:58.966 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:18:58 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[156506]: ts=2023-09-23T20:18:58.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.18\", device_class=\"ssd\", hostname=\"smithi018\", instance=\"smithi149.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.18\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.18\", device_class=\"ssd\", hostname=\"smithi018\", instance=\"smithi018:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.18\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T20:18:59.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:18:59 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:18:59.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:18:59 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:18:59.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:18:59 smithi018 ceph-mon[184144]: Reconfiguring osd.2 (monmap changed)... 2023-09-23T20:18:59.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:18:59 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-09-23T20:18:59.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:18:59 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:18:59.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:18:59 smithi018 ceph-mon[184144]: Reconfiguring daemon osd.2 on smithi018 2023-09-23T20:18:59.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:18:59 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:18:59.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:18:59 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:18:59.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:18:59 smithi018 ceph-mon[179647]: Reconfiguring osd.2 (monmap changed)... 2023-09-23T20:18:59.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:18:59 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-09-23T20:18:59.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:18:59 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:18:59.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:18:59 smithi018 ceph-mon[179647]: Reconfiguring daemon osd.2 on smithi018 2023-09-23T20:18:59.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:59 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:18:59.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:59 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:18:59.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:59 smithi149 ceph-mon[170394]: Reconfiguring osd.2 (monmap changed)... 2023-09-23T20:18:59.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:59 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-09-23T20:18:59.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:59 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:18:59.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:18:59 smithi149 ceph-mon[170394]: Reconfiguring daemon osd.2 on smithi018 2023-09-23T20:19:00.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:19:00 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:19:00.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:19:00 smithi018 ceph-mon[184144]: pgmap v93: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:19:00.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:19:00 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:19:00.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:19:00 smithi018 ceph-mon[184144]: Reconfiguring osd.3 (monmap changed)... 2023-09-23T20:19:00.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:19:00 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-09-23T20:19:00.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:19:00 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:19:00.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:19:00 smithi018 ceph-mon[184144]: Reconfiguring daemon osd.3 on smithi018 2023-09-23T20:19:00.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:19:00 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:19:00.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:19:00 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:19:00.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:19:00 smithi018 ceph-mon[179647]: pgmap v93: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:19:00.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:19:00 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:19:00.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:19:00 smithi018 ceph-mon[179647]: Reconfiguring osd.3 (monmap changed)... 2023-09-23T20:19:00.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:19:00 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-09-23T20:19:00.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:19:00 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:19:00.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:19:00 smithi018 ceph-mon[179647]: Reconfiguring daemon osd.3 on smithi018 2023-09-23T20:19:00.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:19:00 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:19:00.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:19:00 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:19:00.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:19:00 smithi149 ceph-mon[170394]: pgmap v93: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:19:00.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:19:00 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:19:00.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:19:00 smithi149 ceph-mon[170394]: Reconfiguring osd.3 (monmap changed)... 2023-09-23T20:19:00.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:19:00 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-09-23T20:19:00.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:19:00 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:19:00.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:19:00 smithi149 ceph-mon[170394]: Reconfiguring daemon osd.3 on smithi018 2023-09-23T20:19:00.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:19:00 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:19:01.617 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:19:01 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:19:01.617 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:19:01 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:19:01.617 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:19:01 smithi018 ceph-mon[184144]: Reconfiguring mds.foo.smithi018.hduiol (monmap changed)... 2023-09-23T20:19:01.617 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:19:01 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi018.hduiol", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T20:19:01.617 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:19:01 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:19:01.617 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:19:01 smithi018 ceph-mon[184144]: Reconfiguring daemon mds.foo.smithi018.hduiol on smithi018 2023-09-23T20:19:01.618 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:19:01 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:19:01.618 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:19:01 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:19:01.618 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:19:01 smithi018 ceph-mon[179647]: Reconfiguring mds.foo.smithi018.hduiol (monmap changed)... 2023-09-23T20:19:01.618 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:19:01 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi018.hduiol", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T20:19:01.618 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:19:01 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:19:01.618 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:19:01 smithi018 ceph-mon[179647]: Reconfiguring daemon mds.foo.smithi018.hduiol on smithi018 2023-09-23T20:19:01.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:19:01 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:19:01.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:19:01 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:19:01.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:19:01 smithi149 ceph-mon[170394]: Reconfiguring mds.foo.smithi018.hduiol (monmap changed)... 2023-09-23T20:19:01.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:19:01 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi018.hduiol", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T20:19:01.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:19:01 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:19:01.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:19:01 smithi149 ceph-mon[170394]: Reconfiguring daemon mds.foo.smithi018.hduiol on smithi018 2023-09-23T20:19:02.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:19:02 smithi018 ceph-mon[184144]: pgmap v94: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:19:02.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:19:02 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:19:02.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:19:02 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:19:02.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:19:02 smithi018 ceph-mon[184144]: Reconfiguring mon.b (monmap changed)... 2023-09-23T20:19:02.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:19:02 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T20:19:02.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:19:02 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T20:19:02.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:19:02 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:19:02.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:19:02 smithi018 ceph-mon[184144]: Reconfiguring daemon mon.b on smithi149 2023-09-23T20:19:02.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:19:02 smithi018 ceph-mon[179647]: pgmap v94: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:19:02.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:19:02 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:19:02.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:19:02 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:19:02.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:19:02 smithi018 ceph-mon[179647]: Reconfiguring mon.b (monmap changed)... 2023-09-23T20:19:02.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:19:02 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T20:19:02.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:19:02 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T20:19:02.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:19:02 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:19:02.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:19:02 smithi018 ceph-mon[179647]: Reconfiguring daemon mon.b on smithi149 2023-09-23T20:19:02.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:19:02 smithi149 ceph-mon[170394]: pgmap v94: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:19:02.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:19:02 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:19:02.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:19:02 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:19:02.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:19:02 smithi149 ceph-mon[170394]: Reconfiguring mon.b (monmap changed)... 2023-09-23T20:19:02.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:19:02 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T20:19:02.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:19:02 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T20:19:02.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:19:02 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:19:02.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:19:02 smithi149 ceph-mon[170394]: Reconfiguring daemon mon.b on smithi149 2023-09-23T20:19:03.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:19:03 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:19:03.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:19:03 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:19:03.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:19:03 smithi149 ceph-mon[170394]: Reconfiguring mgr.x (monmap changed)... 2023-09-23T20:19:03.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:19:03 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T20:19:03.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:19:03 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T20:19:03.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:19:03 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:19:03.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:19:03 smithi149 ceph-mon[170394]: Reconfiguring daemon mgr.x on smithi149 2023-09-23T20:19:04.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:19:03 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:19:04.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:19:03 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:19:04.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:19:03 smithi018 ceph-mon[184144]: Reconfiguring mgr.x (monmap changed)... 2023-09-23T20:19:04.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:19:03 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T20:19:04.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:19:03 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T20:19:04.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:19:03 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:19:04.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:19:03 smithi018 ceph-mon[184144]: Reconfiguring daemon mgr.x on smithi149 2023-09-23T20:19:04.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:19:03 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:19:04.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:19:03 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:19:04.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:19:03 smithi018 ceph-mon[179647]: Reconfiguring mgr.x (monmap changed)... 2023-09-23T20:19:04.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:19:03 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T20:19:04.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:19:03 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T20:19:04.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:19:03 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:19:04.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:19:03 smithi018 ceph-mon[179647]: Reconfiguring daemon mgr.x on smithi149 2023-09-23T20:19:05.127 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:19:04 smithi149 ceph-mon[170394]: pgmap v95: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:19:05.128 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:19:04 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:19:05.128 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:19:04 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:19:05.128 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:19:04 smithi149 ceph-mon[170394]: Reconfiguring osd.4 (monmap changed)... 2023-09-23T20:19:05.128 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:19:04 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-09-23T20:19:05.128 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:19:04 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:19:05.128 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:19:04 smithi149 ceph-mon[170394]: Reconfiguring daemon osd.4 on smithi149 2023-09-23T20:19:05.391 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:19:04 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:19:04] "GET /metrics HTTP/1.1" 200 33906 "" "Prometheus/2.43.0" 2023-09-23T20:19:05.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:19:04 smithi018 ceph-mon[179647]: pgmap v95: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:19:05.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:19:04 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:19:05.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:19:04 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:19:05.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:19:04 smithi018 ceph-mon[179647]: Reconfiguring osd.4 (monmap changed)... 2023-09-23T20:19:05.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:19:04 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-09-23T20:19:05.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:19:04 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:19:05.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:19:04 smithi018 ceph-mon[179647]: Reconfiguring daemon osd.4 on smithi149 2023-09-23T20:19:05.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:19:04 smithi018 ceph-mon[184144]: pgmap v95: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:19:05.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:19:04 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:19:05.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:19:04 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:19:05.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:19:04 smithi018 ceph-mon[184144]: Reconfiguring osd.4 (monmap changed)... 2023-09-23T20:19:05.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:19:04 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-09-23T20:19:05.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:19:04 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:19:05.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:19:04 smithi018 ceph-mon[184144]: Reconfiguring daemon osd.4 on smithi149 2023-09-23T20:19:06.379 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:19:06 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:19:06.380 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:19:06 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:19:06.380 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:19:06 smithi149 ceph-mon[170394]: Reconfiguring osd.5 (monmap changed)... 2023-09-23T20:19:06.380 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:19:06 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-09-23T20:19:06.380 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:19:06 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:19:06.380 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:19:06 smithi149 ceph-mon[170394]: Reconfiguring daemon osd.5 on smithi149 2023-09-23T20:19:06.380 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:19:06 smithi149 ceph-mon[170394]: pgmap v96: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:19:06.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:19:06 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:19:06.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:19:06 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:19:06.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:19:06 smithi018 ceph-mon[184144]: Reconfiguring osd.5 (monmap changed)... 2023-09-23T20:19:06.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:19:06 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-09-23T20:19:06.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:19:06 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:19:06.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:19:06 smithi018 ceph-mon[184144]: Reconfiguring daemon osd.5 on smithi149 2023-09-23T20:19:06.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:19:06 smithi018 ceph-mon[184144]: pgmap v96: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:19:06.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:19:06 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:19:06.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:19:06 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:19:06.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:19:06 smithi018 ceph-mon[179647]: Reconfiguring osd.5 (monmap changed)... 2023-09-23T20:19:06.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:19:06 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-09-23T20:19:06.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:19:06 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:19:06.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:19:06 smithi018 ceph-mon[179647]: Reconfiguring daemon osd.5 on smithi149 2023-09-23T20:19:06.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:19:06 smithi018 ceph-mon[179647]: pgmap v96: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:19:07.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:19:07 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:19:07.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:19:07 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:19:07.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:19:07 smithi149 ceph-mon[170394]: Reconfiguring osd.6 (monmap changed)... 2023-09-23T20:19:07.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:19:07 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-09-23T20:19:07.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:19:07 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:19:07.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:19:07 smithi149 ceph-mon[170394]: Reconfiguring daemon osd.6 on smithi149 2023-09-23T20:19:07.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:19:07 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:19:07.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:19:07 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:19:07.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:19:07 smithi018 ceph-mon[184144]: Reconfiguring osd.6 (monmap changed)... 2023-09-23T20:19:07.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:19:07 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-09-23T20:19:07.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:19:07 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:19:07.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:19:07 smithi018 ceph-mon[184144]: Reconfiguring daemon osd.6 on smithi149 2023-09-23T20:19:07.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:19:07 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:19:07.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:19:07 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:19:07.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:19:07 smithi018 ceph-mon[179647]: Reconfiguring osd.6 (monmap changed)... 2023-09-23T20:19:07.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:19:07 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-09-23T20:19:07.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:19:07 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:19:07.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:19:07 smithi018 ceph-mon[179647]: Reconfiguring daemon osd.6 on smithi149 2023-09-23T20:19:08.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:19:08 smithi018 ceph-mon[184144]: pgmap v97: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:19:08.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:19:08 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:19:08.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:19:08 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:19:08.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:19:08 smithi018 ceph-mon[184144]: Reconfiguring osd.7 (monmap changed)... 2023-09-23T20:19:08.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:19:08 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-09-23T20:19:08.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:19:08 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:19:08.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:19:08 smithi018 ceph-mon[184144]: Reconfiguring daemon osd.7 on smithi149 2023-09-23T20:19:08.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:19:08 smithi018 ceph-mon[179647]: pgmap v97: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:19:08.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:19:08 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:19:08.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:19:08 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:19:08.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:19:08 smithi018 ceph-mon[179647]: Reconfiguring osd.7 (monmap changed)... 2023-09-23T20:19:08.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:19:08 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-09-23T20:19:08.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:19:08 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:19:08.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:19:08 smithi018 ceph-mon[179647]: Reconfiguring daemon osd.7 on smithi149 2023-09-23T20:19:08.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:19:08 smithi149 ceph-mon[170394]: pgmap v97: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:19:08.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:19:08 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:19:08.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:19:08 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:19:08.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:19:08 smithi149 ceph-mon[170394]: Reconfiguring osd.7 (monmap changed)... 2023-09-23T20:19:08.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:19:08 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-09-23T20:19:08.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:19:08 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:19:08.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:19:08 smithi149 ceph-mon[170394]: Reconfiguring daemon osd.7 on smithi149 2023-09-23T20:19:08.967 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:19:08 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[156506]: ts=2023-09-23T20:19:08.658Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.18\", device_class=\"ssd\", hostname=\"smithi018\", instance=\"smithi149.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.18\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.18\", device_class=\"ssd\", hostname=\"smithi018\", instance=\"smithi018:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.18\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T20:19:10.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:19:09 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:19:10.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:19:09 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:19:10.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:19:09 smithi149 ceph-mon[170394]: Reconfiguring mds.foo.smithi149.urtezf (monmap changed)... 2023-09-23T20:19:10.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:19:09 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi149.urtezf", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T20:19:10.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:19:09 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:19:10.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:19:09 smithi149 ceph-mon[170394]: Reconfiguring daemon mds.foo.smithi149.urtezf on smithi149 2023-09-23T20:19:10.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:19:09 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:19:10.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:19:09 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:19:10.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:19:09 smithi018 ceph-mon[184144]: Reconfiguring mds.foo.smithi149.urtezf (monmap changed)... 2023-09-23T20:19:10.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:19:09 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi149.urtezf", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T20:19:10.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:19:09 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:19:10.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:19:09 smithi018 ceph-mon[184144]: Reconfiguring daemon mds.foo.smithi149.urtezf on smithi149 2023-09-23T20:19:10.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:19:09 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:19:10.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:19:09 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:19:10.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:19:09 smithi018 ceph-mon[179647]: Reconfiguring mds.foo.smithi149.urtezf (monmap changed)... 2023-09-23T20:19:10.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:19:09 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi149.urtezf", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T20:19:10.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:19:09 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:19:10.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:19:09 smithi018 ceph-mon[179647]: Reconfiguring daemon mds.foo.smithi149.urtezf on smithi149 2023-09-23T20:19:11.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:19:11 smithi018 ceph-mon[184144]: pgmap v98: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:19:11.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:19:11 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:19:11.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:19:11 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:19:11.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:19:11 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:19:11.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:19:11 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:19:11.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:19:11 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:19:11.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:19:11 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:19:11.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:19:11 smithi018 ceph-mon[179647]: pgmap v98: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:19:11.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:19:11 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:19:11.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:19:11 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:19:11.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:19:11 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:19:11.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:19:11 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:19:11.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:19:11 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:19:11.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:19:11 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:19:11.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:19:11 smithi149 ceph-mon[170394]: pgmap v98: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:19:11.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:19:11 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:19:11.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:19:11 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:19:11.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:19:11 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:19:11.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:19:11 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:19:11.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:19:11 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:19:11.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:19:11 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:19:13.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:19:13 smithi018 ceph-mon[179647]: pgmap v99: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:19:13.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:19:13 smithi018 ceph-mon[184144]: pgmap v99: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:19:13.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:19:13 smithi149 ceph-mon[170394]: pgmap v99: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:19:15.391 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:19:14 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:19:14] "GET /metrics HTTP/1.1" 200 33907 "" "Prometheus/2.43.0" 2023-09-23T20:19:15.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:19:15 smithi018 ceph-mon[184144]: pgmap v100: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:19:15.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:19:15 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:19:15.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:19:15 smithi018 ceph-mon[179647]: pgmap v100: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:19:15.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:19:15 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:19:15.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:19:15 smithi149 ceph-mon[170394]: pgmap v100: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:19:15.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:19:15 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:19:17.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:19:17 smithi018 ceph-mon[184144]: pgmap v101: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:19:17.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:19:17 smithi018 ceph-mon[179647]: pgmap v101: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:19:17.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:19:17 smithi149 ceph-mon[170394]: pgmap v101: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:19:18.966 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:19:18 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[156506]: ts=2023-09-23T20:19:18.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.18\", device_class=\"ssd\", hostname=\"smithi018\", instance=\"smithi149.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.18\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.18\", device_class=\"ssd\", hostname=\"smithi018\", instance=\"smithi018:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.18\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T20:19:19.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:19:19 smithi018 ceph-mon[179647]: pgmap v102: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:19:19.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:19:19 smithi018 ceph-mon[184144]: pgmap v102: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:19:19.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:19:19 smithi149 ceph-mon[170394]: pgmap v102: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:19:21.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:19:21 smithi018 ceph-mon[184144]: pgmap v103: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:19:21.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:19:21 smithi018 ceph-mon[179647]: pgmap v103: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:19:21.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:19:21 smithi149 ceph-mon[170394]: pgmap v103: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:19:23.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:19:23 smithi018 ceph-mon[184144]: pgmap v104: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:19:23.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:19:23 smithi018 ceph-mon[179647]: pgmap v104: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:19:23.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:19:23 smithi149 ceph-mon[170394]: pgmap v104: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:19:25.391 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:19:24 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:19:24] "GET /metrics HTTP/1.1" 200 33907 "" "Prometheus/2.43.0" 2023-09-23T20:19:25.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:19:25 smithi018 ceph-mon[184144]: pgmap v105: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:19:25.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:19:25 smithi018 ceph-mon[179647]: pgmap v105: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:19:25.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:19:25 smithi149 ceph-mon[170394]: pgmap v105: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:19:27.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:19:27 smithi018 ceph-mon[184144]: pgmap v106: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:19:27.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:19:27 smithi018 ceph-mon[179647]: pgmap v106: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:19:27.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:19:27 smithi149 ceph-mon[170394]: pgmap v106: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:19:28.966 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:19:28 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[156506]: ts=2023-09-23T20:19:28.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.18\", device_class=\"ssd\", hostname=\"smithi018\", instance=\"smithi149.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.18\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.18\", device_class=\"ssd\", hostname=\"smithi018\", instance=\"smithi018:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.18\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T20:19:29.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:19:29 smithi018 ceph-mon[179647]: pgmap v107: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:19:29.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:19:29 smithi018 ceph-mon[184144]: pgmap v107: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:19:29.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:19:29 smithi149 ceph-mon[170394]: pgmap v107: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:19:30.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:19:30 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:19:30.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:19:30 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:19:30.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:19:30 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:19:31.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:19:31 smithi018 ceph-mon[184144]: pgmap v108: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:19:31.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:19:31 smithi018 ceph-mon[179647]: pgmap v108: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:19:31.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:19:31 smithi149 ceph-mon[170394]: pgmap v108: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:19:33.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:19:33 smithi018 ceph-mon[184144]: pgmap v109: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:19:33.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:19:33 smithi018 ceph-mon[179647]: pgmap v109: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:19:33.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:19:33 smithi149 ceph-mon[170394]: pgmap v109: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:19:35.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:19:35 smithi018 ceph-mon[184144]: pgmap v110: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:19:35.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:19:35 smithi018 ceph-mon[179647]: pgmap v110: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:19:35.391 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:19:34 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:19:34] "GET /metrics HTTP/1.1" 200 33909 "" "Prometheus/2.43.0" 2023-09-23T20:19:35.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:19:35 smithi149 ceph-mon[170394]: pgmap v110: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:19:37.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:19:36 smithi018 ceph-mon[184144]: pgmap v111: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:19:37.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:19:36 smithi018 ceph-mon[179647]: pgmap v111: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:19:37.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:19:36 smithi149 ceph-mon[170394]: pgmap v111: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:19:38.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:19:38 smithi149 ceph-mon[170394]: pgmap v112: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:19:38.966 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:19:38 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[156506]: ts=2023-09-23T20:19:38.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.18\", device_class=\"ssd\", hostname=\"smithi018\", instance=\"smithi149.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.18\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.18\", device_class=\"ssd\", hostname=\"smithi018\", instance=\"smithi018:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.18\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T20:19:39.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:19:38 smithi018 ceph-mon[184144]: pgmap v112: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:19:39.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:19:38 smithi018 ceph-mon[179647]: pgmap v112: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:19:41.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:19:40 smithi018 ceph-mon[184144]: pgmap v113: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:19:41.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:19:40 smithi018 ceph-mon[179647]: pgmap v113: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:19:41.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:19:40 smithi149 ceph-mon[170394]: pgmap v113: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:19:43.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:19:42 smithi018 ceph-mon[179647]: pgmap v114: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:19:43.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:19:42 smithi018 ceph-mon[184144]: pgmap v114: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:19:43.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:19:42 smithi149 ceph-mon[170394]: pgmap v114: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:19:45.141 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:19:44 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:19:44] "GET /metrics HTTP/1.1" 200 33907 "" "Prometheus/2.43.0" 2023-09-23T20:19:45.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:19:44 smithi018 ceph-mon[179647]: pgmap v115: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:19:45.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:19:44 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:19:45.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:19:44 smithi018 ceph-mon[184144]: pgmap v115: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:19:45.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:19:44 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:19:45.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:19:44 smithi149 ceph-mon[170394]: pgmap v115: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:19:45.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:19:44 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:19:47.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:19:46 smithi018 ceph-mon[184144]: pgmap v116: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:19:47.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:19:46 smithi018 ceph-mon[179647]: pgmap v116: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:19:47.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:19:46 smithi149 ceph-mon[170394]: pgmap v116: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:19:48.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:19:48 smithi149 ceph-mon[170394]: pgmap v117: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:19:48.966 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:19:48 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[156506]: ts=2023-09-23T20:19:48.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.18\", device_class=\"ssd\", hostname=\"smithi018\", instance=\"smithi149.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.18\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.18\", device_class=\"ssd\", hostname=\"smithi018\", instance=\"smithi018:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.18\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T20:19:49.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:19:48 smithi018 ceph-mon[184144]: pgmap v117: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:19:49.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:19:48 smithi018 ceph-mon[179647]: pgmap v117: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:19:51.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:19:50 smithi018 ceph-mon[184144]: pgmap v118: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:19:51.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:19:50 smithi018 ceph-mon[179647]: pgmap v118: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:19:51.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:19:50 smithi149 ceph-mon[170394]: pgmap v118: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:19:52.054 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch upgrade resume' 2023-09-23T20:19:52.663 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:19:53.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:19:52 smithi018 ceph-mon[184144]: pgmap v119: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:19:53.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:19:52 smithi018 ceph-mon[179647]: pgmap v119: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:19:53.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:19:52 smithi149 ceph-mon[170394]: pgmap v119: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:19:54.080 INFO:teuthology.orchestra.run.smithi018.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T20:19:54.515 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'echo wait for half of osds to upgrade' 2023-09-23T20:19:55.122 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:19:55.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:19:55 smithi018 ceph-mon[179647]: pgmap v120: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:19:55.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:19:55 smithi018 ceph-mon[179647]: from='client.44144 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:19:55.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:19:55 smithi018 ceph-mon[179647]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T20:19:55.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:19:55 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:19:55.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:19:55 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:19:55.392 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:19:54 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:19:54] "GET /metrics HTTP/1.1" 200 33907 "" "Prometheus/2.43.0" 2023-09-23T20:19:55.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:19:55 smithi018 ceph-mon[184144]: pgmap v120: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:19:55.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:19:55 smithi018 ceph-mon[184144]: from='client.44144 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:19:55.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:19:55 smithi018 ceph-mon[184144]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T20:19:55.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:19:55 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:19:55.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:19:55 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:19:55.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:19:55 smithi149 ceph-mon[170394]: pgmap v120: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:19:55.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:19:55 smithi149 ceph-mon[170394]: from='client.44144 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:19:55.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:19:55 smithi149 ceph-mon[170394]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T20:19:55.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:19:55 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:19:55.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:19:55 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:19:56.217 INFO:teuthology.orchestra.run.smithi018.stdout:wait for half of osds to upgrade 2023-09-23T20:19:56.669 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'while ! ceph osd versions | grep $sha1 | egrep '"'"': [45678]'"'"'; do sleep 2 ; done' 2023-09-23T20:19:57.301 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:19:57.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:19:57 smithi018 ceph-mon[184144]: pgmap v121: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:19:57.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:19:57 smithi018 ceph-mon[179647]: pgmap v121: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:19:57.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:19:57 smithi149 ceph-mon[170394]: pgmap v121: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:19:59.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:19:59 smithi018 ceph-mon[184144]: pgmap v122: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:19:59.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:19:59 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/936865384' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:19:59.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:19:59 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:19:59.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:19:59 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:19:59.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:19:59 smithi018 ceph-mon[179647]: pgmap v122: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:19:59.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:19:59 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/936865384' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:19:59.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:19:59 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:19:59.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:19:59 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:19:59.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:19:59 smithi149 ceph-mon[170394]: pgmap v122: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:19:59.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:19:59 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/936865384' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:19:59.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:19:59 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:19:59.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:19:59 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:20:00.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:00 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:20:00.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:00 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:20:00.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:00 smithi018 ceph-mon[184144]: pgmap v123: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:20:00.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:00 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:20:00.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:00 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:20:00.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:00 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:20:00.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:00 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:20:00.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:00 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:20:00.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:00 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:20:00.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:00 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:20:00.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:00 smithi018 ceph-mon[184144]: Upgrade: Setting container_image for all mon 2023-09-23T20:20:00.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:00 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:20:00.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:00 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2023-09-23T20:20:00.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:00 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2023-09-23T20:20:00.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:00 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2023-09-23T20:20:00.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:00 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2023-09-23T20:20:00.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:00 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2023-09-23T20:20:00.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:00 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2023-09-23T20:20:00.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:00 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:20:00.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:00 smithi018 ceph-mon[184144]: Upgrade: Setting container_image for all crash 2023-09-23T20:20:00.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:00 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:20:00.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:00 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-09-23T20:20:00.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:00 smithi018 ceph-mon[184144]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-09-23T20:20:00.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:00 smithi018 ceph-mon[184144]: Upgrade: osd.0 is safe to restart 2023-09-23T20:20:00.394 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:00 smithi018 ceph-mon[184144]: overall HEALTH_OK 2023-09-23T20:20:00.394 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:00 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:20:00.394 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:00 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:20:00.394 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:00 smithi018 ceph-mon[179647]: pgmap v123: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:20:00.394 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:00 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:20:00.394 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:00 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:20:00.394 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:00 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:20:00.394 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:00 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:20:00.394 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:00 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:20:00.394 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:00 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:20:00.395 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:00 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:20:00.395 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:00 smithi018 ceph-mon[179647]: Upgrade: Setting container_image for all mon 2023-09-23T20:20:00.395 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:00 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:20:00.395 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:00 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2023-09-23T20:20:00.395 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:00 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2023-09-23T20:20:00.395 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:00 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2023-09-23T20:20:00.395 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:00 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2023-09-23T20:20:00.395 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:00 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2023-09-23T20:20:00.395 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:00 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2023-09-23T20:20:00.395 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:00 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:20:00.395 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:00 smithi018 ceph-mon[179647]: Upgrade: Setting container_image for all crash 2023-09-23T20:20:00.396 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:00 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:20:00.396 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:00 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-09-23T20:20:00.396 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:00 smithi018 ceph-mon[179647]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-09-23T20:20:00.396 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:00 smithi018 ceph-mon[179647]: Upgrade: osd.0 is safe to restart 2023-09-23T20:20:00.396 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:00 smithi018 ceph-mon[179647]: overall HEALTH_OK 2023-09-23T20:20:00.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:00 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:20:00.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:00 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:20:00.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:00 smithi149 ceph-mon[170394]: pgmap v123: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:20:00.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:00 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:20:00.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:00 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:20:00.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:00 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:20:00.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:00 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:20:00.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:00 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:20:00.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:00 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:20:00.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:00 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:20:00.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:00 smithi149 ceph-mon[170394]: Upgrade: Setting container_image for all mon 2023-09-23T20:20:00.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:00 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:20:00.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:00 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2023-09-23T20:20:00.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:00 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2023-09-23T20:20:00.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:00 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2023-09-23T20:20:00.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:00 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2023-09-23T20:20:00.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:00 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2023-09-23T20:20:00.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:00 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2023-09-23T20:20:00.468 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:00 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:20:00.468 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:00 smithi149 ceph-mon[170394]: Upgrade: Setting container_image for all crash 2023-09-23T20:20:00.468 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:00 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:20:00.468 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:00 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-09-23T20:20:00.468 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:00 smithi149 ceph-mon[170394]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-09-23T20:20:00.468 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:00 smithi149 ceph-mon[170394]: Upgrade: osd.0 is safe to restart 2023-09-23T20:20:00.468 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:00 smithi149 ceph-mon[170394]: overall HEALTH_OK 2023-09-23T20:20:02.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:02 smithi018 ceph-mon[184144]: Upgrade: Updating osd.0 2023-09-23T20:20:02.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:02 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:20:02.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:02 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-09-23T20:20:02.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:02 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:20:02.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:02 smithi018 ceph-mon[184144]: Deploying daemon osd.0 on smithi018 2023-09-23T20:20:02.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:02 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1828148481' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:20:02.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:02 smithi018 ceph-mon[184144]: pgmap v124: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:20:02.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:02 smithi018 ceph-mon[179647]: Upgrade: Updating osd.0 2023-09-23T20:20:02.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:02 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:20:02.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:02 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-09-23T20:20:02.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:02 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:20:02.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:02 smithi018 ceph-mon[179647]: Deploying daemon osd.0 on smithi018 2023-09-23T20:20:02.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:02 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1828148481' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:20:02.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:02 smithi018 ceph-mon[179647]: pgmap v124: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:20:02.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:02 smithi149 ceph-mon[170394]: Upgrade: Updating osd.0 2023-09-23T20:20:02.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:02 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:20:02.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:02 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-09-23T20:20:02.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:02 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:20:02.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:02 smithi149 ceph-mon[170394]: Deploying daemon osd.0 on smithi018 2023-09-23T20:20:02.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:02 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1828148481' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:20:02.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:02 smithi149 ceph-mon[170394]: pgmap v124: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:20:03.765 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 23 20:20:03 smithi018 systemd[1]: Stopping Ceph osd.0 for a3b3b17e-5a4c-11ee-8db4-212e2dc638e7... 2023-09-23T20:20:04.141 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 23 20:20:03 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-0[113592]: 2023-09-23T20:20:03.763+0000 7f07a5f49700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T20:20:04.141 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 23 20:20:03 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-0[113592]: 2023-09-23T20:20:03.763+0000 7f07a5f49700 -1 osd.0 90 *** Got signal Terminated *** 2023-09-23T20:20:04.141 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 23 20:20:03 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-0[113592]: 2023-09-23T20:20:03.763+0000 7f07a5f49700 -1 osd.0 90 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T20:20:04.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:04 smithi149 ceph-mon[170394]: pgmap v125: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:20:04.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:04 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2872736366' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:20:04.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:04 smithi149 ceph-mon[170394]: osd.0 marked itself down and dead 2023-09-23T20:20:04.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:04 smithi018 ceph-mon[179647]: pgmap v125: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:20:04.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:04 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2872736366' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:20:04.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:04 smithi018 ceph-mon[179647]: osd.0 marked itself down and dead 2023-09-23T20:20:04.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:04 smithi018 ceph-mon[184144]: pgmap v125: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:20:04.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:04 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2872736366' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:20:04.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:04 smithi018 ceph-mon[184144]: osd.0 marked itself down and dead 2023-09-23T20:20:05.391 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 23 20:20:05 smithi018 bash[194313]: ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-0 2023-09-23T20:20:05.391 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:20:04 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:20:04] "GET /metrics HTTP/1.1" 200 33908 "" "Prometheus/2.43.0" 2023-09-23T20:20:05.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:05 smithi149 ceph-mon[170394]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T20:20:05.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:05 smithi149 ceph-mon[170394]: failed to encode map e91 with expected crc 2023-09-23T20:20:05.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:05 smithi149 ceph-mon[170394]: failed to encode map e91 with expected crc 2023-09-23T20:20:05.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:05 smithi149 ceph-mon[170394]: osdmap e91: 8 total, 7 up, 8 in 2023-09-23T20:20:05.861 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:05 smithi018 ceph-mon[184144]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T20:20:05.862 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:05 smithi018 ceph-mon[184144]: failed to encode map e91 with expected crc 2023-09-23T20:20:05.862 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:05 smithi018 ceph-mon[184144]: failed to encode map e91 with expected crc 2023-09-23T20:20:05.862 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:05 smithi018 ceph-mon[184144]: osdmap e91: 8 total, 7 up, 8 in 2023-09-23T20:20:05.862 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:05 smithi018 ceph-mon[179647]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T20:20:05.862 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:05 smithi018 ceph-mon[179647]: failed to encode map e91 with expected crc 2023-09-23T20:20:05.862 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:05 smithi018 ceph-mon[179647]: failed to encode map e91 with expected crc 2023-09-23T20:20:05.862 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:05 smithi018 ceph-mon[179647]: osdmap e91: 8 total, 7 up, 8 in 2023-09-23T20:20:06.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:06 smithi018 ceph-mon[184144]: pgmap v127: 73 pgs: 26 peering, 47 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:20:06.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:06 smithi018 ceph-mon[184144]: Health check failed: Reduced data availability: 9 pgs inactive, 26 pgs peering (PG_AVAILABILITY) 2023-09-23T20:20:06.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:06 smithi018 ceph-mon[184144]: failed to encode map e92 with expected crc 2023-09-23T20:20:06.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:06 smithi018 ceph-mon[184144]: failed to encode map e92 with expected crc 2023-09-23T20:20:06.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:06 smithi018 ceph-mon[184144]: osdmap e92: 8 total, 7 up, 8 in 2023-09-23T20:20:06.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:06 smithi018 ceph-mon[184144]: failed to encode map e92 with expected crc 2023-09-23T20:20:06.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:06 smithi018 ceph-mon[184144]: failed to encode map e92 with expected crc 2023-09-23T20:20:06.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:06 smithi018 ceph-mon[184144]: failed to encode map e92 with expected crc 2023-09-23T20:20:06.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:06 smithi018 ceph-mon[184144]: failed to encode map e92 with expected crc 2023-09-23T20:20:06.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:06 smithi018 ceph-mon[184144]: failed to encode map e92 with expected crc 2023-09-23T20:20:06.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:06 smithi018 ceph-mon[184144]: failed to encode map e92 with expected crc 2023-09-23T20:20:06.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:06 smithi018 ceph-mon[184144]: failed to encode map e92 with expected crc 2023-09-23T20:20:06.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:06 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2523358659' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:20:06.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:06 smithi018 ceph-mon[179647]: pgmap v127: 73 pgs: 26 peering, 47 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:20:06.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:06 smithi018 ceph-mon[179647]: Health check failed: Reduced data availability: 9 pgs inactive, 26 pgs peering (PG_AVAILABILITY) 2023-09-23T20:20:06.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:06 smithi018 ceph-mon[179647]: failed to encode map e92 with expected crc 2023-09-23T20:20:06.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:06 smithi018 ceph-mon[179647]: failed to encode map e92 with expected crc 2023-09-23T20:20:06.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:06 smithi018 ceph-mon[179647]: osdmap e92: 8 total, 7 up, 8 in 2023-09-23T20:20:06.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:06 smithi018 ceph-mon[179647]: failed to encode map e92 with expected crc 2023-09-23T20:20:06.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:06 smithi018 ceph-mon[179647]: failed to encode map e92 with expected crc 2023-09-23T20:20:06.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:06 smithi018 ceph-mon[179647]: failed to encode map e92 with expected crc 2023-09-23T20:20:06.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:06 smithi018 ceph-mon[179647]: failed to encode map e92 with expected crc 2023-09-23T20:20:06.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:06 smithi018 ceph-mon[179647]: failed to encode map e92 with expected crc 2023-09-23T20:20:06.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:06 smithi018 ceph-mon[179647]: failed to encode map e92 with expected crc 2023-09-23T20:20:06.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:06 smithi018 ceph-mon[179647]: failed to encode map e92 with expected crc 2023-09-23T20:20:06.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:06 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2523358659' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:20:06.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:06 smithi149 ceph-mon[170394]: pgmap v127: 73 pgs: 26 peering, 47 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:20:06.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:06 smithi149 ceph-mon[170394]: Health check failed: Reduced data availability: 9 pgs inactive, 26 pgs peering (PG_AVAILABILITY) 2023-09-23T20:20:06.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:06 smithi149 ceph-mon[170394]: failed to encode map e92 with expected crc 2023-09-23T20:20:06.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:06 smithi149 ceph-mon[170394]: failed to encode map e92 with expected crc 2023-09-23T20:20:06.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:06 smithi149 ceph-mon[170394]: osdmap e92: 8 total, 7 up, 8 in 2023-09-23T20:20:06.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:06 smithi149 ceph-mon[170394]: failed to encode map e92 with expected crc 2023-09-23T20:20:06.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:06 smithi149 ceph-mon[170394]: failed to encode map e92 with expected crc 2023-09-23T20:20:06.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:06 smithi149 ceph-mon[170394]: failed to encode map e92 with expected crc 2023-09-23T20:20:06.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:06 smithi149 ceph-mon[170394]: failed to encode map e92 with expected crc 2023-09-23T20:20:06.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:06 smithi149 ceph-mon[170394]: failed to encode map e92 with expected crc 2023-09-23T20:20:06.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:06 smithi149 ceph-mon[170394]: failed to encode map e92 with expected crc 2023-09-23T20:20:06.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:06 smithi149 ceph-mon[170394]: failed to encode map e92 with expected crc 2023-09-23T20:20:06.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:06 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2523358659' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:20:07.141 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 23 20:20:06 smithi018 systemd[1]: ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7@osd.0.service: Succeeded. 2023-09-23T20:20:07.141 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 23 20:20:06 smithi018 systemd[1]: Stopped Ceph osd.0 for a3b3b17e-5a4c-11ee-8db4-212e2dc638e7. 2023-09-23T20:20:07.141 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 23 20:20:06 smithi018 systemd[1]: Starting Ceph osd.0 for a3b3b17e-5a4c-11ee-8db4-212e2dc638e7... 2023-09-23T20:20:08.141 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 23 20:20:07 smithi018 bash[194615]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-09-23T20:20:08.141 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 23 20:20:07 smithi018 bash[194615]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2023-09-23T20:20:08.141 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 23 20:20:07 smithi018 bash[194615]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_4 --path /var/lib/ceph/osd/ceph-0 --no-mon-config 2023-09-23T20:20:08.141 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 23 20:20:07 smithi018 bash[194615]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_4 /var/lib/ceph/osd/ceph-0/block 2023-09-23T20:20:08.141 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 23 20:20:07 smithi018 bash[194615]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-0/block 2023-09-23T20:20:08.141 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 23 20:20:07 smithi018 bash[194615]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2023-09-23T20:20:08.142 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 23 20:20:07 smithi018 bash[194615]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2023-09-23T20:20:08.142 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 23 20:20:07 smithi018 bash[194615]: --> ceph-volume lvm activate successful for osd ID: 0 2023-09-23T20:20:08.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:08 smithi149 ceph-mon[170394]: pgmap v129: 73 pgs: 26 peering, 47 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:20:08.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:08 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3678809367' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:20:08.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:08 smithi018 ceph-mon[179647]: pgmap v129: 73 pgs: 26 peering, 47 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:20:08.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:08 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3678809367' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:20:08.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:08 smithi018 ceph-mon[184144]: pgmap v129: 73 pgs: 26 peering, 47 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:20:08.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:08 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3678809367' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:20:08.892 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 23 20:20:08 smithi018 podman[194906]: 2023-09-23T20:20:08.892 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 23 20:20:08 smithi018 ceph-osd[194946]: -- 172.21.15.18:0/3531395889 <== mon.0 v2:172.21.15.18:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55c2c3fc84e0 con 0x55c2c3f99000 2023-09-23T20:20:08.892 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 23 20:20:08 smithi018 bash[194906]: 9c05a0fd7c04cb7f13897028eddd53cbdb8e18ee3c04759db7f0576668f10075 2023-09-23T20:20:08.892 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 23 20:20:08 smithi018 systemd[1]: Started Ceph osd.0 for a3b3b17e-5a4c-11ee-8db4-212e2dc638e7. 2023-09-23T20:20:09.391 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 23 20:20:09 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-0[194922]: 2023-09-23T20:20:09.096+0000 7f5e321db640 -1 Falling back to public interface 2023-09-23T20:20:09.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:09 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:20:09.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:09 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:20:09.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:09 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:20:10.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:09 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:20:10.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:09 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:20:10.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:09 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:20:10.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:09 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:20:10.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:09 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:20:10.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:09 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:20:10.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:10 smithi149 ceph-mon[170394]: pgmap v130: 73 pgs: 26 peering, 47 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:20:10.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:10 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/4271962975' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:20:11.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:10 smithi018 ceph-mon[184144]: pgmap v130: 73 pgs: 26 peering, 47 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:20:11.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:10 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/4271962975' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:20:11.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:10 smithi018 ceph-mon[179647]: pgmap v130: 73 pgs: 26 peering, 47 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:20:11.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:10 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/4271962975' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:20:12.141 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 23 20:20:11 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-0[194922]: 2023-09-23T20:20:11.850+0000 7f5e321db640 -1 osd.0 90 log_to_monitors true 2023-09-23T20:20:12.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:11 smithi018 ceph-mon[179647]: Health check failed: Degraded data redundancy: 13/84 objects degraded (15.476%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T20:20:12.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:11 smithi018 ceph-mon[179647]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 9 pgs inactive, 26 pgs peering) 2023-09-23T20:20:12.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:11 smithi018 ceph-mon[184144]: Health check failed: Degraded data redundancy: 13/84 objects degraded (15.476%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T20:20:12.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:11 smithi018 ceph-mon[184144]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 9 pgs inactive, 26 pgs peering) 2023-09-23T20:20:12.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:11 smithi149 ceph-mon[170394]: Health check failed: Degraded data redundancy: 13/84 objects degraded (15.476%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T20:20:12.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:11 smithi149 ceph-mon[170394]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 9 pgs inactive, 26 pgs peering) 2023-09-23T20:20:13.141 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 23 20:20:12 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-0[194922]: 2023-09-23T20:20:12.878+0000 7f5e20642700 -1 osd.0 90 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T20:20:13.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:12 smithi018 ceph-mon[184144]: pgmap v131: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 85 MiB used, 715 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2023-09-23T20:20:13.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:12 smithi018 ceph-mon[184144]: from='osd.0 [v2:172.21.15.18:6802/990458588,v1:172.21.15.18:6803/990458588]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-23T20:20:13.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:12 smithi018 ceph-mon[179647]: pgmap v131: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 85 MiB used, 715 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2023-09-23T20:20:13.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:12 smithi018 ceph-mon[179647]: from='osd.0 [v2:172.21.15.18:6802/990458588,v1:172.21.15.18:6803/990458588]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-23T20:20:13.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:12 smithi149 ceph-mon[170394]: pgmap v131: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 85 MiB used, 715 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2023-09-23T20:20:13.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:12 smithi149 ceph-mon[170394]: from='osd.0 [v2:172.21.15.18:6802/990458588,v1:172.21.15.18:6803/990458588]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-23T20:20:14.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:13 smithi018 ceph-mon[179647]: failed to encode map e93 with expected crc 2023-09-23T20:20:14.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:13 smithi018 ceph-mon[179647]: failed to encode map e93 with expected crc 2023-09-23T20:20:14.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:13 smithi018 ceph-mon[179647]: from='osd.0 [v2:172.21.15.18:6802/990458588,v1:172.21.15.18:6803/990458588]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-09-23T20:20:14.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:13 smithi018 ceph-mon[179647]: osdmap e93: 8 total, 7 up, 8 in 2023-09-23T20:20:14.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:13 smithi018 ceph-mon[179647]: from='osd.0 [v2:172.21.15.18:6802/990458588,v1:172.21.15.18:6803/990458588]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi018", "root=default"]}]: dispatch 2023-09-23T20:20:14.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:13 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1317090496' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:20:14.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:13 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:20:14.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:13 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:20:14.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:13 smithi018 ceph-mon[184144]: failed to encode map e93 with expected crc 2023-09-23T20:20:14.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:13 smithi018 ceph-mon[184144]: failed to encode map e93 with expected crc 2023-09-23T20:20:14.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:13 smithi018 ceph-mon[184144]: from='osd.0 [v2:172.21.15.18:6802/990458588,v1:172.21.15.18:6803/990458588]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-09-23T20:20:14.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:13 smithi018 ceph-mon[184144]: osdmap e93: 8 total, 7 up, 8 in 2023-09-23T20:20:14.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:13 smithi018 ceph-mon[184144]: from='osd.0 [v2:172.21.15.18:6802/990458588,v1:172.21.15.18:6803/990458588]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi018", "root=default"]}]: dispatch 2023-09-23T20:20:14.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:13 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1317090496' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:20:14.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:13 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:20:14.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:13 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:20:14.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:13 smithi149 ceph-mon[170394]: failed to encode map e93 with expected crc 2023-09-23T20:20:14.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:13 smithi149 ceph-mon[170394]: failed to encode map e93 with expected crc 2023-09-23T20:20:14.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:13 smithi149 ceph-mon[170394]: from='osd.0 [v2:172.21.15.18:6802/990458588,v1:172.21.15.18:6803/990458588]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-09-23T20:20:14.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:13 smithi149 ceph-mon[170394]: osdmap e93: 8 total, 7 up, 8 in 2023-09-23T20:20:14.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:13 smithi149 ceph-mon[170394]: from='osd.0 [v2:172.21.15.18:6802/990458588,v1:172.21.15.18:6803/990458588]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi018", "root=default"]}]: dispatch 2023-09-23T20:20:14.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:13 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1317090496' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:20:14.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:13 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:20:14.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:13 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:20:15.141 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:20:14 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:20:14] "GET /metrics HTTP/1.1" 200 33971 "" "Prometheus/2.43.0" 2023-09-23T20:20:15.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:14 smithi018 ceph-mon[179647]: pgmap v133: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 100 MiB used, 715 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2023-09-23T20:20:15.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:14 smithi018 ceph-mon[179647]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T20:20:15.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:14 smithi018 ceph-mon[179647]: failed to encode map e94 with expected crc 2023-09-23T20:20:15.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:14 smithi018 ceph-mon[179647]: failed to encode map e94 with expected crc 2023-09-23T20:20:15.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:14 smithi018 ceph-mon[179647]: osd.0 [v2:172.21.15.18:6802/990458588,v1:172.21.15.18:6803/990458588] boot 2023-09-23T20:20:15.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:14 smithi018 ceph-mon[179647]: osdmap e94: 8 total, 8 up, 8 in 2023-09-23T20:20:15.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:14 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T20:20:15.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:14 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:20:15.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:14 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:20:15.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:14 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:20:15.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:14 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:20:15.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:14 smithi018 ceph-mon[184144]: pgmap v133: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 100 MiB used, 715 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2023-09-23T20:20:15.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:14 smithi018 ceph-mon[184144]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T20:20:15.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:14 smithi018 ceph-mon[184144]: failed to encode map e94 with expected crc 2023-09-23T20:20:15.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:14 smithi018 ceph-mon[184144]: failed to encode map e94 with expected crc 2023-09-23T20:20:15.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:14 smithi018 ceph-mon[184144]: osd.0 [v2:172.21.15.18:6802/990458588,v1:172.21.15.18:6803/990458588] boot 2023-09-23T20:20:15.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:14 smithi018 ceph-mon[184144]: osdmap e94: 8 total, 8 up, 8 in 2023-09-23T20:20:15.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:14 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T20:20:15.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:14 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:20:15.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:14 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:20:15.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:14 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:20:15.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:14 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:20:15.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:14 smithi149 ceph-mon[170394]: pgmap v133: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 100 MiB used, 715 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2023-09-23T20:20:15.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:14 smithi149 ceph-mon[170394]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T20:20:15.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:14 smithi149 ceph-mon[170394]: failed to encode map e94 with expected crc 2023-09-23T20:20:15.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:14 smithi149 ceph-mon[170394]: failed to encode map e94 with expected crc 2023-09-23T20:20:15.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:14 smithi149 ceph-mon[170394]: osd.0 [v2:172.21.15.18:6802/990458588,v1:172.21.15.18:6803/990458588] boot 2023-09-23T20:20:15.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:14 smithi149 ceph-mon[170394]: osdmap e94: 8 total, 8 up, 8 in 2023-09-23T20:20:15.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:14 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T20:20:15.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:14 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:20:15.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:14 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:20:15.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:14 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:20:15.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:14 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:20:16.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:15 smithi018 ceph-mon[179647]: failed to encode map e95 with expected crc 2023-09-23T20:20:16.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:15 smithi018 ceph-mon[179647]: failed to encode map e95 with expected crc 2023-09-23T20:20:16.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:15 smithi018 ceph-mon[179647]: osdmap e95: 8 total, 8 up, 8 in 2023-09-23T20:20:16.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:15 smithi018 ceph-mon[179647]: failed to encode map e95 with expected crc 2023-09-23T20:20:16.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:15 smithi018 ceph-mon[179647]: failed to encode map e95 with expected crc 2023-09-23T20:20:16.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:15 smithi018 ceph-mon[179647]: failed to encode map e95 with expected crc 2023-09-23T20:20:16.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:15 smithi018 ceph-mon[179647]: failed to encode map e95 with expected crc 2023-09-23T20:20:16.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:15 smithi018 ceph-mon[179647]: failed to encode map e95 with expected crc 2023-09-23T20:20:16.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:15 smithi018 ceph-mon[179647]: failed to encode map e95 with expected crc 2023-09-23T20:20:16.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:15 smithi018 ceph-mon[179647]: failed to encode map e95 with expected crc 2023-09-23T20:20:16.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:15 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2047057576' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:20:16.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:15 smithi018 ceph-mon[184144]: failed to encode map e95 with expected crc 2023-09-23T20:20:16.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:15 smithi018 ceph-mon[184144]: failed to encode map e95 with expected crc 2023-09-23T20:20:16.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:15 smithi018 ceph-mon[184144]: osdmap e95: 8 total, 8 up, 8 in 2023-09-23T20:20:16.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:15 smithi018 ceph-mon[184144]: failed to encode map e95 with expected crc 2023-09-23T20:20:16.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:15 smithi018 ceph-mon[184144]: failed to encode map e95 with expected crc 2023-09-23T20:20:16.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:15 smithi018 ceph-mon[184144]: failed to encode map e95 with expected crc 2023-09-23T20:20:16.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:15 smithi018 ceph-mon[184144]: failed to encode map e95 with expected crc 2023-09-23T20:20:16.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:15 smithi018 ceph-mon[184144]: failed to encode map e95 with expected crc 2023-09-23T20:20:16.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:15 smithi018 ceph-mon[184144]: failed to encode map e95 with expected crc 2023-09-23T20:20:16.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:15 smithi018 ceph-mon[184144]: failed to encode map e95 with expected crc 2023-09-23T20:20:16.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:15 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2047057576' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:20:16.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:15 smithi149 ceph-mon[170394]: failed to encode map e95 with expected crc 2023-09-23T20:20:16.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:15 smithi149 ceph-mon[170394]: failed to encode map e95 with expected crc 2023-09-23T20:20:16.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:15 smithi149 ceph-mon[170394]: osdmap e95: 8 total, 8 up, 8 in 2023-09-23T20:20:16.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:15 smithi149 ceph-mon[170394]: failed to encode map e95 with expected crc 2023-09-23T20:20:16.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:15 smithi149 ceph-mon[170394]: failed to encode map e95 with expected crc 2023-09-23T20:20:16.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:15 smithi149 ceph-mon[170394]: failed to encode map e95 with expected crc 2023-09-23T20:20:16.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:15 smithi149 ceph-mon[170394]: failed to encode map e95 with expected crc 2023-09-23T20:20:16.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:15 smithi149 ceph-mon[170394]: failed to encode map e95 with expected crc 2023-09-23T20:20:16.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:15 smithi149 ceph-mon[170394]: failed to encode map e95 with expected crc 2023-09-23T20:20:16.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:15 smithi149 ceph-mon[170394]: failed to encode map e95 with expected crc 2023-09-23T20:20:16.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:15 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2047057576' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:20:17.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:16 smithi018 ceph-mon[179647]: pgmap v136: 73 pgs: 2 active+undersized+degraded, 9 active+undersized, 62 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2023-09-23T20:20:17.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:16 smithi018 ceph-mon[184144]: pgmap v136: 73 pgs: 2 active+undersized+degraded, 9 active+undersized, 62 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2023-09-23T20:20:17.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:16 smithi149 ceph-mon[170394]: pgmap v136: 73 pgs: 2 active+undersized+degraded, 9 active+undersized, 62 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2023-09-23T20:20:17.641 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:20:17 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: 2023-09-23T20:20:17.175+0000 7f953be6a700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-09-23T20:20:18.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:18 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:20:18.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:18 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:20:18.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:18 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:20:18.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:18 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:20:18.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:18 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:20:18.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:18 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:20:18.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:18 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:20:18.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:18 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:20:18.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:18 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:20:18.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:18 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T20:20:18.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:18 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/63740083' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:20:18.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:18 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:20:18.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:18 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:20:18.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:18 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:20:18.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:18 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:20:18.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:18 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:20:18.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:18 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:20:18.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:18 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:20:18.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:18 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:20:18.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:18 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:20:18.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:18 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T20:20:18.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:18 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/63740083' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:20:18.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:18 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:20:18.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:18 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:20:18.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:18 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:20:18.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:18 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:20:18.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:18 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:20:18.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:18 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:20:18.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:18 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:20:18.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:18 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:20:18.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:18 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:20:18.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:18 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T20:20:18.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:18 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/63740083' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:20:19.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:19 smithi018 ceph-mon[179647]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T20:20:19.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:19 smithi018 ceph-mon[179647]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-09-23T20:20:19.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:19 smithi018 ceph-mon[179647]: pgmap v137: 73 pgs: 2 active+undersized+degraded, 9 active+undersized, 62 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2023-09-23T20:20:19.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:19 smithi018 ceph-mon[184144]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T20:20:19.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:19 smithi018 ceph-mon[184144]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-09-23T20:20:19.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:19 smithi018 ceph-mon[184144]: pgmap v137: 73 pgs: 2 active+undersized+degraded, 9 active+undersized, 62 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2023-09-23T20:20:19.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:19 smithi149 ceph-mon[170394]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T20:20:19.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:19 smithi149 ceph-mon[170394]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-09-23T20:20:19.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:19 smithi149 ceph-mon[170394]: pgmap v137: 73 pgs: 2 active+undersized+degraded, 9 active+undersized, 62 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2023-09-23T20:20:20.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:20 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/282798616' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:20:20.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:20 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/282798616' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:20:20.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:20 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/282798616' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:20:21.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:21 smithi018 ceph-mon[184144]: pgmap v138: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:20:21.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:21 smithi018 ceph-mon[184144]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded) 2023-09-23T20:20:21.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:21 smithi018 ceph-mon[184144]: Cluster is now healthy 2023-09-23T20:20:21.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:21 smithi018 ceph-mon[179647]: pgmap v138: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:20:21.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:21 smithi018 ceph-mon[179647]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded) 2023-09-23T20:20:21.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:21 smithi018 ceph-mon[179647]: Cluster is now healthy 2023-09-23T20:20:21.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:21 smithi149 ceph-mon[170394]: pgmap v138: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:20:21.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:21 smithi149 ceph-mon[170394]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded) 2023-09-23T20:20:21.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:21 smithi149 ceph-mon[170394]: Cluster is now healthy 2023-09-23T20:20:23.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:23 smithi018 ceph-mon[179647]: pgmap v139: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:20:23.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:23 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/710868884' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:20:23.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:23 smithi018 ceph-mon[184144]: pgmap v139: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:20:23.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:23 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/710868884' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:20:23.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:23 smithi149 ceph-mon[170394]: pgmap v139: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:20:23.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:23 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/710868884' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:20:25.391 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:20:24 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:20:24] "GET /metrics HTTP/1.1" 200 33971 "" "Prometheus/2.43.0" 2023-09-23T20:20:25.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:25 smithi018 ceph-mon[179647]: pgmap v140: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:20:25.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:25 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:20:25.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:25 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3349860222' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:20:25.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:25 smithi018 ceph-mon[184144]: pgmap v140: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:20:25.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:25 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:20:25.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:25 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3349860222' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:20:25.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:25 smithi149 ceph-mon[170394]: pgmap v140: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:20:25.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:25 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:20:25.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:25 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3349860222' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:20:27.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:27 smithi018 ceph-mon[184144]: pgmap v141: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:20:27.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:27 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2644991527' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:20:27.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:27 smithi018 ceph-mon[179647]: pgmap v141: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:20:27.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:27 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2644991527' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:20:27.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:27 smithi149 ceph-mon[170394]: pgmap v141: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:20:27.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:27 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2644991527' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:20:29.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:29 smithi018 ceph-mon[184144]: pgmap v142: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:20:29.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:29 smithi018 ceph-mon[179647]: pgmap v142: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:20:29.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:29 smithi149 ceph-mon[170394]: pgmap v142: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:20:30.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:30 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/211608137' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:20:30.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:30 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:20:30.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:30 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:20:30.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:30 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/211608137' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:20:30.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:30 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:20:30.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:30 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:20:30.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:30 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/211608137' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:20:30.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:30 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:20:30.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:30 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:20:31.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:31 smithi018 ceph-mon[184144]: pgmap v143: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:20:31.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:31 smithi018 ceph-mon[179647]: pgmap v143: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:20:31.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:31 smithi149 ceph-mon[170394]: pgmap v143: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:20:32.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:32 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2929547855' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:20:32.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:32 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2929547855' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:20:32.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:32 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2929547855' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:20:33.199 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:33 smithi018 ceph-mon[184144]: pgmap v144: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:20:33.199 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:33 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T20:20:33.200 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:33 smithi018 ceph-mon[179647]: pgmap v144: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:20:33.200 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:33 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T20:20:33.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:33 smithi149 ceph-mon[170394]: pgmap v144: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:20:33.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:33 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T20:20:34.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:34 smithi018 ceph-mon[184144]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T20:20:34.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:34 smithi018 ceph-mon[184144]: Upgrade: osd.1 is safe to restart 2023-09-23T20:20:34.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:34 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:20:34.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:34 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-09-23T20:20:34.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:34 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:20:34.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:34 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2719439282' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:20:34.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:34 smithi018 ceph-mon[179647]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T20:20:34.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:34 smithi018 ceph-mon[179647]: Upgrade: osd.1 is safe to restart 2023-09-23T20:20:34.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:34 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:20:34.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:34 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-09-23T20:20:34.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:34 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:20:34.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:34 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2719439282' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:20:34.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:34 smithi149 ceph-mon[170394]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T20:20:34.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:34 smithi149 ceph-mon[170394]: Upgrade: osd.1 is safe to restart 2023-09-23T20:20:34.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:34 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:20:34.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:34 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-09-23T20:20:34.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:34 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:20:34.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:34 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2719439282' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:20:35.041 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:20:34 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:20:34] "GET /metrics HTTP/1.1" 200 33977 "" "Prometheus/2.43.0" 2023-09-23T20:20:35.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:35 smithi018 ceph-mon[184144]: pgmap v145: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:20:35.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:35 smithi018 ceph-mon[184144]: Upgrade: Updating osd.1 2023-09-23T20:20:35.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:35 smithi018 ceph-mon[184144]: Deploying daemon osd.1 on smithi018 2023-09-23T20:20:35.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:35 smithi018 ceph-mon[179647]: pgmap v145: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:20:35.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:35 smithi018 ceph-mon[179647]: Upgrade: Updating osd.1 2023-09-23T20:20:35.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:35 smithi018 ceph-mon[179647]: Deploying daemon osd.1 on smithi018 2023-09-23T20:20:35.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:35 smithi149 ceph-mon[170394]: pgmap v145: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:20:35.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:35 smithi149 ceph-mon[170394]: Upgrade: Updating osd.1 2023-09-23T20:20:35.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:35 smithi149 ceph-mon[170394]: Deploying daemon osd.1 on smithi018 2023-09-23T20:20:36.141 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 23 20:20:35 smithi018 systemd[1]: Stopping Ceph osd.1 for a3b3b17e-5a4c-11ee-8db4-212e2dc638e7... 2023-09-23T20:20:36.641 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 23 20:20:36 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-1[121464]: 2023-09-23T20:20:36.305+0000 7f63cc58b700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T20:20:36.641 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 23 20:20:36 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-1[121464]: 2023-09-23T20:20:36.305+0000 7f63cc58b700 -1 osd.1 95 *** Got signal Terminated *** 2023-09-23T20:20:36.641 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 23 20:20:36 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-1[121464]: 2023-09-23T20:20:36.305+0000 7f63cc58b700 -1 osd.1 95 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T20:20:37.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:37 smithi018 ceph-mon[179647]: pgmap v146: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:20:37.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:37 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2201420527' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:20:37.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:37 smithi018 ceph-mon[179647]: osd.1 marked itself down and dead 2023-09-23T20:20:37.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:37 smithi018 ceph-mon[184144]: pgmap v146: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:20:37.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:37 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2201420527' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:20:37.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:37 smithi018 ceph-mon[184144]: osd.1 marked itself down and dead 2023-09-23T20:20:37.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:37 smithi149 ceph-mon[170394]: pgmap v146: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:20:37.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:37 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2201420527' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:20:37.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:37 smithi149 ceph-mon[170394]: osd.1 marked itself down and dead 2023-09-23T20:20:37.891 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 23 20:20:37 smithi018 bash[200551]: ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-1 2023-09-23T20:20:38.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:38 smithi018 ceph-mon[184144]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T20:20:38.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:38 smithi018 ceph-mon[184144]: failed to encode map e96 with expected crc 2023-09-23T20:20:38.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:38 smithi018 ceph-mon[184144]: failed to encode map e96 with expected crc 2023-09-23T20:20:38.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:38 smithi018 ceph-mon[184144]: osdmap e96: 8 total, 7 up, 8 in 2023-09-23T20:20:38.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:38 smithi018 ceph-mon[179647]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T20:20:38.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:38 smithi018 ceph-mon[179647]: failed to encode map e96 with expected crc 2023-09-23T20:20:38.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:38 smithi018 ceph-mon[179647]: failed to encode map e96 with expected crc 2023-09-23T20:20:38.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:38 smithi018 ceph-mon[179647]: osdmap e96: 8 total, 7 up, 8 in 2023-09-23T20:20:38.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:38 smithi149 ceph-mon[170394]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T20:20:38.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:38 smithi149 ceph-mon[170394]: failed to encode map e96 with expected crc 2023-09-23T20:20:38.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:38 smithi149 ceph-mon[170394]: failed to encode map e96 with expected crc 2023-09-23T20:20:38.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:38 smithi149 ceph-mon[170394]: osdmap e96: 8 total, 7 up, 8 in 2023-09-23T20:20:39.391 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 23 20:20:38 smithi018 systemd[1]: ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7@osd.1.service: Succeeded. 2023-09-23T20:20:39.391 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 23 20:20:38 smithi018 systemd[1]: Stopped Ceph osd.1 for a3b3b17e-5a4c-11ee-8db4-212e2dc638e7. 2023-09-23T20:20:39.391 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 23 20:20:39 smithi018 systemd[1]: Starting Ceph osd.1 for a3b3b17e-5a4c-11ee-8db4-212e2dc638e7... 2023-09-23T20:20:39.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:39 smithi018 ceph-mon[179647]: pgmap v148: 73 pgs: 9 stale+active+clean, 64 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:20:39.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:39 smithi018 ceph-mon[179647]: failed to encode map e97 with expected crc 2023-09-23T20:20:39.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:39 smithi018 ceph-mon[179647]: failed to encode map e97 with expected crc 2023-09-23T20:20:39.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:39 smithi018 ceph-mon[179647]: osdmap e97: 8 total, 7 up, 8 in 2023-09-23T20:20:39.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:39 smithi018 ceph-mon[179647]: failed to encode map e97 with expected crc 2023-09-23T20:20:39.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:39 smithi018 ceph-mon[179647]: failed to encode map e97 with expected crc 2023-09-23T20:20:39.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:39 smithi018 ceph-mon[179647]: failed to encode map e97 with expected crc 2023-09-23T20:20:39.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:39 smithi018 ceph-mon[179647]: failed to encode map e97 with expected crc 2023-09-23T20:20:39.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:39 smithi018 ceph-mon[179647]: failed to encode map e97 with expected crc 2023-09-23T20:20:39.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:39 smithi018 ceph-mon[179647]: failed to encode map e97 with expected crc 2023-09-23T20:20:39.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:39 smithi018 ceph-mon[179647]: failed to encode map e97 with expected crc 2023-09-23T20:20:39.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:39 smithi018 ceph-mon[179647]: failed to encode map e97 with expected crc 2023-09-23T20:20:39.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:39 smithi018 ceph-mon[179647]: failed to encode map e97 with expected crc 2023-09-23T20:20:39.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:39 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3680176563' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:20:39.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:39 smithi018 ceph-mon[184144]: pgmap v148: 73 pgs: 9 stale+active+clean, 64 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:20:39.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:39 smithi018 ceph-mon[184144]: failed to encode map e97 with expected crc 2023-09-23T20:20:39.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:39 smithi018 ceph-mon[184144]: failed to encode map e97 with expected crc 2023-09-23T20:20:39.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:39 smithi018 ceph-mon[184144]: osdmap e97: 8 total, 7 up, 8 in 2023-09-23T20:20:39.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:39 smithi018 ceph-mon[184144]: failed to encode map e97 with expected crc 2023-09-23T20:20:39.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:39 smithi018 ceph-mon[184144]: failed to encode map e97 with expected crc 2023-09-23T20:20:39.394 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:39 smithi018 ceph-mon[184144]: failed to encode map e97 with expected crc 2023-09-23T20:20:39.394 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:39 smithi018 ceph-mon[184144]: failed to encode map e97 with expected crc 2023-09-23T20:20:39.394 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:39 smithi018 ceph-mon[184144]: failed to encode map e97 with expected crc 2023-09-23T20:20:39.394 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:39 smithi018 ceph-mon[184144]: failed to encode map e97 with expected crc 2023-09-23T20:20:39.394 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:39 smithi018 ceph-mon[184144]: failed to encode map e97 with expected crc 2023-09-23T20:20:39.394 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:39 smithi018 ceph-mon[184144]: failed to encode map e97 with expected crc 2023-09-23T20:20:39.394 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:39 smithi018 ceph-mon[184144]: failed to encode map e97 with expected crc 2023-09-23T20:20:39.394 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:39 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3680176563' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:20:39.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:39 smithi149 ceph-mon[170394]: pgmap v148: 73 pgs: 9 stale+active+clean, 64 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:20:39.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:39 smithi149 ceph-mon[170394]: failed to encode map e97 with expected crc 2023-09-23T20:20:39.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:39 smithi149 ceph-mon[170394]: failed to encode map e97 with expected crc 2023-09-23T20:20:39.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:39 smithi149 ceph-mon[170394]: osdmap e97: 8 total, 7 up, 8 in 2023-09-23T20:20:39.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:39 smithi149 ceph-mon[170394]: failed to encode map e97 with expected crc 2023-09-23T20:20:39.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:39 smithi149 ceph-mon[170394]: failed to encode map e97 with expected crc 2023-09-23T20:20:39.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:39 smithi149 ceph-mon[170394]: failed to encode map e97 with expected crc 2023-09-23T20:20:39.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:39 smithi149 ceph-mon[170394]: failed to encode map e97 with expected crc 2023-09-23T20:20:39.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:39 smithi149 ceph-mon[170394]: failed to encode map e97 with expected crc 2023-09-23T20:20:39.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:39 smithi149 ceph-mon[170394]: failed to encode map e97 with expected crc 2023-09-23T20:20:39.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:39 smithi149 ceph-mon[170394]: failed to encode map e97 with expected crc 2023-09-23T20:20:39.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:39 smithi149 ceph-mon[170394]: failed to encode map e97 with expected crc 2023-09-23T20:20:39.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:39 smithi149 ceph-mon[170394]: failed to encode map e97 with expected crc 2023-09-23T20:20:39.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:39 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3680176563' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:20:40.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:40 smithi018 ceph-mon[179647]: pgmap v150: 73 pgs: 4 peering, 8 stale+active+clean, 61 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:20:40.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:40 smithi018 ceph-mon[179647]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-09-23T20:20:40.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:40 smithi018 ceph-mon[184144]: pgmap v150: 73 pgs: 4 peering, 8 stale+active+clean, 61 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:20:40.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:40 smithi018 ceph-mon[184144]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-09-23T20:20:40.392 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 23 20:20:39 smithi018 bash[200858]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-09-23T20:20:40.392 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 23 20:20:39 smithi018 bash[200858]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2023-09-23T20:20:40.392 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 23 20:20:39 smithi018 bash[200858]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_3 --path /var/lib/ceph/osd/ceph-1 --no-mon-config 2023-09-23T20:20:40.392 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 23 20:20:39 smithi018 bash[200858]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_3 /var/lib/ceph/osd/ceph-1/block 2023-09-23T20:20:40.392 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 23 20:20:39 smithi018 bash[200858]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-1/block 2023-09-23T20:20:40.393 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 23 20:20:39 smithi018 bash[200858]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2023-09-23T20:20:40.393 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 23 20:20:39 smithi018 bash[200858]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2023-09-23T20:20:40.393 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 23 20:20:39 smithi018 bash[200858]: --> ceph-volume lvm activate successful for osd ID: 1 2023-09-23T20:20:40.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:40 smithi149 ceph-mon[170394]: pgmap v150: 73 pgs: 4 peering, 8 stale+active+clean, 61 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:20:40.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:40 smithi149 ceph-mon[170394]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-09-23T20:20:40.863 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 23 20:20:40 smithi018 podman[201106]: 2023-09-23T20:20:40.863 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 23 20:20:40 smithi018 bash[201106]: c71a2c8070c3382c5f92819bd1dc717d8f21aa6bd7a4d1ea5835d3271b49272a 2023-09-23T20:20:40.863 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 23 20:20:40 smithi018 ceph-osd[201169]: -- 172.21.15.18:0/535247368 <== mon.2 v2:172.21.15.18:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x564d519ce4e0 con 0x564d5199f400 2023-09-23T20:20:41.141 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 23 20:20:40 smithi018 systemd[1]: Started Ceph osd.1 for a3b3b17e-5a4c-11ee-8db4-212e2dc638e7. 2023-09-23T20:20:41.641 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 23 20:20:41 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-1[201125]: 2023-09-23T20:20:41.320+0000 7f078180c640 -1 Falling back to public interface 2023-09-23T20:20:42.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:41 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:20:42.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:41 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:20:42.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:41 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:20:42.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:41 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2797639655' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:20:42.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:41 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:20:42.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:41 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:20:42.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:41 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:20:42.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:41 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2797639655' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:20:42.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:41 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:20:42.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:41 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:20:42.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:41 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:20:42.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:41 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2797639655' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:20:43.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:42 smithi149 ceph-mon[170394]: pgmap v151: 73 pgs: 3 active+undersized+degraded, 4 peering, 25 active+undersized, 41 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2023-09-23T20:20:43.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:42 smithi149 ceph-mon[170394]: Health check failed: Degraded data redundancy: 3/84 objects degraded (3.571%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T20:20:43.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:42 smithi018 ceph-mon[184144]: pgmap v151: 73 pgs: 3 active+undersized+degraded, 4 peering, 25 active+undersized, 41 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2023-09-23T20:20:43.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:42 smithi018 ceph-mon[184144]: Health check failed: Degraded data redundancy: 3/84 objects degraded (3.571%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T20:20:43.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:42 smithi018 ceph-mon[179647]: pgmap v151: 73 pgs: 3 active+undersized+degraded, 4 peering, 25 active+undersized, 41 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2023-09-23T20:20:43.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:42 smithi018 ceph-mon[179647]: Health check failed: Degraded data redundancy: 3/84 objects degraded (3.571%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T20:20:44.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:43 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2770317754' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:20:44.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:43 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2770317754' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:20:44.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:43 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2770317754' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:20:44.391 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 23 20:20:44 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-1[201125]: 2023-09-23T20:20:44.058+0000 7f078180c640 -1 osd.1 95 log_to_monitors true 2023-09-23T20:20:45.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:44 smithi149 ceph-mon[170394]: pgmap v152: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-23T20:20:45.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:44 smithi149 ceph-mon[170394]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-09-23T20:20:45.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:44 smithi149 ceph-mon[170394]: from='osd.1 [v2:172.21.15.18:6810/1422592526,v1:172.21.15.18:6811/1422592526]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T20:20:45.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:44 smithi149 ceph-mon[170394]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T20:20:45.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:44 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:20:45.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:44 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:20:45.393 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:20:44 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:20:44] "GET /metrics HTTP/1.1" 200 34040 "" "Prometheus/2.43.0" 2023-09-23T20:20:45.393 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 23 20:20:44 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-1[201125]: 2023-09-23T20:20:44.946+0000 7f0770c75700 -1 osd.1 95 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T20:20:45.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:44 smithi018 ceph-mon[179647]: pgmap v152: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-23T20:20:45.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:44 smithi018 ceph-mon[179647]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-09-23T20:20:45.394 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:44 smithi018 ceph-mon[179647]: from='osd.1 [v2:172.21.15.18:6810/1422592526,v1:172.21.15.18:6811/1422592526]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T20:20:45.394 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:44 smithi018 ceph-mon[179647]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T20:20:45.394 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:44 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:20:45.394 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:44 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:20:45.394 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:44 smithi018 ceph-mon[184144]: pgmap v152: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-23T20:20:45.394 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:44 smithi018 ceph-mon[184144]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-09-23T20:20:45.394 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:44 smithi018 ceph-mon[184144]: from='osd.1 [v2:172.21.15.18:6810/1422592526,v1:172.21.15.18:6811/1422592526]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T20:20:45.394 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:44 smithi018 ceph-mon[184144]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T20:20:45.394 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:44 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:20:45.394 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:44 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:20:46.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:45 smithi149 ceph-mon[170394]: failed to encode map e98 with expected crc 2023-09-23T20:20:46.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:45 smithi149 ceph-mon[170394]: failed to encode map e98 with expected crc 2023-09-23T20:20:46.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:45 smithi149 ceph-mon[170394]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-09-23T20:20:46.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:45 smithi149 ceph-mon[170394]: osdmap e98: 8 total, 7 up, 8 in 2023-09-23T20:20:46.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:45 smithi149 ceph-mon[170394]: from='osd.1 [v2:172.21.15.18:6810/1422592526,v1:172.21.15.18:6811/1422592526]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi018", "root=default"]}]: dispatch 2023-09-23T20:20:46.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:45 smithi149 ceph-mon[170394]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi018", "root=default"]}]: dispatch 2023-09-23T20:20:46.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:45 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:20:46.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:45 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:20:46.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:45 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2656018737' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:20:46.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:45 smithi018 ceph-mon[179647]: failed to encode map e98 with expected crc 2023-09-23T20:20:46.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:45 smithi018 ceph-mon[179647]: failed to encode map e98 with expected crc 2023-09-23T20:20:46.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:45 smithi018 ceph-mon[179647]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-09-23T20:20:46.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:45 smithi018 ceph-mon[179647]: osdmap e98: 8 total, 7 up, 8 in 2023-09-23T20:20:46.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:45 smithi018 ceph-mon[179647]: from='osd.1 [v2:172.21.15.18:6810/1422592526,v1:172.21.15.18:6811/1422592526]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi018", "root=default"]}]: dispatch 2023-09-23T20:20:46.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:45 smithi018 ceph-mon[179647]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi018", "root=default"]}]: dispatch 2023-09-23T20:20:46.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:45 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:20:46.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:45 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:20:46.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:45 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2656018737' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:20:46.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:45 smithi018 ceph-mon[184144]: failed to encode map e98 with expected crc 2023-09-23T20:20:46.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:45 smithi018 ceph-mon[184144]: failed to encode map e98 with expected crc 2023-09-23T20:20:46.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:45 smithi018 ceph-mon[184144]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-09-23T20:20:46.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:45 smithi018 ceph-mon[184144]: osdmap e98: 8 total, 7 up, 8 in 2023-09-23T20:20:46.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:45 smithi018 ceph-mon[184144]: from='osd.1 [v2:172.21.15.18:6810/1422592526,v1:172.21.15.18:6811/1422592526]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi018", "root=default"]}]: dispatch 2023-09-23T20:20:46.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:45 smithi018 ceph-mon[184144]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi018", "root=default"]}]: dispatch 2023-09-23T20:20:46.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:45 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:20:46.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:45 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:20:46.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:45 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2656018737' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:20:47.192 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:46 smithi018 ceph-mon[179647]: pgmap v154: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 121 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-23T20:20:47.192 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:46 smithi018 ceph-mon[179647]: failed to encode map e98 with expected crc 2023-09-23T20:20:47.192 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:46 smithi018 ceph-mon[179647]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T20:20:47.192 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:46 smithi018 ceph-mon[179647]: failed to encode map e98 with expected crc 2023-09-23T20:20:47.192 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:46 smithi018 ceph-mon[179647]: osd.1 [v2:172.21.15.18:6810/1422592526,v1:172.21.15.18:6811/1422592526] boot 2023-09-23T20:20:47.193 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:46 smithi018 ceph-mon[179647]: osdmap e99: 8 total, 8 up, 8 in 2023-09-23T20:20:47.193 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:46 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T20:20:47.193 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:46 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:20:47.193 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:46 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:20:47.193 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:46 smithi018 ceph-mon[179647]: osdmap e100: 8 total, 8 up, 8 in 2023-09-23T20:20:47.193 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:46 smithi018 ceph-mon[184144]: pgmap v154: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 121 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-23T20:20:47.193 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:46 smithi018 ceph-mon[184144]: failed to encode map e98 with expected crc 2023-09-23T20:20:47.193 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:46 smithi018 ceph-mon[184144]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T20:20:47.193 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:46 smithi018 ceph-mon[184144]: failed to encode map e98 with expected crc 2023-09-23T20:20:47.193 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:46 smithi018 ceph-mon[184144]: osd.1 [v2:172.21.15.18:6810/1422592526,v1:172.21.15.18:6811/1422592526] boot 2023-09-23T20:20:47.194 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:46 smithi018 ceph-mon[184144]: osdmap e99: 8 total, 8 up, 8 in 2023-09-23T20:20:47.194 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:46 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T20:20:47.194 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:46 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:20:47.194 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:46 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:20:47.194 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:46 smithi018 ceph-mon[184144]: osdmap e100: 8 total, 8 up, 8 in 2023-09-23T20:20:47.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:46 smithi149 ceph-mon[170394]: pgmap v154: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 121 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-23T20:20:47.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:46 smithi149 ceph-mon[170394]: failed to encode map e98 with expected crc 2023-09-23T20:20:47.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:46 smithi149 ceph-mon[170394]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T20:20:47.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:46 smithi149 ceph-mon[170394]: failed to encode map e98 with expected crc 2023-09-23T20:20:47.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:46 smithi149 ceph-mon[170394]: osd.1 [v2:172.21.15.18:6810/1422592526,v1:172.21.15.18:6811/1422592526] boot 2023-09-23T20:20:47.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:46 smithi149 ceph-mon[170394]: osdmap e99: 8 total, 8 up, 8 in 2023-09-23T20:20:47.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:46 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T20:20:47.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:46 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:20:47.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:46 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:20:47.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:46 smithi149 ceph-mon[170394]: osdmap e100: 8 total, 8 up, 8 in 2023-09-23T20:20:48.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:47 smithi018 ceph-mon[184144]: failed to encode map e100 with expected crc 2023-09-23T20:20:48.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:47 smithi018 ceph-mon[184144]: failed to encode map e100 with expected crc 2023-09-23T20:20:48.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:47 smithi018 ceph-mon[184144]: failed to encode map e100 with expected crc 2023-09-23T20:20:48.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:47 smithi018 ceph-mon[184144]: failed to encode map e100 with expected crc 2023-09-23T20:20:48.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:47 smithi018 ceph-mon[184144]: failed to encode map e100 with expected crc 2023-09-23T20:20:48.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:47 smithi018 ceph-mon[184144]: failed to encode map e100 with expected crc 2023-09-23T20:20:48.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:47 smithi018 ceph-mon[184144]: failed to encode map e100 with expected crc 2023-09-23T20:20:48.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:47 smithi018 ceph-mon[184144]: failed to encode map e100 with expected crc 2023-09-23T20:20:48.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:47 smithi018 ceph-mon[184144]: failed to encode map e100 with expected crc 2023-09-23T20:20:48.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:47 smithi018 ceph-mon[184144]: failed to encode map e100 with expected crc 2023-09-23T20:20:48.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:47 smithi018 ceph-mon[184144]: failed to encode map e100 with expected crc 2023-09-23T20:20:48.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:47 smithi018 ceph-mon[184144]: failed to encode map e100 with expected crc 2023-09-23T20:20:48.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:47 smithi018 ceph-mon[184144]: failed to encode map e100 with expected crc 2023-09-23T20:20:48.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:47 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3314754443' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:20:48.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:47 smithi018 ceph-mon[179647]: failed to encode map e100 with expected crc 2023-09-23T20:20:48.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:47 smithi018 ceph-mon[179647]: failed to encode map e100 with expected crc 2023-09-23T20:20:48.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:47 smithi018 ceph-mon[179647]: failed to encode map e100 with expected crc 2023-09-23T20:20:48.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:47 smithi018 ceph-mon[179647]: failed to encode map e100 with expected crc 2023-09-23T20:20:48.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:47 smithi018 ceph-mon[179647]: failed to encode map e100 with expected crc 2023-09-23T20:20:48.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:47 smithi018 ceph-mon[179647]: failed to encode map e100 with expected crc 2023-09-23T20:20:48.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:47 smithi018 ceph-mon[179647]: failed to encode map e100 with expected crc 2023-09-23T20:20:48.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:47 smithi018 ceph-mon[179647]: failed to encode map e100 with expected crc 2023-09-23T20:20:48.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:47 smithi018 ceph-mon[179647]: failed to encode map e100 with expected crc 2023-09-23T20:20:48.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:47 smithi018 ceph-mon[179647]: failed to encode map e100 with expected crc 2023-09-23T20:20:48.144 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:47 smithi018 ceph-mon[179647]: failed to encode map e100 with expected crc 2023-09-23T20:20:48.144 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:47 smithi018 ceph-mon[179647]: failed to encode map e100 with expected crc 2023-09-23T20:20:48.144 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:47 smithi018 ceph-mon[179647]: failed to encode map e100 with expected crc 2023-09-23T20:20:48.144 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:47 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3314754443' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:20:48.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:47 smithi149 ceph-mon[170394]: failed to encode map e100 with expected crc 2023-09-23T20:20:48.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:47 smithi149 ceph-mon[170394]: failed to encode map e100 with expected crc 2023-09-23T20:20:48.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:47 smithi149 ceph-mon[170394]: failed to encode map e100 with expected crc 2023-09-23T20:20:48.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:47 smithi149 ceph-mon[170394]: failed to encode map e100 with expected crc 2023-09-23T20:20:48.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:47 smithi149 ceph-mon[170394]: failed to encode map e100 with expected crc 2023-09-23T20:20:48.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:47 smithi149 ceph-mon[170394]: failed to encode map e100 with expected crc 2023-09-23T20:20:48.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:47 smithi149 ceph-mon[170394]: failed to encode map e100 with expected crc 2023-09-23T20:20:48.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:47 smithi149 ceph-mon[170394]: failed to encode map e100 with expected crc 2023-09-23T20:20:48.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:47 smithi149 ceph-mon[170394]: failed to encode map e100 with expected crc 2023-09-23T20:20:48.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:47 smithi149 ceph-mon[170394]: failed to encode map e100 with expected crc 2023-09-23T20:20:48.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:47 smithi149 ceph-mon[170394]: failed to encode map e100 with expected crc 2023-09-23T20:20:48.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:47 smithi149 ceph-mon[170394]: failed to encode map e100 with expected crc 2023-09-23T20:20:48.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:47 smithi149 ceph-mon[170394]: failed to encode map e100 with expected crc 2023-09-23T20:20:48.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:47 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3314754443' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:20:49.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:48 smithi149 ceph-mon[170394]: pgmap v157: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 121 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-23T20:20:49.260 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:48 smithi018 ceph-mon[184144]: pgmap v157: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 121 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-23T20:20:49.260 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:48 smithi018 ceph-mon[179647]: pgmap v157: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 121 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-23T20:20:49.641 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:20:49 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: 2023-09-23T20:20:49.258+0000 7f953be6a700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2023-09-23T20:20:50.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:50 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:20:50.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:50 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:20:50.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:50 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:20:50.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:50 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:20:50.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:50 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:20:50.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:50 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:20:50.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:50 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:20:50.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:50 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:20:50.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:50 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:20:50.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:50 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T20:20:50.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:50 smithi018 ceph-mon[179647]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T20:20:50.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:50 smithi018 ceph-mon[179647]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2023-09-23T20:20:50.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:50 smithi018 ceph-mon[179647]: pgmap v158: 73 pgs: 4 active+undersized+degraded, 25 active+undersized, 44 active+clean; 4.6 MiB data, 121 MiB used, 715 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-09-23T20:20:50.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:50 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:20:50.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:50 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:20:50.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:50 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:20:50.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:50 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:20:50.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:50 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:20:50.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:50 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:20:50.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:50 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:20:50.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:50 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:20:50.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:50 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:20:50.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:50 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:20:50.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:50 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T20:20:50.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:50 smithi018 ceph-mon[184144]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T20:20:50.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:50 smithi018 ceph-mon[184144]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2023-09-23T20:20:50.394 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:50 smithi018 ceph-mon[184144]: pgmap v158: 73 pgs: 4 active+undersized+degraded, 25 active+undersized, 44 active+clean; 4.6 MiB data, 121 MiB used, 715 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-09-23T20:20:50.394 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:50 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:20:50.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:50 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:20:50.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:50 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:20:50.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:50 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:20:50.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:50 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:20:50.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:50 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:20:50.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:50 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:20:50.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:50 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:20:50.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:50 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:20:50.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:50 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:20:50.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:50 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T20:20:50.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:50 smithi149 ceph-mon[170394]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T20:20:50.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:50 smithi149 ceph-mon[170394]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2023-09-23T20:20:50.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:50 smithi149 ceph-mon[170394]: pgmap v158: 73 pgs: 4 active+undersized+degraded, 25 active+undersized, 44 active+clean; 4.6 MiB data, 121 MiB used, 715 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-09-23T20:20:50.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:50 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:20:51.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:51 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 4/84 objects degraded (4.762%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T20:20:51.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:51 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1705509469' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:20:51.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:51 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 4/84 objects degraded (4.762%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T20:20:51.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:51 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1705509469' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:20:51.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:51 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 4/84 objects degraded (4.762%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T20:20:51.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:51 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1705509469' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:20:52.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:52 smithi018 ceph-mon[184144]: pgmap v159: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:20:52.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:52 smithi018 ceph-mon[179647]: pgmap v159: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:20:52.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:52 smithi149 ceph-mon[170394]: pgmap v159: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:20:53.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:53 smithi018 ceph-mon[179647]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4/84 objects degraded (4.762%), 4 pgs degraded) 2023-09-23T20:20:53.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:53 smithi018 ceph-mon[179647]: Cluster is now healthy 2023-09-23T20:20:53.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:53 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/137193257' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:20:53.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:53 smithi018 ceph-mon[184144]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4/84 objects degraded (4.762%), 4 pgs degraded) 2023-09-23T20:20:53.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:53 smithi018 ceph-mon[184144]: Cluster is now healthy 2023-09-23T20:20:53.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:53 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/137193257' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:20:53.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:53 smithi149 ceph-mon[170394]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4/84 objects degraded (4.762%), 4 pgs degraded) 2023-09-23T20:20:53.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:53 smithi149 ceph-mon[170394]: Cluster is now healthy 2023-09-23T20:20:53.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:53 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/137193257' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:20:54.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:54 smithi018 ceph-mon[179647]: pgmap v160: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:20:54.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:54 smithi018 ceph-mon[184144]: pgmap v160: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:20:54.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:54 smithi149 ceph-mon[170394]: pgmap v160: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:20:55.391 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:20:54 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:20:54] "GET /metrics HTTP/1.1" 200 34040 "" "Prometheus/2.43.0" 2023-09-23T20:20:55.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:55 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/704212989' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:20:55.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:55 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/704212989' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:20:55.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:55 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/704212989' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:20:57.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:56 smithi018 ceph-mon[184144]: pgmap v161: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:20:57.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:56 smithi018 ceph-mon[179647]: pgmap v161: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:20:57.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:56 smithi149 ceph-mon[170394]: pgmap v161: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:20:58.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:57 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3815701774' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:20:58.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:57 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3815701774' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:20:58.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:57 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3815701774' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:20:59.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:20:58 smithi018 ceph-mon[184144]: pgmap v162: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:20:59.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:20:58 smithi018 ceph-mon[179647]: pgmap v162: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:20:59.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:20:58 smithi149 ceph-mon[170394]: pgmap v162: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:21:00.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:00 smithi149 ceph-mon[170394]: pgmap v163: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:21:00.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:00 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:21:00.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:00 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:21:00.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:00 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:21:00.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:00 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2324237952' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:21:00.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:00 smithi018 ceph-mon[184144]: pgmap v163: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:21:00.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:00 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:21:00.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:00 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:21:00.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:00 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:21:00.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:00 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2324237952' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:21:00.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:00 smithi018 ceph-mon[179647]: pgmap v163: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:21:00.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:00 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:21:00.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:00 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:21:00.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:00 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:21:00.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:00 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2324237952' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:21:03.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:02 smithi018 ceph-mon[184144]: pgmap v164: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:21:03.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:02 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2028205825' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:21:03.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:02 smithi018 ceph-mon[179647]: pgmap v164: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:21:03.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:02 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2028205825' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:21:03.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:02 smithi149 ceph-mon[170394]: pgmap v164: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:21:03.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:02 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2028205825' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:21:05.141 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:21:04 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:21:04] "GET /metrics HTTP/1.1" 200 34032 "" "Prometheus/2.43.0" 2023-09-23T20:21:05.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:04 smithi018 ceph-mon[179647]: pgmap v165: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:21:05.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:04 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3261101853' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:21:05.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:04 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T20:21:05.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:04 smithi018 ceph-mon[184144]: pgmap v165: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:21:05.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:04 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3261101853' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:21:05.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:04 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T20:21:05.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:04 smithi149 ceph-mon[170394]: pgmap v165: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:21:05.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:04 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3261101853' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:21:05.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:04 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T20:21:05.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:05 smithi018 ceph-mon[179647]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T20:21:05.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:05 smithi018 ceph-mon[179647]: Upgrade: osd.2 is safe to restart 2023-09-23T20:21:05.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:05 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:21:05.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:05 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-09-23T20:21:05.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:05 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:21:05.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:05 smithi018 ceph-mon[184144]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T20:21:05.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:05 smithi018 ceph-mon[184144]: Upgrade: osd.2 is safe to restart 2023-09-23T20:21:05.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:05 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:21:05.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:05 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-09-23T20:21:05.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:05 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:21:06.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:05 smithi149 ceph-mon[170394]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T20:21:06.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:05 smithi149 ceph-mon[170394]: Upgrade: osd.2 is safe to restart 2023-09-23T20:21:06.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:05 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:21:06.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:05 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-09-23T20:21:06.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:05 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:21:06.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:06 smithi018 ceph-mon[184144]: pgmap v166: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:21:06.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:06 smithi018 ceph-mon[184144]: Upgrade: Updating osd.2 2023-09-23T20:21:06.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:06 smithi018 ceph-mon[184144]: Deploying daemon osd.2 on smithi018 2023-09-23T20:21:06.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:06 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2521789132' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:21:06.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:06 smithi018 ceph-mon[179647]: pgmap v166: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:21:06.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:06 smithi018 ceph-mon[179647]: Upgrade: Updating osd.2 2023-09-23T20:21:06.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:06 smithi018 ceph-mon[179647]: Deploying daemon osd.2 on smithi018 2023-09-23T20:21:06.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:06 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2521789132' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:21:07.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:06 smithi149 ceph-mon[170394]: pgmap v166: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:21:07.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:06 smithi149 ceph-mon[170394]: Upgrade: Updating osd.2 2023-09-23T20:21:07.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:06 smithi149 ceph-mon[170394]: Deploying daemon osd.2 on smithi018 2023-09-23T20:21:07.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:06 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2521789132' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:21:08.141 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 23 20:21:08 smithi018 systemd[1]: Stopping Ceph osd.2 for a3b3b17e-5a4c-11ee-8db4-212e2dc638e7... 2023-09-23T20:21:08.641 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 23 20:21:08 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-2[129269]: 2023-09-23T20:21:08.349+0000 7f66818f3700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T20:21:08.641 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 23 20:21:08 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-2[129269]: 2023-09-23T20:21:08.349+0000 7f66818f3700 -1 osd.2 100 *** Got signal Terminated *** 2023-09-23T20:21:08.641 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 23 20:21:08 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-2[129269]: 2023-09-23T20:21:08.349+0000 7f66818f3700 -1 osd.2 100 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T20:21:09.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:08 smithi018 ceph-mon[179647]: pgmap v167: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:21:09.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:08 smithi018 ceph-mon[179647]: osd.2 marked itself down and dead 2023-09-23T20:21:09.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:08 smithi018 ceph-mon[184144]: pgmap v167: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:21:09.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:08 smithi018 ceph-mon[184144]: osd.2 marked itself down and dead 2023-09-23T20:21:09.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:08 smithi149 ceph-mon[170394]: pgmap v167: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:21:09.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:08 smithi149 ceph-mon[170394]: osd.2 marked itself down and dead 2023-09-23T20:21:09.850 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 23 20:21:09 smithi018 bash[206579]: ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-2 2023-09-23T20:21:09.852 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:09 smithi018 ceph-mon[179647]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T20:21:09.852 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:09 smithi018 ceph-mon[179647]: failed to encode map e101 with expected crc 2023-09-23T20:21:09.852 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:09 smithi018 ceph-mon[179647]: osdmap e101: 8 total, 7 up, 8 in 2023-09-23T20:21:09.852 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:09 smithi018 ceph-mon[179647]: failed to encode map e101 with expected crc 2023-09-23T20:21:09.852 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:09 smithi018 ceph-mon[179647]: failed to encode map e101 with expected crc 2023-09-23T20:21:09.852 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:09 smithi018 ceph-mon[179647]: failed to encode map e101 with expected crc 2023-09-23T20:21:09.853 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:09 smithi018 ceph-mon[179647]: failed to encode map e101 with expected crc 2023-09-23T20:21:09.853 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:09 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/4164352327' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:21:09.853 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:09 smithi018 ceph-mon[179647]: osdmap e102: 8 total, 7 up, 8 in 2023-09-23T20:21:09.853 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:09 smithi018 ceph-mon[184144]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T20:21:09.853 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:09 smithi018 ceph-mon[184144]: failed to encode map e101 with expected crc 2023-09-23T20:21:09.853 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:09 smithi018 ceph-mon[184144]: osdmap e101: 8 total, 7 up, 8 in 2023-09-23T20:21:09.853 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:09 smithi018 ceph-mon[184144]: failed to encode map e101 with expected crc 2023-09-23T20:21:09.853 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:09 smithi018 ceph-mon[184144]: failed to encode map e101 with expected crc 2023-09-23T20:21:09.853 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:09 smithi018 ceph-mon[184144]: failed to encode map e101 with expected crc 2023-09-23T20:21:09.853 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:09 smithi018 ceph-mon[184144]: failed to encode map e101 with expected crc 2023-09-23T20:21:09.854 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:09 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/4164352327' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:21:09.854 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:09 smithi018 ceph-mon[184144]: osdmap e102: 8 total, 7 up, 8 in 2023-09-23T20:21:10.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:09 smithi149 ceph-mon[170394]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T20:21:10.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:09 smithi149 ceph-mon[170394]: failed to encode map e101 with expected crc 2023-09-23T20:21:10.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:09 smithi149 ceph-mon[170394]: osdmap e101: 8 total, 7 up, 8 in 2023-09-23T20:21:10.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:09 smithi149 ceph-mon[170394]: failed to encode map e101 with expected crc 2023-09-23T20:21:10.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:09 smithi149 ceph-mon[170394]: failed to encode map e101 with expected crc 2023-09-23T20:21:10.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:09 smithi149 ceph-mon[170394]: failed to encode map e101 with expected crc 2023-09-23T20:21:10.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:09 smithi149 ceph-mon[170394]: failed to encode map e101 with expected crc 2023-09-23T20:21:10.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:09 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/4164352327' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:21:10.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:09 smithi149 ceph-mon[170394]: osdmap e102: 8 total, 7 up, 8 in 2023-09-23T20:21:10.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:10 smithi018 ceph-mon[179647]: pgmap v169: 73 pgs: 3 stale+active+clean, 70 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:21:10.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:10 smithi018 ceph-mon[179647]: failed to encode map e102 with expected crc 2023-09-23T20:21:10.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:10 smithi018 ceph-mon[179647]: failed to encode map e102 with expected crc 2023-09-23T20:21:10.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:10 smithi018 ceph-mon[179647]: failed to encode map e102 with expected crc 2023-09-23T20:21:10.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:10 smithi018 ceph-mon[179647]: failed to encode map e102 with expected crc 2023-09-23T20:21:10.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:10 smithi018 ceph-mon[179647]: failed to encode map e102 with expected crc 2023-09-23T20:21:10.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:10 smithi018 ceph-mon[179647]: failed to encode map e102 with expected crc 2023-09-23T20:21:10.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:10 smithi018 ceph-mon[179647]: failed to encode map e102 with expected crc 2023-09-23T20:21:10.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:10 smithi018 ceph-mon[179647]: failed to encode map e102 with expected crc 2023-09-23T20:21:10.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:10 smithi018 ceph-mon[184144]: pgmap v169: 73 pgs: 3 stale+active+clean, 70 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:21:10.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:10 smithi018 ceph-mon[184144]: failed to encode map e102 with expected crc 2023-09-23T20:21:10.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:10 smithi018 ceph-mon[184144]: failed to encode map e102 with expected crc 2023-09-23T20:21:10.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:10 smithi018 ceph-mon[184144]: failed to encode map e102 with expected crc 2023-09-23T20:21:10.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:10 smithi018 ceph-mon[184144]: failed to encode map e102 with expected crc 2023-09-23T20:21:10.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:10 smithi018 ceph-mon[184144]: failed to encode map e102 with expected crc 2023-09-23T20:21:10.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:10 smithi018 ceph-mon[184144]: failed to encode map e102 with expected crc 2023-09-23T20:21:10.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:10 smithi018 ceph-mon[184144]: failed to encode map e102 with expected crc 2023-09-23T20:21:10.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:10 smithi018 ceph-mon[184144]: failed to encode map e102 with expected crc 2023-09-23T20:21:11.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:10 smithi149 ceph-mon[170394]: pgmap v169: 73 pgs: 3 stale+active+clean, 70 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:21:11.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:10 smithi149 ceph-mon[170394]: failed to encode map e102 with expected crc 2023-09-23T20:21:11.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:10 smithi149 ceph-mon[170394]: failed to encode map e102 with expected crc 2023-09-23T20:21:11.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:10 smithi149 ceph-mon[170394]: failed to encode map e102 with expected crc 2023-09-23T20:21:11.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:10 smithi149 ceph-mon[170394]: failed to encode map e102 with expected crc 2023-09-23T20:21:11.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:10 smithi149 ceph-mon[170394]: failed to encode map e102 with expected crc 2023-09-23T20:21:11.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:10 smithi149 ceph-mon[170394]: failed to encode map e102 with expected crc 2023-09-23T20:21:11.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:10 smithi149 ceph-mon[170394]: failed to encode map e102 with expected crc 2023-09-23T20:21:11.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:10 smithi149 ceph-mon[170394]: failed to encode map e102 with expected crc 2023-09-23T20:21:11.392 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 23 20:21:10 smithi018 systemd[1]: ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7@osd.2.service: Succeeded. 2023-09-23T20:21:11.392 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 23 20:21:10 smithi018 systemd[1]: Stopped Ceph osd.2 for a3b3b17e-5a4c-11ee-8db4-212e2dc638e7. 2023-09-23T20:21:11.392 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 23 20:21:11 smithi018 systemd[1]: Starting Ceph osd.2 for a3b3b17e-5a4c-11ee-8db4-212e2dc638e7... 2023-09-23T20:21:12.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:11 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3932319736' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:21:12.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:11 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3932319736' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:21:12.142 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 23 20:21:11 smithi018 bash[206903]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-09-23T20:21:12.142 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 23 20:21:11 smithi018 bash[206903]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2023-09-23T20:21:12.142 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 23 20:21:11 smithi018 bash[206903]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_2 --path /var/lib/ceph/osd/ceph-2 --no-mon-config 2023-09-23T20:21:12.142 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 23 20:21:11 smithi018 bash[206903]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_2 /var/lib/ceph/osd/ceph-2/block 2023-09-23T20:21:12.142 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 23 20:21:11 smithi018 bash[206903]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-2/block 2023-09-23T20:21:12.142 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 23 20:21:11 smithi018 bash[206903]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2023-09-23T20:21:12.142 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 23 20:21:11 smithi018 bash[206903]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2023-09-23T20:21:12.142 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 23 20:21:11 smithi018 bash[206903]: --> ceph-volume lvm activate successful for osd ID: 2 2023-09-23T20:21:12.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:11 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3932319736' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:21:12.857 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 23 20:21:12 smithi018 podman[207170]: 2023-09-23T20:21:12.857 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 23 20:21:12 smithi018 ceph-osd[207210]: -- 172.21.15.18:0/2694038624 <== mon.0 v2:172.21.15.18:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x560dd5ea24e0 con 0x560dd5e73400 2023-09-23T20:21:12.858 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 23 20:21:12 smithi018 bash[207170]: 4b4b70078c4e08a676be9690d301fc1ee19291646ec992db2daeb98e0b2523d3 2023-09-23T20:21:12.858 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 23 20:21:12 smithi018 systemd[1]: Started Ceph osd.2 for a3b3b17e-5a4c-11ee-8db4-212e2dc638e7. 2023-09-23T20:21:13.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:12 smithi018 ceph-mon[184144]: pgmap v171: 73 pgs: 3 active+undersized+degraded, 14 active+undersized, 56 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-09-23T20:21:13.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:12 smithi018 ceph-mon[184144]: Health check failed: Degraded data redundancy: 4/84 objects degraded (4.762%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T20:21:13.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:12 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:21:13.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:12 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:21:13.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:12 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:21:13.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:12 smithi018 ceph-mon[179647]: pgmap v171: 73 pgs: 3 active+undersized+degraded, 14 active+undersized, 56 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-09-23T20:21:13.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:12 smithi018 ceph-mon[179647]: Health check failed: Degraded data redundancy: 4/84 objects degraded (4.762%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T20:21:13.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:12 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:21:13.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:12 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:21:13.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:12 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:21:13.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:12 smithi149 ceph-mon[170394]: pgmap v171: 73 pgs: 3 active+undersized+degraded, 14 active+undersized, 56 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-09-23T20:21:13.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:12 smithi149 ceph-mon[170394]: Health check failed: Degraded data redundancy: 4/84 objects degraded (4.762%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T20:21:13.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:12 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:21:13.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:12 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:21:13.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:12 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:21:13.641 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 23 20:21:13 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-2[207186]: 2023-09-23T20:21:13.257+0000 7f1682d33640 -1 Falling back to public interface 2023-09-23T20:21:14.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:13 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2812036139' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:21:14.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:13 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2812036139' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:21:14.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:13 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2812036139' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:21:15.141 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:21:14 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:21:14] "GET /metrics HTTP/1.1" 200 34032 "" "Prometheus/2.43.0" 2023-09-23T20:21:15.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:14 smithi018 ceph-mon[184144]: pgmap v172: 73 pgs: 4 active+undersized+degraded, 15 active+undersized, 54 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-09-23T20:21:15.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:14 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:21:15.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:14 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:21:15.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:14 smithi018 ceph-mon[179647]: pgmap v172: 73 pgs: 4 active+undersized+degraded, 15 active+undersized, 54 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-09-23T20:21:15.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:14 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:21:15.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:14 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:21:15.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:14 smithi149 ceph-mon[170394]: pgmap v172: 73 pgs: 4 active+undersized+degraded, 15 active+undersized, 54 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-09-23T20:21:15.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:14 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:21:15.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:14 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:21:16.391 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 23 20:21:16 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-2[207186]: 2023-09-23T20:21:16.026+0000 7f1682d33640 -1 osd.2 100 log_to_monitors true 2023-09-23T20:21:17.141 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 23 20:21:16 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-2[207186]: 2023-09-23T20:21:16.885+0000 7f167299d700 -1 osd.2 100 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T20:21:17.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:16 smithi018 ceph-mon[179647]: pgmap v173: 73 pgs: 4 active+undersized+degraded, 15 active+undersized, 54 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-09-23T20:21:17.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:16 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1232117961' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:21:17.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:16 smithi018 ceph-mon[179647]: from='osd.2 [v2:172.21.15.18:6818/2061051826,v1:172.21.15.18:6819/2061051826]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T20:21:17.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:16 smithi018 ceph-mon[184144]: pgmap v173: 73 pgs: 4 active+undersized+degraded, 15 active+undersized, 54 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-09-23T20:21:17.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:16 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1232117961' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:21:17.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:16 smithi018 ceph-mon[184144]: from='osd.2 [v2:172.21.15.18:6818/2061051826,v1:172.21.15.18:6819/2061051826]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T20:21:17.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:16 smithi149 ceph-mon[170394]: pgmap v173: 73 pgs: 4 active+undersized+degraded, 15 active+undersized, 54 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-09-23T20:21:17.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:16 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1232117961' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:21:17.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:16 smithi149 ceph-mon[170394]: from='osd.2 [v2:172.21.15.18:6818/2061051826,v1:172.21.15.18:6819/2061051826]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T20:21:18.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:17 smithi018 ceph-mon[179647]: failed to encode map e103 with expected crc 2023-09-23T20:21:18.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:17 smithi018 ceph-mon[179647]: from='osd.2 [v2:172.21.15.18:6818/2061051826,v1:172.21.15.18:6819/2061051826]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-09-23T20:21:18.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:17 smithi018 ceph-mon[179647]: osdmap e103: 8 total, 7 up, 8 in 2023-09-23T20:21:18.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:17 smithi018 ceph-mon[179647]: from='osd.2 [v2:172.21.15.18:6818/2061051826,v1:172.21.15.18:6819/2061051826]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi018", "root=default"]}]: dispatch 2023-09-23T20:21:18.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:17 smithi018 ceph-mon[179647]: failed to encode map e103 with expected crc 2023-09-23T20:21:18.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:17 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:21:18.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:17 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:21:18.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:17 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:21:18.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:17 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:21:18.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:17 smithi018 ceph-mon[184144]: failed to encode map e103 with expected crc 2023-09-23T20:21:18.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:17 smithi018 ceph-mon[184144]: from='osd.2 [v2:172.21.15.18:6818/2061051826,v1:172.21.15.18:6819/2061051826]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-09-23T20:21:18.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:17 smithi018 ceph-mon[184144]: osdmap e103: 8 total, 7 up, 8 in 2023-09-23T20:21:18.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:17 smithi018 ceph-mon[184144]: from='osd.2 [v2:172.21.15.18:6818/2061051826,v1:172.21.15.18:6819/2061051826]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi018", "root=default"]}]: dispatch 2023-09-23T20:21:18.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:17 smithi018 ceph-mon[184144]: failed to encode map e103 with expected crc 2023-09-23T20:21:18.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:17 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:21:18.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:17 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:21:18.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:17 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:21:18.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:17 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:21:18.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:17 smithi149 ceph-mon[170394]: failed to encode map e103 with expected crc 2023-09-23T20:21:18.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:17 smithi149 ceph-mon[170394]: from='osd.2 [v2:172.21.15.18:6818/2061051826,v1:172.21.15.18:6819/2061051826]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-09-23T20:21:18.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:17 smithi149 ceph-mon[170394]: osdmap e103: 8 total, 7 up, 8 in 2023-09-23T20:21:18.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:17 smithi149 ceph-mon[170394]: from='osd.2 [v2:172.21.15.18:6818/2061051826,v1:172.21.15.18:6819/2061051826]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi018", "root=default"]}]: dispatch 2023-09-23T20:21:18.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:17 smithi149 ceph-mon[170394]: failed to encode map e103 with expected crc 2023-09-23T20:21:18.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:17 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:21:18.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:17 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:21:18.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:17 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:21:18.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:17 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:21:19.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:18 smithi018 ceph-mon[184144]: failed to encode map e103 with expected crc 2023-09-23T20:21:19.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:18 smithi018 ceph-mon[184144]: failed to encode map e103 with expected crc 2023-09-23T20:21:19.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:18 smithi018 ceph-mon[184144]: pgmap v175: 73 pgs: 4 active+undersized+degraded, 15 active+undersized, 54 active+clean; 4.6 MiB data, 138 MiB used, 715 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-09-23T20:21:19.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:18 smithi018 ceph-mon[184144]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T20:21:19.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:18 smithi018 ceph-mon[184144]: failed to encode map e104 with expected crc 2023-09-23T20:21:19.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:18 smithi018 ceph-mon[184144]: failed to encode map e104 with expected crc 2023-09-23T20:21:19.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:18 smithi018 ceph-mon[184144]: osd.2 [v2:172.21.15.18:6818/2061051826,v1:172.21.15.18:6819/2061051826] boot 2023-09-23T20:21:19.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:18 smithi018 ceph-mon[184144]: osdmap e104: 8 total, 8 up, 8 in 2023-09-23T20:21:19.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:18 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T20:21:19.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:18 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3948820655' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:21:19.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:18 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:21:19.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:18 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:21:19.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:18 smithi018 ceph-mon[179647]: failed to encode map e103 with expected crc 2023-09-23T20:21:19.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:18 smithi018 ceph-mon[179647]: failed to encode map e103 with expected crc 2023-09-23T20:21:19.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:18 smithi018 ceph-mon[179647]: pgmap v175: 73 pgs: 4 active+undersized+degraded, 15 active+undersized, 54 active+clean; 4.6 MiB data, 138 MiB used, 715 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-09-23T20:21:19.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:18 smithi018 ceph-mon[179647]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T20:21:19.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:18 smithi018 ceph-mon[179647]: failed to encode map e104 with expected crc 2023-09-23T20:21:19.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:18 smithi018 ceph-mon[179647]: failed to encode map e104 with expected crc 2023-09-23T20:21:19.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:18 smithi018 ceph-mon[179647]: osd.2 [v2:172.21.15.18:6818/2061051826,v1:172.21.15.18:6819/2061051826] boot 2023-09-23T20:21:19.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:18 smithi018 ceph-mon[179647]: osdmap e104: 8 total, 8 up, 8 in 2023-09-23T20:21:19.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:18 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T20:21:19.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:18 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3948820655' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:21:19.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:18 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:21:19.144 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:18 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:21:19.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:18 smithi149 ceph-mon[170394]: failed to encode map e103 with expected crc 2023-09-23T20:21:19.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:18 smithi149 ceph-mon[170394]: failed to encode map e103 with expected crc 2023-09-23T20:21:19.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:18 smithi149 ceph-mon[170394]: pgmap v175: 73 pgs: 4 active+undersized+degraded, 15 active+undersized, 54 active+clean; 4.6 MiB data, 138 MiB used, 715 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-09-23T20:21:19.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:18 smithi149 ceph-mon[170394]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T20:21:19.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:18 smithi149 ceph-mon[170394]: failed to encode map e104 with expected crc 2023-09-23T20:21:19.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:18 smithi149 ceph-mon[170394]: failed to encode map e104 with expected crc 2023-09-23T20:21:19.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:18 smithi149 ceph-mon[170394]: osd.2 [v2:172.21.15.18:6818/2061051826,v1:172.21.15.18:6819/2061051826] boot 2023-09-23T20:21:19.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:18 smithi149 ceph-mon[170394]: osdmap e104: 8 total, 8 up, 8 in 2023-09-23T20:21:19.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:18 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T20:21:19.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:18 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3948820655' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:21:19.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:18 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:21:19.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:18 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:21:19.889 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:19 smithi018 ceph-mon[179647]: failed to encode map e105 with expected crc 2023-09-23T20:21:19.890 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:19 smithi018 ceph-mon[179647]: failed to encode map e105 with expected crc 2023-09-23T20:21:19.890 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:19 smithi018 ceph-mon[179647]: failed to encode map e105 with expected crc 2023-09-23T20:21:19.890 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:19 smithi018 ceph-mon[179647]: osdmap e105: 8 total, 8 up, 8 in 2023-09-23T20:21:19.890 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:19 smithi018 ceph-mon[179647]: failed to encode map e105 with expected crc 2023-09-23T20:21:19.890 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:19 smithi018 ceph-mon[179647]: failed to encode map e105 with expected crc 2023-09-23T20:21:19.890 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:19 smithi018 ceph-mon[179647]: failed to encode map e105 with expected crc 2023-09-23T20:21:19.890 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:19 smithi018 ceph-mon[179647]: failed to encode map e105 with expected crc 2023-09-23T20:21:19.890 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:19 smithi018 ceph-mon[179647]: failed to encode map e105 with expected crc 2023-09-23T20:21:19.890 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:19 smithi018 ceph-mon[179647]: failed to encode map e105 with expected crc 2023-09-23T20:21:19.890 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:19 smithi018 ceph-mon[179647]: failed to encode map e105 with expected crc 2023-09-23T20:21:19.890 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:19 smithi018 ceph-mon[179647]: failed to encode map e105 with expected crc 2023-09-23T20:21:20.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:19 smithi018 ceph-mon[184144]: failed to encode map e105 with expected crc 2023-09-23T20:21:20.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:19 smithi018 ceph-mon[184144]: failed to encode map e105 with expected crc 2023-09-23T20:21:20.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:19 smithi018 ceph-mon[184144]: failed to encode map e105 with expected crc 2023-09-23T20:21:20.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:19 smithi018 ceph-mon[184144]: osdmap e105: 8 total, 8 up, 8 in 2023-09-23T20:21:20.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:19 smithi018 ceph-mon[184144]: failed to encode map e105 with expected crc 2023-09-23T20:21:20.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:19 smithi018 ceph-mon[184144]: failed to encode map e105 with expected crc 2023-09-23T20:21:20.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:19 smithi018 ceph-mon[184144]: failed to encode map e105 with expected crc 2023-09-23T20:21:20.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:19 smithi018 ceph-mon[184144]: failed to encode map e105 with expected crc 2023-09-23T20:21:20.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:19 smithi018 ceph-mon[184144]: failed to encode map e105 with expected crc 2023-09-23T20:21:20.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:19 smithi018 ceph-mon[184144]: failed to encode map e105 with expected crc 2023-09-23T20:21:20.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:19 smithi018 ceph-mon[184144]: failed to encode map e105 with expected crc 2023-09-23T20:21:20.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:19 smithi018 ceph-mon[184144]: failed to encode map e105 with expected crc 2023-09-23T20:21:20.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:19 smithi149 ceph-mon[170394]: failed to encode map e105 with expected crc 2023-09-23T20:21:20.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:19 smithi149 ceph-mon[170394]: failed to encode map e105 with expected crc 2023-09-23T20:21:20.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:19 smithi149 ceph-mon[170394]: failed to encode map e105 with expected crc 2023-09-23T20:21:20.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:19 smithi149 ceph-mon[170394]: osdmap e105: 8 total, 8 up, 8 in 2023-09-23T20:21:20.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:19 smithi149 ceph-mon[170394]: failed to encode map e105 with expected crc 2023-09-23T20:21:20.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:19 smithi149 ceph-mon[170394]: failed to encode map e105 with expected crc 2023-09-23T20:21:20.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:19 smithi149 ceph-mon[170394]: failed to encode map e105 with expected crc 2023-09-23T20:21:20.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:19 smithi149 ceph-mon[170394]: failed to encode map e105 with expected crc 2023-09-23T20:21:20.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:19 smithi149 ceph-mon[170394]: failed to encode map e105 with expected crc 2023-09-23T20:21:20.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:19 smithi149 ceph-mon[170394]: failed to encode map e105 with expected crc 2023-09-23T20:21:20.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:19 smithi149 ceph-mon[170394]: failed to encode map e105 with expected crc 2023-09-23T20:21:20.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:19 smithi149 ceph-mon[170394]: failed to encode map e105 with expected crc 2023-09-23T20:21:21.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:20 smithi018 ceph-mon[184144]: pgmap v178: 73 pgs: 4 active+undersized+degraded, 15 active+undersized, 54 active+clean; 4.6 MiB data, 138 MiB used, 715 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-09-23T20:21:21.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:20 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3379198357' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:21:21.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:20 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 6/84 objects degraded (7.143%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T20:21:21.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:20 smithi018 ceph-mon[179647]: pgmap v178: 73 pgs: 4 active+undersized+degraded, 15 active+undersized, 54 active+clean; 4.6 MiB data, 138 MiB used, 715 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-09-23T20:21:21.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:20 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3379198357' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:21:21.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:20 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 6/84 objects degraded (7.143%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T20:21:21.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:20 smithi149 ceph-mon[170394]: pgmap v178: 73 pgs: 4 active+undersized+degraded, 15 active+undersized, 54 active+clean; 4.6 MiB data, 138 MiB used, 715 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-09-23T20:21:21.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:20 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3379198357' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:21:21.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:20 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 6/84 objects degraded (7.143%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T20:21:21.891 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:21:21 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: 2023-09-23T20:21:21.555+0000 7f953be6a700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-09-23T20:21:22.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:22 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:21:22.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:22 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:21:22.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:22 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:21:22.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:22 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:21:22.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:22 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:21:22.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:22 smithi149 ceph-mon[170394]: pgmap v179: 73 pgs: 1 active+undersized+degraded, 4 active+undersized, 68 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2023-09-23T20:21:22.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:22 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:21:22.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:22 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:21:22.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:22 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:21:22.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:22 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:21:22.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:22 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T20:21:22.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:22 smithi149 ceph-mon[170394]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T20:21:22.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:22 smithi149 ceph-mon[170394]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-09-23T20:21:22.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:22 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:21:22.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:22 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:21:22.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:22 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:21:22.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:22 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:21:22.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:22 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:21:22.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:22 smithi018 ceph-mon[184144]: pgmap v179: 73 pgs: 1 active+undersized+degraded, 4 active+undersized, 68 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2023-09-23T20:21:22.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:22 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:21:22.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:22 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:21:22.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:22 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:21:22.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:22 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:21:22.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:22 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T20:21:22.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:22 smithi018 ceph-mon[184144]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T20:21:22.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:22 smithi018 ceph-mon[184144]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-09-23T20:21:22.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:22 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:21:22.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:22 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:21:22.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:22 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:21:22.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:22 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:21:22.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:22 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:21:22.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:22 smithi018 ceph-mon[179647]: pgmap v179: 73 pgs: 1 active+undersized+degraded, 4 active+undersized, 68 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2023-09-23T20:21:22.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:22 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:21:22.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:22 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:21:22.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:22 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:21:22.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:22 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:21:22.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:22 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T20:21:22.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:22 smithi018 ceph-mon[179647]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T20:21:22.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:22 smithi018 ceph-mon[179647]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-09-23T20:21:23.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:23 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3705212623' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:21:23.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:23 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3705212623' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:21:23.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:23 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3705212623' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:21:24.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:24 smithi149 ceph-mon[170394]: pgmap v180: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:21:24.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:24 smithi018 ceph-mon[184144]: pgmap v180: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:21:24.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:24 smithi018 ceph-mon[179647]: pgmap v180: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:21:25.391 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:21:24 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:21:24] "GET /metrics HTTP/1.1" 200 34032 "" "Prometheus/2.43.0" 2023-09-23T20:21:25.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:25 smithi149 ceph-mon[170394]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/84 objects degraded (2.381%), 1 pg degraded) 2023-09-23T20:21:25.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:25 smithi149 ceph-mon[170394]: Cluster is now healthy 2023-09-23T20:21:25.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:25 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3017472496' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:21:25.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:25 smithi018 ceph-mon[179647]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/84 objects degraded (2.381%), 1 pg degraded) 2023-09-23T20:21:25.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:25 smithi018 ceph-mon[179647]: Cluster is now healthy 2023-09-23T20:21:25.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:25 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3017472496' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:21:25.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:25 smithi018 ceph-mon[184144]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/84 objects degraded (2.381%), 1 pg degraded) 2023-09-23T20:21:25.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:25 smithi018 ceph-mon[184144]: Cluster is now healthy 2023-09-23T20:21:25.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:25 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3017472496' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:21:27.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:26 smithi018 ceph-mon[179647]: pgmap v181: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:21:27.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:26 smithi018 ceph-mon[184144]: pgmap v181: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:21:27.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:26 smithi149 ceph-mon[170394]: pgmap v181: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:21:28.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:27 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1523510129' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:21:28.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:27 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1523510129' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:21:28.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:27 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1523510129' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:21:29.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:28 smithi018 ceph-mon[184144]: pgmap v182: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:21:29.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:28 smithi018 ceph-mon[179647]: pgmap v182: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:21:29.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:28 smithi149 ceph-mon[170394]: pgmap v182: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:21:30.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:30 smithi149 ceph-mon[170394]: pgmap v183: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:21:30.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:30 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:21:30.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:30 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:21:30.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:30 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:21:30.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:30 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/694412520' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:21:30.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:30 smithi018 ceph-mon[179647]: pgmap v183: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:21:30.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:30 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:21:30.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:30 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:21:30.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:30 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:21:30.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:30 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/694412520' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:21:30.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:30 smithi018 ceph-mon[184144]: pgmap v183: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:21:30.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:30 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:21:30.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:30 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:21:30.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:30 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:21:30.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:30 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/694412520' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:21:33.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:32 smithi018 ceph-mon[184144]: pgmap v184: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:21:33.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:32 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3690995707' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:21:33.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:32 smithi018 ceph-mon[179647]: pgmap v184: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:21:33.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:32 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3690995707' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:21:33.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:32 smithi149 ceph-mon[170394]: pgmap v184: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:21:33.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:32 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3690995707' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:21:35.141 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:21:34 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:21:34] "GET /metrics HTTP/1.1" 200 34030 "" "Prometheus/2.43.0" 2023-09-23T20:21:35.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:34 smithi018 ceph-mon[179647]: pgmap v185: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:21:35.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:34 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1818598327' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:21:35.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:34 smithi018 ceph-mon[184144]: pgmap v185: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:21:35.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:34 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1818598327' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:21:35.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:34 smithi149 ceph-mon[170394]: pgmap v185: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:21:35.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:34 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1818598327' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:21:37.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:36 smithi018 ceph-mon[184144]: pgmap v186: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:21:37.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:36 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T20:21:37.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:36 smithi018 ceph-mon[179647]: pgmap v186: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:21:37.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:36 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T20:21:37.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:36 smithi149 ceph-mon[170394]: pgmap v186: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:21:37.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:36 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T20:21:38.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:37 smithi018 ceph-mon[184144]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T20:21:38.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:37 smithi018 ceph-mon[184144]: Upgrade: osd.3 is safe to restart 2023-09-23T20:21:38.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:37 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/4058764980' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:21:38.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:37 smithi018 ceph-mon[179647]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T20:21:38.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:37 smithi018 ceph-mon[179647]: Upgrade: osd.3 is safe to restart 2023-09-23T20:21:38.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:37 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/4058764980' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:21:38.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:37 smithi149 ceph-mon[170394]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T20:21:38.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:37 smithi149 ceph-mon[170394]: Upgrade: osd.3 is safe to restart 2023-09-23T20:21:38.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:37 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/4058764980' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:21:39.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:38 smithi018 ceph-mon[179647]: pgmap v187: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:21:39.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:38 smithi018 ceph-mon[179647]: Upgrade: Updating osd.3 2023-09-23T20:21:39.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:38 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:21:39.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:38 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-09-23T20:21:39.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:38 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:21:39.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:38 smithi018 ceph-mon[179647]: Deploying daemon osd.3 on smithi018 2023-09-23T20:21:39.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:38 smithi018 ceph-mon[184144]: pgmap v187: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:21:39.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:38 smithi018 ceph-mon[184144]: Upgrade: Updating osd.3 2023-09-23T20:21:39.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:38 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:21:39.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:38 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-09-23T20:21:39.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:38 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:21:39.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:38 smithi018 ceph-mon[184144]: Deploying daemon osd.3 on smithi018 2023-09-23T20:21:39.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:38 smithi149 ceph-mon[170394]: pgmap v187: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:21:39.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:38 smithi149 ceph-mon[170394]: Upgrade: Updating osd.3 2023-09-23T20:21:39.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:38 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:21:39.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:38 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-09-23T20:21:39.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:38 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:21:39.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:38 smithi149 ceph-mon[170394]: Deploying daemon osd.3 on smithi018 2023-09-23T20:21:39.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:39 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1700213005' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:21:39.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:39 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1700213005' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:21:40.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:39 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1700213005' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:21:40.503 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 23 20:21:40 smithi018 systemd[1]: Stopping Ceph osd.3 for a3b3b17e-5a4c-11ee-8db4-212e2dc638e7... 2023-09-23T20:21:40.840 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 23 20:21:40 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-3[136408]: 2023-09-23T20:21:40.502+0000 7f4f4eec0700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T20:21:40.840 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 23 20:21:40 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-3[136408]: 2023-09-23T20:21:40.502+0000 7f4f4eec0700 -1 osd.3 105 *** Got signal Terminated *** 2023-09-23T20:21:40.840 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 23 20:21:40 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-3[136408]: 2023-09-23T20:21:40.502+0000 7f4f4eec0700 -1 osd.3 105 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T20:21:41.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:40 smithi018 ceph-mon[179647]: pgmap v188: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:21:41.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:40 smithi018 ceph-mon[179647]: osd.3 marked itself down and dead 2023-09-23T20:21:41.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:40 smithi018 ceph-mon[184144]: pgmap v188: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:21:41.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:40 smithi018 ceph-mon[184144]: osd.3 marked itself down and dead 2023-09-23T20:21:41.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:40 smithi149 ceph-mon[170394]: pgmap v188: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:21:41.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:40 smithi149 ceph-mon[170394]: osd.3 marked itself down and dead 2023-09-23T20:21:41.641 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 23 20:21:41 smithi018 bash[213273]: ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-3 2023-09-23T20:21:42.111 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:41 smithi018 ceph-mon[179647]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T20:21:42.111 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:41 smithi018 ceph-mon[179647]: failed to encode map e106 with expected crc 2023-09-23T20:21:42.111 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:41 smithi018 ceph-mon[179647]: osdmap e106: 8 total, 7 up, 8 in 2023-09-23T20:21:42.112 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:41 smithi018 ceph-mon[179647]: failed to encode map e106 with expected crc 2023-09-23T20:21:42.112 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:41 smithi018 ceph-mon[179647]: failed to encode map e106 with expected crc 2023-09-23T20:21:42.112 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:41 smithi018 ceph-mon[179647]: failed to encode map e106 with expected crc 2023-09-23T20:21:42.112 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:41 smithi018 ceph-mon[179647]: failed to encode map e106 with expected crc 2023-09-23T20:21:42.112 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:41 smithi018 ceph-mon[179647]: failed to encode map e106 with expected crc 2023-09-23T20:21:42.112 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:41 smithi018 ceph-mon[179647]: failed to encode map e106 with expected crc 2023-09-23T20:21:42.112 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:41 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3903569734' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:21:42.112 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:41 smithi018 ceph-mon[184144]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T20:21:42.112 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:41 smithi018 ceph-mon[184144]: failed to encode map e106 with expected crc 2023-09-23T20:21:42.112 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:41 smithi018 ceph-mon[184144]: osdmap e106: 8 total, 7 up, 8 in 2023-09-23T20:21:42.112 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:41 smithi018 ceph-mon[184144]: failed to encode map e106 with expected crc 2023-09-23T20:21:42.113 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:41 smithi018 ceph-mon[184144]: failed to encode map e106 with expected crc 2023-09-23T20:21:42.113 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:41 smithi018 ceph-mon[184144]: failed to encode map e106 with expected crc 2023-09-23T20:21:42.113 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:41 smithi018 ceph-mon[184144]: failed to encode map e106 with expected crc 2023-09-23T20:21:42.113 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:41 smithi018 ceph-mon[184144]: failed to encode map e106 with expected crc 2023-09-23T20:21:42.113 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:41 smithi018 ceph-mon[184144]: failed to encode map e106 with expected crc 2023-09-23T20:21:42.113 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:41 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3903569734' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:21:42.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:41 smithi149 ceph-mon[170394]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T20:21:42.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:41 smithi149 ceph-mon[170394]: failed to encode map e106 with expected crc 2023-09-23T20:21:42.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:41 smithi149 ceph-mon[170394]: osdmap e106: 8 total, 7 up, 8 in 2023-09-23T20:21:42.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:41 smithi149 ceph-mon[170394]: failed to encode map e106 with expected crc 2023-09-23T20:21:42.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:41 smithi149 ceph-mon[170394]: failed to encode map e106 with expected crc 2023-09-23T20:21:42.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:41 smithi149 ceph-mon[170394]: failed to encode map e106 with expected crc 2023-09-23T20:21:42.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:41 smithi149 ceph-mon[170394]: failed to encode map e106 with expected crc 2023-09-23T20:21:42.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:41 smithi149 ceph-mon[170394]: failed to encode map e106 with expected crc 2023-09-23T20:21:42.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:41 smithi149 ceph-mon[170394]: failed to encode map e106 with expected crc 2023-09-23T20:21:42.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:41 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3903569734' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:21:43.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:42 smithi018 ceph-mon[179647]: pgmap v190: 73 pgs: 13 stale+active+clean, 60 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:21:43.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:42 smithi018 ceph-mon[179647]: failed to encode map e107 with expected crc 2023-09-23T20:21:43.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:42 smithi018 ceph-mon[179647]: osdmap e107: 8 total, 7 up, 8 in 2023-09-23T20:21:43.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:42 smithi018 ceph-mon[179647]: failed to encode map e107 with expected crc 2023-09-23T20:21:43.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:42 smithi018 ceph-mon[179647]: failed to encode map e107 with expected crc 2023-09-23T20:21:43.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:42 smithi018 ceph-mon[179647]: failed to encode map e107 with expected crc 2023-09-23T20:21:43.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:42 smithi018 ceph-mon[179647]: failed to encode map e107 with expected crc 2023-09-23T20:21:43.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:42 smithi018 ceph-mon[179647]: failed to encode map e107 with expected crc 2023-09-23T20:21:43.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:42 smithi018 ceph-mon[179647]: failed to encode map e107 with expected crc 2023-09-23T20:21:43.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:42 smithi018 ceph-mon[179647]: failed to encode map e107 with expected crc 2023-09-23T20:21:43.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:42 smithi018 ceph-mon[179647]: failed to encode map e107 with expected crc 2023-09-23T20:21:43.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:42 smithi018 ceph-mon[184144]: pgmap v190: 73 pgs: 13 stale+active+clean, 60 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:21:43.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:42 smithi018 ceph-mon[184144]: failed to encode map e107 with expected crc 2023-09-23T20:21:43.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:42 smithi018 ceph-mon[184144]: osdmap e107: 8 total, 7 up, 8 in 2023-09-23T20:21:43.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:42 smithi018 ceph-mon[184144]: failed to encode map e107 with expected crc 2023-09-23T20:21:43.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:42 smithi018 ceph-mon[184144]: failed to encode map e107 with expected crc 2023-09-23T20:21:43.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:42 smithi018 ceph-mon[184144]: failed to encode map e107 with expected crc 2023-09-23T20:21:43.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:42 smithi018 ceph-mon[184144]: failed to encode map e107 with expected crc 2023-09-23T20:21:43.144 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:42 smithi018 ceph-mon[184144]: failed to encode map e107 with expected crc 2023-09-23T20:21:43.144 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:42 smithi018 ceph-mon[184144]: failed to encode map e107 with expected crc 2023-09-23T20:21:43.144 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:42 smithi018 ceph-mon[184144]: failed to encode map e107 with expected crc 2023-09-23T20:21:43.144 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:42 smithi018 ceph-mon[184144]: failed to encode map e107 with expected crc 2023-09-23T20:21:43.144 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 23 20:21:42 smithi018 systemd[1]: ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7@osd.3.service: Succeeded. 2023-09-23T20:21:43.144 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 23 20:21:42 smithi018 systemd[1]: Stopped Ceph osd.3 for a3b3b17e-5a4c-11ee-8db4-212e2dc638e7. 2023-09-23T20:21:43.144 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 23 20:21:43 smithi018 systemd[1]: Starting Ceph osd.3 for a3b3b17e-5a4c-11ee-8db4-212e2dc638e7... 2023-09-23T20:21:43.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:42 smithi149 ceph-mon[170394]: pgmap v190: 73 pgs: 13 stale+active+clean, 60 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:21:43.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:42 smithi149 ceph-mon[170394]: failed to encode map e107 with expected crc 2023-09-23T20:21:43.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:42 smithi149 ceph-mon[170394]: osdmap e107: 8 total, 7 up, 8 in 2023-09-23T20:21:43.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:42 smithi149 ceph-mon[170394]: failed to encode map e107 with expected crc 2023-09-23T20:21:43.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:42 smithi149 ceph-mon[170394]: failed to encode map e107 with expected crc 2023-09-23T20:21:43.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:42 smithi149 ceph-mon[170394]: failed to encode map e107 with expected crc 2023-09-23T20:21:43.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:42 smithi149 ceph-mon[170394]: failed to encode map e107 with expected crc 2023-09-23T20:21:43.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:42 smithi149 ceph-mon[170394]: failed to encode map e107 with expected crc 2023-09-23T20:21:43.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:42 smithi149 ceph-mon[170394]: failed to encode map e107 with expected crc 2023-09-23T20:21:43.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:42 smithi149 ceph-mon[170394]: failed to encode map e107 with expected crc 2023-09-23T20:21:43.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:42 smithi149 ceph-mon[170394]: failed to encode map e107 with expected crc 2023-09-23T20:21:44.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:43 smithi018 ceph-mon[184144]: Health check failed: Degraded data redundancy: 1/84 objects degraded (1.190%), 1 pg degraded (PG_DEGRADED) 2023-09-23T20:21:44.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:43 smithi018 ceph-mon[179647]: Health check failed: Degraded data redundancy: 1/84 objects degraded (1.190%), 1 pg degraded (PG_DEGRADED) 2023-09-23T20:21:44.142 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 23 20:21:43 smithi018 bash[213568]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-09-23T20:21:44.142 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 23 20:21:43 smithi018 bash[213568]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2023-09-23T20:21:44.142 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 23 20:21:43 smithi018 bash[213568]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_1 --path /var/lib/ceph/osd/ceph-3 --no-mon-config 2023-09-23T20:21:44.142 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 23 20:21:43 smithi018 bash[213568]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_1 /var/lib/ceph/osd/ceph-3/block 2023-09-23T20:21:44.142 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 23 20:21:43 smithi018 bash[213568]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-3/block 2023-09-23T20:21:44.142 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 23 20:21:43 smithi018 bash[213568]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2023-09-23T20:21:44.142 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 23 20:21:43 smithi018 bash[213568]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2023-09-23T20:21:44.142 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 23 20:21:43 smithi018 bash[213568]: --> ceph-volume lvm activate successful for osd ID: 3 2023-09-23T20:21:44.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:43 smithi149 ceph-mon[170394]: Health check failed: Degraded data redundancy: 1/84 objects degraded (1.190%), 1 pg degraded (PG_DEGRADED) 2023-09-23T20:21:44.876 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 23 20:21:44 smithi018 podman[213858]: 2023-09-23T20:21:44.877 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 23 20:21:44 smithi018 ceph-osd[213898]: -- 172.21.15.18:0/925069497 <== mon.0 v2:172.21.15.18:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x560243fb04e0 con 0x560243f81400 2023-09-23T20:21:44.877 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 23 20:21:44 smithi018 bash[213858]: d2f01bb88ee91bcc440187cbb1e5af96c616e6be8b851569729c17dc802d94be 2023-09-23T20:21:45.141 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:21:44 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:21:44] "GET /metrics HTTP/1.1" 200 34028 "" "Prometheus/2.43.0" 2023-09-23T20:21:45.141 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 23 20:21:44 smithi018 systemd[1]: Started Ceph osd.3 for a3b3b17e-5a4c-11ee-8db4-212e2dc638e7. 2023-09-23T20:21:45.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:44 smithi018 ceph-mon[179647]: pgmap v192: 73 pgs: 1 active+undersized+degraded, 10 stale+active+clean, 10 active+undersized, 52 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail; 1/84 objects degraded (1.190%) 2023-09-23T20:21:45.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:44 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1493476119' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:21:45.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:44 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:21:45.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:44 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:21:45.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:44 smithi018 ceph-mon[184144]: pgmap v192: 73 pgs: 1 active+undersized+degraded, 10 stale+active+clean, 10 active+undersized, 52 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail; 1/84 objects degraded (1.190%) 2023-09-23T20:21:45.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:44 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1493476119' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:21:45.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:44 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:21:45.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:44 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:21:45.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:44 smithi149 ceph-mon[170394]: pgmap v192: 73 pgs: 1 active+undersized+degraded, 10 stale+active+clean, 10 active+undersized, 52 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail; 1/84 objects degraded (1.190%) 2023-09-23T20:21:45.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:44 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1493476119' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:21:45.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:44 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:21:45.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:44 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:21:45.641 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 23 20:21:45 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-3[213874]: 2023-09-23T20:21:45.309+0000 7f838499f640 -1 Falling back to public interface 2023-09-23T20:21:46.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:45 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:21:46.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:45 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:21:46.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:45 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:21:46.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:45 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:21:46.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:45 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:21:46.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:45 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:21:46.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:45 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:21:46.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:45 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:21:46.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:45 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:21:47.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:46 smithi149 ceph-mon[170394]: pgmap v193: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-09-23T20:21:47.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:46 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3145217125' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:21:47.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:46 smithi018 ceph-mon[184144]: pgmap v193: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-09-23T20:21:47.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:46 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3145217125' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:21:47.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:46 smithi018 ceph-mon[179647]: pgmap v193: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-09-23T20:21:47.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:46 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3145217125' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:21:48.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:47 smithi018 ceph-mon[184144]: from='osd.3 [v2:172.21.15.18:6826/2876439245,v1:172.21.15.18:6827/2876439245]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-09-23T20:21:48.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:47 smithi018 ceph-mon[179647]: from='osd.3 [v2:172.21.15.18:6826/2876439245,v1:172.21.15.18:6827/2876439245]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-09-23T20:21:48.141 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 23 20:21:47 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-3[213874]: 2023-09-23T20:21:47.836+0000 7f838499f640 -1 osd.3 105 log_to_monitors true 2023-09-23T20:21:48.141 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 23 20:21:47 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-3[213874]: 2023-09-23T20:21:47.958+0000 7f8371e04700 -1 osd.3 105 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T20:21:48.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:47 smithi149 ceph-mon[170394]: from='osd.3 [v2:172.21.15.18:6826/2876439245,v1:172.21.15.18:6827/2876439245]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-09-23T20:21:49.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:48 smithi149 ceph-mon[170394]: pgmap v194: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-09-23T20:21:49.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:48 smithi149 ceph-mon[170394]: from='osd.3 [v2:172.21.15.18:6826/2876439245,v1:172.21.15.18:6827/2876439245]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-09-23T20:21:49.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:48 smithi149 ceph-mon[170394]: osdmap e108: 8 total, 7 up, 8 in 2023-09-23T20:21:49.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:48 smithi149 ceph-mon[170394]: from='osd.3 [v2:172.21.15.18:6826/2876439245,v1:172.21.15.18:6827/2876439245]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi018", "root=default"]}]: dispatch 2023-09-23T20:21:49.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:48 smithi149 ceph-mon[170394]: failed to encode map e108 with expected crc 2023-09-23T20:21:49.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:48 smithi149 ceph-mon[170394]: failed to encode map e108 with expected crc 2023-09-23T20:21:49.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:48 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1694481304' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:21:49.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:48 smithi018 ceph-mon[184144]: pgmap v194: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-09-23T20:21:49.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:48 smithi018 ceph-mon[184144]: from='osd.3 [v2:172.21.15.18:6826/2876439245,v1:172.21.15.18:6827/2876439245]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-09-23T20:21:49.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:48 smithi018 ceph-mon[184144]: osdmap e108: 8 total, 7 up, 8 in 2023-09-23T20:21:49.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:48 smithi018 ceph-mon[184144]: from='osd.3 [v2:172.21.15.18:6826/2876439245,v1:172.21.15.18:6827/2876439245]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi018", "root=default"]}]: dispatch 2023-09-23T20:21:49.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:48 smithi018 ceph-mon[184144]: failed to encode map e108 with expected crc 2023-09-23T20:21:49.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:48 smithi018 ceph-mon[184144]: failed to encode map e108 with expected crc 2023-09-23T20:21:49.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:48 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1694481304' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:21:49.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:48 smithi018 ceph-mon[179647]: pgmap v194: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-09-23T20:21:49.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:48 smithi018 ceph-mon[179647]: from='osd.3 [v2:172.21.15.18:6826/2876439245,v1:172.21.15.18:6827/2876439245]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-09-23T20:21:49.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:48 smithi018 ceph-mon[179647]: osdmap e108: 8 total, 7 up, 8 in 2023-09-23T20:21:49.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:48 smithi018 ceph-mon[179647]: from='osd.3 [v2:172.21.15.18:6826/2876439245,v1:172.21.15.18:6827/2876439245]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi018", "root=default"]}]: dispatch 2023-09-23T20:21:49.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:48 smithi018 ceph-mon[179647]: failed to encode map e108 with expected crc 2023-09-23T20:21:49.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:48 smithi018 ceph-mon[179647]: failed to encode map e108 with expected crc 2023-09-23T20:21:49.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:48 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1694481304' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:21:50.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:49 smithi149 ceph-mon[170394]: failed to encode map e108 with expected crc 2023-09-23T20:21:50.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:49 smithi149 ceph-mon[170394]: failed to encode map e108 with expected crc 2023-09-23T20:21:50.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:49 smithi149 ceph-mon[170394]: failed to encode map e108 with expected crc 2023-09-23T20:21:50.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:49 smithi149 ceph-mon[170394]: failed to encode map e108 with expected crc 2023-09-23T20:21:50.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:49 smithi149 ceph-mon[170394]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T20:21:50.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:49 smithi149 ceph-mon[170394]: failed to encode map e109 with expected crc 2023-09-23T20:21:50.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:49 smithi149 ceph-mon[170394]: osd.3 [v2:172.21.15.18:6826/2876439245,v1:172.21.15.18:6827/2876439245] boot 2023-09-23T20:21:50.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:49 smithi149 ceph-mon[170394]: osdmap e109: 8 total, 8 up, 8 in 2023-09-23T20:21:50.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:49 smithi149 ceph-mon[170394]: failed to encode map e109 with expected crc 2023-09-23T20:21:50.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:49 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T20:21:50.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:49 smithi149 ceph-mon[170394]: failed to encode map e109 with expected crc 2023-09-23T20:21:50.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:49 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:21:50.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:49 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:21:50.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:49 smithi018 ceph-mon[179647]: failed to encode map e108 with expected crc 2023-09-23T20:21:50.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:49 smithi018 ceph-mon[179647]: failed to encode map e108 with expected crc 2023-09-23T20:21:50.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:49 smithi018 ceph-mon[179647]: failed to encode map e108 with expected crc 2023-09-23T20:21:50.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:49 smithi018 ceph-mon[179647]: failed to encode map e108 with expected crc 2023-09-23T20:21:50.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:49 smithi018 ceph-mon[179647]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T20:21:50.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:49 smithi018 ceph-mon[179647]: failed to encode map e109 with expected crc 2023-09-23T20:21:50.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:49 smithi018 ceph-mon[179647]: osd.3 [v2:172.21.15.18:6826/2876439245,v1:172.21.15.18:6827/2876439245] boot 2023-09-23T20:21:50.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:49 smithi018 ceph-mon[179647]: osdmap e109: 8 total, 8 up, 8 in 2023-09-23T20:21:50.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:49 smithi018 ceph-mon[179647]: failed to encode map e109 with expected crc 2023-09-23T20:21:50.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:49 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T20:21:50.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:49 smithi018 ceph-mon[179647]: failed to encode map e109 with expected crc 2023-09-23T20:21:50.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:49 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:21:50.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:49 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:21:50.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:49 smithi018 ceph-mon[184144]: failed to encode map e108 with expected crc 2023-09-23T20:21:50.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:49 smithi018 ceph-mon[184144]: failed to encode map e108 with expected crc 2023-09-23T20:21:50.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:49 smithi018 ceph-mon[184144]: failed to encode map e108 with expected crc 2023-09-23T20:21:50.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:49 smithi018 ceph-mon[184144]: failed to encode map e108 with expected crc 2023-09-23T20:21:50.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:49 smithi018 ceph-mon[184144]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T20:21:50.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:49 smithi018 ceph-mon[184144]: failed to encode map e109 with expected crc 2023-09-23T20:21:50.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:49 smithi018 ceph-mon[184144]: osd.3 [v2:172.21.15.18:6826/2876439245,v1:172.21.15.18:6827/2876439245] boot 2023-09-23T20:21:50.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:49 smithi018 ceph-mon[184144]: osdmap e109: 8 total, 8 up, 8 in 2023-09-23T20:21:50.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:49 smithi018 ceph-mon[184144]: failed to encode map e109 with expected crc 2023-09-23T20:21:50.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:49 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T20:21:50.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:49 smithi018 ceph-mon[184144]: failed to encode map e109 with expected crc 2023-09-23T20:21:50.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:49 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:21:50.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:49 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:21:51.027 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable)": 4 2023-09-23T20:21:51.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:50 smithi149 ceph-mon[170394]: pgmap v197: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-09-23T20:21:51.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:50 smithi149 ceph-mon[170394]: failed to encode map e110 with expected crc 2023-09-23T20:21:51.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:50 smithi149 ceph-mon[170394]: osdmap e110: 8 total, 8 up, 8 in 2023-09-23T20:21:51.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:50 smithi149 ceph-mon[170394]: failed to encode map e110 with expected crc 2023-09-23T20:21:51.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:50 smithi149 ceph-mon[170394]: failed to encode map e110 with expected crc 2023-09-23T20:21:51.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:50 smithi149 ceph-mon[170394]: failed to encode map e110 with expected crc 2023-09-23T20:21:51.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:50 smithi149 ceph-mon[170394]: failed to encode map e110 with expected crc 2023-09-23T20:21:51.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:50 smithi149 ceph-mon[170394]: failed to encode map e110 with expected crc 2023-09-23T20:21:51.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:50 smithi149 ceph-mon[170394]: failed to encode map e110 with expected crc 2023-09-23T20:21:51.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:50 smithi149 ceph-mon[170394]: failed to encode map e110 with expected crc 2023-09-23T20:21:51.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:50 smithi149 ceph-mon[170394]: failed to encode map e110 with expected crc 2023-09-23T20:21:51.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:50 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:21:51.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:50 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:21:51.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:50 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 4/84 objects degraded (4.762%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T20:21:51.389 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:50 smithi018 ceph-mon[184144]: pgmap v197: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-09-23T20:21:51.389 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:50 smithi018 ceph-mon[184144]: failed to encode map e110 with expected crc 2023-09-23T20:21:51.389 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:50 smithi018 ceph-mon[184144]: osdmap e110: 8 total, 8 up, 8 in 2023-09-23T20:21:51.389 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:50 smithi018 ceph-mon[184144]: failed to encode map e110 with expected crc 2023-09-23T20:21:51.390 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:50 smithi018 ceph-mon[184144]: failed to encode map e110 with expected crc 2023-09-23T20:21:51.390 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:50 smithi018 ceph-mon[184144]: failed to encode map e110 with expected crc 2023-09-23T20:21:51.390 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:50 smithi018 ceph-mon[184144]: failed to encode map e110 with expected crc 2023-09-23T20:21:51.390 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:50 smithi018 ceph-mon[184144]: failed to encode map e110 with expected crc 2023-09-23T20:21:51.390 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:50 smithi018 ceph-mon[184144]: failed to encode map e110 with expected crc 2023-09-23T20:21:51.390 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:50 smithi018 ceph-mon[184144]: failed to encode map e110 with expected crc 2023-09-23T20:21:51.390 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:50 smithi018 ceph-mon[184144]: failed to encode map e110 with expected crc 2023-09-23T20:21:51.390 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:50 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:21:51.390 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:50 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:21:51.390 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:50 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 4/84 objects degraded (4.762%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T20:21:51.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:50 smithi018 ceph-mon[179647]: pgmap v197: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-09-23T20:21:51.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:50 smithi018 ceph-mon[179647]: failed to encode map e110 with expected crc 2023-09-23T20:21:51.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:50 smithi018 ceph-mon[179647]: osdmap e110: 8 total, 8 up, 8 in 2023-09-23T20:21:51.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:50 smithi018 ceph-mon[179647]: failed to encode map e110 with expected crc 2023-09-23T20:21:51.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:50 smithi018 ceph-mon[179647]: failed to encode map e110 with expected crc 2023-09-23T20:21:51.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:50 smithi018 ceph-mon[179647]: failed to encode map e110 with expected crc 2023-09-23T20:21:51.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:50 smithi018 ceph-mon[179647]: failed to encode map e110 with expected crc 2023-09-23T20:21:51.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:50 smithi018 ceph-mon[179647]: failed to encode map e110 with expected crc 2023-09-23T20:21:51.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:50 smithi018 ceph-mon[179647]: failed to encode map e110 with expected crc 2023-09-23T20:21:51.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:50 smithi018 ceph-mon[179647]: failed to encode map e110 with expected crc 2023-09-23T20:21:51.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:50 smithi018 ceph-mon[179647]: failed to encode map e110 with expected crc 2023-09-23T20:21:51.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:50 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:21:51.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:50 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:21:51.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:50 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 4/84 objects degraded (4.762%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T20:21:51.714 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch upgrade pause' 2023-09-23T20:21:52.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:51 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3594328791' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:21:52.344 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:21:52.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:51 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3594328791' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:21:52.402 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:51 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3594328791' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T20:21:53.137 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:52 smithi018 ceph-mon[184144]: pgmap v199: 73 pgs: 2 peering, 2 active+undersized+degraded, 19 active+undersized, 50 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2023-09-23T20:21:53.137 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:52 smithi018 ceph-mon[179647]: pgmap v199: 73 pgs: 2 peering, 2 active+undersized+degraded, 19 active+undersized, 50 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2023-09-23T20:21:53.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:52 smithi149 ceph-mon[170394]: pgmap v199: 73 pgs: 2 peering, 2 active+undersized+degraded, 19 active+undersized, 50 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2023-09-23T20:21:53.891 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:21:53 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: 2023-09-23T20:21:53.571+0000 7f953be6a700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-09-23T20:21:54.033 INFO:teuthology.orchestra.run.smithi018.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T20:21:54.455 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch ps' 2023-09-23T20:21:54.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:54 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:21:54.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:54 smithi149 ceph-mon[170394]: pgmap v200: 73 pgs: 2 peering, 71 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:21:54.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:54 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:21:54.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:54 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:21:54.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:54 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:21:54.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:54 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:21:54.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:54 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:21:54.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:54 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:21:54.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:54 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:21:54.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:54 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:21:54.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:54 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-09-23T20:21:54.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:54 smithi149 ceph-mon[170394]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-09-23T20:21:54.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:54 smithi149 ceph-mon[170394]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-09-23T20:21:54.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:54 smithi149 ceph-mon[170394]: from='client.44468 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:21:54.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:54 smithi149 ceph-mon[170394]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T20:21:54.718 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:54 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:21:54.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:54 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:21:54.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:54 smithi018 ceph-mon[184144]: pgmap v200: 73 pgs: 2 peering, 71 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:21:54.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:54 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:21:54.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:54 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:21:54.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:54 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:21:54.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:54 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:21:54.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:54 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:21:54.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:54 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:21:54.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:54 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:21:54.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:54 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:21:54.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:54 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-09-23T20:21:54.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:54 smithi018 ceph-mon[184144]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-09-23T20:21:54.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:54 smithi018 ceph-mon[184144]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-09-23T20:21:54.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:54 smithi018 ceph-mon[184144]: from='client.44468 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:21:54.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:54 smithi018 ceph-mon[184144]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T20:21:54.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:54 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:21:54.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:54 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:21:54.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:54 smithi018 ceph-mon[179647]: pgmap v200: 73 pgs: 2 peering, 71 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:21:54.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:54 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:21:54.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:54 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:21:54.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:54 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:21:54.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:54 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:21:54.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:54 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:21:54.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:54 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:21:54.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:54 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:21:54.895 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:54 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:21:54.895 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:54 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-09-23T20:21:54.895 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:54 smithi018 ceph-mon[179647]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-09-23T20:21:54.895 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:54 smithi018 ceph-mon[179647]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2023-09-23T20:21:54.895 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:54 smithi018 ceph-mon[179647]: from='client.44468 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:21:54.895 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:54 smithi018 ceph-mon[179647]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T20:21:54.895 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:54 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:21:55.062 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:21:55.258 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:21:54 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:21:54] "GET /metrics HTTP/1.1" 200 34028 "" "Prometheus/2.43.0" 2023-09-23T20:21:55.571 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:55 smithi018 ceph-mon[184144]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded) 2023-09-23T20:21:55.571 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:55 smithi018 ceph-mon[184144]: Cluster is now healthy 2023-09-23T20:21:55.572 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:55 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:21:55.572 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:55 smithi018 ceph-mon[179647]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded) 2023-09-23T20:21:55.572 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:55 smithi018 ceph-mon[179647]: Cluster is now healthy 2023-09-23T20:21:55.572 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:55 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:21:55.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:55 smithi149 ceph-mon[170394]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded) 2023-09-23T20:21:55.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:55 smithi149 ceph-mon[170394]: Cluster is now healthy 2023-09-23T20:21:55.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:55 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:21:56.492 INFO:teuthology.orchestra.run.smithi018.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-09-23T20:21:56.492 INFO:teuthology.orchestra.run.smithi018.stdout:alertmanager.a smithi018 *:9093,9094 running (7m) - 9m 28.8M - 0.25.0 c8568f914cd2 57241c9344be 2023-09-23T20:21:56.492 INFO:teuthology.orchestra.run.smithi018.stdout:grafana.a smithi149 *:3000 running (6m) - 9m 94.7M - 9.4.7 2c41d148cca3 62150314e898 2023-09-23T20:21:56.492 INFO:teuthology.orchestra.run.smithi018.stdout:mds.foo.smithi018.hduiol smithi018 running (9m) - 9m 19.1M - 17.2.6-1123-g61431f6b 0db14aa01094 7b6a15d633a4 2023-09-23T20:21:56.493 INFO:teuthology.orchestra.run.smithi018.stdout:mds.foo.smithi149.urtezf smithi149 running (9m) - 9m 18.6M - 17.2.6-1123-g61431f6b 0db14aa01094 6ee4df523f8c 2023-09-23T20:21:56.493 INFO:teuthology.orchestra.run.smithi018.stdout:mgr.x smithi149 *:8443,9283,8765 running (6m) - 13m 450M - 18.2.0-460-gbb0daeff e32c2a64c126 9513b5c2a2f7 2023-09-23T20:21:56.493 INFO:teuthology.orchestra.run.smithi018.stdout:mgr.y smithi018 *:8443,9283,8765 running (6m) - 14m 516M - 18.2.0-460-gbb0daeff e32c2a64c126 5aa74f13c0a1 2023-09-23T20:21:56.493 INFO:teuthology.orchestra.run.smithi018.stdout:mon.a smithi018 running (6m) - 15m 83.5M 2048M 18.2.0-460-gbb0daeff e32c2a64c126 425294bbcbd1 2023-09-23T20:21:56.493 INFO:teuthology.orchestra.run.smithi018.stdout:mon.b smithi149 running (3m) - 13m 48.3M 2048M 18.2.0-460-gbb0daeff e32c2a64c126 b1930bb5f3e1 2023-09-23T20:21:56.493 INFO:teuthology.orchestra.run.smithi018.stdout:mon.c smithi018 running (4m) - 13m 55.7M 2048M 18.2.0-460-gbb0daeff e32c2a64c126 e2701e7c5d35 2023-09-23T20:21:56.493 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter.a smithi018 *:9100 running (7m) - 9m 18.1M - 1.5.0 0da6a335fe13 500f76739e6a 2023-09-23T20:21:56.493 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter.b smithi149 *:9100 running (6m) - 9m 18.0M - 1.5.0 0da6a335fe13 f1789a46736b 2023-09-23T20:21:56.493 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0 smithi018 running (108s) - 12m 61.8M 2501M 18.2.0-460-gbb0daeff e32c2a64c126 9c05a0fd7c04 2023-09-23T20:21:56.493 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1 smithi018 running (76s) - 12m 59.2M 2501M 18.2.0-460-gbb0daeff e32c2a64c126 c71a2c8070c3 2023-09-23T20:21:56.493 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2 smithi018 running (44s) - 12m 48.8M 2501M 18.2.0-460-gbb0daeff e32c2a64c126 4b4b70078c4e 2023-09-23T20:21:56.493 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3 smithi018 running (12s) - 11m 13.8M 2501M 18.2.0-460-gbb0daeff e32c2a64c126 d2f01bb88ee9 2023-09-23T20:21:56.494 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4 smithi149 running (11m) - 11m 95.6M 4096M 17.2.6-1123-g61431f6b 0db14aa01094 b4b39c43de75 2023-09-23T20:21:56.494 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5 smithi149 running (11m) - 11m 101M 4096M 17.2.6-1123-g61431f6b 0db14aa01094 cec31ff38f4c 2023-09-23T20:21:56.494 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6 smithi149 running (10m) - 10m 93.2M 4096M 17.2.6-1123-g61431f6b 0db14aa01094 ce687413759d 2023-09-23T20:21:56.494 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7 smithi149 running (10m) - 10m 108M 4096M 17.2.6-1123-g61431f6b 0db14aa01094 d3145e28f8c3 2023-09-23T20:21:56.494 INFO:teuthology.orchestra.run.smithi018.stdout:prometheus.a smithi149 *:9095 running (6m) - 10m 63.5M - 2.43.0 a07b618ecd1d bb2162e75dc7 2023-09-23T20:21:56.923 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch ps' 2023-09-23T20:21:57.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:56 smithi018 ceph-mon[184144]: pgmap v201: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:21:57.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:56 smithi018 ceph-mon[179647]: pgmap v201: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:21:57.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:56 smithi149 ceph-mon[170394]: pgmap v201: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:21:57.528 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:21:58.027 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:57 smithi018 ceph-mon[184144]: from='client.34572 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:21:58.027 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:57 smithi018 ceph-mon[179647]: from='client.34572 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:21:58.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:57 smithi149 ceph-mon[170394]: from='client.34572 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:21:59.000 INFO:teuthology.orchestra.run.smithi018.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-09-23T20:21:59.000 INFO:teuthology.orchestra.run.smithi018.stdout:alertmanager.a smithi018 *:9093,9094 running (7m) - 9m 28.8M - 0.25.0 c8568f914cd2 57241c9344be 2023-09-23T20:21:59.000 INFO:teuthology.orchestra.run.smithi018.stdout:grafana.a smithi149 *:3000 running (6m) - 9m 94.7M - 9.4.7 2c41d148cca3 62150314e898 2023-09-23T20:21:59.001 INFO:teuthology.orchestra.run.smithi018.stdout:mds.foo.smithi018.hduiol smithi018 running (9m) - 9m 19.1M - 17.2.6-1123-g61431f6b 0db14aa01094 7b6a15d633a4 2023-09-23T20:21:59.001 INFO:teuthology.orchestra.run.smithi018.stdout:mds.foo.smithi149.urtezf smithi149 running (9m) - 9m 18.6M - 17.2.6-1123-g61431f6b 0db14aa01094 6ee4df523f8c 2023-09-23T20:21:59.001 INFO:teuthology.orchestra.run.smithi018.stdout:mgr.x smithi149 *:8443,9283,8765 running (6m) - 13m 450M - 18.2.0-460-gbb0daeff e32c2a64c126 9513b5c2a2f7 2023-09-23T20:21:59.001 INFO:teuthology.orchestra.run.smithi018.stdout:mgr.y smithi018 *:8443,9283,8765 running (6m) - 14m 516M - 18.2.0-460-gbb0daeff e32c2a64c126 5aa74f13c0a1 2023-09-23T20:21:59.001 INFO:teuthology.orchestra.run.smithi018.stdout:mon.a smithi018 running (6m) - 15m 83.5M 2048M 18.2.0-460-gbb0daeff e32c2a64c126 425294bbcbd1 2023-09-23T20:21:59.001 INFO:teuthology.orchestra.run.smithi018.stdout:mon.b smithi149 running (3m) - 13m 48.3M 2048M 18.2.0-460-gbb0daeff e32c2a64c126 b1930bb5f3e1 2023-09-23T20:21:59.001 INFO:teuthology.orchestra.run.smithi018.stdout:mon.c smithi018 running (4m) - 13m 55.7M 2048M 18.2.0-460-gbb0daeff e32c2a64c126 e2701e7c5d35 2023-09-23T20:21:59.001 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter.a smithi018 *:9100 running (7m) - 9m 18.1M - 1.5.0 0da6a335fe13 500f76739e6a 2023-09-23T20:21:59.001 INFO:teuthology.orchestra.run.smithi018.stdout:node-exporter.b smithi149 *:9100 running (6m) - 9m 18.0M - 1.5.0 0da6a335fe13 f1789a46736b 2023-09-23T20:21:59.001 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0 smithi018 running (110s) - 13m 61.8M 2501M 18.2.0-460-gbb0daeff e32c2a64c126 9c05a0fd7c04 2023-09-23T20:21:59.001 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1 smithi018 running (78s) - 12m 59.2M 2501M 18.2.0-460-gbb0daeff e32c2a64c126 c71a2c8070c3 2023-09-23T20:21:59.002 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2 smithi018 running (46s) - 12m 48.8M 2501M 18.2.0-460-gbb0daeff e32c2a64c126 4b4b70078c4e 2023-09-23T20:21:59.002 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3 smithi018 running (14s) - 11m 13.8M 2501M 18.2.0-460-gbb0daeff e32c2a64c126 d2f01bb88ee9 2023-09-23T20:21:59.002 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4 smithi149 running (11m) - 11m 95.6M 4096M 17.2.6-1123-g61431f6b 0db14aa01094 b4b39c43de75 2023-09-23T20:21:59.002 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5 smithi149 running (11m) - 11m 101M 4096M 17.2.6-1123-g61431f6b 0db14aa01094 cec31ff38f4c 2023-09-23T20:21:59.002 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6 smithi149 running (11m) - 11m 93.2M 4096M 17.2.6-1123-g61431f6b 0db14aa01094 ce687413759d 2023-09-23T20:21:59.002 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7 smithi149 running (10m) - 10m 108M 4096M 17.2.6-1123-g61431f6b 0db14aa01094 d3145e28f8c3 2023-09-23T20:21:59.002 INFO:teuthology.orchestra.run.smithi018.stdout:prometheus.a smithi149 *:9095 running (6m) - 10m 63.5M - 2.43.0 a07b618ecd1d bb2162e75dc7 2023-09-23T20:21:59.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:58 smithi018 ceph-mon[184144]: pgmap v202: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:21:59.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:58 smithi018 ceph-mon[179647]: pgmap v202: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:21:59.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:58 smithi149 ceph-mon[170394]: pgmap v202: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:21:59.389 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph versions' 2023-09-23T20:21:59.999 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:22:00.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:59 smithi018 ceph-mon[179647]: from='client.34578 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:22:00.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:59 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:22:00.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:21:59 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:22:00.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:59 smithi018 ceph-mon[184144]: from='client.34578 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:22:00.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:59 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:22:00.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:21:59 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:22:00.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:59 smithi149 ceph-mon[170394]: from='client.34578 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:22:00.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:59 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:22:00.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:21:59 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:22:01.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:22:00 smithi018 ceph-mon[184144]: pgmap v203: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:22:01.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:22:00 smithi018 ceph-mon[179647]: pgmap v203: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:22:01.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:22:00 smithi149 ceph-mon[170394]: pgmap v203: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:22:01.709 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2023-09-23T20:22:01.709 INFO:teuthology.orchestra.run.smithi018.stdout: "mon": { 2023-09-23T20:22:01.710 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable)": 3 2023-09-23T20:22:01.710 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:22:01.710 INFO:teuthology.orchestra.run.smithi018.stdout: "mgr": { 2023-09-23T20:22:01.710 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable)": 2 2023-09-23T20:22:01.710 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:22:01.710 INFO:teuthology.orchestra.run.smithi018.stdout: "osd": { 2023-09-23T20:22:01.710 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)": 4, 2023-09-23T20:22:01.710 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable)": 4 2023-09-23T20:22:01.710 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:22:01.710 INFO:teuthology.orchestra.run.smithi018.stdout: "mds": { 2023-09-23T20:22:01.711 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)": 2 2023-09-23T20:22:01.711 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:22:01.711 INFO:teuthology.orchestra.run.smithi018.stdout: "overall": { 2023-09-23T20:22:01.711 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)": 6, 2023-09-23T20:22:01.711 INFO:teuthology.orchestra.run.smithi018.stdout: "ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable)": 9 2023-09-23T20:22:01.711 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:22:01.711 INFO:teuthology.orchestra.run.smithi018.stdout:} 2023-09-23T20:22:02.059 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:22:01 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/584234106' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:22:02.059 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:22:01 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/584234106' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:22:02.060 DEBUG:teuthology.parallel:result is None 2023-09-23T20:22:02.061 INFO:teuthology.run_tasks:Running task print... 2023-09-23T20:22:02.070 INFO:teuthology.task.print:**** done upgrading first half of cluster **** 2023-09-23T20:22:02.071 INFO:teuthology.run_tasks:Running task ceph.healthy... 2023-09-23T20:22:02.086 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2023-09-23T20:22:02.087 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2023-09-23T20:22:02.087 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph mgr dump --format=json 2023-09-23T20:22:02.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:22:01 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/584234106' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T20:22:02.694 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:22:03.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:22:02 smithi018 ceph-mon[184144]: pgmap v204: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:22:03.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:22:02 smithi018 ceph-mon[179647]: pgmap v204: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:22:03.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:22:02 smithi149 ceph-mon[170394]: pgmap v204: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:22:04.184 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:22:04.541 INFO:teuthology.orchestra.run.smithi018.stdout:{"epoch":36,"active_gid":34107,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6800","nonce":198807647},{"type":"v1","addr":"172.21.15.18:6801","nonce":198807647}]},"active_addr":"172.21.15.18:6801/198807647","active_change":"2023-09-23T20:15:59.309914+0000","active_mgr_features":4540138322906710015,"available":true,"standbys":[{"gid":25808,"name":"x","mgr_features":4540138322906710015,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph:v18","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:0.0.2","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph:v18","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:0.0.2","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.18:8443/","prometheus":"http://172.21.15.18:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":90,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.18:0","nonce":1296405130}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.18:0","nonce":1197208400}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.18:0","nonce":991662839}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.18:0","nonce":54737100}]}]} 2023-09-23T20:22:04.545 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2023-09-23T20:22:04.545 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2023-09-23T20:22:04.546 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd dump --format=json 2023-09-23T20:22:05.141 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:22:04 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:22:04] "GET /metrics HTTP/1.1" 200 34028 "" "Prometheus/2.43.0" 2023-09-23T20:22:05.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:22:04 smithi018 ceph-mon[179647]: pgmap v205: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:22:05.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:22:04 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/829609709' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-09-23T20:22:05.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:22:04 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:22:05.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:22:04 smithi018 ceph-mon[184144]: pgmap v205: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:22:05.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:22:04 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/829609709' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-09-23T20:22:05.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:22:04 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:22:05.153 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:22:05.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:22:04 smithi149 ceph-mon[170394]: pgmap v205: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:22:05.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:22:04 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/829609709' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-09-23T20:22:05.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:22:04 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:22:06.663 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:22:06.664 INFO:teuthology.orchestra.run.smithi018.stdout:{"epoch":110,"fsid":"a3b3b17e-5a4c-11ee-8db4-212e2dc638e7","created":"2023-09-23T20:06:54.359034+0000","modified":"2023-09-23T20:21:49.935938+0000","last_up_change":"2023-09-23T20:21:48.934116+0000","last_in_change":"2023-09-23T20:11:09.202130+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":22,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-09-23T20:09:47.162973+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-09-23T20:11:59.431837+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"86","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":15,"snap_epoch":86,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-09-23T20:12:42.254901+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-09-23T20:12:43.191578+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"69","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"44c8aaa3-bbd1-4330-8d34-73f7a08b0d4d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":90,"up_from":94,"up_thru":109,"down_at":91,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6802","nonce":990458588},{"type":"v1","addr":"172.21.15.18:6803","nonce":990458588}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6804","nonce":990458588},{"type":"v1","addr":"172.21.15.18:6805","nonce":990458588}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6808","nonce":990458588},{"type":"v1","addr":"172.21.15.18:6809","nonce":990458588}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6806","nonce":990458588},{"type":"v1","addr":"172.21.15.18:6807","nonce":990458588}]},"public_addr":"172.21.15.18:6803/990458588","cluster_addr":"172.21.15.18:6805/990458588","heartbeat_back_addr":"172.21.15.18:6809/990458588","heartbeat_front_addr":"172.21.15.18:6807/990458588","state":["exists","up"]},{"osd":1,"uuid":"c0534324-49c8-4804-bced-c4d13510e74a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":95,"up_from":99,"up_thru":109,"down_at":96,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6810","nonce":1422592526},{"type":"v1","addr":"172.21.15.18:6811","nonce":1422592526}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6812","nonce":1422592526},{"type":"v1","addr":"172.21.15.18:6813","nonce":1422592526}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6816","nonce":1422592526},{"type":"v1","addr":"172.21.15.18:6817","nonce":1422592526}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6814","nonce":1422592526},{"type":"v1","addr":"172.21.15.18:6815","nonce":1422592526}]},"public_addr":"172.21.15.18:6811/1422592526","cluster_addr":"172.21.15.18:6813/1422592526","heartbeat_back_addr":"172.21.15.18:6817/1422592526","heartbeat_front_addr":"172.21.15.18:6815/1422592526","state":["exists","up"]},{"osd":2,"uuid":"f3b2116d-a3d5-4279-91dc-eb47458d3239","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":100,"up_from":104,"up_thru":109,"down_at":101,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6818","nonce":2061051826},{"type":"v1","addr":"172.21.15.18:6819","nonce":2061051826}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6820","nonce":2061051826},{"type":"v1","addr":"172.21.15.18:6821","nonce":2061051826}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6824","nonce":2061051826},{"type":"v1","addr":"172.21.15.18:6825","nonce":2061051826}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6822","nonce":2061051826},{"type":"v1","addr":"172.21.15.18:6823","nonce":2061051826}]},"public_addr":"172.21.15.18:6819/2061051826","cluster_addr":"172.21.15.18:6821/2061051826","heartbeat_back_addr":"172.21.15.18:6825/2061051826","heartbeat_front_addr":"172.21.15.18:6823/2061051826","state":["exists","up"]},{"osd":3,"uuid":"8edccb7f-c4c9-47f3-a6f5-38a1a38cb099","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":105,"up_from":109,"up_thru":109,"down_at":106,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6826","nonce":2876439245},{"type":"v1","addr":"172.21.15.18:6827","nonce":2876439245}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6828","nonce":2876439245},{"type":"v1","addr":"172.21.15.18:6829","nonce":2876439245}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6832","nonce":2876439245},{"type":"v1","addr":"172.21.15.18:6833","nonce":2876439245}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6830","nonce":2876439245},{"type":"v1","addr":"172.21.15.18:6831","nonce":2876439245}]},"public_addr":"172.21.15.18:6827/2876439245","cluster_addr":"172.21.15.18:6829/2876439245","heartbeat_back_addr":"172.21.15.18:6833/2876439245","heartbeat_front_addr":"172.21.15.18:6831/2876439245","state":["exists","up"]},{"osd":4,"uuid":"1d5a64ce-fe42-4355-991a-dfb20cbe5423","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":109,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6800","nonce":3640432477},{"type":"v1","addr":"172.21.15.149:6801","nonce":3640432477}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6802","nonce":3640432477},{"type":"v1","addr":"172.21.15.149:6803","nonce":3640432477}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6806","nonce":3640432477},{"type":"v1","addr":"172.21.15.149:6807","nonce":3640432477}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6804","nonce":3640432477},{"type":"v1","addr":"172.21.15.149:6805","nonce":3640432477}]},"public_addr":"172.21.15.149:6801/3640432477","cluster_addr":"172.21.15.149:6803/3640432477","heartbeat_back_addr":"172.21.15.149:6807/3640432477","heartbeat_front_addr":"172.21.15.149:6805/3640432477","state":["exists","up"]},{"osd":5,"uuid":"105ae3b2-b2fc-477d-9134-3d517bc9427e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":109,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6808","nonce":2969709367},{"type":"v1","addr":"172.21.15.149:6809","nonce":2969709367}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6810","nonce":2969709367},{"type":"v1","addr":"172.21.15.149:6811","nonce":2969709367}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6814","nonce":2969709367},{"type":"v1","addr":"172.21.15.149:6815","nonce":2969709367}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6812","nonce":2969709367},{"type":"v1","addr":"172.21.15.149:6813","nonce":2969709367}]},"public_addr":"172.21.15.149:6809/2969709367","cluster_addr":"172.21.15.149:6811/2969709367","heartbeat_back_addr":"172.21.15.149:6815/2969709367","heartbeat_front_addr":"172.21.15.149:6813/2969709367","state":["exists","up"]},{"osd":6,"uuid":"27d2d2a8-e8c6-4a05-a85e-bd54376cf4c1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":106,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6816","nonce":1291453593},{"type":"v1","addr":"172.21.15.149:6817","nonce":1291453593}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6818","nonce":1291453593},{"type":"v1","addr":"172.21.15.149:6819","nonce":1291453593}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6822","nonce":1291453593},{"type":"v1","addr":"172.21.15.149:6823","nonce":1291453593}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6820","nonce":1291453593},{"type":"v1","addr":"172.21.15.149:6821","nonce":1291453593}]},"public_addr":"172.21.15.149:6817/1291453593","cluster_addr":"172.21.15.149:6819/1291453593","heartbeat_back_addr":"172.21.15.149:6823/1291453593","heartbeat_front_addr":"172.21.15.149:6821/1291453593","state":["exists","up"]},{"osd":7,"uuid":"9da636f7-995c-4f29-9264-70d09c6f66c6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":109,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6824","nonce":3117985178},{"type":"v1","addr":"172.21.15.149:6825","nonce":3117985178}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6826","nonce":3117985178},{"type":"v1","addr":"172.21.15.149:6827","nonce":3117985178}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6830","nonce":3117985178},{"type":"v1","addr":"172.21.15.149:6831","nonce":3117985178}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6828","nonce":3117985178},{"type":"v1","addr":"172.21.15.149:6829","nonce":3117985178}]},"public_addr":"172.21.15.149:6825/3117985178","cluster_addr":"172.21.15.149:6827/3117985178","heartbeat_back_addr":"172.21.15.149:6831/3117985178","heartbeat_front_addr":"172.21.15.149:6829/3117985178","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:09:01.034520+0000","dead_epoch":90},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:09:21.795238+0000","dead_epoch":95},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:09:42.987835+0000","dead_epoch":100},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:10:04.092458+0000","dead_epoch":105},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:10:23.309112+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:10:43.348231+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:11:03.128458+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:11:23.084228+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.d","mappings":[{"from":1,"to":5}]},{"pgid":"4.f","mappings":[{"from":1,"to":2}]},{"pgid":"4.18","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.18:6801/2791912398":"2023-09-24T20:15:59.309691+0000","172.21.15.149:0/502112376":"2023-09-24T20:15:19.879895+0000","172.21.15.18:0/69122222":"2023-09-24T20:07:23.919815+0000","172.21.15.149:6835/1754402171":"2023-09-24T20:15:19.879895+0000","172.21.15.18:0/3831267686":"2023-09-24T20:07:48.635784+0000","172.21.15.149:0/3406344898":"2023-09-24T20:15:19.879895+0000","172.21.15.149:6834/1754402171":"2023-09-24T20:15:19.879895+0000","172.21.15.18:0/1988472148":"2023-09-24T20:11:49.652762+0000","172.21.15.18:0/1082665289":"2023-09-24T20:07:48.635784+0000","172.21.15.18:6800/1930027051":"2023-09-24T20:07:23.919815+0000","172.21.15.18:0/2809002822":"2023-09-24T20:14:38.972322+0000","172.21.15.18:6801/2611527229":"2023-09-24T20:11:49.652762+0000","172.21.15.149:0/2486656126":"2023-09-24T20:15:19.879895+0000","172.21.15.18:0/1119348530":"2023-09-24T20:15:59.309691+0000","172.21.15.18:6800/2611527229":"2023-09-24T20:11:49.652762+0000","172.21.15.18:0/2543728509":"2023-09-24T20:14:38.972322+0000","172.21.15.18:0/2993111051":"2023-09-24T20:07:23.919815+0000","172.21.15.18:0/3990640673":"2023-09-24T20:14:38.972322+0000","172.21.15.18:0/1698512393":"2023-09-24T20:11:49.652762+0000","172.21.15.149:0/894377242":"2023-09-24T20:15:19.879895+0000","172.21.15.18:0/3356195690":"2023-09-24T20:14:38.972322+0000","172.21.15.18:6801/1930027051":"2023-09-24T20:07:23.919815+0000","172.21.15.18:6801/380813540":"2023-09-24T20:07:48.635784+0000","172.21.15.18:0/3863388488":"2023-09-24T20:11:49.652762+0000","172.21.15.18:0/2549145961":"2023-09-24T20:15:59.309691+0000","172.21.15.149:0/1078964609":"2023-09-24T20:15:19.879895+0000","172.21.15.18:0/1645667747":"2023-09-24T20:07:48.635784+0000","172.21.15.18:6800/380813540":"2023-09-24T20:07:48.635784+0000","172.21.15.18:6800/115279545":"2023-09-24T20:14:38.972322+0000","172.21.15.18:0/3510203881":"2023-09-24T20:15:59.309691+0000","172.21.15.18:6801/115279545":"2023-09-24T20:14:38.972322+0000","172.21.15.18:0/950924496":"2023-09-24T20:07:23.919815+0000","172.21.15.149:0/1653842667":"2023-09-24T20:15:19.879895+0000","172.21.15.18:0/3334666861":"2023-09-24T20:15:59.309691+0000","172.21.15.18:0/3912151186":"2023-09-24T20:11:49.652762+0000","172.21.15.18:0/872601545":"2023-09-24T20:15:59.309691+0000","172.21.15.18:0/1505013437":"2023-09-24T20:15:59.309691+0000","172.21.15.18:6800/2791912398":"2023-09-24T20:15:59.309691+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-23T20:22:07.020 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:22:06 smithi018 ceph-mon[184144]: pgmap v206: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:22:07.021 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:22:06 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3645895058' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T20:22:07.021 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:22:06 smithi018 ceph-mon[179647]: pgmap v206: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:22:07.021 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:22:06 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3645895058' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T20:22:07.114 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2023-09-23T20:22:07.114 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd dump --format=json 2023-09-23T20:22:07.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:22:06 smithi149 ceph-mon[170394]: pgmap v206: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:22:07.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:22:06 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3645895058' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T20:22:07.718 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:22:09.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:22:08 smithi018 ceph-mon[179647]: pgmap v207: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:22:09.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:22:08 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:22:09.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:22:08 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:22:09.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:22:08 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:22:09.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:22:08 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:22:09.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:22:08 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:22:09.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:22:08 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:22:09.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:22:08 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:22:09.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:22:08 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:22:09.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:22:08 smithi018 ceph-mon[184144]: pgmap v207: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:22:09.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:22:08 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:22:09.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:22:08 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:22:09.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:22:08 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:22:09.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:22:08 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:22:09.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:22:08 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:22:09.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:22:08 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:22:09.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:22:08 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:22:09.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:22:08 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:22:09.188 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:22:09.188 INFO:teuthology.orchestra.run.smithi018.stdout:{"epoch":110,"fsid":"a3b3b17e-5a4c-11ee-8db4-212e2dc638e7","created":"2023-09-23T20:06:54.359034+0000","modified":"2023-09-23T20:21:49.935938+0000","last_up_change":"2023-09-23T20:21:48.934116+0000","last_in_change":"2023-09-23T20:11:09.202130+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":22,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-09-23T20:09:47.162973+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-09-23T20:11:59.431837+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"86","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":15,"snap_epoch":86,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-09-23T20:12:42.254901+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-09-23T20:12:43.191578+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"69","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"44c8aaa3-bbd1-4330-8d34-73f7a08b0d4d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":90,"up_from":94,"up_thru":109,"down_at":91,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6802","nonce":990458588},{"type":"v1","addr":"172.21.15.18:6803","nonce":990458588}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6804","nonce":990458588},{"type":"v1","addr":"172.21.15.18:6805","nonce":990458588}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6808","nonce":990458588},{"type":"v1","addr":"172.21.15.18:6809","nonce":990458588}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6806","nonce":990458588},{"type":"v1","addr":"172.21.15.18:6807","nonce":990458588}]},"public_addr":"172.21.15.18:6803/990458588","cluster_addr":"172.21.15.18:6805/990458588","heartbeat_back_addr":"172.21.15.18:6809/990458588","heartbeat_front_addr":"172.21.15.18:6807/990458588","state":["exists","up"]},{"osd":1,"uuid":"c0534324-49c8-4804-bced-c4d13510e74a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":95,"up_from":99,"up_thru":109,"down_at":96,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6810","nonce":1422592526},{"type":"v1","addr":"172.21.15.18:6811","nonce":1422592526}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6812","nonce":1422592526},{"type":"v1","addr":"172.21.15.18:6813","nonce":1422592526}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6816","nonce":1422592526},{"type":"v1","addr":"172.21.15.18:6817","nonce":1422592526}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6814","nonce":1422592526},{"type":"v1","addr":"172.21.15.18:6815","nonce":1422592526}]},"public_addr":"172.21.15.18:6811/1422592526","cluster_addr":"172.21.15.18:6813/1422592526","heartbeat_back_addr":"172.21.15.18:6817/1422592526","heartbeat_front_addr":"172.21.15.18:6815/1422592526","state":["exists","up"]},{"osd":2,"uuid":"f3b2116d-a3d5-4279-91dc-eb47458d3239","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":100,"up_from":104,"up_thru":109,"down_at":101,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6818","nonce":2061051826},{"type":"v1","addr":"172.21.15.18:6819","nonce":2061051826}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6820","nonce":2061051826},{"type":"v1","addr":"172.21.15.18:6821","nonce":2061051826}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6824","nonce":2061051826},{"type":"v1","addr":"172.21.15.18:6825","nonce":2061051826}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6822","nonce":2061051826},{"type":"v1","addr":"172.21.15.18:6823","nonce":2061051826}]},"public_addr":"172.21.15.18:6819/2061051826","cluster_addr":"172.21.15.18:6821/2061051826","heartbeat_back_addr":"172.21.15.18:6825/2061051826","heartbeat_front_addr":"172.21.15.18:6823/2061051826","state":["exists","up"]},{"osd":3,"uuid":"8edccb7f-c4c9-47f3-a6f5-38a1a38cb099","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":105,"up_from":109,"up_thru":109,"down_at":106,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6826","nonce":2876439245},{"type":"v1","addr":"172.21.15.18:6827","nonce":2876439245}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6828","nonce":2876439245},{"type":"v1","addr":"172.21.15.18:6829","nonce":2876439245}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6832","nonce":2876439245},{"type":"v1","addr":"172.21.15.18:6833","nonce":2876439245}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6830","nonce":2876439245},{"type":"v1","addr":"172.21.15.18:6831","nonce":2876439245}]},"public_addr":"172.21.15.18:6827/2876439245","cluster_addr":"172.21.15.18:6829/2876439245","heartbeat_back_addr":"172.21.15.18:6833/2876439245","heartbeat_front_addr":"172.21.15.18:6831/2876439245","state":["exists","up"]},{"osd":4,"uuid":"1d5a64ce-fe42-4355-991a-dfb20cbe5423","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":109,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6800","nonce":3640432477},{"type":"v1","addr":"172.21.15.149:6801","nonce":3640432477}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6802","nonce":3640432477},{"type":"v1","addr":"172.21.15.149:6803","nonce":3640432477}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6806","nonce":3640432477},{"type":"v1","addr":"172.21.15.149:6807","nonce":3640432477}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6804","nonce":3640432477},{"type":"v1","addr":"172.21.15.149:6805","nonce":3640432477}]},"public_addr":"172.21.15.149:6801/3640432477","cluster_addr":"172.21.15.149:6803/3640432477","heartbeat_back_addr":"172.21.15.149:6807/3640432477","heartbeat_front_addr":"172.21.15.149:6805/3640432477","state":["exists","up"]},{"osd":5,"uuid":"105ae3b2-b2fc-477d-9134-3d517bc9427e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":109,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6808","nonce":2969709367},{"type":"v1","addr":"172.21.15.149:6809","nonce":2969709367}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6810","nonce":2969709367},{"type":"v1","addr":"172.21.15.149:6811","nonce":2969709367}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6814","nonce":2969709367},{"type":"v1","addr":"172.21.15.149:6815","nonce":2969709367}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6812","nonce":2969709367},{"type":"v1","addr":"172.21.15.149:6813","nonce":2969709367}]},"public_addr":"172.21.15.149:6809/2969709367","cluster_addr":"172.21.15.149:6811/2969709367","heartbeat_back_addr":"172.21.15.149:6815/2969709367","heartbeat_front_addr":"172.21.15.149:6813/2969709367","state":["exists","up"]},{"osd":6,"uuid":"27d2d2a8-e8c6-4a05-a85e-bd54376cf4c1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":106,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6816","nonce":1291453593},{"type":"v1","addr":"172.21.15.149:6817","nonce":1291453593}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6818","nonce":1291453593},{"type":"v1","addr":"172.21.15.149:6819","nonce":1291453593}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6822","nonce":1291453593},{"type":"v1","addr":"172.21.15.149:6823","nonce":1291453593}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6820","nonce":1291453593},{"type":"v1","addr":"172.21.15.149:6821","nonce":1291453593}]},"public_addr":"172.21.15.149:6817/1291453593","cluster_addr":"172.21.15.149:6819/1291453593","heartbeat_back_addr":"172.21.15.149:6823/1291453593","heartbeat_front_addr":"172.21.15.149:6821/1291453593","state":["exists","up"]},{"osd":7,"uuid":"9da636f7-995c-4f29-9264-70d09c6f66c6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":109,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6824","nonce":3117985178},{"type":"v1","addr":"172.21.15.149:6825","nonce":3117985178}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6826","nonce":3117985178},{"type":"v1","addr":"172.21.15.149:6827","nonce":3117985178}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6830","nonce":3117985178},{"type":"v1","addr":"172.21.15.149:6831","nonce":3117985178}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6828","nonce":3117985178},{"type":"v1","addr":"172.21.15.149:6829","nonce":3117985178}]},"public_addr":"172.21.15.149:6825/3117985178","cluster_addr":"172.21.15.149:6827/3117985178","heartbeat_back_addr":"172.21.15.149:6831/3117985178","heartbeat_front_addr":"172.21.15.149:6829/3117985178","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:09:01.034520+0000","dead_epoch":90},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:09:21.795238+0000","dead_epoch":95},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:09:42.987835+0000","dead_epoch":100},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:10:04.092458+0000","dead_epoch":105},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:10:23.309112+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:10:43.348231+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:11:03.128458+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:11:23.084228+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.d","mappings":[{"from":1,"to":5}]},{"pgid":"4.f","mappings":[{"from":1,"to":2}]},{"pgid":"4.18","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.18:6801/2791912398":"2023-09-24T20:15:59.309691+0000","172.21.15.149:0/502112376":"2023-09-24T20:15:19.879895+0000","172.21.15.18:0/69122222":"2023-09-24T20:07:23.919815+0000","172.21.15.149:6835/1754402171":"2023-09-24T20:15:19.879895+0000","172.21.15.18:0/3831267686":"2023-09-24T20:07:48.635784+0000","172.21.15.149:0/3406344898":"2023-09-24T20:15:19.879895+0000","172.21.15.149:6834/1754402171":"2023-09-24T20:15:19.879895+0000","172.21.15.18:0/1988472148":"2023-09-24T20:11:49.652762+0000","172.21.15.18:0/1082665289":"2023-09-24T20:07:48.635784+0000","172.21.15.18:6800/1930027051":"2023-09-24T20:07:23.919815+0000","172.21.15.18:0/2809002822":"2023-09-24T20:14:38.972322+0000","172.21.15.18:6801/2611527229":"2023-09-24T20:11:49.652762+0000","172.21.15.149:0/2486656126":"2023-09-24T20:15:19.879895+0000","172.21.15.18:0/1119348530":"2023-09-24T20:15:59.309691+0000","172.21.15.18:6800/2611527229":"2023-09-24T20:11:49.652762+0000","172.21.15.18:0/2543728509":"2023-09-24T20:14:38.972322+0000","172.21.15.18:0/2993111051":"2023-09-24T20:07:23.919815+0000","172.21.15.18:0/3990640673":"2023-09-24T20:14:38.972322+0000","172.21.15.18:0/1698512393":"2023-09-24T20:11:49.652762+0000","172.21.15.149:0/894377242":"2023-09-24T20:15:19.879895+0000","172.21.15.18:0/3356195690":"2023-09-24T20:14:38.972322+0000","172.21.15.18:6801/1930027051":"2023-09-24T20:07:23.919815+0000","172.21.15.18:6801/380813540":"2023-09-24T20:07:48.635784+0000","172.21.15.18:0/3863388488":"2023-09-24T20:11:49.652762+0000","172.21.15.18:0/2549145961":"2023-09-24T20:15:59.309691+0000","172.21.15.149:0/1078964609":"2023-09-24T20:15:19.879895+0000","172.21.15.18:0/1645667747":"2023-09-24T20:07:48.635784+0000","172.21.15.18:6800/380813540":"2023-09-24T20:07:48.635784+0000","172.21.15.18:6800/115279545":"2023-09-24T20:14:38.972322+0000","172.21.15.18:0/3510203881":"2023-09-24T20:15:59.309691+0000","172.21.15.18:6801/115279545":"2023-09-24T20:14:38.972322+0000","172.21.15.18:0/950924496":"2023-09-24T20:07:23.919815+0000","172.21.15.149:0/1653842667":"2023-09-24T20:15:19.879895+0000","172.21.15.18:0/3334666861":"2023-09-24T20:15:59.309691+0000","172.21.15.18:0/3912151186":"2023-09-24T20:11:49.652762+0000","172.21.15.18:0/872601545":"2023-09-24T20:15:59.309691+0000","172.21.15.18:0/1505013437":"2023-09-24T20:15:59.309691+0000","172.21.15.18:6800/2791912398":"2023-09-24T20:15:59.309691+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-23T20:22:09.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:22:08 smithi149 ceph-mon[170394]: pgmap v207: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:22:09.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:22:08 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:22:09.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:22:08 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:22:09.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:22:08 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:22:09.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:22:08 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:22:09.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:22:08 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:22:09.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:22:08 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:22:09.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:22:08 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:22:09.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:22:08 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:22:09.578 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph tell osd.0 flush_pg_stats 2023-09-23T20:22:09.579 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph tell osd.1 flush_pg_stats 2023-09-23T20:22:09.579 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph tell osd.2 flush_pg_stats 2023-09-23T20:22:09.580 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph tell osd.3 flush_pg_stats 2023-09-23T20:22:09.580 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph tell osd.4 flush_pg_stats 2023-09-23T20:22:09.580 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph tell osd.5 flush_pg_stats 2023-09-23T20:22:09.581 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph tell osd.6 flush_pg_stats 2023-09-23T20:22:09.581 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph tell osd.7 flush_pg_stats 2023-09-23T20:22:09.904 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:22:09 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/292824649' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T20:22:10.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:22:09 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/292824649' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T20:22:10.390 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:22:10.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:22:09 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/292824649' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T20:22:10.778 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:22:10.882 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:22:10 smithi018 ceph-mon[179647]: pgmap v208: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:22:10.946 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:22:11.022 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:22:11.123 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:22:11.140 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:22:10 smithi018 ceph-mon[184144]: pgmap v208: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:22:11.170 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:22:11.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:22:10 smithi149 ceph-mon[170394]: pgmap v208: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:22:11.378 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:22:11.582 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:22:12.886 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:22:12 smithi018 ceph-mon[179647]: pgmap v209: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:22:13.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:22:12 smithi018 ceph-mon[184144]: pgmap v209: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:22:13.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:22:12 smithi149 ceph-mon[170394]: pgmap v209: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:22:15.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:22:14 smithi018 ceph-mon[184144]: pgmap v210: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:22:15.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:22:14 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:22:15.141 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:22:14 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:22:14] "GET /metrics HTTP/1.1" 200 34029 "" "Prometheus/2.43.0" 2023-09-23T20:22:15.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:22:14 smithi018 ceph-mon[179647]: pgmap v210: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:22:15.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:22:14 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:22:15.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:22:14 smithi149 ceph-mon[170394]: pgmap v210: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:22:15.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:22:14 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:22:15.298 INFO:teuthology.orchestra.run.smithi018.stdout:425201762323 2023-09-23T20:22:15.298 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.1 2023-09-23T20:22:16.116 INFO:teuthology.orchestra.run.smithi018.stdout:403726925850 2023-09-23T20:22:16.117 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.0 2023-09-23T20:22:17.039 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:22:16 smithi018 ceph-mon[184144]: pgmap v211: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:22:17.039 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:22:16 smithi018 ceph-mon[179647]: pgmap v211: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:22:17.040 INFO:teuthology.orchestra.run.smithi018.stdout:167503724685 2023-09-23T20:22:17.040 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.5 2023-09-23T20:22:17.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:22:16 smithi149 ceph-mon[170394]: pgmap v211: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:22:17.611 INFO:teuthology.orchestra.run.smithi018.stdout:236223201413 2023-09-23T20:22:17.611 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.7 2023-09-23T20:22:17.761 INFO:teuthology.orchestra.run.smithi018.stdout:446676598797 2023-09-23T20:22:17.762 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.2 2023-09-23T20:22:18.389 INFO:teuthology.orchestra.run.smithi018.stdout:201863463049 2023-09-23T20:22:18.389 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.6 2023-09-23T20:22:18.704 INFO:teuthology.orchestra.run.smithi018.stdout:468151435272 2023-09-23T20:22:18.704 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.3 2023-09-23T20:22:18.902 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:22:18.957 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:22:18 smithi018 ceph-mon[184144]: pgmap v212: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:22:18.957 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:22:18 smithi018 ceph-mon[179647]: pgmap v212: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:22:18.958 INFO:teuthology.orchestra.run.smithi018.stdout:128849019025 2023-09-23T20:22:18.959 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.4 2023-09-23T20:22:19.056 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:22:19.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:22:18 smithi149 ceph-mon[170394]: pgmap v212: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:22:19.373 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:22:19.373 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:22:19.430 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:22:19.928 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:22:20.087 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:22:21.076 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:22:20 smithi018 ceph-mon[179647]: pgmap v213: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:22:21.076 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:22:20 smithi018 ceph-mon[184144]: pgmap v213: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:22:21.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:22:20 smithi149 ceph-mon[170394]: pgmap v213: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:22:21.530 INFO:teuthology.orchestra.run.smithi018.stdout:403726925851 2023-09-23T20:22:21.654 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:22:22.097 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:22:21 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1734413189' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T20:22:22.098 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:22:21 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1734413189' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T20:22:22.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:22:21 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1734413189' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T20:22:23.125 INFO:tasks.cephadm.ceph_manager.ceph:need seq 403726925850 got 403726925851 for osd.0 2023-09-23T20:22:23.147 DEBUG:teuthology.parallel:result is None 2023-09-23T20:22:23.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:22:22 smithi149 ceph-mon[170394]: pgmap v214: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:22:23.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:22:22 smithi018 ceph-mon[179647]: pgmap v214: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:22:23.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:22:22 smithi018 ceph-mon[184144]: pgmap v214: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:22:23.562 INFO:teuthology.orchestra.run.smithi018.stdout:425201762325 2023-09-23T20:22:23.987 INFO:tasks.cephadm.ceph_manager.ceph:need seq 425201762323 got 425201762325 for osd.1 2023-09-23T20:22:23.987 DEBUG:teuthology.parallel:result is None 2023-09-23T20:22:24.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:22:23 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3526919213' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T20:22:24.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:22:23 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3526919213' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T20:22:24.180 INFO:teuthology.orchestra.run.smithi018.stdout:236223201414 2023-09-23T20:22:24.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:22:23 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3526919213' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T20:22:24.713 INFO:tasks.cephadm.ceph_manager.ceph:need seq 236223201413 got 236223201414 for osd.7 2023-09-23T20:22:24.713 DEBUG:teuthology.parallel:result is None 2023-09-23T20:22:24.802 INFO:teuthology.orchestra.run.smithi018.stdout:468151435273 2023-09-23T20:22:24.994 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:22:24 smithi018 ceph-mon[179647]: pgmap v215: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:22:24.995 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:22:24 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/334272542' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T20:22:24.995 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:22:24 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1902129011' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T20:22:24.995 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:22:24 smithi018 ceph-mon[184144]: pgmap v215: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:22:24.995 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:22:24 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/334272542' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T20:22:24.995 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:22:24 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1902129011' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T20:22:25.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:22:24 smithi149 ceph-mon[170394]: pgmap v215: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:22:25.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:22:24 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/334272542' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T20:22:25.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:22:24 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1902129011' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T20:22:25.391 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:22:24 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:22:24] "GET /metrics HTTP/1.1" 200 34029 "" "Prometheus/2.43.0" 2023-09-23T20:22:25.662 INFO:teuthology.orchestra.run.smithi018.stdout:446676598799 2023-09-23T20:22:25.706 INFO:teuthology.orchestra.run.smithi018.stdout:167503724687 2023-09-23T20:22:25.807 INFO:tasks.cephadm.ceph_manager.ceph:need seq 468151435272 got 468151435273 for osd.3 2023-09-23T20:22:25.807 DEBUG:teuthology.parallel:result is None 2023-09-23T20:22:25.881 INFO:teuthology.orchestra.run.smithi018.stdout:201863463050 2023-09-23T20:22:26.090 INFO:teuthology.orchestra.run.smithi018.stdout:128849019027 2023-09-23T20:22:26.187 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:22:25 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2012298986' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T20:22:26.187 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:22:25 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/233195037' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T20:22:26.187 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:22:25 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2404826435' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T20:22:26.188 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:22:25 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2012298986' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T20:22:26.188 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:22:25 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/233195037' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T20:22:26.188 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:22:25 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2404826435' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T20:22:26.189 INFO:tasks.cephadm.ceph_manager.ceph:need seq 167503724685 got 167503724687 for osd.5 2023-09-23T20:22:26.189 DEBUG:teuthology.parallel:result is None 2023-09-23T20:22:26.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:22:25 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2012298986' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T20:22:26.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:22:25 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/233195037' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T20:22:26.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:22:25 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2404826435' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T20:22:26.395 INFO:tasks.cephadm.ceph_manager.ceph:need seq 446676598797 got 446676598799 for osd.2 2023-09-23T20:22:26.395 DEBUG:teuthology.parallel:result is None 2023-09-23T20:22:26.687 INFO:tasks.cephadm.ceph_manager.ceph:need seq 201863463049 got 201863463050 for osd.6 2023-09-23T20:22:26.687 DEBUG:teuthology.parallel:result is None 2023-09-23T20:22:26.836 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849019025 got 128849019027 for osd.4 2023-09-23T20:22:26.836 DEBUG:teuthology.parallel:result is None 2023-09-23T20:22:26.836 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2023-09-23T20:22:26.836 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:22:27.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:22:26 smithi018 ceph-mon[179647]: pgmap v216: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:22:27.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:22:26 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3709844942' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T20:22:27.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:22:26 smithi018 ceph-mon[184144]: pgmap v216: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:22:27.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:22:26 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3709844942' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T20:22:27.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:22:26 smithi149 ceph-mon[170394]: pgmap v216: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:22:27.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:22:26 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3709844942' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T20:22:27.446 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:22:28.762 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:22:28.763 INFO:teuthology.orchestra.run.smithi018.stderr:dumped all 2023-09-23T20:22:29.108 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:22:28 smithi018 ceph-mon[184144]: pgmap v217: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:22:29.108 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:22:28 smithi018 ceph-mon[179647]: pgmap v217: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:22:29.196 INFO:teuthology.orchestra.run.smithi018.stdout:{"pg_ready":true,"pg_map":{"version":217,"stamp":"2023-09-23T20:22:27.428156+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790195,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6495,"num_read_kb":14136,"num_write":1146,"num_write_kb":15141,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1192,"ondisk_log_size":1192,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":164132,"kb_used_data":23076,"kb_used_omap":30,"kb_used_meta":140961,"kb_avail":749797084,"statfs":{"total":767960285184,"available":767792214016,"internally_reserved":0,"allocated":23629824,"data_stored":19305714,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31263,"internal_metadata":144344545},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003054"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":79,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.941990+0000","last_change":"2023-09-23T20:21:18.893413+0000","last_active":"2023-09-23T20:21:48.941990+0000","last_peered":"2023-09-23T20:21:48.941990+0000","last_clean":"2023-09-23T20:21:48.941990+0000","last_became_active":"2023-09-23T20:21:18.893008+0000","last_became_peered":"2023-09-23T20:21:18.893008+0000","last_unstale":"2023-09-23T20:21:48.941990+0000","last_undegraded":"2023-09-23T20:21:48.941990+0000","last_fullsized":"2023-09-23T20:21:48.941990+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:50:55.264882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":56,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.938999+0000","last_change":"2023-09-23T20:21:18.889521+0000","last_active":"2023-09-23T20:21:49.938999+0000","last_peered":"2023-09-23T20:21:49.938999+0000","last_clean":"2023-09-23T20:21:49.938999+0000","last_became_active":"2023-09-23T20:21:18.889268+0000","last_became_peered":"2023-09-23T20:21:18.889268+0000","last_unstale":"2023-09-23T20:21:49.938999+0000","last_undegraded":"2023-09-23T20:21:49.938999+0000","last_fullsized":"2023-09-23T20:21:49.938999+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:05:48.805778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":93,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.949944+0000","last_change":"2023-09-23T20:21:49.949944+0000","last_active":"2023-09-23T20:21:49.949944+0000","last_peered":"2023-09-23T20:21:49.949944+0000","last_clean":"2023-09-23T20:21:49.949944+0000","last_became_active":"2023-09-23T20:21:49.949425+0000","last_became_peered":"2023-09-23T20:21:49.949425+0000","last_unstale":"2023-09-23T20:21:49.949944+0000","last_undegraded":"2023-09-23T20:21:49.949944+0000","last_fullsized":"2023-09-23T20:21:49.949944+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:40:56.395791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":78,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.957394+0000","last_change":"2023-09-23T20:21:49.957394+0000","last_active":"2023-09-23T20:21:49.957394+0000","last_peered":"2023-09-23T20:21:49.957394+0000","last_clean":"2023-09-23T20:21:49.957394+0000","last_became_active":"2023-09-23T20:21:49.957016+0000","last_became_peered":"2023-09-23T20:21:49.957016+0000","last_unstale":"2023-09-23T20:21:49.957394+0000","last_undegraded":"2023-09-23T20:21:49.957394+0000","last_fullsized":"2023-09-23T20:21:49.957394+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:49:26.222448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":86,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.953041+0000","last_change":"2023-09-23T20:21:49.953041+0000","last_active":"2023-09-23T20:21:49.953041+0000","last_peered":"2023-09-23T20:21:49.953041+0000","last_clean":"2023-09-23T20:21:49.953041+0000","last_became_active":"2023-09-23T20:21:49.952763+0000","last_became_peered":"2023-09-23T20:21:49.952763+0000","last_unstale":"2023-09-23T20:21:49.953041+0000","last_undegraded":"2023-09-23T20:21:49.953041+0000","last_fullsized":"2023-09-23T20:21:49.953041+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:58:35.321466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":54,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.950333+0000","last_change":"2023-09-23T20:12:43.753383+0000","last_active":"2023-09-23T20:21:48.950333+0000","last_peered":"2023-09-23T20:21:48.950333+0000","last_clean":"2023-09-23T20:21:48.950333+0000","last_became_active":"2023-09-23T20:12:43.752921+0000","last_became_peered":"2023-09-23T20:12:43.752921+0000","last_unstale":"2023-09-23T20:21:48.950333+0000","last_undegraded":"2023-09-23T20:21:48.950333+0000","last_fullsized":"2023-09-23T20:21:48.950333+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:28:26.982998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":86,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.957885+0000","last_change":"2023-09-23T20:21:49.957885+0000","last_active":"2023-09-23T20:21:49.957885+0000","last_peered":"2023-09-23T20:21:49.957885+0000","last_clean":"2023-09-23T20:21:49.957885+0000","last_became_active":"2023-09-23T20:21:49.957492+0000","last_became_peered":"2023-09-23T20:21:49.957492+0000","last_unstale":"2023-09-23T20:21:49.957885+0000","last_undegraded":"2023-09-23T20:21:49.957885+0000","last_fullsized":"2023-09-23T20:21:49.957885+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:24:55.137836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":71,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.949471+0000","last_change":"2023-09-23T20:20:46.938981+0000","last_active":"2023-09-23T20:21:48.949471+0000","last_peered":"2023-09-23T20:21:48.949471+0000","last_clean":"2023-09-23T20:21:48.949471+0000","last_became_active":"2023-09-23T20:20:46.938627+0000","last_became_peered":"2023-09-23T20:20:46.938627+0000","last_unstale":"2023-09-23T20:21:48.949471+0000","last_undegraded":"2023-09-23T20:21:48.949471+0000","last_fullsized":"2023-09-23T20:21:48.949471+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:44:43.177086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":90,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.953299+0000","last_change":"2023-09-23T20:21:49.953299+0000","last_active":"2023-09-23T20:21:49.953299+0000","last_peered":"2023-09-23T20:21:49.953299+0000","last_clean":"2023-09-23T20:21:49.953299+0000","last_became_active":"2023-09-23T20:21:49.952996+0000","last_became_peered":"2023-09-23T20:21:49.952996+0000","last_unstale":"2023-09-23T20:21:49.953299+0000","last_undegraded":"2023-09-23T20:21:49.953299+0000","last_fullsized":"2023-09-23T20:21:49.953299+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:46:51.746281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":40,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.950883+0000","last_change":"2023-09-23T20:20:14.888484+0000","last_active":"2023-09-23T20:21:48.950883+0000","last_peered":"2023-09-23T20:21:48.950883+0000","last_clean":"2023-09-23T20:21:48.950883+0000","last_became_active":"2023-09-23T20:20:14.887219+0000","last_became_peered":"2023-09-23T20:20:14.887219+0000","last_unstale":"2023-09-23T20:21:48.950883+0000","last_undegraded":"2023-09-23T20:21:48.950883+0000","last_fullsized":"2023-09-23T20:21:48.950883+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:13:53.763649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":52,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.943196+0000","last_change":"2023-09-23T20:12:43.765908+0000","last_active":"2023-09-23T20:21:48.943196+0000","last_peered":"2023-09-23T20:21:48.943196+0000","last_clean":"2023-09-23T20:21:48.943196+0000","last_became_active":"2023-09-23T20:12:43.765558+0000","last_became_peered":"2023-09-23T20:12:43.765558+0000","last_unstale":"2023-09-23T20:21:48.943196+0000","last_undegraded":"2023-09-23T20:21:48.943196+0000","last_fullsized":"2023-09-23T20:21:48.943196+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:35:43.809898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":58,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.939966+0000","last_change":"2023-09-23T20:20:46.937163+0000","last_active":"2023-09-23T20:21:49.939966+0000","last_peered":"2023-09-23T20:21:49.939966+0000","last_clean":"2023-09-23T20:21:49.939966+0000","last_became_active":"2023-09-23T20:20:46.936846+0000","last_became_peered":"2023-09-23T20:20:46.936846+0000","last_unstale":"2023-09-23T20:21:49.939966+0000","last_undegraded":"2023-09-23T20:21:49.939966+0000","last_fullsized":"2023-09-23T20:21:49.939966+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:01:32.798855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"86'117","reported_seq":596,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.945893+0000","last_change":"2023-09-23T20:21:18.897269+0000","last_active":"2023-09-23T20:21:48.945893+0000","last_peered":"2023-09-23T20:21:48.945893+0000","last_clean":"2023-09-23T20:21:48.945893+0000","last_became_active":"2023-09-23T20:21:18.896834+0000","last_became_peered":"2023-09-23T20:21:18.896834+0000","last_unstale":"2023-09-23T20:21:48.945893+0000","last_undegraded":"2023-09-23T20:21:48.945893+0000","last_fullsized":"2023-09-23T20:21:48.945893+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:57:17.499025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046891400000000002,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":893,"num_read_kb":922,"num_write":82,"num_write_kb":4193,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":67,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.953565+0000","last_change":"2023-09-23T20:21:49.953565+0000","last_active":"2023-09-23T20:21:49.953565+0000","last_peered":"2023-09-23T20:21:49.953565+0000","last_clean":"2023-09-23T20:21:49.953565+0000","last_became_active":"2023-09-23T20:21:49.953176+0000","last_became_peered":"2023-09-23T20:21:49.953176+0000","last_unstale":"2023-09-23T20:21:49.953565+0000","last_undegraded":"2023-09-23T20:21:49.953565+0000","last_fullsized":"2023-09-23T20:21:49.953565+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:02:35.778207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"86'128","reported_seq":432,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.940390+0000","last_change":"2023-09-23T20:20:46.939256+0000","last_active":"2023-09-23T20:21:48.940390+0000","last_peered":"2023-09-23T20:21:48.940390+0000","last_clean":"2023-09-23T20:21:48.940390+0000","last_became_active":"2023-09-23T20:20:46.938756+0000","last_became_peered":"2023-09-23T20:20:46.938756+0000","last_unstale":"2023-09-23T20:21:48.940390+0000","last_undegraded":"2023-09-23T20:21:48.940390+0000","last_fullsized":"2023-09-23T20:21:48.940390+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:52:18.161436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000293202,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":579,"num_read_kb":1473,"num_write":96,"num_write_kb":1062,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":70,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.957535+0000","last_change":"2023-09-23T20:21:49.957535+0000","last_active":"2023-09-23T20:21:49.957535+0000","last_peered":"2023-09-23T20:21:49.957535+0000","last_clean":"2023-09-23T20:21:49.957535+0000","last_became_active":"2023-09-23T20:21:49.954319+0000","last_became_peered":"2023-09-23T20:21:49.954319+0000","last_unstale":"2023-09-23T20:21:49.957535+0000","last_undegraded":"2023-09-23T20:21:49.957535+0000","last_fullsized":"2023-09-23T20:21:49.957535+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:14:10.293305+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":38,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.938907+0000","last_change":"2023-09-23T20:20:14.888874+0000","last_active":"2023-09-23T20:21:49.938907+0000","last_peered":"2023-09-23T20:21:49.938907+0000","last_clean":"2023-09-23T20:21:49.938907+0000","last_became_active":"2023-09-23T20:20:14.887724+0000","last_became_peered":"2023-09-23T20:20:14.887724+0000","last_unstale":"2023-09-23T20:21:49.938907+0000","last_undegraded":"2023-09-23T20:21:49.938907+0000","last_fullsized":"2023-09-23T20:21:49.938907+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:09:24.510326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":47,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.939386+0000","last_change":"2023-09-23T20:21:18.893457+0000","last_active":"2023-09-23T20:21:48.939386+0000","last_peered":"2023-09-23T20:21:48.939386+0000","last_clean":"2023-09-23T20:21:48.939386+0000","last_became_active":"2023-09-23T20:21:18.893175+0000","last_became_peered":"2023-09-23T20:21:18.893175+0000","last_unstale":"2023-09-23T20:21:48.939386+0000","last_undegraded":"2023-09-23T20:21:48.939386+0000","last_fullsized":"2023-09-23T20:21:48.939386+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:24:37.563758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"86'102","reported_seq":616,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.951036+0000","last_change":"2023-09-23T20:20:14.881975+0000","last_active":"2023-09-23T20:21:48.951036+0000","last_peered":"2023-09-23T20:21:48.951036+0000","last_clean":"2023-09-23T20:21:48.951036+0000","last_became_active":"2023-09-23T20:20:14.881544+0000","last_became_peered":"2023-09-23T20:20:14.881544+0000","last_unstale":"2023-09-23T20:21:48.951036+0000","last_undegraded":"2023-09-23T20:21:48.951036+0000","last_fullsized":"2023-09-23T20:21:48.951036+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":102,"log_dups_size":0,"ondisk_log_size":102,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:34.594972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030656500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":963,"num_read_kb":1234,"num_write":60,"num_write_kb":225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":68,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.940049+0000","last_change":"2023-09-23T20:21:18.887651+0000","last_active":"2023-09-23T20:21:49.940049+0000","last_peered":"2023-09-23T20:21:49.940049+0000","last_clean":"2023-09-23T20:21:49.940049+0000","last_became_active":"2023-09-23T20:21:18.887338+0000","last_became_peered":"2023-09-23T20:21:18.887338+0000","last_unstale":"2023-09-23T20:21:49.940049+0000","last_undegraded":"2023-09-23T20:21:49.940049+0000","last_fullsized":"2023-09-23T20:21:49.940049+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:26:39.067022+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":52,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.952295+0000","last_change":"2023-09-23T20:21:49.951490+0000","last_active":"2023-09-23T20:21:49.952295+0000","last_peered":"2023-09-23T20:21:49.952295+0000","last_clean":"2023-09-23T20:21:49.952295+0000","last_became_active":"2023-09-23T20:21:49.951079+0000","last_became_peered":"2023-09-23T20:21:49.951079+0000","last_unstale":"2023-09-23T20:21:49.952295+0000","last_undegraded":"2023-09-23T20:21:49.952295+0000","last_fullsized":"2023-09-23T20:21:49.952295+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:12:01.053013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"86'80","reported_seq":356,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.952040+0000","last_change":"2023-09-23T20:20:46.950784+0000","last_active":"2023-09-23T20:21:48.952040+0000","last_peered":"2023-09-23T20:21:48.952040+0000","last_clean":"2023-09-23T20:21:48.952040+0000","last_became_active":"2023-09-23T20:20:46.950071+0000","last_became_peered":"2023-09-23T20:20:46.950071+0000","last_unstale":"2023-09-23T20:21:48.952040+0000","last_undegraded":"2023-09-23T20:21:48.952040+0000","last_fullsized":"2023-09-23T20:21:48.952040+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":80,"log_dups_size":0,"ondisk_log_size":80,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:11:06.812404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035115,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":395,"num_read_kb":1370,"num_write":66,"num_write_kb":1061,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":52,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.946301+0000","last_change":"2023-09-23T20:12:45.609829+0000","last_active":"2023-09-23T20:21:48.946301+0000","last_peered":"2023-09-23T20:21:48.946301+0000","last_clean":"2023-09-23T20:21:48.946301+0000","last_became_active":"2023-09-23T20:12:45.609467+0000","last_became_peered":"2023-09-23T20:12:45.609467+0000","last_unstale":"2023-09-23T20:21:48.946301+0000","last_undegraded":"2023-09-23T20:21:48.946301+0000","last_fullsized":"2023-09-23T20:21:48.946301+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:27:31.490555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":22,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.956277+0000","last_change":"2023-09-23T20:21:49.956277+0000","last_active":"2023-09-23T20:21:49.956277+0000","last_peered":"2023-09-23T20:21:49.956277+0000","last_clean":"2023-09-23T20:21:49.956277+0000","last_became_active":"2023-09-23T20:21:49.955974+0000","last_became_peered":"2023-09-23T20:21:49.955974+0000","last_unstale":"2023-09-23T20:21:49.956277+0000","last_undegraded":"2023-09-23T20:21:49.956277+0000","last_fullsized":"2023-09-23T20:21:49.956277+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:11:37.850675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":452,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.951949+0000","last_change":"2023-09-23T20:20:14.882204+0000","last_active":"2023-09-23T20:21:48.951949+0000","last_peered":"2023-09-23T20:21:48.951949+0000","last_clean":"2023-09-23T20:21:48.951949+0000","last_became_active":"2023-09-23T20:20:14.881699+0000","last_became_peered":"2023-09-23T20:20:14.881699+0000","last_unstale":"2023-09-23T20:21:48.951949+0000","last_undegraded":"2023-09-23T20:21:48.951949+0000","last_fullsized":"2023-09-23T20:21:48.951949+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:09:47.213130+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:16:37.341010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"86'194","reported_seq":728,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.948141+0000","last_change":"2023-09-23T20:21:18.892176+0000","last_active":"2023-09-23T20:21:48.948141+0000","last_peered":"2023-09-23T20:21:48.948141+0000","last_clean":"2023-09-23T20:21:48.948141+0000","last_became_active":"2023-09-23T20:21:18.891647+0000","last_became_peered":"2023-09-23T20:21:18.891647+0000","last_unstale":"2023-09-23T20:21:48.948141+0000","last_undegraded":"2023-09-23T20:21:48.948141+0000","last_fullsized":"2023-09-23T20:21:48.948141+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":194,"log_dups_size":0,"ondisk_log_size":194,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:25:52.557392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00037995999999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1025,"num_read_kb":1201,"num_write":173,"num_write_kb":1276,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":76,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.939413+0000","last_change":"2023-09-23T20:20:46.936524+0000","last_active":"2023-09-23T20:21:48.939413+0000","last_peered":"2023-09-23T20:21:48.939413+0000","last_clean":"2023-09-23T20:21:48.939413+0000","last_became_active":"2023-09-23T20:20:46.936267+0000","last_became_peered":"2023-09-23T20:20:46.936267+0000","last_unstale":"2023-09-23T20:21:48.939413+0000","last_undegraded":"2023-09-23T20:21:48.939413+0000","last_fullsized":"2023-09-23T20:21:48.939413+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:32:20.984521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":47,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.937923+0000","last_change":"2023-09-23T20:21:18.891553+0000","last_active":"2023-09-23T20:21:48.937923+0000","last_peered":"2023-09-23T20:21:48.937923+0000","last_clean":"2023-09-23T20:21:48.937923+0000","last_became_active":"2023-09-23T20:21:18.891230+0000","last_became_peered":"2023-09-23T20:21:18.891230+0000","last_unstale":"2023-09-23T20:21:48.937923+0000","last_undegraded":"2023-09-23T20:21:48.937923+0000","last_fullsized":"2023-09-23T20:21:48.937923+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:15:04.114927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"86'130","reported_seq":413,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.954331+0000","last_change":"2023-09-23T20:21:49.954331+0000","last_active":"2023-09-23T20:21:49.954331+0000","last_peered":"2023-09-23T20:21:49.954331+0000","last_clean":"2023-09-23T20:21:49.954331+0000","last_became_active":"2023-09-23T20:21:49.953687+0000","last_became_peered":"2023-09-23T20:21:49.953687+0000","last_unstale":"2023-09-23T20:21:49.954331+0000","last_undegraded":"2023-09-23T20:21:49.954331+0000","last_fullsized":"2023-09-23T20:21:49.954331+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:12.549803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011300049999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":512,"num_read_kb":1820,"num_write":133,"num_write_kb":1347,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":63,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.952961+0000","last_change":"2023-09-23T20:21:49.952961+0000","last_active":"2023-09-23T20:21:49.952961+0000","last_peered":"2023-09-23T20:21:49.952961+0000","last_clean":"2023-09-23T20:21:49.952961+0000","last_became_active":"2023-09-23T20:21:49.952644+0000","last_became_peered":"2023-09-23T20:21:49.952644+0000","last_unstale":"2023-09-23T20:21:49.952961+0000","last_undegraded":"2023-09-23T20:21:49.952961+0000","last_fullsized":"2023-09-23T20:21:49.952961+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:05:30.492540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":70,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.943468+0000","last_change":"2023-09-23T20:20:14.887345+0000","last_active":"2023-09-23T20:21:48.943468+0000","last_peered":"2023-09-23T20:21:48.943468+0000","last_clean":"2023-09-23T20:21:48.943468+0000","last_became_active":"2023-09-23T20:20:14.886970+0000","last_became_peered":"2023-09-23T20:20:14.886970+0000","last_unstale":"2023-09-23T20:21:48.943468+0000","last_undegraded":"2023-09-23T20:21:48.943468+0000","last_fullsized":"2023-09-23T20:21:48.943468+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:00:38.575926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"86'106","reported_seq":734,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.949374+0000","last_change":"2023-09-23T20:20:46.939468+0000","last_active":"2023-09-23T20:21:48.949374+0000","last_peered":"2023-09-23T20:21:48.949374+0000","last_clean":"2023-09-23T20:21:48.949374+0000","last_became_active":"2023-09-23T20:20:46.938954+0000","last_became_peered":"2023-09-23T20:20:46.938954+0000","last_unstale":"2023-09-23T20:21:48.949374+0000","last_undegraded":"2023-09-23T20:21:48.949374+0000","last_fullsized":"2023-09-23T20:21:48.949374+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":106,"log_dups_size":0,"ondisk_log_size":106,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:01:43.761064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00099302599999999993,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1105,"num_read_kb":1689,"num_write":100,"num_write_kb":404,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":33,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.938411+0000","last_change":"2023-09-23T20:20:46.937104+0000","last_active":"2023-09-23T20:21:48.938411+0000","last_peered":"2023-09-23T20:21:48.938411+0000","last_clean":"2023-09-23T20:21:48.938411+0000","last_became_active":"2023-09-23T20:20:46.936820+0000","last_became_peered":"2023-09-23T20:20:46.936820+0000","last_unstale":"2023-09-23T20:21:48.938411+0000","last_undegraded":"2023-09-23T20:21:48.938411+0000","last_fullsized":"2023-09-23T20:21:48.938411+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:22:19.879310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":86,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.953141+0000","last_change":"2023-09-23T20:21:49.953141+0000","last_active":"2023-09-23T20:21:49.953141+0000","last_peered":"2023-09-23T20:21:49.953141+0000","last_clean":"2023-09-23T20:21:49.953141+0000","last_became_active":"2023-09-23T20:21:49.952794+0000","last_became_peered":"2023-09-23T20:21:49.952794+0000","last_unstale":"2023-09-23T20:21:49.953141+0000","last_undegraded":"2023-09-23T20:21:49.953141+0000","last_fullsized":"2023-09-23T20:21:49.953141+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:36:19.919248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"86'119","reported_seq":441,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.939283+0000","last_change":"2023-09-23T20:20:46.937071+0000","last_active":"2023-09-23T20:21:48.939283+0000","last_peered":"2023-09-23T20:21:48.939283+0000","last_clean":"2023-09-23T20:21:48.939283+0000","last_became_active":"2023-09-23T20:20:46.935348+0000","last_became_peered":"2023-09-23T20:20:46.935348+0000","last_unstale":"2023-09-23T20:21:48.939283+0000","last_undegraded":"2023-09-23T20:21:48.939283+0000","last_fullsized":"2023-09-23T20:21:48.939283+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":119,"log_dups_size":0,"ondisk_log_size":119,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:09.829030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032705499999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":523,"num_read_kb":3503,"num_write":136,"num_write_kb":3128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":43,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.959532+0000","last_change":"2023-09-23T20:21:49.959532+0000","last_active":"2023-09-23T20:21:49.959532+0000","last_peered":"2023-09-23T20:21:49.959532+0000","last_clean":"2023-09-23T20:21:49.959532+0000","last_became_active":"2023-09-23T20:21:49.959161+0000","last_became_peered":"2023-09-23T20:21:49.959161+0000","last_unstale":"2023-09-23T20:21:49.959532+0000","last_undegraded":"2023-09-23T20:21:49.959532+0000","last_fullsized":"2023-09-23T20:21:49.959532+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:00:05.013667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":73,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.958915+0000","last_change":"2023-09-23T20:21:49.958915+0000","last_active":"2023-09-23T20:21:49.958915+0000","last_peered":"2023-09-23T20:21:49.958915+0000","last_clean":"2023-09-23T20:21:49.958915+0000","last_became_active":"2023-09-23T20:21:49.958440+0000","last_became_peered":"2023-09-23T20:21:49.958440+0000","last_unstale":"2023-09-23T20:21:49.958915+0000","last_undegraded":"2023-09-23T20:21:49.958915+0000","last_fullsized":"2023-09-23T20:21:49.958915+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:36:48.998945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":69,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.942436+0000","last_change":"2023-09-23T20:20:46.939720+0000","last_active":"2023-09-23T20:21:48.942436+0000","last_peered":"2023-09-23T20:21:48.942436+0000","last_clean":"2023-09-23T20:21:48.942436+0000","last_became_active":"2023-09-23T20:20:46.939357+0000","last_became_peered":"2023-09-23T20:20:46.939357+0000","last_unstale":"2023-09-23T20:21:48.942436+0000","last_undegraded":"2023-09-23T20:21:48.942436+0000","last_fullsized":"2023-09-23T20:21:48.942436+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:56.613212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":70,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.943289+0000","last_change":"2023-09-23T20:21:18.896635+0000","last_active":"2023-09-23T20:21:48.943289+0000","last_peered":"2023-09-23T20:21:48.943289+0000","last_clean":"2023-09-23T20:21:48.943289+0000","last_became_active":"2023-09-23T20:21:18.896335+0000","last_became_peered":"2023-09-23T20:21:18.896335+0000","last_unstale":"2023-09-23T20:21:48.943289+0000","last_undegraded":"2023-09-23T20:21:48.943289+0000","last_fullsized":"2023-09-23T20:21:48.943289+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:46:43.743795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":79,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.944402+0000","last_change":"2023-09-23T20:21:18.891669+0000","last_active":"2023-09-23T20:21:48.944402+0000","last_peered":"2023-09-23T20:21:48.944402+0000","last_clean":"2023-09-23T20:21:48.944402+0000","last_became_active":"2023-09-23T20:21:18.891326+0000","last_became_peered":"2023-09-23T20:21:18.891326+0000","last_unstale":"2023-09-23T20:21:48.944402+0000","last_undegraded":"2023-09-23T20:21:48.944402+0000","last_fullsized":"2023-09-23T20:21:48.944402+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:48:39.893213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":70,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.946401+0000","last_change":"2023-09-23T20:20:46.937163+0000","last_active":"2023-09-23T20:21:48.946401+0000","last_peered":"2023-09-23T20:21:48.946401+0000","last_clean":"2023-09-23T20:21:48.946401+0000","last_became_active":"2023-09-23T20:20:46.936757+0000","last_became_peered":"2023-09-23T20:20:46.936757+0000","last_unstale":"2023-09-23T20:21:48.946401+0000","last_undegraded":"2023-09-23T20:21:48.946401+0000","last_fullsized":"2023-09-23T20:21:48.946401+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:09.401379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":69,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.954710+0000","last_change":"2023-09-23T20:21:49.954710+0000","last_active":"2023-09-23T20:21:49.954710+0000","last_peered":"2023-09-23T20:21:49.954710+0000","last_clean":"2023-09-23T20:21:49.954710+0000","last_became_active":"2023-09-23T20:21:49.954355+0000","last_became_peered":"2023-09-23T20:21:49.954355+0000","last_unstale":"2023-09-23T20:21:49.954710+0000","last_undegraded":"2023-09-23T20:21:49.954710+0000","last_fullsized":"2023-09-23T20:21:49.954710+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:56:16.233285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":70,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.953004+0000","last_change":"2023-09-23T20:21:49.953004+0000","last_active":"2023-09-23T20:21:49.953004+0000","last_peered":"2023-09-23T20:21:49.953004+0000","last_clean":"2023-09-23T20:21:49.953004+0000","last_became_active":"2023-09-23T20:21:49.952625+0000","last_became_peered":"2023-09-23T20:21:49.952625+0000","last_unstale":"2023-09-23T20:21:49.953004+0000","last_undegraded":"2023-09-23T20:21:49.953004+0000","last_fullsized":"2023-09-23T20:21:49.953004+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:48:28.305640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":52,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.950817+0000","last_change":"2023-09-23T20:20:46.935968+0000","last_active":"2023-09-23T20:21:48.950817+0000","last_peered":"2023-09-23T20:21:48.950817+0000","last_clean":"2023-09-23T20:21:48.950817+0000","last_became_active":"2023-09-23T20:20:46.935618+0000","last_became_peered":"2023-09-23T20:20:46.935618+0000","last_unstale":"2023-09-23T20:21:48.950817+0000","last_undegraded":"2023-09-23T20:21:48.950817+0000","last_fullsized":"2023-09-23T20:21:48.950817+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:42:59.867176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":69,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.954875+0000","last_change":"2023-09-23T20:21:49.954875+0000","last_active":"2023-09-23T20:21:49.954875+0000","last_peered":"2023-09-23T20:21:49.954875+0000","last_clean":"2023-09-23T20:21:49.954875+0000","last_became_active":"2023-09-23T20:21:49.954519+0000","last_became_peered":"2023-09-23T20:21:49.954519+0000","last_unstale":"2023-09-23T20:21:49.954875+0000","last_undegraded":"2023-09-23T20:21:49.954875+0000","last_fullsized":"2023-09-23T20:21:49.954875+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:29:52.962029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":69,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.946248+0000","last_change":"2023-09-23T20:20:46.934762+0000","last_active":"2023-09-23T20:21:48.946248+0000","last_peered":"2023-09-23T20:21:48.946248+0000","last_clean":"2023-09-23T20:21:48.946248+0000","last_became_active":"2023-09-23T20:20:46.934472+0000","last_became_peered":"2023-09-23T20:20:46.934472+0000","last_unstale":"2023-09-23T20:21:48.946248+0000","last_undegraded":"2023-09-23T20:21:48.946248+0000","last_fullsized":"2023-09-23T20:21:48.946248+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:28:42.708286+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":54,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.951041+0000","last_change":"2023-09-23T20:12:43.751523+0000","last_active":"2023-09-23T20:21:48.951041+0000","last_peered":"2023-09-23T20:21:48.951041+0000","last_clean":"2023-09-23T20:21:48.951041+0000","last_became_active":"2023-09-23T20:12:43.751257+0000","last_became_peered":"2023-09-23T20:12:43.751257+0000","last_unstale":"2023-09-23T20:21:48.951041+0000","last_undegraded":"2023-09-23T20:21:48.951041+0000","last_fullsized":"2023-09-23T20:21:48.951041+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:50:27.040225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":86,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.957494+0000","last_change":"2023-09-23T20:21:49.957494+0000","last_active":"2023-09-23T20:21:49.957494+0000","last_peered":"2023-09-23T20:21:49.957494+0000","last_clean":"2023-09-23T20:21:49.957494+0000","last_became_active":"2023-09-23T20:21:49.957266+0000","last_became_peered":"2023-09-23T20:21:49.957266+0000","last_unstale":"2023-09-23T20:21:49.957494+0000","last_undegraded":"2023-09-23T20:21:49.957494+0000","last_fullsized":"2023-09-23T20:21:49.957494+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:49:51.483575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":72,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.949331+0000","last_change":"2023-09-23T20:20:46.950784+0000","last_active":"2023-09-23T20:21:48.949331+0000","last_peered":"2023-09-23T20:21:48.949331+0000","last_clean":"2023-09-23T20:21:48.949331+0000","last_became_active":"2023-09-23T20:20:46.950055+0000","last_became_peered":"2023-09-23T20:20:46.950055+0000","last_unstale":"2023-09-23T20:21:48.949331+0000","last_undegraded":"2023-09-23T20:21:48.949331+0000","last_fullsized":"2023-09-23T20:21:48.949331+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:23:07.645427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":51,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.949875+0000","last_change":"2023-09-23T20:12:45.179211+0000","last_active":"2023-09-23T20:21:48.949875+0000","last_peered":"2023-09-23T20:21:48.949875+0000","last_clean":"2023-09-23T20:21:48.949875+0000","last_became_active":"2023-09-23T20:12:45.178966+0000","last_became_peered":"2023-09-23T20:12:45.178966+0000","last_unstale":"2023-09-23T20:21:48.949875+0000","last_undegraded":"2023-09-23T20:21:48.949875+0000","last_fullsized":"2023-09-23T20:21:48.949875+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:58.902140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":71,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.950476+0000","last_change":"2023-09-23T20:20:14.883283+0000","last_active":"2023-09-23T20:21:48.950476+0000","last_peered":"2023-09-23T20:21:48.950476+0000","last_clean":"2023-09-23T20:21:48.950476+0000","last_became_active":"2023-09-23T20:20:14.882806+0000","last_became_peered":"2023-09-23T20:20:14.882806+0000","last_unstale":"2023-09-23T20:21:48.950476+0000","last_undegraded":"2023-09-23T20:21:48.950476+0000","last_fullsized":"2023-09-23T20:21:48.950476+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:23:35.295701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":72,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.958182+0000","last_change":"2023-09-23T20:21:49.958182+0000","last_active":"2023-09-23T20:21:49.958182+0000","last_peered":"2023-09-23T20:21:49.958182+0000","last_clean":"2023-09-23T20:21:49.958182+0000","last_became_active":"2023-09-23T20:21:49.957868+0000","last_became_peered":"2023-09-23T20:21:49.957868+0000","last_unstale":"2023-09-23T20:21:49.958182+0000","last_undegraded":"2023-09-23T20:21:49.958182+0000","last_fullsized":"2023-09-23T20:21:49.958182+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:27:41.577566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":78,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.944990+0000","last_change":"2023-09-23T20:20:14.881009+0000","last_active":"2023-09-23T20:21:48.944990+0000","last_peered":"2023-09-23T20:21:48.944990+0000","last_clean":"2023-09-23T20:21:48.944990+0000","last_became_active":"2023-09-23T20:20:14.880685+0000","last_became_peered":"2023-09-23T20:20:14.880685+0000","last_unstale":"2023-09-23T20:21:48.944990+0000","last_undegraded":"2023-09-23T20:21:48.944990+0000","last_fullsized":"2023-09-23T20:21:48.944990+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:28.010434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":52,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.954242+0000","last_change":"2023-09-23T20:21:49.954242+0000","last_active":"2023-09-23T20:21:49.954242+0000","last_peered":"2023-09-23T20:21:49.954242+0000","last_clean":"2023-09-23T20:21:49.954242+0000","last_became_active":"2023-09-23T20:21:49.953597+0000","last_became_peered":"2023-09-23T20:21:49.953597+0000","last_unstale":"2023-09-23T20:21:49.954242+0000","last_undegraded":"2023-09-23T20:21:49.954242+0000","last_fullsized":"2023-09-23T20:21:49.954242+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:08:46.716466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":53,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.948637+0000","last_change":"2023-09-23T20:12:43.749139+0000","last_active":"2023-09-23T20:21:48.948637+0000","last_peered":"2023-09-23T20:21:48.948637+0000","last_clean":"2023-09-23T20:21:48.948637+0000","last_became_active":"2023-09-23T20:12:43.748854+0000","last_became_peered":"2023-09-23T20:12:43.748854+0000","last_unstale":"2023-09-23T20:21:48.948637+0000","last_undegraded":"2023-09-23T20:21:48.948637+0000","last_fullsized":"2023-09-23T20:21:48.948637+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:57:11.630712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":68,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.952635+0000","last_change":"2023-09-23T20:21:49.952635+0000","last_active":"2023-09-23T20:21:49.952635+0000","last_peered":"2023-09-23T20:21:49.952635+0000","last_clean":"2023-09-23T20:21:49.952635+0000","last_became_active":"2023-09-23T20:21:49.952287+0000","last_became_peered":"2023-09-23T20:21:49.952287+0000","last_unstale":"2023-09-23T20:21:49.952635+0000","last_undegraded":"2023-09-23T20:21:49.952635+0000","last_fullsized":"2023-09-23T20:21:49.952635+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:17:17.370526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":51,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.953201+0000","last_change":"2023-09-23T20:21:49.953201+0000","last_active":"2023-09-23T20:21:49.953201+0000","last_peered":"2023-09-23T20:21:49.953201+0000","last_clean":"2023-09-23T20:21:49.953201+0000","last_became_active":"2023-09-23T20:21:49.952917+0000","last_became_peered":"2023-09-23T20:21:49.952917+0000","last_unstale":"2023-09-23T20:21:49.953201+0000","last_undegraded":"2023-09-23T20:21:49.953201+0000","last_fullsized":"2023-09-23T20:21:49.953201+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:54.579892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":72,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.958515+0000","last_change":"2023-09-23T20:21:49.958515+0000","last_active":"2023-09-23T20:21:49.958515+0000","last_peered":"2023-09-23T20:21:49.958515+0000","last_clean":"2023-09-23T20:21:49.958515+0000","last_became_active":"2023-09-23T20:21:49.958156+0000","last_became_peered":"2023-09-23T20:21:49.958156+0000","last_unstale":"2023-09-23T20:21:49.958515+0000","last_undegraded":"2023-09-23T20:21:49.958515+0000","last_fullsized":"2023-09-23T20:21:49.958515+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:56:47.145702+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":72,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.948079+0000","last_change":"2023-09-23T20:21:18.897418+0000","last_active":"2023-09-23T20:21:48.948079+0000","last_peered":"2023-09-23T20:21:48.948079+0000","last_clean":"2023-09-23T20:21:48.948079+0000","last_became_active":"2023-09-23T20:21:18.896968+0000","last_became_peered":"2023-09-23T20:21:18.896968+0000","last_unstale":"2023-09-23T20:21:48.948079+0000","last_undegraded":"2023-09-23T20:21:48.948079+0000","last_fullsized":"2023-09-23T20:21:48.948079+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:12:44.651568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":69,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.943565+0000","last_change":"2023-09-23T20:20:46.939938+0000","last_active":"2023-09-23T20:21:48.943565+0000","last_peered":"2023-09-23T20:21:48.943565+0000","last_clean":"2023-09-23T20:21:48.943565+0000","last_became_active":"2023-09-23T20:20:46.938777+0000","last_became_peered":"2023-09-23T20:20:46.938777+0000","last_unstale":"2023-09-23T20:21:48.943565+0000","last_undegraded":"2023-09-23T20:21:48.943565+0000","last_fullsized":"2023-09-23T20:21:48.943565+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:45:10.329438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":75,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.943049+0000","last_change":"2023-09-23T20:12:43.750217+0000","last_active":"2023-09-23T20:21:48.943049+0000","last_peered":"2023-09-23T20:21:48.943049+0000","last_clean":"2023-09-23T20:21:48.943049+0000","last_became_active":"2023-09-23T20:12:43.749929+0000","last_became_peered":"2023-09-23T20:12:43.749929+0000","last_unstale":"2023-09-23T20:21:48.943049+0000","last_undegraded":"2023-09-23T20:21:48.943049+0000","last_fullsized":"2023-09-23T20:21:48.943049+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:50:28.470211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":48,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.938497+0000","last_change":"2023-09-23T20:21:18.891438+0000","last_active":"2023-09-23T20:21:48.938497+0000","last_peered":"2023-09-23T20:21:48.938497+0000","last_clean":"2023-09-23T20:21:48.938497+0000","last_became_active":"2023-09-23T20:21:18.891140+0000","last_became_peered":"2023-09-23T20:21:18.891140+0000","last_unstale":"2023-09-23T20:21:48.938497+0000","last_undegraded":"2023-09-23T20:21:48.938497+0000","last_fullsized":"2023-09-23T20:21:48.938497+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:18:44.603577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":73,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.954219+0000","last_change":"2023-09-23T20:21:49.954219+0000","last_active":"2023-09-23T20:21:49.954219+0000","last_peered":"2023-09-23T20:21:49.954219+0000","last_clean":"2023-09-23T20:21:49.954219+0000","last_became_active":"2023-09-23T20:21:49.953827+0000","last_became_peered":"2023-09-23T20:21:49.953827+0000","last_unstale":"2023-09-23T20:21:49.954219+0000","last_undegraded":"2023-09-23T20:21:49.954219+0000","last_fullsized":"2023-09-23T20:21:49.954219+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:46:48.036706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":23,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.955179+0000","last_change":"2023-09-23T20:21:49.955179+0000","last_active":"2023-09-23T20:21:49.955179+0000","last_peered":"2023-09-23T20:21:49.955179+0000","last_clean":"2023-09-23T20:21:49.955179+0000","last_became_active":"2023-09-23T20:21:49.954878+0000","last_became_peered":"2023-09-23T20:21:49.954878+0000","last_unstale":"2023-09-23T20:21:49.955179+0000","last_undegraded":"2023-09-23T20:21:49.955179+0000","last_fullsized":"2023-09-23T20:21:49.955179+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:02:26.579246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":69,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.947957+0000","last_change":"2023-09-23T20:20:46.950281+0000","last_active":"2023-09-23T20:21:48.947957+0000","last_peered":"2023-09-23T20:21:48.947957+0000","last_clean":"2023-09-23T20:21:48.947957+0000","last_became_active":"2023-09-23T20:20:46.949892+0000","last_became_peered":"2023-09-23T20:20:46.949892+0000","last_unstale":"2023-09-23T20:21:48.947957+0000","last_undegraded":"2023-09-23T20:21:48.947957+0000","last_fullsized":"2023-09-23T20:21:48.947957+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:28.763832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":72,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.958837+0000","last_change":"2023-09-23T20:21:49.958837+0000","last_active":"2023-09-23T20:21:49.958837+0000","last_peered":"2023-09-23T20:21:49.958837+0000","last_clean":"2023-09-23T20:21:49.958837+0000","last_became_active":"2023-09-23T20:21:49.958135+0000","last_became_peered":"2023-09-23T20:21:49.958135+0000","last_unstale":"2023-09-23T20:21:49.958837+0000","last_undegraded":"2023-09-23T20:21:49.958837+0000","last_fullsized":"2023-09-23T20:21:49.958837+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:31:52.350275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":51,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.950402+0000","last_change":"2023-09-23T20:21:49.950402+0000","last_active":"2023-09-23T20:21:49.950402+0000","last_peered":"2023-09-23T20:21:49.950402+0000","last_clean":"2023-09-23T20:21:49.950402+0000","last_became_active":"2023-09-23T20:21:49.950116+0000","last_became_peered":"2023-09-23T20:21:49.950116+0000","last_unstale":"2023-09-23T20:21:49.950402+0000","last_undegraded":"2023-09-23T20:21:49.950402+0000","last_fullsized":"2023-09-23T20:21:49.950402+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:10:20.322615+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":69,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.945804+0000","last_change":"2023-09-23T20:20:46.937089+0000","last_active":"2023-09-23T20:21:48.945804+0000","last_peered":"2023-09-23T20:21:48.945804+0000","last_clean":"2023-09-23T20:21:48.945804+0000","last_became_active":"2023-09-23T20:20:46.936831+0000","last_became_peered":"2023-09-23T20:20:46.936831+0000","last_unstale":"2023-09-23T20:21:48.945804+0000","last_undegraded":"2023-09-23T20:21:48.945804+0000","last_fullsized":"2023-09-23T20:21:48.945804+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:34:56.747010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":90,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.958727+0000","last_change":"2023-09-23T20:21:49.958727+0000","last_active":"2023-09-23T20:21:49.958727+0000","last_peered":"2023-09-23T20:21:49.958727+0000","last_clean":"2023-09-23T20:21:49.958727+0000","last_became_active":"2023-09-23T20:21:49.957722+0000","last_became_peered":"2023-09-23T20:21:49.957722+0000","last_unstale":"2023-09-23T20:21:49.958727+0000","last_undegraded":"2023-09-23T20:21:49.958727+0000","last_fullsized":"2023-09-23T20:21:49.958727+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:47:13.589014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":53,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.956577+0000","last_change":"2023-09-23T20:21:49.954419+0000","last_active":"2023-09-23T20:21:49.956577+0000","last_peered":"2023-09-23T20:21:49.956577+0000","last_clean":"2023-09-23T20:21:49.956577+0000","last_became_active":"2023-09-23T20:21:49.953941+0000","last_became_peered":"2023-09-23T20:21:49.953941+0000","last_unstale":"2023-09-23T20:21:49.956577+0000","last_undegraded":"2023-09-23T20:21:49.956577+0000","last_fullsized":"2023-09-23T20:21:49.956577+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:05:38.809180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":47,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.955411+0000","last_change":"2023-09-23T20:21:49.955411+0000","last_active":"2023-09-23T20:21:49.955411+0000","last_peered":"2023-09-23T20:21:49.955411+0000","last_clean":"2023-09-23T20:21:49.955411+0000","last_became_active":"2023-09-23T20:21:49.955112+0000","last_became_peered":"2023-09-23T20:21:49.955112+0000","last_unstale":"2023-09-23T20:21:49.955411+0000","last_undegraded":"2023-09-23T20:21:49.955411+0000","last_fullsized":"2023-09-23T20:21:49.955411+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:47:53.870056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":72,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.951483+0000","last_change":"2023-09-23T20:21:49.951483+0000","last_active":"2023-09-23T20:21:49.951483+0000","last_peered":"2023-09-23T20:21:49.951483+0000","last_clean":"2023-09-23T20:21:49.951483+0000","last_became_active":"2023-09-23T20:21:49.951167+0000","last_became_peered":"2023-09-23T20:21:49.951167+0000","last_unstale":"2023-09-23T20:21:49.951483+0000","last_undegraded":"2023-09-23T20:21:49.951483+0000","last_fullsized":"2023-09-23T20:21:49.951483+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:08:19.552997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"69'1","reported_seq":88,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.942866+0000","last_change":"2023-09-23T20:21:18.891671+0000","last_active":"2023-09-23T20:21:48.942866+0000","last_peered":"2023-09-23T20:21:48.942866+0000","last_clean":"2023-09-23T20:21:48.942866+0000","last_became_active":"2023-09-23T20:21:18.891285+0000","last_became_peered":"2023-09-23T20:21:18.891285+0000","last_unstale":"2023-09-23T20:21:48.942866+0000","last_undegraded":"2023-09-23T20:21:48.942866+0000","last_fullsized":"2023-09-23T20:21:48.942866+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:04:49.595473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":25181,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6082,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201415,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":16944,"kb_used_data":5800,"kb_used_omap":0,"kb_used_meta":11136,"kb_avail":93728208,"statfs":{"total":95995035648,"available":95977684992,"internally_reserved":0,"allocated":5939200,"data_stored":5402099,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":11403264},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:22:17 2023","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.56499999999999995,"15min":0.56999999999999995},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":0.74199999999999999,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.57599999999999996},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.56200000000000006,"15min":0.56599999999999995},"min":{"1min":0.40600000000000003,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.68300000000000005,"5min":0.93799999999999994,"15min":0.93799999999999994},"last":0.53100000000000003}]},{"osd":1,"last update":"Sat Sep 23 20:21:48 2023","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":0.74399999999999999,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.55600000000000005},{"interface":"front","average":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"min":{"1min":0.443,"5min":0.443,"15min":0.443},"max":{"1min":0.88800000000000001,"5min":0.88800000000000001,"15min":0.88800000000000001},"last":0.51400000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:22:19 2023","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.82599999999999996,"5min":0.82599999999999996,"15min":0.82599999999999996},"last":0.60399999999999998},{"interface":"front","average":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.85999999999999999,"5min":0.85999999999999999,"15min":0.85999999999999999},"last":0.628}]},{"osd":3,"last update":"Sat Sep 23 20:20:44 2023","interfaces":[{"interface":"back","average":{"1min":0.59599999999999997,"5min":0.61199999999999999,"15min":0.57299999999999995},"min":{"1min":0.36699999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.72099999999999997,"5min":1.988,"15min":1.988},"last":0.59399999999999997},{"interface":"front","average":{"1min":0.623,"5min":0.63,"15min":0.61599999999999999},"min":{"1min":0.44,"5min":0.40899999999999997,"15min":0.36199999999999999},"max":{"1min":0.84399999999999997,"5min":0.872,"15min":1.127},"last":0.64200000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:21:48 2023","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.50600000000000001,"15min":0.50600000000000001},"min":{"1min":0.30199999999999999,"5min":0.28799999999999998,"15min":0.26700000000000002},"max":{"1min":0.90100000000000002,"5min":0.90100000000000002,"15min":1.274},"last":0.40400000000000003},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.54300000000000004,"15min":0.53400000000000003},"min":{"1min":0.35799999999999998,"5min":0.33100000000000002,"15min":0.29699999999999999},"max":{"1min":0.80000000000000004,"5min":0.80000000000000004,"15min":2.024},"last":0.33400000000000002}]},{"osd":5,"last update":"Sat Sep 23 20:21:48 2023","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":0.54200000000000004,"15min":0.54100000000000004},"min":{"1min":0.36599999999999999,"5min":0.33500000000000002,"15min":0.32000000000000001},"max":{"1min":0.79300000000000004,"5min":0.79600000000000004,"15min":1.337},"last":0.432},{"interface":"front","average":{"1min":0.629,"5min":0.60099999999999998,"15min":0.58799999999999997},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.31},"max":{"1min":0.89100000000000001,"5min":0.89100000000000001,"15min":1.5660000000000001},"last":0.35799999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:21:48 2023","interfaces":[{"interface":"back","average":{"1min":0.65300000000000002,"5min":0.61899999999999999,"15min":0.59399999999999997},"min":{"1min":0.379,"5min":0.379,"15min":0.34499999999999997},"max":{"1min":0.98699999999999999,"5min":0.98699999999999999,"15min":1.246},"last":0.45700000000000002},{"interface":"front","average":{"1min":0.60799999999999998,"5min":0.60999999999999999,"15min":0.58399999999999996},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.34799999999999998},"max":{"1min":0.873,"5min":0.873,"15min":2.048},"last":0.47899999999999998}]}]},{"osd":6,"up_from":47,"seq":201863463051,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":12392,"kb_used_data":5792,"kb_used_omap":0,"kb_used_meta":6592,"kb_avail":93732760,"statfs":{"total":95995035648,"available":95982346240,"internally_reserved":0,"allocated":5931008,"data_stored":5401972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6750208},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:22:18 2023","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":0.57799999999999996,"15min":0.57799999999999996},"min":{"1min":0.36799999999999999,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.58299999999999996},{"interface":"front","average":{"1min":0.57499999999999996,"5min":0.56499999999999995,"15min":0.56299999999999994},"min":{"1min":0.40500000000000003,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.76600000000000001,"5min":0.82299999999999995,"15min":0.82299999999999995},"last":0.60899999999999999}]},{"osd":1,"last update":"Sat Sep 23 20:21:48 2023","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.76300000000000001,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.44900000000000001},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.77500000000000002,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.55900000000000005}]},{"osd":2,"last update":"Sat Sep 23 20:22:18 2023","interfaces":[{"interface":"back","average":{"1min":0.59499999999999997,"5min":0.59499999999999997,"15min":0.59499999999999997},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.75,"5min":0.75,"15min":0.75},"last":0.48399999999999999},{"interface":"front","average":{"1min":0.60899999999999999,"5min":0.60899999999999999,"15min":0.60899999999999999},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":0.79300000000000004,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.77000000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:21:34 2023","interfaces":[{"interface":"back","average":{"1min":0.63100000000000001,"5min":0.60299999999999998,"15min":0.56100000000000005},"min":{"1min":0.46300000000000002,"5min":0.314,"15min":0.314},"max":{"1min":0.79300000000000004,"5min":0.81799999999999995,"15min":0.86599999999999999},"last":0.52600000000000002},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.59099999999999997,"15min":0.57999999999999996},"min":{"1min":0.40699999999999997,"5min":0.35299999999999998,"15min":0.34999999999999998},"max":{"1min":0.79200000000000004,"5min":0.84099999999999997,"15min":2.0670000000000002},"last":0.65400000000000003}]},{"osd":4,"last update":"Sat Sep 23 20:21:34 2023","interfaces":[{"interface":"back","average":{"1min":0.46700000000000003,"5min":0.46899999999999997,"15min":0.46300000000000002},"min":{"1min":0.38200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.63400000000000001,"5min":0.71799999999999997,"15min":0.72299999999999998},"last":0.67400000000000004},{"interface":"front","average":{"1min":0.52400000000000002,"5min":0.51300000000000001,"15min":0.49299999999999999},"min":{"1min":0.30099999999999999,"5min":0.28599999999999998,"15min":0.28499999999999998},"max":{"1min":0.72499999999999998,"5min":0.83899999999999997,"15min":0.83899999999999997},"last":0.76200000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:21:34 2023","interfaces":[{"interface":"back","average":{"1min":0.497,"5min":0.51700000000000002,"15min":0.49399999999999999},"min":{"1min":0.32600000000000001,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.67100000000000004,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.71599999999999997},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.55500000000000005,"15min":0.52100000000000002},"min":{"1min":0.34300000000000003,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.77900000000000003,"5min":0.874,"15min":0.874},"last":0.69199999999999995}]},{"osd":7,"last update":"Sat Sep 23 20:21:48 2023","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.57499999999999996,"15min":0.54300000000000004},"min":{"1min":0.372,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.85199999999999998,"5min":0.85199999999999998,"15min":0.85199999999999998},"last":0.81100000000000005},{"interface":"front","average":{"1min":0.60199999999999998,"5min":0.60099999999999998,"15min":0.56899999999999995},"min":{"1min":0.38200000000000001,"5min":0.33500000000000002,"15min":0.315},"max":{"1min":0.83099999999999996,"5min":0.90800000000000003,"15min":0.90800000000000003},"last":0.73699999999999999}]}]},{"osd":1,"up_from":99,"seq":425201762326,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28744,"kb_used_data":1152,"kb_used_omap":6,"kb_used_meta":27577,"kb_avail":93716408,"statfs":{"total":95995035648,"available":95965601792,"internally_reserved":0,"allocated":1179648,"data_stored":623002,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6696,"internal_metadata":28239320},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:21:46 2023","interfaces":[{"interface":"back","average":{"1min":0.377,"5min":0.377,"15min":0.377},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.50700000000000001,"5min":0.50700000000000001,"15min":0.50700000000000001},"last":0.501},{"interface":"front","average":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"last":0.626}]},{"osd":2,"last update":"Sat Sep 23 20:22:18 2023","interfaces":[{"interface":"back","average":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.628,"5min":0.628,"15min":0.628},"last":0.60799999999999998},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":3.226,"5min":3.226,"15min":3.226},"last":0.55900000000000005}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.64100000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:21:46 2023","interfaces":[{"interface":"back","average":{"1min":0.61699999999999999,"5min":0.61699999999999999,"15min":0.61699999999999999},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.89100000000000001,"5min":0.89100000000000001,"15min":0.89100000000000001},"last":0.75700000000000001},{"interface":"front","average":{"1min":0.63500000000000001,"5min":0.63500000000000001,"15min":0.63500000000000001},"min":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":0.85199999999999998,"5min":0.85199999999999998,"15min":0.85199999999999998},"last":0.84799999999999998}]},{"osd":5,"last update":"Sat Sep 23 20:21:46 2023","interfaces":[{"interface":"back","average":{"1min":0.64100000000000001,"5min":0.64100000000000001,"15min":0.64100000000000001},"min":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.82899999999999996},{"interface":"front","average":{"1min":0.67600000000000005,"5min":0.67600000000000005,"15min":0.67600000000000005},"min":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"max":{"1min":0.86899999999999999,"5min":0.86899999999999999,"15min":0.86899999999999999},"last":0.81999999999999995}]},{"osd":6,"last update":"Sat Sep 23 20:21:46 2023","interfaces":[{"interface":"back","average":{"1min":0.66800000000000004,"5min":0.66800000000000004,"15min":0.66800000000000004},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.92200000000000004,"5min":0.92200000000000004,"15min":0.92200000000000004},"last":0.77200000000000002},{"interface":"front","average":{"1min":0.63800000000000001,"5min":0.63800000000000001,"15min":0.63800000000000001},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":0.86599999999999999,"5min":0.86599999999999999,"15min":0.86599999999999999},"last":0.86599999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:21:46 2023","interfaces":[{"interface":"back","average":{"1min":0.65200000000000002,"5min":0.65200000000000002,"15min":0.65200000000000002},"min":{"1min":0.47599999999999998,"5min":0.47599999999999998,"15min":0.47599999999999998},"max":{"1min":0.91200000000000003,"5min":0.91200000000000003,"15min":0.91200000000000003},"last":0.88900000000000001},{"interface":"front","average":{"1min":0.68400000000000005,"5min":0.68400000000000005,"15min":0.68400000000000005},"min":{"1min":0.45300000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"max":{"1min":0.85899999999999999,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.78000000000000003}]}]},{"osd":0,"up_from":94,"seq":403726925852,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":29340,"kb_used_data":1748,"kb_used_omap":7,"kb_used_meta":27576,"kb_avail":93715812,"statfs":{"total":95995035648,"available":95964991488,"internally_reserved":0,"allocated":1789952,"data_stored":1215718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":28237841},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:21:48 2023","interfaces":[{"interface":"back","average":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"last":0.29199999999999998},{"interface":"front","average":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":0.52700000000000002,"5min":0.52700000000000002,"15min":0.52700000000000002},"last":0.30299999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:22:18 2023","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":4.6280000000000001,"5min":4.6280000000000001,"15min":4.6280000000000001},"last":0.26800000000000002},{"interface":"front","average":{"1min":0.496,"5min":0.496,"15min":0.496},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":1.921,"5min":1.921,"15min":1.921},"last":0.36899999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:21:15 2023","interfaces":[{"interface":"back","average":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"last":0.33600000000000002},{"interface":"front","average":{"1min":0.435,"5min":0.435,"15min":0.435},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"last":0.41099999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:22:18 2023","interfaces":[{"interface":"back","average":{"1min":0.78800000000000003,"5min":0.63600000000000001,"15min":0.60999999999999999},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":4.7450000000000001,"5min":4.7450000000000001,"15min":4.7450000000000001},"last":0.61399999999999999},{"interface":"front","average":{"1min":0.60799999999999998,"5min":0.61199999999999999,"15min":0.61299999999999999},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.92100000000000004,"5min":0.92100000000000004,"15min":0.92100000000000004},"last":0.55200000000000005}]},{"osd":5,"last update":"Sat Sep 23 20:22:18 2023","interfaces":[{"interface":"back","average":{"1min":0.64500000000000002,"5min":0.65600000000000003,"15min":0.65800000000000003},"min":{"1min":0.47899999999999998,"5min":0.47899999999999998,"15min":0.47899999999999998},"max":{"1min":0.90400000000000003,"5min":0.90400000000000003,"15min":0.90400000000000003},"last":0.69199999999999995},{"interface":"front","average":{"1min":0.67100000000000004,"5min":0.67700000000000005,"15min":0.67800000000000005},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.90600000000000003,"5min":0.93899999999999995,"15min":0.93899999999999995},"last":0.77400000000000002}]},{"osd":6,"last update":"Sat Sep 23 20:22:18 2023","interfaces":[{"interface":"back","average":{"1min":0.68799999999999994,"5min":0.65600000000000003,"15min":0.65100000000000002},"min":{"1min":0.495,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":0.97899999999999998,"5min":0.97899999999999998,"15min":0.97899999999999998},"last":0.72399999999999998},{"interface":"front","average":{"1min":0.68799999999999994,"5min":0.66800000000000004,"15min":0.66400000000000003},"min":{"1min":0.45300000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"max":{"1min":0.89900000000000002,"5min":0.95999999999999996,"15min":0.95999999999999996},"last":0.71499999999999997}]},{"osd":7,"last update":"Sat Sep 23 20:22:18 2023","interfaces":[{"interface":"back","average":{"1min":0.66400000000000003,"5min":0.67600000000000005,"15min":0.67800000000000005},"min":{"1min":0.47199999999999998,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":0.96399999999999997,"5min":0.96399999999999997,"15min":0.96399999999999997},"last":0.73799999999999999},{"interface":"front","average":{"1min":0.86399999999999999,"5min":0.70399999999999996,"15min":0.67800000000000005},"min":{"1min":0.52200000000000002,"5min":0.48499999999999999,"15min":0.48499999999999999},"max":{"1min":4.7140000000000004,"5min":4.7140000000000004,"15min":4.7140000000000004},"last":0.64700000000000002}]}]},{"osd":2,"up_from":104,"seq":446676598800,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32768,"kb_used_data":5240,"kb_used_omap":11,"kb_used_meta":27508,"kb_avail":93712384,"statfs":{"total":95995035648,"available":95961481216,"internally_reserved":0,"allocated":5365760,"data_stored":4815759,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11878,"internal_metadata":28168602},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:22:20 2023","interfaces":[{"interface":"back","average":{"1min":0.377,"5min":0.377,"15min":0.377},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":0.60599999999999998,"5min":0.60599999999999998,"15min":0.60599999999999998},"last":0.27600000000000002},{"interface":"front","average":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.69899999999999995,"5min":0.69899999999999995,"15min":0.69899999999999995},"last":0.51900000000000002}]},{"osd":1,"last update":"Sat Sep 23 20:22:20 2023","interfaces":[{"interface":"back","average":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.68000000000000005,"5min":0.68000000000000005,"15min":0.68000000000000005},"last":0.42999999999999999},{"interface":"front","average":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"last":0.47599999999999998}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.45900000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:22:20 2023","interfaces":[{"interface":"back","average":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.65900000000000003},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.74099999999999999,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.86199999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:22:20 2023","interfaces":[{"interface":"back","average":{"1min":0.65700000000000003,"5min":0.65700000000000003,"15min":0.65700000000000003},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.89100000000000001,"5min":0.89100000000000001,"15min":0.89100000000000001},"last":0.626},{"interface":"front","average":{"1min":0.65700000000000003,"5min":0.65700000000000003,"15min":0.65700000000000003},"min":{"1min":0.46600000000000003,"5min":0.46600000000000003,"15min":0.46600000000000003},"max":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.78600000000000003}]},{"osd":6,"last update":"Sat Sep 23 20:22:20 2023","interfaces":[{"interface":"back","average":{"1min":0.65700000000000003,"5min":0.65700000000000003,"15min":0.65700000000000003},"min":{"1min":0.501,"5min":0.501,"15min":0.501},"max":{"1min":0.94299999999999995,"5min":0.94299999999999995,"15min":0.94299999999999995},"last":0.82499999999999996},{"interface":"front","average":{"1min":0.67300000000000004,"5min":0.67300000000000004,"15min":0.67300000000000004},"min":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":1,"5min":1,"15min":1},"last":0.56999999999999995}]},{"osd":7,"last update":"Sat Sep 23 20:22:20 2023","interfaces":[{"interface":"back","average":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"min":{"1min":0.48399999999999999,"5min":0.48399999999999999,"15min":0.48399999999999999},"max":{"1min":0.96099999999999997,"5min":0.96099999999999997,"15min":0.96099999999999997},"last":0.76800000000000002},{"interface":"front","average":{"1min":0.65700000000000003,"5min":0.65700000000000003,"15min":0.65700000000000003},"min":{"1min":0.49299999999999999,"5min":0.49299999999999999,"15min":0.49299999999999999},"max":{"1min":0.92500000000000004,"5min":0.92500000000000004,"15min":0.92500000000000004},"last":0.59199999999999997}]}]},{"osd":3,"up_from":109,"seq":468151435273,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28648,"kb_used_data":1120,"kb_used_omap":4,"kb_used_meta":27515,"kb_avail":93716504,"statfs":{"total":95995035648,"available":95965700096,"internally_reserved":0,"allocated":1146880,"data_stored":617010,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4514,"internal_metadata":28175966},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.80500000000000005}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.40200000000000002}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34200000000000003}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.76400000000000001}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.61099999999999999}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.71399999999999997}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.66200000000000003}]}]},{"osd":4,"up_from":30,"seq":128849019027,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":7780,"kb_used_data":1116,"kb_used_omap":0,"kb_used_meta":6656,"kb_avail":93737372,"statfs":{"total":95995035648,"available":95987068928,"internally_reserved":0,"allocated":1142784,"data_stored":617034,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6815744},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:22:16 2023","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":0.57599999999999996,"15min":0.57099999999999995},"min":{"1min":0.41099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.85099999999999998,"5min":0.85099999999999998,"15min":0.85099999999999998},"last":0.65500000000000003},{"interface":"front","average":{"1min":0.63500000000000001,"5min":0.64000000000000001,"15min":0.64100000000000001},"min":{"1min":0.47299999999999998,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":0.79500000000000004,"5min":0.84799999999999998,"15min":0.84799999999999998},"last":0.63500000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:21:49 2023","interfaces":[{"interface":"back","average":{"1min":0.626,"5min":0.626,"15min":0.626},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":1.4219999999999999,"5min":1.4219999999999999,"15min":1.4219999999999999},"last":0.52500000000000002},{"interface":"front","average":{"1min":0.626,"5min":0.626,"15min":0.626},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":1.393,"5min":1.393,"15min":1.393},"last":0.68700000000000006}]},{"osd":2,"last update":"Sat Sep 23 20:22:19 2023","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"min":{"1min":0.49299999999999999,"5min":0.49299999999999999,"15min":0.49299999999999999},"max":{"1min":0.78400000000000003,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.51300000000000001},{"interface":"front","average":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":0.73599999999999999,"5min":0.73599999999999999,"15min":0.73599999999999999},"last":0.60999999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:21:40 2023","interfaces":[{"interface":"back","average":{"1min":0.63400000000000001,"5min":0.64700000000000002,"15min":0.58899999999999997},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":1.4079999999999999,"5min":3.0800000000000001,"15min":3.0800000000000001},"last":0.626},{"interface":"front","average":{"1min":0.67100000000000004,"5min":0.64900000000000002,"15min":0.58699999999999997},"min":{"1min":0.435,"5min":0.373,"15min":0.34699999999999998},"max":{"1min":1.4630000000000001,"5min":3.6190000000000002,"15min":3.6190000000000002},"last":0.70999999999999996}]},{"osd":5,"last update":"Sat Sep 23 20:22:05 2023","interfaces":[{"interface":"back","average":{"1min":0.48799999999999999,"5min":0.50600000000000001,"15min":0.45700000000000002},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.26000000000000001},"max":{"1min":0.66100000000000003,"5min":3.5270000000000001,"15min":3.5270000000000001},"last":0.45000000000000001},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.52900000000000003,"15min":0.46700000000000003},"min":{"1min":0.317,"5min":0.317,"15min":0.248},"max":{"1min":0.81299999999999994,"5min":0.86299999999999999,"15min":0.86299999999999999},"last":0.53100000000000003}]},{"osd":6,"last update":"Sat Sep 23 20:21:25 2023","interfaces":[{"interface":"back","average":{"1min":0.51400000000000001,"5min":0.53100000000000003,"15min":0.48499999999999999},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.27500000000000002},"max":{"1min":0.73699999999999999,"5min":0.77900000000000003,"15min":0.82499999999999996},"last":0.47099999999999997},{"interface":"front","average":{"1min":0.54900000000000004,"5min":0.56499999999999995,"15min":0.502},"min":{"1min":0.36599999999999999,"5min":0.32100000000000001,"15min":0.314},"max":{"1min":0.83599999999999997,"5min":3.4820000000000002,"15min":3.4820000000000002},"last":0.495}]},{"osd":7,"last update":"Sat Sep 23 20:21:40 2023","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":0.57199999999999995,"15min":0.54700000000000004},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.33600000000000002},"max":{"1min":0.875,"5min":0.91600000000000004,"15min":0.91600000000000004},"last":0.56499999999999995},{"interface":"front","average":{"1min":0.57999999999999996,"5min":0.61199999999999999,"15min":0.55600000000000005},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.29899999999999999},"max":{"1min":0.80800000000000005,"5min":3.77,"15min":3.77},"last":0.59699999999999998}]}]},{"osd":5,"up_from":39,"seq":167503724688,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":7516,"kb_used_data":1108,"kb_used_omap":0,"kb_used_meta":6400,"kb_avail":93737636,"statfs":{"total":95995035648,"available":95987339264,"internally_reserved":0,"allocated":1134592,"data_stored":613120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6553600},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:22:18 2023","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":0.59599999999999997,"15min":0.59299999999999997},"min":{"1min":0.38900000000000001,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.79800000000000004,"5min":0.83499999999999996,"15min":0.83499999999999996},"last":0.48899999999999999},{"interface":"front","average":{"1min":0.57999999999999996,"5min":0.59599999999999997,"15min":0.59899999999999998},"min":{"1min":0.44,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.82299999999999995,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.56599999999999995}]},{"osd":1,"last update":"Sat Sep 23 20:21:47 2023","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.64600000000000002},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.84999999999999998,"5min":0.84999999999999998,"15min":0.84999999999999998},"last":0.46000000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:22:18 2023","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.70899999999999996,"5min":0.70899999999999996,"15min":0.70899999999999996},"last":0.44400000000000001},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.72899999999999998,"5min":0.72899999999999998,"15min":0.72899999999999998},"last":0.52300000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:21:11 2023","interfaces":[{"interface":"back","average":{"1min":0.60299999999999998,"5min":0.60699999999999998,"15min":0.59099999999999997},"min":{"1min":0.47499999999999998,"5min":0.35299999999999998,"15min":0.34100000000000003},"max":{"1min":0.73199999999999998,"5min":0.93400000000000005,"15min":2.7570000000000001},"last":0.59299999999999997},{"interface":"front","average":{"1min":0.61099999999999999,"5min":0.63,"15min":0.57399999999999995},"min":{"1min":0.40699999999999997,"5min":0.40600000000000003,"15min":0.35599999999999998},"max":{"1min":0.74099999999999999,"5min":0.88200000000000001,"15min":2.2599999999999998},"last":0.61299999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:22:13 2023","interfaces":[{"interface":"back","average":{"1min":0.498,"5min":0.48199999999999998,"15min":0.439},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.26400000000000001},"max":{"1min":0.68200000000000005,"5min":0.76100000000000001,"15min":1.0620000000000001},"last":0.54900000000000004},{"interface":"front","average":{"1min":0.58799999999999997,"5min":0.52700000000000002,"15min":0.47999999999999998},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.28799999999999998},"max":{"1min":0.82299999999999995,"5min":0.82299999999999995,"15min":1.4450000000000001},"last":0.66900000000000004}]},{"osd":6,"last update":"Sat Sep 23 20:21:25 2023","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.51300000000000001,"15min":0.48799999999999999},"min":{"1min":0.38,"5min":0.32500000000000001,"15min":0.29799999999999999},"max":{"1min":0.68500000000000005,"5min":0.73899999999999999,"15min":1.151},"last":0.629},{"interface":"front","average":{"1min":0.61899999999999999,"5min":0.60299999999999998,"15min":0.55500000000000005},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.76200000000000001,"5min":0.82099999999999995,"15min":0.82099999999999995},"last":0.69099999999999995}]},{"osd":7,"last update":"Sat Sep 23 20:21:44 2023","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.56599999999999995,"15min":0.53800000000000003},"min":{"1min":0.41799999999999998,"5min":0.372,"15min":0.33800000000000002},"max":{"1min":0.81999999999999995,"5min":0.92500000000000004,"15min":0.98199999999999998},"last":0.58199999999999996},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.57799999999999996,"15min":0.54500000000000004},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.79200000000000004,"5min":0.79200000000000004,"15min":1.0489999999999999},"last":0.60199999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4722,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1360,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6696,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7156,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3154,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:22:29.202 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:22:29.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:22:28 smithi149 ceph-mon[170394]: pgmap v217: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:22:29.809 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:22:30.195 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:22:29 smithi018 ceph-mon[184144]: from='client.44606 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:22:30.195 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:22:29 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:22:30.195 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:22:29 smithi018 ceph-mon[179647]: from='client.44606 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:22:30.196 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:22:29 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:22:30.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:22:29 smithi149 ceph-mon[170394]: from='client.44606 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:22:30.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:22:29 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:22:31.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:22:30 smithi018 ceph-mon[179647]: pgmap v218: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:22:31.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:22:30 smithi018 ceph-mon[184144]: pgmap v218: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:22:31.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:22:30 smithi149 ceph-mon[170394]: pgmap v218: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:22:31.231 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:22:31.232 INFO:teuthology.orchestra.run.smithi018.stderr:dumped all 2023-09-23T20:22:31.658 INFO:teuthology.orchestra.run.smithi018.stdout:{"pg_ready":true,"pg_map":{"version":218,"stamp":"2023-09-23T20:22:29.429225+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790195,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6495,"num_read_kb":14136,"num_write":1146,"num_write_kb":15141,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1192,"ondisk_log_size":1192,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":164132,"kb_used_data":23076,"kb_used_omap":30,"kb_used_meta":140961,"kb_avail":749797084,"statfs":{"total":767960285184,"available":767792214016,"internally_reserved":0,"allocated":23629824,"data_stored":19305714,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31263,"internal_metadata":144344545},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003640"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":79,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.941990+0000","last_change":"2023-09-23T20:21:18.893413+0000","last_active":"2023-09-23T20:21:48.941990+0000","last_peered":"2023-09-23T20:21:48.941990+0000","last_clean":"2023-09-23T20:21:48.941990+0000","last_became_active":"2023-09-23T20:21:18.893008+0000","last_became_peered":"2023-09-23T20:21:18.893008+0000","last_unstale":"2023-09-23T20:21:48.941990+0000","last_undegraded":"2023-09-23T20:21:48.941990+0000","last_fullsized":"2023-09-23T20:21:48.941990+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:50:55.264882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":56,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.938999+0000","last_change":"2023-09-23T20:21:18.889521+0000","last_active":"2023-09-23T20:21:49.938999+0000","last_peered":"2023-09-23T20:21:49.938999+0000","last_clean":"2023-09-23T20:21:49.938999+0000","last_became_active":"2023-09-23T20:21:18.889268+0000","last_became_peered":"2023-09-23T20:21:18.889268+0000","last_unstale":"2023-09-23T20:21:49.938999+0000","last_undegraded":"2023-09-23T20:21:49.938999+0000","last_fullsized":"2023-09-23T20:21:49.938999+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:05:48.805778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":93,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.949944+0000","last_change":"2023-09-23T20:21:49.949944+0000","last_active":"2023-09-23T20:21:49.949944+0000","last_peered":"2023-09-23T20:21:49.949944+0000","last_clean":"2023-09-23T20:21:49.949944+0000","last_became_active":"2023-09-23T20:21:49.949425+0000","last_became_peered":"2023-09-23T20:21:49.949425+0000","last_unstale":"2023-09-23T20:21:49.949944+0000","last_undegraded":"2023-09-23T20:21:49.949944+0000","last_fullsized":"2023-09-23T20:21:49.949944+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:40:56.395791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":78,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.957394+0000","last_change":"2023-09-23T20:21:49.957394+0000","last_active":"2023-09-23T20:21:49.957394+0000","last_peered":"2023-09-23T20:21:49.957394+0000","last_clean":"2023-09-23T20:21:49.957394+0000","last_became_active":"2023-09-23T20:21:49.957016+0000","last_became_peered":"2023-09-23T20:21:49.957016+0000","last_unstale":"2023-09-23T20:21:49.957394+0000","last_undegraded":"2023-09-23T20:21:49.957394+0000","last_fullsized":"2023-09-23T20:21:49.957394+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:49:26.222448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":86,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.953041+0000","last_change":"2023-09-23T20:21:49.953041+0000","last_active":"2023-09-23T20:21:49.953041+0000","last_peered":"2023-09-23T20:21:49.953041+0000","last_clean":"2023-09-23T20:21:49.953041+0000","last_became_active":"2023-09-23T20:21:49.952763+0000","last_became_peered":"2023-09-23T20:21:49.952763+0000","last_unstale":"2023-09-23T20:21:49.953041+0000","last_undegraded":"2023-09-23T20:21:49.953041+0000","last_fullsized":"2023-09-23T20:21:49.953041+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:58:35.321466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":54,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.950333+0000","last_change":"2023-09-23T20:12:43.753383+0000","last_active":"2023-09-23T20:21:48.950333+0000","last_peered":"2023-09-23T20:21:48.950333+0000","last_clean":"2023-09-23T20:21:48.950333+0000","last_became_active":"2023-09-23T20:12:43.752921+0000","last_became_peered":"2023-09-23T20:12:43.752921+0000","last_unstale":"2023-09-23T20:21:48.950333+0000","last_undegraded":"2023-09-23T20:21:48.950333+0000","last_fullsized":"2023-09-23T20:21:48.950333+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:28:26.982998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":86,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.957885+0000","last_change":"2023-09-23T20:21:49.957885+0000","last_active":"2023-09-23T20:21:49.957885+0000","last_peered":"2023-09-23T20:21:49.957885+0000","last_clean":"2023-09-23T20:21:49.957885+0000","last_became_active":"2023-09-23T20:21:49.957492+0000","last_became_peered":"2023-09-23T20:21:49.957492+0000","last_unstale":"2023-09-23T20:21:49.957885+0000","last_undegraded":"2023-09-23T20:21:49.957885+0000","last_fullsized":"2023-09-23T20:21:49.957885+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:24:55.137836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":71,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.949471+0000","last_change":"2023-09-23T20:20:46.938981+0000","last_active":"2023-09-23T20:21:48.949471+0000","last_peered":"2023-09-23T20:21:48.949471+0000","last_clean":"2023-09-23T20:21:48.949471+0000","last_became_active":"2023-09-23T20:20:46.938627+0000","last_became_peered":"2023-09-23T20:20:46.938627+0000","last_unstale":"2023-09-23T20:21:48.949471+0000","last_undegraded":"2023-09-23T20:21:48.949471+0000","last_fullsized":"2023-09-23T20:21:48.949471+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:44:43.177086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":90,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.953299+0000","last_change":"2023-09-23T20:21:49.953299+0000","last_active":"2023-09-23T20:21:49.953299+0000","last_peered":"2023-09-23T20:21:49.953299+0000","last_clean":"2023-09-23T20:21:49.953299+0000","last_became_active":"2023-09-23T20:21:49.952996+0000","last_became_peered":"2023-09-23T20:21:49.952996+0000","last_unstale":"2023-09-23T20:21:49.953299+0000","last_undegraded":"2023-09-23T20:21:49.953299+0000","last_fullsized":"2023-09-23T20:21:49.953299+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:46:51.746281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":40,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.950883+0000","last_change":"2023-09-23T20:20:14.888484+0000","last_active":"2023-09-23T20:21:48.950883+0000","last_peered":"2023-09-23T20:21:48.950883+0000","last_clean":"2023-09-23T20:21:48.950883+0000","last_became_active":"2023-09-23T20:20:14.887219+0000","last_became_peered":"2023-09-23T20:20:14.887219+0000","last_unstale":"2023-09-23T20:21:48.950883+0000","last_undegraded":"2023-09-23T20:21:48.950883+0000","last_fullsized":"2023-09-23T20:21:48.950883+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:13:53.763649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":52,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.943196+0000","last_change":"2023-09-23T20:12:43.765908+0000","last_active":"2023-09-23T20:21:48.943196+0000","last_peered":"2023-09-23T20:21:48.943196+0000","last_clean":"2023-09-23T20:21:48.943196+0000","last_became_active":"2023-09-23T20:12:43.765558+0000","last_became_peered":"2023-09-23T20:12:43.765558+0000","last_unstale":"2023-09-23T20:21:48.943196+0000","last_undegraded":"2023-09-23T20:21:48.943196+0000","last_fullsized":"2023-09-23T20:21:48.943196+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:35:43.809898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":58,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.939966+0000","last_change":"2023-09-23T20:20:46.937163+0000","last_active":"2023-09-23T20:21:49.939966+0000","last_peered":"2023-09-23T20:21:49.939966+0000","last_clean":"2023-09-23T20:21:49.939966+0000","last_became_active":"2023-09-23T20:20:46.936846+0000","last_became_peered":"2023-09-23T20:20:46.936846+0000","last_unstale":"2023-09-23T20:21:49.939966+0000","last_undegraded":"2023-09-23T20:21:49.939966+0000","last_fullsized":"2023-09-23T20:21:49.939966+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:01:32.798855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"86'117","reported_seq":596,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.945893+0000","last_change":"2023-09-23T20:21:18.897269+0000","last_active":"2023-09-23T20:21:48.945893+0000","last_peered":"2023-09-23T20:21:48.945893+0000","last_clean":"2023-09-23T20:21:48.945893+0000","last_became_active":"2023-09-23T20:21:18.896834+0000","last_became_peered":"2023-09-23T20:21:18.896834+0000","last_unstale":"2023-09-23T20:21:48.945893+0000","last_undegraded":"2023-09-23T20:21:48.945893+0000","last_fullsized":"2023-09-23T20:21:48.945893+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:57:17.499025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046891400000000002,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":893,"num_read_kb":922,"num_write":82,"num_write_kb":4193,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":67,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.953565+0000","last_change":"2023-09-23T20:21:49.953565+0000","last_active":"2023-09-23T20:21:49.953565+0000","last_peered":"2023-09-23T20:21:49.953565+0000","last_clean":"2023-09-23T20:21:49.953565+0000","last_became_active":"2023-09-23T20:21:49.953176+0000","last_became_peered":"2023-09-23T20:21:49.953176+0000","last_unstale":"2023-09-23T20:21:49.953565+0000","last_undegraded":"2023-09-23T20:21:49.953565+0000","last_fullsized":"2023-09-23T20:21:49.953565+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:02:35.778207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"86'128","reported_seq":432,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.940390+0000","last_change":"2023-09-23T20:20:46.939256+0000","last_active":"2023-09-23T20:21:48.940390+0000","last_peered":"2023-09-23T20:21:48.940390+0000","last_clean":"2023-09-23T20:21:48.940390+0000","last_became_active":"2023-09-23T20:20:46.938756+0000","last_became_peered":"2023-09-23T20:20:46.938756+0000","last_unstale":"2023-09-23T20:21:48.940390+0000","last_undegraded":"2023-09-23T20:21:48.940390+0000","last_fullsized":"2023-09-23T20:21:48.940390+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:52:18.161436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000293202,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":579,"num_read_kb":1473,"num_write":96,"num_write_kb":1062,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":70,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.957535+0000","last_change":"2023-09-23T20:21:49.957535+0000","last_active":"2023-09-23T20:21:49.957535+0000","last_peered":"2023-09-23T20:21:49.957535+0000","last_clean":"2023-09-23T20:21:49.957535+0000","last_became_active":"2023-09-23T20:21:49.954319+0000","last_became_peered":"2023-09-23T20:21:49.954319+0000","last_unstale":"2023-09-23T20:21:49.957535+0000","last_undegraded":"2023-09-23T20:21:49.957535+0000","last_fullsized":"2023-09-23T20:21:49.957535+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:14:10.293305+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":38,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.938907+0000","last_change":"2023-09-23T20:20:14.888874+0000","last_active":"2023-09-23T20:21:49.938907+0000","last_peered":"2023-09-23T20:21:49.938907+0000","last_clean":"2023-09-23T20:21:49.938907+0000","last_became_active":"2023-09-23T20:20:14.887724+0000","last_became_peered":"2023-09-23T20:20:14.887724+0000","last_unstale":"2023-09-23T20:21:49.938907+0000","last_undegraded":"2023-09-23T20:21:49.938907+0000","last_fullsized":"2023-09-23T20:21:49.938907+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:09:24.510326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":47,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.939386+0000","last_change":"2023-09-23T20:21:18.893457+0000","last_active":"2023-09-23T20:21:48.939386+0000","last_peered":"2023-09-23T20:21:48.939386+0000","last_clean":"2023-09-23T20:21:48.939386+0000","last_became_active":"2023-09-23T20:21:18.893175+0000","last_became_peered":"2023-09-23T20:21:18.893175+0000","last_unstale":"2023-09-23T20:21:48.939386+0000","last_undegraded":"2023-09-23T20:21:48.939386+0000","last_fullsized":"2023-09-23T20:21:48.939386+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:24:37.563758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"86'102","reported_seq":616,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.951036+0000","last_change":"2023-09-23T20:20:14.881975+0000","last_active":"2023-09-23T20:21:48.951036+0000","last_peered":"2023-09-23T20:21:48.951036+0000","last_clean":"2023-09-23T20:21:48.951036+0000","last_became_active":"2023-09-23T20:20:14.881544+0000","last_became_peered":"2023-09-23T20:20:14.881544+0000","last_unstale":"2023-09-23T20:21:48.951036+0000","last_undegraded":"2023-09-23T20:21:48.951036+0000","last_fullsized":"2023-09-23T20:21:48.951036+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":102,"log_dups_size":0,"ondisk_log_size":102,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:34.594972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030656500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":963,"num_read_kb":1234,"num_write":60,"num_write_kb":225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":68,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.940049+0000","last_change":"2023-09-23T20:21:18.887651+0000","last_active":"2023-09-23T20:21:49.940049+0000","last_peered":"2023-09-23T20:21:49.940049+0000","last_clean":"2023-09-23T20:21:49.940049+0000","last_became_active":"2023-09-23T20:21:18.887338+0000","last_became_peered":"2023-09-23T20:21:18.887338+0000","last_unstale":"2023-09-23T20:21:49.940049+0000","last_undegraded":"2023-09-23T20:21:49.940049+0000","last_fullsized":"2023-09-23T20:21:49.940049+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:26:39.067022+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":52,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.952295+0000","last_change":"2023-09-23T20:21:49.951490+0000","last_active":"2023-09-23T20:21:49.952295+0000","last_peered":"2023-09-23T20:21:49.952295+0000","last_clean":"2023-09-23T20:21:49.952295+0000","last_became_active":"2023-09-23T20:21:49.951079+0000","last_became_peered":"2023-09-23T20:21:49.951079+0000","last_unstale":"2023-09-23T20:21:49.952295+0000","last_undegraded":"2023-09-23T20:21:49.952295+0000","last_fullsized":"2023-09-23T20:21:49.952295+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:12:01.053013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"86'80","reported_seq":356,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.952040+0000","last_change":"2023-09-23T20:20:46.950784+0000","last_active":"2023-09-23T20:21:48.952040+0000","last_peered":"2023-09-23T20:21:48.952040+0000","last_clean":"2023-09-23T20:21:48.952040+0000","last_became_active":"2023-09-23T20:20:46.950071+0000","last_became_peered":"2023-09-23T20:20:46.950071+0000","last_unstale":"2023-09-23T20:21:48.952040+0000","last_undegraded":"2023-09-23T20:21:48.952040+0000","last_fullsized":"2023-09-23T20:21:48.952040+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":80,"log_dups_size":0,"ondisk_log_size":80,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:11:06.812404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035115,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":395,"num_read_kb":1370,"num_write":66,"num_write_kb":1061,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":52,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.946301+0000","last_change":"2023-09-23T20:12:45.609829+0000","last_active":"2023-09-23T20:21:48.946301+0000","last_peered":"2023-09-23T20:21:48.946301+0000","last_clean":"2023-09-23T20:21:48.946301+0000","last_became_active":"2023-09-23T20:12:45.609467+0000","last_became_peered":"2023-09-23T20:12:45.609467+0000","last_unstale":"2023-09-23T20:21:48.946301+0000","last_undegraded":"2023-09-23T20:21:48.946301+0000","last_fullsized":"2023-09-23T20:21:48.946301+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:27:31.490555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":22,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.956277+0000","last_change":"2023-09-23T20:21:49.956277+0000","last_active":"2023-09-23T20:21:49.956277+0000","last_peered":"2023-09-23T20:21:49.956277+0000","last_clean":"2023-09-23T20:21:49.956277+0000","last_became_active":"2023-09-23T20:21:49.955974+0000","last_became_peered":"2023-09-23T20:21:49.955974+0000","last_unstale":"2023-09-23T20:21:49.956277+0000","last_undegraded":"2023-09-23T20:21:49.956277+0000","last_fullsized":"2023-09-23T20:21:49.956277+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:11:37.850675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":452,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.951949+0000","last_change":"2023-09-23T20:20:14.882204+0000","last_active":"2023-09-23T20:21:48.951949+0000","last_peered":"2023-09-23T20:21:48.951949+0000","last_clean":"2023-09-23T20:21:48.951949+0000","last_became_active":"2023-09-23T20:20:14.881699+0000","last_became_peered":"2023-09-23T20:20:14.881699+0000","last_unstale":"2023-09-23T20:21:48.951949+0000","last_undegraded":"2023-09-23T20:21:48.951949+0000","last_fullsized":"2023-09-23T20:21:48.951949+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:09:47.213130+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:16:37.341010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"86'194","reported_seq":728,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.948141+0000","last_change":"2023-09-23T20:21:18.892176+0000","last_active":"2023-09-23T20:21:48.948141+0000","last_peered":"2023-09-23T20:21:48.948141+0000","last_clean":"2023-09-23T20:21:48.948141+0000","last_became_active":"2023-09-23T20:21:18.891647+0000","last_became_peered":"2023-09-23T20:21:18.891647+0000","last_unstale":"2023-09-23T20:21:48.948141+0000","last_undegraded":"2023-09-23T20:21:48.948141+0000","last_fullsized":"2023-09-23T20:21:48.948141+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":194,"log_dups_size":0,"ondisk_log_size":194,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:25:52.557392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00037995999999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1025,"num_read_kb":1201,"num_write":173,"num_write_kb":1276,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":76,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.939413+0000","last_change":"2023-09-23T20:20:46.936524+0000","last_active":"2023-09-23T20:21:48.939413+0000","last_peered":"2023-09-23T20:21:48.939413+0000","last_clean":"2023-09-23T20:21:48.939413+0000","last_became_active":"2023-09-23T20:20:46.936267+0000","last_became_peered":"2023-09-23T20:20:46.936267+0000","last_unstale":"2023-09-23T20:21:48.939413+0000","last_undegraded":"2023-09-23T20:21:48.939413+0000","last_fullsized":"2023-09-23T20:21:48.939413+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:32:20.984521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":47,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.937923+0000","last_change":"2023-09-23T20:21:18.891553+0000","last_active":"2023-09-23T20:21:48.937923+0000","last_peered":"2023-09-23T20:21:48.937923+0000","last_clean":"2023-09-23T20:21:48.937923+0000","last_became_active":"2023-09-23T20:21:18.891230+0000","last_became_peered":"2023-09-23T20:21:18.891230+0000","last_unstale":"2023-09-23T20:21:48.937923+0000","last_undegraded":"2023-09-23T20:21:48.937923+0000","last_fullsized":"2023-09-23T20:21:48.937923+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:15:04.114927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"86'130","reported_seq":413,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.954331+0000","last_change":"2023-09-23T20:21:49.954331+0000","last_active":"2023-09-23T20:21:49.954331+0000","last_peered":"2023-09-23T20:21:49.954331+0000","last_clean":"2023-09-23T20:21:49.954331+0000","last_became_active":"2023-09-23T20:21:49.953687+0000","last_became_peered":"2023-09-23T20:21:49.953687+0000","last_unstale":"2023-09-23T20:21:49.954331+0000","last_undegraded":"2023-09-23T20:21:49.954331+0000","last_fullsized":"2023-09-23T20:21:49.954331+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:12.549803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011300049999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":512,"num_read_kb":1820,"num_write":133,"num_write_kb":1347,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":63,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.952961+0000","last_change":"2023-09-23T20:21:49.952961+0000","last_active":"2023-09-23T20:21:49.952961+0000","last_peered":"2023-09-23T20:21:49.952961+0000","last_clean":"2023-09-23T20:21:49.952961+0000","last_became_active":"2023-09-23T20:21:49.952644+0000","last_became_peered":"2023-09-23T20:21:49.952644+0000","last_unstale":"2023-09-23T20:21:49.952961+0000","last_undegraded":"2023-09-23T20:21:49.952961+0000","last_fullsized":"2023-09-23T20:21:49.952961+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:05:30.492540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":70,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.943468+0000","last_change":"2023-09-23T20:20:14.887345+0000","last_active":"2023-09-23T20:21:48.943468+0000","last_peered":"2023-09-23T20:21:48.943468+0000","last_clean":"2023-09-23T20:21:48.943468+0000","last_became_active":"2023-09-23T20:20:14.886970+0000","last_became_peered":"2023-09-23T20:20:14.886970+0000","last_unstale":"2023-09-23T20:21:48.943468+0000","last_undegraded":"2023-09-23T20:21:48.943468+0000","last_fullsized":"2023-09-23T20:21:48.943468+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:00:38.575926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"86'106","reported_seq":734,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.949374+0000","last_change":"2023-09-23T20:20:46.939468+0000","last_active":"2023-09-23T20:21:48.949374+0000","last_peered":"2023-09-23T20:21:48.949374+0000","last_clean":"2023-09-23T20:21:48.949374+0000","last_became_active":"2023-09-23T20:20:46.938954+0000","last_became_peered":"2023-09-23T20:20:46.938954+0000","last_unstale":"2023-09-23T20:21:48.949374+0000","last_undegraded":"2023-09-23T20:21:48.949374+0000","last_fullsized":"2023-09-23T20:21:48.949374+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":106,"log_dups_size":0,"ondisk_log_size":106,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:01:43.761064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00099302599999999993,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1105,"num_read_kb":1689,"num_write":100,"num_write_kb":404,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":33,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.938411+0000","last_change":"2023-09-23T20:20:46.937104+0000","last_active":"2023-09-23T20:21:48.938411+0000","last_peered":"2023-09-23T20:21:48.938411+0000","last_clean":"2023-09-23T20:21:48.938411+0000","last_became_active":"2023-09-23T20:20:46.936820+0000","last_became_peered":"2023-09-23T20:20:46.936820+0000","last_unstale":"2023-09-23T20:21:48.938411+0000","last_undegraded":"2023-09-23T20:21:48.938411+0000","last_fullsized":"2023-09-23T20:21:48.938411+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:22:19.879310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":86,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.953141+0000","last_change":"2023-09-23T20:21:49.953141+0000","last_active":"2023-09-23T20:21:49.953141+0000","last_peered":"2023-09-23T20:21:49.953141+0000","last_clean":"2023-09-23T20:21:49.953141+0000","last_became_active":"2023-09-23T20:21:49.952794+0000","last_became_peered":"2023-09-23T20:21:49.952794+0000","last_unstale":"2023-09-23T20:21:49.953141+0000","last_undegraded":"2023-09-23T20:21:49.953141+0000","last_fullsized":"2023-09-23T20:21:49.953141+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:36:19.919248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"86'119","reported_seq":441,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.939283+0000","last_change":"2023-09-23T20:20:46.937071+0000","last_active":"2023-09-23T20:21:48.939283+0000","last_peered":"2023-09-23T20:21:48.939283+0000","last_clean":"2023-09-23T20:21:48.939283+0000","last_became_active":"2023-09-23T20:20:46.935348+0000","last_became_peered":"2023-09-23T20:20:46.935348+0000","last_unstale":"2023-09-23T20:21:48.939283+0000","last_undegraded":"2023-09-23T20:21:48.939283+0000","last_fullsized":"2023-09-23T20:21:48.939283+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":119,"log_dups_size":0,"ondisk_log_size":119,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:09.829030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032705499999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":523,"num_read_kb":3503,"num_write":136,"num_write_kb":3128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":43,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.959532+0000","last_change":"2023-09-23T20:21:49.959532+0000","last_active":"2023-09-23T20:21:49.959532+0000","last_peered":"2023-09-23T20:21:49.959532+0000","last_clean":"2023-09-23T20:21:49.959532+0000","last_became_active":"2023-09-23T20:21:49.959161+0000","last_became_peered":"2023-09-23T20:21:49.959161+0000","last_unstale":"2023-09-23T20:21:49.959532+0000","last_undegraded":"2023-09-23T20:21:49.959532+0000","last_fullsized":"2023-09-23T20:21:49.959532+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:00:05.013667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":73,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.958915+0000","last_change":"2023-09-23T20:21:49.958915+0000","last_active":"2023-09-23T20:21:49.958915+0000","last_peered":"2023-09-23T20:21:49.958915+0000","last_clean":"2023-09-23T20:21:49.958915+0000","last_became_active":"2023-09-23T20:21:49.958440+0000","last_became_peered":"2023-09-23T20:21:49.958440+0000","last_unstale":"2023-09-23T20:21:49.958915+0000","last_undegraded":"2023-09-23T20:21:49.958915+0000","last_fullsized":"2023-09-23T20:21:49.958915+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:36:48.998945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":69,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.942436+0000","last_change":"2023-09-23T20:20:46.939720+0000","last_active":"2023-09-23T20:21:48.942436+0000","last_peered":"2023-09-23T20:21:48.942436+0000","last_clean":"2023-09-23T20:21:48.942436+0000","last_became_active":"2023-09-23T20:20:46.939357+0000","last_became_peered":"2023-09-23T20:20:46.939357+0000","last_unstale":"2023-09-23T20:21:48.942436+0000","last_undegraded":"2023-09-23T20:21:48.942436+0000","last_fullsized":"2023-09-23T20:21:48.942436+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:56.613212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":70,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.943289+0000","last_change":"2023-09-23T20:21:18.896635+0000","last_active":"2023-09-23T20:21:48.943289+0000","last_peered":"2023-09-23T20:21:48.943289+0000","last_clean":"2023-09-23T20:21:48.943289+0000","last_became_active":"2023-09-23T20:21:18.896335+0000","last_became_peered":"2023-09-23T20:21:18.896335+0000","last_unstale":"2023-09-23T20:21:48.943289+0000","last_undegraded":"2023-09-23T20:21:48.943289+0000","last_fullsized":"2023-09-23T20:21:48.943289+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:46:43.743795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":79,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.944402+0000","last_change":"2023-09-23T20:21:18.891669+0000","last_active":"2023-09-23T20:21:48.944402+0000","last_peered":"2023-09-23T20:21:48.944402+0000","last_clean":"2023-09-23T20:21:48.944402+0000","last_became_active":"2023-09-23T20:21:18.891326+0000","last_became_peered":"2023-09-23T20:21:18.891326+0000","last_unstale":"2023-09-23T20:21:48.944402+0000","last_undegraded":"2023-09-23T20:21:48.944402+0000","last_fullsized":"2023-09-23T20:21:48.944402+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:48:39.893213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":70,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.946401+0000","last_change":"2023-09-23T20:20:46.937163+0000","last_active":"2023-09-23T20:21:48.946401+0000","last_peered":"2023-09-23T20:21:48.946401+0000","last_clean":"2023-09-23T20:21:48.946401+0000","last_became_active":"2023-09-23T20:20:46.936757+0000","last_became_peered":"2023-09-23T20:20:46.936757+0000","last_unstale":"2023-09-23T20:21:48.946401+0000","last_undegraded":"2023-09-23T20:21:48.946401+0000","last_fullsized":"2023-09-23T20:21:48.946401+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:09.401379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":69,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.954710+0000","last_change":"2023-09-23T20:21:49.954710+0000","last_active":"2023-09-23T20:21:49.954710+0000","last_peered":"2023-09-23T20:21:49.954710+0000","last_clean":"2023-09-23T20:21:49.954710+0000","last_became_active":"2023-09-23T20:21:49.954355+0000","last_became_peered":"2023-09-23T20:21:49.954355+0000","last_unstale":"2023-09-23T20:21:49.954710+0000","last_undegraded":"2023-09-23T20:21:49.954710+0000","last_fullsized":"2023-09-23T20:21:49.954710+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:56:16.233285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":70,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.953004+0000","last_change":"2023-09-23T20:21:49.953004+0000","last_active":"2023-09-23T20:21:49.953004+0000","last_peered":"2023-09-23T20:21:49.953004+0000","last_clean":"2023-09-23T20:21:49.953004+0000","last_became_active":"2023-09-23T20:21:49.952625+0000","last_became_peered":"2023-09-23T20:21:49.952625+0000","last_unstale":"2023-09-23T20:21:49.953004+0000","last_undegraded":"2023-09-23T20:21:49.953004+0000","last_fullsized":"2023-09-23T20:21:49.953004+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:48:28.305640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":53,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.939096+0000","last_change":"2023-09-23T20:20:46.935968+0000","last_active":"2023-09-23T20:21:49.939096+0000","last_peered":"2023-09-23T20:21:49.939096+0000","last_clean":"2023-09-23T20:21:49.939096+0000","last_became_active":"2023-09-23T20:20:46.935618+0000","last_became_peered":"2023-09-23T20:20:46.935618+0000","last_unstale":"2023-09-23T20:21:49.939096+0000","last_undegraded":"2023-09-23T20:21:49.939096+0000","last_fullsized":"2023-09-23T20:21:49.939096+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:42:59.867176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":69,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.954875+0000","last_change":"2023-09-23T20:21:49.954875+0000","last_active":"2023-09-23T20:21:49.954875+0000","last_peered":"2023-09-23T20:21:49.954875+0000","last_clean":"2023-09-23T20:21:49.954875+0000","last_became_active":"2023-09-23T20:21:49.954519+0000","last_became_peered":"2023-09-23T20:21:49.954519+0000","last_unstale":"2023-09-23T20:21:49.954875+0000","last_undegraded":"2023-09-23T20:21:49.954875+0000","last_fullsized":"2023-09-23T20:21:49.954875+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:29:52.962029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":69,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.946248+0000","last_change":"2023-09-23T20:20:46.934762+0000","last_active":"2023-09-23T20:21:48.946248+0000","last_peered":"2023-09-23T20:21:48.946248+0000","last_clean":"2023-09-23T20:21:48.946248+0000","last_became_active":"2023-09-23T20:20:46.934472+0000","last_became_peered":"2023-09-23T20:20:46.934472+0000","last_unstale":"2023-09-23T20:21:48.946248+0000","last_undegraded":"2023-09-23T20:21:48.946248+0000","last_fullsized":"2023-09-23T20:21:48.946248+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:28:42.708286+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":54,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.951041+0000","last_change":"2023-09-23T20:12:43.751523+0000","last_active":"2023-09-23T20:21:48.951041+0000","last_peered":"2023-09-23T20:21:48.951041+0000","last_clean":"2023-09-23T20:21:48.951041+0000","last_became_active":"2023-09-23T20:12:43.751257+0000","last_became_peered":"2023-09-23T20:12:43.751257+0000","last_unstale":"2023-09-23T20:21:48.951041+0000","last_undegraded":"2023-09-23T20:21:48.951041+0000","last_fullsized":"2023-09-23T20:21:48.951041+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:50:27.040225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":86,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.957494+0000","last_change":"2023-09-23T20:21:49.957494+0000","last_active":"2023-09-23T20:21:49.957494+0000","last_peered":"2023-09-23T20:21:49.957494+0000","last_clean":"2023-09-23T20:21:49.957494+0000","last_became_active":"2023-09-23T20:21:49.957266+0000","last_became_peered":"2023-09-23T20:21:49.957266+0000","last_unstale":"2023-09-23T20:21:49.957494+0000","last_undegraded":"2023-09-23T20:21:49.957494+0000","last_fullsized":"2023-09-23T20:21:49.957494+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:49:51.483575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":72,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.949331+0000","last_change":"2023-09-23T20:20:46.950784+0000","last_active":"2023-09-23T20:21:48.949331+0000","last_peered":"2023-09-23T20:21:48.949331+0000","last_clean":"2023-09-23T20:21:48.949331+0000","last_became_active":"2023-09-23T20:20:46.950055+0000","last_became_peered":"2023-09-23T20:20:46.950055+0000","last_unstale":"2023-09-23T20:21:48.949331+0000","last_undegraded":"2023-09-23T20:21:48.949331+0000","last_fullsized":"2023-09-23T20:21:48.949331+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:23:07.645427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":51,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.949875+0000","last_change":"2023-09-23T20:12:45.179211+0000","last_active":"2023-09-23T20:21:48.949875+0000","last_peered":"2023-09-23T20:21:48.949875+0000","last_clean":"2023-09-23T20:21:48.949875+0000","last_became_active":"2023-09-23T20:12:45.178966+0000","last_became_peered":"2023-09-23T20:12:45.178966+0000","last_unstale":"2023-09-23T20:21:48.949875+0000","last_undegraded":"2023-09-23T20:21:48.949875+0000","last_fullsized":"2023-09-23T20:21:48.949875+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:58.902140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":71,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.950476+0000","last_change":"2023-09-23T20:20:14.883283+0000","last_active":"2023-09-23T20:21:48.950476+0000","last_peered":"2023-09-23T20:21:48.950476+0000","last_clean":"2023-09-23T20:21:48.950476+0000","last_became_active":"2023-09-23T20:20:14.882806+0000","last_became_peered":"2023-09-23T20:20:14.882806+0000","last_unstale":"2023-09-23T20:21:48.950476+0000","last_undegraded":"2023-09-23T20:21:48.950476+0000","last_fullsized":"2023-09-23T20:21:48.950476+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:23:35.295701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":72,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.958182+0000","last_change":"2023-09-23T20:21:49.958182+0000","last_active":"2023-09-23T20:21:49.958182+0000","last_peered":"2023-09-23T20:21:49.958182+0000","last_clean":"2023-09-23T20:21:49.958182+0000","last_became_active":"2023-09-23T20:21:49.957868+0000","last_became_peered":"2023-09-23T20:21:49.957868+0000","last_unstale":"2023-09-23T20:21:49.958182+0000","last_undegraded":"2023-09-23T20:21:49.958182+0000","last_fullsized":"2023-09-23T20:21:49.958182+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:27:41.577566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":78,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.944990+0000","last_change":"2023-09-23T20:20:14.881009+0000","last_active":"2023-09-23T20:21:48.944990+0000","last_peered":"2023-09-23T20:21:48.944990+0000","last_clean":"2023-09-23T20:21:48.944990+0000","last_became_active":"2023-09-23T20:20:14.880685+0000","last_became_peered":"2023-09-23T20:20:14.880685+0000","last_unstale":"2023-09-23T20:21:48.944990+0000","last_undegraded":"2023-09-23T20:21:48.944990+0000","last_fullsized":"2023-09-23T20:21:48.944990+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:28.010434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":52,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.954242+0000","last_change":"2023-09-23T20:21:49.954242+0000","last_active":"2023-09-23T20:21:49.954242+0000","last_peered":"2023-09-23T20:21:49.954242+0000","last_clean":"2023-09-23T20:21:49.954242+0000","last_became_active":"2023-09-23T20:21:49.953597+0000","last_became_peered":"2023-09-23T20:21:49.953597+0000","last_unstale":"2023-09-23T20:21:49.954242+0000","last_undegraded":"2023-09-23T20:21:49.954242+0000","last_fullsized":"2023-09-23T20:21:49.954242+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:08:46.716466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":53,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.948637+0000","last_change":"2023-09-23T20:12:43.749139+0000","last_active":"2023-09-23T20:21:48.948637+0000","last_peered":"2023-09-23T20:21:48.948637+0000","last_clean":"2023-09-23T20:21:48.948637+0000","last_became_active":"2023-09-23T20:12:43.748854+0000","last_became_peered":"2023-09-23T20:12:43.748854+0000","last_unstale":"2023-09-23T20:21:48.948637+0000","last_undegraded":"2023-09-23T20:21:48.948637+0000","last_fullsized":"2023-09-23T20:21:48.948637+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:57:11.630712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":68,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.952635+0000","last_change":"2023-09-23T20:21:49.952635+0000","last_active":"2023-09-23T20:21:49.952635+0000","last_peered":"2023-09-23T20:21:49.952635+0000","last_clean":"2023-09-23T20:21:49.952635+0000","last_became_active":"2023-09-23T20:21:49.952287+0000","last_became_peered":"2023-09-23T20:21:49.952287+0000","last_unstale":"2023-09-23T20:21:49.952635+0000","last_undegraded":"2023-09-23T20:21:49.952635+0000","last_fullsized":"2023-09-23T20:21:49.952635+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:17:17.370526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":51,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.953201+0000","last_change":"2023-09-23T20:21:49.953201+0000","last_active":"2023-09-23T20:21:49.953201+0000","last_peered":"2023-09-23T20:21:49.953201+0000","last_clean":"2023-09-23T20:21:49.953201+0000","last_became_active":"2023-09-23T20:21:49.952917+0000","last_became_peered":"2023-09-23T20:21:49.952917+0000","last_unstale":"2023-09-23T20:21:49.953201+0000","last_undegraded":"2023-09-23T20:21:49.953201+0000","last_fullsized":"2023-09-23T20:21:49.953201+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:54.579892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":72,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.958515+0000","last_change":"2023-09-23T20:21:49.958515+0000","last_active":"2023-09-23T20:21:49.958515+0000","last_peered":"2023-09-23T20:21:49.958515+0000","last_clean":"2023-09-23T20:21:49.958515+0000","last_became_active":"2023-09-23T20:21:49.958156+0000","last_became_peered":"2023-09-23T20:21:49.958156+0000","last_unstale":"2023-09-23T20:21:49.958515+0000","last_undegraded":"2023-09-23T20:21:49.958515+0000","last_fullsized":"2023-09-23T20:21:49.958515+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:56:47.145702+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":72,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.948079+0000","last_change":"2023-09-23T20:21:18.897418+0000","last_active":"2023-09-23T20:21:48.948079+0000","last_peered":"2023-09-23T20:21:48.948079+0000","last_clean":"2023-09-23T20:21:48.948079+0000","last_became_active":"2023-09-23T20:21:18.896968+0000","last_became_peered":"2023-09-23T20:21:18.896968+0000","last_unstale":"2023-09-23T20:21:48.948079+0000","last_undegraded":"2023-09-23T20:21:48.948079+0000","last_fullsized":"2023-09-23T20:21:48.948079+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:12:44.651568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":69,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.943565+0000","last_change":"2023-09-23T20:20:46.939938+0000","last_active":"2023-09-23T20:21:48.943565+0000","last_peered":"2023-09-23T20:21:48.943565+0000","last_clean":"2023-09-23T20:21:48.943565+0000","last_became_active":"2023-09-23T20:20:46.938777+0000","last_became_peered":"2023-09-23T20:20:46.938777+0000","last_unstale":"2023-09-23T20:21:48.943565+0000","last_undegraded":"2023-09-23T20:21:48.943565+0000","last_fullsized":"2023-09-23T20:21:48.943565+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:45:10.329438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":75,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.943049+0000","last_change":"2023-09-23T20:12:43.750217+0000","last_active":"2023-09-23T20:21:48.943049+0000","last_peered":"2023-09-23T20:21:48.943049+0000","last_clean":"2023-09-23T20:21:48.943049+0000","last_became_active":"2023-09-23T20:12:43.749929+0000","last_became_peered":"2023-09-23T20:12:43.749929+0000","last_unstale":"2023-09-23T20:21:48.943049+0000","last_undegraded":"2023-09-23T20:21:48.943049+0000","last_fullsized":"2023-09-23T20:21:48.943049+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:50:28.470211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":48,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.938497+0000","last_change":"2023-09-23T20:21:18.891438+0000","last_active":"2023-09-23T20:21:48.938497+0000","last_peered":"2023-09-23T20:21:48.938497+0000","last_clean":"2023-09-23T20:21:48.938497+0000","last_became_active":"2023-09-23T20:21:18.891140+0000","last_became_peered":"2023-09-23T20:21:18.891140+0000","last_unstale":"2023-09-23T20:21:48.938497+0000","last_undegraded":"2023-09-23T20:21:48.938497+0000","last_fullsized":"2023-09-23T20:21:48.938497+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:18:44.603577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":73,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.954219+0000","last_change":"2023-09-23T20:21:49.954219+0000","last_active":"2023-09-23T20:21:49.954219+0000","last_peered":"2023-09-23T20:21:49.954219+0000","last_clean":"2023-09-23T20:21:49.954219+0000","last_became_active":"2023-09-23T20:21:49.953827+0000","last_became_peered":"2023-09-23T20:21:49.953827+0000","last_unstale":"2023-09-23T20:21:49.954219+0000","last_undegraded":"2023-09-23T20:21:49.954219+0000","last_fullsized":"2023-09-23T20:21:49.954219+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:46:48.036706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":23,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.955179+0000","last_change":"2023-09-23T20:21:49.955179+0000","last_active":"2023-09-23T20:21:49.955179+0000","last_peered":"2023-09-23T20:21:49.955179+0000","last_clean":"2023-09-23T20:21:49.955179+0000","last_became_active":"2023-09-23T20:21:49.954878+0000","last_became_peered":"2023-09-23T20:21:49.954878+0000","last_unstale":"2023-09-23T20:21:49.955179+0000","last_undegraded":"2023-09-23T20:21:49.955179+0000","last_fullsized":"2023-09-23T20:21:49.955179+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:02:26.579246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":69,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.947957+0000","last_change":"2023-09-23T20:20:46.950281+0000","last_active":"2023-09-23T20:21:48.947957+0000","last_peered":"2023-09-23T20:21:48.947957+0000","last_clean":"2023-09-23T20:21:48.947957+0000","last_became_active":"2023-09-23T20:20:46.949892+0000","last_became_peered":"2023-09-23T20:20:46.949892+0000","last_unstale":"2023-09-23T20:21:48.947957+0000","last_undegraded":"2023-09-23T20:21:48.947957+0000","last_fullsized":"2023-09-23T20:21:48.947957+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:28.763832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":72,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.958837+0000","last_change":"2023-09-23T20:21:49.958837+0000","last_active":"2023-09-23T20:21:49.958837+0000","last_peered":"2023-09-23T20:21:49.958837+0000","last_clean":"2023-09-23T20:21:49.958837+0000","last_became_active":"2023-09-23T20:21:49.958135+0000","last_became_peered":"2023-09-23T20:21:49.958135+0000","last_unstale":"2023-09-23T20:21:49.958837+0000","last_undegraded":"2023-09-23T20:21:49.958837+0000","last_fullsized":"2023-09-23T20:21:49.958837+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:31:52.350275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":51,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.950402+0000","last_change":"2023-09-23T20:21:49.950402+0000","last_active":"2023-09-23T20:21:49.950402+0000","last_peered":"2023-09-23T20:21:49.950402+0000","last_clean":"2023-09-23T20:21:49.950402+0000","last_became_active":"2023-09-23T20:21:49.950116+0000","last_became_peered":"2023-09-23T20:21:49.950116+0000","last_unstale":"2023-09-23T20:21:49.950402+0000","last_undegraded":"2023-09-23T20:21:49.950402+0000","last_fullsized":"2023-09-23T20:21:49.950402+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:10:20.322615+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":69,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.945804+0000","last_change":"2023-09-23T20:20:46.937089+0000","last_active":"2023-09-23T20:21:48.945804+0000","last_peered":"2023-09-23T20:21:48.945804+0000","last_clean":"2023-09-23T20:21:48.945804+0000","last_became_active":"2023-09-23T20:20:46.936831+0000","last_became_peered":"2023-09-23T20:20:46.936831+0000","last_unstale":"2023-09-23T20:21:48.945804+0000","last_undegraded":"2023-09-23T20:21:48.945804+0000","last_fullsized":"2023-09-23T20:21:48.945804+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:34:56.747010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":90,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.958727+0000","last_change":"2023-09-23T20:21:49.958727+0000","last_active":"2023-09-23T20:21:49.958727+0000","last_peered":"2023-09-23T20:21:49.958727+0000","last_clean":"2023-09-23T20:21:49.958727+0000","last_became_active":"2023-09-23T20:21:49.957722+0000","last_became_peered":"2023-09-23T20:21:49.957722+0000","last_unstale":"2023-09-23T20:21:49.958727+0000","last_undegraded":"2023-09-23T20:21:49.958727+0000","last_fullsized":"2023-09-23T20:21:49.958727+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:47:13.589014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":53,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.956577+0000","last_change":"2023-09-23T20:21:49.954419+0000","last_active":"2023-09-23T20:21:49.956577+0000","last_peered":"2023-09-23T20:21:49.956577+0000","last_clean":"2023-09-23T20:21:49.956577+0000","last_became_active":"2023-09-23T20:21:49.953941+0000","last_became_peered":"2023-09-23T20:21:49.953941+0000","last_unstale":"2023-09-23T20:21:49.956577+0000","last_undegraded":"2023-09-23T20:21:49.956577+0000","last_fullsized":"2023-09-23T20:21:49.956577+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:05:38.809180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":47,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.955411+0000","last_change":"2023-09-23T20:21:49.955411+0000","last_active":"2023-09-23T20:21:49.955411+0000","last_peered":"2023-09-23T20:21:49.955411+0000","last_clean":"2023-09-23T20:21:49.955411+0000","last_became_active":"2023-09-23T20:21:49.955112+0000","last_became_peered":"2023-09-23T20:21:49.955112+0000","last_unstale":"2023-09-23T20:21:49.955411+0000","last_undegraded":"2023-09-23T20:21:49.955411+0000","last_fullsized":"2023-09-23T20:21:49.955411+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:47:53.870056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":72,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.951483+0000","last_change":"2023-09-23T20:21:49.951483+0000","last_active":"2023-09-23T20:21:49.951483+0000","last_peered":"2023-09-23T20:21:49.951483+0000","last_clean":"2023-09-23T20:21:49.951483+0000","last_became_active":"2023-09-23T20:21:49.951167+0000","last_became_peered":"2023-09-23T20:21:49.951167+0000","last_unstale":"2023-09-23T20:21:49.951483+0000","last_undegraded":"2023-09-23T20:21:49.951483+0000","last_fullsized":"2023-09-23T20:21:49.951483+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:08:19.552997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"69'1","reported_seq":88,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.942866+0000","last_change":"2023-09-23T20:21:18.891671+0000","last_active":"2023-09-23T20:21:48.942866+0000","last_peered":"2023-09-23T20:21:48.942866+0000","last_clean":"2023-09-23T20:21:48.942866+0000","last_became_active":"2023-09-23T20:21:18.891285+0000","last_became_peered":"2023-09-23T20:21:18.891285+0000","last_unstale":"2023-09-23T20:21:48.942866+0000","last_undegraded":"2023-09-23T20:21:48.942866+0000","last_fullsized":"2023-09-23T20:21:48.942866+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:04:49.595473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":25181,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6082,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201415,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":16944,"kb_used_data":5800,"kb_used_omap":0,"kb_used_meta":11136,"kb_avail":93728208,"statfs":{"total":95995035648,"available":95977684992,"internally_reserved":0,"allocated":5939200,"data_stored":5402099,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":11403264},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:22:17 2023","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.56499999999999995,"15min":0.56999999999999995},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":0.74199999999999999,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.57599999999999996},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.56200000000000006,"15min":0.56599999999999995},"min":{"1min":0.40600000000000003,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.68300000000000005,"5min":0.93799999999999994,"15min":0.93799999999999994},"last":0.53100000000000003}]},{"osd":1,"last update":"Sat Sep 23 20:21:48 2023","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":0.74399999999999999,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.55600000000000005},{"interface":"front","average":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"min":{"1min":0.443,"5min":0.443,"15min":0.443},"max":{"1min":0.88800000000000001,"5min":0.88800000000000001,"15min":0.88800000000000001},"last":0.51400000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:22:19 2023","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.82599999999999996,"5min":0.82599999999999996,"15min":0.82599999999999996},"last":0.60399999999999998},{"interface":"front","average":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.85999999999999999,"5min":0.85999999999999999,"15min":0.85999999999999999},"last":0.628}]},{"osd":3,"last update":"Sat Sep 23 20:20:44 2023","interfaces":[{"interface":"back","average":{"1min":0.59599999999999997,"5min":0.61199999999999999,"15min":0.57299999999999995},"min":{"1min":0.36699999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.72099999999999997,"5min":1.988,"15min":1.988},"last":0.59399999999999997},{"interface":"front","average":{"1min":0.623,"5min":0.63,"15min":0.61599999999999999},"min":{"1min":0.44,"5min":0.40899999999999997,"15min":0.36199999999999999},"max":{"1min":0.84399999999999997,"5min":0.872,"15min":1.127},"last":0.64200000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:21:48 2023","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.50600000000000001,"15min":0.50600000000000001},"min":{"1min":0.30199999999999999,"5min":0.28799999999999998,"15min":0.26700000000000002},"max":{"1min":0.90100000000000002,"5min":0.90100000000000002,"15min":1.274},"last":0.40400000000000003},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.54300000000000004,"15min":0.53400000000000003},"min":{"1min":0.35799999999999998,"5min":0.33100000000000002,"15min":0.29699999999999999},"max":{"1min":0.80000000000000004,"5min":0.80000000000000004,"15min":2.024},"last":0.33400000000000002}]},{"osd":5,"last update":"Sat Sep 23 20:21:48 2023","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":0.54200000000000004,"15min":0.54100000000000004},"min":{"1min":0.36599999999999999,"5min":0.33500000000000002,"15min":0.32000000000000001},"max":{"1min":0.79300000000000004,"5min":0.79600000000000004,"15min":1.337},"last":0.432},{"interface":"front","average":{"1min":0.629,"5min":0.60099999999999998,"15min":0.58799999999999997},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.31},"max":{"1min":0.89100000000000001,"5min":0.89100000000000001,"15min":1.5660000000000001},"last":0.35799999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:21:48 2023","interfaces":[{"interface":"back","average":{"1min":0.65300000000000002,"5min":0.61899999999999999,"15min":0.59399999999999997},"min":{"1min":0.379,"5min":0.379,"15min":0.34499999999999997},"max":{"1min":0.98699999999999999,"5min":0.98699999999999999,"15min":1.246},"last":0.45700000000000002},{"interface":"front","average":{"1min":0.60799999999999998,"5min":0.60999999999999999,"15min":0.58399999999999996},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.34799999999999998},"max":{"1min":0.873,"5min":0.873,"15min":2.048},"last":0.47899999999999998}]}]},{"osd":6,"up_from":47,"seq":201863463051,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":12392,"kb_used_data":5792,"kb_used_omap":0,"kb_used_meta":6592,"kb_avail":93732760,"statfs":{"total":95995035648,"available":95982346240,"internally_reserved":0,"allocated":5931008,"data_stored":5401972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6750208},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:22:18 2023","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":0.57799999999999996,"15min":0.57799999999999996},"min":{"1min":0.36799999999999999,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.58299999999999996},{"interface":"front","average":{"1min":0.57499999999999996,"5min":0.56499999999999995,"15min":0.56299999999999994},"min":{"1min":0.40500000000000003,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.76600000000000001,"5min":0.82299999999999995,"15min":0.82299999999999995},"last":0.60899999999999999}]},{"osd":1,"last update":"Sat Sep 23 20:21:48 2023","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.76300000000000001,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.44900000000000001},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.77500000000000002,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.55900000000000005}]},{"osd":2,"last update":"Sat Sep 23 20:22:18 2023","interfaces":[{"interface":"back","average":{"1min":0.59499999999999997,"5min":0.59499999999999997,"15min":0.59499999999999997},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.75,"5min":0.75,"15min":0.75},"last":0.48399999999999999},{"interface":"front","average":{"1min":0.60899999999999999,"5min":0.60899999999999999,"15min":0.60899999999999999},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":0.79300000000000004,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.77000000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:21:34 2023","interfaces":[{"interface":"back","average":{"1min":0.63100000000000001,"5min":0.60299999999999998,"15min":0.56100000000000005},"min":{"1min":0.46300000000000002,"5min":0.314,"15min":0.314},"max":{"1min":0.79300000000000004,"5min":0.81799999999999995,"15min":0.86599999999999999},"last":0.52600000000000002},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.59099999999999997,"15min":0.57999999999999996},"min":{"1min":0.40699999999999997,"5min":0.35299999999999998,"15min":0.34999999999999998},"max":{"1min":0.79200000000000004,"5min":0.84099999999999997,"15min":2.0670000000000002},"last":0.65400000000000003}]},{"osd":4,"last update":"Sat Sep 23 20:21:34 2023","interfaces":[{"interface":"back","average":{"1min":0.46700000000000003,"5min":0.46899999999999997,"15min":0.46300000000000002},"min":{"1min":0.38200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.63400000000000001,"5min":0.71799999999999997,"15min":0.72299999999999998},"last":0.67400000000000004},{"interface":"front","average":{"1min":0.52400000000000002,"5min":0.51300000000000001,"15min":0.49299999999999999},"min":{"1min":0.30099999999999999,"5min":0.28599999999999998,"15min":0.28499999999999998},"max":{"1min":0.72499999999999998,"5min":0.83899999999999997,"15min":0.83899999999999997},"last":0.76200000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:21:34 2023","interfaces":[{"interface":"back","average":{"1min":0.497,"5min":0.51700000000000002,"15min":0.49399999999999999},"min":{"1min":0.32600000000000001,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.67100000000000004,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.71599999999999997},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.55500000000000005,"15min":0.52100000000000002},"min":{"1min":0.34300000000000003,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.77900000000000003,"5min":0.874,"15min":0.874},"last":0.69199999999999995}]},{"osd":7,"last update":"Sat Sep 23 20:21:48 2023","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.57499999999999996,"15min":0.54300000000000004},"min":{"1min":0.372,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.85199999999999998,"5min":0.85199999999999998,"15min":0.85199999999999998},"last":0.81100000000000005},{"interface":"front","average":{"1min":0.60199999999999998,"5min":0.60099999999999998,"15min":0.56899999999999995},"min":{"1min":0.38200000000000001,"5min":0.33500000000000002,"15min":0.315},"max":{"1min":0.83099999999999996,"5min":0.90800000000000003,"15min":0.90800000000000003},"last":0.73699999999999999}]}]},{"osd":1,"up_from":99,"seq":425201762326,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28744,"kb_used_data":1152,"kb_used_omap":6,"kb_used_meta":27577,"kb_avail":93716408,"statfs":{"total":95995035648,"available":95965601792,"internally_reserved":0,"allocated":1179648,"data_stored":623002,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6696,"internal_metadata":28239320},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:21:46 2023","interfaces":[{"interface":"back","average":{"1min":0.377,"5min":0.377,"15min":0.377},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.50700000000000001,"5min":0.50700000000000001,"15min":0.50700000000000001},"last":0.501},{"interface":"front","average":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"last":0.626}]},{"osd":2,"last update":"Sat Sep 23 20:22:18 2023","interfaces":[{"interface":"back","average":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.628,"5min":0.628,"15min":0.628},"last":0.60799999999999998},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":3.226,"5min":3.226,"15min":3.226},"last":0.55900000000000005}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.64100000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:21:46 2023","interfaces":[{"interface":"back","average":{"1min":0.61699999999999999,"5min":0.61699999999999999,"15min":0.61699999999999999},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.89100000000000001,"5min":0.89100000000000001,"15min":0.89100000000000001},"last":0.75700000000000001},{"interface":"front","average":{"1min":0.63500000000000001,"5min":0.63500000000000001,"15min":0.63500000000000001},"min":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":0.85199999999999998,"5min":0.85199999999999998,"15min":0.85199999999999998},"last":0.84799999999999998}]},{"osd":5,"last update":"Sat Sep 23 20:21:46 2023","interfaces":[{"interface":"back","average":{"1min":0.64100000000000001,"5min":0.64100000000000001,"15min":0.64100000000000001},"min":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.82899999999999996},{"interface":"front","average":{"1min":0.67600000000000005,"5min":0.67600000000000005,"15min":0.67600000000000005},"min":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"max":{"1min":0.86899999999999999,"5min":0.86899999999999999,"15min":0.86899999999999999},"last":0.81999999999999995}]},{"osd":6,"last update":"Sat Sep 23 20:21:46 2023","interfaces":[{"interface":"back","average":{"1min":0.66800000000000004,"5min":0.66800000000000004,"15min":0.66800000000000004},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.92200000000000004,"5min":0.92200000000000004,"15min":0.92200000000000004},"last":0.77200000000000002},{"interface":"front","average":{"1min":0.63800000000000001,"5min":0.63800000000000001,"15min":0.63800000000000001},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":0.86599999999999999,"5min":0.86599999999999999,"15min":0.86599999999999999},"last":0.86599999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:21:46 2023","interfaces":[{"interface":"back","average":{"1min":0.65200000000000002,"5min":0.65200000000000002,"15min":0.65200000000000002},"min":{"1min":0.47599999999999998,"5min":0.47599999999999998,"15min":0.47599999999999998},"max":{"1min":0.91200000000000003,"5min":0.91200000000000003,"15min":0.91200000000000003},"last":0.88900000000000001},{"interface":"front","average":{"1min":0.68400000000000005,"5min":0.68400000000000005,"15min":0.68400000000000005},"min":{"1min":0.45300000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"max":{"1min":0.85899999999999999,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.78000000000000003}]}]},{"osd":0,"up_from":94,"seq":403726925853,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":29340,"kb_used_data":1748,"kb_used_omap":7,"kb_used_meta":27576,"kb_avail":93715812,"statfs":{"total":95995035648,"available":95964991488,"internally_reserved":0,"allocated":1789952,"data_stored":1215718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":28237841},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:21:48 2023","interfaces":[{"interface":"back","average":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"last":0.34000000000000002},{"interface":"front","average":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":0.52700000000000002,"5min":0.52700000000000002,"15min":0.52700000000000002},"last":0.47399999999999998}]},{"osd":2,"last update":"Sat Sep 23 20:22:18 2023","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":4.6280000000000001,"5min":4.6280000000000001,"15min":4.6280000000000001},"last":0.38100000000000001},{"interface":"front","average":{"1min":0.496,"5min":0.496,"15min":0.496},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":1.921,"5min":1.921,"15min":1.921},"last":0.317}]},{"osd":3,"last update":"Sat Sep 23 20:21:15 2023","interfaces":[{"interface":"back","average":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"last":0.35699999999999998},{"interface":"front","average":{"1min":0.435,"5min":0.435,"15min":0.435},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"last":0.51900000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:22:18 2023","interfaces":[{"interface":"back","average":{"1min":0.78800000000000003,"5min":0.63600000000000001,"15min":0.60999999999999999},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":4.7450000000000001,"5min":4.7450000000000001,"15min":4.7450000000000001},"last":0.41199999999999998},{"interface":"front","average":{"1min":0.60799999999999998,"5min":0.61199999999999999,"15min":0.61299999999999999},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.92100000000000004,"5min":0.92100000000000004,"15min":0.92100000000000004},"last":0.61499999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:22:18 2023","interfaces":[{"interface":"back","average":{"1min":0.64500000000000002,"5min":0.65600000000000003,"15min":0.65800000000000003},"min":{"1min":0.47899999999999998,"5min":0.47899999999999998,"15min":0.47899999999999998},"max":{"1min":0.90400000000000003,"5min":0.90400000000000003,"15min":0.90400000000000003},"last":0.57899999999999996},{"interface":"front","average":{"1min":0.67100000000000004,"5min":0.67700000000000005,"15min":0.67800000000000005},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.90600000000000003,"5min":0.93899999999999995,"15min":0.93899999999999995},"last":0.53900000000000003}]},{"osd":6,"last update":"Sat Sep 23 20:22:18 2023","interfaces":[{"interface":"back","average":{"1min":0.68799999999999994,"5min":0.65600000000000003,"15min":0.65100000000000002},"min":{"1min":0.495,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":0.97899999999999998,"5min":0.97899999999999998,"15min":0.97899999999999998},"last":0.65000000000000002},{"interface":"front","average":{"1min":0.68799999999999994,"5min":0.66800000000000004,"15min":0.66400000000000003},"min":{"1min":0.45300000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"max":{"1min":0.89900000000000002,"5min":0.95999999999999996,"15min":0.95999999999999996},"last":0.59999999999999998}]},{"osd":7,"last update":"Sat Sep 23 20:22:18 2023","interfaces":[{"interface":"back","average":{"1min":0.66400000000000003,"5min":0.67600000000000005,"15min":0.67800000000000005},"min":{"1min":0.47199999999999998,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":0.96399999999999997,"5min":0.96399999999999997,"15min":0.96399999999999997},"last":0.63300000000000001},{"interface":"front","average":{"1min":0.86399999999999999,"5min":0.70399999999999996,"15min":0.67800000000000005},"min":{"1min":0.52200000000000002,"5min":0.48499999999999999,"15min":0.48499999999999999},"max":{"1min":4.7140000000000004,"5min":4.7140000000000004,"15min":4.7140000000000004},"last":0.51100000000000001}]}]},{"osd":2,"up_from":104,"seq":446676598800,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32768,"kb_used_data":5240,"kb_used_omap":11,"kb_used_meta":27508,"kb_avail":93712384,"statfs":{"total":95995035648,"available":95961481216,"internally_reserved":0,"allocated":5365760,"data_stored":4815759,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11878,"internal_metadata":28168602},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:22:20 2023","interfaces":[{"interface":"back","average":{"1min":0.377,"5min":0.377,"15min":0.377},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":0.60599999999999998,"5min":0.60599999999999998,"15min":0.60599999999999998},"last":0.27600000000000002},{"interface":"front","average":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.69899999999999995,"5min":0.69899999999999995,"15min":0.69899999999999995},"last":0.51900000000000002}]},{"osd":1,"last update":"Sat Sep 23 20:22:20 2023","interfaces":[{"interface":"back","average":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.68000000000000005,"5min":0.68000000000000005,"15min":0.68000000000000005},"last":0.42999999999999999},{"interface":"front","average":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"last":0.47599999999999998}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.45900000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:22:20 2023","interfaces":[{"interface":"back","average":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.65900000000000003},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.74099999999999999,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.86199999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:22:20 2023","interfaces":[{"interface":"back","average":{"1min":0.65700000000000003,"5min":0.65700000000000003,"15min":0.65700000000000003},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.89100000000000001,"5min":0.89100000000000001,"15min":0.89100000000000001},"last":0.626},{"interface":"front","average":{"1min":0.65700000000000003,"5min":0.65700000000000003,"15min":0.65700000000000003},"min":{"1min":0.46600000000000003,"5min":0.46600000000000003,"15min":0.46600000000000003},"max":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.78600000000000003}]},{"osd":6,"last update":"Sat Sep 23 20:22:20 2023","interfaces":[{"interface":"back","average":{"1min":0.65700000000000003,"5min":0.65700000000000003,"15min":0.65700000000000003},"min":{"1min":0.501,"5min":0.501,"15min":0.501},"max":{"1min":0.94299999999999995,"5min":0.94299999999999995,"15min":0.94299999999999995},"last":0.82499999999999996},{"interface":"front","average":{"1min":0.67300000000000004,"5min":0.67300000000000004,"15min":0.67300000000000004},"min":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":1,"5min":1,"15min":1},"last":0.56999999999999995}]},{"osd":7,"last update":"Sat Sep 23 20:22:20 2023","interfaces":[{"interface":"back","average":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"min":{"1min":0.48399999999999999,"5min":0.48399999999999999,"15min":0.48399999999999999},"max":{"1min":0.96099999999999997,"5min":0.96099999999999997,"15min":0.96099999999999997},"last":0.76800000000000002},{"interface":"front","average":{"1min":0.65700000000000003,"5min":0.65700000000000003,"15min":0.65700000000000003},"min":{"1min":0.49299999999999999,"5min":0.49299999999999999,"15min":0.49299999999999999},"max":{"1min":0.92500000000000004,"5min":0.92500000000000004,"15min":0.92500000000000004},"last":0.59199999999999997}]}]},{"osd":3,"up_from":109,"seq":468151435274,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28648,"kb_used_data":1120,"kb_used_omap":4,"kb_used_meta":27515,"kb_avail":93716504,"statfs":{"total":95995035648,"available":95965700096,"internally_reserved":0,"allocated":1146880,"data_stored":617010,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4514,"internal_metadata":28175966},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.36799999999999999}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.39700000000000002}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34699999999999998}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.63500000000000001}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.53700000000000003}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.66300000000000003}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.76600000000000001}]}]},{"osd":4,"up_from":30,"seq":128849019027,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":7780,"kb_used_data":1116,"kb_used_omap":0,"kb_used_meta":6656,"kb_avail":93737372,"statfs":{"total":95995035648,"available":95987068928,"internally_reserved":0,"allocated":1142784,"data_stored":617034,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6815744},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:22:16 2023","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":0.57599999999999996,"15min":0.57099999999999995},"min":{"1min":0.41099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.85099999999999998,"5min":0.85099999999999998,"15min":0.85099999999999998},"last":0.65500000000000003},{"interface":"front","average":{"1min":0.63500000000000001,"5min":0.64000000000000001,"15min":0.64100000000000001},"min":{"1min":0.47299999999999998,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":0.79500000000000004,"5min":0.84799999999999998,"15min":0.84799999999999998},"last":0.63500000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:21:49 2023","interfaces":[{"interface":"back","average":{"1min":0.626,"5min":0.626,"15min":0.626},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":1.4219999999999999,"5min":1.4219999999999999,"15min":1.4219999999999999},"last":0.52500000000000002},{"interface":"front","average":{"1min":0.626,"5min":0.626,"15min":0.626},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":1.393,"5min":1.393,"15min":1.393},"last":0.68700000000000006}]},{"osd":2,"last update":"Sat Sep 23 20:22:19 2023","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"min":{"1min":0.49299999999999999,"5min":0.49299999999999999,"15min":0.49299999999999999},"max":{"1min":0.78400000000000003,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.51300000000000001},{"interface":"front","average":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":0.73599999999999999,"5min":0.73599999999999999,"15min":0.73599999999999999},"last":0.60999999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:21:40 2023","interfaces":[{"interface":"back","average":{"1min":0.63400000000000001,"5min":0.64700000000000002,"15min":0.58899999999999997},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":1.4079999999999999,"5min":3.0800000000000001,"15min":3.0800000000000001},"last":0.626},{"interface":"front","average":{"1min":0.67100000000000004,"5min":0.64900000000000002,"15min":0.58699999999999997},"min":{"1min":0.435,"5min":0.373,"15min":0.34699999999999998},"max":{"1min":1.4630000000000001,"5min":3.6190000000000002,"15min":3.6190000000000002},"last":0.70999999999999996}]},{"osd":5,"last update":"Sat Sep 23 20:22:05 2023","interfaces":[{"interface":"back","average":{"1min":0.48799999999999999,"5min":0.50600000000000001,"15min":0.45700000000000002},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.26000000000000001},"max":{"1min":0.66100000000000003,"5min":3.5270000000000001,"15min":3.5270000000000001},"last":0.45000000000000001},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.52900000000000003,"15min":0.46700000000000003},"min":{"1min":0.317,"5min":0.317,"15min":0.248},"max":{"1min":0.81299999999999994,"5min":0.86299999999999999,"15min":0.86299999999999999},"last":0.53100000000000003}]},{"osd":6,"last update":"Sat Sep 23 20:21:25 2023","interfaces":[{"interface":"back","average":{"1min":0.51400000000000001,"5min":0.53100000000000003,"15min":0.48499999999999999},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.27500000000000002},"max":{"1min":0.73699999999999999,"5min":0.77900000000000003,"15min":0.82499999999999996},"last":0.47099999999999997},{"interface":"front","average":{"1min":0.54900000000000004,"5min":0.56499999999999995,"15min":0.502},"min":{"1min":0.36599999999999999,"5min":0.32100000000000001,"15min":0.314},"max":{"1min":0.83599999999999997,"5min":3.4820000000000002,"15min":3.4820000000000002},"last":0.495}]},{"osd":7,"last update":"Sat Sep 23 20:21:40 2023","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":0.57199999999999995,"15min":0.54700000000000004},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.33600000000000002},"max":{"1min":0.875,"5min":0.91600000000000004,"15min":0.91600000000000004},"last":0.56499999999999995},{"interface":"front","average":{"1min":0.57999999999999996,"5min":0.61199999999999999,"15min":0.55600000000000005},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.29899999999999999},"max":{"1min":0.80800000000000005,"5min":3.77,"15min":3.77},"last":0.59699999999999998}]}]},{"osd":5,"up_from":39,"seq":167503724688,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":7516,"kb_used_data":1108,"kb_used_omap":0,"kb_used_meta":6400,"kb_avail":93737636,"statfs":{"total":95995035648,"available":95987339264,"internally_reserved":0,"allocated":1134592,"data_stored":613120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6553600},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:22:18 2023","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":0.59599999999999997,"15min":0.59299999999999997},"min":{"1min":0.38900000000000001,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.79800000000000004,"5min":0.83499999999999996,"15min":0.83499999999999996},"last":0.48899999999999999},{"interface":"front","average":{"1min":0.57999999999999996,"5min":0.59599999999999997,"15min":0.59899999999999998},"min":{"1min":0.44,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.82299999999999995,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.56599999999999995}]},{"osd":1,"last update":"Sat Sep 23 20:21:47 2023","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.64600000000000002},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.84999999999999998,"5min":0.84999999999999998,"15min":0.84999999999999998},"last":0.46000000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:22:18 2023","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.70899999999999996,"5min":0.70899999999999996,"15min":0.70899999999999996},"last":0.44400000000000001},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.72899999999999998,"5min":0.72899999999999998,"15min":0.72899999999999998},"last":0.52300000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:21:11 2023","interfaces":[{"interface":"back","average":{"1min":0.60299999999999998,"5min":0.60699999999999998,"15min":0.59099999999999997},"min":{"1min":0.47499999999999998,"5min":0.35299999999999998,"15min":0.34100000000000003},"max":{"1min":0.73199999999999998,"5min":0.93400000000000005,"15min":2.7570000000000001},"last":0.59299999999999997},{"interface":"front","average":{"1min":0.61099999999999999,"5min":0.63,"15min":0.57399999999999995},"min":{"1min":0.40699999999999997,"5min":0.40600000000000003,"15min":0.35599999999999998},"max":{"1min":0.74099999999999999,"5min":0.88200000000000001,"15min":2.2599999999999998},"last":0.61299999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:22:13 2023","interfaces":[{"interface":"back","average":{"1min":0.498,"5min":0.48199999999999998,"15min":0.439},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.26400000000000001},"max":{"1min":0.68200000000000005,"5min":0.76100000000000001,"15min":1.0620000000000001},"last":0.54900000000000004},{"interface":"front","average":{"1min":0.58799999999999997,"5min":0.52700000000000002,"15min":0.47999999999999998},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.28799999999999998},"max":{"1min":0.82299999999999995,"5min":0.82299999999999995,"15min":1.4450000000000001},"last":0.66900000000000004}]},{"osd":6,"last update":"Sat Sep 23 20:21:25 2023","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.51300000000000001,"15min":0.48799999999999999},"min":{"1min":0.38,"5min":0.32500000000000001,"15min":0.29799999999999999},"max":{"1min":0.68500000000000005,"5min":0.73899999999999999,"15min":1.151},"last":0.629},{"interface":"front","average":{"1min":0.61899999999999999,"5min":0.60299999999999998,"15min":0.55500000000000005},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.76200000000000001,"5min":0.82099999999999995,"15min":0.82099999999999995},"last":0.69099999999999995}]},{"osd":7,"last update":"Sat Sep 23 20:21:44 2023","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.56599999999999995,"15min":0.53800000000000003},"min":{"1min":0.41799999999999998,"5min":0.372,"15min":0.33800000000000002},"max":{"1min":0.81999999999999995,"5min":0.92500000000000004,"15min":0.98199999999999998},"last":0.58199999999999996},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.57799999999999996,"15min":0.54500000000000004},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.79200000000000004,"5min":0.79200000000000004,"15min":1.0489999999999999},"last":0.60199999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4722,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1360,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6696,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7156,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3154,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:22:31.665 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2023-09-23T20:22:31.665 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2023-09-23T20:22:31.665 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2023-09-23T20:22:31.665 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph health --format=json 2023-09-23T20:22:32.272 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:22:33.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:22:32 smithi018 ceph-mon[179647]: from='client.34710 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:22:33.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:22:32 smithi018 ceph-mon[179647]: pgmap v219: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:22:33.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:22:32 smithi018 ceph-mon[184144]: from='client.34710 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:22:33.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:22:32 smithi018 ceph-mon[184144]: pgmap v219: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:22:33.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:22:32 smithi149 ceph-mon[170394]: from='client.34710 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:22:33.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:22:32 smithi149 ceph-mon[170394]: pgmap v219: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:22:33.742 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:22:33.742 INFO:teuthology.orchestra.run.smithi018.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2023-09-23T20:22:34.082 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:22:33 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/590192516' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-09-23T20:22:34.082 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:22:33 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/590192516' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-09-23T20:22:34.133 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2023-09-23T20:22:34.133 INFO:teuthology.run_tasks:Running task print... 2023-09-23T20:22:34.142 INFO:teuthology.task.print:**** applying stress + thrashing to mixed-version cluster **** 2023-09-23T20:22:34.142 INFO:teuthology.run_tasks:Running task parallel... 2023-09-23T20:22:34.186 INFO:teuthology.task.parallel:starting parallel... 2023-09-23T20:22:34.186 INFO:teuthology.task.parallel:In parallel, running task sequential... 2023-09-23T20:22:34.186 INFO:teuthology.task.sequential:In sequential, running task thrashosds... 2023-09-23T20:22:34.187 INFO:tasks.thrashosds:config is {'aggressive_pg_num_changes': False, 'chance_force_recovery': 0, 'chance_pgnum_grow': 1, 'chance_pgpnum_fix': 1, 'chance_thrash_cluster_full': 0, 'chance_thrash_pg_upmap': 0, 'chance_thrash_pg_upmap_items': 0, 'disable_objectstore_tool_tests': True, 'timeout': 1200, 'sighup_delay': 0.1, 'optrack_toggle_delay': 2.0, 'dump_ops_enable': 'true', 'noscrub_toggle_delay': 2.0, 'random_eio': 0.0} 2023-09-23T20:22:34.188 INFO:tasks.thrashosds:overrides is {'admin_socket': {'branch': 'reef'}, 'ceph': {'conf': {'global': {'mon election default strategy': 1}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}}, 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'sha1': 'bb0daeff97f020f686596fca6eba20c310e26fb7'}, 'ceph-deploy': {'conf': {'client': {'log file': '/var/log/ceph/ceph-$name.$pid.log'}, 'mon': {}}}, 'install': {'ceph': {'flavor': 'default', 'sha1': 'bb0daeff97f020f686596fca6eba20c310e26fb7'}}, 'selinux': {'whitelist': ['scontext=system_u:system_r:logrotate_t:s0']}, 'workunit': {'branch': 'reef', 'sha1': 'bb0daeff97f020f686596fca6eba20c310e26fb7'}} 2023-09-23T20:22:34.188 INFO:tasks.thrashosds:config is {'aggressive_pg_num_changes': False, 'chance_force_recovery': 0, 'chance_pgnum_grow': 1, 'chance_pgpnum_fix': 1, 'chance_thrash_cluster_full': 0, 'chance_thrash_pg_upmap': 0, 'chance_thrash_pg_upmap_items': 0, 'disable_objectstore_tool_tests': True, 'timeout': 1200, 'sighup_delay': 0.1, 'optrack_toggle_delay': 2.0, 'dump_ops_enable': 'true', 'noscrub_toggle_delay': 2.0, 'random_eio': 0.0} 2023-09-23T20:22:34.188 INFO:tasks.thrashosds:Beginning thrashosds... 2023-09-23T20:22:34.188 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2023-09-23T20:22:34.188 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:22:34.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:22:33 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/590192516' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-09-23T20:22:34.793 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:22:35.201 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:22:34 smithi018 ceph-mon[184144]: pgmap v220: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:22:35.202 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:22:34 smithi018 ceph-mon[179647]: pgmap v220: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:22:35.202 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:22:34 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:22:34] "GET /metrics HTTP/1.1" 200 34027 "" "Prometheus/2.43.0" 2023-09-23T20:22:35.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:22:34 smithi149 ceph-mon[170394]: pgmap v220: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:22:36.331 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:22:36.332 INFO:teuthology.orchestra.run.smithi018.stderr:dumped all 2023-09-23T20:22:36.743 INFO:teuthology.orchestra.run.smithi018.stdout:{"pg_ready":true,"pg_map":{"version":221,"stamp":"2023-09-23T20:22:35.430532+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790195,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6495,"num_read_kb":14136,"num_write":1146,"num_write_kb":15141,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1192,"ondisk_log_size":1192,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":164132,"kb_used_data":23076,"kb_used_omap":30,"kb_used_meta":140961,"kb_avail":749797084,"statfs":{"total":767960285184,"available":767792214016,"internally_reserved":0,"allocated":23629824,"data_stored":19305714,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31263,"internal_metadata":144344545},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003391"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":79,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.941990+0000","last_change":"2023-09-23T20:21:18.893413+0000","last_active":"2023-09-23T20:21:48.941990+0000","last_peered":"2023-09-23T20:21:48.941990+0000","last_clean":"2023-09-23T20:21:48.941990+0000","last_became_active":"2023-09-23T20:21:18.893008+0000","last_became_peered":"2023-09-23T20:21:18.893008+0000","last_unstale":"2023-09-23T20:21:48.941990+0000","last_undegraded":"2023-09-23T20:21:48.941990+0000","last_fullsized":"2023-09-23T20:21:48.941990+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:50:55.264882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":56,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.938999+0000","last_change":"2023-09-23T20:21:18.889521+0000","last_active":"2023-09-23T20:21:49.938999+0000","last_peered":"2023-09-23T20:21:49.938999+0000","last_clean":"2023-09-23T20:21:49.938999+0000","last_became_active":"2023-09-23T20:21:18.889268+0000","last_became_peered":"2023-09-23T20:21:18.889268+0000","last_unstale":"2023-09-23T20:21:49.938999+0000","last_undegraded":"2023-09-23T20:21:49.938999+0000","last_fullsized":"2023-09-23T20:21:49.938999+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:05:48.805778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":93,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.949944+0000","last_change":"2023-09-23T20:21:49.949944+0000","last_active":"2023-09-23T20:21:49.949944+0000","last_peered":"2023-09-23T20:21:49.949944+0000","last_clean":"2023-09-23T20:21:49.949944+0000","last_became_active":"2023-09-23T20:21:49.949425+0000","last_became_peered":"2023-09-23T20:21:49.949425+0000","last_unstale":"2023-09-23T20:21:49.949944+0000","last_undegraded":"2023-09-23T20:21:49.949944+0000","last_fullsized":"2023-09-23T20:21:49.949944+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:40:56.395791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":78,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.957394+0000","last_change":"2023-09-23T20:21:49.957394+0000","last_active":"2023-09-23T20:21:49.957394+0000","last_peered":"2023-09-23T20:21:49.957394+0000","last_clean":"2023-09-23T20:21:49.957394+0000","last_became_active":"2023-09-23T20:21:49.957016+0000","last_became_peered":"2023-09-23T20:21:49.957016+0000","last_unstale":"2023-09-23T20:21:49.957394+0000","last_undegraded":"2023-09-23T20:21:49.957394+0000","last_fullsized":"2023-09-23T20:21:49.957394+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:49:26.222448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":86,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.953041+0000","last_change":"2023-09-23T20:21:49.953041+0000","last_active":"2023-09-23T20:21:49.953041+0000","last_peered":"2023-09-23T20:21:49.953041+0000","last_clean":"2023-09-23T20:21:49.953041+0000","last_became_active":"2023-09-23T20:21:49.952763+0000","last_became_peered":"2023-09-23T20:21:49.952763+0000","last_unstale":"2023-09-23T20:21:49.953041+0000","last_undegraded":"2023-09-23T20:21:49.953041+0000","last_fullsized":"2023-09-23T20:21:49.953041+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:58:35.321466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":54,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.950333+0000","last_change":"2023-09-23T20:12:43.753383+0000","last_active":"2023-09-23T20:21:48.950333+0000","last_peered":"2023-09-23T20:21:48.950333+0000","last_clean":"2023-09-23T20:21:48.950333+0000","last_became_active":"2023-09-23T20:12:43.752921+0000","last_became_peered":"2023-09-23T20:12:43.752921+0000","last_unstale":"2023-09-23T20:21:48.950333+0000","last_undegraded":"2023-09-23T20:21:48.950333+0000","last_fullsized":"2023-09-23T20:21:48.950333+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:28:26.982998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":86,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.957885+0000","last_change":"2023-09-23T20:21:49.957885+0000","last_active":"2023-09-23T20:21:49.957885+0000","last_peered":"2023-09-23T20:21:49.957885+0000","last_clean":"2023-09-23T20:21:49.957885+0000","last_became_active":"2023-09-23T20:21:49.957492+0000","last_became_peered":"2023-09-23T20:21:49.957492+0000","last_unstale":"2023-09-23T20:21:49.957885+0000","last_undegraded":"2023-09-23T20:21:49.957885+0000","last_fullsized":"2023-09-23T20:21:49.957885+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:24:55.137836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":71,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.949471+0000","last_change":"2023-09-23T20:20:46.938981+0000","last_active":"2023-09-23T20:21:48.949471+0000","last_peered":"2023-09-23T20:21:48.949471+0000","last_clean":"2023-09-23T20:21:48.949471+0000","last_became_active":"2023-09-23T20:20:46.938627+0000","last_became_peered":"2023-09-23T20:20:46.938627+0000","last_unstale":"2023-09-23T20:21:48.949471+0000","last_undegraded":"2023-09-23T20:21:48.949471+0000","last_fullsized":"2023-09-23T20:21:48.949471+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:44:43.177086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":90,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.953299+0000","last_change":"2023-09-23T20:21:49.953299+0000","last_active":"2023-09-23T20:21:49.953299+0000","last_peered":"2023-09-23T20:21:49.953299+0000","last_clean":"2023-09-23T20:21:49.953299+0000","last_became_active":"2023-09-23T20:21:49.952996+0000","last_became_peered":"2023-09-23T20:21:49.952996+0000","last_unstale":"2023-09-23T20:21:49.953299+0000","last_undegraded":"2023-09-23T20:21:49.953299+0000","last_fullsized":"2023-09-23T20:21:49.953299+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:46:51.746281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":40,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.950883+0000","last_change":"2023-09-23T20:20:14.888484+0000","last_active":"2023-09-23T20:21:48.950883+0000","last_peered":"2023-09-23T20:21:48.950883+0000","last_clean":"2023-09-23T20:21:48.950883+0000","last_became_active":"2023-09-23T20:20:14.887219+0000","last_became_peered":"2023-09-23T20:20:14.887219+0000","last_unstale":"2023-09-23T20:21:48.950883+0000","last_undegraded":"2023-09-23T20:21:48.950883+0000","last_fullsized":"2023-09-23T20:21:48.950883+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:13:53.763649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":52,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.943196+0000","last_change":"2023-09-23T20:12:43.765908+0000","last_active":"2023-09-23T20:21:48.943196+0000","last_peered":"2023-09-23T20:21:48.943196+0000","last_clean":"2023-09-23T20:21:48.943196+0000","last_became_active":"2023-09-23T20:12:43.765558+0000","last_became_peered":"2023-09-23T20:12:43.765558+0000","last_unstale":"2023-09-23T20:21:48.943196+0000","last_undegraded":"2023-09-23T20:21:48.943196+0000","last_fullsized":"2023-09-23T20:21:48.943196+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:35:43.809898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":58,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.939966+0000","last_change":"2023-09-23T20:20:46.937163+0000","last_active":"2023-09-23T20:21:49.939966+0000","last_peered":"2023-09-23T20:21:49.939966+0000","last_clean":"2023-09-23T20:21:49.939966+0000","last_became_active":"2023-09-23T20:20:46.936846+0000","last_became_peered":"2023-09-23T20:20:46.936846+0000","last_unstale":"2023-09-23T20:21:49.939966+0000","last_undegraded":"2023-09-23T20:21:49.939966+0000","last_fullsized":"2023-09-23T20:21:49.939966+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:01:32.798855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"86'117","reported_seq":596,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.945893+0000","last_change":"2023-09-23T20:21:18.897269+0000","last_active":"2023-09-23T20:21:48.945893+0000","last_peered":"2023-09-23T20:21:48.945893+0000","last_clean":"2023-09-23T20:21:48.945893+0000","last_became_active":"2023-09-23T20:21:18.896834+0000","last_became_peered":"2023-09-23T20:21:18.896834+0000","last_unstale":"2023-09-23T20:21:48.945893+0000","last_undegraded":"2023-09-23T20:21:48.945893+0000","last_fullsized":"2023-09-23T20:21:48.945893+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:57:17.499025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046891400000000002,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":893,"num_read_kb":922,"num_write":82,"num_write_kb":4193,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":67,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.953565+0000","last_change":"2023-09-23T20:21:49.953565+0000","last_active":"2023-09-23T20:21:49.953565+0000","last_peered":"2023-09-23T20:21:49.953565+0000","last_clean":"2023-09-23T20:21:49.953565+0000","last_became_active":"2023-09-23T20:21:49.953176+0000","last_became_peered":"2023-09-23T20:21:49.953176+0000","last_unstale":"2023-09-23T20:21:49.953565+0000","last_undegraded":"2023-09-23T20:21:49.953565+0000","last_fullsized":"2023-09-23T20:21:49.953565+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:02:35.778207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"86'128","reported_seq":432,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.940390+0000","last_change":"2023-09-23T20:20:46.939256+0000","last_active":"2023-09-23T20:21:48.940390+0000","last_peered":"2023-09-23T20:21:48.940390+0000","last_clean":"2023-09-23T20:21:48.940390+0000","last_became_active":"2023-09-23T20:20:46.938756+0000","last_became_peered":"2023-09-23T20:20:46.938756+0000","last_unstale":"2023-09-23T20:21:48.940390+0000","last_undegraded":"2023-09-23T20:21:48.940390+0000","last_fullsized":"2023-09-23T20:21:48.940390+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:52:18.161436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000293202,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":579,"num_read_kb":1473,"num_write":96,"num_write_kb":1062,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":70,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.957535+0000","last_change":"2023-09-23T20:21:49.957535+0000","last_active":"2023-09-23T20:21:49.957535+0000","last_peered":"2023-09-23T20:21:49.957535+0000","last_clean":"2023-09-23T20:21:49.957535+0000","last_became_active":"2023-09-23T20:21:49.954319+0000","last_became_peered":"2023-09-23T20:21:49.954319+0000","last_unstale":"2023-09-23T20:21:49.957535+0000","last_undegraded":"2023-09-23T20:21:49.957535+0000","last_fullsized":"2023-09-23T20:21:49.957535+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:14:10.293305+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":38,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.938907+0000","last_change":"2023-09-23T20:20:14.888874+0000","last_active":"2023-09-23T20:21:49.938907+0000","last_peered":"2023-09-23T20:21:49.938907+0000","last_clean":"2023-09-23T20:21:49.938907+0000","last_became_active":"2023-09-23T20:20:14.887724+0000","last_became_peered":"2023-09-23T20:20:14.887724+0000","last_unstale":"2023-09-23T20:21:49.938907+0000","last_undegraded":"2023-09-23T20:21:49.938907+0000","last_fullsized":"2023-09-23T20:21:49.938907+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:09:24.510326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":47,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.939386+0000","last_change":"2023-09-23T20:21:18.893457+0000","last_active":"2023-09-23T20:21:48.939386+0000","last_peered":"2023-09-23T20:21:48.939386+0000","last_clean":"2023-09-23T20:21:48.939386+0000","last_became_active":"2023-09-23T20:21:18.893175+0000","last_became_peered":"2023-09-23T20:21:18.893175+0000","last_unstale":"2023-09-23T20:21:48.939386+0000","last_undegraded":"2023-09-23T20:21:48.939386+0000","last_fullsized":"2023-09-23T20:21:48.939386+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:24:37.563758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"86'102","reported_seq":616,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.951036+0000","last_change":"2023-09-23T20:20:14.881975+0000","last_active":"2023-09-23T20:21:48.951036+0000","last_peered":"2023-09-23T20:21:48.951036+0000","last_clean":"2023-09-23T20:21:48.951036+0000","last_became_active":"2023-09-23T20:20:14.881544+0000","last_became_peered":"2023-09-23T20:20:14.881544+0000","last_unstale":"2023-09-23T20:21:48.951036+0000","last_undegraded":"2023-09-23T20:21:48.951036+0000","last_fullsized":"2023-09-23T20:21:48.951036+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":102,"log_dups_size":0,"ondisk_log_size":102,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:34.594972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030656500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":963,"num_read_kb":1234,"num_write":60,"num_write_kb":225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":68,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.940049+0000","last_change":"2023-09-23T20:21:18.887651+0000","last_active":"2023-09-23T20:21:49.940049+0000","last_peered":"2023-09-23T20:21:49.940049+0000","last_clean":"2023-09-23T20:21:49.940049+0000","last_became_active":"2023-09-23T20:21:18.887338+0000","last_became_peered":"2023-09-23T20:21:18.887338+0000","last_unstale":"2023-09-23T20:21:49.940049+0000","last_undegraded":"2023-09-23T20:21:49.940049+0000","last_fullsized":"2023-09-23T20:21:49.940049+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:26:39.067022+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":52,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.952295+0000","last_change":"2023-09-23T20:21:49.951490+0000","last_active":"2023-09-23T20:21:49.952295+0000","last_peered":"2023-09-23T20:21:49.952295+0000","last_clean":"2023-09-23T20:21:49.952295+0000","last_became_active":"2023-09-23T20:21:49.951079+0000","last_became_peered":"2023-09-23T20:21:49.951079+0000","last_unstale":"2023-09-23T20:21:49.952295+0000","last_undegraded":"2023-09-23T20:21:49.952295+0000","last_fullsized":"2023-09-23T20:21:49.952295+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:12:01.053013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"86'80","reported_seq":356,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.952040+0000","last_change":"2023-09-23T20:20:46.950784+0000","last_active":"2023-09-23T20:21:48.952040+0000","last_peered":"2023-09-23T20:21:48.952040+0000","last_clean":"2023-09-23T20:21:48.952040+0000","last_became_active":"2023-09-23T20:20:46.950071+0000","last_became_peered":"2023-09-23T20:20:46.950071+0000","last_unstale":"2023-09-23T20:21:48.952040+0000","last_undegraded":"2023-09-23T20:21:48.952040+0000","last_fullsized":"2023-09-23T20:21:48.952040+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":80,"log_dups_size":0,"ondisk_log_size":80,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:11:06.812404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035115,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":395,"num_read_kb":1370,"num_write":66,"num_write_kb":1061,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":52,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.946301+0000","last_change":"2023-09-23T20:12:45.609829+0000","last_active":"2023-09-23T20:21:48.946301+0000","last_peered":"2023-09-23T20:21:48.946301+0000","last_clean":"2023-09-23T20:21:48.946301+0000","last_became_active":"2023-09-23T20:12:45.609467+0000","last_became_peered":"2023-09-23T20:12:45.609467+0000","last_unstale":"2023-09-23T20:21:48.946301+0000","last_undegraded":"2023-09-23T20:21:48.946301+0000","last_fullsized":"2023-09-23T20:21:48.946301+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:27:31.490555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":22,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.956277+0000","last_change":"2023-09-23T20:21:49.956277+0000","last_active":"2023-09-23T20:21:49.956277+0000","last_peered":"2023-09-23T20:21:49.956277+0000","last_clean":"2023-09-23T20:21:49.956277+0000","last_became_active":"2023-09-23T20:21:49.955974+0000","last_became_peered":"2023-09-23T20:21:49.955974+0000","last_unstale":"2023-09-23T20:21:49.956277+0000","last_undegraded":"2023-09-23T20:21:49.956277+0000","last_fullsized":"2023-09-23T20:21:49.956277+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:11:37.850675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":452,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.951949+0000","last_change":"2023-09-23T20:20:14.882204+0000","last_active":"2023-09-23T20:21:48.951949+0000","last_peered":"2023-09-23T20:21:48.951949+0000","last_clean":"2023-09-23T20:21:48.951949+0000","last_became_active":"2023-09-23T20:20:14.881699+0000","last_became_peered":"2023-09-23T20:20:14.881699+0000","last_unstale":"2023-09-23T20:21:48.951949+0000","last_undegraded":"2023-09-23T20:21:48.951949+0000","last_fullsized":"2023-09-23T20:21:48.951949+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:09:47.213130+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:16:37.341010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"86'194","reported_seq":728,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.948141+0000","last_change":"2023-09-23T20:21:18.892176+0000","last_active":"2023-09-23T20:21:48.948141+0000","last_peered":"2023-09-23T20:21:48.948141+0000","last_clean":"2023-09-23T20:21:48.948141+0000","last_became_active":"2023-09-23T20:21:18.891647+0000","last_became_peered":"2023-09-23T20:21:18.891647+0000","last_unstale":"2023-09-23T20:21:48.948141+0000","last_undegraded":"2023-09-23T20:21:48.948141+0000","last_fullsized":"2023-09-23T20:21:48.948141+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":194,"log_dups_size":0,"ondisk_log_size":194,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:25:52.557392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00037995999999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1025,"num_read_kb":1201,"num_write":173,"num_write_kb":1276,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":76,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.939413+0000","last_change":"2023-09-23T20:20:46.936524+0000","last_active":"2023-09-23T20:21:48.939413+0000","last_peered":"2023-09-23T20:21:48.939413+0000","last_clean":"2023-09-23T20:21:48.939413+0000","last_became_active":"2023-09-23T20:20:46.936267+0000","last_became_peered":"2023-09-23T20:20:46.936267+0000","last_unstale":"2023-09-23T20:21:48.939413+0000","last_undegraded":"2023-09-23T20:21:48.939413+0000","last_fullsized":"2023-09-23T20:21:48.939413+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:32:20.984521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":47,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.937923+0000","last_change":"2023-09-23T20:21:18.891553+0000","last_active":"2023-09-23T20:21:48.937923+0000","last_peered":"2023-09-23T20:21:48.937923+0000","last_clean":"2023-09-23T20:21:48.937923+0000","last_became_active":"2023-09-23T20:21:18.891230+0000","last_became_peered":"2023-09-23T20:21:18.891230+0000","last_unstale":"2023-09-23T20:21:48.937923+0000","last_undegraded":"2023-09-23T20:21:48.937923+0000","last_fullsized":"2023-09-23T20:21:48.937923+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:15:04.114927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"86'130","reported_seq":413,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.954331+0000","last_change":"2023-09-23T20:21:49.954331+0000","last_active":"2023-09-23T20:21:49.954331+0000","last_peered":"2023-09-23T20:21:49.954331+0000","last_clean":"2023-09-23T20:21:49.954331+0000","last_became_active":"2023-09-23T20:21:49.953687+0000","last_became_peered":"2023-09-23T20:21:49.953687+0000","last_unstale":"2023-09-23T20:21:49.954331+0000","last_undegraded":"2023-09-23T20:21:49.954331+0000","last_fullsized":"2023-09-23T20:21:49.954331+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:12.549803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011300049999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":512,"num_read_kb":1820,"num_write":133,"num_write_kb":1347,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":63,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.952961+0000","last_change":"2023-09-23T20:21:49.952961+0000","last_active":"2023-09-23T20:21:49.952961+0000","last_peered":"2023-09-23T20:21:49.952961+0000","last_clean":"2023-09-23T20:21:49.952961+0000","last_became_active":"2023-09-23T20:21:49.952644+0000","last_became_peered":"2023-09-23T20:21:49.952644+0000","last_unstale":"2023-09-23T20:21:49.952961+0000","last_undegraded":"2023-09-23T20:21:49.952961+0000","last_fullsized":"2023-09-23T20:21:49.952961+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:05:30.492540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":70,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.943468+0000","last_change":"2023-09-23T20:20:14.887345+0000","last_active":"2023-09-23T20:21:48.943468+0000","last_peered":"2023-09-23T20:21:48.943468+0000","last_clean":"2023-09-23T20:21:48.943468+0000","last_became_active":"2023-09-23T20:20:14.886970+0000","last_became_peered":"2023-09-23T20:20:14.886970+0000","last_unstale":"2023-09-23T20:21:48.943468+0000","last_undegraded":"2023-09-23T20:21:48.943468+0000","last_fullsized":"2023-09-23T20:21:48.943468+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:00:38.575926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"86'106","reported_seq":734,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.949374+0000","last_change":"2023-09-23T20:20:46.939468+0000","last_active":"2023-09-23T20:21:48.949374+0000","last_peered":"2023-09-23T20:21:48.949374+0000","last_clean":"2023-09-23T20:21:48.949374+0000","last_became_active":"2023-09-23T20:20:46.938954+0000","last_became_peered":"2023-09-23T20:20:46.938954+0000","last_unstale":"2023-09-23T20:21:48.949374+0000","last_undegraded":"2023-09-23T20:21:48.949374+0000","last_fullsized":"2023-09-23T20:21:48.949374+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":106,"log_dups_size":0,"ondisk_log_size":106,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:01:43.761064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00099302599999999993,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1105,"num_read_kb":1689,"num_write":100,"num_write_kb":404,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":33,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.938411+0000","last_change":"2023-09-23T20:20:46.937104+0000","last_active":"2023-09-23T20:21:48.938411+0000","last_peered":"2023-09-23T20:21:48.938411+0000","last_clean":"2023-09-23T20:21:48.938411+0000","last_became_active":"2023-09-23T20:20:46.936820+0000","last_became_peered":"2023-09-23T20:20:46.936820+0000","last_unstale":"2023-09-23T20:21:48.938411+0000","last_undegraded":"2023-09-23T20:21:48.938411+0000","last_fullsized":"2023-09-23T20:21:48.938411+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:22:19.879310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":86,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.953141+0000","last_change":"2023-09-23T20:21:49.953141+0000","last_active":"2023-09-23T20:21:49.953141+0000","last_peered":"2023-09-23T20:21:49.953141+0000","last_clean":"2023-09-23T20:21:49.953141+0000","last_became_active":"2023-09-23T20:21:49.952794+0000","last_became_peered":"2023-09-23T20:21:49.952794+0000","last_unstale":"2023-09-23T20:21:49.953141+0000","last_undegraded":"2023-09-23T20:21:49.953141+0000","last_fullsized":"2023-09-23T20:21:49.953141+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:36:19.919248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"86'119","reported_seq":441,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.939283+0000","last_change":"2023-09-23T20:20:46.937071+0000","last_active":"2023-09-23T20:21:48.939283+0000","last_peered":"2023-09-23T20:21:48.939283+0000","last_clean":"2023-09-23T20:21:48.939283+0000","last_became_active":"2023-09-23T20:20:46.935348+0000","last_became_peered":"2023-09-23T20:20:46.935348+0000","last_unstale":"2023-09-23T20:21:48.939283+0000","last_undegraded":"2023-09-23T20:21:48.939283+0000","last_fullsized":"2023-09-23T20:21:48.939283+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":119,"log_dups_size":0,"ondisk_log_size":119,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:09.829030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032705499999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":523,"num_read_kb":3503,"num_write":136,"num_write_kb":3128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":43,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.959532+0000","last_change":"2023-09-23T20:21:49.959532+0000","last_active":"2023-09-23T20:21:49.959532+0000","last_peered":"2023-09-23T20:21:49.959532+0000","last_clean":"2023-09-23T20:21:49.959532+0000","last_became_active":"2023-09-23T20:21:49.959161+0000","last_became_peered":"2023-09-23T20:21:49.959161+0000","last_unstale":"2023-09-23T20:21:49.959532+0000","last_undegraded":"2023-09-23T20:21:49.959532+0000","last_fullsized":"2023-09-23T20:21:49.959532+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:00:05.013667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":73,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.958915+0000","last_change":"2023-09-23T20:21:49.958915+0000","last_active":"2023-09-23T20:21:49.958915+0000","last_peered":"2023-09-23T20:21:49.958915+0000","last_clean":"2023-09-23T20:21:49.958915+0000","last_became_active":"2023-09-23T20:21:49.958440+0000","last_became_peered":"2023-09-23T20:21:49.958440+0000","last_unstale":"2023-09-23T20:21:49.958915+0000","last_undegraded":"2023-09-23T20:21:49.958915+0000","last_fullsized":"2023-09-23T20:21:49.958915+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:36:48.998945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":69,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.942436+0000","last_change":"2023-09-23T20:20:46.939720+0000","last_active":"2023-09-23T20:21:48.942436+0000","last_peered":"2023-09-23T20:21:48.942436+0000","last_clean":"2023-09-23T20:21:48.942436+0000","last_became_active":"2023-09-23T20:20:46.939357+0000","last_became_peered":"2023-09-23T20:20:46.939357+0000","last_unstale":"2023-09-23T20:21:48.942436+0000","last_undegraded":"2023-09-23T20:21:48.942436+0000","last_fullsized":"2023-09-23T20:21:48.942436+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:56.613212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":70,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.943289+0000","last_change":"2023-09-23T20:21:18.896635+0000","last_active":"2023-09-23T20:21:48.943289+0000","last_peered":"2023-09-23T20:21:48.943289+0000","last_clean":"2023-09-23T20:21:48.943289+0000","last_became_active":"2023-09-23T20:21:18.896335+0000","last_became_peered":"2023-09-23T20:21:18.896335+0000","last_unstale":"2023-09-23T20:21:48.943289+0000","last_undegraded":"2023-09-23T20:21:48.943289+0000","last_fullsized":"2023-09-23T20:21:48.943289+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:46:43.743795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":79,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.944402+0000","last_change":"2023-09-23T20:21:18.891669+0000","last_active":"2023-09-23T20:21:48.944402+0000","last_peered":"2023-09-23T20:21:48.944402+0000","last_clean":"2023-09-23T20:21:48.944402+0000","last_became_active":"2023-09-23T20:21:18.891326+0000","last_became_peered":"2023-09-23T20:21:18.891326+0000","last_unstale":"2023-09-23T20:21:48.944402+0000","last_undegraded":"2023-09-23T20:21:48.944402+0000","last_fullsized":"2023-09-23T20:21:48.944402+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:48:39.893213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":70,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.946401+0000","last_change":"2023-09-23T20:20:46.937163+0000","last_active":"2023-09-23T20:21:48.946401+0000","last_peered":"2023-09-23T20:21:48.946401+0000","last_clean":"2023-09-23T20:21:48.946401+0000","last_became_active":"2023-09-23T20:20:46.936757+0000","last_became_peered":"2023-09-23T20:20:46.936757+0000","last_unstale":"2023-09-23T20:21:48.946401+0000","last_undegraded":"2023-09-23T20:21:48.946401+0000","last_fullsized":"2023-09-23T20:21:48.946401+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:09.401379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":69,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.954710+0000","last_change":"2023-09-23T20:21:49.954710+0000","last_active":"2023-09-23T20:21:49.954710+0000","last_peered":"2023-09-23T20:21:49.954710+0000","last_clean":"2023-09-23T20:21:49.954710+0000","last_became_active":"2023-09-23T20:21:49.954355+0000","last_became_peered":"2023-09-23T20:21:49.954355+0000","last_unstale":"2023-09-23T20:21:49.954710+0000","last_undegraded":"2023-09-23T20:21:49.954710+0000","last_fullsized":"2023-09-23T20:21:49.954710+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:56:16.233285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":70,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.953004+0000","last_change":"2023-09-23T20:21:49.953004+0000","last_active":"2023-09-23T20:21:49.953004+0000","last_peered":"2023-09-23T20:21:49.953004+0000","last_clean":"2023-09-23T20:21:49.953004+0000","last_became_active":"2023-09-23T20:21:49.952625+0000","last_became_peered":"2023-09-23T20:21:49.952625+0000","last_unstale":"2023-09-23T20:21:49.953004+0000","last_undegraded":"2023-09-23T20:21:49.953004+0000","last_fullsized":"2023-09-23T20:21:49.953004+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:48:28.305640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":53,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.939096+0000","last_change":"2023-09-23T20:20:46.935968+0000","last_active":"2023-09-23T20:21:49.939096+0000","last_peered":"2023-09-23T20:21:49.939096+0000","last_clean":"2023-09-23T20:21:49.939096+0000","last_became_active":"2023-09-23T20:20:46.935618+0000","last_became_peered":"2023-09-23T20:20:46.935618+0000","last_unstale":"2023-09-23T20:21:49.939096+0000","last_undegraded":"2023-09-23T20:21:49.939096+0000","last_fullsized":"2023-09-23T20:21:49.939096+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:42:59.867176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":69,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.954875+0000","last_change":"2023-09-23T20:21:49.954875+0000","last_active":"2023-09-23T20:21:49.954875+0000","last_peered":"2023-09-23T20:21:49.954875+0000","last_clean":"2023-09-23T20:21:49.954875+0000","last_became_active":"2023-09-23T20:21:49.954519+0000","last_became_peered":"2023-09-23T20:21:49.954519+0000","last_unstale":"2023-09-23T20:21:49.954875+0000","last_undegraded":"2023-09-23T20:21:49.954875+0000","last_fullsized":"2023-09-23T20:21:49.954875+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:29:52.962029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":69,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.946248+0000","last_change":"2023-09-23T20:20:46.934762+0000","last_active":"2023-09-23T20:21:48.946248+0000","last_peered":"2023-09-23T20:21:48.946248+0000","last_clean":"2023-09-23T20:21:48.946248+0000","last_became_active":"2023-09-23T20:20:46.934472+0000","last_became_peered":"2023-09-23T20:20:46.934472+0000","last_unstale":"2023-09-23T20:21:48.946248+0000","last_undegraded":"2023-09-23T20:21:48.946248+0000","last_fullsized":"2023-09-23T20:21:48.946248+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:28:42.708286+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":54,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.951041+0000","last_change":"2023-09-23T20:12:43.751523+0000","last_active":"2023-09-23T20:21:48.951041+0000","last_peered":"2023-09-23T20:21:48.951041+0000","last_clean":"2023-09-23T20:21:48.951041+0000","last_became_active":"2023-09-23T20:12:43.751257+0000","last_became_peered":"2023-09-23T20:12:43.751257+0000","last_unstale":"2023-09-23T20:21:48.951041+0000","last_undegraded":"2023-09-23T20:21:48.951041+0000","last_fullsized":"2023-09-23T20:21:48.951041+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:50:27.040225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":86,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.957494+0000","last_change":"2023-09-23T20:21:49.957494+0000","last_active":"2023-09-23T20:21:49.957494+0000","last_peered":"2023-09-23T20:21:49.957494+0000","last_clean":"2023-09-23T20:21:49.957494+0000","last_became_active":"2023-09-23T20:21:49.957266+0000","last_became_peered":"2023-09-23T20:21:49.957266+0000","last_unstale":"2023-09-23T20:21:49.957494+0000","last_undegraded":"2023-09-23T20:21:49.957494+0000","last_fullsized":"2023-09-23T20:21:49.957494+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:49:51.483575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":72,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.949331+0000","last_change":"2023-09-23T20:20:46.950784+0000","last_active":"2023-09-23T20:21:48.949331+0000","last_peered":"2023-09-23T20:21:48.949331+0000","last_clean":"2023-09-23T20:21:48.949331+0000","last_became_active":"2023-09-23T20:20:46.950055+0000","last_became_peered":"2023-09-23T20:20:46.950055+0000","last_unstale":"2023-09-23T20:21:48.949331+0000","last_undegraded":"2023-09-23T20:21:48.949331+0000","last_fullsized":"2023-09-23T20:21:48.949331+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:23:07.645427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":51,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.949875+0000","last_change":"2023-09-23T20:12:45.179211+0000","last_active":"2023-09-23T20:21:48.949875+0000","last_peered":"2023-09-23T20:21:48.949875+0000","last_clean":"2023-09-23T20:21:48.949875+0000","last_became_active":"2023-09-23T20:12:45.178966+0000","last_became_peered":"2023-09-23T20:12:45.178966+0000","last_unstale":"2023-09-23T20:21:48.949875+0000","last_undegraded":"2023-09-23T20:21:48.949875+0000","last_fullsized":"2023-09-23T20:21:48.949875+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:58.902140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":71,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.950476+0000","last_change":"2023-09-23T20:20:14.883283+0000","last_active":"2023-09-23T20:21:48.950476+0000","last_peered":"2023-09-23T20:21:48.950476+0000","last_clean":"2023-09-23T20:21:48.950476+0000","last_became_active":"2023-09-23T20:20:14.882806+0000","last_became_peered":"2023-09-23T20:20:14.882806+0000","last_unstale":"2023-09-23T20:21:48.950476+0000","last_undegraded":"2023-09-23T20:21:48.950476+0000","last_fullsized":"2023-09-23T20:21:48.950476+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:23:35.295701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":72,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.958182+0000","last_change":"2023-09-23T20:21:49.958182+0000","last_active":"2023-09-23T20:21:49.958182+0000","last_peered":"2023-09-23T20:21:49.958182+0000","last_clean":"2023-09-23T20:21:49.958182+0000","last_became_active":"2023-09-23T20:21:49.957868+0000","last_became_peered":"2023-09-23T20:21:49.957868+0000","last_unstale":"2023-09-23T20:21:49.958182+0000","last_undegraded":"2023-09-23T20:21:49.958182+0000","last_fullsized":"2023-09-23T20:21:49.958182+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:27:41.577566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":78,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.944990+0000","last_change":"2023-09-23T20:20:14.881009+0000","last_active":"2023-09-23T20:21:48.944990+0000","last_peered":"2023-09-23T20:21:48.944990+0000","last_clean":"2023-09-23T20:21:48.944990+0000","last_became_active":"2023-09-23T20:20:14.880685+0000","last_became_peered":"2023-09-23T20:20:14.880685+0000","last_unstale":"2023-09-23T20:21:48.944990+0000","last_undegraded":"2023-09-23T20:21:48.944990+0000","last_fullsized":"2023-09-23T20:21:48.944990+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:28.010434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":52,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.954242+0000","last_change":"2023-09-23T20:21:49.954242+0000","last_active":"2023-09-23T20:21:49.954242+0000","last_peered":"2023-09-23T20:21:49.954242+0000","last_clean":"2023-09-23T20:21:49.954242+0000","last_became_active":"2023-09-23T20:21:49.953597+0000","last_became_peered":"2023-09-23T20:21:49.953597+0000","last_unstale":"2023-09-23T20:21:49.954242+0000","last_undegraded":"2023-09-23T20:21:49.954242+0000","last_fullsized":"2023-09-23T20:21:49.954242+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:08:46.716466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":53,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.948637+0000","last_change":"2023-09-23T20:12:43.749139+0000","last_active":"2023-09-23T20:21:48.948637+0000","last_peered":"2023-09-23T20:21:48.948637+0000","last_clean":"2023-09-23T20:21:48.948637+0000","last_became_active":"2023-09-23T20:12:43.748854+0000","last_became_peered":"2023-09-23T20:12:43.748854+0000","last_unstale":"2023-09-23T20:21:48.948637+0000","last_undegraded":"2023-09-23T20:21:48.948637+0000","last_fullsized":"2023-09-23T20:21:48.948637+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:57:11.630712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":68,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.952635+0000","last_change":"2023-09-23T20:21:49.952635+0000","last_active":"2023-09-23T20:21:49.952635+0000","last_peered":"2023-09-23T20:21:49.952635+0000","last_clean":"2023-09-23T20:21:49.952635+0000","last_became_active":"2023-09-23T20:21:49.952287+0000","last_became_peered":"2023-09-23T20:21:49.952287+0000","last_unstale":"2023-09-23T20:21:49.952635+0000","last_undegraded":"2023-09-23T20:21:49.952635+0000","last_fullsized":"2023-09-23T20:21:49.952635+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:17:17.370526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":52,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.955345+0000","last_change":"2023-09-23T20:21:49.953201+0000","last_active":"2023-09-23T20:21:49.955345+0000","last_peered":"2023-09-23T20:21:49.955345+0000","last_clean":"2023-09-23T20:21:49.955345+0000","last_became_active":"2023-09-23T20:21:49.952917+0000","last_became_peered":"2023-09-23T20:21:49.952917+0000","last_unstale":"2023-09-23T20:21:49.955345+0000","last_undegraded":"2023-09-23T20:21:49.955345+0000","last_fullsized":"2023-09-23T20:21:49.955345+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:54.579892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":72,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.958515+0000","last_change":"2023-09-23T20:21:49.958515+0000","last_active":"2023-09-23T20:21:49.958515+0000","last_peered":"2023-09-23T20:21:49.958515+0000","last_clean":"2023-09-23T20:21:49.958515+0000","last_became_active":"2023-09-23T20:21:49.958156+0000","last_became_peered":"2023-09-23T20:21:49.958156+0000","last_unstale":"2023-09-23T20:21:49.958515+0000","last_undegraded":"2023-09-23T20:21:49.958515+0000","last_fullsized":"2023-09-23T20:21:49.958515+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:56:47.145702+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":72,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.948079+0000","last_change":"2023-09-23T20:21:18.897418+0000","last_active":"2023-09-23T20:21:48.948079+0000","last_peered":"2023-09-23T20:21:48.948079+0000","last_clean":"2023-09-23T20:21:48.948079+0000","last_became_active":"2023-09-23T20:21:18.896968+0000","last_became_peered":"2023-09-23T20:21:18.896968+0000","last_unstale":"2023-09-23T20:21:48.948079+0000","last_undegraded":"2023-09-23T20:21:48.948079+0000","last_fullsized":"2023-09-23T20:21:48.948079+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:12:44.651568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":69,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.943565+0000","last_change":"2023-09-23T20:20:46.939938+0000","last_active":"2023-09-23T20:21:48.943565+0000","last_peered":"2023-09-23T20:21:48.943565+0000","last_clean":"2023-09-23T20:21:48.943565+0000","last_became_active":"2023-09-23T20:20:46.938777+0000","last_became_peered":"2023-09-23T20:20:46.938777+0000","last_unstale":"2023-09-23T20:21:48.943565+0000","last_undegraded":"2023-09-23T20:21:48.943565+0000","last_fullsized":"2023-09-23T20:21:48.943565+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:45:10.329438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":75,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.943049+0000","last_change":"2023-09-23T20:12:43.750217+0000","last_active":"2023-09-23T20:21:48.943049+0000","last_peered":"2023-09-23T20:21:48.943049+0000","last_clean":"2023-09-23T20:21:48.943049+0000","last_became_active":"2023-09-23T20:12:43.749929+0000","last_became_peered":"2023-09-23T20:12:43.749929+0000","last_unstale":"2023-09-23T20:21:48.943049+0000","last_undegraded":"2023-09-23T20:21:48.943049+0000","last_fullsized":"2023-09-23T20:21:48.943049+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:50:28.470211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":48,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.938497+0000","last_change":"2023-09-23T20:21:18.891438+0000","last_active":"2023-09-23T20:21:48.938497+0000","last_peered":"2023-09-23T20:21:48.938497+0000","last_clean":"2023-09-23T20:21:48.938497+0000","last_became_active":"2023-09-23T20:21:18.891140+0000","last_became_peered":"2023-09-23T20:21:18.891140+0000","last_unstale":"2023-09-23T20:21:48.938497+0000","last_undegraded":"2023-09-23T20:21:48.938497+0000","last_fullsized":"2023-09-23T20:21:48.938497+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:18:44.603577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":73,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.954219+0000","last_change":"2023-09-23T20:21:49.954219+0000","last_active":"2023-09-23T20:21:49.954219+0000","last_peered":"2023-09-23T20:21:49.954219+0000","last_clean":"2023-09-23T20:21:49.954219+0000","last_became_active":"2023-09-23T20:21:49.953827+0000","last_became_peered":"2023-09-23T20:21:49.953827+0000","last_unstale":"2023-09-23T20:21:49.954219+0000","last_undegraded":"2023-09-23T20:21:49.954219+0000","last_fullsized":"2023-09-23T20:21:49.954219+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:46:48.036706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":23,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.955179+0000","last_change":"2023-09-23T20:21:49.955179+0000","last_active":"2023-09-23T20:21:49.955179+0000","last_peered":"2023-09-23T20:21:49.955179+0000","last_clean":"2023-09-23T20:21:49.955179+0000","last_became_active":"2023-09-23T20:21:49.954878+0000","last_became_peered":"2023-09-23T20:21:49.954878+0000","last_unstale":"2023-09-23T20:21:49.955179+0000","last_undegraded":"2023-09-23T20:21:49.955179+0000","last_fullsized":"2023-09-23T20:21:49.955179+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:02:26.579246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":69,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.947957+0000","last_change":"2023-09-23T20:20:46.950281+0000","last_active":"2023-09-23T20:21:48.947957+0000","last_peered":"2023-09-23T20:21:48.947957+0000","last_clean":"2023-09-23T20:21:48.947957+0000","last_became_active":"2023-09-23T20:20:46.949892+0000","last_became_peered":"2023-09-23T20:20:46.949892+0000","last_unstale":"2023-09-23T20:21:48.947957+0000","last_undegraded":"2023-09-23T20:21:48.947957+0000","last_fullsized":"2023-09-23T20:21:48.947957+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:28.763832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":72,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.958837+0000","last_change":"2023-09-23T20:21:49.958837+0000","last_active":"2023-09-23T20:21:49.958837+0000","last_peered":"2023-09-23T20:21:49.958837+0000","last_clean":"2023-09-23T20:21:49.958837+0000","last_became_active":"2023-09-23T20:21:49.958135+0000","last_became_peered":"2023-09-23T20:21:49.958135+0000","last_unstale":"2023-09-23T20:21:49.958837+0000","last_undegraded":"2023-09-23T20:21:49.958837+0000","last_fullsized":"2023-09-23T20:21:49.958837+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:31:52.350275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":51,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.950402+0000","last_change":"2023-09-23T20:21:49.950402+0000","last_active":"2023-09-23T20:21:49.950402+0000","last_peered":"2023-09-23T20:21:49.950402+0000","last_clean":"2023-09-23T20:21:49.950402+0000","last_became_active":"2023-09-23T20:21:49.950116+0000","last_became_peered":"2023-09-23T20:21:49.950116+0000","last_unstale":"2023-09-23T20:21:49.950402+0000","last_undegraded":"2023-09-23T20:21:49.950402+0000","last_fullsized":"2023-09-23T20:21:49.950402+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:10:20.322615+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":69,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.945804+0000","last_change":"2023-09-23T20:20:46.937089+0000","last_active":"2023-09-23T20:21:48.945804+0000","last_peered":"2023-09-23T20:21:48.945804+0000","last_clean":"2023-09-23T20:21:48.945804+0000","last_became_active":"2023-09-23T20:20:46.936831+0000","last_became_peered":"2023-09-23T20:20:46.936831+0000","last_unstale":"2023-09-23T20:21:48.945804+0000","last_undegraded":"2023-09-23T20:21:48.945804+0000","last_fullsized":"2023-09-23T20:21:48.945804+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:34:56.747010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":90,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.958727+0000","last_change":"2023-09-23T20:21:49.958727+0000","last_active":"2023-09-23T20:21:49.958727+0000","last_peered":"2023-09-23T20:21:49.958727+0000","last_clean":"2023-09-23T20:21:49.958727+0000","last_became_active":"2023-09-23T20:21:49.957722+0000","last_became_peered":"2023-09-23T20:21:49.957722+0000","last_unstale":"2023-09-23T20:21:49.958727+0000","last_undegraded":"2023-09-23T20:21:49.958727+0000","last_fullsized":"2023-09-23T20:21:49.958727+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:47:13.589014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":53,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.956577+0000","last_change":"2023-09-23T20:21:49.954419+0000","last_active":"2023-09-23T20:21:49.956577+0000","last_peered":"2023-09-23T20:21:49.956577+0000","last_clean":"2023-09-23T20:21:49.956577+0000","last_became_active":"2023-09-23T20:21:49.953941+0000","last_became_peered":"2023-09-23T20:21:49.953941+0000","last_unstale":"2023-09-23T20:21:49.956577+0000","last_undegraded":"2023-09-23T20:21:49.956577+0000","last_fullsized":"2023-09-23T20:21:49.956577+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:05:38.809180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":47,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.955411+0000","last_change":"2023-09-23T20:21:49.955411+0000","last_active":"2023-09-23T20:21:49.955411+0000","last_peered":"2023-09-23T20:21:49.955411+0000","last_clean":"2023-09-23T20:21:49.955411+0000","last_became_active":"2023-09-23T20:21:49.955112+0000","last_became_peered":"2023-09-23T20:21:49.955112+0000","last_unstale":"2023-09-23T20:21:49.955411+0000","last_undegraded":"2023-09-23T20:21:49.955411+0000","last_fullsized":"2023-09-23T20:21:49.955411+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:47:53.870056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":72,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.951483+0000","last_change":"2023-09-23T20:21:49.951483+0000","last_active":"2023-09-23T20:21:49.951483+0000","last_peered":"2023-09-23T20:21:49.951483+0000","last_clean":"2023-09-23T20:21:49.951483+0000","last_became_active":"2023-09-23T20:21:49.951167+0000","last_became_peered":"2023-09-23T20:21:49.951167+0000","last_unstale":"2023-09-23T20:21:49.951483+0000","last_undegraded":"2023-09-23T20:21:49.951483+0000","last_fullsized":"2023-09-23T20:21:49.951483+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:08:19.552997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"69'1","reported_seq":88,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.942866+0000","last_change":"2023-09-23T20:21:18.891671+0000","last_active":"2023-09-23T20:21:48.942866+0000","last_peered":"2023-09-23T20:21:48.942866+0000","last_clean":"2023-09-23T20:21:48.942866+0000","last_became_active":"2023-09-23T20:21:18.891285+0000","last_became_peered":"2023-09-23T20:21:18.891285+0000","last_unstale":"2023-09-23T20:21:48.942866+0000","last_undegraded":"2023-09-23T20:21:48.942866+0000","last_fullsized":"2023-09-23T20:21:48.942866+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:04:49.595473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":25181,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6082,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201417,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":16944,"kb_used_data":5800,"kb_used_omap":0,"kb_used_meta":11136,"kb_avail":93728208,"statfs":{"total":95995035648,"available":95977684992,"internally_reserved":0,"allocated":5939200,"data_stored":5402099,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":11403264},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:22:17 2023","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.56499999999999995,"15min":0.56999999999999995},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":0.74199999999999999,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":1.5049999999999999},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.56200000000000006,"15min":0.56599999999999995},"min":{"1min":0.40600000000000003,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.68300000000000005,"5min":0.93799999999999994,"15min":0.93799999999999994},"last":1.5209999999999999}]},{"osd":1,"last update":"Sat Sep 23 20:21:48 2023","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":0.74399999999999999,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":1.399},{"interface":"front","average":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"min":{"1min":0.443,"5min":0.443,"15min":0.443},"max":{"1min":0.88800000000000001,"5min":0.88800000000000001,"15min":0.88800000000000001},"last":1.5569999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:22:19 2023","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.82599999999999996,"5min":0.82599999999999996,"15min":0.82599999999999996},"last":1.448},{"interface":"front","average":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.85999999999999999,"5min":0.85999999999999999,"15min":0.85999999999999999},"last":1.4670000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:20:44 2023","interfaces":[{"interface":"back","average":{"1min":0.59599999999999997,"5min":0.61199999999999999,"15min":0.57299999999999995},"min":{"1min":0.36699999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.72099999999999997,"5min":1.988,"15min":1.988},"last":1.5369999999999999},{"interface":"front","average":{"1min":0.623,"5min":0.63,"15min":0.61599999999999999},"min":{"1min":0.44,"5min":0.40899999999999997,"15min":0.36199999999999999},"max":{"1min":0.84399999999999997,"5min":0.872,"15min":1.127},"last":1.492}]},{"osd":4,"last update":"Sat Sep 23 20:21:48 2023","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.50600000000000001,"15min":0.50600000000000001},"min":{"1min":0.30199999999999999,"5min":0.28799999999999998,"15min":0.26700000000000002},"max":{"1min":0.90100000000000002,"5min":0.90100000000000002,"15min":1.274},"last":0.51100000000000001},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.54300000000000004,"15min":0.53400000000000003},"min":{"1min":0.35799999999999998,"5min":0.33100000000000002,"15min":0.29699999999999999},"max":{"1min":0.80000000000000004,"5min":0.80000000000000004,"15min":2.024},"last":0.53400000000000003}]},{"osd":5,"last update":"Sat Sep 23 20:21:48 2023","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":0.54200000000000004,"15min":0.54100000000000004},"min":{"1min":0.36599999999999999,"5min":0.33500000000000002,"15min":0.32000000000000001},"max":{"1min":0.79300000000000004,"5min":0.79600000000000004,"15min":1.337},"last":0.495},{"interface":"front","average":{"1min":0.629,"5min":0.60099999999999998,"15min":0.58799999999999997},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.31},"max":{"1min":0.89100000000000001,"5min":0.89100000000000001,"15min":1.5660000000000001},"last":0.60299999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:21:48 2023","interfaces":[{"interface":"back","average":{"1min":0.65300000000000002,"5min":0.61899999999999999,"15min":0.59399999999999997},"min":{"1min":0.379,"5min":0.379,"15min":0.34499999999999997},"max":{"1min":0.98699999999999999,"5min":0.98699999999999999,"15min":1.246},"last":0.56200000000000006},{"interface":"front","average":{"1min":0.60799999999999998,"5min":0.60999999999999999,"15min":0.58399999999999996},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.34799999999999998},"max":{"1min":0.873,"5min":0.873,"15min":2.048},"last":0.58799999999999997}]}]},{"osd":6,"up_from":47,"seq":201863463053,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":12392,"kb_used_data":5792,"kb_used_omap":0,"kb_used_meta":6592,"kb_avail":93732760,"statfs":{"total":95995035648,"available":95982346240,"internally_reserved":0,"allocated":5931008,"data_stored":5401972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6750208},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:22:18 2023","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":0.57799999999999996,"15min":0.57799999999999996},"min":{"1min":0.36799999999999999,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.628},{"interface":"front","average":{"1min":0.57499999999999996,"5min":0.56499999999999995,"15min":0.56299999999999994},"min":{"1min":0.40500000000000003,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.76600000000000001,"5min":0.82299999999999995,"15min":0.82299999999999995},"last":0.79500000000000004}]},{"osd":1,"last update":"Sat Sep 23 20:21:48 2023","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.76300000000000001,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.59099999999999997},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.77500000000000002,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.65500000000000003}]},{"osd":2,"last update":"Sat Sep 23 20:22:18 2023","interfaces":[{"interface":"back","average":{"1min":0.59499999999999997,"5min":0.59499999999999997,"15min":0.59499999999999997},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.75,"5min":0.75,"15min":0.75},"last":0.60999999999999999},{"interface":"front","average":{"1min":0.60899999999999999,"5min":0.60899999999999999,"15min":0.60899999999999999},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":0.79300000000000004,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.73599999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:21:34 2023","interfaces":[{"interface":"back","average":{"1min":0.63100000000000001,"5min":0.60299999999999998,"15min":0.56100000000000005},"min":{"1min":0.46300000000000002,"5min":0.314,"15min":0.314},"max":{"1min":0.79300000000000004,"5min":0.81799999999999995,"15min":0.86599999999999999},"last":0.64100000000000001},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.59099999999999997,"15min":0.57999999999999996},"min":{"1min":0.40699999999999997,"5min":0.35299999999999998,"15min":0.34999999999999998},"max":{"1min":0.79200000000000004,"5min":0.84099999999999997,"15min":2.0670000000000002},"last":0.67200000000000004}]},{"osd":4,"last update":"Sat Sep 23 20:21:34 2023","interfaces":[{"interface":"back","average":{"1min":0.46700000000000003,"5min":0.46899999999999997,"15min":0.46300000000000002},"min":{"1min":0.38200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.63400000000000001,"5min":0.71799999999999997,"15min":0.72299999999999998},"last":0.56899999999999995},{"interface":"front","average":{"1min":0.52400000000000002,"5min":0.51300000000000001,"15min":0.49299999999999999},"min":{"1min":0.30099999999999999,"5min":0.28599999999999998,"15min":0.28499999999999998},"max":{"1min":0.72499999999999998,"5min":0.83899999999999997,"15min":0.83899999999999997},"last":0.72199999999999998}]},{"osd":5,"last update":"Sat Sep 23 20:21:34 2023","interfaces":[{"interface":"back","average":{"1min":0.497,"5min":0.51700000000000002,"15min":0.49399999999999999},"min":{"1min":0.32600000000000001,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.67100000000000004,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.68799999999999994},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.55500000000000005,"15min":0.52100000000000002},"min":{"1min":0.34300000000000003,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.77900000000000003,"5min":0.874,"15min":0.874},"last":0.76000000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:21:48 2023","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.57499999999999996,"15min":0.54300000000000004},"min":{"1min":0.372,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.85199999999999998,"5min":0.85199999999999998,"15min":0.85199999999999998},"last":0.78000000000000003},{"interface":"front","average":{"1min":0.60199999999999998,"5min":0.60099999999999998,"15min":0.56899999999999995},"min":{"1min":0.38200000000000001,"5min":0.33500000000000002,"15min":0.315},"max":{"1min":0.83099999999999996,"5min":0.90800000000000003,"15min":0.90800000000000003},"last":0.82599999999999996}]}]},{"osd":1,"up_from":99,"seq":425201762328,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28744,"kb_used_data":1152,"kb_used_omap":6,"kb_used_meta":27577,"kb_avail":93716408,"statfs":{"total":95995035648,"available":95965601792,"internally_reserved":0,"allocated":1179648,"data_stored":623002,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6696,"internal_metadata":28239320},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:21:46 2023","interfaces":[{"interface":"back","average":{"1min":0.377,"5min":0.377,"15min":0.377},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.50700000000000001,"5min":0.50700000000000001,"15min":0.50700000000000001},"last":0.26200000000000001},{"interface":"front","average":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"last":0.34000000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:22:18 2023","interfaces":[{"interface":"back","average":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.628,"5min":0.628,"15min":0.628},"last":0.35199999999999998},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":3.226,"5min":3.226,"15min":3.226},"last":0.309}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.40300000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:21:46 2023","interfaces":[{"interface":"back","average":{"1min":0.61699999999999999,"5min":0.61699999999999999,"15min":0.61699999999999999},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.89100000000000001,"5min":0.89100000000000001,"15min":0.89100000000000001},"last":0.84899999999999998},{"interface":"front","average":{"1min":0.63500000000000001,"5min":0.63500000000000001,"15min":0.63500000000000001},"min":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":0.85199999999999998,"5min":0.85199999999999998,"15min":0.85199999999999998},"last":0.60199999999999998}]},{"osd":5,"last update":"Sat Sep 23 20:21:46 2023","interfaces":[{"interface":"back","average":{"1min":0.64100000000000001,"5min":0.64100000000000001,"15min":0.64100000000000001},"min":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.76200000000000001},{"interface":"front","average":{"1min":0.67600000000000005,"5min":0.67600000000000005,"15min":0.67600000000000005},"min":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"max":{"1min":0.86899999999999999,"5min":0.86899999999999999,"15min":0.86899999999999999},"last":0.755}]},{"osd":6,"last update":"Sat Sep 23 20:21:46 2023","interfaces":[{"interface":"back","average":{"1min":0.66800000000000004,"5min":0.66800000000000004,"15min":0.66800000000000004},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.92200000000000004,"5min":0.92200000000000004,"15min":0.92200000000000004},"last":0.80700000000000005},{"interface":"front","average":{"1min":0.63800000000000001,"5min":0.63800000000000001,"15min":0.63800000000000001},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":0.86599999999999999,"5min":0.86599999999999999,"15min":0.86599999999999999},"last":0.64000000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:21:46 2023","interfaces":[{"interface":"back","average":{"1min":0.65200000000000002,"5min":0.65200000000000002,"15min":0.65200000000000002},"min":{"1min":0.47599999999999998,"5min":0.47599999999999998,"15min":0.47599999999999998},"max":{"1min":0.91200000000000003,"5min":0.91200000000000003,"15min":0.91200000000000003},"last":0.61699999999999999},{"interface":"front","average":{"1min":0.68400000000000005,"5min":0.68400000000000005,"15min":0.68400000000000005},"min":{"1min":0.45300000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"max":{"1min":0.85899999999999999,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.746}]}]},{"osd":0,"up_from":94,"seq":403726925854,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":29340,"kb_used_data":1748,"kb_used_omap":7,"kb_used_meta":27576,"kb_avail":93715812,"statfs":{"total":95995035648,"available":95964991488,"internally_reserved":0,"allocated":1789952,"data_stored":1215718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":28237841},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:21:48 2023","interfaces":[{"interface":"back","average":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"last":0.35599999999999998},{"interface":"front","average":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":0.52700000000000002,"5min":0.52700000000000002,"15min":0.52700000000000002},"last":0.438}]},{"osd":2,"last update":"Sat Sep 23 20:22:18 2023","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":4.6280000000000001,"5min":4.6280000000000001,"15min":4.6280000000000001},"last":0.45200000000000001},{"interface":"front","average":{"1min":0.496,"5min":0.496,"15min":0.496},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":1.921,"5min":1.921,"15min":1.921},"last":0.40000000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:21:15 2023","interfaces":[{"interface":"back","average":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"last":0.38600000000000001},{"interface":"front","average":{"1min":0.435,"5min":0.435,"15min":0.435},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"last":0.41499999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:22:18 2023","interfaces":[{"interface":"back","average":{"1min":0.78800000000000003,"5min":0.63600000000000001,"15min":0.60999999999999999},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":4.7450000000000001,"5min":4.7450000000000001,"15min":4.7450000000000001},"last":0.59999999999999998},{"interface":"front","average":{"1min":0.60799999999999998,"5min":0.61199999999999999,"15min":0.61299999999999999},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.92100000000000004,"5min":0.92100000000000004,"15min":0.92100000000000004},"last":0.65200000000000002}]},{"osd":5,"last update":"Sat Sep 23 20:22:18 2023","interfaces":[{"interface":"back","average":{"1min":0.64500000000000002,"5min":0.65600000000000003,"15min":0.65800000000000003},"min":{"1min":0.47899999999999998,"5min":0.47899999999999998,"15min":0.47899999999999998},"max":{"1min":0.90400000000000003,"5min":0.90400000000000003,"15min":0.90400000000000003},"last":0.61099999999999999},{"interface":"front","average":{"1min":0.67100000000000004,"5min":0.67700000000000005,"15min":0.67800000000000005},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.90600000000000003,"5min":0.93899999999999995,"15min":0.93899999999999995},"last":0.66600000000000004}]},{"osd":6,"last update":"Sat Sep 23 20:22:18 2023","interfaces":[{"interface":"back","average":{"1min":0.68799999999999994,"5min":0.65600000000000003,"15min":0.65100000000000002},"min":{"1min":0.495,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":0.97899999999999998,"5min":0.97899999999999998,"15min":0.97899999999999998},"last":0.55000000000000004},{"interface":"front","average":{"1min":0.68799999999999994,"5min":0.66800000000000004,"15min":0.66400000000000003},"min":{"1min":0.45300000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"max":{"1min":0.89900000000000002,"5min":0.95999999999999996,"15min":0.95999999999999996},"last":0.63600000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:22:18 2023","interfaces":[{"interface":"back","average":{"1min":0.66400000000000003,"5min":0.67600000000000005,"15min":0.67800000000000005},"min":{"1min":0.47199999999999998,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":0.96399999999999997,"5min":0.96399999999999997,"15min":0.96399999999999997},"last":0.68500000000000005},{"interface":"front","average":{"1min":0.86399999999999999,"5min":0.70399999999999996,"15min":0.67800000000000005},"min":{"1min":0.52200000000000002,"5min":0.48499999999999999,"15min":0.48499999999999999},"max":{"1min":4.7140000000000004,"5min":4.7140000000000004,"15min":4.7140000000000004},"last":0.64700000000000002}]}]},{"osd":2,"up_from":104,"seq":446676598801,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32768,"kb_used_data":5240,"kb_used_omap":11,"kb_used_meta":27508,"kb_avail":93712384,"statfs":{"total":95995035648,"available":95961481216,"internally_reserved":0,"allocated":5365760,"data_stored":4815759,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11878,"internal_metadata":28168602},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:22:20 2023","interfaces":[{"interface":"back","average":{"1min":0.377,"5min":0.377,"15min":0.377},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":0.60599999999999998,"5min":0.60599999999999998,"15min":0.60599999999999998},"last":0.48399999999999999},{"interface":"front","average":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.69899999999999995,"5min":0.69899999999999995,"15min":0.69899999999999995},"last":0.51500000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:22:20 2023","interfaces":[{"interface":"back","average":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.68000000000000005,"5min":0.68000000000000005,"15min":0.68000000000000005},"last":0.52600000000000002},{"interface":"front","average":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"last":0.46800000000000003}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.45000000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:22:20 2023","interfaces":[{"interface":"back","average":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.61099999999999999},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.74099999999999999,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.71499999999999997}]},{"osd":5,"last update":"Sat Sep 23 20:22:20 2023","interfaces":[{"interface":"back","average":{"1min":0.65700000000000003,"5min":0.65700000000000003,"15min":0.65700000000000003},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.89100000000000001,"5min":0.89100000000000001,"15min":0.89100000000000001},"last":0.58599999999999997},{"interface":"front","average":{"1min":0.65700000000000003,"5min":0.65700000000000003,"15min":0.65700000000000003},"min":{"1min":0.46600000000000003,"5min":0.46600000000000003,"15min":0.46600000000000003},"max":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.74099999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:22:20 2023","interfaces":[{"interface":"back","average":{"1min":0.65700000000000003,"5min":0.65700000000000003,"15min":0.65700000000000003},"min":{"1min":0.501,"5min":0.501,"15min":0.501},"max":{"1min":0.94299999999999995,"5min":0.94299999999999995,"15min":0.94299999999999995},"last":0.68200000000000005},{"interface":"front","average":{"1min":0.67300000000000004,"5min":0.67300000000000004,"15min":0.67300000000000004},"min":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":1,"5min":1,"15min":1},"last":0.59899999999999998}]},{"osd":7,"last update":"Sat Sep 23 20:22:20 2023","interfaces":[{"interface":"back","average":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"min":{"1min":0.48399999999999999,"5min":0.48399999999999999,"15min":0.48399999999999999},"max":{"1min":0.96099999999999997,"5min":0.96099999999999997,"15min":0.96099999999999997},"last":0.749},{"interface":"front","average":{"1min":0.65700000000000003,"5min":0.65700000000000003,"15min":0.65700000000000003},"min":{"1min":0.49299999999999999,"5min":0.49299999999999999,"15min":0.49299999999999999},"max":{"1min":0.92500000000000004,"5min":0.92500000000000004,"15min":0.92500000000000004},"last":0.69399999999999995}]}]},{"osd":3,"up_from":109,"seq":468151435275,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28648,"kb_used_data":1120,"kb_used_omap":4,"kb_used_meta":27515,"kb_avail":93716504,"statfs":{"total":95995035648,"available":95965700096,"internally_reserved":0,"allocated":1146880,"data_stored":617010,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4514,"internal_metadata":28175966},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.32000000000000001}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.30099999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.57899999999999996}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.60999999999999999}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.65800000000000003}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.622}]}]},{"osd":4,"up_from":30,"seq":128849019029,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":7780,"kb_used_data":1116,"kb_used_omap":0,"kb_used_meta":6656,"kb_avail":93737372,"statfs":{"total":95995035648,"available":95987068928,"internally_reserved":0,"allocated":1142784,"data_stored":617034,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6815744},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:22:16 2023","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":0.57599999999999996,"15min":0.57099999999999995},"min":{"1min":0.41099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.85099999999999998,"5min":0.85099999999999998,"15min":0.85099999999999998},"last":0.54200000000000004},{"interface":"front","average":{"1min":0.63500000000000001,"5min":0.64000000000000001,"15min":0.64100000000000001},"min":{"1min":0.47299999999999998,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":0.79500000000000004,"5min":0.84799999999999998,"15min":0.84799999999999998},"last":0.5}]},{"osd":1,"last update":"Sat Sep 23 20:21:49 2023","interfaces":[{"interface":"back","average":{"1min":0.626,"5min":0.626,"15min":0.626},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":1.4219999999999999,"5min":1.4219999999999999,"15min":1.4219999999999999},"last":0.46400000000000002},{"interface":"front","average":{"1min":0.626,"5min":0.626,"15min":0.626},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":1.393,"5min":1.393,"15min":1.393},"last":0.434}]},{"osd":2,"last update":"Sat Sep 23 20:22:19 2023","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"min":{"1min":0.49299999999999999,"5min":0.49299999999999999,"15min":0.49299999999999999},"max":{"1min":0.78400000000000003,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.60199999999999998},{"interface":"front","average":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":0.73599999999999999,"5min":0.73599999999999999,"15min":0.73599999999999999},"last":0.58999999999999997}]},{"osd":3,"last update":"Sat Sep 23 20:21:40 2023","interfaces":[{"interface":"back","average":{"1min":0.63400000000000001,"5min":0.64700000000000002,"15min":0.58899999999999997},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":1.4079999999999999,"5min":3.0800000000000001,"15min":3.0800000000000001},"last":0.66800000000000004},{"interface":"front","average":{"1min":0.67100000000000004,"5min":0.64900000000000002,"15min":0.58699999999999997},"min":{"1min":0.435,"5min":0.373,"15min":0.34699999999999998},"max":{"1min":1.4630000000000001,"5min":3.6190000000000002,"15min":3.6190000000000002},"last":0.71599999999999997}]},{"osd":5,"last update":"Sat Sep 23 20:22:05 2023","interfaces":[{"interface":"back","average":{"1min":0.48799999999999999,"5min":0.50600000000000001,"15min":0.45700000000000002},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.26000000000000001},"max":{"1min":0.66100000000000003,"5min":3.5270000000000001,"15min":3.5270000000000001},"last":0.623},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.52900000000000003,"15min":0.46700000000000003},"min":{"1min":0.317,"5min":0.317,"15min":0.248},"max":{"1min":0.81299999999999994,"5min":0.86299999999999999,"15min":0.86299999999999999},"last":0.64400000000000002}]},{"osd":6,"last update":"Sat Sep 23 20:22:27 2023","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":0.54600000000000004,"15min":0.497},"min":{"1min":0.46999999999999997,"5min":0.35599999999999998,"15min":0.27500000000000002},"max":{"1min":0.76000000000000001,"5min":0.77900000000000003,"15min":0.82499999999999996},"last":0.69999999999999996},{"interface":"front","average":{"1min":0.56999999999999995,"5min":0.56599999999999995,"15min":0.51000000000000001},"min":{"1min":0.41499999999999998,"5min":0.32100000000000001,"15min":0.314},"max":{"1min":0.73299999999999998,"5min":3.4820000000000002,"15min":3.4820000000000002},"last":0.73199999999999998}]},{"osd":7,"last update":"Sat Sep 23 20:21:40 2023","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":0.57199999999999995,"15min":0.54700000000000004},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.33600000000000002},"max":{"1min":0.875,"5min":0.91600000000000004,"15min":0.91600000000000004},"last":0.76200000000000001},{"interface":"front","average":{"1min":0.57999999999999996,"5min":0.61199999999999999,"15min":0.55600000000000005},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.29899999999999999},"max":{"1min":0.80800000000000005,"5min":3.77,"15min":3.77},"last":0.80200000000000005}]}]},{"osd":5,"up_from":39,"seq":167503724690,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":7516,"kb_used_data":1108,"kb_used_omap":0,"kb_used_meta":6400,"kb_avail":93737636,"statfs":{"total":95995035648,"available":95987339264,"internally_reserved":0,"allocated":1134592,"data_stored":613120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6553600},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:22:18 2023","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":0.59599999999999997,"15min":0.59299999999999997},"min":{"1min":0.38900000000000001,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.79800000000000004,"5min":0.83499999999999996,"15min":0.83499999999999996},"last":0.625},{"interface":"front","average":{"1min":0.57999999999999996,"5min":0.59599999999999997,"15min":0.59899999999999998},"min":{"1min":0.44,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.82299999999999995,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.56000000000000005}]},{"osd":1,"last update":"Sat Sep 23 20:21:47 2023","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.57399999999999995},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.84999999999999998,"5min":0.84999999999999998,"15min":0.84999999999999998},"last":0.51600000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:22:18 2023","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.70899999999999996,"5min":0.70899999999999996,"15min":0.70899999999999996},"last":0.65000000000000002},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.72899999999999998,"5min":0.72899999999999998,"15min":0.72899999999999998},"last":0.46800000000000003}]},{"osd":3,"last update":"Sat Sep 23 20:21:11 2023","interfaces":[{"interface":"back","average":{"1min":0.60299999999999998,"5min":0.60699999999999998,"15min":0.59099999999999997},"min":{"1min":0.47499999999999998,"5min":0.35299999999999998,"15min":0.34100000000000003},"max":{"1min":0.73199999999999998,"5min":0.93400000000000005,"15min":2.7570000000000001},"last":0.48599999999999999},{"interface":"front","average":{"1min":0.61099999999999999,"5min":0.63,"15min":0.57399999999999995},"min":{"1min":0.40699999999999997,"5min":0.40600000000000003,"15min":0.35599999999999998},"max":{"1min":0.74099999999999999,"5min":0.88200000000000001,"15min":2.2599999999999998},"last":0.60099999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:22:13 2023","interfaces":[{"interface":"back","average":{"1min":0.498,"5min":0.48199999999999998,"15min":0.439},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.26400000000000001},"max":{"1min":0.68200000000000005,"5min":0.76100000000000001,"15min":1.0620000000000001},"last":0.45100000000000001},{"interface":"front","average":{"1min":0.58799999999999997,"5min":0.52700000000000002,"15min":0.47999999999999998},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.28799999999999998},"max":{"1min":0.82299999999999995,"5min":0.82299999999999995,"15min":1.4450000000000001},"last":0.499}]},{"osd":6,"last update":"Sat Sep 23 20:22:27 2023","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":0.52000000000000002,"15min":0.49399999999999999},"min":{"1min":0.44500000000000001,"5min":0.32500000000000001,"15min":0.29799999999999999},"max":{"1min":0.80000000000000004,"5min":0.80000000000000004,"15min":1.151},"last":0.54300000000000004},{"interface":"front","average":{"1min":0.65700000000000003,"5min":0.61399999999999999,"15min":0.56299999999999994},"min":{"1min":0.375,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.84899999999999998,"5min":0.84899999999999998,"15min":0.84899999999999998},"last":0.53300000000000003}]},{"osd":7,"last update":"Sat Sep 23 20:21:44 2023","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.56599999999999995,"15min":0.53800000000000003},"min":{"1min":0.41799999999999998,"5min":0.372,"15min":0.33800000000000002},"max":{"1min":0.81999999999999995,"5min":0.92500000000000004,"15min":0.98199999999999998},"last":0.42499999999999999},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.57799999999999996,"15min":0.54500000000000004},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.79200000000000004,"5min":0.79200000000000004,"15min":1.0489999999999999},"last":0.38700000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4722,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1360,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6696,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7156,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3154,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:22:36.748 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:22:37.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:22:36 smithi018 ceph-mon[184144]: pgmap v221: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:22:37.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:22:36 smithi018 ceph-mon[179647]: pgmap v221: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:22:37.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:22:36 smithi149 ceph-mon[170394]: pgmap v221: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:22:37.354 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:22:38.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:22:37 smithi149 ceph-mon[170394]: from='client.44624 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:22:38.386 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:22:37 smithi018 ceph-mon[179647]: from='client.44624 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:22:38.386 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:22:37 smithi018 ceph-mon[184144]: from='client.44624 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:22:38.791 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:22:38.793 INFO:teuthology.orchestra.run.smithi018.stderr:dumped all 2023-09-23T20:22:39.095 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:22:38 smithi018 ceph-mon[184144]: pgmap v222: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:22:39.095 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:22:38 smithi018 ceph-mon[179647]: pgmap v222: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:22:39.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:22:38 smithi149 ceph-mon[170394]: pgmap v222: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:22:39.313 INFO:teuthology.orchestra.run.smithi018.stdout:{"pg_ready":true,"pg_map":{"version":222,"stamp":"2023-09-23T20:22:37.431011+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790195,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6495,"num_read_kb":14136,"num_write":1146,"num_write_kb":15141,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1192,"ondisk_log_size":1192,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":164132,"kb_used_data":23076,"kb_used_omap":30,"kb_used_meta":140961,"kb_avail":749797084,"statfs":{"total":767960285184,"available":767792214016,"internally_reserved":0,"allocated":23629824,"data_stored":19305714,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31263,"internal_metadata":144344545},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003451"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":79,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.941990+0000","last_change":"2023-09-23T20:21:18.893413+0000","last_active":"2023-09-23T20:21:48.941990+0000","last_peered":"2023-09-23T20:21:48.941990+0000","last_clean":"2023-09-23T20:21:48.941990+0000","last_became_active":"2023-09-23T20:21:18.893008+0000","last_became_peered":"2023-09-23T20:21:18.893008+0000","last_unstale":"2023-09-23T20:21:48.941990+0000","last_undegraded":"2023-09-23T20:21:48.941990+0000","last_fullsized":"2023-09-23T20:21:48.941990+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:50:55.264882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":56,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.938999+0000","last_change":"2023-09-23T20:21:18.889521+0000","last_active":"2023-09-23T20:21:49.938999+0000","last_peered":"2023-09-23T20:21:49.938999+0000","last_clean":"2023-09-23T20:21:49.938999+0000","last_became_active":"2023-09-23T20:21:18.889268+0000","last_became_peered":"2023-09-23T20:21:18.889268+0000","last_unstale":"2023-09-23T20:21:49.938999+0000","last_undegraded":"2023-09-23T20:21:49.938999+0000","last_fullsized":"2023-09-23T20:21:49.938999+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:05:48.805778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":93,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.949944+0000","last_change":"2023-09-23T20:21:49.949944+0000","last_active":"2023-09-23T20:21:49.949944+0000","last_peered":"2023-09-23T20:21:49.949944+0000","last_clean":"2023-09-23T20:21:49.949944+0000","last_became_active":"2023-09-23T20:21:49.949425+0000","last_became_peered":"2023-09-23T20:21:49.949425+0000","last_unstale":"2023-09-23T20:21:49.949944+0000","last_undegraded":"2023-09-23T20:21:49.949944+0000","last_fullsized":"2023-09-23T20:21:49.949944+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:40:56.395791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":78,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.957394+0000","last_change":"2023-09-23T20:21:49.957394+0000","last_active":"2023-09-23T20:21:49.957394+0000","last_peered":"2023-09-23T20:21:49.957394+0000","last_clean":"2023-09-23T20:21:49.957394+0000","last_became_active":"2023-09-23T20:21:49.957016+0000","last_became_peered":"2023-09-23T20:21:49.957016+0000","last_unstale":"2023-09-23T20:21:49.957394+0000","last_undegraded":"2023-09-23T20:21:49.957394+0000","last_fullsized":"2023-09-23T20:21:49.957394+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:49:26.222448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":86,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.953041+0000","last_change":"2023-09-23T20:21:49.953041+0000","last_active":"2023-09-23T20:21:49.953041+0000","last_peered":"2023-09-23T20:21:49.953041+0000","last_clean":"2023-09-23T20:21:49.953041+0000","last_became_active":"2023-09-23T20:21:49.952763+0000","last_became_peered":"2023-09-23T20:21:49.952763+0000","last_unstale":"2023-09-23T20:21:49.953041+0000","last_undegraded":"2023-09-23T20:21:49.953041+0000","last_fullsized":"2023-09-23T20:21:49.953041+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:58:35.321466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":54,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.950333+0000","last_change":"2023-09-23T20:12:43.753383+0000","last_active":"2023-09-23T20:21:48.950333+0000","last_peered":"2023-09-23T20:21:48.950333+0000","last_clean":"2023-09-23T20:21:48.950333+0000","last_became_active":"2023-09-23T20:12:43.752921+0000","last_became_peered":"2023-09-23T20:12:43.752921+0000","last_unstale":"2023-09-23T20:21:48.950333+0000","last_undegraded":"2023-09-23T20:21:48.950333+0000","last_fullsized":"2023-09-23T20:21:48.950333+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:28:26.982998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":86,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.957885+0000","last_change":"2023-09-23T20:21:49.957885+0000","last_active":"2023-09-23T20:21:49.957885+0000","last_peered":"2023-09-23T20:21:49.957885+0000","last_clean":"2023-09-23T20:21:49.957885+0000","last_became_active":"2023-09-23T20:21:49.957492+0000","last_became_peered":"2023-09-23T20:21:49.957492+0000","last_unstale":"2023-09-23T20:21:49.957885+0000","last_undegraded":"2023-09-23T20:21:49.957885+0000","last_fullsized":"2023-09-23T20:21:49.957885+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:24:55.137836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":71,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.949471+0000","last_change":"2023-09-23T20:20:46.938981+0000","last_active":"2023-09-23T20:21:48.949471+0000","last_peered":"2023-09-23T20:21:48.949471+0000","last_clean":"2023-09-23T20:21:48.949471+0000","last_became_active":"2023-09-23T20:20:46.938627+0000","last_became_peered":"2023-09-23T20:20:46.938627+0000","last_unstale":"2023-09-23T20:21:48.949471+0000","last_undegraded":"2023-09-23T20:21:48.949471+0000","last_fullsized":"2023-09-23T20:21:48.949471+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:44:43.177086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":90,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.953299+0000","last_change":"2023-09-23T20:21:49.953299+0000","last_active":"2023-09-23T20:21:49.953299+0000","last_peered":"2023-09-23T20:21:49.953299+0000","last_clean":"2023-09-23T20:21:49.953299+0000","last_became_active":"2023-09-23T20:21:49.952996+0000","last_became_peered":"2023-09-23T20:21:49.952996+0000","last_unstale":"2023-09-23T20:21:49.953299+0000","last_undegraded":"2023-09-23T20:21:49.953299+0000","last_fullsized":"2023-09-23T20:21:49.953299+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:46:51.746281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":40,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.950883+0000","last_change":"2023-09-23T20:20:14.888484+0000","last_active":"2023-09-23T20:21:48.950883+0000","last_peered":"2023-09-23T20:21:48.950883+0000","last_clean":"2023-09-23T20:21:48.950883+0000","last_became_active":"2023-09-23T20:20:14.887219+0000","last_became_peered":"2023-09-23T20:20:14.887219+0000","last_unstale":"2023-09-23T20:21:48.950883+0000","last_undegraded":"2023-09-23T20:21:48.950883+0000","last_fullsized":"2023-09-23T20:21:48.950883+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:13:53.763649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":52,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.943196+0000","last_change":"2023-09-23T20:12:43.765908+0000","last_active":"2023-09-23T20:21:48.943196+0000","last_peered":"2023-09-23T20:21:48.943196+0000","last_clean":"2023-09-23T20:21:48.943196+0000","last_became_active":"2023-09-23T20:12:43.765558+0000","last_became_peered":"2023-09-23T20:12:43.765558+0000","last_unstale":"2023-09-23T20:21:48.943196+0000","last_undegraded":"2023-09-23T20:21:48.943196+0000","last_fullsized":"2023-09-23T20:21:48.943196+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:35:43.809898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":58,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.939966+0000","last_change":"2023-09-23T20:20:46.937163+0000","last_active":"2023-09-23T20:21:49.939966+0000","last_peered":"2023-09-23T20:21:49.939966+0000","last_clean":"2023-09-23T20:21:49.939966+0000","last_became_active":"2023-09-23T20:20:46.936846+0000","last_became_peered":"2023-09-23T20:20:46.936846+0000","last_unstale":"2023-09-23T20:21:49.939966+0000","last_undegraded":"2023-09-23T20:21:49.939966+0000","last_fullsized":"2023-09-23T20:21:49.939966+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:01:32.798855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"86'117","reported_seq":596,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.945893+0000","last_change":"2023-09-23T20:21:18.897269+0000","last_active":"2023-09-23T20:21:48.945893+0000","last_peered":"2023-09-23T20:21:48.945893+0000","last_clean":"2023-09-23T20:21:48.945893+0000","last_became_active":"2023-09-23T20:21:18.896834+0000","last_became_peered":"2023-09-23T20:21:18.896834+0000","last_unstale":"2023-09-23T20:21:48.945893+0000","last_undegraded":"2023-09-23T20:21:48.945893+0000","last_fullsized":"2023-09-23T20:21:48.945893+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:57:17.499025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046891400000000002,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":893,"num_read_kb":922,"num_write":82,"num_write_kb":4193,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":67,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.953565+0000","last_change":"2023-09-23T20:21:49.953565+0000","last_active":"2023-09-23T20:21:49.953565+0000","last_peered":"2023-09-23T20:21:49.953565+0000","last_clean":"2023-09-23T20:21:49.953565+0000","last_became_active":"2023-09-23T20:21:49.953176+0000","last_became_peered":"2023-09-23T20:21:49.953176+0000","last_unstale":"2023-09-23T20:21:49.953565+0000","last_undegraded":"2023-09-23T20:21:49.953565+0000","last_fullsized":"2023-09-23T20:21:49.953565+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:02:35.778207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"86'128","reported_seq":432,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.940390+0000","last_change":"2023-09-23T20:20:46.939256+0000","last_active":"2023-09-23T20:21:48.940390+0000","last_peered":"2023-09-23T20:21:48.940390+0000","last_clean":"2023-09-23T20:21:48.940390+0000","last_became_active":"2023-09-23T20:20:46.938756+0000","last_became_peered":"2023-09-23T20:20:46.938756+0000","last_unstale":"2023-09-23T20:21:48.940390+0000","last_undegraded":"2023-09-23T20:21:48.940390+0000","last_fullsized":"2023-09-23T20:21:48.940390+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:52:18.161436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000293202,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":579,"num_read_kb":1473,"num_write":96,"num_write_kb":1062,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":70,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.957535+0000","last_change":"2023-09-23T20:21:49.957535+0000","last_active":"2023-09-23T20:21:49.957535+0000","last_peered":"2023-09-23T20:21:49.957535+0000","last_clean":"2023-09-23T20:21:49.957535+0000","last_became_active":"2023-09-23T20:21:49.954319+0000","last_became_peered":"2023-09-23T20:21:49.954319+0000","last_unstale":"2023-09-23T20:21:49.957535+0000","last_undegraded":"2023-09-23T20:21:49.957535+0000","last_fullsized":"2023-09-23T20:21:49.957535+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:14:10.293305+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":38,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.938907+0000","last_change":"2023-09-23T20:20:14.888874+0000","last_active":"2023-09-23T20:21:49.938907+0000","last_peered":"2023-09-23T20:21:49.938907+0000","last_clean":"2023-09-23T20:21:49.938907+0000","last_became_active":"2023-09-23T20:20:14.887724+0000","last_became_peered":"2023-09-23T20:20:14.887724+0000","last_unstale":"2023-09-23T20:21:49.938907+0000","last_undegraded":"2023-09-23T20:21:49.938907+0000","last_fullsized":"2023-09-23T20:21:49.938907+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:09:24.510326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":47,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.939386+0000","last_change":"2023-09-23T20:21:18.893457+0000","last_active":"2023-09-23T20:21:48.939386+0000","last_peered":"2023-09-23T20:21:48.939386+0000","last_clean":"2023-09-23T20:21:48.939386+0000","last_became_active":"2023-09-23T20:21:18.893175+0000","last_became_peered":"2023-09-23T20:21:18.893175+0000","last_unstale":"2023-09-23T20:21:48.939386+0000","last_undegraded":"2023-09-23T20:21:48.939386+0000","last_fullsized":"2023-09-23T20:21:48.939386+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:24:37.563758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"86'102","reported_seq":616,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.951036+0000","last_change":"2023-09-23T20:20:14.881975+0000","last_active":"2023-09-23T20:21:48.951036+0000","last_peered":"2023-09-23T20:21:48.951036+0000","last_clean":"2023-09-23T20:21:48.951036+0000","last_became_active":"2023-09-23T20:20:14.881544+0000","last_became_peered":"2023-09-23T20:20:14.881544+0000","last_unstale":"2023-09-23T20:21:48.951036+0000","last_undegraded":"2023-09-23T20:21:48.951036+0000","last_fullsized":"2023-09-23T20:21:48.951036+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":102,"log_dups_size":0,"ondisk_log_size":102,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:34.594972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030656500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":963,"num_read_kb":1234,"num_write":60,"num_write_kb":225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":68,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.940049+0000","last_change":"2023-09-23T20:21:18.887651+0000","last_active":"2023-09-23T20:21:49.940049+0000","last_peered":"2023-09-23T20:21:49.940049+0000","last_clean":"2023-09-23T20:21:49.940049+0000","last_became_active":"2023-09-23T20:21:18.887338+0000","last_became_peered":"2023-09-23T20:21:18.887338+0000","last_unstale":"2023-09-23T20:21:49.940049+0000","last_undegraded":"2023-09-23T20:21:49.940049+0000","last_fullsized":"2023-09-23T20:21:49.940049+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:26:39.067022+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":52,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.952295+0000","last_change":"2023-09-23T20:21:49.951490+0000","last_active":"2023-09-23T20:21:49.952295+0000","last_peered":"2023-09-23T20:21:49.952295+0000","last_clean":"2023-09-23T20:21:49.952295+0000","last_became_active":"2023-09-23T20:21:49.951079+0000","last_became_peered":"2023-09-23T20:21:49.951079+0000","last_unstale":"2023-09-23T20:21:49.952295+0000","last_undegraded":"2023-09-23T20:21:49.952295+0000","last_fullsized":"2023-09-23T20:21:49.952295+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:12:01.053013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"86'80","reported_seq":356,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.952040+0000","last_change":"2023-09-23T20:20:46.950784+0000","last_active":"2023-09-23T20:21:48.952040+0000","last_peered":"2023-09-23T20:21:48.952040+0000","last_clean":"2023-09-23T20:21:48.952040+0000","last_became_active":"2023-09-23T20:20:46.950071+0000","last_became_peered":"2023-09-23T20:20:46.950071+0000","last_unstale":"2023-09-23T20:21:48.952040+0000","last_undegraded":"2023-09-23T20:21:48.952040+0000","last_fullsized":"2023-09-23T20:21:48.952040+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":80,"log_dups_size":0,"ondisk_log_size":80,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:11:06.812404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035115,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":395,"num_read_kb":1370,"num_write":66,"num_write_kb":1061,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":52,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.946301+0000","last_change":"2023-09-23T20:12:45.609829+0000","last_active":"2023-09-23T20:21:48.946301+0000","last_peered":"2023-09-23T20:21:48.946301+0000","last_clean":"2023-09-23T20:21:48.946301+0000","last_became_active":"2023-09-23T20:12:45.609467+0000","last_became_peered":"2023-09-23T20:12:45.609467+0000","last_unstale":"2023-09-23T20:21:48.946301+0000","last_undegraded":"2023-09-23T20:21:48.946301+0000","last_fullsized":"2023-09-23T20:21:48.946301+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:27:31.490555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":22,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.956277+0000","last_change":"2023-09-23T20:21:49.956277+0000","last_active":"2023-09-23T20:21:49.956277+0000","last_peered":"2023-09-23T20:21:49.956277+0000","last_clean":"2023-09-23T20:21:49.956277+0000","last_became_active":"2023-09-23T20:21:49.955974+0000","last_became_peered":"2023-09-23T20:21:49.955974+0000","last_unstale":"2023-09-23T20:21:49.956277+0000","last_undegraded":"2023-09-23T20:21:49.956277+0000","last_fullsized":"2023-09-23T20:21:49.956277+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:11:37.850675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":452,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.951949+0000","last_change":"2023-09-23T20:20:14.882204+0000","last_active":"2023-09-23T20:21:48.951949+0000","last_peered":"2023-09-23T20:21:48.951949+0000","last_clean":"2023-09-23T20:21:48.951949+0000","last_became_active":"2023-09-23T20:20:14.881699+0000","last_became_peered":"2023-09-23T20:20:14.881699+0000","last_unstale":"2023-09-23T20:21:48.951949+0000","last_undegraded":"2023-09-23T20:21:48.951949+0000","last_fullsized":"2023-09-23T20:21:48.951949+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:09:47.213130+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:16:37.341010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"86'194","reported_seq":728,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.948141+0000","last_change":"2023-09-23T20:21:18.892176+0000","last_active":"2023-09-23T20:21:48.948141+0000","last_peered":"2023-09-23T20:21:48.948141+0000","last_clean":"2023-09-23T20:21:48.948141+0000","last_became_active":"2023-09-23T20:21:18.891647+0000","last_became_peered":"2023-09-23T20:21:18.891647+0000","last_unstale":"2023-09-23T20:21:48.948141+0000","last_undegraded":"2023-09-23T20:21:48.948141+0000","last_fullsized":"2023-09-23T20:21:48.948141+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":194,"log_dups_size":0,"ondisk_log_size":194,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:25:52.557392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00037995999999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1025,"num_read_kb":1201,"num_write":173,"num_write_kb":1276,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":76,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.939413+0000","last_change":"2023-09-23T20:20:46.936524+0000","last_active":"2023-09-23T20:21:48.939413+0000","last_peered":"2023-09-23T20:21:48.939413+0000","last_clean":"2023-09-23T20:21:48.939413+0000","last_became_active":"2023-09-23T20:20:46.936267+0000","last_became_peered":"2023-09-23T20:20:46.936267+0000","last_unstale":"2023-09-23T20:21:48.939413+0000","last_undegraded":"2023-09-23T20:21:48.939413+0000","last_fullsized":"2023-09-23T20:21:48.939413+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:32:20.984521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":47,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.937923+0000","last_change":"2023-09-23T20:21:18.891553+0000","last_active":"2023-09-23T20:21:48.937923+0000","last_peered":"2023-09-23T20:21:48.937923+0000","last_clean":"2023-09-23T20:21:48.937923+0000","last_became_active":"2023-09-23T20:21:18.891230+0000","last_became_peered":"2023-09-23T20:21:18.891230+0000","last_unstale":"2023-09-23T20:21:48.937923+0000","last_undegraded":"2023-09-23T20:21:48.937923+0000","last_fullsized":"2023-09-23T20:21:48.937923+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:15:04.114927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"86'130","reported_seq":413,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.954331+0000","last_change":"2023-09-23T20:21:49.954331+0000","last_active":"2023-09-23T20:21:49.954331+0000","last_peered":"2023-09-23T20:21:49.954331+0000","last_clean":"2023-09-23T20:21:49.954331+0000","last_became_active":"2023-09-23T20:21:49.953687+0000","last_became_peered":"2023-09-23T20:21:49.953687+0000","last_unstale":"2023-09-23T20:21:49.954331+0000","last_undegraded":"2023-09-23T20:21:49.954331+0000","last_fullsized":"2023-09-23T20:21:49.954331+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:12.549803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011300049999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":512,"num_read_kb":1820,"num_write":133,"num_write_kb":1347,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":63,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.952961+0000","last_change":"2023-09-23T20:21:49.952961+0000","last_active":"2023-09-23T20:21:49.952961+0000","last_peered":"2023-09-23T20:21:49.952961+0000","last_clean":"2023-09-23T20:21:49.952961+0000","last_became_active":"2023-09-23T20:21:49.952644+0000","last_became_peered":"2023-09-23T20:21:49.952644+0000","last_unstale":"2023-09-23T20:21:49.952961+0000","last_undegraded":"2023-09-23T20:21:49.952961+0000","last_fullsized":"2023-09-23T20:21:49.952961+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:05:30.492540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":70,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.943468+0000","last_change":"2023-09-23T20:20:14.887345+0000","last_active":"2023-09-23T20:21:48.943468+0000","last_peered":"2023-09-23T20:21:48.943468+0000","last_clean":"2023-09-23T20:21:48.943468+0000","last_became_active":"2023-09-23T20:20:14.886970+0000","last_became_peered":"2023-09-23T20:20:14.886970+0000","last_unstale":"2023-09-23T20:21:48.943468+0000","last_undegraded":"2023-09-23T20:21:48.943468+0000","last_fullsized":"2023-09-23T20:21:48.943468+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:00:38.575926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"86'106","reported_seq":734,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.949374+0000","last_change":"2023-09-23T20:20:46.939468+0000","last_active":"2023-09-23T20:21:48.949374+0000","last_peered":"2023-09-23T20:21:48.949374+0000","last_clean":"2023-09-23T20:21:48.949374+0000","last_became_active":"2023-09-23T20:20:46.938954+0000","last_became_peered":"2023-09-23T20:20:46.938954+0000","last_unstale":"2023-09-23T20:21:48.949374+0000","last_undegraded":"2023-09-23T20:21:48.949374+0000","last_fullsized":"2023-09-23T20:21:48.949374+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":106,"log_dups_size":0,"ondisk_log_size":106,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:01:43.761064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00099302599999999993,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1105,"num_read_kb":1689,"num_write":100,"num_write_kb":404,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":33,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.938411+0000","last_change":"2023-09-23T20:20:46.937104+0000","last_active":"2023-09-23T20:21:48.938411+0000","last_peered":"2023-09-23T20:21:48.938411+0000","last_clean":"2023-09-23T20:21:48.938411+0000","last_became_active":"2023-09-23T20:20:46.936820+0000","last_became_peered":"2023-09-23T20:20:46.936820+0000","last_unstale":"2023-09-23T20:21:48.938411+0000","last_undegraded":"2023-09-23T20:21:48.938411+0000","last_fullsized":"2023-09-23T20:21:48.938411+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:22:19.879310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":86,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.953141+0000","last_change":"2023-09-23T20:21:49.953141+0000","last_active":"2023-09-23T20:21:49.953141+0000","last_peered":"2023-09-23T20:21:49.953141+0000","last_clean":"2023-09-23T20:21:49.953141+0000","last_became_active":"2023-09-23T20:21:49.952794+0000","last_became_peered":"2023-09-23T20:21:49.952794+0000","last_unstale":"2023-09-23T20:21:49.953141+0000","last_undegraded":"2023-09-23T20:21:49.953141+0000","last_fullsized":"2023-09-23T20:21:49.953141+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:36:19.919248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"86'119","reported_seq":441,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.939283+0000","last_change":"2023-09-23T20:20:46.937071+0000","last_active":"2023-09-23T20:21:48.939283+0000","last_peered":"2023-09-23T20:21:48.939283+0000","last_clean":"2023-09-23T20:21:48.939283+0000","last_became_active":"2023-09-23T20:20:46.935348+0000","last_became_peered":"2023-09-23T20:20:46.935348+0000","last_unstale":"2023-09-23T20:21:48.939283+0000","last_undegraded":"2023-09-23T20:21:48.939283+0000","last_fullsized":"2023-09-23T20:21:48.939283+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":119,"log_dups_size":0,"ondisk_log_size":119,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:09.829030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032705499999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":523,"num_read_kb":3503,"num_write":136,"num_write_kb":3128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":43,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.959532+0000","last_change":"2023-09-23T20:21:49.959532+0000","last_active":"2023-09-23T20:21:49.959532+0000","last_peered":"2023-09-23T20:21:49.959532+0000","last_clean":"2023-09-23T20:21:49.959532+0000","last_became_active":"2023-09-23T20:21:49.959161+0000","last_became_peered":"2023-09-23T20:21:49.959161+0000","last_unstale":"2023-09-23T20:21:49.959532+0000","last_undegraded":"2023-09-23T20:21:49.959532+0000","last_fullsized":"2023-09-23T20:21:49.959532+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:00:05.013667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":73,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.958915+0000","last_change":"2023-09-23T20:21:49.958915+0000","last_active":"2023-09-23T20:21:49.958915+0000","last_peered":"2023-09-23T20:21:49.958915+0000","last_clean":"2023-09-23T20:21:49.958915+0000","last_became_active":"2023-09-23T20:21:49.958440+0000","last_became_peered":"2023-09-23T20:21:49.958440+0000","last_unstale":"2023-09-23T20:21:49.958915+0000","last_undegraded":"2023-09-23T20:21:49.958915+0000","last_fullsized":"2023-09-23T20:21:49.958915+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:36:48.998945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":69,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.942436+0000","last_change":"2023-09-23T20:20:46.939720+0000","last_active":"2023-09-23T20:21:48.942436+0000","last_peered":"2023-09-23T20:21:48.942436+0000","last_clean":"2023-09-23T20:21:48.942436+0000","last_became_active":"2023-09-23T20:20:46.939357+0000","last_became_peered":"2023-09-23T20:20:46.939357+0000","last_unstale":"2023-09-23T20:21:48.942436+0000","last_undegraded":"2023-09-23T20:21:48.942436+0000","last_fullsized":"2023-09-23T20:21:48.942436+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:56.613212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":70,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.943289+0000","last_change":"2023-09-23T20:21:18.896635+0000","last_active":"2023-09-23T20:21:48.943289+0000","last_peered":"2023-09-23T20:21:48.943289+0000","last_clean":"2023-09-23T20:21:48.943289+0000","last_became_active":"2023-09-23T20:21:18.896335+0000","last_became_peered":"2023-09-23T20:21:18.896335+0000","last_unstale":"2023-09-23T20:21:48.943289+0000","last_undegraded":"2023-09-23T20:21:48.943289+0000","last_fullsized":"2023-09-23T20:21:48.943289+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:46:43.743795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":79,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.944402+0000","last_change":"2023-09-23T20:21:18.891669+0000","last_active":"2023-09-23T20:21:48.944402+0000","last_peered":"2023-09-23T20:21:48.944402+0000","last_clean":"2023-09-23T20:21:48.944402+0000","last_became_active":"2023-09-23T20:21:18.891326+0000","last_became_peered":"2023-09-23T20:21:18.891326+0000","last_unstale":"2023-09-23T20:21:48.944402+0000","last_undegraded":"2023-09-23T20:21:48.944402+0000","last_fullsized":"2023-09-23T20:21:48.944402+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:48:39.893213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":70,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.946401+0000","last_change":"2023-09-23T20:20:46.937163+0000","last_active":"2023-09-23T20:21:48.946401+0000","last_peered":"2023-09-23T20:21:48.946401+0000","last_clean":"2023-09-23T20:21:48.946401+0000","last_became_active":"2023-09-23T20:20:46.936757+0000","last_became_peered":"2023-09-23T20:20:46.936757+0000","last_unstale":"2023-09-23T20:21:48.946401+0000","last_undegraded":"2023-09-23T20:21:48.946401+0000","last_fullsized":"2023-09-23T20:21:48.946401+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:09.401379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":69,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.954710+0000","last_change":"2023-09-23T20:21:49.954710+0000","last_active":"2023-09-23T20:21:49.954710+0000","last_peered":"2023-09-23T20:21:49.954710+0000","last_clean":"2023-09-23T20:21:49.954710+0000","last_became_active":"2023-09-23T20:21:49.954355+0000","last_became_peered":"2023-09-23T20:21:49.954355+0000","last_unstale":"2023-09-23T20:21:49.954710+0000","last_undegraded":"2023-09-23T20:21:49.954710+0000","last_fullsized":"2023-09-23T20:21:49.954710+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:56:16.233285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":70,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.953004+0000","last_change":"2023-09-23T20:21:49.953004+0000","last_active":"2023-09-23T20:21:49.953004+0000","last_peered":"2023-09-23T20:21:49.953004+0000","last_clean":"2023-09-23T20:21:49.953004+0000","last_became_active":"2023-09-23T20:21:49.952625+0000","last_became_peered":"2023-09-23T20:21:49.952625+0000","last_unstale":"2023-09-23T20:21:49.953004+0000","last_undegraded":"2023-09-23T20:21:49.953004+0000","last_fullsized":"2023-09-23T20:21:49.953004+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:48:28.305640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":53,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.939096+0000","last_change":"2023-09-23T20:20:46.935968+0000","last_active":"2023-09-23T20:21:49.939096+0000","last_peered":"2023-09-23T20:21:49.939096+0000","last_clean":"2023-09-23T20:21:49.939096+0000","last_became_active":"2023-09-23T20:20:46.935618+0000","last_became_peered":"2023-09-23T20:20:46.935618+0000","last_unstale":"2023-09-23T20:21:49.939096+0000","last_undegraded":"2023-09-23T20:21:49.939096+0000","last_fullsized":"2023-09-23T20:21:49.939096+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:42:59.867176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":69,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.954875+0000","last_change":"2023-09-23T20:21:49.954875+0000","last_active":"2023-09-23T20:21:49.954875+0000","last_peered":"2023-09-23T20:21:49.954875+0000","last_clean":"2023-09-23T20:21:49.954875+0000","last_became_active":"2023-09-23T20:21:49.954519+0000","last_became_peered":"2023-09-23T20:21:49.954519+0000","last_unstale":"2023-09-23T20:21:49.954875+0000","last_undegraded":"2023-09-23T20:21:49.954875+0000","last_fullsized":"2023-09-23T20:21:49.954875+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:29:52.962029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":69,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.946248+0000","last_change":"2023-09-23T20:20:46.934762+0000","last_active":"2023-09-23T20:21:48.946248+0000","last_peered":"2023-09-23T20:21:48.946248+0000","last_clean":"2023-09-23T20:21:48.946248+0000","last_became_active":"2023-09-23T20:20:46.934472+0000","last_became_peered":"2023-09-23T20:20:46.934472+0000","last_unstale":"2023-09-23T20:21:48.946248+0000","last_undegraded":"2023-09-23T20:21:48.946248+0000","last_fullsized":"2023-09-23T20:21:48.946248+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:28:42.708286+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":54,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.951041+0000","last_change":"2023-09-23T20:12:43.751523+0000","last_active":"2023-09-23T20:21:48.951041+0000","last_peered":"2023-09-23T20:21:48.951041+0000","last_clean":"2023-09-23T20:21:48.951041+0000","last_became_active":"2023-09-23T20:12:43.751257+0000","last_became_peered":"2023-09-23T20:12:43.751257+0000","last_unstale":"2023-09-23T20:21:48.951041+0000","last_undegraded":"2023-09-23T20:21:48.951041+0000","last_fullsized":"2023-09-23T20:21:48.951041+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:50:27.040225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":86,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.957494+0000","last_change":"2023-09-23T20:21:49.957494+0000","last_active":"2023-09-23T20:21:49.957494+0000","last_peered":"2023-09-23T20:21:49.957494+0000","last_clean":"2023-09-23T20:21:49.957494+0000","last_became_active":"2023-09-23T20:21:49.957266+0000","last_became_peered":"2023-09-23T20:21:49.957266+0000","last_unstale":"2023-09-23T20:21:49.957494+0000","last_undegraded":"2023-09-23T20:21:49.957494+0000","last_fullsized":"2023-09-23T20:21:49.957494+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:49:51.483575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":72,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.949331+0000","last_change":"2023-09-23T20:20:46.950784+0000","last_active":"2023-09-23T20:21:48.949331+0000","last_peered":"2023-09-23T20:21:48.949331+0000","last_clean":"2023-09-23T20:21:48.949331+0000","last_became_active":"2023-09-23T20:20:46.950055+0000","last_became_peered":"2023-09-23T20:20:46.950055+0000","last_unstale":"2023-09-23T20:21:48.949331+0000","last_undegraded":"2023-09-23T20:21:48.949331+0000","last_fullsized":"2023-09-23T20:21:48.949331+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:23:07.645427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":51,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.949875+0000","last_change":"2023-09-23T20:12:45.179211+0000","last_active":"2023-09-23T20:21:48.949875+0000","last_peered":"2023-09-23T20:21:48.949875+0000","last_clean":"2023-09-23T20:21:48.949875+0000","last_became_active":"2023-09-23T20:12:45.178966+0000","last_became_peered":"2023-09-23T20:12:45.178966+0000","last_unstale":"2023-09-23T20:21:48.949875+0000","last_undegraded":"2023-09-23T20:21:48.949875+0000","last_fullsized":"2023-09-23T20:21:48.949875+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:58.902140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":71,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.950476+0000","last_change":"2023-09-23T20:20:14.883283+0000","last_active":"2023-09-23T20:21:48.950476+0000","last_peered":"2023-09-23T20:21:48.950476+0000","last_clean":"2023-09-23T20:21:48.950476+0000","last_became_active":"2023-09-23T20:20:14.882806+0000","last_became_peered":"2023-09-23T20:20:14.882806+0000","last_unstale":"2023-09-23T20:21:48.950476+0000","last_undegraded":"2023-09-23T20:21:48.950476+0000","last_fullsized":"2023-09-23T20:21:48.950476+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:23:35.295701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":72,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.958182+0000","last_change":"2023-09-23T20:21:49.958182+0000","last_active":"2023-09-23T20:21:49.958182+0000","last_peered":"2023-09-23T20:21:49.958182+0000","last_clean":"2023-09-23T20:21:49.958182+0000","last_became_active":"2023-09-23T20:21:49.957868+0000","last_became_peered":"2023-09-23T20:21:49.957868+0000","last_unstale":"2023-09-23T20:21:49.958182+0000","last_undegraded":"2023-09-23T20:21:49.958182+0000","last_fullsized":"2023-09-23T20:21:49.958182+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:27:41.577566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":78,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.944990+0000","last_change":"2023-09-23T20:20:14.881009+0000","last_active":"2023-09-23T20:21:48.944990+0000","last_peered":"2023-09-23T20:21:48.944990+0000","last_clean":"2023-09-23T20:21:48.944990+0000","last_became_active":"2023-09-23T20:20:14.880685+0000","last_became_peered":"2023-09-23T20:20:14.880685+0000","last_unstale":"2023-09-23T20:21:48.944990+0000","last_undegraded":"2023-09-23T20:21:48.944990+0000","last_fullsized":"2023-09-23T20:21:48.944990+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:28.010434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":52,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.954242+0000","last_change":"2023-09-23T20:21:49.954242+0000","last_active":"2023-09-23T20:21:49.954242+0000","last_peered":"2023-09-23T20:21:49.954242+0000","last_clean":"2023-09-23T20:21:49.954242+0000","last_became_active":"2023-09-23T20:21:49.953597+0000","last_became_peered":"2023-09-23T20:21:49.953597+0000","last_unstale":"2023-09-23T20:21:49.954242+0000","last_undegraded":"2023-09-23T20:21:49.954242+0000","last_fullsized":"2023-09-23T20:21:49.954242+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:08:46.716466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":53,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.948637+0000","last_change":"2023-09-23T20:12:43.749139+0000","last_active":"2023-09-23T20:21:48.948637+0000","last_peered":"2023-09-23T20:21:48.948637+0000","last_clean":"2023-09-23T20:21:48.948637+0000","last_became_active":"2023-09-23T20:12:43.748854+0000","last_became_peered":"2023-09-23T20:12:43.748854+0000","last_unstale":"2023-09-23T20:21:48.948637+0000","last_undegraded":"2023-09-23T20:21:48.948637+0000","last_fullsized":"2023-09-23T20:21:48.948637+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:57:11.630712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":68,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.952635+0000","last_change":"2023-09-23T20:21:49.952635+0000","last_active":"2023-09-23T20:21:49.952635+0000","last_peered":"2023-09-23T20:21:49.952635+0000","last_clean":"2023-09-23T20:21:49.952635+0000","last_became_active":"2023-09-23T20:21:49.952287+0000","last_became_peered":"2023-09-23T20:21:49.952287+0000","last_unstale":"2023-09-23T20:21:49.952635+0000","last_undegraded":"2023-09-23T20:21:49.952635+0000","last_fullsized":"2023-09-23T20:21:49.952635+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:17:17.370526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":52,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.955345+0000","last_change":"2023-09-23T20:21:49.953201+0000","last_active":"2023-09-23T20:21:49.955345+0000","last_peered":"2023-09-23T20:21:49.955345+0000","last_clean":"2023-09-23T20:21:49.955345+0000","last_became_active":"2023-09-23T20:21:49.952917+0000","last_became_peered":"2023-09-23T20:21:49.952917+0000","last_unstale":"2023-09-23T20:21:49.955345+0000","last_undegraded":"2023-09-23T20:21:49.955345+0000","last_fullsized":"2023-09-23T20:21:49.955345+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:54.579892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":72,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.958515+0000","last_change":"2023-09-23T20:21:49.958515+0000","last_active":"2023-09-23T20:21:49.958515+0000","last_peered":"2023-09-23T20:21:49.958515+0000","last_clean":"2023-09-23T20:21:49.958515+0000","last_became_active":"2023-09-23T20:21:49.958156+0000","last_became_peered":"2023-09-23T20:21:49.958156+0000","last_unstale":"2023-09-23T20:21:49.958515+0000","last_undegraded":"2023-09-23T20:21:49.958515+0000","last_fullsized":"2023-09-23T20:21:49.958515+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:56:47.145702+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":72,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.948079+0000","last_change":"2023-09-23T20:21:18.897418+0000","last_active":"2023-09-23T20:21:48.948079+0000","last_peered":"2023-09-23T20:21:48.948079+0000","last_clean":"2023-09-23T20:21:48.948079+0000","last_became_active":"2023-09-23T20:21:18.896968+0000","last_became_peered":"2023-09-23T20:21:18.896968+0000","last_unstale":"2023-09-23T20:21:48.948079+0000","last_undegraded":"2023-09-23T20:21:48.948079+0000","last_fullsized":"2023-09-23T20:21:48.948079+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:12:44.651568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":69,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.943565+0000","last_change":"2023-09-23T20:20:46.939938+0000","last_active":"2023-09-23T20:21:48.943565+0000","last_peered":"2023-09-23T20:21:48.943565+0000","last_clean":"2023-09-23T20:21:48.943565+0000","last_became_active":"2023-09-23T20:20:46.938777+0000","last_became_peered":"2023-09-23T20:20:46.938777+0000","last_unstale":"2023-09-23T20:21:48.943565+0000","last_undegraded":"2023-09-23T20:21:48.943565+0000","last_fullsized":"2023-09-23T20:21:48.943565+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:45:10.329438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":75,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.943049+0000","last_change":"2023-09-23T20:12:43.750217+0000","last_active":"2023-09-23T20:21:48.943049+0000","last_peered":"2023-09-23T20:21:48.943049+0000","last_clean":"2023-09-23T20:21:48.943049+0000","last_became_active":"2023-09-23T20:12:43.749929+0000","last_became_peered":"2023-09-23T20:12:43.749929+0000","last_unstale":"2023-09-23T20:21:48.943049+0000","last_undegraded":"2023-09-23T20:21:48.943049+0000","last_fullsized":"2023-09-23T20:21:48.943049+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:50:28.470211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":48,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.938497+0000","last_change":"2023-09-23T20:21:18.891438+0000","last_active":"2023-09-23T20:21:48.938497+0000","last_peered":"2023-09-23T20:21:48.938497+0000","last_clean":"2023-09-23T20:21:48.938497+0000","last_became_active":"2023-09-23T20:21:18.891140+0000","last_became_peered":"2023-09-23T20:21:18.891140+0000","last_unstale":"2023-09-23T20:21:48.938497+0000","last_undegraded":"2023-09-23T20:21:48.938497+0000","last_fullsized":"2023-09-23T20:21:48.938497+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:18:44.603577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":73,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.954219+0000","last_change":"2023-09-23T20:21:49.954219+0000","last_active":"2023-09-23T20:21:49.954219+0000","last_peered":"2023-09-23T20:21:49.954219+0000","last_clean":"2023-09-23T20:21:49.954219+0000","last_became_active":"2023-09-23T20:21:49.953827+0000","last_became_peered":"2023-09-23T20:21:49.953827+0000","last_unstale":"2023-09-23T20:21:49.954219+0000","last_undegraded":"2023-09-23T20:21:49.954219+0000","last_fullsized":"2023-09-23T20:21:49.954219+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:46:48.036706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":23,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.955179+0000","last_change":"2023-09-23T20:21:49.955179+0000","last_active":"2023-09-23T20:21:49.955179+0000","last_peered":"2023-09-23T20:21:49.955179+0000","last_clean":"2023-09-23T20:21:49.955179+0000","last_became_active":"2023-09-23T20:21:49.954878+0000","last_became_peered":"2023-09-23T20:21:49.954878+0000","last_unstale":"2023-09-23T20:21:49.955179+0000","last_undegraded":"2023-09-23T20:21:49.955179+0000","last_fullsized":"2023-09-23T20:21:49.955179+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:02:26.579246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":69,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.947957+0000","last_change":"2023-09-23T20:20:46.950281+0000","last_active":"2023-09-23T20:21:48.947957+0000","last_peered":"2023-09-23T20:21:48.947957+0000","last_clean":"2023-09-23T20:21:48.947957+0000","last_became_active":"2023-09-23T20:20:46.949892+0000","last_became_peered":"2023-09-23T20:20:46.949892+0000","last_unstale":"2023-09-23T20:21:48.947957+0000","last_undegraded":"2023-09-23T20:21:48.947957+0000","last_fullsized":"2023-09-23T20:21:48.947957+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:28.763832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":72,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.958837+0000","last_change":"2023-09-23T20:21:49.958837+0000","last_active":"2023-09-23T20:21:49.958837+0000","last_peered":"2023-09-23T20:21:49.958837+0000","last_clean":"2023-09-23T20:21:49.958837+0000","last_became_active":"2023-09-23T20:21:49.958135+0000","last_became_peered":"2023-09-23T20:21:49.958135+0000","last_unstale":"2023-09-23T20:21:49.958837+0000","last_undegraded":"2023-09-23T20:21:49.958837+0000","last_fullsized":"2023-09-23T20:21:49.958837+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:31:52.350275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":51,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.950402+0000","last_change":"2023-09-23T20:21:49.950402+0000","last_active":"2023-09-23T20:21:49.950402+0000","last_peered":"2023-09-23T20:21:49.950402+0000","last_clean":"2023-09-23T20:21:49.950402+0000","last_became_active":"2023-09-23T20:21:49.950116+0000","last_became_peered":"2023-09-23T20:21:49.950116+0000","last_unstale":"2023-09-23T20:21:49.950402+0000","last_undegraded":"2023-09-23T20:21:49.950402+0000","last_fullsized":"2023-09-23T20:21:49.950402+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:10:20.322615+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":69,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.945804+0000","last_change":"2023-09-23T20:20:46.937089+0000","last_active":"2023-09-23T20:21:48.945804+0000","last_peered":"2023-09-23T20:21:48.945804+0000","last_clean":"2023-09-23T20:21:48.945804+0000","last_became_active":"2023-09-23T20:20:46.936831+0000","last_became_peered":"2023-09-23T20:20:46.936831+0000","last_unstale":"2023-09-23T20:21:48.945804+0000","last_undegraded":"2023-09-23T20:21:48.945804+0000","last_fullsized":"2023-09-23T20:21:48.945804+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:34:56.747010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":90,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.958727+0000","last_change":"2023-09-23T20:21:49.958727+0000","last_active":"2023-09-23T20:21:49.958727+0000","last_peered":"2023-09-23T20:21:49.958727+0000","last_clean":"2023-09-23T20:21:49.958727+0000","last_became_active":"2023-09-23T20:21:49.957722+0000","last_became_peered":"2023-09-23T20:21:49.957722+0000","last_unstale":"2023-09-23T20:21:49.958727+0000","last_undegraded":"2023-09-23T20:21:49.958727+0000","last_fullsized":"2023-09-23T20:21:49.958727+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:47:13.589014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":53,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.956577+0000","last_change":"2023-09-23T20:21:49.954419+0000","last_active":"2023-09-23T20:21:49.956577+0000","last_peered":"2023-09-23T20:21:49.956577+0000","last_clean":"2023-09-23T20:21:49.956577+0000","last_became_active":"2023-09-23T20:21:49.953941+0000","last_became_peered":"2023-09-23T20:21:49.953941+0000","last_unstale":"2023-09-23T20:21:49.956577+0000","last_undegraded":"2023-09-23T20:21:49.956577+0000","last_fullsized":"2023-09-23T20:21:49.956577+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:05:38.809180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":47,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.955411+0000","last_change":"2023-09-23T20:21:49.955411+0000","last_active":"2023-09-23T20:21:49.955411+0000","last_peered":"2023-09-23T20:21:49.955411+0000","last_clean":"2023-09-23T20:21:49.955411+0000","last_became_active":"2023-09-23T20:21:49.955112+0000","last_became_peered":"2023-09-23T20:21:49.955112+0000","last_unstale":"2023-09-23T20:21:49.955411+0000","last_undegraded":"2023-09-23T20:21:49.955411+0000","last_fullsized":"2023-09-23T20:21:49.955411+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:47:53.870056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":72,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T20:21:49.951483+0000","last_change":"2023-09-23T20:21:49.951483+0000","last_active":"2023-09-23T20:21:49.951483+0000","last_peered":"2023-09-23T20:21:49.951483+0000","last_clean":"2023-09-23T20:21:49.951483+0000","last_became_active":"2023-09-23T20:21:49.951167+0000","last_became_peered":"2023-09-23T20:21:49.951167+0000","last_unstale":"2023-09-23T20:21:49.951483+0000","last_undegraded":"2023-09-23T20:21:49.951483+0000","last_fullsized":"2023-09-23T20:21:49.951483+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:08:19.552997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"69'1","reported_seq":88,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T20:21:48.942866+0000","last_change":"2023-09-23T20:21:18.891671+0000","last_active":"2023-09-23T20:21:48.942866+0000","last_peered":"2023-09-23T20:21:48.942866+0000","last_clean":"2023-09-23T20:21:48.942866+0000","last_became_active":"2023-09-23T20:21:18.891285+0000","last_became_peered":"2023-09-23T20:21:18.891285+0000","last_unstale":"2023-09-23T20:21:48.942866+0000","last_undegraded":"2023-09-23T20:21:48.942866+0000","last_fullsized":"2023-09-23T20:21:48.942866+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:04:49.595473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":25181,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6082,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201417,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":16944,"kb_used_data":5800,"kb_used_omap":0,"kb_used_meta":11136,"kb_avail":93728208,"statfs":{"total":95995035648,"available":95977684992,"internally_reserved":0,"allocated":5939200,"data_stored":5402099,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":11403264},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:22:17 2023","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.56499999999999995,"15min":0.56999999999999995},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":0.74199999999999999,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":1.5049999999999999},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.56200000000000006,"15min":0.56599999999999995},"min":{"1min":0.40600000000000003,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.68300000000000005,"5min":0.93799999999999994,"15min":0.93799999999999994},"last":1.5209999999999999}]},{"osd":1,"last update":"Sat Sep 23 20:21:48 2023","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":0.74399999999999999,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":1.399},{"interface":"front","average":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"min":{"1min":0.443,"5min":0.443,"15min":0.443},"max":{"1min":0.88800000000000001,"5min":0.88800000000000001,"15min":0.88800000000000001},"last":1.5569999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:22:19 2023","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.82599999999999996,"5min":0.82599999999999996,"15min":0.82599999999999996},"last":1.448},{"interface":"front","average":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.85999999999999999,"5min":0.85999999999999999,"15min":0.85999999999999999},"last":1.4670000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:20:44 2023","interfaces":[{"interface":"back","average":{"1min":0.59599999999999997,"5min":0.61199999999999999,"15min":0.57299999999999995},"min":{"1min":0.36699999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.72099999999999997,"5min":1.988,"15min":1.988},"last":1.5369999999999999},{"interface":"front","average":{"1min":0.623,"5min":0.63,"15min":0.61599999999999999},"min":{"1min":0.44,"5min":0.40899999999999997,"15min":0.36199999999999999},"max":{"1min":0.84399999999999997,"5min":0.872,"15min":1.127},"last":1.492}]},{"osd":4,"last update":"Sat Sep 23 20:21:48 2023","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.50600000000000001,"15min":0.50600000000000001},"min":{"1min":0.30199999999999999,"5min":0.28799999999999998,"15min":0.26700000000000002},"max":{"1min":0.90100000000000002,"5min":0.90100000000000002,"15min":1.274},"last":0.51100000000000001},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.54300000000000004,"15min":0.53400000000000003},"min":{"1min":0.35799999999999998,"5min":0.33100000000000002,"15min":0.29699999999999999},"max":{"1min":0.80000000000000004,"5min":0.80000000000000004,"15min":2.024},"last":0.53400000000000003}]},{"osd":5,"last update":"Sat Sep 23 20:21:48 2023","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":0.54200000000000004,"15min":0.54100000000000004},"min":{"1min":0.36599999999999999,"5min":0.33500000000000002,"15min":0.32000000000000001},"max":{"1min":0.79300000000000004,"5min":0.79600000000000004,"15min":1.337},"last":0.495},{"interface":"front","average":{"1min":0.629,"5min":0.60099999999999998,"15min":0.58799999999999997},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.31},"max":{"1min":0.89100000000000001,"5min":0.89100000000000001,"15min":1.5660000000000001},"last":0.60299999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:21:48 2023","interfaces":[{"interface":"back","average":{"1min":0.65300000000000002,"5min":0.61899999999999999,"15min":0.59399999999999997},"min":{"1min":0.379,"5min":0.379,"15min":0.34499999999999997},"max":{"1min":0.98699999999999999,"5min":0.98699999999999999,"15min":1.246},"last":0.56200000000000006},{"interface":"front","average":{"1min":0.60799999999999998,"5min":0.60999999999999999,"15min":0.58399999999999996},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.34799999999999998},"max":{"1min":0.873,"5min":0.873,"15min":2.048},"last":0.58799999999999997}]}]},{"osd":6,"up_from":47,"seq":201863463053,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":12392,"kb_used_data":5792,"kb_used_omap":0,"kb_used_meta":6592,"kb_avail":93732760,"statfs":{"total":95995035648,"available":95982346240,"internally_reserved":0,"allocated":5931008,"data_stored":5401972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6750208},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:22:18 2023","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":0.57799999999999996,"15min":0.57799999999999996},"min":{"1min":0.36799999999999999,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.628},{"interface":"front","average":{"1min":0.57499999999999996,"5min":0.56499999999999995,"15min":0.56299999999999994},"min":{"1min":0.40500000000000003,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.76600000000000001,"5min":0.82299999999999995,"15min":0.82299999999999995},"last":0.79500000000000004}]},{"osd":1,"last update":"Sat Sep 23 20:21:48 2023","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.76300000000000001,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.59099999999999997},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.77500000000000002,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.65500000000000003}]},{"osd":2,"last update":"Sat Sep 23 20:22:18 2023","interfaces":[{"interface":"back","average":{"1min":0.59499999999999997,"5min":0.59499999999999997,"15min":0.59499999999999997},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.75,"5min":0.75,"15min":0.75},"last":0.60999999999999999},{"interface":"front","average":{"1min":0.60899999999999999,"5min":0.60899999999999999,"15min":0.60899999999999999},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":0.79300000000000004,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.73599999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:21:34 2023","interfaces":[{"interface":"back","average":{"1min":0.63100000000000001,"5min":0.60299999999999998,"15min":0.56100000000000005},"min":{"1min":0.46300000000000002,"5min":0.314,"15min":0.314},"max":{"1min":0.79300000000000004,"5min":0.81799999999999995,"15min":0.86599999999999999},"last":0.64100000000000001},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.59099999999999997,"15min":0.57999999999999996},"min":{"1min":0.40699999999999997,"5min":0.35299999999999998,"15min":0.34999999999999998},"max":{"1min":0.79200000000000004,"5min":0.84099999999999997,"15min":2.0670000000000002},"last":0.67200000000000004}]},{"osd":4,"last update":"Sat Sep 23 20:21:34 2023","interfaces":[{"interface":"back","average":{"1min":0.46700000000000003,"5min":0.46899999999999997,"15min":0.46300000000000002},"min":{"1min":0.38200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.63400000000000001,"5min":0.71799999999999997,"15min":0.72299999999999998},"last":0.56899999999999995},{"interface":"front","average":{"1min":0.52400000000000002,"5min":0.51300000000000001,"15min":0.49299999999999999},"min":{"1min":0.30099999999999999,"5min":0.28599999999999998,"15min":0.28499999999999998},"max":{"1min":0.72499999999999998,"5min":0.83899999999999997,"15min":0.83899999999999997},"last":0.72199999999999998}]},{"osd":5,"last update":"Sat Sep 23 20:21:34 2023","interfaces":[{"interface":"back","average":{"1min":0.497,"5min":0.51700000000000002,"15min":0.49399999999999999},"min":{"1min":0.32600000000000001,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.67100000000000004,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.68799999999999994},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.55500000000000005,"15min":0.52100000000000002},"min":{"1min":0.34300000000000003,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.77900000000000003,"5min":0.874,"15min":0.874},"last":0.76000000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:21:48 2023","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.57499999999999996,"15min":0.54300000000000004},"min":{"1min":0.372,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.85199999999999998,"5min":0.85199999999999998,"15min":0.85199999999999998},"last":0.78000000000000003},{"interface":"front","average":{"1min":0.60199999999999998,"5min":0.60099999999999998,"15min":0.56899999999999995},"min":{"1min":0.38200000000000001,"5min":0.33500000000000002,"15min":0.315},"max":{"1min":0.83099999999999996,"5min":0.90800000000000003,"15min":0.90800000000000003},"last":0.82599999999999996}]}]},{"osd":1,"up_from":99,"seq":425201762328,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28744,"kb_used_data":1152,"kb_used_omap":6,"kb_used_meta":27577,"kb_avail":93716408,"statfs":{"total":95995035648,"available":95965601792,"internally_reserved":0,"allocated":1179648,"data_stored":623002,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6696,"internal_metadata":28239320},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:21:46 2023","interfaces":[{"interface":"back","average":{"1min":0.377,"5min":0.377,"15min":0.377},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.50700000000000001,"5min":0.50700000000000001,"15min":0.50700000000000001},"last":0.26200000000000001},{"interface":"front","average":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"last":0.34000000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:22:18 2023","interfaces":[{"interface":"back","average":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.628,"5min":0.628,"15min":0.628},"last":0.35199999999999998},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":3.226,"5min":3.226,"15min":3.226},"last":0.309}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.40300000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:21:46 2023","interfaces":[{"interface":"back","average":{"1min":0.61699999999999999,"5min":0.61699999999999999,"15min":0.61699999999999999},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.89100000000000001,"5min":0.89100000000000001,"15min":0.89100000000000001},"last":0.84899999999999998},{"interface":"front","average":{"1min":0.63500000000000001,"5min":0.63500000000000001,"15min":0.63500000000000001},"min":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":0.85199999999999998,"5min":0.85199999999999998,"15min":0.85199999999999998},"last":0.60199999999999998}]},{"osd":5,"last update":"Sat Sep 23 20:21:46 2023","interfaces":[{"interface":"back","average":{"1min":0.64100000000000001,"5min":0.64100000000000001,"15min":0.64100000000000001},"min":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.76200000000000001},{"interface":"front","average":{"1min":0.67600000000000005,"5min":0.67600000000000005,"15min":0.67600000000000005},"min":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"max":{"1min":0.86899999999999999,"5min":0.86899999999999999,"15min":0.86899999999999999},"last":0.755}]},{"osd":6,"last update":"Sat Sep 23 20:21:46 2023","interfaces":[{"interface":"back","average":{"1min":0.66800000000000004,"5min":0.66800000000000004,"15min":0.66800000000000004},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.92200000000000004,"5min":0.92200000000000004,"15min":0.92200000000000004},"last":0.80700000000000005},{"interface":"front","average":{"1min":0.63800000000000001,"5min":0.63800000000000001,"15min":0.63800000000000001},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":0.86599999999999999,"5min":0.86599999999999999,"15min":0.86599999999999999},"last":0.64000000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:21:46 2023","interfaces":[{"interface":"back","average":{"1min":0.65200000000000002,"5min":0.65200000000000002,"15min":0.65200000000000002},"min":{"1min":0.47599999999999998,"5min":0.47599999999999998,"15min":0.47599999999999998},"max":{"1min":0.91200000000000003,"5min":0.91200000000000003,"15min":0.91200000000000003},"last":0.61699999999999999},{"interface":"front","average":{"1min":0.68400000000000005,"5min":0.68400000000000005,"15min":0.68400000000000005},"min":{"1min":0.45300000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"max":{"1min":0.85899999999999999,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.746}]}]},{"osd":0,"up_from":94,"seq":403726925854,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":29340,"kb_used_data":1748,"kb_used_omap":7,"kb_used_meta":27576,"kb_avail":93715812,"statfs":{"total":95995035648,"available":95964991488,"internally_reserved":0,"allocated":1789952,"data_stored":1215718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":28237841},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:21:48 2023","interfaces":[{"interface":"back","average":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"last":0.35599999999999998},{"interface":"front","average":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":0.52700000000000002,"5min":0.52700000000000002,"15min":0.52700000000000002},"last":0.438}]},{"osd":2,"last update":"Sat Sep 23 20:22:18 2023","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":4.6280000000000001,"5min":4.6280000000000001,"15min":4.6280000000000001},"last":0.45200000000000001},{"interface":"front","average":{"1min":0.496,"5min":0.496,"15min":0.496},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":1.921,"5min":1.921,"15min":1.921},"last":0.40000000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:21:15 2023","interfaces":[{"interface":"back","average":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"last":0.38600000000000001},{"interface":"front","average":{"1min":0.435,"5min":0.435,"15min":0.435},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"last":0.41499999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:22:18 2023","interfaces":[{"interface":"back","average":{"1min":0.78800000000000003,"5min":0.63600000000000001,"15min":0.60999999999999999},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":4.7450000000000001,"5min":4.7450000000000001,"15min":4.7450000000000001},"last":0.59999999999999998},{"interface":"front","average":{"1min":0.60799999999999998,"5min":0.61199999999999999,"15min":0.61299999999999999},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.92100000000000004,"5min":0.92100000000000004,"15min":0.92100000000000004},"last":0.65200000000000002}]},{"osd":5,"last update":"Sat Sep 23 20:22:18 2023","interfaces":[{"interface":"back","average":{"1min":0.64500000000000002,"5min":0.65600000000000003,"15min":0.65800000000000003},"min":{"1min":0.47899999999999998,"5min":0.47899999999999998,"15min":0.47899999999999998},"max":{"1min":0.90400000000000003,"5min":0.90400000000000003,"15min":0.90400000000000003},"last":0.61099999999999999},{"interface":"front","average":{"1min":0.67100000000000004,"5min":0.67700000000000005,"15min":0.67800000000000005},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.90600000000000003,"5min":0.93899999999999995,"15min":0.93899999999999995},"last":0.66600000000000004}]},{"osd":6,"last update":"Sat Sep 23 20:22:18 2023","interfaces":[{"interface":"back","average":{"1min":0.68799999999999994,"5min":0.65600000000000003,"15min":0.65100000000000002},"min":{"1min":0.495,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":0.97899999999999998,"5min":0.97899999999999998,"15min":0.97899999999999998},"last":0.55000000000000004},{"interface":"front","average":{"1min":0.68799999999999994,"5min":0.66800000000000004,"15min":0.66400000000000003},"min":{"1min":0.45300000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"max":{"1min":0.89900000000000002,"5min":0.95999999999999996,"15min":0.95999999999999996},"last":0.63600000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:22:18 2023","interfaces":[{"interface":"back","average":{"1min":0.66400000000000003,"5min":0.67600000000000005,"15min":0.67800000000000005},"min":{"1min":0.47199999999999998,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":0.96399999999999997,"5min":0.96399999999999997,"15min":0.96399999999999997},"last":0.68500000000000005},{"interface":"front","average":{"1min":0.86399999999999999,"5min":0.70399999999999996,"15min":0.67800000000000005},"min":{"1min":0.52200000000000002,"5min":0.48499999999999999,"15min":0.48499999999999999},"max":{"1min":4.7140000000000004,"5min":4.7140000000000004,"15min":4.7140000000000004},"last":0.64700000000000002}]}]},{"osd":2,"up_from":104,"seq":446676598802,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32768,"kb_used_data":5240,"kb_used_omap":11,"kb_used_meta":27508,"kb_avail":93712384,"statfs":{"total":95995035648,"available":95961481216,"internally_reserved":0,"allocated":5365760,"data_stored":4815759,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11878,"internal_metadata":28168602},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:22:20 2023","interfaces":[{"interface":"back","average":{"1min":0.377,"5min":0.377,"15min":0.377},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":0.60599999999999998,"5min":0.60599999999999998,"15min":0.60599999999999998},"last":0.32600000000000001},{"interface":"front","average":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.69899999999999995,"5min":0.69899999999999995,"15min":0.69899999999999995},"last":0.38900000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:22:20 2023","interfaces":[{"interface":"back","average":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.68000000000000005,"5min":0.68000000000000005,"15min":0.68000000000000005},"last":0.438},{"interface":"front","average":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"last":0.34000000000000002}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.377}]},{"osd":4,"last update":"Sat Sep 23 20:22:20 2023","interfaces":[{"interface":"back","average":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.53700000000000003},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.74099999999999999,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.49199999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:22:20 2023","interfaces":[{"interface":"back","average":{"1min":0.65700000000000003,"5min":0.65700000000000003,"15min":0.65700000000000003},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.89100000000000001,"5min":0.89100000000000001,"15min":0.89100000000000001},"last":0.58199999999999996},{"interface":"front","average":{"1min":0.65700000000000003,"5min":0.65700000000000003,"15min":0.65700000000000003},"min":{"1min":0.46600000000000003,"5min":0.46600000000000003,"15min":0.46600000000000003},"max":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.58899999999999997}]},{"osd":6,"last update":"Sat Sep 23 20:22:20 2023","interfaces":[{"interface":"back","average":{"1min":0.65700000000000003,"5min":0.65700000000000003,"15min":0.65700000000000003},"min":{"1min":0.501,"5min":0.501,"15min":0.501},"max":{"1min":0.94299999999999995,"5min":0.94299999999999995,"15min":0.94299999999999995},"last":0.46999999999999997},{"interface":"front","average":{"1min":0.67300000000000004,"5min":0.67300000000000004,"15min":0.67300000000000004},"min":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":1,"5min":1,"15min":1},"last":0.56999999999999995}]},{"osd":7,"last update":"Sat Sep 23 20:22:20 2023","interfaces":[{"interface":"back","average":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"min":{"1min":0.48399999999999999,"5min":0.48399999999999999,"15min":0.48399999999999999},"max":{"1min":0.96099999999999997,"5min":0.96099999999999997,"15min":0.96099999999999997},"last":0.54500000000000004},{"interface":"front","average":{"1min":0.65700000000000003,"5min":0.65700000000000003,"15min":0.65700000000000003},"min":{"1min":0.49299999999999999,"5min":0.49299999999999999,"15min":0.49299999999999999},"max":{"1min":0.92500000000000004,"5min":0.92500000000000004,"15min":0.92500000000000004},"last":0.60199999999999998}]}]},{"osd":3,"up_from":109,"seq":468151435275,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28648,"kb_used_data":1120,"kb_used_omap":4,"kb_used_meta":27515,"kb_avail":93716504,"statfs":{"total":95995035648,"available":95965700096,"internally_reserved":0,"allocated":1146880,"data_stored":617010,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4514,"internal_metadata":28175966},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.32000000000000001}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.30099999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26300000000000001}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.57899999999999996}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.60999999999999999}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.65800000000000003}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.622}]}]},{"osd":4,"up_from":30,"seq":128849019029,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":7780,"kb_used_data":1116,"kb_used_omap":0,"kb_used_meta":6656,"kb_avail":93737372,"statfs":{"total":95995035648,"available":95987068928,"internally_reserved":0,"allocated":1142784,"data_stored":617034,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6815744},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:22:16 2023","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":0.57599999999999996,"15min":0.57099999999999995},"min":{"1min":0.41099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.85099999999999998,"5min":0.85099999999999998,"15min":0.85099999999999998},"last":0.54200000000000004},{"interface":"front","average":{"1min":0.63500000000000001,"5min":0.64000000000000001,"15min":0.64100000000000001},"min":{"1min":0.47299999999999998,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":0.79500000000000004,"5min":0.84799999999999998,"15min":0.84799999999999998},"last":0.5}]},{"osd":1,"last update":"Sat Sep 23 20:21:49 2023","interfaces":[{"interface":"back","average":{"1min":0.626,"5min":0.626,"15min":0.626},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":1.4219999999999999,"5min":1.4219999999999999,"15min":1.4219999999999999},"last":0.46400000000000002},{"interface":"front","average":{"1min":0.626,"5min":0.626,"15min":0.626},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":1.393,"5min":1.393,"15min":1.393},"last":0.434}]},{"osd":2,"last update":"Sat Sep 23 20:22:19 2023","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"min":{"1min":0.49299999999999999,"5min":0.49299999999999999,"15min":0.49299999999999999},"max":{"1min":0.78400000000000003,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.60199999999999998},{"interface":"front","average":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":0.73599999999999999,"5min":0.73599999999999999,"15min":0.73599999999999999},"last":0.58999999999999997}]},{"osd":3,"last update":"Sat Sep 23 20:21:40 2023","interfaces":[{"interface":"back","average":{"1min":0.63400000000000001,"5min":0.64700000000000002,"15min":0.58899999999999997},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":1.4079999999999999,"5min":3.0800000000000001,"15min":3.0800000000000001},"last":0.66800000000000004},{"interface":"front","average":{"1min":0.67100000000000004,"5min":0.64900000000000002,"15min":0.58699999999999997},"min":{"1min":0.435,"5min":0.373,"15min":0.34699999999999998},"max":{"1min":1.4630000000000001,"5min":3.6190000000000002,"15min":3.6190000000000002},"last":0.71599999999999997}]},{"osd":5,"last update":"Sat Sep 23 20:22:05 2023","interfaces":[{"interface":"back","average":{"1min":0.48799999999999999,"5min":0.50600000000000001,"15min":0.45700000000000002},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.26000000000000001},"max":{"1min":0.66100000000000003,"5min":3.5270000000000001,"15min":3.5270000000000001},"last":0.623},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.52900000000000003,"15min":0.46700000000000003},"min":{"1min":0.317,"5min":0.317,"15min":0.248},"max":{"1min":0.81299999999999994,"5min":0.86299999999999999,"15min":0.86299999999999999},"last":0.64400000000000002}]},{"osd":6,"last update":"Sat Sep 23 20:22:27 2023","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":0.54600000000000004,"15min":0.497},"min":{"1min":0.46999999999999997,"5min":0.35599999999999998,"15min":0.27500000000000002},"max":{"1min":0.76000000000000001,"5min":0.77900000000000003,"15min":0.82499999999999996},"last":0.69999999999999996},{"interface":"front","average":{"1min":0.56999999999999995,"5min":0.56599999999999995,"15min":0.51000000000000001},"min":{"1min":0.41499999999999998,"5min":0.32100000000000001,"15min":0.314},"max":{"1min":0.73299999999999998,"5min":3.4820000000000002,"15min":3.4820000000000002},"last":0.73199999999999998}]},{"osd":7,"last update":"Sat Sep 23 20:21:40 2023","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":0.57199999999999995,"15min":0.54700000000000004},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.33600000000000002},"max":{"1min":0.875,"5min":0.91600000000000004,"15min":0.91600000000000004},"last":0.76200000000000001},{"interface":"front","average":{"1min":0.57999999999999996,"5min":0.61199999999999999,"15min":0.55600000000000005},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.29899999999999999},"max":{"1min":0.80800000000000005,"5min":3.77,"15min":3.77},"last":0.80200000000000005}]}]},{"osd":5,"up_from":39,"seq":167503724690,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":7516,"kb_used_data":1108,"kb_used_omap":0,"kb_used_meta":6400,"kb_avail":93737636,"statfs":{"total":95995035648,"available":95987339264,"internally_reserved":0,"allocated":1134592,"data_stored":613120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6553600},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:22:18 2023","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":0.59599999999999997,"15min":0.59299999999999997},"min":{"1min":0.38900000000000001,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.79800000000000004,"5min":0.83499999999999996,"15min":0.83499999999999996},"last":0.625},{"interface":"front","average":{"1min":0.57999999999999996,"5min":0.59599999999999997,"15min":0.59899999999999998},"min":{"1min":0.44,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.82299999999999995,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.56000000000000005}]},{"osd":1,"last update":"Sat Sep 23 20:21:47 2023","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.57399999999999995},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.84999999999999998,"5min":0.84999999999999998,"15min":0.84999999999999998},"last":0.51600000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:22:18 2023","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.70899999999999996,"5min":0.70899999999999996,"15min":0.70899999999999996},"last":0.65000000000000002},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.72899999999999998,"5min":0.72899999999999998,"15min":0.72899999999999998},"last":0.46800000000000003}]},{"osd":3,"last update":"Sat Sep 23 20:21:11 2023","interfaces":[{"interface":"back","average":{"1min":0.60299999999999998,"5min":0.60699999999999998,"15min":0.59099999999999997},"min":{"1min":0.47499999999999998,"5min":0.35299999999999998,"15min":0.34100000000000003},"max":{"1min":0.73199999999999998,"5min":0.93400000000000005,"15min":2.7570000000000001},"last":0.48599999999999999},{"interface":"front","average":{"1min":0.61099999999999999,"5min":0.63,"15min":0.57399999999999995},"min":{"1min":0.40699999999999997,"5min":0.40600000000000003,"15min":0.35599999999999998},"max":{"1min":0.74099999999999999,"5min":0.88200000000000001,"15min":2.2599999999999998},"last":0.60099999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:22:13 2023","interfaces":[{"interface":"back","average":{"1min":0.498,"5min":0.48199999999999998,"15min":0.439},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.26400000000000001},"max":{"1min":0.68200000000000005,"5min":0.76100000000000001,"15min":1.0620000000000001},"last":0.45100000000000001},{"interface":"front","average":{"1min":0.58799999999999997,"5min":0.52700000000000002,"15min":0.47999999999999998},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.28799999999999998},"max":{"1min":0.82299999999999995,"5min":0.82299999999999995,"15min":1.4450000000000001},"last":0.499}]},{"osd":6,"last update":"Sat Sep 23 20:22:27 2023","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":0.52000000000000002,"15min":0.49399999999999999},"min":{"1min":0.44500000000000001,"5min":0.32500000000000001,"15min":0.29799999999999999},"max":{"1min":0.80000000000000004,"5min":0.80000000000000004,"15min":1.151},"last":0.54300000000000004},{"interface":"front","average":{"1min":0.65700000000000003,"5min":0.61399999999999999,"15min":0.56299999999999994},"min":{"1min":0.375,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.84899999999999998,"5min":0.84899999999999998,"15min":0.84899999999999998},"last":0.53300000000000003}]},{"osd":7,"last update":"Sat Sep 23 20:21:44 2023","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.56599999999999995,"15min":0.53800000000000003},"min":{"1min":0.41799999999999998,"5min":0.372,"15min":0.33800000000000002},"max":{"1min":0.81999999999999995,"5min":0.92500000000000004,"15min":0.98199999999999998},"last":0.42499999999999999},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.57799999999999996,"15min":0.54500000000000004},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.79200000000000004,"5min":0.79200000000000004,"15min":1.0489999999999999},"last":0.38700000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4722,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1360,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6696,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7156,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3154,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:22:39.321 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2023-09-23T20:22:39.321 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd dump 2023-09-23T20:22:39.927 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:22:40.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:22:39 smithi149 ceph-mon[170394]: from='client.34728 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:22:40.311 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:22:39 smithi018 ceph-mon[184144]: from='client.34728 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:22:40.311 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:22:39 smithi018 ceph-mon[179647]: from='client.34728 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:22:41.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:22:40 smithi149 ceph-mon[170394]: pgmap v223: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:22:41.315 INFO:teuthology.orchestra.run.smithi018.stdout:epoch 110 2023-09-23T20:22:41.315 INFO:teuthology.orchestra.run.smithi018.stdout:fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 2023-09-23T20:22:41.315 INFO:teuthology.orchestra.run.smithi018.stdout:created 2023-09-23T20:06:54.359034+0000 2023-09-23T20:22:41.315 INFO:teuthology.orchestra.run.smithi018.stdout:modified 2023-09-23T20:21:49.935938+0000 2023-09-23T20:22:41.316 INFO:teuthology.orchestra.run.smithi018.stdout:flags sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit 2023-09-23T20:22:41.316 INFO:teuthology.orchestra.run.smithi018.stdout:crush_version 22 2023-09-23T20:22:41.316 INFO:teuthology.orchestra.run.smithi018.stdout:full_ratio 0.95 2023-09-23T20:22:41.316 INFO:teuthology.orchestra.run.smithi018.stdout:backfillfull_ratio 0.9 2023-09-23T20:22:41.316 INFO:teuthology.orchestra.run.smithi018.stdout:nearfull_ratio 0.85 2023-09-23T20:22:41.316 INFO:teuthology.orchestra.run.smithi018.stdout:require_min_compat_client luminous 2023-09-23T20:22:41.316 INFO:teuthology.orchestra.run.smithi018.stdout:min_compat_client luminous 2023-09-23T20:22:41.316 INFO:teuthology.orchestra.run.smithi018.stdout:require_osd_release quincy 2023-09-23T20:22:41.316 INFO:teuthology.orchestra.run.smithi018.stdout:stretch_mode_enabled false 2023-09-23T20:22:41.316 INFO:teuthology.orchestra.run.smithi018.stdout:pool 1 '.mgr' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 1 pgp_num 1 autoscale_mode off last_change 22 flags hashpspool stripe_width 0 pg_num_max 32 pg_num_min 1 application mgr read_balance_score 7.89 2023-09-23T20:22:41.316 INFO:teuthology.orchestra.run.smithi018.stdout:pool 2 'rbd' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 8 pgp_num 8 autoscale_mode off last_change 86 flags hashpspool,selfmanaged_snaps stripe_width 0 application rbd read_balance_score 2.00 2023-09-23T20:22:41.316 INFO:teuthology.orchestra.run.smithi018.stdout:pool 3 'cephfs.foo.meta' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode off last_change 68 flags hashpspool stripe_width 0 pg_autoscale_bias 4 pg_num_min 16 recovery_priority 5 application cephfs read_balance_score 1.50 2023-09-23T20:22:41.317 INFO:teuthology.orchestra.run.smithi018.stdout:pool 4 'cephfs.foo.data' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode off last_change 69 flags hashpspool stripe_width 0 application cephfs read_balance_score 2.25 2023-09-23T20:22:41.317 INFO:teuthology.orchestra.run.smithi018.stdout:max_osd 8 2023-09-23T20:22:41.317 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0 up in weight 1 up_from 94 up_thru 109 down_at 91 last_clean_interval [8,90) [v2:172.21.15.18:6802/990458588,v1:172.21.15.18:6803/990458588] [v2:172.21.15.18:6804/990458588,v1:172.21.15.18:6805/990458588] exists,up 44c8aaa3-bbd1-4330-8d34-73f7a08b0d4d 2023-09-23T20:22:41.317 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1 up in weight 1 up_from 99 up_thru 109 down_at 96 last_clean_interval [13,95) [v2:172.21.15.18:6810/1422592526,v1:172.21.15.18:6811/1422592526] [v2:172.21.15.18:6812/1422592526,v1:172.21.15.18:6813/1422592526] exists,up c0534324-49c8-4804-bced-c4d13510e74a 2023-09-23T20:22:41.317 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2 up in weight 1 up_from 104 up_thru 109 down_at 101 last_clean_interval [18,100) [v2:172.21.15.18:6818/2061051826,v1:172.21.15.18:6819/2061051826] [v2:172.21.15.18:6820/2061051826,v1:172.21.15.18:6821/2061051826] exists,up f3b2116d-a3d5-4279-91dc-eb47458d3239 2023-09-23T20:22:41.317 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3 up in weight 1 up_from 109 up_thru 109 down_at 106 last_clean_interval [25,105) [v2:172.21.15.18:6826/2876439245,v1:172.21.15.18:6827/2876439245] [v2:172.21.15.18:6828/2876439245,v1:172.21.15.18:6829/2876439245] exists,up 8edccb7f-c4c9-47f3-a6f5-38a1a38cb099 2023-09-23T20:22:41.317 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4 up in weight 1 up_from 30 up_thru 109 down_at 0 last_clean_interval [0,0) [v2:172.21.15.149:6800/3640432477,v1:172.21.15.149:6801/3640432477] [v2:172.21.15.149:6802/3640432477,v1:172.21.15.149:6803/3640432477] exists,up 1d5a64ce-fe42-4355-991a-dfb20cbe5423 2023-09-23T20:22:41.317 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5 up in weight 1 up_from 39 up_thru 109 down_at 0 last_clean_interval [0,0) [v2:172.21.15.149:6808/2969709367,v1:172.21.15.149:6809/2969709367] [v2:172.21.15.149:6810/2969709367,v1:172.21.15.149:6811/2969709367] exists,up 105ae3b2-b2fc-477d-9134-3d517bc9427e 2023-09-23T20:22:41.317 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6 up in weight 1 up_from 47 up_thru 106 down_at 0 last_clean_interval [0,0) [v2:172.21.15.149:6816/1291453593,v1:172.21.15.149:6817/1291453593] [v2:172.21.15.149:6818/1291453593,v1:172.21.15.149:6819/1291453593] exists,up 27d2d2a8-e8c6-4a05-a85e-bd54376cf4c1 2023-09-23T20:22:41.317 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7 up in weight 1 up_from 55 up_thru 109 down_at 0 last_clean_interval [0,0) [v2:172.21.15.149:6824/3117985178,v1:172.21.15.149:6825/3117985178] [v2:172.21.15.149:6826/3117985178,v1:172.21.15.149:6827/3117985178] exists,up 9da636f7-995c-4f29-9264-70d09c6f66c6 2023-09-23T20:22:41.317 INFO:teuthology.orchestra.run.smithi018.stdout:pg_upmap_items 4.d [1,5] 2023-09-23T20:22:41.317 INFO:teuthology.orchestra.run.smithi018.stdout:pg_upmap_items 4.f [1,2] 2023-09-23T20:22:41.318 INFO:teuthology.orchestra.run.smithi018.stdout:pg_upmap_items 4.18 [1,2] 2023-09-23T20:22:41.318 INFO:teuthology.orchestra.run.smithi018.stdout:blocklist 172.21.15.18:6801/2791912398 expires 2023-09-24T20:15:59.309691+0000 2023-09-23T20:22:41.318 INFO:teuthology.orchestra.run.smithi018.stdout:blocklist 172.21.15.149:0/502112376 expires 2023-09-24T20:15:19.879895+0000 2023-09-23T20:22:41.318 INFO:teuthology.orchestra.run.smithi018.stdout:blocklist 172.21.15.18:0/69122222 expires 2023-09-24T20:07:23.919815+0000 2023-09-23T20:22:41.318 INFO:teuthology.orchestra.run.smithi018.stdout:blocklist 172.21.15.149:6835/1754402171 expires 2023-09-24T20:15:19.879895+0000 2023-09-23T20:22:41.318 INFO:teuthology.orchestra.run.smithi018.stdout:blocklist 172.21.15.18:0/3831267686 expires 2023-09-24T20:07:48.635784+0000 2023-09-23T20:22:41.318 INFO:teuthology.orchestra.run.smithi018.stdout:blocklist 172.21.15.149:0/3406344898 expires 2023-09-24T20:15:19.879895+0000 2023-09-23T20:22:41.318 INFO:teuthology.orchestra.run.smithi018.stdout:blocklist 172.21.15.149:6834/1754402171 expires 2023-09-24T20:15:19.879895+0000 2023-09-23T20:22:41.318 INFO:teuthology.orchestra.run.smithi018.stdout:blocklist 172.21.15.18:0/1988472148 expires 2023-09-24T20:11:49.652762+0000 2023-09-23T20:22:41.318 INFO:teuthology.orchestra.run.smithi018.stdout:blocklist 172.21.15.18:0/1082665289 expires 2023-09-24T20:07:48.635784+0000 2023-09-23T20:22:41.318 INFO:teuthology.orchestra.run.smithi018.stdout:blocklist 172.21.15.18:6800/1930027051 expires 2023-09-24T20:07:23.919815+0000 2023-09-23T20:22:41.318 INFO:teuthology.orchestra.run.smithi018.stdout:blocklist 172.21.15.18:0/2809002822 expires 2023-09-24T20:14:38.972322+0000 2023-09-23T20:22:41.319 INFO:teuthology.orchestra.run.smithi018.stdout:blocklist 172.21.15.18:6801/2611527229 expires 2023-09-24T20:11:49.652762+0000 2023-09-23T20:22:41.319 INFO:teuthology.orchestra.run.smithi018.stdout:blocklist 172.21.15.149:0/2486656126 expires 2023-09-24T20:15:19.879895+0000 2023-09-23T20:22:41.319 INFO:teuthology.orchestra.run.smithi018.stdout:blocklist 172.21.15.18:0/1119348530 expires 2023-09-24T20:15:59.309691+0000 2023-09-23T20:22:41.319 INFO:teuthology.orchestra.run.smithi018.stdout:blocklist 172.21.15.18:6800/2611527229 expires 2023-09-24T20:11:49.652762+0000 2023-09-23T20:22:41.319 INFO:teuthology.orchestra.run.smithi018.stdout:blocklist 172.21.15.18:0/2543728509 expires 2023-09-24T20:14:38.972322+0000 2023-09-23T20:22:41.319 INFO:teuthology.orchestra.run.smithi018.stdout:blocklist 172.21.15.18:0/2993111051 expires 2023-09-24T20:07:23.919815+0000 2023-09-23T20:22:41.319 INFO:teuthology.orchestra.run.smithi018.stdout:blocklist 172.21.15.18:0/3990640673 expires 2023-09-24T20:14:38.972322+0000 2023-09-23T20:22:41.319 INFO:teuthology.orchestra.run.smithi018.stdout:blocklist 172.21.15.18:0/1698512393 expires 2023-09-24T20:11:49.652762+0000 2023-09-23T20:22:41.319 INFO:teuthology.orchestra.run.smithi018.stdout:blocklist 172.21.15.149:0/894377242 expires 2023-09-24T20:15:19.879895+0000 2023-09-23T20:22:41.319 INFO:teuthology.orchestra.run.smithi018.stdout:blocklist 172.21.15.18:0/3356195690 expires 2023-09-24T20:14:38.972322+0000 2023-09-23T20:22:41.319 INFO:teuthology.orchestra.run.smithi018.stdout:blocklist 172.21.15.18:6801/1930027051 expires 2023-09-24T20:07:23.919815+0000 2023-09-23T20:22:41.319 INFO:teuthology.orchestra.run.smithi018.stdout:blocklist 172.21.15.18:6801/380813540 expires 2023-09-24T20:07:48.635784+0000 2023-09-23T20:22:41.319 INFO:teuthology.orchestra.run.smithi018.stdout:blocklist 172.21.15.18:0/3863388488 expires 2023-09-24T20:11:49.652762+0000 2023-09-23T20:22:41.320 INFO:teuthology.orchestra.run.smithi018.stdout:blocklist 172.21.15.18:0/2549145961 expires 2023-09-24T20:15:59.309691+0000 2023-09-23T20:22:41.320 INFO:teuthology.orchestra.run.smithi018.stdout:blocklist 172.21.15.149:0/1078964609 expires 2023-09-24T20:15:19.879895+0000 2023-09-23T20:22:41.320 INFO:teuthology.orchestra.run.smithi018.stdout:blocklist 172.21.15.18:0/1645667747 expires 2023-09-24T20:07:48.635784+0000 2023-09-23T20:22:41.320 INFO:teuthology.orchestra.run.smithi018.stdout:blocklist 172.21.15.18:6800/380813540 expires 2023-09-24T20:07:48.635784+0000 2023-09-23T20:22:41.320 INFO:teuthology.orchestra.run.smithi018.stdout:blocklist 172.21.15.18:6800/115279545 expires 2023-09-24T20:14:38.972322+0000 2023-09-23T20:22:41.320 INFO:teuthology.orchestra.run.smithi018.stdout:blocklist 172.21.15.18:0/3510203881 expires 2023-09-24T20:15:59.309691+0000 2023-09-23T20:22:41.320 INFO:teuthology.orchestra.run.smithi018.stdout:blocklist 172.21.15.18:6801/115279545 expires 2023-09-24T20:14:38.972322+0000 2023-09-23T20:22:41.320 INFO:teuthology.orchestra.run.smithi018.stdout:blocklist 172.21.15.18:0/950924496 expires 2023-09-24T20:07:23.919815+0000 2023-09-23T20:22:41.320 INFO:teuthology.orchestra.run.smithi018.stdout:blocklist 172.21.15.149:0/1653842667 expires 2023-09-24T20:15:19.879895+0000 2023-09-23T20:22:41.320 INFO:teuthology.orchestra.run.smithi018.stdout:blocklist 172.21.15.18:0/3334666861 expires 2023-09-24T20:15:59.309691+0000 2023-09-23T20:22:41.320 INFO:teuthology.orchestra.run.smithi018.stdout:blocklist 172.21.15.18:0/3912151186 expires 2023-09-24T20:11:49.652762+0000 2023-09-23T20:22:41.320 INFO:teuthology.orchestra.run.smithi018.stdout:blocklist 172.21.15.18:0/872601545 expires 2023-09-24T20:15:59.309691+0000 2023-09-23T20:22:41.320 INFO:teuthology.orchestra.run.smithi018.stdout:blocklist 172.21.15.18:0/1505013437 expires 2023-09-24T20:15:59.309691+0000 2023-09-23T20:22:41.321 INFO:teuthology.orchestra.run.smithi018.stdout:blocklist 172.21.15.18:6800/2791912398 expires 2023-09-24T20:15:59.309691+0000 2023-09-23T20:22:41.344 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:22:40 smithi018 ceph-mon[184144]: pgmap v223: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:22:41.344 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:22:40 smithi018 ceph-mon[179647]: pgmap v223: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:22:41.779 INFO:tasks.cephadm.ceph_manager.ceph:['osd.0 up in weight 1 up_from 94 up_thru 109 down_at 91 last_clean_interval [8,90) [v2:172.21.15.18:6802/990458588,v1:172.21.15.18:6803/990458588] [v2:172.21.15.18:6804/990458588,v1:172.21.15.18:6805/990458588] exists,up 44c8aaa3-bbd1-4330-8d34-73f7a08b0d4d', 'osd.1 up in weight 1 up_from 99 up_thru 109 down_at 96 last_clean_interval [13,95) [v2:172.21.15.18:6810/1422592526,v1:172.21.15.18:6811/1422592526] [v2:172.21.15.18:6812/1422592526,v1:172.21.15.18:6813/1422592526] exists,up c0534324-49c8-4804-bced-c4d13510e74a', 'osd.2 up in weight 1 up_from 104 up_thru 109 down_at 101 last_clean_interval [18,100) [v2:172.21.15.18:6818/2061051826,v1:172.21.15.18:6819/2061051826] [v2:172.21.15.18:6820/2061051826,v1:172.21.15.18:6821/2061051826] exists,up f3b2116d-a3d5-4279-91dc-eb47458d3239', 'osd.3 up in weight 1 up_from 109 up_thru 109 down_at 106 last_clean_interval [25,105) [v2:172.21.15.18:6826/2876439245,v1:172.21.15.18:6827/2876439245] [v2:172.21.15.18:6828/2876439245,v1:172.21.15.18:6829/2876439245] exists,up 8edccb7f-c4c9-47f3-a6f5-38a1a38cb099', 'osd.4 up in weight 1 up_from 30 up_thru 109 down_at 0 last_clean_interval [0,0) [v2:172.21.15.149:6800/3640432477,v1:172.21.15.149:6801/3640432477] [v2:172.21.15.149:6802/3640432477,v1:172.21.15.149:6803/3640432477] exists,up 1d5a64ce-fe42-4355-991a-dfb20cbe5423', 'osd.5 up in weight 1 up_from 39 up_thru 109 down_at 0 last_clean_interval [0,0) [v2:172.21.15.149:6808/2969709367,v1:172.21.15.149:6809/2969709367] [v2:172.21.15.149:6810/2969709367,v1:172.21.15.149:6811/2969709367] exists,up 105ae3b2-b2fc-477d-9134-3d517bc9427e', 'osd.6 up in weight 1 up_from 47 up_thru 106 down_at 0 last_clean_interval [0,0) [v2:172.21.15.149:6816/1291453593,v1:172.21.15.149:6817/1291453593] [v2:172.21.15.149:6818/1291453593,v1:172.21.15.149:6819/1291453593] exists,up 27d2d2a8-e8c6-4a05-a85e-bd54376cf4c1', 'osd.7 up in weight 1 up_from 55 up_thru 109 down_at 0 last_clean_interval [0,0) [v2:172.21.15.149:6824/3117985178,v1:172.21.15.149:6825/3117985178] [v2:172.21.15.149:6826/3117985178,v1:172.21.15.149:6827/3117985178] exists,up 9da636f7-995c-4f29-9264-70d09c6f66c6'] 2023-09-23T20:22:41.779 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon mon.a config show 2023-09-23T20:22:42.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:22:41 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1440441914' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T20:22:42.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:22:41 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1440441914' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T20:22:42.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:22:41 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1440441914' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T20:22:42.387 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:22:43.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:22:42 smithi018 ceph-mon[184144]: pgmap v224: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:22:43.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:22:42 smithi018 ceph-mon[179647]: pgmap v224: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:22:43.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:22:42 smithi149 ceph-mon[170394]: pgmap v224: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:22:43.609 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2023-09-23T20:22:43.609 INFO:teuthology.orchestra.run.smithi018.stdout: "name": "mon.a", 2023-09-23T20:22:43.609 INFO:teuthology.orchestra.run.smithi018.stdout: "cluster": "ceph", 2023-09-23T20:22:43.609 INFO:teuthology.orchestra.run.smithi018.stdout: "admin_socket": "/var/run/ceph/ceph-mon.a.asok", 2023-09-23T20:22:43.609 INFO:teuthology.orchestra.run.smithi018.stdout: "admin_socket_mode": "", 2023-09-23T20:22:43.610 INFO:teuthology.orchestra.run.smithi018.stdout: "auth_allow_insecure_global_id_reclaim": "false", 2023-09-23T20:22:43.610 INFO:teuthology.orchestra.run.smithi018.stdout: "auth_client_required": "cephx, none", 2023-09-23T20:22:43.610 INFO:teuthology.orchestra.run.smithi018.stdout: "auth_cluster_required": "cephx", 2023-09-23T20:22:43.610 INFO:teuthology.orchestra.run.smithi018.stdout: "auth_debug": "true", 2023-09-23T20:22:43.610 INFO:teuthology.orchestra.run.smithi018.stdout: "auth_expose_insecure_global_id_reclaim": "true", 2023-09-23T20:22:43.610 INFO:teuthology.orchestra.run.smithi018.stdout: "auth_mon_ticket_ttl": "660.000000", 2023-09-23T20:22:43.610 INFO:teuthology.orchestra.run.smithi018.stdout: "auth_service_required": "cephx", 2023-09-23T20:22:43.610 INFO:teuthology.orchestra.run.smithi018.stdout: "auth_service_ticket_ttl": "240.000000", 2023-09-23T20:22:43.610 INFO:teuthology.orchestra.run.smithi018.stdout: "auth_supported": "", 2023-09-23T20:22:43.610 INFO:teuthology.orchestra.run.smithi018.stdout: "bdev_aio": "true", 2023-09-23T20:22:43.610 INFO:teuthology.orchestra.run.smithi018.stdout: "bdev_aio_max_queue_depth": "1024", 2023-09-23T20:22:43.610 INFO:teuthology.orchestra.run.smithi018.stdout: "bdev_aio_poll_ms": "250", 2023-09-23T20:22:43.611 INFO:teuthology.orchestra.run.smithi018.stdout: "bdev_aio_reap_max": "16", 2023-09-23T20:22:43.611 INFO:teuthology.orchestra.run.smithi018.stdout: "bdev_async_discard": "false", 2023-09-23T20:22:43.613 INFO:teuthology.orchestra.run.smithi018.stdout: "bdev_block_size": "4096", 2023-09-23T20:22:43.613 INFO:teuthology.orchestra.run.smithi018.stdout: "bdev_debug_aio": "false", 2023-09-23T20:22:43.613 INFO:teuthology.orchestra.run.smithi018.stdout: "bdev_debug_aio_log_age": "5.000000", 2023-09-23T20:22:43.613 INFO:teuthology.orchestra.run.smithi018.stdout: "bdev_debug_aio_suicide_timeout": "60.000000", 2023-09-23T20:22:43.613 INFO:teuthology.orchestra.run.smithi018.stdout: "bdev_debug_inflight_ios": "false", 2023-09-23T20:22:43.613 INFO:teuthology.orchestra.run.smithi018.stdout: "bdev_enable_discard": "false", 2023-09-23T20:22:43.613 INFO:teuthology.orchestra.run.smithi018.stdout: "bdev_flock_retry": "3", 2023-09-23T20:22:43.613 INFO:teuthology.orchestra.run.smithi018.stdout: "bdev_flock_retry_interval": "0.100000", 2023-09-23T20:22:43.613 INFO:teuthology.orchestra.run.smithi018.stdout: "bdev_inject_crash": "0", 2023-09-23T20:22:43.613 INFO:teuthology.orchestra.run.smithi018.stdout: "bdev_inject_crash_flush_delay": "2", 2023-09-23T20:22:43.614 INFO:teuthology.orchestra.run.smithi018.stdout: "bdev_ioring": "false", 2023-09-23T20:22:43.614 INFO:teuthology.orchestra.run.smithi018.stdout: "bdev_ioring_hipri": "false", 2023-09-23T20:22:43.614 INFO:teuthology.orchestra.run.smithi018.stdout: "bdev_ioring_sqthread_poll": "false", 2023-09-23T20:22:43.614 INFO:teuthology.orchestra.run.smithi018.stdout: "bdev_nvme_unbind_from_kernel": "false", 2023-09-23T20:22:43.614 INFO:teuthology.orchestra.run.smithi018.stdout: "bdev_read_buffer_alignment": "4096", 2023-09-23T20:22:43.614 INFO:teuthology.orchestra.run.smithi018.stdout: "bdev_read_preallocated_huge_buffers": "", 2023-09-23T20:22:43.614 INFO:teuthology.orchestra.run.smithi018.stdout: "bdev_type": "", 2023-09-23T20:22:43.614 INFO:teuthology.orchestra.run.smithi018.stdout: "bluefs_alloc_size": "1048576", 2023-09-23T20:22:43.614 INFO:teuthology.orchestra.run.smithi018.stdout: "bluefs_allocator": "hybrid", 2023-09-23T20:22:43.614 INFO:teuthology.orchestra.run.smithi018.stdout: "bluefs_buffered_io": "true", 2023-09-23T20:22:43.614 INFO:teuthology.orchestra.run.smithi018.stdout: "bluefs_check_for_zeros": "false", 2023-09-23T20:22:43.615 INFO:teuthology.orchestra.run.smithi018.stdout: "bluefs_check_volume_selector_often": "false", 2023-09-23T20:22:43.615 INFO:teuthology.orchestra.run.smithi018.stdout: "bluefs_check_volume_selector_on_umount": "false", 2023-09-23T20:22:43.615 INFO:teuthology.orchestra.run.smithi018.stdout: "bluefs_compact_log_sync": "false", 2023-09-23T20:22:43.615 INFO:teuthology.orchestra.run.smithi018.stdout: "bluefs_failed_shared_alloc_cooldown": "600.000000", 2023-09-23T20:22:43.615 INFO:teuthology.orchestra.run.smithi018.stdout: "bluefs_log_compact_min_ratio": "5.000000", 2023-09-23T20:22:43.615 INFO:teuthology.orchestra.run.smithi018.stdout: "bluefs_log_compact_min_size": "16777216", 2023-09-23T20:22:43.615 INFO:teuthology.orchestra.run.smithi018.stdout: "bluefs_log_replay_check_allocations": "true", 2023-09-23T20:22:43.615 INFO:teuthology.orchestra.run.smithi018.stdout: "bluefs_max_log_runway": "4194304", 2023-09-23T20:22:43.615 INFO:teuthology.orchestra.run.smithi018.stdout: "bluefs_max_prefetch": "1048576", 2023-09-23T20:22:43.615 INFO:teuthology.orchestra.run.smithi018.stdout: "bluefs_min_flush_size": "524288", 2023-09-23T20:22:43.615 INFO:teuthology.orchestra.run.smithi018.stdout: "bluefs_min_log_runway": "1048576", 2023-09-23T20:22:43.615 INFO:teuthology.orchestra.run.smithi018.stdout: "bluefs_replay_recovery": "false", 2023-09-23T20:22:43.615 INFO:teuthology.orchestra.run.smithi018.stdout: "bluefs_replay_recovery_disable_compact": "false", 2023-09-23T20:22:43.616 INFO:teuthology.orchestra.run.smithi018.stdout: "bluefs_shared_alloc_size": "65536", 2023-09-23T20:22:43.616 INFO:teuthology.orchestra.run.smithi018.stdout: "bluefs_sync_write": "false", 2023-09-23T20:22:43.616 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_2q_cache_kin_ratio": "0.500000", 2023-09-23T20:22:43.616 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_2q_cache_kout_ratio": "0.500000", 2023-09-23T20:22:43.616 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_alloc_stats_dump_interval": "86400.000000", 2023-09-23T20:22:43.616 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_allocation_from_file": "true", 2023-09-23T20:22:43.616 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_allocator": "hybrid", 2023-09-23T20:22:43.616 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_avl_alloc_bf_free_pct": "4", 2023-09-23T20:22:43.616 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_avl_alloc_bf_threshold": "131072", 2023-09-23T20:22:43.616 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_avl_alloc_ff_max_search_bytes": "16777216", 2023-09-23T20:22:43.616 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_avl_alloc_ff_max_search_count": "100", 2023-09-23T20:22:43.617 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_bitmapallocator_blocks_per_zone": "1024", 2023-09-23T20:22:43.617 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_bitmapallocator_span_size": "1024", 2023-09-23T20:22:43.617 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_blobid_prealloc": "10240", 2023-09-23T20:22:43.617 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_block_create": "true", 2023-09-23T20:22:43.617 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_block_db_create": "false", 2023-09-23T20:22:43.617 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_block_db_path": "", 2023-09-23T20:22:43.617 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_block_db_size": "0", 2023-09-23T20:22:43.617 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_block_path": "", 2023-09-23T20:22:43.617 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_block_preallocate_file": "false", 2023-09-23T20:22:43.617 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_block_size": "107374182400", 2023-09-23T20:22:43.617 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_block_wal_create": "false", 2023-09-23T20:22:43.617 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_block_wal_path": "", 2023-09-23T20:22:43.618 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_block_wal_size": "100663296", 2023-09-23T20:22:43.618 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_bluefs": "true", 2023-09-23T20:22:43.618 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_bluefs_alloc_failure_dump_interval": "0.000000", 2023-09-23T20:22:43.618 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_bluefs_env_mirror": "false", 2023-09-23T20:22:43.618 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_bluefs_max_free": "10737418240", 2023-09-23T20:22:43.618 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_cache_age_bin_interval": "1.000000", 2023-09-23T20:22:43.618 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_cache_age_bins_data": "1 2 6 24 120 720 0 0 0 0", 2023-09-23T20:22:43.618 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_cache_age_bins_kv": "1 2 6 24 120 720 0 0 0 0", 2023-09-23T20:22:43.618 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_cache_age_bins_kv_onode": "0 0 0 0 0 0 0 0 0 720", 2023-09-23T20:22:43.618 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_cache_age_bins_meta": "1 2 6 24 120 720 0 0 0 0", 2023-09-23T20:22:43.618 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_cache_autotune": "true", 2023-09-23T20:22:43.619 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_cache_autotune_interval": "5.000000", 2023-09-23T20:22:43.619 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_cache_kv_onode_ratio": "0.040000", 2023-09-23T20:22:43.619 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_cache_kv_ratio": "0.450000", 2023-09-23T20:22:43.619 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_cache_meta_ratio": "0.450000", 2023-09-23T20:22:43.619 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_cache_size": "0", 2023-09-23T20:22:43.619 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_cache_size_hdd": "1073741824", 2023-09-23T20:22:43.619 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_cache_size_ssd": "3221225472", 2023-09-23T20:22:43.619 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_cache_trim_interval": "0.050000", 2023-09-23T20:22:43.619 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_cache_trim_max_skip_pinned": "1000", 2023-09-23T20:22:43.619 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_cache_type": "2q", 2023-09-23T20:22:43.619 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_cleaner_sleep_interval": "5.000000", 2023-09-23T20:22:43.619 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_clone_cow": "true", 2023-09-23T20:22:43.620 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_compression_algorithm": "snappy", 2023-09-23T20:22:43.620 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_compression_max_blob_size": "0", 2023-09-23T20:22:43.620 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_compression_max_blob_size_hdd": "65536", 2023-09-23T20:22:43.620 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_compression_max_blob_size_ssd": "65536", 2023-09-23T20:22:43.620 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_compression_min_blob_size": "0", 2023-09-23T20:22:43.620 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_compression_min_blob_size_hdd": "8192", 2023-09-23T20:22:43.620 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_compression_min_blob_size_ssd": "65536", 2023-09-23T20:22:43.620 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_compression_mode": "none", 2023-09-23T20:22:43.620 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_compression_required_ratio": "0.875000", 2023-09-23T20:22:43.620 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_csum_type": "crc32c", 2023-09-23T20:22:43.620 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_debug_enforce_settings": "default", 2023-09-23T20:22:43.620 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_debug_freelist": "false", 2023-09-23T20:22:43.621 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_debug_fsck_abort": "false", 2023-09-23T20:22:43.621 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_debug_inject_allocation_from_file_failure": "0.000000", 2023-09-23T20:22:43.621 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_debug_inject_bug21040": "false", 2023-09-23T20:22:43.621 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_debug_inject_csum_err_probability": "0.000000", 2023-09-23T20:22:43.621 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_debug_inject_read_err": "false", 2023-09-23T20:22:43.621 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_debug_legacy_omap": "false", 2023-09-23T20:22:43.621 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_debug_no_reuse_blocks": "false", 2023-09-23T20:22:43.621 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_debug_omit_block_device_write": "false", 2023-09-23T20:22:43.621 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_debug_omit_kv_commit": "false", 2023-09-23T20:22:43.621 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_debug_permit_any_bdev_label": "false", 2023-09-23T20:22:43.621 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_debug_prefill": "0.000000", 2023-09-23T20:22:43.621 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_debug_prefragment_max": "1048576", 2023-09-23T20:22:43.622 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_debug_random_read_err": "0.000000", 2023-09-23T20:22:43.622 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_debug_randomize_serial_transaction": "0", 2023-09-23T20:22:43.622 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_debug_small_allocations": "0", 2023-09-23T20:22:43.622 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_debug_too_many_blobs_threshold": "24576", 2023-09-23T20:22:43.622 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_default_buffered_read": "true", 2023-09-23T20:22:43.622 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_default_buffered_write": "false", 2023-09-23T20:22:43.622 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_deferred_batch_ops": "0", 2023-09-23T20:22:43.622 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_deferred_batch_ops_hdd": "64", 2023-09-23T20:22:43.622 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_deferred_batch_ops_ssd": "16", 2023-09-23T20:22:43.622 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_extent_map_inline_shard_prealloc_size": "256", 2023-09-23T20:22:43.622 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_extent_map_shard_max_size": "1200", 2023-09-23T20:22:43.623 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_extent_map_shard_min_size": "150", 2023-09-23T20:22:43.623 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_extent_map_shard_target_size": "500", 2023-09-23T20:22:43.623 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_extent_map_shard_target_size_slop": "0.200000", 2023-09-23T20:22:43.623 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_fail_eio": "false", 2023-09-23T20:22:43.623 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_freelist_blocks_per_key": "128", 2023-09-23T20:22:43.623 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_fsck_error_on_no_per_pg_omap": "false", 2023-09-23T20:22:43.623 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_fsck_error_on_no_per_pool_omap": "false", 2023-09-23T20:22:43.623 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_fsck_error_on_no_per_pool_stats": "false", 2023-09-23T20:22:43.623 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_fsck_on_mkfs": "true", 2023-09-23T20:22:43.623 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_fsck_on_mkfs_deep": "false", 2023-09-23T20:22:43.623 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_fsck_on_mount": "false", 2023-09-23T20:22:43.623 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_fsck_on_mount_deep": "false", 2023-09-23T20:22:43.623 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_fsck_on_umount": "false", 2023-09-23T20:22:43.624 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_fsck_on_umount_deep": "false", 2023-09-23T20:22:43.624 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_fsck_quick_fix_on_mount": "false", 2023-09-23T20:22:43.624 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_fsck_quick_fix_threads": "2", 2023-09-23T20:22:43.624 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_fsck_read_bytes_cap": "67108864", 2023-09-23T20:22:43.624 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_fsck_shared_blob_tracker_size": "0.031250", 2023-09-23T20:22:43.624 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_gc_enable_blob_threshold": "0", 2023-09-23T20:22:43.624 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_gc_enable_total_threshold": "0", 2023-09-23T20:22:43.624 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_hybrid_alloc_mem_cap": "67108864", 2023-09-23T20:22:43.624 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_ignore_data_csum": "false", 2023-09-23T20:22:43.624 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_kv_sync_util_logging_s": "10.000000", 2023-09-23T20:22:43.624 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_kvbackend": "rocksdb", 2023-09-23T20:22:43.625 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_log_collection_list_age": "60.000000", 2023-09-23T20:22:43.625 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_log_omap_iterator_age": "5.000000", 2023-09-23T20:22:43.625 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_log_op_age": "5.000000", 2023-09-23T20:22:43.625 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_max_alloc_size": "0", 2023-09-23T20:22:43.625 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_max_blob_size": "0", 2023-09-23T20:22:43.625 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_max_blob_size_hdd": "65536", 2023-09-23T20:22:43.625 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_max_blob_size_ssd": "65536", 2023-09-23T20:22:43.625 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_max_defer_interval": "3.000000", 2023-09-23T20:22:43.626 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_max_deferred_txc": "32", 2023-09-23T20:22:43.626 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_min_alloc_size": "0", 2023-09-23T20:22:43.626 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_min_alloc_size_hdd": "4096", 2023-09-23T20:22:43.626 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_min_alloc_size_ssd": "4096", 2023-09-23T20:22:43.626 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_nid_prealloc": "1024", 2023-09-23T20:22:43.626 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_prefer_deferred_size": "0", 2023-09-23T20:22:43.626 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_prefer_deferred_size_hdd": "65536", 2023-09-23T20:22:43.626 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_prefer_deferred_size_ssd": "0", 2023-09-23T20:22:43.626 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_qfsck_on_mount": "true", 2023-09-23T20:22:43.626 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_retry_disk_reads": "3", 2023-09-23T20:22:43.626 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_rocksdb_cf": "true", 2023-09-23T20:22:43.626 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_rocksdb_cfs": "m(3) p(3,0-12) O(3,0-13)=block_cache={type=binned_lru} L=min_write_buffer_number_to_merge=32 P=min_write_buffer_number_to_merge=32", 2023-09-23T20:22:43.627 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_rocksdb_options": "compression=kNoCompression,max_write_buffer_number=64,min_write_buffer_number_to_merge=6,compaction_style=kCompactionStyleLevel,write_buffer_size=16777216,max_background_jobs=4,level0_file_num_compaction_trigger=8,max_bytes_for_level_base=1073741824,max_bytes_for_level_multiplier=8,compaction_readahead_size=2MB,max_total_wal_size=1073741824,writable_file_max_buffer_size=0", 2023-09-23T20:22:43.627 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_rocksdb_options_annex": "", 2023-09-23T20:22:43.627 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_spdk_coremask": "0x1", 2023-09-23T20:22:43.627 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_spdk_io_sleep": "5", 2023-09-23T20:22:43.627 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_spdk_max_io_completion": "0", 2023-09-23T20:22:43.627 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_spdk_mem": "512", 2023-09-23T20:22:43.627 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_sync_submit_transaction": "false", 2023-09-23T20:22:43.627 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_throttle_bytes": "67108864", 2023-09-23T20:22:43.627 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_throttle_cost_per_io": "0", 2023-09-23T20:22:43.627 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_throttle_cost_per_io_hdd": "670000", 2023-09-23T20:22:43.627 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_throttle_cost_per_io_ssd": "4000", 2023-09-23T20:22:43.628 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_throttle_deferred_bytes": "134217728", 2023-09-23T20:22:43.628 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_throttle_trace_rate": "0.000000", 2023-09-23T20:22:43.628 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_tracing": "false", 2023-09-23T20:22:43.628 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_use_optimal_io_size_for_min_alloc_size": "false", 2023-09-23T20:22:43.628 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_volume_selection_policy": "use_some_extra", 2023-09-23T20:22:43.628 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_volume_selection_reserved": "0", 2023-09-23T20:22:43.628 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_volume_selection_reserved_factor": "2.000000", 2023-09-23T20:22:43.628 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_warn_on_bluefs_spillover": "true", 2023-09-23T20:22:43.628 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_warn_on_legacy_statfs": "true", 2023-09-23T20:22:43.628 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_warn_on_no_per_pg_omap": "false", 2023-09-23T20:22:43.628 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_warn_on_no_per_pool_omap": "true", 2023-09-23T20:22:43.628 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_warn_on_spurious_read_errors": "true", 2023-09-23T20:22:43.629 INFO:teuthology.orchestra.run.smithi018.stdout: "bluestore_zero_block_detection": "false", 2023-09-23T20:22:43.629 INFO:teuthology.orchestra.run.smithi018.stdout: "cephadm_path": "/usr/sbin/cephadm", 2023-09-23T20:22:43.629 INFO:teuthology.orchestra.run.smithi018.stdout: "cephfs_mirror_action_update_interval": "2", 2023-09-23T20:22:43.629 INFO:teuthology.orchestra.run.smithi018.stdout: "cephfs_mirror_directory_scan_interval": "10", 2023-09-23T20:22:43.629 INFO:teuthology.orchestra.run.smithi018.stdout: "cephfs_mirror_max_concurrent_directory_syncs": "3", 2023-09-23T20:22:43.629 INFO:teuthology.orchestra.run.smithi018.stdout: "cephfs_mirror_max_consecutive_failures_per_directory": "10", 2023-09-23T20:22:43.629 INFO:teuthology.orchestra.run.smithi018.stdout: "cephfs_mirror_max_snapshot_sync_per_cycle": "3", 2023-09-23T20:22:43.629 INFO:teuthology.orchestra.run.smithi018.stdout: "cephfs_mirror_mount_timeout": "10", 2023-09-23T20:22:43.629 INFO:teuthology.orchestra.run.smithi018.stdout: "cephfs_mirror_restart_mirror_on_blocklist_interval": "30", 2023-09-23T20:22:43.629 INFO:teuthology.orchestra.run.smithi018.stdout: "cephfs_mirror_restart_mirror_on_failure_interval": "20", 2023-09-23T20:22:43.629 INFO:teuthology.orchestra.run.smithi018.stdout: "cephfs_mirror_retry_failed_directories_interval": "60", 2023-09-23T20:22:43.629 INFO:teuthology.orchestra.run.smithi018.stdout: "cephsqlite_blocklist_dead_locker": "true", 2023-09-23T20:22:43.630 INFO:teuthology.orchestra.run.smithi018.stdout: "cephsqlite_lock_renewal_interval": "2000", 2023-09-23T20:22:43.630 INFO:teuthology.orchestra.run.smithi018.stdout: "cephsqlite_lock_renewal_timeout": "30000", 2023-09-23T20:22:43.630 INFO:teuthology.orchestra.run.smithi018.stdout: "cephx_cluster_require_signatures": "false", 2023-09-23T20:22:43.630 INFO:teuthology.orchestra.run.smithi018.stdout: "cephx_cluster_require_version": "2", 2023-09-23T20:22:43.630 INFO:teuthology.orchestra.run.smithi018.stdout: "cephx_require_signatures": "false", 2023-09-23T20:22:43.630 INFO:teuthology.orchestra.run.smithi018.stdout: "cephx_require_version": "2", 2023-09-23T20:22:43.630 INFO:teuthology.orchestra.run.smithi018.stdout: "cephx_service_require_signatures": "false", 2023-09-23T20:22:43.630 INFO:teuthology.orchestra.run.smithi018.stdout: "cephx_service_require_version": "2", 2023-09-23T20:22:43.630 INFO:teuthology.orchestra.run.smithi018.stdout: "cephx_sign_messages": "true", 2023-09-23T20:22:43.630 INFO:teuthology.orchestra.run.smithi018.stdout: "chdir": "", 2023-09-23T20:22:43.630 INFO:teuthology.orchestra.run.smithi018.stdout: "client_acl_type": "", 2023-09-23T20:22:43.631 INFO:teuthology.orchestra.run.smithi018.stdout: "client_asio_thread_count": "2", 2023-09-23T20:22:43.631 INFO:teuthology.orchestra.run.smithi018.stdout: "client_cache_mid": "0.750000", 2023-09-23T20:22:43.631 INFO:teuthology.orchestra.run.smithi018.stdout: "client_cache_size": "16384", 2023-09-23T20:22:43.631 INFO:teuthology.orchestra.run.smithi018.stdout: "client_caps_release_delay": "5", 2023-09-23T20:22:43.631 INFO:teuthology.orchestra.run.smithi018.stdout: "client_check_pool_perm": "true", 2023-09-23T20:22:43.631 INFO:teuthology.orchestra.run.smithi018.stdout: "client_collect_and_send_global_metrics": "false", 2023-09-23T20:22:43.631 INFO:teuthology.orchestra.run.smithi018.stdout: "client_debug_force_sync_read": "false", 2023-09-23T20:22:43.631 INFO:teuthology.orchestra.run.smithi018.stdout: "client_debug_getattr_caps": "false", 2023-09-23T20:22:43.631 INFO:teuthology.orchestra.run.smithi018.stdout: "client_debug_inject_tick_delay": "0", 2023-09-23T20:22:43.631 INFO:teuthology.orchestra.run.smithi018.stdout: "client_die_on_failed_dentry_invalidate": "true", 2023-09-23T20:22:43.632 INFO:teuthology.orchestra.run.smithi018.stdout: "client_die_on_failed_remount": "false", 2023-09-23T20:22:43.632 INFO:teuthology.orchestra.run.smithi018.stdout: "client_dirsize_rbytes": "true", 2023-09-23T20:22:43.632 INFO:teuthology.orchestra.run.smithi018.stdout: "client_force_lazyio": "false", 2023-09-23T20:22:43.632 INFO:teuthology.orchestra.run.smithi018.stdout: "client_fs": "", 2023-09-23T20:22:43.632 INFO:teuthology.orchestra.run.smithi018.stdout: "client_inject_fixed_oldest_tid": "false", 2023-09-23T20:22:43.632 INFO:teuthology.orchestra.run.smithi018.stdout: "client_inject_release_failure": "false", 2023-09-23T20:22:43.632 INFO:teuthology.orchestra.run.smithi018.stdout: "client_max_inline_size": "4096", 2023-09-23T20:22:43.632 INFO:teuthology.orchestra.run.smithi018.stdout: "client_max_retries_on_remount_failure": "5", 2023-09-23T20:22:43.632 INFO:teuthology.orchestra.run.smithi018.stdout: "client_mds_namespace": "", 2023-09-23T20:22:43.632 INFO:teuthology.orchestra.run.smithi018.stdout: "client_metadata": "", 2023-09-23T20:22:43.632 INFO:teuthology.orchestra.run.smithi018.stdout: "client_mount_gid": "-1", 2023-09-23T20:22:43.632 INFO:teuthology.orchestra.run.smithi018.stdout: "client_mount_timeout": "300", 2023-09-23T20:22:43.633 INFO:teuthology.orchestra.run.smithi018.stdout: "client_mount_uid": "-1", 2023-09-23T20:22:43.633 INFO:teuthology.orchestra.run.smithi018.stdout: "client_mountpoint": "/", 2023-09-23T20:22:43.633 INFO:teuthology.orchestra.run.smithi018.stdout: "client_notify_timeout": "10", 2023-09-23T20:22:43.633 INFO:teuthology.orchestra.run.smithi018.stdout: "client_oc": "true", 2023-09-23T20:22:43.633 INFO:teuthology.orchestra.run.smithi018.stdout: "client_oc_max_dirty": "104857600", 2023-09-23T20:22:43.633 INFO:teuthology.orchestra.run.smithi018.stdout: "client_oc_max_dirty_age": "5.000000", 2023-09-23T20:22:43.633 INFO:teuthology.orchestra.run.smithi018.stdout: "client_oc_max_objects": "1000", 2023-09-23T20:22:43.633 INFO:teuthology.orchestra.run.smithi018.stdout: "client_oc_size": "209715200", 2023-09-23T20:22:43.633 INFO:teuthology.orchestra.run.smithi018.stdout: "client_oc_target_dirty": "8388608", 2023-09-23T20:22:43.633 INFO:teuthology.orchestra.run.smithi018.stdout: "client_permissions": "true", 2023-09-23T20:22:43.633 INFO:teuthology.orchestra.run.smithi018.stdout: "client_quota": "true", 2023-09-23T20:22:43.634 INFO:teuthology.orchestra.run.smithi018.stdout: "client_quota_df": "true", 2023-09-23T20:22:43.634 INFO:teuthology.orchestra.run.smithi018.stdout: "client_readahead_max_bytes": "0", 2023-09-23T20:22:43.634 INFO:teuthology.orchestra.run.smithi018.stdout: "client_readahead_max_periods": "4", 2023-09-23T20:22:43.634 INFO:teuthology.orchestra.run.smithi018.stdout: "client_readahead_min": "131072", 2023-09-23T20:22:43.634 INFO:teuthology.orchestra.run.smithi018.stdout: "client_reconnect_stale": "false", 2023-09-23T20:22:43.634 INFO:teuthology.orchestra.run.smithi018.stdout: "client_shutdown_timeout": "30", 2023-09-23T20:22:43.634 INFO:teuthology.orchestra.run.smithi018.stdout: "client_snapdir": ".snap", 2023-09-23T20:22:43.634 INFO:teuthology.orchestra.run.smithi018.stdout: "client_tick_interval": "1", 2023-09-23T20:22:43.634 INFO:teuthology.orchestra.run.smithi018.stdout: "client_trace": "", 2023-09-23T20:22:43.634 INFO:teuthology.orchestra.run.smithi018.stdout: "client_try_dentry_invalidate": "false", 2023-09-23T20:22:43.634 INFO:teuthology.orchestra.run.smithi018.stdout: "client_use_faked_inos": "false", 2023-09-23T20:22:43.635 INFO:teuthology.orchestra.run.smithi018.stdout: "client_use_random_mds": "false", 2023-09-23T20:22:43.635 INFO:teuthology.orchestra.run.smithi018.stdout: "clog_to_graylog": "false", 2023-09-23T20:22:43.635 INFO:teuthology.orchestra.run.smithi018.stdout: "clog_to_graylog_host": "127.0.0.1", 2023-09-23T20:22:43.635 INFO:teuthology.orchestra.run.smithi018.stdout: "clog_to_graylog_port": "12201", 2023-09-23T20:22:43.635 INFO:teuthology.orchestra.run.smithi018.stdout: "clog_to_monitors": "default=true", 2023-09-23T20:22:43.635 INFO:teuthology.orchestra.run.smithi018.stdout: "clog_to_syslog": "false", 2023-09-23T20:22:43.635 INFO:teuthology.orchestra.run.smithi018.stdout: "clog_to_syslog_facility": "default=daemon audit=local0", 2023-09-23T20:22:43.635 INFO:teuthology.orchestra.run.smithi018.stdout: "clog_to_syslog_level": "info", 2023-09-23T20:22:43.635 INFO:teuthology.orchestra.run.smithi018.stdout: "cluster_addr": "-", 2023-09-23T20:22:43.635 INFO:teuthology.orchestra.run.smithi018.stdout: "cluster_network": "", 2023-09-23T20:22:43.635 INFO:teuthology.orchestra.run.smithi018.stdout: "cluster_network_interface": "", 2023-09-23T20:22:43.636 INFO:teuthology.orchestra.run.smithi018.stdout: "compressor_zlib_isal": "false", 2023-09-23T20:22:43.636 INFO:teuthology.orchestra.run.smithi018.stdout: "compressor_zlib_level": "5", 2023-09-23T20:22:43.636 INFO:teuthology.orchestra.run.smithi018.stdout: "compressor_zlib_winsize": "-15", 2023-09-23T20:22:43.636 INFO:teuthology.orchestra.run.smithi018.stdout: "compressor_zstd_level": "1", 2023-09-23T20:22:43.636 INFO:teuthology.orchestra.run.smithi018.stdout: "container_image": "quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d", 2023-09-23T20:22:43.636 INFO:teuthology.orchestra.run.smithi018.stdout: "crash_dir": "/var/lib/ceph/crash", 2023-09-23T20:22:43.636 INFO:teuthology.orchestra.run.smithi018.stdout: "crimson_alien_op_num_threads": "6", 2023-09-23T20:22:43.636 INFO:teuthology.orchestra.run.smithi018.stdout: "crimson_alien_thread_cpu_cores": "", 2023-09-23T20:22:43.636 INFO:teuthology.orchestra.run.smithi018.stdout: "crimson_osd_obc_lru_size": "10", 2023-09-23T20:22:43.636 INFO:teuthology.orchestra.run.smithi018.stdout: "crimson_osd_scheduler_concurrency": "0", 2023-09-23T20:22:43.636 INFO:teuthology.orchestra.run.smithi018.stdout: "crimson_seastar_smp": "1", 2023-09-23T20:22:43.636 INFO:teuthology.orchestra.run.smithi018.stdout: "crush_location": "", 2023-09-23T20:22:43.637 INFO:teuthology.orchestra.run.smithi018.stdout: "crush_location_hook": "", 2023-09-23T20:22:43.637 INFO:teuthology.orchestra.run.smithi018.stdout: "crush_location_hook_timeout": "10", 2023-09-23T20:22:43.637 INFO:teuthology.orchestra.run.smithi018.stdout: "daemonize": "false", 2023-09-23T20:22:43.637 INFO:teuthology.orchestra.run.smithi018.stdout: "daos_pool": "tank", 2023-09-23T20:22:43.637 INFO:teuthology.orchestra.run.smithi018.stdout: "dbstore_config_uri": "file:/var/lib/ceph/radosgw/dbstore-config.db", 2023-09-23T20:22:43.637 INFO:teuthology.orchestra.run.smithi018.stdout: "dbstore_db_dir": "/var/lib/ceph/radosgw", 2023-09-23T20:22:43.637 INFO:teuthology.orchestra.run.smithi018.stdout: "dbstore_db_name_prefix": "dbstore", 2023-09-23T20:22:43.637 INFO:teuthology.orchestra.run.smithi018.stdout: "debug_alienstore": "0/5", 2023-09-23T20:22:43.637 INFO:teuthology.orchestra.run.smithi018.stdout: "debug_allow_any_pool_priority": "false", 2023-09-23T20:22:43.637 INFO:teuthology.orchestra.run.smithi018.stdout: "debug_asok": "1/5", 2023-09-23T20:22:43.637 INFO:teuthology.orchestra.run.smithi018.stdout: "debug_asok_assert_abort": "false", 2023-09-23T20:22:43.637 INFO:teuthology.orchestra.run.smithi018.stdout: "debug_asserts_on_shutdown": "true", 2023-09-23T20:22:43.638 INFO:teuthology.orchestra.run.smithi018.stdout: "debug_auth": "1/5", 2023-09-23T20:22:43.638 INFO:teuthology.orchestra.run.smithi018.stdout: "debug_bdev": "1/3", 2023-09-23T20:22:43.638 INFO:teuthology.orchestra.run.smithi018.stdout: "debug_bluefs": "1/5", 2023-09-23T20:22:43.638 INFO:teuthology.orchestra.run.smithi018.stdout: "debug_bluestore": "1/5", 2023-09-23T20:22:43.638 INFO:teuthology.orchestra.run.smithi018.stdout: "debug_buffer": "0/1", 2023-09-23T20:22:43.638 INFO:teuthology.orchestra.run.smithi018.stdout: "debug_ceph_exporter": "1/5", 2023-09-23T20:22:43.638 INFO:teuthology.orchestra.run.smithi018.stdout: "debug_cephfs_mirror": "0/5", 2023-09-23T20:22:43.638 INFO:teuthology.orchestra.run.smithi018.stdout: "debug_cephsqlite": "0/5", 2023-09-23T20:22:43.638 INFO:teuthology.orchestra.run.smithi018.stdout: "debug_client": "0/5", 2023-09-23T20:22:43.638 INFO:teuthology.orchestra.run.smithi018.stdout: "debug_compressor": "1/5", 2023-09-23T20:22:43.638 INFO:teuthology.orchestra.run.smithi018.stdout: "debug_context": "0/1", 2023-09-23T20:22:43.639 INFO:teuthology.orchestra.run.smithi018.stdout: "debug_crush": "1/1", 2023-09-23T20:22:43.639 INFO:teuthology.orchestra.run.smithi018.stdout: "debug_crypto": "1/5", 2023-09-23T20:22:43.639 INFO:teuthology.orchestra.run.smithi018.stdout: "debug_cyanstore": "0/5", 2023-09-23T20:22:43.639 INFO:teuthology.orchestra.run.smithi018.stdout: "debug_deliberately_leak_memory": "false", 2023-09-23T20:22:43.639 INFO:teuthology.orchestra.run.smithi018.stdout: "debug_disable_randomized_ping": "false", 2023-09-23T20:22:43.639 INFO:teuthology.orchestra.run.smithi018.stdout: "debug_dpdk": "1/5", 2023-09-23T20:22:43.639 INFO:teuthology.orchestra.run.smithi018.stdout: "debug_eventtrace": "1/5", 2023-09-23T20:22:43.639 INFO:teuthology.orchestra.run.smithi018.stdout: "debug_filer": "0/1", 2023-09-23T20:22:43.639 INFO:teuthology.orchestra.run.smithi018.stdout: "debug_filestore": "1/3", 2023-09-23T20:22:43.639 INFO:teuthology.orchestra.run.smithi018.stdout: "debug_finisher": "1/1", 2023-09-23T20:22:43.639 INFO:teuthology.orchestra.run.smithi018.stdout: "debug_fuse": "1/5", 2023-09-23T20:22:43.639 INFO:teuthology.orchestra.run.smithi018.stdout: "debug_heartbeat_testing_span": "0", 2023-09-23T20:22:43.640 INFO:teuthology.orchestra.run.smithi018.stdout: "debug_heartbeatmap": "1/5", 2023-09-23T20:22:43.640 INFO:teuthology.orchestra.run.smithi018.stdout: "debug_immutable_obj_cache": "0/5", 2023-09-23T20:22:43.640 INFO:teuthology.orchestra.run.smithi018.stdout: "debug_javaclient": "1/5", 2023-09-23T20:22:43.640 INFO:teuthology.orchestra.run.smithi018.stdout: "debug_journal": "1/3", 2023-09-23T20:22:43.640 INFO:teuthology.orchestra.run.smithi018.stdout: "debug_journaler": "0/5", 2023-09-23T20:22:43.640 INFO:teuthology.orchestra.run.smithi018.stdout: "debug_kstore": "1/5", 2023-09-23T20:22:43.640 INFO:teuthology.orchestra.run.smithi018.stdout: "debug_leveldb": "4/5", 2023-09-23T20:22:43.640 INFO:teuthology.orchestra.run.smithi018.stdout: "debug_lockdep": "0/1", 2023-09-23T20:22:43.640 INFO:teuthology.orchestra.run.smithi018.stdout: "debug_mclock": "1/5", 2023-09-23T20:22:43.640 INFO:teuthology.orchestra.run.smithi018.stdout: "debug_mds": "1/5", 2023-09-23T20:22:43.641 INFO:teuthology.orchestra.run.smithi018.stdout: "debug_mds_balancer": "1/5", 2023-09-23T20:22:43.641 INFO:teuthology.orchestra.run.smithi018.stdout: "debug_mds_locker": "1/5", 2023-09-23T20:22:43.641 INFO:teuthology.orchestra.run.smithi018.stdout: "debug_mds_log": "1/5", 2023-09-23T20:22:43.641 INFO:teuthology.orchestra.run.smithi018.stdout: "debug_mds_log_expire": "1/5", 2023-09-23T20:22:43.641 INFO:teuthology.orchestra.run.smithi018.stdout: "debug_mds_migrator": "1/5", 2023-09-23T20:22:43.641 INFO:teuthology.orchestra.run.smithi018.stdout: "debug_memstore": "1/5", 2023-09-23T20:22:43.641 INFO:teuthology.orchestra.run.smithi018.stdout: "debug_mgr": "2/5", 2023-09-23T20:22:43.641 INFO:teuthology.orchestra.run.smithi018.stdout: "debug_mgrc": "1/5", 2023-09-23T20:22:43.641 INFO:teuthology.orchestra.run.smithi018.stdout: "debug_mon": "20/20", 2023-09-23T20:22:43.641 INFO:teuthology.orchestra.run.smithi018.stdout: "debug_monc": "0/10", 2023-09-23T20:22:43.641 INFO:teuthology.orchestra.run.smithi018.stdout: "debug_ms": "1/1", 2023-09-23T20:22:43.642 INFO:teuthology.orchestra.run.smithi018.stdout: "debug_none": "0/5", 2023-09-23T20:22:43.642 INFO:teuthology.orchestra.run.smithi018.stdout: "debug_objclass": "0/5", 2023-09-23T20:22:43.642 INFO:teuthology.orchestra.run.smithi018.stdout: "debug_objectcacher": "0/5", 2023-09-23T20:22:43.642 INFO:teuthology.orchestra.run.smithi018.stdout: "debug_objecter": "0/1", 2023-09-23T20:22:43.642 INFO:teuthology.orchestra.run.smithi018.stdout: "debug_optracker": "0/5", 2023-09-23T20:22:43.642 INFO:teuthology.orchestra.run.smithi018.stdout: "debug_osd": "1/5", 2023-09-23T20:22:43.642 INFO:teuthology.orchestra.run.smithi018.stdout: "debug_paxos": "20/20", 2023-09-23T20:22:43.642 INFO:teuthology.orchestra.run.smithi018.stdout: "debug_perfcounter": "1/5", 2023-09-23T20:22:43.642 INFO:teuthology.orchestra.run.smithi018.stdout: "debug_prioritycache": "1/5", 2023-09-23T20:22:43.642 INFO:teuthology.orchestra.run.smithi018.stdout: "debug_rados": "0/5", 2023-09-23T20:22:43.642 INFO:teuthology.orchestra.run.smithi018.stdout: "debug_rbd": "0/5", 2023-09-23T20:22:43.642 INFO:teuthology.orchestra.run.smithi018.stdout: "debug_rbd_mirror": "0/5", 2023-09-23T20:22:43.643 INFO:teuthology.orchestra.run.smithi018.stdout: "debug_rbd_pwl": "0/5", 2023-09-23T20:22:43.643 INFO:teuthology.orchestra.run.smithi018.stdout: "debug_rbd_replay": "0/5", 2023-09-23T20:22:43.643 INFO:teuthology.orchestra.run.smithi018.stdout: "debug_refs": "0/0", 2023-09-23T20:22:43.643 INFO:teuthology.orchestra.run.smithi018.stdout: "debug_reserver": "1/1", 2023-09-23T20:22:43.643 INFO:teuthology.orchestra.run.smithi018.stdout: "debug_rgw": "1/5", 2023-09-23T20:22:43.643 INFO:teuthology.orchestra.run.smithi018.stdout: "debug_rgw_access": "1/5", 2023-09-23T20:22:43.643 INFO:teuthology.orchestra.run.smithi018.stdout: "debug_rgw_datacache": "1/5", 2023-09-23T20:22:43.643 INFO:teuthology.orchestra.run.smithi018.stdout: "debug_rgw_dbstore": "1/5", 2023-09-23T20:22:43.643 INFO:teuthology.orchestra.run.smithi018.stdout: "debug_rgw_flight": "1/5", 2023-09-23T20:22:43.643 INFO:teuthology.orchestra.run.smithi018.stdout: "debug_rgw_sync": "1/5", 2023-09-23T20:22:43.643 INFO:teuthology.orchestra.run.smithi018.stdout: "debug_rocksdb": "4/5", 2023-09-23T20:22:43.643 INFO:teuthology.orchestra.run.smithi018.stdout: "debug_seastore": "0/5", 2023-09-23T20:22:43.644 INFO:teuthology.orchestra.run.smithi018.stdout: "debug_seastore_backref": "0/5", 2023-09-23T20:22:43.644 INFO:teuthology.orchestra.run.smithi018.stdout: "debug_seastore_cache": "0/5", 2023-09-23T20:22:43.644 INFO:teuthology.orchestra.run.smithi018.stdout: "debug_seastore_cleaner": "0/5", 2023-09-23T20:22:43.644 INFO:teuthology.orchestra.run.smithi018.stdout: "debug_seastore_device": "0/5", 2023-09-23T20:22:43.644 INFO:teuthology.orchestra.run.smithi018.stdout: "debug_seastore_epm": "0/5", 2023-09-23T20:22:43.644 INFO:teuthology.orchestra.run.smithi018.stdout: "debug_seastore_fixedkv_tree": "0/5", 2023-09-23T20:22:43.644 INFO:teuthology.orchestra.run.smithi018.stdout: "debug_seastore_journal": "0/5", 2023-09-23T20:22:43.644 INFO:teuthology.orchestra.run.smithi018.stdout: "debug_seastore_lba": "0/5", 2023-09-23T20:22:43.644 INFO:teuthology.orchestra.run.smithi018.stdout: "debug_seastore_odata": "0/5", 2023-09-23T20:22:43.644 INFO:teuthology.orchestra.run.smithi018.stdout: "debug_seastore_omap": "0/5", 2023-09-23T20:22:43.644 INFO:teuthology.orchestra.run.smithi018.stdout: "debug_seastore_onode": "0/5", 2023-09-23T20:22:43.645 INFO:teuthology.orchestra.run.smithi018.stdout: "debug_seastore_t": "0/5", 2023-09-23T20:22:43.645 INFO:teuthology.orchestra.run.smithi018.stdout: "debug_seastore_tm": "0/5", 2023-09-23T20:22:43.645 INFO:teuthology.orchestra.run.smithi018.stdout: "debug_striper": "0/1", 2023-09-23T20:22:43.645 INFO:teuthology.orchestra.run.smithi018.stdout: "debug_test": "0/5", 2023-09-23T20:22:43.645 INFO:teuthology.orchestra.run.smithi018.stdout: "debug_throttle": "1/1", 2023-09-23T20:22:43.645 INFO:teuthology.orchestra.run.smithi018.stdout: "debug_timer": "0/1", 2023-09-23T20:22:43.645 INFO:teuthology.orchestra.run.smithi018.stdout: "debug_tp": "0/5", 2023-09-23T20:22:43.645 INFO:teuthology.orchestra.run.smithi018.stdout: "device_failure_prediction_mode": "none", 2023-09-23T20:22:43.645 INFO:teuthology.orchestra.run.smithi018.stdout: "enable_experimental_unrecoverable_data_corrupting_features": "", 2023-09-23T20:22:43.645 INFO:teuthology.orchestra.run.smithi018.stdout: "erasure_code_dir": "/usr/lib64/ceph/erasure-code", 2023-09-23T20:22:43.645 INFO:teuthology.orchestra.run.smithi018.stdout: "err_to_graylog": "false", 2023-09-23T20:22:43.645 INFO:teuthology.orchestra.run.smithi018.stdout: "err_to_journald": "false", 2023-09-23T20:22:43.646 INFO:teuthology.orchestra.run.smithi018.stdout: "err_to_stderr": "true", 2023-09-23T20:22:43.646 INFO:teuthology.orchestra.run.smithi018.stdout: "err_to_syslog": "false", 2023-09-23T20:22:43.646 INFO:teuthology.orchestra.run.smithi018.stdout: "event_tracing": "false", 2023-09-23T20:22:43.646 INFO:teuthology.orchestra.run.smithi018.stdout: "exporter_addr": "0.0.0.0", 2023-09-23T20:22:43.646 INFO:teuthology.orchestra.run.smithi018.stdout: "exporter_http_port": "9926", 2023-09-23T20:22:43.646 INFO:teuthology.orchestra.run.smithi018.stdout: "exporter_prio_limit": "5", 2023-09-23T20:22:43.646 INFO:teuthology.orchestra.run.smithi018.stdout: "exporter_sock_dir": "/var/run/ceph/", 2023-09-23T20:22:43.646 INFO:teuthology.orchestra.run.smithi018.stdout: "exporter_sort_metrics": "true", 2023-09-23T20:22:43.646 INFO:teuthology.orchestra.run.smithi018.stdout: "exporter_stats_period": "5", 2023-09-23T20:22:43.646 INFO:teuthology.orchestra.run.smithi018.stdout: "fake_statfs_for_testing": "0", 2023-09-23T20:22:43.646 INFO:teuthology.orchestra.run.smithi018.stdout: "fatal_signal_handlers": "true", 2023-09-23T20:22:43.646 INFO:teuthology.orchestra.run.smithi018.stdout: "filer_max_purge_ops": "10", 2023-09-23T20:22:43.647 INFO:teuthology.orchestra.run.smithi018.stdout: "filer_max_truncate_ops": "128", 2023-09-23T20:22:43.647 INFO:teuthology.orchestra.run.smithi018.stdout: "filestore_apply_finisher_threads": "1", 2023-09-23T20:22:43.647 INFO:teuthology.orchestra.run.smithi018.stdout: "filestore_blackhole": "false", 2023-09-23T20:22:43.647 INFO:teuthology.orchestra.run.smithi018.stdout: "filestore_btrfs_clone_range": "true", 2023-09-23T20:22:43.647 INFO:teuthology.orchestra.run.smithi018.stdout: "filestore_btrfs_snap": "true", 2023-09-23T20:22:43.647 INFO:teuthology.orchestra.run.smithi018.stdout: "filestore_caller_concurrency": "10", 2023-09-23T20:22:43.647 INFO:teuthology.orchestra.run.smithi018.stdout: "filestore_collect_device_partition_information": "true", 2023-09-23T20:22:43.647 INFO:teuthology.orchestra.run.smithi018.stdout: "filestore_commit_timeout": "600.000000", 2023-09-23T20:22:43.647 INFO:teuthology.orchestra.run.smithi018.stdout: "filestore_debug_inject_read_err": "false", 2023-09-23T20:22:43.647 INFO:teuthology.orchestra.run.smithi018.stdout: "filestore_debug_omap_check": "false", 2023-09-23T20:22:43.647 INFO:teuthology.orchestra.run.smithi018.stdout: "filestore_debug_random_read_err": "0.000000", 2023-09-23T20:22:43.647 INFO:teuthology.orchestra.run.smithi018.stdout: "filestore_debug_verify_split": "false", 2023-09-23T20:22:43.648 INFO:teuthology.orchestra.run.smithi018.stdout: "filestore_dump_file": "", 2023-09-23T20:22:43.648 INFO:teuthology.orchestra.run.smithi018.stdout: "filestore_expected_throughput_bytes": "209715200.000000", 2023-09-23T20:22:43.648 INFO:teuthology.orchestra.run.smithi018.stdout: "filestore_expected_throughput_ops": "200.000000", 2023-09-23T20:22:43.648 INFO:teuthology.orchestra.run.smithi018.stdout: "filestore_fadvise": "true", 2023-09-23T20:22:43.648 INFO:teuthology.orchestra.run.smithi018.stdout: "filestore_fail_eio": "true", 2023-09-23T20:22:43.648 INFO:teuthology.orchestra.run.smithi018.stdout: "filestore_fd_cache_shards": "16", 2023-09-23T20:22:43.648 INFO:teuthology.orchestra.run.smithi018.stdout: "filestore_fd_cache_size": "128", 2023-09-23T20:22:43.648 INFO:teuthology.orchestra.run.smithi018.stdout: "filestore_fiemap": "false", 2023-09-23T20:22:43.648 INFO:teuthology.orchestra.run.smithi018.stdout: "filestore_fiemap_threshold": "4096", 2023-09-23T20:22:43.648 INFO:teuthology.orchestra.run.smithi018.stdout: "filestore_fsync_flushes_journal_data": "false", 2023-09-23T20:22:43.648 INFO:teuthology.orchestra.run.smithi018.stdout: "filestore_index_retry_probability": "0.000000", 2023-09-23T20:22:43.649 INFO:teuthology.orchestra.run.smithi018.stdout: "filestore_inject_stall": "0", 2023-09-23T20:22:43.649 INFO:teuthology.orchestra.run.smithi018.stdout: "filestore_journal_parallel": "false", 2023-09-23T20:22:43.649 INFO:teuthology.orchestra.run.smithi018.stdout: "filestore_journal_trailing": "false", 2023-09-23T20:22:43.649 INFO:teuthology.orchestra.run.smithi018.stdout: "filestore_journal_writeahead": "false", 2023-09-23T20:22:43.649 INFO:teuthology.orchestra.run.smithi018.stdout: "filestore_kill_at": "0", 2023-09-23T20:22:43.649 INFO:teuthology.orchestra.run.smithi018.stdout: "filestore_max_alloc_hint_size": "1048576", 2023-09-23T20:22:43.649 INFO:teuthology.orchestra.run.smithi018.stdout: "filestore_max_inline_xattr_size": "0", 2023-09-23T20:22:43.649 INFO:teuthology.orchestra.run.smithi018.stdout: "filestore_max_inline_xattr_size_btrfs": "2048", 2023-09-23T20:22:43.649 INFO:teuthology.orchestra.run.smithi018.stdout: "filestore_max_inline_xattr_size_other": "512", 2023-09-23T20:22:43.649 INFO:teuthology.orchestra.run.smithi018.stdout: "filestore_max_inline_xattr_size_xfs": "65536", 2023-09-23T20:22:43.649 INFO:teuthology.orchestra.run.smithi018.stdout: "filestore_max_inline_xattrs": "0", 2023-09-23T20:22:43.649 INFO:teuthology.orchestra.run.smithi018.stdout: "filestore_max_inline_xattrs_btrfs": "10", 2023-09-23T20:22:43.650 INFO:teuthology.orchestra.run.smithi018.stdout: "filestore_max_inline_xattrs_other": "2", 2023-09-23T20:22:43.650 INFO:teuthology.orchestra.run.smithi018.stdout: "filestore_max_inline_xattrs_xfs": "10", 2023-09-23T20:22:43.650 INFO:teuthology.orchestra.run.smithi018.stdout: "filestore_max_sync_interval": "5.000000", 2023-09-23T20:22:43.650 INFO:teuthology.orchestra.run.smithi018.stdout: "filestore_max_xattr_value_size": "0", 2023-09-23T20:22:43.650 INFO:teuthology.orchestra.run.smithi018.stdout: "filestore_max_xattr_value_size_btrfs": "65536", 2023-09-23T20:22:43.650 INFO:teuthology.orchestra.run.smithi018.stdout: "filestore_max_xattr_value_size_other": "1024", 2023-09-23T20:22:43.650 INFO:teuthology.orchestra.run.smithi018.stdout: "filestore_max_xattr_value_size_xfs": "65536", 2023-09-23T20:22:43.650 INFO:teuthology.orchestra.run.smithi018.stdout: "filestore_merge_threshold": "-10", 2023-09-23T20:22:43.650 INFO:teuthology.orchestra.run.smithi018.stdout: "filestore_min_sync_interval": "0.010000", 2023-09-23T20:22:43.650 INFO:teuthology.orchestra.run.smithi018.stdout: "filestore_odsync_write": "false", 2023-09-23T20:22:43.650 INFO:teuthology.orchestra.run.smithi018.stdout: "filestore_omap_backend": "rocksdb", 2023-09-23T20:22:43.650 INFO:teuthology.orchestra.run.smithi018.stdout: "filestore_omap_backend_path": "", 2023-09-23T20:22:43.651 INFO:teuthology.orchestra.run.smithi018.stdout: "filestore_omap_header_cache_size": "1024", 2023-09-23T20:22:43.651 INFO:teuthology.orchestra.run.smithi018.stdout: "filestore_ondisk_finisher_threads": "1", 2023-09-23T20:22:43.651 INFO:teuthology.orchestra.run.smithi018.stdout: "filestore_op_thread_suicide_timeout": "180", 2023-09-23T20:22:43.651 INFO:teuthology.orchestra.run.smithi018.stdout: "filestore_op_thread_timeout": "60", 2023-09-23T20:22:43.651 INFO:teuthology.orchestra.run.smithi018.stdout: "filestore_op_threads": "2", 2023-09-23T20:22:43.651 INFO:teuthology.orchestra.run.smithi018.stdout: "filestore_punch_hole": "false", 2023-09-23T20:22:43.651 INFO:teuthology.orchestra.run.smithi018.stdout: "filestore_queue_high_delay_multiple": "0.000000", 2023-09-23T20:22:43.651 INFO:teuthology.orchestra.run.smithi018.stdout: "filestore_queue_high_delay_multiple_bytes": "0.000000", 2023-09-23T20:22:43.651 INFO:teuthology.orchestra.run.smithi018.stdout: "filestore_queue_high_delay_multiple_ops": "0.000000", 2023-09-23T20:22:43.651 INFO:teuthology.orchestra.run.smithi018.stdout: "filestore_queue_high_threshhold": "0.900000", 2023-09-23T20:22:43.651 INFO:teuthology.orchestra.run.smithi018.stdout: "filestore_queue_low_threshhold": "0.300000", 2023-09-23T20:22:43.651 INFO:teuthology.orchestra.run.smithi018.stdout: "filestore_queue_max_bytes": "104857600", 2023-09-23T20:22:43.652 INFO:teuthology.orchestra.run.smithi018.stdout: "filestore_queue_max_delay_multiple": "0.000000", 2023-09-23T20:22:43.652 INFO:teuthology.orchestra.run.smithi018.stdout: "filestore_queue_max_delay_multiple_bytes": "0.000000", 2023-09-23T20:22:43.652 INFO:teuthology.orchestra.run.smithi018.stdout: "filestore_queue_max_delay_multiple_ops": "0.000000", 2023-09-23T20:22:43.652 INFO:teuthology.orchestra.run.smithi018.stdout: "filestore_queue_max_ops": "50", 2023-09-23T20:22:43.652 INFO:teuthology.orchestra.run.smithi018.stdout: "filestore_rocksdb_options": "max_background_jobs=10,compaction_readahead_size=2097152,compression=kNoCompression", 2023-09-23T20:22:43.652 INFO:teuthology.orchestra.run.smithi018.stdout: "filestore_seek_data_hole": "false", 2023-09-23T20:22:43.652 INFO:teuthology.orchestra.run.smithi018.stdout: "filestore_sloppy_crc": "false", 2023-09-23T20:22:43.652 INFO:teuthology.orchestra.run.smithi018.stdout: "filestore_sloppy_crc_block_size": "65536", 2023-09-23T20:22:43.652 INFO:teuthology.orchestra.run.smithi018.stdout: "filestore_splice": "false", 2023-09-23T20:22:43.652 INFO:teuthology.orchestra.run.smithi018.stdout: "filestore_split_multiple": "2", 2023-09-23T20:22:43.652 INFO:teuthology.orchestra.run.smithi018.stdout: "filestore_split_rand_factor": "20", 2023-09-23T20:22:43.652 INFO:teuthology.orchestra.run.smithi018.stdout: "filestore_update_to": "1000", 2023-09-23T20:22:43.653 INFO:teuthology.orchestra.run.smithi018.stdout: "filestore_wbthrottle_btrfs_bytes_hard_limit": "419430400", 2023-09-23T20:22:43.653 INFO:teuthology.orchestra.run.smithi018.stdout: "filestore_wbthrottle_btrfs_bytes_start_flusher": "41943040", 2023-09-23T20:22:43.653 INFO:teuthology.orchestra.run.smithi018.stdout: "filestore_wbthrottle_btrfs_inodes_hard_limit": "5000", 2023-09-23T20:22:43.653 INFO:teuthology.orchestra.run.smithi018.stdout: "filestore_wbthrottle_btrfs_inodes_start_flusher": "500", 2023-09-23T20:22:43.653 INFO:teuthology.orchestra.run.smithi018.stdout: "filestore_wbthrottle_btrfs_ios_hard_limit": "5000", 2023-09-23T20:22:43.653 INFO:teuthology.orchestra.run.smithi018.stdout: "filestore_wbthrottle_btrfs_ios_start_flusher": "500", 2023-09-23T20:22:43.653 INFO:teuthology.orchestra.run.smithi018.stdout: "filestore_wbthrottle_enable": "true", 2023-09-23T20:22:43.653 INFO:teuthology.orchestra.run.smithi018.stdout: "filestore_wbthrottle_xfs_bytes_hard_limit": "419430400", 2023-09-23T20:22:43.653 INFO:teuthology.orchestra.run.smithi018.stdout: "filestore_wbthrottle_xfs_bytes_start_flusher": "41943040", 2023-09-23T20:22:43.653 INFO:teuthology.orchestra.run.smithi018.stdout: "filestore_wbthrottle_xfs_inodes_hard_limit": "5000", 2023-09-23T20:22:43.653 INFO:teuthology.orchestra.run.smithi018.stdout: "filestore_wbthrottle_xfs_inodes_start_flusher": "500", 2023-09-23T20:22:43.653 INFO:teuthology.orchestra.run.smithi018.stdout: "filestore_wbthrottle_xfs_ios_hard_limit": "5000", 2023-09-23T20:22:43.654 INFO:teuthology.orchestra.run.smithi018.stdout: "filestore_wbthrottle_xfs_ios_start_flusher": "500", 2023-09-23T20:22:43.654 INFO:teuthology.orchestra.run.smithi018.stdout: "filestore_xfs_extsize": "false", 2023-09-23T20:22:43.654 INFO:teuthology.orchestra.run.smithi018.stdout: "filestore_zfs_snap": "false", 2023-09-23T20:22:43.654 INFO:teuthology.orchestra.run.smithi018.stdout: "fio_dir": "/tmp/fio", 2023-09-23T20:22:43.654 INFO:teuthology.orchestra.run.smithi018.stdout: "fsid": "a3b3b17e-5a4c-11ee-8db4-212e2dc638e7", 2023-09-23T20:22:43.654 INFO:teuthology.orchestra.run.smithi018.stdout: "fuse_allow_other": "true", 2023-09-23T20:22:43.654 INFO:teuthology.orchestra.run.smithi018.stdout: "fuse_atomic_o_trunc": "true", 2023-09-23T20:22:43.654 INFO:teuthology.orchestra.run.smithi018.stdout: "fuse_big_writes": "true", 2023-09-23T20:22:43.654 INFO:teuthology.orchestra.run.smithi018.stdout: "fuse_debug": "false", 2023-09-23T20:22:43.654 INFO:teuthology.orchestra.run.smithi018.stdout: "fuse_default_permissions": "false", 2023-09-23T20:22:43.654 INFO:teuthology.orchestra.run.smithi018.stdout: "fuse_disable_pagecache": "false", 2023-09-23T20:22:43.654 INFO:teuthology.orchestra.run.smithi018.stdout: "fuse_max_write": "0", 2023-09-23T20:22:43.655 INFO:teuthology.orchestra.run.smithi018.stdout: "fuse_multithreaded": "true", 2023-09-23T20:22:43.655 INFO:teuthology.orchestra.run.smithi018.stdout: "fuse_require_active_mds": "true", 2023-09-23T20:22:43.655 INFO:teuthology.orchestra.run.smithi018.stdout: "fuse_set_user_groups": "true", 2023-09-23T20:22:43.655 INFO:teuthology.orchestra.run.smithi018.stdout: "fuse_splice_move": "true", 2023-09-23T20:22:43.655 INFO:teuthology.orchestra.run.smithi018.stdout: "fuse_splice_read": "true", 2023-09-23T20:22:43.655 INFO:teuthology.orchestra.run.smithi018.stdout: "fuse_splice_write": "true", 2023-09-23T20:22:43.655 INFO:teuthology.orchestra.run.smithi018.stdout: "fuse_syncfs_on_mksnap": "true", 2023-09-23T20:22:43.655 INFO:teuthology.orchestra.run.smithi018.stdout: "fuse_use_invalidate_cb": "true", 2023-09-23T20:22:43.655 INFO:teuthology.orchestra.run.smithi018.stdout: "gss_ktab_client_file": "/var/lib/ceph/mon.a/gss_client_mon.a.ktab", 2023-09-23T20:22:43.655 INFO:teuthology.orchestra.run.smithi018.stdout: "gss_target_name": "ceph", 2023-09-23T20:22:43.655 INFO:teuthology.orchestra.run.smithi018.stdout: "heartbeat_file": "", 2023-09-23T20:22:43.655 INFO:teuthology.orchestra.run.smithi018.stdout: "heartbeat_inject_failure": "0", 2023-09-23T20:22:43.656 INFO:teuthology.orchestra.run.smithi018.stdout: "heartbeat_interval": "5", 2023-09-23T20:22:43.656 INFO:teuthology.orchestra.run.smithi018.stdout: "host": "smithi018", 2023-09-23T20:22:43.656 INFO:teuthology.orchestra.run.smithi018.stdout: "immutable_object_cache_client_dedicated_thread_num": "2", 2023-09-23T20:22:43.656 INFO:teuthology.orchestra.run.smithi018.stdout: "immutable_object_cache_max_inflight_ops": "128", 2023-09-23T20:22:43.656 INFO:teuthology.orchestra.run.smithi018.stdout: "immutable_object_cache_max_size": "1073741824", 2023-09-23T20:22:43.656 INFO:teuthology.orchestra.run.smithi018.stdout: "immutable_object_cache_path": "/tmp/ceph_immutable_object_cache", 2023-09-23T20:22:43.656 INFO:teuthology.orchestra.run.smithi018.stdout: "immutable_object_cache_qos_bps_burst": "0", 2023-09-23T20:22:43.656 INFO:teuthology.orchestra.run.smithi018.stdout: "immutable_object_cache_qos_bps_burst_seconds": "1", 2023-09-23T20:22:43.656 INFO:teuthology.orchestra.run.smithi018.stdout: "immutable_object_cache_qos_bps_limit": "0", 2023-09-23T20:22:43.656 INFO:teuthology.orchestra.run.smithi018.stdout: "immutable_object_cache_qos_iops_burst": "0", 2023-09-23T20:22:43.656 INFO:teuthology.orchestra.run.smithi018.stdout: "immutable_object_cache_qos_iops_burst_seconds": "1", 2023-09-23T20:22:43.656 INFO:teuthology.orchestra.run.smithi018.stdout: "immutable_object_cache_qos_iops_limit": "0", 2023-09-23T20:22:43.657 INFO:teuthology.orchestra.run.smithi018.stdout: "immutable_object_cache_qos_schedule_tick_min": "50", 2023-09-23T20:22:43.657 INFO:teuthology.orchestra.run.smithi018.stdout: "immutable_object_cache_sock": "/var/run/ceph/immutable_object_cache_sock", 2023-09-23T20:22:43.657 INFO:teuthology.orchestra.run.smithi018.stdout: "immutable_object_cache_watermark": "0.900000", 2023-09-23T20:22:43.657 INFO:teuthology.orchestra.run.smithi018.stdout: "inject_early_sigterm": "false", 2023-09-23T20:22:43.657 INFO:teuthology.orchestra.run.smithi018.stdout: "jaeger_agent_port": "6799", 2023-09-23T20:22:43.657 INFO:teuthology.orchestra.run.smithi018.stdout: "jaeger_tracing_enable": "false", 2023-09-23T20:22:43.657 INFO:teuthology.orchestra.run.smithi018.stdout: "journal_aio": "true", 2023-09-23T20:22:43.657 INFO:teuthology.orchestra.run.smithi018.stdout: "journal_align_min_size": "65536", 2023-09-23T20:22:43.657 INFO:teuthology.orchestra.run.smithi018.stdout: "journal_block_align": "true", 2023-09-23T20:22:43.657 INFO:teuthology.orchestra.run.smithi018.stdout: "journal_block_size": "4096", 2023-09-23T20:22:43.657 INFO:teuthology.orchestra.run.smithi018.stdout: "journal_dio": "true", 2023-09-23T20:22:43.657 INFO:teuthology.orchestra.run.smithi018.stdout: "journal_discard": "false", 2023-09-23T20:22:43.658 INFO:teuthology.orchestra.run.smithi018.stdout: "journal_force_aio": "false", 2023-09-23T20:22:43.658 INFO:teuthology.orchestra.run.smithi018.stdout: "journal_ignore_corruption": "false", 2023-09-23T20:22:43.658 INFO:teuthology.orchestra.run.smithi018.stdout: "journal_max_write_bytes": "10485760", 2023-09-23T20:22:43.658 INFO:teuthology.orchestra.run.smithi018.stdout: "journal_max_write_entries": "100", 2023-09-23T20:22:43.658 INFO:teuthology.orchestra.run.smithi018.stdout: "journal_replay_from": "0", 2023-09-23T20:22:43.658 INFO:teuthology.orchestra.run.smithi018.stdout: "journal_throttle_high_multiple": "0.000000", 2023-09-23T20:22:43.658 INFO:teuthology.orchestra.run.smithi018.stdout: "journal_throttle_high_threshhold": "0.900000", 2023-09-23T20:22:43.658 INFO:teuthology.orchestra.run.smithi018.stdout: "journal_throttle_low_threshhold": "0.600000", 2023-09-23T20:22:43.658 INFO:teuthology.orchestra.run.smithi018.stdout: "journal_throttle_max_multiple": "0.000000", 2023-09-23T20:22:43.658 INFO:teuthology.orchestra.run.smithi018.stdout: "journal_write_header_frequency": "0", 2023-09-23T20:22:43.658 INFO:teuthology.orchestra.run.smithi018.stdout: "journal_zero_on_create": "false", 2023-09-23T20:22:43.659 INFO:teuthology.orchestra.run.smithi018.stdout: "journaler_prefetch_periods": "10", 2023-09-23T20:22:43.659 INFO:teuthology.orchestra.run.smithi018.stdout: "journaler_prezero_periods": "5", 2023-09-23T20:22:43.659 INFO:teuthology.orchestra.run.smithi018.stdout: "journaler_write_head_interval": "15", 2023-09-23T20:22:43.659 INFO:teuthology.orchestra.run.smithi018.stdout: "key": "", 2023-09-23T20:22:43.659 INFO:teuthology.orchestra.run.smithi018.stdout: "keyfile": "", 2023-09-23T20:22:43.659 INFO:teuthology.orchestra.run.smithi018.stdout: "keyring": "/var/lib/ceph/mon/ceph-a/keyring", 2023-09-23T20:22:43.659 INFO:teuthology.orchestra.run.smithi018.stdout: "kstore_backend": "rocksdb", 2023-09-23T20:22:43.659 INFO:teuthology.orchestra.run.smithi018.stdout: "kstore_default_stripe_size": "65536", 2023-09-23T20:22:43.659 INFO:teuthology.orchestra.run.smithi018.stdout: "kstore_fsck_on_mount": "false", 2023-09-23T20:22:43.659 INFO:teuthology.orchestra.run.smithi018.stdout: "kstore_fsck_on_mount_deep": "true", 2023-09-23T20:22:43.659 INFO:teuthology.orchestra.run.smithi018.stdout: "kstore_max_bytes": "67108864", 2023-09-23T20:22:43.659 INFO:teuthology.orchestra.run.smithi018.stdout: "kstore_max_ops": "512", 2023-09-23T20:22:43.659 INFO:teuthology.orchestra.run.smithi018.stdout: "kstore_nid_prealloc": "1024", 2023-09-23T20:22:43.660 INFO:teuthology.orchestra.run.smithi018.stdout: "kstore_onode_map_size": "1024", 2023-09-23T20:22:43.660 INFO:teuthology.orchestra.run.smithi018.stdout: "kstore_rocksdb_options": "compression=kNoCompression", 2023-09-23T20:22:43.660 INFO:teuthology.orchestra.run.smithi018.stdout: "kstore_sync_submit_transaction": "false", 2023-09-23T20:22:43.660 INFO:teuthology.orchestra.run.smithi018.stdout: "kstore_sync_transaction": "false", 2023-09-23T20:22:43.660 INFO:teuthology.orchestra.run.smithi018.stdout: "leveldb_block_size": "65536", 2023-09-23T20:22:43.660 INFO:teuthology.orchestra.run.smithi018.stdout: "leveldb_bloom_size": "0", 2023-09-23T20:22:43.660 INFO:teuthology.orchestra.run.smithi018.stdout: "leveldb_cache_size": "536870912", 2023-09-23T20:22:43.660 INFO:teuthology.orchestra.run.smithi018.stdout: "leveldb_compact_on_mount": "false", 2023-09-23T20:22:43.660 INFO:teuthology.orchestra.run.smithi018.stdout: "leveldb_compression": "false", 2023-09-23T20:22:43.660 INFO:teuthology.orchestra.run.smithi018.stdout: "leveldb_log": "", 2023-09-23T20:22:43.660 INFO:teuthology.orchestra.run.smithi018.stdout: "leveldb_log_to_ceph_log": "true", 2023-09-23T20:22:43.660 INFO:teuthology.orchestra.run.smithi018.stdout: "leveldb_max_open_files": "0", 2023-09-23T20:22:43.661 INFO:teuthology.orchestra.run.smithi018.stdout: "leveldb_paranoid": "false", 2023-09-23T20:22:43.661 INFO:teuthology.orchestra.run.smithi018.stdout: "leveldb_write_buffer_size": "33554432", 2023-09-23T20:22:43.661 INFO:teuthology.orchestra.run.smithi018.stdout: "librados_thread_count": "2", 2023-09-23T20:22:43.661 INFO:teuthology.orchestra.run.smithi018.stdout: "lockdep": "false", 2023-09-23T20:22:43.661 INFO:teuthology.orchestra.run.smithi018.stdout: "lockdep_force_backtrace": "false", 2023-09-23T20:22:43.661 INFO:teuthology.orchestra.run.smithi018.stdout: "log_coarse_timestamps": "true", 2023-09-23T20:22:43.661 INFO:teuthology.orchestra.run.smithi018.stdout: "log_file": "/var/log/ceph/ceph-mon.a.log", 2023-09-23T20:22:43.661 INFO:teuthology.orchestra.run.smithi018.stdout: "log_flush_on_exit": "false", 2023-09-23T20:22:43.661 INFO:teuthology.orchestra.run.smithi018.stdout: "log_graylog_host": "127.0.0.1", 2023-09-23T20:22:43.661 INFO:teuthology.orchestra.run.smithi018.stdout: "log_graylog_port": "12201", 2023-09-23T20:22:43.661 INFO:teuthology.orchestra.run.smithi018.stdout: "log_max_new": "1000", 2023-09-23T20:22:43.661 INFO:teuthology.orchestra.run.smithi018.stdout: "log_max_recent": "10000", 2023-09-23T20:22:43.662 INFO:teuthology.orchestra.run.smithi018.stdout: "log_stderr_prefix": "", 2023-09-23T20:22:43.662 INFO:teuthology.orchestra.run.smithi018.stdout: "log_stop_at_utilization": "0.970000", 2023-09-23T20:22:43.662 INFO:teuthology.orchestra.run.smithi018.stdout: "log_to_file": "true", 2023-09-23T20:22:43.662 INFO:teuthology.orchestra.run.smithi018.stdout: "log_to_graylog": "false", 2023-09-23T20:22:43.662 INFO:teuthology.orchestra.run.smithi018.stdout: "log_to_journald": "false", 2023-09-23T20:22:43.662 INFO:teuthology.orchestra.run.smithi018.stdout: "log_to_stderr": "false", 2023-09-23T20:22:43.662 INFO:teuthology.orchestra.run.smithi018.stdout: "log_to_syslog": "false", 2023-09-23T20:22:43.662 INFO:teuthology.orchestra.run.smithi018.stdout: "max_rotating_auth_attempts": "10", 2023-09-23T20:22:43.662 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_abort_on_newly_corrupt_dentry": "true", 2023-09-23T20:22:43.662 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_action_on_write_error": "1", 2023-09-23T20:22:43.662 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_alternate_name_max": "8192", 2023-09-23T20:22:43.663 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_asio_thread_count": "2", 2023-09-23T20:22:43.663 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_bal_export_pin": "true", 2023-09-23T20:22:43.663 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_bal_fragment_dirs": "true", 2023-09-23T20:22:43.663 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_bal_fragment_fast_factor": "1.500000", 2023-09-23T20:22:43.663 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_bal_fragment_interval": "5", 2023-09-23T20:22:43.663 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_bal_fragment_size_max": "100000", 2023-09-23T20:22:43.663 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_bal_idle_threshold": "0.000000", 2023-09-23T20:22:43.663 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_bal_interval": "10", 2023-09-23T20:22:43.663 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_bal_max": "-1", 2023-09-23T20:22:43.663 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_bal_max_until": "-1", 2023-09-23T20:22:43.663 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_bal_merge_size": "50", 2023-09-23T20:22:43.663 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_bal_midchunk": "0.300000", 2023-09-23T20:22:43.664 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_bal_min_rebalance": "0.100000", 2023-09-23T20:22:43.664 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_bal_min_start": "0.200000", 2023-09-23T20:22:43.664 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_bal_minchunk": "0.001000", 2023-09-23T20:22:43.664 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_bal_mode": "0", 2023-09-23T20:22:43.664 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_bal_need_max": "1.200000", 2023-09-23T20:22:43.664 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_bal_need_min": "0.800000", 2023-09-23T20:22:43.664 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_bal_replicate_threshold": "8000.000000", 2023-09-23T20:22:43.664 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_bal_sample_interval": "3.000000", 2023-09-23T20:22:43.664 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_bal_split_bits": "3", 2023-09-23T20:22:43.664 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_bal_split_rd": "25000.000000", 2023-09-23T20:22:43.664 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_bal_split_size": "10000", 2023-09-23T20:22:43.664 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_bal_split_wr": "10000.000000", 2023-09-23T20:22:43.665 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_bal_target_decay": "10.000000", 2023-09-23T20:22:43.665 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_bal_unreplicate_threshold": "0.000000", 2023-09-23T20:22:43.665 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_beacon_grace": "15.000000", 2023-09-23T20:22:43.665 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_beacon_interval": "4.000000", 2023-09-23T20:22:43.665 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_beacon_mon_down_grace": "60", 2023-09-23T20:22:43.665 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_cache_memory_limit": "4294967296", 2023-09-23T20:22:43.665 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_cache_mid": "0.700000", 2023-09-23T20:22:43.665 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_cache_release_free_interval": "10", 2023-09-23T20:22:43.665 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_cache_reservation": "0.050000", 2023-09-23T20:22:43.665 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_cache_trim_decay_rate": "1.000000", 2023-09-23T20:22:43.665 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_cache_trim_interval": "1", 2023-09-23T20:22:43.665 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_cache_trim_threshold": "262144", 2023-09-23T20:22:43.666 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_cap_acquisition_throttle_retry_request_timeout": "0.500000", 2023-09-23T20:22:43.666 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_cap_revoke_eviction_timeout": "0.000000", 2023-09-23T20:22:43.666 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_client_delegate_inos_pct": "50", 2023-09-23T20:22:43.666 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_client_prealloc_inos": "1000", 2023-09-23T20:22:43.666 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_client_writeable_range_max_inc_objs": "1024", 2023-09-23T20:22:43.666 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_connect_bootstrapping": "false", 2023-09-23T20:22:43.666 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_damage_table_max_entries": "10000", 2023-09-23T20:22:43.666 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_data": "/var/lib/ceph/mds/ceph-a", 2023-09-23T20:22:43.666 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_debug_auth_pins": "false", 2023-09-23T20:22:43.666 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_debug_frag": "false", 2023-09-23T20:22:43.666 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_debug_scatterstat": "false", 2023-09-23T20:22:43.666 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_debug_subtrees": "false", 2023-09-23T20:22:43.667 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_decay_halflife": "5.000000", 2023-09-23T20:22:43.667 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_default_dir_hash": "2", 2023-09-23T20:22:43.667 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_defer_session_stale": "true", 2023-09-23T20:22:43.667 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_deny_all_reconnect": "false", 2023-09-23T20:22:43.667 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_dir_keys_per_op": "16384", 2023-09-23T20:22:43.667 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_dir_max_commit_size": "10", 2023-09-23T20:22:43.667 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_dir_max_entries": "0", 2023-09-23T20:22:43.667 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_dir_prefetch": "true", 2023-09-23T20:22:43.667 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_dirstat_min_interval": "1.000000", 2023-09-23T20:22:43.667 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_dump_cache_after_rejoin": "false", 2023-09-23T20:22:43.667 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_dump_cache_on_map": "false", 2023-09-23T20:22:43.667 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_dump_cache_threshold_file": "0", 2023-09-23T20:22:43.668 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_dump_cache_threshold_formatter": "1073741824", 2023-09-23T20:22:43.668 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_early_reply": "true", 2023-09-23T20:22:43.668 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_enable_op_tracker": "true", 2023-09-23T20:22:43.668 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_enforce_unique_name": "true", 2023-09-23T20:22:43.668 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_export_ephemeral_distributed": "true", 2023-09-23T20:22:43.668 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_export_ephemeral_distributed_factor": "2.000000", 2023-09-23T20:22:43.668 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_export_ephemeral_random": "true", 2023-09-23T20:22:43.668 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_export_ephemeral_random_max": "0.010000", 2023-09-23T20:22:43.668 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_extraordinary_events_dump_interval": "0", 2023-09-23T20:22:43.668 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_forward_all_requests_to_auth": "false", 2023-09-23T20:22:43.668 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_freeze_tree_timeout": "30.000000", 2023-09-23T20:22:43.668 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_fscrypt_last_block_max_size": "4096", 2023-09-23T20:22:43.669 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_go_bad_corrupt_dentry": "true", 2023-09-23T20:22:43.669 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_hack_allow_loading_invalid_metadata": "false", 2023-09-23T20:22:43.669 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_health_cache_threshold": "1.500000", 2023-09-23T20:22:43.669 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_health_summarize_threshold": "10", 2023-09-23T20:22:43.669 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_heartbeat_grace": "15.000000", 2023-09-23T20:22:43.669 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_heartbeat_reset_grace": "1000", 2023-09-23T20:22:43.669 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_inject_health_dummy": "false", 2023-09-23T20:22:43.669 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_inject_journal_corrupt_dentry_first": "0.000000", 2023-09-23T20:22:43.669 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_inject_migrator_session_race": "false", 2023-09-23T20:22:43.669 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_inject_rename_corrupt_dentry_first": "0.000000", 2023-09-23T20:22:43.669 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_inject_skip_replaying_inotable": "false", 2023-09-23T20:22:43.669 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_inject_traceless_reply_probability": "0.000000", 2023-09-23T20:22:43.670 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_join_fs": "", 2023-09-23T20:22:43.670 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_journal_format": "1", 2023-09-23T20:22:43.670 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_kill_create_at": "0", 2023-09-23T20:22:43.670 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_kill_export_at": "0", 2023-09-23T20:22:43.670 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_kill_import_at": "0", 2023-09-23T20:22:43.670 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_kill_journal_at": "0", 2023-09-23T20:22:43.670 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_kill_journal_expire_at": "0", 2023-09-23T20:22:43.670 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_kill_journal_replay_at": "0", 2023-09-23T20:22:43.670 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_kill_link_at": "0", 2023-09-23T20:22:43.670 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_kill_mdstable_at": "0", 2023-09-23T20:22:43.670 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_kill_openc_at": "0", 2023-09-23T20:22:43.670 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_kill_rename_at": "0", 2023-09-23T20:22:43.671 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_kill_skip_replaying_inotable": "false", 2023-09-23T20:22:43.671 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_log_events_per_segment": "1024", 2023-09-23T20:22:43.671 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_log_max_events": "-1", 2023-09-23T20:22:43.671 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_log_max_segments": "128", 2023-09-23T20:22:43.671 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_log_pause": "false", 2023-09-23T20:22:43.671 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_log_segment_size": "0", 2023-09-23T20:22:43.671 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_log_skip_corrupt_events": "false", 2023-09-23T20:22:43.671 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_log_warn_factor": "2.000000", 2023-09-23T20:22:43.671 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_max_caps_per_client": "1048576", 2023-09-23T20:22:43.671 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_max_completed_flushes": "100000", 2023-09-23T20:22:43.671 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_max_completed_requests": "100000", 2023-09-23T20:22:43.672 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_max_export_size": "20971520", 2023-09-23T20:22:43.672 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_max_file_recover": "32", 2023-09-23T20:22:43.672 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_max_purge_files": "64", 2023-09-23T20:22:43.672 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_max_purge_ops": "8192", 2023-09-23T20:22:43.672 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_max_purge_ops_per_pg": "0.500000", 2023-09-23T20:22:43.672 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_max_scrub_ops_in_progress": "5", 2023-09-23T20:22:43.672 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_max_snaps_per_dir": "100", 2023-09-23T20:22:43.672 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_max_xattr_pairs_size": "65536", 2023-09-23T20:22:43.672 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_metrics_update_interval": "2", 2023-09-23T20:22:43.672 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_min_caps_per_client": "100", 2023-09-23T20:22:43.673 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_min_caps_working_set": "10000", 2023-09-23T20:22:43.673 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_mon_shutdown_timeout": "5.000000", 2023-09-23T20:22:43.673 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_numa_node": "-1", 2023-09-23T20:22:43.673 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_oft_prefetch_dirfrags": "false", 2023-09-23T20:22:43.673 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_op_complaint_time": "30.000000", 2023-09-23T20:22:43.673 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_op_history_duration": "600", 2023-09-23T20:22:43.673 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_op_history_size": "20", 2023-09-23T20:22:43.673 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_op_log_threshold": "5", 2023-09-23T20:22:43.673 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_ping_grace": "15", 2023-09-23T20:22:43.673 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_ping_interval": "5", 2023-09-23T20:22:43.674 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_purge_queue_busy_flush_period": "1.000000", 2023-09-23T20:22:43.674 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_recall_global_max_decay_threshold": "131072", 2023-09-23T20:22:43.674 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_recall_max_caps": "30000", 2023-09-23T20:22:43.674 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_recall_max_decay_rate": "1.500000", 2023-09-23T20:22:43.674 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_recall_max_decay_threshold": "131072", 2023-09-23T20:22:43.674 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_recall_warning_decay_rate": "60.000000", 2023-09-23T20:22:43.674 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_recall_warning_threshold": "262144", 2023-09-23T20:22:43.674 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_reconnect_timeout": "45.000000", 2023-09-23T20:22:43.674 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_replay_interval": "1.000000", 2023-09-23T20:22:43.674 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_replay_unsafe_with_closed_session": "false", 2023-09-23T20:22:43.674 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_request_load_average_decay_rate": "60.000000", 2023-09-23T20:22:43.675 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_root_ino_gid": "0", 2023-09-23T20:22:43.675 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_root_ino_uid": "0", 2023-09-23T20:22:43.675 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_scatter_nudge_interval": "5.000000", 2023-09-23T20:22:43.675 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_session_blocklist_on_evict": "true", 2023-09-23T20:22:43.675 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_session_blocklist_on_timeout": "true", 2023-09-23T20:22:43.675 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_session_cache_liveness_decay_rate": "300.000000", 2023-09-23T20:22:43.675 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_session_cache_liveness_magnitude": "10", 2023-09-23T20:22:43.675 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_session_cap_acquisition_decay_rate": "30.000000", 2023-09-23T20:22:43.675 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_session_cap_acquisition_throttle": "100000", 2023-09-23T20:22:43.675 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_session_max_caps_throttle_ratio": "1.100000", 2023-09-23T20:22:43.676 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_sessionmap_keys_per_op": "1024", 2023-09-23T20:22:43.676 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_shutdown_check": "0", 2023-09-23T20:22:43.676 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_skip_ino": "0", 2023-09-23T20:22:43.676 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_sleep_rank_change": "0.000000", 2023-09-23T20:22:43.676 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_snap_max_uid": "4294967294", 2023-09-23T20:22:43.676 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_snap_min_uid": "0", 2023-09-23T20:22:43.676 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_snap_rstat": "false", 2023-09-23T20:22:43.676 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_standby_replay_damaged": "false", 2023-09-23T20:22:43.676 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_symlink_recovery": "true", 2023-09-23T20:22:43.676 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_task_status_update_interval": "2.000000", 2023-09-23T20:22:43.677 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_thrash_exports": "0", 2023-09-23T20:22:43.677 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_thrash_fragments": "0", 2023-09-23T20:22:43.677 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_tick_interval": "5.000000", 2023-09-23T20:22:43.677 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_valgrind_exit": "false", 2023-09-23T20:22:43.677 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_verify_backtrace": "1", 2023-09-23T20:22:43.677 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_verify_scatter": "false", 2023-09-23T20:22:43.677 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_wipe_ino_prealloc": "false", 2023-09-23T20:22:43.677 INFO:teuthology.orchestra.run.smithi018.stdout: "mds_wipe_sessions": "false", 2023-09-23T20:22:43.677 INFO:teuthology.orchestra.run.smithi018.stdout: "mempool_debug": "false", 2023-09-23T20:22:43.677 INFO:teuthology.orchestra.run.smithi018.stdout: "memstore_debug_omit_block_device_write": "false", 2023-09-23T20:22:43.677 INFO:teuthology.orchestra.run.smithi018.stdout: "memstore_device_bytes": "1073741824", 2023-09-23T20:22:43.678 INFO:teuthology.orchestra.run.smithi018.stdout: "memstore_page_set": "false", 2023-09-23T20:22:43.678 INFO:teuthology.orchestra.run.smithi018.stdout: "memstore_page_size": "65536", 2023-09-23T20:22:43.678 INFO:teuthology.orchestra.run.smithi018.stdout: "mgr_client_bytes": "134217728", 2023-09-23T20:22:43.678 INFO:teuthology.orchestra.run.smithi018.stdout: "mgr_client_messages": "512", 2023-09-23T20:22:43.678 INFO:teuthology.orchestra.run.smithi018.stdout: "mgr_client_service_daemon_unregister_timeout": "1.000000", 2023-09-23T20:22:43.678 INFO:teuthology.orchestra.run.smithi018.stdout: "mgr_connect_retry_interval": "1.000000", 2023-09-23T20:22:43.678 INFO:teuthology.orchestra.run.smithi018.stdout: "mgr_data": "/var/lib/ceph/mgr/ceph-a", 2023-09-23T20:22:43.678 INFO:teuthology.orchestra.run.smithi018.stdout: "mgr_debug_aggressive_pg_num_changes": "false", 2023-09-23T20:22:43.678 INFO:teuthology.orchestra.run.smithi018.stdout: "mgr_disabled_modules": "", 2023-09-23T20:22:43.678 INFO:teuthology.orchestra.run.smithi018.stdout: "mgr_initial_modules": "restful iostat nfs", 2023-09-23T20:22:43.678 INFO:teuthology.orchestra.run.smithi018.stdout: "mgr_max_pg_num_change": "128", 2023-09-23T20:22:43.678 INFO:teuthology.orchestra.run.smithi018.stdout: "mgr_mds_bytes": "134217728", 2023-09-23T20:22:43.679 INFO:teuthology.orchestra.run.smithi018.stdout: "mgr_mds_messages": "128", 2023-09-23T20:22:43.679 INFO:teuthology.orchestra.run.smithi018.stdout: "mgr_module_path": "/usr/share/ceph/mgr", 2023-09-23T20:22:43.679 INFO:teuthology.orchestra.run.smithi018.stdout: "mgr_mon_bytes": "134217728", 2023-09-23T20:22:43.679 INFO:teuthology.orchestra.run.smithi018.stdout: "mgr_mon_messages": "128", 2023-09-23T20:22:43.679 INFO:teuthology.orchestra.run.smithi018.stdout: "mgr_osd_bytes": "536870912", 2023-09-23T20:22:43.679 INFO:teuthology.orchestra.run.smithi018.stdout: "mgr_osd_messages": "8192", 2023-09-23T20:22:43.679 INFO:teuthology.orchestra.run.smithi018.stdout: "mgr_pool": "true", 2023-09-23T20:22:43.679 INFO:teuthology.orchestra.run.smithi018.stdout: "mgr_service_beacon_grace": "60.000000", 2023-09-23T20:22:43.679 INFO:teuthology.orchestra.run.smithi018.stdout: "mgr_standby_modules": "true", 2023-09-23T20:22:43.679 INFO:teuthology.orchestra.run.smithi018.stdout: "mgr_stats_period": "5", 2023-09-23T20:22:43.679 INFO:teuthology.orchestra.run.smithi018.stdout: "mgr_stats_threshold": "5", 2023-09-23T20:22:43.681 INFO:teuthology.orchestra.run.smithi018.stdout: "mgr_tick_period": "2", 2023-09-23T20:22:43.681 INFO:teuthology.orchestra.run.smithi018.stdout: "mgr_ttl_cache_expire_seconds": "0", 2023-09-23T20:22:43.681 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_accept_timeout_factor": "2.000000", 2023-09-23T20:22:43.681 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_allow_pool_delete": "true", 2023-09-23T20:22:43.681 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_allow_pool_size_one": "false", 2023-09-23T20:22:43.681 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_auth_validate_all_caps": "true", 2023-09-23T20:22:43.681 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_cache_target_full_warn_ratio": "0.660000", 2023-09-23T20:22:43.681 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_clean_pg_upmaps_per_chunk": "256", 2023-09-23T20:22:43.681 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_client_bytes": "104857600", 2023-09-23T20:22:43.681 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_client_directed_command_retry": "2", 2023-09-23T20:22:43.681 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_client_hunt_interval": "3.000000", 2023-09-23T20:22:43.682 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_client_hunt_interval_backoff": "1.500000", 2023-09-23T20:22:43.682 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_client_hunt_interval_max_multiple": "10.000000", 2023-09-23T20:22:43.682 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_client_hunt_interval_min_multiple": "1.000000", 2023-09-23T20:22:43.682 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_client_hunt_parallel": "3", 2023-09-23T20:22:43.682 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_client_log_interval": "1.000000", 2023-09-23T20:22:43.682 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_client_max_log_entries_per_message": "1000", 2023-09-23T20:22:43.682 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_client_ping_interval": "10.000000", 2023-09-23T20:22:43.682 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_client_ping_timeout": "30.000000", 2023-09-23T20:22:43.682 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_clock_drift_allowed": "1.000000", 2023-09-23T20:22:43.683 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_clock_drift_warn_backoff": "5.000000", 2023-09-23T20:22:43.683 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_cluster_log_file": "default=/var/log/ceph/ceph.$channel.log cluster=/var/log/ceph/ceph.log", 2023-09-23T20:22:43.683 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_cluster_log_file_level": "debug", 2023-09-23T20:22:43.683 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_cluster_log_to_file": "false", 2023-09-23T20:22:43.683 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_cluster_log_to_graylog": "false", 2023-09-23T20:22:43.683 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_cluster_log_to_graylog_host": "127.0.0.1", 2023-09-23T20:22:43.683 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_cluster_log_to_graylog_port": "12201", 2023-09-23T20:22:43.683 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_cluster_log_to_journald": "true", 2023-09-23T20:22:43.683 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_cluster_log_to_stderr": "false", 2023-09-23T20:22:43.683 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_cluster_log_to_syslog": "default=false", 2023-09-23T20:22:43.684 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_cluster_log_to_syslog_facility": "daemon", 2023-09-23T20:22:43.684 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_cluster_log_to_syslog_level": "info", 2023-09-23T20:22:43.684 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_compact_on_bootstrap": "false", 2023-09-23T20:22:43.684 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_compact_on_start": "false", 2023-09-23T20:22:43.684 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_compact_on_trim": "true", 2023-09-23T20:22:43.684 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_con_tracker_persist_interval": "10", 2023-09-23T20:22:43.684 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_con_tracker_score_halflife": "43200", 2023-09-23T20:22:43.684 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_config_key_max_entry_size": "65536", 2023-09-23T20:22:43.684 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_cpu_threads": "4", 2023-09-23T20:22:43.684 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_crush_min_required_version": "hammer", 2023-09-23T20:22:43.685 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_daemon_bytes": "419430400", 2023-09-23T20:22:43.685 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_data": "/var/lib/ceph/mon/ceph-a", 2023-09-23T20:22:43.685 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_data_avail_crit": "5", 2023-09-23T20:22:43.685 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_data_avail_warn": "5", 2023-09-23T20:22:43.685 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_data_size_warn": "16106127360", 2023-09-23T20:22:43.685 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_debug_block_osdmap_trim": "false", 2023-09-23T20:22:43.685 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_debug_deprecated_as_obsolete": "false", 2023-09-23T20:22:43.685 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_debug_dump_json": "false", 2023-09-23T20:22:43.685 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_debug_dump_location": "/var/log/ceph/ceph-mon.a.tdump", 2023-09-23T20:22:43.686 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_debug_dump_transactions": "false", 2023-09-23T20:22:43.686 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_debug_extra_checks": "false", 2023-09-23T20:22:43.686 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_debug_no_initial_persistent_features": "false", 2023-09-23T20:22:43.686 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_debug_no_require_bluestore_for_ec_overwrites": "false", 2023-09-23T20:22:43.686 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_debug_no_require_quincy": "false", 2023-09-23T20:22:43.686 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_debug_no_require_reef": "false", 2023-09-23T20:22:43.686 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_debug_unsafe_allow_tier_with_nonempty_snaps": "false", 2023-09-23T20:22:43.686 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_delta_reset_interval": "10.000000", 2023-09-23T20:22:43.686 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_dns_srv_name": "ceph-mon", 2023-09-23T20:22:43.686 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_down_mkfs_grace": "60", 2023-09-23T20:22:43.686 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_election_default_strategy": "1", 2023-09-23T20:22:43.686 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_election_timeout": "5.000000", 2023-09-23T20:22:43.687 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_elector_ignore_propose_margin": "0.000500", 2023-09-23T20:22:43.687 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_elector_ping_divisor": "2", 2023-09-23T20:22:43.687 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_elector_ping_timeout": "2.000000", 2023-09-23T20:22:43.687 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_enable_op_tracker": "true", 2023-09-23T20:22:43.687 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_fake_pool_delete": "false", 2023-09-23T20:22:43.687 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_force_quorum_join": "false", 2023-09-23T20:22:43.687 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_globalid_prealloc": "10000", 2023-09-23T20:22:43.687 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_health_detail_to_clog": "true", 2023-09-23T20:22:43.687 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_health_log_update_period": "5", 2023-09-23T20:22:43.687 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_health_max_detail": "50", 2023-09-23T20:22:43.687 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_health_to_clog": "true", 2023-09-23T20:22:43.688 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_health_to_clog_interval": "600", 2023-09-23T20:22:43.688 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_health_to_clog_tick_interval": "60.000000", 2023-09-23T20:22:43.688 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_host": "[v2:172.21.15.18:3300/0,v1:172.21.15.18:6789/0] [v2:172.21.15.149:3300/0,v1:172.21.15.149:6789/0] [v2:172.21.15.18:3301/0,v1:172.21.15.18:6790/0]", 2023-09-23T20:22:43.688 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_host_override": "", 2023-09-23T20:22:43.688 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_initial_members": "", 2023-09-23T20:22:43.688 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_inject_pg_merge_bounce_probability": "0.000000", 2023-09-23T20:22:43.688 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_inject_sync_get_chunk_delay": "0.000000", 2023-09-23T20:22:43.688 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_inject_transaction_delay_max": "10.000000", 2023-09-23T20:22:43.688 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_inject_transaction_delay_probability": "0.000000", 2023-09-23T20:22:43.688 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_keyvaluedb": "rocksdb", 2023-09-23T20:22:43.688 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_lease": "5.000000", 2023-09-23T20:22:43.688 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_lease_ack_timeout_factor": "2.000000", 2023-09-23T20:22:43.689 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_lease_renew_interval_factor": "0.600000", 2023-09-23T20:22:43.689 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_log_full_interval": "50", 2023-09-23T20:22:43.689 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_log_max": "10000", 2023-09-23T20:22:43.689 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_log_max_summary": "50", 2023-09-23T20:22:43.689 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_max_log_entries_per_event": "4096", 2023-09-23T20:22:43.689 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_max_log_epochs": "500", 2023-09-23T20:22:43.689 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_max_mdsmap_epochs": "500", 2023-09-23T20:22:43.689 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_max_mgrmap_epochs": "500", 2023-09-23T20:22:43.689 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_max_osd": "10000", 2023-09-23T20:22:43.689 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_max_pg_per_osd": "10000", 2023-09-23T20:22:43.690 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_max_pool_pg_num": "65536", 2023-09-23T20:22:43.690 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_max_snap_prune_per_epoch": "100", 2023-09-23T20:22:43.690 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_mds_blocklist_interval": "86400.000000", 2023-09-23T20:22:43.690 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_mds_force_trim_to": "0", 2023-09-23T20:22:43.690 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_mds_skip_sanity": "true", 2023-09-23T20:22:43.690 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_memory_autotune": "true", 2023-09-23T20:22:43.690 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_memory_target": "2147483648", 2023-09-23T20:22:43.690 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_mgr_beacon_grace": "30", 2023-09-23T20:22:43.690 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_mgr_blocklist_interval": "86400.000000", 2023-09-23T20:22:43.690 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_mgr_digest_period": "5", 2023-09-23T20:22:43.691 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_mgr_inactive_grace": "60", 2023-09-23T20:22:43.691 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_mgr_mkfs_grace": "240", 2023-09-23T20:22:43.691 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_mgr_proxy_client_bytes_ratio": "0.300000", 2023-09-23T20:22:43.691 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_min_osdmap_epochs": "500", 2023-09-23T20:22:43.691 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_op_complaint_time": "30", 2023-09-23T20:22:43.691 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_op_history_duration": "600", 2023-09-23T20:22:43.691 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_op_history_size": "20", 2023-09-23T20:22:43.691 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_op_history_slow_op_size": "20", 2023-09-23T20:22:43.691 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_op_history_slow_op_threshold": "10", 2023-09-23T20:22:43.691 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_op_log_threshold": "5", 2023-09-23T20:22:43.692 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_osd_adjust_down_out_interval": "true", 2023-09-23T20:22:43.692 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_osd_adjust_heartbeat_grace": "true", 2023-09-23T20:22:43.692 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_osd_auto_mark_auto_out_in": "true", 2023-09-23T20:22:43.692 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_osd_auto_mark_in": "false", 2023-09-23T20:22:43.692 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_osd_auto_mark_new_in": "true", 2023-09-23T20:22:43.692 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_osd_backfillfull_ratio": "0.900000", 2023-09-23T20:22:43.692 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_osd_blocklist_default_expire": "3600.000000", 2023-09-23T20:22:43.692 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_osd_cache_size": "500", 2023-09-23T20:22:43.692 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_osd_cache_size_min": "134217728", 2023-09-23T20:22:43.693 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_osd_crush_smoke_test": "true", 2023-09-23T20:22:43.693 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_osd_destroyed_out_interval": "600", 2023-09-23T20:22:43.693 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_osd_down_out_interval": "600", 2023-09-23T20:22:43.693 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_osd_down_out_subtree_limit": "rack", 2023-09-23T20:22:43.693 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_osd_err_op_age_ratio": "128.000000", 2023-09-23T20:22:43.693 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_osd_force_trim_to": "0", 2023-09-23T20:22:43.693 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_osd_full_ratio": "0.950000", 2023-09-23T20:22:43.693 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_osd_initial_require_min_compat_client": "luminous", 2023-09-23T20:22:43.693 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_osd_laggy_halflife": "3600", 2023-09-23T20:22:43.693 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_osd_laggy_max_interval": "300", 2023-09-23T20:22:43.694 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_osd_laggy_weight": "0.300000", 2023-09-23T20:22:43.694 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_osd_mapping_pgs_per_chunk": "4096", 2023-09-23T20:22:43.694 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_osd_max_creating_pgs": "1024", 2023-09-23T20:22:43.694 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_osd_max_initial_pgs": "1024", 2023-09-23T20:22:43.694 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_osd_min_down_reporters": "2", 2023-09-23T20:22:43.694 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_osd_min_in_ratio": "0.750000", 2023-09-23T20:22:43.694 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_osd_min_up_ratio": "0.300000", 2023-09-23T20:22:43.694 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_osd_nearfull_ratio": "0.850000", 2023-09-23T20:22:43.694 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_osd_prime_pg_temp": "true", 2023-09-23T20:22:43.694 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_osd_prime_pg_temp_max_estimate": "0.250000", 2023-09-23T20:22:43.695 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_osd_prime_pg_temp_max_time": "0.500000", 2023-09-23T20:22:43.695 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_osd_report_timeout": "900", 2023-09-23T20:22:43.695 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_osd_reporter_subtree_level": "osd", 2023-09-23T20:22:43.695 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_osd_snap_trim_queue_warn_on": "32768", 2023-09-23T20:22:43.695 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_osd_warn_num_repaired": "10", 2023-09-23T20:22:43.695 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_osd_warn_op_age": "32.000000", 2023-09-23T20:22:43.695 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_osdmap_full_prune_enabled": "true", 2023-09-23T20:22:43.695 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_osdmap_full_prune_interval": "10", 2023-09-23T20:22:43.695 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_osdmap_full_prune_min": "10000", 2023-09-23T20:22:43.695 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_osdmap_full_prune_txsize": "100", 2023-09-23T20:22:43.696 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_pg_check_down_all_threshold": "0.500000", 2023-09-23T20:22:43.696 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_pg_stuck_threshold": "60", 2023-09-23T20:22:43.696 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_pg_warn_max_object_skew": "0.000000", 2023-09-23T20:22:43.696 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_pg_warn_min_objects": "10000", 2023-09-23T20:22:43.696 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_pg_warn_min_per_osd": "0", 2023-09-23T20:22:43.696 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_pg_warn_min_pool_objects": "1000", 2023-09-23T20:22:43.696 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_pool_quota_crit_threshold": "0", 2023-09-23T20:22:43.696 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_pool_quota_warn_threshold": "0", 2023-09-23T20:22:43.696 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_probe_timeout": "2.000000", 2023-09-23T20:22:43.696 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_reweight_max_change": "0.050000", 2023-09-23T20:22:43.697 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_reweight_max_osds": "4", 2023-09-23T20:22:43.697 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_reweight_min_bytes_per_osd": "10", 2023-09-23T20:22:43.697 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_reweight_min_pgs_per_osd": "4", 2023-09-23T20:22:43.697 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_rocksdb_options": "write_buffer_size=33554432,compression=kNoCompression,level_compaction_dynamic_level_bytes=true", 2023-09-23T20:22:43.697 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_scrub_inject_crc_mismatch": "0.000000", 2023-09-23T20:22:43.697 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_scrub_inject_missing_keys": "0.000000", 2023-09-23T20:22:43.697 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_scrub_interval": "86400", 2023-09-23T20:22:43.697 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_scrub_max_keys": "100", 2023-09-23T20:22:43.697 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_scrub_timeout": "300", 2023-09-23T20:22:43.697 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_session_timeout": "300", 2023-09-23T20:22:43.697 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_smart_report_timeout": "5", 2023-09-23T20:22:43.698 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_stat_smooth_intervals": "6", 2023-09-23T20:22:43.698 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_stretch_cluster_recovery_ratio": "0.600000", 2023-09-23T20:22:43.698 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_stretch_pool_min_size": "2", 2023-09-23T20:22:43.698 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_stretch_pool_size": "4", 2023-09-23T20:22:43.698 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_stretch_recovery_min_wait": "15.000000", 2023-09-23T20:22:43.698 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_subscribe_interval": "86400.000000", 2023-09-23T20:22:43.698 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_sync_debug": "false", 2023-09-23T20:22:43.698 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_sync_max_payload_keys": "2000", 2023-09-23T20:22:43.698 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_sync_max_payload_size": "1048576", 2023-09-23T20:22:43.699 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_sync_provider_kill_at": "0", 2023-09-23T20:22:43.699 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_sync_requester_kill_at": "0", 2023-09-23T20:22:43.699 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_sync_timeout": "60.000000", 2023-09-23T20:22:43.699 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_target_pg_per_osd": "100", 2023-09-23T20:22:43.699 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_tick_interval": "5", 2023-09-23T20:22:43.699 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_timecheck_interval": "300.000000", 2023-09-23T20:22:43.699 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_timecheck_skew_interval": "30.000000", 2023-09-23T20:22:43.699 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_use_min_delay_socket": "false", 2023-09-23T20:22:43.699 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_warn_older_version_delay": "604800", 2023-09-23T20:22:43.699 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_warn_on_cache_pools_without_hit_sets": "true", 2023-09-23T20:22:43.699 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_warn_on_crush_straw_calc_version_zero": "false", 2023-09-23T20:22:43.700 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_warn_on_degraded_stretch_mode": "true", 2023-09-23T20:22:43.700 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_warn_on_filestore_osds": "true", 2023-09-23T20:22:43.700 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_warn_on_insecure_global_id_reclaim": "false", 2023-09-23T20:22:43.700 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_warn_on_insecure_global_id_reclaim_allowed": "false", 2023-09-23T20:22:43.700 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_warn_on_legacy_crush_tunables": "false", 2023-09-23T20:22:43.700 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_warn_on_misplaced": "false", 2023-09-23T20:22:43.700 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_warn_on_msgr2_not_enabled": "true", 2023-09-23T20:22:43.701 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_warn_on_older_version": "true", 2023-09-23T20:22:43.701 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_warn_on_osd_down_out_interval_zero": "false", 2023-09-23T20:22:43.701 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_warn_on_pool_no_app": "true", 2023-09-23T20:22:43.701 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_warn_on_pool_no_redundancy": "true", 2023-09-23T20:22:43.701 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_warn_on_pool_pg_num_not_power_of_two": "false", 2023-09-23T20:22:43.701 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_warn_on_slow_ping_ratio": "0.050000", 2023-09-23T20:22:43.701 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_warn_on_slow_ping_time": "0.000000", 2023-09-23T20:22:43.701 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_warn_on_too_few_osds": "false", 2023-09-23T20:22:43.701 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_warn_pg_not_deep_scrubbed_ratio": "0.750000", 2023-09-23T20:22:43.701 INFO:teuthology.orchestra.run.smithi018.stdout: "mon_warn_pg_not_scrubbed_ratio": "0.500000", 2023-09-23T20:22:43.702 INFO:teuthology.orchestra.run.smithi018.stdout: "monmap": "", 2023-09-23T20:22:43.702 INFO:teuthology.orchestra.run.smithi018.stdout: "motr_admin_endpoint": "192.168.180.182@tcp:12345:4:1", 2023-09-23T20:22:43.702 INFO:teuthology.orchestra.run.smithi018.stdout: "motr_admin_fid": "0x7200000000000001:0x0", 2023-09-23T20:22:43.702 INFO:teuthology.orchestra.run.smithi018.stdout: "motr_ha_endpoint": "192.168.180.182@tcp:12345:1:1", 2023-09-23T20:22:43.702 INFO:teuthology.orchestra.run.smithi018.stdout: "motr_my_endpoint": "192.168.180.182@tcp:12345:4:1", 2023-09-23T20:22:43.702 INFO:teuthology.orchestra.run.smithi018.stdout: "motr_my_fid": "0x7200000000000001:0x0", 2023-09-23T20:22:43.702 INFO:teuthology.orchestra.run.smithi018.stdout: "motr_profile_fid": "0x7000000000000001:0x0", 2023-09-23T20:22:43.702 INFO:teuthology.orchestra.run.smithi018.stdout: "motr_tracing_enabled": "false", 2023-09-23T20:22:43.702 INFO:teuthology.orchestra.run.smithi018.stdout: "ms_async_op_threads": "3", 2023-09-23T20:22:43.702 INFO:teuthology.orchestra.run.smithi018.stdout: "ms_async_rdma_buffer_size": "131072", 2023-09-23T20:22:43.703 INFO:teuthology.orchestra.run.smithi018.stdout: "ms_async_rdma_cm": "false", 2023-09-23T20:22:43.703 INFO:teuthology.orchestra.run.smithi018.stdout: "ms_async_rdma_device_name": "", 2023-09-23T20:22:43.703 INFO:teuthology.orchestra.run.smithi018.stdout: "ms_async_rdma_dscp": "96", 2023-09-23T20:22:43.703 INFO:teuthology.orchestra.run.smithi018.stdout: "ms_async_rdma_enable_hugepage": "false", 2023-09-23T20:22:43.703 INFO:teuthology.orchestra.run.smithi018.stdout: "ms_async_rdma_gid_idx": "0", 2023-09-23T20:22:43.703 INFO:teuthology.orchestra.run.smithi018.stdout: "ms_async_rdma_local_gid": "", 2023-09-23T20:22:43.703 INFO:teuthology.orchestra.run.smithi018.stdout: "ms_async_rdma_polling_us": "1000", 2023-09-23T20:22:43.703 INFO:teuthology.orchestra.run.smithi018.stdout: "ms_async_rdma_port_num": "1", 2023-09-23T20:22:43.703 INFO:teuthology.orchestra.run.smithi018.stdout: "ms_async_rdma_receive_buffers": "32768", 2023-09-23T20:22:43.703 INFO:teuthology.orchestra.run.smithi018.stdout: "ms_async_rdma_receive_queue_len": "4096", 2023-09-23T20:22:43.704 INFO:teuthology.orchestra.run.smithi018.stdout: "ms_async_rdma_roce_ver": "1", 2023-09-23T20:22:43.704 INFO:teuthology.orchestra.run.smithi018.stdout: "ms_async_rdma_send_buffers": "1024", 2023-09-23T20:22:43.704 INFO:teuthology.orchestra.run.smithi018.stdout: "ms_async_rdma_sl": "3", 2023-09-23T20:22:43.704 INFO:teuthology.orchestra.run.smithi018.stdout: "ms_async_rdma_support_srq": "true", 2023-09-23T20:22:43.704 INFO:teuthology.orchestra.run.smithi018.stdout: "ms_async_rdma_type": "ib", 2023-09-23T20:22:43.704 INFO:teuthology.orchestra.run.smithi018.stdout: "ms_async_reap_threshold": "5", 2023-09-23T20:22:43.704 INFO:teuthology.orchestra.run.smithi018.stdout: "ms_bind_before_connect": "false", 2023-09-23T20:22:43.704 INFO:teuthology.orchestra.run.smithi018.stdout: "ms_bind_ipv4": "true", 2023-09-23T20:22:43.704 INFO:teuthology.orchestra.run.smithi018.stdout: "ms_bind_ipv6": "false", 2023-09-23T20:22:43.704 INFO:teuthology.orchestra.run.smithi018.stdout: "ms_bind_msgr1": "true", 2023-09-23T20:22:43.704 INFO:teuthology.orchestra.run.smithi018.stdout: "ms_bind_msgr2": "true", 2023-09-23T20:22:43.704 INFO:teuthology.orchestra.run.smithi018.stdout: "ms_bind_port_max": "7568", 2023-09-23T20:22:43.705 INFO:teuthology.orchestra.run.smithi018.stdout: "ms_bind_port_min": "6800", 2023-09-23T20:22:43.705 INFO:teuthology.orchestra.run.smithi018.stdout: "ms_bind_prefer_ipv4": "false", 2023-09-23T20:22:43.705 INFO:teuthology.orchestra.run.smithi018.stdout: "ms_bind_retry_count": "3", 2023-09-23T20:22:43.705 INFO:teuthology.orchestra.run.smithi018.stdout: "ms_bind_retry_delay": "5", 2023-09-23T20:22:43.705 INFO:teuthology.orchestra.run.smithi018.stdout: "ms_blackhole_client": "false", 2023-09-23T20:22:43.705 INFO:teuthology.orchestra.run.smithi018.stdout: "ms_blackhole_mds": "false", 2023-09-23T20:22:43.705 INFO:teuthology.orchestra.run.smithi018.stdout: "ms_blackhole_mgr": "false", 2023-09-23T20:22:43.705 INFO:teuthology.orchestra.run.smithi018.stdout: "ms_blackhole_mon": "false", 2023-09-23T20:22:43.705 INFO:teuthology.orchestra.run.smithi018.stdout: "ms_blackhole_osd": "false", 2023-09-23T20:22:43.705 INFO:teuthology.orchestra.run.smithi018.stdout: "ms_client_mode": "crc secure", 2023-09-23T20:22:43.706 INFO:teuthology.orchestra.run.smithi018.stdout: "ms_cluster_mode": "crc secure", 2023-09-23T20:22:43.706 INFO:teuthology.orchestra.run.smithi018.stdout: "ms_cluster_type": "", 2023-09-23T20:22:43.706 INFO:teuthology.orchestra.run.smithi018.stdout: "ms_compress_secure": "false", 2023-09-23T20:22:43.706 INFO:teuthology.orchestra.run.smithi018.stdout: "ms_connection_idle_timeout": "900", 2023-09-23T20:22:43.706 INFO:teuthology.orchestra.run.smithi018.stdout: "ms_connection_ready_timeout": "10", 2023-09-23T20:22:43.706 INFO:teuthology.orchestra.run.smithi018.stdout: "ms_crc_data": "true", 2023-09-23T20:22:43.706 INFO:teuthology.orchestra.run.smithi018.stdout: "ms_crc_header": "true", 2023-09-23T20:22:43.706 INFO:teuthology.orchestra.run.smithi018.stdout: "ms_die_on_bad_msg": "false", 2023-09-23T20:22:43.706 INFO:teuthology.orchestra.run.smithi018.stdout: "ms_die_on_bug": "true", 2023-09-23T20:22:43.706 INFO:teuthology.orchestra.run.smithi018.stdout: "ms_die_on_old_message": "true", 2023-09-23T20:22:43.707 INFO:teuthology.orchestra.run.smithi018.stdout: "ms_die_on_skipped_message": "false", 2023-09-23T20:22:43.707 INFO:teuthology.orchestra.run.smithi018.stdout: "ms_die_on_unhandled_msg": "false", 2023-09-23T20:22:43.707 INFO:teuthology.orchestra.run.smithi018.stdout: "ms_dispatch_throttle_bytes": "104857600", 2023-09-23T20:22:43.707 INFO:teuthology.orchestra.run.smithi018.stdout: "ms_dpdk_coremask": "0xF", 2023-09-23T20:22:43.707 INFO:teuthology.orchestra.run.smithi018.stdout: "ms_dpdk_debug_allow_loopback": "false", 2023-09-23T20:22:43.707 INFO:teuthology.orchestra.run.smithi018.stdout: "ms_dpdk_devs_allowlist": "", 2023-09-23T20:22:43.707 INFO:teuthology.orchestra.run.smithi018.stdout: "ms_dpdk_enable_tso": "true", 2023-09-23T20:22:43.707 INFO:teuthology.orchestra.run.smithi018.stdout: "ms_dpdk_gateway_ipv4_addr": "", 2023-09-23T20:22:43.707 INFO:teuthology.orchestra.run.smithi018.stdout: "ms_dpdk_host_ipv4_addr": "", 2023-09-23T20:22:43.707 INFO:teuthology.orchestra.run.smithi018.stdout: "ms_dpdk_hugepages": "", 2023-09-23T20:22:43.708 INFO:teuthology.orchestra.run.smithi018.stdout: "ms_dpdk_hw_flow_control": "true", 2023-09-23T20:22:43.708 INFO:teuthology.orchestra.run.smithi018.stdout: "ms_dpdk_hw_queue_weight": "1.000000", 2023-09-23T20:22:43.708 INFO:teuthology.orchestra.run.smithi018.stdout: "ms_dpdk_lro": "true", 2023-09-23T20:22:43.708 INFO:teuthology.orchestra.run.smithi018.stdout: "ms_dpdk_memory_channel": "4", 2023-09-23T20:22:43.708 INFO:teuthology.orchestra.run.smithi018.stdout: "ms_dpdk_netmask_ipv4_addr": "", 2023-09-23T20:22:43.708 INFO:teuthology.orchestra.run.smithi018.stdout: "ms_dpdk_pmd": "", 2023-09-23T20:22:43.708 INFO:teuthology.orchestra.run.smithi018.stdout: "ms_dpdk_port_id": "0", 2023-09-23T20:22:43.708 INFO:teuthology.orchestra.run.smithi018.stdout: "ms_dpdk_rx_buffer_count_per_core": "8192", 2023-09-23T20:22:43.708 INFO:teuthology.orchestra.run.smithi018.stdout: "ms_dump_corrupt_message_level": "1", 2023-09-23T20:22:43.708 INFO:teuthology.orchestra.run.smithi018.stdout: "ms_dump_on_send": "false", 2023-09-23T20:22:43.708 INFO:teuthology.orchestra.run.smithi018.stdout: "ms_initial_backoff": "0.200000", 2023-09-23T20:22:43.708 INFO:teuthology.orchestra.run.smithi018.stdout: "ms_inject_delay_max": "1.000000", 2023-09-23T20:22:43.709 INFO:teuthology.orchestra.run.smithi018.stdout: "ms_inject_delay_probability": "0.000000", 2023-09-23T20:22:43.709 INFO:teuthology.orchestra.run.smithi018.stdout: "ms_inject_delay_type": "", 2023-09-23T20:22:43.709 INFO:teuthology.orchestra.run.smithi018.stdout: "ms_inject_internal_delays": "0.000000", 2023-09-23T20:22:43.709 INFO:teuthology.orchestra.run.smithi018.stdout: "ms_inject_network_congestion": "0", 2023-09-23T20:22:43.709 INFO:teuthology.orchestra.run.smithi018.stdout: "ms_inject_socket_failures": "0", 2023-09-23T20:22:43.709 INFO:teuthology.orchestra.run.smithi018.stdout: "ms_learn_addr_from_peer": "true", 2023-09-23T20:22:43.709 INFO:teuthology.orchestra.run.smithi018.stdout: "ms_max_accept_failures": "4", 2023-09-23T20:22:43.709 INFO:teuthology.orchestra.run.smithi018.stdout: "ms_max_backoff": "15.000000", 2023-09-23T20:22:43.709 INFO:teuthology.orchestra.run.smithi018.stdout: "ms_mon_client_mode": "secure crc", 2023-09-23T20:22:43.709 INFO:teuthology.orchestra.run.smithi018.stdout: "ms_mon_cluster_mode": "secure crc", 2023-09-23T20:22:43.710 INFO:teuthology.orchestra.run.smithi018.stdout: "ms_mon_service_mode": "secure crc", 2023-09-23T20:22:43.710 INFO:teuthology.orchestra.run.smithi018.stdout: "ms_osd_compress_min_size": "1024", 2023-09-23T20:22:43.710 INFO:teuthology.orchestra.run.smithi018.stdout: "ms_osd_compress_mode": "none", 2023-09-23T20:22:43.710 INFO:teuthology.orchestra.run.smithi018.stdout: "ms_osd_compression_algorithm": "snappy", 2023-09-23T20:22:43.710 INFO:teuthology.orchestra.run.smithi018.stdout: "ms_pq_max_tokens_per_priority": "16777216", 2023-09-23T20:22:43.710 INFO:teuthology.orchestra.run.smithi018.stdout: "ms_pq_min_cost": "65536", 2023-09-23T20:22:43.710 INFO:teuthology.orchestra.run.smithi018.stdout: "ms_public_type": "", 2023-09-23T20:22:43.710 INFO:teuthology.orchestra.run.smithi018.stdout: "ms_service_mode": "crc secure", 2023-09-23T20:22:43.710 INFO:teuthology.orchestra.run.smithi018.stdout: "ms_tcp_listen_backlog": "512", 2023-09-23T20:22:43.710 INFO:teuthology.orchestra.run.smithi018.stdout: "ms_tcp_nodelay": "true", 2023-09-23T20:22:43.710 INFO:teuthology.orchestra.run.smithi018.stdout: "ms_tcp_prefetch_max_size": "4096", 2023-09-23T20:22:43.711 INFO:teuthology.orchestra.run.smithi018.stdout: "ms_tcp_rcvbuf": "0", 2023-09-23T20:22:43.711 INFO:teuthology.orchestra.run.smithi018.stdout: "ms_type": "async+posix", 2023-09-23T20:22:43.711 INFO:teuthology.orchestra.run.smithi018.stdout: "no_config_file": "false", 2023-09-23T20:22:43.711 INFO:teuthology.orchestra.run.smithi018.stdout: "objecter_completion_locks_per_session": "32", 2023-09-23T20:22:43.711 INFO:teuthology.orchestra.run.smithi018.stdout: "objecter_debug_inject_relock_delay": "false", 2023-09-23T20:22:43.711 INFO:teuthology.orchestra.run.smithi018.stdout: "objecter_inflight_op_bytes": "104857600", 2023-09-23T20:22:43.711 INFO:teuthology.orchestra.run.smithi018.stdout: "objecter_inflight_ops": "1024", 2023-09-23T20:22:43.711 INFO:teuthology.orchestra.run.smithi018.stdout: "objecter_inject_no_watch_ping": "false", 2023-09-23T20:22:43.711 INFO:teuthology.orchestra.run.smithi018.stdout: "objecter_retry_writes_after_first_reply": "false", 2023-09-23T20:22:43.711 INFO:teuthology.orchestra.run.smithi018.stdout: "objecter_tick_interval": "5.000000", 2023-09-23T20:22:43.712 INFO:teuthology.orchestra.run.smithi018.stdout: "objecter_timeout": "10.000000", 2023-09-23T20:22:43.712 INFO:teuthology.orchestra.run.smithi018.stdout: "objectstore_blackhole": "false", 2023-09-23T20:22:43.712 INFO:teuthology.orchestra.run.smithi018.stdout: "openssl_engine_opts": "", 2023-09-23T20:22:43.712 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_agent_delay_time": "5.000000", 2023-09-23T20:22:43.712 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_agent_hist_halflife": "1000", 2023-09-23T20:22:43.712 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_agent_max_low_ops": "2", 2023-09-23T20:22:43.713 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_agent_max_ops": "4", 2023-09-23T20:22:43.713 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_agent_min_evict_effort": "0.100000", 2023-09-23T20:22:43.713 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_agent_quantize_effort": "0.100000", 2023-09-23T20:22:43.713 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_agent_slop": "0.020000", 2023-09-23T20:22:43.713 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_aggregated_slow_ops_logging": "true", 2023-09-23T20:22:43.713 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_allow_recovery_below_min_size": "true", 2023-09-23T20:22:43.713 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_asio_thread_count": "2", 2023-09-23T20:22:43.714 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_async_recovery_min_cost": "100", 2023-09-23T20:22:43.714 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_auto_mark_unfound_lost": "false", 2023-09-23T20:22:43.714 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_backfill_retry_interval": "30.000000", 2023-09-23T20:22:43.714 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_backfill_scan_max": "512", 2023-09-23T20:22:43.714 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_backfill_scan_min": "64", 2023-09-23T20:22:43.714 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_backoff_on_degraded": "false", 2023-09-23T20:22:43.714 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_backoff_on_peering": "false", 2023-09-23T20:22:43.714 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_backoff_on_unfound": "true", 2023-09-23T20:22:43.715 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_beacon_report_interval": "300", 2023-09-23T20:22:43.715 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_bench_duration": "30", 2023-09-23T20:22:43.715 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_bench_large_size_max_throughput": "104857600", 2023-09-23T20:22:43.715 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_bench_max_block_size": "67108864", 2023-09-23T20:22:43.715 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_bench_small_size_max_iops": "100", 2023-09-23T20:22:43.715 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_blkin_trace_all": "false", 2023-09-23T20:22:43.715 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_blocked_scrub_grace_period": "120", 2023-09-23T20:22:43.716 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_calc_pg_upmaps_aggressively": "true", 2023-09-23T20:22:43.716 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_calc_pg_upmaps_aggressively_fast": "true", 2023-09-23T20:22:43.716 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_calc_pg_upmaps_local_fallback_retries": "100", 2023-09-23T20:22:43.716 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_check_for_log_corruption": "false", 2023-09-23T20:22:43.716 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_check_max_object_name_len_on_startup": "true", 2023-09-23T20:22:43.716 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_class_default_list": "cephfs hello journal lock log numops otp rbd refcount rgw rgw_gc timeindex user version cas cmpomap queue 2pc_queue fifo", 2023-09-23T20:22:43.716 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_class_dir": "/usr/lib64/rados-classes", 2023-09-23T20:22:43.717 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_class_load_list": "cephfs hello journal lock log numops otp rbd refcount rgw rgw_gc timeindex user version cas cmpomap queue 2pc_queue fifo", 2023-09-23T20:22:43.717 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_class_update_on_start": "true", 2023-09-23T20:22:43.717 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_client_message_cap": "256", 2023-09-23T20:22:43.717 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_client_message_size_cap": "524288000", 2023-09-23T20:22:43.717 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_client_op_priority": "63", 2023-09-23T20:22:43.717 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_client_watch_timeout": "30", 2023-09-23T20:22:43.717 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_command_max_records": "256", 2023-09-23T20:22:43.718 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_compact_on_start": "false", 2023-09-23T20:22:43.718 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_copyfrom_max_chunk": "8388608", 2023-09-23T20:22:43.718 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_crush_chooseleaf_type": "1", 2023-09-23T20:22:43.718 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_crush_initial_weight": "-1.000000", 2023-09-23T20:22:43.718 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_crush_update_on_start": "true", 2023-09-23T20:22:43.718 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_crush_update_weight_set": "true", 2023-09-23T20:22:43.718 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_data": "/var/lib/ceph/osd/ceph-a", 2023-09-23T20:22:43.719 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_debug_crash_on_ignored_backoff": "false", 2023-09-23T20:22:43.719 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_debug_deep_scrub_sleep": "0.000000", 2023-09-23T20:22:43.719 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_debug_drop_ping_duration": "0", 2023-09-23T20:22:43.719 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_debug_drop_ping_probability": "0.000000", 2023-09-23T20:22:43.719 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_debug_feed_pullee": "-1", 2023-09-23T20:22:43.719 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_debug_inject_copyfrom_error": "false", 2023-09-23T20:22:43.719 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_debug_inject_dispatch_delay_duration": "0.100000", 2023-09-23T20:22:43.720 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_debug_inject_dispatch_delay_probability": "0.000000", 2023-09-23T20:22:43.720 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_debug_misdirected_ops": "false", 2023-09-23T20:22:43.720 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_debug_no_acting_change": "false", 2023-09-23T20:22:43.720 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_debug_no_purge_strays": "false", 2023-09-23T20:22:43.720 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_debug_op_order": "false", 2023-09-23T20:22:43.720 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_debug_pg_log_writeout": "false", 2023-09-23T20:22:43.720 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_debug_pretend_recovery_active": "false", 2023-09-23T20:22:43.720 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_debug_random_push_read_error": "0.000000", 2023-09-23T20:22:43.721 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_debug_reject_backfill_probability": "0.000000", 2023-09-23T20:22:43.721 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_debug_shutdown": "false", 2023-09-23T20:22:43.721 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_debug_skip_full_check_in_backfill_reservation": "false", 2023-09-23T20:22:43.721 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_debug_skip_full_check_in_recovery": "false", 2023-09-23T20:22:43.721 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_debug_verify_cached_snaps": "false", 2023-09-23T20:22:43.721 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_debug_verify_missing_on_start": "false", 2023-09-23T20:22:43.722 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_debug_verify_snaps": "false", 2023-09-23T20:22:43.722 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_debug_verify_stray_on_activate": "false", 2023-09-23T20:22:43.722 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_deep_scrub_interval": "604800.000000", 2023-09-23T20:22:43.722 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_deep_scrub_keys": "1024", 2023-09-23T20:22:43.722 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_deep_scrub_large_omap_object_key_threshold": "200000", 2023-09-23T20:22:43.722 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_deep_scrub_large_omap_object_value_sum_threshold": "1073741824", 2023-09-23T20:22:43.722 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_deep_scrub_randomize_ratio": "0.150000", 2023-09-23T20:22:43.723 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_deep_scrub_stride": "524288", 2023-09-23T20:22:43.723 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_deep_scrub_update_digest_min_age": "7200", 2023-09-23T20:22:43.723 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_default_data_pool_replay_window": "45", 2023-09-23T20:22:43.723 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_default_notify_timeout": "30", 2023-09-23T20:22:43.723 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_delete_sleep": "0.000000", 2023-09-23T20:22:43.723 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_delete_sleep_hdd": "5.000000", 2023-09-23T20:22:43.723 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_delete_sleep_hybrid": "1.000000", 2023-09-23T20:22:43.724 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_delete_sleep_ssd": "1.000000", 2023-09-23T20:22:43.724 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_discard_disconnected_ops": "true", 2023-09-23T20:22:43.724 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_enable_op_tracker": "true", 2023-09-23T20:22:43.724 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_erasure_code_plugins": "jerasure lrc", 2023-09-23T20:22:43.724 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_extblkdev_plugins": "vdo", 2023-09-23T20:22:43.724 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_failsafe_full_ratio": "0.970000", 2023-09-23T20:22:43.724 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_fast_fail_on_connection_refused": "true", 2023-09-23T20:22:43.725 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_fast_info": "true", 2023-09-23T20:22:43.725 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_fast_shutdown": "true", 2023-09-23T20:22:43.725 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_fast_shutdown_notify_mon": "true", 2023-09-23T20:22:43.725 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_fast_shutdown_timeout": "15", 2023-09-23T20:22:43.725 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_find_best_info_ignore_history_les": "false", 2023-09-23T20:22:43.725 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_force_auth_primary_missing_objects": "100", 2023-09-23T20:22:43.725 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_force_recovery_pg_log_entries_factor": "1.300000", 2023-09-23T20:22:43.725 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_function_tracing": "false", 2023-09-23T20:22:43.726 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_heartbeat_grace": "20", 2023-09-23T20:22:43.726 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_heartbeat_interval": "6", 2023-09-23T20:22:43.726 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_heartbeat_min_healthy_ratio": "0.330000", 2023-09-23T20:22:43.726 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_heartbeat_min_peers": "10", 2023-09-23T20:22:43.726 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_heartbeat_min_size": "2000", 2023-09-23T20:22:43.727 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_heartbeat_stale": "600", 2023-09-23T20:22:43.727 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_heartbeat_use_min_delay_socket": "false", 2023-09-23T20:22:43.727 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_hit_set_max_size": "100000", 2023-09-23T20:22:43.727 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_hit_set_min_size": "1000", 2023-09-23T20:22:43.727 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_hit_set_namespace": ".ceph-internal", 2023-09-23T20:22:43.727 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_ignore_stale_divergent_priors": "false", 2023-09-23T20:22:43.727 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_inject_bad_map_crc_probability": "0.000000", 2023-09-23T20:22:43.728 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_inject_failure_on_pg_removal": "false", 2023-09-23T20:22:43.728 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_journal": "/var/lib/ceph/osd/ceph-a/journal", 2023-09-23T20:22:43.728 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_journal_flush_on_shutdown": "true", 2023-09-23T20:22:43.728 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_journal_size": "5120", 2023-09-23T20:22:43.728 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_kill_backfill_at": "0", 2023-09-23T20:22:43.728 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_loop_before_reset_tphandle": "64", 2023-09-23T20:22:43.728 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_map_cache_size": "50", 2023-09-23T20:22:43.729 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_map_dedup": "true", 2023-09-23T20:22:43.729 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_map_message_max": "40", 2023-09-23T20:22:43.729 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_map_message_max_bytes": "10485760", 2023-09-23T20:22:43.729 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_map_share_max_epochs": "40", 2023-09-23T20:22:43.729 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_max_attr_name_len": "100", 2023-09-23T20:22:43.729 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_max_attr_size": "0", 2023-09-23T20:22:43.729 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_max_backfills": "1", 2023-09-23T20:22:43.730 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_max_markdown_count": "5", 2023-09-23T20:22:43.730 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_max_markdown_period": "600", 2023-09-23T20:22:43.730 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_max_object_name_len": "2048", 2023-09-23T20:22:43.730 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_max_object_namespace_len": "256", 2023-09-23T20:22:43.730 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_max_object_size": "134217728", 2023-09-23T20:22:43.730 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_max_omap_bytes_per_request": "1073741824", 2023-09-23T20:22:43.731 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_max_omap_entries_per_request": "1024", 2023-09-23T20:22:43.731 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_max_pg_blocked_by": "16", 2023-09-23T20:22:43.731 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_max_pg_log_entries": "10000", 2023-09-23T20:22:43.731 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_max_pg_per_osd_hard_ratio": "3.000000", 2023-09-23T20:22:43.731 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_max_pgls": "1024", 2023-09-23T20:22:43.731 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_max_push_cost": "8388608", 2023-09-23T20:22:43.731 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_max_push_objects": "10", 2023-09-23T20:22:43.732 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_max_scrubs": "1", 2023-09-23T20:22:43.732 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_max_snap_prune_intervals_per_epoch": "512", 2023-09-23T20:22:43.732 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_max_trimming_pgs": "2", 2023-09-23T20:22:43.732 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_max_write_op_reply_len": "64", 2023-09-23T20:22:43.732 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_max_write_size": "90", 2023-09-23T20:22:43.732 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_mclock_force_run_benchmark_on_init": "false", 2023-09-23T20:22:43.733 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_mclock_iops_capacity_threshold_hdd": "500.000000", 2023-09-23T20:22:43.733 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_mclock_iops_capacity_threshold_ssd": "80000.000000", 2023-09-23T20:22:43.733 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_mclock_max_capacity_iops_hdd": "315.000000", 2023-09-23T20:22:43.733 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_mclock_max_capacity_iops_ssd": "21500.000000", 2023-09-23T20:22:43.733 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_mclock_max_sequential_bandwidth_hdd": "157286400", 2023-09-23T20:22:43.733 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_mclock_max_sequential_bandwidth_ssd": "1258291200", 2023-09-23T20:22:43.734 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_mclock_override_recovery_settings": "false", 2023-09-23T20:22:43.734 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_mclock_profile": "balanced", 2023-09-23T20:22:43.734 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_mclock_scheduler_anticipation_timeout": "0.000000", 2023-09-23T20:22:43.734 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_mclock_scheduler_background_best_effort_lim": "0.000000", 2023-09-23T20:22:43.734 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_mclock_scheduler_background_best_effort_res": "0.000000", 2023-09-23T20:22:43.734 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_mclock_scheduler_background_best_effort_wgt": "1", 2023-09-23T20:22:43.734 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_mclock_scheduler_background_recovery_lim": "0.000000", 2023-09-23T20:22:43.735 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_mclock_scheduler_background_recovery_res": "0.000000", 2023-09-23T20:22:43.735 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_mclock_scheduler_background_recovery_wgt": "1", 2023-09-23T20:22:43.735 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_mclock_scheduler_client_lim": "0.000000", 2023-09-23T20:22:43.735 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_mclock_scheduler_client_res": "0.000000", 2023-09-23T20:22:43.735 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_mclock_scheduler_client_wgt": "1", 2023-09-23T20:22:43.735 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_mclock_skip_benchmark": "false", 2023-09-23T20:22:43.736 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_memory_base": "805306368", 2023-09-23T20:22:43.736 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_memory_cache_min": "134217728", 2023-09-23T20:22:43.736 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_memory_cache_resize_interval": "1.000000", 2023-09-23T20:22:43.736 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_memory_expected_fragmentation": "0.150000", 2023-09-23T20:22:43.736 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_memory_target": "4294967296", 2023-09-23T20:22:43.736 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_memory_target_autotune": "false", 2023-09-23T20:22:43.736 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_memory_target_cgroup_limit_ratio": "0.800000", 2023-09-23T20:22:43.737 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_min_pg_log_entries": "250", 2023-09-23T20:22:43.737 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_min_recovery_priority": "0", 2023-09-23T20:22:43.737 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_mon_heartbeat_interval": "30", 2023-09-23T20:22:43.737 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_mon_heartbeat_stat_stale": "3600", 2023-09-23T20:22:43.737 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_mon_report_interval": "5", 2023-09-23T20:22:43.737 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_mon_report_max_in_flight": "2", 2023-09-23T20:22:43.737 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_mon_shutdown_timeout": "5.000000", 2023-09-23T20:22:43.737 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_num_cache_shards": "32", 2023-09-23T20:22:43.738 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_num_op_tracker_shard": "32", 2023-09-23T20:22:43.738 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_numa_auto_affinity": "true", 2023-09-23T20:22:43.738 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_numa_node": "-1", 2023-09-23T20:22:43.738 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_numa_prefer_iface": "true", 2023-09-23T20:22:43.738 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_object_clean_region_max_num_intervals": "10", 2023-09-23T20:22:43.738 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_objecter_finishers": "1", 2023-09-23T20:22:43.739 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_objectstore": "bluestore", 2023-09-23T20:22:43.739 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_objectstore_fuse": "false", 2023-09-23T20:22:43.739 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_objectstore_tracing": "false", 2023-09-23T20:22:43.739 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_op_complaint_time": "30.000000", 2023-09-23T20:22:43.739 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_op_history_duration": "600", 2023-09-23T20:22:43.739 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_op_history_size": "20", 2023-09-23T20:22:43.739 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_op_history_slow_op_size": "20", 2023-09-23T20:22:43.740 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_op_history_slow_op_threshold": "10.000000", 2023-09-23T20:22:43.740 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_op_log_threshold": "5", 2023-09-23T20:22:43.740 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_op_num_shards": "0", 2023-09-23T20:22:43.740 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_op_num_shards_hdd": "5", 2023-09-23T20:22:43.740 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_op_num_shards_ssd": "8", 2023-09-23T20:22:43.740 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_op_num_threads_per_shard": "0", 2023-09-23T20:22:43.741 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_op_num_threads_per_shard_hdd": "1", 2023-09-23T20:22:43.741 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_op_num_threads_per_shard_ssd": "2", 2023-09-23T20:22:43.741 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_op_pq_max_tokens_per_priority": "4194304", 2023-09-23T20:22:43.741 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_op_pq_min_cost": "65536", 2023-09-23T20:22:43.741 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_op_queue": "mclock_scheduler", 2023-09-23T20:22:43.741 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_op_queue_cut_off": "high", 2023-09-23T20:22:43.742 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_op_thread_suicide_timeout": "150", 2023-09-23T20:22:43.742 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_op_thread_timeout": "15", 2023-09-23T20:22:43.742 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_open_classes_on_start": "true", 2023-09-23T20:22:43.742 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_os_flags": "0", 2023-09-23T20:22:43.742 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_peering_op_priority": "255", 2023-09-23T20:22:43.742 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_pg_delete_cost": "1048576", 2023-09-23T20:22:43.742 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_pg_delete_priority": "5", 2023-09-23T20:22:43.743 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_pg_epoch_max_lag_factor": "2.000000", 2023-09-23T20:22:43.743 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_pg_epoch_persisted_max_stale": "40", 2023-09-23T20:22:43.743 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_pg_log_dups_tracked": "3000", 2023-09-23T20:22:43.743 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_pg_log_trim_max": "10000", 2023-09-23T20:22:43.743 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_pg_log_trim_min": "100", 2023-09-23T20:22:43.743 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_pg_max_concurrent_snap_trims": "2", 2023-09-23T20:22:43.743 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_pg_object_context_cache_count": "64", 2023-09-23T20:22:43.743 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_pg_stat_report_interval_max": "500", 2023-09-23T20:22:43.744 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_pool_default_cache_max_evict_check_size": "10", 2023-09-23T20:22:43.744 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_pool_default_cache_min_evict_age": "0", 2023-09-23T20:22:43.744 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_pool_default_cache_min_flush_age": "0", 2023-09-23T20:22:43.744 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_pool_default_cache_target_dirty_high_ratio": "0.600000", 2023-09-23T20:22:43.744 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_pool_default_cache_target_dirty_ratio": "0.400000", 2023-09-23T20:22:43.744 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_pool_default_cache_target_full_ratio": "0.800000", 2023-09-23T20:22:43.744 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_pool_default_crimson": "false", 2023-09-23T20:22:43.744 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_pool_default_crush_rule": "-1", 2023-09-23T20:22:43.744 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_pool_default_ec_fast_read": "false", 2023-09-23T20:22:43.744 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_pool_default_erasure_code_profile": "plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd", 2023-09-23T20:22:43.744 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_pool_default_flag_bulk": "false", 2023-09-23T20:22:43.745 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_pool_default_flag_hashpspool": "true", 2023-09-23T20:22:43.745 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_pool_default_flag_nodelete": "false", 2023-09-23T20:22:43.745 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_pool_default_flag_nopgchange": "false", 2023-09-23T20:22:43.745 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_pool_default_flag_nosizechange": "false", 2023-09-23T20:22:43.745 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_pool_default_flags": "0", 2023-09-23T20:22:43.745 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_pool_default_hit_set_bloom_fpp": "0.050000", 2023-09-23T20:22:43.745 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_pool_default_min_size": "0", 2023-09-23T20:22:43.745 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_pool_default_pg_autoscale_mode": "off", 2023-09-23T20:22:43.745 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_pool_default_pg_num": "32", 2023-09-23T20:22:43.745 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_pool_default_pgp_num": "0", 2023-09-23T20:22:43.745 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_pool_default_read_lease_ratio": "0.800000", 2023-09-23T20:22:43.745 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_pool_default_size": "3", 2023-09-23T20:22:43.746 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_pool_default_type": "replicated", 2023-09-23T20:22:43.746 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_pool_erasure_code_stripe_unit": "4096", 2023-09-23T20:22:43.746 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_pool_use_gmt_hitset": "true", 2023-09-23T20:22:43.746 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_push_per_object_cost": "1000", 2023-09-23T20:22:43.746 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_read_ec_check_for_errors": "false", 2023-09-23T20:22:43.746 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_recover_clone_overlap": "true", 2023-09-23T20:22:43.746 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_recover_clone_overlap_limit": "10", 2023-09-23T20:22:43.746 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_recovery_cost": "20971520", 2023-09-23T20:22:43.746 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_recovery_delay_start": "0.000000", 2023-09-23T20:22:43.746 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_recovery_max_active": "0", 2023-09-23T20:22:43.746 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_recovery_max_active_hdd": "3", 2023-09-23T20:22:43.746 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_recovery_max_active_ssd": "10", 2023-09-23T20:22:43.747 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_recovery_max_chunk": "8388608", 2023-09-23T20:22:43.747 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_recovery_max_omap_entries_per_chunk": "8096", 2023-09-23T20:22:43.747 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_recovery_max_single_start": "1", 2023-09-23T20:22:43.747 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_recovery_op_priority": "3", 2023-09-23T20:22:43.747 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_recovery_op_warn_multiple": "16", 2023-09-23T20:22:43.747 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_recovery_priority": "5", 2023-09-23T20:22:43.747 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_recovery_retry_interval": "30.000000", 2023-09-23T20:22:43.747 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_recovery_sleep": "0.000000", 2023-09-23T20:22:43.747 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_recovery_sleep_hdd": "0.100000", 2023-09-23T20:22:43.747 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_recovery_sleep_hybrid": "0.025000", 2023-09-23T20:22:43.747 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_recovery_sleep_ssd": "0.000000", 2023-09-23T20:22:43.748 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_repair_during_recovery": "false", 2023-09-23T20:22:43.748 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_requested_scrub_priority": "120", 2023-09-23T20:22:43.748 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_rocksdb_iterator_bounds_enabled": "true", 2023-09-23T20:22:43.748 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_rollback_to_cluster_snap": "", 2023-09-23T20:22:43.748 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_scrub_auto_repair": "false", 2023-09-23T20:22:43.748 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_scrub_auto_repair_num_errors": "5", 2023-09-23T20:22:43.748 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_scrub_backoff_ratio": "0.660000", 2023-09-23T20:22:43.748 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_scrub_begin_hour": "0", 2023-09-23T20:22:43.748 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_scrub_begin_week_day": "0", 2023-09-23T20:22:43.748 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_scrub_chunk_max": "25", 2023-09-23T20:22:43.749 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_scrub_chunk_min": "5", 2023-09-23T20:22:43.749 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_scrub_cost": "52428800", 2023-09-23T20:22:43.749 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_scrub_during_recovery": "false", 2023-09-23T20:22:43.749 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_scrub_end_hour": "0", 2023-09-23T20:22:43.749 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_scrub_end_week_day": "0", 2023-09-23T20:22:43.749 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_scrub_event_cost": "4096", 2023-09-23T20:22:43.749 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_scrub_extended_sleep": "0.000000", 2023-09-23T20:22:43.749 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_scrub_interval_randomize_ratio": "0.500000", 2023-09-23T20:22:43.749 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_scrub_invalid_stats": "true", 2023-09-23T20:22:43.749 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_scrub_load_threshold": "0.500000", 2023-09-23T20:22:43.749 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_scrub_max_interval": "604800.000000", 2023-09-23T20:22:43.749 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_scrub_max_preemptions": "5", 2023-09-23T20:22:43.750 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_scrub_min_interval": "86400.000000", 2023-09-23T20:22:43.750 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_scrub_priority": "5", 2023-09-23T20:22:43.750 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_scrub_reservation_timeout": "5000", 2023-09-23T20:22:43.750 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_scrub_sleep": "0.000000", 2023-09-23T20:22:43.750 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_scrub_slow_reservation_response": "2200", 2023-09-23T20:22:43.750 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_shallow_scrub_chunk_max": "100", 2023-09-23T20:22:43.750 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_shallow_scrub_chunk_min": "50", 2023-09-23T20:22:43.750 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_shutdown_pgref_assert": "false", 2023-09-23T20:22:43.750 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_skip_data_digest": "false", 2023-09-23T20:22:43.750 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_smart_report_timeout": "5", 2023-09-23T20:22:43.750 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_snap_trim_cost": "1048576", 2023-09-23T20:22:43.751 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_snap_trim_priority": "5", 2023-09-23T20:22:43.751 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_snap_trim_sleep": "0.000000", 2023-09-23T20:22:43.751 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_snap_trim_sleep_hdd": "5.000000", 2023-09-23T20:22:43.751 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_snap_trim_sleep_hybrid": "2.000000", 2023-09-23T20:22:43.751 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_snap_trim_sleep_ssd": "0.000000", 2023-09-23T20:22:43.751 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_stats_update_period_not_scrubbing": "120", 2023-09-23T20:22:43.751 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_stats_update_period_scrubbing": "15", 2023-09-23T20:22:43.751 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_target_pg_log_entries_per_osd": "300000", 2023-09-23T20:22:43.751 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_target_transaction_size": "30", 2023-09-23T20:22:43.751 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_tier_default_cache_hit_set_count": "4", 2023-09-23T20:22:43.751 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_tier_default_cache_hit_set_grade_decay_rate": "20", 2023-09-23T20:22:43.751 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_tier_default_cache_hit_set_period": "1200", 2023-09-23T20:22:43.752 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_tier_default_cache_hit_set_search_last_n": "1", 2023-09-23T20:22:43.752 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_tier_default_cache_hit_set_type": "bloom", 2023-09-23T20:22:43.752 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_tier_default_cache_min_read_recency_for_promote": "1", 2023-09-23T20:22:43.752 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_tier_default_cache_min_write_recency_for_promote": "1", 2023-09-23T20:22:43.752 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_tier_default_cache_mode": "writeback", 2023-09-23T20:22:43.752 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_tier_promote_max_bytes_sec": "5242880", 2023-09-23T20:22:43.752 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_tier_promote_max_objects_sec": "25", 2023-09-23T20:22:43.752 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_tracing": "false", 2023-09-23T20:22:43.752 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_use_stale_snap": "false", 2023-09-23T20:22:43.752 INFO:teuthology.orchestra.run.smithi018.stdout: "osd_uuid": "00000000-0000-0000-0000-000000000000", 2023-09-23T20:22:43.752 INFO:teuthology.orchestra.run.smithi018.stdout: "osdc_blkin_trace_all": "false", 2023-09-23T20:22:43.753 INFO:teuthology.orchestra.run.smithi018.stdout: "paxos_kill_at": "0", 2023-09-23T20:22:43.753 INFO:teuthology.orchestra.run.smithi018.stdout: "paxos_max_join_drift": "10", 2023-09-23T20:22:43.753 INFO:teuthology.orchestra.run.smithi018.stdout: "paxos_min": "500", 2023-09-23T20:22:43.753 INFO:teuthology.orchestra.run.smithi018.stdout: "paxos_min_wait": "0.050000", 2023-09-23T20:22:43.753 INFO:teuthology.orchestra.run.smithi018.stdout: "paxos_propose_interval": "1.000000", 2023-09-23T20:22:43.753 INFO:teuthology.orchestra.run.smithi018.stdout: "paxos_service_trim_max": "500", 2023-09-23T20:22:43.753 INFO:teuthology.orchestra.run.smithi018.stdout: "paxos_service_trim_max_multiplier": "20", 2023-09-23T20:22:43.753 INFO:teuthology.orchestra.run.smithi018.stdout: "paxos_service_trim_min": "250", 2023-09-23T20:22:43.753 INFO:teuthology.orchestra.run.smithi018.stdout: "paxos_stash_full_interval": "25", 2023-09-23T20:22:43.753 INFO:teuthology.orchestra.run.smithi018.stdout: "paxos_trim_max": "500", 2023-09-23T20:22:43.753 INFO:teuthology.orchestra.run.smithi018.stdout: "paxos_trim_min": "250", 2023-09-23T20:22:43.753 INFO:teuthology.orchestra.run.smithi018.stdout: "perf": "true", 2023-09-23T20:22:43.754 INFO:teuthology.orchestra.run.smithi018.stdout: "pid_file": "", 2023-09-23T20:22:43.754 INFO:teuthology.orchestra.run.smithi018.stdout: "plugin_crypto_accelerator": "crypto_isal", 2023-09-23T20:22:43.754 INFO:teuthology.orchestra.run.smithi018.stdout: "plugin_dir": "/usr/lib64/ceph", 2023-09-23T20:22:43.754 INFO:teuthology.orchestra.run.smithi018.stdout: "public_addr": "-", 2023-09-23T20:22:43.754 INFO:teuthology.orchestra.run.smithi018.stdout: "public_addrv": "", 2023-09-23T20:22:43.754 INFO:teuthology.orchestra.run.smithi018.stdout: "public_bind_addr": "-", 2023-09-23T20:22:43.754 INFO:teuthology.orchestra.run.smithi018.stdout: "public_network": "172.21.0.0/20", 2023-09-23T20:22:43.754 INFO:teuthology.orchestra.run.smithi018.stdout: "public_network_interface": "", 2023-09-23T20:22:43.754 INFO:teuthology.orchestra.run.smithi018.stdout: "qat_compressor_enabled": "false", 2023-09-23T20:22:43.754 INFO:teuthology.orchestra.run.smithi018.stdout: "qat_compressor_session_max_number": "256", 2023-09-23T20:22:43.755 INFO:teuthology.orchestra.run.smithi018.stdout: "rados_mon_op_timeout": "0", 2023-09-23T20:22:43.755 INFO:teuthology.orchestra.run.smithi018.stdout: "rados_osd_op_timeout": "0", 2023-09-23T20:22:43.755 INFO:teuthology.orchestra.run.smithi018.stdout: "rados_tracing": "false", 2023-09-23T20:22:43.755 INFO:teuthology.orchestra.run.smithi018.stdout: "rbd_atime_update_interval": "60", 2023-09-23T20:22:43.755 INFO:teuthology.orchestra.run.smithi018.stdout: "rbd_auto_exclusive_lock_until_manual_request": "true", 2023-09-23T20:22:43.755 INFO:teuthology.orchestra.run.smithi018.stdout: "rbd_balance_parent_reads": "false", 2023-09-23T20:22:43.755 INFO:teuthology.orchestra.run.smithi018.stdout: "rbd_balance_snap_reads": "false", 2023-09-23T20:22:43.755 INFO:teuthology.orchestra.run.smithi018.stdout: "rbd_blkin_trace_all": "false", 2023-09-23T20:22:43.755 INFO:teuthology.orchestra.run.smithi018.stdout: "rbd_blocklist_expire_seconds": "0", 2023-09-23T20:22:43.755 INFO:teuthology.orchestra.run.smithi018.stdout: "rbd_blocklist_on_break_lock": "true", 2023-09-23T20:22:43.755 INFO:teuthology.orchestra.run.smithi018.stdout: "rbd_cache": "true", 2023-09-23T20:22:43.755 INFO:teuthology.orchestra.run.smithi018.stdout: "rbd_cache_block_writes_upfront": "false", 2023-09-23T20:22:43.756 INFO:teuthology.orchestra.run.smithi018.stdout: "rbd_cache_max_dirty": "25165824", 2023-09-23T20:22:43.756 INFO:teuthology.orchestra.run.smithi018.stdout: "rbd_cache_max_dirty_age": "1.000000", 2023-09-23T20:22:43.756 INFO:teuthology.orchestra.run.smithi018.stdout: "rbd_cache_max_dirty_object": "0", 2023-09-23T20:22:43.756 INFO:teuthology.orchestra.run.smithi018.stdout: "rbd_cache_policy": "writearound", 2023-09-23T20:22:43.756 INFO:teuthology.orchestra.run.smithi018.stdout: "rbd_cache_size": "33554432", 2023-09-23T20:22:43.756 INFO:teuthology.orchestra.run.smithi018.stdout: "rbd_cache_target_dirty": "16777216", 2023-09-23T20:22:43.756 INFO:teuthology.orchestra.run.smithi018.stdout: "rbd_cache_writethrough_until_flush": "true", 2023-09-23T20:22:43.756 INFO:teuthology.orchestra.run.smithi018.stdout: "rbd_clone_copy_on_read": "false", 2023-09-23T20:22:43.756 INFO:teuthology.orchestra.run.smithi018.stdout: "rbd_compression_hint": "none", 2023-09-23T20:22:43.756 INFO:teuthology.orchestra.run.smithi018.stdout: "rbd_concurrent_management_ops": "10", 2023-09-23T20:22:43.756 INFO:teuthology.orchestra.run.smithi018.stdout: "rbd_config_pool_override_update_timestamp": "0", 2023-09-23T20:22:43.757 INFO:teuthology.orchestra.run.smithi018.stdout: "rbd_default_clone_format": "auto", 2023-09-23T20:22:43.757 INFO:teuthology.orchestra.run.smithi018.stdout: "rbd_default_data_pool": "", 2023-09-23T20:22:43.757 INFO:teuthology.orchestra.run.smithi018.stdout: "rbd_default_features": "61", 2023-09-23T20:22:43.757 INFO:teuthology.orchestra.run.smithi018.stdout: "rbd_default_format": "2", 2023-09-23T20:22:43.757 INFO:teuthology.orchestra.run.smithi018.stdout: "rbd_default_map_options": "", 2023-09-23T20:22:43.757 INFO:teuthology.orchestra.run.smithi018.stdout: "rbd_default_order": "22", 2023-09-23T20:22:43.757 INFO:teuthology.orchestra.run.smithi018.stdout: "rbd_default_pool": "rbd", 2023-09-23T20:22:43.757 INFO:teuthology.orchestra.run.smithi018.stdout: "rbd_default_snapshot_quiesce_mode": "required", 2023-09-23T20:22:43.757 INFO:teuthology.orchestra.run.smithi018.stdout: "rbd_default_stripe_count": "0", 2023-09-23T20:22:43.757 INFO:teuthology.orchestra.run.smithi018.stdout: "rbd_default_stripe_unit": "0", 2023-09-23T20:22:43.757 INFO:teuthology.orchestra.run.smithi018.stdout: "rbd_disable_zero_copy_writes": "true", 2023-09-23T20:22:43.757 INFO:teuthology.orchestra.run.smithi018.stdout: "rbd_discard_granularity_bytes": "65536", 2023-09-23T20:22:43.758 INFO:teuthology.orchestra.run.smithi018.stdout: "rbd_discard_on_zeroed_write_same": "true", 2023-09-23T20:22:43.758 INFO:teuthology.orchestra.run.smithi018.stdout: "rbd_enable_alloc_hint": "true", 2023-09-23T20:22:43.758 INFO:teuthology.orchestra.run.smithi018.stdout: "rbd_invalidate_object_map_on_timeout": "true", 2023-09-23T20:22:43.758 INFO:teuthology.orchestra.run.smithi018.stdout: "rbd_io_scheduler": "simple", 2023-09-23T20:22:43.758 INFO:teuthology.orchestra.run.smithi018.stdout: "rbd_io_scheduler_simple_max_delay": "0", 2023-09-23T20:22:43.758 INFO:teuthology.orchestra.run.smithi018.stdout: "rbd_journal_commit_age": "5.000000", 2023-09-23T20:22:43.758 INFO:teuthology.orchestra.run.smithi018.stdout: "rbd_journal_max_concurrent_object_sets": "0", 2023-09-23T20:22:43.758 INFO:teuthology.orchestra.run.smithi018.stdout: "rbd_journal_max_payload_bytes": "16384", 2023-09-23T20:22:43.758 INFO:teuthology.orchestra.run.smithi018.stdout: "rbd_journal_object_flush_age": "0.000000", 2023-09-23T20:22:43.758 INFO:teuthology.orchestra.run.smithi018.stdout: "rbd_journal_object_flush_bytes": "1048576", 2023-09-23T20:22:43.758 INFO:teuthology.orchestra.run.smithi018.stdout: "rbd_journal_object_flush_interval": "0", 2023-09-23T20:22:43.759 INFO:teuthology.orchestra.run.smithi018.stdout: "rbd_journal_object_max_in_flight_appends": "0", 2023-09-23T20:22:43.759 INFO:teuthology.orchestra.run.smithi018.stdout: "rbd_journal_object_writethrough_until_flush": "true", 2023-09-23T20:22:43.759 INFO:teuthology.orchestra.run.smithi018.stdout: "rbd_journal_order": "24", 2023-09-23T20:22:43.759 INFO:teuthology.orchestra.run.smithi018.stdout: "rbd_journal_pool": "", 2023-09-23T20:22:43.759 INFO:teuthology.orchestra.run.smithi018.stdout: "rbd_journal_splay_width": "4", 2023-09-23T20:22:43.759 INFO:teuthology.orchestra.run.smithi018.stdout: "rbd_localize_parent_reads": "false", 2023-09-23T20:22:43.759 INFO:teuthology.orchestra.run.smithi018.stdout: "rbd_localize_snap_reads": "false", 2023-09-23T20:22:43.759 INFO:teuthology.orchestra.run.smithi018.stdout: "rbd_mirror_concurrent_image_deletions": "1", 2023-09-23T20:22:43.759 INFO:teuthology.orchestra.run.smithi018.stdout: "rbd_mirror_concurrent_image_syncs": "5", 2023-09-23T20:22:43.759 INFO:teuthology.orchestra.run.smithi018.stdout: "rbd_mirror_delete_retry_interval": "30.000000", 2023-09-23T20:22:43.759 INFO:teuthology.orchestra.run.smithi018.stdout: "rbd_mirror_image_perf_stats_prio": "5", 2023-09-23T20:22:43.759 INFO:teuthology.orchestra.run.smithi018.stdout: "rbd_mirror_image_policy_migration_throttle": "300", 2023-09-23T20:22:43.760 INFO:teuthology.orchestra.run.smithi018.stdout: "rbd_mirror_image_policy_rebalance_timeout": "0.000000", 2023-09-23T20:22:43.760 INFO:teuthology.orchestra.run.smithi018.stdout: "rbd_mirror_image_policy_type": "simple", 2023-09-23T20:22:43.760 INFO:teuthology.orchestra.run.smithi018.stdout: "rbd_mirror_image_policy_update_throttle_interval": "1.000000", 2023-09-23T20:22:43.760 INFO:teuthology.orchestra.run.smithi018.stdout: "rbd_mirror_image_state_check_interval": "30", 2023-09-23T20:22:43.760 INFO:teuthology.orchestra.run.smithi018.stdout: "rbd_mirror_journal_commit_age": "5.000000", 2023-09-23T20:22:43.760 INFO:teuthology.orchestra.run.smithi018.stdout: "rbd_mirror_journal_poll_age": "5.000000", 2023-09-23T20:22:43.760 INFO:teuthology.orchestra.run.smithi018.stdout: "rbd_mirror_leader_heartbeat_interval": "5", 2023-09-23T20:22:43.760 INFO:teuthology.orchestra.run.smithi018.stdout: "rbd_mirror_leader_max_acquire_attempts_before_break": "3", 2023-09-23T20:22:43.760 INFO:teuthology.orchestra.run.smithi018.stdout: "rbd_mirror_leader_max_missed_heartbeats": "2", 2023-09-23T20:22:43.760 INFO:teuthology.orchestra.run.smithi018.stdout: "rbd_mirror_memory_autotune": "true", 2023-09-23T20:22:43.760 INFO:teuthology.orchestra.run.smithi018.stdout: "rbd_mirror_memory_base": "805306368", 2023-09-23T20:22:43.760 INFO:teuthology.orchestra.run.smithi018.stdout: "rbd_mirror_memory_cache_autotune_interval": "30.000000", 2023-09-23T20:22:43.761 INFO:teuthology.orchestra.run.smithi018.stdout: "rbd_mirror_memory_cache_min": "134217728", 2023-09-23T20:22:43.761 INFO:teuthology.orchestra.run.smithi018.stdout: "rbd_mirror_memory_cache_resize_interval": "5.000000", 2023-09-23T20:22:43.761 INFO:teuthology.orchestra.run.smithi018.stdout: "rbd_mirror_memory_expected_fragmentation": "0.150000", 2023-09-23T20:22:43.761 INFO:teuthology.orchestra.run.smithi018.stdout: "rbd_mirror_memory_target": "4294967296", 2023-09-23T20:22:43.761 INFO:teuthology.orchestra.run.smithi018.stdout: "rbd_mirror_perf_stats_prio": "5", 2023-09-23T20:22:43.761 INFO:teuthology.orchestra.run.smithi018.stdout: "rbd_mirror_pool_replayers_refresh_interval": "30", 2023-09-23T20:22:43.761 INFO:teuthology.orchestra.run.smithi018.stdout: "rbd_mirror_sync_point_update_age": "30.000000", 2023-09-23T20:22:43.761 INFO:teuthology.orchestra.run.smithi018.stdout: "rbd_mirroring_delete_delay": "0", 2023-09-23T20:22:43.761 INFO:teuthology.orchestra.run.smithi018.stdout: "rbd_mirroring_max_mirroring_snapshots": "5", 2023-09-23T20:22:43.761 INFO:teuthology.orchestra.run.smithi018.stdout: "rbd_mirroring_replay_delay": "0", 2023-09-23T20:22:43.761 INFO:teuthology.orchestra.run.smithi018.stdout: "rbd_mirroring_resync_after_disconnect": "false", 2023-09-23T20:22:43.762 INFO:teuthology.orchestra.run.smithi018.stdout: "rbd_move_parent_to_trash_on_remove": "false", 2023-09-23T20:22:43.762 INFO:teuthology.orchestra.run.smithi018.stdout: "rbd_move_to_trash_on_remove": "false", 2023-09-23T20:22:43.762 INFO:teuthology.orchestra.run.smithi018.stdout: "rbd_move_to_trash_on_remove_expire_seconds": "0", 2023-09-23T20:22:43.762 INFO:teuthology.orchestra.run.smithi018.stdout: "rbd_mtime_update_interval": "60", 2023-09-23T20:22:43.762 INFO:teuthology.orchestra.run.smithi018.stdout: "rbd_non_blocking_aio": "true", 2023-09-23T20:22:43.762 INFO:teuthology.orchestra.run.smithi018.stdout: "rbd_op_thread_timeout": "60", 2023-09-23T20:22:43.762 INFO:teuthology.orchestra.run.smithi018.stdout: "rbd_op_threads": "1", 2023-09-23T20:22:43.762 INFO:teuthology.orchestra.run.smithi018.stdout: "rbd_parent_cache_enabled": "false", 2023-09-23T20:22:43.762 INFO:teuthology.orchestra.run.smithi018.stdout: "rbd_persistent_cache_mode": "disabled", 2023-09-23T20:22:43.762 INFO:teuthology.orchestra.run.smithi018.stdout: "rbd_persistent_cache_path": "/tmp", 2023-09-23T20:22:43.762 INFO:teuthology.orchestra.run.smithi018.stdout: "rbd_persistent_cache_size": "1073741824", 2023-09-23T20:22:43.762 INFO:teuthology.orchestra.run.smithi018.stdout: "rbd_plugins": "", 2023-09-23T20:22:43.763 INFO:teuthology.orchestra.run.smithi018.stdout: "rbd_qos_bps_burst": "0", 2023-09-23T20:22:43.763 INFO:teuthology.orchestra.run.smithi018.stdout: "rbd_qos_bps_burst_seconds": "1", 2023-09-23T20:22:43.763 INFO:teuthology.orchestra.run.smithi018.stdout: "rbd_qos_bps_limit": "0", 2023-09-23T20:22:43.763 INFO:teuthology.orchestra.run.smithi018.stdout: "rbd_qos_exclude_ops": "0", 2023-09-23T20:22:43.763 INFO:teuthology.orchestra.run.smithi018.stdout: "rbd_qos_iops_burst": "0", 2023-09-23T20:22:43.763 INFO:teuthology.orchestra.run.smithi018.stdout: "rbd_qos_iops_burst_seconds": "1", 2023-09-23T20:22:43.763 INFO:teuthology.orchestra.run.smithi018.stdout: "rbd_qos_iops_limit": "0", 2023-09-23T20:22:43.763 INFO:teuthology.orchestra.run.smithi018.stdout: "rbd_qos_read_bps_burst": "0", 2023-09-23T20:22:43.763 INFO:teuthology.orchestra.run.smithi018.stdout: "rbd_qos_read_bps_burst_seconds": "1", 2023-09-23T20:22:43.763 INFO:teuthology.orchestra.run.smithi018.stdout: "rbd_qos_read_bps_limit": "0", 2023-09-23T20:22:43.763 INFO:teuthology.orchestra.run.smithi018.stdout: "rbd_qos_read_iops_burst": "0", 2023-09-23T20:22:43.764 INFO:teuthology.orchestra.run.smithi018.stdout: "rbd_qos_read_iops_burst_seconds": "1", 2023-09-23T20:22:43.764 INFO:teuthology.orchestra.run.smithi018.stdout: "rbd_qos_read_iops_limit": "0", 2023-09-23T20:22:43.764 INFO:teuthology.orchestra.run.smithi018.stdout: "rbd_qos_schedule_tick_min": "50", 2023-09-23T20:22:43.764 INFO:teuthology.orchestra.run.smithi018.stdout: "rbd_qos_write_bps_burst": "0", 2023-09-23T20:22:43.764 INFO:teuthology.orchestra.run.smithi018.stdout: "rbd_qos_write_bps_burst_seconds": "1", 2023-09-23T20:22:43.764 INFO:teuthology.orchestra.run.smithi018.stdout: "rbd_qos_write_bps_limit": "0", 2023-09-23T20:22:43.764 INFO:teuthology.orchestra.run.smithi018.stdout: "rbd_qos_write_iops_burst": "0", 2023-09-23T20:22:43.764 INFO:teuthology.orchestra.run.smithi018.stdout: "rbd_qos_write_iops_burst_seconds": "1", 2023-09-23T20:22:43.764 INFO:teuthology.orchestra.run.smithi018.stdout: "rbd_qos_write_iops_limit": "0", 2023-09-23T20:22:43.764 INFO:teuthology.orchestra.run.smithi018.stdout: "rbd_quiesce_notification_attempts": "10", 2023-09-23T20:22:43.765 INFO:teuthology.orchestra.run.smithi018.stdout: "rbd_read_from_replica_policy": "default", 2023-09-23T20:22:43.765 INFO:teuthology.orchestra.run.smithi018.stdout: "rbd_readahead_disable_after_bytes": "52428800", 2023-09-23T20:22:43.765 INFO:teuthology.orchestra.run.smithi018.stdout: "rbd_readahead_max_bytes": "524288", 2023-09-23T20:22:43.765 INFO:teuthology.orchestra.run.smithi018.stdout: "rbd_readahead_trigger_requests": "10", 2023-09-23T20:22:43.765 INFO:teuthology.orchestra.run.smithi018.stdout: "rbd_request_timed_out_seconds": "30", 2023-09-23T20:22:43.765 INFO:teuthology.orchestra.run.smithi018.stdout: "rbd_skip_partial_discard": "true", 2023-09-23T20:22:43.765 INFO:teuthology.orchestra.run.smithi018.stdout: "rbd_sparse_read_threshold_bytes": "65536", 2023-09-23T20:22:43.765 INFO:teuthology.orchestra.run.smithi018.stdout: "rbd_tracing": "false", 2023-09-23T20:22:43.765 INFO:teuthology.orchestra.run.smithi018.stdout: "rbd_validate_names": "true", 2023-09-23T20:22:43.765 INFO:teuthology.orchestra.run.smithi018.stdout: "rbd_validate_pool": "true", 2023-09-23T20:22:43.765 INFO:teuthology.orchestra.run.smithi018.stdout: "restapi_base_url": "", 2023-09-23T20:22:43.765 INFO:teuthology.orchestra.run.smithi018.stdout: "restapi_log_level": "", 2023-09-23T20:22:43.766 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_acl_grants_max_num": "100", 2023-09-23T20:22:43.766 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_admin_entry": "admin", 2023-09-23T20:22:43.766 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_allow_notification_secrets_in_cleartext": "false", 2023-09-23T20:22:43.766 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_backend_store": "rados", 2023-09-23T20:22:43.766 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_barbican_url": "", 2023-09-23T20:22:43.766 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_beast_enable_async": "true", 2023-09-23T20:22:43.766 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_bucket_default_quota_max_objects": "-1", 2023-09-23T20:22:43.766 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_bucket_default_quota_max_size": "-1", 2023-09-23T20:22:43.766 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_bucket_index_max_aio": "128", 2023-09-23T20:22:43.766 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_bucket_index_transaction_instrumentation": "false", 2023-09-23T20:22:43.766 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_bucket_quota_cache_size": "10000", 2023-09-23T20:22:43.767 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_bucket_quota_ttl": "600", 2023-09-23T20:22:43.767 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_bucket_sync_spawn_window": "20", 2023-09-23T20:22:43.767 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_cache_enabled": "true", 2023-09-23T20:22:43.767 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_cache_expiry_interval": "900", 2023-09-23T20:22:43.767 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_cache_lru_size": "10000", 2023-09-23T20:22:43.767 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_config_store": "rados", 2023-09-23T20:22:43.767 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_content_length_compat": "false", 2023-09-23T20:22:43.767 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_copy_obj_progress": "true", 2023-09-23T20:22:43.767 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_copy_obj_progress_every_bytes": "1048576", 2023-09-23T20:22:43.767 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_cors_rules_max_num": "100", 2023-09-23T20:22:43.767 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_cross_domain_policy": "", 2023-09-23T20:22:43.767 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_crypt_default_encryption_key": "", 2023-09-23T20:22:43.768 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_crypt_kmip_addr": "", 2023-09-23T20:22:43.768 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_crypt_kmip_ca_path": "", 2023-09-23T20:22:43.768 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_crypt_kmip_client_cert": "", 2023-09-23T20:22:43.768 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_crypt_kmip_client_key": "", 2023-09-23T20:22:43.768 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_crypt_kmip_kms_key_template": "", 2023-09-23T20:22:43.768 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_crypt_kmip_password": "", 2023-09-23T20:22:43.768 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_crypt_kmip_s3_key_template": "$keyid", 2023-09-23T20:22:43.768 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_crypt_kmip_username": "", 2023-09-23T20:22:43.768 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_crypt_require_ssl": "true", 2023-09-23T20:22:43.768 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_crypt_s3_kms_backend": "barbican", 2023-09-23T20:22:43.768 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_crypt_s3_kms_encryption_keys": "", 2023-09-23T20:22:43.768 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_crypt_sse_s3_backend": "vault", 2023-09-23T20:22:43.769 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_crypt_sse_s3_key_template": "%bucket_id", 2023-09-23T20:22:43.769 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_crypt_sse_s3_vault_addr": "", 2023-09-23T20:22:43.769 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_crypt_sse_s3_vault_auth": "token", 2023-09-23T20:22:43.769 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_crypt_sse_s3_vault_namespace": "", 2023-09-23T20:22:43.769 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_crypt_sse_s3_vault_prefix": "", 2023-09-23T20:22:43.769 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_crypt_sse_s3_vault_secret_engine": "transit", 2023-09-23T20:22:43.769 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_crypt_sse_s3_vault_ssl_cacert": "", 2023-09-23T20:22:43.769 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_crypt_sse_s3_vault_ssl_clientcert": "", 2023-09-23T20:22:43.769 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_crypt_sse_s3_vault_ssl_clientkey": "", 2023-09-23T20:22:43.769 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_crypt_sse_s3_vault_token_file": "", 2023-09-23T20:22:43.769 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_crypt_sse_s3_vault_verify_ssl": "true", 2023-09-23T20:22:43.769 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_crypt_suppress_logs": "true", 2023-09-23T20:22:43.770 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_crypt_vault_addr": "", 2023-09-23T20:22:43.770 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_crypt_vault_auth": "token", 2023-09-23T20:22:43.770 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_crypt_vault_namespace": "", 2023-09-23T20:22:43.770 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_crypt_vault_prefix": "", 2023-09-23T20:22:43.770 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_crypt_vault_secret_engine": "transit", 2023-09-23T20:22:43.770 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_crypt_vault_ssl_cacert": "", 2023-09-23T20:22:43.770 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_crypt_vault_ssl_clientcert": "", 2023-09-23T20:22:43.770 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_crypt_vault_ssl_clientkey": "", 2023-09-23T20:22:43.770 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_crypt_vault_token_file": "", 2023-09-23T20:22:43.770 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_crypt_vault_verify_ssl": "true", 2023-09-23T20:22:43.770 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_curl_buffersize": "524288", 2023-09-23T20:22:43.771 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_curl_low_speed_limit": "1024", 2023-09-23T20:22:43.771 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_curl_low_speed_time": "300", 2023-09-23T20:22:43.771 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_curl_tcp_keepalive": "0", 2023-09-23T20:22:43.771 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_curl_wait_timeout_ms": "1000", 2023-09-23T20:22:43.771 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_d3n_l1_datacache_persistent_path": "/tmp/rgw_datacache/", 2023-09-23T20:22:43.771 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_d3n_l1_datacache_size": "1073741824", 2023-09-23T20:22:43.771 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_d3n_l1_evict_cache_on_start": "true", 2023-09-23T20:22:43.771 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_d3n_l1_eviction_policy": "lru", 2023-09-23T20:22:43.771 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_d3n_l1_fadvise": "4", 2023-09-23T20:22:43.771 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_d3n_l1_local_datacache_enabled": "false", 2023-09-23T20:22:43.771 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_d3n_libaio_aio_num": "64", 2023-09-23T20:22:43.771 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_d3n_libaio_aio_threads": "20", 2023-09-23T20:22:43.772 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_data": "/var/lib/ceph/radosgw/ceph-a", 2023-09-23T20:22:43.772 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_data_log_changes_size": "1000", 2023-09-23T20:22:43.772 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_data_log_num_shards": "128", 2023-09-23T20:22:43.772 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_data_log_obj_prefix": "data_log", 2023-09-23T20:22:43.772 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_data_log_window": "30", 2023-09-23T20:22:43.772 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_data_notify_interval_msec": "0", 2023-09-23T20:22:43.772 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_data_sync_poll_interval": "20", 2023-09-23T20:22:43.772 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_data_sync_spawn_window": "20", 2023-09-23T20:22:43.772 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_debug_inject_olh_cancel_modification_err": "false", 2023-09-23T20:22:43.772 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_debug_inject_set_olh_err": "0", 2023-09-23T20:22:43.772 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_default_data_log_backing": "fifo", 2023-09-23T20:22:43.773 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_default_realm_info_oid": "default.realm", 2023-09-23T20:22:43.773 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_default_region_info_oid": "default.region", 2023-09-23T20:22:43.773 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_default_zone_info_oid": "default.zone", 2023-09-23T20:22:43.773 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_default_zonegroup_info_oid": "default.zonegroup", 2023-09-23T20:22:43.773 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_defer_to_bucket_acls": "", 2023-09-23T20:22:43.773 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_delete_multi_obj_max_num": "1000", 2023-09-23T20:22:43.773 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_dmclock_admin_lim": "0.000000", 2023-09-23T20:22:43.773 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_dmclock_admin_res": "100.000000", 2023-09-23T20:22:43.773 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_dmclock_admin_wgt": "100.000000", 2023-09-23T20:22:43.773 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_dmclock_auth_lim": "0.000000", 2023-09-23T20:22:43.773 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_dmclock_auth_res": "200.000000", 2023-09-23T20:22:43.773 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_dmclock_auth_wgt": "100.000000", 2023-09-23T20:22:43.774 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_dmclock_data_lim": "0.000000", 2023-09-23T20:22:43.774 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_dmclock_data_res": "500.000000", 2023-09-23T20:22:43.774 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_dmclock_data_wgt": "500.000000", 2023-09-23T20:22:43.774 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_dmclock_metadata_lim": "0.000000", 2023-09-23T20:22:43.774 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_dmclock_metadata_res": "500.000000", 2023-09-23T20:22:43.774 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_dmclock_metadata_wgt": "500.000000", 2023-09-23T20:22:43.774 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_dns_name": "", 2023-09-23T20:22:43.774 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_dns_s3website_name": "", 2023-09-23T20:22:43.774 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_dynamic_resharding": "true", 2023-09-23T20:22:43.774 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_enable_apis": "s3, s3website, swift, swift_auth, admin, sts, iam, notifications", 2023-09-23T20:22:43.774 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_enable_gc_threads": "true", 2023-09-23T20:22:43.774 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_enable_lc_threads": "true", 2023-09-23T20:22:43.775 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_enable_ops_log": "false", 2023-09-23T20:22:43.775 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_enable_quota_threads": "true", 2023-09-23T20:22:43.775 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_enable_static_website": "false", 2023-09-23T20:22:43.775 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_enable_usage_log": "false", 2023-09-23T20:22:43.775 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_enforce_swift_acls": "true", 2023-09-23T20:22:43.775 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_exit_timeout_secs": "120", 2023-09-23T20:22:43.775 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_expose_bucket": "false", 2023-09-23T20:22:43.775 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_extended_http_attrs": "", 2023-09-23T20:22:43.775 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_filter": "none", 2023-09-23T20:22:43.775 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_frontend_defaults": "beast ssl_certificate=config://rgw/cert/$realm/$zone.crt ssl_private_key=config://rgw/cert/$realm/$zone.key", 2023-09-23T20:22:43.775 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_frontends": "beast port=7480", 2023-09-23T20:22:43.776 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_gc_max_concurrent_io": "10", 2023-09-23T20:22:43.776 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_gc_max_deferred": "50", 2023-09-23T20:22:43.776 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_gc_max_deferred_entries_size": "3072", 2023-09-23T20:22:43.776 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_gc_max_objs": "32", 2023-09-23T20:22:43.776 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_gc_max_queue_size": "134213632", 2023-09-23T20:22:43.776 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_gc_max_trim_chunk": "16", 2023-09-23T20:22:43.776 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_gc_obj_min_wait": "7200", 2023-09-23T20:22:43.776 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_gc_processor_max_time": "3600", 2023-09-23T20:22:43.776 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_gc_processor_period": "3600", 2023-09-23T20:22:43.776 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_get_obj_max_req_size": "4194304", 2023-09-23T20:22:43.776 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_get_obj_window_size": "16777216", 2023-09-23T20:22:43.776 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_healthcheck_disabling_path": "", 2023-09-23T20:22:43.777 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_ignore_get_invalid_range": "false", 2023-09-23T20:22:43.777 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_init_timeout": "300", 2023-09-23T20:22:43.777 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_inject_notify_timeout_probability": "0.000000", 2023-09-23T20:22:43.777 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_json_config": "/var/lib/ceph/radosgw/config.json", 2023-09-23T20:22:43.777 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_keystone_accepted_admin_roles": "", 2023-09-23T20:22:43.777 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_keystone_accepted_roles": "Member, admin", 2023-09-23T20:22:43.777 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_keystone_admin_domain": "", 2023-09-23T20:22:43.777 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_keystone_admin_password": "", 2023-09-23T20:22:43.777 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_keystone_admin_password_path": "", 2023-09-23T20:22:43.777 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_keystone_admin_project": "", 2023-09-23T20:22:43.777 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_keystone_admin_tenant": "", 2023-09-23T20:22:43.777 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_keystone_admin_token": "", 2023-09-23T20:22:43.778 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_keystone_admin_token_path": "", 2023-09-23T20:22:43.778 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_keystone_admin_user": "", 2023-09-23T20:22:43.778 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_keystone_api_version": "2", 2023-09-23T20:22:43.778 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_keystone_barbican_domain": "", 2023-09-23T20:22:43.778 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_keystone_barbican_password": "", 2023-09-23T20:22:43.778 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_keystone_barbican_project": "", 2023-09-23T20:22:43.778 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_keystone_barbican_tenant": "", 2023-09-23T20:22:43.778 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_keystone_barbican_user": "", 2023-09-23T20:22:43.778 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_keystone_expired_token_cache_expiration": "3600", 2023-09-23T20:22:43.778 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_keystone_implicit_tenants": "false", 2023-09-23T20:22:43.778 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_keystone_service_token_accepted_roles": "admin", 2023-09-23T20:22:43.779 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_keystone_service_token_enabled": "false", 2023-09-23T20:22:43.779 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_keystone_token_cache_size": "10000", 2023-09-23T20:22:43.779 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_keystone_url": "", 2023-09-23T20:22:43.779 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_keystone_verify_ssl": "true", 2023-09-23T20:22:43.779 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_lc_debug_interval": "-1", 2023-09-23T20:22:43.779 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_lc_lock_max_time": "90", 2023-09-23T20:22:43.779 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_lc_max_objs": "32", 2023-09-23T20:22:43.779 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_lc_max_rules": "1000", 2023-09-23T20:22:43.779 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_lc_max_worker": "3", 2023-09-23T20:22:43.779 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_lc_max_wp_worker": "3", 2023-09-23T20:22:43.779 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_lc_thread_delay": "0", 2023-09-23T20:22:43.779 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_ldap_binddn": "uid=admin,cn=users,dc=example,dc=com", 2023-09-23T20:22:43.780 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_ldap_dnattr": "uid", 2023-09-23T20:22:43.780 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_ldap_searchdn": "cn=users,cn=accounts,dc=example,dc=com", 2023-09-23T20:22:43.780 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_ldap_searchfilter": "", 2023-09-23T20:22:43.780 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_ldap_secret": "/etc/openldap/secret", 2023-09-23T20:22:43.780 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_ldap_uri": "ldaps://", 2023-09-23T20:22:43.780 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_lifecycle_work_time": "00:00-06:00", 2023-09-23T20:22:43.780 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_list_bucket_min_readahead": "1000", 2023-09-23T20:22:43.780 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_list_buckets_max_chunk": "1000", 2023-09-23T20:22:43.780 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_log_http_headers": "", 2023-09-23T20:22:43.780 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_log_nonexistent_bucket": "false", 2023-09-23T20:22:43.780 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_log_object_name": "%Y-%m-%d-%H-%i-%n", 2023-09-23T20:22:43.780 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_log_object_name_utc": "false", 2023-09-23T20:22:43.781 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_luarocks_location": "/tmp/luarocks", 2023-09-23T20:22:43.781 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_max_attr_name_len": "0", 2023-09-23T20:22:43.781 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_max_attr_size": "0", 2023-09-23T20:22:43.781 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_max_attrs_num_in_req": "0", 2023-09-23T20:22:43.781 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_max_chunk_size": "4194304", 2023-09-23T20:22:43.781 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_max_concurrent_requests": "1024", 2023-09-23T20:22:43.781 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_max_copy_obj_concurrent_io": "10", 2023-09-23T20:22:43.781 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_max_dynamic_shards": "1999", 2023-09-23T20:22:43.781 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_max_listing_results": "1000", 2023-09-23T20:22:43.781 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_max_notify_retries": "10", 2023-09-23T20:22:43.781 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_max_objs_per_shard": "100000", 2023-09-23T20:22:43.782 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_max_put_param_size": "1048576", 2023-09-23T20:22:43.782 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_max_put_size": "5368709120", 2023-09-23T20:22:43.782 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_max_slo_entries": "1000", 2023-09-23T20:22:43.782 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_md_log_max_shards": "64", 2023-09-23T20:22:43.782 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_md_notify_interval_msec": "200", 2023-09-23T20:22:43.782 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_meta_sync_poll_interval": "20", 2023-09-23T20:22:43.782 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_meta_sync_spawn_window": "20", 2023-09-23T20:22:43.782 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_mime_types_file": "/etc/mime.types", 2023-09-23T20:22:43.782 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_mp_lock_max_time": "600", 2023-09-23T20:22:43.782 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_multi_obj_del_max_aio": "16", 2023-09-23T20:22:43.782 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_multipart_min_part_size": "5242880", 2023-09-23T20:22:43.782 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_multipart_part_upload_limit": "10000", 2023-09-23T20:22:43.783 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_nfs_fhcache_partitions": "3", 2023-09-23T20:22:43.783 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_nfs_fhcache_size": "2017", 2023-09-23T20:22:43.783 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_nfs_frontends": "rgw-nfs", 2023-09-23T20:22:43.783 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_nfs_lru_lane_hiwat": "911", 2023-09-23T20:22:43.783 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_nfs_lru_lanes": "5", 2023-09-23T20:22:43.783 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_nfs_max_gc": "300", 2023-09-23T20:22:43.783 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_nfs_namespace_expire_secs": "300", 2023-09-23T20:22:43.783 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_nfs_run_gc_threads": "false", 2023-09-23T20:22:43.783 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_nfs_run_lc_threads": "false", 2023-09-23T20:22:43.783 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_nfs_run_quota_threads": "false", 2023-09-23T20:22:43.783 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_nfs_run_sync_thread": "false", 2023-09-23T20:22:43.784 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_nfs_s3_fast_attrs": "false", 2023-09-23T20:22:43.784 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_nfs_write_completion_interval_s": "10", 2023-09-23T20:22:43.784 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_num_async_rados_threads": "32", 2023-09-23T20:22:43.784 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_num_control_oids": "8", 2023-09-23T20:22:43.784 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_numa_node": "-1", 2023-09-23T20:22:43.784 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_obj_stripe_size": "4194304", 2023-09-23T20:22:43.784 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_obj_tombstone_cache_size": "1000", 2023-09-23T20:22:43.784 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_objexp_chunk_size": "100", 2023-09-23T20:22:43.784 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_objexp_gc_interval": "600", 2023-09-23T20:22:43.784 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_objexp_hints_num_shards": "127", 2023-09-23T20:22:43.784 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_olh_pending_timeout_sec": "3600", 2023-09-23T20:22:43.784 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_op_thread_suicide_timeout": "0", 2023-09-23T20:22:43.785 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_op_thread_timeout": "600", 2023-09-23T20:22:43.785 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_op_tracing": "false", 2023-09-23T20:22:43.785 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_opa_token": "", 2023-09-23T20:22:43.785 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_opa_url": "", 2023-09-23T20:22:43.785 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_opa_verify_ssl": "true", 2023-09-23T20:22:43.785 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_ops_log_data_backlog": "5242880", 2023-09-23T20:22:43.785 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_ops_log_file_path": "/var/log/ceph/ops-log-ceph-mon.a.log", 2023-09-23T20:22:43.785 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_ops_log_rados": "false", 2023-09-23T20:22:43.785 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_ops_log_socket_path": "", 2023-09-23T20:22:43.785 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_override_bucket_index_max_shards": "0", 2023-09-23T20:22:43.785 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_pending_bucket_index_op_expiration": "120", 2023-09-23T20:22:43.785 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_period_latest_epoch_info_oid": ".latest_epoch", 2023-09-23T20:22:43.786 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_period_push_interval": "2.000000", 2023-09-23T20:22:43.786 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_period_push_interval_max": "30.000000", 2023-09-23T20:22:43.786 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_period_root_pool": ".rgw.root", 2023-09-23T20:22:43.786 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_policy_reject_invalid_principals": "true", 2023-09-23T20:22:43.786 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_print_continue": "true", 2023-09-23T20:22:43.786 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_print_prohibited_content_length": "false", 2023-09-23T20:22:43.786 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_put_obj_max_window_size": "67108864", 2023-09-23T20:22:43.786 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_put_obj_min_window_size": "16777216", 2023-09-23T20:22:43.786 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_rados_pool_autoscale_bias": "4.000000", 2023-09-23T20:22:43.786 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_rados_pool_recovery_priority": "5", 2023-09-23T20:22:43.786 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_rados_tracing": "false", 2023-09-23T20:22:43.787 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_realm": "", 2023-09-23T20:22:43.787 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_realm_id": "", 2023-09-23T20:22:43.787 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_realm_root_pool": ".rgw.root", 2023-09-23T20:22:43.787 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_region": "", 2023-09-23T20:22:43.787 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_region_root_pool": ".rgw.root", 2023-09-23T20:22:43.787 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_relaxed_region_enforcement": "false", 2023-09-23T20:22:43.787 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_relaxed_s3_bucket_names": "false", 2023-09-23T20:22:43.787 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_remote_addr_param": "REMOTE_ADDR", 2023-09-23T20:22:43.787 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_request_uri": "", 2023-09-23T20:22:43.787 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_reshard_batch_size": "64", 2023-09-23T20:22:43.787 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_reshard_bucket_lock_duration": "360", 2023-09-23T20:22:43.787 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_reshard_max_aio": "128", 2023-09-23T20:22:43.788 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_reshard_num_logs": "16", 2023-09-23T20:22:43.788 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_reshard_thread_interval": "600", 2023-09-23T20:22:43.788 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_resolve_cname": "false", 2023-09-23T20:22:43.788 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_rest_getusage_op_compat": "false", 2023-09-23T20:22:43.788 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_run_sync_thread": "true", 2023-09-23T20:22:43.788 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_s3_auth_order": "sts, external, local", 2023-09-23T20:22:43.788 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_s3_auth_use_keystone": "false", 2023-09-23T20:22:43.788 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_s3_auth_use_ldap": "false", 2023-09-23T20:22:43.788 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_s3_auth_use_rados": "true", 2023-09-23T20:22:43.788 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_s3_auth_use_sts": "false", 2023-09-23T20:22:43.788 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_s3_client_max_sig_ver": "-1", 2023-09-23T20:22:43.789 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_s3_success_create_obj_status": "0", 2023-09-23T20:22:43.789 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_safe_max_objects_per_shard": "102400", 2023-09-23T20:22:43.789 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_scheduler_type": "throttler", 2023-09-23T20:22:43.789 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_script_uri": "", 2023-09-23T20:22:43.789 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_service_provider_name": "", 2023-09-23T20:22:43.789 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_shard_warning_threshold": "90.000000", 2023-09-23T20:22:43.789 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_sts_client_id": "", 2023-09-23T20:22:43.789 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_sts_client_secret": "", 2023-09-23T20:22:43.789 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_sts_entry": "sts", 2023-09-23T20:22:43.789 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_sts_key": "sts", 2023-09-23T20:22:43.789 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_sts_max_session_duration": "43200", 2023-09-23T20:22:43.790 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_sts_min_session_duration": "900", 2023-09-23T20:22:43.790 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_sts_token_introspection_url": "", 2023-09-23T20:22:43.790 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_swift_account_in_url": "false", 2023-09-23T20:22:43.790 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_swift_auth_entry": "auth", 2023-09-23T20:22:43.790 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_swift_auth_url": "", 2023-09-23T20:22:43.790 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_swift_custom_header": "", 2023-09-23T20:22:43.790 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_swift_enforce_content_length": "false", 2023-09-23T20:22:43.790 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_swift_need_stats": "true", 2023-09-23T20:22:43.790 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_swift_tenant_name": "", 2023-09-23T20:22:43.790 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_swift_token_expiration": "86400", 2023-09-23T20:22:43.790 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_swift_url": "", 2023-09-23T20:22:43.790 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_swift_url_prefix": "swift", 2023-09-23T20:22:43.791 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_swift_versioning_enabled": "false", 2023-09-23T20:22:43.791 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_sync_data_full_inject_err_probability": "0.000000", 2023-09-23T20:22:43.791 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_sync_data_inject_err_probability": "0.000000", 2023-09-23T20:22:43.791 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_sync_lease_period": "120", 2023-09-23T20:22:43.791 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_sync_log_trim_concurrent_buckets": "4", 2023-09-23T20:22:43.791 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_sync_log_trim_interval": "1200", 2023-09-23T20:22:43.791 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_sync_log_trim_max_buckets": "16", 2023-09-23T20:22:43.791 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_sync_log_trim_min_cold_buckets": "4", 2023-09-23T20:22:43.791 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_sync_meta_inject_err_probability": "0.000000", 2023-09-23T20:22:43.791 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_sync_obj_etag_verify": "false", 2023-09-23T20:22:43.791 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_sync_trace_history_size": "4096", 2023-09-23T20:22:43.792 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_sync_trace_per_node_log_size": "32", 2023-09-23T20:22:43.792 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_sync_trace_servicemap_update_interval": "10", 2023-09-23T20:22:43.792 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_thread_pool_size": "512", 2023-09-23T20:22:43.792 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_torrent_comment": "", 2023-09-23T20:22:43.792 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_torrent_createby": "", 2023-09-23T20:22:43.792 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_torrent_encoding": "", 2023-09-23T20:22:43.792 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_torrent_flag": "false", 2023-09-23T20:22:43.792 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_torrent_origin": "", 2023-09-23T20:22:43.792 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_torrent_sha_unit": "524288", 2023-09-23T20:22:43.792 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_torrent_tracker": "", 2023-09-23T20:22:43.792 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_trust_forwarded_https": "false", 2023-09-23T20:22:43.792 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_usage_log_flush_threshold": "1024", 2023-09-23T20:22:43.793 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_usage_log_tick_interval": "30", 2023-09-23T20:22:43.793 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_usage_max_shards": "32", 2023-09-23T20:22:43.793 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_usage_max_user_shards": "1", 2023-09-23T20:22:43.793 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_use_opa_authz": "false", 2023-09-23T20:22:43.793 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_user_default_quota_max_objects": "-1", 2023-09-23T20:22:43.793 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_user_default_quota_max_size": "-1", 2023-09-23T20:22:43.793 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_user_max_buckets": "1000", 2023-09-23T20:22:43.793 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_user_policies_max_num": "100", 2023-09-23T20:22:43.793 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_user_quota_bucket_sync_interval": "180", 2023-09-23T20:22:43.793 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_user_quota_sync_idle_users": "false", 2023-09-23T20:22:43.793 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_user_quota_sync_interval": "86400", 2023-09-23T20:22:43.793 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_user_quota_sync_wait_time": "86400", 2023-09-23T20:22:43.794 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_user_unique_email": "true", 2023-09-23T20:22:43.794 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_verify_ssl": "true", 2023-09-23T20:22:43.794 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_website_routing_rules_max_num": "50", 2023-09-23T20:22:43.794 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_zone": "", 2023-09-23T20:22:43.794 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_zone_id": "", 2023-09-23T20:22:43.794 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_zone_root_pool": ".rgw.root", 2023-09-23T20:22:43.794 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_zonegroup": "", 2023-09-23T20:22:43.794 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_zonegroup_id": "", 2023-09-23T20:22:43.794 INFO:teuthology.orchestra.run.smithi018.stdout: "rgw_zonegroup_root_pool": ".rgw.root", 2023-09-23T20:22:43.794 INFO:teuthology.orchestra.run.smithi018.stdout: "rgwlc_auto_session_clear": "true", 2023-09-23T20:22:43.794 INFO:teuthology.orchestra.run.smithi018.stdout: "rgwlc_skip_bucket_step": "false", 2023-09-23T20:22:43.795 INFO:teuthology.orchestra.run.smithi018.stdout: "rocksdb_block_size": "4096", 2023-09-23T20:22:43.795 INFO:teuthology.orchestra.run.smithi018.stdout: "rocksdb_bloom_bits_per_key": "20", 2023-09-23T20:22:43.795 INFO:teuthology.orchestra.run.smithi018.stdout: "rocksdb_cache_index_and_filter_blocks": "true", 2023-09-23T20:22:43.795 INFO:teuthology.orchestra.run.smithi018.stdout: "rocksdb_cache_index_and_filter_blocks_with_high_priority": "false", 2023-09-23T20:22:43.795 INFO:teuthology.orchestra.run.smithi018.stdout: "rocksdb_cache_row_ratio": "0.000000", 2023-09-23T20:22:43.795 INFO:teuthology.orchestra.run.smithi018.stdout: "rocksdb_cache_shard_bits": "4", 2023-09-23T20:22:43.795 INFO:teuthology.orchestra.run.smithi018.stdout: "rocksdb_cache_size": "536870912", 2023-09-23T20:22:43.795 INFO:teuthology.orchestra.run.smithi018.stdout: "rocksdb_cache_type": "binned_lru", 2023-09-23T20:22:43.795 INFO:teuthology.orchestra.run.smithi018.stdout: "rocksdb_cf_compact_on_deletion": "true", 2023-09-23T20:22:43.795 INFO:teuthology.orchestra.run.smithi018.stdout: "rocksdb_cf_compact_on_deletion_sliding_window": "32768", 2023-09-23T20:22:43.795 INFO:teuthology.orchestra.run.smithi018.stdout: "rocksdb_cf_compact_on_deletion_trigger": "16384", 2023-09-23T20:22:43.795 INFO:teuthology.orchestra.run.smithi018.stdout: "rocksdb_collect_compaction_stats": "false", 2023-09-23T20:22:43.795 INFO:teuthology.orchestra.run.smithi018.stdout: "rocksdb_collect_extended_stats": "false", 2023-09-23T20:22:43.796 INFO:teuthology.orchestra.run.smithi018.stdout: "rocksdb_collect_memory_stats": "false", 2023-09-23T20:22:43.796 INFO:teuthology.orchestra.run.smithi018.stdout: "rocksdb_delete_range_threshold": "1048576", 2023-09-23T20:22:43.796 INFO:teuthology.orchestra.run.smithi018.stdout: "rocksdb_index_type": "binary_search", 2023-09-23T20:22:43.796 INFO:teuthology.orchestra.run.smithi018.stdout: "rocksdb_log_to_ceph_log": "true", 2023-09-23T20:22:43.796 INFO:teuthology.orchestra.run.smithi018.stdout: "rocksdb_metadata_block_size": "4096", 2023-09-23T20:22:43.796 INFO:teuthology.orchestra.run.smithi018.stdout: "rocksdb_partition_filters": "false", 2023-09-23T20:22:43.796 INFO:teuthology.orchestra.run.smithi018.stdout: "rocksdb_perf": "false", 2023-09-23T20:22:43.796 INFO:teuthology.orchestra.run.smithi018.stdout: "rocksdb_pin_l0_filter_and_index_blocks_in_cache": "false", 2023-09-23T20:22:43.796 INFO:teuthology.orchestra.run.smithi018.stdout: "rotating_keys_bootstrap_timeout": "30", 2023-09-23T20:22:43.796 INFO:teuthology.orchestra.run.smithi018.stdout: "rotating_keys_renewal_timeout": "10", 2023-09-23T20:22:43.796 INFO:teuthology.orchestra.run.smithi018.stdout: "run_dir": "/var/run/ceph", 2023-09-23T20:22:43.796 INFO:teuthology.orchestra.run.smithi018.stdout: "seastore_block_create": "true", 2023-09-23T20:22:43.797 INFO:teuthology.orchestra.run.smithi018.stdout: "seastore_cache_lru_size": "67108864", 2023-09-23T20:22:43.797 INFO:teuthology.orchestra.run.smithi018.stdout: "seastore_cbjournal_size": "5368709120", 2023-09-23T20:22:43.797 INFO:teuthology.orchestra.run.smithi018.stdout: "seastore_default_max_object_size": "16777216", 2023-09-23T20:22:43.797 INFO:teuthology.orchestra.run.smithi018.stdout: "seastore_default_object_metadata_reservation": "16777216", 2023-09-23T20:22:43.797 INFO:teuthology.orchestra.run.smithi018.stdout: "seastore_device_size": "53687091200", 2023-09-23T20:22:43.797 INFO:teuthology.orchestra.run.smithi018.stdout: "seastore_journal_batch_capacity": "16", 2023-09-23T20:22:43.797 INFO:teuthology.orchestra.run.smithi018.stdout: "seastore_journal_batch_flush_size": "16777216", 2023-09-23T20:22:43.797 INFO:teuthology.orchestra.run.smithi018.stdout: "seastore_journal_batch_preferred_fullness": "0.950000", 2023-09-23T20:22:43.797 INFO:teuthology.orchestra.run.smithi018.stdout: "seastore_journal_iodepth_limit": "5", 2023-09-23T20:22:43.797 INFO:teuthology.orchestra.run.smithi018.stdout: "seastore_main_device_type": "SSD", 2023-09-23T20:22:43.797 INFO:teuthology.orchestra.run.smithi018.stdout: "seastore_max_concurrent_transactions": "8", 2023-09-23T20:22:43.798 INFO:teuthology.orchestra.run.smithi018.stdout: "seastore_multiple_tiers_default_evict_ratio": "0.600000", 2023-09-23T20:22:43.798 INFO:teuthology.orchestra.run.smithi018.stdout: "seastore_multiple_tiers_fast_evict_ratio": "0.700000", 2023-09-23T20:22:43.798 INFO:teuthology.orchestra.run.smithi018.stdout: "seastore_multiple_tiers_stop_evict_ratio": "0.500000", 2023-09-23T20:22:43.798 INFO:teuthology.orchestra.run.smithi018.stdout: "seastore_obj_data_write_amplification": "1.250000", 2023-09-23T20:22:43.798 INFO:teuthology.orchestra.run.smithi018.stdout: "seastore_segment_size": "67108864", 2023-09-23T20:22:43.798 INFO:teuthology.orchestra.run.smithi018.stdout: "set_keepcaps": "false", 2023-09-23T20:22:43.798 INFO:teuthology.orchestra.run.smithi018.stdout: "setgroup": "ceph", 2023-09-23T20:22:43.798 INFO:teuthology.orchestra.run.smithi018.stdout: "setuser": "ceph", 2023-09-23T20:22:43.798 INFO:teuthology.orchestra.run.smithi018.stdout: "setuser_match_path": "", 2023-09-23T20:22:43.798 INFO:teuthology.orchestra.run.smithi018.stdout: "target_max_misplaced_ratio": "0.050000", 2023-09-23T20:22:43.798 INFO:teuthology.orchestra.run.smithi018.stdout: "thp": "false", 2023-09-23T20:22:43.798 INFO:teuthology.orchestra.run.smithi018.stdout: "threadpool_default_timeout": "60", 2023-09-23T20:22:43.799 INFO:teuthology.orchestra.run.smithi018.stdout: "threadpool_empty_queue_max_wait": "2", 2023-09-23T20:22:43.799 INFO:teuthology.orchestra.run.smithi018.stdout: "throttler_perf_counter": "true" 2023-09-23T20:22:43.799 INFO:teuthology.orchestra.run.smithi018.stdout:} 2023-09-23T20:22:44.048 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'mon.*' injectargs --mon_osd_down_out_interval=0 2023-09-23T20:22:44.653 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:22:45.136 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:22:44 smithi018 ceph-mon[184144]: pgmap v225: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:22:45.136 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:22:44 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:22:45.136 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:22:44 smithi018 ceph-mon[179647]: pgmap v225: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:22:45.136 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:22:44 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:22:45.136 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:22:44 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:22:44] "GET /metrics HTTP/1.1" 200 34024 "" "Prometheus/2.43.0" 2023-09-23T20:22:45.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:22:44 smithi149 ceph-mon[170394]: pgmap v225: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:22:45.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:22:44 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:22:45.894 INFO:teuthology.orchestra.run.smithi018.stdout:mon.a: {} 2023-09-23T20:22:45.904 INFO:teuthology.orchestra.run.smithi018.stdout:mon.b: {} 2023-09-23T20:22:45.914 INFO:teuthology.orchestra.run.smithi018.stdout:mon.c: {} 2023-09-23T20:22:45.915 INFO:teuthology.orchestra.run.smithi018.stderr:mon.a: mon_osd_down_out_interval = '0' (not observed, change may require restart) 2023-09-23T20:22:45.915 INFO:teuthology.orchestra.run.smithi018.stderr:mon.b: mon_osd_down_out_interval = '0' (not observed, change may require restart) 2023-09-23T20:22:45.915 INFO:teuthology.orchestra.run.smithi018.stderr:mon.c: mon_osd_down_out_interval = '0' (not observed, change may require restart) 2023-09-23T20:22:46.309 INFO:tasks.thrashosds.thrasher:Unable to test ceph-objectstore-tool, not available on all OSD nodes 2023-09-23T20:22:46.310 INFO:teuthology.task.sequential:In sequential, running task full_sequential... 2023-09-23T20:22:46.310 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-09-23T20:22:46.311 INFO:tasks.radosbench:Beginning radosbench... 2023-09-23T20:22:46.312 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_0 2023-09-23T20:22:46.312 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd pool create unique_pool_0 16 2023-09-23T20:22:46.312 INFO:tasks.thrashosds.thrasher:starting do_thrash 2023-09-23T20:22:46.312 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-23T20:22:46.312 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:22:46.313 INFO:tasks.thrashosds.thrasher:starting do_sighup with a delay of 0.1 2023-09-23T20:22:46.313 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:22:46.313 INFO:tasks.thrashosds.thrasher:starting do_optrack_toggle with a delay of 2.0 2023-09-23T20:22:46.313 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:22:46.314 INFO:tasks.thrashosds.thrasher:starting do_dump_ops 2023-09-23T20:22:46.314 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T20:22:46.314 INFO:tasks.thrashosds.thrasher:starting do_noscrub_toggle with a delay of 2.0 2023-09-23T20:22:46.314 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T20:22:46.357 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:46.414 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:22:46.418 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:46.419 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:22:46.519 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:22:46.594 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:46.635 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:22:46.637 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:46.637 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:22:46.738 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:22:46.767 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:46.801 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:22:46.803 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:46.804 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:22:46.905 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:22:46.936 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:46.966 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:22:47.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:22:46 smithi018 ceph-mon[184144]: pgmap v226: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:22:47.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:22:46 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1178365918' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T20:22:47.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:22:46 smithi018 ceph-mon[179647]: pgmap v226: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:22:47.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:22:46 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1178365918' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T20:22:47.142 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:22:47.144 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:22:47.146 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:47.147 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:22:47.147 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:22:47.147 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:22:47.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:22:46 smithi149 ceph-mon[170394]: pgmap v226: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:22:47.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:22:46 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1178365918' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T20:22:47.248 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:22:47.278 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:47.321 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:22:47.326 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:47.327 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:22:47.427 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:22:47.457 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:48.198 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:22:48.202 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:48.202 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:22:48.303 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:22:48.336 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:48.372 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:22:48.374 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:48.375 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:22:48.475 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:22:48.503 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:48.537 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:22:48.539 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:48.540 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:22:48.641 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:22:48.667 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:48.699 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:22:48.702 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:48.702 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:22:48.803 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:22:48.828 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:48.861 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:22:48.864 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:48.864 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:22:48.965 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:22:48.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:22:48 smithi149 ceph-mon[170394]: pgmap v227: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:22:48.991 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:49.023 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:22:49.025 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:49.026 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:22:49.118 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:22:48 smithi018 ceph-mon[179647]: pgmap v227: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:22:49.118 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:22:48 smithi018 ceph-mon[184144]: pgmap v227: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:22:49.127 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:22:49.148 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:22:49.161 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:49.169 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:22:49.190 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:22:49.210 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:22:49.230 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:22:49.249 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:22:49.268 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:22:49.289 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:22:49.290 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:22:49.290 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:22:49.290 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:22:49.290 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:22:49.290 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:22:49.290 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:22:49.290 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:22:49.290 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:22:49.594 INFO:teuthology.orchestra.run.smithi018.stdout:ERROR: (22) Invalid argument 2023-09-23T20:22:49.595 INFO:teuthology.orchestra.run.smithi018.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:22:49.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:22:49 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1307432447' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:22:49.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:22:49 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1307432447' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:22:50.135 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:22:50.138 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:50.139 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:22:50.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:22:49 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1307432447' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:22:50.239 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:22:50.270 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:50.291 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T20:22:50.304 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:22:50.306 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:50.307 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:22:50.407 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:22:50.435 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:50.529 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:22:50.532 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:50.532 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:22:50.633 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:22:50.669 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:50.766 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:22:50.769 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:50.770 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:22:50.870 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:22:50.897 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:50.932 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:22:50.934 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:22:50 smithi018 ceph-mon[184144]: pgmap v228: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:22:50.934 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:22:50 smithi018 ceph-mon[179647]: pgmap v228: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:22:50.934 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:50.935 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:22:50.939 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:22:51.036 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:22:51.063 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:51.162 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:22:51.165 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:51.165 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:22:51.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:22:50 smithi149 ceph-mon[170394]: pgmap v228: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:22:51.266 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:22:51.297 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:51.335 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:22:51.337 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:51.338 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:22:51.439 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:22:51.468 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:51.576 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:22:51.578 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:51.579 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:22:51.680 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:22:51.711 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:51.747 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:22:51.750 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:51.750 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:22:51.848 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:22:51 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3598166834' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:22:51.849 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:22:51 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/4219112720' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-09-23T20:22:51.849 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:22:51 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-09-23T20:22:51.851 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:22:51.863 INFO:teuthology.orchestra.run.smithi018.stderr:pool 'unique_pool_0' created 2023-09-23T20:22:51.863 INFO:teuthology.orchestra.run.smithi018.stderr:noscrub is set 2023-09-23T20:22:51.894 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:51.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:22:51 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3598166834' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:22:51.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:22:51 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/4219112720' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-09-23T20:22:51.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:22:51 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-09-23T20:22:52.091 INFO:teuthology.orchestra.run.smithi018.stdout:ERROR: (22) Invalid argument 2023-09-23T20:22:52.091 INFO:teuthology.orchestra.run.smithi018.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:22:52.113 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:22:51 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3598166834' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:22:52.114 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:22:51 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/4219112720' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-09-23T20:22:52.114 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:22:51 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-09-23T20:22:52.396 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:22:52.483 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:22:52.486 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:52.486 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:22:52.587 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:22:52.618 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:52.662 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd pool application enable unique_pool_0 rados --yes-i-really-mean-it || true 2023-09-23T20:22:52.698 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T20:22:52.944 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:22:52.947 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:22:52 smithi018 ceph-mon[184144]: pgmap v229: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:22:52.947 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:22:52 smithi018 ceph-mon[184144]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:22:52.947 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:22:52 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3598166834' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:22:52.947 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:22:52 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2023-09-23T20:22:52.947 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:22:52 smithi018 ceph-mon[184144]: osdmap e111: 8 total, 8 up, 8 in 2023-09-23T20:22:52.947 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:22:52 smithi018 ceph-mon[184144]: failed to encode map e111 with expected crc 2023-09-23T20:22:52.947 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:22:52 smithi018 ceph-mon[184144]: failed to encode map e111 with expected crc 2023-09-23T20:22:52.947 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:22:52 smithi018 ceph-mon[184144]: failed to encode map e111 with expected crc 2023-09-23T20:22:52.948 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:22:52 smithi018 ceph-mon[184144]: failed to encode map e111 with expected crc 2023-09-23T20:22:52.948 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:22:52 smithi018 ceph-mon[179647]: pgmap v229: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:22:52.948 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:22:52 smithi018 ceph-mon[179647]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:22:52.948 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:22:52 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3598166834' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:22:52.948 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:22:52 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2023-09-23T20:22:52.948 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:22:52 smithi018 ceph-mon[179647]: osdmap e111: 8 total, 8 up, 8 in 2023-09-23T20:22:52.948 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:22:52 smithi018 ceph-mon[179647]: failed to encode map e111 with expected crc 2023-09-23T20:22:52.948 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:22:52 smithi018 ceph-mon[179647]: failed to encode map e111 with expected crc 2023-09-23T20:22:52.948 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:22:52 smithi018 ceph-mon[179647]: failed to encode map e111 with expected crc 2023-09-23T20:22:52.948 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:22:52 smithi018 ceph-mon[179647]: failed to encode map e111 with expected crc 2023-09-23T20:22:52.949 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:52.949 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:22:53.050 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:22:53.077 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:53.105 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:22:53.118 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:22:53.122 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:53.123 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:22:53.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:22:52 smithi149 ceph-mon[170394]: pgmap v229: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:22:53.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:22:52 smithi149 ceph-mon[170394]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:22:53.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:22:52 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3598166834' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:22:53.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:22:52 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2023-09-23T20:22:53.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:22:52 smithi149 ceph-mon[170394]: osdmap e111: 8 total, 8 up, 8 in 2023-09-23T20:22:53.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:22:52 smithi149 ceph-mon[170394]: failed to encode map e111 with expected crc 2023-09-23T20:22:53.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:22:52 smithi149 ceph-mon[170394]: failed to encode map e111 with expected crc 2023-09-23T20:22:53.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:22:52 smithi149 ceph-mon[170394]: failed to encode map e111 with expected crc 2023-09-23T20:22:53.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:22:52 smithi149 ceph-mon[170394]: failed to encode map e111 with expected crc 2023-09-23T20:22:53.223 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:22:53.254 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:53.289 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:22:53.291 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:53.292 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:22:53.393 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:22:53.420 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:53.453 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:22:53.456 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:53.457 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:22:53.523 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:22:53.523 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:22:53.558 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:22:53.589 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:53.625 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:22:53.627 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:53.628 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:22:53.729 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:22:53.756 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:53.832 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:22:53.835 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:53.835 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:22:53.859 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:22:53 smithi018 ceph-mon[179647]: osdmap e112: 8 total, 8 up, 8 in 2023-09-23T20:22:53.859 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:22:53 smithi018 ceph-mon[179647]: failed to encode map e112 with expected crc 2023-09-23T20:22:53.859 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:22:53 smithi018 ceph-mon[179647]: failed to encode map e112 with expected crc 2023-09-23T20:22:53.859 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:22:53 smithi018 ceph-mon[179647]: failed to encode map e112 with expected crc 2023-09-23T20:22:53.937 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:22:53.964 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:53.998 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:22:54.000 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:54.001 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:22:54.101 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:22:54.117 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:22:53 smithi018 ceph-mon[179647]: failed to encode map e112 with expected crc 2023-09-23T20:22:54.117 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:22:53 smithi018 ceph-mon[184144]: osdmap e112: 8 total, 8 up, 8 in 2023-09-23T20:22:54.117 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:22:53 smithi018 ceph-mon[184144]: failed to encode map e112 with expected crc 2023-09-23T20:22:54.117 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:22:53 smithi018 ceph-mon[184144]: failed to encode map e112 with expected crc 2023-09-23T20:22:54.117 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:22:53 smithi018 ceph-mon[184144]: failed to encode map e112 with expected crc 2023-09-23T20:22:54.118 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:22:53 smithi018 ceph-mon[184144]: failed to encode map e112 with expected crc 2023-09-23T20:22:54.136 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:54.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:22:53 smithi149 ceph-mon[170394]: osdmap e112: 8 total, 8 up, 8 in 2023-09-23T20:22:54.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:22:53 smithi149 ceph-mon[170394]: failed to encode map e112 with expected crc 2023-09-23T20:22:54.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:22:53 smithi149 ceph-mon[170394]: failed to encode map e112 with expected crc 2023-09-23T20:22:54.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:22:53 smithi149 ceph-mon[170394]: failed to encode map e112 with expected crc 2023-09-23T20:22:54.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:22:53 smithi149 ceph-mon[170394]: failed to encode map e112 with expected crc 2023-09-23T20:22:54.227 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:22:54.230 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:54.230 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:22:54.331 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:22:54.362 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:54.404 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:22:54.407 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:54.407 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:22:54.508 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:22:54.537 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:54.674 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:22:54.677 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:54.678 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:22:54.779 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:22:54.806 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:54.975 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:22:54 smithi018 ceph-mon[184144]: pgmap v232: 89 pgs: 5 creating+peering, 8 unknown, 76 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:22:54.975 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:22:54 smithi018 ceph-mon[184144]: failed to encode map e113 with expected crc 2023-09-23T20:22:54.975 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:22:54 smithi018 ceph-mon[184144]: osdmap e113: 8 total, 8 up, 8 in 2023-09-23T20:22:54.975 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:22:54 smithi018 ceph-mon[184144]: failed to encode map e113 with expected crc 2023-09-23T20:22:54.975 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:22:54 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1690739306' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:22:54.975 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:22:54 smithi018 ceph-mon[179647]: pgmap v232: 89 pgs: 5 creating+peering, 8 unknown, 76 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:22:54.976 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:22:54 smithi018 ceph-mon[179647]: failed to encode map e113 with expected crc 2023-09-23T20:22:54.976 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:22:54 smithi018 ceph-mon[179647]: osdmap e113: 8 total, 8 up, 8 in 2023-09-23T20:22:54.976 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:22:54 smithi018 ceph-mon[179647]: failed to encode map e113 with expected crc 2023-09-23T20:22:54.976 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:22:54 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1690739306' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:22:54.999 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T20:22:55.053 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:22:55.053 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:22:55.053 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:22:55.054 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:22:55.054 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:22:55.054 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:22:55.054 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:22:55.054 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:22:55.054 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:22:55.054 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:22:55.054 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:22:55.054 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:22:55.055 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:22:55.055 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:22:55.055 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:22:55.055 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:22:55.099 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:22:55.102 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:55.102 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:22:55.203 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:22:55.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:22:54 smithi149 ceph-mon[170394]: pgmap v232: 89 pgs: 5 creating+peering, 8 unknown, 76 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:22:55.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:22:54 smithi149 ceph-mon[170394]: failed to encode map e113 with expected crc 2023-09-23T20:22:55.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:22:54 smithi149 ceph-mon[170394]: osdmap e113: 8 total, 8 up, 8 in 2023-09-23T20:22:55.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:22:54 smithi149 ceph-mon[170394]: failed to encode map e113 with expected crc 2023-09-23T20:22:55.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:22:54 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1690739306' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:22:55.235 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:55.270 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:22:55.273 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:55.274 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:22:55.374 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:22:55.390 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:22:54 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:22:54] "GET /metrics HTTP/1.1" 200 34024 "" "Prometheus/2.43.0" 2023-09-23T20:22:55.404 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:55.503 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:22:55.505 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:55.506 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:22:55.606 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:22:55.637 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:55.674 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:22:55.676 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:55.677 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:22:55.777 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:22:55.806 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:55.841 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:22:55.843 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:55.844 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:22:55.944 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:22:55.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:22:55 smithi149 ceph-mon[170394]: failed to encode map e113 with expected crc 2023-09-23T20:22:55.971 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:56.004 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:22:56.006 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:56.007 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:22:56.057 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2023-09-23T20:22:56.057 INFO:teuthology.orchestra.run.smithi018.stdout: "size": 20, 2023-09-23T20:22:56.057 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 600, 2023-09-23T20:22:56.057 INFO:teuthology.orchestra.run.smithi018.stdout: "ops": [] 2023-09-23T20:22:56.057 INFO:teuthology.orchestra.run.smithi018.stdout:} 2023-09-23T20:22:56.082 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:22:55 smithi018 ceph-mon[179647]: failed to encode map e113 with expected crc 2023-09-23T20:22:56.082 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:22:55 smithi018 ceph-mon[184144]: failed to encode map e113 with expected crc 2023-09-23T20:22:56.108 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:22:56.134 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:56.165 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:22:56.168 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:56.168 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:22:56.174 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:22:56.269 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:22:56.298 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:56.459 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:22:56.462 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:56.462 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:22:56.488 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T20:22:56.563 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:22:56.601 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:56.636 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:22:56.638 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:56.638 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:22:56.739 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:22:56.767 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:56.799 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:22:56.802 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:56.802 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:22:56.877 INFO:teuthology.orchestra.run.smithi018.stderr:enabled application 'rados' on pool 'unique_pool_0' 2023-09-23T20:22:56.903 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:22:56.930 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:57.084 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:22:56 smithi018 ceph-mon[184144]: pgmap v234: 89 pgs: 5 creating+peering, 84 active+clean; 4.6 MiB data, 165 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:22:57.084 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:22:56 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/700378247' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T20:22:57.084 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:22:56 smithi018 ceph-mon[179647]: pgmap v234: 89 pgs: 5 creating+peering, 84 active+clean; 4.6 MiB data, 165 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:22:57.084 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:22:56 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/700378247' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T20:22:57.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:22:56 smithi149 ceph-mon[170394]: pgmap v234: 89 pgs: 5 creating+peering, 84 active+clean; 4.6 MiB data, 165 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:22:57.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:22:56 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/700378247' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T20:22:57.321 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:22:57.324 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:57.325 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:22:57.426 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:22:57.454 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:57.485 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:22:57.486 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:22:57.489 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:57.489 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:22:57.533 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:22:57.590 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:22:57.620 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:57.763 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:22:57.766 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:57.766 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:22:57.867 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:22:57.901 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:58.022 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:22:57 smithi018 ceph-mon[184144]: failed to encode map e114 with expected crc 2023-09-23T20:22:58.022 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:22:57 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/700378247' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T20:22:58.023 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:22:57 smithi018 ceph-mon[184144]: osdmap e114: 8 total, 8 up, 8 in 2023-09-23T20:22:58.023 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:22:57 smithi018 ceph-mon[179647]: failed to encode map e114 with expected crc 2023-09-23T20:22:58.023 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:22:57 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/700378247' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T20:22:58.023 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:22:57 smithi018 ceph-mon[179647]: osdmap e114: 8 total, 8 up, 8 in 2023-09-23T20:22:58.139 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:22:58.141 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:58.142 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:22:58.192 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:22:58.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:22:57 smithi149 ceph-mon[170394]: failed to encode map e114 with expected crc 2023-09-23T20:22:58.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:22:57 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/700378247' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T20:22:58.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:22:57 smithi149 ceph-mon[170394]: osdmap e114: 8 total, 8 up, 8 in 2023-09-23T20:22:58.243 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:22:58.273 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:58.309 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:22:58.311 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:58.312 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:22:58.399 DEBUG:teuthology.orchestra.run.smithi018:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_0 bench 90 write' 2023-09-23T20:22:58.400 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_0 2023-09-23T20:22:58.400 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:22:58.402 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-09-23T20:22:58.402 INFO:teuthology.orchestra.run:waiting for 3000 2023-09-23T20:22:58.412 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:22:58.440 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:58.451 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:hints = 1 2023-09-23T20:22:58.451 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-09-23T20:22:58.451 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Object prefix: benchmark_data_smithi018_228451 2023-09-23T20:22:58.451 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T20:22:58.452 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 0 0 0 0 0 0 - 0 2023-09-23T20:22:58.491 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:22:58.495 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:58.495 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:22:58.596 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:22:58.694 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:58.876 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:22:58.881 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:58.881 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:22:58.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:22:58 smithi018 ceph-mon[179647]: pgmap v236: 89 pgs: 89 active+clean; 4.6 MiB data, 165 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:22:58.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:22:58 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/4280064532' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:22:58.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:22:58 smithi018 ceph-mon[184144]: pgmap v236: 89 pgs: 89 active+clean; 4.6 MiB data, 165 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:22:58.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:22:58 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/4280064532' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:22:58.922 INFO:teuthology.orchestra.run.smithi018.stderr:nodeep-scrub is set 2023-09-23T20:22:58.983 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:22:59.041 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:59.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:22:58 smithi149 ceph-mon[170394]: pgmap v236: 89 pgs: 89 active+clean; 4.6 MiB data, 165 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:22:59.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:22:58 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/4280064532' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:22:59.452 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 1 16 3400 3384 211.489 211.5 0.00550326 0.00470389 2023-09-23T20:22:59.709 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:22:59.713 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:22:59.714 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:22:59.815 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:22:59.856 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2023-09-23T20:22:59.856 INFO:teuthology.orchestra.run.smithi018.stdout: "ops": [ 2023-09-23T20:22:59.856 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:22:59.856 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.34791.0:4703 5.c 5:3d505036:::benchmark_data_smithi018_228451_object4702:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e115)", 2023-09-23T20:22:59.857 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:22:59.850947+0000", 2023-09-23T20:22:59.857 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.001303982, 2023-09-23T20:22:59.857 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.001341959, 2023-09-23T20:22:59.857 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:22:59.857 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:22:59.857 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:22:59.857 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.34791", 2023-09-23T20:22:59.857 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:22:59.857 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 4703 2023-09-23T20:22:59.857 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:22:59.857 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:22:59.857 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:22:59.858 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:22:59.858 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:22:59.850947+0000", 2023-09-23T20:22:59.858 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:22:59.858 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:22:59.858 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:22:59.858 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:22:59.858 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:22:59.850947+0000", 2023-09-23T20:22:59.858 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:22:59.858 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:22:59.858 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:22:59.858 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:22:59.859 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:22:59.850945+0000", 2023-09-23T20:22:59.859 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:22:59.859 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:22:59.859 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:22:59.859 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:22:59.859 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:22:59.850974+0000", 2023-09-23T20:22:59.859 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.9074e-05 2023-09-23T20:22:59.859 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:22:59.859 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:22:59.859 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:22:59.859 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:22:59.850993+0000", 2023-09-23T20:22:59.860 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.9091000000000001e-05 2023-09-23T20:22:59.860 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:22:59.860 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:22:59.860 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:22:59.860 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:22:59.851004+0000", 2023-09-23T20:22:59.860 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0834e-05 2023-09-23T20:22:59.860 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:22:59.860 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:22:59.860 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:22:59.860 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:22:59.851062+0000", 2023-09-23T20:22:59.860 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.7797999999999998e-05 2023-09-23T20:22:59.861 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:22:59.861 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:22:59.861 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:22:59.861 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:22:59.851229+0000", 2023-09-23T20:22:59.861 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00016716499999999999 2023-09-23T20:22:59.861 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:22:59.861 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:22:59.861 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 0,4", 2023-09-23T20:22:59.861 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:22:59.851384+0000", 2023-09-23T20:22:59.862 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00015496800000000001 2023-09-23T20:22:59.862 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:22:59.862 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:22:59.862 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:22:59.862 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:22:59.862 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:22:59.862 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_repop(client.34791.0:4706 5.0 e115/111)", 2023-09-23T20:22:59.862 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:22:59.851987+0000", 2023-09-23T20:22:59.862 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.00026412700000000001, 2023-09-23T20:22:59.862 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00038691099999999998, 2023-09-23T20:22:59.863 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:22:59.863 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "queued for pg", 2023-09-23T20:22:59.863 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:22:59.863 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:22:59.863 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:22:59.863 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:22:59.851987+0000", 2023-09-23T20:22:59.863 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:22:59.863 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:22:59.863 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:22:59.863 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:22:59.863 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:22:59.851987+0000", 2023-09-23T20:22:59.864 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:22:59.864 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:22:59.864 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:22:59.864 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:22:59.864 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:22:59.851986+0000", 2023-09-23T20:22:59.864 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.999999 2023-09-23T20:22:59.864 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:22:59.864 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:22:59.864 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:22:59.864 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:22:59.852010+0000", 2023-09-23T20:22:59.865 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.4097e-05 2023-09-23T20:22:59.865 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:22:59.865 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:22:59.865 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:22:59.865 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:22:59.852025+0000", 2023-09-23T20:22:59.865 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.4836e-05 2023-09-23T20:22:59.865 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:22:59.865 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:22:59.865 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:22:59.865 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:22:59.852032+0000", 2023-09-23T20:22:59.866 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.6459999999999993e-06 2023-09-23T20:22:59.866 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:22:59.866 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:22:59.866 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:22:59.866 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:22:59.866 INFO:teuthology.orchestra.run.smithi018.stdout: ], 2023-09-23T20:22:59.866 INFO:teuthology.orchestra.run.smithi018.stdout: "num_ops": 2 2023-09-23T20:22:59.866 INFO:teuthology.orchestra.run.smithi018.stdout:} 2023-09-23T20:22:59.890 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:22:59.909 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:22:59.980 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:22:59 smithi018 ceph-mon[179647]: failed to encode map e114 with expected crc 2023-09-23T20:22:59.981 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:22:59 smithi018 ceph-mon[179647]: failed to encode map e114 with expected crc 2023-09-23T20:22:59.981 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:22:59 smithi018 ceph-mon[179647]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:22:59.981 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:22:59 smithi018 ceph-mon[179647]: failed to encode map e115 with expected crc 2023-09-23T20:22:59.981 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:22:59 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/4280064532' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:22:59.981 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:22:59 smithi018 ceph-mon[179647]: osdmap e115: 8 total, 8 up, 8 in 2023-09-23T20:22:59.981 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:22:59 smithi018 ceph-mon[179647]: failed to encode map e115 with expected crc 2023-09-23T20:22:59.981 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:22:59 smithi018 ceph-mon[179647]: failed to encode map e115 with expected crc 2023-09-23T20:22:59.981 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:22:59 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:22:59.981 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:22:59 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:22:59.981 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:22:59 smithi018 ceph-mon[184144]: failed to encode map e114 with expected crc 2023-09-23T20:22:59.982 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:22:59 smithi018 ceph-mon[184144]: failed to encode map e114 with expected crc 2023-09-23T20:22:59.982 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:22:59 smithi018 ceph-mon[184144]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:22:59.982 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:22:59 smithi018 ceph-mon[184144]: failed to encode map e115 with expected crc 2023-09-23T20:22:59.982 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:22:59 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/4280064532' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:22:59.982 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:22:59 smithi018 ceph-mon[184144]: osdmap e115: 8 total, 8 up, 8 in 2023-09-23T20:22:59.982 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:22:59 smithi018 ceph-mon[184144]: failed to encode map e115 with expected crc 2023-09-23T20:22:59.982 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:22:59 smithi018 ceph-mon[184144]: failed to encode map e115 with expected crc 2023-09-23T20:22:59.982 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:22:59 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:22:59.982 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:22:59 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:23:00.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:22:59 smithi149 ceph-mon[170394]: failed to encode map e114 with expected crc 2023-09-23T20:23:00.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:22:59 smithi149 ceph-mon[170394]: failed to encode map e114 with expected crc 2023-09-23T20:23:00.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:22:59 smithi149 ceph-mon[170394]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:23:00.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:22:59 smithi149 ceph-mon[170394]: failed to encode map e115 with expected crc 2023-09-23T20:23:00.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:22:59 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/4280064532' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:23:00.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:22:59 smithi149 ceph-mon[170394]: osdmap e115: 8 total, 8 up, 8 in 2023-09-23T20:23:00.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:22:59 smithi149 ceph-mon[170394]: failed to encode map e115 with expected crc 2023-09-23T20:23:00.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:22:59 smithi149 ceph-mon[170394]: failed to encode map e115 with expected crc 2023-09-23T20:23:00.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:22:59 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:23:00.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:22:59 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:23:00.221 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:23:00.224 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:00.225 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:23:00.326 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:23:00.365 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T20:23:00.381 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:00.449 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:23:00.451 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 2 16 6596 6580 205.609 199.75 0.0049907 0.00483776 2023-09-23T20:23:00.457 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:00.457 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:23:00.558 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:23:00.637 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:00.786 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:23:00.792 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:00.793 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:23:00.833 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:23:00.877 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:23:00.890 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:00 smithi018 ceph-mon[184144]: pgmap v238: 89 pgs: 89 active+clean; 4.6 MiB data, 165 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:23:00.890 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:00 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1609355430' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:23:00.893 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:23:00.913 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:23:00.950 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:00.954 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:23:01.005 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:23:01.041 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:23:01.083 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:23:01.119 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:23:01.124 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:23:01.124 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:23:01.124 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:23:01.124 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:23:01.124 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:23:01.124 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:23:01.124 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:23:01.124 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:23:01.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:00 smithi018 ceph-mon[179647]: pgmap v238: 89 pgs: 89 active+clean; 4.6 MiB data, 165 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:23:01.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:00 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1609355430' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:23:01.149 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:23:01.152 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:01.153 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:23:01.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:00 smithi149 ceph-mon[170394]: pgmap v238: 89 pgs: 89 active+clean; 4.6 MiB data, 165 MiB used, 715 GiB / 715 GiB avail 2023-09-23T20:23:01.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:00 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1609355430' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:23:01.253 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:23:01.294 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:01.352 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:23:01.359 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:01.359 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:23:01.452 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 3 15 9120 9105 189.672 157.812 0.00574909 0.00526494 2023-09-23T20:23:01.460 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:23:01.496 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:01.545 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:23:01.552 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:01.552 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:23:01.653 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:23:01.688 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:01.743 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:23:01.748 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:01.748 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:23:01.784 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T20:23:01.799 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:23:01.849 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:23:01.899 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:01.954 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:23:01.958 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:01.958 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:23:02.059 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:23:02.187 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:02.359 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:23:02.367 INFO:teuthology.orchestra.run.smithi018.stderr:dumped all 2023-09-23T20:23:02.452 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 4 16 11608 11592 181.097 155.438 0.00766208 0.00551401 2023-09-23T20:23:02.506 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:23:02.510 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:02.510 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:23:02.611 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:23:02.659 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:02.722 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:23:02.727 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:02.727 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:23:02.828 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:23:02.900 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:02.913 INFO:teuthology.orchestra.run.smithi018.stdout:{"pg_ready":true,"pg_map":{"version":239,"stamp":"2023-09-23T20:23:01.436501+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":208934835,"num_objects":3143,"num_object_clones":0,"num_object_copies":9429,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3143,"num_whiteouts":0,"num_read":6495,"num_read_kb":14136,"num_write":4261,"num_write_kb":214501,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4307,"ondisk_log_size":4307,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":813480,"kb_used_data":627304,"kb_used_omap":30,"kb_used_meta":186081,"kb_avail":749147736,"statfs":{"total":767960285184,"available":767127281664,"internally_reserved":0,"allocated":642359296,"data_stored":637984901,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31263,"internal_metadata":190547425},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,1,3],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":204144640,"num_objects":3115,"num_object_clones":0,"num_object_copies":9345,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3115,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3115,"num_write_kb":199360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"7.577708"},"pg_stats":[{"pgid":"3.1a","version":"69'1","reported_seq":93,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-09-23T20:22:57.592578+0000","last_change":"2023-09-23T20:21:18.891671+0000","last_active":"2023-09-23T20:22:57.592578+0000","last_peered":"2023-09-23T20:22:57.592578+0000","last_clean":"2023-09-23T20:22:57.592578+0000","last_became_active":"2023-09-23T20:21:18.891285+0000","last_became_peered":"2023-09-23T20:21:18.891285+0000","last_unstale":"2023-09-23T20:22:57.592578+0000","last_undegraded":"2023-09-23T20:22:57.592578+0000","last_fullsized":"2023-09-23T20:22:57.592578+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:04:49.595473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":75,"reported_epoch":112,"state":"active+clean","last_fresh":"2023-09-23T20:22:52.858993+0000","last_change":"2023-09-23T20:21:49.951483+0000","last_active":"2023-09-23T20:22:52.858993+0000","last_peered":"2023-09-23T20:22:52.858993+0000","last_clean":"2023-09-23T20:22:52.858993+0000","last_became_active":"2023-09-23T20:21:49.951167+0000","last_became_peered":"2023-09-23T20:21:49.951167+0000","last_unstale":"2023-09-23T20:22:52.858993+0000","last_undegraded":"2023-09-23T20:22:52.858993+0000","last_fullsized":"2023-09-23T20:22:52.858993+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:08:19.552997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":52,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-09-23T20:22:56.871525+0000","last_change":"2023-09-23T20:21:49.955411+0000","last_active":"2023-09-23T20:22:56.871525+0000","last_peered":"2023-09-23T20:22:56.871525+0000","last_clean":"2023-09-23T20:22:56.871525+0000","last_became_active":"2023-09-23T20:21:49.955112+0000","last_became_peered":"2023-09-23T20:21:49.955112+0000","last_unstale":"2023-09-23T20:22:56.871525+0000","last_undegraded":"2023-09-23T20:22:56.871525+0000","last_fullsized":"2023-09-23T20:22:56.871525+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:47:53.870056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":57,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-09-23T20:22:53.880548+0000","last_change":"2023-09-23T20:21:49.954419+0000","last_active":"2023-09-23T20:22:53.880548+0000","last_peered":"2023-09-23T20:22:53.880548+0000","last_clean":"2023-09-23T20:22:53.880548+0000","last_became_active":"2023-09-23T20:21:49.953941+0000","last_became_peered":"2023-09-23T20:21:49.953941+0000","last_unstale":"2023-09-23T20:22:53.880548+0000","last_undegraded":"2023-09-23T20:22:53.880548+0000","last_fullsized":"2023-09-23T20:22:53.880548+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:05:38.809180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":93,"reported_epoch":112,"state":"active+clean","last_fresh":"2023-09-23T20:22:52.859800+0000","last_change":"2023-09-23T20:21:49.958727+0000","last_active":"2023-09-23T20:22:52.859800+0000","last_peered":"2023-09-23T20:22:52.859800+0000","last_clean":"2023-09-23T20:22:52.859800+0000","last_became_active":"2023-09-23T20:21:49.957722+0000","last_became_peered":"2023-09-23T20:21:49.957722+0000","last_unstale":"2023-09-23T20:22:52.859800+0000","last_undegraded":"2023-09-23T20:22:52.859800+0000","last_fullsized":"2023-09-23T20:22:52.859800+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:47:13.589014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":74,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-09-23T20:22:58.346732+0000","last_change":"2023-09-23T20:20:46.937089+0000","last_active":"2023-09-23T20:22:58.346732+0000","last_peered":"2023-09-23T20:22:58.346732+0000","last_clean":"2023-09-23T20:22:58.346732+0000","last_became_active":"2023-09-23T20:20:46.936831+0000","last_became_peered":"2023-09-23T20:20:46.936831+0000","last_unstale":"2023-09-23T20:22:58.346732+0000","last_undegraded":"2023-09-23T20:22:58.346732+0000","last_fullsized":"2023-09-23T20:22:58.346732+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:34:56.747010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":56,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-09-23T20:22:53.880566+0000","last_change":"2023-09-23T20:21:49.950402+0000","last_active":"2023-09-23T20:22:53.880566+0000","last_peered":"2023-09-23T20:22:53.880566+0000","last_clean":"2023-09-23T20:22:53.880566+0000","last_became_active":"2023-09-23T20:21:49.950116+0000","last_became_peered":"2023-09-23T20:21:49.950116+0000","last_unstale":"2023-09-23T20:22:53.880566+0000","last_undegraded":"2023-09-23T20:22:53.880566+0000","last_fullsized":"2023-09-23T20:22:53.880566+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:10:20.322615+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":75,"reported_epoch":112,"state":"active+clean","last_fresh":"2023-09-23T20:22:52.859709+0000","last_change":"2023-09-23T20:21:49.958837+0000","last_active":"2023-09-23T20:22:52.859709+0000","last_peered":"2023-09-23T20:22:52.859709+0000","last_clean":"2023-09-23T20:22:52.859709+0000","last_became_active":"2023-09-23T20:21:49.958135+0000","last_became_peered":"2023-09-23T20:21:49.958135+0000","last_unstale":"2023-09-23T20:22:52.859709+0000","last_undegraded":"2023-09-23T20:22:52.859709+0000","last_fullsized":"2023-09-23T20:22:52.859709+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:31:52.350275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":74,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-09-23T20:22:56.873864+0000","last_change":"2023-09-23T20:20:46.950281+0000","last_active":"2023-09-23T20:22:56.873864+0000","last_peered":"2023-09-23T20:22:56.873864+0000","last_clean":"2023-09-23T20:22:56.873864+0000","last_became_active":"2023-09-23T20:20:46.949892+0000","last_became_peered":"2023-09-23T20:20:46.949892+0000","last_unstale":"2023-09-23T20:22:56.873864+0000","last_undegraded":"2023-09-23T20:22:56.873864+0000","last_fullsized":"2023-09-23T20:22:56.873864+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:28.763832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":26,"reported_epoch":112,"state":"active+clean","last_fresh":"2023-09-23T20:22:52.858433+0000","last_change":"2023-09-23T20:21:49.955179+0000","last_active":"2023-09-23T20:22:52.858433+0000","last_peered":"2023-09-23T20:22:52.858433+0000","last_clean":"2023-09-23T20:22:52.858433+0000","last_became_active":"2023-09-23T20:21:49.954878+0000","last_became_peered":"2023-09-23T20:21:49.954878+0000","last_unstale":"2023-09-23T20:22:52.858433+0000","last_undegraded":"2023-09-23T20:22:52.858433+0000","last_fullsized":"2023-09-23T20:22:52.858433+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:02:26.579246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":78,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-09-23T20:22:58.346557+0000","last_change":"2023-09-23T20:21:49.954219+0000","last_active":"2023-09-23T20:22:58.346557+0000","last_peered":"2023-09-23T20:22:58.346557+0000","last_clean":"2023-09-23T20:22:58.346557+0000","last_became_active":"2023-09-23T20:21:49.953827+0000","last_became_peered":"2023-09-23T20:21:49.953827+0000","last_unstale":"2023-09-23T20:22:58.346557+0000","last_undegraded":"2023-09-23T20:22:58.346557+0000","last_fullsized":"2023-09-23T20:22:58.346557+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:46:48.036706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":53,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-09-23T20:22:56.871823+0000","last_change":"2023-09-23T20:21:18.891438+0000","last_active":"2023-09-23T20:22:56.871823+0000","last_peered":"2023-09-23T20:22:56.871823+0000","last_clean":"2023-09-23T20:22:56.871823+0000","last_became_active":"2023-09-23T20:21:18.891140+0000","last_became_peered":"2023-09-23T20:21:18.891140+0000","last_unstale":"2023-09-23T20:22:56.871823+0000","last_undegraded":"2023-09-23T20:22:56.871823+0000","last_fullsized":"2023-09-23T20:22:56.871823+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:18:44.603577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":80,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-09-23T20:22:57.592800+0000","last_change":"2023-09-23T20:12:43.750217+0000","last_active":"2023-09-23T20:22:57.592800+0000","last_peered":"2023-09-23T20:22:57.592800+0000","last_clean":"2023-09-23T20:22:57.592800+0000","last_became_active":"2023-09-23T20:12:43.749929+0000","last_became_peered":"2023-09-23T20:12:43.749929+0000","last_unstale":"2023-09-23T20:22:57.592800+0000","last_undegraded":"2023-09-23T20:22:57.592800+0000","last_fullsized":"2023-09-23T20:22:57.592800+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:50:28.470211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":74,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-09-23T20:22:57.592876+0000","last_change":"2023-09-23T20:20:46.939938+0000","last_active":"2023-09-23T20:22:57.592876+0000","last_peered":"2023-09-23T20:22:57.592876+0000","last_clean":"2023-09-23T20:22:57.592876+0000","last_became_active":"2023-09-23T20:20:46.938777+0000","last_became_peered":"2023-09-23T20:20:46.938777+0000","last_unstale":"2023-09-23T20:22:57.592876+0000","last_undegraded":"2023-09-23T20:22:57.592876+0000","last_fullsized":"2023-09-23T20:22:57.592876+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:45:10.329438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":77,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-09-23T20:22:58.346214+0000","last_change":"2023-09-23T20:21:18.897418+0000","last_active":"2023-09-23T20:22:58.346214+0000","last_peered":"2023-09-23T20:22:58.346214+0000","last_clean":"2023-09-23T20:22:58.346214+0000","last_became_active":"2023-09-23T20:21:18.896968+0000","last_became_peered":"2023-09-23T20:21:18.896968+0000","last_unstale":"2023-09-23T20:22:58.346214+0000","last_undegraded":"2023-09-23T20:22:58.346214+0000","last_fullsized":"2023-09-23T20:22:58.346214+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:12:44.651568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":75,"reported_epoch":112,"state":"active+clean","last_fresh":"2023-09-23T20:22:52.858875+0000","last_change":"2023-09-23T20:21:49.958515+0000","last_active":"2023-09-23T20:22:52.858875+0000","last_peered":"2023-09-23T20:22:52.858875+0000","last_clean":"2023-09-23T20:22:52.858875+0000","last_became_active":"2023-09-23T20:21:49.958156+0000","last_became_peered":"2023-09-23T20:21:49.958156+0000","last_unstale":"2023-09-23T20:22:52.858875+0000","last_undegraded":"2023-09-23T20:22:52.858875+0000","last_fullsized":"2023-09-23T20:22:52.858875+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:56:47.145702+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":56,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-09-23T20:22:53.880244+0000","last_change":"2023-09-23T20:21:49.953201+0000","last_active":"2023-09-23T20:22:53.880244+0000","last_peered":"2023-09-23T20:22:53.880244+0000","last_clean":"2023-09-23T20:22:53.880244+0000","last_became_active":"2023-09-23T20:21:49.952917+0000","last_became_peered":"2023-09-23T20:21:49.952917+0000","last_unstale":"2023-09-23T20:22:53.880244+0000","last_undegraded":"2023-09-23T20:22:53.880244+0000","last_fullsized":"2023-09-23T20:22:53.880244+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:54.579892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":73,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-09-23T20:22:56.873846+0000","last_change":"2023-09-23T20:21:49.952635+0000","last_active":"2023-09-23T20:22:56.873846+0000","last_peered":"2023-09-23T20:22:56.873846+0000","last_clean":"2023-09-23T20:22:56.873846+0000","last_became_active":"2023-09-23T20:21:49.952287+0000","last_became_peered":"2023-09-23T20:21:49.952287+0000","last_unstale":"2023-09-23T20:22:56.873846+0000","last_undegraded":"2023-09-23T20:22:56.873846+0000","last_fullsized":"2023-09-23T20:22:56.873846+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:17:17.370526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":58,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-09-23T20:22:58.346176+0000","last_change":"2023-09-23T20:12:43.749139+0000","last_active":"2023-09-23T20:22:58.346176+0000","last_peered":"2023-09-23T20:22:58.346176+0000","last_clean":"2023-09-23T20:22:58.346176+0000","last_became_active":"2023-09-23T20:12:43.748854+0000","last_became_peered":"2023-09-23T20:12:43.748854+0000","last_unstale":"2023-09-23T20:22:58.346176+0000","last_undegraded":"2023-09-23T20:22:58.346176+0000","last_fullsized":"2023-09-23T20:22:58.346176+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:57:11.630712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":57,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-09-23T20:22:53.880463+0000","last_change":"2023-09-23T20:21:49.954242+0000","last_active":"2023-09-23T20:22:53.880463+0000","last_peered":"2023-09-23T20:22:53.880463+0000","last_clean":"2023-09-23T20:22:53.880463+0000","last_became_active":"2023-09-23T20:21:49.953597+0000","last_became_peered":"2023-09-23T20:21:49.953597+0000","last_unstale":"2023-09-23T20:22:53.880463+0000","last_undegraded":"2023-09-23T20:22:53.880463+0000","last_fullsized":"2023-09-23T20:22:53.880463+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:08:46.716466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":83,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-09-23T20:22:58.346055+0000","last_change":"2023-09-23T20:20:14.881009+0000","last_active":"2023-09-23T20:22:58.346055+0000","last_peered":"2023-09-23T20:22:58.346055+0000","last_clean":"2023-09-23T20:22:58.346055+0000","last_became_active":"2023-09-23T20:20:14.880685+0000","last_became_peered":"2023-09-23T20:20:14.880685+0000","last_unstale":"2023-09-23T20:22:58.346055+0000","last_undegraded":"2023-09-23T20:22:58.346055+0000","last_fullsized":"2023-09-23T20:22:58.346055+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:28.010434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":75,"reported_epoch":112,"state":"active+clean","last_fresh":"2023-09-23T20:22:52.859206+0000","last_change":"2023-09-23T20:21:49.958182+0000","last_active":"2023-09-23T20:22:52.859206+0000","last_peered":"2023-09-23T20:22:52.859206+0000","last_clean":"2023-09-23T20:22:52.859206+0000","last_became_active":"2023-09-23T20:21:49.957868+0000","last_became_peered":"2023-09-23T20:21:49.957868+0000","last_unstale":"2023-09-23T20:22:52.859206+0000","last_undegraded":"2023-09-23T20:22:52.859206+0000","last_fullsized":"2023-09-23T20:22:52.859206+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:27:41.577566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.9","version":"115'364","reported_seq":376,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:23:00.358955+0000","last_change":"2023-09-23T20:22:52.867071+0000","last_active":"2023-09-23T20:23:00.358955+0000","last_peered":"2023-09-23T20:23:00.358955+0000","last_clean":"2023-09-23T20:23:00.358955+0000","last_became_active":"2023-09-23T20:22:52.866494+0000","last_became_peered":"2023-09-23T20:22:52.866494+0000","last_unstale":"2023-09-23T20:23:00.358955+0000","last_undegraded":"2023-09-23T20:23:00.358955+0000","last_fullsized":"2023-09-23T20:23:00.358955+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:22:51.846999+0000","objects_scrubbed":0,"log_size":364,"log_dups_size":0,"ondisk_log_size":364,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:04:42.023719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":23855104,"num_objects":364,"num_object_clones":0,"num_object_copies":1092,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":364,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":364,"num_write_kb":23296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":76,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-09-23T20:22:58.346198+0000","last_change":"2023-09-23T20:20:14.883283+0000","last_active":"2023-09-23T20:22:58.346198+0000","last_peered":"2023-09-23T20:22:58.346198+0000","last_clean":"2023-09-23T20:22:58.346198+0000","last_became_active":"2023-09-23T20:20:14.882806+0000","last_became_peered":"2023-09-23T20:20:14.882806+0000","last_unstale":"2023-09-23T20:22:58.346198+0000","last_undegraded":"2023-09-23T20:22:58.346198+0000","last_fullsized":"2023-09-23T20:22:58.346198+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:23:35.295701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":56,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-09-23T20:22:56.873539+0000","last_change":"2023-09-23T20:12:45.179211+0000","last_active":"2023-09-23T20:22:56.873539+0000","last_peered":"2023-09-23T20:22:56.873539+0000","last_clean":"2023-09-23T20:22:56.873539+0000","last_became_active":"2023-09-23T20:12:45.178966+0000","last_became_peered":"2023-09-23T20:12:45.178966+0000","last_unstale":"2023-09-23T20:22:56.873539+0000","last_undegraded":"2023-09-23T20:22:56.873539+0000","last_fullsized":"2023-09-23T20:22:56.873539+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:58.902140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.8","version":"0'0","reported_seq":11,"reported_epoch":112,"state":"active+clean","last_fresh":"2023-09-23T20:22:52.865248+0000","last_change":"2023-09-23T20:22:52.864948+0000","last_active":"2023-09-23T20:22:52.865248+0000","last_peered":"2023-09-23T20:22:52.865248+0000","last_clean":"2023-09-23T20:22:52.865248+0000","last_became_active":"2023-09-23T20:22:52.864633+0000","last_became_peered":"2023-09-23T20:22:52.864633+0000","last_unstale":"2023-09-23T20:22:52.865248+0000","last_undegraded":"2023-09-23T20:22:52.865248+0000","last_fullsized":"2023-09-23T20:22:52.865248+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:22:51.846999+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:47:54.574720+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":77,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-09-23T20:22:58.346496+0000","last_change":"2023-09-23T20:20:46.950784+0000","last_active":"2023-09-23T20:22:58.346496+0000","last_peered":"2023-09-23T20:22:58.346496+0000","last_clean":"2023-09-23T20:22:58.346496+0000","last_became_active":"2023-09-23T20:20:46.950055+0000","last_became_peered":"2023-09-23T20:20:46.950055+0000","last_unstale":"2023-09-23T20:22:58.346496+0000","last_undegraded":"2023-09-23T20:22:58.346496+0000","last_fullsized":"2023-09-23T20:22:58.346496+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:23:07.645427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":91,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-09-23T20:22:57.592652+0000","last_change":"2023-09-23T20:21:49.957494+0000","last_active":"2023-09-23T20:22:57.592652+0000","last_peered":"2023-09-23T20:22:57.592652+0000","last_clean":"2023-09-23T20:22:57.592652+0000","last_became_active":"2023-09-23T20:21:49.957266+0000","last_became_peered":"2023-09-23T20:21:49.957266+0000","last_unstale":"2023-09-23T20:22:57.592652+0000","last_undegraded":"2023-09-23T20:22:57.592652+0000","last_fullsized":"2023-09-23T20:22:57.592652+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:49:51.483575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.b","version":"0'0","reported_seq":11,"reported_epoch":112,"state":"active+clean","last_fresh":"2023-09-23T20:22:52.871211+0000","last_change":"2023-09-23T20:22:52.869646+0000","last_active":"2023-09-23T20:22:52.871211+0000","last_peered":"2023-09-23T20:22:52.871211+0000","last_clean":"2023-09-23T20:22:52.871211+0000","last_became_active":"2023-09-23T20:22:52.869345+0000","last_became_peered":"2023-09-23T20:22:52.869345+0000","last_unstale":"2023-09-23T20:22:52.871211+0000","last_undegraded":"2023-09-23T20:22:52.871211+0000","last_fullsized":"2023-09-23T20:22:52.871211+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:22:51.846999+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:16:39.278977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":59,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-09-23T20:22:58.346656+0000","last_change":"2023-09-23T20:12:43.751523+0000","last_active":"2023-09-23T20:22:58.346656+0000","last_peered":"2023-09-23T20:22:58.346656+0000","last_clean":"2023-09-23T20:22:58.346656+0000","last_became_active":"2023-09-23T20:12:43.751257+0000","last_became_peered":"2023-09-23T20:12:43.751257+0000","last_unstale":"2023-09-23T20:22:58.346656+0000","last_undegraded":"2023-09-23T20:22:58.346656+0000","last_fullsized":"2023-09-23T20:22:58.346656+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:50:27.040225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":74,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-09-23T20:22:58.346752+0000","last_change":"2023-09-23T20:20:46.934762+0000","last_active":"2023-09-23T20:22:58.346752+0000","last_peered":"2023-09-23T20:22:58.346752+0000","last_clean":"2023-09-23T20:22:58.346752+0000","last_became_active":"2023-09-23T20:20:46.934472+0000","last_became_peered":"2023-09-23T20:20:46.934472+0000","last_unstale":"2023-09-23T20:22:58.346752+0000","last_undegraded":"2023-09-23T20:22:58.346752+0000","last_fullsized":"2023-09-23T20:22:58.346752+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:28:42.708286+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.a","version":"0'0","reported_seq":11,"reported_epoch":112,"state":"active+clean","last_fresh":"2023-09-23T20:22:52.865733+0000","last_change":"2023-09-23T20:22:52.865524+0000","last_active":"2023-09-23T20:22:52.865733+0000","last_peered":"2023-09-23T20:22:52.865733+0000","last_clean":"2023-09-23T20:22:52.865733+0000","last_became_active":"2023-09-23T20:22:52.865271+0000","last_became_peered":"2023-09-23T20:22:52.865271+0000","last_unstale":"2023-09-23T20:22:52.865733+0000","last_undegraded":"2023-09-23T20:22:52.865733+0000","last_fullsized":"2023-09-23T20:22:52.865733+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:22:51.846999+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:01:09.151325+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":74,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-09-23T20:22:56.873747+0000","last_change":"2023-09-23T20:21:49.954875+0000","last_active":"2023-09-23T20:22:56.873747+0000","last_peered":"2023-09-23T20:22:56.873747+0000","last_clean":"2023-09-23T20:22:56.873747+0000","last_became_active":"2023-09-23T20:21:49.954519+0000","last_became_peered":"2023-09-23T20:21:49.954519+0000","last_unstale":"2023-09-23T20:22:56.873747+0000","last_undegraded":"2023-09-23T20:22:56.873747+0000","last_fullsized":"2023-09-23T20:22:56.873747+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:29:52.962029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":57,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-09-23T20:22:53.880289+0000","last_change":"2023-09-23T20:20:46.935968+0000","last_active":"2023-09-23T20:22:53.880289+0000","last_peered":"2023-09-23T20:22:53.880289+0000","last_clean":"2023-09-23T20:22:53.880289+0000","last_became_active":"2023-09-23T20:20:46.935618+0000","last_became_peered":"2023-09-23T20:20:46.935618+0000","last_unstale":"2023-09-23T20:22:53.880289+0000","last_undegraded":"2023-09-23T20:22:53.880289+0000","last_fullsized":"2023-09-23T20:22:53.880289+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:42:59.867176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.d","version":"0'0","reported_seq":11,"reported_epoch":112,"state":"active+clean","last_fresh":"2023-09-23T20:22:52.871032+0000","last_change":"2023-09-23T20:22:52.869690+0000","last_active":"2023-09-23T20:22:52.871032+0000","last_peered":"2023-09-23T20:22:52.871032+0000","last_clean":"2023-09-23T20:22:52.871032+0000","last_became_active":"2023-09-23T20:22:52.869366+0000","last_became_peered":"2023-09-23T20:22:52.869366+0000","last_unstale":"2023-09-23T20:22:52.871032+0000","last_undegraded":"2023-09-23T20:22:52.871032+0000","last_fullsized":"2023-09-23T20:22:52.871032+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:22:51.846999+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:16:02.423281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":73,"reported_epoch":112,"state":"active+clean","last_fresh":"2023-09-23T20:22:52.858681+0000","last_change":"2023-09-23T20:21:49.953004+0000","last_active":"2023-09-23T20:22:52.858681+0000","last_peered":"2023-09-23T20:22:52.858681+0000","last_clean":"2023-09-23T20:22:52.858681+0000","last_became_active":"2023-09-23T20:21:49.952625+0000","last_became_peered":"2023-09-23T20:21:49.952625+0000","last_unstale":"2023-09-23T20:22:52.858681+0000","last_undegraded":"2023-09-23T20:22:52.858681+0000","last_fullsized":"2023-09-23T20:22:52.858681+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:48:28.305640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":74,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-09-23T20:22:57.592668+0000","last_change":"2023-09-23T20:21:49.954710+0000","last_active":"2023-09-23T20:22:57.592668+0000","last_peered":"2023-09-23T20:22:57.592668+0000","last_clean":"2023-09-23T20:22:57.592668+0000","last_became_active":"2023-09-23T20:21:49.954355+0000","last_became_peered":"2023-09-23T20:21:49.954355+0000","last_unstale":"2023-09-23T20:22:57.592668+0000","last_undegraded":"2023-09-23T20:22:57.592668+0000","last_fullsized":"2023-09-23T20:22:57.592668+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:56:16.233285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.c","version":"115'310","reported_seq":323,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:22:59.938229+0000","last_change":"2023-09-23T20:22:52.866632+0000","last_active":"2023-09-23T20:22:59.938229+0000","last_peered":"2023-09-23T20:22:59.938229+0000","last_clean":"2023-09-23T20:22:59.938229+0000","last_became_active":"2023-09-23T20:22:52.866356+0000","last_became_peered":"2023-09-23T20:22:52.866356+0000","last_unstale":"2023-09-23T20:22:59.938229+0000","last_undegraded":"2023-09-23T20:22:59.938229+0000","last_fullsized":"2023-09-23T20:22:59.938229+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:22:51.846999+0000","objects_scrubbed":0,"log_size":310,"log_dups_size":0,"ondisk_log_size":310,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:10:12.356062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":20316160,"num_objects":310,"num_object_clones":0,"num_object_copies":930,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":310,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":310,"num_write_kb":19840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":75,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-09-23T20:22:58.346817+0000","last_change":"2023-09-23T20:20:46.937163+0000","last_active":"2023-09-23T20:22:58.346817+0000","last_peered":"2023-09-23T20:22:58.346817+0000","last_clean":"2023-09-23T20:22:58.346817+0000","last_became_active":"2023-09-23T20:20:46.936757+0000","last_became_peered":"2023-09-23T20:20:46.936757+0000","last_unstale":"2023-09-23T20:22:58.346817+0000","last_undegraded":"2023-09-23T20:22:58.346817+0000","last_fullsized":"2023-09-23T20:22:58.346817+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:09.401379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":84,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-09-23T20:22:57.592779+0000","last_change":"2023-09-23T20:21:18.891669+0000","last_active":"2023-09-23T20:22:57.592779+0000","last_peered":"2023-09-23T20:22:57.592779+0000","last_clean":"2023-09-23T20:22:57.592779+0000","last_became_active":"2023-09-23T20:21:18.891326+0000","last_became_peered":"2023-09-23T20:21:18.891326+0000","last_unstale":"2023-09-23T20:22:57.592779+0000","last_undegraded":"2023-09-23T20:22:57.592779+0000","last_fullsized":"2023-09-23T20:22:57.592779+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:48:39.893213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.f","version":"115'434","reported_seq":446,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:23:00.358895+0000","last_change":"2023-09-23T20:22:52.872510+0000","last_active":"2023-09-23T20:23:00.358895+0000","last_peered":"2023-09-23T20:23:00.358895+0000","last_clean":"2023-09-23T20:23:00.358895+0000","last_became_active":"2023-09-23T20:22:52.872179+0000","last_became_peered":"2023-09-23T20:22:52.872179+0000","last_unstale":"2023-09-23T20:23:00.358895+0000","last_undegraded":"2023-09-23T20:23:00.358895+0000","last_fullsized":"2023-09-23T20:23:00.358895+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:22:51.846999+0000","objects_scrubbed":0,"log_size":434,"log_dups_size":0,"ondisk_log_size":434,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:53.012272+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":28442624,"num_objects":434,"num_object_clones":0,"num_object_copies":1302,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":434,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":434,"num_write_kb":27776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":75,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-09-23T20:22:57.592482+0000","last_change":"2023-09-23T20:21:18.896635+0000","last_active":"2023-09-23T20:22:57.592482+0000","last_peered":"2023-09-23T20:22:57.592482+0000","last_clean":"2023-09-23T20:22:57.592482+0000","last_became_active":"2023-09-23T20:21:18.896335+0000","last_became_peered":"2023-09-23T20:21:18.896335+0000","last_unstale":"2023-09-23T20:22:57.592482+0000","last_undegraded":"2023-09-23T20:22:57.592482+0000","last_fullsized":"2023-09-23T20:22:57.592482+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:46:43.743795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":74,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-09-23T20:22:57.592872+0000","last_change":"2023-09-23T20:20:46.939720+0000","last_active":"2023-09-23T20:22:57.592872+0000","last_peered":"2023-09-23T20:22:57.592872+0000","last_clean":"2023-09-23T20:22:57.592872+0000","last_became_active":"2023-09-23T20:20:46.939357+0000","last_became_peered":"2023-09-23T20:20:46.939357+0000","last_unstale":"2023-09-23T20:22:57.592872+0000","last_undegraded":"2023-09-23T20:22:57.592872+0000","last_fullsized":"2023-09-23T20:22:57.592872+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:56.613212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.e","version":"115'420","reported_seq":432,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:23:00.354215+0000","last_change":"2023-09-23T20:22:52.871704+0000","last_active":"2023-09-23T20:23:00.354215+0000","last_peered":"2023-09-23T20:23:00.354215+0000","last_clean":"2023-09-23T20:23:00.354215+0000","last_became_active":"2023-09-23T20:22:52.871267+0000","last_became_peered":"2023-09-23T20:22:52.871267+0000","last_unstale":"2023-09-23T20:23:00.354215+0000","last_undegraded":"2023-09-23T20:23:00.354215+0000","last_fullsized":"2023-09-23T20:23:00.354215+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:22:51.846999+0000","objects_scrubbed":0,"log_size":420,"log_dups_size":0,"ondisk_log_size":420,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:50:07.619801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":27525120,"num_objects":420,"num_object_clones":0,"num_object_copies":1260,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":420,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":420,"num_write_kb":26880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":76,"reported_epoch":112,"state":"active+clean","last_fresh":"2023-09-23T20:22:52.859926+0000","last_change":"2023-09-23T20:21:49.958915+0000","last_active":"2023-09-23T20:22:52.859926+0000","last_peered":"2023-09-23T20:22:52.859926+0000","last_clean":"2023-09-23T20:22:52.859926+0000","last_became_active":"2023-09-23T20:21:49.958440+0000","last_became_peered":"2023-09-23T20:21:49.958440+0000","last_unstale":"2023-09-23T20:22:52.859926+0000","last_undegraded":"2023-09-23T20:22:52.859926+0000","last_fullsized":"2023-09-23T20:22:52.859926+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:36:48.998945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":46,"reported_epoch":112,"state":"active+clean","last_fresh":"2023-09-23T20:22:52.862733+0000","last_change":"2023-09-23T20:21:49.959532+0000","last_active":"2023-09-23T20:22:52.862733+0000","last_peered":"2023-09-23T20:22:52.862733+0000","last_clean":"2023-09-23T20:22:52.862733+0000","last_became_active":"2023-09-23T20:21:49.959161+0000","last_became_peered":"2023-09-23T20:21:49.959161+0000","last_unstale":"2023-09-23T20:22:52.862733+0000","last_undegraded":"2023-09-23T20:22:52.862733+0000","last_fullsized":"2023-09-23T20:22:52.862733+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:00:05.013667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"86'119","reported_seq":447,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-09-23T20:22:56.872071+0000","last_change":"2023-09-23T20:20:46.937071+0000","last_active":"2023-09-23T20:22:56.872071+0000","last_peered":"2023-09-23T20:22:56.872071+0000","last_clean":"2023-09-23T20:22:56.872071+0000","last_became_active":"2023-09-23T20:20:46.935348+0000","last_became_peered":"2023-09-23T20:20:46.935348+0000","last_unstale":"2023-09-23T20:22:56.872071+0000","last_undegraded":"2023-09-23T20:22:56.872071+0000","last_fullsized":"2023-09-23T20:22:56.872071+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":119,"log_dups_size":0,"ondisk_log_size":119,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:09.829030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032705499999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":523,"num_read_kb":3503,"num_write":136,"num_write_kb":3128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":91,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-09-23T20:22:56.873857+0000","last_change":"2023-09-23T20:21:49.953141+0000","last_active":"2023-09-23T20:22:56.873857+0000","last_peered":"2023-09-23T20:22:56.873857+0000","last_clean":"2023-09-23T20:22:56.873857+0000","last_became_active":"2023-09-23T20:21:49.952794+0000","last_became_peered":"2023-09-23T20:21:49.952794+0000","last_unstale":"2023-09-23T20:22:56.873857+0000","last_undegraded":"2023-09-23T20:22:56.873857+0000","last_fullsized":"2023-09-23T20:22:56.873857+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:36:19.919248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":39,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-09-23T20:22:56.871832+0000","last_change":"2023-09-23T20:20:46.937104+0000","last_active":"2023-09-23T20:22:56.871832+0000","last_peered":"2023-09-23T20:22:56.871832+0000","last_clean":"2023-09-23T20:22:56.871832+0000","last_became_active":"2023-09-23T20:20:46.936820+0000","last_became_peered":"2023-09-23T20:20:46.936820+0000","last_unstale":"2023-09-23T20:22:56.871832+0000","last_undegraded":"2023-09-23T20:22:56.871832+0000","last_fullsized":"2023-09-23T20:22:56.871832+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:22:19.879310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.3","version":"0'0","reported_seq":12,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-09-23T20:22:53.880514+0000","last_change":"2023-09-23T20:22:52.870842+0000","last_active":"2023-09-23T20:22:53.880514+0000","last_peered":"2023-09-23T20:22:53.880514+0000","last_clean":"2023-09-23T20:22:53.880514+0000","last_became_active":"2023-09-23T20:22:52.870364+0000","last_became_peered":"2023-09-23T20:22:52.870364+0000","last_unstale":"2023-09-23T20:22:53.880514+0000","last_undegraded":"2023-09-23T20:22:53.880514+0000","last_fullsized":"2023-09-23T20:22:53.880514+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:22:51.846999+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:38:59.773675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.5","version":"0'0","reported_seq":12,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-09-23T20:22:53.880343+0000","last_change":"2023-09-23T20:22:52.867106+0000","last_active":"2023-09-23T20:22:53.880343+0000","last_peered":"2023-09-23T20:22:53.880343+0000","last_clean":"2023-09-23T20:22:53.880343+0000","last_became_active":"2023-09-23T20:22:52.866781+0000","last_became_peered":"2023-09-23T20:22:52.866781+0000","last_unstale":"2023-09-23T20:22:53.880343+0000","last_undegraded":"2023-09-23T20:22:53.880343+0000","last_fullsized":"2023-09-23T20:22:53.880343+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:22:51.846999+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:32:01.131890+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"86'106","reported_seq":739,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-09-23T20:22:56.873656+0000","last_change":"2023-09-23T20:20:46.939468+0000","last_active":"2023-09-23T20:22:56.873656+0000","last_peered":"2023-09-23T20:22:56.873656+0000","last_clean":"2023-09-23T20:22:56.873656+0000","last_became_active":"2023-09-23T20:20:46.938954+0000","last_became_peered":"2023-09-23T20:20:46.938954+0000","last_unstale":"2023-09-23T20:22:56.873656+0000","last_undegraded":"2023-09-23T20:22:56.873656+0000","last_fullsized":"2023-09-23T20:22:56.873656+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":106,"log_dups_size":0,"ondisk_log_size":106,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:01:43.761064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00099302599999999993,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1105,"num_read_kb":1689,"num_write":100,"num_write_kb":404,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":75,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-09-23T20:22:57.592676+0000","last_change":"2023-09-23T20:20:14.887345+0000","last_active":"2023-09-23T20:22:57.592676+0000","last_peered":"2023-09-23T20:22:57.592676+0000","last_clean":"2023-09-23T20:22:57.592676+0000","last_became_active":"2023-09-23T20:20:14.886970+0000","last_became_peered":"2023-09-23T20:20:14.886970+0000","last_unstale":"2023-09-23T20:22:57.592676+0000","last_undegraded":"2023-09-23T20:22:57.592676+0000","last_fullsized":"2023-09-23T20:22:57.592676+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:00:38.575926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":69,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-09-23T20:22:56.872061+0000","last_change":"2023-09-23T20:21:49.952961+0000","last_active":"2023-09-23T20:22:56.872061+0000","last_peered":"2023-09-23T20:22:56.872061+0000","last_clean":"2023-09-23T20:22:56.872061+0000","last_became_active":"2023-09-23T20:21:49.952644+0000","last_became_peered":"2023-09-23T20:21:49.952644+0000","last_unstale":"2023-09-23T20:22:56.872061+0000","last_undegraded":"2023-09-23T20:22:56.872061+0000","last_fullsized":"2023-09-23T20:22:56.872061+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:05:30.492540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.6","version":"0'0","reported_seq":11,"reported_epoch":112,"state":"active+clean","last_fresh":"2023-09-23T20:22:52.871018+0000","last_change":"2023-09-23T20:22:52.870321+0000","last_active":"2023-09-23T20:22:52.871018+0000","last_peered":"2023-09-23T20:22:52.871018+0000","last_clean":"2023-09-23T20:22:52.871018+0000","last_became_active":"2023-09-23T20:22:52.870089+0000","last_became_peered":"2023-09-23T20:22:52.870089+0000","last_unstale":"2023-09-23T20:22:52.871018+0000","last_undegraded":"2023-09-23T20:22:52.871018+0000","last_fullsized":"2023-09-23T20:22:52.871018+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:22:51.846999+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:01:42.591394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"86'130","reported_seq":416,"reported_epoch":112,"state":"active+clean","last_fresh":"2023-09-23T20:22:52.862202+0000","last_change":"2023-09-23T20:21:49.954331+0000","last_active":"2023-09-23T20:22:52.862202+0000","last_peered":"2023-09-23T20:22:52.862202+0000","last_clean":"2023-09-23T20:22:52.862202+0000","last_became_active":"2023-09-23T20:21:49.953687+0000","last_became_peered":"2023-09-23T20:21:49.953687+0000","last_unstale":"2023-09-23T20:22:52.862202+0000","last_undegraded":"2023-09-23T20:22:52.862202+0000","last_fullsized":"2023-09-23T20:22:52.862202+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:12.549803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011300049999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":512,"num_read_kb":1820,"num_write":133,"num_write_kb":1347,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":53,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-09-23T20:22:56.871589+0000","last_change":"2023-09-23T20:21:18.891553+0000","last_active":"2023-09-23T20:22:56.871589+0000","last_peered":"2023-09-23T20:22:56.871589+0000","last_clean":"2023-09-23T20:22:56.871589+0000","last_became_active":"2023-09-23T20:21:18.891230+0000","last_became_peered":"2023-09-23T20:21:18.891230+0000","last_unstale":"2023-09-23T20:22:56.871589+0000","last_undegraded":"2023-09-23T20:22:56.871589+0000","last_fullsized":"2023-09-23T20:22:56.871589+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:15:04.114927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":82,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-09-23T20:22:56.872369+0000","last_change":"2023-09-23T20:20:46.936524+0000","last_active":"2023-09-23T20:22:56.872369+0000","last_peered":"2023-09-23T20:22:56.872369+0000","last_clean":"2023-09-23T20:22:56.872369+0000","last_became_active":"2023-09-23T20:20:46.936267+0000","last_became_peered":"2023-09-23T20:20:46.936267+0000","last_unstale":"2023-09-23T20:22:56.872369+0000","last_undegraded":"2023-09-23T20:22:56.872369+0000","last_fullsized":"2023-09-23T20:22:56.872369+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:32:20.984521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"86'194","reported_seq":733,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-09-23T20:22:56.873653+0000","last_change":"2023-09-23T20:21:18.892176+0000","last_active":"2023-09-23T20:22:56.873653+0000","last_peered":"2023-09-23T20:22:56.873653+0000","last_clean":"2023-09-23T20:22:56.873653+0000","last_became_active":"2023-09-23T20:21:18.891647+0000","last_became_peered":"2023-09-23T20:21:18.891647+0000","last_unstale":"2023-09-23T20:22:56.873653+0000","last_undegraded":"2023-09-23T20:22:56.873653+0000","last_fullsized":"2023-09-23T20:22:56.873653+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":194,"log_dups_size":0,"ondisk_log_size":194,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:25:52.557392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00037995999999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1025,"num_read_kb":1201,"num_write":173,"num_write_kb":1276,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":457,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-09-23T20:22:58.345982+0000","last_change":"2023-09-23T20:20:14.882204+0000","last_active":"2023-09-23T20:22:58.345982+0000","last_peered":"2023-09-23T20:22:58.345982+0000","last_clean":"2023-09-23T20:22:58.345982+0000","last_became_active":"2023-09-23T20:20:14.881699+0000","last_became_peered":"2023-09-23T20:20:14.881699+0000","last_unstale":"2023-09-23T20:22:58.345982+0000","last_undegraded":"2023-09-23T20:22:58.345982+0000","last_fullsized":"2023-09-23T20:22:58.345982+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:09:47.213130+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:16:37.341010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":25,"reported_epoch":112,"state":"active+clean","last_fresh":"2023-09-23T20:22:52.858572+0000","last_change":"2023-09-23T20:21:49.956277+0000","last_active":"2023-09-23T20:22:52.858572+0000","last_peered":"2023-09-23T20:22:52.858572+0000","last_clean":"2023-09-23T20:22:52.858572+0000","last_became_active":"2023-09-23T20:21:49.955974+0000","last_became_peered":"2023-09-23T20:21:49.955974+0000","last_unstale":"2023-09-23T20:22:52.858572+0000","last_undegraded":"2023-09-23T20:22:52.858572+0000","last_fullsized":"2023-09-23T20:22:52.858572+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:11:37.850675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":57,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-09-23T20:22:58.346526+0000","last_change":"2023-09-23T20:12:45.609829+0000","last_active":"2023-09-23T20:22:58.346526+0000","last_peered":"2023-09-23T20:22:58.346526+0000","last_clean":"2023-09-23T20:22:58.346526+0000","last_became_active":"2023-09-23T20:12:45.609467+0000","last_became_peered":"2023-09-23T20:12:45.609467+0000","last_unstale":"2023-09-23T20:22:58.346526+0000","last_undegraded":"2023-09-23T20:22:58.346526+0000","last_fullsized":"2023-09-23T20:22:58.346526+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:27:31.490555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.4","version":"115'383","reported_seq":395,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:23:00.353269+0000","last_change":"2023-09-23T20:22:52.870145+0000","last_active":"2023-09-23T20:23:00.353269+0000","last_peered":"2023-09-23T20:23:00.353269+0000","last_clean":"2023-09-23T20:23:00.353269+0000","last_became_active":"2023-09-23T20:22:52.869818+0000","last_became_peered":"2023-09-23T20:22:52.869818+0000","last_unstale":"2023-09-23T20:23:00.353269+0000","last_undegraded":"2023-09-23T20:23:00.353269+0000","last_fullsized":"2023-09-23T20:23:00.353269+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:22:51.846999+0000","objects_scrubbed":0,"log_size":383,"log_dups_size":0,"ondisk_log_size":383,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:57:09.153239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":25100288,"num_objects":383,"num_object_clones":0,"num_object_copies":1149,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":383,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":383,"num_write_kb":24512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.7","version":"115'389","reported_seq":401,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:23:00.356512+0000","last_change":"2023-09-23T20:22:52.870580+0000","last_active":"2023-09-23T20:23:00.356512+0000","last_peered":"2023-09-23T20:23:00.356512+0000","last_clean":"2023-09-23T20:23:00.356512+0000","last_became_active":"2023-09-23T20:22:52.870287+0000","last_became_peered":"2023-09-23T20:22:52.870287+0000","last_unstale":"2023-09-23T20:23:00.356512+0000","last_undegraded":"2023-09-23T20:23:00.356512+0000","last_fullsized":"2023-09-23T20:23:00.356512+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:22:51.846999+0000","objects_scrubbed":0,"log_size":389,"log_dups_size":0,"ondisk_log_size":389,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:40:25.674976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":25493504,"num_objects":389,"num_object_clones":0,"num_object_copies":1167,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":389,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":389,"num_write_kb":24896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.0","version":"86'80","reported_seq":361,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-09-23T20:22:58.346068+0000","last_change":"2023-09-23T20:20:46.950784+0000","last_active":"2023-09-23T20:22:58.346068+0000","last_peered":"2023-09-23T20:22:58.346068+0000","last_clean":"2023-09-23T20:22:58.346068+0000","last_became_active":"2023-09-23T20:20:46.950071+0000","last_became_peered":"2023-09-23T20:20:46.950071+0000","last_unstale":"2023-09-23T20:22:58.346068+0000","last_undegraded":"2023-09-23T20:22:58.346068+0000","last_fullsized":"2023-09-23T20:22:58.346068+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":80,"log_dups_size":0,"ondisk_log_size":80,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:11:06.812404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035115,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":395,"num_read_kb":1370,"num_write":66,"num_write_kb":1061,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":56,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-09-23T20:22:53.880117+0000","last_change":"2023-09-23T20:21:49.951490+0000","last_active":"2023-09-23T20:22:53.880117+0000","last_peered":"2023-09-23T20:22:53.880117+0000","last_clean":"2023-09-23T20:22:53.880117+0000","last_became_active":"2023-09-23T20:21:49.951079+0000","last_became_peered":"2023-09-23T20:21:49.951079+0000","last_unstale":"2023-09-23T20:22:53.880117+0000","last_undegraded":"2023-09-23T20:22:53.880117+0000","last_fullsized":"2023-09-23T20:22:53.880117+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:12:01.053013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":72,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-09-23T20:22:53.880699+0000","last_change":"2023-09-23T20:21:18.887651+0000","last_active":"2023-09-23T20:22:53.880699+0000","last_peered":"2023-09-23T20:22:53.880699+0000","last_clean":"2023-09-23T20:22:53.880699+0000","last_became_active":"2023-09-23T20:21:18.887338+0000","last_became_peered":"2023-09-23T20:21:18.887338+0000","last_unstale":"2023-09-23T20:22:53.880699+0000","last_undegraded":"2023-09-23T20:22:53.880699+0000","last_fullsized":"2023-09-23T20:22:53.880699+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:26:39.067022+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"86'102","reported_seq":621,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-09-23T20:22:58.346608+0000","last_change":"2023-09-23T20:20:14.881975+0000","last_active":"2023-09-23T20:22:58.346608+0000","last_peered":"2023-09-23T20:22:58.346608+0000","last_clean":"2023-09-23T20:22:58.346608+0000","last_became_active":"2023-09-23T20:20:14.881544+0000","last_became_peered":"2023-09-23T20:20:14.881544+0000","last_unstale":"2023-09-23T20:22:58.346608+0000","last_undegraded":"2023-09-23T20:22:58.346608+0000","last_fullsized":"2023-09-23T20:22:58.346608+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":102,"log_dups_size":0,"ondisk_log_size":102,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:34.594972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030656500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":963,"num_read_kb":1234,"num_write":60,"num_write_kb":225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":53,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-09-23T20:22:56.872154+0000","last_change":"2023-09-23T20:21:18.893457+0000","last_active":"2023-09-23T20:22:56.872154+0000","last_peered":"2023-09-23T20:22:56.872154+0000","last_clean":"2023-09-23T20:22:56.872154+0000","last_became_active":"2023-09-23T20:21:18.893175+0000","last_became_peered":"2023-09-23T20:21:18.893175+0000","last_unstale":"2023-09-23T20:22:56.872154+0000","last_undegraded":"2023-09-23T20:22:56.872154+0000","last_fullsized":"2023-09-23T20:22:56.872154+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:24:37.563758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":42,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-09-23T20:22:53.880390+0000","last_change":"2023-09-23T20:20:14.888874+0000","last_active":"2023-09-23T20:22:53.880390+0000","last_peered":"2023-09-23T20:22:53.880390+0000","last_clean":"2023-09-23T20:22:53.880390+0000","last_became_active":"2023-09-23T20:20:14.887724+0000","last_became_peered":"2023-09-23T20:20:14.887724+0000","last_unstale":"2023-09-23T20:22:53.880390+0000","last_undegraded":"2023-09-23T20:22:53.880390+0000","last_fullsized":"2023-09-23T20:22:53.880390+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:09:24.510326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.2","version":"115'422","reported_seq":434,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:23:00.337833+0000","last_change":"2023-09-23T20:22:52.871970+0000","last_active":"2023-09-23T20:23:00.337833+0000","last_peered":"2023-09-23T20:23:00.337833+0000","last_clean":"2023-09-23T20:23:00.337833+0000","last_became_active":"2023-09-23T20:22:52.871608+0000","last_became_peered":"2023-09-23T20:22:52.871608+0000","last_unstale":"2023-09-23T20:23:00.337833+0000","last_undegraded":"2023-09-23T20:23:00.337833+0000","last_fullsized":"2023-09-23T20:23:00.337833+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:22:51.846999+0000","objects_scrubbed":0,"log_size":422,"log_dups_size":0,"ondisk_log_size":422,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:24:02.901091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":27656192,"num_objects":422,"num_object_clones":0,"num_object_copies":1266,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":422,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":422,"num_write_kb":27008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":73,"reported_epoch":112,"state":"active+clean","last_fresh":"2023-09-23T20:22:52.859256+0000","last_change":"2023-09-23T20:21:49.957535+0000","last_active":"2023-09-23T20:22:52.859256+0000","last_peered":"2023-09-23T20:22:52.859256+0000","last_clean":"2023-09-23T20:22:52.859256+0000","last_became_active":"2023-09-23T20:21:49.954319+0000","last_became_peered":"2023-09-23T20:21:49.954319+0000","last_unstale":"2023-09-23T20:22:52.859256+0000","last_undegraded":"2023-09-23T20:22:52.859256+0000","last_fullsized":"2023-09-23T20:22:52.859256+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:14:10.293305+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"86'128","reported_seq":438,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-09-23T20:22:56.872601+0000","last_change":"2023-09-23T20:20:46.939256+0000","last_active":"2023-09-23T20:22:56.872601+0000","last_peered":"2023-09-23T20:22:56.872601+0000","last_clean":"2023-09-23T20:22:56.872601+0000","last_became_active":"2023-09-23T20:20:46.938756+0000","last_became_peered":"2023-09-23T20:20:46.938756+0000","last_unstale":"2023-09-23T20:22:56.872601+0000","last_undegraded":"2023-09-23T20:22:56.872601+0000","last_fullsized":"2023-09-23T20:22:56.872601+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:52:18.161436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000293202,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":579,"num_read_kb":1473,"num_write":96,"num_write_kb":1062,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":70,"reported_epoch":112,"state":"active+clean","last_fresh":"2023-09-23T20:22:52.859880+0000","last_change":"2023-09-23T20:21:49.953565+0000","last_active":"2023-09-23T20:22:52.859880+0000","last_peered":"2023-09-23T20:22:52.859880+0000","last_clean":"2023-09-23T20:22:52.859880+0000","last_became_active":"2023-09-23T20:21:49.953176+0000","last_became_peered":"2023-09-23T20:21:49.953176+0000","last_unstale":"2023-09-23T20:22:52.859880+0000","last_undegraded":"2023-09-23T20:22:52.859880+0000","last_fullsized":"2023-09-23T20:22:52.859880+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:02:35.778207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.1","version":"115'393","reported_seq":405,"reported_epoch":115,"state":"active+clean","last_fresh":"2023-09-23T20:23:00.352530+0000","last_change":"2023-09-23T20:22:52.868465+0000","last_active":"2023-09-23T20:23:00.352530+0000","last_peered":"2023-09-23T20:23:00.352530+0000","last_clean":"2023-09-23T20:23:00.352530+0000","last_became_active":"2023-09-23T20:22:52.868201+0000","last_became_peered":"2023-09-23T20:22:52.868201+0000","last_unstale":"2023-09-23T20:23:00.352530+0000","last_undegraded":"2023-09-23T20:23:00.352530+0000","last_fullsized":"2023-09-23T20:23:00.352530+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:22:51.846999+0000","objects_scrubbed":0,"log_size":393,"log_dups_size":0,"ondisk_log_size":393,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:12:09.300199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":25755648,"num_objects":393,"num_object_clones":0,"num_object_copies":1179,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":393,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":393,"num_write_kb":25152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"86'117","reported_seq":601,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-09-23T20:22:58.346663+0000","last_change":"2023-09-23T20:21:18.897269+0000","last_active":"2023-09-23T20:22:58.346663+0000","last_peered":"2023-09-23T20:22:58.346663+0000","last_clean":"2023-09-23T20:22:58.346663+0000","last_became_active":"2023-09-23T20:21:18.896834+0000","last_became_peered":"2023-09-23T20:21:18.896834+0000","last_unstale":"2023-09-23T20:22:58.346663+0000","last_undegraded":"2023-09-23T20:22:58.346663+0000","last_fullsized":"2023-09-23T20:22:58.346663+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:57:17.499025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046891400000000002,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":893,"num_read_kb":922,"num_write":82,"num_write_kb":4193,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":62,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-09-23T20:22:53.880639+0000","last_change":"2023-09-23T20:20:46.937163+0000","last_active":"2023-09-23T20:22:53.880639+0000","last_peered":"2023-09-23T20:22:53.880639+0000","last_clean":"2023-09-23T20:22:53.880639+0000","last_became_active":"2023-09-23T20:20:46.936846+0000","last_became_peered":"2023-09-23T20:20:46.936846+0000","last_unstale":"2023-09-23T20:22:53.880639+0000","last_undegraded":"2023-09-23T20:22:53.880639+0000","last_fullsized":"2023-09-23T20:22:53.880639+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:01:32.798855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":57,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-09-23T20:22:57.592558+0000","last_change":"2023-09-23T20:12:43.765908+0000","last_active":"2023-09-23T20:22:57.592558+0000","last_peered":"2023-09-23T20:22:57.592558+0000","last_clean":"2023-09-23T20:22:57.592558+0000","last_became_active":"2023-09-23T20:12:43.765558+0000","last_became_peered":"2023-09-23T20:12:43.765558+0000","last_unstale":"2023-09-23T20:22:57.592558+0000","last_undegraded":"2023-09-23T20:22:57.592558+0000","last_fullsized":"2023-09-23T20:22:57.592558+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:35:43.809898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.0","version":"0'0","reported_seq":11,"reported_epoch":112,"state":"active+clean","last_fresh":"2023-09-23T20:22:52.954576+0000","last_change":"2023-09-23T20:22:52.867218+0000","last_active":"2023-09-23T20:22:52.954576+0000","last_peered":"2023-09-23T20:22:52.954576+0000","last_clean":"2023-09-23T20:22:52.954576+0000","last_became_active":"2023-09-23T20:22:52.866992+0000","last_became_peered":"2023-09-23T20:22:52.866992+0000","last_unstale":"2023-09-23T20:22:52.954576+0000","last_undegraded":"2023-09-23T20:22:52.954576+0000","last_fullsized":"2023-09-23T20:22:52.954576+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:22:51.846999+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:12:49.188276+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":45,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-09-23T20:22:53.880396+0000","last_change":"2023-09-23T20:20:14.888484+0000","last_active":"2023-09-23T20:22:53.880396+0000","last_peered":"2023-09-23T20:22:53.880396+0000","last_clean":"2023-09-23T20:22:53.880396+0000","last_became_active":"2023-09-23T20:20:14.887219+0000","last_became_peered":"2023-09-23T20:20:14.887219+0000","last_unstale":"2023-09-23T20:22:53.880396+0000","last_undegraded":"2023-09-23T20:22:53.880396+0000","last_fullsized":"2023-09-23T20:22:53.880396+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:13:53.763649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":93,"reported_epoch":112,"state":"active+clean","last_fresh":"2023-09-23T20:22:52.862307+0000","last_change":"2023-09-23T20:21:49.953299+0000","last_active":"2023-09-23T20:22:52.862307+0000","last_peered":"2023-09-23T20:22:52.862307+0000","last_clean":"2023-09-23T20:22:52.862307+0000","last_became_active":"2023-09-23T20:21:49.952996+0000","last_became_peered":"2023-09-23T20:21:49.952996+0000","last_unstale":"2023-09-23T20:22:52.862307+0000","last_undegraded":"2023-09-23T20:22:52.862307+0000","last_fullsized":"2023-09-23T20:22:52.862307+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:46:51.746281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":76,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-09-23T20:22:56.873845+0000","last_change":"2023-09-23T20:20:46.938981+0000","last_active":"2023-09-23T20:22:56.873845+0000","last_peered":"2023-09-23T20:22:56.873845+0000","last_clean":"2023-09-23T20:22:56.873845+0000","last_became_active":"2023-09-23T20:20:46.938627+0000","last_became_peered":"2023-09-23T20:20:46.938627+0000","last_unstale":"2023-09-23T20:22:56.873845+0000","last_undegraded":"2023-09-23T20:22:56.873845+0000","last_fullsized":"2023-09-23T20:22:56.873845+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:44:43.177086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":91,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-09-23T20:22:57.592952+0000","last_change":"2023-09-23T20:21:49.957885+0000","last_active":"2023-09-23T20:22:57.592952+0000","last_peered":"2023-09-23T20:22:57.592952+0000","last_clean":"2023-09-23T20:22:57.592952+0000","last_became_active":"2023-09-23T20:21:49.957492+0000","last_became_peered":"2023-09-23T20:21:49.957492+0000","last_unstale":"2023-09-23T20:22:57.592952+0000","last_undegraded":"2023-09-23T20:22:57.592952+0000","last_fullsized":"2023-09-23T20:22:57.592952+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:24:55.137836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":59,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-09-23T20:22:56.873932+0000","last_change":"2023-09-23T20:12:43.753383+0000","last_active":"2023-09-23T20:22:56.873932+0000","last_peered":"2023-09-23T20:22:56.873932+0000","last_clean":"2023-09-23T20:22:56.873932+0000","last_became_active":"2023-09-23T20:12:43.752921+0000","last_became_peered":"2023-09-23T20:12:43.752921+0000","last_unstale":"2023-09-23T20:22:56.873932+0000","last_undegraded":"2023-09-23T20:22:56.873932+0000","last_fullsized":"2023-09-23T20:22:56.873932+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:28:26.982998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":91,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-09-23T20:22:57.592620+0000","last_change":"2023-09-23T20:21:49.953041+0000","last_active":"2023-09-23T20:22:57.592620+0000","last_peered":"2023-09-23T20:22:57.592620+0000","last_clean":"2023-09-23T20:22:57.592620+0000","last_became_active":"2023-09-23T20:21:49.952763+0000","last_became_peered":"2023-09-23T20:21:49.952763+0000","last_unstale":"2023-09-23T20:22:57.592620+0000","last_undegraded":"2023-09-23T20:22:57.592620+0000","last_fullsized":"2023-09-23T20:22:57.592620+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:58:35.321466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":81,"reported_epoch":112,"state":"active+clean","last_fresh":"2023-09-23T20:22:52.859100+0000","last_change":"2023-09-23T20:21:49.957394+0000","last_active":"2023-09-23T20:22:52.859100+0000","last_peered":"2023-09-23T20:22:52.859100+0000","last_clean":"2023-09-23T20:22:52.859100+0000","last_became_active":"2023-09-23T20:21:49.957016+0000","last_became_peered":"2023-09-23T20:21:49.957016+0000","last_unstale":"2023-09-23T20:22:52.859100+0000","last_undegraded":"2023-09-23T20:22:52.859100+0000","last_fullsized":"2023-09-23T20:22:52.859100+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:49:26.222448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":96,"reported_epoch":112,"state":"active+clean","last_fresh":"2023-09-23T20:22:52.858393+0000","last_change":"2023-09-23T20:21:49.949944+0000","last_active":"2023-09-23T20:22:52.858393+0000","last_peered":"2023-09-23T20:22:52.858393+0000","last_clean":"2023-09-23T20:22:52.858393+0000","last_became_active":"2023-09-23T20:21:49.949425+0000","last_became_peered":"2023-09-23T20:21:49.949425+0000","last_unstale":"2023-09-23T20:22:52.858393+0000","last_undegraded":"2023-09-23T20:22:52.858393+0000","last_fullsized":"2023-09-23T20:22:52.858393+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:40:56.395791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":60,"reported_epoch":113,"state":"active+clean","last_fresh":"2023-09-23T20:22:53.880582+0000","last_change":"2023-09-23T20:21:18.889521+0000","last_active":"2023-09-23T20:22:53.880582+0000","last_peered":"2023-09-23T20:22:53.880582+0000","last_clean":"2023-09-23T20:22:53.880582+0000","last_became_active":"2023-09-23T20:21:18.889268+0000","last_became_peered":"2023-09-23T20:21:18.889268+0000","last_unstale":"2023-09-23T20:22:53.880582+0000","last_undegraded":"2023-09-23T20:22:53.880582+0000","last_fullsized":"2023-09-23T20:22:53.880582+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:05:48.805778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":84,"reported_epoch":114,"state":"active+clean","last_fresh":"2023-09-23T20:22:57.592377+0000","last_change":"2023-09-23T20:21:18.893413+0000","last_active":"2023-09-23T20:22:57.592377+0000","last_peered":"2023-09-23T20:22:57.592377+0000","last_clean":"2023-09-23T20:22:57.592377+0000","last_became_active":"2023-09-23T20:21:18.893008+0000","last_became_peered":"2023-09-23T20:21:18.893008+0000","last_unstale":"2023-09-23T20:22:57.592377+0000","last_undegraded":"2023-09-23T20:22:57.592377+0000","last_fullsized":"2023-09-23T20:22:57.592377+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:50:55.264882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":5,"num_pg":16,"stat_sum":{"num_bytes":204144640,"num_objects":3115,"num_object_clones":0,"num_object_copies":9345,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3115,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3115,"num_write_kb":199360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":778240000,"data_stored":778240000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3115,"ondisk_log_size":3115,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":25181,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6082,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201422,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":129624,"kb_used_data":112656,"kb_used_omap":0,"kb_used_meta":16960,"kb_avail":93615528,"statfs":{"total":95995035648,"available":95862300672,"internally_reserved":0,"allocated":115359744,"data_stored":114822014,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":17367040},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,1,3],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:22:17 2023","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.56499999999999995,"15min":0.56999999999999995},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":0.74199999999999999,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.372},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.56200000000000006,"15min":0.56599999999999995},"min":{"1min":0.40600000000000003,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.68300000000000005,"5min":0.93799999999999994,"15min":0.93799999999999994},"last":0.61699999999999999}]},{"osd":1,"last update":"Sat Sep 23 20:22:49 2023","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":0.56599999999999995,"15min":0.56399999999999995},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":1.399,"5min":1.399,"15min":1.399},"last":0.34699999999999998},{"interface":"front","average":{"1min":0.60499999999999998,"5min":0.60999999999999999,"15min":0.61099999999999999},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":1.5569999999999999,"5min":1.5569999999999999,"15min":1.5569999999999999},"last":0.47699999999999998}]},{"osd":2,"last update":"Sat Sep 23 20:22:19 2023","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.82599999999999996,"5min":0.82599999999999996,"15min":0.82599999999999996},"last":0.72899999999999998},{"interface":"front","average":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.85999999999999999,"5min":0.85999999999999999,"15min":0.85999999999999999},"last":0.42299999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:22:49 2023","interfaces":[{"interface":"back","average":{"1min":0.65500000000000003,"5min":0.65500000000000003,"15min":0.65500000000000003},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":1.5369999999999999,"5min":1.5369999999999999,"15min":1.5369999999999999},"last":0.45500000000000002},{"interface":"front","average":{"1min":0.67400000000000004,"5min":0.67400000000000004,"15min":0.67400000000000004},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":1.492,"5min":1.492,"15min":1.492},"last":0.69799999999999995}]},{"osd":4,"last update":"Sat Sep 23 20:22:49 2023","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":0.503,"15min":0.50600000000000001},"min":{"1min":0.314,"5min":0.28799999999999998,"15min":0.26700000000000002},"max":{"1min":0.76600000000000001,"5min":0.90100000000000002,"15min":1.274},"last":0.60099999999999998},{"interface":"front","average":{"1min":0.61099999999999999,"5min":0.55000000000000004,"15min":0.53900000000000003},"min":{"1min":0.33400000000000002,"5min":0.33100000000000002,"15min":0.29699999999999999},"max":{"1min":1.131,"5min":1.131,"15min":2.024},"last":1.159}]},{"osd":5,"last update":"Sat Sep 23 20:22:49 2023","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":0.53600000000000003,"15min":0.54200000000000004},"min":{"1min":0.36199999999999999,"5min":0.33500000000000002,"15min":0.32000000000000001},"max":{"1min":0.83199999999999996,"5min":0.83199999999999996,"15min":1.337},"last":0.88500000000000001},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.58999999999999997,"15min":0.58699999999999997},"min":{"1min":0.35799999999999998,"5min":0.34399999999999997,"15min":0.31},"max":{"1min":0.82199999999999995,"5min":0.89100000000000001,"15min":1.5660000000000001},"last":0.56899999999999995}]},{"osd":6,"last update":"Sat Sep 23 20:22:49 2023","interfaces":[{"interface":"back","average":{"1min":0.63800000000000001,"5min":0.61499999999999999,"15min":0.59699999999999998},"min":{"1min":0.45700000000000002,"5min":0.379,"15min":0.34499999999999997},"max":{"1min":0.82299999999999995,"5min":0.98699999999999999,"15min":1.246},"last":0.92200000000000004},{"interface":"front","average":{"1min":0.64800000000000002,"5min":0.60899999999999999,"15min":0.58999999999999997},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.88700000000000001,"5min":0.88700000000000001,"15min":2.048},"last":1.3620000000000001}]}]},{"osd":6,"up_from":47,"seq":201863463058,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":76624,"kb_used_data":64968,"kb_used_omap":0,"kb_used_meta":11648,"kb_avail":93668528,"statfs":{"total":95995035648,"available":95916572672,"internally_reserved":0,"allocated":66527232,"data_stored":65997567,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":11927552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:22:18 2023","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":0.57799999999999996,"15min":0.57799999999999996},"min":{"1min":0.36799999999999999,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.33300000000000002},{"interface":"front","average":{"1min":0.57499999999999996,"5min":0.56499999999999995,"15min":0.56299999999999994},"min":{"1min":0.40500000000000003,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.76600000000000001,"5min":0.82299999999999995,"15min":0.82299999999999995},"last":0.94799999999999995}]},{"osd":1,"last update":"Sat Sep 23 20:22:49 2023","interfaces":[{"interface":"back","average":{"1min":0.58199999999999996,"5min":0.57599999999999996,"15min":0.57499999999999996},"min":{"1min":0.443,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.72499999999999998,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.31},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.41199999999999998,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.68899999999999995,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.81399999999999995}]},{"osd":2,"last update":"Sat Sep 23 20:22:18 2023","interfaces":[{"interface":"back","average":{"1min":0.59499999999999997,"5min":0.59499999999999997,"15min":0.59499999999999997},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.75,"5min":0.75,"15min":0.75},"last":0.76200000000000001},{"interface":"front","average":{"1min":0.60899999999999999,"5min":0.60899999999999999,"15min":0.60899999999999999},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":0.79300000000000004,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.47799999999999998}]},{"osd":3,"last update":"Sat Sep 23 20:22:49 2023","interfaces":[{"interface":"back","average":{"1min":0.61299999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"min":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":0.749,"5min":0.749,"15min":0.749},"last":0.40699999999999997},{"interface":"front","average":{"1min":0.67700000000000005,"5min":0.67700000000000005,"15min":0.67700000000000005},"min":{"1min":0.54400000000000004,"5min":0.54400000000000004,"15min":0.54400000000000004},"max":{"1min":0.93899999999999995,"5min":0.93899999999999995,"15min":0.93899999999999995},"last":0.73499999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:22:38 2023","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":0.48799999999999999,"15min":0.46899999999999997},"min":{"1min":0.34699999999999998,"5min":0.312,"15min":0.26200000000000001},"max":{"1min":0.74299999999999999,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.69799999999999995},{"interface":"front","average":{"1min":0.55600000000000005,"5min":0.51900000000000002,"15min":0.497},"min":{"1min":0.34200000000000003,"5min":0.30099999999999999,"15min":0.28499999999999998},"max":{"1min":0.81299999999999994,"5min":0.81299999999999994,"15min":0.83899999999999997},"last":0.44600000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:22:38 2023","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":0.53000000000000003,"15min":0.5},"min":{"1min":0.36399999999999999,"5min":0.32600000000000001,"15min":0.29799999999999999},"max":{"1min":0.81899999999999995,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.38600000000000001},{"interface":"front","average":{"1min":0.63400000000000001,"5min":0.57399999999999995,"15min":0.52900000000000003},"min":{"1min":0.42599999999999999,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.82299999999999995,"5min":0.85599999999999998,"15min":0.874},"last":0.85699999999999998}]},{"osd":7,"last update":"Sat Sep 23 20:22:49 2023","interfaces":[{"interface":"back","average":{"1min":0.64100000000000001,"5min":0.58299999999999996,"15min":0.55000000000000004},"min":{"1min":0.443,"5min":0.36799999999999999,"15min":0.32800000000000001},"max":{"1min":0.81100000000000005,"5min":0.85199999999999998,"15min":0.85199999999999998},"last":0.78600000000000003},{"interface":"front","average":{"1min":0.66500000000000004,"5min":0.61099999999999999,"15min":0.57599999999999996},"min":{"1min":0.46200000000000002,"5min":0.33500000000000002,"15min":0.315},"max":{"1min":0.83999999999999997,"5min":0.84799999999999998,"15min":0.90800000000000003},"last":0.79900000000000004}]}]},{"osd":1,"up_from":99,"seq":425201762333,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":112580,"kb_used_data":80892,"kb_used_omap":6,"kb_used_meta":31673,"kb_avail":93632572,"statfs":{"total":95995035648,"available":95879753728,"internally_reserved":0,"allocated":82833408,"data_stored":82258391,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6696,"internal_metadata":32433624},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:22:47 2023","interfaces":[{"interface":"back","average":{"1min":0.33700000000000002,"5min":0.36899999999999999,"15min":0.374},"min":{"1min":0.19600000000000001,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"last":0.27100000000000002},{"interface":"front","average":{"1min":0.36599999999999999,"5min":0.39300000000000002,"15min":0.39700000000000002},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":0.71299999999999997,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":0.45100000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:22:18 2023","interfaces":[{"interface":"back","average":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.628,"5min":0.628,"15min":0.628},"last":0.498},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":3.226,"5min":3.226,"15min":3.226},"last":0.35399999999999998}]},{"osd":3,"last update":"Sat Sep 23 20:22:52 2023","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":3.6059999999999999,"5min":3.6059999999999999,"15min":3.6059999999999999},"last":0.32200000000000001},{"interface":"front","average":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.66900000000000004,"5min":0.66900000000000004,"15min":0.66900000000000004},"last":0.53500000000000003}]},{"osd":4,"last update":"Sat Sep 23 20:22:47 2023","interfaces":[{"interface":"back","average":{"1min":0.624,"5min":0.61799999999999999,"15min":0.61699999999999999},"min":{"1min":0.41599999999999998,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.93300000000000005,"5min":0.93300000000000005,"15min":0.93300000000000005},"last":0.55700000000000005},{"interface":"front","average":{"1min":0.625,"5min":0.63300000000000001,"15min":0.63400000000000001},"min":{"1min":0.49299999999999999,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":0.84799999999999998,"5min":0.85199999999999998,"15min":0.85199999999999998},"last":0.377}]},{"osd":5,"last update":"Sat Sep 23 20:22:47 2023","interfaces":[{"interface":"back","average":{"1min":0.65300000000000002,"5min":0.64300000000000002,"15min":0.64100000000000001},"min":{"1min":0.45600000000000002,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":0.89500000000000002,"5min":0.89500000000000002,"15min":0.89500000000000002},"last":0.50900000000000001},{"interface":"front","average":{"1min":0.66600000000000004,"5min":0.67400000000000004,"15min":0.67500000000000004},"min":{"1min":0.46899999999999997,"5min":0.46400000000000002,"15min":0.46400000000000002},"max":{"1min":0.96999999999999997,"5min":0.96999999999999997,"15min":0.96999999999999997},"last":0.47199999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:22:47 2023","interfaces":[{"interface":"back","average":{"1min":0.629,"5min":0.66000000000000003,"15min":0.66500000000000004},"min":{"1min":0.42699999999999999,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.84799999999999998,"5min":0.92200000000000004,"15min":0.92200000000000004},"last":0.48999999999999999},{"interface":"front","average":{"1min":0.63900000000000001,"5min":0.63800000000000001,"15min":0.63800000000000001},"min":{"1min":0.47799999999999998,"5min":0.433,"15min":0.433},"max":{"1min":1,"5min":1,"15min":1},"last":0.51900000000000002}]},{"osd":7,"last update":"Sat Sep 23 20:22:47 2023","interfaces":[{"interface":"back","average":{"1min":0.63700000000000001,"5min":0.64900000000000002,"15min":0.65100000000000002},"min":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"max":{"1min":1.0309999999999999,"5min":1.0309999999999999,"15min":1.0309999999999999},"last":0.58799999999999997},{"interface":"front","average":{"1min":0.64600000000000002,"5min":0.67600000000000005,"15min":0.68100000000000005},"min":{"1min":0.48499999999999999,"5min":0.45300000000000001,"15min":0.45300000000000001},"max":{"1min":0.86499999999999999,"5min":0.86499999999999999,"15min":0.86499999999999999},"last":0.42299999999999999}]}]},{"osd":0,"up_from":94,"seq":403726925859,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":29376,"kb_used_data":1784,"kb_used_omap":7,"kb_used_meta":27576,"kb_avail":93715776,"statfs":{"total":95995035648,"available":95964954624,"internally_reserved":0,"allocated":1826816,"data_stored":1241645,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":28237841},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:22:52 2023","interfaces":[{"interface":"back","average":{"1min":0.34300000000000003,"5min":0.35499999999999998,"15min":0.35699999999999998},"min":{"1min":0.17699999999999999,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.52900000000000003,"5min":0.55400000000000005,"15min":0.55400000000000005},"last":0.38900000000000001},{"interface":"front","average":{"1min":0.39600000000000002,"5min":0.38800000000000001,"15min":0.38600000000000001},"min":{"1min":0.27400000000000002,"5min":0.253,"15min":0.253},"max":{"1min":0.63,"5min":0.63,"15min":0.63},"last":0.28899999999999998}]},{"osd":2,"last update":"Sat Sep 23 20:22:18 2023","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":4.6280000000000001,"5min":4.6280000000000001,"15min":4.6280000000000001},"last":0.63200000000000001},{"interface":"front","average":{"1min":0.496,"5min":0.496,"15min":0.496},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":1.921,"5min":1.921,"15min":1.921},"last":0.79900000000000004}]},{"osd":3,"last update":"Sat Sep 23 20:22:52 2023","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":4.6820000000000004,"5min":4.6820000000000004,"15min":4.6820000000000004},"last":0.48099999999999998},{"interface":"front","average":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"last":0.308}]},{"osd":4,"last update":"Sat Sep 23 20:22:18 2023","interfaces":[{"interface":"back","average":{"1min":0.78800000000000003,"5min":0.63600000000000001,"15min":0.60999999999999999},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":4.7450000000000001,"5min":4.7450000000000001,"15min":4.7450000000000001},"last":0.55000000000000004},{"interface":"front","average":{"1min":0.60799999999999998,"5min":0.61199999999999999,"15min":0.61299999999999999},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.92100000000000004,"5min":0.92100000000000004,"15min":0.92100000000000004},"last":0.51600000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:22:18 2023","interfaces":[{"interface":"back","average":{"1min":0.64500000000000002,"5min":0.65600000000000003,"15min":0.65800000000000003},"min":{"1min":0.47899999999999998,"5min":0.47899999999999998,"15min":0.47899999999999998},"max":{"1min":0.90400000000000003,"5min":0.90400000000000003,"15min":0.90400000000000003},"last":0.56499999999999995},{"interface":"front","average":{"1min":0.67100000000000004,"5min":0.67700000000000005,"15min":0.67800000000000005},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.90600000000000003,"5min":0.93899999999999995,"15min":0.93899999999999995},"last":0.53900000000000003}]},{"osd":6,"last update":"Sat Sep 23 20:22:18 2023","interfaces":[{"interface":"back","average":{"1min":0.68799999999999994,"5min":0.65600000000000003,"15min":0.65100000000000002},"min":{"1min":0.495,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":0.97899999999999998,"5min":0.97899999999999998,"15min":0.97899999999999998},"last":0.64600000000000002},{"interface":"front","average":{"1min":0.68799999999999994,"5min":0.66800000000000004,"15min":0.66400000000000003},"min":{"1min":0.45300000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"max":{"1min":0.89900000000000002,"5min":0.95999999999999996,"15min":0.95999999999999996},"last":0.65400000000000003}]},{"osd":7,"last update":"Sat Sep 23 20:22:18 2023","interfaces":[{"interface":"back","average":{"1min":0.66400000000000003,"5min":0.67600000000000005,"15min":0.67800000000000005},"min":{"1min":0.47199999999999998,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":0.96399999999999997,"5min":0.96399999999999997,"15min":0.96399999999999997},"last":0.504},{"interface":"front","average":{"1min":0.86399999999999999,"5min":0.70399999999999996,"15min":0.67800000000000005},"min":{"1min":0.52200000000000002,"5min":0.48499999999999999,"15min":0.48499999999999999},"max":{"1min":4.7140000000000004,"5min":4.7140000000000004,"15min":4.7140000000000004},"last":0.59999999999999998}]}]},{"osd":2,"up_from":104,"seq":446676598806,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32800,"kb_used_data":5272,"kb_used_omap":11,"kb_used_meta":27508,"kb_avail":93712352,"statfs":{"total":95995035648,"available":95961448448,"internally_reserved":0,"allocated":5398528,"data_stored":4841003,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11878,"internal_metadata":28168602},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:22:20 2023","interfaces":[{"interface":"back","average":{"1min":0.377,"5min":0.377,"15min":0.377},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":0.60599999999999998,"5min":0.60599999999999998,"15min":0.60599999999999998},"last":0.26200000000000001},{"interface":"front","average":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.69899999999999995,"5min":0.69899999999999995,"15min":0.69899999999999995},"last":0.28100000000000003}]},{"osd":1,"last update":"Sat Sep 23 20:22:20 2023","interfaces":[{"interface":"back","average":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.68000000000000005,"5min":0.68000000000000005,"15min":0.68000000000000005},"last":0.308},{"interface":"front","average":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"last":0.24399999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:22:49 2023","interfaces":[{"interface":"back","average":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.504,"5min":0.504,"15min":0.504},"last":0.39700000000000002},{"interface":"front","average":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.50600000000000001,"5min":0.50600000000000001,"15min":0.50600000000000001},"last":0.34100000000000003}]},{"osd":4,"last update":"Sat Sep 23 20:22:20 2023","interfaces":[{"interface":"back","average":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.35299999999999998},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.74099999999999999,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.42299999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:22:20 2023","interfaces":[{"interface":"back","average":{"1min":0.65700000000000003,"5min":0.65700000000000003,"15min":0.65700000000000003},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.89100000000000001,"5min":0.89100000000000001,"15min":0.89100000000000001},"last":0.53600000000000003},{"interface":"front","average":{"1min":0.65700000000000003,"5min":0.65700000000000003,"15min":0.65700000000000003},"min":{"1min":0.46600000000000003,"5min":0.46600000000000003,"15min":0.46600000000000003},"max":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.78600000000000003}]},{"osd":6,"last update":"Sat Sep 23 20:22:20 2023","interfaces":[{"interface":"back","average":{"1min":0.65700000000000003,"5min":0.65700000000000003,"15min":0.65700000000000003},"min":{"1min":0.501,"5min":0.501,"15min":0.501},"max":{"1min":0.94299999999999995,"5min":0.94299999999999995,"15min":0.94299999999999995},"last":0.64700000000000002},{"interface":"front","average":{"1min":0.67300000000000004,"5min":0.67300000000000004,"15min":0.67300000000000004},"min":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":1,"5min":1,"15min":1},"last":0.57399999999999995}]},{"osd":7,"last update":"Sat Sep 23 20:22:20 2023","interfaces":[{"interface":"back","average":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"min":{"1min":0.48399999999999999,"5min":0.48399999999999999,"15min":0.48399999999999999},"max":{"1min":0.96099999999999997,"5min":0.96099999999999997,"15min":0.96099999999999997},"last":0.60099999999999998},{"interface":"front","average":{"1min":0.65700000000000003,"5min":0.65700000000000003,"15min":0.65700000000000003},"min":{"1min":0.49299999999999999,"5min":0.49299999999999999,"15min":0.49299999999999999},"max":{"1min":0.92500000000000004,"5min":0.92500000000000004,"15min":0.92500000000000004},"last":0.58599999999999997}]}]},{"osd":3,"up_from":109,"seq":468151435280,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28684,"kb_used_data":1156,"kb_used_omap":4,"kb_used_meta":27515,"kb_avail":93716468,"statfs":{"total":95995035648,"available":95965663232,"internally_reserved":0,"allocated":1183744,"data_stored":642937,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4514,"internal_metadata":28175966},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:22:49 2023","interfaces":[{"interface":"back","average":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.80500000000000005,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":0.26900000000000002},{"interface":"front","average":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.67200000000000004,"5min":0.67200000000000004,"15min":0.67200000000000004},"last":0.379}]},{"osd":1,"last update":"Sat Sep 23 20:22:49 2023","interfaces":[{"interface":"back","average":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"min":{"1min":0.221,"5min":0.221,"15min":0.221},"max":{"1min":1.145,"5min":1.145,"15min":1.145},"last":0.33800000000000002},{"interface":"front","average":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.44500000000000001},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":1.1930000000000001,"5min":1.1930000000000001,"15min":1.1930000000000001},"last":0.39300000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:22:49 2023","interfaces":[{"interface":"back","average":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.55700000000000005,"5min":0.55700000000000005,"15min":0.55700000000000005},"last":0.32400000000000001},{"interface":"front","average":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":1.2270000000000001,"5min":1.2270000000000001,"15min":1.2270000000000001},"last":0.36499999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:22:49 2023","interfaces":[{"interface":"back","average":{"1min":0.60499999999999998,"5min":0.60499999999999998,"15min":0.60499999999999998},"min":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":0.93799999999999994,"5min":0.93799999999999994,"15min":0.93799999999999994},"last":0.58299999999999996},{"interface":"front","average":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":1.2829999999999999,"5min":1.2829999999999999,"15min":1.2829999999999999},"last":0.66300000000000003}]},{"osd":5,"last update":"Sat Sep 23 20:22:49 2023","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.97299999999999998,"5min":0.97299999999999998,"15min":0.97299999999999998},"last":0.67400000000000004},{"interface":"front","average":{"1min":0.63100000000000001,"5min":0.63100000000000001,"15min":0.63100000000000001},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":0.90800000000000003,"5min":0.90800000000000003,"15min":0.90800000000000003},"last":0.59699999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:22:49 2023","interfaces":[{"interface":"back","average":{"1min":0.63800000000000001,"5min":0.63800000000000001,"15min":0.63800000000000001},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.92700000000000005,"5min":0.92700000000000005,"15min":0.92700000000000005},"last":0.64800000000000002},{"interface":"front","average":{"1min":0.67200000000000004,"5min":0.67200000000000004,"15min":0.67200000000000004},"min":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":1.6020000000000001,"5min":1.6020000000000001,"15min":1.6020000000000001},"last":0.57199999999999995}]},{"osd":7,"last update":"Sat Sep 23 20:22:49 2023","interfaces":[{"interface":"back","average":{"1min":0.70299999999999996,"5min":0.70299999999999996,"15min":0.70299999999999996},"min":{"1min":0.439,"5min":0.439,"15min":0.439},"max":{"1min":1.6439999999999999,"5min":1.6439999999999999,"15min":1.6439999999999999},"last":0.621},{"interface":"front","average":{"1min":0.64200000000000002,"5min":0.64200000000000002,"15min":0.64200000000000002},"min":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":1.006,"5min":1.006,"15min":1.006},"last":0.71199999999999997}]}]},{"osd":4,"up_from":30,"seq":128849019034,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":200268,"kb_used_data":178564,"kb_used_omap":0,"kb_used_meta":21696,"kb_avail":93544884,"statfs":{"total":95995035648,"available":95789961216,"internally_reserved":0,"allocated":182849536,"data_stored":182323157,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":22216704},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:22:16 2023","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":0.57599999999999996,"15min":0.57099999999999995},"min":{"1min":0.41099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.85099999999999998,"5min":0.85099999999999998,"15min":0.85099999999999998},"last":0.627},{"interface":"front","average":{"1min":0.63500000000000001,"5min":0.64000000000000001,"15min":0.64100000000000001},"min":{"1min":0.47299999999999998,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":0.79500000000000004,"5min":0.84799999999999998,"15min":0.84799999999999998},"last":0.71899999999999997}]},{"osd":1,"last update":"Sat Sep 23 20:22:51 2023","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.61199999999999999,"15min":0.621},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.749,"5min":1.4219999999999999,"15min":1.4219999999999999},"last":0.63600000000000001},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.61099999999999999,"15min":0.621},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.74199999999999999,"5min":1.393,"15min":1.393},"last":0.68600000000000005}]},{"osd":2,"last update":"Sat Sep 23 20:22:19 2023","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"min":{"1min":0.49299999999999999,"5min":0.49299999999999999,"15min":0.49299999999999999},"max":{"1min":0.78400000000000003,"5min":0.78400000000000003,"15min":0.78400000000000003},"last":0.70799999999999996},{"interface":"front","average":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":0.73599999999999999,"5min":0.73599999999999999,"15min":0.73599999999999999},"last":0.67200000000000004}]},{"osd":3,"last update":"Sat Sep 23 20:22:51 2023","interfaces":[{"interface":"back","average":{"1min":0.64100000000000001,"5min":0.64100000000000001,"15min":0.64100000000000001},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.751,"5min":0.751,"15min":0.751},"last":0.65700000000000003},{"interface":"front","average":{"1min":0.66500000000000004,"5min":0.66500000000000004,"15min":0.66500000000000004},"min":{"1min":0.51000000000000001,"5min":0.51000000000000001,"15min":0.51000000000000001},"max":{"1min":0.82599999999999996,"5min":0.82599999999999996,"15min":0.82599999999999996},"last":0.59199999999999997}]},{"osd":5,"last update":"Sat Sep 23 20:22:05 2023","interfaces":[{"interface":"back","average":{"1min":0.48799999999999999,"5min":0.50600000000000001,"15min":0.45700000000000002},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.26000000000000001},"max":{"1min":0.66100000000000003,"5min":3.5270000000000001,"15min":3.5270000000000001},"last":0.47199999999999998},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.52900000000000003,"15min":0.46700000000000003},"min":{"1min":0.317,"5min":0.317,"15min":0.248},"max":{"1min":0.81299999999999994,"5min":0.86299999999999999,"15min":0.86299999999999999},"last":0.51200000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:22:27 2023","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":0.54600000000000004,"15min":0.497},"min":{"1min":0.46999999999999997,"5min":0.35599999999999998,"15min":0.27500000000000002},"max":{"1min":0.76000000000000001,"5min":0.77900000000000003,"15min":0.82499999999999996},"last":0.42099999999999999},{"interface":"front","average":{"1min":0.56999999999999995,"5min":0.56599999999999995,"15min":0.51000000000000001},"min":{"1min":0.41499999999999998,"5min":0.32100000000000001,"15min":0.314},"max":{"1min":0.73299999999999998,"5min":3.4820000000000002,"15min":3.4820000000000002},"last":0.39400000000000002}]},{"osd":7,"last update":"Sat Sep 23 20:22:41 2023","interfaces":[{"interface":"back","average":{"1min":0.66000000000000003,"5min":0.58699999999999997,"15min":0.55500000000000005},"min":{"1min":0.48999999999999999,"5min":0.38700000000000001,"15min":0.33600000000000002},"max":{"1min":0.84499999999999997,"5min":0.91600000000000004,"15min":0.91600000000000004},"last":0.45700000000000002},{"interface":"front","average":{"1min":0.63600000000000001,"5min":0.61799999999999999,"15min":0.56299999999999994},"min":{"1min":0.47799999999999998,"5min":0.39600000000000002,"15min":0.29899999999999999},"max":{"1min":0.80200000000000005,"5min":3.77,"15min":3.77},"last":0.439}]}]},{"osd":5,"up_from":39,"seq":167503724695,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":203524,"kb_used_data":182012,"kb_used_omap":0,"kb_used_meta":21504,"kb_avail":93541628,"statfs":{"total":95995035648,"available":95786627072,"internally_reserved":0,"allocated":186380288,"data_stored":185858187,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":22020096},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:22:18 2023","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":0.59599999999999997,"15min":0.59299999999999997},"min":{"1min":0.38900000000000001,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.79800000000000004,"5min":0.83499999999999996,"15min":0.83499999999999996},"last":0.57299999999999995},{"interface":"front","average":{"1min":0.57999999999999996,"5min":0.59599999999999997,"15min":0.59899999999999998},"min":{"1min":0.44,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.82299999999999995,"5min":0.82399999999999995,"15min":0.82399999999999995},"last":0.53300000000000003}]},{"osd":1,"last update":"Sat Sep 23 20:22:50 2023","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.54600000000000004,"15min":0.54500000000000004},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":0.72199999999999998,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.52400000000000002},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"min":{"1min":0.46000000000000002,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.69199999999999995,"5min":0.84999999999999998,"15min":0.84999999999999998},"last":0.56000000000000005}]},{"osd":2,"last update":"Sat Sep 23 20:22:18 2023","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.70899999999999996,"5min":0.70899999999999996,"15min":0.70899999999999996},"last":0.48799999999999999},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.72899999999999998,"5min":0.72899999999999998,"15min":0.72899999999999998},"last":0.59399999999999997}]},{"osd":3,"last update":"Sat Sep 23 20:22:50 2023","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.72799999999999998,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":0.498},{"interface":"front","average":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.78100000000000003,"5min":0.78100000000000003,"15min":0.78100000000000003},"last":0.60499999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:22:13 2023","interfaces":[{"interface":"back","average":{"1min":0.498,"5min":0.48199999999999998,"15min":0.439},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.26400000000000001},"max":{"1min":0.68200000000000005,"5min":0.76100000000000001,"15min":1.0620000000000001},"last":0.39400000000000002},{"interface":"front","average":{"1min":0.58799999999999997,"5min":0.52700000000000002,"15min":0.47999999999999998},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.28799999999999998},"max":{"1min":0.82299999999999995,"5min":0.82299999999999995,"15min":1.4450000000000001},"last":0.378}]},{"osd":6,"last update":"Sat Sep 23 20:22:27 2023","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":0.52000000000000002,"15min":0.49399999999999999},"min":{"1min":0.44500000000000001,"5min":0.32500000000000001,"15min":0.29799999999999999},"max":{"1min":0.80000000000000004,"5min":0.80000000000000004,"15min":1.151},"last":0.33900000000000002},{"interface":"front","average":{"1min":0.65700000000000003,"5min":0.61399999999999999,"15min":0.56299999999999994},"min":{"1min":0.375,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.84899999999999998,"5min":0.84899999999999998,"15min":0.84899999999999998},"last":0.45400000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:22:50 2023","interfaces":[{"interface":"back","average":{"1min":0.625,"5min":0.57799999999999996,"15min":0.54400000000000004},"min":{"1min":0.42499999999999999,"5min":0.372,"15min":0.33800000000000002},"max":{"1min":0.873,"5min":0.873,"15min":0.98199999999999998},"last":0.42699999999999999},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.58199999999999996,"15min":0.54900000000000004},"min":{"1min":0.38700000000000001,"5min":0.315,"15min":0.315},"max":{"1min":0.81399999999999995,"5min":0.81399999999999995,"15min":1.0489999999999999},"last":0.41499999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4722,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1360,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6696,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7156,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3154,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":98762752,"data_stored":98762752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":210960384,"data_stored":210960384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":236847104,"data_stored":236847104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":106102784,"data_stored":106102784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":125566976,"data_stored":125566976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:23:02.920 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-09-23T20:23:02.920 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd pool set unique_pool_0 pg_num 26 2023-09-23T20:23:02.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:02 smithi149 ceph-mon[170394]: pgmap v239: 89 pgs: 89 active+clean; 199 MiB data, 794 MiB used, 714 GiB / 715 GiB avail; 26 MiB/s wr, 411 op/s 2023-09-23T20:23:02.967 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:23:02.972 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:02.972 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:23:03.073 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:23:03.118 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:03.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:02 smithi018 ceph-mon[184144]: pgmap v239: 89 pgs: 89 active+clean; 199 MiB data, 794 MiB used, 714 GiB / 715 GiB avail; 26 MiB/s wr, 411 op/s 2023-09-23T20:23:03.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:02 smithi018 ceph-mon[179647]: pgmap v239: 89 pgs: 89 active+clean; 199 MiB data, 794 MiB used, 714 GiB / 715 GiB avail; 26 MiB/s wr, 411 op/s 2023-09-23T20:23:03.181 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:23:03.185 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:03.185 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:23:03.287 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:23:03.318 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:23:03.327 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:03.377 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:23:03.383 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:03.384 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:23:03.453 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 5 16 14316 14300 178.721 169.25 0.00528583 0.00558973 2023-09-23T20:23:03.484 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:23:03.541 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:03.593 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:23:03.707 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:23:03.721 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:03.722 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:23:03.801 INFO:teuthology.orchestra.run.smithi018.stdout:ERROR: (22) Invalid argument 2023-09-23T20:23:03.801 INFO:teuthology.orchestra.run.smithi018.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:23:03.824 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:23:03.937 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:04.018 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:03 smithi018 ceph-mon[179647]: from='client.44702 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:23:04.019 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:03 smithi018 ceph-mon[184144]: from='client.44702 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:23:04.218 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:03 smithi149 ceph-mon[170394]: from='client.44702 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:23:04.355 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:23:04.367 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:04.368 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:23:04.453 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 6 16 16775 16759 174.526 153.688 0.00353789 0.0057234 2023-09-23T20:23:04.469 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:23:04.533 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:04.741 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:23:04.746 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:04.747 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:23:04.756 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:23:04.782 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T20:23:04.848 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:23:04.893 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:04.943 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:23:04.947 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:04.948 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:23:05.049 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:23:05.115 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:04 smithi018 ceph-mon[184144]: pgmap v240: 89 pgs: 89 active+clean; 550 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-09-23T20:23:05.115 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:04 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:23:05.122 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:04 smithi018 ceph-mon[179647]: pgmap v240: 89 pgs: 89 active+clean; 550 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-09-23T20:23:05.122 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:04 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:23:05.122 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:23:04 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:23:04] "GET /metrics HTTP/1.1" 200 36385 "" "Prometheus/2.43.0" 2023-09-23T20:23:05.122 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:05.196 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:23:05.207 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:05.207 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:23:05.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:04 smithi149 ceph-mon[170394]: pgmap v240: 89 pgs: 89 active+clean; 550 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-09-23T20:23:05.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:04 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:23:05.243 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:23:05.308 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:23:05.387 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:05.453 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 7 16 18987 18971 169.343 138.25 0.00568433 0.00589948 2023-09-23T20:23:05.929 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:23:05.931 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:05.931 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:23:05.943 INFO:teuthology.orchestra.run.smithi018.stderr:noscrub is unset 2023-09-23T20:23:06.032 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:23:06.079 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:06.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:05 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/264816245' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:23:06.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:05 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/264816245' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:23:06.145 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:23:06.151 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:06.151 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:23:06.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:05 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/264816245' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:23:06.253 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:23:06.298 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:06.350 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:23:06.354 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:06.355 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:23:06.455 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 8 15 21427 21412 167.21 152.562 0.003237 0.00597433 2023-09-23T20:23:06.455 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:23:06.500 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:06.558 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:23:06.564 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:06.565 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:23:06.665 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:23:06.712 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:06.759 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:23:06.763 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:06.764 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:23:06.864 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:23:06.933 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:07.069 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:23:07.073 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:07.074 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:23:07.123 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:23:07.175 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:23:07.213 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:06 smithi149 ceph-mon[170394]: pgmap v241: 89 pgs: 89 active+clean; 937 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 109 MiB/s wr, 1.74k op/s 2023-09-23T20:23:07.213 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:06 smithi149 ceph-mon[170394]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:23:07.213 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:06 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/264816245' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:23:07.213 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:06 smithi149 ceph-mon[170394]: osdmap e116: 8 total, 8 up, 8 in 2023-09-23T20:23:07.213 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:06 smithi149 ceph-mon[170394]: failed to encode map e116 with expected crc 2023-09-23T20:23:07.213 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:06 smithi149 ceph-mon[170394]: failed to encode map e116 with expected crc 2023-09-23T20:23:07.213 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:06 smithi149 ceph-mon[170394]: failed to encode map e116 with expected crc 2023-09-23T20:23:07.213 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:06 smithi149 ceph-mon[170394]: failed to encode map e116 with expected crc 2023-09-23T20:23:07.218 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:07.275 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:23:07.279 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:07.279 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:23:07.380 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:23:07.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:06 smithi018 ceph-mon[179647]: pgmap v241: 89 pgs: 89 active+clean; 937 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 109 MiB/s wr, 1.74k op/s 2023-09-23T20:23:07.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:06 smithi018 ceph-mon[179647]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:23:07.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:06 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/264816245' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:23:07.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:06 smithi018 ceph-mon[179647]: osdmap e116: 8 total, 8 up, 8 in 2023-09-23T20:23:07.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:06 smithi018 ceph-mon[179647]: failed to encode map e116 with expected crc 2023-09-23T20:23:07.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:06 smithi018 ceph-mon[179647]: failed to encode map e116 with expected crc 2023-09-23T20:23:07.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:06 smithi018 ceph-mon[179647]: failed to encode map e116 with expected crc 2023-09-23T20:23:07.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:06 smithi018 ceph-mon[179647]: failed to encode map e116 with expected crc 2023-09-23T20:23:07.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:06 smithi018 ceph-mon[184144]: pgmap v241: 89 pgs: 89 active+clean; 937 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 109 MiB/s wr, 1.74k op/s 2023-09-23T20:23:07.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:06 smithi018 ceph-mon[184144]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:23:07.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:06 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/264816245' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:23:07.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:06 smithi018 ceph-mon[184144]: osdmap e116: 8 total, 8 up, 8 in 2023-09-23T20:23:07.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:06 smithi018 ceph-mon[184144]: failed to encode map e116 with expected crc 2023-09-23T20:23:07.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:06 smithi018 ceph-mon[184144]: failed to encode map e116 with expected crc 2023-09-23T20:23:07.394 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:06 smithi018 ceph-mon[184144]: failed to encode map e116 with expected crc 2023-09-23T20:23:07.394 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:06 smithi018 ceph-mon[184144]: failed to encode map e116 with expected crc 2023-09-23T20:23:07.435 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:07.455 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 9 15 23627 23612 163.909 137.5 0.0102035 0.00609513 2023-09-23T20:23:07.522 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:23:07.526 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:07.527 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:23:07.627 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:23:07.685 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:07.955 INFO:teuthology.orchestra.run.smithi018.stderr:set pool 5 pg_num to 26 2023-09-23T20:23:08.209 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:23:08.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:07 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3606018094' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T20:23:08.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:07 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T20:23:08.246 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:23:08.286 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:23:08.329 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:23:08.366 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:23:08.390 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:07 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3606018094' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T20:23:08.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:07 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T20:23:08.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:07 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3606018094' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T20:23:08.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:07 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T20:23:08.413 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:23:08.454 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:23:08.455 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 10 16 25766 25750 160.88 133.625 0.00772455 0.00620969 2023-09-23T20:23:08.462 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:23:08.467 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:08.468 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:23:08.491 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:23:08.499 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:23:08.499 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:23:08.499 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:23:08.499 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:23:08.499 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:23:08.499 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:23:08.499 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:23:08.499 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:23:08.569 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:23:08.622 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:08.686 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:23:08.691 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:08.692 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:23:08.759 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T20:23:08.792 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:23:08.832 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:08.880 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:23:08.885 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:08.886 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:23:08.987 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:23:09.067 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:09.108 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:23:09.114 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:09.115 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:23:09.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:08 smithi018 ceph-mon[179647]: pgmap v243: 89 pgs: 89 active+clean; 1.2 GiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 138 MiB/s wr, 2.20k op/s 2023-09-23T20:23:09.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:08 smithi018 ceph-mon[179647]: failed to encode map e117 with expected crc 2023-09-23T20:23:09.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:08 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "26"}]': finished 2023-09-23T20:23:09.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:08 smithi018 ceph-mon[179647]: osdmap e117: 8 total, 8 up, 8 in 2023-09-23T20:23:09.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:08 smithi018 ceph-mon[179647]: failed to encode map e117 with expected crc 2023-09-23T20:23:09.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:08 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2154581970' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:23:09.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:08 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:23:09.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:08 smithi018 ceph-mon[184144]: pgmap v243: 89 pgs: 89 active+clean; 1.2 GiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 138 MiB/s wr, 2.20k op/s 2023-09-23T20:23:09.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:08 smithi018 ceph-mon[184144]: failed to encode map e117 with expected crc 2023-09-23T20:23:09.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:08 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "26"}]': finished 2023-09-23T20:23:09.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:08 smithi018 ceph-mon[184144]: osdmap e117: 8 total, 8 up, 8 in 2023-09-23T20:23:09.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:08 smithi018 ceph-mon[184144]: failed to encode map e117 with expected crc 2023-09-23T20:23:09.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:08 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2154581970' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:23:09.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:08 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:23:09.215 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:23:09.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:08 smithi149 ceph-mon[170394]: pgmap v243: 89 pgs: 89 active+clean; 1.2 GiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 138 MiB/s wr, 2.20k op/s 2023-09-23T20:23:09.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:08 smithi149 ceph-mon[170394]: failed to encode map e117 with expected crc 2023-09-23T20:23:09.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:08 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "26"}]': finished 2023-09-23T20:23:09.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:08 smithi149 ceph-mon[170394]: osdmap e117: 8 total, 8 up, 8 in 2023-09-23T20:23:09.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:08 smithi149 ceph-mon[170394]: failed to encode map e117 with expected crc 2023-09-23T20:23:09.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:08 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2154581970' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:23:09.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:08 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:23:09.267 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:09.331 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:23:09.338 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:09.338 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:23:09.439 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:23:09.487 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 11 16 27781 27765 157.703 125.938 0.00482067 0.00633589 2023-09-23T20:23:09.537 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:10.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:09 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-09-23T20:23:10.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:09 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-09-23T20:23:10.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:09 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-09-23T20:23:10.455 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 12 16 28913 28897 150.458 70.75 0.0479345 0.00637478 2023-09-23T20:23:11.080 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:23:11.084 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:11.085 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:23:11.185 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:23:11.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:10 smithi149 ceph-mon[170394]: pgmap v245: 89 pgs: 89 active+clean; 1.3 GiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 139 MiB/s wr, 2.23k op/s 2023-09-23T20:23:11.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:10 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "26"}]': finished 2023-09-23T20:23:11.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:10 smithi149 ceph-mon[170394]: osdmap e118: 8 total, 8 up, 8 in 2023-09-23T20:23:11.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:10 smithi149 ceph-mon[170394]: failed to encode map e118 with expected crc 2023-09-23T20:23:11.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:10 smithi149 ceph-mon[170394]: failed to encode map e118 with expected crc 2023-09-23T20:23:11.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:10 smithi149 ceph-mon[170394]: failed to encode map e118 with expected crc 2023-09-23T20:23:11.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:10 smithi149 ceph-mon[170394]: failed to encode map e118 with expected crc 2023-09-23T20:23:11.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:10 smithi149 ceph-mon[170394]: failed to encode map e118 with expected crc 2023-09-23T20:23:11.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:10 smithi149 ceph-mon[170394]: failed to encode map e118 with expected crc 2023-09-23T20:23:11.228 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:10 smithi018 ceph-mon[179647]: pgmap v245: 89 pgs: 89 active+clean; 1.3 GiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 139 MiB/s wr, 2.23k op/s 2023-09-23T20:23:11.229 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:10 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "26"}]': finished 2023-09-23T20:23:11.229 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:10 smithi018 ceph-mon[179647]: osdmap e118: 8 total, 8 up, 8 in 2023-09-23T20:23:11.229 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:10 smithi018 ceph-mon[179647]: failed to encode map e118 with expected crc 2023-09-23T20:23:11.229 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:10 smithi018 ceph-mon[179647]: failed to encode map e118 with expected crc 2023-09-23T20:23:11.229 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:10 smithi018 ceph-mon[179647]: failed to encode map e118 with expected crc 2023-09-23T20:23:11.229 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:10 smithi018 ceph-mon[179647]: failed to encode map e118 with expected crc 2023-09-23T20:23:11.229 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:10 smithi018 ceph-mon[179647]: failed to encode map e118 with expected crc 2023-09-23T20:23:11.229 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:10 smithi018 ceph-mon[179647]: failed to encode map e118 with expected crc 2023-09-23T20:23:11.229 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:10 smithi018 ceph-mon[184144]: pgmap v245: 89 pgs: 89 active+clean; 1.3 GiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 139 MiB/s wr, 2.23k op/s 2023-09-23T20:23:11.229 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:10 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "26"}]': finished 2023-09-23T20:23:11.230 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:10 smithi018 ceph-mon[184144]: osdmap e118: 8 total, 8 up, 8 in 2023-09-23T20:23:11.230 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:10 smithi018 ceph-mon[184144]: failed to encode map e118 with expected crc 2023-09-23T20:23:11.230 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:10 smithi018 ceph-mon[184144]: failed to encode map e118 with expected crc 2023-09-23T20:23:11.230 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:10 smithi018 ceph-mon[184144]: failed to encode map e118 with expected crc 2023-09-23T20:23:11.230 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:10 smithi018 ceph-mon[184144]: failed to encode map e118 with expected crc 2023-09-23T20:23:11.230 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:10 smithi018 ceph-mon[184144]: failed to encode map e118 with expected crc 2023-09-23T20:23:11.230 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:10 smithi018 ceph-mon[184144]: failed to encode map e118 with expected crc 2023-09-23T20:23:11.233 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:11.455 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 13 16 29824 29808 143.265 56.9375 0.00727306 0.00697497 2023-09-23T20:23:12.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:11 smithi018 ceph-mon[179647]: failed to encode map e119 with expected crc 2023-09-23T20:23:12.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:11 smithi018 ceph-mon[179647]: failed to encode map e119 with expected crc 2023-09-23T20:23:12.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:11 smithi018 ceph-mon[179647]: osdmap e119: 8 total, 8 up, 8 in 2023-09-23T20:23:12.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:11 smithi018 ceph-mon[179647]: failed to encode map e119 with expected crc 2023-09-23T20:23:12.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:11 smithi018 ceph-mon[179647]: failed to encode map e119 with expected crc 2023-09-23T20:23:12.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:11 smithi018 ceph-mon[179647]: failed to encode map e119 with expected crc 2023-09-23T20:23:12.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:11 smithi018 ceph-mon[179647]: failed to encode map e119 with expected crc 2023-09-23T20:23:12.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:11 smithi018 ceph-mon[179647]: failed to encode map e119 with expected crc 2023-09-23T20:23:12.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:11 smithi018 ceph-mon[179647]: failed to encode map e119 with expected crc 2023-09-23T20:23:12.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:11 smithi018 ceph-mon[179647]: 5.1 scrub starts 2023-09-23T20:23:12.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:11 smithi018 ceph-mon[179647]: 5.5 scrub starts 2023-09-23T20:23:12.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:11 smithi018 ceph-mon[179647]: 5.1 scrub ok 2023-09-23T20:23:12.394 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:11 smithi018 ceph-mon[184144]: failed to encode map e119 with expected crc 2023-09-23T20:23:12.394 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:11 smithi018 ceph-mon[184144]: failed to encode map e119 with expected crc 2023-09-23T20:23:12.394 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:11 smithi018 ceph-mon[184144]: osdmap e119: 8 total, 8 up, 8 in 2023-09-23T20:23:12.394 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:11 smithi018 ceph-mon[184144]: failed to encode map e119 with expected crc 2023-09-23T20:23:12.395 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:11 smithi018 ceph-mon[184144]: failed to encode map e119 with expected crc 2023-09-23T20:23:12.395 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:11 smithi018 ceph-mon[184144]: failed to encode map e119 with expected crc 2023-09-23T20:23:12.395 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:11 smithi018 ceph-mon[184144]: failed to encode map e119 with expected crc 2023-09-23T20:23:12.395 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:11 smithi018 ceph-mon[184144]: failed to encode map e119 with expected crc 2023-09-23T20:23:12.395 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:11 smithi018 ceph-mon[184144]: failed to encode map e119 with expected crc 2023-09-23T20:23:12.395 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:11 smithi018 ceph-mon[184144]: 5.1 scrub starts 2023-09-23T20:23:12.395 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:11 smithi018 ceph-mon[184144]: 5.5 scrub starts 2023-09-23T20:23:12.395 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:11 smithi018 ceph-mon[184144]: 5.1 scrub ok 2023-09-23T20:23:12.455 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 14 16 31984 31968 142.674 135 0.00468142 0.00700344 2023-09-23T20:23:12.470 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:11 smithi149 ceph-mon[170394]: failed to encode map e119 with expected crc 2023-09-23T20:23:12.470 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:11 smithi149 ceph-mon[170394]: failed to encode map e119 with expected crc 2023-09-23T20:23:12.470 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:11 smithi149 ceph-mon[170394]: osdmap e119: 8 total, 8 up, 8 in 2023-09-23T20:23:12.470 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:11 smithi149 ceph-mon[170394]: failed to encode map e119 with expected crc 2023-09-23T20:23:12.470 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:11 smithi149 ceph-mon[170394]: failed to encode map e119 with expected crc 2023-09-23T20:23:12.470 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:11 smithi149 ceph-mon[170394]: failed to encode map e119 with expected crc 2023-09-23T20:23:12.470 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:11 smithi149 ceph-mon[170394]: failed to encode map e119 with expected crc 2023-09-23T20:23:12.470 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:11 smithi149 ceph-mon[170394]: failed to encode map e119 with expected crc 2023-09-23T20:23:12.470 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:11 smithi149 ceph-mon[170394]: failed to encode map e119 with expected crc 2023-09-23T20:23:12.470 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:11 smithi149 ceph-mon[170394]: 5.1 scrub starts 2023-09-23T20:23:12.470 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:11 smithi149 ceph-mon[170394]: 5.5 scrub starts 2023-09-23T20:23:12.471 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:11 smithi149 ceph-mon[170394]: 5.1 scrub ok 2023-09-23T20:23:12.728 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:23:12.733 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:12.734 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:23:12.834 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:23:12.897 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:12.968 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:23:12.982 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:12.983 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:23:13.084 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:23:13.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:12 smithi018 ceph-mon[184144]: pgmap v248: 99 pgs: 5 peering, 10 unknown, 84 active+clean; 1.3 GiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 75 MiB/s wr, 1.19k op/s 2023-09-23T20:23:13.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:12 smithi018 ceph-mon[184144]: 5.6 scrub starts 2023-09-23T20:23:13.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:12 smithi018 ceph-mon[184144]: 5.11 scrub starts 2023-09-23T20:23:13.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:12 smithi018 ceph-mon[184144]: 5.11 scrub ok 2023-09-23T20:23:13.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:12 smithi018 ceph-mon[179647]: pgmap v248: 99 pgs: 5 peering, 10 unknown, 84 active+clean; 1.3 GiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 75 MiB/s wr, 1.19k op/s 2023-09-23T20:23:13.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:12 smithi018 ceph-mon[179647]: 5.6 scrub starts 2023-09-23T20:23:13.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:12 smithi018 ceph-mon[179647]: 5.11 scrub starts 2023-09-23T20:23:13.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:12 smithi018 ceph-mon[179647]: 5.11 scrub ok 2023-09-23T20:23:13.146 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:12 smithi149 ceph-mon[170394]: pgmap v248: 99 pgs: 5 peering, 10 unknown, 84 active+clean; 1.3 GiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 75 MiB/s wr, 1.19k op/s 2023-09-23T20:23:13.146 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:12 smithi149 ceph-mon[170394]: 5.6 scrub starts 2023-09-23T20:23:13.146 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:12 smithi149 ceph-mon[170394]: 5.11 scrub starts 2023-09-23T20:23:13.146 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:12 smithi149 ceph-mon[170394]: 5.11 scrub ok 2023-09-23T20:23:13.153 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:13.223 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:23:13.228 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:13.229 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:23:13.329 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:23:13.380 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:13.438 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:23:13.443 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:13.444 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:23:13.455 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 15 16 34215 34199 142.457 139.438 0.00238782 0.00700868 2023-09-23T20:23:13.545 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:23:13.599 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:13.807 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:23:13.814 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:13.814 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:23:13.915 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:23:13.973 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:14.032 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:23:14.037 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:14.037 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:23:14.138 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:23:14.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:13 smithi018 ceph-mon[179647]: 5.6 scrub ok 2023-09-23T20:23:14.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:13 smithi018 ceph-mon[179647]: 5.2 scrub starts 2023-09-23T20:23:14.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:13 smithi018 ceph-mon[179647]: 5.2 scrub ok 2023-09-23T20:23:14.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:13 smithi018 ceph-mon[179647]: 5.4 scrub starts 2023-09-23T20:23:14.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:13 smithi018 ceph-mon[179647]: 5.0 scrub starts 2023-09-23T20:23:14.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:13 smithi018 ceph-mon[184144]: 5.6 scrub ok 2023-09-23T20:23:14.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:13 smithi018 ceph-mon[184144]: 5.2 scrub starts 2023-09-23T20:23:14.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:13 smithi018 ceph-mon[184144]: 5.2 scrub ok 2023-09-23T20:23:14.144 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:13 smithi018 ceph-mon[184144]: 5.4 scrub starts 2023-09-23T20:23:14.144 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:13 smithi018 ceph-mon[184144]: 5.0 scrub starts 2023-09-23T20:23:14.187 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:14.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:13 smithi149 ceph-mon[170394]: 5.6 scrub ok 2023-09-23T20:23:14.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:13 smithi149 ceph-mon[170394]: 5.2 scrub starts 2023-09-23T20:23:14.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:13 smithi149 ceph-mon[170394]: 5.2 scrub ok 2023-09-23T20:23:14.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:13 smithi149 ceph-mon[170394]: 5.4 scrub starts 2023-09-23T20:23:14.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:13 smithi149 ceph-mon[170394]: 5.0 scrub starts 2023-09-23T20:23:14.248 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2023-09-23T20:23:14.248 INFO:teuthology.orchestra.run.smithi018.stdout: "size": 20, 2023-09-23T20:23:14.249 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 600, 2023-09-23T20:23:14.249 INFO:teuthology.orchestra.run.smithi018.stdout: "ops": [ 2023-09-23T20:23:14.249 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.249 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.34791.0:1394 5.c 5:39266cf1:::benchmark_data_smithi018_228451_object1393:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e114)", 2023-09-23T20:23:14.249 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:22:58.884546+0000", 2023-09-23T20:23:14.249 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 15.359305055, 2023-09-23T20:23:14.249 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.018868965000000001, 2023-09-23T20:23:14.249 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:23:14.249 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:23:14.249 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:23:14.249 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.34791", 2023-09-23T20:23:14.249 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:23:14.250 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 1394 2023-09-23T20:23:14.250 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.250 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:23:14.250 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.250 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:23:14.250 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:22:58.884546+0000", 2023-09-23T20:23:14.250 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:23:14.250 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.250 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.250 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:23:14.250 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:22:58.884546+0000", 2023-09-23T20:23:14.251 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:23:14.251 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.251 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.251 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:23:14.251 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:22:58.884544+0000", 2023-09-23T20:23:14.251 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:23:14.251 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.251 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.251 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:23:14.251 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:22:58.884568+0000", 2023-09-23T20:23:14.251 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.3363999999999999e-05 2023-09-23T20:23:14.251 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.252 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.252 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:23:14.252 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:22:58.884584+0000", 2023-09-23T20:23:14.252 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.6606999999999999e-05 2023-09-23T20:23:14.252 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.252 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.252 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:23:14.252 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:22:58.884599+0000", 2023-09-23T20:23:14.252 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.5228000000000001e-05 2023-09-23T20:23:14.252 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.252 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.252 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:23:14.253 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:22:58.884652+0000", 2023-09-23T20:23:14.253 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.2537999999999997e-05 2023-09-23T20:23:14.253 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.253 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.253 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:23:14.253 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:22:58.884805+0000", 2023-09-23T20:23:14.253 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000153559 2023-09-23T20:23:14.253 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.253 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.253 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 0,4", 2023-09-23T20:23:14.253 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:22:58.884958+0000", 2023-09-23T20:23:14.254 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00015283000000000001 2023-09-23T20:23:14.254 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.254 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.254 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:23:14.254 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:22:58.886015+0000", 2023-09-23T20:23:14.254 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0010571389999999999 2023-09-23T20:23:14.254 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.254 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.254 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:23:14.254 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:22:58.886591+0000", 2023-09-23T20:23:14.254 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00057508899999999998 2023-09-23T20:23:14.254 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.255 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.255 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:23:14.255 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:22:58.903322+0000", 2023-09-23T20:23:14.255 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.016731204999999999 2023-09-23T20:23:14.255 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.255 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.255 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:23:14.255 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:22:58.903372+0000", 2023-09-23T20:23:14.255 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.0623e-05 2023-09-23T20:23:14.255 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.255 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.255 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:23:14.255 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:22:58.903415+0000", 2023-09-23T20:23:14.256 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.2144999999999997e-05 2023-09-23T20:23:14.256 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:23:14.256 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:23:14.256 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:23:14.256 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.256 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.256 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.34791.0:4703 5.c 5:3d505036:::benchmark_data_smithi018_228451_object4702:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e115)", 2023-09-23T20:23:14.256 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:22:59.850947+0000", 2023-09-23T20:23:14.256 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 14.392903548, 2023-09-23T20:23:14.256 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.017444899, 2023-09-23T20:23:14.256 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:23:14.256 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:23:14.257 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:23:14.257 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.34791", 2023-09-23T20:23:14.257 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:23:14.257 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 4703 2023-09-23T20:23:14.257 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.257 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:23:14.257 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.257 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:23:14.257 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:22:59.850947+0000", 2023-09-23T20:23:14.257 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:23:14.257 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.257 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.257 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:23:14.258 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:22:59.850947+0000", 2023-09-23T20:23:14.258 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:23:14.258 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.258 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.258 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:23:14.258 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:22:59.850945+0000", 2023-09-23T20:23:14.258 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:23:14.258 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.258 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.258 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:23:14.258 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:22:59.850974+0000", 2023-09-23T20:23:14.258 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.9074e-05 2023-09-23T20:23:14.259 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.259 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.259 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:23:14.259 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:22:59.850993+0000", 2023-09-23T20:23:14.259 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.9091000000000001e-05 2023-09-23T20:23:14.259 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.259 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.259 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:23:14.259 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:22:59.851004+0000", 2023-09-23T20:23:14.259 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0834e-05 2023-09-23T20:23:14.259 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.259 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.259 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:23:14.260 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:22:59.851062+0000", 2023-09-23T20:23:14.260 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.7797999999999998e-05 2023-09-23T20:23:14.260 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.260 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.260 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:23:14.260 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:22:59.851229+0000", 2023-09-23T20:23:14.260 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00016716499999999999 2023-09-23T20:23:14.260 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.260 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.260 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 0,4", 2023-09-23T20:23:14.260 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:22:59.851384+0000", 2023-09-23T20:23:14.260 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00015496800000000001 2023-09-23T20:23:14.261 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.261 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.261 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:23:14.261 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:22:59.852538+0000", 2023-09-23T20:23:14.261 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.001153369 2023-09-23T20:23:14.261 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.261 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.261 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:23:14.261 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:22:59.854216+0000", 2023-09-23T20:23:14.261 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0016787029999999999 2023-09-23T20:23:14.261 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.261 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.262 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:23:14.262 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:22:59.868278+0000", 2023-09-23T20:23:14.262 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.014062204999999999 2023-09-23T20:23:14.262 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.262 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.262 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:23:14.262 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:22:59.868344+0000", 2023-09-23T20:23:14.262 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.5454e-05 2023-09-23T20:23:14.262 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.262 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.262 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:23:14.262 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:22:59.868392+0000", 2023-09-23T20:23:14.263 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.8018e-05 2023-09-23T20:23:14.263 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:23:14.263 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:23:14.263 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:23:14.263 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.263 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.263 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.34791.0:26134 5.c 5:3431952c:::benchmark_data_smithi018_228451_object26133:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2023-09-23T20:23:14.263 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:23:08.630850+0000", 2023-09-23T20:23:14.263 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 5.6130005760000001, 2023-09-23T20:23:14.263 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.021078897999999999, 2023-09-23T20:23:14.263 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:23:14.263 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:23:14.263 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:23:14.264 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.34791", 2023-09-23T20:23:14.264 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:23:14.264 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 26134 2023-09-23T20:23:14.264 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.264 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:23:14.264 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.264 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:23:14.264 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:08.630850+0000", 2023-09-23T20:23:14.264 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:23:14.264 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.264 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.264 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:23:14.265 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:08.630850+0000", 2023-09-23T20:23:14.265 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:23:14.265 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.265 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.265 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:23:14.265 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:08.630848+0000", 2023-09-23T20:23:14.265 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:23:14.265 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.265 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.265 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:23:14.266 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:08.630874+0000", 2023-09-23T20:23:14.266 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.5055999999999999e-05 2023-09-23T20:23:14.266 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.266 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.266 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:23:14.266 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:08.630887+0000", 2023-09-23T20:23:14.266 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3159000000000001e-05 2023-09-23T20:23:14.266 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.266 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.266 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:23:14.266 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:08.630895+0000", 2023-09-23T20:23:14.266 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.2679999999999994e-06 2023-09-23T20:23:14.267 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.267 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.267 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:23:14.267 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:08.630942+0000", 2023-09-23T20:23:14.267 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.7345999999999997e-05 2023-09-23T20:23:14.267 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.267 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.267 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:23:14.267 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:08.631069+0000", 2023-09-23T20:23:14.267 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00012715200000000001 2023-09-23T20:23:14.267 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.267 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.267 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 0,4", 2023-09-23T20:23:14.268 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:08.631207+0000", 2023-09-23T20:23:14.268 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00013730700000000001 2023-09-23T20:23:14.268 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.268 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.268 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:23:14.268 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:08.634838+0000", 2023-09-23T20:23:14.268 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0036308289999999999 2023-09-23T20:23:14.268 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.268 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.268 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:23:14.268 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:08.634985+0000", 2023-09-23T20:23:14.268 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000146976 2023-09-23T20:23:14.269 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.269 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.269 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:23:14.269 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:08.651844+0000", 2023-09-23T20:23:14.269 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.016859365000000001 2023-09-23T20:23:14.269 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.269 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.269 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:23:14.269 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:08.651893+0000", 2023-09-23T20:23:14.269 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.8798999999999998e-05 2023-09-23T20:23:14.269 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.269 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.270 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:23:14.270 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:08.651929+0000", 2023-09-23T20:23:14.270 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.6186999999999998e-05 2023-09-23T20:23:14.270 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:23:14.270 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:23:14.270 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:23:14.270 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.270 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.270 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_repop(client.34791.0:26125 5.0 e117/111 5:014b74ea:::benchmark_data_smithi018_228451_object26124:head v 117'1649, mlcod=117'1647)", 2023-09-23T20:23:14.270 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:23:08.633834+0000", 2023-09-23T20:23:14.270 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 5.61001656, 2023-09-23T20:23:14.270 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.016650966, 2023-09-23T20:23:14.271 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:23:14.271 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:23:14.271 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:23:14.271 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.271 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:23:14.271 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:08.633834+0000", 2023-09-23T20:23:14.271 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:23:14.271 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.271 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.271 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:23:14.271 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:08.633834+0000", 2023-09-23T20:23:14.271 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:23:14.272 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.272 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.272 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:23:14.272 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:08.633833+0000", 2023-09-23T20:23:14.272 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.999999 2023-09-23T20:23:14.272 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.272 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.272 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:23:14.272 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:08.633861+0000", 2023-09-23T20:23:14.272 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.8011999999999999e-05 2023-09-23T20:23:14.272 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.272 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.272 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:23:14.273 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:08.633874+0000", 2023-09-23T20:23:14.273 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2690999999999999e-05 2023-09-23T20:23:14.273 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.273 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.273 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:23:14.273 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:08.633882+0000", 2023-09-23T20:23:14.273 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.0420000000000003e-06 2023-09-23T20:23:14.273 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.273 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.273 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:23:14.273 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:08.633936+0000", 2023-09-23T20:23:14.273 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.3752999999999998e-05 2023-09-23T20:23:14.274 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.274 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.274 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:23:14.274 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:08.633958+0000", 2023-09-23T20:23:14.274 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.2033999999999999e-05 2023-09-23T20:23:14.274 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.274 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.274 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:23:14.274 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:08.650443+0000", 2023-09-23T20:23:14.274 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.016485278999999999 2023-09-23T20:23:14.274 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.274 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.274 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:23:14.275 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:08.650485+0000", 2023-09-23T20:23:14.275 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.2114999999999999e-05 2023-09-23T20:23:14.275 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:23:14.275 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:23:14.275 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:23:14.275 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.275 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.275 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_repop(client.34791.0:26137 5.5 e117/111 5:acd89411:::benchmark_data_smithi018_228451_object26136:head v 117'1589, mlcod=117'1586)", 2023-09-23T20:23:14.275 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:23:08.634508+0000", 2023-09-23T20:23:14.275 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 5.6093426639999997, 2023-09-23T20:23:14.275 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.016301886000000002, 2023-09-23T20:23:14.275 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:23:14.276 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:23:14.276 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:23:14.276 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.276 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:23:14.276 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:08.634508+0000", 2023-09-23T20:23:14.276 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:23:14.276 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.276 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.276 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:23:14.276 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:08.634508+0000", 2023-09-23T20:23:14.276 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:23:14.276 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.276 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.277 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:23:14.277 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:08.634508+0000", 2023-09-23T20:23:14.277 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:23:14.277 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.277 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.277 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:23:14.277 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:08.634532+0000", 2023-09-23T20:23:14.277 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.4705000000000001e-05 2023-09-23T20:23:14.277 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.277 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.277 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:23:14.277 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:08.634539+0000", 2023-09-23T20:23:14.277 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.9e-06 2023-09-23T20:23:14.278 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.278 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.278 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:23:14.278 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:08.634544+0000", 2023-09-23T20:23:14.278 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.7940000000000002e-06 2023-09-23T20:23:14.278 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.278 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.278 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:23:14.278 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:08.635324+0000", 2023-09-23T20:23:14.278 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00077980599999999997 2023-09-23T20:23:14.278 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.278 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.279 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:23:14.279 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:08.635346+0000", 2023-09-23T20:23:14.279 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.2580999999999999e-05 2023-09-23T20:23:14.279 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.279 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.279 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:23:14.279 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:08.650761+0000", 2023-09-23T20:23:14.279 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.015414476 2023-09-23T20:23:14.279 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.279 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.279 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:23:14.279 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:08.650810+0000", 2023-09-23T20:23:14.280 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.8915999999999997e-05 2023-09-23T20:23:14.280 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:23:14.280 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:23:14.280 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:23:14.280 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.280 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.280 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.34791.0:26141 5.c 5:3108b05c:::benchmark_data_smithi018_228451_object26140:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2023-09-23T20:23:14.280 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:23:08.636765+0000", 2023-09-23T20:23:14.280 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 5.6070857429999998, 2023-09-23T20:23:14.280 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.020557955999999999, 2023-09-23T20:23:14.280 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:23:14.280 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:23:14.281 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:23:14.281 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.34791", 2023-09-23T20:23:14.281 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:23:14.281 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 26141 2023-09-23T20:23:14.281 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.281 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:23:14.281 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.281 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:23:14.281 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:08.636765+0000", 2023-09-23T20:23:14.281 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:23:14.281 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.281 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.281 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:23:14.282 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:08.636765+0000", 2023-09-23T20:23:14.282 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:23:14.282 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.282 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.282 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:23:14.282 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:08.636763+0000", 2023-09-23T20:23:14.282 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:23:14.282 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.282 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.282 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:23:14.282 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:08.636793+0000", 2023-09-23T20:23:14.283 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.9204999999999999e-05 2023-09-23T20:23:14.283 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.283 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.283 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:23:14.283 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:08.650428+0000", 2023-09-23T20:23:14.283 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.013635171999999999 2023-09-23T20:23:14.283 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.283 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.283 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:23:14.283 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:08.650447+0000", 2023-09-23T20:23:14.283 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.9602e-05 2023-09-23T20:23:14.284 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.284 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.284 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:23:14.284 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:08.650511+0000", 2023-09-23T20:23:14.284 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.41e-05 2023-09-23T20:23:14.284 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.284 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.284 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:23:14.284 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:08.650664+0000", 2023-09-23T20:23:14.284 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00015220000000000001 2023-09-23T20:23:14.284 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.284 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.284 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 0,4", 2023-09-23T20:23:14.285 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:08.650829+0000", 2023-09-23T20:23:14.285 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000165833 2023-09-23T20:23:14.285 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.285 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.285 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:23:14.285 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:08.652677+0000", 2023-09-23T20:23:14.285 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0018472219999999999 2023-09-23T20:23:14.285 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.285 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.285 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:23:14.285 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:08.652761+0000", 2023-09-23T20:23:14.285 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.4363999999999994e-05 2023-09-23T20:23:14.285 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.286 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.286 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:23:14.286 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:08.657230+0000", 2023-09-23T20:23:14.286 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0044693739999999999 2023-09-23T20:23:14.286 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.286 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.286 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:23:14.286 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:08.657286+0000", 2023-09-23T20:23:14.286 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.6020000000000002e-05 2023-09-23T20:23:14.286 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.286 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.286 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:23:14.287 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:08.657323+0000", 2023-09-23T20:23:14.287 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.6396000000000002e-05 2023-09-23T20:23:14.287 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:23:14.287 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:23:14.287 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:23:14.287 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.287 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.287 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_repop(client.34791.0:28891 5.8 e117/111 5:116f3062:::benchmark_data_smithi018_228451_object28890:head v 117'1812, mlcod=117'1809)", 2023-09-23T20:23:14.287 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:23:09.952836+0000", 2023-09-23T20:23:14.287 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 4.291014176, 2023-09-23T20:23:14.287 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.067661234000000001, 2023-09-23T20:23:14.287 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:23:14.287 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "started", 2023-09-23T20:23:14.288 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:23:14.288 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.288 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:23:14.288 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.952836+0000", 2023-09-23T20:23:14.288 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:23:14.288 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.288 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.288 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:23:14.288 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.952836+0000", 2023-09-23T20:23:14.288 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:23:14.288 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.288 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.288 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:23:14.289 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.952836+0000", 2023-09-23T20:23:14.289 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.999999 2023-09-23T20:23:14.289 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.289 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.289 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:23:14.289 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.952862+0000", 2023-09-23T20:23:14.289 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.6084999999999998e-05 2023-09-23T20:23:14.289 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.289 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.289 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:23:14.289 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.952873+0000", 2023-09-23T20:23:14.289 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1042e-05 2023-09-23T20:23:14.290 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.290 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.290 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:23:14.290 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.952880+0000", 2023-09-23T20:23:14.290 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.9800000000000001e-06 2023-09-23T20:23:14.290 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.290 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.290 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:23:14.290 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.955910+0000", 2023-09-23T20:23:14.290 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0030306830000000002 2023-09-23T20:23:14.290 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.290 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.291 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:23:14.291 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.955934+0000", 2023-09-23T20:23:14.291 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.3961e-05 2023-09-23T20:23:14.291 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.291 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.291 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:23:14.291 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:10.020498+0000", 2023-09-23T20:23:14.291 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.064563279000000001 2023-09-23T20:23:14.291 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:23:14.291 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:23:14.291 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:23:14.291 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.292 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.292 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_repop(client.34791.0:28895 5.8 e117/111)", 2023-09-23T20:23:14.292 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:23:09.959846+0000", 2023-09-23T20:23:14.292 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 4.2840043139999997, 2023-09-23T20:23:14.292 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.060793900999999997, 2023-09-23T20:23:14.292 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:23:14.292 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:23:14.292 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:23:14.292 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.292 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:23:14.292 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.959846+0000", 2023-09-23T20:23:14.292 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:23:14.292 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.293 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.293 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:23:14.293 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.959846+0000", 2023-09-23T20:23:14.293 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:23:14.293 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.293 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.293 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:23:14.293 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.959846+0000", 2023-09-23T20:23:14.293 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:23:14.293 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.293 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.293 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:23:14.294 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.959882+0000", 2023-09-23T20:23:14.294 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.6146000000000003e-05 2023-09-23T20:23:14.294 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.294 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.294 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:23:14.294 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.959893+0000", 2023-09-23T20:23:14.294 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0581000000000001e-05 2023-09-23T20:23:14.294 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.294 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.294 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:23:14.294 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.959901+0000", 2023-09-23T20:23:14.294 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.9230000000000002e-06 2023-09-23T20:23:14.294 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.295 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.295 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:23:14.295 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:10.020477+0000", 2023-09-23T20:23:14.295 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.060576020000000001 2023-09-23T20:23:14.295 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.295 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.295 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:23:14.295 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:10.020640+0000", 2023-09-23T20:23:14.295 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00016356699999999999 2023-09-23T20:23:14.295 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:23:14.295 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:23:14.295 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:23:14.295 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.296 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.296 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_repop(client.34791.0:28900 5.7 e117/111)", 2023-09-23T20:23:14.296 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:23:09.964108+0000", 2023-09-23T20:23:14.296 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 4.279742894, 2023-09-23T20:23:14.296 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.047487036000000003, 2023-09-23T20:23:14.296 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:23:14.296 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:23:14.296 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:23:14.296 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.296 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:23:14.296 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.964108+0000", 2023-09-23T20:23:14.296 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:23:14.297 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.297 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.297 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:23:14.297 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.964108+0000", 2023-09-23T20:23:14.297 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:23:14.297 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.297 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.297 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:23:14.297 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.964107+0000", 2023-09-23T20:23:14.297 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.999999 2023-09-23T20:23:14.297 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.297 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.298 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:23:14.298 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.964151+0000", 2023-09-23T20:23:14.298 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.3955000000000002e-05 2023-09-23T20:23:14.298 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.298 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.298 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:23:14.298 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.964162+0000", 2023-09-23T20:23:14.298 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0716e-05 2023-09-23T20:23:14.298 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.298 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.298 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:23:14.298 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.964170+0000", 2023-09-23T20:23:14.298 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.1000000000000004e-06 2023-09-23T20:23:14.299 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.299 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.299 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:23:14.299 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:10.011574+0000", 2023-09-23T20:23:14.299 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.047403985000000003 2023-09-23T20:23:14.299 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.299 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.299 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:23:14.299 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:10.011595+0000", 2023-09-23T20:23:14.299 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.1073e-05 2023-09-23T20:23:14.299 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:23:14.299 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:23:14.300 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:23:14.300 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.300 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.300 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_repop(client.34791.0:28897 5.8 e117/111)", 2023-09-23T20:23:14.300 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:23:09.966391+0000", 2023-09-23T20:23:14.300 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 4.2774596850000002, 2023-09-23T20:23:14.300 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.056564050999999997, 2023-09-23T20:23:14.300 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:23:14.300 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:23:14.300 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:23:14.300 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.300 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:23:14.301 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.966391+0000", 2023-09-23T20:23:14.301 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:23:14.301 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.301 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.301 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:23:14.301 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.966391+0000", 2023-09-23T20:23:14.301 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:23:14.301 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.301 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.301 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:23:14.301 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.966390+0000", 2023-09-23T20:23:14.301 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.999999 2023-09-23T20:23:14.301 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.302 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.302 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:23:14.302 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.966429+0000", 2023-09-23T20:23:14.302 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.8685e-05 2023-09-23T20:23:14.302 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.302 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.302 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:23:14.302 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.966446+0000", 2023-09-23T20:23:14.302 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.7322000000000001e-05 2023-09-23T20:23:14.302 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.302 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.302 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:23:14.302 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.966456+0000", 2023-09-23T20:23:14.303 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0003000000000001e-05 2023-09-23T20:23:14.303 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.303 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.303 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:23:14.303 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:10.022942+0000", 2023-09-23T20:23:14.303 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.056486249000000002 2023-09-23T20:23:14.303 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.303 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.303 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:23:14.303 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:10.022955+0000", 2023-09-23T20:23:14.303 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2693e-05 2023-09-23T20:23:14.303 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:23:14.304 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:23:14.304 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:23:14.304 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.304 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.304 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_repop(client.34791.0:28899 5.8 e117/111)", 2023-09-23T20:23:14.304 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:23:09.971941+0000", 2023-09-23T20:23:14.304 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 4.2719095420000004, 2023-09-23T20:23:14.304 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.051043534000000002, 2023-09-23T20:23:14.304 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:23:14.304 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:23:14.304 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:23:14.304 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.305 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:23:14.305 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.971941+0000", 2023-09-23T20:23:14.305 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:23:14.305 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.305 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.305 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:23:14.305 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.971941+0000", 2023-09-23T20:23:14.305 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:23:14.305 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.305 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.305 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:23:14.305 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.971940+0000", 2023-09-23T20:23:14.306 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.999999 2023-09-23T20:23:14.306 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.306 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.306 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:23:14.306 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.971971+0000", 2023-09-23T20:23:14.306 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.1235999999999997e-05 2023-09-23T20:23:14.306 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.306 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.306 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:23:14.306 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.971987+0000", 2023-09-23T20:23:14.306 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.5943999999999998e-05 2023-09-23T20:23:14.306 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.307 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.307 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:23:14.307 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.971995+0000", 2023-09-23T20:23:14.307 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.4540000000000001e-06 2023-09-23T20:23:14.307 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.307 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.307 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:23:14.307 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:10.022973+0000", 2023-09-23T20:23:14.307 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.050978185000000002 2023-09-23T20:23:14.307 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.307 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.307 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:23:14.307 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:10.022985+0000", 2023-09-23T20:23:14.308 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1792e-05 2023-09-23T20:23:14.308 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:23:14.308 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:23:14.308 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:23:14.308 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.308 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.308 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.34791.0:29826 5.c 5:34559a89:::benchmark_data_smithi018_228451_object29825:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e119)", 2023-09-23T20:23:14.308 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:23:11.456518+0000", 2023-09-23T20:23:14.308 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 2.787332331, 2023-09-23T20:23:14.308 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.017112361, 2023-09-23T20:23:14.308 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:23:14.308 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:23:14.308 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:23:14.309 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.34791", 2023-09-23T20:23:14.309 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:23:14.309 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 29826 2023-09-23T20:23:14.309 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.309 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:23:14.309 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.309 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:23:14.309 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.456518+0000", 2023-09-23T20:23:14.309 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:23:14.309 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.309 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.310 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:23:14.310 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.456518+0000", 2023-09-23T20:23:14.310 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:23:14.310 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.310 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.310 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:23:14.310 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.456517+0000", 2023-09-23T20:23:14.310 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:23:14.310 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.310 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.310 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:23:14.310 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.456545+0000", 2023-09-23T20:23:14.310 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.8598999999999998e-05 2023-09-23T20:23:14.311 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.311 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.311 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:23:14.311 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.456559+0000", 2023-09-23T20:23:14.311 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.4042999999999999e-05 2023-09-23T20:23:14.311 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.311 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.311 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:23:14.311 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.456568+0000", 2023-09-23T20:23:14.311 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.1779999999999992e-06 2023-09-23T20:23:14.311 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.311 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.312 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:23:14.312 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.456661+0000", 2023-09-23T20:23:14.312 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.3060999999999996e-05 2023-09-23T20:23:14.312 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.312 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.312 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:23:14.312 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.456780+0000", 2023-09-23T20:23:14.312 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000118348 2023-09-23T20:23:14.312 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.312 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.312 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 0,4", 2023-09-23T20:23:14.312 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.456919+0000", 2023-09-23T20:23:14.312 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000138899 2023-09-23T20:23:14.313 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.313 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.313 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:23:14.313 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.458687+0000", 2023-09-23T20:23:14.313 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.001768517 2023-09-23T20:23:14.313 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.313 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.313 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:23:14.313 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.458877+0000", 2023-09-23T20:23:14.313 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00018957300000000001 2023-09-23T20:23:14.313 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.313 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.314 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:23:14.314 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.473540+0000", 2023-09-23T20:23:14.314 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.014662840999999999 2023-09-23T20:23:14.314 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.314 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.314 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:23:14.314 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.473588+0000", 2023-09-23T20:23:14.314 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.7979999999999998e-05 2023-09-23T20:23:14.314 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.314 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.314 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:23:14.314 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.473631+0000", 2023-09-23T20:23:14.315 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.3044999999999999e-05 2023-09-23T20:23:14.315 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:23:14.315 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:23:14.315 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:23:14.315 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.315 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.315 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.34791.0:29831 5.c 5:308103cd:::benchmark_data_smithi018_228451_object29830:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e119)", 2023-09-23T20:23:14.315 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:23:11.459180+0000", 2023-09-23T20:23:14.315 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 2.784671098, 2023-09-23T20:23:14.315 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.016467611, 2023-09-23T20:23:14.315 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:23:14.315 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:23:14.315 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:23:14.316 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.34791", 2023-09-23T20:23:14.316 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:23:14.316 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 29831 2023-09-23T20:23:14.316 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.316 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:23:14.316 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.316 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:23:14.316 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.459180+0000", 2023-09-23T20:23:14.316 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:23:14.316 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.316 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.316 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:23:14.316 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.459180+0000", 2023-09-23T20:23:14.317 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:23:14.317 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.317 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.317 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:23:14.317 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.459178+0000", 2023-09-23T20:23:14.317 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:23:14.317 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.317 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.317 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:23:14.317 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.459202+0000", 2023-09-23T20:23:14.317 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.4142e-05 2023-09-23T20:23:14.317 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.318 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.318 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:23:14.318 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.459220+0000", 2023-09-23T20:23:14.318 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.7668000000000001e-05 2023-09-23T20:23:14.318 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.318 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.318 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:23:14.318 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.459229+0000", 2023-09-23T20:23:14.318 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.5100000000000004e-06 2023-09-23T20:23:14.318 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.318 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.318 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:23:14.318 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.459346+0000", 2023-09-23T20:23:14.319 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000116569 2023-09-23T20:23:14.319 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.319 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.319 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:23:14.319 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.459505+0000", 2023-09-23T20:23:14.319 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00015899999999999999 2023-09-23T20:23:14.319 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.319 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.319 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 0,4", 2023-09-23T20:23:14.319 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.459652+0000", 2023-09-23T20:23:14.319 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00014682399999999999 2023-09-23T20:23:14.319 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.320 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.320 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:23:14.320 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.463511+0000", 2023-09-23T20:23:14.320 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0038596910000000002 2023-09-23T20:23:14.320 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.320 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.320 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:23:14.320 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.464754+0000", 2023-09-23T20:23:14.320 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.001242562 2023-09-23T20:23:14.320 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.320 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.320 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:23:14.320 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.475557+0000", 2023-09-23T20:23:14.321 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.010802719000000001 2023-09-23T20:23:14.321 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.321 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.321 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:23:14.321 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.475609+0000", 2023-09-23T20:23:14.321 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.1972999999999999e-05 2023-09-23T20:23:14.321 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.321 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.321 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:23:14.321 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.475647+0000", 2023-09-23T20:23:14.321 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.8615000000000002e-05 2023-09-23T20:23:14.321 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:23:14.322 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:23:14.322 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:23:14.322 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.322 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.322 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_repop(client.34791.0:30049 5.0 e119/118 5:02143b9a:::benchmark_data_smithi018_228451_object30048:head v 119'1881, mlcod=119'1879)", 2023-09-23T20:23:14.322 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:23:11.600490+0000", 2023-09-23T20:23:14.322 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 2.6433605140000003, 2023-09-23T20:23:14.322 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.065576586000000006, 2023-09-23T20:23:14.322 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:23:14.322 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:23:14.322 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:23:14.322 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.323 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:23:14.323 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.600490+0000", 2023-09-23T20:23:14.323 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:23:14.323 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.323 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.323 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:23:14.323 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.600490+0000", 2023-09-23T20:23:14.323 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:23:14.323 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.323 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.323 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:23:14.323 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.600489+0000", 2023-09-23T20:23:14.324 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:23:14.324 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.324 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.324 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:23:14.324 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.600513+0000", 2023-09-23T20:23:14.324 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.3672999999999998e-05 2023-09-23T20:23:14.324 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.324 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.324 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:23:14.324 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.600524+0000", 2023-09-23T20:23:14.324 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0762e-05 2023-09-23T20:23:14.324 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.324 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.325 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:23:14.325 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.600530+0000", 2023-09-23T20:23:14.325 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.6460000000000004e-06 2023-09-23T20:23:14.325 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.325 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.325 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:23:14.325 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.664459+0000", 2023-09-23T20:23:14.325 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.063928296999999995 2023-09-23T20:23:14.325 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.325 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.325 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:23:14.325 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.664482+0000", 2023-09-23T20:23:14.326 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.3598999999999999e-05 2023-09-23T20:23:14.326 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.326 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.326 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:23:14.326 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.666044+0000", 2023-09-23T20:23:14.326 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.001561839 2023-09-23T20:23:14.326 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.326 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.326 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:23:14.326 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.666067+0000", 2023-09-23T20:23:14.326 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.2456999999999999e-05 2023-09-23T20:23:14.326 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:23:14.326 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:23:14.327 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:23:14.327 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.327 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.327 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_repop(client.34791.0:30048 5.7 e119/118 5:e4e7bb94:::benchmark_data_smithi018_228451_object30047:head v 119'1875, mlcod=119'1873)", 2023-09-23T20:23:14.327 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:23:11.601095+0000", 2023-09-23T20:23:14.327 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 2.6427556559999998, 2023-09-23T20:23:14.327 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.065242429000000005, 2023-09-23T20:23:14.327 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:23:14.327 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:23:14.327 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:23:14.327 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.328 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:23:14.328 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.601095+0000", 2023-09-23T20:23:14.328 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:23:14.328 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.328 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.328 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:23:14.328 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.601095+0000", 2023-09-23T20:23:14.328 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:23:14.328 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.328 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.328 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:23:14.328 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.601094+0000", 2023-09-23T20:23:14.328 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:23:14.329 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.329 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.329 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:23:14.329 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.601148+0000", 2023-09-23T20:23:14.329 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.4033999999999998e-05 2023-09-23T20:23:14.329 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.329 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.329 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:23:14.329 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.664431+0000", 2023-09-23T20:23:14.329 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.063283060000000002 2023-09-23T20:23:14.329 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.329 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.329 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:23:14.330 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.664439+0000", 2023-09-23T20:23:14.330 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.8960000000000003e-06 2023-09-23T20:23:14.330 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.330 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.330 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:23:14.330 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.664486+0000", 2023-09-23T20:23:14.330 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.6499000000000003e-05 2023-09-23T20:23:14.330 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.330 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.330 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:23:14.330 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.664508+0000", 2023-09-23T20:23:14.331 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.1818999999999999e-05 2023-09-23T20:23:14.331 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.331 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.331 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:23:14.331 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.666319+0000", 2023-09-23T20:23:14.331 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0018118100000000001 2023-09-23T20:23:14.331 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.331 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.331 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:23:14.331 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.666337+0000", 2023-09-23T20:23:14.331 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.7941e-05 2023-09-23T20:23:14.331 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:23:14.331 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:23:14.332 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:23:14.332 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.332 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.332 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_repop(client.34791.0:30056 5.8 e119/118 5:17db1733:::benchmark_data_smithi018_228451_object30055:head v 119'1852, mlcod=119'1850)", 2023-09-23T20:23:14.332 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:23:11.602466+0000", 2023-09-23T20:23:14.332 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 2.6413841840000001, 2023-09-23T20:23:14.332 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.063574579000000006, 2023-09-23T20:23:14.332 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:23:14.332 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:23:14.332 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:23:14.332 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.332 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:23:14.333 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.602466+0000", 2023-09-23T20:23:14.333 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:23:14.333 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.333 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.333 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:23:14.333 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.602466+0000", 2023-09-23T20:23:14.333 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:23:14.333 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.333 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.333 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:23:14.333 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.602466+0000", 2023-09-23T20:23:14.333 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.999999 2023-09-23T20:23:14.333 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.334 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.334 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:23:14.334 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.602492+0000", 2023-09-23T20:23:14.334 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.6409000000000001e-05 2023-09-23T20:23:14.334 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.334 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.334 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:23:14.334 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.664428+0000", 2023-09-23T20:23:14.334 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.061935839999999999 2023-09-23T20:23:14.334 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.334 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.334 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:23:14.335 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.664437+0000", 2023-09-23T20:23:14.335 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.4730000000000006e-06 2023-09-23T20:23:14.335 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.335 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.335 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:23:14.335 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.664555+0000", 2023-09-23T20:23:14.335 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000117469 2023-09-23T20:23:14.335 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.335 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.335 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:23:14.335 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.664575+0000", 2023-09-23T20:23:14.335 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.9847000000000002e-05 2023-09-23T20:23:14.336 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.336 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.336 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:23:14.336 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.666004+0000", 2023-09-23T20:23:14.336 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0014292689999999999 2023-09-23T20:23:14.336 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.336 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.336 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:23:14.336 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.666041+0000", 2023-09-23T20:23:14.336 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.7038e-05 2023-09-23T20:23:14.336 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:23:14.336 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:23:14.336 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:23:14.337 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.337 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.337 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_repop(client.34791.0:30066 5.15 e119/118 5:ad3ada56:::benchmark_data_smithi018_228451_object30065:head v 119'1806, mlcod=119'1804)", 2023-09-23T20:23:14.337 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:23:11.606339+0000", 2023-09-23T20:23:14.337 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 2.6375114860000002, 2023-09-23T20:23:14.337 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.059641397999999998, 2023-09-23T20:23:14.337 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:23:14.337 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:23:14.337 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:23:14.337 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.337 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:23:14.337 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.606339+0000", 2023-09-23T20:23:14.337 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:23:14.338 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.338 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.338 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:23:14.338 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.606339+0000", 2023-09-23T20:23:14.338 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:23:14.338 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.338 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.338 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:23:14.338 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.606338+0000", 2023-09-23T20:23:14.338 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:23:14.338 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.338 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.339 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:23:14.339 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.606365+0000", 2023-09-23T20:23:14.339 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.6925999999999999e-05 2023-09-23T20:23:14.339 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.339 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.339 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:23:14.339 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.664423+0000", 2023-09-23T20:23:14.339 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.058058222 2023-09-23T20:23:14.339 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.339 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.339 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:23:14.339 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.664438+0000", 2023-09-23T20:23:14.339 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.5089000000000001e-05 2023-09-23T20:23:14.340 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.340 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.340 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:23:14.340 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.664485+0000", 2023-09-23T20:23:14.340 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.6584e-05 2023-09-23T20:23:14.340 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.340 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.340 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:23:14.340 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.664506+0000", 2023-09-23T20:23:14.340 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.1421e-05 2023-09-23T20:23:14.340 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.340 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.341 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:23:14.341 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.665946+0000", 2023-09-23T20:23:14.341 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.001439748 2023-09-23T20:23:14.341 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.341 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.341 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:23:14.341 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.665981+0000", 2023-09-23T20:23:14.341 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.4736000000000001e-05 2023-09-23T20:23:14.341 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:23:14.341 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:23:14.341 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:23:14.342 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.342 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.342 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.34791.0:33553 5.c 5:32af92e8:::benchmark_data_smithi018_228451_object33552:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e119)", 2023-09-23T20:23:14.342 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:23:13.132625+0000", 2023-09-23T20:23:14.342 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 1.1112254260000001, 2023-09-23T20:23:14.342 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.020257374000000002, 2023-09-23T20:23:14.342 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:23:14.342 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:23:14.342 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:23:14.342 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.34791", 2023-09-23T20:23:14.342 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:23:14.342 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 33553 2023-09-23T20:23:14.343 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.343 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:23:14.343 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.343 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:23:14.343 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:13.132625+0000", 2023-09-23T20:23:14.343 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:23:14.343 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.343 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.343 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:23:14.343 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:13.132625+0000", 2023-09-23T20:23:14.343 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:23:14.343 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.344 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.344 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:23:14.344 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:13.132624+0000", 2023-09-23T20:23:14.344 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:23:14.344 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.344 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.344 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:23:14.344 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:13.132648+0000", 2023-09-23T20:23:14.344 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.4056000000000001e-05 2023-09-23T20:23:14.344 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.344 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.344 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:23:14.345 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:13.132661+0000", 2023-09-23T20:23:14.345 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.309e-05 2023-09-23T20:23:14.345 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.345 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.345 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:23:14.345 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:13.132669+0000", 2023-09-23T20:23:14.345 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.0490000000000004e-06 2023-09-23T20:23:14.345 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.345 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.345 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:23:14.345 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:13.132725+0000", 2023-09-23T20:23:14.345 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.6558999999999998e-05 2023-09-23T20:23:14.345 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.346 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.346 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:23:14.346 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:13.132877+0000", 2023-09-23T20:23:14.346 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00015186299999999999 2023-09-23T20:23:14.346 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.346 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.346 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 0,4", 2023-09-23T20:23:14.346 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:13.133048+0000", 2023-09-23T20:23:14.346 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00017094500000000001 2023-09-23T20:23:14.346 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.346 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.346 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:23:14.346 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:13.134591+0000", 2023-09-23T20:23:14.347 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.001542505 2023-09-23T20:23:14.347 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.347 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.347 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:23:14.347 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:13.137348+0000", 2023-09-23T20:23:14.347 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0027571470000000002 2023-09-23T20:23:14.347 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.347 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.347 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:23:14.347 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:13.152784+0000", 2023-09-23T20:23:14.347 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.015436228 2023-09-23T20:23:14.348 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.348 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.348 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:23:14.348 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:13.152843+0000", 2023-09-23T20:23:14.348 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.8965000000000002e-05 2023-09-23T20:23:14.348 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.348 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.348 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:23:14.348 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:13.152883+0000", 2023-09-23T20:23:14.348 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.9629000000000003e-05 2023-09-23T20:23:14.348 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:23:14.348 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:23:14.348 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:23:14.349 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.349 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.349 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.34791.0:34195 5.c 5:31faae24:::benchmark_data_smithi018_228451_object34194:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e119)", 2023-09-23T20:23:14.349 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:23:13.434104+0000", 2023-09-23T20:23:14.349 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.80974649300000001, 2023-09-23T20:23:14.349 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.028755393000000001, 2023-09-23T20:23:14.349 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:23:14.349 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:23:14.349 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:23:14.349 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.34791", 2023-09-23T20:23:14.349 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:23:14.349 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 34195 2023-09-23T20:23:14.349 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.350 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:23:14.350 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.350 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:23:14.350 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:13.434104+0000", 2023-09-23T20:23:14.350 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:23:14.350 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.350 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.350 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:23:14.350 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:13.434104+0000", 2023-09-23T20:23:14.350 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:23:14.350 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.350 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.351 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:23:14.351 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:13.434103+0000", 2023-09-23T20:23:14.351 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:23:14.351 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.351 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.351 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:23:14.351 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:13.434127+0000", 2023-09-23T20:23:14.351 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.3252999999999999e-05 2023-09-23T20:23:14.351 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.351 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.351 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:23:14.351 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:13.434134+0000", 2023-09-23T20:23:14.351 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.4250000000000001e-06 2023-09-23T20:23:14.352 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.352 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.352 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:23:14.352 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:13.434140+0000", 2023-09-23T20:23:14.352 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.1210000000000003e-06 2023-09-23T20:23:14.352 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.352 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.352 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:23:14.352 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:13.437428+0000", 2023-09-23T20:23:14.352 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0032876680000000001 2023-09-23T20:23:14.352 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.353 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.353 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:23:14.353 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:13.437552+0000", 2023-09-23T20:23:14.353 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00012441700000000001 2023-09-23T20:23:14.353 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.353 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.353 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 0,4", 2023-09-23T20:23:14.353 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:13.437655+0000", 2023-09-23T20:23:14.353 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00010306099999999999 2023-09-23T20:23:14.353 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.353 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.353 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:23:14.353 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:13.440787+0000", 2023-09-23T20:23:14.354 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0031312829999999999 2023-09-23T20:23:14.354 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.354 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.354 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:23:14.354 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:13.441967+0000", 2023-09-23T20:23:14.354 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0011806830000000001 2023-09-23T20:23:14.354 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.354 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.354 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:23:14.354 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:13.462716+0000", 2023-09-23T20:23:14.354 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.020748144999999999 2023-09-23T20:23:14.355 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.355 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.355 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:23:14.355 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:13.462770+0000", 2023-09-23T20:23:14.355 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.3961000000000001e-05 2023-09-23T20:23:14.355 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.355 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.355 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:23:14.355 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:13.462860+0000", 2023-09-23T20:23:14.355 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.9988999999999995e-05 2023-09-23T20:23:14.355 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:23:14.355 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:23:14.356 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:23:14.356 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.356 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.356 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.34791.0:34207 5.c 5:39729a72:::benchmark_data_smithi018_228451_object34206:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e119)", 2023-09-23T20:23:14.356 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:23:13.437803+0000", 2023-09-23T20:23:14.356 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.80604726299999996, 2023-09-23T20:23:14.356 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.025165557000000002, 2023-09-23T20:23:14.356 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:23:14.356 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:23:14.356 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:23:14.356 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.34791", 2023-09-23T20:23:14.356 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:23:14.357 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 34207 2023-09-23T20:23:14.357 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.357 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:23:14.357 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.357 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:23:14.357 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:13.437803+0000", 2023-09-23T20:23:14.357 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:23:14.357 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.357 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.357 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:23:14.357 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:13.437803+0000", 2023-09-23T20:23:14.358 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:23:14.358 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.358 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.358 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:23:14.358 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:13.437802+0000", 2023-09-23T20:23:14.358 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:23:14.358 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.358 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.358 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:23:14.358 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:13.437831+0000", 2023-09-23T20:23:14.358 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.9533999999999999e-05 2023-09-23T20:23:14.358 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.359 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.359 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:23:14.359 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:13.437849+0000", 2023-09-23T20:23:14.359 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.7390000000000001e-05 2023-09-23T20:23:14.359 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.359 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.359 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:23:14.359 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:13.437858+0000", 2023-09-23T20:23:14.359 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.2760000000000006e-06 2023-09-23T20:23:14.359 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.359 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.359 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:23:14.360 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:13.439369+0000", 2023-09-23T20:23:14.360 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0015107359999999999 2023-09-23T20:23:14.360 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.360 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.360 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:23:14.360 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:13.439493+0000", 2023-09-23T20:23:14.360 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00012423600000000001 2023-09-23T20:23:14.360 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.360 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.360 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 0,4", 2023-09-23T20:23:14.360 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:13.439626+0000", 2023-09-23T20:23:14.361 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000133211 2023-09-23T20:23:14.361 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.361 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.361 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:23:14.361 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:13.440941+0000", 2023-09-23T20:23:14.361 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0013146010000000001 2023-09-23T20:23:14.361 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.361 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.361 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:23:14.361 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:13.446837+0000", 2023-09-23T20:23:14.361 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0058959420000000004 2023-09-23T20:23:14.361 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.362 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.362 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:23:14.362 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:13.462897+0000", 2023-09-23T20:23:14.362 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.016060857000000001 2023-09-23T20:23:14.362 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.362 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.362 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:23:14.362 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:13.462938+0000", 2023-09-23T20:23:14.362 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.0720000000000003e-05 2023-09-23T20:23:14.362 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:14.362 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:14.362 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:23:14.363 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:13.462969+0000", 2023-09-23T20:23:14.363 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.0799000000000001e-05 2023-09-23T20:23:14.363 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:23:14.363 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:23:14.363 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:23:14.363 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:23:14.363 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:23:14.363 INFO:teuthology.orchestra.run.smithi018.stdout:} 2023-09-23T20:23:14.364 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:23:14.364 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:14.365 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:23:14.365 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:23:14.456 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 16 16 36337 36321 141.837 132.625 0.00796281 0.00704587 2023-09-23T20:23:14.466 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:23:14.518 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:14.631 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:23:14.636 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:14.636 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:23:14.677 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T20:23:14.738 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:23:14.890 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:14.977 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:23:14.987 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:14.987 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:23:15.088 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:23:15.145 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:23:15 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:23:15] "GET /metrics HTTP/1.1" 200 36458 "" "Prometheus/2.43.0" 2023-09-23T20:23:15.145 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:15 smithi018 ceph-mon[184144]: 5.4 scrub ok 2023-09-23T20:23:15.145 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:15 smithi018 ceph-mon[184144]: pgmap v249: 99 pgs: 1 active+clean+scrubbing, 5 peering, 5 unknown, 88 active+clean; 1.6 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s; 32 KiB/s, 0 objects/s recovering 2023-09-23T20:23:15.146 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:15 smithi018 ceph-mon[184144]: 5.8 scrub starts 2023-09-23T20:23:15.146 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:15 smithi018 ceph-mon[184144]: 5.0 scrub ok 2023-09-23T20:23:15.146 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:15 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:23:15.146 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:15 smithi018 ceph-mon[184144]: 5.10 scrub starts 2023-09-23T20:23:15.146 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:15 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:23:15.146 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:15 smithi018 ceph-mon[179647]: 5.4 scrub ok 2023-09-23T20:23:15.147 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:15 smithi018 ceph-mon[179647]: pgmap v249: 99 pgs: 1 active+clean+scrubbing, 5 peering, 5 unknown, 88 active+clean; 1.6 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s; 32 KiB/s, 0 objects/s recovering 2023-09-23T20:23:15.147 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:15 smithi018 ceph-mon[179647]: 5.8 scrub starts 2023-09-23T20:23:15.147 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:15 smithi018 ceph-mon[179647]: 5.0 scrub ok 2023-09-23T20:23:15.147 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:15 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:23:15.147 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:15 smithi018 ceph-mon[179647]: 5.10 scrub starts 2023-09-23T20:23:15.147 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:15 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:23:15.171 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:15.395 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:23:15.401 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:15.402 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:23:15.455 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:23:15.456 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 17 16 38065 38049 139.846 108 0.00655713 0.00714608 2023-09-23T20:23:15.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:15 smithi149 ceph-mon[170394]: 5.4 scrub ok 2023-09-23T20:23:15.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:15 smithi149 ceph-mon[170394]: pgmap v249: 99 pgs: 1 active+clean+scrubbing, 5 peering, 5 unknown, 88 active+clean; 1.6 GiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s; 32 KiB/s, 0 objects/s recovering 2023-09-23T20:23:15.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:15 smithi149 ceph-mon[170394]: 5.8 scrub starts 2023-09-23T20:23:15.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:15 smithi149 ceph-mon[170394]: 5.0 scrub ok 2023-09-23T20:23:15.468 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:15 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:23:15.468 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:15 smithi149 ceph-mon[170394]: 5.10 scrub starts 2023-09-23T20:23:15.468 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:15 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:23:15.503 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:23:15.584 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:15.834 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:23:15.842 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:15.843 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:23:15.943 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:23:16.011 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:23:16.021 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:16.028 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:16 smithi018 ceph-mon[179647]: 5.16 scrub starts 2023-09-23T20:23:16.029 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:16 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:23:16.029 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:16 smithi018 ceph-mon[179647]: 5.10 scrub ok 2023-09-23T20:23:16.029 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:16 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "17"}]: dispatch 2023-09-23T20:23:16.135 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:23:16.142 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:16.143 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:23:16.243 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:23:16.295 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:16 smithi018 ceph-mon[184144]: 5.16 scrub starts 2023-09-23T20:23:16.295 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:16 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:23:16.295 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:16 smithi018 ceph-mon[184144]: 5.10 scrub ok 2023-09-23T20:23:16.295 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:16 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "17"}]: dispatch 2023-09-23T20:23:16.297 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:16.413 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:23:16.418 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:16.419 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:23:16.456 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 18 16 39597 39581 137.396 95.75 0.00195584 0.00720571 2023-09-23T20:23:16.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:16 smithi149 ceph-mon[170394]: 5.16 scrub starts 2023-09-23T20:23:16.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:16 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:23:16.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:16 smithi149 ceph-mon[170394]: 5.10 scrub ok 2023-09-23T20:23:16.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:16 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "17"}]: dispatch 2023-09-23T20:23:16.507 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-23T20:23:16.507 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:23:16.508 INFO:tasks.thrashosds.thrasher:inject_pause on osd.4 2023-09-23T20:23:16.508 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T20:23:16.508 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T20:23:16.508 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 config set filestore_inject_stall 3 2023-09-23T20:23:16.519 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:23:16.554 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:16.606 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:23:16.908 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:23:16.912 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:16.913 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:23:17.013 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:23:17.047 INFO:teuthology.orchestra.run.smithi018.stderr:nodeep-scrub is unset 2023-09-23T20:23:17.069 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:17.102 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:23:17.261 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:17 smithi018 ceph-mon[179647]: 5.16 scrub ok 2023-09-23T20:23:17.261 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:17 smithi018 ceph-mon[179647]: pgmap v250: 99 pgs: 1 active+clean+scrubbing, 98 active+clean; 2.1 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s; 34 KiB/s, 0 objects/s recovering 2023-09-23T20:23:17.261 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:17 smithi018 ceph-mon[179647]: 5.12 scrub starts 2023-09-23T20:23:17.261 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:17 smithi018 ceph-mon[179647]: 5.12 scrub ok 2023-09-23T20:23:17.261 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:17 smithi018 ceph-mon[179647]: 5.7 scrub starts 2023-09-23T20:23:17.261 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:17 smithi018 ceph-mon[179647]: 5.18 scrub starts 2023-09-23T20:23:17.261 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:17 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "17"}]': finished 2023-09-23T20:23:17.262 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:17 smithi018 ceph-mon[179647]: osdmap e120: 8 total, 8 up, 8 in 2023-09-23T20:23:17.262 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:17 smithi018 ceph-mon[179647]: failed to encode map e120 with expected crc 2023-09-23T20:23:17.262 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:17 smithi018 ceph-mon[179647]: 5.7 scrub ok 2023-09-23T20:23:17.262 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:17 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2429329726' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:23:17.262 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:17 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:23:17.262 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:17 smithi018 ceph-mon[184144]: 5.16 scrub ok 2023-09-23T20:23:17.262 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:17 smithi018 ceph-mon[184144]: pgmap v250: 99 pgs: 1 active+clean+scrubbing, 98 active+clean; 2.1 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s; 34 KiB/s, 0 objects/s recovering 2023-09-23T20:23:17.262 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:17 smithi018 ceph-mon[184144]: 5.12 scrub starts 2023-09-23T20:23:17.262 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:17 smithi018 ceph-mon[184144]: 5.12 scrub ok 2023-09-23T20:23:17.262 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:17 smithi018 ceph-mon[184144]: 5.7 scrub starts 2023-09-23T20:23:17.263 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:17 smithi018 ceph-mon[184144]: 5.18 scrub starts 2023-09-23T20:23:17.263 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:17 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "17"}]': finished 2023-09-23T20:23:17.263 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:17 smithi018 ceph-mon[184144]: osdmap e120: 8 total, 8 up, 8 in 2023-09-23T20:23:17.263 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:17 smithi018 ceph-mon[184144]: failed to encode map e120 with expected crc 2023-09-23T20:23:17.263 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:17 smithi018 ceph-mon[184144]: 5.7 scrub ok 2023-09-23T20:23:17.263 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:17 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2429329726' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:23:17.263 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:17 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:23:17.312 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:17 smithi149 ceph-mon[170394]: 5.16 scrub ok 2023-09-23T20:23:17.312 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:17 smithi149 ceph-mon[170394]: pgmap v250: 99 pgs: 1 active+clean+scrubbing, 98 active+clean; 2.1 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s; 34 KiB/s, 0 objects/s recovering 2023-09-23T20:23:17.312 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:17 smithi149 ceph-mon[170394]: 5.12 scrub starts 2023-09-23T20:23:17.312 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:17 smithi149 ceph-mon[170394]: 5.12 scrub ok 2023-09-23T20:23:17.312 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:17 smithi149 ceph-mon[170394]: 5.7 scrub starts 2023-09-23T20:23:17.312 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:17 smithi149 ceph-mon[170394]: 5.18 scrub starts 2023-09-23T20:23:17.312 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:17 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "17"}]': finished 2023-09-23T20:23:17.312 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:17 smithi149 ceph-mon[170394]: osdmap e120: 8 total, 8 up, 8 in 2023-09-23T20:23:17.312 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:17 smithi149 ceph-mon[170394]: failed to encode map e120 with expected crc 2023-09-23T20:23:17.313 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:17 smithi149 ceph-mon[170394]: 5.7 scrub ok 2023-09-23T20:23:17.313 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:17 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2429329726' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:23:17.313 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:17 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:23:17.456 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 19 16 39689 39673 130.469 5.75 0.00255793 0.00719465 2023-09-23T20:23:17.630 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:23:17.633 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:17.634 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:23:17.668 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2023-09-23T20:23:17.668 INFO:teuthology.orchestra.run.smithi018.stdout: "ops": [], 2023-09-23T20:23:17.668 INFO:teuthology.orchestra.run.smithi018.stdout: "num_ops": 0 2023-09-23T20:23:17.668 INFO:teuthology.orchestra.run.smithi018.stdout:} 2023-09-23T20:23:17.735 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:23:17.766 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:18.004 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:23:18.007 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:18.008 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:23:18.109 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:23:18.138 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:18 smithi018 ceph-mon[179647]: 5.8 scrub starts 2023-09-23T20:23:18.138 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:18 smithi018 ceph-mon[179647]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:23:18.138 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:18 smithi018 ceph-mon[179647]: Cluster is now healthy 2023-09-23T20:23:18.138 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:18 smithi018 ceph-mon[179647]: failed to encode map e121 with expected crc 2023-09-23T20:23:18.138 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:18 smithi018 ceph-mon[179647]: failed to encode map e121 with expected crc 2023-09-23T20:23:18.138 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:18 smithi018 ceph-mon[179647]: failed to encode map e121 with expected crc 2023-09-23T20:23:18.138 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:18 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:23:18.138 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:18 smithi018 ceph-mon[179647]: osdmap e121: 8 total, 8 up, 8 in 2023-09-23T20:23:18.138 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:18 smithi018 ceph-mon[179647]: failed to encode map e121 with expected crc 2023-09-23T20:23:18.138 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:18 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "18"}]: dispatch 2023-09-23T20:23:18.139 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:18 smithi018 ceph-mon[184144]: 5.8 scrub starts 2023-09-23T20:23:18.139 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:18 smithi018 ceph-mon[184144]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:23:18.139 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:18 smithi018 ceph-mon[184144]: Cluster is now healthy 2023-09-23T20:23:18.139 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:18 smithi018 ceph-mon[184144]: failed to encode map e121 with expected crc 2023-09-23T20:23:18.139 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:18 smithi018 ceph-mon[184144]: failed to encode map e121 with expected crc 2023-09-23T20:23:18.139 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:18 smithi018 ceph-mon[184144]: failed to encode map e121 with expected crc 2023-09-23T20:23:18.139 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:18 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:23:18.139 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:18 smithi018 ceph-mon[184144]: osdmap e121: 8 total, 8 up, 8 in 2023-09-23T20:23:18.139 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:18 smithi018 ceph-mon[184144]: failed to encode map e121 with expected crc 2023-09-23T20:23:18.139 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:18 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "18"}]: dispatch 2023-09-23T20:23:18.164 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:18.227 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T20:23:18.252 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:23:18.258 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:18.259 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:23:18.359 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:23:18.388 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:18 smithi149 ceph-mon[170394]: 5.8 scrub starts 2023-09-23T20:23:18.388 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:18 smithi149 ceph-mon[170394]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:23:18.388 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:18 smithi149 ceph-mon[170394]: Cluster is now healthy 2023-09-23T20:23:18.388 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:18 smithi149 ceph-mon[170394]: failed to encode map e121 with expected crc 2023-09-23T20:23:18.388 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:18 smithi149 ceph-mon[170394]: failed to encode map e121 with expected crc 2023-09-23T20:23:18.388 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:18 smithi149 ceph-mon[170394]: failed to encode map e121 with expected crc 2023-09-23T20:23:18.389 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:18 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:23:18.389 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:18 smithi149 ceph-mon[170394]: osdmap e121: 8 total, 8 up, 8 in 2023-09-23T20:23:18.389 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:18 smithi149 ceph-mon[170394]: failed to encode map e121 with expected crc 2023-09-23T20:23:18.389 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:18 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "18"}]: dispatch 2023-09-23T20:23:18.391 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:18.456 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:2023-09-23T20:23:18.457533+0000 min lat: 0.00112599 max lat: 2.03187 avg lat: 0.00791318 2023-09-23T20:23:18.457 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T20:23:18.457 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 20 16 40037 40021 125.034 21.75 0.00176934 0.00791318 2023-09-23T20:23:18.829 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:23:18.832 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:18.832 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:23:18.922 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:23:18.933 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:23:18.950 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:23:18.970 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:18.973 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:23:18.980 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:23:18.998 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:23:19.012 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:23:19.016 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:19.016 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:23:19.023 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:23:19.046 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:23:19.071 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:23:19.095 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:23:19.096 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:23:19.096 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:23:19.096 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:23:19.096 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:23:19.096 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:23:19.096 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:23:19.097 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:23:19.097 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:23:19.117 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:23:19.143 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:19 smithi149 ceph-mon[170394]: 5.8 scrub ok 2023-09-23T20:23:19.143 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:19 smithi149 ceph-mon[170394]: pgmap v253: 99 pgs: 99 active+clean; 2.3 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s; 39 KiB/s, 0 objects/s recovering 2023-09-23T20:23:19.143 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:19 smithi149 ceph-mon[170394]: 5.18 scrub starts 2023-09-23T20:23:19.144 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:19 smithi149 ceph-mon[170394]: failed to encode map e121 with expected crc 2023-09-23T20:23:19.144 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:19 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "18"}]': finished 2023-09-23T20:23:19.144 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:19 smithi149 ceph-mon[170394]: osdmap e122: 8 total, 8 up, 8 in 2023-09-23T20:23:19.144 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:19 smithi149 ceph-mon[170394]: failed to encode map e122 with expected crc 2023-09-23T20:23:19.144 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:19 smithi149 ceph-mon[170394]: failed to encode map e122 with expected crc 2023-09-23T20:23:19.144 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:19 smithi149 ceph-mon[170394]: failed to encode map e122 with expected crc 2023-09-23T20:23:19.144 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:19 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3299046889' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:23:19.146 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:19.235 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:19 smithi018 ceph-mon[184144]: 5.8 scrub ok 2023-09-23T20:23:19.235 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:19 smithi018 ceph-mon[184144]: pgmap v253: 99 pgs: 99 active+clean; 2.3 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s; 39 KiB/s, 0 objects/s recovering 2023-09-23T20:23:19.235 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:19 smithi018 ceph-mon[184144]: 5.18 scrub starts 2023-09-23T20:23:19.235 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:19 smithi018 ceph-mon[184144]: failed to encode map e121 with expected crc 2023-09-23T20:23:19.235 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:19 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "18"}]': finished 2023-09-23T20:23:19.235 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:19 smithi018 ceph-mon[184144]: osdmap e122: 8 total, 8 up, 8 in 2023-09-23T20:23:19.236 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:19 smithi018 ceph-mon[184144]: failed to encode map e122 with expected crc 2023-09-23T20:23:19.236 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:19 smithi018 ceph-mon[184144]: failed to encode map e122 with expected crc 2023-09-23T20:23:19.236 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:19 smithi018 ceph-mon[184144]: failed to encode map e122 with expected crc 2023-09-23T20:23:19.236 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:19 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3299046889' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:23:19.236 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:19 smithi018 ceph-mon[179647]: 5.8 scrub ok 2023-09-23T20:23:19.236 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:19 smithi018 ceph-mon[179647]: pgmap v253: 99 pgs: 99 active+clean; 2.3 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s; 39 KiB/s, 0 objects/s recovering 2023-09-23T20:23:19.236 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:19 smithi018 ceph-mon[179647]: 5.18 scrub starts 2023-09-23T20:23:19.236 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:19 smithi018 ceph-mon[179647]: failed to encode map e121 with expected crc 2023-09-23T20:23:19.236 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:19 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "18"}]': finished 2023-09-23T20:23:19.237 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:19 smithi018 ceph-mon[179647]: osdmap e122: 8 total, 8 up, 8 in 2023-09-23T20:23:19.237 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:19 smithi018 ceph-mon[179647]: failed to encode map e122 with expected crc 2023-09-23T20:23:19.237 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:19 smithi018 ceph-mon[179647]: failed to encode map e122 with expected crc 2023-09-23T20:23:19.237 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:19 smithi018 ceph-mon[179647]: failed to encode map e122 with expected crc 2023-09-23T20:23:19.237 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:19 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3299046889' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:23:19.456 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 21 16 40067 40051 119.17 1.875 0.00193176 0.00790888 2023-09-23T20:23:19.777 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T20:23:20.348 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:20 smithi018 ceph-mon[184144]: 5.18 scrub ok 2023-09-23T20:23:20.348 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:20 smithi018 ceph-mon[184144]: failed to encode map e123 with expected crc 2023-09-23T20:23:20.348 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:20 smithi018 ceph-mon[184144]: osdmap e123: 8 total, 8 up, 8 in 2023-09-23T20:23:20.348 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:20 smithi018 ceph-mon[184144]: failed to encode map e123 with expected crc 2023-09-23T20:23:20.348 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:20 smithi018 ceph-mon[184144]: failed to encode map e123 with expected crc 2023-09-23T20:23:20.348 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:20 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "19"}]: dispatch 2023-09-23T20:23:20.348 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:20 smithi018 ceph-mon[179647]: 5.18 scrub ok 2023-09-23T20:23:20.348 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:20 smithi018 ceph-mon[179647]: failed to encode map e123 with expected crc 2023-09-23T20:23:20.349 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:20 smithi018 ceph-mon[179647]: osdmap e123: 8 total, 8 up, 8 in 2023-09-23T20:23:20.349 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:20 smithi018 ceph-mon[179647]: failed to encode map e123 with expected crc 2023-09-23T20:23:20.349 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:20 smithi018 ceph-mon[179647]: failed to encode map e123 with expected crc 2023-09-23T20:23:20.349 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:20 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "19"}]: dispatch 2023-09-23T20:23:20.457 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 22 16 40539 40523 115.095 29.5 0.00210697 0.00861895 2023-09-23T20:23:20.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:20 smithi149 ceph-mon[170394]: 5.18 scrub ok 2023-09-23T20:23:20.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:20 smithi149 ceph-mon[170394]: failed to encode map e123 with expected crc 2023-09-23T20:23:20.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:20 smithi149 ceph-mon[170394]: osdmap e123: 8 total, 8 up, 8 in 2023-09-23T20:23:20.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:20 smithi149 ceph-mon[170394]: failed to encode map e123 with expected crc 2023-09-23T20:23:20.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:20 smithi149 ceph-mon[170394]: failed to encode map e123 with expected crc 2023-09-23T20:23:20.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:20 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "19"}]: dispatch 2023-09-23T20:23:20.636 INFO:teuthology.orchestra.run.smithi018.stdout:ERROR: (22) Invalid argument 2023-09-23T20:23:20.637 INFO:teuthology.orchestra.run.smithi018.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:23:20.910 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:23:20.913 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:20.914 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:23:21.014 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:23:21.049 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:21.086 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:23:21.088 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:21.089 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:23:21.116 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T20:23:21.150 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:23:21.189 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:23:21.215 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:21 smithi149 ceph-mon[170394]: pgmap v256: 99 pgs: 99 active+clean; 2.4 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 9.2 MiB/s wr, 147 op/s 2023-09-23T20:23:21.215 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:21 smithi149 ceph-mon[170394]: 5.3 scrub starts 2023-09-23T20:23:21.215 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:21 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "19"}]': finished 2023-09-23T20:23:21.215 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:21 smithi149 ceph-mon[170394]: osdmap e124: 8 total, 8 up, 8 in 2023-09-23T20:23:21.215 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:21 smithi149 ceph-mon[170394]: failed to encode map e124 with expected crc 2023-09-23T20:23:21.215 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:21 smithi149 ceph-mon[170394]: failed to encode map e124 with expected crc 2023-09-23T20:23:21.215 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:21 smithi149 ceph-mon[170394]: failed to encode map e124 with expected crc 2023-09-23T20:23:21.215 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:21 smithi149 ceph-mon[170394]: failed to encode map e124 with expected crc 2023-09-23T20:23:21.215 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:21 smithi149 ceph-mon[170394]: osdmap e125: 8 total, 8 up, 8 in 2023-09-23T20:23:21.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:21 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:23:21.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:21 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:23:21.218 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:21.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:21 smithi018 ceph-mon[179647]: pgmap v256: 99 pgs: 99 active+clean; 2.4 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 9.2 MiB/s wr, 147 op/s 2023-09-23T20:23:21.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:21 smithi018 ceph-mon[179647]: 5.3 scrub starts 2023-09-23T20:23:21.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:21 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "19"}]': finished 2023-09-23T20:23:21.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:21 smithi018 ceph-mon[179647]: osdmap e124: 8 total, 8 up, 8 in 2023-09-23T20:23:21.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:21 smithi018 ceph-mon[179647]: failed to encode map e124 with expected crc 2023-09-23T20:23:21.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:21 smithi018 ceph-mon[179647]: failed to encode map e124 with expected crc 2023-09-23T20:23:21.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:21 smithi018 ceph-mon[179647]: failed to encode map e124 with expected crc 2023-09-23T20:23:21.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:21 smithi018 ceph-mon[179647]: failed to encode map e124 with expected crc 2023-09-23T20:23:21.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:21 smithi018 ceph-mon[179647]: osdmap e125: 8 total, 8 up, 8 in 2023-09-23T20:23:21.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:21 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:23:21.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:21 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:23:21.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:21 smithi018 ceph-mon[184144]: pgmap v256: 99 pgs: 99 active+clean; 2.4 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 9.2 MiB/s wr, 147 op/s 2023-09-23T20:23:21.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:21 smithi018 ceph-mon[184144]: 5.3 scrub starts 2023-09-23T20:23:21.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:21 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "19"}]': finished 2023-09-23T20:23:21.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:21 smithi018 ceph-mon[184144]: osdmap e124: 8 total, 8 up, 8 in 2023-09-23T20:23:21.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:21 smithi018 ceph-mon[184144]: failed to encode map e124 with expected crc 2023-09-23T20:23:21.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:21 smithi018 ceph-mon[184144]: failed to encode map e124 with expected crc 2023-09-23T20:23:21.394 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:21 smithi018 ceph-mon[184144]: failed to encode map e124 with expected crc 2023-09-23T20:23:21.394 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:21 smithi018 ceph-mon[184144]: failed to encode map e124 with expected crc 2023-09-23T20:23:21.394 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:21 smithi018 ceph-mon[184144]: osdmap e125: 8 total, 8 up, 8 in 2023-09-23T20:23:21.394 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:21 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:23:21.394 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:21 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:23:21.457 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 23 16 40539 40523 110.091 0 - 0.00861895 2023-09-23T20:23:21.758 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:23:21.860 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:23:21.862 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:21.863 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:23:21.928 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:23:21.963 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:23:22.062 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:22.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:22 smithi149 ceph-mon[170394]: failed to encode map e125 with expected crc 2023-09-23T20:23:22.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:22 smithi149 ceph-mon[170394]: failed to encode map e125 with expected crc 2023-09-23T20:23:22.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:22 smithi149 ceph-mon[170394]: failed to encode map e125 with expected crc 2023-09-23T20:23:22.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:22 smithi149 ceph-mon[170394]: failed to encode map e125 with expected crc 2023-09-23T20:23:22.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:22 smithi149 ceph-mon[170394]: 5.17 deep-scrub starts 2023-09-23T20:23:22.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:22 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "20"}]: dispatch 2023-09-23T20:23:22.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:22 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:23:22.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:22 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:23:22.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:22 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:23:22.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:22 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "20"}]': finished 2023-09-23T20:23:22.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:22 smithi149 ceph-mon[170394]: osdmap e126: 8 total, 8 up, 8 in 2023-09-23T20:23:22.296 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:23:22.300 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:22.300 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:23:22.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:22 smithi018 ceph-mon[179647]: failed to encode map e125 with expected crc 2023-09-23T20:23:22.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:22 smithi018 ceph-mon[179647]: failed to encode map e125 with expected crc 2023-09-23T20:23:22.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:22 smithi018 ceph-mon[179647]: failed to encode map e125 with expected crc 2023-09-23T20:23:22.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:22 smithi018 ceph-mon[179647]: failed to encode map e125 with expected crc 2023-09-23T20:23:22.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:22 smithi018 ceph-mon[179647]: 5.17 deep-scrub starts 2023-09-23T20:23:22.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:22 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "20"}]: dispatch 2023-09-23T20:23:22.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:22 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:23:22.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:22 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:23:22.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:22 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:23:22.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:22 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "20"}]': finished 2023-09-23T20:23:22.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:22 smithi018 ceph-mon[179647]: osdmap e126: 8 total, 8 up, 8 in 2023-09-23T20:23:22.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:22 smithi018 ceph-mon[184144]: failed to encode map e125 with expected crc 2023-09-23T20:23:22.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:22 smithi018 ceph-mon[184144]: failed to encode map e125 with expected crc 2023-09-23T20:23:22.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:22 smithi018 ceph-mon[184144]: failed to encode map e125 with expected crc 2023-09-23T20:23:22.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:22 smithi018 ceph-mon[184144]: failed to encode map e125 with expected crc 2023-09-23T20:23:22.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:22 smithi018 ceph-mon[184144]: 5.17 deep-scrub starts 2023-09-23T20:23:22.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:22 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "20"}]: dispatch 2023-09-23T20:23:22.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:22 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:23:22.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:22 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:23:22.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:22 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:23:22.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:22 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "20"}]': finished 2023-09-23T20:23:22.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:22 smithi018 ceph-mon[184144]: osdmap e126: 8 total, 8 up, 8 in 2023-09-23T20:23:22.402 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:23:22.446 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:22.457 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 24 16 41066 41050 106.877 16.4688 0.00321034 0.00926058 2023-09-23T20:23:22.495 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:23:22.498 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:22.499 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:23:22.599 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:23:22.633 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:22.861 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:23:22.865 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:22.866 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:23:22.967 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:23:23.004 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:23.051 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:23:23.055 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:23.056 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:23:23.063 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:23:23.157 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:23:23.167 INFO:teuthology.orchestra.run.smithi149.stdout:{ 2023-09-23T20:23:23.167 INFO:teuthology.orchestra.run.smithi149.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-09-23T20:23:23.167 INFO:teuthology.orchestra.run.smithi149.stdout:} 2023-09-23T20:23:23.190 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:23 smithi149 ceph-mon[170394]: pgmap v259: 99 pgs: 1 active+recovering+undersized+remapped, 98 active+clean; 2.4 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 1810/119643 objects misplaced (1.513%); 5.5 MiB/s, 87 objects/s recovering 2023-09-23T20:23:23.190 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:23 smithi149 ceph-mon[170394]: failed to encode map e126 with expected crc 2023-09-23T20:23:23.190 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:23 smithi149 ceph-mon[170394]: failed to encode map e126 with expected crc 2023-09-23T20:23:23.190 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:23 smithi149 ceph-mon[170394]: failed to encode map e126 with expected crc 2023-09-23T20:23:23.191 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:23 smithi149 ceph-mon[170394]: 5.17 scrub starts 2023-09-23T20:23:23.191 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:23 smithi149 ceph-mon[170394]: osdmap e127: 8 total, 8 up, 8 in 2023-09-23T20:23:23.194 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:23.233 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:23:23.239 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:23.239 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:23:23.255 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:23 smithi018 ceph-mon[184144]: pgmap v259: 99 pgs: 1 active+recovering+undersized+remapped, 98 active+clean; 2.4 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 1810/119643 objects misplaced (1.513%); 5.5 MiB/s, 87 objects/s recovering 2023-09-23T20:23:23.255 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:23 smithi018 ceph-mon[184144]: failed to encode map e126 with expected crc 2023-09-23T20:23:23.255 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:23 smithi018 ceph-mon[184144]: failed to encode map e126 with expected crc 2023-09-23T20:23:23.256 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:23 smithi018 ceph-mon[184144]: failed to encode map e126 with expected crc 2023-09-23T20:23:23.256 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:23 smithi018 ceph-mon[184144]: 5.17 scrub starts 2023-09-23T20:23:23.256 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:23 smithi018 ceph-mon[184144]: osdmap e127: 8 total, 8 up, 8 in 2023-09-23T20:23:23.256 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:23 smithi018 ceph-mon[179647]: pgmap v259: 99 pgs: 1 active+recovering+undersized+remapped, 98 active+clean; 2.4 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 1810/119643 objects misplaced (1.513%); 5.5 MiB/s, 87 objects/s recovering 2023-09-23T20:23:23.256 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:23 smithi018 ceph-mon[179647]: failed to encode map e126 with expected crc 2023-09-23T20:23:23.256 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:23 smithi018 ceph-mon[179647]: failed to encode map e126 with expected crc 2023-09-23T20:23:23.256 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:23 smithi018 ceph-mon[179647]: failed to encode map e126 with expected crc 2023-09-23T20:23:23.256 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:23 smithi018 ceph-mon[179647]: 5.17 scrub starts 2023-09-23T20:23:23.256 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:23 smithi018 ceph-mon[179647]: osdmap e127: 8 total, 8 up, 8 in 2023-09-23T20:23:23.340 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:23:23.369 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:23.457 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 25 16 41092 41076 102.667 1.625 0.0033707 0.0092566 2023-09-23T20:23:23.513 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:23:23.516 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:23.516 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:23:23.528 INFO:teuthology.orchestra.run.smithi018.stdout:ERROR: (22) Invalid argument 2023-09-23T20:23:23.528 INFO:teuthology.orchestra.run.smithi018.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:23:23.617 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:23:23.654 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:23.871 INFO:teuthology.orchestra.run.smithi018.stderr:noscrub is set 2023-09-23T20:23:23.897 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:23:23.901 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:23.901 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:23:24.000 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T20:23:24.002 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:23:24.119 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:24.228 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:23:24.233 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:24.233 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:23:24.334 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:23:24.361 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:24 smithi149 ceph-mon[170394]: failed to encode map e127 with expected crc 2023-09-23T20:23:24.361 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:24 smithi149 ceph-mon[170394]: failed to encode map e127 with expected crc 2023-09-23T20:23:24.361 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:24 smithi149 ceph-mon[170394]: failed to encode map e127 with expected crc 2023-09-23T20:23:24.361 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:24 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/921331371' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:23:24.361 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:24 smithi149 ceph-mon[170394]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:23:24.361 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:24 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/921331371' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:23:24.362 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:24 smithi149 ceph-mon[170394]: osdmap e128: 8 total, 8 up, 8 in 2023-09-23T20:23:24.364 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:24.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:24 smithi018 ceph-mon[184144]: failed to encode map e127 with expected crc 2023-09-23T20:23:24.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:24 smithi018 ceph-mon[184144]: failed to encode map e127 with expected crc 2023-09-23T20:23:24.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:24 smithi018 ceph-mon[184144]: failed to encode map e127 with expected crc 2023-09-23T20:23:24.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:24 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/921331371' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:23:24.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:24 smithi018 ceph-mon[184144]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:23:24.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:24 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/921331371' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:23:24.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:24 smithi018 ceph-mon[184144]: osdmap e128: 8 total, 8 up, 8 in 2023-09-23T20:23:24.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:24 smithi018 ceph-mon[179647]: failed to encode map e127 with expected crc 2023-09-23T20:23:24.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:24 smithi018 ceph-mon[179647]: failed to encode map e127 with expected crc 2023-09-23T20:23:24.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:24 smithi018 ceph-mon[179647]: failed to encode map e127 with expected crc 2023-09-23T20:23:24.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:24 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/921331371' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:23:24.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:24 smithi018 ceph-mon[179647]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:23:24.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:24 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/921331371' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:23:24.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:24 smithi018 ceph-mon[179647]: osdmap e128: 8 total, 8 up, 8 in 2023-09-23T20:23:24.402 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:23:24.405 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:24.405 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:23:24.457 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 26 16 41516 41500 99.7379 26.5 0.0026051 0.00989877 2023-09-23T20:23:24.507 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:23:24.542 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:24.688 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:23:24.692 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:24.692 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:23:24.794 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:23:24.821 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:24.855 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:23:24.857 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:24.858 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:23:24.903 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:23:24.959 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:23:24.997 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:25.048 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:23:25.054 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:25.054 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:23:25.064 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:23:25.103 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:23:25.149 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:23:25.155 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:23:25.204 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:23:25.243 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:25.251 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:23:25.296 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:23:25.319 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:23:25.325 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:25 smithi018 ceph-mon[179647]: pgmap v262: 99 pgs: 1 remapped+peering, 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 95 active+clean; 2.5 GiB data, 7.7 GiB used, 707 GiB / 715 GiB avail; 5302/121737 objects misplaced (4.355%); 7.3 MiB/s, 116 objects/s recovering 2023-09-23T20:23:25.325 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:25 smithi018 ceph-mon[179647]: failed to encode map e128 with expected crc 2023-09-23T20:23:25.325 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:25 smithi018 ceph-mon[179647]: failed to encode map e128 with expected crc 2023-09-23T20:23:25.325 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:25 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:23:25.325 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:25 smithi018 ceph-mon[179647]: osdmap e129: 8 total, 8 up, 8 in 2023-09-23T20:23:25.325 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:25 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2554839329' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:23:25.326 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:25.326 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:23:25.326 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:23:24 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:23:24] "GET /metrics HTTP/1.1" 200 36458 "" "Prometheus/2.43.0" 2023-09-23T20:23:25.327 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:25 smithi018 ceph-mon[184144]: pgmap v262: 99 pgs: 1 remapped+peering, 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 95 active+clean; 2.5 GiB data, 7.7 GiB used, 707 GiB / 715 GiB avail; 5302/121737 objects misplaced (4.355%); 7.3 MiB/s, 116 objects/s recovering 2023-09-23T20:23:25.327 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:25 smithi018 ceph-mon[184144]: failed to encode map e128 with expected crc 2023-09-23T20:23:25.327 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:25 smithi018 ceph-mon[184144]: failed to encode map e128 with expected crc 2023-09-23T20:23:25.327 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:25 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:23:25.327 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:25 smithi018 ceph-mon[184144]: osdmap e129: 8 total, 8 up, 8 in 2023-09-23T20:23:25.327 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:25 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2554839329' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:23:25.340 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:23:25.380 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:23:25.387 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:23:25.387 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:23:25.387 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:23:25.387 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:23:25.387 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:23:25.387 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:23:25.388 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:23:25.388 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:23:25.427 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:23:25.459 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 27 16 42405 42389 98.0959 55.5625 0.0130817 0.010189 2023-09-23T20:23:25.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:25 smithi149 ceph-mon[170394]: pgmap v262: 99 pgs: 1 remapped+peering, 1 active+recovery_wait+remapped, 2 active+recovering+undersized+remapped, 95 active+clean; 2.5 GiB data, 7.7 GiB used, 707 GiB / 715 GiB avail; 5302/121737 objects misplaced (4.355%); 7.3 MiB/s, 116 objects/s recovering 2023-09-23T20:23:25.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:25 smithi149 ceph-mon[170394]: failed to encode map e128 with expected crc 2023-09-23T20:23:25.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:25 smithi149 ceph-mon[170394]: failed to encode map e128 with expected crc 2023-09-23T20:23:25.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:25 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:23:25.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:25 smithi149 ceph-mon[170394]: osdmap e129: 8 total, 8 up, 8 in 2023-09-23T20:23:25.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:25 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2554839329' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:23:25.493 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:26.050 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:23:26.053 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:26.054 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:23:26.155 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:23:26.226 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:26.346 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:23:26.352 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:26.352 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:26 smithi018 ceph-mon[179647]: failed to encode map e129 with expected crc 2023-09-23T20:23:26.352 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:26 smithi018 ceph-mon[179647]: failed to encode map e129 with expected crc 2023-09-23T20:23:26.353 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:26 smithi018 ceph-mon[184144]: failed to encode map e129 with expected crc 2023-09-23T20:23:26.353 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:26 smithi018 ceph-mon[184144]: failed to encode map e129 with expected crc 2023-09-23T20:23:26.353 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:23:26.454 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:23:26.461 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 28 16 44136 44120 98.4482 108.188 0.0179935 0.0101518 2023-09-23T20:23:26.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:26 smithi149 ceph-mon[170394]: failed to encode map e129 with expected crc 2023-09-23T20:23:26.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:26 smithi149 ceph-mon[170394]: failed to encode map e129 with expected crc 2023-09-23T20:23:26.497 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:26.560 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:23:26.566 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:26.567 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:23:26.668 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:23:26.746 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:26.754 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T20:23:26.829 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:23:26.833 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2023-09-23T20:23:26.833 INFO:teuthology.orchestra.run.smithi018.stdout: "ops": [ 2023-09-23T20:23:26.834 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:26.834 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.34791.0:44694 5.0 5:0784aa94:::benchmark_data_smithi018_228451_object44693:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e129)", 2023-09-23T20:23:26.834 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:23:26.828837+0000", 2023-09-23T20:23:26.834 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.0046479060000000003, 2023-09-23T20:23:26.834 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0046840859999999996, 2023-09-23T20:23:26.834 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:23:26.834 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:23:26.834 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:23:26.834 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.34791", 2023-09-23T20:23:26.835 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:23:26.835 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 44694 2023-09-23T20:23:26.835 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:26.835 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:23:26.835 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:26.835 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:23:26.835 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:26.828837+0000", 2023-09-23T20:23:26.835 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:23:26.835 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:26.835 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:26.836 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:23:26.836 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:26.828837+0000", 2023-09-23T20:23:26.836 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:23:26.836 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:26.836 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:26.836 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:23:26.836 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:26.828836+0000", 2023-09-23T20:23:26.836 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:23:26.836 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:26.836 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:26.837 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:23:26.837 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:26.828866+0000", 2023-09-23T20:23:26.837 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.0416000000000002e-05 2023-09-23T20:23:26.837 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:26.837 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:26.837 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:23:26.837 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:26.828884+0000", 2023-09-23T20:23:26.837 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.7671000000000001e-05 2023-09-23T20:23:26.837 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:26.837 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:26.838 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:23:26.838 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:26.828893+0000", 2023-09-23T20:23:26.838 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.5089999999999999e-06 2023-09-23T20:23:26.838 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:26.838 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:26.838 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:23:26.838 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:26.828946+0000", 2023-09-23T20:23:26.838 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.2509999999999997e-05 2023-09-23T20:23:26.838 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:26.838 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:26.839 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:23:26.839 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:26.829097+0000", 2023-09-23T20:23:26.839 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00015146800000000001 2023-09-23T20:23:26.839 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:26.839 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:26.839 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,4", 2023-09-23T20:23:26.839 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:26.829249+0000", 2023-09-23T20:23:26.839 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000152237 2023-09-23T20:23:26.839 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:26.839 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:26.840 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:23:26.840 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:26.831093+0000", 2023-09-23T20:23:26.840 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0018433939999999999 2023-09-23T20:23:26.840 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:26.840 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:26.840 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:23:26.840 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:26.832070+0000", 2023-09-23T20:23:26.840 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00097696200000000006 2023-09-23T20:23:26.840 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:23:26.840 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:23:26.841 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:23:26.841 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:26.841 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:26.841 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.34791.0:44707 5.11 5:8d58c443:::benchmark_data_smithi018_228451_object44706:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e129)", 2023-09-23T20:23:26.841 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:23:26.832243+0000", 2023-09-23T20:23:26.841 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.001242202, 2023-09-23T20:23:26.841 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.001341472, 2023-09-23T20:23:26.841 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:23:26.841 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:23:26.841 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:23:26.841 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.34791", 2023-09-23T20:23:26.842 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:23:26.842 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 44707 2023-09-23T20:23:26.842 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:26.842 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:23:26.842 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:26.842 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:23:26.842 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:26.832243+0000", 2023-09-23T20:23:26.842 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:23:26.842 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:26.842 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:26.843 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:23:26.843 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:26.832243+0000", 2023-09-23T20:23:26.843 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:23:26.843 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:26.843 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:26.843 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:23:26.843 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:26.832242+0000", 2023-09-23T20:23:26.843 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:23:26.843 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:26.843 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:26.844 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:23:26.844 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:26.832263+0000", 2023-09-23T20:23:26.844 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.0996e-05 2023-09-23T20:23:26.844 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:26.844 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:26.844 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:23:26.844 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:26.832273+0000", 2023-09-23T20:23:26.844 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0389e-05 2023-09-23T20:23:26.844 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:26.844 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:26.845 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:23:26.845 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:26.832278+0000", 2023-09-23T20:23:26.845 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.2739999999999998e-06 2023-09-23T20:23:26.845 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:26.845 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:26.845 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:23:26.845 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:26.832680+0000", 2023-09-23T20:23:26.845 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00040163500000000001 2023-09-23T20:23:26.845 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:26.845 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:26.846 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:23:26.846 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:26.832841+0000", 2023-09-23T20:23:26.846 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00016080599999999999 2023-09-23T20:23:26.846 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:23:26.846 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:23:26.846 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 4,6,7", 2023-09-23T20:23:26.846 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:26.832999+0000", 2023-09-23T20:23:26.846 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00015810499999999999 2023-09-23T20:23:26.846 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:23:26.846 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:23:26.847 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:23:26.847 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:23:26.847 INFO:teuthology.orchestra.run.smithi018.stdout: ], 2023-09-23T20:23:26.847 INFO:teuthology.orchestra.run.smithi018.stdout: "num_ops": 2 2023-09-23T20:23:26.847 INFO:teuthology.orchestra.run.smithi018.stdout:} 2023-09-23T20:23:26.847 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:26.848 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:23:26.949 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:23:26.988 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:27.041 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:23:27.044 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:27.044 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:23:27.145 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:23:27.219 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:27.295 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:27 smithi018 ceph-mon[179647]: pgmap v265: 99 pgs: 1 remapped+peering, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 96 active+clean; 2.5 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 3492/124173 objects misplaced (2.812%); 16 MiB/s, 255 objects/s recovering 2023-09-23T20:23:27.296 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:27 smithi018 ceph-mon[184144]: pgmap v265: 99 pgs: 1 remapped+peering, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 96 active+clean; 2.5 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 3492/124173 objects misplaced (2.812%); 16 MiB/s, 255 objects/s recovering 2023-09-23T20:23:27.296 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:23:27.308 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T20:23:27.311 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:27.311 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:23:27.413 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:23:27.461 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 29 16 45631 45615 98.2751 93.4375 0.0120176 0.0101703 2023-09-23T20:23:27.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:27 smithi149 ceph-mon[170394]: pgmap v265: 99 pgs: 1 remapped+peering, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 96 active+clean; 2.5 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 3492/124173 objects misplaced (2.812%); 16 MiB/s, 255 objects/s recovering 2023-09-23T20:23:27.532 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:27.605 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:23:27.611 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:27.612 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:23:27.627 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:23:27.713 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:23:27.760 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:27.813 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:23:27.819 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:27.820 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:23:27.921 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:23:27.984 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:28.058 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:23:28.061 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:28.061 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:23:28.145 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:23:28.162 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:23:28.201 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:28.249 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:23:28.254 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:28.254 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:23:28.355 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:23:28.391 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:28.456 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:23:28.461 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:28.462 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:23:28.462 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 30 16 47174 47158 98.2136 96.4375 0.0103793 0.0101765 2023-09-23T20:23:28.562 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:23:28.577 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-23T20:23:28.577 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:23:28.577 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_0 2023-09-23T20:23:28.578 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:23:28.605 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:23:28.646 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:28.781 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:23:28.790 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:28.791 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:23:28.891 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:23:28.935 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:28.983 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:23:28.986 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:28.986 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:23:29.087 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:23:29.124 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:29.173 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:23:29.178 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:29.178 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:23:29.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:29 smithi149 ceph-mon[170394]: pgmap v266: 99 pgs: 1 remapped+peering, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 96 active+clean; 2.6 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 3492/128208 objects misplaced (2.724%); 11 MiB/s, 182 objects/s recovering 2023-09-23T20:23:29.279 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:23:29.283 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:29 smithi018 ceph-mon[184144]: pgmap v266: 99 pgs: 1 remapped+peering, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 96 active+clean; 2.6 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 3492/128208 objects misplaced (2.724%); 11 MiB/s, 182 objects/s recovering 2023-09-23T20:23:29.283 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:29 smithi018 ceph-mon[179647]: pgmap v266: 99 pgs: 1 remapped+peering, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 96 active+clean; 2.6 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 3492/128208 objects misplaced (2.724%); 11 MiB/s, 182 objects/s recovering 2023-09-23T20:23:29.298 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:23:29.386 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:29.464 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 31 16 48466 48450 97.6415 80.75 0.00552192 0.0102363 2023-09-23T20:23:30.064 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:23:30.069 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:30.069 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:23:30.098 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:23:30.152 INFO:teuthology.orchestra.run.smithi018.stderr:nodeep-scrub is set 2023-09-23T20:23:30.170 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:23:30.207 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:30.250 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:23:30.255 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:30.255 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:23:30.356 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:23:30.377 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:30 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:23:30.377 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:30 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1548989782' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:23:30.377 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:30 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:23:30.377 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:30 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1548989782' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:23:30.390 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:30 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:23:30.390 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:30 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1548989782' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:23:30.394 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:30.433 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:23:30.436 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:30.436 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:23:30.464 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 32 16 50065 50049 97.713 99.9375 0.00198117 0.0101856 2023-09-23T20:23:30.538 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:23:30.566 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:30.602 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:23:30.606 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:30.606 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:23:30.707 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:23:30.733 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:30.764 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:23:30.767 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:30.767 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:23:30.867 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:23:30.906 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:30.998 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:23:31.002 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:31.002 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:23:31.103 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:23:31.136 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:31.268 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:31 smithi018 ceph-mon[184144]: pgmap v267: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 96 active+clean; 2.7 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 24 MiB/s wr, 387 op/s; 2620/131634 objects degraded (1.990%); 1320/131634 objects misplaced (1.003%); 25 MiB/s, 393 objects/s recovering 2023-09-23T20:23:31.269 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:31 smithi018 ceph-mon[184144]: Health check failed: Degraded data redundancy: 2620/131634 objects degraded (1.990%), 1 pg degraded (PG_DEGRADED) 2023-09-23T20:23:31.269 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:31 smithi018 ceph-mon[184144]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:23:31.269 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:31 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1548989782' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:23:31.269 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:31 smithi018 ceph-mon[184144]: osdmap e130: 8 total, 8 up, 8 in 2023-09-23T20:23:31.269 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:31 smithi018 ceph-mon[184144]: failed to encode map e130 with expected crc 2023-09-23T20:23:31.269 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:31 smithi018 ceph-mon[184144]: failed to encode map e130 with expected crc 2023-09-23T20:23:31.270 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:31 smithi018 ceph-mon[179647]: pgmap v267: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 96 active+clean; 2.7 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 24 MiB/s wr, 387 op/s; 2620/131634 objects degraded (1.990%); 1320/131634 objects misplaced (1.003%); 25 MiB/s, 393 objects/s recovering 2023-09-23T20:23:31.270 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:31 smithi018 ceph-mon[179647]: Health check failed: Degraded data redundancy: 2620/131634 objects degraded (1.990%), 1 pg degraded (PG_DEGRADED) 2023-09-23T20:23:31.270 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:31 smithi018 ceph-mon[179647]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:23:31.270 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:31 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1548989782' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:23:31.270 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:31 smithi018 ceph-mon[179647]: osdmap e130: 8 total, 8 up, 8 in 2023-09-23T20:23:31.270 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:31 smithi018 ceph-mon[179647]: failed to encode map e130 with expected crc 2023-09-23T20:23:31.270 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:31 smithi018 ceph-mon[179647]: failed to encode map e130 with expected crc 2023-09-23T20:23:31.464 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 33 16 50619 50603 95.8017 34.625 0.0027977 0.0103946 2023-09-23T20:23:31.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:31 smithi149 ceph-mon[170394]: pgmap v267: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+remapped, 96 active+clean; 2.7 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 24 MiB/s wr, 387 op/s; 2620/131634 objects degraded (1.990%); 1320/131634 objects misplaced (1.003%); 25 MiB/s, 393 objects/s recovering 2023-09-23T20:23:31.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:31 smithi149 ceph-mon[170394]: Health check failed: Degraded data redundancy: 2620/131634 objects degraded (1.990%), 1 pg degraded (PG_DEGRADED) 2023-09-23T20:23:31.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:31 smithi149 ceph-mon[170394]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:23:31.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:31 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1548989782' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:23:31.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:31 smithi149 ceph-mon[170394]: osdmap e130: 8 total, 8 up, 8 in 2023-09-23T20:23:31.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:31 smithi149 ceph-mon[170394]: failed to encode map e130 with expected crc 2023-09-23T20:23:31.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:31 smithi149 ceph-mon[170394]: failed to encode map e130 with expected crc 2023-09-23T20:23:31.484 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:23:31.489 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:31.489 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:23:31.545 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2023-09-23T20:23:31.546 INFO:teuthology.orchestra.run.smithi018.stdout: "ops": [], 2023-09-23T20:23:31.546 INFO:teuthology.orchestra.run.smithi018.stdout: "complaint_time": 30, 2023-09-23T20:23:31.546 INFO:teuthology.orchestra.run.smithi018.stdout: "num_blocked_ops": 0 2023-09-23T20:23:31.546 INFO:teuthology.orchestra.run.smithi018.stdout:} 2023-09-23T20:23:31.589 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:23:31.636 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:31.943 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:23:31.947 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:31.948 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:23:31.971 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T20:23:32.049 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:23:32.084 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:32.179 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:23:32.183 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:32.184 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:23:32.184 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:32 smithi018 ceph-mon[184144]: failed to encode map e131 with expected crc 2023-09-23T20:23:32.185 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:32 smithi018 ceph-mon[184144]: osdmap e131: 8 total, 8 up, 8 in 2023-09-23T20:23:32.185 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:32 smithi018 ceph-mon[184144]: failed to encode map e131 with expected crc 2023-09-23T20:23:32.185 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:32 smithi018 ceph-mon[184144]: failed to encode map e131 with expected crc 2023-09-23T20:23:32.185 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:32 smithi018 ceph-mon[184144]: failed to encode map e131 with expected crc 2023-09-23T20:23:32.185 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:32 smithi018 ceph-mon[184144]: failed to encode map e131 with expected crc 2023-09-23T20:23:32.185 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:32 smithi018 ceph-mon[184144]: pgmap v270: 99 pgs: 1 creating+peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 96 active+clean; 2.9 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s; 2620/143040 objects degraded (1.832%); 1000/143040 objects misplaced (0.699%); 26 MiB/s, 423 objects/s recovering 2023-09-23T20:23:32.185 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:32 smithi018 ceph-mon[179647]: failed to encode map e131 with expected crc 2023-09-23T20:23:32.186 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:32 smithi018 ceph-mon[179647]: osdmap e131: 8 total, 8 up, 8 in 2023-09-23T20:23:32.186 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:32 smithi018 ceph-mon[179647]: failed to encode map e131 with expected crc 2023-09-23T20:23:32.186 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:32 smithi018 ceph-mon[179647]: failed to encode map e131 with expected crc 2023-09-23T20:23:32.186 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:32 smithi018 ceph-mon[179647]: failed to encode map e131 with expected crc 2023-09-23T20:23:32.186 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:32 smithi018 ceph-mon[179647]: failed to encode map e131 with expected crc 2023-09-23T20:23:32.186 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:32 smithi018 ceph-mon[179647]: pgmap v270: 99 pgs: 1 creating+peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 96 active+clean; 2.9 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s; 2620/143040 objects degraded (1.832%); 1000/143040 objects misplaced (0.699%); 26 MiB/s, 423 objects/s recovering 2023-09-23T20:23:32.285 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:23:32.373 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:32.464 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 34 16 51095 51079 93.8595 29.75 0.0263214 0.0106484 2023-09-23T20:23:32.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:32 smithi149 ceph-mon[170394]: failed to encode map e131 with expected crc 2023-09-23T20:23:32.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:32 smithi149 ceph-mon[170394]: osdmap e131: 8 total, 8 up, 8 in 2023-09-23T20:23:32.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:32 smithi149 ceph-mon[170394]: failed to encode map e131 with expected crc 2023-09-23T20:23:32.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:32 smithi149 ceph-mon[170394]: failed to encode map e131 with expected crc 2023-09-23T20:23:32.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:32 smithi149 ceph-mon[170394]: failed to encode map e131 with expected crc 2023-09-23T20:23:32.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:32 smithi149 ceph-mon[170394]: failed to encode map e131 with expected crc 2023-09-23T20:23:32.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:32 smithi149 ceph-mon[170394]: pgmap v270: 99 pgs: 1 creating+peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 96 active+clean; 2.9 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s; 2620/143040 objects degraded (1.832%); 1000/143040 objects misplaced (0.699%); 26 MiB/s, 423 objects/s recovering 2023-09-23T20:23:32.527 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:23:32.539 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:32.540 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:23:32.546 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:23:32.597 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:23:32.641 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:23:32.642 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:23:32.674 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:32.695 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:23:32.733 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:23:32.733 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:23:32.740 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:32.740 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:23:32.776 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:23:32.779 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:23:32.825 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:23:32.841 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:23:32.864 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:23:32.876 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:32.878 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:23:32.879 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:23:32.879 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:23:32.879 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:23:32.879 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:23:32.879 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:23:32.879 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:23:32.879 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:23:32.924 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:23:32.927 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:32.928 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:23:33.003 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:23:33.007 INFO:teuthology.orchestra.run.smithi018.stderr:dumped all 2023-09-23T20:23:33.028 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:23:33.045 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T20:23:33.115 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:33.163 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:33 smithi018 ceph-mon[179647]: failed to encode map e132 with expected crc 2023-09-23T20:23:33.163 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:33 smithi018 ceph-mon[179647]: failed to encode map e132 with expected crc 2023-09-23T20:23:33.163 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:33 smithi018 ceph-mon[179647]: osdmap e132: 8 total, 8 up, 8 in 2023-09-23T20:23:33.163 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:33 smithi018 ceph-mon[179647]: failed to encode map e132 with expected crc 2023-09-23T20:23:33.163 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:33 smithi018 ceph-mon[179647]: failed to encode map e132 with expected crc 2023-09-23T20:23:33.163 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:33 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/743992404' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:23:33.164 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:33 smithi018 ceph-mon[179647]: from='client.44747 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:23:33.164 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:33 smithi018 ceph-mon[184144]: failed to encode map e132 with expected crc 2023-09-23T20:23:33.164 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:33 smithi018 ceph-mon[184144]: failed to encode map e132 with expected crc 2023-09-23T20:23:33.164 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:33 smithi018 ceph-mon[184144]: osdmap e132: 8 total, 8 up, 8 in 2023-09-23T20:23:33.164 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:33 smithi018 ceph-mon[184144]: failed to encode map e132 with expected crc 2023-09-23T20:23:33.164 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:33 smithi018 ceph-mon[184144]: failed to encode map e132 with expected crc 2023-09-23T20:23:33.164 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:33 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/743992404' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:23:33.164 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:33 smithi018 ceph-mon[184144]: from='client.44747 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:23:33.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:33 smithi149 ceph-mon[170394]: failed to encode map e132 with expected crc 2023-09-23T20:23:33.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:33 smithi149 ceph-mon[170394]: failed to encode map e132 with expected crc 2023-09-23T20:23:33.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:33 smithi149 ceph-mon[170394]: osdmap e132: 8 total, 8 up, 8 in 2023-09-23T20:23:33.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:33 smithi149 ceph-mon[170394]: failed to encode map e132 with expected crc 2023-09-23T20:23:33.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:33 smithi149 ceph-mon[170394]: failed to encode map e132 with expected crc 2023-09-23T20:23:33.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:33 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/743992404' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:23:33.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:33 smithi149 ceph-mon[170394]: from='client.44747 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:23:33.449 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:23:33.455 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:33.456 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:23:33.464 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 35 16 52523 52507 93.7276 89.25 0.0137802 0.0106638 2023-09-23T20:23:33.557 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:23:33.622 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:33.679 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:23:33.684 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:33.685 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:23:33.786 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:23:33.851 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:33.975 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:23:33.981 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:33.982 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:23:34.067 INFO:teuthology.orchestra.run.smithi018.stdout:{"pg_ready":true,"pg_map":{"version":271,"stamp":"2023-09-23T20:23:32.142581+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3245250483,"num_objects":49474,"num_object_clones":0,"num_object_copies":148422,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2620,"num_objects_misplaced":1000,"num_objects_unfound":0,"num_objects_dirty":49474,"num_whiteouts":0,"num_read":6495,"num_read_kb":14136,"num_write":37115,"num_write_kb":2317125,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3267,"num_bytes_recovered":215484000,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":68829,"ondisk_log_size":68829,"up":297,"acting":295,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":304,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":10116240,"kb_used_data":9777040,"kb_used_omap":30,"kb_used_meta":339105,"kb_avail":739844976,"statfs":{"total":767960285184,"available":757601255424,"internally_reserved":0,"allocated":10011688960,"data_stored":10007110755,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31263,"internal_metadata":347244001},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[10,3,1,0,5],"upper_bound":32},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":529727488,"num_objects":8083,"num_object_clones":0,"num_object_copies":24249,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2620,"num_objects_misplaced":-2492,"num_objects_unfound":0,"num_objects_dirty":8083,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8083,"num_write_kb":517312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1887,"num_bytes_recovered":123666432,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.695172"},"pg_stats":[{"pgid":"3.1a","version":"69'1","reported_seq":107,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T20:23:24.866672+0000","last_change":"2023-09-23T20:21:18.891671+0000","last_active":"2023-09-23T20:23:24.866672+0000","last_peered":"2023-09-23T20:23:24.866672+0000","last_clean":"2023-09-23T20:23:24.866672+0000","last_became_active":"2023-09-23T20:21:18.891285+0000","last_became_peered":"2023-09-23T20:21:18.891285+0000","last_unstale":"2023-09-23T20:23:24.866672+0000","last_undegraded":"2023-09-23T20:23:24.866672+0000","last_fullsized":"2023-09-23T20:23:24.866672+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:04:49.595473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":91,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T20:23:23.881443+0000","last_change":"2023-09-23T20:21:49.951483+0000","last_active":"2023-09-23T20:23:23.881443+0000","last_peered":"2023-09-23T20:23:23.881443+0000","last_clean":"2023-09-23T20:23:23.881443+0000","last_became_active":"2023-09-23T20:21:49.951167+0000","last_became_peered":"2023-09-23T20:21:49.951167+0000","last_unstale":"2023-09-23T20:23:23.881443+0000","last_undegraded":"2023-09-23T20:23:23.881443+0000","last_fullsized":"2023-09-23T20:23:23.881443+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:08:19.552997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":67,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T20:23:23.866880+0000","last_change":"2023-09-23T20:21:49.955411+0000","last_active":"2023-09-23T20:23:23.866880+0000","last_peered":"2023-09-23T20:23:23.866880+0000","last_clean":"2023-09-23T20:23:23.866880+0000","last_became_active":"2023-09-23T20:21:49.955112+0000","last_became_peered":"2023-09-23T20:21:49.955112+0000","last_unstale":"2023-09-23T20:23:23.866880+0000","last_undegraded":"2023-09-23T20:23:23.866880+0000","last_fullsized":"2023-09-23T20:23:23.866880+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:47:53.870056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":72,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T20:23:23.861761+0000","last_change":"2023-09-23T20:21:49.954419+0000","last_active":"2023-09-23T20:23:23.861761+0000","last_peered":"2023-09-23T20:23:23.861761+0000","last_clean":"2023-09-23T20:23:23.861761+0000","last_became_active":"2023-09-23T20:21:49.953941+0000","last_became_peered":"2023-09-23T20:21:49.953941+0000","last_unstale":"2023-09-23T20:23:23.861761+0000","last_undegraded":"2023-09-23T20:23:23.861761+0000","last_fullsized":"2023-09-23T20:23:23.861761+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:05:38.809180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":109,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T20:23:23.877295+0000","last_change":"2023-09-23T20:21:49.958727+0000","last_active":"2023-09-23T20:23:23.877295+0000","last_peered":"2023-09-23T20:23:23.877295+0000","last_clean":"2023-09-23T20:23:23.877295+0000","last_became_active":"2023-09-23T20:21:49.957722+0000","last_became_peered":"2023-09-23T20:21:49.957722+0000","last_unstale":"2023-09-23T20:23:23.877295+0000","last_undegraded":"2023-09-23T20:23:23.877295+0000","last_fullsized":"2023-09-23T20:23:23.877295+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:47:13.589014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":89,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T20:23:24.866024+0000","last_change":"2023-09-23T20:20:46.937089+0000","last_active":"2023-09-23T20:23:24.866024+0000","last_peered":"2023-09-23T20:23:24.866024+0000","last_clean":"2023-09-23T20:23:24.866024+0000","last_became_active":"2023-09-23T20:20:46.936831+0000","last_became_peered":"2023-09-23T20:20:46.936831+0000","last_unstale":"2023-09-23T20:23:24.866024+0000","last_undegraded":"2023-09-23T20:23:24.866024+0000","last_fullsized":"2023-09-23T20:23:24.866024+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:34:56.747010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.11","version":"129'2527","reported_seq":5188,"reported_epoch":130,"state":"creating+peering","last_fresh":"2023-09-23T20:23:30.148943+0000","last_change":"2023-09-23T20:23:30.148802+0000","last_active":"2023-09-23T20:23:12.353414+0000","last_peered":"2023-09-23T20:23:12.353414+0000","last_clean":"2023-09-23T20:23:12.353414+0000","last_became_active":"2023-09-23T20:23:20.088723+0000","last_became_peered":"2023-09-23T20:23:20.088723+0000","last_unstale":"2023-09-23T20:23:30.148943+0000","last_undegraded":"2023-09-23T20:23:30.148943+0000","last_fullsized":"2023-09-23T20:23:30.148943+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":118,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":5,"last_scrub":"119'1977","last_scrub_stamp":"2023-09-23T20:23:12.353414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:23:12.353414+0000","objects_scrubbed":1018,"log_size":2527,"log_dups_size":0,"ondisk_log_size":2527,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:55:34.477458+0000","scrub_duration":0.26235630100000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":103415808,"num_objects":1578,"num_object_clones":0,"num_object_copies":4734,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1578,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":560,"num_write_kb":35840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1282,"num_bytes_recovered":84017152,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[3],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":71,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T20:23:23.862190+0000","last_change":"2023-09-23T20:21:49.950402+0000","last_active":"2023-09-23T20:23:23.862190+0000","last_peered":"2023-09-23T20:23:23.862190+0000","last_clean":"2023-09-23T20:23:23.862190+0000","last_became_active":"2023-09-23T20:21:49.950116+0000","last_became_peered":"2023-09-23T20:21:49.950116+0000","last_unstale":"2023-09-23T20:23:23.862190+0000","last_undegraded":"2023-09-23T20:23:23.862190+0000","last_fullsized":"2023-09-23T20:23:23.862190+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:10:20.322615+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":91,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T20:23:23.877215+0000","last_change":"2023-09-23T20:21:49.958837+0000","last_active":"2023-09-23T20:23:23.877215+0000","last_peered":"2023-09-23T20:23:23.877215+0000","last_clean":"2023-09-23T20:23:23.877215+0000","last_became_active":"2023-09-23T20:21:49.958135+0000","last_became_peered":"2023-09-23T20:21:49.958135+0000","last_unstale":"2023-09-23T20:23:23.877215+0000","last_undegraded":"2023-09-23T20:23:23.877215+0000","last_fullsized":"2023-09-23T20:23:23.877215+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:31:52.350275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.10","version":"130'2523","reported_seq":5456,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-09-23T20:23:30.302715+0000","last_change":"2023-09-23T20:23:24.880162+0000","last_active":"2023-09-23T20:23:30.302715+0000","last_peered":"2023-09-23T20:23:30.302715+0000","last_clean":"2023-09-23T20:23:30.302715+0000","last_became_active":"2023-09-23T20:23:24.867240+0000","last_became_peered":"2023-09-23T20:23:24.867240+0000","last_unstale":"2023-09-23T20:23:30.302715+0000","last_undegraded":"2023-09-23T20:23:30.302715+0000","last_fullsized":"2023-09-23T20:23:30.302715+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":118,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":5,"last_scrub":"119'2122","last_scrub_stamp":"2023-09-23T20:23:15.108101+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:23:15.108101+0000","objects_scrubbed":1199,"log_size":2523,"log_dups_size":0,"ondisk_log_size":2523,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:10:57.070009+0000","scrub_duration":0.30476455699999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":105119744,"num_objects":1604,"num_object_clones":0,"num_object_copies":4812,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1604,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":405,"num_write_kb":25920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1255,"num_bytes_recovered":82247680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":89,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T20:23:24.884866+0000","last_change":"2023-09-23T20:20:46.950281+0000","last_active":"2023-09-23T20:23:24.884866+0000","last_peered":"2023-09-23T20:23:24.884866+0000","last_clean":"2023-09-23T20:23:24.884866+0000","last_became_active":"2023-09-23T20:20:46.949892+0000","last_became_peered":"2023-09-23T20:20:46.949892+0000","last_unstale":"2023-09-23T20:23:24.884866+0000","last_undegraded":"2023-09-23T20:23:24.884866+0000","last_fullsized":"2023-09-23T20:23:24.884866+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:28.763832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":42,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T20:23:23.877940+0000","last_change":"2023-09-23T20:21:49.955179+0000","last_active":"2023-09-23T20:23:23.877940+0000","last_peered":"2023-09-23T20:23:23.877940+0000","last_clean":"2023-09-23T20:23:23.877940+0000","last_became_active":"2023-09-23T20:21:49.954878+0000","last_became_peered":"2023-09-23T20:21:49.954878+0000","last_unstale":"2023-09-23T20:23:23.877940+0000","last_undegraded":"2023-09-23T20:23:23.877940+0000","last_fullsized":"2023-09-23T20:23:23.877940+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:02:26.579246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.13","version":"129'2352","reported_seq":2393,"reported_epoch":129,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:23:27.859944+0000","last_change":"2023-09-23T20:23:23.931641+0000","last_active":"2023-09-23T20:23:27.859944+0000","last_peered":"2023-09-23T20:23:27.859944+0000","last_clean":"2023-09-23T20:23:20.858048+0000","last_became_active":"2023-09-23T20:23:23.928214+0000","last_became_peered":"2023-09-23T20:23:23.928214+0000","last_unstale":"2023-09-23T20:23:27.859944+0000","last_undegraded":"2023-09-23T20:23:23.902947+0000","last_fullsized":"2023-09-23T20:23:23.859127+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":118,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:22:51.846999+0000","objects_scrubbed":0,"log_size":2352,"log_dups_size":0,"ondisk_log_size":2352,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":94994432,"num_objects":1450,"num_object_clones":0,"num_object_copies":4350,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2620,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1450,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1450,"num_write_kb":92768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[0,1],"avail_no_missing":["0","5","6"],"object_location_counts":[{"shards":"0,1","objects":140},{"shards":"0,5,6","objects":1310}],"blocked_by":[],"up_primary":3,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":93,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T20:23:24.864806+0000","last_change":"2023-09-23T20:21:49.954219+0000","last_active":"2023-09-23T20:23:24.864806+0000","last_peered":"2023-09-23T20:23:24.864806+0000","last_clean":"2023-09-23T20:23:24.864806+0000","last_became_active":"2023-09-23T20:21:49.953827+0000","last_became_peered":"2023-09-23T20:21:49.953827+0000","last_unstale":"2023-09-23T20:23:24.864806+0000","last_undegraded":"2023-09-23T20:23:24.864806+0000","last_fullsized":"2023-09-23T20:23:24.864806+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:46:48.036706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":68,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T20:23:23.867117+0000","last_change":"2023-09-23T20:21:18.891438+0000","last_active":"2023-09-23T20:23:23.867117+0000","last_peered":"2023-09-23T20:23:23.867117+0000","last_clean":"2023-09-23T20:23:23.867117+0000","last_became_active":"2023-09-23T20:21:18.891140+0000","last_became_peered":"2023-09-23T20:21:18.891140+0000","last_unstale":"2023-09-23T20:23:23.867117+0000","last_undegraded":"2023-09-23T20:23:23.867117+0000","last_fullsized":"2023-09-23T20:23:23.867117+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:18:44.603577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.12","version":"129'2334","reported_seq":4606,"reported_epoch":129,"state":"active+recovering+undersized+remapped","last_fresh":"2023-09-23T20:23:27.889247+0000","last_change":"2023-09-23T20:23:21.956630+0000","last_active":"2023-09-23T20:23:27.889247+0000","last_peered":"2023-09-23T20:23:27.889247+0000","last_clean":"2023-09-23T20:23:18.340339+0000","last_became_active":"2023-09-23T20:23:21.914020+0000","last_became_peered":"2023-09-23T20:23:21.914020+0000","last_unstale":"2023-09-23T20:23:27.889247+0000","last_undegraded":"2023-09-23T20:23:27.889247+0000","last_fullsized":"2023-09-23T20:23:21.858786+0000","mapping_epoch":125,"log_start":"0'0","ondisk_log_start":"0'0","created":118,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":5,"last_scrub":"119'2087","last_scrub_stamp":"2023-09-23T20:23:15.737284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:23:15.737284+0000","objects_scrubbed":1166,"log_size":2334,"log_dups_size":0,"ondisk_log_size":2334,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:50:04.178108+0000","scrub_duration":0.27092271000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":93454336,"num_objects":1426,"num_object_clones":0,"num_object_copies":4278,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1000,"num_objects_unfound":0,"num_objects_dirty":1426,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":260,"num_write_kb":16640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":724,"num_bytes_recovered":47448064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[0,5],"avail_no_missing":["0","5","6"],"object_location_counts":[{"shards":"0,5","objects":926},{"shards":"0,5,6","objects":500}],"blocked_by":[],"up_primary":1,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":94,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T20:23:24.866421+0000","last_change":"2023-09-23T20:12:43.750217+0000","last_active":"2023-09-23T20:23:24.866421+0000","last_peered":"2023-09-23T20:23:24.866421+0000","last_clean":"2023-09-23T20:23:24.866421+0000","last_became_active":"2023-09-23T20:12:43.749929+0000","last_became_peered":"2023-09-23T20:12:43.749929+0000","last_unstale":"2023-09-23T20:23:24.866421+0000","last_undegraded":"2023-09-23T20:23:24.866421+0000","last_fullsized":"2023-09-23T20:23:24.866421+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:50:28.470211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":88,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T20:23:24.865677+0000","last_change":"2023-09-23T20:20:46.939938+0000","last_active":"2023-09-23T20:23:24.865677+0000","last_peered":"2023-09-23T20:23:24.865677+0000","last_clean":"2023-09-23T20:23:24.865677+0000","last_became_active":"2023-09-23T20:20:46.938777+0000","last_became_peered":"2023-09-23T20:20:46.938777+0000","last_unstale":"2023-09-23T20:23:24.865677+0000","last_undegraded":"2023-09-23T20:23:24.865677+0000","last_fullsized":"2023-09-23T20:23:24.865677+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:45:10.329438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.15","version":"129'2296","reported_seq":2333,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T20:23:27.880419+0000","last_change":"2023-09-23T20:23:11.073735+0000","last_active":"2023-09-23T20:23:27.880419+0000","last_peered":"2023-09-23T20:23:27.880419+0000","last_clean":"2023-09-23T20:23:27.880419+0000","last_became_active":"2023-09-23T20:23:11.072549+0000","last_became_peered":"2023-09-23T20:23:11.072549+0000","last_unstale":"2023-09-23T20:23:27.880419+0000","last_undegraded":"2023-09-23T20:23:27.880419+0000","last_fullsized":"2023-09-23T20:23:27.880419+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":118,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:22:51.846999+0000","objects_scrubbed":0,"log_size":2296,"log_dups_size":0,"ondisk_log_size":2296,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":92602368,"num_objects":1413,"num_object_clones":0,"num_object_copies":4239,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1413,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1413,"num_write_kb":90432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":92,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T20:23:24.864213+0000","last_change":"2023-09-23T20:21:18.897418+0000","last_active":"2023-09-23T20:23:24.864213+0000","last_peered":"2023-09-23T20:23:24.864213+0000","last_clean":"2023-09-23T20:23:24.864213+0000","last_became_active":"2023-09-23T20:21:18.896968+0000","last_became_peered":"2023-09-23T20:21:18.896968+0000","last_unstale":"2023-09-23T20:23:24.864213+0000","last_undegraded":"2023-09-23T20:23:24.864213+0000","last_fullsized":"2023-09-23T20:23:24.864213+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:12:44.651568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":91,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T20:23:23.881286+0000","last_change":"2023-09-23T20:21:49.958515+0000","last_active":"2023-09-23T20:23:23.881286+0000","last_peered":"2023-09-23T20:23:23.881286+0000","last_clean":"2023-09-23T20:23:23.881286+0000","last_became_active":"2023-09-23T20:21:49.958156+0000","last_became_peered":"2023-09-23T20:21:49.958156+0000","last_unstale":"2023-09-23T20:23:23.881286+0000","last_undegraded":"2023-09-23T20:23:23.881286+0000","last_fullsized":"2023-09-23T20:23:23.881286+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:56:47.145702+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.14","version":"130'2430","reported_seq":2467,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-09-23T20:23:30.338879+0000","last_change":"2023-09-23T20:23:11.068938+0000","last_active":"2023-09-23T20:23:30.338879+0000","last_peered":"2023-09-23T20:23:30.338879+0000","last_clean":"2023-09-23T20:23:30.338879+0000","last_became_active":"2023-09-23T20:23:11.067774+0000","last_became_peered":"2023-09-23T20:23:11.067774+0000","last_unstale":"2023-09-23T20:23:30.338879+0000","last_undegraded":"2023-09-23T20:23:30.338879+0000","last_fullsized":"2023-09-23T20:23:30.338879+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":118,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:22:51.846999+0000","objects_scrubbed":0,"log_size":2430,"log_dups_size":0,"ondisk_log_size":2430,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":100368384,"num_objects":1532,"num_object_clones":0,"num_object_copies":4596,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1532,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1532,"num_write_kb":98016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":71,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T20:23:23.861599+0000","last_change":"2023-09-23T20:21:49.953201+0000","last_active":"2023-09-23T20:23:23.861599+0000","last_peered":"2023-09-23T20:23:23.861599+0000","last_clean":"2023-09-23T20:23:23.861599+0000","last_became_active":"2023-09-23T20:21:49.952917+0000","last_became_peered":"2023-09-23T20:21:49.952917+0000","last_unstale":"2023-09-23T20:23:23.861599+0000","last_undegraded":"2023-09-23T20:23:23.861599+0000","last_fullsized":"2023-09-23T20:23:23.861599+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:54.579892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":88,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T20:23:24.883014+0000","last_change":"2023-09-23T20:21:49.952635+0000","last_active":"2023-09-23T20:23:24.883014+0000","last_peered":"2023-09-23T20:23:24.883014+0000","last_clean":"2023-09-23T20:23:24.883014+0000","last_became_active":"2023-09-23T20:21:49.952287+0000","last_became_peered":"2023-09-23T20:21:49.952287+0000","last_unstale":"2023-09-23T20:23:24.883014+0000","last_undegraded":"2023-09-23T20:23:24.883014+0000","last_fullsized":"2023-09-23T20:23:24.883014+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:17:17.370526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.17","version":"130'2465","reported_seq":2507,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-09-23T20:23:30.284401+0000","last_change":"2023-09-23T20:23:21.884086+0000","last_active":"2023-09-23T20:23:30.284401+0000","last_peered":"2023-09-23T20:23:30.284401+0000","last_clean":"2023-09-23T20:23:30.284401+0000","last_became_active":"2023-09-23T20:23:11.076350+0000","last_became_peered":"2023-09-23T20:23:11.076350+0000","last_unstale":"2023-09-23T20:23:30.284401+0000","last_undegraded":"2023-09-23T20:23:30.284401+0000","last_fullsized":"2023-09-23T20:23:30.284401+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":118,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:22:51.846999+0000","objects_scrubbed":0,"log_size":2465,"log_dups_size":0,"ondisk_log_size":2465,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":101384192,"num_objects":1547,"num_object_clones":0,"num_object_copies":4641,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1547,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1547,"num_write_kb":99008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":73,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T20:23:24.864026+0000","last_change":"2023-09-23T20:12:43.749139+0000","last_active":"2023-09-23T20:23:24.864026+0000","last_peered":"2023-09-23T20:23:24.864026+0000","last_clean":"2023-09-23T20:23:24.864026+0000","last_became_active":"2023-09-23T20:12:43.748854+0000","last_became_peered":"2023-09-23T20:12:43.748854+0000","last_unstale":"2023-09-23T20:23:24.864026+0000","last_undegraded":"2023-09-23T20:23:24.864026+0000","last_fullsized":"2023-09-23T20:23:24.864026+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:57:11.630712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":72,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T20:23:23.861688+0000","last_change":"2023-09-23T20:21:49.954242+0000","last_active":"2023-09-23T20:23:23.861688+0000","last_peered":"2023-09-23T20:23:23.861688+0000","last_clean":"2023-09-23T20:23:23.861688+0000","last_became_active":"2023-09-23T20:21:49.953597+0000","last_became_peered":"2023-09-23T20:21:49.953597+0000","last_unstale":"2023-09-23T20:23:23.861688+0000","last_undegraded":"2023-09-23T20:23:23.861688+0000","last_fullsized":"2023-09-23T20:23:23.861688+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:08:46.716466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.16","version":"131'2563","reported_seq":2640,"reported_epoch":131,"state":"active+clean","last_fresh":"2023-09-23T20:23:31.452280+0000","last_change":"2023-09-23T20:23:15.242095+0000","last_active":"2023-09-23T20:23:31.452280+0000","last_peered":"2023-09-23T20:23:31.452280+0000","last_clean":"2023-09-23T20:23:31.452280+0000","last_became_active":"2023-09-23T20:23:11.069819+0000","last_became_peered":"2023-09-23T20:23:11.069819+0000","last_unstale":"2023-09-23T20:23:31.452280+0000","last_undegraded":"2023-09-23T20:23:31.452280+0000","last_fullsized":"2023-09-23T20:23:31.452280+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":118,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":5,"last_scrub":"119'2163","last_scrub_stamp":"2023-09-23T20:23:15.241911+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:23:15.241911+0000","objects_scrubbed":1192,"log_size":2563,"log_dups_size":0,"ondisk_log_size":2563,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:57:32.114635+0000","scrub_duration":0.33323710899999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":105054208,"num_objects":1603,"num_object_clones":0,"num_object_copies":4809,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1603,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":411,"num_write_kb":26304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":98,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T20:23:24.865518+0000","last_change":"2023-09-23T20:20:14.881009+0000","last_active":"2023-09-23T20:23:24.865518+0000","last_peered":"2023-09-23T20:23:24.865518+0000","last_clean":"2023-09-23T20:23:24.865518+0000","last_became_active":"2023-09-23T20:20:14.880685+0000","last_became_peered":"2023-09-23T20:20:14.880685+0000","last_unstale":"2023-09-23T20:23:24.865518+0000","last_undegraded":"2023-09-23T20:23:24.865518+0000","last_fullsized":"2023-09-23T20:23:24.865518+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:28.010434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":91,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T20:23:23.881561+0000","last_change":"2023-09-23T20:21:49.958182+0000","last_active":"2023-09-23T20:23:23.881561+0000","last_peered":"2023-09-23T20:23:23.881561+0000","last_clean":"2023-09-23T20:23:23.881561+0000","last_became_active":"2023-09-23T20:21:49.957868+0000","last_became_peered":"2023-09-23T20:21:49.957868+0000","last_unstale":"2023-09-23T20:23:23.881561+0000","last_undegraded":"2023-09-23T20:23:23.881561+0000","last_fullsized":"2023-09-23T20:23:23.881561+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:27:41.577566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.9","version":"130'2421","reported_seq":2459,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-09-23T20:23:30.344179+0000","last_change":"2023-09-23T20:23:10.973039+0000","last_active":"2023-09-23T20:23:30.344179+0000","last_peered":"2023-09-23T20:23:30.344179+0000","last_clean":"2023-09-23T20:23:30.344179+0000","last_became_active":"2023-09-23T20:23:10.971930+0000","last_became_peered":"2023-09-23T20:23:10.971930+0000","last_unstale":"2023-09-23T20:23:30.344179+0000","last_undegraded":"2023-09-23T20:23:30.344179+0000","last_fullsized":"2023-09-23T20:23:30.344179+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":111,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:22:51.846999+0000","objects_scrubbed":0,"log_size":2421,"log_dups_size":0,"ondisk_log_size":2421,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":100827136,"num_objects":1538,"num_object_clones":0,"num_object_copies":4614,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1538,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1538,"num_write_kb":98464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":91,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T20:23:24.864580+0000","last_change":"2023-09-23T20:20:14.883283+0000","last_active":"2023-09-23T20:23:24.864580+0000","last_peered":"2023-09-23T20:23:24.864580+0000","last_clean":"2023-09-23T20:23:24.864580+0000","last_became_active":"2023-09-23T20:20:14.882806+0000","last_became_peered":"2023-09-23T20:20:14.882806+0000","last_unstale":"2023-09-23T20:23:24.864580+0000","last_undegraded":"2023-09-23T20:23:24.864580+0000","last_fullsized":"2023-09-23T20:23:24.864580+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:23:35.295701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":71,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T20:23:24.882680+0000","last_change":"2023-09-23T20:12:45.179211+0000","last_active":"2023-09-23T20:23:24.882680+0000","last_peered":"2023-09-23T20:23:24.882680+0000","last_clean":"2023-09-23T20:23:24.882680+0000","last_became_active":"2023-09-23T20:12:45.178966+0000","last_became_peered":"2023-09-23T20:12:45.178966+0000","last_unstale":"2023-09-23T20:23:24.882680+0000","last_undegraded":"2023-09-23T20:23:24.882680+0000","last_fullsized":"2023-09-23T20:23:24.882680+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:58.902140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.8","version":"131'2487","reported_seq":2566,"reported_epoch":131,"state":"active+clean","last_fresh":"2023-09-23T20:23:31.536035+0000","last_change":"2023-09-23T20:23:17.129472+0000","last_active":"2023-09-23T20:23:31.536035+0000","last_peered":"2023-09-23T20:23:31.536035+0000","last_clean":"2023-09-23T20:23:31.536035+0000","last_became_active":"2023-09-23T20:23:11.067277+0000","last_became_peered":"2023-09-23T20:23:11.067277+0000","last_unstale":"2023-09-23T20:23:31.536035+0000","last_undegraded":"2023-09-23T20:23:31.536035+0000","last_fullsized":"2023-09-23T20:23:31.536035+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":111,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"120'2144","last_scrub_stamp":"2023-09-23T20:23:17.129360+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:23:17.129360+0000","objects_scrubbed":1209,"log_size":2487,"log_dups_size":0,"ondisk_log_size":2487,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:33:11.603231+0000","scrub_duration":0.17393181099999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":101711872,"num_objects":1552,"num_object_clones":0,"num_object_copies":4656,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1552,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":343,"num_write_kb":21952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":92,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T20:23:24.864626+0000","last_change":"2023-09-23T20:20:46.950784+0000","last_active":"2023-09-23T20:23:24.864626+0000","last_peered":"2023-09-23T20:23:24.864626+0000","last_clean":"2023-09-23T20:23:24.864626+0000","last_became_active":"2023-09-23T20:20:46.950055+0000","last_became_peered":"2023-09-23T20:20:46.950055+0000","last_unstale":"2023-09-23T20:23:24.864626+0000","last_undegraded":"2023-09-23T20:23:24.864626+0000","last_fullsized":"2023-09-23T20:23:24.864626+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:23:07.645427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":105,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T20:23:24.865915+0000","last_change":"2023-09-23T20:21:49.957494+0000","last_active":"2023-09-23T20:23:24.865915+0000","last_peered":"2023-09-23T20:23:24.865915+0000","last_clean":"2023-09-23T20:23:24.865915+0000","last_became_active":"2023-09-23T20:21:49.957266+0000","last_became_peered":"2023-09-23T20:21:49.957266+0000","last_unstale":"2023-09-23T20:23:24.865915+0000","last_undegraded":"2023-09-23T20:23:24.865915+0000","last_fullsized":"2023-09-23T20:23:24.865915+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:49:51.483575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.b","version":"131'3158","reported_seq":3187,"reported_epoch":131,"state":"active+clean","last_fresh":"2023-09-23T20:23:31.496004+0000","last_change":"2023-09-23T20:22:52.869646+0000","last_active":"2023-09-23T20:23:31.496004+0000","last_peered":"2023-09-23T20:23:31.496004+0000","last_clean":"2023-09-23T20:23:31.496004+0000","last_became_active":"2023-09-23T20:22:52.869345+0000","last_became_peered":"2023-09-23T20:22:52.869345+0000","last_unstale":"2023-09-23T20:23:31.496004+0000","last_undegraded":"2023-09-23T20:23:31.496004+0000","last_fullsized":"2023-09-23T20:23:31.496004+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:22:51.846999+0000","objects_scrubbed":0,"log_size":3158,"log_dups_size":0,"ondisk_log_size":3158,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:16:39.278977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":206962688,"num_objects":3158,"num_object_clones":0,"num_object_copies":9474,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3158,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3158,"num_write_kb":202112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":74,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T20:23:24.864932+0000","last_change":"2023-09-23T20:12:43.751523+0000","last_active":"2023-09-23T20:23:24.864932+0000","last_peered":"2023-09-23T20:23:24.864932+0000","last_clean":"2023-09-23T20:23:24.864932+0000","last_became_active":"2023-09-23T20:12:43.751257+0000","last_became_peered":"2023-09-23T20:12:43.751257+0000","last_unstale":"2023-09-23T20:23:24.864932+0000","last_undegraded":"2023-09-23T20:23:24.864932+0000","last_fullsized":"2023-09-23T20:23:24.864932+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:50:27.040225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":89,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T20:23:24.865708+0000","last_change":"2023-09-23T20:20:46.934762+0000","last_active":"2023-09-23T20:23:24.865708+0000","last_peered":"2023-09-23T20:23:24.865708+0000","last_clean":"2023-09-23T20:23:24.865708+0000","last_became_active":"2023-09-23T20:20:46.934472+0000","last_became_peered":"2023-09-23T20:20:46.934472+0000","last_unstale":"2023-09-23T20:23:24.865708+0000","last_undegraded":"2023-09-23T20:23:24.865708+0000","last_fullsized":"2023-09-23T20:23:24.865708+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:28:42.708286+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.a","version":"131'3188","reported_seq":3217,"reported_epoch":131,"state":"active+clean","last_fresh":"2023-09-23T20:23:31.583504+0000","last_change":"2023-09-23T20:22:52.865524+0000","last_active":"2023-09-23T20:23:31.583504+0000","last_peered":"2023-09-23T20:23:31.583504+0000","last_clean":"2023-09-23T20:23:31.583504+0000","last_became_active":"2023-09-23T20:22:52.865271+0000","last_became_peered":"2023-09-23T20:22:52.865271+0000","last_unstale":"2023-09-23T20:23:31.583504+0000","last_undegraded":"2023-09-23T20:23:31.583504+0000","last_fullsized":"2023-09-23T20:23:31.583504+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:22:51.846999+0000","objects_scrubbed":0,"log_size":3188,"log_dups_size":0,"ondisk_log_size":3188,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:01:09.151325+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":208928768,"num_objects":3188,"num_object_clones":0,"num_object_copies":9564,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3188,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3188,"num_write_kb":204032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":89,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T20:23:24.882925+0000","last_change":"2023-09-23T20:21:49.954875+0000","last_active":"2023-09-23T20:23:24.882925+0000","last_peered":"2023-09-23T20:23:24.882925+0000","last_clean":"2023-09-23T20:23:24.882925+0000","last_became_active":"2023-09-23T20:21:49.954519+0000","last_became_peered":"2023-09-23T20:21:49.954519+0000","last_unstale":"2023-09-23T20:23:24.882925+0000","last_undegraded":"2023-09-23T20:23:24.882925+0000","last_fullsized":"2023-09-23T20:23:24.882925+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:29:52.962029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":72,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T20:23:23.860848+0000","last_change":"2023-09-23T20:20:46.935968+0000","last_active":"2023-09-23T20:23:23.860848+0000","last_peered":"2023-09-23T20:23:23.860848+0000","last_clean":"2023-09-23T20:23:23.860848+0000","last_became_active":"2023-09-23T20:20:46.935618+0000","last_became_peered":"2023-09-23T20:20:46.935618+0000","last_unstale":"2023-09-23T20:23:23.860848+0000","last_undegraded":"2023-09-23T20:23:23.860848+0000","last_fullsized":"2023-09-23T20:23:23.860848+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:42:59.867176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.d","version":"131'3160","reported_seq":3189,"reported_epoch":131,"state":"active+clean","last_fresh":"2023-09-23T20:23:31.571541+0000","last_change":"2023-09-23T20:22:52.869690+0000","last_active":"2023-09-23T20:23:31.571541+0000","last_peered":"2023-09-23T20:23:31.571541+0000","last_clean":"2023-09-23T20:23:31.571541+0000","last_became_active":"2023-09-23T20:22:52.869366+0000","last_became_peered":"2023-09-23T20:22:52.869366+0000","last_unstale":"2023-09-23T20:23:31.571541+0000","last_undegraded":"2023-09-23T20:23:31.571541+0000","last_fullsized":"2023-09-23T20:23:31.571541+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:22:51.846999+0000","objects_scrubbed":0,"log_size":3160,"log_dups_size":0,"ondisk_log_size":3160,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:16:02.423281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":207093760,"num_objects":3160,"num_object_clones":0,"num_object_copies":9480,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3160,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3160,"num_write_kb":202240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":89,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T20:23:23.915251+0000","last_change":"2023-09-23T20:21:49.953004+0000","last_active":"2023-09-23T20:23:23.915251+0000","last_peered":"2023-09-23T20:23:23.915251+0000","last_clean":"2023-09-23T20:23:23.915251+0000","last_became_active":"2023-09-23T20:21:49.952625+0000","last_became_peered":"2023-09-23T20:21:49.952625+0000","last_unstale":"2023-09-23T20:23:23.915251+0000","last_undegraded":"2023-09-23T20:23:23.915251+0000","last_fullsized":"2023-09-23T20:23:23.915251+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:48:28.305640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":88,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T20:23:24.866262+0000","last_change":"2023-09-23T20:21:49.954710+0000","last_active":"2023-09-23T20:23:24.866262+0000","last_peered":"2023-09-23T20:23:24.866262+0000","last_clean":"2023-09-23T20:23:24.866262+0000","last_became_active":"2023-09-23T20:21:49.954355+0000","last_became_peered":"2023-09-23T20:21:49.954355+0000","last_unstale":"2023-09-23T20:23:24.866262+0000","last_undegraded":"2023-09-23T20:23:24.866262+0000","last_fullsized":"2023-09-23T20:23:24.866262+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:56:16.233285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.c","version":"129'3089","reported_seq":3116,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T20:23:29.943332+0000","last_change":"2023-09-23T20:22:52.866632+0000","last_active":"2023-09-23T20:23:29.943332+0000","last_peered":"2023-09-23T20:23:29.943332+0000","last_clean":"2023-09-23T20:23:29.943332+0000","last_became_active":"2023-09-23T20:22:52.866356+0000","last_became_peered":"2023-09-23T20:22:52.866356+0000","last_unstale":"2023-09-23T20:23:29.943332+0000","last_undegraded":"2023-09-23T20:23:29.943332+0000","last_fullsized":"2023-09-23T20:23:29.943332+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:22:51.846999+0000","objects_scrubbed":0,"log_size":3089,"log_dups_size":0,"ondisk_log_size":3089,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:10:12.356062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":202440704,"num_objects":3089,"num_object_clones":0,"num_object_copies":9267,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3089,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3089,"num_write_kb":197696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":90,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T20:23:24.865552+0000","last_change":"2023-09-23T20:20:46.937163+0000","last_active":"2023-09-23T20:23:24.865552+0000","last_peered":"2023-09-23T20:23:24.865552+0000","last_clean":"2023-09-23T20:23:24.865552+0000","last_became_active":"2023-09-23T20:20:46.936757+0000","last_became_peered":"2023-09-23T20:20:46.936757+0000","last_unstale":"2023-09-23T20:23:24.865552+0000","last_undegraded":"2023-09-23T20:23:24.865552+0000","last_fullsized":"2023-09-23T20:23:24.865552+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:09.401379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":98,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T20:23:24.866144+0000","last_change":"2023-09-23T20:21:18.891669+0000","last_active":"2023-09-23T20:23:24.866144+0000","last_peered":"2023-09-23T20:23:24.866144+0000","last_clean":"2023-09-23T20:23:24.866144+0000","last_became_active":"2023-09-23T20:21:18.891326+0000","last_became_peered":"2023-09-23T20:21:18.891326+0000","last_unstale":"2023-09-23T20:23:24.866144+0000","last_undegraded":"2023-09-23T20:23:24.866144+0000","last_fullsized":"2023-09-23T20:23:24.866144+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:48:39.893213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.f","version":"130'3116","reported_seq":3143,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-09-23T20:23:30.296752+0000","last_change":"2023-09-23T20:22:52.872510+0000","last_active":"2023-09-23T20:23:30.296752+0000","last_peered":"2023-09-23T20:23:30.296752+0000","last_clean":"2023-09-23T20:23:30.296752+0000","last_became_active":"2023-09-23T20:22:52.872179+0000","last_became_peered":"2023-09-23T20:22:52.872179+0000","last_unstale":"2023-09-23T20:23:30.296752+0000","last_undegraded":"2023-09-23T20:23:30.296752+0000","last_fullsized":"2023-09-23T20:23:30.296752+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:22:51.846999+0000","objects_scrubbed":0,"log_size":3116,"log_dups_size":0,"ondisk_log_size":3116,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:53.012272+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":204210176,"num_objects":3116,"num_object_clones":0,"num_object_copies":9348,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3116,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3116,"num_write_kb":199424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":89,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T20:23:24.865470+0000","last_change":"2023-09-23T20:21:18.896635+0000","last_active":"2023-09-23T20:23:24.865470+0000","last_peered":"2023-09-23T20:23:24.865470+0000","last_clean":"2023-09-23T20:23:24.865470+0000","last_became_active":"2023-09-23T20:21:18.896335+0000","last_became_peered":"2023-09-23T20:21:18.896335+0000","last_unstale":"2023-09-23T20:23:24.865470+0000","last_undegraded":"2023-09-23T20:23:24.865470+0000","last_fullsized":"2023-09-23T20:23:24.865470+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:46:43.743795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":88,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T20:23:24.867009+0000","last_change":"2023-09-23T20:20:46.939720+0000","last_active":"2023-09-23T20:23:24.867009+0000","last_peered":"2023-09-23T20:23:24.867009+0000","last_clean":"2023-09-23T20:23:24.867009+0000","last_became_active":"2023-09-23T20:20:46.939357+0000","last_became_peered":"2023-09-23T20:20:46.939357+0000","last_unstale":"2023-09-23T20:23:24.867009+0000","last_undegraded":"2023-09-23T20:23:24.867009+0000","last_fullsized":"2023-09-23T20:23:24.867009+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:56.613212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.e","version":"130'3025","reported_seq":3051,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-09-23T20:23:30.350848+0000","last_change":"2023-09-23T20:22:52.871704+0000","last_active":"2023-09-23T20:23:30.350848+0000","last_peered":"2023-09-23T20:23:30.350848+0000","last_clean":"2023-09-23T20:23:30.350848+0000","last_became_active":"2023-09-23T20:22:52.871267+0000","last_became_peered":"2023-09-23T20:22:52.871267+0000","last_unstale":"2023-09-23T20:23:30.350848+0000","last_undegraded":"2023-09-23T20:23:30.350848+0000","last_fullsized":"2023-09-23T20:23:30.350848+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:22:51.846999+0000","objects_scrubbed":0,"log_size":3025,"log_dups_size":0,"ondisk_log_size":3025,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:50:07.619801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":198246400,"num_objects":3025,"num_object_clones":0,"num_object_copies":9075,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3025,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3025,"num_write_kb":193600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":92,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T20:23:23.877440+0000","last_change":"2023-09-23T20:21:49.958915+0000","last_active":"2023-09-23T20:23:23.877440+0000","last_peered":"2023-09-23T20:23:23.877440+0000","last_clean":"2023-09-23T20:23:23.877440+0000","last_became_active":"2023-09-23T20:21:49.958440+0000","last_became_peered":"2023-09-23T20:21:49.958440+0000","last_unstale":"2023-09-23T20:23:23.877440+0000","last_undegraded":"2023-09-23T20:23:23.877440+0000","last_fullsized":"2023-09-23T20:23:23.877440+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:36:48.998945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":65,"reported_epoch":131,"state":"active+clean","last_fresh":"2023-09-23T20:23:31.164774+0000","last_change":"2023-09-23T20:21:49.959532+0000","last_active":"2023-09-23T20:23:31.164774+0000","last_peered":"2023-09-23T20:23:31.164774+0000","last_clean":"2023-09-23T20:23:31.164774+0000","last_became_active":"2023-09-23T20:21:49.959161+0000","last_became_peered":"2023-09-23T20:21:49.959161+0000","last_unstale":"2023-09-23T20:23:31.164774+0000","last_undegraded":"2023-09-23T20:23:31.164774+0000","last_fullsized":"2023-09-23T20:23:31.164774+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:00:05.013667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"86'119","reported_seq":461,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T20:23:23.867424+0000","last_change":"2023-09-23T20:20:46.937071+0000","last_active":"2023-09-23T20:23:23.867424+0000","last_peered":"2023-09-23T20:23:23.867424+0000","last_clean":"2023-09-23T20:23:23.867424+0000","last_became_active":"2023-09-23T20:20:46.935348+0000","last_became_peered":"2023-09-23T20:20:46.935348+0000","last_unstale":"2023-09-23T20:23:23.867424+0000","last_undegraded":"2023-09-23T20:23:23.867424+0000","last_fullsized":"2023-09-23T20:23:23.867424+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":119,"log_dups_size":0,"ondisk_log_size":119,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:09.829030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032705499999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":523,"num_read_kb":3503,"num_write":136,"num_write_kb":3128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":106,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T20:23:24.883098+0000","last_change":"2023-09-23T20:21:49.953141+0000","last_active":"2023-09-23T20:23:24.883098+0000","last_peered":"2023-09-23T20:23:24.883098+0000","last_clean":"2023-09-23T20:23:24.883098+0000","last_became_active":"2023-09-23T20:21:49.952794+0000","last_became_peered":"2023-09-23T20:21:49.952794+0000","last_unstale":"2023-09-23T20:23:24.883098+0000","last_undegraded":"2023-09-23T20:23:24.883098+0000","last_fullsized":"2023-09-23T20:23:24.883098+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:36:19.919248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":53,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T20:23:23.867803+0000","last_change":"2023-09-23T20:20:46.937104+0000","last_active":"2023-09-23T20:23:23.867803+0000","last_peered":"2023-09-23T20:23:23.867803+0000","last_clean":"2023-09-23T20:23:23.867803+0000","last_became_active":"2023-09-23T20:20:46.936820+0000","last_became_peered":"2023-09-23T20:20:46.936820+0000","last_unstale":"2023-09-23T20:23:23.867803+0000","last_undegraded":"2023-09-23T20:23:23.867803+0000","last_fullsized":"2023-09-23T20:23:23.867803+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:22:19.879310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.3","version":"129'2357","reported_seq":2395,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T20:23:27.833685+0000","last_change":"2023-09-23T20:23:20.060302+0000","last_active":"2023-09-23T20:23:27.833685+0000","last_peered":"2023-09-23T20:23:27.833685+0000","last_clean":"2023-09-23T20:23:27.833685+0000","last_became_active":"2023-09-23T20:23:11.059880+0000","last_became_peered":"2023-09-23T20:23:11.059880+0000","last_unstale":"2023-09-23T20:23:27.833685+0000","last_undegraded":"2023-09-23T20:23:27.833685+0000","last_fullsized":"2023-09-23T20:23:27.833685+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":111,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:22:51.846999+0000","objects_scrubbed":0,"log_size":2357,"log_dups_size":0,"ondisk_log_size":2357,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":95322112,"num_objects":1454,"num_object_clones":0,"num_object_copies":4362,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1454,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1454,"num_write_kb":93088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.5","version":"129'2308","reported_seq":2346,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T20:23:27.841661+0000","last_change":"2023-09-23T20:23:11.152272+0000","last_active":"2023-09-23T20:23:27.841661+0000","last_peered":"2023-09-23T20:23:27.841661+0000","last_clean":"2023-09-23T20:23:27.841661+0000","last_became_active":"2023-09-23T20:23:11.055735+0000","last_became_peered":"2023-09-23T20:23:11.055735+0000","last_unstale":"2023-09-23T20:23:27.841661+0000","last_undegraded":"2023-09-23T20:23:27.841661+0000","last_fullsized":"2023-09-23T20:23:27.841661+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":111,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:22:51.846999+0000","objects_scrubbed":0,"log_size":2308,"log_dups_size":0,"ondisk_log_size":2308,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":93388800,"num_objects":1425,"num_object_clones":0,"num_object_copies":4275,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1425,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1425,"num_write_kb":91200,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"86'106","reported_seq":754,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T20:23:24.883170+0000","last_change":"2023-09-23T20:20:46.939468+0000","last_active":"2023-09-23T20:23:24.883170+0000","last_peered":"2023-09-23T20:23:24.883170+0000","last_clean":"2023-09-23T20:23:24.883170+0000","last_became_active":"2023-09-23T20:20:46.938954+0000","last_became_peered":"2023-09-23T20:20:46.938954+0000","last_unstale":"2023-09-23T20:23:24.883170+0000","last_undegraded":"2023-09-23T20:23:24.883170+0000","last_fullsized":"2023-09-23T20:23:24.883170+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":106,"log_dups_size":0,"ondisk_log_size":106,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:01:43.761064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00099302599999999993,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1105,"num_read_kb":1689,"num_write":100,"num_write_kb":404,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":89,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T20:23:24.865598+0000","last_change":"2023-09-23T20:20:14.887345+0000","last_active":"2023-09-23T20:23:24.865598+0000","last_peered":"2023-09-23T20:23:24.865598+0000","last_clean":"2023-09-23T20:23:24.865598+0000","last_became_active":"2023-09-23T20:20:14.886970+0000","last_became_peered":"2023-09-23T20:20:14.886970+0000","last_unstale":"2023-09-23T20:23:24.865598+0000","last_undegraded":"2023-09-23T20:23:24.865598+0000","last_fullsized":"2023-09-23T20:23:24.865598+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:00:38.575926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":83,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T20:23:23.867311+0000","last_change":"2023-09-23T20:21:49.952961+0000","last_active":"2023-09-23T20:23:23.867311+0000","last_peered":"2023-09-23T20:23:23.867311+0000","last_clean":"2023-09-23T20:23:23.867311+0000","last_became_active":"2023-09-23T20:21:49.952644+0000","last_became_peered":"2023-09-23T20:21:49.952644+0000","last_unstale":"2023-09-23T20:23:23.867311+0000","last_undegraded":"2023-09-23T20:23:23.867311+0000","last_fullsized":"2023-09-23T20:23:23.867311+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:05:30.492540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.6","version":"131'2557","reported_seq":2631,"reported_epoch":131,"state":"active+clean","last_fresh":"2023-09-23T20:23:31.545125+0000","last_change":"2023-09-23T20:23:12.084592+0000","last_active":"2023-09-23T20:23:31.545125+0000","last_peered":"2023-09-23T20:23:31.545125+0000","last_clean":"2023-09-23T20:23:31.545125+0000","last_became_active":"2023-09-23T20:23:11.061368+0000","last_became_peered":"2023-09-23T20:23:11.061368+0000","last_unstale":"2023-09-23T20:23:31.545125+0000","last_undegraded":"2023-09-23T20:23:31.545125+0000","last_fullsized":"2023-09-23T20:23:31.545125+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":111,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"119'1954","last_scrub_stamp":"2023-09-23T20:23:12.084143+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:23:12.084143+0000","objects_scrubbed":1042,"log_size":2557,"log_dups_size":0,"ondisk_log_size":2557,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:05:54.522913+0000","scrub_duration":0.23299162700000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":108068864,"num_objects":1649,"num_object_clones":0,"num_object_copies":4947,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1649,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":607,"num_write_kb":38848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"86'130","reported_seq":435,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-09-23T20:23:30.138757+0000","last_change":"2023-09-23T20:21:49.954331+0000","last_active":"2023-09-23T20:23:30.138757+0000","last_peered":"2023-09-23T20:23:30.138757+0000","last_clean":"2023-09-23T20:23:30.138757+0000","last_became_active":"2023-09-23T20:21:49.953687+0000","last_became_peered":"2023-09-23T20:21:49.953687+0000","last_unstale":"2023-09-23T20:23:30.138757+0000","last_undegraded":"2023-09-23T20:23:30.138757+0000","last_fullsized":"2023-09-23T20:23:30.138757+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:12.549803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011300049999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":512,"num_read_kb":1820,"num_write":133,"num_write_kb":1347,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":67,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T20:23:23.867107+0000","last_change":"2023-09-23T20:21:18.891553+0000","last_active":"2023-09-23T20:23:23.867107+0000","last_peered":"2023-09-23T20:23:23.867107+0000","last_clean":"2023-09-23T20:23:23.867107+0000","last_became_active":"2023-09-23T20:21:18.891230+0000","last_became_peered":"2023-09-23T20:21:18.891230+0000","last_unstale":"2023-09-23T20:23:23.867107+0000","last_undegraded":"2023-09-23T20:23:23.867107+0000","last_fullsized":"2023-09-23T20:23:23.867107+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:15:04.114927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":96,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T20:23:23.867683+0000","last_change":"2023-09-23T20:20:46.936524+0000","last_active":"2023-09-23T20:23:23.867683+0000","last_peered":"2023-09-23T20:23:23.867683+0000","last_clean":"2023-09-23T20:23:23.867683+0000","last_became_active":"2023-09-23T20:20:46.936267+0000","last_became_peered":"2023-09-23T20:20:46.936267+0000","last_unstale":"2023-09-23T20:23:23.867683+0000","last_undegraded":"2023-09-23T20:23:23.867683+0000","last_fullsized":"2023-09-23T20:23:23.867683+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:32:20.984521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"86'194","reported_seq":748,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T20:23:24.883003+0000","last_change":"2023-09-23T20:21:18.892176+0000","last_active":"2023-09-23T20:23:24.883003+0000","last_peered":"2023-09-23T20:23:24.883003+0000","last_clean":"2023-09-23T20:23:24.883003+0000","last_became_active":"2023-09-23T20:21:18.891647+0000","last_became_peered":"2023-09-23T20:21:18.891647+0000","last_unstale":"2023-09-23T20:23:24.883003+0000","last_undegraded":"2023-09-23T20:23:24.883003+0000","last_fullsized":"2023-09-23T20:23:24.883003+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":194,"log_dups_size":0,"ondisk_log_size":194,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:25:52.557392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00037995999999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1025,"num_read_kb":1201,"num_write":173,"num_write_kb":1276,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":472,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T20:23:24.864518+0000","last_change":"2023-09-23T20:20:14.882204+0000","last_active":"2023-09-23T20:23:24.864518+0000","last_peered":"2023-09-23T20:23:24.864518+0000","last_clean":"2023-09-23T20:23:24.864518+0000","last_became_active":"2023-09-23T20:20:14.881699+0000","last_became_peered":"2023-09-23T20:20:14.881699+0000","last_unstale":"2023-09-23T20:23:24.864518+0000","last_undegraded":"2023-09-23T20:23:24.864518+0000","last_fullsized":"2023-09-23T20:23:24.864518+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:09:47.213130+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:16:37.341010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":41,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T20:23:23.876322+0000","last_change":"2023-09-23T20:21:49.956277+0000","last_active":"2023-09-23T20:23:23.876322+0000","last_peered":"2023-09-23T20:23:23.876322+0000","last_clean":"2023-09-23T20:23:23.876322+0000","last_became_active":"2023-09-23T20:21:49.955974+0000","last_became_peered":"2023-09-23T20:21:49.955974+0000","last_unstale":"2023-09-23T20:23:23.876322+0000","last_undegraded":"2023-09-23T20:23:23.876322+0000","last_fullsized":"2023-09-23T20:23:23.876322+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:11:37.850675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":72,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T20:23:24.865667+0000","last_change":"2023-09-23T20:12:45.609829+0000","last_active":"2023-09-23T20:23:24.865667+0000","last_peered":"2023-09-23T20:23:24.865667+0000","last_clean":"2023-09-23T20:23:24.865667+0000","last_became_active":"2023-09-23T20:12:45.609467+0000","last_became_peered":"2023-09-23T20:12:45.609467+0000","last_unstale":"2023-09-23T20:23:24.865667+0000","last_undegraded":"2023-09-23T20:23:24.865667+0000","last_fullsized":"2023-09-23T20:23:24.865667+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:27:31.490555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.4","version":"130'2475","reported_seq":2565,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-09-23T20:23:30.293802+0000","last_change":"2023-09-23T20:23:13.408752+0000","last_active":"2023-09-23T20:23:30.293802+0000","last_peered":"2023-09-23T20:23:30.293802+0000","last_clean":"2023-09-23T20:23:30.293802+0000","last_became_active":"2023-09-23T20:23:11.063664+0000","last_became_peered":"2023-09-23T20:23:11.063664+0000","last_unstale":"2023-09-23T20:23:30.293802+0000","last_undegraded":"2023-09-23T20:23:30.293802+0000","last_fullsized":"2023-09-23T20:23:30.293802+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":111,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"119'1973","last_scrub_stamp":"2023-09-23T20:23:13.408658+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:23:13.408658+0000","objects_scrubbed":1060,"log_size":2475,"log_dups_size":0,"ondisk_log_size":2475,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:51:04.986370+0000","scrub_duration":0.28811426499999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":102629376,"num_objects":1566,"num_object_clones":0,"num_object_copies":4698,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1566,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":506,"num_write_kb":32384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.7","version":"130'2535","reported_seq":2637,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-09-23T20:23:30.348014+0000","last_change":"2023-09-23T20:23:16.284169+0000","last_active":"2023-09-23T20:23:30.348014+0000","last_peered":"2023-09-23T20:23:30.348014+0000","last_clean":"2023-09-23T20:23:30.348014+0000","last_became_active":"2023-09-23T20:23:11.069066+0000","last_became_peered":"2023-09-23T20:23:11.069066+0000","last_unstale":"2023-09-23T20:23:30.348014+0000","last_undegraded":"2023-09-23T20:23:30.348014+0000","last_fullsized":"2023-09-23T20:23:30.348014+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":111,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"120'2196","last_scrub_stamp":"2023-09-23T20:23:16.284112+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:23:16.284112+0000","objects_scrubbed":1268,"log_size":2535,"log_dups_size":0,"ondisk_log_size":2535,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:41:59.721972+0000","scrub_duration":0.33229684999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":105906176,"num_objects":1616,"num_object_clones":0,"num_object_copies":4848,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1616,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":348,"num_write_kb":22272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.0","version":"86'80","reported_seq":376,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T20:23:24.864552+0000","last_change":"2023-09-23T20:20:46.950784+0000","last_active":"2023-09-23T20:23:24.864552+0000","last_peered":"2023-09-23T20:23:24.864552+0000","last_clean":"2023-09-23T20:23:24.864552+0000","last_became_active":"2023-09-23T20:20:46.950071+0000","last_became_peered":"2023-09-23T20:20:46.950071+0000","last_unstale":"2023-09-23T20:23:24.864552+0000","last_undegraded":"2023-09-23T20:23:24.864552+0000","last_fullsized":"2023-09-23T20:23:24.864552+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":80,"log_dups_size":0,"ondisk_log_size":80,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:11:06.812404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035115,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":395,"num_read_kb":1370,"num_write":66,"num_write_kb":1061,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":71,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T20:23:23.859119+0000","last_change":"2023-09-23T20:21:49.951490+0000","last_active":"2023-09-23T20:23:23.859119+0000","last_peered":"2023-09-23T20:23:23.859119+0000","last_clean":"2023-09-23T20:23:23.859119+0000","last_became_active":"2023-09-23T20:21:49.951079+0000","last_became_peered":"2023-09-23T20:21:49.951079+0000","last_unstale":"2023-09-23T20:23:23.859119+0000","last_undegraded":"2023-09-23T20:23:23.859119+0000","last_fullsized":"2023-09-23T20:23:23.859119+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:12:01.053013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":87,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T20:23:23.861927+0000","last_change":"2023-09-23T20:21:18.887651+0000","last_active":"2023-09-23T20:23:23.861927+0000","last_peered":"2023-09-23T20:23:23.861927+0000","last_clean":"2023-09-23T20:23:23.861927+0000","last_became_active":"2023-09-23T20:21:18.887338+0000","last_became_peered":"2023-09-23T20:21:18.887338+0000","last_unstale":"2023-09-23T20:23:23.861927+0000","last_undegraded":"2023-09-23T20:23:23.861927+0000","last_fullsized":"2023-09-23T20:23:23.861927+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:26:39.067022+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"86'102","reported_seq":636,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T20:23:24.864862+0000","last_change":"2023-09-23T20:20:14.881975+0000","last_active":"2023-09-23T20:23:24.864862+0000","last_peered":"2023-09-23T20:23:24.864862+0000","last_clean":"2023-09-23T20:23:24.864862+0000","last_became_active":"2023-09-23T20:20:14.881544+0000","last_became_peered":"2023-09-23T20:20:14.881544+0000","last_unstale":"2023-09-23T20:23:24.864862+0000","last_undegraded":"2023-09-23T20:23:24.864862+0000","last_fullsized":"2023-09-23T20:23:24.864862+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":102,"log_dups_size":0,"ondisk_log_size":102,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:34.594972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030656500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":963,"num_read_kb":1234,"num_write":60,"num_write_kb":225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":67,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T20:23:23.867433+0000","last_change":"2023-09-23T20:21:18.893457+0000","last_active":"2023-09-23T20:23:23.867433+0000","last_peered":"2023-09-23T20:23:23.867433+0000","last_clean":"2023-09-23T20:23:23.867433+0000","last_became_active":"2023-09-23T20:21:18.893175+0000","last_became_peered":"2023-09-23T20:21:18.893175+0000","last_unstale":"2023-09-23T20:23:23.867433+0000","last_undegraded":"2023-09-23T20:23:23.867433+0000","last_fullsized":"2023-09-23T20:23:23.867433+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:24:37.563758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":57,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T20:23:23.861063+0000","last_change":"2023-09-23T20:20:14.888874+0000","last_active":"2023-09-23T20:23:23.861063+0000","last_peered":"2023-09-23T20:23:23.861063+0000","last_clean":"2023-09-23T20:23:23.861063+0000","last_became_active":"2023-09-23T20:20:14.887724+0000","last_became_peered":"2023-09-23T20:20:14.887724+0000","last_unstale":"2023-09-23T20:23:23.861063+0000","last_undegraded":"2023-09-23T20:23:23.861063+0000","last_fullsized":"2023-09-23T20:23:23.861063+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:09:24.510326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.2","version":"130'2455","reported_seq":2543,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-09-23T20:23:30.348005+0000","last_change":"2023-09-23T20:23:12.771560+0000","last_active":"2023-09-23T20:23:30.348005+0000","last_peered":"2023-09-23T20:23:30.348005+0000","last_clean":"2023-09-23T20:23:30.348005+0000","last_became_active":"2023-09-23T20:23:11.058929+0000","last_became_peered":"2023-09-23T20:23:11.058929+0000","last_unstale":"2023-09-23T20:23:30.348005+0000","last_undegraded":"2023-09-23T20:23:30.348005+0000","last_fullsized":"2023-09-23T20:23:30.348005+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":111,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"119'1924","last_scrub_stamp":"2023-09-23T20:23:12.771453+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:23:12.771453+0000","objects_scrubbed":1010,"log_size":2455,"log_dups_size":0,"ondisk_log_size":2455,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:49:31.924585+0000","scrub_duration":0.26141313799999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":101515264,"num_objects":1549,"num_object_clones":0,"num_object_copies":4647,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1549,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":539,"num_write_kb":34496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":89,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T20:23:23.881653+0000","last_change":"2023-09-23T20:21:49.957535+0000","last_active":"2023-09-23T20:23:23.881653+0000","last_peered":"2023-09-23T20:23:23.881653+0000","last_clean":"2023-09-23T20:23:23.881653+0000","last_became_active":"2023-09-23T20:21:49.954319+0000","last_became_peered":"2023-09-23T20:21:49.954319+0000","last_unstale":"2023-09-23T20:23:23.881653+0000","last_undegraded":"2023-09-23T20:23:23.881653+0000","last_fullsized":"2023-09-23T20:23:23.881653+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:14:10.293305+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"86'128","reported_seq":452,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T20:23:23.868909+0000","last_change":"2023-09-23T20:20:46.939256+0000","last_active":"2023-09-23T20:23:23.868909+0000","last_peered":"2023-09-23T20:23:23.868909+0000","last_clean":"2023-09-23T20:23:23.868909+0000","last_became_active":"2023-09-23T20:20:46.938756+0000","last_became_peered":"2023-09-23T20:20:46.938756+0000","last_unstale":"2023-09-23T20:23:23.868909+0000","last_undegraded":"2023-09-23T20:23:23.868909+0000","last_fullsized":"2023-09-23T20:23:23.868909+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:52:18.161436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000293202,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":579,"num_read_kb":1473,"num_write":96,"num_write_kb":1062,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":86,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T20:23:23.877366+0000","last_change":"2023-09-23T20:21:49.953565+0000","last_active":"2023-09-23T20:23:23.877366+0000","last_peered":"2023-09-23T20:23:23.877366+0000","last_clean":"2023-09-23T20:23:23.877366+0000","last_became_active":"2023-09-23T20:21:49.953176+0000","last_became_peered":"2023-09-23T20:21:49.953176+0000","last_unstale":"2023-09-23T20:23:23.877366+0000","last_undegraded":"2023-09-23T20:23:23.877366+0000","last_fullsized":"2023-09-23T20:23:23.877366+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:02:35.778207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.1","version":"130'2501","reported_seq":2586,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-09-23T20:23:30.291090+0000","last_change":"2023-09-23T20:23:11.363740+0000","last_active":"2023-09-23T20:23:30.291090+0000","last_peered":"2023-09-23T20:23:30.291090+0000","last_clean":"2023-09-23T20:23:30.291090+0000","last_became_active":"2023-09-23T20:23:11.053830+0000","last_became_peered":"2023-09-23T20:23:11.053830+0000","last_unstale":"2023-09-23T20:23:30.291090+0000","last_undegraded":"2023-09-23T20:23:30.291090+0000","last_fullsized":"2023-09-23T20:23:30.291090+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":111,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"119'1889","last_scrub_stamp":"2023-09-23T20:23:11.363674+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:23:11.363674+0000","objects_scrubbed":958,"log_size":2501,"log_dups_size":0,"ondisk_log_size":2501,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:02:44.132086+0000","scrub_duration":0.26718564099999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":103415808,"num_objects":1578,"num_object_clones":0,"num_object_copies":4734,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1578,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":620,"num_write_kb":39680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"86'117","reported_seq":616,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T20:23:24.865943+0000","last_change":"2023-09-23T20:21:18.897269+0000","last_active":"2023-09-23T20:23:24.865943+0000","last_peered":"2023-09-23T20:23:24.865943+0000","last_clean":"2023-09-23T20:23:24.865943+0000","last_became_active":"2023-09-23T20:21:18.896834+0000","last_became_peered":"2023-09-23T20:21:18.896834+0000","last_unstale":"2023-09-23T20:23:24.865943+0000","last_undegraded":"2023-09-23T20:23:24.865943+0000","last_fullsized":"2023-09-23T20:23:24.865943+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:57:17.499025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046891400000000002,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":893,"num_read_kb":922,"num_write":82,"num_write_kb":4193,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":77,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T20:23:23.861857+0000","last_change":"2023-09-23T20:20:46.937163+0000","last_active":"2023-09-23T20:23:23.861857+0000","last_peered":"2023-09-23T20:23:23.861857+0000","last_clean":"2023-09-23T20:23:23.861857+0000","last_became_active":"2023-09-23T20:20:46.936846+0000","last_became_peered":"2023-09-23T20:20:46.936846+0000","last_unstale":"2023-09-23T20:23:23.861857+0000","last_undegraded":"2023-09-23T20:23:23.861857+0000","last_fullsized":"2023-09-23T20:23:23.861857+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:01:32.798855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":71,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T20:23:24.865780+0000","last_change":"2023-09-23T20:12:43.765908+0000","last_active":"2023-09-23T20:23:24.865780+0000","last_peered":"2023-09-23T20:23:24.865780+0000","last_clean":"2023-09-23T20:23:24.865780+0000","last_became_active":"2023-09-23T20:12:43.765558+0000","last_became_peered":"2023-09-23T20:12:43.765558+0000","last_unstale":"2023-09-23T20:23:24.865780+0000","last_undegraded":"2023-09-23T20:23:24.865780+0000","last_fullsized":"2023-09-23T20:23:24.865780+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:35:43.809898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.0","version":"129'2387","reported_seq":2455,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T20:23:27.940971+0000","last_change":"2023-09-23T20:23:13.995643+0000","last_active":"2023-09-23T20:23:27.940971+0000","last_peered":"2023-09-23T20:23:27.940971+0000","last_clean":"2023-09-23T20:23:27.940971+0000","last_became_active":"2023-09-23T20:23:11.065489+0000","last_became_peered":"2023-09-23T20:23:11.065489+0000","last_unstale":"2023-09-23T20:23:27.940971+0000","last_undegraded":"2023-09-23T20:23:27.940971+0000","last_fullsized":"2023-09-23T20:23:27.940971+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":111,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"119'2020","last_scrub_stamp":"2023-09-23T20:23:13.995392+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:23:13.995392+0000","objects_scrubbed":1088,"log_size":2387,"log_dups_size":0,"ondisk_log_size":2387,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:09:18.391697+0000","scrub_duration":0.22507750400000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":95944704,"num_objects":1464,"num_object_clones":0,"num_object_copies":4392,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1464,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":376,"num_write_kb":24064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":60,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T20:23:23.860993+0000","last_change":"2023-09-23T20:20:14.888484+0000","last_active":"2023-09-23T20:23:23.860993+0000","last_peered":"2023-09-23T20:23:23.860993+0000","last_clean":"2023-09-23T20:23:23.860993+0000","last_became_active":"2023-09-23T20:20:14.887219+0000","last_became_peered":"2023-09-23T20:20:14.887219+0000","last_unstale":"2023-09-23T20:23:23.860993+0000","last_undegraded":"2023-09-23T20:23:23.860993+0000","last_fullsized":"2023-09-23T20:23:23.860993+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:13:53.763649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":111,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-09-23T20:23:30.139028+0000","last_change":"2023-09-23T20:21:49.953299+0000","last_active":"2023-09-23T20:23:30.139028+0000","last_peered":"2023-09-23T20:23:30.139028+0000","last_clean":"2023-09-23T20:23:30.139028+0000","last_became_active":"2023-09-23T20:21:49.952996+0000","last_became_peered":"2023-09-23T20:21:49.952996+0000","last_unstale":"2023-09-23T20:23:30.139028+0000","last_undegraded":"2023-09-23T20:23:30.139028+0000","last_fullsized":"2023-09-23T20:23:30.139028+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:46:51.746281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":91,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T20:23:24.883045+0000","last_change":"2023-09-23T20:20:46.938981+0000","last_active":"2023-09-23T20:23:24.883045+0000","last_peered":"2023-09-23T20:23:24.883045+0000","last_clean":"2023-09-23T20:23:24.883045+0000","last_became_active":"2023-09-23T20:20:46.938627+0000","last_became_peered":"2023-09-23T20:20:46.938627+0000","last_unstale":"2023-09-23T20:23:24.883045+0000","last_undegraded":"2023-09-23T20:23:24.883045+0000","last_fullsized":"2023-09-23T20:23:24.883045+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:44:43.177086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":105,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T20:23:24.865813+0000","last_change":"2023-09-23T20:21:49.957885+0000","last_active":"2023-09-23T20:23:24.865813+0000","last_peered":"2023-09-23T20:23:24.865813+0000","last_clean":"2023-09-23T20:23:24.865813+0000","last_became_active":"2023-09-23T20:21:49.957492+0000","last_became_peered":"2023-09-23T20:21:49.957492+0000","last_unstale":"2023-09-23T20:23:24.865813+0000","last_undegraded":"2023-09-23T20:23:24.865813+0000","last_fullsized":"2023-09-23T20:23:24.865813+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:24:55.137836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":74,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T20:23:24.883248+0000","last_change":"2023-09-23T20:12:43.753383+0000","last_active":"2023-09-23T20:23:24.883248+0000","last_peered":"2023-09-23T20:23:24.883248+0000","last_clean":"2023-09-23T20:23:24.883248+0000","last_became_active":"2023-09-23T20:12:43.752921+0000","last_became_peered":"2023-09-23T20:12:43.752921+0000","last_unstale":"2023-09-23T20:23:24.883248+0000","last_undegraded":"2023-09-23T20:23:24.883248+0000","last_fullsized":"2023-09-23T20:23:24.883248+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:28:26.982998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":105,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T20:23:24.866786+0000","last_change":"2023-09-23T20:21:49.953041+0000","last_active":"2023-09-23T20:23:24.866786+0000","last_peered":"2023-09-23T20:23:24.866786+0000","last_clean":"2023-09-23T20:23:24.866786+0000","last_became_active":"2023-09-23T20:21:49.952763+0000","last_became_peered":"2023-09-23T20:21:49.952763+0000","last_unstale":"2023-09-23T20:23:24.866786+0000","last_undegraded":"2023-09-23T20:23:24.866786+0000","last_fullsized":"2023-09-23T20:23:24.866786+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:58:35.321466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.18","version":"131'2488","reported_seq":2564,"reported_epoch":131,"state":"active+clean","last_fresh":"2023-09-23T20:23:31.498993+0000","last_change":"2023-09-23T20:23:18.112051+0000","last_active":"2023-09-23T20:23:31.498993+0000","last_peered":"2023-09-23T20:23:31.498993+0000","last_clean":"2023-09-23T20:23:31.498993+0000","last_became_active":"2023-09-23T20:23:11.077074+0000","last_became_peered":"2023-09-23T20:23:11.077074+0000","last_unstale":"2023-09-23T20:23:31.498993+0000","last_undegraded":"2023-09-23T20:23:31.498993+0000","last_fullsized":"2023-09-23T20:23:31.498993+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":118,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":5,"last_scrub":"120'2147","last_scrub_stamp":"2023-09-23T20:23:18.111873+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:23:18.111873+0000","objects_scrubbed":1267,"log_size":2488,"log_dups_size":0,"ondisk_log_size":2488,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:35:23.572513+0000","scrub_duration":0.17815524799999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":105381888,"num_objects":1608,"num_object_clones":0,"num_object_copies":4824,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1608,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":341,"num_write_kb":21824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":97,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T20:23:23.881033+0000","last_change":"2023-09-23T20:21:49.957394+0000","last_active":"2023-09-23T20:23:23.881033+0000","last_peered":"2023-09-23T20:23:23.881033+0000","last_clean":"2023-09-23T20:23:23.881033+0000","last_became_active":"2023-09-23T20:21:49.957016+0000","last_became_peered":"2023-09-23T20:21:49.957016+0000","last_unstale":"2023-09-23T20:23:23.881033+0000","last_undegraded":"2023-09-23T20:23:23.881033+0000","last_fullsized":"2023-09-23T20:23:23.881033+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:49:26.222448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":112,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T20:23:23.877803+0000","last_change":"2023-09-23T20:21:49.949944+0000","last_active":"2023-09-23T20:23:23.877803+0000","last_peered":"2023-09-23T20:23:23.877803+0000","last_clean":"2023-09-23T20:23:23.877803+0000","last_became_active":"2023-09-23T20:21:49.949425+0000","last_became_peered":"2023-09-23T20:21:49.949425+0000","last_unstale":"2023-09-23T20:23:23.877803+0000","last_undegraded":"2023-09-23T20:23:23.877803+0000","last_fullsized":"2023-09-23T20:23:23.877803+0000","mapping_epoch":109,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:40:56.395791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.19","version":"130'2440","reported_seq":2477,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-09-23T20:23:30.344290+0000","last_change":"2023-09-23T20:23:10.978296+0000","last_active":"2023-09-23T20:23:30.344290+0000","last_peered":"2023-09-23T20:23:30.344290+0000","last_clean":"2023-09-23T20:23:30.344290+0000","last_became_active":"2023-09-23T20:23:10.977578+0000","last_became_peered":"2023-09-23T20:23:10.977578+0000","last_unstale":"2023-09-23T20:23:30.344290+0000","last_undegraded":"2023-09-23T20:23:30.344290+0000","last_fullsized":"2023-09-23T20:23:30.344290+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":118,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:22:51.846999+0000","objects_scrubbed":0,"log_size":2440,"log_dups_size":0,"ondisk_log_size":2440,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":102072320,"num_objects":1558,"num_object_clones":0,"num_object_copies":4674,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1558,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1558,"num_write_kb":99680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":75,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T20:23:23.862288+0000","last_change":"2023-09-23T20:21:18.889521+0000","last_active":"2023-09-23T20:23:23.862288+0000","last_peered":"2023-09-23T20:23:23.862288+0000","last_clean":"2023-09-23T20:23:23.862288+0000","last_became_active":"2023-09-23T20:21:18.889268+0000","last_became_peered":"2023-09-23T20:21:18.889268+0000","last_unstale":"2023-09-23T20:23:23.862288+0000","last_undegraded":"2023-09-23T20:23:23.862288+0000","last_fullsized":"2023-09-23T20:23:23.862288+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:05:48.805778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":98,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T20:23:24.865419+0000","last_change":"2023-09-23T20:21:18.893413+0000","last_active":"2023-09-23T20:23:24.865419+0000","last_peered":"2023-09-23T20:23:24.865419+0000","last_clean":"2023-09-23T20:23:24.865419+0000","last_became_active":"2023-09-23T20:21:18.893008+0000","last_became_peered":"2023-09-23T20:21:18.893008+0000","last_unstale":"2023-09-23T20:23:24.865419+0000","last_undegraded":"2023-09-23T20:23:24.865419+0000","last_fullsized":"2023-09-23T20:23:24.865419+0000","mapping_epoch":104,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:50:55.264882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":5,"num_pg":26,"stat_sum":{"num_bytes":3240460288,"num_objects":49446,"num_object_clones":0,"num_object_copies":148338,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2620,"num_objects_misplaced":1000,"num_objects_unfound":0,"num_objects_dirty":49446,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":35969,"num_write_kb":2301984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3261,"num_bytes_recovered":213712896,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":10046406656,"data_stored":10046406656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":67637,"ondisk_log_size":67637,"up":78,"acting":76,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":25181,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6082,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201428,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1149328,"kb_used_data":1108680,"kb_used_omap":0,"kb_used_meta":40640,"kb_avail":92595824,"statfs":{"total":95995035648,"available":94818123776,"internally_reserved":0,"allocated":1135288320,"data_stored":1134749492,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":41615360},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:23:17 2023","interfaces":[{"interface":"back","average":{"1min":0.58499999999999996,"5min":0.56699999999999995,"15min":0.57099999999999995},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":1.5049999999999999,"5min":1.5049999999999999,"15min":1.5049999999999999},"last":3.5960000000000001},{"interface":"front","average":{"1min":0.59899999999999998,"5min":0.56799999999999995,"15min":0.56799999999999995},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":1.5209999999999999,"5min":1.5209999999999999,"15min":1.5209999999999999},"last":1.3080000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:22:49 2023","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":0.56599999999999995,"15min":0.56399999999999995},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":1.399,"5min":1.399,"15min":1.399},"last":0.54100000000000004},{"interface":"front","average":{"1min":0.60499999999999998,"5min":0.60999999999999999,"15min":0.61099999999999999},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":1.5569999999999999,"5min":1.5569999999999999,"15min":1.5569999999999999},"last":1.179}]},{"osd":2,"last update":"Sat Sep 23 20:23:23 2023","interfaces":[{"interface":"back","average":{"1min":0.67200000000000004,"5min":0.58899999999999997,"15min":0.57499999999999996},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":1.448,"5min":1.448,"15min":1.448},"last":1.6779999999999999},{"interface":"front","average":{"1min":0.80400000000000005,"5min":0.63,"15min":0.60099999999999998},"min":{"1min":0.42299999999999999,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":3.7410000000000001,"5min":3.7410000000000001,"15min":3.7410000000000001},"last":0.56799999999999995}]},{"osd":3,"last update":"Sat Sep 23 20:22:49 2023","interfaces":[{"interface":"back","average":{"1min":0.65500000000000003,"5min":0.65500000000000003,"15min":0.65500000000000003},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":1.5369999999999999,"5min":1.5369999999999999,"15min":1.5369999999999999},"last":0.60299999999999998},{"interface":"front","average":{"1min":0.67400000000000004,"5min":0.67400000000000004,"15min":0.67400000000000004},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":1.492,"5min":1.492,"15min":1.492},"last":0.27600000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:22:49 2023","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":0.503,"15min":0.50600000000000001},"min":{"1min":0.314,"5min":0.28799999999999998,"15min":0.26700000000000002},"max":{"1min":0.76600000000000001,"5min":0.90100000000000002,"15min":1.274},"last":0.34599999999999997},{"interface":"front","average":{"1min":0.61099999999999999,"5min":0.55000000000000004,"15min":0.53900000000000003},"min":{"1min":0.33400000000000002,"5min":0.33100000000000002,"15min":0.29699999999999999},"max":{"1min":1.131,"5min":1.131,"15min":2.024},"last":0.31}]},{"osd":5,"last update":"Sat Sep 23 20:22:49 2023","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":0.53600000000000003,"15min":0.54200000000000004},"min":{"1min":0.36199999999999999,"5min":0.33500000000000002,"15min":0.32000000000000001},"max":{"1min":0.83199999999999996,"5min":0.83199999999999996,"15min":1.337},"last":0.505},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.58999999999999997,"15min":0.58699999999999997},"min":{"1min":0.35799999999999998,"5min":0.34399999999999997,"15min":0.31},"max":{"1min":0.82199999999999995,"5min":0.89100000000000001,"15min":1.5660000000000001},"last":0.39700000000000002}]},{"osd":6,"last update":"Sat Sep 23 20:22:49 2023","interfaces":[{"interface":"back","average":{"1min":0.63800000000000001,"5min":0.61499999999999999,"15min":0.59699999999999998},"min":{"1min":0.45700000000000002,"5min":0.379,"15min":0.34499999999999997},"max":{"1min":0.82299999999999995,"5min":0.98699999999999999,"15min":1.246},"last":0.42899999999999999},{"interface":"front","average":{"1min":0.64800000000000002,"5min":0.60899999999999999,"15min":0.58999999999999997},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.88700000000000001,"5min":0.88700000000000001,"15min":2.048},"last":1.042}]}]},{"osd":6,"up_from":47,"seq":201863463064,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":999368,"kb_used_data":964480,"kb_used_omap":0,"kb_used_meta":34880,"kb_avail":92745784,"statfs":{"total":95995035648,"available":94971682816,"internally_reserved":0,"allocated":987627520,"data_stored":987096757,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":35717120},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:23:20 2023","interfaces":[{"interface":"back","average":{"1min":0.66700000000000004,"5min":0.59599999999999997,"15min":0.58399999999999996},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":1.841,"5min":1.841,"15min":1.841},"last":0.89100000000000001},{"interface":"front","average":{"1min":0.75700000000000001,"5min":0.60399999999999998,"15min":0.57599999999999996},"min":{"1min":0.39600000000000002,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":3.246,"5min":3.246,"15min":3.246},"last":1.0860000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:22:49 2023","interfaces":[{"interface":"back","average":{"1min":0.58199999999999996,"5min":0.57599999999999996,"15min":0.57499999999999996},"min":{"1min":0.443,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.72499999999999998,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":2.2709999999999999},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.41199999999999998,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.68899999999999995,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":1.101}]},{"osd":2,"last update":"Sat Sep 23 20:23:20 2023","interfaces":[{"interface":"back","average":{"1min":0.63300000000000001,"5min":0.60199999999999998,"15min":0.59699999999999998},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":1.9430000000000001,"5min":1.9430000000000001,"15min":1.9430000000000001},"last":0.78900000000000003},{"interface":"front","average":{"1min":0.73899999999999999,"5min":0.63500000000000001,"15min":0.61699999999999999},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":1.9890000000000001,"5min":1.9890000000000001,"15min":1.9890000000000001},"last":0.84499999999999997}]},{"osd":3,"last update":"Sat Sep 23 20:22:49 2023","interfaces":[{"interface":"back","average":{"1min":0.61299999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"min":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":0.749,"5min":0.749,"15min":0.749},"last":2.4750000000000001},{"interface":"front","average":{"1min":0.67700000000000005,"5min":0.67700000000000005,"15min":0.67700000000000005},"min":{"1min":0.54400000000000004,"5min":0.54400000000000004,"15min":0.54400000000000004},"max":{"1min":0.93899999999999995,"5min":0.93899999999999995,"15min":0.93899999999999995},"last":1.175}]},{"osd":4,"last update":"Sat Sep 23 20:22:38 2023","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":0.48799999999999999,"15min":0.46899999999999997},"min":{"1min":0.34699999999999998,"5min":0.312,"15min":0.26200000000000001},"max":{"1min":0.74299999999999999,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":1.073},{"interface":"front","average":{"1min":0.55600000000000005,"5min":0.51900000000000002,"15min":0.497},"min":{"1min":0.34200000000000003,"5min":0.30099999999999999,"15min":0.28499999999999998},"max":{"1min":0.81299999999999994,"5min":0.81299999999999994,"15min":0.83899999999999997},"last":1.222}]},{"osd":5,"last update":"Sat Sep 23 20:22:38 2023","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":0.53000000000000003,"15min":0.5},"min":{"1min":0.36399999999999999,"5min":0.32600000000000001,"15min":0.29799999999999999},"max":{"1min":0.81899999999999995,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.38500000000000001},{"interface":"front","average":{"1min":0.63400000000000001,"5min":0.57399999999999995,"15min":0.52900000000000003},"min":{"1min":0.42599999999999999,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.82299999999999995,"5min":0.85599999999999998,"15min":0.874},"last":1.121}]},{"osd":7,"last update":"Sat Sep 23 20:22:49 2023","interfaces":[{"interface":"back","average":{"1min":0.64100000000000001,"5min":0.58299999999999996,"15min":0.55000000000000004},"min":{"1min":0.443,"5min":0.36799999999999999,"15min":0.32800000000000001},"max":{"1min":0.81100000000000005,"5min":0.85199999999999998,"15min":0.85199999999999998},"last":1.038},{"interface":"front","average":{"1min":0.66500000000000004,"5min":0.61099999999999999,"15min":0.57599999999999996},"min":{"1min":0.46200000000000002,"5min":0.33500000000000002,"15min":0.315},"max":{"1min":0.83999999999999997,"5min":0.84799999999999998,"15min":0.90800000000000003},"last":1.1479999999999999}]}]},{"osd":1,"up_from":99,"seq":425201762339,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1108076,"kb_used_data":1054244,"kb_used_omap":6,"kb_used_meta":53817,"kb_avail":92637076,"statfs":{"total":95995035648,"available":94860365824,"internally_reserved":0,"allocated":1079545856,"data_stored":1078919412,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6696,"internal_metadata":55109080},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:22:47 2023","interfaces":[{"interface":"back","average":{"1min":0.33700000000000002,"5min":0.36899999999999999,"15min":0.374},"min":{"1min":0.19600000000000001,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"last":2.681},{"interface":"front","average":{"1min":0.36599999999999999,"5min":0.39300000000000002,"15min":0.39700000000000002},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":0.71299999999999997,"5min":0.71299999999999997,"15min":0.71299999999999997},"last":1.452}]},{"osd":2,"last update":"Sat Sep 23 20:23:19 2023","interfaces":[{"interface":"back","average":{"1min":1.218,"5min":0.54900000000000004,"15min":0.437},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":7.2770000000000001,"5min":7.2770000000000001,"15min":7.2770000000000001},"last":2.714},{"interface":"front","average":{"1min":1.038,"5min":0.64900000000000002,"15min":0.58399999999999996},"min":{"1min":0.307,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":7.5389999999999997,"5min":7.5389999999999997,"15min":7.5389999999999997},"last":1.087}]},{"osd":3,"last update":"Sat Sep 23 20:22:52 2023","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":3.6059999999999999,"5min":3.6059999999999999,"15min":3.6059999999999999},"last":0.751},{"interface":"front","average":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.66900000000000004,"5min":0.66900000000000004,"15min":0.66900000000000004},"last":0.26900000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:22:47 2023","interfaces":[{"interface":"back","average":{"1min":0.624,"5min":0.61799999999999999,"15min":0.61699999999999999},"min":{"1min":0.41599999999999998,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.93300000000000005,"5min":0.93300000000000005,"15min":0.93300000000000005},"last":4.1189999999999998},{"interface":"front","average":{"1min":0.625,"5min":0.63300000000000001,"15min":0.63400000000000001},"min":{"1min":0.49299999999999999,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":0.84799999999999998,"5min":0.85199999999999998,"15min":0.85199999999999998},"last":0.78600000000000003}]},{"osd":5,"last update":"Sat Sep 23 20:22:47 2023","interfaces":[{"interface":"back","average":{"1min":0.65300000000000002,"5min":0.64300000000000002,"15min":0.64100000000000001},"min":{"1min":0.45600000000000002,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":0.89500000000000002,"5min":0.89500000000000002,"15min":0.89500000000000002},"last":0.55400000000000005},{"interface":"front","average":{"1min":0.66600000000000004,"5min":0.67400000000000004,"15min":0.67500000000000004},"min":{"1min":0.46899999999999997,"5min":0.46400000000000002,"15min":0.46400000000000002},"max":{"1min":0.96999999999999997,"5min":0.96999999999999997,"15min":0.96999999999999997},"last":0.93000000000000005}]},{"osd":6,"last update":"Sat Sep 23 20:22:47 2023","interfaces":[{"interface":"back","average":{"1min":0.629,"5min":0.66000000000000003,"15min":0.66500000000000004},"min":{"1min":0.42699999999999999,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.84799999999999998,"5min":0.92200000000000004,"15min":0.92200000000000004},"last":0.52100000000000002},{"interface":"front","average":{"1min":0.63900000000000001,"5min":0.63800000000000001,"15min":0.63800000000000001},"min":{"1min":0.47799999999999998,"5min":0.433,"15min":0.433},"max":{"1min":1,"5min":1,"15min":1},"last":0.89500000000000002}]},{"osd":7,"last update":"Sat Sep 23 20:22:47 2023","interfaces":[{"interface":"back","average":{"1min":0.63700000000000001,"5min":0.64900000000000002,"15min":0.65100000000000002},"min":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"max":{"1min":1.0309999999999999,"5min":1.0309999999999999,"15min":1.0309999999999999},"last":0.96899999999999997},{"interface":"front","average":{"1min":0.64600000000000002,"5min":0.67600000000000005,"15min":0.68100000000000005},"min":{"1min":0.48499999999999999,"5min":0.45300000000000001,"15min":0.45300000000000001},"max":{"1min":0.86499999999999999,"5min":0.86499999999999999,"15min":0.86499999999999999},"last":1.1899999999999999}]}]},{"osd":0,"up_from":94,"seq":403726925865,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1523768,"kb_used_data":1474032,"kb_used_omap":7,"kb_used_meta":49720,"kb_avail":92221384,"statfs":{"total":95995035648,"available":94434697216,"internally_reserved":0,"allocated":1509408768,"data_stored":1508771885,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":50913297},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:22:52 2023","interfaces":[{"interface":"back","average":{"1min":0.34300000000000003,"5min":0.35499999999999998,"15min":0.35699999999999998},"min":{"1min":0.17699999999999999,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":0.52900000000000003,"5min":0.55400000000000005,"15min":0.55400000000000005},"last":1.1739999999999999},{"interface":"front","average":{"1min":0.39600000000000002,"5min":0.38800000000000001,"15min":0.38600000000000001},"min":{"1min":0.27400000000000002,"5min":0.253,"15min":0.253},"max":{"1min":0.63,"5min":0.63,"15min":0.63},"last":0.749}]},{"osd":2,"last update":"Sat Sep 23 20:23:19 2023","interfaces":[{"interface":"back","average":{"1min":0.64900000000000002,"5min":0.60699999999999998,"15min":0.59999999999999998},"min":{"1min":0.249,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":3.6219999999999999,"5min":4.6280000000000001,"15min":4.6280000000000001},"last":1.9279999999999999},{"interface":"front","average":{"1min":0.748,"5min":0.54600000000000004,"15min":0.51200000000000001},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":3.4049999999999998,"5min":3.4049999999999998,"15min":3.4049999999999998},"last":0.86799999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:22:52 2023","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":4.6820000000000004,"5min":4.6820000000000004,"15min":4.6820000000000004},"last":1.891},{"interface":"front","average":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"last":0.54700000000000004}]},{"osd":4,"last update":"Sat Sep 23 20:23:19 2023","interfaces":[{"interface":"back","average":{"1min":0.63200000000000001,"5min":0.64200000000000002,"15min":0.61199999999999999},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":1.423,"5min":4.7450000000000001,"15min":4.7450000000000001},"last":0.97399999999999998},{"interface":"front","average":{"1min":0.65100000000000002,"5min":0.62,"15min":0.61599999999999999},"min":{"1min":0.42999999999999999,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":1.177,"5min":1.177,"15min":1.177},"last":0.51300000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:23:19 2023","interfaces":[{"interface":"back","average":{"1min":0.67600000000000005,"5min":0.65900000000000003,"15min":0.65900000000000003},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":1.208,"5min":1.208,"15min":1.208},"last":0.57899999999999996},{"interface":"front","average":{"1min":0.65500000000000003,"5min":0.67200000000000004,"15min":0.67600000000000005},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":1.4630000000000001,"5min":1.4630000000000001,"15min":1.4630000000000001},"last":0.78200000000000003}]},{"osd":6,"last update":"Sat Sep 23 20:23:19 2023","interfaces":[{"interface":"back","average":{"1min":0.72799999999999998,"5min":0.67200000000000004,"15min":0.65600000000000003},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":1.4430000000000001,"5min":1.4430000000000001,"15min":1.4430000000000001},"last":0.60199999999999998},{"interface":"front","average":{"1min":0.68600000000000005,"5min":0.67200000000000004,"15min":0.66600000000000004},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":1.524,"5min":1.524,"15min":1.524},"last":0.81399999999999995}]},{"osd":7,"last update":"Sat Sep 23 20:23:19 2023","interfaces":[{"interface":"back","average":{"1min":0.68899999999999995,"5min":0.67800000000000005,"15min":0.67800000000000005},"min":{"1min":0.46200000000000002,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":1.5660000000000001,"5min":1.5660000000000001,"15min":1.5660000000000001},"last":0.84399999999999997},{"interface":"front","average":{"1min":0.66700000000000004,"5min":0.70499999999999996,"15min":0.67800000000000005},"min":{"1min":0.436,"5min":0.436,"15min":0.436},"max":{"1min":1.46,"5min":4.7140000000000004,"15min":4.7140000000000004},"last":1.016}]}]},{"osd":2,"up_from":104,"seq":446676598813,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1267740,"kb_used_data":1213972,"kb_used_omap":11,"kb_used_meta":53748,"kb_avail":92477412,"statfs":{"total":95995035648,"available":94696869888,"internally_reserved":0,"allocated":1243107328,"data_stored":1242501444,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11878,"internal_metadata":55038362},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,0,0,3],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:23:21 2023","interfaces":[{"interface":"back","average":{"1min":0.98099999999999998,"5min":0.497,"15min":0.41699999999999998},"min":{"1min":0.216,"5min":0.216,"15min":0.216},"max":{"1min":9.1519999999999992,"5min":9.1519999999999992,"15min":9.1519999999999992},"last":2.0059999999999998},{"interface":"front","average":{"1min":0.98999999999999999,"5min":0.51400000000000001,"15min":0.435},"min":{"1min":0.28100000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":10.371,"5min":10.371,"15min":10.371},"last":2.1859999999999999}]},{"osd":1,"last update":"Sat Sep 23 20:23:21 2023","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.41999999999999998,"15min":0.40200000000000002},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":2.6000000000000001,"5min":2.6000000000000001,"15min":2.6000000000000001},"last":0.57499999999999996},{"interface":"front","average":{"1min":0.60099999999999998,"5min":0.42599999999999999,"15min":0.39700000000000002},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":2.859,"5min":2.859,"15min":2.859},"last":2.1360000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:22:49 2023","interfaces":[{"interface":"back","average":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.504,"5min":0.504,"15min":0.504},"last":2.6429999999999998},{"interface":"front","average":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.50600000000000001,"5min":0.50600000000000001,"15min":0.50600000000000001},"last":0.51300000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:23:21 2023","interfaces":[{"interface":"back","average":{"1min":0.76000000000000001,"5min":0.61899999999999999,"15min":0.59499999999999997},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":2.1760000000000002,"5min":2.1760000000000002,"15min":2.1760000000000002},"last":1.431},{"interface":"front","average":{"1min":0.77800000000000002,"5min":0.623,"15min":0.59699999999999998},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":2.1970000000000001,"5min":2.1970000000000001,"15min":2.1970000000000001},"last":2.0579999999999998}]},{"osd":5,"last update":"Sat Sep 23 20:23:21 2023","interfaces":[{"interface":"back","average":{"1min":0.83999999999999997,"5min":0.69299999999999995,"15min":0.66900000000000004},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":2.2160000000000002,"5min":2.2160000000000002,"15min":2.2160000000000002},"last":1.353},{"interface":"front","average":{"1min":0.96599999999999997,"5min":0.71799999999999997,"15min":0.67700000000000005},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":4.1920000000000002,"5min":4.1920000000000002,"15min":4.1920000000000002},"last":2.0979999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:23:21 2023","interfaces":[{"interface":"back","average":{"1min":0.88500000000000001,"5min":0.70199999999999996,"15min":0.67200000000000004},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":4.0869999999999997,"5min":4.0869999999999997,"15min":4.0869999999999997},"last":2.5510000000000002},{"interface":"front","average":{"1min":0.74399999999999999,"5min":0.68700000000000006,"15min":0.67700000000000005},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":2.4489999999999998,"5min":2.4489999999999998,"15min":2.4489999999999998},"last":0.60699999999999998}]},{"osd":7,"last update":"Sat Sep 23 20:23:21 2023","interfaces":[{"interface":"back","average":{"1min":0.97499999999999998,"5min":0.71299999999999997,"15min":0.66900000000000004},"min":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":4.3040000000000003,"5min":4.3040000000000003,"15min":4.3040000000000003},"last":0.63900000000000001},{"interface":"front","average":{"1min":1.0189999999999999,"5min":0.72899999999999998,"15min":0.68100000000000005},"min":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":4.6100000000000003,"5min":4.6100000000000003,"15min":4.6100000000000003},"last":1.3919999999999999}]}]},{"osd":3,"up_from":109,"seq":468151435286,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":638464,"kb_used_data":610936,"kb_used_omap":4,"kb_used_meta":27515,"kb_avail":93106688,"statfs":{"total":95995035648,"available":95341248512,"internally_reserved":0,"allocated":625598464,"data_stored":625009673,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4514,"internal_metadata":28175966},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:22:49 2023","interfaces":[{"interface":"back","average":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.80500000000000005,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":2.0579999999999998},{"interface":"front","average":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.67200000000000004,"5min":0.67200000000000004,"15min":0.67200000000000004},"last":1.974}]},{"osd":1,"last update":"Sat Sep 23 20:22:49 2023","interfaces":[{"interface":"back","average":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"min":{"1min":0.221,"5min":0.221,"15min":0.221},"max":{"1min":1.145,"5min":1.145,"15min":1.145},"last":0.90000000000000002},{"interface":"front","average":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.44500000000000001},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":1.1930000000000001,"5min":1.1930000000000001,"15min":1.1930000000000001},"last":0.95999999999999996}]},{"osd":2,"last update":"Sat Sep 23 20:22:49 2023","interfaces":[{"interface":"back","average":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.55700000000000005,"5min":0.55700000000000005,"15min":0.55700000000000005},"last":1.5009999999999999},{"interface":"front","average":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":1.2270000000000001,"5min":1.2270000000000001,"15min":1.2270000000000001},"last":2.9369999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:22:49 2023","interfaces":[{"interface":"back","average":{"1min":0.60499999999999998,"5min":0.60499999999999998,"15min":0.60499999999999998},"min":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":0.93799999999999994,"5min":0.93799999999999994,"15min":0.93799999999999994},"last":0.35699999999999998},{"interface":"front","average":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":1.2829999999999999,"5min":1.2829999999999999,"15min":1.2829999999999999},"last":0.93200000000000005}]},{"osd":5,"last update":"Sat Sep 23 20:22:49 2023","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.97299999999999998,"5min":0.97299999999999998,"15min":0.97299999999999998},"last":0.38600000000000001},{"interface":"front","average":{"1min":0.63100000000000001,"5min":0.63100000000000001,"15min":0.63100000000000001},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":0.90800000000000003,"5min":0.90800000000000003,"15min":0.90800000000000003},"last":1.5309999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:22:49 2023","interfaces":[{"interface":"back","average":{"1min":0.63800000000000001,"5min":0.63800000000000001,"15min":0.63800000000000001},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.92700000000000005,"5min":0.92700000000000005,"15min":0.92700000000000005},"last":1.5580000000000001},{"interface":"front","average":{"1min":0.67200000000000004,"5min":0.67200000000000004,"15min":0.67200000000000004},"min":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":1.6020000000000001,"5min":1.6020000000000001,"15min":1.6020000000000001},"last":1.069}]},{"osd":7,"last update":"Sat Sep 23 20:22:49 2023","interfaces":[{"interface":"back","average":{"1min":0.70299999999999996,"5min":0.70299999999999996,"15min":0.70299999999999996},"min":{"1min":0.439,"5min":0.439,"15min":0.439},"max":{"1min":1.6439999999999999,"5min":1.6439999999999999,"15min":1.6439999999999999},"last":1.0369999999999999},{"interface":"front","average":{"1min":0.64200000000000002,"5min":0.64200000000000002,"15min":0.64200000000000002},"min":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":1.006,"5min":1.006,"15min":1.006},"last":0.41099999999999998}]}]},{"osd":4,"up_from":30,"seq":128849019040,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1624772,"kb_used_data":1590908,"kb_used_omap":0,"kb_used_meta":33856,"kb_avail":92120380,"statfs":{"total":95995035648,"available":94331269120,"internally_reserved":0,"allocated":1629089792,"data_stored":1628562315,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":34668544},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:23:17 2023","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":0.57299999999999995,"15min":0.56999999999999995},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":0.79200000000000004,"5min":0.85099999999999998,"15min":0.85099999999999998},"last":0.48899999999999999},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.629,"15min":0.63700000000000001},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.94499999999999995,"5min":0.94499999999999995,"15min":0.94499999999999995},"last":0.54200000000000004}]},{"osd":1,"last update":"Sat Sep 23 20:22:51 2023","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.61199999999999999,"15min":0.621},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.749,"5min":1.4219999999999999,"15min":1.4219999999999999},"last":0.434},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.61099999999999999,"15min":0.621},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.74199999999999999,"5min":1.393,"15min":1.393},"last":4.0789999999999997}]},{"osd":2,"last update":"Sat Sep 23 20:23:22 2023","interfaces":[{"interface":"back","average":{"1min":0.68500000000000005,"5min":0.623,"15min":0.61299999999999999},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":1.865,"5min":1.865,"15min":1.865},"last":2.4420000000000002},{"interface":"front","average":{"1min":0.997,"5min":0.67000000000000004,"15min":0.61599999999999999},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":2.794,"5min":2.794,"15min":2.794},"last":0.85199999999999998}]},{"osd":3,"last update":"Sat Sep 23 20:22:51 2023","interfaces":[{"interface":"back","average":{"1min":0.64100000000000001,"5min":0.64100000000000001,"15min":0.64100000000000001},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.751,"5min":0.751,"15min":0.751},"last":0.97099999999999997},{"interface":"front","average":{"1min":0.66500000000000004,"5min":0.66500000000000004,"15min":0.66500000000000004},"min":{"1min":0.51000000000000001,"5min":0.51000000000000001,"15min":0.51000000000000001},"max":{"1min":0.82599999999999996,"5min":0.82599999999999996,"15min":0.82599999999999996},"last":0.52100000000000002}]},{"osd":5,"last update":"Sat Sep 23 20:23:09 2023","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.51200000000000001,"15min":0.46500000000000002},"min":{"1min":0.313,"5min":0.313,"15min":0.26000000000000001},"max":{"1min":0.97899999999999998,"5min":3.5270000000000001,"15min":3.5270000000000001},"last":0.46600000000000003},{"interface":"front","average":{"1min":0.53000000000000003,"5min":0.52300000000000002,"15min":0.47599999999999998},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.248},"max":{"1min":0.82199999999999995,"5min":0.86299999999999999,"15min":0.86299999999999999},"last":0.40400000000000003}]},{"osd":6,"last update":"Sat Sep 23 20:22:27 2023","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":0.54600000000000004,"15min":0.497},"min":{"1min":0.46999999999999997,"5min":0.35599999999999998,"15min":0.27500000000000002},"max":{"1min":0.76000000000000001,"5min":0.77900000000000003,"15min":0.82499999999999996},"last":0.58299999999999996},{"interface":"front","average":{"1min":0.56999999999999995,"5min":0.56599999999999995,"15min":0.51000000000000001},"min":{"1min":0.41499999999999998,"5min":0.32100000000000001,"15min":0.314},"max":{"1min":0.73299999999999998,"5min":3.4820000000000002,"15min":3.4820000000000002},"last":0.503}]},{"osd":7,"last update":"Sat Sep 23 20:22:41 2023","interfaces":[{"interface":"back","average":{"1min":0.66000000000000003,"5min":0.58699999999999997,"15min":0.55500000000000005},"min":{"1min":0.48999999999999999,"5min":0.38700000000000001,"15min":0.33600000000000002},"max":{"1min":0.84499999999999997,"5min":0.91600000000000004,"15min":0.91600000000000004},"last":1.0469999999999999},{"interface":"front","average":{"1min":0.63600000000000001,"5min":0.61799999999999999,"15min":0.56299999999999994},"min":{"1min":0.47799999999999998,"5min":0.39600000000000002,"15min":0.29899999999999999},"max":{"1min":0.80200000000000005,"5min":3.77,"15min":3.77},"last":0.53100000000000003}]}]},{"osd":5,"up_from":39,"seq":167503724701,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1804724,"kb_used_data":1759788,"kb_used_omap":0,"kb_used_meta":44928,"kb_avail":91940428,"statfs":{"total":95995035648,"available":94146998272,"internally_reserved":0,"allocated":1802022912,"data_stored":1801499777,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":46006272},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:23:19 2023","interfaces":[{"interface":"back","average":{"1min":0.629,"5min":0.60399999999999998,"15min":0.59599999999999997},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":1.7629999999999999,"5min":1.7629999999999999,"15min":1.7629999999999999},"last":0.252},{"interface":"front","average":{"1min":0.78400000000000003,"5min":0.63300000000000001,"15min":0.61099999999999999},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":2.056,"5min":2.056,"15min":2.056},"last":4.2249999999999996}]},{"osd":1,"last update":"Sat Sep 23 20:22:50 2023","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.54600000000000004,"15min":0.54500000000000004},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":0.72199999999999998,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.44900000000000001},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"min":{"1min":0.46000000000000002,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.69199999999999995,"5min":0.84999999999999998,"15min":0.84999999999999998},"last":1.1439999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:23:19 2023","interfaces":[{"interface":"back","average":{"1min":0.878,"5min":0.61499999999999999,"15min":0.57099999999999995},"min":{"1min":0.38,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":2.8660000000000001,"5min":2.8660000000000001,"15min":2.8660000000000001},"last":0.46800000000000003},{"interface":"front","average":{"1min":0.81299999999999994,"5min":0.60899999999999999,"15min":0.57499999999999996},"min":{"1min":0.39000000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":2.0840000000000001,"5min":2.0840000000000001,"15min":2.0840000000000001},"last":1.4510000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:22:50 2023","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.72799999999999998,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":0.40100000000000002},{"interface":"front","average":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":0.78100000000000003,"5min":0.78100000000000003,"15min":0.78100000000000003},"last":0.30599999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:23:14 2023","interfaces":[{"interface":"back","average":{"1min":0.81499999999999995,"5min":0.55400000000000005,"15min":0.46700000000000003},"min":{"1min":0.33200000000000002,"5min":0.32700000000000001,"15min":0.26400000000000001},"max":{"1min":2.4369999999999998,"5min":2.4369999999999998,"15min":2.4369999999999998},"last":0.29099999999999998},{"interface":"front","average":{"1min":0.78500000000000003,"5min":0.58299999999999996,"15min":0.503},"min":{"1min":0.34000000000000002,"5min":0.29299999999999998,"15min":0.28799999999999998},"max":{"1min":2.496,"5min":2.496,"15min":2.496},"last":0.34499999999999997}]},{"osd":6,"last update":"Sat Sep 23 20:23:27 2023","interfaces":[{"interface":"back","average":{"1min":0.78300000000000003,"5min":0.57899999999999996,"15min":0.51500000000000001},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.29799999999999999},"max":{"1min":2.4319999999999999,"5min":2.4319999999999999,"15min":2.4319999999999999},"last":0.36799999999999999},{"interface":"front","average":{"1min":0.82299999999999995,"5min":0.65900000000000003,"15min":0.58299999999999996},"min":{"1min":0.378,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":2.5219999999999998,"5min":2.5219999999999998,"15min":2.5219999999999998},"last":0.41299999999999998}]},{"osd":7,"last update":"Sat Sep 23 20:22:50 2023","interfaces":[{"interface":"back","average":{"1min":0.625,"5min":0.57799999999999996,"15min":0.54400000000000004},"min":{"1min":0.42499999999999999,"5min":0.372,"15min":0.33800000000000002},"max":{"1min":0.873,"5min":0.873,"15min":0.98199999999999998},"last":0.38600000000000001},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.58199999999999996,"15min":0.54900000000000004},"min":{"1min":0.38700000000000001,"5min":0.315,"15min":0.315},"max":{"1min":0.81399999999999995,"5min":0.81399999999999995,"15min":1.0489999999999999},"last":1.095}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4722,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1360,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6696,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7156,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3154,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1507655680,"data_stored":1507655680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1082130432,"data_stored":1082130432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1250557952,"data_stored":1250557952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":634060800,"data_stored":634060800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1631911936,"data_stored":1631911936,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1822359552,"data_stored":1822359552,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":984678400,"data_stored":984678400,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1133051904,"data_stored":1133051904,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:23:34.083 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:23:34.167 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:34.290 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:23:34.294 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:34.295 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:23:34.344 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:23:34.395 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:23:34.433 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:34 smithi149 ceph-mon[170394]: pgmap v272: 99 pgs: 1 creating+peering, 2 active+recovering+undersized+remapped, 96 active+clean; 3.1 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s; 3156/151080 objects misplaced (2.089%); 22 MiB/s, 352 objects/s recovering 2023-09-23T20:23:34.440 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:34.451 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:34 smithi018 ceph-mon[184144]: pgmap v272: 99 pgs: 1 creating+peering, 2 active+recovering+undersized+remapped, 96 active+clean; 3.1 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s; 3156/151080 objects misplaced (2.089%); 22 MiB/s, 352 objects/s recovering 2023-09-23T20:23:34.451 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:34 smithi018 ceph-mon[179647]: pgmap v272: 99 pgs: 1 creating+peering, 2 active+recovering+undersized+remapped, 96 active+clean; 3.1 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s; 3156/151080 objects misplaced (2.089%); 22 MiB/s, 352 objects/s recovering 2023-09-23T20:23:34.464 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 36 16 54162 54146 93.9692 102.438 0.00819213 0.0106371 2023-09-23T20:23:34.487 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:23:34.489 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:34.490 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:23:34.590 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:23:34.670 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:34.773 INFO:teuthology.orchestra.run.smithi018.stdout:ERROR: (22) Invalid argument 2023-09-23T20:23:34.773 INFO:teuthology.orchestra.run.smithi018.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:23:34.787 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:23:34.792 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:34.793 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:23:34.893 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:23:34.970 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:35.045 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:23:34 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:23:34] "GET /metrics HTTP/1.1" 200 36474 "" "Prometheus/2.43.0" 2023-09-23T20:23:35.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:35 smithi018 ceph-mon[179647]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2620/143040 objects degraded (1.832%), 1 pg degraded) 2023-09-23T20:23:35.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:35 smithi018 ceph-mon[184144]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2620/143040 objects degraded (1.832%), 1 pg degraded) 2023-09-23T20:23:35.434 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:23:35.466 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 37 16 55828 55812 94.2405 104.125 0.0190274 0.0106063 2023-09-23T20:23:35.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:35 smithi149 ceph-mon[170394]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2620/143040 objects degraded (1.832%), 1 pg degraded) 2023-09-23T20:23:36.450 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:23:36.453 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:36 smithi018 ceph-mon[179647]: pgmap v273: 99 pgs: 1 active+recovering+undersized+remapped, 98 active+clean; 3.3 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 100 MiB/s wr, 1.59k op/s; 2156/160335 objects misplaced (1.345%); 11 MiB/s, 175 objects/s recovering 2023-09-23T20:23:36.453 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:36 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "21"}]: dispatch 2023-09-23T20:23:36.454 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:36.454 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:23:36.454 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:36 smithi018 ceph-mon[184144]: pgmap v273: 99 pgs: 1 active+recovering+undersized+remapped, 98 active+clean; 3.3 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 100 MiB/s wr, 1.59k op/s; 2156/160335 objects misplaced (1.345%); 11 MiB/s, 175 objects/s recovering 2023-09-23T20:23:36.455 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:36 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "21"}]: dispatch 2023-09-23T20:23:36.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:36 smithi149 ceph-mon[170394]: pgmap v273: 99 pgs: 1 active+recovering+undersized+remapped, 98 active+clean; 3.3 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 100 MiB/s wr, 1.59k op/s; 2156/160335 objects misplaced (1.345%); 11 MiB/s, 175 objects/s recovering 2023-09-23T20:23:36.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:36 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "21"}]: dispatch 2023-09-23T20:23:36.466 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 38 16 57468 57452 94.4575 102.5 0.00225816 0.0105534 2023-09-23T20:23:36.497 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T20:23:36.555 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:23:36.583 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:36.618 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:23:36.621 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:36.621 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:23:36.722 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:23:36.747 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:36.779 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:23:36.781 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:36.782 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:23:36.882 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:23:36.914 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:37.108 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:23:37.174 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:37 smithi018 ceph-mon[184144]: failed to encode map e133 with expected crc 2023-09-23T20:23:37.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:37 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "21"}]': finished 2023-09-23T20:23:37.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:37 smithi018 ceph-mon[184144]: osdmap e133: 8 total, 8 up, 8 in 2023-09-23T20:23:37.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:37 smithi018 ceph-mon[179647]: failed to encode map e133 with expected crc 2023-09-23T20:23:37.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:37 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "21"}]': finished 2023-09-23T20:23:37.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:37 smithi018 ceph-mon[179647]: osdmap e133: 8 total, 8 up, 8 in 2023-09-23T20:23:37.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:37 smithi149 ceph-mon[170394]: failed to encode map e133 with expected crc 2023-09-23T20:23:37.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:37 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "21"}]': finished 2023-09-23T20:23:37.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:37 smithi149 ceph-mon[170394]: osdmap e133: 8 total, 8 up, 8 in 2023-09-23T20:23:37.467 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 39 16 57550 57534 92.1676 5.125 0.00161698 0.0105425 2023-09-23T20:23:38.256 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:23:38.259 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:38.260 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:23:38.360 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:23:38.377 INFO:teuthology.orchestra.run.smithi149.stdout:ERROR: (22) Invalid argument 2023-09-23T20:23:38.377 INFO:teuthology.orchestra.run.smithi149.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:23:38.408 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:38 smithi149 ceph-mon[170394]: osdmap e134: 8 total, 8 up, 8 in 2023-09-23T20:23:38.408 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:38 smithi149 ceph-mon[170394]: failed to encode map e134 with expected crc 2023-09-23T20:23:38.408 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:38 smithi149 ceph-mon[170394]: failed to encode map e134 with expected crc 2023-09-23T20:23:38.409 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:38 smithi149 ceph-mon[170394]: pgmap v276: 99 pgs: 1 active+recovering+undersized+remapped, 98 active+clean; 3.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 82 MiB/s wr, 1.32k op/s; 2156/166746 objects misplaced (1.293%); 7.6 MiB/s, 121 objects/s recovering 2023-09-23T20:23:38.409 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:38 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "22"}]: dispatch 2023-09-23T20:23:38.429 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:38.472 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:2023-09-23T20:23:38.472880+0000 min lat: 0.00112599 max lat: 2.05382 avg lat: 0.0110563 2023-09-23T20:23:38.472 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T20:23:38.472 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 40 16 57847 57831 90.3148 18.5625 0.00294604 0.0110563 2023-09-23T20:23:38.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:38 smithi018 ceph-mon[179647]: osdmap e134: 8 total, 8 up, 8 in 2023-09-23T20:23:38.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:38 smithi018 ceph-mon[179647]: failed to encode map e134 with expected crc 2023-09-23T20:23:38.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:38 smithi018 ceph-mon[179647]: failed to encode map e134 with expected crc 2023-09-23T20:23:38.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:38 smithi018 ceph-mon[179647]: pgmap v276: 99 pgs: 1 active+recovering+undersized+remapped, 98 active+clean; 3.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 82 MiB/s wr, 1.32k op/s; 2156/166746 objects misplaced (1.293%); 7.6 MiB/s, 121 objects/s recovering 2023-09-23T20:23:38.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:38 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "22"}]: dispatch 2023-09-23T20:23:38.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:38 smithi018 ceph-mon[184144]: osdmap e134: 8 total, 8 up, 8 in 2023-09-23T20:23:38.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:38 smithi018 ceph-mon[184144]: failed to encode map e134 with expected crc 2023-09-23T20:23:38.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:38 smithi018 ceph-mon[184144]: failed to encode map e134 with expected crc 2023-09-23T20:23:38.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:38 smithi018 ceph-mon[184144]: pgmap v276: 99 pgs: 1 active+recovering+undersized+remapped, 98 active+clean; 3.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 82 MiB/s wr, 1.32k op/s; 2156/166746 objects misplaced (1.293%); 7.6 MiB/s, 121 objects/s recovering 2023-09-23T20:23:38.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:38 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "22"}]: dispatch 2023-09-23T20:23:39.061 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T20:23:39.075 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-23T20:23:39.075 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:23:39.076 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-09-23T20:23:39.076 INFO:tasks.thrashosds.thrasher:Setting osd 0 primary_affinity to 1.000000 2023-09-23T20:23:39.076 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 0 1 2023-09-23T20:23:39.323 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:23:39.326 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:39.327 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:23:39.327 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:23:39.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:39 smithi018 ceph-mon[184144]: failed to encode map e134 with expected crc 2023-09-23T20:23:39.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:39 smithi018 ceph-mon[184144]: failed to encode map e134 with expected crc 2023-09-23T20:23:39.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:39 smithi018 ceph-mon[184144]: failed to encode map e135 with expected crc 2023-09-23T20:23:39.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:39 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "22"}]': finished 2023-09-23T20:23:39.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:39 smithi018 ceph-mon[184144]: osdmap e135: 8 total, 8 up, 8 in 2023-09-23T20:23:39.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:39 smithi018 ceph-mon[184144]: failed to encode map e135 with expected crc 2023-09-23T20:23:39.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:39 smithi018 ceph-mon[184144]: failed to encode map e135 with expected crc 2023-09-23T20:23:39.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:39 smithi018 ceph-mon[179647]: failed to encode map e134 with expected crc 2023-09-23T20:23:39.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:39 smithi018 ceph-mon[179647]: failed to encode map e134 with expected crc 2023-09-23T20:23:39.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:39 smithi018 ceph-mon[179647]: failed to encode map e135 with expected crc 2023-09-23T20:23:39.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:39 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "22"}]': finished 2023-09-23T20:23:39.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:39 smithi018 ceph-mon[179647]: osdmap e135: 8 total, 8 up, 8 in 2023-09-23T20:23:39.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:39 smithi018 ceph-mon[179647]: failed to encode map e135 with expected crc 2023-09-23T20:23:39.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:39 smithi018 ceph-mon[179647]: failed to encode map e135 with expected crc 2023-09-23T20:23:39.427 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:23:39.453 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:39 smithi149 ceph-mon[170394]: failed to encode map e134 with expected crc 2023-09-23T20:23:39.453 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:39 smithi149 ceph-mon[170394]: failed to encode map e134 with expected crc 2023-09-23T20:23:39.453 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:39 smithi149 ceph-mon[170394]: failed to encode map e135 with expected crc 2023-09-23T20:23:39.453 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:39 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "22"}]': finished 2023-09-23T20:23:39.453 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:39 smithi149 ceph-mon[170394]: osdmap e135: 8 total, 8 up, 8 in 2023-09-23T20:23:39.453 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:39 smithi149 ceph-mon[170394]: failed to encode map e135 with expected crc 2023-09-23T20:23:39.453 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:39 smithi149 ceph-mon[170394]: failed to encode map e135 with expected crc 2023-09-23T20:23:39.455 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:39.472 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 41 16 58138 58122 88.5562 18.1875 0.00191851 0.0110275 2023-09-23T20:23:39.487 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:23:39.490 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:39.490 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:23:39.591 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:23:39.618 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:39.650 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:23:39.652 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:39.652 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:23:39.659 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:23:39.754 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:23:39.780 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:39.862 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:23:39.865 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:39.865 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:23:39.966 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:23:39.995 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:40.286 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:40 smithi018 ceph-mon[184144]: failed to encode map e135 with expected crc 2023-09-23T20:23:40.286 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:40 smithi018 ceph-mon[184144]: osdmap e136: 8 total, 8 up, 8 in 2023-09-23T20:23:40.286 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:40 smithi018 ceph-mon[184144]: failed to encode map e136 with expected crc 2023-09-23T20:23:40.286 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:40 smithi018 ceph-mon[184144]: failed to encode map e136 with expected crc 2023-09-23T20:23:40.286 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:40 smithi018 ceph-mon[184144]: failed to encode map e136 with expected crc 2023-09-23T20:23:40.286 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:40 smithi018 ceph-mon[184144]: pgmap v279: 99 pgs: 1 active+recovering+undersized+remapped, 98 active+clean; 3.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 48 MiB/s wr, 764 op/s; 560/169509 objects misplaced (0.330%); 12 MiB/s, 199 objects/s recovering 2023-09-23T20:23:40.286 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:40 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "23"}]: dispatch 2023-09-23T20:23:40.287 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:40 smithi018 ceph-mon[179647]: failed to encode map e135 with expected crc 2023-09-23T20:23:40.287 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:40 smithi018 ceph-mon[179647]: osdmap e136: 8 total, 8 up, 8 in 2023-09-23T20:23:40.287 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:40 smithi018 ceph-mon[179647]: failed to encode map e136 with expected crc 2023-09-23T20:23:40.287 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:40 smithi018 ceph-mon[179647]: failed to encode map e136 with expected crc 2023-09-23T20:23:40.287 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:40 smithi018 ceph-mon[179647]: failed to encode map e136 with expected crc 2023-09-23T20:23:40.287 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:40 smithi018 ceph-mon[179647]: pgmap v279: 99 pgs: 1 active+recovering+undersized+remapped, 98 active+clean; 3.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 48 MiB/s wr, 764 op/s; 560/169509 objects misplaced (0.330%); 12 MiB/s, 199 objects/s recovering 2023-09-23T20:23:40.287 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:40 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "23"}]: dispatch 2023-09-23T20:23:40.287 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:23:40.296 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:40.296 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:23:40.397 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:23:40.418 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:40 smithi149 ceph-mon[170394]: failed to encode map e135 with expected crc 2023-09-23T20:23:40.473 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:40 smithi149 ceph-mon[170394]: osdmap e136: 8 total, 8 up, 8 in 2023-09-23T20:23:40.473 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:40 smithi149 ceph-mon[170394]: failed to encode map e136 with expected crc 2023-09-23T20:23:40.473 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:40 smithi149 ceph-mon[170394]: failed to encode map e136 with expected crc 2023-09-23T20:23:40.474 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:40 smithi149 ceph-mon[170394]: failed to encode map e136 with expected crc 2023-09-23T20:23:40.474 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:40 smithi149 ceph-mon[170394]: pgmap v279: 99 pgs: 1 active+recovering+undersized+remapped, 98 active+clean; 3.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 48 MiB/s wr, 764 op/s; 560/169509 objects misplaced (0.330%); 12 MiB/s, 199 objects/s recovering 2023-09-23T20:23:40.474 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:40 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "23"}]: dispatch 2023-09-23T20:23:40.474 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:40.475 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 42 16 58433 58417 86.8874 18.4375 0.0119902 0.0114845 2023-09-23T20:23:40.610 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:23:40.613 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:40.614 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:23:40.714 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:23:40.761 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:40.808 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:23:40.811 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:40.812 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:23:40.912 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:23:40.945 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:40.946 INFO:teuthology.orchestra.run.smithi149.stdout:ERROR: (22) Invalid argument 2023-09-23T20:23:40.946 INFO:teuthology.orchestra.run.smithi149.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:23:40.982 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:23:40.985 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:40.986 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:23:41.013 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:23:41.087 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:23:41.118 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:41.250 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T20:23:41.255 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:23:41.257 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:41.257 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:23:41.328 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:41 smithi149 ceph-mon[170394]: failed to encode map e136 with expected crc 2023-09-23T20:23:41.328 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:41 smithi149 ceph-mon[170394]: failed to encode map e137 with expected crc 2023-09-23T20:23:41.328 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:41 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "23"}]': finished 2023-09-23T20:23:41.328 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:41 smithi149 ceph-mon[170394]: osdmap e137: 8 total, 8 up, 8 in 2023-09-23T20:23:41.328 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:41 smithi149 ceph-mon[170394]: failed to encode map e137 with expected crc 2023-09-23T20:23:41.329 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:41 smithi149 ceph-mon[170394]: failed to encode map e137 with expected crc 2023-09-23T20:23:41.329 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:41 smithi149 ceph-mon[170394]: osdmap e138: 8 total, 8 up, 8 in 2023-09-23T20:23:41.329 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:41 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1293740956' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:23:41.329 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:41 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:23:41.336 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:41 smithi018 ceph-mon[184144]: failed to encode map e136 with expected crc 2023-09-23T20:23:41.336 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:41 smithi018 ceph-mon[184144]: failed to encode map e137 with expected crc 2023-09-23T20:23:41.336 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:41 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "23"}]': finished 2023-09-23T20:23:41.336 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:41 smithi018 ceph-mon[184144]: osdmap e137: 8 total, 8 up, 8 in 2023-09-23T20:23:41.336 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:41 smithi018 ceph-mon[184144]: failed to encode map e137 with expected crc 2023-09-23T20:23:41.336 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:41 smithi018 ceph-mon[184144]: failed to encode map e137 with expected crc 2023-09-23T20:23:41.336 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:41 smithi018 ceph-mon[184144]: osdmap e138: 8 total, 8 up, 8 in 2023-09-23T20:23:41.336 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:41 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1293740956' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:23:41.337 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:41 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:23:41.337 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:41 smithi018 ceph-mon[179647]: failed to encode map e136 with expected crc 2023-09-23T20:23:41.337 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:41 smithi018 ceph-mon[179647]: failed to encode map e137 with expected crc 2023-09-23T20:23:41.337 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:41 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "23"}]': finished 2023-09-23T20:23:41.337 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:41 smithi018 ceph-mon[179647]: osdmap e137: 8 total, 8 up, 8 in 2023-09-23T20:23:41.337 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:41 smithi018 ceph-mon[179647]: failed to encode map e137 with expected crc 2023-09-23T20:23:41.337 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:41 smithi018 ceph-mon[179647]: failed to encode map e137 with expected crc 2023-09-23T20:23:41.338 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:41 smithi018 ceph-mon[179647]: osdmap e138: 8 total, 8 up, 8 in 2023-09-23T20:23:41.338 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:41 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1293740956' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:23:41.338 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:41 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:23:41.358 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:23:41.388 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:41.422 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:23:41.424 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:41.425 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:23:41.473 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 43 16 58641 58625 85.1695 13 0.00681769 0.0114598 2023-09-23T20:23:41.525 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:23:41.554 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:41.587 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:23:41.589 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:41.590 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:23:41.690 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:23:41.717 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:41.752 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:23:41.755 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:41.755 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:23:41.856 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:23:41.871 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:23:41.878 INFO:teuthology.orchestra.run.smithi018.stderr:noscrub is unset 2023-09-23T20:23:41.888 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:41.998 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:23:42.003 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:42.004 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:23:42.105 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:23:42.166 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:42.223 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:23:42.226 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:42.227 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:23:42.327 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:23:42.402 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:42.413 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:23:42.448 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:23:42.460 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:42 smithi149 ceph-mon[170394]: failed to encode map e138 with expected crc 2023-09-23T20:23:42.460 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:42 smithi149 ceph-mon[170394]: failed to encode map e138 with expected crc 2023-09-23T20:23:42.460 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:42 smithi149 ceph-mon[170394]: failed to encode map e138 with expected crc 2023-09-23T20:23:42.460 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:42 smithi149 ceph-mon[170394]: pgmap v282: 99 pgs: 1 peering, 1 active+recovering+undersized+remapped, 97 active+clean; 3.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 36 MiB/s wr, 578 op/s; 560/173688 objects misplaced (0.322%); 12 MiB/s, 199 objects/s recovering 2023-09-23T20:23:42.460 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:42 smithi149 ceph-mon[170394]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:23:42.461 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:42 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:23:42.461 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:42 smithi149 ceph-mon[170394]: osdmap e139: 8 total, 8 up, 8 in 2023-09-23T20:23:42.468 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:23:42.475 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 44 16 59296 59280 84.1597 40.9375 0.0211572 0.0118774 2023-09-23T20:23:42.475 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:42 smithi018 ceph-mon[179647]: failed to encode map e138 with expected crc 2023-09-23T20:23:42.475 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:42 smithi018 ceph-mon[179647]: failed to encode map e138 with expected crc 2023-09-23T20:23:42.475 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:42 smithi018 ceph-mon[179647]: failed to encode map e138 with expected crc 2023-09-23T20:23:42.476 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:42 smithi018 ceph-mon[179647]: pgmap v282: 99 pgs: 1 peering, 1 active+recovering+undersized+remapped, 97 active+clean; 3.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 36 MiB/s wr, 578 op/s; 560/173688 objects misplaced (0.322%); 12 MiB/s, 199 objects/s recovering 2023-09-23T20:23:42.476 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:42 smithi018 ceph-mon[179647]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:23:42.476 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:42 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:23:42.476 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:42 smithi018 ceph-mon[179647]: osdmap e139: 8 total, 8 up, 8 in 2023-09-23T20:23:42.476 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:42 smithi018 ceph-mon[184144]: failed to encode map e138 with expected crc 2023-09-23T20:23:42.476 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:42 smithi018 ceph-mon[184144]: failed to encode map e138 with expected crc 2023-09-23T20:23:42.476 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:42 smithi018 ceph-mon[184144]: failed to encode map e138 with expected crc 2023-09-23T20:23:42.477 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:42 smithi018 ceph-mon[184144]: pgmap v282: 99 pgs: 1 peering, 1 active+recovering+undersized+remapped, 97 active+clean; 3.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 36 MiB/s wr, 578 op/s; 560/173688 objects misplaced (0.322%); 12 MiB/s, 199 objects/s recovering 2023-09-23T20:23:42.477 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:42 smithi018 ceph-mon[184144]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:23:42.477 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:42 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:23:42.477 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:42 smithi018 ceph-mon[184144]: osdmap e139: 8 total, 8 up, 8 in 2023-09-23T20:23:42.477 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:42.478 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:23:42.492 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:23:42.527 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:23:42.579 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:23:42.583 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:23:42.617 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:42.633 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:23:42.672 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:23:42.683 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:23:42.688 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:42.689 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:23:42.711 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:23:42.721 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:23:42.721 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:23:42.721 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:23:42.722 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:23:42.722 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:23:42.722 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:23:42.722 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:23:42.722 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:23:42.789 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:23:42.877 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:43.144 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:23:43.154 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:43.155 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:23:43.255 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:23:43.321 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:43.358 INFO:teuthology.orchestra.run.smithi149.stdout:{ 2023-09-23T20:23:43.358 INFO:teuthology.orchestra.run.smithi149.stdout: "size": 20, 2023-09-23T20:23:43.358 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 600, 2023-09-23T20:23:43.358 INFO:teuthology.orchestra.run.smithi149.stdout: "ops": [ 2023-09-23T20:23:43.358 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.359 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:26558 5.e 5:7c2f1774:::benchmark_data_smithi018_228451_object26557:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2023-09-23T20:23:43.359 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:08.819503+0000", 2023-09-23T20:23:43.359 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 34.535344807000001, 2023-09-23T20:23:43.359 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.023925767000000001, 2023-09-23T20:23:43.359 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:23:43.359 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:23:43.359 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:23:43.359 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:23:43.359 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:23:43.359 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 26558 2023-09-23T20:23:43.359 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.360 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:23:43.360 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.360 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:23:43.360 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:08.819503+0000", 2023-09-23T20:23:43.360 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:23:43.360 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.360 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.360 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:23:43.360 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:08.819503+0000", 2023-09-23T20:23:43.360 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:23:43.360 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.360 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.361 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:23:43.361 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:08.819503+0000", 2023-09-23T20:23:43.361 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:23:43.361 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.361 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.361 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:23:43.361 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:08.819532+0000", 2023-09-23T20:23:43.361 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.9291000000000001e-05 2023-09-23T20:23:43.361 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.361 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.361 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:23:43.361 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:08.819541+0000", 2023-09-23T20:23:43.362 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.7090000000000008e-06 2023-09-23T20:23:43.362 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.362 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.362 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:23:43.362 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:08.819546+0000", 2023-09-23T20:23:43.362 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.0780000000000003e-06 2023-09-23T20:23:43.362 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.362 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.362 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:23:43.362 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:08.819590+0000", 2023-09-23T20:23:43.362 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.3721999999999999e-05 2023-09-23T20:23:43.362 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.363 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.363 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:23:43.363 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:08.819704+0000", 2023-09-23T20:23:43.363 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00011435200000000001 2023-09-23T20:23:43.363 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.363 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.363 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:23:43.363 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:08.819856+0000", 2023-09-23T20:23:43.363 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00015259 2023-09-23T20:23:43.363 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.363 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.364 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:23:43.364 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:08.822808+0000", 2023-09-23T20:23:43.364 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0029512779999999999 2023-09-23T20:23:43.364 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.364 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.364 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:23:43.364 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:08.824448+0000", 2023-09-23T20:23:43.364 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0016405250000000001 2023-09-23T20:23:43.364 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.364 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.364 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:23:43.364 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:08.843334+0000", 2023-09-23T20:23:43.364 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.018885366000000001 2023-09-23T20:23:43.365 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.365 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.365 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:23:43.365 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:08.843387+0000", 2023-09-23T20:23:43.365 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.3179e-05 2023-09-23T20:23:43.365 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.365 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.365 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:23:43.365 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:08.843429+0000", 2023-09-23T20:23:43.365 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.2202e-05 2023-09-23T20:23:43.365 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:23:43.366 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:23:43.366 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:23:43.366 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.366 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.366 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:28903 5.e 5:72d8cd02:::benchmark_data_smithi018_228451_object28902:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e118)", 2023-09-23T20:23:43.366 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:09.965660+0000", 2023-09-23T20:23:43.366 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 33.389188533999999, 2023-09-23T20:23:43.366 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.054136203000000001, 2023-09-23T20:23:43.366 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:23:43.366 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:23:43.366 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:23:43.366 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:23:43.367 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:23:43.367 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 28903 2023-09-23T20:23:43.367 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.367 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:23:43.367 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.367 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:23:43.367 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:09.965660+0000", 2023-09-23T20:23:43.367 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:23:43.367 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.367 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.367 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:23:43.367 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:09.965660+0000", 2023-09-23T20:23:43.368 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:23:43.368 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.368 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.368 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:23:43.368 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:09.965658+0000", 2023-09-23T20:23:43.368 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:23:43.368 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.368 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.368 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:23:43.368 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:09.965715+0000", 2023-09-23T20:23:43.368 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.6619000000000001e-05 2023-09-23T20:23:43.368 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.369 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.369 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:23:43.369 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:09.965726+0000", 2023-09-23T20:23:43.369 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1061e-05 2023-09-23T20:23:43.369 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.369 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.369 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:23:43.369 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:09.965731+0000", 2023-09-23T20:23:43.369 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.6360000000000002e-06 2023-09-23T20:23:43.369 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.369 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.370 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:23:43.370 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:09.965766+0000", 2023-09-23T20:23:43.370 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.5153e-05 2023-09-23T20:23:43.370 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.370 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.370 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op must wait for map", 2023-09-23T20:23:43.370 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:09.965770+0000", 2023-09-23T20:23:43.370 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.031e-06 2023-09-23T20:23:43.370 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.370 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.370 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:23:43.370 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:09.980648+0000", 2023-09-23T20:23:43.371 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.014878245999999999 2023-09-23T20:23:43.371 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.371 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.371 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:23:43.371 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:09.980853+0000", 2023-09-23T20:23:43.371 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00020477700000000001 2023-09-23T20:23:43.371 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.371 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.371 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:23:43.371 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:09.981046+0000", 2023-09-23T20:23:43.371 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000193117 2023-09-23T20:23:43.371 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.372 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.372 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:23:43.372 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:09.985936+0000", 2023-09-23T20:23:43.372 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.004889783 2023-09-23T20:23:43.372 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.372 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.372 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:23:43.372 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:09.986389+0000", 2023-09-23T20:23:43.372 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00045317900000000001 2023-09-23T20:23:43.372 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.372 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.372 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:23:43.373 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:10.019678+0000", 2023-09-23T20:23:43.373 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.033289232000000002 2023-09-23T20:23:43.373 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.373 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.373 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:23:43.373 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:10.019750+0000", 2023-09-23T20:23:43.373 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.1699999999999995e-05 2023-09-23T20:23:43.373 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.373 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.373 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:23:43.373 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:10.019796+0000", 2023-09-23T20:23:43.373 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.5837999999999997e-05 2023-09-23T20:23:43.374 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:23:43.374 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:23:43.374 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:23:43.374 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.374 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.374 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:28909 5.1 5:825aa99e:::benchmark_data_smithi018_228451_object28908:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e118)", 2023-09-23T20:23:43.374 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:09.981257+0000", 2023-09-23T20:23:43.374 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 33.373591179999998, 2023-09-23T20:23:43.374 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0826966440000001, 2023-09-23T20:23:43.374 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:23:43.374 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:23:43.374 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:23:43.375 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:23:43.375 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:23:43.375 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 28909 2023-09-23T20:23:43.375 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.375 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:23:43.375 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.375 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:23:43.375 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:09.981257+0000", 2023-09-23T20:23:43.375 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:23:43.375 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.375 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.375 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:23:43.376 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:09.981257+0000", 2023-09-23T20:23:43.376 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:23:43.376 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.376 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.376 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:23:43.376 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:09.981255+0000", 2023-09-23T20:23:43.376 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999981 2023-09-23T20:23:43.376 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.376 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.376 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:23:43.376 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:09.981345+0000", 2023-09-23T20:23:43.377 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.9339999999999995e-05 2023-09-23T20:23:43.377 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.377 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.377 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:23:43.377 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:09.981360+0000", 2023-09-23T20:23:43.377 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.5591999999999999e-05 2023-09-23T20:23:43.377 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.377 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.377 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:23:43.377 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:09.981370+0000", 2023-09-23T20:23:43.377 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.6069999999999997e-06 2023-09-23T20:23:43.377 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.377 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.378 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:23:43.378 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:10.015283+0000", 2023-09-23T20:23:43.378 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.033912963999999997 2023-09-23T20:23:43.378 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.378 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.378 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:23:43.378 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:10.015297+0000", 2023-09-23T20:23:43.378 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.4477e-05 2023-09-23T20:23:43.378 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.378 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.378 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:23:43.379 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.054705+0000", 2023-09-23T20:23:43.379 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.03940805 2023-09-23T20:23:43.379 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.379 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.379 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:23:43.379 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.054798+0000", 2023-09-23T20:23:43.379 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.2872000000000001e-05 2023-09-23T20:23:43.379 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.379 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.379 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 3,7", 2023-09-23T20:23:43.379 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.054907+0000", 2023-09-23T20:23:43.379 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000108952 2023-09-23T20:23:43.380 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.380 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.380 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:23:43.380 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.056066+0000", 2023-09-23T20:23:43.380 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0011585460000000001 2023-09-23T20:23:43.380 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.380 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.380 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:23:43.380 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.056406+0000", 2023-09-23T20:23:43.380 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000340064 2023-09-23T20:23:43.380 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.380 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.381 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:23:43.381 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.063876+0000", 2023-09-23T20:23:43.381 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.007470661 2023-09-23T20:23:43.381 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.381 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.381 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:23:43.381 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.063917+0000", 2023-09-23T20:23:43.381 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.1151e-05 2023-09-23T20:23:43.381 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.381 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.381 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:23:43.381 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.063954+0000", 2023-09-23T20:23:43.382 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.6075999999999998e-05 2023-09-23T20:23:43.382 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:23:43.382 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:23:43.382 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:23:43.382 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.382 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.382 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_repop(client.34791.0:28912 5.f e118/111 5:ffc31d86:::benchmark_data_smithi018_228451_object28911:head v 118'1814, mlcod=117'1812)", 2023-09-23T20:23:43.382 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:10.043765+0000", 2023-09-23T20:23:43.382 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 33.311082796999997, 2023-09-23T20:23:43.382 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.020684111000000002, 2023-09-23T20:23:43.382 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:23:43.382 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:23:43.383 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:23:43.383 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.383 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:23:43.383 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:10.043765+0000", 2023-09-23T20:23:43.383 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:23:43.383 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.383 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.383 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:23:43.383 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:10.043765+0000", 2023-09-23T20:23:43.383 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:23:43.383 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.383 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.384 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:23:43.384 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:10.043764+0000", 2023-09-23T20:23:43.384 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:23:43.384 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.384 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.384 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:23:43.384 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:10.043816+0000", 2023-09-23T20:23:43.384 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.2253999999999998e-05 2023-09-23T20:23:43.384 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.384 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.384 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:23:43.384 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:10.046290+0000", 2023-09-23T20:23:43.385 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0024743019999999998 2023-09-23T20:23:43.385 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.385 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.385 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:23:43.385 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:10.046309+0000", 2023-09-23T20:23:43.385 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.8703e-05 2023-09-23T20:23:43.385 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.385 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.385 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:23:43.385 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:10.050193+0000", 2023-09-23T20:23:43.385 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0038842669999999998 2023-09-23T20:23:43.385 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.385 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.386 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:23:43.386 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:10.050229+0000", 2023-09-23T20:23:43.386 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.6226999999999997e-05 2023-09-23T20:23:43.386 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.386 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.386 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:23:43.386 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:10.064369+0000", 2023-09-23T20:23:43.386 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.014139347 2023-09-23T20:23:43.386 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.386 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.386 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:23:43.387 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:10.064449+0000", 2023-09-23T20:23:43.387 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.0606999999999996e-05 2023-09-23T20:23:43.387 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:23:43.387 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:23:43.387 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:23:43.387 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.387 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.387 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:34208 5.e 5:79181d83:::benchmark_data_smithi018_228451_object34207:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e119)", 2023-09-23T20:23:43.387 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:13.441874+0000", 2023-09-23T20:23:43.387 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 29.912974301999999, 2023-09-23T20:23:43.387 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.020561856, 2023-09-23T20:23:43.387 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:23:43.388 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:23:43.388 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:23:43.388 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:23:43.388 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:23:43.388 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 34208 2023-09-23T20:23:43.388 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.388 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:23:43.388 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.388 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:23:43.388 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:13.441874+0000", 2023-09-23T20:23:43.388 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:23:43.388 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.388 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.389 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:23:43.389 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:13.441874+0000", 2023-09-23T20:23:43.389 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:23:43.389 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.389 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.389 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:23:43.389 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:13.441871+0000", 2023-09-23T20:23:43.389 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999976 2023-09-23T20:23:43.389 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.389 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.389 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:23:43.389 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:13.441922+0000", 2023-09-23T20:23:43.390 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.0376e-05 2023-09-23T20:23:43.390 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.390 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.390 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:23:43.390 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:13.441940+0000", 2023-09-23T20:23:43.390 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.8624999999999999e-05 2023-09-23T20:23:43.390 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.390 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.390 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:23:43.390 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:13.441948+0000", 2023-09-23T20:23:43.390 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.8469999999999996e-06 2023-09-23T20:23:43.391 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.391 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.391 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:23:43.391 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:13.442010+0000", 2023-09-23T20:23:43.391 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.2042e-05 2023-09-23T20:23:43.391 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.391 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.391 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:23:43.391 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:13.442164+0000", 2023-09-23T20:23:43.391 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000154151 2023-09-23T20:23:43.391 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.391 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.391 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:23:43.392 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:13.442375+0000", 2023-09-23T20:23:43.392 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00021088699999999999 2023-09-23T20:23:43.392 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.392 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.392 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:23:43.392 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:13.445241+0000", 2023-09-23T20:23:43.392 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0028662420000000002 2023-09-23T20:23:43.392 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.392 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.392 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:23:43.392 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:13.447861+0000", 2023-09-23T20:23:43.392 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.002619601 2023-09-23T20:23:43.393 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.393 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.393 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:23:43.393 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:13.462343+0000", 2023-09-23T20:23:43.393 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.01448227 2023-09-23T20:23:43.393 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.393 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.393 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:23:43.393 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:13.462402+0000", 2023-09-23T20:23:43.393 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.8427000000000002e-05 2023-09-23T20:23:43.393 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.393 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.394 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:23:43.394 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:13.462436+0000", 2023-09-23T20:23:43.394 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.3872999999999999e-05 2023-09-23T20:23:43.394 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:23:43.394 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:23:43.394 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:23:43.394 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.394 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.394 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:34211 5.e 5:79d93b74:::benchmark_data_smithi018_228451_object34210:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e119)", 2023-09-23T20:23:43.394 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:13.442379+0000", 2023-09-23T20:23:43.394 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 29.912469163000001, 2023-09-23T20:23:43.394 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.020123412, 2023-09-23T20:23:43.395 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:23:43.395 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:23:43.395 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:23:43.395 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:23:43.395 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:23:43.395 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 34211 2023-09-23T20:23:43.395 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.395 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:23:43.395 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.395 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:23:43.395 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:13.442379+0000", 2023-09-23T20:23:43.395 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:23:43.396 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.396 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.396 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:23:43.396 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:13.442379+0000", 2023-09-23T20:23:43.396 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:23:43.396 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.396 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.396 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:23:43.396 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:13.442378+0000", 2023-09-23T20:23:43.396 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:23:43.396 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.396 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.397 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:23:43.397 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:13.442471+0000", 2023-09-23T20:23:43.397 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.2441000000000005e-05 2023-09-23T20:23:43.397 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.397 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.397 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:23:43.397 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:13.442483+0000", 2023-09-23T20:23:43.397 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.2182e-05 2023-09-23T20:23:43.397 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.397 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.397 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:23:43.397 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:13.442489+0000", 2023-09-23T20:23:43.398 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.9009999999999999e-06 2023-09-23T20:23:43.398 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.398 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.398 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:23:43.398 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:13.443869+0000", 2023-09-23T20:23:43.398 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.001380629 2023-09-23T20:23:43.398 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.398 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.398 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:23:43.398 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:13.443977+0000", 2023-09-23T20:23:43.398 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000107897 2023-09-23T20:23:43.398 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.399 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.399 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:23:43.399 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:13.444106+0000", 2023-09-23T20:23:43.399 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00012888799999999999 2023-09-23T20:23:43.399 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.399 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.399 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:23:43.399 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:13.445461+0000", 2023-09-23T20:23:43.399 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0013550299999999999 2023-09-23T20:23:43.399 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.399 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.400 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:23:43.400 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:13.447896+0000", 2023-09-23T20:23:43.400 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0024353310000000002 2023-09-23T20:23:43.400 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.400 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.400 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:23:43.400 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:13.462459+0000", 2023-09-23T20:23:43.400 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.014562340999999999 2023-09-23T20:23:43.400 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.400 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.400 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:23:43.400 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:13.462483+0000", 2023-09-23T20:23:43.400 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.4309000000000001e-05 2023-09-23T20:23:43.401 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.401 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.401 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:23:43.401 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:13.462502+0000", 2023-09-23T20:23:43.401 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.9253999999999999e-05 2023-09-23T20:23:43.401 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:23:43.401 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:23:43.401 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:23:43.401 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.401 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.401 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:37273 5.1 5:8695d2dd:::benchmark_data_smithi018_228451_object37272:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e119)", 2023-09-23T20:23:43.401 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:14.987897+0000", 2023-09-23T20:23:43.402 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 28.366951496999999, 2023-09-23T20:23:43.402 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.020245102000000001, 2023-09-23T20:23:43.402 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:23:43.402 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:23:43.402 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:23:43.402 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:23:43.402 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:23:43.402 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 37273 2023-09-23T20:23:43.402 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.402 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:23:43.402 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.402 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:23:43.403 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:14.987897+0000", 2023-09-23T20:23:43.403 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:23:43.403 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.403 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.403 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:23:43.403 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:14.987897+0000", 2023-09-23T20:23:43.403 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:23:43.403 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.403 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.403 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:23:43.403 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:14.987895+0000", 2023-09-23T20:23:43.403 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:23:43.404 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.404 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.404 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:23:43.404 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:14.987927+0000", 2023-09-23T20:23:43.404 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.1908e-05 2023-09-23T20:23:43.404 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.404 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.404 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:23:43.404 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:14.987942+0000", 2023-09-23T20:23:43.404 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.4337e-05 2023-09-23T20:23:43.404 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.404 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.405 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:23:43.405 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:14.987948+0000", 2023-09-23T20:23:43.405 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.1809999999999998e-06 2023-09-23T20:23:43.405 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.405 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.405 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:23:43.405 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:14.987981+0000", 2023-09-23T20:23:43.405 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.3470000000000003e-05 2023-09-23T20:23:43.405 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.405 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.405 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:23:43.406 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:14.988085+0000", 2023-09-23T20:23:43.406 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000103785 2023-09-23T20:23:43.406 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.406 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.406 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 3,7", 2023-09-23T20:23:43.406 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:14.988201+0000", 2023-09-23T20:23:43.406 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000116031 2023-09-23T20:23:43.406 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.406 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.406 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:23:43.406 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:14.989364+0000", 2023-09-23T20:23:43.406 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.001162747 2023-09-23T20:23:43.407 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.407 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.407 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:23:43.407 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:14.990340+0000", 2023-09-23T20:23:43.407 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00097665199999999995 2023-09-23T20:23:43.407 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.407 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.407 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:23:43.407 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:15.008045+0000", 2023-09-23T20:23:43.407 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.017704395000000001 2023-09-23T20:23:43.407 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.407 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.408 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:23:43.408 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:15.008098+0000", 2023-09-23T20:23:43.408 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.3431999999999999e-05 2023-09-23T20:23:43.408 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.408 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.408 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:23:43.408 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:15.008142+0000", 2023-09-23T20:23:43.408 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.3460000000000001e-05 2023-09-23T20:23:43.408 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:23:43.408 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:23:43.408 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:23:43.408 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.409 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.409 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:37405 5.11 5:8d12bb3a:::benchmark_data_smithi018_228451_object37404:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e119)", 2023-09-23T20:23:43.409 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:15.084637+0000", 2023-09-23T20:23:43.409 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 28.270210728999999, 2023-09-23T20:23:43.409 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.029495468, 2023-09-23T20:23:43.409 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:23:43.409 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:23:43.409 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:23:43.409 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:23:43.409 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:23:43.409 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 37405 2023-09-23T20:23:43.409 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.410 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:23:43.410 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.410 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:23:43.410 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:15.084637+0000", 2023-09-23T20:23:43.410 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:23:43.410 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.410 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.410 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:23:43.410 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:15.084637+0000", 2023-09-23T20:23:43.410 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:23:43.411 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.411 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.411 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:23:43.411 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:15.084637+0000", 2023-09-23T20:23:43.411 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:23:43.411 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.411 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.411 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:23:43.411 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:15.084683+0000", 2023-09-23T20:23:43.411 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.6119999999999999e-05 2023-09-23T20:23:43.411 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.411 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.412 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:23:43.412 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:15.084689+0000", 2023-09-23T20:23:43.412 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.0000000000000002e-06 2023-09-23T20:23:43.412 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.412 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.412 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:23:43.412 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:15.084694+0000", 2023-09-23T20:23:43.412 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.0549999999999999e-06 2023-09-23T20:23:43.412 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.412 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.412 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:23:43.412 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:15.084753+0000", 2023-09-23T20:23:43.412 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.8922000000000003e-05 2023-09-23T20:23:43.413 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.413 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.413 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:23:43.413 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:15.084836+0000", 2023-09-23T20:23:43.413 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.3125000000000001e-05 2023-09-23T20:23:43.413 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.413 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.413 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 3,7", 2023-09-23T20:23:43.413 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:15.084941+0000", 2023-09-23T20:23:43.413 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00010488300000000001 2023-09-23T20:23:43.413 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.413 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.414 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:23:43.414 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:15.089357+0000", 2023-09-23T20:23:43.414 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0044161130000000002 2023-09-23T20:23:43.414 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.414 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.414 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:23:43.414 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:15.089375+0000", 2023-09-23T20:23:43.414 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.8343e-05 2023-09-23T20:23:43.414 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.414 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.414 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:23:43.414 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:15.114073+0000", 2023-09-23T20:23:43.415 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.024697244 2023-09-23T20:23:43.415 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.415 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.415 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:23:43.415 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:15.114107+0000", 2023-09-23T20:23:43.415 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.4184999999999999e-05 2023-09-23T20:23:43.415 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.415 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.415 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:23:43.415 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:15.114133+0000", 2023-09-23T20:23:43.415 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.5999e-05 2023-09-23T20:23:43.415 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:23:43.416 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:23:43.416 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:23:43.416 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.416 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.416 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:38742 5.e 5:7364816c:::benchmark_data_smithi018_228451_object38741:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e119)", 2023-09-23T20:23:43.416 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:15.862423+0000", 2023-09-23T20:23:43.416 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 27.492424956000001, 2023-09-23T20:23:43.416 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.021062811000000001, 2023-09-23T20:23:43.416 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:23:43.416 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:23:43.416 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:23:43.416 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:23:43.417 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:23:43.417 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 38742 2023-09-23T20:23:43.417 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.417 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:23:43.417 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.417 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:23:43.417 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:15.862423+0000", 2023-09-23T20:23:43.417 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:23:43.417 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.417 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.417 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:23:43.418 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:15.862423+0000", 2023-09-23T20:23:43.418 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:23:43.418 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.418 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.418 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:23:43.418 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:15.862422+0000", 2023-09-23T20:23:43.418 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:23:43.418 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.418 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.418 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:23:43.418 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:15.862482+0000", 2023-09-23T20:23:43.419 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.9636000000000003e-05 2023-09-23T20:23:43.419 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.419 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.419 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:23:43.419 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:15.862493+0000", 2023-09-23T20:23:43.419 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0871e-05 2023-09-23T20:23:43.419 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.419 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.419 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:23:43.419 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:15.862499+0000", 2023-09-23T20:23:43.419 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.511e-06 2023-09-23T20:23:43.419 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.419 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.420 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:23:43.420 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:15.863811+0000", 2023-09-23T20:23:43.420 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.001311755 2023-09-23T20:23:43.420 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.420 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.420 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:23:43.420 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:15.863898+0000", 2023-09-23T20:23:43.420 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.6835999999999993e-05 2023-09-23T20:23:43.420 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.420 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.420 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:23:43.421 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:15.864005+0000", 2023-09-23T20:23:43.421 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000107346 2023-09-23T20:23:43.421 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.421 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.421 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:23:43.421 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:15.865421+0000", 2023-09-23T20:23:43.421 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0014155459999999999 2023-09-23T20:23:43.421 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.421 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.421 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:23:43.421 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:15.866193+0000", 2023-09-23T20:23:43.421 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000772227 2023-09-23T20:23:43.422 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.422 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.422 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:23:43.422 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:15.883403+0000", 2023-09-23T20:23:43.422 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.017210196000000001 2023-09-23T20:23:43.422 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.422 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.422 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:23:43.422 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:15.883452+0000", 2023-09-23T20:23:43.422 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.8633e-05 2023-09-23T20:23:43.422 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.422 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.423 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:23:43.423 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:15.883486+0000", 2023-09-23T20:23:43.423 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.4093e-05 2023-09-23T20:23:43.423 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:23:43.423 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:23:43.423 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:23:43.423 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.423 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.423 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:45335 5.e 5:79f1e7ee:::benchmark_data_smithi018_228451_object45334:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e129)", 2023-09-23T20:23:43.423 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:27.262621+0000", 2023-09-23T20:23:43.423 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 16.092227312999999, 2023-09-23T20:23:43.423 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.025852976, 2023-09-23T20:23:43.424 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:23:43.424 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:23:43.424 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:23:43.424 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:23:43.424 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:23:43.424 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 45335 2023-09-23T20:23:43.424 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.424 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:23:43.424 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.424 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:23:43.424 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:27.262621+0000", 2023-09-23T20:23:43.425 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:23:43.425 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.425 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.425 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:23:43.425 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:27.262621+0000", 2023-09-23T20:23:43.425 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:23:43.425 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.425 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.425 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:23:43.425 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:27.262619+0000", 2023-09-23T20:23:43.425 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999981 2023-09-23T20:23:43.425 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.426 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.426 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:23:43.426 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:27.262733+0000", 2023-09-23T20:23:43.426 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000114039 2023-09-23T20:23:43.426 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.426 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.426 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:23:43.426 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:27.262749+0000", 2023-09-23T20:23:43.426 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.6237999999999999e-05 2023-09-23T20:23:43.426 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.426 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.426 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:23:43.427 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:27.262758+0000", 2023-09-23T20:23:43.427 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.1379999999999996e-06 2023-09-23T20:23:43.427 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.427 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.427 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:23:43.427 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:27.262893+0000", 2023-09-23T20:23:43.427 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00013443799999999999 2023-09-23T20:23:43.427 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.427 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.427 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:23:43.427 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:27.263001+0000", 2023-09-23T20:23:43.427 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000108799 2023-09-23T20:23:43.428 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.428 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.428 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:23:43.428 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:27.263190+0000", 2023-09-23T20:23:43.428 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000188708 2023-09-23T20:23:43.428 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.428 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.428 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:23:43.428 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:27.264767+0000", 2023-09-23T20:23:43.428 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0015766980000000001 2023-09-23T20:23:43.428 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.428 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.429 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:23:43.429 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:27.265278+0000", 2023-09-23T20:23:43.429 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00051085200000000001 2023-09-23T20:23:43.429 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.429 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.429 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:23:43.429 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:27.288413+0000", 2023-09-23T20:23:43.429 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.023135448999999999 2023-09-23T20:23:43.429 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.429 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.429 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:23:43.429 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:27.288449+0000", 2023-09-23T20:23:43.429 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.5855000000000001e-05 2023-09-23T20:23:43.430 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.430 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.430 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:23:43.430 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:27.288474+0000", 2023-09-23T20:23:43.430 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.4672e-05 2023-09-23T20:23:43.430 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:23:43.430 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:23:43.430 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:23:43.430 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.430 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.430 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:46259 5.e 5:72552d35:::benchmark_data_smithi018_228451_object46258:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e129)", 2023-09-23T20:23:43.431 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:27.866046+0000", 2023-09-23T20:23:43.431 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 15.488802464999999, 2023-09-23T20:23:43.431 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.019953200000000001, 2023-09-23T20:23:43.431 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:23:43.431 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:23:43.431 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:23:43.431 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:23:43.431 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:23:43.431 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 46259 2023-09-23T20:23:43.431 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.431 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:23:43.431 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.431 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:23:43.432 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:27.866046+0000", 2023-09-23T20:23:43.432 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:23:43.432 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.432 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.432 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:23:43.432 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:27.866046+0000", 2023-09-23T20:23:43.432 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:23:43.432 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.432 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.432 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:23:43.432 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:27.866045+0000", 2023-09-23T20:23:43.432 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:23:43.433 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.433 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.433 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:23:43.433 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:27.866072+0000", 2023-09-23T20:23:43.433 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.7373e-05 2023-09-23T20:23:43.433 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.433 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.433 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:23:43.433 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:27.866082+0000", 2023-09-23T20:23:43.433 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.9660000000000008e-06 2023-09-23T20:23:43.433 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.433 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.434 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:23:43.434 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:27.866088+0000", 2023-09-23T20:23:43.434 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.9009999999999999e-06 2023-09-23T20:23:43.434 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.434 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.434 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:23:43.434 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:27.867823+0000", 2023-09-23T20:23:43.434 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0017352470000000001 2023-09-23T20:23:43.434 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.434 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.434 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:23:43.434 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:27.867945+0000", 2023-09-23T20:23:43.435 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00012195 2023-09-23T20:23:43.435 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.435 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.435 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:23:43.435 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:27.868089+0000", 2023-09-23T20:23:43.435 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00014378899999999999 2023-09-23T20:23:43.435 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.435 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.435 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:23:43.435 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:27.870018+0000", 2023-09-23T20:23:43.435 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.001928475 2023-09-23T20:23:43.435 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.436 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.436 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:23:43.436 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:27.874814+0000", 2023-09-23T20:23:43.436 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0047964779999999999 2023-09-23T20:23:43.436 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.436 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.436 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:23:43.436 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:27.885910+0000", 2023-09-23T20:23:43.436 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.011095384999999999 2023-09-23T20:23:43.436 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.436 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.436 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:23:43.437 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:27.885960+0000", 2023-09-23T20:23:43.437 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.0605000000000001e-05 2023-09-23T20:23:43.437 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.437 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.437 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:23:43.437 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:27.885999+0000", 2023-09-23T20:23:43.437 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.8655999999999997e-05 2023-09-23T20:23:43.437 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:23:43.437 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:23:43.437 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:23:43.437 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.437 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.438 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:47438 5.e 5:76a66ff7:::benchmark_data_smithi018_228451_object47437:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e129)", 2023-09-23T20:23:43.438 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:28.674569+0000", 2023-09-23T20:23:43.438 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 14.680278929, 2023-09-23T20:23:43.438 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.029309139000000001, 2023-09-23T20:23:43.438 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:23:43.438 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:23:43.438 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:23:43.438 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:23:43.438 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:23:43.438 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 47438 2023-09-23T20:23:43.438 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.438 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:23:43.439 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.439 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:23:43.439 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:28.674569+0000", 2023-09-23T20:23:43.439 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:23:43.439 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.439 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.439 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:23:43.439 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:28.674569+0000", 2023-09-23T20:23:43.439 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:23:43.439 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.439 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.439 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:23:43.440 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:28.674568+0000", 2023-09-23T20:23:43.440 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:23:43.440 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.440 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.440 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:23:43.440 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:28.674657+0000", 2023-09-23T20:23:43.440 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.9034000000000002e-05 2023-09-23T20:23:43.440 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.440 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.440 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:23:43.440 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:28.674672+0000", 2023-09-23T20:23:43.440 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.5264000000000002e-05 2023-09-23T20:23:43.441 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.441 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.441 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:23:43.441 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:28.674679+0000", 2023-09-23T20:23:43.441 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.8619999999999997e-06 2023-09-23T20:23:43.441 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.441 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.441 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:23:43.441 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:28.674724+0000", 2023-09-23T20:23:43.441 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.5204000000000003e-05 2023-09-23T20:23:43.441 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.441 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.442 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:23:43.442 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:28.674842+0000", 2023-09-23T20:23:43.442 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000117413 2023-09-23T20:23:43.442 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.442 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.442 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:23:43.442 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:28.674969+0000", 2023-09-23T20:23:43.442 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00012704200000000001 2023-09-23T20:23:43.442 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.442 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.442 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:23:43.442 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:28.676647+0000", 2023-09-23T20:23:43.443 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.001677811 2023-09-23T20:23:43.443 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.443 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.443 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:23:43.443 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:28.678453+0000", 2023-09-23T20:23:43.443 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0018068870000000001 2023-09-23T20:23:43.443 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.443 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.443 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:23:43.443 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:28.703815+0000", 2023-09-23T20:23:43.443 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.025362053999999998 2023-09-23T20:23:43.443 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.444 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.444 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:23:43.444 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:28.703852+0000", 2023-09-23T20:23:43.444 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.6906000000000002e-05 2023-09-23T20:23:43.444 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.444 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.444 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:23:43.444 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:28.703878+0000", 2023-09-23T20:23:43.444 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.5891999999999999e-05 2023-09-23T20:23:43.444 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:23:43.444 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:23:43.444 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:23:43.445 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.445 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.445 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:47452 5.e 5:7b4c9ec7:::benchmark_data_smithi018_228451_object47451:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e129)", 2023-09-23T20:23:43.445 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:28.684836+0000", 2023-09-23T20:23:43.445 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 14.670012436, 2023-09-23T20:23:43.445 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.025947984, 2023-09-23T20:23:43.445 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:23:43.445 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:23:43.445 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:23:43.445 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:23:43.445 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:23:43.445 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 47452 2023-09-23T20:23:43.446 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.446 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:23:43.446 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.446 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:23:43.446 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:28.684836+0000", 2023-09-23T20:23:43.446 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:23:43.446 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.446 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.446 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:23:43.446 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:28.684836+0000", 2023-09-23T20:23:43.446 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:23:43.446 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.447 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.447 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:23:43.447 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:28.684835+0000", 2023-09-23T20:23:43.447 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:23:43.447 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.447 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.447 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:23:43.447 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:28.684896+0000", 2023-09-23T20:23:43.447 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.1441999999999999e-05 2023-09-23T20:23:43.447 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.447 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.447 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:23:43.448 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:28.684908+0000", 2023-09-23T20:23:43.448 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.147e-05 2023-09-23T20:23:43.448 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.448 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.448 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:23:43.448 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:28.684914+0000", 2023-09-23T20:23:43.448 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.657e-06 2023-09-23T20:23:43.448 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.448 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.448 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:23:43.448 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:28.684952+0000", 2023-09-23T20:23:43.448 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.8090000000000003e-05 2023-09-23T20:23:43.449 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.449 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.449 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:23:43.449 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:28.685051+0000", 2023-09-23T20:23:43.449 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.8308999999999997e-05 2023-09-23T20:23:43.449 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.449 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.449 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:23:43.449 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:28.685153+0000", 2023-09-23T20:23:43.449 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00010268 2023-09-23T20:23:43.449 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.449 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.450 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:23:43.450 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:28.686431+0000", 2023-09-23T20:23:43.450 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0012774730000000001 2023-09-23T20:23:43.450 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.450 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.450 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:23:43.450 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:28.686824+0000", 2023-09-23T20:23:43.450 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00039282199999999997 2023-09-23T20:23:43.450 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.450 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.450 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:23:43.450 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:28.710724+0000", 2023-09-23T20:23:43.451 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.023900108999999999 2023-09-23T20:23:43.451 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.451 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.451 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:23:43.451 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:28.710759+0000", 2023-09-23T20:23:43.451 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.5308000000000001e-05 2023-09-23T20:23:43.451 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.451 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.451 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:23:43.451 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:28.710784+0000", 2023-09-23T20:23:43.451 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.4550999999999999e-05 2023-09-23T20:23:43.451 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:23:43.452 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:23:43.452 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:23:43.452 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.452 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.452 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:47457 5.e 5:7a945d4a:::benchmark_data_smithi018_228451_object47456:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e129)", 2023-09-23T20:23:43.452 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:28.691149+0000", 2023-09-23T20:23:43.452 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 14.663698749, 2023-09-23T20:23:43.452 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.019724508000000002, 2023-09-23T20:23:43.452 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:23:43.452 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:23:43.452 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:23:43.452 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:23:43.453 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:23:43.453 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 47457 2023-09-23T20:23:43.453 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.453 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:23:43.453 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.453 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:23:43.453 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:28.691149+0000", 2023-09-23T20:23:43.453 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:23:43.453 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.453 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.453 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:23:43.453 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:28.691149+0000", 2023-09-23T20:23:43.454 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:23:43.454 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.454 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.454 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:23:43.454 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:28.691149+0000", 2023-09-23T20:23:43.454 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:23:43.454 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.454 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.454 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:23:43.454 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:28.691250+0000", 2023-09-23T20:23:43.454 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000101661 2023-09-23T20:23:43.455 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.455 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.455 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:23:43.455 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:28.691263+0000", 2023-09-23T20:23:43.455 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.2397e-05 2023-09-23T20:23:43.455 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.455 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.455 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:23:43.455 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:28.691277+0000", 2023-09-23T20:23:43.455 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.4192000000000001e-05 2023-09-23T20:23:43.455 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.455 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.456 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:23:43.456 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:28.691333+0000", 2023-09-23T20:23:43.456 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.5723999999999997e-05 2023-09-23T20:23:43.456 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.456 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.456 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:23:43.456 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:28.691464+0000", 2023-09-23T20:23:43.456 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00013133200000000001 2023-09-23T20:23:43.456 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.456 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.456 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:23:43.456 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:28.691596+0000", 2023-09-23T20:23:43.457 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00013243200000000001 2023-09-23T20:23:43.457 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.457 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.457 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:23:43.457 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:28.694025+0000", 2023-09-23T20:23:43.457 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0024286540000000001 2023-09-23T20:23:43.457 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.457 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.457 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:23:43.457 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:28.694399+0000", 2023-09-23T20:23:43.457 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000374483 2023-09-23T20:23:43.457 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.458 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.458 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:23:43.458 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:28.710821+0000", 2023-09-23T20:23:43.458 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.016421126000000001 2023-09-23T20:23:43.458 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.458 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.458 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:23:43.458 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:28.710852+0000", 2023-09-23T20:23:43.458 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.1615000000000001e-05 2023-09-23T20:23:43.458 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.458 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.459 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:23:43.459 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:28.710874+0000", 2023-09-23T20:23:43.459 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.1624000000000002e-05 2023-09-23T20:23:43.459 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:23:43.459 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:23:43.459 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:23:43.459 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.459 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.459 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:47825 5.1 5:8141e520:::benchmark_data_smithi018_228451_object47824:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e129)", 2023-09-23T20:23:43.459 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:28.931240+0000", 2023-09-23T20:23:43.459 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 14.423607887999999, 2023-09-23T20:23:43.459 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.019726536999999999, 2023-09-23T20:23:43.460 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:23:43.460 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:23:43.460 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:23:43.460 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:23:43.460 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:23:43.460 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 47825 2023-09-23T20:23:43.460 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.460 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:23:43.460 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.460 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:23:43.460 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:28.931240+0000", 2023-09-23T20:23:43.460 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:23:43.461 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.461 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.461 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:23:43.461 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:28.931240+0000", 2023-09-23T20:23:43.461 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:23:43.461 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.461 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.461 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:23:43.461 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:28.931238+0000", 2023-09-23T20:23:43.461 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999981 2023-09-23T20:23:43.461 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.461 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.462 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:23:43.462 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:28.931274+0000", 2023-09-23T20:23:43.462 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.5179999999999999e-05 2023-09-23T20:23:43.462 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.462 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.462 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:23:43.462 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:28.931284+0000", 2023-09-23T20:23:43.462 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0467e-05 2023-09-23T20:23:43.462 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.462 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.462 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:23:43.462 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:28.931290+0000", 2023-09-23T20:23:43.463 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.1619999999999996e-06 2023-09-23T20:23:43.463 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.463 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.463 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:23:43.463 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:28.931361+0000", 2023-09-23T20:23:43.463 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.1186999999999995e-05 2023-09-23T20:23:43.463 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.463 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.463 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:23:43.463 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:28.931481+0000", 2023-09-23T20:23:43.463 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000119756 2023-09-23T20:23:43.464 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.464 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.464 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 3,7", 2023-09-23T20:23:43.464 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:28.931620+0000", 2023-09-23T20:23:43.464 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00013924200000000001 2023-09-23T20:23:43.464 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.464 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.464 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:23:43.464 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:28.934022+0000", 2023-09-23T20:23:43.464 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.002402071 2023-09-23T20:23:43.464 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.464 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.465 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:23:43.465 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:28.937375+0000", 2023-09-23T20:23:43.465 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0033525389999999999 2023-09-23T20:23:43.465 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.465 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.465 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:23:43.465 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:28.950872+0000", 2023-09-23T20:23:43.465 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.013496920000000001 2023-09-23T20:23:43.465 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.465 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.465 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:23:43.465 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:28.950923+0000", 2023-09-23T20:23:43.466 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.0834999999999997e-05 2023-09-23T20:23:43.466 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.466 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.466 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:23:43.466 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:28.950967+0000", 2023-09-23T20:23:43.466 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.3915999999999998e-05 2023-09-23T20:23:43.466 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:23:43.466 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:23:43.466 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:23:43.466 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.466 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.466 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:47826 5.1 5:81ab35d2:::benchmark_data_smithi018_228451_object47825:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e129)", 2023-09-23T20:23:43.467 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:28.931298+0000", 2023-09-23T20:23:43.467 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 14.423550208, 2023-09-23T20:23:43.467 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.019758438, 2023-09-23T20:23:43.467 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:23:43.467 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:23:43.467 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:23:43.467 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:23:43.467 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:23:43.467 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 47826 2023-09-23T20:23:43.467 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.467 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:23:43.467 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.468 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:23:43.468 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:28.931298+0000", 2023-09-23T20:23:43.468 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:23:43.468 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.468 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.468 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:23:43.468 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:28.931298+0000", 2023-09-23T20:23:43.468 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:23:43.468 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.468 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.468 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:23:43.468 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:28.931297+0000", 2023-09-23T20:23:43.469 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:23:43.469 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.469 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.469 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:23:43.469 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:28.931334+0000", 2023-09-23T20:23:43.469 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.6498000000000002e-05 2023-09-23T20:23:43.469 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.469 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.469 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:23:43.469 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:28.931342+0000", 2023-09-23T20:23:43.469 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.7219999999999999e-06 2023-09-23T20:23:43.469 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.470 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.470 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:23:43.470 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:28.931347+0000", 2023-09-23T20:23:43.470 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.9849999999999997e-06 2023-09-23T20:23:43.470 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.470 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.470 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:23:43.470 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:28.932721+0000", 2023-09-23T20:23:43.470 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.001374433 2023-09-23T20:23:43.470 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.470 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.470 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:23:43.471 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:28.932848+0000", 2023-09-23T20:23:43.471 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00012739899999999999 2023-09-23T20:23:43.471 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.471 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.471 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 3,7", 2023-09-23T20:23:43.471 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:28.932981+0000", 2023-09-23T20:23:43.471 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00013259099999999999 2023-09-23T20:23:43.471 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.471 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.471 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:23:43.471 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:28.934028+0000", 2023-09-23T20:23:43.471 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0010467090000000001 2023-09-23T20:23:43.472 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.472 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.472 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:23:43.472 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:28.937487+0000", 2023-09-23T20:23:43.472 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0034590739999999999 2023-09-23T20:23:43.472 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.472 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.472 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:23:43.472 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:28.950997+0000", 2023-09-23T20:23:43.472 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.013509939 2023-09-23T20:23:43.472 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.472 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.473 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:23:43.473 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:28.951030+0000", 2023-09-23T20:23:43.473 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.2957999999999999e-05 2023-09-23T20:23:43.473 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.473 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.473 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:23:43.473 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:28.951056+0000", 2023-09-23T20:23:43.473 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.6648e-05 2023-09-23T20:23:43.473 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:23:43.473 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:23:43.473 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:23:43.473 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.474 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.474 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:48312 5.e 5:779af618:::benchmark_data_smithi018_228451_object48311:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e129)", 2023-09-23T20:23:43.474 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:29.307192+0000", 2023-09-23T20:23:43.474 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 14.047656484999999, 2023-09-23T20:23:43.474 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.020429767000000001, 2023-09-23T20:23:43.474 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:23:43.474 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:23:43.474 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:23:43.474 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:23:43.474 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:23:43.474 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 48312 2023-09-23T20:23:43.474 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.475 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:23:43.475 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.475 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:23:43.475 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:29.307192+0000", 2023-09-23T20:23:43.475 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:23:43.475 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.475 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.475 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:23:43.475 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:29.307192+0000", 2023-09-23T20:23:43.475 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:23:43.475 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.475 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.476 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:23:43.476 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:29.307190+0000", 2023-09-23T20:23:43.476 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:23:43.476 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.476 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.476 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:23:43.476 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:29.307300+0000", 2023-09-23T20:23:43.476 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000110345 2023-09-23T20:23:43.476 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.476 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.476 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:23:43.476 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:29.307322+0000", 2023-09-23T20:23:43.477 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.2010000000000001e-05 2023-09-23T20:23:43.477 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.477 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.477 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:23:43.477 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:29.307330+0000", 2023-09-23T20:23:43.477 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.8280000000000003e-06 2023-09-23T20:23:43.477 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.477 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.477 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:23:43.477 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:29.307380+0000", 2023-09-23T20:23:43.477 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.9629999999999997e-05 2023-09-23T20:23:43.477 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.478 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.478 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:23:43.478 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:29.307527+0000", 2023-09-23T20:23:43.478 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00014735200000000001 2023-09-23T20:23:43.478 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.478 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.478 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:23:43.478 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:29.307662+0000", 2023-09-23T20:23:43.478 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00013504399999999999 2023-09-23T20:23:43.478 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.478 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.478 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:23:43.479 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:29.309643+0000", 2023-09-23T20:23:43.479 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0019810029999999998 2023-09-23T20:23:43.479 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.479 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.479 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:23:43.479 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:29.311123+0000", 2023-09-23T20:23:43.479 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00148011 2023-09-23T20:23:43.479 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.479 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.479 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:23:43.479 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:29.327541+0000", 2023-09-23T20:23:43.479 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.016417442000000001 2023-09-23T20:23:43.480 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.480 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.480 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:23:43.480 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:29.327582+0000", 2023-09-23T20:23:43.480 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.1596e-05 2023-09-23T20:23:43.480 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.480 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.480 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:23:43.480 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:29.327621+0000", 2023-09-23T20:23:43.480 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.8954e-05 2023-09-23T20:23:43.480 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:23:43.480 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:23:43.481 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:23:43.481 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.481 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.481 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:51061 5.e 5:7fe83d2c:::benchmark_data_smithi018_228451_object51060:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e132)", 2023-09-23T20:23:43.481 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:32.428495+0000", 2023-09-23T20:23:43.481 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 10.926352601, 2023-09-23T20:23:43.481 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.024212648, 2023-09-23T20:23:43.481 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:23:43.481 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:23:43.481 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:23:43.481 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:23:43.482 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:23:43.482 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 51061 2023-09-23T20:23:43.482 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.482 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:23:43.482 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.482 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:23:43.482 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:32.428495+0000", 2023-09-23T20:23:43.482 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:23:43.482 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.482 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.482 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:23:43.482 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:32.428495+0000", 2023-09-23T20:23:43.483 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:23:43.483 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.483 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.483 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:23:43.483 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:32.428495+0000", 2023-09-23T20:23:43.483 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:23:43.483 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.483 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.483 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:23:43.483 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:32.428602+0000", 2023-09-23T20:23:43.483 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000106866 2023-09-23T20:23:43.484 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.484 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.484 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:23:43.484 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:32.428613+0000", 2023-09-23T20:23:43.484 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.146e-05 2023-09-23T20:23:43.484 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.484 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.484 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:23:43.484 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:32.428618+0000", 2023-09-23T20:23:43.484 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.6659999999999999e-06 2023-09-23T20:23:43.484 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.484 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.485 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:23:43.485 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:32.430311+0000", 2023-09-23T20:23:43.485 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0016932080000000001 2023-09-23T20:23:43.485 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.485 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.485 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:23:43.485 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:32.430437+0000", 2023-09-23T20:23:43.485 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00012517100000000001 2023-09-23T20:23:43.485 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.485 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.485 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:23:43.486 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:32.430582+0000", 2023-09-23T20:23:43.486 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000145747 2023-09-23T20:23:43.486 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.486 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.486 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:23:43.486 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:32.432203+0000", 2023-09-23T20:23:43.486 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0016205740000000001 2023-09-23T20:23:43.486 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.486 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.486 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:23:43.486 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:32.432937+0000", 2023-09-23T20:23:43.487 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00073371899999999997 2023-09-23T20:23:43.487 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.487 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.487 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:23:43.487 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:32.452617+0000", 2023-09-23T20:23:43.487 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.019680458000000001 2023-09-23T20:23:43.487 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.487 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.487 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:23:43.487 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:32.452674+0000", 2023-09-23T20:23:43.487 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.6567000000000002e-05 2023-09-23T20:23:43.487 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.488 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.488 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:23:43.488 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:32.452708+0000", 2023-09-23T20:23:43.488 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.4541e-05 2023-09-23T20:23:43.488 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:23:43.488 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:23:43.488 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:23:43.488 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.488 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.488 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:59839 5.e 5:701840a9:::benchmark_data_smithi018_228451_object59838:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e139)", 2023-09-23T20:23:43.488 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:42.886527+0000", 2023-09-23T20:23:43.488 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 0.46832085299999998, 2023-09-23T20:23:43.489 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.023023136999999999, 2023-09-23T20:23:43.489 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:23:43.489 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:23:43.489 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:23:43.489 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:23:43.489 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:23:43.489 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 59839 2023-09-23T20:23:43.489 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.489 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:23:43.489 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.489 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:23:43.489 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:42.886527+0000", 2023-09-23T20:23:43.490 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:23:43.490 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.490 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.490 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:23:43.490 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:42.886527+0000", 2023-09-23T20:23:43.490 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:23:43.490 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.490 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.490 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:23:43.490 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:42.886527+0000", 2023-09-23T20:23:43.490 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:23:43.490 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.491 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.491 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:23:43.491 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:42.886555+0000", 2023-09-23T20:23:43.491 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.8187e-05 2023-09-23T20:23:43.491 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.491 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.491 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:23:43.491 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:42.886568+0000", 2023-09-23T20:23:43.491 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.2785e-05 2023-09-23T20:23:43.491 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.491 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.491 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:23:43.492 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:42.886573+0000", 2023-09-23T20:23:43.492 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.4979999999999997e-06 2023-09-23T20:23:43.492 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.492 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.492 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:23:43.492 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:42.888717+0000", 2023-09-23T20:23:43.492 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0021437420000000001 2023-09-23T20:23:43.492 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.492 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.492 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:23:43.492 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:42.888819+0000", 2023-09-23T20:23:43.492 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000102131 2023-09-23T20:23:43.493 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.493 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.493 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:23:43.493 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:42.888910+0000", 2023-09-23T20:23:43.493 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.1123000000000002e-05 2023-09-23T20:23:43.493 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.493 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.493 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:23:43.493 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:42.890775+0000", 2023-09-23T20:23:43.493 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0018654019999999999 2023-09-23T20:23:43.493 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.493 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.494 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:23:43.494 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:42.892048+0000", 2023-09-23T20:23:43.494 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0012726250000000001 2023-09-23T20:23:43.494 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.494 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.494 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:23:43.494 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:42.909454+0000", 2023-09-23T20:23:43.494 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.017405786999999999 2023-09-23T20:23:43.494 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.494 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.494 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:23:43.494 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:42.909509+0000", 2023-09-23T20:23:43.495 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.5213999999999997e-05 2023-09-23T20:23:43.495 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.495 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.495 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:23:43.495 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:42.909550+0000", 2023-09-23T20:23:43.495 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.1227999999999999e-05 2023-09-23T20:23:43.495 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:23:43.495 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:23:43.495 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:23:43.495 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.495 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.495 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:60280 5.e 5:7786a1f4:::benchmark_data_smithi018_228451_object60279:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e139)", 2023-09-23T20:23:43.496 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:43.216090+0000", 2023-09-23T20:23:43.496 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 0.13875784099999999, 2023-09-23T20:23:43.496 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.019836801000000001, 2023-09-23T20:23:43.496 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:23:43.496 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:23:43.496 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:23:43.496 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:23:43.496 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:23:43.496 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 60280 2023-09-23T20:23:43.496 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.496 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:23:43.496 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.497 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:23:43.497 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:43.216090+0000", 2023-09-23T20:23:43.497 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:23:43.497 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.497 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.497 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:23:43.497 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:43.216090+0000", 2023-09-23T20:23:43.497 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:23:43.497 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.497 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.497 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:23:43.497 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:43.216090+0000", 2023-09-23T20:23:43.498 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:23:43.498 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.498 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.498 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:23:43.498 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:43.216183+0000", 2023-09-23T20:23:43.498 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.2764999999999997e-05 2023-09-23T20:23:43.498 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.498 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.498 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:23:43.498 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:43.216190+0000", 2023-09-23T20:23:43.498 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.1230000000000002e-06 2023-09-23T20:23:43.498 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.499 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.499 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:23:43.499 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:43.216199+0000", 2023-09-23T20:23:43.499 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.9500000000000007e-06 2023-09-23T20:23:43.499 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.499 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.499 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:23:43.499 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:43.220567+0000", 2023-09-23T20:23:43.499 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0043683180000000004 2023-09-23T20:23:43.499 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.499 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.499 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:23:43.499 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:43.220690+0000", 2023-09-23T20:23:43.500 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00012288500000000001 2023-09-23T20:23:43.500 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.500 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.500 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:23:43.500 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:43.220845+0000", 2023-09-23T20:23:43.500 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00015525900000000001 2023-09-23T20:23:43.500 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.500 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.500 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:23:43.500 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:43.224391+0000", 2023-09-23T20:23:43.500 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0035463119999999998 2023-09-23T20:23:43.500 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.501 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.501 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:23:43.501 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:43.227723+0000", 2023-09-23T20:23:43.501 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0033316259999999999 2023-09-23T20:23:43.501 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.501 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.501 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:23:43.501 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:43.235869+0000", 2023-09-23T20:23:43.501 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0081455280000000008 2023-09-23T20:23:43.501 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.501 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.501 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:23:43.502 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:43.235903+0000", 2023-09-23T20:23:43.502 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.4529000000000001e-05 2023-09-23T20:23:43.502 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:43.502 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:43.502 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:23:43.502 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:43.235927+0000", 2023-09-23T20:23:43.502 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.3952e-05 2023-09-23T20:23:43.502 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:23:43.502 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:23:43.502 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:23:43.502 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:23:43.502 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:23:43.503 INFO:teuthology.orchestra.run.smithi149.stdout:} 2023-09-23T20:23:43.503 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:43 smithi149 ceph-mon[170394]: failed to encode map e139 with expected crc 2023-09-23T20:23:43.503 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:43 smithi149 ceph-mon[170394]: failed to encode map e139 with expected crc 2023-09-23T20:23:43.503 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:43 smithi149 ceph-mon[170394]: failed to encode map e139 with expected crc 2023-09-23T20:23:43.503 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:43 smithi149 ceph-mon[170394]: failed to encode map e139 with expected crc 2023-09-23T20:23:43.504 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:43 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1572670285' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:23:43.504 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:43.504 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:23:43.505 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:23:43.505 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 45 16 60634 60618 84.1476 83.625 0.0094595 0.0118788 2023-09-23T20:23:43.605 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:23:43.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:43 smithi018 ceph-mon[184144]: failed to encode map e139 with expected crc 2023-09-23T20:23:43.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:43 smithi018 ceph-mon[184144]: failed to encode map e139 with expected crc 2023-09-23T20:23:43.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:43 smithi018 ceph-mon[184144]: failed to encode map e139 with expected crc 2023-09-23T20:23:43.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:43 smithi018 ceph-mon[184144]: failed to encode map e139 with expected crc 2023-09-23T20:23:43.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:43 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1572670285' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:23:43.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:43 smithi018 ceph-mon[179647]: failed to encode map e139 with expected crc 2023-09-23T20:23:43.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:43 smithi018 ceph-mon[179647]: failed to encode map e139 with expected crc 2023-09-23T20:23:43.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:43 smithi018 ceph-mon[179647]: failed to encode map e139 with expected crc 2023-09-23T20:23:43.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:43 smithi018 ceph-mon[179647]: failed to encode map e139 with expected crc 2023-09-23T20:23:43.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:43 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1572670285' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:23:43.667 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:43.725 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:23:43.732 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:43.733 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:23:43.795 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T20:23:43.833 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:23:43.893 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:43.975 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:23:43.979 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:43.979 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:23:44.080 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:23:44.139 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:44.203 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:23:44.211 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:44.212 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:23:44.249 INFO:teuthology.orchestra.run.smithi018.stderr:set osd.0 primary-affinity to 1 (8655362) 2023-09-23T20:23:44.277 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:44 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/696994058' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T20:23:44.277 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:44 smithi018 ceph-mon[184144]: pgmap v284: 99 pgs: 2 active+recovering+undersized+remapped, 1 peering, 96 active+clean; 3.6 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 32 MiB/s wr, 509 op/s; 2212/176022 objects misplaced (1.257%); 23 MiB/s, 362 objects/s recovering 2023-09-23T20:23:44.278 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:44 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/696994058' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T20:23:44.278 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:44 smithi018 ceph-mon[179647]: pgmap v284: 99 pgs: 2 active+recovering+undersized+remapped, 1 peering, 96 active+clean; 3.6 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 32 MiB/s wr, 509 op/s; 2212/176022 objects misplaced (1.257%); 23 MiB/s, 362 objects/s recovering 2023-09-23T20:23:44.312 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:23:44.357 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:44.415 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:23:44.421 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:44.421 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:23:44.475 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 46 16 62099 62083 84.3086 91.5625 0.00977583 0.0118565 2023-09-23T20:23:44.521 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:23:44.575 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:44 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/696994058' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T20:23:44.575 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:44 smithi149 ceph-mon[170394]: pgmap v284: 99 pgs: 2 active+recovering+undersized+remapped, 1 peering, 96 active+clean; 3.6 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 32 MiB/s wr, 509 op/s; 2212/176022 objects misplaced (1.257%); 23 MiB/s, 362 objects/s recovering 2023-09-23T20:23:44.576 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:44.584 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:23:44.590 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T20:23:44.648 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:23:44.655 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:44.655 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:23:44.756 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:23:44.860 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:44.939 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:23:44.947 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:44.948 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:23:45.049 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:23:45.114 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:45.170 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:23:45.176 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:45.177 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:23:45.227 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:45 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/696994058' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2023-09-23T20:23:45.227 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:45 smithi018 ceph-mon[184144]: osdmap e140: 8 total, 8 up, 8 in 2023-09-23T20:23:45.228 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:45 smithi018 ceph-mon[184144]: failed to encode map e140 with expected crc 2023-09-23T20:23:45.228 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:45 smithi018 ceph-mon[184144]: failed to encode map e140 with expected crc 2023-09-23T20:23:45.228 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:45 smithi018 ceph-mon[184144]: failed to encode map e140 with expected crc 2023-09-23T20:23:45.228 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:45 smithi018 ceph-mon[184144]: failed to encode map e140 with expected crc 2023-09-23T20:23:45.228 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:45 smithi018 ceph-mon[184144]: failed to encode map e140 with expected crc 2023-09-23T20:23:45.228 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:45 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:23:45.229 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:23:44 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:23:44] "GET /metrics HTTP/1.1" 200 36477 "" "Prometheus/2.43.0" 2023-09-23T20:23:45.244 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:23:45.278 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:23:45.392 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:45.453 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:23:45.461 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:23:45.475 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 47 16 63390 63374 84.2314 80.6875 0.012952 0.0118659 2023-09-23T20:23:45.476 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:45.476 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:23:45.477 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:45 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/696994058' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2023-09-23T20:23:45.477 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:45 smithi018 ceph-mon[179647]: osdmap e140: 8 total, 8 up, 8 in 2023-09-23T20:23:45.477 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:45 smithi018 ceph-mon[179647]: failed to encode map e140 with expected crc 2023-09-23T20:23:45.477 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:45 smithi018 ceph-mon[179647]: failed to encode map e140 with expected crc 2023-09-23T20:23:45.477 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:45 smithi018 ceph-mon[179647]: failed to encode map e140 with expected crc 2023-09-23T20:23:45.477 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:45 smithi018 ceph-mon[179647]: failed to encode map e140 with expected crc 2023-09-23T20:23:45.477 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:45 smithi018 ceph-mon[179647]: failed to encode map e140 with expected crc 2023-09-23T20:23:45.477 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:45 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:23:45.577 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:23:45.635 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:45.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:45 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/696994058' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2023-09-23T20:23:45.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:45 smithi149 ceph-mon[170394]: osdmap e140: 8 total, 8 up, 8 in 2023-09-23T20:23:45.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:45 smithi149 ceph-mon[170394]: failed to encode map e140 with expected crc 2023-09-23T20:23:45.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:45 smithi149 ceph-mon[170394]: failed to encode map e140 with expected crc 2023-09-23T20:23:45.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:45 smithi149 ceph-mon[170394]: failed to encode map e140 with expected crc 2023-09-23T20:23:45.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:45 smithi149 ceph-mon[170394]: failed to encode map e140 with expected crc 2023-09-23T20:23:45.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:45 smithi149 ceph-mon[170394]: failed to encode map e140 with expected crc 2023-09-23T20:23:45.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:45 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:23:45.746 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:23:45.755 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:45.756 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:23:45.856 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:23:45.898 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:45.939 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:23:45.944 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:45.944 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:23:46.045 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:23:46.131 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:46.350 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:23:46.357 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:46.357 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:23:46.378 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:23:46.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:46 smithi018 ceph-mon[179647]: failed to encode map e140 with expected crc 2023-09-23T20:23:46.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:46 smithi018 ceph-mon[179647]: pgmap v286: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 96 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 54 MiB/s wr, 859 op/s; 3648/183882 objects degraded (1.984%); 2212/183882 objects misplaced (1.203%); 18 MiB/s, 293 objects/s recovering 2023-09-23T20:23:46.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:46 smithi018 ceph-mon[184144]: failed to encode map e140 with expected crc 2023-09-23T20:23:46.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:46 smithi018 ceph-mon[184144]: pgmap v286: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 96 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 54 MiB/s wr, 859 op/s; 3648/183882 objects degraded (1.984%); 2212/183882 objects misplaced (1.203%); 18 MiB/s, 293 objects/s recovering 2023-09-23T20:23:46.458 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:23:46.475 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 48 16 64677 64661 84.1521 80.4375 0.00950951 0.011878 2023-09-23T20:23:46.538 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:46.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:46 smithi149 ceph-mon[170394]: failed to encode map e140 with expected crc 2023-09-23T20:23:46.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:46 smithi149 ceph-mon[170394]: pgmap v286: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 96 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 54 MiB/s wr, 859 op/s; 3648/183882 objects degraded (1.984%); 2212/183882 objects misplaced (1.203%); 18 MiB/s, 293 objects/s recovering 2023-09-23T20:23:46.747 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:23:46.756 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:46.757 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:23:46.857 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:23:46.940 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:47.383 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:23:47.387 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:47.387 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:23:47.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:47 smithi149 ceph-mon[170394]: Health check failed: Degraded data redundancy: 3648/183882 objects degraded (1.984%), 1 pg degraded (PG_DEGRADED) 2023-09-23T20:23:47.476 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 49 16 66000 65984 84.1214 82.6875 0.0103792 0.011882 2023-09-23T20:23:47.488 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:23:47.538 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:47.643 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:47 smithi018 ceph-mon[184144]: Health check failed: Degraded data redundancy: 3648/183882 objects degraded (1.984%), 1 pg degraded (PG_DEGRADED) 2023-09-23T20:23:47.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:47 smithi018 ceph-mon[179647]: Health check failed: Degraded data redundancy: 3648/183882 objects degraded (1.984%), 1 pg degraded (PG_DEGRADED) 2023-09-23T20:23:47.709 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:23:47.714 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:47.714 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:23:47.815 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:23:47.895 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:47.975 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:23:47.980 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:47.980 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:23:47.997 INFO:teuthology.orchestra.run.smithi149.stdout:{ 2023-09-23T20:23:47.998 INFO:teuthology.orchestra.run.smithi149.stdout: "ops": [ 2023-09-23T20:23:47.998 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:47.998 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:66706 5.f 5:ff47fb8a:::benchmark_data_smithi018_228451_object66705:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e140)", 2023-09-23T20:23:47.998 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:47.994739+0000", 2023-09-23T20:23:47.998 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 0.0019873830000000001, 2023-09-23T20:23:47.998 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0020154719999999999, 2023-09-23T20:23:47.998 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:23:47.998 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:23:47.998 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:23:47.998 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:23:47.998 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:23:47.999 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 66706 2023-09-23T20:23:47.999 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:47.999 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:23:47.999 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:47.999 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:23:47.999 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:47.994739+0000", 2023-09-23T20:23:47.999 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:23:47.999 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:47.999 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:47.999 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:23:47.999 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:47.994739+0000", 2023-09-23T20:23:47.999 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:23:48.000 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:48.000 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:48.000 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:23:48.000 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:47.994738+0000", 2023-09-23T20:23:48.000 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:23:48.000 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:48.000 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:48.000 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:23:48.000 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:47.994763+0000", 2023-09-23T20:23:48.000 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.5442000000000001e-05 2023-09-23T20:23:48.000 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:48.000 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:48.000 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:23:48.001 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:47.994771+0000", 2023-09-23T20:23:48.001 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.5279999999999998e-06 2023-09-23T20:23:48.001 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:48.001 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:48.001 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:23:48.001 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:47.994775+0000", 2023-09-23T20:23:48.001 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.2849999999999996e-06 2023-09-23T20:23:48.001 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:48.001 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:48.001 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:23:48.001 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:47.994826+0000", 2023-09-23T20:23:48.002 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.1542999999999998e-05 2023-09-23T20:23:48.002 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:48.002 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:48.002 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:23:48.002 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:47.994979+0000", 2023-09-23T20:23:48.002 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000153108 2023-09-23T20:23:48.002 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:48.002 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:48.002 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 4,6", 2023-09-23T20:23:48.002 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:47.995152+0000", 2023-09-23T20:23:48.002 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00017279599999999999 2023-09-23T20:23:48.002 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:23:48.003 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:23:48.003 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:23:48.003 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:48.003 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:48.003 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:66708 5.f 5.880dbfbf (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e140)", 2023-09-23T20:23:48.003 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:47.994783+0000", 2023-09-23T20:23:48.003 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 0.0019427789999999999, 2023-09-23T20:23:48.003 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.002021012, 2023-09-23T20:23:48.003 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:23:48.003 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "queued for pg", 2023-09-23T20:23:48.003 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:23:48.003 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:23:48.004 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:23:48.004 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 66708 2023-09-23T20:23:48.004 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:48.004 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:23:48.004 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:48.004 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:23:48.004 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:47.994783+0000", 2023-09-23T20:23:48.004 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:23:48.004 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:48.004 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:48.004 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:23:48.005 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:47.994783+0000", 2023-09-23T20:23:48.005 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:23:48.005 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:48.005 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:48.005 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:23:48.005 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:47.994783+0000", 2023-09-23T20:23:48.005 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:23:48.005 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:48.005 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:48.005 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:23:48.005 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:47.994807+0000", 2023-09-23T20:23:48.005 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.4298e-05 2023-09-23T20:23:48.006 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:48.006 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:48.006 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:23:48.006 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:47.994813+0000", 2023-09-23T20:23:48.006 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.117e-06 2023-09-23T20:23:48.006 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:48.006 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:48.006 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:23:48.006 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:47.994816+0000", 2023-09-23T20:23:48.006 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.7609999999999999e-06 2023-09-23T20:23:48.006 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:23:48.006 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:23:48.007 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:23:48.007 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:48.007 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:48.007 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:66710 5.f 5.d92041df (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e140)", 2023-09-23T20:23:48.007 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:47.994828+0000", 2023-09-23T20:23:48.007 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 0.001898121, 2023-09-23T20:23:48.007 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0020093099999999998, 2023-09-23T20:23:48.007 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:23:48.007 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "queued for pg", 2023-09-23T20:23:48.007 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:23:48.007 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:23:48.007 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:23:48.007 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 66710 2023-09-23T20:23:48.008 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:48.008 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:23:48.008 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:48.008 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:23:48.008 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:47.994828+0000", 2023-09-23T20:23:48.008 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:23:48.008 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:48.008 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:48.008 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:23:48.008 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:47.994828+0000", 2023-09-23T20:23:48.008 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:23:48.008 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:48.009 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:48.009 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:23:48.009 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:47.994827+0000", 2023-09-23T20:23:48.009 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:23:48.009 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:48.009 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:48.009 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:23:48.009 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:47.994854+0000", 2023-09-23T20:23:48.009 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.6429000000000001e-05 2023-09-23T20:23:48.009 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:48.009 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:48.009 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:23:48.010 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:47.994860+0000", 2023-09-23T20:23:48.010 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.1109999999999996e-06 2023-09-23T20:23:48.010 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:48.010 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:48.010 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:23:48.010 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:47.994863+0000", 2023-09-23T20:23:48.010 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.146e-06 2023-09-23T20:23:48.010 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:23:48.010 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:23:48.010 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:23:48.010 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:48.010 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:48.011 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_repop(client.34791.0:66707 5.12 e140/131 5:4c43b03b:::benchmark_data_smithi018_228451_object66706:head v 140'2976, mlcod=140'2971)", 2023-09-23T20:23:48.011 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:47.995391+0000", 2023-09-23T20:23:48.011 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 0.0013352310000000001, 2023-09-23T20:23:48.011 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.001478952, 2023-09-23T20:23:48.011 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:23:48.011 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "started", 2023-09-23T20:23:48.011 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:23:48.011 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:48.011 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:23:48.011 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:47.995391+0000", 2023-09-23T20:23:48.011 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:23:48.011 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:48.011 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:48.012 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:23:48.012 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:47.995391+0000", 2023-09-23T20:23:48.012 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:23:48.012 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:48.012 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:48.012 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:23:48.012 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:47.995390+0000", 2023-09-23T20:23:48.012 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:23:48.012 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:48.012 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:48.012 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:23:48.013 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:47.995431+0000", 2023-09-23T20:23:48.013 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.1318e-05 2023-09-23T20:23:48.013 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:48.013 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:48.013 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:23:48.013 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:47.995439+0000", 2023-09-23T20:23:48.013 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.8699999999999992e-06 2023-09-23T20:23:48.013 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:48.013 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:48.013 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:23:48.013 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:47.995443+0000", 2023-09-23T20:23:48.013 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.0759999999999996e-06 2023-09-23T20:23:48.014 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:48.014 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:48.014 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:23:48.014 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:47.995482+0000", 2023-09-23T20:23:48.014 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.854e-05 2023-09-23T20:23:48.014 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:48.014 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:48.014 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:23:48.014 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:47.995501+0000", 2023-09-23T20:23:48.014 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.9349999999999999e-05 2023-09-23T20:23:48.014 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:23:48.015 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:23:48.015 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:23:48.015 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:23:48.015 INFO:teuthology.orchestra.run.smithi149.stdout: ], 2023-09-23T20:23:48.015 INFO:teuthology.orchestra.run.smithi149.stdout: "num_ops": 4 2023-09-23T20:23:48.015 INFO:teuthology.orchestra.run.smithi149.stdout:} 2023-09-23T20:23:48.075 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:23:48.081 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:23:48.104 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:23:48.136 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:23:48.149 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:48.172 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:23:48.209 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:23:48.244 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:23:48.290 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:23:48.292 INFO:teuthology.orchestra.run.smithi018.stderr:nodeep-scrub is unset 2023-09-23T20:23:48.329 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:23:48.336 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:23:48.336 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:23:48.336 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:23:48.336 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:23:48.336 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:23:48.336 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:23:48.337 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:23:48.337 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:23:48.418 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:23:48.420 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:48 smithi149 ceph-mon[170394]: pgmap v287: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 96 active+clean; 3.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 51 MiB/s wr, 822 op/s; 3648/189957 objects degraded (1.920%); 1490/189957 objects misplaced (0.784%); 22 MiB/s, 344 objects/s recovering 2023-09-23T20:23:48.420 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:48 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2444400847' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:23:48.420 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:48 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2440934300' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:23:48.421 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:48.421 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:23:48.476 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 50 16 67386 67370 84.1713 86.625 0.00661563 0.0118646 2023-09-23T20:23:48.522 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:23:48.531 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T20:23:48.566 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:48.631 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:23:48.637 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:48.637 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:23:48.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:48 smithi018 ceph-mon[184144]: pgmap v287: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 96 active+clean; 3.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 51 MiB/s wr, 822 op/s; 3648/189957 objects degraded (1.920%); 1490/189957 objects misplaced (0.784%); 22 MiB/s, 344 objects/s recovering 2023-09-23T20:23:48.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:48 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2444400847' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:23:48.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:48 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2440934300' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:23:48.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:48 smithi018 ceph-mon[179647]: pgmap v287: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 96 active+clean; 3.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 51 MiB/s wr, 822 op/s; 3648/189957 objects degraded (1.920%); 1490/189957 objects misplaced (0.784%); 22 MiB/s, 344 objects/s recovering 2023-09-23T20:23:48.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:48 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2444400847' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:23:48.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:48 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2440934300' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:23:48.738 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:23:48.772 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:48.815 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:23:48.822 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:48.823 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:23:48.924 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:23:48.957 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:48.996 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:23:48.999 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:48.999 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:23:49.100 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:23:49.128 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:49.161 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:23:49.164 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:49.164 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:23:49.257 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:49 smithi018 ceph-mon[179647]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:23:49.257 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:49 smithi018 ceph-mon[179647]: failed to encode map e141 with expected crc 2023-09-23T20:23:49.257 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:49 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2444400847' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:23:49.258 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:49 smithi018 ceph-mon[179647]: osdmap e141: 8 total, 8 up, 8 in 2023-09-23T20:23:49.258 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:49 smithi018 ceph-mon[179647]: failed to encode map e141 with expected crc 2023-09-23T20:23:49.258 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:23:49.265 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:23:49.313 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:49.375 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:23:49.379 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:49.380 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:23:49.476 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 51 16 68122 68106 83.4231 46 0.00765038 0.0119831 2023-09-23T20:23:49.480 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:23:49.540 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:49 smithi149 ceph-mon[170394]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:23:49.540 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:49 smithi149 ceph-mon[170394]: failed to encode map e141 with expected crc 2023-09-23T20:23:49.540 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:49 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2444400847' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:23:49.540 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:49 smithi149 ceph-mon[170394]: osdmap e141: 8 total, 8 up, 8 in 2023-09-23T20:23:49.540 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:49 smithi149 ceph-mon[170394]: failed to encode map e141 with expected crc 2023-09-23T20:23:49.550 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:49.610 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:23:49.614 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:49.615 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:23:49.640 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:49 smithi018 ceph-mon[184144]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:23:49.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:49 smithi018 ceph-mon[184144]: failed to encode map e141 with expected crc 2023-09-23T20:23:49.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:49 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2444400847' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:23:49.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:49 smithi018 ceph-mon[184144]: osdmap e141: 8 total, 8 up, 8 in 2023-09-23T20:23:49.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:49 smithi018 ceph-mon[184144]: failed to encode map e141 with expected crc 2023-09-23T20:23:49.667 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-23T20:23:49.667 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:23:49.668 INFO:tasks.thrashosds.thrasher:inject_pause on osd.7 2023-09-23T20:23:49.668 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T20:23:49.668 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T20:23:49.668 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 config set filestore_inject_stall 3 2023-09-23T20:23:49.716 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:23:49.786 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:50.112 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:23:50.117 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:50.118 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:23:50.219 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:23:50.296 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:50.371 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:23:50.378 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:50.378 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:23:50.379 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:50 smithi149 ceph-mon[170394]: failed to encode map e142 with expected crc 2023-09-23T20:23:50.379 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:50 smithi149 ceph-mon[170394]: osdmap e142: 8 total, 8 up, 8 in 2023-09-23T20:23:50.379 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:50 smithi149 ceph-mon[170394]: failed to encode map e142 with expected crc 2023-09-23T20:23:50.379 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:50 smithi149 ceph-mon[170394]: failed to encode map e142 with expected crc 2023-09-23T20:23:50.379 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:50 smithi149 ceph-mon[170394]: failed to encode map e142 with expected crc 2023-09-23T20:23:50.379 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:50 smithi149 ceph-mon[170394]: pgmap v290: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 96 active+clean; 3.9 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 59 MiB/s wr, 950 op/s; 3648/193128 objects degraded (1.889%); 986/193128 objects misplaced (0.511%); 13 MiB/s, 204 objects/s recovering 2023-09-23T20:23:50.379 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:50 smithi149 ceph-mon[170394]: 5.13 scrub starts 2023-09-23T20:23:50.476 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 52 16 69573 69557 83.5625 90.6875 0.0109583 0.0119624 2023-09-23T20:23:50.479 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:23:50.559 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:50.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:50 smithi018 ceph-mon[179647]: failed to encode map e142 with expected crc 2023-09-23T20:23:50.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:50 smithi018 ceph-mon[179647]: osdmap e142: 8 total, 8 up, 8 in 2023-09-23T20:23:50.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:50 smithi018 ceph-mon[179647]: failed to encode map e142 with expected crc 2023-09-23T20:23:50.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:50 smithi018 ceph-mon[179647]: failed to encode map e142 with expected crc 2023-09-23T20:23:50.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:50 smithi018 ceph-mon[179647]: failed to encode map e142 with expected crc 2023-09-23T20:23:50.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:50 smithi018 ceph-mon[179647]: pgmap v290: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 96 active+clean; 3.9 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 59 MiB/s wr, 950 op/s; 3648/193128 objects degraded (1.889%); 986/193128 objects misplaced (0.511%); 13 MiB/s, 204 objects/s recovering 2023-09-23T20:23:50.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:50 smithi018 ceph-mon[179647]: 5.13 scrub starts 2023-09-23T20:23:50.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:50 smithi018 ceph-mon[184144]: failed to encode map e142 with expected crc 2023-09-23T20:23:50.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:50 smithi018 ceph-mon[184144]: osdmap e142: 8 total, 8 up, 8 in 2023-09-23T20:23:50.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:50 smithi018 ceph-mon[184144]: failed to encode map e142 with expected crc 2023-09-23T20:23:50.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:50 smithi018 ceph-mon[184144]: failed to encode map e142 with expected crc 2023-09-23T20:23:50.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:50 smithi018 ceph-mon[184144]: failed to encode map e142 with expected crc 2023-09-23T20:23:50.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:50 smithi018 ceph-mon[184144]: pgmap v290: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 96 active+clean; 3.9 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 59 MiB/s wr, 950 op/s; 3648/193128 objects degraded (1.889%); 986/193128 objects misplaced (0.511%); 13 MiB/s, 204 objects/s recovering 2023-09-23T20:23:50.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:50 smithi018 ceph-mon[184144]: 5.13 scrub starts 2023-09-23T20:23:50.740 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:23:50.745 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:50.745 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:23:50.804 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:23:50.846 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:23:50.856 INFO:teuthology.orchestra.run.smithi149.stdout:ERROR: (22) Invalid argument 2023-09-23T20:23:50.856 INFO:teuthology.orchestra.run.smithi149.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:23:50.906 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T20:23:50.917 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:50.985 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:23:51.085 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:23:51.095 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:51.095 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:23:51.196 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:23:51.240 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:51.298 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:23:51.306 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:51.306 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:23:51.345 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T20:23:51.407 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:23:51.476 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 53 16 70905 70889 83.5565 83.25 0.00720246 0.0119639 2023-09-23T20:23:51.513 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:51.643 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:23:51.650 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:51.651 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:23:51.752 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:23:51.764 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:23:51.810 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:51.817 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:23:51.936 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:23:51.940 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:51.940 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:23:51.942 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:51 smithi149 ceph-mon[170394]: 5.14 scrub starts 2023-09-23T20:23:52.041 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:23:52.126 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:52.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:51 smithi018 ceph-mon[184144]: 5.14 scrub starts 2023-09-23T20:23:52.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:51 smithi018 ceph-mon[179647]: 5.14 scrub starts 2023-09-23T20:23:52.292 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:23:52.301 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:52.302 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:23:52.403 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:23:52.404 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:23:52.476 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 54 16 72242 72226 83.5565 83.5625 0.0179036 0.0119633 2023-09-23T20:23:52.483 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:52.554 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:23:52.562 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:52.563 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:23:52.664 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:23:52.718 INFO:teuthology.orchestra.run.smithi149.stdout:{ 2023-09-23T20:23:52.718 INFO:teuthology.orchestra.run.smithi149.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-09-23T20:23:52.718 INFO:teuthology.orchestra.run.smithi149.stdout:} 2023-09-23T20:23:52.726 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:52.845 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:23:52.851 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:52.852 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:23:52.952 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:23:52.969 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:52 smithi018 ceph-mon[179647]: 5.14 scrub ok 2023-09-23T20:23:52.969 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:52 smithi018 ceph-mon[179647]: pgmap v291: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 96 active+clean; 4.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s; 3648/202722 objects degraded (1.800%); 986/202722 objects misplaced (0.486%); 11 MiB/s, 169 objects/s recovering 2023-09-23T20:23:52.969 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:52 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 3648/202722 objects degraded (1.800%), 1 pg degraded (PG_DEGRADED) 2023-09-23T20:23:52.969 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:52 smithi018 ceph-mon[184144]: 5.14 scrub ok 2023-09-23T20:23:52.969 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:52 smithi018 ceph-mon[184144]: pgmap v291: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 96 active+clean; 4.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s; 3648/202722 objects degraded (1.800%); 986/202722 objects misplaced (0.486%); 11 MiB/s, 169 objects/s recovering 2023-09-23T20:23:52.969 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:52 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 3648/202722 objects degraded (1.800%), 1 pg degraded (PG_DEGRADED) 2023-09-23T20:23:53.038 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:53.125 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:52 smithi149 ceph-mon[170394]: 5.14 scrub ok 2023-09-23T20:23:53.125 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:52 smithi149 ceph-mon[170394]: pgmap v291: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 96 active+clean; 4.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s; 3648/202722 objects degraded (1.800%); 986/202722 objects misplaced (0.486%); 11 MiB/s, 169 objects/s recovering 2023-09-23T20:23:53.125 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:52 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 3648/202722 objects degraded (1.800%), 1 pg degraded (PG_DEGRADED) 2023-09-23T20:23:53.240 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:23:53.246 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:53.247 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:23:53.348 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:23:53.437 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:53.476 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 55 16 73590 73574 83.569 84.25 0.0101514 0.0119608 2023-09-23T20:23:53.558 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:23:53.571 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:53.572 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:23:53.673 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:23:53.740 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:53.815 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:23:53.828 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:53.828 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:23:53.911 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:23:53.929 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:23:53.947 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:23:53.980 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:23:54.005 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:54.011 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:23:54.042 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:23:54.078 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:23:54.080 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:23:54.087 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:53 smithi149 ceph-mon[170394]: 5.13 scrub starts 2023-09-23T20:23:54.087 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:53 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3053774062' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:23:54.087 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:54.088 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:23:54.108 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:23:54.112 INFO:teuthology.orchestra.run.smithi149.stdout:{ 2023-09-23T20:23:54.112 INFO:teuthology.orchestra.run.smithi149.stdout: "size": 20, 2023-09-23T20:23:54.112 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 600, 2023-09-23T20:23:54.112 INFO:teuthology.orchestra.run.smithi149.stdout: "ops": [ 2023-09-23T20:23:54.112 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.112 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:28900 5.7 5:e711fc21:::benchmark_data_smithi018_228451_object28899:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2023-09-23T20:23:54.113 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:09.962001+0000", 2023-09-23T20:23:54.113 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 44.14655526, 2023-09-23T20:23:54.113 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.066552317, 2023-09-23T20:23:54.113 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:23:54.113 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:23:54.113 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:23:54.113 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:23:54.113 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:23:54.113 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 28900 2023-09-23T20:23:54.113 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.113 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:23:54.113 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.113 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:23:54.114 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:09.962001+0000", 2023-09-23T20:23:54.114 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:23:54.114 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.114 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.114 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:23:54.114 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:09.962001+0000", 2023-09-23T20:23:54.114 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:23:54.114 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.114 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.114 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:23:54.114 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:09.962000+0000", 2023-09-23T20:23:54.114 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:23:54.115 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.115 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.115 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:23:54.115 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:09.962049+0000", 2023-09-23T20:23:54.115 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.9487999999999998e-05 2023-09-23T20:23:54.115 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.115 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.115 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:23:54.115 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:09.962064+0000", 2023-09-23T20:23:54.115 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.4771e-05 2023-09-23T20:23:54.115 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.115 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.116 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:23:54.116 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:09.962071+0000", 2023-09-23T20:23:54.116 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.7089999999999996e-06 2023-09-23T20:23:54.116 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.116 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.116 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:23:54.116 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:09.962124+0000", 2023-09-23T20:23:54.116 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.3349e-05 2023-09-23T20:23:54.116 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.116 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.116 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:23:54.116 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:09.962259+0000", 2023-09-23T20:23:54.117 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00013512800000000001 2023-09-23T20:23:54.117 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.117 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.117 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,1", 2023-09-23T20:23:54.117 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:09.962409+0000", 2023-09-23T20:23:54.117 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00014919900000000001 2023-09-23T20:23:54.117 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.117 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.117 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:23:54.117 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:09.963457+0000", 2023-09-23T20:23:54.117 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0010485500000000001 2023-09-23T20:23:54.117 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.117 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.118 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:23:54.118 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:10.028538+0000", 2023-09-23T20:23:54.118 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.065080959999999993 2023-09-23T20:23:54.118 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.118 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.118 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:23:54.118 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:10.028553+0000", 2023-09-23T20:23:54.118 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.5347000000000001e-05 2023-09-23T20:23:54.118 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:23:54.118 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:23:54.118 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:23:54.118 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.119 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.119 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:28900 5.7 5:e711fc21:::benchmark_data_smithi018_228451_object28899:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e118)", 2023-09-23T20:23:54.119 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:09.965170+0000", 2023-09-23T20:23:54.119 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 44.14338652, 2023-09-23T20:23:54.119 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.11416534, 2023-09-23T20:23:54.119 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:23:54.119 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:23:54.119 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:23:54.119 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:23:54.119 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:23:54.119 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 28900 2023-09-23T20:23:54.119 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.120 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:23:54.120 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.120 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:23:54.120 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:09.965170+0000", 2023-09-23T20:23:54.120 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:23:54.120 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.120 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.120 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:23:54.120 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:09.965170+0000", 2023-09-23T20:23:54.120 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:23:54.120 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.120 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.120 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:23:54.121 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:09.965169+0000", 2023-09-23T20:23:54.121 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:23:54.121 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.121 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.121 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:23:54.121 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:09.965230+0000", 2023-09-23T20:23:54.121 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.1246000000000003e-05 2023-09-23T20:23:54.121 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.121 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.121 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:23:54.121 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:09.965242+0000", 2023-09-23T20:23:54.121 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1778e-05 2023-09-23T20:23:54.122 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.122 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.122 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:23:54.122 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:09.965247+0000", 2023-09-23T20:23:54.122 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.0220000000000004e-06 2023-09-23T20:23:54.122 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.122 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.122 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:23:54.122 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:09.965283+0000", 2023-09-23T20:23:54.122 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.553e-05 2023-09-23T20:23:54.122 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.122 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.122 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op must wait for map", 2023-09-23T20:23:54.123 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:09.965287+0000", 2023-09-23T20:23:54.123 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.233e-06 2023-09-23T20:23:54.123 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.123 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.123 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:23:54.123 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:10.029527+0000", 2023-09-23T20:23:54.123 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.064239990999999996 2023-09-23T20:23:54.123 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.123 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.123 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:23:54.123 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:10.029536+0000", 2023-09-23T20:23:54.123 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.9590000000000001e-06 2023-09-23T20:23:54.124 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.124 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.124 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:23:54.124 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.072695+0000", 2023-09-23T20:23:54.124 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.043158966 2023-09-23T20:23:54.124 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.124 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.124 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for degraded object", 2023-09-23T20:23:54.124 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.073612+0000", 2023-09-23T20:23:54.124 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00091741399999999997 2023-09-23T20:23:54.124 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.124 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.124 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:23:54.125 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.079262+0000", 2023-09-23T20:23:54.125 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0056501490000000001 2023-09-23T20:23:54.125 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.125 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.125 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:23:54.125 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.079335+0000", 2023-09-23T20:23:54.125 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.2902000000000001e-05 2023-09-23T20:23:54.125 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:23:54.125 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:23:54.125 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:23:54.125 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.125 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.126 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:28904 5.7 5:e741b35c:::benchmark_data_smithi018_228451_object28903:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e118)", 2023-09-23T20:23:54.126 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:09.965427+0000", 2023-09-23T20:23:54.126 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 44.143129633000001, 2023-09-23T20:23:54.126 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.113551148, 2023-09-23T20:23:54.126 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:23:54.126 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:23:54.126 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:23:54.126 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:23:54.126 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:23:54.126 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 28904 2023-09-23T20:23:54.126 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.126 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:23:54.126 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.127 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:23:54.127 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:09.965427+0000", 2023-09-23T20:23:54.127 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:23:54.127 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.127 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.127 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:23:54.127 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:09.965427+0000", 2023-09-23T20:23:54.127 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:23:54.127 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.127 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.127 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:23:54.127 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:09.965426+0000", 2023-09-23T20:23:54.128 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:23:54.128 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.128 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.128 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:23:54.128 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:09.965494+0000", 2023-09-23T20:23:54.128 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.7729000000000002e-05 2023-09-23T20:23:54.128 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.128 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.128 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:23:54.128 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:09.965502+0000", 2023-09-23T20:23:54.128 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.4840000000000004e-06 2023-09-23T20:23:54.128 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.128 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.129 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:23:54.129 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:09.965506+0000", 2023-09-23T20:23:54.129 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.7220000000000001e-06 2023-09-23T20:23:54.129 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.129 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.129 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:23:54.129 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:09.965534+0000", 2023-09-23T20:23:54.129 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.7809000000000001e-05 2023-09-23T20:23:54.129 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.129 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.129 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting_for_map not empty", 2023-09-23T20:23:54.129 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:09.965538+0000", 2023-09-23T20:23:54.130 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.7299999999999999e-06 2023-09-23T20:23:54.130 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.130 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.130 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:23:54.130 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:10.031769+0000", 2023-09-23T20:23:54.130 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.066231060999999994 2023-09-23T20:23:54.130 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.130 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.130 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:23:54.130 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:10.031783+0000", 2023-09-23T20:23:54.130 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.4116e-05 2023-09-23T20:23:54.130 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.130 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.131 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:23:54.131 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.073645+0000", 2023-09-23T20:23:54.131 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0418621770000001 2023-09-23T20:23:54.131 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.131 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.131 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:23:54.131 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.073797+0000", 2023-09-23T20:23:54.131 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00015189500000000001 2023-09-23T20:23:54.131 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.131 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.131 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,1", 2023-09-23T20:23:54.131 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.073947+0000", 2023-09-23T20:23:54.132 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00014965900000000001 2023-09-23T20:23:54.132 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.132 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.132 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:23:54.132 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.075945+0000", 2023-09-23T20:23:54.132 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0019980919999999999 2023-09-23T20:23:54.132 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.132 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.132 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:23:54.132 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.078775+0000", 2023-09-23T20:23:54.132 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.002830122 2023-09-23T20:23:54.132 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.133 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.133 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:23:54.133 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.078844+0000", 2023-09-23T20:23:54.133 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.8878e-05 2023-09-23T20:23:54.133 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.133 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.133 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:23:54.133 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.078894+0000", 2023-09-23T20:23:54.133 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.0593999999999997e-05 2023-09-23T20:23:54.133 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.133 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.133 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:23:54.133 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.078978+0000", 2023-09-23T20:23:54.134 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.3540000000000003e-05 2023-09-23T20:23:54.134 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:23:54.134 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:23:54.134 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:23:54.134 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.134 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.134 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:28910 5.17 5:efedae70:::benchmark_data_smithi018_228451_object28909:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e118)", 2023-09-23T20:23:54.134 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:09.976859+0000", 2023-09-23T20:23:54.134 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 44.131697113999998, 2023-09-23T20:23:54.134 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1056038930000001, 2023-09-23T20:23:54.134 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:23:54.134 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:23:54.135 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:23:54.135 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:23:54.135 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:23:54.135 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 28910 2023-09-23T20:23:54.135 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.135 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:23:54.135 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.135 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:23:54.135 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:09.976859+0000", 2023-09-23T20:23:54.135 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:23:54.135 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.135 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.136 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:23:54.136 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:09.976859+0000", 2023-09-23T20:23:54.136 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:23:54.136 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.136 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.136 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:23:54.136 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:09.976856+0000", 2023-09-23T20:23:54.136 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999971 2023-09-23T20:23:54.136 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.136 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.136 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:23:54.136 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:09.976937+0000", 2023-09-23T20:23:54.136 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.0599999999999994e-05 2023-09-23T20:23:54.137 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.137 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.137 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:23:54.137 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:09.976958+0000", 2023-09-23T20:23:54.137 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.0599999999999999e-05 2023-09-23T20:23:54.137 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.137 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.137 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:23:54.137 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:09.976968+0000", 2023-09-23T20:23:54.137 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0611000000000001e-05 2023-09-23T20:23:54.137 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.137 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.137 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:23:54.138 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:10.029408+0000", 2023-09-23T20:23:54.138 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.052440242999999997 2023-09-23T20:23:54.138 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.138 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.138 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:23:54.138 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:10.029420+0000", 2023-09-23T20:23:54.138 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1399e-05 2023-09-23T20:23:54.138 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.138 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.138 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:23:54.138 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.077194+0000", 2023-09-23T20:23:54.138 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0477743589999999 2023-09-23T20:23:54.139 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.139 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.139 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:23:54.139 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.077406+0000", 2023-09-23T20:23:54.139 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00021226799999999999 2023-09-23T20:23:54.139 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.139 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.139 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,1", 2023-09-23T20:23:54.139 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.077553+0000", 2023-09-23T20:23:54.139 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00014669800000000001 2023-09-23T20:23:54.139 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.139 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.139 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:23:54.140 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.078739+0000", 2023-09-23T20:23:54.140 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.001185385 2023-09-23T20:23:54.140 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.140 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.140 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:23:54.140 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.080743+0000", 2023-09-23T20:23:54.140 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0020049310000000002 2023-09-23T20:23:54.140 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.140 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.140 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:23:54.140 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.082384+0000", 2023-09-23T20:23:54.140 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.001640664 2023-09-23T20:23:54.141 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.141 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.141 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:23:54.141 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.082428+0000", 2023-09-23T20:23:54.141 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.3615000000000002e-05 2023-09-23T20:23:54.141 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.141 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.141 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:23:54.141 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.082463+0000", 2023-09-23T20:23:54.141 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.5348000000000001e-05 2023-09-23T20:23:54.141 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:23:54.141 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:23:54.141 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:23:54.142 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.142 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.142 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:28911 5.17 5:e90730f8:::benchmark_data_smithi018_228451_object28910:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e118)", 2023-09-23T20:23:54.142 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:09.994956+0000", 2023-09-23T20:23:54.142 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 44.113600097999999, 2023-09-23T20:23:54.142 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0910110099999999, 2023-09-23T20:23:54.142 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:23:54.142 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:23:54.142 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:23:54.142 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:23:54.142 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:23:54.142 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 28911 2023-09-23T20:23:54.143 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.143 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:23:54.143 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.143 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:23:54.143 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:09.994956+0000", 2023-09-23T20:23:54.143 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:23:54.143 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.143 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.143 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:23:54.143 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:09.994956+0000", 2023-09-23T20:23:54.143 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:23:54.143 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.143 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.144 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:23:54.144 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:09.994953+0000", 2023-09-23T20:23:54.144 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999967 2023-09-23T20:23:54.144 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.144 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.144 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:23:54.144 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:09.995026+0000", 2023-09-23T20:23:54.144 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.3279000000000001e-05 2023-09-23T20:23:54.144 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.144 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.144 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:23:54.144 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:09.995055+0000", 2023-09-23T20:23:54.145 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.8319000000000001e-05 2023-09-23T20:23:54.145 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.145 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.145 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:23:54.145 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:09.995069+0000", 2023-09-23T20:23:54.145 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.4309e-05 2023-09-23T20:23:54.145 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.145 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.145 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:23:54.145 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:10.029439+0000", 2023-09-23T20:23:54.145 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.034370276999999998 2023-09-23T20:23:54.145 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.146 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.146 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:23:54.146 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:10.029444+0000", 2023-09-23T20:23:54.146 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.0470000000000001e-06 2023-09-23T20:23:54.146 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.146 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.146 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:23:54.146 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.079384+0000", 2023-09-23T20:23:54.146 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0499395030000001 2023-09-23T20:23:54.146 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.146 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.146 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:23:54.147 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.079449+0000", 2023-09-23T20:23:54.147 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.4920999999999997e-05 2023-09-23T20:23:54.147 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.147 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.147 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,1", 2023-09-23T20:23:54.147 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.079565+0000", 2023-09-23T20:23:54.147 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000116261 2023-09-23T20:23:54.147 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.147 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.147 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:23:54.147 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.080749+0000", 2023-09-23T20:23:54.147 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0011843750000000001 2023-09-23T20:23:54.148 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.148 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.148 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:23:54.148 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.083421+0000", 2023-09-23T20:23:54.148 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0026717690000000001 2023-09-23T20:23:54.148 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.148 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.148 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:23:54.148 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.085866+0000", 2023-09-23T20:23:54.148 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0024445579999999999 2023-09-23T20:23:54.148 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.148 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.149 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:23:54.149 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.085919+0000", 2023-09-23T20:23:54.149 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.3133000000000001e-05 2023-09-23T20:23:54.149 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.149 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.149 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:23:54.149 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.085967+0000", 2023-09-23T20:23:54.149 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.8559e-05 2023-09-23T20:23:54.149 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:23:54.149 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:23:54.149 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:23:54.149 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.149 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.150 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:29966 5.7 5:e2d2f422:::benchmark_data_smithi018_228451_object29965:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e119)", 2023-09-23T20:23:54.150 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:11.526539+0000", 2023-09-23T20:23:54.150 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 42.582016987000003, 2023-09-23T20:23:54.150 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.049077708999999997, 2023-09-23T20:23:54.150 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:23:54.150 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:23:54.150 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:23:54.150 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:23:54.150 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:23:54.150 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 29966 2023-09-23T20:23:54.150 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.150 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:23:54.151 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.151 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:23:54.151 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.526539+0000", 2023-09-23T20:23:54.151 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:23:54.151 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.151 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.151 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:23:54.151 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.526539+0000", 2023-09-23T20:23:54.151 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:23:54.151 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.151 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.151 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:23:54.151 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.526539+0000", 2023-09-23T20:23:54.152 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:23:54.152 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.152 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.152 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:23:54.152 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.526569+0000", 2023-09-23T20:23:54.152 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.0436000000000001e-05 2023-09-23T20:23:54.152 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.152 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.152 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:23:54.152 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.526577+0000", 2023-09-23T20:23:54.152 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.2490000000000004e-06 2023-09-23T20:23:54.152 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.153 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.153 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:23:54.153 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.526582+0000", 2023-09-23T20:23:54.153 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.9629999999999997e-06 2023-09-23T20:23:54.153 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.153 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.153 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:23:54.153 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.527504+0000", 2023-09-23T20:23:54.153 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00092279299999999999 2023-09-23T20:23:54.153 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.153 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.153 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:23:54.153 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.527922+0000", 2023-09-23T20:23:54.154 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00041762100000000002 2023-09-23T20:23:54.154 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.154 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.154 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,1", 2023-09-23T20:23:54.154 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.528059+0000", 2023-09-23T20:23:54.154 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000137346 2023-09-23T20:23:54.154 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.154 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.154 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:23:54.154 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.528782+0000", 2023-09-23T20:23:54.154 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000722311 2023-09-23T20:23:54.154 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.155 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.155 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:23:54.155 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.530049+0000", 2023-09-23T20:23:54.155 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0012674839999999999 2023-09-23T20:23:54.155 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.155 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.155 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:23:54.155 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.575553+0000", 2023-09-23T20:23:54.155 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.045504078000000003 2023-09-23T20:23:54.155 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.155 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.155 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:23:54.156 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.575588+0000", 2023-09-23T20:23:54.156 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.4736000000000001e-05 2023-09-23T20:23:54.156 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.156 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.156 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:23:54.156 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.575617+0000", 2023-09-23T20:23:54.156 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.9166000000000002e-05 2023-09-23T20:23:54.156 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:23:54.156 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:23:54.156 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:23:54.156 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.156 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.156 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:30048 5.7 5:e4e7bb94:::benchmark_data_smithi018_228451_object30047:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e119)", 2023-09-23T20:23:54.157 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:11.600486+0000", 2023-09-23T20:23:54.157 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 42.508070410999999, 2023-09-23T20:23:54.157 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.066197917999999994, 2023-09-23T20:23:54.157 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:23:54.157 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:23:54.157 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:23:54.157 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:23:54.157 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:23:54.157 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 30048 2023-09-23T20:23:54.157 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.157 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:23:54.157 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.158 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:23:54.158 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.600486+0000", 2023-09-23T20:23:54.158 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:23:54.158 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.158 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.158 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:23:54.158 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.600486+0000", 2023-09-23T20:23:54.158 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:23:54.158 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.158 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.158 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:23:54.158 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.600485+0000", 2023-09-23T20:23:54.158 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:23:54.159 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.159 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.159 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:23:54.159 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.600517+0000", 2023-09-23T20:23:54.159 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.2496000000000002e-05 2023-09-23T20:23:54.159 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.159 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.159 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:23:54.159 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.600526+0000", 2023-09-23T20:23:54.159 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.3030000000000005e-06 2023-09-23T20:23:54.159 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.159 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.160 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:23:54.160 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.600532+0000", 2023-09-23T20:23:54.160 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.1479999999999997e-06 2023-09-23T20:23:54.160 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.160 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.160 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:23:54.160 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.600590+0000", 2023-09-23T20:23:54.160 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.8121000000000001e-05 2023-09-23T20:23:54.160 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.160 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.160 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:23:54.160 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.600698+0000", 2023-09-23T20:23:54.161 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000108149 2023-09-23T20:23:54.161 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.161 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.161 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,1", 2023-09-23T20:23:54.161 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.600833+0000", 2023-09-23T20:23:54.161 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00013561700000000001 2023-09-23T20:23:54.161 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.161 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.161 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:23:54.161 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.602384+0000", 2023-09-23T20:23:54.161 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0015502510000000001 2023-09-23T20:23:54.161 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.162 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.162 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:23:54.162 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.603154+0000", 2023-09-23T20:23:54.162 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000770264 2023-09-23T20:23:54.162 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.162 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.162 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:23:54.162 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.666629+0000", 2023-09-23T20:23:54.162 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.063475153000000006 2023-09-23T20:23:54.162 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.162 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.162 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:23:54.162 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.666658+0000", 2023-09-23T20:23:54.163 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.9283000000000001e-05 2023-09-23T20:23:54.163 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.163 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.163 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:23:54.163 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.666684+0000", 2023-09-23T20:23:54.163 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.5449999999999999e-05 2023-09-23T20:23:54.163 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:23:54.163 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:23:54.163 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:23:54.163 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.163 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.163 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:30052 5.17 5:e970ad06:::benchmark_data_smithi018_228451_object30051:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e119)", 2023-09-23T20:23:54.164 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:11.600946+0000", 2023-09-23T20:23:54.164 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 42.507610092, 2023-09-23T20:23:54.164 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.065737426000000002, 2023-09-23T20:23:54.164 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:23:54.164 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:23:54.164 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:23:54.164 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:23:54.164 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:23:54.164 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 30052 2023-09-23T20:23:54.164 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.164 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:23:54.164 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.165 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:23:54.165 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.600946+0000", 2023-09-23T20:23:54.165 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:23:54.165 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.165 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.165 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:23:54.165 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.600946+0000", 2023-09-23T20:23:54.165 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:23:54.165 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.165 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.165 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:23:54.165 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.600945+0000", 2023-09-23T20:23:54.165 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:23:54.166 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.166 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.166 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:23:54.166 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.601021+0000", 2023-09-23T20:23:54.166 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.5463999999999994e-05 2023-09-23T20:23:54.166 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.166 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.166 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:23:54.166 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.601036+0000", 2023-09-23T20:23:54.166 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.4952e-05 2023-09-23T20:23:54.166 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.166 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.167 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:23:54.167 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.601043+0000", 2023-09-23T20:23:54.167 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.058e-06 2023-09-23T20:23:54.167 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.167 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.167 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:23:54.167 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.601102+0000", 2023-09-23T20:23:54.167 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.9562999999999999e-05 2023-09-23T20:23:54.167 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.167 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.167 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:23:54.167 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.601220+0000", 2023-09-23T20:23:54.167 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000117835 2023-09-23T20:23:54.168 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.168 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.168 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,1", 2023-09-23T20:23:54.168 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.601358+0000", 2023-09-23T20:23:54.168 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000138052 2023-09-23T20:23:54.168 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.168 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.168 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:23:54.168 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.603159+0000", 2023-09-23T20:23:54.168 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0018007050000000001 2023-09-23T20:23:54.168 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.168 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.169 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:23:54.169 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.604725+0000", 2023-09-23T20:23:54.169 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0015660959999999999 2023-09-23T20:23:54.169 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.169 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.169 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:23:54.169 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.666616+0000", 2023-09-23T20:23:54.169 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.061891109 2023-09-23T20:23:54.169 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.169 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.169 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:23:54.169 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.666655+0000", 2023-09-23T20:23:54.169 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.8930999999999997e-05 2023-09-23T20:23:54.170 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.170 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.170 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:23:54.170 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.666684+0000", 2023-09-23T20:23:54.170 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.8537000000000001e-05 2023-09-23T20:23:54.170 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:23:54.170 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:23:54.170 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:23:54.170 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.170 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.170 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:30053 5.17 5:ece8a473:::benchmark_data_smithi018_228451_object30052:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e119)", 2023-09-23T20:23:54.170 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:11.601053+0000", 2023-09-23T20:23:54.171 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 42.507503010999997, 2023-09-23T20:23:54.171 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.065697773000000001, 2023-09-23T20:23:54.171 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:23:54.171 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:23:54.171 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:23:54.171 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:23:54.171 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:23:54.171 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 30053 2023-09-23T20:23:54.171 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.171 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:23:54.171 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.171 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:23:54.172 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.601053+0000", 2023-09-23T20:23:54.172 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:23:54.172 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.172 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.172 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:23:54.172 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.601053+0000", 2023-09-23T20:23:54.172 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:23:54.172 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.172 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.172 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:23:54.172 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.601053+0000", 2023-09-23T20:23:54.172 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:23:54.173 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.173 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.173 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:23:54.173 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.601172+0000", 2023-09-23T20:23:54.173 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000119479 2023-09-23T20:23:54.173 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.173 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.173 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:23:54.173 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.601180+0000", 2023-09-23T20:23:54.173 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.1049999999999995e-06 2023-09-23T20:23:54.173 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.173 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.174 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:23:54.174 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.601186+0000", 2023-09-23T20:23:54.174 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.4079999999999997e-06 2023-09-23T20:23:54.174 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.174 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.174 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:23:54.174 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.601871+0000", 2023-09-23T20:23:54.174 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000685533 2023-09-23T20:23:54.174 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.174 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.174 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:23:54.174 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.601970+0000", 2023-09-23T20:23:54.174 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.9182e-05 2023-09-23T20:23:54.175 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.175 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.175 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,1", 2023-09-23T20:23:54.175 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.602078+0000", 2023-09-23T20:23:54.175 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00010724400000000001 2023-09-23T20:23:54.175 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.175 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.175 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:23:54.175 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.604732+0000", 2023-09-23T20:23:54.175 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0026540729999999998 2023-09-23T20:23:54.175 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.175 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.176 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:23:54.176 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.605027+0000", 2023-09-23T20:23:54.176 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000294813 2023-09-23T20:23:54.176 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.176 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.176 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:23:54.176 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.666710+0000", 2023-09-23T20:23:54.176 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.061683267999999999 2023-09-23T20:23:54.176 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.176 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.176 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:23:54.176 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.666733+0000", 2023-09-23T20:23:54.177 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.2699000000000001e-05 2023-09-23T20:23:54.177 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.177 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.177 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:23:54.177 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.666751+0000", 2023-09-23T20:23:54.177 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.8522999999999999e-05 2023-09-23T20:23:54.177 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:23:54.177 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:23:54.177 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:23:54.177 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.177 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.178 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:30067 5.17 5:e8cc916b:::benchmark_data_smithi018_228451_object30066:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e119)", 2023-09-23T20:23:54.178 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:11.605138+0000", 2023-09-23T20:23:54.178 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 42.503418345, 2023-09-23T20:23:54.178 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.061700006000000002, 2023-09-23T20:23:54.178 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:23:54.178 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:23:54.178 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:23:54.178 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:23:54.178 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:23:54.178 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 30067 2023-09-23T20:23:54.178 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.178 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:23:54.178 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.179 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:23:54.179 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.605138+0000", 2023-09-23T20:23:54.179 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:23:54.179 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.179 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.179 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:23:54.179 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.605138+0000", 2023-09-23T20:23:54.179 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:23:54.179 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.179 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.179 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:23:54.179 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.605136+0000", 2023-09-23T20:23:54.180 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999981 2023-09-23T20:23:54.180 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.180 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.180 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:23:54.180 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.605166+0000", 2023-09-23T20:23:54.180 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.9580000000000001e-05 2023-09-23T20:23:54.180 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.180 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.180 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:23:54.180 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.605176+0000", 2023-09-23T20:23:54.180 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.025e-05 2023-09-23T20:23:54.180 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.180 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.181 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:23:54.181 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.605182+0000", 2023-09-23T20:23:54.181 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.7219999999999996e-06 2023-09-23T20:23:54.181 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.181 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.181 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:23:54.181 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.605236+0000", 2023-09-23T20:23:54.181 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.4567000000000001e-05 2023-09-23T20:23:54.181 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.181 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.181 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:23:54.181 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.605363+0000", 2023-09-23T20:23:54.181 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000126811 2023-09-23T20:23:54.182 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.182 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.182 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,1", 2023-09-23T20:23:54.182 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.605536+0000", 2023-09-23T20:23:54.182 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00017262199999999999 2023-09-23T20:23:54.182 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.182 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.182 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:23:54.182 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.606673+0000", 2023-09-23T20:23:54.182 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0011375720000000001 2023-09-23T20:23:54.182 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.182 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.183 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:23:54.183 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.606912+0000", 2023-09-23T20:23:54.183 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00023875200000000001 2023-09-23T20:23:54.183 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.183 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.183 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:23:54.183 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.666781+0000", 2023-09-23T20:23:54.183 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.059868882999999998 2023-09-23T20:23:54.183 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.183 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.183 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:23:54.183 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.666808+0000", 2023-09-23T20:23:54.184 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.7307000000000001e-05 2023-09-23T20:23:54.184 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.184 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.184 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:23:54.184 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.666838+0000", 2023-09-23T20:23:54.184 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.9655999999999999e-05 2023-09-23T20:23:54.184 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:23:54.184 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:23:54.184 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:23:54.184 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.184 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.184 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:30078 5.7 5:e38ec444:::benchmark_data_smithi018_228451_object30077:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e119)", 2023-09-23T20:23:54.185 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:11.610121+0000", 2023-09-23T20:23:54.185 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 42.498435643000001, 2023-09-23T20:23:54.185 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.056720310000000003, 2023-09-23T20:23:54.185 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:23:54.185 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:23:54.185 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:23:54.185 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:23:54.185 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:23:54.185 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 30078 2023-09-23T20:23:54.185 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.185 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:23:54.185 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.185 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:23:54.186 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.610121+0000", 2023-09-23T20:23:54.186 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:23:54.186 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.186 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.186 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:23:54.186 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.610121+0000", 2023-09-23T20:23:54.186 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:23:54.186 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.186 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.186 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:23:54.186 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.610119+0000", 2023-09-23T20:23:54.186 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:23:54.186 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.187 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.187 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:23:54.187 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.610156+0000", 2023-09-23T20:23:54.187 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.6544000000000001e-05 2023-09-23T20:23:54.187 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.187 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.187 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:23:54.187 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.610181+0000", 2023-09-23T20:23:54.187 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.4777e-05 2023-09-23T20:23:54.187 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.187 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.187 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:23:54.187 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.610188+0000", 2023-09-23T20:23:54.188 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.9920000000000001e-06 2023-09-23T20:23:54.188 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.188 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.188 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:23:54.188 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.610246+0000", 2023-09-23T20:23:54.188 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.7806999999999997e-05 2023-09-23T20:23:54.188 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.188 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.188 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:23:54.188 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.610351+0000", 2023-09-23T20:23:54.188 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00010531699999999999 2023-09-23T20:23:54.188 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.189 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.189 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,1", 2023-09-23T20:23:54.189 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.610444+0000", 2023-09-23T20:23:54.189 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.2659999999999997e-05 2023-09-23T20:23:54.189 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.189 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.189 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:23:54.189 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.611917+0000", 2023-09-23T20:23:54.189 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0014739029999999999 2023-09-23T20:23:54.189 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.189 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.189 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:23:54.189 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.612567+0000", 2023-09-23T20:23:54.190 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00064996699999999999 2023-09-23T20:23:54.190 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.190 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.190 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:23:54.190 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.666777+0000", 2023-09-23T20:23:54.190 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.054209211 2023-09-23T20:23:54.190 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.190 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.190 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:23:54.190 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.666815+0000", 2023-09-23T20:23:54.190 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.8071999999999997e-05 2023-09-23T20:23:54.190 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.191 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.191 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:23:54.191 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.666841+0000", 2023-09-23T20:23:54.191 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.6270000000000001e-05 2023-09-23T20:23:54.191 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:23:54.191 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:23:54.191 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:23:54.191 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.191 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.191 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:30079 5.7 5:e4ff7c58:::benchmark_data_smithi018_228451_object30078:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e119)", 2023-09-23T20:23:54.191 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:11.610398+0000", 2023-09-23T20:23:54.191 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 42.49815804, 2023-09-23T20:23:54.192 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.056548500000000002, 2023-09-23T20:23:54.192 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:23:54.192 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:23:54.192 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:23:54.192 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:23:54.192 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:23:54.192 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 30079 2023-09-23T20:23:54.192 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.192 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:23:54.192 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.192 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:23:54.192 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.610398+0000", 2023-09-23T20:23:54.193 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:23:54.193 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.193 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.193 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:23:54.193 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.610398+0000", 2023-09-23T20:23:54.193 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:23:54.193 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.193 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.193 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:23:54.193 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.610398+0000", 2023-09-23T20:23:54.193 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:23:54.193 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.193 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.194 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:23:54.194 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.610468+0000", 2023-09-23T20:23:54.194 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.0433999999999997e-05 2023-09-23T20:23:54.194 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.194 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.194 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:23:54.194 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.610476+0000", 2023-09-23T20:23:54.194 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.4940000000000003e-06 2023-09-23T20:23:54.194 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.194 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.194 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:23:54.194 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.610483+0000", 2023-09-23T20:23:54.195 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.3820000000000001e-06 2023-09-23T20:23:54.195 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.195 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.195 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:23:54.195 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.611059+0000", 2023-09-23T20:23:54.195 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000576481 2023-09-23T20:23:54.195 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.195 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.195 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:23:54.195 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.611192+0000", 2023-09-23T20:23:54.195 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00013255600000000001 2023-09-23T20:23:54.195 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.195 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.196 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,1", 2023-09-23T20:23:54.196 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.611364+0000", 2023-09-23T20:23:54.196 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00017208100000000001 2023-09-23T20:23:54.196 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.196 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.196 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:23:54.196 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.612123+0000", 2023-09-23T20:23:54.196 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000758682 2023-09-23T20:23:54.196 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.196 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.196 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:23:54.196 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.612594+0000", 2023-09-23T20:23:54.197 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000471564 2023-09-23T20:23:54.197 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.197 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.197 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:23:54.197 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.666872+0000", 2023-09-23T20:23:54.197 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.054277893000000001 2023-09-23T20:23:54.197 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.197 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.197 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:23:54.197 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.666915+0000", 2023-09-23T20:23:54.197 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.2565000000000003e-05 2023-09-23T20:23:54.197 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.197 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.198 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:23:54.198 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.666947+0000", 2023-09-23T20:23:54.198 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.2107999999999998e-05 2023-09-23T20:23:54.198 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:23:54.198 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:23:54.198 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:23:54.198 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.198 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.198 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_repop(client.34791.0:63101 5.d e140/111 5:b74b02af:::benchmark_data_smithi018_228451_object63100:head v 140'3963, mlcod=140'3961)", 2023-09-23T20:23:54.198 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:45.210004+0000", 2023-09-23T20:23:54.198 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 8.8985522049999997, 2023-09-23T20:23:54.198 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.051712539000000002, 2023-09-23T20:23:54.199 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:23:54.199 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:23:54.199 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:23:54.199 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.199 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:23:54.199 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:45.210004+0000", 2023-09-23T20:23:54.199 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:23:54.199 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.199 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.199 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:23:54.199 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:45.210004+0000", 2023-09-23T20:23:54.199 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:23:54.199 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.200 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.200 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:23:54.200 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:45.210004+0000", 2023-09-23T20:23:54.200 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:23:54.200 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.200 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.200 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:23:54.200 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:45.210031+0000", 2023-09-23T20:23:54.200 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.6880999999999999e-05 2023-09-23T20:23:54.200 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.200 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.200 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:23:54.201 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:45.210037+0000", 2023-09-23T20:23:54.201 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.4960000000000001e-06 2023-09-23T20:23:54.201 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.201 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.201 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:23:54.201 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:45.210041+0000", 2023-09-23T20:23:54.201 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.0160000000000002e-06 2023-09-23T20:23:54.201 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.201 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.201 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:23:54.201 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:45.212236+0000", 2023-09-23T20:23:54.201 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0021946280000000001 2023-09-23T20:23:54.201 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.202 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.202 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:23:54.202 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:45.212256+0000", 2023-09-23T20:23:54.202 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.0231e-05 2023-09-23T20:23:54.202 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.202 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.202 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:23:54.202 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:45.261677+0000", 2023-09-23T20:23:54.202 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.049421101000000002 2023-09-23T20:23:54.202 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.202 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.202 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:23:54.202 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:45.261717+0000", 2023-09-23T20:23:54.203 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.9527000000000003e-05 2023-09-23T20:23:54.203 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:23:54.203 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:23:54.203 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:23:54.203 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.203 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.203 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_repop(client.34791.0:63107 5.e e140/111 5:70890534:::benchmark_data_smithi018_228451_object63106:head v 140'3841, mlcod=140'3838)", 2023-09-23T20:23:54.203 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:45.214650+0000", 2023-09-23T20:23:54.203 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 8.8939065419999999, 2023-09-23T20:23:54.203 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.054976848000000002, 2023-09-23T20:23:54.203 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:23:54.203 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:23:54.204 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:23:54.204 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.204 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:23:54.204 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:45.214650+0000", 2023-09-23T20:23:54.204 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:23:54.204 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.204 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.204 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:23:54.204 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:45.214650+0000", 2023-09-23T20:23:54.204 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:23:54.204 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.204 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.204 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:23:54.205 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:45.214649+0000", 2023-09-23T20:23:54.205 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:23:54.205 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.205 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.205 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:23:54.205 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:45.214671+0000", 2023-09-23T20:23:54.205 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.1818e-05 2023-09-23T20:23:54.205 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.205 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.205 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:23:54.205 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:45.214682+0000", 2023-09-23T20:23:54.205 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0913e-05 2023-09-23T20:23:54.206 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.206 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.206 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:23:54.206 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:45.214687+0000", 2023-09-23T20:23:54.206 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.5829999999999997e-06 2023-09-23T20:23:54.206 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.206 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.206 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:23:54.206 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:45.214726+0000", 2023-09-23T20:23:54.206 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.9116999999999998e-05 2023-09-23T20:23:54.206 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.206 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.206 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:23:54.207 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:45.214744+0000", 2023-09-23T20:23:54.207 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.7827000000000001e-05 2023-09-23T20:23:54.207 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.207 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.207 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:23:54.207 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:45.269585+0000", 2023-09-23T20:23:54.207 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.054840853000000002 2023-09-23T20:23:54.207 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.207 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.207 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:23:54.207 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:45.269627+0000", 2023-09-23T20:23:54.208 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.1569000000000001e-05 2023-09-23T20:23:54.208 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:23:54.208 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:23:54.208 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:23:54.208 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.208 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.208 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_repop(client.34791.0:63110 5.e e140/111 5:7f49d411:::benchmark_data_smithi018_228451_object63109:head v 140'3842, mlcod=140'3838)", 2023-09-23T20:23:54.208 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:45.216979+0000", 2023-09-23T20:23:54.208 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 8.8915777200000008, 2023-09-23T20:23:54.208 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.052676958000000003, 2023-09-23T20:23:54.208 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:23:54.208 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:23:54.208 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:23:54.209 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.209 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:23:54.209 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:45.216979+0000", 2023-09-23T20:23:54.209 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:23:54.209 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.209 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.209 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:23:54.209 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:45.216979+0000", 2023-09-23T20:23:54.209 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:23:54.209 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.209 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.209 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:23:54.209 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:45.216978+0000", 2023-09-23T20:23:54.210 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:23:54.210 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.210 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.210 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:23:54.210 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:45.217002+0000", 2023-09-23T20:23:54.210 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.3479e-05 2023-09-23T20:23:54.210 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.210 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.210 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:23:54.210 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:45.217010+0000", 2023-09-23T20:23:54.210 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.5529999999999993e-06 2023-09-23T20:23:54.210 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.211 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.211 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:23:54.211 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:45.217015+0000", 2023-09-23T20:23:54.211 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.7010000000000004e-06 2023-09-23T20:23:54.211 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.211 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.211 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:23:54.211 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:45.217054+0000", 2023-09-23T20:23:54.211 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.9186000000000001e-05 2023-09-23T20:23:54.211 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.211 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.211 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:23:54.212 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:45.217073+0000", 2023-09-23T20:23:54.212 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.8584999999999999e-05 2023-09-23T20:23:54.212 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.212 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.212 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:23:54.212 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:45.269631+0000", 2023-09-23T20:23:54.212 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.052558768999999998 2023-09-23T20:23:54.212 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.212 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.212 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:23:54.212 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:45.269656+0000", 2023-09-23T20:23:54.212 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.4193999999999999e-05 2023-09-23T20:23:54.212 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:23:54.213 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:23:54.213 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:23:54.213 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.213 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.213 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:63893 5.7 5:e3b2a1bf:::benchmark_data_smithi018_228451_object63892:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e140)", 2023-09-23T20:23:54.213 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:45.877301+0000", 2023-09-23T20:23:54.213 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 8.2312557759999994, 2023-09-23T20:23:54.213 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.064876473000000004, 2023-09-23T20:23:54.213 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:23:54.213 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:23:54.213 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:23:54.213 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:23:54.214 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:23:54.214 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 63893 2023-09-23T20:23:54.214 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.214 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:23:54.214 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.214 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:23:54.214 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:45.877301+0000", 2023-09-23T20:23:54.214 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:23:54.214 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.214 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.214 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:23:54.214 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:45.877301+0000", 2023-09-23T20:23:54.214 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:23:54.215 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.215 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.215 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:23:54.215 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:45.877299+0000", 2023-09-23T20:23:54.215 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:23:54.215 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.215 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.215 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:23:54.215 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:45.877356+0000", 2023-09-23T20:23:54.215 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.6518999999999999e-05 2023-09-23T20:23:54.215 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.215 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.216 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:23:54.216 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:45.877365+0000", 2023-09-23T20:23:54.216 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.0650000000000005e-06 2023-09-23T20:23:54.216 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.216 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.216 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:23:54.216 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:45.877369+0000", 2023-09-23T20:23:54.216 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.5460000000000002e-06 2023-09-23T20:23:54.216 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.216 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.216 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:23:54.216 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:45.877399+0000", 2023-09-23T20:23:54.217 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.9981999999999999e-05 2023-09-23T20:23:54.217 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.217 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.217 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:23:54.217 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:45.877485+0000", 2023-09-23T20:23:54.217 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.5821000000000004e-05 2023-09-23T20:23:54.217 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.217 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.217 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,1", 2023-09-23T20:23:54.217 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:45.877574+0000", 2023-09-23T20:23:54.217 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.8764999999999995e-05 2023-09-23T20:23:54.217 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.217 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.218 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:23:54.218 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:45.880356+0000", 2023-09-23T20:23:54.218 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0027823689999999998 2023-09-23T20:23:54.218 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.218 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.218 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:23:54.218 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:45.881924+0000", 2023-09-23T20:23:54.218 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0015682820000000001 2023-09-23T20:23:54.218 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.218 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.218 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:23:54.218 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:45.942085+0000", 2023-09-23T20:23:54.219 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.060160080999999997 2023-09-23T20:23:54.219 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.219 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.219 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:23:54.219 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:45.942135+0000", 2023-09-23T20:23:54.219 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.0565999999999997e-05 2023-09-23T20:23:54.219 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.219 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.219 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:23:54.219 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:45.942177+0000", 2023-09-23T20:23:54.219 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.1903000000000002e-05 2023-09-23T20:23:54.219 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:23:54.220 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:23:54.220 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:23:54.220 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.220 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.220 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:63900 5.7 5:e29fa24d:::benchmark_data_smithi018_228451_object63899:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e140)", 2023-09-23T20:23:54.220 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:45.881870+0000", 2023-09-23T20:23:54.220 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 8.2266868160000008, 2023-09-23T20:23:54.220 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.061485024999999999, 2023-09-23T20:23:54.220 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:23:54.220 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:23:54.220 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:23:54.220 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:23:54.220 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:23:54.221 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 63900 2023-09-23T20:23:54.221 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.221 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:23:54.221 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.221 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:23:54.221 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:45.881870+0000", 2023-09-23T20:23:54.221 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:23:54.221 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.221 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.221 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:23:54.221 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:45.881870+0000", 2023-09-23T20:23:54.221 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:23:54.222 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.222 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.222 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:23:54.222 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:45.881868+0000", 2023-09-23T20:23:54.222 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:23:54.222 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.222 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.222 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:23:54.222 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:45.881912+0000", 2023-09-23T20:23:54.222 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.3996999999999999e-05 2023-09-23T20:23:54.222 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.222 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.222 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:23:54.223 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:45.881923+0000", 2023-09-23T20:23:54.223 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0946e-05 2023-09-23T20:23:54.223 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.223 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.223 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:23:54.223 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:45.881929+0000", 2023-09-23T20:23:54.223 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.9460000000000003e-06 2023-09-23T20:23:54.223 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.223 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.223 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:23:54.223 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:45.881966+0000", 2023-09-23T20:23:54.223 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.7222999999999999e-05 2023-09-23T20:23:54.224 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.224 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.224 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:23:54.224 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:45.938939+0000", 2023-09-23T20:23:54.224 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.056972415999999998 2023-09-23T20:23:54.224 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.224 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.224 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,1", 2023-09-23T20:23:54.224 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:45.939129+0000", 2023-09-23T20:23:54.224 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00019020800000000001 2023-09-23T20:23:54.224 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.224 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.225 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:23:54.225 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:45.942197+0000", 2023-09-23T20:23:54.225 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0030685729999999998 2023-09-23T20:23:54.225 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.225 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.225 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:23:54.225 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:45.942228+0000", 2023-09-23T20:23:54.225 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.0616e-05 2023-09-23T20:23:54.225 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.225 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.225 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:23:54.225 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:45.943260+0000", 2023-09-23T20:23:54.225 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.001031836 2023-09-23T20:23:54.226 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.226 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.226 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:23:54.226 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:45.943311+0000", 2023-09-23T20:23:54.226 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.1119999999999998e-05 2023-09-23T20:23:54.226 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.226 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.226 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:23:54.226 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:45.943355+0000", 2023-09-23T20:23:54.226 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.3643000000000002e-05 2023-09-23T20:23:54.226 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:23:54.226 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:23:54.226 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:23:54.227 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.227 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.227 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:63901 5.7 5:e24b0eca:::benchmark_data_smithi018_228451_object63900:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e140)", 2023-09-23T20:23:54.227 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:45.881947+0000", 2023-09-23T20:23:54.227 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 8.2266088669999995, 2023-09-23T20:23:54.227 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.063658610000000004, 2023-09-23T20:23:54.227 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:23:54.227 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:23:54.227 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:23:54.227 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:23:54.227 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:23:54.227 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 63901 2023-09-23T20:23:54.228 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.228 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:23:54.228 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.228 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:23:54.228 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:45.881947+0000", 2023-09-23T20:23:54.228 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:23:54.228 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.228 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.228 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:23:54.228 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:45.881947+0000", 2023-09-23T20:23:54.228 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:23:54.228 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.229 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.229 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:23:54.229 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:45.881947+0000", 2023-09-23T20:23:54.229 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:23:54.229 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.229 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.229 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:23:54.229 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:45.881983+0000", 2023-09-23T20:23:54.229 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.6298999999999999e-05 2023-09-23T20:23:54.229 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.229 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.229 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:23:54.229 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:45.881990+0000", 2023-09-23T20:23:54.230 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.7150000000000001e-06 2023-09-23T20:23:54.230 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.230 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.230 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:23:54.230 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:45.881995+0000", 2023-09-23T20:23:54.230 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.8450000000000002e-06 2023-09-23T20:23:54.230 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.230 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.230 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:23:54.230 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:45.940553+0000", 2023-09-23T20:23:54.230 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.058558336000000002 2023-09-23T20:23:54.230 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.231 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.231 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:23:54.231 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:45.940682+0000", 2023-09-23T20:23:54.231 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00012888099999999999 2023-09-23T20:23:54.231 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.231 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.231 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,1", 2023-09-23T20:23:54.231 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:45.940829+0000", 2023-09-23T20:23:54.231 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00014673400000000001 2023-09-23T20:23:54.231 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.231 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.231 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:23:54.231 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:45.944875+0000", 2023-09-23T20:23:54.232 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0040465680000000004 2023-09-23T20:23:54.232 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.232 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.232 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:23:54.232 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:45.944929+0000", 2023-09-23T20:23:54.232 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.3442000000000001e-05 2023-09-23T20:23:54.232 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.232 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.232 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:23:54.232 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:45.945538+0000", 2023-09-23T20:23:54.232 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00060949500000000005 2023-09-23T20:23:54.232 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.233 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.233 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:23:54.233 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:45.945572+0000", 2023-09-23T20:23:54.233 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.4109999999999997e-05 2023-09-23T20:23:54.233 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.233 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.233 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:23:54.233 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:45.945606+0000", 2023-09-23T20:23:54.233 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.3902000000000002e-05 2023-09-23T20:23:54.233 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:23:54.233 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:23:54.233 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:23:54.233 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.234 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.234 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_repop(client.34791.0:63897 5.3 e140/118 5:c6316350:::benchmark_data_smithi018_228451_object63896:head v 140'2918, mlcod=140'2916)", 2023-09-23T20:23:54.234 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:45.882405+0000", 2023-09-23T20:23:54.234 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 8.2261510839999996, 2023-09-23T20:23:54.234 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.060728074, 2023-09-23T20:23:54.234 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:23:54.234 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:23:54.234 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:23:54.234 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.234 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:23:54.234 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:45.882405+0000", 2023-09-23T20:23:54.234 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:23:54.235 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.235 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.235 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:23:54.235 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:45.882405+0000", 2023-09-23T20:23:54.235 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:23:54.235 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.235 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.235 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:23:54.235 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:45.882405+0000", 2023-09-23T20:23:54.235 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:23:54.235 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.235 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.236 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:23:54.236 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:45.882503+0000", 2023-09-23T20:23:54.236 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.8387000000000005e-05 2023-09-23T20:23:54.236 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.236 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.236 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:23:54.236 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:45.938686+0000", 2023-09-23T20:23:54.236 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.056183134000000003 2023-09-23T20:23:54.236 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.236 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.236 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:23:54.236 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:45.938713+0000", 2023-09-23T20:23:54.237 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.6774000000000001e-05 2023-09-23T20:23:54.237 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.237 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.237 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:23:54.237 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:45.939221+0000", 2023-09-23T20:23:54.237 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00050845399999999998 2023-09-23T20:23:54.237 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.237 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.237 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:23:54.237 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:45.939242+0000", 2023-09-23T20:23:54.237 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.073e-05 2023-09-23T20:23:54.237 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.237 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.238 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:23:54.238 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:45.943104+0000", 2023-09-23T20:23:54.238 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0038622779999999998 2023-09-23T20:23:54.238 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.238 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.238 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:23:54.238 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:45.943133+0000", 2023-09-23T20:23:54.238 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.9006e-05 2023-09-23T20:23:54.238 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:23:54.238 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:23:54.238 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:23:54.238 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.238 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.239 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_repop(client.34791.0:63904 5.d e140/111 5:bf155307:::benchmark_data_smithi018_228451_object63903:head v 140'4007, mlcod=140'4005)", 2023-09-23T20:23:54.239 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:45.885513+0000", 2023-09-23T20:23:54.239 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 8.2230431310000007, 2023-09-23T20:23:54.239 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.058035059, 2023-09-23T20:23:54.239 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:23:54.239 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:23:54.239 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:23:54.239 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.239 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:23:54.239 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:45.885513+0000", 2023-09-23T20:23:54.239 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:23:54.239 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.240 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.240 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:23:54.240 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:45.885513+0000", 2023-09-23T20:23:54.240 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:23:54.240 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.240 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.240 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:23:54.240 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:45.885512+0000", 2023-09-23T20:23:54.240 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:23:54.240 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.240 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.240 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:23:54.241 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:45.885639+0000", 2023-09-23T20:23:54.241 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000126549 2023-09-23T20:23:54.241 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.241 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.241 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:23:54.241 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:45.938768+0000", 2023-09-23T20:23:54.241 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.053129786999999998 2023-09-23T20:23:54.241 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.241 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.241 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:23:54.241 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:45.938781+0000", 2023-09-23T20:23:54.241 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.2248e-05 2023-09-23T20:23:54.241 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.242 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.242 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:23:54.242 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:45.939059+0000", 2023-09-23T20:23:54.242 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00027843799999999999 2023-09-23T20:23:54.242 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.242 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.242 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:23:54.242 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:45.939085+0000", 2023-09-23T20:23:54.242 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.5661e-05 2023-09-23T20:23:54.242 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.242 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.242 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:23:54.242 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:45.943516+0000", 2023-09-23T20:23:54.243 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0044310929999999997 2023-09-23T20:23:54.243 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:23:54.243 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:23:54.243 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:23:54.243 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:45.943548+0000", 2023-09-23T20:23:54.243 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.2382000000000003e-05 2023-09-23T20:23:54.243 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:23:54.243 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:23:54.243 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:23:54.243 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:23:54.243 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:23:54.243 INFO:teuthology.orchestra.run.smithi149.stdout:} 2023-09-23T20:23:54.244 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:23:54.244 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:53 smithi018 ceph-mon[179647]: 5.13 scrub starts 2023-09-23T20:23:54.244 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:53 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3053774062' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:23:54.245 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:53 smithi018 ceph-mon[184144]: 5.13 scrub starts 2023-09-23T20:23:54.245 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:53 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3053774062' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:23:54.245 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:23:54.245 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:23:54.245 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:23:54.245 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:23:54.246 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:23:54.246 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:23:54.246 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:23:54.246 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:23:54.246 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:23:54.286 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:54.472 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T20:23:54.476 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 56 16 74590 74574 83.1927 62.5 0.00284893 0.0119433 2023-09-23T20:23:54.492 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:23:54.495 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:54.495 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:23:54.596 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:23:54.625 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:54.661 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:23:54.663 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:54.664 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:23:54.765 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:23:54.791 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:54.827 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:23:54.830 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:54.830 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:23:54.913 INFO:teuthology.orchestra.run.smithi018.stderr:noscrub is set 2023-09-23T20:23:54.931 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:23:54.996 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:55.075 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:23:55.086 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:54 smithi149 ceph-mon[170394]: pgmap v292: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 56 MiB/s wr, 899 op/s; 3648/211971 objects degraded (1.721%); 104/211971 objects misplaced (0.049%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T20:23:55.086 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:54 smithi149 ceph-mon[170394]: osdmap e143: 8 total, 8 up, 8 in 2023-09-23T20:23:55.086 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:54 smithi149 ceph-mon[170394]: failed to encode map e143 with expected crc 2023-09-23T20:23:55.086 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:54 smithi149 ceph-mon[170394]: failed to encode map e143 with expected crc 2023-09-23T20:23:55.086 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:54 smithi149 ceph-mon[170394]: failed to encode map e143 with expected crc 2023-09-23T20:23:55.086 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:54 smithi149 ceph-mon[170394]: failed to encode map e143 with expected crc 2023-09-23T20:23:55.087 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:54 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1431898118' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:23:55.087 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:54 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:23:55.088 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:55.088 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:23:55.148 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:23:55.190 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:23:55.259 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:55.280 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:23:54 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:23:54] "GET /metrics HTTP/1.1" 200 36477 "" "Prometheus/2.43.0" 2023-09-23T20:23:55.281 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:54 smithi018 ceph-mon[179647]: pgmap v292: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 56 MiB/s wr, 899 op/s; 3648/211971 objects degraded (1.721%); 104/211971 objects misplaced (0.049%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T20:23:55.281 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:54 smithi018 ceph-mon[179647]: osdmap e143: 8 total, 8 up, 8 in 2023-09-23T20:23:55.281 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:54 smithi018 ceph-mon[179647]: failed to encode map e143 with expected crc 2023-09-23T20:23:55.281 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:54 smithi018 ceph-mon[179647]: failed to encode map e143 with expected crc 2023-09-23T20:23:55.281 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:54 smithi018 ceph-mon[179647]: failed to encode map e143 with expected crc 2023-09-23T20:23:55.281 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:54 smithi018 ceph-mon[179647]: failed to encode map e143 with expected crc 2023-09-23T20:23:55.281 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:54 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1431898118' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:23:55.281 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:54 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:23:55.281 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:54 smithi018 ceph-mon[184144]: pgmap v292: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 56 MiB/s wr, 899 op/s; 3648/211971 objects degraded (1.721%); 104/211971 objects misplaced (0.049%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T20:23:55.282 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:54 smithi018 ceph-mon[184144]: osdmap e143: 8 total, 8 up, 8 in 2023-09-23T20:23:55.282 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:54 smithi018 ceph-mon[184144]: failed to encode map e143 with expected crc 2023-09-23T20:23:55.282 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:54 smithi018 ceph-mon[184144]: failed to encode map e143 with expected crc 2023-09-23T20:23:55.282 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:54 smithi018 ceph-mon[184144]: failed to encode map e143 with expected crc 2023-09-23T20:23:55.282 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:54 smithi018 ceph-mon[184144]: failed to encode map e143 with expected crc 2023-09-23T20:23:55.282 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:54 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1431898118' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:23:55.282 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:54 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:23:55.333 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:23:55.338 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:55.339 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:23:55.439 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:23:55.477 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 57 16 75323 75307 82.537 45.8125 0.0115143 0.0121111 2023-09-23T20:23:55.493 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:55.556 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:23:55.560 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:55.561 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:23:55.662 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:23:55.705 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:55.772 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:23:55.776 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:55.776 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:23:55.877 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:23:55.925 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:55.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:55 smithi149 ceph-mon[170394]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:23:55.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:55 smithi149 ceph-mon[170394]: failed to encode map e144 with expected crc 2023-09-23T20:23:55.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:55 smithi149 ceph-mon[170394]: failed to encode map e144 with expected crc 2023-09-23T20:23:55.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:55 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:23:55.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:55 smithi149 ceph-mon[170394]: osdmap e144: 8 total, 8 up, 8 in 2023-09-23T20:23:55.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:55 smithi149 ceph-mon[170394]: failed to encode map e144 with expected crc 2023-09-23T20:23:55.968 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:55 smithi149 ceph-mon[170394]: failed to encode map e144 with expected crc 2023-09-23T20:23:55.968 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:55 smithi149 ceph-mon[170394]: failed to encode map e144 with expected crc 2023-09-23T20:23:55.968 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:55 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "24"}]: dispatch 2023-09-23T20:23:55.988 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:23:55.992 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:55.992 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:23:56.093 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:23:56.140 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:56.189 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:23:56.193 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:56.193 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:55 smithi018 ceph-mon[179647]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:23:56.193 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:55 smithi018 ceph-mon[179647]: failed to encode map e144 with expected crc 2023-09-23T20:23:56.194 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:55 smithi018 ceph-mon[179647]: failed to encode map e144 with expected crc 2023-09-23T20:23:56.194 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:55 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:23:56.194 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:55 smithi018 ceph-mon[179647]: osdmap e144: 8 total, 8 up, 8 in 2023-09-23T20:23:56.194 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:55 smithi018 ceph-mon[179647]: failed to encode map e144 with expected crc 2023-09-23T20:23:56.194 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:55 smithi018 ceph-mon[179647]: failed to encode map e144 with expected crc 2023-09-23T20:23:56.194 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:55 smithi018 ceph-mon[179647]: failed to encode map e144 with expected crc 2023-09-23T20:23:56.195 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:55 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "24"}]: dispatch 2023-09-23T20:23:56.195 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:55 smithi018 ceph-mon[184144]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:23:56.195 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:55 smithi018 ceph-mon[184144]: failed to encode map e144 with expected crc 2023-09-23T20:23:56.195 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:55 smithi018 ceph-mon[184144]: failed to encode map e144 with expected crc 2023-09-23T20:23:56.196 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:55 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:23:56.196 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:55 smithi018 ceph-mon[184144]: osdmap e144: 8 total, 8 up, 8 in 2023-09-23T20:23:56.196 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:55 smithi018 ceph-mon[184144]: failed to encode map e144 with expected crc 2023-09-23T20:23:56.196 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:55 smithi018 ceph-mon[184144]: failed to encode map e144 with expected crc 2023-09-23T20:23:56.196 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:55 smithi018 ceph-mon[184144]: failed to encode map e144 with expected crc 2023-09-23T20:23:56.197 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:55 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "24"}]: dispatch 2023-09-23T20:23:56.197 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:23:56.298 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:23:56.347 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:56.389 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:23:56.392 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:56.392 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:23:56.477 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 58 16 76576 76560 82.4641 78.3125 0.00215588 0.0120724 2023-09-23T20:23:56.493 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:23:56.529 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:56.555 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:23:56.581 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:23:56.588 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:56.588 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:23:56.689 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:23:56.690 INFO:teuthology.orchestra.run.smithi149.stdout:{ 2023-09-23T20:23:56.690 INFO:teuthology.orchestra.run.smithi149.stdout: "ops": [], 2023-09-23T20:23:56.690 INFO:teuthology.orchestra.run.smithi149.stdout: "num_ops": 0 2023-09-23T20:23:56.690 INFO:teuthology.orchestra.run.smithi149.stdout:} 2023-09-23T20:23:56.717 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:56.751 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:23:56.754 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:56.755 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:23:56.855 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:23:56.883 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:56.926 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:23:56.930 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:56.931 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:23:56.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:56 smithi149 ceph-mon[170394]: pgmap v295: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 97 active+clean; 4.5 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 73 MiB/s wr, 1.16k op/s; 3338/221208 objects misplaced (1.509%) 2023-09-23T20:23:56.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:56 smithi149 ceph-mon[170394]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3648/211971 objects degraded (1.721%), 1 pg degraded) 2023-09-23T20:23:56.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:56 smithi149 ceph-mon[170394]: failed to encode map e145 with expected crc 2023-09-23T20:23:56.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:56 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "24"}]': finished 2023-09-23T20:23:56.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:56 smithi149 ceph-mon[170394]: osdmap e145: 8 total, 8 up, 8 in 2023-09-23T20:23:57.032 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:23:57.059 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:57.099 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T20:23:57.099 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:23:57.101 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:57.101 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:23:57.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:56 smithi018 ceph-mon[179647]: pgmap v295: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 97 active+clean; 4.5 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 73 MiB/s wr, 1.16k op/s; 3338/221208 objects misplaced (1.509%) 2023-09-23T20:23:57.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:56 smithi018 ceph-mon[179647]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3648/211971 objects degraded (1.721%), 1 pg degraded) 2023-09-23T20:23:57.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:56 smithi018 ceph-mon[179647]: failed to encode map e145 with expected crc 2023-09-23T20:23:57.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:56 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "24"}]': finished 2023-09-23T20:23:57.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:56 smithi018 ceph-mon[179647]: osdmap e145: 8 total, 8 up, 8 in 2023-09-23T20:23:57.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:56 smithi018 ceph-mon[184144]: pgmap v295: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 97 active+clean; 4.5 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 73 MiB/s wr, 1.16k op/s; 3338/221208 objects misplaced (1.509%) 2023-09-23T20:23:57.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:56 smithi018 ceph-mon[184144]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3648/211971 objects degraded (1.721%), 1 pg degraded) 2023-09-23T20:23:57.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:56 smithi018 ceph-mon[184144]: failed to encode map e145 with expected crc 2023-09-23T20:23:57.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:56 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "24"}]': finished 2023-09-23T20:23:57.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:56 smithi018 ceph-mon[184144]: osdmap e145: 8 total, 8 up, 8 in 2023-09-23T20:23:57.196 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:23:57.203 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:23:57.230 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:57.264 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:23:57.267 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:57.267 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:23:57.368 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:23:57.381 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T20:23:57.395 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:57.430 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:23:57.432 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:57.433 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:23:57.477 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 59 16 76662 76646 81.1579 5.375 0.0021889 0.0120626 2023-09-23T20:23:57.533 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:23:57.562 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:57.759 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:23:58.211 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:57 smithi149 ceph-mon[170394]: osdmap e146: 8 total, 8 up, 8 in 2023-09-23T20:23:58.211 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:57 smithi149 ceph-mon[170394]: failed to encode map e146 with expected crc 2023-09-23T20:23:58.211 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:57 smithi149 ceph-mon[170394]: failed to encode map e146 with expected crc 2023-09-23T20:23:58.211 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:57 smithi149 ceph-mon[170394]: failed to encode map e146 with expected crc 2023-09-23T20:23:58.212 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:57 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "25"}]: dispatch 2023-09-23T20:23:58.315 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:23:58.318 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:58.319 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:23:58.319 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:57 smithi018 ceph-mon[184144]: osdmap e146: 8 total, 8 up, 8 in 2023-09-23T20:23:58.319 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:57 smithi018 ceph-mon[184144]: failed to encode map e146 with expected crc 2023-09-23T20:23:58.319 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:57 smithi018 ceph-mon[184144]: failed to encode map e146 with expected crc 2023-09-23T20:23:58.319 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:57 smithi018 ceph-mon[184144]: failed to encode map e146 with expected crc 2023-09-23T20:23:58.320 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:57 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "25"}]: dispatch 2023-09-23T20:23:58.320 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:57 smithi018 ceph-mon[179647]: osdmap e146: 8 total, 8 up, 8 in 2023-09-23T20:23:58.320 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:57 smithi018 ceph-mon[179647]: failed to encode map e146 with expected crc 2023-09-23T20:23:58.320 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:57 smithi018 ceph-mon[179647]: failed to encode map e146 with expected crc 2023-09-23T20:23:58.320 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:57 smithi018 ceph-mon[179647]: failed to encode map e146 with expected crc 2023-09-23T20:23:58.320 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:57 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "25"}]: dispatch 2023-09-23T20:23:58.330 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-23T20:23:58.330 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:23:58.331 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '110', 'type': 'test-reweight-by-utilization'} 2023-09-23T20:23:58.331 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd test-reweight-by-utilization 110 3.0 2023-09-23T20:23:58.420 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:23:58.461 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:58.477 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:2023-09-23T20:23:58.477872+0000 min lat: 0.00112599 max lat: 2.10781 avg lat: 0.0124264 2023-09-23T20:23:58.477 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T20:23:58.477 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 60 16 77147 77131 80.3107 30.3125 0.00647308 0.0124264 2023-09-23T20:23:58.723 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:23:58.730 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:58.731 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:23:58.831 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:23:58.871 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:58.919 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:23:58.923 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:58.923 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:23:59.024 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:23:59.050 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:58 smithi018 ceph-mon[184144]: failed to encode map e146 with expected crc 2023-09-23T20:23:59.050 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:58 smithi018 ceph-mon[184144]: pgmap v298: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 97 active+clean; 4.6 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s; 3338/225642 objects misplaced (1.479%); 4.1 MiB/s, 64 objects/s recovering 2023-09-23T20:23:59.050 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:58 smithi018 ceph-mon[184144]: failed to encode map e147 with expected crc 2023-09-23T20:23:59.051 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:58 smithi018 ceph-mon[184144]: failed to encode map e147 with expected crc 2023-09-23T20:23:59.051 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:58 smithi018 ceph-mon[184144]: failed to encode map e147 with expected crc 2023-09-23T20:23:59.051 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:58 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "25"}]': finished 2023-09-23T20:23:59.051 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:58 smithi018 ceph-mon[184144]: osdmap e147: 8 total, 8 up, 8 in 2023-09-23T20:23:59.051 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:58 smithi018 ceph-mon[184144]: failed to encode map e147 with expected crc 2023-09-23T20:23:59.051 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:58 smithi018 ceph-mon[184144]: failed to encode map e147 with expected crc 2023-09-23T20:23:59.051 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:58 smithi018 ceph-mon[179647]: failed to encode map e146 with expected crc 2023-09-23T20:23:59.051 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:58 smithi018 ceph-mon[179647]: pgmap v298: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 97 active+clean; 4.6 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s; 3338/225642 objects misplaced (1.479%); 4.1 MiB/s, 64 objects/s recovering 2023-09-23T20:23:59.051 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:58 smithi018 ceph-mon[179647]: failed to encode map e147 with expected crc 2023-09-23T20:23:59.052 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:58 smithi018 ceph-mon[179647]: failed to encode map e147 with expected crc 2023-09-23T20:23:59.052 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:58 smithi018 ceph-mon[179647]: failed to encode map e147 with expected crc 2023-09-23T20:23:59.052 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:58 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "25"}]': finished 2023-09-23T20:23:59.052 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:58 smithi018 ceph-mon[179647]: osdmap e147: 8 total, 8 up, 8 in 2023-09-23T20:23:59.052 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:58 smithi018 ceph-mon[179647]: failed to encode map e147 with expected crc 2023-09-23T20:23:59.052 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:58 smithi018 ceph-mon[179647]: failed to encode map e147 with expected crc 2023-09-23T20:23:59.053 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:59.125 INFO:teuthology.orchestra.run.smithi149.stdout:{ 2023-09-23T20:23:59.125 INFO:teuthology.orchestra.run.smithi149.stdout: "ops": [], 2023-09-23T20:23:59.125 INFO:teuthology.orchestra.run.smithi149.stdout: "complaint_time": 30, 2023-09-23T20:23:59.125 INFO:teuthology.orchestra.run.smithi149.stdout: "num_blocked_ops": 0 2023-09-23T20:23:59.125 INFO:teuthology.orchestra.run.smithi149.stdout:} 2023-09-23T20:23:59.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:58 smithi149 ceph-mon[170394]: failed to encode map e146 with expected crc 2023-09-23T20:23:59.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:58 smithi149 ceph-mon[170394]: pgmap v298: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 97 active+clean; 4.6 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s; 3338/225642 objects misplaced (1.479%); 4.1 MiB/s, 64 objects/s recovering 2023-09-23T20:23:59.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:58 smithi149 ceph-mon[170394]: failed to encode map e147 with expected crc 2023-09-23T20:23:59.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:58 smithi149 ceph-mon[170394]: failed to encode map e147 with expected crc 2023-09-23T20:23:59.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:58 smithi149 ceph-mon[170394]: failed to encode map e147 with expected crc 2023-09-23T20:23:59.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:58 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num_actual", "val": "25"}]': finished 2023-09-23T20:23:59.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:58 smithi149 ceph-mon[170394]: osdmap e147: 8 total, 8 up, 8 in 2023-09-23T20:23:59.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:58 smithi149 ceph-mon[170394]: failed to encode map e147 with expected crc 2023-09-23T20:23:59.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:58 smithi149 ceph-mon[170394]: failed to encode map e147 with expected crc 2023-09-23T20:23:59.391 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:23:59.394 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:59.394 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:23:59.478 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 61 16 77501 77485 79.3563 22.125 0.00380807 0.0123932 2023-09-23T20:23:59.495 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:23:59.526 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:59.544 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T20:23:59.613 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:23:59.615 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:59.616 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:23:59.716 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:23:59.743 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:59.782 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:23:59.785 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:59.786 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:23:59.886 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:23:59.913 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:23:59.922 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:23:59.922 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:23:59.952 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:23:59.956 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:23:59.956 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:24:00.057 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:24:00.087 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:00.171 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:24:00.174 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:00.175 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:24:00.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:59 smithi149 ceph-mon[170394]: osdmap e148: 8 total, 8 up, 8 in 2023-09-23T20:24:00.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:59 smithi149 ceph-mon[170394]: failed to encode map e148 with expected crc 2023-09-23T20:24:00.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:59 smithi149 ceph-mon[170394]: failed to encode map e148 with expected crc 2023-09-23T20:24:00.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:59 smithi149 ceph-mon[170394]: failed to encode map e148 with expected crc 2023-09-23T20:24:00.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:59 smithi149 ceph-mon[170394]: failed to encode map e148 with expected crc 2023-09-23T20:24:00.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:23:59 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:24:00.222 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:24:00.275 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:24:00.301 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:59 smithi018 ceph-mon[184144]: osdmap e148: 8 total, 8 up, 8 in 2023-09-23T20:24:00.301 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:59 smithi018 ceph-mon[184144]: failed to encode map e148 with expected crc 2023-09-23T20:24:00.302 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:59 smithi018 ceph-mon[184144]: failed to encode map e148 with expected crc 2023-09-23T20:24:00.302 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:59 smithi018 ceph-mon[184144]: failed to encode map e148 with expected crc 2023-09-23T20:24:00.302 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:59 smithi018 ceph-mon[184144]: failed to encode map e148 with expected crc 2023-09-23T20:24:00.302 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:23:59 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:24:00.302 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:59 smithi018 ceph-mon[179647]: osdmap e148: 8 total, 8 up, 8 in 2023-09-23T20:24:00.302 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:59 smithi018 ceph-mon[179647]: failed to encode map e148 with expected crc 2023-09-23T20:24:00.302 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:59 smithi018 ceph-mon[179647]: failed to encode map e148 with expected crc 2023-09-23T20:24:00.302 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:59 smithi018 ceph-mon[179647]: failed to encode map e148 with expected crc 2023-09-23T20:24:00.302 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:59 smithi018 ceph-mon[179647]: failed to encode map e148 with expected crc 2023-09-23T20:24:00.302 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:23:59 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:24:00.304 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:00.405 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:24:00.407 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:00.408 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:24:00.478 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 62 16 77501 77485 78.0768 0 - 0.0123932 2023-09-23T20:24:00.509 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:24:00.538 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:00.623 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:24:00.627 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:00.627 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:24:00.728 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:24:00.757 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:00.950 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:24:00.955 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:00.956 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:24:01.002 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:24:01.002 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:24:01.002 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:24:01.003 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:24:01.003 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:24:01.003 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:24:01.043 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:24:01.056 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:24:01.079 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:24:01.085 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:24:01.085 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:24:01.086 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:24:01.086 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:24:01.086 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:24:01.086 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:24:01.086 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:24:01.086 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:24:01.119 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:00 smithi018 ceph-mon[179647]: pgmap v301: 99 pgs: 1 remapped+peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 96 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 33 MiB/s wr, 525 op/s; 3338/227517 objects misplaced (1.467%) 2023-09-23T20:24:01.119 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:00 smithi018 ceph-mon[179647]: failed to encode map e149 with expected crc 2023-09-23T20:24:01.119 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:00 smithi018 ceph-mon[179647]: osdmap e149: 8 total, 8 up, 8 in 2023-09-23T20:24:01.119 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:00 smithi018 ceph-mon[179647]: failed to encode map e149 with expected crc 2023-09-23T20:24:01.119 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:00 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1475263082' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:24:01.120 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:00 smithi018 ceph-mon[184144]: pgmap v301: 99 pgs: 1 remapped+peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 96 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 33 MiB/s wr, 525 op/s; 3338/227517 objects misplaced (1.467%) 2023-09-23T20:24:01.120 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:00 smithi018 ceph-mon[184144]: failed to encode map e149 with expected crc 2023-09-23T20:24:01.120 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:00 smithi018 ceph-mon[184144]: osdmap e149: 8 total, 8 up, 8 in 2023-09-23T20:24:01.120 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:00 smithi018 ceph-mon[184144]: failed to encode map e149 with expected crc 2023-09-23T20:24:01.120 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:00 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1475263082' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:24:01.129 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:01.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:00 smithi149 ceph-mon[170394]: pgmap v301: 99 pgs: 1 remapped+peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 96 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 33 MiB/s wr, 525 op/s; 3338/227517 objects misplaced (1.467%) 2023-09-23T20:24:01.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:00 smithi149 ceph-mon[170394]: failed to encode map e149 with expected crc 2023-09-23T20:24:01.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:00 smithi149 ceph-mon[170394]: osdmap e149: 8 total, 8 up, 8 in 2023-09-23T20:24:01.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:00 smithi149 ceph-mon[170394]: failed to encode map e149 with expected crc 2023-09-23T20:24:01.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:00 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1475263082' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:24:01.447 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:24:01.450 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:01.451 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:24:01.478 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 63 16 78186 78170 77.5172 21.4062 0.00886341 0.0128959 2023-09-23T20:24:01.551 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:24:01.619 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:01.622 INFO:teuthology.orchestra.run.smithi149.stdout:ERROR: (22) Invalid argument 2023-09-23T20:24:01.622 INFO:teuthology.orchestra.run.smithi149.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:24:01.850 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:24:01.854 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:01.855 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:24:01.956 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:24:02.037 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:02.071 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T20:24:02.111 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:24:02.115 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:02.116 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:24:02.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:01 smithi018 ceph-mon[179647]: failed to encode map e149 with expected crc 2023-09-23T20:24:02.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:01 smithi018 ceph-mon[179647]: failed to encode map e149 with expected crc 2023-09-23T20:24:02.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:01 smithi018 ceph-mon[184144]: failed to encode map e149 with expected crc 2023-09-23T20:24:02.144 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:01 smithi018 ceph-mon[184144]: failed to encode map e149 with expected crc 2023-09-23T20:24:02.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:01 smithi149 ceph-mon[170394]: failed to encode map e149 with expected crc 2023-09-23T20:24:02.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:01 smithi149 ceph-mon[170394]: failed to encode map e149 with expected crc 2023-09-23T20:24:02.216 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:24:02.313 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:02.406 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:24:02.414 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:02.415 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:24:02.478 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 64 16 79385 79369 77.4768 74.9375 0.0127474 0.0129015 2023-09-23T20:24:02.516 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:24:02.594 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:02.661 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:24:02.667 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:02.668 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:24:02.768 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:24:02.863 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:02.927 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:24:02.932 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:02.932 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:24:02.935 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:24:02.984 INFO:teuthology.orchestra.run.smithi018.stderr:nodeep-scrub is set 2023-09-23T20:24:02.992 INFO:teuthology.orchestra.run.smithi018.stdout:moved 26 / 297 (8.75421%) 2023-09-23T20:24:02.993 INFO:teuthology.orchestra.run.smithi018.stdout:avg 37.125 2023-09-23T20:24:02.993 INFO:teuthology.orchestra.run.smithi018.stdout:stddev 5.64441 -> 6.50841 (expected baseline 5.69951) 2023-09-23T20:24:02.993 INFO:teuthology.orchestra.run.smithi018.stdout:min osd.2 with 26 -> 32 pgs (0.700337 -> 0.861953 * mean) 2023-09-23T20:24:02.993 INFO:teuthology.orchestra.run.smithi018.stdout:max osd.4 with 45 -> 37 pgs (1.21212 -> 0.996633 * mean) 2023-09-23T20:24:02.993 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:24:02.993 INFO:teuthology.orchestra.run.smithi018.stdout:oload 110 2023-09-23T20:24:02.993 INFO:teuthology.orchestra.run.smithi018.stdout:max_change 3 2023-09-23T20:24:02.993 INFO:teuthology.orchestra.run.smithi018.stdout:max_change_osds 4 2023-09-23T20:24:02.993 INFO:teuthology.orchestra.run.smithi018.stdout:average_utilization 0.0213 2023-09-23T20:24:02.993 INFO:teuthology.orchestra.run.smithi018.stdout:overload_utilization 0.0235 2023-09-23T20:24:02.993 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5 weight 1.0000 -> 0.7006 2023-09-23T20:24:02.994 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0 weight 1.0000 -> 0.7967 2023-09-23T20:24:02.994 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4 weight 1.0000 -> 0.8030 2023-09-23T20:24:02.998 INFO:teuthology.orchestra.run.smithi018.stderr:no change 2023-09-23T20:24:03.034 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:24:03.094 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:03.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:02 smithi149 ceph-mon[170394]: pgmap v303: 99 pgs: 1 remapped+peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 96 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 26 MiB/s wr, 420 op/s; 1994/231369 objects misplaced (0.862%); 9.2 MiB/s, 147 objects/s recovering 2023-09-23T20:24:03.218 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:02 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/571176572' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:24:03.218 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:02 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:24:03.323 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:24:02 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: 2023-09-23T20:24:02.989+0000 7f953be6a700 -1 mgr.server reply reply (3) No such process no change 2023-09-23T20:24:03.324 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:02 smithi018 ceph-mon[184144]: pgmap v303: 99 pgs: 1 remapped+peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 96 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 26 MiB/s wr, 420 op/s; 1994/231369 objects misplaced (0.862%); 9.2 MiB/s, 147 objects/s recovering 2023-09-23T20:24:03.324 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:02 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/571176572' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:24:03.324 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:02 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:24:03.324 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:02 smithi018 ceph-mon[179647]: pgmap v303: 99 pgs: 1 remapped+peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 96 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 26 MiB/s wr, 420 op/s; 1994/231369 objects misplaced (0.862%); 9.2 MiB/s, 147 objects/s recovering 2023-09-23T20:24:03.324 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:02 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/571176572' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:24:03.324 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:02 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:24:03.461 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '1000', 'type': 'reweight-by-utilization'} 2023-09-23T20:24:03.461 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd reweight-by-utilization 1000 3.0 2023-09-23T20:24:03.481 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 65 16 80656 80640 77.503 79.4375 0.0107042 0.0128976 2023-09-23T20:24:03.642 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:24:04.049 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:24:04.053 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:04.054 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:24:04.154 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:24:04.218 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:04.289 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:24:04.291 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:03 smithi018 ceph-mon[179647]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:24:04.291 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:03 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:24:04.291 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:03 smithi018 ceph-mon[179647]: osdmap e150: 8 total, 8 up, 8 in 2023-09-23T20:24:04.291 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:03 smithi018 ceph-mon[179647]: failed to encode map e150 with expected crc 2023-09-23T20:24:04.292 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:03 smithi018 ceph-mon[179647]: failed to encode map e150 with expected crc 2023-09-23T20:24:04.292 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:03 smithi018 ceph-mon[179647]: failed to encode map e150 with expected crc 2023-09-23T20:24:04.292 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:03 smithi018 ceph-mon[179647]: failed to encode map e150 with expected crc 2023-09-23T20:24:04.292 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:03 smithi018 ceph-mon[179647]: from='client.54661 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:24:04.292 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:04.293 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:24:04.293 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:03 smithi018 ceph-mon[184144]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:24:04.293 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:03 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:24:04.293 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:03 smithi018 ceph-mon[184144]: osdmap e150: 8 total, 8 up, 8 in 2023-09-23T20:24:04.293 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:03 smithi018 ceph-mon[184144]: failed to encode map e150 with expected crc 2023-09-23T20:24:04.293 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:03 smithi018 ceph-mon[184144]: failed to encode map e150 with expected crc 2023-09-23T20:24:04.294 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:03 smithi018 ceph-mon[184144]: failed to encode map e150 with expected crc 2023-09-23T20:24:04.294 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:03 smithi018 ceph-mon[184144]: failed to encode map e150 with expected crc 2023-09-23T20:24:04.294 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:03 smithi018 ceph-mon[184144]: from='client.54661 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:24:04.307 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:24:04.394 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:24:04.446 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:24:04.454 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:03 smithi149 ceph-mon[170394]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:24:04.455 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:03 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:24:04.455 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:03 smithi149 ceph-mon[170394]: osdmap e150: 8 total, 8 up, 8 in 2023-09-23T20:24:04.455 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:03 smithi149 ceph-mon[170394]: failed to encode map e150 with expected crc 2023-09-23T20:24:04.455 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:03 smithi149 ceph-mon[170394]: failed to encode map e150 with expected crc 2023-09-23T20:24:04.455 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:03 smithi149 ceph-mon[170394]: failed to encode map e150 with expected crc 2023-09-23T20:24:04.455 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:03 smithi149 ceph-mon[170394]: failed to encode map e150 with expected crc 2023-09-23T20:24:04.455 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:03 smithi149 ceph-mon[170394]: from='client.54661 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:24:04.462 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:04.482 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 66 16 81710 81694 77.3259 65.875 0.0135448 0.0129275 2023-09-23T20:24:04.981 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:24:04.994 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:04.994 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:24:05.095 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:24:05.167 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:04 smithi018 ceph-mon[179647]: pgmap v305: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 95 active+clean; 4.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 32 MiB/s wr, 518 op/s; 4758/235110 objects degraded (2.024%); 6752/235110 objects misplaced (2.872%); 7.6 MiB/s, 121 objects/s recovering 2023-09-23T20:24:05.167 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:04 smithi018 ceph-mon[179647]: Health check failed: Degraded data redundancy: 4758/235110 objects degraded (2.024%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T20:24:05.168 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:04 smithi018 ceph-mon[184144]: pgmap v305: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 95 active+clean; 4.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 32 MiB/s wr, 518 op/s; 4758/235110 objects degraded (2.024%); 6752/235110 objects misplaced (2.872%); 7.6 MiB/s, 121 objects/s recovering 2023-09-23T20:24:05.168 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:04 smithi018 ceph-mon[184144]: Health check failed: Degraded data redundancy: 4758/235110 objects degraded (2.024%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T20:24:05.168 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:24:04 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:24:04] "GET /metrics HTTP/1.1" 200 36471 "" "Prometheus/2.43.0" 2023-09-23T20:24:05.180 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:05.248 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:04 smithi149 ceph-mon[170394]: pgmap v305: 99 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 95 active+clean; 4.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 32 MiB/s wr, 518 op/s; 4758/235110 objects degraded (2.024%); 6752/235110 objects misplaced (2.872%); 7.6 MiB/s, 121 objects/s recovering 2023-09-23T20:24:05.248 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:04 smithi149 ceph-mon[170394]: Health check failed: Degraded data redundancy: 4758/235110 objects degraded (2.024%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T20:24:05.340 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:24:05.343 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:05.344 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:24:05.445 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:24:05.483 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 67 16 83000 82984 77.3738 80.625 0.0130985 0.0129198 2023-09-23T20:24:05.522 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:05.623 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:24:05.630 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:05.630 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:24:05.722 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T20:24:05.731 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:24:05.797 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:05.859 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:24:05.867 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:05.868 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:24:05.968 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:24:06.031 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:06.084 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:24:06.090 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:06.091 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:24:06.191 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:24:06.274 INFO:teuthology.orchestra.run.smithi149.stdout:ERROR: (22) Invalid argument 2023-09-23T20:24:06.274 INFO:teuthology.orchestra.run.smithi149.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:24:06.285 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:06.363 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:24:06.370 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:06.370 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:24:06.385 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:24:06.425 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:24:06.463 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:24:06.472 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:24:06.493 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 68 16 84066 84050 77.2156 66.625 0.0188035 0.0129466 2023-09-23T20:24:06.516 INFO:teuthology.orchestra.run.smithi018.stdout:moved 0 / 297 (0%) 2023-09-23T20:24:06.517 INFO:teuthology.orchestra.run.smithi018.stdout:avg 37.125 2023-09-23T20:24:06.517 INFO:teuthology.orchestra.run.smithi018.stdout:stddev 5.64441 -> 5.64441 (expected baseline 5.69951) 2023-09-23T20:24:06.517 INFO:teuthology.orchestra.run.smithi018.stdout:min osd.2 with 26 -> 26 pgs (0.700337 -> 0.700337 * mean) 2023-09-23T20:24:06.517 INFO:teuthology.orchestra.run.smithi018.stdout:max osd.4 with 45 -> 45 pgs (1.21212 -> 1.21212 * mean) 2023-09-23T20:24:06.517 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:24:06.517 INFO:teuthology.orchestra.run.smithi018.stdout:oload 1000 2023-09-23T20:24:06.517 INFO:teuthology.orchestra.run.smithi018.stdout:max_change 3 2023-09-23T20:24:06.517 INFO:teuthology.orchestra.run.smithi018.stdout:max_change_osds 4 2023-09-23T20:24:06.517 INFO:teuthology.orchestra.run.smithi018.stdout:average_utilization 0.0223 2023-09-23T20:24:06.517 INFO:teuthology.orchestra.run.smithi018.stdout:overload_utilization 0.2228 2023-09-23T20:24:06.518 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:24:06.525 INFO:teuthology.orchestra.run.smithi018.stderr:no change 2023-09-23T20:24:06.560 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:06.578 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:24:06.614 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T20:24:06.642 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:24:06.643 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:24:06.650 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:06.651 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:24:06.677 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:24:06.721 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:24:06.725 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:24:06.726 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:24:06.726 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:24:06.726 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:24:06.726 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:24:06.726 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:24:06.726 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:24:06.726 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:24:06.752 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:24:06.799 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:07.225 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:24:07.225 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:07 smithi018 ceph-mon[184144]: pgmap v306: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 51 MiB/s wr, 820 op/s; 4758/243609 objects degraded (1.953%); 5844/243609 objects misplaced (2.399%); 11 MiB/s, 172 objects/s recovering 2023-09-23T20:24:07.226 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:07 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1068393939' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:24:07.231 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:07.231 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:07 smithi018 ceph-mon[179647]: pgmap v306: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 51 MiB/s wr, 820 op/s; 4758/243609 objects degraded (1.953%); 5844/243609 objects misplaced (2.399%); 11 MiB/s, 172 objects/s recovering 2023-09-23T20:24:07.231 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:07 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1068393939' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:24:07.232 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:24:07.249 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '1000', 'type': 'reweight-by-utilization'} 2023-09-23T20:24:07.249 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd reweight-by-utilization 1000 1.0 2023-09-23T20:24:07.314 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:24:07.332 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:24:07.353 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:24:07.388 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:07 smithi149 ceph-mon[170394]: pgmap v306: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 51 MiB/s wr, 820 op/s; 4758/243609 objects degraded (1.953%); 5844/243609 objects misplaced (2.399%); 11 MiB/s, 172 objects/s recovering 2023-09-23T20:24:07.388 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:07 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1068393939' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:24:07.390 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:07.483 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 69 16 85190 85174 77.1146 70.25 0.0130917 0.0129634 2023-09-23T20:24:07.514 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:24:07.520 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:07.520 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:24:07.621 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:24:07.690 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:07.757 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:24:07.764 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:07.764 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:24:07.865 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:24:07.945 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:08.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:08 smithi018 ceph-mon[179647]: from='client.44807 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:24:08.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:08 smithi018 ceph-mon[184144]: from='client.44807 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:24:08.166 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:24:08.170 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:08.171 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:24:08.189 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:08 smithi149 ceph-mon[170394]: from='client.44807 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:24:08.272 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:24:08.346 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:08.415 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:24:08.484 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 70 16 86367 86351 77.0638 73.5625 0.0159402 0.0129712 2023-09-23T20:24:08.517 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:24:08.522 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:08.522 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:24:08.623 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:24:08.694 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:08.862 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:24:08.871 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:08.871 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:24:08.972 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:24:08.973 INFO:teuthology.orchestra.run.smithi149.stdout:{ 2023-09-23T20:24:08.973 INFO:teuthology.orchestra.run.smithi149.stdout: "ops": [], 2023-09-23T20:24:08.973 INFO:teuthology.orchestra.run.smithi149.stdout: "complaint_time": 30, 2023-09-23T20:24:08.973 INFO:teuthology.orchestra.run.smithi149.stdout: "num_blocked_ops": 0 2023-09-23T20:24:08.973 INFO:teuthology.orchestra.run.smithi149.stdout:} 2023-09-23T20:24:09.043 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:09 smithi018 ceph-mon[179647]: pgmap v307: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 53 MiB/s wr, 847 op/s; 4758/247869 objects degraded (1.920%); 5844/247869 objects misplaced (2.358%); 8.8 MiB/s, 140 objects/s recovering 2023-09-23T20:24:09.043 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:09 smithi018 ceph-mon[184144]: pgmap v307: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 53 MiB/s wr, 847 op/s; 4758/247869 objects degraded (1.920%); 5844/247869 objects misplaced (2.358%); 8.8 MiB/s, 140 objects/s recovering 2023-09-23T20:24:09.047 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:09.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:09 smithi149 ceph-mon[170394]: pgmap v307: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 53 MiB/s wr, 847 op/s; 4758/247869 objects degraded (1.920%); 5844/247869 objects misplaced (2.358%); 8.8 MiB/s, 140 objects/s recovering 2023-09-23T20:24:09.301 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:24:09.402 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:24:09.405 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:09.406 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:24:09.459 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T20:24:09.484 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 71 16 87548 87532 77.018 73.8125 0.0180698 0.0129791 2023-09-23T20:24:09.506 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:24:09.577 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:09.666 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:24:09.681 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:09.681 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:24:09.782 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:24:09.853 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:09.918 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:24:09.922 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:09.922 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:24:10.023 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:24:10.097 INFO:teuthology.orchestra.run.smithi018.stderr:noscrub is unset 2023-09-23T20:24:10.130 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:10 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3863092262' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:24:10.130 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:10 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:24:10.131 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:10 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3863092262' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:24:10.131 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:10 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:24:10.136 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:10.225 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:24:10.234 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:10.235 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:24:10.251 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:24:10.274 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:24:10.336 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:24:10.400 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:10.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:10 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3863092262' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:24:10.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:10 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:24:10.484 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 72 16 88647 88631 76.902 68.6875 0.0094197 0.0129985 2023-09-23T20:24:10.494 INFO:teuthology.orchestra.run.smithi018.stdout:moved 0 / 297 (0%) 2023-09-23T20:24:10.495 INFO:teuthology.orchestra.run.smithi018.stdout:avg 37.125 2023-09-23T20:24:10.495 INFO:teuthology.orchestra.run.smithi018.stdout:stddev 5.64441 -> 5.64441 (expected baseline 5.69951) 2023-09-23T20:24:10.495 INFO:teuthology.orchestra.run.smithi018.stdout:min osd.2 with 26 -> 26 pgs (0.700337 -> 0.700337 * mean) 2023-09-23T20:24:10.495 INFO:teuthology.orchestra.run.smithi018.stdout:max osd.4 with 45 -> 45 pgs (1.21212 -> 1.21212 * mean) 2023-09-23T20:24:10.495 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:24:10.495 INFO:teuthology.orchestra.run.smithi018.stdout:oload 1000 2023-09-23T20:24:10.495 INFO:teuthology.orchestra.run.smithi018.stdout:max_change 1 2023-09-23T20:24:10.495 INFO:teuthology.orchestra.run.smithi018.stdout:max_change_osds 4 2023-09-23T20:24:10.495 INFO:teuthology.orchestra.run.smithi018.stdout:average_utilization 0.0231 2023-09-23T20:24:10.495 INFO:teuthology.orchestra.run.smithi018.stdout:overload_utilization 0.2305 2023-09-23T20:24:10.516 INFO:teuthology.orchestra.run.smithi018.stderr:no change 2023-09-23T20:24:10.925 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:24:10.933 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:10.934 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:24:11.034 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:24:11.090 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:11.152 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:24:11.157 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:11.157 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:24:11.233 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:11 smithi018 ceph-mon[179647]: pgmap v308: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 53 MiB/s wr, 840 op/s; 4758/251547 objects degraded (1.891%); 5844/251547 objects misplaced (2.323%); 7.4 MiB/s, 118 objects/s recovering 2023-09-23T20:24:11.233 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:11 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 4758/251547 objects degraded (1.891%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T20:24:11.233 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:11 smithi018 ceph-mon[179647]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:24:11.233 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:11 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:24:11.233 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:11 smithi018 ceph-mon[179647]: osdmap e151: 8 total, 8 up, 8 in 2023-09-23T20:24:11.233 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:11 smithi018 ceph-mon[179647]: failed to encode map e151 with expected crc 2023-09-23T20:24:11.233 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:11 smithi018 ceph-mon[179647]: failed to encode map e151 with expected crc 2023-09-23T20:24:11.233 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:11 smithi018 ceph-mon[179647]: failed to encode map e151 with expected crc 2023-09-23T20:24:11.234 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:11 smithi018 ceph-mon[184144]: pgmap v308: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 53 MiB/s wr, 840 op/s; 4758/251547 objects degraded (1.891%); 5844/251547 objects misplaced (2.323%); 7.4 MiB/s, 118 objects/s recovering 2023-09-23T20:24:11.234 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:11 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 4758/251547 objects degraded (1.891%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T20:24:11.234 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:11 smithi018 ceph-mon[184144]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:24:11.234 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:11 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:24:11.234 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:11 smithi018 ceph-mon[184144]: osdmap e151: 8 total, 8 up, 8 in 2023-09-23T20:24:11.234 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:11 smithi018 ceph-mon[184144]: failed to encode map e151 with expected crc 2023-09-23T20:24:11.234 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:11 smithi018 ceph-mon[184144]: failed to encode map e151 with expected crc 2023-09-23T20:24:11.234 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:11 smithi018 ceph-mon[184144]: failed to encode map e151 with expected crc 2023-09-23T20:24:11.258 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:24:11.313 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:11 smithi149 ceph-mon[170394]: pgmap v308: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 53 MiB/s wr, 840 op/s; 4758/251547 objects degraded (1.891%); 5844/251547 objects misplaced (2.323%); 7.4 MiB/s, 118 objects/s recovering 2023-09-23T20:24:11.313 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:11 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 4758/251547 objects degraded (1.891%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T20:24:11.313 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:11 smithi149 ceph-mon[170394]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:24:11.313 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:11 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:24:11.313 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:11 smithi149 ceph-mon[170394]: osdmap e151: 8 total, 8 up, 8 in 2023-09-23T20:24:11.314 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:11 smithi149 ceph-mon[170394]: failed to encode map e151 with expected crc 2023-09-23T20:24:11.314 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:11 smithi149 ceph-mon[170394]: failed to encode map e151 with expected crc 2023-09-23T20:24:11.314 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:11 smithi149 ceph-mon[170394]: failed to encode map e151 with expected crc 2023-09-23T20:24:11.318 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:11.410 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:24:11.417 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:11.417 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:24:11.484 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 73 16 89742 89726 76.7861 68.4375 0.0052356 0.013019 2023-09-23T20:24:11.518 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:24:11.560 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '110', 'type': 'test-reweight-by-utilization'} 2023-09-23T20:24:11.560 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd test-reweight-by-utilization 110 1.0 2023-09-23T20:24:11.617 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:11.667 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:24:11.676 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:11.676 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:24:11.777 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:24:11.819 INFO:teuthology.orchestra.run.smithi149.stdout:{ 2023-09-23T20:24:11.819 INFO:teuthology.orchestra.run.smithi149.stdout: "size": 20, 2023-09-23T20:24:11.819 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 600, 2023-09-23T20:24:11.819 INFO:teuthology.orchestra.run.smithi149.stdout: "ops": [ 2023-09-23T20:24:11.819 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.820 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:29961 5.4 5:20b9d156:::benchmark_data_smithi018_228451_object29960:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e119)", 2023-09-23T20:24:11.820 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:11.525212+0000", 2023-09-23T20:24:11.820 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 60.291130922000001, 2023-09-23T20:24:11.820 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.05059744, 2023-09-23T20:24:11.820 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:24:11.820 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:24:11.820 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:24:11.820 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:24:11.820 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:24:11.821 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 29961 2023-09-23T20:24:11.821 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.821 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:24:11.821 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.821 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:24:11.821 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.525212+0000", 2023-09-23T20:24:11.821 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:24:11.821 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.821 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.821 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:24:11.822 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.525212+0000", 2023-09-23T20:24:11.822 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:24:11.822 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.822 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.822 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:24:11.822 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.525211+0000", 2023-09-23T20:24:11.822 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:24:11.822 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.822 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.822 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:24:11.823 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.525280+0000", 2023-09-23T20:24:11.823 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.8520000000000001e-05 2023-09-23T20:24:11.823 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.823 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.823 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:24:11.823 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.525292+0000", 2023-09-23T20:24:11.823 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.2339e-05 2023-09-23T20:24:11.823 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.823 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.823 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:24:11.823 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.525298+0000", 2023-09-23T20:24:11.824 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.2759999999999997e-06 2023-09-23T20:24:11.824 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.824 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.824 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:24:11.824 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.525356+0000", 2023-09-23T20:24:11.824 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.7763000000000002e-05 2023-09-23T20:24:11.824 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.824 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.824 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:24:11.824 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.525468+0000", 2023-09-23T20:24:11.825 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000112149 2023-09-23T20:24:11.825 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.825 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.825 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 2,5", 2023-09-23T20:24:11.825 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.525596+0000", 2023-09-23T20:24:11.825 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000128009 2023-09-23T20:24:11.825 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.825 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.825 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:24:11.825 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.526691+0000", 2023-09-23T20:24:11.826 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.001094582 2023-09-23T20:24:11.826 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.826 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.826 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:24:11.826 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.527735+0000", 2023-09-23T20:24:11.826 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.001043958 2023-09-23T20:24:11.826 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.826 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.826 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:24:11.826 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.575728+0000", 2023-09-23T20:24:11.827 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.047993418000000003 2023-09-23T20:24:11.827 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.827 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.827 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:24:11.827 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.575782+0000", 2023-09-23T20:24:11.827 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.3942e-05 2023-09-23T20:24:11.827 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.827 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.827 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:24:11.827 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.575810+0000", 2023-09-23T20:24:11.828 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.777e-05 2023-09-23T20:24:11.828 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:24:11.828 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:24:11.828 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:24:11.828 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.828 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.828 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:29972 5.14 5:2e3b60b1:::benchmark_data_smithi018_228451_object29971:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e119)", 2023-09-23T20:24:11.828 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:11.528938+0000", 2023-09-23T20:24:11.828 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 60.287405513000003, 2023-09-23T20:24:11.828 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.046754209999999997, 2023-09-23T20:24:11.828 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:24:11.829 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:24:11.829 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:24:11.829 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:24:11.829 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:24:11.829 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 29972 2023-09-23T20:24:11.829 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.829 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:24:11.829 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.829 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:24:11.829 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.528938+0000", 2023-09-23T20:24:11.830 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:24:11.830 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.830 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.830 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:24:11.830 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.528938+0000", 2023-09-23T20:24:11.830 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:24:11.830 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.830 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.830 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:24:11.831 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.528936+0000", 2023-09-23T20:24:11.831 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:24:11.831 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.831 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.831 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:24:11.831 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.529004+0000", 2023-09-23T20:24:11.831 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.7397999999999994e-05 2023-09-23T20:24:11.831 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.831 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.831 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:24:11.832 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.529027+0000", 2023-09-23T20:24:11.832 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.3499999999999999e-05 2023-09-23T20:24:11.832 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.832 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.832 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:24:11.832 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.529035+0000", 2023-09-23T20:24:11.832 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.4270000000000002e-06 2023-09-23T20:24:11.832 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.832 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.832 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:24:11.832 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.529086+0000", 2023-09-23T20:24:11.833 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.1669000000000003e-05 2023-09-23T20:24:11.833 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.833 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.833 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:24:11.833 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.529214+0000", 2023-09-23T20:24:11.833 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00012748800000000001 2023-09-23T20:24:11.833 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.833 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.833 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 2,5", 2023-09-23T20:24:11.833 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.529387+0000", 2023-09-23T20:24:11.834 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00017375299999999999 2023-09-23T20:24:11.834 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.834 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.834 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:24:11.834 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.530210+0000", 2023-09-23T20:24:11.834 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00082250800000000001 2023-09-23T20:24:11.834 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.834 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.834 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:24:11.834 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.530834+0000", 2023-09-23T20:24:11.835 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00062393799999999999 2023-09-23T20:24:11.835 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.835 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.835 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:24:11.835 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.575603+0000", 2023-09-23T20:24:11.835 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.044769309 2023-09-23T20:24:11.835 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.835 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.835 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:24:11.835 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.575655+0000", 2023-09-23T20:24:11.836 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.2198999999999999e-05 2023-09-23T20:24:11.836 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.836 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.836 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:24:11.836 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.575692+0000", 2023-09-23T20:24:11.836 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.6538999999999997e-05 2023-09-23T20:24:11.836 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:24:11.836 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:24:11.836 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:24:11.836 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.837 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.837 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:39077 5.10 5.43acc570 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e120)", 2023-09-23T20:24:11.837 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:16.079010+0000", 2023-09-23T20:24:11.837 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 55.737333388000003, 2023-09-23T20:24:11.837 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.96266576800000003, 2023-09-23T20:24:11.837 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:24:11.837 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:24:11.837 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:24:11.837 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:24:11.838 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:24:11.838 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 39077 2023-09-23T20:24:11.838 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.838 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:24:11.838 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.838 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:24:11.838 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.079010+0000", 2023-09-23T20:24:11.838 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:24:11.838 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.838 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.839 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:24:11.839 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.079010+0000", 2023-09-23T20:24:11.839 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:24:11.839 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.839 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.839 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:24:11.839 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.079009+0000", 2023-09-23T20:24:11.839 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:24:11.839 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.839 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.839 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:24:11.840 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.079042+0000", 2023-09-23T20:24:11.840 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.2978000000000002e-05 2023-09-23T20:24:11.840 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.840 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.840 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:24:11.840 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.079050+0000", 2023-09-23T20:24:11.840 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.729e-06 2023-09-23T20:24:11.840 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.840 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.840 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:24:11.841 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.079055+0000", 2023-09-23T20:24:11.841 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.5709999999999997e-06 2023-09-23T20:24:11.841 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.841 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.841 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:24:11.841 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.079112+0000", 2023-09-23T20:24:11.841 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.6051999999999998e-05 2023-09-23T20:24:11.841 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.841 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.841 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:24:11.841 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.079119+0000", 2023-09-23T20:24:11.841 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.2189999999999998e-06 2023-09-23T20:24:11.842 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.842 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.842 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:24:11.842 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:17.041662+0000", 2023-09-23T20:24:11.842 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.96254368199999996 2023-09-23T20:24:11.842 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.842 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.842 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:24:11.842 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:17.041676+0000", 2023-09-23T20:24:11.842 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.3243e-05 2023-09-23T20:24:11.843 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:24:11.843 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:24:11.843 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:24:11.843 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.843 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.843 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:39130 5.10 5.c968f510 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e120)", 2023-09-23T20:24:11.843 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:16.108071+0000", 2023-09-23T20:24:11.843 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 55.708272250999997, 2023-09-23T20:24:11.843 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.93362244800000005, 2023-09-23T20:24:11.843 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:24:11.844 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:24:11.844 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:24:11.844 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:24:11.844 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:24:11.844 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 39130 2023-09-23T20:24:11.844 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.844 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:24:11.844 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.844 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:24:11.844 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.108071+0000", 2023-09-23T20:24:11.845 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:24:11.845 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.845 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.845 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:24:11.845 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.108071+0000", 2023-09-23T20:24:11.845 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:24:11.845 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.845 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.845 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:24:11.845 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.108070+0000", 2023-09-23T20:24:11.846 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:24:11.846 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.846 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.846 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:24:11.846 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.108130+0000", 2023-09-23T20:24:11.846 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.9410000000000002e-05 2023-09-23T20:24:11.846 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.846 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.846 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:24:11.847 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.108141+0000", 2023-09-23T20:24:11.847 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1477999999999999e-05 2023-09-23T20:24:11.847 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.847 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.847 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:24:11.847 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.108147+0000", 2023-09-23T20:24:11.847 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.1319999999999999e-06 2023-09-23T20:24:11.847 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.847 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.847 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:24:11.848 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.108206+0000", 2023-09-23T20:24:11.848 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.8393999999999997e-05 2023-09-23T20:24:11.848 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.848 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.848 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:24:11.848 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.108214+0000", 2023-09-23T20:24:11.848 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.9610000000000005e-06 2023-09-23T20:24:11.848 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.848 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.848 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:24:11.848 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:17.041688+0000", 2023-09-23T20:24:11.849 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.93347436800000005 2023-09-23T20:24:11.849 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.849 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.849 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:24:11.849 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:17.041693+0000", 2023-09-23T20:24:11.849 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.3739999999999997e-06 2023-09-23T20:24:11.849 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:24:11.849 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:24:11.849 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:24:11.849 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.849 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.850 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:39199 5.10 5.b79ac790 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e120)", 2023-09-23T20:24:11.850 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:16.154492+0000", 2023-09-23T20:24:11.850 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 55.661851483, 2023-09-23T20:24:11.850 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.88722685300000004, 2023-09-23T20:24:11.850 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:24:11.850 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:24:11.850 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:24:11.850 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:24:11.850 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:24:11.851 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 39199 2023-09-23T20:24:11.851 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.851 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:24:11.851 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.851 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:24:11.851 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.154492+0000", 2023-09-23T20:24:11.851 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:24:11.851 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.851 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.852 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:24:11.852 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.154492+0000", 2023-09-23T20:24:11.852 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:24:11.852 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.852 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.852 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:24:11.852 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.154491+0000", 2023-09-23T20:24:11.852 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:24:11.852 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.852 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.852 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:24:11.853 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.154534+0000", 2023-09-23T20:24:11.853 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.3306999999999997e-05 2023-09-23T20:24:11.853 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.853 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.853 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:24:11.853 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.154549+0000", 2023-09-23T20:24:11.853 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.4786e-05 2023-09-23T20:24:11.853 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.853 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.853 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:24:11.854 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.154556+0000", 2023-09-23T20:24:11.854 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.9430000000000003e-06 2023-09-23T20:24:11.854 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.854 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.854 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:24:11.854 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.154607+0000", 2023-09-23T20:24:11.854 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.1183000000000001e-05 2023-09-23T20:24:11.854 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.854 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.855 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:24:11.855 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.154614+0000", 2023-09-23T20:24:11.855 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.5900000000000002e-06 2023-09-23T20:24:11.855 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.855 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.855 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:24:11.855 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:17.041709+0000", 2023-09-23T20:24:11.855 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.88709450599999995 2023-09-23T20:24:11.855 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.855 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.856 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:24:11.856 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:17.041719+0000", 2023-09-23T20:24:11.856 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.6639999999999993e-06 2023-09-23T20:24:11.856 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:24:11.856 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:24:11.856 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:24:11.856 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.856 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.856 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:39218 5.10 5.57c41cb0 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e120)", 2023-09-23T20:24:11.856 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:16.163064+0000", 2023-09-23T20:24:11.856 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 55.653279196, 2023-09-23T20:24:11.857 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.87866627399999997, 2023-09-23T20:24:11.857 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:24:11.857 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:24:11.857 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:24:11.857 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:24:11.857 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:24:11.857 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 39218 2023-09-23T20:24:11.857 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.857 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:24:11.857 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.858 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:24:11.858 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.163064+0000", 2023-09-23T20:24:11.858 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:24:11.858 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.858 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.858 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:24:11.858 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.163064+0000", 2023-09-23T20:24:11.858 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:24:11.858 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.859 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.859 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:24:11.859 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.163064+0000", 2023-09-23T20:24:11.859 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:24:11.859 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.859 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.859 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:24:11.859 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.163128+0000", 2023-09-23T20:24:11.859 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.4064000000000002e-05 2023-09-23T20:24:11.859 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.859 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.859 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:24:11.860 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.163136+0000", 2023-09-23T20:24:11.860 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.9310000000000008e-06 2023-09-23T20:24:11.860 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.860 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.860 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:24:11.860 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.163144+0000", 2023-09-23T20:24:11.860 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.1310000000000006e-06 2023-09-23T20:24:11.860 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.860 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.860 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:24:11.861 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.163208+0000", 2023-09-23T20:24:11.861 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.4208000000000006e-05 2023-09-23T20:24:11.861 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.861 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.861 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:24:11.861 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.163216+0000", 2023-09-23T20:24:11.861 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.0879999999999995e-06 2023-09-23T20:24:11.861 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.861 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.861 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:24:11.862 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:17.041727+0000", 2023-09-23T20:24:11.862 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.87851053899999998 2023-09-23T20:24:11.862 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.862 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.862 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:24:11.862 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:17.041730+0000", 2023-09-23T20:24:11.862 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.817e-06 2023-09-23T20:24:11.862 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:24:11.863 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:24:11.863 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:24:11.863 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.863 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.863 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:39310 5.10 5.66681090 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e120)", 2023-09-23T20:24:11.863 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:16.217162+0000", 2023-09-23T20:24:11.863 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 55.599180820000001, 2023-09-23T20:24:11.863 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.82458225600000001, 2023-09-23T20:24:11.863 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:24:11.863 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:24:11.863 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:24:11.864 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:24:11.864 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:24:11.864 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 39310 2023-09-23T20:24:11.864 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.864 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:24:11.864 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.864 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:24:11.864 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.217162+0000", 2023-09-23T20:24:11.864 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:24:11.865 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.865 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.865 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:24:11.865 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.217162+0000", 2023-09-23T20:24:11.865 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:24:11.865 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.865 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.865 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:24:11.865 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.217162+0000", 2023-09-23T20:24:11.865 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:24:11.865 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.866 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.866 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:24:11.866 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.217195+0000", 2023-09-23T20:24:11.866 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.3138e-05 2023-09-23T20:24:11.866 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.866 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.866 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:24:11.866 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.217203+0000", 2023-09-23T20:24:11.866 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.8620000000000003e-06 2023-09-23T20:24:11.866 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.867 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.867 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:24:11.867 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.217208+0000", 2023-09-23T20:24:11.867 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.8960000000000002e-06 2023-09-23T20:24:11.867 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.867 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.867 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:24:11.867 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.217264+0000", 2023-09-23T20:24:11.867 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.5713e-05 2023-09-23T20:24:11.867 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.867 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.868 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:24:11.868 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.217271+0000", 2023-09-23T20:24:11.868 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.8310000000000001e-06 2023-09-23T20:24:11.868 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.868 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.868 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:24:11.868 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:17.041741+0000", 2023-09-23T20:24:11.868 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.82447004400000001 2023-09-23T20:24:11.868 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.868 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.868 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:24:11.869 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:17.041745+0000", 2023-09-23T20:24:11.869 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.3179999999999998e-06 2023-09-23T20:24:11.869 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:24:11.869 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:24:11.869 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:24:11.869 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.869 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.869 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:39313 5.10 5.c9b04c50 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e120)", 2023-09-23T20:24:11.869 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:16.219810+0000", 2023-09-23T20:24:11.869 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 55.596533145000002, 2023-09-23T20:24:11.869 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.82197555200000005, 2023-09-23T20:24:11.870 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:24:11.870 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:24:11.870 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:24:11.870 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:24:11.870 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:24:11.870 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 39313 2023-09-23T20:24:11.870 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.870 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:24:11.870 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.870 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:24:11.871 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.219810+0000", 2023-09-23T20:24:11.871 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:24:11.871 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.871 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.871 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:24:11.871 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.219810+0000", 2023-09-23T20:24:11.871 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:24:11.871 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.871 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.871 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:24:11.871 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.219809+0000", 2023-09-23T20:24:11.872 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:24:11.872 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.872 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.872 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:24:11.872 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.219853+0000", 2023-09-23T20:24:11.872 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.3417999999999997e-05 2023-09-23T20:24:11.872 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.872 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.872 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:24:11.872 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.219865+0000", 2023-09-23T20:24:11.872 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.2101e-05 2023-09-23T20:24:11.873 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.873 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.873 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:24:11.873 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.219871+0000", 2023-09-23T20:24:11.873 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.6200000000000001e-06 2023-09-23T20:24:11.873 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.873 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.873 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:24:11.873 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.219911+0000", 2023-09-23T20:24:11.873 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.9589000000000003e-05 2023-09-23T20:24:11.873 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.874 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.874 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:24:11.874 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.219916+0000", 2023-09-23T20:24:11.874 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.3850000000000001e-06 2023-09-23T20:24:11.874 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.874 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.874 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:24:11.874 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:17.041778+0000", 2023-09-23T20:24:11.874 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.82186138099999995 2023-09-23T20:24:11.875 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.875 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.875 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:24:11.875 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:17.041786+0000", 2023-09-23T20:24:11.875 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.9580000000000007e-06 2023-09-23T20:24:11.875 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:24:11.875 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:24:11.875 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:24:11.875 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.875 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.875 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:39369 5.10 5.55eb7150 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e120)", 2023-09-23T20:24:11.876 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:16.252179+0000", 2023-09-23T20:24:11.876 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 55.564164605000002, 2023-09-23T20:24:11.876 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.78962392599999998, 2023-09-23T20:24:11.876 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:24:11.876 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:24:11.876 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:24:11.876 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:24:11.876 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:24:11.876 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 39369 2023-09-23T20:24:11.876 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.877 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:24:11.877 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.877 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:24:11.877 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.252179+0000", 2023-09-23T20:24:11.877 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:24:11.877 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.877 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.877 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:24:11.877 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.252179+0000", 2023-09-23T20:24:11.878 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:24:11.878 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.878 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.878 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:24:11.878 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.252177+0000", 2023-09-23T20:24:11.878 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:24:11.878 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.878 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.878 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:24:11.879 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.252236+0000", 2023-09-23T20:24:11.879 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.8325999999999997e-05 2023-09-23T20:24:11.879 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.879 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.879 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:24:11.879 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.252251+0000", 2023-09-23T20:24:11.879 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.5480000000000001e-05 2023-09-23T20:24:11.879 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.879 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.879 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:24:11.880 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.252258+0000", 2023-09-23T20:24:11.880 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.0609999999999998e-06 2023-09-23T20:24:11.880 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.880 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.880 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:24:11.880 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.252303+0000", 2023-09-23T20:24:11.880 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.4892999999999999e-05 2023-09-23T20:24:11.880 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.881 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.881 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:24:11.881 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.252308+0000", 2023-09-23T20:24:11.881 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.2290000000000002e-06 2023-09-23T20:24:11.881 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.881 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.881 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:24:11.881 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:17.041797+0000", 2023-09-23T20:24:11.881 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.78948882399999998 2023-09-23T20:24:11.881 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.882 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.882 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:24:11.882 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:17.041803+0000", 2023-09-23T20:24:11.882 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.4519999999999996e-06 2023-09-23T20:24:11.882 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:24:11.882 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:24:11.882 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:24:11.882 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.882 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.883 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:39381 5.10 5.cc5a9050 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e120)", 2023-09-23T20:24:11.883 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:16.258586+0000", 2023-09-23T20:24:11.883 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 55.557756957999999, 2023-09-23T20:24:11.883 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.78325138500000002, 2023-09-23T20:24:11.883 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:24:11.883 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:24:11.883 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:24:11.883 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:24:11.883 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:24:11.884 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 39381 2023-09-23T20:24:11.884 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.884 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:24:11.884 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.884 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:24:11.884 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.258586+0000", 2023-09-23T20:24:11.884 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:24:11.884 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.884 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.884 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:24:11.885 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.258586+0000", 2023-09-23T20:24:11.885 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:24:11.885 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.885 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.885 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:24:11.885 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.258585+0000", 2023-09-23T20:24:11.885 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:24:11.885 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.885 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.885 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:24:11.886 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.258657+0000", 2023-09-23T20:24:11.886 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.1934000000000006e-05 2023-09-23T20:24:11.886 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.886 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.886 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:24:11.886 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.258674+0000", 2023-09-23T20:24:11.886 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.7422999999999999e-05 2023-09-23T20:24:11.886 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.886 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.887 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:24:11.887 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.258681+0000", 2023-09-23T20:24:11.887 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.9110000000000004e-06 2023-09-23T20:24:11.887 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.887 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.887 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:24:11.887 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.258739+0000", 2023-09-23T20:24:11.887 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.8155999999999997e-05 2023-09-23T20:24:11.887 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.887 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.888 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:24:11.888 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.258748+0000", 2023-09-23T20:24:11.888 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.1810000000000007e-06 2023-09-23T20:24:11.888 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.888 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.888 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:24:11.888 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:17.041828+0000", 2023-09-23T20:24:11.888 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.78307920499999994 2023-09-23T20:24:11.888 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.888 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.889 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:24:11.889 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:17.041838+0000", 2023-09-23T20:24:11.889 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0108999999999999e-05 2023-09-23T20:24:11.889 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:24:11.889 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:24:11.889 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:24:11.889 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.889 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.889 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:39421 5.10 5.c19f050 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e120)", 2023-09-23T20:24:11.889 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:16.286823+0000", 2023-09-23T20:24:11.889 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 55.529520241, 2023-09-23T20:24:11.890 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.75502831000000004, 2023-09-23T20:24:11.890 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:24:11.890 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:24:11.890 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:24:11.890 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:24:11.890 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:24:11.890 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 39421 2023-09-23T20:24:11.890 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.890 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:24:11.891 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.891 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:24:11.891 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.286823+0000", 2023-09-23T20:24:11.891 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:24:11.891 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.891 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.891 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:24:11.891 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.286823+0000", 2023-09-23T20:24:11.891 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:24:11.891 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.892 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.892 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:24:11.892 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.286821+0000", 2023-09-23T20:24:11.892 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999981 2023-09-23T20:24:11.892 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.892 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.892 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:24:11.892 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.286886+0000", 2023-09-23T20:24:11.892 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.4592999999999996e-05 2023-09-23T20:24:11.893 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.893 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.893 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:24:11.893 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.286898+0000", 2023-09-23T20:24:11.893 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.2387e-05 2023-09-23T20:24:11.893 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.893 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.893 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:24:11.893 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.286904+0000", 2023-09-23T20:24:11.894 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.1619999999999996e-06 2023-09-23T20:24:11.894 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.894 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.894 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:24:11.894 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.286970+0000", 2023-09-23T20:24:11.894 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.5284999999999996e-05 2023-09-23T20:24:11.894 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.894 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.894 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:24:11.894 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.286980+0000", 2023-09-23T20:24:11.895 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0562e-05 2023-09-23T20:24:11.895 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.895 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.895 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:24:11.895 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:17.041848+0000", 2023-09-23T20:24:11.895 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.75486723700000002 2023-09-23T20:24:11.895 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.895 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.895 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:24:11.895 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:17.041851+0000", 2023-09-23T20:24:11.896 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.7689999999999998e-06 2023-09-23T20:24:11.896 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:24:11.896 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:24:11.896 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:24:11.896 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.896 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.896 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:39462 5.10 5.dbc892b0 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e120)", 2023-09-23T20:24:11.896 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:16.320756+0000", 2023-09-23T20:24:11.896 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 55.495586940000003, 2023-09-23T20:24:11.896 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.72112014800000002, 2023-09-23T20:24:11.897 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:24:11.897 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:24:11.897 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:24:11.897 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:24:11.897 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:24:11.897 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 39462 2023-09-23T20:24:11.897 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.897 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:24:11.897 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.897 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:24:11.898 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.320756+0000", 2023-09-23T20:24:11.898 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:24:11.898 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.898 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.898 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:24:11.898 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.320756+0000", 2023-09-23T20:24:11.898 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:24:11.898 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.898 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.899 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:24:11.899 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.320755+0000", 2023-09-23T20:24:11.899 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:24:11.899 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.899 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.899 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:24:11.899 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.320838+0000", 2023-09-23T20:24:11.899 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.2861999999999994e-05 2023-09-23T20:24:11.899 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.899 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.899 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:24:11.900 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.320851+0000", 2023-09-23T20:24:11.900 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.2989999999999999e-05 2023-09-23T20:24:11.900 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.900 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.900 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:24:11.900 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.320856+0000", 2023-09-23T20:24:11.900 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.9960000000000001e-06 2023-09-23T20:24:11.900 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.900 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.900 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:24:11.901 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.320914+0000", 2023-09-23T20:24:11.901 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.7859000000000003e-05 2023-09-23T20:24:11.901 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.901 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.901 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:24:11.901 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.320925+0000", 2023-09-23T20:24:11.901 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0871e-05 2023-09-23T20:24:11.901 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.901 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.901 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:24:11.902 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:17.041867+0000", 2023-09-23T20:24:11.902 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.72094236199999995 2023-09-23T20:24:11.902 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.902 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.902 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:24:11.902 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:17.041876+0000", 2023-09-23T20:24:11.902 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.2599999999999994e-06 2023-09-23T20:24:11.902 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:24:11.902 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:24:11.903 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:24:11.903 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.903 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.903 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:39488 5.10 5.85dc4890 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e120)", 2023-09-23T20:24:11.903 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:16.340478+0000", 2023-09-23T20:24:11.903 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 55.475865333999998, 2023-09-23T20:24:11.903 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.701410168, 2023-09-23T20:24:11.903 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:24:11.903 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:24:11.903 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:24:11.904 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:24:11.904 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:24:11.904 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 39488 2023-09-23T20:24:11.904 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.904 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:24:11.904 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.904 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:24:11.904 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.340478+0000", 2023-09-23T20:24:11.904 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:24:11.904 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.904 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.905 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:24:11.905 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.340478+0000", 2023-09-23T20:24:11.905 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:24:11.905 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.905 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.905 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:24:11.905 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.340477+0000", 2023-09-23T20:24:11.905 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:24:11.905 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.905 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.906 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:24:11.906 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.340527+0000", 2023-09-23T20:24:11.906 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.0659999999999999e-05 2023-09-23T20:24:11.906 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.906 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.906 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:24:11.906 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.340543+0000", 2023-09-23T20:24:11.906 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.5267000000000001e-05 2023-09-23T20:24:11.906 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.906 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.907 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:24:11.907 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.340549+0000", 2023-09-23T20:24:11.907 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.63e-06 2023-09-23T20:24:11.907 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.907 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.907 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:24:11.907 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.340599+0000", 2023-09-23T20:24:11.907 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.9500999999999999e-05 2023-09-23T20:24:11.907 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.907 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.908 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:24:11.908 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.340606+0000", 2023-09-23T20:24:11.908 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.4490000000000002e-06 2023-09-23T20:24:11.908 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.908 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.908 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:24:11.908 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:17.041884+0000", 2023-09-23T20:24:11.908 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.70127816099999996 2023-09-23T20:24:11.908 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.908 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.908 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:24:11.909 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:17.041888+0000", 2023-09-23T20:24:11.909 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.8580000000000002e-06 2023-09-23T20:24:11.909 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:24:11.909 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:24:11.909 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:24:11.909 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.909 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.909 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:39536 5.10 5.6cf39eb0 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e120)", 2023-09-23T20:24:11.909 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:16.392766+0000", 2023-09-23T20:24:11.909 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 55.423576932000003, 2023-09-23T20:24:11.910 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.649134455, 2023-09-23T20:24:11.910 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:24:11.910 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:24:11.910 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:24:11.910 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:24:11.910 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:24:11.910 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 39536 2023-09-23T20:24:11.910 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.910 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:24:11.910 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.910 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:24:11.911 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.392766+0000", 2023-09-23T20:24:11.911 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:24:11.911 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.911 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.911 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:24:11.911 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.392766+0000", 2023-09-23T20:24:11.911 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:24:11.911 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.911 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.911 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:24:11.911 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.392766+0000", 2023-09-23T20:24:11.912 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:24:11.912 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.912 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.912 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:24:11.912 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.392783+0000", 2023-09-23T20:24:11.912 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.7799e-05 2023-09-23T20:24:11.912 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.912 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.912 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:24:11.912 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.392795+0000", 2023-09-23T20:24:11.913 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.2002e-05 2023-09-23T20:24:11.913 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.913 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.913 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:24:11.913 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.392815+0000", 2023-09-23T20:24:11.913 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.9208999999999999e-05 2023-09-23T20:24:11.913 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.913 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.913 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:24:11.913 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.392863+0000", 2023-09-23T20:24:11.914 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.8807000000000002e-05 2023-09-23T20:24:11.914 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.914 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.914 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:24:11.914 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.392871+0000", 2023-09-23T20:24:11.914 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.4100000000000002e-06 2023-09-23T20:24:11.914 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.914 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.914 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:24:11.915 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:17.041898+0000", 2023-09-23T20:24:11.915 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.64902672900000002 2023-09-23T20:24:11.915 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.915 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.915 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:24:11.915 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:17.041901+0000", 2023-09-23T20:24:11.915 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.2799999999999999e-06 2023-09-23T20:24:11.915 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:24:11.915 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:24:11.916 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:24:11.916 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.916 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.916 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:39555 5.10 5.575b1e10 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e120)", 2023-09-23T20:24:11.916 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:16.410224+0000", 2023-09-23T20:24:11.916 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 55.406118786999997, 2023-09-23T20:24:11.916 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.63169022600000002, 2023-09-23T20:24:11.916 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:24:11.916 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:24:11.916 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:24:11.917 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:24:11.917 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:24:11.917 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 39555 2023-09-23T20:24:11.917 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.917 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:24:11.917 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.917 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:24:11.917 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.410224+0000", 2023-09-23T20:24:11.917 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:24:11.917 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.917 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.918 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:24:11.918 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.410224+0000", 2023-09-23T20:24:11.918 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:24:11.918 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.918 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.918 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:24:11.918 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.410224+0000", 2023-09-23T20:24:11.918 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:24:11.918 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.919 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.919 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:24:11.919 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.410345+0000", 2023-09-23T20:24:11.919 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00012100900000000001 2023-09-23T20:24:11.919 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.919 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.919 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:24:11.919 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.410356+0000", 2023-09-23T20:24:11.919 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1387e-05 2023-09-23T20:24:11.920 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.920 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.920 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:24:11.920 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.410362+0000", 2023-09-23T20:24:11.920 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.1500000000000004e-06 2023-09-23T20:24:11.920 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.920 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.920 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:24:11.920 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.410417+0000", 2023-09-23T20:24:11.921 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.5081999999999999e-05 2023-09-23T20:24:11.921 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.921 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.921 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:24:11.921 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.410425+0000", 2023-09-23T20:24:11.921 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.6990000000000003e-06 2023-09-23T20:24:11.921 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.921 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.921 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:24:11.921 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:17.041911+0000", 2023-09-23T20:24:11.922 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.63148628600000001 2023-09-23T20:24:11.922 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.922 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.922 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:24:11.922 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:17.041915+0000", 2023-09-23T20:24:11.922 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.4070000000000002e-06 2023-09-23T20:24:11.922 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:24:11.922 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:24:11.923 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:24:11.923 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.923 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.923 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:39663 5.10 5.2022fef0 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e120)", 2023-09-23T20:24:11.923 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:16.512807+0000", 2023-09-23T20:24:11.923 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 55.303535855, 2023-09-23T20:24:11.923 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.52913794800000002, 2023-09-23T20:24:11.923 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:24:11.923 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:24:11.924 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:24:11.924 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:24:11.924 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:24:11.924 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 39663 2023-09-23T20:24:11.924 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.924 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:24:11.924 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.924 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:24:11.924 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.512807+0000", 2023-09-23T20:24:11.924 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:24:11.924 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.925 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.925 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:24:11.925 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.512807+0000", 2023-09-23T20:24:11.925 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:24:11.925 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.925 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.925 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:24:11.925 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.512806+0000", 2023-09-23T20:24:11.925 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:24:11.925 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.926 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.926 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:24:11.926 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.512855+0000", 2023-09-23T20:24:11.926 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.8955000000000001e-05 2023-09-23T20:24:11.926 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.926 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.926 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:24:11.926 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.512869+0000", 2023-09-23T20:24:11.926 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.3247999999999999e-05 2023-09-23T20:24:11.926 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.927 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.927 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:24:11.927 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.512874+0000", 2023-09-23T20:24:11.927 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.4600000000000002e-06 2023-09-23T20:24:11.927 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.927 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.927 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:24:11.927 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.512930+0000", 2023-09-23T20:24:11.927 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.5869000000000003e-05 2023-09-23T20:24:11.927 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.927 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.928 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:24:11.928 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.512940+0000", 2023-09-23T20:24:11.928 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.9520000000000006e-06 2023-09-23T20:24:11.928 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.928 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.928 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:24:11.928 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:17.041937+0000", 2023-09-23T20:24:11.928 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.52899666099999998 2023-09-23T20:24:11.928 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.928 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.928 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:24:11.929 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:17.041945+0000", 2023-09-23T20:24:11.929 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.8270000000000004e-06 2023-09-23T20:24:11.929 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:24:11.929 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:24:11.929 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:24:11.929 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.929 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.929 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:39688 5.10 5.190a9f30 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e120)", 2023-09-23T20:24:11.929 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:16.539222+0000", 2023-09-23T20:24:11.929 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 55.277121250999997, 2023-09-23T20:24:11.929 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.50273684200000002, 2023-09-23T20:24:11.930 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:24:11.930 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:24:11.930 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:24:11.930 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:24:11.930 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:24:11.930 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 39688 2023-09-23T20:24:11.930 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.930 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:24:11.930 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.930 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:24:11.931 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.539222+0000", 2023-09-23T20:24:11.931 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:24:11.931 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.931 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.931 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:24:11.931 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.539222+0000", 2023-09-23T20:24:11.931 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:24:11.931 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.931 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.931 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:24:11.932 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.539220+0000", 2023-09-23T20:24:11.932 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:24:11.932 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.932 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.932 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:24:11.932 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.539277+0000", 2023-09-23T20:24:11.932 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.6338000000000002e-05 2023-09-23T20:24:11.932 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.932 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.932 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:24:11.932 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.539293+0000", 2023-09-23T20:24:11.933 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.6657999999999999e-05 2023-09-23T20:24:11.933 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.933 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.933 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:24:11.933 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.539300+0000", 2023-09-23T20:24:11.933 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.7730000000000002e-06 2023-09-23T20:24:11.933 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.933 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.933 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:24:11.933 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.539376+0000", 2023-09-23T20:24:11.933 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.5721000000000002e-05 2023-09-23T20:24:11.934 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.934 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.934 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:24:11.934 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.539382+0000", 2023-09-23T20:24:11.934 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.9279999999999998e-06 2023-09-23T20:24:11.934 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.934 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.934 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:24:11.934 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:17.041955+0000", 2023-09-23T20:24:11.934 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.50257318399999995 2023-09-23T20:24:11.935 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.935 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.935 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:24:11.935 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:17.041959+0000", 2023-09-23T20:24:11.935 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.8e-06 2023-09-23T20:24:11.935 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:24:11.935 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:24:11.935 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:24:11.935 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.935 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.936 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:39689 5.10 5.fcdff2b0 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e120)", 2023-09-23T20:24:11.936 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:16.541271+0000", 2023-09-23T20:24:11.936 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 55.275072391999998, 2023-09-23T20:24:11.936 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.50071250700000003, 2023-09-23T20:24:11.936 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:24:11.936 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:24:11.936 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:24:11.936 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:24:11.936 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:24:11.936 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 39689 2023-09-23T20:24:11.937 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.937 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:24:11.937 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.937 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:24:11.937 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.541271+0000", 2023-09-23T20:24:11.937 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:24:11.937 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.937 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.937 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:24:11.937 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.541271+0000", 2023-09-23T20:24:11.938 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:24:11.938 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.938 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.938 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:24:11.938 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.541270+0000", 2023-09-23T20:24:11.938 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:24:11.938 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.938 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.938 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:24:11.939 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.541314+0000", 2023-09-23T20:24:11.939 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.3504999999999998e-05 2023-09-23T20:24:11.939 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.939 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.939 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:24:11.939 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.541334+0000", 2023-09-23T20:24:11.939 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.0132e-05 2023-09-23T20:24:11.939 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.939 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.939 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:24:11.939 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.541339+0000", 2023-09-23T20:24:11.940 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.4620000000000004e-06 2023-09-23T20:24:11.940 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.940 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.940 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:24:11.940 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.541375+0000", 2023-09-23T20:24:11.940 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.6004999999999998e-05 2023-09-23T20:24:11.940 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.940 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.940 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:24:11.940 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:16.541380+0000", 2023-09-23T20:24:11.940 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.1800000000000004e-06 2023-09-23T20:24:11.941 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.941 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.941 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:24:11.941 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:17.041974+0000", 2023-09-23T20:24:11.941 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.50059391499999994 2023-09-23T20:24:11.941 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.941 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.941 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:24:11.941 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:17.041983+0000", 2023-09-23T20:24:11.942 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.1730000000000001e-06 2023-09-23T20:24:11.942 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:24:11.942 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:24:11.942 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:24:11.942 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.942 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.942 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:63124 5.4 5:2334d985:::benchmark_data_smithi018_228451_object63123:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e140)", 2023-09-23T20:24:11.942 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:45.222279+0000", 2023-09-23T20:24:11.942 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 26.594064629999998, 2023-09-23T20:24:11.942 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.040489402000000001, 2023-09-23T20:24:11.943 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:24:11.943 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:24:11.943 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:24:11.943 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:24:11.943 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:24:11.943 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 63124 2023-09-23T20:24:11.943 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.943 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:24:11.943 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.943 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:24:11.944 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:45.222279+0000", 2023-09-23T20:24:11.944 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:24:11.944 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.944 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.944 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:24:11.944 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:45.222279+0000", 2023-09-23T20:24:11.944 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:24:11.944 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.944 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.944 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:24:11.944 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:45.222278+0000", 2023-09-23T20:24:11.945 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:24:11.945 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.945 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.945 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:24:11.945 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:45.222344+0000", 2023-09-23T20:24:11.945 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.6161999999999994e-05 2023-09-23T20:24:11.945 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.945 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.945 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:24:11.945 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:45.222354+0000", 2023-09-23T20:24:11.945 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0329000000000001e-05 2023-09-23T20:24:11.946 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.946 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.946 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:24:11.946 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:45.222361+0000", 2023-09-23T20:24:11.946 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.3289999999999999e-06 2023-09-23T20:24:11.946 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.946 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.946 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:24:11.946 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:45.222395+0000", 2023-09-23T20:24:11.946 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.4956000000000002e-05 2023-09-23T20:24:11.947 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.947 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.947 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:24:11.947 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:45.222508+0000", 2023-09-23T20:24:11.947 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000112212 2023-09-23T20:24:11.947 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.947 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.947 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 2,5", 2023-09-23T20:24:11.947 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:45.222642+0000", 2023-09-23T20:24:11.947 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00013433499999999999 2023-09-23T20:24:11.947 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.948 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.948 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:24:11.948 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:45.223759+0000", 2023-09-23T20:24:11.948 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.001117119 2023-09-23T20:24:11.948 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.948 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.948 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:24:11.948 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:45.225249+0000", 2023-09-23T20:24:11.948 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.001489752 2023-09-23T20:24:11.948 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.949 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.949 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:24:11.949 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:45.262681+0000", 2023-09-23T20:24:11.949 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.037432278999999999 2023-09-23T20:24:11.949 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.949 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.949 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:24:11.949 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:45.262731+0000", 2023-09-23T20:24:11.949 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.9582000000000001e-05 2023-09-23T20:24:11.949 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.950 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.950 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:24:11.950 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:45.262768+0000", 2023-09-23T20:24:11.950 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.7299999999999999e-05 2023-09-23T20:24:11.950 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:24:11.950 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:24:11.950 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:24:11.950 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.950 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.951 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:63576 5.4 5:21c68274:::benchmark_data_smithi018_228451_object63575:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e140)", 2023-09-23T20:24:11.951 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:45.610086+0000", 2023-09-23T20:24:11.951 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 26.206257252, 2023-09-23T20:24:11.951 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.043691071999999997, 2023-09-23T20:24:11.951 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:24:11.951 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:24:11.951 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:24:11.951 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:24:11.951 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:24:11.951 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 63576 2023-09-23T20:24:11.951 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.952 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:24:11.952 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.952 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:24:11.952 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:45.610086+0000", 2023-09-23T20:24:11.952 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:24:11.952 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.952 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.952 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:24:11.952 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:45.610086+0000", 2023-09-23T20:24:11.952 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:24:11.952 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.953 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.953 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:24:11.953 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:45.610086+0000", 2023-09-23T20:24:11.953 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:24:11.953 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.953 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.953 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:24:11.953 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:45.610143+0000", 2023-09-23T20:24:11.953 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.7262000000000002e-05 2023-09-23T20:24:11.953 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.954 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.954 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:24:11.954 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:45.610150+0000", 2023-09-23T20:24:11.954 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.1060000000000002e-06 2023-09-23T20:24:11.954 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.954 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.954 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:24:11.954 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:45.610155+0000", 2023-09-23T20:24:11.954 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.989e-06 2023-09-23T20:24:11.954 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.955 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.955 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:24:11.955 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:45.610181+0000", 2023-09-23T20:24:11.955 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.6304000000000001e-05 2023-09-23T20:24:11.955 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.955 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.955 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:24:11.955 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:45.610254+0000", 2023-09-23T20:24:11.955 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.2334000000000002e-05 2023-09-23T20:24:11.955 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.956 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.956 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 2,5", 2023-09-23T20:24:11.956 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:45.610351+0000", 2023-09-23T20:24:11.956 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.7496000000000003e-05 2023-09-23T20:24:11.956 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.956 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.956 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:24:11.956 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:45.611615+0000", 2023-09-23T20:24:11.956 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0012641480000000001 2023-09-23T20:24:11.956 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.957 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.957 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:24:11.957 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:45.612442+0000", 2023-09-23T20:24:11.957 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00082655999999999995 2023-09-23T20:24:11.957 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.957 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.957 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:24:11.957 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:45.653703+0000", 2023-09-23T20:24:11.957 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.041261680000000002 2023-09-23T20:24:11.957 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.958 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.958 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:24:11.958 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:45.653740+0000", 2023-09-23T20:24:11.958 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.6868e-05 2023-09-23T20:24:11.958 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:24:11.958 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:24:11.958 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:24:11.958 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:45.653777+0000", 2023-09-23T20:24:11.958 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.6789000000000003e-05 2023-09-23T20:24:11.959 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:24:11.959 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:24:11.959 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:24:11.959 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:24:11.959 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:24:11.959 INFO:teuthology.orchestra.run.smithi149.stdout:} 2023-09-23T20:24:11.960 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:11.960 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:11.961 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:24:11.961 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:24:12.061 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:24:12.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:12 smithi018 ceph-mon[184144]: from='client.54673 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:24:12.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:12 smithi018 ceph-mon[179647]: from='client.54673 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:24:12.150 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:12.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:12 smithi149 ceph-mon[170394]: from='client.54673 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:24:12.265 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T20:24:12.322 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:24:12.333 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:12.334 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:24:12.434 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:24:12.485 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 74 16 90763 90747 76.6099 63.8125 0.00791879 0.0130274 2023-09-23T20:24:12.497 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:24:12.522 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:12.625 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:24:12.651 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:24:12.654 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:12.655 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:24:12.658 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:24:12.694 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:24:12.725 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:24:12.756 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:24:12.757 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:24:12.787 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:24:12.798 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:12.813 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:24:12.835 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:24:12.837 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:24:12.837 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:24:12.837 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:24:12.837 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:24:12.837 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:24:12.838 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:24:12.838 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:24:12.838 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:24:12.838 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:24:12.840 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:12.841 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:24:12.941 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:24:12.978 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:13.097 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:13 smithi018 ceph-mon[184144]: pgmap v310: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s; 4758/260688 objects degraded (1.825%); 4990/260688 objects misplaced (1.914%); 6.5 MiB/s, 103 objects/s recovering 2023-09-23T20:24:13.097 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:13 smithi018 ceph-mon[184144]: osdmap e152: 8 total, 8 up, 8 in 2023-09-23T20:24:13.097 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:13 smithi018 ceph-mon[184144]: failed to encode map e152 with expected crc 2023-09-23T20:24:13.097 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:13 smithi018 ceph-mon[184144]: failed to encode map e152 with expected crc 2023-09-23T20:24:13.098 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:13 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2809277634' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:24:13.098 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:13 smithi018 ceph-mon[179647]: pgmap v310: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s; 4758/260688 objects degraded (1.825%); 4990/260688 objects misplaced (1.914%); 6.5 MiB/s, 103 objects/s recovering 2023-09-23T20:24:13.098 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:13 smithi018 ceph-mon[179647]: osdmap e152: 8 total, 8 up, 8 in 2023-09-23T20:24:13.098 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:13 smithi018 ceph-mon[179647]: failed to encode map e152 with expected crc 2023-09-23T20:24:13.098 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:13 smithi018 ceph-mon[179647]: failed to encode map e152 with expected crc 2023-09-23T20:24:13.098 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:13 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2809277634' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:24:13.272 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:24:13.276 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:13.277 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:24:13.377 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:24:13.449 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:13.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:13 smithi149 ceph-mon[170394]: pgmap v310: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s; 4758/260688 objects degraded (1.825%); 4990/260688 objects misplaced (1.914%); 6.5 MiB/s, 103 objects/s recovering 2023-09-23T20:24:13.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:13 smithi149 ceph-mon[170394]: osdmap e152: 8 total, 8 up, 8 in 2023-09-23T20:24:13.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:13 smithi149 ceph-mon[170394]: failed to encode map e152 with expected crc 2023-09-23T20:24:13.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:13 smithi149 ceph-mon[170394]: failed to encode map e152 with expected crc 2023-09-23T20:24:13.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:13 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2809277634' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:24:13.485 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 75 16 91355 91339 76.0819 37 0.00995536 0.013139 2023-09-23T20:24:13.519 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:24:13.522 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:13.523 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:24:13.618 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T20:24:13.624 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:24:13.687 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:13.758 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:24:13.763 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:13.763 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:24:13.778 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:24:13.864 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:24:13.964 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:14.179 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:24:14.188 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:14.189 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:24:14.290 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:24:14.298 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:14 smithi018 ceph-mon[184144]: failed to encode map e153 with expected crc 2023-09-23T20:24:14.298 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:14 smithi018 ceph-mon[184144]: failed to encode map e153 with expected crc 2023-09-23T20:24:14.298 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:14 smithi018 ceph-mon[184144]: osdmap e153: 8 total, 8 up, 8 in 2023-09-23T20:24:14.298 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:14 smithi018 ceph-mon[184144]: failed to encode map e153 with expected crc 2023-09-23T20:24:14.298 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:14 smithi018 ceph-mon[184144]: failed to encode map e153 with expected crc 2023-09-23T20:24:14.299 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:14 smithi018 ceph-mon[179647]: failed to encode map e153 with expected crc 2023-09-23T20:24:14.299 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:14 smithi018 ceph-mon[179647]: failed to encode map e153 with expected crc 2023-09-23T20:24:14.299 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:14 smithi018 ceph-mon[179647]: osdmap e153: 8 total, 8 up, 8 in 2023-09-23T20:24:14.299 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:14 smithi018 ceph-mon[179647]: failed to encode map e153 with expected crc 2023-09-23T20:24:14.299 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:14 smithi018 ceph-mon[179647]: failed to encode map e153 with expected crc 2023-09-23T20:24:14.333 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:14.381 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:24:14.386 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:14.386 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:24:14.387 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:14 smithi149 ceph-mon[170394]: failed to encode map e153 with expected crc 2023-09-23T20:24:14.387 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:14 smithi149 ceph-mon[170394]: failed to encode map e153 with expected crc 2023-09-23T20:24:14.387 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:14 smithi149 ceph-mon[170394]: osdmap e153: 8 total, 8 up, 8 in 2023-09-23T20:24:14.387 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:14 smithi149 ceph-mon[170394]: failed to encode map e153 with expected crc 2023-09-23T20:24:14.387 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:14 smithi149 ceph-mon[170394]: failed to encode map e153 with expected crc 2023-09-23T20:24:14.485 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 76 16 92439 92423 75.9722 67.75 0.0190925 0.0131588 2023-09-23T20:24:14.487 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:24:14.543 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:14.603 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:24:14.613 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:14.613 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:24:14.714 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:24:14.768 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:14.835 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:24:14.842 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:14.842 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:24:14.943 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:24:14.950 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:24:14.981 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:15.032 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:24:15.035 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:15.036 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:24:15.048 INFO:teuthology.orchestra.run.smithi018.stdout:moved 26 / 297 (8.75421%) 2023-09-23T20:24:15.048 INFO:teuthology.orchestra.run.smithi018.stdout:avg 37.125 2023-09-23T20:24:15.048 INFO:teuthology.orchestra.run.smithi018.stdout:stddev 5.64441 -> 6.31343 (expected baseline 5.69951) 2023-09-23T20:24:15.048 INFO:teuthology.orchestra.run.smithi018.stdout:min osd.2 with 26 -> 32 pgs (0.700337 -> 0.861953 * mean) 2023-09-23T20:24:15.049 INFO:teuthology.orchestra.run.smithi018.stdout:max osd.4 with 45 -> 37 pgs (1.21212 -> 0.996633 * mean) 2023-09-23T20:24:15.049 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:24:15.049 INFO:teuthology.orchestra.run.smithi018.stdout:oload 110 2023-09-23T20:24:15.049 INFO:teuthology.orchestra.run.smithi018.stdout:max_change 1 2023-09-23T20:24:15.049 INFO:teuthology.orchestra.run.smithi018.stdout:max_change_osds 4 2023-09-23T20:24:15.049 INFO:teuthology.orchestra.run.smithi018.stdout:average_utilization 0.0243 2023-09-23T20:24:15.049 INFO:teuthology.orchestra.run.smithi018.stdout:overload_utilization 0.0267 2023-09-23T20:24:15.049 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5 weight 1.0000 -> 0.7157 2023-09-23T20:24:15.049 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0 weight 1.0000 -> 0.7997 2023-09-23T20:24:15.049 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4 weight 1.0000 -> 0.8014 2023-09-23T20:24:15.052 INFO:teuthology.orchestra.run.smithi018.stderr:no change 2023-09-23T20:24:15.137 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:24:15.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:15 smithi018 ceph-mon[184144]: pgmap v313: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 1 active+recovery_wait, 95 active+clean; 5.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 70 MiB/s wr, 1.13k op/s; 2426/268185 objects degraded (0.905%); 7172/268185 objects misplaced (2.674%); 5.2 MiB/s, 83 objects/s recovering 2023-09-23T20:24:15.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:15 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:24:15.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:15 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:24:15.142 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:24:14 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:24:14] "GET /metrics HTTP/1.1" 200 36472 "" "Prometheus/2.43.0" 2023-09-23T20:24:15.142 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:24:15 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: 2023-09-23T20:24:15.046+0000 7f953be6a700 -1 mgr.server reply reply (3) No such process no change 2023-09-23T20:24:15.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:15 smithi018 ceph-mon[179647]: pgmap v313: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 1 active+recovery_wait, 95 active+clean; 5.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 70 MiB/s wr, 1.13k op/s; 2426/268185 objects degraded (0.905%); 7172/268185 objects misplaced (2.674%); 5.2 MiB/s, 83 objects/s recovering 2023-09-23T20:24:15.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:15 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:24:15.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:15 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:24:15.192 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:15.258 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:24:15.269 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:15.270 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:24:15.370 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:24:15.458 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:15.460 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:24:15.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:15 smithi149 ceph-mon[170394]: pgmap v313: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 1 active+recovery_wait, 95 active+clean; 5.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 70 MiB/s wr, 1.13k op/s; 2426/268185 objects degraded (0.905%); 7172/268185 objects misplaced (2.674%); 5.2 MiB/s, 83 objects/s recovering 2023-09-23T20:24:15.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:15 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:24:15.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:15 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:24:15.485 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 77 16 93533 93517 75.8735 68.375 0.033803 0.0131745 2023-09-23T20:24:15.575 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:24:15.583 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:15.583 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:24:15.684 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:24:15.818 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:15.876 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '1000', 'type': 'reweight-by-utilization'} 2023-09-23T20:24:15.876 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd reweight-by-utilization 1000 1.0 2023-09-23T20:24:15.950 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:24:15.959 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:15.960 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:24:16.061 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:24:16.100 INFO:teuthology.orchestra.run.smithi018.stdout:ERROR: (22) Invalid argument 2023-09-23T20:24:16.100 INFO:teuthology.orchestra.run.smithi018.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:24:16.128 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:16 smithi018 ceph-mon[184144]: from='client.34905 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:24:16.128 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:16 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 2426/268185 objects degraded (0.905%), 1 pg degraded (PG_DEGRADED) 2023-09-23T20:24:16.128 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:16 smithi018 ceph-mon[179647]: from='client.34905 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:24:16.128 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:16 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 2426/268185 objects degraded (0.905%), 1 pg degraded (PG_DEGRADED) 2023-09-23T20:24:16.152 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:16.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:16 smithi149 ceph-mon[170394]: from='client.34905 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:24:16.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:16 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 2426/268185 objects degraded (0.905%), 1 pg degraded (PG_DEGRADED) 2023-09-23T20:24:16.485 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 78 16 94568 94552 75.7301 64.6875 0.00772522 0.0132004 2023-09-23T20:24:17.394 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:17 smithi018 ceph-mon[179647]: pgmap v314: 99 pgs: 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 96 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 86 MiB/s wr, 1.37k op/s; 2426/276237 objects degraded (0.878%); 6940/276237 objects misplaced (2.512%); 6.4 MiB/s, 102 objects/s recovering 2023-09-23T20:24:17.394 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:17 smithi018 ceph-mon[184144]: pgmap v314: 99 pgs: 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 96 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 86 MiB/s wr, 1.37k op/s; 2426/276237 objects degraded (0.878%); 6940/276237 objects misplaced (2.512%); 6.4 MiB/s, 102 objects/s recovering 2023-09-23T20:24:17.468 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:17 smithi149 ceph-mon[170394]: pgmap v314: 99 pgs: 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 96 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 86 MiB/s wr, 1.37k op/s; 2426/276237 objects degraded (0.878%); 6940/276237 objects misplaced (2.512%); 6.4 MiB/s, 102 objects/s recovering 2023-09-23T20:24:17.485 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 79 16 95805 95789 75.75 77.3125 0.00669189 0.0131964 2023-09-23T20:24:18.487 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:2023-09-23T20:24:18.487474+0000 min lat: 0.00112599 max lat: 2.83721 avg lat: 0.0131952 2023-09-23T20:24:18.487 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T20:24:18.487 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 80 16 97031 97015 75.7598 76.625 0.00856688 0.0131952 2023-09-23T20:24:18.970 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:24:18.979 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:18.980 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:24:19.081 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:24:19.147 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:19.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:19 smithi018 ceph-mon[184144]: pgmap v315: 99 pgs: 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 96 active+clean; 5.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s; 2426/279912 objects degraded (0.867%); 6940/279912 objects misplaced (2.479%); 5.2 MiB/s, 83 objects/s recovering 2023-09-23T20:24:19.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:19 smithi018 ceph-mon[179647]: pgmap v315: 99 pgs: 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 96 active+clean; 5.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s; 2426/279912 objects degraded (0.867%); 6940/279912 objects misplaced (2.479%); 5.2 MiB/s, 83 objects/s recovering 2023-09-23T20:24:19.468 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:19 smithi149 ceph-mon[170394]: pgmap v315: 99 pgs: 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 96 active+clean; 5.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s; 2426/279912 objects degraded (0.867%); 6940/279912 objects misplaced (2.479%); 5.2 MiB/s, 83 objects/s recovering 2023-09-23T20:24:19.486 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 81 16 98190 98174 75.7187 72.4375 0.0127625 0.0132019 2023-09-23T20:24:19.909 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:24:19.911 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:19.912 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:24:20.013 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:24:20.074 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:20.203 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:20 smithi018 ceph-mon[179647]: pgmap v316: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 96 active+clean; 5.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 59 MiB/s wr, 949 op/s; 2426/283488 objects degraded (0.856%); 6392/283488 objects misplaced (2.255%); 3.6 MiB/s, 58 objects/s recovering 2023-09-23T20:24:20.203 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:20 smithi018 ceph-mon[184144]: pgmap v316: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 96 active+clean; 5.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 59 MiB/s wr, 949 op/s; 2426/283488 objects degraded (0.856%); 6392/283488 objects misplaced (2.255%); 3.6 MiB/s, 58 objects/s recovering 2023-09-23T20:24:20.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:20 smithi149 ceph-mon[170394]: pgmap v316: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 96 active+clean; 5.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 59 MiB/s wr, 949 op/s; 2426/283488 objects degraded (0.856%); 6392/283488 objects misplaced (2.255%); 3.6 MiB/s, 58 objects/s recovering 2023-09-23T20:24:20.487 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 82 16 99316 99300 75.6535 70.375 0.013275 0.0132133 2023-09-23T20:24:20.603 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:24:20.618 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:20.619 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:24:20.648 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T20:24:20.720 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:24:20.824 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:20.904 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:24:20.910 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:24:21.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:21 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 2426/283488 objects degraded (0.856%), 1 pg degraded (PG_DEGRADED) 2023-09-23T20:24:21.488 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 83 16 100422 100406 75.5738 69.125 0.014279 0.0132268 2023-09-23T20:24:21.647 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:21 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 2426/283488 objects degraded (0.856%), 1 pg degraded (PG_DEGRADED) 2023-09-23T20:24:21.647 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:21 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 2426/283488 objects degraded (0.856%), 1 pg degraded (PG_DEGRADED) 2023-09-23T20:24:21.669 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:24:21.679 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:21.680 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:24:21.781 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:24:21.838 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:22.015 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:24:22.038 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:22.038 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:24:22.139 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:24:22.190 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:22.246 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:24:22.250 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:22.250 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:24:22.276 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:22 smithi018 ceph-mon[179647]: pgmap v317: 99 pgs: 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 96 active+clean; 6.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 62 MiB/s wr, 991 op/s; 2426/292773 objects degraded (0.829%); 6392/292773 objects misplaced (2.183%); 3.1 MiB/s, 49 objects/s recovering 2023-09-23T20:24:22.276 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:22 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:24:22.276 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:22 smithi018 ceph-mon[184144]: pgmap v317: 99 pgs: 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 96 active+clean; 6.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 62 MiB/s wr, 991 op/s; 2426/292773 objects degraded (0.829%); 6392/292773 objects misplaced (2.183%); 3.1 MiB/s, 49 objects/s recovering 2023-09-23T20:24:22.277 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:22 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:24:22.351 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:24:22.391 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:22.446 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:24:22.451 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:22.451 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:24:22.451 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:22 smithi149 ceph-mon[170394]: pgmap v317: 99 pgs: 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 96 active+clean; 6.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 62 MiB/s wr, 991 op/s; 2426/292773 objects degraded (0.829%); 6392/292773 objects misplaced (2.183%); 3.1 MiB/s, 49 objects/s recovering 2023-09-23T20:24:22.452 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:22 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:24:22.489 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 84 16 101342 101326 75.3572 57.5 0.00930759 0.0132651 2023-09-23T20:24:22.552 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:24:22.622 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:23.077 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:24:23.082 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:23.082 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:24:23.156 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:24:23.183 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:24:23.260 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:23 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1711832200' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:24:23.260 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:23 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:24:23.261 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:23 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1711832200' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:24:23.261 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:23 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:24:23.269 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:23.275 INFO:teuthology.orchestra.run.smithi018.stderr:nodeep-scrub is unset 2023-09-23T20:24:23.490 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 85 16 102282 102266 75.1618 58.75 0.0196457 0.0132997 2023-09-23T20:24:23.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:23 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1711832200' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:24:23.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:23 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:24:23.817 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:24:23.826 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:23.826 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:24:23.927 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:24:23.989 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:24.085 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:24:24.092 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:24.093 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:24:24.153 INFO:teuthology.orchestra.run.smithi018.stdout:moved 0 / 297 (0%) 2023-09-23T20:24:24.153 INFO:teuthology.orchestra.run.smithi018.stdout:avg 37.125 2023-09-23T20:24:24.153 INFO:teuthology.orchestra.run.smithi018.stdout:stddev 5.64441 -> 5.64441 (expected baseline 5.69951) 2023-09-23T20:24:24.154 INFO:teuthology.orchestra.run.smithi018.stdout:min osd.2 with 26 -> 26 pgs (0.700337 -> 0.700337 * mean) 2023-09-23T20:24:24.154 INFO:teuthology.orchestra.run.smithi018.stdout:max osd.4 with 45 -> 45 pgs (1.21212 -> 1.21212 * mean) 2023-09-23T20:24:24.154 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:24:24.154 INFO:teuthology.orchestra.run.smithi018.stdout:oload 1000 2023-09-23T20:24:24.154 INFO:teuthology.orchestra.run.smithi018.stdout:max_change 1 2023-09-23T20:24:24.154 INFO:teuthology.orchestra.run.smithi018.stdout:max_change_osds 4 2023-09-23T20:24:24.154 INFO:teuthology.orchestra.run.smithi018.stdout:average_utilization 0.0271 2023-09-23T20:24:24.154 INFO:teuthology.orchestra.run.smithi018.stdout:overload_utilization 0.2710 2023-09-23T20:24:24.156 INFO:teuthology.orchestra.run.smithi018.stderr:no change 2023-09-23T20:24:24.194 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:24:24.274 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:24 smithi018 ceph-mon[179647]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:24:24.274 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:24 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:24:24.274 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:24 smithi018 ceph-mon[179647]: osdmap e154: 8 total, 8 up, 8 in 2023-09-23T20:24:24.274 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:24 smithi018 ceph-mon[179647]: failed to encode map e154 with expected crc 2023-09-23T20:24:24.274 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:24 smithi018 ceph-mon[179647]: pgmap v319: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 96 active+clean; 6.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s; 2426/300510 objects degraded (0.807%); 5824/300510 objects misplaced (1.938%); 4.2 MiB/s, 67 objects/s recovering 2023-09-23T20:24:24.275 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:24 smithi018 ceph-mon[179647]: from='client.54694 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:24:24.275 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:24 smithi018 ceph-mon[184144]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:24:24.275 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:24 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:24:24.275 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:24 smithi018 ceph-mon[184144]: osdmap e154: 8 total, 8 up, 8 in 2023-09-23T20:24:24.275 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:24 smithi018 ceph-mon[184144]: failed to encode map e154 with expected crc 2023-09-23T20:24:24.275 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:24 smithi018 ceph-mon[184144]: pgmap v319: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 96 active+clean; 6.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s; 2426/300510 objects degraded (0.807%); 5824/300510 objects misplaced (1.938%); 4.2 MiB/s, 67 objects/s recovering 2023-09-23T20:24:24.275 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:24 smithi018 ceph-mon[184144]: from='client.54694 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:24:24.277 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:24.490 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 86 16 103279 103263 75.0124 62.3125 0.0221863 0.0133267 2023-09-23T20:24:24.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:24 smithi149 ceph-mon[170394]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:24:24.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:24 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:24:24.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:24 smithi149 ceph-mon[170394]: osdmap e154: 8 total, 8 up, 8 in 2023-09-23T20:24:24.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:24 smithi149 ceph-mon[170394]: failed to encode map e154 with expected crc 2023-09-23T20:24:24.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:24 smithi149 ceph-mon[170394]: pgmap v319: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 96 active+clean; 6.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s; 2426/300510 objects degraded (0.807%); 5824/300510 objects misplaced (1.938%); 4.2 MiB/s, 67 objects/s recovering 2023-09-23T20:24:24.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:24 smithi149 ceph-mon[170394]: from='client.54694 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:24:24.726 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:24:24.772 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:24:24.820 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:24:24.841 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:24:24.846 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:24.846 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:24:24.858 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:24:24.923 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:24:24.947 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:24:24.957 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:24:25.001 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:24:25.011 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:25.050 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:24:25.052 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:24:25.052 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:24:25.052 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:24:25.052 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:24:25.052 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:24:25.052 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:24:25.052 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:24:25.052 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:24:25.096 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:24:25.104 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:25.104 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:24:25.144 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:24:24 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:24:24] "GET /metrics HTTP/1.1" 200 36472 "" "Prometheus/2.43.0" 2023-09-23T20:24:25.205 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:24:25.261 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:25.397 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:24:25.411 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:25.411 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:25 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3487863877' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:24:25.412 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:24:25.490 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 87 16 104239 104223 74.8398 60 0.020764 0.0133573 2023-09-23T20:24:25.494 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:25 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3487863877' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:24:25.494 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:25 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3487863877' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:24:25.513 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:24:25.528 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2023-09-23T20:24:25.528 INFO:teuthology.orchestra.run.smithi018.stdout: "ops": [], 2023-09-23T20:24:25.528 INFO:teuthology.orchestra.run.smithi018.stdout: "complaint_time": 30, 2023-09-23T20:24:25.528 INFO:teuthology.orchestra.run.smithi018.stdout: "num_blocked_ops": 0 2023-09-23T20:24:25.528 INFO:teuthology.orchestra.run.smithi018.stdout:} 2023-09-23T20:24:25.582 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:25.736 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:24:25.741 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:25.741 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:24:25.842 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:24:25.915 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:26.038 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T20:24:26.105 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:24:26.117 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:26.118 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:24:26.137 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T20:24:26.218 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:24:26.331 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:26.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:26 smithi018 ceph-mon[184144]: pgmap v320: 99 pgs: 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 96 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s; 2426/308298 objects degraded (0.787%); 5824/308298 objects misplaced (1.889%); 3.5 MiB/s, 55 objects/s recovering 2023-09-23T20:24:26.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:26 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 2426/300510 objects degraded (0.807%), 1 pg degraded (PG_DEGRADED) 2023-09-23T20:24:26.395 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:26 smithi018 ceph-mon[179647]: pgmap v320: 99 pgs: 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 96 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s; 2426/308298 objects degraded (0.787%); 5824/308298 objects misplaced (1.889%); 3.5 MiB/s, 55 objects/s recovering 2023-09-23T20:24:26.395 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:26 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 2426/300510 objects degraded (0.807%), 1 pg degraded (PG_DEGRADED) 2023-09-23T20:24:26.425 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:24:26.433 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:26.433 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:24:26.490 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 88 16 105172 105156 74.652 58.3125 0.0511438 0.0133914 2023-09-23T20:24:26.535 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:24:26.584 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:26 smithi149 ceph-mon[170394]: pgmap v320: 99 pgs: 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 96 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s; 2426/308298 objects degraded (0.787%); 5824/308298 objects misplaced (1.889%); 3.5 MiB/s, 55 objects/s recovering 2023-09-23T20:24:26.584 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:26 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 2426/300510 objects degraded (0.807%), 1 pg degraded (PG_DEGRADED) 2023-09-23T20:24:26.589 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:26.657 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:24:26.662 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:26.663 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:24:26.763 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:24:26.844 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:26.895 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:24:26.930 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:24:26.939 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:26.939 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:24:27.040 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:24:27.092 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:27.192 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:24:27.218 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:24:27.232 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:27.233 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:24:27.333 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:24:27.403 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:27.492 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 89 16 106114 106098 74.4748 58.875 0.0151242 0.0134224 2023-09-23T20:24:27.592 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:24:27.598 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:27.599 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:24:27.699 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:24:27.774 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:27.846 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:24:27.851 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:27.852 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:24:27.952 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:24:28.020 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:28.086 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:24:28.089 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:24:28.094 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:28.094 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:24:28.195 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:24:28.256 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:28.320 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:24:28.327 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:28.327 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:24:28.428 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:24:28.485 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:28.490 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Total time run: 90.0154 2023-09-23T20:24:28.490 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Total writes made: 107058 2023-09-23T20:24:28.490 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Write size: 65536 2023-09-23T20:24:28.490 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Object size: 65536 2023-09-23T20:24:28.491 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Bandwidth (MB/sec): 74.3331 2023-09-23T20:24:28.491 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Stddev Bandwidth: 45.3079 2023-09-23T20:24:28.491 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Max bandwidth (MB/sec): 211.5 2023-09-23T20:24:28.491 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Min bandwidth (MB/sec): 0 2023-09-23T20:24:28.491 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Average IOPS: 1189 2023-09-23T20:24:28.491 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Stddev IOPS: 724.94 2023-09-23T20:24:28.491 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Max IOPS: 3384 2023-09-23T20:24:28.491 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Min IOPS: 0 2023-09-23T20:24:28.491 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Average Latency(s): 0.0134485 2023-09-23T20:24:28.491 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Stddev Latency(s): 0.0732044 2023-09-23T20:24:28.491 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Max latency(s): 2.83721 2023-09-23T20:24:28.492 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Min latency(s): 0.00112599 2023-09-23T20:24:28.496 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Cleaning up (deleting benchmark objects) 2023-09-23T20:24:28.589 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:24:28.600 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:28.601 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:24:28.701 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:24:28.781 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:28.855 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:24:28.859 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:28 smithi018 ceph-mon[179647]: pgmap v321: 99 pgs: 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 96 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s; 2426/311958 objects degraded (0.778%); 5824/311958 objects misplaced (1.867%); 3.5 MiB/s, 55 objects/s recovering 2023-09-23T20:24:28.859 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:28 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:24:28.859 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:28 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:24:28.860 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:28.861 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:24:28.961 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:24:29.008 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:29.064 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:24:29.069 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:29.073 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:24:29.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:28 smithi018 ceph-mon[184144]: pgmap v321: 99 pgs: 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 96 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s; 2426/311958 objects degraded (0.778%); 5824/311958 objects misplaced (1.867%); 3.5 MiB/s, 55 objects/s recovering 2023-09-23T20:24:29.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:28 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:24:29.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:28 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:24:29.174 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:24:29.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:28 smithi149 ceph-mon[170394]: pgmap v321: 99 pgs: 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 96 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s; 2426/311958 objects degraded (0.778%); 5824/311958 objects misplaced (1.867%); 3.5 MiB/s, 55 objects/s recovering 2023-09-23T20:24:29.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:28 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:24:29.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:28 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:24:29.339 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:29.375 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2023-09-23T20:24:29.375 INFO:teuthology.orchestra.run.smithi018.stdout: "size": 20, 2023-09-23T20:24:29.375 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 600, 2023-09-23T20:24:29.375 INFO:teuthology.orchestra.run.smithi018.stdout: "ops": [ 2023-09-23T20:24:29.376 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.376 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.34791.0:28906 5.3 5:c272d99a:::benchmark_data_smithi018_228451_object28905:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e118)", 2023-09-23T20:24:29.376 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:23:09.973303+0000", 2023-09-23T20:24:29.376 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 79.386863324999993, 2023-09-23T20:24:29.376 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.095172016, 2023-09-23T20:24:29.376 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:24:29.376 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:24:29.376 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:24:29.376 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.34791", 2023-09-23T20:24:29.376 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:24:29.376 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 28906 2023-09-23T20:24:29.376 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.377 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:24:29.377 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.377 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:24:29.377 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.973303+0000", 2023-09-23T20:24:29.377 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:24:29.377 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.377 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.377 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:24:29.377 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.973303+0000", 2023-09-23T20:24:29.377 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:24:29.377 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.377 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.378 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:24:29.378 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.973300+0000", 2023-09-23T20:24:29.378 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999976 2023-09-23T20:24:29.378 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.378 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.378 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:24:29.378 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.973331+0000", 2023-09-23T20:24:29.378 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.0750000000000002e-05 2023-09-23T20:24:29.378 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.378 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.378 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:24:29.379 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.973347+0000", 2023-09-23T20:24:29.379 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.5458999999999999e-05 2023-09-23T20:24:29.379 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.379 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.379 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:24:29.379 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.973357+0000", 2023-09-23T20:24:29.379 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0498e-05 2023-09-23T20:24:29.379 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.379 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.379 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:24:29.379 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:10.023773+0000", 2023-09-23T20:24:29.379 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.050416004 2023-09-23T20:24:29.380 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.380 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.380 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:24:29.380 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:10.023793+0000", 2023-09-23T20:24:29.380 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.9508999999999999e-05 2023-09-23T20:24:29.380 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.380 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.380 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:24:29.380 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.061439+0000", 2023-09-23T20:24:29.380 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.037646359 2023-09-23T20:24:29.380 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.381 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.381 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:24:29.381 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.061615+0000", 2023-09-23T20:24:29.381 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000176505 2023-09-23T20:24:29.381 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.381 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.381 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 5,6", 2023-09-23T20:24:29.381 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.061827+0000", 2023-09-23T20:24:29.381 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00021147200000000001 2023-09-23T20:24:29.381 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.381 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.381 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:24:29.382 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.063990+0000", 2023-09-23T20:24:29.382 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0021635019999999999 2023-09-23T20:24:29.382 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.382 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.382 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:24:29.382 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.065413+0000", 2023-09-23T20:24:29.382 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0014226180000000001 2023-09-23T20:24:29.382 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.382 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.382 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:24:29.382 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.068371+0000", 2023-09-23T20:24:29.383 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.002957777 2023-09-23T20:24:29.383 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.383 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.383 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:24:29.383 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.068433+0000", 2023-09-23T20:24:29.383 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.2032000000000005e-05 2023-09-23T20:24:29.383 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.383 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.383 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:24:29.383 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.068475+0000", 2023-09-23T20:24:29.383 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.1687000000000003e-05 2023-09-23T20:24:29.384 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:24:29.384 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:24:29.384 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:24:29.384 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.384 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.384 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.34791.0:75986 5.17 5:e9503909:::benchmark_data_smithi018_228451_object75985:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e146)", 2023-09-23T20:24:29.384 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:23:56.930198+0000", 2023-09-23T20:24:29.384 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 32.429967974, 2023-09-23T20:24:29.384 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1330166429999999, 2023-09-23T20:24:29.384 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:24:29.384 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:24:29.385 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:24:29.385 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.34791", 2023-09-23T20:24:29.385 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:24:29.385 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 75986 2023-09-23T20:24:29.385 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.385 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:24:29.385 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.385 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:24:29.385 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.930198+0000", 2023-09-23T20:24:29.385 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:24:29.385 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.385 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.386 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:24:29.386 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.930198+0000", 2023-09-23T20:24:29.386 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:24:29.386 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.386 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.386 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:24:29.386 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.930196+0000", 2023-09-23T20:24:29.386 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:24:29.386 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.386 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.386 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:24:29.387 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.930222+0000", 2023-09-23T20:24:29.387 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.5797000000000001e-05 2023-09-23T20:24:29.387 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.387 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.387 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:24:29.387 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.930236+0000", 2023-09-23T20:24:29.387 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.4181e-05 2023-09-23T20:24:29.387 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.387 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.387 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:24:29.387 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.930245+0000", 2023-09-23T20:24:29.387 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.2379999999999997e-06 2023-09-23T20:24:29.388 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.388 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.388 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:24:29.388 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.930304+0000", 2023-09-23T20:24:29.388 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.9571000000000003e-05 2023-09-23T20:24:29.388 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.388 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.388 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:24:29.388 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.930320+0000", 2023-09-23T20:24:29.388 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.5308e-05 2023-09-23T20:24:29.388 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.388 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.389 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:24:29.389 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.055795+0000", 2023-09-23T20:24:29.389 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1254757980000001 2023-09-23T20:24:29.389 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.389 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.389 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:24:29.389 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.055914+0000", 2023-09-23T20:24:29.389 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000118748 2023-09-23T20:24:29.389 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.389 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.389 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,3,7", 2023-09-23T20:24:29.390 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.056027+0000", 2023-09-23T20:24:29.390 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00011285000000000001 2023-09-23T20:24:29.390 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.390 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.390 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:24:29.390 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.059412+0000", 2023-09-23T20:24:29.390 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0033849510000000002 2023-09-23T20:24:29.390 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.390 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.390 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:24:29.390 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.059444+0000", 2023-09-23T20:24:29.391 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.1903000000000003e-05 2023-09-23T20:24:29.391 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.391 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.391 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:24:29.391 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.059453+0000", 2023-09-23T20:24:29.391 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.1479999999999995e-06 2023-09-23T20:24:29.391 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.391 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.391 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:24:29.391 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.062037+0000", 2023-09-23T20:24:29.391 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0025838139999999998 2023-09-23T20:24:29.392 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.392 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.392 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:24:29.392 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.062074+0000", 2023-09-23T20:24:29.392 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.7474000000000001e-05 2023-09-23T20:24:29.392 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.392 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.392 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:24:29.392 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.063215+0000", 2023-09-23T20:24:29.392 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0011402879999999999 2023-09-23T20:24:29.392 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:24:29.393 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:24:29.393 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:24:29.393 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.393 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.393 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.34791.0:75990 5.17 5:ee7f398d:::benchmark_data_smithi018_228451_object75989:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e146)", 2023-09-23T20:24:29.393 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:23:56.930258+0000", 2023-09-23T20:24:29.393 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 32.429907880000002, 2023-09-23T20:24:29.393 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.134089846, 2023-09-23T20:24:29.393 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:24:29.393 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:24:29.393 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:24:29.393 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.34791", 2023-09-23T20:24:29.394 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:24:29.394 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 75990 2023-09-23T20:24:29.394 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.394 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:24:29.394 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.394 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:24:29.394 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.930258+0000", 2023-09-23T20:24:29.394 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:24:29.394 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.394 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.395 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:24:29.395 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.930258+0000", 2023-09-23T20:24:29.395 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:24:29.395 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.395 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.395 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:24:29.395 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.930257+0000", 2023-09-23T20:24:29.395 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:24:29.395 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.395 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.396 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:24:29.396 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.930279+0000", 2023-09-23T20:24:29.396 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.1834000000000001e-05 2023-09-23T20:24:29.396 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.396 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.396 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:24:29.396 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.930287+0000", 2023-09-23T20:24:29.396 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.6899999999999992e-06 2023-09-23T20:24:29.396 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.396 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.396 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:24:29.397 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.930293+0000", 2023-09-23T20:24:29.397 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.4359999999999999e-06 2023-09-23T20:24:29.397 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.397 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.397 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:24:29.397 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.930346+0000", 2023-09-23T20:24:29.397 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.2324000000000003e-05 2023-09-23T20:24:29.397 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.397 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.397 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:24:29.397 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.930354+0000", 2023-09-23T20:24:29.397 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.3410000000000003e-06 2023-09-23T20:24:29.397 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.398 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.398 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:24:29.398 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.058079+0000", 2023-09-23T20:24:29.398 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1277248049999999 2023-09-23T20:24:29.398 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.398 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.398 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:24:29.398 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.058216+0000", 2023-09-23T20:24:29.398 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00013760399999999999 2023-09-23T20:24:29.398 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.399 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.399 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,3,7", 2023-09-23T20:24:29.399 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.058352+0000", 2023-09-23T20:24:29.399 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00013527 2023-09-23T20:24:29.399 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.399 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.399 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:24:29.399 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.061985+0000", 2023-09-23T20:24:29.399 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0036338130000000001 2023-09-23T20:24:29.399 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.399 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.400 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:24:29.400 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.061993+0000", 2023-09-23T20:24:29.400 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.8350000000000004e-06 2023-09-23T20:24:29.400 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.400 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.400 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:24:29.400 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.063268+0000", 2023-09-23T20:24:29.400 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0012749809999999999 2023-09-23T20:24:29.400 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.400 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.400 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:24:29.400 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.063382+0000", 2023-09-23T20:24:29.401 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000113581 2023-09-23T20:24:29.401 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.401 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.401 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:24:29.401 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.063437+0000", 2023-09-23T20:24:29.401 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.5414000000000002e-05 2023-09-23T20:24:29.401 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.401 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.401 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:24:29.401 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.064348+0000", 2023-09-23T20:24:29.401 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00091062399999999996 2023-09-23T20:24:29.402 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:24:29.402 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:24:29.402 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:24:29.402 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.402 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.402 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.34791.0:76086 5.17 5:efb66307:::benchmark_data_smithi018_228451_object76085:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e146)", 2023-09-23T20:24:29.402 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:23:56.930299+0000", 2023-09-23T20:24:29.402 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 32.429866675, 2023-09-23T20:24:29.402 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1350808800000001, 2023-09-23T20:24:29.402 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:24:29.402 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:24:29.403 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:24:29.403 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.34791", 2023-09-23T20:24:29.403 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:24:29.403 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 76086 2023-09-23T20:24:29.403 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.403 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:24:29.403 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.403 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:24:29.403 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.930299+0000", 2023-09-23T20:24:29.403 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:24:29.403 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.403 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.404 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:24:29.404 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.930299+0000", 2023-09-23T20:24:29.404 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:24:29.404 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.404 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.404 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:24:29.404 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.930299+0000", 2023-09-23T20:24:29.404 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:24:29.404 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.404 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.404 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:24:29.405 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.930320+0000", 2023-09-23T20:24:29.405 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.0982e-05 2023-09-23T20:24:29.405 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.405 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.405 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:24:29.405 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.930328+0000", 2023-09-23T20:24:29.405 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.5439999999999999e-06 2023-09-23T20:24:29.405 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.405 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.405 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:24:29.405 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.930334+0000", 2023-09-23T20:24:29.405 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.1909999999999997e-06 2023-09-23T20:24:29.406 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.406 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.406 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:24:29.406 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.930377+0000", 2023-09-23T20:24:29.406 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.2737000000000001e-05 2023-09-23T20:24:29.406 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.406 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.406 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:24:29.406 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.930385+0000", 2023-09-23T20:24:29.406 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.9789999999999993e-06 2023-09-23T20:24:29.407 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.407 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.407 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:24:29.407 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.059544+0000", 2023-09-23T20:24:29.407 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.129159094 2023-09-23T20:24:29.407 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.407 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.407 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:24:29.407 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.059640+0000", 2023-09-23T20:24:29.407 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.5759000000000003e-05 2023-09-23T20:24:29.407 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.408 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.408 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,3,7", 2023-09-23T20:24:29.408 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.059762+0000", 2023-09-23T20:24:29.408 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000122024 2023-09-23T20:24:29.408 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.408 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.408 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:24:29.408 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.063277+0000", 2023-09-23T20:24:29.408 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0035149420000000001 2023-09-23T20:24:29.408 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.408 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.408 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:24:29.409 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.063314+0000", 2023-09-23T20:24:29.409 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.6888999999999999e-05 2023-09-23T20:24:29.409 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.409 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.409 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:24:29.409 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.064409+0000", 2023-09-23T20:24:29.409 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0010948920000000001 2023-09-23T20:24:29.409 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.409 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.409 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:24:29.409 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.064458+0000", 2023-09-23T20:24:29.409 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.8826000000000003e-05 2023-09-23T20:24:29.409 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.410 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.410 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:24:29.410 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.064511+0000", 2023-09-23T20:24:29.410 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.3640999999999997e-05 2023-09-23T20:24:29.410 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.410 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.410 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:24:29.410 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.065380+0000", 2023-09-23T20:24:29.410 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00086886199999999998 2023-09-23T20:24:29.410 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:24:29.410 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:24:29.411 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:24:29.411 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.411 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.411 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.34791.0:76104 5.17 5:eb4afaee:::benchmark_data_smithi018_228451_object76103:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e146)", 2023-09-23T20:24:29.411 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:23:56.930340+0000", 2023-09-23T20:24:29.411 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 32.429825780000002, 2023-09-23T20:24:29.411 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.13575136, 2023-09-23T20:24:29.411 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:24:29.411 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:24:29.411 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:24:29.411 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.34791", 2023-09-23T20:24:29.412 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:24:29.412 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 76104 2023-09-23T20:24:29.412 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.412 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:24:29.412 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.412 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:24:29.412 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.930340+0000", 2023-09-23T20:24:29.412 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:24:29.413 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.413 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.413 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:24:29.413 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.930340+0000", 2023-09-23T20:24:29.413 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:24:29.413 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.413 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.413 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:24:29.413 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.930340+0000", 2023-09-23T20:24:29.414 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:24:29.414 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.414 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.414 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:24:29.414 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.930359+0000", 2023-09-23T20:24:29.414 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.9777000000000001e-05 2023-09-23T20:24:29.414 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.414 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.414 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:24:29.414 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.930366+0000", 2023-09-23T20:24:29.414 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.0520000000000004e-06 2023-09-23T20:24:29.415 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.415 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.415 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:24:29.415 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.930372+0000", 2023-09-23T20:24:29.415 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.5890000000000002e-06 2023-09-23T20:24:29.415 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.415 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.415 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:24:29.415 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.930408+0000", 2023-09-23T20:24:29.415 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.5859999999999999e-05 2023-09-23T20:24:29.415 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.415 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.416 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:24:29.416 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.930415+0000", 2023-09-23T20:24:29.416 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.8070000000000002e-06 2023-09-23T20:24:29.416 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.416 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.416 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:24:29.416 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.060624+0000", 2023-09-23T20:24:29.416 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1302093689999999 2023-09-23T20:24:29.416 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.416 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.416 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:24:29.416 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.060751+0000", 2023-09-23T20:24:29.417 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00012704799999999999 2023-09-23T20:24:29.417 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.417 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.417 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,3,7", 2023-09-23T20:24:29.417 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.060916+0000", 2023-09-23T20:24:29.417 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00016477399999999999 2023-09-23T20:24:29.417 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.417 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.417 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:24:29.417 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.063337+0000", 2023-09-23T20:24:29.417 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0024210210000000002 2023-09-23T20:24:29.417 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.417 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.418 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:24:29.418 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.063343+0000", 2023-09-23T20:24:29.418 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.4779999999999996e-06 2023-09-23T20:24:29.418 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.418 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.418 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:24:29.418 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.065427+0000", 2023-09-23T20:24:29.418 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.002083349 2023-09-23T20:24:29.418 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.418 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.418 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:24:29.418 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.065463+0000", 2023-09-23T20:24:29.419 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.6013999999999998e-05 2023-09-23T20:24:29.419 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.419 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.419 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:24:29.419 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.065515+0000", 2023-09-23T20:24:29.419 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.1765999999999999e-05 2023-09-23T20:24:29.419 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.419 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.419 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:24:29.419 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.066091+0000", 2023-09-23T20:24:29.419 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00057691600000000002 2023-09-23T20:24:29.420 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:24:29.420 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:24:29.420 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:24:29.420 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.420 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.420 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.34791.0:76249 5.17 5:ef69b942:::benchmark_data_smithi018_228451_object76248:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e146)", 2023-09-23T20:24:29.420 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:23:56.930378+0000", 2023-09-23T20:24:29.420 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 32.429788111000001, 2023-09-23T20:24:29.420 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1644411269999999, 2023-09-23T20:24:29.420 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:24:29.420 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:24:29.420 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:24:29.420 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.34791", 2023-09-23T20:24:29.421 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:24:29.421 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 76249 2023-09-23T20:24:29.421 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.421 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:24:29.421 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.421 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:24:29.421 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.930378+0000", 2023-09-23T20:24:29.421 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:24:29.421 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.421 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.421 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:24:29.422 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.930378+0000", 2023-09-23T20:24:29.422 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:24:29.422 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.422 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.422 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:24:29.422 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.930377+0000", 2023-09-23T20:24:29.422 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:24:29.422 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.422 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.422 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:24:29.422 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.930397+0000", 2023-09-23T20:24:29.422 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.9236999999999999e-05 2023-09-23T20:24:29.423 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.423 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.423 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:24:29.423 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.930404+0000", 2023-09-23T20:24:29.423 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.9589999999999998e-06 2023-09-23T20:24:29.423 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.423 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.423 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:24:29.423 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.930409+0000", 2023-09-23T20:24:29.423 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.7440000000000004e-06 2023-09-23T20:24:29.423 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.423 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.424 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:24:29.424 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.930438+0000", 2023-09-23T20:24:29.424 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.9e-05 2023-09-23T20:24:29.424 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.424 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.424 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:24:29.424 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.930446+0000", 2023-09-23T20:24:29.424 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.3459999999999996e-06 2023-09-23T20:24:29.424 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.425 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.425 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:24:29.425 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.066130+0000", 2023-09-23T20:24:29.425 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.135683878 2023-09-23T20:24:29.425 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.425 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.425 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:24:29.425 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.066263+0000", 2023-09-23T20:24:29.425 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00013328 2023-09-23T20:24:29.425 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.425 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.426 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,3,7", 2023-09-23T20:24:29.426 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.066386+0000", 2023-09-23T20:24:29.426 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000123156 2023-09-23T20:24:29.426 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.426 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.426 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:24:29.426 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.093159+0000", 2023-09-23T20:24:29.426 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.026772906999999999 2023-09-23T20:24:29.426 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.426 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.426 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:24:29.427 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.093171+0000", 2023-09-23T20:24:29.427 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1985e-05 2023-09-23T20:24:29.427 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.427 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.427 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:24:29.427 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.093295+0000", 2023-09-23T20:24:29.427 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00012436200000000001 2023-09-23T20:24:29.427 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.427 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.427 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:24:29.427 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.093370+0000", 2023-09-23T20:24:29.427 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.5061000000000005e-05 2023-09-23T20:24:29.428 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.428 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.428 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:24:29.428 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.093435+0000", 2023-09-23T20:24:29.428 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.4698999999999998e-05 2023-09-23T20:24:29.428 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.428 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.428 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:24:29.428 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.094819+0000", 2023-09-23T20:24:29.428 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00138393 2023-09-23T20:24:29.428 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:24:29.428 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:24:29.429 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:24:29.429 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.429 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.429 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.34791.0:76255 5.17 5:edaf6009:::benchmark_data_smithi018_228451_object76254:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e146)", 2023-09-23T20:24:29.429 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:23:56.930415+0000", 2023-09-23T20:24:29.429 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 32.429751017000001, 2023-09-23T20:24:29.429 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.170445253, 2023-09-23T20:24:29.429 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:24:29.429 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:24:29.429 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:24:29.429 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.34791", 2023-09-23T20:24:29.429 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:24:29.429 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 76255 2023-09-23T20:24:29.430 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.430 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:24:29.430 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.430 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:24:29.430 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.930415+0000", 2023-09-23T20:24:29.430 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:24:29.430 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.430 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.430 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:24:29.430 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.930415+0000", 2023-09-23T20:24:29.430 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:24:29.430 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.431 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.431 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:24:29.431 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.930414+0000", 2023-09-23T20:24:29.431 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:24:29.431 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.431 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.431 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:24:29.431 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.930434+0000", 2023-09-23T20:24:29.431 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.9207000000000001e-05 2023-09-23T20:24:29.431 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.431 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.431 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:24:29.431 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.930440+0000", 2023-09-23T20:24:29.432 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.6680000000000004e-06 2023-09-23T20:24:29.432 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.432 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.432 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:24:29.432 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.930446+0000", 2023-09-23T20:24:29.432 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.5879999999999997e-06 2023-09-23T20:24:29.432 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.432 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.432 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:24:29.432 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.930494+0000", 2023-09-23T20:24:29.432 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.8109000000000003e-05 2023-09-23T20:24:29.432 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.433 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.433 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:24:29.433 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.930505+0000", 2023-09-23T20:24:29.433 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0813e-05 2023-09-23T20:24:29.433 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.433 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.433 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:24:29.433 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.067503+0000", 2023-09-23T20:24:29.433 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1369980989999999 2023-09-23T20:24:29.433 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.433 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.433 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:24:29.434 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.067643+0000", 2023-09-23T20:24:29.434 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00013979300000000001 2023-09-23T20:24:29.434 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.434 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.434 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,3,7", 2023-09-23T20:24:29.434 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.067809+0000", 2023-09-23T20:24:29.434 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00016613199999999999 2023-09-23T20:24:29.434 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.434 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.434 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:24:29.434 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.093250+0000", 2023-09-23T20:24:29.435 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.025440746 2023-09-23T20:24:29.435 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.435 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.435 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:24:29.435 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.093333+0000", 2023-09-23T20:24:29.435 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.3744000000000003e-05 2023-09-23T20:24:29.435 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.435 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.435 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:24:29.435 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.094864+0000", 2023-09-23T20:24:29.435 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.001531113 2023-09-23T20:24:29.435 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.436 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.436 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:24:29.436 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.099642+0000", 2023-09-23T20:24:29.436 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0047778359999999997 2023-09-23T20:24:29.436 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.436 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.436 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:24:29.436 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.099694+0000", 2023-09-23T20:24:29.436 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.1907000000000003e-05 2023-09-23T20:24:29.436 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.436 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.437 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:24:29.437 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.100860+0000", 2023-09-23T20:24:29.437 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0011659420000000001 2023-09-23T20:24:29.437 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:24:29.437 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:24:29.437 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:24:29.437 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.437 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.437 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.34791.0:76279 5.17 5:ede9227d:::benchmark_data_smithi018_228451_object76278:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e146)", 2023-09-23T20:24:29.437 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:23:56.930459+0000", 2023-09-23T20:24:29.437 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 32.429707026000003, 2023-09-23T20:24:29.437 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1748222939999999, 2023-09-23T20:24:29.438 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:24:29.438 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:24:29.438 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:24:29.438 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.34791", 2023-09-23T20:24:29.438 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:24:29.438 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 76279 2023-09-23T20:24:29.438 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.438 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:24:29.438 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.438 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:24:29.438 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.930459+0000", 2023-09-23T20:24:29.439 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:24:29.439 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.439 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.439 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:24:29.439 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.930459+0000", 2023-09-23T20:24:29.439 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:24:29.439 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.439 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.439 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:24:29.439 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.930458+0000", 2023-09-23T20:24:29.439 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:24:29.439 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.439 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.440 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:24:29.440 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.930480+0000", 2023-09-23T20:24:29.440 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.1199000000000002e-05 2023-09-23T20:24:29.440 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.440 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.440 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:24:29.440 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.930530+0000", 2023-09-23T20:24:29.440 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.9999e-05 2023-09-23T20:24:29.440 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.440 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.440 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:24:29.440 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.930537+0000", 2023-09-23T20:24:29.441 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.3370000000000002e-06 2023-09-23T20:24:29.441 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.441 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.441 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:24:29.441 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.930583+0000", 2023-09-23T20:24:29.441 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.6041000000000002e-05 2023-09-23T20:24:29.441 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.441 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.441 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:24:29.441 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.930592+0000", 2023-09-23T20:24:29.441 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.7390000000000005e-06 2023-09-23T20:24:29.441 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.442 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.442 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:24:29.442 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.094883+0000", 2023-09-23T20:24:29.442 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.164291594 2023-09-23T20:24:29.442 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.442 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.442 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:24:29.442 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.095063+0000", 2023-09-23T20:24:29.442 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00018021199999999999 2023-09-23T20:24:29.442 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.443 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.443 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,3,7", 2023-09-23T20:24:29.443 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.095277+0000", 2023-09-23T20:24:29.443 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00021343100000000001 2023-09-23T20:24:29.443 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.443 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.443 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:24:29.443 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.099613+0000", 2023-09-23T20:24:29.443 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0043363710000000003 2023-09-23T20:24:29.443 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.443 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.443 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:24:29.444 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.100877+0000", 2023-09-23T20:24:29.444 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0012636570000000001 2023-09-23T20:24:29.444 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.444 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.444 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:24:29.444 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.104056+0000", 2023-09-23T20:24:29.444 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.003179056 2023-09-23T20:24:29.444 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.444 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.444 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:24:29.444 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.104122+0000", 2023-09-23T20:24:29.444 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.6235999999999994e-05 2023-09-23T20:24:29.445 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.445 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.445 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:24:29.445 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.104184+0000", 2023-09-23T20:24:29.445 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.1722000000000003e-05 2023-09-23T20:24:29.445 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.445 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.445 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:24:29.445 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.105281+0000", 2023-09-23T20:24:29.445 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0010971959999999999 2023-09-23T20:24:29.445 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:24:29.446 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:24:29.446 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:24:29.446 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.446 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.446 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.34791.0:76341 5.17 5:efe1c8d4:::benchmark_data_smithi018_228451_object76340:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e146)", 2023-09-23T20:24:29.446 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:23:56.930549+0000", 2023-09-23T20:24:29.446 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 32.429616932999998, 2023-09-23T20:24:29.446 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1756924040000001, 2023-09-23T20:24:29.446 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:24:29.446 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:24:29.446 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:24:29.447 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.34791", 2023-09-23T20:24:29.447 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:24:29.447 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 76341 2023-09-23T20:24:29.447 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.447 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:24:29.447 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.447 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:24:29.447 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.930549+0000", 2023-09-23T20:24:29.447 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:24:29.447 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.447 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.447 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:24:29.447 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.930549+0000", 2023-09-23T20:24:29.448 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:24:29.448 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.448 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.448 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:24:29.448 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.930548+0000", 2023-09-23T20:24:29.448 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:24:29.448 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.448 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.448 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:24:29.448 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.930572+0000", 2023-09-23T20:24:29.448 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.3747000000000002e-05 2023-09-23T20:24:29.448 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.449 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.449 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:24:29.449 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.930589+0000", 2023-09-23T20:24:29.449 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.7288000000000001e-05 2023-09-23T20:24:29.449 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.449 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.449 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:24:29.449 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.930596+0000", 2023-09-23T20:24:29.449 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.6320000000000002e-06 2023-09-23T20:24:29.449 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.449 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.449 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:24:29.450 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.930626+0000", 2023-09-23T20:24:29.450 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.9841000000000001e-05 2023-09-23T20:24:29.450 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.450 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.450 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:24:29.450 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.930633+0000", 2023-09-23T20:24:29.450 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.277e-06 2023-09-23T20:24:29.450 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.450 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.450 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:24:29.450 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.096513+0000", 2023-09-23T20:24:29.450 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1658801350000001 2023-09-23T20:24:29.451 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.451 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.451 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:24:29.451 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.096694+0000", 2023-09-23T20:24:29.451 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00018080200000000001 2023-09-23T20:24:29.451 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.451 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.451 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,3,7", 2023-09-23T20:24:29.451 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.096915+0000", 2023-09-23T20:24:29.451 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000221265 2023-09-23T20:24:29.451 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.451 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.451 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:24:29.452 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.104075+0000", 2023-09-23T20:24:29.452 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0071594099999999997 2023-09-23T20:24:29.452 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.452 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.452 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:24:29.452 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.105333+0000", 2023-09-23T20:24:29.452 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0012584479999999999 2023-09-23T20:24:29.452 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.452 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.452 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:24:29.452 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.105432+0000", 2023-09-23T20:24:29.452 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.9204999999999996e-05 2023-09-23T20:24:29.453 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.453 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.453 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:24:29.453 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.105490+0000", 2023-09-23T20:24:29.453 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.8143000000000002e-05 2023-09-23T20:24:29.453 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.453 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.453 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:24:29.453 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.105550+0000", 2023-09-23T20:24:29.453 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.9407000000000002e-05 2023-09-23T20:24:29.453 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.454 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.454 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:24:29.454 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.106241+0000", 2023-09-23T20:24:29.454 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00069145399999999998 2023-09-23T20:24:29.454 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:24:29.454 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:24:29.454 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:24:29.454 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.454 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.454 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.34791.0:76399 5.17 5:ec2ff4cc:::benchmark_data_smithi018_228451_object76398:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e146)", 2023-09-23T20:24:29.454 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:23:56.930610+0000", 2023-09-23T20:24:29.454 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 32.429556034999997, 2023-09-23T20:24:29.455 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1763586560000001, 2023-09-23T20:24:29.455 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:24:29.455 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:24:29.455 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:24:29.455 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.34791", 2023-09-23T20:24:29.455 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:24:29.455 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 76399 2023-09-23T20:24:29.455 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.455 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:24:29.455 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.455 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:24:29.455 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.930610+0000", 2023-09-23T20:24:29.456 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:24:29.456 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.456 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.456 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:24:29.456 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.930610+0000", 2023-09-23T20:24:29.456 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:24:29.456 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.456 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.456 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:24:29.456 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.930609+0000", 2023-09-23T20:24:29.456 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:24:29.456 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.457 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.457 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:24:29.457 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.930630+0000", 2023-09-23T20:24:29.457 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.1098999999999999e-05 2023-09-23T20:24:29.457 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.457 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.457 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:24:29.457 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.930638+0000", 2023-09-23T20:24:29.457 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.605e-06 2023-09-23T20:24:29.457 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.457 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.457 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:24:29.457 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.930644+0000", 2023-09-23T20:24:29.458 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.8699999999999997e-06 2023-09-23T20:24:29.458 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.458 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.458 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:24:29.458 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.930688+0000", 2023-09-23T20:24:29.458 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.4288e-05 2023-09-23T20:24:29.458 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.458 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.458 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:24:29.458 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.930696+0000", 2023-09-23T20:24:29.458 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.9130000000000003e-06 2023-09-23T20:24:29.459 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.459 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.459 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:24:29.459 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.098150+0000", 2023-09-23T20:24:29.459 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.167453941 2023-09-23T20:24:29.459 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.459 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.459 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:24:29.459 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.098281+0000", 2023-09-23T20:24:29.459 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00013097400000000001 2023-09-23T20:24:29.459 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.459 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.460 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,3,7", 2023-09-23T20:24:29.460 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.098420+0000", 2023-09-23T20:24:29.460 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000138892 2023-09-23T20:24:29.460 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.460 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.460 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:24:29.460 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.105344+0000", 2023-09-23T20:24:29.460 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0069237839999999997 2023-09-23T20:24:29.460 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.460 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.460 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:24:29.461 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.105392+0000", 2023-09-23T20:24:29.461 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.8659000000000003e-05 2023-09-23T20:24:29.461 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.461 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.461 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:24:29.461 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.106285+0000", 2023-09-23T20:24:29.461 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00089303699999999995 2023-09-23T20:24:29.461 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.461 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.461 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:24:29.461 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.106327+0000", 2023-09-23T20:24:29.461 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.1773000000000001e-05 2023-09-23T20:24:29.462 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.462 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.462 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:24:29.462 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.106362+0000", 2023-09-23T20:24:29.462 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.5061000000000002e-05 2023-09-23T20:24:29.462 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.462 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.462 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:24:29.462 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.106969+0000", 2023-09-23T20:24:29.462 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000606318 2023-09-23T20:24:29.462 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:24:29.463 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:24:29.463 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:24:29.463 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.463 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.463 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.34791.0:76438 5.17 5:ed635187:::benchmark_data_smithi018_228451_object76437:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e146)", 2023-09-23T20:24:29.463 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:23:56.930660+0000", 2023-09-23T20:24:29.463 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 32.429505560000003, 2023-09-23T20:24:29.463 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.182794573, 2023-09-23T20:24:29.463 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:24:29.463 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:24:29.463 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:24:29.463 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.34791", 2023-09-23T20:24:29.464 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:24:29.464 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 76438 2023-09-23T20:24:29.464 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.464 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:24:29.464 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.464 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:24:29.464 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.930660+0000", 2023-09-23T20:24:29.464 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:24:29.464 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.464 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.464 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:24:29.464 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.930660+0000", 2023-09-23T20:24:29.465 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:24:29.465 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.465 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.465 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:24:29.465 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.930660+0000", 2023-09-23T20:24:29.465 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:24:29.465 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.465 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.465 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:24:29.465 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.930680+0000", 2023-09-23T20:24:29.466 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.0324999999999999e-05 2023-09-23T20:24:29.466 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.466 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.466 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:24:29.466 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.930689+0000", 2023-09-23T20:24:29.466 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.5909999999999996e-06 2023-09-23T20:24:29.466 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.466 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.466 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:24:29.466 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.930695+0000", 2023-09-23T20:24:29.466 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.1829999999999999e-06 2023-09-23T20:24:29.466 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.467 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.467 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:24:29.467 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.930729+0000", 2023-09-23T20:24:29.467 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.4520999999999997e-05 2023-09-23T20:24:29.467 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.467 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.467 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:24:29.467 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.930736+0000", 2023-09-23T20:24:29.467 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.0709999999999997e-06 2023-09-23T20:24:29.467 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.467 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.467 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:24:29.468 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.100915+0000", 2023-09-23T20:24:29.468 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1701783969999999 2023-09-23T20:24:29.468 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.468 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.468 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:24:29.468 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.101086+0000", 2023-09-23T20:24:29.468 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00017089199999999999 2023-09-23T20:24:29.468 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.468 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.468 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,3,7", 2023-09-23T20:24:29.468 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.101269+0000", 2023-09-23T20:24:29.468 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00018318499999999999 2023-09-23T20:24:29.469 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.469 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.469 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:24:29.469 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.105351+0000", 2023-09-23T20:24:29.469 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0040824090000000004 2023-09-23T20:24:29.469 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.469 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.469 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:24:29.469 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.108786+0000", 2023-09-23T20:24:29.469 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0034345410000000002 2023-09-23T20:24:29.469 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.470 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.470 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:24:29.470 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.108882+0000", 2023-09-23T20:24:29.470 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.5968e-05 2023-09-23T20:24:29.470 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.470 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.470 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:24:29.470 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.112333+0000", 2023-09-23T20:24:29.470 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0034508239999999999 2023-09-23T20:24:29.470 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.470 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.470 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:24:29.470 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.112376+0000", 2023-09-23T20:24:29.471 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.3741e-05 2023-09-23T20:24:29.471 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.471 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.471 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:24:29.471 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.113455+0000", 2023-09-23T20:24:29.471 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.001078451 2023-09-23T20:24:29.471 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:24:29.471 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:24:29.471 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:24:29.471 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.471 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.471 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.34791.0:76447 5.17 5:ebb98b51:::benchmark_data_smithi018_228451_object76446:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e146)", 2023-09-23T20:24:29.472 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:23:56.930704+0000", 2023-09-23T20:24:29.472 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 32.429461476, 2023-09-23T20:24:29.472 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.183893173, 2023-09-23T20:24:29.472 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:24:29.472 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:24:29.472 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:24:29.472 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.34791", 2023-09-23T20:24:29.472 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:24:29.472 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 76447 2023-09-23T20:24:29.472 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.472 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:24:29.472 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.473 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:24:29.473 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.930704+0000", 2023-09-23T20:24:29.473 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:24:29.473 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.473 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.473 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:24:29.473 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.930704+0000", 2023-09-23T20:24:29.473 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:24:29.473 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.473 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.473 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:24:29.473 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.930704+0000", 2023-09-23T20:24:29.474 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:24:29.474 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.474 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.474 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:24:29.474 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.930724+0000", 2023-09-23T20:24:29.474 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.0358999999999999e-05 2023-09-23T20:24:29.474 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.474 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.474 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:24:29.474 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.930732+0000", 2023-09-23T20:24:29.474 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.5889999999999997e-06 2023-09-23T20:24:29.474 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.475 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.475 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:24:29.475 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.930738+0000", 2023-09-23T20:24:29.475 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.8220000000000003e-06 2023-09-23T20:24:29.475 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.475 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.475 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:24:29.475 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.930784+0000", 2023-09-23T20:24:29.475 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.6699999999999997e-05 2023-09-23T20:24:29.475 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.476 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.476 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:24:29.476 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.930807+0000", 2023-09-23T20:24:29.476 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.3110000000000001e-05 2023-09-23T20:24:29.476 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.476 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.476 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:24:29.476 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.102608+0000", 2023-09-23T20:24:29.476 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1718004950000001 2023-09-23T20:24:29.476 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.476 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.476 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:24:29.477 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.102754+0000", 2023-09-23T20:24:29.477 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00014558 2023-09-23T20:24:29.477 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.477 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.477 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,3,7", 2023-09-23T20:24:29.477 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.102934+0000", 2023-09-23T20:24:29.477 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00018002900000000001 2023-09-23T20:24:29.477 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.477 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.477 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:24:29.477 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.106296+0000", 2023-09-23T20:24:29.477 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0033623049999999999 2023-09-23T20:24:29.477 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.478 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.478 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:24:29.478 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.108838+0000", 2023-09-23T20:24:29.478 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0025419959999999999 2023-09-23T20:24:29.478 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.478 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.478 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:24:29.478 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.108917+0000", 2023-09-23T20:24:29.478 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.9291999999999999e-05 2023-09-23T20:24:29.478 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.478 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.478 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:24:29.479 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.113543+0000", 2023-09-23T20:24:29.479 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0046255879999999999 2023-09-23T20:24:29.479 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.479 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.479 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:24:29.479 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.113592+0000", 2023-09-23T20:24:29.479 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.9740000000000001e-05 2023-09-23T20:24:29.479 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.479 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.479 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:24:29.479 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.114598+0000", 2023-09-23T20:24:29.479 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0010050899999999999 2023-09-23T20:24:29.480 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:24:29.480 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:24:29.480 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:24:29.480 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.480 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.480 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.34791.0:76507 5.17 5:eb180571:::benchmark_data_smithi018_228451_object76506:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e146)", 2023-09-23T20:24:29.480 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:23:56.930747+0000", 2023-09-23T20:24:29.480 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 32.429418411, 2023-09-23T20:24:29.480 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.184906432, 2023-09-23T20:24:29.480 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:24:29.480 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:24:29.480 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:24:29.481 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.34791", 2023-09-23T20:24:29.481 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:24:29.481 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 76507 2023-09-23T20:24:29.481 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.481 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:24:29.481 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.481 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:24:29.481 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.930747+0000", 2023-09-23T20:24:29.481 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:24:29.481 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.481 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.481 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:24:29.482 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.930747+0000", 2023-09-23T20:24:29.482 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:24:29.482 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.482 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.482 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:24:29.482 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.930747+0000", 2023-09-23T20:24:29.482 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:24:29.482 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.482 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.482 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:24:29.482 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.930763+0000", 2023-09-23T20:24:29.482 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.6523000000000001e-05 2023-09-23T20:24:29.483 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.483 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.483 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:24:29.483 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.930769+0000", 2023-09-23T20:24:29.483 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.5849999999999999e-06 2023-09-23T20:24:29.483 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.483 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.483 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:24:29.483 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.930773+0000", 2023-09-23T20:24:29.483 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.2039999999999999e-06 2023-09-23T20:24:29.483 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.483 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.484 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:24:29.484 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.930832+0000", 2023-09-23T20:24:29.484 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.8486000000000002e-05 2023-09-23T20:24:29.484 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.484 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.484 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:24:29.484 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.930839+0000", 2023-09-23T20:24:29.484 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.6550000000000004e-06 2023-09-23T20:24:29.484 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.484 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.484 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:24:29.485 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.107007+0000", 2023-09-23T20:24:29.485 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1761680619999999 2023-09-23T20:24:29.485 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.485 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.485 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:24:29.485 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.107179+0000", 2023-09-23T20:24:29.485 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00017187800000000001 2023-09-23T20:24:29.485 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.485 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.485 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,3,7", 2023-09-23T20:24:29.485 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.107374+0000", 2023-09-23T20:24:29.485 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00019490800000000001 2023-09-23T20:24:29.486 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.486 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.486 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:24:29.486 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.114657+0000", 2023-09-23T20:24:29.486 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0072831409999999999 2023-09-23T20:24:29.486 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.486 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.486 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:24:29.486 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.114666+0000", 2023-09-23T20:24:29.486 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.8429999999999999e-06 2023-09-23T20:24:29.487 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.487 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.487 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:24:29.487 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.114714+0000", 2023-09-23T20:24:29.487 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.8266999999999997e-05 2023-09-23T20:24:29.487 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.487 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.487 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:24:29.487 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.114751+0000", 2023-09-23T20:24:29.487 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.6137999999999999e-05 2023-09-23T20:24:29.487 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.488 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.488 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:24:29.488 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.114831+0000", 2023-09-23T20:24:29.488 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.0245000000000002e-05 2023-09-23T20:24:29.488 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.488 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.488 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:24:29.488 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.115654+0000", 2023-09-23T20:24:29.488 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00082313999999999998 2023-09-23T20:24:29.488 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:24:29.488 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:24:29.488 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:24:29.489 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.489 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.489 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.34791.0:76527 5.17 5:efd8728d:::benchmark_data_smithi018_228451_object76526:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e146)", 2023-09-23T20:24:29.489 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:23:56.930780+0000", 2023-09-23T20:24:29.489 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 32.429385736999997, 2023-09-23T20:24:29.489 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1900372930000001, 2023-09-23T20:24:29.489 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:24:29.489 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:24:29.489 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:24:29.489 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.34791", 2023-09-23T20:24:29.489 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:24:29.490 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 76527 2023-09-23T20:24:29.490 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.490 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:24:29.490 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.490 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:24:29.490 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.930780+0000", 2023-09-23T20:24:29.490 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:24:29.490 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.490 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.490 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:24:29.490 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.930780+0000", 2023-09-23T20:24:29.491 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:24:29.491 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.491 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.491 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:24:29.491 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.930780+0000", 2023-09-23T20:24:29.491 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:24:29.491 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.491 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.491 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:24:29.491 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.930840+0000", 2023-09-23T20:24:29.491 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.9892000000000002e-05 2023-09-23T20:24:29.492 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.492 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.492 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:24:29.492 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.930847+0000", 2023-09-23T20:24:29.492 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.8339999999999999e-06 2023-09-23T20:24:29.492 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.492 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.492 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:24:29.492 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.930852+0000", 2023-09-23T20:24:29.492 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.189e-06 2023-09-23T20:24:29.493 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.493 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.493 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:24:29.493 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.930886+0000", 2023-09-23T20:24:29.493 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.3915000000000003e-05 2023-09-23T20:24:29.493 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.493 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.493 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:24:29.493 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.930893+0000", 2023-09-23T20:24:29.493 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.063e-06 2023-09-23T20:24:29.493 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.494 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.494 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:24:29.494 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.108941+0000", 2023-09-23T20:24:29.494 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1780487930000001 2023-09-23T20:24:29.494 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.494 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.494 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:24:29.494 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.109115+0000", 2023-09-23T20:24:29.494 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00017343 2023-09-23T20:24:29.494 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.494 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.495 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,3,7", 2023-09-23T20:24:29.495 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.109312+0000", 2023-09-23T20:24:29.495 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000196708 2023-09-23T20:24:29.495 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.495 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.495 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:24:29.495 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.114673+0000", 2023-09-23T20:24:29.495 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0053610979999999999 2023-09-23T20:24:29.495 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.495 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.495 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:24:29.496 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.115856+0000", 2023-09-23T20:24:29.496 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0011837939999999999 2023-09-23T20:24:29.496 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.496 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.496 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:24:29.496 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.119482+0000", 2023-09-23T20:24:29.496 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0036250760000000001 2023-09-23T20:24:29.496 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.496 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.496 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:24:29.497 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.119550+0000", 2023-09-23T20:24:29.497 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.8917000000000004e-05 2023-09-23T20:24:29.497 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.497 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.497 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:24:29.497 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.119594+0000", 2023-09-23T20:24:29.497 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.3174999999999999e-05 2023-09-23T20:24:29.497 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.497 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.498 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:24:29.498 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.120817+0000", 2023-09-23T20:24:29.498 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0012238430000000001 2023-09-23T20:24:29.498 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:24:29.498 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:24:29.498 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:24:29.498 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.498 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.498 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.34791.0:76538 5.17 5:ea25f855:::benchmark_data_smithi018_228451_object76537:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e146)", 2023-09-23T20:24:29.498 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:23:56.930860+0000", 2023-09-23T20:24:29.498 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 32.429305779000003, 2023-09-23T20:24:29.499 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1909149210000001, 2023-09-23T20:24:29.499 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:24:29.499 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:24:29.499 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:24:29.499 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.34791", 2023-09-23T20:24:29.499 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:24:29.499 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 76538 2023-09-23T20:24:29.499 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.499 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:24:29.499 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.499 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:24:29.500 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.930860+0000", 2023-09-23T20:24:29.500 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:24:29.500 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.500 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.500 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:24:29.500 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.930860+0000", 2023-09-23T20:24:29.500 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:24:29.500 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.500 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.500 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:24:29.500 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.930860+0000", 2023-09-23T20:24:29.501 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:24:29.501 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.501 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.501 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:24:29.501 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.930905+0000", 2023-09-23T20:24:29.501 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.5185999999999997e-05 2023-09-23T20:24:29.501 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.501 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.501 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:24:29.501 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.930913+0000", 2023-09-23T20:24:29.501 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.8949999999999998e-06 2023-09-23T20:24:29.501 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.502 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.502 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:24:29.502 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.930916+0000", 2023-09-23T20:24:29.502 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.7050000000000001e-06 2023-09-23T20:24:29.502 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.502 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.502 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:24:29.502 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.930955+0000", 2023-09-23T20:24:29.502 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.8968999999999999e-05 2023-09-23T20:24:29.502 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.502 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.503 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:24:29.503 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.930965+0000", 2023-09-23T20:24:29.503 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.3899999999999999e-06 2023-09-23T20:24:29.503 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.503 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.503 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:24:29.503 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.110689+0000", 2023-09-23T20:24:29.503 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.179724142 2023-09-23T20:24:29.503 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.503 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.503 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:24:29.503 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.110861+0000", 2023-09-23T20:24:29.504 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00017222 2023-09-23T20:24:29.504 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.504 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.504 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,3,7", 2023-09-23T20:24:29.504 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.111021+0000", 2023-09-23T20:24:29.504 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00016031199999999999 2023-09-23T20:24:29.504 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.504 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.504 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:24:29.504 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.114679+0000", 2023-09-23T20:24:29.504 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0036579690000000001 2023-09-23T20:24:29.504 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.505 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.505 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:24:29.505 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.116173+0000", 2023-09-23T20:24:29.505 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0014932229999999999 2023-09-23T20:24:29.505 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.505 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.505 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:24:29.505 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.119513+0000", 2023-09-23T20:24:29.505 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0033407810000000001 2023-09-23T20:24:29.505 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.505 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.506 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:24:29.506 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.120872+0000", 2023-09-23T20:24:29.506 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0013582710000000001 2023-09-23T20:24:29.506 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.506 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.506 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:24:29.506 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.120909+0000", 2023-09-23T20:24:29.506 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.7762000000000002e-05 2023-09-23T20:24:29.506 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.506 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.507 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:24:29.507 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.121775+0000", 2023-09-23T20:24:29.507 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00086561400000000001 2023-09-23T20:24:29.507 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:24:29.507 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:24:29.507 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:24:29.507 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.507 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.507 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.34791.0:76629 5.17 5:ee2eecd0:::benchmark_data_smithi018_228451_object76628:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e146)", 2023-09-23T20:24:29.507 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:23:56.930923+0000", 2023-09-23T20:24:29.507 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 32.429242555999998, 2023-09-23T20:24:29.507 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1926419539999999, 2023-09-23T20:24:29.508 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:24:29.508 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:24:29.508 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:24:29.508 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.34791", 2023-09-23T20:24:29.508 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:24:29.508 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 76629 2023-09-23T20:24:29.508 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.508 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:24:29.508 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.508 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:24:29.508 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.930923+0000", 2023-09-23T20:24:29.508 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:24:29.509 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.509 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.509 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:24:29.509 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.930923+0000", 2023-09-23T20:24:29.509 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:24:29.509 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.509 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.509 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:24:29.509 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.930923+0000", 2023-09-23T20:24:29.509 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:24:29.509 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.509 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.510 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:24:29.510 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.930960+0000", 2023-09-23T20:24:29.510 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.7706000000000001e-05 2023-09-23T20:24:29.510 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.510 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.510 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:24:29.510 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.930965+0000", 2023-09-23T20:24:29.510 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.9540000000000003e-06 2023-09-23T20:24:29.510 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.510 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.510 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:24:29.511 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.930970+0000", 2023-09-23T20:24:29.511 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.6600000000000003e-06 2023-09-23T20:24:29.511 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.511 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.511 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:24:29.511 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.930999+0000", 2023-09-23T20:24:29.511 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.9482999999999999e-05 2023-09-23T20:24:29.511 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.511 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.511 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:24:29.511 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.931007+0000", 2023-09-23T20:24:29.511 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.2320000000000004e-06 2023-09-23T20:24:29.512 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.512 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.512 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:24:29.512 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.116200+0000", 2023-09-23T20:24:29.512 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.185193298 2023-09-23T20:24:29.512 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.512 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.512 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:24:29.512 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.116380+0000", 2023-09-23T20:24:29.512 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00018027700000000001 2023-09-23T20:24:29.512 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.512 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.513 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,3,7", 2023-09-23T20:24:29.513 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.116579+0000", 2023-09-23T20:24:29.513 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00019830800000000001 2023-09-23T20:24:29.513 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.513 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.513 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:24:29.513 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.122016+0000", 2023-09-23T20:24:29.513 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0054373540000000001 2023-09-23T20:24:29.513 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.513 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.513 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:24:29.514 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.122052+0000", 2023-09-23T20:24:29.514 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.5689999999999999e-05 2023-09-23T20:24:29.514 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.514 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.514 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:24:29.514 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.122061+0000", 2023-09-23T20:24:29.514 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.5410000000000006e-06 2023-09-23T20:24:29.514 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.514 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.514 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:24:29.515 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.122560+0000", 2023-09-23T20:24:29.515 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00049886500000000003 2023-09-23T20:24:29.515 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.515 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.515 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:24:29.515 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.122610+0000", 2023-09-23T20:24:29.515 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.0172e-05 2023-09-23T20:24:29.515 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.515 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.515 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:24:29.515 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.123565+0000", 2023-09-23T20:24:29.516 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00095512500000000003 2023-09-23T20:24:29.516 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:24:29.516 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:24:29.516 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:24:29.516 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.516 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.516 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.34791.0:76662 5.17 5:ee6eaa00:::benchmark_data_smithi018_228451_object76661:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e146)", 2023-09-23T20:24:29.517 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:23:56.930978+0000", 2023-09-23T20:24:29.517 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 32.429188093, 2023-09-23T20:24:29.517 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1935305629999999, 2023-09-23T20:24:29.517 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:24:29.517 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:24:29.517 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:24:29.518 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.34791", 2023-09-23T20:24:29.518 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:24:29.518 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 76662 2023-09-23T20:24:29.518 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.518 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:24:29.518 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.519 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:24:29.519 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.930978+0000", 2023-09-23T20:24:29.519 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:24:29.519 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.519 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.519 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:24:29.520 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.930978+0000", 2023-09-23T20:24:29.520 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:24:29.520 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.520 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.520 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:24:29.520 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.930977+0000", 2023-09-23T20:24:29.521 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:24:29.521 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.521 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.521 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:24:29.521 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.931018+0000", 2023-09-23T20:24:29.521 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.0522000000000003e-05 2023-09-23T20:24:29.522 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.522 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.522 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:24:29.522 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.931023+0000", 2023-09-23T20:24:29.522 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.2870000000000004e-06 2023-09-23T20:24:29.523 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.523 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.523 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:24:29.523 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.931027+0000", 2023-09-23T20:24:29.523 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.7009999999999998e-06 2023-09-23T20:24:29.523 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.524 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.524 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:24:29.524 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.931061+0000", 2023-09-23T20:24:29.524 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.4320999999999999e-05 2023-09-23T20:24:29.524 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.524 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.525 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:24:29.525 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.931070+0000", 2023-09-23T20:24:29.525 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.1619999999999997e-06 2023-09-23T20:24:29.525 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.525 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.525 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:24:29.525 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.117921+0000", 2023-09-23T20:24:29.526 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1868510429999999 2023-09-23T20:24:29.526 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.526 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.526 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:24:29.526 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.118064+0000", 2023-09-23T20:24:29.526 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00014286 2023-09-23T20:24:29.527 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.527 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.527 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,3,7", 2023-09-23T20:24:29.527 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.118235+0000", 2023-09-23T20:24:29.527 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00017098600000000001 2023-09-23T20:24:29.527 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.528 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.528 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:24:29.528 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.122068+0000", 2023-09-23T20:24:29.528 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0038331929999999999 2023-09-23T20:24:29.528 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.528 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.529 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:24:29.529 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.122100+0000", 2023-09-23T20:24:29.529 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.1523000000000003e-05 2023-09-23T20:24:29.529 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.529 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.529 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:24:29.530 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.122423+0000", 2023-09-23T20:24:29.530 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00032268599999999998 2023-09-23T20:24:29.530 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.530 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.530 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:24:29.530 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.123610+0000", 2023-09-23T20:24:29.531 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0011871620000000001 2023-09-23T20:24:29.531 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.531 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.531 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:24:29.531 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.123667+0000", 2023-09-23T20:24:29.531 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.7599000000000002e-05 2023-09-23T20:24:29.531 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.532 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.532 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:24:29.532 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:58.124508+0000", 2023-09-23T20:24:29.532 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00084102700000000005 2023-09-23T20:24:29.532 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:24:29.532 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:24:29.532 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:24:29.533 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.533 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.533 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.34791.0:88807 5.3 5:c1557aa9:::benchmark_data_smithi018_228451_object88806:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e151)", 2023-09-23T20:24:29.533 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:24:10.610486+0000", 2023-09-23T20:24:29.533 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 18.749679627999999, 2023-09-23T20:24:29.534 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.11438723100000001, 2023-09-23T20:24:29.534 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:24:29.534 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:24:29.534 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:24:29.534 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.34791", 2023-09-23T20:24:29.534 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:24:29.535 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 88807 2023-09-23T20:24:29.535 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.535 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:24:29.535 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.535 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:24:29.535 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:10.610486+0000", 2023-09-23T20:24:29.536 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:24:29.536 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.536 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.536 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:24:29.536 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:10.610486+0000", 2023-09-23T20:24:29.537 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:24:29.537 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.537 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.537 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:24:29.537 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:10.610484+0000", 2023-09-23T20:24:29.538 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:24:29.538 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.538 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.538 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:24:29.538 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:10.610512+0000", 2023-09-23T20:24:29.539 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.7851000000000002e-05 2023-09-23T20:24:29.539 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.539 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.539 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:24:29.539 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:10.610524+0000", 2023-09-23T20:24:29.539 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2082000000000001e-05 2023-09-23T20:24:29.540 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.540 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.540 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:24:29.540 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:10.610530+0000", 2023-09-23T20:24:29.540 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.925e-06 2023-09-23T20:24:29.541 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.541 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.541 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:24:29.541 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:10.611949+0000", 2023-09-23T20:24:29.541 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.001418908 2023-09-23T20:24:29.541 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.542 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.542 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:24:29.542 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:10.612074+0000", 2023-09-23T20:24:29.542 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00012517499999999999 2023-09-23T20:24:29.542 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.543 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.543 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 5,6", 2023-09-23T20:24:29.543 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:10.612206+0000", 2023-09-23T20:24:29.543 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000131323 2023-09-23T20:24:29.543 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.544 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.544 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:24:29.544 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:10.614121+0000", 2023-09-23T20:24:29.544 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0019150390000000001 2023-09-23T20:24:29.544 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.544 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.545 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:24:29.545 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:10.619284+0000", 2023-09-23T20:24:29.545 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0051638279999999996 2023-09-23T20:24:29.545 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.545 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.545 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:24:29.545 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:10.724766+0000", 2023-09-23T20:24:29.545 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.105481609 2023-09-23T20:24:29.546 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.546 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.546 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:24:29.546 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:10.724828+0000", 2023-09-23T20:24:29.546 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.1623000000000003e-05 2023-09-23T20:24:29.546 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.546 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.547 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:24:29.547 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:10.724873+0000", 2023-09-23T20:24:29.547 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.5856999999999998e-05 2023-09-23T20:24:29.547 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:24:29.547 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:24:29.547 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:24:29.547 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.548 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.548 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.34791.0:88808 5.5 5:a05dfd91:::benchmark_data_smithi018_228451_object88807:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e151)", 2023-09-23T20:24:29.548 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:24:10.611856+0000", 2023-09-23T20:24:29.548 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 18.748310218, 2023-09-23T20:24:29.548 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.11362781499999999, 2023-09-23T20:24:29.548 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:24:29.548 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:24:29.548 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:24:29.549 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.34791", 2023-09-23T20:24:29.549 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:24:29.549 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 88808 2023-09-23T20:24:29.549 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.549 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:24:29.549 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.549 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:24:29.549 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:10.611856+0000", 2023-09-23T20:24:29.549 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:24:29.549 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.549 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.550 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:24:29.550 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:10.611856+0000", 2023-09-23T20:24:29.550 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:24:29.550 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.550 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.550 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:24:29.550 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:10.611855+0000", 2023-09-23T20:24:29.550 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.999999 2023-09-23T20:24:29.550 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.550 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.551 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:24:29.551 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:10.611877+0000", 2023-09-23T20:24:29.551 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.2041000000000001e-05 2023-09-23T20:24:29.551 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.551 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.551 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:24:29.551 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:10.611888+0000", 2023-09-23T20:24:29.551 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1234e-05 2023-09-23T20:24:29.551 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.551 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.551 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:24:29.551 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:10.611895+0000", 2023-09-23T20:24:29.551 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.0559999999999999e-06 2023-09-23T20:24:29.552 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.552 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.552 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:24:29.552 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:10.611968+0000", 2023-09-23T20:24:29.552 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.3188000000000005e-05 2023-09-23T20:24:29.552 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.552 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.552 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:24:29.552 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:10.612081+0000", 2023-09-23T20:24:29.552 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000112679 2023-09-23T20:24:29.552 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.552 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.552 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,4", 2023-09-23T20:24:29.553 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:10.612206+0000", 2023-09-23T20:24:29.553 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00012457899999999999 2023-09-23T20:24:29.553 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.553 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.553 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:24:29.553 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:10.614444+0000", 2023-09-23T20:24:29.553 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00223873 2023-09-23T20:24:29.553 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.553 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.553 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:24:29.553 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:10.617413+0000", 2023-09-23T20:24:29.553 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0029683000000000001 2023-09-23T20:24:29.554 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.554 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.554 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:24:29.554 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:10.725394+0000", 2023-09-23T20:24:29.554 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.107981699 2023-09-23T20:24:29.554 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.554 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.554 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:24:29.555 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:10.725445+0000", 2023-09-23T20:24:29.555 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.0801999999999999e-05 2023-09-23T20:24:29.555 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.555 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.555 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:24:29.555 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:10.725483+0000", 2023-09-23T20:24:29.555 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.8417999999999997e-05 2023-09-23T20:24:29.555 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:24:29.555 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:24:29.555 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:24:29.555 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.555 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.556 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.34791.0:88814 5.17 5:ee898aee:::benchmark_data_smithi018_228451_object88813:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e151)", 2023-09-23T20:24:29.556 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:24:10.614697+0000", 2023-09-23T20:24:29.556 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 18.745468498000001, 2023-09-23T20:24:29.556 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.112850018, 2023-09-23T20:24:29.556 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:24:29.556 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:24:29.556 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:24:29.556 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.34791", 2023-09-23T20:24:29.556 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:24:29.556 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 88814 2023-09-23T20:24:29.556 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.556 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:24:29.556 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.557 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:24:29.557 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:10.614697+0000", 2023-09-23T20:24:29.557 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:24:29.557 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.557 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.557 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:24:29.557 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:10.614697+0000", 2023-09-23T20:24:29.557 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:24:29.557 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.557 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.557 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:24:29.557 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:10.614696+0000", 2023-09-23T20:24:29.558 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:24:29.558 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.558 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.558 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:24:29.558 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:10.614716+0000", 2023-09-23T20:24:29.558 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.0259999999999999e-05 2023-09-23T20:24:29.558 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.558 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.558 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:24:29.559 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:10.614725+0000", 2023-09-23T20:24:29.559 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.7050000000000005e-06 2023-09-23T20:24:29.559 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.559 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.559 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:24:29.559 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:10.614730+0000", 2023-09-23T20:24:29.559 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.3299999999999998e-06 2023-09-23T20:24:29.559 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.559 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.559 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:24:29.559 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:10.614783+0000", 2023-09-23T20:24:29.559 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.2973000000000003e-05 2023-09-23T20:24:29.559 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.560 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.560 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:24:29.560 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:10.614948+0000", 2023-09-23T20:24:29.560 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000164699 2023-09-23T20:24:29.560 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.560 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.560 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,3,7", 2023-09-23T20:24:29.560 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:10.615093+0000", 2023-09-23T20:24:29.560 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00014474999999999999 2023-09-23T20:24:29.560 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.560 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.560 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:24:29.561 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:10.616974+0000", 2023-09-23T20:24:29.561 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.001881733 2023-09-23T20:24:29.561 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.561 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.561 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:24:29.561 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:10.617398+0000", 2023-09-23T20:24:29.561 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00042315199999999999 2023-09-23T20:24:29.561 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.561 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.561 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:24:29.561 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:10.619431+0000", 2023-09-23T20:24:29.561 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0020337290000000002 2023-09-23T20:24:29.561 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.562 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.562 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:24:29.562 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:10.726232+0000", 2023-09-23T20:24:29.562 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.10680112 2023-09-23T20:24:29.562 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.562 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.562 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:24:29.562 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:10.726280+0000", 2023-09-23T20:24:29.562 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.8010999999999998e-05 2023-09-23T20:24:29.562 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:29.562 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:29.563 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:24:29.563 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:10.727547+0000", 2023-09-23T20:24:29.563 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.001266945 2023-09-23T20:24:29.563 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:24:29.563 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:24:29.563 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:24:29.563 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:24:29.563 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:24:29.563 INFO:teuthology.orchestra.run.smithi018.stdout:} 2023-09-23T20:24:29.564 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:24:29.564 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:29.565 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:24:29.565 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:24:29.666 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:24:29.717 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:29.775 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:24:29.782 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:29.782 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:24:29.883 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:24:29.895 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T20:24:29.920 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-23T20:24:29.920 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:24:29.920 INFO:tasks.thrashosds.thrasher:Removing osd 3, in_osds are: [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-23T20:24:29.920 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd out 3 2023-09-23T20:24:29.995 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:30.085 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:24:30.098 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:30.099 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:24:30.199 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:24:30.256 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:30.309 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:24:30.313 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:30.314 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:24:30.415 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:24:30.450 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:30.471 INFO:teuthology.orchestra.run.smithi018.stderr:noscrub is set 2023-09-23T20:24:30.502 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:24:30.508 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:30 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:24:30.508 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:30 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:24:30.508 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:30 smithi149 ceph-mon[170394]: pgmap v322: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 96 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s; 2426/315246 objects degraded (0.770%); 5252/315246 objects misplaced (1.666%); 3.6 MiB/s, 56 objects/s recovering 2023-09-23T20:24:30.508 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:30 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3405904859' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:24:30.508 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:30.509 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:24:30.609 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:24:30.644 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:30 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:24:30.645 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:30 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:24:30.645 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:30 smithi018 ceph-mon[179647]: pgmap v322: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 96 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s; 2426/315246 objects degraded (0.770%); 5252/315246 objects misplaced (1.666%); 3.6 MiB/s, 56 objects/s recovering 2023-09-23T20:24:30.645 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:30 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3405904859' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:24:30.645 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:30 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:24:30.645 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:30 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:24:30.645 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:30 smithi018 ceph-mon[184144]: pgmap v322: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 96 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s; 2426/315246 objects degraded (0.770%); 5252/315246 objects misplaced (1.666%); 3.6 MiB/s, 56 objects/s recovering 2023-09-23T20:24:30.645 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:30 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3405904859' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:24:30.667 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:30.735 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:24:30.738 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:30.739 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:24:30.840 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:24:30.897 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:30.965 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:24:30.973 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:30.974 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:24:31.074 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:24:31.120 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:31.185 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:24:31.190 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:31.190 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:24:31.291 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:24:31.373 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:31.492 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:24:31.542 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:24:31.547 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:31.548 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:24:31.648 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:24:31.708 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:31 smithi018 ceph-mon[179647]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:24:31.708 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:31 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3405904859' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:24:31.708 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:31 smithi018 ceph-mon[179647]: osdmap e155: 8 total, 8 up, 8 in 2023-09-23T20:24:31.708 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:31 smithi018 ceph-mon[179647]: failed to encode map e155 with expected crc 2023-09-23T20:24:31.708 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:31 smithi018 ceph-mon[179647]: failed to encode map e155 with expected crc 2023-09-23T20:24:31.708 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:31 smithi018 ceph-mon[179647]: failed to encode map e155 with expected crc 2023-09-23T20:24:31.708 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:31 smithi018 ceph-mon[179647]: failed to encode map e155 with expected crc 2023-09-23T20:24:31.709 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:31 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 2426/315246 objects degraded (0.770%), 1 pg degraded (PG_DEGRADED) 2023-09-23T20:24:31.709 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:31 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:24:31.709 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:31 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:24:31.709 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:31 smithi018 ceph-mon[184144]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:24:31.709 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:31 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3405904859' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:24:31.709 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:31 smithi018 ceph-mon[184144]: osdmap e155: 8 total, 8 up, 8 in 2023-09-23T20:24:31.709 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:31 smithi018 ceph-mon[184144]: failed to encode map e155 with expected crc 2023-09-23T20:24:31.709 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:31 smithi018 ceph-mon[184144]: failed to encode map e155 with expected crc 2023-09-23T20:24:31.709 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:31 smithi018 ceph-mon[184144]: failed to encode map e155 with expected crc 2023-09-23T20:24:31.709 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:31 smithi018 ceph-mon[184144]: failed to encode map e155 with expected crc 2023-09-23T20:24:31.710 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:31 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 2426/315246 objects degraded (0.770%), 1 pg degraded (PG_DEGRADED) 2023-09-23T20:24:31.710 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:31 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:24:31.710 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:31 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:24:31.711 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:31.719 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:31 smithi149 ceph-mon[170394]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:24:31.719 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:31 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3405904859' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:24:31.719 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:31 smithi149 ceph-mon[170394]: osdmap e155: 8 total, 8 up, 8 in 2023-09-23T20:24:31.719 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:31 smithi149 ceph-mon[170394]: failed to encode map e155 with expected crc 2023-09-23T20:24:31.719 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:31 smithi149 ceph-mon[170394]: failed to encode map e155 with expected crc 2023-09-23T20:24:31.719 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:31 smithi149 ceph-mon[170394]: failed to encode map e155 with expected crc 2023-09-23T20:24:31.719 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:31 smithi149 ceph-mon[170394]: failed to encode map e155 with expected crc 2023-09-23T20:24:31.719 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:31 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 2426/315246 objects degraded (0.770%), 1 pg degraded (PG_DEGRADED) 2023-09-23T20:24:31.719 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:31 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:24:31.720 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:31 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:24:31.878 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:24:31.900 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:24:31.910 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:31.910 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:24:31.960 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:24:31.996 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:24:32.011 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:24:32.043 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:24:32.086 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:32.092 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:24:32.134 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:24:32.175 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:24:32.221 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:24:32.263 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:24:32.284 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:24:32.284 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:24:32.284 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:24:32.284 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:24:32.284 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:24:32.285 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:24:32.285 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:24:32.285 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:24:32.719 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:32 smithi149 ceph-mon[170394]: pgmap v324: 99 pgs: 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 96 active+clean; 6.5 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 49 MiB/s wr, 898 op/s; 2426/316944 objects degraded (0.765%); 5252/316944 objects misplaced (1.657%); 2.2 MiB/s, 34 objects/s recovering 2023-09-23T20:24:32.720 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:32 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2472371301' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:24:32.720 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:32 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:24:32.720 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:32 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:24:32.720 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:32 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:24:32.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:32 smithi018 ceph-mon[184144]: pgmap v324: 99 pgs: 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 96 active+clean; 6.5 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 49 MiB/s wr, 898 op/s; 2426/316944 objects degraded (0.765%); 5252/316944 objects misplaced (1.657%); 2.2 MiB/s, 34 objects/s recovering 2023-09-23T20:24:32.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:32 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2472371301' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:24:32.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:32 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:24:32.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:32 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:24:32.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:32 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:24:32.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:32 smithi018 ceph-mon[179647]: pgmap v324: 99 pgs: 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 96 active+clean; 6.5 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 49 MiB/s wr, 898 op/s; 2426/316944 objects degraded (0.765%); 5252/316944 objects misplaced (1.657%); 2.2 MiB/s, 34 objects/s recovering 2023-09-23T20:24:32.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:32 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2472371301' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:24:32.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:32 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:24:32.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:32 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:24:32.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:32 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:24:33.123 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T20:24:33.969 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:24:33.972 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:33.973 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:24:34.074 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:24:34.137 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:34.198 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:24:34.203 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:34.204 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:24:34.304 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:24:34.378 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:34.694 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:24:34.697 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:34.698 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:24:34.798 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:24:34.837 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:34 smithi149 ceph-mon[170394]: pgmap v325: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 96 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 102 B/s rd, 43 MiB/s wr, 963 op/s; 2426/313107 objects degraded (0.775%); 4694/313107 objects misplaced (1.499%); 3.5 MiB/s, 56 objects/s recovering 2023-09-23T20:24:34.843 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:34.862 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:34 smithi018 ceph-mon[179647]: pgmap v325: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 96 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 102 B/s rd, 43 MiB/s wr, 963 op/s; 2426/313107 objects degraded (0.775%); 4694/313107 objects misplaced (1.499%); 3.5 MiB/s, 56 objects/s recovering 2023-09-23T20:24:34.863 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:34 smithi018 ceph-mon[184144]: pgmap v325: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 96 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 102 B/s rd, 43 MiB/s wr, 963 op/s; 2426/313107 objects degraded (0.775%); 4694/313107 objects misplaced (1.499%); 3.5 MiB/s, 56 objects/s recovering 2023-09-23T20:24:34.895 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:24:34.900 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:34.900 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:24:35.001 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:24:35.060 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:35.114 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:24:35.137 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:24:35.142 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:24:34 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:24:34] "GET /metrics HTTP/1.1" 200 36474 "" "Prometheus/2.43.0" 2023-09-23T20:24:35.145 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:35.145 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:24:35.246 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:24:35.311 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:35.382 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:24:35.394 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:35.394 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:24:35.495 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:24:35.561 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:35.628 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:24:35.636 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:35.636 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:24:35.737 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:24:35.878 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:35.955 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:24:35.961 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:35.961 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:24:35.962 INFO:teuthology.orchestra.run.smithi018.stdout:ERROR: (22) Invalid argument 2023-09-23T20:24:35.962 INFO:teuthology.orchestra.run.smithi018.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:24:36.063 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:24:36.117 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:36.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:35 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mon-a[179623]: 2023-09-23T20:24:35.924+0000 7f160fef9700 -1 mon.a@0(leader).osd e155 definitely_dead 0 2023-09-23T20:24:36.343 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:24:36.347 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:36.348 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:24:36.378 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:24:36.448 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:24:36.536 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:36.680 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:24:36.685 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:36.686 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:24:36.725 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T20:24:36.731 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:36 smithi018 ceph-mon[184144]: pgmap v326: 99 pgs: 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 96 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 102 B/s rd, 27 MiB/s wr, 976 op/s; 2426/304911 objects degraded (0.796%); 4694/304911 objects misplaced (1.539%); 3.5 MiB/s, 56 objects/s recovering 2023-09-23T20:24:36.731 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:36 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 2426/304911 objects degraded (0.796%), 1 pg degraded (PG_DEGRADED) 2023-09-23T20:24:36.731 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:36 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3891932599' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["3"]}]: dispatch 2023-09-23T20:24:36.732 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:36 smithi018 ceph-mon[184144]: Client client.admin marked osd.3 out, while it was still marked up 2023-09-23T20:24:36.732 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:36 smithi018 ceph-mon[179647]: pgmap v326: 99 pgs: 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 96 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 102 B/s rd, 27 MiB/s wr, 976 op/s; 2426/304911 objects degraded (0.796%); 4694/304911 objects misplaced (1.539%); 3.5 MiB/s, 56 objects/s recovering 2023-09-23T20:24:36.732 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:36 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 2426/304911 objects degraded (0.796%), 1 pg degraded (PG_DEGRADED) 2023-09-23T20:24:36.732 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:36 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3891932599' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["3"]}]: dispatch 2023-09-23T20:24:36.732 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:36 smithi018 ceph-mon[179647]: Client client.admin marked osd.3 out, while it was still marked up 2023-09-23T20:24:36.787 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:24:36.843 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:36 smithi149 ceph-mon[170394]: pgmap v326: 99 pgs: 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 96 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 102 B/s rd, 27 MiB/s wr, 976 op/s; 2426/304911 objects degraded (0.796%); 4694/304911 objects misplaced (1.539%); 3.5 MiB/s, 56 objects/s recovering 2023-09-23T20:24:36.843 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:36 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 2426/304911 objects degraded (0.796%), 1 pg degraded (PG_DEGRADED) 2023-09-23T20:24:36.843 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:36 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3891932599' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["3"]}]: dispatch 2023-09-23T20:24:36.843 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:36 smithi149 ceph-mon[170394]: Client client.admin marked osd.3 out, while it was still marked up 2023-09-23T20:24:36.846 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:36.904 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:24:36.914 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:36.915 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:24:36.953 INFO:teuthology.orchestra.run.smithi018.stderr:marked out osd.3. 2023-09-23T20:24:37.016 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:24:37.058 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:37.156 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:24:37.158 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:37.159 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:24:37.259 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:24:37.285 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:37.320 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:24:37.322 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:37.323 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:24:37.423 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:24:37.452 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:37.595 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:24:37.599 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:24:37.602 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:37.603 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:24:37.704 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:24:37.730 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:37.761 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:24:37.764 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:37.765 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:24:37.865 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:24:37.892 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:37.938 INFO:teuthology.orchestra.run.smithi018.stderr:nodeep-scrub is set 2023-09-23T20:24:37.942 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:24:37.944 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:37.945 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:24:38.041 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:24:38.045 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:24:38.080 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:37 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3891932599' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["3"]}]': finished 2023-09-23T20:24:38.081 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:37 smithi018 ceph-mon[179647]: osdmap e156: 8 total, 8 up, 7 in 2023-09-23T20:24:38.081 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:37 smithi018 ceph-mon[179647]: failed to encode map e156 with expected crc 2023-09-23T20:24:38.081 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:37 smithi018 ceph-mon[179647]: failed to encode map e156 with expected crc 2023-09-23T20:24:38.081 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:37 smithi018 ceph-mon[179647]: failed to encode map e156 with expected crc 2023-09-23T20:24:38.081 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:37 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1367656665' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:24:38.081 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:37 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:24:38.081 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:37 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3891932599' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["3"]}]': finished 2023-09-23T20:24:38.081 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:37 smithi018 ceph-mon[184144]: osdmap e156: 8 total, 8 up, 7 in 2023-09-23T20:24:38.081 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:37 smithi018 ceph-mon[184144]: failed to encode map e156 with expected crc 2023-09-23T20:24:38.081 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:37 smithi018 ceph-mon[184144]: failed to encode map e156 with expected crc 2023-09-23T20:24:38.082 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:37 smithi018 ceph-mon[184144]: failed to encode map e156 with expected crc 2023-09-23T20:24:38.082 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:37 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1367656665' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:24:38.082 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:37 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:24:38.084 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:38.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:37 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3891932599' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["3"]}]': finished 2023-09-23T20:24:38.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:37 smithi149 ceph-mon[170394]: osdmap e156: 8 total, 8 up, 7 in 2023-09-23T20:24:38.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:37 smithi149 ceph-mon[170394]: failed to encode map e156 with expected crc 2023-09-23T20:24:38.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:37 smithi149 ceph-mon[170394]: failed to encode map e156 with expected crc 2023-09-23T20:24:38.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:37 smithi149 ceph-mon[170394]: failed to encode map e156 with expected crc 2023-09-23T20:24:38.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:37 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1367656665' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:24:38.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:37 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:24:38.388 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:24:38.390 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:38.390 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:24:38.491 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:24:38.518 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:38.551 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:24:38.554 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:38.555 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:24:38.656 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:24:38.687 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:38.972 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:38 smithi018 ceph-mon[179647]: pgmap v328: 99 pgs: 4 peering, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 92 active+clean; 6.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 127 B/s rd, 16 MiB/s wr, 1.09k op/s; 2426/301023 objects degraded (0.806%); 4694/301023 objects misplaced (1.559%); 2.2 MiB/s, 34 objects/s recovering 2023-09-23T20:24:38.972 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:38 smithi018 ceph-mon[179647]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T20:24:38.972 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:38 smithi018 ceph-mon[179647]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:24:38.972 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:38 smithi018 ceph-mon[179647]: failed to encode map e157 with expected crc 2023-09-23T20:24:38.972 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:38 smithi018 ceph-mon[179647]: failed to encode map e157 with expected crc 2023-09-23T20:24:38.973 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:38 smithi018 ceph-mon[179647]: failed to encode map e157 with expected crc 2023-09-23T20:24:38.973 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:38 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:24:38.973 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:38 smithi018 ceph-mon[179647]: failed to encode map e157 with expected crc 2023-09-23T20:24:38.973 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:38 smithi018 ceph-mon[179647]: osdmap e157: 8 total, 8 up, 7 in 2023-09-23T20:24:38.973 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:38 smithi018 ceph-mon[179647]: failed to encode map e157 with expected crc 2023-09-23T20:24:38.973 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:38 smithi018 ceph-mon[179647]: failed to encode map e157 with expected crc 2023-09-23T20:24:38.973 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:38 smithi018 ceph-mon[179647]: failed to encode map e157 with expected crc 2023-09-23T20:24:38.973 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:38 smithi018 ceph-mon[179647]: failed to encode map e157 with expected crc 2023-09-23T20:24:38.973 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:38 smithi018 ceph-mon[179647]: failed to encode map e157 with expected crc 2023-09-23T20:24:38.974 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:38 smithi018 ceph-mon[184144]: pgmap v328: 99 pgs: 4 peering, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 92 active+clean; 6.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 127 B/s rd, 16 MiB/s wr, 1.09k op/s; 2426/301023 objects degraded (0.806%); 4694/301023 objects misplaced (1.559%); 2.2 MiB/s, 34 objects/s recovering 2023-09-23T20:24:38.974 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:38 smithi018 ceph-mon[184144]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T20:24:38.975 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:38 smithi018 ceph-mon[184144]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:24:38.975 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:38 smithi018 ceph-mon[184144]: failed to encode map e157 with expected crc 2023-09-23T20:24:38.975 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:38 smithi018 ceph-mon[184144]: failed to encode map e157 with expected crc 2023-09-23T20:24:38.975 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:38 smithi018 ceph-mon[184144]: failed to encode map e157 with expected crc 2023-09-23T20:24:38.975 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:38 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:24:38.975 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:38 smithi018 ceph-mon[184144]: failed to encode map e157 with expected crc 2023-09-23T20:24:38.975 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:38 smithi018 ceph-mon[184144]: osdmap e157: 8 total, 8 up, 7 in 2023-09-23T20:24:38.975 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:38 smithi018 ceph-mon[184144]: failed to encode map e157 with expected crc 2023-09-23T20:24:38.975 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:38 smithi018 ceph-mon[184144]: failed to encode map e157 with expected crc 2023-09-23T20:24:38.975 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:38 smithi018 ceph-mon[184144]: failed to encode map e157 with expected crc 2023-09-23T20:24:38.975 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:38 smithi018 ceph-mon[184144]: failed to encode map e157 with expected crc 2023-09-23T20:24:38.975 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:38 smithi018 ceph-mon[184144]: failed to encode map e157 with expected crc 2023-09-23T20:24:39.087 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:24:39.098 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:39.098 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:24:39.199 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:24:39.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:38 smithi149 ceph-mon[170394]: pgmap v328: 99 pgs: 4 peering, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 92 active+clean; 6.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 127 B/s rd, 16 MiB/s wr, 1.09k op/s; 2426/301023 objects degraded (0.806%); 4694/301023 objects misplaced (1.559%); 2.2 MiB/s, 34 objects/s recovering 2023-09-23T20:24:39.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:38 smithi149 ceph-mon[170394]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T20:24:39.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:38 smithi149 ceph-mon[170394]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:24:39.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:38 smithi149 ceph-mon[170394]: failed to encode map e157 with expected crc 2023-09-23T20:24:39.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:38 smithi149 ceph-mon[170394]: failed to encode map e157 with expected crc 2023-09-23T20:24:39.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:38 smithi149 ceph-mon[170394]: failed to encode map e157 with expected crc 2023-09-23T20:24:39.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:38 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:24:39.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:38 smithi149 ceph-mon[170394]: failed to encode map e157 with expected crc 2023-09-23T20:24:39.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:38 smithi149 ceph-mon[170394]: osdmap e157: 8 total, 8 up, 7 in 2023-09-23T20:24:39.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:38 smithi149 ceph-mon[170394]: failed to encode map e157 with expected crc 2023-09-23T20:24:39.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:38 smithi149 ceph-mon[170394]: failed to encode map e157 with expected crc 2023-09-23T20:24:39.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:38 smithi149 ceph-mon[170394]: failed to encode map e157 with expected crc 2023-09-23T20:24:39.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:38 smithi149 ceph-mon[170394]: failed to encode map e157 with expected crc 2023-09-23T20:24:39.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:38 smithi149 ceph-mon[170394]: failed to encode map e157 with expected crc 2023-09-23T20:24:39.240 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:40.093 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:24:40.096 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:40.096 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:24:40.197 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:24:40.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:39 smithi149 ceph-mon[170394]: failed to encode map e158 with expected crc 2023-09-23T20:24:40.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:39 smithi149 ceph-mon[170394]: osdmap e158: 8 total, 8 up, 7 in 2023-09-23T20:24:40.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:39 smithi149 ceph-mon[170394]: failed to encode map e158 with expected crc 2023-09-23T20:24:40.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:39 smithi149 ceph-mon[170394]: failed to encode map e158 with expected crc 2023-09-23T20:24:40.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:39 smithi149 ceph-mon[170394]: failed to encode map e158 with expected crc 2023-09-23T20:24:40.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:39 smithi149 ceph-mon[170394]: failed to encode map e158 with expected crc 2023-09-23T20:24:40.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:39 smithi149 ceph-mon[170394]: failed to encode map e158 with expected crc 2023-09-23T20:24:40.223 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:40.256 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:24:40.258 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:40.259 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:24:40.360 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:24:40.386 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:40.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:39 smithi018 ceph-mon[179647]: failed to encode map e158 with expected crc 2023-09-23T20:24:40.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:39 smithi018 ceph-mon[179647]: osdmap e158: 8 total, 8 up, 7 in 2023-09-23T20:24:40.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:39 smithi018 ceph-mon[179647]: failed to encode map e158 with expected crc 2023-09-23T20:24:40.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:39 smithi018 ceph-mon[179647]: failed to encode map e158 with expected crc 2023-09-23T20:24:40.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:39 smithi018 ceph-mon[179647]: failed to encode map e158 with expected crc 2023-09-23T20:24:40.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:39 smithi018 ceph-mon[179647]: failed to encode map e158 with expected crc 2023-09-23T20:24:40.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:39 smithi018 ceph-mon[179647]: failed to encode map e158 with expected crc 2023-09-23T20:24:40.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:39 smithi018 ceph-mon[184144]: failed to encode map e158 with expected crc 2023-09-23T20:24:40.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:39 smithi018 ceph-mon[184144]: osdmap e158: 8 total, 8 up, 7 in 2023-09-23T20:24:40.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:39 smithi018 ceph-mon[184144]: failed to encode map e158 with expected crc 2023-09-23T20:24:40.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:39 smithi018 ceph-mon[184144]: failed to encode map e158 with expected crc 2023-09-23T20:24:40.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:39 smithi018 ceph-mon[184144]: failed to encode map e158 with expected crc 2023-09-23T20:24:40.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:39 smithi018 ceph-mon[184144]: failed to encode map e158 with expected crc 2023-09-23T20:24:40.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:39 smithi018 ceph-mon[184144]: failed to encode map e158 with expected crc 2023-09-23T20:24:40.419 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:24:40.421 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:40.422 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:24:40.522 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:24:40.558 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:40.757 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:24:40.757 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:24:40.757 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:24:40.757 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:24:40.758 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:24:40.758 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:24:40.758 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:24:40.758 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:24:40.758 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:24:40.758 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:24:40.759 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:24:40.759 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:24:40.759 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:24:40.759 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:24:40.759 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:24:40.759 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:24:40.950 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:24:40.953 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:40.953 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:24:41.054 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:24:41.081 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:41.114 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:24:41.117 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:41.117 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:24:41.171 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T20:24:41.200 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:40 smithi018 ceph-mon[184144]: pgmap v331: 99 pgs: 1 remapped+peering, 4 peering, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 91 active+clean; 6.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 759 op/s; 285/299430 objects degraded (0.095%); 6623/299430 objects misplaced (2.212%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T20:24:41.200 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:40 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1907111120' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:24:41.200 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:40 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 285/299430 objects degraded (0.095%), 1 pg degraded (PG_DEGRADED) 2023-09-23T20:24:41.200 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:40 smithi018 ceph-mon[179647]: pgmap v331: 99 pgs: 1 remapped+peering, 4 peering, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 91 active+clean; 6.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 759 op/s; 285/299430 objects degraded (0.095%); 6623/299430 objects misplaced (2.212%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T20:24:41.200 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:40 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1907111120' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:24:41.201 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:40 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 285/299430 objects degraded (0.095%), 1 pg degraded (PG_DEGRADED) 2023-09-23T20:24:41.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:40 smithi149 ceph-mon[170394]: pgmap v331: 99 pgs: 1 remapped+peering, 4 peering, 1 active+recovery_wait, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 91 active+clean; 6.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 759 op/s; 285/299430 objects degraded (0.095%); 6623/299430 objects misplaced (2.212%); 1.1 MiB/s, 17 objects/s recovering 2023-09-23T20:24:41.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:40 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1907111120' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:24:41.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:40 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 285/299430 objects degraded (0.095%), 1 pg degraded (PG_DEGRADED) 2023-09-23T20:24:41.218 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:24:41.249 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:41.926 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:24:41.930 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:41.931 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:24:42.031 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:24:42.062 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:42.253 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2023-09-23T20:24:42.254 INFO:teuthology.orchestra.run.smithi018.stdout: "ops": [], 2023-09-23T20:24:42.254 INFO:teuthology.orchestra.run.smithi018.stdout: "complaint_time": 30, 2023-09-23T20:24:42.254 INFO:teuthology.orchestra.run.smithi018.stdout: "num_blocked_ops": 0 2023-09-23T20:24:42.254 INFO:teuthology.orchestra.run.smithi018.stdout:} 2023-09-23T20:24:42.300 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:24:42.304 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:42.304 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:24:42.405 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:24:42.457 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:42.636 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 4, 5, 6, 7] out_osds: [3] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-23T20:24:42.636 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:24:42.637 INFO:tasks.thrashosds.thrasher:Removing osd 2, in_osds are: [0, 1, 2, 4, 5, 6, 7] 2023-09-23T20:24:42.637 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd out 2 2023-09-23T20:24:42.663 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:24:42.673 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:42.674 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:24:42.719 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:24:42.743 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T20:24:42.775 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:24:42.821 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:42.920 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:42 smithi018 ceph-mon[184144]: pgmap v332: 99 pgs: 87 active+clean, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 4 peering, 1 remapped+peering; 6.0 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 523 op/s; 17736/295479 objects degraded (6.002%); 4355/295479 objects misplaced (1.474%); 3.9 MiB/s, 63 objects/s recovering 2023-09-23T20:24:42.920 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:42 smithi018 ceph-mon[179647]: pgmap v332: 99 pgs: 87 active+clean, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 4 peering, 1 remapped+peering; 6.0 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 523 op/s; 17736/295479 objects degraded (6.002%); 4355/295479 objects misplaced (1.474%); 3.9 MiB/s, 63 objects/s recovering 2023-09-23T20:24:42.973 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:24:42.976 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:42.977 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:24:43.077 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:24:43.104 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:43.137 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:24:43.140 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:43.140 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:24:43.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:42 smithi149 ceph-mon[170394]: pgmap v332: 99 pgs: 87 active+clean, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 4 peering, 1 remapped+peering; 6.0 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 523 op/s; 17736/295479 objects degraded (6.002%); 4355/295479 objects misplaced (1.474%); 3.9 MiB/s, 63 objects/s recovering 2023-09-23T20:24:43.241 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:24:43.278 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:43.503 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:24:43.511 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:43.512 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:24:43.555 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:24:43.613 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:24:43.640 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:43.675 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:24:43.677 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:43.678 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:24:43.767 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:24:43.779 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:24:43.823 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:43.999 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:24:44.005 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:44.005 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:24:44.031 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:43 smithi018 ceph-mon[184144]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T20:24:44.031 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:43 smithi018 ceph-mon[179647]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T20:24:44.106 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:24:44.142 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:44.350 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:24:44.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:43 smithi149 ceph-mon[170394]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T20:24:45.003 INFO:teuthology.orchestra.run.smithi018.stderr:noscrub is unset 2023-09-23T20:24:45.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:44 smithi018 ceph-mon[179647]: pgmap v333: 99 pgs: 89 active+clean, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 remapped+peering; 6.0 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 482 op/s; 24466/295371 objects degraded (8.283%); 4355/295371 objects misplaced (1.474%); 3.6 MiB/s, 57 objects/s recovering 2023-09-23T20:24:45.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:44 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2713509329' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:24:45.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:44 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:24:45.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:44 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:24:45.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:44 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:24:45.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:44 smithi018 ceph-mon[184144]: pgmap v333: 99 pgs: 89 active+clean, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 remapped+peering; 6.0 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 482 op/s; 24466/295371 objects degraded (8.283%); 4355/295371 objects misplaced (1.474%); 3.6 MiB/s, 57 objects/s recovering 2023-09-23T20:24:45.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:44 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2713509329' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:24:45.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:44 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:24:45.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:44 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:24:45.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:44 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:24:45.392 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:24:44 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:24:44] "GET /metrics HTTP/1.1" 200 36499 "" "Prometheus/2.43.0" 2023-09-23T20:24:45.471 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:44 smithi149 ceph-mon[170394]: pgmap v333: 99 pgs: 89 active+clean, 8 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 remapped+peering; 6.0 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 482 op/s; 24466/295371 objects degraded (8.283%); 4355/295371 objects misplaced (1.474%); 3.6 MiB/s, 57 objects/s recovering 2023-09-23T20:24:45.471 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:44 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2713509329' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:24:45.471 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:44 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:24:45.471 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:44 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:24:45.472 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:44 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:24:46.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:46 smithi018 ceph-mon[184144]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:24:46.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:46 smithi018 ceph-mon[184144]: failed to encode map e159 with expected crc 2023-09-23T20:24:46.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:46 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:24:46.394 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:46 smithi018 ceph-mon[184144]: osdmap e159: 8 total, 8 up, 7 in 2023-09-23T20:24:46.394 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:46 smithi018 ceph-mon[184144]: failed to encode map e159 with expected crc 2023-09-23T20:24:46.394 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:46 smithi018 ceph-mon[184144]: failed to encode map e159 with expected crc 2023-09-23T20:24:46.394 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:46 smithi018 ceph-mon[184144]: failed to encode map e159 with expected crc 2023-09-23T20:24:46.394 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:46 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 24466/295371 objects degraded (8.283%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T20:24:46.394 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:46 smithi018 ceph-mon[179647]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:24:46.394 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:46 smithi018 ceph-mon[179647]: failed to encode map e159 with expected crc 2023-09-23T20:24:46.394 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:46 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:24:46.395 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:46 smithi018 ceph-mon[179647]: osdmap e159: 8 total, 8 up, 7 in 2023-09-23T20:24:46.395 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:46 smithi018 ceph-mon[179647]: failed to encode map e159 with expected crc 2023-09-23T20:24:46.395 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:46 smithi018 ceph-mon[179647]: failed to encode map e159 with expected crc 2023-09-23T20:24:46.395 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:46 smithi018 ceph-mon[179647]: failed to encode map e159 with expected crc 2023-09-23T20:24:46.395 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:46 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 24466/295371 objects degraded (8.283%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T20:24:46.468 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:46 smithi149 ceph-mon[170394]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:24:46.469 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:46 smithi149 ceph-mon[170394]: failed to encode map e159 with expected crc 2023-09-23T20:24:46.469 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:46 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:24:46.469 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:46 smithi149 ceph-mon[170394]: osdmap e159: 8 total, 8 up, 7 in 2023-09-23T20:24:46.469 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:46 smithi149 ceph-mon[170394]: failed to encode map e159 with expected crc 2023-09-23T20:24:46.469 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:46 smithi149 ceph-mon[170394]: failed to encode map e159 with expected crc 2023-09-23T20:24:46.470 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:46 smithi149 ceph-mon[170394]: failed to encode map e159 with expected crc 2023-09-23T20:24:46.470 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:46 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 24466/295371 objects degraded (8.283%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T20:24:47.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:47 smithi018 ceph-mon[179647]: pgmap v335: 99 pgs: 1 activating+undersized+degraded+remapped, 89 active+clean, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 6.0 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 278 op/s; 27864/294678 objects degraded (9.456%); 2743/294678 objects misplaced (0.931%); 9.7 MiB/s, 156 objects/s recovering 2023-09-23T20:24:47.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:47 smithi018 ceph-mon[184144]: pgmap v335: 99 pgs: 1 activating+undersized+degraded+remapped, 89 active+clean, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 6.0 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 278 op/s; 27864/294678 objects degraded (9.456%); 2743/294678 objects misplaced (0.931%); 9.7 MiB/s, 156 objects/s recovering 2023-09-23T20:24:47.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:47 smithi149 ceph-mon[170394]: pgmap v335: 99 pgs: 1 activating+undersized+degraded+remapped, 89 active+clean, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 6.0 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 278 op/s; 27864/294678 objects degraded (9.456%); 2743/294678 objects misplaced (0.931%); 9.7 MiB/s, 156 objects/s recovering 2023-09-23T20:24:48.825 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:24:48.831 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:48.831 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:24:48.932 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:24:49.016 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:49.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:49 smithi018 ceph-mon[179647]: pgmap v336: 99 pgs: 1 activating+undersized+degraded+remapped, 89 active+clean, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 6.0 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 244 op/s; 27864/293127 objects degraded (9.506%); 2743/293127 objects misplaced (0.936%); 7.8 MiB/s, 125 objects/s recovering 2023-09-23T20:24:49.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:49 smithi018 ceph-mon[184144]: pgmap v336: 99 pgs: 1 activating+undersized+degraded+remapped, 89 active+clean, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 6.0 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 244 op/s; 27864/293127 objects degraded (9.506%); 2743/293127 objects misplaced (0.936%); 7.8 MiB/s, 125 objects/s recovering 2023-09-23T20:24:49.469 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:49 smithi149 ceph-mon[170394]: pgmap v336: 99 pgs: 1 activating+undersized+degraded+remapped, 89 active+clean, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 6.0 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 244 op/s; 27864/293127 objects degraded (9.506%); 2743/293127 objects misplaced (0.936%); 7.8 MiB/s, 125 objects/s recovering 2023-09-23T20:24:49.979 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:24:49.985 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:49.986 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:24:50.087 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:24:50.157 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:50.373 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:24:50.386 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:50.386 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:24:50.487 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:24:50.542 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:50.611 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:24:50.614 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:50.615 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:24:50.715 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:24:50.777 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:51.045 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:51 smithi018 ceph-mon[179647]: pgmap v337: 99 pgs: 1 activating+undersized+degraded+remapped, 89 active+clean, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 6.0 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 210 op/s; 27864/293127 objects degraded (9.506%); 2743/293127 objects misplaced (0.936%); 6.7 MiB/s, 107 objects/s recovering 2023-09-23T20:24:51.045 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:51 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 27864/293127 objects degraded (9.506%), 10 pgs degraded (PG_DEGRADED) 2023-09-23T20:24:51.046 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:51 smithi018 ceph-mon[184144]: pgmap v337: 99 pgs: 1 activating+undersized+degraded+remapped, 89 active+clean, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 6.0 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 210 op/s; 27864/293127 objects degraded (9.506%); 2743/293127 objects misplaced (0.936%); 6.7 MiB/s, 107 objects/s recovering 2023-09-23T20:24:51.046 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:51 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 27864/293127 objects degraded (9.506%), 10 pgs degraded (PG_DEGRADED) 2023-09-23T20:24:51.058 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:24:51.062 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:51.062 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:24:51.109 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:24:51.164 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:24:51.199 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:51.252 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:24:51.258 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:51.259 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:24:51.359 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:24:51.401 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:51 smithi149 ceph-mon[170394]: pgmap v337: 99 pgs: 1 activating+undersized+degraded+remapped, 89 active+clean, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 6.0 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 210 op/s; 27864/293127 objects degraded (9.506%); 2743/293127 objects misplaced (0.936%); 6.7 MiB/s, 107 objects/s recovering 2023-09-23T20:24:51.402 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:51 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 27864/293127 objects degraded (9.506%), 10 pgs degraded (PG_DEGRADED) 2023-09-23T20:24:51.408 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:51.479 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:24:51.487 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:51.487 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:24:51.533 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2023-09-23T20:24:51.533 INFO:teuthology.orchestra.run.smithi018.stdout: "size": 20, 2023-09-23T20:24:51.533 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 600, 2023-09-23T20:24:51.534 INFO:teuthology.orchestra.run.smithi018.stdout: "ops": [ 2023-09-23T20:24:51.534 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.534 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.34791.0:39077 5.10 5:0ea335c2:::benchmark_data_smithi018_228451_object39076:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e121)", 2023-09-23T20:24:51.534 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:23:17.046212+0000", 2023-09-23T20:24:51.534 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 94.483117913000001, 2023-09-23T20:24:51.534 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0643534059999999, 2023-09-23T20:24:51.534 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:24:51.534 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:24:51.534 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:24:51.534 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.34791", 2023-09-23T20:24:51.535 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:24:51.535 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 39077 2023-09-23T20:24:51.535 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.535 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:24:51.535 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.535 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:24:51.535 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.046212+0000", 2023-09-23T20:24:51.535 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:24:51.535 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.535 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.535 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:24:51.535 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.046212+0000", 2023-09-23T20:24:51.536 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:24:51.536 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.536 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.536 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:24:51.536 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.046209+0000", 2023-09-23T20:24:51.536 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999967 2023-09-23T20:24:51.536 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.536 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.536 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:24:51.536 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.046257+0000", 2023-09-23T20:24:51.536 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.8041999999999998e-05 2023-09-23T20:24:51.536 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.537 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.537 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:24:51.537 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.046290+0000", 2023-09-23T20:24:51.537 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.2882999999999997e-05 2023-09-23T20:24:51.537 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.537 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.537 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:24:51.537 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.046309+0000", 2023-09-23T20:24:51.537 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.8708000000000001e-05 2023-09-23T20:24:51.537 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.537 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.538 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:24:51.538 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.047312+0000", 2023-09-23T20:24:51.538 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.001003887 2023-09-23T20:24:51.538 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.538 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.538 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:24:51.538 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.047326+0000", 2023-09-23T20:24:51.538 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3237999999999999e-05 2023-09-23T20:24:51.538 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.538 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.538 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:24:51.539 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.105379+0000", 2023-09-23T20:24:51.539 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0580528220000001 2023-09-23T20:24:51.539 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.539 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.539 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:24:51.539 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.105553+0000", 2023-09-23T20:24:51.539 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00017465499999999999 2023-09-23T20:24:51.539 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.539 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.539 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 4,6,7", 2023-09-23T20:24:51.539 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.105672+0000", 2023-09-23T20:24:51.539 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000118594 2023-09-23T20:24:51.539 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.540 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.540 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:24:51.540 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.107999+0000", 2023-09-23T20:24:51.540 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0023273980000000001 2023-09-23T20:24:51.540 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.540 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.540 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:24:51.540 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.108010+0000", 2023-09-23T20:24:51.540 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1039000000000001e-05 2023-09-23T20:24:51.540 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.540 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.540 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:24:51.540 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.109991+0000", 2023-09-23T20:24:51.541 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0019804359999999999 2023-09-23T20:24:51.541 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.541 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.541 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:24:51.541 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.110042+0000", 2023-09-23T20:24:51.541 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.1693999999999997e-05 2023-09-23T20:24:51.541 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.541 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.541 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:24:51.541 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.110117+0000", 2023-09-23T20:24:51.541 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.4426999999999997e-05 2023-09-23T20:24:51.541 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.541 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.542 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:24:51.542 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.110566+0000", 2023-09-23T20:24:51.542 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00044872599999999999 2023-09-23T20:24:51.542 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:24:51.542 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:24:51.542 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:24:51.542 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.542 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.542 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.34791.0:39130 5.10 5:08af1693:::benchmark_data_smithi018_228451_object39129:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e121)", 2023-09-23T20:24:51.542 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:23:17.046331+0000", 2023-09-23T20:24:51.542 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 94.482999442999997, 2023-09-23T20:24:51.543 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0647581580000001, 2023-09-23T20:24:51.543 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:24:51.543 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:24:51.543 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:24:51.543 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.34791", 2023-09-23T20:24:51.543 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:24:51.543 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 39130 2023-09-23T20:24:51.543 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.543 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:24:51.543 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.544 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:24:51.544 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.046331+0000", 2023-09-23T20:24:51.544 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:24:51.544 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.544 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.544 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:24:51.544 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.046331+0000", 2023-09-23T20:24:51.544 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:24:51.544 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.544 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.544 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:24:51.544 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.046330+0000", 2023-09-23T20:24:51.545 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.999999 2023-09-23T20:24:51.545 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.545 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.545 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:24:51.545 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.046363+0000", 2023-09-23T20:24:51.545 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.3501000000000003e-05 2023-09-23T20:24:51.545 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.545 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.545 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:24:51.545 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.046372+0000", 2023-09-23T20:24:51.545 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.8519999999999993e-06 2023-09-23T20:24:51.545 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.545 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.546 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:24:51.546 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.046379+0000", 2023-09-23T20:24:51.546 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.3620000000000002e-06 2023-09-23T20:24:51.546 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.546 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.546 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:24:51.546 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.047351+0000", 2023-09-23T20:24:51.546 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00097203600000000002 2023-09-23T20:24:51.546 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.546 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.546 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:24:51.547 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.047360+0000", 2023-09-23T20:24:51.547 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.0890000000000006e-06 2023-09-23T20:24:51.547 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.547 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.547 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:24:51.547 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.106657+0000", 2023-09-23T20:24:51.547 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.059297632 2023-09-23T20:24:51.547 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.547 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.547 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:24:51.547 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.106758+0000", 2023-09-23T20:24:51.547 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000100306 2023-09-23T20:24:51.547 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.548 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.548 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 4,6,7", 2023-09-23T20:24:51.548 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.106882+0000", 2023-09-23T20:24:51.548 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00012447900000000001 2023-09-23T20:24:51.548 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.548 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.548 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:24:51.548 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.109210+0000", 2023-09-23T20:24:51.548 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0023281460000000001 2023-09-23T20:24:51.548 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.548 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.548 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:24:51.548 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.110617+0000", 2023-09-23T20:24:51.549 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0014068489999999999 2023-09-23T20:24:51.549 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.549 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.549 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:24:51.549 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.110687+0000", 2023-09-23T20:24:51.549 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.9468000000000006e-05 2023-09-23T20:24:51.549 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.549 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.549 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:24:51.549 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.110766+0000", 2023-09-23T20:24:51.549 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.962e-05 2023-09-23T20:24:51.549 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.550 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.550 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:24:51.550 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.110816+0000", 2023-09-23T20:24:51.550 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.003e-05 2023-09-23T20:24:51.550 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.550 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.550 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:24:51.550 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.111089+0000", 2023-09-23T20:24:51.550 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00027253200000000003 2023-09-23T20:24:51.550 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:24:51.550 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:24:51.551 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:24:51.551 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.551 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.551 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.34791.0:39199 5.10 5:09e359ed:::benchmark_data_smithi018_228451_object39198:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e121)", 2023-09-23T20:24:51.551 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:23:17.046384+0000", 2023-09-23T20:24:51.551 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 94.482945927000003, 2023-09-23T20:24:51.551 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.065389938, 2023-09-23T20:24:51.551 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:24:51.551 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:24:51.551 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:24:51.551 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.34791", 2023-09-23T20:24:51.551 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:24:51.551 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 39199 2023-09-23T20:24:51.552 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.552 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:24:51.552 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.552 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:24:51.552 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.046384+0000", 2023-09-23T20:24:51.552 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:24:51.552 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.552 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.552 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:24:51.552 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.046384+0000", 2023-09-23T20:24:51.552 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:24:51.552 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.552 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.553 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:24:51.553 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.046384+0000", 2023-09-23T20:24:51.553 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:24:51.553 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.553 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.553 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:24:51.553 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.046414+0000", 2023-09-23T20:24:51.553 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.0431999999999999e-05 2023-09-23T20:24:51.553 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.553 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.553 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:24:51.553 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.046422+0000", 2023-09-23T20:24:51.554 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.8820000000000001e-06 2023-09-23T20:24:51.554 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.554 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.554 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:24:51.554 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.046428+0000", 2023-09-23T20:24:51.554 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.1460000000000001e-06 2023-09-23T20:24:51.554 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.554 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.554 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:24:51.554 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.047381+0000", 2023-09-23T20:24:51.555 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00095294100000000001 2023-09-23T20:24:51.555 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.555 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.555 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:24:51.555 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.047389+0000", 2023-09-23T20:24:51.555 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.6939999999999995e-06 2023-09-23T20:24:51.555 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.555 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.555 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:24:51.555 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.107338+0000", 2023-09-23T20:24:51.555 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0599495240000001 2023-09-23T20:24:51.555 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.555 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.556 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:24:51.556 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.107438+0000", 2023-09-23T20:24:51.556 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.9877999999999995e-05 2023-09-23T20:24:51.556 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.556 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.556 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 4,6,7", 2023-09-23T20:24:51.556 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.107546+0000", 2023-09-23T20:24:51.556 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000108045 2023-09-23T20:24:51.556 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.556 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.556 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:24:51.557 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.109954+0000", 2023-09-23T20:24:51.557 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0024078950000000002 2023-09-23T20:24:51.557 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.557 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.557 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:24:51.557 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.111161+0000", 2023-09-23T20:24:51.557 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0012072459999999999 2023-09-23T20:24:51.557 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.557 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.557 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:24:51.557 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.111187+0000", 2023-09-23T20:24:51.557 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.5720000000000001e-05 2023-09-23T20:24:51.558 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.558 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.558 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:24:51.558 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.111335+0000", 2023-09-23T20:24:51.558 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00014750899999999999 2023-09-23T20:24:51.558 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.558 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.558 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:24:51.558 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.111390+0000", 2023-09-23T20:24:51.558 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.5009000000000001e-05 2023-09-23T20:24:51.558 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.559 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.559 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:24:51.559 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.111774+0000", 2023-09-23T20:24:51.559 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00038447699999999999 2023-09-23T20:24:51.559 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:24:51.559 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:24:51.559 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:24:51.559 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.559 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.559 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.34791.0:39218 5.10 5:0d3823ea:::benchmark_data_smithi018_228451_object39217:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e121)", 2023-09-23T20:24:51.559 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:23:17.046433+0000", 2023-09-23T20:24:51.559 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 94.482896749000005, 2023-09-23T20:24:51.560 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0660785699999999, 2023-09-23T20:24:51.560 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:24:51.560 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:24:51.560 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:24:51.560 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.34791", 2023-09-23T20:24:51.560 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:24:51.560 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 39218 2023-09-23T20:24:51.560 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.560 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:24:51.560 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.560 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:24:51.560 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.046433+0000", 2023-09-23T20:24:51.561 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:24:51.561 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.561 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.561 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:24:51.561 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.046433+0000", 2023-09-23T20:24:51.561 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:24:51.561 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.561 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.561 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:24:51.561 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.046433+0000", 2023-09-23T20:24:51.562 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:24:51.562 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.562 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.562 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:24:51.562 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.046463+0000", 2023-09-23T20:24:51.562 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.0289000000000001e-05 2023-09-23T20:24:51.562 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.562 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.562 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:24:51.562 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.046471+0000", 2023-09-23T20:24:51.563 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.7610000000000007e-06 2023-09-23T20:24:51.563 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.563 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.563 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:24:51.563 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.046477+0000", 2023-09-23T20:24:51.563 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.9309999999999996e-06 2023-09-23T20:24:51.563 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.563 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.563 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:24:51.563 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.047414+0000", 2023-09-23T20:24:51.564 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00093689999999999995 2023-09-23T20:24:51.564 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.564 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.564 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:24:51.564 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.047421+0000", 2023-09-23T20:24:51.564 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.4120000000000004e-06 2023-09-23T20:24:51.564 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.564 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.564 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:24:51.564 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.108123+0000", 2023-09-23T20:24:51.564 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.060702008 2023-09-23T20:24:51.564 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.565 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.565 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:24:51.565 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.108291+0000", 2023-09-23T20:24:51.565 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00016749700000000001 2023-09-23T20:24:51.565 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.565 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.565 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 4,6,7", 2023-09-23T20:24:51.565 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.108509+0000", 2023-09-23T20:24:51.565 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00021819100000000001 2023-09-23T20:24:51.565 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.565 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.566 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:24:51.566 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.110587+0000", 2023-09-23T20:24:51.566 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.002078388 2023-09-23T20:24:51.566 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.566 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.566 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:24:51.566 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.111233+0000", 2023-09-23T20:24:51.566 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00064565300000000005 2023-09-23T20:24:51.566 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.566 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.566 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:24:51.567 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.111936+0000", 2023-09-23T20:24:51.567 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00070322400000000001 2023-09-23T20:24:51.567 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.567 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.567 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:24:51.567 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.112090+0000", 2023-09-23T20:24:51.567 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000153552 2023-09-23T20:24:51.567 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.567 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.567 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:24:51.567 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.112160+0000", 2023-09-23T20:24:51.567 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.0017000000000004e-05 2023-09-23T20:24:51.568 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.568 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.568 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:24:51.568 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.112512+0000", 2023-09-23T20:24:51.568 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00035219400000000001 2023-09-23T20:24:51.568 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:24:51.568 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:24:51.568 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:24:51.568 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.568 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.568 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.34791.0:39310 5.10 5:09081666:::benchmark_data_smithi018_228451_object39309:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e121)", 2023-09-23T20:24:51.568 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:23:17.046482+0000", 2023-09-23T20:24:51.569 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 94.482847953999993, 2023-09-23T20:24:51.569 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0670518259999999, 2023-09-23T20:24:51.569 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:24:51.569 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:24:51.569 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:24:51.569 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.34791", 2023-09-23T20:24:51.569 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:24:51.569 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 39310 2023-09-23T20:24:51.569 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.569 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:24:51.569 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.569 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:24:51.569 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.046482+0000", 2023-09-23T20:24:51.570 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:24:51.570 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.570 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.570 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:24:51.570 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.046482+0000", 2023-09-23T20:24:51.570 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:24:51.570 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.570 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.570 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:24:51.570 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.046481+0000", 2023-09-23T20:24:51.570 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:24:51.571 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.571 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.571 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:24:51.571 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.046510+0000", 2023-09-23T20:24:51.571 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.8085999999999998e-05 2023-09-23T20:24:51.571 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.572 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.572 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:24:51.572 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.046517+0000", 2023-09-23T20:24:51.572 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.153e-06 2023-09-23T20:24:51.572 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.572 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.572 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:24:51.572 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.046522+0000", 2023-09-23T20:24:51.572 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.4110000000000004e-06 2023-09-23T20:24:51.572 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.572 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.572 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:24:51.573 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.047448+0000", 2023-09-23T20:24:51.573 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00092549400000000001 2023-09-23T20:24:51.573 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.573 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.573 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:24:51.573 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.047455+0000", 2023-09-23T20:24:51.573 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.4270000000000002e-06 2023-09-23T20:24:51.573 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.573 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.573 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:24:51.573 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.109246+0000", 2023-09-23T20:24:51.574 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0617913370000001 2023-09-23T20:24:51.574 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.574 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.574 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:24:51.574 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.109386+0000", 2023-09-23T20:24:51.574 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00014002900000000001 2023-09-23T20:24:51.574 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.574 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.574 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 4,6,7", 2023-09-23T20:24:51.574 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.109501+0000", 2023-09-23T20:24:51.574 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000114571 2023-09-23T20:24:51.575 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.575 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.575 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:24:51.575 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.110701+0000", 2023-09-23T20:24:51.575 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0011995739999999999 2023-09-23T20:24:51.575 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.575 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.575 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:24:51.575 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.111285+0000", 2023-09-23T20:24:51.575 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00058425800000000004 2023-09-23T20:24:51.575 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.575 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.575 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:24:51.576 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.112587+0000", 2023-09-23T20:24:51.576 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.001301838 2023-09-23T20:24:51.576 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.576 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.576 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:24:51.576 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.113122+0000", 2023-09-23T20:24:51.576 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00053523899999999996 2023-09-23T20:24:51.576 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.576 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.576 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:24:51.576 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.113175+0000", 2023-09-23T20:24:51.576 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.3004999999999998e-05 2023-09-23T20:24:51.577 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.577 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.577 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:24:51.577 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.113534+0000", 2023-09-23T20:24:51.577 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00035903299999999998 2023-09-23T20:24:51.577 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:24:51.577 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:24:51.577 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:24:51.577 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.577 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.577 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.34791.0:39313 5.10 5:0a320d93:::benchmark_data_smithi018_228451_object39312:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e121)", 2023-09-23T20:24:51.577 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:23:17.046527+0000", 2023-09-23T20:24:51.577 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 94.482802949000003, 2023-09-23T20:24:51.578 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0775630709999999, 2023-09-23T20:24:51.578 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:24:51.578 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:24:51.578 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:24:51.578 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.34791", 2023-09-23T20:24:51.578 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:24:51.578 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 39313 2023-09-23T20:24:51.578 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.578 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:24:51.578 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.578 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:24:51.578 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.046527+0000", 2023-09-23T20:24:51.579 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:24:51.579 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.579 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.579 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:24:51.579 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.046527+0000", 2023-09-23T20:24:51.579 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:24:51.579 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.579 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.579 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:24:51.579 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.046526+0000", 2023-09-23T20:24:51.579 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.999999 2023-09-23T20:24:51.579 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.579 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.580 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:24:51.580 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.046555+0000", 2023-09-23T20:24:51.580 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.9037e-05 2023-09-23T20:24:51.580 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.580 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.580 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:24:51.580 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.046563+0000", 2023-09-23T20:24:51.580 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.2960000000000001e-06 2023-09-23T20:24:51.580 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.580 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.580 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:24:51.581 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.046568+0000", 2023-09-23T20:24:51.581 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.5489999999999997e-06 2023-09-23T20:24:51.581 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.581 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.581 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:24:51.581 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.047480+0000", 2023-09-23T20:24:51.581 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00091210300000000005 2023-09-23T20:24:51.581 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.581 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.581 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:24:51.581 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.047488+0000", 2023-09-23T20:24:51.581 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.4719999999999998e-06 2023-09-23T20:24:51.582 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.582 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.582 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:24:51.582 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.113570+0000", 2023-09-23T20:24:51.582 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.066082043 2023-09-23T20:24:51.582 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.582 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.582 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:24:51.582 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.113710+0000", 2023-09-23T20:24:51.582 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00014017200000000001 2023-09-23T20:24:51.582 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.583 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.583 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 4,6,7", 2023-09-23T20:24:51.583 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.113824+0000", 2023-09-23T20:24:51.583 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000113897 2023-09-23T20:24:51.583 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.583 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.583 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:24:51.583 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.118618+0000", 2023-09-23T20:24:51.583 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0047939669999999997 2023-09-23T20:24:51.583 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.583 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.583 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:24:51.584 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.118664+0000", 2023-09-23T20:24:51.584 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.6458999999999997e-05 2023-09-23T20:24:51.584 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.584 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.584 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:24:51.584 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.120499+0000", 2023-09-23T20:24:51.584 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0018351190000000001 2023-09-23T20:24:51.584 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.584 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.584 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:24:51.584 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.123510+0000", 2023-09-23T20:24:51.584 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0030101569999999999 2023-09-23T20:24:51.585 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.585 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.585 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:24:51.585 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.123569+0000", 2023-09-23T20:24:51.585 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.9886999999999998e-05 2023-09-23T20:24:51.585 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.585 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.585 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:24:51.585 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.124090+0000", 2023-09-23T20:24:51.585 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00052065399999999995 2023-09-23T20:24:51.586 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:24:51.586 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:24:51.586 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:24:51.586 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.586 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.586 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.34791.0:39369 5.10 5:0a8ed7aa:::benchmark_data_smithi018_228451_object39368:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e121)", 2023-09-23T20:24:51.586 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:23:17.046573+0000", 2023-09-23T20:24:51.586 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 94.482756658, 2023-09-23T20:24:51.586 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.08553827, 2023-09-23T20:24:51.586 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:24:51.586 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:24:51.587 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:24:51.587 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.34791", 2023-09-23T20:24:51.587 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:24:51.587 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 39369 2023-09-23T20:24:51.587 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.587 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:24:51.587 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.587 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:24:51.587 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.046573+0000", 2023-09-23T20:24:51.587 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:24:51.587 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.587 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.588 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:24:51.588 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.046573+0000", 2023-09-23T20:24:51.588 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:24:51.588 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.588 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.588 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:24:51.588 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.046573+0000", 2023-09-23T20:24:51.588 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:24:51.588 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.588 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.588 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:24:51.588 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.046609+0000", 2023-09-23T20:24:51.589 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.5871999999999998e-05 2023-09-23T20:24:51.589 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.589 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.589 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:24:51.589 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.046617+0000", 2023-09-23T20:24:51.589 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.841e-06 2023-09-23T20:24:51.589 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.589 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.589 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:24:51.589 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.046623+0000", 2023-09-23T20:24:51.589 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.0270000000000001e-06 2023-09-23T20:24:51.590 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.590 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.590 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:24:51.590 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.047510+0000", 2023-09-23T20:24:51.590 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00088703199999999995 2023-09-23T20:24:51.590 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.590 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.590 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:24:51.590 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.047519+0000", 2023-09-23T20:24:51.590 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.7710000000000001e-06 2023-09-23T20:24:51.590 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.590 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.591 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:24:51.591 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.114518+0000", 2023-09-23T20:24:51.591 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0669989609999999 2023-09-23T20:24:51.591 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.591 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.591 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:24:51.591 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.114677+0000", 2023-09-23T20:24:51.591 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00015815 2023-09-23T20:24:51.591 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.591 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.591 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 4,6,7", 2023-09-23T20:24:51.591 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.114884+0000", 2023-09-23T20:24:51.592 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000207386 2023-09-23T20:24:51.592 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.592 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.592 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:24:51.592 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.118708+0000", 2023-09-23T20:24:51.592 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.003823608 2023-09-23T20:24:51.592 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.592 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.592 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:24:51.592 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.120506+0000", 2023-09-23T20:24:51.592 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0017987529999999999 2023-09-23T20:24:51.592 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.593 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.593 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:24:51.593 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.124167+0000", 2023-09-23T20:24:51.593 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0036609559999999999 2023-09-23T20:24:51.593 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.593 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.593 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:24:51.593 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.131452+0000", 2023-09-23T20:24:51.593 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00728512 2023-09-23T20:24:51.593 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.593 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.594 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:24:51.594 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.131508+0000", 2023-09-23T20:24:51.594 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.5665999999999999e-05 2023-09-23T20:24:51.594 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.594 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.594 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:24:51.594 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.132112+0000", 2023-09-23T20:24:51.594 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00060357800000000001 2023-09-23T20:24:51.594 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:24:51.594 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:24:51.594 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:24:51.594 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.595 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.595 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.34791.0:39381 5.10 5:0a095a33:::benchmark_data_smithi018_228451_object39380:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e121)", 2023-09-23T20:24:51.595 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:23:17.046628+0000", 2023-09-23T20:24:51.595 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 94.482701985999995, 2023-09-23T20:24:51.595 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0863524579999999, 2023-09-23T20:24:51.595 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:24:51.595 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:24:51.595 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:24:51.595 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.34791", 2023-09-23T20:24:51.595 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:24:51.595 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 39381 2023-09-23T20:24:51.595 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.595 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:24:51.596 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.596 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:24:51.596 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.046628+0000", 2023-09-23T20:24:51.596 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:24:51.596 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.596 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.596 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:24:51.596 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.046628+0000", 2023-09-23T20:24:51.596 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:24:51.596 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.596 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.596 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:24:51.596 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.046628+0000", 2023-09-23T20:24:51.597 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:24:51.597 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.597 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.597 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:24:51.597 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.046680+0000", 2023-09-23T20:24:51.597 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.2605999999999997e-05 2023-09-23T20:24:51.597 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.597 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.597 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:24:51.597 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.046688+0000", 2023-09-23T20:24:51.597 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.6459999999999993e-06 2023-09-23T20:24:51.597 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.597 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.598 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:24:51.598 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.046693+0000", 2023-09-23T20:24:51.598 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.6509999999999998e-06 2023-09-23T20:24:51.598 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.598 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.598 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:24:51.598 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.047543+0000", 2023-09-23T20:24:51.598 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00085002299999999999 2023-09-23T20:24:51.598 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.599 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.599 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:24:51.599 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.047551+0000", 2023-09-23T20:24:51.599 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.1720000000000001e-06 2023-09-23T20:24:51.599 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.599 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.599 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:24:51.599 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.115665+0000", 2023-09-23T20:24:51.599 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0681141380000001 2023-09-23T20:24:51.599 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.600 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.600 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:24:51.600 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.115810+0000", 2023-09-23T20:24:51.600 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000144912 2023-09-23T20:24:51.600 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.600 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.600 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 4,6,7", 2023-09-23T20:24:51.600 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.115962+0000", 2023-09-23T20:24:51.600 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00015245600000000001 2023-09-23T20:24:51.600 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.600 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.600 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:24:51.600 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.120513+0000", 2023-09-23T20:24:51.601 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0045509510000000001 2023-09-23T20:24:51.601 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.601 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.601 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:24:51.601 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.120576+0000", 2023-09-23T20:24:51.601 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.3190999999999999e-05 2023-09-23T20:24:51.601 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.601 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.601 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:24:51.601 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.124249+0000", 2023-09-23T20:24:51.601 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.003672799 2023-09-23T20:24:51.601 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.601 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.602 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:24:51.602 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.132308+0000", 2023-09-23T20:24:51.602 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0080585139999999993 2023-09-23T20:24:51.602 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.602 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.602 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:24:51.602 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.132390+0000", 2023-09-23T20:24:51.602 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.2897000000000003e-05 2023-09-23T20:24:51.602 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.602 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.603 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:24:51.603 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.132980+0000", 2023-09-23T20:24:51.603 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00059002700000000002 2023-09-23T20:24:51.603 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:24:51.603 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:24:51.603 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:24:51.603 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.603 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.603 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.34791.0:39421 5.10 5:0a0f9830:::benchmark_data_smithi018_228451_object39420:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e121)", 2023-09-23T20:24:51.603 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:23:17.046699+0000", 2023-09-23T20:24:51.603 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 94.482631237000007, 2023-09-23T20:24:51.603 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.086892612, 2023-09-23T20:24:51.604 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:24:51.604 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:24:51.604 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:24:51.604 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.34791", 2023-09-23T20:24:51.604 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:24:51.604 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 39421 2023-09-23T20:24:51.604 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.604 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:24:51.604 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.604 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:24:51.604 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.046699+0000", 2023-09-23T20:24:51.605 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:24:51.605 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.605 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.605 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:24:51.605 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.046699+0000", 2023-09-23T20:24:51.605 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:24:51.605 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.605 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.605 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:24:51.605 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.046698+0000", 2023-09-23T20:24:51.605 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:24:51.605 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.605 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.606 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:24:51.606 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.046745+0000", 2023-09-23T20:24:51.606 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.702e-05 2023-09-23T20:24:51.606 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.606 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.606 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:24:51.606 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.046759+0000", 2023-09-23T20:24:51.606 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3199e-05 2023-09-23T20:24:51.606 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.606 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.606 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:24:51.607 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.046765+0000", 2023-09-23T20:24:51.607 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.037e-06 2023-09-23T20:24:51.607 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.607 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.607 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:24:51.607 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.047573+0000", 2023-09-23T20:24:51.607 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00080837699999999999 2023-09-23T20:24:51.607 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.607 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.607 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:24:51.607 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.047580+0000", 2023-09-23T20:24:51.608 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.2420000000000003e-06 2023-09-23T20:24:51.608 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.608 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.608 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:24:51.608 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.116716+0000", 2023-09-23T20:24:51.608 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0691352940000001 2023-09-23T20:24:51.608 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.608 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.608 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:24:51.608 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.116847+0000", 2023-09-23T20:24:51.608 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000131543 2023-09-23T20:24:51.608 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.609 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.609 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 4,6,7", 2023-09-23T20:24:51.609 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.117034+0000", 2023-09-23T20:24:51.609 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00018665399999999999 2023-09-23T20:24:51.609 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.609 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.609 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:24:51.609 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.120518+0000", 2023-09-23T20:24:51.609 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0034838099999999999 2023-09-23T20:24:51.609 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.609 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.609 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:24:51.610 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.124213+0000", 2023-09-23T20:24:51.610 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.003694953 2023-09-23T20:24:51.610 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.610 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.610 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:24:51.610 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.125022+0000", 2023-09-23T20:24:51.610 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00080983599999999998 2023-09-23T20:24:51.610 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.610 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.611 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:24:51.611 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.133115+0000", 2023-09-23T20:24:51.611 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0080930419999999999 2023-09-23T20:24:51.611 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.611 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.611 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:24:51.611 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.133178+0000", 2023-09-23T20:24:51.611 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.2149000000000004e-05 2023-09-23T20:24:51.611 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.611 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.611 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:24:51.611 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.133591+0000", 2023-09-23T20:24:51.612 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00041385700000000001 2023-09-23T20:24:51.612 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:24:51.612 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:24:51.612 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:24:51.612 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.612 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.612 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.34791.0:39462 5.10 5:0d4913db:::benchmark_data_smithi018_228451_object39461:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e121)", 2023-09-23T20:24:51.612 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:23:17.046770+0000", 2023-09-23T20:24:51.612 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 94.482560195000005, 2023-09-23T20:24:51.612 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.087333044, 2023-09-23T20:24:51.612 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:24:51.612 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:24:51.612 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:24:51.613 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.34791", 2023-09-23T20:24:51.613 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:24:51.613 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 39462 2023-09-23T20:24:51.613 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.613 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:24:51.613 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.613 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:24:51.613 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.046770+0000", 2023-09-23T20:24:51.613 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:24:51.613 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.613 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.614 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:24:51.614 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.046770+0000", 2023-09-23T20:24:51.614 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:24:51.614 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.614 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.614 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:24:51.614 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.046769+0000", 2023-09-23T20:24:51.614 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:24:51.614 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.614 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.614 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:24:51.615 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.046824+0000", 2023-09-23T20:24:51.615 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.4347e-05 2023-09-23T20:24:51.615 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.615 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.615 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:24:51.615 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.046832+0000", 2023-09-23T20:24:51.615 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.0120000000000005e-06 2023-09-23T20:24:51.615 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.615 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.615 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:24:51.615 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.046838+0000", 2023-09-23T20:24:51.615 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.3849999999999999e-06 2023-09-23T20:24:51.616 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.616 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.616 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:24:51.616 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.047603+0000", 2023-09-23T20:24:51.616 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00076497499999999999 2023-09-23T20:24:51.616 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.616 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.616 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:24:51.616 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.047610+0000", 2023-09-23T20:24:51.616 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.3100000000000003e-06 2023-09-23T20:24:51.616 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.616 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.617 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:24:51.617 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.117700+0000", 2023-09-23T20:24:51.617 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0700895720000001 2023-09-23T20:24:51.617 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.617 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.617 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:24:51.617 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.117869+0000", 2023-09-23T20:24:51.617 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00016904299999999999 2023-09-23T20:24:51.617 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.617 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.617 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 4,6,7", 2023-09-23T20:24:51.617 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.118015+0000", 2023-09-23T20:24:51.617 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00014622999999999999 2023-09-23T20:24:51.618 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.618 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.618 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:24:51.618 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.121536+0000", 2023-09-23T20:24:51.618 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0035209989999999999 2023-09-23T20:24:51.618 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.618 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.618 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:24:51.618 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.124291+0000", 2023-09-23T20:24:51.618 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.002754547 2023-09-23T20:24:51.618 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.619 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.619 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:24:51.619 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.131406+0000", 2023-09-23T20:24:51.619 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0071151850000000004 2023-09-23T20:24:51.619 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.619 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.619 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:24:51.619 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.133680+0000", 2023-09-23T20:24:51.619 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0022737830000000001 2023-09-23T20:24:51.619 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.619 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.619 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:24:51.620 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.133729+0000", 2023-09-23T20:24:51.620 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.8915000000000002e-05 2023-09-23T20:24:51.620 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.620 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.620 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:24:51.620 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.134103+0000", 2023-09-23T20:24:51.620 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00037416100000000002 2023-09-23T20:24:51.620 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:24:51.620 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:24:51.620 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:24:51.620 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.620 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.621 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.34791.0:39488 5.10 5:09123ba1:::benchmark_data_smithi018_228451_object39487:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e121)", 2023-09-23T20:24:51.621 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:23:17.046843+0000", 2023-09-23T20:24:51.621 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 94.482486898000005, 2023-09-23T20:24:51.621 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0878474629999999, 2023-09-23T20:24:51.621 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:24:51.621 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:24:51.621 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:24:51.621 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.34791", 2023-09-23T20:24:51.621 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:24:51.621 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 39488 2023-09-23T20:24:51.621 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.621 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:24:51.622 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.622 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:24:51.622 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.046843+0000", 2023-09-23T20:24:51.622 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:24:51.622 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.622 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.622 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:24:51.622 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.046843+0000", 2023-09-23T20:24:51.622 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:24:51.622 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.622 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.622 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:24:51.623 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.046843+0000", 2023-09-23T20:24:51.623 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:24:51.623 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.623 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.623 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:24:51.623 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.046890+0000", 2023-09-23T20:24:51.623 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.7080999999999999e-05 2023-09-23T20:24:51.623 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.623 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.623 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:24:51.623 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.046898+0000", 2023-09-23T20:24:51.624 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.943e-06 2023-09-23T20:24:51.624 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.624 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.624 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:24:51.624 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.046903+0000", 2023-09-23T20:24:51.624 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.7400000000000001e-06 2023-09-23T20:24:51.624 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.624 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.624 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:24:51.624 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.047632+0000", 2023-09-23T20:24:51.624 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000728164 2023-09-23T20:24:51.624 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.625 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.625 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:24:51.625 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.047640+0000", 2023-09-23T20:24:51.625 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.6589999999999996e-06 2023-09-23T20:24:51.625 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.625 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.625 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:24:51.625 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.118738+0000", 2023-09-23T20:24:51.625 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.071097956 2023-09-23T20:24:51.625 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.625 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.625 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:24:51.626 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.118936+0000", 2023-09-23T20:24:51.626 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00019784699999999999 2023-09-23T20:24:51.626 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.626 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.626 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 4,6,7", 2023-09-23T20:24:51.626 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.119122+0000", 2023-09-23T20:24:51.626 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00018633000000000001 2023-09-23T20:24:51.626 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.626 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.626 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:24:51.626 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.121545+0000", 2023-09-23T20:24:51.627 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0024223259999999998 2023-09-23T20:24:51.627 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.627 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.627 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:24:51.627 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.125094+0000", 2023-09-23T20:24:51.627 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.003548879 2023-09-23T20:24:51.627 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.627 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.627 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:24:51.627 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.132252+0000", 2023-09-23T20:24:51.627 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0071583940000000002 2023-09-23T20:24:51.627 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.628 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.628 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:24:51.628 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.134238+0000", 2023-09-23T20:24:51.628 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0019857630000000002 2023-09-23T20:24:51.628 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.628 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.628 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:24:51.628 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.134301+0000", 2023-09-23T20:24:51.628 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.3622999999999997e-05 2023-09-23T20:24:51.628 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.628 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.629 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:24:51.629 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.134691+0000", 2023-09-23T20:24:51.629 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00038920300000000002 2023-09-23T20:24:51.629 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:24:51.629 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:24:51.629 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:24:51.629 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.629 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.630 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.34791.0:39536 5.10 5:0d79cf36:::benchmark_data_smithi018_228451_object39535:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e121)", 2023-09-23T20:24:51.630 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:23:17.046908+0000", 2023-09-23T20:24:51.630 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 94.482421707, 2023-09-23T20:24:51.630 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0884368179999999, 2023-09-23T20:24:51.630 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:24:51.630 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:24:51.630 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:24:51.630 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.34791", 2023-09-23T20:24:51.631 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:24:51.631 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 39536 2023-09-23T20:24:51.631 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.631 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:24:51.631 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.631 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:24:51.631 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.046908+0000", 2023-09-23T20:24:51.631 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:24:51.631 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.631 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.632 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:24:51.632 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.046908+0000", 2023-09-23T20:24:51.632 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:24:51.632 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.632 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.632 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:24:51.632 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.046908+0000", 2023-09-23T20:24:51.632 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:24:51.632 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.632 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.632 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:24:51.632 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.046951+0000", 2023-09-23T20:24:51.633 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.2963999999999997e-05 2023-09-23T20:24:51.633 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.633 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.633 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:24:51.633 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.046957+0000", 2023-09-23T20:24:51.633 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.4640000000000003e-06 2023-09-23T20:24:51.633 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.633 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.633 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:24:51.633 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.046963+0000", 2023-09-23T20:24:51.633 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.7039999999999999e-06 2023-09-23T20:24:51.634 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.634 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.634 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:24:51.634 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.047663+0000", 2023-09-23T20:24:51.634 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00069969100000000001 2023-09-23T20:24:51.634 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.634 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.634 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:24:51.634 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.047670+0000", 2023-09-23T20:24:51.634 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.2280000000000001e-06 2023-09-23T20:24:51.634 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.635 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.635 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:24:51.635 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.119624+0000", 2023-09-23T20:24:51.635 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0719541050000001 2023-09-23T20:24:51.635 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.635 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.635 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:24:51.635 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.119775+0000", 2023-09-23T20:24:51.635 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00015126699999999999 2023-09-23T20:24:51.635 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.635 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.635 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 4,6,7", 2023-09-23T20:24:51.636 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.119938+0000", 2023-09-23T20:24:51.636 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00016282699999999999 2023-09-23T20:24:51.636 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.636 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.636 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:24:51.636 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.124112+0000", 2023-09-23T20:24:51.636 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0041736300000000002 2023-09-23T20:24:51.636 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.636 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.636 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:24:51.636 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.125136+0000", 2023-09-23T20:24:51.636 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0010240080000000001 2023-09-23T20:24:51.637 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.637 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.637 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:24:51.637 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.133066+0000", 2023-09-23T20:24:51.637 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0079299660000000001 2023-09-23T20:24:51.637 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.637 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.637 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:24:51.637 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.134881+0000", 2023-09-23T20:24:51.637 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0018154790000000001 2023-09-23T20:24:51.637 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.637 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.638 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:24:51.638 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.134946+0000", 2023-09-23T20:24:51.638 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.4577000000000002e-05 2023-09-23T20:24:51.638 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.638 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.638 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:24:51.638 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.135345+0000", 2023-09-23T20:24:51.638 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00039939199999999999 2023-09-23T20:24:51.638 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:24:51.638 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:24:51.638 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:24:51.638 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.639 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.639 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.34791.0:39555 5.10 5:0878daea:::benchmark_data_smithi018_228451_object39554:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e121)", 2023-09-23T20:24:51.639 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:23:17.046967+0000", 2023-09-23T20:24:51.639 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 94.482362619, 2023-09-23T20:24:51.639 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.088860621, 2023-09-23T20:24:51.639 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:24:51.639 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:24:51.639 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:24:51.639 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.34791", 2023-09-23T20:24:51.639 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:24:51.639 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 39555 2023-09-23T20:24:51.639 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.640 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:24:51.640 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.640 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:24:51.640 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.046967+0000", 2023-09-23T20:24:51.640 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:24:51.640 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.640 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.640 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:24:51.640 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.046967+0000", 2023-09-23T20:24:51.640 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:24:51.640 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.641 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.641 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:24:51.641 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.046967+0000", 2023-09-23T20:24:51.641 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:24:51.641 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.641 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.641 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:24:51.641 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.047010+0000", 2023-09-23T20:24:51.641 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.2651999999999998e-05 2023-09-23T20:24:51.641 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.641 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.641 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:24:51.641 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.047016+0000", 2023-09-23T20:24:51.642 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.4099999999999996e-06 2023-09-23T20:24:51.642 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.642 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.642 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:24:51.642 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.047021+0000", 2023-09-23T20:24:51.642 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.9899999999999997e-06 2023-09-23T20:24:51.642 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.642 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.642 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:24:51.642 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.047691+0000", 2023-09-23T20:24:51.642 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00066963899999999998 2023-09-23T20:24:51.643 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.643 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.643 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:24:51.643 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.047701+0000", 2023-09-23T20:24:51.643 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0121e-05 2023-09-23T20:24:51.643 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.643 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.643 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:24:51.643 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.120613+0000", 2023-09-23T20:24:51.643 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0729125340000001 2023-09-23T20:24:51.643 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.643 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.644 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:24:51.644 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.120771+0000", 2023-09-23T20:24:51.644 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00015753499999999999 2023-09-23T20:24:51.644 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.644 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.644 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 4,6,7", 2023-09-23T20:24:51.644 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.120973+0000", 2023-09-23T20:24:51.644 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00020160999999999999 2023-09-23T20:24:51.644 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.644 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.644 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:24:51.645 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.125038+0000", 2023-09-23T20:24:51.645 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0040653900000000003 2023-09-23T20:24:51.645 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.645 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.645 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:24:51.645 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.125174+0000", 2023-09-23T20:24:51.645 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00013604599999999999 2023-09-23T20:24:51.645 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.645 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.645 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:24:51.645 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.133642+0000", 2023-09-23T20:24:51.645 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0084676119999999994 2023-09-23T20:24:51.646 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.646 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.646 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:24:51.646 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.135423+0000", 2023-09-23T20:24:51.646 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0017813289999999999 2023-09-23T20:24:51.646 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.646 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.646 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:24:51.646 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.135480+0000", 2023-09-23T20:24:51.646 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.7268999999999997e-05 2023-09-23T20:24:51.646 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.647 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.647 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:24:51.647 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.135828+0000", 2023-09-23T20:24:51.647 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00034786600000000001 2023-09-23T20:24:51.647 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:24:51.647 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:24:51.647 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:24:51.647 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.647 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.647 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.34791.0:39663 5.10 5:0f7f4404:::benchmark_data_smithi018_228451_object39662:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e121)", 2023-09-23T20:24:51.647 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:23:17.047026+0000", 2023-09-23T20:24:51.647 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 94.482304380000002, 2023-09-23T20:24:51.648 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0891852280000001, 2023-09-23T20:24:51.648 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:24:51.648 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:24:51.648 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:24:51.648 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.34791", 2023-09-23T20:24:51.648 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:24:51.648 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 39663 2023-09-23T20:24:51.648 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.648 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:24:51.648 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.648 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:24:51.648 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.047026+0000", 2023-09-23T20:24:51.649 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:24:51.649 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.649 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.649 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:24:51.649 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.047026+0000", 2023-09-23T20:24:51.649 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:24:51.649 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.649 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.649 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:24:51.649 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.047025+0000", 2023-09-23T20:24:51.649 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:24:51.650 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.650 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.650 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:24:51.650 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.047068+0000", 2023-09-23T20:24:51.650 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.2718e-05 2023-09-23T20:24:51.650 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.650 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.650 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:24:51.650 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.047075+0000", 2023-09-23T20:24:51.650 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.6819999999999997e-06 2023-09-23T20:24:51.650 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.650 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.651 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:24:51.651 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.047079+0000", 2023-09-23T20:24:51.651 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.6990000000000002e-06 2023-09-23T20:24:51.651 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.651 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.651 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:24:51.651 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.047725+0000", 2023-09-23T20:24:51.651 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00064525100000000001 2023-09-23T20:24:51.651 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.651 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.651 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:24:51.651 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.047733+0000", 2023-09-23T20:24:51.652 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.7460000000000006e-06 2023-09-23T20:24:51.652 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.652 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.652 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:24:51.652 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.121584+0000", 2023-09-23T20:24:51.652 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.073850768 2023-09-23T20:24:51.652 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.652 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.652 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:24:51.652 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.121747+0000", 2023-09-23T20:24:51.652 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000163134 2023-09-23T20:24:51.652 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.653 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.653 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 4,6,7", 2023-09-23T20:24:51.653 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.121954+0000", 2023-09-23T20:24:51.653 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000206877 2023-09-23T20:24:51.653 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.653 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.653 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:24:51.653 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.125045+0000", 2023-09-23T20:24:51.653 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0030906290000000001 2023-09-23T20:24:51.653 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.653 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.653 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:24:51.653 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.131361+0000", 2023-09-23T20:24:51.654 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0063161520000000002 2023-09-23T20:24:51.654 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.654 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.654 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:24:51.654 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.134184+0000", 2023-09-23T20:24:51.654 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.002823285 2023-09-23T20:24:51.654 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.654 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.654 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:24:51.654 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.135867+0000", 2023-09-23T20:24:51.654 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0016828699999999999 2023-09-23T20:24:51.655 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.655 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.655 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:24:51.655 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.135912+0000", 2023-09-23T20:24:51.655 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.4749000000000001e-05 2023-09-23T20:24:51.655 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.655 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.655 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:24:51.655 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.136211+0000", 2023-09-23T20:24:51.655 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00029910599999999999 2023-09-23T20:24:51.655 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:24:51.655 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:24:51.656 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:24:51.656 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.656 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.656 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.34791.0:39688 5.10 5:0cf95098:::benchmark_data_smithi018_228451_object39687:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e121)", 2023-09-23T20:24:51.656 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:23:17.047084+0000", 2023-09-23T20:24:51.656 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 94.482246380000007, 2023-09-23T20:24:51.656 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0895501940000001, 2023-09-23T20:24:51.656 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:24:51.656 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:24:51.656 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:24:51.656 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.34791", 2023-09-23T20:24:51.656 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:24:51.656 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 39688 2023-09-23T20:24:51.657 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.657 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:24:51.657 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.657 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:24:51.657 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.047084+0000", 2023-09-23T20:24:51.657 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:24:51.657 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.657 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.657 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:24:51.657 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.047084+0000", 2023-09-23T20:24:51.657 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:24:51.657 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.658 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.658 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:24:51.658 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.047083+0000", 2023-09-23T20:24:51.658 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:24:51.658 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.658 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.658 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:24:51.658 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.047126+0000", 2023-09-23T20:24:51.658 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.2305999999999998e-05 2023-09-23T20:24:51.658 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.658 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.659 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:24:51.659 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.047133+0000", 2023-09-23T20:24:51.659 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.9870000000000002e-06 2023-09-23T20:24:51.659 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.659 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.659 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:24:51.659 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.047138+0000", 2023-09-23T20:24:51.659 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.1320000000000002e-06 2023-09-23T20:24:51.659 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.659 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.659 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:24:51.659 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.047755+0000", 2023-09-23T20:24:51.659 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00061696099999999996 2023-09-23T20:24:51.660 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.660 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.660 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:24:51.660 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.047765+0000", 2023-09-23T20:24:51.660 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.9469999999999998e-06 2023-09-23T20:24:51.660 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.660 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.660 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:24:51.660 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.122486+0000", 2023-09-23T20:24:51.660 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.074721459 2023-09-23T20:24:51.660 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.660 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.661 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:24:51.661 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.122631+0000", 2023-09-23T20:24:51.661 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00014520599999999999 2023-09-23T20:24:51.661 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.661 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.661 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 4,6,7", 2023-09-23T20:24:51.661 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.122830+0000", 2023-09-23T20:24:51.661 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00019834900000000001 2023-09-23T20:24:51.661 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.661 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.661 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:24:51.661 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.125051+0000", 2023-09-23T20:24:51.662 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0022214639999999998 2023-09-23T20:24:51.662 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.662 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.662 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:24:51.662 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.132184+0000", 2023-09-23T20:24:51.662 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0071326310000000004 2023-09-23T20:24:51.662 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.662 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.663 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:24:51.663 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.134816+0000", 2023-09-23T20:24:51.663 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0026327479999999999 2023-09-23T20:24:51.663 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.663 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.663 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:24:51.663 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.136264+0000", 2023-09-23T20:24:51.663 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.001447537 2023-09-23T20:24:51.663 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.663 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.664 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:24:51.664 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.136302+0000", 2023-09-23T20:24:51.664 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.8455999999999999e-05 2023-09-23T20:24:51.664 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.664 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.664 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:24:51.664 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.136634+0000", 2023-09-23T20:24:51.664 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00033138599999999997 2023-09-23T20:24:51.665 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:24:51.665 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:24:51.665 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:24:51.665 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.665 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.665 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.34791.0:39689 5.10 5:0d4ffb3f:::benchmark_data_smithi018_228451_object39688:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e121)", 2023-09-23T20:24:51.665 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:23:17.047142+0000", 2023-09-23T20:24:51.665 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 94.482187809999999, 2023-09-23T20:24:51.666 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0923403490000001, 2023-09-23T20:24:51.666 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:24:51.666 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:24:51.666 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:24:51.666 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.34791", 2023-09-23T20:24:51.666 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:24:51.666 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 39689 2023-09-23T20:24:51.666 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.666 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:24:51.666 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.666 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:24:51.666 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.047142+0000", 2023-09-23T20:24:51.667 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:24:51.667 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.667 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.667 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:24:51.667 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.047142+0000", 2023-09-23T20:24:51.667 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:24:51.667 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.667 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.667 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:24:51.667 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.047142+0000", 2023-09-23T20:24:51.667 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:24:51.667 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.668 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.668 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:24:51.668 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.047182+0000", 2023-09-23T20:24:51.668 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.0222000000000002e-05 2023-09-23T20:24:51.668 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.668 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.668 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:24:51.668 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.047189+0000", 2023-09-23T20:24:51.668 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.4899999999999997e-06 2023-09-23T20:24:51.668 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.668 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.669 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:24:51.669 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.047194+0000", 2023-09-23T20:24:51.669 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.2190000000000003e-06 2023-09-23T20:24:51.669 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.669 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.669 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:24:51.669 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.047795+0000", 2023-09-23T20:24:51.669 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00060106199999999997 2023-09-23T20:24:51.669 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.669 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.669 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:24:51.670 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:17.047802+0000", 2023-09-23T20:24:51.670 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.9090000000000003e-06 2023-09-23T20:24:51.670 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.670 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.670 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:24:51.670 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.130059+0000", 2023-09-23T20:24:51.670 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.082256876 2023-09-23T20:24:51.670 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.670 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.670 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:24:51.670 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.130201+0000", 2023-09-23T20:24:51.671 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00014283699999999999 2023-09-23T20:24:51.671 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.671 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.671 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 4,6,7", 2023-09-23T20:24:51.671 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.130378+0000", 2023-09-23T20:24:51.671 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000176763 2023-09-23T20:24:51.671 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.671 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.671 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:24:51.671 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.133075+0000", 2023-09-23T20:24:51.671 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0026969989999999998 2023-09-23T20:24:51.671 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.672 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.672 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:24:51.672 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.134764+0000", 2023-09-23T20:24:51.672 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.001688429 2023-09-23T20:24:51.672 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.672 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.672 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:24:51.672 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.135389+0000", 2023-09-23T20:24:51.672 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00062552799999999998 2023-09-23T20:24:51.672 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.672 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.672 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:24:51.673 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.138950+0000", 2023-09-23T20:24:51.673 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0035604009999999999 2023-09-23T20:24:51.673 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.673 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.673 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:24:51.673 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.139012+0000", 2023-09-23T20:24:51.673 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.2891000000000005e-05 2023-09-23T20:24:51.673 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.673 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.673 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:24:51.673 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:18.139483+0000", 2023-09-23T20:24:51.673 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00047011699999999998 2023-09-23T20:24:51.674 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:24:51.674 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:24:51.674 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:24:51.674 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.674 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.674 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.34791.0:50327 5.12 5:4affa94e:::benchmark_data_smithi018_228451_object50326:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e131)", 2023-09-23T20:24:51.674 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:23:31.235046+0000", 2023-09-23T20:24:51.674 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 80.294283999000001, 2023-09-23T20:24:51.675 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.917343723, 2023-09-23T20:24:51.675 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:24:51.675 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:24:51.675 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:24:51.675 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.34791", 2023-09-23T20:24:51.675 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:24:51.675 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 50327 2023-09-23T20:24:51.675 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.676 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:24:51.676 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.676 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:24:51.676 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:31.235046+0000", 2023-09-23T20:24:51.676 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:24:51.676 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.676 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.677 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:24:51.677 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:31.235046+0000", 2023-09-23T20:24:51.677 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:24:51.677 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.677 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.677 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:24:51.677 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:31.235045+0000", 2023-09-23T20:24:51.677 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:24:51.678 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.678 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.678 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:24:51.678 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:31.235072+0000", 2023-09-23T20:24:51.678 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.6423000000000001e-05 2023-09-23T20:24:51.678 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.678 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.679 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:24:51.679 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:31.235080+0000", 2023-09-23T20:24:51.679 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.8350000000000004e-06 2023-09-23T20:24:51.679 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.679 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.679 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:24:51.679 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:31.235086+0000", 2023-09-23T20:24:51.679 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.0979999999999999e-06 2023-09-23T20:24:51.680 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.680 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.680 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:24:51.680 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:31.235465+0000", 2023-09-23T20:24:51.680 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00037880699999999999 2023-09-23T20:24:51.680 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.680 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.680 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:24:51.681 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:31.235475+0000", 2023-09-23T20:24:51.681 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0445000000000001e-05 2023-09-23T20:24:51.681 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.681 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.681 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:24:51.681 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:32.148741+0000", 2023-09-23T20:24:51.681 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.91326574599999999 2023-09-23T20:24:51.681 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.682 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.682 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:24:51.682 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:32.148893+0000", 2023-09-23T20:24:51.682 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00015223299999999999 2023-09-23T20:24:51.682 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.682 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.682 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 3,5", 2023-09-23T20:24:51.683 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:32.149064+0000", 2023-09-23T20:24:51.683 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000171087 2023-09-23T20:24:51.683 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.683 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.683 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:24:51.683 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:32.150447+0000", 2023-09-23T20:24:51.683 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0013826369999999999 2023-09-23T20:24:51.683 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.684 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.684 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:24:51.684 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:32.152124+0000", 2023-09-23T20:24:51.684 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00167761 2023-09-23T20:24:51.684 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.684 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.684 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:24:51.684 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:32.152283+0000", 2023-09-23T20:24:51.685 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00015837800000000001 2023-09-23T20:24:51.685 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.685 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.685 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:24:51.685 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:32.152340+0000", 2023-09-23T20:24:51.685 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.7275999999999998e-05 2023-09-23T20:24:51.686 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.686 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.686 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:24:51.686 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:32.152390+0000", 2023-09-23T20:24:51.686 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.9709999999999997e-05 2023-09-23T20:24:51.686 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:24:51.686 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:24:51.686 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:24:51.687 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.687 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.687 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.34791.0:50340 5.12 5:4bc95138:::benchmark_data_smithi018_228451_object50339:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e131)", 2023-09-23T20:24:51.687 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:23:31.245956+0000", 2023-09-23T20:24:51.687 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 80.283373779000001, 2023-09-23T20:24:51.687 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.90665291000000003, 2023-09-23T20:24:51.687 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:24:51.688 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:24:51.688 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:24:51.688 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.34791", 2023-09-23T20:24:51.688 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:24:51.688 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 50340 2023-09-23T20:24:51.688 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.688 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:24:51.688 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.689 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:24:51.689 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:31.245956+0000", 2023-09-23T20:24:51.689 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:24:51.689 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.689 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.689 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:24:51.689 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:31.245956+0000", 2023-09-23T20:24:51.690 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:24:51.690 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.690 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.690 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:24:51.690 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:31.245955+0000", 2023-09-23T20:24:51.690 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:24:51.690 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.690 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.691 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:24:51.691 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:31.245984+0000", 2023-09-23T20:24:51.691 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.9139e-05 2023-09-23T20:24:51.691 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.691 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.691 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:24:51.691 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:31.246002+0000", 2023-09-23T20:24:51.692 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.8366e-05 2023-09-23T20:24:51.692 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.692 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.692 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:24:51.692 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:31.246011+0000", 2023-09-23T20:24:51.692 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.3929999999999997e-06 2023-09-23T20:24:51.692 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.692 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.693 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:24:51.693 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:31.246146+0000", 2023-09-23T20:24:51.693 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000134141 2023-09-23T20:24:51.693 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.693 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.693 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:24:51.693 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:31.246158+0000", 2023-09-23T20:24:51.693 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2605e-05 2023-09-23T20:24:51.694 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.694 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.694 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:24:51.694 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:32.150475+0000", 2023-09-23T20:24:51.694 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.90431637499999995 2023-09-23T20:24:51.694 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.694 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.694 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:24:51.694 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:32.150607+0000", 2023-09-23T20:24:51.694 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00013200000000000001 2023-09-23T20:24:51.695 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.695 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.695 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 3,5", 2023-09-23T20:24:51.695 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:32.150742+0000", 2023-09-23T20:24:51.695 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000135809 2023-09-23T20:24:51.695 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.695 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.695 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:24:51.695 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:32.152445+0000", 2023-09-23T20:24:51.695 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0017024399999999999 2023-09-23T20:24:51.695 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.695 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.696 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:24:51.696 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:32.152456+0000", 2023-09-23T20:24:51.696 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.147e-05 2023-09-23T20:24:51.696 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.696 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.696 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:24:51.696 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:32.152504+0000", 2023-09-23T20:24:51.696 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.7991999999999997e-05 2023-09-23T20:24:51.696 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.696 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.696 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:24:51.696 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:32.152563+0000", 2023-09-23T20:24:51.696 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.9138000000000002e-05 2023-09-23T20:24:51.697 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.697 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.697 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:24:51.697 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:32.152609+0000", 2023-09-23T20:24:51.697 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.5729000000000002e-05 2023-09-23T20:24:51.697 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:24:51.697 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:24:51.697 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:24:51.697 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.697 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.697 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.34791.0:50361 5.12 5:4e2cf6f1:::benchmark_data_smithi018_228451_object50360:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e131)", 2023-09-23T20:24:51.698 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:23:31.260079+0000", 2023-09-23T20:24:51.698 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 80.269251480999998, 2023-09-23T20:24:51.698 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.89374765700000003, 2023-09-23T20:24:51.698 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:24:51.698 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:24:51.698 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:24:51.698 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.34791", 2023-09-23T20:24:51.698 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:24:51.698 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 50361 2023-09-23T20:24:51.698 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.698 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:24:51.699 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.699 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:24:51.699 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:31.260079+0000", 2023-09-23T20:24:51.699 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:24:51.699 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.699 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.699 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:24:51.699 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:31.260079+0000", 2023-09-23T20:24:51.699 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:24:51.699 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.699 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.699 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:24:51.699 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:31.260077+0000", 2023-09-23T20:24:51.700 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:24:51.700 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.700 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.700 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:24:51.700 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:31.260102+0000", 2023-09-23T20:24:51.700 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.5151e-05 2023-09-23T20:24:51.700 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.700 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.700 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:24:51.700 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:31.260116+0000", 2023-09-23T20:24:51.700 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3854e-05 2023-09-23T20:24:51.700 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.700 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.701 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:24:51.701 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:31.260124+0000", 2023-09-23T20:24:51.701 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.4309999999999994e-06 2023-09-23T20:24:51.701 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.701 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.701 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:24:51.701 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:31.260178+0000", 2023-09-23T20:24:51.701 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.3974999999999997e-05 2023-09-23T20:24:51.702 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.702 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.702 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:24:51.702 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:31.260188+0000", 2023-09-23T20:24:51.702 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.4639999999999995e-06 2023-09-23T20:24:51.702 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.702 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.702 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:24:51.703 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:32.151391+0000", 2023-09-23T20:24:51.703 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.89120316300000002 2023-09-23T20:24:51.703 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.703 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.703 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:24:51.703 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:32.151536+0000", 2023-09-23T20:24:51.703 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00014492899999999999 2023-09-23T20:24:51.703 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.704 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.704 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 3,5", 2023-09-23T20:24:51.704 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:32.151640+0000", 2023-09-23T20:24:51.704 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000104845 2023-09-23T20:24:51.704 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.704 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.704 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:24:51.704 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:32.152463+0000", 2023-09-23T20:24:51.705 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00082301 2023-09-23T20:24:51.705 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.705 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.705 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:24:51.705 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:32.153657+0000", 2023-09-23T20:24:51.705 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.001193288 2023-09-23T20:24:51.705 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.705 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.706 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:24:51.706 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:32.153722+0000", 2023-09-23T20:24:51.706 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.4768000000000001e-05 2023-09-23T20:24:51.706 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.706 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.706 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:24:51.706 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:32.153779+0000", 2023-09-23T20:24:51.707 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.7442999999999999e-05 2023-09-23T20:24:51.707 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.707 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.707 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:24:51.707 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:32.153826+0000", 2023-09-23T20:24:51.707 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.7225999999999998e-05 2023-09-23T20:24:51.707 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:24:51.707 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:24:51.708 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:24:51.708 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.708 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.708 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.34791.0:50363 5.12 5:4b08984f:::benchmark_data_smithi018_228451_object50362:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e131)", 2023-09-23T20:24:51.708 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:23:31.260142+0000", 2023-09-23T20:24:51.708 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 80.269188301, 2023-09-23T20:24:51.708 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.89592204600000003, 2023-09-23T20:24:51.708 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:24:51.709 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:24:51.709 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:24:51.709 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.34791", 2023-09-23T20:24:51.709 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:24:51.709 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 50363 2023-09-23T20:24:51.709 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.709 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:24:51.709 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.710 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:24:51.710 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:31.260142+0000", 2023-09-23T20:24:51.710 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:24:51.710 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.710 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.710 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:24:51.710 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:31.260142+0000", 2023-09-23T20:24:51.710 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:24:51.711 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.711 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.711 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:24:51.711 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:31.260141+0000", 2023-09-23T20:24:51.711 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:24:51.711 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.711 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.711 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:24:51.711 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:31.260160+0000", 2023-09-23T20:24:51.711 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.914e-05 2023-09-23T20:24:51.711 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.711 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.711 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:24:51.712 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:31.260168+0000", 2023-09-23T20:24:51.712 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.3719999999999999e-06 2023-09-23T20:24:51.712 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.712 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.712 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:24:51.712 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:31.260174+0000", 2023-09-23T20:24:51.712 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.7570000000000001e-06 2023-09-23T20:24:51.712 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.712 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.712 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:24:51.712 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:31.260224+0000", 2023-09-23T20:24:51.712 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.0673000000000001e-05 2023-09-23T20:24:51.712 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.713 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.713 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:24:51.713 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:31.260234+0000", 2023-09-23T20:24:51.713 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.7680000000000003e-06 2023-09-23T20:24:51.713 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.713 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.713 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:24:51.713 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:32.152665+0000", 2023-09-23T20:24:51.713 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.89243123700000004 2023-09-23T20:24:51.713 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.713 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.713 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:24:51.713 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:32.152806+0000", 2023-09-23T20:24:51.714 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000140383 2023-09-23T20:24:51.714 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.714 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.714 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 3,5", 2023-09-23T20:24:51.714 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:32.152963+0000", 2023-09-23T20:24:51.714 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000157699 2023-09-23T20:24:51.714 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.714 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.714 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:24:51.714 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:32.154898+0000", 2023-09-23T20:24:51.714 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.001935134 2023-09-23T20:24:51.715 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.715 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.715 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:24:51.715 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:32.155925+0000", 2023-09-23T20:24:51.715 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.001026215 2023-09-23T20:24:51.715 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.715 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.715 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:24:51.715 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:32.155971+0000", 2023-09-23T20:24:51.715 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.6118999999999997e-05 2023-09-23T20:24:51.715 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.715 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.715 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:24:51.715 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:32.156023+0000", 2023-09-23T20:24:51.716 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.2556000000000003e-05 2023-09-23T20:24:51.716 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:24:51.716 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:24:51.716 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:24:51.716 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:32.156064+0000", 2023-09-23T20:24:51.716 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.0463999999999997e-05 2023-09-23T20:24:51.716 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:24:51.716 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:24:51.716 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:24:51.716 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:24:51.716 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:24:51.716 INFO:teuthology.orchestra.run.smithi018.stdout:} 2023-09-23T20:24:51.717 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:24:51.724 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:51 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mon-a[179623]: 2023-09-23T20:24:51.464+0000 7f160fef9700 -1 mon.a@0(leader).osd e159 definitely_dead 0 2023-09-23T20:24:51.787 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:52.054 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:52 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3892442966' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-09-23T20:24:52.054 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:52 smithi018 ceph-mon[179647]: Client client.admin marked osd.2 out, while it was still marked up 2023-09-23T20:24:52.060 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T20:24:52.061 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:24:52.080 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:52.081 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:24:52.097 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T20:24:52.104 INFO:teuthology.orchestra.run.smithi018.stderr:marked out osd.2. 2023-09-23T20:24:52.182 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:24:52.236 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:52.348 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:52 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3892442966' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-09-23T20:24:52.349 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:52 smithi018 ceph-mon[184144]: Client client.admin marked osd.2 out, while it was still marked up 2023-09-23T20:24:52.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:52 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3892442966' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-09-23T20:24:52.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:52 smithi149 ceph-mon[170394]: Client client.admin marked osd.2 out, while it was still marked up 2023-09-23T20:24:52.632 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:24:52.635 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:52.636 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:24:52.737 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:24:52.764 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:52.800 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:24:52.803 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:52.803 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:24:52.904 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:24:52.956 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:53.001 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:24:53.005 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:53.005 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:24:53.081 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:53 smithi149 ceph-mon[170394]: pgmap v338: 99 pgs: 89 active+clean, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped; 5.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 439 op/s; 27861/282279 objects degraded (9.870%); 2101/282279 objects misplaced (0.744%); 7.0 MiB/s, 112 objects/s recovering 2023-09-23T20:24:53.081 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:53 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3892442966' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["2"]}]': finished 2023-09-23T20:24:53.106 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:24:53.124 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:24:53.140 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:53.227 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:24:53.247 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:24:53.251 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:24:53.254 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:53 smithi018 ceph-mon[184144]: pgmap v338: 99 pgs: 89 active+clean, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped; 5.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 439 op/s; 27861/282279 objects degraded (9.870%); 2101/282279 objects misplaced (0.744%); 7.0 MiB/s, 112 objects/s recovering 2023-09-23T20:24:53.254 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:53 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3892442966' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["2"]}]': finished 2023-09-23T20:24:53.254 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:53 smithi018 ceph-mon[184144]: osdmap e160: 8 total, 8 up, 6 in 2023-09-23T20:24:53.254 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:53 smithi018 ceph-mon[184144]: failed to encode map e160 with expected crc 2023-09-23T20:24:53.254 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:53 smithi018 ceph-mon[184144]: failed to encode map e160 with expected crc 2023-09-23T20:24:53.254 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:53 smithi018 ceph-mon[179647]: pgmap v338: 99 pgs: 89 active+clean, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped; 5.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 439 op/s; 27861/282279 objects degraded (9.870%); 2101/282279 objects misplaced (0.744%); 7.0 MiB/s, 112 objects/s recovering 2023-09-23T20:24:53.254 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:53 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3892442966' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["2"]}]': finished 2023-09-23T20:24:53.255 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:53 smithi018 ceph-mon[179647]: osdmap e160: 8 total, 8 up, 6 in 2023-09-23T20:24:53.255 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:53 smithi018 ceph-mon[179647]: failed to encode map e160 with expected crc 2023-09-23T20:24:53.255 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:53 smithi018 ceph-mon[179647]: failed to encode map e160 with expected crc 2023-09-23T20:24:53.255 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:24:53.255 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:53.256 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:24:53.268 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:24:53.291 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:24:53.338 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:24:53.339 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:24:53.354 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:24:53.357 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:24:53.374 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:24:53.376 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:24:53.376 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:24:53.376 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:24:53.376 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:24:53.376 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:24:53.377 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:24:53.377 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:24:53.377 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:24:53.388 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:53.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:53 smithi149 ceph-mon[170394]: osdmap e160: 8 total, 8 up, 6 in 2023-09-23T20:24:53.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:53 smithi149 ceph-mon[170394]: failed to encode map e160 with expected crc 2023-09-23T20:24:53.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:53 smithi149 ceph-mon[170394]: failed to encode map e160 with expected crc 2023-09-23T20:24:53.576 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:24:53.578 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:53.579 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:24:53.679 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:24:53.707 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:53.743 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:24:53.747 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:53.747 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:24:53.848 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:24:53.877 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:54.091 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:54 smithi018 ceph-mon[179647]: failed to encode map e161 with expected crc 2023-09-23T20:24:54.091 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:54 smithi018 ceph-mon[179647]: osdmap e161: 8 total, 8 up, 6 in 2023-09-23T20:24:54.091 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:54 smithi018 ceph-mon[179647]: failed to encode map e161 with expected crc 2023-09-23T20:24:54.091 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:54 smithi018 ceph-mon[179647]: failed to encode map e161 with expected crc 2023-09-23T20:24:54.091 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:54 smithi018 ceph-mon[179647]: failed to encode map e161 with expected crc 2023-09-23T20:24:54.091 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:54 smithi018 ceph-mon[179647]: failed to encode map e161 with expected crc 2023-09-23T20:24:54.091 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:54 smithi018 ceph-mon[179647]: failed to encode map e161 with expected crc 2023-09-23T20:24:54.091 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:54 smithi018 ceph-mon[179647]: failed to encode map e161 with expected crc 2023-09-23T20:24:54.091 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:54 smithi018 ceph-mon[179647]: failed to encode map e161 with expected crc 2023-09-23T20:24:54.091 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:54 smithi018 ceph-mon[179647]: failed to encode map e161 with expected crc 2023-09-23T20:24:54.092 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:54 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2855010210' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:24:54.092 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:54 smithi018 ceph-mon[179647]: osdmap e162: 8 total, 8 up, 6 in 2023-09-23T20:24:54.092 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:54 smithi018 ceph-mon[184144]: failed to encode map e161 with expected crc 2023-09-23T20:24:54.092 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:54 smithi018 ceph-mon[184144]: osdmap e161: 8 total, 8 up, 6 in 2023-09-23T20:24:54.092 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:54 smithi018 ceph-mon[184144]: failed to encode map e161 with expected crc 2023-09-23T20:24:54.092 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:54 smithi018 ceph-mon[184144]: failed to encode map e161 with expected crc 2023-09-23T20:24:54.092 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:54 smithi018 ceph-mon[184144]: failed to encode map e161 with expected crc 2023-09-23T20:24:54.092 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:54 smithi018 ceph-mon[184144]: failed to encode map e161 with expected crc 2023-09-23T20:24:54.092 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:54 smithi018 ceph-mon[184144]: failed to encode map e161 with expected crc 2023-09-23T20:24:54.093 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:54 smithi018 ceph-mon[184144]: failed to encode map e161 with expected crc 2023-09-23T20:24:54.093 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:54 smithi018 ceph-mon[184144]: failed to encode map e161 with expected crc 2023-09-23T20:24:54.093 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:54 smithi018 ceph-mon[184144]: failed to encode map e161 with expected crc 2023-09-23T20:24:54.093 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:54 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2855010210' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:24:54.093 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:54 smithi018 ceph-mon[184144]: osdmap e162: 8 total, 8 up, 6 in 2023-09-23T20:24:54.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:54 smithi149 ceph-mon[170394]: failed to encode map e161 with expected crc 2023-09-23T20:24:54.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:54 smithi149 ceph-mon[170394]: osdmap e161: 8 total, 8 up, 6 in 2023-09-23T20:24:54.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:54 smithi149 ceph-mon[170394]: failed to encode map e161 with expected crc 2023-09-23T20:24:54.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:54 smithi149 ceph-mon[170394]: failed to encode map e161 with expected crc 2023-09-23T20:24:54.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:54 smithi149 ceph-mon[170394]: failed to encode map e161 with expected crc 2023-09-23T20:24:54.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:54 smithi149 ceph-mon[170394]: failed to encode map e161 with expected crc 2023-09-23T20:24:54.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:54 smithi149 ceph-mon[170394]: failed to encode map e161 with expected crc 2023-09-23T20:24:54.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:54 smithi149 ceph-mon[170394]: failed to encode map e161 with expected crc 2023-09-23T20:24:54.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:54 smithi149 ceph-mon[170394]: failed to encode map e161 with expected crc 2023-09-23T20:24:54.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:54 smithi149 ceph-mon[170394]: failed to encode map e161 with expected crc 2023-09-23T20:24:54.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:54 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2855010210' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:24:54.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:54 smithi149 ceph-mon[170394]: osdmap e162: 8 total, 8 up, 6 in 2023-09-23T20:24:54.476 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:24:54.483 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:54.483 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:24:54.584 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:24:54.657 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:54.737 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:24:54.749 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:54.750 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:24:54.850 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:24:54.937 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:55.038 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:24:55.049 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:55.050 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:24:55.141 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:24:54 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:24:54] "GET /metrics HTTP/1.1" 200 36499 "" "Prometheus/2.43.0" 2023-09-23T20:24:55.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:55 smithi018 ceph-mon[184144]: pgmap v341: 99 pgs: 89 active+clean, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped; 5.7 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 666 op/s; 27861/278667 objects degraded (9.998%); 2101/278667 objects misplaced (0.754%); 2.5 MiB/s, 40 objects/s recovering 2023-09-23T20:24:55.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:55 smithi018 ceph-mon[184144]: failed to encode map e162 with expected crc 2023-09-23T20:24:55.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:55 smithi018 ceph-mon[184144]: failed to encode map e162 with expected crc 2023-09-23T20:24:55.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:55 smithi018 ceph-mon[184144]: failed to encode map e162 with expected crc 2023-09-23T20:24:55.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:55 smithi018 ceph-mon[184144]: failed to encode map e162 with expected crc 2023-09-23T20:24:55.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:55 smithi018 ceph-mon[184144]: failed to encode map e162 with expected crc 2023-09-23T20:24:55.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:55 smithi018 ceph-mon[184144]: failed to encode map e162 with expected crc 2023-09-23T20:24:55.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:55 smithi018 ceph-mon[184144]: failed to encode map e162 with expected crc 2023-09-23T20:24:55.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:55 smithi018 ceph-mon[179647]: pgmap v341: 99 pgs: 89 active+clean, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped; 5.7 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 666 op/s; 27861/278667 objects degraded (9.998%); 2101/278667 objects misplaced (0.754%); 2.5 MiB/s, 40 objects/s recovering 2023-09-23T20:24:55.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:55 smithi018 ceph-mon[179647]: failed to encode map e162 with expected crc 2023-09-23T20:24:55.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:55 smithi018 ceph-mon[179647]: failed to encode map e162 with expected crc 2023-09-23T20:24:55.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:55 smithi018 ceph-mon[179647]: failed to encode map e162 with expected crc 2023-09-23T20:24:55.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:55 smithi018 ceph-mon[179647]: failed to encode map e162 with expected crc 2023-09-23T20:24:55.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:55 smithi018 ceph-mon[179647]: failed to encode map e162 with expected crc 2023-09-23T20:24:55.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:55 smithi018 ceph-mon[179647]: failed to encode map e162 with expected crc 2023-09-23T20:24:55.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:55 smithi018 ceph-mon[179647]: failed to encode map e162 with expected crc 2023-09-23T20:24:55.150 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:24:55.201 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:55.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:55 smithi149 ceph-mon[170394]: pgmap v341: 99 pgs: 89 active+clean, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped; 5.7 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 666 op/s; 27861/278667 objects degraded (9.998%); 2101/278667 objects misplaced (0.754%); 2.5 MiB/s, 40 objects/s recovering 2023-09-23T20:24:55.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:55 smithi149 ceph-mon[170394]: failed to encode map e162 with expected crc 2023-09-23T20:24:55.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:55 smithi149 ceph-mon[170394]: failed to encode map e162 with expected crc 2023-09-23T20:24:55.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:55 smithi149 ceph-mon[170394]: failed to encode map e162 with expected crc 2023-09-23T20:24:55.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:55 smithi149 ceph-mon[170394]: failed to encode map e162 with expected crc 2023-09-23T20:24:55.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:55 smithi149 ceph-mon[170394]: failed to encode map e162 with expected crc 2023-09-23T20:24:55.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:55 smithi149 ceph-mon[170394]: failed to encode map e162 with expected crc 2023-09-23T20:24:55.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:55 smithi149 ceph-mon[170394]: failed to encode map e162 with expected crc 2023-09-23T20:24:55.275 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:24:55.281 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:55.282 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:24:55.382 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:24:55.458 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:55.543 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:24:55.551 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:55.551 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:24:55.553 INFO:teuthology.orchestra.run.smithi018.stdout:ERROR: (22) Invalid argument 2023-09-23T20:24:55.553 INFO:teuthology.orchestra.run.smithi018.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:24:55.652 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:24:55.771 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:55.850 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:24:55.858 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:55.859 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:24:55.960 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:24:55.963 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:24:56.015 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T20:24:56.018 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:56.073 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:24:56.077 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:56.078 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:24:56.141 INFO:teuthology.orchestra.run.smithi018.stderr:nodeep-scrub is unset 2023-09-23T20:24:56.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:56 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2479392027' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:24:56.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:56 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:24:56.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:56 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 27861/278667 objects degraded (9.998%), 10 pgs degraded (PG_DEGRADED) 2023-09-23T20:24:56.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:56 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2479392027' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:24:56.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:56 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:24:56.177 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:56 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 27861/278667 objects degraded (9.998%), 10 pgs degraded (PG_DEGRADED) 2023-09-23T20:24:56.179 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:24:56.233 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:56.295 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:24:56.301 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:56.301 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:24:56.402 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:24:56.438 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:56.469 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:56 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2479392027' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:24:56.469 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:56 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:24:56.469 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:56 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 27861/278667 objects degraded (9.998%), 10 pgs degraded (PG_DEGRADED) 2023-09-23T20:24:56.505 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:24:56.508 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:56.508 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:24:56.563 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:24:56.594 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:24:56.609 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:24:56.670 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:56.929 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:24:56.932 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:56.932 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:24:57.034 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:24:57.063 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:57.168 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:24:57.177 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:57.177 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:24:57.236 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:57 smithi018 ceph-mon[184144]: pgmap v343: 99 pgs: 1 active+recovering+undersized+remapped, 82 active+clean, 1 active+recovering+undersized+degraded+remapped, 15 active+recovery_wait+undersized+degraded+remapped; 5.6 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 1.10k op/s; 57910/273306 objects degraded (21.189%); 3691/273306 objects misplaced (1.351%); 17 MiB/s, 1 keys/s, 271 objects/s recovering 2023-09-23T20:24:57.237 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:57 smithi018 ceph-mon[184144]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:24:57.237 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:57 smithi018 ceph-mon[184144]: failed to encode map e163 with expected crc 2023-09-23T20:24:57.237 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:57 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:24:57.237 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:57 smithi018 ceph-mon[184144]: osdmap e163: 8 total, 8 up, 6 in 2023-09-23T20:24:57.237 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:57 smithi018 ceph-mon[184144]: failed to encode map e163 with expected crc 2023-09-23T20:24:57.237 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:57 smithi018 ceph-mon[184144]: failed to encode map e163 with expected crc 2023-09-23T20:24:57.237 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:57 smithi018 ceph-mon[179647]: pgmap v343: 99 pgs: 1 active+recovering+undersized+remapped, 82 active+clean, 1 active+recovering+undersized+degraded+remapped, 15 active+recovery_wait+undersized+degraded+remapped; 5.6 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 1.10k op/s; 57910/273306 objects degraded (21.189%); 3691/273306 objects misplaced (1.351%); 17 MiB/s, 1 keys/s, 271 objects/s recovering 2023-09-23T20:24:57.238 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:57 smithi018 ceph-mon[179647]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:24:57.238 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:57 smithi018 ceph-mon[179647]: failed to encode map e163 with expected crc 2023-09-23T20:24:57.238 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:57 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:24:57.238 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:57 smithi018 ceph-mon[179647]: osdmap e163: 8 total, 8 up, 6 in 2023-09-23T20:24:57.238 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:57 smithi018 ceph-mon[179647]: failed to encode map e163 with expected crc 2023-09-23T20:24:57.238 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:57 smithi018 ceph-mon[179647]: failed to encode map e163 with expected crc 2023-09-23T20:24:57.279 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:24:57.340 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:57.468 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:57 smithi149 ceph-mon[170394]: pgmap v343: 99 pgs: 1 active+recovering+undersized+remapped, 82 active+clean, 1 active+recovering+undersized+degraded+remapped, 15 active+recovery_wait+undersized+degraded+remapped; 5.6 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 1.10k op/s; 57910/273306 objects degraded (21.189%); 3691/273306 objects misplaced (1.351%); 17 MiB/s, 1 keys/s, 271 objects/s recovering 2023-09-23T20:24:57.468 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:57 smithi149 ceph-mon[170394]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:24:57.468 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:57 smithi149 ceph-mon[170394]: failed to encode map e163 with expected crc 2023-09-23T20:24:57.468 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:57 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:24:57.468 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:57 smithi149 ceph-mon[170394]: osdmap e163: 8 total, 8 up, 6 in 2023-09-23T20:24:57.468 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:57 smithi149 ceph-mon[170394]: failed to encode map e163 with expected crc 2023-09-23T20:24:57.468 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:57 smithi149 ceph-mon[170394]: failed to encode map e163 with expected crc 2023-09-23T20:24:57.579 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:24:57.587 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:57.587 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:24:57.688 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:24:57.710 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 4, 5, 6, 7] out_osds: [3, 2] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-23T20:24:57.710 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:24:57.710 INFO:tasks.thrashosds.thrasher:Adding osd 3 2023-09-23T20:24:57.711 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd in 3 2023-09-23T20:24:57.786 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:57.873 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:24:57.886 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:57.886 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:24:57.987 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:24:58.041 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:58.280 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:58 smithi018 ceph-mon[179647]: failed to encode map e164 with expected crc 2023-09-23T20:24:58.280 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:58 smithi018 ceph-mon[179647]: osdmap e164: 8 total, 8 up, 6 in 2023-09-23T20:24:58.280 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:58 smithi018 ceph-mon[179647]: failed to encode map e164 with expected crc 2023-09-23T20:24:58.280 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:58 smithi018 ceph-mon[179647]: failed to encode map e164 with expected crc 2023-09-23T20:24:58.280 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:58 smithi018 ceph-mon[184144]: failed to encode map e164 with expected crc 2023-09-23T20:24:58.280 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:58 smithi018 ceph-mon[184144]: osdmap e164: 8 total, 8 up, 6 in 2023-09-23T20:24:58.280 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:58 smithi018 ceph-mon[184144]: failed to encode map e164 with expected crc 2023-09-23T20:24:58.280 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:58 smithi018 ceph-mon[184144]: failed to encode map e164 with expected crc 2023-09-23T20:24:58.288 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:24:58.290 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:58.290 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:24:58.390 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:24:58.445 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:58 smithi149 ceph-mon[170394]: failed to encode map e164 with expected crc 2023-09-23T20:24:58.446 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:58 smithi149 ceph-mon[170394]: osdmap e164: 8 total, 8 up, 6 in 2023-09-23T20:24:58.446 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:58 smithi149 ceph-mon[170394]: failed to encode map e164 with expected crc 2023-09-23T20:24:58.446 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:58 smithi149 ceph-mon[170394]: failed to encode map e164 with expected crc 2023-09-23T20:24:58.463 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:58.509 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:24:58.538 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:24:58.555 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:24:58.558 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:58.559 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:24:58.573 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:24:58.601 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:24:58.639 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:24:58.659 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:24:58.679 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:24:58.717 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:58.728 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:24:58.777 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:24:58.782 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:24:58.782 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:24:58.782 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:24:58.783 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:24:58.783 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:24:58.783 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:24:58.783 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:24:58.783 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:24:58.783 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:24:58.788 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:58.788 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:24:58.839 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:24:58.889 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:24:58.936 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2023-09-23T20:24:58.936 INFO:teuthology.orchestra.run.smithi018.stdout: "ops": [], 2023-09-23T20:24:58.936 INFO:teuthology.orchestra.run.smithi018.stdout: "complaint_time": 30, 2023-09-23T20:24:58.936 INFO:teuthology.orchestra.run.smithi018.stdout: "num_blocked_ops": 0 2023-09-23T20:24:58.936 INFO:teuthology.orchestra.run.smithi018.stdout:} 2023-09-23T20:24:58.957 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:58.978 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T20:24:59.222 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:59 smithi018 ceph-mon[184144]: pgmap v346: 99 pgs: 1 active+recovering+undersized+remapped, 82 active+clean, 1 active+recovering+undersized+degraded+remapped, 15 active+recovery_wait+undersized+degraded+remapped; 5.6 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 403 op/s; 57910/273306 objects degraded (21.189%); 3691/273306 objects misplaced (1.351%); 18 MiB/s, 2 keys/s, 295 objects/s recovering 2023-09-23T20:24:59.222 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:24:59 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/49343701' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:24:59.222 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:59 smithi018 ceph-mon[179647]: pgmap v346: 99 pgs: 1 active+recovering+undersized+remapped, 82 active+clean, 1 active+recovering+undersized+degraded+remapped, 15 active+recovery_wait+undersized+degraded+remapped; 5.6 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 403 op/s; 57910/273306 objects degraded (21.189%); 3691/273306 objects misplaced (1.351%); 18 MiB/s, 2 keys/s, 295 objects/s recovering 2023-09-23T20:24:59.222 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:24:59 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/49343701' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:24:59.229 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:24:59.237 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:59.238 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:24:59.339 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:24:59.377 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:59.470 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:59 smithi149 ceph-mon[170394]: pgmap v346: 99 pgs: 1 active+recovering+undersized+remapped, 82 active+clean, 1 active+recovering+undersized+degraded+remapped, 15 active+recovery_wait+undersized+degraded+remapped; 5.6 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 403 op/s; 57910/273306 objects degraded (21.189%); 3691/273306 objects misplaced (1.351%); 18 MiB/s, 2 keys/s, 295 objects/s recovering 2023-09-23T20:24:59.470 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:24:59 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/49343701' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:24:59.583 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:24:59.586 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:59.587 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:24:59.665 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T20:24:59.687 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:24:59.778 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:24:59.871 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:24:59.884 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:24:59.884 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:24:59.985 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:25:00.102 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:00.175 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:25:00.185 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:00.185 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:25:00.286 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:25:00.345 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:00.347 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:25:00.440 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:00 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:25:00.440 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:00 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:25:00.440 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:00 smithi149 ceph-mon[170394]: pgmap v347: 99 pgs: 1 active+recovering+undersized+remapped, 82 active+clean, 1 active+recovering+undersized+degraded+remapped, 15 active+recovery_wait+undersized+degraded+remapped; 5.6 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 297 op/s; 57910/273306 objects degraded (21.189%); 3691/273306 objects misplaced (1.351%); 14 MiB/s, 1 keys/s, 217 objects/s recovering 2023-09-23T20:25:00.445 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:25:00.455 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:00.455 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:25:00.556 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:25:00.633 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:00.694 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:00 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:25:00.694 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:00 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:25:00.695 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:00 smithi018 ceph-mon[184144]: pgmap v347: 99 pgs: 1 active+recovering+undersized+remapped, 82 active+clean, 1 active+recovering+undersized+degraded+remapped, 15 active+recovery_wait+undersized+degraded+remapped; 5.6 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 297 op/s; 57910/273306 objects degraded (21.189%); 3691/273306 objects misplaced (1.351%); 14 MiB/s, 1 keys/s, 217 objects/s recovering 2023-09-23T20:25:00.695 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:00 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:25:00.695 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:00 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:25:00.695 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:00 smithi018 ceph-mon[179647]: pgmap v347: 99 pgs: 1 active+recovering+undersized+remapped, 82 active+clean, 1 active+recovering+undersized+degraded+remapped, 15 active+recovery_wait+undersized+degraded+remapped; 5.6 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 297 op/s; 57910/273306 objects degraded (21.189%); 3691/273306 objects misplaced (1.351%); 14 MiB/s, 1 keys/s, 217 objects/s recovering 2023-09-23T20:25:00.738 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:25:00.747 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:00.748 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:25:00.848 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:25:00.906 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:25:00.907 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:00.973 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:25:00.979 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:00.980 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:25:01.080 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:25:01.176 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:01.243 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:25:01.251 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:01.251 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:25:01.269 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:25:01.318 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:01 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mon-a[179623]: 2023-09-23T20:25:01.051+0000 7f160fef9700 -1 mon.a@0(leader).osd e164 definitely_dead 0 2023-09-23T20:25:01.352 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:25:01.411 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:01.452 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:01 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 57910/273306 objects degraded (21.189%), 16 pgs degraded (PG_DEGRADED) 2023-09-23T20:25:01.452 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:01 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3452599321' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["3"]}]: dispatch 2023-09-23T20:25:01.509 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:25:01.516 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:01.517 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:25:01.589 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:01 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 57910/273306 objects degraded (21.189%), 16 pgs degraded (PG_DEGRADED) 2023-09-23T20:25:01.589 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:01 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3452599321' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["3"]}]: dispatch 2023-09-23T20:25:01.589 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:01 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 57910/273306 objects degraded (21.189%), 16 pgs degraded (PG_DEGRADED) 2023-09-23T20:25:01.590 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:01 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3452599321' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["3"]}]: dispatch 2023-09-23T20:25:01.618 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:25:01.673 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:01.931 INFO:teuthology.orchestra.run.smithi018.stderr:marked in osd.3. 2023-09-23T20:25:02.377 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:25:02.380 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:02.380 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:25:02.481 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:25:02.509 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:02.548 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:25:02.551 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:02.551 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:25:02.652 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:25:02.681 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:02.931 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:02 smithi018 ceph-mon[179647]: pgmap v348: 99 pgs: 83 active+clean, 2 active+recovering+undersized+degraded+remapped, 14 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 626 op/s; 54319/264711 objects degraded (20.520%); 5837/264711 objects misplaced (2.205%); 18 MiB/s, 1 keys/s, 295 objects/s recovering 2023-09-23T20:25:02.931 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:02 smithi018 ceph-mon[179647]: failed to encode map e165 with expected crc 2023-09-23T20:25:02.931 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:02 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3452599321' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["3"]}]': finished 2023-09-23T20:25:02.931 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:02 smithi018 ceph-mon[179647]: osdmap e165: 8 total, 8 up, 7 in 2023-09-23T20:25:02.931 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:02 smithi018 ceph-mon[179647]: failed to encode map e165 with expected crc 2023-09-23T20:25:02.932 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:02 smithi018 ceph-mon[184144]: pgmap v348: 99 pgs: 83 active+clean, 2 active+recovering+undersized+degraded+remapped, 14 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 626 op/s; 54319/264711 objects degraded (20.520%); 5837/264711 objects misplaced (2.205%); 18 MiB/s, 1 keys/s, 295 objects/s recovering 2023-09-23T20:25:02.932 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:02 smithi018 ceph-mon[184144]: failed to encode map e165 with expected crc 2023-09-23T20:25:02.932 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:02 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3452599321' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["3"]}]': finished 2023-09-23T20:25:02.932 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:02 smithi018 ceph-mon[184144]: osdmap e165: 8 total, 8 up, 7 in 2023-09-23T20:25:02.932 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:02 smithi018 ceph-mon[184144]: failed to encode map e165 with expected crc 2023-09-23T20:25:02.933 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:02 smithi149 ceph-mon[170394]: pgmap v348: 99 pgs: 83 active+clean, 2 active+recovering+undersized+degraded+remapped, 14 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 626 op/s; 54319/264711 objects degraded (20.520%); 5837/264711 objects misplaced (2.205%); 18 MiB/s, 1 keys/s, 295 objects/s recovering 2023-09-23T20:25:02.933 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:02 smithi149 ceph-mon[170394]: failed to encode map e165 with expected crc 2023-09-23T20:25:02.933 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:02 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3452599321' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["3"]}]': finished 2023-09-23T20:25:02.933 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:02 smithi149 ceph-mon[170394]: osdmap e165: 8 total, 8 up, 7 in 2023-09-23T20:25:02.933 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:02 smithi149 ceph-mon[170394]: failed to encode map e165 with expected crc 2023-09-23T20:25:03.355 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:25:03.358 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:03.358 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:25:03.459 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:25:03.489 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:03.530 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:25:03.532 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:03.533 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:25:03.620 INFO:tasks.thrashosds.thrasher:Added osd 3 2023-09-23T20:25:03.634 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:25:03.665 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:03.705 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:25:03.708 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:03.709 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:25:03.809 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:25:03.837 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:03.874 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:25:03.876 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:03.877 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:25:03.920 INFO:teuthology.orchestra.run.smithi018.stderr:noscrub is set 2023-09-23T20:25:03.977 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:25:04.015 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:25:04.031 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:04.065 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2023-09-23T20:25:04.065 INFO:teuthology.orchestra.run.smithi018.stdout: "size": 20, 2023-09-23T20:25:04.065 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 600, 2023-09-23T20:25:04.065 INFO:teuthology.orchestra.run.smithi018.stdout: "ops": [ 2023-09-23T20:25:04.066 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.066 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.34791.0:28891 5.8 5:116f3062:::benchmark_data_smithi018_228451_object28890:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e118)", 2023-09-23T20:25:04.066 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:23:09.968465+0000", 2023-09-23T20:25:04.066 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 114.093647299, 2023-09-23T20:25:04.066 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1002673000000001, 2023-09-23T20:25:04.066 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:25:04.066 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:25:04.066 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:25:04.066 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.34791", 2023-09-23T20:25:04.067 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:25:04.067 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 28891 2023-09-23T20:25:04.067 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.067 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:25:04.067 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.067 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:25:04.067 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.968465+0000", 2023-09-23T20:25:04.067 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:25:04.067 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.067 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.067 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:25:04.068 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.968465+0000", 2023-09-23T20:25:04.068 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:25:04.068 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.068 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.068 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:25:04.068 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.968464+0000", 2023-09-23T20:25:04.068 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:25:04.068 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.068 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.068 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:25:04.068 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.968506+0000", 2023-09-23T20:25:04.068 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.2608999999999998e-05 2023-09-23T20:25:04.069 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.069 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.069 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:25:04.069 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.968529+0000", 2023-09-23T20:25:04.069 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.2875999999999999e-05 2023-09-23T20:25:04.069 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.069 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.069 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:25:04.069 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.968543+0000", 2023-09-23T20:25:04.069 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.4151000000000001e-05 2023-09-23T20:25:04.069 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.069 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.070 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:25:04.070 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.968601+0000", 2023-09-23T20:25:04.070 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.8199000000000003e-05 2023-09-23T20:25:04.070 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.070 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.070 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op must wait for map", 2023-09-23T20:25:04.070 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.968609+0000", 2023-09-23T20:25:04.070 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.1509999999999998e-06 2023-09-23T20:25:04.070 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.070 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.070 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:25:04.071 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:10.024810+0000", 2023-09-23T20:25:04.071 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.056201444000000003 2023-09-23T20:25:04.071 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.071 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.071 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:25:04.071 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:10.024819+0000", 2023-09-23T20:25:04.071 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.2210000000000003e-06 2023-09-23T20:25:04.071 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.071 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.071 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:25:04.071 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.068626+0000", 2023-09-23T20:25:04.071 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0438065649999999 2023-09-23T20:25:04.071 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.072 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.072 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:25:04.072 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.068733+0000", 2023-09-23T20:25:04.072 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000106909 2023-09-23T20:25:04.072 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:25:04.072 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:25:04.072 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:25:04.072 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.072 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.072 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.34791.0:28895 5.8 5:1248f566:::benchmark_data_smithi018_228451_object28894:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e118)", 2023-09-23T20:25:04.072 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:23:09.968563+0000", 2023-09-23T20:25:04.072 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 114.093550206, 2023-09-23T20:25:04.073 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.106749932, 2023-09-23T20:25:04.073 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:25:04.073 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:25:04.073 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:25:04.073 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.34791", 2023-09-23T20:25:04.073 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:25:04.073 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 28895 2023-09-23T20:25:04.073 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.073 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:25:04.073 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.073 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:25:04.073 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.968563+0000", 2023-09-23T20:25:04.074 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:25:04.074 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.074 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.074 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:25:04.074 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.968563+0000", 2023-09-23T20:25:04.074 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:25:04.074 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.074 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.074 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:25:04.074 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.968562+0000", 2023-09-23T20:25:04.074 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.999999 2023-09-23T20:25:04.075 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.075 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.075 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:25:04.075 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.968606+0000", 2023-09-23T20:25:04.075 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.4425999999999998e-05 2023-09-23T20:25:04.075 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.075 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.075 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:25:04.075 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.968616+0000", 2023-09-23T20:25:04.075 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.8810000000000008e-06 2023-09-23T20:25:04.075 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.076 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.076 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:25:04.076 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.968623+0000", 2023-09-23T20:25:04.076 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.652e-06 2023-09-23T20:25:04.076 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.076 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.076 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:25:04.076 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.968652+0000", 2023-09-23T20:25:04.076 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.9105e-05 2023-09-23T20:25:04.076 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.076 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.076 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting_for_map not empty", 2023-09-23T20:25:04.077 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.968656+0000", 2023-09-23T20:25:04.077 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.4340000000000002e-06 2023-09-23T20:25:04.077 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.077 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.077 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:25:04.077 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:10.024864+0000", 2023-09-23T20:25:04.077 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.056207319999999998 2023-09-23T20:25:04.077 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.077 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.077 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:25:04.077 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:10.024873+0000", 2023-09-23T20:25:04.077 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.0820000000000005e-06 2023-09-23T20:25:04.077 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.078 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.078 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:25:04.078 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.068764+0000", 2023-09-23T20:25:04.078 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.043891471 2023-09-23T20:25:04.078 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.078 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.078 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for degraded object", 2023-09-23T20:25:04.078 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.069685+0000", 2023-09-23T20:25:04.078 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00092115299999999997 2023-09-23T20:25:04.078 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.078 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.079 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:25:04.079 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.075213+0000", 2023-09-23T20:25:04.079 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0055280440000000002 2023-09-23T20:25:04.079 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.079 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.079 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:25:04.079 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.075312+0000", 2023-09-23T20:25:04.079 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.9123000000000006e-05 2023-09-23T20:25:04.079 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:25:04.079 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:25:04.079 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:25:04.079 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.080 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.080 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.34791.0:28897 5.18 5:1dddaaad:::benchmark_data_smithi018_228451_object28896:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e118)", 2023-09-23T20:25:04.080 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:23:09.968628+0000", 2023-09-23T20:25:04.080 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 114.09348457599999, 2023-09-23T20:25:04.080 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1136079919999999, 2023-09-23T20:25:04.080 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:25:04.080 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:25:04.080 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:25:04.080 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.34791", 2023-09-23T20:25:04.080 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:25:04.080 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 28897 2023-09-23T20:25:04.080 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.081 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:25:04.081 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.081 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:25:04.081 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.968628+0000", 2023-09-23T20:25:04.081 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:25:04.081 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.081 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.081 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:25:04.081 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.968628+0000", 2023-09-23T20:25:04.081 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:25:04.081 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.082 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.082 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:25:04.082 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.968628+0000", 2023-09-23T20:25:04.082 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:25:04.082 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.082 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.082 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:25:04.082 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.968656+0000", 2023-09-23T20:25:04.082 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.7968999999999999e-05 2023-09-23T20:25:04.082 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.082 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.082 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:25:04.083 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.968664+0000", 2023-09-23T20:25:04.083 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.4509999999999992e-06 2023-09-23T20:25:04.083 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.083 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.083 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:25:04.083 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.968670+0000", 2023-09-23T20:25:04.083 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.5589999999999996e-06 2023-09-23T20:25:04.083 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.083 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.083 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:25:04.083 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:10.024730+0000", 2023-09-23T20:25:04.084 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.056060655000000001 2023-09-23T20:25:04.084 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.084 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.084 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:25:04.084 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:10.024738+0000", 2023-09-23T20:25:04.084 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.1130000000000001e-06 2023-09-23T20:25:04.084 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.084 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.084 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:25:04.084 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.078174+0000", 2023-09-23T20:25:04.084 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.053435809 2023-09-23T20:25:04.084 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.084 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.085 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for degraded object", 2023-09-23T20:25:04.085 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.078877+0000", 2023-09-23T20:25:04.085 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000702913 2023-09-23T20:25:04.085 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.085 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.085 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:25:04.085 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.082148+0000", 2023-09-23T20:25:04.085 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0032704880000000002 2023-09-23T20:25:04.085 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.085 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.085 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:25:04.086 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.082236+0000", 2023-09-23T20:25:04.086 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.8488999999999999e-05 2023-09-23T20:25:04.086 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:25:04.086 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:25:04.086 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:25:04.086 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.086 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.086 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.34791.0:28898 5.16 5:6da0b3ba:::benchmark_data_smithi018_228451_object28897:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e118)", 2023-09-23T20:25:04.086 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:23:09.969024+0000", 2023-09-23T20:25:04.086 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 114.09308920300001, 2023-09-23T20:25:04.086 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.102100836, 2023-09-23T20:25:04.087 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:25:04.087 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:25:04.087 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:25:04.087 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.34791", 2023-09-23T20:25:04.087 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:25:04.087 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 28898 2023-09-23T20:25:04.087 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.087 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:25:04.087 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.087 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:25:04.087 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.969024+0000", 2023-09-23T20:25:04.087 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:25:04.088 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.088 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.088 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:25:04.088 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.969024+0000", 2023-09-23T20:25:04.088 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:25:04.088 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.088 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.088 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:25:04.088 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.969023+0000", 2023-09-23T20:25:04.088 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:25:04.088 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.088 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.088 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:25:04.089 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.969054+0000", 2023-09-23T20:25:04.089 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.1010000000000003e-05 2023-09-23T20:25:04.089 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.089 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.089 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:25:04.089 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.969066+0000", 2023-09-23T20:25:04.089 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1771e-05 2023-09-23T20:25:04.089 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.089 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.089 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:25:04.089 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.969075+0000", 2023-09-23T20:25:04.089 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.5040000000000008e-06 2023-09-23T20:25:04.090 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.090 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.090 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:25:04.090 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:10.010679+0000", 2023-09-23T20:25:04.090 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.041603581000000001 2023-09-23T20:25:04.090 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.090 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.090 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:25:04.090 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:10.010687+0000", 2023-09-23T20:25:04.090 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.6880000000000005e-06 2023-09-23T20:25:04.090 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.091 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.091 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:25:04.091 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.071051+0000", 2023-09-23T20:25:04.091 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.060363755 2023-09-23T20:25:04.091 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.091 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.091 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:25:04.091 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.071124+0000", 2023-09-23T20:25:04.091 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.3145000000000005e-05 2023-09-23T20:25:04.091 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:25:04.091 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:25:04.091 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:25:04.092 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.092 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.092 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.34791.0:28899 5.8 5:162cc4a5:::benchmark_data_smithi018_228451_object28898:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e118)", 2023-09-23T20:25:04.092 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:23:09.969091+0000", 2023-09-23T20:25:04.092 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 114.09302129300001, 2023-09-23T20:25:04.092 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.107737296, 2023-09-23T20:25:04.092 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:25:04.092 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:25:04.092 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:25:04.092 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.34791", 2023-09-23T20:25:04.092 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:25:04.092 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 28899 2023-09-23T20:25:04.093 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.093 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:25:04.093 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.093 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:25:04.093 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.969091+0000", 2023-09-23T20:25:04.093 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:25:04.093 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.093 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.093 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:25:04.093 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.969091+0000", 2023-09-23T20:25:04.093 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:25:04.093 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.094 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.094 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:25:04.094 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.969091+0000", 2023-09-23T20:25:04.094 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:25:04.094 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.094 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.094 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:25:04.094 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.969122+0000", 2023-09-23T20:25:04.094 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.1510000000000002e-05 2023-09-23T20:25:04.094 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.094 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.094 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:25:04.095 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.969130+0000", 2023-09-23T20:25:04.095 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.1980000000000001e-06 2023-09-23T20:25:04.095 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.095 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.095 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:25:04.095 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.969137+0000", 2023-09-23T20:25:04.095 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.9120000000000001e-06 2023-09-23T20:25:04.095 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.095 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.095 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:25:04.095 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.969293+0000", 2023-09-23T20:25:04.096 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000155568 2023-09-23T20:25:04.096 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.096 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.096 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting_for_map not empty", 2023-09-23T20:25:04.096 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.969302+0000", 2023-09-23T20:25:04.096 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.2180000000000005e-06 2023-09-23T20:25:04.096 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.096 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.096 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:25:04.096 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:10.024903+0000", 2023-09-23T20:25:04.096 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.055601105999999997 2023-09-23T20:25:04.096 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.097 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.097 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:25:04.097 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:10.024911+0000", 2023-09-23T20:25:04.097 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.6499999999999996e-06 2023-09-23T20:25:04.097 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.097 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.097 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:25:04.097 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.069774+0000", 2023-09-23T20:25:04.097 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.044863152 2023-09-23T20:25:04.097 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.097 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.097 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for degraded object", 2023-09-23T20:25:04.098 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.070421+0000", 2023-09-23T20:25:04.098 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00064713200000000002 2023-09-23T20:25:04.098 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.098 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.098 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:25:04.098 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.076740+0000", 2023-09-23T20:25:04.098 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0063188300000000001 2023-09-23T20:25:04.098 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.098 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.098 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:25:04.098 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.076829+0000", 2023-09-23T20:25:04.098 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.8709999999999996e-05 2023-09-23T20:25:04.099 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:25:04.099 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:25:04.099 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:25:04.099 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.099 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.099 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.34791.0:28902 5.6 5:650457bb:::benchmark_data_smithi018_228451_object28901:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e118)", 2023-09-23T20:25:04.099 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:23:09.969142+0000", 2023-09-23T20:25:04.099 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 114.09297035500001, 2023-09-23T20:25:04.099 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0945083250000001, 2023-09-23T20:25:04.099 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:25:04.099 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:25:04.100 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:25:04.100 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.34791", 2023-09-23T20:25:04.100 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:25:04.100 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 28902 2023-09-23T20:25:04.100 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.100 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:25:04.100 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.100 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:25:04.100 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.969142+0000", 2023-09-23T20:25:04.100 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:25:04.100 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.100 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.100 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:25:04.101 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.969142+0000", 2023-09-23T20:25:04.101 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:25:04.101 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.101 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.101 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:25:04.101 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.969142+0000", 2023-09-23T20:25:04.101 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:25:04.101 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.101 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.101 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:25:04.101 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.969171+0000", 2023-09-23T20:25:04.101 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.9196999999999998e-05 2023-09-23T20:25:04.102 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.102 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.102 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:25:04.102 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.969178+0000", 2023-09-23T20:25:04.102 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.2350000000000002e-06 2023-09-23T20:25:04.102 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.102 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.102 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:25:04.102 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.969184+0000", 2023-09-23T20:25:04.102 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.0329999999999997e-06 2023-09-23T20:25:04.102 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.102 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.103 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:25:04.103 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.970393+0000", 2023-09-23T20:25:04.103 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.001209183 2023-09-23T20:25:04.103 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.103 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.103 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op must wait for map", 2023-09-23T20:25:04.103 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.970400+0000", 2023-09-23T20:25:04.103 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.2720000000000003e-06 2023-09-23T20:25:04.103 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.103 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.103 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:25:04.103 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:10.010509+0000", 2023-09-23T20:25:04.103 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.04010975 2023-09-23T20:25:04.104 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.104 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.104 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:25:04.104 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:10.010524+0000", 2023-09-23T20:25:04.104 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.4620999999999999e-05 2023-09-23T20:25:04.104 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.104 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.104 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:25:04.104 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.063562+0000", 2023-09-23T20:25:04.104 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0530376589999999 2023-09-23T20:25:04.104 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.104 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.105 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:25:04.105 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.063651+0000", 2023-09-23T20:25:04.105 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.8992000000000004e-05 2023-09-23T20:25:04.105 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:25:04.105 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:25:04.105 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:25:04.105 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.105 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.105 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.34791.0:88809 5.a 5:5ab11718:::benchmark_data_smithi018_228451_object88808:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e151)", 2023-09-23T20:25:04.105 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:24:10.611557+0000", 2023-09-23T20:25:04.105 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 53.450555776999998, 2023-09-23T20:25:04.105 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.114272402, 2023-09-23T20:25:04.106 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:25:04.106 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:25:04.106 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:25:04.106 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.34791", 2023-09-23T20:25:04.106 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:25:04.106 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 88809 2023-09-23T20:25:04.106 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.106 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:25:04.106 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.106 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:25:04.106 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:10.611557+0000", 2023-09-23T20:25:04.106 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:25:04.107 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.107 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.107 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:25:04.107 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:10.611557+0000", 2023-09-23T20:25:04.107 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:25:04.107 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.107 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.107 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:25:04.107 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:10.611556+0000", 2023-09-23T20:25:04.107 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.999999 2023-09-23T20:25:04.107 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.107 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.108 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:25:04.108 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:10.611578+0000", 2023-09-23T20:25:04.108 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.2201999999999999e-05 2023-09-23T20:25:04.108 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.108 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.108 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:25:04.108 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:10.611587+0000", 2023-09-23T20:25:04.108 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.0820000000000005e-06 2023-09-23T20:25:04.108 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.108 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.108 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:25:04.108 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:10.611592+0000", 2023-09-23T20:25:04.109 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.259e-06 2023-09-23T20:25:04.109 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.109 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.109 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:25:04.109 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:10.611632+0000", 2023-09-23T20:25:04.109 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.9798000000000001e-05 2023-09-23T20:25:04.109 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.109 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.109 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:25:04.109 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:10.611727+0000", 2023-09-23T20:25:04.109 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.4843e-05 2023-09-23T20:25:04.109 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.109 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.110 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:25:04.110 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:10.611841+0000", 2023-09-23T20:25:04.110 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00011386400000000001 2023-09-23T20:25:04.110 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.110 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.110 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:25:04.110 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:10.614839+0000", 2023-09-23T20:25:04.110 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0029978840000000001 2023-09-23T20:25:04.110 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.110 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.110 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:25:04.110 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:10.614970+0000", 2023-09-23T20:25:04.111 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00013094100000000001 2023-09-23T20:25:04.111 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.111 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.111 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:25:04.111 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:10.725735+0000", 2023-09-23T20:25:04.111 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.110765216 2023-09-23T20:25:04.111 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.111 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.111 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:25:04.111 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:10.725781+0000", 2023-09-23T20:25:04.111 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.5655000000000002e-05 2023-09-23T20:25:04.111 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.112 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.112 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:25:04.112 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:10.725829+0000", 2023-09-23T20:25:04.112 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.8690999999999998e-05 2023-09-23T20:25:04.112 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:25:04.112 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:25:04.112 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:25:04.112 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.112 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.112 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.34791.0:88821 5.b 5:d71c7e15:::benchmark_data_smithi018_228451_object88820:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e151)", 2023-09-23T20:25:04.112 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:24:10.621243+0000", 2023-09-23T20:25:04.112 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 53.440869503000002, 2023-09-23T20:25:04.113 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.112987824, 2023-09-23T20:25:04.113 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:25:04.113 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:25:04.113 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:25:04.113 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.34791", 2023-09-23T20:25:04.113 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:25:04.113 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 88821 2023-09-23T20:25:04.113 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.113 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:25:04.113 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.113 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:25:04.113 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:10.621243+0000", 2023-09-23T20:25:04.114 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:25:04.114 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.114 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.114 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:25:04.114 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:10.621243+0000", 2023-09-23T20:25:04.114 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:25:04.114 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.114 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.114 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:25:04.114 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:10.621242+0000", 2023-09-23T20:25:04.114 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.999999 2023-09-23T20:25:04.114 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.115 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.115 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:25:04.115 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:10.621271+0000", 2023-09-23T20:25:04.115 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.8954000000000001e-05 2023-09-23T20:25:04.115 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.115 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.115 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:25:04.115 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:10.621283+0000", 2023-09-23T20:25:04.115 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1837e-05 2023-09-23T20:25:04.115 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.115 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.115 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:25:04.116 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:10.621291+0000", 2023-09-23T20:25:04.116 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.9950000000000005e-06 2023-09-23T20:25:04.116 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.116 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.116 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:25:04.116 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:10.722642+0000", 2023-09-23T20:25:04.116 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.10135113900000001 2023-09-23T20:25:04.116 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.116 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.116 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:25:04.116 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:10.722816+0000", 2023-09-23T20:25:04.116 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00017411 2023-09-23T20:25:04.117 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.117 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.117 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:25:04.117 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:10.722978+0000", 2023-09-23T20:25:04.117 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00016133400000000001 2023-09-23T20:25:04.117 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.117 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.117 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:25:04.117 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:10.728944+0000", 2023-09-23T20:25:04.117 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.005966191 2023-09-23T20:25:04.117 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.118 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.118 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:25:04.118 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:10.730054+0000", 2023-09-23T20:25:04.118 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0011101710000000001 2023-09-23T20:25:04.118 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.118 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.118 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:25:04.118 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:10.734152+0000", 2023-09-23T20:25:04.118 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0040975910000000003 2023-09-23T20:25:04.118 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.118 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.118 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:25:04.119 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:10.734196+0000", 2023-09-23T20:25:04.119 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.4839999999999998e-05 2023-09-23T20:25:04.119 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.119 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.119 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:25:04.119 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:10.734231+0000", 2023-09-23T20:25:04.119 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.4604999999999998e-05 2023-09-23T20:25:04.119 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:25:04.119 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:25:04.119 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:25:04.119 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.119 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.120 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.34791.0:88825 5.8 5:13e07650:::benchmark_data_smithi018_228451_object88824:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e151)", 2023-09-23T20:25:04.120 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:24:10.626599+0000", 2023-09-23T20:25:04.120 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 53.435513853000003, 2023-09-23T20:25:04.120 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.109427939, 2023-09-23T20:25:04.120 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:25:04.120 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:25:04.120 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:25:04.120 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.34791", 2023-09-23T20:25:04.120 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:25:04.120 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 88825 2023-09-23T20:25:04.120 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.120 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:25:04.121 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.121 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:25:04.121 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:10.626599+0000", 2023-09-23T20:25:04.121 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:25:04.121 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.121 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.121 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:25:04.121 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:10.626599+0000", 2023-09-23T20:25:04.121 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:25:04.121 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.121 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.121 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:25:04.121 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:10.626598+0000", 2023-09-23T20:25:04.122 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.999999 2023-09-23T20:25:04.122 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.122 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.122 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:25:04.122 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:10.626617+0000", 2023-09-23T20:25:04.122 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.9516000000000001e-05 2023-09-23T20:25:04.122 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.122 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.122 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:25:04.122 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:10.724786+0000", 2023-09-23T20:25:04.122 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.098168431 2023-09-23T20:25:04.122 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.123 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.123 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:25:04.123 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:10.724804+0000", 2023-09-23T20:25:04.123 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.8046999999999998e-05 2023-09-23T20:25:04.123 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.123 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.123 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:25:04.123 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:10.727060+0000", 2023-09-23T20:25:04.123 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.002256311 2023-09-23T20:25:04.123 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.123 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.123 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:25:04.124 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:10.727172+0000", 2023-09-23T20:25:04.124 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000112014 2023-09-23T20:25:04.124 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.124 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.124 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 0,1", 2023-09-23T20:25:04.124 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:10.727292+0000", 2023-09-23T20:25:04.124 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00012037899999999999 2023-09-23T20:25:04.124 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.124 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.124 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:25:04.124 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:10.730946+0000", 2023-09-23T20:25:04.124 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.003653997 2023-09-23T20:25:04.125 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.125 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.125 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:25:04.125 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:10.734740+0000", 2023-09-23T20:25:04.125 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0037931509999999998 2023-09-23T20:25:04.125 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.125 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.125 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:25:04.125 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:10.735942+0000", 2023-09-23T20:25:04.125 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0012020609999999999 2023-09-23T20:25:04.125 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.125 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.125 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:25:04.126 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:10.735986+0000", 2023-09-23T20:25:04.126 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.4224999999999998e-05 2023-09-23T20:25:04.126 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.126 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.126 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:25:04.126 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:10.736027+0000", 2023-09-23T20:25:04.126 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.0992000000000003e-05 2023-09-23T20:25:04.126 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:25:04.126 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:25:04.126 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:25:04.126 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.126 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.127 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.34791.0:115711 5.b 5:d0314ec1:::benchmark_data_smithi018_228451_object8650:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e158)", 2023-09-23T20:25:04.127 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:24:44.792263+0000", 2023-09-23T20:25:04.127 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 19.269849695000001, 2023-09-23T20:25:04.127 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.17426969, 2023-09-23T20:25:04.127 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:25:04.127 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:25:04.127 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:25:04.127 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.34791", 2023-09-23T20:25:04.127 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:25:04.127 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 115711 2023-09-23T20:25:04.127 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.127 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:25:04.128 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.128 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:25:04.128 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:44.792263+0000", 2023-09-23T20:25:04.128 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:25:04.128 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.128 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.128 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:25:04.128 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:44.792263+0000", 2023-09-23T20:25:04.128 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:25:04.128 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.128 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.128 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:25:04.129 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:44.792262+0000", 2023-09-23T20:25:04.129 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.999999 2023-09-23T20:25:04.129 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.129 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.129 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:25:04.129 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:44.792268+0000", 2023-09-23T20:25:04.129 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.9580000000000004e-06 2023-09-23T20:25:04.129 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.129 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.129 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:25:04.129 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:44.792280+0000", 2023-09-23T20:25:04.129 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2616000000000001e-05 2023-09-23T20:25:04.129 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.130 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.130 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:25:04.130 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:44.792288+0000", 2023-09-23T20:25:04.130 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.6240000000000002e-06 2023-09-23T20:25:04.130 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.130 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.130 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:25:04.130 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:44.792332+0000", 2023-09-23T20:25:04.130 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.3705999999999998e-05 2023-09-23T20:25:04.130 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.130 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.130 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:25:04.131 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:44.792457+0000", 2023-09-23T20:25:04.131 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00012493399999999999 2023-09-23T20:25:04.131 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.131 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.131 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:25:04.131 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:44.792568+0000", 2023-09-23T20:25:04.131 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000111208 2023-09-23T20:25:04.131 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.131 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.131 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:25:04.131 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:44.796427+0000", 2023-09-23T20:25:04.131 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0038591530000000001 2023-09-23T20:25:04.132 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.132 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.132 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:25:04.132 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:44.798952+0000", 2023-09-23T20:25:04.132 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0025252249999999999 2023-09-23T20:25:04.132 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.132 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.132 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:25:04.132 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:44.966445+0000", 2023-09-23T20:25:04.132 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.167493109 2023-09-23T20:25:04.132 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.132 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.133 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:25:04.133 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:44.966499+0000", 2023-09-23T20:25:04.133 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.3919000000000003e-05 2023-09-23T20:25:04.133 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.133 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.133 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:25:04.133 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:44.966533+0000", 2023-09-23T20:25:04.133 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.3426999999999997e-05 2023-09-23T20:25:04.133 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:25:04.133 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:25:04.133 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:25:04.133 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.133 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.134 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.34791.0:117728 5.a 5:57bfb76d:::benchmark_data_smithi018_228451_object10667:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e159)", 2023-09-23T20:25:04.134 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:24:46.926020+0000", 2023-09-23T20:25:04.134 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 17.136093196000001, 2023-09-23T20:25:04.134 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.12683606, 2023-09-23T20:25:04.134 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:25:04.134 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:25:04.134 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:25:04.134 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.34791", 2023-09-23T20:25:04.134 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:25:04.134 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 117728 2023-09-23T20:25:04.134 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.135 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:25:04.135 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.135 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:25:04.135 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:46.926020+0000", 2023-09-23T20:25:04.135 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:25:04.135 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.135 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.135 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:25:04.135 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:46.926020+0000", 2023-09-23T20:25:04.135 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:25:04.135 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.135 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.136 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:25:04.136 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:46.926019+0000", 2023-09-23T20:25:04.136 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.999999 2023-09-23T20:25:04.136 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.136 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.136 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:25:04.136 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:46.926024+0000", 2023-09-23T20:25:04.136 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.792e-06 2023-09-23T20:25:04.136 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.136 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.136 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:25:04.136 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:46.926034+0000", 2023-09-23T20:25:04.136 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0051000000000001e-05 2023-09-23T20:25:04.137 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.137 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.137 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:25:04.137 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:46.926040+0000", 2023-09-23T20:25:04.137 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.8310000000000003e-06 2023-09-23T20:25:04.137 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.137 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.137 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:25:04.137 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:46.934921+0000", 2023-09-23T20:25:04.137 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0088809179999999998 2023-09-23T20:25:04.137 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.137 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.138 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:25:04.138 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:46.935093+0000", 2023-09-23T20:25:04.138 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000171993 2023-09-23T20:25:04.138 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.138 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.138 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 0,4", 2023-09-23T20:25:04.138 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:46.935248+0000", 2023-09-23T20:25:04.138 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00015415399999999999 2023-09-23T20:25:04.138 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.138 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.138 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:25:04.138 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:46.938741+0000", 2023-09-23T20:25:04.139 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0034931480000000002 2023-09-23T20:25:04.139 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.139 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.139 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:25:04.139 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:46.943110+0000", 2023-09-23T20:25:04.139 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0043693969999999997 2023-09-23T20:25:04.139 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.139 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.139 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:25:04.139 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:47.047751+0000", 2023-09-23T20:25:04.139 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.10464069500000001 2023-09-23T20:25:04.139 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.140 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.140 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:25:04.140 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:47.047816+0000", 2023-09-23T20:25:04.140 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.5541000000000001e-05 2023-09-23T20:25:04.140 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.140 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.140 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:25:04.140 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:47.052856+0000", 2023-09-23T20:25:04.140 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0050392989999999997 2023-09-23T20:25:04.140 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:25:04.140 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:25:04.140 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:25:04.140 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.141 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.141 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.34791.0:117742 5.6 5:65d3913c:::benchmark_data_smithi018_228451_object10681:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e159)", 2023-09-23T20:25:04.141 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:24:46.939052+0000", 2023-09-23T20:25:04.141 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 17.123060622000001, 2023-09-23T20:25:04.141 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.10870558600000001, 2023-09-23T20:25:04.141 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:25:04.141 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:25:04.141 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:25:04.141 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.34791", 2023-09-23T20:25:04.141 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:25:04.141 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 117742 2023-09-23T20:25:04.142 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.142 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:25:04.142 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.142 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:25:04.142 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:46.939052+0000", 2023-09-23T20:25:04.142 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:25:04.142 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.142 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.142 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:25:04.142 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:46.939052+0000", 2023-09-23T20:25:04.142 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:25:04.142 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.143 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.143 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:25:04.143 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:46.939051+0000", 2023-09-23T20:25:04.143 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:25:04.143 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.143 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.143 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:25:04.143 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:46.939056+0000", 2023-09-23T20:25:04.143 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.476e-06 2023-09-23T20:25:04.143 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.143 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.144 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:25:04.144 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:46.939064+0000", 2023-09-23T20:25:04.144 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.5689999999999999e-06 2023-09-23T20:25:04.144 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.144 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.144 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:25:04.144 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:46.939068+0000", 2023-09-23T20:25:04.144 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.7149999999999997e-06 2023-09-23T20:25:04.144 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.144 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.144 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:25:04.145 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:46.939116+0000", 2023-09-23T20:25:04.145 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.7394000000000001e-05 2023-09-23T20:25:04.145 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.145 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.145 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:25:04.145 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:46.939226+0000", 2023-09-23T20:25:04.145 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00011065000000000001 2023-09-23T20:25:04.145 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.145 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.145 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 5,7", 2023-09-23T20:25:04.145 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:46.939327+0000", 2023-09-23T20:25:04.145 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00010100800000000001 2023-09-23T20:25:04.145 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.146 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.146 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:25:04.146 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:46.941489+0000", 2023-09-23T20:25:04.146 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.002161708 2023-09-23T20:25:04.146 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.146 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.146 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:25:04.146 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:47.047544+0000", 2023-09-23T20:25:04.146 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.106054763 2023-09-23T20:25:04.146 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.146 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.146 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:25:04.147 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:47.047666+0000", 2023-09-23T20:25:04.147 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00012227000000000001 2023-09-23T20:25:04.147 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.147 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.147 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:25:04.147 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:47.047714+0000", 2023-09-23T20:25:04.147 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.7942999999999998e-05 2023-09-23T20:25:04.147 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.147 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.147 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:25:04.147 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:47.047758+0000", 2023-09-23T20:25:04.147 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.3705999999999998e-05 2023-09-23T20:25:04.148 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:25:04.148 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:25:04.148 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:25:04.148 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.148 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.148 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.34791.0:117743 5.a 5:58e1e67c:::benchmark_data_smithi018_228451_object10682:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e159)", 2023-09-23T20:25:04.148 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:24:46.939441+0000", 2023-09-23T20:25:04.148 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 17.122671711999999, 2023-09-23T20:25:04.148 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.11830555700000001, 2023-09-23T20:25:04.148 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:25:04.148 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:25:04.148 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:25:04.149 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.34791", 2023-09-23T20:25:04.149 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:25:04.149 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 117743 2023-09-23T20:25:04.149 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.149 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:25:04.149 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.149 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:25:04.149 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:46.939441+0000", 2023-09-23T20:25:04.149 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:25:04.149 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.149 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.149 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:25:04.150 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:46.939441+0000", 2023-09-23T20:25:04.150 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:25:04.150 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.150 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.150 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:25:04.150 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:46.939440+0000", 2023-09-23T20:25:04.150 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.999999 2023-09-23T20:25:04.150 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.150 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.150 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:25:04.150 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:46.939445+0000", 2023-09-23T20:25:04.151 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.7090000000000001e-06 2023-09-23T20:25:04.151 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.151 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.151 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:25:04.151 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:46.939455+0000", 2023-09-23T20:25:04.151 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.9210000000000004e-06 2023-09-23T20:25:04.151 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.151 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.151 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:25:04.151 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:46.939461+0000", 2023-09-23T20:25:04.151 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.5799999999999997e-06 2023-09-23T20:25:04.151 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.152 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.152 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:25:04.152 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:46.939505+0000", 2023-09-23T20:25:04.152 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.3504000000000002e-05 2023-09-23T20:25:04.152 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.152 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.152 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:25:04.152 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:46.939600+0000", 2023-09-23T20:25:04.152 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.5085000000000002e-05 2023-09-23T20:25:04.152 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.152 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.152 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 0,4", 2023-09-23T20:25:04.153 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:46.939725+0000", 2023-09-23T20:25:04.153 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000125248 2023-09-23T20:25:04.153 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.153 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.153 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:25:04.153 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:46.943160+0000", 2023-09-23T20:25:04.153 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.003435067 2023-09-23T20:25:04.153 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.153 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.153 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:25:04.153 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:47.047482+0000", 2023-09-23T20:25:04.153 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.10432166900000001 2023-09-23T20:25:04.154 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.154 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.154 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:25:04.154 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:47.052917+0000", 2023-09-23T20:25:04.154 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0054350630000000004 2023-09-23T20:25:04.154 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.154 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.154 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:25:04.154 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:47.052979+0000", 2023-09-23T20:25:04.154 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.1644999999999997e-05 2023-09-23T20:25:04.154 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.155 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.155 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:25:04.155 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:47.057747+0000", 2023-09-23T20:25:04.155 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0047677869999999999 2023-09-23T20:25:04.155 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:25:04.155 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:25:04.155 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:25:04.155 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.155 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.155 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.34791.0:119798 5.a 5:507139a1:::benchmark_data_smithi018_228451_object12737:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e159)", 2023-09-23T20:25:04.155 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:24:49.061477+0000", 2023-09-23T20:25:04.155 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 15.000635970999999, 2023-09-23T20:25:04.156 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.111888767, 2023-09-23T20:25:04.156 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:25:04.156 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:25:04.156 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:25:04.156 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.34791", 2023-09-23T20:25:04.156 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:25:04.156 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 119798 2023-09-23T20:25:04.156 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.156 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:25:04.156 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.156 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:25:04.157 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:49.061477+0000", 2023-09-23T20:25:04.157 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:25:04.157 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.157 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.157 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:25:04.157 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:49.061477+0000", 2023-09-23T20:25:04.157 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:25:04.157 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.157 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.157 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:25:04.157 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:49.061476+0000", 2023-09-23T20:25:04.158 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.999999 2023-09-23T20:25:04.158 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.158 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.158 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:25:04.158 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:49.061481+0000", 2023-09-23T20:25:04.158 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.5380000000000002e-06 2023-09-23T20:25:04.158 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.158 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.158 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:25:04.158 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:49.061492+0000", 2023-09-23T20:25:04.158 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0322e-05 2023-09-23T20:25:04.158 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.159 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.159 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:25:04.159 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:49.061497+0000", 2023-09-23T20:25:04.159 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.3839999999999996e-06 2023-09-23T20:25:04.159 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.159 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.159 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:25:04.159 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:49.061530+0000", 2023-09-23T20:25:04.159 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.3160000000000001e-05 2023-09-23T20:25:04.159 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.159 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.160 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:25:04.160 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:49.061640+0000", 2023-09-23T20:25:04.160 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000109663 2023-09-23T20:25:04.160 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.160 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.160 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 0,4", 2023-09-23T20:25:04.160 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:49.061731+0000", 2023-09-23T20:25:04.160 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.1391000000000006e-05 2023-09-23T20:25:04.160 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.160 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.160 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:25:04.160 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:49.064689+0000", 2023-09-23T20:25:04.161 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0029582499999999999 2023-09-23T20:25:04.161 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.161 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.161 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:25:04.161 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:49.066208+0000", 2023-09-23T20:25:04.161 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0015184370000000001 2023-09-23T20:25:04.161 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.161 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.161 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:25:04.161 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:49.168464+0000", 2023-09-23T20:25:04.161 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.10225582800000001 2023-09-23T20:25:04.161 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.162 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.162 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:25:04.162 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:49.168512+0000", 2023-09-23T20:25:04.162 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.7939999999999998e-05 2023-09-23T20:25:04.162 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.162 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.162 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:25:04.162 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:49.173366+0000", 2023-09-23T20:25:04.162 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0048539129999999996 2023-09-23T20:25:04.162 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:25:04.162 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:25:04.163 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:25:04.163 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.163 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.163 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.34791.0:122477 5.a 5:5f61046a:::benchmark_data_smithi018_228451_object15416:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e159)", 2023-09-23T20:25:04.163 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:24:52.040682+0000", 2023-09-23T20:25:04.163 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 12.0214309, 2023-09-23T20:25:04.163 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0382268649999999, 2023-09-23T20:25:04.163 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:25:04.163 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:25:04.163 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:25:04.163 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.34791", 2023-09-23T20:25:04.163 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:25:04.164 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 122477 2023-09-23T20:25:04.164 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.164 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:25:04.164 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.164 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:25:04.164 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:52.040682+0000", 2023-09-23T20:25:04.164 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:25:04.164 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.164 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.164 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:25:04.164 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:52.040682+0000", 2023-09-23T20:25:04.164 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:25:04.165 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.165 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.165 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:25:04.165 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:52.040680+0000", 2023-09-23T20:25:04.165 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:25:04.165 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.165 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.165 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:25:04.165 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:52.040685+0000", 2023-09-23T20:25:04.165 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.9540000000000003e-06 2023-09-23T20:25:04.165 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.165 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.166 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:25:04.166 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:52.040695+0000", 2023-09-23T20:25:04.166 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.8600000000000005e-06 2023-09-23T20:25:04.166 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.166 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.166 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:25:04.166 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:52.040702+0000", 2023-09-23T20:25:04.166 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.0809999999999996e-06 2023-09-23T20:25:04.166 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.166 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.166 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:25:04.166 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:52.046630+0000", 2023-09-23T20:25:04.167 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0059274710000000001 2023-09-23T20:25:04.167 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.167 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.167 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:25:04.167 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:52.046727+0000", 2023-09-23T20:25:04.167 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.7386999999999994e-05 2023-09-23T20:25:04.167 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.167 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.167 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 0,4", 2023-09-23T20:25:04.167 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:52.046829+0000", 2023-09-23T20:25:04.167 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000101801 2023-09-23T20:25:04.168 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.168 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.168 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:25:04.168 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:52.055043+0000", 2023-09-23T20:25:04.168 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0082142259999999998 2023-09-23T20:25:04.168 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.168 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.168 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:25:04.168 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:52.063458+0000", 2023-09-23T20:25:04.168 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0084143670000000007 2023-09-23T20:25:04.168 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.169 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.169 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:25:04.169 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:52.090685+0000", 2023-09-23T20:25:04.169 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.027227429000000001 2023-09-23T20:25:04.169 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.169 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.169 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:25:04.169 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:52.090696+0000", 2023-09-23T20:25:04.169 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0993e-05 2023-09-23T20:25:04.169 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.169 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.169 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:25:04.170 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:53.078892+0000", 2023-09-23T20:25:04.170 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.988195822 2023-09-23T20:25:04.170 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.170 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.170 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:25:04.170 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:53.078909+0000", 2023-09-23T20:25:04.170 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.6815000000000001e-05 2023-09-23T20:25:04.170 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:25:04.170 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:25:04.170 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:25:04.171 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.171 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.171 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.34791.0:122487 5.a 5:53c53dfe:::benchmark_data_smithi018_228451_object15426:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e159)", 2023-09-23T20:25:04.171 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:24:52.046007+0000", 2023-09-23T20:25:04.171 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 12.016105924, 2023-09-23T20:25:04.171 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0329430020000001, 2023-09-23T20:25:04.171 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:25:04.171 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:25:04.171 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:25:04.171 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.34791", 2023-09-23T20:25:04.171 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:25:04.171 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 122487 2023-09-23T20:25:04.172 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.172 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:25:04.172 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.172 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:25:04.172 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:52.046007+0000", 2023-09-23T20:25:04.172 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:25:04.172 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.172 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.172 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:25:04.172 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:52.046007+0000", 2023-09-23T20:25:04.172 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:25:04.172 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.173 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.173 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:25:04.173 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:52.046006+0000", 2023-09-23T20:25:04.173 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.999999 2023-09-23T20:25:04.173 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.173 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.173 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:25:04.173 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:52.046011+0000", 2023-09-23T20:25:04.173 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.6459999999999998e-06 2023-09-23T20:25:04.173 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.173 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.173 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:25:04.174 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:52.046021+0000", 2023-09-23T20:25:04.174 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.3589999999999997e-06 2023-09-23T20:25:04.174 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.174 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.174 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:25:04.174 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:52.046028+0000", 2023-09-23T20:25:04.174 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.9290000000000001e-06 2023-09-23T20:25:04.174 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.174 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.174 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:25:04.174 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:52.050156+0000", 2023-09-23T20:25:04.175 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0041279200000000002 2023-09-23T20:25:04.175 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.175 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.175 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:25:04.175 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:52.050339+0000", 2023-09-23T20:25:04.175 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00018302999999999999 2023-09-23T20:25:04.175 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.175 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.175 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 0,4", 2023-09-23T20:25:04.175 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:52.050509+0000", 2023-09-23T20:25:04.175 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00017038000000000001 2023-09-23T20:25:04.175 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.176 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.176 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:25:04.176 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:52.058430+0000", 2023-09-23T20:25:04.176 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0079205839999999996 2023-09-23T20:25:04.176 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.176 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.176 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:25:04.176 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:52.063549+0000", 2023-09-23T20:25:04.176 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0051190029999999996 2023-09-23T20:25:04.176 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.176 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.177 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:25:04.177 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:52.090721+0000", 2023-09-23T20:25:04.177 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.027172321999999999 2023-09-23T20:25:04.177 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.177 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.177 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:25:04.177 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:52.090729+0000", 2023-09-23T20:25:04.177 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.695e-06 2023-09-23T20:25:04.177 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.177 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.177 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:25:04.177 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:53.078939+0000", 2023-09-23T20:25:04.178 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.98820984999999995 2023-09-23T20:25:04.178 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.178 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.178 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:25:04.178 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:53.078950+0000", 2023-09-23T20:25:04.178 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1259e-05 2023-09-23T20:25:04.178 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:25:04.178 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:25:04.178 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:25:04.178 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.178 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.178 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.34791.0:122497 5.a 5:52cb959d:::benchmark_data_smithi018_228451_object15436:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e159)", 2023-09-23T20:25:04.179 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:24:52.063924+0000", 2023-09-23T20:25:04.179 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 11.998188860999999, 2023-09-23T20:25:04.179 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0150647070000001, 2023-09-23T20:25:04.179 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:25:04.179 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:25:04.179 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:25:04.179 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.34791", 2023-09-23T20:25:04.179 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:25:04.179 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 122497 2023-09-23T20:25:04.179 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.179 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:25:04.179 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.180 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:25:04.180 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:52.063924+0000", 2023-09-23T20:25:04.180 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:25:04.180 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.180 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.180 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:25:04.180 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:52.063924+0000", 2023-09-23T20:25:04.180 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:25:04.180 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.180 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.180 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:25:04.180 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:52.063922+0000", 2023-09-23T20:25:04.181 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:25:04.181 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.181 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.181 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:25:04.181 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:52.063929+0000", 2023-09-23T20:25:04.181 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.109e-06 2023-09-23T20:25:04.181 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.181 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.181 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:25:04.181 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:52.063945+0000", 2023-09-23T20:25:04.181 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.5875999999999998e-05 2023-09-23T20:25:04.181 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.182 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.182 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:25:04.182 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:52.063954+0000", 2023-09-23T20:25:04.182 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.4659999999999999e-06 2023-09-23T20:25:04.182 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.182 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.182 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:25:04.182 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:52.064052+0000", 2023-09-23T20:25:04.182 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.8398000000000002e-05 2023-09-23T20:25:04.182 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.182 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.182 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:25:04.183 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:52.064239+0000", 2023-09-23T20:25:04.183 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00018665399999999999 2023-09-23T20:25:04.183 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.183 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.183 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 0,4", 2023-09-23T20:25:04.183 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:52.064381+0000", 2023-09-23T20:25:04.183 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000142464 2023-09-23T20:25:04.183 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.183 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.183 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:25:04.183 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:52.090753+0000", 2023-09-23T20:25:04.183 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.026371896999999998 2023-09-23T20:25:04.184 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.184 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.184 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:25:04.184 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:52.090761+0000", 2023-09-23T20:25:04.184 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.6909999999999997e-06 2023-09-23T20:25:04.184 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.184 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.184 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:25:04.184 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:53.078976+0000", 2023-09-23T20:25:04.184 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.98821585199999995 2023-09-23T20:25:04.184 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.184 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.184 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:25:04.185 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:53.078989+0000", 2023-09-23T20:25:04.185 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2106000000000001e-05 2023-09-23T20:25:04.185 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:25:04.185 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:25:04.185 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:25:04.185 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.185 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.185 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.34791.0:122499 5.a 5.db9883aa (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e159)", 2023-09-23T20:25:04.185 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:24:52.063982+0000", 2023-09-23T20:25:04.185 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 11.998131019000001, 2023-09-23T20:25:04.185 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.01503842, 2023-09-23T20:25:04.186 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:25:04.186 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:25:04.186 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:25:04.186 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.34791", 2023-09-23T20:25:04.186 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:25:04.186 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 122499 2023-09-23T20:25:04.186 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.186 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:25:04.186 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.186 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:25:04.186 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:52.063982+0000", 2023-09-23T20:25:04.187 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:25:04.187 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.187 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.187 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:25:04.187 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:52.063982+0000", 2023-09-23T20:25:04.187 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:25:04.187 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.187 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.187 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:25:04.187 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:52.063981+0000", 2023-09-23T20:25:04.188 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:25:04.188 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.188 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.188 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:25:04.188 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:52.063986+0000", 2023-09-23T20:25:04.188 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.8600000000000001e-06 2023-09-23T20:25:04.188 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.188 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.188 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:25:04.188 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:52.063997+0000", 2023-09-23T20:25:04.188 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1418e-05 2023-09-23T20:25:04.189 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.189 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.189 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:25:04.189 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:52.064004+0000", 2023-09-23T20:25:04.189 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.0219999999999999e-06 2023-09-23T20:25:04.189 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.189 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.189 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:25:04.189 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:52.090781+0000", 2023-09-23T20:25:04.189 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.026776423000000001 2023-09-23T20:25:04.189 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.189 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.189 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:25:04.190 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:52.090797+0000", 2023-09-23T20:25:04.190 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.6223999999999999e-05 2023-09-23T20:25:04.190 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.190 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.190 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:25:04.190 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:53.079010+0000", 2023-09-23T20:25:04.190 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.98821315700000001 2023-09-23T20:25:04.190 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.190 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.190 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:25:04.190 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:53.079020+0000", 2023-09-23T20:25:04.191 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0013999999999999e-05 2023-09-23T20:25:04.191 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:25:04.191 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:25:04.191 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:25:04.191 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.191 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.191 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.34791.0:122509 5.a 5.9a5288fa (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e159)", 2023-09-23T20:25:04.191 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:24:52.093777+0000", 2023-09-23T20:25:04.191 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 11.968335461000001, 2023-09-23T20:25:04.191 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.98527409700000002, 2023-09-23T20:25:04.191 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:25:04.191 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:25:04.191 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:25:04.192 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.34791", 2023-09-23T20:25:04.192 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:25:04.192 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 122509 2023-09-23T20:25:04.192 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.192 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:25:04.192 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.192 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:25:04.192 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:52.093777+0000", 2023-09-23T20:25:04.192 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:25:04.192 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.192 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.192 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:25:04.193 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:52.093777+0000", 2023-09-23T20:25:04.193 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:25:04.193 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.193 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.193 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:25:04.193 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:52.093776+0000", 2023-09-23T20:25:04.193 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:25:04.193 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.193 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.193 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:25:04.193 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:52.093782+0000", 2023-09-23T20:25:04.194 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.9599999999999997e-06 2023-09-23T20:25:04.194 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.194 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.194 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:25:04.194 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:52.093800+0000", 2023-09-23T20:25:04.194 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.7878000000000001e-05 2023-09-23T20:25:04.194 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.194 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.194 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:25:04.194 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:52.093819+0000", 2023-09-23T20:25:04.194 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.9227000000000001e-05 2023-09-23T20:25:04.194 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.195 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.195 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:25:04.195 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:52.094412+0000", 2023-09-23T20:25:04.195 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00059324900000000001 2023-09-23T20:25:04.195 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.195 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.195 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:25:04.195 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:52.094423+0000", 2023-09-23T20:25:04.195 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0997e-05 2023-09-23T20:25:04.195 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.196 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.196 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:25:04.196 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:53.079042+0000", 2023-09-23T20:25:04.196 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.98461890399999996 2023-09-23T20:25:04.196 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.196 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.196 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:25:04.196 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:53.079051+0000", 2023-09-23T20:25:04.196 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.4509999999999998e-06 2023-09-23T20:25:04.196 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:25:04.196 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:25:04.196 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:25:04.197 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.197 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.197 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.34791.0:122516 5.a 5.76cbecaa (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e160)", 2023-09-23T20:25:04.197 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:24:52.104509+0000", 2023-09-23T20:25:04.197 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 11.957603602000001, 2023-09-23T20:25:04.197 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.974572522, 2023-09-23T20:25:04.197 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:25:04.197 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:25:04.197 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:25:04.197 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.34791", 2023-09-23T20:25:04.197 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:25:04.197 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 122516 2023-09-23T20:25:04.197 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.198 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:25:04.198 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.198 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:25:04.198 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:52.104509+0000", 2023-09-23T20:25:04.198 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:25:04.198 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.198 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.198 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:25:04.198 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:52.104509+0000", 2023-09-23T20:25:04.198 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:25:04.198 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.198 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.199 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:25:04.199 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:52.104507+0000", 2023-09-23T20:25:04.199 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:25:04.199 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.199 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.199 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:25:04.199 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:52.104516+0000", 2023-09-23T20:25:04.199 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.0939999999999997e-06 2023-09-23T20:25:04.199 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.199 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.199 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:25:04.199 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:52.104531+0000", 2023-09-23T20:25:04.200 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.4579999999999999e-05 2023-09-23T20:25:04.200 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.200 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.200 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:25:04.200 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:52.104540+0000", 2023-09-23T20:25:04.200 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.1910000000000006e-06 2023-09-23T20:25:04.200 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.200 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.200 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:25:04.200 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:52.104587+0000", 2023-09-23T20:25:04.200 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.6489999999999997e-05 2023-09-23T20:25:04.200 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.200 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.201 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:25:04.201 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:52.104596+0000", 2023-09-23T20:25:04.201 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.3009999999999995e-06 2023-09-23T20:25:04.201 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.201 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.201 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:25:04.201 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:53.079072+0000", 2023-09-23T20:25:04.201 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.97447639500000005 2023-09-23T20:25:04.201 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:04.201 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:04.201 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:25:04.201 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:53.079082+0000", 2023-09-23T20:25:04.201 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.4150000000000005e-06 2023-09-23T20:25:04.202 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:25:04.202 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:25:04.202 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:25:04.202 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:25:04.202 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:25:04.202 INFO:teuthology.orchestra.run.smithi018.stdout:} 2023-09-23T20:25:04.203 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:25:04.203 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:03 smithi018 ceph-mon[184144]: failed to encode map e166 with expected crc 2023-09-23T20:25:04.203 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:03 smithi018 ceph-mon[184144]: failed to encode map e166 with expected crc 2023-09-23T20:25:04.203 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:03 smithi018 ceph-mon[184144]: osdmap e166: 8 total, 8 up, 7 in 2023-09-23T20:25:04.203 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:03 smithi018 ceph-mon[184144]: failed to encode map e166 with expected crc 2023-09-23T20:25:04.203 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:03 smithi018 ceph-mon[184144]: failed to encode map e166 with expected crc 2023-09-23T20:25:04.204 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:03 smithi018 ceph-mon[184144]: failed to encode map e166 with expected crc 2023-09-23T20:25:04.204 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:03 smithi018 ceph-mon[184144]: failed to encode map e166 with expected crc 2023-09-23T20:25:04.204 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:03 smithi018 ceph-mon[184144]: failed to encode map e166 with expected crc 2023-09-23T20:25:04.204 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:03 smithi018 ceph-mon[184144]: failed to encode map e166 with expected crc 2023-09-23T20:25:04.204 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:03 smithi018 ceph-mon[184144]: failed to encode map e166 with expected crc 2023-09-23T20:25:04.204 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:03 smithi018 ceph-mon[184144]: failed to encode map e166 with expected crc 2023-09-23T20:25:04.204 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:03 smithi018 ceph-mon[184144]: failed to encode map e166 with expected crc 2023-09-23T20:25:04.204 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:03 smithi018 ceph-mon[184144]: failed to encode map e166 with expected crc 2023-09-23T20:25:04.204 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:03 smithi018 ceph-mon[184144]: failed to encode map e166 with expected crc 2023-09-23T20:25:04.204 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:03 smithi018 ceph-mon[184144]: failed to encode map e166 with expected crc 2023-09-23T20:25:04.204 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:03 smithi018 ceph-mon[184144]: failed to encode map e166 with expected crc 2023-09-23T20:25:04.204 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:03 smithi018 ceph-mon[184144]: failed to encode map e166 with expected crc 2023-09-23T20:25:04.205 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:03 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2566469059' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:25:04.205 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:03 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:25:04.205 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:03 smithi018 ceph-mon[184144]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:25:04.205 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:03 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:25:04.205 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:03 smithi018 ceph-mon[184144]: osdmap e167: 8 total, 8 up, 7 in 2023-09-23T20:25:04.205 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:03 smithi149 ceph-mon[170394]: failed to encode map e166 with expected crc 2023-09-23T20:25:04.205 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:03 smithi149 ceph-mon[170394]: failed to encode map e166 with expected crc 2023-09-23T20:25:04.205 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:03 smithi149 ceph-mon[170394]: osdmap e166: 8 total, 8 up, 7 in 2023-09-23T20:25:04.205 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:03 smithi149 ceph-mon[170394]: failed to encode map e166 with expected crc 2023-09-23T20:25:04.206 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:03 smithi149 ceph-mon[170394]: failed to encode map e166 with expected crc 2023-09-23T20:25:04.206 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:03 smithi149 ceph-mon[170394]: failed to encode map e166 with expected crc 2023-09-23T20:25:04.206 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:03 smithi149 ceph-mon[170394]: failed to encode map e166 with expected crc 2023-09-23T20:25:04.206 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:03 smithi149 ceph-mon[170394]: failed to encode map e166 with expected crc 2023-09-23T20:25:04.206 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:03 smithi149 ceph-mon[170394]: failed to encode map e166 with expected crc 2023-09-23T20:25:04.206 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:03 smithi149 ceph-mon[170394]: failed to encode map e166 with expected crc 2023-09-23T20:25:04.206 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:03 smithi149 ceph-mon[170394]: failed to encode map e166 with expected crc 2023-09-23T20:25:04.206 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:03 smithi149 ceph-mon[170394]: failed to encode map e166 with expected crc 2023-09-23T20:25:04.206 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:03 smithi149 ceph-mon[170394]: failed to encode map e166 with expected crc 2023-09-23T20:25:04.206 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:03 smithi149 ceph-mon[170394]: failed to encode map e166 with expected crc 2023-09-23T20:25:04.206 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:03 smithi149 ceph-mon[170394]: failed to encode map e166 with expected crc 2023-09-23T20:25:04.206 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:03 smithi149 ceph-mon[170394]: failed to encode map e166 with expected crc 2023-09-23T20:25:04.207 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:03 smithi149 ceph-mon[170394]: failed to encode map e166 with expected crc 2023-09-23T20:25:04.207 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:03 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2566469059' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:25:04.207 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:03 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:25:04.207 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:03 smithi149 ceph-mon[170394]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:25:04.207 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:03 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:25:04.207 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:03 smithi149 ceph-mon[170394]: osdmap e167: 8 total, 8 up, 7 in 2023-09-23T20:25:04.207 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:03 smithi018 ceph-mon[179647]: failed to encode map e166 with expected crc 2023-09-23T20:25:04.207 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:03 smithi018 ceph-mon[179647]: failed to encode map e166 with expected crc 2023-09-23T20:25:04.207 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:03 smithi018 ceph-mon[179647]: osdmap e166: 8 total, 8 up, 7 in 2023-09-23T20:25:04.207 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:03 smithi018 ceph-mon[179647]: failed to encode map e166 with expected crc 2023-09-23T20:25:04.208 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:03 smithi018 ceph-mon[179647]: failed to encode map e166 with expected crc 2023-09-23T20:25:04.208 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:03 smithi018 ceph-mon[179647]: failed to encode map e166 with expected crc 2023-09-23T20:25:04.208 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:03 smithi018 ceph-mon[179647]: failed to encode map e166 with expected crc 2023-09-23T20:25:04.208 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:03 smithi018 ceph-mon[179647]: failed to encode map e166 with expected crc 2023-09-23T20:25:04.208 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:03 smithi018 ceph-mon[179647]: failed to encode map e166 with expected crc 2023-09-23T20:25:04.208 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:03 smithi018 ceph-mon[179647]: failed to encode map e166 with expected crc 2023-09-23T20:25:04.208 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:03 smithi018 ceph-mon[179647]: failed to encode map e166 with expected crc 2023-09-23T20:25:04.208 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:03 smithi018 ceph-mon[179647]: failed to encode map e166 with expected crc 2023-09-23T20:25:04.208 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:03 smithi018 ceph-mon[179647]: failed to encode map e166 with expected crc 2023-09-23T20:25:04.208 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:03 smithi018 ceph-mon[179647]: failed to encode map e166 with expected crc 2023-09-23T20:25:04.208 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:03 smithi018 ceph-mon[179647]: failed to encode map e166 with expected crc 2023-09-23T20:25:04.209 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:03 smithi018 ceph-mon[179647]: failed to encode map e166 with expected crc 2023-09-23T20:25:04.209 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:03 smithi018 ceph-mon[179647]: failed to encode map e166 with expected crc 2023-09-23T20:25:04.209 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:03 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2566469059' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:25:04.209 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:03 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:25:04.209 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:03 smithi018 ceph-mon[179647]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:25:04.209 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:03 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:25:04.209 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:03 smithi018 ceph-mon[179647]: osdmap e167: 8 total, 8 up, 7 in 2023-09-23T20:25:04.209 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:04.210 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:25:04.311 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:25:04.369 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:04.494 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:25:04.502 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:04.502 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:25:04.603 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:25:04.645 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:04.832 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T20:25:04.858 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:25:04.861 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:04.861 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:25:04.962 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:25:05.031 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:05.207 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:04 smithi018 ceph-mon[179647]: pgmap v351: 99 pgs: 1 active+recovery_wait+degraded, 1 peering, 81 active+clean, 1 active+recovering, 2 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 448 op/s; 54288/264711 objects degraded (20.508%); 5838/264711 objects misplaced (2.205%); 8.7 MiB/s, 139 objects/s recovering 2023-09-23T20:25:05.207 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:04 smithi018 ceph-mon[179647]: failed to encode map e167 with expected crc 2023-09-23T20:25:05.207 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:04 smithi018 ceph-mon[179647]: failed to encode map e167 with expected crc 2023-09-23T20:25:05.207 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:04 smithi018 ceph-mon[179647]: failed to encode map e167 with expected crc 2023-09-23T20:25:05.208 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:04 smithi018 ceph-mon[179647]: failed to encode map e167 with expected crc 2023-09-23T20:25:05.208 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:04 smithi018 ceph-mon[179647]: failed to encode map e167 with expected crc 2023-09-23T20:25:05.208 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:04 smithi018 ceph-mon[179647]: 5.17 starting backfill to osd.3 from (0'0,155'4470] MIN to 164'4829 2023-09-23T20:25:05.208 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:25:04 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:25:04] "GET /metrics HTTP/1.1" 200 36513 "" "Prometheus/2.43.0" 2023-09-23T20:25:05.208 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:04 smithi018 ceph-mon[184144]: pgmap v351: 99 pgs: 1 active+recovery_wait+degraded, 1 peering, 81 active+clean, 1 active+recovering, 2 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 448 op/s; 54288/264711 objects degraded (20.508%); 5838/264711 objects misplaced (2.205%); 8.7 MiB/s, 139 objects/s recovering 2023-09-23T20:25:05.208 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:04 smithi018 ceph-mon[184144]: failed to encode map e167 with expected crc 2023-09-23T20:25:05.208 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:04 smithi018 ceph-mon[184144]: failed to encode map e167 with expected crc 2023-09-23T20:25:05.209 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:04 smithi018 ceph-mon[184144]: failed to encode map e167 with expected crc 2023-09-23T20:25:05.209 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:04 smithi018 ceph-mon[184144]: failed to encode map e167 with expected crc 2023-09-23T20:25:05.209 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:04 smithi018 ceph-mon[184144]: failed to encode map e167 with expected crc 2023-09-23T20:25:05.209 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:04 smithi018 ceph-mon[184144]: 5.17 starting backfill to osd.3 from (0'0,155'4470] MIN to 164'4829 2023-09-23T20:25:05.209 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:25:05.211 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:05.212 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:25:05.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:04 smithi149 ceph-mon[170394]: pgmap v351: 99 pgs: 1 active+recovery_wait+degraded, 1 peering, 81 active+clean, 1 active+recovering, 2 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 448 op/s; 54288/264711 objects degraded (20.508%); 5838/264711 objects misplaced (2.205%); 8.7 MiB/s, 139 objects/s recovering 2023-09-23T20:25:05.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:04 smithi149 ceph-mon[170394]: failed to encode map e167 with expected crc 2023-09-23T20:25:05.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:04 smithi149 ceph-mon[170394]: failed to encode map e167 with expected crc 2023-09-23T20:25:05.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:04 smithi149 ceph-mon[170394]: failed to encode map e167 with expected crc 2023-09-23T20:25:05.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:04 smithi149 ceph-mon[170394]: failed to encode map e167 with expected crc 2023-09-23T20:25:05.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:04 smithi149 ceph-mon[170394]: failed to encode map e167 with expected crc 2023-09-23T20:25:05.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:04 smithi149 ceph-mon[170394]: 5.17 starting backfill to osd.3 from (0'0,155'4470] MIN to 164'4829 2023-09-23T20:25:05.312 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:25:05.365 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:05.444 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:25:05.453 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:05.453 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:25:05.554 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:25:05.609 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:05.672 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:25:05.682 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:05.683 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:25:05.783 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:25:05.806 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:25:05.823 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:25:05.838 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:05.859 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:25:05.889 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:25:05.914 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:25:05.919 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:25:05.923 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:05.923 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:25:05.958 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:25:06.000 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:25:06.024 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:25:06.046 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:25:06.091 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:25:06.096 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:25:06.096 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:25:06.096 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:25:06.097 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:25:06.097 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:25:06.097 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:25:06.097 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:25:06.097 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:25:06.101 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:06.175 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:25:06.191 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:06.191 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:25:06.196 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:05 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2959231739' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:25:06.196 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:05 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 54288/264711 objects degraded (20.508%), 16 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T20:25:06.293 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:25:06.297 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:05 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2959231739' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:25:06.297 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:05 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 54288/264711 objects degraded (20.508%), 16 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T20:25:06.298 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:05 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2959231739' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:25:06.298 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:05 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 54288/264711 objects degraded (20.508%), 16 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T20:25:06.358 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T20:25:06.384 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:06.466 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:25:06.476 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:06.476 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:25:06.577 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:25:06.644 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:06.718 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:25:06.725 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:06.726 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:25:06.827 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:25:06.883 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:06.950 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:25:06.963 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:06.964 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:25:07.064 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:25:07.133 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:07.201 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:25:07.211 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:07.212 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:25:07.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:06 smithi149 ceph-mon[170394]: pgmap v353: 99 pgs: 1 active+recovery_wait+degraded, 1 peering, 80 active+clean, 1 active+remapped+backfill_wait, 1 active+recovering, 2 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped; 5.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 813 op/s; 29453/258657 objects degraded (11.387%); 9157/258657 objects misplaced (3.540%); 18 MiB/s, 295 objects/s recovering 2023-09-23T20:25:07.312 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:25:07.322 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:25:07.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:06 smithi018 ceph-mon[179647]: pgmap v353: 99 pgs: 1 active+recovery_wait+degraded, 1 peering, 80 active+clean, 1 active+remapped+backfill_wait, 1 active+recovering, 2 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped; 5.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 813 op/s; 29453/258657 objects degraded (11.387%); 9157/258657 objects misplaced (3.540%); 18 MiB/s, 295 objects/s recovering 2023-09-23T20:25:07.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:06 smithi018 ceph-mon[184144]: pgmap v353: 99 pgs: 1 active+recovery_wait+degraded, 1 peering, 80 active+clean, 1 active+remapped+backfill_wait, 1 active+recovering, 2 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped; 5.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 813 op/s; 29453/258657 objects degraded (11.387%); 9157/258657 objects misplaced (3.540%); 18 MiB/s, 295 objects/s recovering 2023-09-23T20:25:07.394 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:07.445 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:25:07.450 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:07.450 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:25:07.550 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:25:07.613 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:07.677 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:25:07.687 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:07.688 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:25:07.754 INFO:teuthology.orchestra.run.smithi018.stdout:ERROR: (22) Invalid argument 2023-09-23T20:25:07.754 INFO:teuthology.orchestra.run.smithi018.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:25:07.789 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:25:07.864 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:07.931 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:25:07.942 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:07.942 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:25:08.043 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:25:08.112 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:25:07 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[153650]: logger=sqlstore.transactions t=2023-09-23T20:25:07.924549115Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2023-09-23T20:25:08.113 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:25:07 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[153650]: logger=cleanup t=2023-09-23T20:25:07.926584973Z level=info msg="Completed cleanup jobs" duration=12.456744ms 2023-09-23T20:25:08.121 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:08.185 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:25:08.191 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:08.192 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:25:08.292 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:25:08.363 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:08.429 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:25:08.442 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:08.442 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:25:08.543 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:25:08.599 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:08.606 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:25:08.621 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 4, 5, 6, 7, 3] out_osds: [2] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-23T20:25:08.621 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:25:08.622 INFO:tasks.thrashosds.thrasher:Growing pool .mgr 2023-09-23T20:25:08.622 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:25:08.984 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:08 smithi018 ceph-mon[179647]: pgmap v354: 99 pgs: 1 active+recovery_wait+degraded, 1 peering, 80 active+clean, 1 active+remapped+backfill_wait, 1 active+recovering, 2 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped; 5.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 336 op/s; 29453/258657 objects degraded (11.387%); 9157/258657 objects misplaced (3.540%); 9.2 MiB/s, 147 objects/s recovering 2023-09-23T20:25:08.984 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:08 smithi018 ceph-mon[184144]: pgmap v354: 99 pgs: 1 active+recovery_wait+degraded, 1 peering, 80 active+clean, 1 active+remapped+backfill_wait, 1 active+recovering, 2 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped; 5.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 336 op/s; 29453/258657 objects degraded (11.387%); 9157/258657 objects misplaced (3.540%); 9.2 MiB/s, 147 objects/s recovering 2023-09-23T20:25:09.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:08 smithi149 ceph-mon[170394]: pgmap v354: 99 pgs: 1 active+recovery_wait+degraded, 1 peering, 80 active+clean, 1 active+remapped+backfill_wait, 1 active+recovering, 2 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped; 5.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 336 op/s; 29453/258657 objects degraded (11.387%); 9157/258657 objects misplaced (3.540%); 9.2 MiB/s, 147 objects/s recovering 2023-09-23T20:25:10.171 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T20:25:10.230 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:25:10.240 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:10.240 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:25:10.341 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:25:10.397 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:10.480 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:25:10.484 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:25:10.498 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:10.498 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:25:10.579 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:25:10.599 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:25:10.687 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:10.840 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:25:10.846 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:10.847 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:25:10.947 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:25:11.011 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:11.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:11 smithi018 ceph-mon[179647]: pgmap v355: 99 pgs: 1 active+recovery_wait+degraded, 82 active+clean, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped; 5.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 265 op/s; 29468/258657 objects degraded (11.393%); 9156/258657 objects misplaced (3.540%); 7.2 MiB/s, 116 objects/s recovering 2023-09-23T20:25:11.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:11 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 29468/258657 objects degraded (11.393%), 16 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T20:25:11.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:11 smithi018 ceph-mon[184144]: pgmap v355: 99 pgs: 1 active+recovery_wait+degraded, 82 active+clean, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped; 5.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 265 op/s; 29468/258657 objects degraded (11.393%); 9156/258657 objects misplaced (3.540%); 7.2 MiB/s, 116 objects/s recovering 2023-09-23T20:25:11.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:11 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 29468/258657 objects degraded (11.393%), 16 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T20:25:11.470 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:11 smithi149 ceph-mon[170394]: pgmap v355: 99 pgs: 1 active+recovery_wait+degraded, 82 active+clean, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped; 5.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 265 op/s; 29468/258657 objects degraded (11.393%); 9156/258657 objects misplaced (3.540%); 7.2 MiB/s, 116 objects/s recovering 2023-09-23T20:25:11.470 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:11 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 29468/258657 objects degraded (11.393%), 16 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T20:25:11.534 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:25:11.538 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:11.538 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:25:11.639 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:25:11.683 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:25:11.738 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:11.818 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:25:11.824 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:11.825 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:25:11.926 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:25:11.977 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:12.005 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:12 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3882137377' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:25:12.033 INFO:teuthology.orchestra.run.smithi018.stderr:nodeep-scrub is set 2023-09-23T20:25:12.048 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:25:12.052 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:12.052 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:25:12.153 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:25:12.227 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:12.277 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:12 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3882137377' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:25:12.278 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:12 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3882137377' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:25:12.312 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:25:12.324 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:12.324 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:25:12.425 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:25:12.503 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:12.574 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:25:12.579 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:12.580 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:25:12.681 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:25:12.742 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:12.858 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:25:12.865 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:12.865 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:25:12.966 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:25:13.038 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:13.141 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:25:13.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:13 smithi018 ceph-mon[179647]: pgmap v356: 99 pgs: 1 active+recovery_wait+degraded, 82 active+clean, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped; 5.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 641 op/s; 29290/248166 objects degraded (11.803%); 8057/248166 objects misplaced (3.247%); 12 MiB/s, 191 objects/s recovering 2023-09-23T20:25:13.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:13 smithi018 ceph-mon[179647]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:25:13.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:13 smithi018 ceph-mon[179647]: failed to encode map e168 with expected crc 2023-09-23T20:25:13.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:13 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3882137377' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:25:13.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:13 smithi018 ceph-mon[179647]: osdmap e168: 8 total, 8 up, 7 in 2023-09-23T20:25:13.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:13 smithi018 ceph-mon[179647]: failed to encode map e168 with expected crc 2023-09-23T20:25:13.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:13 smithi018 ceph-mon[179647]: failed to encode map e168 with expected crc 2023-09-23T20:25:13.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:13 smithi018 ceph-mon[179647]: failed to encode map e168 with expected crc 2023-09-23T20:25:13.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:13 smithi018 ceph-mon[179647]: failed to encode map e168 with expected crc 2023-09-23T20:25:13.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:13 smithi018 ceph-mon[184144]: pgmap v356: 99 pgs: 1 active+recovery_wait+degraded, 82 active+clean, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped; 5.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 641 op/s; 29290/248166 objects degraded (11.803%); 8057/248166 objects misplaced (3.247%); 12 MiB/s, 191 objects/s recovering 2023-09-23T20:25:13.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:13 smithi018 ceph-mon[184144]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:25:13.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:13 smithi018 ceph-mon[184144]: failed to encode map e168 with expected crc 2023-09-23T20:25:13.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:13 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3882137377' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:25:13.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:13 smithi018 ceph-mon[184144]: osdmap e168: 8 total, 8 up, 7 in 2023-09-23T20:25:13.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:13 smithi018 ceph-mon[184144]: failed to encode map e168 with expected crc 2023-09-23T20:25:13.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:13 smithi018 ceph-mon[184144]: failed to encode map e168 with expected crc 2023-09-23T20:25:13.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:13 smithi018 ceph-mon[184144]: failed to encode map e168 with expected crc 2023-09-23T20:25:13.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:13 smithi018 ceph-mon[184144]: failed to encode map e168 with expected crc 2023-09-23T20:25:13.149 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:13.149 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:25:13.231 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:25:13.234 INFO:teuthology.orchestra.run.smithi018.stderr:dumped all 2023-09-23T20:25:13.250 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:25:13.330 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:13 smithi149 ceph-mon[170394]: pgmap v356: 99 pgs: 1 active+recovery_wait+degraded, 82 active+clean, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped; 5.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 641 op/s; 29290/248166 objects degraded (11.803%); 8057/248166 objects misplaced (3.247%); 12 MiB/s, 191 objects/s recovering 2023-09-23T20:25:13.330 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:13 smithi149 ceph-mon[170394]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:25:13.330 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:13 smithi149 ceph-mon[170394]: failed to encode map e168 with expected crc 2023-09-23T20:25:13.330 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:13 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3882137377' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:25:13.330 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:13 smithi149 ceph-mon[170394]: osdmap e168: 8 total, 8 up, 7 in 2023-09-23T20:25:13.331 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:13 smithi149 ceph-mon[170394]: failed to encode map e168 with expected crc 2023-09-23T20:25:13.331 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:13 smithi149 ceph-mon[170394]: failed to encode map e168 with expected crc 2023-09-23T20:25:13.331 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:13 smithi149 ceph-mon[170394]: failed to encode map e168 with expected crc 2023-09-23T20:25:13.331 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:13 smithi149 ceph-mon[170394]: failed to encode map e168 with expected crc 2023-09-23T20:25:13.334 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:13.400 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:25:13.408 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:25:13.412 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:13.413 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:25:13.434 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:25:13.466 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:25:13.493 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:25:13.513 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:25:13.560 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:13.566 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:25:13.602 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:25:13.616 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:25:13.627 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:13.628 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:25:13.635 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:25:13.661 INFO:teuthology.orchestra.run.smithi018.stdout:{"pg_ready":true,"pg_map":{"version":357,"stamp":"2023-09-23T20:25:12.020191+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5424158667,"num_objects":82722,"num_object_clones":0,"num_object_copies":248166,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":29290,"num_objects_misplaced":8057,"num_objects_unfound":0,"num_objects_dirty":82722,"num_whiteouts":0,"num_read":6496,"num_read_kb":14137,"num_write":116930,"num_write_kb":5865958,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":15176,"num_bytes_recovered":992741728,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139077,"ondisk_log_size":139077,"up":297,"acting":282,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":332,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":15878984,"kb_used_data":15213968,"kb_used_omap":18,"kb_used_meta":664557,"kb_avail":640337080,"statfs":{"total":671965249536,"available":655705169920,"internally_reserved":0,"allocated":15579103232,"data_stored":15574443262,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":19385,"internal_metadata":680506439},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-361431040,"num_objects":-5515,"num_object_clones":0,"num_object_copies":-16545,"num_objects_missing_on_primary":-2,"num_objects_missing":-2,"num_objects_degraded":-24998,"num_objects_misplaced":2219,"num_objects_unfound":0,"num_objects_dirty":-5515,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5515,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1646,"num_bytes_recovered":105513050,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.527900"},"pg_stats":[{"pgid":"3.1a","version":"69'1","reported_seq":154,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-23T20:25:03.095521+0000","last_change":"2023-09-23T20:24:53.087034+0000","last_active":"2023-09-23T20:25:03.095521+0000","last_peered":"2023-09-23T20:25:03.095521+0000","last_clean":"2023-09-23T20:25:03.095521+0000","last_became_active":"2023-09-23T20:24:53.086545+0000","last_became_peered":"2023-09-23T20:24:53.086545+0000","last_unstale":"2023-09-23T20:25:03.095521+0000","last_undegraded":"2023-09-23T20:25:03.095521+0000","last_fullsized":"2023-09-23T20:25:03.095521+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:45:19.147638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":12,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-23T20:25:02.999297+0000","last_change":"2023-09-23T20:25:02.932223+0000","last_active":"2023-09-23T20:25:02.999297+0000","last_peered":"2023-09-23T20:25:02.999297+0000","last_clean":"2023-09-23T20:25:02.999297+0000","last_became_active":"2023-09-23T20:25:02.931024+0000","last_became_peered":"2023-09-23T20:25:02.931024+0000","last_unstale":"2023-09-23T20:25:02.999297+0000","last_undegraded":"2023-09-23T20:25:02.999297+0000","last_fullsized":"2023-09-23T20:25:02.999297+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:16:00.586211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":123,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-23T20:25:02.942597+0000","last_change":"2023-09-23T20:25:02.934543+0000","last_active":"2023-09-23T20:25:02.942597+0000","last_peered":"2023-09-23T20:25:02.942597+0000","last_clean":"2023-09-23T20:25:02.942597+0000","last_became_active":"2023-09-23T20:25:02.934178+0000","last_became_peered":"2023-09-23T20:25:02.934178+0000","last_unstale":"2023-09-23T20:25:02.942597+0000","last_undegraded":"2023-09-23T20:25:02.942597+0000","last_fullsized":"2023-09-23T20:25:02.942597+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:27:38.453052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":129,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-23T20:25:02.961017+0000","last_change":"2023-09-23T20:25:02.957185+0000","last_active":"2023-09-23T20:25:02.961017+0000","last_peered":"2023-09-23T20:25:02.961017+0000","last_clean":"2023-09-23T20:25:02.961017+0000","last_became_active":"2023-09-23T20:25:02.956689+0000","last_became_peered":"2023-09-23T20:25:02.956689+0000","last_unstale":"2023-09-23T20:25:02.961017+0000","last_undegraded":"2023-09-23T20:25:02.961017+0000","last_fullsized":"2023-09-23T20:25:02.961017+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:04:28.524015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":12,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-23T20:25:02.999062+0000","last_change":"2023-09-23T20:25:02.932054+0000","last_active":"2023-09-23T20:25:02.999062+0000","last_peered":"2023-09-23T20:25:02.999062+0000","last_clean":"2023-09-23T20:25:02.999062+0000","last_became_active":"2023-09-23T20:25:02.931309+0000","last_became_peered":"2023-09-23T20:25:02.931309+0000","last_unstale":"2023-09-23T20:25:02.999062+0000","last_undegraded":"2023-09-23T20:25:02.999062+0000","last_fullsized":"2023-09-23T20:25:02.999062+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:57:53.378499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":126,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-23T20:25:02.925622+0000","last_change":"2023-09-23T20:20:46.937089+0000","last_active":"2023-09-23T20:25:02.925622+0000","last_peered":"2023-09-23T20:25:02.925622+0000","last_clean":"2023-09-23T20:25:02.925622+0000","last_became_active":"2023-09-23T20:20:46.936831+0000","last_became_peered":"2023-09-23T20:20:46.936831+0000","last_unstale":"2023-09-23T20:25:02.925622+0000","last_undegraded":"2023-09-23T20:25:02.925622+0000","last_fullsized":"2023-09-23T20:25:02.925622+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:34:56.747010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.11","version":"167'5138","reported_seq":7844,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T20:25:10.366078+0000","last_change":"2023-09-23T20:23:31.161216+0000","last_active":"2023-09-23T20:25:10.366078+0000","last_peered":"2023-09-23T20:25:10.366078+0000","last_clean":"2023-09-23T20:25:10.366078+0000","last_became_active":"2023-09-23T20:23:31.146532+0000","last_became_peered":"2023-09-23T20:23:31.146532+0000","last_unstale":"2023-09-23T20:25:10.366078+0000","last_undegraded":"2023-09-23T20:25:10.366078+0000","last_fullsized":"2023-09-23T20:25:10.366078+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":118,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":5,"last_scrub":"119'1977","last_scrub_stamp":"2023-09-23T20:23:12.353414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:23:12.353414+0000","objects_scrubbed":1018,"log_size":5138,"log_dups_size":0,"ondisk_log_size":5138,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:28:29.630274+0000","scrub_duration":0.26235630100000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":171638784,"num_objects":2619,"num_object_clones":0,"num_object_copies":7857,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2619,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3171,"num_write_kb":152704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1282,"num_bytes_recovered":84017152,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":128,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T20:25:03.919518+0000","last_change":"2023-09-23T20:25:02.969432+0000","last_active":"2023-09-23T20:25:03.919518+0000","last_peered":"2023-09-23T20:25:03.919518+0000","last_clean":"2023-09-23T20:25:03.919518+0000","last_became_active":"2023-09-23T20:25:02.969118+0000","last_became_peered":"2023-09-23T20:25:02.969118+0000","last_unstale":"2023-09-23T20:25:03.919518+0000","last_undegraded":"2023-09-23T20:25:03.919518+0000","last_fullsized":"2023-09-23T20:25:03.919518+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:02:22.453831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":12,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-23T20:25:02.999046+0000","last_change":"2023-09-23T20:25:02.932225+0000","last_active":"2023-09-23T20:25:02.999046+0000","last_peered":"2023-09-23T20:25:02.999046+0000","last_clean":"2023-09-23T20:25:02.999046+0000","last_became_active":"2023-09-23T20:25:02.931746+0000","last_became_peered":"2023-09-23T20:25:02.931746+0000","last_unstale":"2023-09-23T20:25:02.999046+0000","last_undegraded":"2023-09-23T20:25:02.999046+0000","last_fullsized":"2023-09-23T20:25:02.999046+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:57:24.470179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.10","version":"167'5039","reported_seq":8009,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T20:25:10.250582+0000","last_change":"2023-09-23T20:23:24.880162+0000","last_active":"2023-09-23T20:25:10.250582+0000","last_peered":"2023-09-23T20:25:10.250582+0000","last_clean":"2023-09-23T20:25:10.250582+0000","last_became_active":"2023-09-23T20:23:24.867240+0000","last_became_peered":"2023-09-23T20:23:24.867240+0000","last_unstale":"2023-09-23T20:25:10.250582+0000","last_undegraded":"2023-09-23T20:25:10.250582+0000","last_fullsized":"2023-09-23T20:25:10.250582+0000","mapping_epoch":128,"log_start":"0'0","ondisk_log_start":"0'0","created":118,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":5,"last_scrub":"119'2122","last_scrub_stamp":"2023-09-23T20:23:15.108101+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:23:15.108101+0000","objects_scrubbed":1199,"log_size":5039,"log_dups_size":0,"ondisk_log_size":5039,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:10:57.070009+0000","scrub_duration":0.30476455699999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":169082880,"num_objects":2580,"num_object_clones":0,"num_object_copies":7740,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2580,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2921,"num_write_kb":137664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1255,"num_bytes_recovered":82247680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":126,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-23T20:25:02.958395+0000","last_change":"2023-09-23T20:20:46.950281+0000","last_active":"2023-09-23T20:25:02.958395+0000","last_peered":"2023-09-23T20:25:02.958395+0000","last_clean":"2023-09-23T20:25:02.958395+0000","last_became_active":"2023-09-23T20:20:46.949892+0000","last_became_peered":"2023-09-23T20:20:46.949892+0000","last_unstale":"2023-09-23T20:25:02.958395+0000","last_undegraded":"2023-09-23T20:25:02.958395+0000","last_fullsized":"2023-09-23T20:25:02.958395+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:28.763832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":12,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-23T20:25:02.999121+0000","last_change":"2023-09-23T20:25:02.947669+0000","last_active":"2023-09-23T20:25:02.999121+0000","last_peered":"2023-09-23T20:25:02.999121+0000","last_clean":"2023-09-23T20:25:02.999121+0000","last_became_active":"2023-09-23T20:25:02.947368+0000","last_became_peered":"2023-09-23T20:25:02.947368+0000","last_unstale":"2023-09-23T20:25:02.999121+0000","last_undegraded":"2023-09-23T20:25:02.999121+0000","last_fullsized":"2023-09-23T20:25:02.999121+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:52:53.813761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.13","version":"167'5045","reported_seq":9088,"reported_epoch":167,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:25:09.699982+0000","last_change":"2023-09-23T20:25:02.952793+0000","last_active":"2023-09-23T20:25:09.699982+0000","last_peered":"2023-09-23T20:25:09.699982+0000","last_clean":"2023-09-23T20:24:36.774911+0000","last_became_active":"2023-09-23T20:25:02.945089+0000","last_became_peered":"2023-09-23T20:25:02.945089+0000","last_unstale":"2023-09-23T20:25:09.699982+0000","last_undegraded":"2023-09-23T20:25:02.911244+0000","last_fullsized":"2023-09-23T20:25:02.906341+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":118,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:22:51.846999+0000","objects_scrubbed":0,"log_size":5045,"log_dups_size":0,"ondisk_log_size":5045,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":170557440,"num_objects":2603,"num_object_clones":0,"num_object_copies":7809,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":360,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2603,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4143,"num_write_kb":215840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1311,"num_bytes_recovered":85852160,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[0,1],"avail_no_missing":["0","1"],"object_location_counts":[{"shards":"0,1","objects":2603}],"blocked_by":[],"up_primary":3,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":162,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-23T20:25:03.048498+0000","last_change":"2023-09-23T20:25:03.038511+0000","last_active":"2023-09-23T20:25:03.048498+0000","last_peered":"2023-09-23T20:25:03.048498+0000","last_clean":"2023-09-23T20:25:03.048498+0000","last_became_active":"2023-09-23T20:25:02.976811+0000","last_became_peered":"2023-09-23T20:25:02.976811+0000","last_unstale":"2023-09-23T20:25:03.048498+0000","last_undegraded":"2023-09-23T20:25:03.048498+0000","last_fullsized":"2023-09-23T20:25:03.048498+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:28:06.605731+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":115,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-23T20:25:02.905499+0000","last_change":"2023-09-23T20:24:53.090001+0000","last_active":"2023-09-23T20:25:02.905499+0000","last_peered":"2023-09-23T20:25:02.905499+0000","last_clean":"2023-09-23T20:25:02.905499+0000","last_became_active":"2023-09-23T20:24:53.089655+0000","last_became_peered":"2023-09-23T20:24:53.089655+0000","last_unstale":"2023-09-23T20:25:02.905499+0000","last_undegraded":"2023-09-23T20:25:02.905499+0000","last_fullsized":"2023-09-23T20:25:02.905499+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:48:51.700559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.12","version":"167'4953","reported_seq":8275,"reported_epoch":167,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:25:09.946168+0000","last_change":"2023-09-23T20:25:02.998320+0000","last_active":"2023-09-23T20:25:09.946168+0000","last_peered":"2023-09-23T20:25:09.946168+0000","last_clean":"2023-09-23T20:24:36.852780+0000","last_became_active":"2023-09-23T20:25:02.989287+0000","last_became_peered":"2023-09-23T20:25:02.989287+0000","last_unstale":"2023-09-23T20:25:09.946168+0000","last_undegraded":"2023-09-23T20:25:02.916498+0000","last_fullsized":"2023-09-23T20:25:02.905610+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":118,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":5,"last_scrub":"119'2087","last_scrub_stamp":"2023-09-23T20:23:15.737284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:23:15.737284+0000","objects_scrubbed":1166,"log_size":4953,"log_dups_size":0,"ondisk_log_size":4953,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:42:16.611213+0000","scrub_duration":0.27092271000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":164298752,"num_objects":2507,"num_object_clones":0,"num_object_copies":7521,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":366,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2507,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2879,"num_write_kb":135040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1224,"num_bytes_recovered":80216064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5],"avail_no_missing":["1","5"],"object_location_counts":[{"shards":"1,5","objects":2507}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":131,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-23T20:25:02.922112+0000","last_change":"2023-09-23T20:12:43.750217+0000","last_active":"2023-09-23T20:25:02.922112+0000","last_peered":"2023-09-23T20:25:02.922112+0000","last_clean":"2023-09-23T20:25:02.922112+0000","last_became_active":"2023-09-23T20:12:43.749929+0000","last_became_peered":"2023-09-23T20:12:43.749929+0000","last_unstale":"2023-09-23T20:25:02.922112+0000","last_undegraded":"2023-09-23T20:25:02.922112+0000","last_fullsized":"2023-09-23T20:25:02.922112+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:50:28.470211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":125,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-23T20:25:02.917613+0000","last_change":"2023-09-23T20:20:46.939938+0000","last_active":"2023-09-23T20:25:02.917613+0000","last_peered":"2023-09-23T20:25:02.917613+0000","last_clean":"2023-09-23T20:25:02.917613+0000","last_became_active":"2023-09-23T20:20:46.938777+0000","last_became_peered":"2023-09-23T20:20:46.938777+0000","last_unstale":"2023-09-23T20:25:02.917613+0000","last_undegraded":"2023-09-23T20:25:02.917613+0000","last_fullsized":"2023-09-23T20:25:02.917613+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:45:10.329438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.15","version":"167'4905","reported_seq":8456,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T20:25:10.351384+0000","last_change":"2023-09-23T20:24:36.924738+0000","last_active":"2023-09-23T20:25:10.351384+0000","last_peered":"2023-09-23T20:25:10.351384+0000","last_clean":"2023-09-23T20:25:10.351384+0000","last_became_active":"2023-09-23T20:23:54.922070+0000","last_became_peered":"2023-09-23T20:23:54.922070+0000","last_unstale":"2023-09-23T20:25:10.351384+0000","last_undegraded":"2023-09-23T20:25:10.351384+0000","last_fullsized":"2023-09-23T20:25:10.351384+0000","mapping_epoch":143,"log_start":"0'0","ondisk_log_start":"0'0","created":118,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:22:51.846999+0000","objects_scrubbed":0,"log_size":4905,"log_dups_size":0,"ondisk_log_size":4905,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":168820736,"num_objects":2576,"num_object_clones":0,"num_object_copies":7728,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2576,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4022,"num_write_kb":211136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1727,"num_bytes_recovered":113180672,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":145,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-23T20:25:02.935809+0000","last_change":"2023-09-23T20:24:53.095197+0000","last_active":"2023-09-23T20:25:02.935809+0000","last_peered":"2023-09-23T20:25:02.935809+0000","last_clean":"2023-09-23T20:25:02.935809+0000","last_became_active":"2023-09-23T20:24:53.089668+0000","last_became_peered":"2023-09-23T20:24:53.089668+0000","last_unstale":"2023-09-23T20:25:02.935809+0000","last_undegraded":"2023-09-23T20:25:02.935809+0000","last_fullsized":"2023-09-23T20:25:02.935809+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:40:12.888818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":12,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-23T20:25:02.999278+0000","last_change":"2023-09-23T20:25:02.962047+0000","last_active":"2023-09-23T20:25:02.999278+0000","last_peered":"2023-09-23T20:25:02.999278+0000","last_clean":"2023-09-23T20:25:02.999278+0000","last_became_active":"2023-09-23T20:25:02.961523+0000","last_became_peered":"2023-09-23T20:25:02.961523+0000","last_unstale":"2023-09-23T20:25:02.999278+0000","last_undegraded":"2023-09-23T20:25:02.999278+0000","last_fullsized":"2023-09-23T20:25:02.999278+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:18:11.830822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.14","version":"167'4939","reported_seq":9041,"reported_epoch":167,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:25:10.136972+0000","last_change":"2023-09-23T20:25:04.064961+0000","last_active":"2023-09-23T20:25:10.136972+0000","last_peered":"2023-09-23T20:25:10.136972+0000","last_clean":"2023-09-23T20:24:36.883442+0000","last_became_active":"2023-09-23T20:25:04.010012+0000","last_became_peered":"2023-09-23T20:25:04.010012+0000","last_unstale":"2023-09-23T20:25:10.136972+0000","last_undegraded":"2023-09-23T20:25:03.973999+0000","last_fullsized":"2023-09-23T20:25:03.909780+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":118,"last_epoch_clean":142,"parent":"0.0","parent_split_bits":5,"last_scrub":"142'3061","last_scrub_stamp":"2023-09-23T20:23:51.058360+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:23:51.058360+0000","objects_scrubbed":2161,"log_size":4939,"log_dups_size":0,"ondisk_log_size":4939,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:04:23.969154+0000","scrub_duration":0.44581189700000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":166526976,"num_objects":2541,"num_object_clones":0,"num_object_copies":7623,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3503,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2541,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1890,"num_write_kb":72640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":141,"num_bytes_recovered":9175040,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[7,3],"avail_no_missing":["7","2","5"],"object_location_counts":[{"shards":"2,3,5,7","objects":998},{"shards":"2,3,7","objects":1543},{"shards":"3,7","objects":-154}],"blocked_by":[],"up_primary":3,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":128,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-23T20:25:02.958489+0000","last_change":"2023-09-23T20:25:02.953993+0000","last_active":"2023-09-23T20:25:02.958489+0000","last_peered":"2023-09-23T20:25:02.958489+0000","last_clean":"2023-09-23T20:25:02.958489+0000","last_became_active":"2023-09-23T20:25:02.953672+0000","last_became_peered":"2023-09-23T20:25:02.953672+0000","last_unstale":"2023-09-23T20:25:02.958489+0000","last_undegraded":"2023-09-23T20:25:02.958489+0000","last_fullsized":"2023-09-23T20:25:02.958489+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:26:08.603577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":145,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-23T20:25:02.955571+0000","last_change":"2023-09-23T20:25:02.949584+0000","last_active":"2023-09-23T20:25:02.955571+0000","last_peered":"2023-09-23T20:25:02.955571+0000","last_clean":"2023-09-23T20:25:02.955571+0000","last_became_active":"2023-09-23T20:25:02.949132+0000","last_became_peered":"2023-09-23T20:25:02.949132+0000","last_unstale":"2023-09-23T20:25:02.955571+0000","last_undegraded":"2023-09-23T20:25:02.955571+0000","last_fullsized":"2023-09-23T20:25:02.955571+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:34:20.509823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.17","version":"167'4964","reported_seq":11190,"reported_epoch":167,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T20:25:09.806201+0000","last_change":"2023-09-23T20:25:03.985160+0000","last_active":"2023-09-23T20:25:09.806201+0000","last_peered":"2023-09-23T20:25:09.806201+0000","last_clean":"2023-09-23T20:25:01.825968+0000","last_became_active":"2023-09-23T20:25:03.975878+0000","last_became_peered":"2023-09-23T20:25:03.975878+0000","last_unstale":"2023-09-23T20:25:09.806201+0000","last_undegraded":"2023-09-23T20:25:09.806201+0000","last_fullsized":"2023-09-23T20:25:09.806201+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":118,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:22:51.846999+0000","objects_scrubbed":0,"log_size":4964,"log_dups_size":0,"ondisk_log_size":4964,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":166592512,"num_objects":2542,"num_object_clones":0,"num_object_copies":7626,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2542,"num_objects_unfound":0,"num_objects_dirty":2542,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4046,"num_write_kb":210816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2326,"num_bytes_recovered":152436736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[0,1,7],"avail_no_missing":["0","1","7"],"object_location_counts":[{"shards":"0,1,7","objects":2542}],"blocked_by":[],"up_primary":3,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":110,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-23T20:25:02.935126+0000","last_change":"2023-09-23T20:12:43.749139+0000","last_active":"2023-09-23T20:25:02.935126+0000","last_peered":"2023-09-23T20:25:02.935126+0000","last_clean":"2023-09-23T20:25:02.935126+0000","last_became_active":"2023-09-23T20:12:43.748854+0000","last_became_peered":"2023-09-23T20:12:43.748854+0000","last_unstale":"2023-09-23T20:25:02.935126+0000","last_undegraded":"2023-09-23T20:25:02.935126+0000","last_fullsized":"2023-09-23T20:25:02.935126+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:57:11.630712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":138,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T20:25:03.911566+0000","last_change":"2023-09-23T20:25:02.957010+0000","last_active":"2023-09-23T20:25:03.911566+0000","last_peered":"2023-09-23T20:25:03.911566+0000","last_clean":"2023-09-23T20:25:03.911566+0000","last_became_active":"2023-09-23T20:25:02.956267+0000","last_became_peered":"2023-09-23T20:25:02.956267+0000","last_unstale":"2023-09-23T20:25:03.911566+0000","last_undegraded":"2023-09-23T20:25:03.911566+0000","last_fullsized":"2023-09-23T20:25:03.911566+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:10:55.299422+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.16","version":"167'5087","reported_seq":11785,"reported_epoch":167,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:25:10.377425+0000","last_change":"2023-09-23T20:25:03.028250+0000","last_active":"2023-09-23T20:25:10.377425+0000","last_peered":"2023-09-23T20:25:10.377425+0000","last_clean":"2023-09-23T20:24:36.863809+0000","last_became_active":"2023-09-23T20:25:03.020723+0000","last_became_peered":"2023-09-23T20:25:03.020723+0000","last_unstale":"2023-09-23T20:25:10.377425+0000","last_undegraded":"2023-09-23T20:25:02.955253+0000","last_fullsized":"2023-09-23T20:25:02.922712+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":118,"last_epoch_clean":153,"parent":"0.0","parent_split_bits":5,"last_scrub":"119'2163","last_scrub_stamp":"2023-09-23T20:23:15.241911+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:23:15.241911+0000","objects_scrubbed":1192,"log_size":5087,"log_dups_size":0,"ondisk_log_size":5087,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:27:20.087257+0000","scrub_duration":0.33323710899999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":169803776,"num_objects":2591,"num_object_clones":0,"num_object_copies":7773,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":359,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2591,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2935,"num_write_kb":138688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2361,"num_bytes_recovered":154599424,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,1],"avail_no_missing":["5","1","2","7"],"object_location_counts":[{"shards":"1,5","objects":2591}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":135,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-23T20:25:02.924199+0000","last_change":"2023-09-23T20:20:14.881009+0000","last_active":"2023-09-23T20:25:02.924199+0000","last_peered":"2023-09-23T20:25:02.924199+0000","last_clean":"2023-09-23T20:25:02.924199+0000","last_became_active":"2023-09-23T20:20:14.880685+0000","last_became_peered":"2023-09-23T20:20:14.880685+0000","last_unstale":"2023-09-23T20:25:02.924199+0000","last_undegraded":"2023-09-23T20:25:02.924199+0000","last_fullsized":"2023-09-23T20:25:02.924199+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:28.010434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":12,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-23T20:25:02.999334+0000","last_change":"2023-09-23T20:25:02.977462+0000","last_active":"2023-09-23T20:25:02.999334+0000","last_peered":"2023-09-23T20:25:02.999334+0000","last_clean":"2023-09-23T20:25:02.999334+0000","last_became_active":"2023-09-23T20:25:02.969910+0000","last_became_peered":"2023-09-23T20:25:02.969910+0000","last_unstale":"2023-09-23T20:25:02.999334+0000","last_undegraded":"2023-09-23T20:25:02.999334+0000","last_fullsized":"2023-09-23T20:25:02.999334+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:22:55.644955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.9","version":"167'4889","reported_seq":4964,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T20:25:10.303211+0000","last_change":"2023-09-23T20:23:10.973039+0000","last_active":"2023-09-23T20:25:10.303211+0000","last_peered":"2023-09-23T20:25:10.303211+0000","last_clean":"2023-09-23T20:25:10.303211+0000","last_became_active":"2023-09-23T20:23:10.971930+0000","last_became_peered":"2023-09-23T20:23:10.971930+0000","last_unstale":"2023-09-23T20:25:10.303211+0000","last_undegraded":"2023-09-23T20:25:10.303211+0000","last_fullsized":"2023-09-23T20:25:10.303211+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":111,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:22:51.846999+0000","objects_scrubbed":0,"log_size":4889,"log_dups_size":0,"ondisk_log_size":4889,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":167018496,"num_objects":2548,"num_object_clones":0,"num_object_copies":7644,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2548,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4006,"num_write_kb":209760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":128,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-23T20:25:02.937299+0000","last_change":"2023-09-23T20:20:14.883283+0000","last_active":"2023-09-23T20:25:02.937299+0000","last_peered":"2023-09-23T20:25:02.937299+0000","last_clean":"2023-09-23T20:25:02.937299+0000","last_became_active":"2023-09-23T20:20:14.882806+0000","last_became_peered":"2023-09-23T20:20:14.882806+0000","last_unstale":"2023-09-23T20:25:02.937299+0000","last_undegraded":"2023-09-23T20:25:02.937299+0000","last_fullsized":"2023-09-23T20:25:02.937299+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:23:35.295701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":108,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-23T20:25:02.918741+0000","last_change":"2023-09-23T20:12:45.179211+0000","last_active":"2023-09-23T20:25:02.918741+0000","last_peered":"2023-09-23T20:25:02.918741+0000","last_clean":"2023-09-23T20:25:02.918741+0000","last_became_active":"2023-09-23T20:12:45.178966+0000","last_became_peered":"2023-09-23T20:12:45.178966+0000","last_unstale":"2023-09-23T20:25:02.918741+0000","last_undegraded":"2023-09-23T20:25:02.918741+0000","last_fullsized":"2023-09-23T20:25:02.918741+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:58.902140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.8","version":"167'4986","reported_seq":5107,"reported_epoch":167,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:25:09.773816+0000","last_change":"2023-09-23T20:24:54.251778+0000","last_active":"2023-09-23T20:25:09.773816+0000","last_peered":"2023-09-23T20:25:09.773816+0000","last_clean":"2023-09-23T20:24:51.985226+0000","last_became_active":"2023-09-23T20:24:54.238441+0000","last_became_peered":"2023-09-23T20:24:54.238441+0000","last_unstale":"2023-09-23T20:25:09.773816+0000","last_undegraded":"2023-09-23T20:24:54.104438+0000","last_fullsized":"2023-09-23T20:24:54.068189+0000","mapping_epoch":161,"log_start":"0'0","ondisk_log_start":"0'0","created":111,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"120'2144","last_scrub_stamp":"2023-09-23T20:23:17.129360+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:23:17.129360+0000","objects_scrubbed":1209,"log_size":4986,"log_dups_size":0,"ondisk_log_size":4986,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:30:47.560705+0000","scrub_duration":0.17393181099999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":171769856,"num_objects":2621,"num_object_clones":0,"num_object_copies":7863,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3336,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2621,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2842,"num_write_kb":136128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1],"avail_no_missing":["0","1","2"],"object_location_counts":[{"shards":"0,1","objects":-249},{"shards":"0,1,2","objects":2621}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":129,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-23T20:25:02.935873+0000","last_change":"2023-09-23T20:20:46.950784+0000","last_active":"2023-09-23T20:25:02.935873+0000","last_peered":"2023-09-23T20:25:02.935873+0000","last_clean":"2023-09-23T20:25:02.935873+0000","last_became_active":"2023-09-23T20:20:46.950055+0000","last_became_peered":"2023-09-23T20:20:46.950055+0000","last_unstale":"2023-09-23T20:25:02.935873+0000","last_undegraded":"2023-09-23T20:25:02.935873+0000","last_fullsized":"2023-09-23T20:25:02.935873+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:23:07.645427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":162,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-23T20:25:02.944119+0000","last_change":"2023-09-23T20:25:02.943743+0000","last_active":"2023-09-23T20:25:02.944119+0000","last_peered":"2023-09-23T20:25:02.944119+0000","last_clean":"2023-09-23T20:25:02.944119+0000","last_became_active":"2023-09-23T20:25:02.943279+0000","last_became_peered":"2023-09-23T20:25:02.943279+0000","last_unstale":"2023-09-23T20:25:02.944119+0000","last_undegraded":"2023-09-23T20:25:02.944119+0000","last_fullsized":"2023-09-23T20:25:02.944119+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:21:36.128366+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.b","version":"167'8108","reported_seq":8179,"reported_epoch":167,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:25:09.827442+0000","last_change":"2023-09-23T20:24:54.402158+0000","last_active":"2023-09-23T20:25:09.827442+0000","last_peered":"2023-09-23T20:25:09.827442+0000","last_clean":"2023-09-23T20:24:52.031628+0000","last_became_active":"2023-09-23T20:24:54.372312+0000","last_became_peered":"2023-09-23T20:24:54.372312+0000","last_unstale":"2023-09-23T20:25:09.827442+0000","last_undegraded":"2023-09-23T20:24:54.118218+0000","last_fullsized":"2023-09-23T20:24:54.073487+0000","mapping_epoch":161,"log_start":"119'1966","ondisk_log_start":"119'1966","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:22:51.846999+0000","objects_scrubbed":0,"log_size":6142,"log_dups_size":0,"ondisk_log_size":6142,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:50:06.279186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":335806464,"num_objects":5124,"num_object_clones":0,"num_object_copies":15372,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6616,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5124,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8108,"num_write_kb":423424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5],"avail_no_missing":["0","2","5"],"object_location_counts":[{"shards":"0,2,5","objects":5124},{"shards":"0,5","objects":-520}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":111,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-23T20:25:02.935992+0000","last_change":"2023-09-23T20:12:43.751523+0000","last_active":"2023-09-23T20:25:02.935992+0000","last_peered":"2023-09-23T20:25:02.935992+0000","last_clean":"2023-09-23T20:25:02.935992+0000","last_became_active":"2023-09-23T20:12:43.751257+0000","last_became_peered":"2023-09-23T20:12:43.751257+0000","last_unstale":"2023-09-23T20:25:02.935992+0000","last_undegraded":"2023-09-23T20:25:02.935992+0000","last_fullsized":"2023-09-23T20:25:02.935992+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:50:27.040225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":126,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-23T20:25:02.925093+0000","last_change":"2023-09-23T20:20:46.934762+0000","last_active":"2023-09-23T20:25:02.925093+0000","last_peered":"2023-09-23T20:25:02.925093+0000","last_clean":"2023-09-23T20:25:02.925093+0000","last_became_active":"2023-09-23T20:20:46.934472+0000","last_became_peered":"2023-09-23T20:20:46.934472+0000","last_unstale":"2023-09-23T20:25:02.925093+0000","last_undegraded":"2023-09-23T20:25:02.925093+0000","last_fullsized":"2023-09-23T20:25:02.925093+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:28:42.708286+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.a","version":"167'8277","reported_seq":9839,"reported_epoch":167,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:25:10.491042+0000","last_change":"2023-09-23T20:25:03.180899+0000","last_active":"2023-09-23T20:25:10.491042+0000","last_peered":"2023-09-23T20:25:10.491042+0000","last_clean":"2023-09-23T20:24:36.892269+0000","last_became_active":"2023-09-23T20:25:03.105771+0000","last_became_peered":"2023-09-23T20:25:03.105771+0000","last_unstale":"2023-09-23T20:25:10.491042+0000","last_undegraded":"2023-09-23T20:25:03.091729+0000","last_fullsized":"2023-09-23T20:25:02.915832+0000","mapping_epoch":165,"log_start":"119'2113","ondisk_log_start":"119'2113","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:22:51.846999+0000","objects_scrubbed":0,"log_size":6164,"log_dups_size":0,"ondisk_log_size":6164,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:39:43.225804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":337575936,"num_objects":5151,"num_object_clones":0,"num_object_copies":15453,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6842,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5151,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8277,"num_write_kb":429696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":599,"num_bytes_recovered":39059456,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,0],"avail_no_missing":["4","2"],"object_location_counts":[{"shards":"0,4","objects":1},{"shards":"2,3,4","objects":5150}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":146,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-23T20:25:02.947954+0000","last_change":"2023-09-23T20:25:02.947631+0000","last_active":"2023-09-23T20:25:02.947954+0000","last_peered":"2023-09-23T20:25:02.947954+0000","last_clean":"2023-09-23T20:25:02.947954+0000","last_became_active":"2023-09-23T20:25:02.947268+0000","last_became_peered":"2023-09-23T20:25:02.947268+0000","last_unstale":"2023-09-23T20:25:02.947954+0000","last_undegraded":"2023-09-23T20:25:02.947954+0000","last_fullsized":"2023-09-23T20:25:02.947954+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:19:43.643985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":111,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-23T20:25:02.919600+0000","last_change":"2023-09-23T20:20:46.935968+0000","last_active":"2023-09-23T20:25:02.919600+0000","last_peered":"2023-09-23T20:25:02.919600+0000","last_clean":"2023-09-23T20:25:02.919600+0000","last_became_active":"2023-09-23T20:20:46.935618+0000","last_became_peered":"2023-09-23T20:20:46.935618+0000","last_unstale":"2023-09-23T20:25:02.919600+0000","last_undegraded":"2023-09-23T20:25:02.919600+0000","last_fullsized":"2023-09-23T20:25:02.919600+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:42:59.867176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.d","version":"167'8348","reported_seq":8850,"reported_epoch":167,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T20:25:10.399178+0000","last_change":"2023-09-23T20:25:03.013605+0000","last_active":"2023-09-23T20:25:10.399178+0000","last_peered":"2023-09-23T20:25:10.399178+0000","last_clean":"2023-09-23T20:24:52.040908+0000","last_became_active":"2023-09-23T20:24:54.341971+0000","last_became_peered":"2023-09-23T20:24:54.341971+0000","last_unstale":"2023-09-23T20:25:10.399178+0000","last_undegraded":"2023-09-23T20:24:54.128184+0000","last_fullsized":"2023-09-23T20:24:54.071764+0000","mapping_epoch":161,"log_start":"119'2267","ondisk_log_start":"119'2267","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:22:51.846999+0000","objects_scrubbed":0,"log_size":6081,"log_dups_size":0,"ondisk_log_size":6081,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:46:47.433617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":344653848,"num_objects":5260,"num_object_clones":0,"num_object_copies":15780,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":370,"num_objects_misplaced":5258,"num_objects_unfound":0,"num_objects_dirty":5260,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8348,"num_write_kb":435393,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":213,"num_bytes_recovered":13959168,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,0],"acting":[7,5],"avail_no_missing":["7","2","5"],"object_location_counts":[{"shards":"2,5,7","objects":5258},{"shards":"5,7","objects":-368}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":16,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-23T20:25:03.001231+0000","last_change":"2023-09-23T20:25:03.000957+0000","last_active":"2023-09-23T20:25:03.001231+0000","last_peered":"2023-09-23T20:25:03.001231+0000","last_clean":"2023-09-23T20:25:03.001231+0000","last_became_active":"2023-09-23T20:25:02.939903+0000","last_became_peered":"2023-09-23T20:25:02.939903+0000","last_unstale":"2023-09-23T20:25:03.001231+0000","last_undegraded":"2023-09-23T20:25:03.001231+0000","last_fullsized":"2023-09-23T20:25:03.001231+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:51:14.981461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":145,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-23T20:25:02.929853+0000","last_change":"2023-09-23T20:25:02.929447+0000","last_active":"2023-09-23T20:25:02.929853+0000","last_peered":"2023-09-23T20:25:02.929853+0000","last_clean":"2023-09-23T20:25:02.929853+0000","last_became_active":"2023-09-23T20:25:02.929173+0000","last_became_peered":"2023-09-23T20:25:02.929173+0000","last_unstale":"2023-09-23T20:25:02.929853+0000","last_undegraded":"2023-09-23T20:25:02.929853+0000","last_fullsized":"2023-09-23T20:25:02.929853+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:32:09.280647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.c","version":"167'8216","reported_seq":8281,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T20:25:09.975849+0000","last_change":"2023-09-23T20:22:52.866632+0000","last_active":"2023-09-23T20:25:09.975849+0000","last_peered":"2023-09-23T20:25:09.975849+0000","last_clean":"2023-09-23T20:25:09.975849+0000","last_became_active":"2023-09-23T20:22:52.866356+0000","last_became_peered":"2023-09-23T20:22:52.866356+0000","last_unstale":"2023-09-23T20:25:09.975849+0000","last_undegraded":"2023-09-23T20:25:09.975849+0000","last_fullsized":"2023-09-23T20:25:09.975849+0000","mapping_epoch":111,"log_start":"129'2673","ondisk_log_start":"129'2673","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:22:51.846999+0000","objects_scrubbed":0,"log_size":5543,"log_dups_size":0,"ondisk_log_size":5543,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:10:12.356062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":341311488,"num_objects":5208,"num_object_clones":0,"num_object_copies":15624,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5208,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8216,"num_write_kb":429568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":127,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-23T20:25:02.925031+0000","last_change":"2023-09-23T20:20:46.937163+0000","last_active":"2023-09-23T20:25:02.925031+0000","last_peered":"2023-09-23T20:25:02.925031+0000","last_clean":"2023-09-23T20:25:02.925031+0000","last_became_active":"2023-09-23T20:20:46.936757+0000","last_became_peered":"2023-09-23T20:20:46.936757+0000","last_unstale":"2023-09-23T20:25:02.925031+0000","last_undegraded":"2023-09-23T20:25:02.925031+0000","last_fullsized":"2023-09-23T20:25:02.925031+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:09.401379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":145,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-23T20:25:02.922574+0000","last_change":"2023-09-23T20:24:53.088424+0000","last_active":"2023-09-23T20:25:02.922574+0000","last_peered":"2023-09-23T20:25:02.922574+0000","last_clean":"2023-09-23T20:25:02.922574+0000","last_became_active":"2023-09-23T20:24:53.088096+0000","last_became_peered":"2023-09-23T20:24:53.088096+0000","last_unstale":"2023-09-23T20:25:02.922574+0000","last_undegraded":"2023-09-23T20:25:02.922574+0000","last_fullsized":"2023-09-23T20:25:02.922574+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:49:22.479999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.f","version":"167'8188","reported_seq":8251,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T20:25:10.384072+0000","last_change":"2023-09-23T20:22:52.872510+0000","last_active":"2023-09-23T20:25:10.384072+0000","last_peered":"2023-09-23T20:25:10.384072+0000","last_clean":"2023-09-23T20:25:10.384072+0000","last_became_active":"2023-09-23T20:22:52.872179+0000","last_became_peered":"2023-09-23T20:22:52.872179+0000","last_unstale":"2023-09-23T20:25:10.384072+0000","last_undegraded":"2023-09-23T20:25:10.384072+0000","last_fullsized":"2023-09-23T20:25:10.384072+0000","mapping_epoch":111,"log_start":"115'695","ondisk_log_start":"115'695","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:22:51.846999+0000","objects_scrubbed":0,"log_size":7493,"log_dups_size":0,"ondisk_log_size":7493,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:53.012272+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":331087872,"num_objects":5052,"num_object_clones":0,"num_object_copies":15156,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5052,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8188,"num_write_kb":423680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":136,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-23T20:25:02.930939+0000","last_change":"2023-09-23T20:24:53.086258+0000","last_active":"2023-09-23T20:25:02.930939+0000","last_peered":"2023-09-23T20:25:02.930939+0000","last_clean":"2023-09-23T20:25:02.930939+0000","last_became_active":"2023-09-23T20:24:53.085899+0000","last_became_peered":"2023-09-23T20:24:53.085899+0000","last_unstale":"2023-09-23T20:25:02.930939+0000","last_undegraded":"2023-09-23T20:25:02.930939+0000","last_fullsized":"2023-09-23T20:25:02.930939+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:58:18.246920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":125,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-23T20:25:02.922400+0000","last_change":"2023-09-23T20:20:46.939720+0000","last_active":"2023-09-23T20:25:02.922400+0000","last_peered":"2023-09-23T20:25:02.922400+0000","last_clean":"2023-09-23T20:25:02.922400+0000","last_became_active":"2023-09-23T20:20:46.939357+0000","last_became_peered":"2023-09-23T20:20:46.939357+0000","last_unstale":"2023-09-23T20:25:02.922400+0000","last_undegraded":"2023-09-23T20:25:02.922400+0000","last_fullsized":"2023-09-23T20:25:02.922400+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:56.613212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.e","version":"167'8096","reported_seq":8159,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T20:25:10.447447+0000","last_change":"2023-09-23T20:22:52.871704+0000","last_active":"2023-09-23T20:25:10.447447+0000","last_peered":"2023-09-23T20:25:10.447447+0000","last_clean":"2023-09-23T20:25:10.447447+0000","last_became_active":"2023-09-23T20:22:52.871267+0000","last_became_peered":"2023-09-23T20:22:52.871267+0000","last_unstale":"2023-09-23T20:25:10.447447+0000","last_undegraded":"2023-09-23T20:25:10.447447+0000","last_fullsized":"2023-09-23T20:25:10.447447+0000","mapping_epoch":111,"log_start":"119'2004","ondisk_log_start":"119'2004","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:22:51.846999+0000","objects_scrubbed":0,"log_size":6092,"log_dups_size":0,"ondisk_log_size":6092,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:50:07.619801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":340131840,"num_objects":5190,"num_object_clones":0,"num_object_copies":15570,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5190,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8096,"num_write_kb":425152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":12,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-23T20:25:02.999072+0000","last_change":"2023-09-23T20:25:02.978173+0000","last_active":"2023-09-23T20:25:02.999072+0000","last_peered":"2023-09-23T20:25:02.999072+0000","last_clean":"2023-09-23T20:25:02.999072+0000","last_became_active":"2023-09-23T20:25:02.977856+0000","last_became_peered":"2023-09-23T20:25:02.977856+0000","last_unstale":"2023-09-23T20:25:02.999072+0000","last_undegraded":"2023-09-23T20:25:02.999072+0000","last_fullsized":"2023-09-23T20:25:02.999072+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:19.861943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":26,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-23T20:25:02.937088+0000","last_change":"2023-09-23T20:25:02.924911+0000","last_active":"2023-09-23T20:25:02.937088+0000","last_peered":"2023-09-23T20:25:02.937088+0000","last_clean":"2023-09-23T20:25:02.937088+0000","last_became_active":"2023-09-23T20:25:02.924586+0000","last_became_peered":"2023-09-23T20:25:02.924586+0000","last_unstale":"2023-09-23T20:25:02.937088+0000","last_undegraded":"2023-09-23T20:25:02.937088+0000","last_fullsized":"2023-09-23T20:25:02.937088+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:24:44.176766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"86'119","reported_seq":500,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-23T20:25:02.910555+0000","last_change":"2023-09-23T20:20:46.937071+0000","last_active":"2023-09-23T20:25:02.910555+0000","last_peered":"2023-09-23T20:25:02.910555+0000","last_clean":"2023-09-23T20:25:02.910555+0000","last_became_active":"2023-09-23T20:20:46.935348+0000","last_became_peered":"2023-09-23T20:20:46.935348+0000","last_unstale":"2023-09-23T20:25:02.910555+0000","last_undegraded":"2023-09-23T20:25:02.910555+0000","last_fullsized":"2023-09-23T20:25:02.910555+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":119,"log_dups_size":0,"ondisk_log_size":119,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:09.829030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032705499999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":523,"num_read_kb":3503,"num_write":136,"num_write_kb":3128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":173,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-23T20:25:02.955523+0000","last_change":"2023-09-23T20:25:02.949419+0000","last_active":"2023-09-23T20:25:02.955523+0000","last_peered":"2023-09-23T20:25:02.955523+0000","last_clean":"2023-09-23T20:25:02.955523+0000","last_became_active":"2023-09-23T20:25:02.948712+0000","last_became_peered":"2023-09-23T20:25:02.948712+0000","last_unstale":"2023-09-23T20:25:02.955523+0000","last_undegraded":"2023-09-23T20:25:02.955523+0000","last_fullsized":"2023-09-23T20:25:02.955523+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:30:14.591799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":92,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-23T20:25:02.905704+0000","last_change":"2023-09-23T20:20:46.937104+0000","last_active":"2023-09-23T20:25:02.905704+0000","last_peered":"2023-09-23T20:25:02.905704+0000","last_clean":"2023-09-23T20:25:02.905704+0000","last_became_active":"2023-09-23T20:20:46.936820+0000","last_became_peered":"2023-09-23T20:20:46.936820+0000","last_unstale":"2023-09-23T20:25:02.905704+0000","last_undegraded":"2023-09-23T20:25:02.905704+0000","last_fullsized":"2023-09-23T20:25:02.905704+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:22:19.879310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.3","version":"167'5055","reported_seq":5132,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T20:25:09.677075+0000","last_change":"2023-09-23T20:23:20.060302+0000","last_active":"2023-09-23T20:25:09.677075+0000","last_peered":"2023-09-23T20:25:09.677075+0000","last_clean":"2023-09-23T20:25:09.677075+0000","last_became_active":"2023-09-23T20:23:11.059880+0000","last_became_peered":"2023-09-23T20:23:11.059880+0000","last_unstale":"2023-09-23T20:25:09.677075+0000","last_undegraded":"2023-09-23T20:25:09.677075+0000","last_fullsized":"2023-09-23T20:25:09.677075+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":111,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:22:51.846999+0000","objects_scrubbed":0,"log_size":5055,"log_dups_size":0,"ondisk_log_size":5055,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":177111040,"num_objects":2702,"num_object_clones":0,"num_object_copies":8106,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2702,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4152,"num_write_kb":219360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.5","version":"167'4973","reported_seq":5050,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T20:25:09.834992+0000","last_change":"2023-09-23T20:23:11.152272+0000","last_active":"2023-09-23T20:25:09.834992+0000","last_peered":"2023-09-23T20:25:09.834992+0000","last_clean":"2023-09-23T20:25:09.834992+0000","last_became_active":"2023-09-23T20:23:11.055735+0000","last_became_peered":"2023-09-23T20:23:11.055735+0000","last_unstale":"2023-09-23T20:25:09.834992+0000","last_undegraded":"2023-09-23T20:25:09.834992+0000","last_fullsized":"2023-09-23T20:25:09.834992+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":111,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:22:51.846999+0000","objects_scrubbed":0,"log_size":4973,"log_dups_size":0,"ondisk_log_size":4973,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":171048960,"num_objects":2610,"num_object_clones":0,"num_object_copies":7830,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2610,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4090,"num_write_kb":214400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"86'106","reported_seq":791,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-23T20:25:02.920575+0000","last_change":"2023-09-23T20:20:46.939468+0000","last_active":"2023-09-23T20:25:02.920575+0000","last_peered":"2023-09-23T20:25:02.920575+0000","last_clean":"2023-09-23T20:25:02.920575+0000","last_became_active":"2023-09-23T20:20:46.938954+0000","last_became_peered":"2023-09-23T20:20:46.938954+0000","last_unstale":"2023-09-23T20:25:02.920575+0000","last_undegraded":"2023-09-23T20:25:02.920575+0000","last_fullsized":"2023-09-23T20:25:02.920575+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":106,"log_dups_size":0,"ondisk_log_size":106,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:01:43.761064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00099302599999999993,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1105,"num_read_kb":1689,"num_write":100,"num_write_kb":404,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":126,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-23T20:25:02.917533+0000","last_change":"2023-09-23T20:20:14.887345+0000","last_active":"2023-09-23T20:25:02.917533+0000","last_peered":"2023-09-23T20:25:02.917533+0000","last_clean":"2023-09-23T20:25:02.917533+0000","last_became_active":"2023-09-23T20:20:14.886970+0000","last_became_peered":"2023-09-23T20:20:14.886970+0000","last_unstale":"2023-09-23T20:25:02.917533+0000","last_undegraded":"2023-09-23T20:25:02.917533+0000","last_fullsized":"2023-09-23T20:25:02.917533+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:00:38.575926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":149,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-23T20:25:02.948436+0000","last_change":"2023-09-23T20:25:02.942681+0000","last_active":"2023-09-23T20:25:02.948436+0000","last_peered":"2023-09-23T20:25:02.948436+0000","last_clean":"2023-09-23T20:25:02.948436+0000","last_became_active":"2023-09-23T20:25:02.942161+0000","last_became_peered":"2023-09-23T20:25:02.942161+0000","last_unstale":"2023-09-23T20:25:02.948436+0000","last_undegraded":"2023-09-23T20:25:02.948436+0000","last_fullsized":"2023-09-23T20:25:02.948436+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:54:11.686892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.6","version":"167'5072","reported_seq":5430,"reported_epoch":167,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:25:10.320540+0000","last_change":"2023-09-23T20:25:02.979495+0000","last_active":"2023-09-23T20:25:10.320540+0000","last_peered":"2023-09-23T20:25:10.320540+0000","last_clean":"2023-09-23T20:24:51.979668+0000","last_became_active":"2023-09-23T20:24:54.304218+0000","last_became_peered":"2023-09-23T20:24:54.304218+0000","last_unstale":"2023-09-23T20:25:10.320540+0000","last_undegraded":"2023-09-23T20:24:54.098533+0000","last_fullsized":"2023-09-23T20:24:54.076592+0000","mapping_epoch":161,"log_start":"0'0","ondisk_log_start":"0'0","created":111,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"119'1954","last_scrub_stamp":"2023-09-23T20:23:12.084143+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:23:12.084143+0000","objects_scrubbed":1042,"log_size":5072,"log_dups_size":0,"ondisk_log_size":5072,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:17:58.572964+0000","scrub_duration":0.23299162700000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":164364288,"num_objects":2508,"num_object_clones":0,"num_object_copies":7524,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3218,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2508,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3122,"num_write_kb":146816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":118,"num_bytes_recovered":7667712,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7],"avail_no_missing":["5","2","7"],"object_location_counts":[{"shards":"2,5,7","objects":2508},{"shards":"5,7","objects":-187}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"86'130","reported_seq":373,"reported_epoch":167,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T20:25:03.924463+0000","last_change":"2023-09-23T20:25:03.924645+0000","last_active":"2023-09-23T20:25:03.924463+0000","last_peered":"2023-09-23T20:25:03.924463+0000","last_clean":"2023-09-23T20:13:52.310702+0000","last_became_active":"2023-09-23T20:25:03.924117+0000","last_became_peered":"2023-09-23T20:25:03.924117+0000","last_unstale":"2023-09-23T20:25:03.924463+0000","last_undegraded":"2023-09-23T20:25:03.909544+0000","last_fullsized":"2023-09-23T20:25:03.924463+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":110,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:53:23.259685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011300049999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":16,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":512,"num_read_kb":1820,"num_write":133,"num_write_kb":1347,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":["3","0","2"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":115,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-23T20:25:02.904646+0000","last_change":"2023-09-23T20:24:53.090659+0000","last_active":"2023-09-23T20:25:02.904646+0000","last_peered":"2023-09-23T20:25:02.904646+0000","last_clean":"2023-09-23T20:25:02.904646+0000","last_became_active":"2023-09-23T20:24:53.090354+0000","last_became_peered":"2023-09-23T20:24:53.090354+0000","last_unstale":"2023-09-23T20:25:02.904646+0000","last_undegraded":"2023-09-23T20:25:02.904646+0000","last_fullsized":"2023-09-23T20:25:02.904646+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:27:39.275622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":135,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T20:25:03.907108+0000","last_change":"2023-09-23T20:20:46.936524+0000","last_active":"2023-09-23T20:25:03.907108+0000","last_peered":"2023-09-23T20:25:03.907108+0000","last_clean":"2023-09-23T20:25:03.907108+0000","last_became_active":"2023-09-23T20:20:46.936267+0000","last_became_peered":"2023-09-23T20:20:46.936267+0000","last_unstale":"2023-09-23T20:25:03.907108+0000","last_undegraded":"2023-09-23T20:25:03.907108+0000","last_fullsized":"2023-09-23T20:25:03.907108+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:32:20.984521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"86'194","reported_seq":803,"reported_epoch":166,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:25:02.992677+0000","last_change":"2023-09-23T20:25:02.992677+0000","last_active":"2023-09-23T20:25:02.992677+0000","last_peered":"2023-09-23T20:25:02.992677+0000","last_clean":"2023-09-23T20:24:44.985617+0000","last_became_active":"2023-09-23T20:25:02.992336+0000","last_became_peered":"2023-09-23T20:25:02.992336+0000","last_unstale":"2023-09-23T20:25:02.992677+0000","last_undegraded":"2023-09-23T20:25:02.948123+0000","last_fullsized":"2023-09-23T20:25:02.947375+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":194,"log_dups_size":0,"ondisk_log_size":194,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:40:10.848836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00037995999999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1025,"num_read_kb":1201,"num_write":173,"num_write_kb":1276,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7],"avail_no_missing":["5","2","7"],"object_location_counts":[{"shards":"2,5,7","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":509,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-23T20:25:02.936082+0000","last_change":"2023-09-23T20:20:14.882204+0000","last_active":"2023-09-23T20:25:02.936082+0000","last_peered":"2023-09-23T20:25:02.936082+0000","last_clean":"2023-09-23T20:25:02.936082+0000","last_became_active":"2023-09-23T20:20:14.881699+0000","last_became_peered":"2023-09-23T20:20:14.881699+0000","last_unstale":"2023-09-23T20:25:02.936082+0000","last_undegraded":"2023-09-23T20:25:02.936082+0000","last_fullsized":"2023-09-23T20:25:02.936082+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:09:47.213130+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:16:37.341010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":12,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-23T20:25:02.999170+0000","last_change":"2023-09-23T20:25:02.984205+0000","last_active":"2023-09-23T20:25:02.999170+0000","last_peered":"2023-09-23T20:25:02.999170+0000","last_clean":"2023-09-23T20:25:02.999170+0000","last_became_active":"2023-09-23T20:25:02.983758+0000","last_became_peered":"2023-09-23T20:25:02.983758+0000","last_unstale":"2023-09-23T20:25:02.999170+0000","last_undegraded":"2023-09-23T20:25:02.999170+0000","last_fullsized":"2023-09-23T20:25:02.999170+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:35:14.573101+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":109,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-23T20:25:02.923616+0000","last_change":"2023-09-23T20:12:45.609829+0000","last_active":"2023-09-23T20:25:02.923616+0000","last_peered":"2023-09-23T20:25:02.923616+0000","last_clean":"2023-09-23T20:25:02.923616+0000","last_became_active":"2023-09-23T20:12:45.609467+0000","last_became_peered":"2023-09-23T20:12:45.609467+0000","last_unstale":"2023-09-23T20:25:02.923616+0000","last_undegraded":"2023-09-23T20:25:02.923616+0000","last_fullsized":"2023-09-23T20:25:02.923616+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:27:31.490555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.4","version":"167'5066","reported_seq":5444,"reported_epoch":167,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:25:10.221753+0000","last_change":"2023-09-23T20:25:02.956346+0000","last_active":"2023-09-23T20:25:10.221753+0000","last_peered":"2023-09-23T20:25:10.221753+0000","last_clean":"2023-09-23T20:24:52.062405+0000","last_became_active":"2023-09-23T20:24:54.375235+0000","last_became_peered":"2023-09-23T20:24:54.375235+0000","last_unstale":"2023-09-23T20:25:10.221753+0000","last_undegraded":"2023-09-23T20:24:54.227166+0000","last_fullsized":"2023-09-23T20:24:54.199160+0000","mapping_epoch":161,"log_start":"0'0","ondisk_log_start":"0'0","created":111,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"119'1973","last_scrub_stamp":"2023-09-23T20:23:13.408658+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:23:13.408658+0000","objects_scrubbed":1060,"log_size":5066,"log_dups_size":0,"ondisk_log_size":5066,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:00:39.267224+0000","scrub_duration":0.28811426499999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":175046656,"num_objects":2671,"num_object_clones":0,"num_object_copies":8013,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3295,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2671,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3097,"num_write_kb":150656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":119,"num_bytes_recovered":7798784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,0],"acting":[7,5],"avail_no_missing":["7","2","5"],"object_location_counts":[{"shards":"2,5,7","objects":2671},{"shards":"5,7","objects":-157}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.7","version":"167'5117","reported_seq":5256,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T20:25:10.387947+0000","last_change":"2023-09-23T20:23:16.284169+0000","last_active":"2023-09-23T20:25:10.387947+0000","last_peered":"2023-09-23T20:25:10.387947+0000","last_clean":"2023-09-23T20:25:10.387947+0000","last_became_active":"2023-09-23T20:23:11.069066+0000","last_became_peered":"2023-09-23T20:23:11.069066+0000","last_unstale":"2023-09-23T20:25:10.387947+0000","last_undegraded":"2023-09-23T20:25:10.387947+0000","last_fullsized":"2023-09-23T20:25:10.387947+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":111,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"120'2196","last_scrub_stamp":"2023-09-23T20:23:16.284112+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:23:16.284112+0000","objects_scrubbed":1268,"log_size":5117,"log_dups_size":0,"ondisk_log_size":5117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:41:59.721972+0000","scrub_duration":0.33229684999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":174325760,"num_objects":2660,"num_object_clones":0,"num_object_copies":7980,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2660,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2930,"num_write_kb":138304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.0","version":"86'80","reported_seq":413,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-23T20:25:02.935980+0000","last_change":"2023-09-23T20:20:46.950784+0000","last_active":"2023-09-23T20:25:02.935980+0000","last_peered":"2023-09-23T20:25:02.935980+0000","last_clean":"2023-09-23T20:25:02.935980+0000","last_became_active":"2023-09-23T20:20:46.950071+0000","last_became_peered":"2023-09-23T20:20:46.950071+0000","last_unstale":"2023-09-23T20:25:02.935980+0000","last_undegraded":"2023-09-23T20:25:02.935980+0000","last_fullsized":"2023-09-23T20:25:02.935980+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":80,"log_dups_size":0,"ondisk_log_size":80,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:11:06.812404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035115,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":395,"num_read_kb":1370,"num_write":66,"num_write_kb":1061,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":128,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-23T20:25:02.943221+0000","last_change":"2023-09-23T20:25:02.942709+0000","last_active":"2023-09-23T20:25:02.943221+0000","last_peered":"2023-09-23T20:25:02.943221+0000","last_clean":"2023-09-23T20:25:02.943221+0000","last_became_active":"2023-09-23T20:25:02.942323+0000","last_became_peered":"2023-09-23T20:25:02.942323+0000","last_unstale":"2023-09-23T20:25:02.943221+0000","last_undegraded":"2023-09-23T20:25:02.943221+0000","last_fullsized":"2023-09-23T20:25:02.943221+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:26:12.039101+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":135,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-23T20:25:02.915175+0000","last_change":"2023-09-23T20:24:53.094930+0000","last_active":"2023-09-23T20:25:02.915175+0000","last_peered":"2023-09-23T20:25:02.915175+0000","last_clean":"2023-09-23T20:25:02.915175+0000","last_became_active":"2023-09-23T20:24:53.094200+0000","last_became_peered":"2023-09-23T20:24:53.094200+0000","last_unstale":"2023-09-23T20:25:02.915175+0000","last_undegraded":"2023-09-23T20:25:02.915175+0000","last_fullsized":"2023-09-23T20:25:02.915175+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:48:49.948329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"86'102","reported_seq":673,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-23T20:25:02.936062+0000","last_change":"2023-09-23T20:20:14.881975+0000","last_active":"2023-09-23T20:25:02.936062+0000","last_peered":"2023-09-23T20:25:02.936062+0000","last_clean":"2023-09-23T20:25:02.936062+0000","last_became_active":"2023-09-23T20:20:14.881544+0000","last_became_peered":"2023-09-23T20:20:14.881544+0000","last_unstale":"2023-09-23T20:25:02.936062+0000","last_undegraded":"2023-09-23T20:25:02.936062+0000","last_fullsized":"2023-09-23T20:25:02.936062+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":102,"log_dups_size":0,"ondisk_log_size":102,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:34.594972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030656500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":963,"num_read_kb":1234,"num_write":60,"num_write_kb":225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":115,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-23T20:25:02.910670+0000","last_change":"2023-09-23T20:24:53.077730+0000","last_active":"2023-09-23T20:25:02.910670+0000","last_peered":"2023-09-23T20:25:02.910670+0000","last_clean":"2023-09-23T20:25:02.910670+0000","last_became_active":"2023-09-23T20:24:53.077091+0000","last_became_peered":"2023-09-23T20:24:53.077091+0000","last_unstale":"2023-09-23T20:25:02.910670+0000","last_undegraded":"2023-09-23T20:25:02.910670+0000","last_fullsized":"2023-09-23T20:25:02.910670+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:41:18.221370+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":96,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-23T20:25:02.919833+0000","last_change":"2023-09-23T20:20:14.888874+0000","last_active":"2023-09-23T20:25:02.919833+0000","last_peered":"2023-09-23T20:25:02.919833+0000","last_clean":"2023-09-23T20:25:02.919833+0000","last_became_active":"2023-09-23T20:20:14.887724+0000","last_became_peered":"2023-09-23T20:20:14.887724+0000","last_unstale":"2023-09-23T20:25:02.919833+0000","last_undegraded":"2023-09-23T20:25:02.919833+0000","last_fullsized":"2023-09-23T20:25:02.919833+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:09:24.510326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.2","version":"167'5049","reported_seq":5174,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T20:25:10.341482+0000","last_change":"2023-09-23T20:23:12.771560+0000","last_active":"2023-09-23T20:25:10.341482+0000","last_peered":"2023-09-23T20:25:10.341482+0000","last_clean":"2023-09-23T20:25:10.341482+0000","last_became_active":"2023-09-23T20:23:11.058929+0000","last_became_peered":"2023-09-23T20:23:11.058929+0000","last_unstale":"2023-09-23T20:25:10.341482+0000","last_undegraded":"2023-09-23T20:25:10.341482+0000","last_fullsized":"2023-09-23T20:25:10.341482+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":111,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"119'1924","last_scrub_stamp":"2023-09-23T20:23:12.771453+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:23:12.771453+0000","objects_scrubbed":1010,"log_size":5049,"log_dups_size":0,"ondisk_log_size":5049,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:49:31.924585+0000","scrub_duration":0.26141313799999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":169541632,"num_objects":2587,"num_object_clones":0,"num_object_copies":7761,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2587,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3133,"num_write_kb":150720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":16,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-23T20:25:03.007959+0000","last_change":"2023-09-23T20:25:03.007736+0000","last_active":"2023-09-23T20:25:03.007959+0000","last_peered":"2023-09-23T20:25:03.007959+0000","last_clean":"2023-09-23T20:25:03.007959+0000","last_became_active":"2023-09-23T20:25:02.973689+0000","last_became_peered":"2023-09-23T20:25:02.973689+0000","last_unstale":"2023-09-23T20:25:03.007959+0000","last_undegraded":"2023-09-23T20:25:03.007959+0000","last_fullsized":"2023-09-23T20:25:03.007959+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:37:25.643774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"86'128","reported_seq":491,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T20:25:03.907608+0000","last_change":"2023-09-23T20:20:46.939256+0000","last_active":"2023-09-23T20:25:03.907608+0000","last_peered":"2023-09-23T20:25:03.907608+0000","last_clean":"2023-09-23T20:25:03.907608+0000","last_became_active":"2023-09-23T20:20:46.938756+0000","last_became_peered":"2023-09-23T20:20:46.938756+0000","last_unstale":"2023-09-23T20:25:03.907608+0000","last_undegraded":"2023-09-23T20:25:03.907608+0000","last_fullsized":"2023-09-23T20:25:03.907608+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:52:18.161436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000293202,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":579,"num_read_kb":1473,"num_write":96,"num_write_kb":1062,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":12,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-23T20:25:02.999082+0000","last_change":"2023-09-23T20:25:02.971740+0000","last_active":"2023-09-23T20:25:02.999082+0000","last_peered":"2023-09-23T20:25:02.999082+0000","last_clean":"2023-09-23T20:25:02.999082+0000","last_became_active":"2023-09-23T20:25:02.970412+0000","last_became_peered":"2023-09-23T20:25:02.970412+0000","last_unstale":"2023-09-23T20:25:02.999082+0000","last_undegraded":"2023-09-23T20:25:02.999082+0000","last_fullsized":"2023-09-23T20:25:02.999082+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:53:17.247216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.1","version":"167'5055","reported_seq":5821,"reported_epoch":167,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:25:10.399052+0000","last_change":"2023-09-23T20:25:03.007104+0000","last_active":"2023-09-23T20:25:10.399052+0000","last_peered":"2023-09-23T20:25:10.399052+0000","last_clean":"2023-09-23T20:24:36.885029+0000","last_became_active":"2023-09-23T20:25:02.989894+0000","last_became_peered":"2023-09-23T20:25:02.989894+0000","last_unstale":"2023-09-23T20:25:10.399052+0000","last_undegraded":"2023-09-23T20:25:02.930513+0000","last_fullsized":"2023-09-23T20:25:02.916357+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":111,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"119'1889","last_scrub_stamp":"2023-09-23T20:23:11.363674+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:23:11.363674+0000","objects_scrubbed":958,"log_size":5055,"log_dups_size":0,"ondisk_log_size":5055,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:39:55.591205+0000","scrub_duration":0.26718564099999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163053568,"num_objects":2488,"num_object_clones":0,"num_object_copies":7464,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":380,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2488,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3174,"num_write_kb":150528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":312,"num_bytes_recovered":20447232,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,7],"avail_no_missing":["4","7"],"object_location_counts":[{"shards":"4,7","objects":2488}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"86'117","reported_seq":674,"reported_epoch":166,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:25:03.009705+0000","last_change":"2023-09-23T20:25:03.009705+0000","last_active":"2023-09-23T20:25:03.009705+0000","last_peered":"2023-09-23T20:25:03.009705+0000","last_clean":"2023-09-23T20:24:44.992366+0000","last_became_active":"2023-09-23T20:25:02.950112+0000","last_became_peered":"2023-09-23T20:25:02.950112+0000","last_unstale":"2023-09-23T20:25:03.009705+0000","last_undegraded":"2023-09-23T20:25:03.009482+0000","last_fullsized":"2023-09-23T20:25:02.924010+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":105,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:05:25.448625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046891400000000002,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":893,"num_read_kb":922,"num_write":82,"num_write_kb":4193,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7],"avail_no_missing":["6","2","7"],"object_location_counts":[{"shards":"2,6,7","objects":1}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":116,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-23T20:25:02.915024+0000","last_change":"2023-09-23T20:20:46.937163+0000","last_active":"2023-09-23T20:25:02.915024+0000","last_peered":"2023-09-23T20:25:02.915024+0000","last_clean":"2023-09-23T20:25:02.915024+0000","last_became_active":"2023-09-23T20:20:46.936846+0000","last_became_peered":"2023-09-23T20:20:46.936846+0000","last_unstale":"2023-09-23T20:25:02.915024+0000","last_undegraded":"2023-09-23T20:25:02.915024+0000","last_fullsized":"2023-09-23T20:25:02.915024+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:01:32.798855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":108,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-23T20:25:02.930986+0000","last_change":"2023-09-23T20:12:43.765908+0000","last_active":"2023-09-23T20:25:02.930986+0000","last_peered":"2023-09-23T20:25:02.930986+0000","last_clean":"2023-09-23T20:25:02.930986+0000","last_became_active":"2023-09-23T20:12:43.765558+0000","last_became_peered":"2023-09-23T20:12:43.765558+0000","last_unstale":"2023-09-23T20:25:02.930986+0000","last_undegraded":"2023-09-23T20:25:02.930986+0000","last_fullsized":"2023-09-23T20:25:02.930986+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:35:43.809898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.0","version":"167'5026","reported_seq":5147,"reported_epoch":167,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:25:09.921638+0000","last_change":"2023-09-23T20:25:02.978067+0000","last_active":"2023-09-23T20:25:09.921638+0000","last_peered":"2023-09-23T20:25:09.921638+0000","last_clean":"2023-09-23T20:24:36.869854+0000","last_became_active":"2023-09-23T20:25:02.971037+0000","last_became_peered":"2023-09-23T20:25:02.971037+0000","last_unstale":"2023-09-23T20:25:09.921638+0000","last_undegraded":"2023-09-23T20:25:02.913577+0000","last_fullsized":"2023-09-23T20:25:02.903982+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":111,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"119'2020","last_scrub_stamp":"2023-09-23T20:23:13.995392+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:23:13.995392+0000","objects_scrubbed":1088,"log_size":5026,"log_dups_size":0,"ondisk_log_size":5026,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:46:39.920415+0000","scrub_duration":0.22507750400000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":170459136,"num_objects":2601,"num_object_clones":0,"num_object_copies":7803,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":353,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2601,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3015,"num_write_kb":144896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[{"shards":"1,4","objects":2601}],"blocked_by":[],"up_primary":3,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":98,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-23T20:25:02.919718+0000","last_change":"2023-09-23T20:20:14.888484+0000","last_active":"2023-09-23T20:25:02.919718+0000","last_peered":"2023-09-23T20:25:02.919718+0000","last_clean":"2023-09-23T20:25:02.919718+0000","last_became_active":"2023-09-23T20:20:14.887219+0000","last_became_peered":"2023-09-23T20:20:14.887219+0000","last_unstale":"2023-09-23T20:25:02.919718+0000","last_undegraded":"2023-09-23T20:25:02.919718+0000","last_fullsized":"2023-09-23T20:25:02.919718+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:13:53.763649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":165,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-23T20:25:02.948369+0000","last_change":"2023-09-23T20:25:02.929763+0000","last_active":"2023-09-23T20:25:02.948369+0000","last_peered":"2023-09-23T20:25:02.948369+0000","last_clean":"2023-09-23T20:25:02.948369+0000","last_became_active":"2023-09-23T20:25:02.929481+0000","last_became_peered":"2023-09-23T20:25:02.929481+0000","last_unstale":"2023-09-23T20:25:02.948369+0000","last_undegraded":"2023-09-23T20:25:02.948369+0000","last_fullsized":"2023-09-23T20:25:02.948369+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:28:02.272044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":128,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-23T20:25:02.928209+0000","last_change":"2023-09-23T20:20:46.938981+0000","last_active":"2023-09-23T20:25:02.928209+0000","last_peered":"2023-09-23T20:25:02.928209+0000","last_clean":"2023-09-23T20:25:02.928209+0000","last_became_active":"2023-09-23T20:20:46.938627+0000","last_became_peered":"2023-09-23T20:20:46.938627+0000","last_unstale":"2023-09-23T20:25:02.928209+0000","last_undegraded":"2023-09-23T20:25:02.928209+0000","last_fullsized":"2023-09-23T20:25:02.928209+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:44:43.177086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":172,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-23T20:25:02.955666+0000","last_change":"2023-09-23T20:25:02.945142+0000","last_active":"2023-09-23T20:25:02.955666+0000","last_peered":"2023-09-23T20:25:02.955666+0000","last_clean":"2023-09-23T20:25:02.955666+0000","last_became_active":"2023-09-23T20:25:02.944653+0000","last_became_peered":"2023-09-23T20:25:02.944653+0000","last_unstale":"2023-09-23T20:25:02.955666+0000","last_undegraded":"2023-09-23T20:25:02.955666+0000","last_fullsized":"2023-09-23T20:25:02.955666+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:38:12.927982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":111,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-23T20:25:02.918894+0000","last_change":"2023-09-23T20:12:43.753383+0000","last_active":"2023-09-23T20:25:02.918894+0000","last_peered":"2023-09-23T20:25:02.918894+0000","last_clean":"2023-09-23T20:25:02.918894+0000","last_became_active":"2023-09-23T20:12:43.752921+0000","last_became_peered":"2023-09-23T20:12:43.752921+0000","last_unstale":"2023-09-23T20:25:02.918894+0000","last_undegraded":"2023-09-23T20:25:02.918894+0000","last_fullsized":"2023-09-23T20:25:02.918894+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:28:26.982998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":172,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-23T20:25:03.125791+0000","last_change":"2023-09-23T20:25:03.103656+0000","last_active":"2023-09-23T20:25:03.125791+0000","last_peered":"2023-09-23T20:25:03.125791+0000","last_clean":"2023-09-23T20:25:03.125791+0000","last_became_active":"2023-09-23T20:25:03.103311+0000","last_became_peered":"2023-09-23T20:25:03.103311+0000","last_unstale":"2023-09-23T20:25:03.125791+0000","last_undegraded":"2023-09-23T20:25:03.125791+0000","last_fullsized":"2023-09-23T20:25:03.125791+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:55:54.677030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.18","version":"167'5024","reported_seq":11655,"reported_epoch":167,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T20:25:09.840624+0000","last_change":"2023-09-23T20:25:03.043851+0000","last_active":"2023-09-23T20:25:09.840624+0000","last_peered":"2023-09-23T20:25:09.840624+0000","last_clean":"2023-09-23T20:23:56.635121+0000","last_became_active":"2023-09-23T20:24:00.896245+0000","last_became_peered":"2023-09-23T20:24:00.896245+0000","last_unstale":"2023-09-23T20:25:09.840624+0000","last_undegraded":"2023-09-23T20:24:00.007633+0000","last_fullsized":"2023-09-23T20:23:59.938511+0000","mapping_epoch":148,"log_start":"0'0","ondisk_log_start":"0'0","created":118,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":5,"last_scrub":"120'2147","last_scrub_stamp":"2023-09-23T20:23:18.111873+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:23:18.111873+0000","objects_scrubbed":1267,"log_size":5024,"log_dups_size":0,"ondisk_log_size":5024,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:37:58.644082+0000","scrub_duration":0.17815524799999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":167903232,"num_objects":2562,"num_object_clones":0,"num_object_copies":7686,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":257,"num_objects_misplaced":257,"num_objects_unfound":0,"num_objects_dirty":2562,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2877,"num_write_kb":133504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2169,"num_bytes_recovered":140312576,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[0,1],"avail_no_missing":["0","1","2"],"object_location_counts":[{"shards":"0,1","objects":2305}],"blocked_by":[],"up_primary":4,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":12,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-23T20:25:02.999314+0000","last_change":"2023-09-23T20:25:02.927991+0000","last_active":"2023-09-23T20:25:02.999314+0000","last_peered":"2023-09-23T20:25:02.999314+0000","last_clean":"2023-09-23T20:25:02.999314+0000","last_became_active":"2023-09-23T20:25:02.927636+0000","last_became_peered":"2023-09-23T20:25:02.927636+0000","last_unstale":"2023-09-23T20:25:02.999314+0000","last_undegraded":"2023-09-23T20:25:02.999314+0000","last_fullsized":"2023-09-23T20:25:02.999314+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:35:17.487950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":12,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-23T20:25:02.999103+0000","last_change":"2023-09-23T20:25:02.945926+0000","last_active":"2023-09-23T20:25:02.999103+0000","last_peered":"2023-09-23T20:25:02.999103+0000","last_clean":"2023-09-23T20:25:02.999103+0000","last_became_active":"2023-09-23T20:25:02.945639+0000","last_became_peered":"2023-09-23T20:25:02.945639+0000","last_unstale":"2023-09-23T20:25:02.999103+0000","last_undegraded":"2023-09-23T20:25:02.999103+0000","last_fullsized":"2023-09-23T20:25:02.999103+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:19:36.481337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.19","version":"167'4988","reported_seq":5062,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T20:25:10.376856+0000","last_change":"2023-09-23T20:23:10.978296+0000","last_active":"2023-09-23T20:25:10.376856+0000","last_peered":"2023-09-23T20:25:10.376856+0000","last_clean":"2023-09-23T20:25:10.376856+0000","last_became_active":"2023-09-23T20:23:10.977578+0000","last_became_peered":"2023-09-23T20:23:10.977578+0000","last_unstale":"2023-09-23T20:25:10.376856+0000","last_undegraded":"2023-09-23T20:25:10.376856+0000","last_fullsized":"2023-09-23T20:25:10.376856+0000","mapping_epoch":118,"log_start":"0'0","ondisk_log_start":"0'0","created":118,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:22:51.846999+0000","objects_scrubbed":0,"log_size":4988,"log_dups_size":0,"ondisk_log_size":4988,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":169836544,"num_objects":2592,"num_object_clones":0,"num_object_copies":7776,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2592,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4106,"num_write_kb":214304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":129,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-23T20:25:02.915736+0000","last_change":"2023-09-23T20:24:53.103929+0000","last_active":"2023-09-23T20:25:02.915736+0000","last_peered":"2023-09-23T20:25:02.915736+0000","last_clean":"2023-09-23T20:25:02.915736+0000","last_became_active":"2023-09-23T20:24:53.088125+0000","last_became_peered":"2023-09-23T20:24:53.088125+0000","last_unstale":"2023-09-23T20:25:02.915736+0000","last_undegraded":"2023-09-23T20:25:02.915736+0000","last_fullsized":"2023-09-23T20:25:02.915736+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:29:54.036628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":145,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-23T20:25:02.916189+0000","last_change":"2023-09-23T20:24:53.089249+0000","last_active":"2023-09-23T20:25:02.916189+0000","last_peered":"2023-09-23T20:25:02.916189+0000","last_clean":"2023-09-23T20:25:02.916189+0000","last_became_active":"2023-09-23T20:24:53.088952+0000","last_became_peered":"2023-09-23T20:24:53.088952+0000","last_unstale":"2023-09-23T20:25:02.916189+0000","last_undegraded":"2023-09-23T20:25:02.916189+0000","last_fullsized":"2023-09-23T20:25:02.916189+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:38:03.216069+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":5,"num_pg":26,"stat_sum":{"num_bytes":5419368472,"num_objects":82694,"num_object_clones":0,"num_object_copies":248082,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":29255,"num_objects_misplaced":8057,"num_objects_unfound":0,"num_objects_dirty":82694,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":115784,"num_write_kb":5850817,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":15159,"num_bytes_recovered":990969856,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":17611698176,"data_stored":17611685960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":137885,"ondisk_log_size":137885,"up":78,"acting":65,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":768,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":110592,"data_stored":17190,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":25181,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":35,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6082,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":22,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201448,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1966632,"kb_used_data":1890592,"kb_used_omap":0,"kb_used_meta":76032,"kb_avail":91778520,"statfs":{"total":95995035648,"available":93981204480,"internally_reserved":0,"allocated":1935966208,"data_stored":1935380181,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":77856768},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:24:21 2023","interfaces":[{"interface":"back","average":{"1min":1.3089999999999999,"5min":0.71399999999999997,"15min":0.62},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":4.3479999999999999,"5min":4.3479999999999999,"15min":4.3479999999999999},"last":0.27200000000000002},{"interface":"front","average":{"1min":1.714,"5min":0.79800000000000004,"15min":0.64400000000000002},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":8.3689999999999998,"5min":8.3689999999999998,"15min":8.3689999999999998},"last":0.34399999999999997}]},{"osd":1,"last update":"Sat Sep 23 20:24:56 2023","interfaces":[{"interface":"back","average":{"1min":2.1579999999999999,"5min":0.93600000000000005,"15min":0.68700000000000006},"min":{"1min":0.33300000000000002,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":17.795999999999999,"5min":17.795999999999999,"15min":17.795999999999999},"last":3.8580000000000001},{"interface":"front","average":{"1min":1.9199999999999999,"5min":0.89400000000000002,"15min":0.70599999999999996},"min":{"1min":0.38100000000000001,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":13.250999999999999,"5min":13.250999999999999,"15min":13.250999999999999},"last":0.33000000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:24:24 2023","interfaces":[{"interface":"back","average":{"1min":1.258,"5min":0.72699999999999998,"15min":0.621},"min":{"1min":0.36099999999999999,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":7.8890000000000002,"5min":7.8890000000000002,"15min":7.8890000000000002},"last":0.42299999999999999},{"interface":"front","average":{"1min":2.036,"5min":0.92000000000000004,"15min":0.69799999999999995},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":11.423999999999999,"5min":11.423999999999999,"15min":11.423999999999999},"last":3.8809999999999998}]},{"osd":3,"last update":"Sat Sep 23 20:24:56 2023","interfaces":[{"interface":"back","average":{"1min":1.238,"5min":0.86399999999999999,"15min":0.72399999999999998},"min":{"1min":0.38700000000000001,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":3.3290000000000002,"5min":3.7949999999999999,"15min":3.7949999999999999},"last":3.8319999999999999},{"interface":"front","average":{"1min":1.0089999999999999,"5min":0.75800000000000001,"15min":0.70199999999999996},"min":{"1min":0.35099999999999998,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":3.9609999999999999,"5min":3.9609999999999999,"15min":3.9609999999999999},"last":0.498}]},{"osd":4,"last update":"Sat Sep 23 20:24:56 2023","interfaces":[{"interface":"back","average":{"1min":1.3360000000000001,"5min":0.73699999999999999,"15min":0.57499999999999996},"min":{"1min":0.31900000000000001,"5min":0.27000000000000002,"15min":0.26700000000000002},"max":{"1min":13.141999999999999,"5min":13.141999999999999,"15min":13.141999999999999},"last":0.47599999999999998},{"interface":"front","average":{"1min":1.2210000000000001,"5min":0.748,"15min":0.60099999999999998},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":5.2779999999999996,"5min":5.2779999999999996,"15min":5.2779999999999996},"last":1.905}]},{"osd":5,"last update":"Sat Sep 23 20:24:56 2023","interfaces":[{"interface":"back","average":{"1min":1.4470000000000001,"5min":0.83199999999999996,"15min":0.63400000000000001},"min":{"1min":0.30299999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":3.952,"5min":4.4790000000000001,"15min":4.4790000000000001},"last":3.7890000000000001},{"interface":"front","average":{"1min":1.698,"5min":0.874,"15min":0.67700000000000005},"min":{"1min":0.374,"5min":0.34399999999999997,"15min":0.31},"max":{"1min":13.292,"5min":13.292,"15min":13.292},"last":0.54600000000000004}]},{"osd":6,"last update":"Sat Sep 23 20:24:56 2023","interfaces":[{"interface":"back","average":{"1min":1.391,"5min":0.79600000000000004,"15min":0.65600000000000003},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.34499999999999997},"max":{"1min":13.185,"5min":13.185,"15min":13.185},"last":0.52100000000000002},{"interface":"front","average":{"1min":0.90800000000000003,"5min":0.76800000000000002,"15min":0.64400000000000002},"min":{"1min":0.33000000000000002,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":4.1280000000000001,"5min":4.1280000000000001,"15min":4.1280000000000001},"last":0.53300000000000003}]}]},{"osd":6,"up_from":47,"seq":201863463084,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1583524,"kb_used_data":1527388,"kb_used_omap":0,"kb_used_meta":56128,"kb_avail":92161628,"statfs":{"total":95995035648,"available":94373507072,"internally_reserved":0,"allocated":1564045312,"data_stored":1563463818,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":57475072},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:24:21 2023","interfaces":[{"interface":"back","average":{"1min":1.716,"5min":0.82299999999999995,"15min":0.65900000000000003},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":7.3849999999999998,"5min":7.3849999999999998,"15min":7.3849999999999998},"last":0.47799999999999998},{"interface":"front","average":{"1min":1.8660000000000001,"5min":0.86399999999999999,"15min":0.66300000000000003},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":6.9420000000000002,"5min":6.9420000000000002,"15min":6.9420000000000002},"last":2.7639999999999998}]},{"osd":1,"last update":"Sat Sep 23 20:24:57 2023","interfaces":[{"interface":"back","average":{"1min":2.383,"5min":1.0509999999999999,"15min":0.73299999999999998},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":18.001000000000001,"5min":18.001000000000001,"15min":18.001000000000001},"last":4.8810000000000002},{"interface":"front","average":{"1min":1.2549999999999999,"5min":0.76900000000000002,"15min":0.621},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":4.0880000000000001,"5min":4.0880000000000001,"15min":4.0880000000000001},"last":4.819}]},{"osd":2,"last update":"Sat Sep 23 20:24:21 2023","interfaces":[{"interface":"back","average":{"1min":1.1890000000000001,"5min":0.72099999999999997,"15min":0.63700000000000001},"min":{"1min":0.379,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":4.1289999999999996,"5min":4.1289999999999996,"15min":4.1289999999999996},"last":0.96199999999999997},{"interface":"front","average":{"1min":1.3859999999999999,"5min":0.79000000000000004,"15min":0.66900000000000004},"min":{"1min":0.41499999999999998,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":5.2220000000000004,"5min":5.2220000000000004,"15min":5.2220000000000004},"last":0.45000000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:24:57 2023","interfaces":[{"interface":"back","average":{"1min":2.105,"5min":1.0209999999999999,"15min":0.749},"min":{"1min":0.39000000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":7.9359999999999999,"5min":7.9359999999999999,"15min":7.9359999999999999},"last":1.0429999999999999},{"interface":"front","average":{"1min":1.615,"5min":1.022,"15min":0.79200000000000004},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":4.3940000000000001,"5min":8.6050000000000004,"15min":8.6050000000000004},"last":1.077}]},{"osd":4,"last update":"Sat Sep 23 20:24:43 2023","interfaces":[{"interface":"back","average":{"1min":1.2929999999999999,"5min":0.67000000000000004,"15min":0.53100000000000003},"min":{"1min":0.29399999999999998,"5min":0.28299999999999997,"15min":0.26200000000000001},"max":{"1min":5.1379999999999999,"5min":5.1379999999999999,"15min":5.1379999999999999},"last":0.39100000000000001},{"interface":"front","average":{"1min":1.5920000000000001,"5min":0.76300000000000001,"15min":0.57999999999999996},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.28499999999999998},"max":{"1min":4.8280000000000003,"5min":4.8280000000000003,"15min":4.8280000000000003},"last":0.995}]},{"osd":5,"last update":"Sat Sep 23 20:24:43 2023","interfaces":[{"interface":"back","average":{"1min":1.966,"5min":0.82199999999999995,"15min":0.60399999999999998},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.29799999999999999},"max":{"1min":9.6669999999999998,"5min":9.6669999999999998,"15min":9.6669999999999998},"last":0.54500000000000004},{"interface":"front","average":{"1min":1.7050000000000001,"5min":0.84599999999999997,"15min":0.626},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.29499999999999998},"max":{"1min":6.9109999999999996,"5min":6.9109999999999996,"15min":6.9109999999999996},"last":1.024}]},{"osd":7,"last update":"Sat Sep 23 20:24:57 2023","interfaces":[{"interface":"back","average":{"1min":1.367,"5min":0.77500000000000002,"15min":0.61699999999999999},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.32800000000000001},"max":{"1min":5.4589999999999996,"5min":5.4589999999999996,"15min":5.4589999999999996},"last":0.42799999999999999},{"interface":"front","average":{"1min":1.4930000000000001,"5min":0.86299999999999999,"15min":0.66200000000000003},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.315},"max":{"1min":7.3979999999999997,"5min":7.3979999999999997,"15min":7.3979999999999997},"last":1.0589999999999999}]}]},{"osd":1,"up_from":99,"seq":425201762359,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2130336,"kb_used_data":2000216,"kb_used_omap":6,"kb_used_meta":130105,"kb_avail":91614816,"statfs":{"total":95995035648,"available":93813571584,"internally_reserved":0,"allocated":2048221184,"data_stored":2047413387,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6696,"internal_metadata":133227992},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:24:51 2023","interfaces":[{"interface":"back","average":{"1min":5.056,"5min":1.4119999999999999,"15min":0.72199999999999998},"min":{"1min":0.247,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":40.113,"5min":40.113,"15min":40.113},"last":0.215},{"interface":"front","average":{"1min":3.2909999999999999,"5min":1.0429999999999999,"15min":0.61399999999999999},"min":{"1min":0.22900000000000001,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":42.994999999999997,"5min":42.994999999999997,"15min":42.994999999999997},"last":0.311}]},{"osd":2,"last update":"Sat Sep 23 20:24:22 2023","interfaces":[{"interface":"back","average":{"1min":1.569,"5min":0.78600000000000003,"15min":0.51600000000000001},"min":{"1min":0.27900000000000003,"5min":0.253,"15min":0.253},"max":{"1min":8.827,"5min":8.827,"15min":8.827},"last":0.35499999999999998},{"interface":"front","average":{"1min":0.94999999999999996,"5min":0.72799999999999998,"15min":0.60999999999999999},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":5.5300000000000002,"5min":7.5389999999999997,"15min":7.5389999999999997},"last":0.34000000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:24:57 2023","interfaces":[{"interface":"back","average":{"1min":2.3490000000000002,"5min":0.98099999999999998,"15min":0.69299999999999995},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":8.7420000000000009,"5min":8.7420000000000009,"15min":8.7420000000000009},"last":0.29199999999999998},{"interface":"front","average":{"1min":1.3740000000000001,"5min":0.71599999999999997,"15min":0.51000000000000001},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":9.1859999999999999,"5min":9.1859999999999999,"15min":9.1859999999999999},"last":0.26000000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:24:51 2023","interfaces":[{"interface":"back","average":{"1min":1.8200000000000001,"5min":0.91600000000000004,"15min":0.71599999999999997},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":7.601,"5min":7.601,"15min":7.601},"last":0.36899999999999999},{"interface":"front","average":{"1min":2.7000000000000002,"5min":1.0840000000000001,"15min":0.78400000000000003},"min":{"1min":0.42599999999999999,"5min":0.312,"15min":0.312},"max":{"1min":8.7899999999999991,"5min":8.7899999999999991,"15min":8.7899999999999991},"last":4.2690000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:24:51 2023","interfaces":[{"interface":"back","average":{"1min":1.9350000000000001,"5min":0.97899999999999998,"15min":0.753},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":12.590999999999999,"5min":12.590999999999999,"15min":12.590999999999999},"last":0.42899999999999999},{"interface":"front","average":{"1min":4.7329999999999997,"5min":1.5409999999999999,"15min":0.96399999999999997},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":39.579000000000001,"5min":39.579000000000001,"15min":39.579000000000001},"last":0.437}]},{"osd":6,"last update":"Sat Sep 23 20:24:51 2023","interfaces":[{"interface":"back","average":{"1min":1.0209999999999999,"5min":0.76800000000000002,"15min":0.70099999999999996},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":6.6669999999999998,"5min":6.6669999999999998,"15min":6.6669999999999998},"last":1.091},{"interface":"front","average":{"1min":2.6469999999999998,"5min":1.087,"15min":0.78700000000000003},"min":{"1min":0.443,"5min":0.377,"15min":0.377},"max":{"1min":9.0920000000000005,"5min":9.0920000000000005,"15min":9.0920000000000005},"last":3.419}]},{"osd":7,"last update":"Sat Sep 23 20:24:51 2023","interfaces":[{"interface":"back","average":{"1min":2.4020000000000001,"5min":1.0700000000000001,"15min":0.79100000000000004},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":7.5469999999999997,"5min":7.5469999999999997,"15min":7.5469999999999997},"last":1.2769999999999999},{"interface":"front","average":{"1min":2.5249999999999999,"5min":1.125,"15min":0.83099999999999996},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":8.859,"5min":8.859,"15min":8.859},"last":3.6080000000000001}]}]},{"osd":0,"up_from":94,"seq":403726925885,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2653076,"kb_used_data":2535180,"kb_used_omap":7,"kb_used_meta":117816,"kb_avail":91092076,"statfs":{"total":95995035648,"available":93278285824,"internally_reserved":0,"allocated":2596024320,"data_stored":2595228314,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":120643601},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:24:55 2023","interfaces":[{"interface":"back","average":{"1min":1.397,"5min":0.63200000000000001,"15min":0.45000000000000001},"min":{"1min":0.20499999999999999,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":5.3609999999999998,"5min":5.3609999999999998,"15min":5.3609999999999998},"last":0.219},{"interface":"front","average":{"1min":1.5329999999999999,"5min":0.72699999999999998,"15min":0.499},"min":{"1min":0.248,"5min":0.222,"15min":0.222},"max":{"1min":8.9770000000000003,"5min":8.9770000000000003,"15min":8.9770000000000003},"last":0.23499999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:24:21 2023","interfaces":[{"interface":"back","average":{"1min":1.641,"5min":0.81599999999999995,"15min":0.67000000000000004},"min":{"1min":0.30199999999999999,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":5.1429999999999998,"5min":5.1429999999999998,"15min":5.1429999999999998},"last":0.254},{"interface":"front","average":{"1min":2.3690000000000002,"5min":0.92100000000000004,"15min":0.63700000000000001},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":9.1940000000000008,"5min":9.1940000000000008,"15min":9.1940000000000008},"last":0.31900000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:24:55 2023","interfaces":[{"interface":"back","average":{"1min":1.998,"5min":0.91200000000000003,"15min":0.70199999999999996},"min":{"1min":0.32300000000000001,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":5.2080000000000002,"5min":5.2080000000000002,"15min":5.2080000000000002},"last":0.28799999999999998},{"interface":"front","average":{"1min":1.2070000000000001,"5min":0.624,"15min":0.48899999999999999},"min":{"1min":0.28000000000000003,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":5.4210000000000003,"5min":5.4210000000000003,"15min":5.4210000000000003},"last":0.38700000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:24:21 2023","interfaces":[{"interface":"back","average":{"1min":1.409,"5min":0.80500000000000005,"15min":0.66700000000000004},"min":{"1min":0.40200000000000002,"5min":0.37,"15min":0.37},"max":{"1min":4.8490000000000002,"5min":4.8490000000000002,"15min":4.8490000000000002},"last":1.466},{"interface":"front","average":{"1min":1.145,"5min":0.72599999999999998,"15min":0.65100000000000002},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":5.2670000000000003,"5min":5.2670000000000003,"15min":5.2670000000000003},"last":6.2370000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:24:21 2023","interfaces":[{"interface":"back","average":{"1min":2.7570000000000001,"5min":1.079,"15min":0.79900000000000004},"min":{"1min":0.39800000000000002,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":11.224,"5min":11.224,"15min":11.224},"last":10.868},{"interface":"front","average":{"1min":2.8330000000000002,"5min":1.103,"15min":0.81999999999999995},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":11.167999999999999,"5min":11.167999999999999,"15min":11.167999999999999},"last":10.823}]},{"osd":6,"last update":"Sat Sep 23 20:24:21 2023","interfaces":[{"interface":"back","average":{"1min":0.98799999999999999,"5min":0.73999999999999999,"15min":0.67900000000000005},"min":{"1min":0.49199999999999999,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":5.3920000000000003,"5min":5.3920000000000003,"15min":5.3920000000000003},"last":1.3500000000000001},{"interface":"front","average":{"1min":1.1240000000000001,"5min":0.76400000000000001,"15min":0.69599999999999995},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":5.3410000000000002,"5min":5.3410000000000002,"15min":5.3410000000000002},"last":1.383}]},{"osd":7,"last update":"Sat Sep 23 20:24:21 2023","interfaces":[{"interface":"back","average":{"1min":1.4419999999999999,"5min":0.82999999999999996,"15min":0.72899999999999998},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":5.4219999999999997,"5min":5.4219999999999997,"15min":5.4219999999999997},"last":7.3079999999999998},{"interface":"front","average":{"1min":1.51,"5min":0.874,"15min":0.73399999999999999},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":5.181,"5min":5.181,"15min":5.181},"last":7.3230000000000004}]}]},{"osd":2,"up_from":0,"seq":446676598833,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":109,"seq":468151435306,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1773340,"kb_used_data":1661332,"kb_used_omap":4,"kb_used_meta":111995,"kb_avail":91971812,"statfs":{"total":95995035648,"available":94179135488,"internally_reserved":0,"allocated":1701203968,"data_stored":1700456324,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4514,"internal_metadata":114683486},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:24:53 2023","interfaces":[{"interface":"back","average":{"1min":1.72,"5min":0.93200000000000005,"15min":0.54500000000000004},"min":{"1min":0.23599999999999999,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":6.3730000000000002,"5min":17.239000000000001,"15min":17.239000000000001},"last":0.27100000000000002},{"interface":"front","average":{"1min":1.75,"5min":0.97699999999999998,"15min":0.58199999999999996},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":8.0259999999999998,"5min":17.297000000000001,"15min":17.297000000000001},"last":0.36499999999999999}]},{"osd":1,"last update":"Sat Sep 23 20:24:53 2023","interfaces":[{"interface":"back","average":{"1min":1.046,"5min":0.79700000000000004,"15min":0.53000000000000003},"min":{"1min":0.22900000000000001,"5min":0.214,"15min":0.214},"max":{"1min":3.7570000000000001,"5min":11.537000000000001,"15min":11.537000000000001},"last":0.38100000000000001},{"interface":"front","average":{"1min":0.94299999999999995,"5min":0.78500000000000003,"15min":0.55800000000000005},"min":{"1min":0.29899999999999999,"5min":0.245,"15min":0.245},"max":{"1min":2.7879999999999998,"5min":11.5,"15min":11.5},"last":0.41599999999999998}]},{"osd":2,"last update":"Sat Sep 23 20:24:53 2023","interfaces":[{"interface":"back","average":{"1min":1.798,"5min":1.105,"15min":0.60299999999999998},"min":{"1min":0.26900000000000002,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":10.244,"5min":17.335000000000001,"15min":17.335000000000001},"last":0.30199999999999999},{"interface":"front","average":{"1min":1.7709999999999999,"5min":1.0269999999999999,"15min":0.65100000000000002},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":10.185,"5min":17.684000000000001,"15min":17.684000000000001},"last":0.32500000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:24:53 2023","interfaces":[{"interface":"back","average":{"1min":1.3939999999999999,"5min":0.82199999999999995,"15min":0.67700000000000005},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":5.5309999999999997,"5min":5.5309999999999997,"15min":5.5309999999999997},"last":0.48199999999999998},{"interface":"front","average":{"1min":1.024,"5min":0.75,"15min":0.66000000000000003},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":2.7509999999999999,"5min":2.895,"15min":2.895},"last":1.95}]},{"osd":5,"last update":"Sat Sep 23 20:24:53 2023","interfaces":[{"interface":"back","average":{"1min":1.6499999999999999,"5min":0.99199999999999999,"15min":0.73499999999999999},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":4.3620000000000001,"5min":7.0490000000000004,"15min":7.0490000000000004},"last":0.68000000000000005},{"interface":"front","average":{"1min":0.95999999999999996,"5min":0.87,"15min":0.70999999999999996},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":2.802,"5min":6.4450000000000003,"15min":6.4450000000000003},"last":2.9340000000000002}]},{"osd":6,"last update":"Sat Sep 23 20:24:53 2023","interfaces":[{"interface":"back","average":{"1min":0.99399999999999999,"5min":0.873,"15min":0.71599999999999997},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":3.3799999999999999,"5min":5.5819999999999999,"15min":5.5819999999999999},"last":0.96699999999999997},{"interface":"front","average":{"1min":1.103,"5min":0.81899999999999995,"15min":0.72099999999999997},"min":{"1min":0.39900000000000002,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":3.863,"5min":3.863,"15min":3.863},"last":0.89300000000000002}]},{"osd":7,"last update":"Sat Sep 23 20:24:53 2023","interfaces":[{"interface":"back","average":{"1min":1.3500000000000001,"5min":0.88800000000000001,"15min":0.76400000000000001},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":4.9089999999999998,"5min":4.9089999999999998,"15min":4.9089999999999998},"last":0.73899999999999999},{"interface":"front","average":{"1min":1.899,"5min":1.0529999999999999,"15min":0.77900000000000003},"min":{"1min":0.42999999999999999,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":8.6180000000000003,"5min":8.6180000000000003,"15min":8.6180000000000003},"last":0.50600000000000001}]}]},{"osd":4,"up_from":30,"seq":128849019060,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2763608,"kb_used_data":2681424,"kb_used_omap":0,"kb_used_meta":82048,"kb_avail":90981544,"statfs":{"total":95995035648,"available":93165101056,"internally_reserved":0,"allocated":2745778176,"data_stored":2745207572,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":84017152},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:24:19 2023","interfaces":[{"interface":"back","average":{"1min":2.0649999999999999,"5min":0.872,"15min":0.67000000000000004},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":20.411999999999999,"5min":20.411999999999999,"15min":20.411999999999999},"last":0.40300000000000002},{"interface":"front","average":{"1min":2.1389999999999998,"5min":0.92800000000000005,"15min":0.73699999999999999},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":20.454999999999998,"5min":20.454999999999998,"15min":20.454999999999998},"last":0.29199999999999998}]},{"osd":1,"last update":"Sat Sep 23 20:24:56 2023","interfaces":[{"interface":"back","average":{"1min":1.0489999999999999,"5min":0.70399999999999996,"15min":0.65200000000000002},"min":{"1min":0.28000000000000003,"5min":0.255,"15min":0.255},"max":{"1min":6.8070000000000004,"5min":6.8070000000000004,"15min":6.8070000000000004},"last":0.38900000000000001},{"interface":"front","average":{"1min":1.3160000000000001,"5min":0.84399999999999997,"15min":0.69799999999999995},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":4.6799999999999997,"5min":7.1230000000000002,"15min":7.1230000000000002},"last":0.44400000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:24:26 2023","interfaces":[{"interface":"back","average":{"1min":2.2869999999999999,"5min":0.95899999999999996,"15min":0.72499999999999998},"min":{"1min":0.36199999999999999,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":21.462,"5min":21.462,"15min":21.462},"last":0.46899999999999997},{"interface":"front","average":{"1min":2.1469999999999998,"5min":0.98199999999999998,"15min":0.71999999999999997},"min":{"1min":0.40000000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":22.593,"5min":22.593,"15min":22.593},"last":0.33000000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:24:56 2023","interfaces":[{"interface":"back","average":{"1min":2.577,"5min":1.056,"15min":0.77900000000000003},"min":{"1min":0.33200000000000002,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":24.891999999999999,"5min":24.891999999999999,"15min":24.891999999999999},"last":0.51900000000000002},{"interface":"front","average":{"1min":2.4319999999999999,"5min":1.0309999999999999,"15min":0.78700000000000003},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":21.922000000000001,"5min":21.922000000000001,"15min":21.922000000000001},"last":0.49199999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:25:09 2023","interfaces":[{"interface":"back","average":{"1min":1.0449999999999999,"5min":0.77400000000000002,"15min":0.56000000000000005},"min":{"1min":0.27400000000000002,"5min":0.26800000000000002,"15min":0.26000000000000001},"max":{"1min":5.2699999999999996,"5min":8.0109999999999992,"15min":8.0109999999999992},"last":5.3659999999999997},{"interface":"front","average":{"1min":1.7050000000000001,"5min":0.83899999999999997,"15min":0.59499999999999997},"min":{"1min":0.29599999999999999,"5min":0.28000000000000003,"15min":0.248},"max":{"1min":8.016,"5min":8.016,"15min":8.016},"last":6.8710000000000004}]},{"osd":6,"last update":"Sat Sep 23 20:24:32 2023","interfaces":[{"interface":"back","average":{"1min":2.601,"5min":0.95799999999999996,"15min":0.64800000000000002},"min":{"1min":0.33100000000000002,"5min":0.32900000000000001,"15min":0.27500000000000002},"max":{"1min":41.371000000000002,"5min":41.371000000000002,"15min":41.371000000000002},"last":0.57499999999999996},{"interface":"front","average":{"1min":2.395,"5min":0.95799999999999996,"15min":0.65000000000000002},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":41.140999999999998,"5min":41.140999999999998,"15min":41.140999999999998},"last":0.42199999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:24:42 2023","interfaces":[{"interface":"back","average":{"1min":1.611,"5min":0.80900000000000005,"15min":0.63200000000000001},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.33600000000000002},"max":{"1min":8.1910000000000007,"5min":8.1910000000000007,"15min":8.1910000000000007},"last":2.0619999999999998},{"interface":"front","average":{"1min":1.4430000000000001,"5min":0.78900000000000003,"15min":0.629},"min":{"1min":0.39500000000000002,"5min":0.34300000000000003,"15min":0.29899999999999999},"max":{"1min":5.1909999999999998,"5min":5.1909999999999998,"15min":5.1909999999999998},"last":1.9419999999999999}]}]},{"osd":5,"up_from":39,"seq":167503724721,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3008468,"kb_used_data":2917836,"kb_used_omap":0,"kb_used_meta":90432,"kb_avail":90736684,"statfs":{"total":95995035648,"available":92914364416,"internally_reserved":0,"allocated":2987864064,"data_stored":2987293666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":92602368},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:24:19 2023","interfaces":[{"interface":"back","average":{"1min":1.2470000000000001,"5min":0.73499999999999999,"15min":0.63900000000000001},"min":{"1min":0.252,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":3.6909999999999998,"5min":3.6909999999999998,"15min":3.6909999999999998},"last":1.095},{"interface":"front","average":{"1min":1.9570000000000001,"5min":0.90400000000000003,"15min":0.70199999999999996},"min":{"1min":0.20599999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":12.215,"5min":12.215,"15min":12.215},"last":3.165}]},{"osd":1,"last update":"Sat Sep 23 20:24:55 2023","interfaces":[{"interface":"back","average":{"1min":2.9529999999999998,"5min":1.0589999999999999,"15min":0.71599999999999997},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":32.243000000000002,"5min":32.243000000000002,"15min":32.243000000000002},"last":0.245},{"interface":"front","average":{"1min":3.0219999999999998,"5min":1.101,"15min":0.73099999999999998},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":32.231000000000002,"5min":32.231000000000002,"15min":32.231000000000002},"last":0.32400000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:24:19 2023","interfaces":[{"interface":"back","average":{"1min":2.2389999999999999,"5min":0.95299999999999996,"15min":0.68400000000000005},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":13.834,"5min":13.834,"15min":13.834},"last":0.46600000000000003},{"interface":"front","average":{"1min":2.3769999999999998,"5min":0.97299999999999998,"15min":0.69699999999999995},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":12.284000000000001,"5min":12.284000000000001,"15min":12.284000000000001},"last":0.35699999999999998}]},{"osd":3,"last update":"Sat Sep 23 20:24:55 2023","interfaces":[{"interface":"back","average":{"1min":1.736,"5min":0.89000000000000001,"15min":0.66800000000000004},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":7.3799999999999999,"5min":7.3799999999999999,"15min":7.3799999999999999},"last":0.42399999999999999},{"interface":"front","average":{"1min":1.7789999999999999,"5min":0.88100000000000001,"15min":0.70399999999999996},"min":{"1min":0.33400000000000002,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":8.7260000000000009,"5min":8.7260000000000009,"15min":8.7260000000000009},"last":0.96999999999999997}]},{"osd":4,"last update":"Sat Sep 23 20:24:15 2023","interfaces":[{"interface":"back","average":{"1min":0.97599999999999998,"5min":0.65700000000000003,"15min":0.50600000000000001},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.26400000000000001},"max":{"1min":8.0079999999999991,"5min":8.0079999999999991,"15min":8.0079999999999991},"last":0.63900000000000001},{"interface":"front","average":{"1min":0.96799999999999997,"5min":0.67900000000000005,"15min":0.53800000000000003},"min":{"1min":0.34499999999999997,"5min":0.29299999999999998,"15min":0.28799999999999998},"max":{"1min":2.8929999999999998,"5min":2.8929999999999998,"15min":2.8929999999999998},"last":0.308}]},{"osd":6,"last update":"Sat Sep 23 20:24:27 2023","interfaces":[{"interface":"back","average":{"1min":1.8560000000000001,"5min":0.85199999999999998,"15min":0.60799999999999998},"min":{"1min":0.35299999999999998,"5min":0.30499999999999999,"15min":0.29799999999999999},"max":{"1min":11.545999999999999,"5min":11.545999999999999,"15min":11.545999999999999},"last":0.45300000000000001},{"interface":"front","average":{"1min":1.712,"5min":0.88300000000000001,"15min":0.66200000000000003},"min":{"1min":0.39900000000000002,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":7.8200000000000003,"5min":7.8200000000000003,"15min":7.8200000000000003},"last":0.40200000000000002}]},{"osd":7,"last update":"Sat Sep 23 20:24:55 2023","interfaces":[{"interface":"back","average":{"1min":1.202,"5min":0.78900000000000003,"15min":0.61199999999999999},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.33800000000000002},"max":{"1min":5.0469999999999997,"5min":5.0469999999999997,"15min":5.0469999999999997},"last":0.433},{"interface":"front","average":{"1min":1.641,"5min":0.86299999999999999,"15min":0.64500000000000002},"min":{"1min":0.371,"5min":0.315,"15min":0.315},"max":{"1min":12.368,"5min":12.368,"15min":12.368},"last":3.2120000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4722,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1360,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6696,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7156,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3154,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2573074432,"data_stored":2573074432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2039021568,"data_stored":2039021568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2112032768,"data_stored":2112028696,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1694040064,"data_stored":1694040064,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2743271424,"data_stored":2743271424,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2975535104,"data_stored":2975531032,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1545666560,"data_stored":1545666560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1929056256,"data_stored":1929052184,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:25:13.668 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-09-23T20:25:13.668 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd pool set .mgr pg_num 11 2023-09-23T20:25:13.673 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:25:13.677 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:25:13.677 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:25:13.677 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:25:13.677 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:25:13.677 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:25:13.677 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:25:13.677 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:25:13.677 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:25:13.729 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:25:13.789 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:13.840 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:25:13.842 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:13.842 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:25:13.943 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:25:14.010 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:14.127 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2023-09-23T20:25:14.127 INFO:teuthology.orchestra.run.smithi018.stdout: "ops": [], 2023-09-23T20:25:14.127 INFO:teuthology.orchestra.run.smithi018.stdout: "complaint_time": 30, 2023-09-23T20:25:14.127 INFO:teuthology.orchestra.run.smithi018.stdout: "num_blocked_ops": 0 2023-09-23T20:25:14.127 INFO:teuthology.orchestra.run.smithi018.stdout:} 2023-09-23T20:25:14.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:14 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/808378543' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:25:14.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:14 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/808378543' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:25:14.264 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:25:14.269 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:14.269 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:25:14.370 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:25:14.416 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:14.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:14 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/808378543' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:25:14.516 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:25:14.520 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:14.520 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:25:14.609 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T20:25:14.621 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:25:14.646 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T20:25:14.657 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:14.718 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:25:14.725 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:25:14.727 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:14.727 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:25:14.828 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:25:14.861 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:14.956 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:25:14.958 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:14.959 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:25:15.060 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:25:15.094 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:15.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:15 smithi018 ceph-mon[184144]: from='client.34995 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:25:15.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:15 smithi018 ceph-mon[184144]: pgmap v358: 99 pgs: 1 active+recovery_wait+degraded, 82 active+clean, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped; 5.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 574 op/s; 29290/248166 objects degraded (11.803%); 8057/248166 objects misplaced (3.247%); 10 MiB/s, 171 objects/s recovering 2023-09-23T20:25:15.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:15 smithi018 ceph-mon[184144]: osdmap e169: 8 total, 8 up, 7 in 2023-09-23T20:25:15.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:15 smithi018 ceph-mon[184144]: failed to encode map e169 with expected crc 2023-09-23T20:25:15.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:15 smithi018 ceph-mon[184144]: failed to encode map e169 with expected crc 2023-09-23T20:25:15.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:15 smithi018 ceph-mon[184144]: failed to encode map e169 with expected crc 2023-09-23T20:25:15.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:15 smithi018 ceph-mon[184144]: failed to encode map e169 with expected crc 2023-09-23T20:25:15.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:15 smithi018 ceph-mon[184144]: failed to encode map e169 with expected crc 2023-09-23T20:25:15.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:15 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:25:15.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:15 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:25:15.392 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:25:14 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:25:14] "GET /metrics HTTP/1.1" 200 36502 "" "Prometheus/2.43.0" 2023-09-23T20:25:15.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:15 smithi018 ceph-mon[179647]: from='client.34995 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:25:15.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:15 smithi018 ceph-mon[179647]: pgmap v358: 99 pgs: 1 active+recovery_wait+degraded, 82 active+clean, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped; 5.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 574 op/s; 29290/248166 objects degraded (11.803%); 8057/248166 objects misplaced (3.247%); 10 MiB/s, 171 objects/s recovering 2023-09-23T20:25:15.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:15 smithi018 ceph-mon[179647]: osdmap e169: 8 total, 8 up, 7 in 2023-09-23T20:25:15.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:15 smithi018 ceph-mon[179647]: failed to encode map e169 with expected crc 2023-09-23T20:25:15.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:15 smithi018 ceph-mon[179647]: failed to encode map e169 with expected crc 2023-09-23T20:25:15.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:15 smithi018 ceph-mon[179647]: failed to encode map e169 with expected crc 2023-09-23T20:25:15.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:15 smithi018 ceph-mon[179647]: failed to encode map e169 with expected crc 2023-09-23T20:25:15.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:15 smithi018 ceph-mon[179647]: failed to encode map e169 with expected crc 2023-09-23T20:25:15.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:15 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:25:15.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:15 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:25:15.474 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:15 smithi149 ceph-mon[170394]: from='client.34995 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:25:15.474 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:15 smithi149 ceph-mon[170394]: pgmap v358: 99 pgs: 1 active+recovery_wait+degraded, 82 active+clean, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 13 active+recovery_wait+undersized+degraded+remapped; 5.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 574 op/s; 29290/248166 objects degraded (11.803%); 8057/248166 objects misplaced (3.247%); 10 MiB/s, 171 objects/s recovering 2023-09-23T20:25:15.475 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:15 smithi149 ceph-mon[170394]: osdmap e169: 8 total, 8 up, 7 in 2023-09-23T20:25:15.475 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:15 smithi149 ceph-mon[170394]: failed to encode map e169 with expected crc 2023-09-23T20:25:15.475 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:15 smithi149 ceph-mon[170394]: failed to encode map e169 with expected crc 2023-09-23T20:25:15.475 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:15 smithi149 ceph-mon[170394]: failed to encode map e169 with expected crc 2023-09-23T20:25:15.475 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:15 smithi149 ceph-mon[170394]: failed to encode map e169 with expected crc 2023-09-23T20:25:15.475 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:15 smithi149 ceph-mon[170394]: failed to encode map e169 with expected crc 2023-09-23T20:25:15.475 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:15 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:25:15.475 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:15 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:25:15.499 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:25:15.504 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:15.506 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:25:15.606 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:25:15.643 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:15.788 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:25:15.792 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:15.793 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:25:15.893 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:25:15.908 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:25:15.948 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:15.955 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:25:16.061 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:16 smithi018 ceph-mon[184144]: failed to encode map e170 with expected crc 2023-09-23T20:25:16.061 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:16 smithi018 ceph-mon[184144]: osdmap e170: 8 total, 8 up, 7 in 2023-09-23T20:25:16.061 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:16 smithi018 ceph-mon[184144]: failed to encode map e170 with expected crc 2023-09-23T20:25:16.061 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:16 smithi018 ceph-mon[184144]: failed to encode map e170 with expected crc 2023-09-23T20:25:16.061 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:16 smithi018 ceph-mon[184144]: failed to encode map e170 with expected crc 2023-09-23T20:25:16.061 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:16 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 29290/248166 objects degraded (11.803%), 16 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T20:25:16.062 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:16 smithi018 ceph-mon[179647]: failed to encode map e170 with expected crc 2023-09-23T20:25:16.062 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:16 smithi018 ceph-mon[179647]: osdmap e170: 8 total, 8 up, 7 in 2023-09-23T20:25:16.062 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:16 smithi018 ceph-mon[179647]: failed to encode map e170 with expected crc 2023-09-23T20:25:16.062 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:16 smithi018 ceph-mon[179647]: failed to encode map e170 with expected crc 2023-09-23T20:25:16.062 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:16 smithi018 ceph-mon[179647]: failed to encode map e170 with expected crc 2023-09-23T20:25:16.062 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:16 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 29290/248166 objects degraded (11.803%), 16 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T20:25:16.341 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:25:16.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:16 smithi149 ceph-mon[170394]: failed to encode map e170 with expected crc 2023-09-23T20:25:16.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:16 smithi149 ceph-mon[170394]: osdmap e170: 8 total, 8 up, 7 in 2023-09-23T20:25:16.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:16 smithi149 ceph-mon[170394]: failed to encode map e170 with expected crc 2023-09-23T20:25:16.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:16 smithi149 ceph-mon[170394]: failed to encode map e170 with expected crc 2023-09-23T20:25:16.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:16 smithi149 ceph-mon[170394]: failed to encode map e170 with expected crc 2023-09-23T20:25:16.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:16 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 29290/248166 objects degraded (11.803%), 16 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T20:25:16.790 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:25:16.794 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:16.795 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:25:16.895 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:25:16.993 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:17.047 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:25:17.055 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:17.055 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:25:17.156 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:25:17.203 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:17.385 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:17 smithi018 ceph-mon[184144]: pgmap v361: 99 pgs: 1 active+recovery_wait+degraded, 82 active+clean, 1 active+remapped+backfill_wait, 3 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped; 4.9 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 1.03k op/s; 26365/240105 objects degraded (10.981%); 10079/240105 objects misplaced (4.198%); 10 MiB/s, 169 objects/s recovering 2023-09-23T20:25:17.386 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:17 smithi018 ceph-mon[179647]: pgmap v361: 99 pgs: 1 active+recovery_wait+degraded, 82 active+clean, 1 active+remapped+backfill_wait, 3 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped; 4.9 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 1.03k op/s; 26365/240105 objects degraded (10.981%); 10079/240105 objects misplaced (4.198%); 10 MiB/s, 169 objects/s recovering 2023-09-23T20:25:17.419 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:25:17.434 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:17.434 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:25:17.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:17 smithi149 ceph-mon[170394]: pgmap v361: 99 pgs: 1 active+recovery_wait+degraded, 82 active+clean, 1 active+remapped+backfill_wait, 3 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped; 4.9 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 1.03k op/s; 26365/240105 objects degraded (10.981%); 10079/240105 objects misplaced (4.198%); 10 MiB/s, 169 objects/s recovering 2023-09-23T20:25:17.535 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:25:17.655 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:17.725 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:25:17.736 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:17.736 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:25:17.837 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:25:17.893 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:18.018 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:25:18.022 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:18.022 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:25:18.048 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:25:18.100 INFO:teuthology.orchestra.run.smithi018.stderr:set pool 1 pg_num to 11 2023-09-23T20:25:18.124 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:25:18.181 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:18.246 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:25:18.252 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:18.253 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:25:18.260 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:18 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3616411178' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]: dispatch 2023-09-23T20:25:18.260 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:18 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]: dispatch 2023-09-23T20:25:18.260 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:18 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3616411178' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]: dispatch 2023-09-23T20:25:18.261 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:18 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]: dispatch 2023-09-23T20:25:18.354 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:25:18.393 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:18.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:18 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3616411178' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]: dispatch 2023-09-23T20:25:18.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:18 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]: dispatch 2023-09-23T20:25:18.486 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:25:18.492 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:18.492 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:25:18.593 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:25:18.635 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2023-09-23T20:25:18.636 INFO:teuthology.orchestra.run.smithi018.stdout: "size": 20, 2023-09-23T20:25:18.636 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 600, 2023-09-23T20:25:18.636 INFO:teuthology.orchestra.run.smithi018.stdout: "ops": [ 2023-09-23T20:25:18.636 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.636 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.34791.0:28889 5.10 5:0f2ad13f:::benchmark_data_smithi018_228451_object28888:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e118)", 2023-09-23T20:25:18.636 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:23:09.972687+0000", 2023-09-23T20:25:18.636 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 128.66078303500001, 2023-09-23T20:25:18.636 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0965630740000001, 2023-09-23T20:25:18.636 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:25:18.636 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:25:18.636 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:25:18.636 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.34791", 2023-09-23T20:25:18.637 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:25:18.637 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 28889 2023-09-23T20:25:18.637 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.637 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:25:18.637 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.637 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:25:18.637 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.972687+0000", 2023-09-23T20:25:18.637 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:25:18.637 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.637 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.637 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:25:18.637 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.972687+0000", 2023-09-23T20:25:18.637 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:25:18.638 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.638 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.638 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:25:18.638 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.972685+0000", 2023-09-23T20:25:18.638 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:25:18.638 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.638 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.638 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:25:18.638 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.972723+0000", 2023-09-23T20:25:18.638 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.7648999999999998e-05 2023-09-23T20:25:18.638 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.638 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.639 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:25:18.639 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.972743+0000", 2023-09-23T20:25:18.639 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.9748000000000001e-05 2023-09-23T20:25:18.639 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.639 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.639 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:25:18.639 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.972754+0000", 2023-09-23T20:25:18.639 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1235e-05 2023-09-23T20:25:18.639 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.639 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.639 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:25:18.639 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:10.007585+0000", 2023-09-23T20:25:18.639 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.034831612999999997 2023-09-23T20:25:18.640 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.640 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.640 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:25:18.640 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:10.007595+0000", 2023-09-23T20:25:18.640 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.5759999999999995e-06 2023-09-23T20:25:18.640 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.640 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.640 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:25:18.640 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.069163+0000", 2023-09-23T20:25:18.640 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0615679520000001 2023-09-23T20:25:18.640 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.641 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.641 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:25:18.641 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.069250+0000", 2023-09-23T20:25:18.641 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.7239999999999998e-05 2023-09-23T20:25:18.641 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:25:18.641 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:25:18.641 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:25:18.641 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.641 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.641 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.34791.0:28905 5.0 5:069f0235:::benchmark_data_smithi018_228451_object28904:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e118)", 2023-09-23T20:25:18.641 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:23:09.973354+0000", 2023-09-23T20:25:18.641 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 128.660116061, 2023-09-23T20:25:18.642 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0987403600000001, 2023-09-23T20:25:18.642 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:25:18.642 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:25:18.642 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:25:18.642 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.34791", 2023-09-23T20:25:18.642 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:25:18.642 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 28905 2023-09-23T20:25:18.642 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.642 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:25:18.642 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.642 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:25:18.642 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.973354+0000", 2023-09-23T20:25:18.643 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:25:18.643 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.643 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.643 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:25:18.643 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.973354+0000", 2023-09-23T20:25:18.643 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:25:18.643 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.643 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.643 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:25:18.643 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.973353+0000", 2023-09-23T20:25:18.643 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:25:18.643 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.644 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.644 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:25:18.644 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.973390+0000", 2023-09-23T20:25:18.644 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.7347e-05 2023-09-23T20:25:18.644 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.644 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.644 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:25:18.644 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.973402+0000", 2023-09-23T20:25:18.644 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2091e-05 2023-09-23T20:25:18.644 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.644 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.644 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:25:18.644 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.973411+0000", 2023-09-23T20:25:18.645 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.5140000000000001e-06 2023-09-23T20:25:18.645 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.645 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.645 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:25:18.645 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:10.008100+0000", 2023-09-23T20:25:18.645 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.034689603999999999 2023-09-23T20:25:18.645 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.645 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.645 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:25:18.645 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:10.008109+0000", 2023-09-23T20:25:18.645 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.0499999999999997e-06 2023-09-23T20:25:18.645 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.646 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.646 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:25:18.646 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.066580+0000", 2023-09-23T20:25:18.646 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0584706420000001 2023-09-23T20:25:18.646 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.646 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.646 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:25:18.646 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.066831+0000", 2023-09-23T20:25:18.646 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00025123499999999999 2023-09-23T20:25:18.646 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.646 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.647 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,4", 2023-09-23T20:25:18.647 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.066984+0000", 2023-09-23T20:25:18.647 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00015263599999999999 2023-09-23T20:25:18.647 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.647 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.647 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:25:18.647 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.068869+0000", 2023-09-23T20:25:18.647 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0018853699999999999 2023-09-23T20:25:18.647 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.647 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.647 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:25:18.647 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.071781+0000", 2023-09-23T20:25:18.647 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.002912235 2023-09-23T20:25:18.648 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.648 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.648 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:25:18.648 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.072001+0000", 2023-09-23T20:25:18.648 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000219118 2023-09-23T20:25:18.648 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.648 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.648 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:25:18.648 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.072053+0000", 2023-09-23T20:25:18.648 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.2621000000000003e-05 2023-09-23T20:25:18.648 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.648 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.649 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:25:18.649 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.072094+0000", 2023-09-23T20:25:18.649 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.1186999999999997e-05 2023-09-23T20:25:18.649 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:25:18.649 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:25:18.649 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:25:18.649 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.649 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.649 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.34791.0:28913 5.10 5:08948f05:::benchmark_data_smithi018_228451_object28912:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e118)", 2023-09-23T20:25:18.649 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:23:10.068876+0000", 2023-09-23T20:25:18.649 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 128.564593739, 2023-09-23T20:25:18.649 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.00850116, 2023-09-23T20:25:18.650 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:25:18.650 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:25:18.650 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:25:18.650 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.34791", 2023-09-23T20:25:18.650 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:25:18.650 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 28913 2023-09-23T20:25:18.650 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.650 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:25:18.650 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.650 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:25:18.650 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:10.068876+0000", 2023-09-23T20:25:18.650 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:25:18.651 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.651 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.651 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:25:18.651 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:10.068876+0000", 2023-09-23T20:25:18.651 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:25:18.651 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.651 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.651 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:25:18.651 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:10.068875+0000", 2023-09-23T20:25:18.651 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:25:18.651 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.651 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.651 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:25:18.652 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:10.068903+0000", 2023-09-23T20:25:18.652 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.8710000000000001e-05 2023-09-23T20:25:18.652 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.652 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.652 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:25:18.652 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:10.068925+0000", 2023-09-23T20:25:18.652 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.128e-05 2023-09-23T20:25:18.652 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.652 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.652 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:25:18.652 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:10.068936+0000", 2023-09-23T20:25:18.652 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1348999999999999e-05 2023-09-23T20:25:18.653 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.653 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.653 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:25:18.653 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:10.069003+0000", 2023-09-23T20:25:18.653 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.6523e-05 2023-09-23T20:25:18.653 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.653 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.653 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:25:18.653 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:10.069017+0000", 2023-09-23T20:25:18.653 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.4270000000000001e-05 2023-09-23T20:25:18.653 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.653 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.654 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:25:18.654 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.069308+0000", 2023-09-23T20:25:18.654 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0002914000000001 2023-09-23T20:25:18.654 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.654 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.654 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:25:18.654 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.069443+0000", 2023-09-23T20:25:18.654 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000135038 2023-09-23T20:25:18.654 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.654 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.654 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,4", 2023-09-23T20:25:18.654 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.069691+0000", 2023-09-23T20:25:18.655 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00024818099999999999 2023-09-23T20:25:18.655 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.655 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.655 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:25:18.655 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.072098+0000", 2023-09-23T20:25:18.655 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.002407003 2023-09-23T20:25:18.655 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.655 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.655 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:25:18.655 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.072544+0000", 2023-09-23T20:25:18.655 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00044530200000000001 2023-09-23T20:25:18.655 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.656 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.656 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:25:18.656 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.077159+0000", 2023-09-23T20:25:18.656 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0046152789999999999 2023-09-23T20:25:18.656 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.656 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.656 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:25:18.656 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.077338+0000", 2023-09-23T20:25:18.656 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00017922200000000001 2023-09-23T20:25:18.656 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.656 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.656 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:25:18.656 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.077378+0000", 2023-09-23T20:25:18.657 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.9271999999999999e-05 2023-09-23T20:25:18.657 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:25:18.657 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:25:18.657 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:25:18.657 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.657 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.657 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.34791.0:75986 5.17 5.909c0a97 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e145)", 2023-09-23T20:25:18.657 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:23:55.954505+0000", 2023-09-23T20:25:18.657 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 82.678964827000001, 2023-09-23T20:25:18.657 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.97247618800000002, 2023-09-23T20:25:18.657 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:25:18.657 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:25:18.657 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:25:18.658 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.34791", 2023-09-23T20:25:18.658 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:25:18.658 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 75986 2023-09-23T20:25:18.658 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.658 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:25:18.658 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.658 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:25:18.658 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:55.954505+0000", 2023-09-23T20:25:18.658 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:25:18.658 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.658 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.659 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:25:18.659 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:55.954505+0000", 2023-09-23T20:25:18.659 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:25:18.659 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.659 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.659 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:25:18.659 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:55.954504+0000", 2023-09-23T20:25:18.659 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:25:18.659 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.659 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.659 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:25:18.659 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:55.954529+0000", 2023-09-23T20:25:18.660 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.5401e-05 2023-09-23T20:25:18.660 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.660 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.660 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:25:18.660 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:55.954543+0000", 2023-09-23T20:25:18.660 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3626e-05 2023-09-23T20:25:18.660 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.660 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.660 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:25:18.660 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:55.954553+0000", 2023-09-23T20:25:18.660 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.01e-05 2023-09-23T20:25:18.660 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.660 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.661 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:25:18.661 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:55.954611+0000", 2023-09-23T20:25:18.661 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.7877000000000002e-05 2023-09-23T20:25:18.661 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.661 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.661 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:25:18.661 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:55.954620+0000", 2023-09-23T20:25:18.661 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.6929999999999996e-06 2023-09-23T20:25:18.661 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.661 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.661 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:25:18.661 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.926972+0000", 2023-09-23T20:25:18.662 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.97235204099999994 2023-09-23T20:25:18.662 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.662 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.662 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:25:18.662 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.926981+0000", 2023-09-23T20:25:18.662 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.7210000000000006e-06 2023-09-23T20:25:18.662 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:25:18.662 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:25:18.662 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:25:18.662 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.662 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.663 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.34791.0:75990 5.17 5.b19cfe77 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e145)", 2023-09-23T20:25:18.663 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:23:55.958067+0000", 2023-09-23T20:25:18.663 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 82.675403334999999, 2023-09-23T20:25:18.663 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.96897582500000001, 2023-09-23T20:25:18.663 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:25:18.663 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:25:18.663 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:25:18.663 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.34791", 2023-09-23T20:25:18.663 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:25:18.663 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 75990 2023-09-23T20:25:18.663 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.663 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:25:18.664 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.664 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:25:18.664 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:55.958067+0000", 2023-09-23T20:25:18.664 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:25:18.664 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.664 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.664 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:25:18.664 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:55.958067+0000", 2023-09-23T20:25:18.664 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:25:18.664 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.664 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.664 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:25:18.665 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:55.958065+0000", 2023-09-23T20:25:18.665 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:25:18.665 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.665 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.665 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:25:18.665 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:55.958090+0000", 2023-09-23T20:25:18.665 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.4776000000000001e-05 2023-09-23T20:25:18.665 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.665 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.665 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:25:18.665 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:55.958108+0000", 2023-09-23T20:25:18.666 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.7484000000000001e-05 2023-09-23T20:25:18.666 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.666 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.666 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:25:18.666 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:55.958116+0000", 2023-09-23T20:25:18.666 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.5369999999999997e-06 2023-09-23T20:25:18.666 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.666 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.666 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:25:18.666 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:55.958166+0000", 2023-09-23T20:25:18.666 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.9548000000000001e-05 2023-09-23T20:25:18.667 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.667 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.667 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:25:18.667 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:55.958182+0000", 2023-09-23T20:25:18.667 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.6149000000000001e-05 2023-09-23T20:25:18.667 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.667 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.667 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:25:18.667 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.927026+0000", 2023-09-23T20:25:18.667 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.96884431199999999 2023-09-23T20:25:18.667 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.667 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.668 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:25:18.668 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.927043+0000", 2023-09-23T20:25:18.668 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.6376e-05 2023-09-23T20:25:18.668 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:25:18.668 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:25:18.668 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:25:18.668 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.668 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.668 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.34791.0:76086 5.17 5.e0c66df7 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e145)", 2023-09-23T20:25:18.668 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:23:56.031309+0000", 2023-09-23T20:25:18.668 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 82.602161593000005, 2023-09-23T20:25:18.668 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.89577783200000005, 2023-09-23T20:25:18.669 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:25:18.669 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:25:18.669 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:25:18.669 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.34791", 2023-09-23T20:25:18.669 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:25:18.669 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 76086 2023-09-23T20:25:18.669 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.669 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:25:18.669 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.669 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:25:18.669 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.031309+0000", 2023-09-23T20:25:18.669 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:25:18.670 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.670 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.670 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:25:18.670 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.031309+0000", 2023-09-23T20:25:18.670 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:25:18.670 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.670 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.670 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:25:18.670 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.031307+0000", 2023-09-23T20:25:18.670 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:25:18.670 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.671 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.671 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:25:18.671 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.031333+0000", 2023-09-23T20:25:18.671 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.5408000000000001e-05 2023-09-23T20:25:18.671 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.671 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.671 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:25:18.671 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.031349+0000", 2023-09-23T20:25:18.671 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.6552e-05 2023-09-23T20:25:18.671 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.671 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.671 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:25:18.671 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.031358+0000", 2023-09-23T20:25:18.672 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.9150000000000002e-06 2023-09-23T20:25:18.672 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.672 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.672 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:25:18.672 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.031432+0000", 2023-09-23T20:25:18.672 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.3740999999999998e-05 2023-09-23T20:25:18.672 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.672 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.672 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:25:18.672 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.031440+0000", 2023-09-23T20:25:18.672 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.1100000000000003e-06 2023-09-23T20:25:18.672 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.673 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.673 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:25:18.673 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.927075+0000", 2023-09-23T20:25:18.673 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.89563524800000005 2023-09-23T20:25:18.673 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.673 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.673 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:25:18.673 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.927086+0000", 2023-09-23T20:25:18.673 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.114e-05 2023-09-23T20:25:18.673 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:25:18.673 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:25:18.673 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:25:18.673 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.674 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.674 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.34791.0:76104 5.17 5.775f52d7 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e145)", 2023-09-23T20:25:18.674 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:23:56.042174+0000", 2023-09-23T20:25:18.674 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 82.591296475999997, 2023-09-23T20:25:18.674 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.88495434900000003, 2023-09-23T20:25:18.674 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:25:18.674 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:25:18.674 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:25:18.674 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.34791", 2023-09-23T20:25:18.674 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:25:18.674 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 76104 2023-09-23T20:25:18.674 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.675 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:25:18.675 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.675 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:25:18.675 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.042174+0000", 2023-09-23T20:25:18.675 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:25:18.675 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.675 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.675 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:25:18.675 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.042174+0000", 2023-09-23T20:25:18.675 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:25:18.675 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.676 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.676 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:25:18.676 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.042172+0000", 2023-09-23T20:25:18.676 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:25:18.676 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.676 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.676 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:25:18.676 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.042200+0000", 2023-09-23T20:25:18.676 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.7487999999999998e-05 2023-09-23T20:25:18.676 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.676 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.676 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:25:18.677 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.042214+0000", 2023-09-23T20:25:18.677 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.4535999999999999e-05 2023-09-23T20:25:18.677 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.677 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.677 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:25:18.677 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.042222+0000", 2023-09-23T20:25:18.677 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.8150000000000006e-06 2023-09-23T20:25:18.677 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.677 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.677 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:25:18.677 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.042292+0000", 2023-09-23T20:25:18.677 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.0086000000000006e-05 2023-09-23T20:25:18.678 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.678 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.678 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:25:18.678 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.042308+0000", 2023-09-23T20:25:18.678 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.5389000000000001e-05 2023-09-23T20:25:18.678 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.678 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.678 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:25:18.678 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.927115+0000", 2023-09-23T20:25:18.678 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.88480730699999999 2023-09-23T20:25:18.678 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.678 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.679 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:25:18.679 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.927128+0000", 2023-09-23T20:25:18.679 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3074000000000001e-05 2023-09-23T20:25:18.679 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:25:18.679 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:25:18.679 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:25:18.679 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.679 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.679 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.34791.0:76249 5.17 5.429d96f7 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e145)", 2023-09-23T20:25:18.679 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:23:56.151472+0000", 2023-09-23T20:25:18.679 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 82.481998204999996, 2023-09-23T20:25:18.679 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.77570009299999998, 2023-09-23T20:25:18.680 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:25:18.680 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:25:18.680 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:25:18.680 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.34791", 2023-09-23T20:25:18.680 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:25:18.680 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 76249 2023-09-23T20:25:18.680 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.680 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:25:18.680 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.680 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:25:18.680 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.151472+0000", 2023-09-23T20:25:18.680 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:25:18.681 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.681 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.681 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:25:18.681 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.151472+0000", 2023-09-23T20:25:18.681 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:25:18.681 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.681 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.681 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:25:18.681 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.151469+0000", 2023-09-23T20:25:18.681 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999976 2023-09-23T20:25:18.681 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.681 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.682 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:25:18.682 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.151508+0000", 2023-09-23T20:25:18.682 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.9032000000000002e-05 2023-09-23T20:25:18.682 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.682 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.682 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:25:18.682 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.151529+0000", 2023-09-23T20:25:18.682 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.0716e-05 2023-09-23T20:25:18.682 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.682 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.682 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:25:18.682 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.151540+0000", 2023-09-23T20:25:18.683 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1258e-05 2023-09-23T20:25:18.683 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.683 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.683 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:25:18.683 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.151604+0000", 2023-09-23T20:25:18.683 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.3281000000000006e-05 2023-09-23T20:25:18.683 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.683 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.683 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:25:18.683 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.151615+0000", 2023-09-23T20:25:18.683 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1430999999999999e-05 2023-09-23T20:25:18.683 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.684 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.684 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:25:18.684 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.927160+0000", 2023-09-23T20:25:18.684 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.77554441399999996 2023-09-23T20:25:18.684 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.684 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.684 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:25:18.684 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.927172+0000", 2023-09-23T20:25:18.684 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2427e-05 2023-09-23T20:25:18.684 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:25:18.684 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:25:18.684 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:25:18.685 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.685 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.685 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.34791.0:76255 5.17 5.9006f5b7 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e145)", 2023-09-23T20:25:18.685 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:23:56.155426+0000", 2023-09-23T20:25:18.685 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 82.478044350999994, 2023-09-23T20:25:18.685 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.77179339999999996, 2023-09-23T20:25:18.685 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:25:18.685 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:25:18.685 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:25:18.685 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.34791", 2023-09-23T20:25:18.685 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:25:18.685 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 76255 2023-09-23T20:25:18.686 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.686 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:25:18.686 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.686 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:25:18.686 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.155426+0000", 2023-09-23T20:25:18.686 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:25:18.686 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.686 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.686 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:25:18.686 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.155426+0000", 2023-09-23T20:25:18.686 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:25:18.687 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.687 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.687 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:25:18.687 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.155425+0000", 2023-09-23T20:25:18.687 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.999999 2023-09-23T20:25:18.687 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.687 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.687 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:25:18.687 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.155447+0000", 2023-09-23T20:25:18.687 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.2459e-05 2023-09-23T20:25:18.687 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.687 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.687 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:25:18.688 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.155459+0000", 2023-09-23T20:25:18.688 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2155e-05 2023-09-23T20:25:18.688 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.688 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.688 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:25:18.688 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.155467+0000", 2023-09-23T20:25:18.688 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.5730000000000002e-06 2023-09-23T20:25:18.688 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.688 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.688 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:25:18.688 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.155524+0000", 2023-09-23T20:25:18.688 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.7108000000000003e-05 2023-09-23T20:25:18.689 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.689 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.689 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:25:18.689 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.155534+0000", 2023-09-23T20:25:18.689 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.6139999999999998e-06 2023-09-23T20:25:18.689 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.689 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.689 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:25:18.689 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.927206+0000", 2023-09-23T20:25:18.689 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.77167270899999996 2023-09-23T20:25:18.689 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.689 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.690 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:25:18.690 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.927219+0000", 2023-09-23T20:25:18.690 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2828e-05 2023-09-23T20:25:18.690 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:25:18.690 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:25:18.690 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:25:18.690 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.690 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.690 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.34791.0:76279 5.17 5.be4497b7 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e145)", 2023-09-23T20:25:18.690 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:23:56.175291+0000", 2023-09-23T20:25:18.690 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 82.458178974999996, 2023-09-23T20:25:18.690 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.75196661499999995, 2023-09-23T20:25:18.691 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:25:18.691 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:25:18.691 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:25:18.691 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.34791", 2023-09-23T20:25:18.691 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:25:18.691 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 76279 2023-09-23T20:25:18.691 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.691 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:25:18.691 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.691 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:25:18.691 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.175291+0000", 2023-09-23T20:25:18.691 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:25:18.692 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.692 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.692 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:25:18.692 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.175291+0000", 2023-09-23T20:25:18.692 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:25:18.692 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.692 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.692 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:25:18.692 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.175290+0000", 2023-09-23T20:25:18.692 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:25:18.692 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.692 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.693 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:25:18.693 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.175315+0000", 2023-09-23T20:25:18.693 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.4998e-05 2023-09-23T20:25:18.693 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.693 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.693 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:25:18.693 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.175327+0000", 2023-09-23T20:25:18.693 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2405000000000001e-05 2023-09-23T20:25:18.693 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.693 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.693 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:25:18.693 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.175333+0000", 2023-09-23T20:25:18.694 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.9000000000000003e-06 2023-09-23T20:25:18.694 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.694 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.694 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:25:18.694 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.175382+0000", 2023-09-23T20:25:18.694 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.9166000000000003e-05 2023-09-23T20:25:18.694 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.694 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.694 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:25:18.694 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.175401+0000", 2023-09-23T20:25:18.694 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.8689999999999999e-05 2023-09-23T20:25:18.695 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.695 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.695 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:25:18.695 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.927246+0000", 2023-09-23T20:25:18.695 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.75184536999999996 2023-09-23T20:25:18.695 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.695 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.695 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:25:18.695 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.927258+0000", 2023-09-23T20:25:18.695 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1518999999999999e-05 2023-09-23T20:25:18.695 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:25:18.695 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:25:18.695 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:25:18.695 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.696 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.696 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.34791.0:76341 5.17 5.2b1387f7 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e145)", 2023-09-23T20:25:18.696 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:23:56.230973+0000", 2023-09-23T20:25:18.696 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 82.402496732000003, 2023-09-23T20:25:18.696 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.69632106199999999, 2023-09-23T20:25:18.696 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:25:18.696 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:25:18.696 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:25:18.696 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.34791", 2023-09-23T20:25:18.696 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:25:18.696 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 76341 2023-09-23T20:25:18.696 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.697 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:25:18.697 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.697 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:25:18.697 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.230973+0000", 2023-09-23T20:25:18.697 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:25:18.697 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.697 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.697 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:25:18.697 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.230973+0000", 2023-09-23T20:25:18.697 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:25:18.697 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.697 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.698 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:25:18.698 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.230972+0000", 2023-09-23T20:25:18.698 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.999999 2023-09-23T20:25:18.698 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.698 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.698 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:25:18.698 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.230996+0000", 2023-09-23T20:25:18.698 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.3439999999999999e-05 2023-09-23T20:25:18.698 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.698 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.698 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:25:18.698 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.231012+0000", 2023-09-23T20:25:18.699 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.6152000000000001e-05 2023-09-23T20:25:18.699 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.699 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.699 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:25:18.699 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.231019+0000", 2023-09-23T20:25:18.699 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.9569999999999996e-06 2023-09-23T20:25:18.699 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.699 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.699 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:25:18.699 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.231081+0000", 2023-09-23T20:25:18.699 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.1940999999999995e-05 2023-09-23T20:25:18.699 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.699 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.700 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:25:18.700 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.231090+0000", 2023-09-23T20:25:18.700 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.8259999999999999e-06 2023-09-23T20:25:18.700 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.700 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.700 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:25:18.700 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.927285+0000", 2023-09-23T20:25:18.700 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.69619489099999998 2023-09-23T20:25:18.700 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.700 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.700 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:25:18.700 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.927294+0000", 2023-09-23T20:25:18.701 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.8689999999999999e-06 2023-09-23T20:25:18.701 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:25:18.701 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:25:18.701 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:25:18.701 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.701 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.701 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.34791.0:76399 5.17 5.332ff437 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e145)", 2023-09-23T20:25:18.701 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:23:56.277880+0000", 2023-09-23T20:25:18.701 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 82.355589706000004, 2023-09-23T20:25:18.701 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.64945012400000002, 2023-09-23T20:25:18.701 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:25:18.701 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:25:18.702 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:25:18.702 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.34791", 2023-09-23T20:25:18.702 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:25:18.702 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 76399 2023-09-23T20:25:18.702 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.702 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:25:18.702 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.702 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:25:18.702 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.277880+0000", 2023-09-23T20:25:18.702 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:25:18.702 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.702 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.703 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:25:18.703 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.277880+0000", 2023-09-23T20:25:18.703 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:25:18.703 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.703 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.703 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:25:18.703 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.277880+0000", 2023-09-23T20:25:18.703 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:25:18.703 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.703 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.703 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:25:18.703 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.277925+0000", 2023-09-23T20:25:18.704 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.5504999999999999e-05 2023-09-23T20:25:18.704 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.704 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.704 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:25:18.704 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.277934+0000", 2023-09-23T20:25:18.704 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.8200000000000003e-06 2023-09-23T20:25:18.704 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.704 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.704 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:25:18.704 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.277940+0000", 2023-09-23T20:25:18.704 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.7769999999999999e-06 2023-09-23T20:25:18.704 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.704 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.705 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:25:18.705 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.277997+0000", 2023-09-23T20:25:18.705 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.7187e-05 2023-09-23T20:25:18.705 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.705 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.705 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:25:18.705 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.278006+0000", 2023-09-23T20:25:18.705 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.1929999999999999e-06 2023-09-23T20:25:18.705 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.705 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.705 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:25:18.705 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.927320+0000", 2023-09-23T20:25:18.705 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.64931412399999999 2023-09-23T20:25:18.706 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.706 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.706 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:25:18.706 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.927331+0000", 2023-09-23T20:25:18.706 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0122000000000001e-05 2023-09-23T20:25:18.706 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:25:18.706 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:25:18.706 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:25:18.706 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.706 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.706 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.34791.0:76438 5.17 5.e18ac6b7 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e145)", 2023-09-23T20:25:18.706 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:23:56.310541+0000", 2023-09-23T20:25:18.706 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 82.322929228000007, 2023-09-23T20:25:18.707 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.61683234799999997, 2023-09-23T20:25:18.707 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:25:18.707 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:25:18.707 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:25:18.707 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.34791", 2023-09-23T20:25:18.707 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:25:18.707 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 76438 2023-09-23T20:25:18.707 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.707 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:25:18.707 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.707 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:25:18.707 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.310541+0000", 2023-09-23T20:25:18.708 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:25:18.708 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.708 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.708 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:25:18.708 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.310541+0000", 2023-09-23T20:25:18.708 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:25:18.708 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.708 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.708 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:25:18.708 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.310539+0000", 2023-09-23T20:25:18.708 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:25:18.708 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.709 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.709 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:25:18.709 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.310564+0000", 2023-09-23T20:25:18.709 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.4213e-05 2023-09-23T20:25:18.709 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.709 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.709 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:25:18.709 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.310584+0000", 2023-09-23T20:25:18.709 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.0318000000000001e-05 2023-09-23T20:25:18.709 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.709 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.709 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:25:18.709 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.310595+0000", 2023-09-23T20:25:18.709 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1078e-05 2023-09-23T20:25:18.710 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.710 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.710 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:25:18.710 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.310654+0000", 2023-09-23T20:25:18.710 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.9277999999999997e-05 2023-09-23T20:25:18.710 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.710 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.710 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:25:18.710 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.310667+0000", 2023-09-23T20:25:18.710 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2651e-05 2023-09-23T20:25:18.710 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.710 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.711 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:25:18.711 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.927360+0000", 2023-09-23T20:25:18.711 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.61669307200000001 2023-09-23T20:25:18.711 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.711 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.711 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:25:18.711 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.927373+0000", 2023-09-23T20:25:18.711 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.327e-05 2023-09-23T20:25:18.711 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:25:18.711 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:25:18.711 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:25:18.711 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.711 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.712 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.34791.0:76447 5.17 5.8ad19dd7 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e145)", 2023-09-23T20:25:18.712 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:23:56.318593+0000", 2023-09-23T20:25:18.712 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 82.314876812999998, 2023-09-23T20:25:18.712 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.60882077800000001, 2023-09-23T20:25:18.712 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:25:18.712 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:25:18.712 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:25:18.712 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.34791", 2023-09-23T20:25:18.712 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:25:18.712 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 76447 2023-09-23T20:25:18.712 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.712 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:25:18.712 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.713 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:25:18.713 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.318593+0000", 2023-09-23T20:25:18.713 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:25:18.713 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.713 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.713 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:25:18.713 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.318593+0000", 2023-09-23T20:25:18.713 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:25:18.713 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.713 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.713 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:25:18.713 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.318591+0000", 2023-09-23T20:25:18.714 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:25:18.714 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.714 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.714 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:25:18.714 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.318621+0000", 2023-09-23T20:25:18.714 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.9802e-05 2023-09-23T20:25:18.714 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.714 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.714 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:25:18.714 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.318639+0000", 2023-09-23T20:25:18.714 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.7300999999999999e-05 2023-09-23T20:25:18.714 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.715 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.715 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:25:18.715 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.318647+0000", 2023-09-23T20:25:18.715 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.7150000000000004e-06 2023-09-23T20:25:18.715 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.715 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.715 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:25:18.715 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.318696+0000", 2023-09-23T20:25:18.715 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.8730999999999998e-05 2023-09-23T20:25:18.715 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.715 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.715 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:25:18.715 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.318705+0000", 2023-09-23T20:25:18.715 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.7420000000000003e-06 2023-09-23T20:25:18.716 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.716 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.716 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:25:18.716 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.927404+0000", 2023-09-23T20:25:18.716 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.60869895100000004 2023-09-23T20:25:18.716 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.716 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.716 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:25:18.716 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.927414+0000", 2023-09-23T20:25:18.716 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0383999999999999e-05 2023-09-23T20:25:18.716 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:25:18.716 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:25:18.716 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:25:18.717 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.717 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.717 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.34791.0:76507 5.17 5.8ea018d7 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e145)", 2023-09-23T20:25:18.717 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:23:56.381055+0000", 2023-09-23T20:25:18.717 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 82.252414846999997, 2023-09-23T20:25:18.717 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.54639771500000001, 2023-09-23T20:25:18.717 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:25:18.717 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:25:18.717 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:25:18.717 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.34791", 2023-09-23T20:25:18.717 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:25:18.717 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 76507 2023-09-23T20:25:18.718 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.718 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:25:18.718 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.718 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:25:18.718 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.381055+0000", 2023-09-23T20:25:18.718 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:25:18.718 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.718 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.718 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:25:18.718 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.381055+0000", 2023-09-23T20:25:18.718 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:25:18.719 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.719 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.719 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:25:18.719 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.381053+0000", 2023-09-23T20:25:18.719 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:25:18.719 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.719 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.719 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:25:18.719 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.381079+0000", 2023-09-23T20:25:18.719 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.6174999999999999e-05 2023-09-23T20:25:18.719 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.719 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.720 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:25:18.720 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.381099+0000", 2023-09-23T20:25:18.720 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.9290999999999999e-05 2023-09-23T20:25:18.720 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.720 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.720 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:25:18.720 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.381108+0000", 2023-09-23T20:25:18.720 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.0990000000000005e-06 2023-09-23T20:25:18.720 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.720 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.720 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:25:18.720 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.381166+0000", 2023-09-23T20:25:18.720 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.7890000000000003e-05 2023-09-23T20:25:18.721 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.721 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.721 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:25:18.721 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.381177+0000", 2023-09-23T20:25:18.721 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1238e-05 2023-09-23T20:25:18.721 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.721 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.721 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:25:18.721 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.927440+0000", 2023-09-23T20:25:18.721 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.546263206 2023-09-23T20:25:18.721 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.721 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.721 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:25:18.722 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.927453+0000", 2023-09-23T20:25:18.722 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2947e-05 2023-09-23T20:25:18.722 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:25:18.722 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:25:18.722 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:25:18.722 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.722 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.722 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.34791.0:76527 5.17 5.b14e1bf7 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e145)", 2023-09-23T20:25:18.722 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:23:56.401261+0000", 2023-09-23T20:25:18.722 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 82.232209362000006, 2023-09-23T20:25:18.722 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.52623452699999995, 2023-09-23T20:25:18.722 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:25:18.723 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:25:18.723 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:25:18.723 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.34791", 2023-09-23T20:25:18.723 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:25:18.723 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 76527 2023-09-23T20:25:18.723 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.723 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:25:18.723 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.723 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:25:18.723 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.401261+0000", 2023-09-23T20:25:18.723 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:25:18.723 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.724 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.724 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:25:18.724 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.401261+0000", 2023-09-23T20:25:18.724 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:25:18.724 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.724 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.724 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:25:18.724 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.401260+0000", 2023-09-23T20:25:18.724 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.999999 2023-09-23T20:25:18.724 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.724 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.724 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:25:18.725 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.401279+0000", 2023-09-23T20:25:18.725 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.8757999999999999e-05 2023-09-23T20:25:18.725 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.725 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.725 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:25:18.725 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.401290+0000", 2023-09-23T20:25:18.725 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1058e-05 2023-09-23T20:25:18.725 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.725 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.725 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:25:18.725 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.401295+0000", 2023-09-23T20:25:18.725 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.6840000000000001e-06 2023-09-23T20:25:18.726 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.726 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.726 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:25:18.726 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.401349+0000", 2023-09-23T20:25:18.726 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.3266000000000001e-05 2023-09-23T20:25:18.726 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.726 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.726 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:25:18.726 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.401358+0000", 2023-09-23T20:25:18.726 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.2930000000000006e-06 2023-09-23T20:25:18.726 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.726 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.727 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:25:18.727 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.927483+0000", 2023-09-23T20:25:18.727 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.52612539400000002 2023-09-23T20:25:18.727 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.727 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.727 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:25:18.727 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.927495+0000", 2023-09-23T20:25:18.727 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1844e-05 2023-09-23T20:25:18.727 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:25:18.727 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:25:18.727 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:25:18.727 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.727 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.728 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.34791.0:76538 5.17 5.aa1fa457 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e145)", 2023-09-23T20:25:18.728 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:23:56.416592+0000", 2023-09-23T20:25:18.728 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 82.216877724, 2023-09-23T20:25:18.728 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.51094999100000005, 2023-09-23T20:25:18.728 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:25:18.728 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:25:18.728 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:25:18.728 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.34791", 2023-09-23T20:25:18.728 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:25:18.728 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 76538 2023-09-23T20:25:18.728 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.728 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:25:18.728 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.729 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:25:18.729 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.416592+0000", 2023-09-23T20:25:18.729 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:25:18.729 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.729 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.729 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:25:18.729 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.416592+0000", 2023-09-23T20:25:18.729 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:25:18.729 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.729 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.729 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:25:18.729 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.416592+0000", 2023-09-23T20:25:18.730 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.999999 2023-09-23T20:25:18.730 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.730 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.730 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:25:18.730 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.416635+0000", 2023-09-23T20:25:18.730 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.3720000000000002e-05 2023-09-23T20:25:18.730 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.730 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.730 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:25:18.730 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.416649+0000", 2023-09-23T20:25:18.730 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3997e-05 2023-09-23T20:25:18.730 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.731 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.731 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:25:18.731 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.416658+0000", 2023-09-23T20:25:18.731 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.5269999999999998e-06 2023-09-23T20:25:18.731 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.731 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.731 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:25:18.731 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.416712+0000", 2023-09-23T20:25:18.731 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.4292999999999997e-05 2023-09-23T20:25:18.731 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.731 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.731 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:25:18.732 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.416724+0000", 2023-09-23T20:25:18.732 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1642e-05 2023-09-23T20:25:18.732 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.732 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.732 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:25:18.732 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.927529+0000", 2023-09-23T20:25:18.732 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.51080517299999995 2023-09-23T20:25:18.732 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.732 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.732 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:25:18.732 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.927542+0000", 2023-09-23T20:25:18.732 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3438000000000001e-05 2023-09-23T20:25:18.732 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:25:18.733 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:25:18.733 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:25:18.733 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.733 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.733 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.34791.0:76629 5.17 5.b377477 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e145)", 2023-09-23T20:25:18.733 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:23:56.571938+0000", 2023-09-23T20:25:18.733 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 82.061532596000006, 2023-09-23T20:25:18.733 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.35564711900000001, 2023-09-23T20:25:18.733 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:25:18.733 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:25:18.733 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:25:18.733 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.34791", 2023-09-23T20:25:18.733 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:25:18.734 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 76629 2023-09-23T20:25:18.734 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.734 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:25:18.734 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.734 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:25:18.734 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.571938+0000", 2023-09-23T20:25:18.734 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:25:18.734 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.734 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.734 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:25:18.734 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.571938+0000", 2023-09-23T20:25:18.735 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:25:18.735 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.735 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.735 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:25:18.735 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.571936+0000", 2023-09-23T20:25:18.735 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:25:18.735 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.735 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.735 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:25:18.735 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.571959+0000", 2023-09-23T20:25:18.735 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.2770000000000001e-05 2023-09-23T20:25:18.735 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.735 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.735 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:25:18.736 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.571973+0000", 2023-09-23T20:25:18.736 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3852999999999999e-05 2023-09-23T20:25:18.736 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.736 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.736 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:25:18.736 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.571979+0000", 2023-09-23T20:25:18.736 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.5409999999999997e-06 2023-09-23T20:25:18.736 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.736 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.736 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:25:18.736 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.572022+0000", 2023-09-23T20:25:18.736 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.2991999999999998e-05 2023-09-23T20:25:18.736 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.737 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.737 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:25:18.737 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.572036+0000", 2023-09-23T20:25:18.737 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3753e-05 2023-09-23T20:25:18.737 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.737 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.737 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:25:18.737 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.927572+0000", 2023-09-23T20:25:18.737 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.35553553399999999 2023-09-23T20:25:18.737 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.737 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.737 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:25:18.737 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:56.927585+0000", 2023-09-23T20:25:18.738 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3094e-05 2023-09-23T20:25:18.738 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:25:18.738 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:25:18.738 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:25:18.738 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.738 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.738 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.34791.0:126924 5.17 5.44e46a57 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e165)", 2023-09-23T20:25:18.738 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:25:01.938216+0000", 2023-09-23T20:25:18.738 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 16.695253767000001, 2023-09-23T20:25:18.738 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.97596713099999999, 2023-09-23T20:25:18.738 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:25:18.738 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:25:18.739 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:25:18.739 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.34791", 2023-09-23T20:25:18.739 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:25:18.739 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 126924 2023-09-23T20:25:18.739 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.739 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:25:18.739 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.739 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:25:18.739 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:25:01.938216+0000", 2023-09-23T20:25:18.739 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:25:18.739 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.739 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.739 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:25:18.739 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:25:01.938216+0000", 2023-09-23T20:25:18.740 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:25:18.740 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.740 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.740 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:25:18.740 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:25:01.938215+0000", 2023-09-23T20:25:18.740 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:25:18.740 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.740 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.740 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:25:18.740 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:25:01.938221+0000", 2023-09-23T20:25:18.740 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.8599999999999998e-06 2023-09-23T20:25:18.740 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.741 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.741 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:25:18.741 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:25:01.938230+0000", 2023-09-23T20:25:18.741 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.6910000000000003e-06 2023-09-23T20:25:18.741 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.741 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.741 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:25:18.741 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:25:01.938238+0000", 2023-09-23T20:25:18.741 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.3950000000000001e-06 2023-09-23T20:25:18.741 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.741 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.741 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:25:18.742 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:25:01.938299+0000", 2023-09-23T20:25:18.742 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.0990999999999999e-05 2023-09-23T20:25:18.742 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.742 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.742 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:25:18.742 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:25:01.938312+0000", 2023-09-23T20:25:18.742 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2652e-05 2023-09-23T20:25:18.742 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.742 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.742 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:25:18.742 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:25:02.914169+0000", 2023-09-23T20:25:18.742 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.975857631 2023-09-23T20:25:18.743 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.743 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.743 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:25:18.743 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:25:02.914183+0000", 2023-09-23T20:25:18.743 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.4209000000000001e-05 2023-09-23T20:25:18.743 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:25:18.743 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:25:18.743 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:25:18.743 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.743 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.743 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.34791.0:126942 5.17 5.ee5de657 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e165)", 2023-09-23T20:25:18.743 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:25:01.988081+0000", 2023-09-23T20:25:18.743 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 16.645389051999999, 2023-09-23T20:25:18.744 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.92613754400000003, 2023-09-23T20:25:18.744 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:25:18.744 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:25:18.744 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:25:18.744 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.34791", 2023-09-23T20:25:18.744 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:25:18.744 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 126942 2023-09-23T20:25:18.744 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.744 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:25:18.744 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.744 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:25:18.744 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:25:01.988081+0000", 2023-09-23T20:25:18.744 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:25:18.745 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.745 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.745 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:25:18.745 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:25:01.988081+0000", 2023-09-23T20:25:18.745 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:25:18.745 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.745 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.745 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:25:18.745 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:25:01.988080+0000", 2023-09-23T20:25:18.745 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:25:18.745 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.745 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.746 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:25:18.746 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:25:01.988087+0000", 2023-09-23T20:25:18.746 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.1590000000000004e-06 2023-09-23T20:25:18.746 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.746 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.746 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:25:18.746 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:25:01.988105+0000", 2023-09-23T20:25:18.746 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.7997000000000001e-05 2023-09-23T20:25:18.746 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.746 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.746 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:25:18.746 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:25:01.988115+0000", 2023-09-23T20:25:18.747 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.8879999999999992e-06 2023-09-23T20:25:18.747 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.747 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.747 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:25:18.747 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:25:01.988170+0000", 2023-09-23T20:25:18.747 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.5707e-05 2023-09-23T20:25:18.747 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.747 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.747 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:25:18.747 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:25:01.988182+0000", 2023-09-23T20:25:18.747 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1633e-05 2023-09-23T20:25:18.747 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.747 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.748 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:25:18.748 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:25:02.914209+0000", 2023-09-23T20:25:18.748 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.92602680800000003 2023-09-23T20:25:18.748 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:25:18.748 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:25:18.748 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:25:18.748 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:25:02.914219+0000", 2023-09-23T20:25:18.748 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.8409999999999995e-06 2023-09-23T20:25:18.748 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:25:18.748 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:25:18.748 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:25:18.748 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:25:18.748 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:25:18.749 INFO:teuthology.orchestra.run.smithi018.stdout:} 2023-09-23T20:25:18.749 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:18.749 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:25:18.755 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:18.755 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:25:18.856 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:25:18.913 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:19.322 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:19 smithi018 ceph-mon[179647]: pgmap v362: 99 pgs: 1 active+recovery_wait+degraded, 83 active+clean, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped; 4.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 550 op/s; 25906/238254 objects degraded (10.873%); 9822/238254 objects misplaced (4.122%); 4.8 MiB/s, 119 objects/s recovering 2023-09-23T20:25:19.322 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:19 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]': finished 2023-09-23T20:25:19.323 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:19 smithi018 ceph-mon[179647]: osdmap e171: 8 total, 8 up, 7 in 2023-09-23T20:25:19.323 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:19 smithi018 ceph-mon[179647]: failed to encode map e171 with expected crc 2023-09-23T20:25:19.323 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:19 smithi018 ceph-mon[179647]: failed to encode map e171 with expected crc 2023-09-23T20:25:19.323 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:19 smithi018 ceph-mon[179647]: failed to encode map e171 with expected crc 2023-09-23T20:25:19.323 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:19 smithi018 ceph-mon[179647]: failed to encode map e171 with expected crc 2023-09-23T20:25:19.323 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:19 smithi018 ceph-mon[179647]: failed to encode map e171 with expected crc 2023-09-23T20:25:19.323 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:19 smithi018 ceph-mon[179647]: failed to encode map e171 with expected crc 2023-09-23T20:25:19.325 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:19 smithi018 ceph-mon[184144]: pgmap v362: 99 pgs: 1 active+recovery_wait+degraded, 83 active+clean, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped; 4.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 550 op/s; 25906/238254 objects degraded (10.873%); 9822/238254 objects misplaced (4.122%); 4.8 MiB/s, 119 objects/s recovering 2023-09-23T20:25:19.325 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:19 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]': finished 2023-09-23T20:25:19.325 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:19 smithi018 ceph-mon[184144]: osdmap e171: 8 total, 8 up, 7 in 2023-09-23T20:25:19.325 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:19 smithi018 ceph-mon[184144]: failed to encode map e171 with expected crc 2023-09-23T20:25:19.326 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:19 smithi018 ceph-mon[184144]: failed to encode map e171 with expected crc 2023-09-23T20:25:19.326 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:19 smithi018 ceph-mon[184144]: failed to encode map e171 with expected crc 2023-09-23T20:25:19.326 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:19 smithi018 ceph-mon[184144]: failed to encode map e171 with expected crc 2023-09-23T20:25:19.326 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:19 smithi018 ceph-mon[184144]: failed to encode map e171 with expected crc 2023-09-23T20:25:19.326 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:19 smithi018 ceph-mon[184144]: failed to encode map e171 with expected crc 2023-09-23T20:25:19.369 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:25:19.389 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:19.390 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:25:19.397 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T20:25:19.459 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:19 smithi149 ceph-mon[170394]: pgmap v362: 99 pgs: 1 active+recovery_wait+degraded, 83 active+clean, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped; 4.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 550 op/s; 25906/238254 objects degraded (10.873%); 9822/238254 objects misplaced (4.122%); 4.8 MiB/s, 119 objects/s recovering 2023-09-23T20:25:19.460 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:19 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]': finished 2023-09-23T20:25:19.460 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:19 smithi149 ceph-mon[170394]: osdmap e171: 8 total, 8 up, 7 in 2023-09-23T20:25:19.460 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:19 smithi149 ceph-mon[170394]: failed to encode map e171 with expected crc 2023-09-23T20:25:19.460 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:19 smithi149 ceph-mon[170394]: failed to encode map e171 with expected crc 2023-09-23T20:25:19.460 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:19 smithi149 ceph-mon[170394]: failed to encode map e171 with expected crc 2023-09-23T20:25:19.460 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:19 smithi149 ceph-mon[170394]: failed to encode map e171 with expected crc 2023-09-23T20:25:19.460 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:19 smithi149 ceph-mon[170394]: failed to encode map e171 with expected crc 2023-09-23T20:25:19.461 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:19 smithi149 ceph-mon[170394]: failed to encode map e171 with expected crc 2023-09-23T20:25:19.491 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:25:19.531 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:19.577 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:25:19.582 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:19.583 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:25:19.683 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:25:19.721 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:19.880 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:25:19.884 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:19.884 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:25:19.985 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:25:20.054 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:20.121 INFO:teuthology.orchestra.run.smithi018.stderr:noscrub is unset 2023-09-23T20:25:20.140 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:25:20.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:20 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2529420371' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:25:20.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:20 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]: dispatch 2023-09-23T20:25:20.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:20 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2529420371' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:25:20.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:20 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]: dispatch 2023-09-23T20:25:20.143 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:20.143 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:25:20.244 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:25:20.290 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:20.333 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:25:20.344 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:25:20.351 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:20.352 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:25:20.452 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:25:20.471 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:20 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2529420371' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:25:20.471 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:20 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]: dispatch 2023-09-23T20:25:20.492 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:20.633 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:25:20.655 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:25:20.678 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:25:20.704 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:25:20.744 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:25:20.768 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:25:20.773 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:25:20.773 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:20.774 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:25:20.809 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:25:20.842 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:25:20.844 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:25:20.844 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:25:20.844 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:25:20.845 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:25:20.845 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:25:20.845 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:25:20.845 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:25:20.845 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:25:20.874 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:25:20.982 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:21.102 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:21 smithi018 ceph-mon[184144]: pgmap v364: 99 pgs: 1 active+recovery_wait+degraded, 83 active+clean, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped; 4.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 550 op/s; 25906/238254 objects degraded (10.873%); 9822/238254 objects misplaced (4.122%); 4.8 MiB/s, 119 objects/s recovering 2023-09-23T20:25:21.102 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:21 smithi018 ceph-mon[184144]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:25:21.102 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:21 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2529420371' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:25:21.102 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:21 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]': finished 2023-09-23T20:25:21.102 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:21 smithi018 ceph-mon[184144]: osdmap e172: 8 total, 8 up, 7 in 2023-09-23T20:25:21.102 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:21 smithi018 ceph-mon[184144]: failed to encode map e172 with expected crc 2023-09-23T20:25:21.102 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:21 smithi018 ceph-mon[184144]: failed to encode map e172 with expected crc 2023-09-23T20:25:21.102 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:21 smithi018 ceph-mon[184144]: failed to encode map e172 with expected crc 2023-09-23T20:25:21.103 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:21 smithi018 ceph-mon[184144]: failed to encode map e172 with expected crc 2023-09-23T20:25:21.103 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:21 smithi018 ceph-mon[184144]: failed to encode map e172 with expected crc 2023-09-23T20:25:21.103 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:21 smithi018 ceph-mon[184144]: failed to encode map e172 with expected crc 2023-09-23T20:25:21.103 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:21 smithi018 ceph-mon[184144]: failed to encode map e172 with expected crc 2023-09-23T20:25:21.103 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:21 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2052930895' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:25:21.103 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:21 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 25906/238254 objects degraded (10.873%), 15 pgs degraded (PG_DEGRADED) 2023-09-23T20:25:21.103 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:21 smithi018 ceph-mon[179647]: pgmap v364: 99 pgs: 1 active+recovery_wait+degraded, 83 active+clean, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped; 4.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 550 op/s; 25906/238254 objects degraded (10.873%); 9822/238254 objects misplaced (4.122%); 4.8 MiB/s, 119 objects/s recovering 2023-09-23T20:25:21.103 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:21 smithi018 ceph-mon[179647]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:25:21.103 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:21 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2529420371' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:25:21.103 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:21 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]': finished 2023-09-23T20:25:21.104 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:21 smithi018 ceph-mon[179647]: osdmap e172: 8 total, 8 up, 7 in 2023-09-23T20:25:21.104 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:21 smithi018 ceph-mon[179647]: failed to encode map e172 with expected crc 2023-09-23T20:25:21.104 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:21 smithi018 ceph-mon[179647]: failed to encode map e172 with expected crc 2023-09-23T20:25:21.104 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:21 smithi018 ceph-mon[179647]: failed to encode map e172 with expected crc 2023-09-23T20:25:21.104 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:21 smithi018 ceph-mon[179647]: failed to encode map e172 with expected crc 2023-09-23T20:25:21.104 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:21 smithi018 ceph-mon[179647]: failed to encode map e172 with expected crc 2023-09-23T20:25:21.104 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:21 smithi018 ceph-mon[179647]: failed to encode map e172 with expected crc 2023-09-23T20:25:21.104 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:21 smithi018 ceph-mon[179647]: failed to encode map e172 with expected crc 2023-09-23T20:25:21.104 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:21 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2052930895' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:25:21.104 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:21 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 25906/238254 objects degraded (10.873%), 15 pgs degraded (PG_DEGRADED) 2023-09-23T20:25:21.218 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:21 smithi149 ceph-mon[170394]: pgmap v364: 99 pgs: 1 active+recovery_wait+degraded, 83 active+clean, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped; 4.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 550 op/s; 25906/238254 objects degraded (10.873%); 9822/238254 objects misplaced (4.122%); 4.8 MiB/s, 119 objects/s recovering 2023-09-23T20:25:21.218 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:21 smithi149 ceph-mon[170394]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:25:21.218 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:21 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2529420371' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:25:21.218 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:21 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]': finished 2023-09-23T20:25:21.218 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:21 smithi149 ceph-mon[170394]: osdmap e172: 8 total, 8 up, 7 in 2023-09-23T20:25:21.218 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:21 smithi149 ceph-mon[170394]: failed to encode map e172 with expected crc 2023-09-23T20:25:21.219 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:21 smithi149 ceph-mon[170394]: failed to encode map e172 with expected crc 2023-09-23T20:25:21.219 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:21 smithi149 ceph-mon[170394]: failed to encode map e172 with expected crc 2023-09-23T20:25:21.219 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:21 smithi149 ceph-mon[170394]: failed to encode map e172 with expected crc 2023-09-23T20:25:21.219 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:21 smithi149 ceph-mon[170394]: failed to encode map e172 with expected crc 2023-09-23T20:25:21.219 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:21 smithi149 ceph-mon[170394]: failed to encode map e172 with expected crc 2023-09-23T20:25:21.219 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:21 smithi149 ceph-mon[170394]: failed to encode map e172 with expected crc 2023-09-23T20:25:21.219 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:21 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2052930895' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:25:21.219 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:21 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 25906/238254 objects degraded (10.873%), 15 pgs degraded (PG_DEGRADED) 2023-09-23T20:25:21.376 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:25:21.386 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:21.386 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:25:21.487 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:25:21.524 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:21.573 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:25:21.582 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:21.583 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:25:21.684 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:25:21.718 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:21.763 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:25:21.767 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:21.768 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:25:21.868 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:25:21.907 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:21.960 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:25:21.965 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:21.966 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:25:22.066 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:25:22.107 INFO:teuthology.orchestra.run.smithi149.stdout:ERROR: (22) Invalid argument 2023-09-23T20:25:22.107 INFO:teuthology.orchestra.run.smithi149.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:25:22.154 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:22.244 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:25:22.259 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:22.260 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:25:22.360 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:25:22.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:22 smithi018 ceph-mon[184144]: osdmap e173: 8 total, 8 up, 7 in 2023-09-23T20:25:22.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:22 smithi018 ceph-mon[184144]: failed to encode map e173 with expected crc 2023-09-23T20:25:22.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:22 smithi018 ceph-mon[184144]: failed to encode map e173 with expected crc 2023-09-23T20:25:22.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:22 smithi018 ceph-mon[184144]: failed to encode map e173 with expected crc 2023-09-23T20:25:22.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:22 smithi018 ceph-mon[179647]: osdmap e173: 8 total, 8 up, 7 in 2023-09-23T20:25:22.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:22 smithi018 ceph-mon[179647]: failed to encode map e173 with expected crc 2023-09-23T20:25:22.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:22 smithi018 ceph-mon[179647]: failed to encode map e173 with expected crc 2023-09-23T20:25:22.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:22 smithi018 ceph-mon[179647]: failed to encode map e173 with expected crc 2023-09-23T20:25:22.404 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:22.463 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:25:22.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:22 smithi149 ceph-mon[170394]: osdmap e173: 8 total, 8 up, 7 in 2023-09-23T20:25:22.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:22 smithi149 ceph-mon[170394]: failed to encode map e173 with expected crc 2023-09-23T20:25:22.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:22 smithi149 ceph-mon[170394]: failed to encode map e173 with expected crc 2023-09-23T20:25:22.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:22 smithi149 ceph-mon[170394]: failed to encode map e173 with expected crc 2023-09-23T20:25:22.467 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:22.468 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:25:22.568 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:25:22.595 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T20:25:22.653 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:22.749 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:25:22.759 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:22.759 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:25:22.827 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T20:25:22.861 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:25:22.977 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:23.073 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:25:23.101 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:23.101 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:25:23.202 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:25:23.242 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:23.286 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:25:23.288 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:25:23.291 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:23.292 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:25:23.392 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:25:23.476 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:23 smithi149 ceph-mon[170394]: pgmap v367: 109 pgs: 1 peering, 10 unknown, 1 active+recovery_wait+degraded, 82 active+clean, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped; 4.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 625 op/s; 25797/228183 objects degraded (11.305%); 9270/228183 objects misplaced (4.063%); 7.6 MiB/s, 167 objects/s recovering 2023-09-23T20:25:23.521 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:23.554 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:25:23.578 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:25:23.582 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:23.582 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:25:23.606 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:25:23.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:23 smithi018 ceph-mon[184144]: pgmap v367: 109 pgs: 1 peering, 10 unknown, 1 active+recovery_wait+degraded, 82 active+clean, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped; 4.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 625 op/s; 25797/228183 objects degraded (11.305%); 9270/228183 objects misplaced (4.063%); 7.6 MiB/s, 167 objects/s recovering 2023-09-23T20:25:23.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:23 smithi018 ceph-mon[179647]: pgmap v367: 109 pgs: 1 peering, 10 unknown, 1 active+recovery_wait+degraded, 82 active+clean, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped; 4.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 625 op/s; 25797/228183 objects degraded (11.305%); 9270/228183 objects misplaced (4.063%); 7.6 MiB/s, 167 objects/s recovering 2023-09-23T20:25:23.683 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:25:23.714 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:24.091 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 4, 5, 6, 7, 3] out_osds: [2] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-23T20:25:24.091 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:25:24.091 INFO:tasks.thrashosds.thrasher:inject_pause on osd.1 2023-09-23T20:25:24.091 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T20:25:24.091 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T20:25:24.092 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 config set filestore_inject_stall 3 2023-09-23T20:25:24.765 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:25:24.768 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:24.769 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:25:24.869 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:25:24.903 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:25.141 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:25:24 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:25:24] "GET /metrics HTTP/1.1" 200 36502 "" "Prometheus/2.43.0" 2023-09-23T20:25:25.185 INFO:teuthology.orchestra.run.smithi149.stdout:ERROR: (22) Invalid argument 2023-09-23T20:25:25.186 INFO:teuthology.orchestra.run.smithi149.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:25:25.468 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:25 smithi149 ceph-mon[170394]: pgmap v368: 109 pgs: 1 active+recovery_wait+degraded, 1 peering, 10 unknown, 82 active+clean, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped; 4.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 522 op/s; 25797/228183 objects degraded (11.305%); 9270/228183 objects misplaced (4.063%); 5.5 MiB/s, 91 objects/s recovering 2023-09-23T20:25:25.537 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:25 smithi018 ceph-mon[179647]: pgmap v368: 109 pgs: 1 active+recovery_wait+degraded, 1 peering, 10 unknown, 82 active+clean, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped; 4.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 522 op/s; 25797/228183 objects degraded (11.305%); 9270/228183 objects misplaced (4.063%); 5.5 MiB/s, 91 objects/s recovering 2023-09-23T20:25:25.537 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:25 smithi018 ceph-mon[184144]: pgmap v368: 109 pgs: 1 active+recovery_wait+degraded, 1 peering, 10 unknown, 82 active+clean, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped; 4.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 522 op/s; 25797/228183 objects degraded (11.305%); 9270/228183 objects misplaced (4.063%); 5.5 MiB/s, 91 objects/s recovering 2023-09-23T20:25:25.551 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:25:25.555 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:25.556 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:25:25.656 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:25:25.733 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T20:25:25.778 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:25.842 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:25:25.854 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:25.855 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:25:25.855 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:25:25.956 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:25:25.997 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:26.234 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:25:26.237 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:26.238 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:25:26.280 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:25:26.339 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:25:26.433 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:26 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 25797/228183 objects degraded (11.305%), 15 pgs degraded (PG_DEGRADED) 2023-09-23T20:25:26.434 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:26 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 25797/228183 objects degraded (11.305%), 15 pgs degraded (PG_DEGRADED) 2023-09-23T20:25:26.445 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:26 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 25797/228183 objects degraded (11.305%), 15 pgs degraded (PG_DEGRADED) 2023-09-23T20:25:26.453 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:26.523 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:25:26.538 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:26.539 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:25:26.587 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:25:26.639 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:25:26.739 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:26.807 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:25:26.815 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:26.816 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:25:26.916 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:25:27.005 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:27.117 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:25:27.130 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:27.131 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:25:27.232 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:25:27.379 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:27 smithi149 ceph-mon[170394]: pgmap v369: 109 pgs: 1 active+recovery_wait+degraded, 1 peering, 10 unknown, 82 active+clean, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped; 4.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 684 op/s; 25797/222360 objects degraded (11.601%); 8943/222360 objects misplaced (4.022%); 6.4 MiB/s, 104 objects/s recovering 2023-09-23T20:25:27.398 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:27.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:27 smithi018 ceph-mon[184144]: pgmap v369: 109 pgs: 1 active+recovery_wait+degraded, 1 peering, 10 unknown, 82 active+clean, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped; 4.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 684 op/s; 25797/222360 objects degraded (11.601%); 8943/222360 objects misplaced (4.022%); 6.4 MiB/s, 104 objects/s recovering 2023-09-23T20:25:27.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:27 smithi018 ceph-mon[179647]: pgmap v369: 109 pgs: 1 active+recovery_wait+degraded, 1 peering, 10 unknown, 82 active+clean, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped; 4.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 684 op/s; 25797/222360 objects degraded (11.601%); 8943/222360 objects misplaced (4.022%); 6.4 MiB/s, 104 objects/s recovering 2023-09-23T20:25:28.223 INFO:teuthology.orchestra.run.smithi018.stderr:nodeep-scrub is unset 2023-09-23T20:25:28.242 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:28 smithi018 ceph-mon[184144]: pgmap v370: 109 pgs: 1 active+recovery_wait+degraded, 93 active+clean, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped; 4.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 835 op/s; 25655/218205 objects degraded (11.757%); 8739/218205 objects misplaced (4.005%); 8.8 MiB/s, 140 objects/s recovering 2023-09-23T20:25:28.242 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:28 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2500684226' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:25:28.246 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:28 smithi018 ceph-mon[179647]: pgmap v370: 109 pgs: 1 active+recovery_wait+degraded, 93 active+clean, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped; 4.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 835 op/s; 25655/218205 objects degraded (11.757%); 8739/218205 objects misplaced (4.005%); 8.8 MiB/s, 140 objects/s recovering 2023-09-23T20:25:28.246 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:28 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2500684226' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:25:28.271 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:25:28.302 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:25:28.335 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:25:28.376 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:25:28.408 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:25:28.441 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:25:28.476 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:25:28.511 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:28 smithi149 ceph-mon[170394]: pgmap v370: 109 pgs: 1 active+recovery_wait+degraded, 93 active+clean, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped; 4.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 835 op/s; 25655/218205 objects degraded (11.757%); 8739/218205 objects misplaced (4.005%); 8.8 MiB/s, 140 objects/s recovering 2023-09-23T20:25:28.511 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:28 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2500684226' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:25:28.528 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:25:28.532 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:25:28.532 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:25:28.532 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:25:28.533 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:25:28.533 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:25:28.533 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:25:28.533 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:25:28.533 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:25:28.583 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2023-09-23T20:25:28.583 INFO:teuthology.orchestra.run.smithi018.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-09-23T20:25:28.583 INFO:teuthology.orchestra.run.smithi018.stdout:} 2023-09-23T20:25:29.121 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:25:29.166 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:29.167 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:25:29.215 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:29 smithi018 ceph-mon[179647]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:25:29.215 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:29 smithi018 ceph-mon[179647]: failed to encode map e174 with expected crc 2023-09-23T20:25:29.215 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:29 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2500684226' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:25:29.215 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:29 smithi018 ceph-mon[179647]: osdmap e174: 8 total, 8 up, 7 in 2023-09-23T20:25:29.215 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:29 smithi018 ceph-mon[179647]: failed to encode map e174 with expected crc 2023-09-23T20:25:29.215 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:29 smithi018 ceph-mon[179647]: failed to encode map e174 with expected crc 2023-09-23T20:25:29.215 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:29 smithi018 ceph-mon[179647]: failed to encode map e174 with expected crc 2023-09-23T20:25:29.215 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:29 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2385470730' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:25:29.268 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:25:29.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:29 smithi149 ceph-mon[170394]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:25:29.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:29 smithi149 ceph-mon[170394]: failed to encode map e174 with expected crc 2023-09-23T20:25:29.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:29 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2500684226' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:25:29.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:29 smithi149 ceph-mon[170394]: osdmap e174: 8 total, 8 up, 7 in 2023-09-23T20:25:29.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:29 smithi149 ceph-mon[170394]: failed to encode map e174 with expected crc 2023-09-23T20:25:29.468 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:29 smithi149 ceph-mon[170394]: failed to encode map e174 with expected crc 2023-09-23T20:25:29.468 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:29 smithi149 ceph-mon[170394]: failed to encode map e174 with expected crc 2023-09-23T20:25:29.468 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:29 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2385470730' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:25:29.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:29 smithi018 ceph-mon[184144]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:25:29.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:29 smithi018 ceph-mon[184144]: failed to encode map e174 with expected crc 2023-09-23T20:25:29.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:29 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2500684226' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:25:29.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:29 smithi018 ceph-mon[184144]: osdmap e174: 8 total, 8 up, 7 in 2023-09-23T20:25:29.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:29 smithi018 ceph-mon[184144]: failed to encode map e174 with expected crc 2023-09-23T20:25:29.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:29 smithi018 ceph-mon[184144]: failed to encode map e174 with expected crc 2023-09-23T20:25:29.643 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:29 smithi018 ceph-mon[184144]: failed to encode map e174 with expected crc 2023-09-23T20:25:29.643 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:29 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2385470730' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:25:30.218 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:30.723 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:30 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:25:30.723 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:30 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:25:30.723 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:30 smithi149 ceph-mon[170394]: pgmap v372: 109 pgs: 1 active+recovery_wait+degraded, 93 active+clean, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped; 4.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 100 KiB/s rd, 262 KiB/s wr, 477 op/s; 25655/218205 objects degraded (11.757%); 8739/218205 objects misplaced (4.005%); 4.5 MiB/s, 69 objects/s recovering 2023-09-23T20:25:30.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:30 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:25:30.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:30 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:25:30.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:30 smithi018 ceph-mon[179647]: pgmap v372: 109 pgs: 1 active+recovery_wait+degraded, 93 active+clean, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped; 4.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 100 KiB/s rd, 262 KiB/s wr, 477 op/s; 25655/218205 objects degraded (11.757%); 8739/218205 objects misplaced (4.005%); 4.5 MiB/s, 69 objects/s recovering 2023-09-23T20:25:30.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:30 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:25:30.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:30 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:25:30.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:30 smithi018 ceph-mon[184144]: pgmap v372: 109 pgs: 1 active+recovery_wait+degraded, 93 active+clean, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped; 4.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 100 KiB/s rd, 262 KiB/s wr, 477 op/s; 25655/218205 objects degraded (11.757%); 8739/218205 objects misplaced (4.005%); 4.5 MiB/s, 69 objects/s recovering 2023-09-23T20:25:30.965 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T20:25:31.237 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:25:31.249 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:31.250 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:25:31.325 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:25:31.351 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:25:31.414 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:31.474 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:25:31.481 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:31.482 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:25:31.583 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:25:31.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:31 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 25655/218205 objects degraded (11.757%), 15 pgs degraded (PG_DEGRADED) 2023-09-23T20:25:31.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:31 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 25655/218205 objects degraded (11.757%), 15 pgs degraded (PG_DEGRADED) 2023-09-23T20:25:31.663 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:31.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:31 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 25655/218205 objects degraded (11.757%), 15 pgs degraded (PG_DEGRADED) 2023-09-23T20:25:31.720 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:25:31.726 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:31.726 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:25:31.765 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:25:31.827 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:25:31.874 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:32.020 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:25:32.024 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:25:32.025 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:32.026 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:25:32.127 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:25:32.207 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:32.363 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:25:32.374 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:32.374 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:25:32.475 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:25:32.611 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:32 smithi149 ceph-mon[170394]: pgmap v373: 109 pgs: 1 active+recovery_wait+degraded, 93 active+clean, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped; 4.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 84 KiB/s rd, 220 KiB/s wr, 746 op/s; 25551/207819 objects degraded (12.295%); 8176/207819 objects misplaced (3.934%); 7.3 MiB/s, 114 objects/s recovering 2023-09-23T20:25:32.611 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:32 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:25:32.622 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:32.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:32 smithi018 ceph-mon[184144]: pgmap v373: 109 pgs: 1 active+recovery_wait+degraded, 93 active+clean, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped; 4.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 84 KiB/s rd, 220 KiB/s wr, 746 op/s; 25551/207819 objects degraded (12.295%); 8176/207819 objects misplaced (3.934%); 7.3 MiB/s, 114 objects/s recovering 2023-09-23T20:25:32.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:32 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:25:32.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:32 smithi018 ceph-mon[179647]: pgmap v373: 109 pgs: 1 active+recovery_wait+degraded, 93 active+clean, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped; 4.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 84 KiB/s rd, 220 KiB/s wr, 746 op/s; 25551/207819 objects degraded (12.295%); 8176/207819 objects misplaced (3.934%); 7.3 MiB/s, 114 objects/s recovering 2023-09-23T20:25:32.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:32 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:25:32.787 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:25:32.792 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:32.792 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:25:32.893 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:25:32.962 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:33.053 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:25:33.065 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:33.066 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:25:33.166 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:25:33.458 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:33.593 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:25:33.608 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:33.608 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:25:33.709 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:25:33.775 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:33.832 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:25:33.837 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:33.838 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:25:33.897 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:25:33.936 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:25:33.938 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:25:33.960 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:25:33.980 INFO:teuthology.orchestra.run.smithi149.stdout:{ 2023-09-23T20:25:33.980 INFO:teuthology.orchestra.run.smithi149.stdout: "size": 20, 2023-09-23T20:25:33.981 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 600, 2023-09-23T20:25:33.981 INFO:teuthology.orchestra.run.smithi149.stdout: "ops": [ 2023-09-23T20:25:33.981 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:33.981 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:28909 5.1 5:825aa99e:::benchmark_data_smithi018_228451_object28908:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e118)", 2023-09-23T20:25:33.981 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:09.981257+0000", 2023-09-23T20:25:33.981 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 143.99431871799999, 2023-09-23T20:25:33.981 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0826966440000001, 2023-09-23T20:25:33.981 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:25:33.981 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:25:33.981 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:25:33.981 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:25:33.981 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:25:33.982 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 28909 2023-09-23T20:25:33.982 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:33.982 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:25:33.982 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:33.982 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:25:33.982 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:09.981257+0000", 2023-09-23T20:25:33.982 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:25:33.982 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:33.982 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:33.982 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:25:33.982 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:09.981257+0000", 2023-09-23T20:25:33.982 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:25:33.983 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:33.983 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:33.983 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:25:33.983 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:09.981255+0000", 2023-09-23T20:25:33.983 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999981 2023-09-23T20:25:33.983 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:33.983 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:33.983 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:25:33.983 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:09.981345+0000", 2023-09-23T20:25:33.983 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.9339999999999995e-05 2023-09-23T20:25:33.983 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:33.983 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:33.984 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:25:33.984 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:09.981360+0000", 2023-09-23T20:25:33.984 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.5591999999999999e-05 2023-09-23T20:25:33.984 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:33.984 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:33.984 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:25:33.984 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:09.981370+0000", 2023-09-23T20:25:33.984 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.6069999999999997e-06 2023-09-23T20:25:33.984 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:33.984 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:33.984 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:25:33.984 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:10.015283+0000", 2023-09-23T20:25:33.985 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.033912963999999997 2023-09-23T20:25:33.985 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:33.985 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:33.985 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:25:33.985 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:10.015297+0000", 2023-09-23T20:25:33.985 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.4477e-05 2023-09-23T20:25:33.985 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:33.985 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:33.985 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:25:33.985 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.054705+0000", 2023-09-23T20:25:33.985 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.03940805 2023-09-23T20:25:33.985 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:33.985 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:33.986 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:25:33.986 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.054798+0000", 2023-09-23T20:25:33.986 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.2872000000000001e-05 2023-09-23T20:25:33.986 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:33.986 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:33.986 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 3,7", 2023-09-23T20:25:33.986 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.054907+0000", 2023-09-23T20:25:33.986 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000108952 2023-09-23T20:25:33.986 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:33.986 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:33.986 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:25:33.987 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.056066+0000", 2023-09-23T20:25:33.987 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0011585460000000001 2023-09-23T20:25:33.987 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:33.987 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:33.987 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:25:33.987 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.056406+0000", 2023-09-23T20:25:33.987 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000340064 2023-09-23T20:25:33.987 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:33.987 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:33.987 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:25:33.987 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.063876+0000", 2023-09-23T20:25:33.987 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.007470661 2023-09-23T20:25:33.987 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:33.987 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:33.988 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:25:33.988 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.063917+0000", 2023-09-23T20:25:33.988 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.1151e-05 2023-09-23T20:25:33.988 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:33.988 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:33.988 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:25:33.988 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.063954+0000", 2023-09-23T20:25:33.988 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.6075999999999998e-05 2023-09-23T20:25:33.988 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:25:33.988 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:25:33.988 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:25:33.988 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:33.988 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:33.989 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:76669 5.18 5.af594e58 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e147)", 2023-09-23T20:25:33.989 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:58.071853+0000", 2023-09-23T20:25:33.989 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 95.903722115999997, 2023-09-23T20:25:33.989 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.86360499599999996, 2023-09-23T20:25:33.989 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:25:33.989 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:25:33.989 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:25:33.989 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:25:33.989 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:25:33.989 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 76669 2023-09-23T20:25:33.989 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:33.989 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:25:33.989 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:33.989 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:25:33.990 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:58.071853+0000", 2023-09-23T20:25:33.990 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:25:33.990 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:33.990 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:33.990 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:25:33.990 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:58.071853+0000", 2023-09-23T20:25:33.990 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:25:33.990 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:33.990 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:33.990 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:25:33.990 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:58.071853+0000", 2023-09-23T20:25:33.990 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:25:33.991 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:33.991 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:33.991 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:25:33.991 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:58.071910+0000", 2023-09-23T20:25:33.991 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.7513000000000003e-05 2023-09-23T20:25:33.991 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:33.991 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:33.991 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:25:33.991 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:58.071922+0000", 2023-09-23T20:25:33.991 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.2041e-05 2023-09-23T20:25:33.991 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:33.991 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:33.991 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:25:33.992 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:58.071926+0000", 2023-09-23T20:25:33.992 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.2660000000000003e-06 2023-09-23T20:25:33.992 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:33.992 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:33.992 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:25:33.992 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:58.071963+0000", 2023-09-23T20:25:33.992 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.6186999999999998e-05 2023-09-23T20:25:33.992 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:33.992 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:33.992 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:25:33.992 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:58.071968+0000", 2023-09-23T20:25:33.992 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.9200000000000003e-06 2023-09-23T20:25:33.992 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:33.992 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:33.993 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:25:33.993 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:58.935445+0000", 2023-09-23T20:25:33.993 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.86347702999999998 2023-09-23T20:25:33.993 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:33.993 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:33.993 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:25:33.993 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:58.935458+0000", 2023-09-23T20:25:33.993 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.3918e-05 2023-09-23T20:25:33.993 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:25:33.993 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:25:33.993 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:25:33.993 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:33.993 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:33.994 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:76679 5.18 5.48e67dd8 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e147)", 2023-09-23T20:25:33.994 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:58.103169+0000", 2023-09-23T20:25:33.994 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 95.872407022999994, 2023-09-23T20:25:33.994 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.83523706399999997, 2023-09-23T20:25:33.994 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:25:33.994 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:25:33.994 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:25:33.994 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:25:33.994 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:25:33.994 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 76679 2023-09-23T20:25:33.994 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:33.994 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:25:33.995 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:33.995 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:25:33.995 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:58.103169+0000", 2023-09-23T20:25:33.995 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:25:33.995 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:33.995 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:33.995 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:25:33.995 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:58.103169+0000", 2023-09-23T20:25:33.995 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:25:33.995 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:33.995 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:33.995 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:25:33.995 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:58.103167+0000", 2023-09-23T20:25:33.996 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:25:33.996 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:33.996 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:33.996 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:25:33.997 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:25:33.997 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:58.103270+0000", 2023-09-23T20:25:33.997 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00010306000000000001 2023-09-23T20:25:33.997 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:33.997 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:33.997 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:25:33.997 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:58.103287+0000", 2023-09-23T20:25:33.998 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.7147e-05 2023-09-23T20:25:33.998 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:33.998 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:33.998 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:25:33.998 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:58.103294+0000", 2023-09-23T20:25:33.998 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.8090000000000004e-06 2023-09-23T20:25:33.998 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:33.998 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:33.998 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:25:33.998 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:58.103352+0000", 2023-09-23T20:25:33.998 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.7491000000000002e-05 2023-09-23T20:25:33.998 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:33.999 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:33.999 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:25:33.999 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:58.103359+0000", 2023-09-23T20:25:33.999 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.1999999999999997e-06 2023-09-23T20:25:33.999 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:33.999 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:33.999 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:25:33.999 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:58.938390+0000", 2023-09-23T20:25:33.999 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.83503123000000001 2023-09-23T20:25:33.999 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:33.999 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:33.999 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:25:33.999 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:58.938406+0000", 2023-09-23T20:25:34.000 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.5665e-05 2023-09-23T20:25:34.000 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:25:34.000 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:25:34.000 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:25:34.000 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.000 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.000 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:76873 5.18 5.9a8cb918 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e147)", 2023-09-23T20:25:34.000 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:58.246134+0000", 2023-09-23T20:25:34.000 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 95.729441498, 2023-09-23T20:25:34.000 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.69232187000000001, 2023-09-23T20:25:34.000 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:25:34.000 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:25:34.000 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:25:34.001 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:25:34.001 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:25:34.001 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 76873 2023-09-23T20:25:34.001 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.001 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:25:34.001 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.001 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:25:34.001 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:58.246134+0000", 2023-09-23T20:25:34.001 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:25:34.001 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.001 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.001 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:25:34.001 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:58.246134+0000", 2023-09-23T20:25:34.002 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:25:34.002 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.002 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.002 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:25:34.002 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:58.246133+0000", 2023-09-23T20:25:34.002 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:25:34.002 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.002 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.002 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:25:34.002 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:58.246193+0000", 2023-09-23T20:25:34.002 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.0713e-05 2023-09-23T20:25:34.002 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.003 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.003 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:25:34.003 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:58.246209+0000", 2023-09-23T20:25:34.003 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.6067e-05 2023-09-23T20:25:34.003 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.003 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.003 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:25:34.003 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:58.246216+0000", 2023-09-23T20:25:34.003 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.2790000000000004e-06 2023-09-23T20:25:34.003 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.003 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.003 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:25:34.003 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:58.246271+0000", 2023-09-23T20:25:34.004 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.499e-05 2023-09-23T20:25:34.004 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.004 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.004 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:25:34.004 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:58.246280+0000", 2023-09-23T20:25:34.004 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.3619999999999995e-06 2023-09-23T20:25:34.004 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.004 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.004 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:25:34.004 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:58.938442+0000", 2023-09-23T20:25:34.004 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.69216180900000002 2023-09-23T20:25:34.004 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.005 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.005 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:25:34.005 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:58.938456+0000", 2023-09-23T20:25:34.005 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.412e-05 2023-09-23T20:25:34.005 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:25:34.005 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:25:34.005 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:25:34.005 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.005 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.005 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:76939 5.18 5.b6118d18 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e147)", 2023-09-23T20:25:34.005 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:58.304520+0000", 2023-09-23T20:25:34.005 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 95.671055222999996, 2023-09-23T20:25:34.005 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.63397631499999996, 2023-09-23T20:25:34.006 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:25:34.006 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:25:34.006 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:25:34.006 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:25:34.006 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:25:34.006 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 76939 2023-09-23T20:25:34.006 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.006 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:25:34.006 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.006 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:25:34.006 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:58.304520+0000", 2023-09-23T20:25:34.006 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:25:34.006 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.007 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.007 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:25:34.007 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:58.304520+0000", 2023-09-23T20:25:34.007 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:25:34.007 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.007 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.007 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:25:34.007 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:58.304518+0000", 2023-09-23T20:25:34.007 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999981 2023-09-23T20:25:34.007 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.007 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.007 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:25:34.007 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:58.304606+0000", 2023-09-23T20:25:34.008 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.7961999999999996e-05 2023-09-23T20:25:34.008 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.008 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.008 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:25:34.008 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:58.304623+0000", 2023-09-23T20:25:34.008 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.6586999999999999e-05 2023-09-23T20:25:34.008 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.008 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.008 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:25:34.008 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:58.304630+0000", 2023-09-23T20:25:34.008 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.7920000000000004e-06 2023-09-23T20:25:34.008 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.008 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.009 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:25:34.009 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:58.304749+0000", 2023-09-23T20:25:34.009 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000119045 2023-09-23T20:25:34.009 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.009 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.009 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:25:34.009 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:58.304756+0000", 2023-09-23T20:25:34.009 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.4789999999999999e-06 2023-09-23T20:25:34.009 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.009 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.009 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:25:34.009 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:58.938478+0000", 2023-09-23T20:25:34.009 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.63372185000000003 2023-09-23T20:25:34.010 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.010 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.010 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:25:34.010 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:58.938497+0000", 2023-09-23T20:25:34.010 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.8476999999999999e-05 2023-09-23T20:25:34.010 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:25:34.010 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:25:34.010 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:25:34.010 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.010 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.010 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:76943 5.18 5.104746d8 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e147)", 2023-09-23T20:25:34.010 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:58.304682+0000", 2023-09-23T20:25:34.010 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 95.670893113999995, 2023-09-23T20:25:34.011 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.63384238500000001, 2023-09-23T20:25:34.011 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:25:34.011 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:25:34.011 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:25:34.011 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:25:34.011 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:25:34.011 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 76943 2023-09-23T20:25:34.011 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.011 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:25:34.011 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.011 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:25:34.011 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:58.304682+0000", 2023-09-23T20:25:34.011 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:25:34.012 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.012 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.012 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:25:34.012 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:58.304682+0000", 2023-09-23T20:25:34.012 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:25:34.012 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.012 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.012 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:25:34.012 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:58.304682+0000", 2023-09-23T20:25:34.012 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:25:34.012 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.012 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.013 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:25:34.013 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:58.304762+0000", 2023-09-23T20:25:34.013 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.0663999999999999e-05 2023-09-23T20:25:34.013 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.013 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.013 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:25:34.013 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:58.304773+0000", 2023-09-23T20:25:34.013 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0195e-05 2023-09-23T20:25:34.013 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.013 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.013 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:25:34.013 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:58.304778+0000", 2023-09-23T20:25:34.014 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.9229999999999999e-06 2023-09-23T20:25:34.014 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.014 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.014 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:25:34.014 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:58.304822+0000", 2023-09-23T20:25:34.014 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.4020000000000002e-05 2023-09-23T20:25:34.014 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.014 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.014 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:25:34.014 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:58.304828+0000", 2023-09-23T20:25:34.014 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.2310000000000004e-06 2023-09-23T20:25:34.015 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.015 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.015 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:25:34.015 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:58.938516+0000", 2023-09-23T20:25:34.015 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.63368784099999997 2023-09-23T20:25:34.015 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.015 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.015 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:25:34.015 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:58.938525+0000", 2023-09-23T20:25:34.015 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.3170000000000008e-06 2023-09-23T20:25:34.015 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:25:34.015 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:25:34.015 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:25:34.016 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.016 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.016 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:76979 5.18 5.937ed098 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e147)", 2023-09-23T20:25:34.016 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:58.336467+0000", 2023-09-23T20:25:34.016 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 95.639108876999998, 2023-09-23T20:25:34.016 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.60208404699999996, 2023-09-23T20:25:34.016 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:25:34.016 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:25:34.016 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:25:34.016 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:25:34.016 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:25:34.016 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 76979 2023-09-23T20:25:34.016 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.017 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:25:34.017 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.017 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:25:34.017 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:58.336467+0000", 2023-09-23T20:25:34.017 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:25:34.017 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.017 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.017 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:25:34.017 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:58.336467+0000", 2023-09-23T20:25:34.017 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:25:34.017 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.017 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.018 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:25:34.018 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:58.336465+0000", 2023-09-23T20:25:34.018 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:25:34.018 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.018 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.018 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:25:34.018 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:58.336504+0000", 2023-09-23T20:25:34.018 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.8692000000000002e-05 2023-09-23T20:25:34.018 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.018 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.018 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:25:34.019 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:58.336518+0000", 2023-09-23T20:25:34.019 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.3771000000000001e-05 2023-09-23T20:25:34.019 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.019 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.019 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:25:34.019 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:58.336523+0000", 2023-09-23T20:25:34.019 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.9969999999999998e-06 2023-09-23T20:25:34.019 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.019 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.019 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:25:34.019 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:58.336566+0000", 2023-09-23T20:25:34.019 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.3182000000000001e-05 2023-09-23T20:25:34.019 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.020 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.020 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:25:34.020 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:58.336576+0000", 2023-09-23T20:25:34.020 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.8260000000000005e-06 2023-09-23T20:25:34.020 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.020 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.020 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:25:34.020 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:58.938542+0000", 2023-09-23T20:25:34.020 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.60196588600000001 2023-09-23T20:25:34.020 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.020 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.020 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:25:34.021 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:58.938551+0000", 2023-09-23T20:25:34.021 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.2280000000000004e-06 2023-09-23T20:25:34.021 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:25:34.021 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:25:34.021 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:25:34.021 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.021 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.021 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:126867 5.a 5:50925f4d:::benchmark_data_smithi018_228451_object19806:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e164)", 2023-09-23T20:25:34.021 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:25:01.833816+0000", 2023-09-23T20:25:34.021 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 32.141759157999999, 2023-09-23T20:25:34.021 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.442573232, 2023-09-23T20:25:34.021 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:25:34.022 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:25:34.022 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:25:34.022 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:25:34.022 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:25:34.022 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 126867 2023-09-23T20:25:34.022 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.022 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:25:34.022 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.022 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:25:34.022 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.833816+0000", 2023-09-23T20:25:34.022 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:25:34.023 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.023 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.023 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:25:34.023 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.833816+0000", 2023-09-23T20:25:34.023 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:25:34.023 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.023 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.023 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:25:34.023 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.833815+0000", 2023-09-23T20:25:34.023 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:25:34.023 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.023 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.024 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:25:34.024 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.833821+0000", 2023-09-23T20:25:34.024 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.122e-06 2023-09-23T20:25:34.024 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.024 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.024 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:25:34.024 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.833832+0000", 2023-09-23T20:25:34.024 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1338999999999999e-05 2023-09-23T20:25:34.024 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.024 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.024 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:25:34.024 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.833839+0000", 2023-09-23T20:25:34.025 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.0419999999999999e-06 2023-09-23T20:25:34.025 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.025 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.025 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:25:34.025 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.898829+0000", 2023-09-23T20:25:34.025 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.064990508000000002 2023-09-23T20:25:34.025 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.025 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.025 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for degraded object", 2023-09-23T20:25:34.025 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.899653+0000", 2023-09-23T20:25:34.025 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00082397299999999998 2023-09-23T20:25:34.025 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.026 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.026 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:25:34.026 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.945495+0000", 2023-09-23T20:25:34.026 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.045841848999999997 2023-09-23T20:25:34.026 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.026 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.026 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:25:34.026 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.945501+0000", 2023-09-23T20:25:34.026 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.3770000000000001e-06 2023-09-23T20:25:34.026 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.026 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.026 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:25:34.027 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.124574+0000", 2023-09-23T20:25:34.027 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1790724560000001 2023-09-23T20:25:34.027 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.027 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.027 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for degraded object", 2023-09-23T20:25:34.027 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.125110+0000", 2023-09-23T20:25:34.027 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00053609000000000003 2023-09-23T20:25:34.027 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.027 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.027 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:25:34.027 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.258824+0000", 2023-09-23T20:25:34.027 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.133714534 2023-09-23T20:25:34.027 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.028 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.028 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:25:34.028 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.258902+0000", 2023-09-23T20:25:34.028 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.7299000000000006e-05 2023-09-23T20:25:34.028 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.028 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.028 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:25:34.028 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.259000+0000", 2023-09-23T20:25:34.028 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.8595e-05 2023-09-23T20:25:34.028 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.028 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.028 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:25:34.029 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.269120+0000", 2023-09-23T20:25:34.029 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.010120013000000001 2023-09-23T20:25:34.029 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.029 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.029 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:25:34.029 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.272674+0000", 2023-09-23T20:25:34.029 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0035536449999999998 2023-09-23T20:25:34.029 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.029 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.029 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:25:34.029 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.272702+0000", 2023-09-23T20:25:34.029 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.8435000000000001e-05 2023-09-23T20:25:34.030 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.030 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.030 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:25:34.030 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.272739+0000", 2023-09-23T20:25:34.030 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.6727999999999998e-05 2023-09-23T20:25:34.030 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.030 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.030 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:25:34.030 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.276390+0000", 2023-09-23T20:25:34.030 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0036506640000000001 2023-09-23T20:25:34.030 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:25:34.031 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:25:34.031 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:25:34.031 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.031 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.031 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:126894 5.a 5:5259ce05:::benchmark_data_smithi018_228451_object19833:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e164)", 2023-09-23T20:25:34.031 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:25:01.880498+0000", 2023-09-23T20:25:34.031 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 32.095077701999998, 2023-09-23T20:25:34.031 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.3921141399999999, 2023-09-23T20:25:34.031 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:25:34.031 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:25:34.031 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:25:34.031 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:25:34.031 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:25:34.032 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 126894 2023-09-23T20:25:34.032 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.032 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:25:34.032 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.032 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:25:34.032 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.880498+0000", 2023-09-23T20:25:34.032 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:25:34.032 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.032 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.032 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:25:34.032 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.880498+0000", 2023-09-23T20:25:34.032 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:25:34.032 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.033 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.033 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:25:34.033 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.880496+0000", 2023-09-23T20:25:34.033 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:25:34.033 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.033 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.033 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:25:34.033 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.880501+0000", 2023-09-23T20:25:34.033 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.904e-06 2023-09-23T20:25:34.033 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.033 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.034 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:25:34.034 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.880511+0000", 2023-09-23T20:25:34.034 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.5380000000000008e-06 2023-09-23T20:25:34.034 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.034 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.034 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:25:34.034 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.880516+0000", 2023-09-23T20:25:34.034 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.9980000000000003e-06 2023-09-23T20:25:34.034 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.034 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.034 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:25:34.035 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.899727+0000", 2023-09-23T20:25:34.035 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.019211236999999999 2023-09-23T20:25:34.035 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.035 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.035 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for degraded object", 2023-09-23T20:25:34.035 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.900501+0000", 2023-09-23T20:25:34.035 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00077373899999999996 2023-09-23T20:25:34.035 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.035 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.035 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:25:34.035 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.945461+0000", 2023-09-23T20:25:34.035 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.044960444000000002 2023-09-23T20:25:34.036 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.036 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.036 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:25:34.036 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.945472+0000", 2023-09-23T20:25:34.036 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0738e-05 2023-09-23T20:25:34.036 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.036 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.036 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:25:34.036 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.124180+0000", 2023-09-23T20:25:34.036 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1787084809999999 2023-09-23T20:25:34.036 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.036 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.037 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for degraded object", 2023-09-23T20:25:34.037 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.124532+0000", 2023-09-23T20:25:34.037 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00035113800000000003 2023-09-23T20:25:34.037 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.037 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.037 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:25:34.037 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.250481+0000", 2023-09-23T20:25:34.037 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.12594944799999999 2023-09-23T20:25:34.037 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.037 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.037 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:25:34.037 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.250553+0000", 2023-09-23T20:25:34.038 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.1903000000000006e-05 2023-09-23T20:25:34.038 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.038 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.038 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:25:34.038 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.250646+0000", 2023-09-23T20:25:34.038 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.3293000000000003e-05 2023-09-23T20:25:34.038 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.038 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.038 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:25:34.038 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.269045+0000", 2023-09-23T20:25:34.038 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.018399174000000001 2023-09-23T20:25:34.038 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.039 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.039 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:25:34.039 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.269114+0000", 2023-09-23T20:25:34.039 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.8677000000000007e-05 2023-09-23T20:25:34.039 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.039 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.039 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:25:34.039 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.269161+0000", 2023-09-23T20:25:34.039 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.7015999999999998e-05 2023-09-23T20:25:34.039 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.039 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.039 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:25:34.039 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.269214+0000", 2023-09-23T20:25:34.040 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.3128999999999999e-05 2023-09-23T20:25:34.040 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.040 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.040 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:25:34.040 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.272612+0000", 2023-09-23T20:25:34.040 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0033978659999999998 2023-09-23T20:25:34.040 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:25:34.040 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:25:34.040 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:25:34.040 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.040 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.040 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:126925 5.a 5:5e6040ba:::benchmark_data_smithi018_228451_object19864:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e165)", 2023-09-23T20:25:34.041 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:25:01.937985+0000", 2023-09-23T20:25:34.041 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 32.037590575999999, 2023-09-23T20:25:34.041 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.341994892, 2023-09-23T20:25:34.041 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:25:34.041 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:25:34.041 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:25:34.041 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:25:34.041 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:25:34.041 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 126925 2023-09-23T20:25:34.041 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.041 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:25:34.041 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.041 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:25:34.042 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.937985+0000", 2023-09-23T20:25:34.042 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:25:34.042 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.042 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.042 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:25:34.042 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.937985+0000", 2023-09-23T20:25:34.042 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:25:34.042 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.042 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.042 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:25:34.042 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.937983+0000", 2023-09-23T20:25:34.042 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999981 2023-09-23T20:25:34.043 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.043 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.043 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:25:34.043 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.937989+0000", 2023-09-23T20:25:34.043 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.1920000000000002e-06 2023-09-23T20:25:34.043 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.043 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.043 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:25:34.043 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.937998+0000", 2023-09-23T20:25:34.043 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.5250000000000005e-06 2023-09-23T20:25:34.043 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.043 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.044 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:25:34.044 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.938003+0000", 2023-09-23T20:25:34.044 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.2129999999999999e-06 2023-09-23T20:25:34.044 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.044 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.044 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:25:34.044 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.947727+0000", 2023-09-23T20:25:34.044 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0097239900000000001 2023-09-23T20:25:34.044 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.044 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.044 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:25:34.045 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.947739+0000", 2023-09-23T20:25:34.045 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1355000000000001e-05 2023-09-23T20:25:34.045 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.045 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.045 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:25:34.045 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.125146+0000", 2023-09-23T20:25:34.045 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1774076529999999 2023-09-23T20:25:34.045 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.045 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.045 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for degraded object", 2023-09-23T20:25:34.045 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.125724+0000", 2023-09-23T20:25:34.045 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00057758600000000005 2023-09-23T20:25:34.046 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.046 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.046 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:25:34.046 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.266528+0000", 2023-09-23T20:25:34.046 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.14080421800000001 2023-09-23T20:25:34.046 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.046 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.046 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:25:34.046 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.266606+0000", 2023-09-23T20:25:34.046 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.7725999999999994e-05 2023-09-23T20:25:34.046 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.046 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.047 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:25:34.047 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.266716+0000", 2023-09-23T20:25:34.047 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00011035999999999999 2023-09-23T20:25:34.047 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.047 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.047 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:25:34.047 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.276441+0000", 2023-09-23T20:25:34.047 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0097248620000000008 2023-09-23T20:25:34.047 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.047 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.047 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:25:34.047 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.276449+0000", 2023-09-23T20:25:34.048 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.0299999999999994e-06 2023-09-23T20:25:34.048 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.048 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.048 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:25:34.048 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.276482+0000", 2023-09-23T20:25:34.048 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.2796999999999998e-05 2023-09-23T20:25:34.048 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.048 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.048 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:25:34.048 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.276533+0000", 2023-09-23T20:25:34.048 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.1218000000000003e-05 2023-09-23T20:25:34.048 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.049 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.049 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:25:34.049 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.279980+0000", 2023-09-23T20:25:34.049 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0034469639999999998 2023-09-23T20:25:34.049 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:25:34.049 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:25:34.049 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:25:34.049 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.049 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.049 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:126935 5.1 5:8421d65c:::benchmark_data_smithi018_228451_object19874:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e165)", 2023-09-23T20:25:34.049 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:25:01.964401+0000", 2023-09-23T20:25:34.049 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 32.011175084000001, 2023-09-23T20:25:34.050 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0449979250000001, 2023-09-23T20:25:34.050 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:25:34.050 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:25:34.050 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:25:34.050 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:25:34.050 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:25:34.050 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 126935 2023-09-23T20:25:34.050 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.050 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:25:34.050 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.050 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:25:34.051 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.964401+0000", 2023-09-23T20:25:34.051 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:25:34.051 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.051 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.051 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:25:34.051 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.964401+0000", 2023-09-23T20:25:34.051 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:25:34.051 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.051 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.051 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:25:34.051 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.964398+0000", 2023-09-23T20:25:34.051 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999976 2023-09-23T20:25:34.052 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.052 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.052 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:25:34.052 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.964405+0000", 2023-09-23T20:25:34.052 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.8800000000000002e-06 2023-09-23T20:25:34.052 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.052 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.052 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:25:34.052 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.964423+0000", 2023-09-23T20:25:34.052 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.7864e-05 2023-09-23T20:25:34.052 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.052 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.052 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:25:34.053 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.964429+0000", 2023-09-23T20:25:34.053 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.3910000000000003e-06 2023-09-23T20:25:34.053 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.053 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.053 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:25:34.053 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.970829+0000", 2023-09-23T20:25:34.053 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0063990360000000003 2023-09-23T20:25:34.053 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.053 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.053 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:25:34.053 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.970843+0000", 2023-09-23T20:25:34.053 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.49e-05 2023-09-23T20:25:34.054 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.054 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.054 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:25:34.054 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.002512+0000", 2023-09-23T20:25:34.054 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.031668544 2023-09-23T20:25:34.054 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.054 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.054 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:25:34.054 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.002617+0000", 2023-09-23T20:25:34.054 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00010461 2023-09-23T20:25:34.054 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.054 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.055 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 3,7", 2023-09-23T20:25:34.055 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.002730+0000", 2023-09-23T20:25:34.055 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00011356700000000001 2023-09-23T20:25:34.055 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.055 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.055 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:25:34.055 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.007261+0000", 2023-09-23T20:25:34.055 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0045309240000000004 2023-09-23T20:25:34.055 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.055 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.055 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:25:34.055 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.008570+0000", 2023-09-23T20:25:34.056 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.001309225 2023-09-23T20:25:34.056 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.056 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.056 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:25:34.056 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.009210+0000", 2023-09-23T20:25:34.056 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00063932400000000003 2023-09-23T20:25:34.056 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.056 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.056 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:25:34.056 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.009257+0000", 2023-09-23T20:25:34.056 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.7692999999999999e-05 2023-09-23T20:25:34.056 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.057 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.057 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:25:34.057 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.009398+0000", 2023-09-23T20:25:34.057 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000141123 2023-09-23T20:25:34.057 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:25:34.057 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:25:34.057 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:25:34.057 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.057 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.057 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:126945 5.1 5:856ddac8:::benchmark_data_smithi018_228451_object19884:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e165)", 2023-09-23T20:25:34.057 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:25:01.991896+0000", 2023-09-23T20:25:34.057 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 31.983679989999999, 2023-09-23T20:25:34.057 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0216826400000001, 2023-09-23T20:25:34.058 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:25:34.058 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:25:34.058 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:25:34.058 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:25:34.058 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:25:34.058 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 126945 2023-09-23T20:25:34.058 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.058 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:25:34.058 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.058 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:25:34.058 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.991896+0000", 2023-09-23T20:25:34.059 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:25:34.059 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.059 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.059 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:25:34.059 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.991896+0000", 2023-09-23T20:25:34.059 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:25:34.059 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.059 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.059 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:25:34.059 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.991894+0000", 2023-09-23T20:25:34.059 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:25:34.059 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.060 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.060 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:25:34.060 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.991900+0000", 2023-09-23T20:25:34.060 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.2790000000000004e-06 2023-09-23T20:25:34.060 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.060 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.060 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:25:34.060 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.991914+0000", 2023-09-23T20:25:34.060 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.3226e-05 2023-09-23T20:25:34.060 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.060 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.060 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:25:34.061 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.991920+0000", 2023-09-23T20:25:34.061 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.7710000000000001e-06 2023-09-23T20:25:34.061 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.061 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.061 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:25:34.061 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.991975+0000", 2023-09-23T20:25:34.061 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.5021000000000001e-05 2023-09-23T20:25:34.061 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.061 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.061 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:25:34.061 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.991985+0000", 2023-09-23T20:25:34.061 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.5999999999999996e-06 2023-09-23T20:25:34.062 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.062 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.062 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:25:34.062 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.004845+0000", 2023-09-23T20:25:34.062 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0128602600000001 2023-09-23T20:25:34.062 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.062 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.062 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:25:34.062 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.004955+0000", 2023-09-23T20:25:34.062 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000109419 2023-09-23T20:25:34.062 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.063 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.063 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 3,7", 2023-09-23T20:25:34.063 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.005108+0000", 2023-09-23T20:25:34.063 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00015359099999999999 2023-09-23T20:25:34.063 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.063 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.063 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:25:34.063 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.007360+0000", 2023-09-23T20:25:34.063 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0022521389999999998 2023-09-23T20:25:34.063 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.063 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.063 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:25:34.063 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.008577+0000", 2023-09-23T20:25:34.064 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00121667 2023-09-23T20:25:34.064 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.064 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.064 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:25:34.064 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.013397+0000", 2023-09-23T20:25:34.064 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0048202760000000001 2023-09-23T20:25:34.064 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.064 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.064 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:25:34.064 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.013444+0000", 2023-09-23T20:25:34.064 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.6585000000000002e-05 2023-09-23T20:25:34.064 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.065 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.065 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:25:34.065 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.013578+0000", 2023-09-23T20:25:34.065 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00013424899999999999 2023-09-23T20:25:34.065 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:25:34.065 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:25:34.065 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:25:34.065 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.065 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.065 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:134103 5.18 5:1940be66:::benchmark_data_smithi018_228451_object27042:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e169)", 2023-09-23T20:25:34.065 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:25:14.071252+0000", 2023-09-23T20:25:34.065 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 19.904323512000001, 2023-09-23T20:25:34.066 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0302794790000001, 2023-09-23T20:25:34.066 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:25:34.066 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:25:34.066 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:25:34.066 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:25:34.066 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:25:34.066 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 134103 2023-09-23T20:25:34.066 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.066 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:25:34.066 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.066 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:25:34.067 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.071252+0000", 2023-09-23T20:25:34.067 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:25:34.067 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.067 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.067 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:25:34.067 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.071252+0000", 2023-09-23T20:25:34.067 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:25:34.067 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.067 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.067 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:25:34.067 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.071251+0000", 2023-09-23T20:25:34.067 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:25:34.068 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.068 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.068 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:25:34.068 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.071255+0000", 2023-09-23T20:25:34.068 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.335e-06 2023-09-23T20:25:34.068 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.068 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.068 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:25:34.068 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.071266+0000", 2023-09-23T20:25:34.068 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0402e-05 2023-09-23T20:25:34.068 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.068 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.068 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:25:34.069 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.071272+0000", 2023-09-23T20:25:34.069 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.3609999999999997e-06 2023-09-23T20:25:34.069 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.069 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.069 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:25:34.069 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.071501+0000", 2023-09-23T20:25:34.069 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00022861499999999999 2023-09-23T20:25:34.069 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.069 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.069 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:25:34.069 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.071508+0000", 2023-09-23T20:25:34.069 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.8290000000000008e-06 2023-09-23T20:25:34.069 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.069 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.070 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:25:34.070 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.073848+0000", 2023-09-23T20:25:34.070 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0023397380000001 2023-09-23T20:25:34.070 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.070 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.070 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:25:34.070 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.073991+0000", 2023-09-23T20:25:34.070 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00014334999999999999 2023-09-23T20:25:34.070 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.070 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.070 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 1,6", 2023-09-23T20:25:34.071 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.074073+0000", 2023-09-23T20:25:34.071 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.1357e-05 2023-09-23T20:25:34.071 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.071 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.071 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:25:34.071 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.086054+0000", 2023-09-23T20:25:34.071 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.011981103 2023-09-23T20:25:34.071 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.071 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.071 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:25:34.071 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.101412+0000", 2023-09-23T20:25:34.071 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.015358449999999999 2023-09-23T20:25:34.072 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.072 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.072 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:25:34.072 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.101441+0000", 2023-09-23T20:25:34.072 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.8416999999999999e-05 2023-09-23T20:25:34.072 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.072 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.072 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:25:34.072 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.101492+0000", 2023-09-23T20:25:34.072 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.1372000000000002e-05 2023-09-23T20:25:34.072 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.072 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.073 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:25:34.073 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.101532+0000", 2023-09-23T20:25:34.073 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.9385999999999999e-05 2023-09-23T20:25:34.073 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:25:34.073 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:25:34.073 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:25:34.073 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.073 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.073 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:134131 5.18 5:1c0b9433:::benchmark_data_smithi018_228451_object27070:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e169)", 2023-09-23T20:25:34.073 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:25:14.103862+0000", 2023-09-23T20:25:34.073 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 19.871714006000001, 2023-09-23T20:25:34.073 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.99783719299999996, 2023-09-23T20:25:34.074 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:25:34.074 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:25:34.074 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:25:34.074 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:25:34.074 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:25:34.074 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 134131 2023-09-23T20:25:34.074 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.074 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:25:34.074 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.074 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:25:34.074 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.103862+0000", 2023-09-23T20:25:34.075 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:25:34.075 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.075 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.075 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:25:34.075 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.103862+0000", 2023-09-23T20:25:34.075 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:25:34.075 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.075 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.075 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:25:34.075 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.103861+0000", 2023-09-23T20:25:34.075 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:25:34.075 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.075 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.076 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:25:34.076 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.103864+0000", 2023-09-23T20:25:34.076 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.4109999999999999e-06 2023-09-23T20:25:34.076 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.076 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.076 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:25:34.076 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.103870+0000", 2023-09-23T20:25:34.076 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.0190000000000003e-06 2023-09-23T20:25:34.076 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.076 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.076 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:25:34.077 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.103873+0000", 2023-09-23T20:25:34.077 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.6049999999999998e-06 2023-09-23T20:25:34.077 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.077 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.077 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:25:34.077 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.103936+0000", 2023-09-23T20:25:34.077 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.2351e-05 2023-09-23T20:25:34.077 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.077 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.077 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:25:34.077 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.103942+0000", 2023-09-23T20:25:34.077 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.7009999999999999e-06 2023-09-23T20:25:34.077 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.078 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.078 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:25:34.078 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.076330+0000", 2023-09-23T20:25:34.078 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.97238725299999995 2023-09-23T20:25:34.078 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.078 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.078 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:25:34.078 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.076434+0000", 2023-09-23T20:25:34.078 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00010414 2023-09-23T20:25:34.078 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.078 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.078 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 1,6", 2023-09-23T20:25:34.079 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.076538+0000", 2023-09-23T20:25:34.079 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000104041 2023-09-23T20:25:34.079 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.079 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.079 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:25:34.079 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.086064+0000", 2023-09-23T20:25:34.079 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0095265880000000008 2023-09-23T20:25:34.079 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.079 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.079 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:25:34.079 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.101562+0000", 2023-09-23T20:25:34.079 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.015497858 2023-09-23T20:25:34.079 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.080 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.080 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:25:34.080 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.101613+0000", 2023-09-23T20:25:34.080 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.0283e-05 2023-09-23T20:25:34.080 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.080 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.080 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:25:34.080 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.101660+0000", 2023-09-23T20:25:34.080 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.7633999999999998e-05 2023-09-23T20:25:34.080 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.080 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.080 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:25:34.081 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.101699+0000", 2023-09-23T20:25:34.081 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.8646999999999998e-05 2023-09-23T20:25:34.081 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:25:34.081 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:25:34.081 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:25:34.081 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.081 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.081 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:134133 5.18 5:1bf4905b:::benchmark_data_smithi018_228451_object27072:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e169)", 2023-09-23T20:25:34.081 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:25:14.109028+0000", 2023-09-23T20:25:34.081 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 19.866548002999998, 2023-09-23T20:25:34.081 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.99277708499999995, 2023-09-23T20:25:34.081 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:25:34.081 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:25:34.081 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:25:34.082 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:25:34.082 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:25:34.082 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 134133 2023-09-23T20:25:34.082 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.082 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:25:34.082 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.082 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:25:34.082 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.109028+0000", 2023-09-23T20:25:34.082 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:25:34.082 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.082 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.083 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:25:34.083 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.109028+0000", 2023-09-23T20:25:34.083 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:25:34.083 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.083 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.083 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:25:34.083 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.109027+0000", 2023-09-23T20:25:34.083 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:25:34.083 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.083 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.083 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:25:34.083 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.109031+0000", 2023-09-23T20:25:34.084 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.4290000000000003e-06 2023-09-23T20:25:34.084 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.084 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.084 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:25:34.084 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.109047+0000", 2023-09-23T20:25:34.084 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.6314999999999999e-05 2023-09-23T20:25:34.084 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.084 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.084 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:25:34.084 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.109054+0000", 2023-09-23T20:25:34.084 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.6370000000000002e-06 2023-09-23T20:25:34.085 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.085 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.085 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:25:34.085 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.109142+0000", 2023-09-23T20:25:34.085 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.7954999999999994e-05 2023-09-23T20:25:34.085 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.085 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.085 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:25:34.085 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.109150+0000", 2023-09-23T20:25:34.085 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.4999999999999999e-06 2023-09-23T20:25:34.085 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.085 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.085 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:25:34.086 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.078060+0000", 2023-09-23T20:25:34.086 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.96890956900000003 2023-09-23T20:25:34.086 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.086 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.086 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:25:34.086 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.078173+0000", 2023-09-23T20:25:34.086 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000113341 2023-09-23T20:25:34.086 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.086 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.086 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 1,6", 2023-09-23T20:25:34.086 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.078295+0000", 2023-09-23T20:25:34.087 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000121894 2023-09-23T20:25:34.087 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.087 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.087 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:25:34.087 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.086074+0000", 2023-09-23T20:25:34.087 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0077791140000000002 2023-09-23T20:25:34.087 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.087 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.087 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:25:34.087 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.101588+0000", 2023-09-23T20:25:34.087 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.015513968 2023-09-23T20:25:34.087 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.088 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.088 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:25:34.088 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.101732+0000", 2023-09-23T20:25:34.088 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00014366499999999999 2023-09-23T20:25:34.088 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.088 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.088 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:25:34.088 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.101772+0000", 2023-09-23T20:25:34.088 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.0204999999999999e-05 2023-09-23T20:25:34.088 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.088 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.088 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:25:34.088 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.101805+0000", 2023-09-23T20:25:34.089 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.2564000000000002e-05 2023-09-23T20:25:34.089 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:25:34.089 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:25:34.089 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:25:34.089 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.089 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.089 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:134135 5.18 5:1df619a8:::benchmark_data_smithi018_228451_object27074:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e169)", 2023-09-23T20:25:34.089 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:25:14.109193+0000", 2023-09-23T20:25:34.089 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 19.866382774000002, 2023-09-23T20:25:34.089 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.99274740800000005, 2023-09-23T20:25:34.089 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:25:34.089 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:25:34.089 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:25:34.090 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:25:34.090 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:25:34.090 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 134135 2023-09-23T20:25:34.090 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.090 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:25:34.090 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.090 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:25:34.090 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.109193+0000", 2023-09-23T20:25:34.090 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:25:34.090 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.090 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.090 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:25:34.091 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.109193+0000", 2023-09-23T20:25:34.091 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:25:34.091 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.091 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.091 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:25:34.091 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.109192+0000", 2023-09-23T20:25:34.091 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:25:34.091 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.091 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.091 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:25:34.091 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.109196+0000", 2023-09-23T20:25:34.091 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.2740000000000001e-06 2023-09-23T20:25:34.092 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.092 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.092 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:25:34.092 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.109205+0000", 2023-09-23T20:25:34.092 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.0970000000000006e-06 2023-09-23T20:25:34.092 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.092 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.092 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:25:34.092 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.109211+0000", 2023-09-23T20:25:34.092 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.5400000000000001e-06 2023-09-23T20:25:34.092 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.093 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.093 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:25:34.093 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.109256+0000", 2023-09-23T20:25:34.093 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.5410000000000001e-05 2023-09-23T20:25:34.093 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.093 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.093 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:25:34.093 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.109267+0000", 2023-09-23T20:25:34.093 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.004e-05 2023-09-23T20:25:34.093 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.093 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.093 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:25:34.094 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.079350+0000", 2023-09-23T20:25:34.094 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.970083841 2023-09-23T20:25:34.094 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.094 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.094 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:25:34.094 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.079462+0000", 2023-09-23T20:25:34.094 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00011156 2023-09-23T20:25:34.094 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.094 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.094 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 1,6", 2023-09-23T20:25:34.094 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.079535+0000", 2023-09-23T20:25:34.095 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.2638000000000005e-05 2023-09-23T20:25:34.095 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.095 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.095 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:25:34.095 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.086084+0000", 2023-09-23T20:25:34.095 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.006549109 2023-09-23T20:25:34.095 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.095 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.095 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:25:34.095 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.101837+0000", 2023-09-23T20:25:34.095 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.015753775000000001 2023-09-23T20:25:34.095 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.096 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.096 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:25:34.096 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.101896+0000", 2023-09-23T20:25:34.096 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.8525e-05 2023-09-23T20:25:34.096 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.096 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.096 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:25:34.096 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.101921+0000", 2023-09-23T20:25:34.096 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.4905e-05 2023-09-23T20:25:34.096 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.096 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.096 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:25:34.097 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.101940+0000", 2023-09-23T20:25:34.097 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.9368000000000002e-05 2023-09-23T20:25:34.097 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:25:34.097 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:25:34.097 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:25:34.097 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.097 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.097 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:134176 5.18 5:1f2720a7:::benchmark_data_smithi018_228451_object27115:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e169)", 2023-09-23T20:25:34.097 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:25:14.171790+0000", 2023-09-23T20:25:34.097 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 19.803785614999999, 2023-09-23T20:25:34.097 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.93025216899999996, 2023-09-23T20:25:34.097 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:25:34.097 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:25:34.098 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:25:34.098 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:25:34.098 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:25:34.098 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 134176 2023-09-23T20:25:34.098 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.098 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:25:34.098 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.098 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:25:34.098 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.171790+0000", 2023-09-23T20:25:34.098 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:25:34.098 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.098 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.099 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:25:34.099 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.171790+0000", 2023-09-23T20:25:34.099 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:25:34.099 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.099 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.099 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:25:34.099 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.171790+0000", 2023-09-23T20:25:34.099 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:25:34.099 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.099 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.101 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:25:34.101 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.171792+0000", 2023-09-23T20:25:34.101 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.5330000000000001e-06 2023-09-23T20:25:34.101 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.101 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.101 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:25:34.101 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.171797+0000", 2023-09-23T20:25:34.101 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.2109999999999997e-06 2023-09-23T20:25:34.101 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.101 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.102 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:25:34.102 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.171801+0000", 2023-09-23T20:25:34.102 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.709e-06 2023-09-23T20:25:34.102 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.102 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.102 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:25:34.102 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.172312+0000", 2023-09-23T20:25:34.102 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00051119399999999997 2023-09-23T20:25:34.102 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.102 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.102 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:25:34.102 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.172327+0000", 2023-09-23T20:25:34.103 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.4389000000000001e-05 2023-09-23T20:25:34.103 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.103 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.103 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:25:34.103 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.080465+0000", 2023-09-23T20:25:34.103 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.90813792800000004 2023-09-23T20:25:34.103 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.103 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.103 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:25:34.104 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.080593+0000", 2023-09-23T20:25:34.104 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00012820799999999999 2023-09-23T20:25:34.104 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.104 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.104 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 1,6", 2023-09-23T20:25:34.104 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.080723+0000", 2023-09-23T20:25:34.104 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000130362 2023-09-23T20:25:34.104 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.104 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.104 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:25:34.104 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.090073+0000", 2023-09-23T20:25:34.104 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0093501950000000004 2023-09-23T20:25:34.105 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.105 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.105 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:25:34.105 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.101864+0000", 2023-09-23T20:25:34.105 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.011790429 2023-09-23T20:25:34.105 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.105 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.105 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:25:34.105 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.102002+0000", 2023-09-23T20:25:34.105 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00013855900000000001 2023-09-23T20:25:34.105 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.106 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.106 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:25:34.106 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.102024+0000", 2023-09-23T20:25:34.106 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.1926e-05 2023-09-23T20:25:34.106 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.106 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.106 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:25:34.106 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.102042+0000", 2023-09-23T20:25:34.106 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.7946000000000001e-05 2023-09-23T20:25:34.106 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:25:34.106 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:25:34.106 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:25:34.107 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.107 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.107 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:134225 5.18 5:18f2ddba:::benchmark_data_smithi018_228451_object27164:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e169)", 2023-09-23T20:25:34.107 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:25:14.238878+0000", 2023-09-23T20:25:34.107 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 19.736697483, 2023-09-23T20:25:34.107 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.86321889900000004, 2023-09-23T20:25:34.107 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:25:34.107 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:25:34.107 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:25:34.107 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:25:34.107 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:25:34.107 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 134225 2023-09-23T20:25:34.108 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.108 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:25:34.108 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.108 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:25:34.108 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.238878+0000", 2023-09-23T20:25:34.108 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:25:34.108 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.108 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.108 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:25:34.108 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.238878+0000", 2023-09-23T20:25:34.108 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:25:34.109 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.109 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.109 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:25:34.109 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.238877+0000", 2023-09-23T20:25:34.109 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:25:34.109 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.109 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.109 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:25:34.109 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.238881+0000", 2023-09-23T20:25:34.109 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.42e-06 2023-09-23T20:25:34.109 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.109 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.110 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:25:34.110 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.238893+0000", 2023-09-23T20:25:34.110 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1618e-05 2023-09-23T20:25:34.110 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.110 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.110 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:25:34.110 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.238898+0000", 2023-09-23T20:25:34.110 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.2800000000000003e-06 2023-09-23T20:25:34.110 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.110 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.110 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:25:34.110 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.238954+0000", 2023-09-23T20:25:34.111 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.5578999999999998e-05 2023-09-23T20:25:34.111 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.111 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.111 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:25:34.111 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.238965+0000", 2023-09-23T20:25:34.111 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0740999999999999e-05 2023-09-23T20:25:34.111 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.111 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.111 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:25:34.111 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.082298+0000", 2023-09-23T20:25:34.111 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.843333361 2023-09-23T20:25:34.111 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.112 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.112 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:25:34.112 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.082458+0000", 2023-09-23T20:25:34.112 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000159937 2023-09-23T20:25:34.112 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.112 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.112 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 1,6", 2023-09-23T20:25:34.112 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.082595+0000", 2023-09-23T20:25:34.112 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00013719000000000001 2023-09-23T20:25:34.112 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.112 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.112 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:25:34.113 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.090083+0000", 2023-09-23T20:25:34.113 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0074878849999999997 2023-09-23T20:25:34.113 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.113 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.113 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:25:34.113 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.101879+0000", 2023-09-23T20:25:34.113 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.011796152000000001 2023-09-23T20:25:34.113 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.113 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.113 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:25:34.113 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.102058+0000", 2023-09-23T20:25:34.113 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00017879199999999999 2023-09-23T20:25:34.114 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.114 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.114 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:25:34.114 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.102079+0000", 2023-09-23T20:25:34.114 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.1396e-05 2023-09-23T20:25:34.114 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.114 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.114 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:25:34.114 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.102097+0000", 2023-09-23T20:25:34.114 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.7615999999999999e-05 2023-09-23T20:25:34.114 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:25:34.115 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:25:34.115 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:25:34.115 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.115 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.115 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:134291 5.18 5:1e60ddf2:::benchmark_data_smithi018_228451_object27230:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e169)", 2023-09-23T20:25:34.115 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:25:14.332140+0000", 2023-09-23T20:25:34.115 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 19.643435806999999, 2023-09-23T20:25:34.115 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.77005607600000003, 2023-09-23T20:25:34.115 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:25:34.115 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:25:34.115 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:25:34.115 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:25:34.116 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:25:34.116 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 134291 2023-09-23T20:25:34.116 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.116 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:25:34.116 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.116 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:25:34.116 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.332140+0000", 2023-09-23T20:25:34.116 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:25:34.116 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.116 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.116 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:25:34.116 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.332140+0000", 2023-09-23T20:25:34.116 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:25:34.117 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.117 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.117 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:25:34.117 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.332139+0000", 2023-09-23T20:25:34.117 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:25:34.117 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.117 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.117 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:25:34.117 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.332143+0000", 2023-09-23T20:25:34.117 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.8080000000000004e-06 2023-09-23T20:25:34.117 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.117 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.118 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:25:34.118 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.332155+0000", 2023-09-23T20:25:34.118 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1216999999999999e-05 2023-09-23T20:25:34.118 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.118 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.118 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:25:34.118 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.332160+0000", 2023-09-23T20:25:34.118 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.4450000000000004e-06 2023-09-23T20:25:34.118 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.118 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.118 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:25:34.118 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.332209+0000", 2023-09-23T20:25:34.119 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.8822000000000001e-05 2023-09-23T20:25:34.119 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.119 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.119 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:25:34.119 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.332216+0000", 2023-09-23T20:25:34.119 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.0809999999999996e-06 2023-09-23T20:25:34.119 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.119 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.119 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:25:34.119 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.084295+0000", 2023-09-23T20:25:34.119 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.75207854200000002 2023-09-23T20:25:34.119 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.120 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.120 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:25:34.120 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.084424+0000", 2023-09-23T20:25:34.120 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00012909099999999999 2023-09-23T20:25:34.120 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.120 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.120 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 1,6", 2023-09-23T20:25:34.120 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.084534+0000", 2023-09-23T20:25:34.120 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000110449 2023-09-23T20:25:34.120 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.120 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.120 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:25:34.121 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.093553+0000", 2023-09-23T20:25:34.121 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0090184629999999991 2023-09-23T20:25:34.121 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.121 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.121 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:25:34.121 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.101957+0000", 2023-09-23T20:25:34.121 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0084048639999999997 2023-09-23T20:25:34.121 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.121 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.121 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:25:34.121 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.102157+0000", 2023-09-23T20:25:34.121 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00019960400000000001 2023-09-23T20:25:34.121 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.122 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.122 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:25:34.122 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.102178+0000", 2023-09-23T20:25:34.122 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.1035000000000001e-05 2023-09-23T20:25:34.122 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.122 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.122 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:25:34.122 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.102196+0000", 2023-09-23T20:25:34.122 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.7788e-05 2023-09-23T20:25:34.122 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:25:34.122 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:25:34.122 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:25:34.123 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.123 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.123 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:134367 5.18 5:1c202e23:::benchmark_data_smithi018_228451_object27306:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e169)", 2023-09-23T20:25:34.123 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:25:14.443008+0000", 2023-09-23T20:25:34.123 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 19.532567265000001, 2023-09-23T20:25:34.123 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.65924484999999999, 2023-09-23T20:25:34.123 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:25:34.123 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:25:34.123 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:25:34.123 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:25:34.123 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:25:34.123 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 134367 2023-09-23T20:25:34.123 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.124 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:25:34.124 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.124 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:25:34.124 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.443008+0000", 2023-09-23T20:25:34.124 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:25:34.124 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.124 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.124 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:25:34.124 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.443008+0000", 2023-09-23T20:25:34.124 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:25:34.125 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.125 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.125 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:25:34.125 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.443007+0000", 2023-09-23T20:25:34.125 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:25:34.125 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.125 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.125 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:25:34.125 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.443013+0000", 2023-09-23T20:25:34.125 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.0000000000000002e-06 2023-09-23T20:25:34.125 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.125 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.126 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:25:34.126 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.443030+0000", 2023-09-23T20:25:34.126 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.7093000000000001e-05 2023-09-23T20:25:34.126 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.126 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.126 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:25:34.126 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.443037+0000", 2023-09-23T20:25:34.126 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.3679999999999996e-06 2023-09-23T20:25:34.126 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.126 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.126 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:25:34.127 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.443107+0000", 2023-09-23T20:25:34.127 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.9842e-05 2023-09-23T20:25:34.127 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.127 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.127 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:25:34.127 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.443118+0000", 2023-09-23T20:25:34.127 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0825999999999999e-05 2023-09-23T20:25:34.127 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.127 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.127 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:25:34.127 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.086130+0000", 2023-09-23T20:25:34.127 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.64301198999999998 2023-09-23T20:25:34.128 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.128 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.128 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:25:34.128 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.086234+0000", 2023-09-23T20:25:34.128 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000104379 2023-09-23T20:25:34.128 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.128 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.128 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 1,6", 2023-09-23T20:25:34.128 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.086307+0000", 2023-09-23T20:25:34.128 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.2948999999999996e-05 2023-09-23T20:25:34.128 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.128 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.129 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:25:34.129 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.093560+0000", 2023-09-23T20:25:34.129 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.007252975 2023-09-23T20:25:34.129 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.129 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.129 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:25:34.129 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.101973+0000", 2023-09-23T20:25:34.129 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.008412915 2023-09-23T20:25:34.129 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.129 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.129 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:25:34.129 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.102213+0000", 2023-09-23T20:25:34.130 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000239822 2023-09-23T20:25:34.130 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.130 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.130 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:25:34.130 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.102234+0000", 2023-09-23T20:25:34.130 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.0934e-05 2023-09-23T20:25:34.130 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:34.130 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:34.130 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:25:34.130 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.102253+0000", 2023-09-23T20:25:34.130 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.9196999999999999e-05 2023-09-23T20:25:34.130 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:25:34.131 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:25:34.131 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:25:34.131 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:25:34.131 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:25:34.131 INFO:teuthology.orchestra.run.smithi149.stdout:} 2023-09-23T20:25:34.131 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 4, 5, 6, 7, 3] out_osds: [2] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-23T20:25:34.131 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:25:34.131 INFO:tasks.thrashosds.thrasher:Adding osd 2 2023-09-23T20:25:34.132 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd in 2 2023-09-23T20:25:34.132 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:34.133 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:25:34.133 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:25:34.133 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:25:34.138 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:25:34.140 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:25:34.145 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:34.146 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:25:34.146 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:25:34.146 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:25:34.146 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:25:34.146 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:25:34.146 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:25:34.146 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:25:34.146 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:25:34.147 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:25:34.248 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:25:34.308 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:34.496 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:25:34.499 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:34.500 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:25:34.576 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:34 smithi149 ceph-mon[170394]: pgmap v374: 109 pgs: 1 active+recovery_wait+degraded, 93 active+clean, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped; 4.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 84 KiB/s rd, 220 KiB/s wr, 746 op/s; 25551/207819 objects degraded (12.295%); 8176/207819 objects misplaced (3.934%); 7.3 MiB/s, 114 objects/s recovering 2023-09-23T20:25:34.577 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:34 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2225383861' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:25:34.577 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:34 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3327262729' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:25:34.583 INFO:teuthology.orchestra.run.smithi018.stderr:noscrub is set 2023-09-23T20:25:34.601 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:25:34.656 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:34.771 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:25:34.777 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:34 smithi018 ceph-mon[184144]: pgmap v374: 109 pgs: 1 active+recovery_wait+degraded, 93 active+clean, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped; 4.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 84 KiB/s rd, 220 KiB/s wr, 746 op/s; 25551/207819 objects degraded (12.295%); 8176/207819 objects misplaced (3.934%); 7.3 MiB/s, 114 objects/s recovering 2023-09-23T20:25:34.777 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:34 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2225383861' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:25:34.777 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:34 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3327262729' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:25:34.777 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:34 smithi018 ceph-mon[179647]: pgmap v374: 109 pgs: 1 active+recovery_wait+degraded, 93 active+clean, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped; 4.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 84 KiB/s rd, 220 KiB/s wr, 746 op/s; 25551/207819 objects degraded (12.295%); 8176/207819 objects misplaced (3.934%); 7.3 MiB/s, 114 objects/s recovering 2023-09-23T20:25:34.777 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:34 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2225383861' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:25:34.778 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:34 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3327262729' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:25:34.778 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:34.778 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:25:34.880 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:25:34.974 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:35.085 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:25:34 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:25:34] "GET /metrics HTTP/1.1" 200 36506 "" "Prometheus/2.43.0" 2023-09-23T20:25:35.380 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:25:35.852 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:35 smithi018 ceph-mon[184144]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:25:35.852 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:35 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3327262729' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:25:35.852 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:35 smithi018 ceph-mon[184144]: osdmap e175: 8 total, 8 up, 7 in 2023-09-23T20:25:35.852 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:35 smithi018 ceph-mon[184144]: failed to encode map e175 with expected crc 2023-09-23T20:25:35.852 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:35 smithi018 ceph-mon[184144]: failed to encode map e175 with expected crc 2023-09-23T20:25:35.853 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:35 smithi018 ceph-mon[184144]: failed to encode map e175 with expected crc 2023-09-23T20:25:35.853 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:35 smithi018 ceph-mon[184144]: failed to encode map e175 with expected crc 2023-09-23T20:25:35.853 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:35 smithi018 ceph-mon[179647]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:25:35.853 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:35 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3327262729' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:25:35.853 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:35 smithi018 ceph-mon[179647]: osdmap e175: 8 total, 8 up, 7 in 2023-09-23T20:25:35.853 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:35 smithi018 ceph-mon[179647]: failed to encode map e175 with expected crc 2023-09-23T20:25:35.853 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:35 smithi018 ceph-mon[179647]: failed to encode map e175 with expected crc 2023-09-23T20:25:35.853 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:35 smithi018 ceph-mon[179647]: failed to encode map e175 with expected crc 2023-09-23T20:25:35.853 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:35 smithi018 ceph-mon[179647]: failed to encode map e175 with expected crc 2023-09-23T20:25:35.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:35 smithi149 ceph-mon[170394]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:25:35.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:35 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3327262729' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:25:35.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:35 smithi149 ceph-mon[170394]: osdmap e175: 8 total, 8 up, 7 in 2023-09-23T20:25:35.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:35 smithi149 ceph-mon[170394]: failed to encode map e175 with expected crc 2023-09-23T20:25:35.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:35 smithi149 ceph-mon[170394]: failed to encode map e175 with expected crc 2023-09-23T20:25:35.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:35 smithi149 ceph-mon[170394]: failed to encode map e175 with expected crc 2023-09-23T20:25:35.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:35 smithi149 ceph-mon[170394]: failed to encode map e175 with expected crc 2023-09-23T20:25:36.491 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:25:36.496 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:36.496 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:25:36.519 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T20:25:36.593 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:36 smithi018 ceph-mon[179647]: pgmap v376: 109 pgs: 1 active+recovery_wait+degraded, 93 active+clean, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped; 4.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 694 op/s; 25551/201522 objects degraded (12.679%); 7834/201522 objects misplaced (3.887%); 6.4 MiB/s, 101 objects/s recovering 2023-09-23T20:25:36.593 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:36 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 25551/201522 objects degraded (12.679%), 15 pgs degraded (PG_DEGRADED) 2023-09-23T20:25:36.597 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:25:36.664 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:36.777 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:25:36.793 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:36.794 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:36 smithi149 ceph-mon[170394]: pgmap v376: 109 pgs: 1 active+recovery_wait+degraded, 93 active+clean, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped; 4.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 694 op/s; 25551/201522 objects degraded (12.679%); 7834/201522 objects misplaced (3.887%); 6.4 MiB/s, 101 objects/s recovering 2023-09-23T20:25:36.794 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:36 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 25551/201522 objects degraded (12.679%), 15 pgs degraded (PG_DEGRADED) 2023-09-23T20:25:36.794 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:25:36.829 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:25:36.860 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:36 smithi018 ceph-mon[184144]: pgmap v376: 109 pgs: 1 active+recovery_wait+degraded, 93 active+clean, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped; 4.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 694 op/s; 25551/201522 objects degraded (12.679%); 7834/201522 objects misplaced (3.887%); 6.4 MiB/s, 101 objects/s recovering 2023-09-23T20:25:36.861 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:36 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 25551/201522 objects degraded (12.679%), 15 pgs degraded (PG_DEGRADED) 2023-09-23T20:25:36.895 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:25:36.935 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:37.197 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T20:25:37.288 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:25:37.291 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:37.292 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:25:37.392 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:25:37.472 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:37.586 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:25:37.595 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:37.595 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:25:37.696 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:25:37.727 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:25:37.739 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:37.990 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:25:37.995 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:37.995 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:25:38.045 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:25:38.097 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:25:38.152 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:38.154 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:25:38.276 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:25:38.285 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:38.285 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:25:38.386 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:25:38.441 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:38 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mon-a[179623]: 2023-09-23T20:25:38.340+0000 7f160fef9700 -1 mon.a@0(leader).osd e175 definitely_dead 0 2023-09-23T20:25:38.447 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:38.604 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:25:38.614 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:38.614 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:25:38.627 INFO:teuthology.orchestra.run.smithi018.stderr:marked in osd.2. 2023-09-23T20:25:38.715 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:25:38.742 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:38.815 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:25:38.817 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:38.818 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:25:38.844 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:38 smithi018 ceph-mon[184144]: pgmap v377: 109 pgs: 1 active+recovery_wait+degraded, 93 active+clean, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped; 4.0 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 740 op/s; 25388/197550 objects degraded (12.851%); 7632/197550 objects misplaced (3.863%); 7.9 MiB/s, 126 objects/s recovering 2023-09-23T20:25:38.844 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:38 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/432114282' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["2"]}]: dispatch 2023-09-23T20:25:38.844 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:38 smithi018 ceph-mon[179647]: pgmap v377: 109 pgs: 1 active+recovery_wait+degraded, 93 active+clean, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped; 4.0 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 740 op/s; 25388/197550 objects degraded (12.851%); 7632/197550 objects misplaced (3.863%); 7.9 MiB/s, 126 objects/s recovering 2023-09-23T20:25:38.844 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:38 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/432114282' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["2"]}]: dispatch 2023-09-23T20:25:38.919 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:25:38.943 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:38 smithi149 ceph-mon[170394]: pgmap v377: 109 pgs: 1 active+recovery_wait+degraded, 93 active+clean, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped; 4.0 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 740 op/s; 25388/197550 objects degraded (12.851%); 7632/197550 objects misplaced (3.863%); 7.9 MiB/s, 126 objects/s recovering 2023-09-23T20:25:38.943 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:38 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/432114282' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["2"]}]: dispatch 2023-09-23T20:25:38.945 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:39.332 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:25:39.335 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:39.335 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:25:39.436 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:25:39.467 INFO:tasks.thrashosds.thrasher:Added osd 2 2023-09-23T20:25:39.469 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:39.560 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:25:39.564 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:39.564 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:25:39.665 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:25:39.739 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:39 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/432114282' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["2"]}]': finished 2023-09-23T20:25:39.739 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:39 smithi018 ceph-mon[179647]: osdmap e176: 8 total, 8 up, 8 in 2023-09-23T20:25:39.739 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:39 smithi018 ceph-mon[179647]: failed to encode map e176 with expected crc 2023-09-23T20:25:39.739 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:39 smithi018 ceph-mon[179647]: failed to encode map e176 with expected crc 2023-09-23T20:25:39.739 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:39 smithi018 ceph-mon[179647]: failed to encode map e176 with expected crc 2023-09-23T20:25:39.740 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:39 smithi018 ceph-mon[179647]: failed to encode map e176 with expected crc 2023-09-23T20:25:39.740 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:39 smithi018 ceph-mon[179647]: failed to encode map e176 with expected crc 2023-09-23T20:25:39.740 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:39 smithi018 ceph-mon[179647]: failed to encode map e176 with expected crc 2023-09-23T20:25:39.740 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:39 smithi018 ceph-mon[179647]: failed to encode map e176 with expected crc 2023-09-23T20:25:39.740 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:39 smithi018 ceph-mon[179647]: failed to encode map e176 with expected crc 2023-09-23T20:25:39.740 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:39 smithi018 ceph-mon[179647]: failed to encode map e176 with expected crc 2023-09-23T20:25:39.740 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:39 smithi018 ceph-mon[179647]: failed to encode map e176 with expected crc 2023-09-23T20:25:39.740 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:39 smithi018 ceph-mon[179647]: failed to encode map e176 with expected crc 2023-09-23T20:25:39.740 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:39 smithi018 ceph-mon[179647]: failed to encode map e176 with expected crc 2023-09-23T20:25:39.740 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:39 smithi018 ceph-mon[179647]: osdmap e177: 8 total, 8 up, 8 in 2023-09-23T20:25:39.740 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:39 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/432114282' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["2"]}]': finished 2023-09-23T20:25:39.741 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:39 smithi018 ceph-mon[184144]: osdmap e176: 8 total, 8 up, 8 in 2023-09-23T20:25:39.741 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:39 smithi018 ceph-mon[184144]: failed to encode map e176 with expected crc 2023-09-23T20:25:39.741 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:39 smithi018 ceph-mon[184144]: failed to encode map e176 with expected crc 2023-09-23T20:25:39.741 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:39 smithi018 ceph-mon[184144]: failed to encode map e176 with expected crc 2023-09-23T20:25:39.741 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:39 smithi018 ceph-mon[184144]: failed to encode map e176 with expected crc 2023-09-23T20:25:39.741 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:39 smithi018 ceph-mon[184144]: failed to encode map e176 with expected crc 2023-09-23T20:25:39.741 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:39 smithi018 ceph-mon[184144]: failed to encode map e176 with expected crc 2023-09-23T20:25:39.741 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:39 smithi018 ceph-mon[184144]: failed to encode map e176 with expected crc 2023-09-23T20:25:39.741 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:39 smithi018 ceph-mon[184144]: failed to encode map e176 with expected crc 2023-09-23T20:25:39.741 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:39 smithi018 ceph-mon[184144]: failed to encode map e176 with expected crc 2023-09-23T20:25:39.741 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:39 smithi018 ceph-mon[184144]: failed to encode map e176 with expected crc 2023-09-23T20:25:39.742 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:39 smithi018 ceph-mon[184144]: failed to encode map e176 with expected crc 2023-09-23T20:25:39.742 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:39 smithi018 ceph-mon[184144]: failed to encode map e176 with expected crc 2023-09-23T20:25:39.742 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:39 smithi018 ceph-mon[184144]: osdmap e177: 8 total, 8 up, 8 in 2023-09-23T20:25:39.742 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:39.834 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:25:39.838 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:39.838 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:25:39.939 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:25:39.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:39 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/432114282' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["2"]}]': finished 2023-09-23T20:25:39.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:39 smithi149 ceph-mon[170394]: osdmap e176: 8 total, 8 up, 8 in 2023-09-23T20:25:39.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:39 smithi149 ceph-mon[170394]: failed to encode map e176 with expected crc 2023-09-23T20:25:39.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:39 smithi149 ceph-mon[170394]: failed to encode map e176 with expected crc 2023-09-23T20:25:39.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:39 smithi149 ceph-mon[170394]: failed to encode map e176 with expected crc 2023-09-23T20:25:39.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:39 smithi149 ceph-mon[170394]: failed to encode map e176 with expected crc 2023-09-23T20:25:39.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:39 smithi149 ceph-mon[170394]: failed to encode map e176 with expected crc 2023-09-23T20:25:39.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:39 smithi149 ceph-mon[170394]: failed to encode map e176 with expected crc 2023-09-23T20:25:39.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:39 smithi149 ceph-mon[170394]: failed to encode map e176 with expected crc 2023-09-23T20:25:39.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:39 smithi149 ceph-mon[170394]: failed to encode map e176 with expected crc 2023-09-23T20:25:39.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:39 smithi149 ceph-mon[170394]: failed to encode map e176 with expected crc 2023-09-23T20:25:39.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:39 smithi149 ceph-mon[170394]: failed to encode map e176 with expected crc 2023-09-23T20:25:39.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:39 smithi149 ceph-mon[170394]: failed to encode map e176 with expected crc 2023-09-23T20:25:39.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:39 smithi149 ceph-mon[170394]: failed to encode map e176 with expected crc 2023-09-23T20:25:39.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:39 smithi149 ceph-mon[170394]: osdmap e177: 8 total, 8 up, 8 in 2023-09-23T20:25:39.974 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:40.020 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:25:40.023 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:40.023 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:25:40.066 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:25:40.103 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:25:40.124 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:25:40.128 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:25:40.149 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:25:40.163 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:40.169 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:25:40.195 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:25:40.222 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:25:40.249 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:25:40.252 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:25:40.252 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:25:40.252 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:25:40.253 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:25:40.253 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:25:40.253 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:25:40.253 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:25:40.253 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:25:40.802 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:25:40.805 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:40.805 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:25:40.906 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:25:40.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:40 smithi149 ceph-mon[170394]: pgmap v379: 109 pgs: 1 active+recovery_wait+degraded, 93 active+clean, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped; 4.0 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 427 op/s; 25388/197550 objects degraded (12.851%); 7632/197550 objects misplaced (3.863%); 4.8 MiB/s, 76 objects/s recovering 2023-09-23T20:25:40.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:40 smithi149 ceph-mon[170394]: failed to encode map e177 with expected crc 2023-09-23T20:25:40.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:40 smithi149 ceph-mon[170394]: failed to encode map e177 with expected crc 2023-09-23T20:25:40.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:40 smithi149 ceph-mon[170394]: failed to encode map e177 with expected crc 2023-09-23T20:25:40.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:40 smithi149 ceph-mon[170394]: failed to encode map e177 with expected crc 2023-09-23T20:25:40.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:40 smithi149 ceph-mon[170394]: failed to encode map e177 with expected crc 2023-09-23T20:25:40.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:40 smithi149 ceph-mon[170394]: failed to encode map e177 with expected crc 2023-09-23T20:25:40.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:40 smithi149 ceph-mon[170394]: failed to encode map e177 with expected crc 2023-09-23T20:25:40.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:40 smithi149 ceph-mon[170394]: failed to encode map e177 with expected crc 2023-09-23T20:25:40.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:40 smithi149 ceph-mon[170394]: failed to encode map e177 with expected crc 2023-09-23T20:25:40.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:40 smithi149 ceph-mon[170394]: failed to encode map e177 with expected crc 2023-09-23T20:25:40.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:40 smithi149 ceph-mon[170394]: failed to encode map e177 with expected crc 2023-09-23T20:25:40.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:40 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3548663935' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:25:40.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:40 smithi149 ceph-mon[170394]: osdmap e178: 8 total, 8 up, 8 in 2023-09-23T20:25:40.972 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:40 smithi018 ceph-mon[179647]: pgmap v379: 109 pgs: 1 active+recovery_wait+degraded, 93 active+clean, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped; 4.0 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 427 op/s; 25388/197550 objects degraded (12.851%); 7632/197550 objects misplaced (3.863%); 4.8 MiB/s, 76 objects/s recovering 2023-09-23T20:25:40.972 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:40 smithi018 ceph-mon[179647]: failed to encode map e177 with expected crc 2023-09-23T20:25:40.972 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:40 smithi018 ceph-mon[179647]: failed to encode map e177 with expected crc 2023-09-23T20:25:40.972 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:40 smithi018 ceph-mon[179647]: failed to encode map e177 with expected crc 2023-09-23T20:25:40.972 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:40 smithi018 ceph-mon[179647]: failed to encode map e177 with expected crc 2023-09-23T20:25:40.972 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:40 smithi018 ceph-mon[179647]: failed to encode map e177 with expected crc 2023-09-23T20:25:40.972 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:40 smithi018 ceph-mon[179647]: failed to encode map e177 with expected crc 2023-09-23T20:25:40.972 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:40 smithi018 ceph-mon[179647]: failed to encode map e177 with expected crc 2023-09-23T20:25:40.973 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:40 smithi018 ceph-mon[179647]: failed to encode map e177 with expected crc 2023-09-23T20:25:40.973 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:40 smithi018 ceph-mon[179647]: failed to encode map e177 with expected crc 2023-09-23T20:25:40.973 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:40 smithi018 ceph-mon[179647]: failed to encode map e177 with expected crc 2023-09-23T20:25:40.973 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:40 smithi018 ceph-mon[179647]: failed to encode map e177 with expected crc 2023-09-23T20:25:40.973 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:40 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3548663935' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:25:40.973 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:40 smithi018 ceph-mon[179647]: osdmap e178: 8 total, 8 up, 8 in 2023-09-23T20:25:40.973 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:40 smithi018 ceph-mon[184144]: pgmap v379: 109 pgs: 1 active+recovery_wait+degraded, 93 active+clean, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 12 active+recovery_wait+undersized+degraded+remapped; 4.0 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 427 op/s; 25388/197550 objects degraded (12.851%); 7632/197550 objects misplaced (3.863%); 4.8 MiB/s, 76 objects/s recovering 2023-09-23T20:25:40.974 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:40 smithi018 ceph-mon[184144]: failed to encode map e177 with expected crc 2023-09-23T20:25:40.974 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:40 smithi018 ceph-mon[184144]: failed to encode map e177 with expected crc 2023-09-23T20:25:40.974 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:40 smithi018 ceph-mon[184144]: failed to encode map e177 with expected crc 2023-09-23T20:25:40.974 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:40 smithi018 ceph-mon[184144]: failed to encode map e177 with expected crc 2023-09-23T20:25:40.974 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:40 smithi018 ceph-mon[184144]: failed to encode map e177 with expected crc 2023-09-23T20:25:40.974 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:40 smithi018 ceph-mon[184144]: failed to encode map e177 with expected crc 2023-09-23T20:25:40.974 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:40 smithi018 ceph-mon[184144]: failed to encode map e177 with expected crc 2023-09-23T20:25:40.974 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:40 smithi018 ceph-mon[184144]: failed to encode map e177 with expected crc 2023-09-23T20:25:40.975 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:40 smithi018 ceph-mon[184144]: failed to encode map e177 with expected crc 2023-09-23T20:25:40.975 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:40 smithi018 ceph-mon[184144]: failed to encode map e177 with expected crc 2023-09-23T20:25:40.975 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:40 smithi018 ceph-mon[184144]: failed to encode map e177 with expected crc 2023-09-23T20:25:40.975 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:40 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3548663935' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:25:40.975 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:40 smithi018 ceph-mon[184144]: osdmap e178: 8 total, 8 up, 8 in 2023-09-23T20:25:40.978 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:41.084 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:25:41.091 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:41.092 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:25:41.192 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:25:41.261 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:41.316 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:25:41.320 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:41.320 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:25:41.397 INFO:teuthology.orchestra.run.smithi149.stdout:{ 2023-09-23T20:25:41.397 INFO:teuthology.orchestra.run.smithi149.stdout: "ops": [ 2023-09-23T20:25:41.398 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:41.398 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:150871 5.7 5:e7c017d6:::benchmark_data_smithi018_228451_object43810:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e178)", 2023-09-23T20:25:41.398 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:25:41.390964+0000", 2023-09-23T20:25:41.398 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 0.0065696770000000003, 2023-09-23T20:25:41.398 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.006603345, 2023-09-23T20:25:41.398 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:25:41.398 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:25:41.398 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:25:41.398 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:25:41.398 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:25:41.399 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 150871 2023-09-23T20:25:41.399 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:41.399 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:25:41.399 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:41.399 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:25:41.399 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:41.390964+0000", 2023-09-23T20:25:41.399 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:25:41.399 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:41.399 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:41.399 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:25:41.399 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:41.390964+0000", 2023-09-23T20:25:41.399 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:25:41.400 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:41.400 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:41.400 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:25:41.400 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:41.390963+0000", 2023-09-23T20:25:41.400 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:25:41.400 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:41.400 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:41.400 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:25:41.400 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:41.390968+0000", 2023-09-23T20:25:41.400 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.7670000000000003e-06 2023-09-23T20:25:41.400 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:41.400 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:41.401 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:25:41.401 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:41.390986+0000", 2023-09-23T20:25:41.401 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.7875000000000001e-05 2023-09-23T20:25:41.401 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:41.401 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:41.401 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:25:41.401 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:41.390990+0000", 2023-09-23T20:25:41.401 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.1989999999999999e-06 2023-09-23T20:25:41.401 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:41.401 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:41.401 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:25:41.401 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:41.392840+0000", 2023-09-23T20:25:41.402 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0018496560000000001 2023-09-23T20:25:41.402 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:41.402 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:41.402 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:25:41.402 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:41.392917+0000", 2023-09-23T20:25:41.402 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.6928999999999995e-05 2023-09-23T20:25:41.402 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:41.402 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:41.402 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,1", 2023-09-23T20:25:41.402 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:41.392977+0000", 2023-09-23T20:25:41.402 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.0157e-05 2023-09-23T20:25:41.403 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:41.403 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:41.403 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:25:41.403 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:41.395620+0000", 2023-09-23T20:25:41.403 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0026434290000000001 2023-09-23T20:25:41.403 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:25:41.403 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:25:41.403 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:25:41.403 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:25:41.403 INFO:teuthology.orchestra.run.smithi149.stdout: ], 2023-09-23T20:25:41.403 INFO:teuthology.orchestra.run.smithi149.stdout: "num_ops": 1 2023-09-23T20:25:41.403 INFO:teuthology.orchestra.run.smithi149.stdout:} 2023-09-23T20:25:41.421 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:25:41.476 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:41.619 INFO:teuthology.orchestra.run.smithi018.stderr:nodeep-scrub is set 2023-09-23T20:25:41.720 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:41 smithi149 ceph-mon[170394]: failed to encode map e178 with expected crc 2023-09-23T20:25:41.720 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:41 smithi149 ceph-mon[170394]: failed to encode map e178 with expected crc 2023-09-23T20:25:41.720 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:41 smithi149 ceph-mon[170394]: failed to encode map e178 with expected crc 2023-09-23T20:25:41.720 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:41 smithi149 ceph-mon[170394]: failed to encode map e178 with expected crc 2023-09-23T20:25:41.720 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:41 smithi149 ceph-mon[170394]: failed to encode map e178 with expected crc 2023-09-23T20:25:41.720 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:41 smithi149 ceph-mon[170394]: failed to encode map e178 with expected crc 2023-09-23T20:25:41.721 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:41 smithi149 ceph-mon[170394]: failed to encode map e178 with expected crc 2023-09-23T20:25:41.721 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:41 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2848124545' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:25:41.721 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:41 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:25:41.721 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:41 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 25388/197550 objects degraded (12.851%), 15 pgs degraded (PG_DEGRADED) 2023-09-23T20:25:41.721 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:41 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:25:41.721 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:41 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:25:41.721 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:41 smithi149 ceph-mon[170394]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:25:41.721 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:41 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:25:41.721 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:41 smithi149 ceph-mon[170394]: osdmap e179: 8 total, 8 up, 8 in 2023-09-23T20:25:41.904 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:25:41.913 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:41.913 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:25:42.006 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T20:25:42.014 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:25:42.087 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:42.117 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:41 smithi018 ceph-mon[184144]: failed to encode map e178 with expected crc 2023-09-23T20:25:42.117 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:41 smithi018 ceph-mon[184144]: failed to encode map e178 with expected crc 2023-09-23T20:25:42.118 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:41 smithi018 ceph-mon[184144]: failed to encode map e178 with expected crc 2023-09-23T20:25:42.118 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:41 smithi018 ceph-mon[184144]: failed to encode map e178 with expected crc 2023-09-23T20:25:42.118 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:41 smithi018 ceph-mon[184144]: failed to encode map e178 with expected crc 2023-09-23T20:25:42.118 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:41 smithi018 ceph-mon[184144]: failed to encode map e178 with expected crc 2023-09-23T20:25:42.118 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:41 smithi018 ceph-mon[184144]: failed to encode map e178 with expected crc 2023-09-23T20:25:42.118 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:41 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2848124545' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:25:42.118 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:41 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:25:42.118 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:41 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 25388/197550 objects degraded (12.851%), 15 pgs degraded (PG_DEGRADED) 2023-09-23T20:25:42.118 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:41 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:25:42.118 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:41 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:25:42.119 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:41 smithi018 ceph-mon[184144]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:25:42.119 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:41 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:25:42.119 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:41 smithi018 ceph-mon[184144]: osdmap e179: 8 total, 8 up, 8 in 2023-09-23T20:25:42.124 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:41 smithi018 ceph-mon[179647]: failed to encode map e178 with expected crc 2023-09-23T20:25:42.124 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:41 smithi018 ceph-mon[179647]: failed to encode map e178 with expected crc 2023-09-23T20:25:42.124 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:41 smithi018 ceph-mon[179647]: failed to encode map e178 with expected crc 2023-09-23T20:25:42.124 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:41 smithi018 ceph-mon[179647]: failed to encode map e178 with expected crc 2023-09-23T20:25:42.124 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:41 smithi018 ceph-mon[179647]: failed to encode map e178 with expected crc 2023-09-23T20:25:42.124 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:41 smithi018 ceph-mon[179647]: failed to encode map e178 with expected crc 2023-09-23T20:25:42.124 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:41 smithi018 ceph-mon[179647]: failed to encode map e178 with expected crc 2023-09-23T20:25:42.124 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:41 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2848124545' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:25:42.124 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:41 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:25:42.124 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:41 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 25388/197550 objects degraded (12.851%), 15 pgs degraded (PG_DEGRADED) 2023-09-23T20:25:42.124 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:41 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:25:42.125 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:41 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:25:42.125 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:41 smithi018 ceph-mon[179647]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:25:42.125 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:41 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:25:42.125 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:41 smithi018 ceph-mon[179647]: osdmap e179: 8 total, 8 up, 8 in 2023-09-23T20:25:42.141 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:25:42.147 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:42.148 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:25:42.248 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:25:42.336 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:42.710 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:25:42.713 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:42.713 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:25:42.814 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:25:42.905 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:43.030 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:25:43.045 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:43.045 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:25:43.114 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:25:43.146 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:25:43.210 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:25:43.211 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:43.294 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:25:43.301 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:43.302 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:25:43.302 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:43 smithi149 ceph-mon[170394]: pgmap v382: 109 pgs: 1 remapped+peering, 1 active+recovering+undersized+remapped, 2 peering, 1 active+recovery_wait+degraded, 93 active+clean, 1 active+undersized+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 3.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 561 op/s; 10204/191415 objects degraded (5.331%); 1946/191415 objects misplaced (1.017%); 7.2 MiB/s, 1 keys/s, 214 objects/s recovering 2023-09-23T20:25:43.302 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:43 smithi149 ceph-mon[170394]: failed to encode map e179 with expected crc 2023-09-23T20:25:43.302 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:43 smithi149 ceph-mon[170394]: failed to encode map e179 with expected crc 2023-09-23T20:25:43.302 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:43 smithi149 ceph-mon[170394]: failed to encode map e179 with expected crc 2023-09-23T20:25:43.302 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:43 smithi149 ceph-mon[170394]: failed to encode map e179 with expected crc 2023-09-23T20:25:43.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:43 smithi018 ceph-mon[184144]: pgmap v382: 109 pgs: 1 remapped+peering, 1 active+recovering+undersized+remapped, 2 peering, 1 active+recovery_wait+degraded, 93 active+clean, 1 active+undersized+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 3.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 561 op/s; 10204/191415 objects degraded (5.331%); 1946/191415 objects misplaced (1.017%); 7.2 MiB/s, 1 keys/s, 214 objects/s recovering 2023-09-23T20:25:43.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:43 smithi018 ceph-mon[184144]: failed to encode map e179 with expected crc 2023-09-23T20:25:43.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:43 smithi018 ceph-mon[184144]: failed to encode map e179 with expected crc 2023-09-23T20:25:43.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:43 smithi018 ceph-mon[184144]: failed to encode map e179 with expected crc 2023-09-23T20:25:43.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:43 smithi018 ceph-mon[184144]: failed to encode map e179 with expected crc 2023-09-23T20:25:43.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:43 smithi018 ceph-mon[179647]: pgmap v382: 109 pgs: 1 remapped+peering, 1 active+recovering+undersized+remapped, 2 peering, 1 active+recovery_wait+degraded, 93 active+clean, 1 active+undersized+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 3.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 561 op/s; 10204/191415 objects degraded (5.331%); 1946/191415 objects misplaced (1.017%); 7.2 MiB/s, 1 keys/s, 214 objects/s recovering 2023-09-23T20:25:43.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:43 smithi018 ceph-mon[179647]: failed to encode map e179 with expected crc 2023-09-23T20:25:43.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:43 smithi018 ceph-mon[179647]: failed to encode map e179 with expected crc 2023-09-23T20:25:43.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:43 smithi018 ceph-mon[179647]: failed to encode map e179 with expected crc 2023-09-23T20:25:43.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:43 smithi018 ceph-mon[179647]: failed to encode map e179 with expected crc 2023-09-23T20:25:43.402 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:25:43.453 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:43.508 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:25:43.518 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:43.519 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:25:43.619 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:25:43.677 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:43.814 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:25:43.827 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:43.828 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:25:43.929 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:25:43.930 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:25:44.015 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:44.118 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T20:25:44.468 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 4, 5, 6, 7, 3, 2] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-23T20:25:44.468 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:25:44.468 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-09-23T20:25:44.468 INFO:tasks.thrashosds.thrasher:Setting osd 2 primary_affinity to 1.000000 2023-09-23T20:25:44.468 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 2 1 2023-09-23T20:25:45.197 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:25:45.206 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:45.206 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:25:45.307 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:25:45.332 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:45 smithi149 ceph-mon[170394]: pgmap v384: 109 pgs: 1 remapped+peering, 1 active+recovering+undersized+remapped, 2 peering, 95 active+clean, 1 active+undersized+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 465 op/s; 7549/190707 objects degraded (3.958%); 1946/190707 objects misplaced (1.020%); 4.2 MiB/s, 2 keys/s, 259 objects/s recovering 2023-09-23T20:25:45.332 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:45 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:25:45.333 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:45 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:25:45.335 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:45.391 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:25:44 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:25:44] "GET /metrics HTTP/1.1" 200 36484 "" "Prometheus/2.43.0" 2023-09-23T20:25:45.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:45 smithi018 ceph-mon[179647]: pgmap v384: 109 pgs: 1 remapped+peering, 1 active+recovering+undersized+remapped, 2 peering, 95 active+clean, 1 active+undersized+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 465 op/s; 7549/190707 objects degraded (3.958%); 1946/190707 objects misplaced (1.020%); 4.2 MiB/s, 2 keys/s, 259 objects/s recovering 2023-09-23T20:25:45.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:45 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:25:45.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:45 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:25:45.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:45 smithi018 ceph-mon[184144]: pgmap v384: 109 pgs: 1 remapped+peering, 1 active+recovering+undersized+remapped, 2 peering, 95 active+clean, 1 active+undersized+remapped, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 465 op/s; 7549/190707 objects degraded (3.958%); 1946/190707 objects misplaced (1.020%); 4.2 MiB/s, 2 keys/s, 259 objects/s recovering 2023-09-23T20:25:45.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:45 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:25:45.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:45 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:25:45.538 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:25:45.546 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:45.546 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:25:45.647 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:25:45.748 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:45.836 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:25:45.839 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:45.840 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:25:45.940 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:25:46.020 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:46.110 INFO:teuthology.orchestra.run.smithi149.stdout:{ 2023-09-23T20:25:46.111 INFO:teuthology.orchestra.run.smithi149.stdout: "ops": [], 2023-09-23T20:25:46.111 INFO:teuthology.orchestra.run.smithi149.stdout: "complaint_time": 30, 2023-09-23T20:25:46.111 INFO:teuthology.orchestra.run.smithi149.stdout: "num_blocked_ops": 0 2023-09-23T20:25:46.111 INFO:teuthology.orchestra.run.smithi149.stdout:} 2023-09-23T20:25:46.140 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:25:46.151 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:46.152 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:25:46.252 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:25:46.294 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:46 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 7549/190707 objects degraded (3.958%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T20:25:46.294 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:46 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 7549/190707 objects degraded (3.958%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T20:25:46.313 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:46.315 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:46 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 7549/190707 objects degraded (3.958%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T20:25:46.439 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:25:46.452 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:46.453 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:25:46.554 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:25:46.582 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:46.701 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:25:46.711 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:46.712 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:25:46.756 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:25:46.813 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:25:46.886 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:46.925 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:25:47.009 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:25:47.019 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:47.019 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:25:47.120 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:25:47.146 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:47 smithi018 ceph-mon[184144]: pgmap v385: 109 pgs: 1 remapped+peering, 1 active+recovering+undersized+remapped, 97 active+clean, 1 active+remapped+backfill_wait, 1 active+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 3.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 720 op/s; 7459/184581 objects degraded (4.041%); 1842/184581 objects misplaced (0.998%); 3.4 MiB/s, 1 keys/s, 227 objects/s recovering 2023-09-23T20:25:47.147 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:47 smithi018 ceph-mon[179647]: pgmap v385: 109 pgs: 1 remapped+peering, 1 active+recovering+undersized+remapped, 97 active+clean, 1 active+remapped+backfill_wait, 1 active+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 3.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 720 op/s; 7459/184581 objects degraded (4.041%); 1842/184581 objects misplaced (0.998%); 3.4 MiB/s, 1 keys/s, 227 objects/s recovering 2023-09-23T20:25:47.211 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:47.389 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:47 smithi149 ceph-mon[170394]: pgmap v385: 109 pgs: 1 remapped+peering, 1 active+recovering+undersized+remapped, 97 active+clean, 1 active+remapped+backfill_wait, 1 active+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 3.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 720 op/s; 7459/184581 objects degraded (4.041%); 1842/184581 objects misplaced (0.998%); 3.4 MiB/s, 1 keys/s, 227 objects/s recovering 2023-09-23T20:25:47.467 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T20:25:47.507 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:25:47.514 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:47.514 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:25:47.615 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:25:47.672 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:25:47.672 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:25:47.689 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:47.709 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:25:47.753 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:25:47.817 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:25:47.826 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:25:47.833 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:47.834 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:25:47.851 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:25:47.891 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:25:47.935 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:25:47.936 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:25:47.953 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:25:47.953 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:25:47.954 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:25:47.954 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:25:47.954 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:25:47.954 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:25:47.954 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:25:47.954 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:25:48.003 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:48.073 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:25:48.079 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:48 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1504230887' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:25:48.080 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:48 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:25:48.080 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:48 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:25:48.080 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:48 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1504230887' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:25:48.080 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:48 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:25:48.080 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:48 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:25:48.080 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:48.081 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:25:48.182 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:25:48.249 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:48.359 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:25:48.361 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:48 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1504230887' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:25:48.361 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:48 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:25:48.361 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:48 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:25:48.362 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:48.362 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:25:48.403 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:25:48.464 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:25:48.548 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:48.936 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:25:48.960 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:48.960 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:25:49.061 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:25:49.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:49 smithi018 ceph-mon[184144]: pgmap v386: 109 pgs: 1 active+recovering+undersized+remapped, 98 active+clean, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 741 op/s; 8063/179970 objects degraded (4.480%); 1842/179970 objects misplaced (1.024%); 2.6 MiB/s, 1 keys/s, 219 objects/s recovering 2023-09-23T20:25:49.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:49 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:25:49.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:49 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:25:49.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:49 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:25:49.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:49 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1123394028' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T20:25:49.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:49 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T20:25:49.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:49 smithi018 ceph-mon[179647]: pgmap v386: 109 pgs: 1 active+recovering+undersized+remapped, 98 active+clean, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 741 op/s; 8063/179970 objects degraded (4.480%); 1842/179970 objects misplaced (1.024%); 2.6 MiB/s, 1 keys/s, 219 objects/s recovering 2023-09-23T20:25:49.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:49 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:25:49.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:49 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:25:49.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:49 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:25:49.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:49 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1123394028' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T20:25:49.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:49 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T20:25:49.152 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:49.217 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:25:49.219 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:49.219 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:25:49.320 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:25:49.389 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:49.458 INFO:teuthology.orchestra.run.smithi018.stderr:set osd.2 primary-affinity to 1 (8655362) 2023-09-23T20:25:49.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:49 smithi149 ceph-mon[170394]: pgmap v386: 109 pgs: 1 active+recovering+undersized+remapped, 98 active+clean, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 741 op/s; 8063/179970 objects degraded (4.480%); 1842/179970 objects misplaced (1.024%); 2.6 MiB/s, 1 keys/s, 219 objects/s recovering 2023-09-23T20:25:49.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:49 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:25:49.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:49 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:25:49.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:49 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:25:49.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:49 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1123394028' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T20:25:49.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:49 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T20:25:49.470 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:25:49.484 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:49.484 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:25:49.585 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:25:49.644 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:50.167 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:25:50.170 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:50.171 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:25:50.271 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:25:50.383 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:50.452 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:50 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2023-09-23T20:25:50.452 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:50 smithi149 ceph-mon[170394]: osdmap e180: 8 total, 8 up, 8 in 2023-09-23T20:25:50.452 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:50 smithi149 ceph-mon[170394]: failed to encode map e180 with expected crc 2023-09-23T20:25:50.452 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:50 smithi149 ceph-mon[170394]: pgmap v388: 109 pgs: 1 active+recovering+undersized+remapped, 98 active+clean, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 509 op/s; 8063/179175 objects degraded (4.500%); 1842/179175 objects misplaced (1.028%); 0 B/s, 101 objects/s recovering 2023-09-23T20:25:50.453 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:50 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2986527386' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:25:50.453 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:50 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:25:50.476 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:25:50.476 INFO:teuthology.orchestra.run.smithi018.stderr:noscrub is unset 2023-09-23T20:25:50.489 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:50.489 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:25:50.590 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:25:50.670 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:50.751 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:25:50.755 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:50.756 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:25:50.856 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:50 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2023-09-23T20:25:50.857 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:50 smithi018 ceph-mon[179647]: osdmap e180: 8 total, 8 up, 8 in 2023-09-23T20:25:50.857 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:50 smithi018 ceph-mon[179647]: failed to encode map e180 with expected crc 2023-09-23T20:25:50.857 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:50 smithi018 ceph-mon[179647]: pgmap v388: 109 pgs: 1 active+recovering+undersized+remapped, 98 active+clean, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 509 op/s; 8063/179175 objects degraded (4.500%); 1842/179175 objects misplaced (1.028%); 0 B/s, 101 objects/s recovering 2023-09-23T20:25:50.857 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:50 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2986527386' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:25:50.857 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:50 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:25:50.857 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:50 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2023-09-23T20:25:50.857 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:50 smithi018 ceph-mon[184144]: osdmap e180: 8 total, 8 up, 8 in 2023-09-23T20:25:50.858 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:50 smithi018 ceph-mon[184144]: failed to encode map e180 with expected crc 2023-09-23T20:25:50.858 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:50 smithi018 ceph-mon[184144]: pgmap v388: 109 pgs: 1 active+recovering+undersized+remapped, 98 active+clean, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 509 op/s; 8063/179175 objects degraded (4.500%); 1842/179175 objects misplaced (1.028%); 0 B/s, 101 objects/s recovering 2023-09-23T20:25:50.858 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:50 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2986527386' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:25:50.858 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:50 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:25:50.858 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:25:50.906 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:50.910 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:25:50.942 INFO:teuthology.orchestra.run.smithi149.stdout:ERROR: (22) Invalid argument 2023-09-23T20:25:50.942 INFO:teuthology.orchestra.run.smithi149.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:25:50.975 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:25:50.982 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:50.982 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:25:51.083 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:25:51.146 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:51.328 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:25:51.332 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:51.332 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:25:51.427 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T20:25:51.433 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:25:51.535 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:51.603 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:25:51.610 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:51.610 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:25:51.711 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:25:51.722 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:51 smithi149 ceph-mon[170394]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:25:51.722 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:51 smithi149 ceph-mon[170394]: failed to encode map e181 with expected crc 2023-09-23T20:25:51.723 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:51 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:25:51.723 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:51 smithi149 ceph-mon[170394]: osdmap e181: 8 total, 8 up, 8 in 2023-09-23T20:25:51.723 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:51 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 8063/179175 objects degraded (4.500%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T20:25:51.735 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:25:51.759 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:51 smithi018 ceph-mon[179647]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:25:51.759 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:51 smithi018 ceph-mon[179647]: failed to encode map e181 with expected crc 2023-09-23T20:25:51.759 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:51 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:25:51.759 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:51 smithi018 ceph-mon[179647]: osdmap e181: 8 total, 8 up, 8 in 2023-09-23T20:25:51.760 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:51 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 8063/179175 objects degraded (4.500%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T20:25:51.760 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:51 smithi018 ceph-mon[184144]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:25:51.760 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:51 smithi018 ceph-mon[184144]: failed to encode map e181 with expected crc 2023-09-23T20:25:51.760 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:51 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:25:51.760 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:51 smithi018 ceph-mon[184144]: osdmap e181: 8 total, 8 up, 8 in 2023-09-23T20:25:51.760 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:51 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 8063/179175 objects degraded (4.500%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T20:25:51.767 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:51.883 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:25:51.888 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:51.889 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:25:51.990 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:25:52.041 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:52.116 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:25:52.120 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:52.120 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:25:52.221 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:25:52.285 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:52.288 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:25:52.431 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:25:52.437 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:52.437 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:25:52.538 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:25:52.551 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:52 smithi149 ceph-mon[170394]: pgmap v390: 109 pgs: 98 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 3.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 969 op/s; 8063/167430 objects degraded (4.816%); 1724/167430 objects misplaced (1.030%); 0 B/s, 91 objects/s recovering 2023-09-23T20:25:52.595 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:52 smithi018 ceph-mon[179647]: pgmap v390: 109 pgs: 98 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 3.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 969 op/s; 8063/167430 objects degraded (4.816%); 1724/167430 objects misplaced (1.030%); 0 B/s, 91 objects/s recovering 2023-09-23T20:25:52.596 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:52 smithi018 ceph-mon[184144]: pgmap v390: 109 pgs: 98 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 3.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 969 op/s; 8063/167430 objects degraded (4.816%); 1724/167430 objects misplaced (1.030%); 0 B/s, 91 objects/s recovering 2023-09-23T20:25:52.597 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:52.662 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:25:52.665 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:52.665 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:25:52.766 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:25:52.821 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:52.857 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T20:25:53.002 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:25:53.011 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:53.012 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:25:53.112 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:25:53.201 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:53.291 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:25:53.302 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:53.303 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:25:53.363 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:25:53.400 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:25:53.403 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:25:53.450 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:25:53.495 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:25:53.501 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:53.538 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:25:53.565 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:25:53.569 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:53.569 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:25:53.573 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:25:53.610 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:25:53.670 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:25:53.717 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:25:53.760 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:53 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/584839966' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:25:53.764 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:53.808 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:25:53.812 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:25:53.812 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:25:53.812 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:25:53.812 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:25:53.812 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:25:53.812 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:25:53.812 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:25:53.812 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:25:53.839 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:53 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/584839966' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:25:53.839 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:53 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/584839966' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:25:54.495 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:54 smithi018 ceph-mon[179647]: pgmap v391: 109 pgs: 98 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 747 op/s; 8063/166620 objects degraded (4.839%); 1724/166620 objects misplaced (1.035%); 0 B/s, 79 objects/s recovering 2023-09-23T20:25:54.497 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:54 smithi018 ceph-mon[184144]: pgmap v391: 109 pgs: 98 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 747 op/s; 8063/166620 objects degraded (4.839%); 1724/166620 objects misplaced (1.035%); 0 B/s, 79 objects/s recovering 2023-09-23T20:25:54.687 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:25:54.706 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:54.707 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:25:54.808 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:25:54.867 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:54 smithi149 ceph-mon[170394]: pgmap v391: 109 pgs: 98 active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 747 op/s; 8063/166620 objects degraded (4.839%); 1724/166620 objects misplaced (1.035%); 0 B/s, 79 objects/s recovering 2023-09-23T20:25:54.869 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 4, 5, 6, 7, 3, 2] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-23T20:25:54.869 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd reweight 0 1 2023-09-23T20:25:54.877 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:54.943 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:25:54.952 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:54.953 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:25:55.054 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:25:55.149 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:55.193 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:25:54 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:25:54] "GET /metrics HTTP/1.1" 200 36484 "" "Prometheus/2.43.0" 2023-09-23T20:25:55.232 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:25:55.236 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:55.236 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:25:55.337 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:25:55.361 INFO:teuthology.orchestra.run.smithi149.stdout:{ 2023-09-23T20:25:55.361 INFO:teuthology.orchestra.run.smithi149.stdout: "ops": [ 2023-09-23T20:25:55.361 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:55.361 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:162921 5.2 5:45868fd6:::benchmark_data_smithi018_228451_object55860:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e181)", 2023-09-23T20:25:55.361 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:25:55.358405+0000", 2023-09-23T20:25:55.361 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 0.0027030689999999998, 2023-09-23T20:25:55.361 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.002736676, 2023-09-23T20:25:55.362 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:25:55.362 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:25:55.362 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:25:55.362 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:25:55.362 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:25:55.362 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 162921 2023-09-23T20:25:55.362 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:55.362 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:25:55.362 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:55.362 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:25:55.362 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:55.358405+0000", 2023-09-23T20:25:55.363 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:25:55.363 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:55.363 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:55.363 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:25:55.363 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:55.358405+0000", 2023-09-23T20:25:55.363 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:25:55.363 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:55.363 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:55.363 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:25:55.363 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:55.358403+0000", 2023-09-23T20:25:55.363 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:25:55.363 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:55.364 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:55.364 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:25:55.364 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:55.358411+0000", 2023-09-23T20:25:55.364 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.6469999999999998e-06 2023-09-23T20:25:55.364 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:55.364 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:55.364 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:25:55.364 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:55.358427+0000", 2023-09-23T20:25:55.364 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.5950000000000001e-05 2023-09-23T20:25:55.364 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:55.364 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:55.364 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:25:55.365 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:55.358433+0000", 2023-09-23T20:25:55.365 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.7290000000000003e-06 2023-09-23T20:25:55.365 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:55.365 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:55.365 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:25:55.365 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:55.358476+0000", 2023-09-23T20:25:55.365 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.2740000000000001e-05 2023-09-23T20:25:55.365 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:55.365 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:55.365 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:25:55.365 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:55.358597+0000", 2023-09-23T20:25:55.365 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000120622 2023-09-23T20:25:55.366 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:25:55.366 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:25:55.366 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:25:55.366 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:55.358722+0000", 2023-09-23T20:25:55.366 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00012554800000000001 2023-09-23T20:25:55.366 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:25:55.366 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:25:55.366 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:25:55.366 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:25:55.366 INFO:teuthology.orchestra.run.smithi149.stdout: ], 2023-09-23T20:25:55.366 INFO:teuthology.orchestra.run.smithi149.stdout: "num_ops": 1 2023-09-23T20:25:55.367 INFO:teuthology.orchestra.run.smithi149.stdout:} 2023-09-23T20:25:55.405 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:55.478 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:25:55.484 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:55.485 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:25:55.585 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:25:55.650 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:55.821 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:25:55.831 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:55.832 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:25:55.835 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:25:55.877 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T20:25:55.932 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:25:56.013 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:56.081 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:25:56.099 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:56.100 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:25:56.200 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:25:56.269 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:56.345 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:25:56.346 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:25:56.354 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:56.354 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:25:56.455 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:25:56.520 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:56.555 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:25:56.558 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:56.558 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:25:56.608 INFO:teuthology.orchestra.run.smithi018.stderr:nodeep-scrub is unset 2023-09-23T20:25:56.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:56 smithi018 ceph-mon[179647]: pgmap v392: 109 pgs: 98 active+clean, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 868 op/s; 8063/159102 objects degraded (5.068%); 1603/159102 objects misplaced (1.008%); 0 B/s, 33 objects/s recovering 2023-09-23T20:25:56.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:56 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/962872389' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:25:56.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:56 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:25:56.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:56 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 8063/159102 objects degraded (5.068%), 10 pgs degraded (PG_DEGRADED) 2023-09-23T20:25:56.643 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:56 smithi018 ceph-mon[184144]: pgmap v392: 109 pgs: 98 active+clean, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 868 op/s; 8063/159102 objects degraded (5.068%); 1603/159102 objects misplaced (1.008%); 0 B/s, 33 objects/s recovering 2023-09-23T20:25:56.643 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:56 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/962872389' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:25:56.643 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:56 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:25:56.643 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:56 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 8063/159102 objects degraded (5.068%), 10 pgs degraded (PG_DEGRADED) 2023-09-23T20:25:56.659 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:25:56.707 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:56.784 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:25:56.790 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:56.790 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:56 smithi149 ceph-mon[170394]: pgmap v392: 109 pgs: 98 active+clean, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 868 op/s; 8063/159102 objects degraded (5.068%); 1603/159102 objects misplaced (1.008%); 0 B/s, 33 objects/s recovering 2023-09-23T20:25:56.790 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:56 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/962872389' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:25:56.790 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:56 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:25:56.790 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:56 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 8063/159102 objects degraded (5.068%), 10 pgs degraded (PG_DEGRADED) 2023-09-23T20:25:56.791 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:25:56.824 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:25:56.892 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:25:56.937 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:57.122 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:25:57.127 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:57.128 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:25:57.229 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:25:57.305 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:57.372 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:25:57.375 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:57.376 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:25:57.477 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:25:57.533 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:57.639 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:25:57.642 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:57.642 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:25:57.743 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:25:57.749 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:25:57.801 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:57 smithi018 ceph-mon[179647]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:25:57.801 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:57 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:25:57.801 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:57 smithi018 ceph-mon[179647]: osdmap e182: 8 total, 8 up, 8 in 2023-09-23T20:25:57.801 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:57 smithi018 ceph-mon[179647]: failed to encode map e182 with expected crc 2023-09-23T20:25:57.801 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:57 smithi018 ceph-mon[179647]: failed to encode map e182 with expected crc 2023-09-23T20:25:57.801 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:57 smithi018 ceph-mon[179647]: failed to encode map e182 with expected crc 2023-09-23T20:25:57.801 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:57 smithi018 ceph-mon[179647]: failed to encode map e182 with expected crc 2023-09-23T20:25:57.802 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:57 smithi018 ceph-mon[184144]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:25:57.802 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:57 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:25:57.802 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:57 smithi018 ceph-mon[184144]: osdmap e182: 8 total, 8 up, 8 in 2023-09-23T20:25:57.802 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:57 smithi018 ceph-mon[184144]: failed to encode map e182 with expected crc 2023-09-23T20:25:57.802 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:57 smithi018 ceph-mon[184144]: failed to encode map e182 with expected crc 2023-09-23T20:25:57.802 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:57 smithi018 ceph-mon[184144]: failed to encode map e182 with expected crc 2023-09-23T20:25:57.802 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:57 smithi018 ceph-mon[184144]: failed to encode map e182 with expected crc 2023-09-23T20:25:57.817 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:57.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:57 smithi149 ceph-mon[170394]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:25:57.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:57 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:25:57.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:57 smithi149 ceph-mon[170394]: osdmap e182: 8 total, 8 up, 8 in 2023-09-23T20:25:57.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:57 smithi149 ceph-mon[170394]: failed to encode map e182 with expected crc 2023-09-23T20:25:57.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:57 smithi149 ceph-mon[170394]: failed to encode map e182 with expected crc 2023-09-23T20:25:57.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:57 smithi149 ceph-mon[170394]: failed to encode map e182 with expected crc 2023-09-23T20:25:57.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:57 smithi149 ceph-mon[170394]: failed to encode map e182 with expected crc 2023-09-23T20:25:58.120 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:25:58.122 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:58.123 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:25:58.223 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:25:58.294 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:58.363 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:25:58.371 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:58.372 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:25:58.472 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:25:58.555 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:58.628 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:25:58.631 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:58.631 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:25:58.651 INFO:teuthology.orchestra.run.smithi018.stderr:reweighted osd.0 to 1 (10000) 2023-09-23T20:25:58.686 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:58 smithi018 ceph-mon[179647]: pgmap v394: 109 pgs: 1 active+recovering+undersized+remapped, 98 active+clean, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 3.1 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s; 8063/154170 objects degraded (5.230%); 1603/154170 objects misplaced (1.040%); 0 B/s, 72 objects/s recovering 2023-09-23T20:25:58.687 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:58 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2711876147' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T20:25:58.687 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:58 smithi018 ceph-mon[184144]: pgmap v394: 109 pgs: 1 active+recovering+undersized+remapped, 98 active+clean, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 3.1 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s; 8063/154170 objects degraded (5.230%); 1603/154170 objects misplaced (1.040%); 0 B/s, 72 objects/s recovering 2023-09-23T20:25:58.687 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:58 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2711876147' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T20:25:58.732 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:25:58.825 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:58.919 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:25:58.923 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:58 smithi149 ceph-mon[170394]: pgmap v394: 109 pgs: 1 active+recovering+undersized+remapped, 98 active+clean, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 3.1 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s; 8063/154170 objects degraded (5.230%); 1603/154170 objects misplaced (1.040%); 0 B/s, 72 objects/s recovering 2023-09-23T20:25:58.924 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:58 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2711876147' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T20:25:58.925 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:25:58.925 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:25:59.026 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:25:59.108 INFO:teuthology.orchestra.run.smithi149.stdout:{ 2023-09-23T20:25:59.108 INFO:teuthology.orchestra.run.smithi149.stdout: "ops": [], 2023-09-23T20:25:59.109 INFO:teuthology.orchestra.run.smithi149.stdout: "complaint_time": 30, 2023-09-23T20:25:59.109 INFO:teuthology.orchestra.run.smithi149.stdout: "num_blocked_ops": 0 2023-09-23T20:25:59.109 INFO:teuthology.orchestra.run.smithi149.stdout:} 2023-09-23T20:25:59.124 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:25:59.131 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd reweight 1 1 2023-09-23T20:25:59.501 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T20:25:59.544 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:25:59.571 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:25:59.607 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:25:59.662 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:25:59.701 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:25:59.737 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:25:59.777 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:25:59.821 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:25:59.846 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:25:59.846 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:25:59.847 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:25:59.847 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:25:59.847 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:25:59.847 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:25:59.847 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:25:59.847 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:25:59.887 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:59 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2711876147' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T20:25:59.887 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:59 smithi018 ceph-mon[179647]: osdmap e183: 8 total, 8 up, 8 in 2023-09-23T20:25:59.887 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:59 smithi018 ceph-mon[179647]: failed to encode map e183 with expected crc 2023-09-23T20:25:59.888 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:59 smithi018 ceph-mon[179647]: failed to encode map e183 with expected crc 2023-09-23T20:25:59.888 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:59 smithi018 ceph-mon[179647]: failed to encode map e183 with expected crc 2023-09-23T20:25:59.888 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:59 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:25:59.888 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:59 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:25:59.888 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:25:59 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/207501669' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:25:59.888 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:59 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2711876147' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T20:25:59.888 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:59 smithi018 ceph-mon[184144]: osdmap e183: 8 total, 8 up, 8 in 2023-09-23T20:25:59.888 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:59 smithi018 ceph-mon[184144]: failed to encode map e183 with expected crc 2023-09-23T20:25:59.889 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:59 smithi018 ceph-mon[184144]: failed to encode map e183 with expected crc 2023-09-23T20:25:59.889 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:59 smithi018 ceph-mon[184144]: failed to encode map e183 with expected crc 2023-09-23T20:25:59.889 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:59 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:25:59.889 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:59 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:25:59.889 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:25:59 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/207501669' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:25:59.933 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:25:59.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:59 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2711876147' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T20:25:59.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:59 smithi149 ceph-mon[170394]: osdmap e183: 8 total, 8 up, 8 in 2023-09-23T20:25:59.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:59 smithi149 ceph-mon[170394]: failed to encode map e183 with expected crc 2023-09-23T20:25:59.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:59 smithi149 ceph-mon[170394]: failed to encode map e183 with expected crc 2023-09-23T20:25:59.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:59 smithi149 ceph-mon[170394]: failed to encode map e183 with expected crc 2023-09-23T20:25:59.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:59 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:25:59.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:59 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:25:59.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:25:59 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/207501669' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:26:00.223 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:26:00.229 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:00.230 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:26:00.314 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T20:26:00.330 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:26:00.409 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:00.481 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:26:00.489 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:00.489 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:26:00.590 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:26:00.696 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:00.764 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:26:00.767 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:00.767 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:26:00.868 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:26:00.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:00 smithi018 ceph-mon[184144]: pgmap v396: 109 pgs: 1 active+recovering+undersized+remapped, 98 active+clean, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 3.1 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 587 op/s; 8063/153321 objects degraded (5.259%); 1603/153321 objects misplaced (1.046%); 0 B/s, 38 objects/s recovering 2023-09-23T20:26:00.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:00 smithi018 ceph-mon[179647]: pgmap v396: 109 pgs: 1 active+recovering+undersized+remapped, 98 active+clean, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 3.1 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 587 op/s; 8063/153321 objects degraded (5.259%); 1603/153321 objects misplaced (1.046%); 0 B/s, 38 objects/s recovering 2023-09-23T20:26:00.925 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:00.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:00 smithi149 ceph-mon[170394]: pgmap v396: 109 pgs: 1 active+recovering+undersized+remapped, 98 active+clean, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 3.1 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 587 op/s; 8063/153321 objects degraded (5.259%); 1603/153321 objects misplaced (1.046%); 0 B/s, 38 objects/s recovering 2023-09-23T20:26:01.055 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:26:01.056 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:26:01.062 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:01.062 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:26:01.163 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:26:01.205 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:26:01.229 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:01.314 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:26:01.319 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:01.319 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:26:01.420 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:26:01.515 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:01.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:01 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 8063/153321 objects degraded (5.259%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T20:26:01.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:01 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 8063/153321 objects degraded (5.259%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T20:26:01.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:01 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 8063/153321 objects degraded (5.259%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T20:26:01.984 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:26:01.988 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:01.989 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:26:02.089 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:26:02.182 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:02.422 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:26:02.949 INFO:teuthology.orchestra.run.smithi018.stderr:reweighted osd.1 to 1 (10000) 2023-09-23T20:26:02.957 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:26:02.967 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:02.968 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:26:03.069 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:26:03.164 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:03.219 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:02 smithi149 ceph-mon[170394]: pgmap v397: 109 pgs: 1 active+recovering+undersized+remapped, 98 active+clean, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s; 8063/141123 objects degraded (5.713%); 1452/141123 objects misplaced (1.029%); 0 B/s, 86 objects/s recovering 2023-09-23T20:26:03.219 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:02 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1212307323' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T20:26:03.297 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:26:03.311 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:03.311 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:26:03.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:02 smithi018 ceph-mon[184144]: pgmap v397: 109 pgs: 1 active+recovering+undersized+remapped, 98 active+clean, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s; 8063/141123 objects degraded (5.713%); 1452/141123 objects misplaced (1.029%); 0 B/s, 86 objects/s recovering 2023-09-23T20:26:03.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:02 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1212307323' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T20:26:03.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:02 smithi018 ceph-mon[179647]: pgmap v397: 109 pgs: 1 active+recovering+undersized+remapped, 98 active+clean, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s; 8063/141123 objects degraded (5.713%); 1452/141123 objects misplaced (1.029%); 0 B/s, 86 objects/s recovering 2023-09-23T20:26:03.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:02 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1212307323' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T20:26:03.411 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:26:03.516 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:03.646 INFO:teuthology.orchestra.run.smithi149.stdout:ERROR: (22) Invalid argument 2023-09-23T20:26:03.647 INFO:teuthology.orchestra.run.smithi149.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:26:04.170 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:26:04.174 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:04.175 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:26:04.209 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:03 smithi018 ceph-mon[184144]: failed to encode map e184 with expected crc 2023-09-23T20:26:04.209 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:03 smithi018 ceph-mon[184144]: failed to encode map e184 with expected crc 2023-09-23T20:26:04.209 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:03 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1212307323' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T20:26:04.209 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:03 smithi018 ceph-mon[184144]: osdmap e184: 8 total, 8 up, 8 in 2023-09-23T20:26:04.210 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:03 smithi018 ceph-mon[184144]: failed to encode map e184 with expected crc 2023-09-23T20:26:04.210 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:03 smithi018 ceph-mon[184144]: failed to encode map e184 with expected crc 2023-09-23T20:26:04.210 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:03 smithi018 ceph-mon[184144]: failed to encode map e184 with expected crc 2023-09-23T20:26:04.210 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:03 smithi018 ceph-mon[184144]: failed to encode map e184 with expected crc 2023-09-23T20:26:04.210 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:03 smithi018 ceph-mon[179647]: failed to encode map e184 with expected crc 2023-09-23T20:26:04.210 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:03 smithi018 ceph-mon[179647]: failed to encode map e184 with expected crc 2023-09-23T20:26:04.210 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:03 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1212307323' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T20:26:04.210 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:03 smithi018 ceph-mon[179647]: osdmap e184: 8 total, 8 up, 8 in 2023-09-23T20:26:04.211 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:03 smithi018 ceph-mon[179647]: failed to encode map e184 with expected crc 2023-09-23T20:26:04.211 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:03 smithi018 ceph-mon[179647]: failed to encode map e184 with expected crc 2023-09-23T20:26:04.211 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:03 smithi018 ceph-mon[179647]: failed to encode map e184 with expected crc 2023-09-23T20:26:04.211 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:03 smithi018 ceph-mon[179647]: failed to encode map e184 with expected crc 2023-09-23T20:26:04.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:03 smithi149 ceph-mon[170394]: failed to encode map e184 with expected crc 2023-09-23T20:26:04.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:03 smithi149 ceph-mon[170394]: failed to encode map e184 with expected crc 2023-09-23T20:26:04.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:03 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1212307323' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T20:26:04.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:03 smithi149 ceph-mon[170394]: osdmap e184: 8 total, 8 up, 8 in 2023-09-23T20:26:04.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:03 smithi149 ceph-mon[170394]: failed to encode map e184 with expected crc 2023-09-23T20:26:04.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:03 smithi149 ceph-mon[170394]: failed to encode map e184 with expected crc 2023-09-23T20:26:04.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:03 smithi149 ceph-mon[170394]: failed to encode map e184 with expected crc 2023-09-23T20:26:04.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:03 smithi149 ceph-mon[170394]: failed to encode map e184 with expected crc 2023-09-23T20:26:04.276 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:26:04.340 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:04.583 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd reweight 4 1 2023-09-23T20:26:04.622 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:26:04.629 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:04.630 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:26:04.730 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:26:04.754 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T20:26:04.831 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:04.927 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:26:04.931 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:04.932 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:26:04.942 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:26:05.033 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:26:05.081 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:05.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:04 smithi018 ceph-mon[179647]: pgmap v399: 109 pgs: 1 active+recovering+undersized+remapped, 98 active+clean, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 667 op/s; 8063/140301 objects degraded (5.747%); 1452/140301 objects misplaced (1.035%); 0 B/s, 55 objects/s recovering 2023-09-23T20:26:05.141 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:26:04 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:26:04] "GET /metrics HTTP/1.1" 200 36470 "" "Prometheus/2.43.0" 2023-09-23T20:26:05.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:04 smithi018 ceph-mon[184144]: pgmap v399: 109 pgs: 1 active+recovering+undersized+remapped, 98 active+clean, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 667 op/s; 8063/140301 objects degraded (5.747%); 1452/140301 objects misplaced (1.035%); 0 B/s, 55 objects/s recovering 2023-09-23T20:26:05.158 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:26:05.163 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:05.163 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:26:05.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:04 smithi149 ceph-mon[170394]: pgmap v399: 109 pgs: 1 active+recovering+undersized+remapped, 98 active+clean, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 667 op/s; 8063/140301 objects degraded (5.747%); 1452/140301 objects misplaced (1.035%); 0 B/s, 55 objects/s recovering 2023-09-23T20:26:05.264 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:26:05.335 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:05.546 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:26:05.601 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:26:05.608 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:05.608 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:26:05.709 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:26:05.801 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:05.855 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:26:05.877 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:26:05.883 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:05.883 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:26:05.984 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:26:05.996 INFO:teuthology.orchestra.run.smithi018.stderr:noscrub is set 2023-09-23T20:26:06.052 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:05 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3167908510' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:26:06.052 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:05 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 8063/140301 objects degraded (5.747%), 9 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:26:06.060 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:06.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:05 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3167908510' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:26:06.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:05 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 8063/140301 objects degraded (5.747%), 9 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:26:06.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:05 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3167908510' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:26:06.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:05 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 8063/140301 objects degraded (5.747%), 9 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:26:06.173 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:26:06.183 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:06.184 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:26:06.285 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:26:06.382 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:07.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:06 smithi018 ceph-mon[184144]: pgmap v400: 109 pgs: 1 active+recovering+undersized+remapped, 98 active+clean, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 904 op/s; 8063/132444 objects degraded (6.088%); 1304/132444 objects misplaced (0.985%); 0 B/s, 47 objects/s recovering 2023-09-23T20:26:07.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:06 smithi018 ceph-mon[184144]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:26:07.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:06 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3167908510' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:26:07.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:06 smithi018 ceph-mon[184144]: osdmap e185: 8 total, 8 up, 8 in 2023-09-23T20:26:07.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:06 smithi018 ceph-mon[184144]: failed to encode map e185 with expected crc 2023-09-23T20:26:07.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:06 smithi018 ceph-mon[184144]: failed to encode map e185 with expected crc 2023-09-23T20:26:07.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:06 smithi018 ceph-mon[184144]: failed to encode map e185 with expected crc 2023-09-23T20:26:07.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:06 smithi018 ceph-mon[184144]: failed to encode map e185 with expected crc 2023-09-23T20:26:07.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:06 smithi018 ceph-mon[184144]: failed to encode map e185 with expected crc 2023-09-23T20:26:07.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:06 smithi018 ceph-mon[179647]: pgmap v400: 109 pgs: 1 active+recovering+undersized+remapped, 98 active+clean, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 904 op/s; 8063/132444 objects degraded (6.088%); 1304/132444 objects misplaced (0.985%); 0 B/s, 47 objects/s recovering 2023-09-23T20:26:07.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:06 smithi018 ceph-mon[179647]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:26:07.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:06 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3167908510' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:26:07.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:06 smithi018 ceph-mon[179647]: osdmap e185: 8 total, 8 up, 8 in 2023-09-23T20:26:07.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:06 smithi018 ceph-mon[179647]: failed to encode map e185 with expected crc 2023-09-23T20:26:07.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:06 smithi018 ceph-mon[179647]: failed to encode map e185 with expected crc 2023-09-23T20:26:07.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:06 smithi018 ceph-mon[179647]: failed to encode map e185 with expected crc 2023-09-23T20:26:07.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:06 smithi018 ceph-mon[179647]: failed to encode map e185 with expected crc 2023-09-23T20:26:07.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:06 smithi018 ceph-mon[179647]: failed to encode map e185 with expected crc 2023-09-23T20:26:07.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:06 smithi149 ceph-mon[170394]: pgmap v400: 109 pgs: 1 active+recovering+undersized+remapped, 98 active+clean, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 904 op/s; 8063/132444 objects degraded (6.088%); 1304/132444 objects misplaced (0.985%); 0 B/s, 47 objects/s recovering 2023-09-23T20:26:07.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:06 smithi149 ceph-mon[170394]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:26:07.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:06 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3167908510' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:26:07.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:06 smithi149 ceph-mon[170394]: osdmap e185: 8 total, 8 up, 8 in 2023-09-23T20:26:07.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:06 smithi149 ceph-mon[170394]: failed to encode map e185 with expected crc 2023-09-23T20:26:07.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:06 smithi149 ceph-mon[170394]: failed to encode map e185 with expected crc 2023-09-23T20:26:07.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:06 smithi149 ceph-mon[170394]: failed to encode map e185 with expected crc 2023-09-23T20:26:07.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:06 smithi149 ceph-mon[170394]: failed to encode map e185 with expected crc 2023-09-23T20:26:07.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:06 smithi149 ceph-mon[170394]: failed to encode map e185 with expected crc 2023-09-23T20:26:07.260 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:26:07.265 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:07.266 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:26:07.275 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:26:07.308 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:26:07.358 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:26:07.366 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:26:07.406 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:26:07.443 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:26:07.447 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:07.490 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:26:07.540 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:26:07.564 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:26:07.575 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:07.575 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:26:07.582 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:26:07.584 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:26:07.584 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:26:07.585 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:26:07.585 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:26:07.585 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:26:07.585 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:26:07.585 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:26:07.585 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:26:07.676 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:26:07.785 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:07.965 INFO:teuthology.orchestra.run.smithi149.stdout:{ 2023-09-23T20:26:07.965 INFO:teuthology.orchestra.run.smithi149.stdout: "ops": [ 2023-09-23T20:26:07.965 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:26:07.965 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:174246 5.10 5:0dc3f61a:::benchmark_data_smithi018_228451_object67185:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e185)", 2023-09-23T20:26:07.965 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:26:07.954447+0000", 2023-09-23T20:26:07.965 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 0.0071901329999999996, 2023-09-23T20:26:07.965 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.007233036, 2023-09-23T20:26:07.966 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:26:07.966 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:26:07.966 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:26:07.966 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:26:07.966 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:26:07.966 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 174246 2023-09-23T20:26:07.966 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:26:07.966 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:26:07.966 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:26:07.966 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:26:07.966 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:26:07.954447+0000", 2023-09-23T20:26:07.967 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:26:07.967 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:26:07.967 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:26:07.967 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:26:07.967 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:26:07.954447+0000", 2023-09-23T20:26:07.967 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:26:07.967 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:26:07.967 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:26:07.967 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:26:07.967 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:26:07.954446+0000", 2023-09-23T20:26:07.967 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:26:07.967 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:26:07.968 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:26:07.968 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:26:07.968 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:26:07.954452+0000", 2023-09-23T20:26:07.968 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.1720000000000004e-06 2023-09-23T20:26:07.968 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:26:07.968 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:26:07.968 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:26:07.968 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:26:07.954462+0000", 2023-09-23T20:26:07.968 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0414000000000001e-05 2023-09-23T20:26:07.968 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:26:07.968 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:26:07.969 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:26:07.969 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:26:07.954468+0000", 2023-09-23T20:26:07.969 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.8850000000000004e-06 2023-09-23T20:26:07.969 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:26:07.969 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:26:07.969 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:26:07.969 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:26:07.956238+0000", 2023-09-23T20:26:07.969 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0017699879999999999 2023-09-23T20:26:07.969 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:26:07.969 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:26:07.969 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:26:07.969 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:26:07.956371+0000", 2023-09-23T20:26:07.969 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00013228900000000001 2023-09-23T20:26:07.970 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:26:07.970 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:26:07.970 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 4,6", 2023-09-23T20:26:07.970 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:26:07.956657+0000", 2023-09-23T20:26:07.970 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00028610300000000001 2023-09-23T20:26:07.970 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:26:07.970 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:26:07.970 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:26:07.970 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:26:07.959157+0000", 2023-09-23T20:26:07.970 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0025004129999999999 2023-09-23T20:26:07.970 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:26:07.971 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:26:07.971 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:26:07.971 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:26:07.971 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:26:07.971 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:174254 5.d 5:b4eeed85:::benchmark_data_smithi018_228451_object67193:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e185)", 2023-09-23T20:26:07.971 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:26:07.956694+0000", 2023-09-23T20:26:07.971 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 0.0049432039999999997, 2023-09-23T20:26:07.971 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0050839910000000004, 2023-09-23T20:26:07.971 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:26:07.971 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:26:07.971 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:26:07.971 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:26:07.972 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:26:07.972 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 174254 2023-09-23T20:26:07.972 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:26:07.972 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:26:07.972 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:26:07.972 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:26:07.972 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:26:07.956694+0000", 2023-09-23T20:26:07.972 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:26:07.972 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:26:07.972 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:26:07.972 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:26:07.972 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:26:07.956694+0000", 2023-09-23T20:26:07.973 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:26:07.973 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:26:07.973 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:26:07.973 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:26:07.973 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:26:07.956693+0000", 2023-09-23T20:26:07.973 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:26:07.973 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:26:07.973 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:26:07.973 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:26:07.973 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:26:07.956699+0000", 2023-09-23T20:26:07.973 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.6449999999999999e-06 2023-09-23T20:26:07.973 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:26:07.974 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:26:07.974 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:26:07.974 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:26:07.956711+0000", 2023-09-23T20:26:07.974 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.2132e-05 2023-09-23T20:26:07.974 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:26:07.974 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:26:07.974 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:26:07.974 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:26:07.956718+0000", 2023-09-23T20:26:07.974 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.5799999999999997e-06 2023-09-23T20:26:07.974 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:26:07.974 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:26:07.974 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:26:07.975 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:26:07.956806+0000", 2023-09-23T20:26:07.975 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.8080999999999999e-05 2023-09-23T20:26:07.975 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:26:07.975 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:26:07.975 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:26:07.975 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:26:07.956939+0000", 2023-09-23T20:26:07.975 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000132803 2023-09-23T20:26:07.975 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:26:07.975 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:26:07.975 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 2,5", 2023-09-23T20:26:07.975 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:26:07.957060+0000", 2023-09-23T20:26:07.976 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000120752 2023-09-23T20:26:07.976 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:26:07.976 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:26:07.976 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:26:07.976 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:26:07.960452+0000", 2023-09-23T20:26:07.976 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0033927839999999998 2023-09-23T20:26:07.976 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:26:07.976 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:26:07.976 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:26:07.976 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:26:07.976 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:26:07.976 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_repop_reply(client.34791.0:174242 5.4 e185/176)", 2023-09-23T20:26:07.977 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:26:07.961776+0000", 2023-09-23T20:26:07.977 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 4294967295.9998608, 2023-09-23T20:26:07.977 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.1982000000000002e-05, 2023-09-23T20:26:07.977 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:26:07.977 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "queued for pg", 2023-09-23T20:26:07.977 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:26:07.977 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:26:07.977 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:26:07.977 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:26:07.961776+0000", 2023-09-23T20:26:07.977 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:26:07.977 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:26:07.977 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:26:07.978 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:26:07.978 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:26:07.961776+0000", 2023-09-23T20:26:07.978 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:26:07.978 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:26:07.978 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:26:07.978 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:26:07.978 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:26:07.961775+0000", 2023-09-23T20:26:07.978 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:26:07.978 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:26:07.978 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:26:07.978 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:26:07.979 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:26:07.961782+0000", 2023-09-23T20:26:07.979 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.1410000000000002e-06 2023-09-23T20:26:07.979 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:26:07.979 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:26:07.979 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:26:07.979 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:26:07.961800+0000", 2023-09-23T20:26:07.979 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.8901000000000001e-05 2023-09-23T20:26:07.979 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:26:07.979 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:26:07.979 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:26:07.979 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:26:07.961807+0000", 2023-09-23T20:26:07.979 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.5599999999999999e-06 2023-09-23T20:26:07.980 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:26:07.980 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:26:07.980 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:26:07.980 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:26:07.980 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:26:07.980 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:174242 5.4 5:21d2826e:::benchmark_data_smithi018_228451_object67181:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e185)", 2023-09-23T20:26:07.980 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:26:07.945483+0000", 2023-09-23T20:26:07.980 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 0.01615457, 2023-09-23T20:26:07.980 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.016471963999999999, 2023-09-23T20:26:07.980 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:26:07.980 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:26:07.980 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:26:07.981 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:26:07.981 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:26:07.981 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 174242 2023-09-23T20:26:07.981 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:26:07.981 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:26:07.981 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:26:07.981 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:26:07.981 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:26:07.945483+0000", 2023-09-23T20:26:07.981 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:26:07.981 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:26:07.981 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:26:07.981 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:26:07.982 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:26:07.945483+0000", 2023-09-23T20:26:07.982 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:26:07.982 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:26:07.982 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:26:07.982 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:26:07.982 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:26:07.945482+0000", 2023-09-23T20:26:07.982 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:26:07.982 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:26:07.982 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:26:07.983 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:26:07.983 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:26:07.945486+0000", 2023-09-23T20:26:07.983 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.4150000000000001e-06 2023-09-23T20:26:07.983 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:26:07.983 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:26:07.983 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:26:07.983 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:26:07.945495+0000", 2023-09-23T20:26:07.983 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.0669999999999998e-06 2023-09-23T20:26:07.983 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:26:07.983 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:26:07.983 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:26:07.983 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:26:07.945500+0000", 2023-09-23T20:26:07.984 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.0760000000000002e-06 2023-09-23T20:26:07.984 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:26:07.984 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:26:07.984 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:26:07.984 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:26:07.945560+0000", 2023-09-23T20:26:07.984 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.9908999999999998e-05 2023-09-23T20:26:07.984 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:26:07.984 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:26:07.984 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:26:07.984 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:26:07.945701+0000", 2023-09-23T20:26:07.984 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00014078800000000001 2023-09-23T20:26:07.984 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:26:07.985 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:26:07.985 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 2,5", 2023-09-23T20:26:07.985 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:26:07.946227+0000", 2023-09-23T20:26:07.985 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000525732 2023-09-23T20:26:07.985 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:26:07.985 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:26:07.985 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:26:07.985 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:26:07.949488+0000", 2023-09-23T20:26:07.985 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0032611850000000002 2023-09-23T20:26:07.985 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:26:07.985 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:26:07.985 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:26:07.985 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:26:07.952205+0000", 2023-09-23T20:26:07.986 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0027167820000000001 2023-09-23T20:26:07.986 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:26:07.986 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:26:07.986 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:26:07.986 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:26:07.986 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:26:07.986 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:174244 5.10 5:0b55ccf3:::benchmark_data_smithi018_228451_object67183:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e185)", 2023-09-23T20:26:07.986 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:26:07.950884+0000", 2023-09-23T20:26:07.986 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 0.010753175, 2023-09-23T20:26:07.986 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.011122906, 2023-09-23T20:26:07.987 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:26:07.987 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:26:07.987 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:26:07.987 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:26:07.987 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:26:07.987 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 174244 2023-09-23T20:26:07.987 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:26:07.987 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:26:07.987 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:26:07.988 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:26:07.988 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:26:07.950884+0000", 2023-09-23T20:26:07.988 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:26:07.988 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:26:07.988 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:26:07.988 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:26:07.988 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:26:07.950884+0000", 2023-09-23T20:26:07.988 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:26:07.988 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:26:07.988 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:26:07.988 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:26:07.988 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:26:07.950883+0000", 2023-09-23T20:26:07.989 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:26:07.989 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:26:07.989 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:26:07.989 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:26:07.989 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:26:07.950889+0000", 2023-09-23T20:26:07.989 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.9800000000000003e-06 2023-09-23T20:26:07.989 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:26:07.989 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:26:07.989 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:26:07.989 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:26:07.950902+0000", 2023-09-23T20:26:07.989 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.3009e-05 2023-09-23T20:26:07.989 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:26:07.990 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:26:07.990 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:26:07.990 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:26:07.950908+0000", 2023-09-23T20:26:07.990 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.5400000000000001e-06 2023-09-23T20:26:07.990 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:26:07.990 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:26:07.990 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:26:07.990 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:26:07.950963+0000", 2023-09-23T20:26:07.990 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.4455e-05 2023-09-23T20:26:07.990 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:26:07.990 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:26:07.991 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:26:07.991 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:26:07.951096+0000", 2023-09-23T20:26:07.991 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000133421 2023-09-23T20:26:07.991 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:26:07.991 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:26:07.991 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 4,6", 2023-09-23T20:26:07.991 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:26:07.951534+0000", 2023-09-23T20:26:07.991 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00043807600000000003 2023-09-23T20:26:07.991 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:26:07.991 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:26:07.991 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:26:07.992 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:26:07.959115+0000", 2023-09-23T20:26:07.992 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0075809550000000003 2023-09-23T20:26:07.992 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:26:07.992 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:26:07.992 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:26:07.992 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:26:07.959147+0000", 2023-09-23T20:26:07.992 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.1473000000000002e-05 2023-09-23T20:26:07.992 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:26:07.992 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:26:07.992 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:26:07.992 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:26:07.992 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:26:07.993 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:174245 5.10 5:0ca684fa:::benchmark_data_smithi018_228451_object67184:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e185)", 2023-09-23T20:26:07.993 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:26:07.951539+0000", 2023-09-23T20:26:07.993 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 0.010097801999999999, 2023-09-23T20:26:07.993 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.010540815, 2023-09-23T20:26:07.993 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:26:07.993 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:26:07.993 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:26:07.993 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:26:07.993 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:26:07.993 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 174245 2023-09-23T20:26:07.993 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:26:07.993 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:26:07.994 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:26:07.994 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:26:07.994 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:26:07.951539+0000", 2023-09-23T20:26:07.994 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:26:07.994 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:26:07.994 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:26:07.994 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:26:07.994 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:26:07.951539+0000", 2023-09-23T20:26:07.994 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:26:07.994 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:26:07.994 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:26:07.995 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:26:07.995 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:26:07.951539+0000", 2023-09-23T20:26:07.995 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:26:07.995 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:26:07.995 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:26:07.995 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:26:07.995 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:26:07.951543+0000", 2023-09-23T20:26:07.995 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.4789999999999998e-06 2023-09-23T20:26:07.995 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:26:07.995 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:26:07.995 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:26:07.995 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:26:07.951552+0000", 2023-09-23T20:26:07.995 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.2110000000000004e-06 2023-09-23T20:26:07.996 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:26:07.996 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:26:07.996 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:26:07.996 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:26:07.951558+0000", 2023-09-23T20:26:07.996 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.7830000000000004e-06 2023-09-23T20:26:07.996 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:26:07.996 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:26:07.996 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:26:07.996 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:26:07.953925+0000", 2023-09-23T20:26:07.996 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0023668970000000002 2023-09-23T20:26:07.996 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:26:07.996 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:26:07.997 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:26:07.997 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:26:07.954029+0000", 2023-09-23T20:26:07.997 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000103719 2023-09-23T20:26:07.997 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:26:07.997 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:26:07.997 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 4,6", 2023-09-23T20:26:07.997 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:26:07.954325+0000", 2023-09-23T20:26:07.997 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00029586499999999997 2023-09-23T20:26:07.997 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:26:07.997 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:26:07.997 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:26:07.997 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:26:07.959153+0000", 2023-09-23T20:26:07.998 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0048280789999999999 2023-09-23T20:26:07.998 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:26:07.998 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:26:07.998 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:26:07.998 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:26:07.998 INFO:teuthology.orchestra.run.smithi149.stdout: ], 2023-09-23T20:26:07.998 INFO:teuthology.orchestra.run.smithi149.stdout: "num_ops": 6 2023-09-23T20:26:07.998 INFO:teuthology.orchestra.run.smithi149.stdout:} 2023-09-23T20:26:07.999 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:07.999 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:26:07.999 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:26:08.101 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:26:08.172 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:08.327 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:26:08.330 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:08.331 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:07 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3341815281' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:26:08.331 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:26:08.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:07 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3341815281' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:26:08.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:07 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3341815281' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:26:08.393 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T20:26:08.432 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:26:08.494 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:08.554 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:26:08.559 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:08.559 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:26:08.660 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:26:08.714 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:08.777 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:26:08.785 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:08.785 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:26:08.886 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:26:08.916 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T20:26:08.942 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:08.997 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:26:09.009 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:09.009 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:26:09.023 INFO:teuthology.orchestra.run.smithi018.stderr:reweighted osd.4 to 1 (10000) 2023-09-23T20:26:09.056 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:08 smithi018 ceph-mon[184144]: pgmap v402: 109 pgs: 1 active+recovering+undersized+remapped, 98 active+clean, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 1.09k op/s; 8063/127236 objects degraded (6.337%); 1304/127236 objects misplaced (1.025%); 0 B/s, 96 objects/s recovering 2023-09-23T20:26:09.056 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:08 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/226855360' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T20:26:09.056 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:08 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T20:26:09.056 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:08 smithi018 ceph-mon[179647]: pgmap v402: 109 pgs: 1 active+recovering+undersized+remapped, 98 active+clean, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 1.09k op/s; 8063/127236 objects degraded (6.337%); 1304/127236 objects misplaced (1.025%); 0 B/s, 96 objects/s recovering 2023-09-23T20:26:09.056 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:08 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/226855360' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T20:26:09.056 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:08 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T20:26:09.110 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:26:09.144 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:09.191 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:26:09.204 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:09.204 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:26:09.212 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:26:09.305 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:26:09.389 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:08 smithi149 ceph-mon[170394]: pgmap v402: 109 pgs: 1 active+recovering+undersized+remapped, 98 active+clean, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 1.09k op/s; 8063/127236 objects degraded (6.337%); 1304/127236 objects misplaced (1.025%); 0 B/s, 96 objects/s recovering 2023-09-23T20:26:09.389 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:08 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/226855360' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T20:26:09.389 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:08 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T20:26:09.395 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:09.464 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd reweight 5 1 2023-09-23T20:26:09.464 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:26:09.473 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:09.474 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:26:09.574 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:26:09.637 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:09.848 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:26:09.954 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:26:09.961 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:09.961 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:26:10.009 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:26:10.062 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:26:10.144 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:10.231 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:26:10.241 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:10.242 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:26:10.270 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:10 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T20:26:10.270 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:10 smithi018 ceph-mon[184144]: osdmap e186: 8 total, 8 up, 8 in 2023-09-23T20:26:10.270 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:10 smithi018 ceph-mon[184144]: failed to encode map e186 with expected crc 2023-09-23T20:26:10.271 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:10 smithi018 ceph-mon[184144]: failed to encode map e186 with expected crc 2023-09-23T20:26:10.271 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:10 smithi018 ceph-mon[184144]: failed to encode map e186 with expected crc 2023-09-23T20:26:10.273 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:10 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T20:26:10.273 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:10 smithi018 ceph-mon[179647]: osdmap e186: 8 total, 8 up, 8 in 2023-09-23T20:26:10.273 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:10 smithi018 ceph-mon[179647]: failed to encode map e186 with expected crc 2023-09-23T20:26:10.273 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:10 smithi018 ceph-mon[179647]: failed to encode map e186 with expected crc 2023-09-23T20:26:10.273 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:10 smithi018 ceph-mon[179647]: failed to encode map e186 with expected crc 2023-09-23T20:26:10.343 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:26:10.380 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:10 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T20:26:10.380 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:10 smithi149 ceph-mon[170394]: osdmap e186: 8 total, 8 up, 8 in 2023-09-23T20:26:10.380 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:10 smithi149 ceph-mon[170394]: failed to encode map e186 with expected crc 2023-09-23T20:26:10.380 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:10 smithi149 ceph-mon[170394]: failed to encode map e186 with expected crc 2023-09-23T20:26:10.381 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:10 smithi149 ceph-mon[170394]: failed to encode map e186 with expected crc 2023-09-23T20:26:10.388 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:10.752 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:26:10.758 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:10.759 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:26:10.859 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:26:10.920 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:10.930 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:26:10.974 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:26:10.984 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:10.984 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:26:11.012 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:11 smithi149 ceph-mon[170394]: pgmap v404: 109 pgs: 1 active+recovering+undersized+remapped, 98 active+clean, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 2.6 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 746 op/s; 8063/126393 objects degraded (6.379%); 1304/126393 objects misplaced (1.032%); 0 B/s, 58 objects/s recovering 2023-09-23T20:26:11.012 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:11 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 8063/126393 objects degraded (6.379%), 9 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:26:11.085 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:26:11.095 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:26:11.160 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:11.228 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:26:11.233 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:11.233 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:26:11.269 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:11 smithi018 ceph-mon[179647]: pgmap v404: 109 pgs: 1 active+recovering+undersized+remapped, 98 active+clean, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 2.6 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 746 op/s; 8063/126393 objects degraded (6.379%); 1304/126393 objects misplaced (1.032%); 0 B/s, 58 objects/s recovering 2023-09-23T20:26:11.269 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:11 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 8063/126393 objects degraded (6.379%), 9 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:26:11.269 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:11 smithi018 ceph-mon[184144]: pgmap v404: 109 pgs: 1 active+recovering+undersized+remapped, 98 active+clean, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 2.6 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 746 op/s; 8063/126393 objects degraded (6.379%); 1304/126393 objects misplaced (1.032%); 0 B/s, 58 objects/s recovering 2023-09-23T20:26:11.269 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:11 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 8063/126393 objects degraded (6.379%), 9 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:26:11.334 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:26:11.390 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:11.451 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:26:11.455 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:11.456 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:26:11.557 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:26:11.641 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:11.727 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:26:11.741 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:11.742 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:26:11.791 INFO:teuthology.orchestra.run.smithi149.stdout:{ 2023-09-23T20:26:11.791 INFO:teuthology.orchestra.run.smithi149.stdout: "ops": [], 2023-09-23T20:26:11.791 INFO:teuthology.orchestra.run.smithi149.stdout: "complaint_time": 30, 2023-09-23T20:26:11.791 INFO:teuthology.orchestra.run.smithi149.stdout: "num_blocked_ops": 0 2023-09-23T20:26:11.791 INFO:teuthology.orchestra.run.smithi149.stdout:} 2023-09-23T20:26:11.843 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:26:11.912 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:12.190 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:26:12.194 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:12.194 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:26:12.294 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T20:26:12.294 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:26:12.328 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:12.372 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:26:12.377 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:12.377 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:26:12.478 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:26:12.504 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:12.539 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:26:12.541 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:12.542 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:26:12.643 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:26:12.678 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:12.818 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:26:12.824 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:12.824 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:26:12.925 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:26:12.926 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:26:12.953 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:13.027 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:26:13.057 INFO:teuthology.orchestra.run.smithi018.stderr:nodeep-scrub is set 2023-09-23T20:26:13.057 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:26:13.080 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:13 smithi018 ceph-mon[184144]: pgmap v405: 109 pgs: 1 active+recovering+undersized+remapped, 98 active+clean, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 2.3 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s; 8063/113742 objects degraded (7.089%); 1175/113742 objects misplaced (1.033%); 0 B/s, 106 objects/s recovering 2023-09-23T20:26:13.080 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:13 smithi018 ceph-mon[184144]: osdmap e187: 8 total, 8 up, 8 in 2023-09-23T20:26:13.080 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:13 smithi018 ceph-mon[184144]: failed to encode map e187 with expected crc 2023-09-23T20:26:13.080 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:13 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1486619900' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:26:13.080 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:13 smithi018 ceph-mon[184144]: failed to encode map e187 with expected crc 2023-09-23T20:26:13.080 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:13 smithi018 ceph-mon[184144]: failed to encode map e187 with expected crc 2023-09-23T20:26:13.080 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:13 smithi018 ceph-mon[184144]: failed to encode map e187 with expected crc 2023-09-23T20:26:13.080 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:13 smithi018 ceph-mon[184144]: failed to encode map e187 with expected crc 2023-09-23T20:26:13.080 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:13 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2739169888' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:26:13.081 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:13 smithi018 ceph-mon[179647]: pgmap v405: 109 pgs: 1 active+recovering+undersized+remapped, 98 active+clean, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 2.3 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s; 8063/113742 objects degraded (7.089%); 1175/113742 objects misplaced (1.033%); 0 B/s, 106 objects/s recovering 2023-09-23T20:26:13.081 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:13 smithi018 ceph-mon[179647]: osdmap e187: 8 total, 8 up, 8 in 2023-09-23T20:26:13.081 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:13 smithi018 ceph-mon[179647]: failed to encode map e187 with expected crc 2023-09-23T20:26:13.081 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:13 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1486619900' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:26:13.081 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:13 smithi018 ceph-mon[179647]: failed to encode map e187 with expected crc 2023-09-23T20:26:13.081 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:13 smithi018 ceph-mon[179647]: failed to encode map e187 with expected crc 2023-09-23T20:26:13.081 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:13 smithi018 ceph-mon[179647]: failed to encode map e187 with expected crc 2023-09-23T20:26:13.081 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:13 smithi018 ceph-mon[179647]: failed to encode map e187 with expected crc 2023-09-23T20:26:13.081 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:13 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2739169888' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:26:13.083 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:26:13.086 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:13.087 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:26:13.092 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:26:13.136 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:26:13.179 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:26:13.188 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:26:13.223 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:13 smithi149 ceph-mon[170394]: pgmap v405: 109 pgs: 1 active+recovering+undersized+remapped, 98 active+clean, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 2.3 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s; 8063/113742 objects degraded (7.089%); 1175/113742 objects misplaced (1.033%); 0 B/s, 106 objects/s recovering 2023-09-23T20:26:13.223 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:13 smithi149 ceph-mon[170394]: osdmap e187: 8 total, 8 up, 8 in 2023-09-23T20:26:13.223 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:13 smithi149 ceph-mon[170394]: failed to encode map e187 with expected crc 2023-09-23T20:26:13.223 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:13 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1486619900' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:26:13.223 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:13 smithi149 ceph-mon[170394]: failed to encode map e187 with expected crc 2023-09-23T20:26:13.223 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:13 smithi149 ceph-mon[170394]: failed to encode map e187 with expected crc 2023-09-23T20:26:13.224 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:13 smithi149 ceph-mon[170394]: failed to encode map e187 with expected crc 2023-09-23T20:26:13.224 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:13 smithi149 ceph-mon[170394]: failed to encode map e187 with expected crc 2023-09-23T20:26:13.224 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:13 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2739169888' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:26:13.232 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:26:13.265 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:13.268 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:26:13.304 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:26:13.315 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:26:13.315 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:26:13.315 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:26:13.315 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:26:13.316 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:26:13.316 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:26:13.316 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:26:13.316 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:26:13.325 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:26:13.334 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:13.335 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:26:13.435 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:26:13.501 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:13.566 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:26:13.574 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:13.575 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:26:13.675 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:26:13.715 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:13.764 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:26:13.769 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:13.770 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:26:13.871 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:26:13.907 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:13.944 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:26:13.948 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:13.948 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:26:14.047 INFO:teuthology.orchestra.run.smithi018.stderr:reweighted osd.5 to 1 (10000) 2023-09-23T20:26:14.049 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:26:14.098 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:14.173 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:14 smithi149 ceph-mon[170394]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:26:14.173 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:14 smithi149 ceph-mon[170394]: failed to encode map e188 with expected crc 2023-09-23T20:26:14.173 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:14 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1486619900' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:26:14.173 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:14 smithi149 ceph-mon[170394]: osdmap e188: 8 total, 8 up, 8 in 2023-09-23T20:26:14.173 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:14 smithi149 ceph-mon[170394]: failed to encode map e188 with expected crc 2023-09-23T20:26:14.173 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:14 smithi149 ceph-mon[170394]: failed to encode map e188 with expected crc 2023-09-23T20:26:14.174 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:14 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1301828548' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T20:26:14.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:14 smithi018 ceph-mon[184144]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:26:14.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:14 smithi018 ceph-mon[184144]: failed to encode map e188 with expected crc 2023-09-23T20:26:14.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:14 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1486619900' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:26:14.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:14 smithi018 ceph-mon[184144]: osdmap e188: 8 total, 8 up, 8 in 2023-09-23T20:26:14.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:14 smithi018 ceph-mon[184144]: failed to encode map e188 with expected crc 2023-09-23T20:26:14.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:14 smithi018 ceph-mon[184144]: failed to encode map e188 with expected crc 2023-09-23T20:26:14.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:14 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1301828548' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T20:26:14.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:14 smithi018 ceph-mon[179647]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:26:14.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:14 smithi018 ceph-mon[179647]: failed to encode map e188 with expected crc 2023-09-23T20:26:14.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:14 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1486619900' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:26:14.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:14 smithi018 ceph-mon[179647]: osdmap e188: 8 total, 8 up, 8 in 2023-09-23T20:26:14.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:14 smithi018 ceph-mon[179647]: failed to encode map e188 with expected crc 2023-09-23T20:26:14.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:14 smithi018 ceph-mon[179647]: failed to encode map e188 with expected crc 2023-09-23T20:26:14.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:14 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1301828548' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T20:26:14.417 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:26:14.425 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:14.426 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:26:14.477 INFO:teuthology.orchestra.run.smithi149.stdout:ERROR: (22) Invalid argument 2023-09-23T20:26:14.477 INFO:teuthology.orchestra.run.smithi149.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:26:14.514 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd reweight 6 1 2023-09-23T20:26:14.527 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:26:14.626 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:14.892 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:26:14.895 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:14.896 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:26:14.997 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:26:15.053 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:15.211 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:15 smithi149 ceph-mon[170394]: pgmap v408: 109 pgs: 1 active+recovering+undersized+remapped, 98 active+clean, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 2.3 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 788 op/s; 8063/113025 objects degraded (7.134%); 1175/113025 objects misplaced (1.040%); 0 B/s, 77 objects/s recovering 2023-09-23T20:26:15.212 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:15 smithi149 ceph-mon[170394]: failed to encode map e189 with expected crc 2023-09-23T20:26:15.212 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:15 smithi149 ceph-mon[170394]: failed to encode map e189 with expected crc 2023-09-23T20:26:15.212 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:15 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1301828548' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T20:26:15.212 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:15 smithi149 ceph-mon[170394]: osdmap e189: 8 total, 8 up, 8 in 2023-09-23T20:26:15.212 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:15 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:26:15.212 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:15 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:26:15.252 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:26:15.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:15 smithi018 ceph-mon[179647]: pgmap v408: 109 pgs: 1 active+recovering+undersized+remapped, 98 active+clean, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 2.3 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 788 op/s; 8063/113025 objects degraded (7.134%); 1175/113025 objects misplaced (1.040%); 0 B/s, 77 objects/s recovering 2023-09-23T20:26:15.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:15 smithi018 ceph-mon[179647]: failed to encode map e189 with expected crc 2023-09-23T20:26:15.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:15 smithi018 ceph-mon[179647]: failed to encode map e189 with expected crc 2023-09-23T20:26:15.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:15 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1301828548' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T20:26:15.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:15 smithi018 ceph-mon[179647]: osdmap e189: 8 total, 8 up, 8 in 2023-09-23T20:26:15.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:15 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:26:15.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:15 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:26:15.393 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:26:14 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:26:14] "GET /metrics HTTP/1.1" 200 36463 "" "Prometheus/2.43.0" 2023-09-23T20:26:15.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:15 smithi018 ceph-mon[184144]: pgmap v408: 109 pgs: 1 active+recovering+undersized+remapped, 98 active+clean, 1 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped; 2.3 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 788 op/s; 8063/113025 objects degraded (7.134%); 1175/113025 objects misplaced (1.040%); 0 B/s, 77 objects/s recovering 2023-09-23T20:26:15.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:15 smithi018 ceph-mon[184144]: failed to encode map e189 with expected crc 2023-09-23T20:26:15.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:15 smithi018 ceph-mon[184144]: failed to encode map e189 with expected crc 2023-09-23T20:26:15.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:15 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1301828548' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T20:26:15.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:15 smithi018 ceph-mon[184144]: osdmap e189: 8 total, 8 up, 8 in 2023-09-23T20:26:15.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:15 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:26:15.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:15 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:26:15.624 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T20:26:15.812 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:26:16.068 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:16 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 8063/113025 objects degraded (7.134%), 9 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:26:16.068 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:16 smithi018 ceph-mon[179647]: osdmap e190: 8 total, 8 up, 8 in 2023-09-23T20:26:16.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:16 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 8063/113025 objects degraded (7.134%), 9 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:26:16.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:16 smithi018 ceph-mon[184144]: osdmap e190: 8 total, 8 up, 8 in 2023-09-23T20:26:16.393 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:26:16.398 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:16 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 8063/113025 objects degraded (7.134%), 9 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:26:16.398 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:16 smithi149 ceph-mon[170394]: osdmap e190: 8 total, 8 up, 8 in 2023-09-23T20:26:16.399 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:16.399 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:26:16.433 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T20:26:16.500 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:26:16.603 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:16.760 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:26:16.761 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:26:16.765 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:16.765 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:26:16.866 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:26:16.902 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:16.942 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:26:16.946 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:16.947 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:26:17.022 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:26:17.047 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:26:17.085 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:17 smithi018 ceph-mon[179647]: pgmap v410: 109 pgs: 1 active+recovering+undersized+remapped, 99 active+clean, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped; 2.2 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 947 op/s; 7697/109320 objects degraded (7.041%); 1079/109320 objects misplaced (0.987%); 0 B/s, 138 objects/s recovering 2023-09-23T20:26:17.085 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:17 smithi018 ceph-mon[179647]: failed to encode map e190 with expected crc 2023-09-23T20:26:17.086 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:17 smithi018 ceph-mon[179647]: failed to encode map e190 with expected crc 2023-09-23T20:26:17.086 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:17 smithi018 ceph-mon[179647]: failed to encode map e190 with expected crc 2023-09-23T20:26:17.086 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:17 smithi018 ceph-mon[179647]: osdmap e191: 8 total, 8 up, 8 in 2023-09-23T20:26:17.137 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:17.230 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:26:17.236 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:17.236 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:26:17.337 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:26:17.374 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:17 smithi018 ceph-mon[184144]: pgmap v410: 109 pgs: 1 active+recovering+undersized+remapped, 99 active+clean, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped; 2.2 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 947 op/s; 7697/109320 objects degraded (7.041%); 1079/109320 objects misplaced (0.987%); 0 B/s, 138 objects/s recovering 2023-09-23T20:26:17.374 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:17 smithi018 ceph-mon[184144]: failed to encode map e190 with expected crc 2023-09-23T20:26:17.374 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:17 smithi018 ceph-mon[184144]: failed to encode map e190 with expected crc 2023-09-23T20:26:17.375 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:17 smithi018 ceph-mon[184144]: failed to encode map e190 with expected crc 2023-09-23T20:26:17.375 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:17 smithi018 ceph-mon[184144]: osdmap e191: 8 total, 8 up, 8 in 2023-09-23T20:26:17.400 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:17.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:17 smithi149 ceph-mon[170394]: pgmap v410: 109 pgs: 1 active+recovering+undersized+remapped, 99 active+clean, 1 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped; 2.2 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 947 op/s; 7697/109320 objects degraded (7.041%); 1079/109320 objects misplaced (0.987%); 0 B/s, 138 objects/s recovering 2023-09-23T20:26:17.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:17 smithi149 ceph-mon[170394]: failed to encode map e190 with expected crc 2023-09-23T20:26:17.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:17 smithi149 ceph-mon[170394]: failed to encode map e190 with expected crc 2023-09-23T20:26:17.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:17 smithi149 ceph-mon[170394]: failed to encode map e190 with expected crc 2023-09-23T20:26:17.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:17 smithi149 ceph-mon[170394]: osdmap e191: 8 total, 8 up, 8 in 2023-09-23T20:26:17.499 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:26:17.504 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:17.505 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:26:17.605 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:26:17.683 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:17.743 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:26:17.751 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:17.752 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:26:17.852 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:26:17.948 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:18.104 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:18 smithi018 ceph-mon[179647]: failed to encode map e191 with expected crc 2023-09-23T20:26:18.104 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:18 smithi018 ceph-mon[179647]: failed to encode map e191 with expected crc 2023-09-23T20:26:18.105 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:18 smithi018 ceph-mon[179647]: failed to encode map e191 with expected crc 2023-09-23T20:26:18.105 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:18 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1182573744' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T20:26:18.105 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:18 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T20:26:18.107 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:26:18.120 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:18.120 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:26:18.125 INFO:teuthology.orchestra.run.smithi018.stderr:reweighted osd.6 to 1 (10000) 2023-09-23T20:26:18.185 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:26:18.221 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:26:18.314 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:18.391 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:26:18.401 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:18 smithi018 ceph-mon[184144]: failed to encode map e191 with expected crc 2023-09-23T20:26:18.402 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:18 smithi018 ceph-mon[184144]: failed to encode map e191 with expected crc 2023-09-23T20:26:18.402 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:18 smithi018 ceph-mon[184144]: failed to encode map e191 with expected crc 2023-09-23T20:26:18.402 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:18 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1182573744' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T20:26:18.402 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:18 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T20:26:18.406 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:18.406 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:26:18.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:18 smithi149 ceph-mon[170394]: failed to encode map e191 with expected crc 2023-09-23T20:26:18.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:18 smithi149 ceph-mon[170394]: failed to encode map e191 with expected crc 2023-09-23T20:26:18.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:18 smithi149 ceph-mon[170394]: failed to encode map e191 with expected crc 2023-09-23T20:26:18.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:18 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1182573744' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T20:26:18.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:18 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T20:26:18.506 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:26:18.584 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:18.646 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:26:18.652 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:18.653 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:26:18.753 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:26:18.806 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:18.869 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:26:18.875 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:18.875 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:26:18.976 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:26:19.040 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:19.071 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd reweight 7 1 2023-09-23T20:26:19.093 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:26:19.096 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:19.097 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:26:19.198 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:26:19.218 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:19 smithi149 ceph-mon[170394]: pgmap v413: 109 pgs: 1 active+recovering+undersized+remapped, 100 active+clean, 1 active+undersized+remapped, 1 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped; 2.1 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 769 op/s; 5596/103362 objects degraded (5.414%); 1079/103362 objects misplaced (1.044%); 0 B/s, 230 objects/s recovering 2023-09-23T20:26:19.219 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:19 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T20:26:19.219 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:19 smithi149 ceph-mon[170394]: osdmap e192: 8 total, 8 up, 8 in 2023-09-23T20:26:19.219 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:19 smithi149 ceph-mon[170394]: failed to encode map e192 with expected crc 2023-09-23T20:26:19.219 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:19 smithi149 ceph-mon[170394]: failed to encode map e192 with expected crc 2023-09-23T20:26:19.219 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:19 smithi149 ceph-mon[170394]: failed to encode map e192 with expected crc 2023-09-23T20:26:19.220 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:19 smithi149 ceph-mon[170394]: failed to encode map e192 with expected crc 2023-09-23T20:26:19.220 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:19 smithi149 ceph-mon[170394]: failed to encode map e192 with expected crc 2023-09-23T20:26:19.267 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:19.311 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:26:19.319 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:26:19.322 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:19.322 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:26:19.360 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:26:19.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:19 smithi018 ceph-mon[184144]: pgmap v413: 109 pgs: 1 active+recovering+undersized+remapped, 100 active+clean, 1 active+undersized+remapped, 1 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped; 2.1 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 769 op/s; 5596/103362 objects degraded (5.414%); 1079/103362 objects misplaced (1.044%); 0 B/s, 230 objects/s recovering 2023-09-23T20:26:19.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:19 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T20:26:19.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:19 smithi018 ceph-mon[184144]: osdmap e192: 8 total, 8 up, 8 in 2023-09-23T20:26:19.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:19 smithi018 ceph-mon[184144]: failed to encode map e192 with expected crc 2023-09-23T20:26:19.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:19 smithi018 ceph-mon[184144]: failed to encode map e192 with expected crc 2023-09-23T20:26:19.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:19 smithi018 ceph-mon[184144]: failed to encode map e192 with expected crc 2023-09-23T20:26:19.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:19 smithi018 ceph-mon[184144]: failed to encode map e192 with expected crc 2023-09-23T20:26:19.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:19 smithi018 ceph-mon[184144]: failed to encode map e192 with expected crc 2023-09-23T20:26:19.394 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:19 smithi018 ceph-mon[179647]: pgmap v413: 109 pgs: 1 active+recovering+undersized+remapped, 100 active+clean, 1 active+undersized+remapped, 1 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped; 2.1 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 769 op/s; 5596/103362 objects degraded (5.414%); 1079/103362 objects misplaced (1.044%); 0 B/s, 230 objects/s recovering 2023-09-23T20:26:19.394 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:19 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T20:26:19.394 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:19 smithi018 ceph-mon[179647]: osdmap e192: 8 total, 8 up, 8 in 2023-09-23T20:26:19.394 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:19 smithi018 ceph-mon[179647]: failed to encode map e192 with expected crc 2023-09-23T20:26:19.394 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:19 smithi018 ceph-mon[179647]: failed to encode map e192 with expected crc 2023-09-23T20:26:19.395 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:19 smithi018 ceph-mon[179647]: failed to encode map e192 with expected crc 2023-09-23T20:26:19.395 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:19 smithi018 ceph-mon[179647]: failed to encode map e192 with expected crc 2023-09-23T20:26:19.395 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:19 smithi018 ceph-mon[179647]: failed to encode map e192 with expected crc 2023-09-23T20:26:19.402 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:26:19.423 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:26:19.436 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:26:19.475 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:19.477 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:26:19.504 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:26:19.517 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:26:19.520 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:19.521 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:26:19.538 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:26:19.565 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:26:19.567 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:26:19.568 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:26:19.568 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:26:19.568 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:26:19.568 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:26:19.568 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:26:19.568 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:26:19.569 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:26:19.622 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:26:19.653 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:19.714 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:26:19.717 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:19.718 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:26:19.819 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:26:19.845 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:19.879 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:26:19.881 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:19.882 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:26:19.983 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:26:20.017 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:20.103 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:26:20.107 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:20.107 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:26:20.146 INFO:teuthology.orchestra.run.smithi018.stderr:noscrub is unset 2023-09-23T20:26:20.204 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:26:20.208 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:26:20.255 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:20.319 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:26:20.326 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:20.327 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:26:20.351 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2023-09-23T20:26:20.351 INFO:teuthology.orchestra.run.smithi018.stdout: "ops": [ 2023-09-23T20:26:20.351 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:26:20.351 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.34791.0:182982 5.3 5:c26a037e:::benchmark_data_smithi018_228451_object75921:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e194)", 2023-09-23T20:26:20.352 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:26:20.333786+0000", 2023-09-23T20:26:20.352 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.014533656000000001, 2023-09-23T20:26:20.352 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.014568500999999999, 2023-09-23T20:26:20.352 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:26:20.352 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:26:20.352 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:26:20.353 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.34791", 2023-09-23T20:26:20.353 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:26:20.353 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 182982 2023-09-23T20:26:20.353 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:26:20.353 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:26:20.353 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:26:20.353 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:26:20.354 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:26:20.333786+0000", 2023-09-23T20:26:20.354 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:26:20.354 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:26:20.354 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:26:20.354 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:26:20.354 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:26:20.333786+0000", 2023-09-23T20:26:20.354 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:26:20.355 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:26:20.355 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:26:20.355 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:26:20.355 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:26:20.333785+0000", 2023-09-23T20:26:20.355 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:26:20.355 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:26:20.355 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:26:20.356 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:26:20.356 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:26:20.333797+0000", 2023-09-23T20:26:20.356 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2456000000000001e-05 2023-09-23T20:26:20.356 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:26:20.356 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:26:20.356 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:26:20.356 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:26:20.333838+0000", 2023-09-23T20:26:20.357 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.0812000000000002e-05 2023-09-23T20:26:20.357 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:26:20.357 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:26:20.357 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:26:20.357 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:26:20.333846+0000", 2023-09-23T20:26:20.357 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.1459999999999996e-06 2023-09-23T20:26:20.357 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:26:20.358 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:26:20.358 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:26:20.358 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:26:20.333896+0000", 2023-09-23T20:26:20.358 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.9910000000000002e-05 2023-09-23T20:26:20.358 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:26:20.358 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:26:20.358 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:26:20.359 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:26:20.334020+0000", 2023-09-23T20:26:20.359 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000124141 2023-09-23T20:26:20.359 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:26:20.359 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:26:20.359 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 5,6", 2023-09-23T20:26:20.359 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:26:20.334495+0000", 2023-09-23T20:26:20.359 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000475017 2023-09-23T20:26:20.360 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:26:20.360 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:26:20.360 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:26:20.360 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:26:20.336936+0000", 2023-09-23T20:26:20.360 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.002440458 2023-09-23T20:26:20.360 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:26:20.360 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:26:20.361 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:26:20.361 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:26:20.361 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:26:20.361 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.34791.0:182987 5.13 5:cf1143b5:::benchmark_data_smithi018_228451_object75926:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e194)", 2023-09-23T20:26:20.361 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:26:20.344862+0000", 2023-09-23T20:26:20.361 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.0034584239999999999, 2023-09-23T20:26:20.361 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0035861170000000002, 2023-09-23T20:26:20.362 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:26:20.362 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:26:20.362 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:26:20.362 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.34791", 2023-09-23T20:26:20.362 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:26:20.362 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 182987 2023-09-23T20:26:20.362 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:26:20.363 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:26:20.363 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:26:20.363 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:26:20.363 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:26:20.344862+0000", 2023-09-23T20:26:20.363 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:26:20.363 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:26:20.363 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:26:20.364 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:26:20.364 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:26:20.344862+0000", 2023-09-23T20:26:20.364 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:26:20.364 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:26:20.364 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:26:20.364 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:26:20.365 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:26:20.344860+0000", 2023-09-23T20:26:20.365 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:26:20.365 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:26:20.365 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:26:20.365 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:26:20.365 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:26:20.344867+0000", 2023-09-23T20:26:20.365 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.5969999999999997e-06 2023-09-23T20:26:20.366 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:26:20.366 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:26:20.366 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:26:20.366 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:26:20.344884+0000", 2023-09-23T20:26:20.366 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.7700999999999999e-05 2023-09-23T20:26:20.366 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:26:20.366 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:26:20.367 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:26:20.367 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:26:20.344894+0000", 2023-09-23T20:26:20.367 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.3419999999999997e-06 2023-09-23T20:26:20.367 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:26:20.367 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:26:20.367 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:26:20.367 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:26:20.344954+0000", 2023-09-23T20:26:20.367 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.0523999999999999e-05 2023-09-23T20:26:20.368 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:26:20.368 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:26:20.368 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:26:20.368 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:26:20.345109+0000", 2023-09-23T20:26:20.368 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000155098 2023-09-23T20:26:20.368 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:26:20.368 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:26:20.369 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:26:20.369 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:26:20.345631+0000", 2023-09-23T20:26:20.369 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00052116599999999999 2023-09-23T20:26:20.369 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:26:20.369 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:26:20.369 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:26:20.370 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:26:20.370 INFO:teuthology.orchestra.run.smithi018.stdout: ], 2023-09-23T20:26:20.370 INFO:teuthology.orchestra.run.smithi018.stdout: "num_ops": 2 2023-09-23T20:26:20.370 INFO:teuthology.orchestra.run.smithi018.stdout:} 2023-09-23T20:26:20.370 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:20 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/865876219' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:26:20.370 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:20 smithi018 ceph-mon[179647]: osdmap e193: 8 total, 8 up, 8 in 2023-09-23T20:26:20.371 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:20 smithi018 ceph-mon[179647]: failed to encode map e193 with expected crc 2023-09-23T20:26:20.371 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:20 smithi018 ceph-mon[179647]: failed to encode map e193 with expected crc 2023-09-23T20:26:20.371 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:20 smithi018 ceph-mon[179647]: failed to encode map e193 with expected crc 2023-09-23T20:26:20.371 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:20 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2897068676' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:26:20.371 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:20 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/865876219' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:26:20.371 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:20 smithi018 ceph-mon[184144]: osdmap e193: 8 total, 8 up, 8 in 2023-09-23T20:26:20.371 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:20 smithi018 ceph-mon[184144]: failed to encode map e193 with expected crc 2023-09-23T20:26:20.371 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:20 smithi018 ceph-mon[184144]: failed to encode map e193 with expected crc 2023-09-23T20:26:20.372 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:20 smithi018 ceph-mon[184144]: failed to encode map e193 with expected crc 2023-09-23T20:26:20.372 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:20 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2897068676' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:26:20.427 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:26:20.468 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:20 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/865876219' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:26:20.469 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:20 smithi149 ceph-mon[170394]: osdmap e193: 8 total, 8 up, 8 in 2023-09-23T20:26:20.469 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:20 smithi149 ceph-mon[170394]: failed to encode map e193 with expected crc 2023-09-23T20:26:20.469 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:20 smithi149 ceph-mon[170394]: failed to encode map e193 with expected crc 2023-09-23T20:26:20.469 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:20 smithi149 ceph-mon[170394]: failed to encode map e193 with expected crc 2023-09-23T20:26:20.469 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:20 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2897068676' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:26:20.498 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:20.565 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:26:20.574 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:20.574 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:26:20.675 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:26:20.739 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:20.981 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:26:20.985 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:20.985 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:26:21.086 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:26:21.148 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:21.224 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:26:21.231 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:21.231 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:26:21.304 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:21 smithi018 ceph-mon[179647]: pgmap v416: 109 pgs: 1 active+recovering+undersized+remapped, 100 active+clean, 1 active+undersized+remapped, 1 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped; 2.1 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 551 op/s; 5596/102705 objects degraded (5.449%); 1079/102705 objects misplaced (1.051%); 0 B/s, 167 objects/s recovering 2023-09-23T20:26:21.304 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:21 smithi018 ceph-mon[179647]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:26:21.304 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:21 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/865876219' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:26:21.304 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:21 smithi018 ceph-mon[179647]: osdmap e194: 8 total, 8 up, 8 in 2023-09-23T20:26:21.304 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:21 smithi018 ceph-mon[179647]: failed to encode map e194 with expected crc 2023-09-23T20:26:21.304 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:21 smithi018 ceph-mon[179647]: failed to encode map e194 with expected crc 2023-09-23T20:26:21.304 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:21 smithi018 ceph-mon[179647]: failed to encode map e194 with expected crc 2023-09-23T20:26:21.304 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:21 smithi018 ceph-mon[179647]: failed to encode map e194 with expected crc 2023-09-23T20:26:21.305 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:21 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 5596/102705 objects degraded (5.449%), 6 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T20:26:21.305 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:21 smithi018 ceph-mon[184144]: pgmap v416: 109 pgs: 1 active+recovering+undersized+remapped, 100 active+clean, 1 active+undersized+remapped, 1 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped; 2.1 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 551 op/s; 5596/102705 objects degraded (5.449%); 1079/102705 objects misplaced (1.051%); 0 B/s, 167 objects/s recovering 2023-09-23T20:26:21.305 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:21 smithi018 ceph-mon[184144]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:26:21.305 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:21 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/865876219' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:26:21.305 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:21 smithi018 ceph-mon[184144]: osdmap e194: 8 total, 8 up, 8 in 2023-09-23T20:26:21.305 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:21 smithi018 ceph-mon[184144]: failed to encode map e194 with expected crc 2023-09-23T20:26:21.305 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:21 smithi018 ceph-mon[184144]: failed to encode map e194 with expected crc 2023-09-23T20:26:21.305 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:21 smithi018 ceph-mon[184144]: failed to encode map e194 with expected crc 2023-09-23T20:26:21.305 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:21 smithi018 ceph-mon[184144]: failed to encode map e194 with expected crc 2023-09-23T20:26:21.306 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:21 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 5596/102705 objects degraded (5.449%), 6 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T20:26:21.332 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:26:21.386 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:21.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:21 smithi149 ceph-mon[170394]: pgmap v416: 109 pgs: 1 active+recovering+undersized+remapped, 100 active+clean, 1 active+undersized+remapped, 1 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped; 2.1 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 551 op/s; 5596/102705 objects degraded (5.449%); 1079/102705 objects misplaced (1.051%); 0 B/s, 167 objects/s recovering 2023-09-23T20:26:21.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:21 smithi149 ceph-mon[170394]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:26:21.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:21 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/865876219' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:26:21.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:21 smithi149 ceph-mon[170394]: osdmap e194: 8 total, 8 up, 8 in 2023-09-23T20:26:21.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:21 smithi149 ceph-mon[170394]: failed to encode map e194 with expected crc 2023-09-23T20:26:21.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:21 smithi149 ceph-mon[170394]: failed to encode map e194 with expected crc 2023-09-23T20:26:21.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:21 smithi149 ceph-mon[170394]: failed to encode map e194 with expected crc 2023-09-23T20:26:21.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:21 smithi149 ceph-mon[170394]: failed to encode map e194 with expected crc 2023-09-23T20:26:21.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:21 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 5596/102705 objects degraded (5.449%), 6 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T20:26:21.932 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:26:23.125 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:26:23.130 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:23.130 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:26:23.231 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:26:23.279 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:23.354 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:26:23.365 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:23.366 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:26:23.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:23 smithi018 ceph-mon[184144]: pgmap v418: 109 pgs: 1 active+recovering+undersized+remapped, 101 active+clean, 1 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped; 1.9 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 716 op/s; 5596/94269 objects degraded (5.936%); 1079/94269 objects misplaced (1.145%); 0 B/s, 133 objects/s recovering 2023-09-23T20:26:23.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:23 smithi018 ceph-mon[179647]: pgmap v418: 109 pgs: 1 active+recovering+undersized+remapped, 101 active+clean, 1 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped; 1.9 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 716 op/s; 5596/94269 objects degraded (5.936%); 1079/94269 objects misplaced (1.145%); 0 B/s, 133 objects/s recovering 2023-09-23T20:26:23.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:23 smithi149 ceph-mon[170394]: pgmap v418: 109 pgs: 1 active+recovering+undersized+remapped, 101 active+clean, 1 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped; 1.9 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 716 op/s; 5596/94269 objects degraded (5.936%); 1079/94269 objects misplaced (1.145%); 0 B/s, 133 objects/s recovering 2023-09-23T20:26:23.466 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:26:23.529 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:23.714 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:26:23.724 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:23.724 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:26:23.760 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T20:26:23.826 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:26:23.853 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:23.904 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:26:23.907 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:23.907 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:26:24.008 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:26:24.106 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:24.694 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:26:24.701 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:24.702 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:26:24.802 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:26:24.889 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:24.953 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:26:24.963 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:24.964 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:26:24.964 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T20:26:24.987 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:26:25.003 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:26:24 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:26:24] "GET /metrics HTTP/1.1" 200 36463 "" "Prometheus/2.43.0" 2023-09-23T20:26:25.064 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:26:25.103 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:25.165 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:26:25.173 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:25.173 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:26:25.274 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:26:25.324 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:25.380 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:26:25.384 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:26:25.385 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:25.385 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:26:25.400 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:25 smithi018 ceph-mon[179647]: pgmap v419: 109 pgs: 1 active+recovering+undersized+remapped, 102 active+clean, 6 active+recovery_wait+undersized+degraded+remapped; 1.9 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 614 op/s; 5596/92298 objects degraded (6.063%); 1.5 MiB/s, 110 objects/s recovering 2023-09-23T20:26:25.400 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:25 smithi018 ceph-mon[184144]: pgmap v419: 109 pgs: 1 active+recovering+undersized+remapped, 102 active+clean, 6 active+recovery_wait+undersized+degraded+remapped; 1.9 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 614 op/s; 5596/92298 objects degraded (6.063%); 1.5 MiB/s, 110 objects/s recovering 2023-09-23T20:26:25.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:25 smithi149 ceph-mon[170394]: pgmap v419: 109 pgs: 1 active+recovering+undersized+remapped, 102 active+clean, 6 active+recovery_wait+undersized+degraded+remapped; 1.9 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 614 op/s; 5596/92298 objects degraded (6.063%); 1.5 MiB/s, 110 objects/s recovering 2023-09-23T20:26:25.486 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:26:25.552 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:25.621 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:26:25.632 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:25.633 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:26:25.734 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:26:25.848 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:25.973 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:26:25.983 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:25.984 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:26:26.085 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:26:26.172 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:26.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:26 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 5596/92298 objects degraded (6.063%), 6 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T20:26:26.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:26 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 5596/92298 objects degraded (6.063%), 6 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T20:26:26.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:26 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 5596/92298 objects degraded (6.063%), 6 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T20:26:26.652 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:26:26.658 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:26.659 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:26:26.760 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:26:26.875 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:26.985 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:26:27.106 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:26:27.116 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:27.117 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:26:27.198 INFO:teuthology.orchestra.run.smithi018.stderr:reweighted osd.7 to 1 (10000) 2023-09-23T20:26:27.217 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:26:27.276 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:27 smithi018 ceph-mon[179647]: pgmap v420: 109 pgs: 1 active+recovering+undersized+remapped, 102 active+clean, 6 active+recovery_wait+undersized+degraded+remapped; 1.8 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 676 op/s; 5596/87627 objects degraded (6.386%); 1.2 MiB/s, 89 objects/s recovering 2023-09-23T20:26:27.276 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:27 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/417467015' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T20:26:27.276 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:27 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T20:26:27.277 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:27 smithi018 ceph-mon[184144]: pgmap v420: 109 pgs: 1 active+recovering+undersized+remapped, 102 active+clean, 6 active+recovery_wait+undersized+degraded+remapped; 1.8 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 676 op/s; 5596/87627 objects degraded (6.386%); 1.2 MiB/s, 89 objects/s recovering 2023-09-23T20:26:27.277 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:27 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/417467015' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T20:26:27.277 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:27 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T20:26:27.289 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:27.437 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:26:27.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:27 smithi149 ceph-mon[170394]: pgmap v420: 109 pgs: 1 active+recovering+undersized+remapped, 102 active+clean, 6 active+recovery_wait+undersized+degraded+remapped; 1.8 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 676 op/s; 5596/87627 objects degraded (6.386%); 1.2 MiB/s, 89 objects/s recovering 2023-09-23T20:26:27.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:27 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/417467015' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T20:26:27.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:27 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T20:26:27.486 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:26:27.523 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:26:27.565 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:26:27.566 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:26:27.569 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:27.570 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:26:27.599 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:26:27.631 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:26:27.671 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:26:27.677 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:26:27.741 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:26:27.745 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd reweight 3 1 2023-09-23T20:26:27.769 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:27.769 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:26:27.769 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:26:27.770 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:26:27.770 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:26:27.770 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:26:27.770 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:26:27.770 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:26:27.770 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:26:28.378 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:26:28.384 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:28.385 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:26:28.445 INFO:teuthology.orchestra.run.smithi018.stdout:ERROR: (22) Invalid argument 2023-09-23T20:26:28.445 INFO:teuthology.orchestra.run.smithi018.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:26:28.460 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:28 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T20:26:28.460 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:28 smithi018 ceph-mon[184144]: osdmap e195: 8 total, 8 up, 8 in 2023-09-23T20:26:28.460 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:28 smithi018 ceph-mon[184144]: failed to encode map e195 with expected crc 2023-09-23T20:26:28.460 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:28 smithi018 ceph-mon[184144]: failed to encode map e195 with expected crc 2023-09-23T20:26:28.460 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:28 smithi018 ceph-mon[184144]: failed to encode map e195 with expected crc 2023-09-23T20:26:28.460 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:28 smithi018 ceph-mon[184144]: failed to encode map e195 with expected crc 2023-09-23T20:26:28.461 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:28 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2968639643' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:26:28.461 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:28 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T20:26:28.461 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:28 smithi018 ceph-mon[179647]: osdmap e195: 8 total, 8 up, 8 in 2023-09-23T20:26:28.461 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:28 smithi018 ceph-mon[179647]: failed to encode map e195 with expected crc 2023-09-23T20:26:28.461 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:28 smithi018 ceph-mon[179647]: failed to encode map e195 with expected crc 2023-09-23T20:26:28.461 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:28 smithi018 ceph-mon[179647]: failed to encode map e195 with expected crc 2023-09-23T20:26:28.462 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:28 smithi018 ceph-mon[179647]: failed to encode map e195 with expected crc 2023-09-23T20:26:28.462 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:28 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2968639643' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:26:28.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:28 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T20:26:28.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:28 smithi149 ceph-mon[170394]: osdmap e195: 8 total, 8 up, 8 in 2023-09-23T20:26:28.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:28 smithi149 ceph-mon[170394]: failed to encode map e195 with expected crc 2023-09-23T20:26:28.468 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:28 smithi149 ceph-mon[170394]: failed to encode map e195 with expected crc 2023-09-23T20:26:28.468 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:28 smithi149 ceph-mon[170394]: failed to encode map e195 with expected crc 2023-09-23T20:26:28.468 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:28 smithi149 ceph-mon[170394]: failed to encode map e195 with expected crc 2023-09-23T20:26:28.468 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:28 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2968639643' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:26:28.486 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:26:28.550 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:28.619 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:26:28.638 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:28.639 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:26:28.739 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:26:28.800 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:28.873 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:26:28.879 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:28.880 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:26:28.980 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:26:29.037 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:29.165 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T20:26:29.193 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:29 smithi149 ceph-mon[170394]: pgmap v422: 109 pgs: 1 active+recovering+undersized+remapped, 102 active+clean, 6 active+recovery_wait+undersized+degraded+remapped; 1.6 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 962 op/s; 5596/79593 objects degraded (7.031%); 1.1 MiB/s, 139 objects/s recovering 2023-09-23T20:26:29.234 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:26:29.240 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:29.240 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:26:29.290 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:29 smithi018 ceph-mon[179647]: pgmap v422: 109 pgs: 1 active+recovering+undersized+remapped, 102 active+clean, 6 active+recovery_wait+undersized+degraded+remapped; 1.6 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 962 op/s; 5596/79593 objects degraded (7.031%); 1.1 MiB/s, 139 objects/s recovering 2023-09-23T20:26:29.290 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:29 smithi018 ceph-mon[184144]: pgmap v422: 109 pgs: 1 active+recovering+undersized+remapped, 102 active+clean, 6 active+recovery_wait+undersized+degraded+remapped; 1.6 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 962 op/s; 5596/79593 objects degraded (7.031%); 1.1 MiB/s, 139 objects/s recovering 2023-09-23T20:26:29.341 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:26:29.392 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:29.430 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:26:29.458 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:26:29.463 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:29.464 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:26:29.565 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:26:29.630 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:29.684 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:26:29.689 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:29.689 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:26:29.790 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:26:29.830 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:29.917 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:26:29.926 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:29.926 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:26:30.027 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:26:30.092 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:30.232 INFO:teuthology.orchestra.run.smithi018.stderr:nodeep-scrub is unset 2023-09-23T20:26:30.470 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:30 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:26:30.471 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:30 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1564606580' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:26:30.471 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:30 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:26:30.471 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:26:30.498 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:26:30.505 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:30.506 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:26:30.506 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:30 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:26:30.506 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:30 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1564606580' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:26:30.506 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:30 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:26:30.506 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:30 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:26:30.506 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:30 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1564606580' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:26:30.507 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:30 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:26:30.607 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:26:30.708 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:26:30.718 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:30.927 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:26:30.933 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:30.934 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:26:31.035 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:26:31.076 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:31.119 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:26:31.128 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:31.128 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:26:31.229 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:26:31.269 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:31.325 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:26:31.329 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:31 smithi149 ceph-mon[170394]: pgmap v423: 109 pgs: 1 active+recovering+undersized+remapped, 102 active+clean, 6 active+recovery_wait+undersized+degraded+remapped; 1.6 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 759 op/s; 5596/78366 objects degraded (7.141%); 961 KiB/s, 118 objects/s recovering 2023-09-23T20:26:31.329 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:31 smithi149 ceph-mon[170394]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:26:31.329 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:31 smithi149 ceph-mon[170394]: failed to encode map e196 with expected crc 2023-09-23T20:26:31.330 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:31 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:26:31.330 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:31 smithi149 ceph-mon[170394]: osdmap e196: 8 total, 8 up, 8 in 2023-09-23T20:26:31.330 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:31 smithi149 ceph-mon[170394]: failed to encode map e196 with expected crc 2023-09-23T20:26:31.330 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:31 smithi149 ceph-mon[170394]: failed to encode map e196 with expected crc 2023-09-23T20:26:31.330 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:31 smithi149 ceph-mon[170394]: failed to encode map e196 with expected crc 2023-09-23T20:26:31.330 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:31 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 5596/78366 objects degraded (7.141%), 6 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T20:26:31.331 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:31.331 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:26:31.432 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:26:31.481 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:31.488 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:26:31.545 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:26:31.549 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:31.549 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:26:31.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:31 smithi018 ceph-mon[179647]: pgmap v423: 109 pgs: 1 active+recovering+undersized+remapped, 102 active+clean, 6 active+recovery_wait+undersized+degraded+remapped; 1.6 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 759 op/s; 5596/78366 objects degraded (7.141%); 961 KiB/s, 118 objects/s recovering 2023-09-23T20:26:31.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:31 smithi018 ceph-mon[179647]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:26:31.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:31 smithi018 ceph-mon[179647]: failed to encode map e196 with expected crc 2023-09-23T20:26:31.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:31 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:26:31.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:31 smithi018 ceph-mon[179647]: osdmap e196: 8 total, 8 up, 8 in 2023-09-23T20:26:31.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:31 smithi018 ceph-mon[179647]: failed to encode map e196 with expected crc 2023-09-23T20:26:31.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:31 smithi018 ceph-mon[179647]: failed to encode map e196 with expected crc 2023-09-23T20:26:31.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:31 smithi018 ceph-mon[179647]: failed to encode map e196 with expected crc 2023-09-23T20:26:31.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:31 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 5596/78366 objects degraded (7.141%), 6 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T20:26:31.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:31 smithi018 ceph-mon[184144]: pgmap v423: 109 pgs: 1 active+recovering+undersized+remapped, 102 active+clean, 6 active+recovery_wait+undersized+degraded+remapped; 1.6 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 759 op/s; 5596/78366 objects degraded (7.141%); 961 KiB/s, 118 objects/s recovering 2023-09-23T20:26:31.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:31 smithi018 ceph-mon[184144]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:26:31.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:31 smithi018 ceph-mon[184144]: failed to encode map e196 with expected crc 2023-09-23T20:26:31.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:31 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:26:31.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:31 smithi018 ceph-mon[184144]: osdmap e196: 8 total, 8 up, 8 in 2023-09-23T20:26:31.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:31 smithi018 ceph-mon[184144]: failed to encode map e196 with expected crc 2023-09-23T20:26:31.643 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:31 smithi018 ceph-mon[184144]: failed to encode map e196 with expected crc 2023-09-23T20:26:31.643 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:31 smithi018 ceph-mon[184144]: failed to encode map e196 with expected crc 2023-09-23T20:26:31.643 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:31 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 5596/78366 objects degraded (7.141%), 6 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T20:26:31.650 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:26:31.721 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:31.851 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:26:31.858 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:31.859 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:26:31.960 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:26:32.073 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:32.251 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:26:32.261 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:32 smithi018 ceph-mon[184144]: pgmap v425: 109 pgs: 1 active+recovering+undersized+remapped, 102 active+clean, 6 active+recovery_wait+undersized+degraded+remapped; 1.4 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s; 5596/67143 objects degraded (8.334%); 0 B/s, 109 objects/s recovering 2023-09-23T20:26:32.261 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:32 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2937262789' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T20:26:32.266 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:32.266 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:32 smithi018 ceph-mon[179647]: pgmap v425: 109 pgs: 1 active+recovering+undersized+remapped, 102 active+clean, 6 active+recovery_wait+undersized+degraded+remapped; 1.4 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s; 5596/67143 objects degraded (8.334%); 0 B/s, 109 objects/s recovering 2023-09-23T20:26:32.266 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:32 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2937262789' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T20:26:32.267 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:26:32.295 INFO:teuthology.orchestra.run.smithi018.stderr:reweighted osd.3 to 1 (10000) 2023-09-23T20:26:32.367 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:26:32.397 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:32.447 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:26:32.453 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:32.453 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:26:32.554 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:26:32.568 INFO:teuthology.orchestra.run.smithi018.stdout:ERROR: (22) Invalid argument 2023-09-23T20:26:32.568 INFO:teuthology.orchestra.run.smithi018.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:26:32.579 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:32 smithi149 ceph-mon[170394]: pgmap v425: 109 pgs: 1 active+recovering+undersized+remapped, 102 active+clean, 6 active+recovery_wait+undersized+degraded+remapped; 1.4 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s; 5596/67143 objects degraded (8.334%); 0 B/s, 109 objects/s recovering 2023-09-23T20:26:32.579 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:32 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2937262789' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T20:26:32.582 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:32.615 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:26:32.618 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:32.618 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:26:32.719 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:26:32.750 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T20:26:32.751 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:32.946 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T20:26:33.047 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:26:33.049 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:33.050 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:26:33.150 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:26:33.179 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:33.213 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:26:33.216 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:33.217 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:26:33.317 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:26:33.368 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:33.427 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd reweight 2 1 2023-09-23T20:26:33.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:33 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2937262789' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T20:26:33.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:33 smithi149 ceph-mon[170394]: osdmap e197: 8 total, 8 up, 8 in 2023-09-23T20:26:33.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:33 smithi149 ceph-mon[170394]: failed to encode map e197 with expected crc 2023-09-23T20:26:33.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:33 smithi149 ceph-mon[170394]: failed to encode map e197 with expected crc 2023-09-23T20:26:33.469 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:26:33.480 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:33.480 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:26:33.531 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:33 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2937262789' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T20:26:33.531 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:33 smithi018 ceph-mon[179647]: osdmap e197: 8 total, 8 up, 8 in 2023-09-23T20:26:33.531 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:33 smithi018 ceph-mon[179647]: failed to encode map e197 with expected crc 2023-09-23T20:26:33.531 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:33 smithi018 ceph-mon[179647]: failed to encode map e197 with expected crc 2023-09-23T20:26:33.531 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:33 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2937262789' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T20:26:33.532 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:33 smithi018 ceph-mon[184144]: osdmap e197: 8 total, 8 up, 8 in 2023-09-23T20:26:33.532 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:33 smithi018 ceph-mon[184144]: failed to encode map e197 with expected crc 2023-09-23T20:26:33.532 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:33 smithi018 ceph-mon[184144]: failed to encode map e197 with expected crc 2023-09-23T20:26:33.582 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:26:33.642 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:33.689 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:26:33.696 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:33.697 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:26:33.771 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:26:33.798 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:26:33.853 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:33.896 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:26:33.905 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:33.905 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:26:34.006 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:26:34.094 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:34.124 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:26:34.201 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:26:34.208 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:26:34.209 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:34.210 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:26:34.255 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:26:34.290 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:26:34.311 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:26:34.331 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:26:34.365 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:34 smithi018 ceph-mon[179647]: failed to encode map e198 with expected crc 2023-09-23T20:26:34.365 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:34 smithi018 ceph-mon[179647]: osdmap e198: 8 total, 8 up, 8 in 2023-09-23T20:26:34.365 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:34 smithi018 ceph-mon[179647]: failed to encode map e198 with expected crc 2023-09-23T20:26:34.365 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:34 smithi018 ceph-mon[179647]: failed to encode map e198 with expected crc 2023-09-23T20:26:34.365 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:34 smithi018 ceph-mon[179647]: failed to encode map e198 with expected crc 2023-09-23T20:26:34.365 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:34 smithi018 ceph-mon[179647]: pgmap v428: 109 pgs: 2 active+recovering+undersized+remapped, 102 active+clean, 5 active+recovery_wait+undersized+degraded+remapped; 1.3 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 798 op/s; 4379/65205 objects degraded (6.716%); 0 B/s, 110 objects/s recovering 2023-09-23T20:26:34.365 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:34 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1982592483' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:26:34.366 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:34 smithi018 ceph-mon[184144]: failed to encode map e198 with expected crc 2023-09-23T20:26:34.366 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:34 smithi018 ceph-mon[184144]: osdmap e198: 8 total, 8 up, 8 in 2023-09-23T20:26:34.366 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:34 smithi018 ceph-mon[184144]: failed to encode map e198 with expected crc 2023-09-23T20:26:34.366 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:34 smithi018 ceph-mon[184144]: failed to encode map e198 with expected crc 2023-09-23T20:26:34.366 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:34 smithi018 ceph-mon[184144]: failed to encode map e198 with expected crc 2023-09-23T20:26:34.366 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:34 smithi018 ceph-mon[184144]: pgmap v428: 109 pgs: 2 active+recovering+undersized+remapped, 102 active+clean, 5 active+recovery_wait+undersized+degraded+remapped; 1.3 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 798 op/s; 4379/65205 objects degraded (6.716%); 0 B/s, 110 objects/s recovering 2023-09-23T20:26:34.366 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:34 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1982592483' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:26:34.367 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:34.378 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:26:34.423 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:26:34.472 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:26:34.506 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:26:34.506 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:26:34.506 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:26:34.506 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:26:34.507 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:26:34.507 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:26:34.507 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:26:34.507 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:26:34.507 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:26:34.650 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:26:34.656 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:34.656 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:26:34.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:34 smithi149 ceph-mon[170394]: failed to encode map e198 with expected crc 2023-09-23T20:26:34.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:34 smithi149 ceph-mon[170394]: osdmap e198: 8 total, 8 up, 8 in 2023-09-23T20:26:34.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:34 smithi149 ceph-mon[170394]: failed to encode map e198 with expected crc 2023-09-23T20:26:34.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:34 smithi149 ceph-mon[170394]: failed to encode map e198 with expected crc 2023-09-23T20:26:34.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:34 smithi149 ceph-mon[170394]: failed to encode map e198 with expected crc 2023-09-23T20:26:34.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:34 smithi149 ceph-mon[170394]: pgmap v428: 109 pgs: 2 active+recovering+undersized+remapped, 102 active+clean, 5 active+recovery_wait+undersized+degraded+remapped; 1.3 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 798 op/s; 4379/65205 objects degraded (6.716%); 0 B/s, 110 objects/s recovering 2023-09-23T20:26:34.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:34 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1982592483' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:26:34.757 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:26:34.843 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:35.070 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:26:35.080 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:35.081 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:26:35.182 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:26:35.238 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:35.303 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:26:35.311 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:35.311 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:26:35.391 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:26:34 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:26:34] "GET /metrics HTTP/1.1" 200 36472 "" "Prometheus/2.43.0" 2023-09-23T20:26:35.412 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:26:35.475 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:36.260 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:26:36.274 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:26:36.280 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:36.281 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:26:36.382 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:26:36.469 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:36.686 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:26:36.694 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:36.695 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:26:36.796 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:26:36.817 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:36 smithi018 ceph-mon[179647]: pgmap v429: 109 pgs: 2 active+recovering+undersized+remapped, 102 active+clean, 5 active+recovery_wait+undersized+degraded+remapped; 1.3 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 936 op/s; 4379/61494 objects degraded (7.121%); 0 B/s, 110 objects/s recovering 2023-09-23T20:26:36.817 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:36 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 4379/61494 objects degraded (7.121%), 5 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T20:26:36.821 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:36 smithi018 ceph-mon[184144]: pgmap v429: 109 pgs: 2 active+recovering+undersized+remapped, 102 active+clean, 5 active+recovery_wait+undersized+degraded+remapped; 1.3 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 936 op/s; 4379/61494 objects degraded (7.121%); 0 B/s, 110 objects/s recovering 2023-09-23T20:26:36.821 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:36 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 4379/61494 objects degraded (7.121%), 5 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T20:26:36.910 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:36.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:36 smithi149 ceph-mon[170394]: pgmap v429: 109 pgs: 2 active+recovering+undersized+remapped, 102 active+clean, 5 active+recovery_wait+undersized+degraded+remapped; 1.3 GiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 936 op/s; 4379/61494 objects degraded (7.121%); 0 B/s, 110 objects/s recovering 2023-09-23T20:26:36.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:36 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 4379/61494 objects degraded (7.121%), 5 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T20:26:37.054 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:26:37.386 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:26:37.389 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:37.389 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:26:37.490 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:26:37.613 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:37.733 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:26:37.740 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:37.740 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:26:37.780 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2023-09-23T20:26:37.780 INFO:teuthology.orchestra.run.smithi018.stdout: "ops": [ 2023-09-23T20:26:37.780 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:26:37.780 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_repop(client.34791.0:197941 5.18 e198/169 5:1e91305e:::benchmark_data_smithi018_228451_object90880:head v 198'7070, mlcod=198'7066)", 2023-09-23T20:26:37.781 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:26:37.762739+0000", 2023-09-23T20:26:37.781 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.013955024999999999, 2023-09-23T20:26:37.781 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.013986854, 2023-09-23T20:26:37.781 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:26:37.781 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "started", 2023-09-23T20:26:37.781 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:26:37.781 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:26:37.781 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:26:37.781 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:26:37.762739+0000", 2023-09-23T20:26:37.781 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:26:37.782 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:26:37.782 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:26:37.782 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:26:37.782 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:26:37.762739+0000", 2023-09-23T20:26:37.782 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:26:37.782 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:26:37.782 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:26:37.782 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:26:37.782 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:26:37.762738+0000", 2023-09-23T20:26:37.782 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:26:37.783 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:26:37.783 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:26:37.783 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:26:37.783 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:26:37.762745+0000", 2023-09-23T20:26:37.783 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.1409999999999999e-06 2023-09-23T20:26:37.783 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:26:37.783 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:26:37.783 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:26:37.783 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:26:37.762756+0000", 2023-09-23T20:26:37.783 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0947999999999999e-05 2023-09-23T20:26:37.784 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:26:37.784 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:26:37.784 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:26:37.784 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:26:37.762763+0000", 2023-09-23T20:26:37.784 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.5599999999999999e-06 2023-09-23T20:26:37.784 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:26:37.784 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:26:37.784 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:26:37.784 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:26:37.771205+0000", 2023-09-23T20:26:37.784 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0084416179999999997 2023-09-23T20:26:37.784 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:26:37.784 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:26:37.785 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:26:37.785 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:26:37.771224+0000", 2023-09-23T20:26:37.785 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.9681999999999999e-05 2023-09-23T20:26:37.785 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:26:37.785 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:26:37.785 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:26:37.785 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:26:37.785 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:26:37.785 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.34791.0:197946 5.c 5:35402352:::benchmark_data_smithi018_228451_object90885:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e198)", 2023-09-23T20:26:37.785 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:26:37.772766+0000", 2023-09-23T20:26:37.785 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.0039277670000000004, 2023-09-23T20:26:37.785 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0040519570000000001, 2023-09-23T20:26:37.786 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:26:37.786 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:26:37.786 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:26:37.786 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.34791", 2023-09-23T20:26:37.786 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:26:37.786 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 197946 2023-09-23T20:26:37.786 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:26:37.786 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:26:37.786 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:26:37.786 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:26:37.786 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:26:37.772766+0000", 2023-09-23T20:26:37.786 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:26:37.787 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:26:37.787 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:26:37.787 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:26:37.787 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:26:37.772766+0000", 2023-09-23T20:26:37.787 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:26:37.787 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:26:37.787 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:26:37.787 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:26:37.787 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:26:37.772765+0000", 2023-09-23T20:26:37.787 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:26:37.787 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:26:37.787 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:26:37.788 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:26:37.788 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:26:37.772772+0000", 2023-09-23T20:26:37.788 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.4340000000000003e-06 2023-09-23T20:26:37.788 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:26:37.788 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:26:37.788 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:26:37.788 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:26:37.772801+0000", 2023-09-23T20:26:37.788 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.8636000000000002e-05 2023-09-23T20:26:37.788 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:26:37.788 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:26:37.788 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:26:37.789 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:26:37.772812+0000", 2023-09-23T20:26:37.789 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0795000000000001e-05 2023-09-23T20:26:37.789 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:26:37.789 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:26:37.789 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:26:37.789 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:26:37.773935+0000", 2023-09-23T20:26:37.789 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0011230560000000001 2023-09-23T20:26:37.789 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:26:37.789 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:26:37.789 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:26:37.789 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:26:37.774050+0000", 2023-09-23T20:26:37.789 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000114733 2023-09-23T20:26:37.790 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:26:37.790 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:26:37.790 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 0,4", 2023-09-23T20:26:37.790 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:26:37.774160+0000", 2023-09-23T20:26:37.790 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00011079299999999999 2023-09-23T20:26:37.790 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:26:37.790 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:26:37.790 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:26:37.790 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:26:37.790 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:26:37.790 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.34791.0:197949 5.12 5:4afb8d6d:::benchmark_data_smithi018_228451_object90888:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e198)", 2023-09-23T20:26:37.791 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:26:37.772838+0000", 2023-09-23T20:26:37.791 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.0038558160000000002, 2023-09-23T20:26:37.791 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0040742590000000002, 2023-09-23T20:26:37.791 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:26:37.791 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:26:37.791 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:26:37.791 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.34791", 2023-09-23T20:26:37.791 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:26:37.791 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 197949 2023-09-23T20:26:37.791 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:26:37.791 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:26:37.791 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:26:37.792 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:26:37.792 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:26:37.772838+0000", 2023-09-23T20:26:37.792 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:26:37.792 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:26:37.792 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:26:37.792 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:26:37.792 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:26:37.772838+0000", 2023-09-23T20:26:37.792 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:26:37.792 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:26:37.792 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:26:37.792 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:26:37.792 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:26:37.772838+0000", 2023-09-23T20:26:37.792 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:26:37.793 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:26:37.793 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:26:37.793 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:26:37.793 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:26:37.772842+0000", 2023-09-23T20:26:37.793 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.1479999999999999e-06 2023-09-23T20:26:37.793 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:26:37.793 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:26:37.793 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:26:37.793 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:26:37.772855+0000", 2023-09-23T20:26:37.793 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2873e-05 2023-09-23T20:26:37.793 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:26:37.793 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:26:37.794 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:26:37.794 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:26:37.772861+0000", 2023-09-23T20:26:37.794 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.5719999999999999e-06 2023-09-23T20:26:37.794 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:26:37.794 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:26:37.794 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:26:37.794 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:26:37.772912+0000", 2023-09-23T20:26:37.794 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.1354000000000003e-05 2023-09-23T20:26:37.794 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:26:37.794 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:26:37.794 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:26:37.794 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:26:37.773043+0000", 2023-09-23T20:26:37.795 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00013070900000000001 2023-09-23T20:26:37.795 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:26:37.795 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:26:37.795 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 3,5", 2023-09-23T20:26:37.795 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:26:37.773507+0000", 2023-09-23T20:26:37.795 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00046396799999999999 2023-09-23T20:26:37.795 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:26:37.795 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:26:37.795 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:26:37.795 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:26:37.776657+0000", 2023-09-23T20:26:37.795 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0031498469999999999 2023-09-23T20:26:37.795 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:26:37.796 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:26:37.796 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:26:37.796 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:26:37.796 INFO:teuthology.orchestra.run.smithi018.stdout: ], 2023-09-23T20:26:37.796 INFO:teuthology.orchestra.run.smithi018.stdout: "num_ops": 3 2023-09-23T20:26:37.796 INFO:teuthology.orchestra.run.smithi018.stdout:} 2023-09-23T20:26:37.841 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:26:37.940 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:38.199 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:26:38.206 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:38.206 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:26:38.280 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T20:26:38.307 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:26:38.336 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:26:38.429 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:38.506 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:26:38.525 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:38.525 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:26:38.626 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:26:38.668 INFO:teuthology.orchestra.run.smithi018.stderr:noscrub is set 2023-09-23T20:26:38.751 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:38.794 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:38 smithi018 ceph-mon[179647]: pgmap v430: 109 pgs: 1 active+recovering+undersized+remapped, 103 active+clean, 5 active+recovery_wait+undersized+degraded+remapped; 1.1 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 895 op/s; 4379/54933 objects degraded (7.972%); 0 B/s, 106 objects/s recovering 2023-09-23T20:26:38.794 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:38 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2295038345' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:26:38.794 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:38 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:26:38.796 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:38 smithi018 ceph-mon[184144]: pgmap v430: 109 pgs: 1 active+recovering+undersized+remapped, 103 active+clean, 5 active+recovery_wait+undersized+degraded+remapped; 1.1 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 895 op/s; 4379/54933 objects degraded (7.972%); 0 B/s, 106 objects/s recovering 2023-09-23T20:26:38.796 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:38 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2295038345' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:26:38.796 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:38 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:26:38.911 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:26:38.918 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:38.918 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:26:38.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:38 smithi149 ceph-mon[170394]: pgmap v430: 109 pgs: 1 active+recovering+undersized+remapped, 103 active+clean, 5 active+recovery_wait+undersized+degraded+remapped; 1.1 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 895 op/s; 4379/54933 objects degraded (7.972%); 0 B/s, 106 objects/s recovering 2023-09-23T20:26:38.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:38 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2295038345' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:26:38.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:38 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:26:39.020 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:26:39.089 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:39.164 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:26:39.174 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:39.175 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:26:39.275 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:26:39.350 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:39.427 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:26:39.435 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:39.436 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:26:39.537 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:26:39.594 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:39.680 INFO:teuthology.orchestra.run.smithi018.stderr:reweighted osd.2 to 1 (10000) 2023-09-23T20:26:39.730 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:26:39.741 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:39.741 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:26:39.744 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:26:39.843 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:26:39.863 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:39 smithi018 ceph-mon[184144]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:26:39.863 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:39 smithi018 ceph-mon[184144]: failed to encode map e199 with expected crc 2023-09-23T20:26:39.863 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:39 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:26:39.863 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:39 smithi018 ceph-mon[184144]: osdmap e199: 8 total, 8 up, 8 in 2023-09-23T20:26:39.864 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:39 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1809381079' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T20:26:39.864 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:39 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T20:26:39.864 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:39 smithi018 ceph-mon[179647]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:26:39.864 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:39 smithi018 ceph-mon[179647]: failed to encode map e199 with expected crc 2023-09-23T20:26:39.864 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:39 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:26:39.864 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:39 smithi018 ceph-mon[179647]: osdmap e199: 8 total, 8 up, 8 in 2023-09-23T20:26:39.865 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:39 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1809381079' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T20:26:39.865 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:39 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T20:26:39.916 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:39.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:39 smithi149 ceph-mon[170394]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:26:39.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:39 smithi149 ceph-mon[170394]: failed to encode map e199 with expected crc 2023-09-23T20:26:39.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:39 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:26:39.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:39 smithi149 ceph-mon[170394]: osdmap e199: 8 total, 8 up, 8 in 2023-09-23T20:26:39.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:39 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1809381079' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T20:26:39.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:39 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T20:26:40.117 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:26:40.120 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:40.121 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:26:40.221 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:26:40.229 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-09-23T20:26:40.229 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:26:40.281 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:40.384 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:26:40.389 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:40.390 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:26:40.490 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:26:40.593 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:40.672 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:26:40.690 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:40.691 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:26:40.791 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:26:40.829 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:40.862 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:40 smithi018 ceph-mon[179647]: pgmap v432: 109 pgs: 1 active+recovering+undersized+remapped, 103 active+clean, 5 active+recovery_wait+undersized+degraded+remapped; 1.1 GiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 566 op/s; 4379/53913 objects degraded (8.122%); 0 B/s, 103 objects/s recovering 2023-09-23T20:26:40.862 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:40 smithi018 ceph-mon[179647]: failed to encode map e200 with expected crc 2023-09-23T20:26:40.862 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:40 smithi018 ceph-mon[179647]: failed to encode map e200 with expected crc 2023-09-23T20:26:40.862 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:40 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T20:26:40.862 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:40 smithi018 ceph-mon[179647]: osdmap e200: 8 total, 8 up, 8 in 2023-09-23T20:26:40.862 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:40 smithi018 ceph-mon[179647]: failed to encode map e200 with expected crc 2023-09-23T20:26:40.862 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:40 smithi018 ceph-mon[179647]: failed to encode map e200 with expected crc 2023-09-23T20:26:40.862 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:40 smithi018 ceph-mon[179647]: failed to encode map e200 with expected crc 2023-09-23T20:26:40.868 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:40 smithi018 ceph-mon[184144]: pgmap v432: 109 pgs: 1 active+recovering+undersized+remapped, 103 active+clean, 5 active+recovery_wait+undersized+degraded+remapped; 1.1 GiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 566 op/s; 4379/53913 objects degraded (8.122%); 0 B/s, 103 objects/s recovering 2023-09-23T20:26:40.868 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:40 smithi018 ceph-mon[184144]: failed to encode map e200 with expected crc 2023-09-23T20:26:40.868 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:40 smithi018 ceph-mon[184144]: failed to encode map e200 with expected crc 2023-09-23T20:26:40.868 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:40 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T20:26:40.869 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:40 smithi018 ceph-mon[184144]: osdmap e200: 8 total, 8 up, 8 in 2023-09-23T20:26:40.869 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:40 smithi018 ceph-mon[184144]: failed to encode map e200 with expected crc 2023-09-23T20:26:40.869 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:40 smithi018 ceph-mon[184144]: failed to encode map e200 with expected crc 2023-09-23T20:26:40.869 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:40 smithi018 ceph-mon[184144]: failed to encode map e200 with expected crc 2023-09-23T20:26:40.894 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:26:40.902 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:40.902 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:26:40.907 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:26:40.944 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:26:40.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:40 smithi149 ceph-mon[170394]: pgmap v432: 109 pgs: 1 active+recovering+undersized+remapped, 103 active+clean, 5 active+recovery_wait+undersized+degraded+remapped; 1.1 GiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 566 op/s; 4379/53913 objects degraded (8.122%); 0 B/s, 103 objects/s recovering 2023-09-23T20:26:40.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:40 smithi149 ceph-mon[170394]: failed to encode map e200 with expected crc 2023-09-23T20:26:40.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:40 smithi149 ceph-mon[170394]: failed to encode map e200 with expected crc 2023-09-23T20:26:40.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:40 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T20:26:40.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:40 smithi149 ceph-mon[170394]: osdmap e200: 8 total, 8 up, 8 in 2023-09-23T20:26:40.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:40 smithi149 ceph-mon[170394]: failed to encode map e200 with expected crc 2023-09-23T20:26:40.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:40 smithi149 ceph-mon[170394]: failed to encode map e200 with expected crc 2023-09-23T20:26:40.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:40 smithi149 ceph-mon[170394]: failed to encode map e200 with expected crc 2023-09-23T20:26:40.994 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:26:41.003 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:26:41.030 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:26:41.072 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:26:41.091 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:41.132 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:26:41.177 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:26:41.189 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:41.189 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:26:41.190 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:26:41.227 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:26:41.249 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:26:41.249 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:26:41.249 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:26:41.249 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:26:41.249 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:26:41.249 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:26:41.249 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:26:41.249 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:26:41.262 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T20:26:41.290 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:26:41.324 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:41.375 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:26:41.383 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:41.383 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:26:41.448 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:26:41.484 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:26:41.526 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:41.573 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:26:41.585 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:41.585 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:26:41.686 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:26:41.730 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:41 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3015794880' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:26:41.730 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:41 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 4379/53913 objects degraded (8.122%), 5 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T20:26:41.731 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:41.816 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:26:41.826 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:41.826 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:26:41.859 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:41 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3015794880' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:26:41.859 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:41 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 4379/53913 objects degraded (8.122%), 5 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T20:26:41.867 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:41 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3015794880' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:26:41.867 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:41 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 4379/53913 objects degraded (8.122%), 5 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T20:26:41.927 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:26:41.988 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:42.065 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:26:42.071 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:42.071 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:26:42.172 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:26:42.229 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:42.295 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:26:42.298 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:42.299 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:26:42.399 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:26:42.470 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:42.671 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:26:42.679 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:42.679 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:26:42.736 INFO:teuthology.orchestra.run.smithi018.stdout:ERROR: (22) Invalid argument 2023-09-23T20:26:42.736 INFO:teuthology.orchestra.run.smithi018.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:26:42.780 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:26:42.813 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:42.864 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:26:42.867 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:42.867 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:26:42.968 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:26:43.034 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:43.118 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:26:43.125 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:43.126 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:26:43.126 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:42 smithi149 ceph-mon[170394]: pgmap v434: 109 pgs: 1 active+recovering+undersized+remapped, 103 active+clean, 5 active+recovery_wait+undersized+degraded+remapped; 911 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 899 op/s; 4379/43614 objects degraded (10.040%); 0 B/s, 64 objects/s recovering 2023-09-23T20:26:43.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:42 smithi018 ceph-mon[179647]: pgmap v434: 109 pgs: 1 active+recovering+undersized+remapped, 103 active+clean, 5 active+recovery_wait+undersized+degraded+remapped; 911 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 899 op/s; 4379/43614 objects degraded (10.040%); 0 B/s, 64 objects/s recovering 2023-09-23T20:26:43.144 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:42 smithi018 ceph-mon[184144]: pgmap v434: 109 pgs: 1 active+recovering+undersized+remapped, 103 active+clean, 5 active+recovery_wait+undersized+degraded+remapped; 911 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 899 op/s; 4379/43614 objects degraded (10.040%); 0 B/s, 64 objects/s recovering 2023-09-23T20:26:43.227 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:26:43.262 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:43.313 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:26:43.320 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:43.321 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:26:43.344 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T20:26:43.421 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:26:43.473 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:43.520 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:26:43.523 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:43.523 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:26:43.615 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:26:43.624 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:26:43.654 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:43.691 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:26:43.697 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:43.698 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:26:43.798 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:26:43.824 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:43.857 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:26:43.860 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:43.860 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:26:43.961 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:26:43.989 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:44.025 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:26:44.034 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:44.035 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:26:44.135 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:26:44.209 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:43 smithi018 ceph-mon[184144]: osdmap e201: 8 total, 8 up, 8 in 2023-09-23T20:26:44.209 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:43 smithi018 ceph-mon[184144]: failed to encode map e201 with expected crc 2023-09-23T20:26:44.209 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:43 smithi018 ceph-mon[184144]: failed to encode map e201 with expected crc 2023-09-23T20:26:44.210 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:43 smithi018 ceph-mon[179647]: osdmap e201: 8 total, 8 up, 8 in 2023-09-23T20:26:44.210 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:43 smithi018 ceph-mon[179647]: failed to encode map e201 with expected crc 2023-09-23T20:26:44.210 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:43 smithi018 ceph-mon[179647]: failed to encode map e201 with expected crc 2023-09-23T20:26:44.214 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:44.219 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:43 smithi149 ceph-mon[170394]: osdmap e201: 8 total, 8 up, 8 in 2023-09-23T20:26:44.219 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:43 smithi149 ceph-mon[170394]: failed to encode map e201 with expected crc 2023-09-23T20:26:44.219 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:43 smithi149 ceph-mon[170394]: failed to encode map e201 with expected crc 2023-09-23T20:26:44.401 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:26:44.410 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:44.410 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:26:44.511 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:26:44.598 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:44.684 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:26:44.696 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:44.696 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:26:44.755 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:26:44.764 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:26:44.797 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:26:44.840 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:44.864 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:26:44.881 INFO:teuthology.orchestra.run.smithi018.stderr:dumped all 2023-09-23T20:26:44.885 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:26:44.894 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:44.894 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:26:44.995 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:26:45.024 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:45.081 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:26:45.089 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:44 smithi149 ceph-mon[170394]: pgmap v436: 109 pgs: 1 peering, 103 active+clean, 5 active+recovery_wait+undersized+degraded+remapped; 853 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 784 op/s; 4379/40800 objects degraded (10.733%); 0 B/s, 139 objects/s recovering 2023-09-23T20:26:45.089 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:44 smithi149 ceph-mon[170394]: failed to encode map e202 with expected crc 2023-09-23T20:26:45.089 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:44 smithi149 ceph-mon[170394]: osdmap e202: 8 total, 8 up, 8 in 2023-09-23T20:26:45.089 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:44 smithi149 ceph-mon[170394]: failed to encode map e202 with expected crc 2023-09-23T20:26:45.090 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:44 smithi149 ceph-mon[170394]: failed to encode map e202 with expected crc 2023-09-23T20:26:45.090 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:44 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:26:45.090 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:45.091 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:26:45.147 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:44 smithi018 ceph-mon[179647]: pgmap v436: 109 pgs: 1 peering, 103 active+clean, 5 active+recovery_wait+undersized+degraded+remapped; 853 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 784 op/s; 4379/40800 objects degraded (10.733%); 0 B/s, 139 objects/s recovering 2023-09-23T20:26:45.147 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:44 smithi018 ceph-mon[179647]: failed to encode map e202 with expected crc 2023-09-23T20:26:45.147 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:44 smithi018 ceph-mon[179647]: osdmap e202: 8 total, 8 up, 8 in 2023-09-23T20:26:45.148 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:44 smithi018 ceph-mon[179647]: failed to encode map e202 with expected crc 2023-09-23T20:26:45.148 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:44 smithi018 ceph-mon[179647]: failed to encode map e202 with expected crc 2023-09-23T20:26:45.148 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:44 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:26:45.148 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:26:44 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:26:44] "GET /metrics HTTP/1.1" 200 36455 "" "Prometheus/2.43.0" 2023-09-23T20:26:45.148 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:44 smithi018 ceph-mon[184144]: pgmap v436: 109 pgs: 1 peering, 103 active+clean, 5 active+recovery_wait+undersized+degraded+remapped; 853 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 784 op/s; 4379/40800 objects degraded (10.733%); 0 B/s, 139 objects/s recovering 2023-09-23T20:26:45.148 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:44 smithi018 ceph-mon[184144]: failed to encode map e202 with expected crc 2023-09-23T20:26:45.149 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:44 smithi018 ceph-mon[184144]: osdmap e202: 8 total, 8 up, 8 in 2023-09-23T20:26:45.149 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:44 smithi018 ceph-mon[184144]: failed to encode map e202 with expected crc 2023-09-23T20:26:45.149 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:44 smithi018 ceph-mon[184144]: failed to encode map e202 with expected crc 2023-09-23T20:26:45.149 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:44 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:26:45.191 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:26:45.278 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:45.471 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:26:45.481 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:45.481 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:26:45.551 INFO:teuthology.orchestra.run.smithi018.stdout:{"pg_ready":true,"pg_map":{"version":437,"stamp":"2023-09-23T20:26:43.969694+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":894179275,"num_objects":13600,"num_object_clones":0,"num_object_copies":40800,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4379,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":13600,"num_whiteouts":0,"num_read":6496,"num_read_kb":14137,"num_write":186052,"num_write_kb":5865958,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":25523,"num_bytes_recovered":1204947884,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":176621,"ondisk_log_size":176621,"up":327,"acting":322,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":348,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":4503648,"kb_used_data":3522336,"kb_used_omap":30,"kb_used_meta":981153,"kb_avail":745457568,"statfs":{"total":767960285184,"available":763348549632,"internally_reserved":0,"allocated":3606872064,"data_stored":3600039945,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31263,"internal_metadata":1004701153},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,1,3,3,9,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":9,"apply_latency_ms":9,"commit_latency_ns":9000000,"apply_latency_ns":9000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-286457856,"num_objects":-4371,"num_object_clones":0,"num_object_copies":-13113,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-4371,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4371,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":444,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"5.355667"},"pg_stats":[{"pgid":"3.1a","version":"69'1","reported_seq":197,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T20:26:38.630852+0000","last_change":"2023-09-23T20:25:39.632437+0000","last_active":"2023-09-23T20:26:38.630852+0000","last_peered":"2023-09-23T20:26:38.630852+0000","last_clean":"2023-09-23T20:26:38.630852+0000","last_became_active":"2023-09-23T20:25:39.632074+0000","last_became_peered":"2023-09-23T20:25:39.632074+0000","last_unstale":"2023-09-23T20:26:38.630852+0000","last_undegraded":"2023-09-23T20:26:38.630852+0000","last_fullsized":"2023-09-23T20:26:38.630852+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:32:13.543799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":46,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-09-23T20:26:39.675087+0000","last_change":"2023-09-23T20:25:02.932223+0000","last_active":"2023-09-23T20:26:39.675087+0000","last_peered":"2023-09-23T20:26:39.675087+0000","last_clean":"2023-09-23T20:26:39.675087+0000","last_became_active":"2023-09-23T20:25:02.931024+0000","last_became_peered":"2023-09-23T20:25:02.931024+0000","last_unstale":"2023-09-23T20:26:39.675087+0000","last_undegraded":"2023-09-23T20:26:39.675087+0000","last_fullsized":"2023-09-23T20:26:39.675087+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:16:00.586211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":157,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T20:26:38.630032+0000","last_change":"2023-09-23T20:25:02.934543+0000","last_active":"2023-09-23T20:26:38.630032+0000","last_peered":"2023-09-23T20:26:38.630032+0000","last_clean":"2023-09-23T20:26:38.630032+0000","last_became_active":"2023-09-23T20:25:02.934178+0000","last_became_peered":"2023-09-23T20:25:02.934178+0000","last_unstale":"2023-09-23T20:26:38.630032+0000","last_undegraded":"2023-09-23T20:26:38.630032+0000","last_fullsized":"2023-09-23T20:26:38.630032+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:27:38.453052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":163,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T20:26:38.654909+0000","last_change":"2023-09-23T20:25:02.957185+0000","last_active":"2023-09-23T20:26:38.654909+0000","last_peered":"2023-09-23T20:26:38.654909+0000","last_clean":"2023-09-23T20:26:38.654909+0000","last_became_active":"2023-09-23T20:25:02.956689+0000","last_became_peered":"2023-09-23T20:25:02.956689+0000","last_unstale":"2023-09-23T20:26:38.654909+0000","last_undegraded":"2023-09-23T20:26:38.654909+0000","last_fullsized":"2023-09-23T20:26:38.654909+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:04:28.524015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":46,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-09-23T20:26:39.674472+0000","last_change":"2023-09-23T20:25:02.932054+0000","last_active":"2023-09-23T20:26:39.674472+0000","last_peered":"2023-09-23T20:26:39.674472+0000","last_clean":"2023-09-23T20:26:39.674472+0000","last_became_active":"2023-09-23T20:25:02.931309+0000","last_became_peered":"2023-09-23T20:25:02.931309+0000","last_unstale":"2023-09-23T20:26:39.674472+0000","last_undegraded":"2023-09-23T20:26:39.674472+0000","last_fullsized":"2023-09-23T20:26:39.674472+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:57:53.378499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":159,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T20:26:38.635363+0000","last_change":"2023-09-23T20:20:46.937089+0000","last_active":"2023-09-23T20:26:38.635363+0000","last_peered":"2023-09-23T20:26:38.635363+0000","last_clean":"2023-09-23T20:26:38.635363+0000","last_became_active":"2023-09-23T20:20:46.936831+0000","last_became_peered":"2023-09-23T20:20:46.936831+0000","last_unstale":"2023-09-23T20:26:38.635363+0000","last_undegraded":"2023-09-23T20:26:38.635363+0000","last_fullsized":"2023-09-23T20:26:38.635363+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:34:56.747010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.11","version":"200'7334","reported_seq":10073,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-09-23T20:26:40.440447+0000","last_change":"2023-09-23T20:23:31.161216+0000","last_active":"2023-09-23T20:26:40.440447+0000","last_peered":"2023-09-23T20:26:40.440447+0000","last_clean":"2023-09-23T20:26:40.440447+0000","last_became_active":"2023-09-23T20:23:31.146532+0000","last_became_peered":"2023-09-23T20:23:31.146532+0000","last_unstale":"2023-09-23T20:26:40.440447+0000","last_undegraded":"2023-09-23T20:26:40.440447+0000","last_fullsized":"2023-09-23T20:26:40.440447+0000","mapping_epoch":130,"log_start":"0'0","ondisk_log_start":"0'0","created":118,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":5,"last_scrub":"119'1977","last_scrub_stamp":"2023-09-23T20:23:12.353414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:23:12.353414+0000","objects_scrubbed":1018,"log_size":7334,"log_dups_size":0,"ondisk_log_size":7334,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:28:29.630274+0000","scrub_duration":0.26235630100000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":27721728,"num_objects":423,"num_object_clones":0,"num_object_copies":1269,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":423,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5367,"num_write_kb":152704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1282,"num_bytes_recovered":84017152,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":161,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T20:26:38.655361+0000","last_change":"2023-09-23T20:25:02.969432+0000","last_active":"2023-09-23T20:26:38.655361+0000","last_peered":"2023-09-23T20:26:38.655361+0000","last_clean":"2023-09-23T20:26:38.655361+0000","last_became_active":"2023-09-23T20:25:02.969118+0000","last_became_peered":"2023-09-23T20:25:02.969118+0000","last_unstale":"2023-09-23T20:26:38.655361+0000","last_undegraded":"2023-09-23T20:26:38.655361+0000","last_fullsized":"2023-09-23T20:26:38.655361+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:02:22.453831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":46,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-09-23T20:26:39.674380+0000","last_change":"2023-09-23T20:25:02.932225+0000","last_active":"2023-09-23T20:26:39.674380+0000","last_peered":"2023-09-23T20:26:39.674380+0000","last_clean":"2023-09-23T20:26:39.674380+0000","last_became_active":"2023-09-23T20:25:02.931746+0000","last_became_peered":"2023-09-23T20:25:02.931746+0000","last_unstale":"2023-09-23T20:26:39.674380+0000","last_undegraded":"2023-09-23T20:26:39.674380+0000","last_fullsized":"2023-09-23T20:26:39.674380+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:57:24.470179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.10","version":"200'7207","reported_seq":10210,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-09-23T20:26:40.342423+0000","last_change":"2023-09-23T20:23:24.880162+0000","last_active":"2023-09-23T20:26:40.342423+0000","last_peered":"2023-09-23T20:26:40.342423+0000","last_clean":"2023-09-23T20:26:40.342423+0000","last_became_active":"2023-09-23T20:23:24.867240+0000","last_became_peered":"2023-09-23T20:23:24.867240+0000","last_unstale":"2023-09-23T20:26:40.342423+0000","last_undegraded":"2023-09-23T20:26:40.342423+0000","last_fullsized":"2023-09-23T20:26:40.342423+0000","mapping_epoch":128,"log_start":"116'1248","ondisk_log_start":"116'1248","created":118,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":5,"last_scrub":"119'2122","last_scrub_stamp":"2023-09-23T20:23:15.108101+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:23:15.108101+0000","objects_scrubbed":1199,"log_size":5959,"log_dups_size":0,"ondisk_log_size":5959,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:10:57.070009+0000","scrub_duration":0.30476455699999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":27000832,"num_objects":412,"num_object_clones":0,"num_object_copies":1236,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":412,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5089,"num_write_kb":137664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1255,"num_bytes_recovered":82247680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":159,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T20:26:38.627729+0000","last_change":"2023-09-23T20:20:46.950281+0000","last_active":"2023-09-23T20:26:38.627729+0000","last_peered":"2023-09-23T20:26:38.627729+0000","last_clean":"2023-09-23T20:26:38.627729+0000","last_became_active":"2023-09-23T20:20:46.949892+0000","last_became_peered":"2023-09-23T20:20:46.949892+0000","last_unstale":"2023-09-23T20:26:38.627729+0000","last_undegraded":"2023-09-23T20:26:38.627729+0000","last_fullsized":"2023-09-23T20:26:38.627729+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:28.763832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":46,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-09-23T20:26:39.675015+0000","last_change":"2023-09-23T20:25:02.947669+0000","last_active":"2023-09-23T20:26:39.675015+0000","last_peered":"2023-09-23T20:26:39.675015+0000","last_clean":"2023-09-23T20:26:39.675015+0000","last_became_active":"2023-09-23T20:25:02.947368+0000","last_became_peered":"2023-09-23T20:25:02.947368+0000","last_unstale":"2023-09-23T20:26:39.675015+0000","last_undegraded":"2023-09-23T20:26:39.675015+0000","last_fullsized":"2023-09-23T20:26:39.675015+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:52:53.813761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.13","version":"200'7224","reported_seq":11484,"reported_epoch":200,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:26:39.842474+0000","last_change":"2023-09-23T20:25:40.848149+0000","last_active":"2023-09-23T20:26:39.842474+0000","last_peered":"2023-09-23T20:26:39.842474+0000","last_clean":"2023-09-23T20:24:36.774911+0000","last_became_active":"2023-09-23T20:25:02.945089+0000","last_became_peered":"2023-09-23T20:25:02.945089+0000","last_unstale":"2023-09-23T20:26:39.842474+0000","last_undegraded":"2023-09-23T20:25:40.847992+0000","last_fullsized":"2023-09-23T20:25:02.906341+0000","mapping_epoch":165,"log_start":"116'1347","ondisk_log_start":"116'1347","created":118,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:22:51.846999+0000","objects_scrubbed":0,"log_size":5877,"log_dups_size":0,"ondisk_log_size":5877,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":27754496,"num_objects":424,"num_object_clones":0,"num_object_copies":1272,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":270,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":424,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6322,"num_write_kb":215840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1401,"num_bytes_recovered":85852160,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[0,1],"avail_no_missing":["0","1"],"object_location_counts":[{"shards":"0,1","objects":424}],"blocked_by":[],"up_primary":3,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":195,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T20:26:38.629370+0000","last_change":"2023-09-23T20:25:03.038511+0000","last_active":"2023-09-23T20:26:38.629370+0000","last_peered":"2023-09-23T20:26:38.629370+0000","last_clean":"2023-09-23T20:26:38.629370+0000","last_became_active":"2023-09-23T20:25:02.976811+0000","last_became_peered":"2023-09-23T20:25:02.976811+0000","last_unstale":"2023-09-23T20:26:38.629370+0000","last_undegraded":"2023-09-23T20:26:38.629370+0000","last_fullsized":"2023-09-23T20:26:38.629370+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:28:06.605731+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":158,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T20:26:38.647437+0000","last_change":"2023-09-23T20:25:39.646452+0000","last_active":"2023-09-23T20:26:38.647437+0000","last_peered":"2023-09-23T20:26:38.647437+0000","last_clean":"2023-09-23T20:26:38.647437+0000","last_became_active":"2023-09-23T20:25:39.646175+0000","last_became_peered":"2023-09-23T20:25:39.646175+0000","last_unstale":"2023-09-23T20:26:38.647437+0000","last_undegraded":"2023-09-23T20:26:38.647437+0000","last_fullsized":"2023-09-23T20:26:38.647437+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:33:02.099951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.12","version":"200'7052","reported_seq":11151,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-09-23T20:26:39.922023+0000","last_change":"2023-09-23T20:26:14.067112+0000","last_active":"2023-09-23T20:26:39.922023+0000","last_peered":"2023-09-23T20:26:39.922023+0000","last_clean":"2023-09-23T20:26:39.922023+0000","last_became_active":"2023-09-23T20:26:14.046688+0000","last_became_peered":"2023-09-23T20:26:14.046688+0000","last_unstale":"2023-09-23T20:26:39.922023+0000","last_undegraded":"2023-09-23T20:26:39.922023+0000","last_fullsized":"2023-09-23T20:26:39.922023+0000","mapping_epoch":188,"log_start":"0'0","ondisk_log_start":"0'0","created":118,"last_epoch_clean":189,"parent":"0.0","parent_split_bits":5,"last_scrub":"119'2087","last_scrub_stamp":"2023-09-23T20:23:15.737284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:23:15.737284+0000","objects_scrubbed":1166,"log_size":7052,"log_dups_size":0,"ondisk_log_size":7052,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:39:04.257684+0000","scrub_duration":0.27092271000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26738688,"num_objects":408,"num_object_clones":0,"num_object_copies":1224,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":408,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4978,"num_write_kb":135040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1590,"num_bytes_recovered":80216064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":164,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T20:26:38.631275+0000","last_change":"2023-09-23T20:12:43.750217+0000","last_active":"2023-09-23T20:26:38.631275+0000","last_peered":"2023-09-23T20:26:38.631275+0000","last_clean":"2023-09-23T20:26:38.631275+0000","last_became_active":"2023-09-23T20:12:43.749929+0000","last_became_peered":"2023-09-23T20:12:43.749929+0000","last_unstale":"2023-09-23T20:26:38.631275+0000","last_undegraded":"2023-09-23T20:26:38.631275+0000","last_fullsized":"2023-09-23T20:26:38.631275+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:50:28.470211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":158,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T20:26:38.630949+0000","last_change":"2023-09-23T20:20:46.939938+0000","last_active":"2023-09-23T20:26:38.630949+0000","last_peered":"2023-09-23T20:26:38.630949+0000","last_clean":"2023-09-23T20:26:38.630949+0000","last_became_active":"2023-09-23T20:20:46.938777+0000","last_became_peered":"2023-09-23T20:20:46.938777+0000","last_unstale":"2023-09-23T20:26:38.630949+0000","last_undegraded":"2023-09-23T20:26:38.630949+0000","last_fullsized":"2023-09-23T20:26:38.630949+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:45:10.329438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.15","version":"200'7028","reported_seq":10612,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-09-23T20:26:40.495388+0000","last_change":"2023-09-23T20:24:36.924738+0000","last_active":"2023-09-23T20:26:40.495388+0000","last_peered":"2023-09-23T20:26:40.495388+0000","last_clean":"2023-09-23T20:26:40.495388+0000","last_became_active":"2023-09-23T20:23:54.922070+0000","last_became_peered":"2023-09-23T20:23:54.922070+0000","last_unstale":"2023-09-23T20:26:40.495388+0000","last_undegraded":"2023-09-23T20:26:40.495388+0000","last_fullsized":"2023-09-23T20:26:40.495388+0000","mapping_epoch":143,"log_start":"0'0","ondisk_log_start":"0'0","created":118,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:22:51.846999+0000","objects_scrubbed":0,"log_size":7028,"log_dups_size":0,"ondisk_log_size":7028,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":29687808,"num_objects":453,"num_object_clones":0,"num_object_copies":1359,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":453,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6145,"num_write_kb":211136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1727,"num_bytes_recovered":113180672,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":194,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T20:26:38.628579+0000","last_change":"2023-09-23T20:25:39.654026+0000","last_active":"2023-09-23T20:26:38.628579+0000","last_peered":"2023-09-23T20:26:38.628579+0000","last_clean":"2023-09-23T20:26:38.628579+0000","last_became_active":"2023-09-23T20:25:39.626849+0000","last_became_peered":"2023-09-23T20:25:39.626849+0000","last_unstale":"2023-09-23T20:26:38.628579+0000","last_undegraded":"2023-09-23T20:26:38.628579+0000","last_fullsized":"2023-09-23T20:26:38.628579+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:22:53.957630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":46,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-09-23T20:26:39.682050+0000","last_change":"2023-09-23T20:25:02.962047+0000","last_active":"2023-09-23T20:26:39.682050+0000","last_peered":"2023-09-23T20:26:39.682050+0000","last_clean":"2023-09-23T20:26:39.682050+0000","last_became_active":"2023-09-23T20:25:02.961523+0000","last_became_peered":"2023-09-23T20:25:02.961523+0000","last_unstale":"2023-09-23T20:26:39.682050+0000","last_undegraded":"2023-09-23T20:26:39.682050+0000","last_fullsized":"2023-09-23T20:26:39.682050+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:18:11.830822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.14","version":"200'7103","reported_seq":13543,"reported_epoch":201,"state":"peering","last_fresh":"2023-09-23T20:26:42.981298+0000","last_change":"2023-09-23T20:26:42.981400+0000","last_active":"2023-09-23T20:26:42.981298+0000","last_peered":"2023-09-23T20:26:42.935498+0000","last_clean":"2023-09-23T20:24:36.883442+0000","last_became_active":"2023-09-23T20:25:40.725928+0000","last_became_peered":"2023-09-23T20:25:40.725928+0000","last_unstale":"2023-09-23T20:26:42.981298+0000","last_undegraded":"2023-09-23T20:26:42.981298+0000","last_fullsized":"2023-09-23T20:26:42.981298+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":118,"last_epoch_clean":142,"parent":"0.0","parent_split_bits":5,"last_scrub":"142'3061","last_scrub_stamp":"2023-09-23T20:23:51.058360+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:23:51.058360+0000","objects_scrubbed":2161,"log_size":7103,"log_dups_size":0,"ondisk_log_size":7103,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:50:31.351984+0000","scrub_duration":0.44581189700000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":24707072,"num_objects":377,"num_object_clones":0,"num_object_copies":1131,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":377,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4054,"num_write_kb":72640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1218,"num_bytes_recovered":9175040,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[1,2,7],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":161,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T20:26:38.654310+0000","last_change":"2023-09-23T20:25:02.953993+0000","last_active":"2023-09-23T20:26:38.654310+0000","last_peered":"2023-09-23T20:26:38.654310+0000","last_clean":"2023-09-23T20:26:38.654310+0000","last_became_active":"2023-09-23T20:25:02.953672+0000","last_became_peered":"2023-09-23T20:25:02.953672+0000","last_unstale":"2023-09-23T20:26:38.654310+0000","last_undegraded":"2023-09-23T20:26:38.654310+0000","last_fullsized":"2023-09-23T20:26:38.654310+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:26:08.603577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":178,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T20:26:38.627535+0000","last_change":"2023-09-23T20:25:02.949584+0000","last_active":"2023-09-23T20:26:38.627535+0000","last_peered":"2023-09-23T20:26:38.627535+0000","last_clean":"2023-09-23T20:26:38.627535+0000","last_became_active":"2023-09-23T20:25:02.949132+0000","last_became_peered":"2023-09-23T20:25:02.949132+0000","last_unstale":"2023-09-23T20:26:38.627535+0000","last_undegraded":"2023-09-23T20:26:38.627535+0000","last_fullsized":"2023-09-23T20:26:38.627535+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:34:20.509823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.17","version":"201'7132","reported_seq":13686,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-09-23T20:26:42.987638+0000","last_change":"2023-09-23T20:26:20.168261+0000","last_active":"2023-09-23T20:26:42.987638+0000","last_peered":"2023-09-23T20:26:42.987638+0000","last_clean":"2023-09-23T20:26:42.987638+0000","last_became_active":"2023-09-23T20:26:20.147495+0000","last_became_peered":"2023-09-23T20:26:20.147495+0000","last_unstale":"2023-09-23T20:26:42.987638+0000","last_undegraded":"2023-09-23T20:26:42.987638+0000","last_fullsized":"2023-09-23T20:26:42.987638+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":118,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:22:51.846999+0000","objects_scrubbed":0,"log_size":7132,"log_dups_size":0,"ondisk_log_size":7132,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":24510464,"num_objects":374,"num_object_clones":0,"num_object_copies":1122,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":374,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6214,"num_write_kb":210816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2467,"num_bytes_recovered":161677312,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":143,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T20:26:38.628570+0000","last_change":"2023-09-23T20:12:43.749139+0000","last_active":"2023-09-23T20:26:38.628570+0000","last_peered":"2023-09-23T20:26:38.628570+0000","last_clean":"2023-09-23T20:26:38.628570+0000","last_became_active":"2023-09-23T20:12:43.748854+0000","last_became_peered":"2023-09-23T20:12:43.748854+0000","last_unstale":"2023-09-23T20:26:38.628570+0000","last_undegraded":"2023-09-23T20:26:38.628570+0000","last_fullsized":"2023-09-23T20:26:38.628570+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:57:11.630712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":171,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T20:26:38.654837+0000","last_change":"2023-09-23T20:25:02.957010+0000","last_active":"2023-09-23T20:26:38.654837+0000","last_peered":"2023-09-23T20:26:38.654837+0000","last_clean":"2023-09-23T20:26:38.654837+0000","last_became_active":"2023-09-23T20:25:02.956267+0000","last_became_peered":"2023-09-23T20:25:02.956267+0000","last_unstale":"2023-09-23T20:26:38.654837+0000","last_undegraded":"2023-09-23T20:26:38.654837+0000","last_fullsized":"2023-09-23T20:26:38.654837+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:10:55.299422+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.16","version":"200'7265","reported_seq":14724,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-09-23T20:26:40.386475+0000","last_change":"2023-09-23T20:26:16.974059+0000","last_active":"2023-09-23T20:26:40.386475+0000","last_peered":"2023-09-23T20:26:40.386475+0000","last_clean":"2023-09-23T20:26:40.386475+0000","last_became_active":"2023-09-23T20:26:16.954812+0000","last_became_peered":"2023-09-23T20:26:16.954812+0000","last_unstale":"2023-09-23T20:26:40.386475+0000","last_undegraded":"2023-09-23T20:26:40.386475+0000","last_fullsized":"2023-09-23T20:26:40.386475+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":118,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":5,"last_scrub":"119'2163","last_scrub_stamp":"2023-09-23T20:23:15.241911+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:23:15.241911+0000","objects_scrubbed":1192,"log_size":7265,"log_dups_size":0,"ondisk_log_size":7265,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:52:46.936495+0000","scrub_duration":0.33323710899999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":27066368,"num_objects":413,"num_object_clones":0,"num_object_copies":1239,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":413,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5113,"num_write_kb":138688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2720,"num_bytes_recovered":154599424,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":168,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T20:26:38.634347+0000","last_change":"2023-09-23T20:20:14.881009+0000","last_active":"2023-09-23T20:26:38.634347+0000","last_peered":"2023-09-23T20:26:38.634347+0000","last_clean":"2023-09-23T20:26:38.634347+0000","last_became_active":"2023-09-23T20:20:14.880685+0000","last_became_peered":"2023-09-23T20:20:14.880685+0000","last_unstale":"2023-09-23T20:26:38.634347+0000","last_undegraded":"2023-09-23T20:26:38.634347+0000","last_fullsized":"2023-09-23T20:26:38.634347+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:28.010434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":46,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-09-23T20:26:39.675160+0000","last_change":"2023-09-23T20:25:02.977462+0000","last_active":"2023-09-23T20:26:39.675160+0000","last_peered":"2023-09-23T20:26:39.675160+0000","last_clean":"2023-09-23T20:26:39.675160+0000","last_became_active":"2023-09-23T20:25:02.969910+0000","last_became_peered":"2023-09-23T20:25:02.969910+0000","last_unstale":"2023-09-23T20:26:39.675160+0000","last_undegraded":"2023-09-23T20:26:39.675160+0000","last_fullsized":"2023-09-23T20:26:39.675160+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:22:55.644955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.9","version":"200'7015","reported_seq":7123,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-09-23T20:26:40.430119+0000","last_change":"2023-09-23T20:23:10.973039+0000","last_active":"2023-09-23T20:26:40.430119+0000","last_peered":"2023-09-23T20:26:40.430119+0000","last_clean":"2023-09-23T20:26:40.430119+0000","last_became_active":"2023-09-23T20:23:10.971930+0000","last_became_peered":"2023-09-23T20:23:10.971930+0000","last_unstale":"2023-09-23T20:26:40.430119+0000","last_undegraded":"2023-09-23T20:26:40.430119+0000","last_fullsized":"2023-09-23T20:26:40.430119+0000","mapping_epoch":111,"log_start":"115'940","ondisk_log_start":"115'940","created":111,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:22:51.846999+0000","objects_scrubbed":0,"log_size":6075,"log_dups_size":0,"ondisk_log_size":6075,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":27688960,"num_objects":422,"num_object_clones":0,"num_object_copies":1266,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":422,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6132,"num_write_kb":209760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":161,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T20:26:38.629109+0000","last_change":"2023-09-23T20:20:14.883283+0000","last_active":"2023-09-23T20:26:38.629109+0000","last_peered":"2023-09-23T20:26:38.629109+0000","last_clean":"2023-09-23T20:26:38.629109+0000","last_became_active":"2023-09-23T20:20:14.882806+0000","last_became_peered":"2023-09-23T20:20:14.882806+0000","last_unstale":"2023-09-23T20:26:38.629109+0000","last_undegraded":"2023-09-23T20:26:38.629109+0000","last_fullsized":"2023-09-23T20:26:38.629109+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:23:35.295701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":141,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T20:26:38.626905+0000","last_change":"2023-09-23T20:12:45.179211+0000","last_active":"2023-09-23T20:26:38.626905+0000","last_peered":"2023-09-23T20:26:38.626905+0000","last_clean":"2023-09-23T20:26:38.626905+0000","last_became_active":"2023-09-23T20:12:45.178966+0000","last_became_peered":"2023-09-23T20:12:45.178966+0000","last_unstale":"2023-09-23T20:26:38.626905+0000","last_undegraded":"2023-09-23T20:26:38.626905+0000","last_fullsized":"2023-09-23T20:26:38.626905+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:58.902140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.8","version":"200'7168","reported_seq":7331,"reported_epoch":200,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:26:39.825059+0000","last_change":"2023-09-23T20:25:39.718803+0000","last_active":"2023-09-23T20:26:39.825059+0000","last_peered":"2023-09-23T20:26:39.825059+0000","last_clean":"2023-09-23T20:24:51.985226+0000","last_became_active":"2023-09-23T20:25:39.704786+0000","last_became_peered":"2023-09-23T20:25:39.704786+0000","last_unstale":"2023-09-23T20:26:39.825059+0000","last_undegraded":"2023-09-23T20:25:39.625409+0000","last_fullsized":"2023-09-23T20:25:39.610997+0000","mapping_epoch":176,"log_start":"115'1198","ondisk_log_start":"115'1198","created":111,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"120'2144","last_scrub_stamp":"2023-09-23T20:23:17.129360+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:23:17.129360+0000","objects_scrubbed":1209,"log_size":5970,"log_dups_size":0,"ondisk_log_size":5970,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:22:27.232625+0000","scrub_duration":0.17393181099999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":28770304,"num_objects":439,"num_object_clones":0,"num_object_copies":1317,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":868,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":439,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5024,"num_write_kb":136128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[0,1],"avail_no_missing":["0","1"],"object_location_counts":[{"shards":"0,1","objects":439}],"blocked_by":[],"up_primary":2,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":162,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T20:26:38.628645+0000","last_change":"2023-09-23T20:20:46.950784+0000","last_active":"2023-09-23T20:26:38.628645+0000","last_peered":"2023-09-23T20:26:38.628645+0000","last_clean":"2023-09-23T20:26:38.628645+0000","last_became_active":"2023-09-23T20:20:46.950055+0000","last_became_peered":"2023-09-23T20:20:46.950055+0000","last_unstale":"2023-09-23T20:26:38.628645+0000","last_undegraded":"2023-09-23T20:26:38.628645+0000","last_fullsized":"2023-09-23T20:26:38.628645+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:23:07.645427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":195,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T20:26:38.630780+0000","last_change":"2023-09-23T20:25:02.943743+0000","last_active":"2023-09-23T20:26:38.630780+0000","last_peered":"2023-09-23T20:26:38.630780+0000","last_clean":"2023-09-23T20:26:38.630780+0000","last_became_active":"2023-09-23T20:25:02.943279+0000","last_became_peered":"2023-09-23T20:25:02.943279+0000","last_unstale":"2023-09-23T20:26:38.630780+0000","last_undegraded":"2023-09-23T20:26:38.630780+0000","last_fullsized":"2023-09-23T20:26:38.630780+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:21:36.128366+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.b","version":"200'12324","reported_seq":12435,"reported_epoch":200,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:26:39.811054+0000","last_change":"2023-09-23T20:25:39.764057+0000","last_active":"2023-09-23T20:26:39.811054+0000","last_peered":"2023-09-23T20:26:39.811054+0000","last_clean":"2023-09-23T20:24:52.031628+0000","last_became_active":"2023-09-23T20:25:39.743845+0000","last_became_peered":"2023-09-23T20:25:39.743845+0000","last_unstale":"2023-09-23T20:26:39.811054+0000","last_undegraded":"2023-09-23T20:25:39.622936+0000","last_fullsized":"2023-09-23T20:25:39.612475+0000","mapping_epoch":176,"log_start":"154'6572","ondisk_log_start":"154'6572","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:22:51.846999+0000","objects_scrubbed":0,"log_size":5752,"log_dups_size":0,"ondisk_log_size":5752,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:56:57.182858+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":59506688,"num_objects":908,"num_object_clones":0,"num_object_copies":2724,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1733,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":908,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12324,"num_write_kb":423424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,5","objects":908}],"blocked_by":[],"up_primary":2,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":144,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T20:26:38.629227+0000","last_change":"2023-09-23T20:12:43.751523+0000","last_active":"2023-09-23T20:26:38.629227+0000","last_peered":"2023-09-23T20:26:38.629227+0000","last_clean":"2023-09-23T20:26:38.629227+0000","last_became_active":"2023-09-23T20:12:43.751257+0000","last_became_peered":"2023-09-23T20:12:43.751257+0000","last_unstale":"2023-09-23T20:26:38.629227+0000","last_undegraded":"2023-09-23T20:26:38.629227+0000","last_fullsized":"2023-09-23T20:26:38.629227+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:50:27.040225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":159,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T20:26:38.634933+0000","last_change":"2023-09-23T20:20:46.934762+0000","last_active":"2023-09-23T20:26:38.634933+0000","last_peered":"2023-09-23T20:26:38.634933+0000","last_clean":"2023-09-23T20:26:38.634933+0000","last_became_active":"2023-09-23T20:20:46.934472+0000","last_became_peered":"2023-09-23T20:20:46.934472+0000","last_unstale":"2023-09-23T20:26:38.634933+0000","last_undegraded":"2023-09-23T20:26:38.634933+0000","last_fullsized":"2023-09-23T20:26:38.634933+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:28:42.708286+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.a","version":"200'12589","reported_seq":21232,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-09-23T20:26:41.957295+0000","last_change":"2023-09-23T20:26:13.071682+0000","last_active":"2023-09-23T20:26:41.957295+0000","last_peered":"2023-09-23T20:26:41.957295+0000","last_clean":"2023-09-23T20:26:41.957295+0000","last_became_active":"2023-09-23T20:26:13.034652+0000","last_became_peered":"2023-09-23T20:26:13.034652+0000","last_unstale":"2023-09-23T20:26:41.957295+0000","last_undegraded":"2023-09-23T20:26:41.957295+0000","last_fullsized":"2023-09-23T20:26:41.957295+0000","mapping_epoch":187,"log_start":"144'4713","ondisk_log_start":"144'4713","created":111,"last_epoch_clean":188,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:22:51.846999+0000","objects_scrubbed":0,"log_size":7876,"log_dups_size":0,"ondisk_log_size":7876,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:05:48.610901+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":54984704,"num_objects":839,"num_object_clones":0,"num_object_copies":2517,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":839,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12589,"num_write_kb":429696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3973,"num_bytes_recovered":115343360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":179,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T20:26:38.627384+0000","last_change":"2023-09-23T20:25:02.947631+0000","last_active":"2023-09-23T20:26:38.627384+0000","last_peered":"2023-09-23T20:26:38.627384+0000","last_clean":"2023-09-23T20:26:38.627384+0000","last_became_active":"2023-09-23T20:25:02.947268+0000","last_became_peered":"2023-09-23T20:25:02.947268+0000","last_unstale":"2023-09-23T20:26:38.627384+0000","last_undegraded":"2023-09-23T20:26:38.627384+0000","last_fullsized":"2023-09-23T20:26:38.627384+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:19:43.643985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":144,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T20:26:38.655741+0000","last_change":"2023-09-23T20:20:46.935968+0000","last_active":"2023-09-23T20:26:38.655741+0000","last_peered":"2023-09-23T20:26:38.655741+0000","last_clean":"2023-09-23T20:26:38.655741+0000","last_became_active":"2023-09-23T20:20:46.935618+0000","last_became_peered":"2023-09-23T20:20:46.935618+0000","last_unstale":"2023-09-23T20:26:38.655741+0000","last_undegraded":"2023-09-23T20:26:38.655741+0000","last_fullsized":"2023-09-23T20:26:38.655741+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:42:59.867176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":552,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T20:26:38.628211+0000","last_change":"2023-09-23T20:25:21.167520+0000","last_active":"2023-09-23T20:26:38.628211+0000","last_peered":"2023-09-23T20:26:38.628211+0000","last_clean":"2023-09-23T20:26:38.628211+0000","last_became_active":"2023-09-23T20:25:21.166811+0000","last_became_peered":"2023-09-23T20:25:21.166811+0000","last_unstale":"2023-09-23T20:26:38.628211+0000","last_undegraded":"2023-09-23T20:26:38.628211+0000","last_fullsized":"2023-09-23T20:26:38.628211+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:09:47.213130+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.d","version":"200'12798","reported_seq":18278,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-09-23T20:26:41.977598+0000","last_change":"2023-09-23T20:26:33.302039+0000","last_active":"2023-09-23T20:26:41.977598+0000","last_peered":"2023-09-23T20:26:41.977598+0000","last_clean":"2023-09-23T20:26:41.977598+0000","last_became_active":"2023-09-23T20:26:33.259615+0000","last_became_peered":"2023-09-23T20:26:33.259615+0000","last_unstale":"2023-09-23T20:26:41.977598+0000","last_undegraded":"2023-09-23T20:26:41.977598+0000","last_fullsized":"2023-09-23T20:26:41.977598+0000","mapping_epoch":197,"log_start":"154'6618","ondisk_log_start":"154'6618","created":111,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:22:51.846999+0000","objects_scrubbed":0,"log_size":6180,"log_dups_size":0,"ondisk_log_size":6180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:43:31.739694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53018648,"num_objects":810,"num_object_clones":0,"num_object_copies":2430,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":810,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":12798,"num_write_kb":435393,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2675,"num_bytes_recovered":61145088,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":50,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-09-23T20:26:39.679893+0000","last_change":"2023-09-23T20:25:03.000957+0000","last_active":"2023-09-23T20:26:39.679893+0000","last_peered":"2023-09-23T20:26:39.679893+0000","last_clean":"2023-09-23T20:26:39.679893+0000","last_became_active":"2023-09-23T20:25:02.939903+0000","last_became_peered":"2023-09-23T20:25:02.939903+0000","last_unstale":"2023-09-23T20:26:39.679893+0000","last_undegraded":"2023-09-23T20:26:39.679893+0000","last_fullsized":"2023-09-23T20:26:39.679893+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:51:14.981461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":178,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T20:26:38.631167+0000","last_change":"2023-09-23T20:25:02.929447+0000","last_active":"2023-09-23T20:26:38.631167+0000","last_peered":"2023-09-23T20:26:38.631167+0000","last_clean":"2023-09-23T20:26:38.631167+0000","last_became_active":"2023-09-23T20:25:02.929173+0000","last_became_peered":"2023-09-23T20:25:02.929173+0000","last_unstale":"2023-09-23T20:26:38.631167+0000","last_undegraded":"2023-09-23T20:26:38.631167+0000","last_fullsized":"2023-09-23T20:26:38.631167+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:32:09.280647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":552,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T20:26:38.628149+0000","last_change":"2023-09-23T20:25:21.166037+0000","last_active":"2023-09-23T20:26:38.628149+0000","last_peered":"2023-09-23T20:26:38.628149+0000","last_clean":"2023-09-23T20:26:38.628149+0000","last_became_active":"2023-09-23T20:25:21.165726+0000","last_became_peered":"2023-09-23T20:25:21.165726+0000","last_unstale":"2023-09-23T20:26:38.628149+0000","last_undegraded":"2023-09-23T20:26:38.628149+0000","last_fullsized":"2023-09-23T20:26:38.628149+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:09:47.213130+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":22,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.c","version":"200'12529","reported_seq":12628,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-09-23T20:26:40.029258+0000","last_change":"2023-09-23T20:22:52.866632+0000","last_active":"2023-09-23T20:26:40.029258+0000","last_peered":"2023-09-23T20:26:40.029258+0000","last_clean":"2023-09-23T20:26:40.029258+0000","last_became_active":"2023-09-23T20:22:52.866356+0000","last_became_peered":"2023-09-23T20:22:52.866356+0000","last_unstale":"2023-09-23T20:26:40.029258+0000","last_undegraded":"2023-09-23T20:26:40.029258+0000","last_fullsized":"2023-09-23T20:26:40.029258+0000","mapping_epoch":111,"log_start":"153'5973","ondisk_log_start":"153'5973","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:22:51.846999+0000","objects_scrubbed":0,"log_size":6556,"log_dups_size":0,"ondisk_log_size":6556,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:10:12.356062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":58654720,"num_objects":895,"num_object_clones":0,"num_object_copies":2685,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":895,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12529,"num_write_kb":429568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":160,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T20:26:38.634798+0000","last_change":"2023-09-23T20:20:46.937163+0000","last_active":"2023-09-23T20:26:38.634798+0000","last_peered":"2023-09-23T20:26:38.634798+0000","last_clean":"2023-09-23T20:26:38.634798+0000","last_became_active":"2023-09-23T20:20:46.936757+0000","last_became_peered":"2023-09-23T20:20:46.936757+0000","last_unstale":"2023-09-23T20:26:38.634798+0000","last_undegraded":"2023-09-23T20:26:38.634798+0000","last_fullsized":"2023-09-23T20:26:38.634798+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:09.401379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":188,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T20:26:38.631214+0000","last_change":"2023-09-23T20:25:39.624636+0000","last_active":"2023-09-23T20:26:38.631214+0000","last_peered":"2023-09-23T20:26:38.631214+0000","last_clean":"2023-09-23T20:26:38.631214+0000","last_became_active":"2023-09-23T20:25:39.624297+0000","last_became_peered":"2023-09-23T20:25:39.624297+0000","last_unstale":"2023-09-23T20:26:38.631214+0000","last_undegraded":"2023-09-23T20:26:38.631214+0000","last_fullsized":"2023-09-23T20:26:38.631214+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:33:47.448737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.f","version":"200'12400","reported_seq":12497,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-09-23T20:26:40.590402+0000","last_change":"2023-09-23T20:22:52.872510+0000","last_active":"2023-09-23T20:26:40.590402+0000","last_peered":"2023-09-23T20:26:40.590402+0000","last_clean":"2023-09-23T20:26:40.590402+0000","last_became_active":"2023-09-23T20:22:52.872179+0000","last_became_peered":"2023-09-23T20:22:52.872179+0000","last_unstale":"2023-09-23T20:26:40.590402+0000","last_undegraded":"2023-09-23T20:26:40.590402+0000","last_fullsized":"2023-09-23T20:26:40.590402+0000","mapping_epoch":111,"log_start":"149'4895","ondisk_log_start":"149'4895","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:22:51.846999+0000","objects_scrubbed":0,"log_size":7505,"log_dups_size":0,"ondisk_log_size":7505,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:53.012272+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":55050240,"num_objects":840,"num_object_clones":0,"num_object_copies":2520,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":840,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12400,"num_write_kb":423680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":179,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T20:26:38.630701+0000","last_change":"2023-09-23T20:25:39.628097+0000","last_active":"2023-09-23T20:26:38.630701+0000","last_peered":"2023-09-23T20:26:38.630701+0000","last_clean":"2023-09-23T20:26:38.630701+0000","last_became_active":"2023-09-23T20:25:39.627806+0000","last_became_peered":"2023-09-23T20:25:39.627806+0000","last_unstale":"2023-09-23T20:26:38.630701+0000","last_undegraded":"2023-09-23T20:26:38.630701+0000","last_fullsized":"2023-09-23T20:26:38.630701+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:53:44.398854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":158,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T20:26:38.631367+0000","last_change":"2023-09-23T20:20:46.939720+0000","last_active":"2023-09-23T20:26:38.631367+0000","last_peered":"2023-09-23T20:26:38.631367+0000","last_clean":"2023-09-23T20:26:38.631367+0000","last_became_active":"2023-09-23T20:20:46.939357+0000","last_became_peered":"2023-09-23T20:20:46.939357+0000","last_unstale":"2023-09-23T20:26:38.631367+0000","last_undegraded":"2023-09-23T20:26:38.631367+0000","last_fullsized":"2023-09-23T20:26:38.631367+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:56.613212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":552,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T20:26:38.628353+0000","last_change":"2023-09-23T20:25:21.167297+0000","last_active":"2023-09-23T20:26:38.628353+0000","last_peered":"2023-09-23T20:26:38.628353+0000","last_clean":"2023-09-23T20:26:38.628353+0000","last_became_active":"2023-09-23T20:25:21.167006+0000","last_became_peered":"2023-09-23T20:25:21.167006+0000","last_unstale":"2023-09-23T20:26:38.628353+0000","last_undegraded":"2023-09-23T20:26:38.628353+0000","last_fullsized":"2023-09-23T20:26:38.628353+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:09:47.213130+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.e","version":"200'12408","reported_seq":12504,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-09-23T20:26:40.611896+0000","last_change":"2023-09-23T20:22:52.871704+0000","last_active":"2023-09-23T20:26:40.611896+0000","last_peered":"2023-09-23T20:26:40.611896+0000","last_clean":"2023-09-23T20:26:40.611896+0000","last_became_active":"2023-09-23T20:22:52.871267+0000","last_became_peered":"2023-09-23T20:22:52.871267+0000","last_unstale":"2023-09-23T20:26:40.611896+0000","last_undegraded":"2023-09-23T20:26:40.611896+0000","last_fullsized":"2023-09-23T20:26:40.611896+0000","mapping_epoch":111,"log_start":"153'6004","ondisk_log_start":"153'6004","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:22:51.846999+0000","objects_scrubbed":0,"log_size":6404,"log_dups_size":0,"ondisk_log_size":6404,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:50:07.619801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":57540608,"num_objects":878,"num_object_clones":0,"num_object_copies":2634,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":878,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12408,"num_write_kb":425152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":46,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-09-23T20:26:39.674554+0000","last_change":"2023-09-23T20:25:02.978173+0000","last_active":"2023-09-23T20:26:39.674554+0000","last_peered":"2023-09-23T20:26:39.674554+0000","last_clean":"2023-09-23T20:26:39.674554+0000","last_became_active":"2023-09-23T20:25:02.977856+0000","last_became_peered":"2023-09-23T20:25:02.977856+0000","last_unstale":"2023-09-23T20:26:39.674554+0000","last_undegraded":"2023-09-23T20:26:39.674554+0000","last_fullsized":"2023-09-23T20:26:39.674554+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:19.861943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":59,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T20:26:38.653281+0000","last_change":"2023-09-23T20:25:02.924911+0000","last_active":"2023-09-23T20:26:38.653281+0000","last_peered":"2023-09-23T20:26:38.653281+0000","last_clean":"2023-09-23T20:26:38.653281+0000","last_became_active":"2023-09-23T20:25:02.924586+0000","last_became_peered":"2023-09-23T20:25:02.924586+0000","last_unstale":"2023-09-23T20:26:38.653281+0000","last_undegraded":"2023-09-23T20:26:38.653281+0000","last_fullsized":"2023-09-23T20:26:38.653281+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:24:44.176766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":552,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T20:26:38.628474+0000","last_change":"2023-09-23T20:25:21.167164+0000","last_active":"2023-09-23T20:26:38.628474+0000","last_peered":"2023-09-23T20:26:38.628474+0000","last_clean":"2023-09-23T20:26:38.628474+0000","last_became_active":"2023-09-23T20:25:21.166869+0000","last_became_peered":"2023-09-23T20:25:21.166869+0000","last_unstale":"2023-09-23T20:26:38.628474+0000","last_undegraded":"2023-09-23T20:26:38.628474+0000","last_fullsized":"2023-09-23T20:26:38.628474+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:09:47.213130+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"86'119","reported_seq":533,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T20:26:38.653743+0000","last_change":"2023-09-23T20:20:46.937071+0000","last_active":"2023-09-23T20:26:38.653743+0000","last_peered":"2023-09-23T20:26:38.653743+0000","last_clean":"2023-09-23T20:26:38.653743+0000","last_became_active":"2023-09-23T20:20:46.935348+0000","last_became_peered":"2023-09-23T20:20:46.935348+0000","last_unstale":"2023-09-23T20:26:38.653743+0000","last_undegraded":"2023-09-23T20:26:38.653743+0000","last_fullsized":"2023-09-23T20:26:38.653743+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":119,"log_dups_size":0,"ondisk_log_size":119,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:09.829030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032705499999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":523,"num_read_kb":3503,"num_write":136,"num_write_kb":3128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":216,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T20:26:38.627459+0000","last_change":"2023-09-23T20:25:39.637160+0000","last_active":"2023-09-23T20:26:38.627459+0000","last_peered":"2023-09-23T20:26:38.627459+0000","last_clean":"2023-09-23T20:26:38.627459+0000","last_became_active":"2023-09-23T20:25:39.636860+0000","last_became_peered":"2023-09-23T20:25:39.636860+0000","last_unstale":"2023-09-23T20:26:38.627459+0000","last_undegraded":"2023-09-23T20:26:38.627459+0000","last_fullsized":"2023-09-23T20:26:38.627459+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:21:47.860063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":125,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T20:26:38.647590+0000","last_change":"2023-09-23T20:20:46.937104+0000","last_active":"2023-09-23T20:26:38.647590+0000","last_peered":"2023-09-23T20:26:38.647590+0000","last_clean":"2023-09-23T20:26:38.647590+0000","last_became_active":"2023-09-23T20:20:46.936820+0000","last_became_peered":"2023-09-23T20:20:46.936820+0000","last_unstale":"2023-09-23T20:26:38.647590+0000","last_undegraded":"2023-09-23T20:26:38.647590+0000","last_fullsized":"2023-09-23T20:26:38.647590+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:22:19.879310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.3","version":"200'7294","reported_seq":7404,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-09-23T20:26:39.878320+0000","last_change":"2023-09-23T20:23:20.060302+0000","last_active":"2023-09-23T20:26:39.878320+0000","last_peered":"2023-09-23T20:26:39.878320+0000","last_clean":"2023-09-23T20:26:39.878320+0000","last_became_active":"2023-09-23T20:23:11.059880+0000","last_became_peered":"2023-09-23T20:23:11.059880+0000","last_unstale":"2023-09-23T20:26:39.878320+0000","last_undegraded":"2023-09-23T20:26:39.878320+0000","last_fullsized":"2023-09-23T20:26:39.878320+0000","mapping_epoch":111,"log_start":"116'1448","ondisk_log_start":"116'1448","created":111,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:22:51.846999+0000","objects_scrubbed":0,"log_size":5846,"log_dups_size":0,"ondisk_log_size":5846,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":30375936,"num_objects":463,"num_object_clones":0,"num_object_copies":1389,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":463,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6391,"num_write_kb":219360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":551,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T20:26:38.628324+0000","last_change":"2023-09-23T20:25:21.160602+0000","last_active":"2023-09-23T20:26:38.628324+0000","last_peered":"2023-09-23T20:26:38.628324+0000","last_clean":"2023-09-23T20:26:38.628324+0000","last_became_active":"2023-09-23T20:25:21.160325+0000","last_became_peered":"2023-09-23T20:25:21.160325+0000","last_unstale":"2023-09-23T20:26:38.628324+0000","last_undegraded":"2023-09-23T20:26:38.628324+0000","last_fullsized":"2023-09-23T20:26:38.628324+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:09:47.213130+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.5","version":"200'7167","reported_seq":7277,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-09-23T20:26:39.842538+0000","last_change":"2023-09-23T20:23:11.152272+0000","last_active":"2023-09-23T20:26:39.842538+0000","last_peered":"2023-09-23T20:26:39.842538+0000","last_clean":"2023-09-23T20:26:39.842538+0000","last_became_active":"2023-09-23T20:23:11.055735+0000","last_became_peered":"2023-09-23T20:23:11.055735+0000","last_unstale":"2023-09-23T20:26:39.842538+0000","last_undegraded":"2023-09-23T20:26:39.842538+0000","last_fullsized":"2023-09-23T20:26:39.842538+0000","mapping_epoch":111,"log_start":"116'1299","ondisk_log_start":"116'1299","created":111,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:22:51.846999+0000","objects_scrubbed":0,"log_size":5868,"log_dups_size":0,"ondisk_log_size":5868,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":27262976,"num_objects":416,"num_object_clones":0,"num_object_copies":1248,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":416,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6284,"num_write_kb":214400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"86'106","reported_seq":824,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T20:26:38.627194+0000","last_change":"2023-09-23T20:20:46.939468+0000","last_active":"2023-09-23T20:26:38.627194+0000","last_peered":"2023-09-23T20:26:38.627194+0000","last_clean":"2023-09-23T20:26:38.627194+0000","last_became_active":"2023-09-23T20:20:46.938954+0000","last_became_peered":"2023-09-23T20:20:46.938954+0000","last_unstale":"2023-09-23T20:26:38.627194+0000","last_undegraded":"2023-09-23T20:26:38.627194+0000","last_fullsized":"2023-09-23T20:26:38.627194+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":106,"log_dups_size":0,"ondisk_log_size":106,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:01:43.761064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00099302599999999993,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1105,"num_read_kb":1689,"num_write":100,"num_write_kb":404,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":159,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T20:26:38.630939+0000","last_change":"2023-09-23T20:20:14.887345+0000","last_active":"2023-09-23T20:26:38.630939+0000","last_peered":"2023-09-23T20:26:38.630939+0000","last_clean":"2023-09-23T20:26:38.630939+0000","last_became_active":"2023-09-23T20:20:14.886970+0000","last_became_peered":"2023-09-23T20:20:14.886970+0000","last_unstale":"2023-09-23T20:26:38.630939+0000","last_undegraded":"2023-09-23T20:26:38.630939+0000","last_fullsized":"2023-09-23T20:26:38.630939+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:00:38.575926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":191,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T20:26:38.653661+0000","last_change":"2023-09-23T20:25:39.631133+0000","last_active":"2023-09-23T20:26:38.653661+0000","last_peered":"2023-09-23T20:26:38.653661+0000","last_clean":"2023-09-23T20:26:38.653661+0000","last_became_active":"2023-09-23T20:25:39.630779+0000","last_became_peered":"2023-09-23T20:25:39.630779+0000","last_unstale":"2023-09-23T20:26:38.653661+0000","last_undegraded":"2023-09-23T20:26:38.653661+0000","last_fullsized":"2023-09-23T20:26:38.653661+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:39:52.635649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":552,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T20:26:38.628980+0000","last_change":"2023-09-23T20:25:21.167528+0000","last_active":"2023-09-23T20:26:38.628980+0000","last_peered":"2023-09-23T20:26:38.628980+0000","last_clean":"2023-09-23T20:26:38.628980+0000","last_became_active":"2023-09-23T20:25:21.167249+0000","last_became_peered":"2023-09-23T20:25:21.167249+0000","last_unstale":"2023-09-23T20:26:38.628980+0000","last_undegraded":"2023-09-23T20:26:38.628980+0000","last_fullsized":"2023-09-23T20:26:38.628980+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:09:47.213130+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.6","version":"200'7199","reported_seq":10026,"reported_epoch":200,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:26:40.518110+0000","last_change":"2023-09-23T20:25:39.715951+0000","last_active":"2023-09-23T20:26:40.518110+0000","last_peered":"2023-09-23T20:26:40.518110+0000","last_clean":"2023-09-23T20:24:51.979668+0000","last_became_active":"2023-09-23T20:25:39.704394+0000","last_became_peered":"2023-09-23T20:25:39.704394+0000","last_unstale":"2023-09-23T20:26:40.518110+0000","last_undegraded":"2023-09-23T20:25:39.621653+0000","last_fullsized":"2023-09-23T20:25:39.614489+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":111,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"119'1954","last_scrub_stamp":"2023-09-23T20:23:12.084143+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:23:12.084143+0000","objects_scrubbed":1042,"log_size":7199,"log_dups_size":0,"ondisk_log_size":7199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:27:04.787533+0000","scrub_duration":0.23299162700000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":24969216,"num_objects":381,"num_object_clones":0,"num_object_copies":1143,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":904,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":381,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5249,"num_write_kb":146816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1331,"num_bytes_recovered":87162880,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[5,7],"avail_no_missing":["5","7"],"object_location_counts":[{"shards":"5,7","objects":381}],"blocked_by":[],"up_primary":2,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"86'130","reported_seq":509,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T20:26:38.655196+0000","last_change":"2023-09-23T20:25:39.637458+0000","last_active":"2023-09-23T20:26:38.655196+0000","last_peered":"2023-09-23T20:26:38.655196+0000","last_clean":"2023-09-23T20:26:38.655196+0000","last_became_active":"2023-09-23T20:25:39.636773+0000","last_became_peered":"2023-09-23T20:25:39.636773+0000","last_unstale":"2023-09-23T20:26:38.655196+0000","last_undegraded":"2023-09-23T20:26:38.655196+0000","last_fullsized":"2023-09-23T20:26:38.655196+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:00:32.810564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011300049999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":512,"num_read_kb":1820,"num_write":133,"num_write_kb":1347,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":157,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T20:26:38.630200+0000","last_change":"2023-09-23T20:25:39.632283+0000","last_active":"2023-09-23T20:26:38.630200+0000","last_peered":"2023-09-23T20:26:38.630200+0000","last_clean":"2023-09-23T20:26:38.630200+0000","last_became_active":"2023-09-23T20:25:39.631819+0000","last_became_peered":"2023-09-23T20:25:39.631819+0000","last_unstale":"2023-09-23T20:26:38.630200+0000","last_undegraded":"2023-09-23T20:26:38.630200+0000","last_fullsized":"2023-09-23T20:26:38.630200+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:25:21.815555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":168,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T20:26:38.653031+0000","last_change":"2023-09-23T20:20:46.936524+0000","last_active":"2023-09-23T20:26:38.653031+0000","last_peered":"2023-09-23T20:26:38.653031+0000","last_clean":"2023-09-23T20:26:38.653031+0000","last_became_active":"2023-09-23T20:20:46.936267+0000","last_became_peered":"2023-09-23T20:20:46.936267+0000","last_unstale":"2023-09-23T20:26:38.653031+0000","last_undegraded":"2023-09-23T20:26:38.653031+0000","last_fullsized":"2023-09-23T20:26:38.653031+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:32:20.984521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"86'194","reported_seq":848,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T20:26:38.627236+0000","last_change":"2023-09-23T20:25:40.618665+0000","last_active":"2023-09-23T20:26:38.627236+0000","last_peered":"2023-09-23T20:26:38.627236+0000","last_clean":"2023-09-23T20:26:38.627236+0000","last_became_active":"2023-09-23T20:25:40.618087+0000","last_became_peered":"2023-09-23T20:25:40.618087+0000","last_unstale":"2023-09-23T20:26:38.627236+0000","last_undegraded":"2023-09-23T20:26:38.627236+0000","last_fullsized":"2023-09-23T20:26:38.627236+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":194,"log_dups_size":0,"ondisk_log_size":194,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:07:16.114968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00037995999999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1025,"num_read_kb":1201,"num_write":173,"num_write_kb":1276,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":553,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T20:26:38.628242+0000","last_change":"2023-09-23T20:25:21.159705+0000","last_active":"2023-09-23T20:26:38.628242+0000","last_peered":"2023-09-23T20:26:38.628242+0000","last_clean":"2023-09-23T20:26:38.628242+0000","last_became_active":"2023-09-23T20:25:21.159200+0000","last_became_peered":"2023-09-23T20:25:21.159200+0000","last_unstale":"2023-09-23T20:26:38.628242+0000","last_undegraded":"2023-09-23T20:26:38.628242+0000","last_fullsized":"2023-09-23T20:26:38.628242+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:09:47.213130+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":46,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-09-23T20:26:39.680092+0000","last_change":"2023-09-23T20:25:02.984205+0000","last_active":"2023-09-23T20:26:39.680092+0000","last_peered":"2023-09-23T20:26:39.680092+0000","last_clean":"2023-09-23T20:26:39.680092+0000","last_became_active":"2023-09-23T20:25:02.983758+0000","last_became_peered":"2023-09-23T20:25:02.983758+0000","last_unstale":"2023-09-23T20:26:39.680092+0000","last_undegraded":"2023-09-23T20:26:39.680092+0000","last_fullsized":"2023-09-23T20:26:39.680092+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:35:14.573101+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":142,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T20:26:38.633830+0000","last_change":"2023-09-23T20:12:45.609829+0000","last_active":"2023-09-23T20:26:38.633830+0000","last_peered":"2023-09-23T20:26:38.633830+0000","last_clean":"2023-09-23T20:26:38.633830+0000","last_became_active":"2023-09-23T20:12:45.609467+0000","last_became_peered":"2023-09-23T20:12:45.609467+0000","last_unstale":"2023-09-23T20:26:38.633830+0000","last_undegraded":"2023-09-23T20:26:38.633830+0000","last_fullsized":"2023-09-23T20:26:38.633830+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:27:31.490555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.4","version":"200'7285","reported_seq":8248,"reported_epoch":200,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:26:40.488984+0000","last_change":"2023-09-23T20:25:40.829173+0000","last_active":"2023-09-23T20:26:40.488984+0000","last_peered":"2023-09-23T20:26:40.488984+0000","last_clean":"2023-09-23T20:24:52.062405+0000","last_became_active":"2023-09-23T20:25:39.693392+0000","last_became_peered":"2023-09-23T20:25:39.693392+0000","last_unstale":"2023-09-23T20:26:40.488984+0000","last_undegraded":"2023-09-23T20:25:40.828975+0000","last_fullsized":"2023-09-23T20:25:39.617392+0000","mapping_epoch":176,"log_start":"116'1437","ondisk_log_start":"116'1437","created":111,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"119'1973","last_scrub_stamp":"2023-09-23T20:23:13.408658+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:23:13.408658+0000","objects_scrubbed":1060,"log_size":5848,"log_dups_size":0,"ondisk_log_size":5848,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:18:48.846593+0000","scrub_duration":0.28811426499999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":29622272,"num_objects":452,"num_object_clones":0,"num_object_copies":1356,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":604,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":452,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5316,"num_write_kb":150656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":389,"num_bytes_recovered":7798784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,5],"avail_no_missing":["7","5"],"object_location_counts":[{"shards":"5,7","objects":452}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":552,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T20:26:38.628483+0000","last_change":"2023-09-23T20:25:21.172709+0000","last_active":"2023-09-23T20:26:38.628483+0000","last_peered":"2023-09-23T20:26:38.628483+0000","last_clean":"2023-09-23T20:26:38.628483+0000","last_became_active":"2023-09-23T20:25:21.171557+0000","last_became_peered":"2023-09-23T20:25:21.171557+0000","last_unstale":"2023-09-23T20:26:38.628483+0000","last_undegraded":"2023-09-23T20:26:38.628483+0000","last_fullsized":"2023-09-23T20:26:38.628483+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:09:47.213130+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.7","version":"200'7347","reported_seq":7519,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-09-23T20:26:40.489869+0000","last_change":"2023-09-23T20:23:16.284169+0000","last_active":"2023-09-23T20:26:40.489869+0000","last_peered":"2023-09-23T20:26:40.489869+0000","last_clean":"2023-09-23T20:26:40.489869+0000","last_became_active":"2023-09-23T20:23:11.069066+0000","last_became_peered":"2023-09-23T20:23:11.069066+0000","last_unstale":"2023-09-23T20:26:40.489869+0000","last_undegraded":"2023-09-23T20:26:40.489869+0000","last_fullsized":"2023-09-23T20:26:40.489869+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":111,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"120'2196","last_scrub_stamp":"2023-09-23T20:23:16.284112+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:23:16.284112+0000","objects_scrubbed":1268,"log_size":7347,"log_dups_size":0,"ondisk_log_size":7347,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:41:59.721972+0000","scrub_duration":0.33229684999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":28180480,"num_objects":430,"num_object_clones":0,"num_object_copies":1290,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":430,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5160,"num_write_kb":138304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.0","version":"86'80","reported_seq":446,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T20:26:38.628150+0000","last_change":"2023-09-23T20:20:46.950784+0000","last_active":"2023-09-23T20:26:38.628150+0000","last_peered":"2023-09-23T20:26:38.628150+0000","last_clean":"2023-09-23T20:26:38.628150+0000","last_became_active":"2023-09-23T20:20:46.950071+0000","last_became_peered":"2023-09-23T20:20:46.950071+0000","last_unstale":"2023-09-23T20:26:38.628150+0000","last_undegraded":"2023-09-23T20:26:38.628150+0000","last_fullsized":"2023-09-23T20:26:38.628150+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":80,"log_dups_size":0,"ondisk_log_size":80,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:11:06.812404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035115,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":395,"num_read_kb":1370,"num_write":66,"num_write_kb":1061,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":162,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T20:26:38.654154+0000","last_change":"2023-09-23T20:25:02.942709+0000","last_active":"2023-09-23T20:26:38.654154+0000","last_peered":"2023-09-23T20:26:38.654154+0000","last_clean":"2023-09-23T20:26:38.654154+0000","last_became_active":"2023-09-23T20:25:02.942323+0000","last_became_peered":"2023-09-23T20:25:02.942323+0000","last_unstale":"2023-09-23T20:26:38.654154+0000","last_undegraded":"2023-09-23T20:26:38.654154+0000","last_fullsized":"2023-09-23T20:26:38.654154+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:26:12.039101+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":178,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-09-23T20:26:39.670533+0000","last_change":"2023-09-23T20:25:39.641452+0000","last_active":"2023-09-23T20:26:39.670533+0000","last_peered":"2023-09-23T20:26:39.670533+0000","last_clean":"2023-09-23T20:26:39.670533+0000","last_became_active":"2023-09-23T20:25:39.638987+0000","last_became_peered":"2023-09-23T20:25:39.638987+0000","last_unstale":"2023-09-23T20:26:39.670533+0000","last_undegraded":"2023-09-23T20:26:39.670533+0000","last_fullsized":"2023-09-23T20:26:39.670533+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:33:59.955461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":552,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T20:26:38.628506+0000","last_change":"2023-09-23T20:25:21.165430+0000","last_active":"2023-09-23T20:26:38.628506+0000","last_peered":"2023-09-23T20:26:38.628506+0000","last_clean":"2023-09-23T20:26:38.628506+0000","last_became_active":"2023-09-23T20:25:21.165109+0000","last_became_peered":"2023-09-23T20:25:21.165109+0000","last_unstale":"2023-09-23T20:26:38.628506+0000","last_undegraded":"2023-09-23T20:26:38.628506+0000","last_fullsized":"2023-09-23T20:26:38.628506+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:09:47.213130+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.5","version":"86'102","reported_seq":706,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T20:26:38.629295+0000","last_change":"2023-09-23T20:20:14.881975+0000","last_active":"2023-09-23T20:26:38.629295+0000","last_peered":"2023-09-23T20:26:38.629295+0000","last_clean":"2023-09-23T20:26:38.629295+0000","last_became_active":"2023-09-23T20:20:14.881544+0000","last_became_peered":"2023-09-23T20:20:14.881544+0000","last_unstale":"2023-09-23T20:26:38.629295+0000","last_undegraded":"2023-09-23T20:26:38.629295+0000","last_fullsized":"2023-09-23T20:26:38.629295+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":102,"log_dups_size":0,"ondisk_log_size":102,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:34.594972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030656500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":963,"num_read_kb":1234,"num_write":60,"num_write_kb":225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":157,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T20:26:38.653840+0000","last_change":"2023-09-23T20:25:39.631751+0000","last_active":"2023-09-23T20:26:38.653840+0000","last_peered":"2023-09-23T20:26:38.653840+0000","last_clean":"2023-09-23T20:26:38.653840+0000","last_became_active":"2023-09-23T20:25:39.631136+0000","last_became_peered":"2023-09-23T20:25:39.631136+0000","last_unstale":"2023-09-23T20:26:38.653840+0000","last_undegraded":"2023-09-23T20:26:38.653840+0000","last_fullsized":"2023-09-23T20:26:38.653840+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:08:17.020722+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":130,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T20:26:38.656590+0000","last_change":"2023-09-23T20:20:14.888874+0000","last_active":"2023-09-23T20:26:38.656590+0000","last_peered":"2023-09-23T20:26:38.656590+0000","last_clean":"2023-09-23T20:26:38.656590+0000","last_became_active":"2023-09-23T20:20:14.887724+0000","last_became_peered":"2023-09-23T20:20:14.887724+0000","last_unstale":"2023-09-23T20:26:38.656590+0000","last_undegraded":"2023-09-23T20:26:38.656590+0000","last_fullsized":"2023-09-23T20:26:38.656590+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:09:24.510326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.2","version":"200'7192","reported_seq":7350,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-09-23T20:26:40.435376+0000","last_change":"2023-09-23T20:23:12.771560+0000","last_active":"2023-09-23T20:26:40.435376+0000","last_peered":"2023-09-23T20:26:40.435376+0000","last_clean":"2023-09-23T20:26:40.435376+0000","last_became_active":"2023-09-23T20:23:11.058929+0000","last_became_peered":"2023-09-23T20:23:11.058929+0000","last_unstale":"2023-09-23T20:26:40.435376+0000","last_undegraded":"2023-09-23T20:26:40.435376+0000","last_fullsized":"2023-09-23T20:26:40.435376+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":111,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"119'1924","last_scrub_stamp":"2023-09-23T20:23:12.771453+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:23:12.771453+0000","objects_scrubbed":1010,"log_size":7192,"log_dups_size":0,"ondisk_log_size":7192,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:49:31.924585+0000","scrub_duration":0.26141313799999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":29097984,"num_objects":444,"num_object_clones":0,"num_object_copies":1332,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":444,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5276,"num_write_kb":150720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":552,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T20:26:38.629129+0000","last_change":"2023-09-23T20:25:21.200271+0000","last_active":"2023-09-23T20:26:38.629129+0000","last_peered":"2023-09-23T20:26:38.629129+0000","last_clean":"2023-09-23T20:26:38.629129+0000","last_became_active":"2023-09-23T20:25:21.197719+0000","last_became_peered":"2023-09-23T20:25:21.197719+0000","last_unstale":"2023-09-23T20:26:38.629129+0000","last_undegraded":"2023-09-23T20:26:38.629129+0000","last_fullsized":"2023-09-23T20:26:38.629129+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:09:47.213130+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":50,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-09-23T20:26:39.675217+0000","last_change":"2023-09-23T20:25:03.007736+0000","last_active":"2023-09-23T20:26:39.675217+0000","last_peered":"2023-09-23T20:26:39.675217+0000","last_clean":"2023-09-23T20:26:39.675217+0000","last_became_active":"2023-09-23T20:25:02.973689+0000","last_became_peered":"2023-09-23T20:25:02.973689+0000","last_unstale":"2023-09-23T20:26:39.675217+0000","last_undegraded":"2023-09-23T20:26:39.675217+0000","last_fullsized":"2023-09-23T20:26:39.675217+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:37:25.643774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"86'128","reported_seq":524,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T20:26:38.651931+0000","last_change":"2023-09-23T20:20:46.939256+0000","last_active":"2023-09-23T20:26:38.651931+0000","last_peered":"2023-09-23T20:26:38.651931+0000","last_clean":"2023-09-23T20:26:38.651931+0000","last_became_active":"2023-09-23T20:20:46.938756+0000","last_became_peered":"2023-09-23T20:20:46.938756+0000","last_unstale":"2023-09-23T20:26:38.651931+0000","last_undegraded":"2023-09-23T20:26:38.651931+0000","last_fullsized":"2023-09-23T20:26:38.651931+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:52:18.161436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000293202,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":579,"num_read_kb":1473,"num_write":96,"num_write_kb":1062,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":46,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-09-23T20:26:39.674630+0000","last_change":"2023-09-23T20:25:02.971740+0000","last_active":"2023-09-23T20:26:39.674630+0000","last_peered":"2023-09-23T20:26:39.674630+0000","last_clean":"2023-09-23T20:26:39.674630+0000","last_became_active":"2023-09-23T20:25:02.970412+0000","last_became_peered":"2023-09-23T20:25:02.970412+0000","last_unstale":"2023-09-23T20:26:39.674630+0000","last_undegraded":"2023-09-23T20:26:39.674630+0000","last_fullsized":"2023-09-23T20:26:39.674630+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:53:17.247216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.1","version":"200'7115","reported_seq":8686,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-09-23T20:26:40.610199+0000","last_change":"2023-09-23T20:25:41.656824+0000","last_active":"2023-09-23T20:26:40.610199+0000","last_peered":"2023-09-23T20:26:40.610199+0000","last_clean":"2023-09-23T20:26:40.610199+0000","last_became_active":"2023-09-23T20:25:41.621039+0000","last_became_peered":"2023-09-23T20:25:41.621039+0000","last_unstale":"2023-09-23T20:26:40.610199+0000","last_undegraded":"2023-09-23T20:26:40.610199+0000","last_fullsized":"2023-09-23T20:26:40.610199+0000","mapping_epoch":178,"log_start":"0'0","ondisk_log_start":"0'0","created":111,"last_epoch_clean":179,"parent":"0.0","parent_split_bits":0,"last_scrub":"119'1889","last_scrub_stamp":"2023-09-23T20:23:11.363674+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:23:11.363674+0000","objects_scrubbed":958,"log_size":7115,"log_dups_size":0,"ondisk_log_size":7115,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:06:09.763416+0000","scrub_duration":0.26718564099999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":28049408,"num_objects":428,"num_object_clones":0,"num_object_copies":1284,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":428,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5234,"num_write_kb":150528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":692,"num_bytes_recovered":20447232,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"90'161","reported_seq":552,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T20:26:38.629125+0000","last_change":"2023-09-23T20:25:21.172983+0000","last_active":"2023-09-23T20:26:38.629125+0000","last_peered":"2023-09-23T20:26:38.629125+0000","last_clean":"2023-09-23T20:26:38.629125+0000","last_became_active":"2023-09-23T20:25:21.172575+0000","last_became_peered":"2023-09-23T20:25:21.172575+0000","last_unstale":"2023-09-23T20:26:38.629125+0000","last_undegraded":"2023-09-23T20:26:38.629125+0000","last_fullsized":"2023-09-23T20:26:38.629125+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:09:47.213130+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.7","version":"86'117","reported_seq":719,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T20:26:38.635278+0000","last_change":"2023-09-23T20:25:40.617386+0000","last_active":"2023-09-23T20:26:38.635278+0000","last_peered":"2023-09-23T20:26:38.635278+0000","last_clean":"2023-09-23T20:26:38.635278+0000","last_became_active":"2023-09-23T20:25:40.616747+0000","last_became_peered":"2023-09-23T20:25:40.616747+0000","last_unstale":"2023-09-23T20:26:38.635278+0000","last_undegraded":"2023-09-23T20:26:38.635278+0000","last_fullsized":"2023-09-23T20:26:38.635278+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:46:44.551561+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046891400000000002,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":893,"num_read_kb":922,"num_write":82,"num_write_kb":4193,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":150,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-09-23T20:26:39.670490+0000","last_change":"2023-09-23T20:20:46.937163+0000","last_active":"2023-09-23T20:26:39.670490+0000","last_peered":"2023-09-23T20:26:39.670490+0000","last_clean":"2023-09-23T20:26:39.670490+0000","last_became_active":"2023-09-23T20:20:46.936846+0000","last_became_peered":"2023-09-23T20:20:46.936846+0000","last_unstale":"2023-09-23T20:26:39.670490+0000","last_undegraded":"2023-09-23T20:26:39.670490+0000","last_fullsized":"2023-09-23T20:26:39.670490+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:01:32.798855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":141,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T20:26:38.630725+0000","last_change":"2023-09-23T20:12:43.765908+0000","last_active":"2023-09-23T20:26:38.630725+0000","last_peered":"2023-09-23T20:26:38.630725+0000","last_clean":"2023-09-23T20:26:38.630725+0000","last_became_active":"2023-09-23T20:12:43.765558+0000","last_became_peered":"2023-09-23T20:12:43.765558+0000","last_unstale":"2023-09-23T20:26:38.630725+0000","last_undegraded":"2023-09-23T20:26:38.630725+0000","last_fullsized":"2023-09-23T20:26:38.630725+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:35:43.809898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.0","version":"201'7263","reported_seq":7428,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-09-23T20:26:43.000715+0000","last_change":"2023-09-23T20:25:40.633699+0000","last_active":"2023-09-23T20:26:43.000715+0000","last_peered":"2023-09-23T20:26:43.000715+0000","last_clean":"2023-09-23T20:26:43.000715+0000","last_became_active":"2023-09-23T20:25:40.618344+0000","last_became_peered":"2023-09-23T20:25:40.618344+0000","last_unstale":"2023-09-23T20:26:43.000715+0000","last_undegraded":"2023-09-23T20:26:43.000715+0000","last_fullsized":"2023-09-23T20:26:43.000715+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":111,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"119'2020","last_scrub_stamp":"2023-09-23T20:23:13.995392+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:23:13.995392+0000","objects_scrubbed":1088,"log_size":7263,"log_dups_size":0,"ondisk_log_size":7263,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:11:19.162515+0000","scrub_duration":0.22507750400000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":23855104,"num_objects":364,"num_object_clones":0,"num_object_copies":1092,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":364,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5252,"num_write_kb":144896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":355,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":132,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T20:26:38.656504+0000","last_change":"2023-09-23T20:20:14.888484+0000","last_active":"2023-09-23T20:26:38.656504+0000","last_peered":"2023-09-23T20:26:38.656504+0000","last_clean":"2023-09-23T20:26:38.656504+0000","last_became_active":"2023-09-23T20:20:14.887219+0000","last_became_peered":"2023-09-23T20:20:14.887219+0000","last_unstale":"2023-09-23T20:26:38.656504+0000","last_undegraded":"2023-09-23T20:26:38.656504+0000","last_fullsized":"2023-09-23T20:26:38.656504+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:13:53.763649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":34,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T20:26:38.656856+0000","last_change":"2023-09-23T20:25:39.639268+0000","last_active":"2023-09-23T20:26:38.656856+0000","last_peered":"2023-09-23T20:26:38.656856+0000","last_clean":"2023-09-23T20:26:38.656856+0000","last_became_active":"2023-09-23T20:25:39.638922+0000","last_became_peered":"2023-09-23T20:25:39.638922+0000","last_unstale":"2023-09-23T20:26:38.656856+0000","last_undegraded":"2023-09-23T20:26:38.656856+0000","last_fullsized":"2023-09-23T20:26:38.656856+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:09:04.040098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":161,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T20:26:38.627439+0000","last_change":"2023-09-23T20:20:46.938981+0000","last_active":"2023-09-23T20:26:38.627439+0000","last_peered":"2023-09-23T20:26:38.627439+0000","last_clean":"2023-09-23T20:26:38.627439+0000","last_became_active":"2023-09-23T20:20:46.938627+0000","last_became_peered":"2023-09-23T20:20:46.938627+0000","last_unstale":"2023-09-23T20:26:38.627439+0000","last_undegraded":"2023-09-23T20:26:38.627439+0000","last_fullsized":"2023-09-23T20:26:38.627439+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:44:43.177086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":205,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T20:26:38.631029+0000","last_change":"2023-09-23T20:25:02.945142+0000","last_active":"2023-09-23T20:26:38.631029+0000","last_peered":"2023-09-23T20:26:38.631029+0000","last_clean":"2023-09-23T20:26:38.631029+0000","last_became_active":"2023-09-23T20:25:02.944653+0000","last_became_peered":"2023-09-23T20:25:02.944653+0000","last_unstale":"2023-09-23T20:26:38.631029+0000","last_undegraded":"2023-09-23T20:26:38.631029+0000","last_fullsized":"2023-09-23T20:26:38.631029+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:38:12.927982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":144,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T20:26:38.627307+0000","last_change":"2023-09-23T20:12:43.753383+0000","last_active":"2023-09-23T20:26:38.627307+0000","last_peered":"2023-09-23T20:26:38.627307+0000","last_clean":"2023-09-23T20:26:38.627307+0000","last_became_active":"2023-09-23T20:12:43.752921+0000","last_became_peered":"2023-09-23T20:12:43.752921+0000","last_unstale":"2023-09-23T20:26:38.627307+0000","last_undegraded":"2023-09-23T20:26:38.627307+0000","last_fullsized":"2023-09-23T20:26:38.627307+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:28:26.982998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":205,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T20:26:38.630760+0000","last_change":"2023-09-23T20:25:03.103656+0000","last_active":"2023-09-23T20:26:38.630760+0000","last_peered":"2023-09-23T20:26:38.630760+0000","last_clean":"2023-09-23T20:26:38.630760+0000","last_became_active":"2023-09-23T20:25:03.103311+0000","last_became_peered":"2023-09-23T20:25:03.103311+0000","last_unstale":"2023-09-23T20:26:38.630760+0000","last_undegraded":"2023-09-23T20:26:38.630760+0000","last_fullsized":"2023-09-23T20:26:38.630760+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:55:54.677030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.18","version":"200'7136","reported_seq":14583,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-09-23T20:26:40.566102+0000","last_change":"2023-09-23T20:25:15.101300+0000","last_active":"2023-09-23T20:26:40.566102+0000","last_peered":"2023-09-23T20:26:40.566102+0000","last_clean":"2023-09-23T20:26:40.566102+0000","last_became_active":"2023-09-23T20:25:15.063524+0000","last_became_peered":"2023-09-23T20:25:15.063524+0000","last_unstale":"2023-09-23T20:26:40.566102+0000","last_undegraded":"2023-09-23T20:26:40.566102+0000","last_fullsized":"2023-09-23T20:26:40.566102+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":118,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":5,"last_scrub":"120'2147","last_scrub_stamp":"2023-09-23T20:23:18.111873+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:23:18.111873+0000","objects_scrubbed":1267,"log_size":7136,"log_dups_size":0,"ondisk_log_size":7136,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:27:49.823665+0000","scrub_duration":0.17815524799999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":29491200,"num_objects":450,"num_object_clones":0,"num_object_copies":1350,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":450,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4989,"num_write_kb":133504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2426,"num_bytes_recovered":140312576,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":55,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-09-23T20:26:39.675248+0000","last_change":"2023-09-23T20:25:39.629869+0000","last_active":"2023-09-23T20:26:39.675248+0000","last_peered":"2023-09-23T20:26:39.675248+0000","last_clean":"2023-09-23T20:26:39.675248+0000","last_became_active":"2023-09-23T20:25:39.629543+0000","last_became_peered":"2023-09-23T20:25:39.629543+0000","last_unstale":"2023-09-23T20:26:39.675248+0000","last_undegraded":"2023-09-23T20:26:39.675248+0000","last_fullsized":"2023-09-23T20:26:39.675248+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:24:48.220679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":55,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-09-23T20:26:39.674937+0000","last_change":"2023-09-23T20:25:39.622819+0000","last_active":"2023-09-23T20:26:39.674937+0000","last_peered":"2023-09-23T20:26:39.674937+0000","last_clean":"2023-09-23T20:26:39.674937+0000","last_became_active":"2023-09-23T20:25:39.622484+0000","last_became_peered":"2023-09-23T20:25:39.622484+0000","last_unstale":"2023-09-23T20:26:39.674937+0000","last_undegraded":"2023-09-23T20:26:39.674937+0000","last_fullsized":"2023-09-23T20:26:39.674937+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:51:53.973261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.19","version":"200'7151","reported_seq":7258,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-09-23T20:26:40.505462+0000","last_change":"2023-09-23T20:23:10.978296+0000","last_active":"2023-09-23T20:26:40.505462+0000","last_peered":"2023-09-23T20:26:40.505462+0000","last_clean":"2023-09-23T20:26:40.505462+0000","last_became_active":"2023-09-23T20:23:10.977578+0000","last_became_peered":"2023-09-23T20:23:10.977578+0000","last_unstale":"2023-09-23T20:26:40.505462+0000","last_undegraded":"2023-09-23T20:26:40.505462+0000","last_fullsized":"2023-09-23T20:26:40.505462+0000","mapping_epoch":118,"log_start":"115'1214","ondisk_log_start":"115'1214","created":118,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:22:51.846999+0000","objects_scrubbed":0,"log_size":5937,"log_dups_size":0,"ondisk_log_size":5937,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":28082176,"num_objects":429,"num_object_clones":0,"num_object_copies":1287,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":429,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6269,"num_write_kb":214304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":178,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T20:26:38.655439+0000","last_change":"2023-09-23T20:25:39.668882+0000","last_active":"2023-09-23T20:26:38.655439+0000","last_peered":"2023-09-23T20:26:38.655439+0000","last_clean":"2023-09-23T20:26:38.655439+0000","last_became_active":"2023-09-23T20:25:39.637128+0000","last_became_peered":"2023-09-23T20:25:39.637128+0000","last_unstale":"2023-09-23T20:26:38.655439+0000","last_undegraded":"2023-09-23T20:26:38.655439+0000","last_fullsized":"2023-09-23T20:26:38.655439+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:56.406298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":178,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T20:26:38.630575+0000","last_change":"2023-09-23T20:24:53.089249+0000","last_active":"2023-09-23T20:26:38.630575+0000","last_peered":"2023-09-23T20:26:38.630575+0000","last_clean":"2023-09-23T20:26:38.630575+0000","last_became_active":"2023-09-23T20:24:53.088952+0000","last_became_peered":"2023-09-23T20:24:53.088952+0000","last_unstale":"2023-09-23T20:26:38.630575+0000","last_undegraded":"2023-09-23T20:26:38.630575+0000","last_fullsized":"2023-09-23T20:26:38.630575+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:38:03.216069+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":5,"num_pg":26,"stat_sum":{"num_bytes":889389080,"num_objects":13572,"num_object_clones":0,"num_object_copies":40716,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4379,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":13572,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":184906,"num_write_kb":5850817,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":25502,"num_bytes_recovered":1203175424,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3488034816,"data_stored":3488022600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":173819,"ondisk_log_size":173819,"up":78,"acting":73,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":15,"num_bytes_recovered":1356,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":25181,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1771,"ondisk_log_size":1771,"up":33,"acting":33,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6082,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201466,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":484468,"kb_used_data":382700,"kb_used_omap":0,"kb_used_meta":101696,"kb_avail":93260684,"statfs":{"total":95995035648,"available":95498940416,"internally_reserved":0,"allocated":391884800,"data_stored":391169008,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":104136704},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:26:23 2023","interfaces":[{"interface":"back","average":{"1min":1.861,"5min":1.2150000000000001,"15min":0.78700000000000003},"min":{"1min":0.247,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":10.907,"5min":13.092000000000001,"15min":13.092000000000001},"last":0.55100000000000005},{"interface":"front","average":{"1min":3.5369999999999999,"5min":1.5940000000000001,"15min":0.91000000000000003},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":9.8610000000000007,"5min":9.8610000000000007,"15min":9.8610000000000007},"last":0.251}]},{"osd":1,"last update":"Sat Sep 23 20:25:57 2023","interfaces":[{"interface":"back","average":{"1min":3.4020000000000001,"5min":1.504,"15min":0.876},"min":{"1min":0.26900000000000002,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":7.0279999999999996,"5min":17.795999999999999,"15min":17.795999999999999},"last":0.46999999999999997},{"interface":"front","average":{"1min":1.976,"5min":1.167,"15min":0.79700000000000004},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":6.8520000000000003,"5min":13.250999999999999,"15min":13.250999999999999},"last":0.40699999999999997}]},{"osd":2,"last update":"Sat Sep 23 20:26:28 2023","interfaces":[{"interface":"back","average":{"1min":3.2610000000000001,"5min":1.6080000000000001,"15min":0.91500000000000004},"min":{"1min":0.39000000000000001,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":8.1270000000000007,"5min":9.8300000000000001,"15min":9.8300000000000001},"last":0.38900000000000001},{"interface":"front","average":{"1min":3.1850000000000001,"5min":1.708,"15min":0.95999999999999996},"min":{"1min":0.48099999999999998,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":6.6699999999999999,"5min":11.423999999999999,"15min":11.423999999999999},"last":0.41499999999999998}]},{"osd":3,"last update":"Sat Sep 23 20:25:57 2023","interfaces":[{"interface":"back","average":{"1min":3.367,"5min":1.4059999999999999,"15min":0.90500000000000003},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":6.7699999999999996,"5min":6.7699999999999996,"15min":6.7699999999999996},"last":6.7409999999999997},{"interface":"front","average":{"1min":3.2650000000000001,"5min":1.276,"15min":0.874},"min":{"1min":0.32500000000000001,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":9.7840000000000007,"5min":9.7840000000000007,"15min":9.7840000000000007},"last":6.7220000000000004}]},{"osd":4,"last update":"Sat Sep 23 20:25:57 2023","interfaces":[{"interface":"back","average":{"1min":1.569,"5min":0.93899999999999995,"15min":0.64400000000000002},"min":{"1min":0.36099999999999999,"5min":0.27000000000000002,"15min":0.26700000000000002},"max":{"1min":5.6799999999999997,"5min":13.141999999999999,"15min":13.141999999999999},"last":0.35899999999999999},{"interface":"front","average":{"1min":4.2130000000000001,"5min":1.4810000000000001,"15min":0.84599999999999997},"min":{"1min":0.30299999999999999,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":10.853,"5min":10.853,"15min":10.853},"last":0.436}]},{"osd":5,"last update":"Sat Sep 23 20:25:57 2023","interfaces":[{"interface":"back","average":{"1min":3.1779999999999999,"5min":1.3560000000000001,"15min":0.80900000000000005},"min":{"1min":0.44,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":10.563000000000001,"5min":10.563000000000001,"15min":10.563000000000001},"last":0.51200000000000001},{"interface":"front","average":{"1min":2.7519999999999998,"5min":1.302,"15min":0.82099999999999995},"min":{"1min":0.34699999999999998,"5min":0.34399999999999997,"15min":0.31},"max":{"1min":12.131,"5min":13.292,"15min":13.292},"last":0.45400000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:25:57 2023","interfaces":[{"interface":"back","average":{"1min":2.399,"5min":1.151,"15min":0.77600000000000002},"min":{"1min":0.40500000000000003,"5min":0.35699999999999998,"15min":0.34499999999999997},"max":{"1min":11.583,"5min":13.185,"15min":13.185},"last":0.496},{"interface":"front","average":{"1min":2.9300000000000002,"5min":1.226,"15min":0.80100000000000005},"min":{"1min":0.41999999999999998,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":9.9120000000000008,"5min":9.9120000000000008,"15min":9.9120000000000008},"last":0.48599999999999999}]}]},{"osd":6,"up_from":47,"seq":201863463102,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":341992,"kb_used_data":262624,"kb_used_omap":0,"kb_used_meta":79360,"kb_avail":93403160,"statfs":{"total":95995035648,"available":95644835840,"internally_reserved":0,"allocated":268926976,"data_stored":268223321,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":81264640},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:26:26 2023","interfaces":[{"interface":"back","average":{"1min":1.7270000000000001,"5min":1.23,"15min":0.79500000000000004},"min":{"1min":0.221,"5min":0.221,"15min":0.221},"max":{"1min":8.1940000000000008,"5min":8.1940000000000008,"15min":8.1940000000000008},"last":0.54600000000000004},{"interface":"front","average":{"1min":1.3069999999999999,"5min":1.4850000000000001,"15min":0.87},"min":{"1min":0.26300000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":4.0609999999999999,"5min":12.512,"15min":12.512},"last":1.357}]},{"osd":1,"last update":"Sat Sep 23 20:25:57 2023","interfaces":[{"interface":"back","average":{"1min":2.6760000000000002,"5min":1.4710000000000001,"15min":0.873},"min":{"1min":0.30099999999999999,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":7.8929999999999998,"5min":18.001000000000001,"15min":18.001000000000001},"last":1.2170000000000001},{"interface":"front","average":{"1min":2.7040000000000002,"5min":1.2,"15min":0.76500000000000001},"min":{"1min":0.28699999999999998,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":13.91,"5min":13.91,"15min":13.91},"last":4.9349999999999996}]},{"osd":2,"last update":"Sat Sep 23 20:26:26 2023","interfaces":[{"interface":"back","average":{"1min":2.153,"5min":1.385,"15min":0.85799999999999998},"min":{"1min":0.32800000000000001,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":8.0090000000000003,"5min":13.856,"15min":13.856},"last":4.407},{"interface":"front","average":{"1min":2.0720000000000001,"5min":1.284,"15min":0.83399999999999996},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":6.8140000000000001,"5min":6.8140000000000001,"15min":6.8140000000000001},"last":0.44500000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:25:57 2023","interfaces":[{"interface":"back","average":{"1min":3.1840000000000002,"5min":1.5349999999999999,"15min":0.92000000000000004},"min":{"1min":0.39200000000000002,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":9.4239999999999995,"5min":9.4239999999999995,"15min":9.4239999999999995},"last":1.444},{"interface":"front","average":{"1min":2.6309999999999998,"5min":1.413,"15min":0.92200000000000004},"min":{"1min":0.42799999999999999,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":7.8700000000000001,"5min":8.6050000000000004,"15min":8.6050000000000004},"last":1.3300000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:25:45 2023","interfaces":[{"interface":"back","average":{"1min":2.4060000000000001,"5min":1.0589999999999999,"15min":0.66100000000000003},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.26200000000000001},"max":{"1min":7.2969999999999997,"5min":7.2969999999999997,"15min":7.2969999999999997},"last":0.71799999999999997},{"interface":"front","average":{"1min":2.0219999999999998,"5min":1.0660000000000001,"15min":0.68200000000000005},"min":{"1min":0.34399999999999997,"5min":0.29799999999999999,"15min":0.28499999999999998},"max":{"1min":5.8460000000000001,"5min":5.8460000000000001,"15min":5.8460000000000001},"last":1.252}]},{"osd":5,"last update":"Sat Sep 23 20:25:45 2023","interfaces":[{"interface":"back","average":{"1min":2.7599999999999998,"5min":1.2709999999999999,"15min":0.755},"min":{"1min":0.377,"5min":0.31900000000000001,"15min":0.29799999999999999},"max":{"1min":14.085000000000001,"5min":14.085000000000001,"15min":14.085000000000001},"last":0.48699999999999999},{"interface":"front","average":{"1min":3.3969999999999998,"5min":1.407,"15min":0.81799999999999995},"min":{"1min":0.372,"5min":0.33100000000000002,"15min":0.29499999999999998},"max":{"1min":13.045,"5min":13.045,"15min":13.045},"last":1.284}]},{"osd":7,"last update":"Sat Sep 23 20:25:57 2023","interfaces":[{"interface":"back","average":{"1min":3.02,"5min":1.262,"15min":0.78300000000000003},"min":{"1min":0.42299999999999999,"5min":0.33500000000000002,"15min":0.32800000000000001},"max":{"1min":10.217000000000001,"5min":10.217000000000001,"15min":10.217000000000001},"last":0.61299999999999999},{"interface":"front","average":{"1min":3.2749999999999999,"5min":1.393,"15min":0.84199999999999997},"min":{"1min":0.36599999999999999,"5min":0.34999999999999998,"15min":0.315},"max":{"1min":9.7360000000000007,"5min":9.7360000000000007,"15min":9.7360000000000007},"last":1.3080000000000001}]}]},{"osd":1,"up_from":99,"seq":425201762377,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":498528,"kb_used_data":350360,"kb_used_omap":6,"kb_used_meta":148153,"kb_avail":93246624,"statfs":{"total":95995035648,"available":95484542976,"internally_reserved":0,"allocated":358768640,"data_stored":357714465,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6696,"internal_metadata":151709144},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2],"upper_bound":2},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:25:52 2023","interfaces":[{"interface":"back","average":{"1min":1.292,"5min":1.595,"15min":0.78300000000000003},"min":{"1min":0.20599999999999999,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":7.6429999999999998,"5min":40.113,"15min":40.113},"last":7.5},{"interface":"front","average":{"1min":1.4570000000000001,"5min":1.2549999999999999,"15min":0.68500000000000005},"min":{"1min":0.245,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":8.8219999999999992,"5min":42.994999999999997,"15min":42.994999999999997},"last":22.210000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:26:27 2023","interfaces":[{"interface":"back","average":{"1min":1.6479999999999999,"5min":1.8029999999999999,"15min":0.85499999999999998},"min":{"1min":0.27900000000000003,"5min":0.253,"15min":0.253},"max":{"1min":9.6839999999999993,"5min":39.962000000000003,"15min":39.962000000000003},"last":6.2859999999999996},{"interface":"front","average":{"1min":2.806,"5min":1.794,"15min":0.96599999999999997},"min":{"1min":0.26700000000000002,"5min":0.246,"15min":0.246},"max":{"1min":8.7910000000000004,"5min":47.442,"15min":47.442},"last":15.090999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:25:58 2023","interfaces":[{"interface":"back","average":{"1min":1.3029999999999999,"5min":1.1319999999999999,"15min":0.74399999999999999},"min":{"1min":0.28499999999999998,"5min":0.246,"15min":0.246},"max":{"1min":4.6130000000000004,"5min":8.7420000000000009,"15min":8.7420000000000009},"last":6.2009999999999996},{"interface":"front","average":{"1min":2.1619999999999999,"5min":1.0660000000000001,"15min":0.627},"min":{"1min":0.26000000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":8.7400000000000002,"5min":9.1859999999999999,"15min":9.1859999999999999},"last":15.147}]},{"osd":4,"last update":"Sat Sep 23 20:25:52 2023","interfaces":[{"interface":"back","average":{"1min":2.0259999999999998,"5min":1.1970000000000001,"15min":0.81000000000000005},"min":{"1min":0.36699999999999999,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":6.9109999999999996,"5min":7.601,"15min":7.601},"last":15.353},{"interface":"front","average":{"1min":1.768,"5min":1.3100000000000001,"15min":0.85999999999999999},"min":{"1min":0.34499999999999997,"5min":0.312,"15min":0.312},"max":{"1min":4.4509999999999996,"5min":8.7899999999999991,"15min":8.7899999999999991},"last":6.2270000000000003}]},{"osd":5,"last update":"Sat Sep 23 20:25:52 2023","interfaces":[{"interface":"back","average":{"1min":1.857,"5min":1.2230000000000001,"15min":0.83499999999999996},"min":{"1min":0.35699999999999998,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":6.7229999999999999,"5min":12.590999999999999,"15min":12.590999999999999},"last":7.1980000000000004},{"interface":"front","average":{"1min":2.0819999999999999,"5min":1.8220000000000001,"15min":1.0580000000000001},"min":{"1min":0.38600000000000001,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":11.919,"5min":39.579000000000001,"15min":39.579000000000001},"last":6.1159999999999997}]},{"osd":6,"last update":"Sat Sep 23 20:25:52 2023","interfaces":[{"interface":"back","average":{"1min":4.516,"5min":1.5369999999999999,"15min":0.95699999999999996},"min":{"1min":0.432,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":34.409999999999997,"5min":34.409999999999997,"15min":34.409999999999997},"last":7.226},{"interface":"front","average":{"1min":4.0140000000000002,"5min":1.762,"15min":1.012},"min":{"1min":0.48699999999999999,"5min":0.377,"15min":0.377},"max":{"1min":35.554000000000002,"5min":35.554000000000002,"15min":35.554000000000002},"last":15.313000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:25:52 2023","interfaces":[{"interface":"back","average":{"1min":2.323,"5min":1.4039999999999999,"15min":0.90200000000000002},"min":{"1min":0.51900000000000002,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":6.742,"5min":7.5469999999999997,"15min":7.5469999999999997},"last":6.0810000000000004},{"interface":"front","average":{"1min":2.2919999999999998,"5min":1.446,"15min":0.93799999999999994},"min":{"1min":0.48299999999999998,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":9.2279999999999998,"5min":9.2279999999999998,"15min":9.2279999999999998},"last":6.2519999999999998}]}]},{"osd":0,"up_from":94,"seq":403726925903,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":660976,"kb_used_data":498856,"kb_used_omap":7,"kb_used_meta":162104,"kb_avail":93084176,"statfs":{"total":95995035648,"available":95318196224,"internally_reserved":0,"allocated":510828544,"data_stored":509828003,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":165994513},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,0,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:25:55 2023","interfaces":[{"interface":"back","average":{"1min":1.1799999999999999,"5min":0.79600000000000004,"15min":0.504},"min":{"1min":0.20799999999999999,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":5.1699999999999999,"5min":5.3609999999999998,"15min":5.3609999999999998},"last":0.58599999999999997},{"interface":"front","average":{"1min":0.72099999999999997,"5min":0.79400000000000004,"15min":0.52200000000000002},"min":{"1min":0.23400000000000001,"5min":0.222,"15min":0.222},"max":{"1min":2.27,"5min":8.9770000000000003,"15min":8.9770000000000003},"last":15.994999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:26:22 2023","interfaces":[{"interface":"back","average":{"1min":4.3319999999999999,"5min":1.8200000000000001,"15min":1.004},"min":{"1min":0.251,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":23.832999999999998,"5min":23.832999999999998,"15min":23.832999999999998},"last":7.5199999999999996},{"interface":"front","average":{"1min":1.3779999999999999,"5min":1.256,"15min":0.749},"min":{"1min":0.223,"5min":0.223,"15min":0.223},"max":{"1min":6.0629999999999997,"5min":9.1940000000000008,"15min":9.1940000000000008},"last":15.948}]},{"osd":3,"last update":"Sat Sep 23 20:25:55 2023","interfaces":[{"interface":"back","average":{"1min":1.857,"5min":1.1639999999999999,"15min":0.78600000000000003},"min":{"1min":0.27800000000000002,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":5.976,"5min":5.976,"15min":5.976},"last":0.76500000000000001},{"interface":"front","average":{"1min":1.341,"5min":0.80700000000000005,"15min":0.55000000000000004},"min":{"1min":0.28499999999999998,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":5.194,"5min":5.4210000000000003,"15min":5.4210000000000003},"last":9.3670000000000009}]},{"osd":4,"last update":"Sat Sep 23 20:26:22 2023","interfaces":[{"interface":"back","average":{"1min":4.6619999999999999,"5min":1.9430000000000001,"15min":1.046},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":23.876999999999999,"5min":23.876999999999999,"15min":23.876999999999999},"last":3.5},{"interface":"front","average":{"1min":2.8839999999999999,"5min":1.587,"15min":0.93799999999999994},"min":{"1min":0.34799999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":23.681999999999999,"5min":23.681999999999999,"15min":23.681999999999999},"last":6.0300000000000002}]},{"osd":5,"last update":"Sat Sep 23 20:26:22 2023","interfaces":[{"interface":"back","average":{"1min":4.3259999999999996,"5min":2.2290000000000001,"15min":1.1819999999999999},"min":{"1min":0.39100000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":24.198,"5min":24.198,"15min":24.198},"last":9.3369999999999997},{"interface":"front","average":{"1min":2.0169999999999999,"5min":1.754,"15min":1.0369999999999999},"min":{"1min":0.37,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":6.9269999999999996,"5min":11.167999999999999,"15min":11.167999999999999},"last":6.1749999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:26:22 2023","interfaces":[{"interface":"back","average":{"1min":3.694,"5min":1.5760000000000001,"15min":0.95799999999999996},"min":{"1min":0.45400000000000001,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":24.408000000000001,"5min":24.408000000000001,"15min":24.408000000000001},"last":9.2750000000000004},{"interface":"front","average":{"1min":3.5230000000000001,"5min":1.526,"15min":0.94999999999999996},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":25.271000000000001,"5min":25.271000000000001,"15min":25.271000000000001},"last":6.1319999999999997}]},{"osd":7,"last update":"Sat Sep 23 20:26:22 2023","interfaces":[{"interface":"back","average":{"1min":3.3199999999999998,"5min":1.7190000000000001,"15min":1.0249999999999999},"min":{"1min":0.442,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":25.359000000000002,"5min":25.359000000000002,"15min":25.359000000000002},"last":6.2039999999999997},{"interface":"front","average":{"1min":5.298,"5min":2.181,"15min":1.1699999999999999},"min":{"1min":0.499,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":23.905999999999999,"5min":23.905999999999999,"15min":23.905999999999999},"last":2.0329999999999999}]}]},{"osd":2,"up_from":104,"seq":446676598851,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":679692,"kb_used_data":561924,"kb_used_omap":11,"kb_used_meta":117748,"kb_avail":93065460,"statfs":{"total":95995035648,"available":95299031040,"internally_reserved":0,"allocated":575410176,"data_stored":574439357,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11878,"internal_metadata":120574362},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:26:30 2023","interfaces":[{"interface":"back","average":{"1min":2.8090000000000002,"5min":1.706,"15min":0.81999999999999995},"min":{"1min":0.21099999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":9.5980000000000008,"5min":12.811,"15min":12.811},"last":1.8879999999999999},{"interface":"front","average":{"1min":2.1019999999999999,"5min":1.2709999999999999,"15min":0.68700000000000006},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":15.746,"5min":15.746,"15min":15.746},"last":0.29199999999999998}]},{"osd":1,"last update":"Sat Sep 23 20:26:30 2023","interfaces":[{"interface":"back","average":{"1min":2.9390000000000001,"5min":1.524,"15min":0.77000000000000002},"min":{"1min":0.28599999999999998,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":14.135999999999999,"5min":14.135999999999999,"15min":14.135999999999999},"last":1.96},{"interface":"front","average":{"1min":2.617,"5min":1.226,"15min":0.66400000000000003},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":13.718,"5min":13.718,"15min":13.718},"last":5.4400000000000004}]},{"osd":3,"last update":"Sat Sep 23 20:25:54 2023","interfaces":[{"interface":"back","average":{"1min":0.72799999999999998,"5min":0.97199999999999998,"15min":0.56499999999999995},"min":{"1min":0.32600000000000001,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":1.9650000000000001,"5min":15.319000000000001,"15min":15.319000000000001},"last":0.56100000000000005},{"interface":"front","average":{"1min":2.1880000000000002,"5min":1.1719999999999999,"15min":0.65600000000000003},"min":{"1min":0.26100000000000001,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":9.6850000000000005,"5min":9.6850000000000005,"15min":9.6850000000000005},"last":1.9279999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:26:30 2023","interfaces":[{"interface":"back","average":{"1min":1.996,"5min":1.234,"15min":0.80000000000000004},"min":{"1min":0.36299999999999999,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":8.2300000000000004,"5min":8.2300000000000004,"15min":8.2300000000000004},"last":0.67300000000000004},{"interface":"front","average":{"1min":2.3679999999999999,"5min":1.3979999999999999,"15min":0.85599999999999998},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":8.6530000000000005,"5min":8.6530000000000005,"15min":8.6530000000000005},"last":0.40300000000000002}]},{"osd":5,"last update":"Sat Sep 23 20:26:30 2023","interfaces":[{"interface":"back","average":{"1min":2.8319999999999999,"5min":1.577,"15min":0.96299999999999997},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":11.237,"5min":11.237,"15min":11.237},"last":0.76300000000000001},{"interface":"front","average":{"1min":3.452,"5min":1.6850000000000001,"15min":0.999},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":11.266999999999999,"5min":11.266999999999999,"15min":11.266999999999999},"last":0.432}]},{"osd":6,"last update":"Sat Sep 23 20:26:30 2023","interfaces":[{"interface":"back","average":{"1min":2.0129999999999999,"5min":1.294,"15min":0.86899999999999999},"min":{"1min":0.49099999999999999,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":6.2599999999999998,"5min":6.7939999999999996,"15min":6.7939999999999996},"last":0.45800000000000002},{"interface":"front","average":{"1min":2.7410000000000001,"5min":1.585,"15min":0.97699999999999998},"min":{"1min":0.41099999999999998,"5min":0.377,"15min":0.377},"max":{"1min":9.6600000000000001,"5min":9.6600000000000001,"15min":9.6600000000000001},"last":1.8100000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:26:30 2023","interfaces":[{"interface":"back","average":{"1min":3.1150000000000002,"5min":1.712,"15min":1.002},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":15.343999999999999,"5min":15.343999999999999,"15min":15.343999999999999},"last":1.8520000000000001},{"interface":"front","average":{"1min":2.6299999999999999,"5min":1.5840000000000001,"15min":0.96599999999999997},"min":{"1min":0.48899999999999999,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":9.1440000000000001,"5min":10.064,"15min":10.064},"last":0.71499999999999997}]}]},{"osd":3,"up_from":109,"seq":468151435325,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":359908,"kb_used_data":229852,"kb_used_omap":4,"kb_used_meta":130043,"kb_avail":93385244,"statfs":{"total":95995035648,"available":95626489856,"internally_reserved":0,"allocated":235368448,"data_stored":234381843,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4514,"internal_metadata":133164638},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:25:56 2023","interfaces":[{"interface":"back","average":{"1min":4.3869999999999996,"5min":1.7390000000000001,"15min":0.81399999999999995},"min":{"1min":0.20399999999999999,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":67.305999999999997,"5min":67.305999999999997,"15min":67.305999999999997},"last":0.33000000000000002},{"interface":"front","average":{"1min":4.5739999999999998,"5min":1.8149999999999999,"15min":0.86099999999999999},"min":{"1min":0.26300000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":67.403999999999996,"5min":67.403999999999996,"15min":67.403999999999996},"last":1.0760000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:25:56 2023","interfaces":[{"interface":"back","average":{"1min":1.3240000000000001,"5min":0.98199999999999998,"15min":0.59199999999999997},"min":{"1min":0.249,"5min":0.214,"15min":0.214},"max":{"1min":4.3440000000000003,"5min":11.537000000000001,"15min":11.537000000000001},"last":0.35499999999999998},{"interface":"front","average":{"1min":1.3819999999999999,"5min":0.97199999999999998,"15min":0.62},"min":{"1min":0.27900000000000003,"5min":0.245,"15min":0.245},"max":{"1min":6.9480000000000004,"5min":11.5,"15min":11.5},"last":0.433}]},{"osd":2,"last update":"Sat Sep 23 20:25:56 2023","interfaces":[{"interface":"back","average":{"1min":0.63900000000000001,"5min":1.1619999999999999,"15min":0.622},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":2.907,"5min":17.335000000000001,"15min":17.335000000000001},"last":1.1499999999999999},{"interface":"front","average":{"1min":1.419,"5min":1.218,"15min":0.71499999999999997},"min":{"1min":0.30099999999999999,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":6.5700000000000003,"5min":17.684000000000001,"15min":17.684000000000001},"last":4.282}]},{"osd":4,"last update":"Sat Sep 23 20:25:56 2023","interfaces":[{"interface":"back","average":{"1min":1.5169999999999999,"5min":1.0049999999999999,"15min":0.73799999999999999},"min":{"1min":0.32600000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":7.1600000000000001,"5min":7.1600000000000001,"15min":7.1600000000000001},"last":0.55300000000000005},{"interface":"front","average":{"1min":2.8730000000000002,"5min":1.202,"15min":0.81100000000000005},"min":{"1min":0.376,"5min":0.371,"15min":0.371},"max":{"1min":7.5069999999999997,"5min":7.5069999999999997,"15min":7.5069999999999997},"last":0.47499999999999998}]},{"osd":5,"last update":"Sat Sep 23 20:25:56 2023","interfaces":[{"interface":"back","average":{"1min":2.4199999999999999,"5min":1.3540000000000001,"15min":0.85599999999999998},"min":{"1min":0.36599999999999999,"5min":0.311,"15min":0.311},"max":{"1min":9.8089999999999993,"5min":9.8089999999999993,"15min":9.8089999999999993},"last":4.3639999999999999},{"interface":"front","average":{"1min":1.5600000000000001,"5min":1.056,"15min":0.77200000000000002},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":6.2430000000000003,"5min":6.4450000000000003,"15min":6.4450000000000003},"last":1.1220000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:25:56 2023","interfaces":[{"interface":"back","average":{"1min":2.802,"5min":1.306,"15min":0.85999999999999999},"min":{"1min":0.40999999999999998,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":8.2789999999999999,"5min":8.2789999999999999,"15min":8.2789999999999999},"last":6.3319999999999999},{"interface":"front","average":{"1min":2.3740000000000001,"5min":1.159,"15min":0.83399999999999996},"min":{"1min":0.39900000000000002,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":8.7759999999999998,"5min":8.7759999999999998,"15min":8.7759999999999998},"last":4.3209999999999997}]},{"osd":7,"last update":"Sat Sep 23 20:25:56 2023","interfaces":[{"interface":"back","average":{"1min":1.8959999999999999,"5min":1.1259999999999999,"15min":0.84399999999999997},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":6.5259999999999998,"5min":6.5259999999999998,"15min":6.5259999999999998},"last":1.5980000000000001},{"interface":"front","average":{"1min":2.4580000000000002,"5min":1.4159999999999999,"15min":0.90000000000000002},"min":{"1min":0.41099999999999998,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":8.4009999999999998,"5min":8.6180000000000003,"15min":8.6180000000000003},"last":4.3920000000000003}]}]},{"osd":4,"up_from":30,"seq":128849019078,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":628132,"kb_used_data":512924,"kb_used_omap":0,"kb_used_meta":115200,"kb_avail":93117020,"statfs":{"total":95995035648,"available":95351828480,"internally_reserved":0,"allocated":525234176,"data_stored":524533807,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":117964800},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,3,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:26:22 2023","interfaces":[{"interface":"back","average":{"1min":2.258,"5min":1.411,"15min":0.84899999999999998},"min":{"1min":0.25800000000000001,"5min":0.222,"15min":0.222},"max":{"1min":11.331,"5min":20.411999999999999,"15min":20.411999999999999},"last":5.7590000000000003},{"interface":"front","average":{"1min":2.6629999999999998,"5min":1.5489999999999999,"15min":0.94399999999999995},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":13.595000000000001,"5min":20.454999999999998,"15min":20.454999999999998},"last":4.3780000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:25:58 2023","interfaces":[{"interface":"back","average":{"1min":2.3940000000000001,"5min":1.0580000000000001,"15min":0.77000000000000002},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":9.5419999999999998,"5min":9.5419999999999998,"15min":9.5419999999999998},"last":0.85099999999999998},{"interface":"front","average":{"1min":2.3530000000000002,"5min":1.1890000000000001,"15min":0.81299999999999994},"min":{"1min":0.28399999999999997,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":11.455,"5min":11.455,"15min":11.455},"last":5.8099999999999996}]},{"osd":2,"last update":"Sat Sep 23 20:26:32 2023","interfaces":[{"interface":"back","average":{"1min":2.1779999999999999,"5min":1.51,"15min":0.90800000000000003},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":11.412000000000001,"5min":21.462,"15min":21.462},"last":5.7919999999999998},{"interface":"front","average":{"1min":3.29,"5min":1.8660000000000001,"15min":1.014},"min":{"1min":0.28999999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":13.567,"5min":22.593,"15min":22.593},"last":4.3559999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:25:58 2023","interfaces":[{"interface":"back","average":{"1min":2.452,"5min":1.4179999999999999,"15min":0.90000000000000002},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":11.377000000000001,"5min":24.891999999999999,"15min":24.891999999999999},"last":5.8710000000000004},{"interface":"front","average":{"1min":2.867,"5min":1.472,"15min":0.93400000000000005},"min":{"1min":0.373,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":11.439,"5min":21.922000000000001,"15min":21.922000000000001},"last":5.8319999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:26:10 2023","interfaces":[{"interface":"back","average":{"1min":5.0119999999999996,"5min":1.649,"15min":0.86799999999999999},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":17.704000000000001,"5min":17.704000000000001,"15min":17.704000000000001},"last":7.9480000000000004},{"interface":"front","average":{"1min":3.1600000000000001,"5min":1.361,"15min":0.77900000000000003},"min":{"1min":0.32500000000000001,"5min":0.28000000000000003,"15min":0.248},"max":{"1min":15.329000000000001,"5min":15.329000000000001,"15min":15.329000000000001},"last":8.2110000000000003}]},{"osd":6,"last update":"Sat Sep 23 20:26:34 2023","interfaces":[{"interface":"back","average":{"1min":3.3639999999999999,"5min":1.9239999999999999,"15min":0.98099999999999998},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.27500000000000002},"max":{"1min":15.252000000000001,"5min":41.371000000000002,"15min":41.371000000000002},"last":4.3259999999999996},{"interface":"front","average":{"1min":1.9179999999999999,"5min":1.593,"15min":0.88200000000000001},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":9.3350000000000009,"5min":41.140999999999998,"15min":41.140999999999998},"last":7.7729999999999997}]},{"osd":7,"last update":"Sat Sep 23 20:25:45 2023","interfaces":[{"interface":"back","average":{"1min":2.2120000000000002,"5min":1.1379999999999999,"15min":0.74399999999999999},"min":{"1min":0.37,"5min":0.35699999999999998,"15min":0.33600000000000002},"max":{"1min":6.3390000000000004,"5min":8.1910000000000007,"15min":8.1910000000000007},"last":7.7089999999999996},{"interface":"front","average":{"1min":2.8180000000000001,"5min":1.212,"15min":0.78200000000000003},"min":{"1min":0.374,"5min":0.34300000000000003,"15min":0.29899999999999999},"max":{"1min":19.018999999999998,"5min":19.018999999999998,"15min":19.018999999999998},"last":0.81399999999999995}]}]},{"osd":5,"up_from":39,"seq":167503724739,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":849952,"kb_used_data":723096,"kb_used_omap":0,"kb_used_meta":126848,"kb_avail":92895200,"statfs":{"total":95995035648,"available":95124684800,"internally_reserved":0,"allocated":740450304,"data_stored":739750141,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":129892352},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,0,3,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:26:22 2023","interfaces":[{"interface":"back","average":{"1min":2.3380000000000001,"5min":1.548,"15min":0.91000000000000003},"min":{"1min":0.29499999999999998,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":11.346,"5min":32.475000000000001,"15min":32.475000000000001},"last":0.41699999999999998},{"interface":"front","average":{"1min":1.9650000000000001,"5min":1.601,"15min":0.93400000000000005},"min":{"1min":0.245,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":5.6879999999999997,"5min":32.457000000000001,"15min":32.457000000000001},"last":0.40899999999999997}]},{"osd":1,"last update":"Sat Sep 23 20:26:00 2023","interfaces":[{"interface":"back","average":{"1min":2.4950000000000001,"5min":1.4490000000000001,"15min":0.84599999999999997},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":11.295,"5min":32.243000000000002,"15min":32.243000000000002},"last":0.28100000000000003},{"interface":"front","average":{"1min":1.228,"5min":1.2370000000000001,"15min":0.77700000000000002},"min":{"1min":0.32300000000000001,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":7.1319999999999997,"5min":32.231000000000002,"15min":32.231000000000002},"last":0.376}]},{"osd":2,"last update":"Sat Sep 23 20:26:22 2023","interfaces":[{"interface":"back","average":{"1min":1.9970000000000001,"5min":1.891,"15min":0.997},"min":{"1min":0.30099999999999999,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":11.407,"5min":34.514000000000003,"15min":34.514000000000003},"last":0.311},{"interface":"front","average":{"1min":2.105,"5min":1.732,"15min":0.94999999999999996},"min":{"1min":0.32100000000000001,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":11.574999999999999,"5min":32.515000000000001,"15min":32.515000000000001},"last":0.46500000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:26:00 2023","interfaces":[{"interface":"back","average":{"1min":2.1019999999999999,"5min":1.1990000000000001,"15min":0.77100000000000002},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":7.0309999999999997,"5min":7.3799999999999999,"15min":7.3799999999999999},"last":0.36599999999999999},{"interface":"front","average":{"1min":2.831,"5min":1.3240000000000001,"15min":0.85199999999999998},"min":{"1min":0.36699999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":11.379,"5min":11.379,"15min":11.379},"last":0.45000000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:26:18 2023","interfaces":[{"interface":"back","average":{"1min":3.391,"5min":1.4339999999999999,"15min":0.78000000000000003},"min":{"1min":0.29199999999999998,"5min":0.26700000000000002,"15min":0.26400000000000001},"max":{"1min":14.218999999999999,"5min":14.218999999999999,"15min":14.218999999999999},"last":0.33600000000000002},{"interface":"front","average":{"1min":2.6720000000000002,"5min":1.3420000000000001,"15min":0.77000000000000002},"min":{"1min":0.34300000000000003,"5min":0.29099999999999998,"15min":0.28799999999999998},"max":{"1min":11.664,"5min":11.664,"15min":11.664},"last":0.39100000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:26:32 2023","interfaces":[{"interface":"back","average":{"1min":2.633,"5min":1.575,"15min":0.85599999999999998},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.29799999999999999},"max":{"1min":8.7949999999999999,"5min":11.545999999999999,"15min":11.545999999999999},"last":0.53900000000000003},{"interface":"front","average":{"1min":1.6870000000000001,"5min":1.5,"15min":0.879},"min":{"1min":0.376,"5min":0.35999999999999999,"15min":0.34499999999999997},"max":{"1min":8.4120000000000008,"5min":11.456,"15min":11.456},"last":0.503}]},{"osd":7,"last update":"Sat Sep 23 20:26:00 2023","interfaces":[{"interface":"back","average":{"1min":2.9199999999999999,"5min":1.2470000000000001,"15min":0.77200000000000002},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33800000000000002},"max":{"1min":11.430999999999999,"5min":11.430999999999999,"15min":11.430999999999999},"last":0.626},{"interface":"front","average":{"1min":1.6479999999999999,"5min":1.0669999999999999,"15min":0.71899999999999997},"min":{"1min":0.35699999999999998,"5min":0.315,"15min":0.315},"max":{"1min":7.0890000000000004,"5min":12.368,"15min":12.368},"last":0.434}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4722,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1360,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6696,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7156,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3154,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":506593280,"data_stored":506593280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":346423296,"data_stored":346423296,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":558764032,"data_stored":558759960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":232718336,"data_stored":232718336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":494665728,"data_stored":494665728,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":715198464,"data_stored":715194392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":253034496,"data_stored":253034496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":380637184,"data_stored":380633112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:26:45.558 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:26:45.583 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:26:45.681 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:46.027 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:26:46.046 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:46.047 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:26:46.148 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:26:46.187 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:46.239 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:26:46.242 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:46.243 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:26:46.255 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:26:46.343 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:26:46.396 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:45 smithi018 ceph-mon[179647]: from='client.54907 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:26:46.396 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:45 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 4379/40800 objects degraded (10.733%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T20:26:46.396 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:46 smithi018 ceph-mon[184144]: from='client.54907 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:26:46.396 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:46 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 4379/40800 objects degraded (10.733%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T20:26:46.407 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:46 smithi149 ceph-mon[170394]: from='client.54907 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:26:46.408 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:46 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 4379/40800 objects degraded (10.733%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T20:26:46.413 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:46.470 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:26:46.477 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:46.477 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:26:46.578 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:26:46.642 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:46.693 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:26:46.700 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:46.700 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:26:46.773 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:26:46.801 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:26:46.815 INFO:teuthology.orchestra.run.smithi018.stdout:ERROR: (22) Invalid argument 2023-09-23T20:26:46.815 INFO:teuthology.orchestra.run.smithi018.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:26:46.926 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:47.033 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:47 smithi018 ceph-mon[179647]: pgmap v438: 109 pgs: 1 active+recovering+undersized+remapped, 1 peering, 103 active+clean, 4 active+recovery_wait+undersized+degraded+remapped; 767 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 956 op/s; 3511/36690 objects degraded (9.569%); 0 B/s, 120 objects/s recovering 2023-09-23T20:26:47.033 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:47 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2939281243' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:26:47.054 INFO:teuthology.orchestra.run.smithi018.stderr:nodeep-scrub is set 2023-09-23T20:26:47.354 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:47 smithi018 ceph-mon[184144]: pgmap v438: 109 pgs: 1 active+recovering+undersized+remapped, 1 peering, 103 active+clean, 4 active+recovery_wait+undersized+degraded+remapped; 767 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 956 op/s; 3511/36690 objects degraded (9.569%); 0 B/s, 120 objects/s recovering 2023-09-23T20:26:47.354 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:47 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2939281243' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:26:47.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:47 smithi149 ceph-mon[170394]: pgmap v438: 109 pgs: 1 active+recovering+undersized+remapped, 1 peering, 103 active+clean, 4 active+recovery_wait+undersized+degraded+remapped; 767 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 956 op/s; 3511/36690 objects degraded (9.569%); 0 B/s, 120 objects/s recovering 2023-09-23T20:26:47.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:47 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2939281243' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:26:47.612 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:26:47.621 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:47.621 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:26:47.637 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T20:26:47.723 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:26:47.765 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:47.837 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:26:47.842 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:47.843 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:26:47.943 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:26:48.069 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:48.206 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:26:48.214 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:48.214 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:26:48.315 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:26:48.372 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:48 smithi018 ceph-mon[184144]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:26:48.373 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:48 smithi018 ceph-mon[184144]: failed to encode map e203 with expected crc 2023-09-23T20:26:48.373 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:48 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2939281243' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:26:48.373 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:48 smithi018 ceph-mon[184144]: osdmap e203: 8 total, 8 up, 8 in 2023-09-23T20:26:48.373 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:48 smithi018 ceph-mon[184144]: failed to encode map e203 with expected crc 2023-09-23T20:26:48.373 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:48 smithi018 ceph-mon[179647]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:26:48.373 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:48 smithi018 ceph-mon[179647]: failed to encode map e203 with expected crc 2023-09-23T20:26:48.374 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:48 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2939281243' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:26:48.374 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:48 smithi018 ceph-mon[179647]: osdmap e203: 8 total, 8 up, 8 in 2023-09-23T20:26:48.374 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:48 smithi018 ceph-mon[179647]: failed to encode map e203 with expected crc 2023-09-23T20:26:48.374 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:48.469 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:48 smithi149 ceph-mon[170394]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:26:48.469 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:48 smithi149 ceph-mon[170394]: failed to encode map e203 with expected crc 2023-09-23T20:26:48.469 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:48 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2939281243' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:26:48.469 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:48 smithi149 ceph-mon[170394]: osdmap e203: 8 total, 8 up, 8 in 2023-09-23T20:26:48.469 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:48 smithi149 ceph-mon[170394]: failed to encode map e203 with expected crc 2023-09-23T20:26:48.540 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:26:48.545 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:48.545 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:26:48.646 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:26:48.720 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:48.786 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:26:48.792 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:48.792 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:26:48.893 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:26:48.954 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:49.024 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:26:49.035 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:49.036 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:26:49.136 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:26:49.215 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:49.278 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:26:49.285 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:49.285 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:26:49.286 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:49 smithi149 ceph-mon[170394]: pgmap v440: 109 pgs: 1 active+recovering+undersized+remapped, 1 peering, 103 active+clean, 4 active+recovery_wait+undersized+degraded+remapped; 630 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 750 op/s; 3511/30090 objects degraded (11.668%); 0 B/s, 120 objects/s recovering 2023-09-23T20:26:49.286 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:49 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:26:49.386 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:26:49.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:49 smithi018 ceph-mon[179647]: pgmap v440: 109 pgs: 1 active+recovering+undersized+remapped, 1 peering, 103 active+clean, 4 active+recovery_wait+undersized+degraded+remapped; 630 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 750 op/s; 3511/30090 objects degraded (11.668%); 0 B/s, 120 objects/s recovering 2023-09-23T20:26:49.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:49 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:26:49.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:49 smithi018 ceph-mon[184144]: pgmap v440: 109 pgs: 1 active+recovering+undersized+remapped, 1 peering, 103 active+clean, 4 active+recovery_wait+undersized+degraded+remapped; 630 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 750 op/s; 3511/30090 objects degraded (11.668%); 0 B/s, 120 objects/s recovering 2023-09-23T20:26:49.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:49 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:26:49.477 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:49.563 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:26:49.567 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:49.568 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:26:49.668 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:26:49.730 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T20:26:49.748 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:49.800 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:26:49.803 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:49.804 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:26:49.904 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:26:49.972 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:50.513 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:26:50.526 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:50.527 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:26:50.627 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:26:50.683 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:50.745 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:26:50.756 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:50.757 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:26:50.858 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:26:50.937 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:50.999 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:26:51.016 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:51.016 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:26:51.117 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:26:51.167 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:51.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:51 smithi018 ceph-mon[184144]: pgmap v441: 109 pgs: 1 active+recovering+undersized+remapped, 104 active+clean, 4 active+recovery_wait+undersized+degraded+remapped; 606 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 662 op/s; 3511/28974 objects degraded (12.118%); 0 B/s, 110 objects/s recovering 2023-09-23T20:26:51.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:51 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 3511/28974 objects degraded (12.118%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T20:26:51.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:51 smithi018 ceph-mon[184144]: osdmap e204: 8 total, 8 up, 8 in 2023-09-23T20:26:51.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:51 smithi018 ceph-mon[179647]: pgmap v441: 109 pgs: 1 active+recovering+undersized+remapped, 104 active+clean, 4 active+recovery_wait+undersized+degraded+remapped; 606 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 662 op/s; 3511/28974 objects degraded (12.118%); 0 B/s, 110 objects/s recovering 2023-09-23T20:26:51.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:51 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 3511/28974 objects degraded (12.118%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T20:26:51.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:51 smithi018 ceph-mon[179647]: osdmap e204: 8 total, 8 up, 8 in 2023-09-23T20:26:51.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:51 smithi149 ceph-mon[170394]: pgmap v441: 109 pgs: 1 active+recovering+undersized+remapped, 104 active+clean, 4 active+recovery_wait+undersized+degraded+remapped; 606 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 662 op/s; 3511/28974 objects degraded (12.118%); 0 B/s, 110 objects/s recovering 2023-09-23T20:26:51.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:51 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 3511/28974 objects degraded (12.118%), 4 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T20:26:51.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:51 smithi149 ceph-mon[170394]: osdmap e204: 8 total, 8 up, 8 in 2023-09-23T20:26:52.378 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:52 smithi018 ceph-mon[184144]: failed to encode map e204 with expected crc 2023-09-23T20:26:52.378 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:52 smithi018 ceph-mon[184144]: failed to encode map e204 with expected crc 2023-09-23T20:26:52.379 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:52 smithi018 ceph-mon[184144]: failed to encode map e204 with expected crc 2023-09-23T20:26:52.379 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:52 smithi018 ceph-mon[184144]: failed to encode map e204 with expected crc 2023-09-23T20:26:52.379 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:52 smithi018 ceph-mon[184144]: failed to encode map e204 with expected crc 2023-09-23T20:26:52.379 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:52 smithi018 ceph-mon[184144]: osdmap e205: 8 total, 8 up, 8 in 2023-09-23T20:26:52.379 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:52 smithi018 ceph-mon[179647]: failed to encode map e204 with expected crc 2023-09-23T20:26:52.379 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:52 smithi018 ceph-mon[179647]: failed to encode map e204 with expected crc 2023-09-23T20:26:52.379 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:52 smithi018 ceph-mon[179647]: failed to encode map e204 with expected crc 2023-09-23T20:26:52.379 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:52 smithi018 ceph-mon[179647]: failed to encode map e204 with expected crc 2023-09-23T20:26:52.379 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:52 smithi018 ceph-mon[179647]: failed to encode map e204 with expected crc 2023-09-23T20:26:52.379 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:52 smithi018 ceph-mon[179647]: osdmap e205: 8 total, 8 up, 8 in 2023-09-23T20:26:52.428 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:26:52.435 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:52.436 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:26:52.469 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:52 smithi149 ceph-mon[170394]: failed to encode map e204 with expected crc 2023-09-23T20:26:52.469 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:52 smithi149 ceph-mon[170394]: failed to encode map e204 with expected crc 2023-09-23T20:26:52.469 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:52 smithi149 ceph-mon[170394]: failed to encode map e204 with expected crc 2023-09-23T20:26:52.469 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:52 smithi149 ceph-mon[170394]: failed to encode map e204 with expected crc 2023-09-23T20:26:52.469 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:52 smithi149 ceph-mon[170394]: failed to encode map e204 with expected crc 2023-09-23T20:26:52.469 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:52 smithi149 ceph-mon[170394]: osdmap e205: 8 total, 8 up, 8 in 2023-09-23T20:26:52.537 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:26:52.608 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:52.664 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:26:52.672 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:52.672 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:26:52.773 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:26:52.842 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:53.366 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:26:53.369 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:53 smithi149 ceph-mon[170394]: pgmap v443: 109 pgs: 2 active+recovering+undersized+remapped, 104 active+clean, 3 active+recovery_wait+undersized+degraded+remapped; 398 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 961 op/s; 2607/18963 objects degraded (13.748%); 0 B/s, 111 objects/s recovering 2023-09-23T20:26:53.369 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:53 smithi149 ceph-mon[170394]: failed to encode map e205 with expected crc 2023-09-23T20:26:53.369 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:53 smithi149 ceph-mon[170394]: failed to encode map e205 with expected crc 2023-09-23T20:26:53.369 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:53 smithi149 ceph-mon[170394]: failed to encode map e205 with expected crc 2023-09-23T20:26:53.369 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:53 smithi149 ceph-mon[170394]: failed to encode map e205 with expected crc 2023-09-23T20:26:53.369 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:53.370 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:26:53.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:53 smithi018 ceph-mon[184144]: pgmap v443: 109 pgs: 2 active+recovering+undersized+remapped, 104 active+clean, 3 active+recovery_wait+undersized+degraded+remapped; 398 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 961 op/s; 2607/18963 objects degraded (13.748%); 0 B/s, 111 objects/s recovering 2023-09-23T20:26:53.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:53 smithi018 ceph-mon[184144]: failed to encode map e205 with expected crc 2023-09-23T20:26:53.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:53 smithi018 ceph-mon[184144]: failed to encode map e205 with expected crc 2023-09-23T20:26:53.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:53 smithi018 ceph-mon[184144]: failed to encode map e205 with expected crc 2023-09-23T20:26:53.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:53 smithi018 ceph-mon[184144]: failed to encode map e205 with expected crc 2023-09-23T20:26:53.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:53 smithi018 ceph-mon[179647]: pgmap v443: 109 pgs: 2 active+recovering+undersized+remapped, 104 active+clean, 3 active+recovery_wait+undersized+degraded+remapped; 398 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 961 op/s; 2607/18963 objects degraded (13.748%); 0 B/s, 111 objects/s recovering 2023-09-23T20:26:53.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:53 smithi018 ceph-mon[179647]: failed to encode map e205 with expected crc 2023-09-23T20:26:53.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:53 smithi018 ceph-mon[179647]: failed to encode map e205 with expected crc 2023-09-23T20:26:53.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:53 smithi018 ceph-mon[179647]: failed to encode map e205 with expected crc 2023-09-23T20:26:53.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:53 smithi018 ceph-mon[179647]: failed to encode map e205 with expected crc 2023-09-23T20:26:53.470 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:26:53.502 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:26:53.517 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:54.645 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:26:54.705 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:26:54.711 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:54.711 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:26:54.794 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:26:54.812 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:26:54.838 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:26:54.878 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:26:54.897 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:54.915 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:26:54.978 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:26:55.003 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:26:55.009 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:55.010 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:26:55.021 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:26:55.057 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:26:55.101 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:26:55.111 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:26:55.131 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:26:55.131 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:26:55.131 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:26:55.131 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:26:55.131 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:26:55.131 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:26:55.131 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:26:55.131 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:26:55.140 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:55.196 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:26:55.214 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:55.215 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:26:55.256 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:55 smithi018 ceph-mon[179647]: pgmap v445: 109 pgs: 1 active+recovering+undersized+remapped, 105 active+clean, 3 active+recovery_wait+undersized+degraded+remapped; 341 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 788 op/s; 2607/16233 objects degraded (16.060%); 0 B/s, 93 objects/s recovering 2023-09-23T20:26:55.256 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:55 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3487299629' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:26:55.257 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:26:54 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:26:54] "GET /metrics HTTP/1.1" 200 36455 "" "Prometheus/2.43.0" 2023-09-23T20:26:55.258 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:55 smithi018 ceph-mon[184144]: pgmap v445: 109 pgs: 1 active+recovering+undersized+remapped, 105 active+clean, 3 active+recovery_wait+undersized+degraded+remapped; 341 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 788 op/s; 2607/16233 objects degraded (16.060%); 0 B/s, 93 objects/s recovering 2023-09-23T20:26:55.258 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:55 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3487299629' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:26:55.315 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:26:55.414 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:55.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:55 smithi149 ceph-mon[170394]: pgmap v445: 109 pgs: 1 active+recovering+undersized+remapped, 105 active+clean, 3 active+recovery_wait+undersized+degraded+remapped; 341 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 788 op/s; 2607/16233 objects degraded (16.060%); 0 B/s, 93 objects/s recovering 2023-09-23T20:26:55.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:55 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3487299629' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:26:55.553 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:26:55.558 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:55.559 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:26:55.631 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:26:55.654 INFO:teuthology.orchestra.run.smithi018.stderr:dumped all 2023-09-23T20:26:55.660 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:26:55.726 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:55.942 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:26:55.954 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:55.954 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:26:56.055 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:26:56.128 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:56 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 2607/16233 objects degraded (16.060%), 3 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T20:26:56.154 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:56.384 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:56 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 2607/16233 objects degraded (16.060%), 3 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T20:26:56.448 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:26:56.455 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:56.455 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:26:56.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:56 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 2607/16233 objects degraded (16.060%), 3 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T20:26:56.556 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:26:56.623 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:56.686 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:26:56.693 INFO:teuthology.orchestra.run.smithi018.stdout:{"pg_ready":true,"pg_map":{"version":446,"stamp":"2023-09-23T20:26:55.545860+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":281221067,"num_objects":4247,"num_object_clones":0,"num_object_copies":12741,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2607,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4247,"num_whiteouts":0,"num_read":6496,"num_read_kb":14137,"num_write":195405,"num_write_kb":5865958,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26413,"num_bytes_recovered":1204947884,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":179729,"ondisk_log_size":179729,"up":327,"acting":323,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":346,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2569252,"kb_used_data":1595812,"kb_used_omap":30,"kb_used_meta":973217,"kb_avail":747391964,"statfs":{"total":767960285184,"available":765329371136,"internally_reserved":0,"allocated":1634111488,"data_stored":1627221650,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31263,"internal_metadata":996574689},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-378994688,"num_objects":-5783,"num_object_clones":0,"num_object_copies":-17349,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-904,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-5783,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5783,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":609,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.006233"},"pg_stats":[{"pgid":"3.1a","version":"69'1","reported_seq":200,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-09-23T20:26:43.976592+0000","last_change":"2023-09-23T20:25:39.632437+0000","last_active":"2023-09-23T20:26:43.976592+0000","last_peered":"2023-09-23T20:26:43.976592+0000","last_clean":"2023-09-23T20:26:43.976592+0000","last_became_active":"2023-09-23T20:25:39.632074+0000","last_became_peered":"2023-09-23T20:25:39.632074+0000","last_unstale":"2023-09-23T20:26:43.976592+0000","last_undegraded":"2023-09-23T20:26:43.976592+0000","last_fullsized":"2023-09-23T20:26:43.976592+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:32:13.543799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":50,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T20:26:50.955271+0000","last_change":"2023-09-23T20:25:02.932223+0000","last_active":"2023-09-23T20:26:50.955271+0000","last_peered":"2023-09-23T20:26:50.955271+0000","last_clean":"2023-09-23T20:26:50.955271+0000","last_became_active":"2023-09-23T20:25:02.931024+0000","last_became_peered":"2023-09-23T20:25:02.931024+0000","last_unstale":"2023-09-23T20:26:50.955271+0000","last_undegraded":"2023-09-23T20:26:50.955271+0000","last_fullsized":"2023-09-23T20:26:50.955271+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:16:00.586211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":162,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T20:26:50.943044+0000","last_change":"2023-09-23T20:25:02.934543+0000","last_active":"2023-09-23T20:26:50.943044+0000","last_peered":"2023-09-23T20:26:50.943044+0000","last_clean":"2023-09-23T20:26:50.943044+0000","last_became_active":"2023-09-23T20:25:02.934178+0000","last_became_peered":"2023-09-23T20:25:02.934178+0000","last_unstale":"2023-09-23T20:26:50.943044+0000","last_undegraded":"2023-09-23T20:26:50.943044+0000","last_fullsized":"2023-09-23T20:26:50.943044+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:27:38.453052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":168,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T20:26:50.946238+0000","last_change":"2023-09-23T20:25:02.957185+0000","last_active":"2023-09-23T20:26:50.946238+0000","last_peered":"2023-09-23T20:26:50.946238+0000","last_clean":"2023-09-23T20:26:50.946238+0000","last_became_active":"2023-09-23T20:25:02.956689+0000","last_became_peered":"2023-09-23T20:25:02.956689+0000","last_unstale":"2023-09-23T20:26:50.946238+0000","last_undegraded":"2023-09-23T20:26:50.946238+0000","last_fullsized":"2023-09-23T20:26:50.946238+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:04:28.524015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":50,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T20:26:50.956197+0000","last_change":"2023-09-23T20:25:02.932054+0000","last_active":"2023-09-23T20:26:50.956197+0000","last_peered":"2023-09-23T20:26:50.956197+0000","last_clean":"2023-09-23T20:26:50.956197+0000","last_became_active":"2023-09-23T20:25:02.931309+0000","last_became_peered":"2023-09-23T20:25:02.931309+0000","last_unstale":"2023-09-23T20:26:50.956197+0000","last_undegraded":"2023-09-23T20:26:50.956197+0000","last_fullsized":"2023-09-23T20:26:50.956197+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:57:53.378499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":164,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T20:26:50.968823+0000","last_change":"2023-09-23T20:20:46.937089+0000","last_active":"2023-09-23T20:26:50.968823+0000","last_peered":"2023-09-23T20:26:50.968823+0000","last_clean":"2023-09-23T20:26:50.968823+0000","last_became_active":"2023-09-23T20:20:46.936831+0000","last_became_peered":"2023-09-23T20:20:46.936831+0000","last_unstale":"2023-09-23T20:26:50.968823+0000","last_undegraded":"2023-09-23T20:26:50.968823+0000","last_fullsized":"2023-09-23T20:26:50.968823+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:34:56.747010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.11","version":"205'7684","reported_seq":10428,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T20:26:55.362537+0000","last_change":"2023-09-23T20:23:31.161216+0000","last_active":"2023-09-23T20:26:55.362537+0000","last_peered":"2023-09-23T20:26:55.362537+0000","last_clean":"2023-09-23T20:26:55.362537+0000","last_became_active":"2023-09-23T20:23:31.146532+0000","last_became_peered":"2023-09-23T20:23:31.146532+0000","last_unstale":"2023-09-23T20:26:55.362537+0000","last_undegraded":"2023-09-23T20:26:55.362537+0000","last_fullsized":"2023-09-23T20:26:55.362537+0000","mapping_epoch":130,"log_start":"115'143","ondisk_log_start":"115'143","created":118,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":5,"last_scrub":"119'1977","last_scrub_stamp":"2023-09-23T20:23:12.353414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:23:12.353414+0000","objects_scrubbed":1018,"log_size":7541,"log_dups_size":0,"ondisk_log_size":7541,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:28:29.630274+0000","scrub_duration":0.26235630100000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4784128,"num_objects":73,"num_object_clones":0,"num_object_copies":219,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":73,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5717,"num_write_kb":152704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1282,"num_bytes_recovered":84017152,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":166,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T20:26:50.945480+0000","last_change":"2023-09-23T20:25:02.969432+0000","last_active":"2023-09-23T20:26:50.945480+0000","last_peered":"2023-09-23T20:26:50.945480+0000","last_clean":"2023-09-23T20:26:50.945480+0000","last_became_active":"2023-09-23T20:25:02.969118+0000","last_became_peered":"2023-09-23T20:25:02.969118+0000","last_unstale":"2023-09-23T20:26:50.945480+0000","last_undegraded":"2023-09-23T20:26:50.945480+0000","last_fullsized":"2023-09-23T20:26:50.945480+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:02:22.453831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":50,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T20:26:50.956116+0000","last_change":"2023-09-23T20:25:02.932225+0000","last_active":"2023-09-23T20:26:50.956116+0000","last_peered":"2023-09-23T20:26:50.956116+0000","last_clean":"2023-09-23T20:26:50.956116+0000","last_became_active":"2023-09-23T20:25:02.931746+0000","last_became_peered":"2023-09-23T20:25:02.931746+0000","last_unstale":"2023-09-23T20:26:50.956116+0000","last_undegraded":"2023-09-23T20:26:50.956116+0000","last_fullsized":"2023-09-23T20:26:50.956116+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:57:24.470179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.10","version":"203'7455","reported_seq":10461,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-09-23T20:26:50.475418+0000","last_change":"2023-09-23T20:23:24.880162+0000","last_active":"2023-09-23T20:26:50.475418+0000","last_peered":"2023-09-23T20:26:50.475418+0000","last_clean":"2023-09-23T20:26:50.475418+0000","last_became_active":"2023-09-23T20:23:24.867240+0000","last_became_peered":"2023-09-23T20:23:24.867240+0000","last_unstale":"2023-09-23T20:26:50.475418+0000","last_undegraded":"2023-09-23T20:26:50.475418+0000","last_fullsized":"2023-09-23T20:26:50.475418+0000","mapping_epoch":128,"log_start":"117'1704","ondisk_log_start":"117'1704","created":118,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":5,"last_scrub":"119'2122","last_scrub_stamp":"2023-09-23T20:23:15.108101+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:23:15.108101+0000","objects_scrubbed":1199,"log_size":5751,"log_dups_size":0,"ondisk_log_size":5751,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:10:57.070009+0000","scrub_duration":0.30476455699999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":10747904,"num_objects":164,"num_object_clones":0,"num_object_copies":492,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":164,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5337,"num_write_kb":137664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1255,"num_bytes_recovered":82247680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":162,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-09-23T20:26:43.978782+0000","last_change":"2023-09-23T20:20:46.950281+0000","last_active":"2023-09-23T20:26:43.978782+0000","last_peered":"2023-09-23T20:26:43.978782+0000","last_clean":"2023-09-23T20:26:43.978782+0000","last_became_active":"2023-09-23T20:20:46.949892+0000","last_became_peered":"2023-09-23T20:20:46.949892+0000","last_unstale":"2023-09-23T20:26:43.978782+0000","last_undegraded":"2023-09-23T20:26:43.978782+0000","last_fullsized":"2023-09-23T20:26:43.978782+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:28.763832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":50,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T20:26:50.955822+0000","last_change":"2023-09-23T20:25:02.947669+0000","last_active":"2023-09-23T20:26:50.955822+0000","last_peered":"2023-09-23T20:26:50.955822+0000","last_clean":"2023-09-23T20:26:50.955822+0000","last_became_active":"2023-09-23T20:25:02.947368+0000","last_became_peered":"2023-09-23T20:25:02.947368+0000","last_unstale":"2023-09-23T20:26:50.955822+0000","last_undegraded":"2023-09-23T20:26:50.955822+0000","last_fullsized":"2023-09-23T20:26:50.955822+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:52:53.813761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.13","version":"205'7592","reported_seq":11857,"reported_epoch":205,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:26:54.885243+0000","last_change":"2023-09-23T20:25:40.848149+0000","last_active":"2023-09-23T20:26:54.885243+0000","last_peered":"2023-09-23T20:26:54.885243+0000","last_clean":"2023-09-23T20:24:36.774911+0000","last_became_active":"2023-09-23T20:25:02.945089+0000","last_became_peered":"2023-09-23T20:25:02.945089+0000","last_unstale":"2023-09-23T20:26:54.885243+0000","last_undegraded":"2023-09-23T20:25:40.847992+0000","last_fullsized":"2023-09-23T20:25:02.906341+0000","mapping_epoch":165,"log_start":"119'1870","ondisk_log_start":"119'1870","created":118,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:22:51.846999+0000","objects_scrubbed":0,"log_size":5722,"log_dups_size":0,"ondisk_log_size":5722,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":3637248,"num_objects":56,"num_object_clones":0,"num_object_copies":168,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":270,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":56,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6690,"num_write_kb":215840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1401,"num_bytes_recovered":85852160,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[0,1],"avail_no_missing":["0","1"],"object_location_counts":[{"shards":"0,1","objects":56}],"blocked_by":[],"up_primary":3,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":198,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-09-23T20:26:43.981290+0000","last_change":"2023-09-23T20:25:03.038511+0000","last_active":"2023-09-23T20:26:43.981290+0000","last_peered":"2023-09-23T20:26:43.981290+0000","last_clean":"2023-09-23T20:26:43.981290+0000","last_became_active":"2023-09-23T20:25:02.976811+0000","last_became_peered":"2023-09-23T20:25:02.976811+0000","last_unstale":"2023-09-23T20:26:43.981290+0000","last_undegraded":"2023-09-23T20:26:43.981290+0000","last_fullsized":"2023-09-23T20:26:43.981290+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:28:06.605731+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":163,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T20:26:50.942540+0000","last_change":"2023-09-23T20:25:39.646452+0000","last_active":"2023-09-23T20:26:50.942540+0000","last_peered":"2023-09-23T20:26:50.942540+0000","last_clean":"2023-09-23T20:26:50.942540+0000","last_became_active":"2023-09-23T20:25:39.646175+0000","last_became_peered":"2023-09-23T20:25:39.646175+0000","last_unstale":"2023-09-23T20:26:50.942540+0000","last_undegraded":"2023-09-23T20:26:50.942540+0000","last_fullsized":"2023-09-23T20:26:50.942540+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:33:02.099951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.12","version":"205'7397","reported_seq":11501,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T20:26:55.025372+0000","last_change":"2023-09-23T20:26:14.067112+0000","last_active":"2023-09-23T20:26:55.025372+0000","last_peered":"2023-09-23T20:26:55.025372+0000","last_clean":"2023-09-23T20:26:55.025372+0000","last_became_active":"2023-09-23T20:26:14.046688+0000","last_became_peered":"2023-09-23T20:26:14.046688+0000","last_unstale":"2023-09-23T20:26:55.025372+0000","last_undegraded":"2023-09-23T20:26:55.025372+0000","last_fullsized":"2023-09-23T20:26:55.025372+0000","mapping_epoch":188,"log_start":"0'0","ondisk_log_start":"0'0","created":118,"last_epoch_clean":189,"parent":"0.0","parent_split_bits":5,"last_scrub":"119'2087","last_scrub_stamp":"2023-09-23T20:23:15.737284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:23:15.737284+0000","objects_scrubbed":1166,"log_size":7397,"log_dups_size":0,"ondisk_log_size":7397,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:39:04.257684+0000","scrub_duration":0.27092271000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4128768,"num_objects":63,"num_object_clones":0,"num_object_copies":189,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":63,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5323,"num_write_kb":135040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1590,"num_bytes_recovered":80216064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":167,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-09-23T20:26:43.977109+0000","last_change":"2023-09-23T20:12:43.750217+0000","last_active":"2023-09-23T20:26:43.977109+0000","last_peered":"2023-09-23T20:26:43.977109+0000","last_clean":"2023-09-23T20:26:43.977109+0000","last_became_active":"2023-09-23T20:12:43.749929+0000","last_became_peered":"2023-09-23T20:12:43.749929+0000","last_unstale":"2023-09-23T20:26:43.977109+0000","last_undegraded":"2023-09-23T20:26:43.977109+0000","last_fullsized":"2023-09-23T20:26:43.977109+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:50:28.470211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":161,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-09-23T20:26:43.976699+0000","last_change":"2023-09-23T20:20:46.939938+0000","last_active":"2023-09-23T20:26:43.976699+0000","last_peered":"2023-09-23T20:26:43.976699+0000","last_clean":"2023-09-23T20:26:43.976699+0000","last_became_active":"2023-09-23T20:20:46.938777+0000","last_became_peered":"2023-09-23T20:20:46.938777+0000","last_unstale":"2023-09-23T20:26:43.976699+0000","last_undegraded":"2023-09-23T20:26:43.976699+0000","last_fullsized":"2023-09-23T20:26:43.976699+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:45:10.329438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.15","version":"203'7282","reported_seq":10869,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-09-23T20:26:50.500471+0000","last_change":"2023-09-23T20:24:36.924738+0000","last_active":"2023-09-23T20:26:50.500471+0000","last_peered":"2023-09-23T20:26:50.500471+0000","last_clean":"2023-09-23T20:26:50.500471+0000","last_became_active":"2023-09-23T20:23:54.922070+0000","last_became_peered":"2023-09-23T20:23:54.922070+0000","last_unstale":"2023-09-23T20:26:50.500471+0000","last_undegraded":"2023-09-23T20:26:50.500471+0000","last_fullsized":"2023-09-23T20:26:50.500471+0000","mapping_epoch":143,"log_start":"0'0","ondisk_log_start":"0'0","created":118,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:22:51.846999+0000","objects_scrubbed":0,"log_size":7282,"log_dups_size":0,"ondisk_log_size":7282,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13041664,"num_objects":199,"num_object_clones":0,"num_object_copies":597,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":199,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6399,"num_write_kb":211136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1727,"num_bytes_recovered":113180672,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":197,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-09-23T20:26:43.981110+0000","last_change":"2023-09-23T20:25:39.654026+0000","last_active":"2023-09-23T20:26:43.981110+0000","last_peered":"2023-09-23T20:26:43.981110+0000","last_clean":"2023-09-23T20:26:43.981110+0000","last_became_active":"2023-09-23T20:25:39.626849+0000","last_became_peered":"2023-09-23T20:25:39.626849+0000","last_unstale":"2023-09-23T20:26:43.981110+0000","last_undegraded":"2023-09-23T20:26:43.981110+0000","last_fullsized":"2023-09-23T20:26:43.981110+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:22:53.957630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":50,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T20:26:50.958989+0000","last_change":"2023-09-23T20:25:02.962047+0000","last_active":"2023-09-23T20:26:50.958989+0000","last_peered":"2023-09-23T20:26:50.958989+0000","last_clean":"2023-09-23T20:26:50.958989+0000","last_became_active":"2023-09-23T20:25:02.961523+0000","last_became_peered":"2023-09-23T20:25:02.961523+0000","last_unstale":"2023-09-23T20:26:50.958989+0000","last_undegraded":"2023-09-23T20:26:50.958989+0000","last_fullsized":"2023-09-23T20:26:50.958989+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:18:11.830822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.14","version":"205'7359","reported_seq":13811,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T20:26:53.026775+0000","last_change":"2023-09-23T20:26:44.006941+0000","last_active":"2023-09-23T20:26:53.026775+0000","last_peered":"2023-09-23T20:26:53.026775+0000","last_clean":"2023-09-23T20:26:53.026775+0000","last_became_active":"2023-09-23T20:26:43.982576+0000","last_became_peered":"2023-09-23T20:26:43.982576+0000","last_unstale":"2023-09-23T20:26:53.026775+0000","last_undegraded":"2023-09-23T20:26:53.026775+0000","last_fullsized":"2023-09-23T20:26:53.026775+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":118,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":5,"last_scrub":"142'3061","last_scrub_stamp":"2023-09-23T20:23:51.058360+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:23:51.058360+0000","objects_scrubbed":2161,"log_size":7359,"log_dups_size":0,"ondisk_log_size":7359,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:39:55.532369+0000","scrub_duration":0.44581189700000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":7929856,"num_objects":121,"num_object_clones":0,"num_object_copies":363,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":121,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4310,"num_write_kb":72640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1218,"num_bytes_recovered":9175040,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":166,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T20:26:50.944636+0000","last_change":"2023-09-23T20:25:02.953993+0000","last_active":"2023-09-23T20:26:50.944636+0000","last_peered":"2023-09-23T20:26:50.944636+0000","last_clean":"2023-09-23T20:26:50.944636+0000","last_became_active":"2023-09-23T20:25:02.953672+0000","last_became_peered":"2023-09-23T20:25:02.953672+0000","last_unstale":"2023-09-23T20:26:50.944636+0000","last_undegraded":"2023-09-23T20:26:50.944636+0000","last_fullsized":"2023-09-23T20:26:50.944636+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:26:08.603577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":181,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-09-23T20:26:43.978817+0000","last_change":"2023-09-23T20:25:02.949584+0000","last_active":"2023-09-23T20:26:43.978817+0000","last_peered":"2023-09-23T20:26:43.978817+0000","last_clean":"2023-09-23T20:26:43.978817+0000","last_became_active":"2023-09-23T20:25:02.949132+0000","last_became_peered":"2023-09-23T20:25:02.949132+0000","last_unstale":"2023-09-23T20:26:43.978817+0000","last_undegraded":"2023-09-23T20:26:43.978817+0000","last_fullsized":"2023-09-23T20:26:43.978817+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:34:20.509823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.17","version":"205'7371","reported_seq":13929,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T20:26:52.903053+0000","last_change":"2023-09-23T20:26:20.168261+0000","last_active":"2023-09-23T20:26:52.903053+0000","last_peered":"2023-09-23T20:26:52.903053+0000","last_clean":"2023-09-23T20:26:52.903053+0000","last_became_active":"2023-09-23T20:26:20.147495+0000","last_became_peered":"2023-09-23T20:26:20.147495+0000","last_unstale":"2023-09-23T20:26:52.903053+0000","last_undegraded":"2023-09-23T20:26:52.903053+0000","last_fullsized":"2023-09-23T20:26:52.903053+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":118,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:22:51.846999+0000","objects_scrubbed":0,"log_size":7371,"log_dups_size":0,"ondisk_log_size":7371,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8847360,"num_objects":135,"num_object_clones":0,"num_object_copies":405,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":135,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6453,"num_write_kb":210816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2467,"num_bytes_recovered":161677312,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":146,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-09-23T20:26:43.981006+0000","last_change":"2023-09-23T20:12:43.749139+0000","last_active":"2023-09-23T20:26:43.981006+0000","last_peered":"2023-09-23T20:26:43.981006+0000","last_clean":"2023-09-23T20:26:43.981006+0000","last_became_active":"2023-09-23T20:12:43.748854+0000","last_became_peered":"2023-09-23T20:12:43.748854+0000","last_unstale":"2023-09-23T20:26:43.981006+0000","last_undegraded":"2023-09-23T20:26:43.981006+0000","last_fullsized":"2023-09-23T20:26:43.981006+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:57:11.630712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":176,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T20:26:50.946173+0000","last_change":"2023-09-23T20:25:02.957010+0000","last_active":"2023-09-23T20:26:50.946173+0000","last_peered":"2023-09-23T20:26:50.946173+0000","last_clean":"2023-09-23T20:26:50.946173+0000","last_became_active":"2023-09-23T20:25:02.956267+0000","last_became_peered":"2023-09-23T20:25:02.956267+0000","last_unstale":"2023-09-23T20:26:50.946173+0000","last_undegraded":"2023-09-23T20:26:50.946173+0000","last_fullsized":"2023-09-23T20:26:50.946173+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:10:55.299422+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.16","version":"203'7517","reported_seq":14979,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-09-23T20:26:50.566741+0000","last_change":"2023-09-23T20:26:16.974059+0000","last_active":"2023-09-23T20:26:50.566741+0000","last_peered":"2023-09-23T20:26:50.566741+0000","last_clean":"2023-09-23T20:26:50.566741+0000","last_became_active":"2023-09-23T20:26:16.954812+0000","last_became_peered":"2023-09-23T20:26:16.954812+0000","last_unstale":"2023-09-23T20:26:50.566741+0000","last_undegraded":"2023-09-23T20:26:50.566741+0000","last_fullsized":"2023-09-23T20:26:50.566741+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":118,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":5,"last_scrub":"119'2163","last_scrub_stamp":"2023-09-23T20:23:15.241911+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:23:15.241911+0000","objects_scrubbed":1192,"log_size":7517,"log_dups_size":0,"ondisk_log_size":7517,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:52:46.936495+0000","scrub_duration":0.33323710899999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":10551296,"num_objects":161,"num_object_clones":0,"num_object_copies":483,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":161,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5365,"num_write_kb":138688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2720,"num_bytes_recovered":154599424,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":173,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T20:26:50.967552+0000","last_change":"2023-09-23T20:20:14.881009+0000","last_active":"2023-09-23T20:26:50.967552+0000","last_peered":"2023-09-23T20:26:50.967552+0000","last_clean":"2023-09-23T20:26:50.967552+0000","last_became_active":"2023-09-23T20:20:14.880685+0000","last_became_peered":"2023-09-23T20:20:14.880685+0000","last_unstale":"2023-09-23T20:26:50.967552+0000","last_undegraded":"2023-09-23T20:26:50.967552+0000","last_fullsized":"2023-09-23T20:26:50.967552+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:28.010434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":50,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T20:26:50.957377+0000","last_change":"2023-09-23T20:25:02.977462+0000","last_active":"2023-09-23T20:26:50.957377+0000","last_peered":"2023-09-23T20:26:50.957377+0000","last_clean":"2023-09-23T20:26:50.957377+0000","last_became_active":"2023-09-23T20:25:02.969910+0000","last_became_peered":"2023-09-23T20:25:02.969910+0000","last_unstale":"2023-09-23T20:26:50.957377+0000","last_undegraded":"2023-09-23T20:26:50.957377+0000","last_fullsized":"2023-09-23T20:26:50.957377+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:22:55.644955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.9","version":"203'7265","reported_seq":7376,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-09-23T20:26:50.487954+0000","last_change":"2023-09-23T20:23:10.973039+0000","last_active":"2023-09-23T20:26:50.487954+0000","last_peered":"2023-09-23T20:26:50.487954+0000","last_clean":"2023-09-23T20:26:50.487954+0000","last_became_active":"2023-09-23T20:23:10.971930+0000","last_became_peered":"2023-09-23T20:23:10.971930+0000","last_unstale":"2023-09-23T20:26:50.487954+0000","last_undegraded":"2023-09-23T20:26:50.487954+0000","last_fullsized":"2023-09-23T20:26:50.487954+0000","mapping_epoch":111,"log_start":"116'1414","ondisk_log_start":"116'1414","created":111,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:22:51.846999+0000","objects_scrubbed":0,"log_size":5851,"log_dups_size":0,"ondisk_log_size":5851,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":11304960,"num_objects":172,"num_object_clones":0,"num_object_copies":516,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":172,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6382,"num_write_kb":209760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":164,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-09-23T20:26:43.981508+0000","last_change":"2023-09-23T20:20:14.883283+0000","last_active":"2023-09-23T20:26:43.981508+0000","last_peered":"2023-09-23T20:26:43.981508+0000","last_clean":"2023-09-23T20:26:43.981508+0000","last_became_active":"2023-09-23T20:20:14.882806+0000","last_became_peered":"2023-09-23T20:20:14.882806+0000","last_unstale":"2023-09-23T20:26:43.981508+0000","last_undegraded":"2023-09-23T20:26:43.981508+0000","last_fullsized":"2023-09-23T20:26:43.981508+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:23:35.295701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":144,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-09-23T20:26:43.978576+0000","last_change":"2023-09-23T20:12:45.179211+0000","last_active":"2023-09-23T20:26:43.978576+0000","last_peered":"2023-09-23T20:26:43.978576+0000","last_clean":"2023-09-23T20:26:43.978576+0000","last_became_active":"2023-09-23T20:12:45.178966+0000","last_became_peered":"2023-09-23T20:12:45.178966+0000","last_unstale":"2023-09-23T20:26:43.978576+0000","last_undegraded":"2023-09-23T20:26:43.978576+0000","last_fullsized":"2023-09-23T20:26:43.978576+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:58.902140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.8","version":"205'7470","reported_seq":9382,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T20:26:52.024218+0000","last_change":"2023-09-23T20:26:51.971852+0000","last_active":"2023-09-23T20:26:52.024218+0000","last_peered":"2023-09-23T20:26:52.024218+0000","last_clean":"2023-09-23T20:26:52.024218+0000","last_became_active":"2023-09-23T20:26:51.952240+0000","last_became_peered":"2023-09-23T20:26:51.952240+0000","last_unstale":"2023-09-23T20:26:52.024218+0000","last_undegraded":"2023-09-23T20:26:52.024218+0000","last_fullsized":"2023-09-23T20:26:52.024218+0000","mapping_epoch":204,"log_start":"117'1724","ondisk_log_start":"117'1724","created":111,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"120'2144","last_scrub_stamp":"2023-09-23T20:23:17.129360+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:23:17.129360+0000","objects_scrubbed":1209,"log_size":5746,"log_dups_size":0,"ondisk_log_size":5746,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:00:36.612715+0000","scrub_duration":0.17393181099999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8978432,"num_objects":137,"num_object_clones":0,"num_object_copies":411,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":137,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5326,"num_write_kb":136128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":869,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":165,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-09-23T20:26:43.981455+0000","last_change":"2023-09-23T20:20:46.950784+0000","last_active":"2023-09-23T20:26:43.981455+0000","last_peered":"2023-09-23T20:26:43.981455+0000","last_clean":"2023-09-23T20:26:43.981455+0000","last_became_active":"2023-09-23T20:20:46.950055+0000","last_became_peered":"2023-09-23T20:20:46.950055+0000","last_unstale":"2023-09-23T20:26:43.981455+0000","last_undegraded":"2023-09-23T20:26:43.981455+0000","last_fullsized":"2023-09-23T20:26:43.981455+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:23:07.645427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":198,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-09-23T20:26:43.976661+0000","last_change":"2023-09-23T20:25:02.943743+0000","last_active":"2023-09-23T20:26:43.976661+0000","last_peered":"2023-09-23T20:26:43.976661+0000","last_clean":"2023-09-23T20:26:43.976661+0000","last_became_active":"2023-09-23T20:25:02.943279+0000","last_became_peered":"2023-09-23T20:25:02.943279+0000","last_unstale":"2023-09-23T20:26:43.976661+0000","last_undegraded":"2023-09-23T20:26:43.976661+0000","last_fullsized":"2023-09-23T20:26:43.976661+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:21:36.128366+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.b","version":"205'13073","reported_seq":13190,"reported_epoch":205,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:26:54.833511+0000","last_change":"2023-09-23T20:25:39.764057+0000","last_active":"2023-09-23T20:26:54.833511+0000","last_peered":"2023-09-23T20:26:54.833511+0000","last_clean":"2023-09-23T20:24:52.031628+0000","last_became_active":"2023-09-23T20:25:39.743845+0000","last_became_peered":"2023-09-23T20:25:39.743845+0000","last_unstale":"2023-09-23T20:26:54.833511+0000","last_undegraded":"2023-09-23T20:25:39.622936+0000","last_fullsized":"2023-09-23T20:25:39.612475+0000","mapping_epoch":176,"log_start":"159'7372","ondisk_log_start":"159'7372","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:22:51.846999+0000","objects_scrubbed":0,"log_size":5701,"log_dups_size":0,"ondisk_log_size":5701,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:56:57.182858+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":10420224,"num_objects":159,"num_object_clones":0,"num_object_copies":477,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1733,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":159,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13073,"num_write_kb":423424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,5","objects":159}],"blocked_by":[],"up_primary":2,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":147,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-09-23T20:26:43.981114+0000","last_change":"2023-09-23T20:12:43.751523+0000","last_active":"2023-09-23T20:26:43.981114+0000","last_peered":"2023-09-23T20:26:43.981114+0000","last_clean":"2023-09-23T20:26:43.981114+0000","last_became_active":"2023-09-23T20:12:43.751257+0000","last_became_peered":"2023-09-23T20:12:43.751257+0000","last_unstale":"2023-09-23T20:26:43.981114+0000","last_undegraded":"2023-09-23T20:26:43.981114+0000","last_fullsized":"2023-09-23T20:26:43.981114+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:50:27.040225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":164,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T20:26:50.969205+0000","last_change":"2023-09-23T20:20:46.934762+0000","last_active":"2023-09-23T20:26:50.969205+0000","last_peered":"2023-09-23T20:26:50.969205+0000","last_clean":"2023-09-23T20:26:50.969205+0000","last_became_active":"2023-09-23T20:20:46.934472+0000","last_became_peered":"2023-09-23T20:20:46.934472+0000","last_unstale":"2023-09-23T20:26:50.969205+0000","last_undegraded":"2023-09-23T20:26:50.969205+0000","last_fullsized":"2023-09-23T20:26:50.969205+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:28:42.708286+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.a","version":"205'13094","reported_seq":21742,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T20:26:51.983251+0000","last_change":"2023-09-23T20:26:13.071682+0000","last_active":"2023-09-23T20:26:51.983251+0000","last_peered":"2023-09-23T20:26:51.983251+0000","last_clean":"2023-09-23T20:26:51.983251+0000","last_became_active":"2023-09-23T20:26:13.034652+0000","last_became_peered":"2023-09-23T20:26:13.034652+0000","last_unstale":"2023-09-23T20:26:51.983251+0000","last_undegraded":"2023-09-23T20:26:51.983251+0000","last_fullsized":"2023-09-23T20:26:51.983251+0000","mapping_epoch":187,"log_start":"144'4713","ondisk_log_start":"144'4713","created":111,"last_epoch_clean":188,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:22:51.846999+0000","objects_scrubbed":0,"log_size":8381,"log_dups_size":0,"ondisk_log_size":8381,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:05:48.610901+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":21889024,"num_objects":334,"num_object_clones":0,"num_object_copies":1002,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":334,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13094,"num_write_kb":429696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3973,"num_bytes_recovered":115343360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":182,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-09-23T20:26:43.978470+0000","last_change":"2023-09-23T20:25:02.947631+0000","last_active":"2023-09-23T20:26:43.978470+0000","last_peered":"2023-09-23T20:26:43.978470+0000","last_clean":"2023-09-23T20:26:43.978470+0000","last_became_active":"2023-09-23T20:25:02.947268+0000","last_became_peered":"2023-09-23T20:25:02.947268+0000","last_unstale":"2023-09-23T20:26:43.978470+0000","last_undegraded":"2023-09-23T20:26:43.978470+0000","last_fullsized":"2023-09-23T20:26:43.978470+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:19:43.643985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":149,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T20:26:50.945502+0000","last_change":"2023-09-23T20:20:46.935968+0000","last_active":"2023-09-23T20:26:50.945502+0000","last_peered":"2023-09-23T20:26:50.945502+0000","last_clean":"2023-09-23T20:26:50.945502+0000","last_became_active":"2023-09-23T20:20:46.935618+0000","last_became_peered":"2023-09-23T20:20:46.935618+0000","last_unstale":"2023-09-23T20:26:50.945502+0000","last_undegraded":"2023-09-23T20:26:50.945502+0000","last_fullsized":"2023-09-23T20:26:50.945502+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:42:59.867176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":555,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-09-23T20:26:43.980737+0000","last_change":"2023-09-23T20:25:21.167520+0000","last_active":"2023-09-23T20:26:43.980737+0000","last_peered":"2023-09-23T20:26:43.980737+0000","last_clean":"2023-09-23T20:26:43.980737+0000","last_became_active":"2023-09-23T20:25:21.166811+0000","last_became_peered":"2023-09-23T20:25:21.166811+0000","last_unstale":"2023-09-23T20:26:43.980737+0000","last_undegraded":"2023-09-23T20:26:43.980737+0000","last_fullsized":"2023-09-23T20:26:43.980737+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:09:47.213130+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.d","version":"205'13316","reported_seq":18801,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T20:26:52.001555+0000","last_change":"2023-09-23T20:26:33.302039+0000","last_active":"2023-09-23T20:26:52.001555+0000","last_peered":"2023-09-23T20:26:52.001555+0000","last_clean":"2023-09-23T20:26:52.001555+0000","last_became_active":"2023-09-23T20:26:33.259615+0000","last_became_peered":"2023-09-23T20:26:33.259615+0000","last_unstale":"2023-09-23T20:26:52.001555+0000","last_undegraded":"2023-09-23T20:26:52.001555+0000","last_fullsized":"2023-09-23T20:26:52.001555+0000","mapping_epoch":197,"log_start":"154'6618","ondisk_log_start":"154'6618","created":111,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:22:51.846999+0000","objects_scrubbed":0,"log_size":6698,"log_dups_size":0,"ondisk_log_size":6698,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:43:31.739694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19071000,"num_objects":292,"num_object_clones":0,"num_object_copies":876,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":292,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":13316,"num_write_kb":435393,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2675,"num_bytes_recovered":61145088,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":54,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T20:26:50.954979+0000","last_change":"2023-09-23T20:25:03.000957+0000","last_active":"2023-09-23T20:26:50.954979+0000","last_peered":"2023-09-23T20:26:50.954979+0000","last_clean":"2023-09-23T20:26:50.954979+0000","last_became_active":"2023-09-23T20:25:02.939903+0000","last_became_peered":"2023-09-23T20:25:02.939903+0000","last_unstale":"2023-09-23T20:26:50.954979+0000","last_undegraded":"2023-09-23T20:26:50.954979+0000","last_fullsized":"2023-09-23T20:26:50.954979+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:51:14.981461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":181,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-09-23T20:26:43.977082+0000","last_change":"2023-09-23T20:25:02.929447+0000","last_active":"2023-09-23T20:26:43.977082+0000","last_peered":"2023-09-23T20:26:43.977082+0000","last_clean":"2023-09-23T20:26:43.977082+0000","last_became_active":"2023-09-23T20:25:02.929173+0000","last_became_peered":"2023-09-23T20:25:02.929173+0000","last_unstale":"2023-09-23T20:26:43.977082+0000","last_undegraded":"2023-09-23T20:26:43.977082+0000","last_fullsized":"2023-09-23T20:26:43.977082+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:32:09.280647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":555,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-09-23T20:26:43.980672+0000","last_change":"2023-09-23T20:25:21.166037+0000","last_active":"2023-09-23T20:26:43.980672+0000","last_peered":"2023-09-23T20:26:43.980672+0000","last_clean":"2023-09-23T20:26:43.980672+0000","last_became_active":"2023-09-23T20:25:21.165726+0000","last_became_peered":"2023-09-23T20:25:21.165726+0000","last_unstale":"2023-09-23T20:26:43.980672+0000","last_undegraded":"2023-09-23T20:26:43.980672+0000","last_fullsized":"2023-09-23T20:26:43.980672+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:09:47.213130+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":22,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.c","version":"205'13306","reported_seq":13410,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T20:26:55.026997+0000","last_change":"2023-09-23T20:22:52.866632+0000","last_active":"2023-09-23T20:26:55.026997+0000","last_peered":"2023-09-23T20:26:55.026997+0000","last_clean":"2023-09-23T20:26:55.026997+0000","last_became_active":"2023-09-23T20:22:52.866356+0000","last_became_peered":"2023-09-23T20:22:52.866356+0000","last_unstale":"2023-09-23T20:26:55.026997+0000","last_undegraded":"2023-09-23T20:26:55.026997+0000","last_fullsized":"2023-09-23T20:26:55.026997+0000","mapping_epoch":111,"log_start":"154'6573","ondisk_log_start":"154'6573","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:22:51.846999+0000","objects_scrubbed":0,"log_size":6733,"log_dups_size":0,"ondisk_log_size":6733,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:10:12.356062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":7733248,"num_objects":118,"num_object_clones":0,"num_object_copies":354,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":118,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13306,"num_write_kb":429568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":165,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T20:26:50.969080+0000","last_change":"2023-09-23T20:20:46.937163+0000","last_active":"2023-09-23T20:26:50.969080+0000","last_peered":"2023-09-23T20:26:50.969080+0000","last_clean":"2023-09-23T20:26:50.969080+0000","last_became_active":"2023-09-23T20:20:46.936757+0000","last_became_peered":"2023-09-23T20:20:46.936757+0000","last_unstale":"2023-09-23T20:26:50.969080+0000","last_undegraded":"2023-09-23T20:26:50.969080+0000","last_fullsized":"2023-09-23T20:26:50.969080+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:09.401379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":191,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-09-23T20:26:43.976995+0000","last_change":"2023-09-23T20:25:39.624636+0000","last_active":"2023-09-23T20:26:43.976995+0000","last_peered":"2023-09-23T20:26:43.976995+0000","last_clean":"2023-09-23T20:26:43.976995+0000","last_became_active":"2023-09-23T20:25:39.624297+0000","last_became_peered":"2023-09-23T20:25:39.624297+0000","last_unstale":"2023-09-23T20:26:43.976995+0000","last_undegraded":"2023-09-23T20:26:43.976995+0000","last_fullsized":"2023-09-23T20:26:43.976995+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:33:47.448737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.f","version":"203'12905","reported_seq":13005,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-09-23T20:26:50.574128+0000","last_change":"2023-09-23T20:22:52.872510+0000","last_active":"2023-09-23T20:26:50.574128+0000","last_peered":"2023-09-23T20:26:50.574128+0000","last_clean":"2023-09-23T20:26:50.574128+0000","last_became_active":"2023-09-23T20:22:52.872179+0000","last_became_peered":"2023-09-23T20:22:52.872179+0000","last_unstale":"2023-09-23T20:26:50.574128+0000","last_undegraded":"2023-09-23T20:26:50.574128+0000","last_fullsized":"2023-09-23T20:26:50.574128+0000","mapping_epoch":111,"log_start":"150'5395","ondisk_log_start":"150'5395","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:22:51.846999+0000","objects_scrubbed":0,"log_size":7510,"log_dups_size":0,"ondisk_log_size":7510,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:53.012272+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":21954560,"num_objects":335,"num_object_clones":0,"num_object_copies":1005,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":335,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12905,"num_write_kb":423680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":182,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-09-23T20:26:43.976528+0000","last_change":"2023-09-23T20:25:39.628097+0000","last_active":"2023-09-23T20:26:43.976528+0000","last_peered":"2023-09-23T20:26:43.976528+0000","last_clean":"2023-09-23T20:26:43.976528+0000","last_became_active":"2023-09-23T20:25:39.627806+0000","last_became_peered":"2023-09-23T20:25:39.627806+0000","last_unstale":"2023-09-23T20:26:43.976528+0000","last_undegraded":"2023-09-23T20:26:43.976528+0000","last_fullsized":"2023-09-23T20:26:43.976528+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:53:44.398854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":161,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-09-23T20:26:43.977086+0000","last_change":"2023-09-23T20:20:46.939720+0000","last_active":"2023-09-23T20:26:43.977086+0000","last_peered":"2023-09-23T20:26:43.977086+0000","last_clean":"2023-09-23T20:26:43.977086+0000","last_became_active":"2023-09-23T20:20:46.939357+0000","last_became_peered":"2023-09-23T20:20:46.939357+0000","last_unstale":"2023-09-23T20:26:43.977086+0000","last_undegraded":"2023-09-23T20:26:43.977086+0000","last_fullsized":"2023-09-23T20:26:43.977086+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:56.613212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":555,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-09-23T20:26:43.980849+0000","last_change":"2023-09-23T20:25:21.167297+0000","last_active":"2023-09-23T20:26:43.980849+0000","last_peered":"2023-09-23T20:26:43.980849+0000","last_clean":"2023-09-23T20:26:43.980849+0000","last_became_active":"2023-09-23T20:25:21.167006+0000","last_became_peered":"2023-09-23T20:25:21.167006+0000","last_unstale":"2023-09-23T20:26:43.980849+0000","last_undegraded":"2023-09-23T20:26:43.980849+0000","last_fullsized":"2023-09-23T20:26:43.980849+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:09:47.213130+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.e","version":"203'12949","reported_seq":13048,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-09-23T20:26:50.608921+0000","last_change":"2023-09-23T20:22:52.871704+0000","last_active":"2023-09-23T20:26:50.608921+0000","last_peered":"2023-09-23T20:26:50.608921+0000","last_clean":"2023-09-23T20:26:50.608921+0000","last_became_active":"2023-09-23T20:22:52.871267+0000","last_became_peered":"2023-09-23T20:22:52.871267+0000","last_unstale":"2023-09-23T20:26:50.608921+0000","last_undegraded":"2023-09-23T20:26:50.608921+0000","last_fullsized":"2023-09-23T20:26:50.608921+0000","mapping_epoch":111,"log_start":"154'6404","ondisk_log_start":"154'6404","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:22:51.846999+0000","objects_scrubbed":0,"log_size":6545,"log_dups_size":0,"ondisk_log_size":6545,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:50:07.619801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22085632,"num_objects":337,"num_object_clones":0,"num_object_copies":1011,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":337,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12949,"num_write_kb":425152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":50,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T20:26:50.956270+0000","last_change":"2023-09-23T20:25:02.978173+0000","last_active":"2023-09-23T20:26:50.956270+0000","last_peered":"2023-09-23T20:26:50.956270+0000","last_clean":"2023-09-23T20:26:50.956270+0000","last_became_active":"2023-09-23T20:25:02.977856+0000","last_became_peered":"2023-09-23T20:25:02.977856+0000","last_unstale":"2023-09-23T20:26:50.956270+0000","last_undegraded":"2023-09-23T20:26:50.956270+0000","last_fullsized":"2023-09-23T20:26:50.956270+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:19.861943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":64,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T20:26:50.943487+0000","last_change":"2023-09-23T20:25:02.924911+0000","last_active":"2023-09-23T20:26:50.943487+0000","last_peered":"2023-09-23T20:26:50.943487+0000","last_clean":"2023-09-23T20:26:50.943487+0000","last_became_active":"2023-09-23T20:25:02.924586+0000","last_became_peered":"2023-09-23T20:25:02.924586+0000","last_unstale":"2023-09-23T20:26:50.943487+0000","last_undegraded":"2023-09-23T20:26:50.943487+0000","last_fullsized":"2023-09-23T20:26:50.943487+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:24:44.176766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":555,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-09-23T20:26:43.981463+0000","last_change":"2023-09-23T20:25:21.167164+0000","last_active":"2023-09-23T20:26:43.981463+0000","last_peered":"2023-09-23T20:26:43.981463+0000","last_clean":"2023-09-23T20:26:43.981463+0000","last_became_active":"2023-09-23T20:25:21.166869+0000","last_became_peered":"2023-09-23T20:25:21.166869+0000","last_unstale":"2023-09-23T20:26:43.981463+0000","last_undegraded":"2023-09-23T20:26:43.981463+0000","last_fullsized":"2023-09-23T20:26:43.981463+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:09:47.213130+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"86'119","reported_seq":538,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T20:26:50.943498+0000","last_change":"2023-09-23T20:20:46.937071+0000","last_active":"2023-09-23T20:26:50.943498+0000","last_peered":"2023-09-23T20:26:50.943498+0000","last_clean":"2023-09-23T20:26:50.943498+0000","last_became_active":"2023-09-23T20:20:46.935348+0000","last_became_peered":"2023-09-23T20:20:46.935348+0000","last_unstale":"2023-09-23T20:26:50.943498+0000","last_undegraded":"2023-09-23T20:26:50.943498+0000","last_fullsized":"2023-09-23T20:26:50.943498+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":119,"log_dups_size":0,"ondisk_log_size":119,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:09.829030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032705499999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":523,"num_read_kb":3503,"num_write":136,"num_write_kb":3128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":219,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-09-23T20:26:43.978754+0000","last_change":"2023-09-23T20:25:39.637160+0000","last_active":"2023-09-23T20:26:43.978754+0000","last_peered":"2023-09-23T20:26:43.978754+0000","last_clean":"2023-09-23T20:26:43.978754+0000","last_became_active":"2023-09-23T20:25:39.636860+0000","last_became_peered":"2023-09-23T20:25:39.636860+0000","last_unstale":"2023-09-23T20:26:43.978754+0000","last_undegraded":"2023-09-23T20:26:43.978754+0000","last_fullsized":"2023-09-23T20:26:43.978754+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:21:47.860063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":130,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T20:26:50.942697+0000","last_change":"2023-09-23T20:20:46.937104+0000","last_active":"2023-09-23T20:26:50.942697+0000","last_peered":"2023-09-23T20:26:50.942697+0000","last_clean":"2023-09-23T20:26:50.942697+0000","last_became_active":"2023-09-23T20:20:46.936820+0000","last_became_peered":"2023-09-23T20:20:46.936820+0000","last_unstale":"2023-09-23T20:26:50.942697+0000","last_undegraded":"2023-09-23T20:26:50.942697+0000","last_fullsized":"2023-09-23T20:26:50.942697+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:22:19.879310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.3","version":"205'7654","reported_seq":7769,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T20:26:54.844617+0000","last_change":"2023-09-23T20:23:20.060302+0000","last_active":"2023-09-23T20:26:54.844617+0000","last_peered":"2023-09-23T20:26:54.844617+0000","last_clean":"2023-09-23T20:26:54.844617+0000","last_became_active":"2023-09-23T20:23:11.059880+0000","last_became_peered":"2023-09-23T20:23:11.059880+0000","last_unstale":"2023-09-23T20:26:54.844617+0000","last_undegraded":"2023-09-23T20:26:54.844617+0000","last_fullsized":"2023-09-23T20:26:54.844617+0000","mapping_epoch":111,"log_start":"119'1903","ondisk_log_start":"119'1903","created":111,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:22:51.846999+0000","objects_scrubbed":0,"log_size":5751,"log_dups_size":0,"ondisk_log_size":5751,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":6782976,"num_objects":103,"num_object_clones":0,"num_object_copies":309,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":103,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6751,"num_write_kb":219360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":554,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-09-23T20:26:43.981631+0000","last_change":"2023-09-23T20:25:21.160602+0000","last_active":"2023-09-23T20:26:43.981631+0000","last_peered":"2023-09-23T20:26:43.981631+0000","last_clean":"2023-09-23T20:26:43.981631+0000","last_became_active":"2023-09-23T20:25:21.160325+0000","last_became_peered":"2023-09-23T20:25:21.160325+0000","last_unstale":"2023-09-23T20:26:43.981631+0000","last_undegraded":"2023-09-23T20:26:43.981631+0000","last_fullsized":"2023-09-23T20:26:43.981631+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:09:47.213130+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.5","version":"205'7545","reported_seq":7660,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T20:26:54.858720+0000","last_change":"2023-09-23T20:23:11.152272+0000","last_active":"2023-09-23T20:26:54.858720+0000","last_peered":"2023-09-23T20:26:54.858720+0000","last_clean":"2023-09-23T20:26:54.858720+0000","last_became_active":"2023-09-23T20:23:11.055735+0000","last_became_peered":"2023-09-23T20:23:11.055735+0000","last_unstale":"2023-09-23T20:26:54.858720+0000","last_undegraded":"2023-09-23T20:26:54.858720+0000","last_fullsized":"2023-09-23T20:26:54.858720+0000","mapping_epoch":111,"log_start":"119'1795","ondisk_log_start":"119'1795","created":111,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:22:51.846999+0000","objects_scrubbed":0,"log_size":5750,"log_dups_size":0,"ondisk_log_size":5750,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":2490368,"num_objects":38,"num_object_clones":0,"num_object_copies":114,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":38,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6662,"num_write_kb":214400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"86'106","reported_seq":827,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-09-23T20:26:43.978497+0000","last_change":"2023-09-23T20:20:46.939468+0000","last_active":"2023-09-23T20:26:43.978497+0000","last_peered":"2023-09-23T20:26:43.978497+0000","last_clean":"2023-09-23T20:26:43.978497+0000","last_became_active":"2023-09-23T20:20:46.938954+0000","last_became_peered":"2023-09-23T20:20:46.938954+0000","last_unstale":"2023-09-23T20:26:43.978497+0000","last_undegraded":"2023-09-23T20:26:43.978497+0000","last_fullsized":"2023-09-23T20:26:43.978497+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":106,"log_dups_size":0,"ondisk_log_size":106,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:01:43.761064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00099302599999999993,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1105,"num_read_kb":1689,"num_write":100,"num_write_kb":404,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":162,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-09-23T20:26:43.976662+0000","last_change":"2023-09-23T20:20:14.887345+0000","last_active":"2023-09-23T20:26:43.976662+0000","last_peered":"2023-09-23T20:26:43.976662+0000","last_clean":"2023-09-23T20:26:43.976662+0000","last_became_active":"2023-09-23T20:20:14.886970+0000","last_became_peered":"2023-09-23T20:20:14.886970+0000","last_unstale":"2023-09-23T20:26:43.976662+0000","last_undegraded":"2023-09-23T20:26:43.976662+0000","last_fullsized":"2023-09-23T20:26:43.976662+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:00:38.575926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":196,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T20:26:50.943421+0000","last_change":"2023-09-23T20:25:39.631133+0000","last_active":"2023-09-23T20:26:50.943421+0000","last_peered":"2023-09-23T20:26:50.943421+0000","last_clean":"2023-09-23T20:26:50.943421+0000","last_became_active":"2023-09-23T20:25:39.630779+0000","last_became_peered":"2023-09-23T20:25:39.630779+0000","last_unstale":"2023-09-23T20:26:50.943421+0000","last_undegraded":"2023-09-23T20:26:50.943421+0000","last_fullsized":"2023-09-23T20:26:50.943421+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:39:52.635649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":555,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-09-23T20:26:43.981022+0000","last_change":"2023-09-23T20:25:21.167528+0000","last_active":"2023-09-23T20:26:43.981022+0000","last_peered":"2023-09-23T20:26:43.981022+0000","last_clean":"2023-09-23T20:26:43.981022+0000","last_became_active":"2023-09-23T20:25:21.167249+0000","last_became_peered":"2023-09-23T20:25:21.167249+0000","last_unstale":"2023-09-23T20:26:43.981022+0000","last_undegraded":"2023-09-23T20:26:43.981022+0000","last_fullsized":"2023-09-23T20:26:43.981022+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:09:47.213130+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.6","version":"203'7427","reported_seq":10302,"reported_epoch":203,"state":"active+recovering+undersized+remapped","last_fresh":"2023-09-23T20:26:50.609027+0000","last_change":"2023-09-23T20:26:50.327314+0000","last_active":"2023-09-23T20:26:50.609027+0000","last_peered":"2023-09-23T20:26:50.609027+0000","last_clean":"2023-09-23T20:24:51.979668+0000","last_became_active":"2023-09-23T20:25:39.704394+0000","last_became_peered":"2023-09-23T20:25:39.704394+0000","last_unstale":"2023-09-23T20:26:50.609027+0000","last_undegraded":"2023-09-23T20:26:50.609027+0000","last_fullsized":"2023-09-23T20:25:39.614489+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":111,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"119'1954","last_scrub_stamp":"2023-09-23T20:23:12.084143+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:23:12.084143+0000","objects_scrubbed":1042,"log_size":7427,"log_dups_size":0,"ondisk_log_size":7427,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:27:04.787533+0000","scrub_duration":0.23299162700000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":10027008,"num_objects":153,"num_object_clones":0,"num_object_copies":459,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":153,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5477,"num_write_kb":146816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1353,"num_bytes_recovered":87162880,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[5,7],"avail_no_missing":["5","7"],"object_location_counts":[{"shards":"5,7","objects":153}],"blocked_by":[],"up_primary":2,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"86'130","reported_seq":514,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T20:26:50.948841+0000","last_change":"2023-09-23T20:25:39.637458+0000","last_active":"2023-09-23T20:26:50.948841+0000","last_peered":"2023-09-23T20:26:50.948841+0000","last_clean":"2023-09-23T20:26:50.948841+0000","last_became_active":"2023-09-23T20:25:39.636773+0000","last_became_peered":"2023-09-23T20:25:39.636773+0000","last_unstale":"2023-09-23T20:26:50.948841+0000","last_undegraded":"2023-09-23T20:26:50.948841+0000","last_fullsized":"2023-09-23T20:26:50.948841+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:00:32.810564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011300049999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":512,"num_read_kb":1820,"num_write":133,"num_write_kb":1347,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":163,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T20:26:50.942597+0000","last_change":"2023-09-23T20:25:39.632283+0000","last_active":"2023-09-23T20:26:50.942597+0000","last_peered":"2023-09-23T20:26:50.942597+0000","last_clean":"2023-09-23T20:26:50.942597+0000","last_became_active":"2023-09-23T20:25:39.631819+0000","last_became_peered":"2023-09-23T20:25:39.631819+0000","last_unstale":"2023-09-23T20:26:50.942597+0000","last_undegraded":"2023-09-23T20:26:50.942597+0000","last_fullsized":"2023-09-23T20:26:50.942597+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:25:21.815555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":173,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T20:26:50.943247+0000","last_change":"2023-09-23T20:20:46.936524+0000","last_active":"2023-09-23T20:26:50.943247+0000","last_peered":"2023-09-23T20:26:50.943247+0000","last_clean":"2023-09-23T20:26:50.943247+0000","last_became_active":"2023-09-23T20:20:46.936267+0000","last_became_peered":"2023-09-23T20:20:46.936267+0000","last_unstale":"2023-09-23T20:26:50.943247+0000","last_undegraded":"2023-09-23T20:26:50.943247+0000","last_fullsized":"2023-09-23T20:26:50.943247+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:32:20.984521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"86'194","reported_seq":851,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-09-23T20:26:43.978431+0000","last_change":"2023-09-23T20:25:40.618665+0000","last_active":"2023-09-23T20:26:43.978431+0000","last_peered":"2023-09-23T20:26:43.978431+0000","last_clean":"2023-09-23T20:26:43.978431+0000","last_became_active":"2023-09-23T20:25:40.618087+0000","last_became_peered":"2023-09-23T20:25:40.618087+0000","last_unstale":"2023-09-23T20:26:43.978431+0000","last_undegraded":"2023-09-23T20:26:43.978431+0000","last_fullsized":"2023-09-23T20:26:43.978431+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":194,"log_dups_size":0,"ondisk_log_size":194,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:07:16.114968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00037995999999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1025,"num_read_kb":1201,"num_write":173,"num_write_kb":1276,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":556,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-09-23T20:26:43.980776+0000","last_change":"2023-09-23T20:25:21.159705+0000","last_active":"2023-09-23T20:26:43.980776+0000","last_peered":"2023-09-23T20:26:43.980776+0000","last_clean":"2023-09-23T20:26:43.980776+0000","last_became_active":"2023-09-23T20:25:21.159200+0000","last_became_peered":"2023-09-23T20:25:21.159200+0000","last_unstale":"2023-09-23T20:26:43.980776+0000","last_undegraded":"2023-09-23T20:26:43.980776+0000","last_fullsized":"2023-09-23T20:26:43.980776+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:09:47.213130+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":50,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T20:26:50.958957+0000","last_change":"2023-09-23T20:25:02.984205+0000","last_active":"2023-09-23T20:26:50.958957+0000","last_peered":"2023-09-23T20:26:50.958957+0000","last_clean":"2023-09-23T20:26:50.958957+0000","last_became_active":"2023-09-23T20:25:02.983758+0000","last_became_peered":"2023-09-23T20:25:02.983758+0000","last_unstale":"2023-09-23T20:26:50.958957+0000","last_undegraded":"2023-09-23T20:26:50.958957+0000","last_fullsized":"2023-09-23T20:26:50.958957+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:35:14.573101+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":147,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T20:26:50.966341+0000","last_change":"2023-09-23T20:12:45.609829+0000","last_active":"2023-09-23T20:26:50.966341+0000","last_peered":"2023-09-23T20:26:50.966341+0000","last_clean":"2023-09-23T20:26:50.966341+0000","last_became_active":"2023-09-23T20:12:45.609467+0000","last_became_peered":"2023-09-23T20:12:45.609467+0000","last_unstale":"2023-09-23T20:26:50.966341+0000","last_undegraded":"2023-09-23T20:26:50.966341+0000","last_fullsized":"2023-09-23T20:26:50.966341+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:27:31.490555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.4","version":"203'7555","reported_seq":8521,"reported_epoch":203,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:26:50.474489+0000","last_change":"2023-09-23T20:25:40.829173+0000","last_active":"2023-09-23T20:26:50.474489+0000","last_peered":"2023-09-23T20:26:50.474489+0000","last_clean":"2023-09-23T20:24:52.062405+0000","last_became_active":"2023-09-23T20:25:39.693392+0000","last_became_peered":"2023-09-23T20:25:39.693392+0000","last_unstale":"2023-09-23T20:26:50.474489+0000","last_undegraded":"2023-09-23T20:25:40.828975+0000","last_fullsized":"2023-09-23T20:25:39.617392+0000","mapping_epoch":176,"log_start":"119'1846","ondisk_log_start":"119'1846","created":111,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"119'1973","last_scrub_stamp":"2023-09-23T20:23:13.408658+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:23:13.408658+0000","objects_scrubbed":1060,"log_size":5709,"log_dups_size":0,"ondisk_log_size":5709,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:18:48.846593+0000","scrub_duration":0.28811426499999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":11927552,"num_objects":182,"num_object_clones":0,"num_object_copies":546,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":604,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":182,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5586,"num_write_kb":150656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":389,"num_bytes_recovered":7798784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,5],"avail_no_missing":["7","5"],"object_location_counts":[{"shards":"5,7","objects":182}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":555,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-09-23T20:26:43.981016+0000","last_change":"2023-09-23T20:25:21.172709+0000","last_active":"2023-09-23T20:26:43.981016+0000","last_peered":"2023-09-23T20:26:43.981016+0000","last_clean":"2023-09-23T20:26:43.981016+0000","last_became_active":"2023-09-23T20:25:21.171557+0000","last_became_peered":"2023-09-23T20:25:21.171557+0000","last_unstale":"2023-09-23T20:26:43.981016+0000","last_undegraded":"2023-09-23T20:26:43.981016+0000","last_fullsized":"2023-09-23T20:26:43.981016+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:09:47.213130+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.7","version":"203'7598","reported_seq":7773,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-09-23T20:26:50.546985+0000","last_change":"2023-09-23T20:23:16.284169+0000","last_active":"2023-09-23T20:26:50.546985+0000","last_peered":"2023-09-23T20:26:50.546985+0000","last_clean":"2023-09-23T20:26:50.546985+0000","last_became_active":"2023-09-23T20:23:11.069066+0000","last_became_peered":"2023-09-23T20:23:11.069066+0000","last_unstale":"2023-09-23T20:26:50.546985+0000","last_undegraded":"2023-09-23T20:26:50.546985+0000","last_fullsized":"2023-09-23T20:26:50.546985+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":111,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"120'2196","last_scrub_stamp":"2023-09-23T20:23:16.284112+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:23:16.284112+0000","objects_scrubbed":1268,"log_size":7598,"log_dups_size":0,"ondisk_log_size":7598,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:41:59.721972+0000","scrub_duration":0.33229684999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":11730944,"num_objects":179,"num_object_clones":0,"num_object_copies":537,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":179,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5411,"num_write_kb":138304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.0","version":"86'80","reported_seq":449,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-09-23T20:26:43.980670+0000","last_change":"2023-09-23T20:20:46.950784+0000","last_active":"2023-09-23T20:26:43.980670+0000","last_peered":"2023-09-23T20:26:43.980670+0000","last_clean":"2023-09-23T20:26:43.980670+0000","last_became_active":"2023-09-23T20:20:46.950071+0000","last_became_peered":"2023-09-23T20:20:46.950071+0000","last_unstale":"2023-09-23T20:26:43.980670+0000","last_undegraded":"2023-09-23T20:26:43.980670+0000","last_fullsized":"2023-09-23T20:26:43.980670+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":80,"log_dups_size":0,"ondisk_log_size":80,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:11:06.812404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035115,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":395,"num_read_kb":1370,"num_write":66,"num_write_kb":1061,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":167,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T20:26:50.944433+0000","last_change":"2023-09-23T20:25:02.942709+0000","last_active":"2023-09-23T20:26:50.944433+0000","last_peered":"2023-09-23T20:26:50.944433+0000","last_clean":"2023-09-23T20:26:50.944433+0000","last_became_active":"2023-09-23T20:25:02.942323+0000","last_became_peered":"2023-09-23T20:25:02.942323+0000","last_unstale":"2023-09-23T20:26:50.944433+0000","last_undegraded":"2023-09-23T20:26:50.944433+0000","last_fullsized":"2023-09-23T20:26:50.944433+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:26:12.039101+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":183,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T20:26:50.946381+0000","last_change":"2023-09-23T20:25:39.641452+0000","last_active":"2023-09-23T20:26:50.946381+0000","last_peered":"2023-09-23T20:26:50.946381+0000","last_clean":"2023-09-23T20:26:50.946381+0000","last_became_active":"2023-09-23T20:25:39.638987+0000","last_became_peered":"2023-09-23T20:25:39.638987+0000","last_unstale":"2023-09-23T20:26:50.946381+0000","last_undegraded":"2023-09-23T20:26:50.946381+0000","last_fullsized":"2023-09-23T20:26:50.946381+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:33:59.955461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":555,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-09-23T20:26:43.981103+0000","last_change":"2023-09-23T20:25:21.165430+0000","last_active":"2023-09-23T20:26:43.981103+0000","last_peered":"2023-09-23T20:26:43.981103+0000","last_clean":"2023-09-23T20:26:43.981103+0000","last_became_active":"2023-09-23T20:25:21.165109+0000","last_became_peered":"2023-09-23T20:25:21.165109+0000","last_unstale":"2023-09-23T20:26:43.981103+0000","last_undegraded":"2023-09-23T20:26:43.981103+0000","last_fullsized":"2023-09-23T20:26:43.981103+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:09:47.213130+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.5","version":"86'102","reported_seq":709,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-09-23T20:26:43.981216+0000","last_change":"2023-09-23T20:20:14.881975+0000","last_active":"2023-09-23T20:26:43.981216+0000","last_peered":"2023-09-23T20:26:43.981216+0000","last_clean":"2023-09-23T20:26:43.981216+0000","last_became_active":"2023-09-23T20:20:14.881544+0000","last_became_peered":"2023-09-23T20:20:14.881544+0000","last_unstale":"2023-09-23T20:26:43.981216+0000","last_undegraded":"2023-09-23T20:26:43.981216+0000","last_fullsized":"2023-09-23T20:26:43.981216+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":102,"log_dups_size":0,"ondisk_log_size":102,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:34.594972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030656500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":963,"num_read_kb":1234,"num_write":60,"num_write_kb":225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":162,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T20:26:50.943581+0000","last_change":"2023-09-23T20:25:39.631751+0000","last_active":"2023-09-23T20:26:50.943581+0000","last_peered":"2023-09-23T20:26:50.943581+0000","last_clean":"2023-09-23T20:26:50.943581+0000","last_became_active":"2023-09-23T20:25:39.631136+0000","last_became_peered":"2023-09-23T20:25:39.631136+0000","last_unstale":"2023-09-23T20:26:50.943581+0000","last_undegraded":"2023-09-23T20:26:50.943581+0000","last_fullsized":"2023-09-23T20:26:50.943581+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:08:17.020722+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":135,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T20:26:50.945708+0000","last_change":"2023-09-23T20:20:14.888874+0000","last_active":"2023-09-23T20:26:50.945708+0000","last_peered":"2023-09-23T20:26:50.945708+0000","last_clean":"2023-09-23T20:26:50.945708+0000","last_became_active":"2023-09-23T20:20:14.887724+0000","last_became_peered":"2023-09-23T20:20:14.887724+0000","last_unstale":"2023-09-23T20:26:50.945708+0000","last_undegraded":"2023-09-23T20:26:50.945708+0000","last_fullsized":"2023-09-23T20:26:50.945708+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:09:24.510326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.2","version":"205'7586","reported_seq":7749,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T20:26:55.345330+0000","last_change":"2023-09-23T20:23:12.771560+0000","last_active":"2023-09-23T20:26:55.345330+0000","last_peered":"2023-09-23T20:26:55.345330+0000","last_clean":"2023-09-23T20:26:55.345330+0000","last_became_active":"2023-09-23T20:23:11.058929+0000","last_became_peered":"2023-09-23T20:23:11.058929+0000","last_unstale":"2023-09-23T20:26:55.345330+0000","last_undegraded":"2023-09-23T20:26:55.345330+0000","last_fullsized":"2023-09-23T20:26:55.345330+0000","mapping_epoch":111,"log_start":"114'20","ondisk_log_start":"114'20","created":111,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"119'1924","last_scrub_stamp":"2023-09-23T20:23:12.771453+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:23:12.771453+0000","objects_scrubbed":1010,"log_size":7566,"log_dups_size":0,"ondisk_log_size":7566,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:49:31.924585+0000","scrub_duration":0.26141313799999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":3276800,"num_objects":50,"num_object_clones":0,"num_object_copies":150,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":50,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5670,"num_write_kb":150720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":555,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-09-23T20:26:43.981021+0000","last_change":"2023-09-23T20:25:21.200271+0000","last_active":"2023-09-23T20:26:43.981021+0000","last_peered":"2023-09-23T20:26:43.981021+0000","last_clean":"2023-09-23T20:26:43.981021+0000","last_became_active":"2023-09-23T20:25:21.197719+0000","last_became_peered":"2023-09-23T20:25:21.197719+0000","last_unstale":"2023-09-23T20:26:43.981021+0000","last_undegraded":"2023-09-23T20:26:43.981021+0000","last_fullsized":"2023-09-23T20:26:43.981021+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:09:47.213130+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":54,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T20:26:50.957460+0000","last_change":"2023-09-23T20:25:03.007736+0000","last_active":"2023-09-23T20:26:50.957460+0000","last_peered":"2023-09-23T20:26:50.957460+0000","last_clean":"2023-09-23T20:26:50.957460+0000","last_became_active":"2023-09-23T20:25:02.973689+0000","last_became_peered":"2023-09-23T20:25:02.973689+0000","last_unstale":"2023-09-23T20:26:50.957460+0000","last_undegraded":"2023-09-23T20:26:50.957460+0000","last_fullsized":"2023-09-23T20:26:50.957460+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:37:25.643774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"86'128","reported_seq":529,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T20:26:50.943739+0000","last_change":"2023-09-23T20:20:46.939256+0000","last_active":"2023-09-23T20:26:50.943739+0000","last_peered":"2023-09-23T20:26:50.943739+0000","last_clean":"2023-09-23T20:26:50.943739+0000","last_became_active":"2023-09-23T20:20:46.938756+0000","last_became_peered":"2023-09-23T20:20:46.938756+0000","last_unstale":"2023-09-23T20:26:50.943739+0000","last_undegraded":"2023-09-23T20:26:50.943739+0000","last_fullsized":"2023-09-23T20:26:50.943739+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:52:18.161436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000293202,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":579,"num_read_kb":1473,"num_write":96,"num_write_kb":1062,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":50,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T20:26:50.956336+0000","last_change":"2023-09-23T20:25:02.971740+0000","last_active":"2023-09-23T20:26:50.956336+0000","last_peered":"2023-09-23T20:26:50.956336+0000","last_clean":"2023-09-23T20:26:50.956336+0000","last_became_active":"2023-09-23T20:25:02.970412+0000","last_became_peered":"2023-09-23T20:25:02.970412+0000","last_unstale":"2023-09-23T20:26:50.956336+0000","last_undegraded":"2023-09-23T20:26:50.956336+0000","last_fullsized":"2023-09-23T20:26:50.956336+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:53:17.247216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.1","version":"203'7368","reported_seq":8942,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-09-23T20:26:50.513562+0000","last_change":"2023-09-23T20:25:41.656824+0000","last_active":"2023-09-23T20:26:50.513562+0000","last_peered":"2023-09-23T20:26:50.513562+0000","last_clean":"2023-09-23T20:26:50.513562+0000","last_became_active":"2023-09-23T20:25:41.621039+0000","last_became_peered":"2023-09-23T20:25:41.621039+0000","last_unstale":"2023-09-23T20:26:50.513562+0000","last_undegraded":"2023-09-23T20:26:50.513562+0000","last_fullsized":"2023-09-23T20:26:50.513562+0000","mapping_epoch":178,"log_start":"0'0","ondisk_log_start":"0'0","created":111,"last_epoch_clean":179,"parent":"0.0","parent_split_bits":0,"last_scrub":"119'1889","last_scrub_stamp":"2023-09-23T20:23:11.363674+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:23:11.363674+0000","objects_scrubbed":958,"log_size":7368,"log_dups_size":0,"ondisk_log_size":7368,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:06:09.763416+0000","scrub_duration":0.26718564099999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":11468800,"num_objects":175,"num_object_clones":0,"num_object_copies":525,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":175,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5487,"num_write_kb":150528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":692,"num_bytes_recovered":20447232,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"90'161","reported_seq":555,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-09-23T20:26:43.980884+0000","last_change":"2023-09-23T20:25:21.172983+0000","last_active":"2023-09-23T20:26:43.980884+0000","last_peered":"2023-09-23T20:26:43.980884+0000","last_clean":"2023-09-23T20:26:43.980884+0000","last_became_active":"2023-09-23T20:25:21.172575+0000","last_became_peered":"2023-09-23T20:25:21.172575+0000","last_unstale":"2023-09-23T20:26:43.980884+0000","last_undegraded":"2023-09-23T20:26:43.980884+0000","last_fullsized":"2023-09-23T20:26:43.980884+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:09:47.213130+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.7","version":"86'117","reported_seq":724,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T20:26:50.968755+0000","last_change":"2023-09-23T20:25:40.617386+0000","last_active":"2023-09-23T20:26:50.968755+0000","last_peered":"2023-09-23T20:26:50.968755+0000","last_clean":"2023-09-23T20:26:50.968755+0000","last_became_active":"2023-09-23T20:25:40.616747+0000","last_became_peered":"2023-09-23T20:25:40.616747+0000","last_unstale":"2023-09-23T20:26:50.968755+0000","last_undegraded":"2023-09-23T20:26:50.968755+0000","last_fullsized":"2023-09-23T20:26:50.968755+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:46:44.551561+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046891400000000002,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":893,"num_read_kb":922,"num_write":82,"num_write_kb":4193,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":155,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T20:26:50.946314+0000","last_change":"2023-09-23T20:20:46.937163+0000","last_active":"2023-09-23T20:26:50.946314+0000","last_peered":"2023-09-23T20:26:50.946314+0000","last_clean":"2023-09-23T20:26:50.946314+0000","last_became_active":"2023-09-23T20:20:46.936846+0000","last_became_peered":"2023-09-23T20:20:46.936846+0000","last_unstale":"2023-09-23T20:26:50.946314+0000","last_undegraded":"2023-09-23T20:26:50.946314+0000","last_fullsized":"2023-09-23T20:26:50.946314+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:01:32.798855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":144,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-09-23T20:26:43.976579+0000","last_change":"2023-09-23T20:12:43.765908+0000","last_active":"2023-09-23T20:26:43.976579+0000","last_peered":"2023-09-23T20:26:43.976579+0000","last_clean":"2023-09-23T20:26:43.976579+0000","last_became_active":"2023-09-23T20:12:43.765558+0000","last_became_peered":"2023-09-23T20:12:43.765558+0000","last_unstale":"2023-09-23T20:26:43.976579+0000","last_undegraded":"2023-09-23T20:26:43.976579+0000","last_fullsized":"2023-09-23T20:26:43.976579+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:35:43.809898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.0","version":"205'7505","reported_seq":7674,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T20:26:52.934758+0000","last_change":"2023-09-23T20:25:40.633699+0000","last_active":"2023-09-23T20:26:52.934758+0000","last_peered":"2023-09-23T20:26:52.934758+0000","last_clean":"2023-09-23T20:26:52.934758+0000","last_became_active":"2023-09-23T20:25:40.618344+0000","last_became_peered":"2023-09-23T20:25:40.618344+0000","last_unstale":"2023-09-23T20:26:52.934758+0000","last_undegraded":"2023-09-23T20:26:52.934758+0000","last_fullsized":"2023-09-23T20:26:52.934758+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":111,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"119'2020","last_scrub_stamp":"2023-09-23T20:23:13.995392+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:23:13.995392+0000","objects_scrubbed":1088,"log_size":7505,"log_dups_size":0,"ondisk_log_size":7505,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:11:19.162515+0000","scrub_duration":0.22507750400000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":7995392,"num_objects":122,"num_object_clones":0,"num_object_copies":366,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":122,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5494,"num_write_kb":144896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":355,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":137,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T20:26:50.945639+0000","last_change":"2023-09-23T20:20:14.888484+0000","last_active":"2023-09-23T20:26:50.945639+0000","last_peered":"2023-09-23T20:26:50.945639+0000","last_clean":"2023-09-23T20:26:50.945639+0000","last_became_active":"2023-09-23T20:20:14.887219+0000","last_became_peered":"2023-09-23T20:20:14.887219+0000","last_unstale":"2023-09-23T20:26:50.945639+0000","last_undegraded":"2023-09-23T20:26:50.945639+0000","last_fullsized":"2023-09-23T20:26:50.945639+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:13:53.763649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":39,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T20:26:50.948818+0000","last_change":"2023-09-23T20:25:39.639268+0000","last_active":"2023-09-23T20:26:50.948818+0000","last_peered":"2023-09-23T20:26:50.948818+0000","last_clean":"2023-09-23T20:26:50.948818+0000","last_became_active":"2023-09-23T20:25:39.638922+0000","last_became_peered":"2023-09-23T20:25:39.638922+0000","last_unstale":"2023-09-23T20:26:50.948818+0000","last_undegraded":"2023-09-23T20:26:50.948818+0000","last_fullsized":"2023-09-23T20:26:50.948818+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:09:04.040098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":164,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-09-23T20:26:43.978569+0000","last_change":"2023-09-23T20:20:46.938981+0000","last_active":"2023-09-23T20:26:43.978569+0000","last_peered":"2023-09-23T20:26:43.978569+0000","last_clean":"2023-09-23T20:26:43.978569+0000","last_became_active":"2023-09-23T20:20:46.938627+0000","last_became_peered":"2023-09-23T20:20:46.938627+0000","last_unstale":"2023-09-23T20:26:43.978569+0000","last_undegraded":"2023-09-23T20:26:43.978569+0000","last_fullsized":"2023-09-23T20:26:43.978569+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:44:43.177086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":208,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-09-23T20:26:43.976796+0000","last_change":"2023-09-23T20:25:02.945142+0000","last_active":"2023-09-23T20:26:43.976796+0000","last_peered":"2023-09-23T20:26:43.976796+0000","last_clean":"2023-09-23T20:26:43.976796+0000","last_became_active":"2023-09-23T20:25:02.944653+0000","last_became_peered":"2023-09-23T20:25:02.944653+0000","last_unstale":"2023-09-23T20:26:43.976796+0000","last_undegraded":"2023-09-23T20:26:43.976796+0000","last_fullsized":"2023-09-23T20:26:43.976796+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:38:12.927982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":147,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-09-23T20:26:43.979081+0000","last_change":"2023-09-23T20:12:43.753383+0000","last_active":"2023-09-23T20:26:43.979081+0000","last_peered":"2023-09-23T20:26:43.979081+0000","last_clean":"2023-09-23T20:26:43.979081+0000","last_became_active":"2023-09-23T20:12:43.752921+0000","last_became_peered":"2023-09-23T20:12:43.752921+0000","last_unstale":"2023-09-23T20:26:43.979081+0000","last_undegraded":"2023-09-23T20:26:43.979081+0000","last_fullsized":"2023-09-23T20:26:43.979081+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:28:26.982998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":208,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-09-23T20:26:43.976703+0000","last_change":"2023-09-23T20:25:03.103656+0000","last_active":"2023-09-23T20:26:43.976703+0000","last_peered":"2023-09-23T20:26:43.976703+0000","last_clean":"2023-09-23T20:26:43.976703+0000","last_became_active":"2023-09-23T20:25:03.103311+0000","last_became_peered":"2023-09-23T20:25:03.103311+0000","last_unstale":"2023-09-23T20:26:43.976703+0000","last_undegraded":"2023-09-23T20:26:43.976703+0000","last_fullsized":"2023-09-23T20:26:43.976703+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:55:54.677030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.18","version":"203'7398","reported_seq":14848,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-09-23T20:26:50.594280+0000","last_change":"2023-09-23T20:25:15.101300+0000","last_active":"2023-09-23T20:26:50.594280+0000","last_peered":"2023-09-23T20:26:50.594280+0000","last_clean":"2023-09-23T20:26:50.594280+0000","last_became_active":"2023-09-23T20:25:15.063524+0000","last_became_peered":"2023-09-23T20:25:15.063524+0000","last_unstale":"2023-09-23T20:26:50.594280+0000","last_undegraded":"2023-09-23T20:26:50.594280+0000","last_fullsized":"2023-09-23T20:26:50.594280+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":118,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":5,"last_scrub":"120'2147","last_scrub_stamp":"2023-09-23T20:23:18.111873+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:23:18.111873+0000","objects_scrubbed":1267,"log_size":7398,"log_dups_size":0,"ondisk_log_size":7398,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:27:49.823665+0000","scrub_duration":0.17815524799999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":12320768,"num_objects":188,"num_object_clones":0,"num_object_copies":564,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":188,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5251,"num_write_kb":133504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2426,"num_bytes_recovered":140312576,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":59,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T20:26:50.955421+0000","last_change":"2023-09-23T20:25:39.629869+0000","last_active":"2023-09-23T20:26:50.955421+0000","last_peered":"2023-09-23T20:26:50.955421+0000","last_clean":"2023-09-23T20:26:50.955421+0000","last_became_active":"2023-09-23T20:25:39.629543+0000","last_became_peered":"2023-09-23T20:25:39.629543+0000","last_unstale":"2023-09-23T20:26:50.955421+0000","last_undegraded":"2023-09-23T20:26:50.955421+0000","last_fullsized":"2023-09-23T20:26:50.955421+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:24:48.220679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":59,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T20:26:50.955738+0000","last_change":"2023-09-23T20:25:39.622819+0000","last_active":"2023-09-23T20:26:50.955738+0000","last_peered":"2023-09-23T20:26:50.955738+0000","last_clean":"2023-09-23T20:26:50.955738+0000","last_became_active":"2023-09-23T20:25:39.622484+0000","last_became_peered":"2023-09-23T20:25:39.622484+0000","last_unstale":"2023-09-23T20:26:50.955738+0000","last_undegraded":"2023-09-23T20:26:50.955738+0000","last_fullsized":"2023-09-23T20:26:50.955738+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:51:53.973261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.19","version":"203'7407","reported_seq":7517,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-09-23T20:26:50.513457+0000","last_change":"2023-09-23T20:23:10.978296+0000","last_active":"2023-09-23T20:26:50.513457+0000","last_peered":"2023-09-23T20:26:50.513457+0000","last_clean":"2023-09-23T20:26:50.513457+0000","last_became_active":"2023-09-23T20:23:10.977578+0000","last_became_peered":"2023-09-23T20:23:10.977578+0000","last_unstale":"2023-09-23T20:26:50.513457+0000","last_undegraded":"2023-09-23T20:26:50.513457+0000","last_fullsized":"2023-09-23T20:26:50.513457+0000","mapping_epoch":118,"log_start":"117'1657","ondisk_log_start":"117'1657","created":118,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:22:51.846999+0000","objects_scrubbed":0,"log_size":5750,"log_dups_size":0,"ondisk_log_size":5750,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":11304960,"num_objects":173,"num_object_clones":0,"num_object_copies":519,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":173,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6525,"num_write_kb":214304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":183,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T20:26:50.945563+0000","last_change":"2023-09-23T20:25:39.668882+0000","last_active":"2023-09-23T20:26:50.945563+0000","last_peered":"2023-09-23T20:26:50.945563+0000","last_clean":"2023-09-23T20:26:50.945563+0000","last_became_active":"2023-09-23T20:25:39.637128+0000","last_became_peered":"2023-09-23T20:25:39.637128+0000","last_unstale":"2023-09-23T20:26:50.945563+0000","last_undegraded":"2023-09-23T20:26:50.945563+0000","last_fullsized":"2023-09-23T20:26:50.945563+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:56.406298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":181,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-09-23T20:26:43.976462+0000","last_change":"2023-09-23T20:24:53.089249+0000","last_active":"2023-09-23T20:26:43.976462+0000","last_peered":"2023-09-23T20:26:43.976462+0000","last_clean":"2023-09-23T20:26:43.976462+0000","last_became_active":"2023-09-23T20:24:53.088952+0000","last_became_peered":"2023-09-23T20:24:53.088952+0000","last_unstale":"2023-09-23T20:26:43.976462+0000","last_undegraded":"2023-09-23T20:26:43.976462+0000","last_fullsized":"2023-09-23T20:26:43.976462+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:38:03.216069+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":5,"num_pg":26,"stat_sum":{"num_bytes":276430872,"num_objects":4219,"num_object_clones":0,"num_object_copies":12657,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2607,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4219,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":194259,"num_write_kb":5850817,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26392,"num_bytes_recovered":1203175424,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1500327936,"data_stored":1500315720,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":176927,"ondisk_log_size":176927,"up":78,"acting":74,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":15,"num_bytes_recovered":1356,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":25181,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1771,"ondisk_log_size":1771,"up":33,"acting":33,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6082,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201468,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":303052,"kb_used_data":202756,"kb_used_omap":0,"kb_used_meta":100224,"kb_avail":93442100,"statfs":{"total":95995035648,"available":95684710400,"internally_reserved":0,"allocated":207622144,"data_stored":206906343,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":102629376},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:26:23 2023","interfaces":[{"interface":"back","average":{"1min":1.861,"5min":1.2150000000000001,"15min":0.78700000000000003},"min":{"1min":0.247,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":10.907,"5min":13.092000000000001,"15min":13.092000000000001},"last":48.798999999999999},{"interface":"front","average":{"1min":3.5369999999999999,"5min":1.5940000000000001,"15min":0.91000000000000003},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":9.8610000000000007,"5min":9.8610000000000007,"15min":9.8610000000000007},"last":48.247}]},{"osd":1,"last update":"Sat Sep 23 20:25:57 2023","interfaces":[{"interface":"back","average":{"1min":3.4020000000000001,"5min":1.504,"15min":0.876},"min":{"1min":0.26900000000000002,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":7.0279999999999996,"5min":17.795999999999999,"15min":17.795999999999999},"last":0.434},{"interface":"front","average":{"1min":1.976,"5min":1.167,"15min":0.79700000000000004},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":6.8520000000000003,"5min":13.250999999999999,"15min":13.250999999999999},"last":0.41999999999999998}]},{"osd":2,"last update":"Sat Sep 23 20:26:28 2023","interfaces":[{"interface":"back","average":{"1min":3.2610000000000001,"5min":1.6080000000000001,"15min":0.91500000000000004},"min":{"1min":0.39000000000000001,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":8.1270000000000007,"5min":9.8300000000000001,"15min":9.8300000000000001},"last":0.48199999999999998},{"interface":"front","average":{"1min":3.1850000000000001,"5min":1.708,"15min":0.95999999999999996},"min":{"1min":0.48099999999999998,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":6.6699999999999999,"5min":11.423999999999999,"15min":11.423999999999999},"last":0.39200000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:25:57 2023","interfaces":[{"interface":"back","average":{"1min":3.367,"5min":1.4059999999999999,"15min":0.90500000000000003},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":6.7699999999999996,"5min":6.7699999999999996,"15min":6.7699999999999996},"last":0.45800000000000002},{"interface":"front","average":{"1min":3.2650000000000001,"5min":1.276,"15min":0.874},"min":{"1min":0.32500000000000001,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":9.7840000000000007,"5min":9.7840000000000007,"15min":9.7840000000000007},"last":0.40999999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:25:57 2023","interfaces":[{"interface":"back","average":{"1min":1.569,"5min":0.93899999999999995,"15min":0.64400000000000002},"min":{"1min":0.36099999999999999,"5min":0.27000000000000002,"15min":0.26700000000000002},"max":{"1min":5.6799999999999997,"5min":13.141999999999999,"15min":13.141999999999999},"last":0.312},{"interface":"front","average":{"1min":4.2130000000000001,"5min":1.4810000000000001,"15min":0.84599999999999997},"min":{"1min":0.30299999999999999,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":10.853,"5min":10.853,"15min":10.853},"last":0.36799999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:25:57 2023","interfaces":[{"interface":"back","average":{"1min":3.1779999999999999,"5min":1.3560000000000001,"15min":0.80900000000000005},"min":{"1min":0.44,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":10.563000000000001,"5min":10.563000000000001,"15min":10.563000000000001},"last":0.34999999999999998},{"interface":"front","average":{"1min":2.7519999999999998,"5min":1.302,"15min":0.82099999999999995},"min":{"1min":0.34699999999999998,"5min":0.34399999999999997,"15min":0.31},"max":{"1min":12.131,"5min":13.292,"15min":13.292},"last":0.46800000000000003}]},{"osd":6,"last update":"Sat Sep 23 20:25:57 2023","interfaces":[{"interface":"back","average":{"1min":2.399,"5min":1.151,"15min":0.77600000000000002},"min":{"1min":0.40500000000000003,"5min":0.35699999999999998,"15min":0.34499999999999997},"max":{"1min":11.583,"5min":13.185,"15min":13.185},"last":0.505},{"interface":"front","average":{"1min":2.9300000000000002,"5min":1.226,"15min":0.80100000000000005},"min":{"1min":0.41999999999999998,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":9.9120000000000008,"5min":9.9120000000000008,"15min":9.9120000000000008},"last":0.44700000000000001}]}]},{"osd":6,"up_from":47,"seq":201863463105,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":126160,"kb_used_data":47944,"kb_used_omap":0,"kb_used_meta":78208,"kb_avail":93618992,"statfs":{"total":95995035648,"available":95865847808,"internally_reserved":0,"allocated":49094656,"data_stored":48390928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":80084992},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:26:26 2023","interfaces":[{"interface":"back","average":{"1min":1.7270000000000001,"5min":1.23,"15min":0.79500000000000004},"min":{"1min":0.221,"5min":0.221,"15min":0.221},"max":{"1min":8.1940000000000008,"5min":8.1940000000000008,"15min":8.1940000000000008},"last":1.6739999999999999},{"interface":"front","average":{"1min":1.3069999999999999,"5min":1.4850000000000001,"15min":0.87},"min":{"1min":0.26300000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":4.0609999999999999,"5min":12.512,"15min":12.512},"last":0.55300000000000005}]},{"osd":1,"last update":"Sat Sep 23 20:25:57 2023","interfaces":[{"interface":"back","average":{"1min":2.6760000000000002,"5min":1.4710000000000001,"15min":0.873},"min":{"1min":0.30099999999999999,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":7.8929999999999998,"5min":18.001000000000001,"15min":18.001000000000001},"last":0.58299999999999996},{"interface":"front","average":{"1min":2.7040000000000002,"5min":1.2,"15min":0.76500000000000001},"min":{"1min":0.28699999999999998,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":13.91,"5min":13.91,"15min":13.91},"last":0.96299999999999997}]},{"osd":2,"last update":"Sat Sep 23 20:26:26 2023","interfaces":[{"interface":"back","average":{"1min":2.153,"5min":1.385,"15min":0.85799999999999998},"min":{"1min":0.32800000000000001,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":8.0090000000000003,"5min":13.856,"15min":13.856},"last":0.67800000000000005},{"interface":"front","average":{"1min":2.0720000000000001,"5min":1.284,"15min":0.83399999999999996},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":6.8140000000000001,"5min":6.8140000000000001,"15min":6.8140000000000001},"last":2.585}]},{"osd":3,"last update":"Sat Sep 23 20:25:57 2023","interfaces":[{"interface":"back","average":{"1min":3.1840000000000002,"5min":1.5349999999999999,"15min":0.92000000000000004},"min":{"1min":0.39200000000000002,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":9.4239999999999995,"5min":9.4239999999999995,"15min":9.4239999999999995},"last":0.60899999999999999},{"interface":"front","average":{"1min":2.6309999999999998,"5min":1.413,"15min":0.92200000000000004},"min":{"1min":0.42799999999999999,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":7.8700000000000001,"5min":8.6050000000000004,"15min":8.6050000000000004},"last":4.7060000000000004}]},{"osd":4,"last update":"Sat Sep 23 20:26:48 2023","interfaces":[{"interface":"back","average":{"1min":1.5049999999999999,"5min":1.2669999999999999,"15min":0.72999999999999998},"min":{"1min":0.27600000000000002,"5min":0.27200000000000002,"15min":0.26200000000000001},"max":{"1min":10.173,"5min":10.173,"15min":10.173},"last":2.1829999999999998},{"interface":"front","average":{"1min":1.962,"5min":1.3540000000000001,"15min":0.78000000000000003},"min":{"1min":0.312,"5min":0.29799999999999999,"15min":0.28499999999999998},"max":{"1min":8.3249999999999993,"5min":8.3249999999999993,"15min":8.3249999999999993},"last":0.64900000000000002}]},{"osd":5,"last update":"Sat Sep 23 20:26:48 2023","interfaces":[{"interface":"back","average":{"1min":1.202,"5min":1.4119999999999999,"15min":0.80300000000000005},"min":{"1min":0.31,"5min":0.31,"15min":0.29799999999999999},"max":{"1min":4.4580000000000002,"5min":14.085000000000001,"15min":14.085000000000001},"last":2.048},{"interface":"front","average":{"1min":2.831,"5min":1.8660000000000001,"15min":0.97299999999999998},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.29499999999999998},"max":{"1min":18.192,"5min":18.192,"15min":18.192},"last":6.3860000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:25:57 2023","interfaces":[{"interface":"back","average":{"1min":3.02,"5min":1.262,"15min":0.78300000000000003},"min":{"1min":0.42299999999999999,"5min":0.33500000000000002,"15min":0.32800000000000001},"max":{"1min":10.217000000000001,"5min":10.217000000000001,"15min":10.217000000000001},"last":1.879},{"interface":"front","average":{"1min":3.2749999999999999,"5min":1.393,"15min":0.84199999999999997},"min":{"1min":0.36599999999999999,"5min":0.34999999999999998,"15min":0.315},"max":{"1min":9.7360000000000007,"5min":9.7360000000000007,"15min":9.7360000000000007},"last":0.629}]}]},{"osd":1,"up_from":99,"seq":425201762380,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":213740,"kb_used_data":65508,"kb_used_omap":6,"kb_used_meta":148153,"kb_avail":93531412,"statfs":{"total":95995035648,"available":95776165888,"internally_reserved":0,"allocated":67080192,"data_stored":65999018,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6696,"internal_metadata":151709144},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:26:54 2023","interfaces":[{"interface":"back","average":{"1min":2.6600000000000001,"5min":2.052,"15min":0.93500000000000005},"min":{"1min":0.215,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":14.384,"5min":40.113,"15min":40.113},"last":0.72099999999999997},{"interface":"front","average":{"1min":3.1869999999999998,"5min":1.8120000000000001,"15min":0.87},"min":{"1min":0.23699999999999999,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":22.210000000000001,"5min":42.994999999999997,"15min":42.994999999999997},"last":0.23699999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:26:27 2023","interfaces":[{"interface":"back","average":{"1min":1.6479999999999999,"5min":1.8029999999999999,"15min":0.85499999999999998},"min":{"1min":0.27900000000000003,"5min":0.253,"15min":0.253},"max":{"1min":9.6839999999999993,"5min":39.962000000000003,"15min":39.962000000000003},"last":0.76300000000000001},{"interface":"front","average":{"1min":2.806,"5min":1.794,"15min":0.96599999999999997},"min":{"1min":0.26700000000000002,"5min":0.246,"15min":0.246},"max":{"1min":8.7910000000000004,"5min":47.442,"15min":47.442},"last":0.66600000000000004}]},{"osd":3,"last update":"Sat Sep 23 20:25:58 2023","interfaces":[{"interface":"back","average":{"1min":1.3029999999999999,"5min":1.1319999999999999,"15min":0.74399999999999999},"min":{"1min":0.28499999999999998,"5min":0.246,"15min":0.246},"max":{"1min":4.6130000000000004,"5min":8.7420000000000009,"15min":8.7420000000000009},"last":0.59699999999999998},{"interface":"front","average":{"1min":2.1619999999999999,"5min":1.0660000000000001,"15min":0.627},"min":{"1min":0.26000000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":8.7400000000000002,"5min":9.1859999999999999,"15min":9.1859999999999999},"last":0.46899999999999997}]},{"osd":4,"last update":"Sat Sep 23 20:26:54 2023","interfaces":[{"interface":"back","average":{"1min":2.5790000000000002,"5min":1.5900000000000001,"15min":0.94099999999999995},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":15.353,"5min":15.353,"15min":15.353},"last":1.1659999999999999},{"interface":"front","average":{"1min":1.7609999999999999,"5min":1.536,"15min":0.93500000000000005},"min":{"1min":0.32400000000000001,"5min":0.312,"15min":0.312},"max":{"1min":6.2270000000000003,"5min":8.7899999999999991,"15min":8.7899999999999991},"last":0.64500000000000002}]},{"osd":5,"last update":"Sat Sep 23 20:26:54 2023","interfaces":[{"interface":"back","average":{"1min":2.6360000000000001,"5min":1.6220000000000001,"15min":0.96799999999999997},"min":{"1min":0.33000000000000002,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":7.2000000000000002,"5min":12.590999999999999,"15min":12.590999999999999},"last":2.3769999999999998},{"interface":"front","average":{"1min":2.2530000000000001,"5min":2.137,"15min":1.163},"min":{"1min":0.38300000000000001,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":6.3140000000000001,"5min":39.579000000000001,"15min":39.579000000000001},"last":3.0939999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:26:54 2023","interfaces":[{"interface":"back","average":{"1min":2.9630000000000001,"5min":1.996,"15min":1.1100000000000001},"min":{"1min":0.42999999999999999,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":7.226,"5min":34.409999999999997,"15min":34.409999999999997},"last":0.52300000000000002},{"interface":"front","average":{"1min":2.964,"5min":2.2269999999999999,"15min":1.167},"min":{"1min":0.38100000000000001,"5min":0.377,"15min":0.377},"max":{"1min":15.313000000000001,"5min":35.554000000000002,"15min":35.554000000000002},"last":3.056}]},{"osd":7,"last update":"Sat Sep 23 20:26:54 2023","interfaces":[{"interface":"back","average":{"1min":2.8719999999999999,"5min":1.8480000000000001,"15min":1.05},"min":{"1min":0.42499999999999999,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":9.7439999999999998,"5min":9.7439999999999998,"15min":9.7439999999999998},"last":0.54700000000000004},{"interface":"front","average":{"1min":2.3180000000000001,"5min":1.7729999999999999,"15min":1.0469999999999999},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":6.2519999999999998,"5min":9.2279999999999998,"15min":9.2279999999999998},"last":2.5680000000000001}]}]},{"osd":0,"up_from":94,"seq":403726925906,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":339296,"kb_used_data":177176,"kb_used_omap":7,"kb_used_meta":162104,"kb_avail":93405856,"statfs":{"total":95995035648,"available":95647596544,"internally_reserved":0,"allocated":181428224,"data_stored":180420001,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":165994513},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:25:55 2023","interfaces":[{"interface":"back","average":{"1min":1.1799999999999999,"5min":0.79600000000000004,"15min":0.504},"min":{"1min":0.20799999999999999,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":5.1699999999999999,"5min":5.3609999999999998,"15min":5.3609999999999998},"last":1.0600000000000001},{"interface":"front","average":{"1min":0.72099999999999997,"5min":0.79400000000000004,"15min":0.52200000000000002},"min":{"1min":0.23400000000000001,"5min":0.222,"15min":0.222},"max":{"1min":2.27,"5min":8.9770000000000003,"15min":8.9770000000000003},"last":1.29}]},{"osd":2,"last update":"Sat Sep 23 20:26:22 2023","interfaces":[{"interface":"back","average":{"1min":4.3319999999999999,"5min":1.8200000000000001,"15min":1.004},"min":{"1min":0.251,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":23.832999999999998,"5min":23.832999999999998,"15min":23.832999999999998},"last":0.90100000000000002},{"interface":"front","average":{"1min":1.3779999999999999,"5min":1.256,"15min":0.749},"min":{"1min":0.223,"5min":0.223,"15min":0.223},"max":{"1min":6.0629999999999997,"5min":9.1940000000000008,"15min":9.1940000000000008},"last":0.64800000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:25:55 2023","interfaces":[{"interface":"back","average":{"1min":1.857,"5min":1.1639999999999999,"15min":0.78600000000000003},"min":{"1min":0.27800000000000002,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":5.976,"5min":5.976,"15min":5.976},"last":0.999},{"interface":"front","average":{"1min":1.341,"5min":0.80700000000000005,"15min":0.55000000000000004},"min":{"1min":0.28499999999999998,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":5.194,"5min":5.4210000000000003,"15min":5.4210000000000003},"last":1.0920000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:26:22 2023","interfaces":[{"interface":"back","average":{"1min":4.6619999999999999,"5min":1.9430000000000001,"15min":1.046},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":23.876999999999999,"5min":23.876999999999999,"15min":23.876999999999999},"last":0.93899999999999995},{"interface":"front","average":{"1min":2.8839999999999999,"5min":1.587,"15min":0.93799999999999994},"min":{"1min":0.34799999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":23.681999999999999,"5min":23.681999999999999,"15min":23.681999999999999},"last":1.3460000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:26:22 2023","interfaces":[{"interface":"back","average":{"1min":4.3259999999999996,"5min":2.2290000000000001,"15min":1.1819999999999999},"min":{"1min":0.39100000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":24.198,"5min":24.198,"15min":24.198},"last":1.129},{"interface":"front","average":{"1min":2.0169999999999999,"5min":1.754,"15min":1.0369999999999999},"min":{"1min":0.37,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":6.9269999999999996,"5min":11.167999999999999,"15min":11.167999999999999},"last":0.55200000000000005}]},{"osd":6,"last update":"Sat Sep 23 20:26:22 2023","interfaces":[{"interface":"back","average":{"1min":3.694,"5min":1.5760000000000001,"15min":0.95799999999999996},"min":{"1min":0.45400000000000001,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":24.408000000000001,"5min":24.408000000000001,"15min":24.408000000000001},"last":1.165},{"interface":"front","average":{"1min":3.5230000000000001,"5min":1.526,"15min":0.94999999999999996},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":25.271000000000001,"5min":25.271000000000001,"15min":25.271000000000001},"last":0.62}]},{"osd":7,"last update":"Sat Sep 23 20:26:22 2023","interfaces":[{"interface":"back","average":{"1min":3.3199999999999998,"5min":1.7190000000000001,"15min":1.0249999999999999},"min":{"1min":0.442,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":25.359000000000002,"5min":25.359000000000002,"15min":25.359000000000002},"last":0.58999999999999997},{"interface":"front","average":{"1min":5.298,"5min":2.181,"15min":1.1699999999999999},"min":{"1min":0.499,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":23.905999999999999,"5min":23.905999999999999,"15min":23.905999999999999},"last":0.96799999999999997}]}]},{"osd":2,"up_from":104,"seq":446676598853,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":425912,"kb_used_data":308144,"kb_used_omap":11,"kb_used_meta":117748,"kb_avail":93319240,"statfs":{"total":95995035648,"available":95558901760,"internally_reserved":0,"allocated":315539456,"data_stored":314557193,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11878,"internal_metadata":120574362},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:26:30 2023","interfaces":[{"interface":"back","average":{"1min":2.8090000000000002,"5min":1.706,"15min":0.81999999999999995},"min":{"1min":0.21099999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":9.5980000000000008,"5min":12.811,"15min":12.811},"last":0.90000000000000002},{"interface":"front","average":{"1min":2.1019999999999999,"5min":1.2709999999999999,"15min":0.68700000000000006},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":15.746,"5min":15.746,"15min":15.746},"last":0.92100000000000004}]},{"osd":1,"last update":"Sat Sep 23 20:26:30 2023","interfaces":[{"interface":"back","average":{"1min":2.9390000000000001,"5min":1.524,"15min":0.77000000000000002},"min":{"1min":0.28599999999999998,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":14.135999999999999,"5min":14.135999999999999,"15min":14.135999999999999},"last":1.3819999999999999},{"interface":"front","average":{"1min":2.617,"5min":1.226,"15min":0.66400000000000003},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":13.718,"5min":13.718,"15min":13.718},"last":1.456}]},{"osd":3,"last update":"Sat Sep 23 20:25:54 2023","interfaces":[{"interface":"back","average":{"1min":0.72799999999999998,"5min":0.97199999999999998,"15min":0.56499999999999995},"min":{"1min":0.32600000000000001,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":1.9650000000000001,"5min":15.319000000000001,"15min":15.319000000000001},"last":0.54100000000000004},{"interface":"front","average":{"1min":2.1880000000000002,"5min":1.1719999999999999,"15min":0.65600000000000003},"min":{"1min":0.26100000000000001,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":9.6850000000000005,"5min":9.6850000000000005,"15min":9.6850000000000005},"last":0.55800000000000005}]},{"osd":4,"last update":"Sat Sep 23 20:26:30 2023","interfaces":[{"interface":"back","average":{"1min":1.996,"5min":1.234,"15min":0.80000000000000004},"min":{"1min":0.36299999999999999,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":8.2300000000000004,"5min":8.2300000000000004,"15min":8.2300000000000004},"last":5.8799999999999999},{"interface":"front","average":{"1min":2.3679999999999999,"5min":1.3979999999999999,"15min":0.85599999999999998},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":8.6530000000000005,"5min":8.6530000000000005,"15min":8.6530000000000005},"last":0.61899999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:26:30 2023","interfaces":[{"interface":"back","average":{"1min":2.8319999999999999,"5min":1.577,"15min":0.96299999999999997},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":11.237,"5min":11.237,"15min":11.237},"last":0.51800000000000002},{"interface":"front","average":{"1min":3.452,"5min":1.6850000000000001,"15min":0.999},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":11.266999999999999,"5min":11.266999999999999,"15min":11.266999999999999},"last":0.57999999999999996}]},{"osd":6,"last update":"Sat Sep 23 20:26:30 2023","interfaces":[{"interface":"back","average":{"1min":2.0129999999999999,"5min":1.294,"15min":0.86899999999999999},"min":{"1min":0.49099999999999999,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":6.2599999999999998,"5min":6.7939999999999996,"15min":6.7939999999999996},"last":5.1139999999999999},{"interface":"front","average":{"1min":2.7410000000000001,"5min":1.585,"15min":0.97699999999999998},"min":{"1min":0.41099999999999998,"5min":0.377,"15min":0.377},"max":{"1min":9.6600000000000001,"5min":9.6600000000000001,"15min":9.6600000000000001},"last":6.0099999999999998}]},{"osd":7,"last update":"Sat Sep 23 20:26:30 2023","interfaces":[{"interface":"back","average":{"1min":3.1150000000000002,"5min":1.712,"15min":1.002},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":15.343999999999999,"5min":15.343999999999999,"15min":15.343999999999999},"last":5.9649999999999999},{"interface":"front","average":{"1min":2.6299999999999999,"5min":1.5840000000000001,"15min":0.96599999999999997},"min":{"1min":0.48899999999999999,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":9.1440000000000001,"5min":10.064,"15min":10.064},"last":0.69099999999999995}]}]},{"osd":3,"up_from":109,"seq":468151435327,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":213200,"kb_used_data":91336,"kb_used_omap":4,"kb_used_meta":121851,"kb_avail":93531952,"statfs":{"total":95995035648,"available":95776718848,"internally_reserved":0,"allocated":93528064,"data_stored":92529965,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4514,"internal_metadata":124776030},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:25:56 2023","interfaces":[{"interface":"back","average":{"1min":4.3869999999999996,"5min":1.7390000000000001,"15min":0.81399999999999995},"min":{"1min":0.20399999999999999,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":67.305999999999997,"5min":67.305999999999997,"15min":67.305999999999997},"last":3.6070000000000002},{"interface":"front","average":{"1min":4.5739999999999998,"5min":1.8149999999999999,"15min":0.86099999999999999},"min":{"1min":0.26300000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":67.403999999999996,"5min":67.403999999999996,"15min":67.403999999999996},"last":0.34599999999999997}]},{"osd":1,"last update":"Sat Sep 23 20:25:56 2023","interfaces":[{"interface":"back","average":{"1min":1.3240000000000001,"5min":0.98199999999999998,"15min":0.59199999999999997},"min":{"1min":0.249,"5min":0.214,"15min":0.214},"max":{"1min":4.3440000000000003,"5min":11.537000000000001,"15min":11.537000000000001},"last":2.734},{"interface":"front","average":{"1min":1.3819999999999999,"5min":0.97199999999999998,"15min":0.62},"min":{"1min":0.27900000000000003,"5min":0.245,"15min":0.245},"max":{"1min":6.9480000000000004,"5min":11.5,"15min":11.5},"last":3.7440000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:25:56 2023","interfaces":[{"interface":"back","average":{"1min":0.63900000000000001,"5min":1.1619999999999999,"15min":0.622},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":2.907,"5min":17.335000000000001,"15min":17.335000000000001},"last":0.51000000000000001},{"interface":"front","average":{"1min":1.419,"5min":1.218,"15min":0.71499999999999997},"min":{"1min":0.30099999999999999,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":6.5700000000000003,"5min":17.684000000000001,"15min":17.684000000000001},"last":2.6869999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:25:56 2023","interfaces":[{"interface":"back","average":{"1min":1.5169999999999999,"5min":1.0049999999999999,"15min":0.73799999999999999},"min":{"1min":0.32600000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":7.1600000000000001,"5min":7.1600000000000001,"15min":7.1600000000000001},"last":3.855},{"interface":"front","average":{"1min":2.8730000000000002,"5min":1.202,"15min":0.81100000000000005},"min":{"1min":0.376,"5min":0.371,"15min":0.371},"max":{"1min":7.5069999999999997,"5min":7.5069999999999997,"15min":7.5069999999999997},"last":2.7589999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:25:56 2023","interfaces":[{"interface":"back","average":{"1min":2.4199999999999999,"5min":1.3540000000000001,"15min":0.85599999999999998},"min":{"1min":0.36599999999999999,"5min":0.311,"15min":0.311},"max":{"1min":9.8089999999999993,"5min":9.8089999999999993,"15min":9.8089999999999993},"last":5.665},{"interface":"front","average":{"1min":1.5600000000000001,"5min":1.056,"15min":0.77200000000000002},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":6.2430000000000003,"5min":6.4450000000000003,"15min":6.4450000000000003},"last":0.42299999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:25:56 2023","interfaces":[{"interface":"back","average":{"1min":2.802,"5min":1.306,"15min":0.85999999999999999},"min":{"1min":0.40999999999999998,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":8.2789999999999999,"5min":8.2789999999999999,"15min":8.2789999999999999},"last":1.0940000000000001},{"interface":"front","average":{"1min":2.3740000000000001,"5min":1.159,"15min":0.83399999999999996},"min":{"1min":0.39900000000000002,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":8.7759999999999998,"5min":8.7759999999999998,"15min":8.7759999999999998},"last":2.782}]},{"osd":7,"last update":"Sat Sep 23 20:25:56 2023","interfaces":[{"interface":"back","average":{"1min":1.8959999999999999,"5min":1.1259999999999999,"15min":0.84399999999999997},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":6.5259999999999998,"5min":6.5259999999999998,"15min":6.5259999999999998},"last":2.8069999999999999},{"interface":"front","average":{"1min":2.4580000000000002,"5min":1.4159999999999999,"15min":0.90000000000000002},"min":{"1min":0.41099999999999998,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":8.4009999999999998,"5min":8.6180000000000003,"15min":8.6180000000000003},"last":3.7229999999999999}]}]},{"osd":4,"up_from":30,"seq":128849019080,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":360828,"kb_used_data":241844,"kb_used_omap":0,"kb_used_meta":118976,"kb_avail":93384324,"statfs":{"total":95995035648,"available":95625547776,"internally_reserved":0,"allocated":247648256,"data_stored":246947878,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":121831424},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:26:22 2023","interfaces":[{"interface":"back","average":{"1min":2.258,"5min":1.411,"15min":0.84899999999999998},"min":{"1min":0.25800000000000001,"5min":0.222,"15min":0.222},"max":{"1min":11.331,"5min":20.411999999999999,"15min":20.411999999999999},"last":2.6480000000000001},{"interface":"front","average":{"1min":2.6629999999999998,"5min":1.5489999999999999,"15min":0.94399999999999995},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":13.595000000000001,"5min":20.454999999999998,"15min":20.454999999999998},"last":2.6579999999999999}]},{"osd":1,"last update":"Sat Sep 23 20:25:58 2023","interfaces":[{"interface":"back","average":{"1min":2.3940000000000001,"5min":1.0580000000000001,"15min":0.77000000000000002},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":9.5419999999999998,"5min":9.5419999999999998,"15min":9.5419999999999998},"last":0.40200000000000002},{"interface":"front","average":{"1min":2.3530000000000002,"5min":1.1890000000000001,"15min":0.81299999999999994},"min":{"1min":0.28399999999999997,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":11.455,"5min":11.455,"15min":11.455},"last":1.506}]},{"osd":2,"last update":"Sat Sep 23 20:26:32 2023","interfaces":[{"interface":"back","average":{"1min":2.1779999999999999,"5min":1.51,"15min":0.90800000000000003},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":11.412000000000001,"5min":21.462,"15min":21.462},"last":1.5669999999999999},{"interface":"front","average":{"1min":3.29,"5min":1.8660000000000001,"15min":1.014},"min":{"1min":0.28999999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":13.567,"5min":22.593,"15min":22.593},"last":2.1549999999999998}]},{"osd":3,"last update":"Sat Sep 23 20:25:58 2023","interfaces":[{"interface":"back","average":{"1min":2.452,"5min":1.4179999999999999,"15min":0.90000000000000002},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":11.377000000000001,"5min":24.891999999999999,"15min":24.891999999999999},"last":1.893},{"interface":"front","average":{"1min":2.867,"5min":1.472,"15min":0.93400000000000005},"min":{"1min":0.373,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":11.439,"5min":21.922000000000001,"15min":21.922000000000001},"last":1.5429999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:26:10 2023","interfaces":[{"interface":"back","average":{"1min":5.0119999999999996,"5min":1.649,"15min":0.86799999999999999},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":17.704000000000001,"5min":17.704000000000001,"15min":17.704000000000001},"last":0.247},{"interface":"front","average":{"1min":3.1600000000000001,"5min":1.361,"15min":0.77900000000000003},"min":{"1min":0.32500000000000001,"5min":0.28000000000000003,"15min":0.248},"max":{"1min":15.329000000000001,"5min":15.329000000000001,"15min":15.329000000000001},"last":2.1240000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:26:34 2023","interfaces":[{"interface":"back","average":{"1min":3.3639999999999999,"5min":1.9239999999999999,"15min":0.98099999999999998},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.27500000000000002},"max":{"1min":15.252000000000001,"5min":41.371000000000002,"15min":41.371000000000002},"last":2.1779999999999999},{"interface":"front","average":{"1min":1.9179999999999999,"5min":1.593,"15min":0.88200000000000001},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":9.3350000000000009,"5min":41.140999999999998,"15min":41.140999999999998},"last":0.79200000000000004}]},{"osd":7,"last update":"Sat Sep 23 20:26:48 2023","interfaces":[{"interface":"back","average":{"1min":4.3280000000000003,"5min":1.8859999999999999,"15min":0.997},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.33600000000000002},"max":{"1min":16.427,"5min":16.427,"15min":16.427},"last":2.21},{"interface":"front","average":{"1min":3.2890000000000001,"5min":1.754,"15min":0.96599999999999997},"min":{"1min":0.36399999999999999,"5min":0.34300000000000003,"15min":0.29899999999999999},"max":{"1min":15.645,"5min":19.018999999999998,"15min":19.018999999999998},"last":0.376}]}]},{"osd":5,"up_from":39,"seq":167503724741,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":587064,"kb_used_data":461104,"kb_used_omap":0,"kb_used_meta":125952,"kb_avail":93158088,"statfs":{"total":95995035648,"available":95393882112,"internally_reserved":0,"allocated":472170496,"data_stored":471470324,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":128974848},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:26:22 2023","interfaces":[{"interface":"back","average":{"1min":2.3380000000000001,"5min":1.548,"15min":0.91000000000000003},"min":{"1min":0.29499999999999998,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":11.346,"5min":32.475000000000001,"15min":32.475000000000001},"last":5.915},{"interface":"front","average":{"1min":1.9650000000000001,"5min":1.601,"15min":0.93400000000000005},"min":{"1min":0.245,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":5.6879999999999997,"5min":32.457000000000001,"15min":32.457000000000001},"last":1.615}]},{"osd":1,"last update":"Sat Sep 23 20:26:00 2023","interfaces":[{"interface":"back","average":{"1min":2.4950000000000001,"5min":1.4490000000000001,"15min":0.84599999999999997},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":11.295,"5min":32.243000000000002,"15min":32.243000000000002},"last":9.7639999999999993},{"interface":"front","average":{"1min":1.228,"5min":1.2370000000000001,"15min":0.77700000000000002},"min":{"1min":0.32300000000000001,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":7.1319999999999997,"5min":32.231000000000002,"15min":32.231000000000002},"last":0.374}]},{"osd":2,"last update":"Sat Sep 23 20:26:22 2023","interfaces":[{"interface":"back","average":{"1min":1.9970000000000001,"5min":1.891,"15min":0.997},"min":{"1min":0.30099999999999999,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":11.407,"5min":34.514000000000003,"15min":34.514000000000003},"last":5.952},{"interface":"front","average":{"1min":2.105,"5min":1.732,"15min":0.94999999999999996},"min":{"1min":0.32100000000000001,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":11.574999999999999,"5min":32.515000000000001,"15min":32.515000000000001},"last":3.0339999999999998}]},{"osd":3,"last update":"Sat Sep 23 20:26:00 2023","interfaces":[{"interface":"back","average":{"1min":2.1019999999999999,"5min":1.1990000000000001,"15min":0.77100000000000002},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":7.0309999999999997,"5min":7.3799999999999999,"15min":7.3799999999999999},"last":0.45500000000000002},{"interface":"front","average":{"1min":2.831,"5min":1.3240000000000001,"15min":0.85199999999999998},"min":{"1min":0.36699999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":11.379,"5min":11.379,"15min":11.379},"last":6.0049999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:26:18 2023","interfaces":[{"interface":"back","average":{"1min":3.391,"5min":1.4339999999999999,"15min":0.78000000000000003},"min":{"1min":0.29199999999999998,"5min":0.26700000000000002,"15min":0.26400000000000001},"max":{"1min":14.218999999999999,"5min":14.218999999999999,"15min":14.218999999999999},"last":5.4089999999999998},{"interface":"front","average":{"1min":2.6720000000000002,"5min":1.3420000000000001,"15min":0.77000000000000002},"min":{"1min":0.34300000000000003,"5min":0.29099999999999998,"15min":0.28799999999999998},"max":{"1min":11.664,"5min":11.664,"15min":11.664},"last":5.9779999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:26:32 2023","interfaces":[{"interface":"back","average":{"1min":2.633,"5min":1.575,"15min":0.85599999999999998},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.29799999999999999},"max":{"1min":8.7949999999999999,"5min":11.545999999999999,"15min":11.545999999999999},"last":5.444},{"interface":"front","average":{"1min":1.6870000000000001,"5min":1.5,"15min":0.879},"min":{"1min":0.376,"5min":0.35999999999999999,"15min":0.34499999999999997},"max":{"1min":8.4120000000000008,"5min":11.456,"15min":11.456},"last":6.0339999999999998}]},{"osd":7,"last update":"Sat Sep 23 20:26:00 2023","interfaces":[{"interface":"back","average":{"1min":2.9199999999999999,"5min":1.2470000000000001,"15min":0.77200000000000002},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33800000000000002},"max":{"1min":11.430999999999999,"5min":11.430999999999999,"15min":11.430999999999999},"last":6.0730000000000004},{"interface":"front","average":{"1min":1.6479999999999999,"5min":1.0669999999999999,"15min":0.71899999999999997},"min":{"1min":0.35699999999999998,"5min":0.315,"15min":0.315},"max":{"1min":7.0890000000000004,"5min":12.368,"15min":12.368},"last":2.6230000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4722,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1360,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6696,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7156,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3154,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":153223168,"data_stored":153223168,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":52559872,"data_stored":52559872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":299700224,"data_stored":299696152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":90963968,"data_stored":90963968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":224198656,"data_stored":224198656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":447090688,"data_stored":447086616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":35389440,"data_stored":35389440,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":197201920,"data_stored":197197848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:26:56.701 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-23T20:26:56.701 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:56.702 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:26:56.803 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:26:56.932 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:57.129 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:57 smithi018 ceph-mon[179647]: pgmap v446: 109 pgs: 1 active+recovering+undersized+remapped, 105 active+clean, 3 active+recovery_wait+undersized+degraded+remapped; 268 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 722 op/s; 2607/12741 objects degraded (20.462%); 0 B/s, 76 objects/s recovering 2023-09-23T20:26:57.173 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:26:57.176 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:57.177 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:26:57.277 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:26:57.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:57 smithi018 ceph-mon[179647]: from='client.35184 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:26:57.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:57 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:26:57.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:57 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:26:57.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:57 smithi018 ceph-mon[184144]: pgmap v446: 109 pgs: 1 active+recovering+undersized+remapped, 105 active+clean, 3 active+recovery_wait+undersized+degraded+remapped; 268 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 722 op/s; 2607/12741 objects degraded (20.462%); 0 B/s, 76 objects/s recovering 2023-09-23T20:26:57.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:57 smithi018 ceph-mon[184144]: from='client.35184 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:26:57.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:57 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:26:57.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:57 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:26:57.405 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:57.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:57 smithi149 ceph-mon[170394]: pgmap v446: 109 pgs: 1 active+recovering+undersized+remapped, 105 active+clean, 3 active+recovery_wait+undersized+degraded+remapped; 268 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 722 op/s; 2607/12741 objects degraded (20.462%); 0 B/s, 76 objects/s recovering 2023-09-23T20:26:57.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:57 smithi149 ceph-mon[170394]: from='client.35184 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:26:57.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:57 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:26:57.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:57 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:26:57.623 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:26:57.630 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:57.631 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:26:57.732 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:26:57.820 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:26:57.833 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:57.869 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Removed 107058 objects 2023-09-23T20:26:57.869 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Clean up completed and total clean up time :149.373 2023-09-23T20:26:57.892 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:26:57.895 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:57.895 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:26:57.921 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2023-09-23T20:26:57.921 INFO:teuthology.orchestra.run.smithi018.stdout: "ops": [], 2023-09-23T20:26:57.921 INFO:teuthology.orchestra.run.smithi018.stdout: "num_ops": 0 2023-09-23T20:26:57.921 INFO:teuthology.orchestra.run.smithi018.stdout:} 2023-09-23T20:26:57.997 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:26:58.032 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:58.144 INFO:teuthology.orchestra.run.smithi018.stderr:noscrub is unset 2023-09-23T20:26:58.198 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:58 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi149", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:26:58.199 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:58 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1597584273' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:26:58.199 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:58 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi149", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:26:58.199 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:58 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1597584273' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:26:58.386 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:26:58.423 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:26:58.428 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:58.429 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:26:58.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:58 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi149", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:26:58.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:58 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1597584273' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:26:58.531 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:26:58.565 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:58.936 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:26:58.940 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:58.940 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:26:58.943 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T20:26:59.041 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:26:59.069 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:59.106 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:26:59.109 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:59.110 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:26:59.210 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:26:59.248 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:59.289 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:26:59.291 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:59 smithi018 ceph-mon[184144]: pgmap v447: 109 pgs: 1 active+recovering+undersized+remapped, 105 active+clean, 3 active+recovery_wait+undersized+degraded+remapped; 125 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 961 op/s; 2607/5871 objects degraded (44.405%); 0 B/s, 144 objects/s recovering 2023-09-23T20:26:59.291 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:59 smithi018 ceph-mon[184144]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:26:59.292 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:59 smithi018 ceph-mon[184144]: failed to encode map e206 with expected crc 2023-09-23T20:26:59.292 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:59 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1597584273' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:26:59.292 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:26:59 smithi018 ceph-mon[184144]: osdmap e206: 8 total, 8 up, 8 in 2023-09-23T20:26:59.292 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:59 smithi018 ceph-mon[179647]: pgmap v447: 109 pgs: 1 active+recovering+undersized+remapped, 105 active+clean, 3 active+recovery_wait+undersized+degraded+remapped; 125 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 961 op/s; 2607/5871 objects degraded (44.405%); 0 B/s, 144 objects/s recovering 2023-09-23T20:26:59.292 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:59 smithi018 ceph-mon[179647]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:26:59.292 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:59 smithi018 ceph-mon[179647]: failed to encode map e206 with expected crc 2023-09-23T20:26:59.292 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:59 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1597584273' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:26:59.292 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:26:59 smithi018 ceph-mon[179647]: osdmap e206: 8 total, 8 up, 8 in 2023-09-23T20:26:59.293 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:59.293 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:26:59.394 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:26:59.417 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:59 smithi149 ceph-mon[170394]: pgmap v447: 109 pgs: 1 active+recovering+undersized+remapped, 105 active+clean, 3 active+recovery_wait+undersized+degraded+remapped; 125 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 961 op/s; 2607/5871 objects degraded (44.405%); 0 B/s, 144 objects/s recovering 2023-09-23T20:26:59.418 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:59 smithi149 ceph-mon[170394]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:26:59.418 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:59 smithi149 ceph-mon[170394]: failed to encode map e206 with expected crc 2023-09-23T20:26:59.418 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:59 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1597584273' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:26:59.418 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:26:59 smithi149 ceph-mon[170394]: osdmap e206: 8 total, 8 up, 8 in 2023-09-23T20:26:59.420 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:59.453 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:26:59.454 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:26:59.457 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:59.457 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:26:59.558 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:26:59.592 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:59.684 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:26:59.688 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:26:59.689 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:26:59.789 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:26:59.820 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:26:59.884 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:26:59.884 INFO:teuthology.orchestra.run.smithi018.stdout:{"fsid":"a3b3b17e-5a4c-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodeep-scrub flag(s) set","count":12},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 2607/5871 objects degraded (44.405%), 3 pgs degraded, 4 pgs undersized","count":7},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":496,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":207,"num_osds":8,"num_up_osds":8,"osd_up_since":1695500508,"num_in_osds":8,"osd_in_since":1695500738,"num_remapped_pgs":3},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":105},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":3},{"state_name":"active+recovering+undersized+remapped","count":1}],"num_pgs":109,"num_pools":5,"num_objects":1957,"data_bytes":131143627,"bytes_used":2207744000,"bytes_avail":765752541184,"bytes_total":767960285184,"degraded_objects":2607,"degraded_total":5871,"degraded_ratio":0.44404701073071029,"recovering_objects_per_sec":144,"recovering_bytes_per_sec":0,"recovering_keys_per_sec":0,"num_objects_recovered":1156,"num_bytes_recovered":0,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":961},"fsmap":{"epoch":9,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi018.hduiol","status":"up:active","gid":24566}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.18:8443/","prometheus":"http://172.21.15.18:9283/"}},"servicemap":{"epoch":20,"modified":"2023-09-23T20:22:49.433880+0000","services":{}},"progress_events":{"62c6450e-d8fa-4178-b11a-c1b35654ee03":{"message":"Global Recovery Event (3m)\n [==========================..] (remaining: 8s)","progress":0.96330273151397705,"add_to_ceph_s":true},"8e7eff18-0e45-4130-a6e1-662b3902f0d2":{"message":"Upgrade to 18.2.0-460-gbb0daeff (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2023-09-23T20:27:00.012 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:27:00.016 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:00.017 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:27:00.118 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:27:00.152 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:00.276 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:00 smithi018 ceph-mon[184144]: 5.17 scrub starts 2023-09-23T20:27:00.276 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:00 smithi018 ceph-mon[184144]: osdmap e207: 8 total, 8 up, 8 in 2023-09-23T20:27:00.277 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:00 smithi018 ceph-mon[184144]: failed to encode map e207 with expected crc 2023-09-23T20:27:00.277 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:00 smithi018 ceph-mon[184144]: failed to encode map e207 with expected crc 2023-09-23T20:27:00.277 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:00 smithi018 ceph-mon[184144]: failed to encode map e207 with expected crc 2023-09-23T20:27:00.277 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:00 smithi018 ceph-mon[184144]: failed to encode map e207 with expected crc 2023-09-23T20:27:00.277 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:00 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:27:00.277 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:00 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3527526023' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T20:27:00.277 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:00 smithi018 ceph-mon[179647]: 5.17 scrub starts 2023-09-23T20:27:00.278 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:00 smithi018 ceph-mon[179647]: osdmap e207: 8 total, 8 up, 8 in 2023-09-23T20:27:00.278 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:00 smithi018 ceph-mon[179647]: failed to encode map e207 with expected crc 2023-09-23T20:27:00.278 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:00 smithi018 ceph-mon[179647]: failed to encode map e207 with expected crc 2023-09-23T20:27:00.278 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:00 smithi018 ceph-mon[179647]: failed to encode map e207 with expected crc 2023-09-23T20:27:00.278 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:00 smithi018 ceph-mon[179647]: failed to encode map e207 with expected crc 2023-09-23T20:27:00.278 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:00 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:27:00.278 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:00 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3527526023' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T20:27:00.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:00 smithi149 ceph-mon[170394]: 5.17 scrub starts 2023-09-23T20:27:00.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:00 smithi149 ceph-mon[170394]: osdmap e207: 8 total, 8 up, 8 in 2023-09-23T20:27:00.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:00 smithi149 ceph-mon[170394]: failed to encode map e207 with expected crc 2023-09-23T20:27:00.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:00 smithi149 ceph-mon[170394]: failed to encode map e207 with expected crc 2023-09-23T20:27:00.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:00 smithi149 ceph-mon[170394]: failed to encode map e207 with expected crc 2023-09-23T20:27:00.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:00 smithi149 ceph-mon[170394]: failed to encode map e207 with expected crc 2023-09-23T20:27:00.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:00 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:27:00.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:00 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3527526023' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T20:27:00.548 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:27:00.550 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:27:00.552 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:00.552 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:27:00.598 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-09-23T20:27:00.598 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:27:00.653 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:27:00.681 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:00.715 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:27:00.717 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:00.718 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:27:00.819 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:27:00.853 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:00.983 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:27:00.989 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:00.989 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:27:01.090 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:27:01.131 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:01.180 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:27:01.182 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:01.183 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:27:01.184 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T20:27:01.284 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:27:01.321 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:01.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:01 smithi018 ceph-mon[184144]: pgmap v450: 109 pgs: 1 active+recovering+undersized+remapped, 105 active+clean, 3 active+recovery_wait+undersized+degraded+remapped; 101 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 623 op/s; 2607/4737 objects degraded (55.035%); 0 B/s, 78 objects/s recovering 2023-09-23T20:27:01.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:01 smithi018 ceph-mon[184144]: failed to encode map e207 with expected crc 2023-09-23T20:27:01.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:01 smithi018 ceph-mon[184144]: 5.17 scrub starts 2023-09-23T20:27:01.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:01 smithi018 ceph-mon[184144]: failed to encode map e208 with expected crc 2023-09-23T20:27:01.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:01 smithi018 ceph-mon[184144]: failed to encode map e208 with expected crc 2023-09-23T20:27:01.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:01 smithi018 ceph-mon[184144]: failed to encode map e208 with expected crc 2023-09-23T20:27:01.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:01 smithi018 ceph-mon[184144]: osdmap e208: 8 total, 8 up, 8 in 2023-09-23T20:27:01.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:01 smithi018 ceph-mon[184144]: failed to encode map e208 with expected crc 2023-09-23T20:27:01.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:01 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 2607/4737 objects degraded (55.035%), 3 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T20:27:01.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:01 smithi018 ceph-mon[179647]: pgmap v450: 109 pgs: 1 active+recovering+undersized+remapped, 105 active+clean, 3 active+recovery_wait+undersized+degraded+remapped; 101 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 623 op/s; 2607/4737 objects degraded (55.035%); 0 B/s, 78 objects/s recovering 2023-09-23T20:27:01.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:01 smithi018 ceph-mon[179647]: failed to encode map e207 with expected crc 2023-09-23T20:27:01.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:01 smithi018 ceph-mon[179647]: 5.17 scrub starts 2023-09-23T20:27:01.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:01 smithi018 ceph-mon[179647]: failed to encode map e208 with expected crc 2023-09-23T20:27:01.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:01 smithi018 ceph-mon[179647]: failed to encode map e208 with expected crc 2023-09-23T20:27:01.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:01 smithi018 ceph-mon[179647]: failed to encode map e208 with expected crc 2023-09-23T20:27:01.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:01 smithi018 ceph-mon[179647]: osdmap e208: 8 total, 8 up, 8 in 2023-09-23T20:27:01.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:01 smithi018 ceph-mon[179647]: failed to encode map e208 with expected crc 2023-09-23T20:27:01.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:01 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 2607/4737 objects degraded (55.035%), 3 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T20:27:01.440 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:27:01.453 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:27:01.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:01 smithi149 ceph-mon[170394]: pgmap v450: 109 pgs: 1 active+recovering+undersized+remapped, 105 active+clean, 3 active+recovery_wait+undersized+degraded+remapped; 101 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 623 op/s; 2607/4737 objects degraded (55.035%); 0 B/s, 78 objects/s recovering 2023-09-23T20:27:01.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:01 smithi149 ceph-mon[170394]: failed to encode map e207 with expected crc 2023-09-23T20:27:01.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:01 smithi149 ceph-mon[170394]: 5.17 scrub starts 2023-09-23T20:27:01.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:01 smithi149 ceph-mon[170394]: failed to encode map e208 with expected crc 2023-09-23T20:27:01.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:01 smithi149 ceph-mon[170394]: failed to encode map e208 with expected crc 2023-09-23T20:27:01.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:01 smithi149 ceph-mon[170394]: failed to encode map e208 with expected crc 2023-09-23T20:27:01.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:01 smithi149 ceph-mon[170394]: osdmap e208: 8 total, 8 up, 8 in 2023-09-23T20:27:01.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:01 smithi149 ceph-mon[170394]: failed to encode map e208 with expected crc 2023-09-23T20:27:01.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:01 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 2607/4737 objects degraded (55.035%), 3 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T20:27:01.478 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:27:01.490 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:27:01.495 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:01.496 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:27:01.508 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:27:01.533 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:27:01.553 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:27:01.574 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:27:01.594 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:27:01.596 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:27:01.596 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:27:01.596 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:27:01.596 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:27:01.596 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:27:01.596 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:27:01.596 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:27:01.596 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:27:01.597 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:27:01.626 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:01.818 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:27:01.821 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:01.822 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:27:01.923 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:27:01.954 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:02.056 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:27:02.058 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:02.058 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:27:02.159 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:27:02.190 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:02.227 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:27:02.230 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:02.230 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:27:02.331 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:27:02.358 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:02.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:02 smithi018 ceph-mon[184144]: 5.9 scrub starts 2023-09-23T20:27:02.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:02 smithi018 ceph-mon[184144]: 5.17 scrub starts 2023-09-23T20:27:02.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:02 smithi018 ceph-mon[184144]: 5.17 scrub ok 2023-09-23T20:27:02.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:02 smithi018 ceph-mon[184144]: failed to encode map e209 with expected crc 2023-09-23T20:27:02.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:02 smithi018 ceph-mon[184144]: osdmap e209: 8 total, 8 up, 8 in 2023-09-23T20:27:02.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:02 smithi018 ceph-mon[184144]: failed to encode map e209 with expected crc 2023-09-23T20:27:02.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:02 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1702836917' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:27:02.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:02 smithi018 ceph-mon[179647]: 5.9 scrub starts 2023-09-23T20:27:02.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:02 smithi018 ceph-mon[179647]: 5.17 scrub starts 2023-09-23T20:27:02.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:02 smithi018 ceph-mon[179647]: 5.17 scrub ok 2023-09-23T20:27:02.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:02 smithi018 ceph-mon[179647]: failed to encode map e209 with expected crc 2023-09-23T20:27:02.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:02 smithi018 ceph-mon[179647]: osdmap e209: 8 total, 8 up, 8 in 2023-09-23T20:27:02.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:02 smithi018 ceph-mon[179647]: failed to encode map e209 with expected crc 2023-09-23T20:27:02.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:02 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1702836917' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:27:02.392 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:27:02.395 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:02.395 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:27:02.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:02 smithi149 ceph-mon[170394]: 5.9 scrub starts 2023-09-23T20:27:02.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:02 smithi149 ceph-mon[170394]: 5.17 scrub starts 2023-09-23T20:27:02.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:02 smithi149 ceph-mon[170394]: 5.17 scrub ok 2023-09-23T20:27:02.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:02 smithi149 ceph-mon[170394]: failed to encode map e209 with expected crc 2023-09-23T20:27:02.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:02 smithi149 ceph-mon[170394]: osdmap e209: 8 total, 8 up, 8 in 2023-09-23T20:27:02.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:02 smithi149 ceph-mon[170394]: failed to encode map e209 with expected crc 2023-09-23T20:27:02.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:02 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1702836917' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:27:02.469 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:27:02.472 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_0 2023-09-23T20:27:02.472 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd pool rm unique_pool_0 unique_pool_0 --yes-i-really-really-mean-it 2023-09-23T20:27:02.496 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:27:02.580 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:02.583 INFO:teuthology.orchestra.run.smithi018.stdout:ERROR: (22) Invalid argument 2023-09-23T20:27:02.584 INFO:teuthology.orchestra.run.smithi018.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:27:02.621 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:27:02.630 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:27:02.634 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:02.635 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:27:02.736 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:27:02.763 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:02.795 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:27:02.798 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:02.798 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:27:02.899 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:27:02.925 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:02.958 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:27:02.960 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:02.961 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:27:03.061 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:27:03.094 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:03.303 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:03 smithi018 ceph-mon[184144]: pgmap v453: 109 pgs: 1 peering, 1 active+recovering+undersized+remapped, 105 active+clean, 2 active+recovery_wait+undersized+degraded+remapped; 18 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 447 op/s; 2003/717 objects degraded (279.358%); 0 B/s, 150 objects/s recovering 2023-09-23T20:27:03.303 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:03 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:27:03.303 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:03 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:27:03.303 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:03 smithi018 ceph-mon[179647]: pgmap v453: 109 pgs: 1 peering, 1 active+recovering+undersized+remapped, 105 active+clean, 2 active+recovery_wait+undersized+degraded+remapped; 18 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 447 op/s; 2003/717 objects degraded (279.358%); 0 B/s, 150 objects/s recovering 2023-09-23T20:27:03.303 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:03 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:27:03.303 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:03 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:27:03.343 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T20:27:03.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:03 smithi149 ceph-mon[170394]: pgmap v453: 109 pgs: 1 peering, 1 active+recovering+undersized+remapped, 105 active+clean, 2 active+recovery_wait+undersized+degraded+remapped; 18 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 0 B/s wr, 447 op/s; 2003/717 objects degraded (279.358%); 0 B/s, 150 objects/s recovering 2023-09-23T20:27:03.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:03 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:27:03.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:03 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:27:03.737 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:27:03.741 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:03.741 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:27:03.842 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:27:03.870 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:03.957 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:27:03.961 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:03.961 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:27:04.062 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:27:04.090 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:04.101 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:27:04.123 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:27:04.126 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:04.126 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:27:04.177 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:27:04.185 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:04 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:27:04.185 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:04 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:27:04.185 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:04 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:27:04.185 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:04 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:27:04.186 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:04 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:27:04.186 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:04 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:27:04.227 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:27:04.253 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:04.286 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:27:04.288 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:04.289 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:27:04.389 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:27:04.423 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:04.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:04 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:27:04.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:04 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:27:04.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:04 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:27:04.667 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:27:04.881 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:27:04.884 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:04.884 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:27:04.985 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:27:05.011 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:05.025 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:27:05.027 INFO:teuthology.orchestra.run.smithi018.stderr:dumped all 2023-09-23T20:27:05.044 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:27:05.045 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:27:04 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:27:04] "GET /metrics HTTP/1.1" 200 36454 "" "Prometheus/2.43.0" 2023-09-23T20:27:05.046 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:05.047 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:27:05.148 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:27:05.174 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:05.205 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:27:05.208 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:05.209 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:27:05.239 INFO:teuthology.orchestra.run.smithi018.stderr:nodeep-scrub is unset 2023-09-23T20:27:05.309 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:27:05.339 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:05 smithi018 ceph-mon[184144]: 5.19 scrub starts 2023-09-23T20:27:05.339 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:05 smithi018 ceph-mon[184144]: 5.19 scrub ok 2023-09-23T20:27:05.339 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:05 smithi018 ceph-mon[184144]: pgmap v454: 109 pgs: 1 peering, 106 active+clean, 2 active+recovery_wait+undersized+degraded+remapped; 9.1 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 2003/300 objects degraded (667.667%) 2023-09-23T20:27:05.339 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:05 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3641861129' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:27:05.339 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:05 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:27:05.340 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:05 smithi018 ceph-mon[179647]: 5.19 scrub starts 2023-09-23T20:27:05.340 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:05 smithi018 ceph-mon[179647]: 5.19 scrub ok 2023-09-23T20:27:05.340 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:05 smithi018 ceph-mon[179647]: pgmap v454: 109 pgs: 1 peering, 106 active+clean, 2 active+recovery_wait+undersized+degraded+remapped; 9.1 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 2003/300 objects degraded (667.667%) 2023-09-23T20:27:05.340 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:05 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3641861129' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:27:05.340 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:05 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:27:05.343 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:05.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:05 smithi149 ceph-mon[170394]: 5.19 scrub starts 2023-09-23T20:27:05.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:05 smithi149 ceph-mon[170394]: 5.19 scrub ok 2023-09-23T20:27:05.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:05 smithi149 ceph-mon[170394]: pgmap v454: 109 pgs: 1 peering, 106 active+clean, 2 active+recovery_wait+undersized+degraded+remapped; 9.1 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 2003/300 objects degraded (667.667%) 2023-09-23T20:27:05.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:05 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3641861129' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:27:05.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:05 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:27:05.834 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:27:05.837 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:05.837 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:27:05.938 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:27:05.967 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:06.002 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:27:06.004 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:06.005 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:27:06.010 INFO:teuthology.orchestra.run.smithi018.stdout:{"pg_ready":true,"pg_map":{"version":454,"stamp":"2023-09-23T20:27:03.549039+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":9574323,"num_objects":100,"num_object_clones":0,"num_object_copies":300,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2003,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":6496,"num_read_kb":14137,"num_write":193036,"num_write_kb":5655142,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":25432,"num_bytes_recovered":1043270572,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":181274,"ondisk_log_size":181274,"up":327,"acting":325,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":345,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":1585428,"kb_used_data":575252,"kb_used_omap":30,"kb_used_meta":1010081,"kb_avail":748375788,"statfs":{"total":767960285184,"available":766336806912,"internally_reserved":0,"allocated":589058048,"data_stored":582152358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31263,"internal_metadata":1034323425},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,7,2,3],"upper_bound":32},"perf_stat":{"commit_latency_ms":9,"apply_latency_ms":9,"commit_latency_ns":9000000,"apply_latency_ns":9000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-96796696,"num_objects":-1479,"num_object_clones":0,"num_object_copies":-4437,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-604,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-1479,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":-5038,"num_write_kb":-210816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-1528,"num_bytes_recovered":-161677312,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"5.416088"},"pg_stats":[{"pgid":"3.1a","version":"69'1","reported_seq":205,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T20:26:59.153634+0000","last_change":"2023-09-23T20:25:39.632437+0000","last_active":"2023-09-23T20:26:59.153634+0000","last_peered":"2023-09-23T20:26:59.153634+0000","last_clean":"2023-09-23T20:26:59.153634+0000","last_became_active":"2023-09-23T20:25:39.632074+0000","last_became_peered":"2023-09-23T20:25:39.632074+0000","last_unstale":"2023-09-23T20:26:59.153634+0000","last_undegraded":"2023-09-23T20:26:59.153634+0000","last_fullsized":"2023-09-23T20:26:59.153634+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:32:13.543799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":54,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T20:27:00.222305+0000","last_change":"2023-09-23T20:25:02.932223+0000","last_active":"2023-09-23T20:27:00.222305+0000","last_peered":"2023-09-23T20:27:00.222305+0000","last_clean":"2023-09-23T20:27:00.222305+0000","last_became_active":"2023-09-23T20:25:02.931024+0000","last_became_peered":"2023-09-23T20:25:02.931024+0000","last_unstale":"2023-09-23T20:27:00.222305+0000","last_undegraded":"2023-09-23T20:27:00.222305+0000","last_fullsized":"2023-09-23T20:27:00.222305+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:16:00.586211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":164,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-09-23T20:26:58.136181+0000","last_change":"2023-09-23T20:25:02.934543+0000","last_active":"2023-09-23T20:26:58.136181+0000","last_peered":"2023-09-23T20:26:58.136181+0000","last_clean":"2023-09-23T20:26:58.136181+0000","last_became_active":"2023-09-23T20:25:02.934178+0000","last_became_peered":"2023-09-23T20:25:02.934178+0000","last_unstale":"2023-09-23T20:26:58.136181+0000","last_undegraded":"2023-09-23T20:26:58.136181+0000","last_fullsized":"2023-09-23T20:26:58.136181+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:27:38.453052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":170,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-09-23T20:26:58.141117+0000","last_change":"2023-09-23T20:25:02.957185+0000","last_active":"2023-09-23T20:26:58.141117+0000","last_peered":"2023-09-23T20:26:58.141117+0000","last_clean":"2023-09-23T20:26:58.141117+0000","last_became_active":"2023-09-23T20:25:02.956689+0000","last_became_peered":"2023-09-23T20:25:02.956689+0000","last_unstale":"2023-09-23T20:26:58.141117+0000","last_undegraded":"2023-09-23T20:26:58.141117+0000","last_fullsized":"2023-09-23T20:26:58.141117+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:04:28.524015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":54,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T20:27:00.221812+0000","last_change":"2023-09-23T20:25:02.932054+0000","last_active":"2023-09-23T20:27:00.221812+0000","last_peered":"2023-09-23T20:27:00.221812+0000","last_clean":"2023-09-23T20:27:00.221812+0000","last_became_active":"2023-09-23T20:25:02.931309+0000","last_became_peered":"2023-09-23T20:25:02.931309+0000","last_unstale":"2023-09-23T20:27:00.221812+0000","last_undegraded":"2023-09-23T20:27:00.221812+0000","last_fullsized":"2023-09-23T20:27:00.221812+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:57:53.378499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":167,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T20:26:59.854376+0000","last_change":"2023-09-23T20:20:46.937089+0000","last_active":"2023-09-23T20:26:59.854376+0000","last_peered":"2023-09-23T20:26:59.854376+0000","last_clean":"2023-09-23T20:26:59.854376+0000","last_became_active":"2023-09-23T20:20:46.936831+0000","last_became_peered":"2023-09-23T20:20:46.936831+0000","last_unstale":"2023-09-23T20:26:59.854376+0000","last_undegraded":"2023-09-23T20:26:59.854376+0000","last_fullsized":"2023-09-23T20:26:59.854376+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:34:56.747010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.11","version":"205'7757","reported_seq":10504,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T20:26:59.853675+0000","last_change":"2023-09-23T20:23:31.161216+0000","last_active":"2023-09-23T20:26:59.853675+0000","last_peered":"2023-09-23T20:26:59.853675+0000","last_clean":"2023-09-23T20:26:59.853675+0000","last_became_active":"2023-09-23T20:23:31.146532+0000","last_became_peered":"2023-09-23T20:23:31.146532+0000","last_unstale":"2023-09-23T20:26:59.853675+0000","last_undegraded":"2023-09-23T20:26:59.853675+0000","last_fullsized":"2023-09-23T20:26:59.853675+0000","mapping_epoch":130,"log_start":"115'303","ondisk_log_start":"115'303","created":118,"last_epoch_clean":131,"parent":"0.0","parent_split_bits":5,"last_scrub":"119'1977","last_scrub_stamp":"2023-09-23T20:23:12.353414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:23:12.353414+0000","objects_scrubbed":1018,"log_size":7454,"log_dups_size":0,"ondisk_log_size":7454,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:28:29.630274+0000","scrub_duration":0.26235630100000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5790,"num_write_kb":152704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1282,"num_bytes_recovered":84017152,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":168,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-09-23T20:26:58.142562+0000","last_change":"2023-09-23T20:25:02.969432+0000","last_active":"2023-09-23T20:26:58.142562+0000","last_peered":"2023-09-23T20:26:58.142562+0000","last_clean":"2023-09-23T20:26:58.142562+0000","last_became_active":"2023-09-23T20:25:02.969118+0000","last_became_peered":"2023-09-23T20:25:02.969118+0000","last_unstale":"2023-09-23T20:26:58.142562+0000","last_undegraded":"2023-09-23T20:26:58.142562+0000","last_fullsized":"2023-09-23T20:26:58.142562+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:02:22.453831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":54,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T20:27:00.221783+0000","last_change":"2023-09-23T20:25:02.932225+0000","last_active":"2023-09-23T20:27:00.221783+0000","last_peered":"2023-09-23T20:27:00.221783+0000","last_clean":"2023-09-23T20:27:00.221783+0000","last_became_active":"2023-09-23T20:25:02.931746+0000","last_became_peered":"2023-09-23T20:25:02.931746+0000","last_unstale":"2023-09-23T20:27:00.221783+0000","last_undegraded":"2023-09-23T20:27:00.221783+0000","last_fullsized":"2023-09-23T20:27:00.221783+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:57:24.470179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.10","version":"205'7619","reported_seq":10630,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T20:26:59.152250+0000","last_change":"2023-09-23T20:23:24.880162+0000","last_active":"2023-09-23T20:26:59.152250+0000","last_peered":"2023-09-23T20:26:59.152250+0000","last_clean":"2023-09-23T20:26:59.152250+0000","last_became_active":"2023-09-23T20:23:24.867240+0000","last_became_peered":"2023-09-23T20:23:24.867240+0000","last_unstale":"2023-09-23T20:26:59.152250+0000","last_undegraded":"2023-09-23T20:26:59.152250+0000","last_fullsized":"2023-09-23T20:26:59.152250+0000","mapping_epoch":128,"log_start":"119'1861","ondisk_log_start":"119'1861","created":118,"last_epoch_clean":129,"parent":"0.0","parent_split_bits":5,"last_scrub":"119'2122","last_scrub_stamp":"2023-09-23T20:23:15.108101+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:23:15.108101+0000","objects_scrubbed":1199,"log_size":5758,"log_dups_size":0,"ondisk_log_size":5758,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:10:57.070009+0000","scrub_duration":0.30476455699999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5501,"num_write_kb":137664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1255,"num_bytes_recovered":82247680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":167,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T20:26:59.149683+0000","last_change":"2023-09-23T20:20:46.950281+0000","last_active":"2023-09-23T20:26:59.149683+0000","last_peered":"2023-09-23T20:26:59.149683+0000","last_clean":"2023-09-23T20:26:59.149683+0000","last_became_active":"2023-09-23T20:20:46.949892+0000","last_became_peered":"2023-09-23T20:20:46.949892+0000","last_unstale":"2023-09-23T20:26:59.149683+0000","last_undegraded":"2023-09-23T20:26:59.149683+0000","last_fullsized":"2023-09-23T20:26:59.149683+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:28.763832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":54,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T20:27:00.222014+0000","last_change":"2023-09-23T20:25:02.947669+0000","last_active":"2023-09-23T20:27:00.222014+0000","last_peered":"2023-09-23T20:27:00.222014+0000","last_clean":"2023-09-23T20:27:00.222014+0000","last_became_active":"2023-09-23T20:25:02.947368+0000","last_became_peered":"2023-09-23T20:25:02.947368+0000","last_unstale":"2023-09-23T20:27:00.222014+0000","last_undegraded":"2023-09-23T20:27:00.222014+0000","last_fullsized":"2023-09-23T20:27:00.222014+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:52:53.813761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.13","version":"205'7671","reported_seq":11938,"reported_epoch":206,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:26:58.140495+0000","last_change":"2023-09-23T20:25:40.848149+0000","last_active":"2023-09-23T20:26:58.140495+0000","last_peered":"2023-09-23T20:26:58.140495+0000","last_clean":"2023-09-23T20:24:36.774911+0000","last_became_active":"2023-09-23T20:25:02.945089+0000","last_became_peered":"2023-09-23T20:25:02.945089+0000","last_unstale":"2023-09-23T20:26:58.140495+0000","last_undegraded":"2023-09-23T20:25:40.847992+0000","last_fullsized":"2023-09-23T20:25:02.906341+0000","mapping_epoch":165,"log_start":"119'1970","ondisk_log_start":"119'1970","created":118,"last_epoch_clean":137,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:22:51.846999+0000","objects_scrubbed":0,"log_size":5701,"log_dups_size":0,"ondisk_log_size":5701,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":270,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6769,"num_write_kb":215840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1401,"num_bytes_recovered":85852160,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[0,1],"avail_no_missing":["0","1"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":203,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T20:26:59.153020+0000","last_change":"2023-09-23T20:25:03.038511+0000","last_active":"2023-09-23T20:26:59.153020+0000","last_peered":"2023-09-23T20:26:59.153020+0000","last_clean":"2023-09-23T20:26:59.153020+0000","last_became_active":"2023-09-23T20:25:02.976811+0000","last_became_peered":"2023-09-23T20:25:02.976811+0000","last_unstale":"2023-09-23T20:26:59.153020+0000","last_undegraded":"2023-09-23T20:26:59.153020+0000","last_fullsized":"2023-09-23T20:26:59.153020+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:28:06.605731+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":165,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-09-23T20:26:58.136291+0000","last_change":"2023-09-23T20:25:39.646452+0000","last_active":"2023-09-23T20:26:58.136291+0000","last_peered":"2023-09-23T20:26:58.136291+0000","last_clean":"2023-09-23T20:26:58.136291+0000","last_became_active":"2023-09-23T20:25:39.646175+0000","last_became_peered":"2023-09-23T20:25:39.646175+0000","last_unstale":"2023-09-23T20:26:58.136291+0000","last_undegraded":"2023-09-23T20:26:58.136291+0000","last_fullsized":"2023-09-23T20:26:58.136291+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:33:02.099951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.12","version":"205'7460","reported_seq":11566,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-09-23T20:26:58.136389+0000","last_change":"2023-09-23T20:26:14.067112+0000","last_active":"2023-09-23T20:26:58.136389+0000","last_peered":"2023-09-23T20:26:58.136389+0000","last_clean":"2023-09-23T20:26:58.136389+0000","last_became_active":"2023-09-23T20:26:14.046688+0000","last_became_peered":"2023-09-23T20:26:14.046688+0000","last_unstale":"2023-09-23T20:26:58.136389+0000","last_undegraded":"2023-09-23T20:26:58.136389+0000","last_fullsized":"2023-09-23T20:26:58.136389+0000","mapping_epoch":188,"log_start":"0'0","ondisk_log_start":"0'0","created":118,"last_epoch_clean":189,"parent":"0.0","parent_split_bits":5,"last_scrub":"119'2087","last_scrub_stamp":"2023-09-23T20:23:15.737284+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:23:15.737284+0000","objects_scrubbed":1166,"log_size":7460,"log_dups_size":0,"ondisk_log_size":7460,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:39:04.257684+0000","scrub_duration":0.27092271000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5386,"num_write_kb":135040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1590,"num_bytes_recovered":80216064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":172,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T20:26:59.153958+0000","last_change":"2023-09-23T20:12:43.750217+0000","last_active":"2023-09-23T20:26:59.153958+0000","last_peered":"2023-09-23T20:26:59.153958+0000","last_clean":"2023-09-23T20:26:59.153958+0000","last_became_active":"2023-09-23T20:12:43.749929+0000","last_became_peered":"2023-09-23T20:12:43.749929+0000","last_unstale":"2023-09-23T20:26:59.153958+0000","last_undegraded":"2023-09-23T20:26:59.153958+0000","last_fullsized":"2023-09-23T20:26:59.153958+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:50:28.470211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":166,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T20:26:59.153382+0000","last_change":"2023-09-23T20:20:46.939938+0000","last_active":"2023-09-23T20:26:59.153382+0000","last_peered":"2023-09-23T20:26:59.153382+0000","last_clean":"2023-09-23T20:26:59.153382+0000","last_became_active":"2023-09-23T20:20:46.938777+0000","last_became_peered":"2023-09-23T20:20:46.938777+0000","last_unstale":"2023-09-23T20:26:59.153382+0000","last_undegraded":"2023-09-23T20:26:59.153382+0000","last_fullsized":"2023-09-23T20:26:59.153382+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:45:10.329438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.15","version":"205'7451","reported_seq":11043,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T20:26:59.146586+0000","last_change":"2023-09-23T20:24:36.924738+0000","last_active":"2023-09-23T20:26:59.146586+0000","last_peered":"2023-09-23T20:26:59.146586+0000","last_clean":"2023-09-23T20:26:59.146586+0000","last_became_active":"2023-09-23T20:23:54.922070+0000","last_became_peered":"2023-09-23T20:23:54.922070+0000","last_unstale":"2023-09-23T20:26:59.146586+0000","last_undegraded":"2023-09-23T20:26:59.146586+0000","last_fullsized":"2023-09-23T20:26:59.146586+0000","mapping_epoch":143,"log_start":"0'0","ondisk_log_start":"0'0","created":118,"last_epoch_clean":156,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:22:51.846999+0000","objects_scrubbed":0,"log_size":7451,"log_dups_size":0,"ondisk_log_size":7451,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1966080,"num_objects":30,"num_object_clones":0,"num_object_copies":90,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":30,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6568,"num_write_kb":211136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1727,"num_bytes_recovered":113180672,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":202,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T20:26:59.152814+0000","last_change":"2023-09-23T20:25:39.654026+0000","last_active":"2023-09-23T20:26:59.152814+0000","last_peered":"2023-09-23T20:26:59.152814+0000","last_clean":"2023-09-23T20:26:59.152814+0000","last_became_active":"2023-09-23T20:25:39.626849+0000","last_became_peered":"2023-09-23T20:25:39.626849+0000","last_unstale":"2023-09-23T20:26:59.152814+0000","last_undegraded":"2023-09-23T20:26:59.152814+0000","last_fullsized":"2023-09-23T20:26:59.152814+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:22:53.957630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":54,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T20:27:00.222323+0000","last_change":"2023-09-23T20:25:02.962047+0000","last_active":"2023-09-23T20:27:00.222323+0000","last_peered":"2023-09-23T20:27:00.222323+0000","last_clean":"2023-09-23T20:27:00.222323+0000","last_became_active":"2023-09-23T20:25:02.961523+0000","last_became_peered":"2023-09-23T20:25:02.961523+0000","last_unstale":"2023-09-23T20:27:00.222323+0000","last_undegraded":"2023-09-23T20:27:00.222323+0000","last_fullsized":"2023-09-23T20:27:00.222323+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:18:11.830822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.14","version":"205'7480","reported_seq":13936,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T20:27:00.222491+0000","last_change":"2023-09-23T20:26:44.006941+0000","last_active":"2023-09-23T20:27:00.222491+0000","last_peered":"2023-09-23T20:27:00.222491+0000","last_clean":"2023-09-23T20:27:00.222491+0000","last_became_active":"2023-09-23T20:26:43.982576+0000","last_became_peered":"2023-09-23T20:26:43.982576+0000","last_unstale":"2023-09-23T20:27:00.222491+0000","last_undegraded":"2023-09-23T20:27:00.222491+0000","last_fullsized":"2023-09-23T20:27:00.222491+0000","mapping_epoch":201,"log_start":"0'0","ondisk_log_start":"0'0","created":118,"last_epoch_clean":202,"parent":"0.0","parent_split_bits":5,"last_scrub":"142'3061","last_scrub_stamp":"2023-09-23T20:23:51.058360+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:23:51.058360+0000","objects_scrubbed":2161,"log_size":7480,"log_dups_size":0,"ondisk_log_size":7480,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:39:55.532369+0000","scrub_duration":0.44581189700000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4431,"num_write_kb":72640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1218,"num_bytes_recovered":9175040,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":168,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-09-23T20:26:58.140755+0000","last_change":"2023-09-23T20:25:02.953993+0000","last_active":"2023-09-23T20:26:58.140755+0000","last_peered":"2023-09-23T20:26:58.140755+0000","last_clean":"2023-09-23T20:26:58.140755+0000","last_became_active":"2023-09-23T20:25:02.953672+0000","last_became_peered":"2023-09-23T20:25:02.953672+0000","last_unstale":"2023-09-23T20:26:58.140755+0000","last_undegraded":"2023-09-23T20:26:58.140755+0000","last_fullsized":"2023-09-23T20:26:58.140755+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:26:08.603577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":186,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T20:26:59.146701+0000","last_change":"2023-09-23T20:25:02.949584+0000","last_active":"2023-09-23T20:26:59.146701+0000","last_peered":"2023-09-23T20:26:59.146701+0000","last_clean":"2023-09-23T20:26:59.146701+0000","last_became_active":"2023-09-23T20:25:02.949132+0000","last_became_peered":"2023-09-23T20:25:02.949132+0000","last_unstale":"2023-09-23T20:26:59.146701+0000","last_undegraded":"2023-09-23T20:26:59.146701+0000","last_fullsized":"2023-09-23T20:26:59.146701+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:34:20.509823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.17","version":"205'7507","reported_seq":14083,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T20:27:01.076646+0000","last_change":"2023-09-23T20:27:01.076646+0000","last_active":"2023-09-23T20:27:01.076646+0000","last_peered":"2023-09-23T20:27:01.076646+0000","last_clean":"2023-09-23T20:27:01.076646+0000","last_became_active":"2023-09-23T20:26:20.147495+0000","last_became_peered":"2023-09-23T20:26:20.147495+0000","last_unstale":"2023-09-23T20:27:01.076646+0000","last_undegraded":"2023-09-23T20:27:01.076646+0000","last_fullsized":"2023-09-23T20:27:01.076646+0000","mapping_epoch":193,"log_start":"0'0","ondisk_log_start":"0'0","created":118,"last_epoch_clean":194,"parent":"0.0","parent_split_bits":5,"last_scrub":"205'7507","last_scrub_stamp":"2023-09-23T20:27:01.076520+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:27:01.076520+0000","objects_scrubbed":0,"log_size":7507,"log_dups_size":0,"ondisk_log_size":7507,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:30.902597+0000","scrub_duration":0.0039890009999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":151,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T20:26:59.152708+0000","last_change":"2023-09-23T20:12:43.749139+0000","last_active":"2023-09-23T20:26:59.152708+0000","last_peered":"2023-09-23T20:26:59.152708+0000","last_clean":"2023-09-23T20:26:59.152708+0000","last_became_active":"2023-09-23T20:12:43.748854+0000","last_became_peered":"2023-09-23T20:12:43.748854+0000","last_unstale":"2023-09-23T20:26:59.152708+0000","last_undegraded":"2023-09-23T20:26:59.152708+0000","last_fullsized":"2023-09-23T20:26:59.152708+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:57:11.630712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":178,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-09-23T20:26:58.141051+0000","last_change":"2023-09-23T20:25:02.957010+0000","last_active":"2023-09-23T20:26:58.141051+0000","last_peered":"2023-09-23T20:26:58.141051+0000","last_clean":"2023-09-23T20:26:58.141051+0000","last_became_active":"2023-09-23T20:25:02.956267+0000","last_became_peered":"2023-09-23T20:25:02.956267+0000","last_unstale":"2023-09-23T20:26:58.141051+0000","last_undegraded":"2023-09-23T20:26:58.141051+0000","last_fullsized":"2023-09-23T20:26:58.141051+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:10:55.299422+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.16","version":"205'7678","reported_seq":15145,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T20:26:59.146361+0000","last_change":"2023-09-23T20:26:16.974059+0000","last_active":"2023-09-23T20:26:59.146361+0000","last_peered":"2023-09-23T20:26:59.146361+0000","last_clean":"2023-09-23T20:26:59.146361+0000","last_became_active":"2023-09-23T20:26:16.954812+0000","last_became_peered":"2023-09-23T20:26:16.954812+0000","last_unstale":"2023-09-23T20:26:59.146361+0000","last_undegraded":"2023-09-23T20:26:59.146361+0000","last_fullsized":"2023-09-23T20:26:59.146361+0000","mapping_epoch":190,"log_start":"0'0","ondisk_log_start":"0'0","created":118,"last_epoch_clean":191,"parent":"0.0","parent_split_bits":5,"last_scrub":"119'2163","last_scrub_stamp":"2023-09-23T20:23:15.241911+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:23:15.241911+0000","objects_scrubbed":1192,"log_size":7678,"log_dups_size":0,"ondisk_log_size":7678,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:52:46.936495+0000","scrub_duration":0.33323710899999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5526,"num_write_kb":138688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2720,"num_bytes_recovered":154599424,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":176,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T20:26:59.853913+0000","last_change":"2023-09-23T20:20:14.881009+0000","last_active":"2023-09-23T20:26:59.853913+0000","last_peered":"2023-09-23T20:26:59.853913+0000","last_clean":"2023-09-23T20:26:59.853913+0000","last_became_active":"2023-09-23T20:20:14.880685+0000","last_became_peered":"2023-09-23T20:20:14.880685+0000","last_unstale":"2023-09-23T20:26:59.853913+0000","last_undegraded":"2023-09-23T20:26:59.853913+0000","last_fullsized":"2023-09-23T20:26:59.853913+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:28.010434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":54,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T20:27:00.223067+0000","last_change":"2023-09-23T20:25:02.977462+0000","last_active":"2023-09-23T20:27:00.223067+0000","last_peered":"2023-09-23T20:27:00.223067+0000","last_clean":"2023-09-23T20:27:00.223067+0000","last_became_active":"2023-09-23T20:25:02.969910+0000","last_became_peered":"2023-09-23T20:25:02.969910+0000","last_unstale":"2023-09-23T20:27:00.223067+0000","last_undegraded":"2023-09-23T20:27:00.223067+0000","last_fullsized":"2023-09-23T20:27:00.223067+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:22:55.644955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.9","version":"205'7418","reported_seq":7537,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T20:27:00.454332+0000","last_change":"2023-09-23T20:27:00.454332+0000","last_active":"2023-09-23T20:27:00.454332+0000","last_peered":"2023-09-23T20:27:00.454332+0000","last_clean":"2023-09-23T20:27:00.454332+0000","last_became_active":"2023-09-23T20:23:10.971930+0000","last_became_peered":"2023-09-23T20:23:10.971930+0000","last_unstale":"2023-09-23T20:27:00.454332+0000","last_undegraded":"2023-09-23T20:27:00.454332+0000","last_fullsized":"2023-09-23T20:27:00.454332+0000","mapping_epoch":111,"log_start":"117'1659","ondisk_log_start":"117'1659","created":111,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:22:51.846999+0000","objects_scrubbed":0,"log_size":5759,"log_dups_size":0,"ondisk_log_size":5759,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1277952,"num_objects":19,"num_object_clones":0,"num_object_copies":57,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":19,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6535,"num_write_kb":209760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":169,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T20:26:59.154224+0000","last_change":"2023-09-23T20:20:14.883283+0000","last_active":"2023-09-23T20:26:59.154224+0000","last_peered":"2023-09-23T20:26:59.154224+0000","last_clean":"2023-09-23T20:26:59.154224+0000","last_became_active":"2023-09-23T20:20:14.882806+0000","last_became_peered":"2023-09-23T20:20:14.882806+0000","last_unstale":"2023-09-23T20:26:59.154224+0000","last_undegraded":"2023-09-23T20:26:59.154224+0000","last_fullsized":"2023-09-23T20:26:59.154224+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:23:35.295701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":149,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T20:26:59.146143+0000","last_change":"2023-09-23T20:12:45.179211+0000","last_active":"2023-09-23T20:26:59.146143+0000","last_peered":"2023-09-23T20:26:59.146143+0000","last_clean":"2023-09-23T20:26:59.146143+0000","last_became_active":"2023-09-23T20:12:45.178966+0000","last_became_peered":"2023-09-23T20:12:45.178966+0000","last_unstale":"2023-09-23T20:26:59.146143+0000","last_undegraded":"2023-09-23T20:26:59.146143+0000","last_fullsized":"2023-09-23T20:26:59.146143+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:58.902140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.8","version":"205'7607","reported_seq":9526,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T20:27:00.143621+0000","last_change":"2023-09-23T20:26:51.971852+0000","last_active":"2023-09-23T20:27:00.143621+0000","last_peered":"2023-09-23T20:27:00.143621+0000","last_clean":"2023-09-23T20:27:00.143621+0000","last_became_active":"2023-09-23T20:26:51.952240+0000","last_became_peered":"2023-09-23T20:26:51.952240+0000","last_unstale":"2023-09-23T20:27:00.143621+0000","last_undegraded":"2023-09-23T20:27:00.143621+0000","last_fullsized":"2023-09-23T20:27:00.143621+0000","mapping_epoch":204,"log_start":"117'1724","ondisk_log_start":"117'1724","created":111,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"120'2144","last_scrub_stamp":"2023-09-23T20:23:17.129360+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:23:17.129360+0000","objects_scrubbed":1209,"log_size":5883,"log_dups_size":0,"ondisk_log_size":5883,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:00:36.612715+0000","scrub_duration":0.17393181099999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5463,"num_write_kb":136128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":869,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":170,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T20:26:59.154135+0000","last_change":"2023-09-23T20:20:46.950784+0000","last_active":"2023-09-23T20:26:59.154135+0000","last_peered":"2023-09-23T20:26:59.154135+0000","last_clean":"2023-09-23T20:26:59.154135+0000","last_became_active":"2023-09-23T20:20:46.950055+0000","last_became_peered":"2023-09-23T20:20:46.950055+0000","last_unstale":"2023-09-23T20:26:59.154135+0000","last_undegraded":"2023-09-23T20:26:59.154135+0000","last_fullsized":"2023-09-23T20:26:59.154135+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:23:07.645427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":203,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T20:26:59.154542+0000","last_change":"2023-09-23T20:25:02.943743+0000","last_active":"2023-09-23T20:26:59.154542+0000","last_peered":"2023-09-23T20:26:59.154542+0000","last_clean":"2023-09-23T20:26:59.154542+0000","last_became_active":"2023-09-23T20:25:02.943279+0000","last_became_peered":"2023-09-23T20:25:02.943279+0000","last_unstale":"2023-09-23T20:26:59.154542+0000","last_undegraded":"2023-09-23T20:26:59.154542+0000","last_fullsized":"2023-09-23T20:26:59.154542+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:21:36.128366+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.b","version":"205'13232","reported_seq":13351,"reported_epoch":206,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:26:58.141040+0000","last_change":"2023-09-23T20:25:39.764057+0000","last_active":"2023-09-23T20:26:58.141040+0000","last_peered":"2023-09-23T20:26:58.141040+0000","last_clean":"2023-09-23T20:24:52.031628+0000","last_became_active":"2023-09-23T20:25:39.743845+0000","last_became_peered":"2023-09-23T20:25:39.743845+0000","last_unstale":"2023-09-23T20:26:58.141040+0000","last_undegraded":"2023-09-23T20:25:39.622936+0000","last_fullsized":"2023-09-23T20:25:39.612475+0000","mapping_epoch":176,"log_start":"159'7472","ondisk_log_start":"159'7472","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:22:51.846999+0000","objects_scrubbed":0,"log_size":5760,"log_dups_size":0,"ondisk_log_size":5760,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:56:57.182858+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1733,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13232,"num_write_kb":423424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":152,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T20:26:59.152923+0000","last_change":"2023-09-23T20:12:43.751523+0000","last_active":"2023-09-23T20:26:59.152923+0000","last_peered":"2023-09-23T20:26:59.152923+0000","last_clean":"2023-09-23T20:26:59.152923+0000","last_became_active":"2023-09-23T20:12:43.751257+0000","last_became_peered":"2023-09-23T20:12:43.751257+0000","last_unstale":"2023-09-23T20:26:59.152923+0000","last_undegraded":"2023-09-23T20:26:59.152923+0000","last_fullsized":"2023-09-23T20:26:59.152923+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:50:27.040225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":167,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T20:26:59.854188+0000","last_change":"2023-09-23T20:20:46.934762+0000","last_active":"2023-09-23T20:26:59.854188+0000","last_peered":"2023-09-23T20:26:59.854188+0000","last_clean":"2023-09-23T20:26:59.854188+0000","last_became_active":"2023-09-23T20:20:46.934472+0000","last_became_peered":"2023-09-23T20:20:46.934472+0000","last_unstale":"2023-09-23T20:26:59.854188+0000","last_undegraded":"2023-09-23T20:26:59.854188+0000","last_fullsized":"2023-09-23T20:26:59.854188+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:28:42.708286+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.a","version":"205'13428","reported_seq":22080,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T20:27:00.144987+0000","last_change":"2023-09-23T20:26:13.071682+0000","last_active":"2023-09-23T20:27:00.144987+0000","last_peered":"2023-09-23T20:27:00.144987+0000","last_clean":"2023-09-23T20:27:00.144987+0000","last_became_active":"2023-09-23T20:26:13.034652+0000","last_became_peered":"2023-09-23T20:26:13.034652+0000","last_unstale":"2023-09-23T20:27:00.144987+0000","last_undegraded":"2023-09-23T20:27:00.144987+0000","last_fullsized":"2023-09-23T20:27:00.144987+0000","mapping_epoch":187,"log_start":"144'4713","ondisk_log_start":"144'4713","created":111,"last_epoch_clean":188,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:22:51.846999+0000","objects_scrubbed":0,"log_size":8715,"log_dups_size":0,"ondisk_log_size":8715,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:05:48.610901+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13428,"num_write_kb":429696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3973,"num_bytes_recovered":115343360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":187,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T20:26:59.146296+0000","last_change":"2023-09-23T20:25:02.947631+0000","last_active":"2023-09-23T20:26:59.146296+0000","last_peered":"2023-09-23T20:26:59.146296+0000","last_clean":"2023-09-23T20:26:59.146296+0000","last_became_active":"2023-09-23T20:25:02.947268+0000","last_became_peered":"2023-09-23T20:25:02.947268+0000","last_unstale":"2023-09-23T20:26:59.146296+0000","last_undegraded":"2023-09-23T20:26:59.146296+0000","last_fullsized":"2023-09-23T20:26:59.146296+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:19:43.643985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":152,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T20:26:59.148738+0000","last_change":"2023-09-23T20:20:46.935968+0000","last_active":"2023-09-23T20:26:59.148738+0000","last_peered":"2023-09-23T20:26:59.148738+0000","last_clean":"2023-09-23T20:26:59.148738+0000","last_became_active":"2023-09-23T20:20:46.935618+0000","last_became_peered":"2023-09-23T20:20:46.935618+0000","last_unstale":"2023-09-23T20:26:59.148738+0000","last_undegraded":"2023-09-23T20:26:59.148738+0000","last_fullsized":"2023-09-23T20:26:59.148738+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:42:59.867176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":560,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T20:26:59.152376+0000","last_change":"2023-09-23T20:25:21.167520+0000","last_active":"2023-09-23T20:26:59.152376+0000","last_peered":"2023-09-23T20:26:59.152376+0000","last_clean":"2023-09-23T20:26:59.152376+0000","last_became_active":"2023-09-23T20:25:21.166811+0000","last_became_peered":"2023-09-23T20:25:21.166811+0000","last_unstale":"2023-09-23T20:26:59.152376+0000","last_undegraded":"2023-09-23T20:26:59.152376+0000","last_fullsized":"2023-09-23T20:26:59.152376+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:09:47.213130+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.d","version":"205'13608","reported_seq":19097,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T20:27:00.145040+0000","last_change":"2023-09-23T20:26:33.302039+0000","last_active":"2023-09-23T20:27:00.145040+0000","last_peered":"2023-09-23T20:27:00.145040+0000","last_clean":"2023-09-23T20:27:00.145040+0000","last_became_active":"2023-09-23T20:26:33.259615+0000","last_became_peered":"2023-09-23T20:26:33.259615+0000","last_unstale":"2023-09-23T20:27:00.145040+0000","last_undegraded":"2023-09-23T20:27:00.145040+0000","last_fullsized":"2023-09-23T20:27:00.145040+0000","mapping_epoch":197,"log_start":"154'6618","ondisk_log_start":"154'6618","created":111,"last_epoch_clean":198,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:22:51.846999+0000","objects_scrubbed":0,"log_size":6990,"log_dups_size":0,"ondisk_log_size":6990,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:43:31.739694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":13608,"num_write_kb":435393,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2675,"num_bytes_recovered":61145088,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":58,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T20:27:00.222128+0000","last_change":"2023-09-23T20:25:03.000957+0000","last_active":"2023-09-23T20:27:00.222128+0000","last_peered":"2023-09-23T20:27:00.222128+0000","last_clean":"2023-09-23T20:27:00.222128+0000","last_became_active":"2023-09-23T20:25:02.939903+0000","last_became_peered":"2023-09-23T20:25:02.939903+0000","last_unstale":"2023-09-23T20:27:00.222128+0000","last_undegraded":"2023-09-23T20:27:00.222128+0000","last_fullsized":"2023-09-23T20:27:00.222128+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:51:14.981461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":186,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T20:26:59.153763+0000","last_change":"2023-09-23T20:25:02.929447+0000","last_active":"2023-09-23T20:26:59.153763+0000","last_peered":"2023-09-23T20:26:59.153763+0000","last_clean":"2023-09-23T20:26:59.153763+0000","last_became_active":"2023-09-23T20:25:02.929173+0000","last_became_peered":"2023-09-23T20:25:02.929173+0000","last_unstale":"2023-09-23T20:26:59.153763+0000","last_undegraded":"2023-09-23T20:26:59.153763+0000","last_fullsized":"2023-09-23T20:26:59.153763+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:32:09.280647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":560,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T20:26:59.151884+0000","last_change":"2023-09-23T20:25:21.166037+0000","last_active":"2023-09-23T20:26:59.151884+0000","last_peered":"2023-09-23T20:26:59.151884+0000","last_clean":"2023-09-23T20:26:59.151884+0000","last_became_active":"2023-09-23T20:25:21.165726+0000","last_became_peered":"2023-09-23T20:25:21.165726+0000","last_unstale":"2023-09-23T20:26:59.151884+0000","last_undegraded":"2023-09-23T20:26:59.151884+0000","last_fullsized":"2023-09-23T20:26:59.151884+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:09:47.213130+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":22,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.c","version":"205'13424","reported_seq":13530,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-09-23T20:26:58.136275+0000","last_change":"2023-09-23T20:22:52.866632+0000","last_active":"2023-09-23T20:26:58.136275+0000","last_peered":"2023-09-23T20:26:58.136275+0000","last_clean":"2023-09-23T20:26:58.136275+0000","last_became_active":"2023-09-23T20:22:52.866356+0000","last_became_peered":"2023-09-23T20:22:52.866356+0000","last_unstale":"2023-09-23T20:26:58.136275+0000","last_undegraded":"2023-09-23T20:26:58.136275+0000","last_fullsized":"2023-09-23T20:26:58.136275+0000","mapping_epoch":111,"log_start":"154'6673","ondisk_log_start":"154'6673","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:22:51.846999+0000","objects_scrubbed":0,"log_size":6751,"log_dups_size":0,"ondisk_log_size":6751,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:10:12.356062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13424,"num_write_kb":429568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":168,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T20:26:59.854066+0000","last_change":"2023-09-23T20:20:46.937163+0000","last_active":"2023-09-23T20:26:59.854066+0000","last_peered":"2023-09-23T20:26:59.854066+0000","last_clean":"2023-09-23T20:26:59.854066+0000","last_became_active":"2023-09-23T20:20:46.936757+0000","last_became_peered":"2023-09-23T20:20:46.936757+0000","last_unstale":"2023-09-23T20:26:59.854066+0000","last_undegraded":"2023-09-23T20:26:59.854066+0000","last_fullsized":"2023-09-23T20:26:59.854066+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:09.401379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":196,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T20:26:59.153309+0000","last_change":"2023-09-23T20:25:39.624636+0000","last_active":"2023-09-23T20:26:59.153309+0000","last_peered":"2023-09-23T20:26:59.153309+0000","last_clean":"2023-09-23T20:26:59.153309+0000","last_became_active":"2023-09-23T20:25:39.624297+0000","last_became_peered":"2023-09-23T20:25:39.624297+0000","last_unstale":"2023-09-23T20:26:59.153309+0000","last_undegraded":"2023-09-23T20:26:59.153309+0000","last_fullsized":"2023-09-23T20:26:59.153309+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:33:47.448737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.f","version":"205'13240","reported_seq":13345,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T20:26:59.146653+0000","last_change":"2023-09-23T20:22:52.872510+0000","last_active":"2023-09-23T20:26:59.146653+0000","last_peered":"2023-09-23T20:26:59.146653+0000","last_clean":"2023-09-23T20:26:59.146653+0000","last_became_active":"2023-09-23T20:22:52.872179+0000","last_became_peered":"2023-09-23T20:22:52.872179+0000","last_unstale":"2023-09-23T20:26:59.146653+0000","last_undegraded":"2023-09-23T20:26:59.146653+0000","last_fullsized":"2023-09-23T20:26:59.146653+0000","mapping_epoch":111,"log_start":"153'5695","ondisk_log_start":"153'5695","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:22:51.846999+0000","objects_scrubbed":0,"log_size":7545,"log_dups_size":0,"ondisk_log_size":7545,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:53.012272+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13240,"num_write_kb":423680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":187,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T20:26:59.154334+0000","last_change":"2023-09-23T20:25:39.628097+0000","last_active":"2023-09-23T20:26:59.154334+0000","last_peered":"2023-09-23T20:26:59.154334+0000","last_clean":"2023-09-23T20:26:59.154334+0000","last_became_active":"2023-09-23T20:25:39.627806+0000","last_became_peered":"2023-09-23T20:25:39.627806+0000","last_unstale":"2023-09-23T20:26:59.154334+0000","last_undegraded":"2023-09-23T20:26:59.154334+0000","last_fullsized":"2023-09-23T20:26:59.154334+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:53:44.398854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":166,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T20:26:59.154977+0000","last_change":"2023-09-23T20:20:46.939720+0000","last_active":"2023-09-23T20:26:59.154977+0000","last_peered":"2023-09-23T20:26:59.154977+0000","last_clean":"2023-09-23T20:26:59.154977+0000","last_became_active":"2023-09-23T20:20:46.939357+0000","last_became_peered":"2023-09-23T20:20:46.939357+0000","last_unstale":"2023-09-23T20:26:59.154977+0000","last_undegraded":"2023-09-23T20:26:59.154977+0000","last_fullsized":"2023-09-23T20:26:59.154977+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:56.613212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":560,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T20:26:59.152297+0000","last_change":"2023-09-23T20:25:21.167297+0000","last_active":"2023-09-23T20:26:59.152297+0000","last_peered":"2023-09-23T20:26:59.152297+0000","last_clean":"2023-09-23T20:26:59.152297+0000","last_became_active":"2023-09-23T20:25:21.167006+0000","last_became_peered":"2023-09-23T20:25:21.167006+0000","last_unstale":"2023-09-23T20:26:59.152297+0000","last_undegraded":"2023-09-23T20:26:59.152297+0000","last_fullsized":"2023-09-23T20:26:59.152297+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:09:47.213130+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.e","version":"205'13286","reported_seq":13390,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T20:26:59.155002+0000","last_change":"2023-09-23T20:22:52.871704+0000","last_active":"2023-09-23T20:26:59.155002+0000","last_peered":"2023-09-23T20:26:59.155002+0000","last_clean":"2023-09-23T20:26:59.155002+0000","last_became_active":"2023-09-23T20:22:52.871267+0000","last_became_peered":"2023-09-23T20:22:52.871267+0000","last_unstale":"2023-09-23T20:26:59.155002+0000","last_undegraded":"2023-09-23T20:26:59.155002+0000","last_fullsized":"2023-09-23T20:26:59.155002+0000","mapping_epoch":111,"log_start":"154'6704","ondisk_log_start":"154'6704","created":111,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:22:51.846999+0000","objects_scrubbed":0,"log_size":6582,"log_dups_size":0,"ondisk_log_size":6582,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:50:07.619801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13286,"num_write_kb":425152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":54,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T20:27:00.221882+0000","last_change":"2023-09-23T20:25:02.978173+0000","last_active":"2023-09-23T20:27:00.221882+0000","last_peered":"2023-09-23T20:27:00.221882+0000","last_clean":"2023-09-23T20:27:00.221882+0000","last_became_active":"2023-09-23T20:25:02.977856+0000","last_became_peered":"2023-09-23T20:25:02.977856+0000","last_unstale":"2023-09-23T20:27:00.221882+0000","last_undegraded":"2023-09-23T20:27:00.221882+0000","last_fullsized":"2023-09-23T20:27:00.221882+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:19.861943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":66,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-09-23T20:26:58.137396+0000","last_change":"2023-09-23T20:25:02.924911+0000","last_active":"2023-09-23T20:26:58.137396+0000","last_peered":"2023-09-23T20:26:58.137396+0000","last_clean":"2023-09-23T20:26:58.137396+0000","last_became_active":"2023-09-23T20:25:02.924586+0000","last_became_peered":"2023-09-23T20:25:02.924586+0000","last_unstale":"2023-09-23T20:26:58.137396+0000","last_undegraded":"2023-09-23T20:26:58.137396+0000","last_fullsized":"2023-09-23T20:26:58.137396+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:24:44.176766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":560,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T20:26:59.154221+0000","last_change":"2023-09-23T20:25:21.167164+0000","last_active":"2023-09-23T20:26:59.154221+0000","last_peered":"2023-09-23T20:26:59.154221+0000","last_clean":"2023-09-23T20:26:59.154221+0000","last_became_active":"2023-09-23T20:25:21.166869+0000","last_became_peered":"2023-09-23T20:25:21.166869+0000","last_unstale":"2023-09-23T20:26:59.154221+0000","last_undegraded":"2023-09-23T20:26:59.154221+0000","last_fullsized":"2023-09-23T20:26:59.154221+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:09:47.213130+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"86'119","reported_seq":541,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T20:26:59.143783+0000","last_change":"2023-09-23T20:20:46.937071+0000","last_active":"2023-09-23T20:26:59.143783+0000","last_peered":"2023-09-23T20:26:59.143783+0000","last_clean":"2023-09-23T20:26:59.143783+0000","last_became_active":"2023-09-23T20:20:46.935348+0000","last_became_peered":"2023-09-23T20:20:46.935348+0000","last_unstale":"2023-09-23T20:26:59.143783+0000","last_undegraded":"2023-09-23T20:26:59.143783+0000","last_fullsized":"2023-09-23T20:26:59.143783+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":119,"log_dups_size":0,"ondisk_log_size":119,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:09.829030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032705499999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":523,"num_read_kb":3503,"num_write":136,"num_write_kb":3128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":224,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T20:26:59.146723+0000","last_change":"2023-09-23T20:25:39.637160+0000","last_active":"2023-09-23T20:26:59.146723+0000","last_peered":"2023-09-23T20:26:59.146723+0000","last_clean":"2023-09-23T20:26:59.146723+0000","last_became_active":"2023-09-23T20:25:39.636860+0000","last_became_peered":"2023-09-23T20:25:39.636860+0000","last_unstale":"2023-09-23T20:26:59.146723+0000","last_undegraded":"2023-09-23T20:26:59.146723+0000","last_fullsized":"2023-09-23T20:26:59.146723+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:21:47.860063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":133,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T20:26:59.142928+0000","last_change":"2023-09-23T20:20:46.937104+0000","last_active":"2023-09-23T20:26:59.142928+0000","last_peered":"2023-09-23T20:26:59.142928+0000","last_clean":"2023-09-23T20:26:59.142928+0000","last_became_active":"2023-09-23T20:20:46.936820+0000","last_became_peered":"2023-09-23T20:20:46.936820+0000","last_unstale":"2023-09-23T20:26:59.142928+0000","last_undegraded":"2023-09-23T20:26:59.142928+0000","last_fullsized":"2023-09-23T20:26:59.142928+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:22:19.879310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.3","version":"205'7734","reported_seq":7851,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-09-23T20:26:58.141644+0000","last_change":"2023-09-23T20:23:20.060302+0000","last_active":"2023-09-23T20:26:58.141644+0000","last_peered":"2023-09-23T20:26:58.141644+0000","last_clean":"2023-09-23T20:26:58.141644+0000","last_became_active":"2023-09-23T20:23:11.059880+0000","last_became_peered":"2023-09-23T20:23:11.059880+0000","last_unstale":"2023-09-23T20:26:58.141644+0000","last_undegraded":"2023-09-23T20:26:58.141644+0000","last_fullsized":"2023-09-23T20:26:58.141644+0000","mapping_epoch":111,"log_start":"119'2003","ondisk_log_start":"119'2003","created":111,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:22:51.846999+0000","objects_scrubbed":0,"log_size":5731,"log_dups_size":0,"ondisk_log_size":5731,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1540096,"num_objects":23,"num_object_clones":0,"num_object_copies":69,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":23,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6831,"num_write_kb":219360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":559,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T20:26:59.152477+0000","last_change":"2023-09-23T20:25:21.160602+0000","last_active":"2023-09-23T20:26:59.152477+0000","last_peered":"2023-09-23T20:26:59.152477+0000","last_clean":"2023-09-23T20:26:59.152477+0000","last_became_active":"2023-09-23T20:25:21.160325+0000","last_became_peered":"2023-09-23T20:25:21.160325+0000","last_unstale":"2023-09-23T20:26:59.152477+0000","last_undegraded":"2023-09-23T20:26:59.152477+0000","last_fullsized":"2023-09-23T20:26:59.152477+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:09:47.213130+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.5","version":"205'7613","reported_seq":7730,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-09-23T20:26:58.141566+0000","last_change":"2023-09-23T20:23:11.152272+0000","last_active":"2023-09-23T20:26:58.141566+0000","last_peered":"2023-09-23T20:26:58.141566+0000","last_clean":"2023-09-23T20:26:58.141566+0000","last_became_active":"2023-09-23T20:23:11.055735+0000","last_became_peered":"2023-09-23T20:23:11.055735+0000","last_unstale":"2023-09-23T20:26:58.141566+0000","last_undegraded":"2023-09-23T20:26:58.141566+0000","last_fullsized":"2023-09-23T20:26:58.141566+0000","mapping_epoch":111,"log_start":"119'1895","ondisk_log_start":"119'1895","created":111,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:22:51.846999+0000","objects_scrubbed":0,"log_size":5718,"log_dups_size":0,"ondisk_log_size":5718,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6730,"num_write_kb":214400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"86'106","reported_seq":832,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T20:26:59.146312+0000","last_change":"2023-09-23T20:20:46.939468+0000","last_active":"2023-09-23T20:26:59.146312+0000","last_peered":"2023-09-23T20:26:59.146312+0000","last_clean":"2023-09-23T20:26:59.146312+0000","last_became_active":"2023-09-23T20:20:46.938954+0000","last_became_peered":"2023-09-23T20:20:46.938954+0000","last_unstale":"2023-09-23T20:26:59.146312+0000","last_undegraded":"2023-09-23T20:26:59.146312+0000","last_fullsized":"2023-09-23T20:26:59.146312+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":106,"log_dups_size":0,"ondisk_log_size":106,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:01:43.761064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00099302599999999993,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1105,"num_read_kb":1689,"num_write":100,"num_write_kb":404,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":167,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T20:26:59.153293+0000","last_change":"2023-09-23T20:20:14.887345+0000","last_active":"2023-09-23T20:26:59.153293+0000","last_peered":"2023-09-23T20:26:59.153293+0000","last_clean":"2023-09-23T20:26:59.153293+0000","last_became_active":"2023-09-23T20:20:14.886970+0000","last_became_peered":"2023-09-23T20:20:14.886970+0000","last_unstale":"2023-09-23T20:26:59.153293+0000","last_undegraded":"2023-09-23T20:26:59.153293+0000","last_fullsized":"2023-09-23T20:26:59.153293+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:00:38.575926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":199,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T20:26:59.143706+0000","last_change":"2023-09-23T20:25:39.631133+0000","last_active":"2023-09-23T20:26:59.143706+0000","last_peered":"2023-09-23T20:26:59.143706+0000","last_clean":"2023-09-23T20:26:59.143706+0000","last_became_active":"2023-09-23T20:25:39.630779+0000","last_became_peered":"2023-09-23T20:25:39.630779+0000","last_unstale":"2023-09-23T20:26:59.143706+0000","last_undegraded":"2023-09-23T20:26:59.143706+0000","last_fullsized":"2023-09-23T20:26:59.143706+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:39:52.635649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":560,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T20:26:59.152724+0000","last_change":"2023-09-23T20:25:21.167528+0000","last_active":"2023-09-23T20:26:59.152724+0000","last_peered":"2023-09-23T20:26:59.152724+0000","last_clean":"2023-09-23T20:26:59.152724+0000","last_became_active":"2023-09-23T20:25:21.167249+0000","last_became_peered":"2023-09-23T20:25:21.167249+0000","last_unstale":"2023-09-23T20:26:59.152724+0000","last_undegraded":"2023-09-23T20:26:59.152724+0000","last_fullsized":"2023-09-23T20:26:59.152724+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:09:47.213130+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.6","version":"205'7580","reported_seq":11902,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-09-23T20:27:01.162160+0000","last_change":"2023-09-23T20:27:00.181248+0000","last_active":"2023-09-23T20:27:01.162160+0000","last_peered":"2023-09-23T20:27:01.162160+0000","last_clean":"2023-09-23T20:27:01.162160+0000","last_became_active":"2023-09-23T20:27:00.155704+0000","last_became_peered":"2023-09-23T20:27:00.155704+0000","last_unstale":"2023-09-23T20:27:01.162160+0000","last_undegraded":"2023-09-23T20:27:01.162160+0000","last_fullsized":"2023-09-23T20:27:01.162160+0000","mapping_epoch":207,"log_start":"0'0","ondisk_log_start":"0'0","created":111,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"119'1954","last_scrub_stamp":"2023-09-23T20:23:12.084143+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:23:12.084143+0000","objects_scrubbed":1042,"log_size":7580,"log_dups_size":0,"ondisk_log_size":7580,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:49:22.105023+0000","scrub_duration":0.23299162700000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5630,"num_write_kb":146816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2235,"num_bytes_recovered":87162880,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"86'130","reported_seq":518,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T20:27:00.143844+0000","last_change":"2023-09-23T20:25:39.637458+0000","last_active":"2023-09-23T20:27:00.143844+0000","last_peered":"2023-09-23T20:27:00.143844+0000","last_clean":"2023-09-23T20:27:00.143844+0000","last_became_active":"2023-09-23T20:25:39.636773+0000","last_became_peered":"2023-09-23T20:25:39.636773+0000","last_unstale":"2023-09-23T20:27:00.143844+0000","last_undegraded":"2023-09-23T20:27:00.143844+0000","last_fullsized":"2023-09-23T20:27:00.143844+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:00:32.810564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011300049999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":512,"num_read_kb":1820,"num_write":133,"num_write_kb":1347,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":165,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-09-23T20:26:58.136870+0000","last_change":"2023-09-23T20:25:39.632283+0000","last_active":"2023-09-23T20:26:58.136870+0000","last_peered":"2023-09-23T20:26:58.136870+0000","last_clean":"2023-09-23T20:26:58.136870+0000","last_became_active":"2023-09-23T20:25:39.631819+0000","last_became_peered":"2023-09-23T20:25:39.631819+0000","last_unstale":"2023-09-23T20:26:58.136870+0000","last_undegraded":"2023-09-23T20:26:58.136870+0000","last_fullsized":"2023-09-23T20:26:58.136870+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:25:21.815555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":175,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-09-23T20:26:58.137163+0000","last_change":"2023-09-23T20:20:46.936524+0000","last_active":"2023-09-23T20:26:58.137163+0000","last_peered":"2023-09-23T20:26:58.137163+0000","last_clean":"2023-09-23T20:26:58.137163+0000","last_became_active":"2023-09-23T20:20:46.936267+0000","last_became_peered":"2023-09-23T20:20:46.936267+0000","last_unstale":"2023-09-23T20:26:58.137163+0000","last_undegraded":"2023-09-23T20:26:58.137163+0000","last_fullsized":"2023-09-23T20:26:58.137163+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:32:20.984521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"86'194","reported_seq":856,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T20:26:59.146478+0000","last_change":"2023-09-23T20:25:40.618665+0000","last_active":"2023-09-23T20:26:59.146478+0000","last_peered":"2023-09-23T20:26:59.146478+0000","last_clean":"2023-09-23T20:26:59.146478+0000","last_became_active":"2023-09-23T20:25:40.618087+0000","last_became_peered":"2023-09-23T20:25:40.618087+0000","last_unstale":"2023-09-23T20:26:59.146478+0000","last_undegraded":"2023-09-23T20:26:59.146478+0000","last_fullsized":"2023-09-23T20:26:59.146478+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":194,"log_dups_size":0,"ondisk_log_size":194,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:07:16.114968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00037995999999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1025,"num_read_kb":1201,"num_write":173,"num_write_kb":1276,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":561,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T20:26:59.152014+0000","last_change":"2023-09-23T20:25:21.159705+0000","last_active":"2023-09-23T20:26:59.152014+0000","last_peered":"2023-09-23T20:26:59.152014+0000","last_clean":"2023-09-23T20:26:59.152014+0000","last_became_active":"2023-09-23T20:25:21.159200+0000","last_became_peered":"2023-09-23T20:25:21.159200+0000","last_unstale":"2023-09-23T20:26:59.152014+0000","last_undegraded":"2023-09-23T20:26:59.152014+0000","last_fullsized":"2023-09-23T20:26:59.152014+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:09:47.213130+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":54,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T20:27:00.222043+0000","last_change":"2023-09-23T20:25:02.984205+0000","last_active":"2023-09-23T20:27:00.222043+0000","last_peered":"2023-09-23T20:27:00.222043+0000","last_clean":"2023-09-23T20:27:00.222043+0000","last_became_active":"2023-09-23T20:25:02.983758+0000","last_became_peered":"2023-09-23T20:25:02.983758+0000","last_unstale":"2023-09-23T20:27:00.222043+0000","last_undegraded":"2023-09-23T20:27:00.222043+0000","last_fullsized":"2023-09-23T20:27:00.222043+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:35:14.573101+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":150,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T20:26:59.854372+0000","last_change":"2023-09-23T20:12:45.609829+0000","last_active":"2023-09-23T20:26:59.854372+0000","last_peered":"2023-09-23T20:26:59.854372+0000","last_clean":"2023-09-23T20:26:59.854372+0000","last_became_active":"2023-09-23T20:12:45.609467+0000","last_became_peered":"2023-09-23T20:12:45.609467+0000","last_unstale":"2023-09-23T20:26:59.854372+0000","last_undegraded":"2023-09-23T20:26:59.854372+0000","last_fullsized":"2023-09-23T20:26:59.854372+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:27:31.490555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.4","version":"205'7737","reported_seq":9920,"reported_epoch":208,"state":"peering","last_fresh":"2023-09-23T20:27:00.156093+0000","last_change":"2023-09-23T20:27:00.157695+0000","last_active":"2023-09-23T20:27:00.156064+0000","last_peered":"2023-09-23T20:27:00.102537+0000","last_clean":"2023-09-23T20:24:52.062405+0000","last_became_active":"2023-09-23T20:25:39.693392+0000","last_became_peered":"2023-09-23T20:25:39.693392+0000","last_unstale":"2023-09-23T20:27:00.156093+0000","last_undegraded":"2023-09-23T20:27:00.156093+0000","last_fullsized":"2023-09-23T20:27:00.156093+0000","mapping_epoch":208,"log_start":"119'2046","ondisk_log_start":"119'2046","created":111,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"119'1973","last_scrub_stamp":"2023-09-23T20:23:13.408658+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:23:13.408658+0000","objects_scrubbed":1060,"log_size":5691,"log_dups_size":0,"ondisk_log_size":5691,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:28:24.714922+0000","scrub_duration":0.28811426499999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5768,"num_write_kb":150656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":993,"num_bytes_recovered":7798784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[0,2,5],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":560,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T20:26:59.152424+0000","last_change":"2023-09-23T20:25:21.172709+0000","last_active":"2023-09-23T20:26:59.152424+0000","last_peered":"2023-09-23T20:26:59.152424+0000","last_clean":"2023-09-23T20:26:59.152424+0000","last_became_active":"2023-09-23T20:25:21.171557+0000","last_became_peered":"2023-09-23T20:25:21.171557+0000","last_unstale":"2023-09-23T20:26:59.152424+0000","last_undegraded":"2023-09-23T20:26:59.152424+0000","last_fullsized":"2023-09-23T20:26:59.152424+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:09:47.213130+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.7","version":"205'7777","reported_seq":7957,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T20:26:59.146692+0000","last_change":"2023-09-23T20:23:16.284169+0000","last_active":"2023-09-23T20:26:59.146692+0000","last_peered":"2023-09-23T20:26:59.146692+0000","last_clean":"2023-09-23T20:26:59.146692+0000","last_became_active":"2023-09-23T20:23:11.069066+0000","last_became_peered":"2023-09-23T20:23:11.069066+0000","last_unstale":"2023-09-23T20:26:59.146692+0000","last_undegraded":"2023-09-23T20:26:59.146692+0000","last_fullsized":"2023-09-23T20:26:59.146692+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":111,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"120'2196","last_scrub_stamp":"2023-09-23T20:23:16.284112+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:23:16.284112+0000","objects_scrubbed":1268,"log_size":7777,"log_dups_size":0,"ondisk_log_size":7777,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:41:59.721972+0000","scrub_duration":0.33229684999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5590,"num_write_kb":138304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.0","version":"86'80","reported_seq":454,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T20:26:59.151905+0000","last_change":"2023-09-23T20:20:46.950784+0000","last_active":"2023-09-23T20:26:59.151905+0000","last_peered":"2023-09-23T20:26:59.151905+0000","last_clean":"2023-09-23T20:26:59.151905+0000","last_became_active":"2023-09-23T20:20:46.950071+0000","last_became_peered":"2023-09-23T20:20:46.950071+0000","last_unstale":"2023-09-23T20:26:59.151905+0000","last_undegraded":"2023-09-23T20:26:59.151905+0000","last_fullsized":"2023-09-23T20:26:59.151905+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":80,"log_dups_size":0,"ondisk_log_size":80,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:11:06.812404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035115,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":395,"num_read_kb":1370,"num_write":66,"num_write_kb":1061,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":169,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-09-23T20:26:58.145046+0000","last_change":"2023-09-23T20:25:02.942709+0000","last_active":"2023-09-23T20:26:58.145046+0000","last_peered":"2023-09-23T20:26:58.145046+0000","last_clean":"2023-09-23T20:26:58.145046+0000","last_became_active":"2023-09-23T20:25:02.942323+0000","last_became_peered":"2023-09-23T20:25:02.942323+0000","last_unstale":"2023-09-23T20:26:58.145046+0000","last_undegraded":"2023-09-23T20:26:58.145046+0000","last_fullsized":"2023-09-23T20:26:58.145046+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:26:12.039101+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":185,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-09-23T20:26:58.141263+0000","last_change":"2023-09-23T20:25:39.641452+0000","last_active":"2023-09-23T20:26:58.141263+0000","last_peered":"2023-09-23T20:26:58.141263+0000","last_clean":"2023-09-23T20:26:58.141263+0000","last_became_active":"2023-09-23T20:25:39.638987+0000","last_became_peered":"2023-09-23T20:25:39.638987+0000","last_unstale":"2023-09-23T20:26:58.141263+0000","last_undegraded":"2023-09-23T20:26:58.141263+0000","last_fullsized":"2023-09-23T20:26:58.141263+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:33:59.955461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":560,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T20:26:59.152946+0000","last_change":"2023-09-23T20:25:21.165430+0000","last_active":"2023-09-23T20:26:59.152946+0000","last_peered":"2023-09-23T20:26:59.152946+0000","last_clean":"2023-09-23T20:26:59.152946+0000","last_became_active":"2023-09-23T20:25:21.165109+0000","last_became_peered":"2023-09-23T20:25:21.165109+0000","last_unstale":"2023-09-23T20:26:59.152946+0000","last_undegraded":"2023-09-23T20:26:59.152946+0000","last_fullsized":"2023-09-23T20:26:59.152946+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:09:47.213130+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.5","version":"86'102","reported_seq":714,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T20:26:59.152947+0000","last_change":"2023-09-23T20:20:14.881975+0000","last_active":"2023-09-23T20:26:59.152947+0000","last_peered":"2023-09-23T20:26:59.152947+0000","last_clean":"2023-09-23T20:26:59.152947+0000","last_became_active":"2023-09-23T20:20:14.881544+0000","last_became_peered":"2023-09-23T20:20:14.881544+0000","last_unstale":"2023-09-23T20:26:59.152947+0000","last_undegraded":"2023-09-23T20:26:59.152947+0000","last_fullsized":"2023-09-23T20:26:59.152947+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":102,"log_dups_size":0,"ondisk_log_size":102,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:34.594972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030656500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":963,"num_read_kb":1234,"num_write":60,"num_write_kb":225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":165,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T20:26:59.143868+0000","last_change":"2023-09-23T20:25:39.631751+0000","last_active":"2023-09-23T20:26:59.143868+0000","last_peered":"2023-09-23T20:26:59.143868+0000","last_clean":"2023-09-23T20:26:59.143868+0000","last_became_active":"2023-09-23T20:25:39.631136+0000","last_became_peered":"2023-09-23T20:25:39.631136+0000","last_unstale":"2023-09-23T20:26:59.143868+0000","last_undegraded":"2023-09-23T20:26:59.143868+0000","last_fullsized":"2023-09-23T20:26:59.143868+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:08:17.020722+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":137,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-09-23T20:26:58.141494+0000","last_change":"2023-09-23T20:20:14.888874+0000","last_active":"2023-09-23T20:26:58.141494+0000","last_peered":"2023-09-23T20:26:58.141494+0000","last_clean":"2023-09-23T20:26:58.141494+0000","last_became_active":"2023-09-23T20:20:14.887724+0000","last_became_peered":"2023-09-23T20:20:14.887724+0000","last_unstale":"2023-09-23T20:26:58.141494+0000","last_undegraded":"2023-09-23T20:26:58.141494+0000","last_fullsized":"2023-09-23T20:26:58.141494+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:09:24.510326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.2","version":"205'7636","reported_seq":7802,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T20:26:59.854281+0000","last_change":"2023-09-23T20:23:12.771560+0000","last_active":"2023-09-23T20:26:59.854281+0000","last_peered":"2023-09-23T20:26:59.854281+0000","last_clean":"2023-09-23T20:26:59.854281+0000","last_became_active":"2023-09-23T20:23:11.058929+0000","last_became_peered":"2023-09-23T20:23:11.058929+0000","last_unstale":"2023-09-23T20:26:59.854281+0000","last_undegraded":"2023-09-23T20:26:59.854281+0000","last_fullsized":"2023-09-23T20:26:59.854281+0000","mapping_epoch":111,"log_start":"115'108","ondisk_log_start":"115'108","created":111,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":0,"last_scrub":"119'1924","last_scrub_stamp":"2023-09-23T20:23:12.771453+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:23:12.771453+0000","objects_scrubbed":1010,"log_size":7528,"log_dups_size":0,"ondisk_log_size":7528,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:49:31.924585+0000","scrub_duration":0.26141313799999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5720,"num_write_kb":150720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":560,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T20:26:59.152862+0000","last_change":"2023-09-23T20:25:21.200271+0000","last_active":"2023-09-23T20:26:59.152862+0000","last_peered":"2023-09-23T20:26:59.152862+0000","last_clean":"2023-09-23T20:26:59.152862+0000","last_became_active":"2023-09-23T20:25:21.197719+0000","last_became_peered":"2023-09-23T20:25:21.197719+0000","last_unstale":"2023-09-23T20:26:59.152862+0000","last_undegraded":"2023-09-23T20:26:59.152862+0000","last_fullsized":"2023-09-23T20:26:59.152862+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:09:47.213130+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":58,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T20:27:00.223153+0000","last_change":"2023-09-23T20:25:03.007736+0000","last_active":"2023-09-23T20:27:00.223153+0000","last_peered":"2023-09-23T20:27:00.223153+0000","last_clean":"2023-09-23T20:27:00.223153+0000","last_became_active":"2023-09-23T20:25:02.973689+0000","last_became_peered":"2023-09-23T20:25:02.973689+0000","last_unstale":"2023-09-23T20:27:00.223153+0000","last_undegraded":"2023-09-23T20:27:00.223153+0000","last_fullsized":"2023-09-23T20:27:00.223153+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:37:25.643774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"86'128","reported_seq":531,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-09-23T20:26:58.136631+0000","last_change":"2023-09-23T20:20:46.939256+0000","last_active":"2023-09-23T20:26:58.136631+0000","last_peered":"2023-09-23T20:26:58.136631+0000","last_clean":"2023-09-23T20:26:58.136631+0000","last_became_active":"2023-09-23T20:20:46.938756+0000","last_became_peered":"2023-09-23T20:20:46.938756+0000","last_unstale":"2023-09-23T20:26:58.136631+0000","last_undegraded":"2023-09-23T20:26:58.136631+0000","last_fullsized":"2023-09-23T20:26:58.136631+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:52:18.161436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000293202,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":579,"num_read_kb":1473,"num_write":96,"num_write_kb":1062,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":55,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-09-23T20:27:01.203349+0000","last_change":"2023-09-23T20:25:02.971740+0000","last_active":"2023-09-23T20:27:01.203349+0000","last_peered":"2023-09-23T20:27:01.203349+0000","last_clean":"2023-09-23T20:27:01.203349+0000","last_became_active":"2023-09-23T20:25:02.970412+0000","last_became_peered":"2023-09-23T20:25:02.970412+0000","last_unstale":"2023-09-23T20:27:01.203349+0000","last_undegraded":"2023-09-23T20:27:01.203349+0000","last_fullsized":"2023-09-23T20:27:01.203349+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:53:17.247216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.1","version":"205'7543","reported_seq":9122,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T20:26:59.154246+0000","last_change":"2023-09-23T20:25:41.656824+0000","last_active":"2023-09-23T20:26:59.154246+0000","last_peered":"2023-09-23T20:26:59.154246+0000","last_clean":"2023-09-23T20:26:59.154246+0000","last_became_active":"2023-09-23T20:25:41.621039+0000","last_became_peered":"2023-09-23T20:25:41.621039+0000","last_unstale":"2023-09-23T20:26:59.154246+0000","last_undegraded":"2023-09-23T20:26:59.154246+0000","last_fullsized":"2023-09-23T20:26:59.154246+0000","mapping_epoch":178,"log_start":"115'185","ondisk_log_start":"115'185","created":111,"last_epoch_clean":179,"parent":"0.0","parent_split_bits":0,"last_scrub":"119'1889","last_scrub_stamp":"2023-09-23T20:23:11.363674+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:23:11.363674+0000","objects_scrubbed":958,"log_size":7358,"log_dups_size":0,"ondisk_log_size":7358,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:06:09.763416+0000","scrub_duration":0.26718564099999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5662,"num_write_kb":150528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":692,"num_bytes_recovered":20447232,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"90'161","reported_seq":560,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T20:26:59.152292+0000","last_change":"2023-09-23T20:25:21.172983+0000","last_active":"2023-09-23T20:26:59.152292+0000","last_peered":"2023-09-23T20:26:59.152292+0000","last_clean":"2023-09-23T20:26:59.152292+0000","last_became_active":"2023-09-23T20:25:21.172575+0000","last_became_peered":"2023-09-23T20:25:21.172575+0000","last_unstale":"2023-09-23T20:26:59.152292+0000","last_undegraded":"2023-09-23T20:26:59.152292+0000","last_fullsized":"2023-09-23T20:26:59.152292+0000","mapping_epoch":172,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:09:47.213130+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.7","version":"86'117","reported_seq":727,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T20:26:59.854397+0000","last_change":"2023-09-23T20:25:40.617386+0000","last_active":"2023-09-23T20:26:59.854397+0000","last_peered":"2023-09-23T20:26:59.854397+0000","last_clean":"2023-09-23T20:26:59.854397+0000","last_became_active":"2023-09-23T20:25:40.616747+0000","last_became_peered":"2023-09-23T20:25:40.616747+0000","last_unstale":"2023-09-23T20:26:59.854397+0000","last_undegraded":"2023-09-23T20:26:59.854397+0000","last_fullsized":"2023-09-23T20:26:59.854397+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:46:44.551561+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046891400000000002,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":893,"num_read_kb":922,"num_write":82,"num_write_kb":4193,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":157,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-09-23T20:26:58.141192+0000","last_change":"2023-09-23T20:20:46.937163+0000","last_active":"2023-09-23T20:26:58.141192+0000","last_peered":"2023-09-23T20:26:58.141192+0000","last_clean":"2023-09-23T20:26:58.141192+0000","last_became_active":"2023-09-23T20:20:46.936846+0000","last_became_peered":"2023-09-23T20:20:46.936846+0000","last_unstale":"2023-09-23T20:26:58.141192+0000","last_undegraded":"2023-09-23T20:26:58.141192+0000","last_fullsized":"2023-09-23T20:26:58.141192+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:01:32.798855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":149,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T20:26:59.154408+0000","last_change":"2023-09-23T20:12:43.765908+0000","last_active":"2023-09-23T20:26:59.154408+0000","last_peered":"2023-09-23T20:26:59.154408+0000","last_clean":"2023-09-23T20:26:59.154408+0000","last_became_active":"2023-09-23T20:12:43.765558+0000","last_became_peered":"2023-09-23T20:12:43.765558+0000","last_unstale":"2023-09-23T20:26:59.154408+0000","last_undegraded":"2023-09-23T20:26:59.154408+0000","last_fullsized":"2023-09-23T20:26:59.154408+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:35:43.809898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.0","version":"205'7627","reported_seq":7800,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T20:27:00.221972+0000","last_change":"2023-09-23T20:25:40.633699+0000","last_active":"2023-09-23T20:27:00.221972+0000","last_peered":"2023-09-23T20:27:00.221972+0000","last_clean":"2023-09-23T20:27:00.221972+0000","last_became_active":"2023-09-23T20:25:40.618344+0000","last_became_peered":"2023-09-23T20:25:40.618344+0000","last_unstale":"2023-09-23T20:27:00.221972+0000","last_undegraded":"2023-09-23T20:27:00.221972+0000","last_fullsized":"2023-09-23T20:27:00.221972+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":111,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"119'2020","last_scrub_stamp":"2023-09-23T20:23:13.995392+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:23:13.995392+0000","objects_scrubbed":1088,"log_size":7627,"log_dups_size":0,"ondisk_log_size":7627,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:11:19.162515+0000","scrub_duration":0.22507750400000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5616,"num_write_kb":144896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":355,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":139,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-09-23T20:26:58.141406+0000","last_change":"2023-09-23T20:20:14.888484+0000","last_active":"2023-09-23T20:26:58.141406+0000","last_peered":"2023-09-23T20:26:58.141406+0000","last_clean":"2023-09-23T20:26:58.141406+0000","last_became_active":"2023-09-23T20:20:14.887219+0000","last_became_peered":"2023-09-23T20:20:14.887219+0000","last_unstale":"2023-09-23T20:26:58.141406+0000","last_undegraded":"2023-09-23T20:26:58.141406+0000","last_fullsized":"2023-09-23T20:26:58.141406+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:13:53.763649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":43,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T20:27:00.143981+0000","last_change":"2023-09-23T20:25:39.639268+0000","last_active":"2023-09-23T20:27:00.143981+0000","last_peered":"2023-09-23T20:27:00.143981+0000","last_clean":"2023-09-23T20:27:00.143981+0000","last_became_active":"2023-09-23T20:25:39.638922+0000","last_became_peered":"2023-09-23T20:25:39.638922+0000","last_unstale":"2023-09-23T20:27:00.143981+0000","last_undegraded":"2023-09-23T20:27:00.143981+0000","last_fullsized":"2023-09-23T20:27:00.143981+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:09:04.040098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":169,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T20:26:59.146419+0000","last_change":"2023-09-23T20:20:46.938981+0000","last_active":"2023-09-23T20:26:59.146419+0000","last_peered":"2023-09-23T20:26:59.146419+0000","last_clean":"2023-09-23T20:26:59.146419+0000","last_became_active":"2023-09-23T20:20:46.938627+0000","last_became_peered":"2023-09-23T20:20:46.938627+0000","last_unstale":"2023-09-23T20:26:59.146419+0000","last_undegraded":"2023-09-23T20:26:59.146419+0000","last_fullsized":"2023-09-23T20:26:59.146419+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:44:43.177086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":213,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T20:26:59.153517+0000","last_change":"2023-09-23T20:25:02.945142+0000","last_active":"2023-09-23T20:26:59.153517+0000","last_peered":"2023-09-23T20:26:59.153517+0000","last_clean":"2023-09-23T20:26:59.153517+0000","last_became_active":"2023-09-23T20:25:02.944653+0000","last_became_peered":"2023-09-23T20:25:02.944653+0000","last_unstale":"2023-09-23T20:26:59.153517+0000","last_undegraded":"2023-09-23T20:26:59.153517+0000","last_fullsized":"2023-09-23T20:26:59.153517+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:38:12.927982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":152,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T20:26:59.146629+0000","last_change":"2023-09-23T20:12:43.753383+0000","last_active":"2023-09-23T20:26:59.146629+0000","last_peered":"2023-09-23T20:26:59.146629+0000","last_clean":"2023-09-23T20:26:59.146629+0000","last_became_active":"2023-09-23T20:12:43.752921+0000","last_became_peered":"2023-09-23T20:12:43.752921+0000","last_unstale":"2023-09-23T20:26:59.146629+0000","last_undegraded":"2023-09-23T20:26:59.146629+0000","last_fullsized":"2023-09-23T20:26:59.146629+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:28:26.982998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":213,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T20:26:59.153774+0000","last_change":"2023-09-23T20:25:03.103656+0000","last_active":"2023-09-23T20:26:59.153774+0000","last_peered":"2023-09-23T20:26:59.153774+0000","last_clean":"2023-09-23T20:26:59.153774+0000","last_became_active":"2023-09-23T20:25:03.103311+0000","last_became_peered":"2023-09-23T20:25:03.103311+0000","last_unstale":"2023-09-23T20:26:59.153774+0000","last_undegraded":"2023-09-23T20:26:59.153774+0000","last_fullsized":"2023-09-23T20:26:59.153774+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:55:54.677030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.18","version":"205'7586","reported_seq":15041,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T20:26:59.151937+0000","last_change":"2023-09-23T20:25:15.101300+0000","last_active":"2023-09-23T20:26:59.151937+0000","last_peered":"2023-09-23T20:26:59.151937+0000","last_clean":"2023-09-23T20:26:59.151937+0000","last_became_active":"2023-09-23T20:25:15.063524+0000","last_became_peered":"2023-09-23T20:25:15.063524+0000","last_unstale":"2023-09-23T20:26:59.151937+0000","last_undegraded":"2023-09-23T20:26:59.151937+0000","last_fullsized":"2023-09-23T20:26:59.151937+0000","mapping_epoch":169,"log_start":"115'348","ondisk_log_start":"115'348","created":118,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":5,"last_scrub":"120'2147","last_scrub_stamp":"2023-09-23T20:23:18.111873+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:23:18.111873+0000","objects_scrubbed":1267,"log_size":7238,"log_dups_size":0,"ondisk_log_size":7238,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:27:49.823665+0000","scrub_duration":0.17815524799999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5439,"num_write_kb":133504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2426,"num_bytes_recovered":140312576,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":64,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-09-23T20:27:01.203609+0000","last_change":"2023-09-23T20:25:39.629869+0000","last_active":"2023-09-23T20:27:01.203609+0000","last_peered":"2023-09-23T20:27:01.203609+0000","last_clean":"2023-09-23T20:27:01.203609+0000","last_became_active":"2023-09-23T20:25:39.629543+0000","last_became_peered":"2023-09-23T20:25:39.629543+0000","last_unstale":"2023-09-23T20:27:01.203609+0000","last_undegraded":"2023-09-23T20:27:01.203609+0000","last_fullsized":"2023-09-23T20:27:01.203609+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:24:48.220679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":63,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T20:27:00.221962+0000","last_change":"2023-09-23T20:25:39.622819+0000","last_active":"2023-09-23T20:27:00.221962+0000","last_peered":"2023-09-23T20:27:00.221962+0000","last_clean":"2023-09-23T20:27:00.221962+0000","last_became_active":"2023-09-23T20:25:39.622484+0000","last_became_peered":"2023-09-23T20:25:39.622484+0000","last_unstale":"2023-09-23T20:27:00.221962+0000","last_undegraded":"2023-09-23T20:27:00.221962+0000","last_fullsized":"2023-09-23T20:27:00.221962+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:51:53.973261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.19","version":"205'7599","reported_seq":7714,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T20:26:59.152565+0000","last_change":"2023-09-23T20:23:10.978296+0000","last_active":"2023-09-23T20:26:59.152565+0000","last_peered":"2023-09-23T20:26:59.152565+0000","last_clean":"2023-09-23T20:26:59.152565+0000","last_became_active":"2023-09-23T20:23:10.977578+0000","last_became_peered":"2023-09-23T20:23:10.977578+0000","last_unstale":"2023-09-23T20:26:59.152565+0000","last_undegraded":"2023-09-23T20:26:59.152565+0000","last_fullsized":"2023-09-23T20:26:59.152565+0000","mapping_epoch":118,"log_start":"119'1849","ondisk_log_start":"119'1849","created":118,"last_epoch_clean":119,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:22:51.846999+0000","last_clean_scrub_stamp":"2023-09-23T20:22:51.846999+0000","objects_scrubbed":0,"log_size":5750,"log_dups_size":0,"ondisk_log_size":5750,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6717,"num_write_kb":214304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":185,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-09-23T20:26:58.142642+0000","last_change":"2023-09-23T20:25:39.668882+0000","last_active":"2023-09-23T20:26:58.142642+0000","last_peered":"2023-09-23T20:26:58.142642+0000","last_clean":"2023-09-23T20:26:58.142642+0000","last_became_active":"2023-09-23T20:25:39.637128+0000","last_became_peered":"2023-09-23T20:25:39.637128+0000","last_unstale":"2023-09-23T20:26:58.142642+0000","last_undegraded":"2023-09-23T20:26:58.142642+0000","last_fullsized":"2023-09-23T20:26:58.142642+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:56.406298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":186,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T20:26:59.154740+0000","last_change":"2023-09-23T20:24:53.089249+0000","last_active":"2023-09-23T20:26:59.154740+0000","last_peered":"2023-09-23T20:26:59.154740+0000","last_clean":"2023-09-23T20:26:59.154740+0000","last_became_active":"2023-09-23T20:24:53.088952+0000","last_became_peered":"2023-09-23T20:24:53.088952+0000","last_unstale":"2023-09-23T20:26:59.154740+0000","last_undegraded":"2023-09-23T20:26:59.154740+0000","last_fullsized":"2023-09-23T20:26:59.154740+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:38:03.216069+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":5,"num_pg":26,"stat_sum":{"num_bytes":4784128,"num_objects":72,"num_object_clones":0,"num_object_copies":216,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2003,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":72,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":191890,"num_write_kb":5640001,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":25411,"num_bytes_recovered":1041498112,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":548929536,"data_stored":548929536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":178472,"ondisk_log_size":178472,"up":78,"acting":76,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":15,"num_bytes_recovered":1356,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":25181,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1771,"ondisk_log_size":1771,"up":33,"acting":33,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6082,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201470,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":186288,"kb_used_data":75624,"kb_used_omap":0,"kb_used_meta":110656,"kb_avail":93558864,"statfs":{"total":95995035648,"available":95804276736,"internally_reserved":0,"allocated":77438976,"data_stored":76727000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":113311744},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:26:23 2023","interfaces":[{"interface":"back","average":{"1min":1.861,"5min":1.2150000000000001,"15min":0.78700000000000003},"min":{"1min":0.247,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":10.907,"5min":13.092000000000001,"15min":13.092000000000001},"last":0.29399999999999998},{"interface":"front","average":{"1min":3.5369999999999999,"5min":1.5940000000000001,"15min":0.91000000000000003},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":9.8610000000000007,"5min":9.8610000000000007,"15min":9.8610000000000007},"last":0.35299999999999998}]},{"osd":1,"last update":"Sat Sep 23 20:26:58 2023","interfaces":[{"interface":"back","average":{"1min":1.2929999999999999,"5min":1.6499999999999999,"15min":0.92500000000000004},"min":{"1min":0.28599999999999998,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":3.1920000000000002,"5min":17.795999999999999,"15min":17.795999999999999},"last":0.45900000000000002},{"interface":"front","average":{"1min":2.2989999999999999,"5min":1.504,"15min":0.90900000000000003},"min":{"1min":0.33400000000000002,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":6.4950000000000001,"5min":13.250999999999999,"15min":13.250999999999999},"last":0.33400000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:26:28 2023","interfaces":[{"interface":"back","average":{"1min":3.2610000000000001,"5min":1.6080000000000001,"15min":0.91500000000000004},"min":{"1min":0.39000000000000001,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":8.1270000000000007,"5min":9.8300000000000001,"15min":9.8300000000000001},"last":0.318},{"interface":"front","average":{"1min":3.1850000000000001,"5min":1.708,"15min":0.95999999999999996},"min":{"1min":0.48099999999999998,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":6.6699999999999999,"5min":11.423999999999999,"15min":11.423999999999999},"last":0.502}]},{"osd":3,"last update":"Sat Sep 23 20:26:58 2023","interfaces":[{"interface":"back","average":{"1min":2.4049999999999998,"5min":1.756,"15min":1.022},"min":{"1min":0.38100000000000001,"5min":0.314,"15min":0.314},"max":{"1min":6.891,"5min":6.891,"15min":6.891},"last":0.55900000000000005},{"interface":"front","average":{"1min":2.4319999999999999,"5min":1.6279999999999999,"15min":0.99199999999999999},"min":{"1min":0.374,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":6.9050000000000002,"5min":9.7840000000000007,"15min":9.7840000000000007},"last":0.54600000000000004}]},{"osd":4,"last update":"Sat Sep 23 20:26:58 2023","interfaces":[{"interface":"back","average":{"1min":1.6479999999999999,"5min":1.1619999999999999,"15min":0.71899999999999997},"min":{"1min":0.28499999999999998,"5min":0.27000000000000002,"15min":0.26700000000000002},"max":{"1min":5.3540000000000001,"5min":13.141999999999999,"15min":13.141999999999999},"last":0.373},{"interface":"front","average":{"1min":1.5740000000000001,"5min":1.6819999999999999,"15min":0.91400000000000003},"min":{"1min":0.29499999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":5.9279999999999999,"5min":10.853,"15min":10.853},"last":0.39100000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:26:58 2023","interfaces":[{"interface":"back","average":{"1min":1.9930000000000001,"5min":1.643,"15min":0.90500000000000003},"min":{"1min":0.31,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":10.362,"5min":10.563000000000001,"15min":10.563000000000001},"last":0.43099999999999999},{"interface":"front","average":{"1min":1.956,"5min":1.5669999999999999,"15min":0.91100000000000003},"min":{"1min":0.41799999999999998,"5min":0.34699999999999998,"15min":0.31},"max":{"1min":7.9029999999999996,"5min":13.292,"15min":13.292},"last":0.41799999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:26:58 2023","interfaces":[{"interface":"back","average":{"1min":1.9890000000000001,"5min":1.4179999999999999,"15min":0.86899999999999999},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":6.2210000000000001,"5min":13.185,"15min":13.185},"last":0.47299999999999998},{"interface":"front","average":{"1min":1.9199999999999999,"5min":1.4890000000000001,"15min":0.89100000000000001},"min":{"1min":0.378,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":6.4370000000000003,"5min":9.9120000000000008,"15min":9.9120000000000008},"last":0.48299999999999998}]}]},{"osd":6,"up_from":47,"seq":201863463106,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":85528,"kb_used_data":6736,"kb_used_omap":0,"kb_used_meta":78784,"kb_avail":93659624,"statfs":{"total":95995035648,"available":95907454976,"internally_reserved":0,"allocated":6897664,"data_stored":6193904,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":80674816},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:26:26 2023","interfaces":[{"interface":"back","average":{"1min":1.7270000000000001,"5min":1.23,"15min":0.79500000000000004},"min":{"1min":0.221,"5min":0.221,"15min":0.221},"max":{"1min":8.1940000000000008,"5min":8.1940000000000008,"15min":8.1940000000000008},"last":0.42799999999999999},{"interface":"front","average":{"1min":1.3069999999999999,"5min":1.4850000000000001,"15min":0.87},"min":{"1min":0.26300000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":4.0609999999999999,"5min":12.512,"15min":12.512},"last":0.46500000000000002}]},{"osd":1,"last update":"Sat Sep 23 20:25:57 2023","interfaces":[{"interface":"back","average":{"1min":2.6760000000000002,"5min":1.4710000000000001,"15min":0.873},"min":{"1min":0.30099999999999999,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":7.8929999999999998,"5min":18.001000000000001,"15min":18.001000000000001},"last":0.505},{"interface":"front","average":{"1min":2.7040000000000002,"5min":1.2,"15min":0.76500000000000001},"min":{"1min":0.28699999999999998,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":13.91,"5min":13.91,"15min":13.91},"last":0.34200000000000003}]},{"osd":2,"last update":"Sat Sep 23 20:26:26 2023","interfaces":[{"interface":"back","average":{"1min":2.153,"5min":1.385,"15min":0.85799999999999998},"min":{"1min":0.32800000000000001,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":8.0090000000000003,"5min":13.856,"15min":13.856},"last":0.53900000000000003},{"interface":"front","average":{"1min":2.0720000000000001,"5min":1.284,"15min":0.83399999999999996},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":6.8140000000000001,"5min":6.8140000000000001,"15min":6.8140000000000001},"last":0.52300000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:25:57 2023","interfaces":[{"interface":"back","average":{"1min":3.1840000000000002,"5min":1.5349999999999999,"15min":0.92000000000000004},"min":{"1min":0.39200000000000002,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":9.4239999999999995,"5min":9.4239999999999995,"15min":9.4239999999999995},"last":0.56999999999999995},{"interface":"front","average":{"1min":2.6309999999999998,"5min":1.413,"15min":0.92200000000000004},"min":{"1min":0.42799999999999999,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":7.8700000000000001,"5min":8.6050000000000004,"15min":8.6050000000000004},"last":5.173}]},{"osd":4,"last update":"Sat Sep 23 20:26:48 2023","interfaces":[{"interface":"back","average":{"1min":1.5049999999999999,"5min":1.2669999999999999,"15min":0.72999999999999998},"min":{"1min":0.27600000000000002,"5min":0.27200000000000002,"15min":0.26200000000000001},"max":{"1min":10.173,"5min":10.173,"15min":10.173},"last":0.38600000000000001},{"interface":"front","average":{"1min":1.962,"5min":1.3540000000000001,"15min":0.78000000000000003},"min":{"1min":0.312,"5min":0.29799999999999999,"15min":0.28499999999999998},"max":{"1min":8.3249999999999993,"5min":8.3249999999999993,"15min":8.3249999999999993},"last":0.55300000000000005}]},{"osd":5,"last update":"Sat Sep 23 20:26:48 2023","interfaces":[{"interface":"back","average":{"1min":1.202,"5min":1.4119999999999999,"15min":0.80300000000000005},"min":{"1min":0.31,"5min":0.31,"15min":0.29799999999999999},"max":{"1min":4.4580000000000002,"5min":14.085000000000001,"15min":14.085000000000001},"last":0.35599999999999998},{"interface":"front","average":{"1min":2.831,"5min":1.8660000000000001,"15min":0.97299999999999998},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.29499999999999998},"max":{"1min":18.192,"5min":18.192,"15min":18.192},"last":0.53000000000000003}]},{"osd":7,"last update":"Sat Sep 23 20:25:57 2023","interfaces":[{"interface":"back","average":{"1min":3.02,"5min":1.262,"15min":0.78300000000000003},"min":{"1min":0.42299999999999999,"5min":0.33500000000000002,"15min":0.32800000000000001},"max":{"1min":10.217000000000001,"5min":10.217000000000001,"15min":10.217000000000001},"last":0.40799999999999997},{"interface":"front","average":{"1min":3.2749999999999999,"5min":1.393,"15min":0.84199999999999997},"min":{"1min":0.36599999999999999,"5min":0.34999999999999998,"15min":0.315},"max":{"1min":9.7360000000000007,"5min":9.7360000000000007,"15min":9.7360000000000007},"last":0.66500000000000004}]}]},{"osd":1,"up_from":99,"seq":425201762381,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":154756,"kb_used_data":2492,"kb_used_omap":6,"kb_used_meta":152249,"kb_avail":93590396,"statfs":{"total":95995035648,"available":95836565504,"internally_reserved":0,"allocated":2551808,"data_stored":1462924,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6696,"internal_metadata":155903448},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1,3],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:26:54 2023","interfaces":[{"interface":"back","average":{"1min":2.6600000000000001,"5min":2.052,"15min":0.93500000000000005},"min":{"1min":0.215,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":14.384,"5min":40.113,"15min":40.113},"last":2.694},{"interface":"front","average":{"1min":3.1869999999999998,"5min":1.8120000000000001,"15min":0.87},"min":{"1min":0.23699999999999999,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":22.210000000000001,"5min":42.994999999999997,"15min":42.994999999999997},"last":3.6190000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:26:27 2023","interfaces":[{"interface":"back","average":{"1min":1.6479999999999999,"5min":1.8029999999999999,"15min":0.85499999999999998},"min":{"1min":0.27900000000000003,"5min":0.253,"15min":0.253},"max":{"1min":9.6839999999999993,"5min":39.962000000000003,"15min":39.962000000000003},"last":0.433},{"interface":"front","average":{"1min":2.806,"5min":1.794,"15min":0.96599999999999997},"min":{"1min":0.26700000000000002,"5min":0.246,"15min":0.246},"max":{"1min":8.7910000000000004,"5min":47.442,"15min":47.442},"last":1.7849999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:25:58 2023","interfaces":[{"interface":"back","average":{"1min":1.3029999999999999,"5min":1.1319999999999999,"15min":0.74399999999999999},"min":{"1min":0.28499999999999998,"5min":0.246,"15min":0.246},"max":{"1min":4.6130000000000004,"5min":8.7420000000000009,"15min":8.7420000000000009},"last":0.48699999999999999},{"interface":"front","average":{"1min":2.1619999999999999,"5min":1.0660000000000001,"15min":0.627},"min":{"1min":0.26000000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":8.7400000000000002,"5min":9.1859999999999999,"15min":9.1859999999999999},"last":3.4889999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:26:54 2023","interfaces":[{"interface":"back","average":{"1min":2.5790000000000002,"5min":1.5900000000000001,"15min":0.94099999999999995},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":15.353,"5min":15.353,"15min":15.353},"last":0.53500000000000003},{"interface":"front","average":{"1min":1.7609999999999999,"5min":1.536,"15min":0.93500000000000005},"min":{"1min":0.32400000000000001,"5min":0.312,"15min":0.312},"max":{"1min":6.2270000000000003,"5min":8.7899999999999991,"15min":8.7899999999999991},"last":0.51200000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:26:54 2023","interfaces":[{"interface":"back","average":{"1min":2.6360000000000001,"5min":1.6220000000000001,"15min":0.96799999999999997},"min":{"1min":0.33000000000000002,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":7.2000000000000002,"5min":12.590999999999999,"15min":12.590999999999999},"last":0.66600000000000004},{"interface":"front","average":{"1min":2.2530000000000001,"5min":2.137,"15min":1.163},"min":{"1min":0.38300000000000001,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":6.3140000000000001,"5min":39.579000000000001,"15min":39.579000000000001},"last":4.4960000000000004}]},{"osd":6,"last update":"Sat Sep 23 20:26:54 2023","interfaces":[{"interface":"back","average":{"1min":2.9630000000000001,"5min":1.996,"15min":1.1100000000000001},"min":{"1min":0.42999999999999999,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":7.226,"5min":34.409999999999997,"15min":34.409999999999997},"last":0.91400000000000003},{"interface":"front","average":{"1min":2.964,"5min":2.2269999999999999,"15min":1.167},"min":{"1min":0.38100000000000001,"5min":0.377,"15min":0.377},"max":{"1min":15.313000000000001,"5min":35.554000000000002,"15min":35.554000000000002},"last":4.5300000000000002}]},{"osd":7,"last update":"Sat Sep 23 20:26:54 2023","interfaces":[{"interface":"back","average":{"1min":2.8719999999999999,"5min":1.8480000000000001,"15min":1.05},"min":{"1min":0.42499999999999999,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":9.7439999999999998,"5min":9.7439999999999998,"15min":9.7439999999999998},"last":4.6139999999999999},{"interface":"front","average":{"1min":2.3180000000000001,"5min":1.7729999999999999,"15min":1.0469999999999999},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":6.2519999999999998,"5min":9.2279999999999998,"15min":9.2279999999999998},"last":0.69899999999999995}]}]},{"osd":0,"up_from":94,"seq":403726925907,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":249776,"kb_used_data":83560,"kb_used_omap":7,"kb_used_meta":166200,"kb_avail":93495376,"statfs":{"total":95995035648,"available":95739265024,"internally_reserved":0,"allocated":85565440,"data_stored":84557124,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":170188817},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:26:56 2023","interfaces":[{"interface":"back","average":{"1min":3.0779999999999998,"5min":1.3400000000000001,"15min":0.68600000000000005},"min":{"1min":0.223,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":24.247,"5min":24.247,"15min":24.247},"last":0.26300000000000001},{"interface":"front","average":{"1min":2.9950000000000001,"5min":1.3160000000000001,"15min":0.69599999999999995},"min":{"1min":0.28999999999999998,"5min":0.222,"15min":0.222},"max":{"1min":15.994999999999999,"5min":15.994999999999999,"15min":15.994999999999999},"last":0.29399999999999998}]},{"osd":2,"last update":"Sat Sep 23 20:26:22 2023","interfaces":[{"interface":"back","average":{"1min":4.3319999999999999,"5min":1.8200000000000001,"15min":1.004},"min":{"1min":0.251,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":23.832999999999998,"5min":23.832999999999998,"15min":23.832999999999998},"last":0.23699999999999999},{"interface":"front","average":{"1min":1.3779999999999999,"5min":1.256,"15min":0.749},"min":{"1min":0.223,"5min":0.223,"15min":0.223},"max":{"1min":6.0629999999999997,"5min":9.1940000000000008,"15min":9.1940000000000008},"last":0.38700000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:26:56 2023","interfaces":[{"interface":"back","average":{"1min":3.3109999999999999,"5min":1.7070000000000001,"15min":0.96699999999999997},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":23.748999999999999,"5min":23.748999999999999,"15min":23.748999999999999},"last":0.315},{"interface":"front","average":{"1min":3.9809999999999999,"5min":1.5189999999999999,"15min":0.78700000000000003},"min":{"1min":0.315,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":24.274999999999999,"5min":24.274999999999999,"15min":24.274999999999999},"last":0.40300000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:26:22 2023","interfaces":[{"interface":"back","average":{"1min":4.6619999999999999,"5min":1.9430000000000001,"15min":1.046},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":23.876999999999999,"5min":23.876999999999999,"15min":23.876999999999999},"last":0.46999999999999997},{"interface":"front","average":{"1min":2.8839999999999999,"5min":1.587,"15min":0.93799999999999994},"min":{"1min":0.34799999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":23.681999999999999,"5min":23.681999999999999,"15min":23.681999999999999},"last":0.52200000000000002}]},{"osd":5,"last update":"Sat Sep 23 20:26:22 2023","interfaces":[{"interface":"back","average":{"1min":4.3259999999999996,"5min":2.2290000000000001,"15min":1.1819999999999999},"min":{"1min":0.39100000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":24.198,"5min":24.198,"15min":24.198},"last":0.66400000000000003},{"interface":"front","average":{"1min":2.0169999999999999,"5min":1.754,"15min":1.0369999999999999},"min":{"1min":0.37,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":6.9269999999999996,"5min":11.167999999999999,"15min":11.167999999999999},"last":0.68100000000000005}]},{"osd":6,"last update":"Sat Sep 23 20:26:22 2023","interfaces":[{"interface":"back","average":{"1min":3.694,"5min":1.5760000000000001,"15min":0.95799999999999996},"min":{"1min":0.45400000000000001,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":24.408000000000001,"5min":24.408000000000001,"15min":24.408000000000001},"last":0.63200000000000001},{"interface":"front","average":{"1min":3.5230000000000001,"5min":1.526,"15min":0.94999999999999996},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":25.271000000000001,"5min":25.271000000000001,"15min":25.271000000000001},"last":0.56899999999999995}]},{"osd":7,"last update":"Sat Sep 23 20:26:22 2023","interfaces":[{"interface":"back","average":{"1min":3.3199999999999998,"5min":1.7190000000000001,"15min":1.0249999999999999},"min":{"1min":0.442,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":25.359000000000002,"5min":25.359000000000002,"15min":25.359000000000002},"last":0.72999999999999998},{"interface":"front","average":{"1min":5.298,"5min":2.181,"15min":1.1699999999999999},"min":{"1min":0.499,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":23.905999999999999,"5min":23.905999999999999,"15min":23.905999999999999},"last":0.55200000000000005}]}]},{"osd":2,"up_from":104,"seq":446676598855,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":211820,"kb_used_data":89956,"kb_used_omap":11,"kb_used_meta":121844,"kb_avail":93533332,"statfs":{"total":95995035648,"available":95778131968,"internally_reserved":0,"allocated":92114944,"data_stored":91121230,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11878,"internal_metadata":124768666},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:26:30 2023","interfaces":[{"interface":"back","average":{"1min":2.8090000000000002,"5min":1.706,"15min":0.81999999999999995},"min":{"1min":0.21099999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":9.5980000000000008,"5min":12.811,"15min":12.811},"last":0.29399999999999998},{"interface":"front","average":{"1min":2.1019999999999999,"5min":1.2709999999999999,"15min":0.68700000000000006},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":15.746,"5min":15.746,"15min":15.746},"last":0.318}]},{"osd":1,"last update":"Sat Sep 23 20:26:30 2023","interfaces":[{"interface":"back","average":{"1min":2.9390000000000001,"5min":1.524,"15min":0.77000000000000002},"min":{"1min":0.28599999999999998,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":14.135999999999999,"5min":14.135999999999999,"15min":14.135999999999999},"last":0.33900000000000002},{"interface":"front","average":{"1min":2.617,"5min":1.226,"15min":0.66400000000000003},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":13.718,"5min":13.718,"15min":13.718},"last":0.27500000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:26:55 2023","interfaces":[{"interface":"back","average":{"1min":2.246,"5min":1.349,"15min":0.69099999999999995},"min":{"1min":0.26500000000000001,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":10.916,"5min":15.319000000000001,"15min":15.319000000000001},"last":0.58599999999999997},{"interface":"front","average":{"1min":2.294,"5min":1.552,"15min":0.78200000000000003},"min":{"1min":0.25800000000000001,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":12.494999999999999,"5min":12.494999999999999,"15min":12.494999999999999},"last":0.54300000000000004}]},{"osd":4,"last update":"Sat Sep 23 20:26:30 2023","interfaces":[{"interface":"back","average":{"1min":1.996,"5min":1.234,"15min":0.80000000000000004},"min":{"1min":0.36299999999999999,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":8.2300000000000004,"5min":8.2300000000000004,"15min":8.2300000000000004},"last":0.65800000000000003},{"interface":"front","average":{"1min":2.3679999999999999,"5min":1.3979999999999999,"15min":0.85599999999999998},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":8.6530000000000005,"5min":8.6530000000000005,"15min":8.6530000000000005},"last":0.72499999999999998}]},{"osd":5,"last update":"Sat Sep 23 20:26:30 2023","interfaces":[{"interface":"back","average":{"1min":2.8319999999999999,"5min":1.577,"15min":0.96299999999999997},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":11.237,"5min":11.237,"15min":11.237},"last":0.70299999999999996},{"interface":"front","average":{"1min":3.452,"5min":1.6850000000000001,"15min":0.999},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":11.266999999999999,"5min":11.266999999999999,"15min":11.266999999999999},"last":0.67900000000000005}]},{"osd":6,"last update":"Sat Sep 23 20:26:30 2023","interfaces":[{"interface":"back","average":{"1min":2.0129999999999999,"5min":1.294,"15min":0.86899999999999999},"min":{"1min":0.49099999999999999,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":6.2599999999999998,"5min":6.7939999999999996,"15min":6.7939999999999996},"last":0.76100000000000001},{"interface":"front","average":{"1min":2.7410000000000001,"5min":1.585,"15min":0.97699999999999998},"min":{"1min":0.41099999999999998,"5min":0.377,"15min":0.377},"max":{"1min":9.6600000000000001,"5min":9.6600000000000001,"15min":9.6600000000000001},"last":0.68999999999999995}]},{"osd":7,"last update":"Sat Sep 23 20:26:30 2023","interfaces":[{"interface":"back","average":{"1min":3.1150000000000002,"5min":1.712,"15min":1.002},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":15.343999999999999,"5min":15.343999999999999,"15min":15.343999999999999},"last":0.64100000000000001},{"interface":"front","average":{"1min":2.6299999999999999,"5min":1.5840000000000001,"15min":0.96599999999999997},"min":{"1min":0.48899999999999999,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":9.1440000000000001,"5min":10.064,"15min":10.064},"last":0.59699999999999998}]}]},{"osd":3,"up_from":109,"seq":468151435329,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":145652,"kb_used_data":19692,"kb_used_omap":4,"kb_used_meta":125947,"kb_avail":93599500,"statfs":{"total":95995035648,"available":95845888000,"internally_reserved":0,"allocated":20164608,"data_stored":19162450,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4514,"internal_metadata":128970334},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:26:56 2023","interfaces":[{"interface":"back","average":{"1min":3.2530000000000001,"5min":2.319,"15min":1.0069999999999999},"min":{"1min":0.28999999999999998,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":15.589,"5min":67.305999999999997,"15min":67.305999999999997},"last":0.33900000000000002},{"interface":"front","average":{"1min":1.48,"5min":2.0339999999999998,"15min":0.93400000000000005},"min":{"1min":0.29699999999999999,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":9.0920000000000005,"5min":67.403999999999996,"15min":67.403999999999996},"last":0.29399999999999998}]},{"osd":1,"last update":"Sat Sep 23 20:26:56 2023","interfaces":[{"interface":"back","average":{"1min":3.0230000000000001,"5min":1.5069999999999999,"15min":0.76700000000000002},"min":{"1min":0.25,"5min":0.214,"15min":0.214},"max":{"1min":19.675999999999998,"5min":19.675999999999998,"15min":19.675999999999998},"last":0.42599999999999999},{"interface":"front","average":{"1min":2.4590000000000001,"5min":1.375,"15min":0.755},"min":{"1min":0.33200000000000002,"5min":0.245,"15min":0.245},"max":{"1min":11.644,"5min":11.644,"15min":11.644},"last":0.45100000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:26:56 2023","interfaces":[{"interface":"back","average":{"1min":3.4369999999999998,"5min":1.7789999999999999,"15min":0.82799999999999996},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":24.045000000000002,"5min":24.045000000000002,"15min":24.045000000000002},"last":0.25600000000000001},{"interface":"front","average":{"1min":3.4119999999999999,"5min":1.8080000000000001,"15min":0.91200000000000003},"min":{"1min":0.33700000000000002,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":12.427,"5min":17.684000000000001,"15min":17.684000000000001},"last":0.38600000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:26:56 2023","interfaces":[{"interface":"back","average":{"1min":2.875,"5min":1.4590000000000001,"15min":0.88900000000000001},"min":{"1min":0.33100000000000002,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":11.880000000000001,"5min":11.880000000000001,"15min":11.880000000000001},"last":0.70399999999999996},{"interface":"front","average":{"1min":3.698,"5min":1.8180000000000001,"15min":1.016},"min":{"1min":0.378,"5min":0.371,"15min":0.371},"max":{"1min":21.673999999999999,"5min":21.673999999999999,"15min":21.673999999999999},"last":0.61699999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:26:56 2023","interfaces":[{"interface":"back","average":{"1min":3.117,"5min":1.8560000000000001,"15min":1.0229999999999999},"min":{"1min":0.375,"5min":0.311,"15min":0.311},"max":{"1min":11.839,"5min":11.839,"15min":11.839},"last":0.79900000000000004},{"interface":"front","average":{"1min":1.825,"5min":1.2949999999999999,"15min":0.85199999999999998},"min":{"1min":0.40000000000000002,"5min":0.313,"15min":0.313},"max":{"1min":8.5570000000000004,"5min":8.5570000000000004,"15min":8.5570000000000004},"last":0.48399999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:26:56 2023","interfaces":[{"interface":"back","average":{"1min":2.0609999999999999,"5min":1.5900000000000001,"15min":0.95499999999999996},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":8.6059999999999999,"5min":8.6059999999999999,"15min":8.6059999999999999},"last":0.54400000000000004},{"interface":"front","average":{"1min":2.7469999999999999,"5min":1.5740000000000001,"15min":0.97199999999999998},"min":{"1min":0.40600000000000003,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":9.3179999999999996,"5min":9.3179999999999996,"15min":9.3179999999999996},"last":0.59299999999999997}]},{"osd":7,"last update":"Sat Sep 23 20:26:56 2023","interfaces":[{"interface":"back","average":{"1min":3.0030000000000001,"5min":1.5860000000000001,"15min":0.997},"min":{"1min":0.40999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":9.2750000000000004,"5min":9.2750000000000004,"15min":9.2750000000000004},"last":0.63600000000000001},{"interface":"front","average":{"1min":3.6680000000000001,"5min":2.0209999999999999,"15min":1.101},"min":{"1min":0.40699999999999997,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":12.468999999999999,"5min":12.468999999999999,"15min":12.468999999999999},"last":0.76600000000000001}]}]},{"osd":4,"up_from":30,"seq":128849019082,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":174620,"kb_used_data":45076,"kb_used_omap":0,"kb_used_meta":129536,"kb_avail":93570532,"statfs":{"total":95995035648,"available":95816224768,"internally_reserved":0,"allocated":46157824,"data_stored":45457289,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":132644864},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:26:22 2023","interfaces":[{"interface":"back","average":{"1min":2.258,"5min":1.411,"15min":0.84899999999999998},"min":{"1min":0.25800000000000001,"5min":0.222,"15min":0.222},"max":{"1min":11.331,"5min":20.411999999999999,"15min":20.411999999999999},"last":0.371},{"interface":"front","average":{"1min":2.6629999999999998,"5min":1.5489999999999999,"15min":0.94399999999999995},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":13.595000000000001,"5min":20.454999999999998,"15min":20.454999999999998},"last":0.47699999999999998}]},{"osd":1,"last update":"Sat Sep 23 20:27:00 2023","interfaces":[{"interface":"back","average":{"1min":2.883,"5min":1.5089999999999999,"15min":0.92000000000000004},"min":{"1min":0.24299999999999999,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":9.6660000000000004,"5min":9.6660000000000004,"15min":9.6660000000000004},"last":0.53500000000000003},{"interface":"front","average":{"1min":2.4990000000000001,"5min":1.5640000000000001,"15min":0.93799999999999994},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":12.821999999999999,"5min":12.821999999999999,"15min":12.821999999999999},"last":0.32700000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:26:32 2023","interfaces":[{"interface":"back","average":{"1min":2.1779999999999999,"5min":1.51,"15min":0.90800000000000003},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":11.412000000000001,"5min":21.462,"15min":21.462},"last":0.54600000000000004},{"interface":"front","average":{"1min":3.29,"5min":1.8660000000000001,"15min":1.014},"min":{"1min":0.28999999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":13.567,"5min":22.593,"15min":22.593},"last":0.29799999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:27:00 2023","interfaces":[{"interface":"back","average":{"1min":2.516,"5min":1.7929999999999999,"15min":1.0249999999999999},"min":{"1min":0.32900000000000001,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":10.846,"5min":24.891999999999999,"15min":24.891999999999999},"last":0.504},{"interface":"front","average":{"1min":2.464,"5min":1.831,"15min":1.0529999999999999},"min":{"1min":0.35499999999999998,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":9.2080000000000002,"5min":21.922000000000001,"15min":21.922000000000001},"last":0.42499999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:26:10 2023","interfaces":[{"interface":"back","average":{"1min":5.0119999999999996,"5min":1.649,"15min":0.86799999999999999},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":17.704000000000001,"5min":17.704000000000001,"15min":17.704000000000001},"last":0.40899999999999997},{"interface":"front","average":{"1min":3.1600000000000001,"5min":1.361,"15min":0.77900000000000003},"min":{"1min":0.32500000000000001,"5min":0.28000000000000003,"15min":0.248},"max":{"1min":15.329000000000001,"5min":15.329000000000001,"15min":15.329000000000001},"last":0.34599999999999997}]},{"osd":6,"last update":"Sat Sep 23 20:26:34 2023","interfaces":[{"interface":"back","average":{"1min":3.3639999999999999,"5min":1.9239999999999999,"15min":0.98099999999999998},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.27500000000000002},"max":{"1min":15.252000000000001,"5min":41.371000000000002,"15min":41.371000000000002},"last":0.38600000000000001},{"interface":"front","average":{"1min":1.9179999999999999,"5min":1.593,"15min":0.88200000000000001},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":9.3350000000000009,"5min":41.140999999999998,"15min":41.140999999999998},"last":0.56499999999999995}]},{"osd":7,"last update":"Sat Sep 23 20:26:48 2023","interfaces":[{"interface":"back","average":{"1min":4.3280000000000003,"5min":1.8859999999999999,"15min":0.997},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.33600000000000002},"max":{"1min":16.427,"5min":16.427,"15min":16.427},"last":0.52000000000000002},{"interface":"front","average":{"1min":3.2890000000000001,"5min":1.754,"15min":0.96599999999999997},"min":{"1min":0.36399999999999999,"5min":0.34300000000000003,"15min":0.29899999999999999},"max":{"1min":15.645,"5min":19.018999999999998,"15min":19.018999999999998},"last":0.49299999999999999}]}]},{"osd":5,"up_from":39,"seq":167503724743,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":376988,"kb_used_data":252116,"kb_used_omap":0,"kb_used_meta":124864,"kb_avail":93368164,"statfs":{"total":95995035648,"available":95608999936,"internally_reserved":0,"allocated":258166784,"data_stored":257470437,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":127860736},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,6],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:26:22 2023","interfaces":[{"interface":"back","average":{"1min":2.3380000000000001,"5min":1.548,"15min":0.91000000000000003},"min":{"1min":0.29499999999999998,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":11.346,"5min":32.475000000000001,"15min":32.475000000000001},"last":3.5640000000000001},{"interface":"front","average":{"1min":1.9650000000000001,"5min":1.601,"15min":0.93400000000000005},"min":{"1min":0.245,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":5.6879999999999997,"5min":32.457000000000001,"15min":32.457000000000001},"last":0.48499999999999999}]},{"osd":1,"last update":"Sat Sep 23 20:26:00 2023","interfaces":[{"interface":"back","average":{"1min":2.4950000000000001,"5min":1.4490000000000001,"15min":0.84599999999999997},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":11.295,"5min":32.243000000000002,"15min":32.243000000000002},"last":3.512},{"interface":"front","average":{"1min":1.228,"5min":1.2370000000000001,"15min":0.77700000000000002},"min":{"1min":0.32300000000000001,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":7.1319999999999997,"5min":32.231000000000002,"15min":32.231000000000002},"last":1.1699999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:26:22 2023","interfaces":[{"interface":"back","average":{"1min":1.9970000000000001,"5min":1.891,"15min":0.997},"min":{"1min":0.30099999999999999,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":11.407,"5min":34.514000000000003,"15min":34.514000000000003},"last":3.6190000000000002},{"interface":"front","average":{"1min":2.105,"5min":1.732,"15min":0.94999999999999996},"min":{"1min":0.32100000000000001,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":11.574999999999999,"5min":32.515000000000001,"15min":32.515000000000001},"last":1.0349999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:26:00 2023","interfaces":[{"interface":"back","average":{"1min":2.1019999999999999,"5min":1.1990000000000001,"15min":0.77100000000000002},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":7.0309999999999997,"5min":7.3799999999999999,"15min":7.3799999999999999},"last":1.0680000000000001},{"interface":"front","average":{"1min":2.831,"5min":1.3240000000000001,"15min":0.85199999999999998},"min":{"1min":0.36699999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":11.379,"5min":11.379,"15min":11.379},"last":3.544}]},{"osd":4,"last update":"Sat Sep 23 20:26:18 2023","interfaces":[{"interface":"back","average":{"1min":3.391,"5min":1.4339999999999999,"15min":0.78000000000000003},"min":{"1min":0.29199999999999998,"5min":0.26700000000000002,"15min":0.26400000000000001},"max":{"1min":14.218999999999999,"5min":14.218999999999999,"15min":14.218999999999999},"last":0.86299999999999999},{"interface":"front","average":{"1min":2.6720000000000002,"5min":1.3420000000000001,"15min":0.77000000000000002},"min":{"1min":0.34300000000000003,"5min":0.29099999999999998,"15min":0.28799999999999998},"max":{"1min":11.664,"5min":11.664,"15min":11.664},"last":4.1929999999999996}]},{"osd":6,"last update":"Sat Sep 23 20:26:32 2023","interfaces":[{"interface":"back","average":{"1min":2.633,"5min":1.575,"15min":0.85599999999999998},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.29799999999999999},"max":{"1min":8.7949999999999999,"5min":11.545999999999999,"15min":11.545999999999999},"last":1.091},{"interface":"front","average":{"1min":1.6870000000000001,"5min":1.5,"15min":0.879},"min":{"1min":0.376,"5min":0.35999999999999999,"15min":0.34499999999999997},"max":{"1min":8.4120000000000008,"5min":11.456,"15min":11.456},"last":3.601}]},{"osd":7,"last update":"Sat Sep 23 20:26:00 2023","interfaces":[{"interface":"back","average":{"1min":2.9199999999999999,"5min":1.2470000000000001,"15min":0.77200000000000002},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33800000000000002},"max":{"1min":11.430999999999999,"5min":11.430999999999999,"15min":11.430999999999999},"last":3.5840000000000001},{"interface":"front","average":{"1min":1.6479999999999999,"5min":1.0669999999999999,"15min":0.71899999999999997},"min":{"1min":0.35699999999999998,"5min":0.315,"15min":0.315},"max":{"1min":7.0890000000000004,"5min":12.368,"15min":12.368},"last":0.45400000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4722,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1360,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6696,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7156,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3154,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":82509824,"data_stored":82509824,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":80019456,"data_stored":80019456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":17694720,"data_stored":17694720,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":42139648,"data_stored":42139648,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":256049152,"data_stored":256049152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":70516736,"data_stored":70516736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:27:06.106 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:27:06.137 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:06.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:06 smithi149 ceph-mon[170394]: 1.0 scrub starts 2023-09-23T20:27:06.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:06 smithi149 ceph-mon[170394]: from='client.35211 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:27:06.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:06 smithi149 ceph-mon[170394]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:27:06.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:06 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:27:06.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:06 smithi149 ceph-mon[170394]: osdmap e210: 8 total, 8 up, 8 in 2023-09-23T20:27:06.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:06 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 2003/300 objects degraded (667.667%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T20:27:06.239 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:27:06.242 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:06.243 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:27:06.317 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:06 smithi018 ceph-mon[179647]: 1.0 scrub starts 2023-09-23T20:27:06.317 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:06 smithi018 ceph-mon[179647]: from='client.35211 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:27:06.318 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:06 smithi018 ceph-mon[179647]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:27:06.318 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:06 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:27:06.318 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:06 smithi018 ceph-mon[179647]: osdmap e210: 8 total, 8 up, 8 in 2023-09-23T20:27:06.318 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:06 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 2003/300 objects degraded (667.667%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T20:27:06.318 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:06 smithi018 ceph-mon[184144]: 1.0 scrub starts 2023-09-23T20:27:06.318 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:06 smithi018 ceph-mon[184144]: from='client.35211 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:27:06.318 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:06 smithi018 ceph-mon[184144]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:27:06.319 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:06 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:27:06.319 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:06 smithi018 ceph-mon[184144]: osdmap e210: 8 total, 8 up, 8 in 2023-09-23T20:27:06.319 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:06 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 2003/300 objects degraded (667.667%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T20:27:06.343 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:27:06.371 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:06.408 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:27:06.410 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:27:06.411 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:06.411 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:27:06.512 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:27:06.543 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:06.639 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:27:06.642 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:06.642 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:27:06.743 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:27:06.775 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:06.811 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:27:06.814 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:06.814 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:27:06.915 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:27:06.943 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:06.977 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:27:06.980 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:06.980 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:27:07.015 INFO:teuthology.orchestra.run.smithi018.stderr:pool 'unique_pool_0' removed 2023-09-23T20:27:07.081 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:27:07.133 INFO:teuthology.orchestra.run.smithi018.stdout:ERROR: (22) Invalid argument 2023-09-23T20:27:07.133 INFO:teuthology.orchestra.run.smithi018.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:27:07.183 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:07 smithi149 ceph-mon[170394]: pgmap v456: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 peering, 106 active+clean; 9.1 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 270/300 objects degraded (90.000%) 2023-09-23T20:27:07.183 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:07 smithi149 ceph-mon[170394]: failed to encode map e210 with expected crc 2023-09-23T20:27:07.183 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:07 smithi149 ceph-mon[170394]: failed to encode map e210 with expected crc 2023-09-23T20:27:07.183 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:07 smithi149 ceph-mon[170394]: failed to encode map e210 with expected crc 2023-09-23T20:27:07.183 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:07 smithi149 ceph-mon[170394]: failed to encode map e210 with expected crc 2023-09-23T20:27:07.183 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:07 smithi149 ceph-mon[170394]: failed to encode map e210 with expected crc 2023-09-23T20:27:07.183 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:07 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/317938900' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T20:27:07.183 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:07 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T20:27:07.184 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:07 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T20:27:07.184 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:07 smithi149 ceph-mon[170394]: osdmap e211: 8 total, 8 up, 8 in 2023-09-23T20:27:07.195 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:07.302 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:27:07.304 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:07.305 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:27:07.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:07 smithi018 ceph-mon[184144]: pgmap v456: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 peering, 106 active+clean; 9.1 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 270/300 objects degraded (90.000%) 2023-09-23T20:27:07.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:07 smithi018 ceph-mon[184144]: failed to encode map e210 with expected crc 2023-09-23T20:27:07.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:07 smithi018 ceph-mon[184144]: failed to encode map e210 with expected crc 2023-09-23T20:27:07.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:07 smithi018 ceph-mon[184144]: failed to encode map e210 with expected crc 2023-09-23T20:27:07.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:07 smithi018 ceph-mon[184144]: failed to encode map e210 with expected crc 2023-09-23T20:27:07.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:07 smithi018 ceph-mon[184144]: failed to encode map e210 with expected crc 2023-09-23T20:27:07.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:07 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/317938900' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T20:27:07.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:07 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T20:27:07.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:07 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T20:27:07.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:07 smithi018 ceph-mon[184144]: osdmap e211: 8 total, 8 up, 8 in 2023-09-23T20:27:07.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:07 smithi018 ceph-mon[179647]: pgmap v456: 109 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 1 peering, 106 active+clean; 9.1 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 270/300 objects degraded (90.000%) 2023-09-23T20:27:07.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:07 smithi018 ceph-mon[179647]: failed to encode map e210 with expected crc 2023-09-23T20:27:07.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:07 smithi018 ceph-mon[179647]: failed to encode map e210 with expected crc 2023-09-23T20:27:07.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:07 smithi018 ceph-mon[179647]: failed to encode map e210 with expected crc 2023-09-23T20:27:07.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:07 smithi018 ceph-mon[179647]: failed to encode map e210 with expected crc 2023-09-23T20:27:07.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:07 smithi018 ceph-mon[179647]: failed to encode map e210 with expected crc 2023-09-23T20:27:07.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:07 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/317938900' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T20:27:07.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:07 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T20:27:07.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:07 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T20:27:07.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:07 smithi018 ceph-mon[179647]: osdmap e211: 8 total, 8 up, 8 in 2023-09-23T20:27:07.406 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:27:07.434 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:07.651 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T20:27:07.724 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:27:07.726 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:07.727 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:27:07.827 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:27:07.855 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:07.932 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-09-23T20:27:07.932 INFO:tasks.radosbench:Beginning radosbench... 2023-09-23T20:27:07.933 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_1 2023-09-23T20:27:07.933 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd pool create unique_pool_1 16 2023-09-23T20:27:08.118 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:27:08.121 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:08.121 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:27:08.124 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T20:27:08.222 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:27:08.249 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:08.281 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:27:08.284 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:08.284 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:27:08.385 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:27:08.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:08 smithi018 ceph-mon[179647]: failed to encode map e210 with expected crc 2023-09-23T20:27:08.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:08 smithi018 ceph-mon[179647]: failed to encode map e211 with expected crc 2023-09-23T20:27:08.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:08 smithi018 ceph-mon[179647]: failed to encode map e211 with expected crc 2023-09-23T20:27:08.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:08 smithi018 ceph-mon[179647]: failed to encode map e211 with expected crc 2023-09-23T20:27:08.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:08 smithi018 ceph-mon[179647]: failed to encode map e211 with expected crc 2023-09-23T20:27:08.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:08 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"}]: dispatch 2023-09-23T20:27:08.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:08 smithi018 ceph-mon[184144]: failed to encode map e210 with expected crc 2023-09-23T20:27:08.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:08 smithi018 ceph-mon[184144]: failed to encode map e211 with expected crc 2023-09-23T20:27:08.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:08 smithi018 ceph-mon[184144]: failed to encode map e211 with expected crc 2023-09-23T20:27:08.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:08 smithi018 ceph-mon[184144]: failed to encode map e211 with expected crc 2023-09-23T20:27:08.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:08 smithi018 ceph-mon[184144]: failed to encode map e211 with expected crc 2023-09-23T20:27:08.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:08 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"}]: dispatch 2023-09-23T20:27:08.413 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:08.433 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:27:08.454 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:27:08.457 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:08.458 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:27:08.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:08 smithi149 ceph-mon[170394]: failed to encode map e210 with expected crc 2023-09-23T20:27:08.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:08 smithi149 ceph-mon[170394]: failed to encode map e211 with expected crc 2023-09-23T20:27:08.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:08 smithi149 ceph-mon[170394]: failed to encode map e211 with expected crc 2023-09-23T20:27:08.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:08 smithi149 ceph-mon[170394]: failed to encode map e211 with expected crc 2023-09-23T20:27:08.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:08 smithi149 ceph-mon[170394]: failed to encode map e211 with expected crc 2023-09-23T20:27:08.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:08 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"}]: dispatch 2023-09-23T20:27:08.559 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:27:08.586 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:08.617 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:27:08.620 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:08.621 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:27:08.715 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:27:08.721 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:27:08.749 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:27:08.767 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:08.773 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:27:08.805 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:27:08.810 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:27:08.813 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:08.814 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:27:08.826 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:27:08.845 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:27:08.863 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:27:08.882 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:27:08.883 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:27:08.884 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:27:08.884 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:27:08.884 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:27:08.884 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:27:08.884 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:27:08.884 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:27:08.884 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:27:08.914 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:27:08.918 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:27:08.956 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:08.999 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:27:09.002 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:09.003 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:27:09.017 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:27:09.104 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:27:09.131 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:09.224 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:27:09.227 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:09.227 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:27:09.328 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:27:09.355 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:09 smithi018 ceph-mon[179647]: pgmap v458: 83 pgs: 83 active+clean; 4.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-09-23T20:27:09.355 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:09 smithi018 ceph-mon[179647]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 270/300 objects degraded (90.000%), 1 pg degraded, 2 pgs undersized) 2023-09-23T20:27:09.356 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:09 smithi018 ceph-mon[179647]: Cluster is now healthy 2023-09-23T20:27:09.356 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:09 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"}]': finished 2023-09-23T20:27:09.356 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:09 smithi018 ceph-mon[179647]: osdmap e212: 8 total, 8 up, 8 in 2023-09-23T20:27:09.356 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:09 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1793116899' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:27:09.356 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:09 smithi018 ceph-mon[184144]: pgmap v458: 83 pgs: 83 active+clean; 4.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-09-23T20:27:09.356 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:09 smithi018 ceph-mon[184144]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 270/300 objects degraded (90.000%), 1 pg degraded, 2 pgs undersized) 2023-09-23T20:27:09.356 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:09 smithi018 ceph-mon[184144]: Cluster is now healthy 2023-09-23T20:27:09.356 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:09 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"}]': finished 2023-09-23T20:27:09.356 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:09 smithi018 ceph-mon[184144]: osdmap e212: 8 total, 8 up, 8 in 2023-09-23T20:27:09.356 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:09 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1793116899' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:27:09.358 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:09.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:09 smithi149 ceph-mon[170394]: pgmap v458: 83 pgs: 83 active+clean; 4.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-09-23T20:27:09.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:09 smithi149 ceph-mon[170394]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 270/300 objects degraded (90.000%), 1 pg degraded, 2 pgs undersized) 2023-09-23T20:27:09.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:09 smithi149 ceph-mon[170394]: Cluster is now healthy 2023-09-23T20:27:09.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:09 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"}]': finished 2023-09-23T20:27:09.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:09 smithi149 ceph-mon[170394]: osdmap e212: 8 total, 8 up, 8 in 2023-09-23T20:27:09.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:09 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1793116899' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:27:09.790 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:27:09.794 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:27:09.798 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:09.799 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:27:09.899 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:27:09.931 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:09.965 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:27:09.968 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:09.968 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:27:10.069 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:27:10.096 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:10.130 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:27:10.132 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:10.133 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:27:10.233 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:27:10.260 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:10 smithi018 ceph-mon[184144]: failed to encode map e212 with expected crc 2023-09-23T20:27:10.260 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:10 smithi018 ceph-mon[184144]: failed to encode map e212 with expected crc 2023-09-23T20:27:10.260 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:10 smithi018 ceph-mon[184144]: failed to encode map e213 with expected crc 2023-09-23T20:27:10.261 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:10 smithi018 ceph-mon[184144]: osdmap e213: 8 total, 8 up, 8 in 2023-09-23T20:27:10.261 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:10 smithi018 ceph-mon[184144]: failed to encode map e213 with expected crc 2023-09-23T20:27:10.261 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:10 smithi018 ceph-mon[184144]: failed to encode map e213 with expected crc 2023-09-23T20:27:10.261 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:10 smithi018 ceph-mon[184144]: failed to encode map e213 with expected crc 2023-09-23T20:27:10.261 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:10 smithi018 ceph-mon[184144]: failed to encode map e213 with expected crc 2023-09-23T20:27:10.261 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:10 smithi018 ceph-mon[184144]: failed to encode map e213 with expected crc 2023-09-23T20:27:10.261 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:10 smithi018 ceph-mon[179647]: failed to encode map e212 with expected crc 2023-09-23T20:27:10.261 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:10 smithi018 ceph-mon[179647]: failed to encode map e212 with expected crc 2023-09-23T20:27:10.261 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:10 smithi018 ceph-mon[179647]: failed to encode map e213 with expected crc 2023-09-23T20:27:10.262 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:10 smithi018 ceph-mon[179647]: osdmap e213: 8 total, 8 up, 8 in 2023-09-23T20:27:10.262 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:10 smithi018 ceph-mon[179647]: failed to encode map e213 with expected crc 2023-09-23T20:27:10.262 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:10 smithi018 ceph-mon[179647]: failed to encode map e213 with expected crc 2023-09-23T20:27:10.262 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:10 smithi018 ceph-mon[179647]: failed to encode map e213 with expected crc 2023-09-23T20:27:10.262 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:10 smithi018 ceph-mon[179647]: failed to encode map e213 with expected crc 2023-09-23T20:27:10.262 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:10 smithi018 ceph-mon[179647]: failed to encode map e213 with expected crc 2023-09-23T20:27:10.263 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:10.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:10 smithi149 ceph-mon[170394]: failed to encode map e212 with expected crc 2023-09-23T20:27:10.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:10 smithi149 ceph-mon[170394]: failed to encode map e212 with expected crc 2023-09-23T20:27:10.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:10 smithi149 ceph-mon[170394]: failed to encode map e213 with expected crc 2023-09-23T20:27:10.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:10 smithi149 ceph-mon[170394]: osdmap e213: 8 total, 8 up, 8 in 2023-09-23T20:27:10.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:10 smithi149 ceph-mon[170394]: failed to encode map e213 with expected crc 2023-09-23T20:27:10.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:10 smithi149 ceph-mon[170394]: failed to encode map e213 with expected crc 2023-09-23T20:27:10.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:10 smithi149 ceph-mon[170394]: failed to encode map e213 with expected crc 2023-09-23T20:27:10.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:10 smithi149 ceph-mon[170394]: failed to encode map e213 with expected crc 2023-09-23T20:27:10.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:10 smithi149 ceph-mon[170394]: failed to encode map e213 with expected crc 2023-09-23T20:27:11.055 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:27:11.126 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2023-09-23T20:27:11.126 INFO:teuthology.orchestra.run.smithi018.stdout: "ops": [], 2023-09-23T20:27:11.126 INFO:teuthology.orchestra.run.smithi018.stdout: "num_ops": 0 2023-09-23T20:27:11.127 INFO:teuthology.orchestra.run.smithi018.stdout:} 2023-09-23T20:27:11.185 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:27:11.189 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:11.189 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:27:11.290 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:27:11.321 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:11.380 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:27:11.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:11 smithi018 ceph-mon[184144]: pgmap v461: 83 pgs: 83 active+clean; 4.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-09-23T20:27:11.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:11 smithi018 ceph-mon[184144]: failed to encode map e214 with expected crc 2023-09-23T20:27:11.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:11 smithi018 ceph-mon[184144]: failed to encode map e214 with expected crc 2023-09-23T20:27:11.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:11 smithi018 ceph-mon[184144]: osdmap e214: 8 total, 8 up, 8 in 2023-09-23T20:27:11.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:11 smithi018 ceph-mon[184144]: failed to encode map e214 with expected crc 2023-09-23T20:27:11.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:11 smithi018 ceph-mon[184144]: failed to encode map e214 with expected crc 2023-09-23T20:27:11.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:11 smithi018 ceph-mon[184144]: failed to encode map e214 with expected crc 2023-09-23T20:27:11.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:11 smithi018 ceph-mon[184144]: osdmap e215: 8 total, 8 up, 8 in 2023-09-23T20:27:11.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:11 smithi018 ceph-mon[179647]: pgmap v461: 83 pgs: 83 active+clean; 4.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-09-23T20:27:11.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:11 smithi018 ceph-mon[179647]: failed to encode map e214 with expected crc 2023-09-23T20:27:11.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:11 smithi018 ceph-mon[179647]: failed to encode map e214 with expected crc 2023-09-23T20:27:11.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:11 smithi018 ceph-mon[179647]: osdmap e214: 8 total, 8 up, 8 in 2023-09-23T20:27:11.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:11 smithi018 ceph-mon[179647]: failed to encode map e214 with expected crc 2023-09-23T20:27:11.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:11 smithi018 ceph-mon[179647]: failed to encode map e214 with expected crc 2023-09-23T20:27:11.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:11 smithi018 ceph-mon[179647]: failed to encode map e214 with expected crc 2023-09-23T20:27:11.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:11 smithi018 ceph-mon[179647]: osdmap e215: 8 total, 8 up, 8 in 2023-09-23T20:27:11.406 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:27:11.409 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:11.409 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:27:11.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:11 smithi149 ceph-mon[170394]: pgmap v461: 83 pgs: 83 active+clean; 4.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-09-23T20:27:11.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:11 smithi149 ceph-mon[170394]: failed to encode map e214 with expected crc 2023-09-23T20:27:11.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:11 smithi149 ceph-mon[170394]: failed to encode map e214 with expected crc 2023-09-23T20:27:11.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:11 smithi149 ceph-mon[170394]: osdmap e214: 8 total, 8 up, 8 in 2023-09-23T20:27:11.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:11 smithi149 ceph-mon[170394]: failed to encode map e214 with expected crc 2023-09-23T20:27:11.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:11 smithi149 ceph-mon[170394]: failed to encode map e214 with expected crc 2023-09-23T20:27:11.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:11 smithi149 ceph-mon[170394]: failed to encode map e214 with expected crc 2023-09-23T20:27:11.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:11 smithi149 ceph-mon[170394]: osdmap e215: 8 total, 8 up, 8 in 2023-09-23T20:27:11.510 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:27:11.540 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:11.576 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:27:11.579 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:11.579 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:27:11.680 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:27:11.707 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:11.741 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:27:11.744 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:11.744 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:27:11.845 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:27:11.871 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:11.904 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:27:11.906 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:11.907 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:27:12.007 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:27:12.037 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:12.074 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:27:12.076 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:12.077 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:27:12.109 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T20:27:12.178 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:27:12.219 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:12.282 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:12 smithi018 ceph-mon[179647]: failed to encode map e215 with expected crc 2023-09-23T20:27:12.283 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:12 smithi018 ceph-mon[179647]: failed to encode map e215 with expected crc 2023-09-23T20:27:12.283 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:12 smithi018 ceph-mon[179647]: failed to encode map e215 with expected crc 2023-09-23T20:27:12.283 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:12 smithi018 ceph-mon[179647]: failed to encode map e215 with expected crc 2023-09-23T20:27:12.283 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:12 smithi018 ceph-mon[179647]: failed to encode map e215 with expected crc 2023-09-23T20:27:12.283 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:12 smithi018 ceph-mon[179647]: failed to encode map e215 with expected crc 2023-09-23T20:27:12.283 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:12 smithi018 ceph-mon[179647]: failed to encode map e215 with expected crc 2023-09-23T20:27:12.283 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:12 smithi018 ceph-mon[179647]: osdmap e216: 8 total, 8 up, 8 in 2023-09-23T20:27:12.283 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:12 smithi018 ceph-mon[184144]: failed to encode map e215 with expected crc 2023-09-23T20:27:12.284 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:12 smithi018 ceph-mon[184144]: failed to encode map e215 with expected crc 2023-09-23T20:27:12.284 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:12 smithi018 ceph-mon[184144]: failed to encode map e215 with expected crc 2023-09-23T20:27:12.284 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:12 smithi018 ceph-mon[184144]: failed to encode map e215 with expected crc 2023-09-23T20:27:12.284 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:12 smithi018 ceph-mon[184144]: failed to encode map e215 with expected crc 2023-09-23T20:27:12.284 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:12 smithi018 ceph-mon[184144]: failed to encode map e215 with expected crc 2023-09-23T20:27:12.284 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:12 smithi018 ceph-mon[184144]: failed to encode map e215 with expected crc 2023-09-23T20:27:12.284 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:12 smithi018 ceph-mon[184144]: osdmap e216: 8 total, 8 up, 8 in 2023-09-23T20:27:12.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:12 smithi149 ceph-mon[170394]: failed to encode map e215 with expected crc 2023-09-23T20:27:12.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:12 smithi149 ceph-mon[170394]: failed to encode map e215 with expected crc 2023-09-23T20:27:12.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:12 smithi149 ceph-mon[170394]: failed to encode map e215 with expected crc 2023-09-23T20:27:12.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:12 smithi149 ceph-mon[170394]: failed to encode map e215 with expected crc 2023-09-23T20:27:12.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:12 smithi149 ceph-mon[170394]: failed to encode map e215 with expected crc 2023-09-23T20:27:12.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:12 smithi149 ceph-mon[170394]: failed to encode map e215 with expected crc 2023-09-23T20:27:12.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:12 smithi149 ceph-mon[170394]: failed to encode map e215 with expected crc 2023-09-23T20:27:12.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:12 smithi149 ceph-mon[170394]: osdmap e216: 8 total, 8 up, 8 in 2023-09-23T20:27:12.524 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:27:12.527 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:12.527 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:27:12.628 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:27:12.657 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:12.691 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:27:12.694 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:12.695 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:27:12.795 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:27:12.829 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:12.920 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:27:12.923 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:12.924 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:27:12.971 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:27:13.025 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:27:13.059 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:13.111 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:27:13.114 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:13.115 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:27:13.216 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:27:13.245 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:13.261 INFO:teuthology.orchestra.run.smithi018.stderr:noscrub is set 2023-09-23T20:27:13.267 INFO:teuthology.orchestra.run.smithi018.stderr:pool 'unique_pool_1' created 2023-09-23T20:27:13.285 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:27:13.288 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:13.288 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:27:13.376 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:27:13.377 INFO:teuthology.orchestra.run.smithi018.stderr:dumped all 2023-09-23T20:27:13.389 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:27:13.408 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:27:13.421 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:13.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:13 smithi149 ceph-mon[170394]: pgmap v464: 83 pgs: 4 active+undersized+remapped, 6 remapped+peering, 73 active+clean; 4.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-09-23T20:27:13.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:13 smithi149 ceph-mon[170394]: failed to encode map e216 with expected crc 2023-09-23T20:27:13.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:13 smithi149 ceph-mon[170394]: failed to encode map e216 with expected crc 2023-09-23T20:27:13.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:13 smithi149 ceph-mon[170394]: failed to encode map e216 with expected crc 2023-09-23T20:27:13.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:13 smithi149 ceph-mon[170394]: failed to encode map e216 with expected crc 2023-09-23T20:27:13.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:13 smithi149 ceph-mon[170394]: failed to encode map e216 with expected crc 2023-09-23T20:27:13.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:13 smithi149 ceph-mon[170394]: failed to encode map e216 with expected crc 2023-09-23T20:27:13.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:13 smithi149 ceph-mon[170394]: failed to encode map e216 with expected crc 2023-09-23T20:27:13.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:13 smithi149 ceph-mon[170394]: failed to encode map e216 with expected crc 2023-09-23T20:27:13.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:13 smithi149 ceph-mon[170394]: failed to encode map e216 with expected crc 2023-09-23T20:27:13.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:13 smithi149 ceph-mon[170394]: Health check failed: Reduced data availability: 6 pgs peering (PG_AVAILABILITY) 2023-09-23T20:27:13.524 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:13 smithi018 ceph-mon[184144]: pgmap v464: 83 pgs: 4 active+undersized+remapped, 6 remapped+peering, 73 active+clean; 4.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-09-23T20:27:13.524 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:13 smithi018 ceph-mon[184144]: failed to encode map e216 with expected crc 2023-09-23T20:27:13.524 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:13 smithi018 ceph-mon[184144]: failed to encode map e216 with expected crc 2023-09-23T20:27:13.524 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:13 smithi018 ceph-mon[184144]: failed to encode map e216 with expected crc 2023-09-23T20:27:13.525 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:13 smithi018 ceph-mon[184144]: failed to encode map e216 with expected crc 2023-09-23T20:27:13.525 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:13 smithi018 ceph-mon[184144]: failed to encode map e216 with expected crc 2023-09-23T20:27:13.525 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:13 smithi018 ceph-mon[184144]: failed to encode map e216 with expected crc 2023-09-23T20:27:13.525 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:13 smithi018 ceph-mon[184144]: failed to encode map e216 with expected crc 2023-09-23T20:27:13.525 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:13 smithi018 ceph-mon[184144]: failed to encode map e216 with expected crc 2023-09-23T20:27:13.525 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:13 smithi018 ceph-mon[184144]: failed to encode map e216 with expected crc 2023-09-23T20:27:13.525 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:13 smithi018 ceph-mon[184144]: Health check failed: Reduced data availability: 6 pgs peering (PG_AVAILABILITY) 2023-09-23T20:27:13.525 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:13 smithi018 ceph-mon[179647]: pgmap v464: 83 pgs: 4 active+undersized+remapped, 6 remapped+peering, 73 active+clean; 4.6 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-09-23T20:27:13.525 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:13 smithi018 ceph-mon[179647]: failed to encode map e216 with expected crc 2023-09-23T20:27:13.525 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:13 smithi018 ceph-mon[179647]: failed to encode map e216 with expected crc 2023-09-23T20:27:13.526 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:13 smithi018 ceph-mon[179647]: failed to encode map e216 with expected crc 2023-09-23T20:27:13.526 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:13 smithi018 ceph-mon[179647]: failed to encode map e216 with expected crc 2023-09-23T20:27:13.526 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:13 smithi018 ceph-mon[179647]: failed to encode map e216 with expected crc 2023-09-23T20:27:13.526 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:13 smithi018 ceph-mon[179647]: failed to encode map e216 with expected crc 2023-09-23T20:27:13.526 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:13 smithi018 ceph-mon[179647]: failed to encode map e216 with expected crc 2023-09-23T20:27:13.526 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:13 smithi018 ceph-mon[179647]: failed to encode map e216 with expected crc 2023-09-23T20:27:13.526 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:13 smithi018 ceph-mon[179647]: failed to encode map e216 with expected crc 2023-09-23T20:27:13.526 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:13 smithi018 ceph-mon[179647]: Health check failed: Reduced data availability: 6 pgs peering (PG_AVAILABILITY) 2023-09-23T20:27:14.058 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:27:14.062 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:14.063 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:27:14.164 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:27:14.196 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:14.232 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:27:14.235 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:14.236 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:27:14.242 INFO:teuthology.orchestra.run.smithi018.stdout:{"pg_ready":true,"pg_map":{"version":466,"stamp":"2023-09-23T20:27:13.252165+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790195,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6484,"num_read_kb":14128,"num_write":1146,"num_write_kb":15141,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":2100140,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":2802,"ondisk_log_size":2802,"up":249,"acting":240,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":296,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":1527312,"kb_used_data":396688,"kb_used_omap":30,"kb_used_meta":1130529,"kb_avail":748433904,"statfs":{"total":767960285184,"available":766396317696,"internally_reserved":0,"allocated":406208512,"data_stored":399174875,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31263,"internal_metadata":1157662177},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":108,"apply_latency_ms":108,"commit_latency_ns":108000000,"apply_latency_ns":108000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":-11,"num_read_kb":-8,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"4.069954"},"pg_stats":[{"pgid":"3.1a","version":"69'1","reported_seq":211,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T20:27:09.189432+0000","last_change":"2023-09-23T20:25:39.632437+0000","last_active":"2023-09-23T20:27:09.189432+0000","last_peered":"2023-09-23T20:27:09.189432+0000","last_clean":"2023-09-23T20:27:09.189432+0000","last_became_active":"2023-09-23T20:25:39.632074+0000","last_became_peered":"2023-09-23T20:25:39.632074+0000","last_unstale":"2023-09-23T20:27:09.189432+0000","last_undegraded":"2023-09-23T20:27:09.189432+0000","last_fullsized":"2023-09-23T20:27:09.189432+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:32:13.543799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":61,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T20:27:10.955031+0000","last_change":"2023-09-23T20:25:02.932223+0000","last_active":"2023-09-23T20:27:10.955031+0000","last_peered":"2023-09-23T20:27:10.955031+0000","last_clean":"2023-09-23T20:27:10.955031+0000","last_became_active":"2023-09-23T20:25:02.931024+0000","last_became_peered":"2023-09-23T20:25:02.931024+0000","last_unstale":"2023-09-23T20:27:10.955031+0000","last_undegraded":"2023-09-23T20:27:10.955031+0000","last_fullsized":"2023-09-23T20:27:10.955031+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:16:00.586211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":170,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T20:27:08.178929+0000","last_change":"2023-09-23T20:25:02.934543+0000","last_active":"2023-09-23T20:27:08.178929+0000","last_peered":"2023-09-23T20:27:08.178929+0000","last_clean":"2023-09-23T20:27:08.178929+0000","last_became_active":"2023-09-23T20:25:02.934178+0000","last_became_peered":"2023-09-23T20:25:02.934178+0000","last_unstale":"2023-09-23T20:27:08.178929+0000","last_undegraded":"2023-09-23T20:27:08.178929+0000","last_fullsized":"2023-09-23T20:27:08.178929+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:27:38.453052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":176,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T20:27:08.746935+0000","last_change":"2023-09-23T20:25:02.957185+0000","last_active":"2023-09-23T20:27:08.746935+0000","last_peered":"2023-09-23T20:27:08.746935+0000","last_clean":"2023-09-23T20:27:08.746935+0000","last_became_active":"2023-09-23T20:25:02.956689+0000","last_became_peered":"2023-09-23T20:25:02.956689+0000","last_unstale":"2023-09-23T20:27:08.746935+0000","last_undegraded":"2023-09-23T20:27:08.746935+0000","last_fullsized":"2023-09-23T20:27:08.746935+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:04:28.524015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":61,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T20:27:10.954510+0000","last_change":"2023-09-23T20:25:02.932054+0000","last_active":"2023-09-23T20:27:10.954510+0000","last_peered":"2023-09-23T20:27:10.954510+0000","last_clean":"2023-09-23T20:27:10.954510+0000","last_became_active":"2023-09-23T20:25:02.931309+0000","last_became_peered":"2023-09-23T20:25:02.931309+0000","last_unstale":"2023-09-23T20:27:10.954510+0000","last_undegraded":"2023-09-23T20:27:10.954510+0000","last_fullsized":"2023-09-23T20:27:10.954510+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:57:53.378499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":173,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T20:27:09.190144+0000","last_change":"2023-09-23T20:20:46.937089+0000","last_active":"2023-09-23T20:27:09.190144+0000","last_peered":"2023-09-23T20:27:09.190144+0000","last_clean":"2023-09-23T20:27:09.190144+0000","last_became_active":"2023-09-23T20:20:46.936831+0000","last_became_peered":"2023-09-23T20:20:46.936831+0000","last_unstale":"2023-09-23T20:27:09.190144+0000","last_undegraded":"2023-09-23T20:27:09.190144+0000","last_fullsized":"2023-09-23T20:27:09.190144+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:34:56.747010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":174,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T20:27:08.747296+0000","last_change":"2023-09-23T20:25:02.969432+0000","last_active":"2023-09-23T20:27:08.747296+0000","last_peered":"2023-09-23T20:27:08.747296+0000","last_clean":"2023-09-23T20:27:08.747296+0000","last_became_active":"2023-09-23T20:25:02.969118+0000","last_became_peered":"2023-09-23T20:25:02.969118+0000","last_unstale":"2023-09-23T20:27:08.747296+0000","last_undegraded":"2023-09-23T20:27:08.747296+0000","last_fullsized":"2023-09-23T20:27:08.747296+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:02:22.453831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":61,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T20:27:10.954517+0000","last_change":"2023-09-23T20:25:02.932225+0000","last_active":"2023-09-23T20:27:10.954517+0000","last_peered":"2023-09-23T20:27:10.954517+0000","last_clean":"2023-09-23T20:27:10.954517+0000","last_became_active":"2023-09-23T20:25:02.931746+0000","last_became_peered":"2023-09-23T20:25:02.931746+0000","last_unstale":"2023-09-23T20:27:10.954517+0000","last_undegraded":"2023-09-23T20:27:10.954517+0000","last_fullsized":"2023-09-23T20:27:10.954517+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:57:24.470179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":173,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T20:27:09.195697+0000","last_change":"2023-09-23T20:20:46.950281+0000","last_active":"2023-09-23T20:27:09.195697+0000","last_peered":"2023-09-23T20:27:09.195697+0000","last_clean":"2023-09-23T20:27:09.195697+0000","last_became_active":"2023-09-23T20:20:46.949892+0000","last_became_peered":"2023-09-23T20:20:46.949892+0000","last_unstale":"2023-09-23T20:27:09.195697+0000","last_undegraded":"2023-09-23T20:27:09.195697+0000","last_fullsized":"2023-09-23T20:27:09.195697+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:28.763832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":61,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T20:27:10.955546+0000","last_change":"2023-09-23T20:25:02.947669+0000","last_active":"2023-09-23T20:27:10.955546+0000","last_peered":"2023-09-23T20:27:10.955546+0000","last_clean":"2023-09-23T20:27:10.955546+0000","last_became_active":"2023-09-23T20:25:02.947368+0000","last_became_peered":"2023-09-23T20:25:02.947368+0000","last_unstale":"2023-09-23T20:27:10.955546+0000","last_undegraded":"2023-09-23T20:27:10.955546+0000","last_fullsized":"2023-09-23T20:27:10.955546+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:52:53.813761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":209,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T20:27:09.189703+0000","last_change":"2023-09-23T20:25:03.038511+0000","last_active":"2023-09-23T20:27:09.189703+0000","last_peered":"2023-09-23T20:27:09.189703+0000","last_clean":"2023-09-23T20:27:09.189703+0000","last_became_active":"2023-09-23T20:25:02.976811+0000","last_became_peered":"2023-09-23T20:25:02.976811+0000","last_unstale":"2023-09-23T20:27:09.189703+0000","last_undegraded":"2023-09-23T20:27:09.189703+0000","last_fullsized":"2023-09-23T20:27:09.189703+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:28:06.605731+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":171,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T20:27:08.180198+0000","last_change":"2023-09-23T20:25:39.646452+0000","last_active":"2023-09-23T20:27:08.180198+0000","last_peered":"2023-09-23T20:27:08.180198+0000","last_clean":"2023-09-23T20:27:08.180198+0000","last_became_active":"2023-09-23T20:25:39.646175+0000","last_became_peered":"2023-09-23T20:25:39.646175+0000","last_unstale":"2023-09-23T20:27:08.180198+0000","last_undegraded":"2023-09-23T20:27:08.180198+0000","last_fullsized":"2023-09-23T20:27:08.180198+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:33:02.099951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":178,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T20:27:09.188788+0000","last_change":"2023-09-23T20:12:43.750217+0000","last_active":"2023-09-23T20:27:09.188788+0000","last_peered":"2023-09-23T20:27:09.188788+0000","last_clean":"2023-09-23T20:27:09.188788+0000","last_became_active":"2023-09-23T20:12:43.749929+0000","last_became_peered":"2023-09-23T20:12:43.749929+0000","last_unstale":"2023-09-23T20:27:09.188788+0000","last_undegraded":"2023-09-23T20:27:09.188788+0000","last_fullsized":"2023-09-23T20:27:09.188788+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:50:28.470211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":172,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T20:27:09.188278+0000","last_change":"2023-09-23T20:20:46.939938+0000","last_active":"2023-09-23T20:27:09.188278+0000","last_peered":"2023-09-23T20:27:09.188278+0000","last_clean":"2023-09-23T20:27:09.188278+0000","last_became_active":"2023-09-23T20:20:46.938777+0000","last_became_peered":"2023-09-23T20:20:46.938777+0000","last_unstale":"2023-09-23T20:27:09.188278+0000","last_undegraded":"2023-09-23T20:27:09.188278+0000","last_fullsized":"2023-09-23T20:27:09.188278+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:45:10.329438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":208,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T20:27:09.190416+0000","last_change":"2023-09-23T20:25:39.654026+0000","last_active":"2023-09-23T20:27:09.190416+0000","last_peered":"2023-09-23T20:27:09.190416+0000","last_clean":"2023-09-23T20:27:09.190416+0000","last_became_active":"2023-09-23T20:25:39.626849+0000","last_became_peered":"2023-09-23T20:25:39.626849+0000","last_unstale":"2023-09-23T20:27:09.190416+0000","last_undegraded":"2023-09-23T20:27:09.190416+0000","last_fullsized":"2023-09-23T20:27:09.190416+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:22:53.957630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":61,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T20:27:10.956269+0000","last_change":"2023-09-23T20:25:02.962047+0000","last_active":"2023-09-23T20:27:10.956269+0000","last_peered":"2023-09-23T20:27:10.956269+0000","last_clean":"2023-09-23T20:27:10.956269+0000","last_became_active":"2023-09-23T20:25:02.961523+0000","last_became_peered":"2023-09-23T20:25:02.961523+0000","last_unstale":"2023-09-23T20:27:10.956269+0000","last_undegraded":"2023-09-23T20:27:10.956269+0000","last_fullsized":"2023-09-23T20:27:10.956269+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:18:11.830822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":174,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T20:27:08.743710+0000","last_change":"2023-09-23T20:25:02.953993+0000","last_active":"2023-09-23T20:27:08.743710+0000","last_peered":"2023-09-23T20:27:08.743710+0000","last_clean":"2023-09-23T20:27:08.743710+0000","last_became_active":"2023-09-23T20:25:02.953672+0000","last_became_peered":"2023-09-23T20:25:02.953672+0000","last_unstale":"2023-09-23T20:27:08.743710+0000","last_undegraded":"2023-09-23T20:27:08.743710+0000","last_fullsized":"2023-09-23T20:27:08.743710+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:26:08.603577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":192,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T20:27:09.195758+0000","last_change":"2023-09-23T20:25:02.949584+0000","last_active":"2023-09-23T20:27:09.195758+0000","last_peered":"2023-09-23T20:27:09.195758+0000","last_clean":"2023-09-23T20:27:09.195758+0000","last_became_active":"2023-09-23T20:25:02.949132+0000","last_became_peered":"2023-09-23T20:25:02.949132+0000","last_unstale":"2023-09-23T20:27:09.195758+0000","last_undegraded":"2023-09-23T20:27:09.195758+0000","last_fullsized":"2023-09-23T20:27:09.195758+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:34:20.509823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":157,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T20:27:09.192736+0000","last_change":"2023-09-23T20:12:43.749139+0000","last_active":"2023-09-23T20:27:09.192736+0000","last_peered":"2023-09-23T20:27:09.192736+0000","last_clean":"2023-09-23T20:27:09.192736+0000","last_became_active":"2023-09-23T20:12:43.748854+0000","last_became_peered":"2023-09-23T20:12:43.748854+0000","last_unstale":"2023-09-23T20:27:09.192736+0000","last_undegraded":"2023-09-23T20:27:09.192736+0000","last_fullsized":"2023-09-23T20:27:09.192736+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:57:11.630712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":184,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T20:27:08.746821+0000","last_change":"2023-09-23T20:25:02.957010+0000","last_active":"2023-09-23T20:27:08.746821+0000","last_peered":"2023-09-23T20:27:08.746821+0000","last_clean":"2023-09-23T20:27:08.746821+0000","last_became_active":"2023-09-23T20:25:02.956267+0000","last_became_peered":"2023-09-23T20:25:02.956267+0000","last_unstale":"2023-09-23T20:27:08.746821+0000","last_undegraded":"2023-09-23T20:27:08.746821+0000","last_fullsized":"2023-09-23T20:27:08.746821+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:10:55.299422+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":182,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T20:27:09.190605+0000","last_change":"2023-09-23T20:20:14.881009+0000","last_active":"2023-09-23T20:27:09.190605+0000","last_peered":"2023-09-23T20:27:09.190605+0000","last_clean":"2023-09-23T20:27:09.190605+0000","last_became_active":"2023-09-23T20:20:14.880685+0000","last_became_peered":"2023-09-23T20:20:14.880685+0000","last_unstale":"2023-09-23T20:27:09.190605+0000","last_undegraded":"2023-09-23T20:27:09.190605+0000","last_fullsized":"2023-09-23T20:27:09.190605+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:28.010434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":61,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T20:27:10.956598+0000","last_change":"2023-09-23T20:25:02.977462+0000","last_active":"2023-09-23T20:27:10.956598+0000","last_peered":"2023-09-23T20:27:10.956598+0000","last_clean":"2023-09-23T20:27:10.956598+0000","last_became_active":"2023-09-23T20:25:02.969910+0000","last_became_peered":"2023-09-23T20:25:02.969910+0000","last_unstale":"2023-09-23T20:27:10.956598+0000","last_undegraded":"2023-09-23T20:27:10.956598+0000","last_fullsized":"2023-09-23T20:27:10.956598+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:22:55.644955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.a","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T20:27:13.250180+0000","last_change":"2023-09-23T20:27:13.250180+0000","last_active":"2023-09-23T20:27:13.250180+0000","last_peered":"2023-09-23T20:27:13.250180+0000","last_clean":"2023-09-23T20:27:13.250180+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T20:27:13.250180+0000","last_undegraded":"2023-09-23T20:27:13.250180+0000","last_fullsized":"2023-09-23T20:27:13.250180+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":175,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T20:27:09.191433+0000","last_change":"2023-09-23T20:20:14.883283+0000","last_active":"2023-09-23T20:27:09.191433+0000","last_peered":"2023-09-23T20:27:09.191433+0000","last_clean":"2023-09-23T20:27:09.191433+0000","last_became_active":"2023-09-23T20:20:14.882806+0000","last_became_peered":"2023-09-23T20:20:14.882806+0000","last_unstale":"2023-09-23T20:27:09.191433+0000","last_undegraded":"2023-09-23T20:27:09.191433+0000","last_fullsized":"2023-09-23T20:27:09.191433+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:23:35.295701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":155,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T20:27:09.195792+0000","last_change":"2023-09-23T20:12:45.179211+0000","last_active":"2023-09-23T20:27:09.195792+0000","last_peered":"2023-09-23T20:27:09.195792+0000","last_clean":"2023-09-23T20:27:09.195792+0000","last_became_active":"2023-09-23T20:12:45.178966+0000","last_became_peered":"2023-09-23T20:12:45.178966+0000","last_unstale":"2023-09-23T20:27:09.195792+0000","last_undegraded":"2023-09-23T20:27:09.195792+0000","last_fullsized":"2023-09-23T20:27:09.195792+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:58.902140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T20:27:13.250180+0000","last_change":"2023-09-23T20:27:13.250180+0000","last_active":"2023-09-23T20:27:13.250180+0000","last_peered":"2023-09-23T20:27:13.250180+0000","last_clean":"2023-09-23T20:27:13.250180+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T20:27:13.250180+0000","last_undegraded":"2023-09-23T20:27:13.250180+0000","last_fullsized":"2023-09-23T20:27:13.250180+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":176,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T20:27:09.191355+0000","last_change":"2023-09-23T20:20:46.950784+0000","last_active":"2023-09-23T20:27:09.191355+0000","last_peered":"2023-09-23T20:27:09.191355+0000","last_clean":"2023-09-23T20:27:09.191355+0000","last_became_active":"2023-09-23T20:20:46.950055+0000","last_became_peered":"2023-09-23T20:20:46.950055+0000","last_unstale":"2023-09-23T20:27:09.191355+0000","last_undegraded":"2023-09-23T20:27:09.191355+0000","last_fullsized":"2023-09-23T20:27:09.191355+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:23:07.645427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":209,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T20:27:09.189154+0000","last_change":"2023-09-23T20:25:02.943743+0000","last_active":"2023-09-23T20:27:09.189154+0000","last_peered":"2023-09-23T20:27:09.189154+0000","last_clean":"2023-09-23T20:27:09.189154+0000","last_became_active":"2023-09-23T20:25:02.943279+0000","last_became_peered":"2023-09-23T20:25:02.943279+0000","last_unstale":"2023-09-23T20:27:09.189154+0000","last_undegraded":"2023-09-23T20:27:09.189154+0000","last_fullsized":"2023-09-23T20:27:09.189154+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:21:36.128366+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T20:27:13.250180+0000","last_change":"2023-09-23T20:27:13.250180+0000","last_active":"2023-09-23T20:27:13.250180+0000","last_peered":"2023-09-23T20:27:13.250180+0000","last_clean":"2023-09-23T20:27:13.250180+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T20:27:13.250180+0000","last_undegraded":"2023-09-23T20:27:13.250180+0000","last_fullsized":"2023-09-23T20:27:13.250180+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":158,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T20:27:09.189513+0000","last_change":"2023-09-23T20:12:43.751523+0000","last_active":"2023-09-23T20:27:09.189513+0000","last_peered":"2023-09-23T20:27:09.189513+0000","last_clean":"2023-09-23T20:27:09.189513+0000","last_became_active":"2023-09-23T20:12:43.751257+0000","last_became_peered":"2023-09-23T20:12:43.751257+0000","last_unstale":"2023-09-23T20:27:09.189513+0000","last_undegraded":"2023-09-23T20:27:09.189513+0000","last_fullsized":"2023-09-23T20:27:09.189513+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:50:27.040225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":173,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T20:27:09.191163+0000","last_change":"2023-09-23T20:20:46.934762+0000","last_active":"2023-09-23T20:27:09.191163+0000","last_peered":"2023-09-23T20:27:09.191163+0000","last_clean":"2023-09-23T20:27:09.191163+0000","last_became_active":"2023-09-23T20:20:46.934472+0000","last_became_peered":"2023-09-23T20:20:46.934472+0000","last_unstale":"2023-09-23T20:27:09.191163+0000","last_undegraded":"2023-09-23T20:27:09.191163+0000","last_fullsized":"2023-09-23T20:27:09.191163+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:28:42.708286+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T20:27:13.250180+0000","last_change":"2023-09-23T20:27:13.250180+0000","last_active":"2023-09-23T20:27:13.250180+0000","last_peered":"2023-09-23T20:27:13.250180+0000","last_clean":"2023-09-23T20:27:13.250180+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T20:27:13.250180+0000","last_undegraded":"2023-09-23T20:27:13.250180+0000","last_fullsized":"2023-09-23T20:27:13.250180+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":193,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T20:27:09.195469+0000","last_change":"2023-09-23T20:25:02.947631+0000","last_active":"2023-09-23T20:27:09.195469+0000","last_peered":"2023-09-23T20:27:09.195469+0000","last_clean":"2023-09-23T20:27:09.195469+0000","last_became_active":"2023-09-23T20:25:02.947268+0000","last_became_peered":"2023-09-23T20:25:02.947268+0000","last_unstale":"2023-09-23T20:27:09.195469+0000","last_undegraded":"2023-09-23T20:27:09.195469+0000","last_fullsized":"2023-09-23T20:27:09.195469+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:19:43.643985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":158,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T20:27:08.746419+0000","last_change":"2023-09-23T20:20:46.935968+0000","last_active":"2023-09-23T20:27:08.746419+0000","last_peered":"2023-09-23T20:27:08.746419+0000","last_clean":"2023-09-23T20:27:08.746419+0000","last_became_active":"2023-09-23T20:20:46.935618+0000","last_became_peered":"2023-09-23T20:20:46.935618+0000","last_unstale":"2023-09-23T20:27:08.746419+0000","last_undegraded":"2023-09-23T20:27:08.746419+0000","last_fullsized":"2023-09-23T20:27:08.746419+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:42:59.867176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T20:27:13.250180+0000","last_change":"2023-09-23T20:27:13.250180+0000","last_active":"2023-09-23T20:27:13.250180+0000","last_peered":"2023-09-23T20:27:13.250180+0000","last_clean":"2023-09-23T20:27:13.250180+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T20:27:13.250180+0000","last_undegraded":"2023-09-23T20:27:13.250180+0000","last_fullsized":"2023-09-23T20:27:13.250180+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":576,"reported_epoch":214,"state":"active+undersized+remapped","last_fresh":"2023-09-23T20:27:10.297177+0000","last_change":"2023-09-23T20:27:10.297098+0000","last_active":"2023-09-23T20:27:10.297177+0000","last_peered":"2023-09-23T20:27:10.297177+0000","last_clean":"2023-09-23T20:27:07.046103+0000","last_became_active":"2023-09-23T20:27:10.297098+0000","last_became_peered":"2023-09-23T20:27:10.297098+0000","last_unstale":"2023-09-23T20:27:10.297177+0000","last_undegraded":"2023-09-23T20:27:10.297177+0000","last_fullsized":"2023-09-23T20:27:10.195068+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:09:47.213130+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,2],"avail_no_missing":["7","0","2","3","6"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":65,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T20:27:10.956106+0000","last_change":"2023-09-23T20:25:03.000957+0000","last_active":"2023-09-23T20:27:10.956106+0000","last_peered":"2023-09-23T20:27:10.956106+0000","last_clean":"2023-09-23T20:27:10.956106+0000","last_became_active":"2023-09-23T20:25:02.939903+0000","last_became_peered":"2023-09-23T20:25:02.939903+0000","last_unstale":"2023-09-23T20:27:10.956106+0000","last_undegraded":"2023-09-23T20:27:10.956106+0000","last_fullsized":"2023-09-23T20:27:10.956106+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:51:14.981461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":192,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T20:27:09.188687+0000","last_change":"2023-09-23T20:25:02.929447+0000","last_active":"2023-09-23T20:27:09.188687+0000","last_peered":"2023-09-23T20:27:09.188687+0000","last_clean":"2023-09-23T20:27:09.188687+0000","last_became_active":"2023-09-23T20:25:02.929173+0000","last_became_peered":"2023-09-23T20:25:02.929173+0000","last_unstale":"2023-09-23T20:27:09.188687+0000","last_undegraded":"2023-09-23T20:27:09.188687+0000","last_fullsized":"2023-09-23T20:27:09.188687+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:32:09.280647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T20:27:13.250180+0000","last_change":"2023-09-23T20:27:13.250180+0000","last_active":"2023-09-23T20:27:13.250180+0000","last_peered":"2023-09-23T20:27:13.250180+0000","last_clean":"2023-09-23T20:27:13.250180+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T20:27:13.250180+0000","last_undegraded":"2023-09-23T20:27:13.250180+0000","last_fullsized":"2023-09-23T20:27:13.250180+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":416,"reported_epoch":213,"state":"remapped+peering","last_fresh":"2023-09-23T20:27:09.188873+0000","last_change":"2023-09-23T20:27:09.188873+0000","last_active":"2023-09-23T20:27:09.188873+0000","last_peered":"2023-09-23T20:15:59.452160+0000","last_clean":"2023-09-23T20:15:59.452160+0000","last_became_active":"2023-09-23T20:11:27.848678+0000","last_became_peered":"2023-09-23T20:11:27.848678+0000","last_unstale":"2023-09-23T20:27:09.188873+0000","last_undegraded":"2023-09-23T20:27:09.188873+0000","last_fullsized":"2023-09-23T20:27:09.188873+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:09:47.213130+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":174,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T20:27:09.191011+0000","last_change":"2023-09-23T20:20:46.937163+0000","last_active":"2023-09-23T20:27:09.191011+0000","last_peered":"2023-09-23T20:27:09.191011+0000","last_clean":"2023-09-23T20:27:09.191011+0000","last_became_active":"2023-09-23T20:20:46.936757+0000","last_became_peered":"2023-09-23T20:20:46.936757+0000","last_unstale":"2023-09-23T20:27:09.191011+0000","last_undegraded":"2023-09-23T20:27:09.191011+0000","last_fullsized":"2023-09-23T20:27:09.191011+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:09.401379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":202,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T20:27:09.188797+0000","last_change":"2023-09-23T20:25:39.624636+0000","last_active":"2023-09-23T20:27:09.188797+0000","last_peered":"2023-09-23T20:27:09.188797+0000","last_clean":"2023-09-23T20:27:09.188797+0000","last_became_active":"2023-09-23T20:25:39.624297+0000","last_became_peered":"2023-09-23T20:25:39.624297+0000","last_unstale":"2023-09-23T20:27:09.188797+0000","last_undegraded":"2023-09-23T20:27:09.188797+0000","last_fullsized":"2023-09-23T20:27:09.188797+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:33:47.448737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T20:27:13.250180+0000","last_change":"2023-09-23T20:27:13.250180+0000","last_active":"2023-09-23T20:27:13.250180+0000","last_peered":"2023-09-23T20:27:13.250180+0000","last_clean":"2023-09-23T20:27:13.250180+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T20:27:13.250180+0000","last_undegraded":"2023-09-23T20:27:13.250180+0000","last_fullsized":"2023-09-23T20:27:13.250180+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":193,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T20:27:09.188934+0000","last_change":"2023-09-23T20:25:39.628097+0000","last_active":"2023-09-23T20:27:09.188934+0000","last_peered":"2023-09-23T20:27:09.188934+0000","last_clean":"2023-09-23T20:27:09.188934+0000","last_became_active":"2023-09-23T20:25:39.627806+0000","last_became_peered":"2023-09-23T20:25:39.627806+0000","last_unstale":"2023-09-23T20:27:09.188934+0000","last_undegraded":"2023-09-23T20:27:09.188934+0000","last_fullsized":"2023-09-23T20:27:09.188934+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:53:44.398854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":172,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T20:27:09.189921+0000","last_change":"2023-09-23T20:20:46.939720+0000","last_active":"2023-09-23T20:27:09.189921+0000","last_peered":"2023-09-23T20:27:09.189921+0000","last_clean":"2023-09-23T20:27:09.189921+0000","last_became_active":"2023-09-23T20:20:46.939357+0000","last_became_peered":"2023-09-23T20:20:46.939357+0000","last_unstale":"2023-09-23T20:27:09.189921+0000","last_undegraded":"2023-09-23T20:27:09.189921+0000","last_fullsized":"2023-09-23T20:27:09.189921+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:56.613212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T20:27:13.250180+0000","last_change":"2023-09-23T20:27:13.250180+0000","last_active":"2023-09-23T20:27:13.250180+0000","last_peered":"2023-09-23T20:27:13.250180+0000","last_clean":"2023-09-23T20:27:13.250180+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T20:27:13.250180+0000","last_undegraded":"2023-09-23T20:27:13.250180+0000","last_fullsized":"2023-09-23T20:27:13.250180+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":416,"reported_epoch":213,"state":"remapped+peering","last_fresh":"2023-09-23T20:27:09.190800+0000","last_change":"2023-09-23T20:27:09.190800+0000","last_active":"2023-09-23T20:27:09.190800+0000","last_peered":"2023-09-23T20:15:59.452160+0000","last_clean":"2023-09-23T20:15:59.452160+0000","last_became_active":"2023-09-23T20:11:27.848678+0000","last_became_peered":"2023-09-23T20:11:27.848678+0000","last_unstale":"2023-09-23T20:27:09.190800+0000","last_undegraded":"2023-09-23T20:27:09.190800+0000","last_fullsized":"2023-09-23T20:27:09.190800+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:09:47.213130+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":62,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-09-23T20:27:11.951993+0000","last_change":"2023-09-23T20:25:02.978173+0000","last_active":"2023-09-23T20:27:11.951993+0000","last_peered":"2023-09-23T20:27:11.951993+0000","last_clean":"2023-09-23T20:27:11.951993+0000","last_became_active":"2023-09-23T20:25:02.977856+0000","last_became_peered":"2023-09-23T20:25:02.977856+0000","last_unstale":"2023-09-23T20:27:11.951993+0000","last_undegraded":"2023-09-23T20:27:11.951993+0000","last_fullsized":"2023-09-23T20:27:11.951993+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:19.861943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":73,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T20:27:09.184865+0000","last_change":"2023-09-23T20:25:02.924911+0000","last_active":"2023-09-23T20:27:09.184865+0000","last_peered":"2023-09-23T20:27:09.184865+0000","last_clean":"2023-09-23T20:27:09.184865+0000","last_became_active":"2023-09-23T20:25:02.924586+0000","last_became_peered":"2023-09-23T20:25:02.924586+0000","last_unstale":"2023-09-23T20:27:09.184865+0000","last_undegraded":"2023-09-23T20:27:09.184865+0000","last_fullsized":"2023-09-23T20:27:09.184865+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:24:44.176766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T20:27:13.250180+0000","last_change":"2023-09-23T20:27:13.250180+0000","last_active":"2023-09-23T20:27:13.250180+0000","last_peered":"2023-09-23T20:27:13.250180+0000","last_clean":"2023-09-23T20:27:13.250180+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T20:27:13.250180+0000","last_undegraded":"2023-09-23T20:27:13.250180+0000","last_fullsized":"2023-09-23T20:27:13.250180+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":427,"reported_epoch":214,"state":"active+undersized+remapped","last_fresh":"2023-09-23T20:27:10.278220+0000","last_change":"2023-09-23T20:27:10.278123+0000","last_active":"2023-09-23T20:27:10.278220+0000","last_peered":"2023-09-23T20:27:10.278220+0000","last_clean":"2023-09-23T20:15:59.452160+0000","last_became_active":"2023-09-23T20:27:10.278123+0000","last_became_peered":"2023-09-23T20:27:10.278123+0000","last_unstale":"2023-09-23T20:27:10.278220+0000","last_undegraded":"2023-09-23T20:27:10.278220+0000","last_fullsized":"2023-09-23T20:27:10.193975+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:09:47.213130+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,7],"avail_no_missing":["6","0","3","7"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"86'119","reported_seq":547,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T20:27:08.180719+0000","last_change":"2023-09-23T20:20:46.937071+0000","last_active":"2023-09-23T20:27:08.180719+0000","last_peered":"2023-09-23T20:27:08.180719+0000","last_clean":"2023-09-23T20:27:08.180719+0000","last_became_active":"2023-09-23T20:20:46.935348+0000","last_became_peered":"2023-09-23T20:20:46.935348+0000","last_unstale":"2023-09-23T20:27:08.180719+0000","last_undegraded":"2023-09-23T20:27:08.180719+0000","last_fullsized":"2023-09-23T20:27:08.180719+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":119,"log_dups_size":0,"ondisk_log_size":119,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:09.829030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032705499999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":523,"num_read_kb":3503,"num_write":136,"num_write_kb":3128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":230,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T20:27:09.195672+0000","last_change":"2023-09-23T20:25:39.637160+0000","last_active":"2023-09-23T20:27:09.195672+0000","last_peered":"2023-09-23T20:27:09.195672+0000","last_clean":"2023-09-23T20:27:09.195672+0000","last_became_active":"2023-09-23T20:25:39.636860+0000","last_became_peered":"2023-09-23T20:25:39.636860+0000","last_unstale":"2023-09-23T20:27:09.195672+0000","last_undegraded":"2023-09-23T20:27:09.195672+0000","last_fullsized":"2023-09-23T20:27:09.195672+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:21:47.860063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":139,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T20:27:08.180272+0000","last_change":"2023-09-23T20:20:46.937104+0000","last_active":"2023-09-23T20:27:08.180272+0000","last_peered":"2023-09-23T20:27:08.180272+0000","last_clean":"2023-09-23T20:27:08.180272+0000","last_became_active":"2023-09-23T20:20:46.936820+0000","last_became_peered":"2023-09-23T20:20:46.936820+0000","last_unstale":"2023-09-23T20:27:08.180272+0000","last_undegraded":"2023-09-23T20:27:08.180272+0000","last_fullsized":"2023-09-23T20:27:08.180272+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:22:19.879310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.6","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T20:27:13.250180+0000","last_change":"2023-09-23T20:27:13.250180+0000","last_active":"2023-09-23T20:27:13.250180+0000","last_peered":"2023-09-23T20:27:13.250180+0000","last_clean":"2023-09-23T20:27:13.250180+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T20:27:13.250180+0000","last_undegraded":"2023-09-23T20:27:13.250180+0000","last_fullsized":"2023-09-23T20:27:13.250180+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":416,"reported_epoch":213,"state":"remapped+peering","last_fresh":"2023-09-23T20:27:09.191360+0000","last_change":"2023-09-23T20:27:09.191360+0000","last_active":"2023-09-23T20:27:09.191360+0000","last_peered":"2023-09-23T20:15:59.452160+0000","last_clean":"2023-09-23T20:15:59.452160+0000","last_became_active":"2023-09-23T20:11:27.848678+0000","last_became_peered":"2023-09-23T20:11:27.848678+0000","last_unstale":"2023-09-23T20:27:09.191360+0000","last_undegraded":"2023-09-23T20:27:09.191360+0000","last_fullsized":"2023-09-23T20:27:09.191360+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:09:47.213130+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":45,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"86'106","reported_seq":838,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T20:27:09.195793+0000","last_change":"2023-09-23T20:20:46.939468+0000","last_active":"2023-09-23T20:27:09.195793+0000","last_peered":"2023-09-23T20:27:09.195793+0000","last_clean":"2023-09-23T20:27:09.195793+0000","last_became_active":"2023-09-23T20:20:46.938954+0000","last_became_peered":"2023-09-23T20:20:46.938954+0000","last_unstale":"2023-09-23T20:27:09.195793+0000","last_undegraded":"2023-09-23T20:27:09.195793+0000","last_fullsized":"2023-09-23T20:27:09.195793+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":106,"log_dups_size":0,"ondisk_log_size":106,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:01:43.761064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00099302599999999993,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1105,"num_read_kb":1689,"num_write":100,"num_write_kb":404,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":173,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T20:27:09.188194+0000","last_change":"2023-09-23T20:20:14.887345+0000","last_active":"2023-09-23T20:27:09.188194+0000","last_peered":"2023-09-23T20:27:09.188194+0000","last_clean":"2023-09-23T20:27:09.188194+0000","last_became_active":"2023-09-23T20:20:14.886970+0000","last_became_peered":"2023-09-23T20:20:14.886970+0000","last_unstale":"2023-09-23T20:27:09.188194+0000","last_undegraded":"2023-09-23T20:27:09.188194+0000","last_fullsized":"2023-09-23T20:27:09.188194+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:00:38.575926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":205,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T20:27:08.180639+0000","last_change":"2023-09-23T20:25:39.631133+0000","last_active":"2023-09-23T20:27:08.180639+0000","last_peered":"2023-09-23T20:27:08.180639+0000","last_clean":"2023-09-23T20:27:08.180639+0000","last_became_active":"2023-09-23T20:25:39.630779+0000","last_became_peered":"2023-09-23T20:25:39.630779+0000","last_unstale":"2023-09-23T20:27:08.180639+0000","last_undegraded":"2023-09-23T20:27:08.180639+0000","last_fullsized":"2023-09-23T20:27:08.180639+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:39:52.635649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.5","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T20:27:13.250180+0000","last_change":"2023-09-23T20:27:13.250180+0000","last_active":"2023-09-23T20:27:13.250180+0000","last_peered":"2023-09-23T20:27:13.250180+0000","last_clean":"2023-09-23T20:27:13.250180+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T20:27:13.250180+0000","last_undegraded":"2023-09-23T20:27:13.250180+0000","last_fullsized":"2023-09-23T20:27:13.250180+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":576,"reported_epoch":214,"state":"active+undersized+remapped","last_fresh":"2023-09-23T20:27:10.294526+0000","last_change":"2023-09-23T20:27:10.294464+0000","last_active":"2023-09-23T20:27:10.294526+0000","last_peered":"2023-09-23T20:27:10.294526+0000","last_clean":"2023-09-23T20:27:07.049042+0000","last_became_active":"2023-09-23T20:27:10.294464+0000","last_became_peered":"2023-09-23T20:27:10.294464+0000","last_unstale":"2023-09-23T20:27:10.294526+0000","last_undegraded":"2023-09-23T20:27:10.294526+0000","last_fullsized":"2023-09-23T20:27:10.195623+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:09:47.213130+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0],"avail_no_missing":["7","0","4","6"],"object_location_counts":[{"shards":"0,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"86'130","reported_seq":525,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T20:27:10.957675+0000","last_change":"2023-09-23T20:25:39.637458+0000","last_active":"2023-09-23T20:27:10.957675+0000","last_peered":"2023-09-23T20:27:10.957675+0000","last_clean":"2023-09-23T20:27:10.957675+0000","last_became_active":"2023-09-23T20:25:39.636773+0000","last_became_peered":"2023-09-23T20:25:39.636773+0000","last_unstale":"2023-09-23T20:27:10.957675+0000","last_undegraded":"2023-09-23T20:27:10.957675+0000","last_fullsized":"2023-09-23T20:27:10.957675+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:00:32.810564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011300049999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":512,"num_read_kb":1820,"num_write":133,"num_write_kb":1347,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":171,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T20:27:08.179999+0000","last_change":"2023-09-23T20:25:39.632283+0000","last_active":"2023-09-23T20:27:08.179999+0000","last_peered":"2023-09-23T20:27:08.179999+0000","last_clean":"2023-09-23T20:27:08.179999+0000","last_became_active":"2023-09-23T20:25:39.631819+0000","last_became_peered":"2023-09-23T20:25:39.631819+0000","last_unstale":"2023-09-23T20:27:08.179999+0000","last_undegraded":"2023-09-23T20:27:08.179999+0000","last_fullsized":"2023-09-23T20:27:08.179999+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:25:21.815555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":181,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T20:27:08.179308+0000","last_change":"2023-09-23T20:20:46.936524+0000","last_active":"2023-09-23T20:27:08.179308+0000","last_peered":"2023-09-23T20:27:08.179308+0000","last_clean":"2023-09-23T20:27:08.179308+0000","last_became_active":"2023-09-23T20:20:46.936267+0000","last_became_peered":"2023-09-23T20:20:46.936267+0000","last_unstale":"2023-09-23T20:27:08.179308+0000","last_undegraded":"2023-09-23T20:27:08.179308+0000","last_fullsized":"2023-09-23T20:27:08.179308+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:32:20.984521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T20:27:13.250180+0000","last_change":"2023-09-23T20:27:13.250180+0000","last_active":"2023-09-23T20:27:13.250180+0000","last_peered":"2023-09-23T20:27:13.250180+0000","last_clean":"2023-09-23T20:27:13.250180+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T20:27:13.250180+0000","last_undegraded":"2023-09-23T20:27:13.250180+0000","last_fullsized":"2023-09-23T20:27:13.250180+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.3","version":"86'194","reported_seq":862,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T20:27:09.195917+0000","last_change":"2023-09-23T20:25:40.618665+0000","last_active":"2023-09-23T20:27:09.195917+0000","last_peered":"2023-09-23T20:27:09.195917+0000","last_clean":"2023-09-23T20:27:09.195917+0000","last_became_active":"2023-09-23T20:25:40.618087+0000","last_became_peered":"2023-09-23T20:25:40.618087+0000","last_unstale":"2023-09-23T20:27:09.195917+0000","last_undegraded":"2023-09-23T20:27:09.195917+0000","last_fullsized":"2023-09-23T20:27:09.195917+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":194,"log_dups_size":0,"ondisk_log_size":194,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:07:16.114968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00037995999999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1025,"num_read_kb":1201,"num_write":173,"num_write_kb":1276,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":570,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T20:27:09.190228+0000","last_change":"2023-09-23T20:27:04.493422+0000","last_active":"2023-09-23T20:27:09.190228+0000","last_peered":"2023-09-23T20:27:09.190228+0000","last_clean":"2023-09-23T20:27:09.190228+0000","last_became_active":"2023-09-23T20:25:21.159200+0000","last_became_peered":"2023-09-23T20:25:21.159200+0000","last_unstale":"2023-09-23T20:27:09.190228+0000","last_undegraded":"2023-09-23T20:27:09.190228+0000","last_fullsized":"2023-09-23T20:27:09.190228+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:09:47.213130+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":62,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T20:27:10.955368+0000","last_change":"2023-09-23T20:25:02.984205+0000","last_active":"2023-09-23T20:27:10.955368+0000","last_peered":"2023-09-23T20:27:10.955368+0000","last_clean":"2023-09-23T20:27:10.955368+0000","last_became_active":"2023-09-23T20:25:02.983758+0000","last_became_peered":"2023-09-23T20:25:02.983758+0000","last_unstale":"2023-09-23T20:27:10.955368+0000","last_undegraded":"2023-09-23T20:27:10.955368+0000","last_fullsized":"2023-09-23T20:27:10.955368+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:35:14.573101+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":156,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T20:27:09.188878+0000","last_change":"2023-09-23T20:12:45.609829+0000","last_active":"2023-09-23T20:27:09.188878+0000","last_peered":"2023-09-23T20:27:09.188878+0000","last_clean":"2023-09-23T20:27:09.188878+0000","last_became_active":"2023-09-23T20:12:45.609467+0000","last_became_peered":"2023-09-23T20:12:45.609467+0000","last_unstale":"2023-09-23T20:27:09.188878+0000","last_undegraded":"2023-09-23T20:27:09.188878+0000","last_fullsized":"2023-09-23T20:27:09.188878+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:27:31.490555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.4","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T20:27:13.250180+0000","last_change":"2023-09-23T20:27:13.250180+0000","last_active":"2023-09-23T20:27:13.250180+0000","last_peered":"2023-09-23T20:27:13.250180+0000","last_clean":"2023-09-23T20:27:13.250180+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T20:27:13.250180+0000","last_undegraded":"2023-09-23T20:27:13.250180+0000","last_fullsized":"2023-09-23T20:27:13.250180+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":416,"reported_epoch":213,"state":"remapped+peering","last_fresh":"2023-09-23T20:27:09.188739+0000","last_change":"2023-09-23T20:27:09.188739+0000","last_active":"2023-09-23T20:27:09.188739+0000","last_peered":"2023-09-23T20:15:59.452160+0000","last_clean":"2023-09-23T20:15:59.452160+0000","last_became_active":"2023-09-23T20:11:27.848678+0000","last_became_peered":"2023-09-23T20:11:27.848678+0000","last_unstale":"2023-09-23T20:27:09.188739+0000","last_undegraded":"2023-09-23T20:27:09.188739+0000","last_fullsized":"2023-09-23T20:27:09.188739+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:09:47.213130+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"86'80","reported_seq":460,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T20:27:09.190134+0000","last_change":"2023-09-23T20:20:46.950784+0000","last_active":"2023-09-23T20:27:09.190134+0000","last_peered":"2023-09-23T20:27:09.190134+0000","last_clean":"2023-09-23T20:27:09.190134+0000","last_became_active":"2023-09-23T20:20:46.950071+0000","last_became_peered":"2023-09-23T20:20:46.950071+0000","last_unstale":"2023-09-23T20:27:09.190134+0000","last_undegraded":"2023-09-23T20:27:09.190134+0000","last_fullsized":"2023-09-23T20:27:09.190134+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":80,"log_dups_size":0,"ondisk_log_size":80,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:11:06.812404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035115,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":395,"num_read_kb":1370,"num_write":66,"num_write_kb":1061,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":175,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T20:27:08.743435+0000","last_change":"2023-09-23T20:25:02.942709+0000","last_active":"2023-09-23T20:27:08.743435+0000","last_peered":"2023-09-23T20:27:08.743435+0000","last_clean":"2023-09-23T20:27:08.743435+0000","last_became_active":"2023-09-23T20:25:02.942323+0000","last_became_peered":"2023-09-23T20:25:02.942323+0000","last_unstale":"2023-09-23T20:27:08.743435+0000","last_undegraded":"2023-09-23T20:27:08.743435+0000","last_fullsized":"2023-09-23T20:27:08.743435+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:26:12.039101+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":191,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T20:27:08.747040+0000","last_change":"2023-09-23T20:25:39.641452+0000","last_active":"2023-09-23T20:27:08.747040+0000","last_peered":"2023-09-23T20:27:08.747040+0000","last_clean":"2023-09-23T20:27:08.747040+0000","last_became_active":"2023-09-23T20:25:39.638987+0000","last_became_peered":"2023-09-23T20:25:39.638987+0000","last_unstale":"2023-09-23T20:27:08.747040+0000","last_undegraded":"2023-09-23T20:27:08.747040+0000","last_fullsized":"2023-09-23T20:27:08.747040+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:33:59.955461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T20:27:13.250180+0000","last_change":"2023-09-23T20:27:13.250180+0000","last_active":"2023-09-23T20:27:13.250180+0000","last_peered":"2023-09-23T20:27:13.250180+0000","last_clean":"2023-09-23T20:27:13.250180+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T20:27:13.250180+0000","last_undegraded":"2023-09-23T20:27:13.250180+0000","last_fullsized":"2023-09-23T20:27:13.250180+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":427,"reported_epoch":214,"state":"active+undersized+remapped","last_fresh":"2023-09-23T20:27:10.278316+0000","last_change":"2023-09-23T20:27:10.278210+0000","last_active":"2023-09-23T20:27:10.278316+0000","last_peered":"2023-09-23T20:27:10.278316+0000","last_clean":"2023-09-23T20:15:59.452160+0000","last_became_active":"2023-09-23T20:27:10.278210+0000","last_became_peered":"2023-09-23T20:27:10.278210+0000","last_unstale":"2023-09-23T20:27:10.278316+0000","last_undegraded":"2023-09-23T20:27:10.278316+0000","last_fullsized":"2023-09-23T20:27:10.193508+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:09:47.213130+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0],"avail_no_missing":["6","0","3","7"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"86'102","reported_seq":720,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T20:27:09.189608+0000","last_change":"2023-09-23T20:20:14.881975+0000","last_active":"2023-09-23T20:27:09.189608+0000","last_peered":"2023-09-23T20:27:09.189608+0000","last_clean":"2023-09-23T20:27:09.189608+0000","last_became_active":"2023-09-23T20:20:14.881544+0000","last_became_peered":"2023-09-23T20:20:14.881544+0000","last_unstale":"2023-09-23T20:27:09.189608+0000","last_undegraded":"2023-09-23T20:27:09.189608+0000","last_fullsized":"2023-09-23T20:27:09.189608+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":102,"log_dups_size":0,"ondisk_log_size":102,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:34.594972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030656500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":963,"num_read_kb":1234,"num_write":60,"num_write_kb":225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":171,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T20:27:08.180807+0000","last_change":"2023-09-23T20:25:39.631751+0000","last_active":"2023-09-23T20:27:08.180807+0000","last_peered":"2023-09-23T20:27:08.180807+0000","last_clean":"2023-09-23T20:27:08.180807+0000","last_became_active":"2023-09-23T20:25:39.631136+0000","last_became_peered":"2023-09-23T20:25:39.631136+0000","last_unstale":"2023-09-23T20:27:08.180807+0000","last_undegraded":"2023-09-23T20:27:08.180807+0000","last_fullsized":"2023-09-23T20:27:08.180807+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:08:17.020722+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":143,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T20:27:08.746680+0000","last_change":"2023-09-23T20:20:14.888874+0000","last_active":"2023-09-23T20:27:08.746680+0000","last_peered":"2023-09-23T20:27:08.746680+0000","last_clean":"2023-09-23T20:27:08.746680+0000","last_became_active":"2023-09-23T20:20:14.887724+0000","last_became_peered":"2023-09-23T20:20:14.887724+0000","last_unstale":"2023-09-23T20:27:08.746680+0000","last_undegraded":"2023-09-23T20:27:08.746680+0000","last_fullsized":"2023-09-23T20:27:08.746680+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:09:24.510326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T20:27:13.250180+0000","last_change":"2023-09-23T20:27:13.250180+0000","last_active":"2023-09-23T20:27:13.250180+0000","last_peered":"2023-09-23T20:27:13.250180+0000","last_clean":"2023-09-23T20:27:13.250180+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T20:27:13.250180+0000","last_undegraded":"2023-09-23T20:27:13.250180+0000","last_fullsized":"2023-09-23T20:27:13.250180+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":416,"reported_epoch":213,"state":"remapped+peering","last_fresh":"2023-09-23T20:27:09.192397+0000","last_change":"2023-09-23T20:27:09.192397+0000","last_active":"2023-09-23T20:27:09.192397+0000","last_peered":"2023-09-23T20:15:59.452160+0000","last_clean":"2023-09-23T20:15:59.452160+0000","last_became_active":"2023-09-23T20:11:27.848678+0000","last_became_peered":"2023-09-23T20:11:27.848678+0000","last_unstale":"2023-09-23T20:27:09.192397+0000","last_undegraded":"2023-09-23T20:27:09.192397+0000","last_fullsized":"2023-09-23T20:27:09.192397+0000","mapping_epoch":213,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:09:47.213130+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":66,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-09-23T20:27:11.953279+0000","last_change":"2023-09-23T20:25:03.007736+0000","last_active":"2023-09-23T20:27:11.953279+0000","last_peered":"2023-09-23T20:27:11.953279+0000","last_clean":"2023-09-23T20:27:11.953279+0000","last_became_active":"2023-09-23T20:25:02.973689+0000","last_became_peered":"2023-09-23T20:25:02.973689+0000","last_unstale":"2023-09-23T20:27:11.953279+0000","last_undegraded":"2023-09-23T20:27:11.953279+0000","last_fullsized":"2023-09-23T20:27:11.953279+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:37:25.643774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"86'128","reported_seq":537,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T20:27:08.179637+0000","last_change":"2023-09-23T20:20:46.939256+0000","last_active":"2023-09-23T20:27:08.179637+0000","last_peered":"2023-09-23T20:27:08.179637+0000","last_clean":"2023-09-23T20:27:08.179637+0000","last_became_active":"2023-09-23T20:20:46.938756+0000","last_became_peered":"2023-09-23T20:20:46.938756+0000","last_unstale":"2023-09-23T20:27:08.179637+0000","last_undegraded":"2023-09-23T20:27:08.179637+0000","last_fullsized":"2023-09-23T20:27:08.179637+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:52:18.161436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000293202,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":579,"num_read_kb":1473,"num_write":96,"num_write_kb":1062,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":62,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T20:27:10.954765+0000","last_change":"2023-09-23T20:25:02.971740+0000","last_active":"2023-09-23T20:27:10.954765+0000","last_peered":"2023-09-23T20:27:10.954765+0000","last_clean":"2023-09-23T20:27:10.954765+0000","last_became_active":"2023-09-23T20:25:02.970412+0000","last_became_peered":"2023-09-23T20:25:02.970412+0000","last_unstale":"2023-09-23T20:27:10.954765+0000","last_undegraded":"2023-09-23T20:27:10.954765+0000","last_fullsized":"2023-09-23T20:27:10.954765+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:53:17.247216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T20:27:13.250180+0000","last_change":"2023-09-23T20:27:13.250180+0000","last_active":"2023-09-23T20:27:13.250180+0000","last_peered":"2023-09-23T20:27:13.250180+0000","last_clean":"2023-09-23T20:27:13.250180+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T20:27:13.250180+0000","last_undegraded":"2023-09-23T20:27:13.250180+0000","last_fullsized":"2023-09-23T20:27:13.250180+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.4","version":"90'161","reported_seq":429,"reported_epoch":216,"state":"active+clean","last_fresh":"2023-09-23T20:27:11.961197+0000","last_change":"2023-09-23T20:27:11.960817+0000","last_active":"2023-09-23T20:27:11.961197+0000","last_peered":"2023-09-23T20:27:11.961197+0000","last_clean":"2023-09-23T20:27:11.961197+0000","last_became_active":"2023-09-23T20:27:11.960349+0000","last_became_peered":"2023-09-23T20:27:11.960349+0000","last_unstale":"2023-09-23T20:27:11.961197+0000","last_undegraded":"2023-09-23T20:27:11.961197+0000","last_fullsized":"2023-09-23T20:27:11.961197+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:09:47.213130+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":488690,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"86'117","reported_seq":733,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T20:27:09.190061+0000","last_change":"2023-09-23T20:25:40.617386+0000","last_active":"2023-09-23T20:27:09.190061+0000","last_peered":"2023-09-23T20:27:09.190061+0000","last_clean":"2023-09-23T20:27:09.190061+0000","last_became_active":"2023-09-23T20:25:40.616747+0000","last_became_peered":"2023-09-23T20:25:40.616747+0000","last_unstale":"2023-09-23T20:27:09.190061+0000","last_undegraded":"2023-09-23T20:27:09.190061+0000","last_fullsized":"2023-09-23T20:27:09.190061+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:46:44.551561+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046891400000000002,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":893,"num_read_kb":922,"num_write":82,"num_write_kb":4193,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":163,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T20:27:08.747043+0000","last_change":"2023-09-23T20:20:46.937163+0000","last_active":"2023-09-23T20:27:08.747043+0000","last_peered":"2023-09-23T20:27:08.747043+0000","last_clean":"2023-09-23T20:27:08.747043+0000","last_became_active":"2023-09-23T20:20:46.936846+0000","last_became_peered":"2023-09-23T20:20:46.936846+0000","last_unstale":"2023-09-23T20:27:08.747043+0000","last_undegraded":"2023-09-23T20:27:08.747043+0000","last_fullsized":"2023-09-23T20:27:08.747043+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:01:32.798855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":155,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T20:27:09.189016+0000","last_change":"2023-09-23T20:12:43.765908+0000","last_active":"2023-09-23T20:27:09.189016+0000","last_peered":"2023-09-23T20:27:09.189016+0000","last_clean":"2023-09-23T20:27:09.189016+0000","last_became_active":"2023-09-23T20:12:43.765558+0000","last_became_peered":"2023-09-23T20:12:43.765558+0000","last_unstale":"2023-09-23T20:27:09.189016+0000","last_undegraded":"2023-09-23T20:27:09.189016+0000","last_fullsized":"2023-09-23T20:27:09.189016+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:35:43.809898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":145,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T20:27:08.746680+0000","last_change":"2023-09-23T20:20:14.888484+0000","last_active":"2023-09-23T20:27:08.746680+0000","last_peered":"2023-09-23T20:27:08.746680+0000","last_clean":"2023-09-23T20:27:08.746680+0000","last_became_active":"2023-09-23T20:20:14.887219+0000","last_became_peered":"2023-09-23T20:20:14.887219+0000","last_unstale":"2023-09-23T20:27:08.746680+0000","last_undegraded":"2023-09-23T20:27:08.746680+0000","last_fullsized":"2023-09-23T20:27:08.746680+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:13:53.763649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":50,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T20:27:10.956863+0000","last_change":"2023-09-23T20:25:39.639268+0000","last_active":"2023-09-23T20:27:10.956863+0000","last_peered":"2023-09-23T20:27:10.956863+0000","last_clean":"2023-09-23T20:27:10.956863+0000","last_became_active":"2023-09-23T20:25:39.638922+0000","last_became_peered":"2023-09-23T20:25:39.638922+0000","last_unstale":"2023-09-23T20:27:10.956863+0000","last_undegraded":"2023-09-23T20:27:10.956863+0000","last_fullsized":"2023-09-23T20:27:10.956863+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:09:04.040098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":175,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T20:27:09.195626+0000","last_change":"2023-09-23T20:20:46.938981+0000","last_active":"2023-09-23T20:27:09.195626+0000","last_peered":"2023-09-23T20:27:09.195626+0000","last_clean":"2023-09-23T20:27:09.195626+0000","last_became_active":"2023-09-23T20:20:46.938627+0000","last_became_peered":"2023-09-23T20:20:46.938627+0000","last_unstale":"2023-09-23T20:27:09.195626+0000","last_undegraded":"2023-09-23T20:27:09.195626+0000","last_fullsized":"2023-09-23T20:27:09.195626+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:44:43.177086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":219,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T20:27:09.188445+0000","last_change":"2023-09-23T20:25:02.945142+0000","last_active":"2023-09-23T20:27:09.188445+0000","last_peered":"2023-09-23T20:27:09.188445+0000","last_clean":"2023-09-23T20:27:09.188445+0000","last_became_active":"2023-09-23T20:25:02.944653+0000","last_became_peered":"2023-09-23T20:25:02.944653+0000","last_unstale":"2023-09-23T20:27:09.188445+0000","last_undegraded":"2023-09-23T20:27:09.188445+0000","last_fullsized":"2023-09-23T20:27:09.188445+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:38:12.927982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":158,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T20:27:09.195505+0000","last_change":"2023-09-23T20:12:43.753383+0000","last_active":"2023-09-23T20:27:09.195505+0000","last_peered":"2023-09-23T20:27:09.195505+0000","last_clean":"2023-09-23T20:27:09.195505+0000","last_became_active":"2023-09-23T20:12:43.752921+0000","last_became_peered":"2023-09-23T20:12:43.752921+0000","last_unstale":"2023-09-23T20:27:09.195505+0000","last_undegraded":"2023-09-23T20:27:09.195505+0000","last_fullsized":"2023-09-23T20:27:09.195505+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:28:26.982998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":219,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T20:27:09.189581+0000","last_change":"2023-09-23T20:25:03.103656+0000","last_active":"2023-09-23T20:27:09.189581+0000","last_peered":"2023-09-23T20:27:09.189581+0000","last_clean":"2023-09-23T20:27:09.189581+0000","last_became_active":"2023-09-23T20:25:03.103311+0000","last_became_peered":"2023-09-23T20:25:03.103311+0000","last_unstale":"2023-09-23T20:27:09.189581+0000","last_undegraded":"2023-09-23T20:27:09.189581+0000","last_fullsized":"2023-09-23T20:27:09.189581+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:55:54.677030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":71,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T20:27:10.955866+0000","last_change":"2023-09-23T20:25:39.629869+0000","last_active":"2023-09-23T20:27:10.955866+0000","last_peered":"2023-09-23T20:27:10.955866+0000","last_clean":"2023-09-23T20:27:10.955866+0000","last_became_active":"2023-09-23T20:25:39.629543+0000","last_became_peered":"2023-09-23T20:25:39.629543+0000","last_unstale":"2023-09-23T20:27:10.955866+0000","last_undegraded":"2023-09-23T20:27:10.955866+0000","last_fullsized":"2023-09-23T20:27:10.955866+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:24:48.220679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":70,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T20:27:10.955435+0000","last_change":"2023-09-23T20:25:39.622819+0000","last_active":"2023-09-23T20:27:10.955435+0000","last_peered":"2023-09-23T20:27:10.955435+0000","last_clean":"2023-09-23T20:27:10.955435+0000","last_became_active":"2023-09-23T20:25:39.622484+0000","last_became_peered":"2023-09-23T20:25:39.622484+0000","last_unstale":"2023-09-23T20:27:10.955435+0000","last_undegraded":"2023-09-23T20:27:10.955435+0000","last_fullsized":"2023-09-23T20:27:10.955435+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:51:53.973261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":191,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T20:27:08.747349+0000","last_change":"2023-09-23T20:25:39.668882+0000","last_active":"2023-09-23T20:27:08.747349+0000","last_peered":"2023-09-23T20:27:08.747349+0000","last_clean":"2023-09-23T20:27:08.747349+0000","last_became_active":"2023-09-23T20:25:39.637128+0000","last_became_peered":"2023-09-23T20:25:39.637128+0000","last_unstale":"2023-09-23T20:27:08.747349+0000","last_undegraded":"2023-09-23T20:27:08.747349+0000","last_fullsized":"2023-09-23T20:27:08.747349+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:56.406298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":192,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T20:27:09.187929+0000","last_change":"2023-09-23T20:24:53.089249+0000","last_active":"2023-09-23T20:27:09.187929+0000","last_peered":"2023-09-23T20:27:09.187929+0000","last_clean":"2023-09-23T20:27:09.187929+0000","last_became_active":"2023-09-23T20:24:53.088952+0000","last_became_peered":"2023-09-23T20:24:53.088952+0000","last_unstale":"2023-09-23T20:27:09.187929+0000","last_undegraded":"2023-09-23T20:27:09.187929+0000","last_fullsized":"2023-09-23T20:27:09.187929+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:38:03.216069+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":15,"num_bytes_recovered":1356,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":25181,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":489,"num_read_kb":916,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":2098784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3235840,"data_stored":3214528,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1771,"ondisk_log_size":1771,"up":33,"acting":24,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6082,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201472,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":174684,"kb_used_data":60372,"kb_used_omap":0,"kb_used_meta":114304,"kb_avail":93570468,"statfs":{"total":95995035648,"available":95816159232,"internally_reserved":0,"allocated":61820928,"data_stored":61103792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":117047296},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":13,"apply_latency_ms":13,"commit_latency_ns":13000000,"apply_latency_ns":13000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:26:23 2023","interfaces":[{"interface":"back","average":{"1min":1.861,"5min":1.2150000000000001,"15min":0.78700000000000003},"min":{"1min":0.247,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":10.907,"5min":13.092000000000001,"15min":13.092000000000001},"last":0.39100000000000001},{"interface":"front","average":{"1min":3.5369999999999999,"5min":1.5940000000000001,"15min":0.91000000000000003},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":9.8610000000000007,"5min":9.8610000000000007,"15min":9.8610000000000007},"last":0.41899999999999998}]},{"osd":1,"last update":"Sat Sep 23 20:26:58 2023","interfaces":[{"interface":"back","average":{"1min":1.2929999999999999,"5min":1.6499999999999999,"15min":0.92500000000000004},"min":{"1min":0.28599999999999998,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":3.1920000000000002,"5min":17.795999999999999,"15min":17.795999999999999},"last":0.61899999999999999},{"interface":"front","average":{"1min":2.2989999999999999,"5min":1.504,"15min":0.90900000000000003},"min":{"1min":0.33400000000000002,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":6.4950000000000001,"5min":13.250999999999999,"15min":13.250999999999999},"last":0.44900000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:26:28 2023","interfaces":[{"interface":"back","average":{"1min":3.2610000000000001,"5min":1.6080000000000001,"15min":0.91500000000000004},"min":{"1min":0.39000000000000001,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":8.1270000000000007,"5min":9.8300000000000001,"15min":9.8300000000000001},"last":0.46999999999999997},{"interface":"front","average":{"1min":3.1850000000000001,"5min":1.708,"15min":0.95999999999999996},"min":{"1min":0.48099999999999998,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":6.6699999999999999,"5min":11.423999999999999,"15min":11.423999999999999},"last":0.54700000000000004}]},{"osd":3,"last update":"Sat Sep 23 20:26:58 2023","interfaces":[{"interface":"back","average":{"1min":2.4049999999999998,"5min":1.756,"15min":1.022},"min":{"1min":0.38100000000000001,"5min":0.314,"15min":0.314},"max":{"1min":6.891,"5min":6.891,"15min":6.891},"last":0.50700000000000001},{"interface":"front","average":{"1min":2.4319999999999999,"5min":1.6279999999999999,"15min":0.99199999999999999},"min":{"1min":0.374,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":6.9050000000000002,"5min":9.7840000000000007,"15min":9.7840000000000007},"last":0.53500000000000003}]},{"osd":4,"last update":"Sat Sep 23 20:26:58 2023","interfaces":[{"interface":"back","average":{"1min":1.6479999999999999,"5min":1.1619999999999999,"15min":0.71899999999999997},"min":{"1min":0.28499999999999998,"5min":0.27000000000000002,"15min":0.26700000000000002},"max":{"1min":5.3540000000000001,"5min":13.141999999999999,"15min":13.141999999999999},"last":0.48799999999999999},{"interface":"front","average":{"1min":1.5740000000000001,"5min":1.6819999999999999,"15min":0.91400000000000003},"min":{"1min":0.29499999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":5.9279999999999999,"5min":10.853,"15min":10.853},"last":0.58599999999999997}]},{"osd":5,"last update":"Sat Sep 23 20:26:58 2023","interfaces":[{"interface":"back","average":{"1min":1.9930000000000001,"5min":1.643,"15min":0.90500000000000003},"min":{"1min":0.31,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":10.362,"5min":10.563000000000001,"15min":10.563000000000001},"last":0.65900000000000003},{"interface":"front","average":{"1min":1.956,"5min":1.5669999999999999,"15min":0.91100000000000003},"min":{"1min":0.41799999999999998,"5min":0.34699999999999998,"15min":0.31},"max":{"1min":7.9029999999999996,"5min":13.292,"15min":13.292},"last":0.60499999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:26:58 2023","interfaces":[{"interface":"back","average":{"1min":1.9890000000000001,"5min":1.4179999999999999,"15min":0.86899999999999999},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":6.2210000000000001,"5min":13.185,"15min":13.185},"last":0.56599999999999995},{"interface":"front","average":{"1min":1.9199999999999999,"5min":1.4890000000000001,"15min":0.89100000000000001},"min":{"1min":0.378,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":6.4370000000000003,"5min":9.9120000000000008,"15min":9.9120000000000008},"last":0.64200000000000002}]}]},{"osd":6,"up_from":47,"seq":201863463108,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":87828,"kb_used_data":6796,"kb_used_omap":0,"kb_used_meta":81024,"kb_avail":93657324,"statfs":{"total":95995035648,"available":95905099776,"internally_reserved":0,"allocated":6959104,"data_stored":6250033,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":82968576},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":10,"apply_latency_ms":10,"commit_latency_ns":10000000,"apply_latency_ns":10000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:26:26 2023","interfaces":[{"interface":"back","average":{"1min":1.7270000000000001,"5min":1.23,"15min":0.79500000000000004},"min":{"1min":0.221,"5min":0.221,"15min":0.221},"max":{"1min":8.1940000000000008,"5min":8.1940000000000008,"15min":8.1940000000000008},"last":0.67400000000000004},{"interface":"front","average":{"1min":1.3069999999999999,"5min":1.4850000000000001,"15min":0.87},"min":{"1min":0.26300000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":4.0609999999999999,"5min":12.512,"15min":12.512},"last":0.48699999999999999}]},{"osd":1,"last update":"Sat Sep 23 20:27:02 2023","interfaces":[{"interface":"back","average":{"1min":2.6379999999999999,"5min":1.8839999999999999,"15min":1.0109999999999999},"min":{"1min":0.28899999999999998,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":8.3520000000000003,"5min":18.001000000000001,"15min":18.001000000000001},"last":0.58099999999999996},{"interface":"front","average":{"1min":2.391,"5min":1.569,"15min":0.88800000000000001},"min":{"1min":0.28699999999999998,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":8.532,"5min":13.91,"15min":13.91},"last":0.46300000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:26:26 2023","interfaces":[{"interface":"back","average":{"1min":2.153,"5min":1.385,"15min":0.85799999999999998},"min":{"1min":0.32800000000000001,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":8.0090000000000003,"5min":13.856,"15min":13.856},"last":0.54400000000000004},{"interface":"front","average":{"1min":2.0720000000000001,"5min":1.284,"15min":0.83399999999999996},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":6.8140000000000001,"5min":6.8140000000000001,"15min":6.8140000000000001},"last":0.64200000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:27:02 2023","interfaces":[{"interface":"back","average":{"1min":2.105,"5min":1.833,"15min":1.0189999999999999},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":8.2989999999999995,"5min":9.4239999999999995,"15min":9.4239999999999995},"last":0.746},{"interface":"front","average":{"1min":2.754,"5min":1.8280000000000001,"15min":1.0600000000000001},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":8.3770000000000007,"5min":8.6050000000000004,"15min":8.6050000000000004},"last":0.63}]},{"osd":4,"last update":"Sat Sep 23 20:26:48 2023","interfaces":[{"interface":"back","average":{"1min":1.5049999999999999,"5min":1.2669999999999999,"15min":0.72999999999999998},"min":{"1min":0.27600000000000002,"5min":0.27200000000000002,"15min":0.26200000000000001},"max":{"1min":10.173,"5min":10.173,"15min":10.173},"last":0.504},{"interface":"front","average":{"1min":1.962,"5min":1.3540000000000001,"15min":0.78000000000000003},"min":{"1min":0.312,"5min":0.29799999999999999,"15min":0.28499999999999998},"max":{"1min":8.3249999999999993,"5min":8.3249999999999993,"15min":8.3249999999999993},"last":0.69099999999999995}]},{"osd":5,"last update":"Sat Sep 23 20:26:48 2023","interfaces":[{"interface":"back","average":{"1min":1.202,"5min":1.4119999999999999,"15min":0.80300000000000005},"min":{"1min":0.31,"5min":0.31,"15min":0.29799999999999999},"max":{"1min":4.4580000000000002,"5min":14.085000000000001,"15min":14.085000000000001},"last":0.62},{"interface":"front","average":{"1min":2.831,"5min":1.8660000000000001,"15min":0.97299999999999998},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.29499999999999998},"max":{"1min":18.192,"5min":18.192,"15min":18.192},"last":0.66000000000000003}]},{"osd":7,"last update":"Sat Sep 23 20:27:02 2023","interfaces":[{"interface":"back","average":{"1min":1.946,"5min":1.538,"15min":0.877},"min":{"1min":0.36899999999999999,"5min":0.33500000000000002,"15min":0.32800000000000001},"max":{"1min":8.0350000000000001,"5min":10.217000000000001,"15min":10.217000000000001},"last":0.59999999999999998},{"interface":"front","average":{"1min":1.6459999999999999,"5min":1.6020000000000001,"15min":0.91400000000000003},"min":{"1min":0.36499999999999999,"5min":0.34999999999999998,"15min":0.315},"max":{"1min":8.407,"5min":9.7360000000000007,"15min":9.7360000000000007},"last":0.71899999999999997}]}]},{"osd":1,"up_from":99,"seq":425201762383,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":172880,"kb_used_data":2568,"kb_used_omap":6,"kb_used_meta":170297,"kb_avail":93572272,"statfs":{"total":95995035648,"available":95818006528,"internally_reserved":0,"allocated":2629632,"data_stored":1513371,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6696,"internal_metadata":174384600},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":18,"apply_latency_ms":18,"commit_latency_ns":18000000,"apply_latency_ns":18000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:26:54 2023","interfaces":[{"interface":"back","average":{"1min":2.6600000000000001,"5min":2.052,"15min":0.93500000000000005},"min":{"1min":0.215,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":14.384,"5min":40.113,"15min":40.113},"last":0.224},{"interface":"front","average":{"1min":3.1869999999999998,"5min":1.8120000000000001,"15min":0.87},"min":{"1min":0.23699999999999999,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":22.210000000000001,"5min":42.994999999999997,"15min":42.994999999999997},"last":0.371}]},{"osd":2,"last update":"Sat Sep 23 20:26:27 2023","interfaces":[{"interface":"back","average":{"1min":1.6479999999999999,"5min":1.8029999999999999,"15min":0.85499999999999998},"min":{"1min":0.27900000000000003,"5min":0.253,"15min":0.253},"max":{"1min":9.6839999999999993,"5min":39.962000000000003,"15min":39.962000000000003},"last":0.31900000000000001},{"interface":"front","average":{"1min":2.806,"5min":1.794,"15min":0.96599999999999997},"min":{"1min":0.26700000000000002,"5min":0.246,"15min":0.246},"max":{"1min":8.7910000000000004,"5min":47.442,"15min":47.442},"last":0.28999999999999998}]},{"osd":3,"last update":"Sat Sep 23 20:27:00 2023","interfaces":[{"interface":"back","average":{"1min":1.9099999999999999,"5min":1.4039999999999999,"15min":0.83399999999999996},"min":{"1min":0.309,"5min":0.246,"15min":0.246},"max":{"1min":9.0410000000000004,"5min":9.0410000000000004,"15min":9.0410000000000004},"last":0.35999999999999999},{"interface":"front","average":{"1min":2.4319999999999999,"5min":1.4710000000000001,"15min":0.76200000000000001},"min":{"1min":0.29699999999999999,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":15.147,"5min":15.147,"15min":15.147},"last":0.34000000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:26:54 2023","interfaces":[{"interface":"back","average":{"1min":2.5790000000000002,"5min":1.5900000000000001,"15min":0.94099999999999995},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":15.353,"5min":15.353,"15min":15.353},"last":0.56399999999999995},{"interface":"front","average":{"1min":1.7609999999999999,"5min":1.536,"15min":0.93500000000000005},"min":{"1min":0.32400000000000001,"5min":0.312,"15min":0.312},"max":{"1min":6.2270000000000003,"5min":8.7899999999999991,"15min":8.7899999999999991},"last":0.58899999999999997}]},{"osd":5,"last update":"Sat Sep 23 20:26:54 2023","interfaces":[{"interface":"back","average":{"1min":2.6360000000000001,"5min":1.6220000000000001,"15min":0.96799999999999997},"min":{"1min":0.33000000000000002,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":7.2000000000000002,"5min":12.590999999999999,"15min":12.590999999999999},"last":0.53200000000000003},{"interface":"front","average":{"1min":2.2530000000000001,"5min":2.137,"15min":1.163},"min":{"1min":0.38300000000000001,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":6.3140000000000001,"5min":39.579000000000001,"15min":39.579000000000001},"last":0.59799999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:26:54 2023","interfaces":[{"interface":"back","average":{"1min":2.9630000000000001,"5min":1.996,"15min":1.1100000000000001},"min":{"1min":0.42999999999999999,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":7.226,"5min":34.409999999999997,"15min":34.409999999999997},"last":0.503},{"interface":"front","average":{"1min":2.964,"5min":2.2269999999999999,"15min":1.167},"min":{"1min":0.38100000000000001,"5min":0.377,"15min":0.377},"max":{"1min":15.313000000000001,"5min":35.554000000000002,"15min":35.554000000000002},"last":0.48599999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:26:54 2023","interfaces":[{"interface":"back","average":{"1min":2.8719999999999999,"5min":1.8480000000000001,"15min":1.05},"min":{"1min":0.42499999999999999,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":9.7439999999999998,"5min":9.7439999999999998,"15min":9.7439999999999998},"last":0.51400000000000001},{"interface":"front","average":{"1min":2.3180000000000001,"5min":1.7729999999999999,"15min":1.0469999999999999},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":6.2519999999999998,"5min":9.2279999999999998,"15min":9.2279999999999998},"last":0.55100000000000005}]}]},{"osd":0,"up_from":94,"seq":403726925909,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":278452,"kb_used_data":67500,"kb_used_omap":7,"kb_used_meta":210936,"kb_avail":93466700,"statfs":{"total":95995035648,"available":95709900800,"internally_reserved":0,"allocated":69120000,"data_stored":68091575,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":215998481},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":22,"apply_latency_ms":22,"commit_latency_ns":22000000,"apply_latency_ns":22000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:26:56 2023","interfaces":[{"interface":"back","average":{"1min":3.0779999999999998,"5min":1.3400000000000001,"15min":0.68600000000000005},"min":{"1min":0.223,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":24.247,"5min":24.247,"15min":24.247},"last":0.29999999999999999},{"interface":"front","average":{"1min":2.9950000000000001,"5min":1.3160000000000001,"15min":0.69599999999999995},"min":{"1min":0.28999999999999998,"5min":0.222,"15min":0.222},"max":{"1min":15.994999999999999,"5min":15.994999999999999,"15min":15.994999999999999},"last":0.28100000000000003}]},{"osd":2,"last update":"Sat Sep 23 20:26:22 2023","interfaces":[{"interface":"back","average":{"1min":4.3319999999999999,"5min":1.8200000000000001,"15min":1.004},"min":{"1min":0.251,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":23.832999999999998,"5min":23.832999999999998,"15min":23.832999999999998},"last":0.26400000000000001},{"interface":"front","average":{"1min":1.3779999999999999,"5min":1.256,"15min":0.749},"min":{"1min":0.223,"5min":0.223,"15min":0.223},"max":{"1min":6.0629999999999997,"5min":9.1940000000000008,"15min":9.1940000000000008},"last":0.33700000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:26:56 2023","interfaces":[{"interface":"back","average":{"1min":3.3109999999999999,"5min":1.7070000000000001,"15min":0.96699999999999997},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":23.748999999999999,"5min":23.748999999999999,"15min":23.748999999999999},"last":0.32400000000000001},{"interface":"front","average":{"1min":3.9809999999999999,"5min":1.5189999999999999,"15min":0.78700000000000003},"min":{"1min":0.315,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":24.274999999999999,"5min":24.274999999999999,"15min":24.274999999999999},"last":0.34699999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:26:22 2023","interfaces":[{"interface":"back","average":{"1min":4.6619999999999999,"5min":1.9430000000000001,"15min":1.046},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":23.876999999999999,"5min":23.876999999999999,"15min":23.876999999999999},"last":0.70899999999999996},{"interface":"front","average":{"1min":2.8839999999999999,"5min":1.587,"15min":0.93799999999999994},"min":{"1min":0.34799999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":23.681999999999999,"5min":23.681999999999999,"15min":23.681999999999999},"last":0.59899999999999998}]},{"osd":5,"last update":"Sat Sep 23 20:26:22 2023","interfaces":[{"interface":"back","average":{"1min":4.3259999999999996,"5min":2.2290000000000001,"15min":1.1819999999999999},"min":{"1min":0.39100000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":24.198,"5min":24.198,"15min":24.198},"last":0.64700000000000002},{"interface":"front","average":{"1min":2.0169999999999999,"5min":1.754,"15min":1.0369999999999999},"min":{"1min":0.37,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":6.9269999999999996,"5min":11.167999999999999,"15min":11.167999999999999},"last":0.63}]},{"osd":6,"last update":"Sat Sep 23 20:26:22 2023","interfaces":[{"interface":"back","average":{"1min":3.694,"5min":1.5760000000000001,"15min":0.95799999999999996},"min":{"1min":0.45400000000000001,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":24.408000000000001,"5min":24.408000000000001,"15min":24.408000000000001},"last":0.56499999999999995},{"interface":"front","average":{"1min":3.5230000000000001,"5min":1.526,"15min":0.94999999999999996},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":25.271000000000001,"5min":25.271000000000001,"15min":25.271000000000001},"last":0.72199999999999998}]},{"osd":7,"last update":"Sat Sep 23 20:26:22 2023","interfaces":[{"interface":"back","average":{"1min":3.3199999999999998,"5min":1.7190000000000001,"15min":1.0249999999999999},"min":{"1min":0.442,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":25.359000000000002,"5min":25.359000000000002,"15min":25.359000000000002},"last":0.749},{"interface":"front","average":{"1min":5.298,"5min":2.181,"15min":1.1699999999999999},"min":{"1min":0.499,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":23.905999999999999,"5min":23.905999999999999,"15min":23.905999999999999},"last":0.57999999999999996}]}]},{"osd":2,"up_from":104,"seq":446676598857,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":154936,"kb_used_data":6832,"kb_used_omap":11,"kb_used_meta":148084,"kb_avail":93590216,"statfs":{"total":95995035648,"available":95836381184,"internally_reserved":0,"allocated":6995968,"data_stored":5974781,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11878,"internal_metadata":151638426},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":10,"apply_latency_ms":10,"commit_latency_ns":10000000,"apply_latency_ns":10000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:26:30 2023","interfaces":[{"interface":"back","average":{"1min":2.8090000000000002,"5min":1.706,"15min":0.81999999999999995},"min":{"1min":0.21099999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":9.5980000000000008,"5min":12.811,"15min":12.811},"last":0.70399999999999996},{"interface":"front","average":{"1min":2.1019999999999999,"5min":1.2709999999999999,"15min":0.68700000000000006},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":15.746,"5min":15.746,"15min":15.746},"last":0.55600000000000005}]},{"osd":1,"last update":"Sat Sep 23 20:26:30 2023","interfaces":[{"interface":"back","average":{"1min":2.9390000000000001,"5min":1.524,"15min":0.77000000000000002},"min":{"1min":0.28599999999999998,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":14.135999999999999,"5min":14.135999999999999,"15min":14.135999999999999},"last":0.74399999999999999},{"interface":"front","average":{"1min":2.617,"5min":1.226,"15min":0.66400000000000003},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":13.718,"5min":13.718,"15min":13.718},"last":0.60499999999999998}]},{"osd":3,"last update":"Sat Sep 23 20:26:55 2023","interfaces":[{"interface":"back","average":{"1min":2.246,"5min":1.349,"15min":0.69099999999999995},"min":{"1min":0.26500000000000001,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":10.916,"5min":15.319000000000001,"15min":15.319000000000001},"last":0.65100000000000002},{"interface":"front","average":{"1min":2.294,"5min":1.552,"15min":0.78200000000000003},"min":{"1min":0.25800000000000001,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":12.494999999999999,"5min":12.494999999999999,"15min":12.494999999999999},"last":1.1299999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:26:30 2023","interfaces":[{"interface":"back","average":{"1min":1.996,"5min":1.234,"15min":0.80000000000000004},"min":{"1min":0.36299999999999999,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":8.2300000000000004,"5min":8.2300000000000004,"15min":8.2300000000000004},"last":0.78200000000000003},{"interface":"front","average":{"1min":2.3679999999999999,"5min":1.3979999999999999,"15min":0.85599999999999998},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":8.6530000000000005,"5min":8.6530000000000005,"15min":8.6530000000000005},"last":1.008}]},{"osd":5,"last update":"Sat Sep 23 20:26:30 2023","interfaces":[{"interface":"back","average":{"1min":2.8319999999999999,"5min":1.577,"15min":0.96299999999999997},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":11.237,"5min":11.237,"15min":11.237},"last":0.80800000000000005},{"interface":"front","average":{"1min":3.452,"5min":1.6850000000000001,"15min":0.999},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":11.266999999999999,"5min":11.266999999999999,"15min":11.266999999999999},"last":0.88500000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:26:30 2023","interfaces":[{"interface":"back","average":{"1min":2.0129999999999999,"5min":1.294,"15min":0.86899999999999999},"min":{"1min":0.49099999999999999,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":6.2599999999999998,"5min":6.7939999999999996,"15min":6.7939999999999996},"last":0.86199999999999999},{"interface":"front","average":{"1min":2.7410000000000001,"5min":1.585,"15min":0.97699999999999998},"min":{"1min":0.41099999999999998,"5min":0.377,"15min":0.377},"max":{"1min":9.6600000000000001,"5min":9.6600000000000001,"15min":9.6600000000000001},"last":0.91400000000000003}]},{"osd":7,"last update":"Sat Sep 23 20:26:30 2023","interfaces":[{"interface":"back","average":{"1min":3.1150000000000002,"5min":1.712,"15min":1.002},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":15.343999999999999,"5min":15.343999999999999,"15min":15.343999999999999},"last":0.95899999999999996},{"interface":"front","average":{"1min":2.6299999999999999,"5min":1.5840000000000001,"15min":0.96599999999999997},"min":{"1min":0.48899999999999999,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":9.1440000000000001,"5min":10.064,"15min":10.064},"last":0.754}]}]},{"osd":3,"up_from":109,"seq":468151435331,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":147084,"kb_used_data":3076,"kb_used_omap":4,"kb_used_meta":143995,"kb_avail":93598068,"statfs":{"total":95995035648,"available":95844421632,"internally_reserved":0,"allocated":3149824,"data_stored":2115559,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4514,"internal_metadata":147451486},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:26:56 2023","interfaces":[{"interface":"back","average":{"1min":3.2530000000000001,"5min":2.319,"15min":1.0069999999999999},"min":{"1min":0.28999999999999998,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":15.589,"5min":67.305999999999997,"15min":67.305999999999997},"last":0.41799999999999998},{"interface":"front","average":{"1min":1.48,"5min":2.0339999999999998,"15min":0.93400000000000005},"min":{"1min":0.29699999999999999,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":9.0920000000000005,"5min":67.403999999999996,"15min":67.403999999999996},"last":0.53300000000000003}]},{"osd":1,"last update":"Sat Sep 23 20:26:56 2023","interfaces":[{"interface":"back","average":{"1min":3.0230000000000001,"5min":1.5069999999999999,"15min":0.76700000000000002},"min":{"1min":0.25,"5min":0.214,"15min":0.214},"max":{"1min":19.675999999999998,"5min":19.675999999999998,"15min":19.675999999999998},"last":0.45900000000000002},{"interface":"front","average":{"1min":2.4590000000000001,"5min":1.375,"15min":0.755},"min":{"1min":0.33200000000000002,"5min":0.245,"15min":0.245},"max":{"1min":11.644,"5min":11.644,"15min":11.644},"last":0.51500000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:26:56 2023","interfaces":[{"interface":"back","average":{"1min":3.4369999999999998,"5min":1.7789999999999999,"15min":0.82799999999999996},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":24.045000000000002,"5min":24.045000000000002,"15min":24.045000000000002},"last":0.49099999999999999},{"interface":"front","average":{"1min":3.4119999999999999,"5min":1.8080000000000001,"15min":0.91200000000000003},"min":{"1min":0.33700000000000002,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":12.427,"5min":17.684000000000001,"15min":17.684000000000001},"last":0.64000000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:26:56 2023","interfaces":[{"interface":"back","average":{"1min":2.875,"5min":1.4590000000000001,"15min":0.88900000000000001},"min":{"1min":0.33100000000000002,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":11.880000000000001,"5min":11.880000000000001,"15min":11.880000000000001},"last":0.57999999999999996},{"interface":"front","average":{"1min":3.698,"5min":1.8180000000000001,"15min":1.016},"min":{"1min":0.378,"5min":0.371,"15min":0.371},"max":{"1min":21.673999999999999,"5min":21.673999999999999,"15min":21.673999999999999},"last":0.59599999999999997}]},{"osd":5,"last update":"Sat Sep 23 20:26:56 2023","interfaces":[{"interface":"back","average":{"1min":3.117,"5min":1.8560000000000001,"15min":1.0229999999999999},"min":{"1min":0.375,"5min":0.311,"15min":0.311},"max":{"1min":11.839,"5min":11.839,"15min":11.839},"last":0.55000000000000004},{"interface":"front","average":{"1min":1.825,"5min":1.2949999999999999,"15min":0.85199999999999998},"min":{"1min":0.40000000000000002,"5min":0.313,"15min":0.313},"max":{"1min":8.5570000000000004,"5min":8.5570000000000004,"15min":8.5570000000000004},"last":0.74199999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:26:56 2023","interfaces":[{"interface":"back","average":{"1min":2.0609999999999999,"5min":1.5900000000000001,"15min":0.95499999999999996},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":8.6059999999999999,"5min":8.6059999999999999,"15min":8.6059999999999999},"last":0.78300000000000003},{"interface":"front","average":{"1min":2.7469999999999999,"5min":1.5740000000000001,"15min":0.97199999999999998},"min":{"1min":0.40600000000000003,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":9.3179999999999996,"5min":9.3179999999999996,"15min":9.3179999999999996},"last":0.71699999999999997}]},{"osd":7,"last update":"Sat Sep 23 20:26:56 2023","interfaces":[{"interface":"back","average":{"1min":3.0030000000000001,"5min":1.5860000000000001,"15min":0.997},"min":{"1min":0.40999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":9.2750000000000004,"5min":9.2750000000000004,"15min":9.2750000000000004},"last":0.76500000000000001},{"interface":"front","average":{"1min":3.6680000000000001,"5min":2.0209999999999999,"15min":1.101},"min":{"1min":0.40699999999999997,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":12.468999999999999,"5min":12.468999999999999,"15min":12.468999999999999},"last":0.61899999999999999}]}]},{"osd":4,"up_from":30,"seq":128849019084,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":161104,"kb_used_data":27976,"kb_used_omap":0,"kb_used_meta":133120,"kb_avail":93584048,"statfs":{"total":95995035648,"available":95830065152,"internally_reserved":0,"allocated":28647424,"data_stored":27941639,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":136314880},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":19,"apply_latency_ms":19,"commit_latency_ns":19000000,"apply_latency_ns":19000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:26:22 2023","interfaces":[{"interface":"back","average":{"1min":2.258,"5min":1.411,"15min":0.84899999999999998},"min":{"1min":0.25800000000000001,"5min":0.222,"15min":0.222},"max":{"1min":11.331,"5min":20.411999999999999,"15min":20.411999999999999},"last":0.65800000000000003},{"interface":"front","average":{"1min":2.6629999999999998,"5min":1.5489999999999999,"15min":0.94399999999999995},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":13.595000000000001,"5min":20.454999999999998,"15min":20.454999999999998},"last":0.56399999999999995}]},{"osd":1,"last update":"Sat Sep 23 20:27:00 2023","interfaces":[{"interface":"back","average":{"1min":2.883,"5min":1.5089999999999999,"15min":0.92000000000000004},"min":{"1min":0.24299999999999999,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":9.6660000000000004,"5min":9.6660000000000004,"15min":9.6660000000000004},"last":0.46200000000000002},{"interface":"front","average":{"1min":2.4990000000000001,"5min":1.5640000000000001,"15min":0.93799999999999994},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":12.821999999999999,"5min":12.821999999999999,"15min":12.821999999999999},"last":1.2110000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:26:32 2023","interfaces":[{"interface":"back","average":{"1min":2.1779999999999999,"5min":1.51,"15min":0.90800000000000003},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":11.412000000000001,"5min":21.462,"15min":21.462},"last":0.58999999999999997},{"interface":"front","average":{"1min":3.29,"5min":1.8660000000000001,"15min":1.014},"min":{"1min":0.28999999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":13.567,"5min":22.593,"15min":22.593},"last":0.64100000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:27:00 2023","interfaces":[{"interface":"back","average":{"1min":2.516,"5min":1.7929999999999999,"15min":1.0249999999999999},"min":{"1min":0.32900000000000001,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":10.846,"5min":24.891999999999999,"15min":24.891999999999999},"last":0.63},{"interface":"front","average":{"1min":2.464,"5min":1.831,"15min":1.0529999999999999},"min":{"1min":0.35499999999999998,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":9.2080000000000002,"5min":21.922000000000001,"15min":21.922000000000001},"last":0.68100000000000005}]},{"osd":5,"last update":"Sat Sep 23 20:26:10 2023","interfaces":[{"interface":"back","average":{"1min":5.0119999999999996,"5min":1.649,"15min":0.86799999999999999},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":17.704000000000001,"5min":17.704000000000001,"15min":17.704000000000001},"last":0.50600000000000001},{"interface":"front","average":{"1min":3.1600000000000001,"5min":1.361,"15min":0.77900000000000003},"min":{"1min":0.32500000000000001,"5min":0.28000000000000003,"15min":0.248},"max":{"1min":15.329000000000001,"5min":15.329000000000001,"15min":15.329000000000001},"last":0.48299999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:26:34 2023","interfaces":[{"interface":"back","average":{"1min":3.3639999999999999,"5min":1.9239999999999999,"15min":0.98099999999999998},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.27500000000000002},"max":{"1min":15.252000000000001,"5min":41.371000000000002,"15min":41.371000000000002},"last":0.53600000000000003},{"interface":"front","average":{"1min":1.9179999999999999,"5min":1.593,"15min":0.88200000000000001},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":9.3350000000000009,"5min":41.140999999999998,"15min":41.140999999999998},"last":0.57699999999999996}]},{"osd":7,"last update":"Sat Sep 23 20:26:48 2023","interfaces":[{"interface":"back","average":{"1min":4.3280000000000003,"5min":1.8859999999999999,"15min":0.997},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.33600000000000002},"max":{"1min":16.427,"5min":16.427,"15min":16.427},"last":0.60399999999999998},{"interface":"front","average":{"1min":3.2890000000000001,"5min":1.754,"15min":0.96599999999999997},"min":{"1min":0.36399999999999999,"5min":0.34300000000000003,"15min":0.29899999999999999},"max":{"1min":15.645,"5min":19.018999999999998,"15min":19.018999999999998},"last":0.61699999999999999}]}]},{"osd":5,"up_from":39,"seq":167503724745,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":350344,"kb_used_data":221568,"kb_used_omap":0,"kb_used_meta":128768,"kb_avail":93394808,"statfs":{"total":95995035648,"available":95636283392,"internally_reserved":0,"allocated":226885632,"data_stored":226184125,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":131858432},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":16,"apply_latency_ms":16,"commit_latency_ns":16000000,"apply_latency_ns":16000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:26:22 2023","interfaces":[{"interface":"back","average":{"1min":2.3380000000000001,"5min":1.548,"15min":0.91000000000000003},"min":{"1min":0.29499999999999998,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":11.346,"5min":32.475000000000001,"15min":32.475000000000001},"last":0.42399999999999999},{"interface":"front","average":{"1min":1.9650000000000001,"5min":1.601,"15min":0.93400000000000005},"min":{"1min":0.245,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":5.6879999999999997,"5min":32.457000000000001,"15min":32.457000000000001},"last":0.46300000000000002}]},{"osd":1,"last update":"Sat Sep 23 20:27:01 2023","interfaces":[{"interface":"back","average":{"1min":2.3980000000000001,"5min":1.819,"15min":0.96899999999999997},"min":{"1min":0.253,"5min":0.245,"15min":0.245},"max":{"1min":9.7639999999999993,"5min":32.243000000000002,"15min":32.243000000000002},"last":0.55500000000000005},{"interface":"front","average":{"1min":1.103,"5min":1.3480000000000001,"15min":0.81399999999999995},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":5.7389999999999999,"5min":32.231000000000002,"15min":32.231000000000002},"last":0.40899999999999997}]},{"osd":2,"last update":"Sat Sep 23 20:26:22 2023","interfaces":[{"interface":"back","average":{"1min":1.9970000000000001,"5min":1.891,"15min":0.997},"min":{"1min":0.30099999999999999,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":11.407,"5min":34.514000000000003,"15min":34.514000000000003},"last":0.51400000000000001},{"interface":"front","average":{"1min":2.105,"5min":1.732,"15min":0.94999999999999996},"min":{"1min":0.32100000000000001,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":11.574999999999999,"5min":32.515000000000001,"15min":32.515000000000001},"last":0.33700000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:27:01 2023","interfaces":[{"interface":"back","average":{"1min":2.5129999999999999,"5min":1.5900000000000001,"15min":0.90200000000000002},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":12.846,"5min":12.846,"15min":12.846},"last":0.44500000000000001},{"interface":"front","average":{"1min":1.764,"5min":1.554,"15min":0.92800000000000005},"min":{"1min":0.35599999999999998,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":6.0049999999999999,"5min":11.379,"15min":11.379},"last":0.48199999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:26:18 2023","interfaces":[{"interface":"back","average":{"1min":3.391,"5min":1.4339999999999999,"15min":0.78000000000000003},"min":{"1min":0.29199999999999998,"5min":0.26700000000000002,"15min":0.26400000000000001},"max":{"1min":14.218999999999999,"5min":14.218999999999999,"15min":14.218999999999999},"last":0.39100000000000001},{"interface":"front","average":{"1min":2.6720000000000002,"5min":1.3420000000000001,"15min":0.77000000000000002},"min":{"1min":0.34300000000000003,"5min":0.29099999999999998,"15min":0.28799999999999998},"max":{"1min":11.664,"5min":11.664,"15min":11.664},"last":0.53600000000000003}]},{"osd":6,"last update":"Sat Sep 23 20:26:32 2023","interfaces":[{"interface":"back","average":{"1min":2.633,"5min":1.575,"15min":0.85599999999999998},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.29799999999999999},"max":{"1min":8.7949999999999999,"5min":11.545999999999999,"15min":11.545999999999999},"last":0.496},{"interface":"front","average":{"1min":1.6870000000000001,"5min":1.5,"15min":0.879},"min":{"1min":0.376,"5min":0.35999999999999999,"15min":0.34499999999999997},"max":{"1min":8.4120000000000008,"5min":11.456,"15min":11.456},"last":0.59199999999999997}]},{"osd":7,"last update":"Sat Sep 23 20:27:01 2023","interfaces":[{"interface":"back","average":{"1min":2.2890000000000001,"5min":1.585,"15min":0.88900000000000001},"min":{"1min":0.40100000000000002,"5min":0.33900000000000002,"15min":0.33800000000000002},"max":{"1min":6.0730000000000004,"5min":11.430999999999999,"15min":11.430999999999999},"last":0.57299999999999995},{"interface":"front","average":{"1min":1.216,"5min":1.1930000000000001,"15min":0.76400000000000001},"min":{"1min":0.316,"5min":0.316,"15min":0.315},"max":{"1min":5.835,"5min":12.368,"15min":12.368},"last":0.504}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4722,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1360,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6696,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7156,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3154,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:27:14.248 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-23T20:27:14.307 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:14 smithi018 ceph-mon[184144]: 1.3 scrub starts 2023-09-23T20:27:14.308 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:14 smithi018 ceph-mon[184144]: 1.3 scrub ok 2023-09-23T20:27:14.308 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:14 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2436846768' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-09-23T20:27:14.308 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:14 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-09-23T20:27:14.308 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:14 smithi018 ceph-mon[184144]: 1.5 scrub starts 2023-09-23T20:27:14.308 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:14 smithi018 ceph-mon[184144]: 1.5 scrub ok 2023-09-23T20:27:14.308 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:14 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/4183525032' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:27:14.308 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:14 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:27:14.308 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:14 smithi018 ceph-mon[184144]: 1.4 scrub starts 2023-09-23T20:27:14.308 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:14 smithi018 ceph-mon[184144]: 1.4 scrub ok 2023-09-23T20:27:14.308 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:14 smithi018 ceph-mon[184144]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:27:14.308 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:14 smithi018 ceph-mon[184144]: failed to encode map e217 with expected crc 2023-09-23T20:27:14.308 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:14 smithi018 ceph-mon[184144]: failed to encode map e217 with expected crc 2023-09-23T20:27:14.309 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:14 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2023-09-23T20:27:14.309 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:14 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:27:14.309 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:14 smithi018 ceph-mon[184144]: osdmap e217: 8 total, 8 up, 8 in 2023-09-23T20:27:14.309 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:14 smithi018 ceph-mon[184144]: failed to encode map e217 with expected crc 2023-09-23T20:27:14.309 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:14 smithi018 ceph-mon[184144]: failed to encode map e217 with expected crc 2023-09-23T20:27:14.309 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:14 smithi018 ceph-mon[184144]: failed to encode map e217 with expected crc 2023-09-23T20:27:14.309 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:14 smithi018 ceph-mon[184144]: failed to encode map e217 with expected crc 2023-09-23T20:27:14.309 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:14 smithi018 ceph-mon[179647]: 1.3 scrub starts 2023-09-23T20:27:14.309 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:14 smithi018 ceph-mon[179647]: 1.3 scrub ok 2023-09-23T20:27:14.309 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:14 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2436846768' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-09-23T20:27:14.309 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:14 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-09-23T20:27:14.310 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:14 smithi018 ceph-mon[179647]: 1.5 scrub starts 2023-09-23T20:27:14.310 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:14 smithi018 ceph-mon[179647]: 1.5 scrub ok 2023-09-23T20:27:14.310 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:14 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/4183525032' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:27:14.310 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:14 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:27:14.310 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:14 smithi018 ceph-mon[179647]: 1.4 scrub starts 2023-09-23T20:27:14.310 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:14 smithi018 ceph-mon[179647]: 1.4 scrub ok 2023-09-23T20:27:14.310 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:14 smithi018 ceph-mon[179647]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:27:14.310 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:14 smithi018 ceph-mon[179647]: failed to encode map e217 with expected crc 2023-09-23T20:27:14.310 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:14 smithi018 ceph-mon[179647]: failed to encode map e217 with expected crc 2023-09-23T20:27:14.310 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:14 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2023-09-23T20:27:14.310 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:14 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:27:14.310 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:14 smithi018 ceph-mon[179647]: osdmap e217: 8 total, 8 up, 8 in 2023-09-23T20:27:14.311 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:14 smithi018 ceph-mon[179647]: failed to encode map e217 with expected crc 2023-09-23T20:27:14.311 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:14 smithi018 ceph-mon[179647]: failed to encode map e217 with expected crc 2023-09-23T20:27:14.311 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:14 smithi018 ceph-mon[179647]: failed to encode map e217 with expected crc 2023-09-23T20:27:14.311 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:14 smithi018 ceph-mon[179647]: failed to encode map e217 with expected crc 2023-09-23T20:27:14.336 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:27:14.364 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:14.397 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:27:14.400 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:14.400 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:27:14.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:14 smithi149 ceph-mon[170394]: 1.3 scrub starts 2023-09-23T20:27:14.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:14 smithi149 ceph-mon[170394]: 1.3 scrub ok 2023-09-23T20:27:14.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:14 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2436846768' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-09-23T20:27:14.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:14 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-09-23T20:27:14.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:14 smithi149 ceph-mon[170394]: 1.5 scrub starts 2023-09-23T20:27:14.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:14 smithi149 ceph-mon[170394]: 1.5 scrub ok 2023-09-23T20:27:14.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:14 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/4183525032' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:27:14.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:14 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:27:14.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:14 smithi149 ceph-mon[170394]: 1.4 scrub starts 2023-09-23T20:27:14.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:14 smithi149 ceph-mon[170394]: 1.4 scrub ok 2023-09-23T20:27:14.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:14 smithi149 ceph-mon[170394]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:27:14.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:14 smithi149 ceph-mon[170394]: failed to encode map e217 with expected crc 2023-09-23T20:27:14.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:14 smithi149 ceph-mon[170394]: failed to encode map e217 with expected crc 2023-09-23T20:27:14.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:14 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2023-09-23T20:27:14.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:14 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:27:14.468 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:14 smithi149 ceph-mon[170394]: osdmap e217: 8 total, 8 up, 8 in 2023-09-23T20:27:14.468 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:14 smithi149 ceph-mon[170394]: failed to encode map e217 with expected crc 2023-09-23T20:27:14.468 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:14 smithi149 ceph-mon[170394]: failed to encode map e217 with expected crc 2023-09-23T20:27:14.468 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:14 smithi149 ceph-mon[170394]: failed to encode map e217 with expected crc 2023-09-23T20:27:14.468 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:14 smithi149 ceph-mon[170394]: failed to encode map e217 with expected crc 2023-09-23T20:27:14.501 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:27:14.529 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:14.815 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:27:14.818 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:14.818 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:27:14.867 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd pool application enable unique_pool_1 rados --yes-i-really-mean-it || true 2023-09-23T20:27:14.919 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:27:14.946 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:14.979 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:27:14.982 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:14.982 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:27:15.083 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:27:15.119 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:15.207 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:27:15.210 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:15.211 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:27:15.311 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:27:15.341 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:15 smithi018 ceph-mon[184144]: from='client.45122 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:27:15.341 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:15 smithi018 ceph-mon[184144]: pgmap v467: 99 pgs: 16 unknown, 4 active+undersized+remapped, 5 remapped+peering, 74 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 80 KiB/s, 0 objects/s recovering 2023-09-23T20:27:15.341 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:15 smithi018 ceph-mon[184144]: osdmap e218: 8 total, 8 up, 8 in 2023-09-23T20:27:15.341 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:15 smithi018 ceph-mon[184144]: failed to encode map e218 with expected crc 2023-09-23T20:27:15.341 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:15 smithi018 ceph-mon[184144]: failed to encode map e218 with expected crc 2023-09-23T20:27:15.341 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:15 smithi018 ceph-mon[184144]: failed to encode map e218 with expected crc 2023-09-23T20:27:15.341 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:15 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:27:15.341 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:15 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:27:15.341 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:15 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:27:15.342 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:27:14 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:27:14] "GET /metrics HTTP/1.1" 200 36477 "" "Prometheus/2.43.0" 2023-09-23T20:27:15.342 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:15 smithi018 ceph-mon[179647]: from='client.45122 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:27:15.342 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:15 smithi018 ceph-mon[179647]: pgmap v467: 99 pgs: 16 unknown, 4 active+undersized+remapped, 5 remapped+peering, 74 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 80 KiB/s, 0 objects/s recovering 2023-09-23T20:27:15.342 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:15 smithi018 ceph-mon[179647]: osdmap e218: 8 total, 8 up, 8 in 2023-09-23T20:27:15.342 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:15 smithi018 ceph-mon[179647]: failed to encode map e218 with expected crc 2023-09-23T20:27:15.342 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:15 smithi018 ceph-mon[179647]: failed to encode map e218 with expected crc 2023-09-23T20:27:15.342 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:15 smithi018 ceph-mon[179647]: failed to encode map e218 with expected crc 2023-09-23T20:27:15.342 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:15 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:27:15.343 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:15 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:27:15.343 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:15 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:27:15.343 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:15.394 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:27:15.412 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:27:15.435 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:27:15.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:15 smithi149 ceph-mon[170394]: from='client.45122 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:27:15.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:15 smithi149 ceph-mon[170394]: pgmap v467: 99 pgs: 16 unknown, 4 active+undersized+remapped, 5 remapped+peering, 74 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 80 KiB/s, 0 objects/s recovering 2023-09-23T20:27:15.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:15 smithi149 ceph-mon[170394]: osdmap e218: 8 total, 8 up, 8 in 2023-09-23T20:27:15.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:15 smithi149 ceph-mon[170394]: failed to encode map e218 with expected crc 2023-09-23T20:27:15.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:15 smithi149 ceph-mon[170394]: failed to encode map e218 with expected crc 2023-09-23T20:27:15.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:15 smithi149 ceph-mon[170394]: failed to encode map e218 with expected crc 2023-09-23T20:27:15.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:15 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:27:15.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:15 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:27:15.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:15 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:27:15.470 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:27:15.489 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:27:15.510 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:27:15.532 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:27:15.553 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:27:15.554 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:27:15.554 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:27:15.555 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:27:15.555 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:27:15.555 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:27:15.555 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:27:15.555 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:27:15.555 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:27:15.629 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:27:15.634 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:15.634 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:27:15.730 INFO:teuthology.orchestra.run.smithi018.stdout:ERROR: (22) Invalid argument 2023-09-23T20:27:15.731 INFO:teuthology.orchestra.run.smithi018.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:27:15.732 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:27:15.735 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:27:15.767 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:16.068 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T20:27:16.264 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:16 smithi018 ceph-mon[179647]: failed to encode map e219 with expected crc 2023-09-23T20:27:16.264 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:16 smithi018 ceph-mon[179647]: osdmap e219: 8 total, 8 up, 8 in 2023-09-23T20:27:16.264 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:16 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2437206795' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:27:16.264 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:16 smithi018 ceph-mon[179647]: pgmap v470: 99 pgs: 9 unknown, 4 remapped+peering, 86 active+clean; 4.5 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 41 KiB/s, 0 objects/s recovering 2023-09-23T20:27:16.299 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:27:16.302 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:16.302 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:27:16.386 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:27:16.403 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:27:16.415 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T20:27:16.437 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:16.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:16 smithi018 ceph-mon[184144]: failed to encode map e219 with expected crc 2023-09-23T20:27:16.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:16 smithi018 ceph-mon[184144]: osdmap e219: 8 total, 8 up, 8 in 2023-09-23T20:27:16.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:16 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2437206795' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:27:16.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:16 smithi018 ceph-mon[184144]: pgmap v470: 99 pgs: 9 unknown, 4 remapped+peering, 86 active+clean; 4.5 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 41 KiB/s, 0 objects/s recovering 2023-09-23T20:27:16.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:16 smithi149 ceph-mon[170394]: failed to encode map e219 with expected crc 2023-09-23T20:27:16.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:16 smithi149 ceph-mon[170394]: osdmap e219: 8 total, 8 up, 8 in 2023-09-23T20:27:16.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:16 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2437206795' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:27:16.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:16 smithi149 ceph-mon[170394]: pgmap v470: 99 pgs: 9 unknown, 4 remapped+peering, 86 active+clean; 4.5 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 41 KiB/s, 0 objects/s recovering 2023-09-23T20:27:17.386 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:27:17.387 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:17 smithi018 ceph-mon[184144]: failed to encode map e219 with expected crc 2023-09-23T20:27:17.387 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:17 smithi018 ceph-mon[184144]: failed to encode map e219 with expected crc 2023-09-23T20:27:17.387 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:17 smithi018 ceph-mon[184144]: failed to encode map e219 with expected crc 2023-09-23T20:27:17.387 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:17 smithi018 ceph-mon[179647]: failed to encode map e219 with expected crc 2023-09-23T20:27:17.387 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:17 smithi018 ceph-mon[179647]: failed to encode map e219 with expected crc 2023-09-23T20:27:17.387 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:17 smithi018 ceph-mon[179647]: failed to encode map e219 with expected crc 2023-09-23T20:27:17.393 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:17.393 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:27:17.494 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:27:17.523 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:17.560 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:27:17.562 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:17.563 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:27:17.663 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:27:17.692 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:17 smithi149 ceph-mon[170394]: failed to encode map e219 with expected crc 2023-09-23T20:27:17.692 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:17 smithi149 ceph-mon[170394]: failed to encode map e219 with expected crc 2023-09-23T20:27:17.692 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:17 smithi149 ceph-mon[170394]: failed to encode map e219 with expected crc 2023-09-23T20:27:17.695 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:17.731 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:27:17.734 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:17.735 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:27:17.835 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:27:17.863 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:17.897 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:27:17.900 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:17.901 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:27:18.001 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:27:18.028 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:18.060 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:27:18.063 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:18.063 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:27:18.164 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:27:18.198 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:18.238 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:27:18.244 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:18.245 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:27:18.309 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:27:18.346 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:27:18.379 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:18 smithi018 ceph-mon[184144]: pgmap v471: 99 pgs: 3 unknown, 96 active+clean; 4.5 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 47 KiB/s, 0 objects/s recovering 2023-09-23T20:27:18.379 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:18 smithi018 ceph-mon[179647]: pgmap v471: 99 pgs: 3 unknown, 96 active+clean; 4.5 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 47 KiB/s, 0 objects/s recovering 2023-09-23T20:27:18.381 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:18.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:18 smithi149 ceph-mon[170394]: pgmap v471: 99 pgs: 3 unknown, 96 active+clean; 4.5 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 47 KiB/s, 0 objects/s recovering 2023-09-23T20:27:18.468 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:27:18.525 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:27:18.529 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:18.529 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:27:18.630 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:27:18.661 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:18.668 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:27:18.736 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:27:18.738 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:18.739 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:27:18.840 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:27:18.870 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:19.027 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:27:19.027 INFO:teuthology.orchestra.run.smithi018.stdout:{"fsid":"a3b3b17e-5a4c-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub flag(s) set","count":7},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":515,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":219,"num_osds":8,"num_up_osds":8,"osd_up_since":1695500508,"num_in_osds":8,"osd_in_since":1695500738,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":96},{"state_name":"unknown","count":3}],"num_pgs":99,"num_pools":5,"num_objects":28,"data_bytes":4682855,"bytes_used":1507659776,"bytes_avail":766452625408,"bytes_total":767960285184,"unknown_pgs_ratio":0.030303031206130981,"recovering_objects_per_sec":0,"recovering_bytes_per_sec":47898,"recovering_keys_per_sec":0,"num_objects_recovered":0,"num_bytes_recovered":268348,"num_keys_recovered":0},"fsmap":{"epoch":9,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi018.hduiol","status":"up:active","gid":24566}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.18:8443/","prometheus":"http://172.21.15.18:9283/"}},"servicemap":{"epoch":20,"modified":"2023-09-23T20:22:49.433880+0000","services":{}},"progress_events":{"8e7eff18-0e45-4130-a6e1-662b3902f0d2":{"message":"Upgrade to 18.2.0-460-gbb0daeff (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true},"980bcc76-623f-4499-85ce-ee01a0e3cf55":{"message":"Global Recovery Event (0s)\n [............................] ","progress":0,"add_to_ceph_s":true}}} 2023-09-23T20:27:19.032 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:27:19.036 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:19.036 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:27:19.137 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:27:19.167 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:19.201 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:27:19.204 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:19.204 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:27:19.291 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:19 smithi018 ceph-mon[184144]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs peering) 2023-09-23T20:27:19.292 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:19 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/164190758' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T20:27:19.292 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:19 smithi018 ceph-mon[179647]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs peering) 2023-09-23T20:27:19.292 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:19 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/164190758' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T20:27:19.305 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:27:19.333 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:19.366 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:27:19.369 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:19.369 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:27:19.370 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:27:19.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:19 smithi149 ceph-mon[170394]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs peering) 2023-09-23T20:27:19.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:19 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/164190758' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T20:27:19.470 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:27:19.499 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:19.583 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:27:19.585 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:19.586 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:27:19.686 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:27:19.688 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-09-23T20:27:19.688 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:27:19.755 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:19.939 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:27:19.942 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:19.942 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:27:20.043 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:27:20.074 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:20.109 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:27:20.111 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:20.112 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:27:20.213 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:27:20.243 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:20.283 INFO:teuthology.orchestra.run.smithi018.stderr:enabled application 'rados' on pool 'unique_pool_1' 2023-09-23T20:27:20.307 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:27:20.310 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:20.311 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:27:20.412 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:27:20.440 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:20.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:20 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2242937832' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T20:27:20.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:20 smithi149 ceph-mon[170394]: pgmap v472: 99 pgs: 99 active+clean; 4.7 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-09-23T20:27:20.546 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:20 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2242937832' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T20:27:20.546 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:20 smithi018 ceph-mon[184144]: pgmap v472: 99 pgs: 99 active+clean; 4.7 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-09-23T20:27:20.547 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:20 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2242937832' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T20:27:20.547 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:20 smithi018 ceph-mon[179647]: pgmap v472: 99 pgs: 99 active+clean; 4.7 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-09-23T20:27:20.578 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:27:20.583 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:20.583 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:27:20.684 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:27:20.716 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:20.774 INFO:teuthology.orchestra.run.smithi018.stdout:ERROR: (22) Invalid argument 2023-09-23T20:27:20.774 INFO:teuthology.orchestra.run.smithi018.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:27:21.053 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:27:21.056 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:21.056 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:27:21.157 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:27:21.186 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:21.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:21 smithi018 ceph-mon[184144]: failed to encode map e220 with expected crc 2023-09-23T20:27:21.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:21 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2242937832' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T20:27:21.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:21 smithi018 ceph-mon[184144]: osdmap e220: 8 total, 8 up, 8 in 2023-09-23T20:27:21.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:21 smithi018 ceph-mon[184144]: failed to encode map e220 with expected crc 2023-09-23T20:27:21.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:21 smithi018 ceph-mon[179647]: failed to encode map e220 with expected crc 2023-09-23T20:27:21.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:21 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2242937832' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T20:27:21.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:21 smithi018 ceph-mon[179647]: osdmap e220: 8 total, 8 up, 8 in 2023-09-23T20:27:21.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:21 smithi018 ceph-mon[179647]: failed to encode map e220 with expected crc 2023-09-23T20:27:21.414 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:27:21.417 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:21.418 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:27:21.518 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:27:21.548 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:21.582 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:27:21.586 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:21.587 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:27:21.660 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T20:27:21.689 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:27:21.690 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:21 smithi149 ceph-mon[170394]: failed to encode map e220 with expected crc 2023-09-23T20:27:21.690 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:21 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2242937832' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T20:27:21.690 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:21 smithi149 ceph-mon[170394]: osdmap e220: 8 total, 8 up, 8 in 2023-09-23T20:27:21.690 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:21 smithi149 ceph-mon[170394]: failed to encode map e220 with expected crc 2023-09-23T20:27:21.716 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:21.851 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:27:21.867 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:27:21.870 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:21.871 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:27:21.971 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:27:21.998 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:22.035 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:27:22.038 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:22.038 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:27:22.139 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:27:22.169 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:22.264 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:27:22.290 INFO:teuthology.orchestra.run.smithi018.stderr:nodeep-scrub is set 2023-09-23T20:27:22.310 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:22 smithi018 ceph-mon[184144]: failed to encode map e220 with expected crc 2023-09-23T20:27:22.310 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:22 smithi018 ceph-mon[184144]: failed to encode map e220 with expected crc 2023-09-23T20:27:22.310 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:22 smithi018 ceph-mon[184144]: failed to encode map e220 with expected crc 2023-09-23T20:27:22.310 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:22 smithi018 ceph-mon[184144]: pgmap v474: 99 pgs: 99 active+clean; 4.7 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-09-23T20:27:22.310 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:22 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/682264600' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:27:22.310 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:22 smithi018 ceph-mon[179647]: failed to encode map e220 with expected crc 2023-09-23T20:27:22.310 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:22 smithi018 ceph-mon[179647]: failed to encode map e220 with expected crc 2023-09-23T20:27:22.311 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:22 smithi018 ceph-mon[179647]: failed to encode map e220 with expected crc 2023-09-23T20:27:22.311 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:22 smithi018 ceph-mon[179647]: pgmap v474: 99 pgs: 99 active+clean; 4.7 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-09-23T20:27:22.311 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:22 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/682264600' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:27:22.454 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:27:22.457 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:22.458 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:27:22.499 DEBUG:teuthology.orchestra.run.smithi018:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_1 bench 90 write' 2023-09-23T20:27:22.501 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-09-23T20:27:22.501 INFO:teuthology.orchestra.run:waiting for 3000 2023-09-23T20:27:22.514 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:27:22.534 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:27:22.549 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:hints = 1 2023-09-23T20:27:22.550 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-09-23T20:27:22.550 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Object prefix: benchmark_data_smithi018_270558 2023-09-23T20:27:22.550 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T20:27:22.550 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 0 0 0 0 0 0 - 0 2023-09-23T20:27:22.554 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:22 smithi149 ceph-mon[170394]: failed to encode map e220 with expected crc 2023-09-23T20:27:22.554 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:22 smithi149 ceph-mon[170394]: failed to encode map e220 with expected crc 2023-09-23T20:27:22.554 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:22 smithi149 ceph-mon[170394]: failed to encode map e220 with expected crc 2023-09-23T20:27:22.554 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:22 smithi149 ceph-mon[170394]: pgmap v474: 99 pgs: 99 active+clean; 4.7 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2023-09-23T20:27:22.554 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:22 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/682264600' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:27:22.559 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:27:22.563 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:27:22.606 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:27:22.639 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:22.651 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:27:22.691 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:27:22.730 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:27:22.779 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:27:22.785 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:27:22.785 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:27:22.785 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:27:22.785 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:27:22.785 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:27:22.785 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:27:22.785 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:27:22.786 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:27:22.871 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:27:22.878 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:22.878 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:27:22.979 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:27:23.029 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:23.108 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:27:23.112 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:23.113 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:27:23.213 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:27:23.265 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:23.285 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:23 smithi018 ceph-mon[179647]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:27:23.285 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:23 smithi018 ceph-mon[179647]: failed to encode map e221 with expected crc 2023-09-23T20:27:23.285 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:23 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/682264600' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:27:23.285 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:23 smithi018 ceph-mon[179647]: osdmap e221: 8 total, 8 up, 8 in 2023-09-23T20:27:23.285 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:23 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/829301047' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:27:23.285 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:23 smithi018 ceph-mon[179647]: failed to encode map e221 with expected crc 2023-09-23T20:27:23.369 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:23 smithi149 ceph-mon[170394]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:27:23.369 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:23 smithi149 ceph-mon[170394]: failed to encode map e221 with expected crc 2023-09-23T20:27:23.369 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:23 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/682264600' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:27:23.369 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:23 smithi149 ceph-mon[170394]: osdmap e221: 8 total, 8 up, 8 in 2023-09-23T20:27:23.369 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:23 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/829301047' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:27:23.369 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:23 smithi149 ceph-mon[170394]: failed to encode map e221 with expected crc 2023-09-23T20:27:23.379 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:27:23.383 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:23.383 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:27:23.484 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:27:23.549 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:23 smithi018 ceph-mon[184144]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:27:23.549 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:23 smithi018 ceph-mon[184144]: failed to encode map e221 with expected crc 2023-09-23T20:27:23.549 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:23 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/682264600' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:27:23.549 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:23 smithi018 ceph-mon[184144]: osdmap e221: 8 total, 8 up, 8 in 2023-09-23T20:27:23.549 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:23 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/829301047' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:27:23.549 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:23 smithi018 ceph-mon[184144]: failed to encode map e221 with expected crc 2023-09-23T20:27:23.550 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:23.551 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 1 16 3212 3196 199.517 199.75 0.00975669 0.00498105 2023-09-23T20:27:23.615 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:27:23.619 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:23.620 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:27:23.637 INFO:teuthology.orchestra.run.smithi149.stdout:{ 2023-09-23T20:27:23.638 INFO:teuthology.orchestra.run.smithi149.stdout: "ops": [ 2023-09-23T20:27:23.638 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:23.638 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.45161.0:3452 6.e 6:7b9daa0b:::benchmark_data_smithi018_270558_object3451:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e221)", 2023-09-23T20:27:23.638 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:27:23.633909+0000", 2023-09-23T20:27:23.638 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 0.0034828540000000001, 2023-09-23T20:27:23.638 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.003515425, 2023-09-23T20:27:23.638 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:27:23.638 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:27:23.638 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:27:23.638 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.45161", 2023-09-23T20:27:23.639 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3795614987", 2023-09-23T20:27:23.639 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 3452 2023-09-23T20:27:23.639 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:23.639 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:27:23.639 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:23.639 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:27:23.639 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:27:23.633909+0000", 2023-09-23T20:27:23.639 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:27:23.639 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:23.639 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:23.639 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:27:23.639 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:27:23.633909+0000", 2023-09-23T20:27:23.640 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:27:23.640 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:23.640 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:23.640 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:27:23.640 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:27:23.633908+0000", 2023-09-23T20:27:23.640 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:27:23.640 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:23.640 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:23.640 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:27:23.640 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:27:23.633945+0000", 2023-09-23T20:27:23.640 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.6797e-05 2023-09-23T20:27:23.640 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:23.641 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:23.641 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:27:23.641 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:27:23.633953+0000", 2023-09-23T20:27:23.641 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.1000000000000004e-06 2023-09-23T20:27:23.641 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:23.641 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:23.641 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:27:23.641 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:27:23.633959+0000", 2023-09-23T20:27:23.641 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.2510000000000002e-06 2023-09-23T20:27:23.641 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:23.641 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:23.641 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:27:23.642 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:27:23.634579+0000", 2023-09-23T20:27:23.642 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00062036999999999997 2023-09-23T20:27:23.642 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:23.642 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:23.642 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:27:23.642 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:27:23.634712+0000", 2023-09-23T20:27:23.642 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00013268899999999999 2023-09-23T20:27:23.642 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:23.642 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:23.642 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 1,2", 2023-09-23T20:27:23.642 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:27:23.634841+0000", 2023-09-23T20:27:23.642 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00012975399999999999 2023-09-23T20:27:23.643 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:23.643 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:23.643 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:27:23.643 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:27:23.635312+0000", 2023-09-23T20:27:23.643 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00047105299999999998 2023-09-23T20:27:23.643 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:23.643 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:23.643 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:27:23.643 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:27:23.636696+0000", 2023-09-23T20:27:23.643 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.001384054 2023-09-23T20:27:23.643 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:23.643 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:23.644 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:27:23.644 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:27:23.637348+0000", 2023-09-23T20:27:23.644 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00065148699999999996 2023-09-23T20:27:23.644 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:23.644 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:23.644 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:27:23.644 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:27:23.637395+0000", 2023-09-23T20:27:23.644 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.7068999999999999e-05 2023-09-23T20:27:23.644 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:27:23.644 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:27:23.644 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:27:23.645 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:23.645 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:23.645 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.45161.0:3453 6.e 6:71029e41:::benchmark_data_smithi018_270558_object3452:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e221)", 2023-09-23T20:27:23.645 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:27:23.633964+0000", 2023-09-23T20:27:23.645 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 0.0034276839999999999, 2023-09-23T20:27:23.645 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.003561219, 2023-09-23T20:27:23.645 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:27:23.645 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:27:23.645 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:27:23.645 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.45161", 2023-09-23T20:27:23.645 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3795614987", 2023-09-23T20:27:23.645 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 3453 2023-09-23T20:27:23.646 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:23.646 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:27:23.646 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:23.646 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:27:23.646 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:27:23.633964+0000", 2023-09-23T20:27:23.646 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:27:23.646 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:23.646 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:23.646 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:27:23.646 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:27:23.633964+0000", 2023-09-23T20:27:23.647 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:27:23.647 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:23.647 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:23.647 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:27:23.647 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:27:23.633964+0000", 2023-09-23T20:27:23.647 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:27:23.647 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:23.647 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:23.647 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:27:23.647 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:27:23.633995+0000", 2023-09-23T20:27:23.647 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.1272999999999997e-05 2023-09-23T20:27:23.648 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:23.648 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:23.648 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:27:23.648 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:27:23.634005+0000", 2023-09-23T20:27:23.648 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0209e-05 2023-09-23T20:27:23.648 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:23.648 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:23.648 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:27:23.648 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:27:23.634010+0000", 2023-09-23T20:27:23.648 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.5750000000000002e-06 2023-09-23T20:27:23.648 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:23.648 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:23.648 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:27:23.649 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:27:23.635378+0000", 2023-09-23T20:27:23.649 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.001368665 2023-09-23T20:27:23.649 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:23.649 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:23.649 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:27:23.649 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:27:23.635478+0000", 2023-09-23T20:27:23.649 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.9494000000000007e-05 2023-09-23T20:27:23.649 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:23.649 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:23.649 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 1,2", 2023-09-23T20:27:23.649 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:27:23.635599+0000", 2023-09-23T20:27:23.649 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000121221 2023-09-23T20:27:23.650 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:23.650 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:23.650 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:27:23.650 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:27:23.636063+0000", 2023-09-23T20:27:23.650 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000463886 2023-09-23T20:27:23.650 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:27:23.650 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:27:23.650 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:27:23.650 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:27:23.650 INFO:teuthology.orchestra.run.smithi149.stdout: ], 2023-09-23T20:27:23.650 INFO:teuthology.orchestra.run.smithi149.stdout: "num_ops": 2 2023-09-23T20:27:23.651 INFO:teuthology.orchestra.run.smithi149.stdout:} 2023-09-23T20:27:23.721 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:27:23.788 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:23.854 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:27:23.859 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:23.860 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:27:23.961 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:27:24.025 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:24.087 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:27:24.092 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:24.093 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:27:24.100 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T20:27:24.194 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:27:24.251 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:24.317 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:27:24.322 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:24.322 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:27:24.337 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:27:24.342 INFO:teuthology.orchestra.run.smithi018.stderr:dumped all 2023-09-23T20:27:24.424 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:27:24.469 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:24.551 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 2 16 6149 6133 191.534 183.562 0.00423463 0.0052093 2023-09-23T20:27:24.588 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:27:24.591 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:24 smithi149 ceph-mon[170394]: failed to encode map e221 with expected crc 2023-09-23T20:27:24.591 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:24 smithi149 ceph-mon[170394]: failed to encode map e221 with expected crc 2023-09-23T20:27:24.591 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:24 smithi149 ceph-mon[170394]: pgmap v476: 99 pgs: 99 active+clean; 21 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 2.0 MiB/s wr, 27 op/s 2023-09-23T20:27:24.592 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:24.592 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:27:24.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:24 smithi018 ceph-mon[179647]: failed to encode map e221 with expected crc 2023-09-23T20:27:24.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:24 smithi018 ceph-mon[179647]: failed to encode map e221 with expected crc 2023-09-23T20:27:24.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:24 smithi018 ceph-mon[179647]: pgmap v476: 99 pgs: 99 active+clean; 21 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 2.0 MiB/s wr, 27 op/s 2023-09-23T20:27:24.644 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:24 smithi018 ceph-mon[184144]: failed to encode map e221 with expected crc 2023-09-23T20:27:24.644 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:24 smithi018 ceph-mon[184144]: failed to encode map e221 with expected crc 2023-09-23T20:27:24.644 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:24 smithi018 ceph-mon[184144]: pgmap v476: 99 pgs: 99 active+clean; 21 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 2.0 MiB/s wr, 27 op/s 2023-09-23T20:27:24.693 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:27:24.748 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:24.806 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:27:24.815 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:27:24.821 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:24.822 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:27:24.892 INFO:teuthology.orchestra.run.smithi018.stdout:{"pg_ready":true,"pg_map":{"version":476,"stamp":"2023-09-23T20:27:23.553656+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":22258369,"num_objects":293,"num_object_clones":0,"num_object_copies":879,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":293,"num_whiteouts":0,"num_read":6352,"num_read_kb":13879,"num_write":1344,"num_write_kb":31377,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":19,"num_bytes_recovered":1552118,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3066,"ondisk_log_size":3066,"up":297,"acting":297,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":302,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":1422712,"kb_used_data":291320,"kb_used_omap":30,"kb_used_meta":1131297,"kb_avail":748538504,"statfs":{"total":767960285184,"available":766503428096,"internally_reserved":0,"allocated":298311680,"data_stored":291208158,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31263,"internal_metadata":1158448609},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":17521844,"num_objects":265,"num_object_clones":0,"num_object_copies":795,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":265,"num_whiteouts":0,"num_read":-88,"num_read_kb":-166,"num_write":220,"num_write_kb":16456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-2,"num_bytes_recovered":-387012,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.002012"},"pg_stats":[{"pgid":"3.1a","version":"69'1","reported_seq":216,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-09-23T20:27:14.265265+0000","last_change":"2023-09-23T20:25:39.632437+0000","last_active":"2023-09-23T20:27:14.265265+0000","last_peered":"2023-09-23T20:27:14.265265+0000","last_clean":"2023-09-23T20:27:14.265265+0000","last_became_active":"2023-09-23T20:25:39.632074+0000","last_became_peered":"2023-09-23T20:25:39.632074+0000","last_unstale":"2023-09-23T20:27:14.265265+0000","last_undegraded":"2023-09-23T20:27:14.265265+0000","last_fullsized":"2023-09-23T20:27:14.265265+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:32:13.543799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":66,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T20:27:20.426090+0000","last_change":"2023-09-23T20:25:02.932223+0000","last_active":"2023-09-23T20:27:20.426090+0000","last_peered":"2023-09-23T20:27:20.426090+0000","last_clean":"2023-09-23T20:27:20.426090+0000","last_became_active":"2023-09-23T20:25:02.931024+0000","last_became_peered":"2023-09-23T20:25:02.931024+0000","last_unstale":"2023-09-23T20:27:20.426090+0000","last_undegraded":"2023-09-23T20:27:20.426090+0000","last_fullsized":"2023-09-23T20:27:20.426090+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:16:00.586211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":176,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-09-23T20:27:14.261025+0000","last_change":"2023-09-23T20:25:02.934543+0000","last_active":"2023-09-23T20:27:14.261025+0000","last_peered":"2023-09-23T20:27:14.261025+0000","last_clean":"2023-09-23T20:27:14.261025+0000","last_became_active":"2023-09-23T20:25:02.934178+0000","last_became_peered":"2023-09-23T20:25:02.934178+0000","last_unstale":"2023-09-23T20:27:14.261025+0000","last_undegraded":"2023-09-23T20:27:14.261025+0000","last_fullsized":"2023-09-23T20:27:14.261025+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:27:38.453052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":182,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-09-23T20:27:14.261882+0000","last_change":"2023-09-23T20:25:02.957185+0000","last_active":"2023-09-23T20:27:14.261882+0000","last_peered":"2023-09-23T20:27:14.261882+0000","last_clean":"2023-09-23T20:27:14.261882+0000","last_became_active":"2023-09-23T20:25:02.956689+0000","last_became_peered":"2023-09-23T20:25:02.956689+0000","last_unstale":"2023-09-23T20:27:14.261882+0000","last_undegraded":"2023-09-23T20:27:14.261882+0000","last_fullsized":"2023-09-23T20:27:14.261882+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:04:28.524015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":66,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T20:27:20.425740+0000","last_change":"2023-09-23T20:25:02.932054+0000","last_active":"2023-09-23T20:27:20.425740+0000","last_peered":"2023-09-23T20:27:20.425740+0000","last_clean":"2023-09-23T20:27:20.425740+0000","last_became_active":"2023-09-23T20:25:02.931309+0000","last_became_peered":"2023-09-23T20:25:02.931309+0000","last_unstale":"2023-09-23T20:27:20.425740+0000","last_undegraded":"2023-09-23T20:27:20.425740+0000","last_fullsized":"2023-09-23T20:27:20.425740+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:57:53.378499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":178,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-09-23T20:27:14.711008+0000","last_change":"2023-09-23T20:20:46.937089+0000","last_active":"2023-09-23T20:27:14.711008+0000","last_peered":"2023-09-23T20:27:14.711008+0000","last_clean":"2023-09-23T20:27:14.711008+0000","last_became_active":"2023-09-23T20:20:46.936831+0000","last_became_peered":"2023-09-23T20:20:46.936831+0000","last_unstale":"2023-09-23T20:27:14.711008+0000","last_undegraded":"2023-09-23T20:27:14.711008+0000","last_fullsized":"2023-09-23T20:27:14.711008+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:34:56.747010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":180,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-09-23T20:27:14.262235+0000","last_change":"2023-09-23T20:25:02.969432+0000","last_active":"2023-09-23T20:27:14.262235+0000","last_peered":"2023-09-23T20:27:14.262235+0000","last_clean":"2023-09-23T20:27:14.262235+0000","last_became_active":"2023-09-23T20:25:02.969118+0000","last_became_peered":"2023-09-23T20:25:02.969118+0000","last_unstale":"2023-09-23T20:27:14.262235+0000","last_undegraded":"2023-09-23T20:27:14.262235+0000","last_fullsized":"2023-09-23T20:27:14.262235+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:02:22.453831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":67,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-09-23T20:27:22.554816+0000","last_change":"2023-09-23T20:25:02.932225+0000","last_active":"2023-09-23T20:27:22.554816+0000","last_peered":"2023-09-23T20:27:22.554816+0000","last_clean":"2023-09-23T20:27:22.554816+0000","last_became_active":"2023-09-23T20:25:02.931746+0000","last_became_peered":"2023-09-23T20:25:02.931746+0000","last_unstale":"2023-09-23T20:27:22.554816+0000","last_undegraded":"2023-09-23T20:27:22.554816+0000","last_fullsized":"2023-09-23T20:27:22.554816+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:57:24.470179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":179,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-09-23T20:27:15.262082+0000","last_change":"2023-09-23T20:20:46.950281+0000","last_active":"2023-09-23T20:27:15.262082+0000","last_peered":"2023-09-23T20:27:15.262082+0000","last_clean":"2023-09-23T20:27:15.262082+0000","last_became_active":"2023-09-23T20:20:46.949892+0000","last_became_peered":"2023-09-23T20:20:46.949892+0000","last_unstale":"2023-09-23T20:27:15.262082+0000","last_undegraded":"2023-09-23T20:27:15.262082+0000","last_fullsized":"2023-09-23T20:27:15.262082+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:28.763832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":67,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-09-23T20:27:22.554296+0000","last_change":"2023-09-23T20:25:02.947669+0000","last_active":"2023-09-23T20:27:22.554296+0000","last_peered":"2023-09-23T20:27:22.554296+0000","last_clean":"2023-09-23T20:27:22.554296+0000","last_became_active":"2023-09-23T20:25:02.947368+0000","last_became_peered":"2023-09-23T20:25:02.947368+0000","last_unstale":"2023-09-23T20:27:22.554296+0000","last_undegraded":"2023-09-23T20:27:22.554296+0000","last_fullsized":"2023-09-23T20:27:22.554296+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:52:53.813761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":215,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-09-23T20:27:16.081860+0000","last_change":"2023-09-23T20:25:03.038511+0000","last_active":"2023-09-23T20:27:16.081860+0000","last_peered":"2023-09-23T20:27:16.081860+0000","last_clean":"2023-09-23T20:27:16.081860+0000","last_became_active":"2023-09-23T20:25:02.976811+0000","last_became_peered":"2023-09-23T20:25:02.976811+0000","last_unstale":"2023-09-23T20:27:16.081860+0000","last_undegraded":"2023-09-23T20:27:16.081860+0000","last_fullsized":"2023-09-23T20:27:16.081860+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:28:06.605731+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":177,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-09-23T20:27:14.260414+0000","last_change":"2023-09-23T20:25:39.646452+0000","last_active":"2023-09-23T20:27:14.260414+0000","last_peered":"2023-09-23T20:27:14.260414+0000","last_clean":"2023-09-23T20:27:14.260414+0000","last_became_active":"2023-09-23T20:25:39.646175+0000","last_became_peered":"2023-09-23T20:25:39.646175+0000","last_unstale":"2023-09-23T20:27:14.260414+0000","last_undegraded":"2023-09-23T20:27:14.260414+0000","last_fullsized":"2023-09-23T20:27:14.260414+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:33:02.099951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":183,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-09-23T20:27:14.264038+0000","last_change":"2023-09-23T20:12:43.750217+0000","last_active":"2023-09-23T20:27:14.264038+0000","last_peered":"2023-09-23T20:27:14.264038+0000","last_clean":"2023-09-23T20:27:14.264038+0000","last_became_active":"2023-09-23T20:12:43.749929+0000","last_became_peered":"2023-09-23T20:12:43.749929+0000","last_unstale":"2023-09-23T20:27:14.264038+0000","last_undegraded":"2023-09-23T20:27:14.264038+0000","last_fullsized":"2023-09-23T20:27:14.264038+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:50:28.470211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":177,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-09-23T20:27:14.265991+0000","last_change":"2023-09-23T20:20:46.939938+0000","last_active":"2023-09-23T20:27:14.265991+0000","last_peered":"2023-09-23T20:27:14.265991+0000","last_clean":"2023-09-23T20:27:14.265991+0000","last_became_active":"2023-09-23T20:20:46.938777+0000","last_became_peered":"2023-09-23T20:20:46.938777+0000","last_unstale":"2023-09-23T20:27:14.265991+0000","last_undegraded":"2023-09-23T20:27:14.265991+0000","last_fullsized":"2023-09-23T20:27:14.265991+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:45:10.329438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":214,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-09-23T20:27:16.081588+0000","last_change":"2023-09-23T20:25:39.654026+0000","last_active":"2023-09-23T20:27:16.081588+0000","last_peered":"2023-09-23T20:27:16.081588+0000","last_clean":"2023-09-23T20:27:16.081588+0000","last_became_active":"2023-09-23T20:25:39.626849+0000","last_became_peered":"2023-09-23T20:25:39.626849+0000","last_unstale":"2023-09-23T20:27:16.081588+0000","last_undegraded":"2023-09-23T20:27:16.081588+0000","last_fullsized":"2023-09-23T20:27:16.081588+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:22:53.957630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":67,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-09-23T20:27:22.555151+0000","last_change":"2023-09-23T20:25:02.962047+0000","last_active":"2023-09-23T20:27:22.555151+0000","last_peered":"2023-09-23T20:27:22.555151+0000","last_clean":"2023-09-23T20:27:22.555151+0000","last_became_active":"2023-09-23T20:25:02.961523+0000","last_became_peered":"2023-09-23T20:25:02.961523+0000","last_unstale":"2023-09-23T20:27:22.555151+0000","last_undegraded":"2023-09-23T20:27:22.555151+0000","last_fullsized":"2023-09-23T20:27:22.555151+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:18:11.830822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":181,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-09-23T20:27:14.909617+0000","last_change":"2023-09-23T20:25:02.953993+0000","last_active":"2023-09-23T20:27:14.909617+0000","last_peered":"2023-09-23T20:27:14.909617+0000","last_clean":"2023-09-23T20:27:14.909617+0000","last_became_active":"2023-09-23T20:25:02.953672+0000","last_became_peered":"2023-09-23T20:25:02.953672+0000","last_unstale":"2023-09-23T20:27:14.909617+0000","last_undegraded":"2023-09-23T20:27:14.909617+0000","last_fullsized":"2023-09-23T20:27:14.909617+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:26:08.603577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":198,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-09-23T20:27:15.262167+0000","last_change":"2023-09-23T20:25:02.949584+0000","last_active":"2023-09-23T20:27:15.262167+0000","last_peered":"2023-09-23T20:27:15.262167+0000","last_clean":"2023-09-23T20:27:15.262167+0000","last_became_active":"2023-09-23T20:25:02.949132+0000","last_became_peered":"2023-09-23T20:25:02.949132+0000","last_unstale":"2023-09-23T20:27:15.262167+0000","last_undegraded":"2023-09-23T20:27:15.262167+0000","last_fullsized":"2023-09-23T20:27:15.262167+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:34:20.509823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":163,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-09-23T20:27:16.081092+0000","last_change":"2023-09-23T20:12:43.749139+0000","last_active":"2023-09-23T20:27:16.081092+0000","last_peered":"2023-09-23T20:27:16.081092+0000","last_clean":"2023-09-23T20:27:16.081092+0000","last_became_active":"2023-09-23T20:12:43.748854+0000","last_became_peered":"2023-09-23T20:12:43.748854+0000","last_unstale":"2023-09-23T20:27:16.081092+0000","last_undegraded":"2023-09-23T20:27:16.081092+0000","last_fullsized":"2023-09-23T20:27:16.081092+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:57:11.630712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":190,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-09-23T20:27:14.261854+0000","last_change":"2023-09-23T20:25:02.957010+0000","last_active":"2023-09-23T20:27:14.261854+0000","last_peered":"2023-09-23T20:27:14.261854+0000","last_clean":"2023-09-23T20:27:14.261854+0000","last_became_active":"2023-09-23T20:25:02.956267+0000","last_became_peered":"2023-09-23T20:25:02.956267+0000","last_unstale":"2023-09-23T20:27:14.261854+0000","last_undegraded":"2023-09-23T20:27:14.261854+0000","last_fullsized":"2023-09-23T20:27:14.261854+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:10:55.299422+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":187,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-09-23T20:27:14.710295+0000","last_change":"2023-09-23T20:20:14.881009+0000","last_active":"2023-09-23T20:27:14.710295+0000","last_peered":"2023-09-23T20:27:14.710295+0000","last_clean":"2023-09-23T20:27:14.710295+0000","last_became_active":"2023-09-23T20:20:14.880685+0000","last_became_peered":"2023-09-23T20:20:14.880685+0000","last_unstale":"2023-09-23T20:27:14.710295+0000","last_undegraded":"2023-09-23T20:27:14.710295+0000","last_fullsized":"2023-09-23T20:27:14.710295+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:28.010434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":66,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T20:27:20.426340+0000","last_change":"2023-09-23T20:25:02.977462+0000","last_active":"2023-09-23T20:27:20.426340+0000","last_peered":"2023-09-23T20:27:20.426340+0000","last_clean":"2023-09-23T20:27:20.426340+0000","last_became_active":"2023-09-23T20:25:02.969910+0000","last_became_peered":"2023-09-23T20:25:02.969910+0000","last_unstale":"2023-09-23T20:27:20.426340+0000","last_undegraded":"2023-09-23T20:27:20.426340+0000","last_fullsized":"2023-09-23T20:27:20.426340+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:22:55.644955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.a","version":"0'0","reported_seq":11,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-09-23T20:27:15.261845+0000","last_change":"2023-09-23T20:27:14.711745+0000","last_active":"2023-09-23T20:27:15.261845+0000","last_peered":"2023-09-23T20:27:15.261845+0000","last_clean":"2023-09-23T20:27:15.261845+0000","last_became_active":"2023-09-23T20:27:14.711412+0000","last_became_peered":"2023-09-23T20:27:14.711412+0000","last_unstale":"2023-09-23T20:27:15.261845+0000","last_undegraded":"2023-09-23T20:27:15.261845+0000","last_fullsized":"2023-09-23T20:27:15.261845+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:48:13.653105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":181,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-09-23T20:27:16.081956+0000","last_change":"2023-09-23T20:20:14.883283+0000","last_active":"2023-09-23T20:27:16.081956+0000","last_peered":"2023-09-23T20:27:16.081956+0000","last_clean":"2023-09-23T20:27:16.081956+0000","last_became_active":"2023-09-23T20:20:14.882806+0000","last_became_peered":"2023-09-23T20:20:14.882806+0000","last_unstale":"2023-09-23T20:27:16.081956+0000","last_undegraded":"2023-09-23T20:27:16.081956+0000","last_fullsized":"2023-09-23T20:27:16.081956+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:23:35.295701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":161,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-09-23T20:27:15.261853+0000","last_change":"2023-09-23T20:12:45.179211+0000","last_active":"2023-09-23T20:27:15.261853+0000","last_peered":"2023-09-23T20:27:15.261853+0000","last_clean":"2023-09-23T20:27:15.261853+0000","last_became_active":"2023-09-23T20:12:45.178966+0000","last_became_peered":"2023-09-23T20:12:45.178966+0000","last_unstale":"2023-09-23T20:27:15.261853+0000","last_undegraded":"2023-09-23T20:27:15.261853+0000","last_fullsized":"2023-09-23T20:27:15.261853+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:58.902140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"221'80","reported_seq":93,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-09-23T20:27:23.013456+0000","last_change":"2023-09-23T20:27:14.269442+0000","last_active":"2023-09-23T20:27:23.013456+0000","last_peered":"2023-09-23T20:27:23.013456+0000","last_clean":"2023-09-23T20:27:23.013456+0000","last_became_active":"2023-09-23T20:27:14.269167+0000","last_became_peered":"2023-09-23T20:27:14.269167+0000","last_unstale":"2023-09-23T20:27:23.013456+0000","last_undegraded":"2023-09-23T20:27:23.013456+0000","last_fullsized":"2023-09-23T20:27:23.013456+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":80,"log_dups_size":0,"ondisk_log_size":80,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:57:54.752399+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":5242880,"num_objects":80,"num_object_clones":0,"num_object_copies":240,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":80,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":80,"num_write_kb":5120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":182,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-09-23T20:27:16.081913+0000","last_change":"2023-09-23T20:20:46.950784+0000","last_active":"2023-09-23T20:27:16.081913+0000","last_peered":"2023-09-23T20:27:16.081913+0000","last_clean":"2023-09-23T20:27:16.081913+0000","last_became_active":"2023-09-23T20:20:46.950055+0000","last_became_peered":"2023-09-23T20:20:46.950055+0000","last_unstale":"2023-09-23T20:27:16.081913+0000","last_undegraded":"2023-09-23T20:27:16.081913+0000","last_fullsized":"2023-09-23T20:27:16.081913+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:23:07.645427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":214,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-09-23T20:27:14.264511+0000","last_change":"2023-09-23T20:25:02.943743+0000","last_active":"2023-09-23T20:27:14.264511+0000","last_peered":"2023-09-23T20:27:14.264511+0000","last_clean":"2023-09-23T20:27:14.264511+0000","last_became_active":"2023-09-23T20:25:02.943279+0000","last_became_peered":"2023-09-23T20:25:02.943279+0000","last_unstale":"2023-09-23T20:27:14.264511+0000","last_undegraded":"2023-09-23T20:27:14.264511+0000","last_fullsized":"2023-09-23T20:27:14.264511+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:21:36.128366+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"0'0","reported_seq":11,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-09-23T20:27:16.081171+0000","last_change":"2023-09-23T20:27:14.271145+0000","last_active":"2023-09-23T20:27:16.081171+0000","last_peered":"2023-09-23T20:27:16.081171+0000","last_clean":"2023-09-23T20:27:16.081171+0000","last_became_active":"2023-09-23T20:27:14.270876+0000","last_became_peered":"2023-09-23T20:27:14.270876+0000","last_unstale":"2023-09-23T20:27:16.081171+0000","last_undegraded":"2023-09-23T20:27:16.081171+0000","last_fullsized":"2023-09-23T20:27:16.081171+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:23:40.941512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":164,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-09-23T20:27:16.081092+0000","last_change":"2023-09-23T20:12:43.751523+0000","last_active":"2023-09-23T20:27:16.081092+0000","last_peered":"2023-09-23T20:27:16.081092+0000","last_clean":"2023-09-23T20:27:16.081092+0000","last_became_active":"2023-09-23T20:12:43.751257+0000","last_became_peered":"2023-09-23T20:12:43.751257+0000","last_unstale":"2023-09-23T20:27:16.081092+0000","last_undegraded":"2023-09-23T20:27:16.081092+0000","last_fullsized":"2023-09-23T20:27:16.081092+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:50:27.040225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":178,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-09-23T20:27:14.710591+0000","last_change":"2023-09-23T20:20:46.934762+0000","last_active":"2023-09-23T20:27:14.710591+0000","last_peered":"2023-09-23T20:27:14.710591+0000","last_clean":"2023-09-23T20:27:14.710591+0000","last_became_active":"2023-09-23T20:20:46.934472+0000","last_became_peered":"2023-09-23T20:20:46.934472+0000","last_unstale":"2023-09-23T20:27:14.710591+0000","last_undegraded":"2023-09-23T20:27:14.710591+0000","last_fullsized":"2023-09-23T20:27:14.710591+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:28:42.708286+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"0'0","reported_seq":11,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-09-23T20:27:14.909600+0000","last_change":"2023-09-23T20:27:14.268973+0000","last_active":"2023-09-23T20:27:14.909600+0000","last_peered":"2023-09-23T20:27:14.909600+0000","last_clean":"2023-09-23T20:27:14.909600+0000","last_became_active":"2023-09-23T20:27:14.268670+0000","last_became_peered":"2023-09-23T20:27:14.268670+0000","last_unstale":"2023-09-23T20:27:14.909600+0000","last_undegraded":"2023-09-23T20:27:14.909600+0000","last_fullsized":"2023-09-23T20:27:14.909600+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:37:23.418588+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":199,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-09-23T20:27:15.262085+0000","last_change":"2023-09-23T20:25:02.947631+0000","last_active":"2023-09-23T20:27:15.262085+0000","last_peered":"2023-09-23T20:27:15.262085+0000","last_clean":"2023-09-23T20:27:15.262085+0000","last_became_active":"2023-09-23T20:25:02.947268+0000","last_became_peered":"2023-09-23T20:25:02.947268+0000","last_unstale":"2023-09-23T20:27:15.262085+0000","last_undegraded":"2023-09-23T20:27:15.262085+0000","last_fullsized":"2023-09-23T20:27:15.262085+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:19:43.643985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":164,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-09-23T20:27:14.262293+0000","last_change":"2023-09-23T20:20:46.935968+0000","last_active":"2023-09-23T20:27:14.262293+0000","last_peered":"2023-09-23T20:27:14.262293+0000","last_clean":"2023-09-23T20:27:14.262293+0000","last_became_active":"2023-09-23T20:20:46.935618+0000","last_became_peered":"2023-09-23T20:20:46.935618+0000","last_unstale":"2023-09-23T20:27:14.262293+0000","last_undegraded":"2023-09-23T20:27:14.262293+0000","last_fullsized":"2023-09-23T20:27:14.262293+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:42:59.867176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"0'0","reported_seq":10,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-09-23T20:27:14.270503+0000","last_change":"2023-09-23T20:27:14.270193+0000","last_active":"2023-09-23T20:27:14.270503+0000","last_peered":"2023-09-23T20:27:14.270503+0000","last_clean":"2023-09-23T20:27:14.270503+0000","last_became_active":"2023-09-23T20:27:14.269963+0000","last_became_peered":"2023-09-23T20:27:14.269963+0000","last_unstale":"2023-09-23T20:27:14.270503+0000","last_undegraded":"2023-09-23T20:27:14.270503+0000","last_fullsized":"2023-09-23T20:27:14.270503+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:01:09.891149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":592,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-09-23T20:27:16.081286+0000","last_change":"2023-09-23T20:27:11.961741+0000","last_active":"2023-09-23T20:27:16.081286+0000","last_peered":"2023-09-23T20:27:16.081286+0000","last_clean":"2023-09-23T20:27:16.081286+0000","last_became_active":"2023-09-23T20:27:11.961310+0000","last_became_peered":"2023-09-23T20:27:11.961310+0000","last_unstale":"2023-09-23T20:27:16.081286+0000","last_undegraded":"2023-09-23T20:27:16.081286+0000","last_fullsized":"2023-09-23T20:27:16.081286+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:09:47.213130+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":71,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T20:27:20.426044+0000","last_change":"2023-09-23T20:25:03.000957+0000","last_active":"2023-09-23T20:27:20.426044+0000","last_peered":"2023-09-23T20:27:20.426044+0000","last_clean":"2023-09-23T20:27:20.426044+0000","last_became_active":"2023-09-23T20:25:02.939903+0000","last_became_peered":"2023-09-23T20:25:02.939903+0000","last_unstale":"2023-09-23T20:27:20.426044+0000","last_undegraded":"2023-09-23T20:27:20.426044+0000","last_fullsized":"2023-09-23T20:27:20.426044+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:51:14.981461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":197,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-09-23T20:27:14.263946+0000","last_change":"2023-09-23T20:25:02.929447+0000","last_active":"2023-09-23T20:27:14.263946+0000","last_peered":"2023-09-23T20:27:14.263946+0000","last_clean":"2023-09-23T20:27:14.263946+0000","last_became_active":"2023-09-23T20:25:02.929173+0000","last_became_peered":"2023-09-23T20:25:02.929173+0000","last_unstale":"2023-09-23T20:27:14.263946+0000","last_undegraded":"2023-09-23T20:27:14.263946+0000","last_fullsized":"2023-09-23T20:27:14.263946+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:32:09.280647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"0'0","reported_seq":12,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-09-23T20:27:17.035168+0000","last_change":"2023-09-23T20:27:14.270199+0000","last_active":"2023-09-23T20:27:17.035168+0000","last_peered":"2023-09-23T20:27:17.035168+0000","last_clean":"2023-09-23T20:27:17.035168+0000","last_became_active":"2023-09-23T20:27:14.269858+0000","last_became_peered":"2023-09-23T20:27:14.269858+0000","last_unstale":"2023-09-23T20:27:17.035168+0000","last_undegraded":"2023-09-23T20:27:17.035168+0000","last_fullsized":"2023-09-23T20:27:17.035168+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:47:21.406552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":434,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-09-23T20:27:15.261750+0000","last_change":"2023-09-23T20:27:11.959998+0000","last_active":"2023-09-23T20:27:15.261750+0000","last_peered":"2023-09-23T20:27:15.261750+0000","last_clean":"2023-09-23T20:27:15.261750+0000","last_became_active":"2023-09-23T20:27:11.959516+0000","last_became_peered":"2023-09-23T20:27:11.959516+0000","last_unstale":"2023-09-23T20:27:15.261750+0000","last_undegraded":"2023-09-23T20:27:15.261750+0000","last_fullsized":"2023-09-23T20:27:15.261750+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:09:47.213130+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":423697,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":179,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-09-23T20:27:14.710519+0000","last_change":"2023-09-23T20:20:46.937163+0000","last_active":"2023-09-23T20:27:14.710519+0000","last_peered":"2023-09-23T20:27:14.710519+0000","last_clean":"2023-09-23T20:27:14.710519+0000","last_became_active":"2023-09-23T20:20:46.936757+0000","last_became_peered":"2023-09-23T20:20:46.936757+0000","last_unstale":"2023-09-23T20:27:14.710519+0000","last_undegraded":"2023-09-23T20:27:14.710519+0000","last_fullsized":"2023-09-23T20:27:14.710519+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:09.401379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":207,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-09-23T20:27:14.264523+0000","last_change":"2023-09-23T20:25:39.624636+0000","last_active":"2023-09-23T20:27:14.264523+0000","last_peered":"2023-09-23T20:27:14.264523+0000","last_clean":"2023-09-23T20:27:14.264523+0000","last_became_active":"2023-09-23T20:25:39.624297+0000","last_became_peered":"2023-09-23T20:25:39.624297+0000","last_unstale":"2023-09-23T20:27:14.264523+0000","last_undegraded":"2023-09-23T20:27:14.264523+0000","last_fullsized":"2023-09-23T20:27:14.264523+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:33:47.448737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"221'93","reported_seq":106,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-09-23T20:27:23.036699+0000","last_change":"2023-09-23T20:27:14.712208+0000","last_active":"2023-09-23T20:27:23.036699+0000","last_peered":"2023-09-23T20:27:23.036699+0000","last_clean":"2023-09-23T20:27:23.036699+0000","last_became_active":"2023-09-23T20:27:14.711905+0000","last_became_peered":"2023-09-23T20:27:14.711905+0000","last_unstale":"2023-09-23T20:27:23.036699+0000","last_undegraded":"2023-09-23T20:27:23.036699+0000","last_fullsized":"2023-09-23T20:27:23.036699+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":93,"log_dups_size":0,"ondisk_log_size":93,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:14:54.221825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":6094848,"num_objects":93,"num_object_clones":0,"num_object_copies":279,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":93,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":93,"num_write_kb":5952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":198,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-09-23T20:27:14.264326+0000","last_change":"2023-09-23T20:25:39.628097+0000","last_active":"2023-09-23T20:27:14.264326+0000","last_peered":"2023-09-23T20:27:14.264326+0000","last_clean":"2023-09-23T20:27:14.264326+0000","last_became_active":"2023-09-23T20:25:39.627806+0000","last_became_peered":"2023-09-23T20:25:39.627806+0000","last_unstale":"2023-09-23T20:27:14.264326+0000","last_undegraded":"2023-09-23T20:27:14.264326+0000","last_fullsized":"2023-09-23T20:27:14.264326+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:53:44.398854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":177,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-09-23T20:27:14.265167+0000","last_change":"2023-09-23T20:20:46.939720+0000","last_active":"2023-09-23T20:27:14.265167+0000","last_peered":"2023-09-23T20:27:14.265167+0000","last_clean":"2023-09-23T20:27:14.265167+0000","last_became_active":"2023-09-23T20:20:46.939357+0000","last_became_peered":"2023-09-23T20:20:46.939357+0000","last_unstale":"2023-09-23T20:27:14.265167+0000","last_undegraded":"2023-09-23T20:27:14.265167+0000","last_fullsized":"2023-09-23T20:27:14.265167+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:56.613212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"0'0","reported_seq":11,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-09-23T20:27:15.261995+0000","last_change":"2023-09-23T20:27:14.268955+0000","last_active":"2023-09-23T20:27:15.261995+0000","last_peered":"2023-09-23T20:27:15.261995+0000","last_clean":"2023-09-23T20:27:15.261995+0000","last_became_active":"2023-09-23T20:27:14.268662+0000","last_became_peered":"2023-09-23T20:27:14.268662+0000","last_unstale":"2023-09-23T20:27:15.261995+0000","last_undegraded":"2023-09-23T20:27:15.261995+0000","last_fullsized":"2023-09-23T20:27:15.261995+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:33:52.785540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":434,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-09-23T20:27:15.261840+0000","last_change":"2023-09-23T20:27:11.961446+0000","last_active":"2023-09-23T20:27:15.261840+0000","last_peered":"2023-09-23T20:27:15.261840+0000","last_clean":"2023-09-23T20:27:15.261840+0000","last_became_active":"2023-09-23T20:27:11.961051+0000","last_became_peered":"2023-09-23T20:27:11.961051+0000","last_unstale":"2023-09-23T20:27:15.261840+0000","last_undegraded":"2023-09-23T20:27:15.261840+0000","last_fullsized":"2023-09-23T20:27:15.261840+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:09:47.213130+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":67,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T20:27:20.425699+0000","last_change":"2023-09-23T20:25:02.978173+0000","last_active":"2023-09-23T20:27:20.425699+0000","last_peered":"2023-09-23T20:27:20.425699+0000","last_clean":"2023-09-23T20:27:20.425699+0000","last_became_active":"2023-09-23T20:25:02.977856+0000","last_became_peered":"2023-09-23T20:25:02.977856+0000","last_unstale":"2023-09-23T20:27:20.425699+0000","last_undegraded":"2023-09-23T20:27:20.425699+0000","last_fullsized":"2023-09-23T20:27:20.425699+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:19.861943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":79,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-09-23T20:27:14.263627+0000","last_change":"2023-09-23T20:25:02.924911+0000","last_active":"2023-09-23T20:27:14.263627+0000","last_peered":"2023-09-23T20:27:14.263627+0000","last_clean":"2023-09-23T20:27:14.263627+0000","last_became_active":"2023-09-23T20:25:02.924586+0000","last_became_peered":"2023-09-23T20:25:02.924586+0000","last_unstale":"2023-09-23T20:27:14.263627+0000","last_undegraded":"2023-09-23T20:27:14.263627+0000","last_fullsized":"2023-09-23T20:27:14.263627+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:24:44.176766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"0'0","reported_seq":11,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-09-23T20:27:14.909583+0000","last_change":"2023-09-23T20:27:14.267314+0000","last_active":"2023-09-23T20:27:14.909583+0000","last_peered":"2023-09-23T20:27:14.909583+0000","last_clean":"2023-09-23T20:27:14.909583+0000","last_became_active":"2023-09-23T20:27:14.267011+0000","last_became_peered":"2023-09-23T20:27:14.267011+0000","last_unstale":"2023-09-23T20:27:14.909583+0000","last_undegraded":"2023-09-23T20:27:14.909583+0000","last_fullsized":"2023-09-23T20:27:14.909583+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:47:58.542542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":442,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-09-23T20:27:14.710904+0000","last_change":"2023-09-23T20:27:11.964104+0000","last_active":"2023-09-23T20:27:14.710904+0000","last_peered":"2023-09-23T20:27:14.710904+0000","last_clean":"2023-09-23T20:27:14.710904+0000","last_became_active":"2023-09-23T20:27:11.963740+0000","last_became_peered":"2023-09-23T20:27:11.963740+0000","last_unstale":"2023-09-23T20:27:14.710904+0000","last_undegraded":"2023-09-23T20:27:14.710904+0000","last_fullsized":"2023-09-23T20:27:14.710904+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:09:47.213130+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"86'119","reported_seq":553,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-09-23T20:27:14.260677+0000","last_change":"2023-09-23T20:20:46.937071+0000","last_active":"2023-09-23T20:27:14.260677+0000","last_peered":"2023-09-23T20:27:14.260677+0000","last_clean":"2023-09-23T20:27:14.260677+0000","last_became_active":"2023-09-23T20:20:46.935348+0000","last_became_peered":"2023-09-23T20:20:46.935348+0000","last_unstale":"2023-09-23T20:27:14.260677+0000","last_undegraded":"2023-09-23T20:27:14.260677+0000","last_fullsized":"2023-09-23T20:27:14.260677+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":119,"log_dups_size":0,"ondisk_log_size":119,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:09.829030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032705499999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":523,"num_read_kb":3503,"num_write":136,"num_write_kb":3128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":236,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-09-23T20:27:15.262166+0000","last_change":"2023-09-23T20:25:39.637160+0000","last_active":"2023-09-23T20:27:15.262166+0000","last_peered":"2023-09-23T20:27:15.262166+0000","last_clean":"2023-09-23T20:27:15.262166+0000","last_became_active":"2023-09-23T20:25:39.636860+0000","last_became_peered":"2023-09-23T20:25:39.636860+0000","last_unstale":"2023-09-23T20:27:15.262166+0000","last_undegraded":"2023-09-23T20:27:15.262166+0000","last_fullsized":"2023-09-23T20:27:15.262166+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:21:47.860063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":145,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-09-23T20:27:14.260553+0000","last_change":"2023-09-23T20:20:46.937104+0000","last_active":"2023-09-23T20:27:14.260553+0000","last_peered":"2023-09-23T20:27:14.260553+0000","last_clean":"2023-09-23T20:27:14.260553+0000","last_became_active":"2023-09-23T20:20:46.936820+0000","last_became_peered":"2023-09-23T20:20:46.936820+0000","last_unstale":"2023-09-23T20:27:14.260553+0000","last_undegraded":"2023-09-23T20:27:14.260553+0000","last_fullsized":"2023-09-23T20:27:14.260553+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:22:19.879310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.6","version":"221'91","reported_seq":104,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-09-23T20:27:23.045073+0000","last_change":"2023-09-23T20:27:14.269226+0000","last_active":"2023-09-23T20:27:23.045073+0000","last_peered":"2023-09-23T20:27:23.045073+0000","last_clean":"2023-09-23T20:27:23.045073+0000","last_became_active":"2023-09-23T20:27:14.268927+0000","last_became_peered":"2023-09-23T20:27:14.268927+0000","last_unstale":"2023-09-23T20:27:23.045073+0000","last_undegraded":"2023-09-23T20:27:23.045073+0000","last_fullsized":"2023-09-23T20:27:23.045073+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":91,"log_dups_size":0,"ondisk_log_size":91,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:45:57.844328+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":5963776,"num_objects":91,"num_object_clones":0,"num_object_copies":273,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":91,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":91,"num_write_kb":5824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":434,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-09-23T20:27:15.261814+0000","last_change":"2023-09-23T20:27:11.962544+0000","last_active":"2023-09-23T20:27:15.261814+0000","last_peered":"2023-09-23T20:27:15.261814+0000","last_clean":"2023-09-23T20:27:15.261814+0000","last_became_active":"2023-09-23T20:27:11.961863+0000","last_became_peered":"2023-09-23T20:27:11.961863+0000","last_unstale":"2023-09-23T20:27:15.261814+0000","last_undegraded":"2023-09-23T20:27:15.261814+0000","last_fullsized":"2023-09-23T20:27:15.261814+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:09:47.213130+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":45,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"86'106","reported_seq":844,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-09-23T20:27:15.261998+0000","last_change":"2023-09-23T20:20:46.939468+0000","last_active":"2023-09-23T20:27:15.261998+0000","last_peered":"2023-09-23T20:27:15.261998+0000","last_clean":"2023-09-23T20:27:15.261998+0000","last_became_active":"2023-09-23T20:20:46.938954+0000","last_became_peered":"2023-09-23T20:20:46.938954+0000","last_unstale":"2023-09-23T20:27:15.261998+0000","last_undegraded":"2023-09-23T20:27:15.261998+0000","last_fullsized":"2023-09-23T20:27:15.261998+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":106,"log_dups_size":0,"ondisk_log_size":106,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:01:43.761064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00099302599999999993,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1105,"num_read_kb":1689,"num_write":100,"num_write_kb":404,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":178,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-09-23T20:27:14.265865+0000","last_change":"2023-09-23T20:20:14.887345+0000","last_active":"2023-09-23T20:27:14.265865+0000","last_peered":"2023-09-23T20:27:14.265865+0000","last_clean":"2023-09-23T20:27:14.265865+0000","last_became_active":"2023-09-23T20:20:14.886970+0000","last_became_peered":"2023-09-23T20:20:14.886970+0000","last_unstale":"2023-09-23T20:27:14.265865+0000","last_undegraded":"2023-09-23T20:27:14.265865+0000","last_fullsized":"2023-09-23T20:27:14.265865+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:00:38.575926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":211,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-09-23T20:27:14.260555+0000","last_change":"2023-09-23T20:25:39.631133+0000","last_active":"2023-09-23T20:27:14.260555+0000","last_peered":"2023-09-23T20:27:14.260555+0000","last_clean":"2023-09-23T20:27:14.260555+0000","last_became_active":"2023-09-23T20:25:39.630779+0000","last_became_peered":"2023-09-23T20:25:39.630779+0000","last_unstale":"2023-09-23T20:27:14.260555+0000","last_undegraded":"2023-09-23T20:27:14.260555+0000","last_fullsized":"2023-09-23T20:27:14.260555+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:39:52.635649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.5","version":"0'0","reported_seq":11,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-09-23T20:27:16.081777+0000","last_change":"2023-09-23T20:27:14.711792+0000","last_active":"2023-09-23T20:27:16.081777+0000","last_peered":"2023-09-23T20:27:16.081777+0000","last_clean":"2023-09-23T20:27:16.081777+0000","last_became_active":"2023-09-23T20:27:14.711434+0000","last_became_peered":"2023-09-23T20:27:14.711434+0000","last_unstale":"2023-09-23T20:27:16.081777+0000","last_undegraded":"2023-09-23T20:27:16.081777+0000","last_fullsized":"2023-09-23T20:27:16.081777+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:10:14.885066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":592,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-09-23T20:27:16.081398+0000","last_change":"2023-09-23T20:27:11.961802+0000","last_active":"2023-09-23T20:27:16.081398+0000","last_peered":"2023-09-23T20:27:16.081398+0000","last_clean":"2023-09-23T20:27:16.081398+0000","last_became_active":"2023-09-23T20:27:11.961418+0000","last_became_peered":"2023-09-23T20:27:11.961418+0000","last_unstale":"2023-09-23T20:27:16.081398+0000","last_undegraded":"2023-09-23T20:27:16.081398+0000","last_fullsized":"2023-09-23T20:27:16.081398+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:09:47.213130+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"86'130","reported_seq":529,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-09-23T20:27:16.078317+0000","last_change":"2023-09-23T20:25:39.637458+0000","last_active":"2023-09-23T20:27:16.078317+0000","last_peered":"2023-09-23T20:27:16.078317+0000","last_clean":"2023-09-23T20:27:16.078317+0000","last_became_active":"2023-09-23T20:25:39.636773+0000","last_became_peered":"2023-09-23T20:25:39.636773+0000","last_unstale":"2023-09-23T20:27:16.078317+0000","last_undegraded":"2023-09-23T20:27:16.078317+0000","last_fullsized":"2023-09-23T20:27:16.078317+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:00:32.810564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011300049999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":512,"num_read_kb":1820,"num_write":133,"num_write_kb":1347,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":177,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-09-23T20:27:14.260056+0000","last_change":"2023-09-23T20:25:39.632283+0000","last_active":"2023-09-23T20:27:14.260056+0000","last_peered":"2023-09-23T20:27:14.260056+0000","last_clean":"2023-09-23T20:27:14.260056+0000","last_became_active":"2023-09-23T20:25:39.631819+0000","last_became_peered":"2023-09-23T20:25:39.631819+0000","last_unstale":"2023-09-23T20:27:14.260056+0000","last_undegraded":"2023-09-23T20:27:14.260056+0000","last_fullsized":"2023-09-23T20:27:14.260056+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:25:21.815555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":188,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-09-23T20:27:15.061037+0000","last_change":"2023-09-23T20:20:46.936524+0000","last_active":"2023-09-23T20:27:15.061037+0000","last_peered":"2023-09-23T20:27:15.061037+0000","last_clean":"2023-09-23T20:27:15.061037+0000","last_became_active":"2023-09-23T20:20:46.936267+0000","last_became_peered":"2023-09-23T20:20:46.936267+0000","last_unstale":"2023-09-23T20:27:15.061037+0000","last_undegraded":"2023-09-23T20:27:15.061037+0000","last_fullsized":"2023-09-23T20:27:15.061037+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:32:20.984521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"0'0","reported_seq":11,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-09-23T20:27:15.262075+0000","last_change":"2023-09-23T20:27:14.272939+0000","last_active":"2023-09-23T20:27:15.262075+0000","last_peered":"2023-09-23T20:27:15.262075+0000","last_clean":"2023-09-23T20:27:15.262075+0000","last_became_active":"2023-09-23T20:27:14.272096+0000","last_became_peered":"2023-09-23T20:27:14.272096+0000","last_unstale":"2023-09-23T20:27:15.262075+0000","last_undegraded":"2023-09-23T20:27:15.262075+0000","last_fullsized":"2023-09-23T20:27:15.262075+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:41:48.494925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"86'194","reported_seq":868,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-09-23T20:27:15.261915+0000","last_change":"2023-09-23T20:25:40.618665+0000","last_active":"2023-09-23T20:27:15.261915+0000","last_peered":"2023-09-23T20:27:15.261915+0000","last_clean":"2023-09-23T20:27:15.261915+0000","last_became_active":"2023-09-23T20:25:40.618087+0000","last_became_peered":"2023-09-23T20:25:40.618087+0000","last_unstale":"2023-09-23T20:27:15.261915+0000","last_undegraded":"2023-09-23T20:27:15.261915+0000","last_fullsized":"2023-09-23T20:27:15.261915+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":194,"log_dups_size":0,"ondisk_log_size":194,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:07:16.114968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00037995999999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1025,"num_read_kb":1201,"num_write":173,"num_write_kb":1276,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":576,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-09-23T20:27:16.081302+0000","last_change":"2023-09-23T20:27:04.493422+0000","last_active":"2023-09-23T20:27:16.081302+0000","last_peered":"2023-09-23T20:27:16.081302+0000","last_clean":"2023-09-23T20:27:16.081302+0000","last_became_active":"2023-09-23T20:25:21.159200+0000","last_became_peered":"2023-09-23T20:25:21.159200+0000","last_unstale":"2023-09-23T20:27:16.081302+0000","last_undegraded":"2023-09-23T20:27:16.081302+0000","last_fullsized":"2023-09-23T20:27:16.081302+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:09:47.213130+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":67,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T20:27:20.425931+0000","last_change":"2023-09-23T20:25:02.984205+0000","last_active":"2023-09-23T20:27:20.425931+0000","last_peered":"2023-09-23T20:27:20.425931+0000","last_clean":"2023-09-23T20:27:20.425931+0000","last_became_active":"2023-09-23T20:25:02.983758+0000","last_became_peered":"2023-09-23T20:25:02.983758+0000","last_unstale":"2023-09-23T20:27:20.425931+0000","last_undegraded":"2023-09-23T20:27:20.425931+0000","last_fullsized":"2023-09-23T20:27:20.425931+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:35:14.573101+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":161,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-09-23T20:27:14.710901+0000","last_change":"2023-09-23T20:12:45.609829+0000","last_active":"2023-09-23T20:27:14.710901+0000","last_peered":"2023-09-23T20:27:14.710901+0000","last_clean":"2023-09-23T20:27:14.710901+0000","last_became_active":"2023-09-23T20:12:45.609467+0000","last_became_peered":"2023-09-23T20:12:45.609467+0000","last_unstale":"2023-09-23T20:27:14.710901+0000","last_undegraded":"2023-09-23T20:27:14.710901+0000","last_fullsized":"2023-09-23T20:27:14.710901+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:27:31.490555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.4","version":"0'0","reported_seq":11,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-09-23T20:27:15.060950+0000","last_change":"2023-09-23T20:27:14.267770+0000","last_active":"2023-09-23T20:27:15.060950+0000","last_peered":"2023-09-23T20:27:15.060950+0000","last_clean":"2023-09-23T20:27:15.060950+0000","last_became_active":"2023-09-23T20:27:14.266851+0000","last_became_peered":"2023-09-23T20:27:14.266851+0000","last_unstale":"2023-09-23T20:27:15.060950+0000","last_undegraded":"2023-09-23T20:27:15.060950+0000","last_fullsized":"2023-09-23T20:27:15.060950+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:50:38.592759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":444,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-09-23T20:27:15.060928+0000","last_change":"2023-09-23T20:27:12.386203+0000","last_active":"2023-09-23T20:27:15.060928+0000","last_peered":"2023-09-23T20:27:15.060928+0000","last_clean":"2023-09-23T20:27:15.060928+0000","last_became_active":"2023-09-23T20:27:11.962975+0000","last_became_peered":"2023-09-23T20:27:11.962975+0000","last_unstale":"2023-09-23T20:27:15.060928+0000","last_undegraded":"2023-09-23T20:27:15.060928+0000","last_fullsized":"2023-09-23T20:27:15.060928+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:12.386098+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:12.386098+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:23:49.746955+0000","scrub_duration":0.0019500940000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"86'80","reported_seq":466,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-09-23T20:27:16.081204+0000","last_change":"2023-09-23T20:20:46.950784+0000","last_active":"2023-09-23T20:27:16.081204+0000","last_peered":"2023-09-23T20:27:16.081204+0000","last_clean":"2023-09-23T20:27:16.081204+0000","last_became_active":"2023-09-23T20:20:46.950071+0000","last_became_peered":"2023-09-23T20:20:46.950071+0000","last_unstale":"2023-09-23T20:27:16.081204+0000","last_undegraded":"2023-09-23T20:27:16.081204+0000","last_fullsized":"2023-09-23T20:27:16.081204+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":80,"log_dups_size":0,"ondisk_log_size":80,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:11:06.812404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035115,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":395,"num_read_kb":1370,"num_write":66,"num_write_kb":1061,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":181,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-09-23T20:27:14.262192+0000","last_change":"2023-09-23T20:25:02.942709+0000","last_active":"2023-09-23T20:27:14.262192+0000","last_peered":"2023-09-23T20:27:14.262192+0000","last_clean":"2023-09-23T20:27:14.262192+0000","last_became_active":"2023-09-23T20:25:02.942323+0000","last_became_peered":"2023-09-23T20:25:02.942323+0000","last_unstale":"2023-09-23T20:27:14.262192+0000","last_undegraded":"2023-09-23T20:27:14.262192+0000","last_fullsized":"2023-09-23T20:27:14.262192+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:26:12.039101+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":197,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-09-23T20:27:14.261960+0000","last_change":"2023-09-23T20:25:39.641452+0000","last_active":"2023-09-23T20:27:14.261960+0000","last_peered":"2023-09-23T20:27:14.261960+0000","last_clean":"2023-09-23T20:27:14.261960+0000","last_became_active":"2023-09-23T20:25:39.638987+0000","last_became_peered":"2023-09-23T20:25:39.638987+0000","last_unstale":"2023-09-23T20:27:14.261960+0000","last_undegraded":"2023-09-23T20:27:14.261960+0000","last_fullsized":"2023-09-23T20:27:14.261960+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:33:59.955461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"0'0","reported_seq":11,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-09-23T20:27:15.060850+0000","last_change":"2023-09-23T20:27:14.712242+0000","last_active":"2023-09-23T20:27:15.060850+0000","last_peered":"2023-09-23T20:27:15.060850+0000","last_clean":"2023-09-23T20:27:15.060850+0000","last_became_active":"2023-09-23T20:27:14.711916+0000","last_became_peered":"2023-09-23T20:27:14.711916+0000","last_unstale":"2023-09-23T20:27:15.060850+0000","last_undegraded":"2023-09-23T20:27:15.060850+0000","last_fullsized":"2023-09-23T20:27:15.060850+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:08:51.484867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":442,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-09-23T20:27:14.710939+0000","last_change":"2023-09-23T20:27:11.963158+0000","last_active":"2023-09-23T20:27:14.710939+0000","last_peered":"2023-09-23T20:27:14.710939+0000","last_clean":"2023-09-23T20:27:14.710939+0000","last_became_active":"2023-09-23T20:27:11.962792+0000","last_became_peered":"2023-09-23T20:27:11.962792+0000","last_unstale":"2023-09-23T20:27:14.710939+0000","last_undegraded":"2023-09-23T20:27:14.710939+0000","last_fullsized":"2023-09-23T20:27:14.710939+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:09:47.213130+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"86'102","reported_seq":726,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-09-23T20:27:16.081743+0000","last_change":"2023-09-23T20:20:14.881975+0000","last_active":"2023-09-23T20:27:16.081743+0000","last_peered":"2023-09-23T20:27:16.081743+0000","last_clean":"2023-09-23T20:27:16.081743+0000","last_became_active":"2023-09-23T20:20:14.881544+0000","last_became_peered":"2023-09-23T20:20:14.881544+0000","last_unstale":"2023-09-23T20:27:16.081743+0000","last_undegraded":"2023-09-23T20:27:16.081743+0000","last_fullsized":"2023-09-23T20:27:16.081743+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":102,"log_dups_size":0,"ondisk_log_size":102,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:34.594972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030656500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":963,"num_read_kb":1234,"num_write":60,"num_write_kb":225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":177,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-09-23T20:27:14.260760+0000","last_change":"2023-09-23T20:25:39.631751+0000","last_active":"2023-09-23T20:27:14.260760+0000","last_peered":"2023-09-23T20:27:14.260760+0000","last_clean":"2023-09-23T20:27:14.260760+0000","last_became_active":"2023-09-23T20:25:39.631136+0000","last_became_peered":"2023-09-23T20:25:39.631136+0000","last_unstale":"2023-09-23T20:27:14.260760+0000","last_undegraded":"2023-09-23T20:27:14.260760+0000","last_fullsized":"2023-09-23T20:27:14.260760+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:08:17.020722+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":149,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-09-23T20:27:14.262542+0000","last_change":"2023-09-23T20:20:14.888874+0000","last_active":"2023-09-23T20:27:14.262542+0000","last_peered":"2023-09-23T20:27:14.262542+0000","last_clean":"2023-09-23T20:27:14.262542+0000","last_became_active":"2023-09-23T20:20:14.887724+0000","last_became_peered":"2023-09-23T20:20:14.887724+0000","last_unstale":"2023-09-23T20:27:14.262542+0000","last_undegraded":"2023-09-23T20:27:14.262542+0000","last_fullsized":"2023-09-23T20:27:14.262542+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:09:24.510326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"0'0","reported_seq":10,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-09-23T20:27:14.270496+0000","last_change":"2023-09-23T20:27:14.270161+0000","last_active":"2023-09-23T20:27:14.270496+0000","last_peered":"2023-09-23T20:27:14.270496+0000","last_clean":"2023-09-23T20:27:14.270496+0000","last_became_active":"2023-09-23T20:27:14.269904+0000","last_became_peered":"2023-09-23T20:27:14.269904+0000","last_unstale":"2023-09-23T20:27:14.270496+0000","last_undegraded":"2023-09-23T20:27:14.270496+0000","last_fullsized":"2023-09-23T20:27:14.270496+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:41:02.917630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":443,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-09-23T20:27:14.264342+0000","last_change":"2023-09-23T20:27:12.810010+0000","last_active":"2023-09-23T20:27:14.264342+0000","last_peered":"2023-09-23T20:27:14.264342+0000","last_clean":"2023-09-23T20:27:14.264342+0000","last_became_active":"2023-09-23T20:27:11.959614+0000","last_became_peered":"2023-09-23T20:27:11.959614+0000","last_unstale":"2023-09-23T20:27:14.264342+0000","last_undegraded":"2023-09-23T20:27:14.264342+0000","last_fullsized":"2023-09-23T20:27:14.264342+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:12.809933+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:12.809933+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:48:13.147428+0000","scrub_duration":0.001626628,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":71,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T20:27:20.426333+0000","last_change":"2023-09-23T20:25:03.007736+0000","last_active":"2023-09-23T20:27:20.426333+0000","last_peered":"2023-09-23T20:27:20.426333+0000","last_clean":"2023-09-23T20:27:20.426333+0000","last_became_active":"2023-09-23T20:25:02.973689+0000","last_became_peered":"2023-09-23T20:25:02.973689+0000","last_unstale":"2023-09-23T20:27:20.426333+0000","last_undegraded":"2023-09-23T20:27:20.426333+0000","last_fullsized":"2023-09-23T20:27:20.426333+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:37:25.643774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"86'128","reported_seq":544,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-09-23T20:27:15.061011+0000","last_change":"2023-09-23T20:20:46.939256+0000","last_active":"2023-09-23T20:27:15.061011+0000","last_peered":"2023-09-23T20:27:15.061011+0000","last_clean":"2023-09-23T20:27:15.061011+0000","last_became_active":"2023-09-23T20:20:46.938756+0000","last_became_peered":"2023-09-23T20:20:46.938756+0000","last_unstale":"2023-09-23T20:27:15.061011+0000","last_undegraded":"2023-09-23T20:27:15.061011+0000","last_fullsized":"2023-09-23T20:27:15.061011+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:52:18.161436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000293202,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":579,"num_read_kb":1473,"num_write":96,"num_write_kb":1062,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":67,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T20:27:20.425641+0000","last_change":"2023-09-23T20:25:02.971740+0000","last_active":"2023-09-23T20:27:20.425641+0000","last_peered":"2023-09-23T20:27:20.425641+0000","last_clean":"2023-09-23T20:27:20.425641+0000","last_became_active":"2023-09-23T20:25:02.970412+0000","last_became_peered":"2023-09-23T20:25:02.970412+0000","last_unstale":"2023-09-23T20:27:20.425641+0000","last_undegraded":"2023-09-23T20:27:20.425641+0000","last_fullsized":"2023-09-23T20:27:20.425641+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:53:17.247216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"0'0","reported_seq":11,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-09-23T20:27:16.081507+0000","last_change":"2023-09-23T20:27:14.711764+0000","last_active":"2023-09-23T20:27:16.081507+0000","last_peered":"2023-09-23T20:27:16.081507+0000","last_clean":"2023-09-23T20:27:16.081507+0000","last_became_active":"2023-09-23T20:27:14.711409+0000","last_became_peered":"2023-09-23T20:27:14.711409+0000","last_unstale":"2023-09-23T20:27:16.081507+0000","last_undegraded":"2023-09-23T20:27:16.081507+0000","last_fullsized":"2023-09-23T20:27:16.081507+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:46:35.659960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.4","version":"90'161","reported_seq":446,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T20:27:20.426080+0000","last_change":"2023-09-23T20:27:13.244744+0000","last_active":"2023-09-23T20:27:20.426080+0000","last_peered":"2023-09-23T20:27:20.426080+0000","last_clean":"2023-09-23T20:27:20.426080+0000","last_became_active":"2023-09-23T20:27:11.960349+0000","last_became_peered":"2023-09-23T20:27:11.960349+0000","last_unstale":"2023-09-23T20:27:20.426080+0000","last_undegraded":"2023-09-23T20:27:20.426080+0000","last_fullsized":"2023-09-23T20:27:20.426080+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:13.244651+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.244651+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:22:26.451174+0000","scrub_duration":0.0019955509999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"86'117","reported_seq":738,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-09-23T20:27:14.711029+0000","last_change":"2023-09-23T20:25:40.617386+0000","last_active":"2023-09-23T20:27:14.711029+0000","last_peered":"2023-09-23T20:27:14.711029+0000","last_clean":"2023-09-23T20:27:14.711029+0000","last_became_active":"2023-09-23T20:25:40.616747+0000","last_became_peered":"2023-09-23T20:25:40.616747+0000","last_unstale":"2023-09-23T20:27:14.711029+0000","last_undegraded":"2023-09-23T20:27:14.711029+0000","last_fullsized":"2023-09-23T20:27:14.711029+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:46:44.551561+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046891400000000002,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":893,"num_read_kb":922,"num_write":82,"num_write_kb":4193,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":169,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-09-23T20:27:14.261950+0000","last_change":"2023-09-23T20:20:46.937163+0000","last_active":"2023-09-23T20:27:14.261950+0000","last_peered":"2023-09-23T20:27:14.261950+0000","last_clean":"2023-09-23T20:27:14.261950+0000","last_became_active":"2023-09-23T20:20:46.936846+0000","last_became_peered":"2023-09-23T20:20:46.936846+0000","last_unstale":"2023-09-23T20:27:14.261950+0000","last_undegraded":"2023-09-23T20:27:14.261950+0000","last_fullsized":"2023-09-23T20:27:14.261950+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:01:32.798855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":160,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-09-23T20:27:14.264723+0000","last_change":"2023-09-23T20:12:43.765908+0000","last_active":"2023-09-23T20:27:14.264723+0000","last_peered":"2023-09-23T20:27:14.264723+0000","last_clean":"2023-09-23T20:27:14.264723+0000","last_became_active":"2023-09-23T20:12:43.765558+0000","last_became_peered":"2023-09-23T20:12:43.765558+0000","last_unstale":"2023-09-23T20:27:14.264723+0000","last_undegraded":"2023-09-23T20:27:14.264723+0000","last_fullsized":"2023-09-23T20:27:14.264723+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:35:43.809898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":151,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-09-23T20:27:14.262453+0000","last_change":"2023-09-23T20:20:14.888484+0000","last_active":"2023-09-23T20:27:14.262453+0000","last_peered":"2023-09-23T20:27:14.262453+0000","last_clean":"2023-09-23T20:27:14.262453+0000","last_became_active":"2023-09-23T20:20:14.887219+0000","last_became_peered":"2023-09-23T20:20:14.887219+0000","last_unstale":"2023-09-23T20:27:14.262453+0000","last_undegraded":"2023-09-23T20:27:14.262453+0000","last_fullsized":"2023-09-23T20:27:14.262453+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:13:53.763649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":54,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-09-23T20:27:16.077973+0000","last_change":"2023-09-23T20:25:39.639268+0000","last_active":"2023-09-23T20:27:16.077973+0000","last_peered":"2023-09-23T20:27:16.077973+0000","last_clean":"2023-09-23T20:27:16.077973+0000","last_became_active":"2023-09-23T20:25:39.638922+0000","last_became_peered":"2023-09-23T20:25:39.638922+0000","last_unstale":"2023-09-23T20:27:16.077973+0000","last_undegraded":"2023-09-23T20:27:16.077973+0000","last_fullsized":"2023-09-23T20:27:16.077973+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:09:04.040098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":181,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-09-23T20:27:15.261974+0000","last_change":"2023-09-23T20:20:46.938981+0000","last_active":"2023-09-23T20:27:15.261974+0000","last_peered":"2023-09-23T20:27:15.261974+0000","last_clean":"2023-09-23T20:27:15.261974+0000","last_became_active":"2023-09-23T20:20:46.938627+0000","last_became_peered":"2023-09-23T20:20:46.938627+0000","last_unstale":"2023-09-23T20:27:15.261974+0000","last_undegraded":"2023-09-23T20:27:15.261974+0000","last_fullsized":"2023-09-23T20:27:15.261974+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:44:43.177086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":224,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-09-23T20:27:14.266177+0000","last_change":"2023-09-23T20:25:02.945142+0000","last_active":"2023-09-23T20:27:14.266177+0000","last_peered":"2023-09-23T20:27:14.266177+0000","last_clean":"2023-09-23T20:27:14.266177+0000","last_became_active":"2023-09-23T20:25:02.944653+0000","last_became_peered":"2023-09-23T20:25:02.944653+0000","last_unstale":"2023-09-23T20:27:14.266177+0000","last_undegraded":"2023-09-23T20:27:14.266177+0000","last_fullsized":"2023-09-23T20:27:14.266177+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:38:12.927982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":164,"reported_epoch":219,"state":"active+clean","last_fresh":"2023-09-23T20:27:15.262052+0000","last_change":"2023-09-23T20:12:43.753383+0000","last_active":"2023-09-23T20:27:15.262052+0000","last_peered":"2023-09-23T20:27:15.262052+0000","last_clean":"2023-09-23T20:27:15.262052+0000","last_became_active":"2023-09-23T20:12:43.752921+0000","last_became_peered":"2023-09-23T20:12:43.752921+0000","last_unstale":"2023-09-23T20:27:15.262052+0000","last_undegraded":"2023-09-23T20:27:15.262052+0000","last_fullsized":"2023-09-23T20:27:15.262052+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:28:26.982998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":224,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-09-23T20:27:14.265447+0000","last_change":"2023-09-23T20:25:03.103656+0000","last_active":"2023-09-23T20:27:14.265447+0000","last_peered":"2023-09-23T20:27:14.265447+0000","last_clean":"2023-09-23T20:27:14.265447+0000","last_became_active":"2023-09-23T20:25:03.103311+0000","last_became_peered":"2023-09-23T20:25:03.103311+0000","last_unstale":"2023-09-23T20:27:14.265447+0000","last_undegraded":"2023-09-23T20:27:14.265447+0000","last_fullsized":"2023-09-23T20:27:14.265447+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:55:54.677030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":76,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T20:27:20.426428+0000","last_change":"2023-09-23T20:25:39.629869+0000","last_active":"2023-09-23T20:27:20.426428+0000","last_peered":"2023-09-23T20:27:20.426428+0000","last_clean":"2023-09-23T20:27:20.426428+0000","last_became_active":"2023-09-23T20:25:39.629543+0000","last_became_peered":"2023-09-23T20:25:39.629543+0000","last_unstale":"2023-09-23T20:27:20.426428+0000","last_undegraded":"2023-09-23T20:27:20.426428+0000","last_fullsized":"2023-09-23T20:27:20.426428+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:24:48.220679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":75,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T20:27:20.425786+0000","last_change":"2023-09-23T20:25:39.622819+0000","last_active":"2023-09-23T20:27:20.425786+0000","last_peered":"2023-09-23T20:27:20.425786+0000","last_clean":"2023-09-23T20:27:20.425786+0000","last_became_active":"2023-09-23T20:25:39.622484+0000","last_became_peered":"2023-09-23T20:25:39.622484+0000","last_unstale":"2023-09-23T20:27:20.425786+0000","last_undegraded":"2023-09-23T20:27:20.425786+0000","last_fullsized":"2023-09-23T20:27:20.425786+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:51:53.973261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":197,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-09-23T20:27:14.262329+0000","last_change":"2023-09-23T20:25:39.668882+0000","last_active":"2023-09-23T20:27:14.262329+0000","last_peered":"2023-09-23T20:27:14.262329+0000","last_clean":"2023-09-23T20:27:14.262329+0000","last_became_active":"2023-09-23T20:25:39.637128+0000","last_became_peered":"2023-09-23T20:25:39.637128+0000","last_unstale":"2023-09-23T20:27:14.262329+0000","last_undegraded":"2023-09-23T20:27:14.262329+0000","last_fullsized":"2023-09-23T20:27:14.262329+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:56.406298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":197,"reported_epoch":218,"state":"active+clean","last_fresh":"2023-09-23T20:27:14.264806+0000","last_change":"2023-09-23T20:24:53.089249+0000","last_active":"2023-09-23T20:27:14.264806+0000","last_peered":"2023-09-23T20:27:14.264806+0000","last_clean":"2023-09-23T20:27:14.264806+0000","last_became_active":"2023-09-23T20:24:53.088952+0000","last_became_peered":"2023-09-23T20:24:53.088952+0000","last_unstale":"2023-09-23T20:27:14.264806+0000","last_undegraded":"2023-09-23T20:27:14.264806+0000","last_fullsized":"2023-09-23T20:27:14.264806+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:38:03.216069+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":17301504,"num_objects":264,"num_object_clones":0,"num_object_copies":792,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":264,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":264,"num_write_kb":16896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":63832064,"data_stored":63832064,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":264,"ondisk_log_size":264,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":15,"num_bytes_recovered":1356,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":25181,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":757038,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":357,"num_read_kb":667,"num_write":173,"num_write_kb":1756,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1550762,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1771,"ondisk_log_size":1771,"up":33,"acting":33,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6082,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201474,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":155464,"kb_used_data":40960,"kb_used_omap":0,"kb_used_meta":114496,"kb_avail":93589688,"statfs":{"total":95995035648,"available":95835840512,"internally_reserved":0,"allocated":41943040,"data_stored":41227744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":117243904},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:26:23 2023","interfaces":[{"interface":"back","average":{"1min":1.861,"5min":1.2150000000000001,"15min":0.78700000000000003},"min":{"1min":0.247,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":10.907,"5min":13.092000000000001,"15min":13.092000000000001},"last":0.44500000000000001},{"interface":"front","average":{"1min":3.5369999999999999,"5min":1.5940000000000001,"15min":0.91000000000000003},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":9.8610000000000007,"5min":9.8610000000000007,"15min":9.8610000000000007},"last":0.52600000000000002}]},{"osd":1,"last update":"Sat Sep 23 20:26:58 2023","interfaces":[{"interface":"back","average":{"1min":1.2929999999999999,"5min":1.6499999999999999,"15min":0.92500000000000004},"min":{"1min":0.28599999999999998,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":3.1920000000000002,"5min":17.795999999999999,"15min":17.795999999999999},"last":0.41499999999999998},{"interface":"front","average":{"1min":2.2989999999999999,"5min":1.504,"15min":0.90900000000000003},"min":{"1min":0.33400000000000002,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":6.4950000000000001,"5min":13.250999999999999,"15min":13.250999999999999},"last":0.58799999999999997}]},{"osd":2,"last update":"Sat Sep 23 20:26:28 2023","interfaces":[{"interface":"back","average":{"1min":3.2610000000000001,"5min":1.6080000000000001,"15min":0.91500000000000004},"min":{"1min":0.39000000000000001,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":8.1270000000000007,"5min":9.8300000000000001,"15min":9.8300000000000001},"last":0.48599999999999999},{"interface":"front","average":{"1min":3.1850000000000001,"5min":1.708,"15min":0.95999999999999996},"min":{"1min":0.48099999999999998,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":6.6699999999999999,"5min":11.423999999999999,"15min":11.423999999999999},"last":0.60399999999999998}]},{"osd":3,"last update":"Sat Sep 23 20:26:58 2023","interfaces":[{"interface":"back","average":{"1min":2.4049999999999998,"5min":1.756,"15min":1.022},"min":{"1min":0.38100000000000001,"5min":0.314,"15min":0.314},"max":{"1min":6.891,"5min":6.891,"15min":6.891},"last":0.47199999999999998},{"interface":"front","average":{"1min":2.4319999999999999,"5min":1.6279999999999999,"15min":0.99199999999999999},"min":{"1min":0.374,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":6.9050000000000002,"5min":9.7840000000000007,"15min":9.7840000000000007},"last":0.63}]},{"osd":4,"last update":"Sat Sep 23 20:26:58 2023","interfaces":[{"interface":"back","average":{"1min":1.6479999999999999,"5min":1.1619999999999999,"15min":0.71899999999999997},"min":{"1min":0.28499999999999998,"5min":0.27000000000000002,"15min":0.26700000000000002},"max":{"1min":5.3540000000000001,"5min":13.141999999999999,"15min":13.141999999999999},"last":0.504},{"interface":"front","average":{"1min":1.5740000000000001,"5min":1.6819999999999999,"15min":0.91400000000000003},"min":{"1min":0.29499999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":5.9279999999999999,"5min":10.853,"15min":10.853},"last":0.57399999999999995}]},{"osd":5,"last update":"Sat Sep 23 20:26:58 2023","interfaces":[{"interface":"back","average":{"1min":1.9930000000000001,"5min":1.643,"15min":0.90500000000000003},"min":{"1min":0.31,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":10.362,"5min":10.563000000000001,"15min":10.563000000000001},"last":0.56000000000000005},{"interface":"front","average":{"1min":1.956,"5min":1.5669999999999999,"15min":0.91100000000000003},"min":{"1min":0.41799999999999998,"5min":0.34699999999999998,"15min":0.31},"max":{"1min":7.9029999999999996,"5min":13.292,"15min":13.292},"last":0.64500000000000002}]},{"osd":6,"last update":"Sat Sep 23 20:26:58 2023","interfaces":[{"interface":"back","average":{"1min":1.9890000000000001,"5min":1.4179999999999999,"15min":0.86899999999999999},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":6.2210000000000001,"5min":13.185,"15min":13.185},"last":0.54800000000000004},{"interface":"front","average":{"1min":1.9199999999999999,"5min":1.4890000000000001,"15min":0.89100000000000001},"min":{"1min":0.378,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":6.4370000000000003,"5min":9.9120000000000008,"15min":9.9120000000000008},"last":0.67200000000000004}]}]},{"osd":6,"up_from":47,"seq":201863463110,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":87424,"kb_used_data":6264,"kb_used_omap":0,"kb_used_meta":81152,"kb_avail":93657728,"statfs":{"total":95995035648,"available":95905513472,"internally_reserved":0,"allocated":6414336,"data_stored":5707105,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":83099648},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:26:26 2023","interfaces":[{"interface":"back","average":{"1min":1.7270000000000001,"5min":1.23,"15min":0.79500000000000004},"min":{"1min":0.221,"5min":0.221,"15min":0.221},"max":{"1min":8.1940000000000008,"5min":8.1940000000000008,"15min":8.1940000000000008},"last":0.48599999999999999},{"interface":"front","average":{"1min":1.3069999999999999,"5min":1.4850000000000001,"15min":0.87},"min":{"1min":0.26300000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":4.0609999999999999,"5min":12.512,"15min":12.512},"last":0.53800000000000003}]},{"osd":1,"last update":"Sat Sep 23 20:27:02 2023","interfaces":[{"interface":"back","average":{"1min":2.6379999999999999,"5min":1.8839999999999999,"15min":1.0109999999999999},"min":{"1min":0.28899999999999998,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":8.3520000000000003,"5min":18.001000000000001,"15min":18.001000000000001},"last":0.51800000000000002},{"interface":"front","average":{"1min":2.391,"5min":1.569,"15min":0.88800000000000001},"min":{"1min":0.28699999999999998,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":8.532,"5min":13.91,"15min":13.91},"last":0.42599999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:26:26 2023","interfaces":[{"interface":"back","average":{"1min":2.153,"5min":1.385,"15min":0.85799999999999998},"min":{"1min":0.32800000000000001,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":8.0090000000000003,"5min":13.856,"15min":13.856},"last":0.47399999999999998},{"interface":"front","average":{"1min":2.0720000000000001,"5min":1.284,"15min":0.83399999999999996},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":6.8140000000000001,"5min":6.8140000000000001,"15min":6.8140000000000001},"last":0.504}]},{"osd":3,"last update":"Sat Sep 23 20:27:02 2023","interfaces":[{"interface":"back","average":{"1min":2.105,"5min":1.833,"15min":1.0189999999999999},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":8.2989999999999995,"5min":9.4239999999999995,"15min":9.4239999999999995},"last":0.495},{"interface":"front","average":{"1min":2.754,"5min":1.8280000000000001,"15min":1.0600000000000001},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":8.3770000000000007,"5min":8.6050000000000004,"15min":8.6050000000000004},"last":0.55600000000000005}]},{"osd":4,"last update":"Sat Sep 23 20:26:48 2023","interfaces":[{"interface":"back","average":{"1min":1.5049999999999999,"5min":1.2669999999999999,"15min":0.72999999999999998},"min":{"1min":0.27600000000000002,"5min":0.27200000000000002,"15min":0.26200000000000001},"max":{"1min":10.173,"5min":10.173,"15min":10.173},"last":0.33400000000000002},{"interface":"front","average":{"1min":1.962,"5min":1.3540000000000001,"15min":0.78000000000000003},"min":{"1min":0.312,"5min":0.29799999999999999,"15min":0.28499999999999998},"max":{"1min":8.3249999999999993,"5min":8.3249999999999993,"15min":8.3249999999999993},"last":0.36699999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:26:48 2023","interfaces":[{"interface":"back","average":{"1min":1.202,"5min":1.4119999999999999,"15min":0.80300000000000005},"min":{"1min":0.31,"5min":0.31,"15min":0.29799999999999999},"max":{"1min":4.4580000000000002,"5min":14.085000000000001,"15min":14.085000000000001},"last":0.443},{"interface":"front","average":{"1min":2.831,"5min":1.8660000000000001,"15min":0.97299999999999998},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.29499999999999998},"max":{"1min":18.192,"5min":18.192,"15min":18.192},"last":0.45600000000000002}]},{"osd":7,"last update":"Sat Sep 23 20:27:02 2023","interfaces":[{"interface":"back","average":{"1min":1.946,"5min":1.538,"15min":0.877},"min":{"1min":0.36899999999999999,"5min":0.33500000000000002,"15min":0.32800000000000001},"max":{"1min":8.0350000000000001,"5min":10.217000000000001,"15min":10.217000000000001},"last":0.52900000000000003},{"interface":"front","average":{"1min":1.6459999999999999,"5min":1.6020000000000001,"15min":0.91400000000000003},"min":{"1min":0.36499999999999999,"5min":0.34999999999999998,"15min":0.315},"max":{"1min":8.407,"5min":9.7360000000000007,"15min":9.7360000000000007},"last":0.57799999999999996}]}]},{"osd":1,"up_from":99,"seq":425201762385,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":172952,"kb_used_data":2640,"kb_used_omap":6,"kb_used_meta":170297,"kb_avail":93572200,"statfs":{"total":95995035648,"available":95817932800,"internally_reserved":0,"allocated":2703360,"data_stored":1563490,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6696,"internal_metadata":174384600},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:26:54 2023","interfaces":[{"interface":"back","average":{"1min":2.6600000000000001,"5min":2.052,"15min":0.93500000000000005},"min":{"1min":0.215,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":14.384,"5min":40.113,"15min":40.113},"last":0.42799999999999999},{"interface":"front","average":{"1min":3.1869999999999998,"5min":1.8120000000000001,"15min":0.87},"min":{"1min":0.23699999999999999,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":22.210000000000001,"5min":42.994999999999997,"15min":42.994999999999997},"last":0.505}]},{"osd":2,"last update":"Sat Sep 23 20:26:27 2023","interfaces":[{"interface":"back","average":{"1min":1.6479999999999999,"5min":1.8029999999999999,"15min":0.85499999999999998},"min":{"1min":0.27900000000000003,"5min":0.253,"15min":0.253},"max":{"1min":9.6839999999999993,"5min":39.962000000000003,"15min":39.962000000000003},"last":0.46500000000000002},{"interface":"front","average":{"1min":2.806,"5min":1.794,"15min":0.96599999999999997},"min":{"1min":0.26700000000000002,"5min":0.246,"15min":0.246},"max":{"1min":8.7910000000000004,"5min":47.442,"15min":47.442},"last":0.54800000000000004}]},{"osd":3,"last update":"Sat Sep 23 20:27:00 2023","interfaces":[{"interface":"back","average":{"1min":1.9099999999999999,"5min":1.4039999999999999,"15min":0.83399999999999996},"min":{"1min":0.309,"5min":0.246,"15min":0.246},"max":{"1min":9.0410000000000004,"5min":9.0410000000000004,"15min":9.0410000000000004},"last":0.49399999999999999},{"interface":"front","average":{"1min":2.4319999999999999,"5min":1.4710000000000001,"15min":0.76200000000000001},"min":{"1min":0.29699999999999999,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":15.147,"5min":15.147,"15min":15.147},"last":0.58099999999999996}]},{"osd":4,"last update":"Sat Sep 23 20:26:54 2023","interfaces":[{"interface":"back","average":{"1min":2.5790000000000002,"5min":1.5900000000000001,"15min":0.94099999999999995},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":15.353,"5min":15.353,"15min":15.353},"last":0.628},{"interface":"front","average":{"1min":1.7609999999999999,"5min":1.536,"15min":0.93500000000000005},"min":{"1min":0.32400000000000001,"5min":0.312,"15min":0.312},"max":{"1min":6.2270000000000003,"5min":8.7899999999999991,"15min":8.7899999999999991},"last":0.749}]},{"osd":5,"last update":"Sat Sep 23 20:26:54 2023","interfaces":[{"interface":"back","average":{"1min":2.6360000000000001,"5min":1.6220000000000001,"15min":0.96799999999999997},"min":{"1min":0.33000000000000002,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":7.2000000000000002,"5min":12.590999999999999,"15min":12.590999999999999},"last":0.64600000000000002},{"interface":"front","average":{"1min":2.2530000000000001,"5min":2.137,"15min":1.163},"min":{"1min":0.38300000000000001,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":6.3140000000000001,"5min":39.579000000000001,"15min":39.579000000000001},"last":0.73599999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:26:54 2023","interfaces":[{"interface":"back","average":{"1min":2.9630000000000001,"5min":1.996,"15min":1.1100000000000001},"min":{"1min":0.42999999999999999,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":7.226,"5min":34.409999999999997,"15min":34.409999999999997},"last":0.61599999999999999},{"interface":"front","average":{"1min":2.964,"5min":2.2269999999999999,"15min":1.167},"min":{"1min":0.38100000000000001,"5min":0.377,"15min":0.377},"max":{"1min":15.313000000000001,"5min":35.554000000000002,"15min":35.554000000000002},"last":0.65800000000000003}]},{"osd":7,"last update":"Sat Sep 23 20:26:54 2023","interfaces":[{"interface":"back","average":{"1min":2.8719999999999999,"5min":1.8480000000000001,"15min":1.05},"min":{"1min":0.42499999999999999,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":9.7439999999999998,"5min":9.7439999999999998,"15min":9.7439999999999998},"last":0.70099999999999996},{"interface":"front","average":{"1min":2.3180000000000001,"5min":1.7729999999999999,"15min":1.0469999999999999},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":6.2519999999999998,"5min":9.2279999999999998,"15min":9.2279999999999998},"last":0.70999999999999996}]}]},{"osd":0,"up_from":94,"seq":403726925911,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":251696,"kb_used_data":40744,"kb_used_omap":7,"kb_used_meta":210936,"kb_avail":93493456,"statfs":{"total":95995035648,"available":95737298944,"internally_reserved":0,"allocated":41721856,"data_stored":40680620,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":215998481},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:26:56 2023","interfaces":[{"interface":"back","average":{"1min":3.0779999999999998,"5min":1.3400000000000001,"15min":0.68600000000000005},"min":{"1min":0.223,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":24.247,"5min":24.247,"15min":24.247},"last":0.38700000000000001},{"interface":"front","average":{"1min":2.9950000000000001,"5min":1.3160000000000001,"15min":0.69599999999999995},"min":{"1min":0.28999999999999998,"5min":0.222,"15min":0.222},"max":{"1min":15.994999999999999,"5min":15.994999999999999,"15min":15.994999999999999},"last":0.34599999999999997}]},{"osd":2,"last update":"Sat Sep 23 20:26:22 2023","interfaces":[{"interface":"back","average":{"1min":4.3319999999999999,"5min":1.8200000000000001,"15min":1.004},"min":{"1min":0.251,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":23.832999999999998,"5min":23.832999999999998,"15min":23.832999999999998},"last":0.28599999999999998},{"interface":"front","average":{"1min":1.3779999999999999,"5min":1.256,"15min":0.749},"min":{"1min":0.223,"5min":0.223,"15min":0.223},"max":{"1min":6.0629999999999997,"5min":9.1940000000000008,"15min":9.1940000000000008},"last":0.40100000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:26:56 2023","interfaces":[{"interface":"back","average":{"1min":3.3109999999999999,"5min":1.7070000000000001,"15min":0.96699999999999997},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":23.748999999999999,"5min":23.748999999999999,"15min":23.748999999999999},"last":0.33100000000000002},{"interface":"front","average":{"1min":3.9809999999999999,"5min":1.5189999999999999,"15min":0.78700000000000003},"min":{"1min":0.315,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":24.274999999999999,"5min":24.274999999999999,"15min":24.274999999999999},"last":0.52900000000000003}]},{"osd":4,"last update":"Sat Sep 23 20:26:22 2023","interfaces":[{"interface":"back","average":{"1min":4.6619999999999999,"5min":1.9430000000000001,"15min":1.046},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":23.876999999999999,"5min":23.876999999999999,"15min":23.876999999999999},"last":0.55500000000000005},{"interface":"front","average":{"1min":2.8839999999999999,"5min":1.587,"15min":0.93799999999999994},"min":{"1min":0.34799999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":23.681999999999999,"5min":23.681999999999999,"15min":23.681999999999999},"last":0.69099999999999995}]},{"osd":5,"last update":"Sat Sep 23 20:26:22 2023","interfaces":[{"interface":"back","average":{"1min":4.3259999999999996,"5min":2.2290000000000001,"15min":1.1819999999999999},"min":{"1min":0.39100000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":24.198,"5min":24.198,"15min":24.198},"last":0.73799999999999999},{"interface":"front","average":{"1min":2.0169999999999999,"5min":1.754,"15min":1.0369999999999999},"min":{"1min":0.37,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":6.9269999999999996,"5min":11.167999999999999,"15min":11.167999999999999},"last":0.60699999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:26:22 2023","interfaces":[{"interface":"back","average":{"1min":3.694,"5min":1.5760000000000001,"15min":0.95799999999999996},"min":{"1min":0.45400000000000001,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":24.408000000000001,"5min":24.408000000000001,"15min":24.408000000000001},"last":0.76000000000000001},{"interface":"front","average":{"1min":3.5230000000000001,"5min":1.526,"15min":0.94999999999999996},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":25.271000000000001,"5min":25.271000000000001,"15min":25.271000000000001},"last":0.63500000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:26:22 2023","interfaces":[{"interface":"back","average":{"1min":3.3199999999999998,"5min":1.7190000000000001,"15min":1.0249999999999999},"min":{"1min":0.442,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":25.359000000000002,"5min":25.359000000000002,"15min":25.359000000000002},"last":0.57099999999999995},{"interface":"front","average":{"1min":5.298,"5min":2.181,"15min":1.1699999999999999},"min":{"1min":0.499,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":23.905999999999999,"5min":23.905999999999999,"15min":23.905999999999999},"last":0.71799999999999997}]}]},{"osd":2,"up_from":104,"seq":446676598859,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":154992,"kb_used_data":6888,"kb_used_omap":11,"kb_used_meta":148084,"kb_avail":93590160,"statfs":{"total":95995035648,"available":95836323840,"internally_reserved":0,"allocated":7053312,"data_stored":6016784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11878,"internal_metadata":151638426},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:26:30 2023","interfaces":[{"interface":"back","average":{"1min":2.8090000000000002,"5min":1.706,"15min":0.81999999999999995},"min":{"1min":0.21099999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":9.5980000000000008,"5min":12.811,"15min":12.811},"last":0.59399999999999997},{"interface":"front","average":{"1min":2.1019999999999999,"5min":1.2709999999999999,"15min":0.68700000000000006},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":15.746,"5min":15.746,"15min":15.746},"last":0.58399999999999996}]},{"osd":1,"last update":"Sat Sep 23 20:26:30 2023","interfaces":[{"interface":"back","average":{"1min":2.9390000000000001,"5min":1.524,"15min":0.77000000000000002},"min":{"1min":0.28599999999999998,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":14.135999999999999,"5min":14.135999999999999,"15min":14.135999999999999},"last":0.503},{"interface":"front","average":{"1min":2.617,"5min":1.226,"15min":0.66400000000000003},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":13.718,"5min":13.718,"15min":13.718},"last":0.56100000000000005}]},{"osd":3,"last update":"Sat Sep 23 20:26:55 2023","interfaces":[{"interface":"back","average":{"1min":2.246,"5min":1.349,"15min":0.69099999999999995},"min":{"1min":0.26500000000000001,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":10.916,"5min":15.319000000000001,"15min":15.319000000000001},"last":0.53900000000000003},{"interface":"front","average":{"1min":2.294,"5min":1.552,"15min":0.78200000000000003},"min":{"1min":0.25800000000000001,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":12.494999999999999,"5min":12.494999999999999,"15min":12.494999999999999},"last":0.63700000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:26:30 2023","interfaces":[{"interface":"back","average":{"1min":1.996,"5min":1.234,"15min":0.80000000000000004},"min":{"1min":0.36299999999999999,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":8.2300000000000004,"5min":8.2300000000000004,"15min":8.2300000000000004},"last":0.92900000000000005},{"interface":"front","average":{"1min":2.3679999999999999,"5min":1.3979999999999999,"15min":0.85599999999999998},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":8.6530000000000005,"5min":8.6530000000000005,"15min":8.6530000000000005},"last":0.78000000000000003}]},{"osd":5,"last update":"Sat Sep 23 20:26:30 2023","interfaces":[{"interface":"back","average":{"1min":2.8319999999999999,"5min":1.577,"15min":0.96299999999999997},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":11.237,"5min":11.237,"15min":11.237},"last":0.90700000000000003},{"interface":"front","average":{"1min":3.452,"5min":1.6850000000000001,"15min":0.999},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":11.266999999999999,"5min":11.266999999999999,"15min":11.266999999999999},"last":0.751}]},{"osd":6,"last update":"Sat Sep 23 20:26:30 2023","interfaces":[{"interface":"back","average":{"1min":2.0129999999999999,"5min":1.294,"15min":0.86899999999999999},"min":{"1min":0.49099999999999999,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":6.2599999999999998,"5min":6.7939999999999996,"15min":6.7939999999999996},"last":0.83799999999999997},{"interface":"front","average":{"1min":2.7410000000000001,"5min":1.585,"15min":0.97699999999999998},"min":{"1min":0.41099999999999998,"5min":0.377,"15min":0.377},"max":{"1min":9.6600000000000001,"5min":9.6600000000000001,"15min":9.6600000000000001},"last":0.88}]},{"osd":7,"last update":"Sat Sep 23 20:26:30 2023","interfaces":[{"interface":"back","average":{"1min":3.1150000000000002,"5min":1.712,"15min":1.002},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":15.343999999999999,"5min":15.343999999999999,"15min":15.343999999999999},"last":0.79200000000000004},{"interface":"front","average":{"1min":2.6299999999999999,"5min":1.5840000000000001,"15min":0.96599999999999997},"min":{"1min":0.48899999999999999,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":9.1440000000000001,"5min":10.064,"15min":10.064},"last":0.86599999999999999}]}]},{"osd":3,"up_from":109,"seq":468151435333,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":147132,"kb_used_data":3124,"kb_used_omap":4,"kb_used_meta":143995,"kb_avail":93598020,"statfs":{"total":95995035648,"available":95844372480,"internally_reserved":0,"allocated":3198976,"data_stored":2149883,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4514,"internal_metadata":147451486},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:26:56 2023","interfaces":[{"interface":"back","average":{"1min":3.2530000000000001,"5min":2.319,"15min":1.0069999999999999},"min":{"1min":0.28999999999999998,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":15.589,"5min":67.305999999999997,"15min":67.305999999999997},"last":0.27300000000000002},{"interface":"front","average":{"1min":1.48,"5min":2.0339999999999998,"15min":0.93400000000000005},"min":{"1min":0.29699999999999999,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":9.0920000000000005,"5min":67.403999999999996,"15min":67.403999999999996},"last":0.36799999999999999}]},{"osd":1,"last update":"Sat Sep 23 20:26:56 2023","interfaces":[{"interface":"back","average":{"1min":3.0230000000000001,"5min":1.5069999999999999,"15min":0.76700000000000002},"min":{"1min":0.25,"5min":0.214,"15min":0.214},"max":{"1min":19.675999999999998,"5min":19.675999999999998,"15min":19.675999999999998},"last":0.28799999999999998},{"interface":"front","average":{"1min":2.4590000000000001,"5min":1.375,"15min":0.755},"min":{"1min":0.33200000000000002,"5min":0.245,"15min":0.245},"max":{"1min":11.644,"5min":11.644,"15min":11.644},"last":0.35199999999999998}]},{"osd":2,"last update":"Sat Sep 23 20:26:56 2023","interfaces":[{"interface":"back","average":{"1min":3.4369999999999998,"5min":1.7789999999999999,"15min":0.82799999999999996},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":24.045000000000002,"5min":24.045000000000002,"15min":24.045000000000002},"last":0.40699999999999997},{"interface":"front","average":{"1min":3.4119999999999999,"5min":1.8080000000000001,"15min":0.91200000000000003},"min":{"1min":0.33700000000000002,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":12.427,"5min":17.684000000000001,"15min":17.684000000000001},"last":0.39300000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:26:56 2023","interfaces":[{"interface":"back","average":{"1min":2.875,"5min":1.4590000000000001,"15min":0.88900000000000001},"min":{"1min":0.33100000000000002,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":11.880000000000001,"5min":11.880000000000001,"15min":11.880000000000001},"last":0.625},{"interface":"front","average":{"1min":3.698,"5min":1.8180000000000001,"15min":1.016},"min":{"1min":0.378,"5min":0.371,"15min":0.371},"max":{"1min":21.673999999999999,"5min":21.673999999999999,"15min":21.673999999999999},"last":0.57699999999999996}]},{"osd":5,"last update":"Sat Sep 23 20:26:56 2023","interfaces":[{"interface":"back","average":{"1min":3.117,"5min":1.8560000000000001,"15min":1.0229999999999999},"min":{"1min":0.375,"5min":0.311,"15min":0.311},"max":{"1min":11.839,"5min":11.839,"15min":11.839},"last":0.69599999999999995},{"interface":"front","average":{"1min":1.825,"5min":1.2949999999999999,"15min":0.85199999999999998},"min":{"1min":0.40000000000000002,"5min":0.313,"15min":0.313},"max":{"1min":8.5570000000000004,"5min":8.5570000000000004,"15min":8.5570000000000004},"last":0.79000000000000004}]},{"osd":6,"last update":"Sat Sep 23 20:26:56 2023","interfaces":[{"interface":"back","average":{"1min":2.0609999999999999,"5min":1.5900000000000001,"15min":0.95499999999999996},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":8.6059999999999999,"5min":8.6059999999999999,"15min":8.6059999999999999},"last":0.72899999999999998},{"interface":"front","average":{"1min":2.7469999999999999,"5min":1.5740000000000001,"15min":0.97199999999999998},"min":{"1min":0.40600000000000003,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":9.3179999999999996,"5min":9.3179999999999996,"15min":9.3179999999999996},"last":0.77500000000000002}]},{"osd":7,"last update":"Sat Sep 23 20:26:56 2023","interfaces":[{"interface":"back","average":{"1min":3.0030000000000001,"5min":1.5860000000000001,"15min":0.997},"min":{"1min":0.40999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":9.2750000000000004,"5min":9.2750000000000004,"15min":9.2750000000000004},"last":0.65000000000000002},{"interface":"front","average":{"1min":3.6680000000000001,"5min":2.0209999999999999,"15min":1.101},"min":{"1min":0.40699999999999997,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":12.468999999999999,"5min":12.468999999999999,"15min":12.468999999999999},"last":0.66600000000000004}]}]},{"osd":4,"up_from":30,"seq":128849019086,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":140224,"kb_used_data":6904,"kb_used_omap":0,"kb_used_meta":133312,"kb_avail":93604928,"statfs":{"total":95995035648,"available":95851446272,"internally_reserved":0,"allocated":7069696,"data_stored":6362199,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":136511488},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:26:22 2023","interfaces":[{"interface":"back","average":{"1min":2.258,"5min":1.411,"15min":0.84899999999999998},"min":{"1min":0.25800000000000001,"5min":0.222,"15min":0.222},"max":{"1min":11.331,"5min":20.411999999999999,"15min":20.411999999999999},"last":1.0069999999999999},{"interface":"front","average":{"1min":2.6629999999999998,"5min":1.5489999999999999,"15min":0.94399999999999995},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":13.595000000000001,"5min":20.454999999999998,"15min":20.454999999999998},"last":1.0189999999999999}]},{"osd":1,"last update":"Sat Sep 23 20:27:00 2023","interfaces":[{"interface":"back","average":{"1min":2.883,"5min":1.5089999999999999,"15min":0.92000000000000004},"min":{"1min":0.24299999999999999,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":9.6660000000000004,"5min":9.6660000000000004,"15min":9.6660000000000004},"last":1.1699999999999999},{"interface":"front","average":{"1min":2.4990000000000001,"5min":1.5640000000000001,"15min":0.93799999999999994},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":12.821999999999999,"5min":12.821999999999999,"15min":12.821999999999999},"last":0.69599999999999995}]},{"osd":2,"last update":"Sat Sep 23 20:26:32 2023","interfaces":[{"interface":"back","average":{"1min":2.1779999999999999,"5min":1.51,"15min":0.90800000000000003},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":11.412000000000001,"5min":21.462,"15min":21.462},"last":0.41599999999999998},{"interface":"front","average":{"1min":3.29,"5min":1.8660000000000001,"15min":1.014},"min":{"1min":0.28999999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":13.567,"5min":22.593,"15min":22.593},"last":0.877}]},{"osd":3,"last update":"Sat Sep 23 20:27:00 2023","interfaces":[{"interface":"back","average":{"1min":2.516,"5min":1.7929999999999999,"15min":1.0249999999999999},"min":{"1min":0.32900000000000001,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":10.846,"5min":24.891999999999999,"15min":24.891999999999999},"last":0.74099999999999999},{"interface":"front","average":{"1min":2.464,"5min":1.831,"15min":1.0529999999999999},"min":{"1min":0.35499999999999998,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":9.2080000000000002,"5min":21.922000000000001,"15min":21.922000000000001},"last":0.76900000000000002}]},{"osd":5,"last update":"Sat Sep 23 20:27:11 2023","interfaces":[{"interface":"back","average":{"1min":1.671,"5min":1.8859999999999999,"15min":0.95299999999999996},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":10.252000000000001,"5min":17.704000000000001,"15min":17.704000000000001},"last":0.38800000000000001},{"interface":"front","average":{"1min":1.964,"5min":1.643,"15min":0.88400000000000001},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":8.2110000000000003,"5min":15.329000000000001,"15min":15.329000000000001},"last":0.439}]},{"osd":6,"last update":"Sat Sep 23 20:26:34 2023","interfaces":[{"interface":"back","average":{"1min":3.3639999999999999,"5min":1.9239999999999999,"15min":0.98099999999999998},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.27500000000000002},"max":{"1min":15.252000000000001,"5min":41.371000000000002,"15min":41.371000000000002},"last":0.48699999999999999},{"interface":"front","average":{"1min":1.9179999999999999,"5min":1.593,"15min":0.88200000000000001},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":9.3350000000000009,"5min":41.140999999999998,"15min":41.140999999999998},"last":0.63100000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:26:48 2023","interfaces":[{"interface":"back","average":{"1min":4.3280000000000003,"5min":1.8859999999999999,"15min":0.997},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.33600000000000002},"max":{"1min":16.427,"5min":16.427,"15min":16.427},"last":0.53700000000000003},{"interface":"front","average":{"1min":3.2890000000000001,"5min":1.754,"15min":0.96599999999999997},"min":{"1min":0.36399999999999999,"5min":0.34300000000000003,"15min":0.29899999999999999},"max":{"1min":15.645,"5min":19.018999999999998,"15min":19.018999999999998},"last":0.58999999999999997}]}]},{"osd":5,"up_from":39,"seq":167503724747,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":312828,"kb_used_data":183796,"kb_used_omap":0,"kb_used_meta":129024,"kb_avail":93432324,"statfs":{"total":95995035648,"available":95674699776,"internally_reserved":0,"allocated":188207104,"data_stored":187500333,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":132120576},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:26:22 2023","interfaces":[{"interface":"back","average":{"1min":2.3380000000000001,"5min":1.548,"15min":0.91000000000000003},"min":{"1min":0.29499999999999998,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":11.346,"5min":32.475000000000001,"15min":32.475000000000001},"last":0.63800000000000001},{"interface":"front","average":{"1min":1.9650000000000001,"5min":1.601,"15min":0.93400000000000005},"min":{"1min":0.245,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":5.6879999999999997,"5min":32.457000000000001,"15min":32.457000000000001},"last":0.52700000000000002}]},{"osd":1,"last update":"Sat Sep 23 20:27:01 2023","interfaces":[{"interface":"back","average":{"1min":2.3980000000000001,"5min":1.819,"15min":0.96899999999999997},"min":{"1min":0.253,"5min":0.245,"15min":0.245},"max":{"1min":9.7639999999999993,"5min":32.243000000000002,"15min":32.243000000000002},"last":0.60399999999999998},{"interface":"front","average":{"1min":1.103,"5min":1.3480000000000001,"15min":0.81399999999999995},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":5.7389999999999999,"5min":32.231000000000002,"15min":32.231000000000002},"last":0.58499999999999996}]},{"osd":2,"last update":"Sat Sep 23 20:26:22 2023","interfaces":[{"interface":"back","average":{"1min":1.9970000000000001,"5min":1.891,"15min":0.997},"min":{"1min":0.30099999999999999,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":11.407,"5min":34.514000000000003,"15min":34.514000000000003},"last":0.54500000000000004},{"interface":"front","average":{"1min":2.105,"5min":1.732,"15min":0.94999999999999996},"min":{"1min":0.32100000000000001,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":11.574999999999999,"5min":32.515000000000001,"15min":32.515000000000001},"last":0.47299999999999998}]},{"osd":3,"last update":"Sat Sep 23 20:27:01 2023","interfaces":[{"interface":"back","average":{"1min":2.5129999999999999,"5min":1.5900000000000001,"15min":0.90200000000000002},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":12.846,"5min":12.846,"15min":12.846},"last":0.50700000000000001},{"interface":"front","average":{"1min":1.764,"5min":1.554,"15min":0.92800000000000005},"min":{"1min":0.35599999999999998,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":6.0049999999999999,"5min":11.379,"15min":11.379},"last":0.41699999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:27:18 2023","interfaces":[{"interface":"back","average":{"1min":2.8380000000000001,"5min":1.9019999999999999,"15min":0.94399999999999995},"min":{"1min":0.33600000000000002,"5min":0.26700000000000002,"15min":0.26400000000000001},"max":{"1min":12.984,"5min":14.218999999999999,"15min":14.218999999999999},"last":0.45500000000000002},{"interface":"front","average":{"1min":1.5609999999999999,"5min":1.536,"15min":0.84499999999999997},"min":{"1min":0.38800000000000001,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":5.9779999999999998,"5min":11.664,"15min":11.664},"last":0.67700000000000005}]},{"osd":6,"last update":"Sat Sep 23 20:26:32 2023","interfaces":[{"interface":"back","average":{"1min":2.633,"5min":1.575,"15min":0.85599999999999998},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.29799999999999999},"max":{"1min":8.7949999999999999,"5min":11.545999999999999,"15min":11.545999999999999},"last":0.61799999999999999},{"interface":"front","average":{"1min":1.6870000000000001,"5min":1.5,"15min":0.879},"min":{"1min":0.376,"5min":0.35999999999999999,"15min":0.34499999999999997},"max":{"1min":8.4120000000000008,"5min":11.456,"15min":11.456},"last":0.71199999999999997}]},{"osd":7,"last update":"Sat Sep 23 20:27:01 2023","interfaces":[{"interface":"back","average":{"1min":2.2890000000000001,"5min":1.585,"15min":0.88900000000000001},"min":{"1min":0.40100000000000002,"5min":0.33900000000000002,"15min":0.33800000000000002},"max":{"1min":6.0730000000000004,"5min":11.430999999999999,"15min":11.430999999999999},"last":0.73499999999999999},{"interface":"front","average":{"1min":1.216,"5min":1.1930000000000001,"15min":0.76400000000000001},"min":{"1min":0.316,"5min":0.316,"15min":0.315},"max":{"1min":5.835,"5min":12.368,"15min":12.368},"last":0.69799999999999995}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4722,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1360,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6696,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7156,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3154,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":63832064,"data_stored":63832064,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:27:24.923 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:27:24.971 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:24.998 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:27:24 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:27:24] "GET /metrics HTTP/1.1" 200 36477 "" "Prometheus/2.43.0" 2023-09-23T20:27:25.081 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:27:25.089 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:25.090 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:27:25.190 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:27:25.244 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:25.309 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:27:25.314 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:25.315 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:27:25.415 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:27:25.418 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T20:27:25.430 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:27:25.474 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:25.552 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 3 16 9168 9152 190.573 188.688 0.00455854 0.00523653 2023-09-23T20:27:25.578 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:27:25.589 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:25.589 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:27:25.690 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:27:25.733 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:25.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:25 smithi018 ceph-mon[179647]: from='client.55003 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:27:25.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:25 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:27:25.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:25 smithi018 ceph-mon[184144]: from='client.55003 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:27:25.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:25 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:27:25.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:25 smithi149 ceph-mon[170394]: from='client.55003 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:27:25.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:25 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:27:26.243 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:27:26.320 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:27:26.552 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 4 16 11759 11743 183.413 161.938 0.0092961 0.00544386 2023-09-23T20:27:27.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:26 smithi018 ceph-mon[184144]: pgmap v477: 99 pgs: 99 active+clean; 139 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 17 MiB/s wr, 265 op/s 2023-09-23T20:27:27.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:26 smithi018 ceph-mon[179647]: pgmap v477: 99 pgs: 99 active+clean; 139 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 17 MiB/s wr, 265 op/s 2023-09-23T20:27:27.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:26 smithi149 ceph-mon[170394]: pgmap v477: 99 pgs: 99 active+clean; 139 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 17 MiB/s wr, 265 op/s 2023-09-23T20:27:27.552 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 5 16 14287 14271 178.328 158 0.00586013 0.00559902 2023-09-23T20:27:27.618 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:27:27.622 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:27.622 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:27:27.723 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:27:27.789 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:27.901 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:27:28.552 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 6 16 16354 16338 170.138 129.188 0.00517446 0.00586763 2023-09-23T20:27:28.610 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:27:28.615 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:28.616 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:27:28.626 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:27:28.680 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:27:28.716 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:27:28.719 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:27:28.760 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:27:28.763 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:28.808 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:27:28.852 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:27:28.896 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:27:28.919 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:27:28.927 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:28.928 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:27:28.937 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:27:28.948 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:27:28.948 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:27:28.948 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:27:28.948 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:27:28.948 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:27:28.949 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:27:28.949 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:27:28.949 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:27:29.018 INFO:teuthology.orchestra.run.smithi149.stdout:ERROR: (22) Invalid argument 2023-09-23T20:27:29.018 INFO:teuthology.orchestra.run.smithi149.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:27:29.028 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:27:29.029 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:27:29.072 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:29.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:28 smithi149 ceph-mon[170394]: pgmap v478: 99 pgs: 99 active+clean; 484 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 60 MiB/s wr, 957 op/s 2023-09-23T20:27:29.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:28 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3845158569' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:27:29.293 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:28 smithi018 ceph-mon[184144]: pgmap v478: 99 pgs: 99 active+clean; 484 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 60 MiB/s wr, 957 op/s 2023-09-23T20:27:29.293 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:28 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3845158569' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:27:29.293 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:28 smithi018 ceph-mon[179647]: pgmap v478: 99 pgs: 99 active+clean; 484 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 60 MiB/s wr, 957 op/s 2023-09-23T20:27:29.293 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:28 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3845158569' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:27:29.389 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:27:29.393 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:29.393 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:27:29.465 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T20:27:29.494 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:27:29.552 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 7 16 18416 18400 164.242 128.875 0.00969819 0.00608317 2023-09-23T20:27:29.557 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:29.628 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:27:29.632 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:29.633 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:27:29.733 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:27:29.785 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:29.851 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:27:29.855 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:29.856 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:27:29.956 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:27:30.000 INFO:teuthology.orchestra.run.smithi018.stderr:noscrub is unset 2023-09-23T20:27:30.028 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:30.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:29 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1978705254' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:27:30.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:29 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:27:30.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:29 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:27:30.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:29 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:27:30.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:29 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1978705254' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:27:30.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:29 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:27:30.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:29 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:27:30.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:29 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:27:30.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:29 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1978705254' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:27:30.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:29 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:27:30.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:29 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:27:30.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:29 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:27:30.260 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:27:30.426 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:27:30.433 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:30.434 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:27:30.535 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:27:30.552 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 8 16 20666 20650 161.289 140.625 0.00497108 0.00619515 2023-09-23T20:27:30.598 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:30.769 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:27:30.773 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:30.774 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:27:30.874 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:27:30.919 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:31.061 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:27:31.066 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:31.067 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:27:31.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:30 smithi018 ceph-mon[184144]: pgmap v479: 99 pgs: 99 active+clean; 642 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-09-23T20:27:31.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:30 smithi018 ceph-mon[184144]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:27:31.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:30 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:27:31.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:30 smithi018 ceph-mon[184144]: osdmap e222: 8 total, 8 up, 8 in 2023-09-23T20:27:31.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:30 smithi018 ceph-mon[184144]: failed to encode map e222 with expected crc 2023-09-23T20:27:31.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:30 smithi018 ceph-mon[184144]: failed to encode map e222 with expected crc 2023-09-23T20:27:31.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:30 smithi018 ceph-mon[184144]: failed to encode map e222 with expected crc 2023-09-23T20:27:31.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:30 smithi018 ceph-mon[184144]: failed to encode map e222 with expected crc 2023-09-23T20:27:31.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:30 smithi018 ceph-mon[184144]: failed to encode map e222 with expected crc 2023-09-23T20:27:31.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:30 smithi018 ceph-mon[184144]: failed to encode map e222 with expected crc 2023-09-23T20:27:31.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:30 smithi018 ceph-mon[179647]: pgmap v479: 99 pgs: 99 active+clean; 642 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-09-23T20:27:31.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:30 smithi018 ceph-mon[179647]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:27:31.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:30 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:27:31.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:30 smithi018 ceph-mon[179647]: osdmap e222: 8 total, 8 up, 8 in 2023-09-23T20:27:31.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:30 smithi018 ceph-mon[179647]: failed to encode map e222 with expected crc 2023-09-23T20:27:31.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:30 smithi018 ceph-mon[179647]: failed to encode map e222 with expected crc 2023-09-23T20:27:31.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:30 smithi018 ceph-mon[179647]: failed to encode map e222 with expected crc 2023-09-23T20:27:31.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:30 smithi018 ceph-mon[179647]: failed to encode map e222 with expected crc 2023-09-23T20:27:31.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:30 smithi018 ceph-mon[179647]: failed to encode map e222 with expected crc 2023-09-23T20:27:31.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:30 smithi018 ceph-mon[179647]: failed to encode map e222 with expected crc 2023-09-23T20:27:31.167 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:27:31.215 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:30 smithi149 ceph-mon[170394]: pgmap v479: 99 pgs: 99 active+clean; 642 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-09-23T20:27:31.215 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:30 smithi149 ceph-mon[170394]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:27:31.215 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:30 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:27:31.215 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:30 smithi149 ceph-mon[170394]: osdmap e222: 8 total, 8 up, 8 in 2023-09-23T20:27:31.215 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:30 smithi149 ceph-mon[170394]: failed to encode map e222 with expected crc 2023-09-23T20:27:31.215 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:30 smithi149 ceph-mon[170394]: failed to encode map e222 with expected crc 2023-09-23T20:27:31.215 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:30 smithi149 ceph-mon[170394]: failed to encode map e222 with expected crc 2023-09-23T20:27:31.215 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:30 smithi149 ceph-mon[170394]: failed to encode map e222 with expected crc 2023-09-23T20:27:31.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:30 smithi149 ceph-mon[170394]: failed to encode map e222 with expected crc 2023-09-23T20:27:31.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:30 smithi149 ceph-mon[170394]: failed to encode map e222 with expected crc 2023-09-23T20:27:31.220 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:31.281 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:27:31.287 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:27:31.289 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:31.289 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:27:31.292 INFO:teuthology.orchestra.run.smithi018.stderr:dumped all 2023-09-23T20:27:31.390 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:27:31.436 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:31.459 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:27:31.500 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:27:31.506 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:31.507 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:27:31.552 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 9 16 22698 22682 157.478 127 0.00434185 0.00634553 2023-09-23T20:27:31.608 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:27:31.659 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:31.726 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:27:31.730 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:31.731 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:27:31.786 INFO:teuthology.orchestra.run.smithi149.stdout:ERROR: (22) Invalid argument 2023-09-23T20:27:31.786 INFO:teuthology.orchestra.run.smithi149.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:27:31.792 INFO:teuthology.orchestra.run.smithi018.stdout:{"pg_ready":true,"pg_map":{"version":480,"stamp":"2023-09-23T20:27:29.975890+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":769827521,"num_objects":11700,"num_object_clones":0,"num_object_copies":35100,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":11700,"num_whiteouts":0,"num_read":6352,"num_read_kb":13879,"num_write":12751,"num_write_kb":761425,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":19,"num_bytes_recovered":1552118,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":14473,"ondisk_log_size":14473,"up":297,"acting":297,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":301,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":3335104,"kb_used_data":2190080,"kb_used_omap":30,"kb_used_meta":1144929,"kb_avail":746626112,"statfs":{"total":767960285184,"available":764545138688,"internally_reserved":0,"allocated":2242641920,"data_stored":2235514068,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31263,"internal_metadata":1172407777},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[9,6,5,8],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":764870656,"num_objects":11671,"num_object_clones":0,"num_object_copies":35013,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":11671,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11671,"num_write_kb":746944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.422733"},"pg_stats":[{"pgid":"3.1a","version":"69'1","reported_seq":218,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T20:27:20.872475+0000","last_change":"2023-09-23T20:25:39.632437+0000","last_active":"2023-09-23T20:27:20.872475+0000","last_peered":"2023-09-23T20:27:20.872475+0000","last_clean":"2023-09-23T20:27:20.872475+0000","last_became_active":"2023-09-23T20:25:39.632074+0000","last_became_peered":"2023-09-23T20:25:39.632074+0000","last_unstale":"2023-09-23T20:27:20.872475+0000","last_undegraded":"2023-09-23T20:27:20.872475+0000","last_fullsized":"2023-09-23T20:27:20.872475+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:32:13.543799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":66,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T20:27:20.426090+0000","last_change":"2023-09-23T20:25:02.932223+0000","last_active":"2023-09-23T20:27:20.426090+0000","last_peered":"2023-09-23T20:27:20.426090+0000","last_clean":"2023-09-23T20:27:20.426090+0000","last_became_active":"2023-09-23T20:25:02.931024+0000","last_became_peered":"2023-09-23T20:25:02.931024+0000","last_unstale":"2023-09-23T20:27:20.426090+0000","last_undegraded":"2023-09-23T20:27:20.426090+0000","last_fullsized":"2023-09-23T20:27:20.426090+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:16:00.586211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":178,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T20:27:20.278432+0000","last_change":"2023-09-23T20:25:02.934543+0000","last_active":"2023-09-23T20:27:20.278432+0000","last_peered":"2023-09-23T20:27:20.278432+0000","last_clean":"2023-09-23T20:27:20.278432+0000","last_became_active":"2023-09-23T20:25:02.934178+0000","last_became_peered":"2023-09-23T20:25:02.934178+0000","last_unstale":"2023-09-23T20:27:20.278432+0000","last_undegraded":"2023-09-23T20:27:20.278432+0000","last_fullsized":"2023-09-23T20:27:20.278432+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:27:38.453052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":184,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T20:27:20.944368+0000","last_change":"2023-09-23T20:25:02.957185+0000","last_active":"2023-09-23T20:27:20.944368+0000","last_peered":"2023-09-23T20:27:20.944368+0000","last_clean":"2023-09-23T20:27:20.944368+0000","last_became_active":"2023-09-23T20:25:02.956689+0000","last_became_peered":"2023-09-23T20:25:02.956689+0000","last_unstale":"2023-09-23T20:27:20.944368+0000","last_undegraded":"2023-09-23T20:27:20.944368+0000","last_fullsized":"2023-09-23T20:27:20.944368+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:04:28.524015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":67,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-09-23T20:27:22.554717+0000","last_change":"2023-09-23T20:25:02.932054+0000","last_active":"2023-09-23T20:27:22.554717+0000","last_peered":"2023-09-23T20:27:22.554717+0000","last_clean":"2023-09-23T20:27:22.554717+0000","last_became_active":"2023-09-23T20:25:02.931309+0000","last_became_peered":"2023-09-23T20:25:02.931309+0000","last_unstale":"2023-09-23T20:27:22.554717+0000","last_undegraded":"2023-09-23T20:27:22.554717+0000","last_fullsized":"2023-09-23T20:27:22.554717+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:57:53.378499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":180,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T20:27:20.946038+0000","last_change":"2023-09-23T20:20:46.937089+0000","last_active":"2023-09-23T20:27:20.946038+0000","last_peered":"2023-09-23T20:27:20.946038+0000","last_clean":"2023-09-23T20:27:20.946038+0000","last_became_active":"2023-09-23T20:20:46.936831+0000","last_became_peered":"2023-09-23T20:20:46.936831+0000","last_unstale":"2023-09-23T20:27:20.946038+0000","last_undegraded":"2023-09-23T20:27:20.946038+0000","last_fullsized":"2023-09-23T20:27:20.946038+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:34:56.747010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":183,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-09-23T20:27:22.478988+0000","last_change":"2023-09-23T20:25:02.969432+0000","last_active":"2023-09-23T20:27:22.478988+0000","last_peered":"2023-09-23T20:27:22.478988+0000","last_clean":"2023-09-23T20:27:22.478988+0000","last_became_active":"2023-09-23T20:25:02.969118+0000","last_became_peered":"2023-09-23T20:25:02.969118+0000","last_unstale":"2023-09-23T20:27:22.478988+0000","last_undegraded":"2023-09-23T20:27:22.478988+0000","last_fullsized":"2023-09-23T20:27:22.478988+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:02:22.453831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":67,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-09-23T20:27:22.554816+0000","last_change":"2023-09-23T20:25:02.932225+0000","last_active":"2023-09-23T20:27:22.554816+0000","last_peered":"2023-09-23T20:27:22.554816+0000","last_clean":"2023-09-23T20:27:22.554816+0000","last_became_active":"2023-09-23T20:25:02.931746+0000","last_became_peered":"2023-09-23T20:25:02.931746+0000","last_unstale":"2023-09-23T20:27:22.554816+0000","last_undegraded":"2023-09-23T20:27:22.554816+0000","last_fullsized":"2023-09-23T20:27:22.554816+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:57:24.470179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":180,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T20:27:20.412932+0000","last_change":"2023-09-23T20:20:46.950281+0000","last_active":"2023-09-23T20:27:20.412932+0000","last_peered":"2023-09-23T20:27:20.412932+0000","last_clean":"2023-09-23T20:27:20.412932+0000","last_became_active":"2023-09-23T20:20:46.949892+0000","last_became_peered":"2023-09-23T20:20:46.949892+0000","last_unstale":"2023-09-23T20:27:20.412932+0000","last_undegraded":"2023-09-23T20:27:20.412932+0000","last_fullsized":"2023-09-23T20:27:20.412932+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:28.763832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":67,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-09-23T20:27:22.554296+0000","last_change":"2023-09-23T20:25:02.947669+0000","last_active":"2023-09-23T20:27:22.554296+0000","last_peered":"2023-09-23T20:27:22.554296+0000","last_clean":"2023-09-23T20:27:22.554296+0000","last_became_active":"2023-09-23T20:25:02.947368+0000","last_became_peered":"2023-09-23T20:25:02.947368+0000","last_unstale":"2023-09-23T20:27:22.554296+0000","last_undegraded":"2023-09-23T20:27:22.554296+0000","last_fullsized":"2023-09-23T20:27:22.554296+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:52:53.813761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":216,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T20:27:20.281209+0000","last_change":"2023-09-23T20:25:03.038511+0000","last_active":"2023-09-23T20:27:20.281209+0000","last_peered":"2023-09-23T20:27:20.281209+0000","last_clean":"2023-09-23T20:27:20.281209+0000","last_became_active":"2023-09-23T20:25:02.976811+0000","last_became_peered":"2023-09-23T20:25:02.976811+0000","last_unstale":"2023-09-23T20:27:20.281209+0000","last_undegraded":"2023-09-23T20:27:20.281209+0000","last_fullsized":"2023-09-23T20:27:20.281209+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:28:06.605731+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":179,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T20:27:20.278252+0000","last_change":"2023-09-23T20:25:39.646452+0000","last_active":"2023-09-23T20:27:20.278252+0000","last_peered":"2023-09-23T20:27:20.278252+0000","last_clean":"2023-09-23T20:27:20.278252+0000","last_became_active":"2023-09-23T20:25:39.646175+0000","last_became_peered":"2023-09-23T20:25:39.646175+0000","last_unstale":"2023-09-23T20:27:20.278252+0000","last_undegraded":"2023-09-23T20:27:20.278252+0000","last_fullsized":"2023-09-23T20:27:20.278252+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:33:02.099951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":185,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T20:27:20.872766+0000","last_change":"2023-09-23T20:12:43.750217+0000","last_active":"2023-09-23T20:27:20.872766+0000","last_peered":"2023-09-23T20:27:20.872766+0000","last_clean":"2023-09-23T20:27:20.872766+0000","last_became_active":"2023-09-23T20:12:43.749929+0000","last_became_peered":"2023-09-23T20:12:43.749929+0000","last_unstale":"2023-09-23T20:27:20.872766+0000","last_undegraded":"2023-09-23T20:27:20.872766+0000","last_fullsized":"2023-09-23T20:27:20.872766+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:50:28.470211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":179,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T20:27:20.872592+0000","last_change":"2023-09-23T20:20:46.939938+0000","last_active":"2023-09-23T20:27:20.872592+0000","last_peered":"2023-09-23T20:27:20.872592+0000","last_clean":"2023-09-23T20:27:20.872592+0000","last_became_active":"2023-09-23T20:20:46.938777+0000","last_became_peered":"2023-09-23T20:20:46.938777+0000","last_unstale":"2023-09-23T20:27:20.872592+0000","last_undegraded":"2023-09-23T20:27:20.872592+0000","last_fullsized":"2023-09-23T20:27:20.872592+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:45:10.329438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":215,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T20:27:20.281244+0000","last_change":"2023-09-23T20:25:39.654026+0000","last_active":"2023-09-23T20:27:20.281244+0000","last_peered":"2023-09-23T20:27:20.281244+0000","last_clean":"2023-09-23T20:27:20.281244+0000","last_became_active":"2023-09-23T20:25:39.626849+0000","last_became_peered":"2023-09-23T20:25:39.626849+0000","last_unstale":"2023-09-23T20:27:20.281244+0000","last_undegraded":"2023-09-23T20:27:20.281244+0000","last_fullsized":"2023-09-23T20:27:20.281244+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:22:53.957630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":67,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-09-23T20:27:22.555151+0000","last_change":"2023-09-23T20:25:02.962047+0000","last_active":"2023-09-23T20:27:22.555151+0000","last_peered":"2023-09-23T20:27:22.555151+0000","last_clean":"2023-09-23T20:27:22.555151+0000","last_became_active":"2023-09-23T20:25:02.961523+0000","last_became_peered":"2023-09-23T20:25:02.961523+0000","last_unstale":"2023-09-23T20:27:22.555151+0000","last_undegraded":"2023-09-23T20:27:22.555151+0000","last_fullsized":"2023-09-23T20:27:22.555151+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:18:11.830822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":183,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T20:27:20.944717+0000","last_change":"2023-09-23T20:25:02.953993+0000","last_active":"2023-09-23T20:27:20.944717+0000","last_peered":"2023-09-23T20:27:20.944717+0000","last_clean":"2023-09-23T20:27:20.944717+0000","last_became_active":"2023-09-23T20:25:02.953672+0000","last_became_peered":"2023-09-23T20:25:02.953672+0000","last_unstale":"2023-09-23T20:27:20.944717+0000","last_undegraded":"2023-09-23T20:27:20.944717+0000","last_fullsized":"2023-09-23T20:27:20.944717+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:26:08.603577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":199,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T20:27:20.413015+0000","last_change":"2023-09-23T20:25:02.949584+0000","last_active":"2023-09-23T20:27:20.413015+0000","last_peered":"2023-09-23T20:27:20.413015+0000","last_clean":"2023-09-23T20:27:20.413015+0000","last_became_active":"2023-09-23T20:25:02.949132+0000","last_became_peered":"2023-09-23T20:25:02.949132+0000","last_unstale":"2023-09-23T20:27:20.413015+0000","last_undegraded":"2023-09-23T20:27:20.413015+0000","last_fullsized":"2023-09-23T20:27:20.413015+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:34:20.509823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":164,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T20:27:20.280926+0000","last_change":"2023-09-23T20:12:43.749139+0000","last_active":"2023-09-23T20:27:20.280926+0000","last_peered":"2023-09-23T20:27:20.280926+0000","last_clean":"2023-09-23T20:27:20.280926+0000","last_became_active":"2023-09-23T20:12:43.748854+0000","last_became_peered":"2023-09-23T20:12:43.748854+0000","last_unstale":"2023-09-23T20:27:20.280926+0000","last_undegraded":"2023-09-23T20:27:20.280926+0000","last_fullsized":"2023-09-23T20:27:20.280926+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:57:11.630712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":193,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-09-23T20:27:22.478989+0000","last_change":"2023-09-23T20:25:02.957010+0000","last_active":"2023-09-23T20:27:22.478989+0000","last_peered":"2023-09-23T20:27:22.478989+0000","last_clean":"2023-09-23T20:27:22.478989+0000","last_became_active":"2023-09-23T20:25:02.956267+0000","last_became_peered":"2023-09-23T20:25:02.956267+0000","last_unstale":"2023-09-23T20:27:22.478989+0000","last_undegraded":"2023-09-23T20:27:22.478989+0000","last_fullsized":"2023-09-23T20:27:22.478989+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:10:55.299422+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":189,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T20:27:20.945326+0000","last_change":"2023-09-23T20:20:14.881009+0000","last_active":"2023-09-23T20:27:20.945326+0000","last_peered":"2023-09-23T20:27:20.945326+0000","last_clean":"2023-09-23T20:27:20.945326+0000","last_became_active":"2023-09-23T20:20:14.880685+0000","last_became_peered":"2023-09-23T20:20:14.880685+0000","last_unstale":"2023-09-23T20:27:20.945326+0000","last_undegraded":"2023-09-23T20:27:20.945326+0000","last_fullsized":"2023-09-23T20:27:20.945326+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:28.010434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":67,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-09-23T20:27:22.555448+0000","last_change":"2023-09-23T20:25:02.977462+0000","last_active":"2023-09-23T20:27:22.555448+0000","last_peered":"2023-09-23T20:27:22.555448+0000","last_clean":"2023-09-23T20:27:22.555448+0000","last_became_active":"2023-09-23T20:25:02.969910+0000","last_became_peered":"2023-09-23T20:25:02.969910+0000","last_unstale":"2023-09-23T20:27:22.555448+0000","last_undegraded":"2023-09-23T20:27:22.555448+0000","last_fullsized":"2023-09-23T20:27:22.555448+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:22:55.644955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.a","version":"221'583","reported_seq":595,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-09-23T20:27:25.613046+0000","last_change":"2023-09-23T20:27:14.711745+0000","last_active":"2023-09-23T20:27:25.613046+0000","last_peered":"2023-09-23T20:27:25.613046+0000","last_clean":"2023-09-23T20:27:25.613046+0000","last_became_active":"2023-09-23T20:27:14.711412+0000","last_became_peered":"2023-09-23T20:27:14.711412+0000","last_unstale":"2023-09-23T20:27:25.613046+0000","last_undegraded":"2023-09-23T20:27:25.613046+0000","last_fullsized":"2023-09-23T20:27:25.613046+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":583,"log_dups_size":0,"ondisk_log_size":583,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:48:13.653105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":38207488,"num_objects":583,"num_object_clones":0,"num_object_copies":1749,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":583,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":583,"num_write_kb":37312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":182,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T20:27:20.281329+0000","last_change":"2023-09-23T20:20:14.883283+0000","last_active":"2023-09-23T20:27:20.281329+0000","last_peered":"2023-09-23T20:27:20.281329+0000","last_clean":"2023-09-23T20:27:20.281329+0000","last_became_active":"2023-09-23T20:20:14.882806+0000","last_became_peered":"2023-09-23T20:20:14.882806+0000","last_unstale":"2023-09-23T20:27:20.281329+0000","last_undegraded":"2023-09-23T20:27:20.281329+0000","last_fullsized":"2023-09-23T20:27:20.281329+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:23:35.295701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":162,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T20:27:20.412798+0000","last_change":"2023-09-23T20:12:45.179211+0000","last_active":"2023-09-23T20:27:20.412798+0000","last_peered":"2023-09-23T20:27:20.412798+0000","last_clean":"2023-09-23T20:27:20.412798+0000","last_became_active":"2023-09-23T20:12:45.178966+0000","last_became_peered":"2023-09-23T20:12:45.178966+0000","last_unstale":"2023-09-23T20:27:20.412798+0000","last_undegraded":"2023-09-23T20:27:20.412798+0000","last_fullsized":"2023-09-23T20:27:20.412798+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:58.902140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"221'948","reported_seq":961,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-09-23T20:27:28.034387+0000","last_change":"2023-09-23T20:27:14.269442+0000","last_active":"2023-09-23T20:27:28.034387+0000","last_peered":"2023-09-23T20:27:28.034387+0000","last_clean":"2023-09-23T20:27:28.034387+0000","last_became_active":"2023-09-23T20:27:14.269167+0000","last_became_peered":"2023-09-23T20:27:14.269167+0000","last_unstale":"2023-09-23T20:27:28.034387+0000","last_undegraded":"2023-09-23T20:27:28.034387+0000","last_fullsized":"2023-09-23T20:27:28.034387+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":948,"log_dups_size":0,"ondisk_log_size":948,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:57:54.752399+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":62128128,"num_objects":948,"num_object_clones":0,"num_object_copies":2844,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":948,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":948,"num_write_kb":60672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":183,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T20:27:20.281243+0000","last_change":"2023-09-23T20:20:46.950784+0000","last_active":"2023-09-23T20:27:20.281243+0000","last_peered":"2023-09-23T20:27:20.281243+0000","last_clean":"2023-09-23T20:27:20.281243+0000","last_became_active":"2023-09-23T20:20:46.950055+0000","last_became_peered":"2023-09-23T20:20:46.950055+0000","last_unstale":"2023-09-23T20:27:20.281243+0000","last_undegraded":"2023-09-23T20:27:20.281243+0000","last_fullsized":"2023-09-23T20:27:20.281243+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:23:07.645427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":216,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T20:27:20.872398+0000","last_change":"2023-09-23T20:25:02.943743+0000","last_active":"2023-09-23T20:27:20.872398+0000","last_peered":"2023-09-23T20:27:20.872398+0000","last_clean":"2023-09-23T20:27:20.872398+0000","last_became_active":"2023-09-23T20:25:02.943279+0000","last_became_peered":"2023-09-23T20:25:02.943279+0000","last_unstale":"2023-09-23T20:27:20.872398+0000","last_undegraded":"2023-09-23T20:27:20.872398+0000","last_fullsized":"2023-09-23T20:27:20.872398+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:21:36.128366+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"221'560","reported_seq":572,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-09-23T20:27:25.537788+0000","last_change":"2023-09-23T20:27:14.271145+0000","last_active":"2023-09-23T20:27:25.537788+0000","last_peered":"2023-09-23T20:27:25.537788+0000","last_clean":"2023-09-23T20:27:25.537788+0000","last_became_active":"2023-09-23T20:27:14.270876+0000","last_became_peered":"2023-09-23T20:27:14.270876+0000","last_unstale":"2023-09-23T20:27:25.537788+0000","last_undegraded":"2023-09-23T20:27:25.537788+0000","last_fullsized":"2023-09-23T20:27:25.537788+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":560,"log_dups_size":0,"ondisk_log_size":560,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:23:40.941512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":36700160,"num_objects":560,"num_object_clones":0,"num_object_copies":1680,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":560,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":560,"num_write_kb":35840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":165,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T20:27:20.281115+0000","last_change":"2023-09-23T20:12:43.751523+0000","last_active":"2023-09-23T20:27:20.281115+0000","last_peered":"2023-09-23T20:27:20.281115+0000","last_clean":"2023-09-23T20:27:20.281115+0000","last_became_active":"2023-09-23T20:12:43.751257+0000","last_became_peered":"2023-09-23T20:12:43.751257+0000","last_unstale":"2023-09-23T20:27:20.281115+0000","last_undegraded":"2023-09-23T20:27:20.281115+0000","last_fullsized":"2023-09-23T20:27:20.281115+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:50:27.040225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":180,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T20:27:20.945676+0000","last_change":"2023-09-23T20:20:46.934762+0000","last_active":"2023-09-23T20:27:20.945676+0000","last_peered":"2023-09-23T20:27:20.945676+0000","last_clean":"2023-09-23T20:27:20.945676+0000","last_became_active":"2023-09-23T20:20:46.934472+0000","last_became_peered":"2023-09-23T20:20:46.934472+0000","last_unstale":"2023-09-23T20:27:20.945676+0000","last_undegraded":"2023-09-23T20:27:20.945676+0000","last_fullsized":"2023-09-23T20:27:20.945676+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:28:42.708286+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"221'1232","reported_seq":1245,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-09-23T20:27:29.904393+0000","last_change":"2023-09-23T20:27:14.268973+0000","last_active":"2023-09-23T20:27:29.904393+0000","last_peered":"2023-09-23T20:27:29.904393+0000","last_clean":"2023-09-23T20:27:29.904393+0000","last_became_active":"2023-09-23T20:27:14.268670+0000","last_became_peered":"2023-09-23T20:27:14.268670+0000","last_unstale":"2023-09-23T20:27:29.904393+0000","last_undegraded":"2023-09-23T20:27:29.904393+0000","last_fullsized":"2023-09-23T20:27:29.904393+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":1232,"log_dups_size":0,"ondisk_log_size":1232,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:37:23.418588+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":80740352,"num_objects":1232,"num_object_clones":0,"num_object_copies":3696,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1232,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1232,"num_write_kb":78848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":200,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T20:27:20.412974+0000","last_change":"2023-09-23T20:25:02.947631+0000","last_active":"2023-09-23T20:27:20.412974+0000","last_peered":"2023-09-23T20:27:20.412974+0000","last_clean":"2023-09-23T20:27:20.412974+0000","last_became_active":"2023-09-23T20:25:02.947268+0000","last_became_peered":"2023-09-23T20:25:02.947268+0000","last_unstale":"2023-09-23T20:27:20.412974+0000","last_undegraded":"2023-09-23T20:27:20.412974+0000","last_fullsized":"2023-09-23T20:27:20.412974+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:19:43.643985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":166,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T20:27:20.944137+0000","last_change":"2023-09-23T20:20:46.935968+0000","last_active":"2023-09-23T20:27:20.944137+0000","last_peered":"2023-09-23T20:27:20.944137+0000","last_clean":"2023-09-23T20:27:20.944137+0000","last_became_active":"2023-09-23T20:20:46.935618+0000","last_became_peered":"2023-09-23T20:20:46.935618+0000","last_unstale":"2023-09-23T20:27:20.944137+0000","last_undegraded":"2023-09-23T20:27:20.944137+0000","last_fullsized":"2023-09-23T20:27:20.944137+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:42:59.867176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"221'552","reported_seq":563,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-09-23T20:27:25.678587+0000","last_change":"2023-09-23T20:27:14.270193+0000","last_active":"2023-09-23T20:27:25.678587+0000","last_peered":"2023-09-23T20:27:25.678587+0000","last_clean":"2023-09-23T20:27:25.678587+0000","last_became_active":"2023-09-23T20:27:14.269963+0000","last_became_peered":"2023-09-23T20:27:14.269963+0000","last_unstale":"2023-09-23T20:27:25.678587+0000","last_undegraded":"2023-09-23T20:27:25.678587+0000","last_fullsized":"2023-09-23T20:27:25.678587+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":552,"log_dups_size":0,"ondisk_log_size":552,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:01:09.891149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":36175872,"num_objects":552,"num_object_clones":0,"num_object_copies":1656,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":552,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":552,"num_write_kb":35328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":593,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T20:27:20.280827+0000","last_change":"2023-09-23T20:27:11.961741+0000","last_active":"2023-09-23T20:27:20.280827+0000","last_peered":"2023-09-23T20:27:20.280827+0000","last_clean":"2023-09-23T20:27:20.280827+0000","last_became_active":"2023-09-23T20:27:11.961310+0000","last_became_peered":"2023-09-23T20:27:11.961310+0000","last_unstale":"2023-09-23T20:27:20.280827+0000","last_undegraded":"2023-09-23T20:27:20.280827+0000","last_fullsized":"2023-09-23T20:27:20.280827+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:09:47.213130+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":71,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T20:27:20.426044+0000","last_change":"2023-09-23T20:25:03.000957+0000","last_active":"2023-09-23T20:27:20.426044+0000","last_peered":"2023-09-23T20:27:20.426044+0000","last_clean":"2023-09-23T20:27:20.426044+0000","last_became_active":"2023-09-23T20:25:02.939903+0000","last_became_peered":"2023-09-23T20:25:02.939903+0000","last_unstale":"2023-09-23T20:27:20.426044+0000","last_undegraded":"2023-09-23T20:27:20.426044+0000","last_fullsized":"2023-09-23T20:27:20.426044+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:51:14.981461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":199,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T20:27:20.872688+0000","last_change":"2023-09-23T20:25:02.929447+0000","last_active":"2023-09-23T20:27:20.872688+0000","last_peered":"2023-09-23T20:27:20.872688+0000","last_clean":"2023-09-23T20:27:20.872688+0000","last_became_active":"2023-09-23T20:25:02.929173+0000","last_became_peered":"2023-09-23T20:25:02.929173+0000","last_unstale":"2023-09-23T20:27:20.872688+0000","last_undegraded":"2023-09-23T20:27:20.872688+0000","last_fullsized":"2023-09-23T20:27:20.872688+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:32:09.280647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"221'848","reported_seq":861,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-09-23T20:27:27.034710+0000","last_change":"2023-09-23T20:27:14.270199+0000","last_active":"2023-09-23T20:27:27.034710+0000","last_peered":"2023-09-23T20:27:27.034710+0000","last_clean":"2023-09-23T20:27:27.034710+0000","last_became_active":"2023-09-23T20:27:14.269858+0000","last_became_peered":"2023-09-23T20:27:14.269858+0000","last_unstale":"2023-09-23T20:27:27.034710+0000","last_undegraded":"2023-09-23T20:27:27.034710+0000","last_fullsized":"2023-09-23T20:27:27.034710+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":848,"log_dups_size":0,"ondisk_log_size":848,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:47:21.406552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":55574528,"num_objects":848,"num_object_clones":0,"num_object_copies":2544,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":848,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":848,"num_write_kb":54272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":435,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T20:27:20.412607+0000","last_change":"2023-09-23T20:27:11.959998+0000","last_active":"2023-09-23T20:27:20.412607+0000","last_peered":"2023-09-23T20:27:20.412607+0000","last_clean":"2023-09-23T20:27:20.412607+0000","last_became_active":"2023-09-23T20:27:11.959516+0000","last_became_peered":"2023-09-23T20:27:11.959516+0000","last_unstale":"2023-09-23T20:27:20.412607+0000","last_undegraded":"2023-09-23T20:27:20.412607+0000","last_fullsized":"2023-09-23T20:27:20.412607+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:09:47.213130+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":423697,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":181,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T20:27:20.945647+0000","last_change":"2023-09-23T20:20:46.937163+0000","last_active":"2023-09-23T20:27:20.945647+0000","last_peered":"2023-09-23T20:27:20.945647+0000","last_clean":"2023-09-23T20:27:20.945647+0000","last_became_active":"2023-09-23T20:20:46.936757+0000","last_became_peered":"2023-09-23T20:20:46.936757+0000","last_unstale":"2023-09-23T20:27:20.945647+0000","last_undegraded":"2023-09-23T20:27:20.945647+0000","last_fullsized":"2023-09-23T20:27:20.945647+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:09.401379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":209,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T20:27:20.872781+0000","last_change":"2023-09-23T20:25:39.624636+0000","last_active":"2023-09-23T20:27:20.872781+0000","last_peered":"2023-09-23T20:27:20.872781+0000","last_clean":"2023-09-23T20:27:20.872781+0000","last_became_active":"2023-09-23T20:25:39.624297+0000","last_became_peered":"2023-09-23T20:25:39.624297+0000","last_unstale":"2023-09-23T20:27:20.872781+0000","last_undegraded":"2023-09-23T20:27:20.872781+0000","last_fullsized":"2023-09-23T20:27:20.872781+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:33:47.448737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"221'919","reported_seq":932,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-09-23T20:27:28.038489+0000","last_change":"2023-09-23T20:27:14.712208+0000","last_active":"2023-09-23T20:27:28.038489+0000","last_peered":"2023-09-23T20:27:28.038489+0000","last_clean":"2023-09-23T20:27:28.038489+0000","last_became_active":"2023-09-23T20:27:14.711905+0000","last_became_peered":"2023-09-23T20:27:14.711905+0000","last_unstale":"2023-09-23T20:27:28.038489+0000","last_undegraded":"2023-09-23T20:27:28.038489+0000","last_fullsized":"2023-09-23T20:27:28.038489+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":919,"log_dups_size":0,"ondisk_log_size":919,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:14:54.221825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":60227584,"num_objects":919,"num_object_clones":0,"num_object_copies":2757,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":919,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":919,"num_write_kb":58816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":200,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T20:27:20.872254+0000","last_change":"2023-09-23T20:25:39.628097+0000","last_active":"2023-09-23T20:27:20.872254+0000","last_peered":"2023-09-23T20:27:20.872254+0000","last_clean":"2023-09-23T20:27:20.872254+0000","last_became_active":"2023-09-23T20:25:39.627806+0000","last_became_peered":"2023-09-23T20:25:39.627806+0000","last_unstale":"2023-09-23T20:27:20.872254+0000","last_undegraded":"2023-09-23T20:27:20.872254+0000","last_fullsized":"2023-09-23T20:27:20.872254+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:53:44.398854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":179,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T20:27:20.872936+0000","last_change":"2023-09-23T20:20:46.939720+0000","last_active":"2023-09-23T20:27:20.872936+0000","last_peered":"2023-09-23T20:27:20.872936+0000","last_clean":"2023-09-23T20:27:20.872936+0000","last_became_active":"2023-09-23T20:20:46.939357+0000","last_became_peered":"2023-09-23T20:20:46.939357+0000","last_unstale":"2023-09-23T20:27:20.872936+0000","last_undegraded":"2023-09-23T20:27:20.872936+0000","last_fullsized":"2023-09-23T20:27:20.872936+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:56.613212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"221'596","reported_seq":608,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-09-23T20:27:25.613039+0000","last_change":"2023-09-23T20:27:14.268955+0000","last_active":"2023-09-23T20:27:25.613039+0000","last_peered":"2023-09-23T20:27:25.613039+0000","last_clean":"2023-09-23T20:27:25.613039+0000","last_became_active":"2023-09-23T20:27:14.268662+0000","last_became_peered":"2023-09-23T20:27:14.268662+0000","last_unstale":"2023-09-23T20:27:25.613039+0000","last_undegraded":"2023-09-23T20:27:25.613039+0000","last_fullsized":"2023-09-23T20:27:25.613039+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":596,"log_dups_size":0,"ondisk_log_size":596,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:33:52.785540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":39059456,"num_objects":596,"num_object_clones":0,"num_object_copies":1788,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":596,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":596,"num_write_kb":38144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":435,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T20:27:20.412663+0000","last_change":"2023-09-23T20:27:11.961446+0000","last_active":"2023-09-23T20:27:20.412663+0000","last_peered":"2023-09-23T20:27:20.412663+0000","last_clean":"2023-09-23T20:27:20.412663+0000","last_became_active":"2023-09-23T20:27:11.961051+0000","last_became_peered":"2023-09-23T20:27:11.961051+0000","last_unstale":"2023-09-23T20:27:20.412663+0000","last_undegraded":"2023-09-23T20:27:20.412663+0000","last_fullsized":"2023-09-23T20:27:20.412663+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:09:47.213130+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":67,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T20:27:20.425699+0000","last_change":"2023-09-23T20:25:02.978173+0000","last_active":"2023-09-23T20:27:20.425699+0000","last_peered":"2023-09-23T20:27:20.425699+0000","last_clean":"2023-09-23T20:27:20.425699+0000","last_became_active":"2023-09-23T20:25:02.977856+0000","last_became_peered":"2023-09-23T20:25:02.977856+0000","last_unstale":"2023-09-23T20:27:20.425699+0000","last_undegraded":"2023-09-23T20:27:20.425699+0000","last_fullsized":"2023-09-23T20:27:20.425699+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:19.861943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":81,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T20:27:20.278662+0000","last_change":"2023-09-23T20:25:02.924911+0000","last_active":"2023-09-23T20:27:20.278662+0000","last_peered":"2023-09-23T20:27:20.278662+0000","last_clean":"2023-09-23T20:27:20.278662+0000","last_became_active":"2023-09-23T20:25:02.924586+0000","last_became_peered":"2023-09-23T20:25:02.924586+0000","last_unstale":"2023-09-23T20:27:20.278662+0000","last_undegraded":"2023-09-23T20:27:20.278662+0000","last_fullsized":"2023-09-23T20:27:20.278662+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:24:44.176766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"221'1177","reported_seq":1190,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-09-23T20:27:29.898740+0000","last_change":"2023-09-23T20:27:14.267314+0000","last_active":"2023-09-23T20:27:29.898740+0000","last_peered":"2023-09-23T20:27:29.898740+0000","last_clean":"2023-09-23T20:27:29.898740+0000","last_became_active":"2023-09-23T20:27:14.267011+0000","last_became_peered":"2023-09-23T20:27:14.267011+0000","last_unstale":"2023-09-23T20:27:29.898740+0000","last_undegraded":"2023-09-23T20:27:29.898740+0000","last_fullsized":"2023-09-23T20:27:29.898740+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":1177,"log_dups_size":0,"ondisk_log_size":1177,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:47:58.542542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":77135872,"num_objects":1177,"num_object_clones":0,"num_object_copies":3531,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1177,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1177,"num_write_kb":75328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":444,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T20:27:20.945975+0000","last_change":"2023-09-23T20:27:11.964104+0000","last_active":"2023-09-23T20:27:20.945975+0000","last_peered":"2023-09-23T20:27:20.945975+0000","last_clean":"2023-09-23T20:27:20.945975+0000","last_became_active":"2023-09-23T20:27:11.963740+0000","last_became_peered":"2023-09-23T20:27:11.963740+0000","last_unstale":"2023-09-23T20:27:20.945975+0000","last_undegraded":"2023-09-23T20:27:20.945975+0000","last_fullsized":"2023-09-23T20:27:20.945975+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:09:47.213130+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"86'119","reported_seq":555,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T20:27:20.278982+0000","last_change":"2023-09-23T20:20:46.937071+0000","last_active":"2023-09-23T20:27:20.278982+0000","last_peered":"2023-09-23T20:27:20.278982+0000","last_clean":"2023-09-23T20:27:20.278982+0000","last_became_active":"2023-09-23T20:20:46.935348+0000","last_became_peered":"2023-09-23T20:20:46.935348+0000","last_unstale":"2023-09-23T20:27:20.278982+0000","last_undegraded":"2023-09-23T20:27:20.278982+0000","last_fullsized":"2023-09-23T20:27:20.278982+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":119,"log_dups_size":0,"ondisk_log_size":119,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:09.829030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032705499999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":523,"num_read_kb":3503,"num_write":136,"num_write_kb":3128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":237,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T20:27:20.412996+0000","last_change":"2023-09-23T20:25:39.637160+0000","last_active":"2023-09-23T20:27:20.412996+0000","last_peered":"2023-09-23T20:27:20.412996+0000","last_clean":"2023-09-23T20:27:20.412996+0000","last_became_active":"2023-09-23T20:25:39.636860+0000","last_became_peered":"2023-09-23T20:25:39.636860+0000","last_unstale":"2023-09-23T20:27:20.412996+0000","last_undegraded":"2023-09-23T20:27:20.412996+0000","last_fullsized":"2023-09-23T20:27:20.412996+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:21:47.860063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":147,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T20:27:20.278622+0000","last_change":"2023-09-23T20:20:46.937104+0000","last_active":"2023-09-23T20:27:20.278622+0000","last_peered":"2023-09-23T20:27:20.278622+0000","last_clean":"2023-09-23T20:27:20.278622+0000","last_became_active":"2023-09-23T20:20:46.936820+0000","last_became_peered":"2023-09-23T20:20:46.936820+0000","last_unstale":"2023-09-23T20:27:20.278622+0000","last_undegraded":"2023-09-23T20:27:20.278622+0000","last_fullsized":"2023-09-23T20:27:20.278622+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:22:19.879310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.6","version":"221'925","reported_seq":938,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-09-23T20:27:28.028758+0000","last_change":"2023-09-23T20:27:14.269226+0000","last_active":"2023-09-23T20:27:28.028758+0000","last_peered":"2023-09-23T20:27:28.028758+0000","last_clean":"2023-09-23T20:27:28.028758+0000","last_became_active":"2023-09-23T20:27:14.268927+0000","last_became_peered":"2023-09-23T20:27:14.268927+0000","last_unstale":"2023-09-23T20:27:28.028758+0000","last_undegraded":"2023-09-23T20:27:28.028758+0000","last_fullsized":"2023-09-23T20:27:28.028758+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":925,"log_dups_size":0,"ondisk_log_size":925,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:45:57.844328+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":60620800,"num_objects":925,"num_object_clones":0,"num_object_copies":2775,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":925,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":925,"num_write_kb":59200,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":435,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T20:27:20.412654+0000","last_change":"2023-09-23T20:27:11.962544+0000","last_active":"2023-09-23T20:27:20.412654+0000","last_peered":"2023-09-23T20:27:20.412654+0000","last_clean":"2023-09-23T20:27:20.412654+0000","last_became_active":"2023-09-23T20:27:11.961863+0000","last_became_peered":"2023-09-23T20:27:11.961863+0000","last_unstale":"2023-09-23T20:27:20.412654+0000","last_undegraded":"2023-09-23T20:27:20.412654+0000","last_fullsized":"2023-09-23T20:27:20.412654+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:09:47.213130+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":45,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"86'106","reported_seq":845,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T20:27:20.412892+0000","last_change":"2023-09-23T20:20:46.939468+0000","last_active":"2023-09-23T20:27:20.412892+0000","last_peered":"2023-09-23T20:27:20.412892+0000","last_clean":"2023-09-23T20:27:20.412892+0000","last_became_active":"2023-09-23T20:20:46.938954+0000","last_became_peered":"2023-09-23T20:20:46.938954+0000","last_unstale":"2023-09-23T20:27:20.412892+0000","last_undegraded":"2023-09-23T20:27:20.412892+0000","last_fullsized":"2023-09-23T20:27:20.412892+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":106,"log_dups_size":0,"ondisk_log_size":106,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:01:43.761064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00099302599999999993,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1105,"num_read_kb":1689,"num_write":100,"num_write_kb":404,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":180,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T20:27:20.872582+0000","last_change":"2023-09-23T20:20:14.887345+0000","last_active":"2023-09-23T20:27:20.872582+0000","last_peered":"2023-09-23T20:27:20.872582+0000","last_clean":"2023-09-23T20:27:20.872582+0000","last_became_active":"2023-09-23T20:20:14.886970+0000","last_became_peered":"2023-09-23T20:20:14.886970+0000","last_unstale":"2023-09-23T20:27:20.872582+0000","last_undegraded":"2023-09-23T20:27:20.872582+0000","last_fullsized":"2023-09-23T20:27:20.872582+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:00:38.575926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":213,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T20:27:20.278905+0000","last_change":"2023-09-23T20:25:39.631133+0000","last_active":"2023-09-23T20:27:20.278905+0000","last_peered":"2023-09-23T20:27:20.278905+0000","last_clean":"2023-09-23T20:27:20.278905+0000","last_became_active":"2023-09-23T20:25:39.630779+0000","last_became_peered":"2023-09-23T20:25:39.630779+0000","last_unstale":"2023-09-23T20:27:20.278905+0000","last_undegraded":"2023-09-23T20:27:20.278905+0000","last_fullsized":"2023-09-23T20:27:20.278905+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:39:52.635649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.5","version":"221'575","reported_seq":587,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-09-23T20:27:25.510068+0000","last_change":"2023-09-23T20:27:14.711792+0000","last_active":"2023-09-23T20:27:25.510068+0000","last_peered":"2023-09-23T20:27:25.510068+0000","last_clean":"2023-09-23T20:27:25.510068+0000","last_became_active":"2023-09-23T20:27:14.711434+0000","last_became_peered":"2023-09-23T20:27:14.711434+0000","last_unstale":"2023-09-23T20:27:25.510068+0000","last_undegraded":"2023-09-23T20:27:25.510068+0000","last_fullsized":"2023-09-23T20:27:25.510068+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":575,"log_dups_size":0,"ondisk_log_size":575,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:10:14.885066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":37683200,"num_objects":575,"num_object_clones":0,"num_object_copies":1725,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":575,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":575,"num_write_kb":36800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":593,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T20:27:20.280889+0000","last_change":"2023-09-23T20:27:11.961802+0000","last_active":"2023-09-23T20:27:20.280889+0000","last_peered":"2023-09-23T20:27:20.280889+0000","last_clean":"2023-09-23T20:27:20.280889+0000","last_became_active":"2023-09-23T20:27:11.961418+0000","last_became_peered":"2023-09-23T20:27:11.961418+0000","last_unstale":"2023-09-23T20:27:20.280889+0000","last_undegraded":"2023-09-23T20:27:20.280889+0000","last_fullsized":"2023-09-23T20:27:20.280889+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:09:47.213130+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"86'130","reported_seq":530,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T20:27:20.942881+0000","last_change":"2023-09-23T20:25:39.637458+0000","last_active":"2023-09-23T20:27:20.942881+0000","last_peered":"2023-09-23T20:27:20.942881+0000","last_clean":"2023-09-23T20:27:20.942881+0000","last_became_active":"2023-09-23T20:25:39.636773+0000","last_became_peered":"2023-09-23T20:25:39.636773+0000","last_unstale":"2023-09-23T20:27:20.942881+0000","last_undegraded":"2023-09-23T20:27:20.942881+0000","last_fullsized":"2023-09-23T20:27:20.942881+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:00:32.810564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011300049999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":512,"num_read_kb":1820,"num_write":133,"num_write_kb":1347,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":179,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T20:27:20.278509+0000","last_change":"2023-09-23T20:25:39.632283+0000","last_active":"2023-09-23T20:27:20.278509+0000","last_peered":"2023-09-23T20:27:20.278509+0000","last_clean":"2023-09-23T20:27:20.278509+0000","last_became_active":"2023-09-23T20:25:39.631819+0000","last_became_peered":"2023-09-23T20:25:39.631819+0000","last_unstale":"2023-09-23T20:27:20.278509+0000","last_undegraded":"2023-09-23T20:27:20.278509+0000","last_fullsized":"2023-09-23T20:27:20.278509+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:25:21.815555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":190,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T20:27:20.278538+0000","last_change":"2023-09-23T20:20:46.936524+0000","last_active":"2023-09-23T20:27:20.278538+0000","last_peered":"2023-09-23T20:27:20.278538+0000","last_clean":"2023-09-23T20:27:20.278538+0000","last_became_active":"2023-09-23T20:20:46.936267+0000","last_became_peered":"2023-09-23T20:20:46.936267+0000","last_unstale":"2023-09-23T20:27:20.278538+0000","last_undegraded":"2023-09-23T20:27:20.278538+0000","last_fullsized":"2023-09-23T20:27:20.278538+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:32:20.984521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"221'631","reported_seq":643,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-09-23T20:27:25.604218+0000","last_change":"2023-09-23T20:27:14.272939+0000","last_active":"2023-09-23T20:27:25.604218+0000","last_peered":"2023-09-23T20:27:25.604218+0000","last_clean":"2023-09-23T20:27:25.604218+0000","last_became_active":"2023-09-23T20:27:14.272096+0000","last_became_peered":"2023-09-23T20:27:14.272096+0000","last_unstale":"2023-09-23T20:27:25.604218+0000","last_undegraded":"2023-09-23T20:27:25.604218+0000","last_fullsized":"2023-09-23T20:27:25.604218+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":631,"log_dups_size":0,"ondisk_log_size":631,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:41:48.494925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":41353216,"num_objects":631,"num_object_clones":0,"num_object_copies":1893,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":631,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":631,"num_write_kb":40384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"86'194","reported_seq":869,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T20:27:20.412718+0000","last_change":"2023-09-23T20:25:40.618665+0000","last_active":"2023-09-23T20:27:20.412718+0000","last_peered":"2023-09-23T20:27:20.412718+0000","last_clean":"2023-09-23T20:27:20.412718+0000","last_became_active":"2023-09-23T20:25:40.618087+0000","last_became_peered":"2023-09-23T20:25:40.618087+0000","last_unstale":"2023-09-23T20:27:20.412718+0000","last_undegraded":"2023-09-23T20:27:20.412718+0000","last_fullsized":"2023-09-23T20:27:20.412718+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":194,"log_dups_size":0,"ondisk_log_size":194,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:07:16.114968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00037995999999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1025,"num_read_kb":1201,"num_write":173,"num_write_kb":1276,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":577,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T20:27:20.280814+0000","last_change":"2023-09-23T20:27:04.493422+0000","last_active":"2023-09-23T20:27:20.280814+0000","last_peered":"2023-09-23T20:27:20.280814+0000","last_clean":"2023-09-23T20:27:20.280814+0000","last_became_active":"2023-09-23T20:25:21.159200+0000","last_became_peered":"2023-09-23T20:25:21.159200+0000","last_unstale":"2023-09-23T20:27:20.280814+0000","last_undegraded":"2023-09-23T20:27:20.280814+0000","last_fullsized":"2023-09-23T20:27:20.280814+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:09:47.213130+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":67,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T20:27:20.425931+0000","last_change":"2023-09-23T20:25:02.984205+0000","last_active":"2023-09-23T20:27:20.425931+0000","last_peered":"2023-09-23T20:27:20.425931+0000","last_clean":"2023-09-23T20:27:20.425931+0000","last_became_active":"2023-09-23T20:25:02.983758+0000","last_became_peered":"2023-09-23T20:25:02.983758+0000","last_unstale":"2023-09-23T20:27:20.425931+0000","last_undegraded":"2023-09-23T20:27:20.425931+0000","last_fullsized":"2023-09-23T20:27:20.425931+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:35:14.573101+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":163,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T20:27:20.945972+0000","last_change":"2023-09-23T20:12:45.609829+0000","last_active":"2023-09-23T20:27:20.945972+0000","last_peered":"2023-09-23T20:27:20.945972+0000","last_clean":"2023-09-23T20:27:20.945972+0000","last_became_active":"2023-09-23T20:12:45.609467+0000","last_became_peered":"2023-09-23T20:12:45.609467+0000","last_unstale":"2023-09-23T20:27:20.945972+0000","last_undegraded":"2023-09-23T20:27:20.945972+0000","last_fullsized":"2023-09-23T20:27:20.945972+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:27:31.490555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.4","version":"221'506","reported_seq":519,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-09-23T20:27:25.027178+0000","last_change":"2023-09-23T20:27:14.267770+0000","last_active":"2023-09-23T20:27:25.027178+0000","last_peered":"2023-09-23T20:27:25.027178+0000","last_clean":"2023-09-23T20:27:25.027178+0000","last_became_active":"2023-09-23T20:27:14.266851+0000","last_became_peered":"2023-09-23T20:27:14.266851+0000","last_unstale":"2023-09-23T20:27:25.027178+0000","last_undegraded":"2023-09-23T20:27:25.027178+0000","last_fullsized":"2023-09-23T20:27:25.027178+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":506,"log_dups_size":0,"ondisk_log_size":506,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:50:38.592759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":33161216,"num_objects":506,"num_object_clones":0,"num_object_copies":1518,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":506,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":506,"num_write_kb":32384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":446,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T20:27:20.278911+0000","last_change":"2023-09-23T20:27:12.386203+0000","last_active":"2023-09-23T20:27:20.278911+0000","last_peered":"2023-09-23T20:27:20.278911+0000","last_clean":"2023-09-23T20:27:20.278911+0000","last_became_active":"2023-09-23T20:27:11.962975+0000","last_became_peered":"2023-09-23T20:27:11.962975+0000","last_unstale":"2023-09-23T20:27:20.278911+0000","last_undegraded":"2023-09-23T20:27:20.278911+0000","last_fullsized":"2023-09-23T20:27:20.278911+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:12.386098+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:12.386098+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:23:49.746955+0000","scrub_duration":0.0019500940000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"86'80","reported_seq":467,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T20:27:20.280732+0000","last_change":"2023-09-23T20:20:46.950784+0000","last_active":"2023-09-23T20:27:20.280732+0000","last_peered":"2023-09-23T20:27:20.280732+0000","last_clean":"2023-09-23T20:27:20.280732+0000","last_became_active":"2023-09-23T20:20:46.950071+0000","last_became_peered":"2023-09-23T20:20:46.950071+0000","last_unstale":"2023-09-23T20:27:20.280732+0000","last_undegraded":"2023-09-23T20:27:20.280732+0000","last_fullsized":"2023-09-23T20:27:20.280732+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":80,"log_dups_size":0,"ondisk_log_size":80,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:11:06.812404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035115,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":395,"num_read_kb":1370,"num_write":66,"num_write_kb":1061,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":183,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T20:27:20.944095+0000","last_change":"2023-09-23T20:25:02.942709+0000","last_active":"2023-09-23T20:27:20.944095+0000","last_peered":"2023-09-23T20:27:20.944095+0000","last_clean":"2023-09-23T20:27:20.944095+0000","last_became_active":"2023-09-23T20:25:02.942323+0000","last_became_peered":"2023-09-23T20:25:02.942323+0000","last_unstale":"2023-09-23T20:27:20.944095+0000","last_undegraded":"2023-09-23T20:27:20.944095+0000","last_fullsized":"2023-09-23T20:27:20.944095+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:26:12.039101+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":199,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T20:27:20.944479+0000","last_change":"2023-09-23T20:25:39.641452+0000","last_active":"2023-09-23T20:27:20.944479+0000","last_peered":"2023-09-23T20:27:20.944479+0000","last_clean":"2023-09-23T20:27:20.944479+0000","last_became_active":"2023-09-23T20:25:39.638987+0000","last_became_peered":"2023-09-23T20:25:39.638987+0000","last_unstale":"2023-09-23T20:27:20.944479+0000","last_undegraded":"2023-09-23T20:27:20.944479+0000","last_fullsized":"2023-09-23T20:27:20.944479+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:33:59.955461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"221'445","reported_seq":458,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-09-23T20:27:25.051922+0000","last_change":"2023-09-23T20:27:14.712242+0000","last_active":"2023-09-23T20:27:25.051922+0000","last_peered":"2023-09-23T20:27:25.051922+0000","last_clean":"2023-09-23T20:27:25.051922+0000","last_became_active":"2023-09-23T20:27:14.711916+0000","last_became_peered":"2023-09-23T20:27:14.711916+0000","last_unstale":"2023-09-23T20:27:25.051922+0000","last_undegraded":"2023-09-23T20:27:25.051922+0000","last_fullsized":"2023-09-23T20:27:25.051922+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":445,"log_dups_size":0,"ondisk_log_size":445,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:08:51.484867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":29163520,"num_objects":445,"num_object_clones":0,"num_object_copies":1335,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":445,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":445,"num_write_kb":28480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":444,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T20:27:20.945926+0000","last_change":"2023-09-23T20:27:11.963158+0000","last_active":"2023-09-23T20:27:20.945926+0000","last_peered":"2023-09-23T20:27:20.945926+0000","last_clean":"2023-09-23T20:27:20.945926+0000","last_became_active":"2023-09-23T20:27:11.962792+0000","last_became_peered":"2023-09-23T20:27:11.962792+0000","last_unstale":"2023-09-23T20:27:20.945926+0000","last_undegraded":"2023-09-23T20:27:20.945926+0000","last_fullsized":"2023-09-23T20:27:20.945926+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:09:47.213130+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"86'102","reported_seq":727,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T20:27:20.281195+0000","last_change":"2023-09-23T20:20:14.881975+0000","last_active":"2023-09-23T20:27:20.281195+0000","last_peered":"2023-09-23T20:27:20.281195+0000","last_clean":"2023-09-23T20:27:20.281195+0000","last_became_active":"2023-09-23T20:20:14.881544+0000","last_became_peered":"2023-09-23T20:20:14.881544+0000","last_unstale":"2023-09-23T20:27:20.281195+0000","last_undegraded":"2023-09-23T20:27:20.281195+0000","last_fullsized":"2023-09-23T20:27:20.281195+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":102,"log_dups_size":0,"ondisk_log_size":102,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:34.594972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030656500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":963,"num_read_kb":1234,"num_write":60,"num_write_kb":225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":179,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T20:27:20.279068+0000","last_change":"2023-09-23T20:25:39.631751+0000","last_active":"2023-09-23T20:27:20.279068+0000","last_peered":"2023-09-23T20:27:20.279068+0000","last_clean":"2023-09-23T20:27:20.279068+0000","last_became_active":"2023-09-23T20:25:39.631136+0000","last_became_peered":"2023-09-23T20:25:39.631136+0000","last_unstale":"2023-09-23T20:27:20.279068+0000","last_undegraded":"2023-09-23T20:27:20.279068+0000","last_fullsized":"2023-09-23T20:27:20.279068+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:08:17.020722+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":151,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T20:27:20.944311+0000","last_change":"2023-09-23T20:20:14.888874+0000","last_active":"2023-09-23T20:27:20.944311+0000","last_peered":"2023-09-23T20:27:20.944311+0000","last_clean":"2023-09-23T20:27:20.944311+0000","last_became_active":"2023-09-23T20:20:14.887724+0000","last_became_peered":"2023-09-23T20:20:14.887724+0000","last_unstale":"2023-09-23T20:27:20.944311+0000","last_undegraded":"2023-09-23T20:27:20.944311+0000","last_fullsized":"2023-09-23T20:27:20.944311+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:09:24.510326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"221'607","reported_seq":619,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-09-23T20:27:25.668351+0000","last_change":"2023-09-23T20:27:14.270161+0000","last_active":"2023-09-23T20:27:25.668351+0000","last_peered":"2023-09-23T20:27:25.668351+0000","last_clean":"2023-09-23T20:27:25.668351+0000","last_became_active":"2023-09-23T20:27:14.269904+0000","last_became_peered":"2023-09-23T20:27:14.269904+0000","last_unstale":"2023-09-23T20:27:25.668351+0000","last_undegraded":"2023-09-23T20:27:25.668351+0000","last_fullsized":"2023-09-23T20:27:25.668351+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":607,"log_dups_size":0,"ondisk_log_size":607,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:41:02.917630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":39780352,"num_objects":607,"num_object_clones":0,"num_object_copies":1821,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":607,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":607,"num_write_kb":38848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":445,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T20:27:20.872624+0000","last_change":"2023-09-23T20:27:12.810010+0000","last_active":"2023-09-23T20:27:20.872624+0000","last_peered":"2023-09-23T20:27:20.872624+0000","last_clean":"2023-09-23T20:27:20.872624+0000","last_became_active":"2023-09-23T20:27:11.959614+0000","last_became_peered":"2023-09-23T20:27:11.959614+0000","last_unstale":"2023-09-23T20:27:20.872624+0000","last_undegraded":"2023-09-23T20:27:20.872624+0000","last_fullsized":"2023-09-23T20:27:20.872624+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:12.809933+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:12.809933+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:48:13.147428+0000","scrub_duration":0.001626628,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":71,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T20:27:20.426333+0000","last_change":"2023-09-23T20:25:03.007736+0000","last_active":"2023-09-23T20:27:20.426333+0000","last_peered":"2023-09-23T20:27:20.426333+0000","last_clean":"2023-09-23T20:27:20.426333+0000","last_became_active":"2023-09-23T20:25:02.973689+0000","last_became_peered":"2023-09-23T20:25:02.973689+0000","last_unstale":"2023-09-23T20:27:20.426333+0000","last_undegraded":"2023-09-23T20:27:20.426333+0000","last_fullsized":"2023-09-23T20:27:20.426333+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:37:25.643774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"86'128","reported_seq":546,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T20:27:20.278459+0000","last_change":"2023-09-23T20:20:46.939256+0000","last_active":"2023-09-23T20:27:20.278459+0000","last_peered":"2023-09-23T20:27:20.278459+0000","last_clean":"2023-09-23T20:27:20.278459+0000","last_became_active":"2023-09-23T20:20:46.938756+0000","last_became_peered":"2023-09-23T20:20:46.938756+0000","last_unstale":"2023-09-23T20:27:20.278459+0000","last_undegraded":"2023-09-23T20:27:20.278459+0000","last_fullsized":"2023-09-23T20:27:20.278459+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:52:18.161436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000293202,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":579,"num_read_kb":1473,"num_write":96,"num_write_kb":1062,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":67,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T20:27:20.425641+0000","last_change":"2023-09-23T20:25:02.971740+0000","last_active":"2023-09-23T20:27:20.425641+0000","last_peered":"2023-09-23T20:27:20.425641+0000","last_clean":"2023-09-23T20:27:20.425641+0000","last_became_active":"2023-09-23T20:25:02.970412+0000","last_became_peered":"2023-09-23T20:25:02.970412+0000","last_unstale":"2023-09-23T20:27:20.425641+0000","last_undegraded":"2023-09-23T20:27:20.425641+0000","last_fullsized":"2023-09-23T20:27:20.425641+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:53:17.247216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"221'567","reported_seq":579,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-09-23T20:27:25.545370+0000","last_change":"2023-09-23T20:27:14.711764+0000","last_active":"2023-09-23T20:27:25.545370+0000","last_peered":"2023-09-23T20:27:25.545370+0000","last_clean":"2023-09-23T20:27:25.545370+0000","last_became_active":"2023-09-23T20:27:14.711409+0000","last_became_peered":"2023-09-23T20:27:14.711409+0000","last_unstale":"2023-09-23T20:27:25.545370+0000","last_undegraded":"2023-09-23T20:27:25.545370+0000","last_fullsized":"2023-09-23T20:27:25.545370+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":567,"log_dups_size":0,"ondisk_log_size":567,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:46:35.659960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":37158912,"num_objects":567,"num_object_clones":0,"num_object_copies":1701,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":567,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":567,"num_write_kb":36288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.4","version":"90'161","reported_seq":446,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T20:27:20.426080+0000","last_change":"2023-09-23T20:27:13.244744+0000","last_active":"2023-09-23T20:27:20.426080+0000","last_peered":"2023-09-23T20:27:20.426080+0000","last_clean":"2023-09-23T20:27:20.426080+0000","last_became_active":"2023-09-23T20:27:11.960349+0000","last_became_peered":"2023-09-23T20:27:11.960349+0000","last_unstale":"2023-09-23T20:27:20.426080+0000","last_undegraded":"2023-09-23T20:27:20.426080+0000","last_fullsized":"2023-09-23T20:27:20.426080+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:13.244651+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.244651+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:22:26.451174+0000","scrub_duration":0.0019955509999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"86'117","reported_seq":740,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T20:27:20.945964+0000","last_change":"2023-09-23T20:25:40.617386+0000","last_active":"2023-09-23T20:27:20.945964+0000","last_peered":"2023-09-23T20:27:20.945964+0000","last_clean":"2023-09-23T20:27:20.945964+0000","last_became_active":"2023-09-23T20:25:40.616747+0000","last_became_peered":"2023-09-23T20:25:40.616747+0000","last_unstale":"2023-09-23T20:27:20.945964+0000","last_undegraded":"2023-09-23T20:27:20.945964+0000","last_fullsized":"2023-09-23T20:27:20.945964+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:46:44.551561+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046891400000000002,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":893,"num_read_kb":922,"num_write":82,"num_write_kb":4193,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":171,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T20:27:20.944432+0000","last_change":"2023-09-23T20:20:46.937163+0000","last_active":"2023-09-23T20:27:20.944432+0000","last_peered":"2023-09-23T20:27:20.944432+0000","last_clean":"2023-09-23T20:27:20.944432+0000","last_became_active":"2023-09-23T20:20:46.936846+0000","last_became_peered":"2023-09-23T20:20:46.936846+0000","last_unstale":"2023-09-23T20:27:20.944432+0000","last_undegraded":"2023-09-23T20:27:20.944432+0000","last_fullsized":"2023-09-23T20:27:20.944432+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:01:32.798855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":162,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T20:27:20.872249+0000","last_change":"2023-09-23T20:12:43.765908+0000","last_active":"2023-09-23T20:27:20.872249+0000","last_peered":"2023-09-23T20:27:20.872249+0000","last_clean":"2023-09-23T20:27:20.872249+0000","last_became_active":"2023-09-23T20:12:43.765558+0000","last_became_peered":"2023-09-23T20:12:43.765558+0000","last_unstale":"2023-09-23T20:27:20.872249+0000","last_undegraded":"2023-09-23T20:27:20.872249+0000","last_fullsized":"2023-09-23T20:27:20.872249+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:35:43.809898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":153,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T20:27:20.944243+0000","last_change":"2023-09-23T20:20:14.888484+0000","last_active":"2023-09-23T20:27:20.944243+0000","last_peered":"2023-09-23T20:27:20.944243+0000","last_clean":"2023-09-23T20:27:20.944243+0000","last_became_active":"2023-09-23T20:20:14.887219+0000","last_became_peered":"2023-09-23T20:20:14.887219+0000","last_unstale":"2023-09-23T20:27:20.944243+0000","last_undegraded":"2023-09-23T20:27:20.944243+0000","last_fullsized":"2023-09-23T20:27:20.944243+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:13:53.763649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":55,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T20:27:20.943207+0000","last_change":"2023-09-23T20:25:39.639268+0000","last_active":"2023-09-23T20:27:20.943207+0000","last_peered":"2023-09-23T20:27:20.943207+0000","last_clean":"2023-09-23T20:27:20.943207+0000","last_became_active":"2023-09-23T20:25:39.638922+0000","last_became_peered":"2023-09-23T20:25:39.638922+0000","last_unstale":"2023-09-23T20:27:20.943207+0000","last_undegraded":"2023-09-23T20:27:20.943207+0000","last_fullsized":"2023-09-23T20:27:20.943207+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:09:04.040098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":182,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T20:27:20.412826+0000","last_change":"2023-09-23T20:20:46.938981+0000","last_active":"2023-09-23T20:27:20.412826+0000","last_peered":"2023-09-23T20:27:20.412826+0000","last_clean":"2023-09-23T20:27:20.412826+0000","last_became_active":"2023-09-23T20:20:46.938627+0000","last_became_peered":"2023-09-23T20:20:46.938627+0000","last_unstale":"2023-09-23T20:27:20.412826+0000","last_undegraded":"2023-09-23T20:27:20.412826+0000","last_fullsized":"2023-09-23T20:27:20.412826+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:44:43.177086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":226,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T20:27:20.872703+0000","last_change":"2023-09-23T20:25:02.945142+0000","last_active":"2023-09-23T20:27:20.872703+0000","last_peered":"2023-09-23T20:27:20.872703+0000","last_clean":"2023-09-23T20:27:20.872703+0000","last_became_active":"2023-09-23T20:25:02.944653+0000","last_became_peered":"2023-09-23T20:25:02.944653+0000","last_unstale":"2023-09-23T20:27:20.872703+0000","last_undegraded":"2023-09-23T20:27:20.872703+0000","last_fullsized":"2023-09-23T20:27:20.872703+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:38:12.927982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":165,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T20:27:20.412875+0000","last_change":"2023-09-23T20:12:43.753383+0000","last_active":"2023-09-23T20:27:20.412875+0000","last_peered":"2023-09-23T20:27:20.412875+0000","last_clean":"2023-09-23T20:27:20.412875+0000","last_became_active":"2023-09-23T20:12:43.752921+0000","last_became_peered":"2023-09-23T20:12:43.752921+0000","last_unstale":"2023-09-23T20:27:20.412875+0000","last_undegraded":"2023-09-23T20:27:20.412875+0000","last_fullsized":"2023-09-23T20:27:20.412875+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:28:26.982998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":226,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T20:27:20.872594+0000","last_change":"2023-09-23T20:25:03.103656+0000","last_active":"2023-09-23T20:27:20.872594+0000","last_peered":"2023-09-23T20:27:20.872594+0000","last_clean":"2023-09-23T20:27:20.872594+0000","last_became_active":"2023-09-23T20:25:03.103311+0000","last_became_peered":"2023-09-23T20:25:03.103311+0000","last_unstale":"2023-09-23T20:27:20.872594+0000","last_undegraded":"2023-09-23T20:27:20.872594+0000","last_fullsized":"2023-09-23T20:27:20.872594+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:55:54.677030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":76,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T20:27:20.426428+0000","last_change":"2023-09-23T20:25:39.629869+0000","last_active":"2023-09-23T20:27:20.426428+0000","last_peered":"2023-09-23T20:27:20.426428+0000","last_clean":"2023-09-23T20:27:20.426428+0000","last_became_active":"2023-09-23T20:25:39.629543+0000","last_became_peered":"2023-09-23T20:25:39.629543+0000","last_unstale":"2023-09-23T20:27:20.426428+0000","last_undegraded":"2023-09-23T20:27:20.426428+0000","last_fullsized":"2023-09-23T20:27:20.426428+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:24:48.220679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":76,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-09-23T20:27:22.554221+0000","last_change":"2023-09-23T20:25:39.622819+0000","last_active":"2023-09-23T20:27:22.554221+0000","last_peered":"2023-09-23T20:27:22.554221+0000","last_clean":"2023-09-23T20:27:22.554221+0000","last_became_active":"2023-09-23T20:25:39.622484+0000","last_became_peered":"2023-09-23T20:25:39.622484+0000","last_unstale":"2023-09-23T20:27:22.554221+0000","last_undegraded":"2023-09-23T20:27:22.554221+0000","last_fullsized":"2023-09-23T20:27:22.554221+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:51:53.973261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":199,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T20:27:20.944546+0000","last_change":"2023-09-23T20:25:39.668882+0000","last_active":"2023-09-23T20:27:20.944546+0000","last_peered":"2023-09-23T20:27:20.944546+0000","last_clean":"2023-09-23T20:27:20.944546+0000","last_became_active":"2023-09-23T20:25:39.637128+0000","last_became_peered":"2023-09-23T20:25:39.637128+0000","last_unstale":"2023-09-23T20:27:20.944546+0000","last_undegraded":"2023-09-23T20:27:20.944546+0000","last_fullsized":"2023-09-23T20:27:20.944546+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:56.406298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":199,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T20:27:20.872249+0000","last_change":"2023-09-23T20:24:53.089249+0000","last_active":"2023-09-23T20:27:20.872249+0000","last_peered":"2023-09-23T20:27:20.872249+0000","last_clean":"2023-09-23T20:27:20.872249+0000","last_became_active":"2023-09-23T20:24:53.088952+0000","last_became_peered":"2023-09-23T20:24:53.088952+0000","last_unstale":"2023-09-23T20:27:20.872249+0000","last_undegraded":"2023-09-23T20:27:20.872249+0000","last_fullsized":"2023-09-23T20:27:20.872249+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:38:03.216069+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":764870656,"num_objects":11671,"num_object_clones":0,"num_object_copies":35013,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":11671,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11671,"num_write_kb":746944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2320891904,"data_stored":2320891904,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":11671,"ondisk_log_size":11671,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":15,"num_bytes_recovered":1356,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":25181,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":757038,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":357,"num_read_kb":667,"num_write":173,"num_write_kb":1756,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1550762,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1771,"ondisk_log_size":1771,"up":33,"acting":33,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6082,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201475,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297944,"kb_used_data":189200,"kb_used_omap":0,"kb_used_meta":108736,"kb_avail":93447208,"statfs":{"total":95995035648,"available":95689940992,"internally_reserved":0,"allocated":193740800,"data_stored":193025208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":111345664},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,2,0,3],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:27:24 2023","interfaces":[{"interface":"back","average":{"1min":3.9889999999999999,"5min":1.9059999999999999,"15min":1.0149999999999999},"min":{"1min":0.29399999999999998,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":48.798999999999999,"5min":48.798999999999999,"15min":48.798999999999999},"last":0.69999999999999996},{"interface":"front","average":{"1min":3.9540000000000002,"5min":2.2770000000000001,"15min":1.135},"min":{"1min":0.251,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":48.247,"5min":48.247,"15min":48.247},"last":0.434}]},{"osd":1,"last update":"Sat Sep 23 20:26:58 2023","interfaces":[{"interface":"back","average":{"1min":1.2929999999999999,"5min":1.6499999999999999,"15min":0.92500000000000004},"min":{"1min":0.28599999999999998,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":3.1920000000000002,"5min":17.795999999999999,"15min":17.795999999999999},"last":0.35099999999999998},{"interface":"front","average":{"1min":2.2989999999999999,"5min":1.504,"15min":0.90900000000000003},"min":{"1min":0.33400000000000002,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":6.4950000000000001,"5min":13.250999999999999,"15min":13.250999999999999},"last":0.66400000000000003}]},{"osd":2,"last update":"Sat Sep 23 20:26:28 2023","interfaces":[{"interface":"back","average":{"1min":3.2610000000000001,"5min":1.6080000000000001,"15min":0.91500000000000004},"min":{"1min":0.39000000000000001,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":8.1270000000000007,"5min":9.8300000000000001,"15min":9.8300000000000001},"last":2.9580000000000002},{"interface":"front","average":{"1min":3.1850000000000001,"5min":1.708,"15min":0.95999999999999996},"min":{"1min":0.48099999999999998,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":6.6699999999999999,"5min":11.423999999999999,"15min":11.423999999999999},"last":3.0339999999999998}]},{"osd":3,"last update":"Sat Sep 23 20:26:58 2023","interfaces":[{"interface":"back","average":{"1min":2.4049999999999998,"5min":1.756,"15min":1.022},"min":{"1min":0.38100000000000001,"5min":0.314,"15min":0.314},"max":{"1min":6.891,"5min":6.891,"15min":6.891},"last":0.60399999999999998},{"interface":"front","average":{"1min":2.4319999999999999,"5min":1.6279999999999999,"15min":0.99199999999999999},"min":{"1min":0.374,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":6.9050000000000002,"5min":9.7840000000000007,"15min":9.7840000000000007},"last":0.57299999999999995}]},{"osd":4,"last update":"Sat Sep 23 20:26:58 2023","interfaces":[{"interface":"back","average":{"1min":1.6479999999999999,"5min":1.1619999999999999,"15min":0.71899999999999997},"min":{"1min":0.28499999999999998,"5min":0.27000000000000002,"15min":0.26700000000000002},"max":{"1min":5.3540000000000001,"5min":13.141999999999999,"15min":13.141999999999999},"last":2.2629999999999999},{"interface":"front","average":{"1min":1.5740000000000001,"5min":1.6819999999999999,"15min":0.91400000000000003},"min":{"1min":0.29499999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":5.9279999999999999,"5min":10.853,"15min":10.853},"last":2.206}]},{"osd":5,"last update":"Sat Sep 23 20:26:58 2023","interfaces":[{"interface":"back","average":{"1min":1.9930000000000001,"5min":1.643,"15min":0.90500000000000003},"min":{"1min":0.31,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":10.362,"5min":10.563000000000001,"15min":10.563000000000001},"last":0.40799999999999997},{"interface":"front","average":{"1min":1.956,"5min":1.5669999999999999,"15min":0.91100000000000003},"min":{"1min":0.41799999999999998,"5min":0.34699999999999998,"15min":0.31},"max":{"1min":7.9029999999999996,"5min":13.292,"15min":13.292},"last":1.371}]},{"osd":6,"last update":"Sat Sep 23 20:26:58 2023","interfaces":[{"interface":"back","average":{"1min":1.9890000000000001,"5min":1.4179999999999999,"15min":0.86899999999999999},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":6.2210000000000001,"5min":13.185,"15min":13.185},"last":1.5669999999999999},{"interface":"front","average":{"1min":1.9199999999999999,"5min":1.4890000000000001,"15min":0.89100000000000001},"min":{"1min":0.378,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":6.4370000000000003,"5min":9.9120000000000008,"15min":9.9120000000000008},"last":1.585}]}]},{"osd":6,"up_from":47,"seq":201863463111,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":228624,"kb_used_data":137224,"kb_used_omap":0,"kb_used_meta":91392,"kb_avail":93516528,"statfs":{"total":95995035648,"available":95760924672,"internally_reserved":0,"allocated":140517376,"data_stored":139809849,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":93585408},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,2,2,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:26:26 2023","interfaces":[{"interface":"back","average":{"1min":1.7270000000000001,"5min":1.23,"15min":0.79500000000000004},"min":{"1min":0.221,"5min":0.221,"15min":0.221},"max":{"1min":8.1940000000000008,"5min":8.1940000000000008,"15min":8.1940000000000008},"last":0.28499999999999998},{"interface":"front","average":{"1min":1.3069999999999999,"5min":1.4850000000000001,"15min":0.87},"min":{"1min":0.26300000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":4.0609999999999999,"5min":12.512,"15min":12.512},"last":0.35099999999999998}]},{"osd":1,"last update":"Sat Sep 23 20:27:02 2023","interfaces":[{"interface":"back","average":{"1min":2.6379999999999999,"5min":1.8839999999999999,"15min":1.0109999999999999},"min":{"1min":0.28899999999999998,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":8.3520000000000003,"5min":18.001000000000001,"15min":18.001000000000001},"last":0.48399999999999999},{"interface":"front","average":{"1min":2.391,"5min":1.569,"15min":0.88800000000000001},"min":{"1min":0.28699999999999998,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":8.532,"5min":13.91,"15min":13.91},"last":0.91100000000000003}]},{"osd":2,"last update":"Sat Sep 23 20:26:26 2023","interfaces":[{"interface":"back","average":{"1min":2.153,"5min":1.385,"15min":0.85799999999999998},"min":{"1min":0.32800000000000001,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":8.0090000000000003,"5min":13.856,"15min":13.856},"last":0.877},{"interface":"front","average":{"1min":2.0720000000000001,"5min":1.284,"15min":0.83399999999999996},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":6.8140000000000001,"5min":6.8140000000000001,"15min":6.8140000000000001},"last":0.92500000000000004}]},{"osd":3,"last update":"Sat Sep 23 20:27:02 2023","interfaces":[{"interface":"back","average":{"1min":2.105,"5min":1.833,"15min":1.0189999999999999},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":8.2989999999999995,"5min":9.4239999999999995,"15min":9.4239999999999995},"last":0.78900000000000003},{"interface":"front","average":{"1min":2.754,"5min":1.8280000000000001,"15min":1.0600000000000001},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":8.3770000000000007,"5min":8.6050000000000004,"15min":8.6050000000000004},"last":1.0660000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:26:48 2023","interfaces":[{"interface":"back","average":{"1min":1.5049999999999999,"5min":1.2669999999999999,"15min":0.72999999999999998},"min":{"1min":0.27600000000000002,"5min":0.27200000000000002,"15min":0.26200000000000001},"max":{"1min":10.173,"5min":10.173,"15min":10.173},"last":0.40500000000000003},{"interface":"front","average":{"1min":1.962,"5min":1.3540000000000001,"15min":0.78000000000000003},"min":{"1min":0.312,"5min":0.29799999999999999,"15min":0.28499999999999998},"max":{"1min":8.3249999999999993,"5min":8.3249999999999993,"15min":8.3249999999999993},"last":0.41799999999999998}]},{"osd":5,"last update":"Sat Sep 23 20:26:48 2023","interfaces":[{"interface":"back","average":{"1min":1.202,"5min":1.4119999999999999,"15min":0.80300000000000005},"min":{"1min":0.31,"5min":0.31,"15min":0.29799999999999999},"max":{"1min":4.4580000000000002,"5min":14.085000000000001,"15min":14.085000000000001},"last":0.33000000000000002},{"interface":"front","average":{"1min":2.831,"5min":1.8660000000000001,"15min":0.97299999999999998},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.29499999999999998},"max":{"1min":18.192,"5min":18.192,"15min":18.192},"last":0.46400000000000002}]},{"osd":7,"last update":"Sat Sep 23 20:27:02 2023","interfaces":[{"interface":"back","average":{"1min":1.946,"5min":1.538,"15min":0.877},"min":{"1min":0.36899999999999999,"5min":0.33500000000000002,"15min":0.32800000000000001},"max":{"1min":8.0350000000000001,"5min":10.217000000000001,"15min":10.217000000000001},"last":0.372},{"interface":"front","average":{"1min":1.6459999999999999,"5min":1.6020000000000001,"15min":0.91400000000000003},"min":{"1min":0.36499999999999999,"5min":0.34999999999999998,"15min":0.315},"max":{"1min":8.407,"5min":9.7360000000000007,"15min":9.7360000000000007},"last":0.53800000000000003}]}]},{"osd":1,"up_from":99,"seq":425201762386,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":277040,"kb_used_data":119016,"kb_used_omap":6,"kb_used_meta":158009,"kb_avail":93468112,"statfs":{"total":95995035648,"available":95711346688,"internally_reserved":0,"allocated":121872384,"data_stored":120724912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6696,"internal_metadata":161801688},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,1,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:26:54 2023","interfaces":[{"interface":"back","average":{"1min":2.6600000000000001,"5min":2.052,"15min":0.93500000000000005},"min":{"1min":0.215,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":14.384,"5min":40.113,"15min":40.113},"last":0.56899999999999995},{"interface":"front","average":{"1min":3.1869999999999998,"5min":1.8120000000000001,"15min":0.87},"min":{"1min":0.23699999999999999,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":22.210000000000001,"5min":42.994999999999997,"15min":42.994999999999997},"last":0.73899999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:26:27 2023","interfaces":[{"interface":"back","average":{"1min":1.6479999999999999,"5min":1.8029999999999999,"15min":0.85499999999999998},"min":{"1min":0.27900000000000003,"5min":0.253,"15min":0.253},"max":{"1min":9.6839999999999993,"5min":39.962000000000003,"15min":39.962000000000003},"last":0.33500000000000002},{"interface":"front","average":{"1min":2.806,"5min":1.794,"15min":0.96599999999999997},"min":{"1min":0.26700000000000002,"5min":0.246,"15min":0.246},"max":{"1min":8.7910000000000004,"5min":47.442,"15min":47.442},"last":1.4750000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:27:00 2023","interfaces":[{"interface":"back","average":{"1min":1.9099999999999999,"5min":1.4039999999999999,"15min":0.83399999999999996},"min":{"1min":0.309,"5min":0.246,"15min":0.246},"max":{"1min":9.0410000000000004,"5min":9.0410000000000004,"15min":9.0410000000000004},"last":1.2609999999999999},{"interface":"front","average":{"1min":2.4319999999999999,"5min":1.4710000000000001,"15min":0.76200000000000001},"min":{"1min":0.29699999999999999,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":15.147,"5min":15.147,"15min":15.147},"last":0.93799999999999994}]},{"osd":4,"last update":"Sat Sep 23 20:26:54 2023","interfaces":[{"interface":"back","average":{"1min":2.5790000000000002,"5min":1.5900000000000001,"15min":0.94099999999999995},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":15.353,"5min":15.353,"15min":15.353},"last":2.105},{"interface":"front","average":{"1min":1.7609999999999999,"5min":1.536,"15min":0.93500000000000005},"min":{"1min":0.32400000000000001,"5min":0.312,"15min":0.312},"max":{"1min":6.2270000000000003,"5min":8.7899999999999991,"15min":8.7899999999999991},"last":2.024}]},{"osd":5,"last update":"Sat Sep 23 20:26:54 2023","interfaces":[{"interface":"back","average":{"1min":2.6360000000000001,"5min":1.6220000000000001,"15min":0.96799999999999997},"min":{"1min":0.33000000000000002,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":7.2000000000000002,"5min":12.590999999999999,"15min":12.590999999999999},"last":0.52100000000000002},{"interface":"front","average":{"1min":2.2530000000000001,"5min":2.137,"15min":1.163},"min":{"1min":0.38300000000000001,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":6.3140000000000001,"5min":39.579000000000001,"15min":39.579000000000001},"last":0.68700000000000006}]},{"osd":6,"last update":"Sat Sep 23 20:26:54 2023","interfaces":[{"interface":"back","average":{"1min":2.9630000000000001,"5min":1.996,"15min":1.1100000000000001},"min":{"1min":0.42999999999999999,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":7.226,"5min":34.409999999999997,"15min":34.409999999999997},"last":1.518},{"interface":"front","average":{"1min":2.964,"5min":2.2269999999999999,"15min":1.167},"min":{"1min":0.38100000000000001,"5min":0.377,"15min":0.377},"max":{"1min":15.313000000000001,"5min":35.554000000000002,"15min":35.554000000000002},"last":1.8109999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:26:54 2023","interfaces":[{"interface":"back","average":{"1min":2.8719999999999999,"5min":1.8480000000000001,"15min":1.05},"min":{"1min":0.42499999999999999,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":9.7439999999999998,"5min":9.7439999999999998,"15min":9.7439999999999998},"last":1.6910000000000001},{"interface":"front","average":{"1min":2.3180000000000001,"5min":1.7729999999999999,"15min":1.0469999999999999},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":6.2519999999999998,"5min":9.2279999999999998,"15min":9.2279999999999998},"last":1.6240000000000001}]}]},{"osd":0,"up_from":94,"seq":403726925913,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":533768,"kb_used_data":322816,"kb_used_omap":7,"kb_used_meta":210936,"kb_avail":93211384,"statfs":{"total":95995035648,"available":95448457216,"internally_reserved":0,"allocated":330563584,"data_stored":329514746,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":215998481},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:26:56 2023","interfaces":[{"interface":"back","average":{"1min":3.0779999999999998,"5min":1.3400000000000001,"15min":0.68600000000000005},"min":{"1min":0.223,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":24.247,"5min":24.247,"15min":24.247},"last":0.872},{"interface":"front","average":{"1min":2.9950000000000001,"5min":1.3160000000000001,"15min":0.69599999999999995},"min":{"1min":0.28999999999999998,"5min":0.222,"15min":0.222},"max":{"1min":15.994999999999999,"5min":15.994999999999999,"15min":15.994999999999999},"last":2.1600000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:27:24 2023","interfaces":[{"interface":"back","average":{"1min":1.2190000000000001,"5min":1.9450000000000001,"15min":1.046},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23300000000000001},"max":{"1min":7.5199999999999996,"5min":23.832999999999998,"15min":23.832999999999998},"last":1.514},{"interface":"front","average":{"1min":2.2490000000000001,"5min":1.607,"15min":0.86599999999999999},"min":{"1min":0.19700000000000001,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":15.948,"5min":15.948,"15min":15.948},"last":0.20999999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:26:56 2023","interfaces":[{"interface":"back","average":{"1min":3.3109999999999999,"5min":1.7070000000000001,"15min":0.96699999999999997},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":23.748999999999999,"5min":23.748999999999999,"15min":23.748999999999999},"last":1.552},{"interface":"front","average":{"1min":3.9809999999999999,"5min":1.5189999999999999,"15min":0.78700000000000003},"min":{"1min":0.315,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":24.274999999999999,"5min":24.274999999999999,"15min":24.274999999999999},"last":1.224}]},{"osd":4,"last update":"Sat Sep 23 20:27:24 2023","interfaces":[{"interface":"back","average":{"1min":1.55,"5min":2.0960000000000001,"15min":1.1100000000000001},"min":{"1min":0.36899999999999999,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":6.5899999999999999,"5min":23.876999999999999,"15min":23.876999999999999},"last":0.46000000000000002},{"interface":"front","average":{"1min":1.53,"5min":1.772,"15min":0.999},"min":{"1min":0.39500000000000002,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":6.0300000000000002,"5min":23.681999999999999,"15min":23.681999999999999},"last":0.36799999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:27:24 2023","interfaces":[{"interface":"back","average":{"1min":1.8640000000000001,"5min":2.4729999999999999,"15min":1.262},"min":{"1min":0.47399999999999998,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":9.3369999999999997,"5min":24.198,"15min":24.198},"last":0.71599999999999997},{"interface":"front","average":{"1min":1.1539999999999999,"5min":1.8500000000000001,"15min":1.069},"min":{"1min":0.41799999999999998,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":6.1749999999999998,"5min":11.167999999999999,"15min":11.167999999999999},"last":1.591}]},{"osd":6,"last update":"Sat Sep 23 20:27:24 2023","interfaces":[{"interface":"back","average":{"1min":1.6799999999999999,"5min":1.7749999999999999,"15min":1.0269999999999999},"min":{"1min":0.42299999999999999,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":9.2750000000000004,"5min":24.408000000000001,"15min":24.408000000000001},"last":0.318},{"interface":"front","average":{"1min":1.1259999999999999,"5min":1.6140000000000001,"15min":0.98099999999999998},"min":{"1min":0.442,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":6.1319999999999997,"5min":25.271000000000001,"15min":25.271000000000001},"last":0.38800000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:27:24 2023","interfaces":[{"interface":"back","average":{"1min":1.1619999999999999,"5min":1.8180000000000001,"15min":1.0569999999999999},"min":{"1min":0.47799999999999998,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":6.2039999999999997,"5min":25.359000000000002,"15min":25.359000000000002},"last":0.34499999999999997},{"interface":"front","average":{"1min":1.5169999999999999,"5min":2.3119999999999998,"15min":1.2270000000000001},"min":{"1min":0.45600000000000002,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":9.8239999999999998,"5min":23.905999999999999,"15min":23.905999999999999},"last":0.40699999999999997}]}]},{"osd":2,"up_from":104,"seq":446676598860,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":534972,"kb_used_data":382772,"kb_used_omap":11,"kb_used_meta":152180,"kb_avail":93210180,"statfs":{"total":95995035648,"available":95447224320,"internally_reserved":0,"allocated":391958528,"data_stored":390918029,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11878,"internal_metadata":155832730},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:26:30 2023","interfaces":[{"interface":"back","average":{"1min":2.8090000000000002,"5min":1.706,"15min":0.81999999999999995},"min":{"1min":0.21099999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":9.5980000000000008,"5min":12.811,"15min":12.811},"last":0.33300000000000002},{"interface":"front","average":{"1min":2.1019999999999999,"5min":1.2709999999999999,"15min":0.68700000000000006},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":15.746,"5min":15.746,"15min":15.746},"last":0.32100000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:26:30 2023","interfaces":[{"interface":"back","average":{"1min":2.9390000000000001,"5min":1.524,"15min":0.77000000000000002},"min":{"1min":0.28599999999999998,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":14.135999999999999,"5min":14.135999999999999,"15min":14.135999999999999},"last":0.29799999999999999},{"interface":"front","average":{"1min":2.617,"5min":1.226,"15min":0.66400000000000003},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":13.718,"5min":13.718,"15min":13.718},"last":0.34599999999999997}]},{"osd":3,"last update":"Sat Sep 23 20:26:55 2023","interfaces":[{"interface":"back","average":{"1min":2.246,"5min":1.349,"15min":0.69099999999999995},"min":{"1min":0.26500000000000001,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":10.916,"5min":15.319000000000001,"15min":15.319000000000001},"last":0.28199999999999997},{"interface":"front","average":{"1min":2.294,"5min":1.552,"15min":0.78200000000000003},"min":{"1min":0.25800000000000001,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":12.494999999999999,"5min":12.494999999999999,"15min":12.494999999999999},"last":0.41899999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:26:30 2023","interfaces":[{"interface":"back","average":{"1min":1.996,"5min":1.234,"15min":0.80000000000000004},"min":{"1min":0.36299999999999999,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":8.2300000000000004,"5min":8.2300000000000004,"15min":8.2300000000000004},"last":0.68799999999999994},{"interface":"front","average":{"1min":2.3679999999999999,"5min":1.3979999999999999,"15min":0.85599999999999998},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":8.6530000000000005,"5min":8.6530000000000005,"15min":8.6530000000000005},"last":0.55000000000000004}]},{"osd":5,"last update":"Sat Sep 23 20:26:30 2023","interfaces":[{"interface":"back","average":{"1min":2.8319999999999999,"5min":1.577,"15min":0.96299999999999997},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":11.237,"5min":11.237,"15min":11.237},"last":0.72399999999999998},{"interface":"front","average":{"1min":3.452,"5min":1.6850000000000001,"15min":0.999},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":11.266999999999999,"5min":11.266999999999999,"15min":11.266999999999999},"last":0.70899999999999996}]},{"osd":6,"last update":"Sat Sep 23 20:26:30 2023","interfaces":[{"interface":"back","average":{"1min":2.0129999999999999,"5min":1.294,"15min":0.86899999999999999},"min":{"1min":0.49099999999999999,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":6.2599999999999998,"5min":6.7939999999999996,"15min":6.7939999999999996},"last":0.65100000000000002},{"interface":"front","average":{"1min":2.7410000000000001,"5min":1.585,"15min":0.97699999999999998},"min":{"1min":0.41099999999999998,"5min":0.377,"15min":0.377},"max":{"1min":9.6600000000000001,"5min":9.6600000000000001,"15min":9.6600000000000001},"last":0.68000000000000005}]},{"osd":7,"last update":"Sat Sep 23 20:26:30 2023","interfaces":[{"interface":"back","average":{"1min":3.1150000000000002,"5min":1.712,"15min":1.002},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":15.343999999999999,"5min":15.343999999999999,"15min":15.343999999999999},"last":0.69999999999999996},{"interface":"front","average":{"1min":2.6299999999999999,"5min":1.5840000000000001,"15min":0.96599999999999997},"min":{"1min":0.48899999999999999,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":9.1440000000000001,"5min":10.064,"15min":10.064},"last":0.53900000000000003}]}]},{"osd":3,"up_from":109,"seq":468151435334,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":706056,"kb_used_data":553856,"kb_used_omap":4,"kb_used_meta":152187,"kb_avail":93039096,"statfs":{"total":95995035648,"available":95272034304,"internally_reserved":0,"allocated":567148544,"data_stored":566095480,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4514,"internal_metadata":155840094},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,1,2,3],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:26:56 2023","interfaces":[{"interface":"back","average":{"1min":3.2530000000000001,"5min":2.319,"15min":1.0069999999999999},"min":{"1min":0.28999999999999998,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":15.589,"5min":67.305999999999997,"15min":67.305999999999997},"last":0.72999999999999998},{"interface":"front","average":{"1min":1.48,"5min":2.0339999999999998,"15min":0.93400000000000005},"min":{"1min":0.29699999999999999,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":9.0920000000000005,"5min":67.403999999999996,"15min":67.403999999999996},"last":0.68500000000000005}]},{"osd":1,"last update":"Sat Sep 23 20:26:56 2023","interfaces":[{"interface":"back","average":{"1min":3.0230000000000001,"5min":1.5069999999999999,"15min":0.76700000000000002},"min":{"1min":0.25,"5min":0.214,"15min":0.214},"max":{"1min":19.675999999999998,"5min":19.675999999999998,"15min":19.675999999999998},"last":0.33100000000000002},{"interface":"front","average":{"1min":2.4590000000000001,"5min":1.375,"15min":0.755},"min":{"1min":0.33200000000000002,"5min":0.245,"15min":0.245},"max":{"1min":11.644,"5min":11.644,"15min":11.644},"last":0.29399999999999998}]},{"osd":2,"last update":"Sat Sep 23 20:26:56 2023","interfaces":[{"interface":"back","average":{"1min":3.4369999999999998,"5min":1.7789999999999999,"15min":0.82799999999999996},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":24.045000000000002,"5min":24.045000000000002,"15min":24.045000000000002},"last":0.88500000000000001},{"interface":"front","average":{"1min":3.4119999999999999,"5min":1.8080000000000001,"15min":0.91200000000000003},"min":{"1min":0.33700000000000002,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":12.427,"5min":17.684000000000001,"15min":17.684000000000001},"last":0.38300000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:26:56 2023","interfaces":[{"interface":"back","average":{"1min":2.875,"5min":1.4590000000000001,"15min":0.88900000000000001},"min":{"1min":0.33100000000000002,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":11.880000000000001,"5min":11.880000000000001,"15min":11.880000000000001},"last":0.40600000000000003},{"interface":"front","average":{"1min":3.698,"5min":1.8180000000000001,"15min":1.016},"min":{"1min":0.378,"5min":0.371,"15min":0.371},"max":{"1min":21.673999999999999,"5min":21.673999999999999,"15min":21.673999999999999},"last":1.532}]},{"osd":5,"last update":"Sat Sep 23 20:26:56 2023","interfaces":[{"interface":"back","average":{"1min":3.117,"5min":1.8560000000000001,"15min":1.0229999999999999},"min":{"1min":0.375,"5min":0.311,"15min":0.311},"max":{"1min":11.839,"5min":11.839,"15min":11.839},"last":0.44600000000000001},{"interface":"front","average":{"1min":1.825,"5min":1.2949999999999999,"15min":0.85199999999999998},"min":{"1min":0.40000000000000002,"5min":0.313,"15min":0.313},"max":{"1min":8.5570000000000004,"5min":8.5570000000000004,"15min":8.5570000000000004},"last":0.58899999999999997}]},{"osd":6,"last update":"Sat Sep 23 20:26:56 2023","interfaces":[{"interface":"back","average":{"1min":2.0609999999999999,"5min":1.5900000000000001,"15min":0.95499999999999996},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":8.6059999999999999,"5min":8.6059999999999999,"15min":8.6059999999999999},"last":0.65500000000000003},{"interface":"front","average":{"1min":2.7469999999999999,"5min":1.5740000000000001,"15min":0.97199999999999998},"min":{"1min":0.40600000000000003,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":9.3179999999999996,"5min":9.3179999999999996,"15min":9.3179999999999996},"last":1.4870000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:26:56 2023","interfaces":[{"interface":"back","average":{"1min":3.0030000000000001,"5min":1.5860000000000001,"15min":0.997},"min":{"1min":0.40999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":9.2750000000000004,"5min":9.2750000000000004,"15min":9.2750000000000004},"last":1.569},{"interface":"front","average":{"1min":3.6680000000000001,"5min":2.0209999999999999,"15min":1.101},"min":{"1min":0.40699999999999997,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":12.468999999999999,"5min":12.468999999999999,"15min":12.468999999999999},"last":0.70399999999999996}]}]},{"osd":4,"up_from":30,"seq":128849019087,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":319440,"kb_used_data":187208,"kb_used_omap":0,"kb_used_meta":132224,"kb_avail":93425712,"statfs":{"total":95995035648,"available":95667929088,"internally_reserved":0,"allocated":191700992,"data_stored":190993199,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":135397376},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:27:24 2023","interfaces":[{"interface":"back","average":{"1min":2.028,"5min":1.6950000000000001,"15min":0.94699999999999995},"min":{"1min":0.23699999999999999,"5min":0.222,"15min":0.222},"max":{"1min":6.5780000000000003,"5min":20.411999999999999,"15min":20.411999999999999},"last":0.82599999999999996},{"interface":"front","average":{"1min":1.7170000000000001,"5min":1.7649999999999999,"15min":1.016},"min":{"1min":0.307,"5min":0.253,"15min":0.253},"max":{"1min":6.5919999999999996,"5min":20.454999999999998,"15min":20.454999999999998},"last":0.84499999999999997}]},{"osd":1,"last update":"Sat Sep 23 20:27:00 2023","interfaces":[{"interface":"back","average":{"1min":2.883,"5min":1.5089999999999999,"15min":0.92000000000000004},"min":{"1min":0.24299999999999999,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":9.6660000000000004,"5min":9.6660000000000004,"15min":9.6660000000000004},"last":0.63300000000000001},{"interface":"front","average":{"1min":2.4990000000000001,"5min":1.5640000000000001,"15min":0.93799999999999994},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":12.821999999999999,"5min":12.821999999999999,"15min":12.821999999999999},"last":0.90400000000000003}]},{"osd":2,"last update":"Sat Sep 23 20:26:32 2023","interfaces":[{"interface":"back","average":{"1min":2.1779999999999999,"5min":1.51,"15min":0.90800000000000003},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":11.412000000000001,"5min":21.462,"15min":21.462},"last":0.497},{"interface":"front","average":{"1min":3.29,"5min":1.8660000000000001,"15min":1.014},"min":{"1min":0.28999999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":13.567,"5min":22.593,"15min":22.593},"last":0.41099999999999998}]},{"osd":3,"last update":"Sat Sep 23 20:27:00 2023","interfaces":[{"interface":"back","average":{"1min":2.516,"5min":1.7929999999999999,"15min":1.0249999999999999},"min":{"1min":0.32900000000000001,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":10.846,"5min":24.891999999999999,"15min":24.891999999999999},"last":0.57399999999999995},{"interface":"front","average":{"1min":2.464,"5min":1.831,"15min":1.0529999999999999},"min":{"1min":0.35499999999999998,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":9.2080000000000002,"5min":21.922000000000001,"15min":21.922000000000001},"last":0.53700000000000003}]},{"osd":5,"last update":"Sat Sep 23 20:27:11 2023","interfaces":[{"interface":"back","average":{"1min":1.671,"5min":1.8859999999999999,"15min":0.95299999999999996},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":10.252000000000001,"5min":17.704000000000001,"15min":17.704000000000001},"last":0.61399999999999999},{"interface":"front","average":{"1min":1.964,"5min":1.643,"15min":0.88400000000000001},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":8.2110000000000003,"5min":15.329000000000001,"15min":15.329000000000001},"last":0.48699999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:26:34 2023","interfaces":[{"interface":"back","average":{"1min":3.3639999999999999,"5min":1.9239999999999999,"15min":0.98099999999999998},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.27500000000000002},"max":{"1min":15.252000000000001,"5min":41.371000000000002,"15min":41.371000000000002},"last":0.44400000000000001},{"interface":"front","average":{"1min":1.9179999999999999,"5min":1.593,"15min":0.88200000000000001},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":9.3350000000000009,"5min":41.140999999999998,"15min":41.140999999999998},"last":1.2210000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:26:48 2023","interfaces":[{"interface":"back","average":{"1min":4.3280000000000003,"5min":1.8859999999999999,"15min":0.997},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.33600000000000002},"max":{"1min":16.427,"5min":16.427,"15min":16.427},"last":0.77300000000000002},{"interface":"front","average":{"1min":3.2890000000000001,"5min":1.754,"15min":0.96599999999999997},"min":{"1min":0.36399999999999999,"5min":0.34300000000000003,"15min":0.29899999999999999},"max":{"1min":15.645,"5min":19.018999999999998,"15min":19.018999999999998},"last":0.58699999999999997}]}]},{"osd":5,"up_from":39,"seq":167503724748,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":437260,"kb_used_data":297988,"kb_used_omap":0,"kb_used_meta":139264,"kb_avail":93307892,"statfs":{"total":95995035648,"available":95547281408,"internally_reserved":0,"allocated":305139712,"data_stored":304432645,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":142606336},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:26:22 2023","interfaces":[{"interface":"back","average":{"1min":2.3380000000000001,"5min":1.548,"15min":0.91000000000000003},"min":{"1min":0.29499999999999998,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":11.346,"5min":32.475000000000001,"15min":32.475000000000001},"last":0.443},{"interface":"front","average":{"1min":1.9650000000000001,"5min":1.601,"15min":0.93400000000000005},"min":{"1min":0.245,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":5.6879999999999997,"5min":32.457000000000001,"15min":32.457000000000001},"last":0.623}]},{"osd":1,"last update":"Sat Sep 23 20:27:01 2023","interfaces":[{"interface":"back","average":{"1min":2.3980000000000001,"5min":1.819,"15min":0.96899999999999997},"min":{"1min":0.253,"5min":0.245,"15min":0.245},"max":{"1min":9.7639999999999993,"5min":32.243000000000002,"15min":32.243000000000002},"last":0.56599999999999995},{"interface":"front","average":{"1min":1.103,"5min":1.3480000000000001,"15min":0.81399999999999995},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":5.7389999999999999,"5min":32.231000000000002,"15min":32.231000000000002},"last":0.47299999999999998}]},{"osd":2,"last update":"Sat Sep 23 20:26:22 2023","interfaces":[{"interface":"back","average":{"1min":1.9970000000000001,"5min":1.891,"15min":0.997},"min":{"1min":0.30099999999999999,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":11.407,"5min":34.514000000000003,"15min":34.514000000000003},"last":0.68899999999999995},{"interface":"front","average":{"1min":2.105,"5min":1.732,"15min":0.94999999999999996},"min":{"1min":0.32100000000000001,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":11.574999999999999,"5min":32.515000000000001,"15min":32.515000000000001},"last":0.72099999999999997}]},{"osd":3,"last update":"Sat Sep 23 20:27:01 2023","interfaces":[{"interface":"back","average":{"1min":2.5129999999999999,"5min":1.5900000000000001,"15min":0.90200000000000002},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":12.846,"5min":12.846,"15min":12.846},"last":0.55000000000000004},{"interface":"front","average":{"1min":1.764,"5min":1.554,"15min":0.92800000000000005},"min":{"1min":0.35599999999999998,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":6.0049999999999999,"5min":11.379,"15min":11.379},"last":0.751}]},{"osd":4,"last update":"Sat Sep 23 20:27:18 2023","interfaces":[{"interface":"back","average":{"1min":2.8380000000000001,"5min":1.9019999999999999,"15min":0.94399999999999995},"min":{"1min":0.33600000000000002,"5min":0.26700000000000002,"15min":0.26400000000000001},"max":{"1min":12.984,"5min":14.218999999999999,"15min":14.218999999999999},"last":0.501},{"interface":"front","average":{"1min":1.5609999999999999,"5min":1.536,"15min":0.84499999999999997},"min":{"1min":0.38800000000000001,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":5.9779999999999998,"5min":11.664,"15min":11.664},"last":0.78900000000000003}]},{"osd":6,"last update":"Sat Sep 23 20:26:32 2023","interfaces":[{"interface":"back","average":{"1min":2.633,"5min":1.575,"15min":0.85599999999999998},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.29799999999999999},"max":{"1min":8.7949999999999999,"5min":11.545999999999999,"15min":11.545999999999999},"last":0.65100000000000002},{"interface":"front","average":{"1min":1.6870000000000001,"5min":1.5,"15min":0.879},"min":{"1min":0.376,"5min":0.35999999999999999,"15min":0.34499999999999997},"max":{"1min":8.4120000000000008,"5min":11.456,"15min":11.456},"last":0.81899999999999995}]},{"osd":7,"last update":"Sat Sep 23 20:27:01 2023","interfaces":[{"interface":"back","average":{"1min":2.2890000000000001,"5min":1.585,"15min":0.88900000000000001},"min":{"1min":0.40100000000000002,"5min":0.33900000000000002,"15min":0.33800000000000002},"max":{"1min":6.0730000000000004,"5min":11.430999999999999,"15min":11.430999999999999},"last":0.84499999999999997},{"interface":"front","average":{"1min":1.216,"5min":1.1930000000000001,"15min":0.76400000000000001},"min":{"1min":0.316,"5min":0.316,"15min":0.315},"max":{"1min":5.835,"5min":12.368,"15min":12.368},"last":0.76700000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4722,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1360,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6696,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7156,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3154,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":317521920,"data_stored":317521920,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":154206208,"data_stored":154206208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":421527552,"data_stored":421527552,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":632619008,"data_stored":632619008,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":197132288,"data_stored":197132288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":195100672,"data_stored":195100672,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":180158464,"data_stored":180158464,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":222625792,"data_stored":222625792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:27:31.799 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-09-23T20:27:31.799 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:27:31.799 INFO:tasks.thrashosds.thrasher:inject_pause on osd.4 2023-09-23T20:27:31.799 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T20:27:31.799 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T20:27:31.800 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 config set filestore_inject_stall 3 2023-09-23T20:27:31.831 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:27:31.920 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:32.213 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:27:32.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:32 smithi149 ceph-mon[170394]: 1.a scrub starts 2023-09-23T20:27:32.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:32 smithi149 ceph-mon[170394]: 1.a scrub ok 2023-09-23T20:27:32.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:32 smithi018 ceph-mon[184144]: 1.a scrub starts 2023-09-23T20:27:32.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:32 smithi018 ceph-mon[184144]: 1.a scrub ok 2023-09-23T20:27:32.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:32 smithi018 ceph-mon[179647]: 1.a scrub starts 2023-09-23T20:27:32.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:32 smithi018 ceph-mon[179647]: 1.a scrub ok 2023-09-23T20:27:32.511 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T20:27:32.516 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:27:32.520 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:32.521 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:27:32.552 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 10 16 24758 24742 154.605 128.75 0.00811515 0.00646201 2023-09-23T20:27:32.621 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T20:27:32.622 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:27:32.688 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:32.749 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:27:32.756 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:32.757 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:27:32.858 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:27:32.894 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:32.960 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:27:32.974 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:27:32.981 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:32.982 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:27:33.082 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:27:33.124 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:33.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:33 smithi018 ceph-mon[184144]: from='client.35295 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:27:33.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:33 smithi018 ceph-mon[184144]: pgmap v481: 99 pgs: 99 active+clean; 1.2 GiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 127 MiB/s wr, 2.04k op/s 2023-09-23T20:27:33.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:33 smithi018 ceph-mon[184144]: 1.8 scrub starts 2023-09-23T20:27:33.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:33 smithi018 ceph-mon[184144]: 1.8 scrub ok 2023-09-23T20:27:33.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:33 smithi018 ceph-mon[184144]: 1.6 scrub starts 2023-09-23T20:27:33.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:33 smithi018 ceph-mon[184144]: 1.6 scrub ok 2023-09-23T20:27:33.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:33 smithi018 ceph-mon[179647]: from='client.35295 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:27:33.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:33 smithi018 ceph-mon[179647]: pgmap v481: 99 pgs: 99 active+clean; 1.2 GiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 127 MiB/s wr, 2.04k op/s 2023-09-23T20:27:33.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:33 smithi018 ceph-mon[179647]: 1.8 scrub starts 2023-09-23T20:27:33.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:33 smithi018 ceph-mon[179647]: 1.8 scrub ok 2023-09-23T20:27:33.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:33 smithi018 ceph-mon[179647]: 1.6 scrub starts 2023-09-23T20:27:33.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:33 smithi018 ceph-mon[179647]: 1.6 scrub ok 2023-09-23T20:27:33.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:33 smithi149 ceph-mon[170394]: from='client.35295 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:27:33.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:33 smithi149 ceph-mon[170394]: pgmap v481: 99 pgs: 99 active+clean; 1.2 GiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 127 MiB/s wr, 2.04k op/s 2023-09-23T20:27:33.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:33 smithi149 ceph-mon[170394]: 1.8 scrub starts 2023-09-23T20:27:33.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:33 smithi149 ceph-mon[170394]: 1.8 scrub ok 2023-09-23T20:27:33.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:33 smithi149 ceph-mon[170394]: 1.6 scrub starts 2023-09-23T20:27:33.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:33 smithi149 ceph-mon[170394]: 1.6 scrub ok 2023-09-23T20:27:33.328 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:27:33.553 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 11 16 26563 26547 150.798 112.812 0.00835049 0.00662699 2023-09-23T20:27:33.745 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:27:33.788 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:27:33.833 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:27:33.837 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:33.837 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:27:33.841 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:27:33.862 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:27:33.901 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:27:33.938 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:27:33.941 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:27:33.982 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:27:33.992 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:34.034 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:27:34.048 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:27:34.048 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:27:34.048 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:27:34.048 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:27:34.048 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:27:34.049 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:27:34.049 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:27:34.049 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:27:34.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:34 smithi149 ceph-mon[170394]: 1.7 scrub starts 2023-09-23T20:27:34.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:34 smithi149 ceph-mon[170394]: 1.7 scrub ok 2023-09-23T20:27:34.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:34 smithi149 ceph-mon[170394]: 1.0 scrub starts 2023-09-23T20:27:34.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:34 smithi149 ceph-mon[170394]: 1.0 scrub ok 2023-09-23T20:27:34.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:34 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2544120636' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:27:34.288 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:34 smithi018 ceph-mon[184144]: 1.7 scrub starts 2023-09-23T20:27:34.288 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:34 smithi018 ceph-mon[184144]: 1.7 scrub ok 2023-09-23T20:27:34.288 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:34 smithi018 ceph-mon[184144]: 1.0 scrub starts 2023-09-23T20:27:34.289 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:34 smithi018 ceph-mon[184144]: 1.0 scrub ok 2023-09-23T20:27:34.289 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:34 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2544120636' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:27:34.289 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:34 smithi018 ceph-mon[179647]: 1.7 scrub starts 2023-09-23T20:27:34.290 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:34 smithi018 ceph-mon[179647]: 1.7 scrub ok 2023-09-23T20:27:34.290 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:34 smithi018 ceph-mon[179647]: 1.0 scrub starts 2023-09-23T20:27:34.290 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:34 smithi018 ceph-mon[179647]: 1.0 scrub ok 2023-09-23T20:27:34.290 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:34 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2544120636' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:27:34.553 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 12 16 28364 28348 147.611 112.562 0.00541193 0.00676873 2023-09-23T20:27:34.813 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:27:34.826 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:34.826 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:27:34.927 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:27:34.960 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:35.141 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:27:34 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:27:34] "GET /metrics HTTP/1.1" 200 36514 "" "Prometheus/2.43.0" 2023-09-23T20:27:35.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:35 smithi018 ceph-mon[184144]: pgmap v482: 99 pgs: 99 active+clean; 1.3 GiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 132 MiB/s wr, 2.12k op/s 2023-09-23T20:27:35.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:35 smithi018 ceph-mon[179647]: pgmap v482: 99 pgs: 99 active+clean; 1.3 GiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 132 MiB/s wr, 2.12k op/s 2023-09-23T20:27:35.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:35 smithi149 ceph-mon[170394]: pgmap v482: 99 pgs: 99 active+clean; 1.3 GiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 132 MiB/s wr, 2.12k op/s 2023-09-23T20:27:35.445 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:27:35.463 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:35.464 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:27:35.500 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:27:35.553 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 13 16 30002 29986 144.131 102.375 0.00698908 0.00693057 2023-09-23T20:27:35.564 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:27:35.622 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:35.686 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:27:35.690 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:35.691 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:27:35.791 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:27:35.846 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:35.957 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:27:35.965 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:35.966 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:27:36.066 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:27:36.128 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:36 smithi149 ceph-mon[170394]: 1.1 scrub starts 2023-09-23T20:27:36.128 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:36 smithi149 ceph-mon[170394]: 1.1 scrub ok 2023-09-23T20:27:36.134 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:36.214 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:27:36.220 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:36.221 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:27:36.296 INFO:teuthology.orchestra.run.smithi149.stdout:{ 2023-09-23T20:27:36.296 INFO:teuthology.orchestra.run.smithi149.stdout: "success": "" 2023-09-23T20:27:36.296 INFO:teuthology.orchestra.run.smithi149.stdout:} 2023-09-23T20:27:36.322 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:27:36.369 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:36.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:36 smithi018 ceph-mon[184144]: 1.1 scrub starts 2023-09-23T20:27:36.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:36 smithi018 ceph-mon[184144]: 1.1 scrub ok 2023-09-23T20:27:36.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:36 smithi018 ceph-mon[179647]: 1.1 scrub starts 2023-09-23T20:27:36.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:36 smithi018 ceph-mon[179647]: 1.1 scrub ok 2023-09-23T20:27:36.555 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 14 16 31861 31845 142.111 116.188 0.00518503 0.00703103 2023-09-23T20:27:36.780 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:27:36.788 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:36.789 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:27:36.889 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:27:36.972 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:37.219 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:37 smithi149 ceph-mon[170394]: pgmap v483: 99 pgs: 99 active+clean; 1.6 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 147 MiB/s wr, 2.34k op/s 2023-09-23T20:27:37.219 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:37 smithi149 ceph-mon[170394]: 1.2 scrub starts 2023-09-23T20:27:37.219 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:37 smithi149 ceph-mon[170394]: 1.2 scrub ok 2023-09-23T20:27:37.360 INFO:teuthology.orchestra.run.smithi149.stdout:{ 2023-09-23T20:27:37.360 INFO:teuthology.orchestra.run.smithi149.stdout: "ops": [], 2023-09-23T20:27:37.360 INFO:teuthology.orchestra.run.smithi149.stdout: "num_ops": 0 2023-09-23T20:27:37.360 INFO:teuthology.orchestra.run.smithi149.stdout:} 2023-09-23T20:27:37.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:37 smithi018 ceph-mon[184144]: pgmap v483: 99 pgs: 99 active+clean; 1.6 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 147 MiB/s wr, 2.34k op/s 2023-09-23T20:27:37.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:37 smithi018 ceph-mon[184144]: 1.2 scrub starts 2023-09-23T20:27:37.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:37 smithi018 ceph-mon[184144]: 1.2 scrub ok 2023-09-23T20:27:37.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:37 smithi018 ceph-mon[179647]: pgmap v483: 99 pgs: 99 active+clean; 1.6 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 147 MiB/s wr, 2.34k op/s 2023-09-23T20:27:37.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:37 smithi018 ceph-mon[179647]: 1.2 scrub starts 2023-09-23T20:27:37.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:37 smithi018 ceph-mon[179647]: 1.2 scrub ok 2023-09-23T20:27:37.556 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 15 16 33807 33791 140.745 121.625 0.00884235 0.00709963 2023-09-23T20:27:37.658 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:27:37.662 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:37.662 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:27:37.763 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:27:37.812 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:37.881 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T20:27:37.881 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:27:37.885 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:37.886 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:27:37.987 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:27:38.061 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:38.119 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:27:38.125 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:38.125 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:27:38.226 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:27:38.281 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:38.353 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:27:38.357 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:38.358 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:27:38.458 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:27:38.529 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:38.556 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 16 16 35503 35487 138.574 106 0.014802 0.00720936 2023-09-23T20:27:38.600 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:27:38.602 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:27:38.608 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:38.608 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:27:38.709 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:27:38.767 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:38.839 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:27:38.847 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:38.848 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:27:38.949 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:27:38.998 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:39.070 INFO:teuthology.orchestra.run.smithi018.stderr:nodeep-scrub is unset 2023-09-23T20:27:39.100 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:39 smithi018 ceph-mon[184144]: pgmap v484: 99 pgs: 99 active+clean; 1.8 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 134 MiB/s wr, 2.14k op/s 2023-09-23T20:27:39.100 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:39 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3345544034' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:27:39.100 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:39 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:27:39.100 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:39 smithi018 ceph-mon[179647]: pgmap v484: 99 pgs: 99 active+clean; 1.8 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 134 MiB/s wr, 2.14k op/s 2023-09-23T20:27:39.100 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:39 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3345544034' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:27:39.101 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:39 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:27:39.116 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:27:39.125 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:39.125 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:27:39.181 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:39 smithi149 ceph-mon[170394]: pgmap v484: 99 pgs: 99 active+clean; 1.8 GiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 134 MiB/s wr, 2.14k op/s 2023-09-23T20:27:39.181 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:39 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3345544034' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:27:39.181 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:39 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:27:39.227 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:27:39.285 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:39.445 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:27:39.450 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:39.451 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:27:39.552 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:27:39.556 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 17 16 37184 37168 136.602 105.062 0.0118788 0.0073153 2023-09-23T20:27:39.610 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:39.677 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:27:39.680 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:39.680 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:27:39.720 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:27:39.781 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:27:39.841 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:39.932 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:27:39.938 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:39.938 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:27:39.977 INFO:teuthology.orchestra.run.smithi149.stdout:{ 2023-09-23T20:27:39.977 INFO:teuthology.orchestra.run.smithi149.stdout: "ops": [], 2023-09-23T20:27:39.977 INFO:teuthology.orchestra.run.smithi149.stdout: "complaint_time": 30, 2023-09-23T20:27:39.977 INFO:teuthology.orchestra.run.smithi149.stdout: "num_blocked_ops": 0 2023-09-23T20:27:39.977 INFO:teuthology.orchestra.run.smithi149.stdout:} 2023-09-23T20:27:40.040 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:27:40.065 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:40 smithi018 ceph-mon[179647]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:27:40.065 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:40 smithi018 ceph-mon[179647]: Cluster is now healthy 2023-09-23T20:27:40.065 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:40 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:27:40.066 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:40 smithi018 ceph-mon[179647]: osdmap e223: 8 total, 8 up, 8 in 2023-09-23T20:27:40.066 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:40 smithi018 ceph-mon[179647]: failed to encode map e223 with expected crc 2023-09-23T20:27:40.068 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:40 smithi018 ceph-mon[184144]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:27:40.069 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:40 smithi018 ceph-mon[184144]: Cluster is now healthy 2023-09-23T20:27:40.069 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:40 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:27:40.069 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:40 smithi018 ceph-mon[184144]: osdmap e223: 8 total, 8 up, 8 in 2023-09-23T20:27:40.069 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:40 smithi018 ceph-mon[184144]: failed to encode map e223 with expected crc 2023-09-23T20:27:40.069 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:40 smithi018 ceph-mon[184144]: failed to encode map e223 with expected crc 2023-09-23T20:27:40.069 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:40 smithi018 ceph-mon[184144]: failed to encode map e223 with expected crc 2023-09-23T20:27:40.069 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:40 smithi018 ceph-mon[184144]: failed to encode map e223 with expected crc 2023-09-23T20:27:40.103 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:40.182 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:27:40.188 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:40.189 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:27:40.289 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:27:40.310 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:40 smithi149 ceph-mon[170394]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:27:40.310 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:40 smithi149 ceph-mon[170394]: Cluster is now healthy 2023-09-23T20:27:40.310 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:40 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:27:40.310 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:40 smithi149 ceph-mon[170394]: osdmap e223: 8 total, 8 up, 8 in 2023-09-23T20:27:40.310 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:40 smithi149 ceph-mon[170394]: failed to encode map e223 with expected crc 2023-09-23T20:27:40.310 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:40 smithi149 ceph-mon[170394]: failed to encode map e223 with expected crc 2023-09-23T20:27:40.310 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:40 smithi149 ceph-mon[170394]: failed to encode map e223 with expected crc 2023-09-23T20:27:40.310 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:40 smithi149 ceph-mon[170394]: failed to encode map e223 with expected crc 2023-09-23T20:27:40.340 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:40 smithi018 ceph-mon[179647]: failed to encode map e223 with expected crc 2023-09-23T20:27:40.340 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:40 smithi018 ceph-mon[179647]: failed to encode map e223 with expected crc 2023-09-23T20:27:40.340 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:40 smithi018 ceph-mon[179647]: failed to encode map e223 with expected crc 2023-09-23T20:27:40.346 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T20:27:40.351 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:40.452 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:27:40.456 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:40.457 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:27:40.497 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:27:40.557 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:27:40.558 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 18 16 38759 38743 134.473 98.4375 0.00898105 0.00743036 2023-09-23T20:27:40.613 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:40.739 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:27:40.747 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:40.747 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:27:40.848 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:27:40.918 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:41.038 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:27:41.045 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:41.045 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:27:41.097 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:27:41.146 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:27:41.198 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:41 smithi018 ceph-mon[184144]: pgmap v486: 99 pgs: 99 active+clean; 1.9 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 125 MiB/s wr, 1.99k op/s 2023-09-23T20:27:41.199 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:41 smithi018 ceph-mon[179647]: pgmap v486: 99 pgs: 99 active+clean; 1.9 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 125 MiB/s wr, 1.99k op/s 2023-09-23T20:27:41.203 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:41.317 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:27:41.323 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:41.323 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:27:41.360 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:41 smithi149 ceph-mon[170394]: pgmap v486: 99 pgs: 99 active+clean; 1.9 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 125 MiB/s wr, 1.99k op/s 2023-09-23T20:27:41.424 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:27:41.471 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:41.527 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T20:27:41.557 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 19 16 40355 40339 132.646 99.75 0.00697493 0.00753349 2023-09-23T20:27:41.581 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:27:41.590 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:41.590 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:27:41.691 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:27:41.792 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:41.966 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:27:41.972 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:41.972 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:27:42.004 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 4, 5, 6, 7, 3, 2] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-23T20:27:42.004 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:27:42.004 INFO:tasks.thrashosds.thrasher:Growing pool .mgr 2023-09-23T20:27:42.004 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:27:42.073 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:27:42.114 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:27:42.163 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:27:42.164 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:42.211 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:27:42.257 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:27:42.260 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:27:42.264 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:42.264 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:42 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/443302680' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:27:42.265 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:27:42.273 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:42 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/443302680' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:27:42.298 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:27:42.322 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:27:42.334 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:27:42.366 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:27:42.373 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:27:42.401 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:42 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/443302680' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:27:42.408 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:42.418 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:27:42.438 INFO:teuthology.orchestra.run.smithi149.stdout:ERROR: (22) Invalid argument 2023-09-23T20:27:42.438 INFO:teuthology.orchestra.run.smithi149.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:27:42.442 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:27:42.442 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:27:42.442 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:27:42.442 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:27:42.442 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:27:42.442 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:27:42.443 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:27:42.443 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:27:42.468 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:27:42.474 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:42.474 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:27:42.557 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:2023-09-23T20:27:42.558094+0000 min lat: 0.00113549 max lat: 0.118523 avg lat: 0.00768153 2023-09-23T20:27:42.558 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T20:27:42.558 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 20 16 41658 41642 130.086 81.4375 0.015414 0.00768153 2023-09-23T20:27:42.575 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:27:42.622 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:42.833 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:27:42.837 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:42.837 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:27:42.905 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T20:27:42.938 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:27:42.996 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:43.083 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:27:43.086 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:43.087 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:27:43.155 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:43 smithi018 ceph-mon[179647]: pgmap v487: 99 pgs: 99 active+clean; 2.3 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 115 MiB/s wr, 1.83k op/s 2023-09-23T20:27:43.157 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:43 smithi018 ceph-mon[184144]: pgmap v487: 99 pgs: 99 active+clean; 2.3 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 115 MiB/s wr, 1.83k op/s 2023-09-23T20:27:43.188 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:27:43.261 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:43.333 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:27:43.341 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:43.342 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:27:43.443 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:27:43.470 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:43 smithi149 ceph-mon[170394]: pgmap v487: 99 pgs: 99 active+clean; 2.3 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 115 MiB/s wr, 1.83k op/s 2023-09-23T20:27:43.487 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:43.541 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:27:43.550 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:43.550 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:27:43.558 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 21 16 43167 43151 128.377 94.3125 0.00532244 0.00778473 2023-09-23T20:27:43.651 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:27:43.690 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:27:43.719 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:43.778 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:27:43.821 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:27:43.827 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:43.828 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:27:43.928 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:27:43.987 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:44.053 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:27:44.064 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:44.064 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:27:44.165 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:27:44.195 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:44.391 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:27:44.395 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:44.395 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:27:44.496 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:27:44.536 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:44.560 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 22 16 44509 44493 126.345 83.875 0.00731492 0.00790931 2023-09-23T20:27:44.901 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:27:45.107 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:27:44 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:27:44] "GET /metrics HTTP/1.1" 200 36513 "" "Prometheus/2.43.0" 2023-09-23T20:27:45.278 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:45 smithi149 ceph-mon[170394]: pgmap v488: 99 pgs: 99 active+clean; 2.4 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 112 MiB/s wr, 1.78k op/s 2023-09-23T20:27:45.278 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:45 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:27:45.278 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:45 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:27:45.278 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:45 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2958558223' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:27:45.278 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:45 smithi018 ceph-mon[179647]: pgmap v488: 99 pgs: 99 active+clean; 2.4 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 112 MiB/s wr, 1.78k op/s 2023-09-23T20:27:45.278 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:45 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:27:45.278 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:45 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:27:45.278 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:45 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2958558223' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:27:45.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:45 smithi018 ceph-mon[184144]: pgmap v488: 99 pgs: 99 active+clean; 2.4 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 112 MiB/s wr, 1.78k op/s 2023-09-23T20:27:45.397 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:45 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:27:45.397 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:45 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:27:45.397 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:45 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2958558223' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:27:45.467 INFO:teuthology.orchestra.run.smithi018.stderr:noscrub is set 2023-09-23T20:27:45.562 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 23 16 45834 45818 124.447 82.8125 0.0105378 0.00802722 2023-09-23T20:27:45.886 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:27:45.903 INFO:teuthology.orchestra.run.smithi018.stderr:dumped all 2023-09-23T20:27:46.085 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:27:46.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:46 smithi018 ceph-mon[184144]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:27:46.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:46 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2958558223' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:27:46.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:46 smithi018 ceph-mon[184144]: osdmap e224: 8 total, 8 up, 8 in 2023-09-23T20:27:46.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:46 smithi018 ceph-mon[184144]: failed to encode map e224 with expected crc 2023-09-23T20:27:46.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:46 smithi018 ceph-mon[184144]: failed to encode map e224 with expected crc 2023-09-23T20:27:46.147 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:46 smithi018 ceph-mon[179647]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:27:46.148 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:46 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2958558223' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:27:46.148 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:46 smithi018 ceph-mon[179647]: osdmap e224: 8 total, 8 up, 8 in 2023-09-23T20:27:46.148 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:46 smithi018 ceph-mon[179647]: failed to encode map e224 with expected crc 2023-09-23T20:27:46.148 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:46 smithi018 ceph-mon[179647]: failed to encode map e224 with expected crc 2023-09-23T20:27:46.305 INFO:teuthology.orchestra.run.smithi018.stdout:{"pg_ready":true,"pg_map":{"version":490,"stamp":"2023-09-23T20:27:45.561891+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2788092249,"num_objects":42497,"num_object_clones":0,"num_object_copies":127491,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":42497,"num_whiteouts":0,"num_read":6040,"num_read_kb":13296,"num_write":43397,"num_write_kb":2730960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":15,"num_bytes_recovered":162365,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":45271,"ondisk_log_size":45271,"up":297,"acting":297,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":299,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":9402272,"kb_used_data":8179040,"kb_used_omap":30,"kb_used_meta":1223137,"kb_avail":740558944,"statfs":{"total":767960285184,"available":758332358656,"internally_reserved":0,"allocated":8375336960,"data_stored":8368176156,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31263,"internal_metadata":1252492769},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,6,10,3],"upper_bound":16},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":868953690,"num_objects":13259,"num_object_clones":0,"num_object_copies":39777,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":13259,"num_whiteouts":0,"num_read":-46,"num_read_kb":-84,"num_write":13238,"num_write_kb":848420,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-1,"num_bytes_recovered":-161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.003271"},"pg_stats":[{"pgid":"3.1a","version":"69'1","reported_seq":220,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-09-23T20:27:29.985891+0000","last_change":"2023-09-23T20:25:39.632437+0000","last_active":"2023-09-23T20:27:29.985891+0000","last_peered":"2023-09-23T20:27:29.985891+0000","last_clean":"2023-09-23T20:27:29.985891+0000","last_became_active":"2023-09-23T20:25:39.632074+0000","last_became_peered":"2023-09-23T20:25:39.632074+0000","last_unstale":"2023-09-23T20:27:29.985891+0000","last_undegraded":"2023-09-23T20:27:29.985891+0000","last_fullsized":"2023-09-23T20:27:29.985891+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:32:13.543799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":69,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-09-23T20:27:33.049892+0000","last_change":"2023-09-23T20:25:02.932223+0000","last_active":"2023-09-23T20:27:33.049892+0000","last_peered":"2023-09-23T20:27:33.049892+0000","last_clean":"2023-09-23T20:27:33.049892+0000","last_became_active":"2023-09-23T20:25:02.931024+0000","last_became_peered":"2023-09-23T20:25:02.931024+0000","last_unstale":"2023-09-23T20:27:33.049892+0000","last_undegraded":"2023-09-23T20:27:33.049892+0000","last_fullsized":"2023-09-23T20:27:33.049892+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:16:00.586211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":180,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-09-23T20:27:29.984264+0000","last_change":"2023-09-23T20:25:02.934543+0000","last_active":"2023-09-23T20:27:29.984264+0000","last_peered":"2023-09-23T20:27:29.984264+0000","last_clean":"2023-09-23T20:27:29.984264+0000","last_became_active":"2023-09-23T20:25:02.934178+0000","last_became_peered":"2023-09-23T20:25:02.934178+0000","last_unstale":"2023-09-23T20:27:29.984264+0000","last_undegraded":"2023-09-23T20:27:29.984264+0000","last_fullsized":"2023-09-23T20:27:29.984264+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:27:38.453052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":186,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-09-23T20:27:29.978946+0000","last_change":"2023-09-23T20:25:02.957185+0000","last_active":"2023-09-23T20:27:29.978946+0000","last_peered":"2023-09-23T20:27:29.978946+0000","last_clean":"2023-09-23T20:27:29.978946+0000","last_became_active":"2023-09-23T20:25:02.956689+0000","last_became_peered":"2023-09-23T20:25:02.956689+0000","last_unstale":"2023-09-23T20:27:29.978946+0000","last_undegraded":"2023-09-23T20:27:29.978946+0000","last_fullsized":"2023-09-23T20:27:29.978946+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:04:28.524015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":69,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-09-23T20:27:29.987736+0000","last_change":"2023-09-23T20:25:02.932054+0000","last_active":"2023-09-23T20:27:29.987736+0000","last_peered":"2023-09-23T20:27:29.987736+0000","last_clean":"2023-09-23T20:27:29.987736+0000","last_became_active":"2023-09-23T20:25:02.931309+0000","last_became_peered":"2023-09-23T20:25:02.931309+0000","last_unstale":"2023-09-23T20:27:29.987736+0000","last_undegraded":"2023-09-23T20:27:29.987736+0000","last_fullsized":"2023-09-23T20:27:29.987736+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:57:53.378499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":183,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-09-23T20:27:39.060994+0000","last_change":"2023-09-23T20:20:46.937089+0000","last_active":"2023-09-23T20:27:39.060994+0000","last_peered":"2023-09-23T20:27:39.060994+0000","last_clean":"2023-09-23T20:27:39.060994+0000","last_became_active":"2023-09-23T20:20:46.936831+0000","last_became_peered":"2023-09-23T20:20:46.936831+0000","last_unstale":"2023-09-23T20:27:39.060994+0000","last_undegraded":"2023-09-23T20:27:39.060994+0000","last_fullsized":"2023-09-23T20:27:39.060994+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:34:56.747010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":185,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-09-23T20:27:29.978653+0000","last_change":"2023-09-23T20:25:02.969432+0000","last_active":"2023-09-23T20:27:29.978653+0000","last_peered":"2023-09-23T20:27:29.978653+0000","last_clean":"2023-09-23T20:27:29.978653+0000","last_became_active":"2023-09-23T20:25:02.969118+0000","last_became_peered":"2023-09-23T20:25:02.969118+0000","last_unstale":"2023-09-23T20:27:29.978653+0000","last_undegraded":"2023-09-23T20:27:29.978653+0000","last_fullsized":"2023-09-23T20:27:29.978653+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:02:22.453831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":69,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-09-23T20:27:29.987757+0000","last_change":"2023-09-23T20:25:02.932225+0000","last_active":"2023-09-23T20:27:29.987757+0000","last_peered":"2023-09-23T20:27:29.987757+0000","last_clean":"2023-09-23T20:27:29.987757+0000","last_became_active":"2023-09-23T20:25:02.931746+0000","last_became_peered":"2023-09-23T20:25:02.931746+0000","last_unstale":"2023-09-23T20:27:29.987757+0000","last_undegraded":"2023-09-23T20:27:29.987757+0000","last_fullsized":"2023-09-23T20:27:29.987757+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:57:24.470179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":182,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-09-23T20:27:29.988425+0000","last_change":"2023-09-23T20:20:46.950281+0000","last_active":"2023-09-23T20:27:29.988425+0000","last_peered":"2023-09-23T20:27:29.988425+0000","last_clean":"2023-09-23T20:27:29.988425+0000","last_became_active":"2023-09-23T20:20:46.949892+0000","last_became_peered":"2023-09-23T20:20:46.949892+0000","last_unstale":"2023-09-23T20:27:29.988425+0000","last_undegraded":"2023-09-23T20:27:29.988425+0000","last_fullsized":"2023-09-23T20:27:29.988425+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:28.763832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":69,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-09-23T20:27:29.987200+0000","last_change":"2023-09-23T20:25:02.947669+0000","last_active":"2023-09-23T20:27:29.987200+0000","last_peered":"2023-09-23T20:27:29.987200+0000","last_clean":"2023-09-23T20:27:29.987200+0000","last_became_active":"2023-09-23T20:25:02.947368+0000","last_became_peered":"2023-09-23T20:25:02.947368+0000","last_unstale":"2023-09-23T20:27:29.987200+0000","last_undegraded":"2023-09-23T20:27:29.987200+0000","last_fullsized":"2023-09-23T20:27:29.987200+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:52:53.813761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":219,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-09-23T20:27:39.056534+0000","last_change":"2023-09-23T20:25:03.038511+0000","last_active":"2023-09-23T20:27:39.056534+0000","last_peered":"2023-09-23T20:27:39.056534+0000","last_clean":"2023-09-23T20:27:39.056534+0000","last_became_active":"2023-09-23T20:25:02.976811+0000","last_became_peered":"2023-09-23T20:25:02.976811+0000","last_unstale":"2023-09-23T20:27:39.056534+0000","last_undegraded":"2023-09-23T20:27:39.056534+0000","last_fullsized":"2023-09-23T20:27:39.056534+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:28:06.605731+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":182,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-09-23T20:27:39.050249+0000","last_change":"2023-09-23T20:25:39.646452+0000","last_active":"2023-09-23T20:27:39.050249+0000","last_peered":"2023-09-23T20:27:39.050249+0000","last_clean":"2023-09-23T20:27:39.050249+0000","last_became_active":"2023-09-23T20:25:39.646175+0000","last_became_peered":"2023-09-23T20:25:39.646175+0000","last_unstale":"2023-09-23T20:27:39.050249+0000","last_undegraded":"2023-09-23T20:27:39.050249+0000","last_fullsized":"2023-09-23T20:27:39.050249+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:33:02.099951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":187,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-09-23T20:27:29.986654+0000","last_change":"2023-09-23T20:12:43.750217+0000","last_active":"2023-09-23T20:27:29.986654+0000","last_peered":"2023-09-23T20:27:29.986654+0000","last_clean":"2023-09-23T20:27:29.986654+0000","last_became_active":"2023-09-23T20:12:43.749929+0000","last_became_peered":"2023-09-23T20:12:43.749929+0000","last_unstale":"2023-09-23T20:27:29.986654+0000","last_undegraded":"2023-09-23T20:27:29.986654+0000","last_fullsized":"2023-09-23T20:27:29.986654+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:50:28.470211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":181,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-09-23T20:27:29.986213+0000","last_change":"2023-09-23T20:20:46.939938+0000","last_active":"2023-09-23T20:27:29.986213+0000","last_peered":"2023-09-23T20:27:29.986213+0000","last_clean":"2023-09-23T20:27:29.986213+0000","last_became_active":"2023-09-23T20:20:46.938777+0000","last_became_peered":"2023-09-23T20:20:46.938777+0000","last_unstale":"2023-09-23T20:27:29.986213+0000","last_undegraded":"2023-09-23T20:27:29.986213+0000","last_fullsized":"2023-09-23T20:27:29.986213+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:45:10.329438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":218,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-09-23T20:27:39.056377+0000","last_change":"2023-09-23T20:25:39.654026+0000","last_active":"2023-09-23T20:27:39.056377+0000","last_peered":"2023-09-23T20:27:39.056377+0000","last_clean":"2023-09-23T20:27:39.056377+0000","last_became_active":"2023-09-23T20:25:39.626849+0000","last_became_peered":"2023-09-23T20:25:39.626849+0000","last_unstale":"2023-09-23T20:27:39.056377+0000","last_undegraded":"2023-09-23T20:27:39.056377+0000","last_fullsized":"2023-09-23T20:27:39.056377+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:22:53.957630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":69,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-09-23T20:27:29.988852+0000","last_change":"2023-09-23T20:25:02.962047+0000","last_active":"2023-09-23T20:27:29.988852+0000","last_peered":"2023-09-23T20:27:29.988852+0000","last_clean":"2023-09-23T20:27:29.988852+0000","last_became_active":"2023-09-23T20:25:02.961523+0000","last_became_peered":"2023-09-23T20:25:02.961523+0000","last_unstale":"2023-09-23T20:27:29.988852+0000","last_undegraded":"2023-09-23T20:27:29.988852+0000","last_fullsized":"2023-09-23T20:27:29.988852+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:18:11.830822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":185,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-09-23T20:27:29.978726+0000","last_change":"2023-09-23T20:25:02.953993+0000","last_active":"2023-09-23T20:27:29.978726+0000","last_peered":"2023-09-23T20:27:29.978726+0000","last_clean":"2023-09-23T20:27:29.978726+0000","last_became_active":"2023-09-23T20:25:02.953672+0000","last_became_peered":"2023-09-23T20:25:02.953672+0000","last_unstale":"2023-09-23T20:27:29.978726+0000","last_undegraded":"2023-09-23T20:27:29.978726+0000","last_fullsized":"2023-09-23T20:27:29.978726+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:26:08.603577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":201,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-09-23T20:27:29.987459+0000","last_change":"2023-09-23T20:25:02.949584+0000","last_active":"2023-09-23T20:27:29.987459+0000","last_peered":"2023-09-23T20:27:29.987459+0000","last_clean":"2023-09-23T20:27:29.987459+0000","last_became_active":"2023-09-23T20:25:02.949132+0000","last_became_peered":"2023-09-23T20:25:02.949132+0000","last_unstale":"2023-09-23T20:27:29.987459+0000","last_undegraded":"2023-09-23T20:27:29.987459+0000","last_fullsized":"2023-09-23T20:27:29.987459+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:34:20.509823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":167,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-09-23T20:27:39.055680+0000","last_change":"2023-09-23T20:12:43.749139+0000","last_active":"2023-09-23T20:27:39.055680+0000","last_peered":"2023-09-23T20:27:39.055680+0000","last_clean":"2023-09-23T20:27:39.055680+0000","last_became_active":"2023-09-23T20:12:43.748854+0000","last_became_peered":"2023-09-23T20:12:43.748854+0000","last_unstale":"2023-09-23T20:27:39.055680+0000","last_undegraded":"2023-09-23T20:27:39.055680+0000","last_fullsized":"2023-09-23T20:27:39.055680+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:57:11.630712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":195,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-09-23T20:27:29.978847+0000","last_change":"2023-09-23T20:25:02.957010+0000","last_active":"2023-09-23T20:27:29.978847+0000","last_peered":"2023-09-23T20:27:29.978847+0000","last_clean":"2023-09-23T20:27:29.978847+0000","last_became_active":"2023-09-23T20:25:02.956267+0000","last_became_peered":"2023-09-23T20:25:02.956267+0000","last_unstale":"2023-09-23T20:27:29.978847+0000","last_undegraded":"2023-09-23T20:27:29.978847+0000","last_fullsized":"2023-09-23T20:27:29.978847+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:10:55.299422+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":192,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-09-23T20:27:39.059048+0000","last_change":"2023-09-23T20:20:14.881009+0000","last_active":"2023-09-23T20:27:39.059048+0000","last_peered":"2023-09-23T20:27:39.059048+0000","last_clean":"2023-09-23T20:27:39.059048+0000","last_became_active":"2023-09-23T20:20:14.880685+0000","last_became_peered":"2023-09-23T20:20:14.880685+0000","last_unstale":"2023-09-23T20:27:39.059048+0000","last_undegraded":"2023-09-23T20:27:39.059048+0000","last_fullsized":"2023-09-23T20:27:39.059048+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:28.010434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":69,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-09-23T20:27:29.988013+0000","last_change":"2023-09-23T20:25:02.977462+0000","last_active":"2023-09-23T20:27:29.988013+0000","last_peered":"2023-09-23T20:27:29.988013+0000","last_clean":"2023-09-23T20:27:29.988013+0000","last_became_active":"2023-09-23T20:25:02.969910+0000","last_became_peered":"2023-09-23T20:25:02.969910+0000","last_unstale":"2023-09-23T20:27:29.988013+0000","last_undegraded":"2023-09-23T20:27:29.988013+0000","last_fullsized":"2023-09-23T20:27:29.988013+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:22:55.644955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.a","version":"223'2419","reported_seq":2433,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-09-23T20:27:40.618666+0000","last_change":"2023-09-23T20:27:14.711745+0000","last_active":"2023-09-23T20:27:40.618666+0000","last_peered":"2023-09-23T20:27:40.618666+0000","last_clean":"2023-09-23T20:27:40.618666+0000","last_became_active":"2023-09-23T20:27:14.711412+0000","last_became_peered":"2023-09-23T20:27:14.711412+0000","last_unstale":"2023-09-23T20:27:40.618666+0000","last_undegraded":"2023-09-23T20:27:40.618666+0000","last_fullsized":"2023-09-23T20:27:40.618666+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":2419,"log_dups_size":0,"ondisk_log_size":2419,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:48:13.653105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":158531584,"num_objects":2419,"num_object_clones":0,"num_object_copies":7257,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2419,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2419,"num_write_kb":154816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":185,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-09-23T20:27:39.056117+0000","last_change":"2023-09-23T20:20:14.883283+0000","last_active":"2023-09-23T20:27:39.056117+0000","last_peered":"2023-09-23T20:27:39.056117+0000","last_clean":"2023-09-23T20:27:39.056117+0000","last_became_active":"2023-09-23T20:20:14.882806+0000","last_became_peered":"2023-09-23T20:20:14.882806+0000","last_unstale":"2023-09-23T20:27:39.056117+0000","last_undegraded":"2023-09-23T20:27:39.056117+0000","last_fullsized":"2023-09-23T20:27:39.056117+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:23:35.295701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":164,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-09-23T20:27:29.988572+0000","last_change":"2023-09-23T20:12:45.179211+0000","last_active":"2023-09-23T20:27:29.988572+0000","last_peered":"2023-09-23T20:27:29.988572+0000","last_clean":"2023-09-23T20:27:29.988572+0000","last_became_active":"2023-09-23T20:12:45.178966+0000","last_became_peered":"2023-09-23T20:12:45.178966+0000","last_unstale":"2023-09-23T20:27:29.988572+0000","last_undegraded":"2023-09-23T20:27:29.988572+0000","last_fullsized":"2023-09-23T20:27:29.988572+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:58.902140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"223'2584","reported_seq":2599,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-09-23T20:27:43.040851+0000","last_change":"2023-09-23T20:27:14.269442+0000","last_active":"2023-09-23T20:27:43.040851+0000","last_peered":"2023-09-23T20:27:43.040851+0000","last_clean":"2023-09-23T20:27:43.040851+0000","last_became_active":"2023-09-23T20:27:14.269167+0000","last_became_peered":"2023-09-23T20:27:14.269167+0000","last_unstale":"2023-09-23T20:27:43.040851+0000","last_undegraded":"2023-09-23T20:27:43.040851+0000","last_fullsized":"2023-09-23T20:27:43.040851+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":2584,"log_dups_size":0,"ondisk_log_size":2584,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:57:54.752399+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":169345024,"num_objects":2584,"num_object_clones":0,"num_object_copies":7752,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2584,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2584,"num_write_kb":165376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":186,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-09-23T20:27:39.055952+0000","last_change":"2023-09-23T20:20:46.950784+0000","last_active":"2023-09-23T20:27:39.055952+0000","last_peered":"2023-09-23T20:27:39.055952+0000","last_clean":"2023-09-23T20:27:39.055952+0000","last_became_active":"2023-09-23T20:20:46.950055+0000","last_became_peered":"2023-09-23T20:20:46.950055+0000","last_unstale":"2023-09-23T20:27:39.055952+0000","last_undegraded":"2023-09-23T20:27:39.055952+0000","last_fullsized":"2023-09-23T20:27:39.055952+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:23:07.645427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":218,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-09-23T20:27:29.985809+0000","last_change":"2023-09-23T20:25:02.943743+0000","last_active":"2023-09-23T20:27:29.985809+0000","last_peered":"2023-09-23T20:27:29.985809+0000","last_clean":"2023-09-23T20:27:29.985809+0000","last_became_active":"2023-09-23T20:25:02.943279+0000","last_became_peered":"2023-09-23T20:25:02.943279+0000","last_unstale":"2023-09-23T20:27:29.985809+0000","last_undegraded":"2023-09-23T20:27:29.985809+0000","last_fullsized":"2023-09-23T20:27:29.985809+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:21:36.128366+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"224'2832","reported_seq":2847,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-09-23T20:27:45.543533+0000","last_change":"2023-09-23T20:27:14.271145+0000","last_active":"2023-09-23T20:27:45.543533+0000","last_peered":"2023-09-23T20:27:45.543533+0000","last_clean":"2023-09-23T20:27:45.543533+0000","last_became_active":"2023-09-23T20:27:14.270876+0000","last_became_peered":"2023-09-23T20:27:14.270876+0000","last_unstale":"2023-09-23T20:27:45.543533+0000","last_undegraded":"2023-09-23T20:27:45.543533+0000","last_fullsized":"2023-09-23T20:27:45.543533+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":2832,"log_dups_size":0,"ondisk_log_size":2832,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:23:40.941512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":185597952,"num_objects":2832,"num_object_clones":0,"num_object_copies":8496,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2832,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2832,"num_write_kb":181248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":168,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-09-23T20:27:39.056377+0000","last_change":"2023-09-23T20:12:43.751523+0000","last_active":"2023-09-23T20:27:39.056377+0000","last_peered":"2023-09-23T20:27:39.056377+0000","last_clean":"2023-09-23T20:27:39.056377+0000","last_became_active":"2023-09-23T20:12:43.751257+0000","last_became_peered":"2023-09-23T20:12:43.751257+0000","last_unstale":"2023-09-23T20:27:39.056377+0000","last_undegraded":"2023-09-23T20:27:39.056377+0000","last_fullsized":"2023-09-23T20:27:39.056377+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:50:27.040225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":183,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-09-23T20:27:39.060075+0000","last_change":"2023-09-23T20:20:46.934762+0000","last_active":"2023-09-23T20:27:39.060075+0000","last_peered":"2023-09-23T20:27:39.060075+0000","last_clean":"2023-09-23T20:27:39.060075+0000","last_became_active":"2023-09-23T20:20:46.934472+0000","last_became_peered":"2023-09-23T20:20:46.934472+0000","last_unstale":"2023-09-23T20:27:39.060075+0000","last_undegraded":"2023-09-23T20:27:39.060075+0000","last_fullsized":"2023-09-23T20:27:39.060075+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:28:42.708286+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"223'2857","reported_seq":2872,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-09-23T20:27:44.910434+0000","last_change":"2023-09-23T20:27:14.268973+0000","last_active":"2023-09-23T20:27:44.910434+0000","last_peered":"2023-09-23T20:27:44.910434+0000","last_clean":"2023-09-23T20:27:44.910434+0000","last_became_active":"2023-09-23T20:27:14.268670+0000","last_became_peered":"2023-09-23T20:27:14.268670+0000","last_unstale":"2023-09-23T20:27:44.910434+0000","last_undegraded":"2023-09-23T20:27:44.910434+0000","last_fullsized":"2023-09-23T20:27:44.910434+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":2857,"log_dups_size":0,"ondisk_log_size":2857,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:37:23.418588+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":187236352,"num_objects":2857,"num_object_clones":0,"num_object_copies":8571,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2857,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2857,"num_write_kb":182848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":202,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-09-23T20:27:29.988136+0000","last_change":"2023-09-23T20:25:02.947631+0000","last_active":"2023-09-23T20:27:29.988136+0000","last_peered":"2023-09-23T20:27:29.988136+0000","last_clean":"2023-09-23T20:27:29.988136+0000","last_became_active":"2023-09-23T20:25:02.947268+0000","last_became_peered":"2023-09-23T20:25:02.947268+0000","last_unstale":"2023-09-23T20:27:29.988136+0000","last_undegraded":"2023-09-23T20:27:29.988136+0000","last_fullsized":"2023-09-23T20:27:29.988136+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:19:43.643985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":168,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-09-23T20:27:29.978365+0000","last_change":"2023-09-23T20:20:46.935968+0000","last_active":"2023-09-23T20:27:29.978365+0000","last_peered":"2023-09-23T20:27:29.978365+0000","last_clean":"2023-09-23T20:27:29.978365+0000","last_became_active":"2023-09-23T20:20:46.935618+0000","last_became_peered":"2023-09-23T20:20:46.935618+0000","last_unstale":"2023-09-23T20:27:29.978365+0000","last_undegraded":"2023-09-23T20:27:29.978365+0000","last_fullsized":"2023-09-23T20:27:29.978365+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:42:59.867176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"223'2380","reported_seq":2394,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-09-23T20:27:40.673283+0000","last_change":"2023-09-23T20:27:14.270193+0000","last_active":"2023-09-23T20:27:40.673283+0000","last_peered":"2023-09-23T20:27:40.673283+0000","last_clean":"2023-09-23T20:27:40.673283+0000","last_became_active":"2023-09-23T20:27:14.269963+0000","last_became_peered":"2023-09-23T20:27:14.269963+0000","last_unstale":"2023-09-23T20:27:40.673283+0000","last_undegraded":"2023-09-23T20:27:40.673283+0000","last_fullsized":"2023-09-23T20:27:40.673283+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":2380,"log_dups_size":0,"ondisk_log_size":2380,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:01:09.891149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":155975680,"num_objects":2380,"num_object_clones":0,"num_object_copies":7140,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2380,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2380,"num_write_kb":152320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":596,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-09-23T20:27:39.055613+0000","last_change":"2023-09-23T20:27:11.961741+0000","last_active":"2023-09-23T20:27:39.055613+0000","last_peered":"2023-09-23T20:27:39.055613+0000","last_clean":"2023-09-23T20:27:39.055613+0000","last_became_active":"2023-09-23T20:27:11.961310+0000","last_became_peered":"2023-09-23T20:27:11.961310+0000","last_unstale":"2023-09-23T20:27:39.055613+0000","last_undegraded":"2023-09-23T20:27:39.055613+0000","last_fullsized":"2023-09-23T20:27:39.055613+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:09:47.213130+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":73,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-09-23T20:27:29.987850+0000","last_change":"2023-09-23T20:25:03.000957+0000","last_active":"2023-09-23T20:27:29.987850+0000","last_peered":"2023-09-23T20:27:29.987850+0000","last_clean":"2023-09-23T20:27:29.987850+0000","last_became_active":"2023-09-23T20:25:02.939903+0000","last_became_peered":"2023-09-23T20:25:02.939903+0000","last_unstale":"2023-09-23T20:27:29.987850+0000","last_undegraded":"2023-09-23T20:27:29.987850+0000","last_fullsized":"2023-09-23T20:27:29.987850+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:51:14.981461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":201,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-09-23T20:27:29.986572+0000","last_change":"2023-09-23T20:25:02.929447+0000","last_active":"2023-09-23T20:27:29.986572+0000","last_peered":"2023-09-23T20:27:29.986572+0000","last_clean":"2023-09-23T20:27:29.986572+0000","last_became_active":"2023-09-23T20:25:02.929173+0000","last_became_peered":"2023-09-23T20:25:02.929173+0000","last_unstale":"2023-09-23T20:27:29.986572+0000","last_undegraded":"2023-09-23T20:27:29.986572+0000","last_fullsized":"2023-09-23T20:27:29.986572+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:32:09.280647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"223'2638","reported_seq":2653,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-09-23T20:27:42.002958+0000","last_change":"2023-09-23T20:27:14.270199+0000","last_active":"2023-09-23T20:27:42.002958+0000","last_peered":"2023-09-23T20:27:42.002958+0000","last_clean":"2023-09-23T20:27:42.002958+0000","last_became_active":"2023-09-23T20:27:14.269858+0000","last_became_peered":"2023-09-23T20:27:14.269858+0000","last_unstale":"2023-09-23T20:27:42.002958+0000","last_undegraded":"2023-09-23T20:27:42.002958+0000","last_fullsized":"2023-09-23T20:27:42.002958+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":2638,"log_dups_size":0,"ondisk_log_size":2638,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:47:21.406552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":172883968,"num_objects":2638,"num_object_clones":0,"num_object_copies":7914,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2638,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2638,"num_write_kb":168832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":447,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-09-23T20:27:31.757133+0000","last_change":"2023-09-23T20:27:31.757060+0000","last_active":"2023-09-23T20:27:31.757133+0000","last_peered":"2023-09-23T20:27:31.757133+0000","last_clean":"2023-09-23T20:27:31.757133+0000","last_became_active":"2023-09-23T20:27:11.959516+0000","last_became_peered":"2023-09-23T20:27:11.959516+0000","last_unstale":"2023-09-23T20:27:31.757133+0000","last_undegraded":"2023-09-23T20:27:31.757133+0000","last_fullsized":"2023-09-23T20:27:31.757133+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:31.756974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:31.756974+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:35:11.939335+0000","scrub_duration":0.0031802670000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":184,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-09-23T20:27:39.060210+0000","last_change":"2023-09-23T20:20:46.937163+0000","last_active":"2023-09-23T20:27:39.060210+0000","last_peered":"2023-09-23T20:27:39.060210+0000","last_clean":"2023-09-23T20:27:39.060210+0000","last_became_active":"2023-09-23T20:20:46.936757+0000","last_became_peered":"2023-09-23T20:20:46.936757+0000","last_unstale":"2023-09-23T20:27:39.060210+0000","last_undegraded":"2023-09-23T20:27:39.060210+0000","last_fullsized":"2023-09-23T20:27:39.060210+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:09.401379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":211,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-09-23T20:27:29.986057+0000","last_change":"2023-09-23T20:25:39.624636+0000","last_active":"2023-09-23T20:27:29.986057+0000","last_peered":"2023-09-23T20:27:29.986057+0000","last_clean":"2023-09-23T20:27:29.986057+0000","last_became_active":"2023-09-23T20:25:39.624297+0000","last_became_peered":"2023-09-23T20:25:39.624297+0000","last_unstale":"2023-09-23T20:27:29.986057+0000","last_undegraded":"2023-09-23T20:27:29.986057+0000","last_fullsized":"2023-09-23T20:27:29.986057+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:33:47.448737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"223'2582","reported_seq":2597,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-09-23T20:27:43.036763+0000","last_change":"2023-09-23T20:27:14.712208+0000","last_active":"2023-09-23T20:27:43.036763+0000","last_peered":"2023-09-23T20:27:43.036763+0000","last_clean":"2023-09-23T20:27:43.036763+0000","last_became_active":"2023-09-23T20:27:14.711905+0000","last_became_peered":"2023-09-23T20:27:14.711905+0000","last_unstale":"2023-09-23T20:27:43.036763+0000","last_undegraded":"2023-09-23T20:27:43.036763+0000","last_fullsized":"2023-09-23T20:27:43.036763+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":2582,"log_dups_size":0,"ondisk_log_size":2582,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:14:54.221825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":169213952,"num_objects":2582,"num_object_clones":0,"num_object_copies":7746,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2582,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2582,"num_write_kb":165248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":202,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-09-23T20:27:29.985715+0000","last_change":"2023-09-23T20:25:39.628097+0000","last_active":"2023-09-23T20:27:29.985715+0000","last_peered":"2023-09-23T20:27:29.985715+0000","last_clean":"2023-09-23T20:27:29.985715+0000","last_became_active":"2023-09-23T20:25:39.627806+0000","last_became_peered":"2023-09-23T20:25:39.627806+0000","last_unstale":"2023-09-23T20:27:29.985715+0000","last_undegraded":"2023-09-23T20:27:29.985715+0000","last_fullsized":"2023-09-23T20:27:29.985715+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:53:44.398854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":181,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-09-23T20:27:29.987779+0000","last_change":"2023-09-23T20:20:46.939720+0000","last_active":"2023-09-23T20:27:29.987779+0000","last_peered":"2023-09-23T20:27:29.987779+0000","last_clean":"2023-09-23T20:27:29.987779+0000","last_became_active":"2023-09-23T20:20:46.939357+0000","last_became_peered":"2023-09-23T20:20:46.939357+0000","last_unstale":"2023-09-23T20:27:29.987779+0000","last_undegraded":"2023-09-23T20:27:29.987779+0000","last_fullsized":"2023-09-23T20:27:29.987779+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:56.613212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"223'2493","reported_seq":2507,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-09-23T20:27:40.609023+0000","last_change":"2023-09-23T20:27:14.268955+0000","last_active":"2023-09-23T20:27:40.609023+0000","last_peered":"2023-09-23T20:27:40.609023+0000","last_clean":"2023-09-23T20:27:40.609023+0000","last_became_active":"2023-09-23T20:27:14.268662+0000","last_became_peered":"2023-09-23T20:27:14.268662+0000","last_unstale":"2023-09-23T20:27:40.609023+0000","last_undegraded":"2023-09-23T20:27:40.609023+0000","last_fullsized":"2023-09-23T20:27:40.609023+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":2493,"log_dups_size":0,"ondisk_log_size":2493,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:33:52.785540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163381248,"num_objects":2493,"num_object_clones":0,"num_object_copies":7479,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2493,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2493,"num_write_kb":159552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":447,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-09-23T20:27:30.727452+0000","last_change":"2023-09-23T20:27:30.727372+0000","last_active":"2023-09-23T20:27:30.727452+0000","last_peered":"2023-09-23T20:27:30.727452+0000","last_clean":"2023-09-23T20:27:30.727452+0000","last_became_active":"2023-09-23T20:27:11.961051+0000","last_became_peered":"2023-09-23T20:27:11.961051+0000","last_unstale":"2023-09-23T20:27:30.727452+0000","last_undegraded":"2023-09-23T20:27:30.727452+0000","last_fullsized":"2023-09-23T20:27:30.727452+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:30.727267+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:30.727267+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:28:31.374288+0000","scrub_duration":0.0052560289999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":69,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-09-23T20:27:29.987610+0000","last_change":"2023-09-23T20:25:02.978173+0000","last_active":"2023-09-23T20:27:29.987610+0000","last_peered":"2023-09-23T20:27:29.987610+0000","last_clean":"2023-09-23T20:27:29.987610+0000","last_became_active":"2023-09-23T20:25:02.977856+0000","last_became_peered":"2023-09-23T20:25:02.977856+0000","last_unstale":"2023-09-23T20:27:29.987610+0000","last_undegraded":"2023-09-23T20:27:29.987610+0000","last_fullsized":"2023-09-23T20:27:29.987610+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:19.861943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":83,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-09-23T20:27:29.987049+0000","last_change":"2023-09-23T20:25:02.924911+0000","last_active":"2023-09-23T20:27:29.987049+0000","last_peered":"2023-09-23T20:27:29.987049+0000","last_clean":"2023-09-23T20:27:29.987049+0000","last_became_active":"2023-09-23T20:25:02.924586+0000","last_became_peered":"2023-09-23T20:25:02.924586+0000","last_unstale":"2023-09-23T20:27:29.987049+0000","last_undegraded":"2023-09-23T20:27:29.987049+0000","last_fullsized":"2023-09-23T20:27:29.987049+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:24:44.176766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"223'2800","reported_seq":2815,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-09-23T20:27:44.896603+0000","last_change":"2023-09-23T20:27:14.267314+0000","last_active":"2023-09-23T20:27:44.896603+0000","last_peered":"2023-09-23T20:27:44.896603+0000","last_clean":"2023-09-23T20:27:44.896603+0000","last_became_active":"2023-09-23T20:27:14.267011+0000","last_became_peered":"2023-09-23T20:27:14.267011+0000","last_unstale":"2023-09-23T20:27:44.896603+0000","last_undegraded":"2023-09-23T20:27:44.896603+0000","last_fullsized":"2023-09-23T20:27:44.896603+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":2800,"log_dups_size":0,"ondisk_log_size":2800,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:47:58.542542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":183500800,"num_objects":2800,"num_object_clones":0,"num_object_copies":8400,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2800,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2800,"num_write_kb":179200,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":457,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-09-23T20:27:39.060828+0000","last_change":"2023-09-23T20:27:32.746419+0000","last_active":"2023-09-23T20:27:39.060828+0000","last_peered":"2023-09-23T20:27:39.060828+0000","last_clean":"2023-09-23T20:27:39.060828+0000","last_became_active":"2023-09-23T20:27:11.963740+0000","last_became_peered":"2023-09-23T20:27:11.963740+0000","last_unstale":"2023-09-23T20:27:39.060828+0000","last_undegraded":"2023-09-23T20:27:39.060828+0000","last_fullsized":"2023-09-23T20:27:39.060828+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:32.746340+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:32.746340+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:51:57.739554+0000","scrub_duration":0.001880304,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"86'119","reported_seq":557,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-09-23T20:27:29.983700+0000","last_change":"2023-09-23T20:20:46.937071+0000","last_active":"2023-09-23T20:27:29.983700+0000","last_peered":"2023-09-23T20:27:29.983700+0000","last_clean":"2023-09-23T20:27:29.983700+0000","last_became_active":"2023-09-23T20:20:46.935348+0000","last_became_peered":"2023-09-23T20:20:46.935348+0000","last_unstale":"2023-09-23T20:27:29.983700+0000","last_undegraded":"2023-09-23T20:27:29.983700+0000","last_fullsized":"2023-09-23T20:27:29.983700+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":119,"log_dups_size":0,"ondisk_log_size":119,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:09.829030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032705499999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":523,"num_read_kb":3503,"num_write":136,"num_write_kb":3128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":239,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-09-23T20:27:29.987391+0000","last_change":"2023-09-23T20:25:39.637160+0000","last_active":"2023-09-23T20:27:29.987391+0000","last_peered":"2023-09-23T20:27:29.987391+0000","last_clean":"2023-09-23T20:27:29.987391+0000","last_became_active":"2023-09-23T20:25:39.636860+0000","last_became_peered":"2023-09-23T20:25:39.636860+0000","last_unstale":"2023-09-23T20:27:29.987391+0000","last_undegraded":"2023-09-23T20:27:29.987391+0000","last_fullsized":"2023-09-23T20:27:29.987391+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:21:47.860063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":149,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-09-23T20:27:29.983316+0000","last_change":"2023-09-23T20:20:46.937104+0000","last_active":"2023-09-23T20:27:29.983316+0000","last_peered":"2023-09-23T20:27:29.983316+0000","last_clean":"2023-09-23T20:27:29.983316+0000","last_became_active":"2023-09-23T20:20:46.936820+0000","last_became_peered":"2023-09-23T20:20:46.936820+0000","last_unstale":"2023-09-23T20:27:29.983316+0000","last_undegraded":"2023-09-23T20:27:29.983316+0000","last_fullsized":"2023-09-23T20:27:29.983316+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:22:19.879310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.6","version":"223'2590","reported_seq":2605,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-09-23T20:27:43.027695+0000","last_change":"2023-09-23T20:27:14.269226+0000","last_active":"2023-09-23T20:27:43.027695+0000","last_peered":"2023-09-23T20:27:43.027695+0000","last_clean":"2023-09-23T20:27:43.027695+0000","last_became_active":"2023-09-23T20:27:14.268927+0000","last_became_peered":"2023-09-23T20:27:14.268927+0000","last_unstale":"2023-09-23T20:27:43.027695+0000","last_undegraded":"2023-09-23T20:27:43.027695+0000","last_fullsized":"2023-09-23T20:27:43.027695+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":2590,"log_dups_size":0,"ondisk_log_size":2590,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:45:57.844328+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":169738240,"num_objects":2590,"num_object_clones":0,"num_object_copies":7770,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2590,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2590,"num_write_kb":165760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":447,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-09-23T20:27:34.823329+0000","last_change":"2023-09-23T20:27:34.823244+0000","last_active":"2023-09-23T20:27:34.823329+0000","last_peered":"2023-09-23T20:27:34.823329+0000","last_clean":"2023-09-23T20:27:34.823329+0000","last_became_active":"2023-09-23T20:27:11.961863+0000","last_became_peered":"2023-09-23T20:27:11.961863+0000","last_unstale":"2023-09-23T20:27:34.823329+0000","last_undegraded":"2023-09-23T20:27:34.823329+0000","last_fullsized":"2023-09-23T20:27:34.823329+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:34.821789+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:34.821789+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:24:04.705573+0000","scrub_duration":0.0042962870000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"86'106","reported_seq":847,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-09-23T20:27:29.988917+0000","last_change":"2023-09-23T20:20:46.939468+0000","last_active":"2023-09-23T20:27:29.988917+0000","last_peered":"2023-09-23T20:27:29.988917+0000","last_clean":"2023-09-23T20:27:29.988917+0000","last_became_active":"2023-09-23T20:20:46.938954+0000","last_became_peered":"2023-09-23T20:20:46.938954+0000","last_unstale":"2023-09-23T20:27:29.988917+0000","last_undegraded":"2023-09-23T20:27:29.988917+0000","last_fullsized":"2023-09-23T20:27:29.988917+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":106,"log_dups_size":0,"ondisk_log_size":106,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:01:43.761064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00099302599999999993,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1105,"num_read_kb":1689,"num_write":100,"num_write_kb":404,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":182,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-09-23T20:27:29.986132+0000","last_change":"2023-09-23T20:20:14.887345+0000","last_active":"2023-09-23T20:27:29.986132+0000","last_peered":"2023-09-23T20:27:29.986132+0000","last_clean":"2023-09-23T20:27:29.986132+0000","last_became_active":"2023-09-23T20:20:14.886970+0000","last_became_peered":"2023-09-23T20:20:14.886970+0000","last_unstale":"2023-09-23T20:27:29.986132+0000","last_undegraded":"2023-09-23T20:27:29.986132+0000","last_fullsized":"2023-09-23T20:27:29.986132+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:00:38.575926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":215,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-09-23T20:27:29.983621+0000","last_change":"2023-09-23T20:25:39.631133+0000","last_active":"2023-09-23T20:27:29.983621+0000","last_peered":"2023-09-23T20:27:29.983621+0000","last_clean":"2023-09-23T20:27:29.983621+0000","last_became_active":"2023-09-23T20:25:39.630779+0000","last_became_peered":"2023-09-23T20:25:39.630779+0000","last_unstale":"2023-09-23T20:27:29.983621+0000","last_undegraded":"2023-09-23T20:27:29.983621+0000","last_fullsized":"2023-09-23T20:27:29.983621+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:39:52.635649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.5","version":"224'2861","reported_seq":2876,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-09-23T20:27:45.550091+0000","last_change":"2023-09-23T20:27:14.711792+0000","last_active":"2023-09-23T20:27:45.550091+0000","last_peered":"2023-09-23T20:27:45.550091+0000","last_clean":"2023-09-23T20:27:45.550091+0000","last_became_active":"2023-09-23T20:27:14.711434+0000","last_became_peered":"2023-09-23T20:27:14.711434+0000","last_unstale":"2023-09-23T20:27:45.550091+0000","last_undegraded":"2023-09-23T20:27:45.550091+0000","last_fullsized":"2023-09-23T20:27:45.550091+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":2861,"log_dups_size":0,"ondisk_log_size":2861,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:10:14.885066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":187498496,"num_objects":2861,"num_object_clones":0,"num_object_copies":8583,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2861,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2861,"num_write_kb":183104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":606,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-09-23T20:27:39.056903+0000","last_change":"2023-09-23T20:27:35.734647+0000","last_active":"2023-09-23T20:27:39.056903+0000","last_peered":"2023-09-23T20:27:39.056903+0000","last_clean":"2023-09-23T20:27:39.056903+0000","last_became_active":"2023-09-23T20:27:11.961418+0000","last_became_peered":"2023-09-23T20:27:11.961418+0000","last_unstale":"2023-09-23T20:27:39.056903+0000","last_undegraded":"2023-09-23T20:27:39.056903+0000","last_fullsized":"2023-09-23T20:27:39.056903+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:35.734584+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:35.734584+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:39:21.488793+0000","scrub_duration":0.0035251380000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"86'130","reported_seq":532,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-09-23T20:27:29.989505+0000","last_change":"2023-09-23T20:25:39.637458+0000","last_active":"2023-09-23T20:27:29.989505+0000","last_peered":"2023-09-23T20:27:29.989505+0000","last_clean":"2023-09-23T20:27:29.989505+0000","last_became_active":"2023-09-23T20:25:39.636773+0000","last_became_peered":"2023-09-23T20:25:39.636773+0000","last_unstale":"2023-09-23T20:27:29.989505+0000","last_undegraded":"2023-09-23T20:27:29.989505+0000","last_fullsized":"2023-09-23T20:27:29.989505+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:00:32.810564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011300049999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":512,"num_read_kb":1820,"num_write":133,"num_write_kb":1347,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":181,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-09-23T20:27:29.984264+0000","last_change":"2023-09-23T20:25:39.632283+0000","last_active":"2023-09-23T20:27:29.984264+0000","last_peered":"2023-09-23T20:27:29.984264+0000","last_clean":"2023-09-23T20:27:29.984264+0000","last_became_active":"2023-09-23T20:25:39.631819+0000","last_became_peered":"2023-09-23T20:25:39.631819+0000","last_unstale":"2023-09-23T20:27:29.984264+0000","last_undegraded":"2023-09-23T20:27:29.984264+0000","last_fullsized":"2023-09-23T20:27:29.984264+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:25:21.815555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":192,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-09-23T20:27:29.986926+0000","last_change":"2023-09-23T20:20:46.936524+0000","last_active":"2023-09-23T20:27:29.986926+0000","last_peered":"2023-09-23T20:27:29.986926+0000","last_clean":"2023-09-23T20:27:29.986926+0000","last_became_active":"2023-09-23T20:20:46.936267+0000","last_became_peered":"2023-09-23T20:20:46.936267+0000","last_unstale":"2023-09-23T20:27:29.986926+0000","last_undegraded":"2023-09-23T20:27:29.986926+0000","last_fullsized":"2023-09-23T20:27:29.986926+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:32:20.984521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"223'2536","reported_seq":2550,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-09-23T20:27:40.586039+0000","last_change":"2023-09-23T20:27:14.272939+0000","last_active":"2023-09-23T20:27:40.586039+0000","last_peered":"2023-09-23T20:27:40.586039+0000","last_clean":"2023-09-23T20:27:40.586039+0000","last_became_active":"2023-09-23T20:27:14.272096+0000","last_became_peered":"2023-09-23T20:27:14.272096+0000","last_unstale":"2023-09-23T20:27:40.586039+0000","last_undegraded":"2023-09-23T20:27:40.586039+0000","last_fullsized":"2023-09-23T20:27:40.586039+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":2536,"log_dups_size":0,"ondisk_log_size":2536,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:41:48.494925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":166199296,"num_objects":2536,"num_object_clones":0,"num_object_copies":7608,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2536,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2536,"num_write_kb":162304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"86'194","reported_seq":871,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-09-23T20:27:29.987400+0000","last_change":"2023-09-23T20:25:40.618665+0000","last_active":"2023-09-23T20:27:29.987400+0000","last_peered":"2023-09-23T20:27:29.987400+0000","last_clean":"2023-09-23T20:27:29.987400+0000","last_became_active":"2023-09-23T20:25:40.618087+0000","last_became_peered":"2023-09-23T20:25:40.618087+0000","last_unstale":"2023-09-23T20:27:29.987400+0000","last_undegraded":"2023-09-23T20:27:29.987400+0000","last_fullsized":"2023-09-23T20:27:29.987400+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":194,"log_dups_size":0,"ondisk_log_size":194,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:07:16.114968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00037995999999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1025,"num_read_kb":1201,"num_write":173,"num_write_kb":1276,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":590,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-09-23T20:27:39.054631+0000","last_change":"2023-09-23T20:27:32.801281+0000","last_active":"2023-09-23T20:27:39.054631+0000","last_peered":"2023-09-23T20:27:39.054631+0000","last_clean":"2023-09-23T20:27:39.054631+0000","last_became_active":"2023-09-23T20:25:21.159200+0000","last_became_peered":"2023-09-23T20:25:21.159200+0000","last_unstale":"2023-09-23T20:27:39.054631+0000","last_undegraded":"2023-09-23T20:27:39.054631+0000","last_fullsized":"2023-09-23T20:27:39.054631+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":173,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:32.801168+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:32.801168+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:41:23.687740+0000","scrub_duration":0.0096333110000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":69,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-09-23T20:27:29.989058+0000","last_change":"2023-09-23T20:25:02.984205+0000","last_active":"2023-09-23T20:27:29.989058+0000","last_peered":"2023-09-23T20:27:29.989058+0000","last_clean":"2023-09-23T20:27:29.989058+0000","last_became_active":"2023-09-23T20:25:02.983758+0000","last_became_peered":"2023-09-23T20:25:02.983758+0000","last_unstale":"2023-09-23T20:27:29.989058+0000","last_undegraded":"2023-09-23T20:27:29.989058+0000","last_fullsized":"2023-09-23T20:27:29.989058+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:35:14.573101+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":166,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-09-23T20:27:39.060934+0000","last_change":"2023-09-23T20:12:45.609829+0000","last_active":"2023-09-23T20:27:39.060934+0000","last_peered":"2023-09-23T20:27:39.060934+0000","last_clean":"2023-09-23T20:27:39.060934+0000","last_became_active":"2023-09-23T20:12:45.609467+0000","last_became_peered":"2023-09-23T20:12:45.609467+0000","last_unstale":"2023-09-23T20:27:39.060934+0000","last_undegraded":"2023-09-23T20:27:39.060934+0000","last_fullsized":"2023-09-23T20:27:39.060934+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:27:31.490555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.4","version":"223'2811","reported_seq":2823,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-09-23T20:27:45.056459+0000","last_change":"2023-09-23T20:27:14.267770+0000","last_active":"2023-09-23T20:27:45.056459+0000","last_peered":"2023-09-23T20:27:45.056459+0000","last_clean":"2023-09-23T20:27:45.056459+0000","last_became_active":"2023-09-23T20:27:14.266851+0000","last_became_peered":"2023-09-23T20:27:14.266851+0000","last_unstale":"2023-09-23T20:27:45.056459+0000","last_undegraded":"2023-09-23T20:27:45.056459+0000","last_fullsized":"2023-09-23T20:27:45.056459+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":2811,"log_dups_size":0,"ondisk_log_size":2811,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:50:38.592759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":184221696,"num_objects":2811,"num_object_clones":0,"num_object_copies":8433,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2811,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2811,"num_write_kb":179904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":448,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-09-23T20:27:29.983738+0000","last_change":"2023-09-23T20:27:12.386203+0000","last_active":"2023-09-23T20:27:29.983738+0000","last_peered":"2023-09-23T20:27:29.983738+0000","last_clean":"2023-09-23T20:27:29.983738+0000","last_became_active":"2023-09-23T20:27:11.962975+0000","last_became_peered":"2023-09-23T20:27:11.962975+0000","last_unstale":"2023-09-23T20:27:29.983738+0000","last_undegraded":"2023-09-23T20:27:29.983738+0000","last_fullsized":"2023-09-23T20:27:29.983738+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:12.386098+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:12.386098+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:23:49.746955+0000","scrub_duration":0.0019500940000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"86'80","reported_seq":470,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-09-23T20:27:39.054539+0000","last_change":"2023-09-23T20:20:46.950784+0000","last_active":"2023-09-23T20:27:39.054539+0000","last_peered":"2023-09-23T20:27:39.054539+0000","last_clean":"2023-09-23T20:27:39.054539+0000","last_became_active":"2023-09-23T20:20:46.950071+0000","last_became_peered":"2023-09-23T20:20:46.950071+0000","last_unstale":"2023-09-23T20:27:39.054539+0000","last_undegraded":"2023-09-23T20:27:39.054539+0000","last_fullsized":"2023-09-23T20:27:39.054539+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":80,"log_dups_size":0,"ondisk_log_size":80,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:11:06.812404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035115,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":395,"num_read_kb":1370,"num_write":66,"num_write_kb":1061,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":185,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-09-23T20:27:29.979145+0000","last_change":"2023-09-23T20:25:02.942709+0000","last_active":"2023-09-23T20:27:29.979145+0000","last_peered":"2023-09-23T20:27:29.979145+0000","last_clean":"2023-09-23T20:27:29.979145+0000","last_became_active":"2023-09-23T20:25:02.942323+0000","last_became_peered":"2023-09-23T20:25:02.942323+0000","last_unstale":"2023-09-23T20:27:29.979145+0000","last_undegraded":"2023-09-23T20:27:29.979145+0000","last_fullsized":"2023-09-23T20:27:29.979145+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:26:12.039101+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":201,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-09-23T20:27:29.979095+0000","last_change":"2023-09-23T20:25:39.641452+0000","last_active":"2023-09-23T20:27:29.979095+0000","last_peered":"2023-09-23T20:27:29.979095+0000","last_clean":"2023-09-23T20:27:29.979095+0000","last_became_active":"2023-09-23T20:25:39.638987+0000","last_became_peered":"2023-09-23T20:25:39.638987+0000","last_unstale":"2023-09-23T20:27:29.979095+0000","last_undegraded":"2023-09-23T20:27:29.979095+0000","last_fullsized":"2023-09-23T20:27:29.979095+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:33:59.955461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"223'2797","reported_seq":2812,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-09-23T20:27:45.040553+0000","last_change":"2023-09-23T20:27:14.712242+0000","last_active":"2023-09-23T20:27:45.040553+0000","last_peered":"2023-09-23T20:27:45.040553+0000","last_clean":"2023-09-23T20:27:45.040553+0000","last_became_active":"2023-09-23T20:27:14.711916+0000","last_became_peered":"2023-09-23T20:27:14.711916+0000","last_unstale":"2023-09-23T20:27:45.040553+0000","last_undegraded":"2023-09-23T20:27:45.040553+0000","last_fullsized":"2023-09-23T20:27:45.040553+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":2797,"log_dups_size":0,"ondisk_log_size":2797,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:08:51.484867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":183304192,"num_objects":2797,"num_object_clones":0,"num_object_copies":8391,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2797,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2797,"num_write_kb":179008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":457,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-09-23T20:27:39.059600+0000","last_change":"2023-09-23T20:27:31.769411+0000","last_active":"2023-09-23T20:27:39.059600+0000","last_peered":"2023-09-23T20:27:39.059600+0000","last_clean":"2023-09-23T20:27:39.059600+0000","last_became_active":"2023-09-23T20:27:11.962792+0000","last_became_peered":"2023-09-23T20:27:11.962792+0000","last_unstale":"2023-09-23T20:27:39.059600+0000","last_undegraded":"2023-09-23T20:27:39.059600+0000","last_fullsized":"2023-09-23T20:27:39.059600+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:31.769333+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:31.769333+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:13:31.871002+0000","scrub_duration":0.002679046,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"86'102","reported_seq":730,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-09-23T20:27:39.056454+0000","last_change":"2023-09-23T20:20:14.881975+0000","last_active":"2023-09-23T20:27:39.056454+0000","last_peered":"2023-09-23T20:27:39.056454+0000","last_clean":"2023-09-23T20:27:39.056454+0000","last_became_active":"2023-09-23T20:20:14.881544+0000","last_became_peered":"2023-09-23T20:20:14.881544+0000","last_unstale":"2023-09-23T20:27:39.056454+0000","last_undegraded":"2023-09-23T20:27:39.056454+0000","last_fullsized":"2023-09-23T20:27:39.056454+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":102,"log_dups_size":0,"ondisk_log_size":102,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:34.594972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030656500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":963,"num_read_kb":1234,"num_write":60,"num_write_kb":225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":181,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-09-23T20:27:29.983798+0000","last_change":"2023-09-23T20:25:39.631751+0000","last_active":"2023-09-23T20:27:29.983798+0000","last_peered":"2023-09-23T20:27:29.983798+0000","last_clean":"2023-09-23T20:27:29.983798+0000","last_became_active":"2023-09-23T20:25:39.631136+0000","last_became_peered":"2023-09-23T20:25:39.631136+0000","last_unstale":"2023-09-23T20:27:29.983798+0000","last_undegraded":"2023-09-23T20:27:29.983798+0000","last_fullsized":"2023-09-23T20:27:29.983798+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:08:17.020722+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":153,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-09-23T20:27:29.978599+0000","last_change":"2023-09-23T20:20:14.888874+0000","last_active":"2023-09-23T20:27:29.978599+0000","last_peered":"2023-09-23T20:27:29.978599+0000","last_clean":"2023-09-23T20:27:29.978599+0000","last_became_active":"2023-09-23T20:20:14.887724+0000","last_became_peered":"2023-09-23T20:20:14.887724+0000","last_unstale":"2023-09-23T20:27:29.978599+0000","last_undegraded":"2023-09-23T20:27:29.978599+0000","last_fullsized":"2023-09-23T20:27:29.978599+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:09:24.510326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"223'2496","reported_seq":2510,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-09-23T20:27:40.676763+0000","last_change":"2023-09-23T20:27:14.270161+0000","last_active":"2023-09-23T20:27:40.676763+0000","last_peered":"2023-09-23T20:27:40.676763+0000","last_clean":"2023-09-23T20:27:40.676763+0000","last_became_active":"2023-09-23T20:27:14.269904+0000","last_became_peered":"2023-09-23T20:27:14.269904+0000","last_unstale":"2023-09-23T20:27:40.676763+0000","last_undegraded":"2023-09-23T20:27:40.676763+0000","last_fullsized":"2023-09-23T20:27:40.676763+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":2496,"log_dups_size":0,"ondisk_log_size":2496,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:41:02.917630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163577856,"num_objects":2496,"num_object_clones":0,"num_object_copies":7488,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2496,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2496,"num_write_kb":159744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":447,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-09-23T20:27:29.985954+0000","last_change":"2023-09-23T20:27:12.810010+0000","last_active":"2023-09-23T20:27:29.985954+0000","last_peered":"2023-09-23T20:27:29.985954+0000","last_clean":"2023-09-23T20:27:29.985954+0000","last_became_active":"2023-09-23T20:27:11.959614+0000","last_became_peered":"2023-09-23T20:27:11.959614+0000","last_unstale":"2023-09-23T20:27:29.985954+0000","last_undegraded":"2023-09-23T20:27:29.985954+0000","last_fullsized":"2023-09-23T20:27:29.985954+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:12.809933+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:12.809933+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:48:13.147428+0000","scrub_duration":0.001626628,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":73,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-09-23T20:27:29.987965+0000","last_change":"2023-09-23T20:25:03.007736+0000","last_active":"2023-09-23T20:27:29.987965+0000","last_peered":"2023-09-23T20:27:29.987965+0000","last_clean":"2023-09-23T20:27:29.987965+0000","last_became_active":"2023-09-23T20:25:02.973689+0000","last_became_peered":"2023-09-23T20:25:02.973689+0000","last_unstale":"2023-09-23T20:27:29.987965+0000","last_undegraded":"2023-09-23T20:27:29.987965+0000","last_fullsized":"2023-09-23T20:27:29.987965+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:37:25.643774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"86'128","reported_seq":548,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-09-23T20:27:29.987112+0000","last_change":"2023-09-23T20:20:46.939256+0000","last_active":"2023-09-23T20:27:29.987112+0000","last_peered":"2023-09-23T20:27:29.987112+0000","last_clean":"2023-09-23T20:27:29.987112+0000","last_became_active":"2023-09-23T20:20:46.938756+0000","last_became_peered":"2023-09-23T20:20:46.938756+0000","last_unstale":"2023-09-23T20:27:29.987112+0000","last_undegraded":"2023-09-23T20:27:29.987112+0000","last_fullsized":"2023-09-23T20:27:29.987112+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:52:18.161436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000293202,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":579,"num_read_kb":1473,"num_write":96,"num_write_kb":1062,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":69,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-09-23T20:27:29.987538+0000","last_change":"2023-09-23T20:25:02.971740+0000","last_active":"2023-09-23T20:27:29.987538+0000","last_peered":"2023-09-23T20:27:29.987538+0000","last_clean":"2023-09-23T20:27:29.987538+0000","last_became_active":"2023-09-23T20:25:02.970412+0000","last_became_peered":"2023-09-23T20:25:02.970412+0000","last_unstale":"2023-09-23T20:27:29.987538+0000","last_undegraded":"2023-09-23T20:27:29.987538+0000","last_fullsized":"2023-09-23T20:27:29.987538+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:53:17.247216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"224'2793","reported_seq":2808,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-09-23T20:27:45.539939+0000","last_change":"2023-09-23T20:27:14.711764+0000","last_active":"2023-09-23T20:27:45.539939+0000","last_peered":"2023-09-23T20:27:45.539939+0000","last_clean":"2023-09-23T20:27:45.539939+0000","last_became_active":"2023-09-23T20:27:14.711409+0000","last_became_peered":"2023-09-23T20:27:14.711409+0000","last_unstale":"2023-09-23T20:27:45.539939+0000","last_undegraded":"2023-09-23T20:27:45.539939+0000","last_fullsized":"2023-09-23T20:27:45.539939+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":2793,"log_dups_size":0,"ondisk_log_size":2793,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:46:35.659960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":183042048,"num_objects":2793,"num_object_clones":0,"num_object_copies":8379,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2793,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2793,"num_write_kb":178752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.4","version":"90'161","reported_seq":448,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-09-23T20:27:29.988702+0000","last_change":"2023-09-23T20:27:13.244744+0000","last_active":"2023-09-23T20:27:29.988702+0000","last_peered":"2023-09-23T20:27:29.988702+0000","last_clean":"2023-09-23T20:27:29.988702+0000","last_became_active":"2023-09-23T20:27:11.960349+0000","last_became_peered":"2023-09-23T20:27:11.960349+0000","last_unstale":"2023-09-23T20:27:29.988702+0000","last_undegraded":"2023-09-23T20:27:29.988702+0000","last_fullsized":"2023-09-23T20:27:29.988702+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:13.244651+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.244651+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:22:26.451174+0000","scrub_duration":0.0019955509999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"86'117","reported_seq":743,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-09-23T20:27:39.060917+0000","last_change":"2023-09-23T20:25:40.617386+0000","last_active":"2023-09-23T20:27:39.060917+0000","last_peered":"2023-09-23T20:27:39.060917+0000","last_clean":"2023-09-23T20:27:39.060917+0000","last_became_active":"2023-09-23T20:25:40.616747+0000","last_became_peered":"2023-09-23T20:25:40.616747+0000","last_unstale":"2023-09-23T20:27:39.060917+0000","last_undegraded":"2023-09-23T20:27:39.060917+0000","last_fullsized":"2023-09-23T20:27:39.060917+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:46:44.551561+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046891400000000002,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":893,"num_read_kb":922,"num_write":82,"num_write_kb":4193,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":173,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-09-23T20:27:29.979087+0000","last_change":"2023-09-23T20:20:46.937163+0000","last_active":"2023-09-23T20:27:29.979087+0000","last_peered":"2023-09-23T20:27:29.979087+0000","last_clean":"2023-09-23T20:27:29.979087+0000","last_became_active":"2023-09-23T20:20:46.936846+0000","last_became_peered":"2023-09-23T20:20:46.936846+0000","last_unstale":"2023-09-23T20:27:29.979087+0000","last_undegraded":"2023-09-23T20:27:29.979087+0000","last_fullsized":"2023-09-23T20:27:29.979087+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:01:32.798855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":164,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-09-23T20:27:29.985718+0000","last_change":"2023-09-23T20:12:43.765908+0000","last_active":"2023-09-23T20:27:29.985718+0000","last_peered":"2023-09-23T20:27:29.985718+0000","last_clean":"2023-09-23T20:27:29.985718+0000","last_became_active":"2023-09-23T20:12:43.765558+0000","last_became_peered":"2023-09-23T20:12:43.765558+0000","last_unstale":"2023-09-23T20:27:29.985718+0000","last_undegraded":"2023-09-23T20:27:29.985718+0000","last_fullsized":"2023-09-23T20:27:29.985718+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:35:43.809898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":156,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-09-23T20:27:39.061371+0000","last_change":"2023-09-23T20:20:14.888484+0000","last_active":"2023-09-23T20:27:39.061371+0000","last_peered":"2023-09-23T20:27:39.061371+0000","last_clean":"2023-09-23T20:27:39.061371+0000","last_became_active":"2023-09-23T20:20:14.887219+0000","last_became_peered":"2023-09-23T20:20:14.887219+0000","last_unstale":"2023-09-23T20:27:39.061371+0000","last_undegraded":"2023-09-23T20:27:39.061371+0000","last_fullsized":"2023-09-23T20:27:39.061371+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:13:53.763649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":57,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-09-23T20:27:29.988724+0000","last_change":"2023-09-23T20:25:39.639268+0000","last_active":"2023-09-23T20:27:29.988724+0000","last_peered":"2023-09-23T20:27:29.988724+0000","last_clean":"2023-09-23T20:27:29.988724+0000","last_became_active":"2023-09-23T20:25:39.638922+0000","last_became_peered":"2023-09-23T20:25:39.638922+0000","last_unstale":"2023-09-23T20:27:29.988724+0000","last_undegraded":"2023-09-23T20:27:29.988724+0000","last_fullsized":"2023-09-23T20:27:29.988724+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:09:04.040098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":184,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-09-23T20:27:29.988009+0000","last_change":"2023-09-23T20:20:46.938981+0000","last_active":"2023-09-23T20:27:29.988009+0000","last_peered":"2023-09-23T20:27:29.988009+0000","last_clean":"2023-09-23T20:27:29.988009+0000","last_became_active":"2023-09-23T20:20:46.938627+0000","last_became_peered":"2023-09-23T20:20:46.938627+0000","last_unstale":"2023-09-23T20:27:29.988009+0000","last_undegraded":"2023-09-23T20:27:29.988009+0000","last_fullsized":"2023-09-23T20:27:29.988009+0000","mapping_epoch":99,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:44:43.177086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":228,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-09-23T20:27:29.986370+0000","last_change":"2023-09-23T20:25:02.945142+0000","last_active":"2023-09-23T20:27:29.986370+0000","last_peered":"2023-09-23T20:27:29.986370+0000","last_clean":"2023-09-23T20:27:29.986370+0000","last_became_active":"2023-09-23T20:25:02.944653+0000","last_became_peered":"2023-09-23T20:25:02.944653+0000","last_unstale":"2023-09-23T20:27:29.986370+0000","last_undegraded":"2023-09-23T20:27:29.986370+0000","last_fullsized":"2023-09-23T20:27:29.986370+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:38:12.927982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":167,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-09-23T20:27:29.987252+0000","last_change":"2023-09-23T20:12:43.753383+0000","last_active":"2023-09-23T20:27:29.987252+0000","last_peered":"2023-09-23T20:27:29.987252+0000","last_clean":"2023-09-23T20:27:29.987252+0000","last_became_active":"2023-09-23T20:12:43.752921+0000","last_became_peered":"2023-09-23T20:12:43.752921+0000","last_unstale":"2023-09-23T20:27:29.987252+0000","last_undegraded":"2023-09-23T20:27:29.987252+0000","last_fullsized":"2023-09-23T20:27:29.987252+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:28:26.982998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":228,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-09-23T20:27:29.986052+0000","last_change":"2023-09-23T20:25:03.103656+0000","last_active":"2023-09-23T20:27:29.986052+0000","last_peered":"2023-09-23T20:27:29.986052+0000","last_clean":"2023-09-23T20:27:29.986052+0000","last_became_active":"2023-09-23T20:25:03.103311+0000","last_became_peered":"2023-09-23T20:25:03.103311+0000","last_unstale":"2023-09-23T20:27:29.986052+0000","last_undegraded":"2023-09-23T20:27:29.986052+0000","last_fullsized":"2023-09-23T20:27:29.986052+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:55:54.677030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":78,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-09-23T20:27:29.987625+0000","last_change":"2023-09-23T20:25:39.629869+0000","last_active":"2023-09-23T20:27:29.987625+0000","last_peered":"2023-09-23T20:27:29.987625+0000","last_clean":"2023-09-23T20:27:29.987625+0000","last_became_active":"2023-09-23T20:25:39.629543+0000","last_became_peered":"2023-09-23T20:25:39.629543+0000","last_unstale":"2023-09-23T20:27:29.987625+0000","last_undegraded":"2023-09-23T20:27:29.987625+0000","last_fullsized":"2023-09-23T20:27:29.987625+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:24:48.220679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":78,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-09-23T20:27:29.987120+0000","last_change":"2023-09-23T20:25:39.622819+0000","last_active":"2023-09-23T20:27:29.987120+0000","last_peered":"2023-09-23T20:27:29.987120+0000","last_clean":"2023-09-23T20:27:29.987120+0000","last_became_active":"2023-09-23T20:25:39.622484+0000","last_became_peered":"2023-09-23T20:25:39.622484+0000","last_unstale":"2023-09-23T20:27:29.987120+0000","last_undegraded":"2023-09-23T20:27:29.987120+0000","last_fullsized":"2023-09-23T20:27:29.987120+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:51:53.973261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":201,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-09-23T20:27:29.978763+0000","last_change":"2023-09-23T20:25:39.668882+0000","last_active":"2023-09-23T20:27:29.978763+0000","last_peered":"2023-09-23T20:27:29.978763+0000","last_clean":"2023-09-23T20:27:29.978763+0000","last_became_active":"2023-09-23T20:25:39.637128+0000","last_became_peered":"2023-09-23T20:25:39.637128+0000","last_unstale":"2023-09-23T20:27:29.978763+0000","last_undegraded":"2023-09-23T20:27:29.978763+0000","last_fullsized":"2023-09-23T20:27:29.978763+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:56.406298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":201,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-09-23T20:27:29.985695+0000","last_change":"2023-09-23T20:24:53.089249+0000","last_active":"2023-09-23T20:27:29.985695+0000","last_peered":"2023-09-23T20:27:29.985695+0000","last_clean":"2023-09-23T20:27:29.985695+0000","last_became_active":"2023-09-23T20:24:53.088952+0000","last_became_peered":"2023-09-23T20:24:53.088952+0000","last_unstale":"2023-09-23T20:27:29.985695+0000","last_undegraded":"2023-09-23T20:27:29.985695+0000","last_fullsized":"2023-09-23T20:27:29.985695+0000","mapping_epoch":160,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":161,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:38:03.216069+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":2783248384,"num_objects":42469,"num_object_clones":0,"num_object_copies":127407,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":42469,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":42469,"num_write_kb":2718016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8395751424,"data_stored":8395751424,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":42469,"ondisk_log_size":42469,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":15,"num_bytes_recovered":1356,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":25181,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":644038,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1771,"ondisk_log_size":1771,"up":33,"acting":33,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6082,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201479,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1191272,"kb_used_data":1068512,"kb_used_omap":0,"kb_used_meta":122752,"kb_avail":92553880,"statfs":{"total":95995035648,"available":94775173120,"internally_reserved":0,"allocated":1094156288,"data_stored":1093440400,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":125698048},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:27:24 2023","interfaces":[{"interface":"back","average":{"1min":3.9889999999999999,"5min":1.9059999999999999,"15min":1.0149999999999999},"min":{"1min":0.29399999999999998,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":48.798999999999999,"5min":48.798999999999999,"15min":48.798999999999999},"last":0.51500000000000001},{"interface":"front","average":{"1min":3.9540000000000002,"5min":2.2770000000000001,"15min":1.135},"min":{"1min":0.251,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":48.247,"5min":48.247,"15min":48.247},"last":1.7609999999999999}]},{"osd":1,"last update":"Sat Sep 23 20:26:58 2023","interfaces":[{"interface":"back","average":{"1min":1.2929999999999999,"5min":1.6499999999999999,"15min":0.92500000000000004},"min":{"1min":0.28599999999999998,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":3.1920000000000002,"5min":17.795999999999999,"15min":17.795999999999999},"last":0.87},{"interface":"front","average":{"1min":2.2989999999999999,"5min":1.504,"15min":0.90900000000000003},"min":{"1min":0.33400000000000002,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":6.4950000000000001,"5min":13.250999999999999,"15min":13.250999999999999},"last":1.1990000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:27:28 2023","interfaces":[{"interface":"back","average":{"1min":0.98799999999999999,"5min":1.6919999999999999,"15min":0.94299999999999995},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":2.9580000000000002,"5min":9.8300000000000001,"15min":9.8300000000000001},"last":0.64800000000000002},{"interface":"front","average":{"1min":1.51,"5min":1.8919999999999999,"15min":1.022},"min":{"1min":0.39200000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":7.4720000000000004,"5min":11.423999999999999,"15min":11.423999999999999},"last":1.1140000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:26:58 2023","interfaces":[{"interface":"back","average":{"1min":2.4049999999999998,"5min":1.756,"15min":1.022},"min":{"1min":0.38100000000000001,"5min":0.314,"15min":0.314},"max":{"1min":6.891,"5min":6.891,"15min":6.891},"last":0.80500000000000005},{"interface":"front","average":{"1min":2.4319999999999999,"5min":1.6279999999999999,"15min":0.99199999999999999},"min":{"1min":0.374,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":6.9050000000000002,"5min":9.7840000000000007,"15min":9.7840000000000007},"last":0.68400000000000005}]},{"osd":4,"last update":"Sat Sep 23 20:26:58 2023","interfaces":[{"interface":"back","average":{"1min":1.6479999999999999,"5min":1.1619999999999999,"15min":0.71899999999999997},"min":{"1min":0.28499999999999998,"5min":0.27000000000000002,"15min":0.26700000000000002},"max":{"1min":5.3540000000000001,"5min":13.141999999999999,"15min":13.141999999999999},"last":0.39300000000000002},{"interface":"front","average":{"1min":1.5740000000000001,"5min":1.6819999999999999,"15min":0.91400000000000003},"min":{"1min":0.29499999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":5.9279999999999999,"5min":10.853,"15min":10.853},"last":0.59099999999999997}]},{"osd":5,"last update":"Sat Sep 23 20:26:58 2023","interfaces":[{"interface":"back","average":{"1min":1.9930000000000001,"5min":1.643,"15min":0.90500000000000003},"min":{"1min":0.31,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":10.362,"5min":10.563000000000001,"15min":10.563000000000001},"last":0.45700000000000002},{"interface":"front","average":{"1min":1.956,"5min":1.5669999999999999,"15min":0.91100000000000003},"min":{"1min":0.41799999999999998,"5min":0.34699999999999998,"15min":0.31},"max":{"1min":7.9029999999999996,"5min":13.292,"15min":13.292},"last":0.42699999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:26:58 2023","interfaces":[{"interface":"back","average":{"1min":1.9890000000000001,"5min":1.4179999999999999,"15min":0.86899999999999999},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":6.2210000000000001,"5min":13.185,"15min":13.185},"last":0.69699999999999995},{"interface":"front","average":{"1min":1.9199999999999999,"5min":1.4890000000000001,"15min":0.89100000000000001},"min":{"1min":0.378,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":6.4370000000000003,"5min":9.9120000000000008,"15min":9.9120000000000008},"last":0.47199999999999998}]}]},{"osd":6,"up_from":47,"seq":201863463115,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":980256,"kb_used_data":885528,"kb_used_omap":0,"kb_used_meta":94720,"kb_avail":92764896,"statfs":{"total":95995035648,"available":94991253504,"internally_reserved":0,"allocated":906780672,"data_stored":906072849,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":96993280},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:27:28 2023","interfaces":[{"interface":"back","average":{"1min":1.083,"5min":1.331,"15min":0.82899999999999996},"min":{"1min":0.28499999999999998,"5min":0.221,"15min":0.221},"max":{"1min":6.5529999999999999,"5min":8.1940000000000008,"15min":8.1940000000000008},"last":0.60799999999999998},{"interface":"front","average":{"1min":1.6499999999999999,"5min":1.7,"15min":0.94199999999999995},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":8.6189999999999998,"5min":12.512,"15min":12.512},"last":0.72999999999999998}]},{"osd":1,"last update":"Sat Sep 23 20:27:02 2023","interfaces":[{"interface":"back","average":{"1min":2.6379999999999999,"5min":1.8839999999999999,"15min":1.0109999999999999},"min":{"1min":0.28899999999999998,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":8.3520000000000003,"5min":18.001000000000001,"15min":18.001000000000001},"last":0.51900000000000002},{"interface":"front","average":{"1min":2.391,"5min":1.569,"15min":0.88800000000000001},"min":{"1min":0.28699999999999998,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":8.532,"5min":13.91,"15min":13.91},"last":0.63500000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:27:28 2023","interfaces":[{"interface":"back","average":{"1min":1.4419999999999999,"5min":1.5549999999999999,"15min":0.91500000000000004},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":7.1479999999999997,"5min":13.856,"15min":13.856},"last":1.288},{"interface":"front","average":{"1min":1.327,"5min":1.4279999999999999,"15min":0.88200000000000001},"min":{"1min":0.35499999999999998,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":7.1879999999999997,"5min":7.1879999999999997,"15min":7.1879999999999997},"last":1.6759999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:27:02 2023","interfaces":[{"interface":"back","average":{"1min":2.105,"5min":1.833,"15min":1.0189999999999999},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":8.2989999999999995,"5min":9.4239999999999995,"15min":9.4239999999999995},"last":2.4809999999999999},{"interface":"front","average":{"1min":2.754,"5min":1.8280000000000001,"15min":1.0600000000000001},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":8.3770000000000007,"5min":8.6050000000000004,"15min":8.6050000000000004},"last":0.70199999999999996}]},{"osd":4,"last update":"Sat Sep 23 20:26:48 2023","interfaces":[{"interface":"back","average":{"1min":1.5049999999999999,"5min":1.2669999999999999,"15min":0.72999999999999998},"min":{"1min":0.27600000000000002,"5min":0.27200000000000002,"15min":0.26200000000000001},"max":{"1min":10.173,"5min":10.173,"15min":10.173},"last":0.38100000000000001},{"interface":"front","average":{"1min":1.962,"5min":1.3540000000000001,"15min":0.78000000000000003},"min":{"1min":0.312,"5min":0.29799999999999999,"15min":0.28499999999999998},"max":{"1min":8.3249999999999993,"5min":8.3249999999999993,"15min":8.3249999999999993},"last":0.49099999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:26:48 2023","interfaces":[{"interface":"back","average":{"1min":1.202,"5min":1.4119999999999999,"15min":0.80300000000000005},"min":{"1min":0.31,"5min":0.31,"15min":0.29799999999999999},"max":{"1min":4.4580000000000002,"5min":14.085000000000001,"15min":14.085000000000001},"last":0.34999999999999998},{"interface":"front","average":{"1min":2.831,"5min":1.8660000000000001,"15min":0.97299999999999998},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.29499999999999998},"max":{"1min":18.192,"5min":18.192,"15min":18.192},"last":0.39000000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:27:02 2023","interfaces":[{"interface":"back","average":{"1min":1.946,"5min":1.538,"15min":0.877},"min":{"1min":0.36899999999999999,"5min":0.33500000000000002,"15min":0.32800000000000001},"max":{"1min":8.0350000000000001,"5min":10.217000000000001,"15min":10.217000000000001},"last":0.40899999999999997},{"interface":"front","average":{"1min":1.6459999999999999,"5min":1.6020000000000001,"15min":0.91400000000000003},"min":{"1min":0.36499999999999999,"5min":0.34999999999999998,"15min":0.315},"max":{"1min":8.407,"5min":9.7360000000000007,"15min":9.7360000000000007},"last":1.109}]}]},{"osd":1,"up_from":99,"seq":425201762390,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1054728,"kb_used_data":888512,"kb_used_omap":6,"kb_used_meta":166201,"kb_avail":92690424,"statfs":{"total":95995035648,"available":94914994176,"internally_reserved":0,"allocated":909836288,"data_stored":908680874,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6696,"internal_metadata":170190296},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:26:54 2023","interfaces":[{"interface":"back","average":{"1min":2.6600000000000001,"5min":2.052,"15min":0.93500000000000005},"min":{"1min":0.215,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":14.384,"5min":40.113,"15min":40.113},"last":2.879},{"interface":"front","average":{"1min":3.1869999999999998,"5min":1.8120000000000001,"15min":0.87},"min":{"1min":0.23699999999999999,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":22.210000000000001,"5min":42.994999999999997,"15min":42.994999999999997},"last":2.2250000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:27:30 2023","interfaces":[{"interface":"back","average":{"1min":2.0299999999999998,"5min":2.1320000000000001,"15min":0.96499999999999997},"min":{"1min":0.30099999999999999,"5min":0.253,"15min":0.253},"max":{"1min":17.940000000000001,"5min":39.962000000000003,"15min":39.962000000000003},"last":2.488},{"interface":"front","average":{"1min":2.3730000000000002,"5min":2.1579999999999999,"15min":1.087},"min":{"1min":0.26300000000000001,"5min":0.246,"15min":0.246},"max":{"1min":15.090999999999999,"5min":47.442,"15min":47.442},"last":3.323}]},{"osd":3,"last update":"Sat Sep 23 20:27:00 2023","interfaces":[{"interface":"back","average":{"1min":1.9099999999999999,"5min":1.4039999999999999,"15min":0.83399999999999996},"min":{"1min":0.309,"5min":0.246,"15min":0.246},"max":{"1min":9.0410000000000004,"5min":9.0410000000000004,"15min":9.0410000000000004},"last":3.0270000000000001},{"interface":"front","average":{"1min":2.4319999999999999,"5min":1.4710000000000001,"15min":0.76200000000000001},"min":{"1min":0.29699999999999999,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":15.147,"5min":15.147,"15min":15.147},"last":2.339}]},{"osd":4,"last update":"Sat Sep 23 20:26:54 2023","interfaces":[{"interface":"back","average":{"1min":2.5790000000000002,"5min":1.5900000000000001,"15min":0.94099999999999995},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":15.353,"5min":15.353,"15min":15.353},"last":2.7839999999999998},{"interface":"front","average":{"1min":1.7609999999999999,"5min":1.536,"15min":0.93500000000000005},"min":{"1min":0.32400000000000001,"5min":0.312,"15min":0.312},"max":{"1min":6.2270000000000003,"5min":8.7899999999999991,"15min":8.7899999999999991},"last":2.5190000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:26:54 2023","interfaces":[{"interface":"back","average":{"1min":2.6360000000000001,"5min":1.6220000000000001,"15min":0.96799999999999997},"min":{"1min":0.33000000000000002,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":7.2000000000000002,"5min":12.590999999999999,"15min":12.590999999999999},"last":2.2719999999999998},{"interface":"front","average":{"1min":2.2530000000000001,"5min":2.137,"15min":1.163},"min":{"1min":0.38300000000000001,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":6.3140000000000001,"5min":39.579000000000001,"15min":39.579000000000001},"last":2.7469999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:26:54 2023","interfaces":[{"interface":"back","average":{"1min":2.9630000000000001,"5min":1.996,"15min":1.1100000000000001},"min":{"1min":0.42999999999999999,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":7.226,"5min":34.409999999999997,"15min":34.409999999999997},"last":2.3069999999999999},{"interface":"front","average":{"1min":2.964,"5min":2.2269999999999999,"15min":1.167},"min":{"1min":0.38100000000000001,"5min":0.377,"15min":0.377},"max":{"1min":15.313000000000001,"5min":35.554000000000002,"15min":35.554000000000002},"last":2.8410000000000002}]},{"osd":7,"last update":"Sat Sep 23 20:26:54 2023","interfaces":[{"interface":"back","average":{"1min":2.8719999999999999,"5min":1.8480000000000001,"15min":1.05},"min":{"1min":0.42499999999999999,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":9.7439999999999998,"5min":9.7439999999999998,"15min":9.7439999999999998},"last":2.8119999999999998},{"interface":"front","average":{"1min":2.3180000000000001,"5min":1.7729999999999999,"15min":1.0469999999999999},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":6.2519999999999998,"5min":9.2279999999999998,"15min":9.2279999999999998},"last":2.5510000000000002}]}]},{"osd":0,"up_from":94,"seq":403726925916,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":925920,"kb_used_data":723160,"kb_used_omap":7,"kb_used_meta":202744,"kb_avail":92819232,"statfs":{"total":95995035648,"available":95046893568,"internally_reserved":0,"allocated":740515840,"data_stored":739459060,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":207609873},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:26:56 2023","interfaces":[{"interface":"back","average":{"1min":3.0779999999999998,"5min":1.3400000000000001,"15min":0.68600000000000005},"min":{"1min":0.223,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":24.247,"5min":24.247,"15min":24.247},"last":2.823},{"interface":"front","average":{"1min":2.9950000000000001,"5min":1.3160000000000001,"15min":0.69599999999999995},"min":{"1min":0.28999999999999998,"5min":0.222,"15min":0.222},"max":{"1min":15.994999999999999,"5min":15.994999999999999,"15min":15.994999999999999},"last":0.308}]},{"osd":2,"last update":"Sat Sep 23 20:27:24 2023","interfaces":[{"interface":"back","average":{"1min":1.2190000000000001,"5min":1.9450000000000001,"15min":1.046},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23300000000000001},"max":{"1min":7.5199999999999996,"5min":23.832999999999998,"15min":23.832999999999998},"last":1.1599999999999999},{"interface":"front","average":{"1min":2.2490000000000001,"5min":1.607,"15min":0.86599999999999999},"min":{"1min":0.19700000000000001,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":15.948,"5min":15.948,"15min":15.948},"last":0.36199999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:26:56 2023","interfaces":[{"interface":"back","average":{"1min":3.3109999999999999,"5min":1.7070000000000001,"15min":0.96699999999999997},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":23.748999999999999,"5min":23.748999999999999,"15min":23.748999999999999},"last":1.0429999999999999},{"interface":"front","average":{"1min":3.9809999999999999,"5min":1.5189999999999999,"15min":0.78700000000000003},"min":{"1min":0.315,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":24.274999999999999,"5min":24.274999999999999,"15min":24.274999999999999},"last":2.859}]},{"osd":4,"last update":"Sat Sep 23 20:27:24 2023","interfaces":[{"interface":"back","average":{"1min":1.55,"5min":2.0960000000000001,"15min":1.1100000000000001},"min":{"1min":0.36899999999999999,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":6.5899999999999999,"5min":23.876999999999999,"15min":23.876999999999999},"last":4.3730000000000002},{"interface":"front","average":{"1min":1.53,"5min":1.772,"15min":0.999},"min":{"1min":0.39500000000000002,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":6.0300000000000002,"5min":23.681999999999999,"15min":23.681999999999999},"last":2.903}]},{"osd":5,"last update":"Sat Sep 23 20:27:24 2023","interfaces":[{"interface":"back","average":{"1min":1.8640000000000001,"5min":2.4729999999999999,"15min":1.262},"min":{"1min":0.47399999999999998,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":9.3369999999999997,"5min":24.198,"15min":24.198},"last":4.109},{"interface":"front","average":{"1min":1.1539999999999999,"5min":1.8500000000000001,"15min":1.069},"min":{"1min":0.41799999999999998,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":6.1749999999999998,"5min":11.167999999999999,"15min":11.167999999999999},"last":1.8129999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:27:24 2023","interfaces":[{"interface":"back","average":{"1min":1.6799999999999999,"5min":1.7749999999999999,"15min":1.0269999999999999},"min":{"1min":0.42299999999999999,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":9.2750000000000004,"5min":24.408000000000001,"15min":24.408000000000001},"last":3.6259999999999999},{"interface":"front","average":{"1min":1.1259999999999999,"5min":1.6140000000000001,"15min":0.98099999999999998},"min":{"1min":0.442,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":6.1319999999999997,"5min":25.271000000000001,"15min":25.271000000000001},"last":1.9339999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:27:24 2023","interfaces":[{"interface":"back","average":{"1min":1.1619999999999999,"5min":1.8180000000000001,"15min":1.0569999999999999},"min":{"1min":0.47799999999999998,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":6.2039999999999997,"5min":25.359000000000002,"15min":25.359000000000002},"last":0.51100000000000001},{"interface":"front","average":{"1min":1.5169999999999999,"5min":2.3119999999999998,"15min":1.2270000000000001},"min":{"1min":0.45600000000000002,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":9.8239999999999998,"5min":23.905999999999999,"15min":23.905999999999999},"last":0.46100000000000002}]}]},{"osd":2,"up_from":104,"seq":446676598863,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1457172,"kb_used_data":1295116,"kb_used_omap":11,"kb_used_meta":162036,"kb_avail":92287980,"statfs":{"total":95995035648,"available":94502891520,"internally_reserved":0,"allocated":1326198784,"data_stored":1325150343,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11878,"internal_metadata":165925274},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,2,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:27:30 2023","interfaces":[{"interface":"back","average":{"1min":0.85099999999999998,"5min":1.8009999999999999,"15min":0.85099999999999998},"min":{"1min":0.29399999999999998,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":3.302,"5min":12.811,"15min":12.811},"last":0.47399999999999998},{"interface":"front","average":{"1min":0.76400000000000001,"5min":1.3440000000000001,"15min":0.71199999999999997},"min":{"1min":0.28399999999999997,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":4.0279999999999996,"5min":15.746,"15min":15.746},"last":0.375}]},{"osd":1,"last update":"Sat Sep 23 20:27:30 2023","interfaces":[{"interface":"back","average":{"1min":0.93799999999999994,"5min":1.633,"15min":0.80700000000000005},"min":{"1min":0.29799999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":3.3410000000000002,"5min":14.135999999999999,"15min":14.135999999999999},"last":3.2189999999999999},{"interface":"front","average":{"1min":1.863,"5min":1.522,"15min":0.76200000000000001},"min":{"1min":0.27500000000000002,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":14.201000000000001,"5min":14.201000000000001,"15min":14.201000000000001},"last":0.66000000000000003}]},{"osd":3,"last update":"Sat Sep 23 20:26:55 2023","interfaces":[{"interface":"back","average":{"1min":2.246,"5min":1.349,"15min":0.69099999999999995},"min":{"1min":0.26500000000000001,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":10.916,"5min":15.319000000000001,"15min":15.319000000000001},"last":3.048},{"interface":"front","average":{"1min":2.294,"5min":1.552,"15min":0.78200000000000003},"min":{"1min":0.25800000000000001,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":12.494999999999999,"5min":12.494999999999999,"15min":12.494999999999999},"last":9.0129999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:27:30 2023","interfaces":[{"interface":"back","average":{"1min":1.452,"5min":1.407,"15min":0.85799999999999998},"min":{"1min":0.38100000000000001,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":5.8799999999999999,"5min":8.2300000000000004,"15min":8.2300000000000004},"last":0.70699999999999996},{"interface":"front","average":{"1min":0.64300000000000002,"5min":1.409,"15min":0.85899999999999999},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":1.288,"5min":8.6530000000000005,"15min":8.6530000000000005},"last":3.2429999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:27:30 2023","interfaces":[{"interface":"back","average":{"1min":0.98199999999999998,"5min":1.6419999999999999,"15min":0.98499999999999999},"min":{"1min":0.41099999999999998,"5min":0.309,"15min":0.309},"max":{"1min":4.0810000000000004,"5min":11.237,"15min":11.237},"last":0.77100000000000002},{"interface":"front","average":{"1min":0.628,"5min":1.679,"15min":0.997},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":1.129,"5min":11.266999999999999,"15min":11.266999999999999},"last":3.2850000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:27:30 2023","interfaces":[{"interface":"back","average":{"1min":0.90100000000000002,"5min":1.343,"15min":0.88500000000000001},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":5.1139999999999999,"5min":6.7939999999999996,"15min":6.7939999999999996},"last":3.8100000000000001},{"interface":"front","average":{"1min":1.3919999999999999,"5min":1.7290000000000001,"15min":1.0249999999999999},"min":{"1min":0.43099999999999999,"5min":0.377,"15min":0.377},"max":{"1min":6.0099999999999998,"5min":9.6600000000000001,"15min":9.6600000000000001},"last":3.3250000000000002}]},{"osd":7,"last update":"Sat Sep 23 20:27:30 2023","interfaces":[{"interface":"back","average":{"1min":1.3999999999999999,"5min":1.8620000000000001,"15min":1.052},"min":{"1min":0.48199999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":5.9649999999999999,"5min":15.343999999999999,"15min":15.343999999999999},"last":4.2089999999999996},{"interface":"front","average":{"1min":1.2010000000000001,"5min":1.6930000000000001,"15min":1.002},"min":{"1min":0.51100000000000001,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":4.1459999999999999,"5min":10.064,"15min":10.064},"last":2.556}]}]},{"osd":3,"up_from":109,"seq":468151435337,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1846752,"kb_used_data":1662552,"kb_used_omap":4,"kb_used_meta":184187,"kb_avail":91898400,"statfs":{"total":95995035648,"available":94103961600,"internally_reserved":0,"allocated":1702453248,"data_stored":1701392242,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4514,"internal_metadata":188608094},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,1,6],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:26:56 2023","interfaces":[{"interface":"back","average":{"1min":3.2530000000000001,"5min":2.319,"15min":1.0069999999999999},"min":{"1min":0.28999999999999998,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":15.589,"5min":67.305999999999997,"15min":67.305999999999997},"last":6.5899999999999999},{"interface":"front","average":{"1min":1.48,"5min":2.0339999999999998,"15min":0.93400000000000005},"min":{"1min":0.29699999999999999,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":9.0920000000000005,"5min":67.403999999999996,"15min":67.403999999999996},"last":7.0999999999999996}]},{"osd":1,"last update":"Sat Sep 23 20:26:56 2023","interfaces":[{"interface":"back","average":{"1min":3.0230000000000001,"5min":1.5069999999999999,"15min":0.76700000000000002},"min":{"1min":0.25,"5min":0.214,"15min":0.214},"max":{"1min":19.675999999999998,"5min":19.675999999999998,"15min":19.675999999999998},"last":1.9359999999999999},{"interface":"front","average":{"1min":2.4590000000000001,"5min":1.375,"15min":0.755},"min":{"1min":0.33200000000000002,"5min":0.245,"15min":0.245},"max":{"1min":11.644,"5min":11.644,"15min":11.644},"last":2.6949999999999998}]},{"osd":2,"last update":"Sat Sep 23 20:26:56 2023","interfaces":[{"interface":"back","average":{"1min":3.4369999999999998,"5min":1.7789999999999999,"15min":0.82799999999999996},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":24.045000000000002,"5min":24.045000000000002,"15min":24.045000000000002},"last":7.069},{"interface":"front","average":{"1min":3.4119999999999999,"5min":1.8080000000000001,"15min":0.91200000000000003},"min":{"1min":0.33700000000000002,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":12.427,"5min":17.684000000000001,"15min":17.684000000000001},"last":2.5019999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:26:56 2023","interfaces":[{"interface":"back","average":{"1min":2.875,"5min":1.4590000000000001,"15min":0.88900000000000001},"min":{"1min":0.33100000000000002,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":11.880000000000001,"5min":11.880000000000001,"15min":11.880000000000001},"last":1.2629999999999999},{"interface":"front","average":{"1min":3.698,"5min":1.8180000000000001,"15min":1.016},"min":{"1min":0.378,"5min":0.371,"15min":0.371},"max":{"1min":21.673999999999999,"5min":21.673999999999999,"15min":21.673999999999999},"last":2.0129999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:26:56 2023","interfaces":[{"interface":"back","average":{"1min":3.117,"5min":1.8560000000000001,"15min":1.0229999999999999},"min":{"1min":0.375,"5min":0.311,"15min":0.311},"max":{"1min":11.839,"5min":11.839,"15min":11.839},"last":1.2210000000000001},{"interface":"front","average":{"1min":1.825,"5min":1.2949999999999999,"15min":0.85199999999999998},"min":{"1min":0.40000000000000002,"5min":0.313,"15min":0.313},"max":{"1min":8.5570000000000004,"5min":8.5570000000000004,"15min":8.5570000000000004},"last":6.875}]},{"osd":6,"last update":"Sat Sep 23 20:26:56 2023","interfaces":[{"interface":"back","average":{"1min":2.0609999999999999,"5min":1.5900000000000001,"15min":0.95499999999999996},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":8.6059999999999999,"5min":8.6059999999999999,"15min":8.6059999999999999},"last":6.9180000000000001},{"interface":"front","average":{"1min":2.7469999999999999,"5min":1.5740000000000001,"15min":0.97199999999999998},"min":{"1min":0.40600000000000003,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":9.3179999999999996,"5min":9.3179999999999996,"15min":9.3179999999999996},"last":1.9870000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:26:56 2023","interfaces":[{"interface":"back","average":{"1min":3.0030000000000001,"5min":1.5860000000000001,"15min":0.997},"min":{"1min":0.40999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":9.2750000000000004,"5min":9.2750000000000004,"15min":9.2750000000000004},"last":2.0419999999999998},{"interface":"front","average":{"1min":3.6680000000000001,"5min":2.0209999999999999,"15min":1.101},"min":{"1min":0.40699999999999997,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":12.468999999999999,"5min":12.468999999999999,"15min":12.468999999999999},"last":1.29}]}]},{"osd":4,"up_from":30,"seq":128849019090,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":928864,"kb_used_data":787352,"kb_used_omap":0,"kb_used_meta":141504,"kb_avail":92816288,"statfs":{"total":95995035648,"available":95043878912,"internally_reserved":0,"allocated":806248448,"data_stored":805540359,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":144900096},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:27:24 2023","interfaces":[{"interface":"back","average":{"1min":2.028,"5min":1.6950000000000001,"15min":0.94699999999999995},"min":{"1min":0.23699999999999999,"5min":0.222,"15min":0.222},"max":{"1min":6.5780000000000003,"5min":20.411999999999999,"15min":20.411999999999999},"last":0.57699999999999996},{"interface":"front","average":{"1min":1.7170000000000001,"5min":1.7649999999999999,"15min":1.016},"min":{"1min":0.307,"5min":0.253,"15min":0.253},"max":{"1min":6.5919999999999996,"5min":20.454999999999998,"15min":20.454999999999998},"last":0.89900000000000002}]},{"osd":1,"last update":"Sat Sep 23 20:27:00 2023","interfaces":[{"interface":"back","average":{"1min":2.883,"5min":1.5089999999999999,"15min":0.92000000000000004},"min":{"1min":0.24299999999999999,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":9.6660000000000004,"5min":9.6660000000000004,"15min":9.6660000000000004},"last":0.621},{"interface":"front","average":{"1min":2.4990000000000001,"5min":1.5640000000000001,"15min":0.93799999999999994},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":12.821999999999999,"5min":12.821999999999999,"15min":12.821999999999999},"last":0.60299999999999998}]},{"osd":2,"last update":"Sat Sep 23 20:27:33 2023","interfaces":[{"interface":"back","average":{"1min":2.1179999999999999,"5min":1.8120000000000001,"15min":1.0089999999999999},"min":{"1min":0.28599999999999998,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":6.7279999999999998,"5min":21.462,"15min":21.462},"last":0.40200000000000002},{"interface":"front","average":{"1min":1.7529999999999999,"5min":2.0990000000000002,"15min":1.0920000000000001},"min":{"1min":0.29799999999999999,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":6.1799999999999997,"5min":22.593,"15min":22.593},"last":1.855}]},{"osd":3,"last update":"Sat Sep 23 20:27:00 2023","interfaces":[{"interface":"back","average":{"1min":2.516,"5min":1.7929999999999999,"15min":1.0249999999999999},"min":{"1min":0.32900000000000001,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":10.846,"5min":24.891999999999999,"15min":24.891999999999999},"last":1.8819999999999999},{"interface":"front","average":{"1min":2.464,"5min":1.831,"15min":1.0529999999999999},"min":{"1min":0.35499999999999998,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":9.2080000000000002,"5min":21.922000000000001,"15min":21.922000000000001},"last":0.371}]},{"osd":5,"last update":"Sat Sep 23 20:27:11 2023","interfaces":[{"interface":"back","average":{"1min":1.671,"5min":1.8859999999999999,"15min":0.95299999999999996},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":10.252000000000001,"5min":17.704000000000001,"15min":17.704000000000001},"last":1.7290000000000001},{"interface":"front","average":{"1min":1.964,"5min":1.643,"15min":0.88400000000000001},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":8.2110000000000003,"5min":15.329000000000001,"15min":15.329000000000001},"last":0.86599999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:27:35 2023","interfaces":[{"interface":"back","average":{"1min":1.21,"5min":2.0430000000000001,"15min":1.0329999999999999},"min":{"1min":0.38600000000000001,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":4.3259999999999996,"5min":41.371000000000002,"15min":41.371000000000002},"last":1.6499999999999999},{"interface":"front","average":{"1min":1.6679999999999999,"5min":1.8120000000000001,"15min":0.96299999999999997},"min":{"1min":0.372,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":7.7729999999999997,"5min":41.140999999999998,"15min":41.140999999999998},"last":0.34699999999999998}]},{"osd":7,"last update":"Sat Sep 23 20:26:48 2023","interfaces":[{"interface":"back","average":{"1min":4.3280000000000003,"5min":1.8859999999999999,"15min":0.997},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.33600000000000002},"max":{"1min":16.427,"5min":16.427,"15min":16.427},"last":0.92400000000000004},{"interface":"front","average":{"1min":3.2890000000000001,"5min":1.754,"15min":0.96599999999999997},"min":{"1min":0.36399999999999999,"5min":0.34300000000000003,"15min":0.29899999999999999},"max":{"1min":15.645,"5min":19.018999999999998,"15min":19.018999999999998},"last":2.3149999999999999}]}]},{"osd":5,"up_from":39,"seq":167503724751,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1017308,"kb_used_data":868308,"kb_used_omap":0,"kb_used_meta":148992,"kb_avail":92727844,"statfs":{"total":95995035648,"available":94953312256,"internally_reserved":0,"allocated":889147392,"data_stored":888440029,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":152567808},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:27:27 2023","interfaces":[{"interface":"back","average":{"1min":1.2310000000000001,"5min":1.671,"15min":0.95299999999999996},"min":{"1min":0.28399999999999997,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":5.915,"5min":32.475000000000001,"15min":32.475000000000001},"last":1.397},{"interface":"front","average":{"1min":0.93300000000000005,"5min":1.671,"15min":0.95599999999999996},"min":{"1min":0.248,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":2.3359999999999999,"5min":32.457000000000001,"15min":32.457000000000001},"last":1.379}]},{"osd":1,"last update":"Sat Sep 23 20:27:01 2023","interfaces":[{"interface":"back","average":{"1min":2.3980000000000001,"5min":1.819,"15min":0.96899999999999997},"min":{"1min":0.253,"5min":0.245,"15min":0.245},"max":{"1min":9.7639999999999993,"5min":32.243000000000002,"15min":32.243000000000002},"last":1.0620000000000001},{"interface":"front","average":{"1min":1.103,"5min":1.3480000000000001,"15min":0.81399999999999995},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":5.7389999999999999,"5min":32.231000000000002,"15min":32.231000000000002},"last":1.698}]},{"osd":2,"last update":"Sat Sep 23 20:27:27 2023","interfaces":[{"interface":"back","average":{"1min":1.8109999999999999,"5min":2.1440000000000001,"15min":1.081},"min":{"1min":0.311,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":12.052,"5min":34.514000000000003,"15min":34.514000000000003},"last":0.46800000000000003},{"interface":"front","average":{"1min":1.194,"5min":1.859,"15min":0.99199999999999999},"min":{"1min":0.317,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":4.8470000000000004,"5min":32.515000000000001,"15min":32.515000000000001},"last":0.35999999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:27:01 2023","interfaces":[{"interface":"back","average":{"1min":2.5129999999999999,"5min":1.5900000000000001,"15min":0.90200000000000002},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":12.846,"5min":12.846,"15min":12.846},"last":1.294},{"interface":"front","average":{"1min":1.764,"5min":1.554,"15min":0.92800000000000005},"min":{"1min":0.35599999999999998,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":6.0049999999999999,"5min":11.379,"15min":11.379},"last":0.44600000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:27:18 2023","interfaces":[{"interface":"back","average":{"1min":2.8380000000000001,"5min":1.9019999999999999,"15min":0.94399999999999995},"min":{"1min":0.33600000000000002,"5min":0.26700000000000002,"15min":0.26400000000000001},"max":{"1min":12.984,"5min":14.218999999999999,"15min":14.218999999999999},"last":2.633},{"interface":"front","average":{"1min":1.5609999999999999,"5min":1.536,"15min":0.84499999999999997},"min":{"1min":0.38800000000000001,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":5.9779999999999998,"5min":11.664,"15min":11.664},"last":0.41799999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:27:32 2023","interfaces":[{"interface":"back","average":{"1min":1.732,"5min":1.8089999999999999,"15min":0.94099999999999995},"min":{"1min":0.32400000000000001,"5min":0.30499999999999999,"15min":0.29799999999999999},"max":{"1min":12.890000000000001,"5min":12.890000000000001,"15min":12.890000000000001},"last":5.9450000000000003},{"interface":"front","average":{"1min":1.5049999999999999,"5min":1.6699999999999999,"15min":0.94399999999999995},"min":{"1min":0.42599999999999999,"5min":0.35999999999999999,"15min":0.34499999999999997},"max":{"1min":6.0339999999999998,"5min":11.456,"15min":11.456},"last":5.8920000000000003}]},{"osd":7,"last update":"Sat Sep 23 20:27:01 2023","interfaces":[{"interface":"back","average":{"1min":2.2890000000000001,"5min":1.585,"15min":0.88900000000000001},"min":{"1min":0.40100000000000002,"5min":0.33900000000000002,"15min":0.33800000000000002},"max":{"1min":6.0730000000000004,"5min":11.430999999999999,"15min":11.430999999999999},"last":1.155},{"interface":"front","average":{"1min":1.216,"5min":1.1930000000000001,"15min":0.76400000000000001},"min":{"1min":0.316,"5min":0.316,"15min":0.315},"max":{"1min":5.835,"5min":12.368,"15min":12.368},"last":1.9650000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4722,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1360,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6696,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7156,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3154,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":744423424,"data_stored":744423424,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":922550272,"data_stored":922550272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1342439424,"data_stored":1342439424,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1740111872,"data_stored":1740111872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":806158336,"data_stored":806158336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":802291712,"data_stored":802291712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":923729920,"data_stored":923729920,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1114046464,"data_stored":1114046464,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:27:46.562 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-09-23T20:27:46.562 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd pool set .mgr pg_num 21 2023-09-23T20:27:46.563 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:46 smithi149 ceph-mon[170394]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:27:46.563 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:46 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2958558223' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:27:46.563 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:46 smithi149 ceph-mon[170394]: osdmap e224: 8 total, 8 up, 8 in 2023-09-23T20:27:46.563 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:46 smithi149 ceph-mon[170394]: failed to encode map e224 with expected crc 2023-09-23T20:27:46.564 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:46 smithi149 ceph-mon[170394]: failed to encode map e224 with expected crc 2023-09-23T20:27:46.564 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 24 16 47273 47257 123.01 89.9375 0.0124746 0.00812388 2023-09-23T20:27:46.888 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:27:46.893 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:46.894 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:27:46.995 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:27:47.078 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:47.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:47 smithi018 ceph-mon[179647]: failed to encode map e224 with expected crc 2023-09-23T20:27:47.176 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:47 smithi018 ceph-mon[179647]: failed to encode map e224 with expected crc 2023-09-23T20:27:47.177 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:47 smithi018 ceph-mon[179647]: failed to encode map e224 with expected crc 2023-09-23T20:27:47.177 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:47 smithi018 ceph-mon[179647]: pgmap v490: 99 pgs: 99 active+clean; 2.6 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 104 MiB/s wr, 1.65k op/s 2023-09-23T20:27:47.177 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:47 smithi018 ceph-mon[179647]: from='client.35322 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:27:47.177 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:47 smithi018 ceph-mon[184144]: failed to encode map e224 with expected crc 2023-09-23T20:27:47.177 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:47 smithi018 ceph-mon[184144]: failed to encode map e224 with expected crc 2023-09-23T20:27:47.177 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:47 smithi018 ceph-mon[184144]: failed to encode map e224 with expected crc 2023-09-23T20:27:47.177 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:47 smithi018 ceph-mon[184144]: pgmap v490: 99 pgs: 99 active+clean; 2.6 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 104 MiB/s wr, 1.65k op/s 2023-09-23T20:27:47.177 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:47 smithi018 ceph-mon[184144]: from='client.35322 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:27:47.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:47 smithi149 ceph-mon[170394]: failed to encode map e224 with expected crc 2023-09-23T20:27:47.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:47 smithi149 ceph-mon[170394]: failed to encode map e224 with expected crc 2023-09-23T20:27:47.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:47 smithi149 ceph-mon[170394]: failed to encode map e224 with expected crc 2023-09-23T20:27:47.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:47 smithi149 ceph-mon[170394]: pgmap v490: 99 pgs: 99 active+clean; 2.6 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 104 MiB/s wr, 1.65k op/s 2023-09-23T20:27:47.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:47 smithi149 ceph-mon[170394]: from='client.35322 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:27:47.561 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 25 16 48610 48594 121.432 83.5625 0.0170348 0.00822994 2023-09-23T20:27:47.788 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:27:47.797 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:27:47.800 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:47.800 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:27:47.873 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:27:47.902 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:27:47.912 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:27:47.928 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T20:27:47.952 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:27:47.992 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:48.009 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:27:48.056 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:27:48.271 INFO:teuthology.orchestra.run.smithi149.stdout:{ 2023-09-23T20:27:48.271 INFO:teuthology.orchestra.run.smithi149.stdout: "ops": [], 2023-09-23T20:27:48.271 INFO:teuthology.orchestra.run.smithi149.stdout: "num_ops": 0 2023-09-23T20:27:48.271 INFO:teuthology.orchestra.run.smithi149.stdout:} 2023-09-23T20:27:48.272 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:48.273 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:27:48.273 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:27:48.273 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:27:48.274 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:27:48.274 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:27:48.274 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:27:48.274 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:27:48.274 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:27:48.274 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:27:48.275 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:27:48.275 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:27:48.275 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:27:48.275 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:27:48.314 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:48 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2617673032' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:27:48.314 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:48 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2617673032' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:27:48.374 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:27:48.445 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:48.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:48 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2617673032' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:27:48.561 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 26 16 49766 49750 119.541 72.25 0.0109764 0.00836031 2023-09-23T20:27:49.169 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T20:27:49.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:49 smithi018 ceph-mon[184144]: pgmap v491: 99 pgs: 99 active+clean; 2.8 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 105 MiB/s wr, 1.68k op/s 2023-09-23T20:27:49.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:49 smithi018 ceph-mon[179647]: pgmap v491: 99 pgs: 99 active+clean; 2.8 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 105 MiB/s wr, 1.68k op/s 2023-09-23T20:27:49.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:49 smithi149 ceph-mon[170394]: pgmap v491: 99 pgs: 99 active+clean; 2.8 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 105 MiB/s wr, 1.68k op/s 2023-09-23T20:27:49.562 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 27 16 51117 51101 118.24 84.4375 0.0184688 0.0084519 2023-09-23T20:27:49.981 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:27:50.561 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 28 16 52624 52608 117.381 94.1875 0.0155522 0.00851403 2023-09-23T20:27:50.628 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:27:50.633 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:50.633 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:27:50.734 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:27:50.794 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:51.237 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:27:51.242 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:51.243 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:27:51.343 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:27:51.352 INFO:teuthology.orchestra.run.smithi149.stdout:{ 2023-09-23T20:27:51.352 INFO:teuthology.orchestra.run.smithi149.stdout: "ops": [], 2023-09-23T20:27:51.352 INFO:teuthology.orchestra.run.smithi149.stdout: "complaint_time": 30, 2023-09-23T20:27:51.352 INFO:teuthology.orchestra.run.smithi149.stdout: "num_blocked_ops": 0 2023-09-23T20:27:51.352 INFO:teuthology.orchestra.run.smithi149.stdout:} 2023-09-23T20:27:51.381 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:51 smithi149 ceph-mon[170394]: pgmap v492: 99 pgs: 99 active+clean; 2.8 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 97 MiB/s wr, 1.55k op/s 2023-09-23T20:27:51.389 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:51.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:51 smithi018 ceph-mon[179647]: pgmap v492: 99 pgs: 99 active+clean; 2.8 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 97 MiB/s wr, 1.55k op/s 2023-09-23T20:27:51.401 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:51 smithi018 ceph-mon[184144]: pgmap v492: 99 pgs: 99 active+clean; 2.8 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 97 MiB/s wr, 1.55k op/s 2023-09-23T20:27:51.552 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:27:51.561 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 29 16 53933 53917 116.155 81.8125 0.0119894 0.00860398 2023-09-23T20:27:51.701 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:27:51.714 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:51.714 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:27:51.716 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T20:27:51.815 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:27:51.894 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:51.954 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:27:51.958 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:51.959 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:27:52.059 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:27:52.115 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:52.176 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:27:52.181 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:52.181 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:27:52.282 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:27:52.375 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:52.451 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:27:52.544 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:27:52.547 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:52.548 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:27:52.566 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 30 16 55182 55166 114.869 78.0625 0.0106123 0.00870068 2023-09-23T20:27:52.649 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:27:52.720 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:52.827 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:27:52.835 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:52.836 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:27:52.936 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:27:52.985 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:53.180 INFO:teuthology.orchestra.run.smithi018.stderr:set pool 1 pg_num to 21 2023-09-23T20:27:53.184 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:27:53.242 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:53 smithi018 ceph-mon[184144]: pgmap v493: 99 pgs: 99 active+clean; 3.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s 2023-09-23T20:27:53.242 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:53 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2026770170' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"}]: dispatch 2023-09-23T20:27:53.243 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:53 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"}]: dispatch 2023-09-23T20:27:53.243 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:53 smithi018 ceph-mon[179647]: pgmap v493: 99 pgs: 99 active+clean; 3.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s 2023-09-23T20:27:53.243 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:53 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2026770170' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"}]: dispatch 2023-09-23T20:27:53.243 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:53 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"}]: dispatch 2023-09-23T20:27:53.275 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:53 smithi149 ceph-mon[170394]: pgmap v493: 99 pgs: 99 active+clean; 3.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s 2023-09-23T20:27:53.275 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:53 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2026770170' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"}]: dispatch 2023-09-23T20:27:53.276 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:53 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"}]: dispatch 2023-09-23T20:27:53.315 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:27:53.319 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:53.319 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:27:53.420 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:27:53.481 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:53.556 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:27:53.561 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:53.561 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:27:53.567 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 31 16 56374 56358 113.565 74.5 0.0219114 0.0087995 2023-09-23T20:27:53.662 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:27:53.723 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:53.834 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:27:53.845 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:53.845 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:27:53.945 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:27:53.965 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:27:53.971 INFO:teuthology.orchestra.run.smithi149.stdout:{ 2023-09-23T20:27:53.972 INFO:teuthology.orchestra.run.smithi149.stdout: "size": 20, 2023-09-23T20:27:53.972 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 600, 2023-09-23T20:27:53.972 INFO:teuthology.orchestra.run.smithi149.stdout: "ops": [ 2023-09-23T20:27:53.972 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:53.972 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:28907 5.12 5:4860c7d8:::benchmark_data_smithi018_228451_object28906:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e118)", 2023-09-23T20:27:53.972 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:09.975817+0000", 2023-09-23T20:27:53.972 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 283.99279204099997, 2023-09-23T20:27:53.972 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1014516590000001, 2023-09-23T20:27:53.972 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:27:53.973 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:27:53.973 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:27:53.973 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:27:53.973 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:27:53.973 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 28907 2023-09-23T20:27:53.973 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:53.973 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:27:53.973 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:53.973 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:27:53.973 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:09.975817+0000", 2023-09-23T20:27:53.973 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:27:53.973 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:53.974 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:53.974 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:27:53.974 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:09.975817+0000", 2023-09-23T20:27:53.974 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:27:53.974 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:53.974 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:53.974 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:27:53.974 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:09.975813+0000", 2023-09-23T20:27:53.974 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999952 2023-09-23T20:27:53.974 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:53.974 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:53.974 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:27:53.975 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:09.975911+0000", 2023-09-23T20:27:53.975 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.7844999999999996e-05 2023-09-23T20:27:53.975 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:53.975 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:53.975 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:27:53.975 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:09.975954+0000", 2023-09-23T20:27:53.975 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.3825999999999997e-05 2023-09-23T20:27:53.975 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:53.975 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:53.975 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:27:53.975 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:09.975970+0000", 2023-09-23T20:27:53.975 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.5252999999999999e-05 2023-09-23T20:27:53.976 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:53.976 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:53.976 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:27:53.976 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:10.068105+0000", 2023-09-23T20:27:53.976 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.092135898999999993 2023-09-23T20:27:53.976 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:53.976 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:53.976 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:27:53.976 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:10.068116+0000", 2023-09-23T20:27:53.976 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0644e-05 2023-09-23T20:27:53.976 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:53.976 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:53.977 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:27:53.977 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.073358+0000", 2023-09-23T20:27:53.977 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.00524149 2023-09-23T20:27:53.977 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:53.977 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:53.977 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:27:53.977 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.073492+0000", 2023-09-23T20:27:53.977 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00013435199999999999 2023-09-23T20:27:53.977 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:53.977 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:53.977 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:27:53.977 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.073626+0000", 2023-09-23T20:27:53.978 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000133654 2023-09-23T20:27:53.978 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:53.978 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:53.978 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:27:53.978 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.074639+0000", 2023-09-23T20:27:53.978 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.001013634 2023-09-23T20:27:53.978 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:53.978 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:53.978 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:27:53.978 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.074705+0000", 2023-09-23T20:27:53.978 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.6208e-05 2023-09-23T20:27:53.978 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:53.978 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:53.979 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:27:53.979 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.077176+0000", 2023-09-23T20:27:53.979 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0024700550000000001 2023-09-23T20:27:53.979 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:53.979 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:53.979 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:27:53.979 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.077228+0000", 2023-09-23T20:27:53.979 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.2320000000000001e-05 2023-09-23T20:27:53.979 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:53.979 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:53.979 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:27:53.979 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.077269+0000", 2023-09-23T20:27:53.980 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.1258999999999999e-05 2023-09-23T20:27:53.980 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:27:53.980 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:27:53.980 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:27:53.980 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:53.980 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:53.980 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:39722 5.11 5.6eed25d1 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e122)", 2023-09-23T20:27:53.980 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:18.137113+0000", 2023-09-23T20:27:53.980 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 275.831496888, 2023-09-23T20:27:53.980 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.91218893999999995, 2023-09-23T20:27:53.980 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:27:53.980 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:27:53.981 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:27:53.981 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:27:53.981 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:27:53.981 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 39722 2023-09-23T20:27:53.981 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:53.981 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:27:53.981 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:53.981 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:27:53.981 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.137113+0000", 2023-09-23T20:27:53.981 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:27:53.981 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:53.981 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:53.982 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:27:53.982 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.137113+0000", 2023-09-23T20:27:53.982 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:27:53.982 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:53.983 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:53.983 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:27:53.983 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.137112+0000", 2023-09-23T20:27:53.983 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:27:53.983 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:53.984 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:53.984 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:27:53.984 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.137161+0000", 2023-09-23T20:27:53.984 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.9493999999999997e-05 2023-09-23T20:27:53.984 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:53.984 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:53.984 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:27:53.984 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.137181+0000", 2023-09-23T20:27:53.984 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.0123999999999999e-05 2023-09-23T20:27:53.984 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:53.984 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:53.984 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:27:53.985 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.137186+0000", 2023-09-23T20:27:53.985 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.9629999999999997e-06 2023-09-23T20:27:53.985 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:53.985 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:53.985 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:27:53.985 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.137243+0000", 2023-09-23T20:27:53.985 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.698e-05 2023-09-23T20:27:53.985 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:53.985 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:53.985 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:27:53.985 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.137256+0000", 2023-09-23T20:27:53.985 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.27e-05 2023-09-23T20:27:53.986 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:53.986 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:53.986 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:27:53.986 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:19.049292+0000", 2023-09-23T20:27:53.986 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.91203635400000005 2023-09-23T20:27:53.986 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:53.986 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:53.986 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:27:53.986 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:19.049302+0000", 2023-09-23T20:27:53.986 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.3160000000000003e-06 2023-09-23T20:27:53.986 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:27:53.986 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:27:53.987 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:27:53.987 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:53.987 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:53.987 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:39726 5.11 5.e6874e31 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e122)", 2023-09-23T20:27:53.987 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:18.137208+0000", 2023-09-23T20:27:53.987 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 275.83140164500003, 2023-09-23T20:27:53.987 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.91213475899999996, 2023-09-23T20:27:53.987 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:27:53.987 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:27:53.987 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:27:53.987 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:27:53.987 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:27:53.987 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 39726 2023-09-23T20:27:53.988 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:53.988 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:27:53.988 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:53.988 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:27:53.988 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.137208+0000", 2023-09-23T20:27:53.988 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:27:53.988 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:53.988 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:53.988 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:27:53.988 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.137208+0000", 2023-09-23T20:27:53.988 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:27:53.989 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:53.989 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:53.989 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:27:53.989 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.137207+0000", 2023-09-23T20:27:53.989 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:27:53.989 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:53.989 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:53.989 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:27:53.989 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.137242+0000", 2023-09-23T20:27:53.989 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.4764999999999997e-05 2023-09-23T20:27:53.989 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:53.989 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:53.990 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:27:53.990 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.137253+0000", 2023-09-23T20:27:53.990 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0495e-05 2023-09-23T20:27:53.990 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:53.990 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:53.990 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:27:53.990 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.137261+0000", 2023-09-23T20:27:53.990 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.9500000000000001e-06 2023-09-23T20:27:53.990 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:53.990 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:53.991 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:27:53.991 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.137292+0000", 2023-09-23T20:27:53.991 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.1035999999999999e-05 2023-09-23T20:27:53.991 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:53.991 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:53.991 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:27:53.991 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.137296+0000", 2023-09-23T20:27:53.991 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.4900000000000002e-06 2023-09-23T20:27:53.991 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:53.991 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:53.992 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:27:53.992 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:19.049335+0000", 2023-09-23T20:27:53.992 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.91203900500000001 2023-09-23T20:27:53.992 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:53.992 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:53.992 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:27:53.992 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:19.049343+0000", 2023-09-23T20:27:53.992 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.5290000000000003e-06 2023-09-23T20:27:53.992 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:27:53.992 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:27:53.993 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:27:53.993 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:53.993 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:53.993 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:39760 5.11 5.824f07d1 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e122)", 2023-09-23T20:27:53.993 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:18.156298+0000", 2023-09-23T20:27:53.993 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 275.81231142399997, 2023-09-23T20:27:53.993 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.89306234699999998, 2023-09-23T20:27:53.993 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:27:53.993 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:27:53.994 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:27:53.994 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:27:53.994 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:27:53.994 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 39760 2023-09-23T20:27:53.994 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:53.994 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:27:53.994 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:53.994 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:27:53.994 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.156298+0000", 2023-09-23T20:27:53.994 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:27:53.995 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:53.995 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:53.995 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:27:53.995 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.156298+0000", 2023-09-23T20:27:53.995 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:27:53.995 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:53.995 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:53.995 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:27:53.995 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.156296+0000", 2023-09-23T20:27:53.995 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999981 2023-09-23T20:27:53.995 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:53.995 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:53.996 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:27:53.996 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.156367+0000", 2023-09-23T20:27:53.996 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.0866999999999998e-05 2023-09-23T20:27:53.996 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:53.996 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:53.996 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:27:53.996 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.156392+0000", 2023-09-23T20:27:53.996 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.5219999999999999e-05 2023-09-23T20:27:53.996 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:53.996 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:53.997 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:27:53.997 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.156400+0000", 2023-09-23T20:27:53.997 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.3259999999999996e-06 2023-09-23T20:27:53.997 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:53.997 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:53.997 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:27:53.997 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.156461+0000", 2023-09-23T20:27:53.997 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.0371999999999997e-05 2023-09-23T20:27:53.997 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:53.997 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:53.998 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:27:53.998 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.156469+0000", 2023-09-23T20:27:53.998 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.4959999999999996e-06 2023-09-23T20:27:53.998 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:53.998 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:53.998 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:27:53.998 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:19.049355+0000", 2023-09-23T20:27:53.998 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.89288598100000005 2023-09-23T20:27:53.999 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:53.999 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:53.999 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:27:53.999 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:19.049360+0000", 2023-09-23T20:27:53.999 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.1449999999999999e-06 2023-09-23T20:27:53.999 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:27:53.999 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:27:53.999 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:27:54.000 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.000 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.000 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:39802 5.11 5.6adc9311 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e122)", 2023-09-23T20:27:54.000 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:18.185993+0000", 2023-09-23T20:27:54.000 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 275.782616136, 2023-09-23T20:27:54.000 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.86338408, 2023-09-23T20:27:54.000 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:27:54.000 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:27:54.000 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:27:54.001 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:27:54.001 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:27:54.001 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 39802 2023-09-23T20:27:54.001 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.001 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:27:54.001 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.001 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:27:54.001 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.185993+0000", 2023-09-23T20:27:54.001 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:27:54.001 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.002 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.002 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:27:54.002 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.185993+0000", 2023-09-23T20:27:54.002 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:27:54.002 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.002 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.002 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:27:54.002 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.185991+0000", 2023-09-23T20:27:54.002 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999981 2023-09-23T20:27:54.003 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.003 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.003 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:27:54.003 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.186111+0000", 2023-09-23T20:27:54.003 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000119867 2023-09-23T20:27:54.003 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.003 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.003 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:27:54.003 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.186134+0000", 2023-09-23T20:27:54.003 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.2836e-05 2023-09-23T20:27:54.003 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.004 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.004 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:27:54.004 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.186143+0000", 2023-09-23T20:27:54.004 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.2890000000000003e-06 2023-09-23T20:27:54.004 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.004 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.004 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:27:54.004 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.186204+0000", 2023-09-23T20:27:54.004 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.0875000000000002e-05 2023-09-23T20:27:54.004 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.004 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.004 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:27:54.005 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.186214+0000", 2023-09-23T20:27:54.005 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.3700000000000001e-06 2023-09-23T20:27:54.005 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.005 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.005 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:27:54.005 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:19.049371+0000", 2023-09-23T20:27:54.005 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.86315763099999998 2023-09-23T20:27:54.005 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.005 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.005 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:27:54.005 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:19.049377+0000", 2023-09-23T20:27:54.005 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.1469999999999998e-06 2023-09-23T20:27:54.006 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:27:54.006 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:27:54.006 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:27:54.006 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.006 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.006 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:39822 5.11 5.72f66af1 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e122)", 2023-09-23T20:27:54.006 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:18.195831+0000", 2023-09-23T20:27:54.006 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 275.77277891900002, 2023-09-23T20:27:54.006 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.85358087500000002, 2023-09-23T20:27:54.006 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:27:54.006 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:27:54.006 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:27:54.007 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:27:54.007 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:27:54.007 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 39822 2023-09-23T20:27:54.007 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.007 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:27:54.007 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.007 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:27:54.007 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.195831+0000", 2023-09-23T20:27:54.007 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:27:54.007 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.007 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.007 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:27:54.008 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.195831+0000", 2023-09-23T20:27:54.008 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:27:54.008 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.008 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.008 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:27:54.008 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.195829+0000", 2023-09-23T20:27:54.008 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:27:54.008 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.008 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.008 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:27:54.008 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.195955+0000", 2023-09-23T20:27:54.008 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000125561 2023-09-23T20:27:54.009 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.009 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.009 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:27:54.009 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.195970+0000", 2023-09-23T20:27:54.009 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.4769999999999999e-05 2023-09-23T20:27:54.009 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.009 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.009 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:27:54.009 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.195977+0000", 2023-09-23T20:27:54.009 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.8800000000000002e-06 2023-09-23T20:27:54.009 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.009 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.010 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:27:54.010 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.196021+0000", 2023-09-23T20:27:54.010 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.4672000000000002e-05 2023-09-23T20:27:54.010 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.010 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.010 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:27:54.010 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.196028+0000", 2023-09-23T20:27:54.010 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.2119999999999997e-06 2023-09-23T20:27:54.010 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.010 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.010 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:27:54.010 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:19.049401+0000", 2023-09-23T20:27:54.011 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.85337292600000003 2023-09-23T20:27:54.011 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.011 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.011 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:27:54.011 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:19.049411+0000", 2023-09-23T20:27:54.011 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0093e-05 2023-09-23T20:27:54.011 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:27:54.011 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:27:54.011 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:27:54.011 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.011 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.011 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:39831 5.11 5.3f234b1 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e122)", 2023-09-23T20:27:54.011 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:18.200476+0000", 2023-09-23T20:27:54.012 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 275.76813304699999, 2023-09-23T20:27:54.012 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.84895043999999997, 2023-09-23T20:27:54.012 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:27:54.012 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:27:54.012 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:27:54.012 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:27:54.012 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:27:54.012 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 39831 2023-09-23T20:27:54.012 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.012 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:27:54.012 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.013 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:27:54.013 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.200476+0000", 2023-09-23T20:27:54.013 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:27:54.013 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.013 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.013 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:27:54.013 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.200476+0000", 2023-09-23T20:27:54.013 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:27:54.013 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.013 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.013 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:27:54.013 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.200476+0000", 2023-09-23T20:27:54.014 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:27:54.014 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.014 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.014 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:27:54.014 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.200552+0000", 2023-09-23T20:27:54.014 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.6395000000000003e-05 2023-09-23T20:27:54.014 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.014 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.014 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:27:54.014 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.200561+0000", 2023-09-23T20:27:54.014 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.2099999999999993e-06 2023-09-23T20:27:54.014 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.015 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.015 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:27:54.015 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.200567+0000", 2023-09-23T20:27:54.015 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.3330000000000002e-06 2023-09-23T20:27:54.015 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.015 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.015 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:27:54.015 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.200619+0000", 2023-09-23T20:27:54.015 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.2435000000000002e-05 2023-09-23T20:27:54.015 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.015 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.015 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:27:54.016 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.200626+0000", 2023-09-23T20:27:54.016 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.1099999999999997e-06 2023-09-23T20:27:54.016 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.016 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.016 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:27:54.016 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:19.049420+0000", 2023-09-23T20:27:54.016 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.84879393000000003 2023-09-23T20:27:54.016 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.016 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.016 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:27:54.016 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:19.049427+0000", 2023-09-23T20:27:54.016 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.5180000000000001e-06 2023-09-23T20:27:54.017 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:27:54.017 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:27:54.017 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:27:54.017 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.017 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.017 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:39856 5.11 5.d501d031 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e122)", 2023-09-23T20:27:54.017 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:18.218506+0000", 2023-09-23T20:27:54.017 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 275.75010382300002, 2023-09-23T20:27:54.017 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.83093710200000004, 2023-09-23T20:27:54.017 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:27:54.017 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:27:54.017 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:27:54.017 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:27:54.018 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:27:54.018 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 39856 2023-09-23T20:27:54.018 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.018 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:27:54.018 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.018 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:27:54.018 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.218506+0000", 2023-09-23T20:27:54.018 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:27:54.018 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.018 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.018 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:27:54.018 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.218506+0000", 2023-09-23T20:27:54.019 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:27:54.019 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.019 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.019 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:27:54.019 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.218504+0000", 2023-09-23T20:27:54.019 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:27:54.019 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.019 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.019 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:27:54.019 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.218539+0000", 2023-09-23T20:27:54.019 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.5040999999999999e-05 2023-09-23T20:27:54.019 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.019 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.020 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:27:54.020 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.218558+0000", 2023-09-23T20:27:54.020 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.8372e-05 2023-09-23T20:27:54.020 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.020 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.020 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:27:54.020 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.218566+0000", 2023-09-23T20:27:54.020 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.9149999999999996e-06 2023-09-23T20:27:54.020 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.020 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.020 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:27:54.021 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.218615+0000", 2023-09-23T20:27:54.021 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.9008000000000002e-05 2023-09-23T20:27:54.021 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.021 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.021 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:27:54.021 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.218623+0000", 2023-09-23T20:27:54.021 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.9840000000000001e-06 2023-09-23T20:27:54.021 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.021 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.021 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:27:54.021 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:19.049437+0000", 2023-09-23T20:27:54.021 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.83081455500000001 2023-09-23T20:27:54.021 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.022 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.022 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:27:54.022 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:19.049443+0000", 2023-09-23T20:27:54.022 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.6790000000000002e-06 2023-09-23T20:27:54.022 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:27:54.022 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:27:54.022 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:27:54.022 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.022 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.022 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:39890 5.11 5.15c21ed1 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e122)", 2023-09-23T20:27:54.023 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:18.248436+0000", 2023-09-23T20:27:54.023 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 275.72017382400003, 2023-09-23T20:27:54.023 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.80101856299999996, 2023-09-23T20:27:54.023 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:27:54.023 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:27:54.023 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:27:54.023 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:27:54.023 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:27:54.023 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 39890 2023-09-23T20:27:54.024 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.024 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:27:54.024 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.024 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:27:54.024 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.248436+0000", 2023-09-23T20:27:54.024 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:27:54.024 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.024 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.024 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:27:54.024 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.248436+0000", 2023-09-23T20:27:54.025 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:27:54.025 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.025 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.025 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:27:54.025 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.248434+0000", 2023-09-23T20:27:54.025 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:27:54.025 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.025 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.025 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:27:54.025 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.248491+0000", 2023-09-23T20:27:54.026 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.6669000000000002e-05 2023-09-23T20:27:54.026 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.026 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.026 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:27:54.026 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.248508+0000", 2023-09-23T20:27:54.026 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.6878e-05 2023-09-23T20:27:54.026 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.026 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.026 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:27:54.027 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.248514+0000", 2023-09-23T20:27:54.027 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.0909999999999998e-06 2023-09-23T20:27:54.027 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.027 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.027 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:27:54.027 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.248558+0000", 2023-09-23T20:27:54.027 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.4320999999999998e-05 2023-09-23T20:27:54.027 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.027 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.027 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:27:54.028 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.248566+0000", 2023-09-23T20:27:54.028 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.6879999999999999e-06 2023-09-23T20:27:54.028 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.028 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.028 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:27:54.028 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:19.049450+0000", 2023-09-23T20:27:54.028 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.80088458600000001 2023-09-23T20:27:54.028 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.028 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.028 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:27:54.029 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:19.049454+0000", 2023-09-23T20:27:54.029 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.783e-06 2023-09-23T20:27:54.029 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:27:54.029 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:27:54.029 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:27:54.029 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.029 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.029 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:39893 5.11 5.4eed3c91 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e122)", 2023-09-23T20:27:54.029 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:18.251010+0000", 2023-09-23T20:27:54.029 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 275.71759970099998, 2023-09-23T20:27:54.030 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.79845648700000005, 2023-09-23T20:27:54.030 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:27:54.030 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:27:54.030 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:27:54.030 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:27:54.030 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:27:54.030 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 39893 2023-09-23T20:27:54.030 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.030 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:27:54.031 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.031 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:27:54.031 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.251010+0000", 2023-09-23T20:27:54.031 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:27:54.031 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.031 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.031 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:27:54.031 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.251010+0000", 2023-09-23T20:27:54.031 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:27:54.031 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.031 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.032 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:27:54.032 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.251009+0000", 2023-09-23T20:27:54.032 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:27:54.032 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.032 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.032 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:27:54.032 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.251097+0000", 2023-09-23T20:27:54.032 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.7786000000000003e-05 2023-09-23T20:27:54.032 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.032 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.032 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:27:54.032 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.251109+0000", 2023-09-23T20:27:54.032 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.2044e-05 2023-09-23T20:27:54.033 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.033 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.033 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:27:54.033 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.251115+0000", 2023-09-23T20:27:54.033 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.4420000000000003e-06 2023-09-23T20:27:54.033 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.033 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.033 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:27:54.033 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.251152+0000", 2023-09-23T20:27:54.034 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.6545999999999999e-05 2023-09-23T20:27:54.034 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.034 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.034 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:27:54.034 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.251159+0000", 2023-09-23T20:27:54.034 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.8909999999999998e-06 2023-09-23T20:27:54.034 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.034 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.035 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:27:54.035 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:19.049463+0000", 2023-09-23T20:27:54.035 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.79830458699999995 2023-09-23T20:27:54.035 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.035 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.035 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:27:54.035 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:19.049466+0000", 2023-09-23T20:27:54.035 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.145e-06 2023-09-23T20:27:54.035 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:27:54.035 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:27:54.036 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:27:54.036 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.036 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.036 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:49668 5.11 5:8ffafbda:::benchmark_data_smithi018_228451_object49667:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e130)", 2023-09-23T20:27:54.036 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:30.144126+0000", 2023-09-23T20:27:54.036 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 263.824483601, 2023-09-23T20:27:54.036 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0173537960000001, 2023-09-23T20:27:54.036 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:27:54.036 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:27:54.036 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:27:54.037 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:27:54.037 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:27:54.037 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 49668 2023-09-23T20:27:54.037 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.037 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:27:54.037 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.037 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:27:54.037 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.144126+0000", 2023-09-23T20:27:54.037 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:27:54.037 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.037 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.037 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:27:54.038 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.144126+0000", 2023-09-23T20:27:54.038 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:27:54.038 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.038 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.038 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:27:54.038 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.144124+0000", 2023-09-23T20:27:54.038 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:27:54.038 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.038 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.038 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:27:54.038 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.144171+0000", 2023-09-23T20:27:54.038 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.7029e-05 2023-09-23T20:27:54.039 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.039 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.039 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:27:54.039 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.144188+0000", 2023-09-23T20:27:54.039 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.721e-05 2023-09-23T20:27:54.039 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.039 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.039 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:27:54.039 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.144196+0000", 2023-09-23T20:27:54.039 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.498e-06 2023-09-23T20:27:54.039 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.039 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.039 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:27:54.040 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.144252+0000", 2023-09-23T20:27:54.040 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.5965999999999999e-05 2023-09-23T20:27:54.040 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.040 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.040 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op must wait for map", 2023-09-23T20:27:54.040 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.144260+0000", 2023-09-23T20:27:54.040 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.1920000000000005e-06 2023-09-23T20:27:54.040 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.040 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.040 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:27:54.040 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.149093+0000", 2023-09-23T20:27:54.040 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.004832622 2023-09-23T20:27:54.041 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.041 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.041 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:27:54.041 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.149103+0000", 2023-09-23T20:27:54.041 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.8479999999999996e-06 2023-09-23T20:27:54.041 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.041 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.041 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:27:54.041 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.147686+0000", 2023-09-23T20:27:54.041 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.99858325400000003 2023-09-23T20:27:54.041 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.041 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.042 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:27:54.042 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.147822+0000", 2023-09-23T20:27:54.042 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00013592199999999999 2023-09-23T20:27:54.042 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.042 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.042 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 4,7", 2023-09-23T20:27:54.042 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.148002+0000", 2023-09-23T20:27:54.042 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000180185 2023-09-23T20:27:54.042 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.042 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.042 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:27:54.042 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.152001+0000", 2023-09-23T20:27:54.042 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0039989049999999996 2023-09-23T20:27:54.043 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.043 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.043 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:27:54.043 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.161379+0000", 2023-09-23T20:27:54.043 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0093781179999999995 2023-09-23T20:27:54.043 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.043 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.043 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:27:54.043 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.161407+0000", 2023-09-23T20:27:54.043 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.7951999999999999e-05 2023-09-23T20:27:54.043 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.044 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.044 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:27:54.044 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.161445+0000", 2023-09-23T20:27:54.044 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.7999e-05 2023-09-23T20:27:54.044 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.044 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.044 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:27:54.044 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.161480+0000", 2023-09-23T20:27:54.044 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.4752999999999997e-05 2023-09-23T20:27:54.044 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:27:54.044 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:27:54.044 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:27:54.044 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.045 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.045 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:49677 5.11 5:8cd3bece:::benchmark_data_smithi018_228451_object49676:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-09-23T20:27:54.045 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:30.145601+0000", 2023-09-23T20:27:54.045 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 263.82300873700001, 2023-09-23T20:27:54.045 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0160527699999999, 2023-09-23T20:27:54.045 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:27:54.045 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:27:54.045 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:27:54.045 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:27:54.045 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:27:54.045 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 49677 2023-09-23T20:27:54.045 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.046 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:27:54.046 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.046 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:27:54.046 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.145601+0000", 2023-09-23T20:27:54.046 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:27:54.046 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.046 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.046 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:27:54.046 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.145601+0000", 2023-09-23T20:27:54.046 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:27:54.046 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.046 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.047 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:27:54.047 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.145600+0000", 2023-09-23T20:27:54.047 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:27:54.047 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.047 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.047 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:27:54.047 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.145714+0000", 2023-09-23T20:27:54.047 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000114341 2023-09-23T20:27:54.047 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.047 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.047 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:27:54.047 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.145729+0000", 2023-09-23T20:27:54.047 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.5061e-05 2023-09-23T20:27:54.048 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.048 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.048 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:27:54.048 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.145736+0000", 2023-09-23T20:27:54.048 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.9800000000000001e-06 2023-09-23T20:27:54.048 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.048 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.048 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:27:54.048 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.146231+0000", 2023-09-23T20:27:54.048 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00049503600000000002 2023-09-23T20:27:54.048 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.048 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.049 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting_for_map not empty", 2023-09-23T20:27:54.049 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.146242+0000", 2023-09-23T20:27:54.049 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1114999999999999e-05 2023-09-23T20:27:54.049 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.049 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.049 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:27:54.049 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.149132+0000", 2023-09-23T20:27:54.049 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0028893930000000001 2023-09-23T20:27:54.049 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.049 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.049 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:27:54.049 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.149139+0000", 2023-09-23T20:27:54.050 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.0319999999999998e-06 2023-09-23T20:27:54.050 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.050 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.050 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:27:54.050 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.148678+0000", 2023-09-23T20:27:54.050 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.999538861 2023-09-23T20:27:54.050 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.050 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.050 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:27:54.050 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.148809+0000", 2023-09-23T20:27:54.050 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000130968 2023-09-23T20:27:54.050 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.050 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.051 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 4,7", 2023-09-23T20:27:54.051 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.148965+0000", 2023-09-23T20:27:54.051 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00015674800000000001 2023-09-23T20:27:54.051 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.051 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.051 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:27:54.051 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.152012+0000", 2023-09-23T20:27:54.051 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0030466339999999999 2023-09-23T20:27:54.051 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.051 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.051 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:27:54.051 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.161507+0000", 2023-09-23T20:27:54.052 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0094949679999999995 2023-09-23T20:27:54.052 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.052 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.052 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:27:54.052 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.161588+0000", 2023-09-23T20:27:54.052 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.1589999999999996e-05 2023-09-23T20:27:54.052 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.052 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.052 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:27:54.052 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.161617+0000", 2023-09-23T20:27:54.052 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.8099e-05 2023-09-23T20:27:54.052 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.053 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.053 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:27:54.053 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.161654+0000", 2023-09-23T20:27:54.053 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.6927000000000001e-05 2023-09-23T20:27:54.053 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:27:54.053 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:27:54.053 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:27:54.053 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.053 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.053 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:49755 5.11 5:8b8671c4:::benchmark_data_smithi018_228451_object49754:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-09-23T20:27:54.053 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:30.216172+0000", 2023-09-23T20:27:54.053 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 263.75243702900002, 2023-09-23T20:27:54.053 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.94558546499999996, 2023-09-23T20:27:54.054 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:27:54.054 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:27:54.054 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:27:54.054 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:27:54.054 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:27:54.054 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 49755 2023-09-23T20:27:54.054 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.054 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:27:54.054 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.054 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:27:54.054 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.216172+0000", 2023-09-23T20:27:54.055 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:27:54.055 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.055 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.055 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:27:54.055 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.216172+0000", 2023-09-23T20:27:54.055 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:27:54.055 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.055 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.055 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:27:54.055 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.216170+0000", 2023-09-23T20:27:54.055 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999976 2023-09-23T20:27:54.055 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.055 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.056 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:27:54.056 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.216305+0000", 2023-09-23T20:27:54.056 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00013507699999999999 2023-09-23T20:27:54.056 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.056 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.056 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:27:54.056 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.216341+0000", 2023-09-23T20:27:54.056 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.6109999999999998e-05 2023-09-23T20:27:54.056 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.056 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.056 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:27:54.056 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.216349+0000", 2023-09-23T20:27:54.057 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.123e-06 2023-09-23T20:27:54.057 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.057 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.057 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:27:54.057 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.216426+0000", 2023-09-23T20:27:54.057 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.6604e-05 2023-09-23T20:27:54.057 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.057 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.057 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:27:54.058 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.216440+0000", 2023-09-23T20:27:54.058 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.3787e-05 2023-09-23T20:27:54.058 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.058 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.058 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:27:54.058 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.149653+0000", 2023-09-23T20:27:54.058 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.93321383099999999 2023-09-23T20:27:54.058 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.058 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.058 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:27:54.058 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.149779+0000", 2023-09-23T20:27:54.059 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00012566300000000001 2023-09-23T20:27:54.059 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.059 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.059 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 4,7", 2023-09-23T20:27:54.059 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.149925+0000", 2023-09-23T20:27:54.059 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00014582199999999999 2023-09-23T20:27:54.059 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.059 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.059 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:27:54.060 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.153584+0000", 2023-09-23T20:27:54.060 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0036591560000000002 2023-09-23T20:27:54.060 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.060 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.060 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:27:54.060 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.161679+0000", 2023-09-23T20:27:54.060 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0080945359999999994 2023-09-23T20:27:54.060 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.060 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.060 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:27:54.060 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.161695+0000", 2023-09-23T20:27:54.061 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.6311e-05 2023-09-23T20:27:54.061 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.061 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.061 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:27:54.061 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.161729+0000", 2023-09-23T20:27:54.061 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.4637000000000001e-05 2023-09-23T20:27:54.061 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.061 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.061 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:27:54.062 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.161758+0000", 2023-09-23T20:27:54.062 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.8422999999999999e-05 2023-09-23T20:27:54.062 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:27:54.062 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:27:54.062 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:27:54.062 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.062 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.062 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:49769 5.11 5:8ef0bf42:::benchmark_data_smithi018_228451_object49768:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-09-23T20:27:54.062 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:30.222066+0000", 2023-09-23T20:27:54.063 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 263.74654306399998, 2023-09-23T20:27:54.063 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.93977777699999998, 2023-09-23T20:27:54.063 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:27:54.063 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:27:54.063 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:27:54.063 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:27:54.063 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:27:54.063 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 49769 2023-09-23T20:27:54.063 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.063 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:27:54.064 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.064 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:27:54.064 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.222066+0000", 2023-09-23T20:27:54.064 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:27:54.064 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.064 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.064 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:27:54.064 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.222066+0000", 2023-09-23T20:27:54.065 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:27:54.065 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.065 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.065 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:27:54.065 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.222065+0000", 2023-09-23T20:27:54.065 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:27:54.065 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.065 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.065 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:27:54.066 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.222094+0000", 2023-09-23T20:27:54.066 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.9128999999999998e-05 2023-09-23T20:27:54.066 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.066 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.066 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:27:54.066 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.222112+0000", 2023-09-23T20:27:54.066 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.7099999999999999e-05 2023-09-23T20:27:54.066 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.067 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.067 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:27:54.067 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.222119+0000", 2023-09-23T20:27:54.067 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.1389999999999998e-06 2023-09-23T20:27:54.067 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.067 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.067 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:27:54.067 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.222180+0000", 2023-09-23T20:27:54.067 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.1301000000000001e-05 2023-09-23T20:27:54.067 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.068 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.068 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:27:54.068 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.222192+0000", 2023-09-23T20:27:54.068 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1661999999999999e-05 2023-09-23T20:27:54.068 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.068 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.068 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:27:54.068 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.150500+0000", 2023-09-23T20:27:54.068 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.92830877700000003 2023-09-23T20:27:54.068 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.068 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.068 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:27:54.069 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.150616+0000", 2023-09-23T20:27:54.069 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00011518099999999999 2023-09-23T20:27:54.069 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.069 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.069 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 4,7", 2023-09-23T20:27:54.069 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.150742+0000", 2023-09-23T20:27:54.069 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00012607699999999999 2023-09-23T20:27:54.069 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.069 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.069 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:27:54.069 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.153592+0000", 2023-09-23T20:27:54.070 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0028501569999999999 2023-09-23T20:27:54.070 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.070 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.070 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:27:54.070 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.161779+0000", 2023-09-23T20:27:54.070 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0081868459999999994 2023-09-23T20:27:54.070 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.070 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.070 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:27:54.070 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.161794+0000", 2023-09-23T20:27:54.070 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.5384999999999999e-05 2023-09-23T20:27:54.070 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.071 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.071 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:27:54.071 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.161826+0000", 2023-09-23T20:27:54.071 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.1483999999999999e-05 2023-09-23T20:27:54.071 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.071 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.071 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:27:54.071 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.161844+0000", 2023-09-23T20:27:54.071 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.8658999999999999e-05 2023-09-23T20:27:54.071 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:27:54.071 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:27:54.071 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:27:54.071 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.072 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.072 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:49810 5.11 5:8d760048:::benchmark_data_smithi018_228451_object49809:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-09-23T20:27:54.072 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:30.257088+0000", 2023-09-23T20:27:54.072 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 263.71152177200003, 2023-09-23T20:27:54.072 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.90484562899999998, 2023-09-23T20:27:54.072 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:27:54.072 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:27:54.072 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:27:54.072 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:27:54.072 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:27:54.072 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 49810 2023-09-23T20:27:54.072 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.073 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:27:54.073 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.073 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:27:54.073 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.257088+0000", 2023-09-23T20:27:54.073 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:27:54.073 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.073 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.073 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:27:54.073 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.257088+0000", 2023-09-23T20:27:54.073 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:27:54.073 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.073 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.074 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:27:54.074 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.257086+0000", 2023-09-23T20:27:54.074 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:27:54.074 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.074 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.074 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:27:54.074 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.257121+0000", 2023-09-23T20:27:54.074 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.4424000000000001e-05 2023-09-23T20:27:54.074 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.074 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.074 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:27:54.074 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.257134+0000", 2023-09-23T20:27:54.075 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.3409e-05 2023-09-23T20:27:54.075 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.075 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.075 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:27:54.075 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.257140+0000", 2023-09-23T20:27:54.075 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.9089999999999997e-06 2023-09-23T20:27:54.075 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.075 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.075 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:27:54.075 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.257208+0000", 2023-09-23T20:27:54.075 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.7716e-05 2023-09-23T20:27:54.076 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.076 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.076 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:27:54.076 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.257219+0000", 2023-09-23T20:27:54.076 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1382999999999999e-05 2023-09-23T20:27:54.076 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.076 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.076 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:27:54.076 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.151334+0000", 2023-09-23T20:27:54.076 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.89411527000000002 2023-09-23T20:27:54.076 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.076 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.077 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:27:54.077 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.151410+0000", 2023-09-23T20:27:54.077 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.5368000000000001e-05 2023-09-23T20:27:54.077 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.077 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.077 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 4,7", 2023-09-23T20:27:54.077 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.151519+0000", 2023-09-23T20:27:54.077 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000109481 2023-09-23T20:27:54.077 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.077 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.077 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:27:54.077 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.155912+0000", 2023-09-23T20:27:54.078 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0043927510000000003 2023-09-23T20:27:54.078 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.078 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.078 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:27:54.078 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.161867+0000", 2023-09-23T20:27:54.078 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0059548459999999998 2023-09-23T20:27:54.078 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.078 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.078 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:27:54.078 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.161882+0000", 2023-09-23T20:27:54.078 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.5053e-05 2023-09-23T20:27:54.078 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.079 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.079 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:27:54.079 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.161908+0000", 2023-09-23T20:27:54.079 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.6160999999999999e-05 2023-09-23T20:27:54.079 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.079 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.079 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:27:54.079 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.161933+0000", 2023-09-23T20:27:54.079 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.5448000000000001e-05 2023-09-23T20:27:54.079 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:27:54.079 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:27:54.079 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:27:54.080 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.080 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.080 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:49846 5.11 5:8a7c8f0f:::benchmark_data_smithi018_228451_object49845:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-09-23T20:27:54.080 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:30.283871+0000", 2023-09-23T20:27:54.080 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 263.68473844800002, 2023-09-23T20:27:54.080 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.87818426900000002, 2023-09-23T20:27:54.080 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:27:54.080 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:27:54.080 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:27:54.080 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:27:54.080 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:27:54.080 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 49846 2023-09-23T20:27:54.080 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.081 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:27:54.081 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.081 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:27:54.081 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.283871+0000", 2023-09-23T20:27:54.081 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:27:54.081 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.081 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.081 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:27:54.081 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.283871+0000", 2023-09-23T20:27:54.081 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:27:54.081 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.082 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.082 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:27:54.082 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.283870+0000", 2023-09-23T20:27:54.082 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:27:54.082 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.082 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.082 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:27:54.082 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.283939+0000", 2023-09-23T20:27:54.082 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.9799e-05 2023-09-23T20:27:54.082 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.082 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.082 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:27:54.083 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.283960+0000", 2023-09-23T20:27:54.083 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.0613999999999999e-05 2023-09-23T20:27:54.083 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.083 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.083 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:27:54.083 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.283968+0000", 2023-09-23T20:27:54.083 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.6429999999999995e-06 2023-09-23T20:27:54.083 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.083 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.083 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:27:54.083 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.284026+0000", 2023-09-23T20:27:54.083 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.8850000000000001e-05 2023-09-23T20:27:54.083 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.084 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.084 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:27:54.084 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.284033+0000", 2023-09-23T20:27:54.084 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.9249999999999998e-06 2023-09-23T20:27:54.084 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.084 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.084 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:27:54.084 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.152057+0000", 2023-09-23T20:27:54.084 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.86802386899999995 2023-09-23T20:27:54.084 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.084 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.084 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:27:54.085 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.152189+0000", 2023-09-23T20:27:54.085 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00013166600000000001 2023-09-23T20:27:54.085 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.085 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.085 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 4,7", 2023-09-23T20:27:54.085 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.152356+0000", 2023-09-23T20:27:54.085 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00016740599999999999 2023-09-23T20:27:54.085 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.085 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.085 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:27:54.085 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.156658+0000", 2023-09-23T20:27:54.085 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0043016720000000003 2023-09-23T20:27:54.086 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.086 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.086 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:27:54.086 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.161959+0000", 2023-09-23T20:27:54.086 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0053010699999999997 2023-09-23T20:27:54.086 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.086 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.086 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:27:54.086 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.161975+0000", 2023-09-23T20:27:54.086 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.6016e-05 2023-09-23T20:27:54.086 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.087 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.087 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:27:54.087 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.162015+0000", 2023-09-23T20:27:54.087 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.0240000000000001e-05 2023-09-23T20:27:54.087 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.087 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.087 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:27:54.087 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.162055+0000", 2023-09-23T20:27:54.087 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.9984000000000002e-05 2023-09-23T20:27:54.087 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:27:54.088 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:27:54.088 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:27:54.088 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.088 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.088 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:49862 5.11 5:8acf6c38:::benchmark_data_smithi018_228451_object49861:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-09-23T20:27:54.088 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:30.291752+0000", 2023-09-23T20:27:54.088 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 263.67685707800001, 2023-09-23T20:27:54.088 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.87046705199999996, 2023-09-23T20:27:54.088 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:27:54.089 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:27:54.089 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:27:54.089 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:27:54.089 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:27:54.089 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 49862 2023-09-23T20:27:54.089 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.089 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:27:54.089 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.089 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:27:54.089 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.291752+0000", 2023-09-23T20:27:54.090 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:27:54.090 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.090 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.090 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:27:54.090 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.291752+0000", 2023-09-23T20:27:54.090 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:27:54.090 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.090 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.090 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:27:54.091 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.291751+0000", 2023-09-23T20:27:54.091 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:27:54.091 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.091 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.091 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:27:54.091 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.291796+0000", 2023-09-23T20:27:54.091 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.4452000000000001e-05 2023-09-23T20:27:54.091 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.091 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.091 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:27:54.092 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.291814+0000", 2023-09-23T20:27:54.092 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.7846000000000002e-05 2023-09-23T20:27:54.092 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.092 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.092 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:27:54.092 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.291821+0000", 2023-09-23T20:27:54.092 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.4549999999999998e-06 2023-09-23T20:27:54.092 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.092 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.093 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:27:54.093 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.291932+0000", 2023-09-23T20:27:54.093 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000110945 2023-09-23T20:27:54.093 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.093 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.093 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:27:54.093 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.291942+0000", 2023-09-23T20:27:54.093 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0392e-05 2023-09-23T20:27:54.093 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.093 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.094 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:27:54.094 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.152826+0000", 2023-09-23T20:27:54.094 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.86088340100000005 2023-09-23T20:27:54.094 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.094 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.094 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:27:54.094 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.152946+0000", 2023-09-23T20:27:54.094 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000119963 2023-09-23T20:27:54.094 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.095 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.095 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 4,7", 2023-09-23T20:27:54.095 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.153060+0000", 2023-09-23T20:27:54.095 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00011451300000000001 2023-09-23T20:27:54.095 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.095 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.095 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:27:54.095 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.159094+0000", 2023-09-23T20:27:54.095 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0060340300000000001 2023-09-23T20:27:54.095 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.096 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.096 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:27:54.096 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.162100+0000", 2023-09-23T20:27:54.096 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0030057199999999999 2023-09-23T20:27:54.096 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.096 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.096 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:27:54.096 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.162137+0000", 2023-09-23T20:27:54.096 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.7002999999999998e-05 2023-09-23T20:27:54.096 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.097 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.097 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:27:54.097 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.162183+0000", 2023-09-23T20:27:54.097 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.6199000000000003e-05 2023-09-23T20:27:54.097 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.097 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.097 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:27:54.097 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.162219+0000", 2023-09-23T20:27:54.097 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.6186999999999998e-05 2023-09-23T20:27:54.097 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:27:54.098 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:27:54.098 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:27:54.098 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.098 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.098 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:49899 5.11 5:8d0362d8:::benchmark_data_smithi018_228451_object49898:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-09-23T20:27:54.098 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:30.316014+0000", 2023-09-23T20:27:54.098 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 263.65259545399999, 2023-09-23T20:27:54.098 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.84638897400000002, 2023-09-23T20:27:54.098 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:27:54.099 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:27:54.099 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:27:54.099 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:27:54.099 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:27:54.099 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 49899 2023-09-23T20:27:54.099 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.099 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:27:54.099 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.099 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:27:54.099 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.316014+0000", 2023-09-23T20:27:54.099 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:27:54.100 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.100 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.100 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:27:54.100 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.316014+0000", 2023-09-23T20:27:54.100 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:27:54.100 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.100 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.100 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:27:54.101 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.316007+0000", 2023-09-23T20:27:54.101 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999933 2023-09-23T20:27:54.101 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.101 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.101 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:27:54.101 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.316092+0000", 2023-09-23T20:27:54.101 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.4536999999999994e-05 2023-09-23T20:27:54.101 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.101 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.102 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:27:54.102 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.316110+0000", 2023-09-23T20:27:54.102 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.8204000000000001e-05 2023-09-23T20:27:54.102 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.102 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.102 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:27:54.102 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.316115+0000", 2023-09-23T20:27:54.102 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.2079999999999999e-06 2023-09-23T20:27:54.102 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.102 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.102 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:27:54.102 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.316190+0000", 2023-09-23T20:27:54.103 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.4400999999999995e-05 2023-09-23T20:27:54.103 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.103 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.103 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:27:54.103 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.316197+0000", 2023-09-23T20:27:54.103 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.0620000000000003e-06 2023-09-23T20:27:54.103 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.103 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.103 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:27:54.103 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.153639+0000", 2023-09-23T20:27:54.103 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.83744230799999997 2023-09-23T20:27:54.103 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.104 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.104 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:27:54.104 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.153769+0000", 2023-09-23T20:27:54.104 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00012973699999999999 2023-09-23T20:27:54.104 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.104 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.104 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 4,7", 2023-09-23T20:27:54.104 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.153882+0000", 2023-09-23T20:27:54.104 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00011381400000000001 2023-09-23T20:27:54.104 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.104 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.104 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:27:54.105 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.159107+0000", 2023-09-23T20:27:54.105 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0052248950000000002 2023-09-23T20:27:54.105 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.105 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.105 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:27:54.105 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.162256+0000", 2023-09-23T20:27:54.105 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0031487669999999998 2023-09-23T20:27:54.105 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.105 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.105 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:27:54.105 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.162314+0000", 2023-09-23T20:27:54.105 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.7707999999999997e-05 2023-09-23T20:27:54.106 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.106 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.106 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:27:54.106 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.162367+0000", 2023-09-23T20:27:54.106 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.3381000000000003e-05 2023-09-23T20:27:54.106 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.106 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.106 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:27:54.106 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.162403+0000", 2023-09-23T20:27:54.106 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.5754999999999999e-05 2023-09-23T20:27:54.106 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:27:54.106 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:27:54.107 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:27:54.107 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.107 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.107 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:49914 5.11 5:8de3f302:::benchmark_data_smithi018_228451_object49913:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-09-23T20:27:54.107 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:30.327710+0000", 2023-09-23T20:27:54.107 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 263.64089911600001, 2023-09-23T20:27:54.107 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.83480937700000002, 2023-09-23T20:27:54.107 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:27:54.107 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:27:54.107 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:27:54.107 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:27:54.107 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:27:54.108 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 49914 2023-09-23T20:27:54.108 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.108 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:27:54.108 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.108 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:27:54.108 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.327710+0000", 2023-09-23T20:27:54.108 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:27:54.108 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.108 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.108 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:27:54.108 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.327710+0000", 2023-09-23T20:27:54.108 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:27:54.109 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.109 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.109 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:27:54.109 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.327709+0000", 2023-09-23T20:27:54.109 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:27:54.109 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.109 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.109 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:27:54.109 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.327798+0000", 2023-09-23T20:27:54.109 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.9086999999999996e-05 2023-09-23T20:27:54.109 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.109 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.109 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:27:54.110 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.327814+0000", 2023-09-23T20:27:54.110 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.5974999999999999e-05 2023-09-23T20:27:54.110 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.110 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.110 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:27:54.110 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.327820+0000", 2023-09-23T20:27:54.110 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.614e-06 2023-09-23T20:27:54.110 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.110 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.110 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:27:54.110 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.327881+0000", 2023-09-23T20:27:54.110 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.1271000000000003e-05 2023-09-23T20:27:54.111 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.111 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.111 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:27:54.111 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.327897+0000", 2023-09-23T20:27:54.111 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.5962000000000001e-05 2023-09-23T20:27:54.111 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.111 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.111 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:27:54.111 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.154407+0000", 2023-09-23T20:27:54.111 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.82650979199999997 2023-09-23T20:27:54.111 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.111 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.112 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:27:54.112 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.154489+0000", 2023-09-23T20:27:54.112 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.1904999999999996e-05 2023-09-23T20:27:54.112 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.112 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.112 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 4,7", 2023-09-23T20:27:54.112 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.154587+0000", 2023-09-23T20:27:54.112 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.8391e-05 2023-09-23T20:27:54.112 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.112 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.112 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:27:54.112 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.159115+0000", 2023-09-23T20:27:54.112 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0045281030000000003 2023-09-23T20:27:54.113 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.113 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.113 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:27:54.113 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.162286+0000", 2023-09-23T20:27:54.113 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0031707110000000001 2023-09-23T20:27:54.113 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.113 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.113 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:27:54.113 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.162438+0000", 2023-09-23T20:27:54.113 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000152192 2023-09-23T20:27:54.113 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.113 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.113 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:27:54.114 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.162483+0000", 2023-09-23T20:27:54.114 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.4950000000000002e-05 2023-09-23T20:27:54.114 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.114 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.114 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:27:54.114 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.162520+0000", 2023-09-23T20:27:54.114 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.6643999999999997e-05 2023-09-23T20:27:54.114 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:27:54.114 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:27:54.114 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:27:54.114 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.114 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.115 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:49943 5.11 5:88faa156:::benchmark_data_smithi018_228451_object49942:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-09-23T20:27:54.115 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:30.349163+0000", 2023-09-23T20:27:54.115 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 263.61944641600002, 2023-09-23T20:27:54.115 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.81349829299999998, 2023-09-23T20:27:54.115 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:27:54.115 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:27:54.115 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:27:54.115 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:27:54.115 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:27:54.115 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 49943 2023-09-23T20:27:54.115 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.115 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:27:54.116 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.116 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:27:54.116 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.349163+0000", 2023-09-23T20:27:54.116 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:27:54.116 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.116 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.116 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:27:54.116 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.349163+0000", 2023-09-23T20:27:54.116 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:27:54.116 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.116 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.116 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:27:54.116 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.349162+0000", 2023-09-23T20:27:54.117 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:27:54.117 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.117 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.117 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:27:54.117 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.349210+0000", 2023-09-23T20:27:54.117 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.8068999999999997e-05 2023-09-23T20:27:54.117 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.117 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.117 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:27:54.117 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.349222+0000", 2023-09-23T20:27:54.117 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.2434e-05 2023-09-23T20:27:54.117 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.118 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.118 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:27:54.118 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.349228+0000", 2023-09-23T20:27:54.118 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.0170000000000004e-06 2023-09-23T20:27:54.118 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.118 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.118 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:27:54.118 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.349276+0000", 2023-09-23T20:27:54.118 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.8976e-05 2023-09-23T20:27:54.118 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.118 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.118 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:27:54.118 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.349285+0000", 2023-09-23T20:27:54.119 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.4009999999999997e-06 2023-09-23T20:27:54.119 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.119 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.119 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:27:54.119 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.155211+0000", 2023-09-23T20:27:54.119 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.80592635199999996 2023-09-23T20:27:54.119 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.119 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.119 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:27:54.119 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.155298+0000", 2023-09-23T20:27:54.119 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.6293999999999997e-05 2023-09-23T20:27:54.119 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.120 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.120 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 4,7", 2023-09-23T20:27:54.120 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.155431+0000", 2023-09-23T20:27:54.120 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000133416 2023-09-23T20:27:54.120 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.120 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.120 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:27:54.120 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.159126+0000", 2023-09-23T20:27:54.120 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0036948879999999999 2023-09-23T20:27:54.120 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.120 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.120 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:27:54.121 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.162565+0000", 2023-09-23T20:27:54.121 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.003438853 2023-09-23T20:27:54.121 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.121 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.121 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:27:54.121 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.162594+0000", 2023-09-23T20:27:54.121 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.9000999999999999e-05 2023-09-23T20:27:54.121 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.121 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.121 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:27:54.122 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.162632+0000", 2023-09-23T20:27:54.122 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.8451000000000002e-05 2023-09-23T20:27:54.122 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:27:54.122 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:27:54.122 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:27:54.122 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.162661+0000", 2023-09-23T20:27:54.122 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.9220000000000001e-05 2023-09-23T20:27:54.122 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:27:54.122 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:27:54.122 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:27:54.123 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:27:54.123 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:27:54.123 INFO:teuthology.orchestra.run.smithi149.stdout:} 2023-09-23T20:27:54.123 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:54.139 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:27:54.144 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:54.144 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:27:54.209 INFO:teuthology.orchestra.run.smithi018.stderr:nodeep-scrub is set 2023-09-23T20:27:54.246 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:27:54.292 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:54 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"}]': finished 2023-09-23T20:27:54.292 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:54 smithi018 ceph-mon[184144]: osdmap e225: 8 total, 8 up, 8 in 2023-09-23T20:27:54.292 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:54 smithi018 ceph-mon[184144]: failed to encode map e225 with expected crc 2023-09-23T20:27:54.292 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:54 smithi018 ceph-mon[184144]: failed to encode map e225 with expected crc 2023-09-23T20:27:54.292 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:54 smithi018 ceph-mon[184144]: failed to encode map e225 with expected crc 2023-09-23T20:27:54.293 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:54 smithi018 ceph-mon[184144]: failed to encode map e225 with expected crc 2023-09-23T20:27:54.293 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:54 smithi018 ceph-mon[184144]: failed to encode map e225 with expected crc 2023-09-23T20:27:54.293 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:54 smithi018 ceph-mon[184144]: failed to encode map e225 with expected crc 2023-09-23T20:27:54.293 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:54 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"}]: dispatch 2023-09-23T20:27:54.293 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:54 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2414316103' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:27:54.293 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:54 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:27:54.293 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:54 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"}]': finished 2023-09-23T20:27:54.294 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:54 smithi018 ceph-mon[179647]: osdmap e225: 8 total, 8 up, 8 in 2023-09-23T20:27:54.294 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:54 smithi018 ceph-mon[179647]: failed to encode map e225 with expected crc 2023-09-23T20:27:54.294 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:54 smithi018 ceph-mon[179647]: failed to encode map e225 with expected crc 2023-09-23T20:27:54.294 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:54 smithi018 ceph-mon[179647]: failed to encode map e225 with expected crc 2023-09-23T20:27:54.294 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:54 smithi018 ceph-mon[179647]: failed to encode map e225 with expected crc 2023-09-23T20:27:54.294 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:54 smithi018 ceph-mon[179647]: failed to encode map e225 with expected crc 2023-09-23T20:27:54.294 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:54 smithi018 ceph-mon[179647]: failed to encode map e225 with expected crc 2023-09-23T20:27:54.294 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:54 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"}]: dispatch 2023-09-23T20:27:54.294 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:54 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2414316103' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:27:54.294 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:54 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:27:54.322 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:54.384 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:54 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"}]': finished 2023-09-23T20:27:54.384 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:54 smithi149 ceph-mon[170394]: osdmap e225: 8 total, 8 up, 8 in 2023-09-23T20:27:54.384 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:54 smithi149 ceph-mon[170394]: failed to encode map e225 with expected crc 2023-09-23T20:27:54.384 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:54 smithi149 ceph-mon[170394]: failed to encode map e225 with expected crc 2023-09-23T20:27:54.384 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:54 smithi149 ceph-mon[170394]: failed to encode map e225 with expected crc 2023-09-23T20:27:54.384 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:54 smithi149 ceph-mon[170394]: failed to encode map e225 with expected crc 2023-09-23T20:27:54.384 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:54 smithi149 ceph-mon[170394]: failed to encode map e225 with expected crc 2023-09-23T20:27:54.385 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:54 smithi149 ceph-mon[170394]: failed to encode map e225 with expected crc 2023-09-23T20:27:54.385 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:54 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"}]: dispatch 2023-09-23T20:27:54.385 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:54 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2414316103' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:27:54.385 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:54 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:27:54.386 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T20:27:54.569 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 32 16 57605 57589 112.413 76.9375 0.00846838 0.00889101 2023-09-23T20:27:54.621 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:27:54.632 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:54.633 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:27:54.734 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:27:54.802 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:55.059 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:27:55.066 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:55.066 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:27:55.156 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:27:55.167 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:27:55.249 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:27:54 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:27:54] "GET /metrics HTTP/1.1" 200 36513 "" "Prometheus/2.43.0" 2023-09-23T20:27:55.250 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:55 smithi018 ceph-mon[184144]: pgmap v495: 99 pgs: 99 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 92 MiB/s wr, 1.47k op/s 2023-09-23T20:27:55.250 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:55 smithi018 ceph-mon[184144]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:27:55.250 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:55 smithi018 ceph-mon[184144]: failed to encode map e226 with expected crc 2023-09-23T20:27:55.250 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:55 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"}]': finished 2023-09-23T20:27:55.250 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:55 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:27:55.250 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:55 smithi018 ceph-mon[184144]: osdmap e226: 8 total, 8 up, 8 in 2023-09-23T20:27:55.250 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:55 smithi018 ceph-mon[184144]: failed to encode map e226 with expected crc 2023-09-23T20:27:55.250 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:55 smithi018 ceph-mon[184144]: failed to encode map e226 with expected crc 2023-09-23T20:27:55.251 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:55 smithi018 ceph-mon[184144]: failed to encode map e226 with expected crc 2023-09-23T20:27:55.251 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:55 smithi018 ceph-mon[179647]: pgmap v495: 99 pgs: 99 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 92 MiB/s wr, 1.47k op/s 2023-09-23T20:27:55.251 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:55 smithi018 ceph-mon[179647]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:27:55.251 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:55 smithi018 ceph-mon[179647]: failed to encode map e226 with expected crc 2023-09-23T20:27:55.251 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:55 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"}]': finished 2023-09-23T20:27:55.251 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:55 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:27:55.251 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:55 smithi018 ceph-mon[179647]: osdmap e226: 8 total, 8 up, 8 in 2023-09-23T20:27:55.251 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:55 smithi018 ceph-mon[179647]: failed to encode map e226 with expected crc 2023-09-23T20:27:55.251 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:55 smithi018 ceph-mon[179647]: failed to encode map e226 with expected crc 2023-09-23T20:27:55.251 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:55 smithi018 ceph-mon[179647]: failed to encode map e226 with expected crc 2023-09-23T20:27:55.253 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:55.310 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:27:55.315 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:55.316 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:27:55.416 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:27:55.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:55 smithi149 ceph-mon[170394]: pgmap v495: 99 pgs: 99 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 92 MiB/s wr, 1.47k op/s 2023-09-23T20:27:55.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:55 smithi149 ceph-mon[170394]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:27:55.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:55 smithi149 ceph-mon[170394]: failed to encode map e226 with expected crc 2023-09-23T20:27:55.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:55 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"}]': finished 2023-09-23T20:27:55.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:55 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:27:55.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:55 smithi149 ceph-mon[170394]: osdmap e226: 8 total, 8 up, 8 in 2023-09-23T20:27:55.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:55 smithi149 ceph-mon[170394]: failed to encode map e226 with expected crc 2023-09-23T20:27:55.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:55 smithi149 ceph-mon[170394]: failed to encode map e226 with expected crc 2023-09-23T20:27:55.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:55 smithi149 ceph-mon[170394]: failed to encode map e226 with expected crc 2023-09-23T20:27:55.471 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:55.569 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 33 16 58837 58821 111.34 77 0.0092572 0.00897606 2023-09-23T20:27:55.729 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:27:55.737 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:55.737 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:27:55.777 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:27:55.820 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:27:55.838 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:27:55.870 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:27:55.909 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:27:55.910 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:55.953 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:27:55.984 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:27:55.988 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:55.989 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:27:55.990 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:27:56.034 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:27:56.076 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:27:56.090 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:27:56.095 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:27:56.095 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:27:56.095 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:27:56.095 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:27:56.095 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:27:56.095 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:27:56.095 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:27:56.095 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:27:56.158 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:56.239 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:27:56.242 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:56.243 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:27:56.343 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:27:56.404 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:56.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:56 smithi149 ceph-mon[170394]: osdmap e227: 8 total, 8 up, 8 in 2023-09-23T20:27:56.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:56 smithi149 ceph-mon[170394]: failed to encode map e227 with expected crc 2023-09-23T20:27:56.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:56 smithi149 ceph-mon[170394]: failed to encode map e227 with expected crc 2023-09-23T20:27:56.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:56 smithi149 ceph-mon[170394]: failed to encode map e227 with expected crc 2023-09-23T20:27:56.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:56 smithi149 ceph-mon[170394]: failed to encode map e227 with expected crc 2023-09-23T20:27:56.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:56 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/897709705' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:27:56.569 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 34 16 60091 60075 110.371 78.375 0.0116729 0.00905564 2023-09-23T20:27:56.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:56 smithi018 ceph-mon[179647]: osdmap e227: 8 total, 8 up, 8 in 2023-09-23T20:27:56.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:56 smithi018 ceph-mon[179647]: failed to encode map e227 with expected crc 2023-09-23T20:27:56.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:56 smithi018 ceph-mon[179647]: failed to encode map e227 with expected crc 2023-09-23T20:27:56.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:56 smithi018 ceph-mon[179647]: failed to encode map e227 with expected crc 2023-09-23T20:27:56.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:56 smithi018 ceph-mon[179647]: failed to encode map e227 with expected crc 2023-09-23T20:27:56.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:56 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/897709705' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:27:56.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:56 smithi018 ceph-mon[184144]: osdmap e227: 8 total, 8 up, 8 in 2023-09-23T20:27:56.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:56 smithi018 ceph-mon[184144]: failed to encode map e227 with expected crc 2023-09-23T20:27:56.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:56 smithi018 ceph-mon[184144]: failed to encode map e227 with expected crc 2023-09-23T20:27:56.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:56 smithi018 ceph-mon[184144]: failed to encode map e227 with expected crc 2023-09-23T20:27:56.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:56 smithi018 ceph-mon[184144]: failed to encode map e227 with expected crc 2023-09-23T20:27:56.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:56 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/897709705' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:27:56.702 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T20:27:57.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:57 smithi149 ceph-mon[170394]: pgmap v498: 109 pgs: 1 peering, 6 unknown, 102 active+clean; 3.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s 2023-09-23T20:27:57.569 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 35 16 61254 61238 109.294 72.6875 0.0160188 0.00914444 2023-09-23T20:27:57.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:57 smithi018 ceph-mon[184144]: pgmap v498: 109 pgs: 1 peering, 6 unknown, 102 active+clean; 3.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s 2023-09-23T20:27:57.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:57 smithi018 ceph-mon[179647]: pgmap v498: 109 pgs: 1 peering, 6 unknown, 102 active+clean; 3.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s 2023-09-23T20:27:58.168 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:27:58.173 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:58.174 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:27:58.275 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:27:58.346 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:58.398 INFO:teuthology.orchestra.run.smithi149.stdout:ERROR: (22) Invalid argument 2023-09-23T20:27:58.398 INFO:teuthology.orchestra.run.smithi149.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:27:58.406 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:27:58.412 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:27:58.417 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:58.418 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:27:58.518 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:27:58.569 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 36 16 62514 62498 108.446 78.75 0.00453441 0.0092165 2023-09-23T20:27:58.589 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:58.746 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 4, 5, 6, 7, 3, 2] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-23T20:27:58.746 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:27:58.746 INFO:tasks.thrashosds.thrasher:Removing osd 6, in_osds are: [0, 1, 4, 5, 6, 7, 3, 2] 2023-09-23T20:27:58.746 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd out 6 2023-09-23T20:27:58.791 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T20:27:59.293 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:27:59.300 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:27:59.301 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:27:59.401 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:27:59.458 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:27:59.558 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:27:59.569 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 37 16 63659 63643 107.449 71.5625 0.00812623 0.00930146 2023-09-23T20:27:59.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:27:59 smithi018 ceph-mon[179647]: pgmap v499: 109 pgs: 1 peering, 4 unknown, 104 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-09-23T20:27:59.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:27:59 smithi018 ceph-mon[184144]: pgmap v499: 109 pgs: 1 peering, 4 unknown, 104 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-09-23T20:27:59.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:27:59 smithi149 ceph-mon[170394]: pgmap v499: 109 pgs: 1 peering, 4 unknown, 104 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-09-23T20:28:00.176 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:28:00.231 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:28:00.246 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:00.247 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:28:00.347 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:28:00.410 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:00.539 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:28:00.546 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:00.546 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:28:00.570 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 38 16 64857 64841 106.589 74.875 0.015371 0.00937568 2023-09-23T20:28:00.573 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:00 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:28:00.573 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:00 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:28:00.573 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:00 smithi018 ceph-mon[184144]: pgmap v500: 109 pgs: 1 peering, 2 unknown, 106 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 61 MiB/s wr, 982 op/s 2023-09-23T20:28:00.574 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:00 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:28:00.574 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:00 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:28:00.574 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:00 smithi018 ceph-mon[179647]: pgmap v500: 109 pgs: 1 peering, 2 unknown, 106 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 61 MiB/s wr, 982 op/s 2023-09-23T20:28:00.647 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:28:00.661 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:00 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:28:00.661 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:00 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:28:00.661 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:00 smithi149 ceph-mon[170394]: pgmap v500: 109 pgs: 1 peering, 2 unknown, 106 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 61 MiB/s wr, 982 op/s 2023-09-23T20:28:00.698 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:00.811 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:28:00.815 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:00.816 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:28:00.917 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:28:00.931 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:28:00.943 INFO:teuthology.orchestra.run.smithi149.stdout:ERROR: (22) Invalid argument 2023-09-23T20:28:00.943 INFO:teuthology.orchestra.run.smithi149.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:28:00.964 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:01.017 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:28:01.023 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:01.023 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:28:01.124 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:28:01.214 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:01.352 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:28:01.362 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:01.362 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:28:01.406 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T20:28:01.463 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:28:01.490 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:28:01.571 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 39 16 65958 65942 105.621 68.8125 0.015655 0.00946131 2023-09-23T20:28:01.572 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:01.736 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:28:01.743 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:01.744 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:28:01.844 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:28:01.890 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:01.939 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:28:01.946 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:01.946 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:28:01.996 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:01 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "21"}]: dispatch 2023-09-23T20:28:02.047 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:28:02.091 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:02.141 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:28:02.143 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:01 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "21"}]: dispatch 2023-09-23T20:28:02.144 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:02.145 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:28:02.146 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:28:02.245 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:28:02.278 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:02.296 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:02 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "21"}]: dispatch 2023-09-23T20:28:02.384 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:28:02.390 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:02.391 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:28:02.491 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:28:02.549 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:02.571 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:2023-09-23T20:28:02.571632+0000 min lat: 0.00113549 max lat: 0.140818 avg lat: 0.00955044 2023-09-23T20:28:02.571 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T20:28:02.571 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 40 16 67024 67008 104.646 66.625 0.0119257 0.00955044 2023-09-23T20:28:02.666 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:28:02.672 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:02.673 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:28:02.773 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:28:02.814 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:02.992 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:28:02.996 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:02.997 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:28:03.038 INFO:teuthology.orchestra.run.smithi018.stderr:noscrub is unset 2023-09-23T20:28:03.078 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:03 smithi018 ceph-mon[184144]: pgmap v501: 109 pgs: 109 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 85 MiB/s wr, 1.35k op/s 2023-09-23T20:28:03.078 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:03 smithi018 ceph-mon[184144]: failed to encode map e228 with expected crc 2023-09-23T20:28:03.078 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:03 smithi018 ceph-mon[184144]: failed to encode map e228 with expected crc 2023-09-23T20:28:03.079 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:03 smithi018 ceph-mon[184144]: failed to encode map e228 with expected crc 2023-09-23T20:28:03.079 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:03 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "21"}]': finished 2023-09-23T20:28:03.079 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:03 smithi018 ceph-mon[184144]: osdmap e228: 8 total, 8 up, 8 in 2023-09-23T20:28:03.079 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:03 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/71952743' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:28:03.079 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:03 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:28:03.079 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:03 smithi018 ceph-mon[179647]: pgmap v501: 109 pgs: 109 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 85 MiB/s wr, 1.35k op/s 2023-09-23T20:28:03.079 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:03 smithi018 ceph-mon[179647]: failed to encode map e228 with expected crc 2023-09-23T20:28:03.079 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:03 smithi018 ceph-mon[179647]: failed to encode map e228 with expected crc 2023-09-23T20:28:03.080 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:03 smithi018 ceph-mon[179647]: failed to encode map e228 with expected crc 2023-09-23T20:28:03.080 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:03 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "21"}]': finished 2023-09-23T20:28:03.080 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:03 smithi018 ceph-mon[179647]: osdmap e228: 8 total, 8 up, 8 in 2023-09-23T20:28:03.080 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:03 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/71952743' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:28:03.080 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:03 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:28:03.097 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:28:03.137 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:03.201 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:28:03.206 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:03.207 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:28:03.307 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:28:03.339 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:03 smithi149 ceph-mon[170394]: pgmap v501: 109 pgs: 109 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 85 MiB/s wr, 1.35k op/s 2023-09-23T20:28:03.340 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:03 smithi149 ceph-mon[170394]: failed to encode map e228 with expected crc 2023-09-23T20:28:03.340 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:03 smithi149 ceph-mon[170394]: failed to encode map e228 with expected crc 2023-09-23T20:28:03.340 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:03 smithi149 ceph-mon[170394]: failed to encode map e228 with expected crc 2023-09-23T20:28:03.340 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:03 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "21"}]': finished 2023-09-23T20:28:03.340 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:03 smithi149 ceph-mon[170394]: osdmap e228: 8 total, 8 up, 8 in 2023-09-23T20:28:03.340 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:03 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/71952743' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:28:03.340 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:03 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:28:03.411 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:03.482 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:28:03.531 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:28:03.571 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 41 16 68023 68007 103.617 62.4375 0.0107757 0.00964481 2023-09-23T20:28:03.578 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:28:03.602 INFO:teuthology.orchestra.run.smithi149.stdout:ERROR: (22) Invalid argument 2023-09-23T20:28:03.602 INFO:teuthology.orchestra.run.smithi149.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:28:03.612 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:28:03.627 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:03.628 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:28:03.637 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:28:03.652 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:03 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mon-a[179623]: 2023-09-23T20:28:03.529+0000 7f160fef9700 -1 mon.a@0(leader).osd e229 definitely_dead 0 2023-09-23T20:28:03.693 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:28:03.728 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:28:03.729 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:28:03.774 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:28:03.816 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:03.836 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:28:03.840 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:28:03.840 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:28:03.840 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:28:03.840 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:28:03.840 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:28:03.840 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:28:03.840 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:28:03.840 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:28:03.945 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:28:03.953 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:03.954 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:28:04.039 INFO:teuthology.orchestra.run.smithi018.stderr:marked out osd.6. 2023-09-23T20:28:04.054 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:28:04.070 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T20:28:04.099 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:04.132 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:28:04.135 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:04.136 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:28:04.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:04 smithi149 ceph-mon[170394]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:28:04.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:04 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:28:04.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:04 smithi149 ceph-mon[170394]: osdmap e229: 8 total, 8 up, 8 in 2023-09-23T20:28:04.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:04 smithi149 ceph-mon[170394]: failed to encode map e229 with expected crc 2023-09-23T20:28:04.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:04 smithi149 ceph-mon[170394]: failed to encode map e229 with expected crc 2023-09-23T20:28:04.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:04 smithi149 ceph-mon[170394]: failed to encode map e229 with expected crc 2023-09-23T20:28:04.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:04 smithi149 ceph-mon[170394]: failed to encode map e229 with expected crc 2023-09-23T20:28:04.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:04 smithi149 ceph-mon[170394]: failed to encode map e229 with expected crc 2023-09-23T20:28:04.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:04 smithi149 ceph-mon[170394]: failed to encode map e229 with expected crc 2023-09-23T20:28:04.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:04 smithi149 ceph-mon[170394]: failed to encode map e229 with expected crc 2023-09-23T20:28:04.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:04 smithi149 ceph-mon[170394]: failed to encode map e229 with expected crc 2023-09-23T20:28:04.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:04 smithi149 ceph-mon[170394]: failed to encode map e229 with expected crc 2023-09-23T20:28:04.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:04 smithi149 ceph-mon[170394]: failed to encode map e229 with expected crc 2023-09-23T20:28:04.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:04 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:28:04.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:04 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2442415396' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:28:04.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:04 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3365998805' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-09-23T20:28:04.218 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:04 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-09-23T20:28:04.218 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:04 smithi149 ceph-mon[170394]: Client client.admin marked osd.6 out, while it was still marked up 2023-09-23T20:28:04.218 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:04 smithi149 ceph-mon[170394]: 1.4 scrub starts 2023-09-23T20:28:04.218 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:04 smithi149 ceph-mon[170394]: 1.4 scrub ok 2023-09-23T20:28:04.218 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:04 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2023-09-23T20:28:04.218 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:04 smithi149 ceph-mon[170394]: osdmap e230: 8 total, 8 up, 7 in 2023-09-23T20:28:04.237 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:28:04.248 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:04 smithi018 ceph-mon[184144]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:28:04.249 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:04 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:28:04.249 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:04 smithi018 ceph-mon[184144]: osdmap e229: 8 total, 8 up, 8 in 2023-09-23T20:28:04.249 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:04 smithi018 ceph-mon[184144]: failed to encode map e229 with expected crc 2023-09-23T20:28:04.249 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:04 smithi018 ceph-mon[184144]: failed to encode map e229 with expected crc 2023-09-23T20:28:04.249 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:04 smithi018 ceph-mon[184144]: failed to encode map e229 with expected crc 2023-09-23T20:28:04.249 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:04 smithi018 ceph-mon[184144]: failed to encode map e229 with expected crc 2023-09-23T20:28:04.249 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:04 smithi018 ceph-mon[184144]: failed to encode map e229 with expected crc 2023-09-23T20:28:04.249 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:04 smithi018 ceph-mon[184144]: failed to encode map e229 with expected crc 2023-09-23T20:28:04.249 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:04 smithi018 ceph-mon[184144]: failed to encode map e229 with expected crc 2023-09-23T20:28:04.249 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:04 smithi018 ceph-mon[184144]: failed to encode map e229 with expected crc 2023-09-23T20:28:04.250 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:04 smithi018 ceph-mon[184144]: failed to encode map e229 with expected crc 2023-09-23T20:28:04.250 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:04 smithi018 ceph-mon[184144]: failed to encode map e229 with expected crc 2023-09-23T20:28:04.250 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:04 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:28:04.250 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:04 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2442415396' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:28:04.250 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:04 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3365998805' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-09-23T20:28:04.250 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:04 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-09-23T20:28:04.250 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:04 smithi018 ceph-mon[184144]: Client client.admin marked osd.6 out, while it was still marked up 2023-09-23T20:28:04.250 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:04 smithi018 ceph-mon[184144]: 1.4 scrub starts 2023-09-23T20:28:04.250 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:04 smithi018 ceph-mon[184144]: 1.4 scrub ok 2023-09-23T20:28:04.250 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:04 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2023-09-23T20:28:04.250 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:04 smithi018 ceph-mon[184144]: osdmap e230: 8 total, 8 up, 7 in 2023-09-23T20:28:04.251 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:04 smithi018 ceph-mon[179647]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:28:04.251 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:04 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:28:04.251 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:04 smithi018 ceph-mon[179647]: osdmap e229: 8 total, 8 up, 8 in 2023-09-23T20:28:04.251 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:04 smithi018 ceph-mon[179647]: failed to encode map e229 with expected crc 2023-09-23T20:28:04.251 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:04 smithi018 ceph-mon[179647]: failed to encode map e229 with expected crc 2023-09-23T20:28:04.251 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:04 smithi018 ceph-mon[179647]: failed to encode map e229 with expected crc 2023-09-23T20:28:04.251 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:04 smithi018 ceph-mon[179647]: failed to encode map e229 with expected crc 2023-09-23T20:28:04.251 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:04 smithi018 ceph-mon[179647]: failed to encode map e229 with expected crc 2023-09-23T20:28:04.251 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:04 smithi018 ceph-mon[179647]: failed to encode map e229 with expected crc 2023-09-23T20:28:04.251 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:04 smithi018 ceph-mon[179647]: failed to encode map e229 with expected crc 2023-09-23T20:28:04.253 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:04 smithi018 ceph-mon[179647]: failed to encode map e229 with expected crc 2023-09-23T20:28:04.253 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:04 smithi018 ceph-mon[179647]: failed to encode map e229 with expected crc 2023-09-23T20:28:04.253 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:04 smithi018 ceph-mon[179647]: failed to encode map e229 with expected crc 2023-09-23T20:28:04.253 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:04 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:28:04.253 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:04 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2442415396' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:28:04.254 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:04 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3365998805' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-09-23T20:28:04.254 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:04 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-09-23T20:28:04.254 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:04 smithi018 ceph-mon[179647]: Client client.admin marked osd.6 out, while it was still marked up 2023-09-23T20:28:04.254 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:04 smithi018 ceph-mon[179647]: 1.4 scrub starts 2023-09-23T20:28:04.254 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:04 smithi018 ceph-mon[179647]: 1.4 scrub ok 2023-09-23T20:28:04.254 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:04 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2023-09-23T20:28:04.254 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:04 smithi018 ceph-mon[179647]: osdmap e230: 8 total, 8 up, 7 in 2023-09-23T20:28:04.265 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:04.501 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:28:04.504 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:04.504 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:28:04.571 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 42 16 68499 68483 101.859 29.75 0.00321965 0.0096942 2023-09-23T20:28:04.605 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:28:04.634 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:04.719 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:28:04.722 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:04.722 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:28:04.818 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:28:04.823 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:28:04.849 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:04.984 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:28:04.988 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:04.988 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:28:05.089 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:28:05.115 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:05.154 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:28:05.157 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:05.158 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:28:05.259 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:28:05.289 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:28:04 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:28:04] "GET /metrics HTTP/1.1" 200 36518 "" "Prometheus/2.43.0" 2023-09-23T20:28:05.289 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:05 smithi018 ceph-mon[184144]: pgmap v504: 109 pgs: 1 remapped+peering, 1 unknown, 107 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2023-09-23T20:28:05.289 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:05 smithi018 ceph-mon[184144]: failed to encode map e230 with expected crc 2023-09-23T20:28:05.289 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:05 smithi018 ceph-mon[184144]: failed to encode map e230 with expected crc 2023-09-23T20:28:05.289 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:05 smithi018 ceph-mon[184144]: failed to encode map e230 with expected crc 2023-09-23T20:28:05.289 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:05 smithi018 ceph-mon[184144]: failed to encode map e230 with expected crc 2023-09-23T20:28:05.290 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:05 smithi018 ceph-mon[184144]: failed to encode map e230 with expected crc 2023-09-23T20:28:05.290 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:05 smithi018 ceph-mon[184144]: failed to encode map e230 with expected crc 2023-09-23T20:28:05.290 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:05 smithi018 ceph-mon[184144]: failed to encode map e230 with expected crc 2023-09-23T20:28:05.290 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:05 smithi018 ceph-mon[184144]: failed to encode map e230 with expected crc 2023-09-23T20:28:05.290 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:05 smithi018 ceph-mon[184144]: failed to encode map e230 with expected crc 2023-09-23T20:28:05.290 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:05 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:28:05.290 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:05 smithi018 ceph-mon[184144]: osdmap e231: 8 total, 8 up, 7 in 2023-09-23T20:28:05.290 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:05 smithi018 ceph-mon[179647]: pgmap v504: 109 pgs: 1 remapped+peering, 1 unknown, 107 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2023-09-23T20:28:05.290 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:05 smithi018 ceph-mon[179647]: failed to encode map e230 with expected crc 2023-09-23T20:28:05.291 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:05 smithi018 ceph-mon[179647]: failed to encode map e230 with expected crc 2023-09-23T20:28:05.291 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:05 smithi018 ceph-mon[179647]: failed to encode map e230 with expected crc 2023-09-23T20:28:05.291 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:05 smithi018 ceph-mon[179647]: failed to encode map e230 with expected crc 2023-09-23T20:28:05.291 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:05 smithi018 ceph-mon[179647]: failed to encode map e230 with expected crc 2023-09-23T20:28:05.291 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:05 smithi018 ceph-mon[179647]: failed to encode map e230 with expected crc 2023-09-23T20:28:05.291 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:05 smithi018 ceph-mon[179647]: failed to encode map e230 with expected crc 2023-09-23T20:28:05.291 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:05 smithi018 ceph-mon[179647]: failed to encode map e230 with expected crc 2023-09-23T20:28:05.291 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:05 smithi018 ceph-mon[179647]: failed to encode map e230 with expected crc 2023-09-23T20:28:05.291 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:05 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:28:05.291 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:05 smithi018 ceph-mon[179647]: osdmap e231: 8 total, 8 up, 7 in 2023-09-23T20:28:05.293 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:05.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:05 smithi149 ceph-mon[170394]: pgmap v504: 109 pgs: 1 remapped+peering, 1 unknown, 107 active+clean; 4.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2023-09-23T20:28:05.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:05 smithi149 ceph-mon[170394]: failed to encode map e230 with expected crc 2023-09-23T20:28:05.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:05 smithi149 ceph-mon[170394]: failed to encode map e230 with expected crc 2023-09-23T20:28:05.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:05 smithi149 ceph-mon[170394]: failed to encode map e230 with expected crc 2023-09-23T20:28:05.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:05 smithi149 ceph-mon[170394]: failed to encode map e230 with expected crc 2023-09-23T20:28:05.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:05 smithi149 ceph-mon[170394]: failed to encode map e230 with expected crc 2023-09-23T20:28:05.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:05 smithi149 ceph-mon[170394]: failed to encode map e230 with expected crc 2023-09-23T20:28:05.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:05 smithi149 ceph-mon[170394]: failed to encode map e230 with expected crc 2023-09-23T20:28:05.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:05 smithi149 ceph-mon[170394]: failed to encode map e230 with expected crc 2023-09-23T20:28:05.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:05 smithi149 ceph-mon[170394]: failed to encode map e230 with expected crc 2023-09-23T20:28:05.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:05 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:28:05.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:05 smithi149 ceph-mon[170394]: osdmap e231: 8 total, 8 up, 7 in 2023-09-23T20:28:05.480 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:28:05.483 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:05.483 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:28:05.571 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 43 16 68499 68483 99.4911 0 - 0.0096942 2023-09-23T20:28:05.584 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:28:05.611 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:05.660 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T20:28:05.746 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:28:05.750 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:05.750 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:28:05.851 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:28:05.877 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:05.910 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:28:05.913 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:05.913 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:28:06.014 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:28:06.106 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:06.177 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:28:06.187 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:06.187 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:06 smithi149 ceph-mon[170394]: failed to encode map e231 with expected crc 2023-09-23T20:28:06.187 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:06 smithi149 ceph-mon[170394]: failed to encode map e231 with expected crc 2023-09-23T20:28:06.188 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:06 smithi149 ceph-mon[170394]: failed to encode map e231 with expected crc 2023-09-23T20:28:06.188 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:06 smithi149 ceph-mon[170394]: failed to encode map e231 with expected crc 2023-09-23T20:28:06.188 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:06 smithi149 ceph-mon[170394]: failed to encode map e231 with expected crc 2023-09-23T20:28:06.188 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:06 smithi149 ceph-mon[170394]: failed to encode map e231 with expected crc 2023-09-23T20:28:06.188 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:06 smithi149 ceph-mon[170394]: failed to encode map e231 with expected crc 2023-09-23T20:28:06.188 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:06 smithi149 ceph-mon[170394]: failed to encode map e231 with expected crc 2023-09-23T20:28:06.188 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:06 smithi149 ceph-mon[170394]: failed to encode map e231 with expected crc 2023-09-23T20:28:06.188 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:06 smithi149 ceph-mon[170394]: osdmap e232: 8 total, 8 up, 7 in 2023-09-23T20:28:06.189 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:28:06.289 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:28:06.371 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:06 smithi018 ceph-mon[184144]: failed to encode map e231 with expected crc 2023-09-23T20:28:06.371 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:06 smithi018 ceph-mon[184144]: failed to encode map e231 with expected crc 2023-09-23T20:28:06.371 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:06 smithi018 ceph-mon[184144]: failed to encode map e231 with expected crc 2023-09-23T20:28:06.371 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:06 smithi018 ceph-mon[184144]: failed to encode map e231 with expected crc 2023-09-23T20:28:06.371 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:06 smithi018 ceph-mon[184144]: failed to encode map e231 with expected crc 2023-09-23T20:28:06.371 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:06 smithi018 ceph-mon[184144]: failed to encode map e231 with expected crc 2023-09-23T20:28:06.371 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:06 smithi018 ceph-mon[184144]: failed to encode map e231 with expected crc 2023-09-23T20:28:06.371 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:06 smithi018 ceph-mon[184144]: failed to encode map e231 with expected crc 2023-09-23T20:28:06.371 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:06 smithi018 ceph-mon[184144]: failed to encode map e231 with expected crc 2023-09-23T20:28:06.371 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:06 smithi018 ceph-mon[184144]: osdmap e232: 8 total, 8 up, 7 in 2023-09-23T20:28:06.373 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:06 smithi018 ceph-mon[179647]: failed to encode map e231 with expected crc 2023-09-23T20:28:06.373 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:06 smithi018 ceph-mon[179647]: failed to encode map e231 with expected crc 2023-09-23T20:28:06.373 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:06 smithi018 ceph-mon[179647]: failed to encode map e231 with expected crc 2023-09-23T20:28:06.373 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:06 smithi018 ceph-mon[179647]: failed to encode map e231 with expected crc 2023-09-23T20:28:06.373 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:06 smithi018 ceph-mon[179647]: failed to encode map e231 with expected crc 2023-09-23T20:28:06.373 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:06 smithi018 ceph-mon[179647]: failed to encode map e231 with expected crc 2023-09-23T20:28:06.373 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:06 smithi018 ceph-mon[179647]: failed to encode map e231 with expected crc 2023-09-23T20:28:06.373 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:06 smithi018 ceph-mon[179647]: failed to encode map e231 with expected crc 2023-09-23T20:28:06.373 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:06 smithi018 ceph-mon[179647]: failed to encode map e231 with expected crc 2023-09-23T20:28:06.373 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:06 smithi018 ceph-mon[179647]: osdmap e232: 8 total, 8 up, 7 in 2023-09-23T20:28:06.380 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:06.464 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:28:06.492 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2023-09-23T20:28:06.492 INFO:teuthology.orchestra.run.smithi018.stdout: "ops": [ 2023-09-23T20:28:06.492 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:28:06.492 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.45161.0:68677 6.0 6:00d17762:::benchmark_data_smithi018_270558_object68676:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e232)", 2023-09-23T20:28:06.492 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:28:06.473309+0000", 2023-09-23T20:28:06.493 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.015925073000000001, 2023-09-23T20:28:06.493 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.015964069000000001, 2023-09-23T20:28:06.493 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:28:06.493 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:28:06.493 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:28:06.493 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.45161", 2023-09-23T20:28:06.493 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3795614987", 2023-09-23T20:28:06.493 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 68677 2023-09-23T20:28:06.493 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:28:06.493 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:28:06.493 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:28:06.493 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:28:06.494 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:06.473309+0000", 2023-09-23T20:28:06.494 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:28:06.494 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:28:06.494 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:28:06.494 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:28:06.494 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:06.473309+0000", 2023-09-23T20:28:06.494 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:28:06.494 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:28:06.494 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:28:06.494 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:28:06.494 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:06.473308+0000", 2023-09-23T20:28:06.495 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:28:06.495 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:28:06.495 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:28:06.495 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:28:06.495 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:06.473332+0000", 2023-09-23T20:28:06.495 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.4769e-05 2023-09-23T20:28:06.495 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:28:06.495 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:28:06.495 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:28:06.495 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:06.473347+0000", 2023-09-23T20:28:06.495 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.4714e-05 2023-09-23T20:28:06.495 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:28:06.496 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:28:06.496 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:28:06.496 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:06.473355+0000", 2023-09-23T20:28:06.496 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.4039999999999998e-06 2023-09-23T20:28:06.496 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:28:06.496 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:28:06.496 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:28:06.496 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:06.473400+0000", 2023-09-23T20:28:06.496 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.5256999999999997e-05 2023-09-23T20:28:06.496 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:28:06.496 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:28:06.496 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:28:06.497 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:06.473560+0000", 2023-09-23T20:28:06.497 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00015971099999999999 2023-09-23T20:28:06.497 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:28:06.497 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:28:06.497 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 2,3", 2023-09-23T20:28:06.497 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:06.473708+0000", 2023-09-23T20:28:06.497 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000148875 2023-09-23T20:28:06.497 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:28:06.497 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:28:06.497 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:28:06.497 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:06.482251+0000", 2023-09-23T20:28:06.497 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0085423719999999995 2023-09-23T20:28:06.498 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:28:06.498 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:28:06.498 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:28:06.498 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:06.487265+0000", 2023-09-23T20:28:06.498 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0050139060000000003 2023-09-23T20:28:06.498 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:28:06.498 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:28:06.498 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:28:06.498 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:06.489221+0000", 2023-09-23T20:28:06.498 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0019563699999999998 2023-09-23T20:28:06.498 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:28:06.498 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:28:06.499 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:28:06.499 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:06.489285+0000", 2023-09-23T20:28:06.499 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.4323000000000001e-05 2023-09-23T20:28:06.499 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:28:06.499 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:28:06.499 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:28:06.499 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:28:06.499 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:28:06.499 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.45161.0:68679 6.0 6:090b7eb8:::benchmark_data_smithi018_270558_object68678:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e232)", 2023-09-23T20:28:06.499 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:28:06.473411+0000", 2023-09-23T20:28:06.499 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.015822848, 2023-09-23T20:28:06.499 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.016175119000000002, 2023-09-23T20:28:06.500 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:28:06.500 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:28:06.500 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:28:06.500 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.45161", 2023-09-23T20:28:06.500 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3795614987", 2023-09-23T20:28:06.500 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 68679 2023-09-23T20:28:06.500 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:28:06.500 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:28:06.500 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:28:06.500 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:28:06.500 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:06.473411+0000", 2023-09-23T20:28:06.500 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:28:06.501 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:28:06.501 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:28:06.501 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:28:06.501 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:06.473411+0000", 2023-09-23T20:28:06.501 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:28:06.501 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:28:06.501 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:28:06.501 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:28:06.501 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:06.473411+0000", 2023-09-23T20:28:06.501 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:28:06.501 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:28:06.501 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:28:06.502 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:28:06.502 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:06.473432+0000", 2023-09-23T20:28:06.502 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.0882000000000001e-05 2023-09-23T20:28:06.502 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:28:06.502 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:28:06.502 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:28:06.502 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:06.473440+0000", 2023-09-23T20:28:06.502 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.9440000000000005e-06 2023-09-23T20:28:06.502 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:28:06.502 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:28:06.505 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:28:06.505 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:06.473446+0000", 2023-09-23T20:28:06.505 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.0430000000000004e-06 2023-09-23T20:28:06.505 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:28:06.505 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:28:06.505 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:28:06.505 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:06.476340+0000", 2023-09-23T20:28:06.505 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0028947690000000002 2023-09-23T20:28:06.505 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:28:06.505 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:28:06.506 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:28:06.506 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:06.476493+0000", 2023-09-23T20:28:06.506 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000152442 2023-09-23T20:28:06.506 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:28:06.506 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:28:06.506 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 2,3", 2023-09-23T20:28:06.506 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:06.476639+0000", 2023-09-23T20:28:06.506 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00014632600000000001 2023-09-23T20:28:06.506 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:28:06.506 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:28:06.506 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:28:06.507 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:06.487297+0000", 2023-09-23T20:28:06.507 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.010657994 2023-09-23T20:28:06.507 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:28:06.507 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:28:06.507 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:28:06.507 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:28:06.507 INFO:teuthology.orchestra.run.smithi018.stdout: ], 2023-09-23T20:28:06.507 INFO:teuthology.orchestra.run.smithi018.stdout: "num_ops": 2 2023-09-23T20:28:06.507 INFO:teuthology.orchestra.run.smithi018.stdout:} 2023-09-23T20:28:06.536 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:28:06.548 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:06.549 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:28:06.571 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 44 16 68746 68730 97.5815 7.71875 0.0301381 0.0102409 2023-09-23T20:28:06.588 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:28:06.650 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:28:06.747 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:06.961 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:28:06.970 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:06.971 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:28:07.072 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:28:07.076 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T20:28:07.160 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:07.233 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:07 smithi018 ceph-mon[179647]: pgmap v507: 109 pgs: 4 peering, 2 remapped, 2 active+undersized+remapped, 9 remapped+peering, 2 unknown, 90 active+clean; 4.1 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 62 KiB/s rd, 50 MiB/s wr, 841 op/s; 118 KiB/s, 1 objects/s recovering 2023-09-23T20:28:07.233 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:07 smithi018 ceph-mon[179647]: 1.9 scrub starts 2023-09-23T20:28:07.233 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:07 smithi018 ceph-mon[179647]: 1.9 scrub ok 2023-09-23T20:28:07.233 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:07 smithi018 ceph-mon[179647]: failed to encode map e232 with expected crc 2023-09-23T20:28:07.233 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:07 smithi018 ceph-mon[179647]: failed to encode map e232 with expected crc 2023-09-23T20:28:07.234 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:07 smithi018 ceph-mon[179647]: failed to encode map e232 with expected crc 2023-09-23T20:28:07.234 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:07 smithi018 ceph-mon[179647]: failed to encode map e232 with expected crc 2023-09-23T20:28:07.234 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:07 smithi018 ceph-mon[179647]: failed to encode map e232 with expected crc 2023-09-23T20:28:07.234 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:07 smithi018 ceph-mon[179647]: failed to encode map e232 with expected crc 2023-09-23T20:28:07.234 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:07 smithi018 ceph-mon[179647]: failed to encode map e232 with expected crc 2023-09-23T20:28:07.234 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:07 smithi018 ceph-mon[179647]: failed to encode map e232 with expected crc 2023-09-23T20:28:07.234 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:07 smithi018 ceph-mon[179647]: failed to encode map e232 with expected crc 2023-09-23T20:28:07.234 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:07 smithi018 ceph-mon[179647]: failed to encode map e232 with expected crc 2023-09-23T20:28:07.234 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:07 smithi018 ceph-mon[179647]: Health check failed: Reduced data availability: 3 pgs peering (PG_AVAILABILITY) 2023-09-23T20:28:07.234 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:07 smithi018 ceph-mon[179647]: osdmap e233: 8 total, 8 up, 7 in 2023-09-23T20:28:07.235 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:07 smithi018 ceph-mon[184144]: pgmap v507: 109 pgs: 4 peering, 2 remapped, 2 active+undersized+remapped, 9 remapped+peering, 2 unknown, 90 active+clean; 4.1 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 62 KiB/s rd, 50 MiB/s wr, 841 op/s; 118 KiB/s, 1 objects/s recovering 2023-09-23T20:28:07.235 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:07 smithi018 ceph-mon[184144]: 1.9 scrub starts 2023-09-23T20:28:07.235 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:07 smithi018 ceph-mon[184144]: 1.9 scrub ok 2023-09-23T20:28:07.235 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:07 smithi018 ceph-mon[184144]: failed to encode map e232 with expected crc 2023-09-23T20:28:07.235 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:07 smithi018 ceph-mon[184144]: failed to encode map e232 with expected crc 2023-09-23T20:28:07.235 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:07 smithi018 ceph-mon[184144]: failed to encode map e232 with expected crc 2023-09-23T20:28:07.235 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:07 smithi018 ceph-mon[184144]: failed to encode map e232 with expected crc 2023-09-23T20:28:07.235 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:07 smithi018 ceph-mon[184144]: failed to encode map e232 with expected crc 2023-09-23T20:28:07.235 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:07 smithi018 ceph-mon[184144]: failed to encode map e232 with expected crc 2023-09-23T20:28:07.235 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:07 smithi018 ceph-mon[184144]: failed to encode map e232 with expected crc 2023-09-23T20:28:07.235 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:07 smithi018 ceph-mon[184144]: failed to encode map e232 with expected crc 2023-09-23T20:28:07.236 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:07 smithi018 ceph-mon[184144]: failed to encode map e232 with expected crc 2023-09-23T20:28:07.236 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:07 smithi018 ceph-mon[184144]: failed to encode map e232 with expected crc 2023-09-23T20:28:07.236 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:07 smithi018 ceph-mon[184144]: Health check failed: Reduced data availability: 3 pgs peering (PG_AVAILABILITY) 2023-09-23T20:28:07.236 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:07 smithi018 ceph-mon[184144]: osdmap e233: 8 total, 8 up, 7 in 2023-09-23T20:28:07.319 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:28:07.320 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:07.321 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:28:07.422 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:28:07.453 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:07 smithi149 ceph-mon[170394]: pgmap v507: 109 pgs: 4 peering, 2 remapped, 2 active+undersized+remapped, 9 remapped+peering, 2 unknown, 90 active+clean; 4.1 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 62 KiB/s rd, 50 MiB/s wr, 841 op/s; 118 KiB/s, 1 objects/s recovering 2023-09-23T20:28:07.453 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:07 smithi149 ceph-mon[170394]: 1.9 scrub starts 2023-09-23T20:28:07.453 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:07 smithi149 ceph-mon[170394]: 1.9 scrub ok 2023-09-23T20:28:07.453 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:07 smithi149 ceph-mon[170394]: failed to encode map e232 with expected crc 2023-09-23T20:28:07.453 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:07 smithi149 ceph-mon[170394]: failed to encode map e232 with expected crc 2023-09-23T20:28:07.453 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:07 smithi149 ceph-mon[170394]: failed to encode map e232 with expected crc 2023-09-23T20:28:07.453 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:07 smithi149 ceph-mon[170394]: failed to encode map e232 with expected crc 2023-09-23T20:28:07.453 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:07 smithi149 ceph-mon[170394]: failed to encode map e232 with expected crc 2023-09-23T20:28:07.454 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:07 smithi149 ceph-mon[170394]: failed to encode map e232 with expected crc 2023-09-23T20:28:07.454 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:07 smithi149 ceph-mon[170394]: failed to encode map e232 with expected crc 2023-09-23T20:28:07.454 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:07 smithi149 ceph-mon[170394]: failed to encode map e232 with expected crc 2023-09-23T20:28:07.454 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:07 smithi149 ceph-mon[170394]: failed to encode map e232 with expected crc 2023-09-23T20:28:07.454 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:07 smithi149 ceph-mon[170394]: failed to encode map e232 with expected crc 2023-09-23T20:28:07.454 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:07 smithi149 ceph-mon[170394]: Health check failed: Reduced data availability: 3 pgs peering (PG_AVAILABILITY) 2023-09-23T20:28:07.454 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:07 smithi149 ceph-mon[170394]: osdmap e233: 8 total, 8 up, 7 in 2023-09-23T20:28:07.456 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:07.518 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:28:07.522 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:07.522 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:28:07.573 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 45 16 69388 69372 96.3013 40.125 0.0176734 0.0103766 2023-09-23T20:28:07.623 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:28:07.701 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:07.834 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:28:07.885 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:28:07.898 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:07.899 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:28:07.999 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:28:08.126 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:08.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:08 smithi018 ceph-mon[184144]: 1.e scrub starts 2023-09-23T20:28:08.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:08 smithi018 ceph-mon[184144]: failed to encode map e233 with expected crc 2023-09-23T20:28:08.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:08 smithi018 ceph-mon[184144]: failed to encode map e233 with expected crc 2023-09-23T20:28:08.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:08 smithi018 ceph-mon[184144]: failed to encode map e233 with expected crc 2023-09-23T20:28:08.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:08 smithi018 ceph-mon[184144]: failed to encode map e233 with expected crc 2023-09-23T20:28:08.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:08 smithi018 ceph-mon[184144]: failed to encode map e233 with expected crc 2023-09-23T20:28:08.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:08 smithi018 ceph-mon[184144]: osdmap e234: 8 total, 8 up, 7 in 2023-09-23T20:28:08.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:08 smithi018 ceph-mon[179647]: 1.e scrub starts 2023-09-23T20:28:08.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:08 smithi018 ceph-mon[179647]: failed to encode map e233 with expected crc 2023-09-23T20:28:08.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:08 smithi018 ceph-mon[179647]: failed to encode map e233 with expected crc 2023-09-23T20:28:08.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:08 smithi018 ceph-mon[179647]: failed to encode map e233 with expected crc 2023-09-23T20:28:08.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:08 smithi018 ceph-mon[179647]: failed to encode map e233 with expected crc 2023-09-23T20:28:08.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:08 smithi018 ceph-mon[179647]: failed to encode map e233 with expected crc 2023-09-23T20:28:08.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:08 smithi018 ceph-mon[179647]: osdmap e234: 8 total, 8 up, 7 in 2023-09-23T20:28:08.398 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:28:08.403 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:08.403 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:28:08.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:08 smithi149 ceph-mon[170394]: 1.e scrub starts 2023-09-23T20:28:08.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:08 smithi149 ceph-mon[170394]: failed to encode map e233 with expected crc 2023-09-23T20:28:08.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:08 smithi149 ceph-mon[170394]: failed to encode map e233 with expected crc 2023-09-23T20:28:08.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:08 smithi149 ceph-mon[170394]: failed to encode map e233 with expected crc 2023-09-23T20:28:08.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:08 smithi149 ceph-mon[170394]: failed to encode map e233 with expected crc 2023-09-23T20:28:08.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:08 smithi149 ceph-mon[170394]: failed to encode map e233 with expected crc 2023-09-23T20:28:08.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:08 smithi149 ceph-mon[170394]: osdmap e234: 8 total, 8 up, 7 in 2023-09-23T20:28:08.504 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:28:08.557 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:08.573 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 46 16 70041 70025 95.0954 40.8125 0.018992 0.0105091 2023-09-23T20:28:08.619 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:28:08.627 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:08.627 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:28:08.728 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:28:08.770 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:08.823 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:28:08.831 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:08.832 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:28:08.932 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:28:08.957 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:28:09.064 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:09.086 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:09 smithi149 ceph-mon[170394]: pgmap v510: 109 pgs: 5 peering, 2 remapped, 1 active+undersized+remapped, 13 remapped+peering, 1 unknown, 87 active+clean; 4.2 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 62 KiB/s rd, 48 MiB/s wr, 814 op/s; 119 KiB/s, 2 objects/s recovering 2023-09-23T20:28:09.086 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:09 smithi149 ceph-mon[170394]: failed to encode map e234 with expected crc 2023-09-23T20:28:09.086 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:09 smithi149 ceph-mon[170394]: failed to encode map e234 with expected crc 2023-09-23T20:28:09.086 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:09 smithi149 ceph-mon[170394]: failed to encode map e234 with expected crc 2023-09-23T20:28:09.086 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:09 smithi149 ceph-mon[170394]: failed to encode map e234 with expected crc 2023-09-23T20:28:09.086 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:09 smithi149 ceph-mon[170394]: failed to encode map e234 with expected crc 2023-09-23T20:28:09.086 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:09 smithi149 ceph-mon[170394]: failed to encode map e234 with expected crc 2023-09-23T20:28:09.087 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:09 smithi149 ceph-mon[170394]: failed to encode map e234 with expected crc 2023-09-23T20:28:09.087 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:09 smithi149 ceph-mon[170394]: failed to encode map e234 with expected crc 2023-09-23T20:28:09.087 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:09 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:28:09.087 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:09 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:28:09.396 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:09 smithi018 ceph-mon[184144]: pgmap v510: 109 pgs: 5 peering, 2 remapped, 1 active+undersized+remapped, 13 remapped+peering, 1 unknown, 87 active+clean; 4.2 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 62 KiB/s rd, 48 MiB/s wr, 814 op/s; 119 KiB/s, 2 objects/s recovering 2023-09-23T20:28:09.397 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:09 smithi018 ceph-mon[184144]: failed to encode map e234 with expected crc 2023-09-23T20:28:09.397 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:09 smithi018 ceph-mon[184144]: failed to encode map e234 with expected crc 2023-09-23T20:28:09.397 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:09 smithi018 ceph-mon[184144]: failed to encode map e234 with expected crc 2023-09-23T20:28:09.397 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:09 smithi018 ceph-mon[184144]: failed to encode map e234 with expected crc 2023-09-23T20:28:09.397 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:09 smithi018 ceph-mon[184144]: failed to encode map e234 with expected crc 2023-09-23T20:28:09.397 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:09 smithi018 ceph-mon[184144]: failed to encode map e234 with expected crc 2023-09-23T20:28:09.397 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:09 smithi018 ceph-mon[184144]: failed to encode map e234 with expected crc 2023-09-23T20:28:09.397 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:09 smithi018 ceph-mon[184144]: failed to encode map e234 with expected crc 2023-09-23T20:28:09.397 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:09 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:28:09.398 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:09 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:28:09.398 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:09 smithi018 ceph-mon[179647]: pgmap v510: 109 pgs: 5 peering, 2 remapped, 1 active+undersized+remapped, 13 remapped+peering, 1 unknown, 87 active+clean; 4.2 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 62 KiB/s rd, 48 MiB/s wr, 814 op/s; 119 KiB/s, 2 objects/s recovering 2023-09-23T20:28:09.398 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:09 smithi018 ceph-mon[179647]: failed to encode map e234 with expected crc 2023-09-23T20:28:09.398 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:09 smithi018 ceph-mon[179647]: failed to encode map e234 with expected crc 2023-09-23T20:28:09.398 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:09 smithi018 ceph-mon[179647]: failed to encode map e234 with expected crc 2023-09-23T20:28:09.398 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:09 smithi018 ceph-mon[179647]: failed to encode map e234 with expected crc 2023-09-23T20:28:09.398 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:09 smithi018 ceph-mon[179647]: failed to encode map e234 with expected crc 2023-09-23T20:28:09.398 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:09 smithi018 ceph-mon[179647]: failed to encode map e234 with expected crc 2023-09-23T20:28:09.398 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:09 smithi018 ceph-mon[179647]: failed to encode map e234 with expected crc 2023-09-23T20:28:09.398 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:09 smithi018 ceph-mon[179647]: failed to encode map e234 with expected crc 2023-09-23T20:28:09.398 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:09 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:28:09.399 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:09 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:28:09.449 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:28:09.458 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:09.458 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:28:09.559 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:28:09.566 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 4, 5, 7, 3, 2] out_osds: [6] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-23T20:28:09.566 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:28:09.566 INFO:tasks.thrashosds.thrasher:Killing osd 1, live_osds are [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-23T20:28:09.566 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2023-09-23T20:28:09.567 DEBUG:teuthology.orchestra.run.smithi018:> sudo systemctl stop ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7@osd.1 2023-09-23T20:28:09.573 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 47 16 70730 70714 93.9887 43.0625 0.0138398 0.0106291 2023-09-23T20:28:09.592 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:09.643 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:28:09.646 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:09.647 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:28:09.732 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 23 20:28:09 smithi018 systemd[1]: Stopping Ceph osd.1 for a3b3b17e-5a4c-11ee-8db4-212e2dc638e7... 2023-09-23T20:28:09.748 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:28:09.779 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:09.828 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:28:09.833 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:09.833 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:28:09.934 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:28:09.974 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:10.029 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:28:10.035 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:10.035 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:28:10.136 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:28:10.159 INFO:teuthology.orchestra.run.smithi018.stderr:nodeep-scrub is unset 2023-09-23T20:28:10.172 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:10.222 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:28:10.224 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:10 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2672734324' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:28:10.224 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:10 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:28:10.225 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:10.226 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:28:10.326 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:28:10.365 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:10 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2672734324' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:28:10.365 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:10 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:28:10.366 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:10 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2672734324' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:28:10.366 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:10 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:28:10.366 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:10.438 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:28:10.441 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:10.441 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:28:10.542 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:28:10.573 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 48 16 71375 71359 92.8708 40.3125 0.0145365 0.0107611 2023-09-23T20:28:10.635 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:11.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:11 smithi018 ceph-mon[179647]: pgmap v512: 109 pgs: 1 active+recovering+undersized+remapped, 5 peering, 2 remapped, 1 active+undersized+remapped, 12 remapped+peering, 1 unknown, 87 active+clean; 4.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 18 KiB/s rd, 40 MiB/s wr, 645 op/s; 4095/206343 objects misplaced (1.985%); 862 KiB/s, 13 objects/s recovering 2023-09-23T20:28:11.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:11 smithi018 ceph-mon[179647]: 1.5 scrub starts 2023-09-23T20:28:11.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:11 smithi018 ceph-mon[179647]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:28:11.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:11 smithi018 ceph-mon[179647]: failed to encode map e235 with expected crc 2023-09-23T20:28:11.394 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:11 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:28:11.394 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:11 smithi018 ceph-mon[179647]: osdmap e235: 8 total, 8 up, 7 in 2023-09-23T20:28:11.394 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:11 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:28:11.394 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:11 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:28:11.399 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:11 smithi018 ceph-mon[184144]: pgmap v512: 109 pgs: 1 active+recovering+undersized+remapped, 5 peering, 2 remapped, 1 active+undersized+remapped, 12 remapped+peering, 1 unknown, 87 active+clean; 4.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 18 KiB/s rd, 40 MiB/s wr, 645 op/s; 4095/206343 objects misplaced (1.985%); 862 KiB/s, 13 objects/s recovering 2023-09-23T20:28:11.399 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:11 smithi018 ceph-mon[184144]: 1.5 scrub starts 2023-09-23T20:28:11.399 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:11 smithi018 ceph-mon[184144]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:28:11.399 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:11 smithi018 ceph-mon[184144]: failed to encode map e235 with expected crc 2023-09-23T20:28:11.399 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:11 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:28:11.400 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:11 smithi018 ceph-mon[184144]: osdmap e235: 8 total, 8 up, 7 in 2023-09-23T20:28:11.400 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:11 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:28:11.400 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:11 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:28:11.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:11 smithi149 ceph-mon[170394]: pgmap v512: 109 pgs: 1 active+recovering+undersized+remapped, 5 peering, 2 remapped, 1 active+undersized+remapped, 12 remapped+peering, 1 unknown, 87 active+clean; 4.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 18 KiB/s rd, 40 MiB/s wr, 645 op/s; 4095/206343 objects misplaced (1.985%); 862 KiB/s, 13 objects/s recovering 2023-09-23T20:28:11.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:11 smithi149 ceph-mon[170394]: 1.5 scrub starts 2023-09-23T20:28:11.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:11 smithi149 ceph-mon[170394]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:28:11.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:11 smithi149 ceph-mon[170394]: failed to encode map e235 with expected crc 2023-09-23T20:28:11.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:11 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:28:11.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:11 smithi149 ceph-mon[170394]: osdmap e235: 8 total, 8 up, 7 in 2023-09-23T20:28:11.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:11 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:28:11.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:11 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:28:11.575 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 49 16 72171 72155 91.988 49.75 0.0198811 0.0108653 2023-09-23T20:28:11.757 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:28:11.762 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:11.762 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:28:11.863 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:28:11.892 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 23 20:28:11 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-1[201125]: 2023-09-23T20:28:11.768+0000 7f0777e86700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T20:28:11.893 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 23 20:28:11 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-1[201125]: 2023-09-23T20:28:11.768+0000 7f0777e86700 -1 osd.1 235 *** Got signal Terminated *** 2023-09-23T20:28:11.893 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 23 20:28:11 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-1[201125]: 2023-09-23T20:28:11.768+0000 7f0777e86700 -1 osd.1 235 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T20:28:11.966 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:12.110 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:28:12.110 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:28:12.110 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:28:12.110 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:28:12.110 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:28:12.110 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:28:12.112 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:28:12.123 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:12.123 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:28:12.131 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:28:12.179 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:28:12.186 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:28:12.186 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:28:12.186 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:28:12.186 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:28:12.186 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:28:12.186 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:28:12.186 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:28:12.186 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:28:12.212 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:12 smithi018 ceph-mon[179647]: 1.e scrub starts 2023-09-23T20:28:12.212 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:12 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/39687193' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:28:12.212 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:12 smithi018 ceph-mon[179647]: osd.1 marked itself down and dead 2023-09-23T20:28:12.212 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:12 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:28:12.212 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:12 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:28:12.212 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:12 smithi018 ceph-mon[184144]: 1.e scrub starts 2023-09-23T20:28:12.212 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:12 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/39687193' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:28:12.212 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:12 smithi018 ceph-mon[184144]: osd.1 marked itself down and dead 2023-09-23T20:28:12.213 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:12 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:28:12.213 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:12 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:28:12.224 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:28:12.255 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:12.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:12 smithi149 ceph-mon[170394]: 1.e scrub starts 2023-09-23T20:28:12.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:12 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/39687193' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:28:12.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:12 smithi149 ceph-mon[170394]: osd.1 marked itself down and dead 2023-09-23T20:28:12.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:12 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:28:12.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:12 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:28:12.575 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 50 16 72580 72564 90.66 25.5625 0.00405537 0.0109337 2023-09-23T20:28:12.845 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:28:12.848 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:12.849 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:28:12.950 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:28:12.977 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:13.139 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:28:13.143 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:13.144 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:28:13.244 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:28:13.299 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:13.391 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 23 20:28:13 smithi018 bash[276757]: ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-1 2023-09-23T20:28:13.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:13 smithi018 ceph-mon[184144]: pgmap v514: 109 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 104 active+clean; 4.3 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 29 MiB/s wr, 466 op/s; 12724/212559 objects degraded (5.986%); 8261/212559 objects misplaced (3.886%); 3.0 MiB/s, 40 objects/s recovering 2023-09-23T20:28:13.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:13 smithi018 ceph-mon[184144]: Health check failed: Degraded data redundancy: 12724/212559 objects degraded (5.986%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T20:28:13.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:13 smithi018 ceph-mon[184144]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 4 pgs peering) 2023-09-23T20:28:13.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:13 smithi018 ceph-mon[184144]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T20:28:13.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:13 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:28:13.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:13 smithi018 ceph-mon[184144]: osdmap e236: 8 total, 7 up, 7 in 2023-09-23T20:28:13.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:13 smithi018 ceph-mon[184144]: failed to encode map e236 with expected crc 2023-09-23T20:28:13.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:13 smithi018 ceph-mon[184144]: failed to encode map e236 with expected crc 2023-09-23T20:28:13.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:13 smithi018 ceph-mon[184144]: osd.1 marked itself dead as of e236 2023-09-23T20:28:13.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:13 smithi018 ceph-mon[179647]: pgmap v514: 109 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 104 active+clean; 4.3 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 29 MiB/s wr, 466 op/s; 12724/212559 objects degraded (5.986%); 8261/212559 objects misplaced (3.886%); 3.0 MiB/s, 40 objects/s recovering 2023-09-23T20:28:13.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:13 smithi018 ceph-mon[179647]: Health check failed: Degraded data redundancy: 12724/212559 objects degraded (5.986%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T20:28:13.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:13 smithi018 ceph-mon[179647]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 4 pgs peering) 2023-09-23T20:28:13.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:13 smithi018 ceph-mon[179647]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T20:28:13.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:13 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:28:13.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:13 smithi018 ceph-mon[179647]: osdmap e236: 8 total, 7 up, 7 in 2023-09-23T20:28:13.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:13 smithi018 ceph-mon[179647]: failed to encode map e236 with expected crc 2023-09-23T20:28:13.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:13 smithi018 ceph-mon[179647]: failed to encode map e236 with expected crc 2023-09-23T20:28:13.394 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:13 smithi018 ceph-mon[179647]: osd.1 marked itself dead as of e236 2023-09-23T20:28:13.437 INFO:teuthology.orchestra.run.smithi018.stdout:ERROR: (22) Invalid argument 2023-09-23T20:28:13.437 INFO:teuthology.orchestra.run.smithi018.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:28:13.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:13 smithi149 ceph-mon[170394]: pgmap v514: 109 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 104 active+clean; 4.3 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 29 MiB/s wr, 466 op/s; 12724/212559 objects degraded (5.986%); 8261/212559 objects misplaced (3.886%); 3.0 MiB/s, 40 objects/s recovering 2023-09-23T20:28:13.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:13 smithi149 ceph-mon[170394]: Health check failed: Degraded data redundancy: 12724/212559 objects degraded (5.986%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T20:28:13.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:13 smithi149 ceph-mon[170394]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 4 pgs peering) 2023-09-23T20:28:13.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:13 smithi149 ceph-mon[170394]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T20:28:13.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:13 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:28:13.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:13 smithi149 ceph-mon[170394]: osdmap e236: 8 total, 7 up, 7 in 2023-09-23T20:28:13.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:13 smithi149 ceph-mon[170394]: failed to encode map e236 with expected crc 2023-09-23T20:28:13.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:13 smithi149 ceph-mon[170394]: failed to encode map e236 with expected crc 2023-09-23T20:28:13.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:13 smithi149 ceph-mon[170394]: osd.1 marked itself dead as of e236 2023-09-23T20:28:13.570 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:28:13.574 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:13.575 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:28:13.575 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 51 16 72667 72651 88.9896 5.4375 0.00361484 0.0110699 2023-09-23T20:28:13.676 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:28:13.704 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:13.740 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:28:13.744 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:13.744 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:28:13.845 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:28:13.858 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T20:28:13.880 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:13.926 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:28:13.928 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:13.929 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:28:14.030 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:28:14.064 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:14.107 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:28:14.111 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:14.111 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:28:14.212 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:28:14.238 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:14.274 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:28:14.277 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:14.277 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:28:14.312 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:14 smithi018 ceph-mon[184144]: failed to encode map e237 with expected crc 2023-09-23T20:28:14.312 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:14 smithi018 ceph-mon[184144]: osdmap e237: 8 total, 7 up, 7 in 2023-09-23T20:28:14.312 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:14 smithi018 ceph-mon[184144]: failed to encode map e237 with expected crc 2023-09-23T20:28:14.312 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:14 smithi018 ceph-mon[184144]: failed to encode map e237 with expected crc 2023-09-23T20:28:14.312 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:14 smithi018 ceph-mon[184144]: failed to encode map e237 with expected crc 2023-09-23T20:28:14.312 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:14 smithi018 ceph-mon[184144]: failed to encode map e237 with expected crc 2023-09-23T20:28:14.312 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:14 smithi018 ceph-mon[184144]: failed to encode map e237 with expected crc 2023-09-23T20:28:14.312 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:14 smithi018 ceph-mon[184144]: failed to encode map e237 with expected crc 2023-09-23T20:28:14.312 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:14 smithi018 ceph-mon[184144]: failed to encode map e237 with expected crc 2023-09-23T20:28:14.313 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:14 smithi018 ceph-mon[184144]: failed to encode map e237 with expected crc 2023-09-23T20:28:14.313 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:14 smithi018 ceph-mon[184144]: failed to encode map e237 with expected crc 2023-09-23T20:28:14.313 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:14 smithi018 ceph-mon[179647]: failed to encode map e237 with expected crc 2023-09-23T20:28:14.313 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:14 smithi018 ceph-mon[179647]: osdmap e237: 8 total, 7 up, 7 in 2023-09-23T20:28:14.313 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:14 smithi018 ceph-mon[179647]: failed to encode map e237 with expected crc 2023-09-23T20:28:14.313 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:14 smithi018 ceph-mon[179647]: failed to encode map e237 with expected crc 2023-09-23T20:28:14.313 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:14 smithi018 ceph-mon[179647]: failed to encode map e237 with expected crc 2023-09-23T20:28:14.313 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:14 smithi018 ceph-mon[179647]: failed to encode map e237 with expected crc 2023-09-23T20:28:14.313 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:14 smithi018 ceph-mon[179647]: failed to encode map e237 with expected crc 2023-09-23T20:28:14.313 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:14 smithi018 ceph-mon[179647]: failed to encode map e237 with expected crc 2023-09-23T20:28:14.314 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:14 smithi018 ceph-mon[179647]: failed to encode map e237 with expected crc 2023-09-23T20:28:14.314 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:14 smithi018 ceph-mon[179647]: failed to encode map e237 with expected crc 2023-09-23T20:28:14.314 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:14 smithi018 ceph-mon[179647]: failed to encode map e237 with expected crc 2023-09-23T20:28:14.378 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:28:14.409 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:14.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:14 smithi149 ceph-mon[170394]: failed to encode map e237 with expected crc 2023-09-23T20:28:14.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:14 smithi149 ceph-mon[170394]: osdmap e237: 8 total, 7 up, 7 in 2023-09-23T20:28:14.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:14 smithi149 ceph-mon[170394]: failed to encode map e237 with expected crc 2023-09-23T20:28:14.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:14 smithi149 ceph-mon[170394]: failed to encode map e237 with expected crc 2023-09-23T20:28:14.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:14 smithi149 ceph-mon[170394]: failed to encode map e237 with expected crc 2023-09-23T20:28:14.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:14 smithi149 ceph-mon[170394]: failed to encode map e237 with expected crc 2023-09-23T20:28:14.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:14 smithi149 ceph-mon[170394]: failed to encode map e237 with expected crc 2023-09-23T20:28:14.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:14 smithi149 ceph-mon[170394]: failed to encode map e237 with expected crc 2023-09-23T20:28:14.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:14 smithi149 ceph-mon[170394]: failed to encode map e237 with expected crc 2023-09-23T20:28:14.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:14 smithi149 ceph-mon[170394]: failed to encode map e237 with expected crc 2023-09-23T20:28:14.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:14 smithi149 ceph-mon[170394]: failed to encode map e237 with expected crc 2023-09-23T20:28:14.501 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:28:14.504 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:14.504 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:28:14.579 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 52 16 72686 72670 87.2988 1.1875 0.0124957 0.0112884 2023-09-23T20:28:14.605 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:28:14.681 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:14.737 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:28:14.761 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:28:14.773 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:14.774 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:28:14.875 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:28:14.910 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T20:28:14.991 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:15.083 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:28:15.102 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:15.102 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:28:15.203 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:28:15.226 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:15 smithi018 ceph-mon[184144]: pgmap v517: 109 pgs: 11 peering, 1 stale+active+recovery_wait+undersized+degraded+remapped, 12 stale+active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 81 active+clean; 4.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 36 MiB/s wr, 580 op/s; 12724/214914 objects degraded (5.921%); 8038/214914 objects misplaced (3.740%); 5.5 MiB/s, 80 objects/s recovering 2023-09-23T20:28:15.226 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:15 smithi018 ceph-mon[184144]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-09-23T20:28:15.226 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:15 smithi018 ceph-mon[184144]: osdmap e238: 8 total, 7 up, 7 in 2023-09-23T20:28:15.226 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:15 smithi018 ceph-mon[184144]: failed to encode map e238 with expected crc 2023-09-23T20:28:15.226 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:15 smithi018 ceph-mon[184144]: failed to encode map e238 with expected crc 2023-09-23T20:28:15.226 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:15 smithi018 ceph-mon[184144]: failed to encode map e238 with expected crc 2023-09-23T20:28:15.226 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:15 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:28:15.226 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:15 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:28:15.227 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:15 smithi018 ceph-mon[179647]: pgmap v517: 109 pgs: 11 peering, 1 stale+active+recovery_wait+undersized+degraded+remapped, 12 stale+active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 81 active+clean; 4.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 36 MiB/s wr, 580 op/s; 12724/214914 objects degraded (5.921%); 8038/214914 objects misplaced (3.740%); 5.5 MiB/s, 80 objects/s recovering 2023-09-23T20:28:15.227 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:15 smithi018 ceph-mon[179647]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-09-23T20:28:15.227 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:15 smithi018 ceph-mon[179647]: osdmap e238: 8 total, 7 up, 7 in 2023-09-23T20:28:15.227 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:15 smithi018 ceph-mon[179647]: failed to encode map e238 with expected crc 2023-09-23T20:28:15.227 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:15 smithi018 ceph-mon[179647]: failed to encode map e238 with expected crc 2023-09-23T20:28:15.227 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:15 smithi018 ceph-mon[179647]: failed to encode map e238 with expected crc 2023-09-23T20:28:15.227 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:15 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:28:15.227 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:15 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:28:15.228 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:28:14 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:28:14] "GET /metrics HTTP/1.1" 200 36555 "" "Prometheus/2.43.0" 2023-09-23T20:28:15.313 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:15.350 DEBUG:teuthology.orchestra.run.smithi018:> sudo pkill -f 'journalctl -f -n 0 -u ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7@osd.1.service' 2023-09-23T20:28:15.366 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:28:15.384 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:28:15.394 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:15.395 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:28:15.468 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:15 smithi149 ceph-mon[170394]: pgmap v517: 109 pgs: 11 peering, 1 stale+active+recovery_wait+undersized+degraded+remapped, 12 stale+active+clean, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 81 active+clean; 4.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 36 MiB/s wr, 580 op/s; 12724/214914 objects degraded (5.921%); 8038/214914 objects misplaced (3.740%); 5.5 MiB/s, 80 objects/s recovering 2023-09-23T20:28:15.468 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:15 smithi149 ceph-mon[170394]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-09-23T20:28:15.468 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:15 smithi149 ceph-mon[170394]: osdmap e238: 8 total, 7 up, 7 in 2023-09-23T20:28:15.468 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:15 smithi149 ceph-mon[170394]: failed to encode map e238 with expected crc 2023-09-23T20:28:15.468 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:15 smithi149 ceph-mon[170394]: failed to encode map e238 with expected crc 2023-09-23T20:28:15.468 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:15 smithi149 ceph-mon[170394]: failed to encode map e238 with expected crc 2023-09-23T20:28:15.468 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:15 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:28:15.468 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:15 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:28:15.496 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:28:15.541 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T20:28:15.541 INFO:tasks.cephadm.osd.1:Stopped osd.1 2023-09-23T20:28:15.577 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 53 16 73457 73441 86.5611 48.1875 0.0160858 0.0115444 2023-09-23T20:28:15.639 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:15.852 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:28:15.866 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:15.866 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:28:15.967 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:28:16.048 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:28:16.057 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:16.133 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:28:16.136 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:16.137 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:28:16.237 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:28:16.289 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:16.416 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:28:16.433 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:16.433 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:28:16.506 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:28:16.534 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:28:16.573 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:16.577 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 54 16 74292 74276 85.9247 52.1875 0.0280688 0.011632 2023-09-23T20:28:16.631 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:28:16.634 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:16.634 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:28:16.664 INFO:teuthology.orchestra.run.smithi018.stdout:ERROR: (22) Invalid argument 2023-09-23T20:28:16.664 INFO:teuthology.orchestra.run.smithi018.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:28:16.735 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:28:16.779 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:16.827 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:28:16.839 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:16.840 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:28:16.940 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:28:17.023 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:17.089 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:28:17.096 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:17.096 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:28:17.197 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:28:17.310 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:17.448 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:28:17.454 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:17 smithi018 ceph-mon[184144]: pgmap v519: 109 pgs: 11 peering, 1 stale+active+recovery_wait+undersized+degraded+remapped, 9 stale+active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded, 7 active+undersized, 2 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded+remapped, 74 active+clean; 4.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 33 MiB/s wr, 527 op/s; 8736/216810 objects degraded (4.029%); 12346/216810 objects misplaced (5.694%); 4.9 MiB/s, 70 objects/s recovering 2023-09-23T20:28:17.454 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:17 smithi018 ceph-mon[184144]: 1.5 deep-scrub starts 2023-09-23T20:28:17.454 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:17 smithi018 ceph-mon[179647]: pgmap v519: 109 pgs: 11 peering, 1 stale+active+recovery_wait+undersized+degraded+remapped, 9 stale+active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded, 7 active+undersized, 2 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded+remapped, 74 active+clean; 4.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 33 MiB/s wr, 527 op/s; 8736/216810 objects degraded (4.029%); 12346/216810 objects misplaced (5.694%); 4.9 MiB/s, 70 objects/s recovering 2023-09-23T20:28:17.454 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:17 smithi018 ceph-mon[179647]: 1.5 deep-scrub starts 2023-09-23T20:28:17.455 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:17.455 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:28:17.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:17 smithi149 ceph-mon[170394]: pgmap v519: 109 pgs: 11 peering, 1 stale+active+recovery_wait+undersized+degraded+remapped, 9 stale+active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded, 7 active+undersized, 2 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded+remapped, 74 active+clean; 4.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 33 MiB/s wr, 527 op/s; 8736/216810 objects degraded (4.029%); 12346/216810 objects misplaced (5.694%); 4.9 MiB/s, 70 objects/s recovering 2023-09-23T20:28:17.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:17 smithi149 ceph-mon[170394]: 1.5 deep-scrub starts 2023-09-23T20:28:17.493 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T20:28:17.557 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:28:17.577 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 55 16 75152 75136 85.3398 53.75 0.0154028 0.0117112 2023-09-23T20:28:17.678 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:18.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:18 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 22918/220344 objects degraded (10.401%), 13 pgs degraded (PG_DEGRADED) 2023-09-23T20:28:18.578 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 56 16 76055 76039 84.8238 56.4375 0.0486357 0.0117836 2023-09-23T20:28:18.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:18 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 22918/220344 objects degraded (10.401%), 13 pgs degraded (PG_DEGRADED) 2023-09-23T20:28:18.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:18 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 22918/220344 objects degraded (10.401%), 13 pgs degraded (PG_DEGRADED) 2023-09-23T20:28:18.728 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:28:18.740 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:18.741 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:28:18.842 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:28:18.960 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:19.223 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:28:19.227 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:19.227 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:28:19.253 INFO:teuthology.orchestra.run.smithi018.stderr:noscrub is set 2023-09-23T20:28:19.328 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:28:19.377 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:19.434 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:28:19.443 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:19.443 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:28:19.491 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:19 smithi018 ceph-mon[179647]: pgmap v520: 109 pgs: 11 peering, 1 active+recovery_wait+undersized+degraded+remapped, 10 active+undersized+degraded, 28 active+undersized, 2 active+recovering+undersized+remapped, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 55 active+clean; 4.5 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 4.5 KiB/s rd, 27 MiB/s wr, 435 op/s; 22918/220344 objects degraded (10.401%); 16275/220344 objects misplaced (7.386%); 6.5 MiB/s, 102 objects/s recovering 2023-09-23T20:28:19.491 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:19 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1060901294' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:28:19.491 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:19 smithi018 ceph-mon[184144]: pgmap v520: 109 pgs: 11 peering, 1 active+recovery_wait+undersized+degraded+remapped, 10 active+undersized+degraded, 28 active+undersized, 2 active+recovering+undersized+remapped, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 55 active+clean; 4.5 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 4.5 KiB/s rd, 27 MiB/s wr, 435 op/s; 22918/220344 objects degraded (10.401%); 16275/220344 objects misplaced (7.386%); 6.5 MiB/s, 102 objects/s recovering 2023-09-23T20:28:19.491 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:19 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1060901294' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:28:19.544 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:28:19.580 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 57 16 76914 76898 84.2733 53.6875 0.0340686 0.0118592 2023-09-23T20:28:19.636 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:19.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:19 smithi149 ceph-mon[170394]: pgmap v520: 109 pgs: 11 peering, 1 active+recovery_wait+undersized+degraded+remapped, 10 active+undersized+degraded, 28 active+undersized, 2 active+recovering+undersized+remapped, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 55 active+clean; 4.5 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 4.5 KiB/s rd, 27 MiB/s wr, 435 op/s; 22918/220344 objects degraded (10.401%); 16275/220344 objects misplaced (7.386%); 6.5 MiB/s, 102 objects/s recovering 2023-09-23T20:28:19.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:19 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1060901294' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:28:19.797 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:28:19.805 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:19.806 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:28:19.820 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:28:19.907 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:28:19.989 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:20.116 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:28:20.123 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:20.124 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:28:20.225 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:28:20.226 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:28:20.274 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:28:20.306 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:20.329 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:28:20.365 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:28:20.394 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:20 smithi018 ceph-mon[184144]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:28:20.394 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:20 smithi018 ceph-mon[184144]: failed to encode map e239 with expected crc 2023-09-23T20:28:20.394 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:20 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1060901294' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:28:20.394 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:20 smithi018 ceph-mon[184144]: osdmap e239: 8 total, 7 up, 7 in 2023-09-23T20:28:20.394 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:20 smithi018 ceph-mon[184144]: failed to encode map e239 with expected crc 2023-09-23T20:28:20.394 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:20 smithi018 ceph-mon[184144]: failed to encode map e239 with expected crc 2023-09-23T20:28:20.394 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:20 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/212975586' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:28:20.395 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:20 smithi018 ceph-mon[179647]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:28:20.395 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:20 smithi018 ceph-mon[179647]: failed to encode map e239 with expected crc 2023-09-23T20:28:20.395 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:20 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1060901294' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:28:20.395 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:20 smithi018 ceph-mon[179647]: osdmap e239: 8 total, 7 up, 7 in 2023-09-23T20:28:20.395 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:20 smithi018 ceph-mon[179647]: failed to encode map e239 with expected crc 2023-09-23T20:28:20.395 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:20 smithi018 ceph-mon[179647]: failed to encode map e239 with expected crc 2023-09-23T20:28:20.395 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:20 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/212975586' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:28:20.395 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:28:20.397 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:28:20.399 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:20.400 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:28:20.439 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:28:20.488 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:28:20.489 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:28:20.489 INFO:teuthology.orchestra.run.smithi018.stderr:Error ENXIO: problem getting command descriptions from osd.1 2023-09-23T20:28:20.489 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:28:20.489 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:28:20.489 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:28:20.489 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:28:20.489 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:28:20.489 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:28:20.501 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:28:20.542 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 4, 5, 7, 3, 2] out_osds: [6] dead_osds: [1] live_osds: [0, 2, 3, 4, 5, 6, 7] 2023-09-23T20:28:20.542 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:28:20.542 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_1 2023-09-23T20:28:20.542 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:28:20.580 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:20.582 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 58 16 77697 77681 83.6616 48.9375 0.0123558 0.0119481 2023-09-23T20:28:20.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:20 smithi149 ceph-mon[170394]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:28:20.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:20 smithi149 ceph-mon[170394]: failed to encode map e239 with expected crc 2023-09-23T20:28:20.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:20 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1060901294' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:28:20.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:20 smithi149 ceph-mon[170394]: osdmap e239: 8 total, 7 up, 7 in 2023-09-23T20:28:20.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:20 smithi149 ceph-mon[170394]: failed to encode map e239 with expected crc 2023-09-23T20:28:20.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:20 smithi149 ceph-mon[170394]: failed to encode map e239 with expected crc 2023-09-23T20:28:20.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:20 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/212975586' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:28:20.973 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:28:20.979 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:20.979 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:28:21.080 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:28:21.121 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:21.176 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T20:28:21.177 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T20:28:21.232 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:28:21.239 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:21.240 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:28:21.340 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:28:21.424 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:21.447 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:21 smithi018 ceph-mon[184144]: pgmap v522: 109 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 11 active+undersized+degraded, 38 active+undersized, 1 active+recovering+undersized+remapped, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 55 active+clean; 4.5 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 4.2 KiB/s rd, 23 MiB/s wr, 368 op/s; 31288/221982 objects degraded (14.095%); 12403/221982 objects misplaced (5.587%); 5.1 MiB/s, 80 objects/s recovering 2023-09-23T20:28:21.447 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:21 smithi018 ceph-mon[184144]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2023-09-23T20:28:21.447 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:21 smithi018 ceph-mon[179647]: pgmap v522: 109 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 11 active+undersized+degraded, 38 active+undersized, 1 active+recovering+undersized+remapped, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 55 active+clean; 4.5 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 4.2 KiB/s rd, 23 MiB/s wr, 368 op/s; 31288/221982 objects degraded (14.095%); 12403/221982 objects misplaced (5.587%); 5.1 MiB/s, 80 objects/s recovering 2023-09-23T20:28:21.447 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:21 smithi018 ceph-mon[179647]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2023-09-23T20:28:21.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:21 smithi149 ceph-mon[170394]: pgmap v522: 109 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 11 active+undersized+degraded, 38 active+undersized, 1 active+recovering+undersized+remapped, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 55 active+clean; 4.5 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 4.2 KiB/s rd, 23 MiB/s wr, 368 op/s; 31288/221982 objects degraded (14.095%); 12403/221982 objects misplaced (5.587%); 5.1 MiB/s, 80 objects/s recovering 2023-09-23T20:28:21.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:21 smithi149 ceph-mon[170394]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2023-09-23T20:28:21.487 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:28:21.499 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:21.499 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:28:21.582 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 59 16 78525 78509 83.1209 51.75 0.0107823 0.0120251 2023-09-23T20:28:21.601 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:28:21.653 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:28:21.679 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:21.783 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2023-09-23T20:28:21.784 INFO:teuthology.orchestra.run.smithi018.stdout: "ops": [ 2023-09-23T20:28:21.784 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:28:21.784 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.45161.0:78658 6.1 6:8820fa64:::benchmark_data_smithi018_270558_object78657:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e239)", 2023-09-23T20:28:21.784 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:28:21.768859+0000", 2023-09-23T20:28:21.784 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.010439031, 2023-09-23T20:28:21.784 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.010475882000000001, 2023-09-23T20:28:21.784 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:28:21.784 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:28:21.784 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:28:21.784 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.45161", 2023-09-23T20:28:21.785 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3795614987", 2023-09-23T20:28:21.785 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 78658 2023-09-23T20:28:21.785 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:28:21.785 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:28:21.785 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:28:21.785 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:28:21.785 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:21.768859+0000", 2023-09-23T20:28:21.785 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:28:21.785 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:28:21.785 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:28:21.785 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:28:21.785 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:21.768859+0000", 2023-09-23T20:28:21.786 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:28:21.786 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:28:21.786 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:28:21.786 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:28:21.786 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:21.768858+0000", 2023-09-23T20:28:21.786 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:28:21.786 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:28:21.786 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:28:21.786 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:28:21.786 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:21.768891+0000", 2023-09-23T20:28:21.786 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.3618999999999998e-05 2023-09-23T20:28:21.787 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:28:21.787 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:28:21.787 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:28:21.787 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:21.768905+0000", 2023-09-23T20:28:21.787 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.4090999999999999e-05 2023-09-23T20:28:21.787 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:28:21.787 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:28:21.787 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:28:21.787 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:21.768913+0000", 2023-09-23T20:28:21.787 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.6129999999999998e-06 2023-09-23T20:28:21.787 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:28:21.787 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:28:21.787 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:28:21.788 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:21.769729+0000", 2023-09-23T20:28:21.788 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00081570099999999995 2023-09-23T20:28:21.788 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:28:21.788 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:28:21.788 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:28:21.788 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:21.769958+0000", 2023-09-23T20:28:21.788 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000228944 2023-09-23T20:28:21.788 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:28:21.788 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:28:21.788 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 6,7", 2023-09-23T20:28:21.788 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:21.770122+0000", 2023-09-23T20:28:21.788 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000164143 2023-09-23T20:28:21.788 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:28:21.789 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:28:21.789 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:28:21.789 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:28:21.789 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:28:21.789 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.45161.0:78659 6.1 6:8b142e4e:::benchmark_data_smithi018_270558_object78658:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e239)", 2023-09-23T20:28:21.789 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:28:21.768923+0000", 2023-09-23T20:28:21.789 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.010374764, 2023-09-23T20:28:21.789 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.010506613999999999, 2023-09-23T20:28:21.789 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:28:21.789 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:28:21.789 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:28:21.789 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.45161", 2023-09-23T20:28:21.790 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3795614987", 2023-09-23T20:28:21.790 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 78659 2023-09-23T20:28:21.790 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:28:21.790 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:28:21.790 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:28:21.790 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:28:21.790 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:21.768923+0000", 2023-09-23T20:28:21.790 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:28:21.790 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:28:21.790 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:28:21.790 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:28:21.791 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:21.768923+0000", 2023-09-23T20:28:21.791 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:28:21.791 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:28:21.791 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:28:21.791 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:28:21.791 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:21.768923+0000", 2023-09-23T20:28:21.791 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:28:21.791 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:28:21.791 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:28:21.791 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:28:21.791 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:21.768945+0000", 2023-09-23T20:28:21.791 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.1639000000000001e-05 2023-09-23T20:28:21.791 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:28:21.792 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:28:21.792 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:28:21.792 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:21.768951+0000", 2023-09-23T20:28:21.792 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.7379999999999997e-06 2023-09-23T20:28:21.792 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:28:21.792 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:28:21.792 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:28:21.792 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:21.768957+0000", 2023-09-23T20:28:21.792 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.4840000000000003e-06 2023-09-23T20:28:21.792 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:28:21.792 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:28:21.792 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:28:21.793 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:21.772352+0000", 2023-09-23T20:28:21.793 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0033947510000000001 2023-09-23T20:28:21.793 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:28:21.793 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:28:21.793 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:28:21.793 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:21.772547+0000", 2023-09-23T20:28:21.793 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00019504099999999999 2023-09-23T20:28:21.793 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:28:21.793 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:28:21.793 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 6,7", 2023-09-23T20:28:21.793 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:21.772806+0000", 2023-09-23T20:28:21.793 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000259829 2023-09-23T20:28:21.793 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:28:21.794 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:28:21.794 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:28:21.794 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:28:21.794 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:28:21.794 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.45161.0:78660 6.f 6:f3307c5b:::benchmark_data_smithi018_270558_object78659:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e239)", 2023-09-23T20:28:21.794 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:28:21.774253+0000", 2023-09-23T20:28:21.794 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.0050457189999999997, 2023-09-23T20:28:21.794 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0052646070000000001, 2023-09-23T20:28:21.794 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:28:21.794 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:28:21.794 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:28:21.795 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.45161", 2023-09-23T20:28:21.795 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3795614987", 2023-09-23T20:28:21.795 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 78660 2023-09-23T20:28:21.795 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:28:21.795 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:28:21.795 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:28:21.795 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:28:21.795 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:21.774253+0000", 2023-09-23T20:28:21.795 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:28:21.795 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:28:21.795 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:28:21.795 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:28:21.795 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:21.774253+0000", 2023-09-23T20:28:21.796 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:28:21.796 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:28:21.796 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:28:21.796 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:28:21.796 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:21.774250+0000", 2023-09-23T20:28:21.796 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999976 2023-09-23T20:28:21.796 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:28:21.796 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:28:21.796 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:28:21.796 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:21.774280+0000", 2023-09-23T20:28:21.796 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.0122999999999999e-05 2023-09-23T20:28:21.796 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:28:21.797 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:28:21.797 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:28:21.797 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:21.774296+0000", 2023-09-23T20:28:21.797 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.6044000000000001e-05 2023-09-23T20:28:21.797 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:28:21.797 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:28:21.797 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:28:21.797 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:21.774309+0000", 2023-09-23T20:28:21.797 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2867000000000001e-05 2023-09-23T20:28:21.797 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:28:21.797 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:28:21.797 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:28:21.798 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:21.774428+0000", 2023-09-23T20:28:21.798 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000118776 2023-09-23T20:28:21.798 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:28:21.798 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:28:21.798 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:28:21.798 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:21.774610+0000", 2023-09-23T20:28:21.798 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00018156699999999999 2023-09-23T20:28:21.798 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:28:21.798 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:28:21.798 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:28:21.798 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:21.774760+0000", 2023-09-23T20:28:21.798 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00015058899999999999 2023-09-23T20:28:21.799 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:28:21.799 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:28:21.799 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:28:21.799 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:28:21.799 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:28:21.799 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.45161.0:78663 6.f 6:f0a506cd:::benchmark_data_smithi018_270558_object78662:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e239)", 2023-09-23T20:28:21.799 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:28:21.774334+0000", 2023-09-23T20:28:21.799 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.0049638950000000003, 2023-09-23T20:28:21.799 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0052703029999999996, 2023-09-23T20:28:21.799 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:28:21.799 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:28:21.799 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:28:21.800 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.45161", 2023-09-23T20:28:21.800 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3795614987", 2023-09-23T20:28:21.800 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 78663 2023-09-23T20:28:21.800 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:28:21.800 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:28:21.800 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:28:21.800 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:28:21.800 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:21.774334+0000", 2023-09-23T20:28:21.800 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:28:21.800 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:28:21.800 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:28:21.800 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:28:21.801 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:21.774334+0000", 2023-09-23T20:28:21.801 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:28:21.801 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:28:21.801 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:28:21.801 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:28:21.801 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:21.774334+0000", 2023-09-23T20:28:21.801 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:28:21.801 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:28:21.801 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:28:21.801 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:28:21.801 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:21.774352+0000", 2023-09-23T20:28:21.801 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.8266000000000001e-05 2023-09-23T20:28:21.801 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:28:21.802 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:28:21.802 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:28:21.802 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:21.774360+0000", 2023-09-23T20:28:21.802 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.4260000000000003e-06 2023-09-23T20:28:21.802 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:28:21.802 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:28:21.802 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:28:21.802 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:21.774367+0000", 2023-09-23T20:28:21.802 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.4529999999999999e-06 2023-09-23T20:28:21.802 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:28:21.802 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:28:21.803 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:28:21.803 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:21.777553+0000", 2023-09-23T20:28:21.803 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.003185799 2023-09-23T20:28:21.803 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:28:21.803 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:28:21.803 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:28:21.803 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:21.777686+0000", 2023-09-23T20:28:21.803 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00013365299999999999 2023-09-23T20:28:21.803 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:28:21.803 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:28:21.803 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:28:21.803 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:21.777783+0000", 2023-09-23T20:28:21.804 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.6539000000000005e-05 2023-09-23T20:28:21.804 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:28:21.804 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:28:21.804 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:28:21.804 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:28:21.804 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:28:21.804 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_repop(client.45161.0:78665 6.8 e239/217 6:1bfb5f45:::benchmark_data_smithi018_270558_object78664:head v 239'4934, mlcod=239'4932)", 2023-09-23T20:28:21.804 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:28:21.778309+0000", 2023-09-23T20:28:21.804 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.00098907800000000001, 2023-09-23T20:28:21.804 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0013842080000000001, 2023-09-23T20:28:21.804 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:28:21.804 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "started", 2023-09-23T20:28:21.805 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:28:21.805 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:28:21.805 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:28:21.805 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:21.778309+0000", 2023-09-23T20:28:21.805 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:28:21.805 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:28:21.805 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:28:21.805 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:28:21.805 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:21.778309+0000", 2023-09-23T20:28:21.805 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:28:21.805 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:28:21.806 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:28:21.806 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:28:21.806 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:21.778308+0000", 2023-09-23T20:28:21.806 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.999999 2023-09-23T20:28:21.806 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:28:21.806 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:28:21.806 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:28:21.806 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:21.778367+0000", 2023-09-23T20:28:21.806 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.8919000000000003e-05 2023-09-23T20:28:21.806 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:28:21.806 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:28:21.807 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:28:21.807 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:21.778384+0000", 2023-09-23T20:28:21.807 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.6509999999999999e-05 2023-09-23T20:28:21.807 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:28:21.807 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:28:21.807 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:28:21.807 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:21.778391+0000", 2023-09-23T20:28:21.807 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.6389999999999992e-06 2023-09-23T20:28:21.807 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:28:21.807 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:28:21.807 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:28:21.807 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:21.778535+0000", 2023-09-23T20:28:21.807 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00014335800000000001 2023-09-23T20:28:21.808 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:28:21.808 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:28:21.808 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:28:21.808 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:21.778556+0000", 2023-09-23T20:28:21.808 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.0775e-05 2023-09-23T20:28:21.808 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:28:21.808 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:28:21.808 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:28:21.808 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:28:21.808 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:28:21.808 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_repop(client.45161.0:78670 6.3 e239/237 6:c3c65ce7:::benchmark_data_smithi018_270558_object78669:head v 239'4908, mlcod=229'4253)", 2023-09-23T20:28:21.808 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:28:21.778406+0000", 2023-09-23T20:28:21.809 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.00089269700000000002, 2023-09-23T20:28:21.809 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0013542090000000001, 2023-09-23T20:28:21.809 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:28:21.809 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "started", 2023-09-23T20:28:21.809 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:28:21.809 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:28:21.809 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:28:21.809 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:21.778406+0000", 2023-09-23T20:28:21.809 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:28:21.809 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:28:21.809 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:28:21.809 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:28:21.810 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:21.778406+0000", 2023-09-23T20:28:21.810 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:28:21.810 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:28:21.810 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:28:21.810 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:28:21.810 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:21.778405+0000", 2023-09-23T20:28:21.810 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:28:21.810 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:28:21.810 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:28:21.810 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:28:21.810 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:21.778464+0000", 2023-09-23T20:28:21.811 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.9249000000000001e-05 2023-09-23T20:28:21.811 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:28:21.811 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:28:21.811 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:28:21.811 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:21.778472+0000", 2023-09-23T20:28:21.811 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.6420000000000007e-06 2023-09-23T20:28:21.811 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:28:21.811 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:28:21.811 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:28:21.811 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:21.778480+0000", 2023-09-23T20:28:21.811 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.4490000000000002e-06 2023-09-23T20:28:21.811 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:28:21.812 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:28:21.812 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:28:21.812 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:21.778528+0000", 2023-09-23T20:28:21.812 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.8294999999999998e-05 2023-09-23T20:28:21.812 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:28:21.812 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:28:21.812 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:28:21.812 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:21.778554+0000", 2023-09-23T20:28:21.812 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.6403000000000001e-05 2023-09-23T20:28:21.812 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:28:21.812 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:28:21.812 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:28:21.813 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:28:21.813 INFO:teuthology.orchestra.run.smithi018.stdout: ], 2023-09-23T20:28:21.813 INFO:teuthology.orchestra.run.smithi018.stdout: "num_ops": 6 2023-09-23T20:28:21.813 INFO:teuthology.orchestra.run.smithi018.stdout:} 2023-09-23T20:28:21.846 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T20:28:22.034 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:28:22.053 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:22.054 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:28:22.154 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:28:22.199 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:22.330 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:28:22.341 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:22.342 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:28:22.435 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T20:28:22.443 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:28:22.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:22 smithi149 ceph-mon[170394]: pgmap v523: 109 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 11 active+undersized+degraded, 1 active+recovering+undersized+remapped, 38 active+undersized, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 55 active+clean; 4.7 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 3.4 KiB/s rd, 41 MiB/s wr, 661 op/s; 32090/230751 objects degraded (13.907%); 12508/230751 objects misplaced (5.421%); 5.5 MiB/s, 86 objects/s recovering 2023-09-23T20:28:22.540 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:22 smithi018 ceph-mon[179647]: pgmap v523: 109 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 11 active+undersized+degraded, 1 active+recovering+undersized+remapped, 38 active+undersized, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 55 active+clean; 4.7 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 3.4 KiB/s rd, 41 MiB/s wr, 661 op/s; 32090/230751 objects degraded (13.907%); 12508/230751 objects misplaced (5.421%); 5.5 MiB/s, 86 objects/s recovering 2023-09-23T20:28:22.540 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:22 smithi018 ceph-mon[184144]: pgmap v523: 109 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 11 active+undersized+degraded, 1 active+recovering+undersized+remapped, 38 active+undersized, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 55 active+clean; 4.7 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 3.4 KiB/s rd, 41 MiB/s wr, 661 op/s; 32090/230751 objects degraded (13.907%); 12508/230751 objects misplaced (5.421%); 5.5 MiB/s, 86 objects/s recovering 2023-09-23T20:28:22.579 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:22.583 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:2023-09-23T20:28:22.583580+0000 min lat: 0.00113549 max lat: 2.43464 avg lat: 0.0121071 2023-09-23T20:28:22.583 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T20:28:22.583 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 60 16 79324 79308 82.5678 49.9375 0.0131548 0.0121071 2023-09-23T20:28:22.815 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:28:22.825 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:22.825 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:28:22.902 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:28:22.926 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:28:22.973 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:23.178 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:28:23.321 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:28:23.325 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:23.325 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:28:23.426 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:28:23.516 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:28:23.535 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:23.592 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 61 16 80102 80086 82.0067 48.625 0.0220959 0.0121883 2023-09-23T20:28:23.608 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:28:23.617 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:23.618 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:28:23.719 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:28:23.756 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:23.814 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:28:23.820 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:23.821 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:28:23.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:23 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 32600/234606 objects degraded (13.896%), 15 pgs degraded (PG_DEGRADED) 2023-09-23T20:28:23.895 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:23 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 32600/234606 objects degraded (13.896%), 15 pgs degraded (PG_DEGRADED) 2023-09-23T20:28:23.922 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:28:23.960 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:23 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 32600/234606 objects degraded (13.896%), 15 pgs degraded (PG_DEGRADED) 2023-09-23T20:28:23.960 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:24.021 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:28:24.034 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:24.035 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:28:24.136 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:28:24.177 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:28:24.181 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:24.213 INFO:teuthology.orchestra.run.smithi018.stderr:dumped all 2023-09-23T20:28:24.224 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:28:24.227 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:24.228 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:28:24.249 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:28:24.328 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:28:24.410 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:24.586 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 62 16 80883 80867 81.4715 48.8125 0.0221092 0.0122695 2023-09-23T20:28:24.715 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:24 smithi149 ceph-mon[170394]: pgmap v524: 109 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 11 active+undersized+degraded, 1 active+recovering+undersized+remapped, 38 active+undersized, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 55 active+clean; 4.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 2.9 KiB/s rd, 43 MiB/s wr, 697 op/s; 32600/234606 objects degraded (13.896%); 12475/234606 objects misplaced (5.317%); 5.8 MiB/s, 92 objects/s recovering 2023-09-23T20:28:24.715 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:24 smithi149 ceph-mon[170394]: from='client.35370 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:28:24.770 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:24 smithi018 ceph-mon[179647]: pgmap v524: 109 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 11 active+undersized+degraded, 1 active+recovering+undersized+remapped, 38 active+undersized, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 55 active+clean; 4.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 2.9 KiB/s rd, 43 MiB/s wr, 697 op/s; 32600/234606 objects degraded (13.896%); 12475/234606 objects misplaced (5.317%); 5.8 MiB/s, 92 objects/s recovering 2023-09-23T20:28:24.771 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:24 smithi018 ceph-mon[179647]: from='client.35370 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:28:24.771 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:24 smithi018 ceph-mon[184144]: pgmap v524: 109 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 11 active+undersized+degraded, 1 active+recovering+undersized+remapped, 38 active+undersized, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 55 active+clean; 4.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 2.9 KiB/s rd, 43 MiB/s wr, 697 op/s; 32600/234606 objects degraded (13.896%); 12475/234606 objects misplaced (5.317%); 5.8 MiB/s, 92 objects/s recovering 2023-09-23T20:28:24.771 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:24 smithi018 ceph-mon[184144]: from='client.35370 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:28:25.141 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:28:24 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:28:24] "GET /metrics HTTP/1.1" 200 36555 "" "Prometheus/2.43.0" 2023-09-23T20:28:25.586 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 63 16 81746 81730 81.0346 53.9375 0.0428884 0.0123341 2023-09-23T20:28:26.587 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 64 16 82551 82535 80.5547 50.3125 0.0480985 0.0124074 2023-09-23T20:28:27.337 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:27 smithi018 ceph-mon[179647]: pgmap v525: 109 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 11 active+undersized+degraded, 1 active+recovering+undersized+remapped, 38 active+undersized, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 55 active+clean; 4.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 2.7 KiB/s rd, 40 MiB/s wr, 645 op/s; 32600/236142 objects degraded (13.805%); 12475/236142 objects misplaced (5.283%); 5.5 MiB/s, 87 objects/s recovering 2023-09-23T20:28:27.338 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:27 smithi018 ceph-mon[184144]: pgmap v525: 109 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 11 active+undersized+degraded, 1 active+recovering+undersized+remapped, 38 active+undersized, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 55 active+clean; 4.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 2.7 KiB/s rd, 40 MiB/s wr, 645 op/s; 32600/236142 objects degraded (13.805%); 12475/236142 objects misplaced (5.283%); 5.5 MiB/s, 87 objects/s recovering 2023-09-23T20:28:27.348 INFO:teuthology.orchestra.run.smithi018.stdout:{"pg_ready":true,"pg_map":{"version":524,"stamp":"2023-09-23T20:28:23.588795+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5127993731,"num_objects":78202,"num_object_clones":0,"num_object_copies":234606,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":32600,"num_objects_misplaced":12475,"num_objects_unfound":0,"num_objects_dirty":78202,"num_whiteouts":0,"num_read":6187,"num_read_kb":13572,"num_write":79174,"num_write_kb":5016686,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1351,"num_bytes_recovered":90619279,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":82584,"ondisk_log_size":82584,"up":276,"acting":275,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":330,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":14834656,"kb_used_data":13548584,"kb_used_omap":30,"kb_used_meta":1285985,"kb_avail":641381408,"statfs":{"total":671965249536,"available":656774561792,"internally_reserved":0,"allocated":13873750016,"data_stored":13866930786,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31263,"internal_metadata":1316849121},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,1,2,4],"upper_bound":32},"perf_stat":{"commit_latency_ms":12,"apply_latency_ms":12,"commit_latency_ns":12000000,"apply_latency_ns":12000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":430196194,"num_objects":6564,"num_object_clones":0,"num_object_copies":19692,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":19876,"num_objects_misplaced":4437,"num_objects_unfound":0,"num_objects_dirty":6564,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":6571,"num_write_kb":420169,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":874,"num_bytes_recovered":57790886,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.447554"},"pg_stats":[{"pgid":"3.1a","version":"69'1","reported_seq":243,"reported_epoch":238,"state":"active+undersized","last_fresh":"2023-09-23T20:28:14.518820+0000","last_change":"2023-09-23T20:28:13.154911+0000","last_active":"2023-09-23T20:28:14.518820+0000","last_peered":"2023-09-23T20:28:14.518820+0000","last_clean":"2023-09-23T20:28:10.122878+0000","last_became_active":"2023-09-23T20:28:13.154911+0000","last_became_peered":"2023-09-23T20:28:13.154911+0000","last_unstale":"2023-09-23T20:28:14.518820+0000","last_undegraded":"2023-09-23T20:28:14.518820+0000","last_fullsized":"2023-09-23T20:28:13.148164+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:38:02.482515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":["4","2"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":91,"reported_epoch":238,"state":"active+undersized","last_fresh":"2023-09-23T20:28:14.160361+0000","last_change":"2023-09-23T20:28:13.170504+0000","last_active":"2023-09-23T20:28:14.160361+0000","last_peered":"2023-09-23T20:28:14.160361+0000","last_clean":"2023-09-23T20:28:10.123345+0000","last_became_active":"2023-09-23T20:28:13.170504+0000","last_became_peered":"2023-09-23T20:28:13.170504+0000","last_unstale":"2023-09-23T20:28:14.160361+0000","last_undegraded":"2023-09-23T20:28:14.160361+0000","last_fullsized":"2023-09-23T20:28:13.140712+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:35:11.789454+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":["3","4"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":124,"reported_epoch":238,"state":"active+undersized","last_fresh":"2023-09-23T20:28:14.159570+0000","last_change":"2023-09-23T20:28:13.155496+0000","last_active":"2023-09-23T20:28:14.159570+0000","last_peered":"2023-09-23T20:28:14.159570+0000","last_clean":"2023-09-23T20:24:57.110902+0000","last_became_active":"2023-09-23T20:28:13.155496+0000","last_became_peered":"2023-09-23T20:28:13.155496+0000","last_unstale":"2023-09-23T20:28:14.159570+0000","last_undegraded":"2023-09-23T20:28:14.159570+0000","last_fullsized":"2023-09-23T20:28:13.140169+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:49:18.750327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":["3","4"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":203,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:28:14.938498+0000","last_change":"2023-09-23T20:25:02.957185+0000","last_active":"2023-09-23T20:28:14.938498+0000","last_peered":"2023-09-23T20:28:14.938498+0000","last_clean":"2023-09-23T20:28:14.938498+0000","last_became_active":"2023-09-23T20:25:02.956689+0000","last_became_peered":"2023-09-23T20:25:02.956689+0000","last_unstale":"2023-09-23T20:28:14.938498+0000","last_undegraded":"2023-09-23T20:28:14.938498+0000","last_fullsized":"2023-09-23T20:28:14.938498+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:04:28.524015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":91,"reported_epoch":238,"state":"active+undersized","last_fresh":"2023-09-23T20:28:14.159424+0000","last_change":"2023-09-23T20:28:13.145246+0000","last_active":"2023-09-23T20:28:14.159424+0000","last_peered":"2023-09-23T20:28:14.159424+0000","last_clean":"2023-09-23T20:28:10.123201+0000","last_became_active":"2023-09-23T20:28:13.145246+0000","last_became_peered":"2023-09-23T20:28:13.145246+0000","last_unstale":"2023-09-23T20:28:14.159424+0000","last_undegraded":"2023-09-23T20:28:14.159424+0000","last_fullsized":"2023-09-23T20:28:13.139195+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:33:00.081087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":["3","0"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":78,"reported_epoch":238,"state":"active+undersized","last_fresh":"2023-09-23T20:28:14.155916+0000","last_change":"2023-09-23T20:28:13.163992+0000","last_active":"2023-09-23T20:28:14.155916+0000","last_peered":"2023-09-23T20:28:14.155916+0000","last_clean":"2023-09-23T20:28:10.122013+0000","last_became_active":"2023-09-23T20:28:13.163992+0000","last_became_peered":"2023-09-23T20:28:13.163992+0000","last_unstale":"2023-09-23T20:28:14.155916+0000","last_undegraded":"2023-09-23T20:28:14.155916+0000","last_fullsized":"2023-09-23T20:28:13.153382+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:36:20.431010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":["5","0"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":201,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:28:14.142772+0000","last_change":"2023-09-23T20:25:02.969432+0000","last_active":"2023-09-23T20:28:14.142772+0000","last_peered":"2023-09-23T20:28:14.142772+0000","last_clean":"2023-09-23T20:28:14.142772+0000","last_became_active":"2023-09-23T20:25:02.969118+0000","last_became_peered":"2023-09-23T20:25:02.969118+0000","last_unstale":"2023-09-23T20:28:14.142772+0000","last_undegraded":"2023-09-23T20:28:14.142772+0000","last_fullsized":"2023-09-23T20:28:14.142772+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:02:22.453831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":100,"reported_epoch":238,"state":"active+undersized","last_fresh":"2023-09-23T20:28:14.159682+0000","last_change":"2023-09-23T20:28:13.155103+0000","last_active":"2023-09-23T20:28:14.159682+0000","last_peered":"2023-09-23T20:28:14.159682+0000","last_clean":"2023-09-23T20:28:10.123286+0000","last_became_active":"2023-09-23T20:28:13.155103+0000","last_became_peered":"2023-09-23T20:28:13.155103+0000","last_unstale":"2023-09-23T20:28:14.159682+0000","last_undegraded":"2023-09-23T20:28:14.159682+0000","last_fullsized":"2023-09-23T20:28:13.139476+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:55:15.053789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":["3","4"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"90'161","reported_seq":481,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:28:14.145979+0000","last_change":"2023-09-23T20:28:05.995127+0000","last_active":"2023-09-23T20:28:14.145979+0000","last_peered":"2023-09-23T20:28:14.145979+0000","last_clean":"2023-09-23T20:28:14.145979+0000","last_became_active":"2023-09-23T20:28:05.994757+0000","last_became_peered":"2023-09-23T20:28:05.994757+0000","last_unstale":"2023-09-23T20:28:14.145979+0000","last_undegraded":"2023-09-23T20:28:14.145979+0000","last_fullsized":"2023-09-23T20:28:14.145979+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:13.244651+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.244651+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0019955509999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109227,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":205,"reported_epoch":238,"state":"active+undersized","last_fresh":"2023-09-23T20:28:14.155566+0000","last_change":"2023-09-23T20:28:13.175805+0000","last_active":"2023-09-23T20:28:14.155566+0000","last_peered":"2023-09-23T20:28:14.155566+0000","last_clean":"2023-09-23T20:28:10.121056+0000","last_became_active":"2023-09-23T20:28:13.175805+0000","last_became_peered":"2023-09-23T20:28:13.175805+0000","last_unstale":"2023-09-23T20:28:14.155566+0000","last_undegraded":"2023-09-23T20:28:14.155566+0000","last_fullsized":"2023-09-23T20:28:13.156272+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:57:51.683405+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":["5","7"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":94,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:28:14.159778+0000","last_change":"2023-09-23T20:28:05.036058+0000","last_active":"2023-09-23T20:28:14.159778+0000","last_peered":"2023-09-23T20:28:14.159778+0000","last_clean":"2023-09-23T20:28:14.159778+0000","last_became_active":"2023-09-23T20:28:05.035708+0000","last_became_peered":"2023-09-23T20:28:05.035708+0000","last_unstale":"2023-09-23T20:28:14.159778+0000","last_undegraded":"2023-09-23T20:28:14.159778+0000","last_fullsized":"2023-09-23T20:28:14.159778+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:30:06.458724+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":234,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:28:14.148622+0000","last_change":"2023-09-23T20:25:03.038511+0000","last_active":"2023-09-23T20:28:14.148622+0000","last_peered":"2023-09-23T20:28:14.148622+0000","last_clean":"2023-09-23T20:28:14.148622+0000","last_became_active":"2023-09-23T20:25:02.976811+0000","last_became_peered":"2023-09-23T20:25:02.976811+0000","last_unstale":"2023-09-23T20:28:14.148622+0000","last_undegraded":"2023-09-23T20:28:14.148622+0000","last_fullsized":"2023-09-23T20:28:14.148622+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:28:06.605731+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":201,"reported_epoch":238,"state":"active+undersized","last_fresh":"2023-09-23T20:28:17.073098+0000","last_change":"2023-09-23T20:28:13.145997+0000","last_active":"2023-09-23T20:28:17.073098+0000","last_peered":"2023-09-23T20:28:17.073098+0000","last_clean":"2023-09-23T20:28:02.998115+0000","last_became_active":"2023-09-23T20:28:13.145997+0000","last_became_peered":"2023-09-23T20:28:13.145997+0000","last_unstale":"2023-09-23T20:28:17.073098+0000","last_undegraded":"2023-09-23T20:28:17.073098+0000","last_fullsized":"2023-09-23T20:28:13.141589+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:22:49.831269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":["2","0"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":223,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:28:14.519254+0000","last_change":"2023-09-23T20:28:05.051083+0000","last_active":"2023-09-23T20:28:14.519254+0000","last_peered":"2023-09-23T20:28:14.519254+0000","last_clean":"2023-09-23T20:28:14.519254+0000","last_became_active":"2023-09-23T20:28:05.024014+0000","last_became_peered":"2023-09-23T20:28:05.024014+0000","last_unstale":"2023-09-23T20:28:14.519254+0000","last_undegraded":"2023-09-23T20:28:14.519254+0000","last_fullsized":"2023-09-23T20:28:14.519254+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:09:19.424181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":214,"reported_epoch":238,"state":"active+undersized","last_fresh":"2023-09-23T20:28:14.519363+0000","last_change":"2023-09-23T20:28:13.170352+0000","last_active":"2023-09-23T20:28:14.519363+0000","last_peered":"2023-09-23T20:28:14.519363+0000","last_clean":"2023-09-23T20:28:10.122153+0000","last_became_active":"2023-09-23T20:28:13.170352+0000","last_became_peered":"2023-09-23T20:28:13.170352+0000","last_unstale":"2023-09-23T20:28:14.519363+0000","last_undegraded":"2023-09-23T20:28:14.519363+0000","last_fullsized":"2023-09-23T20:28:13.150349+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:32:21.400258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5],"acting":[4,5],"avail_no_missing":["4","5"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"90'161","reported_seq":452,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:28:14.142493+0000","last_change":"2023-09-23T20:28:05.991681+0000","last_active":"2023-09-23T20:28:14.142493+0000","last_peered":"2023-09-23T20:28:14.142493+0000","last_clean":"2023-09-23T20:28:14.142493+0000","last_became_active":"2023-09-23T20:28:05.991304+0000","last_became_peered":"2023-09-23T20:28:05.991304+0000","last_unstale":"2023-09-23T20:28:14.142493+0000","last_undegraded":"2023-09-23T20:28:14.142493+0000","last_fullsized":"2023-09-23T20:28:14.142493+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:34.821789+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:34.821789+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":233,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:28:14.153473+0000","last_change":"2023-09-23T20:25:39.654026+0000","last_active":"2023-09-23T20:28:14.153473+0000","last_peered":"2023-09-23T20:28:14.153473+0000","last_clean":"2023-09-23T20:28:14.153473+0000","last_became_active":"2023-09-23T20:25:39.626849+0000","last_became_peered":"2023-09-23T20:25:39.626849+0000","last_unstale":"2023-09-23T20:28:14.153473+0000","last_undegraded":"2023-09-23T20:28:14.153473+0000","last_fullsized":"2023-09-23T20:28:14.153473+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:22:53.957630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":91,"reported_epoch":238,"state":"active+undersized","last_fresh":"2023-09-23T20:28:14.161826+0000","last_change":"2023-09-23T20:28:13.152317+0000","last_active":"2023-09-23T20:28:14.161826+0000","last_peered":"2023-09-23T20:28:14.161826+0000","last_clean":"2023-09-23T20:28:10.123633+0000","last_became_active":"2023-09-23T20:28:13.152317+0000","last_became_peered":"2023-09-23T20:28:13.152317+0000","last_unstale":"2023-09-23T20:28:14.161826+0000","last_undegraded":"2023-09-23T20:28:14.161826+0000","last_fullsized":"2023-09-23T20:28:13.142454+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:13:55.507629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"90'161","reported_seq":439,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:28:14.155115+0000","last_change":"2023-09-23T20:28:08.021917+0000","last_active":"2023-09-23T20:28:14.155115+0000","last_peered":"2023-09-23T20:28:14.155115+0000","last_clean":"2023-09-23T20:28:14.155115+0000","last_became_active":"2023-09-23T20:28:08.021588+0000","last_became_peered":"2023-09-23T20:28:08.021588+0000","last_unstale":"2023-09-23T20:28:14.155115+0000","last_undegraded":"2023-09-23T20:28:14.155115+0000","last_fullsized":"2023-09-23T20:28:14.155115+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:32.801168+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:32.801168+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":201,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:28:14.143595+0000","last_change":"2023-09-23T20:25:02.953993+0000","last_active":"2023-09-23T20:28:14.143595+0000","last_peered":"2023-09-23T20:28:14.143595+0000","last_clean":"2023-09-23T20:28:14.143595+0000","last_became_active":"2023-09-23T20:25:02.953672+0000","last_became_peered":"2023-09-23T20:25:02.953672+0000","last_unstale":"2023-09-23T20:28:14.143595+0000","last_undegraded":"2023-09-23T20:28:14.143595+0000","last_fullsized":"2023-09-23T20:28:14.143595+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:26:08.603577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":217,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:28:14.155771+0000","last_change":"2023-09-23T20:25:02.949584+0000","last_active":"2023-09-23T20:28:14.155771+0000","last_peered":"2023-09-23T20:28:14.155771+0000","last_clean":"2023-09-23T20:28:14.155771+0000","last_became_active":"2023-09-23T20:25:02.949132+0000","last_became_peered":"2023-09-23T20:25:02.949132+0000","last_unstale":"2023-09-23T20:28:14.155771+0000","last_undegraded":"2023-09-23T20:28:14.155771+0000","last_fullsized":"2023-09-23T20:28:14.155771+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:34:20.509823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"90'161","reported_seq":476,"reported_epoch":238,"state":"active+undersized","last_fresh":"2023-09-23T20:28:14.159804+0000","last_change":"2023-09-23T20:28:13.172329+0000","last_active":"2023-09-23T20:28:14.159804+0000","last_peered":"2023-09-23T20:28:14.159804+0000","last_clean":"2023-09-23T20:27:55.206508+0000","last_became_active":"2023-09-23T20:28:13.172329+0000","last_became_peered":"2023-09-23T20:28:13.172329+0000","last_unstale":"2023-09-23T20:28:14.159804+0000","last_undegraded":"2023-09-23T20:28:14.159804+0000","last_fullsized":"2023-09-23T20:28:13.140024+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:12.386098+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:12.386098+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0019500940000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":["3","4"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":192,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:28:14.148245+0000","last_change":"2023-09-23T20:28:05.022017+0000","last_active":"2023-09-23T20:28:14.148245+0000","last_peered":"2023-09-23T20:28:14.148245+0000","last_clean":"2023-09-23T20:28:14.148245+0000","last_became_active":"2023-09-23T20:28:05.021659+0000","last_became_peered":"2023-09-23T20:28:05.021659+0000","last_unstale":"2023-09-23T20:28:14.148245+0000","last_undegraded":"2023-09-23T20:28:14.148245+0000","last_fullsized":"2023-09-23T20:28:14.148245+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:19:47.722750+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":211,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:28:14.142862+0000","last_change":"2023-09-23T20:25:02.957010+0000","last_active":"2023-09-23T20:28:14.142862+0000","last_peered":"2023-09-23T20:28:14.142862+0000","last_clean":"2023-09-23T20:28:14.142862+0000","last_became_active":"2023-09-23T20:25:02.956267+0000","last_became_peered":"2023-09-23T20:25:02.956267+0000","last_unstale":"2023-09-23T20:28:14.142862+0000","last_undegraded":"2023-09-23T20:28:14.142862+0000","last_fullsized":"2023-09-23T20:28:14.142862+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:10:55.299422+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"90'161","reported_seq":446,"reported_epoch":238,"state":"active+undersized+degraded","last_fresh":"2023-09-23T20:28:14.158597+0000","last_change":"2023-09-23T20:28:13.176937+0000","last_active":"2023-09-23T20:28:14.158597+0000","last_peered":"2023-09-23T20:28:14.158597+0000","last_clean":"2023-09-23T20:28:10.125672+0000","last_became_active":"2023-09-23T20:28:13.176937+0000","last_became_peered":"2023-09-23T20:28:13.176937+0000","last_unstale":"2023-09-23T20:28:14.158597+0000","last_undegraded":"2023-09-23T20:28:13.154776+0000","last_fullsized":"2023-09-23T20:28:13.154661+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:35.734584+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:35.734584+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":["5","7"],"object_location_counts":[{"shards":"5,7","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":41,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:28:14.142164+0000","last_change":"2023-09-23T20:28:05.041053+0000","last_active":"2023-09-23T20:28:14.142164+0000","last_peered":"2023-09-23T20:28:14.142164+0000","last_clean":"2023-09-23T20:28:14.142164+0000","last_became_active":"2023-09-23T20:28:05.030153+0000","last_became_peered":"2023-09-23T20:28:05.030153+0000","last_unstale":"2023-09-23T20:28:14.142164+0000","last_undegraded":"2023-09-23T20:28:14.142164+0000","last_fullsized":"2023-09-23T20:28:14.142164+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:37:31.689827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":91,"reported_epoch":238,"state":"active+undersized","last_fresh":"2023-09-23T20:28:14.160668+0000","last_change":"2023-09-23T20:28:13.146925+0000","last_active":"2023-09-23T20:28:14.160668+0000","last_peered":"2023-09-23T20:28:14.160668+0000","last_clean":"2023-09-23T20:28:10.124095+0000","last_became_active":"2023-09-23T20:28:13.146925+0000","last_became_peered":"2023-09-23T20:28:13.146925+0000","last_unstale":"2023-09-23T20:28:14.160668+0000","last_undegraded":"2023-09-23T20:28:14.160668+0000","last_fullsized":"2023-09-23T20:28:13.141297+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:26:22.810131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.d","version":"90'161","reported_seq":446,"reported_epoch":238,"state":"active+undersized","last_fresh":"2023-09-23T20:28:14.148305+0000","last_change":"2023-09-23T20:28:13.170455+0000","last_active":"2023-09-23T20:28:14.148305+0000","last_peered":"2023-09-23T20:28:14.148305+0000","last_clean":"2023-09-23T20:28:10.105394+0000","last_became_active":"2023-09-23T20:28:13.170455+0000","last_became_peered":"2023-09-23T20:28:13.170455+0000","last_unstale":"2023-09-23T20:28:14.148305+0000","last_undegraded":"2023-09-23T20:28:14.148305+0000","last_fullsized":"2023-09-23T20:28:13.145373+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:12.809933+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:12.809933+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.a","version":"239'4891","reported_seq":6197,"reported_epoch":239,"state":"active+recovering+undersized+remapped","last_fresh":"2023-09-23T20:28:20.630461+0000","last_change":"2023-09-23T20:28:06.211632+0000","last_active":"2023-09-23T20:28:20.630461+0000","last_peered":"2023-09-23T20:28:20.630461+0000","last_clean":"2023-09-23T20:28:04.001969+0000","last_became_active":"2023-09-23T20:28:06.165624+0000","last_became_peered":"2023-09-23T20:28:06.165624+0000","last_unstale":"2023-09-23T20:28:20.630461+0000","last_undegraded":"2023-09-23T20:28:20.630461+0000","last_fullsized":"2023-09-23T20:28:05.983416+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":4891,"log_dups_size":0,"ondisk_log_size":4891,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:51:55.980497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":320536576,"num_objects":4891,"num_object_clones":0,"num_object_copies":14673,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3678,"num_objects_unfound":0,"num_objects_dirty":4891,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4891,"num_write_kb":313024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":632,"num_bytes_recovered":41418752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0],"avail_no_missing":["5","0","6"],"object_location_counts":[{"shards":"0,5","objects":1213},{"shards":"0,5,6","objects":3678}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":200,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:28:14.148684+0000","last_change":"2023-09-23T20:20:14.883283+0000","last_active":"2023-09-23T20:28:14.148684+0000","last_peered":"2023-09-23T20:28:14.148684+0000","last_clean":"2023-09-23T20:28:14.148684+0000","last_became_active":"2023-09-23T20:20:14.882806+0000","last_became_peered":"2023-09-23T20:20:14.882806+0000","last_unstale":"2023-09-23T20:28:14.148684+0000","last_undegraded":"2023-09-23T20:28:14.148684+0000","last_fullsized":"2023-09-23T20:28:14.148684+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:23:35.295701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":190,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:28:14.155202+0000","last_change":"2023-09-23T20:28:05.037565+0000","last_active":"2023-09-23T20:28:14.155202+0000","last_peered":"2023-09-23T20:28:14.155202+0000","last_clean":"2023-09-23T20:28:14.155202+0000","last_became_active":"2023-09-23T20:28:05.037288+0000","last_became_peered":"2023-09-23T20:28:05.037288+0000","last_unstale":"2023-09-23T20:28:14.155202+0000","last_undegraded":"2023-09-23T20:28:14.155202+0000","last_fullsized":"2023-09-23T20:28:14.155202+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:55:40.576813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"90'161","reported_seq":500,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:28:14.159985+0000","last_change":"2023-09-23T20:28:07.018115+0000","last_active":"2023-09-23T20:28:14.159985+0000","last_peered":"2023-09-23T20:28:14.159985+0000","last_clean":"2023-09-23T20:28:14.159985+0000","last_became_active":"2023-09-23T20:28:07.017652+0000","last_became_peered":"2023-09-23T20:28:07.017652+0000","last_unstale":"2023-09-23T20:28:14.159985+0000","last_undegraded":"2023-09-23T20:28:14.159985+0000","last_fullsized":"2023-09-23T20:28:14.159985+0000","mapping_epoch":232,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":233,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:13.244651+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.244651+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0019955509999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109227,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.b","version":"239'4861","reported_seq":4895,"reported_epoch":239,"state":"active+undersized+degraded","last_fresh":"2023-09-23T20:28:23.057532+0000","last_change":"2023-09-23T20:28:13.151214+0000","last_active":"2023-09-23T20:28:23.057532+0000","last_peered":"2023-09-23T20:28:23.057532+0000","last_clean":"2023-09-23T20:28:12.089537+0000","last_became_active":"2023-09-23T20:28:13.151214+0000","last_became_peered":"2023-09-23T20:28:13.151214+0000","last_unstale":"2023-09-23T20:28:23.057532+0000","last_undegraded":"2023-09-23T20:28:13.141183+0000","last_fullsized":"2023-09-23T20:28:13.141041+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":4861,"log_dups_size":0,"ondisk_log_size":4861,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:25:15.926592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":318570496,"num_objects":4861,"num_object_clones":0,"num_object_copies":14583,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4861,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4861,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4861,"num_write_kb":311104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":4861}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":208,"reported_epoch":238,"state":"active+undersized+degraded","last_fresh":"2023-09-23T20:28:14.148789+0000","last_change":"2023-09-23T20:28:13.175199+0000","last_active":"2023-09-23T20:28:14.148789+0000","last_peered":"2023-09-23T20:28:14.148789+0000","last_clean":"2023-09-23T20:28:10.105377+0000","last_became_active":"2023-09-23T20:28:13.175199+0000","last_became_peered":"2023-09-23T20:28:13.175199+0000","last_unstale":"2023-09-23T20:28:14.148789+0000","last_undegraded":"2023-09-23T20:28:13.146101+0000","last_fullsized":"2023-09-23T20:28:13.145995+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:32:17.555869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[{"shards":"4,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":241,"reported_epoch":238,"state":"active+undersized","last_fresh":"2023-09-23T20:28:14.518727+0000","last_change":"2023-09-23T20:28:13.156552+0000","last_active":"2023-09-23T20:28:14.518727+0000","last_peered":"2023-09-23T20:28:14.518727+0000","last_clean":"2023-09-23T20:28:10.121663+0000","last_became_active":"2023-09-23T20:28:13.156552+0000","last_became_peered":"2023-09-23T20:28:13.156552+0000","last_unstale":"2023-09-23T20:28:14.518727+0000","last_undegraded":"2023-09-23T20:28:14.518727+0000","last_fullsized":"2023-09-23T20:28:13.147617+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:28:11.148006+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":["4","3"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"90'161","reported_seq":500,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:28:14.155678+0000","last_change":"2023-09-23T20:28:08.030382+0000","last_active":"2023-09-23T20:28:14.155678+0000","last_peered":"2023-09-23T20:28:14.155678+0000","last_clean":"2023-09-23T20:28:14.155678+0000","last_became_active":"2023-09-23T20:28:08.030032+0000","last_became_peered":"2023-09-23T20:28:08.030032+0000","last_unstale":"2023-09-23T20:28:14.155678+0000","last_undegraded":"2023-09-23T20:28:14.155678+0000","last_fullsized":"2023-09-23T20:28:14.155678+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:32.746340+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:32.746340+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.8","version":"239'4875","reported_seq":4905,"reported_epoch":239,"state":"active+clean","last_fresh":"2023-09-23T20:28:20.552285+0000","last_change":"2023-09-23T20:27:14.271145+0000","last_active":"2023-09-23T20:28:20.552285+0000","last_peered":"2023-09-23T20:28:20.552285+0000","last_clean":"2023-09-23T20:28:20.552285+0000","last_became_active":"2023-09-23T20:27:14.270876+0000","last_became_peered":"2023-09-23T20:27:14.270876+0000","last_unstale":"2023-09-23T20:28:20.552285+0000","last_undegraded":"2023-09-23T20:28:20.552285+0000","last_fullsized":"2023-09-23T20:28:20.552285+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":4875,"log_dups_size":0,"ondisk_log_size":4875,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:23:40.941512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":319488000,"num_objects":4875,"num_object_clones":0,"num_object_copies":14625,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4875,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4875,"num_write_kb":312000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":197,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:28:14.148463+0000","last_change":"2023-09-23T20:28:05.051786+0000","last_active":"2023-09-23T20:28:14.148463+0000","last_peered":"2023-09-23T20:28:14.148463+0000","last_clean":"2023-09-23T20:28:14.148463+0000","last_became_active":"2023-09-23T20:28:05.035372+0000","last_became_peered":"2023-09-23T20:28:05.035372+0000","last_unstale":"2023-09-23T20:28:14.148463+0000","last_undegraded":"2023-09-23T20:28:14.148463+0000","last_fullsized":"2023-09-23T20:28:14.148463+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:34:29.102528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":64,"reported_epoch":238,"state":"active+undersized","last_fresh":"2023-09-23T20:28:14.148337+0000","last_change":"2023-09-23T20:28:13.171782+0000","last_active":"2023-09-23T20:28:14.148337+0000","last_peered":"2023-09-23T20:28:14.148337+0000","last_clean":"2023-09-23T20:20:14.875676+0000","last_became_active":"2023-09-23T20:28:13.171782+0000","last_became_peered":"2023-09-23T20:28:13.171782+0000","last_unstale":"2023-09-23T20:28:14.148337+0000","last_undegraded":"2023-09-23T20:28:14.148337+0000","last_fullsized":"2023-09-23T20:28:13.145291+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:55:21.847801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5],"avail_no_missing":["7","5"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.e","version":"90'161","reported_seq":442,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:28:14.519713+0000","last_change":"2023-09-23T20:28:10.788870+0000","last_active":"2023-09-23T20:28:14.519713+0000","last_peered":"2023-09-23T20:28:14.519713+0000","last_clean":"2023-09-23T20:28:14.519713+0000","last_became_active":"2023-09-23T20:28:05.990524+0000","last_became_peered":"2023-09-23T20:28:05.990524+0000","last_unstale":"2023-09-23T20:28:14.519713+0000","last_undegraded":"2023-09-23T20:28:14.519713+0000","last_fullsized":"2023-09-23T20:28:14.519713+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:31.769333+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:31.769333+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.9","version":"239'4914","reported_seq":4945,"reported_epoch":239,"state":"active+clean","last_fresh":"2023-09-23T20:28:19.912782+0000","last_change":"2023-09-23T20:27:14.268973+0000","last_active":"2023-09-23T20:28:19.912782+0000","last_peered":"2023-09-23T20:28:19.912782+0000","last_clean":"2023-09-23T20:28:19.912782+0000","last_became_active":"2023-09-23T20:27:14.268670+0000","last_became_peered":"2023-09-23T20:27:14.268670+0000","last_unstale":"2023-09-23T20:28:19.912782+0000","last_undegraded":"2023-09-23T20:28:19.912782+0000","last_fullsized":"2023-09-23T20:28:19.912782+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":4914,"log_dups_size":0,"ondisk_log_size":4914,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:37:23.418588+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":322043904,"num_objects":4914,"num_object_clones":0,"num_object_copies":14742,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4914,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4914,"num_write_kb":314496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":235,"reported_epoch":238,"state":"active+undersized","last_fresh":"2023-09-23T20:28:14.155549+0000","last_change":"2023-09-23T20:28:13.168615+0000","last_active":"2023-09-23T20:28:14.155549+0000","last_peered":"2023-09-23T20:28:14.155549+0000","last_clean":"2023-09-23T20:28:10.121217+0000","last_became_active":"2023-09-23T20:28:13.168615+0000","last_became_peered":"2023-09-23T20:28:13.168615+0000","last_unstale":"2023-09-23T20:28:14.155549+0000","last_undegraded":"2023-09-23T20:28:14.155549+0000","last_fullsized":"2023-09-23T20:28:13.152836+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:24:12.165529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":["5","3"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":190,"reported_epoch":238,"state":"active+undersized","last_fresh":"2023-09-23T20:28:14.142284+0000","last_change":"2023-09-23T20:28:13.172321+0000","last_active":"2023-09-23T20:28:14.142284+0000","last_peered":"2023-09-23T20:28:14.142284+0000","last_clean":"2023-09-23T20:28:10.114667+0000","last_became_active":"2023-09-23T20:28:13.172321+0000","last_became_peered":"2023-09-23T20:28:13.172321+0000","last_unstale":"2023-09-23T20:28:14.142284+0000","last_undegraded":"2023-09-23T20:28:14.142284+0000","last_fullsized":"2023-09-23T20:28:13.141132+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:01:05.541514+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":["0","4"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"239'4856","reported_seq":4893,"reported_epoch":239,"state":"active+undersized+degraded","last_fresh":"2023-09-23T20:28:20.695925+0000","last_change":"2023-09-23T20:28:13.171294+0000","last_active":"2023-09-23T20:28:20.695925+0000","last_peered":"2023-09-23T20:28:20.695925+0000","last_clean":"2023-09-23T20:28:12.106078+0000","last_became_active":"2023-09-23T20:28:13.171294+0000","last_became_peered":"2023-09-23T20:28:13.171294+0000","last_unstale":"2023-09-23T20:28:20.695925+0000","last_undegraded":"2023-09-23T20:28:13.149409+0000","last_fullsized":"2023-09-23T20:28:13.149261+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":4856,"log_dups_size":0,"ondisk_log_size":4856,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:17:53.283303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":318242816,"num_objects":4856,"num_object_clones":0,"num_object_copies":14568,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4856,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4856,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4856,"num_write_kb":310784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":["4","2"],"object_location_counts":[{"shards":"2,4","objects":4856}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":621,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:28:14.147995+0000","last_change":"2023-09-23T20:28:05.872149+0000","last_active":"2023-09-23T20:28:14.147995+0000","last_peered":"2023-09-23T20:28:14.147995+0000","last_clean":"2023-09-23T20:28:14.147995+0000","last_became_active":"2023-09-23T20:27:11.961310+0000","last_became_peered":"2023-09-23T20:27:11.961310+0000","last_unstale":"2023-09-23T20:28:14.147995+0000","last_undegraded":"2023-09-23T20:28:14.147995+0000","last_fullsized":"2023-09-23T20:28:14.147995+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:28:05.872074+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:28:05.872074+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:52:32.495955+0000","scrub_duration":0.0029215980000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":89,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:28:14.160011+0000","last_change":"2023-09-23T20:25:03.000957+0000","last_active":"2023-09-23T20:28:14.160011+0000","last_peered":"2023-09-23T20:28:14.160011+0000","last_clean":"2023-09-23T20:28:14.160011+0000","last_became_active":"2023-09-23T20:25:02.939903+0000","last_became_peered":"2023-09-23T20:25:02.939903+0000","last_unstale":"2023-09-23T20:28:14.160011+0000","last_undegraded":"2023-09-23T20:28:14.160011+0000","last_fullsized":"2023-09-23T20:28:14.160011+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:51:14.981461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":234,"reported_epoch":238,"state":"active+undersized","last_fresh":"2023-09-23T20:28:14.519217+0000","last_change":"2023-09-23T20:28:13.165428+0000","last_active":"2023-09-23T20:28:14.519217+0000","last_peered":"2023-09-23T20:28:14.519217+0000","last_clean":"2023-09-23T20:28:10.122622+0000","last_became_active":"2023-09-23T20:28:13.165428+0000","last_became_peered":"2023-09-23T20:28:13.165428+0000","last_unstale":"2023-09-23T20:28:14.519217+0000","last_undegraded":"2023-09-23T20:28:14.519217+0000","last_fullsized":"2023-09-23T20:28:13.148542+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:56:49.047400+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":["4","3"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"239'4995","reported_seq":5026,"reported_epoch":239,"state":"active+clean","last_fresh":"2023-09-23T20:28:22.077419+0000","last_change":"2023-09-23T20:27:14.270199+0000","last_active":"2023-09-23T20:28:22.077419+0000","last_peered":"2023-09-23T20:28:22.077419+0000","last_clean":"2023-09-23T20:28:22.077419+0000","last_became_active":"2023-09-23T20:27:14.269858+0000","last_became_peered":"2023-09-23T20:27:14.269858+0000","last_unstale":"2023-09-23T20:28:22.077419+0000","last_undegraded":"2023-09-23T20:28:22.077419+0000","last_fullsized":"2023-09-23T20:28:22.077419+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":4995,"log_dups_size":0,"ondisk_log_size":4995,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:47:21.406552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327352320,"num_objects":4995,"num_object_clones":0,"num_object_copies":14985,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4995,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4995,"num_write_kb":319680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":463,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:28:14.155120+0000","last_change":"2023-09-23T20:27:31.757060+0000","last_active":"2023-09-23T20:28:14.155120+0000","last_peered":"2023-09-23T20:28:14.155120+0000","last_clean":"2023-09-23T20:28:14.155120+0000","last_became_active":"2023-09-23T20:27:11.959516+0000","last_became_peered":"2023-09-23T20:27:11.959516+0000","last_unstale":"2023-09-23T20:28:14.155120+0000","last_undegraded":"2023-09-23T20:28:14.155120+0000","last_fullsized":"2023-09-23T20:28:14.155120+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:31.756974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:31.756974+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:35:11.939335+0000","scrub_duration":0.0031802670000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":79,"reported_epoch":238,"state":"active+undersized","last_fresh":"2023-09-23T20:28:14.518747+0000","last_change":"2023-09-23T20:28:13.157583+0000","last_active":"2023-09-23T20:28:14.518747+0000","last_peered":"2023-09-23T20:28:14.518747+0000","last_clean":"2023-09-23T20:28:10.122833+0000","last_became_active":"2023-09-23T20:28:13.157583+0000","last_became_peered":"2023-09-23T20:28:13.157583+0000","last_unstale":"2023-09-23T20:28:14.518747+0000","last_undegraded":"2023-09-23T20:28:14.518747+0000","last_fullsized":"2023-09-23T20:28:13.148515+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:16:09.921700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":["4","3"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":227,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:28:14.519612+0000","last_change":"2023-09-23T20:25:39.624636+0000","last_active":"2023-09-23T20:28:14.519612+0000","last_peered":"2023-09-23T20:28:14.519612+0000","last_clean":"2023-09-23T20:28:14.519612+0000","last_became_active":"2023-09-23T20:25:39.624297+0000","last_became_peered":"2023-09-23T20:25:39.624297+0000","last_unstale":"2023-09-23T20:28:14.519612+0000","last_undegraded":"2023-09-23T20:28:14.519612+0000","last_fullsized":"2023-09-23T20:28:14.519612+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:33:47.448737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"90'161","reported_seq":476,"reported_epoch":238,"state":"active+undersized","last_fresh":"2023-09-23T20:28:14.938328+0000","last_change":"2023-09-23T20:28:13.164275+0000","last_active":"2023-09-23T20:28:14.938328+0000","last_peered":"2023-09-23T20:28:14.938328+0000","last_clean":"2023-09-23T20:27:55.206527+0000","last_became_active":"2023-09-23T20:28:13.164275+0000","last_became_peered":"2023-09-23T20:28:13.164275+0000","last_unstale":"2023-09-23T20:28:14.938328+0000","last_undegraded":"2023-09-23T20:28:14.938328+0000","last_fullsized":"2023-09-23T20:28:13.140861+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:12.386098+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:12.386098+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0019500940000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.c","version":"239'4827","reported_seq":5673,"reported_epoch":239,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:28:23.075741+0000","last_change":"2023-09-23T20:28:14.567450+0000","last_active":"2023-09-23T20:28:23.075741+0000","last_peered":"2023-09-23T20:28:23.075741+0000","last_clean":"2023-09-23T20:28:03.999709+0000","last_became_active":"2023-09-23T20:28:06.198771+0000","last_became_peered":"2023-09-23T20:28:06.198771+0000","last_unstale":"2023-09-23T20:28:23.075741+0000","last_undegraded":"2023-09-23T20:28:14.562024+0000","last_fullsized":"2023-09-23T20:28:05.981540+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":4827,"log_dups_size":0,"ondisk_log_size":4827,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:32:08.136452+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":316342272,"num_objects":4827,"num_object_clones":0,"num_object_copies":14481,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3747,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4827,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4827,"num_write_kb":308928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":402,"num_bytes_recovered":26345472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5],"avail_no_missing":["3","5","6"],"object_location_counts":[{"shards":"3,5","objects":1080},{"shards":"3,5,6","objects":3747}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":218,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:28:14.518494+0000","last_change":"2023-09-23T20:25:39.628097+0000","last_active":"2023-09-23T20:28:14.518494+0000","last_peered":"2023-09-23T20:28:14.518494+0000","last_clean":"2023-09-23T20:28:14.518494+0000","last_became_active":"2023-09-23T20:25:39.627806+0000","last_became_peered":"2023-09-23T20:25:39.627806+0000","last_unstale":"2023-09-23T20:28:14.518494+0000","last_undegraded":"2023-09-23T20:28:14.518494+0000","last_fullsized":"2023-09-23T20:28:14.518494+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:53:44.398854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":214,"reported_epoch":238,"state":"active+undersized","last_fresh":"2023-09-23T20:28:14.519895+0000","last_change":"2023-09-23T20:28:13.174633+0000","last_active":"2023-09-23T20:28:14.519895+0000","last_peered":"2023-09-23T20:28:14.519895+0000","last_clean":"2023-09-23T20:28:10.121563+0000","last_became_active":"2023-09-23T20:28:13.174633+0000","last_became_peered":"2023-09-23T20:28:13.174633+0000","last_unstale":"2023-09-23T20:28:14.519895+0000","last_undegraded":"2023-09-23T20:28:14.519895+0000","last_fullsized":"2023-09-23T20:28:13.150357+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:55:59.664339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":["4","3"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"239'4924","reported_seq":4961,"reported_epoch":239,"state":"active+undersized+degraded","last_fresh":"2023-09-23T20:28:20.596026+0000","last_change":"2023-09-23T20:28:13.164184+0000","last_active":"2023-09-23T20:28:20.596026+0000","last_peered":"2023-09-23T20:28:20.596026+0000","last_clean":"2023-09-23T20:28:12.085747+0000","last_became_active":"2023-09-23T20:28:13.164184+0000","last_became_peered":"2023-09-23T20:28:13.164184+0000","last_unstale":"2023-09-23T20:28:20.596026+0000","last_undegraded":"2023-09-23T20:28:13.153718+0000","last_fullsized":"2023-09-23T20:28:13.153621+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":4924,"log_dups_size":0,"ondisk_log_size":4924,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:13:24.112990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":322699264,"num_objects":4924,"num_object_clones":0,"num_object_copies":14772,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4924,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4924,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4924,"num_write_kb":315136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":["5","0"],"object_location_counts":[{"shards":"0,5","objects":4924}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":483,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:28:14.155666+0000","last_change":"2023-09-23T20:28:08.043585+0000","last_active":"2023-09-23T20:28:14.155666+0000","last_peered":"2023-09-23T20:28:14.155666+0000","last_clean":"2023-09-23T20:28:14.155666+0000","last_became_active":"2023-09-23T20:28:08.032075+0000","last_became_peered":"2023-09-23T20:28:08.032075+0000","last_unstale":"2023-09-23T20:28:14.155666+0000","last_undegraded":"2023-09-23T20:28:14.155666+0000","last_fullsized":"2023-09-23T20:28:14.155666+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:30.727267+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:30.727267+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:01:58.886819+0000","scrub_duration":0.0052560289999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":91,"reported_epoch":238,"state":"active+undersized","last_fresh":"2023-09-23T20:28:14.159414+0000","last_change":"2023-09-23T20:28:13.143589+0000","last_active":"2023-09-23T20:28:14.159414+0000","last_peered":"2023-09-23T20:28:14.159414+0000","last_clean":"2023-09-23T20:28:10.123116+0000","last_became_active":"2023-09-23T20:28:13.143589+0000","last_became_peered":"2023-09-23T20:28:13.143589+0000","last_unstale":"2023-09-23T20:28:14.159414+0000","last_undegraded":"2023-09-23T20:28:14.159414+0000","last_fullsized":"2023-09-23T20:28:13.138889+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:07:48.412859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":27,"reported_epoch":238,"state":"active+undersized","last_fresh":"2023-09-23T20:28:14.160549+0000","last_change":"2023-09-23T20:28:13.167121+0000","last_active":"2023-09-23T20:28:14.160549+0000","last_peered":"2023-09-23T20:28:14.160549+0000","last_clean":"2023-09-23T20:24:57.109980+0000","last_became_active":"2023-09-23T20:28:13.167121+0000","last_became_peered":"2023-09-23T20:28:13.167121+0000","last_unstale":"2023-09-23T20:28:14.160549+0000","last_undegraded":"2023-09-23T20:28:14.160549+0000","last_fullsized":"2023-09-23T20:28:13.140887+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:49:30.695766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":["3","4"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.0","version":"239'4885","reported_seq":4915,"reported_epoch":239,"state":"active+clean","last_fresh":"2023-09-23T20:28:19.920363+0000","last_change":"2023-09-23T20:27:14.267314+0000","last_active":"2023-09-23T20:28:19.920363+0000","last_peered":"2023-09-23T20:28:19.920363+0000","last_clean":"2023-09-23T20:28:19.920363+0000","last_became_active":"2023-09-23T20:27:14.267011+0000","last_became_peered":"2023-09-23T20:27:14.267011+0000","last_unstale":"2023-09-23T20:28:19.920363+0000","last_undegraded":"2023-09-23T20:28:19.920363+0000","last_fullsized":"2023-09-23T20:28:19.920363+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":4885,"log_dups_size":0,"ondisk_log_size":4885,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:47:58.542542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":320143360,"num_objects":4885,"num_object_clones":0,"num_object_copies":14655,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4885,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4885,"num_write_kb":312640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":448,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:28:14.160764+0000","last_change":"2023-09-23T20:28:08.032785+0000","last_active":"2023-09-23T20:28:14.160764+0000","last_peered":"2023-09-23T20:28:14.160764+0000","last_clean":"2023-09-23T20:28:14.160764+0000","last_became_active":"2023-09-23T20:28:08.032209+0000","last_became_peered":"2023-09-23T20:28:08.032209+0000","last_unstale":"2023-09-23T20:28:14.160764+0000","last_undegraded":"2023-09-23T20:28:14.160764+0000","last_fullsized":"2023-09-23T20:28:14.160764+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:32.746340+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:32.746340+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"86'119","reported_seq":408,"reported_epoch":238,"state":"active+undersized+degraded","last_fresh":"2023-09-23T20:28:14.938421+0000","last_change":"2023-09-23T20:28:13.147446+0000","last_active":"2023-09-23T20:28:14.938421+0000","last_peered":"2023-09-23T20:28:14.938421+0000","last_clean":"2023-09-23T20:13:53.395005+0000","last_became_active":"2023-09-23T20:28:13.147446+0000","last_became_peered":"2023-09-23T20:28:13.147446+0000","last_unstale":"2023-09-23T20:28:14.938421+0000","last_undegraded":"2023-09-23T20:28:13.139220+0000","last_fullsized":"2023-09-23T20:28:13.139106+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":119,"log_dups_size":0,"ondisk_log_size":119,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:50:31.476259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032705499999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":523,"num_read_kb":3503,"num_write":136,"num_write_kb":3128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[{"shards":"0,7","objects":1}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":255,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:28:14.156003+0000","last_change":"2023-09-23T20:25:39.637160+0000","last_active":"2023-09-23T20:28:14.156003+0000","last_peered":"2023-09-23T20:28:14.156003+0000","last_clean":"2023-09-23T20:28:14.156003+0000","last_became_active":"2023-09-23T20:25:39.636860+0000","last_became_peered":"2023-09-23T20:25:39.636860+0000","last_unstale":"2023-09-23T20:28:14.156003+0000","last_undegraded":"2023-09-23T20:28:14.156003+0000","last_fullsized":"2023-09-23T20:28:14.156003+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:21:47.860063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":26,"reported_epoch":238,"state":"active+undersized","last_fresh":"2023-09-23T20:28:14.155532+0000","last_change":"2023-09-23T20:28:13.176485+0000","last_active":"2023-09-23T20:28:14.155532+0000","last_peered":"2023-09-23T20:28:14.155532+0000","last_clean":"2023-09-23T20:12:43.732826+0000","last_became_active":"2023-09-23T20:28:13.176485+0000","last_became_peered":"2023-09-23T20:28:13.176485+0000","last_unstale":"2023-09-23T20:28:14.155532+0000","last_undegraded":"2023-09-23T20:28:14.155532+0000","last_fullsized":"2023-09-23T20:28:13.155600+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:59:12.518891+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":["5","4"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.6","version":"239'4927","reported_seq":4958,"reported_epoch":239,"state":"active+clean","last_fresh":"2023-09-23T20:28:23.043753+0000","last_change":"2023-09-23T20:27:14.269226+0000","last_active":"2023-09-23T20:28:23.043753+0000","last_peered":"2023-09-23T20:28:23.043753+0000","last_clean":"2023-09-23T20:28:23.043753+0000","last_became_active":"2023-09-23T20:27:14.268927+0000","last_became_peered":"2023-09-23T20:27:14.268927+0000","last_unstale":"2023-09-23T20:28:23.043753+0000","last_undegraded":"2023-09-23T20:28:23.043753+0000","last_fullsized":"2023-09-23T20:28:23.043753+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":4927,"log_dups_size":0,"ondisk_log_size":4927,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:45:57.844328+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":322895872,"num_objects":4927,"num_object_clones":0,"num_object_copies":14781,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4927,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4927,"num_write_kb":315328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":473,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:28:14.155276+0000","last_change":"2023-09-23T20:27:55.213707+0000","last_active":"2023-09-23T20:28:14.155276+0000","last_peered":"2023-09-23T20:28:14.155276+0000","last_clean":"2023-09-23T20:28:14.155276+0000","last_became_active":"2023-09-23T20:27:55.213394+0000","last_became_peered":"2023-09-23T20:27:55.213394+0000","last_unstale":"2023-09-23T20:28:14.155276+0000","last_undegraded":"2023-09-23T20:28:14.155276+0000","last_fullsized":"2023-09-23T20:28:14.155276+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:34.821789+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:34.821789+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0042962870000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"86'106","reported_seq":912,"reported_epoch":238,"state":"active+undersized+degraded","last_fresh":"2023-09-23T20:28:14.155331+0000","last_change":"2023-09-23T20:28:13.164484+0000","last_active":"2023-09-23T20:28:14.155331+0000","last_peered":"2023-09-23T20:28:14.155331+0000","last_clean":"2023-09-23T20:28:10.123134+0000","last_became_active":"2023-09-23T20:28:13.164484+0000","last_became_peered":"2023-09-23T20:28:13.164484+0000","last_unstale":"2023-09-23T20:28:14.155331+0000","last_undegraded":"2023-09-23T20:28:13.155126+0000","last_fullsized":"2023-09-23T20:28:13.155031+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":106,"log_dups_size":0,"ondisk_log_size":106,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:37.280911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00099302599999999993,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1105,"num_read_kb":1689,"num_write":100,"num_write_kb":404,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":["5","0"],"object_location_counts":[{"shards":"0,5","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":215,"reported_epoch":238,"state":"active+undersized","last_fresh":"2023-09-23T20:28:14.519284+0000","last_change":"2023-09-23T20:28:13.170036+0000","last_active":"2023-09-23T20:28:14.519284+0000","last_peered":"2023-09-23T20:28:14.519284+0000","last_clean":"2023-09-23T20:28:10.122078+0000","last_became_active":"2023-09-23T20:28:13.170036+0000","last_became_peered":"2023-09-23T20:28:13.170036+0000","last_unstale":"2023-09-23T20:28:14.519284+0000","last_undegraded":"2023-09-23T20:28:14.519284+0000","last_fullsized":"2023-09-23T20:28:13.149199+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:13:13.987896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":["4","0"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":170,"reported_epoch":238,"state":"active+undersized","last_fresh":"2023-09-23T20:28:17.073216+0000","last_change":"2023-09-23T20:28:13.152577+0000","last_active":"2023-09-23T20:28:17.073216+0000","last_peered":"2023-09-23T20:28:17.073216+0000","last_clean":"2023-09-23T20:25:34.579305+0000","last_became_active":"2023-09-23T20:28:13.152577+0000","last_became_peered":"2023-09-23T20:28:13.152577+0000","last_unstale":"2023-09-23T20:28:17.073216+0000","last_undegraded":"2023-09-23T20:28:17.073216+0000","last_fullsized":"2023-09-23T20:28:13.139696+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:55:11.899806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"6.5","version":"239'4774","reported_seq":4813,"reported_epoch":239,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:28:20.562652+0000","last_change":"2023-09-23T20:28:06.190483+0000","last_active":"2023-09-23T20:28:20.562652+0000","last_peered":"2023-09-23T20:28:20.562652+0000","last_clean":"2023-09-23T20:28:04.012743+0000","last_became_active":"2023-09-23T20:28:06.171815+0000","last_became_peered":"2023-09-23T20:28:06.171815+0000","last_unstale":"2023-09-23T20:28:20.562652+0000","last_undegraded":"2023-09-23T20:28:06.026412+0000","last_fullsized":"2023-09-23T20:28:05.981687+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":4774,"log_dups_size":0,"ondisk_log_size":4774,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:15:42.482730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":312868864,"num_objects":4774,"num_object_clones":0,"num_object_copies":14322,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4228,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4774,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4774,"num_write_kb":305536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3],"avail_no_missing":["7","3","6"],"object_location_counts":[{"shards":"3,6,7","objects":4228},{"shards":"3,7","objects":546}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":631,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:28:14.148148+0000","last_change":"2023-09-23T20:27:55.209408+0000","last_active":"2023-09-23T20:28:14.148148+0000","last_peered":"2023-09-23T20:28:14.148148+0000","last_clean":"2023-09-23T20:28:14.148148+0000","last_became_active":"2023-09-23T20:27:55.209043+0000","last_became_peered":"2023-09-23T20:27:55.209043+0000","last_unstale":"2023-09-23T20:28:14.148148+0000","last_undegraded":"2023-09-23T20:28:14.148148+0000","last_fullsized":"2023-09-23T20:28:14.148148+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:35.734584+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:35.734584+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0035251380000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"86'130","reported_seq":549,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:28:14.145637+0000","last_change":"2023-09-23T20:25:39.637458+0000","last_active":"2023-09-23T20:28:14.145637+0000","last_peered":"2023-09-23T20:28:14.145637+0000","last_clean":"2023-09-23T20:28:14.145637+0000","last_became_active":"2023-09-23T20:25:39.636773+0000","last_became_peered":"2023-09-23T20:25:39.636773+0000","last_unstale":"2023-09-23T20:28:14.145637+0000","last_undegraded":"2023-09-23T20:28:14.145637+0000","last_fullsized":"2023-09-23T20:28:14.145637+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:00:32.810564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011300049999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":512,"num_read_kb":1820,"num_write":133,"num_write_kb":1347,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":200,"reported_epoch":238,"state":"active+undersized","last_fresh":"2023-09-23T20:28:17.069785+0000","last_change":"2023-09-23T20:28:13.144161+0000","last_active":"2023-09-23T20:28:17.069785+0000","last_peered":"2023-09-23T20:28:17.069785+0000","last_clean":"2023-09-23T20:28:02.997876+0000","last_became_active":"2023-09-23T20:28:13.144161+0000","last_became_peered":"2023-09-23T20:28:13.144161+0000","last_unstale":"2023-09-23T20:28:17.069785+0000","last_undegraded":"2023-09-23T20:28:17.069785+0000","last_fullsized":"2023-09-23T20:28:13.138363+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:26.025784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":["2","0"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":69,"reported_epoch":238,"state":"active+undersized","last_fresh":"2023-09-23T20:28:14.155890+0000","last_change":"2023-09-23T20:28:13.168863+0000","last_active":"2023-09-23T20:28:14.155890+0000","last_peered":"2023-09-23T20:28:14.155890+0000","last_clean":"2023-09-23T20:15:59.442389+0000","last_became_active":"2023-09-23T20:28:13.168863+0000","last_became_peered":"2023-09-23T20:28:13.168863+0000","last_unstale":"2023-09-23T20:28:14.155890+0000","last_undegraded":"2023-09-23T20:28:14.155890+0000","last_fullsized":"2023-09-23T20:28:13.153675+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:30:33.149977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":["5","0"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.7","version":"239'4951","reported_seq":4981,"reported_epoch":239,"state":"active+clean","last_fresh":"2023-09-23T20:28:20.596425+0000","last_change":"2023-09-23T20:27:14.272939+0000","last_active":"2023-09-23T20:28:20.596425+0000","last_peered":"2023-09-23T20:28:20.596425+0000","last_clean":"2023-09-23T20:28:20.596425+0000","last_became_active":"2023-09-23T20:27:14.272096+0000","last_became_peered":"2023-09-23T20:27:14.272096+0000","last_unstale":"2023-09-23T20:28:20.596425+0000","last_undegraded":"2023-09-23T20:28:20.596425+0000","last_fullsized":"2023-09-23T20:28:20.596425+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":4951,"log_dups_size":0,"ondisk_log_size":4951,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:41:48.494925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":324468736,"num_objects":4951,"num_object_clones":0,"num_object_copies":14853,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4951,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4951,"num_write_kb":316864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"86'194","reported_seq":887,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:28:14.155386+0000","last_change":"2023-09-23T20:25:40.618665+0000","last_active":"2023-09-23T20:28:14.155386+0000","last_peered":"2023-09-23T20:28:14.155386+0000","last_clean":"2023-09-23T20:28:14.155386+0000","last_became_active":"2023-09-23T20:25:40.618087+0000","last_became_peered":"2023-09-23T20:25:40.618087+0000","last_unstale":"2023-09-23T20:28:14.155386+0000","last_undegraded":"2023-09-23T20:28:14.155386+0000","last_fullsized":"2023-09-23T20:28:14.155386+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":194,"log_dups_size":0,"ondisk_log_size":194,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:07:16.114968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00037995999999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1025,"num_read_kb":1201,"num_write":173,"num_write_kb":1276,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":642,"reported_epoch":238,"state":"active+undersized","last_fresh":"2023-09-23T20:28:14.147968+0000","last_change":"2023-09-23T20:28:13.146277+0000","last_active":"2023-09-23T20:28:14.147968+0000","last_peered":"2023-09-23T20:28:14.147968+0000","last_clean":"2023-09-23T20:28:10.104150+0000","last_became_active":"2023-09-23T20:28:13.146277+0000","last_became_peered":"2023-09-23T20:28:13.146277+0000","last_unstale":"2023-09-23T20:28:14.147968+0000","last_undegraded":"2023-09-23T20:28:14.147968+0000","last_fullsized":"2023-09-23T20:28:13.141588+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:32.801168+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:32.801168+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0096333110000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":["7","0"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":94,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:28:14.159585+0000","last_change":"2023-09-23T20:28:05.036669+0000","last_active":"2023-09-23T20:28:14.159585+0000","last_peered":"2023-09-23T20:28:14.159585+0000","last_clean":"2023-09-23T20:28:14.159585+0000","last_became_active":"2023-09-23T20:28:05.036379+0000","last_became_peered":"2023-09-23T20:28:05.036379+0000","last_unstale":"2023-09-23T20:28:14.159585+0000","last_undegraded":"2023-09-23T20:28:14.159585+0000","last_fullsized":"2023-09-23T20:28:14.159585+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:35:57.806627+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":22,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:28:14.519807+0000","last_change":"2023-09-23T20:28:05.024905+0000","last_active":"2023-09-23T20:28:14.519807+0000","last_peered":"2023-09-23T20:28:14.519807+0000","last_clean":"2023-09-23T20:28:14.519807+0000","last_became_active":"2023-09-23T20:28:05.024596+0000","last_became_peered":"2023-09-23T20:28:05.024596+0000","last_unstale":"2023-09-23T20:28:14.519807+0000","last_undegraded":"2023-09-23T20:28:14.519807+0000","last_fullsized":"2023-09-23T20:28:14.519807+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:23:18.124135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.4","version":"239'4815","reported_seq":4851,"reported_epoch":239,"state":"active+undersized+degraded","last_fresh":"2023-09-23T20:28:20.573084+0000","last_change":"2023-09-23T20:28:13.166676+0000","last_active":"2023-09-23T20:28:20.573084+0000","last_peered":"2023-09-23T20:28:20.573084+0000","last_clean":"2023-09-23T20:28:12.083147+0000","last_became_active":"2023-09-23T20:28:13.166676+0000","last_became_peered":"2023-09-23T20:28:13.166676+0000","last_unstale":"2023-09-23T20:28:20.573084+0000","last_undegraded":"2023-09-23T20:28:13.152307+0000","last_fullsized":"2023-09-23T20:28:13.152178+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":4815,"log_dups_size":0,"ondisk_log_size":4815,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:57:14.413869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":315555840,"num_objects":4815,"num_object_clones":0,"num_object_copies":14445,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4815,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4815,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4815,"num_write_kb":308160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":["5","3"],"object_location_counts":[{"shards":"3,5","objects":4815}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":428,"reported_epoch":238,"state":"active+undersized","last_fresh":"2023-09-23T20:28:14.519032+0000","last_change":"2023-09-23T20:28:13.170340+0000","last_active":"2023-09-23T20:28:14.519032+0000","last_peered":"2023-09-23T20:28:14.519032+0000","last_clean":"2023-09-23T20:15:59.452160+0000","last_became_active":"2023-09-23T20:28:13.170340+0000","last_became_peered":"2023-09-23T20:28:13.170340+0000","last_unstale":"2023-09-23T20:28:14.519032+0000","last_undegraded":"2023-09-23T20:28:14.519032+0000","last_fullsized":"2023-09-23T20:28:13.149972+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:12.386098+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:12.386098+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5],"acting":[4,5],"avail_no_missing":["4","5"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.0","version":"86'80","reported_seq":492,"reported_epoch":238,"state":"active+undersized","last_fresh":"2023-09-23T20:28:14.147948+0000","last_change":"2023-09-23T20:28:13.144600+0000","last_active":"2023-09-23T20:28:14.147948+0000","last_peered":"2023-09-23T20:28:14.147948+0000","last_clean":"2023-09-23T20:28:10.104142+0000","last_became_active":"2023-09-23T20:28:13.144600+0000","last_became_peered":"2023-09-23T20:28:13.144600+0000","last_unstale":"2023-09-23T20:28:14.147948+0000","last_undegraded":"2023-09-23T20:28:14.147948+0000","last_fullsized":"2023-09-23T20:28:13.141588+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":80,"log_dups_size":0,"ondisk_log_size":80,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:37:42.749177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035115,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":395,"num_read_kb":1370,"num_write":66,"num_write_kb":1061,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":["7","0"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":201,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:28:14.142763+0000","last_change":"2023-09-23T20:25:02.942709+0000","last_active":"2023-09-23T20:28:14.142763+0000","last_peered":"2023-09-23T20:28:14.142763+0000","last_clean":"2023-09-23T20:28:14.142763+0000","last_became_active":"2023-09-23T20:25:02.942323+0000","last_became_peered":"2023-09-23T20:25:02.942323+0000","last_unstale":"2023-09-23T20:28:14.142763+0000","last_undegraded":"2023-09-23T20:28:14.142763+0000","last_fullsized":"2023-09-23T20:28:14.142763+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:26:12.039101+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":223,"reported_epoch":238,"state":"active+undersized","last_fresh":"2023-09-23T20:28:14.143166+0000","last_change":"2023-09-23T20:28:13.144536+0000","last_active":"2023-09-23T20:28:14.143166+0000","last_peered":"2023-09-23T20:28:14.143166+0000","last_clean":"2023-09-23T20:28:10.114404+0000","last_became_active":"2023-09-23T20:28:13.144536+0000","last_became_peered":"2023-09-23T20:28:13.144536+0000","last_unstale":"2023-09-23T20:28:14.143166+0000","last_undegraded":"2023-09-23T20:28:14.143166+0000","last_fullsized":"2023-09-23T20:28:13.140563+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:50:22.944488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2],"avail_no_missing":["0","2"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"239'4901","reported_seq":5479,"reported_epoch":239,"state":"active+recovering+degraded+remapped","last_fresh":"2023-09-23T20:28:22.100889+0000","last_change":"2023-09-23T20:28:14.564013+0000","last_active":"2023-09-23T20:28:22.100889+0000","last_peered":"2023-09-23T20:28:22.100889+0000","last_clean":"2023-09-23T20:28:04.011715+0000","last_became_active":"2023-09-23T20:28:14.533872+0000","last_became_peered":"2023-09-23T20:28:14.533872+0000","last_unstale":"2023-09-23T20:28:22.100889+0000","last_undegraded":"2023-09-23T20:28:14.239168+0000","last_fullsized":"2023-09-23T20:28:22.100889+0000","mapping_epoch":237,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":4901,"log_dups_size":0,"ondisk_log_size":4901,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:32:25.529799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":321191936,"num_objects":4901,"num_object_clones":0,"num_object_copies":14703,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4901,"num_objects_misplaced":4211,"num_objects_unfound":0,"num_objects_dirty":4901,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4901,"num_write_kb":313664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":266,"num_bytes_recovered":17432576,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7],"acting":[2,7,6],"avail_no_missing":["2","6"],"object_location_counts":[{"shards":"2,6","objects":4211},{"shards":"2,6,7","objects":690}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":441,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:28:14.142765+0000","last_change":"2023-09-23T20:28:08.028468+0000","last_active":"2023-09-23T20:28:14.142765+0000","last_peered":"2023-09-23T20:28:14.142765+0000","last_clean":"2023-09-23T20:28:14.142765+0000","last_became_active":"2023-09-23T20:28:08.027976+0000","last_became_peered":"2023-09-23T20:28:08.027976+0000","last_unstale":"2023-09-23T20:28:14.142765+0000","last_undegraded":"2023-09-23T20:28:14.142765+0000","last_fullsized":"2023-09-23T20:28:14.142765+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:31.769333+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:31.769333+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"86'102","reported_seq":745,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:28:14.148539+0000","last_change":"2023-09-23T20:20:14.881975+0000","last_active":"2023-09-23T20:28:14.148539+0000","last_peered":"2023-09-23T20:28:14.148539+0000","last_clean":"2023-09-23T20:28:14.148539+0000","last_became_active":"2023-09-23T20:20:14.881544+0000","last_became_peered":"2023-09-23T20:20:14.881544+0000","last_unstale":"2023-09-23T20:28:14.148539+0000","last_undegraded":"2023-09-23T20:28:14.148539+0000","last_fullsized":"2023-09-23T20:28:14.148539+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":102,"log_dups_size":0,"ondisk_log_size":102,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:34.594972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030656500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":963,"num_read_kb":1234,"num_write":60,"num_write_kb":225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":136,"reported_epoch":238,"state":"active+undersized","last_fresh":"2023-09-23T20:28:17.073144+0000","last_change":"2023-09-23T20:28:13.158522+0000","last_active":"2023-09-23T20:28:17.073144+0000","last_peered":"2023-09-23T20:28:17.073144+0000","last_clean":"2023-09-23T20:25:34.579453+0000","last_became_active":"2023-09-23T20:28:13.158522+0000","last_became_peered":"2023-09-23T20:28:13.158522+0000","last_unstale":"2023-09-23T20:28:17.073144+0000","last_undegraded":"2023-09-23T20:28:17.073144+0000","last_fullsized":"2023-09-23T20:28:13.139418+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:41:36.312605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5],"acting":[2,5],"avail_no_missing":["2","5"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":169,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:28:14.142555+0000","last_change":"2023-09-23T20:20:14.888874+0000","last_active":"2023-09-23T20:28:14.142555+0000","last_peered":"2023-09-23T20:28:14.142555+0000","last_clean":"2023-09-23T20:28:14.142555+0000","last_became_active":"2023-09-23T20:20:14.887724+0000","last_became_peered":"2023-09-23T20:20:14.887724+0000","last_unstale":"2023-09-23T20:28:14.142555+0000","last_undegraded":"2023-09-23T20:28:14.142555+0000","last_fullsized":"2023-09-23T20:28:14.142555+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:09:24.510326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"239'4929","reported_seq":4959,"reported_epoch":239,"state":"active+clean","last_fresh":"2023-09-23T20:28:20.671930+0000","last_change":"2023-09-23T20:27:14.270161+0000","last_active":"2023-09-23T20:28:20.671930+0000","last_peered":"2023-09-23T20:28:20.671930+0000","last_clean":"2023-09-23T20:28:20.671930+0000","last_became_active":"2023-09-23T20:27:14.269904+0000","last_became_peered":"2023-09-23T20:27:14.269904+0000","last_unstale":"2023-09-23T20:28:20.671930+0000","last_undegraded":"2023-09-23T20:28:20.671930+0000","last_fullsized":"2023-09-23T20:28:20.671930+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":4929,"log_dups_size":0,"ondisk_log_size":4929,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:41:02.917630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":323026944,"num_objects":4929,"num_object_clones":0,"num_object_copies":14787,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4929,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4929,"num_write_kb":315456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":499,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:28:15.850461+0000","last_change":"2023-09-23T20:28:15.850428+0000","last_active":"2023-09-23T20:28:15.850461+0000","last_peered":"2023-09-23T20:28:15.850461+0000","last_clean":"2023-09-23T20:28:15.850461+0000","last_became_active":"2023-09-23T20:28:08.033922+0000","last_became_peered":"2023-09-23T20:28:08.033922+0000","last_unstale":"2023-09-23T20:28:15.850461+0000","last_undegraded":"2023-09-23T20:28:15.850461+0000","last_fullsized":"2023-09-23T20:28:15.850461+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:12.809933+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:12.809933+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.001626628,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":89,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:28:14.160621+0000","last_change":"2023-09-23T20:25:03.007736+0000","last_active":"2023-09-23T20:28:14.160621+0000","last_peered":"2023-09-23T20:28:14.160621+0000","last_clean":"2023-09-23T20:28:14.160621+0000","last_became_active":"2023-09-23T20:25:02.973689+0000","last_became_peered":"2023-09-23T20:25:02.973689+0000","last_unstale":"2023-09-23T20:28:14.160621+0000","last_undegraded":"2023-09-23T20:28:14.160621+0000","last_fullsized":"2023-09-23T20:28:14.160621+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:37:25.643774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"86'128","reported_seq":411,"reported_epoch":238,"state":"active+undersized","last_fresh":"2023-09-23T20:28:14.519821+0000","last_change":"2023-09-23T20:28:13.174258+0000","last_active":"2023-09-23T20:28:14.519821+0000","last_peered":"2023-09-23T20:28:14.519821+0000","last_clean":"2023-09-23T20:13:51.938835+0000","last_became_active":"2023-09-23T20:28:13.174258+0000","last_became_peered":"2023-09-23T20:28:13.174258+0000","last_unstale":"2023-09-23T20:28:14.519821+0000","last_undegraded":"2023-09-23T20:28:14.519821+0000","last_fullsized":"2023-09-23T20:28:13.149835+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:09:56.949690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000293202,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":579,"num_read_kb":1473,"num_write":96,"num_write_kb":1062,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":["4","3"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":85,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:28:14.159311+0000","last_change":"2023-09-23T20:25:02.971740+0000","last_active":"2023-09-23T20:28:14.159311+0000","last_peered":"2023-09-23T20:28:14.159311+0000","last_clean":"2023-09-23T20:28:14.159311+0000","last_became_active":"2023-09-23T20:25:02.970412+0000","last_became_peered":"2023-09-23T20:25:02.970412+0000","last_unstale":"2023-09-23T20:28:14.159311+0000","last_undegraded":"2023-09-23T20:28:14.159311+0000","last_fullsized":"2023-09-23T20:28:14.159311+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:53:17.247216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"239'4847","reported_seq":4902,"reported_epoch":239,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T20:28:20.568958+0000","last_change":"2023-09-23T20:28:14.534996+0000","last_active":"2023-09-23T20:28:20.568958+0000","last_peered":"2023-09-23T20:28:20.568958+0000","last_clean":"2023-09-23T20:28:04.010682+0000","last_became_active":"2023-09-23T20:28:14.529925+0000","last_became_peered":"2023-09-23T20:28:14.529925+0000","last_unstale":"2023-09-23T20:28:20.568958+0000","last_undegraded":"2023-09-23T20:28:14.153040+0000","last_fullsized":"2023-09-23T20:28:20.568958+0000","mapping_epoch":237,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":4847,"log_dups_size":0,"ondisk_log_size":4847,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:03:54.745914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":317652992,"num_objects":4847,"num_object_clones":0,"num_object_copies":14541,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":261,"num_objects_misplaced":4586,"num_objects_unfound":0,"num_objects_dirty":4847,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4847,"num_write_kb":310208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":196608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2,6],"avail_no_missing":["7","2"],"object_location_counts":[{"shards":"2,6,7","objects":4586},{"shards":"2,7","objects":261}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.4","version":"90'161","reported_seq":485,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:28:14.161677+0000","last_change":"2023-09-23T20:28:03.553294+0000","last_active":"2023-09-23T20:28:14.161677+0000","last_peered":"2023-09-23T20:28:14.161677+0000","last_clean":"2023-09-23T20:28:14.161677+0000","last_became_active":"2023-09-23T20:27:55.212201+0000","last_became_peered":"2023-09-23T20:27:55.212201+0000","last_unstale":"2023-09-23T20:28:14.161677+0000","last_undegraded":"2023-09-23T20:28:14.161677+0000","last_fullsized":"2023-09-23T20:28:14.161677+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:28:03.553106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:28:03.553106+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:24:06.706940+0000","scrub_duration":0.011176864,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"86'117","reported_seq":601,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:28:14.148710+0000","last_change":"2023-09-23T20:28:08.028985+0000","last_active":"2023-09-23T20:28:14.148710+0000","last_peered":"2023-09-23T20:28:14.148710+0000","last_clean":"2023-09-23T20:28:14.148710+0000","last_became_active":"2023-09-23T20:28:08.028620+0000","last_became_peered":"2023-09-23T20:28:08.028620+0000","last_unstale":"2023-09-23T20:28:14.148710+0000","last_undegraded":"2023-09-23T20:28:14.148710+0000","last_fullsized":"2023-09-23T20:28:14.148710+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:52:44.688867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046891400000000002,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":893,"num_read_kb":922,"num_write":82,"num_write_kb":4193,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":195,"reported_epoch":238,"state":"active+undersized+degraded","last_fresh":"2023-09-23T20:28:14.143035+0000","last_change":"2023-09-23T20:28:13.176296+0000","last_active":"2023-09-23T20:28:14.143035+0000","last_peered":"2023-09-23T20:28:14.143035+0000","last_clean":"2023-09-23T20:28:10.114330+0000","last_became_active":"2023-09-23T20:28:13.176296+0000","last_became_peered":"2023-09-23T20:28:13.176296+0000","last_unstale":"2023-09-23T20:28:14.143035+0000","last_undegraded":"2023-09-23T20:28:13.140349+0000","last_fullsized":"2023-09-23T20:28:13.140255+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:12:18.491192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":["0","4"],"object_location_counts":[{"shards":"0,4","objects":1}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":190,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:28:14.518577+0000","last_change":"2023-09-23T20:28:05.035425+0000","last_active":"2023-09-23T20:28:14.518577+0000","last_peered":"2023-09-23T20:28:14.518577+0000","last_clean":"2023-09-23T20:28:14.518577+0000","last_became_active":"2023-09-23T20:28:05.035079+0000","last_became_peered":"2023-09-23T20:28:05.035079+0000","last_unstale":"2023-09-23T20:28:14.518577+0000","last_undegraded":"2023-09-23T20:28:14.518577+0000","last_fullsized":"2023-09-23T20:28:14.518577+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:34:34.293359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":172,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:28:14.142475+0000","last_change":"2023-09-23T20:20:14.888484+0000","last_active":"2023-09-23T20:28:14.142475+0000","last_peered":"2023-09-23T20:28:14.142475+0000","last_clean":"2023-09-23T20:28:14.142475+0000","last_became_active":"2023-09-23T20:20:14.887219+0000","last_became_peered":"2023-09-23T20:20:14.887219+0000","last_unstale":"2023-09-23T20:28:14.142475+0000","last_undegraded":"2023-09-23T20:28:14.142475+0000","last_fullsized":"2023-09-23T20:28:14.142475+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:13:53.763649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":80,"reported_epoch":238,"state":"active+undersized","last_fresh":"2023-09-23T20:28:14.146203+0000","last_change":"2023-09-23T20:28:13.152589+0000","last_active":"2023-09-23T20:28:14.146203+0000","last_peered":"2023-09-23T20:28:14.146203+0000","last_clean":"2023-09-23T20:28:12.056838+0000","last_became_active":"2023-09-23T20:28:13.152589+0000","last_became_peered":"2023-09-23T20:28:13.152589+0000","last_unstale":"2023-09-23T20:28:14.146203+0000","last_undegraded":"2023-09-23T20:28:14.146203+0000","last_fullsized":"2023-09-23T20:28:13.139976+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:54:38.464315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":207,"reported_epoch":238,"state":"active+undersized+degraded","last_fresh":"2023-09-23T20:28:14.155539+0000","last_change":"2023-09-23T20:28:13.169405+0000","last_active":"2023-09-23T20:28:14.155539+0000","last_peered":"2023-09-23T20:28:14.155539+0000","last_clean":"2023-09-23T20:28:10.121149+0000","last_became_active":"2023-09-23T20:28:13.169405+0000","last_became_peered":"2023-09-23T20:28:13.169405+0000","last_unstale":"2023-09-23T20:28:14.155539+0000","last_undegraded":"2023-09-23T20:28:13.153461+0000","last_fullsized":"2023-09-23T20:28:13.153347+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:03:24.383853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":["5","4"],"object_location_counts":[{"shards":"4,5","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":251,"reported_epoch":238,"state":"active+undersized","last_fresh":"2023-09-23T20:28:14.519406+0000","last_change":"2023-09-23T20:28:13.169986+0000","last_active":"2023-09-23T20:28:14.519406+0000","last_peered":"2023-09-23T20:28:14.519406+0000","last_clean":"2023-09-23T20:28:10.122290+0000","last_became_active":"2023-09-23T20:28:13.169986+0000","last_became_peered":"2023-09-23T20:28:13.169986+0000","last_unstale":"2023-09-23T20:28:14.519406+0000","last_undegraded":"2023-09-23T20:28:14.519406+0000","last_fullsized":"2023-09-23T20:28:13.148881+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:44.116758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":["4","3"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":204,"reported_epoch":238,"state":"active+undersized+degraded","last_fresh":"2023-09-23T20:28:14.155638+0000","last_change":"2023-09-23T20:28:13.171105+0000","last_active":"2023-09-23T20:28:14.155638+0000","last_peered":"2023-09-23T20:28:14.155638+0000","last_clean":"2023-09-23T20:28:10.121557+0000","last_became_active":"2023-09-23T20:28:13.171105+0000","last_became_peered":"2023-09-23T20:28:13.171105+0000","last_unstale":"2023-09-23T20:28:14.155638+0000","last_undegraded":"2023-09-23T20:28:13.154069+0000","last_fullsized":"2023-09-23T20:28:13.153978+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:04:22.559138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":["5","4"],"object_location_counts":[{"shards":"4,5","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":244,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:28:14.518689+0000","last_change":"2023-09-23T20:25:03.103656+0000","last_active":"2023-09-23T20:28:14.518689+0000","last_peered":"2023-09-23T20:28:14.518689+0000","last_clean":"2023-09-23T20:28:14.518689+0000","last_became_active":"2023-09-23T20:25:03.103311+0000","last_became_peered":"2023-09-23T20:25:03.103311+0000","last_unstale":"2023-09-23T20:28:14.518689+0000","last_undegraded":"2023-09-23T20:28:14.518689+0000","last_fullsized":"2023-09-23T20:28:14.518689+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:55:54.677030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":109,"reported_epoch":238,"state":"active+undersized","last_fresh":"2023-09-23T20:28:14.160930+0000","last_change":"2023-09-23T20:28:13.146288+0000","last_active":"2023-09-23T20:28:14.160930+0000","last_peered":"2023-09-23T20:28:14.160930+0000","last_clean":"2023-09-23T20:28:10.123189+0000","last_became_active":"2023-09-23T20:28:13.146288+0000","last_became_peered":"2023-09-23T20:28:13.146288+0000","last_unstale":"2023-09-23T20:28:14.160930+0000","last_undegraded":"2023-09-23T20:28:14.160930+0000","last_fullsized":"2023-09-23T20:28:13.141006+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:49:57.680453+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":["3","2"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":94,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:28:14.159692+0000","last_change":"2023-09-23T20:25:39.622819+0000","last_active":"2023-09-23T20:28:14.159692+0000","last_peered":"2023-09-23T20:28:14.159692+0000","last_clean":"2023-09-23T20:28:14.159692+0000","last_became_active":"2023-09-23T20:25:39.622484+0000","last_became_peered":"2023-09-23T20:25:39.622484+0000","last_unstale":"2023-09-23T20:28:14.159692+0000","last_undegraded":"2023-09-23T20:28:14.159692+0000","last_fullsized":"2023-09-23T20:28:14.159692+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:51:53.973261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":217,"reported_epoch":238,"state":"active+clean","last_fresh":"2023-09-23T20:28:14.142872+0000","last_change":"2023-09-23T20:25:39.668882+0000","last_active":"2023-09-23T20:28:14.142872+0000","last_peered":"2023-09-23T20:28:14.142872+0000","last_clean":"2023-09-23T20:28:14.142872+0000","last_became_active":"2023-09-23T20:25:39.637128+0000","last_became_peered":"2023-09-23T20:25:39.637128+0000","last_unstale":"2023-09-23T20:28:14.142872+0000","last_undegraded":"2023-09-23T20:28:14.142872+0000","last_fullsized":"2023-09-23T20:28:14.142872+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:56.406298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":234,"reported_epoch":238,"state":"active+undersized","last_fresh":"2023-09-23T20:28:14.518455+0000","last_change":"2023-09-23T20:28:13.156720+0000","last_active":"2023-09-23T20:28:14.518455+0000","last_peered":"2023-09-23T20:28:14.518455+0000","last_clean":"2023-09-23T20:28:10.121358+0000","last_became_active":"2023-09-23T20:28:13.156720+0000","last_became_peered":"2023-09-23T20:28:13.156720+0000","last_unstale":"2023-09-23T20:28:14.518455+0000","last_undegraded":"2023-09-23T20:28:14.518455+0000","last_fullsized":"2023-09-23T20:28:13.147190+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:21:27.183630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":["4","3"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":5123080192,"num_objects":78172,"num_object_clones":0,"num_object_copies":234516,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":32593,"num_objects_misplaced":12475,"num_objects_unfound":0,"num_objects_dirty":78172,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":78172,"num_write_kb":5003008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1303,"num_bytes_recovered":85393408,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":15366619136,"data_stored":15366619136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":78172,"ondisk_log_size":78172,"up":42,"acting":41,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":76,"acting":76,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":25,"num_bytes_recovered":6160,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":25181,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":81,"acting":81,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":713712,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":192,"num_read_kb":360,"num_write":95,"num_write_kb":953,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":1025388,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2437120,"data_stored":2426464,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3381,"ondisk_log_size":3381,"up":57,"acting":57,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":4194323,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16793600,"data_stored":16777292,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6082,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":20,"acting":20,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201486,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2080964,"kb_used_data":1942524,"kb_used_omap":0,"kb_used_meta":138432,"kb_avail":91664188,"statfs":{"total":95995035648,"available":93864128512,"internally_reserved":0,"allocated":1989144576,"data_stored":1988400269,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":141754368},"hb_peers":[0,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:27:24 2023","interfaces":[{"interface":"back","average":{"1min":3.9889999999999999,"5min":1.9059999999999999,"15min":1.0149999999999999},"min":{"1min":0.29399999999999998,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":48.798999999999999,"5min":48.798999999999999,"15min":48.798999999999999},"last":2.52},{"interface":"front","average":{"1min":3.9540000000000002,"5min":2.2770000000000001,"15min":1.135},"min":{"1min":0.251,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":48.247,"5min":48.247,"15min":48.247},"last":0.57899999999999996}]},{"osd":1,"last update":"Sat Sep 23 20:27:59 2023","interfaces":[{"interface":"back","average":{"1min":1.1759999999999999,"5min":1.7689999999999999,"15min":0.96599999999999997},"min":{"1min":0.29199999999999998,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":5.1509999999999998,"5min":17.795999999999999,"15min":17.795999999999999},"last":3.1429999999999998},{"interface":"front","average":{"1min":0.75900000000000001,"5min":1.5349999999999999,"15min":0.91900000000000004},"min":{"1min":0.33700000000000002,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":1.9219999999999999,"5min":13.250999999999999,"15min":13.250999999999999},"last":0.33100000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:27:28 2023","interfaces":[{"interface":"back","average":{"1min":0.98799999999999999,"5min":1.6919999999999999,"15min":0.94299999999999995},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":2.9580000000000002,"5min":9.8300000000000001,"15min":9.8300000000000001},"last":5.0599999999999996},{"interface":"front","average":{"1min":1.51,"5min":1.8919999999999999,"15min":1.022},"min":{"1min":0.39200000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":7.4720000000000004,"5min":11.423999999999999,"15min":11.423999999999999},"last":2.5390000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:27:59 2023","interfaces":[{"interface":"back","average":{"1min":1.3120000000000001,"5min":1.8879999999999999,"15min":1.0660000000000001},"min":{"1min":0.46200000000000002,"5min":0.314,"15min":0.314},"max":{"1min":4.7880000000000003,"5min":6.891,"15min":6.891},"last":2.673},{"interface":"front","average":{"1min":1.4299999999999999,"5min":1.7789999999999999,"15min":1.042},"min":{"1min":0.46999999999999997,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":5.8209999999999997,"5min":9.7840000000000007,"15min":9.7840000000000007},"last":2.101}]},{"osd":4,"last update":"Sat Sep 23 20:27:59 2023","interfaces":[{"interface":"back","average":{"1min":0.79100000000000004,"5min":1.2170000000000001,"15min":0.73699999999999999},"min":{"1min":0.27100000000000002,"5min":0.27000000000000002,"15min":0.26700000000000002},"max":{"1min":2.2629999999999999,"5min":13.141999999999999,"15min":13.141999999999999},"last":2.4409999999999998},{"interface":"front","average":{"1min":0.96599999999999997,"5min":1.7529999999999999,"15min":0.94299999999999995},"min":{"1min":0.34300000000000003,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":4.9909999999999997,"5min":10.853,"15min":10.853},"last":0.32400000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:27:59 2023","interfaces":[{"interface":"back","average":{"1min":1.4690000000000001,"5min":1.8240000000000001,"15min":0.96599999999999997},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":13.241,"5min":13.241,"15min":13.241},"last":2.5059999999999998},{"interface":"front","average":{"1min":1.4330000000000001,"5min":1.7370000000000001,"15min":0.96699999999999997},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":14.012,"5min":14.012,"15min":14.012},"last":2.4689999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:27:59 2023","interfaces":[{"interface":"back","average":{"1min":0.78300000000000003,"5min":1.4470000000000001,"15min":0.88200000000000001},"min":{"1min":0.35499999999999998,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":2.7989999999999999,"5min":13.185,"15min":13.185},"last":2.4940000000000002},{"interface":"front","average":{"1min":0.78900000000000003,"5min":1.5169999999999999,"15min":0.90600000000000003},"min":{"1min":0.40000000000000002,"5min":0.33000000000000002,"15min":0.29899999999999999},"max":{"1min":1.968,"5min":9.9120000000000008,"15min":9.9120000000000008},"last":0.55100000000000005}]}]},{"osd":6,"up_from":0,"seq":201863463122,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":99,"seq":425201762395,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1618524,"kb_used_data":1418708,"kb_used_omap":6,"kb_used_meta":199801,"kb_avail":92126628,"statfs":{"total":95995035648,"available":94337667072,"internally_reserved":0,"allocated":1452756992,"data_stored":1451546929,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6696,"internal_metadata":204596696},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:27:55 2023","interfaces":[{"interface":"back","average":{"1min":2.044,"5min":2.3929999999999998,"15min":1.046},"min":{"1min":0.224,"5min":0.20599999999999999,"15min":0.19600000000000001},"max":{"1min":17.904,"5min":40.113,"15min":40.113},"last":0.28199999999999997},{"interface":"front","average":{"1min":1.8799999999999999,"5min":2.1150000000000002,"15min":0.96899999999999997},"min":{"1min":0.192,"5min":0.192,"15min":0.192},"max":{"1min":17.812000000000001,"5min":42.994999999999997,"15min":42.994999999999997},"last":7.4850000000000003}]},{"osd":2,"last update":"Sat Sep 23 20:27:30 2023","interfaces":[{"interface":"back","average":{"1min":2.0299999999999998,"5min":2.1320000000000001,"15min":0.96499999999999997},"min":{"1min":0.30099999999999999,"5min":0.253,"15min":0.253},"max":{"1min":17.940000000000001,"5min":39.962000000000003,"15min":39.962000000000003},"last":0.59099999999999997},{"interface":"front","average":{"1min":2.3730000000000002,"5min":2.1579999999999999,"15min":1.087},"min":{"1min":0.26300000000000001,"5min":0.246,"15min":0.246},"max":{"1min":15.090999999999999,"5min":47.442,"15min":47.442},"last":8.7899999999999991}]},{"osd":3,"last update":"Sat Sep 23 20:28:02 2023","interfaces":[{"interface":"back","average":{"1min":1.361,"5min":1.5660000000000001,"15min":0.88800000000000001},"min":{"1min":0.32900000000000001,"5min":0.246,"15min":0.246},"max":{"1min":4.0030000000000001,"5min":9.0410000000000004,"15min":9.0410000000000004},"last":0.22500000000000001},{"interface":"front","average":{"1min":2.2000000000000002,"5min":1.8300000000000001,"15min":0.88200000000000001},"min":{"1min":0.248,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":17.853000000000002,"5min":17.853000000000002,"15min":17.853000000000002},"last":7.7439999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:27:55 2023","interfaces":[{"interface":"back","average":{"1min":1.359,"5min":1.7370000000000001,"15min":0.98999999999999999},"min":{"1min":0.33600000000000002,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":4.4550000000000001,"5min":15.353,"15min":15.353},"last":0.32300000000000001},{"interface":"front","average":{"1min":1.071,"5min":1.625,"15min":0.96399999999999997},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":3.6480000000000001,"5min":8.7899999999999991,"15min":8.7899999999999991},"last":0.42499999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:27:55 2023","interfaces":[{"interface":"back","average":{"1min":1.143,"5min":1.72,"15min":1.0009999999999999},"min":{"1min":0.32800000000000001,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":4.6920000000000002,"5min":12.590999999999999,"15min":12.590999999999999},"last":7.5460000000000003},{"interface":"front","average":{"1min":1.605,"5min":2.3250000000000002,"15min":1.2250000000000001},"min":{"1min":0.40100000000000002,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":4.5369999999999999,"5min":39.579000000000001,"15min":39.579000000000001},"last":0.35699999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:27:55 2023","interfaces":[{"interface":"back","average":{"1min":1.6379999999999999,"5min":2.198,"15min":1.175},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":9.0310000000000006,"5min":34.409999999999997,"15min":34.409999999999997},"last":7.5750000000000002},{"interface":"front","average":{"1min":1.6140000000000001,"5min":2.4220000000000002,"15min":1.232},"min":{"1min":0.48599999999999999,"5min":0.377,"15min":0.377},"max":{"1min":4.5860000000000003,"5min":35.554000000000002,"15min":35.554000000000002},"last":0.39000000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:27:55 2023","interfaces":[{"interface":"back","average":{"1min":1.617,"5min":2.044,"15min":1.115},"min":{"1min":0.505,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":4.6139999999999999,"5min":9.7439999999999998,"15min":9.7439999999999998},"last":4.1660000000000004},{"interface":"front","average":{"1min":1.3200000000000001,"5min":1.9079999999999999,"15min":1.089},"min":{"1min":0.41299999999999998,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":5.7439999999999998,"5min":9.2279999999999998,"15min":9.2279999999999998},"last":0.40100000000000002}]}]},{"osd":0,"up_from":94,"seq":403726925923,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1505328,"kb_used_data":1254184,"kb_used_omap":7,"kb_used_meta":251128,"kb_avail":92239824,"statfs":{"total":95995035648,"available":94453579776,"internally_reserved":0,"allocated":1284284416,"data_stored":1283152541,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":257155089},"hb_peers":[2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:27:57 2023","interfaces":[{"interface":"back","average":{"1min":0.97099999999999997,"5min":1.4650000000000001,"15min":0.72599999999999998},"min":{"1min":0.26300000000000001,"5min":0.20100000000000001,"15min":0.17699999999999999},"max":{"1min":4.7060000000000004,"5min":24.247,"15min":24.247},"last":4.3479999999999999},{"interface":"front","average":{"1min":0.98899999999999999,"5min":1.4339999999999999,"15min":0.73599999999999999},"min":{"1min":0.27900000000000003,"5min":0.222,"15min":0.222},"max":{"1min":2.9289999999999998,"5min":15.994999999999999,"15min":15.994999999999999},"last":6.4279999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:27:24 2023","interfaces":[{"interface":"back","average":{"1min":1.2190000000000001,"5min":1.9450000000000001,"15min":1.046},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23300000000000001},"max":{"1min":7.5199999999999996,"5min":23.832999999999998,"15min":23.832999999999998},"last":0.39700000000000002},{"interface":"front","average":{"1min":2.2490000000000001,"5min":1.607,"15min":0.86599999999999999},"min":{"1min":0.19700000000000001,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":15.948,"5min":15.948,"15min":15.948},"last":0.27000000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:27:57 2023","interfaces":[{"interface":"back","average":{"1min":0.81499999999999995,"5min":1.7509999999999999,"15min":0.98099999999999998},"min":{"1min":0.28999999999999998,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":2.0539999999999998,"5min":23.748999999999999,"15min":23.748999999999999},"last":0.439},{"interface":"front","average":{"1min":1.123,"5min":1.659,"15min":0.83399999999999996},"min":{"1min":0.29299999999999998,"5min":0.24099999999999999,"15min":0.23899999999999999},"max":{"1min":4.7930000000000001,"5min":24.274999999999999,"15min":24.274999999999999},"last":2.1629999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:27:24 2023","interfaces":[{"interface":"back","average":{"1min":1.55,"5min":2.0960000000000001,"15min":1.1100000000000001},"min":{"1min":0.36899999999999999,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":6.5899999999999999,"5min":23.876999999999999,"15min":23.876999999999999},"last":0.47899999999999998},{"interface":"front","average":{"1min":1.53,"5min":1.772,"15min":0.999},"min":{"1min":0.39500000000000002,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":6.0300000000000002,"5min":23.681999999999999,"15min":23.681999999999999},"last":2.2240000000000002}]},{"osd":5,"last update":"Sat Sep 23 20:27:24 2023","interfaces":[{"interface":"back","average":{"1min":1.8640000000000001,"5min":2.4729999999999999,"15min":1.262},"min":{"1min":0.47399999999999998,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":9.3369999999999997,"5min":24.198,"15min":24.198},"last":3.0089999999999999},{"interface":"front","average":{"1min":1.1539999999999999,"5min":1.8500000000000001,"15min":1.069},"min":{"1min":0.41799999999999998,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":6.1749999999999998,"5min":11.167999999999999,"15min":11.167999999999999},"last":2.6150000000000002}]},{"osd":6,"last update":"Sat Sep 23 20:27:24 2023","interfaces":[{"interface":"back","average":{"1min":1.6799999999999999,"5min":1.7749999999999999,"15min":1.0269999999999999},"min":{"1min":0.42299999999999999,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":9.2750000000000004,"5min":24.408000000000001,"15min":24.408000000000001},"last":2.976},{"interface":"front","average":{"1min":1.1259999999999999,"5min":1.6140000000000001,"15min":0.98099999999999998},"min":{"1min":0.442,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":6.1319999999999997,"5min":25.271000000000001,"15min":25.271000000000001},"last":2.4790000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:27:24 2023","interfaces":[{"interface":"back","average":{"1min":1.1619999999999999,"5min":1.8180000000000001,"15min":1.0569999999999999},"min":{"1min":0.47799999999999998,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":6.2039999999999997,"5min":25.359000000000002,"15min":25.359000000000002},"last":0.56399999999999995},{"interface":"front","average":{"1min":1.5169999999999999,"5min":2.3119999999999998,"15min":1.2270000000000001},"min":{"1min":0.45600000000000002,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":9.8239999999999998,"5min":23.905999999999999,"15min":23.905999999999999},"last":0.51000000000000001}]}]},{"osd":2,"up_from":104,"seq":446676598871,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2858860,"kb_used_data":2660708,"kb_used_omap":11,"kb_used_meta":198132,"kb_avail":90886292,"statfs":{"total":95995035648,"available":93067563008,"internally_reserved":0,"allocated":2724564992,"data_stored":2723439178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11878,"internal_metadata":202887578},"hb_peers":[0,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:27:30 2023","interfaces":[{"interface":"back","average":{"1min":0.85099999999999998,"5min":1.8009999999999999,"15min":0.85099999999999998},"min":{"1min":0.29399999999999998,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":3.302,"5min":12.811,"15min":12.811},"last":0.19500000000000001},{"interface":"front","average":{"1min":0.76400000000000001,"5min":1.3440000000000001,"15min":0.71199999999999997},"min":{"1min":0.28399999999999997,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":4.0279999999999996,"5min":15.746,"15min":15.746},"last":1.6459999999999999}]},{"osd":1,"last update":"Sat Sep 23 20:27:30 2023","interfaces":[{"interface":"back","average":{"1min":0.93799999999999994,"5min":1.633,"15min":0.80700000000000005},"min":{"1min":0.29799999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":3.3410000000000002,"5min":14.135999999999999,"15min":14.135999999999999},"last":8.8049999999999997},{"interface":"front","average":{"1min":1.863,"5min":1.522,"15min":0.76200000000000001},"min":{"1min":0.27500000000000002,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":14.201000000000001,"5min":14.201000000000001,"15min":14.201000000000001},"last":2.1640000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:27:58 2023","interfaces":[{"interface":"back","average":{"1min":1.591,"5min":1.595,"15min":0.77300000000000002},"min":{"1min":0.28199999999999997,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":8.4849999999999994,"5min":15.319000000000001,"15min":15.319000000000001},"last":0.28799999999999998},{"interface":"front","average":{"1min":2.0510000000000002,"5min":1.8819999999999999,"15min":0.89200000000000002},"min":{"1min":0.40500000000000003,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":12.067,"5min":12.494999999999999,"15min":12.494999999999999},"last":0.38900000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:27:30 2023","interfaces":[{"interface":"back","average":{"1min":1.452,"5min":1.407,"15min":0.85799999999999998},"min":{"1min":0.38100000000000001,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":5.8799999999999999,"5min":8.2300000000000004,"15min":8.2300000000000004},"last":1.7989999999999999},{"interface":"front","average":{"1min":0.64300000000000002,"5min":1.409,"15min":0.85899999999999999},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":1.288,"5min":8.6530000000000005,"15min":8.6530000000000005},"last":0.32700000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:27:30 2023","interfaces":[{"interface":"back","average":{"1min":0.98199999999999998,"5min":1.6419999999999999,"15min":0.98499999999999999},"min":{"1min":0.41099999999999998,"5min":0.309,"15min":0.309},"max":{"1min":4.0810000000000004,"5min":11.237,"15min":11.237},"last":1.6819999999999999},{"interface":"front","average":{"1min":0.628,"5min":1.679,"15min":0.997},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":1.129,"5min":11.266999999999999,"15min":11.266999999999999},"last":6.8049999999999997}]},{"osd":6,"last update":"Sat Sep 23 20:27:30 2023","interfaces":[{"interface":"back","average":{"1min":0.90100000000000002,"5min":1.343,"15min":0.88500000000000001},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":5.1139999999999999,"5min":6.7939999999999996,"15min":6.7939999999999996},"last":6.8559999999999999},{"interface":"front","average":{"1min":1.3919999999999999,"5min":1.7290000000000001,"15min":1.0249999999999999},"min":{"1min":0.43099999999999999,"5min":0.377,"15min":0.377},"max":{"1min":6.0099999999999998,"5min":9.6600000000000001,"15min":9.6600000000000001},"last":2.4529999999999998}]},{"osd":7,"last update":"Sat Sep 23 20:27:30 2023","interfaces":[{"interface":"back","average":{"1min":1.3999999999999999,"5min":1.8620000000000001,"15min":1.052},"min":{"1min":0.48199999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":5.9649999999999999,"5min":15.343999999999999,"15min":15.343999999999999},"last":3.8889999999999998},{"interface":"front","average":{"1min":1.2010000000000001,"5min":1.6930000000000001,"15min":1.002},"min":{"1min":0.51100000000000001,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":4.1459999999999999,"5min":10.064,"15min":10.064},"last":1.8520000000000001}]}]},{"osd":3,"up_from":109,"seq":468151435345,"num_pgs":58,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3358960,"kb_used_data":3169000,"kb_used_omap":4,"kb_used_meta":189947,"kb_avail":90386192,"statfs":{"total":95995035648,"available":92555460608,"internally_reserved":0,"allocated":3245056000,"data_stored":3243921119,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4514,"internal_metadata":194506334},"hb_peers":[0,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,0,0,3],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:27:59 2023","interfaces":[{"interface":"back","average":{"1min":1.8340000000000001,"5min":2.6160000000000001,"15min":1.1060000000000001},"min":{"1min":0.23499999999999999,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":6.7460000000000004,"5min":67.305999999999997,"15min":67.305999999999997},"last":5.149},{"interface":"front","average":{"1min":1.409,"5min":2.2389999999999999,"15min":1.0029999999999999},"min":{"1min":0.29399999999999998,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":7.0999999999999996,"5min":67.403999999999996,"15min":67.403999999999996},"last":1.4239999999999999}]},{"osd":1,"last update":"Sat Sep 23 20:27:59 2023","interfaces":[{"interface":"back","average":{"1min":1.026,"5min":1.633,"15min":0.80900000000000005},"min":{"1min":0.221,"5min":0.214,"15min":0.214},"max":{"1min":4.6390000000000002,"5min":19.675999999999998,"15min":19.675999999999998},"last":1.0169999999999999},{"interface":"front","average":{"1min":1.371,"5min":1.5600000000000001,"15min":0.81599999999999995},"min":{"1min":0.29099999999999998,"5min":0.245,"15min":0.245},"max":{"1min":7.0800000000000001,"5min":11.644,"15min":11.644},"last":3.2770000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:27:59 2023","interfaces":[{"interface":"back","average":{"1min":1.4710000000000001,"5min":2.0030000000000001,"15min":0.90300000000000002},"min":{"1min":0.25600000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":7.069,"5min":24.045000000000002,"15min":24.045000000000002},"last":0.55400000000000005},{"interface":"front","average":{"1min":1.236,"5min":1.962,"15min":0.96299999999999997},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":5.5300000000000002,"5min":17.684000000000001,"15min":17.684000000000001},"last":0.67700000000000005}]},{"osd":4,"last update":"Sat Sep 23 20:27:59 2023","interfaces":[{"interface":"back","average":{"1min":3.121,"5min":1.962,"15min":1.0569999999999999},"min":{"1min":0.36899999999999999,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":46.042999999999999,"5min":46.042999999999999,"15min":46.042999999999999},"last":5.2220000000000004},{"interface":"front","average":{"1min":2.9540000000000002,"5min":2.286,"15min":1.1719999999999999},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":46.579999999999998,"5min":46.579999999999998,"15min":46.579999999999998},"last":0.64400000000000002}]},{"osd":5,"last update":"Sat Sep 23 20:27:59 2023","interfaces":[{"interface":"back","average":{"1min":3.1459999999999999,"5min":2.3639999999999999,"15min":1.1919999999999999},"min":{"1min":0.442,"5min":0.311,"15min":0.311},"max":{"1min":45.292000000000002,"5min":45.292000000000002,"15min":45.292000000000002},"last":5.194},{"interface":"front","average":{"1min":3.177,"5min":1.804,"15min":1.022},"min":{"1min":0.42199999999999999,"5min":0.313,"15min":0.313},"max":{"1min":44.679000000000002,"5min":44.679000000000002,"15min":44.679000000000002},"last":2.016}]},{"osd":6,"last update":"Sat Sep 23 20:27:59 2023","interfaces":[{"interface":"back","average":{"1min":1.2689999999999999,"5min":1.7170000000000001,"15min":0.997},"min":{"1min":0.45400000000000001,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":6.9180000000000001,"5min":8.6059999999999999,"15min":8.6059999999999999},"last":5.6890000000000001},{"interface":"front","average":{"1min":1.075,"5min":1.655,"15min":0.999},"min":{"1min":0.39700000000000002,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":3.9329999999999998,"5min":9.3179999999999996,"15min":9.3179999999999996},"last":0.60199999999999998}]},{"osd":7,"last update":"Sat Sep 23 20:27:59 2023","interfaces":[{"interface":"back","average":{"1min":1.294,"5min":1.704,"15min":1.036},"min":{"1min":0.374,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":8.4640000000000004,"5min":9.2750000000000004,"15min":9.2750000000000004},"last":3.3149999999999999},{"interface":"front","average":{"1min":3.444,"5min":2.5819999999999999,"15min":1.288},"min":{"1min":0.41099999999999998,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":45.387999999999998,"5min":45.387999999999998,"15min":45.387999999999998},"last":5.9859999999999998}]}]},{"osd":4,"up_from":30,"seq":128849019098,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1705788,"kb_used_data":1555892,"kb_used_omap":0,"kb_used_meta":149888,"kb_avail":92039364,"statfs":{"total":95995035648,"available":94248308736,"internally_reserved":0,"allocated":1593233408,"data_stored":1592496900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":153485312},"hb_peers":[0,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,2,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:27:24 2023","interfaces":[{"interface":"back","average":{"1min":2.028,"5min":1.6950000000000001,"15min":0.94699999999999995},"min":{"1min":0.23699999999999999,"5min":0.222,"15min":0.222},"max":{"1min":6.5780000000000003,"5min":20.411999999999999,"15min":20.411999999999999},"last":0.29499999999999998},{"interface":"front","average":{"1min":1.7170000000000001,"5min":1.7649999999999999,"15min":1.016},"min":{"1min":0.307,"5min":0.253,"15min":0.253},"max":{"1min":6.5919999999999996,"5min":20.454999999999998,"15min":20.454999999999998},"last":1.2370000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:28:01 2023","interfaces":[{"interface":"back","average":{"1min":1.7509999999999999,"5min":1.748,"15min":0.995},"min":{"1min":0.35699999999999998,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":8.7249999999999996,"5min":9.6660000000000004,"15min":9.6660000000000004},"last":3.4529999999999998},{"interface":"front","average":{"1min":0.70799999999999996,"5min":1.595,"15min":0.94399999999999995},"min":{"1min":0.32700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":2.2149999999999999,"5min":12.821999999999999,"15min":12.821999999999999},"last":0.30599999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:27:33 2023","interfaces":[{"interface":"back","average":{"1min":2.1179999999999999,"5min":1.8120000000000001,"15min":1.0089999999999999},"min":{"1min":0.28599999999999998,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":6.7279999999999998,"5min":21.462,"15min":21.462},"last":0.33100000000000002},{"interface":"front","average":{"1min":1.7529999999999999,"5min":2.0990000000000002,"15min":1.0920000000000001},"min":{"1min":0.29799999999999999,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":6.1799999999999997,"5min":22.593,"15min":22.593},"last":1.292}]},{"osd":3,"last update":"Sat Sep 23 20:28:01 2023","interfaces":[{"interface":"back","average":{"1min":1.3120000000000001,"5min":1.9279999999999999,"15min":1.0700000000000001},"min":{"1min":0.33400000000000002,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":8.0519999999999996,"5min":24.891999999999999,"15min":24.891999999999999},"last":1.8919999999999999},{"interface":"front","average":{"1min":1.212,"5min":1.9410000000000001,"15min":1.0900000000000001},"min":{"1min":0.371,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":4.6520000000000001,"5min":21.922000000000001,"15min":21.922000000000001},"last":2.0499999999999998}]},{"osd":5,"last update":"Sat Sep 23 20:28:15 2023","interfaces":[{"interface":"back","average":{"1min":1.1479999999999999,"5min":2.0099999999999998,"15min":0.997},"min":{"1min":0.26900000000000002,"5min":0.247,"15min":0.247},"max":{"1min":4.149,"5min":17.704000000000001,"15min":17.704000000000001},"last":0.372},{"interface":"front","average":{"1min":1.3640000000000001,"5min":1.8100000000000001,"15min":0.94299999999999995},"min":{"1min":0.35899999999999999,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":8.1379999999999999,"5min":15.329000000000001,"15min":15.329000000000001},"last":1.1970000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:27:35 2023","interfaces":[{"interface":"back","average":{"1min":1.21,"5min":2.0430000000000001,"15min":1.0329999999999999},"min":{"1min":0.38600000000000001,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":4.3259999999999996,"5min":41.371000000000002,"15min":41.371000000000002},"last":1.2649999999999999},{"interface":"front","average":{"1min":1.6679999999999999,"5min":1.8120000000000001,"15min":0.96299999999999997},"min":{"1min":0.372,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":7.7729999999999997,"5min":41.140999999999998,"15min":41.140999999999998},"last":0.40799999999999997}]},{"osd":7,"last update":"Sat Sep 23 20:27:51 2023","interfaces":[{"interface":"back","average":{"1min":0.85499999999999998,"5min":1.925,"15min":1.018},"min":{"1min":0.439,"5min":0.35099999999999998,"15min":0.33600000000000002},"max":{"1min":3.1840000000000002,"5min":16.427,"15min":16.427},"last":1.45},{"interface":"front","average":{"1min":0.86099999999999999,"5min":1.7989999999999999,"15min":0.98799999999999999},"min":{"1min":0.36499999999999999,"5min":0.34300000000000003,"15min":0.30499999999999999},"max":{"1min":2.4159999999999999,"5min":19.018999999999998,"15min":19.018999999999998},"last":0.439}]}]},{"osd":5,"up_from":39,"seq":167503724759,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1706232,"kb_used_data":1547568,"kb_used_omap":0,"kb_used_meta":158656,"kb_avail":92038920,"statfs":{"total":95995035648,"available":94247854080,"internally_reserved":0,"allocated":1584709632,"data_stored":1583973850,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":162463744},"hb_peers":[0,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:27:27 2023","interfaces":[{"interface":"back","average":{"1min":1.2310000000000001,"5min":1.671,"15min":0.95299999999999996},"min":{"1min":0.28399999999999997,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":5.915,"5min":32.475000000000001,"15min":32.475000000000001},"last":0.26100000000000001},{"interface":"front","average":{"1min":0.93300000000000005,"5min":1.671,"15min":0.95599999999999996},"min":{"1min":0.248,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":2.3359999999999999,"5min":32.457000000000001,"15min":32.457000000000001},"last":0.46400000000000002}]},{"osd":1,"last update":"Sat Sep 23 20:28:01 2023","interfaces":[{"interface":"back","average":{"1min":1.486,"5min":2.0059999999999998,"15min":1.032},"min":{"1min":0.25600000000000001,"5min":0.245,"15min":0.245},"max":{"1min":11.958,"5min":32.243000000000002,"15min":32.243000000000002},"last":2.0009999999999999},{"interface":"front","average":{"1min":1.5509999999999999,"5min":1.548,"15min":0.88100000000000001},"min":{"1min":0.29599999999999999,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":12.901,"5min":32.231000000000002,"15min":32.231000000000002},"last":8.532}]},{"osd":2,"last update":"Sat Sep 23 20:27:27 2023","interfaces":[{"interface":"back","average":{"1min":1.8109999999999999,"5min":2.1440000000000001,"15min":1.081},"min":{"1min":0.311,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":12.052,"5min":34.514000000000003,"15min":34.514000000000003},"last":0.82899999999999996},{"interface":"front","average":{"1min":1.194,"5min":1.859,"15min":0.99199999999999999},"min":{"1min":0.317,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":4.8470000000000004,"5min":32.515000000000001,"15min":32.515000000000001},"last":0.501}]},{"osd":3,"last update":"Sat Sep 23 20:28:01 2023","interfaces":[{"interface":"back","average":{"1min":1.4339999999999999,"5min":1.7649999999999999,"15min":0.95999999999999996},"min":{"1min":0.40300000000000002,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":8.391,"5min":12.846,"15min":12.846},"last":2.706},{"interface":"front","average":{"1min":1.516,"5min":1.734,"15min":0.98799999999999999},"min":{"1min":0.41699999999999998,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":12.005000000000001,"5min":12.005000000000001,"15min":12.005000000000001},"last":0.47999999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:27:18 2023","interfaces":[{"interface":"back","average":{"1min":2.8380000000000001,"5min":1.9019999999999999,"15min":0.94399999999999995},"min":{"1min":0.33600000000000002,"5min":0.26700000000000002,"15min":0.26400000000000001},"max":{"1min":12.984,"5min":14.218999999999999,"15min":14.218999999999999},"last":3.726},{"interface":"front","average":{"1min":1.5609999999999999,"5min":1.536,"15min":0.84499999999999997},"min":{"1min":0.38800000000000001,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":5.9779999999999998,"5min":11.664,"15min":11.664},"last":0.33700000000000002}]},{"osd":6,"last update":"Sat Sep 23 20:27:32 2023","interfaces":[{"interface":"back","average":{"1min":1.732,"5min":1.8089999999999999,"15min":0.94099999999999995},"min":{"1min":0.32400000000000001,"5min":0.30499999999999999,"15min":0.29799999999999999},"max":{"1min":12.890000000000001,"5min":12.890000000000001,"15min":12.890000000000001},"last":5.8780000000000001},{"interface":"front","average":{"1min":1.5049999999999999,"5min":1.6699999999999999,"15min":0.94399999999999995},"min":{"1min":0.42599999999999999,"5min":0.35999999999999999,"15min":0.34499999999999997},"max":{"1min":6.0339999999999998,"5min":11.456,"15min":11.456},"last":0.45500000000000002}]},{"osd":7,"last update":"Sat Sep 23 20:28:01 2023","interfaces":[{"interface":"back","average":{"1min":2.9300000000000002,"5min":2.0459999999999998,"15min":1.0489999999999999},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":36.621000000000002,"5min":36.621000000000002,"15min":36.621000000000002},"last":0.376},{"interface":"front","average":{"1min":2.9990000000000001,"5min":1.675,"15min":0.92900000000000005},"min":{"1min":0.38700000000000001,"5min":0.316,"15min":0.315},"max":{"1min":37.331000000000003,"5min":37.331000000000003,"15min":37.331000000000003},"last":1.766}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4722,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1360,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6696,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7156,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3154,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1281425408,"data_stored":1281425408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1455095808,"data_stored":1455095808,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2732523520,"data_stored":2732523520,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3256811520,"data_stored":3256811520,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1603600384,"data_stored":1603600384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1591345152,"data_stored":1591345152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1445986304,"data_stored":1445986304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1999831040,"data_stored":1999831040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:28:27.355 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-09-23T20:28:27.355 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd pool set unique_pool_1 pg_num 26 2023-09-23T20:28:27.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:27 smithi149 ceph-mon[170394]: pgmap v525: 109 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 11 active+undersized+degraded, 1 active+recovering+undersized+remapped, 38 active+undersized, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 55 active+clean; 4.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 2.7 KiB/s rd, 40 MiB/s wr, 645 op/s; 32600/236142 objects degraded (13.805%); 12475/236142 objects misplaced (5.283%); 5.5 MiB/s, 87 objects/s recovering 2023-09-23T20:28:27.589 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 65 16 83373 83357 80.1032 51.375 0.00498256 0.0124784 2023-09-23T20:28:28.589 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 66 16 84258 84242 79.7277 55.3125 0.0141005 0.0125375 2023-09-23T20:28:28.768 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:28:28.774 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:28.774 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:28:28.875 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:28:28.905 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:29.035 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:29 smithi018 ceph-mon[179647]: pgmap v526: 109 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 11 active+undersized+degraded, 1 active+recovering+undersized+remapped, 38 active+undersized, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 55 active+clean; 5.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 50 MiB/s wr, 806 op/s; 33632/244545 objects degraded (13.753%); 12391/244545 objects misplaced (5.067%); 5.2 MiB/s, 82 objects/s recovering 2023-09-23T20:28:29.035 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:29 smithi018 ceph-mon[184144]: pgmap v526: 109 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 11 active+undersized+degraded, 1 active+recovering+undersized+remapped, 38 active+undersized, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 55 active+clean; 5.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 50 MiB/s wr, 806 op/s; 33632/244545 objects degraded (13.753%); 12391/244545 objects misplaced (5.067%); 5.2 MiB/s, 82 objects/s recovering 2023-09-23T20:28:29.295 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:28:29.305 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:29.306 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:28:29.407 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:28:29.445 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:29 smithi149 ceph-mon[170394]: pgmap v526: 109 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 11 active+undersized+degraded, 1 active+recovering+undersized+remapped, 38 active+undersized, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 55 active+clean; 5.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 50 MiB/s wr, 806 op/s; 33632/244545 objects degraded (13.753%); 12391/244545 objects misplaced (5.067%); 5.2 MiB/s, 82 objects/s recovering 2023-09-23T20:28:29.451 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:29.514 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:28:29.518 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:29.518 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:28:29.589 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 67 16 84950 84934 79.1835 43.25 0.0145801 0.0126229 2023-09-23T20:28:29.619 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:28:29.742 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:29.994 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:28:30.004 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:30.004 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:28:30.105 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:28:30.154 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:30.209 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2023-09-23T20:28:30.209 INFO:teuthology.orchestra.run.smithi018.stdout: "ops": [], 2023-09-23T20:28:30.209 INFO:teuthology.orchestra.run.smithi018.stdout: "complaint_time": 30, 2023-09-23T20:28:30.209 INFO:teuthology.orchestra.run.smithi018.stdout: "num_blocked_ops": 0 2023-09-23T20:28:30.210 INFO:teuthology.orchestra.run.smithi018.stdout:} 2023-09-23T20:28:30.210 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:28:30.212 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:28:30.215 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:30.215 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:28:30.316 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:28:30.366 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:30.423 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:28:30.429 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:30.429 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:28:30.458 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:30 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:28:30.458 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:30 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:28:30.458 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:30 smithi018 ceph-mon[179647]: pgmap v527: 109 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 11 active+undersized+degraded, 1 active+recovering+undersized+remapped, 38 active+undersized, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 55 active+clean; 5.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 50 MiB/s wr, 798 op/s; 33887/246807 objects degraded (13.730%); 12391/246807 objects misplaced (5.021%); 4.2 MiB/s, 67 objects/s recovering 2023-09-23T20:28:30.458 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:30 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/4162120646' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:28:30.459 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:30 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:28:30.472 INFO:teuthology.orchestra.run.smithi018.stderr:nodeep-scrub is set 2023-09-23T20:28:30.530 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:28:30.588 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:30.591 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 68 16 85701 85685 78.7074 46.9375 0.0198638 0.0127003 2023-09-23T20:28:30.645 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:28:30.652 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:30.652 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:28:30.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:30 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:28:30.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:30 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:28:30.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:30 smithi149 ceph-mon[170394]: pgmap v527: 109 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 11 active+undersized+degraded, 1 active+recovering+undersized+remapped, 38 active+undersized, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 55 active+clean; 5.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 50 MiB/s wr, 798 op/s; 33887/246807 objects degraded (13.730%); 12391/246807 objects misplaced (5.021%); 4.2 MiB/s, 67 objects/s recovering 2023-09-23T20:28:30.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:30 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/4162120646' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:28:30.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:30 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:28:30.753 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:28:30.795 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:30.820 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:30 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:28:30.820 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:30 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:28:30.820 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:30 smithi018 ceph-mon[184144]: pgmap v527: 109 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 11 active+undersized+degraded, 1 active+recovering+undersized+remapped, 38 active+undersized, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 55 active+clean; 5.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 50 MiB/s wr, 798 op/s; 33887/246807 objects degraded (13.730%); 12391/246807 objects misplaced (5.021%); 4.2 MiB/s, 67 objects/s recovering 2023-09-23T20:28:30.820 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:30 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/4162120646' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:28:30.820 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:30 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:28:30.878 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:28:30.888 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:30.888 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:28:30.985 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T20:28:30.989 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:28:31.137 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:31.422 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:28:31.471 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:28:31.505 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:28:31.551 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:28:31.588 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:28:31.591 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 69 16 86473 86457 78.2662 48.25 0.0232097 0.0127714 2023-09-23T20:28:31.627 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:28:31.659 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:28:31.659 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:28:31.659 INFO:teuthology.orchestra.run.smithi018.stderr:Error ENXIO: problem getting command descriptions from osd.1 2023-09-23T20:28:31.659 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:28:31.660 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:28:31.660 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:28:31.660 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:28:31.660 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:28:31.660 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:28:31.679 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:31 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 33887/246807 objects degraded (13.730%), 15 pgs degraded (PG_DEGRADED) 2023-09-23T20:28:31.680 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:31 smithi018 ceph-mon[179647]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:28:31.680 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:31 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:28:31.680 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:31 smithi018 ceph-mon[179647]: osdmap e240: 8 total, 7 up, 7 in 2023-09-23T20:28:31.680 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:31 smithi018 ceph-mon[179647]: failed to encode map e240 with expected crc 2023-09-23T20:28:31.680 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:31 smithi018 ceph-mon[179647]: failed to encode map e240 with expected crc 2023-09-23T20:28:31.680 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:31 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1183366622' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:28:31.680 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:31 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 33887/246807 objects degraded (13.730%), 15 pgs degraded (PG_DEGRADED) 2023-09-23T20:28:31.680 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:31 smithi018 ceph-mon[184144]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:28:31.680 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:31 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:28:31.681 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:31 smithi018 ceph-mon[184144]: osdmap e240: 8 total, 7 up, 7 in 2023-09-23T20:28:31.681 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:31 smithi018 ceph-mon[184144]: failed to encode map e240 with expected crc 2023-09-23T20:28:31.681 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:31 smithi018 ceph-mon[184144]: failed to encode map e240 with expected crc 2023-09-23T20:28:31.681 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:31 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1183366622' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:28:31.715 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:28:31.718 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:31 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 33887/246807 objects degraded (13.730%), 15 pgs degraded (PG_DEGRADED) 2023-09-23T20:28:31.718 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:31 smithi149 ceph-mon[170394]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:28:31.718 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:31 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:28:31.718 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:31 smithi149 ceph-mon[170394]: osdmap e240: 8 total, 7 up, 7 in 2023-09-23T20:28:31.718 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:31 smithi149 ceph-mon[170394]: failed to encode map e240 with expected crc 2023-09-23T20:28:31.718 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:31 smithi149 ceph-mon[170394]: failed to encode map e240 with expected crc 2023-09-23T20:28:31.719 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:31 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1183366622' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:28:31.726 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:31.726 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:28:31.827 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:28:31.868 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:31.918 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:28:31.927 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:31.928 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:28:32.028 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:28:32.072 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:32.139 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:28:32.145 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:32.145 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:28:32.246 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:28:32.296 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:32.300 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T20:28:32.301 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T20:28:32.360 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:28:32.364 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:32.364 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:28:32.465 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:28:32.514 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:32.568 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:28:32.584 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:28:32.587 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:32.588 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:28:32.596 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 70 16 87275 87259 77.8606 50.125 0.0233066 0.012838 2023-09-23T20:28:32.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:32 smithi018 ceph-mon[184144]: pgmap v529: 109 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 11 active+undersized+degraded, 1 active+recovering+undersized+remapped, 38 active+undersized, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 55 active+clean; 5.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 51 MiB/s wr, 809 op/s; 34610/255042 objects degraded (13.570%); 12439/255042 objects misplaced (4.877%); 4.4 MiB/s, 70 objects/s recovering 2023-09-23T20:28:32.645 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:32 smithi018 ceph-mon[179647]: pgmap v529: 109 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 11 active+undersized+degraded, 1 active+recovering+undersized+remapped, 38 active+undersized, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 55 active+clean; 5.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 51 MiB/s wr, 809 op/s; 34610/255042 objects degraded (13.570%); 12439/255042 objects misplaced (4.877%); 4.4 MiB/s, 70 objects/s recovering 2023-09-23T20:28:32.688 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:28:32.776 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:32.842 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:28:32.848 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:32.849 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:28:32.949 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:28:32.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:32 smithi149 ceph-mon[170394]: pgmap v529: 109 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 11 active+undersized+degraded, 1 active+recovering+undersized+remapped, 38 active+undersized, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 55 active+clean; 5.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 51 MiB/s wr, 809 op/s; 34610/255042 objects degraded (13.570%); 12439/255042 objects misplaced (4.877%); 4.4 MiB/s, 70 objects/s recovering 2023-09-23T20:28:32.998 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:33.067 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:28:33.076 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:33.076 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:28:33.177 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:28:33.222 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:33.282 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:28:33.290 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:33.291 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:28:33.391 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:28:33.444 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:33.459 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T20:28:33.502 INFO:teuthology.orchestra.run.smithi018.stderr:set pool 6 pg_num to 26 2023-09-23T20:28:33.515 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:28:33.520 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:33.520 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:28:33.594 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 71 16 88068 88052 77.4622 49.5625 0.0330495 0.0129031 2023-09-23T20:28:33.621 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:28:33.711 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:33 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3391126720' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T20:28:33.711 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:33 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T20:28:33.712 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:33.715 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:33 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3391126720' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T20:28:33.715 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:33 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T20:28:33.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:33 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3391126720' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T20:28:33.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:33 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T20:28:33.942 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:28:33.945 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:33.946 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:28:34.047 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:28:34.076 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:34.141 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:28:34.148 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:34.149 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:28:34.250 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:28:34.301 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:28:34.303 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:34.349 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:28:34.355 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:34.355 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:28:34.456 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:28:34.500 INFO:teuthology.orchestra.run.smithi018.stdout:ERROR: (22) Invalid argument 2023-09-23T20:28:34.500 INFO:teuthology.orchestra.run.smithi018.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:28:34.597 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 72 16 88780 88764 77.0021 44.5 0.0800385 0.0129685 2023-09-23T20:28:34.620 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:28:34.632 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:34 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num", "val": "26"}]': finished 2023-09-23T20:28:34.632 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:34 smithi018 ceph-mon[179647]: osdmap e241: 8 total, 7 up, 7 in 2023-09-23T20:28:34.632 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:34 smithi018 ceph-mon[179647]: failed to encode map e241 with expected crc 2023-09-23T20:28:34.632 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:34 smithi018 ceph-mon[179647]: pgmap v531: 109 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 11 active+undersized+degraded, 1 active+recovering+undersized+remapped, 38 active+undersized, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 55 active+clean; 5.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 58 MiB/s wr, 935 op/s; 35075/258642 objects degraded (13.561%); 12263/258642 objects misplaced (4.741%); 5.5 MiB/s, 87 objects/s recovering 2023-09-23T20:28:34.632 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:34 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-09-23T20:28:34.633 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:34 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num", "val": "26"}]': finished 2023-09-23T20:28:34.633 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:34 smithi018 ceph-mon[184144]: osdmap e241: 8 total, 7 up, 7 in 2023-09-23T20:28:34.633 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:34 smithi018 ceph-mon[184144]: failed to encode map e241 with expected crc 2023-09-23T20:28:34.633 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:34 smithi018 ceph-mon[184144]: pgmap v531: 109 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 11 active+undersized+degraded, 1 active+recovering+undersized+remapped, 38 active+undersized, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 55 active+clean; 5.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 58 MiB/s wr, 935 op/s; 35075/258642 objects degraded (13.561%); 12263/258642 objects misplaced (4.741%); 5.5 MiB/s, 87 objects/s recovering 2023-09-23T20:28:34.633 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:34 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-09-23T20:28:34.637 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:34.722 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:34 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num", "val": "26"}]': finished 2023-09-23T20:28:34.722 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:34 smithi149 ceph-mon[170394]: osdmap e241: 8 total, 7 up, 7 in 2023-09-23T20:28:34.722 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:34 smithi149 ceph-mon[170394]: failed to encode map e241 with expected crc 2023-09-23T20:28:34.722 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:34 smithi149 ceph-mon[170394]: pgmap v531: 109 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 11 active+undersized+degraded, 1 active+recovering+undersized+remapped, 38 active+undersized, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 55 active+clean; 5.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 58 MiB/s wr, 935 op/s; 35075/258642 objects degraded (13.561%); 12263/258642 objects misplaced (4.741%); 5.5 MiB/s, 87 objects/s recovering 2023-09-23T20:28:34.722 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:34 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-09-23T20:28:35.391 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:28:34 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:28:34] "GET /metrics HTTP/1.1" 200 36547 "" "Prometheus/2.43.0" 2023-09-23T20:28:35.597 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 73 16 88812 88796 75.9752 2 0.00274999 0.0130872 2023-09-23T20:28:35.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:35 smithi018 ceph-mon[179647]: failed to encode map e242 with expected crc 2023-09-23T20:28:35.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:35 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num_actual", "val": "26"}]': finished 2023-09-23T20:28:35.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:35 smithi018 ceph-mon[179647]: osdmap e242: 8 total, 7 up, 7 in 2023-09-23T20:28:35.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:35 smithi018 ceph-mon[179647]: failed to encode map e242 with expected crc 2023-09-23T20:28:35.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:35 smithi018 ceph-mon[184144]: failed to encode map e242 with expected crc 2023-09-23T20:28:35.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:35 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num_actual", "val": "26"}]': finished 2023-09-23T20:28:35.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:35 smithi018 ceph-mon[184144]: osdmap e242: 8 total, 7 up, 7 in 2023-09-23T20:28:35.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:35 smithi018 ceph-mon[184144]: failed to encode map e242 with expected crc 2023-09-23T20:28:35.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:35 smithi149 ceph-mon[170394]: failed to encode map e242 with expected crc 2023-09-23T20:28:35.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:35 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pg_num_actual", "val": "26"}]': finished 2023-09-23T20:28:35.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:35 smithi149 ceph-mon[170394]: osdmap e242: 8 total, 7 up, 7 in 2023-09-23T20:28:35.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:35 smithi149 ceph-mon[170394]: failed to encode map e242 with expected crc 2023-09-23T20:28:36.597 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 74 16 89143 89127 75.2285 20.6875 0.00193831 0.0131499 2023-09-23T20:28:36.765 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:28:36.767 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:36.768 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:28:36.869 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:28:36.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:36 smithi018 ceph-mon[184144]: failed to encode map e243 with expected crc 2023-09-23T20:28:36.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:36 smithi018 ceph-mon[184144]: osdmap e243: 8 total, 7 up, 7 in 2023-09-23T20:28:36.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:36 smithi018 ceph-mon[184144]: failed to encode map e243 with expected crc 2023-09-23T20:28:36.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:36 smithi018 ceph-mon[184144]: failed to encode map e243 with expected crc 2023-09-23T20:28:36.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:36 smithi018 ceph-mon[184144]: failed to encode map e243 with expected crc 2023-09-23T20:28:36.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:36 smithi018 ceph-mon[184144]: failed to encode map e243 with expected crc 2023-09-23T20:28:36.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:36 smithi018 ceph-mon[184144]: failed to encode map e243 with expected crc 2023-09-23T20:28:36.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:36 smithi018 ceph-mon[184144]: failed to encode map e243 with expected crc 2023-09-23T20:28:36.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:36 smithi018 ceph-mon[184144]: failed to encode map e243 with expected crc 2023-09-23T20:28:36.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:36 smithi018 ceph-mon[184144]: pgmap v534: 119 pgs: 2 peering, 10 unknown, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+undersized+degraded, 1 active+recovering+undersized+remapped, 38 active+undersized, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 53 active+clean; 5.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 47 MiB/s wr, 756 op/s; 35075/259938 objects degraded (13.494%); 12263/259938 objects misplaced (4.718%); 4.3 MiB/s, 69 objects/s recovering 2023-09-23T20:28:36.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:36 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 35075/258642 objects degraded (13.561%), 15 pgs degraded (PG_DEGRADED) 2023-09-23T20:28:36.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:36 smithi018 ceph-mon[184144]: failed to encode map e244 with expected crc 2023-09-23T20:28:36.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:36 smithi018 ceph-mon[184144]: osdmap e244: 8 total, 7 up, 7 in 2023-09-23T20:28:36.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:36 smithi018 ceph-mon[184144]: failed to encode map e244 with expected crc 2023-09-23T20:28:36.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:36 smithi018 ceph-mon[184144]: failed to encode map e244 with expected crc 2023-09-23T20:28:36.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:36 smithi018 ceph-mon[179647]: failed to encode map e243 with expected crc 2023-09-23T20:28:36.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:36 smithi018 ceph-mon[179647]: osdmap e243: 8 total, 7 up, 7 in 2023-09-23T20:28:36.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:36 smithi018 ceph-mon[179647]: failed to encode map e243 with expected crc 2023-09-23T20:28:36.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:36 smithi018 ceph-mon[179647]: failed to encode map e243 with expected crc 2023-09-23T20:28:36.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:36 smithi018 ceph-mon[179647]: failed to encode map e243 with expected crc 2023-09-23T20:28:36.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:36 smithi018 ceph-mon[179647]: failed to encode map e243 with expected crc 2023-09-23T20:28:36.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:36 smithi018 ceph-mon[179647]: failed to encode map e243 with expected crc 2023-09-23T20:28:36.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:36 smithi018 ceph-mon[179647]: failed to encode map e243 with expected crc 2023-09-23T20:28:36.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:36 smithi018 ceph-mon[179647]: failed to encode map e243 with expected crc 2023-09-23T20:28:36.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:36 smithi018 ceph-mon[179647]: pgmap v534: 119 pgs: 2 peering, 10 unknown, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+undersized+degraded, 1 active+recovering+undersized+remapped, 38 active+undersized, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 53 active+clean; 5.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 47 MiB/s wr, 756 op/s; 35075/259938 objects degraded (13.494%); 12263/259938 objects misplaced (4.718%); 4.3 MiB/s, 69 objects/s recovering 2023-09-23T20:28:36.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:36 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 35075/258642 objects degraded (13.561%), 15 pgs degraded (PG_DEGRADED) 2023-09-23T20:28:36.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:36 smithi018 ceph-mon[179647]: failed to encode map e244 with expected crc 2023-09-23T20:28:36.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:36 smithi018 ceph-mon[179647]: osdmap e244: 8 total, 7 up, 7 in 2023-09-23T20:28:36.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:36 smithi018 ceph-mon[179647]: failed to encode map e244 with expected crc 2023-09-23T20:28:36.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:36 smithi018 ceph-mon[179647]: failed to encode map e244 with expected crc 2023-09-23T20:28:36.916 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:36 smithi149 ceph-mon[170394]: failed to encode map e243 with expected crc 2023-09-23T20:28:36.916 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:36 smithi149 ceph-mon[170394]: osdmap e243: 8 total, 7 up, 7 in 2023-09-23T20:28:36.916 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:36 smithi149 ceph-mon[170394]: failed to encode map e243 with expected crc 2023-09-23T20:28:36.916 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:36 smithi149 ceph-mon[170394]: failed to encode map e243 with expected crc 2023-09-23T20:28:36.916 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:36 smithi149 ceph-mon[170394]: failed to encode map e243 with expected crc 2023-09-23T20:28:36.916 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:36 smithi149 ceph-mon[170394]: failed to encode map e243 with expected crc 2023-09-23T20:28:36.916 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:36 smithi149 ceph-mon[170394]: failed to encode map e243 with expected crc 2023-09-23T20:28:36.916 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:36 smithi149 ceph-mon[170394]: failed to encode map e243 with expected crc 2023-09-23T20:28:36.916 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:36 smithi149 ceph-mon[170394]: failed to encode map e243 with expected crc 2023-09-23T20:28:36.916 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:36 smithi149 ceph-mon[170394]: pgmap v534: 119 pgs: 2 peering, 10 unknown, 2 active+recovery_wait+undersized+degraded+remapped, 11 active+undersized+degraded, 1 active+recovering+undersized+remapped, 38 active+undersized, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 53 active+clean; 5.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 47 MiB/s wr, 756 op/s; 35075/259938 objects degraded (13.494%); 12263/259938 objects misplaced (4.718%); 4.3 MiB/s, 69 objects/s recovering 2023-09-23T20:28:36.916 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:36 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 35075/258642 objects degraded (13.561%), 15 pgs degraded (PG_DEGRADED) 2023-09-23T20:28:36.917 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:36 smithi149 ceph-mon[170394]: failed to encode map e244 with expected crc 2023-09-23T20:28:36.917 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:36 smithi149 ceph-mon[170394]: osdmap e244: 8 total, 7 up, 7 in 2023-09-23T20:28:36.917 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:36 smithi149 ceph-mon[170394]: failed to encode map e244 with expected crc 2023-09-23T20:28:36.917 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:36 smithi149 ceph-mon[170394]: failed to encode map e244 with expected crc 2023-09-23T20:28:36.918 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:36.962 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:28:36.965 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:36.965 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:28:37.066 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:28:37.186 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:37.283 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:28:37.287 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:37.288 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:28:37.327 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T20:28:37.389 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:28:37.440 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:37.497 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:28:37.502 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:37.503 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:28:37.597 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 75 16 89773 89757 74.7506 39.375 0.00970957 0.0133725 2023-09-23T20:28:37.604 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:28:37.651 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:37.663 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:28:37.699 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:28:37.703 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:37 smithi149 ceph-mon[170394]: failed to encode map e244 with expected crc 2023-09-23T20:28:37.703 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:37 smithi149 ceph-mon[170394]: failed to encode map e244 with expected crc 2023-09-23T20:28:37.704 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:37 smithi149 ceph-mon[170394]: failed to encode map e245 with expected crc 2023-09-23T20:28:37.704 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:37 smithi149 ceph-mon[170394]: osdmap e245: 8 total, 7 up, 7 in 2023-09-23T20:28:37.704 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:37 smithi149 ceph-mon[170394]: failed to encode map e245 with expected crc 2023-09-23T20:28:37.704 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:37 smithi149 ceph-mon[170394]: failed to encode map e245 with expected crc 2023-09-23T20:28:37.704 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:37.705 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:28:37.790 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:37 smithi018 ceph-mon[179647]: failed to encode map e244 with expected crc 2023-09-23T20:28:37.790 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:37 smithi018 ceph-mon[179647]: failed to encode map e244 with expected crc 2023-09-23T20:28:37.791 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:37 smithi018 ceph-mon[179647]: failed to encode map e245 with expected crc 2023-09-23T20:28:37.791 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:37 smithi018 ceph-mon[179647]: osdmap e245: 8 total, 7 up, 7 in 2023-09-23T20:28:37.791 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:37 smithi018 ceph-mon[179647]: failed to encode map e245 with expected crc 2023-09-23T20:28:37.791 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:37 smithi018 ceph-mon[179647]: failed to encode map e245 with expected crc 2023-09-23T20:28:37.791 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:37 smithi018 ceph-mon[184144]: failed to encode map e244 with expected crc 2023-09-23T20:28:37.791 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:37 smithi018 ceph-mon[184144]: failed to encode map e244 with expected crc 2023-09-23T20:28:37.791 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:37 smithi018 ceph-mon[184144]: failed to encode map e245 with expected crc 2023-09-23T20:28:37.791 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:37 smithi018 ceph-mon[184144]: osdmap e245: 8 total, 7 up, 7 in 2023-09-23T20:28:37.791 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:37 smithi018 ceph-mon[184144]: failed to encode map e245 with expected crc 2023-09-23T20:28:37.791 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:37 smithi018 ceph-mon[184144]: failed to encode map e245 with expected crc 2023-09-23T20:28:37.806 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:28:37.881 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:37.949 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:28:37.955 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:37.956 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:28:38.056 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:28:38.109 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:38.161 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:28:38.163 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:38.164 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:28:38.264 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:28:38.327 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:38.400 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:28:38.414 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:38.414 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:28:38.515 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:28:38.559 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:38.598 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 76 16 90811 90795 74.6207 64.875 0.0199156 0.0133966 2023-09-23T20:28:38.599 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:28:38.607 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:28:38.614 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:38.615 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:28:38.715 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:28:38.764 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:38.813 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:28:38.822 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:38.822 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:28:38.824 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:38 smithi149 ceph-mon[170394]: failed to encode map e245 with expected crc 2023-09-23T20:28:38.824 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:38 smithi149 ceph-mon[170394]: failed to encode map e245 with expected crc 2023-09-23T20:28:38.824 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:38 smithi149 ceph-mon[170394]: pgmap v537: 119 pgs: 1 active+recovery_wait+degraded, 4 peering, 3 unknown, 1 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 1 active+recovering+undersized+remapped, 38 active+undersized, 3 active+recovery_wait+degraded+remapped, 56 active+clean; 5.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 39 MiB/s wr, 621 op/s; 31791/266100 objects degraded (11.947%); 11248/266100 objects misplaced (4.227%); 4.9 MiB/s, 78 objects/s recovering 2023-09-23T20:28:38.826 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:38 smithi018 ceph-mon[184144]: failed to encode map e245 with expected crc 2023-09-23T20:28:38.826 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:38 smithi018 ceph-mon[184144]: failed to encode map e245 with expected crc 2023-09-23T20:28:38.827 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:38 smithi018 ceph-mon[184144]: pgmap v537: 119 pgs: 1 active+recovery_wait+degraded, 4 peering, 3 unknown, 1 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 1 active+recovering+undersized+remapped, 38 active+undersized, 3 active+recovery_wait+degraded+remapped, 56 active+clean; 5.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 39 MiB/s wr, 621 op/s; 31791/266100 objects degraded (11.947%); 11248/266100 objects misplaced (4.227%); 4.9 MiB/s, 78 objects/s recovering 2023-09-23T20:28:38.827 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:38 smithi018 ceph-mon[179647]: failed to encode map e245 with expected crc 2023-09-23T20:28:38.827 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:38 smithi018 ceph-mon[179647]: failed to encode map e245 with expected crc 2023-09-23T20:28:38.827 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:38 smithi018 ceph-mon[179647]: pgmap v537: 119 pgs: 1 active+recovery_wait+degraded, 4 peering, 3 unknown, 1 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 1 active+recovering+undersized+remapped, 38 active+undersized, 3 active+recovery_wait+degraded+remapped, 56 active+clean; 5.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 39 MiB/s wr, 621 op/s; 31791/266100 objects degraded (11.947%); 11248/266100 objects misplaced (4.227%); 4.9 MiB/s, 78 objects/s recovering 2023-09-23T20:28:38.923 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:28:39.010 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:39.092 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 4, 5, 7, 3, 2] out_osds: [6] dead_osds: [1] live_osds: [0, 2, 3, 4, 5, 6, 7] 2023-09-23T20:28:39.092 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:28:39.092 INFO:tasks.thrashosds.thrasher:fixing pg num pool .mgr 2023-09-23T20:28:39.093 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:28:39.134 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:28:39.148 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:39.148 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:28:39.249 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:28:39.291 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:39.353 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:28:39.358 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:39.358 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:28:39.459 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:28:39.510 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:39.560 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:28:39.568 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:39.568 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:28:39.596 INFO:teuthology.orchestra.run.smithi018.stderr:noscrub is unset 2023-09-23T20:28:39.597 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 77 16 91745 91729 74.4097 58.375 0.012279 0.0134335 2023-09-23T20:28:39.670 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:28:39.719 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:39.774 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:28:39.778 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:39.778 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:28:39.800 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:28:39.863 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:28:39.879 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:28:39.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:39 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3298678527' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:28:39.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:39 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:28:39.900 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:39 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3298678527' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:28:39.900 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:39 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:28:39.900 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:28:39.922 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:39 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3298678527' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:28:39.922 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:39 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:28:39.926 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:39.939 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:28:39.974 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:28:39.989 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:28:39.997 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:39.998 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:28:40.011 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:28:40.048 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:28:40.048 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:28:40.048 INFO:teuthology.orchestra.run.smithi018.stderr:Error ENXIO: problem getting command descriptions from osd.1 2023-09-23T20:28:40.048 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:28:40.048 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:28:40.048 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:28:40.049 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:28:40.049 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:28:40.049 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:28:40.098 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:28:40.216 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:40.598 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 78 16 92840 92824 74.3324 68.4375 0.0295573 0.0134487 2023-09-23T20:28:40.621 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:40 smithi018 ceph-mon[184144]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:28:40.621 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:40 smithi018 ceph-mon[184144]: failed to encode map e246 with expected crc 2023-09-23T20:28:40.621 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:40 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:28:40.621 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:40 smithi018 ceph-mon[184144]: osdmap e246: 8 total, 7 up, 7 in 2023-09-23T20:28:40.621 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:40 smithi018 ceph-mon[184144]: pgmap v539: 119 pgs: 1 active+recovery_wait+degraded, 4 peering, 2 unknown, 12 active+undersized+degraded, 2 active+recovering+undersized+remapped, 38 active+undersized, 3 active+recovery_wait+degraded+remapped, 57 active+clean; 5.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 39 MiB/s wr, 631 op/s; 28220/267696 objects degraded (10.542%); 14746/267696 objects misplaced (5.508%); 8.6 MiB/s, 137 objects/s recovering 2023-09-23T20:28:40.621 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:40 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1160794050' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:28:40.622 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:40 smithi018 ceph-mon[179647]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:28:40.622 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:40 smithi018 ceph-mon[179647]: failed to encode map e246 with expected crc 2023-09-23T20:28:40.622 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:40 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:28:40.622 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:40 smithi018 ceph-mon[179647]: osdmap e246: 8 total, 7 up, 7 in 2023-09-23T20:28:40.622 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:40 smithi018 ceph-mon[179647]: pgmap v539: 119 pgs: 1 active+recovery_wait+degraded, 4 peering, 2 unknown, 12 active+undersized+degraded, 2 active+recovering+undersized+remapped, 38 active+undersized, 3 active+recovery_wait+degraded+remapped, 57 active+clean; 5.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 39 MiB/s wr, 631 op/s; 28220/267696 objects degraded (10.542%); 14746/267696 objects misplaced (5.508%); 8.6 MiB/s, 137 objects/s recovering 2023-09-23T20:28:40.622 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:40 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1160794050' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:28:40.682 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:28:40.697 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:40.698 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:28:40.752 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:28:40.798 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:28:40.805 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T20:28:40.805 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T20:28:40.840 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:40 smithi149 ceph-mon[170394]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:28:40.840 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:40 smithi149 ceph-mon[170394]: failed to encode map e246 with expected crc 2023-09-23T20:28:40.840 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:40 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:28:40.840 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:40 smithi149 ceph-mon[170394]: osdmap e246: 8 total, 7 up, 7 in 2023-09-23T20:28:40.840 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:40 smithi149 ceph-mon[170394]: pgmap v539: 119 pgs: 1 active+recovery_wait+degraded, 4 peering, 2 unknown, 12 active+undersized+degraded, 2 active+recovering+undersized+remapped, 38 active+undersized, 3 active+recovery_wait+degraded+remapped, 57 active+clean; 5.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 39 MiB/s wr, 631 op/s; 28220/267696 objects degraded (10.542%); 14746/267696 objects misplaced (5.508%); 8.6 MiB/s, 137 objects/s recovering 2023-09-23T20:28:40.840 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:40 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1160794050' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:28:40.849 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:40.901 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:28:40.911 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:40.912 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:28:41.012 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:28:41.073 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:41.131 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:28:41.135 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:41.135 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:28:41.213 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2023-09-23T20:28:41.214 INFO:teuthology.orchestra.run.smithi018.stdout: "ops": [ 2023-09-23T20:28:41.214 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:28:41.214 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.45161.0:93478 6.c 6:3c820264:::benchmark_data_smithi018_270558_object93477:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e246)", 2023-09-23T20:28:41.214 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:28:41.204204+0000", 2023-09-23T20:28:41.214 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.0089104649999999994, 2023-09-23T20:28:41.214 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0089460589999999993, 2023-09-23T20:28:41.214 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:28:41.214 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:28:41.214 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:28:41.214 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.45161", 2023-09-23T20:28:41.214 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3795614987", 2023-09-23T20:28:41.215 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 93478 2023-09-23T20:28:41.215 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:28:41.215 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:28:41.215 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:28:41.215 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:28:41.215 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:41.204204+0000", 2023-09-23T20:28:41.215 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:28:41.215 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:28:41.215 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:28:41.215 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:28:41.215 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:41.204204+0000", 2023-09-23T20:28:41.216 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:28:41.216 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:28:41.216 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:28:41.216 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:28:41.216 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:41.204203+0000", 2023-09-23T20:28:41.216 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:28:41.216 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:28:41.216 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:28:41.216 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:28:41.216 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:41.204228+0000", 2023-09-23T20:28:41.216 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.5244000000000001e-05 2023-09-23T20:28:41.216 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:28:41.217 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:28:41.217 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:28:41.217 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:41.204240+0000", 2023-09-23T20:28:41.217 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2564e-05 2023-09-23T20:28:41.217 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:28:41.217 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:28:41.217 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:28:41.217 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:41.204249+0000", 2023-09-23T20:28:41.217 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.3019999999999995e-06 2023-09-23T20:28:41.217 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:28:41.217 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:28:41.217 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:28:41.217 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:41.207301+0000", 2023-09-23T20:28:41.218 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0030526310000000001 2023-09-23T20:28:41.218 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:28:41.218 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:28:41.218 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:28:41.218 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:41.207454+0000", 2023-09-23T20:28:41.218 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000152986 2023-09-23T20:28:41.218 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:28:41.218 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:28:41.218 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 5,7", 2023-09-23T20:28:41.218 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:41.207639+0000", 2023-09-23T20:28:41.218 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00018453999999999999 2023-09-23T20:28:41.218 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:28:41.219 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:28:41.219 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:28:41.219 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:28:41.219 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:28:41.219 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_repop(client.45161.0:93486 6.14 e246/242 6:2f06d763:::benchmark_data_smithi018_270558_object93485:head v 246'5691, mlcod=246'5689)", 2023-09-23T20:28:41.219 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:28:41.211319+0000", 2023-09-23T20:28:41.219 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.0017950349999999999, 2023-09-23T20:28:41.219 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.001883268, 2023-09-23T20:28:41.219 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:28:41.219 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "started", 2023-09-23T20:28:41.219 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:28:41.220 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:28:41.220 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:28:41.220 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:41.211319+0000", 2023-09-23T20:28:41.220 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:28:41.220 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:28:41.220 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:28:41.220 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:28:41.220 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:41.211319+0000", 2023-09-23T20:28:41.220 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:28:41.220 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:28:41.220 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:28:41.220 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:28:41.221 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:41.211319+0000", 2023-09-23T20:28:41.221 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:28:41.221 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:28:41.221 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:28:41.221 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:28:41.221 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:41.211392+0000", 2023-09-23T20:28:41.221 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.3398000000000004e-05 2023-09-23T20:28:41.221 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:28:41.221 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:28:41.221 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:28:41.221 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:41.211407+0000", 2023-09-23T20:28:41.222 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.4538999999999999e-05 2023-09-23T20:28:41.222 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:28:41.222 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:28:41.222 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:28:41.222 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:41.211412+0000", 2023-09-23T20:28:41.222 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.4469999999999997e-06 2023-09-23T20:28:41.222 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:28:41.222 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:28:41.222 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:28:41.222 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:41.213171+0000", 2023-09-23T20:28:41.222 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.001759105 2023-09-23T20:28:41.223 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:28:41.223 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:28:41.223 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:28:41.223 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:41.213189+0000", 2023-09-23T20:28:41.223 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.7716000000000001e-05 2023-09-23T20:28:41.223 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:28:41.223 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:28:41.223 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:28:41.223 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:28:41.223 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:28:41.223 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_repop_reply(client.45161.0:93478 6.c e246/231)", 2023-09-23T20:28:41.223 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:28:41.212670+0000", 2023-09-23T20:28:41.224 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.00044419499999999998, 2023-09-23T20:28:41.224 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00058176500000000004, 2023-09-23T20:28:41.224 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:28:41.224 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "queued for pg", 2023-09-23T20:28:41.224 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:28:41.224 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:28:41.224 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:28:41.224 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:41.212670+0000", 2023-09-23T20:28:41.224 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:28:41.224 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:28:41.224 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:28:41.224 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:28:41.225 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:41.212670+0000", 2023-09-23T20:28:41.225 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:28:41.225 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:28:41.225 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:28:41.225 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:28:41.225 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:41.212670+0000", 2023-09-23T20:28:41.225 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:28:41.225 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:28:41.225 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:28:41.225 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:28:41.225 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:41.212674+0000", 2023-09-23T20:28:41.226 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.3479999999999997e-06 2023-09-23T20:28:41.226 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:28:41.226 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:28:41.226 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:28:41.226 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:41.212681+0000", 2023-09-23T20:28:41.226 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.1980000000000004e-06 2023-09-23T20:28:41.226 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:28:41.226 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:28:41.226 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:28:41.226 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:41.212686+0000", 2023-09-23T20:28:41.226 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.741e-06 2023-09-23T20:28:41.227 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:28:41.227 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:28:41.227 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:28:41.227 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:28:41.227 INFO:teuthology.orchestra.run.smithi018.stdout: ], 2023-09-23T20:28:41.227 INFO:teuthology.orchestra.run.smithi018.stdout: "num_ops": 3 2023-09-23T20:28:41.227 INFO:teuthology.orchestra.run.smithi018.stdout:} 2023-09-23T20:28:41.235 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:28:41.302 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:41.437 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:28:41.451 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:41.452 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:28:41.553 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:28:41.598 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:41.600 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 79 16 93888 93872 74.2184 65.5 0.00530858 0.0134684 2023-09-23T20:28:41.666 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:28:41.669 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:41.670 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:28:41.771 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:28:41.839 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:41 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 28220/267696 objects degraded (10.542%), 16 pgs degraded (PG_DEGRADED) 2023-09-23T20:28:41.840 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:41 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 28220/267696 objects degraded (10.542%), 16 pgs degraded (PG_DEGRADED) 2023-09-23T20:28:41.849 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:41.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:41 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 28220/267696 objects degraded (10.542%), 16 pgs degraded (PG_DEGRADED) 2023-09-23T20:28:42.085 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T20:28:42.101 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:28:42.120 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:42.120 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:28:42.221 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:28:42.332 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:42.601 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:2023-09-23T20:28:42.601612+0000 min lat: 0.00113549 max lat: 2.50015 avg lat: 0.0134837 2023-09-23T20:28:42.601 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T20:28:42.601 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 80 16 94971 94955 74.1367 67.6875 0.0265473 0.0134837 2023-09-23T20:28:42.798 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T20:28:42.806 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:28:42.849 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:42 smithi018 ceph-mon[179647]: pgmap v540: 119 pgs: 2 active+recovery_wait+degraded, 12 active+undersized+degraded, 2 active+recovering+undersized+remapped, 38 active+undersized, 4 active+recovery_wait+degraded+remapped, 61 active+clean; 5.6 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 54 MiB/s wr, 868 op/s; 31201/275598 objects degraded (11.321%); 17356/275598 objects misplaced (6.298%); 8.9 MiB/s, 142 objects/s recovering 2023-09-23T20:28:42.849 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:42 smithi018 ceph-mon[184144]: pgmap v540: 119 pgs: 2 active+recovery_wait+degraded, 12 active+undersized+degraded, 2 active+recovering+undersized+remapped, 38 active+undersized, 4 active+recovery_wait+degraded+remapped, 61 active+clean; 5.6 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 54 MiB/s wr, 868 op/s; 31201/275598 objects degraded (11.321%); 17356/275598 objects misplaced (6.298%); 8.9 MiB/s, 142 objects/s recovering 2023-09-23T20:28:42.896 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:28:42.917 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:42.918 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:28:42.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:42 smithi149 ceph-mon[170394]: pgmap v540: 119 pgs: 2 active+recovery_wait+degraded, 12 active+undersized+degraded, 2 active+recovering+undersized+remapped, 38 active+undersized, 4 active+recovery_wait+degraded+remapped, 61 active+clean; 5.6 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 54 MiB/s wr, 868 op/s; 31201/275598 objects degraded (11.321%); 17356/275598 objects misplaced (6.298%); 8.9 MiB/s, 142 objects/s recovering 2023-09-23T20:28:43.019 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:28:43.068 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:43.123 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:28:43.129 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:43.130 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:28:43.231 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:28:43.348 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:43.601 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 81 16 95985 95969 74.0039 63.375 0.00480903 0.0135077 2023-09-23T20:28:43.862 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:28:43.867 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:43.868 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:28:43.968 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:28:44.028 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:44.114 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:28:44.124 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:44.124 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:28:44.225 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:28:44.272 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:44.340 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:28:44.346 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:44.346 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:28:44.447 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:28:44.495 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:44.544 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:28:44.547 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:44.547 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:28:44.601 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 82 16 97116 97100 73.9634 70.6875 0.0117531 0.0135157 2023-09-23T20:28:44.648 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:28:44.766 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:44.829 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:28:44.845 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:28:44.853 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:44.853 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:28:44.877 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:28:44.922 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:28:44.945 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:28:44.954 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:28:44.965 INFO:teuthology.orchestra.run.smithi018.stderr:dumped all 2023-09-23T20:28:44.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:44 smithi149 ceph-mon[170394]: pgmap v541: 119 pgs: 2 active+recovery_wait+degraded, 12 active+undersized+degraded, 2 active+recovering+undersized+remapped, 38 active+undersized, 4 active+recovery_wait+degraded+remapped, 61 active+clean; 5.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 42 MiB/s wr, 668 op/s; 31513/280509 objects degraded (11.234%); 17508/280509 objects misplaced (6.242%); 6.5 MiB/s, 104 objects/s recovering 2023-09-23T20:28:44.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:44 smithi149 ceph-mon[170394]: 1.e scrub starts 2023-09-23T20:28:44.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:44 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:28:44.995 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:44 smithi018 ceph-mon[179647]: pgmap v541: 119 pgs: 2 active+recovery_wait+degraded, 12 active+undersized+degraded, 2 active+recovering+undersized+remapped, 38 active+undersized, 4 active+recovery_wait+degraded+remapped, 61 active+clean; 5.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 42 MiB/s wr, 668 op/s; 31513/280509 objects degraded (11.234%); 17508/280509 objects misplaced (6.242%); 6.5 MiB/s, 104 objects/s recovering 2023-09-23T20:28:44.996 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:44 smithi018 ceph-mon[179647]: 1.e scrub starts 2023-09-23T20:28:44.996 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:44 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:28:44.996 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:44 smithi018 ceph-mon[184144]: pgmap v541: 119 pgs: 2 active+recovery_wait+degraded, 12 active+undersized+degraded, 2 active+recovering+undersized+remapped, 38 active+undersized, 4 active+recovery_wait+degraded+remapped, 61 active+clean; 5.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 42 MiB/s wr, 668 op/s; 31513/280509 objects degraded (11.234%); 17508/280509 objects misplaced (6.242%); 6.5 MiB/s, 104 objects/s recovering 2023-09-23T20:28:44.996 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:44 smithi018 ceph-mon[184144]: 1.e scrub starts 2023-09-23T20:28:44.996 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:44 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:28:45.023 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:45.392 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:28:44 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:28:44] "GET /metrics HTTP/1.1" 200 36548 "" "Prometheus/2.43.0" 2023-09-23T20:28:45.403 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:28:45.406 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:45.407 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:28:45.507 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:28:45.551 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:45.567 INFO:teuthology.orchestra.run.smithi018.stdout:{"pg_ready":true,"pg_map":{"version":541,"stamp":"2023-09-23T20:28:43.617872+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":6130760067,"num_objects":93503,"num_object_clones":0,"num_object_copies":280509,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":31513,"num_objects_misplaced":17508,"num_objects_unfound":0,"num_objects_dirty":93503,"num_whiteouts":0,"num_read":6187,"num_read_kb":13572,"num_write":94475,"num_write_kb":5995950,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3081,"num_bytes_recovered":203996559,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":150133,"ondisk_log_size":150133,"up":303,"acting":305,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":355,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":17747296,"kb_used_data":16187560,"kb_used_omap":30,"kb_used_meta":1559649,"kb_avail":638468768,"statfs":{"total":671965249536,"available":653792018432,"internally_reserved":0,"allocated":16576061440,"data_stored":16569058670,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31263,"internal_metadata":1597081057},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,0,1,1,0,4],"upper_bound":64},"perf_stat":{"commit_latency_ms":7,"apply_latency_ms":7,"commit_latency_ns":7000000,"apply_latency_ns":7000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":333971456,"num_objects":5096,"num_object_clones":0,"num_object_copies":15288,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-1879,"num_objects_misplaced":7931,"num_objects_unfound":0,"num_objects_dirty":5096,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5096,"num_write_kb":326144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":798,"num_bytes_recovered":52297728,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"7.620442"},"pg_stats":[{"pgid":"3.1a","version":"69'1","reported_seq":250,"reported_epoch":245,"state":"active+undersized","last_fresh":"2023-09-23T20:28:37.020216+0000","last_change":"2023-09-23T20:28:13.154911+0000","last_active":"2023-09-23T20:28:37.020216+0000","last_peered":"2023-09-23T20:28:37.020216+0000","last_clean":"2023-09-23T20:28:10.122878+0000","last_became_active":"2023-09-23T20:28:13.154911+0000","last_became_peered":"2023-09-23T20:28:13.154911+0000","last_unstale":"2023-09-23T20:28:37.020216+0000","last_undegraded":"2023-09-23T20:28:37.020216+0000","last_fullsized":"2023-09-23T20:28:13.148164+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:38:02.482515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":["4","2"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":98,"reported_epoch":245,"state":"active+undersized","last_fresh":"2023-09-23T20:28:37.006255+0000","last_change":"2023-09-23T20:28:13.170504+0000","last_active":"2023-09-23T20:28:37.006255+0000","last_peered":"2023-09-23T20:28:37.006255+0000","last_clean":"2023-09-23T20:28:10.123345+0000","last_became_active":"2023-09-23T20:28:13.170504+0000","last_became_peered":"2023-09-23T20:28:13.170504+0000","last_unstale":"2023-09-23T20:28:37.006255+0000","last_undegraded":"2023-09-23T20:28:37.006255+0000","last_fullsized":"2023-09-23T20:28:13.140712+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:35:11.789454+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":["3","4"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":131,"reported_epoch":245,"state":"active+undersized","last_fresh":"2023-09-23T20:28:37.005946+0000","last_change":"2023-09-23T20:28:13.155496+0000","last_active":"2023-09-23T20:28:37.005946+0000","last_peered":"2023-09-23T20:28:37.005946+0000","last_clean":"2023-09-23T20:24:57.110902+0000","last_became_active":"2023-09-23T20:28:13.155496+0000","last_became_peered":"2023-09-23T20:28:13.155496+0000","last_unstale":"2023-09-23T20:28:37.005946+0000","last_undegraded":"2023-09-23T20:28:37.005946+0000","last_fullsized":"2023-09-23T20:28:13.140169+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:49:18.750327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":["3","4"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":210,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T20:28:37.030891+0000","last_change":"2023-09-23T20:25:02.957185+0000","last_active":"2023-09-23T20:28:37.030891+0000","last_peered":"2023-09-23T20:28:37.030891+0000","last_clean":"2023-09-23T20:28:37.030891+0000","last_became_active":"2023-09-23T20:25:02.956689+0000","last_became_peered":"2023-09-23T20:25:02.956689+0000","last_unstale":"2023-09-23T20:28:37.030891+0000","last_undegraded":"2023-09-23T20:28:37.030891+0000","last_fullsized":"2023-09-23T20:28:37.030891+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:04:28.524015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":98,"reported_epoch":245,"state":"active+undersized","last_fresh":"2023-09-23T20:28:37.006498+0000","last_change":"2023-09-23T20:28:13.145246+0000","last_active":"2023-09-23T20:28:37.006498+0000","last_peered":"2023-09-23T20:28:37.006498+0000","last_clean":"2023-09-23T20:28:10.123201+0000","last_became_active":"2023-09-23T20:28:13.145246+0000","last_became_peered":"2023-09-23T20:28:13.145246+0000","last_unstale":"2023-09-23T20:28:37.006498+0000","last_undegraded":"2023-09-23T20:28:37.006498+0000","last_fullsized":"2023-09-23T20:28:13.139195+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:33:00.081087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":["3","0"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":85,"reported_epoch":245,"state":"active+undersized","last_fresh":"2023-09-23T20:28:37.008910+0000","last_change":"2023-09-23T20:28:13.163992+0000","last_active":"2023-09-23T20:28:37.008910+0000","last_peered":"2023-09-23T20:28:37.008910+0000","last_clean":"2023-09-23T20:28:10.122013+0000","last_became_active":"2023-09-23T20:28:13.163992+0000","last_became_peered":"2023-09-23T20:28:13.163992+0000","last_unstale":"2023-09-23T20:28:37.008910+0000","last_undegraded":"2023-09-23T20:28:37.008910+0000","last_fullsized":"2023-09-23T20:28:13.153382+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:36:20.431010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":["5","0"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.12","version":"246'5759","reported_seq":5806,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T20:28:40.613551+0000","last_change":"2023-09-23T20:28:35.573470+0000","last_active":"2023-09-23T20:28:40.613551+0000","last_peered":"2023-09-23T20:28:40.613551+0000","last_clean":"2023-09-23T20:28:40.613551+0000","last_became_active":"2023-09-23T20:28:35.568920+0000","last_became_peered":"2023-09-23T20:28:35.568920+0000","last_unstale":"2023-09-23T20:28:40.613551+0000","last_undegraded":"2023-09-23T20:28:40.613551+0000","last_fullsized":"2023-09-23T20:28:40.613551+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":242,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":5759,"log_dups_size":0,"ondisk_log_size":5759,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":193200128,"num_objects":2948,"num_object_clones":0,"num_object_copies":8844,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2948,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2948,"num_write_kb":188672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":208,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T20:28:37.031581+0000","last_change":"2023-09-23T20:25:02.969432+0000","last_active":"2023-09-23T20:28:37.031581+0000","last_peered":"2023-09-23T20:28:37.031581+0000","last_clean":"2023-09-23T20:28:37.031581+0000","last_became_active":"2023-09-23T20:25:02.969118+0000","last_became_peered":"2023-09-23T20:25:02.969118+0000","last_unstale":"2023-09-23T20:28:37.031581+0000","last_undegraded":"2023-09-23T20:28:37.031581+0000","last_fullsized":"2023-09-23T20:28:37.031581+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:02:22.453831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":107,"reported_epoch":245,"state":"active+undersized","last_fresh":"2023-09-23T20:28:37.006396+0000","last_change":"2023-09-23T20:28:13.155103+0000","last_active":"2023-09-23T20:28:37.006396+0000","last_peered":"2023-09-23T20:28:37.006396+0000","last_clean":"2023-09-23T20:28:10.123286+0000","last_became_active":"2023-09-23T20:28:13.155103+0000","last_became_peered":"2023-09-23T20:28:13.155103+0000","last_unstale":"2023-09-23T20:28:37.006396+0000","last_undegraded":"2023-09-23T20:28:37.006396+0000","last_fullsized":"2023-09-23T20:28:13.139476+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:55:15.053789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":["3","4"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.13","version":"246'5668","reported_seq":5739,"reported_epoch":246,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T20:28:40.577819+0000","last_change":"2023-09-23T20:28:37.011895+0000","last_active":"2023-09-23T20:28:40.577819+0000","last_peered":"2023-09-23T20:28:40.577819+0000","last_clean":"2023-09-23T20:28:04.010682+0000","last_became_active":"2023-09-23T20:28:37.009790+0000","last_became_peered":"2023-09-23T20:28:37.009790+0000","last_unstale":"2023-09-23T20:28:40.577819+0000","last_undegraded":"2023-09-23T20:28:37.006376+0000","last_fullsized":"2023-09-23T20:28:40.577819+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":242,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":5668,"log_dups_size":0,"ondisk_log_size":5668,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":190021632,"num_objects":2900,"num_object_clones":0,"num_object_copies":8700,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":113,"num_objects_misplaced":2787,"num_objects_unfound":0,"num_objects_dirty":2900,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2900,"num_write_kb":185568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":98304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2,6],"avail_no_missing":["7","2"],"object_location_counts":[{"shards":"2,6,7","objects":2787},{"shards":"2,7","objects":113}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.14","version":"90'161","reported_seq":488,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T20:28:37.001934+0000","last_change":"2023-09-23T20:28:05.995127+0000","last_active":"2023-09-23T20:28:37.001934+0000","last_peered":"2023-09-23T20:28:37.001934+0000","last_clean":"2023-09-23T20:28:37.001934+0000","last_became_active":"2023-09-23T20:28:05.994757+0000","last_became_peered":"2023-09-23T20:28:05.994757+0000","last_unstale":"2023-09-23T20:28:37.001934+0000","last_undegraded":"2023-09-23T20:28:37.001934+0000","last_fullsized":"2023-09-23T20:28:37.001934+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:13.244651+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.244651+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0019955509999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109227,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":212,"reported_epoch":245,"state":"active+undersized","last_fresh":"2023-09-23T20:28:37.008477+0000","last_change":"2023-09-23T20:28:13.175805+0000","last_active":"2023-09-23T20:28:37.008477+0000","last_peered":"2023-09-23T20:28:37.008477+0000","last_clean":"2023-09-23T20:28:10.121056+0000","last_became_active":"2023-09-23T20:28:13.175805+0000","last_became_peered":"2023-09-23T20:28:13.175805+0000","last_unstale":"2023-09-23T20:28:37.008477+0000","last_undegraded":"2023-09-23T20:28:37.008477+0000","last_fullsized":"2023-09-23T20:28:13.156272+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:57:51.683405+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":["5","7"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":101,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T20:28:37.006064+0000","last_change":"2023-09-23T20:28:05.036058+0000","last_active":"2023-09-23T20:28:37.006064+0000","last_peered":"2023-09-23T20:28:37.006064+0000","last_clean":"2023-09-23T20:28:37.006064+0000","last_became_active":"2023-09-23T20:28:05.035708+0000","last_became_peered":"2023-09-23T20:28:05.035708+0000","last_unstale":"2023-09-23T20:28:37.006064+0000","last_undegraded":"2023-09-23T20:28:37.006064+0000","last_fullsized":"2023-09-23T20:28:37.006064+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:30:06.458724+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.10","version":"246'5727","reported_seq":5775,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T20:28:39.892147+0000","last_change":"2023-09-23T20:28:35.586857+0000","last_active":"2023-09-23T20:28:39.892147+0000","last_peered":"2023-09-23T20:28:39.892147+0000","last_clean":"2023-09-23T20:28:39.892147+0000","last_became_active":"2023-09-23T20:28:35.582165+0000","last_became_peered":"2023-09-23T20:28:35.582165+0000","last_unstale":"2023-09-23T20:28:39.892147+0000","last_undegraded":"2023-09-23T20:28:39.892147+0000","last_fullsized":"2023-09-23T20:28:39.892147+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":242,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":5727,"log_dups_size":0,"ondisk_log_size":5727,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":190611456,"num_objects":2909,"num_object_clones":0,"num_object_copies":8727,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2909,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2909,"num_write_kb":186144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":241,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T20:28:37.006596+0000","last_change":"2023-09-23T20:25:03.038511+0000","last_active":"2023-09-23T20:28:37.006596+0000","last_peered":"2023-09-23T20:28:37.006596+0000","last_clean":"2023-09-23T20:28:37.006596+0000","last_became_active":"2023-09-23T20:25:02.976811+0000","last_became_peered":"2023-09-23T20:25:02.976811+0000","last_unstale":"2023-09-23T20:28:37.006596+0000","last_undegraded":"2023-09-23T20:28:37.006596+0000","last_fullsized":"2023-09-23T20:28:37.006596+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:28:06.605731+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":208,"reported_epoch":245,"state":"active+undersized","last_fresh":"2023-09-23T20:28:37.000088+0000","last_change":"2023-09-23T20:28:13.145997+0000","last_active":"2023-09-23T20:28:37.000088+0000","last_peered":"2023-09-23T20:28:37.000088+0000","last_clean":"2023-09-23T20:28:02.998115+0000","last_became_active":"2023-09-23T20:28:13.145997+0000","last_became_peered":"2023-09-23T20:28:13.145997+0000","last_unstale":"2023-09-23T20:28:37.000088+0000","last_undegraded":"2023-09-23T20:28:37.000088+0000","last_fullsized":"2023-09-23T20:28:13.141589+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:22:49.831269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":["2","0"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"6.11","version":"246'5682","reported_seq":7133,"reported_epoch":246,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T20:28:42.091865+0000","last_change":"2023-09-23T20:28:37.012300+0000","last_active":"2023-09-23T20:28:42.091865+0000","last_peered":"2023-09-23T20:28:42.091865+0000","last_clean":"2023-09-23T20:28:04.011715+0000","last_became_active":"2023-09-23T20:28:37.009435+0000","last_became_peered":"2023-09-23T20:28:37.009435+0000","last_unstale":"2023-09-23T20:28:42.091865+0000","last_undegraded":"2023-09-23T20:28:37.003118+0000","last_fullsized":"2023-09-23T20:28:42.091865+0000","mapping_epoch":244,"log_start":"0'0","ondisk_log_start":"0'0","created":242,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":5682,"log_dups_size":0,"ondisk_log_size":5682,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":191889408,"num_objects":2928,"num_object_clones":0,"num_object_copies":8784,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1878,"num_objects_misplaced":2928,"num_objects_unfound":0,"num_objects_dirty":2928,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2928,"num_write_kb":187392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":347,"num_bytes_recovered":22740992,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7],"acting":[2,7,6],"avail_no_missing":["2","6"],"object_location_counts":[{"shards":"2,6","objects":1878},{"shards":"2,6,7","objects":1050}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":230,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T20:28:37.018820+0000","last_change":"2023-09-23T20:28:05.051083+0000","last_active":"2023-09-23T20:28:37.018820+0000","last_peered":"2023-09-23T20:28:37.018820+0000","last_clean":"2023-09-23T20:28:37.018820+0000","last_became_active":"2023-09-23T20:28:05.024014+0000","last_became_peered":"2023-09-23T20:28:05.024014+0000","last_unstale":"2023-09-23T20:28:37.018820+0000","last_undegraded":"2023-09-23T20:28:37.018820+0000","last_fullsized":"2023-09-23T20:28:37.018820+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:09:19.424181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,2],"acting":[4,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":221,"reported_epoch":245,"state":"active+undersized","last_fresh":"2023-09-23T20:28:37.019442+0000","last_change":"2023-09-23T20:28:13.170352+0000","last_active":"2023-09-23T20:28:37.019442+0000","last_peered":"2023-09-23T20:28:37.019442+0000","last_clean":"2023-09-23T20:28:10.122153+0000","last_became_active":"2023-09-23T20:28:13.170352+0000","last_became_peered":"2023-09-23T20:28:13.170352+0000","last_unstale":"2023-09-23T20:28:37.019442+0000","last_undegraded":"2023-09-23T20:28:37.019442+0000","last_fullsized":"2023-09-23T20:28:13.150349+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:32:21.400258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5],"acting":[4,5],"avail_no_missing":["4","5"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.16","version":"246'5697","reported_seq":5745,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T20:28:43.114041+0000","last_change":"2023-09-23T20:28:35.579111+0000","last_active":"2023-09-23T20:28:43.114041+0000","last_peered":"2023-09-23T20:28:43.114041+0000","last_clean":"2023-09-23T20:28:43.114041+0000","last_became_active":"2023-09-23T20:28:35.577020+0000","last_became_peered":"2023-09-23T20:28:35.577020+0000","last_unstale":"2023-09-23T20:28:43.114041+0000","last_undegraded":"2023-09-23T20:28:43.114041+0000","last_fullsized":"2023-09-23T20:28:43.114041+0000","mapping_epoch":242,"log_start":"221'300","ondisk_log_start":"221'300","created":242,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":5397,"log_dups_size":0,"ondisk_log_size":5397,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":193658880,"num_objects":2955,"num_object_clones":0,"num_object_copies":8865,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2955,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2955,"num_write_kb":189120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.11","version":"90'161","reported_seq":459,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T20:28:37.030025+0000","last_change":"2023-09-23T20:28:05.991681+0000","last_active":"2023-09-23T20:28:37.030025+0000","last_peered":"2023-09-23T20:28:37.030025+0000","last_clean":"2023-09-23T20:28:37.030025+0000","last_became_active":"2023-09-23T20:28:05.991304+0000","last_became_peered":"2023-09-23T20:28:05.991304+0000","last_unstale":"2023-09-23T20:28:37.030025+0000","last_undegraded":"2023-09-23T20:28:37.030025+0000","last_fullsized":"2023-09-23T20:28:37.030025+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:34.821789+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:34.821789+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":240,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T20:28:37.006863+0000","last_change":"2023-09-23T20:25:39.654026+0000","last_active":"2023-09-23T20:28:37.006863+0000","last_peered":"2023-09-23T20:28:37.006863+0000","last_clean":"2023-09-23T20:28:37.006863+0000","last_became_active":"2023-09-23T20:25:39.626849+0000","last_became_peered":"2023-09-23T20:25:39.626849+0000","last_unstale":"2023-09-23T20:28:37.006863+0000","last_undegraded":"2023-09-23T20:28:37.006863+0000","last_fullsized":"2023-09-23T20:28:37.006863+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:22:53.957630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":98,"reported_epoch":245,"state":"active+undersized","last_fresh":"2023-09-23T20:28:37.006594+0000","last_change":"2023-09-23T20:28:13.152317+0000","last_active":"2023-09-23T20:28:37.006594+0000","last_peered":"2023-09-23T20:28:37.006594+0000","last_clean":"2023-09-23T20:28:10.123633+0000","last_became_active":"2023-09-23T20:28:13.152317+0000","last_became_peered":"2023-09-23T20:28:13.152317+0000","last_unstale":"2023-09-23T20:28:37.006594+0000","last_undegraded":"2023-09-23T20:28:37.006594+0000","last_fullsized":"2023-09-23T20:28:13.142454+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:13:55.507629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.17","version":"246'5800","reported_seq":5847,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T20:28:40.588477+0000","last_change":"2023-09-23T20:28:35.586734+0000","last_active":"2023-09-23T20:28:40.588477+0000","last_peered":"2023-09-23T20:28:40.588477+0000","last_clean":"2023-09-23T20:28:40.588477+0000","last_became_active":"2023-09-23T20:28:35.582232+0000","last_became_peered":"2023-09-23T20:28:35.582232+0000","last_unstale":"2023-09-23T20:28:40.588477+0000","last_undegraded":"2023-09-23T20:28:40.588477+0000","last_fullsized":"2023-09-23T20:28:40.588477+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":242,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":5800,"log_dups_size":0,"ondisk_log_size":5800,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":194740224,"num_objects":2972,"num_object_clones":0,"num_object_copies":8916,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2972,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2972,"num_write_kb":190176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.10","version":"90'161","reported_seq":446,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T20:28:37.007989+0000","last_change":"2023-09-23T20:28:08.021917+0000","last_active":"2023-09-23T20:28:37.007989+0000","last_peered":"2023-09-23T20:28:37.007989+0000","last_clean":"2023-09-23T20:28:37.007989+0000","last_became_active":"2023-09-23T20:28:08.021588+0000","last_became_peered":"2023-09-23T20:28:08.021588+0000","last_unstale":"2023-09-23T20:28:37.007989+0000","last_undegraded":"2023-09-23T20:28:37.007989+0000","last_fullsized":"2023-09-23T20:28:37.007989+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:32.801168+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:32.801168+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":208,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T20:28:37.030343+0000","last_change":"2023-09-23T20:25:02.953993+0000","last_active":"2023-09-23T20:28:37.030343+0000","last_peered":"2023-09-23T20:28:37.030343+0000","last_clean":"2023-09-23T20:28:37.030343+0000","last_became_active":"2023-09-23T20:25:02.953672+0000","last_became_peered":"2023-09-23T20:25:02.953672+0000","last_unstale":"2023-09-23T20:28:37.030343+0000","last_undegraded":"2023-09-23T20:28:37.030343+0000","last_fullsized":"2023-09-23T20:28:37.030343+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:26:08.603577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":224,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T20:28:37.008951+0000","last_change":"2023-09-23T20:25:02.949584+0000","last_active":"2023-09-23T20:28:37.008951+0000","last_peered":"2023-09-23T20:28:37.008951+0000","last_clean":"2023-09-23T20:28:37.008951+0000","last_became_active":"2023-09-23T20:25:02.949132+0000","last_became_peered":"2023-09-23T20:25:02.949132+0000","last_unstale":"2023-09-23T20:28:37.008951+0000","last_undegraded":"2023-09-23T20:28:37.008951+0000","last_fullsized":"2023-09-23T20:28:37.008951+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:34:20.509823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.14","version":"246'5662","reported_seq":5712,"reported_epoch":246,"state":"active+undersized+degraded","last_fresh":"2023-09-23T20:28:40.602418+0000","last_change":"2023-09-23T20:28:35.577429+0000","last_active":"2023-09-23T20:28:40.602418+0000","last_peered":"2023-09-23T20:28:40.602418+0000","last_clean":"2023-09-23T20:28:12.083147+0000","last_became_active":"2023-09-23T20:28:35.577429+0000","last_became_peered":"2023-09-23T20:28:35.577429+0000","last_unstale":"2023-09-23T20:28:40.602418+0000","last_undegraded":"2023-09-23T20:28:35.564606+0000","last_fullsized":"2023-09-23T20:28:35.527507+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":242,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":5662,"log_dups_size":0,"ondisk_log_size":5662,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":189923328,"num_objects":2898,"num_object_clones":0,"num_object_copies":8694,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2898,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2898,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2898,"num_write_kb":185472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":["5","3"],"object_location_counts":[{"shards":"3,5","objects":2898}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"90'161","reported_seq":483,"reported_epoch":245,"state":"active+undersized","last_fresh":"2023-09-23T20:28:37.005983+0000","last_change":"2023-09-23T20:28:13.172329+0000","last_active":"2023-09-23T20:28:37.005983+0000","last_peered":"2023-09-23T20:28:37.005983+0000","last_clean":"2023-09-23T20:27:55.206508+0000","last_became_active":"2023-09-23T20:28:13.172329+0000","last_became_peered":"2023-09-23T20:28:13.172329+0000","last_unstale":"2023-09-23T20:28:37.005983+0000","last_undegraded":"2023-09-23T20:28:37.005983+0000","last_fullsized":"2023-09-23T20:28:13.140024+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:12.386098+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:12.386098+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0019500940000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":["3","4"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":199,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T20:28:37.003898+0000","last_change":"2023-09-23T20:28:05.022017+0000","last_active":"2023-09-23T20:28:37.003898+0000","last_peered":"2023-09-23T20:28:37.003898+0000","last_clean":"2023-09-23T20:28:37.003898+0000","last_became_active":"2023-09-23T20:28:05.021659+0000","last_became_peered":"2023-09-23T20:28:05.021659+0000","last_unstale":"2023-09-23T20:28:37.003898+0000","last_undegraded":"2023-09-23T20:28:37.003898+0000","last_fullsized":"2023-09-23T20:28:37.003898+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:19:47.722750+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":218,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T20:28:37.030824+0000","last_change":"2023-09-23T20:25:02.957010+0000","last_active":"2023-09-23T20:28:37.030824+0000","last_peered":"2023-09-23T20:28:37.030824+0000","last_clean":"2023-09-23T20:28:37.030824+0000","last_became_active":"2023-09-23T20:25:02.956267+0000","last_became_peered":"2023-09-23T20:25:02.956267+0000","last_unstale":"2023-09-23T20:28:37.030824+0000","last_undegraded":"2023-09-23T20:28:37.030824+0000","last_fullsized":"2023-09-23T20:28:37.030824+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:10:55.299422+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.15","version":"246'5600","reported_seq":5653,"reported_epoch":246,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T20:28:40.609712+0000","last_change":"2023-09-23T20:28:35.613834+0000","last_active":"2023-09-23T20:28:40.609712+0000","last_peered":"2023-09-23T20:28:40.609712+0000","last_clean":"2023-09-23T20:28:04.012743+0000","last_became_active":"2023-09-23T20:28:35.586625+0000","last_became_peered":"2023-09-23T20:28:35.586625+0000","last_unstale":"2023-09-23T20:28:40.609712+0000","last_undegraded":"2023-09-23T20:28:35.574275+0000","last_fullsized":"2023-09-23T20:28:40.609712+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":242,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":5600,"log_dups_size":0,"ondisk_log_size":5600,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":187400192,"num_objects":2860,"num_object_clones":0,"num_object_copies":8580,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2107,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2860,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2860,"num_write_kb":183008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":["7","3","6"],"object_location_counts":[{"shards":"2,3,7","objects":753},{"shards":"3,6,7","objects":2107}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.12","version":"90'161","reported_seq":453,"reported_epoch":245,"state":"active+undersized+degraded","last_fresh":"2023-09-23T20:28:37.008644+0000","last_change":"2023-09-23T20:28:13.176937+0000","last_active":"2023-09-23T20:28:37.008644+0000","last_peered":"2023-09-23T20:28:37.008644+0000","last_clean":"2023-09-23T20:28:10.125672+0000","last_became_active":"2023-09-23T20:28:13.176937+0000","last_became_peered":"2023-09-23T20:28:13.176937+0000","last_unstale":"2023-09-23T20:28:37.008644+0000","last_undegraded":"2023-09-23T20:28:13.154776+0000","last_fullsized":"2023-09-23T20:28:13.154661+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:35.734584+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:35.734584+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":["5","7"],"object_location_counts":[{"shards":"5,7","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":48,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T20:28:37.030056+0000","last_change":"2023-09-23T20:28:05.041053+0000","last_active":"2023-09-23T20:28:37.030056+0000","last_peered":"2023-09-23T20:28:37.030056+0000","last_clean":"2023-09-23T20:28:37.030056+0000","last_became_active":"2023-09-23T20:28:05.030153+0000","last_became_peered":"2023-09-23T20:28:05.030153+0000","last_unstale":"2023-09-23T20:28:37.030056+0000","last_undegraded":"2023-09-23T20:28:37.030056+0000","last_fullsized":"2023-09-23T20:28:37.030056+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:37:31.689827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":98,"reported_epoch":245,"state":"active+undersized","last_fresh":"2023-09-23T20:28:37.006803+0000","last_change":"2023-09-23T20:28:13.146925+0000","last_active":"2023-09-23T20:28:37.006803+0000","last_peered":"2023-09-23T20:28:37.006803+0000","last_clean":"2023-09-23T20:28:10.124095+0000","last_became_active":"2023-09-23T20:28:13.146925+0000","last_became_peered":"2023-09-23T20:28:13.146925+0000","last_unstale":"2023-09-23T20:28:37.006803+0000","last_undegraded":"2023-09-23T20:28:37.006803+0000","last_fullsized":"2023-09-23T20:28:13.141297+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:26:22.810131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.d","version":"90'161","reported_seq":453,"reported_epoch":245,"state":"active+undersized","last_fresh":"2023-09-23T20:28:37.004135+0000","last_change":"2023-09-23T20:28:13.170455+0000","last_active":"2023-09-23T20:28:37.004135+0000","last_peered":"2023-09-23T20:28:37.004135+0000","last_clean":"2023-09-23T20:28:10.105394+0000","last_became_active":"2023-09-23T20:28:13.170455+0000","last_became_peered":"2023-09-23T20:28:13.170455+0000","last_unstale":"2023-09-23T20:28:37.004135+0000","last_undegraded":"2023-09-23T20:28:37.004135+0000","last_fullsized":"2023-09-23T20:28:13.145373+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:12.809933+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:12.809933+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.a","version":"246'5858","reported_seq":8926,"reported_epoch":246,"state":"active+recovering+undersized+remapped","last_fresh":"2023-09-23T20:28:40.669244+0000","last_change":"2023-09-23T20:28:06.211632+0000","last_active":"2023-09-23T20:28:40.669244+0000","last_peered":"2023-09-23T20:28:40.669244+0000","last_clean":"2023-09-23T20:28:04.001969+0000","last_became_active":"2023-09-23T20:28:06.165624+0000","last_became_peered":"2023-09-23T20:28:06.165624+0000","last_unstale":"2023-09-23T20:28:40.669244+0000","last_undegraded":"2023-09-23T20:28:40.669244+0000","last_fullsized":"2023-09-23T20:28:05.983416+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":5858,"log_dups_size":0,"ondisk_log_size":5858,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:51:55.980497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":383909888,"num_objects":5858,"num_object_clones":0,"num_object_copies":17574,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2800,"num_objects_unfound":0,"num_objects_dirty":5858,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5858,"num_write_kb":374912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1510,"num_bytes_recovered":98959360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0],"avail_no_missing":["5","0","6"],"object_location_counts":[{"shards":"0,5","objects":3058},{"shards":"0,5,6","objects":2800}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":207,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T20:28:37.004157+0000","last_change":"2023-09-23T20:20:14.883283+0000","last_active":"2023-09-23T20:28:37.004157+0000","last_peered":"2023-09-23T20:28:37.004157+0000","last_clean":"2023-09-23T20:28:37.004157+0000","last_became_active":"2023-09-23T20:20:14.882806+0000","last_became_peered":"2023-09-23T20:20:14.882806+0000","last_unstale":"2023-09-23T20:28:37.004157+0000","last_undegraded":"2023-09-23T20:28:37.004157+0000","last_fullsized":"2023-09-23T20:28:37.004157+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:23:35.295701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":197,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T20:28:37.008174+0000","last_change":"2023-09-23T20:28:05.037565+0000","last_active":"2023-09-23T20:28:37.008174+0000","last_peered":"2023-09-23T20:28:37.008174+0000","last_clean":"2023-09-23T20:28:37.008174+0000","last_became_active":"2023-09-23T20:28:05.037288+0000","last_became_peered":"2023-09-23T20:28:05.037288+0000","last_unstale":"2023-09-23T20:28:37.008174+0000","last_undegraded":"2023-09-23T20:28:37.008174+0000","last_fullsized":"2023-09-23T20:28:37.008174+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:55:40.576813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,4],"acting":[5,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"90'161","reported_seq":507,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T20:28:37.006170+0000","last_change":"2023-09-23T20:28:07.018115+0000","last_active":"2023-09-23T20:28:37.006170+0000","last_peered":"2023-09-23T20:28:37.006170+0000","last_clean":"2023-09-23T20:28:37.006170+0000","last_became_active":"2023-09-23T20:28:07.017652+0000","last_became_peered":"2023-09-23T20:28:07.017652+0000","last_unstale":"2023-09-23T20:28:37.006170+0000","last_undegraded":"2023-09-23T20:28:37.006170+0000","last_fullsized":"2023-09-23T20:28:37.006170+0000","mapping_epoch":232,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":233,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:13.244651+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.244651+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0019955509999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109227,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.b","version":"246'5838","reported_seq":5879,"reported_epoch":246,"state":"active+undersized+degraded","last_fresh":"2023-09-23T20:28:43.105320+0000","last_change":"2023-09-23T20:28:13.151214+0000","last_active":"2023-09-23T20:28:43.105320+0000","last_peered":"2023-09-23T20:28:43.105320+0000","last_clean":"2023-09-23T20:28:12.089537+0000","last_became_active":"2023-09-23T20:28:13.151214+0000","last_became_peered":"2023-09-23T20:28:13.151214+0000","last_unstale":"2023-09-23T20:28:43.105320+0000","last_undegraded":"2023-09-23T20:28:13.141183+0000","last_fullsized":"2023-09-23T20:28:13.141041+0000","mapping_epoch":236,"log_start":"222'1200","ondisk_log_start":"222'1200","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":4638,"log_dups_size":0,"ondisk_log_size":4638,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:25:15.926592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":382599168,"num_objects":5838,"num_object_clones":0,"num_object_copies":17514,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5838,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5838,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5838,"num_write_kb":373632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":5838}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":215,"reported_epoch":245,"state":"active+undersized+degraded","last_fresh":"2023-09-23T20:28:37.004191+0000","last_change":"2023-09-23T20:28:13.175199+0000","last_active":"2023-09-23T20:28:37.004191+0000","last_peered":"2023-09-23T20:28:37.004191+0000","last_clean":"2023-09-23T20:28:10.105377+0000","last_became_active":"2023-09-23T20:28:13.175199+0000","last_became_peered":"2023-09-23T20:28:13.175199+0000","last_unstale":"2023-09-23T20:28:37.004191+0000","last_undegraded":"2023-09-23T20:28:13.146101+0000","last_fullsized":"2023-09-23T20:28:13.145995+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:32:17.555869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[{"shards":"4,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":248,"reported_epoch":245,"state":"active+undersized","last_fresh":"2023-09-23T20:28:37.018272+0000","last_change":"2023-09-23T20:28:13.156552+0000","last_active":"2023-09-23T20:28:37.018272+0000","last_peered":"2023-09-23T20:28:37.018272+0000","last_clean":"2023-09-23T20:28:10.121663+0000","last_became_active":"2023-09-23T20:28:13.156552+0000","last_became_peered":"2023-09-23T20:28:13.156552+0000","last_unstale":"2023-09-23T20:28:37.018272+0000","last_undegraded":"2023-09-23T20:28:37.018272+0000","last_fullsized":"2023-09-23T20:28:13.147617+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:28:11.148006+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":["4","3"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"90'161","reported_seq":507,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T20:28:37.008622+0000","last_change":"2023-09-23T20:28:08.030382+0000","last_active":"2023-09-23T20:28:37.008622+0000","last_peered":"2023-09-23T20:28:37.008622+0000","last_clean":"2023-09-23T20:28:37.008622+0000","last_became_active":"2023-09-23T20:28:08.030032+0000","last_became_peered":"2023-09-23T20:28:08.030032+0000","last_unstale":"2023-09-23T20:28:37.008622+0000","last_undegraded":"2023-09-23T20:28:37.008622+0000","last_fullsized":"2023-09-23T20:28:37.008622+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:32.746340+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:32.746340+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.8","version":"246'5701","reported_seq":5749,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T20:28:40.605894+0000","last_change":"2023-09-23T20:28:35.581921+0000","last_active":"2023-09-23T20:28:40.605894+0000","last_peered":"2023-09-23T20:28:40.605894+0000","last_clean":"2023-09-23T20:28:40.605894+0000","last_became_active":"2023-09-23T20:28:35.576360+0000","last_became_peered":"2023-09-23T20:28:35.576360+0000","last_unstale":"2023-09-23T20:28:40.605894+0000","last_undegraded":"2023-09-23T20:28:40.605894+0000","last_fullsized":"2023-09-23T20:28:40.605894+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":5701,"log_dups_size":0,"ondisk_log_size":5701,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":191037440,"num_objects":2915,"num_object_clones":0,"num_object_copies":8745,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2915,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2915,"num_write_kb":186560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":204,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T20:28:37.006437+0000","last_change":"2023-09-23T20:28:05.051786+0000","last_active":"2023-09-23T20:28:37.006437+0000","last_peered":"2023-09-23T20:28:37.006437+0000","last_clean":"2023-09-23T20:28:37.006437+0000","last_became_active":"2023-09-23T20:28:05.035372+0000","last_became_peered":"2023-09-23T20:28:05.035372+0000","last_unstale":"2023-09-23T20:28:37.006437+0000","last_undegraded":"2023-09-23T20:28:37.006437+0000","last_fullsized":"2023-09-23T20:28:37.006437+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:34:29.102528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":71,"reported_epoch":245,"state":"active+undersized","last_fresh":"2023-09-23T20:28:37.003880+0000","last_change":"2023-09-23T20:28:13.171782+0000","last_active":"2023-09-23T20:28:37.003880+0000","last_peered":"2023-09-23T20:28:37.003880+0000","last_clean":"2023-09-23T20:20:14.875676+0000","last_became_active":"2023-09-23T20:28:13.171782+0000","last_became_peered":"2023-09-23T20:28:13.171782+0000","last_unstale":"2023-09-23T20:28:37.003880+0000","last_undegraded":"2023-09-23T20:28:37.003880+0000","last_fullsized":"2023-09-23T20:28:13.145291+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:55:21.847801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5],"avail_no_missing":["7","5"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.e","version":"90'161","reported_seq":449,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T20:28:37.021463+0000","last_change":"2023-09-23T20:28:10.788870+0000","last_active":"2023-09-23T20:28:37.021463+0000","last_peered":"2023-09-23T20:28:37.021463+0000","last_clean":"2023-09-23T20:28:37.021463+0000","last_became_active":"2023-09-23T20:28:05.990524+0000","last_became_peered":"2023-09-23T20:28:05.990524+0000","last_unstale":"2023-09-23T20:28:37.021463+0000","last_undegraded":"2023-09-23T20:28:37.021463+0000","last_fullsized":"2023-09-23T20:28:37.021463+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:31.769333+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:31.769333+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.9","version":"246'5683","reported_seq":5730,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T20:28:39.922277+0000","last_change":"2023-09-23T20:28:35.539652+0000","last_active":"2023-09-23T20:28:39.922277+0000","last_peered":"2023-09-23T20:28:39.922277+0000","last_clean":"2023-09-23T20:28:39.922277+0000","last_became_active":"2023-09-23T20:28:35.536824+0000","last_became_peered":"2023-09-23T20:28:35.536824+0000","last_unstale":"2023-09-23T20:28:39.922277+0000","last_undegraded":"2023-09-23T20:28:39.922277+0000","last_fullsized":"2023-09-23T20:28:39.922277+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":5683,"log_dups_size":0,"ondisk_log_size":5683,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":189202432,"num_objects":2887,"num_object_clones":0,"num_object_copies":8661,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2887,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2887,"num_write_kb":184768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":242,"reported_epoch":245,"state":"active+undersized","last_fresh":"2023-09-23T20:28:37.008830+0000","last_change":"2023-09-23T20:28:13.168615+0000","last_active":"2023-09-23T20:28:37.008830+0000","last_peered":"2023-09-23T20:28:37.008830+0000","last_clean":"2023-09-23T20:28:10.121217+0000","last_became_active":"2023-09-23T20:28:13.168615+0000","last_became_peered":"2023-09-23T20:28:13.168615+0000","last_unstale":"2023-09-23T20:28:37.008830+0000","last_undegraded":"2023-09-23T20:28:37.008830+0000","last_fullsized":"2023-09-23T20:28:13.152836+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:24:12.165529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":["5","3"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":197,"reported_epoch":245,"state":"active+undersized","last_fresh":"2023-09-23T20:28:37.030246+0000","last_change":"2023-09-23T20:28:13.172321+0000","last_active":"2023-09-23T20:28:37.030246+0000","last_peered":"2023-09-23T20:28:37.030246+0000","last_clean":"2023-09-23T20:28:10.114667+0000","last_became_active":"2023-09-23T20:28:13.172321+0000","last_became_peered":"2023-09-23T20:28:13.172321+0000","last_unstale":"2023-09-23T20:28:37.030246+0000","last_undegraded":"2023-09-23T20:28:37.030246+0000","last_fullsized":"2023-09-23T20:28:13.141132+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:01:05.541514+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":["0","4"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"246'5771","reported_seq":5815,"reported_epoch":246,"state":"active+undersized+degraded","last_fresh":"2023-09-23T20:28:40.681741+0000","last_change":"2023-09-23T20:28:13.171294+0000","last_active":"2023-09-23T20:28:40.681741+0000","last_peered":"2023-09-23T20:28:40.681741+0000","last_clean":"2023-09-23T20:28:12.106078+0000","last_became_active":"2023-09-23T20:28:13.171294+0000","last_became_peered":"2023-09-23T20:28:13.171294+0000","last_unstale":"2023-09-23T20:28:40.681741+0000","last_undegraded":"2023-09-23T20:28:13.149409+0000","last_fullsized":"2023-09-23T20:28:13.149261+0000","mapping_epoch":236,"log_start":"221'372","ondisk_log_start":"221'372","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":5399,"log_dups_size":0,"ondisk_log_size":5399,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:17:53.283303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":378208256,"num_objects":5771,"num_object_clones":0,"num_object_copies":17313,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5771,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5771,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5771,"num_write_kb":369344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":["4","2"],"object_location_counts":[{"shards":"2,4","objects":5771}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":628,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T20:28:37.003801+0000","last_change":"2023-09-23T20:28:05.872149+0000","last_active":"2023-09-23T20:28:37.003801+0000","last_peered":"2023-09-23T20:28:37.003801+0000","last_clean":"2023-09-23T20:28:37.003801+0000","last_became_active":"2023-09-23T20:27:11.961310+0000","last_became_peered":"2023-09-23T20:27:11.961310+0000","last_unstale":"2023-09-23T20:28:37.003801+0000","last_undegraded":"2023-09-23T20:28:37.003801+0000","last_fullsized":"2023-09-23T20:28:37.003801+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:28:05.872074+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:28:05.872074+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:52:32.495955+0000","scrub_duration":0.0029215980000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":96,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T20:28:37.006253+0000","last_change":"2023-09-23T20:25:03.000957+0000","last_active":"2023-09-23T20:28:37.006253+0000","last_peered":"2023-09-23T20:28:37.006253+0000","last_clean":"2023-09-23T20:28:37.006253+0000","last_became_active":"2023-09-23T20:25:02.939903+0000","last_became_peered":"2023-09-23T20:25:02.939903+0000","last_unstale":"2023-09-23T20:28:37.006253+0000","last_undegraded":"2023-09-23T20:28:37.006253+0000","last_fullsized":"2023-09-23T20:28:37.006253+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:51:14.981461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":241,"reported_epoch":245,"state":"active+undersized","last_fresh":"2023-09-23T20:28:37.018699+0000","last_change":"2023-09-23T20:28:13.165428+0000","last_active":"2023-09-23T20:28:37.018699+0000","last_peered":"2023-09-23T20:28:37.018699+0000","last_clean":"2023-09-23T20:28:10.122622+0000","last_became_active":"2023-09-23T20:28:13.165428+0000","last_became_peered":"2023-09-23T20:28:13.165428+0000","last_unstale":"2023-09-23T20:28:37.018699+0000","last_undegraded":"2023-09-23T20:28:37.018699+0000","last_fullsized":"2023-09-23T20:28:13.148542+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:56:49.047400+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":["4","3"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"246'5962","reported_seq":6000,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T20:28:42.124894+0000","last_change":"2023-09-23T20:27:14.270199+0000","last_active":"2023-09-23T20:28:42.124894+0000","last_peered":"2023-09-23T20:28:42.124894+0000","last_clean":"2023-09-23T20:28:42.124894+0000","last_became_active":"2023-09-23T20:27:14.269858+0000","last_became_peered":"2023-09-23T20:27:14.269858+0000","last_unstale":"2023-09-23T20:28:42.124894+0000","last_undegraded":"2023-09-23T20:28:42.124894+0000","last_fullsized":"2023-09-23T20:28:42.124894+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":5962,"log_dups_size":0,"ondisk_log_size":5962,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:47:21.406552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":390725632,"num_objects":5962,"num_object_clones":0,"num_object_copies":17886,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5962,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5962,"num_write_kb":381568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":470,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T20:28:37.008000+0000","last_change":"2023-09-23T20:27:31.757060+0000","last_active":"2023-09-23T20:28:37.008000+0000","last_peered":"2023-09-23T20:28:37.008000+0000","last_clean":"2023-09-23T20:28:37.008000+0000","last_became_active":"2023-09-23T20:27:11.959516+0000","last_became_peered":"2023-09-23T20:27:11.959516+0000","last_unstale":"2023-09-23T20:28:37.008000+0000","last_undegraded":"2023-09-23T20:28:37.008000+0000","last_fullsized":"2023-09-23T20:28:37.008000+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:31.756974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:31.756974+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:35:11.939335+0000","scrub_duration":0.0031802670000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":86,"reported_epoch":245,"state":"active+undersized","last_fresh":"2023-09-23T20:28:37.020089+0000","last_change":"2023-09-23T20:28:13.157583+0000","last_active":"2023-09-23T20:28:37.020089+0000","last_peered":"2023-09-23T20:28:37.020089+0000","last_clean":"2023-09-23T20:28:10.122833+0000","last_became_active":"2023-09-23T20:28:13.157583+0000","last_became_peered":"2023-09-23T20:28:13.157583+0000","last_unstale":"2023-09-23T20:28:37.020089+0000","last_undegraded":"2023-09-23T20:28:37.020089+0000","last_fullsized":"2023-09-23T20:28:13.148515+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:16:09.921700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":["4","3"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":234,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T20:28:37.017866+0000","last_change":"2023-09-23T20:25:39.624636+0000","last_active":"2023-09-23T20:28:37.017866+0000","last_peered":"2023-09-23T20:28:37.017866+0000","last_clean":"2023-09-23T20:28:37.017866+0000","last_became_active":"2023-09-23T20:25:39.624297+0000","last_became_peered":"2023-09-23T20:25:39.624297+0000","last_unstale":"2023-09-23T20:28:37.017866+0000","last_undegraded":"2023-09-23T20:28:37.017866+0000","last_fullsized":"2023-09-23T20:28:37.017866+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:33:47.448737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"90'161","reported_seq":483,"reported_epoch":245,"state":"active+undersized","last_fresh":"2023-09-23T20:28:37.030109+0000","last_change":"2023-09-23T20:28:13.164275+0000","last_active":"2023-09-23T20:28:37.030109+0000","last_peered":"2023-09-23T20:28:37.030109+0000","last_clean":"2023-09-23T20:27:55.206527+0000","last_became_active":"2023-09-23T20:28:13.164275+0000","last_became_peered":"2023-09-23T20:28:13.164275+0000","last_unstale":"2023-09-23T20:28:37.030109+0000","last_undegraded":"2023-09-23T20:28:37.030109+0000","last_fullsized":"2023-09-23T20:28:13.140861+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:12.386098+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:12.386098+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0019500940000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.c","version":"246'5854","reported_seq":7555,"reported_epoch":246,"state":"active+recovering+undersized+remapped","last_fresh":"2023-09-23T20:28:43.105277+0000","last_change":"2023-09-23T20:28:34.770363+0000","last_active":"2023-09-23T20:28:43.105277+0000","last_peered":"2023-09-23T20:28:43.105277+0000","last_clean":"2023-09-23T20:28:03.999709+0000","last_became_active":"2023-09-23T20:28:06.198771+0000","last_became_peered":"2023-09-23T20:28:06.198771+0000","last_unstale":"2023-09-23T20:28:43.105277+0000","last_undegraded":"2023-09-23T20:28:43.105277+0000","last_fullsized":"2023-09-23T20:28:05.981540+0000","mapping_epoch":231,"log_start":"222'1198","ondisk_log_start":"222'1198","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":4656,"log_dups_size":0,"ondisk_log_size":4656,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:32:08.136452+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":383647744,"num_objects":5854,"num_object_clones":0,"num_object_copies":17562,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3323,"num_objects_unfound":0,"num_objects_dirty":5854,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5854,"num_write_kb":374656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":826,"num_bytes_recovered":54132736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5],"avail_no_missing":["3","5","6"],"object_location_counts":[{"shards":"3,5","objects":2531},{"shards":"3,5,6","objects":3323}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":225,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T20:28:37.018079+0000","last_change":"2023-09-23T20:25:39.628097+0000","last_active":"2023-09-23T20:28:37.018079+0000","last_peered":"2023-09-23T20:28:37.018079+0000","last_clean":"2023-09-23T20:28:37.018079+0000","last_became_active":"2023-09-23T20:25:39.627806+0000","last_became_peered":"2023-09-23T20:25:39.627806+0000","last_unstale":"2023-09-23T20:28:37.018079+0000","last_undegraded":"2023-09-23T20:28:37.018079+0000","last_fullsized":"2023-09-23T20:28:37.018079+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:53:44.398854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":221,"reported_epoch":245,"state":"active+undersized","last_fresh":"2023-09-23T20:28:37.021583+0000","last_change":"2023-09-23T20:28:13.174633+0000","last_active":"2023-09-23T20:28:37.021583+0000","last_peered":"2023-09-23T20:28:37.021583+0000","last_clean":"2023-09-23T20:28:10.121563+0000","last_became_active":"2023-09-23T20:28:13.174633+0000","last_became_peered":"2023-09-23T20:28:13.174633+0000","last_unstale":"2023-09-23T20:28:37.021583+0000","last_undegraded":"2023-09-23T20:28:37.021583+0000","last_fullsized":"2023-09-23T20:28:13.150357+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:55:59.664339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":["4","3"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"246'5822","reported_seq":5866,"reported_epoch":246,"state":"active+undersized+degraded","last_fresh":"2023-09-23T20:28:40.636160+0000","last_change":"2023-09-23T20:28:13.164184+0000","last_active":"2023-09-23T20:28:40.636160+0000","last_peered":"2023-09-23T20:28:40.636160+0000","last_clean":"2023-09-23T20:28:12.085747+0000","last_became_active":"2023-09-23T20:28:13.164184+0000","last_became_peered":"2023-09-23T20:28:13.164184+0000","last_unstale":"2023-09-23T20:28:40.636160+0000","last_undegraded":"2023-09-23T20:28:13.153718+0000","last_fullsized":"2023-09-23T20:28:13.153621+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":5822,"log_dups_size":0,"ondisk_log_size":5822,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:13:24.112990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":381550592,"num_objects":5822,"num_object_clones":0,"num_object_copies":17466,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5822,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5822,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5822,"num_write_kb":372608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":["5","0"],"object_location_counts":[{"shards":"0,5","objects":5822}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":490,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T20:28:37.008531+0000","last_change":"2023-09-23T20:28:08.043585+0000","last_active":"2023-09-23T20:28:37.008531+0000","last_peered":"2023-09-23T20:28:37.008531+0000","last_clean":"2023-09-23T20:28:37.008531+0000","last_became_active":"2023-09-23T20:28:08.032075+0000","last_became_peered":"2023-09-23T20:28:08.032075+0000","last_unstale":"2023-09-23T20:28:37.008531+0000","last_undegraded":"2023-09-23T20:28:37.008531+0000","last_fullsized":"2023-09-23T20:28:37.008531+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:30.727267+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:30.727267+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:01:58.886819+0000","scrub_duration":0.0052560289999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":98,"reported_epoch":245,"state":"active+undersized","last_fresh":"2023-09-23T20:28:37.006251+0000","last_change":"2023-09-23T20:28:13.143589+0000","last_active":"2023-09-23T20:28:37.006251+0000","last_peered":"2023-09-23T20:28:37.006251+0000","last_clean":"2023-09-23T20:28:10.123116+0000","last_became_active":"2023-09-23T20:28:13.143589+0000","last_became_peered":"2023-09-23T20:28:13.143589+0000","last_unstale":"2023-09-23T20:28:37.006251+0000","last_undegraded":"2023-09-23T20:28:37.006251+0000","last_fullsized":"2023-09-23T20:28:13.138889+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:07:48.412859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":34,"reported_epoch":245,"state":"active+undersized","last_fresh":"2023-09-23T20:28:37.006611+0000","last_change":"2023-09-23T20:28:13.167121+0000","last_active":"2023-09-23T20:28:37.006611+0000","last_peered":"2023-09-23T20:28:37.006611+0000","last_clean":"2023-09-23T20:24:57.109980+0000","last_became_active":"2023-09-23T20:28:13.167121+0000","last_became_peered":"2023-09-23T20:28:13.167121+0000","last_unstale":"2023-09-23T20:28:37.006611+0000","last_undegraded":"2023-09-23T20:28:37.006611+0000","last_fullsized":"2023-09-23T20:28:13.140887+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:49:30.695766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":["3","4"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.0","version":"246'5745","reported_seq":5793,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T20:28:39.926266+0000","last_change":"2023-09-23T20:28:35.536749+0000","last_active":"2023-09-23T20:28:39.926266+0000","last_peered":"2023-09-23T20:28:39.926266+0000","last_clean":"2023-09-23T20:28:39.926266+0000","last_became_active":"2023-09-23T20:28:35.532298+0000","last_became_peered":"2023-09-23T20:28:35.532298+0000","last_unstale":"2023-09-23T20:28:39.926266+0000","last_undegraded":"2023-09-23T20:28:39.926266+0000","last_fullsized":"2023-09-23T20:28:39.926266+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":5745,"log_dups_size":0,"ondisk_log_size":5745,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":191791104,"num_objects":2926,"num_object_clones":0,"num_object_copies":8778,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2926,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2926,"num_write_kb":187296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":455,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T20:28:37.006976+0000","last_change":"2023-09-23T20:28:08.032785+0000","last_active":"2023-09-23T20:28:37.006976+0000","last_peered":"2023-09-23T20:28:37.006976+0000","last_clean":"2023-09-23T20:28:37.006976+0000","last_became_active":"2023-09-23T20:28:08.032209+0000","last_became_peered":"2023-09-23T20:28:08.032209+0000","last_unstale":"2023-09-23T20:28:37.006976+0000","last_undegraded":"2023-09-23T20:28:37.006976+0000","last_fullsized":"2023-09-23T20:28:37.006976+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:32.746340+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:32.746340+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"86'119","reported_seq":415,"reported_epoch":245,"state":"active+undersized+degraded","last_fresh":"2023-09-23T20:28:37.033286+0000","last_change":"2023-09-23T20:28:13.147446+0000","last_active":"2023-09-23T20:28:37.033286+0000","last_peered":"2023-09-23T20:28:37.033286+0000","last_clean":"2023-09-23T20:13:53.395005+0000","last_became_active":"2023-09-23T20:28:13.147446+0000","last_became_peered":"2023-09-23T20:28:13.147446+0000","last_unstale":"2023-09-23T20:28:37.033286+0000","last_undegraded":"2023-09-23T20:28:13.139220+0000","last_fullsized":"2023-09-23T20:28:13.139106+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":119,"log_dups_size":0,"ondisk_log_size":119,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:50:31.476259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032705499999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":523,"num_read_kb":3503,"num_write":136,"num_write_kb":3128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[{"shards":"0,7","objects":1}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":262,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T20:28:37.008869+0000","last_change":"2023-09-23T20:25:39.637160+0000","last_active":"2023-09-23T20:28:37.008869+0000","last_peered":"2023-09-23T20:28:37.008869+0000","last_clean":"2023-09-23T20:28:37.008869+0000","last_became_active":"2023-09-23T20:25:39.636860+0000","last_became_peered":"2023-09-23T20:25:39.636860+0000","last_unstale":"2023-09-23T20:28:37.008869+0000","last_undegraded":"2023-09-23T20:28:37.008869+0000","last_fullsized":"2023-09-23T20:28:37.008869+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:21:47.860063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":33,"reported_epoch":245,"state":"active+undersized","last_fresh":"2023-09-23T20:28:37.008453+0000","last_change":"2023-09-23T20:28:13.176485+0000","last_active":"2023-09-23T20:28:37.008453+0000","last_peered":"2023-09-23T20:28:37.008453+0000","last_clean":"2023-09-23T20:12:43.732826+0000","last_became_active":"2023-09-23T20:28:13.176485+0000","last_became_peered":"2023-09-23T20:28:13.176485+0000","last_unstale":"2023-09-23T20:28:37.008453+0000","last_undegraded":"2023-09-23T20:28:37.008453+0000","last_fullsized":"2023-09-23T20:28:13.155600+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:59:12.518891+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":["5","4"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.6","version":"246'5706","reported_seq":5754,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T20:28:43.032230+0000","last_change":"2023-09-23T20:28:35.569598+0000","last_active":"2023-09-23T20:28:43.032230+0000","last_peered":"2023-09-23T20:28:43.032230+0000","last_clean":"2023-09-23T20:28:43.032230+0000","last_became_active":"2023-09-23T20:28:35.565434+0000","last_became_peered":"2023-09-23T20:28:35.565434+0000","last_unstale":"2023-09-23T20:28:43.032230+0000","last_undegraded":"2023-09-23T20:28:43.032230+0000","last_fullsized":"2023-09-23T20:28:43.032230+0000","mapping_epoch":217,"log_start":"221'300","ondisk_log_start":"221'300","created":217,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":5406,"log_dups_size":0,"ondisk_log_size":5406,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":194248704,"num_objects":2964,"num_object_clones":0,"num_object_copies":8892,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2964,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2964,"num_write_kb":189696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":480,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T20:28:37.008195+0000","last_change":"2023-09-23T20:27:55.213707+0000","last_active":"2023-09-23T20:28:37.008195+0000","last_peered":"2023-09-23T20:28:37.008195+0000","last_clean":"2023-09-23T20:28:37.008195+0000","last_became_active":"2023-09-23T20:27:55.213394+0000","last_became_peered":"2023-09-23T20:27:55.213394+0000","last_unstale":"2023-09-23T20:28:37.008195+0000","last_undegraded":"2023-09-23T20:28:37.008195+0000","last_fullsized":"2023-09-23T20:28:37.008195+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:34.821789+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:34.821789+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0042962870000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"86'106","reported_seq":919,"reported_epoch":245,"state":"active+undersized+degraded","last_fresh":"2023-09-23T20:28:37.008376+0000","last_change":"2023-09-23T20:28:13.164484+0000","last_active":"2023-09-23T20:28:37.008376+0000","last_peered":"2023-09-23T20:28:37.008376+0000","last_clean":"2023-09-23T20:28:10.123134+0000","last_became_active":"2023-09-23T20:28:13.164484+0000","last_became_peered":"2023-09-23T20:28:13.164484+0000","last_unstale":"2023-09-23T20:28:37.008376+0000","last_undegraded":"2023-09-23T20:28:13.155126+0000","last_fullsized":"2023-09-23T20:28:13.155031+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":106,"log_dups_size":0,"ondisk_log_size":106,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:37.280911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00099302599999999993,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1105,"num_read_kb":1689,"num_write":100,"num_write_kb":404,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":["5","0"],"object_location_counts":[{"shards":"0,5","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":222,"reported_epoch":245,"state":"active+undersized","last_fresh":"2023-09-23T20:28:37.019306+0000","last_change":"2023-09-23T20:28:13.170036+0000","last_active":"2023-09-23T20:28:37.019306+0000","last_peered":"2023-09-23T20:28:37.019306+0000","last_clean":"2023-09-23T20:28:10.122078+0000","last_became_active":"2023-09-23T20:28:13.170036+0000","last_became_peered":"2023-09-23T20:28:13.170036+0000","last_unstale":"2023-09-23T20:28:37.019306+0000","last_undegraded":"2023-09-23T20:28:37.019306+0000","last_fullsized":"2023-09-23T20:28:13.149199+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:13:13.987896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":["4","0"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":177,"reported_epoch":245,"state":"active+undersized","last_fresh":"2023-09-23T20:28:37.002041+0000","last_change":"2023-09-23T20:28:13.152577+0000","last_active":"2023-09-23T20:28:37.002041+0000","last_peered":"2023-09-23T20:28:37.002041+0000","last_clean":"2023-09-23T20:25:34.579305+0000","last_became_active":"2023-09-23T20:28:13.152577+0000","last_became_peered":"2023-09-23T20:28:13.152577+0000","last_unstale":"2023-09-23T20:28:37.002041+0000","last_undegraded":"2023-09-23T20:28:37.002041+0000","last_fullsized":"2023-09-23T20:28:13.139696+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:55:11.899806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"6.5","version":"246'5642","reported_seq":5698,"reported_epoch":246,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T20:28:40.607661+0000","last_change":"2023-09-23T20:28:36.053560+0000","last_active":"2023-09-23T20:28:40.607661+0000","last_peered":"2023-09-23T20:28:40.607661+0000","last_clean":"2023-09-23T20:28:04.012743+0000","last_became_active":"2023-09-23T20:28:36.015209+0000","last_became_peered":"2023-09-23T20:28:36.015209+0000","last_unstale":"2023-09-23T20:28:40.607661+0000","last_undegraded":"2023-09-23T20:28:36.007394+0000","last_fullsized":"2023-09-23T20:28:40.607661+0000","mapping_epoch":243,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":5642,"log_dups_size":0,"ondisk_log_size":5642,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":190152704,"num_objects":2901,"num_object_clones":0,"num_object_copies":8703,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2121,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2901,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2901,"num_write_kb":185696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":["7","3","6"],"object_location_counts":[{"shards":"2,3,7","objects":780},{"shards":"3,6,7","objects":2121}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":638,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T20:28:37.003799+0000","last_change":"2023-09-23T20:27:55.209408+0000","last_active":"2023-09-23T20:28:37.003799+0000","last_peered":"2023-09-23T20:28:37.003799+0000","last_clean":"2023-09-23T20:28:37.003799+0000","last_became_active":"2023-09-23T20:27:55.209043+0000","last_became_peered":"2023-09-23T20:27:55.209043+0000","last_unstale":"2023-09-23T20:28:37.003799+0000","last_undegraded":"2023-09-23T20:28:37.003799+0000","last_fullsized":"2023-09-23T20:28:37.003799+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:35.734584+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:35.734584+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0035251380000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"86'130","reported_seq":556,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T20:28:37.000100+0000","last_change":"2023-09-23T20:25:39.637458+0000","last_active":"2023-09-23T20:28:37.000100+0000","last_peered":"2023-09-23T20:28:37.000100+0000","last_clean":"2023-09-23T20:28:37.000100+0000","last_became_active":"2023-09-23T20:25:39.636773+0000","last_became_peered":"2023-09-23T20:25:39.636773+0000","last_unstale":"2023-09-23T20:28:37.000100+0000","last_undegraded":"2023-09-23T20:28:37.000100+0000","last_fullsized":"2023-09-23T20:28:37.000100+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:00:32.810564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011300049999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":512,"num_read_kb":1820,"num_write":133,"num_write_kb":1347,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":207,"reported_epoch":245,"state":"active+undersized","last_fresh":"2023-09-23T20:28:36.999626+0000","last_change":"2023-09-23T20:28:13.144161+0000","last_active":"2023-09-23T20:28:36.999626+0000","last_peered":"2023-09-23T20:28:36.999626+0000","last_clean":"2023-09-23T20:28:02.997876+0000","last_became_active":"2023-09-23T20:28:13.144161+0000","last_became_peered":"2023-09-23T20:28:13.144161+0000","last_unstale":"2023-09-23T20:28:36.999626+0000","last_undegraded":"2023-09-23T20:28:36.999626+0000","last_fullsized":"2023-09-23T20:28:13.138363+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:26.025784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":["2","0"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":76,"reported_epoch":245,"state":"active+undersized","last_fresh":"2023-09-23T20:28:37.008881+0000","last_change":"2023-09-23T20:28:13.168863+0000","last_active":"2023-09-23T20:28:37.008881+0000","last_peered":"2023-09-23T20:28:37.008881+0000","last_clean":"2023-09-23T20:15:59.442389+0000","last_became_active":"2023-09-23T20:28:13.168863+0000","last_became_peered":"2023-09-23T20:28:13.168863+0000","last_unstale":"2023-09-23T20:28:37.008881+0000","last_undegraded":"2023-09-23T20:28:37.008881+0000","last_fullsized":"2023-09-23T20:28:13.153675+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:30:33.149977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":["5","0"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.7","version":"246'5770","reported_seq":5818,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T20:28:40.595023+0000","last_change":"2023-09-23T20:28:35.534346+0000","last_active":"2023-09-23T20:28:40.595023+0000","last_peered":"2023-09-23T20:28:40.595023+0000","last_clean":"2023-09-23T20:28:40.595023+0000","last_became_active":"2023-09-23T20:28:35.531247+0000","last_became_peered":"2023-09-23T20:28:35.531247+0000","last_unstale":"2023-09-23T20:28:40.595023+0000","last_undegraded":"2023-09-23T20:28:40.595023+0000","last_fullsized":"2023-09-23T20:28:40.595023+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":5770,"log_dups_size":0,"ondisk_log_size":5770,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":192774144,"num_objects":2941,"num_object_clones":0,"num_object_copies":8823,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2941,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2941,"num_write_kb":188256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"86'194","reported_seq":894,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T20:28:37.008492+0000","last_change":"2023-09-23T20:25:40.618665+0000","last_active":"2023-09-23T20:28:37.008492+0000","last_peered":"2023-09-23T20:28:37.008492+0000","last_clean":"2023-09-23T20:28:37.008492+0000","last_became_active":"2023-09-23T20:25:40.618087+0000","last_became_peered":"2023-09-23T20:25:40.618087+0000","last_unstale":"2023-09-23T20:28:37.008492+0000","last_undegraded":"2023-09-23T20:28:37.008492+0000","last_fullsized":"2023-09-23T20:28:37.008492+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":194,"log_dups_size":0,"ondisk_log_size":194,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:07:16.114968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00037995999999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1025,"num_read_kb":1201,"num_write":173,"num_write_kb":1276,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":649,"reported_epoch":245,"state":"active+undersized","last_fresh":"2023-09-23T20:28:37.003823+0000","last_change":"2023-09-23T20:28:13.146277+0000","last_active":"2023-09-23T20:28:37.003823+0000","last_peered":"2023-09-23T20:28:37.003823+0000","last_clean":"2023-09-23T20:28:10.104150+0000","last_became_active":"2023-09-23T20:28:13.146277+0000","last_became_peered":"2023-09-23T20:28:13.146277+0000","last_unstale":"2023-09-23T20:28:37.003823+0000","last_undegraded":"2023-09-23T20:28:37.003823+0000","last_fullsized":"2023-09-23T20:28:13.141588+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:32.801168+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:32.801168+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0096333110000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":["7","0"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":101,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T20:28:37.006211+0000","last_change":"2023-09-23T20:28:05.036669+0000","last_active":"2023-09-23T20:28:37.006211+0000","last_peered":"2023-09-23T20:28:37.006211+0000","last_clean":"2023-09-23T20:28:37.006211+0000","last_became_active":"2023-09-23T20:28:05.036379+0000","last_became_peered":"2023-09-23T20:28:05.036379+0000","last_unstale":"2023-09-23T20:28:37.006211+0000","last_undegraded":"2023-09-23T20:28:37.006211+0000","last_fullsized":"2023-09-23T20:28:37.006211+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:35:57.806627+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":29,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T20:28:37.017969+0000","last_change":"2023-09-23T20:28:05.024905+0000","last_active":"2023-09-23T20:28:37.017969+0000","last_peered":"2023-09-23T20:28:37.017969+0000","last_clean":"2023-09-23T20:28:37.017969+0000","last_became_active":"2023-09-23T20:28:05.024596+0000","last_became_peered":"2023-09-23T20:28:05.024596+0000","last_unstale":"2023-09-23T20:28:37.017969+0000","last_undegraded":"2023-09-23T20:28:37.017969+0000","last_fullsized":"2023-09-23T20:28:37.017969+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:23:18.124135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.4","version":"246'5669","reported_seq":5720,"reported_epoch":246,"state":"active+undersized+degraded","last_fresh":"2023-09-23T20:28:40.670797+0000","last_change":"2023-09-23T20:28:35.565448+0000","last_active":"2023-09-23T20:28:40.670797+0000","last_peered":"2023-09-23T20:28:40.670797+0000","last_clean":"2023-09-23T20:28:12.083147+0000","last_became_active":"2023-09-23T20:28:35.565448+0000","last_became_peered":"2023-09-23T20:28:35.565448+0000","last_unstale":"2023-09-23T20:28:40.670797+0000","last_undegraded":"2023-09-23T20:28:35.525526+0000","last_fullsized":"2023-09-23T20:28:35.525410+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":5669,"log_dups_size":0,"ondisk_log_size":5669,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":190382080,"num_objects":2905,"num_object_clones":0,"num_object_copies":8715,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2905,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2905,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2905,"num_write_kb":185920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":["5","3"],"object_location_counts":[{"shards":"3,5","objects":2905}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":435,"reported_epoch":245,"state":"active+undersized","last_fresh":"2023-09-23T20:28:37.019038+0000","last_change":"2023-09-23T20:28:13.170340+0000","last_active":"2023-09-23T20:28:37.019038+0000","last_peered":"2023-09-23T20:28:37.019038+0000","last_clean":"2023-09-23T20:15:59.452160+0000","last_became_active":"2023-09-23T20:28:13.170340+0000","last_became_peered":"2023-09-23T20:28:13.170340+0000","last_unstale":"2023-09-23T20:28:37.019038+0000","last_undegraded":"2023-09-23T20:28:37.019038+0000","last_fullsized":"2023-09-23T20:28:13.149972+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:12.386098+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:12.386098+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5],"acting":[4,5],"avail_no_missing":["4","5"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.0","version":"86'80","reported_seq":499,"reported_epoch":245,"state":"active+undersized","last_fresh":"2023-09-23T20:28:37.006553+0000","last_change":"2023-09-23T20:28:13.144600+0000","last_active":"2023-09-23T20:28:37.006553+0000","last_peered":"2023-09-23T20:28:37.006553+0000","last_clean":"2023-09-23T20:28:10.104142+0000","last_became_active":"2023-09-23T20:28:13.144600+0000","last_became_peered":"2023-09-23T20:28:13.144600+0000","last_unstale":"2023-09-23T20:28:37.006553+0000","last_undegraded":"2023-09-23T20:28:37.006553+0000","last_fullsized":"2023-09-23T20:28:13.141588+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":80,"log_dups_size":0,"ondisk_log_size":80,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:37:42.749177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035115,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":395,"num_read_kb":1370,"num_write":66,"num_write_kb":1061,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":["7","0"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":209,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T20:28:37.030197+0000","last_change":"2023-09-23T20:25:02.942709+0000","last_active":"2023-09-23T20:28:37.030197+0000","last_peered":"2023-09-23T20:28:37.030197+0000","last_clean":"2023-09-23T20:28:37.030197+0000","last_became_active":"2023-09-23T20:25:02.942323+0000","last_became_peered":"2023-09-23T20:25:02.942323+0000","last_unstale":"2023-09-23T20:28:37.030197+0000","last_undegraded":"2023-09-23T20:28:37.030197+0000","last_fullsized":"2023-09-23T20:28:37.030197+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:26:12.039101+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":230,"reported_epoch":245,"state":"active+undersized","last_fresh":"2023-09-23T20:28:37.031105+0000","last_change":"2023-09-23T20:28:13.144536+0000","last_active":"2023-09-23T20:28:37.031105+0000","last_peered":"2023-09-23T20:28:37.031105+0000","last_clean":"2023-09-23T20:28:10.114404+0000","last_became_active":"2023-09-23T20:28:13.144536+0000","last_became_peered":"2023-09-23T20:28:13.144536+0000","last_unstale":"2023-09-23T20:28:37.031105+0000","last_undegraded":"2023-09-23T20:28:37.031105+0000","last_fullsized":"2023-09-23T20:28:13.140563+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:50:22.944488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2],"avail_no_missing":["0","2"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"246'5691","reported_seq":7141,"reported_epoch":246,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T20:28:42.089104+0000","last_change":"2023-09-23T20:28:35.549400+0000","last_active":"2023-09-23T20:28:42.089104+0000","last_peered":"2023-09-23T20:28:42.089104+0000","last_clean":"2023-09-23T20:28:04.011715+0000","last_became_active":"2023-09-23T20:28:35.540341+0000","last_became_peered":"2023-09-23T20:28:35.540341+0000","last_unstale":"2023-09-23T20:28:42.089104+0000","last_undegraded":"2023-09-23T20:28:35.530732+0000","last_fullsized":"2023-09-23T20:28:42.089104+0000","mapping_epoch":237,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":5691,"log_dups_size":0,"ondisk_log_size":5691,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":192479232,"num_objects":2937,"num_object_clones":0,"num_object_copies":8811,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1905,"num_objects_misplaced":2937,"num_objects_unfound":0,"num_objects_dirty":2937,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2937,"num_write_kb":187968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":347,"num_bytes_recovered":22740992,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7],"acting":[2,7,6],"avail_no_missing":["2","6"],"object_location_counts":[{"shards":"2,6","objects":1905},{"shards":"2,6,7","objects":1032}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":448,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T20:28:37.030741+0000","last_change":"2023-09-23T20:28:08.028468+0000","last_active":"2023-09-23T20:28:37.030741+0000","last_peered":"2023-09-23T20:28:37.030741+0000","last_clean":"2023-09-23T20:28:37.030741+0000","last_became_active":"2023-09-23T20:28:08.027976+0000","last_became_peered":"2023-09-23T20:28:08.027976+0000","last_unstale":"2023-09-23T20:28:37.030741+0000","last_undegraded":"2023-09-23T20:28:37.030741+0000","last_fullsized":"2023-09-23T20:28:37.030741+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:31.769333+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:31.769333+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"86'102","reported_seq":752,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T20:28:37.006521+0000","last_change":"2023-09-23T20:20:14.881975+0000","last_active":"2023-09-23T20:28:37.006521+0000","last_peered":"2023-09-23T20:28:37.006521+0000","last_clean":"2023-09-23T20:28:37.006521+0000","last_became_active":"2023-09-23T20:20:14.881544+0000","last_became_peered":"2023-09-23T20:20:14.881544+0000","last_unstale":"2023-09-23T20:28:37.006521+0000","last_undegraded":"2023-09-23T20:28:37.006521+0000","last_fullsized":"2023-09-23T20:28:37.006521+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":102,"log_dups_size":0,"ondisk_log_size":102,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:34.594972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030656500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":963,"num_read_kb":1234,"num_write":60,"num_write_kb":225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":143,"reported_epoch":245,"state":"active+undersized","last_fresh":"2023-09-23T20:28:37.001943+0000","last_change":"2023-09-23T20:28:13.158522+0000","last_active":"2023-09-23T20:28:37.001943+0000","last_peered":"2023-09-23T20:28:37.001943+0000","last_clean":"2023-09-23T20:25:34.579453+0000","last_became_active":"2023-09-23T20:28:13.158522+0000","last_became_peered":"2023-09-23T20:28:13.158522+0000","last_unstale":"2023-09-23T20:28:37.001943+0000","last_undegraded":"2023-09-23T20:28:37.001943+0000","last_fullsized":"2023-09-23T20:28:13.139418+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:41:36.312605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5],"acting":[2,5],"avail_no_missing":["2","5"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":177,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T20:28:37.030549+0000","last_change":"2023-09-23T20:20:14.888874+0000","last_active":"2023-09-23T20:28:37.030549+0000","last_peered":"2023-09-23T20:28:37.030549+0000","last_clean":"2023-09-23T20:28:37.030549+0000","last_became_active":"2023-09-23T20:20:14.887724+0000","last_became_peered":"2023-09-23T20:20:14.887724+0000","last_unstale":"2023-09-23T20:28:37.030549+0000","last_undegraded":"2023-09-23T20:28:37.030549+0000","last_fullsized":"2023-09-23T20:28:37.030549+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:09:24.510326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"246'5752","reported_seq":5800,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T20:28:40.673010+0000","last_change":"2023-09-23T20:28:35.542621+0000","last_active":"2023-09-23T20:28:40.673010+0000","last_peered":"2023-09-23T20:28:40.673010+0000","last_clean":"2023-09-23T20:28:40.673010+0000","last_became_active":"2023-09-23T20:28:35.539470+0000","last_became_peered":"2023-09-23T20:28:35.539470+0000","last_unstale":"2023-09-23T20:28:40.673010+0000","last_undegraded":"2023-09-23T20:28:40.673010+0000","last_fullsized":"2023-09-23T20:28:40.673010+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":5752,"log_dups_size":0,"ondisk_log_size":5752,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":192741376,"num_objects":2941,"num_object_clones":0,"num_object_copies":8823,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2941,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2941,"num_write_kb":188224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":506,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T20:28:37.017805+0000","last_change":"2023-09-23T20:28:15.850428+0000","last_active":"2023-09-23T20:28:37.017805+0000","last_peered":"2023-09-23T20:28:37.017805+0000","last_clean":"2023-09-23T20:28:37.017805+0000","last_became_active":"2023-09-23T20:28:08.033922+0000","last_became_peered":"2023-09-23T20:28:08.033922+0000","last_unstale":"2023-09-23T20:28:37.017805+0000","last_undegraded":"2023-09-23T20:28:37.017805+0000","last_fullsized":"2023-09-23T20:28:37.017805+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:12.809933+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:12.809933+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.001626628,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":96,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T20:28:37.006701+0000","last_change":"2023-09-23T20:25:03.007736+0000","last_active":"2023-09-23T20:28:37.006701+0000","last_peered":"2023-09-23T20:28:37.006701+0000","last_clean":"2023-09-23T20:28:37.006701+0000","last_became_active":"2023-09-23T20:25:02.973689+0000","last_became_peered":"2023-09-23T20:25:02.973689+0000","last_unstale":"2023-09-23T20:28:37.006701+0000","last_undegraded":"2023-09-23T20:28:37.006701+0000","last_fullsized":"2023-09-23T20:28:37.006701+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:37:25.643774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"86'128","reported_seq":418,"reported_epoch":245,"state":"active+undersized","last_fresh":"2023-09-23T20:28:37.021574+0000","last_change":"2023-09-23T20:28:13.174258+0000","last_active":"2023-09-23T20:28:37.021574+0000","last_peered":"2023-09-23T20:28:37.021574+0000","last_clean":"2023-09-23T20:13:51.938835+0000","last_became_active":"2023-09-23T20:28:13.174258+0000","last_became_peered":"2023-09-23T20:28:13.174258+0000","last_unstale":"2023-09-23T20:28:37.021574+0000","last_undegraded":"2023-09-23T20:28:37.021574+0000","last_fullsized":"2023-09-23T20:28:13.149835+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:09:56.949690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000293202,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":579,"num_read_kb":1473,"num_write":96,"num_write_kb":1062,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":["4","3"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":92,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T20:28:37.005898+0000","last_change":"2023-09-23T20:25:02.971740+0000","last_active":"2023-09-23T20:28:37.005898+0000","last_peered":"2023-09-23T20:28:37.005898+0000","last_clean":"2023-09-23T20:28:37.005898+0000","last_became_active":"2023-09-23T20:25:02.970412+0000","last_became_peered":"2023-09-23T20:25:02.970412+0000","last_unstale":"2023-09-23T20:28:37.005898+0000","last_undegraded":"2023-09-23T20:28:37.005898+0000","last_fullsized":"2023-09-23T20:28:37.005898+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:53:17.247216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"246'5650","reported_seq":5720,"reported_epoch":246,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T20:28:40.552142+0000","last_change":"2023-09-23T20:28:35.541869+0000","last_active":"2023-09-23T20:28:40.552142+0000","last_peered":"2023-09-23T20:28:40.552142+0000","last_clean":"2023-09-23T20:28:04.010682+0000","last_became_active":"2023-09-23T20:28:35.538081+0000","last_became_peered":"2023-09-23T20:28:35.538081+0000","last_unstale":"2023-09-23T20:28:40.552142+0000","last_undegraded":"2023-09-23T20:28:35.532327+0000","last_fullsized":"2023-09-23T20:28:40.552142+0000","mapping_epoch":237,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":5650,"log_dups_size":0,"ondisk_log_size":5650,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":188841984,"num_objects":2881,"num_object_clones":0,"num_object_copies":8643,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":148,"num_objects_misplaced":2733,"num_objects_unfound":0,"num_objects_dirty":2881,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2881,"num_write_kb":184416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":98304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2,6],"avail_no_missing":["7","2"],"object_location_counts":[{"shards":"2,6,7","objects":2733},{"shards":"2,7","objects":148}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.4","version":"90'161","reported_seq":492,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T20:28:37.006415+0000","last_change":"2023-09-23T20:28:03.553294+0000","last_active":"2023-09-23T20:28:37.006415+0000","last_peered":"2023-09-23T20:28:37.006415+0000","last_clean":"2023-09-23T20:28:37.006415+0000","last_became_active":"2023-09-23T20:27:55.212201+0000","last_became_peered":"2023-09-23T20:27:55.212201+0000","last_unstale":"2023-09-23T20:28:37.006415+0000","last_undegraded":"2023-09-23T20:28:37.006415+0000","last_fullsized":"2023-09-23T20:28:37.006415+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:28:03.553106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:28:03.553106+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:24:06.706940+0000","scrub_duration":0.011176864,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"86'117","reported_seq":608,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T20:28:37.004228+0000","last_change":"2023-09-23T20:28:08.028985+0000","last_active":"2023-09-23T20:28:37.004228+0000","last_peered":"2023-09-23T20:28:37.004228+0000","last_clean":"2023-09-23T20:28:37.004228+0000","last_became_active":"2023-09-23T20:28:08.028620+0000","last_became_peered":"2023-09-23T20:28:08.028620+0000","last_unstale":"2023-09-23T20:28:37.004228+0000","last_undegraded":"2023-09-23T20:28:37.004228+0000","last_fullsized":"2023-09-23T20:28:37.004228+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:52:44.688867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046891400000000002,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":893,"num_read_kb":922,"num_write":82,"num_write_kb":4193,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":202,"reported_epoch":245,"state":"active+undersized+degraded","last_fresh":"2023-09-23T20:28:37.030982+0000","last_change":"2023-09-23T20:28:13.176296+0000","last_active":"2023-09-23T20:28:37.030982+0000","last_peered":"2023-09-23T20:28:37.030982+0000","last_clean":"2023-09-23T20:28:10.114330+0000","last_became_active":"2023-09-23T20:28:13.176296+0000","last_became_peered":"2023-09-23T20:28:13.176296+0000","last_unstale":"2023-09-23T20:28:37.030982+0000","last_undegraded":"2023-09-23T20:28:13.140349+0000","last_fullsized":"2023-09-23T20:28:13.140255+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:12:18.491192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":["0","4"],"object_location_counts":[{"shards":"0,4","objects":1}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":197,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T20:28:37.018148+0000","last_change":"2023-09-23T20:28:05.035425+0000","last_active":"2023-09-23T20:28:37.018148+0000","last_peered":"2023-09-23T20:28:37.018148+0000","last_clean":"2023-09-23T20:28:37.018148+0000","last_became_active":"2023-09-23T20:28:05.035079+0000","last_became_peered":"2023-09-23T20:28:05.035079+0000","last_unstale":"2023-09-23T20:28:37.018148+0000","last_undegraded":"2023-09-23T20:28:37.018148+0000","last_fullsized":"2023-09-23T20:28:37.018148+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:34:34.293359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":179,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T20:28:37.030491+0000","last_change":"2023-09-23T20:20:14.888484+0000","last_active":"2023-09-23T20:28:37.030491+0000","last_peered":"2023-09-23T20:28:37.030491+0000","last_clean":"2023-09-23T20:28:37.030491+0000","last_became_active":"2023-09-23T20:20:14.887219+0000","last_became_peered":"2023-09-23T20:20:14.887219+0000","last_unstale":"2023-09-23T20:28:37.030491+0000","last_undegraded":"2023-09-23T20:28:37.030491+0000","last_fullsized":"2023-09-23T20:28:37.030491+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:13:53.763649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":87,"reported_epoch":245,"state":"active+undersized","last_fresh":"2023-09-23T20:28:37.002108+0000","last_change":"2023-09-23T20:28:13.152589+0000","last_active":"2023-09-23T20:28:37.002108+0000","last_peered":"2023-09-23T20:28:37.002108+0000","last_clean":"2023-09-23T20:28:12.056838+0000","last_became_active":"2023-09-23T20:28:13.152589+0000","last_became_peered":"2023-09-23T20:28:13.152589+0000","last_unstale":"2023-09-23T20:28:37.002108+0000","last_undegraded":"2023-09-23T20:28:37.002108+0000","last_fullsized":"2023-09-23T20:28:13.139976+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:54:38.464315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"6.19","version":"246'5694","reported_seq":5741,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T20:28:39.906678+0000","last_change":"2023-09-23T20:28:35.571020+0000","last_active":"2023-09-23T20:28:39.906678+0000","last_peered":"2023-09-23T20:28:39.906678+0000","last_clean":"2023-09-23T20:28:39.906678+0000","last_became_active":"2023-09-23T20:28:35.567626+0000","last_became_peered":"2023-09-23T20:28:35.567626+0000","last_unstale":"2023-09-23T20:28:39.906678+0000","last_undegraded":"2023-09-23T20:28:39.906678+0000","last_fullsized":"2023-09-23T20:28:39.906678+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":242,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":5694,"log_dups_size":0,"ondisk_log_size":5694,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":189923328,"num_objects":2898,"num_object_clones":0,"num_object_copies":8694,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2898,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2898,"num_write_kb":185472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":214,"reported_epoch":245,"state":"active+undersized+degraded","last_fresh":"2023-09-23T20:28:37.009191+0000","last_change":"2023-09-23T20:28:13.169405+0000","last_active":"2023-09-23T20:28:37.009191+0000","last_peered":"2023-09-23T20:28:37.009191+0000","last_clean":"2023-09-23T20:28:10.121149+0000","last_became_active":"2023-09-23T20:28:13.169405+0000","last_became_peered":"2023-09-23T20:28:13.169405+0000","last_unstale":"2023-09-23T20:28:37.009191+0000","last_undegraded":"2023-09-23T20:28:13.153461+0000","last_fullsized":"2023-09-23T20:28:13.153347+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:03:24.383853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":["5","4"],"object_location_counts":[{"shards":"4,5","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":258,"reported_epoch":245,"state":"active+undersized","last_fresh":"2023-09-23T20:28:37.019612+0000","last_change":"2023-09-23T20:28:13.169986+0000","last_active":"2023-09-23T20:28:37.019612+0000","last_peered":"2023-09-23T20:28:37.019612+0000","last_clean":"2023-09-23T20:28:10.122290+0000","last_became_active":"2023-09-23T20:28:13.169986+0000","last_became_peered":"2023-09-23T20:28:13.169986+0000","last_unstale":"2023-09-23T20:28:37.019612+0000","last_undegraded":"2023-09-23T20:28:37.019612+0000","last_fullsized":"2023-09-23T20:28:13.148881+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:44.116758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":["4","3"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.18","version":"246'5688","reported_seq":5735,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T20:28:40.622080+0000","last_change":"2023-09-23T20:28:35.587755+0000","last_active":"2023-09-23T20:28:40.622080+0000","last_peered":"2023-09-23T20:28:40.622080+0000","last_clean":"2023-09-23T20:28:40.622080+0000","last_became_active":"2023-09-23T20:28:35.585502+0000","last_became_peered":"2023-09-23T20:28:35.585502+0000","last_unstale":"2023-09-23T20:28:40.622080+0000","last_undegraded":"2023-09-23T20:28:40.622080+0000","last_fullsized":"2023-09-23T20:28:40.622080+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":242,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":5688,"log_dups_size":0,"ondisk_log_size":5688,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":190185472,"num_objects":2902,"num_object_clones":0,"num_object_copies":8706,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2902,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2902,"num_write_kb":185728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":211,"reported_epoch":245,"state":"active+undersized+degraded","last_fresh":"2023-09-23T20:28:37.008670+0000","last_change":"2023-09-23T20:28:13.171105+0000","last_active":"2023-09-23T20:28:37.008670+0000","last_peered":"2023-09-23T20:28:37.008670+0000","last_clean":"2023-09-23T20:28:10.121557+0000","last_became_active":"2023-09-23T20:28:13.171105+0000","last_became_peered":"2023-09-23T20:28:13.171105+0000","last_unstale":"2023-09-23T20:28:37.008670+0000","last_undegraded":"2023-09-23T20:28:13.154069+0000","last_fullsized":"2023-09-23T20:28:13.153978+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:04:22.559138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":["5","4"],"object_location_counts":[{"shards":"4,5","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":251,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T20:28:37.020001+0000","last_change":"2023-09-23T20:25:03.103656+0000","last_active":"2023-09-23T20:28:37.020001+0000","last_peered":"2023-09-23T20:28:37.020001+0000","last_clean":"2023-09-23T20:28:37.020001+0000","last_became_active":"2023-09-23T20:25:03.103311+0000","last_became_peered":"2023-09-23T20:25:03.103311+0000","last_unstale":"2023-09-23T20:28:37.020001+0000","last_undegraded":"2023-09-23T20:28:37.020001+0000","last_fullsized":"2023-09-23T20:28:37.020001+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:55:54.677030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":116,"reported_epoch":245,"state":"active+undersized","last_fresh":"2023-09-23T20:28:37.006566+0000","last_change":"2023-09-23T20:28:13.146288+0000","last_active":"2023-09-23T20:28:37.006566+0000","last_peered":"2023-09-23T20:28:37.006566+0000","last_clean":"2023-09-23T20:28:10.123189+0000","last_became_active":"2023-09-23T20:28:13.146288+0000","last_became_peered":"2023-09-23T20:28:13.146288+0000","last_unstale":"2023-09-23T20:28:37.006566+0000","last_undegraded":"2023-09-23T20:28:37.006566+0000","last_fullsized":"2023-09-23T20:28:13.141006+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:49:57.680453+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":["3","2"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":101,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T20:28:37.005968+0000","last_change":"2023-09-23T20:25:39.622819+0000","last_active":"2023-09-23T20:28:37.005968+0000","last_peered":"2023-09-23T20:28:37.005968+0000","last_clean":"2023-09-23T20:28:37.005968+0000","last_became_active":"2023-09-23T20:25:39.622484+0000","last_became_peered":"2023-09-23T20:25:39.622484+0000","last_unstale":"2023-09-23T20:28:37.005968+0000","last_undegraded":"2023-09-23T20:28:37.005968+0000","last_fullsized":"2023-09-23T20:28:37.005968+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:51:53.973261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":225,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T20:28:37.031602+0000","last_change":"2023-09-23T20:25:39.668882+0000","last_active":"2023-09-23T20:28:37.031602+0000","last_peered":"2023-09-23T20:28:37.031602+0000","last_clean":"2023-09-23T20:28:37.031602+0000","last_became_active":"2023-09-23T20:25:39.637128+0000","last_became_peered":"2023-09-23T20:25:39.637128+0000","last_unstale":"2023-09-23T20:28:37.031602+0000","last_undegraded":"2023-09-23T20:28:37.031602+0000","last_fullsized":"2023-09-23T20:28:37.031602+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:56.406298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":241,"reported_epoch":245,"state":"active+undersized","last_fresh":"2023-09-23T20:28:37.017919+0000","last_change":"2023-09-23T20:28:13.156720+0000","last_active":"2023-09-23T20:28:37.017919+0000","last_peered":"2023-09-23T20:28:37.017919+0000","last_clean":"2023-09-23T20:28:10.121358+0000","last_became_active":"2023-09-23T20:28:13.156720+0000","last_became_peered":"2023-09-23T20:28:13.156720+0000","last_unstale":"2023-09-23T20:28:37.017919+0000","last_undegraded":"2023-09-23T20:28:37.017919+0000","last_fullsized":"2023-09-23T20:28:13.147190+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:21:27.183630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":["4","3"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":26,"stat_sum":{"num_bytes":6125846528,"num_objects":93473,"num_object_clones":0,"num_object_copies":280419,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":31506,"num_objects_misplaced":17508,"num_objects_unfound":0,"num_objects_dirty":93473,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":93473,"num_write_kb":5982272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3033,"num_bytes_recovered":198770688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":18253348864,"data_stored":18253348864,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":145721,"ondisk_log_size":145721,"up":69,"acting":71,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":76,"acting":76,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":25,"num_bytes_recovered":6160,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":25181,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":81,"acting":81,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":713712,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":192,"num_read_kb":360,"num_write":95,"num_write_kb":953,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":1025388,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2109440,"data_stored":2098784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3381,"ondisk_log_size":3381,"up":57,"acting":57,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":4194323,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16793600,"data_stored":16777292,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6082,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":20,"acting":20,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201490,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2640212,"kb_used_data":2467212,"kb_used_omap":0,"kb_used_meta":172992,"kb_avail":91104940,"statfs":{"total":95995035648,"available":93291458560,"internally_reserved":0,"allocated":2526425088,"data_stored":2525656330,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":177143808},"hb_peers":[0,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:28:27 2023","interfaces":[{"interface":"back","average":{"1min":1.073,"5min":2.004,"15min":1.048},"min":{"1min":0.20799999999999999,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":4.4829999999999997,"5min":48.798999999999999,"15min":48.798999999999999},"last":0.91800000000000004},{"interface":"front","average":{"1min":1.0589999999999999,"5min":2.3690000000000002,"15min":1.1679999999999999},"min":{"1min":0.30499999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":3.7839999999999998,"5min":48.247,"15min":48.247},"last":0.73699999999999999}]},{"osd":1,"last update":"Sat Sep 23 20:27:59 2023","interfaces":[{"interface":"back","average":{"1min":1.1759999999999999,"5min":1.7689999999999999,"15min":0.96599999999999997},"min":{"1min":0.29199999999999998,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":5.1509999999999998,"5min":17.795999999999999,"15min":17.795999999999999},"last":3.1429999999999998},{"interface":"front","average":{"1min":0.75900000000000001,"5min":1.5349999999999999,"15min":0.91900000000000004},"min":{"1min":0.33700000000000002,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":1.9219999999999999,"5min":13.250999999999999,"15min":13.250999999999999},"last":0.33100000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:28:32 2023","interfaces":[{"interface":"back","average":{"1min":1.9319999999999999,"5min":1.944,"15min":1.034},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":11.372999999999999,"5min":11.372999999999999,"15min":11.372999999999999},"last":0.76000000000000001},{"interface":"front","average":{"1min":1.621,"5min":2.056,"15min":1.091},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":5.3200000000000003,"5min":11.423999999999999,"15min":11.423999999999999},"last":5.6669999999999998}]},{"osd":3,"last update":"Sat Sep 23 20:27:59 2023","interfaces":[{"interface":"back","average":{"1min":1.3120000000000001,"5min":1.8879999999999999,"15min":1.0660000000000001},"min":{"1min":0.46200000000000002,"5min":0.314,"15min":0.314},"max":{"1min":4.7880000000000003,"5min":6.891,"15min":6.891},"last":0.61099999999999999},{"interface":"front","average":{"1min":1.4299999999999999,"5min":1.7789999999999999,"15min":1.042},"min":{"1min":0.46999999999999997,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":5.8209999999999997,"5min":9.7840000000000007,"15min":9.7840000000000007},"last":0.70199999999999996}]},{"osd":4,"last update":"Sat Sep 23 20:27:59 2023","interfaces":[{"interface":"back","average":{"1min":0.79100000000000004,"5min":1.2170000000000001,"15min":0.73699999999999999},"min":{"1min":0.27100000000000002,"5min":0.27000000000000002,"15min":0.26700000000000002},"max":{"1min":2.2629999999999999,"5min":13.141999999999999,"15min":13.141999999999999},"last":0.89100000000000001},{"interface":"front","average":{"1min":0.96599999999999997,"5min":1.7529999999999999,"15min":0.94299999999999995},"min":{"1min":0.34300000000000003,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":4.9909999999999997,"5min":10.853,"15min":10.853},"last":0.78500000000000003}]},{"osd":5,"last update":"Sat Sep 23 20:27:59 2023","interfaces":[{"interface":"back","average":{"1min":1.4690000000000001,"5min":1.8240000000000001,"15min":0.96599999999999997},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":13.241,"5min":13.241,"15min":13.241},"last":1.034},{"interface":"front","average":{"1min":1.4330000000000001,"5min":1.7370000000000001,"15min":0.96699999999999997},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":14.012,"5min":14.012,"15min":14.012},"last":4.2759999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:27:59 2023","interfaces":[{"interface":"back","average":{"1min":0.78300000000000003,"5min":1.4470000000000001,"15min":0.88200000000000001},"min":{"1min":0.35499999999999998,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":2.7989999999999999,"5min":13.185,"15min":13.185},"last":0.93899999999999995},{"interface":"front","average":{"1min":0.78900000000000003,"5min":1.5169999999999999,"15min":0.90600000000000003},"min":{"1min":0.40000000000000002,"5min":0.33000000000000002,"15min":0.29899999999999999},"max":{"1min":1.968,"5min":9.9120000000000008,"15min":9.9120000000000008},"last":3.8639999999999999}]}]},{"osd":6,"up_from":0,"seq":201863463126,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":1,"up_from":99,"seq":425201762395,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1618524,"kb_used_data":1418708,"kb_used_omap":6,"kb_used_meta":199801,"kb_avail":92126628,"statfs":{"total":95995035648,"available":94337667072,"internally_reserved":0,"allocated":1452756992,"data_stored":1451546929,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6696,"internal_metadata":204596696},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:27:55 2023","interfaces":[{"interface":"back","average":{"1min":2.044,"5min":2.3929999999999998,"15min":1.046},"min":{"1min":0.224,"5min":0.20599999999999999,"15min":0.19600000000000001},"max":{"1min":17.904,"5min":40.113,"15min":40.113},"last":0.28199999999999997},{"interface":"front","average":{"1min":1.8799999999999999,"5min":2.1150000000000002,"15min":0.96899999999999997},"min":{"1min":0.192,"5min":0.192,"15min":0.192},"max":{"1min":17.812000000000001,"5min":42.994999999999997,"15min":42.994999999999997},"last":7.4850000000000003}]},{"osd":2,"last update":"Sat Sep 23 20:27:30 2023","interfaces":[{"interface":"back","average":{"1min":2.0299999999999998,"5min":2.1320000000000001,"15min":0.96499999999999997},"min":{"1min":0.30099999999999999,"5min":0.253,"15min":0.253},"max":{"1min":17.940000000000001,"5min":39.962000000000003,"15min":39.962000000000003},"last":0.59099999999999997},{"interface":"front","average":{"1min":2.3730000000000002,"5min":2.1579999999999999,"15min":1.087},"min":{"1min":0.26300000000000001,"5min":0.246,"15min":0.246},"max":{"1min":15.090999999999999,"5min":47.442,"15min":47.442},"last":8.7899999999999991}]},{"osd":3,"last update":"Sat Sep 23 20:28:02 2023","interfaces":[{"interface":"back","average":{"1min":1.361,"5min":1.5660000000000001,"15min":0.88800000000000001},"min":{"1min":0.32900000000000001,"5min":0.246,"15min":0.246},"max":{"1min":4.0030000000000001,"5min":9.0410000000000004,"15min":9.0410000000000004},"last":0.22500000000000001},{"interface":"front","average":{"1min":2.2000000000000002,"5min":1.8300000000000001,"15min":0.88200000000000001},"min":{"1min":0.248,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":17.853000000000002,"5min":17.853000000000002,"15min":17.853000000000002},"last":7.7439999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:27:55 2023","interfaces":[{"interface":"back","average":{"1min":1.359,"5min":1.7370000000000001,"15min":0.98999999999999999},"min":{"1min":0.33600000000000002,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":4.4550000000000001,"5min":15.353,"15min":15.353},"last":0.32300000000000001},{"interface":"front","average":{"1min":1.071,"5min":1.625,"15min":0.96399999999999997},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":3.6480000000000001,"5min":8.7899999999999991,"15min":8.7899999999999991},"last":0.42499999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:27:55 2023","interfaces":[{"interface":"back","average":{"1min":1.143,"5min":1.72,"15min":1.0009999999999999},"min":{"1min":0.32800000000000001,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":4.6920000000000002,"5min":12.590999999999999,"15min":12.590999999999999},"last":7.5460000000000003},{"interface":"front","average":{"1min":1.605,"5min":2.3250000000000002,"15min":1.2250000000000001},"min":{"1min":0.40100000000000002,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":4.5369999999999999,"5min":39.579000000000001,"15min":39.579000000000001},"last":0.35699999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:27:55 2023","interfaces":[{"interface":"back","average":{"1min":1.6379999999999999,"5min":2.198,"15min":1.175},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":9.0310000000000006,"5min":34.409999999999997,"15min":34.409999999999997},"last":7.5750000000000002},{"interface":"front","average":{"1min":1.6140000000000001,"5min":2.4220000000000002,"15min":1.232},"min":{"1min":0.48599999999999999,"5min":0.377,"15min":0.377},"max":{"1min":4.5860000000000003,"5min":35.554000000000002,"15min":35.554000000000002},"last":0.39000000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:27:55 2023","interfaces":[{"interface":"back","average":{"1min":1.617,"5min":2.044,"15min":1.115},"min":{"1min":0.505,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":4.6139999999999999,"5min":9.7439999999999998,"15min":9.7439999999999998},"last":4.1660000000000004},{"interface":"front","average":{"1min":1.3200000000000001,"5min":1.9079999999999999,"15min":1.089},"min":{"1min":0.41299999999999998,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":5.7439999999999998,"5min":9.2279999999999998,"15min":9.2279999999999998},"last":0.40100000000000002}]}]},{"osd":0,"up_from":94,"seq":403726925927,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1721536,"kb_used_data":1474488,"kb_used_omap":7,"kb_used_meta":247032,"kb_avail":92023616,"statfs":{"total":95995035648,"available":94232182784,"internally_reserved":0,"allocated":1509875712,"data_stored":1508712138,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":252960785},"hb_peers":[2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:27:57 2023","interfaces":[{"interface":"back","average":{"1min":0.97099999999999997,"5min":1.4650000000000001,"15min":0.72599999999999998},"min":{"1min":0.26300000000000001,"5min":0.20100000000000001,"15min":0.17699999999999999},"max":{"1min":4.7060000000000004,"5min":24.247,"15min":24.247},"last":4.3479999999999999},{"interface":"front","average":{"1min":0.98899999999999999,"5min":1.4339999999999999,"15min":0.73599999999999999},"min":{"1min":0.27900000000000003,"5min":0.222,"15min":0.222},"max":{"1min":2.9289999999999998,"5min":15.994999999999999,"15min":15.994999999999999},"last":6.4279999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:28:25 2023","interfaces":[{"interface":"back","average":{"1min":2.8959999999999999,"5min":2.3940000000000001,"15min":1.1990000000000001},"min":{"1min":0.19800000000000001,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":9.4459999999999997,"5min":23.832999999999998,"15min":23.832999999999998},"last":0.62},{"interface":"front","average":{"1min":2.9289999999999998,"5min":2.0430000000000001,"15min":1.028},"min":{"1min":0.20999999999999999,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":13.337,"5min":15.948,"15min":15.948},"last":7.6509999999999998}]},{"osd":3,"last update":"Sat Sep 23 20:27:57 2023","interfaces":[{"interface":"back","average":{"1min":0.81499999999999995,"5min":1.7509999999999999,"15min":0.98099999999999998},"min":{"1min":0.28999999999999998,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":2.0539999999999998,"5min":23.748999999999999,"15min":23.748999999999999},"last":0.64200000000000002},{"interface":"front","average":{"1min":1.123,"5min":1.659,"15min":0.83399999999999996},"min":{"1min":0.29299999999999998,"5min":0.24099999999999999,"15min":0.23899999999999999},"max":{"1min":4.7930000000000001,"5min":24.274999999999999,"15min":24.274999999999999},"last":2.7719999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:28:25 2023","interfaces":[{"interface":"back","average":{"1min":2.5310000000000001,"5min":2.4750000000000001,"15min":1.238},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":9.5459999999999994,"5min":23.876999999999999,"15min":23.876999999999999},"last":0.65900000000000003},{"interface":"front","average":{"1min":1.514,"5min":1.944,"15min":1.0589999999999999},"min":{"1min":0.312,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":4.6609999999999996,"5min":23.681999999999999,"15min":23.681999999999999},"last":2.9940000000000002}]},{"osd":5,"last update":"Sat Sep 23 20:28:25 2023","interfaces":[{"interface":"back","average":{"1min":1.867,"5min":2.7109999999999999,"15min":1.343},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":4.774,"5min":24.198,"15min":24.198},"last":3.032},{"interface":"front","average":{"1min":2.6739999999999999,"5min":2.254,"15min":1.202},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":6.694,"5min":11.167999999999999,"15min":11.167999999999999},"last":1.5960000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:28:25 2023","interfaces":[{"interface":"back","average":{"1min":2.0870000000000002,"5min":2.0470000000000002,"15min":1.1220000000000001},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":4.8360000000000003,"5min":24.408000000000001,"15min":24.408000000000001},"last":2.9399999999999999},{"interface":"front","average":{"1min":2.4409999999999998,"5min":1.9650000000000001,"15min":1.1000000000000001},"min":{"1min":0.38800000000000001,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":6.3620000000000001,"5min":25.271000000000001,"15min":25.271000000000001},"last":1.637}]},{"osd":7,"last update":"Sat Sep 23 20:28:25 2023","interfaces":[{"interface":"back","average":{"1min":2.2639999999999998,"5min":2.133,"15min":1.163},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":7.9429999999999996,"5min":25.359000000000002,"15min":25.359000000000002},"last":1.671},{"interface":"front","average":{"1min":2.6669999999999998,"5min":2.7120000000000002,"15min":1.3600000000000001},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":14.509,"5min":23.905999999999999,"15min":23.905999999999999},"last":0.68000000000000005}]}]},{"osd":2,"up_from":104,"seq":446676598875,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3634060,"kb_used_data":3331844,"kb_used_omap":11,"kb_used_meta":302196,"kb_avail":90111092,"statfs":{"total":95995035648,"available":92273758208,"internally_reserved":0,"allocated":3411808256,"data_stored":3410643206,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11878,"internal_metadata":309449114},"hb_peers":[0,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:28:33 2023","interfaces":[{"interface":"back","average":{"1min":2.7869999999999999,"5min":2.1619999999999999,"15min":1.012},"min":{"1min":0.19500000000000001,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":21.695,"5min":21.695,"15min":21.695},"last":0.53800000000000003},{"interface":"front","average":{"1min":2.109,"5min":1.5680000000000001,"15min":0.82599999999999996},"min":{"1min":0.25900000000000001,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":8.6340000000000003,"5min":15.746,"15min":15.746},"last":0.61199999999999999}]},{"osd":1,"last update":"Sat Sep 23 20:27:30 2023","interfaces":[{"interface":"back","average":{"1min":0.93799999999999994,"5min":1.633,"15min":0.80700000000000005},"min":{"1min":0.29799999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":3.3410000000000002,"5min":14.135999999999999,"15min":14.135999999999999},"last":8.8049999999999997},{"interface":"front","average":{"1min":1.863,"5min":1.522,"15min":0.76200000000000001},"min":{"1min":0.27500000000000002,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":14.201000000000001,"5min":14.201000000000001,"15min":14.201000000000001},"last":2.1640000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:27:58 2023","interfaces":[{"interface":"back","average":{"1min":1.591,"5min":1.595,"15min":0.77300000000000002},"min":{"1min":0.28199999999999997,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":8.4849999999999994,"5min":15.319000000000001,"15min":15.319000000000001},"last":0.50700000000000001},{"interface":"front","average":{"1min":2.0510000000000002,"5min":1.8819999999999999,"15min":0.89200000000000002},"min":{"1min":0.40500000000000003,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":12.067,"5min":12.494999999999999,"15min":12.494999999999999},"last":4.1180000000000003}]},{"osd":4,"last update":"Sat Sep 23 20:28:33 2023","interfaces":[{"interface":"back","average":{"1min":1.889,"5min":1.633,"15min":0.94499999999999995},"min":{"1min":0.44900000000000001,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":10.84,"5min":10.84,"15min":10.84},"last":2.6240000000000001},{"interface":"front","average":{"1min":2.7320000000000002,"5min":1.8,"15min":1.0029999999999999},"min":{"1min":0.28799999999999998,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":14.971,"5min":14.971,"15min":14.971},"last":0.40899999999999997}]},{"osd":5,"last update":"Sat Sep 23 20:28:33 2023","interfaces":[{"interface":"back","average":{"1min":1.5669999999999999,"5min":1.788,"15min":1.046},"min":{"1min":0.42299999999999999,"5min":0.309,"15min":0.309},"max":{"1min":5.399,"5min":11.237,"15min":11.237},"last":0.65800000000000003},{"interface":"front","average":{"1min":2.698,"5min":2.0249999999999999,"15min":1.133},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":15.057,"5min":15.057,"15min":15.057},"last":0.45500000000000002}]},{"osd":6,"last update":"Sat Sep 23 20:28:33 2023","interfaces":[{"interface":"back","average":{"1min":2.8130000000000002,"5min":1.728,"15min":1.0289999999999999},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":15.023999999999999,"5min":15.023999999999999,"15min":15.023999999999999},"last":0.48499999999999999},{"interface":"front","average":{"1min":3.1379999999999999,"5min":2.2080000000000002,"15min":1.1890000000000001},"min":{"1min":0.45600000000000002,"5min":0.377,"15min":0.377},"max":{"1min":21.663,"5min":21.663,"15min":21.663},"last":0.56000000000000005}]},{"osd":7,"last update":"Sat Sep 23 20:28:33 2023","interfaces":[{"interface":"back","average":{"1min":3.2570000000000001,"5min":2.3180000000000001,"15min":1.226},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":21.637,"5min":21.637,"15min":21.637},"last":0.58499999999999996},{"interface":"front","average":{"1min":1.907,"5min":1.8700000000000001,"15min":1.085},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":6.0049999999999999,"5min":10.064,"15min":10.064},"last":0.63500000000000001}]}]},{"osd":3,"up_from":109,"seq":468151435349,"num_pgs":65,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4101648,"kb_used_data":3793160,"kb_used_omap":4,"kb_used_meta":308475,"kb_avail":89643504,"statfs":{"total":95995035648,"available":91794948096,"internally_reserved":0,"allocated":3884195840,"data_stored":3883021707,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4514,"internal_metadata":315879006},"hb_peers":[0,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,0,1,1,0,4],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:27:59 2023","interfaces":[{"interface":"back","average":{"1min":1.8340000000000001,"5min":2.6160000000000001,"15min":1.1060000000000001},"min":{"1min":0.23499999999999999,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":6.7460000000000004,"5min":67.305999999999997,"15min":67.305999999999997},"last":2.633},{"interface":"front","average":{"1min":1.409,"5min":2.2389999999999999,"15min":1.0029999999999999},"min":{"1min":0.29399999999999998,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":7.0999999999999996,"5min":67.403999999999996,"15min":67.403999999999996},"last":2.871}]},{"osd":1,"last update":"Sat Sep 23 20:27:59 2023","interfaces":[{"interface":"back","average":{"1min":1.026,"5min":1.633,"15min":0.80900000000000005},"min":{"1min":0.221,"5min":0.214,"15min":0.214},"max":{"1min":4.6390000000000002,"5min":19.675999999999998,"15min":19.675999999999998},"last":1.0169999999999999},{"interface":"front","average":{"1min":1.371,"5min":1.5600000000000001,"15min":0.81599999999999995},"min":{"1min":0.29099999999999998,"5min":0.245,"15min":0.245},"max":{"1min":7.0800000000000001,"5min":11.644,"15min":11.644},"last":3.2770000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:27:59 2023","interfaces":[{"interface":"back","average":{"1min":1.4710000000000001,"5min":2.0030000000000001,"15min":0.90300000000000002},"min":{"1min":0.25600000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":7.069,"5min":24.045000000000002,"15min":24.045000000000002},"last":2.8100000000000001},{"interface":"front","average":{"1min":1.236,"5min":1.962,"15min":0.96299999999999997},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":5.5300000000000002,"5min":17.684000000000001,"15min":17.684000000000001},"last":2.3639999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:27:59 2023","interfaces":[{"interface":"back","average":{"1min":3.121,"5min":1.962,"15min":1.0569999999999999},"min":{"1min":0.36899999999999999,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":46.042999999999999,"5min":46.042999999999999,"15min":46.042999999999999},"last":2.7130000000000001},{"interface":"front","average":{"1min":2.9540000000000002,"5min":2.286,"15min":1.1719999999999999},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":46.579999999999998,"5min":46.579999999999998,"15min":46.579999999999998},"last":2.4220000000000002}]},{"osd":5,"last update":"Sat Sep 23 20:27:59 2023","interfaces":[{"interface":"back","average":{"1min":3.1459999999999999,"5min":2.3639999999999999,"15min":1.1919999999999999},"min":{"1min":0.442,"5min":0.311,"15min":0.311},"max":{"1min":45.292000000000002,"5min":45.292000000000002,"15min":45.292000000000002},"last":2.79},{"interface":"front","average":{"1min":3.177,"5min":1.804,"15min":1.022},"min":{"1min":0.42199999999999999,"5min":0.313,"15min":0.313},"max":{"1min":44.679000000000002,"5min":44.679000000000002,"15min":44.679000000000002},"last":2.9529999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:27:59 2023","interfaces":[{"interface":"back","average":{"1min":1.2689999999999999,"5min":1.7170000000000001,"15min":0.997},"min":{"1min":0.45400000000000001,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":6.9180000000000001,"5min":8.6059999999999999,"15min":8.6059999999999999},"last":2.9830000000000001},{"interface":"front","average":{"1min":1.075,"5min":1.655,"15min":0.999},"min":{"1min":0.39700000000000002,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":3.9329999999999998,"5min":9.3179999999999996,"15min":9.3179999999999996},"last":2.4649999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:27:59 2023","interfaces":[{"interface":"back","average":{"1min":1.294,"5min":1.704,"15min":1.036},"min":{"1min":0.374,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":8.4640000000000004,"5min":9.2750000000000004,"15min":9.2750000000000004},"last":2.9359999999999999},{"interface":"front","average":{"1min":3.444,"5min":2.5819999999999999,"15min":1.288},"min":{"1min":0.41099999999999998,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":45.387999999999998,"5min":45.387999999999998,"15min":45.387999999999998},"last":2.6760000000000002}]}]},{"osd":4,"up_from":30,"seq":128849019102,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2010636,"kb_used_data":1850500,"kb_used_omap":0,"kb_used_meta":160128,"kb_avail":91734516,"statfs":{"total":95995035648,"available":93936144384,"internally_reserved":0,"allocated":1894912000,"data_stored":1894151041,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":163971072},"hb_peers":[0,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:28:28 2023","interfaces":[{"interface":"back","average":{"1min":1.7070000000000001,"5min":1.9259999999999999,"15min":1.022},"min":{"1min":0.25600000000000001,"5min":0.222,"15min":0.222},"max":{"1min":8.2899999999999991,"5min":20.411999999999999,"15min":20.411999999999999},"last":4.7450000000000001},{"interface":"front","average":{"1min":1.778,"5min":2.004,"15min":1.091},"min":{"1min":0.307,"5min":0.253,"15min":0.253},"max":{"1min":6.899,"5min":20.454999999999998,"15min":20.454999999999998},"last":11.868}]},{"osd":1,"last update":"Sat Sep 23 20:28:01 2023","interfaces":[{"interface":"back","average":{"1min":1.7509999999999999,"5min":1.748,"15min":0.995},"min":{"1min":0.35699999999999998,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":8.7249999999999996,"5min":9.6660000000000004,"15min":9.6660000000000004},"last":3.4529999999999998},{"interface":"front","average":{"1min":0.70799999999999996,"5min":1.595,"15min":0.94399999999999995},"min":{"1min":0.32700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":2.2149999999999999,"5min":12.821999999999999,"15min":12.821999999999999},"last":0.30599999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:28:37 2023","interfaces":[{"interface":"back","average":{"1min":1.194,"5min":1.913,"15min":1.048},"min":{"1min":0.29899999999999999,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":4.7130000000000001,"5min":21.462,"15min":21.462},"last":0.56399999999999995},{"interface":"front","average":{"1min":2.2440000000000002,"5min":2.3490000000000002,"15min":1.202},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.26800000000000002},"max":{"1min":7.6619999999999999,"5min":22.593,"15min":22.593},"last":12.103}]},{"osd":3,"last update":"Sat Sep 23 20:28:01 2023","interfaces":[{"interface":"back","average":{"1min":1.3120000000000001,"5min":1.9279999999999999,"15min":1.0700000000000001},"min":{"1min":0.33400000000000002,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":8.0519999999999996,"5min":24.891999999999999,"15min":24.891999999999999},"last":2.7650000000000001},{"interface":"front","average":{"1min":1.212,"5min":1.9410000000000001,"15min":1.0900000000000001},"min":{"1min":0.371,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":4.6520000000000001,"5min":21.922000000000001,"15min":21.922000000000001},"last":1.9850000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:28:15 2023","interfaces":[{"interface":"back","average":{"1min":1.1479999999999999,"5min":2.0099999999999998,"15min":0.997},"min":{"1min":0.26900000000000002,"5min":0.247,"15min":0.247},"max":{"1min":4.149,"5min":17.704000000000001,"15min":17.704000000000001},"last":2.2709999999999999},{"interface":"front","average":{"1min":1.3640000000000001,"5min":1.8100000000000001,"15min":0.94299999999999995},"min":{"1min":0.35899999999999999,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":8.1379999999999999,"5min":15.329000000000001,"15min":15.329000000000001},"last":2.3420000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:28:37 2023","interfaces":[{"interface":"back","average":{"1min":1.4179999999999999,"5min":2.2189999999999999,"15min":1.095},"min":{"1min":0.33100000000000002,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":3.8140000000000001,"5min":41.371000000000002,"15min":41.371000000000002},"last":2.379},{"interface":"front","average":{"1min":1.341,"5min":1.9570000000000001,"15min":1.018},"min":{"1min":0.318,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":4.2160000000000002,"5min":41.140999999999998,"15min":41.140999999999998},"last":2.306}]},{"osd":7,"last update":"Sat Sep 23 20:27:51 2023","interfaces":[{"interface":"back","average":{"1min":0.85499999999999998,"5min":1.925,"15min":1.018},"min":{"1min":0.439,"5min":0.35099999999999998,"15min":0.33600000000000002},"max":{"1min":3.1840000000000002,"5min":16.427,"15min":16.427},"last":2.5379999999999998},{"interface":"front","average":{"1min":0.86099999999999999,"5min":1.7989999999999999,"15min":0.98799999999999999},"min":{"1min":0.36499999999999999,"5min":0.34300000000000003,"15min":0.30499999999999999},"max":{"1min":2.4159999999999999,"5min":19.018999999999998,"15min":19.018999999999998},"last":2.4049999999999998}]}]},{"osd":5,"up_from":39,"seq":167503724763,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2020680,"kb_used_data":1851648,"kb_used_omap":0,"kb_used_meta":169024,"kb_avail":91724472,"statfs":{"total":95995035648,"available":93925859328,"internally_reserved":0,"allocated":1896087552,"data_stored":1895327319,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":173080576},"hb_peers":[0,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:28:29 2023","interfaces":[{"interface":"back","average":{"1min":1.488,"5min":1.843,"15min":1.0129999999999999},"min":{"1min":0.26100000000000001,"5min":0.24099999999999999,"15min":0.23499999999999999},"max":{"1min":6.6479999999999997,"5min":32.475000000000001,"15min":32.475000000000001},"last":0.22900000000000001},{"interface":"front","average":{"1min":1.728,"5min":1.8600000000000001,"15min":1.0309999999999999},"min":{"1min":0.33600000000000002,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":8.4900000000000002,"5min":32.457000000000001,"15min":32.457000000000001},"last":0.60499999999999998}]},{"osd":1,"last update":"Sat Sep 23 20:28:01 2023","interfaces":[{"interface":"back","average":{"1min":1.486,"5min":2.0059999999999998,"15min":1.032},"min":{"1min":0.25600000000000001,"5min":0.245,"15min":0.245},"max":{"1min":11.958,"5min":32.243000000000002,"15min":32.243000000000002},"last":2.0009999999999999},{"interface":"front","average":{"1min":1.5509999999999999,"5min":1.548,"15min":0.88100000000000001},"min":{"1min":0.29599999999999999,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":12.901,"5min":32.231000000000002,"15min":32.231000000000002},"last":8.532}]},{"osd":2,"last update":"Sat Sep 23 20:28:29 2023","interfaces":[{"interface":"back","average":{"1min":1.079,"5min":2.1840000000000002,"15min":1.1160000000000001},"min":{"1min":0.314,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":3.5,"5min":34.514000000000003,"15min":34.514000000000003},"last":0.28000000000000003},{"interface":"front","average":{"1min":2.093,"5min":2.1150000000000002,"15min":1.0940000000000001},"min":{"1min":0.35999999999999999,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":8.6820000000000004,"5min":32.515000000000001,"15min":32.515000000000001},"last":0.874}]},{"osd":3,"last update":"Sat Sep 23 20:28:01 2023","interfaces":[{"interface":"back","average":{"1min":1.4339999999999999,"5min":1.7649999999999999,"15min":0.95999999999999996},"min":{"1min":0.40300000000000002,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":8.391,"5min":12.846,"15min":12.846},"last":0.28799999999999998},{"interface":"front","average":{"1min":1.516,"5min":1.734,"15min":0.98799999999999999},"min":{"1min":0.41699999999999998,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":12.005000000000001,"5min":12.005000000000001,"15min":12.005000000000001},"last":0.38800000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:28:23 2023","interfaces":[{"interface":"back","average":{"1min":0.93000000000000005,"5min":1.925,"15min":0.97399999999999998},"min":{"1min":0.31,"5min":0.26700000000000002,"15min":0.26400000000000001},"max":{"1min":3.726,"5min":14.218999999999999,"15min":14.218999999999999},"last":0.25900000000000001},{"interface":"front","average":{"1min":0.78100000000000003,"5min":1.536,"15min":0.86499999999999999},"min":{"1min":0.33700000000000002,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":1.9670000000000001,"5min":11.664,"15min":11.664},"last":0.34300000000000003}]},{"osd":6,"last update":"Sat Sep 23 20:28:35 2023","interfaces":[{"interface":"back","average":{"1min":1.3720000000000001,"5min":1.9259999999999999,"15min":1.0009999999999999},"min":{"1min":0.32900000000000001,"5min":0.30499999999999999,"15min":0.29799999999999999},"max":{"1min":5.9450000000000003,"5min":12.890000000000001,"15min":12.890000000000001},"last":0.30599999999999999},{"interface":"front","average":{"1min":1.0109999999999999,"5min":1.708,"15min":0.97699999999999998},"min":{"1min":0.36399999999999999,"5min":0.35999999999999999,"15min":0.34499999999999997},"max":{"1min":5.8920000000000003,"5min":11.456,"15min":11.456},"last":0.41499999999999998}]},{"osd":7,"last update":"Sat Sep 23 20:28:01 2023","interfaces":[{"interface":"back","average":{"1min":2.9300000000000002,"5min":2.0459999999999998,"15min":1.0489999999999999},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":36.621000000000002,"5min":36.621000000000002,"15min":36.621000000000002},"last":0.432},{"interface":"front","average":{"1min":2.9990000000000001,"5min":1.675,"15min":0.92900000000000005},"min":{"1min":0.38700000000000001,"5min":0.316,"15min":0.315},"max":{"1min":37.331000000000003,"5min":37.331000000000003,"15min":37.331000000000003},"last":0.56899999999999995}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4722,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1360,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6696,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7156,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3154,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1520959488,"data_stored":1520959488,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1455095808,"data_stored":1455095808,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":3425632256,"data_stored":3425632256,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3911581696,"data_stored":3911581696,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1914503168,"data_stored":1914503168,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1904934912,"data_stored":1904934912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1570045952,"data_stored":1570045952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2550595584,"data_stored":2550595584,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:28:45.576 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd pool set .mgr pgp_num 21 2023-09-23T20:28:45.604 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 83 16 98104 98088 73.8162 61.75 0.0130349 0.0135423 2023-09-23T20:28:45.607 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:28:45.612 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:45.612 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:28:45.713 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:28:45.770 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:45.824 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:28:45.828 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:45 smithi149 ceph-mon[170394]: from='client.35391 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:28:45.829 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:45.830 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:28:45.930 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:28:46.007 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:45 smithi018 ceph-mon[179647]: from='client.35391 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:28:46.014 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:45 smithi018 ceph-mon[184144]: from='client.35391 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:28:46.040 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:46.277 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:28:46.280 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:46.281 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:28:46.381 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:28:46.426 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:46.486 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:28:46.489 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:46.490 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:28:46.590 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:28:46.602 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 84 16 99042 99026 73.6347 58.625 0.0134108 0.0135758 2023-09-23T20:28:46.656 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:46.733 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:28:46.739 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:46.740 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:28:46.841 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:28:46.896 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:46 smithi018 ceph-mon[179647]: pgmap v542: 119 pgs: 2 active+recovery_wait+degraded, 12 active+undersized+degraded, 2 active+recovering+undersized+remapped, 38 active+undersized, 4 active+recovery_wait+degraded+remapped, 61 active+clean; 5.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 42 MiB/s wr, 664 op/s; 31513/282408 objects degraded (11.159%); 17508/282408 objects misplaced (6.200%); 5.8 MiB/s, 92 objects/s recovering 2023-09-23T20:28:46.896 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:46 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 31513/282408 objects degraded (11.159%), 18 pgs degraded (PG_DEGRADED) 2023-09-23T20:28:46.904 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:46 smithi018 ceph-mon[184144]: pgmap v542: 119 pgs: 2 active+recovery_wait+degraded, 12 active+undersized+degraded, 2 active+recovering+undersized+remapped, 38 active+undersized, 4 active+recovery_wait+degraded+remapped, 61 active+clean; 5.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 42 MiB/s wr, 664 op/s; 31513/282408 objects degraded (11.159%); 17508/282408 objects misplaced (6.200%); 5.8 MiB/s, 92 objects/s recovering 2023-09-23T20:28:46.904 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:46 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 31513/282408 objects degraded (11.159%), 18 pgs degraded (PG_DEGRADED) 2023-09-23T20:28:46.941 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:46.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:46 smithi149 ceph-mon[170394]: pgmap v542: 119 pgs: 2 active+recovery_wait+degraded, 12 active+undersized+degraded, 2 active+recovering+undersized+remapped, 38 active+undersized, 4 active+recovery_wait+degraded+remapped, 61 active+clean; 5.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 42 MiB/s wr, 664 op/s; 31513/282408 objects degraded (11.159%); 17508/282408 objects misplaced (6.200%); 5.8 MiB/s, 92 objects/s recovering 2023-09-23T20:28:46.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:46 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 31513/282408 objects degraded (11.159%), 18 pgs degraded (PG_DEGRADED) 2023-09-23T20:28:47.011 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:28:47.015 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:47.016 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:28:47.116 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:28:47.191 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:47.271 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:28:47.351 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:28:47.354 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:47.354 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:28:47.455 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:28:47.526 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:47.602 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 85 16 99936 99920 73.4258 55.875 0.020614 0.0136136 2023-09-23T20:28:47.763 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:28:47.776 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:47.776 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:28:47.818 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2023-09-23T20:28:47.818 INFO:teuthology.orchestra.run.smithi018.stdout: "ops": [], 2023-09-23T20:28:47.819 INFO:teuthology.orchestra.run.smithi018.stdout: "complaint_time": 30, 2023-09-23T20:28:47.819 INFO:teuthology.orchestra.run.smithi018.stdout: "num_blocked_ops": 0 2023-09-23T20:28:47.819 INFO:teuthology.orchestra.run.smithi018.stdout:} 2023-09-23T20:28:47.877 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:28:47.966 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:48.063 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:28:48.105 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:28:48.153 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:28:48.198 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:28:48.248 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:28:48.283 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:28:48.325 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:28:48.326 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:28:48.326 INFO:teuthology.orchestra.run.smithi018.stderr:Error ENXIO: problem getting command descriptions from osd.1 2023-09-23T20:28:48.326 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:28:48.326 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:28:48.326 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:28:48.326 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:28:48.326 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:28:48.326 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:28:48.602 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 86 16 100881 100865 73.2588 59.0625 0.0124727 0.0136457 2023-09-23T20:28:48.738 INFO:teuthology.orchestra.run.smithi018.stderr:nodeep-scrub is unset 2023-09-23T20:28:48.805 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:28:48.808 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:48.808 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:28:48.909 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:28:48.948 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T20:28:48.949 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T20:28:48.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:48 smithi149 ceph-mon[170394]: pgmap v543: 119 pgs: 2 active+recovery_wait+degraded, 12 active+undersized+degraded, 2 active+recovering+undersized+remapped, 38 active+undersized, 4 active+recovery_wait+degraded+remapped, 61 active+clean; 6.0 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 56 MiB/s wr, 903 op/s; 32552/293244 objects degraded (11.101%); 17911/293244 objects misplaced (6.108%); 5.8 MiB/s, 92 objects/s recovering 2023-09-23T20:28:48.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:48 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/812872225' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:28:48.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:48 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2954669043' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:28:48.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:48 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:28:48.995 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:48 smithi018 ceph-mon[184144]: pgmap v543: 119 pgs: 2 active+recovery_wait+degraded, 12 active+undersized+degraded, 2 active+recovering+undersized+remapped, 38 active+undersized, 4 active+recovery_wait+degraded+remapped, 61 active+clean; 6.0 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 56 MiB/s wr, 903 op/s; 32552/293244 objects degraded (11.101%); 17911/293244 objects misplaced (6.108%); 5.8 MiB/s, 92 objects/s recovering 2023-09-23T20:28:48.995 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:48 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/812872225' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:28:48.995 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:48 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2954669043' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:28:48.995 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:48 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:28:48.998 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:48 smithi018 ceph-mon[179647]: pgmap v543: 119 pgs: 2 active+recovery_wait+degraded, 12 active+undersized+degraded, 2 active+recovering+undersized+remapped, 38 active+undersized, 4 active+recovery_wait+degraded+remapped, 61 active+clean; 6.0 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 56 MiB/s wr, 903 op/s; 32552/293244 objects degraded (11.101%); 17911/293244 objects misplaced (6.108%); 5.8 MiB/s, 92 objects/s recovering 2023-09-23T20:28:48.998 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:48 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/812872225' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:28:48.998 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:48 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2954669043' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:28:48.998 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:48 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:28:49.003 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:49.181 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:28:49.185 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:49.186 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:28:49.287 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:28:49.324 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:49.340 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T20:28:49.390 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:28:49.399 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:49.399 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:28:49.500 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:28:49.540 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:49.595 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:28:49.602 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 87 16 101881 101865 73.1351 62.5 0.00766869 0.0136679 2023-09-23T20:28:49.605 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:49.605 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:28:49.706 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:28:49.798 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:49.906 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:28:49.913 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:49.914 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:28:49.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:49 smithi149 ceph-mon[170394]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:28:49.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:49 smithi149 ceph-mon[170394]: failed to encode map e247 with expected crc 2023-09-23T20:28:49.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:49 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:28:49.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:49 smithi149 ceph-mon[170394]: osdmap e247: 8 total, 7 up, 7 in 2023-09-23T20:28:49.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:49 smithi149 ceph-mon[170394]: failed to encode map e247 with expected crc 2023-09-23T20:28:50.015 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:28:50.094 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:49 smithi018 ceph-mon[179647]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:28:50.094 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:49 smithi018 ceph-mon[179647]: failed to encode map e247 with expected crc 2023-09-23T20:28:50.094 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:49 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:28:50.094 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:49 smithi018 ceph-mon[179647]: osdmap e247: 8 total, 7 up, 7 in 2023-09-23T20:28:50.094 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:49 smithi018 ceph-mon[179647]: failed to encode map e247 with expected crc 2023-09-23T20:28:50.094 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:49 smithi018 ceph-mon[184144]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:28:50.094 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:49 smithi018 ceph-mon[184144]: failed to encode map e247 with expected crc 2023-09-23T20:28:50.095 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:49 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:28:50.095 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:49 smithi018 ceph-mon[184144]: osdmap e247: 8 total, 7 up, 7 in 2023-09-23T20:28:50.095 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:49 smithi018 ceph-mon[184144]: failed to encode map e247 with expected crc 2023-09-23T20:28:50.106 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:50.160 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:28:50.169 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:28:50.180 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:50.181 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:28:50.281 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:28:50.333 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:50.602 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 88 16 102852 102836 72.9937 60.6875 0.0138973 0.013695 2023-09-23T20:28:50.769 INFO:teuthology.orchestra.run.smithi018.stderr:set pool 1 pgp_num to 21 2023-09-23T20:28:50.950 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:28:50.999 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:50 smithi018 ceph-mon[184144]: pgmap v545: 119 pgs: 2 active+recovery_wait+degraded, 12 active+undersized+degraded, 2 active+recovering+undersized+remapped, 38 active+undersized, 4 active+recovery_wait+degraded+remapped, 61 active+clean; 6.0 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 59 MiB/s wr, 942 op/s; 32849/295998 objects degraded (11.098%); 17722/295998 objects misplaced (5.987%); 5.4 MiB/s, 86 objects/s recovering 2023-09-23T20:28:51.000 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:50 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2321753454' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "21"}]: dispatch 2023-09-23T20:28:51.000 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:50 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "21"}]: dispatch 2023-09-23T20:28:51.003 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:50 smithi018 ceph-mon[179647]: pgmap v545: 119 pgs: 2 active+recovery_wait+degraded, 12 active+undersized+degraded, 2 active+recovering+undersized+remapped, 38 active+undersized, 4 active+recovery_wait+degraded+remapped, 61 active+clean; 6.0 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 59 MiB/s wr, 942 op/s; 32849/295998 objects degraded (11.098%); 17722/295998 objects misplaced (5.987%); 5.4 MiB/s, 86 objects/s recovering 2023-09-23T20:28:51.003 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:50 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2321753454' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "21"}]: dispatch 2023-09-23T20:28:51.003 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:50 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "21"}]: dispatch 2023-09-23T20:28:51.171 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T20:28:51.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:50 smithi149 ceph-mon[170394]: pgmap v545: 119 pgs: 2 active+recovery_wait+degraded, 12 active+undersized+degraded, 2 active+recovering+undersized+remapped, 38 active+undersized, 4 active+recovery_wait+degraded+remapped, 61 active+clean; 6.0 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 59 MiB/s wr, 942 op/s; 32849/295998 objects degraded (11.098%); 17722/295998 objects misplaced (5.987%); 5.4 MiB/s, 86 objects/s recovering 2023-09-23T20:28:51.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:50 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2321753454' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "21"}]: dispatch 2023-09-23T20:28:51.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:50 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "21"}]: dispatch 2023-09-23T20:28:51.513 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:28:51.523 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:51.523 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:28:51.602 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 89 16 103780 103764 72.8253 58 0.00332705 0.0137188 2023-09-23T20:28:51.624 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:28:51.649 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:51.683 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:28:51.686 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:51.687 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:28:51.787 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:28:51.853 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:51.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:51 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "21"}]': finished 2023-09-23T20:28:51.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:51 smithi018 ceph-mon[184144]: osdmap e248: 8 total, 7 up, 7 in 2023-09-23T20:28:51.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:51 smithi018 ceph-mon[184144]: failed to encode map e248 with expected crc 2023-09-23T20:28:51.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:51 smithi018 ceph-mon[184144]: failed to encode map e248 with expected crc 2023-09-23T20:28:51.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:51 smithi018 ceph-mon[184144]: failed to encode map e248 with expected crc 2023-09-23T20:28:51.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:51 smithi018 ceph-mon[184144]: failed to encode map e248 with expected crc 2023-09-23T20:28:51.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:51 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 32849/295998 objects degraded (11.098%), 18 pgs degraded (PG_DEGRADED) 2023-09-23T20:28:51.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:51 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "21"}]': finished 2023-09-23T20:28:51.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:51 smithi018 ceph-mon[179647]: osdmap e248: 8 total, 7 up, 7 in 2023-09-23T20:28:51.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:51 smithi018 ceph-mon[179647]: failed to encode map e248 with expected crc 2023-09-23T20:28:51.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:51 smithi018 ceph-mon[179647]: failed to encode map e248 with expected crc 2023-09-23T20:28:51.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:51 smithi018 ceph-mon[179647]: failed to encode map e248 with expected crc 2023-09-23T20:28:51.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:51 smithi018 ceph-mon[179647]: failed to encode map e248 with expected crc 2023-09-23T20:28:51.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:51 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 32849/295998 objects degraded (11.098%), 18 pgs degraded (PG_DEGRADED) 2023-09-23T20:28:51.924 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:28:51.931 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:51.932 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:28:51.932 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:51 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "21"}]': finished 2023-09-23T20:28:51.933 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:51 smithi149 ceph-mon[170394]: osdmap e248: 8 total, 7 up, 7 in 2023-09-23T20:28:51.933 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:51 smithi149 ceph-mon[170394]: failed to encode map e248 with expected crc 2023-09-23T20:28:51.933 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:51 smithi149 ceph-mon[170394]: failed to encode map e248 with expected crc 2023-09-23T20:28:51.933 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:51 smithi149 ceph-mon[170394]: failed to encode map e248 with expected crc 2023-09-23T20:28:51.933 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:51 smithi149 ceph-mon[170394]: failed to encode map e248 with expected crc 2023-09-23T20:28:51.933 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:51 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 32849/295998 objects degraded (11.098%), 18 pgs degraded (PG_DEGRADED) 2023-09-23T20:28:52.033 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:28:52.094 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:52.603 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Total time run: 90.0193 2023-09-23T20:28:52.603 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Total writes made: 104736 2023-09-23T20:28:52.603 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Write size: 65536 2023-09-23T20:28:52.603 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Object size: 65536 2023-09-23T20:28:52.603 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Bandwidth (MB/sec): 72.7177 2023-09-23T20:28:52.603 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Stddev Bandwidth: 39.4246 2023-09-23T20:28:52.603 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Max bandwidth (MB/sec): 199.75 2023-09-23T20:28:52.603 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Min bandwidth (MB/sec): 0 2023-09-23T20:28:52.603 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Average IOPS: 1163 2023-09-23T20:28:52.603 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Stddev IOPS: 630.803 2023-09-23T20:28:52.603 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Max IOPS: 3196 2023-09-23T20:28:52.604 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Min IOPS: 0 2023-09-23T20:28:52.604 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Average Latency(s): 0.0137465 2023-09-23T20:28:52.604 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Stddev Latency(s): 0.0429018 2023-09-23T20:28:52.604 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Max latency(s): 2.50015 2023-09-23T20:28:52.604 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Min latency(s): 0.00113549 2023-09-23T20:28:52.607 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Cleaning up (deleting benchmark objects) 2023-09-23T20:28:53.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:53 smithi018 ceph-mon[179647]: pgmap v547: 119 pgs: 2 active+recovery_wait+degraded, 12 active+undersized+degraded, 2 active+recovering+undersized+remapped, 38 active+undersized, 4 active+recovery_wait+degraded+remapped, 61 active+clean; 6.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 66 MiB/s wr, 1.06k op/s; 33803/305919 objects degraded (11.050%); 17812/305919 objects misplaced (5.822%); 4.8 MiB/s, 76 objects/s recovering 2023-09-23T20:28:53.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:53 smithi018 ceph-mon[184144]: pgmap v547: 119 pgs: 2 active+recovery_wait+degraded, 12 active+undersized+degraded, 2 active+recovering+undersized+remapped, 38 active+undersized, 4 active+recovery_wait+degraded+remapped, 61 active+clean; 6.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 66 MiB/s wr, 1.06k op/s; 33803/305919 objects degraded (11.050%); 17812/305919 objects misplaced (5.822%); 4.8 MiB/s, 76 objects/s recovering 2023-09-23T20:28:53.395 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:28:53.406 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:53.407 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:28:53.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:53 smithi149 ceph-mon[170394]: pgmap v547: 119 pgs: 2 active+recovery_wait+degraded, 12 active+undersized+degraded, 2 active+recovering+undersized+remapped, 38 active+undersized, 4 active+recovery_wait+degraded+remapped, 61 active+clean; 6.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 66 MiB/s wr, 1.06k op/s; 33803/305919 objects degraded (11.050%); 17812/305919 objects misplaced (5.822%); 4.8 MiB/s, 76 objects/s recovering 2023-09-23T20:28:53.507 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:28:53.570 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:53.625 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:28:53.631 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:53.631 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:28:53.732 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:28:53.800 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:54.033 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:54 smithi018 ceph-mon[179647]: 1.5 scrub starts 2023-09-23T20:28:54.252 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:54.252 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:28:54.252 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:28:54.267 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:28:54.324 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:54 smithi018 ceph-mon[184144]: 1.5 scrub starts 2023-09-23T20:28:54.353 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:28:54.427 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:54.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:54 smithi149 ceph-mon[170394]: 1.5 scrub starts 2023-09-23T20:28:54.487 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:28:54.513 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:28:54.521 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:54.522 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:28:54.623 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:28:54.708 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:55.060 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:28:55.073 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:55.074 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:28:55.174 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:28:55.233 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:55.262 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:28:54 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:28:54] "GET /metrics HTTP/1.1" 200 36548 "" "Prometheus/2.43.0" 2023-09-23T20:28:55.263 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:55 smithi018 ceph-mon[184144]: pgmap v548: 119 pgs: 2 active+recovery_wait+degraded, 12 active+undersized+degraded, 2 active+recovering+undersized+remapped, 38 active+undersized, 4 active+recovery_wait+degraded+remapped, 61 active+clean; 6.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 72 MiB/s wr, 1.16k op/s; 34042/309819 objects degraded (10.988%); 17859/309819 objects misplaced (5.764%); 6.5 MiB/s, 103 objects/s recovering 2023-09-23T20:28:55.264 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:55 smithi018 ceph-mon[179647]: pgmap v548: 119 pgs: 2 active+recovery_wait+degraded, 12 active+undersized+degraded, 2 active+recovering+undersized+remapped, 38 active+undersized, 4 active+recovery_wait+degraded+remapped, 61 active+clean; 6.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 72 MiB/s wr, 1.16k op/s; 34042/309819 objects degraded (10.988%); 17859/309819 objects misplaced (5.764%); 6.5 MiB/s, 103 objects/s recovering 2023-09-23T20:28:55.282 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:28:55.285 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:55.286 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:28:55.286 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:55 smithi149 ceph-mon[170394]: pgmap v548: 119 pgs: 2 active+recovery_wait+degraded, 12 active+undersized+degraded, 2 active+recovering+undersized+remapped, 38 active+undersized, 4 active+recovery_wait+degraded+remapped, 61 active+clean; 6.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 72 MiB/s wr, 1.16k op/s; 34042/309819 objects degraded (10.988%); 17859/309819 objects misplaced (5.764%); 6.5 MiB/s, 103 objects/s recovering 2023-09-23T20:28:55.334 INFO:teuthology.orchestra.run.smithi018.stdout:ERROR: (22) Invalid argument 2023-09-23T20:28:55.335 INFO:teuthology.orchestra.run.smithi018.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:28:55.387 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:28:55.438 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:55.496 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:28:55.501 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:55.501 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:28:55.602 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:28:55.678 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:56.002 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:28:56.008 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:56.009 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:28:56.046 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T20:28:56.110 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:28:56.167 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:56.263 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:28:56.270 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:56.270 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:28:56.371 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:28:56.377 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:56 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 34042/309819 objects degraded (10.988%), 18 pgs degraded (PG_DEGRADED) 2023-09-23T20:28:56.377 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:56 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 34042/309819 objects degraded (10.988%), 18 pgs degraded (PG_DEGRADED) 2023-09-23T20:28:56.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:56 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 34042/309819 objects degraded (10.988%), 18 pgs degraded (PG_DEGRADED) 2023-09-23T20:28:56.477 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:56.647 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:28:56.652 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:56.652 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:28:56.753 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:28:56.807 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:56.923 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:28:56.926 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:56.927 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:28:56.973 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:28:57.028 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:28:57.064 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:57 smithi018 ceph-mon[179647]: pgmap v549: 119 pgs: 2 active+recovery_wait+degraded, 12 active+undersized+degraded, 2 active+recovering+undersized+remapped, 38 active+undersized, 4 active+recovery_wait+degraded+remapped, 61 active+clean; 6.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 46 MiB/s wr, 789 op/s; 34042/309852 objects degraded (10.987%); 17859/309852 objects misplaced (5.764%); 4.8 MiB/s, 77 objects/s recovering 2023-09-23T20:28:57.071 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:57.105 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:28:57.129 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:28:57.131 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:57.132 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:28:57.148 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:28:57.179 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:28:57.213 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:28:57.232 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:28:57.264 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:28:57.295 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:57.306 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:28:57.321 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:57 smithi149 ceph-mon[170394]: pgmap v549: 119 pgs: 2 active+recovery_wait+degraded, 12 active+undersized+degraded, 2 active+recovering+undersized+remapped, 38 active+undersized, 4 active+recovery_wait+degraded+remapped, 61 active+clean; 6.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 46 MiB/s wr, 789 op/s; 34042/309852 objects degraded (10.987%); 17859/309852 objects misplaced (5.764%); 4.8 MiB/s, 77 objects/s recovering 2023-09-23T20:28:57.336 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:28:57.336 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:28:57.337 INFO:teuthology.orchestra.run.smithi018.stderr:Error ENXIO: problem getting command descriptions from osd.1 2023-09-23T20:28:57.337 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:28:57.337 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:28:57.337 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:28:57.337 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:28:57.337 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:28:57.337 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:28:57.355 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:57 smithi018 ceph-mon[184144]: pgmap v549: 119 pgs: 2 active+recovery_wait+degraded, 12 active+undersized+degraded, 2 active+recovering+undersized+remapped, 38 active+undersized, 4 active+recovery_wait+degraded+remapped, 61 active+clean; 6.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 46 MiB/s wr, 789 op/s; 34042/309852 objects degraded (10.987%); 17859/309852 objects misplaced (5.764%); 4.8 MiB/s, 77 objects/s recovering 2023-09-23T20:28:57.382 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:28:57.392 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:57.392 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:28:57.493 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:28:57.554 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:57.608 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:28:57.612 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:57.612 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:28:57.713 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:28:57.771 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:57.794 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T20:28:57.795 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T20:28:57.829 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:28:57.834 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:57.835 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:28:57.935 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:28:57.991 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:58.051 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:28:58.059 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:58.060 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:28:58.071 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:58 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2311527900' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:28:58.071 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:58 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/4213916897' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:28:58.071 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:58 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:28:58.072 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:58 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2311527900' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:28:58.072 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:58 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/4213916897' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:28:58.072 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:58 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:28:58.098 INFO:teuthology.orchestra.run.smithi018.stderr:noscrub is set 2023-09-23T20:28:58.160 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:28:58.224 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:58.289 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:28:58.293 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:58.294 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:28:58.294 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:58 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2311527900' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:28:58.295 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:58 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/4213916897' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:28:58.295 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:58 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:28:58.394 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:28:58.453 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:58.468 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 4, 5, 7, 3, 2] out_osds: [6] dead_osds: [1] live_osds: [0, 2, 3, 4, 5, 6, 7] 2023-09-23T20:28:58.469 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:28:58.469 INFO:tasks.thrashosds.thrasher:Adding osd 6 2023-09-23T20:28:58.469 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd in 6 2023-09-23T20:28:58.531 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:28:58.539 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:58.539 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:28:58.640 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:28:58.706 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:58.785 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:28:58.788 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:58.789 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:28:58.889 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:28:58.897 INFO:teuthology.orchestra.run.smithi149.stdout:{ 2023-09-23T20:28:58.897 INFO:teuthology.orchestra.run.smithi149.stdout: "ops": [ 2023-09-23T20:28:58.897 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:28:58.897 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_repop(client.45161.0:111298 6.f e249/217 6:f02b4e47:::benchmark_data_smithi018_270558_object6559:head v 249'7073, mlcod=249'7069)", 2023-09-23T20:28:58.897 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:28:58.895989+0000", 2023-09-23T20:28:58.897 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 0.0011240989999999999, 2023-09-23T20:28:58.898 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.001145292, 2023-09-23T20:28:58.898 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:28:58.898 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "started", 2023-09-23T20:28:58.898 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:28:58.898 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:28:58.898 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:28:58.898 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:58.895989+0000", 2023-09-23T20:28:58.898 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:28:58.898 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:28:58.898 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:28:58.898 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:28:58.898 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:58.895989+0000", 2023-09-23T20:28:58.899 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:28:58.899 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:28:58.899 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:28:58.899 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:28:58.899 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:58.895988+0000", 2023-09-23T20:28:58.899 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:28:58.899 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:28:58.899 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:28:58.899 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:28:58.899 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:58.895999+0000", 2023-09-23T20:28:58.899 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0763e-05 2023-09-23T20:28:58.899 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:28:58.900 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:28:58.900 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:28:58.900 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:58.896016+0000", 2023-09-23T20:28:58.900 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.7204999999999999e-05 2023-09-23T20:28:58.900 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:28:58.900 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:28:58.900 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:28:58.900 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:58.896023+0000", 2023-09-23T20:28:58.900 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.0790000000000003e-06 2023-09-23T20:28:58.900 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:28:58.900 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:28:58.900 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:28:58.901 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:58.896072+0000", 2023-09-23T20:28:58.901 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.901e-05 2023-09-23T20:28:58.901 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:28:58.901 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:28:58.901 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:28:58.901 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:58.896097+0000", 2023-09-23T20:28:58.901 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.5486999999999998e-05 2023-09-23T20:28:58.901 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:28:58.901 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:28:58.901 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:28:58.901 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:28:58.901 INFO:teuthology.orchestra.run.smithi149.stdout: ], 2023-09-23T20:28:58.902 INFO:teuthology.orchestra.run.smithi149.stdout: "num_ops": 1 2023-09-23T20:28:58.902 INFO:teuthology.orchestra.run.smithi149.stdout:} 2023-09-23T20:28:58.944 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:59.261 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:28:59.265 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:28:59.265 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:28:59.330 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:59 smithi149 ceph-mon[170394]: pgmap v550: 119 pgs: 2 active+recovery_wait+degraded, 12 active+undersized+degraded, 2 active+recovering+undersized+remapped, 38 active+undersized, 4 active+recovery_wait+degraded+remapped, 61 active+clean; 6.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 115 B/s rd, 43 MiB/s wr, 1.00k op/s; 33556/305871 objects degraded (10.971%); 17342/305871 objects misplaced (5.670%); 6.1 MiB/s, 97 objects/s recovering 2023-09-23T20:28:59.330 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:59 smithi149 ceph-mon[170394]: 6.2 scrub starts 2023-09-23T20:28:59.330 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:59 smithi149 ceph-mon[170394]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:28:59.330 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:59 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:28:59.331 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:59 smithi149 ceph-mon[170394]: osdmap e249: 8 total, 7 up, 7 in 2023-09-23T20:28:59.331 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:59 smithi149 ceph-mon[170394]: failed to encode map e249 with expected crc 2023-09-23T20:28:59.331 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:59 smithi149 ceph-mon[170394]: failed to encode map e249 with expected crc 2023-09-23T20:28:59.331 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:59 smithi149 ceph-mon[170394]: failed to encode map e249 with expected crc 2023-09-23T20:28:59.331 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:28:59 smithi149 ceph-mon[170394]: failed to encode map e249 with expected crc 2023-09-23T20:28:59.333 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:28:59.337 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T20:28:59.366 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:28:59.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:59 smithi018 ceph-mon[179647]: pgmap v550: 119 pgs: 2 active+recovery_wait+degraded, 12 active+undersized+degraded, 2 active+recovering+undersized+remapped, 38 active+undersized, 4 active+recovery_wait+degraded+remapped, 61 active+clean; 6.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 115 B/s rd, 43 MiB/s wr, 1.00k op/s; 33556/305871 objects degraded (10.971%); 17342/305871 objects misplaced (5.670%); 6.1 MiB/s, 97 objects/s recovering 2023-09-23T20:28:59.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:59 smithi018 ceph-mon[179647]: 6.2 scrub starts 2023-09-23T20:28:59.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:59 smithi018 ceph-mon[179647]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:28:59.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:59 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:28:59.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:59 smithi018 ceph-mon[179647]: osdmap e249: 8 total, 7 up, 7 in 2023-09-23T20:28:59.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:59 smithi018 ceph-mon[179647]: failed to encode map e249 with expected crc 2023-09-23T20:28:59.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:59 smithi018 ceph-mon[179647]: failed to encode map e249 with expected crc 2023-09-23T20:28:59.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:59 smithi018 ceph-mon[179647]: failed to encode map e249 with expected crc 2023-09-23T20:28:59.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:28:59 smithi018 ceph-mon[179647]: failed to encode map e249 with expected crc 2023-09-23T20:28:59.396 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:59 smithi018 ceph-mon[184144]: pgmap v550: 119 pgs: 2 active+recovery_wait+degraded, 12 active+undersized+degraded, 2 active+recovering+undersized+remapped, 38 active+undersized, 4 active+recovery_wait+degraded+remapped, 61 active+clean; 6.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 115 B/s rd, 43 MiB/s wr, 1.00k op/s; 33556/305871 objects degraded (10.971%); 17342/305871 objects misplaced (5.670%); 6.1 MiB/s, 97 objects/s recovering 2023-09-23T20:28:59.396 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:59 smithi018 ceph-mon[184144]: 6.2 scrub starts 2023-09-23T20:28:59.397 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:59 smithi018 ceph-mon[184144]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:28:59.397 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:59 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:28:59.397 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:59 smithi018 ceph-mon[184144]: osdmap e249: 8 total, 7 up, 7 in 2023-09-23T20:28:59.397 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:59 smithi018 ceph-mon[184144]: failed to encode map e249 with expected crc 2023-09-23T20:28:59.397 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:59 smithi018 ceph-mon[184144]: failed to encode map e249 with expected crc 2023-09-23T20:28:59.397 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:59 smithi018 ceph-mon[184144]: failed to encode map e249 with expected crc 2023-09-23T20:28:59.397 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:28:59 smithi018 ceph-mon[184144]: failed to encode map e249 with expected crc 2023-09-23T20:28:59.435 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:28:59.796 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:29:00.106 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:29:00.144 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:00 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:29:00.145 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:00 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:29:00.385 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:29:00.386 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:00 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:29:00.393 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:00.393 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:29:00.494 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:29:00.526 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T20:29:00.566 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:00.652 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:29:00.656 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:00.657 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:29:00.758 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:29:00.889 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:01.143 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:29:01.150 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:01.151 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:29:01.252 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:29:01.292 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:29:01.369 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:01 smithi018 ceph-mon[179647]: pgmap v552: 119 pgs: 2 active+recovery_wait+degraded, 12 active+undersized+degraded, 2 active+recovering+undersized+remapped, 38 active+undersized, 4 active+recovery_wait+degraded+remapped, 61 active+clean; 6.2 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 115 B/s rd, 20 MiB/s wr, 739 op/s; 33253/302970 objects degraded (10.976%); 17130/302970 objects misplaced (5.654%); 6.1 MiB/s, 97 objects/s recovering 2023-09-23T20:29:01.369 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:01 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 33253/302970 objects degraded (10.976%), 18 pgs degraded (PG_DEGRADED) 2023-09-23T20:29:01.370 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:01 smithi018 ceph-mon[184144]: pgmap v552: 119 pgs: 2 active+recovery_wait+degraded, 12 active+undersized+degraded, 2 active+recovering+undersized+remapped, 38 active+undersized, 4 active+recovery_wait+degraded+remapped, 61 active+clean; 6.2 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 115 B/s rd, 20 MiB/s wr, 739 op/s; 33253/302970 objects degraded (10.976%); 17130/302970 objects misplaced (5.654%); 6.1 MiB/s, 97 objects/s recovering 2023-09-23T20:29:01.370 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:01 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 33253/302970 objects degraded (10.976%), 18 pgs degraded (PG_DEGRADED) 2023-09-23T20:29:01.372 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:01.452 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:29:01.461 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:01.461 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:29:01.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:01 smithi149 ceph-mon[170394]: pgmap v552: 119 pgs: 2 active+recovery_wait+degraded, 12 active+undersized+degraded, 2 active+recovering+undersized+remapped, 38 active+undersized, 4 active+recovery_wait+degraded+remapped, 61 active+clean; 6.2 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 115 B/s rd, 20 MiB/s wr, 739 op/s; 33253/302970 objects degraded (10.976%); 17130/302970 objects misplaced (5.654%); 6.1 MiB/s, 97 objects/s recovering 2023-09-23T20:29:01.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:01 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 33253/302970 objects degraded (10.976%), 18 pgs degraded (PG_DEGRADED) 2023-09-23T20:29:01.562 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:29:01.622 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:02.201 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:29:02.633 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:02 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mon-a[179623]: 2023-09-23T20:29:02.412+0000 7f160fef9700 -1 mon.a@0(leader).osd e249 definitely_dead 0 2023-09-23T20:29:03.014 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:29:03.019 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:03.020 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:29:03.117 INFO:teuthology.orchestra.run.smithi018.stderr:marked in osd.6. 2023-09-23T20:29:03.121 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:29:03.180 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:03.370 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:03 smithi018 ceph-mon[184144]: pgmap v553: 119 pgs: 2 active+recovery_wait+degraded, 12 active+undersized+degraded, 2 active+recovering+undersized+remapped, 38 active+undersized, 4 active+recovery_wait+degraded+remapped, 61 active+clean; 5.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 102 B/s rd, 17 MiB/s wr, 1.02k op/s; 31995/292152 objects degraded (10.951%); 16574/292152 objects misplaced (5.673%); 8.9 MiB/s, 142 objects/s recovering 2023-09-23T20:29:03.370 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:03 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/896472025' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-09-23T20:29:03.370 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:03 smithi018 ceph-mon[179647]: pgmap v553: 119 pgs: 2 active+recovery_wait+degraded, 12 active+undersized+degraded, 2 active+recovering+undersized+remapped, 38 active+undersized, 4 active+recovery_wait+degraded+remapped, 61 active+clean; 5.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 102 B/s rd, 17 MiB/s wr, 1.02k op/s; 31995/292152 objects degraded (10.951%); 16574/292152 objects misplaced (5.673%); 8.9 MiB/s, 142 objects/s recovering 2023-09-23T20:29:03.370 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:03 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/896472025' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-09-23T20:29:03.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:03 smithi149 ceph-mon[170394]: pgmap v553: 119 pgs: 2 active+recovery_wait+degraded, 12 active+undersized+degraded, 2 active+recovering+undersized+remapped, 38 active+undersized, 4 active+recovery_wait+degraded+remapped, 61 active+clean; 5.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 102 B/s rd, 17 MiB/s wr, 1.02k op/s; 31995/292152 objects degraded (10.951%); 16574/292152 objects misplaced (5.673%); 8.9 MiB/s, 142 objects/s recovering 2023-09-23T20:29:03.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:03 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/896472025' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-09-23T20:29:03.533 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:29:03.533 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:29:03.534 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:29:03.534 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:29:03.534 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:29:03.534 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:29:03.534 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:29:03.534 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:29:03.534 INFO:teuthology.orchestra.run.smithi018.stderr:Error ENXIO: problem getting command descriptions from osd.1 2023-09-23T20:29:03.534 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:29:03.534 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:29:03.535 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:29:03.535 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:29:03.535 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:29:03.535 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:29:03.717 INFO:tasks.thrashosds.thrasher:Added osd 6 2023-09-23T20:29:03.975 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:29:03.978 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:03.978 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:29:04.079 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:29:04.127 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:04.169 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T20:29:04.170 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T20:29:04.176 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:29:04.180 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:04.180 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:29:04.281 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:29:04.311 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:04.357 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:29:04.360 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:04.360 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:29:04.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:04 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/896472025' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2023-09-23T20:29:04.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:04 smithi018 ceph-mon[184144]: osdmap e250: 8 total, 7 up, 8 in 2023-09-23T20:29:04.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:04 smithi018 ceph-mon[184144]: failed to encode map e250 with expected crc 2023-09-23T20:29:04.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:04 smithi018 ceph-mon[184144]: failed to encode map e250 with expected crc 2023-09-23T20:29:04.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:04 smithi018 ceph-mon[184144]: failed to encode map e250 with expected crc 2023-09-23T20:29:04.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:04 smithi018 ceph-mon[184144]: failed to encode map e250 with expected crc 2023-09-23T20:29:04.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:04 smithi018 ceph-mon[184144]: failed to encode map e250 with expected crc 2023-09-23T20:29:04.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:04 smithi018 ceph-mon[184144]: failed to encode map e250 with expected crc 2023-09-23T20:29:04.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:04 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/125975539' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:29:04.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:04 smithi018 ceph-mon[184144]: osdmap e251: 8 total, 7 up, 8 in 2023-09-23T20:29:04.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:04 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/896472025' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2023-09-23T20:29:04.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:04 smithi018 ceph-mon[179647]: osdmap e250: 8 total, 7 up, 8 in 2023-09-23T20:29:04.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:04 smithi018 ceph-mon[179647]: failed to encode map e250 with expected crc 2023-09-23T20:29:04.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:04 smithi018 ceph-mon[179647]: failed to encode map e250 with expected crc 2023-09-23T20:29:04.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:04 smithi018 ceph-mon[179647]: failed to encode map e250 with expected crc 2023-09-23T20:29:04.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:04 smithi018 ceph-mon[179647]: failed to encode map e250 with expected crc 2023-09-23T20:29:04.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:04 smithi018 ceph-mon[179647]: failed to encode map e250 with expected crc 2023-09-23T20:29:04.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:04 smithi018 ceph-mon[179647]: failed to encode map e250 with expected crc 2023-09-23T20:29:04.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:04 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/125975539' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:29:04.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:04 smithi018 ceph-mon[179647]: osdmap e251: 8 total, 7 up, 8 in 2023-09-23T20:29:04.461 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:29:04.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:04 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/896472025' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2023-09-23T20:29:04.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:04 smithi149 ceph-mon[170394]: osdmap e250: 8 total, 7 up, 8 in 2023-09-23T20:29:04.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:04 smithi149 ceph-mon[170394]: failed to encode map e250 with expected crc 2023-09-23T20:29:04.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:04 smithi149 ceph-mon[170394]: failed to encode map e250 with expected crc 2023-09-23T20:29:04.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:04 smithi149 ceph-mon[170394]: failed to encode map e250 with expected crc 2023-09-23T20:29:04.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:04 smithi149 ceph-mon[170394]: failed to encode map e250 with expected crc 2023-09-23T20:29:04.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:04 smithi149 ceph-mon[170394]: failed to encode map e250 with expected crc 2023-09-23T20:29:04.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:04 smithi149 ceph-mon[170394]: failed to encode map e250 with expected crc 2023-09-23T20:29:04.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:04 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/125975539' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:29:04.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:04 smithi149 ceph-mon[170394]: osdmap e251: 8 total, 7 up, 8 in 2023-09-23T20:29:04.492 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:04.782 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:29:04.785 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:04.785 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:29:04.886 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:29:04.916 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:05.002 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:29:05.005 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:05.005 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:29:05.106 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:29:05.114 INFO:teuthology.orchestra.run.smithi018.stderr:nodeep-scrub is set 2023-09-23T20:29:05.153 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:05 smithi149 ceph-mon[170394]: pgmap v555: 119 pgs: 1 remapped+peering, 5 peering, 2 active+recovery_wait+degraded, 12 active+undersized+degraded, 1 active+recovering+undersized+remapped, 36 active+undersized, 4 active+recovery_wait+degraded+remapped, 58 active+clean; 5.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 127 B/s rd, 8.5 MiB/s wr, 1.20k op/s; 31599/287628 objects degraded (10.986%); 13587/287628 objects misplaced (4.724%); 12 MiB/s, 192 objects/s recovering 2023-09-23T20:29:05.153 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:05 smithi149 ceph-mon[170394]: failed to encode map e251 with expected crc 2023-09-23T20:29:05.153 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:05 smithi149 ceph-mon[170394]: failed to encode map e251 with expected crc 2023-09-23T20:29:05.154 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:05 smithi149 ceph-mon[170394]: failed to encode map e251 with expected crc 2023-09-23T20:29:05.154 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:05 smithi149 ceph-mon[170394]: failed to encode map e251 with expected crc 2023-09-23T20:29:05.154 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:05 smithi149 ceph-mon[170394]: failed to encode map e251 with expected crc 2023-09-23T20:29:05.154 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:05 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/4128778264' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:29:05.154 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:05 smithi149 ceph-mon[170394]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:29:05.154 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:05 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/4128778264' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:29:05.154 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:05 smithi149 ceph-mon[170394]: osdmap e252: 8 total, 7 up, 8 in 2023-09-23T20:29:05.227 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:05.284 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:29:05.291 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:05.291 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:29:05.354 INFO:teuthology.orchestra.run.smithi149.stdout:ERROR: (22) Invalid argument 2023-09-23T20:29:05.354 INFO:teuthology.orchestra.run.smithi149.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:29:05.391 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:29:04 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:29:04] "GET /metrics HTTP/1.1" 200 36554 "" "Prometheus/2.43.0" 2023-09-23T20:29:05.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:05 smithi018 ceph-mon[184144]: pgmap v555: 119 pgs: 1 remapped+peering, 5 peering, 2 active+recovery_wait+degraded, 12 active+undersized+degraded, 1 active+recovering+undersized+remapped, 36 active+undersized, 4 active+recovery_wait+degraded+remapped, 58 active+clean; 5.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 127 B/s rd, 8.5 MiB/s wr, 1.20k op/s; 31599/287628 objects degraded (10.986%); 13587/287628 objects misplaced (4.724%); 12 MiB/s, 192 objects/s recovering 2023-09-23T20:29:05.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:05 smithi018 ceph-mon[184144]: failed to encode map e251 with expected crc 2023-09-23T20:29:05.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:05 smithi018 ceph-mon[184144]: failed to encode map e251 with expected crc 2023-09-23T20:29:05.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:05 smithi018 ceph-mon[184144]: failed to encode map e251 with expected crc 2023-09-23T20:29:05.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:05 smithi018 ceph-mon[184144]: failed to encode map e251 with expected crc 2023-09-23T20:29:05.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:05 smithi018 ceph-mon[184144]: failed to encode map e251 with expected crc 2023-09-23T20:29:05.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:05 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/4128778264' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:29:05.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:05 smithi018 ceph-mon[184144]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:29:05.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:05 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/4128778264' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:29:05.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:05 smithi018 ceph-mon[184144]: osdmap e252: 8 total, 7 up, 8 in 2023-09-23T20:29:05.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:05 smithi018 ceph-mon[179647]: pgmap v555: 119 pgs: 1 remapped+peering, 5 peering, 2 active+recovery_wait+degraded, 12 active+undersized+degraded, 1 active+recovering+undersized+remapped, 36 active+undersized, 4 active+recovery_wait+degraded+remapped, 58 active+clean; 5.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 127 B/s rd, 8.5 MiB/s wr, 1.20k op/s; 31599/287628 objects degraded (10.986%); 13587/287628 objects misplaced (4.724%); 12 MiB/s, 192 objects/s recovering 2023-09-23T20:29:05.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:05 smithi018 ceph-mon[179647]: failed to encode map e251 with expected crc 2023-09-23T20:29:05.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:05 smithi018 ceph-mon[179647]: failed to encode map e251 with expected crc 2023-09-23T20:29:05.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:05 smithi018 ceph-mon[179647]: failed to encode map e251 with expected crc 2023-09-23T20:29:05.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:05 smithi018 ceph-mon[179647]: failed to encode map e251 with expected crc 2023-09-23T20:29:05.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:05 smithi018 ceph-mon[179647]: failed to encode map e251 with expected crc 2023-09-23T20:29:05.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:05 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/4128778264' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:29:05.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:05 smithi018 ceph-mon[179647]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:29:05.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:05 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/4128778264' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:29:05.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:05 smithi018 ceph-mon[179647]: osdmap e252: 8 total, 7 up, 8 in 2023-09-23T20:29:05.393 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:29:05.466 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:05.510 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:29:05.514 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:05.515 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:29:05.615 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:29:05.686 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:05.779 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:29:05.783 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T20:29:05.784 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:05.784 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:29:05.885 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:29:06.005 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:06.120 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:29:06.124 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:06.125 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:29:06.171 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:29:06.225 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:29:06.266 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:06.332 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:29:06.340 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:06.341 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:29:06.442 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:29:06.473 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:06 smithi149 ceph-mon[170394]: failed to encode map e252 with expected crc 2023-09-23T20:29:06.474 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:06 smithi149 ceph-mon[170394]: failed to encode map e252 with expected crc 2023-09-23T20:29:06.474 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:06 smithi149 ceph-mon[170394]: failed to encode map e252 with expected crc 2023-09-23T20:29:06.474 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:06 smithi149 ceph-mon[170394]: failed to encode map e252 with expected crc 2023-09-23T20:29:06.474 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:06 smithi149 ceph-mon[170394]: failed to encode map e252 with expected crc 2023-09-23T20:29:06.474 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:06 smithi149 ceph-mon[170394]: failed to encode map e252 with expected crc 2023-09-23T20:29:06.474 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:06 smithi149 ceph-mon[170394]: failed to encode map e252 with expected crc 2023-09-23T20:29:06.474 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:06 smithi149 ceph-mon[170394]: failed to encode map e252 with expected crc 2023-09-23T20:29:06.474 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:06 smithi149 ceph-mon[170394]: failed to encode map e252 with expected crc 2023-09-23T20:29:06.474 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:06 smithi149 ceph-mon[170394]: failed to encode map e252 with expected crc 2023-09-23T20:29:06.474 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:06 smithi149 ceph-mon[170394]: failed to encode map e252 with expected crc 2023-09-23T20:29:06.474 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:06 smithi149 ceph-mon[170394]: failed to encode map e252 with expected crc 2023-09-23T20:29:06.475 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:06 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 31599/287628 objects degraded (10.986%), 18 pgs degraded (PG_DEGRADED) 2023-09-23T20:29:06.475 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:06 smithi149 ceph-mon[170394]: osdmap e253: 8 total, 7 up, 8 in 2023-09-23T20:29:06.493 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:06 smithi018 ceph-mon[179647]: failed to encode map e252 with expected crc 2023-09-23T20:29:06.493 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:06 smithi018 ceph-mon[179647]: failed to encode map e252 with expected crc 2023-09-23T20:29:06.493 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:06 smithi018 ceph-mon[179647]: failed to encode map e252 with expected crc 2023-09-23T20:29:06.493 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:06 smithi018 ceph-mon[179647]: failed to encode map e252 with expected crc 2023-09-23T20:29:06.493 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:06 smithi018 ceph-mon[179647]: failed to encode map e252 with expected crc 2023-09-23T20:29:06.493 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:06 smithi018 ceph-mon[179647]: failed to encode map e252 with expected crc 2023-09-23T20:29:06.493 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:06 smithi018 ceph-mon[179647]: failed to encode map e252 with expected crc 2023-09-23T20:29:06.493 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:06 smithi018 ceph-mon[179647]: failed to encode map e252 with expected crc 2023-09-23T20:29:06.494 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:06 smithi018 ceph-mon[179647]: failed to encode map e252 with expected crc 2023-09-23T20:29:06.494 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:06 smithi018 ceph-mon[179647]: failed to encode map e252 with expected crc 2023-09-23T20:29:06.494 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:06 smithi018 ceph-mon[179647]: failed to encode map e252 with expected crc 2023-09-23T20:29:06.494 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:06 smithi018 ceph-mon[179647]: failed to encode map e252 with expected crc 2023-09-23T20:29:06.494 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:06 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 31599/287628 objects degraded (10.986%), 18 pgs degraded (PG_DEGRADED) 2023-09-23T20:29:06.494 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:06 smithi018 ceph-mon[179647]: osdmap e253: 8 total, 7 up, 8 in 2023-09-23T20:29:06.494 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:06 smithi018 ceph-mon[184144]: failed to encode map e252 with expected crc 2023-09-23T20:29:06.494 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:06 smithi018 ceph-mon[184144]: failed to encode map e252 with expected crc 2023-09-23T20:29:06.495 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:06 smithi018 ceph-mon[184144]: failed to encode map e252 with expected crc 2023-09-23T20:29:06.495 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:06 smithi018 ceph-mon[184144]: failed to encode map e252 with expected crc 2023-09-23T20:29:06.495 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:06 smithi018 ceph-mon[184144]: failed to encode map e252 with expected crc 2023-09-23T20:29:06.495 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:06 smithi018 ceph-mon[184144]: failed to encode map e252 with expected crc 2023-09-23T20:29:06.495 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:06 smithi018 ceph-mon[184144]: failed to encode map e252 with expected crc 2023-09-23T20:29:06.495 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:06 smithi018 ceph-mon[184144]: failed to encode map e252 with expected crc 2023-09-23T20:29:06.495 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:06 smithi018 ceph-mon[184144]: failed to encode map e252 with expected crc 2023-09-23T20:29:06.495 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:06 smithi018 ceph-mon[184144]: failed to encode map e252 with expected crc 2023-09-23T20:29:06.495 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:06 smithi018 ceph-mon[184144]: failed to encode map e252 with expected crc 2023-09-23T20:29:06.495 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:06 smithi018 ceph-mon[184144]: failed to encode map e252 with expected crc 2023-09-23T20:29:06.495 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:06 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 31599/287628 objects degraded (10.986%), 18 pgs degraded (PG_DEGRADED) 2023-09-23T20:29:06.495 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:06 smithi018 ceph-mon[184144]: osdmap e253: 8 total, 7 up, 8 in 2023-09-23T20:29:06.498 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:06.547 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:29:06.550 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:06.550 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:29:06.562 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:29:06.651 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:29:06.735 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:06.868 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:29:06.873 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:06.874 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:29:06.915 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:29:06.974 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:29:07.057 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:07.146 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:29:07.163 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:07.164 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:29:07.220 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:07 smithi149 ceph-mon[170394]: pgmap v558: 119 pgs: 4 active+clean+remapped, 2 remapped+peering, 5 peering, 2 active+recovery_wait+degraded, 12 active+undersized+degraded, 1 active+recovering+undersized+remapped, 32 active+undersized, 4 active+recovery_wait+degraded+remapped, 57 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 931 op/s; 31516/286191 objects degraded (11.012%); 9901/286191 objects misplaced (3.460%); 10 MiB/s, 162 objects/s recovering 2023-09-23T20:29:07.220 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:07 smithi149 ceph-mon[170394]: failed to encode map e253 with expected crc 2023-09-23T20:29:07.220 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:07 smithi149 ceph-mon[170394]: failed to encode map e253 with expected crc 2023-09-23T20:29:07.220 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:07 smithi149 ceph-mon[170394]: failed to encode map e253 with expected crc 2023-09-23T20:29:07.220 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:07 smithi149 ceph-mon[170394]: failed to encode map e253 with expected crc 2023-09-23T20:29:07.220 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:07 smithi149 ceph-mon[170394]: failed to encode map e253 with expected crc 2023-09-23T20:29:07.220 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:07 smithi149 ceph-mon[170394]: failed to encode map e253 with expected crc 2023-09-23T20:29:07.221 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:07 smithi149 ceph-mon[170394]: failed to encode map e253 with expected crc 2023-09-23T20:29:07.221 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:07 smithi149 ceph-mon[170394]: osdmap e254: 8 total, 7 up, 8 in 2023-09-23T20:29:07.265 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:29:07.306 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:07.497 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T20:29:07.540 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:07 smithi018 ceph-mon[179647]: pgmap v558: 119 pgs: 4 active+clean+remapped, 2 remapped+peering, 5 peering, 2 active+recovery_wait+degraded, 12 active+undersized+degraded, 1 active+recovering+undersized+remapped, 32 active+undersized, 4 active+recovery_wait+degraded+remapped, 57 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 931 op/s; 31516/286191 objects degraded (11.012%); 9901/286191 objects misplaced (3.460%); 10 MiB/s, 162 objects/s recovering 2023-09-23T20:29:07.540 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:07 smithi018 ceph-mon[179647]: failed to encode map e253 with expected crc 2023-09-23T20:29:07.540 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:07 smithi018 ceph-mon[179647]: failed to encode map e253 with expected crc 2023-09-23T20:29:07.540 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:07 smithi018 ceph-mon[179647]: failed to encode map e253 with expected crc 2023-09-23T20:29:07.541 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:07 smithi018 ceph-mon[179647]: failed to encode map e253 with expected crc 2023-09-23T20:29:07.541 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:07 smithi018 ceph-mon[179647]: failed to encode map e253 with expected crc 2023-09-23T20:29:07.541 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:07 smithi018 ceph-mon[179647]: failed to encode map e253 with expected crc 2023-09-23T20:29:07.541 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:07 smithi018 ceph-mon[179647]: failed to encode map e253 with expected crc 2023-09-23T20:29:07.541 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:07 smithi018 ceph-mon[179647]: osdmap e254: 8 total, 7 up, 8 in 2023-09-23T20:29:07.541 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:07 smithi018 ceph-mon[184144]: pgmap v558: 119 pgs: 4 active+clean+remapped, 2 remapped+peering, 5 peering, 2 active+recovery_wait+degraded, 12 active+undersized+degraded, 1 active+recovering+undersized+remapped, 32 active+undersized, 4 active+recovery_wait+degraded+remapped, 57 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 931 op/s; 31516/286191 objects degraded (11.012%); 9901/286191 objects misplaced (3.460%); 10 MiB/s, 162 objects/s recovering 2023-09-23T20:29:07.541 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:07 smithi018 ceph-mon[184144]: failed to encode map e253 with expected crc 2023-09-23T20:29:07.541 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:07 smithi018 ceph-mon[184144]: failed to encode map e253 with expected crc 2023-09-23T20:29:07.541 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:07 smithi018 ceph-mon[184144]: failed to encode map e253 with expected crc 2023-09-23T20:29:07.542 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:07 smithi018 ceph-mon[184144]: failed to encode map e253 with expected crc 2023-09-23T20:29:07.542 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:07 smithi018 ceph-mon[184144]: failed to encode map e253 with expected crc 2023-09-23T20:29:07.542 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:07 smithi018 ceph-mon[184144]: failed to encode map e253 with expected crc 2023-09-23T20:29:07.542 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:07 smithi018 ceph-mon[184144]: failed to encode map e253 with expected crc 2023-09-23T20:29:07.542 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:07 smithi018 ceph-mon[184144]: osdmap e254: 8 total, 7 up, 8 in 2023-09-23T20:29:07.901 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:29:07.907 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:07.907 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:29:08.008 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:29:08.058 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:08.126 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:29:08.132 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:08.133 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:29:08.168 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:08 smithi018 ceph-mon[179647]: failed to encode map e254 with expected crc 2023-09-23T20:29:08.168 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:08 smithi018 ceph-mon[179647]: failed to encode map e254 with expected crc 2023-09-23T20:29:08.168 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:08 smithi018 ceph-mon[179647]: failed to encode map e254 with expected crc 2023-09-23T20:29:08.168 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:08 smithi018 ceph-mon[179647]: failed to encode map e254 with expected crc 2023-09-23T20:29:08.168 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:08 smithi018 ceph-mon[179647]: failed to encode map e254 with expected crc 2023-09-23T20:29:08.169 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:08 smithi018 ceph-mon[184144]: failed to encode map e254 with expected crc 2023-09-23T20:29:08.169 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:08 smithi018 ceph-mon[184144]: failed to encode map e254 with expected crc 2023-09-23T20:29:08.169 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:08 smithi018 ceph-mon[184144]: failed to encode map e254 with expected crc 2023-09-23T20:29:08.169 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:08 smithi018 ceph-mon[184144]: failed to encode map e254 with expected crc 2023-09-23T20:29:08.169 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:08 smithi018 ceph-mon[184144]: failed to encode map e254 with expected crc 2023-09-23T20:29:08.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:08 smithi149 ceph-mon[170394]: failed to encode map e254 with expected crc 2023-09-23T20:29:08.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:08 smithi149 ceph-mon[170394]: failed to encode map e254 with expected crc 2023-09-23T20:29:08.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:08 smithi149 ceph-mon[170394]: failed to encode map e254 with expected crc 2023-09-23T20:29:08.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:08 smithi149 ceph-mon[170394]: failed to encode map e254 with expected crc 2023-09-23T20:29:08.218 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:08 smithi149 ceph-mon[170394]: failed to encode map e254 with expected crc 2023-09-23T20:29:08.234 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:29:08.278 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:08.555 INFO:teuthology.orchestra.run.smithi149.stdout:ERROR: (22) Invalid argument 2023-09-23T20:29:08.555 INFO:teuthology.orchestra.run.smithi149.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:29:08.682 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:29:08.690 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:08.691 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:29:08.717 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 4, 5, 7, 3, 2, 6] out_osds: [] dead_osds: [1] live_osds: [0, 2, 3, 4, 5, 6, 7] 2023-09-23T20:29:08.718 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:29:08.718 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_1 2023-09-23T20:29:08.718 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:29:08.792 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:29:08.880 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:09.215 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:29:09.225 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:09.225 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:29:09.254 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:29:09.256 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:09 smithi018 ceph-mon[179647]: pgmap v561: 119 pgs: 8 active+clean+remapped, 2 remapped+peering, 5 peering, 2 active+recovery_wait+degraded, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+remapped, 9 active+undersized+degraded, 1 active+recovering+undersized+remapped, 23 active+undersized, 2 active+recovery_wait+degraded+remapped, 57 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 547 op/s; 30223/281052 objects degraded (10.754%); 4403/281052 objects misplaced (1.567%); 5.9 MiB/s, 111 objects/s recovering 2023-09-23T20:29:09.257 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:09 smithi018 ceph-mon[184144]: pgmap v561: 119 pgs: 8 active+clean+remapped, 2 remapped+peering, 5 peering, 2 active+recovery_wait+degraded, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+remapped, 9 active+undersized+degraded, 1 active+recovering+undersized+remapped, 23 active+undersized, 2 active+recovery_wait+degraded+remapped, 57 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 547 op/s; 30223/281052 objects degraded (10.754%); 4403/281052 objects misplaced (1.567%); 5.9 MiB/s, 111 objects/s recovering 2023-09-23T20:29:09.326 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:29:09.369 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:09.421 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:09 smithi149 ceph-mon[170394]: pgmap v561: 119 pgs: 8 active+clean+remapped, 2 remapped+peering, 5 peering, 2 active+recovery_wait+degraded, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+remapped, 9 active+undersized+degraded, 1 active+recovering+undersized+remapped, 23 active+undersized, 2 active+recovery_wait+degraded+remapped, 57 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 547 op/s; 30223/281052 objects degraded (10.754%); 4403/281052 objects misplaced (1.567%); 5.9 MiB/s, 111 objects/s recovering 2023-09-23T20:29:09.942 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:29:10.033 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:29:10.063 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:29:10.094 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:29:10.130 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:29:10.161 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:29:10.195 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:29:10.230 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:29:10.230 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:29:10.230 INFO:teuthology.orchestra.run.smithi018.stderr:Error ENXIO: problem getting command descriptions from osd.1 2023-09-23T20:29:10.230 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:29:10.230 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:29:10.230 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:29:10.230 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:29:10.231 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:29:10.231 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:29:10.438 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:10 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/410796376' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:29:10.439 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:10 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/410796376' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:29:10.470 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:10 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/410796376' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:29:10.671 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:29:10.677 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:10.678 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:29:10.778 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T20:29:10.779 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:29:10.835 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:10.947 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T20:29:10.948 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T20:29:11.166 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:29:11.171 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:11.171 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:29:11.272 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:29:11.338 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:11.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:11 smithi149 ceph-mon[170394]: pgmap v562: 119 pgs: 9 active+clean+remapped, 1 remapped+peering, 2 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+remapped, 9 active+undersized+degraded, 1 active+recovery_wait+undersized+remapped, 1 active+recovering+undersized+remapped, 22 active+undersized, 2 active+recovery_wait+degraded+remapped, 60 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 488 op/s; 32989/279528 objects degraded (11.802%); 4410/279528 objects misplaced (1.578%); 4.3 MiB/s, 81 objects/s recovering 2023-09-23T20:29:11.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:11 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 32989/279528 objects degraded (11.802%), 19 pgs degraded (PG_DEGRADED) 2023-09-23T20:29:11.476 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:29:11.480 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:11 smithi018 ceph-mon[184144]: pgmap v562: 119 pgs: 9 active+clean+remapped, 1 remapped+peering, 2 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+remapped, 9 active+undersized+degraded, 1 active+recovery_wait+undersized+remapped, 1 active+recovering+undersized+remapped, 22 active+undersized, 2 active+recovery_wait+degraded+remapped, 60 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 488 op/s; 32989/279528 objects degraded (11.802%); 4410/279528 objects misplaced (1.578%); 4.3 MiB/s, 81 objects/s recovering 2023-09-23T20:29:11.480 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:11 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 32989/279528 objects degraded (11.802%), 19 pgs degraded (PG_DEGRADED) 2023-09-23T20:29:11.481 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:11.481 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:29:11.483 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:11 smithi018 ceph-mon[179647]: pgmap v562: 119 pgs: 9 active+clean+remapped, 1 remapped+peering, 2 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+remapped, 9 active+undersized+degraded, 1 active+recovery_wait+undersized+remapped, 1 active+recovering+undersized+remapped, 22 active+undersized, 2 active+recovery_wait+degraded+remapped, 60 active+clean; 5.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 488 op/s; 32989/279528 objects degraded (11.802%); 4410/279528 objects misplaced (1.578%); 4.3 MiB/s, 81 objects/s recovering 2023-09-23T20:29:11.483 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:11 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 32989/279528 objects degraded (11.802%), 19 pgs degraded (PG_DEGRADED) 2023-09-23T20:29:11.554 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:29:11.582 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:29:11.663 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:11.719 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:29:11.731 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:11.731 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:29:11.832 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:29:11.877 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:11.925 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:29:11.929 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:11.929 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:29:12.030 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:29:12.084 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:12.140 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:29:12.144 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:12.145 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:29:12.218 INFO:teuthology.orchestra.run.smithi018.stderr:noscrub is unset 2023-09-23T20:29:12.246 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:29:12.275 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:29:12.282 INFO:teuthology.orchestra.run.smithi018.stderr:dumped all 2023-09-23T20:29:12.316 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:12.386 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:29:12.399 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:12 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2719363925' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:29:12.399 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:12 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:29:12.399 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:12 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2719363925' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:29:12.399 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:12 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:29:12.400 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:12.400 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:29:12.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:12 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2719363925' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:29:12.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:12 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:29:12.501 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:29:12.616 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:12.785 INFO:teuthology.orchestra.run.smithi018.stdout:{"pg_ready":true,"pg_map":{"version":564,"stamp":"2023-09-23T20:29:12.201156+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5849348507,"num_objects":89210,"num_object_clones":0,"num_object_copies":267630,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":31688,"num_objects_misplaced":4235,"num_objects_unfound":0,"num_objects_dirty":89210,"num_whiteouts":0,"num_read":6188,"num_read_kb":13573,"num_write":121296,"num_write_kb":6716783,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6381,"num_bytes_recovered":415420499,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":171654,"ondisk_log_size":171654,"up":311,"acting":318,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":389,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":19109844,"kb_used_data":17326420,"kb_used_omap":30,"kb_used_meta":1783137,"kb_avail":730851372,"statfs":{"total":767960285184,"available":748391804928,"internally_reserved":0,"allocated":17742254080,"data_stored":17734220637,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31263,"internal_metadata":1825932769},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,6,0,0,0,3],"upper_bound":2048},"perf_stat":{"commit_latency_ms":9,"apply_latency_ms":9,"commit_latency_ns":9000000,"apply_latency_ns":9000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-405471232,"num_objects":-6187,"num_object_clones":0,"num_object_copies":-18561,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":172,"num_objects_misplaced":-5666,"num_objects_unfound":0,"num_objects_dirty":-6187,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6187,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":847,"num_bytes_recovered":50926276,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.571920"},"pg_stats":[{"pgid":"3.1a","version":"69'1","reported_seq":258,"reported_epoch":253,"state":"active+undersized","last_fresh":"2023-09-23T20:29:06.030069+0000","last_change":"2023-09-23T20:28:13.154911+0000","last_active":"2023-09-23T20:29:06.030069+0000","last_peered":"2023-09-23T20:29:06.030069+0000","last_clean":"2023-09-23T20:28:10.122878+0000","last_became_active":"2023-09-23T20:28:13.154911+0000","last_became_peered":"2023-09-23T20:28:13.154911+0000","last_unstale":"2023-09-23T20:29:06.030069+0000","last_undegraded":"2023-09-23T20:29:06.030069+0000","last_fullsized":"2023-09-23T20:28:13.148164+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:38:02.482515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":["4","2"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":106,"reported_epoch":253,"state":"active+undersized","last_fresh":"2023-09-23T20:29:06.014219+0000","last_change":"2023-09-23T20:28:13.170504+0000","last_active":"2023-09-23T20:29:06.014219+0000","last_peered":"2023-09-23T20:29:06.014219+0000","last_clean":"2023-09-23T20:28:10.123345+0000","last_became_active":"2023-09-23T20:28:13.170504+0000","last_became_peered":"2023-09-23T20:28:13.170504+0000","last_unstale":"2023-09-23T20:29:06.014219+0000","last_undegraded":"2023-09-23T20:29:06.014219+0000","last_fullsized":"2023-09-23T20:28:13.140712+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:35:11.789454+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":["3","4"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":139,"reported_epoch":253,"state":"active+undersized","last_fresh":"2023-09-23T20:29:06.013916+0000","last_change":"2023-09-23T20:28:13.155496+0000","last_active":"2023-09-23T20:29:06.013916+0000","last_peered":"2023-09-23T20:29:06.013916+0000","last_clean":"2023-09-23T20:24:57.110902+0000","last_became_active":"2023-09-23T20:28:13.155496+0000","last_became_peered":"2023-09-23T20:28:13.155496+0000","last_unstale":"2023-09-23T20:29:06.013916+0000","last_undegraded":"2023-09-23T20:29:06.013916+0000","last_fullsized":"2023-09-23T20:28:13.140169+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:49:18.750327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":["3","4"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":218,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T20:29:06.012878+0000","last_change":"2023-09-23T20:25:02.957185+0000","last_active":"2023-09-23T20:29:06.012878+0000","last_peered":"2023-09-23T20:29:06.012878+0000","last_clean":"2023-09-23T20:29:06.012878+0000","last_became_active":"2023-09-23T20:25:02.956689+0000","last_became_peered":"2023-09-23T20:25:02.956689+0000","last_unstale":"2023-09-23T20:29:06.012878+0000","last_undegraded":"2023-09-23T20:29:06.012878+0000","last_fullsized":"2023-09-23T20:29:06.012878+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:04:28.524015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":106,"reported_epoch":253,"state":"active+undersized","last_fresh":"2023-09-23T20:29:06.013931+0000","last_change":"2023-09-23T20:28:13.145246+0000","last_active":"2023-09-23T20:29:06.013931+0000","last_peered":"2023-09-23T20:29:06.013931+0000","last_clean":"2023-09-23T20:28:10.123201+0000","last_became_active":"2023-09-23T20:28:13.145246+0000","last_became_peered":"2023-09-23T20:28:13.145246+0000","last_unstale":"2023-09-23T20:29:06.013931+0000","last_undegraded":"2023-09-23T20:29:06.013931+0000","last_fullsized":"2023-09-23T20:28:13.139195+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:33:00.081087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":["3","0"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":15,"reported_epoch":253,"state":"active+clean+remapped","last_fresh":"2023-09-23T20:29:06.038687+0000","last_change":"2023-09-23T20:29:05.139387+0000","last_active":"2023-09-23T20:29:06.038687+0000","last_peered":"2023-09-23T20:29:06.038687+0000","last_clean":"2023-09-23T20:29:06.038687+0000","last_became_active":"2023-09-23T20:29:05.139050+0000","last_became_peered":"2023-09-23T20:29:05.139050+0000","last_unstale":"2023-09-23T20:29:06.038687+0000","last_undegraded":"2023-09-23T20:29:06.038687+0000","last_fullsized":"2023-09-23T20:29:06.038687+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:07:47.841478+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5],"acting":[6,5,0],"avail_no_missing":["6","0","5"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.12","version":"254'6632","reported_seq":6687,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T20:29:10.606633+0000","last_change":"2023-09-23T20:28:35.573470+0000","last_active":"2023-09-23T20:29:10.606633+0000","last_peered":"2023-09-23T20:29:10.606633+0000","last_clean":"2023-09-23T20:29:10.606633+0000","last_became_active":"2023-09-23T20:28:35.568920+0000","last_became_peered":"2023-09-23T20:28:35.568920+0000","last_unstale":"2023-09-23T20:29:10.606633+0000","last_undegraded":"2023-09-23T20:29:10.606633+0000","last_fullsized":"2023-09-23T20:29:10.606633+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":242,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":6632,"log_dups_size":0,"ondisk_log_size":6632,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":186187776,"num_objects":2841,"num_object_clones":0,"num_object_copies":8523,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2841,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3821,"num_write_kb":213184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":216,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T20:29:06.012931+0000","last_change":"2023-09-23T20:25:02.969432+0000","last_active":"2023-09-23T20:29:06.012931+0000","last_peered":"2023-09-23T20:29:06.012931+0000","last_clean":"2023-09-23T20:29:06.012931+0000","last_became_active":"2023-09-23T20:25:02.969118+0000","last_became_peered":"2023-09-23T20:25:02.969118+0000","last_unstale":"2023-09-23T20:29:06.012931+0000","last_undegraded":"2023-09-23T20:29:06.012931+0000","last_fullsized":"2023-09-23T20:29:06.012931+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:02:22.453831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":125,"reported_epoch":253,"state":"active+clean+remapped","last_fresh":"2023-09-23T20:29:06.013969+0000","last_change":"2023-09-23T20:29:05.134007+0000","last_active":"2023-09-23T20:29:06.013969+0000","last_peered":"2023-09-23T20:29:06.013969+0000","last_clean":"2023-09-23T20:29:06.013969+0000","last_became_active":"2023-09-23T20:29:05.133551+0000","last_became_peered":"2023-09-23T20:29:05.133551+0000","last_unstale":"2023-09-23T20:29:06.013969+0000","last_undegraded":"2023-09-23T20:29:06.013969+0000","last_fullsized":"2023-09-23T20:29:06.013969+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:49:43.225611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6,4],"avail_no_missing":["3","4","6"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.13","version":"254'6525","reported_seq":6617,"reported_epoch":254,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T20:29:10.593921+0000","last_change":"2023-09-23T20:29:05.139152+0000","last_active":"2023-09-23T20:29:10.593921+0000","last_peered":"2023-09-23T20:29:10.593921+0000","last_clean":"2023-09-23T20:28:04.010682+0000","last_became_active":"2023-09-23T20:29:05.134128+0000","last_became_peered":"2023-09-23T20:29:05.134128+0000","last_unstale":"2023-09-23T20:29:10.593921+0000","last_undegraded":"2023-09-23T20:29:05.119079+0000","last_fullsized":"2023-09-23T20:29:10.593921+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":242,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":6525,"log_dups_size":0,"ondisk_log_size":6525,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":184975360,"num_objects":2823,"num_object_clones":0,"num_object_copies":8469,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":113,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2823,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3757,"num_write_kb":210528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":98304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":["7","2"],"object_location_counts":[{"shards":"2,6,7","objects":2710},{"shards":"2,7","objects":113}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.14","version":"90'161","reported_seq":496,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T20:29:06.024941+0000","last_change":"2023-09-23T20:28:05.995127+0000","last_active":"2023-09-23T20:29:06.024941+0000","last_peered":"2023-09-23T20:29:06.024941+0000","last_clean":"2023-09-23T20:29:06.024941+0000","last_became_active":"2023-09-23T20:28:05.994757+0000","last_became_peered":"2023-09-23T20:28:05.994757+0000","last_unstale":"2023-09-23T20:29:06.024941+0000","last_undegraded":"2023-09-23T20:29:06.024941+0000","last_fullsized":"2023-09-23T20:29:06.024941+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:13.244651+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.244651+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0019955509999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109227,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":220,"reported_epoch":253,"state":"active+undersized","last_fresh":"2023-09-23T20:29:06.030966+0000","last_change":"2023-09-23T20:28:13.175805+0000","last_active":"2023-09-23T20:29:06.030966+0000","last_peered":"2023-09-23T20:29:06.030966+0000","last_clean":"2023-09-23T20:28:10.121056+0000","last_became_active":"2023-09-23T20:28:13.175805+0000","last_became_peered":"2023-09-23T20:28:13.175805+0000","last_unstale":"2023-09-23T20:29:06.030966+0000","last_undegraded":"2023-09-23T20:29:06.030966+0000","last_fullsized":"2023-09-23T20:28:13.156272+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:57:51.683405+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":["5","7"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":118,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T20:29:06.013878+0000","last_change":"2023-09-23T20:29:04.130512+0000","last_active":"2023-09-23T20:29:06.013878+0000","last_peered":"2023-09-23T20:29:06.013878+0000","last_clean":"2023-09-23T20:29:06.013878+0000","last_became_active":"2023-09-23T20:29:04.130135+0000","last_became_peered":"2023-09-23T20:29:04.130135+0000","last_unstale":"2023-09-23T20:29:06.013878+0000","last_undegraded":"2023-09-23T20:29:06.013878+0000","last_fullsized":"2023-09-23T20:29:06.013878+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:53:59.385893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.10","version":"254'6567","reported_seq":6623,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T20:29:09.928566+0000","last_change":"2023-09-23T20:28:35.586857+0000","last_active":"2023-09-23T20:29:09.928566+0000","last_peered":"2023-09-23T20:29:09.928566+0000","last_clean":"2023-09-23T20:29:09.928566+0000","last_became_active":"2023-09-23T20:28:35.582165+0000","last_became_peered":"2023-09-23T20:28:35.582165+0000","last_unstale":"2023-09-23T20:29:09.928566+0000","last_undegraded":"2023-09-23T20:29:09.928566+0000","last_fullsized":"2023-09-23T20:29:09.928566+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":242,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":6567,"log_dups_size":0,"ondisk_log_size":6567,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":185499648,"num_objects":2831,"num_object_clones":0,"num_object_copies":8493,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2831,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3749,"num_write_kb":210528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":249,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T20:29:06.057166+0000","last_change":"2023-09-23T20:25:03.038511+0000","last_active":"2023-09-23T20:29:06.057166+0000","last_peered":"2023-09-23T20:29:06.057166+0000","last_clean":"2023-09-23T20:29:06.057166+0000","last_became_active":"2023-09-23T20:25:02.976811+0000","last_became_peered":"2023-09-23T20:25:02.976811+0000","last_unstale":"2023-09-23T20:29:06.057166+0000","last_undegraded":"2023-09-23T20:29:06.057166+0000","last_fullsized":"2023-09-23T20:29:06.057166+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:28:06.605731+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":15,"reported_epoch":253,"state":"active+clean+remapped","last_fresh":"2023-09-23T20:29:06.027982+0000","last_change":"2023-09-23T20:29:05.138529+0000","last_active":"2023-09-23T20:29:06.027982+0000","last_peered":"2023-09-23T20:29:06.027982+0000","last_clean":"2023-09-23T20:29:06.027982+0000","last_became_active":"2023-09-23T20:29:05.138106+0000","last_became_peered":"2023-09-23T20:29:05.138106+0000","last_unstale":"2023-09-23T20:29:06.027982+0000","last_undegraded":"2023-09-23T20:29:06.027982+0000","last_fullsized":"2023-09-23T20:29:06.027982+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:22:35.015117+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2],"acting":[6,2,0],"avail_no_missing":["6","0","2"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.11","version":"254'6460","reported_seq":8421,"reported_epoch":254,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T20:29:10.500246+0000","last_change":"2023-09-23T20:29:05.175828+0000","last_active":"2023-09-23T20:29:10.500246+0000","last_peered":"2023-09-23T20:29:10.500246+0000","last_clean":"2023-09-23T20:28:04.011715+0000","last_became_active":"2023-09-23T20:29:05.161790+0000","last_became_peered":"2023-09-23T20:29:05.161790+0000","last_unstale":"2023-09-23T20:29:10.500246+0000","last_undegraded":"2023-09-23T20:29:05.122019+0000","last_fullsized":"2023-09-23T20:29:10.500246+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":242,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":6460,"log_dups_size":0,"ondisk_log_size":6460,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":181010432,"num_objects":2762,"num_object_clones":0,"num_object_copies":8286,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1630,"num_objects_misplaced":1132,"num_objects_unfound":0,"num_objects_dirty":2762,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3706,"num_write_kb":206976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":595,"num_bytes_recovered":38993920,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2],"acting":[6,2,7],"avail_no_missing":["6","2"],"object_location_counts":[{"shards":"2,6","objects":1630},{"shards":"2,6,7","objects":1132}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":258,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T20:29:06.027958+0000","last_change":"2023-09-23T20:29:04.164197+0000","last_active":"2023-09-23T20:29:06.027958+0000","last_peered":"2023-09-23T20:29:06.027958+0000","last_clean":"2023-09-23T20:29:06.027958+0000","last_became_active":"2023-09-23T20:29:04.130352+0000","last_became_peered":"2023-09-23T20:29:04.130352+0000","last_unstale":"2023-09-23T20:29:06.027958+0000","last_undegraded":"2023-09-23T20:29:06.027958+0000","last_fullsized":"2023-09-23T20:29:06.027958+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:41:01.129784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":241,"reported_epoch":253,"state":"active+clean+remapped","last_fresh":"2023-09-23T20:29:06.025241+0000","last_change":"2023-09-23T20:29:05.142589+0000","last_active":"2023-09-23T20:29:06.025241+0000","last_peered":"2023-09-23T20:29:06.025241+0000","last_clean":"2023-09-23T20:29:06.025241+0000","last_became_active":"2023-09-23T20:29:05.141794+0000","last_became_peered":"2023-09-23T20:29:05.141794+0000","last_unstale":"2023-09-23T20:29:06.025241+0000","last_undegraded":"2023-09-23T20:29:06.025241+0000","last_fullsized":"2023-09-23T20:29:06.025241+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:46:30.161271+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6,5],"avail_no_missing":["4","5","6"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.16","version":"254'6406","reported_seq":6462,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T20:29:08.130719+0000","last_change":"2023-09-23T20:28:35.579111+0000","last_active":"2023-09-23T20:29:08.130719+0000","last_peered":"2023-09-23T20:29:08.130719+0000","last_clean":"2023-09-23T20:29:08.130719+0000","last_became_active":"2023-09-23T20:28:35.577020+0000","last_became_peered":"2023-09-23T20:28:35.577020+0000","last_unstale":"2023-09-23T20:29:08.130719+0000","last_undegraded":"2023-09-23T20:29:08.130719+0000","last_fullsized":"2023-09-23T20:29:08.130719+0000","mapping_epoch":242,"log_start":"221'300","ondisk_log_start":"221'300","created":242,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":6106,"log_dups_size":0,"ondisk_log_size":6106,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":186384384,"num_objects":2844,"num_object_clones":0,"num_object_copies":8532,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2844,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3664,"num_write_kb":208256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.11","version":"90'161","reported_seq":467,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T20:29:06.011856+0000","last_change":"2023-09-23T20:28:05.991681+0000","last_active":"2023-09-23T20:29:06.011856+0000","last_peered":"2023-09-23T20:29:06.011856+0000","last_clean":"2023-09-23T20:29:06.011856+0000","last_became_active":"2023-09-23T20:28:05.991304+0000","last_became_peered":"2023-09-23T20:28:05.991304+0000","last_unstale":"2023-09-23T20:29:06.011856+0000","last_undegraded":"2023-09-23T20:29:06.011856+0000","last_fullsized":"2023-09-23T20:29:06.011856+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:34.821789+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:34.821789+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":248,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T20:29:06.055478+0000","last_change":"2023-09-23T20:25:39.654026+0000","last_active":"2023-09-23T20:29:06.055478+0000","last_peered":"2023-09-23T20:29:06.055478+0000","last_clean":"2023-09-23T20:29:06.055478+0000","last_became_active":"2023-09-23T20:25:39.626849+0000","last_became_peered":"2023-09-23T20:25:39.626849+0000","last_unstale":"2023-09-23T20:29:06.055478+0000","last_undegraded":"2023-09-23T20:29:06.055478+0000","last_fullsized":"2023-09-23T20:29:06.055478+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:22:53.957630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":106,"reported_epoch":253,"state":"active+undersized","last_fresh":"2023-09-23T20:29:06.014993+0000","last_change":"2023-09-23T20:28:13.152317+0000","last_active":"2023-09-23T20:29:06.014993+0000","last_peered":"2023-09-23T20:29:06.014993+0000","last_clean":"2023-09-23T20:28:10.123633+0000","last_became_active":"2023-09-23T20:28:13.152317+0000","last_became_peered":"2023-09-23T20:28:13.152317+0000","last_unstale":"2023-09-23T20:29:06.014993+0000","last_undegraded":"2023-09-23T20:29:06.014993+0000","last_fullsized":"2023-09-23T20:28:13.142454+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:13:55.507629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.17","version":"254'6697","reported_seq":6752,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T20:29:10.660804+0000","last_change":"2023-09-23T20:28:35.586734+0000","last_active":"2023-09-23T20:29:10.660804+0000","last_peered":"2023-09-23T20:29:10.660804+0000","last_clean":"2023-09-23T20:29:10.660804+0000","last_became_active":"2023-09-23T20:28:35.582232+0000","last_became_peered":"2023-09-23T20:28:35.582232+0000","last_unstale":"2023-09-23T20:29:10.660804+0000","last_undegraded":"2023-09-23T20:29:10.660804+0000","last_fullsized":"2023-09-23T20:29:10.660804+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":242,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":6697,"log_dups_size":0,"ondisk_log_size":6697,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":181698560,"num_objects":2773,"num_object_clones":0,"num_object_copies":8319,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2773,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3869,"num_write_kb":212512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.10","version":"90'161","reported_seq":474,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T20:29:07.045212+0000","last_change":"2023-09-23T20:29:07.045212+0000","last_active":"2023-09-23T20:29:07.045212+0000","last_peered":"2023-09-23T20:29:07.045212+0000","last_clean":"2023-09-23T20:29:07.045212+0000","last_became_active":"2023-09-23T20:29:07.044892+0000","last_became_peered":"2023-09-23T20:29:07.044892+0000","last_unstale":"2023-09-23T20:29:07.045212+0000","last_undegraded":"2023-09-23T20:29:07.045212+0000","last_fullsized":"2023-09-23T20:29:07.045212+0000","mapping_epoch":253,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":254,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:32.801168+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:32.801168+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":216,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T20:29:06.012252+0000","last_change":"2023-09-23T20:25:02.953993+0000","last_active":"2023-09-23T20:29:06.012252+0000","last_peered":"2023-09-23T20:29:06.012252+0000","last_clean":"2023-09-23T20:29:06.012252+0000","last_became_active":"2023-09-23T20:25:02.953672+0000","last_became_peered":"2023-09-23T20:25:02.953672+0000","last_unstale":"2023-09-23T20:29:06.012252+0000","last_undegraded":"2023-09-23T20:29:06.012252+0000","last_fullsized":"2023-09-23T20:29:06.012252+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:26:08.603577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":232,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T20:29:06.031271+0000","last_change":"2023-09-23T20:25:02.949584+0000","last_active":"2023-09-23T20:29:06.031271+0000","last_peered":"2023-09-23T20:29:06.031271+0000","last_clean":"2023-09-23T20:29:06.031271+0000","last_became_active":"2023-09-23T20:25:02.949132+0000","last_became_peered":"2023-09-23T20:25:02.949132+0000","last_unstale":"2023-09-23T20:29:06.031271+0000","last_undegraded":"2023-09-23T20:29:06.031271+0000","last_fullsized":"2023-09-23T20:29:06.031271+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:34:20.509823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.14","version":"254'6588","reported_seq":6646,"reported_epoch":254,"state":"active+undersized+degraded","last_fresh":"2023-09-23T20:29:10.622675+0000","last_change":"2023-09-23T20:28:35.577429+0000","last_active":"2023-09-23T20:29:10.622675+0000","last_peered":"2023-09-23T20:29:10.622675+0000","last_clean":"2023-09-23T20:28:12.083147+0000","last_became_active":"2023-09-23T20:28:35.577429+0000","last_became_peered":"2023-09-23T20:28:35.577429+0000","last_unstale":"2023-09-23T20:29:10.622675+0000","last_undegraded":"2023-09-23T20:28:35.564606+0000","last_fullsized":"2023-09-23T20:28:35.527507+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":242,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":6588,"log_dups_size":0,"ondisk_log_size":6588,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":179830784,"num_objects":2744,"num_object_clones":0,"num_object_copies":8232,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2744,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2744,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3824,"num_write_kb":210176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":["5","3"],"object_location_counts":[{"shards":"3,5","objects":2744}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"90'161","reported_seq":491,"reported_epoch":253,"state":"active+undersized","last_fresh":"2023-09-23T20:29:06.014133+0000","last_change":"2023-09-23T20:28:13.172329+0000","last_active":"2023-09-23T20:29:06.014133+0000","last_peered":"2023-09-23T20:29:06.014133+0000","last_clean":"2023-09-23T20:27:55.206508+0000","last_became_active":"2023-09-23T20:28:13.172329+0000","last_became_peered":"2023-09-23T20:28:13.172329+0000","last_unstale":"2023-09-23T20:29:06.014133+0000","last_undegraded":"2023-09-23T20:29:06.014133+0000","last_fullsized":"2023-09-23T20:28:13.140024+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:12.386098+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:12.386098+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0019500940000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":["3","4"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":217,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T20:29:06.053066+0000","last_change":"2023-09-23T20:29:04.152650+0000","last_active":"2023-09-23T20:29:06.053066+0000","last_peered":"2023-09-23T20:29:06.053066+0000","last_clean":"2023-09-23T20:29:06.053066+0000","last_became_active":"2023-09-23T20:29:04.152275+0000","last_became_peered":"2023-09-23T20:29:04.152275+0000","last_unstale":"2023-09-23T20:29:06.053066+0000","last_undegraded":"2023-09-23T20:29:06.053066+0000","last_fullsized":"2023-09-23T20:29:06.053066+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:51:59.491211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":226,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T20:29:06.012795+0000","last_change":"2023-09-23T20:25:02.957010+0000","last_active":"2023-09-23T20:29:06.012795+0000","last_peered":"2023-09-23T20:29:06.012795+0000","last_clean":"2023-09-23T20:29:06.012795+0000","last_became_active":"2023-09-23T20:25:02.956267+0000","last_became_peered":"2023-09-23T20:25:02.956267+0000","last_unstale":"2023-09-23T20:29:06.012795+0000","last_undegraded":"2023-09-23T20:29:06.012795+0000","last_fullsized":"2023-09-23T20:29:06.012795+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:10:55.299422+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.15","version":"254'6471","reported_seq":7257,"reported_epoch":254,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:29:10.583572+0000","last_change":"2023-09-23T20:29:05.279868+0000","last_active":"2023-09-23T20:29:10.583572+0000","last_peered":"2023-09-23T20:29:10.583572+0000","last_clean":"2023-09-23T20:28:04.012743+0000","last_became_active":"2023-09-23T20:29:05.259540+0000","last_became_peered":"2023-09-23T20:29:05.259540+0000","last_unstale":"2023-09-23T20:29:10.583572+0000","last_undegraded":"2023-09-23T20:29:05.196818+0000","last_fullsized":"2023-09-23T20:29:05.130142+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":242,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":6471,"log_dups_size":0,"ondisk_log_size":6471,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":175538176,"num_objects":2679,"num_object_clones":0,"num_object_copies":8037,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1481,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2679,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3731,"num_write_kb":205088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":357,"num_bytes_recovered":23265280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,3],"avail_no_missing":["7","3"],"object_location_counts":[{"shards":"2,3,7","objects":1128},{"shards":"3,7","objects":1551}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.12","version":"90'161","reported_seq":481,"reported_epoch":254,"state":"active+clean+remapped","last_fresh":"2023-09-23T20:29:07.054255+0000","last_change":"2023-09-23T20:29:07.054255+0000","last_active":"2023-09-23T20:29:07.054255+0000","last_peered":"2023-09-23T20:29:07.054255+0000","last_clean":"2023-09-23T20:29:07.054255+0000","last_became_active":"2023-09-23T20:29:07.053800+0000","last_became_peered":"2023-09-23T20:29:07.053800+0000","last_unstale":"2023-09-23T20:29:07.054255+0000","last_undegraded":"2023-09-23T20:29:07.054255+0000","last_fullsized":"2023-09-23T20:29:07.054255+0000","mapping_epoch":253,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":254,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:35.734584+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:35.734584+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6],"acting":[5,6,7],"avail_no_missing":["5","6","7"],"object_location_counts":[{"shards":"5,6,7","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":23,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T20:29:06.027057+0000","last_change":"2023-09-23T20:29:04.179171+0000","last_active":"2023-09-23T20:29:06.027057+0000","last_peered":"2023-09-23T20:29:06.027057+0000","last_clean":"2023-09-23T20:29:06.027057+0000","last_became_active":"2023-09-23T20:29:04.128672+0000","last_became_peered":"2023-09-23T20:29:04.128672+0000","last_unstale":"2023-09-23T20:29:06.027057+0000","last_undegraded":"2023-09-23T20:29:06.027057+0000","last_fullsized":"2023-09-23T20:29:06.027057+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:49:15.506106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":106,"reported_epoch":253,"state":"active+undersized","last_fresh":"2023-09-23T20:29:06.014781+0000","last_change":"2023-09-23T20:28:13.146925+0000","last_active":"2023-09-23T20:29:06.014781+0000","last_peered":"2023-09-23T20:29:06.014781+0000","last_clean":"2023-09-23T20:28:10.124095+0000","last_became_active":"2023-09-23T20:28:13.146925+0000","last_became_peered":"2023-09-23T20:28:13.146925+0000","last_unstale":"2023-09-23T20:29:06.014781+0000","last_undegraded":"2023-09-23T20:29:06.014781+0000","last_fullsized":"2023-09-23T20:28:13.141297+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:26:22.810131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.d","version":"90'161","reported_seq":461,"reported_epoch":253,"state":"active+undersized","last_fresh":"2023-09-23T20:29:06.055441+0000","last_change":"2023-09-23T20:28:13.170455+0000","last_active":"2023-09-23T20:29:06.055441+0000","last_peered":"2023-09-23T20:29:06.055441+0000","last_clean":"2023-09-23T20:28:10.105394+0000","last_became_active":"2023-09-23T20:28:13.170455+0000","last_became_peered":"2023-09-23T20:28:13.170455+0000","last_unstale":"2023-09-23T20:29:06.055441+0000","last_undegraded":"2023-09-23T20:29:06.055441+0000","last_fullsized":"2023-09-23T20:28:13.145373+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:12.809933+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:12.809933+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.a","version":"254'7587","reported_seq":13226,"reported_epoch":254,"state":"active+recovering+undersized+remapped","last_fresh":"2023-09-23T20:29:10.707235+0000","last_change":"2023-09-23T20:29:04.331976+0000","last_active":"2023-09-23T20:29:10.707235+0000","last_peered":"2023-09-23T20:29:10.707235+0000","last_clean":"2023-09-23T20:28:04.001969+0000","last_became_active":"2023-09-23T20:29:04.283749+0000","last_became_peered":"2023-09-23T20:29:04.283749+0000","last_unstale":"2023-09-23T20:29:10.707235+0000","last_undegraded":"2023-09-23T20:29:10.707235+0000","last_fullsized":"2023-09-23T20:29:04.108178+0000","mapping_epoch":250,"log_start":"221'600","ondisk_log_start":"221'600","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":6987,"log_dups_size":0,"ondisk_log_size":6987,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:47:15.006372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":365887488,"num_objects":5583,"num_object_clones":0,"num_object_copies":16749,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2000,"num_objects_unfound":0,"num_objects_dirty":5583,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7587,"num_write_kb":421440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2788,"num_bytes_recovered":182583296,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,0],"avail_no_missing":["5","0"],"object_location_counts":[{"shards":"0,2,5","objects":2000},{"shards":"0,5","objects":3583}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":215,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T20:29:06.051163+0000","last_change":"2023-09-23T20:20:14.883283+0000","last_active":"2023-09-23T20:29:06.051163+0000","last_peered":"2023-09-23T20:29:06.051163+0000","last_clean":"2023-09-23T20:29:06.051163+0000","last_became_active":"2023-09-23T20:20:14.882806+0000","last_became_peered":"2023-09-23T20:20:14.882806+0000","last_unstale":"2023-09-23T20:29:06.051163+0000","last_undegraded":"2023-09-23T20:29:06.051163+0000","last_fullsized":"2023-09-23T20:29:06.051163+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:23:35.295701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":215,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T20:29:06.031013+0000","last_change":"2023-09-23T20:29:04.128604+0000","last_active":"2023-09-23T20:29:06.031013+0000","last_peered":"2023-09-23T20:29:06.031013+0000","last_clean":"2023-09-23T20:29:06.031013+0000","last_became_active":"2023-09-23T20:29:04.128003+0000","last_became_peered":"2023-09-23T20:29:04.128003+0000","last_unstale":"2023-09-23T20:29:06.031013+0000","last_undegraded":"2023-09-23T20:29:06.031013+0000","last_fullsized":"2023-09-23T20:29:06.031013+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:50:49.385599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"90'161","reported_seq":523,"reported_epoch":253,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:29:06.014070+0000","last_change":"2023-09-23T20:29:05.147095+0000","last_active":"2023-09-23T20:29:06.014070+0000","last_peered":"2023-09-23T20:29:06.014070+0000","last_clean":"2023-09-23T20:28:58.097229+0000","last_became_active":"2023-09-23T20:29:05.146673+0000","last_became_peered":"2023-09-23T20:29:05.146673+0000","last_unstale":"2023-09-23T20:29:06.014070+0000","last_undegraded":"2023-09-23T20:29:05.106936+0000","last_fullsized":"2023-09-23T20:29:05.106337+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":233,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:13.244651+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.244651+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0019955509999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109227,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0],"avail_no_missing":["3","0","4","5"],"object_location_counts":[{"shards":"0,3,4,5","objects":1}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.b","version":"254'7213","reported_seq":7262,"reported_epoch":254,"state":"active+undersized+degraded","last_fresh":"2023-09-23T20:29:08.117892+0000","last_change":"2023-09-23T20:28:13.151214+0000","last_active":"2023-09-23T20:29:08.117892+0000","last_peered":"2023-09-23T20:29:08.117892+0000","last_clean":"2023-09-23T20:28:12.089537+0000","last_became_active":"2023-09-23T20:28:13.151214+0000","last_became_peered":"2023-09-23T20:28:13.151214+0000","last_unstale":"2023-09-23T20:29:08.117892+0000","last_undegraded":"2023-09-23T20:28:13.141183+0000","last_fullsized":"2023-09-23T20:28:13.141041+0000","mapping_epoch":236,"log_start":"223'2400","ondisk_log_start":"223'2400","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":4813,"log_dups_size":0,"ondisk_log_size":4813,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:25:15.926592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":365887488,"num_objects":5583,"num_object_clones":0,"num_object_copies":16749,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5583,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5583,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7213,"num_write_kb":409472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":5583}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":223,"reported_epoch":253,"state":"active+undersized+degraded","last_fresh":"2023-09-23T20:29:06.051108+0000","last_change":"2023-09-23T20:28:13.175199+0000","last_active":"2023-09-23T20:29:06.051108+0000","last_peered":"2023-09-23T20:29:06.051108+0000","last_clean":"2023-09-23T20:28:10.105377+0000","last_became_active":"2023-09-23T20:28:13.175199+0000","last_became_peered":"2023-09-23T20:28:13.175199+0000","last_unstale":"2023-09-23T20:29:06.051108+0000","last_undegraded":"2023-09-23T20:28:13.146101+0000","last_fullsized":"2023-09-23T20:28:13.145995+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:32:17.555869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[{"shards":"4,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":256,"reported_epoch":253,"state":"active+undersized","last_fresh":"2023-09-23T20:29:06.023833+0000","last_change":"2023-09-23T20:28:13.156552+0000","last_active":"2023-09-23T20:29:06.023833+0000","last_peered":"2023-09-23T20:29:06.023833+0000","last_clean":"2023-09-23T20:28:10.121663+0000","last_became_active":"2023-09-23T20:28:13.156552+0000","last_became_peered":"2023-09-23T20:28:13.156552+0000","last_unstale":"2023-09-23T20:29:06.023833+0000","last_undegraded":"2023-09-23T20:29:06.023833+0000","last_fullsized":"2023-09-23T20:28:13.147617+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:28:11.148006+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":["4","3"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"90'161","reported_seq":436,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T20:29:07.046694+0000","last_change":"2023-09-23T20:29:07.046694+0000","last_active":"2023-09-23T20:29:07.046694+0000","last_peered":"2023-09-23T20:29:07.046694+0000","last_clean":"2023-09-23T20:29:07.046694+0000","last_became_active":"2023-09-23T20:29:07.046383+0000","last_became_peered":"2023-09-23T20:29:07.046383+0000","last_unstale":"2023-09-23T20:29:07.046694+0000","last_undegraded":"2023-09-23T20:29:07.046694+0000","last_fullsized":"2023-09-23T20:29:07.046694+0000","mapping_epoch":253,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":254,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:32.746340+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:32.746340+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.8","version":"254'6549","reported_seq":6605,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T20:29:10.622572+0000","last_change":"2023-09-23T20:28:35.581921+0000","last_active":"2023-09-23T20:29:10.622572+0000","last_peered":"2023-09-23T20:29:10.622572+0000","last_clean":"2023-09-23T20:29:10.622572+0000","last_became_active":"2023-09-23T20:28:35.576360+0000","last_became_peered":"2023-09-23T20:28:35.576360+0000","last_unstale":"2023-09-23T20:29:10.622572+0000","last_undegraded":"2023-09-23T20:29:10.622572+0000","last_fullsized":"2023-09-23T20:29:10.622572+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":6549,"log_dups_size":0,"ondisk_log_size":6549,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":181600256,"num_objects":2771,"num_object_clones":0,"num_object_copies":8313,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2771,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3763,"num_write_kb":209088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":226,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T20:29:06.056990+0000","last_change":"2023-09-23T20:29:04.184893+0000","last_active":"2023-09-23T20:29:06.056990+0000","last_peered":"2023-09-23T20:29:06.056990+0000","last_clean":"2023-09-23T20:29:06.056990+0000","last_became_active":"2023-09-23T20:29:04.152723+0000","last_became_peered":"2023-09-23T20:29:04.152723+0000","last_unstale":"2023-09-23T20:29:06.056990+0000","last_undegraded":"2023-09-23T20:29:06.056990+0000","last_fullsized":"2023-09-23T20:29:06.056990+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:51:27.017151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":15,"reported_epoch":253,"state":"active+clean+remapped","last_fresh":"2023-09-23T20:29:06.027763+0000","last_change":"2023-09-23T20:29:05.147066+0000","last_active":"2023-09-23T20:29:06.027763+0000","last_peered":"2023-09-23T20:29:06.027763+0000","last_clean":"2023-09-23T20:29:06.027763+0000","last_became_active":"2023-09-23T20:29:05.146690+0000","last_became_peered":"2023-09-23T20:29:05.146690+0000","last_unstale":"2023-09-23T20:29:06.027763+0000","last_undegraded":"2023-09-23T20:29:06.027763+0000","last_fullsized":"2023-09-23T20:29:06.027763+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:55:41.016487+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7],"acting":[6,7,5],"avail_no_missing":["6","5","7"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.e","version":"90'161","reported_seq":460,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T20:29:06.025159+0000","last_change":"2023-09-23T20:28:43.709000+0000","last_active":"2023-09-23T20:29:06.025159+0000","last_peered":"2023-09-23T20:29:06.025159+0000","last_clean":"2023-09-23T20:29:06.025159+0000","last_became_active":"2023-09-23T20:28:05.990524+0000","last_became_peered":"2023-09-23T20:28:05.990524+0000","last_unstale":"2023-09-23T20:29:06.025159+0000","last_undegraded":"2023-09-23T20:29:06.025159+0000","last_fullsized":"2023-09-23T20:29:06.025159+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:31.769333+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:31.769333+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.9","version":"254'6577","reported_seq":6632,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T20:29:09.885382+0000","last_change":"2023-09-23T20:28:35.539652+0000","last_active":"2023-09-23T20:29:09.885382+0000","last_peered":"2023-09-23T20:29:09.885382+0000","last_clean":"2023-09-23T20:29:09.885382+0000","last_became_active":"2023-09-23T20:28:35.536824+0000","last_became_peered":"2023-09-23T20:28:35.536824+0000","last_unstale":"2023-09-23T20:29:09.885382+0000","last_undegraded":"2023-09-23T20:29:09.885382+0000","last_fullsized":"2023-09-23T20:29:09.885382+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":6577,"log_dups_size":0,"ondisk_log_size":6577,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":181600256,"num_objects":2771,"num_object_clones":0,"num_object_copies":8313,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2771,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3781,"num_write_kb":209664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":260,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T20:29:06.031845+0000","last_change":"2023-09-23T20:29:04.138284+0000","last_active":"2023-09-23T20:29:06.031845+0000","last_peered":"2023-09-23T20:29:06.031845+0000","last_clean":"2023-09-23T20:29:06.031845+0000","last_became_active":"2023-09-23T20:29:04.137960+0000","last_became_peered":"2023-09-23T20:29:04.137960+0000","last_unstale":"2023-09-23T20:29:06.031845+0000","last_undegraded":"2023-09-23T20:29:06.031845+0000","last_fullsized":"2023-09-23T20:29:06.031845+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:43:33.132755+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":205,"reported_epoch":253,"state":"active+undersized","last_fresh":"2023-09-23T20:29:06.012408+0000","last_change":"2023-09-23T20:28:13.172321+0000","last_active":"2023-09-23T20:29:06.012408+0000","last_peered":"2023-09-23T20:29:06.012408+0000","last_clean":"2023-09-23T20:28:10.114667+0000","last_became_active":"2023-09-23T20:28:13.172321+0000","last_became_peered":"2023-09-23T20:28:13.172321+0000","last_unstale":"2023-09-23T20:29:06.012408+0000","last_undegraded":"2023-09-23T20:29:06.012408+0000","last_fullsized":"2023-09-23T20:28:13.141132+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:01:05.541514+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":["0","4"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"254'7529","reported_seq":7581,"reported_epoch":254,"state":"active+undersized+degraded","last_fresh":"2023-09-23T20:29:10.689236+0000","last_change":"2023-09-23T20:28:13.171294+0000","last_active":"2023-09-23T20:29:10.689236+0000","last_peered":"2023-09-23T20:29:10.689236+0000","last_clean":"2023-09-23T20:28:12.106078+0000","last_became_active":"2023-09-23T20:28:13.171294+0000","last_became_peered":"2023-09-23T20:28:13.171294+0000","last_unstale":"2023-09-23T20:29:10.689236+0000","last_undegraded":"2023-09-23T20:28:13.149409+0000","last_fullsized":"2023-09-23T20:28:13.149261+0000","mapping_epoch":236,"log_start":"222'1772","ondisk_log_start":"222'1772","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":5757,"log_dups_size":0,"ondisk_log_size":5757,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:17:53.283303+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":365232128,"num_objects":5573,"num_object_clones":0,"num_object_copies":16719,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5573,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5573,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7529,"num_write_kb":419264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":["4","2"],"object_location_counts":[{"shards":"2,4","objects":5573}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":636,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T20:29:06.052937+0000","last_change":"2023-09-23T20:28:05.872149+0000","last_active":"2023-09-23T20:29:06.052937+0000","last_peered":"2023-09-23T20:29:06.052937+0000","last_clean":"2023-09-23T20:29:06.052937+0000","last_became_active":"2023-09-23T20:27:11.961310+0000","last_became_peered":"2023-09-23T20:27:11.961310+0000","last_unstale":"2023-09-23T20:29:06.052937+0000","last_undegraded":"2023-09-23T20:29:06.052937+0000","last_fullsized":"2023-09-23T20:29:06.052937+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:28:05.872074+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:28:05.872074+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:52:32.495955+0000","scrub_duration":0.0029215980000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":104,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T20:29:06.014287+0000","last_change":"2023-09-23T20:25:03.000957+0000","last_active":"2023-09-23T20:29:06.014287+0000","last_peered":"2023-09-23T20:29:06.014287+0000","last_clean":"2023-09-23T20:29:06.014287+0000","last_became_active":"2023-09-23T20:25:02.939903+0000","last_became_peered":"2023-09-23T20:25:02.939903+0000","last_unstale":"2023-09-23T20:29:06.014287+0000","last_undegraded":"2023-09-23T20:29:06.014287+0000","last_fullsized":"2023-09-23T20:29:06.014287+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:51:14.981461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":259,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T20:29:06.027865+0000","last_change":"2023-09-23T20:29:04.142188+0000","last_active":"2023-09-23T20:29:06.027865+0000","last_peered":"2023-09-23T20:29:06.027865+0000","last_clean":"2023-09-23T20:29:06.027865+0000","last_became_active":"2023-09-23T20:29:04.141753+0000","last_became_peered":"2023-09-23T20:29:04.141753+0000","last_unstale":"2023-09-23T20:29:06.027865+0000","last_undegraded":"2023-09-23T20:29:06.027865+0000","last_fullsized":"2023-09-23T20:29:06.027865+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:57:06.423293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"254'7770","reported_seq":7816,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T20:29:12.145938+0000","last_change":"2023-09-23T20:27:14.270199+0000","last_active":"2023-09-23T20:29:12.145938+0000","last_peered":"2023-09-23T20:29:12.145938+0000","last_clean":"2023-09-23T20:29:12.145938+0000","last_became_active":"2023-09-23T20:27:14.269858+0000","last_became_peered":"2023-09-23T20:27:14.269858+0000","last_unstale":"2023-09-23T20:29:12.145938+0000","last_undegraded":"2023-09-23T20:29:12.145938+0000","last_fullsized":"2023-09-23T20:29:12.145938+0000","mapping_epoch":217,"log_start":"221'200","ondisk_log_start":"221'200","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":7570,"log_dups_size":0,"ondisk_log_size":7570,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:47:21.406552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":359137280,"num_objects":5480,"num_object_clones":0,"num_object_copies":16440,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5480,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7770,"num_write_kb":424000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":478,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T20:29:06.030859+0000","last_change":"2023-09-23T20:27:31.757060+0000","last_active":"2023-09-23T20:29:06.030859+0000","last_peered":"2023-09-23T20:29:06.030859+0000","last_clean":"2023-09-23T20:29:06.030859+0000","last_became_active":"2023-09-23T20:27:11.959516+0000","last_became_peered":"2023-09-23T20:27:11.959516+0000","last_unstale":"2023-09-23T20:29:06.030859+0000","last_undegraded":"2023-09-23T20:29:06.030859+0000","last_fullsized":"2023-09-23T20:29:06.030859+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:31.756974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:31.756974+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:35:11.939335+0000","scrub_duration":0.0031802670000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":15,"reported_epoch":253,"state":"active+clean+remapped","last_fresh":"2023-09-23T20:29:06.027882+0000","last_change":"2023-09-23T20:29:05.154511+0000","last_active":"2023-09-23T20:29:06.027882+0000","last_peered":"2023-09-23T20:29:06.027882+0000","last_clean":"2023-09-23T20:29:06.027882+0000","last_became_active":"2023-09-23T20:29:05.154146+0000","last_became_peered":"2023-09-23T20:29:05.154146+0000","last_unstale":"2023-09-23T20:29:06.027882+0000","last_undegraded":"2023-09-23T20:29:06.027882+0000","last_fullsized":"2023-09-23T20:29:06.027882+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:42:51.271501+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4,3],"avail_no_missing":["6","3","4"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":242,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T20:29:06.028636+0000","last_change":"2023-09-23T20:25:39.624636+0000","last_active":"2023-09-23T20:29:06.028636+0000","last_peered":"2023-09-23T20:29:06.028636+0000","last_clean":"2023-09-23T20:29:06.028636+0000","last_became_active":"2023-09-23T20:25:39.624297+0000","last_became_peered":"2023-09-23T20:25:39.624297+0000","last_unstale":"2023-09-23T20:29:06.028636+0000","last_undegraded":"2023-09-23T20:29:06.028636+0000","last_fullsized":"2023-09-23T20:29:06.028636+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:33:47.448737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"90'161","reported_seq":491,"reported_epoch":253,"state":"active+undersized","last_fresh":"2023-09-23T20:29:06.011867+0000","last_change":"2023-09-23T20:28:13.164275+0000","last_active":"2023-09-23T20:29:06.011867+0000","last_peered":"2023-09-23T20:29:06.011867+0000","last_clean":"2023-09-23T20:27:55.206527+0000","last_became_active":"2023-09-23T20:28:13.164275+0000","last_became_peered":"2023-09-23T20:28:13.164275+0000","last_unstale":"2023-09-23T20:29:06.011867+0000","last_undegraded":"2023-09-23T20:29:06.011867+0000","last_fullsized":"2023-09-23T20:28:13.140861+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:12.386098+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:12.386098+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0019500940000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.c","version":"254'7259","reported_seq":10710,"reported_epoch":254,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:29:08.116699+0000","last_change":"2023-09-23T20:29:04.320488+0000","last_active":"2023-09-23T20:29:08.116699+0000","last_peered":"2023-09-23T20:29:08.116699+0000","last_clean":"2023-09-23T20:28:03.999709+0000","last_became_active":"2023-09-23T20:29:04.247715+0000","last_became_peered":"2023-09-23T20:29:04.247715+0000","last_unstale":"2023-09-23T20:29:08.116699+0000","last_undegraded":"2023-09-23T20:29:04.318595+0000","last_fullsized":"2023-09-23T20:29:04.105278+0000","mapping_epoch":250,"log_start":"223'2498","ondisk_log_start":"223'2498","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":4761,"log_dups_size":0,"ondisk_log_size":4761,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:01:13.556577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":370343936,"num_objects":5651,"num_object_clones":0,"num_object_copies":16953,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2954,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5651,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7259,"num_write_kb":413120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1692,"num_bytes_recovered":110755840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,5],"avail_no_missing":["3","5"],"object_location_counts":[{"shards":"3,5","objects":3349},{"shards":"3,5,7","objects":2302}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":233,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T20:29:06.023668+0000","last_change":"2023-09-23T20:25:39.628097+0000","last_active":"2023-09-23T20:29:06.023668+0000","last_peered":"2023-09-23T20:29:06.023668+0000","last_clean":"2023-09-23T20:29:06.023668+0000","last_became_active":"2023-09-23T20:25:39.627806+0000","last_became_peered":"2023-09-23T20:25:39.627806+0000","last_unstale":"2023-09-23T20:29:06.023668+0000","last_undegraded":"2023-09-23T20:29:06.023668+0000","last_fullsized":"2023-09-23T20:29:06.023668+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:53:44.398854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":241,"reported_epoch":253,"state":"active+clean+remapped","last_fresh":"2023-09-23T20:29:06.043769+0000","last_change":"2023-09-23T20:29:05.144009+0000","last_active":"2023-09-23T20:29:06.043769+0000","last_peered":"2023-09-23T20:29:06.043769+0000","last_clean":"2023-09-23T20:29:06.043769+0000","last_became_active":"2023-09-23T20:29:05.143685+0000","last_became_peered":"2023-09-23T20:29:05.143685+0000","last_unstale":"2023-09-23T20:29:06.043769+0000","last_undegraded":"2023-09-23T20:29:06.043769+0000","last_fullsized":"2023-09-23T20:29:06.043769+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:02:39.541761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6,3],"avail_no_missing":["4","3","6"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"254'7660","reported_seq":7713,"reported_epoch":254,"state":"active+undersized+degraded","last_fresh":"2023-09-23T20:29:10.703664+0000","last_change":"2023-09-23T20:28:13.164184+0000","last_active":"2023-09-23T20:29:10.703664+0000","last_peered":"2023-09-23T20:29:10.703664+0000","last_clean":"2023-09-23T20:28:12.085747+0000","last_became_active":"2023-09-23T20:28:13.164184+0000","last_became_peered":"2023-09-23T20:28:13.164184+0000","last_unstale":"2023-09-23T20:29:10.703664+0000","last_undegraded":"2023-09-23T20:28:13.153718+0000","last_fullsized":"2023-09-23T20:28:13.153621+0000","mapping_epoch":236,"log_start":"221'600","ondisk_log_start":"221'600","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":7060,"log_dups_size":0,"ondisk_log_size":7060,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:13:24.112990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":364445720,"num_objects":5562,"num_object_clones":0,"num_object_copies":16686,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5562,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5562,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7660,"num_write_kb":423041,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":["5","0"],"object_location_counts":[{"shards":"0,5","objects":5562}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":518,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T20:29:07.053561+0000","last_change":"2023-09-23T20:29:07.053561+0000","last_active":"2023-09-23T20:29:07.053561+0000","last_peered":"2023-09-23T20:29:07.053561+0000","last_clean":"2023-09-23T20:29:07.053561+0000","last_became_active":"2023-09-23T20:29:07.053177+0000","last_became_peered":"2023-09-23T20:29:07.053177+0000","last_unstale":"2023-09-23T20:29:07.053561+0000","last_undegraded":"2023-09-23T20:29:07.053561+0000","last_fullsized":"2023-09-23T20:29:07.053561+0000","mapping_epoch":253,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":254,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:30.727267+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:30.727267+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:04:32.661761+0000","scrub_duration":0.0052560289999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":106,"reported_epoch":253,"state":"active+undersized","last_fresh":"2023-09-23T20:29:06.013823+0000","last_change":"2023-09-23T20:28:13.143589+0000","last_active":"2023-09-23T20:29:06.013823+0000","last_peered":"2023-09-23T20:29:06.013823+0000","last_clean":"2023-09-23T20:28:10.123116+0000","last_became_active":"2023-09-23T20:28:13.143589+0000","last_became_peered":"2023-09-23T20:28:13.143589+0000","last_unstale":"2023-09-23T20:29:06.013823+0000","last_undegraded":"2023-09-23T20:29:06.013823+0000","last_fullsized":"2023-09-23T20:28:13.138889+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:07:48.412859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":42,"reported_epoch":253,"state":"active+undersized","last_fresh":"2023-09-23T20:29:06.014654+0000","last_change":"2023-09-23T20:28:13.167121+0000","last_active":"2023-09-23T20:29:06.014654+0000","last_peered":"2023-09-23T20:29:06.014654+0000","last_clean":"2023-09-23T20:24:57.109980+0000","last_became_active":"2023-09-23T20:28:13.167121+0000","last_became_peered":"2023-09-23T20:28:13.167121+0000","last_unstale":"2023-09-23T20:29:06.014654+0000","last_undegraded":"2023-09-23T20:29:06.014654+0000","last_fullsized":"2023-09-23T20:28:13.140887+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:49:30.695766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":["3","4"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.0","version":"254'6626","reported_seq":6682,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T20:29:09.918606+0000","last_change":"2023-09-23T20:28:35.536749+0000","last_active":"2023-09-23T20:29:09.918606+0000","last_peered":"2023-09-23T20:29:09.918606+0000","last_clean":"2023-09-23T20:29:09.918606+0000","last_became_active":"2023-09-23T20:28:35.532298+0000","last_became_peered":"2023-09-23T20:28:35.532298+0000","last_unstale":"2023-09-23T20:29:09.918606+0000","last_undegraded":"2023-09-23T20:29:09.918606+0000","last_fullsized":"2023-09-23T20:29:09.918606+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":6626,"log_dups_size":0,"ondisk_log_size":6626,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":187138048,"num_objects":2855,"num_object_clones":0,"num_object_copies":8565,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2855,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3807,"num_write_kb":213216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":473,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T20:29:07.115982+0000","last_change":"2023-09-23T20:29:07.115982+0000","last_active":"2023-09-23T20:29:07.115982+0000","last_peered":"2023-09-23T20:29:07.115982+0000","last_clean":"2023-09-23T20:29:07.115982+0000","last_became_active":"2023-09-23T20:29:07.115647+0000","last_became_peered":"2023-09-23T20:29:07.115647+0000","last_unstale":"2023-09-23T20:29:07.115982+0000","last_undegraded":"2023-09-23T20:29:07.115982+0000","last_fullsized":"2023-09-23T20:29:07.115982+0000","mapping_epoch":253,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":254,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:32.746340+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:32.746340+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"86'119","reported_seq":423,"reported_epoch":253,"state":"active+undersized+degraded","last_fresh":"2023-09-23T20:29:06.012226+0000","last_change":"2023-09-23T20:28:13.147446+0000","last_active":"2023-09-23T20:29:06.012226+0000","last_peered":"2023-09-23T20:29:06.012226+0000","last_clean":"2023-09-23T20:13:53.395005+0000","last_became_active":"2023-09-23T20:28:13.147446+0000","last_became_peered":"2023-09-23T20:28:13.147446+0000","last_unstale":"2023-09-23T20:29:06.012226+0000","last_undegraded":"2023-09-23T20:28:13.139220+0000","last_fullsized":"2023-09-23T20:28:13.139106+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":119,"log_dups_size":0,"ondisk_log_size":119,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:50:31.476259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032705499999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":523,"num_read_kb":3503,"num_write":136,"num_write_kb":3128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[{"shards":"0,7","objects":1}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":270,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T20:29:06.031181+0000","last_change":"2023-09-23T20:25:39.637160+0000","last_active":"2023-09-23T20:29:06.031181+0000","last_peered":"2023-09-23T20:29:06.031181+0000","last_clean":"2023-09-23T20:29:06.031181+0000","last_became_active":"2023-09-23T20:25:39.636860+0000","last_became_peered":"2023-09-23T20:25:39.636860+0000","last_unstale":"2023-09-23T20:29:06.031181+0000","last_undegraded":"2023-09-23T20:29:06.031181+0000","last_fullsized":"2023-09-23T20:29:06.031181+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:21:47.860063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":41,"reported_epoch":253,"state":"active+undersized","last_fresh":"2023-09-23T20:29:06.031428+0000","last_change":"2023-09-23T20:28:13.176485+0000","last_active":"2023-09-23T20:29:06.031428+0000","last_peered":"2023-09-23T20:29:06.031428+0000","last_clean":"2023-09-23T20:12:43.732826+0000","last_became_active":"2023-09-23T20:28:13.176485+0000","last_became_peered":"2023-09-23T20:28:13.176485+0000","last_unstale":"2023-09-23T20:29:06.031428+0000","last_undegraded":"2023-09-23T20:29:06.031428+0000","last_fullsized":"2023-09-23T20:28:13.155600+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:59:12.518891+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":["5","4"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.6","version":"254'6393","reported_seq":6449,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T20:29:08.108113+0000","last_change":"2023-09-23T20:28:35.569598+0000","last_active":"2023-09-23T20:29:08.108113+0000","last_peered":"2023-09-23T20:29:08.108113+0000","last_clean":"2023-09-23T20:29:08.108113+0000","last_became_active":"2023-09-23T20:28:35.565434+0000","last_became_peered":"2023-09-23T20:28:35.565434+0000","last_unstale":"2023-09-23T20:29:08.108113+0000","last_undegraded":"2023-09-23T20:29:08.108113+0000","last_fullsized":"2023-09-23T20:29:08.108113+0000","mapping_epoch":217,"log_start":"221'300","ondisk_log_start":"221'300","created":217,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":6093,"log_dups_size":0,"ondisk_log_size":6093,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":185532416,"num_objects":2831,"num_object_clones":0,"num_object_copies":8493,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2831,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3651,"num_write_kb":207424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":488,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T20:29:06.029837+0000","last_change":"2023-09-23T20:27:55.213707+0000","last_active":"2023-09-23T20:29:06.029837+0000","last_peered":"2023-09-23T20:29:06.029837+0000","last_clean":"2023-09-23T20:29:06.029837+0000","last_became_active":"2023-09-23T20:27:55.213394+0000","last_became_peered":"2023-09-23T20:27:55.213394+0000","last_unstale":"2023-09-23T20:29:06.029837+0000","last_undegraded":"2023-09-23T20:29:06.029837+0000","last_fullsized":"2023-09-23T20:29:06.029837+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:34.821789+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:34.821789+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0042962870000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"86'106","reported_seq":937,"reported_epoch":253,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:29:06.031239+0000","last_change":"2023-09-23T20:29:05.152059+0000","last_active":"2023-09-23T20:29:06.031239+0000","last_peered":"2023-09-23T20:29:06.031239+0000","last_clean":"2023-09-23T20:28:10.123134+0000","last_became_active":"2023-09-23T20:29:05.151356+0000","last_became_peered":"2023-09-23T20:29:05.151356+0000","last_unstale":"2023-09-23T20:29:06.031239+0000","last_undegraded":"2023-09-23T20:29:05.124027+0000","last_fullsized":"2023-09-23T20:29:05.123568+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":106,"log_dups_size":0,"ondisk_log_size":106,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:07:47.136100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00099302599999999993,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":10,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1105,"num_read_kb":1689,"num_write":100,"num_write_kb":404,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6],"acting":[5,0],"avail_no_missing":["5","0"],"object_location_counts":[{"shards":"0,5","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":240,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T20:29:06.025162+0000","last_change":"2023-09-23T20:29:04.130669+0000","last_active":"2023-09-23T20:29:06.025162+0000","last_peered":"2023-09-23T20:29:06.025162+0000","last_clean":"2023-09-23T20:29:06.025162+0000","last_became_active":"2023-09-23T20:29:04.130383+0000","last_became_peered":"2023-09-23T20:29:04.130383+0000","last_unstale":"2023-09-23T20:29:06.025162+0000","last_undegraded":"2023-09-23T20:29:06.025162+0000","last_fullsized":"2023-09-23T20:29:06.025162+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:22:29.746615+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":185,"reported_epoch":253,"state":"active+undersized","last_fresh":"2023-09-23T20:29:06.025210+0000","last_change":"2023-09-23T20:28:13.152577+0000","last_active":"2023-09-23T20:29:06.025210+0000","last_peered":"2023-09-23T20:29:06.025210+0000","last_clean":"2023-09-23T20:25:34.579305+0000","last_became_active":"2023-09-23T20:28:13.152577+0000","last_became_peered":"2023-09-23T20:28:13.152577+0000","last_unstale":"2023-09-23T20:29:06.025210+0000","last_undegraded":"2023-09-23T20:29:06.025210+0000","last_fullsized":"2023-09-23T20:28:13.139696+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:55:11.899806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"6.5","version":"254'6494","reported_seq":7185,"reported_epoch":254,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:29:10.528702+0000","last_change":"2023-09-23T20:29:05.278449+0000","last_active":"2023-09-23T20:29:10.528702+0000","last_peered":"2023-09-23T20:29:10.528702+0000","last_clean":"2023-09-23T20:28:04.012743+0000","last_became_active":"2023-09-23T20:29:05.259417+0000","last_became_peered":"2023-09-23T20:29:05.259417+0000","last_unstale":"2023-09-23T20:29:10.528702+0000","last_undegraded":"2023-09-23T20:29:05.129039+0000","last_fullsized":"2023-09-23T20:29:05.113332+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":6494,"log_dups_size":0,"ondisk_log_size":6494,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":182419456,"num_objects":2783,"num_object_clones":0,"num_object_copies":8349,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1425,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2783,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3753,"num_write_kb":209184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":308,"num_bytes_recovered":20185088,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,3],"avail_no_missing":["7","3"],"object_location_counts":[{"shards":"2,3,7","objects":1117},{"shards":"3,7","objects":1666}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":646,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T20:29:06.050468+0000","last_change":"2023-09-23T20:27:55.209408+0000","last_active":"2023-09-23T20:29:06.050468+0000","last_peered":"2023-09-23T20:29:06.050468+0000","last_clean":"2023-09-23T20:29:06.050468+0000","last_became_active":"2023-09-23T20:27:55.209043+0000","last_became_peered":"2023-09-23T20:27:55.209043+0000","last_unstale":"2023-09-23T20:29:06.050468+0000","last_undegraded":"2023-09-23T20:29:06.050468+0000","last_fullsized":"2023-09-23T20:29:06.050468+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:35.734584+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:35.734584+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0035251380000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"86'130","reported_seq":564,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T20:29:06.025050+0000","last_change":"2023-09-23T20:25:39.637458+0000","last_active":"2023-09-23T20:29:06.025050+0000","last_peered":"2023-09-23T20:29:06.025050+0000","last_clean":"2023-09-23T20:29:06.025050+0000","last_became_active":"2023-09-23T20:25:39.636773+0000","last_became_peered":"2023-09-23T20:25:39.636773+0000","last_unstale":"2023-09-23T20:29:06.025050+0000","last_undegraded":"2023-09-23T20:29:06.025050+0000","last_fullsized":"2023-09-23T20:29:06.025050+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:00:32.810564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011300049999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":512,"num_read_kb":1820,"num_write":133,"num_write_kb":1347,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":225,"reported_epoch":253,"state":"active+clean+remapped","last_fresh":"2023-09-23T20:29:06.026078+0000","last_change":"2023-09-23T20:29:05.120706+0000","last_active":"2023-09-23T20:29:06.026078+0000","last_peered":"2023-09-23T20:29:06.026078+0000","last_clean":"2023-09-23T20:29:06.026078+0000","last_became_active":"2023-09-23T20:29:05.120283+0000","last_became_peered":"2023-09-23T20:29:05.120283+0000","last_unstale":"2023-09-23T20:29:06.026078+0000","last_undegraded":"2023-09-23T20:29:06.026078+0000","last_fullsized":"2023-09-23T20:29:06.026078+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:42:42.680732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6],"acting":[2,6,0],"avail_no_missing":["2","0","6"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":84,"reported_epoch":253,"state":"active+undersized","last_fresh":"2023-09-23T20:29:06.033920+0000","last_change":"2023-09-23T20:28:13.168863+0000","last_active":"2023-09-23T20:29:06.033920+0000","last_peered":"2023-09-23T20:29:06.033920+0000","last_clean":"2023-09-23T20:15:59.442389+0000","last_became_active":"2023-09-23T20:28:13.168863+0000","last_became_peered":"2023-09-23T20:28:13.168863+0000","last_unstale":"2023-09-23T20:29:06.033920+0000","last_undegraded":"2023-09-23T20:29:06.033920+0000","last_fullsized":"2023-09-23T20:28:13.153675+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:30:33.149977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":["5","0"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.7","version":"254'6611","reported_seq":6667,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T20:29:10.650451+0000","last_change":"2023-09-23T20:28:35.534346+0000","last_active":"2023-09-23T20:29:10.650451+0000","last_peered":"2023-09-23T20:29:10.650451+0000","last_clean":"2023-09-23T20:29:10.650451+0000","last_became_active":"2023-09-23T20:28:35.531247+0000","last_became_peered":"2023-09-23T20:28:35.531247+0000","last_unstale":"2023-09-23T20:29:10.650451+0000","last_undegraded":"2023-09-23T20:29:10.650451+0000","last_fullsized":"2023-09-23T20:29:10.650451+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":6611,"log_dups_size":0,"ondisk_log_size":6611,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":183271424,"num_objects":2796,"num_object_clones":0,"num_object_copies":8388,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2796,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3782,"num_write_kb":210528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"86'194","reported_seq":902,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T20:29:06.030788+0000","last_change":"2023-09-23T20:25:40.618665+0000","last_active":"2023-09-23T20:29:06.030788+0000","last_peered":"2023-09-23T20:29:06.030788+0000","last_clean":"2023-09-23T20:29:06.030788+0000","last_became_active":"2023-09-23T20:25:40.618087+0000","last_became_peered":"2023-09-23T20:25:40.618087+0000","last_unstale":"2023-09-23T20:29:06.030788+0000","last_undegraded":"2023-09-23T20:29:06.030788+0000","last_fullsized":"2023-09-23T20:29:06.030788+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":194,"log_dups_size":0,"ondisk_log_size":194,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:07:16.114968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00037995999999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1025,"num_read_kb":1201,"num_write":173,"num_write_kb":1276,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":677,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T20:29:07.042758+0000","last_change":"2023-09-23T20:29:07.042758+0000","last_active":"2023-09-23T20:29:07.042758+0000","last_peered":"2023-09-23T20:29:07.042758+0000","last_clean":"2023-09-23T20:29:07.042758+0000","last_became_active":"2023-09-23T20:29:07.042407+0000","last_became_peered":"2023-09-23T20:29:07.042407+0000","last_unstale":"2023-09-23T20:29:07.042758+0000","last_undegraded":"2023-09-23T20:29:07.042758+0000","last_fullsized":"2023-09-23T20:29:07.042758+0000","mapping_epoch":253,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":254,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:32.801168+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:32.801168+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0096333110000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":118,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T20:29:06.014053+0000","last_change":"2023-09-23T20:29:04.216400+0000","last_active":"2023-09-23T20:29:06.014053+0000","last_peered":"2023-09-23T20:29:06.014053+0000","last_clean":"2023-09-23T20:29:06.014053+0000","last_became_active":"2023-09-23T20:29:04.216069+0000","last_became_peered":"2023-09-23T20:29:04.216069+0000","last_unstale":"2023-09-23T20:29:06.014053+0000","last_undegraded":"2023-09-23T20:29:06.014053+0000","last_fullsized":"2023-09-23T20:29:06.014053+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:27:37.433035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":13,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T20:29:06.027423+0000","last_change":"2023-09-23T20:29:04.172672+0000","last_active":"2023-09-23T20:29:06.027423+0000","last_peered":"2023-09-23T20:29:06.027423+0000","last_clean":"2023-09-23T20:29:06.027423+0000","last_became_active":"2023-09-23T20:29:04.162047+0000","last_became_peered":"2023-09-23T20:29:04.162047+0000","last_unstale":"2023-09-23T20:29:06.027423+0000","last_undegraded":"2023-09-23T20:29:06.027423+0000","last_fullsized":"2023-09-23T20:29:06.027423+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:28:28.187096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.4","version":"254'6595","reported_seq":6654,"reported_epoch":254,"state":"active+undersized+degraded","last_fresh":"2023-09-23T20:29:10.697642+0000","last_change":"2023-09-23T20:28:35.565448+0000","last_active":"2023-09-23T20:29:10.697642+0000","last_peered":"2023-09-23T20:29:10.697642+0000","last_clean":"2023-09-23T20:28:12.083147+0000","last_became_active":"2023-09-23T20:28:35.565448+0000","last_became_peered":"2023-09-23T20:28:35.565448+0000","last_unstale":"2023-09-23T20:29:10.697642+0000","last_undegraded":"2023-09-23T20:28:35.525526+0000","last_fullsized":"2023-09-23T20:28:35.525410+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":6595,"log_dups_size":0,"ondisk_log_size":6595,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":183173120,"num_objects":2795,"num_object_clones":0,"num_object_copies":8385,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2795,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2795,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3831,"num_write_kb":212032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":["5","3"],"object_location_counts":[{"shards":"3,5","objects":2795}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":455,"reported_epoch":254,"state":"active+clean+remapped","last_fresh":"2023-09-23T20:29:07.048149+0000","last_change":"2023-09-23T20:29:07.048149+0000","last_active":"2023-09-23T20:29:07.048149+0000","last_peered":"2023-09-23T20:29:07.048149+0000","last_clean":"2023-09-23T20:29:07.048149+0000","last_became_active":"2023-09-23T20:29:07.047719+0000","last_became_peered":"2023-09-23T20:29:07.047719+0000","last_unstale":"2023-09-23T20:29:07.048149+0000","last_undegraded":"2023-09-23T20:29:07.048149+0000","last_fullsized":"2023-09-23T20:29:07.048149+0000","mapping_epoch":253,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":254,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:12.386098+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:12.386098+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4,5],"avail_no_missing":["6","4","5"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"86'80","reported_seq":507,"reported_epoch":253,"state":"active+undersized","last_fresh":"2023-09-23T20:29:06.058104+0000","last_change":"2023-09-23T20:28:13.144600+0000","last_active":"2023-09-23T20:29:06.058104+0000","last_peered":"2023-09-23T20:29:06.058104+0000","last_clean":"2023-09-23T20:28:10.104142+0000","last_became_active":"2023-09-23T20:28:13.144600+0000","last_became_peered":"2023-09-23T20:28:13.144600+0000","last_unstale":"2023-09-23T20:29:06.058104+0000","last_undegraded":"2023-09-23T20:29:06.058104+0000","last_fullsized":"2023-09-23T20:28:13.141588+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":80,"log_dups_size":0,"ondisk_log_size":80,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:37:42.749177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035115,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":395,"num_read_kb":1370,"num_write":66,"num_write_kb":1061,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":["7","0"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":217,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T20:29:06.012035+0000","last_change":"2023-09-23T20:25:02.942709+0000","last_active":"2023-09-23T20:29:06.012035+0000","last_peered":"2023-09-23T20:29:06.012035+0000","last_clean":"2023-09-23T20:29:06.012035+0000","last_became_active":"2023-09-23T20:25:02.942323+0000","last_became_peered":"2023-09-23T20:25:02.942323+0000","last_unstale":"2023-09-23T20:29:06.012035+0000","last_undegraded":"2023-09-23T20:29:06.012035+0000","last_fullsized":"2023-09-23T20:29:06.012035+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:26:12.039101+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":239,"reported_epoch":253,"state":"active+undersized","last_fresh":"2023-09-23T20:29:06.013124+0000","last_change":"2023-09-23T20:28:13.144536+0000","last_active":"2023-09-23T20:29:06.013124+0000","last_peered":"2023-09-23T20:29:06.013124+0000","last_clean":"2023-09-23T20:28:10.114404+0000","last_became_active":"2023-09-23T20:28:13.144536+0000","last_became_peered":"2023-09-23T20:28:13.144536+0000","last_unstale":"2023-09-23T20:29:06.013124+0000","last_undegraded":"2023-09-23T20:29:06.013124+0000","last_fullsized":"2023-09-23T20:28:13.140563+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:50:22.944488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2],"avail_no_missing":["0","2"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"254'6457","reported_seq":8400,"reported_epoch":254,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T20:29:10.500405+0000","last_change":"2023-09-23T20:29:05.164266+0000","last_active":"2023-09-23T20:29:10.500405+0000","last_peered":"2023-09-23T20:29:10.500405+0000","last_clean":"2023-09-23T20:28:04.011715+0000","last_became_active":"2023-09-23T20:29:05.145058+0000","last_became_peered":"2023-09-23T20:29:05.145058+0000","last_unstale":"2023-09-23T20:29:10.500405+0000","last_undegraded":"2023-09-23T20:29:05.116413+0000","last_fullsized":"2023-09-23T20:29:10.500405+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":6457,"log_dups_size":0,"ondisk_log_size":6457,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":180289536,"num_objects":2751,"num_object_clones":0,"num_object_copies":8253,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1666,"num_objects_misplaced":1085,"num_objects_unfound":0,"num_objects_dirty":2751,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3703,"num_write_kb":206528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":586,"num_bytes_recovered":38404096,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2],"acting":[6,2,7],"avail_no_missing":["6","2"],"object_location_counts":[{"shards":"2,6","objects":1666},{"shards":"2,6,7","objects":1085}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":466,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T20:29:07.113747+0000","last_change":"2023-09-23T20:29:07.113747+0000","last_active":"2023-09-23T20:29:07.113747+0000","last_peered":"2023-09-23T20:29:07.113747+0000","last_clean":"2023-09-23T20:29:07.113747+0000","last_became_active":"2023-09-23T20:29:07.113428+0000","last_became_peered":"2023-09-23T20:29:07.113428+0000","last_unstale":"2023-09-23T20:29:07.113747+0000","last_undegraded":"2023-09-23T20:29:07.113747+0000","last_fullsized":"2023-09-23T20:29:07.113747+0000","mapping_epoch":253,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":254,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:31.769333+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:31.769333+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"86'102","reported_seq":760,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T20:29:06.057076+0000","last_change":"2023-09-23T20:20:14.881975+0000","last_active":"2023-09-23T20:29:06.057076+0000","last_peered":"2023-09-23T20:29:06.057076+0000","last_clean":"2023-09-23T20:29:06.057076+0000","last_became_active":"2023-09-23T20:20:14.881544+0000","last_became_peered":"2023-09-23T20:20:14.881544+0000","last_unstale":"2023-09-23T20:29:06.057076+0000","last_undegraded":"2023-09-23T20:29:06.057076+0000","last_fullsized":"2023-09-23T20:29:06.057076+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":102,"log_dups_size":0,"ondisk_log_size":102,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:34.594972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030656500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":963,"num_read_kb":1234,"num_write":60,"num_write_kb":225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":151,"reported_epoch":253,"state":"active+undersized","last_fresh":"2023-09-23T20:29:06.025068+0000","last_change":"2023-09-23T20:28:13.158522+0000","last_active":"2023-09-23T20:29:06.025068+0000","last_peered":"2023-09-23T20:29:06.025068+0000","last_clean":"2023-09-23T20:25:34.579453+0000","last_became_active":"2023-09-23T20:28:13.158522+0000","last_became_peered":"2023-09-23T20:28:13.158522+0000","last_unstale":"2023-09-23T20:29:06.025068+0000","last_undegraded":"2023-09-23T20:29:06.025068+0000","last_fullsized":"2023-09-23T20:28:13.139418+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:41:36.312605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5],"acting":[2,5],"avail_no_missing":["2","5"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":185,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T20:29:06.012586+0000","last_change":"2023-09-23T20:20:14.888874+0000","last_active":"2023-09-23T20:29:06.012586+0000","last_peered":"2023-09-23T20:29:06.012586+0000","last_clean":"2023-09-23T20:29:06.012586+0000","last_became_active":"2023-09-23T20:20:14.887724+0000","last_became_peered":"2023-09-23T20:20:14.887724+0000","last_unstale":"2023-09-23T20:29:06.012586+0000","last_undegraded":"2023-09-23T20:29:06.012586+0000","last_fullsized":"2023-09-23T20:29:06.012586+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:09:24.510326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"254'6634","reported_seq":6693,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T20:29:10.697959+0000","last_change":"2023-09-23T20:28:57.713293+0000","last_active":"2023-09-23T20:29:10.697959+0000","last_peered":"2023-09-23T20:29:10.697959+0000","last_clean":"2023-09-23T20:29:10.697959+0000","last_became_active":"2023-09-23T20:28:35.539470+0000","last_became_peered":"2023-09-23T20:28:35.539470+0000","last_unstale":"2023-09-23T20:29:10.697959+0000","last_undegraded":"2023-09-23T20:29:10.697959+0000","last_fullsized":"2023-09-23T20:29:10.697959+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":6634,"log_dups_size":0,"ondisk_log_size":6634,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":182386688,"num_objects":2783,"num_object_clones":0,"num_object_copies":8349,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2783,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3823,"num_write_kb":211392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":537,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T20:29:07.112390+0000","last_change":"2023-09-23T20:29:07.112390+0000","last_active":"2023-09-23T20:29:07.112390+0000","last_peered":"2023-09-23T20:29:07.112390+0000","last_clean":"2023-09-23T20:29:07.112390+0000","last_became_active":"2023-09-23T20:29:07.112037+0000","last_became_peered":"2023-09-23T20:29:07.112037+0000","last_unstale":"2023-09-23T20:29:07.112390+0000","last_undegraded":"2023-09-23T20:29:07.112390+0000","last_fullsized":"2023-09-23T20:29:07.112390+0000","mapping_epoch":253,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":254,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:12.809933+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:12.809933+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.001626628,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":104,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T20:29:06.014729+0000","last_change":"2023-09-23T20:25:03.007736+0000","last_active":"2023-09-23T20:29:06.014729+0000","last_peered":"2023-09-23T20:29:06.014729+0000","last_clean":"2023-09-23T20:29:06.014729+0000","last_became_active":"2023-09-23T20:25:02.973689+0000","last_became_peered":"2023-09-23T20:25:02.973689+0000","last_unstale":"2023-09-23T20:29:06.014729+0000","last_undegraded":"2023-09-23T20:29:06.014729+0000","last_fullsized":"2023-09-23T20:29:06.014729+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:37:25.643774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"86'128","reported_seq":570,"reported_epoch":254,"state":"active+recovery_wait+undersized+remapped","last_fresh":"2023-09-23T20:29:07.110379+0000","last_change":"2023-09-23T20:29:05.133903+0000","last_active":"2023-09-23T20:29:07.110379+0000","last_peered":"2023-09-23T20:29:07.110379+0000","last_clean":"2023-09-23T20:28:02.998875+0000","last_became_active":"2023-09-23T20:29:05.133509+0000","last_became_peered":"2023-09-23T20:29:05.133509+0000","last_unstale":"2023-09-23T20:29:07.110379+0000","last_undegraded":"2023-09-23T20:29:07.110379+0000","last_fullsized":"2023-09-23T20:29:05.112245+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:52:29.541853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000293202,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":7,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":579,"num_read_kb":1473,"num_write":96,"num_write_kb":1062,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[3,4],"avail_no_missing":["3","4"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":101,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T20:29:06.013620+0000","last_change":"2023-09-23T20:25:02.971740+0000","last_active":"2023-09-23T20:29:06.013620+0000","last_peered":"2023-09-23T20:29:06.013620+0000","last_clean":"2023-09-23T20:29:06.013620+0000","last_became_active":"2023-09-23T20:25:02.970412+0000","last_became_peered":"2023-09-23T20:25:02.970412+0000","last_unstale":"2023-09-23T20:29:06.013620+0000","last_undegraded":"2023-09-23T20:29:06.013620+0000","last_fullsized":"2023-09-23T20:29:06.013620+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:53:17.247216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"254'6528","reported_seq":6615,"reported_epoch":254,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T20:29:10.593723+0000","last_change":"2023-09-23T20:29:05.139845+0000","last_active":"2023-09-23T20:29:10.593723+0000","last_peered":"2023-09-23T20:29:10.593723+0000","last_clean":"2023-09-23T20:28:04.010682+0000","last_became_active":"2023-09-23T20:29:05.134072+0000","last_became_peered":"2023-09-23T20:29:05.134072+0000","last_unstale":"2023-09-23T20:29:10.593723+0000","last_undegraded":"2023-09-23T20:29:05.113897+0000","last_fullsized":"2023-09-23T20:29:10.593723+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":6528,"log_dups_size":0,"ondisk_log_size":6528,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":178749440,"num_objects":2727,"num_object_clones":0,"num_object_copies":8181,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":148,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2727,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3759,"num_write_kb":207584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":98304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":["7","2"],"object_location_counts":[{"shards":"2,6,7","objects":2579},{"shards":"2,7","objects":148}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.4","version":"90'161","reported_seq":500,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T20:29:06.014888+0000","last_change":"2023-09-23T20:28:03.553294+0000","last_active":"2023-09-23T20:29:06.014888+0000","last_peered":"2023-09-23T20:29:06.014888+0000","last_clean":"2023-09-23T20:29:06.014888+0000","last_became_active":"2023-09-23T20:27:55.212201+0000","last_became_peered":"2023-09-23T20:27:55.212201+0000","last_unstale":"2023-09-23T20:29:06.014888+0000","last_undegraded":"2023-09-23T20:29:06.014888+0000","last_fullsized":"2023-09-23T20:29:06.014888+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:28:03.553106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:28:03.553106+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:24:06.706940+0000","scrub_duration":0.011176864,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"86'117","reported_seq":564,"reported_epoch":254,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:29:07.107809+0000","last_change":"2023-09-23T20:29:05.138591+0000","last_active":"2023-09-23T20:29:07.107809+0000","last_peered":"2023-09-23T20:29:07.107809+0000","last_clean":"2023-09-23T20:13:53.392235+0000","last_became_active":"2023-09-23T20:29:05.138184+0000","last_became_peered":"2023-09-23T20:29:05.138184+0000","last_unstale":"2023-09-23T20:29:07.107809+0000","last_undegraded":"2023-09-23T20:29:05.109149+0000","last_fullsized":"2023-09-23T20:29:05.108660+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:50:02.080933+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046891400000000002,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":893,"num_read_kb":922,"num_write":82,"num_write_kb":4193,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[2,7],"avail_no_missing":["2","3","7"],"object_location_counts":[{"shards":"2,3,7","objects":1}],"blocked_by":[],"up_primary":6,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":211,"reported_epoch":253,"state":"active+undersized+degraded","last_fresh":"2023-09-23T20:29:06.012985+0000","last_change":"2023-09-23T20:28:13.176296+0000","last_active":"2023-09-23T20:29:06.012985+0000","last_peered":"2023-09-23T20:29:06.012985+0000","last_clean":"2023-09-23T20:28:10.114330+0000","last_became_active":"2023-09-23T20:28:13.176296+0000","last_became_peered":"2023-09-23T20:28:13.176296+0000","last_unstale":"2023-09-23T20:29:06.012985+0000","last_undegraded":"2023-09-23T20:28:13.140349+0000","last_fullsized":"2023-09-23T20:28:13.140255+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:12:18.491192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":["0","4"],"object_location_counts":[{"shards":"0,4","objects":1}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":215,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T20:29:06.023734+0000","last_change":"2023-09-23T20:29:04.131353+0000","last_active":"2023-09-23T20:29:06.023734+0000","last_peered":"2023-09-23T20:29:06.023734+0000","last_clean":"2023-09-23T20:29:06.023734+0000","last_became_active":"2023-09-23T20:29:04.131083+0000","last_became_peered":"2023-09-23T20:29:04.131083+0000","last_unstale":"2023-09-23T20:29:06.023734+0000","last_undegraded":"2023-09-23T20:29:06.023734+0000","last_fullsized":"2023-09-23T20:29:06.023734+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:20:23.992180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":187,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T20:29:06.012538+0000","last_change":"2023-09-23T20:20:14.888484+0000","last_active":"2023-09-23T20:29:06.012538+0000","last_peered":"2023-09-23T20:29:06.012538+0000","last_clean":"2023-09-23T20:29:06.012538+0000","last_became_active":"2023-09-23T20:20:14.887219+0000","last_became_peered":"2023-09-23T20:20:14.887219+0000","last_unstale":"2023-09-23T20:29:06.012538+0000","last_undegraded":"2023-09-23T20:29:06.012538+0000","last_fullsized":"2023-09-23T20:29:06.012538+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:13:53.763649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":95,"reported_epoch":253,"state":"active+undersized","last_fresh":"2023-09-23T20:29:06.025335+0000","last_change":"2023-09-23T20:28:13.152589+0000","last_active":"2023-09-23T20:29:06.025335+0000","last_peered":"2023-09-23T20:29:06.025335+0000","last_clean":"2023-09-23T20:28:12.056838+0000","last_became_active":"2023-09-23T20:28:13.152589+0000","last_became_peered":"2023-09-23T20:28:13.152589+0000","last_unstale":"2023-09-23T20:29:06.025335+0000","last_undegraded":"2023-09-23T20:29:06.025335+0000","last_fullsized":"2023-09-23T20:28:13.139976+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:54:38.464315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"6.19","version":"254'6547","reported_seq":6602,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T20:29:09.930670+0000","last_change":"2023-09-23T20:28:35.571020+0000","last_active":"2023-09-23T20:29:09.930670+0000","last_peered":"2023-09-23T20:29:09.930670+0000","last_clean":"2023-09-23T20:29:09.930670+0000","last_became_active":"2023-09-23T20:28:35.567626+0000","last_became_peered":"2023-09-23T20:28:35.567626+0000","last_unstale":"2023-09-23T20:29:09.930670+0000","last_undegraded":"2023-09-23T20:29:09.930670+0000","last_fullsized":"2023-09-23T20:29:09.930670+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":242,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":6547,"log_dups_size":0,"ondisk_log_size":6547,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":183959552,"num_objects":2807,"num_object_clones":0,"num_object_copies":8421,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2807,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3751,"num_write_kb":209856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":222,"reported_epoch":253,"state":"active+undersized+degraded","last_fresh":"2023-09-23T20:29:06.031712+0000","last_change":"2023-09-23T20:28:13.169405+0000","last_active":"2023-09-23T20:29:06.031712+0000","last_peered":"2023-09-23T20:29:06.031712+0000","last_clean":"2023-09-23T20:28:10.121149+0000","last_became_active":"2023-09-23T20:28:13.169405+0000","last_became_peered":"2023-09-23T20:28:13.169405+0000","last_unstale":"2023-09-23T20:29:06.031712+0000","last_undegraded":"2023-09-23T20:28:13.153461+0000","last_fullsized":"2023-09-23T20:28:13.153347+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":100,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:03:24.383853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":["5","4"],"object_location_counts":[{"shards":"4,5","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":266,"reported_epoch":253,"state":"active+undersized","last_fresh":"2023-09-23T20:29:06.028138+0000","last_change":"2023-09-23T20:28:13.169986+0000","last_active":"2023-09-23T20:29:06.028138+0000","last_peered":"2023-09-23T20:29:06.028138+0000","last_clean":"2023-09-23T20:28:10.122290+0000","last_became_active":"2023-09-23T20:28:13.169986+0000","last_became_peered":"2023-09-23T20:28:13.169986+0000","last_unstale":"2023-09-23T20:29:06.028138+0000","last_undegraded":"2023-09-23T20:29:06.028138+0000","last_fullsized":"2023-09-23T20:28:13.148881+0000","mapping_epoch":236,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:44.116758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":["4","3"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.18","version":"254'6537","reported_seq":6592,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T20:29:10.566181+0000","last_change":"2023-09-23T20:28:35.587755+0000","last_active":"2023-09-23T20:29:10.566181+0000","last_peered":"2023-09-23T20:29:10.566181+0000","last_clean":"2023-09-23T20:29:10.566181+0000","last_became_active":"2023-09-23T20:28:35.585502+0000","last_became_peered":"2023-09-23T20:28:35.585502+0000","last_unstale":"2023-09-23T20:29:10.566181+0000","last_undegraded":"2023-09-23T20:29:10.566181+0000","last_fullsized":"2023-09-23T20:29:10.566181+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":242,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":6537,"log_dups_size":0,"ondisk_log_size":6537,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":182255616,"num_objects":2781,"num_object_clones":0,"num_object_copies":8343,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2781,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3751,"num_write_kb":209024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":233,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T20:29:06.031033+0000","last_change":"2023-09-23T20:29:04.179666+0000","last_active":"2023-09-23T20:29:06.031033+0000","last_peered":"2023-09-23T20:29:06.031033+0000","last_clean":"2023-09-23T20:29:06.031033+0000","last_became_active":"2023-09-23T20:29:04.134810+0000","last_became_peered":"2023-09-23T20:29:04.134810+0000","last_unstale":"2023-09-23T20:29:06.031033+0000","last_undegraded":"2023-09-23T20:29:06.031033+0000","last_fullsized":"2023-09-23T20:29:06.031033+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:11:30.606652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":259,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T20:29:06.029896+0000","last_change":"2023-09-23T20:25:03.103656+0000","last_active":"2023-09-23T20:29:06.029896+0000","last_peered":"2023-09-23T20:29:06.029896+0000","last_clean":"2023-09-23T20:29:06.029896+0000","last_became_active":"2023-09-23T20:25:03.103311+0000","last_became_peered":"2023-09-23T20:25:03.103311+0000","last_unstale":"2023-09-23T20:29:06.029896+0000","last_undegraded":"2023-09-23T20:29:06.029896+0000","last_fullsized":"2023-09-23T20:29:06.029896+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:55:54.677030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":134,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T20:29:06.014486+0000","last_change":"2023-09-23T20:29:04.138540+0000","last_active":"2023-09-23T20:29:06.014486+0000","last_peered":"2023-09-23T20:29:06.014486+0000","last_clean":"2023-09-23T20:29:06.014486+0000","last_became_active":"2023-09-23T20:29:04.138182+0000","last_became_peered":"2023-09-23T20:29:04.138182+0000","last_unstale":"2023-09-23T20:29:06.014486+0000","last_undegraded":"2023-09-23T20:29:06.014486+0000","last_fullsized":"2023-09-23T20:29:06.014486+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:53:15.700049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":109,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T20:29:06.013770+0000","last_change":"2023-09-23T20:25:39.622819+0000","last_active":"2023-09-23T20:29:06.013770+0000","last_peered":"2023-09-23T20:29:06.013770+0000","last_clean":"2023-09-23T20:29:06.013770+0000","last_became_active":"2023-09-23T20:25:39.622484+0000","last_became_peered":"2023-09-23T20:25:39.622484+0000","last_unstale":"2023-09-23T20:29:06.013770+0000","last_undegraded":"2023-09-23T20:29:06.013770+0000","last_fullsized":"2023-09-23T20:29:06.013770+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:51:53.973261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":233,"reported_epoch":253,"state":"active+clean","last_fresh":"2023-09-23T20:29:06.012988+0000","last_change":"2023-09-23T20:25:39.668882+0000","last_active":"2023-09-23T20:29:06.012988+0000","last_peered":"2023-09-23T20:29:06.012988+0000","last_clean":"2023-09-23T20:29:06.012988+0000","last_became_active":"2023-09-23T20:25:39.637128+0000","last_became_peered":"2023-09-23T20:25:39.637128+0000","last_unstale":"2023-09-23T20:29:06.012988+0000","last_undegraded":"2023-09-23T20:29:06.012988+0000","last_fullsized":"2023-09-23T20:29:06.012988+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:56.406298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":261,"reported_epoch":253,"state":"active+clean+remapped","last_fresh":"2023-09-23T20:29:06.023512+0000","last_change":"2023-09-23T20:29:05.142199+0000","last_active":"2023-09-23T20:29:06.023512+0000","last_peered":"2023-09-23T20:29:06.023512+0000","last_clean":"2023-09-23T20:29:06.023512+0000","last_became_active":"2023-09-23T20:29:05.141832+0000","last_became_peered":"2023-09-23T20:29:05.141832+0000","last_unstale":"2023-09-23T20:29:06.023512+0000","last_undegraded":"2023-09-23T20:29:06.023512+0000","last_fullsized":"2023-09-23T20:29:06.023512+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":252,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:02:18.905969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6,3],"avail_no_missing":["4","3","6"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":26,"stat_sum":{"num_bytes":5844434968,"num_objects":89180,"num_object_clones":0,"num_object_copies":267540,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":31674,"num_objects_misplaced":4217,"num_objects_unfound":0,"num_objects_dirty":89180,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":120294,"num_write_kb":6703105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6331,"num_bytes_recovered":414384128,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":17625194496,"data_stored":17625186352,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":167242,"ondisk_log_size":167242,"up":71,"acting":69,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":77,"acting":84,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":35,"num_bytes_recovered":10964,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":25181,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":85,"acting":87,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":713712,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":1,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":192,"num_read_kb":360,"num_write":95,"num_write_kb":953,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":1025388,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2109440,"data_stored":2098784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3381,"ondisk_log_size":3381,"up":58,"acting":59,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":17,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6082,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":20,"acting":19,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201496,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2693236,"kb_used_data":2501548,"kb_used_omap":0,"kb_used_meta":191616,"kb_avail":91051916,"statfs":{"total":95995035648,"available":93237161984,"internally_reserved":0,"allocated":2561585152,"data_stored":2560786244,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":196214784},"hb_peers":[0,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:28:27 2023","interfaces":[{"interface":"back","average":{"1min":1.073,"5min":2.004,"15min":1.048},"min":{"1min":0.20799999999999999,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":4.4829999999999997,"5min":48.798999999999999,"15min":48.798999999999999},"last":5.4859999999999998},{"interface":"front","average":{"1min":1.0589999999999999,"5min":2.3690000000000002,"15min":1.1679999999999999},"min":{"1min":0.30499999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":3.7839999999999998,"5min":48.247,"15min":48.247},"last":0.55500000000000005}]},{"osd":1,"last update":"Sat Sep 23 20:27:59 2023","interfaces":[{"interface":"back","average":{"1min":1.1759999999999999,"5min":1.7689999999999999,"15min":0.96599999999999997},"min":{"1min":0.29199999999999998,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":5.1509999999999998,"5min":17.795999999999999,"15min":17.795999999999999},"last":3.1429999999999998},{"interface":"front","average":{"1min":0.75900000000000001,"5min":1.5349999999999999,"15min":0.91900000000000004},"min":{"1min":0.33700000000000002,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":1.9219999999999999,"5min":13.250999999999999,"15min":13.250999999999999},"last":0.33100000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:28:32 2023","interfaces":[{"interface":"back","average":{"1min":1.9319999999999999,"5min":1.944,"15min":1.034},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":11.372999999999999,"5min":11.372999999999999,"15min":11.372999999999999},"last":0.5},{"interface":"front","average":{"1min":1.621,"5min":2.056,"15min":1.091},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":5.3200000000000003,"5min":11.423999999999999,"15min":11.423999999999999},"last":0.26600000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:29:01 2023","interfaces":[{"interface":"back","average":{"1min":1.8129999999999999,"5min":2.0270000000000001,"15min":1.143},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":8.7810000000000006,"5min":8.7810000000000006,"15min":8.7810000000000006},"last":0.29199999999999998},{"interface":"front","average":{"1min":2.0339999999999998,"5min":2.0339999999999998,"15min":1.133},"min":{"1min":0.40799999999999997,"5min":0.32500000000000001,"15min":0.27600000000000002},"max":{"1min":8.6419999999999995,"5min":9.7840000000000007,"15min":9.7840000000000007},"last":0.53200000000000003}]},{"osd":4,"last update":"Sat Sep 23 20:29:01 2023","interfaces":[{"interface":"back","average":{"1min":1.153,"5min":1.2989999999999999,"15min":0.78100000000000003},"min":{"1min":0.28499999999999998,"5min":0.27100000000000002,"15min":0.26700000000000002},"max":{"1min":5.8550000000000004,"5min":13.141999999999999,"15min":13.141999999999999},"last":5.3860000000000001},{"interface":"front","average":{"1min":1.399,"5min":1.8740000000000001,"15min":1.0009999999999999},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":11.122,"5min":11.122,"15min":11.122},"last":0.46800000000000003}]},{"osd":5,"last update":"Sat Sep 23 20:29:01 2023","interfaces":[{"interface":"back","average":{"1min":1.107,"5min":1.8380000000000001,"15min":1.0029999999999999},"min":{"1min":0.28999999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":3.4710000000000001,"5min":13.241,"15min":13.241},"last":1.74},{"interface":"front","average":{"1min":1.51,"5min":1.869,"15min":1.0309999999999999},"min":{"1min":0.32200000000000001,"5min":0.315,"15min":0.315},"max":{"1min":6.5490000000000004,"5min":14.012,"15min":14.012},"last":5.4530000000000003}]},{"osd":6,"last update":"Sat Sep 23 20:29:01 2023","interfaces":[{"interface":"back","average":{"1min":1.1419999999999999,"5min":1.54,"15min":0.91900000000000004},"min":{"1min":0.35399999999999998,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":6.4100000000000001,"5min":13.185,"15min":13.185},"last":5.5209999999999999},{"interface":"front","average":{"1min":1.206,"5min":1.55,"15min":0.94799999999999995},"min":{"1min":0.36299999999999999,"5min":0.33000000000000002,"15min":0.29899999999999999},"max":{"1min":6.0529999999999999,"5min":9.9120000000000008,"15min":9.9120000000000008},"last":0.57799999999999996}]}]},{"osd":6,"up_from":47,"seq":201863463132,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1613608,"kb_used_data":1482272,"kb_used_omap":0,"kb_used_meta":131328,"kb_avail":92131544,"statfs":{"total":95995035648,"available":94342701056,"internally_reserved":0,"allocated":1517846528,"data_stored":1517059762,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":134479872},"hb_peers":[0,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:28:29 2023","interfaces":[{"interface":"back","average":{"1min":1.038,"5min":1.405,"15min":0.85899999999999999},"min":{"1min":0.28599999999999998,"5min":0.221,"15min":0.221},"max":{"1min":3.7210000000000001,"5min":8.1940000000000008,"15min":8.1940000000000008},"last":0.26300000000000001},{"interface":"front","average":{"1min":1.8879999999999999,"5min":1.9259999999999999,"15min":1.0309999999999999},"min":{"1min":0.34599999999999997,"5min":0.25,"15min":0.25},"max":{"1min":8.6760000000000002,"5min":12.512,"15min":12.512},"last":0.5}]},{"osd":1,"last update":"Sat Sep 23 20:28:03 2023","interfaces":[{"interface":"back","average":{"1min":1.085,"5min":1.9850000000000001,"15min":1.0449999999999999},"min":{"1min":0.28299999999999997,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":6.149,"5min":18.001000000000001,"15min":18.001000000000001},"last":20.106999999999999},{"interface":"front","average":{"1min":1.244,"5min":1.7070000000000001,"15min":0.93400000000000005},"min":{"1min":0.31,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":7.008,"5min":13.91,"15min":13.91},"last":0.98799999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:28:29 2023","interfaces":[{"interface":"back","average":{"1min":1.907,"5min":1.8100000000000001,"15min":1.002},"min":{"1min":0.33200000000000002,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":4.0679999999999996,"5min":13.856,"15min":13.856},"last":0.45400000000000001},{"interface":"front","average":{"1min":1.4890000000000001,"5min":1.5780000000000001,"15min":0.93999999999999995},"min":{"1min":0.43099999999999999,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":3.4220000000000002,"5min":7.1879999999999997,"15min":7.1879999999999997},"last":0.42399999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:29:05 2023","interfaces":[{"interface":"back","average":{"1min":1.964,"5min":2.0920000000000001,"15min":1.1419999999999999},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":7.7130000000000001,"5min":9.4239999999999995,"15min":9.4239999999999995},"last":0.373},{"interface":"front","average":{"1min":2.27,"5min":2.0299999999999998,"15min":1.1799999999999999},"min":{"1min":0.39800000000000002,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":7.7409999999999997,"5min":8.3770000000000007,"15min":8.6050000000000004},"last":0.47299999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:28:50 2023","interfaces":[{"interface":"back","average":{"1min":0.98599999999999999,"5min":1.3819999999999999,"15min":0.78300000000000003},"min":{"1min":0.28599999999999998,"5min":0.27200000000000002,"15min":0.26200000000000001},"max":{"1min":5.431,"5min":10.173,"15min":10.173},"last":0.39400000000000002},{"interface":"front","average":{"1min":0.83199999999999996,"5min":1.4079999999999999,"15min":0.81299999999999994},"min":{"1min":0.34799999999999998,"5min":0.29799999999999999,"15min":0.28499999999999998},"max":{"1min":2.4009999999999998,"5min":8.3249999999999993,"15min":8.3249999999999993},"last":0.40400000000000003}]},{"osd":5,"last update":"Sat Sep 23 20:28:50 2023","interfaces":[{"interface":"back","average":{"1min":0.57799999999999996,"5min":1.413,"15min":0.81499999999999995},"min":{"1min":0.311,"5min":0.31,"15min":0.29799999999999999},"max":{"1min":2.0670000000000002,"5min":14.085000000000001,"15min":14.085000000000001},"last":0.49299999999999999},{"interface":"front","average":{"1min":1.24,"5min":2.0510000000000002,"15min":1.0589999999999999},"min":{"1min":0.33900000000000002,"5min":0.32800000000000001,"15min":0.29499999999999998},"max":{"1min":4.2640000000000002,"5min":18.192,"15min":18.192},"last":1.1719999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:29:05 2023","interfaces":[{"interface":"back","average":{"1min":1.347,"5min":1.647,"15min":0.93000000000000005},"min":{"1min":0.379,"5min":0.316,"15min":0.316},"max":{"1min":5.3879999999999999,"5min":10.217000000000001,"15min":10.217000000000001},"last":1.125},{"interface":"front","average":{"1min":1.746,"5min":1.7969999999999999,"15min":1.0089999999999999},"min":{"1min":0.39600000000000002,"5min":0.34999999999999998,"15min":0.315},"max":{"1min":7.8120000000000003,"5min":9.7360000000000007,"15min":9.7360000000000007},"last":0.434}]}]},{"osd":1,"up_from":99,"seq":425201762395,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1618524,"kb_used_data":1418708,"kb_used_omap":6,"kb_used_meta":199801,"kb_avail":92126628,"statfs":{"total":95995035648,"available":94337667072,"internally_reserved":0,"allocated":1452756992,"data_stored":1451546929,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6696,"internal_metadata":204596696},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:27:55 2023","interfaces":[{"interface":"back","average":{"1min":2.044,"5min":2.3929999999999998,"15min":1.046},"min":{"1min":0.224,"5min":0.20599999999999999,"15min":0.19600000000000001},"max":{"1min":17.904,"5min":40.113,"15min":40.113},"last":0.28199999999999997},{"interface":"front","average":{"1min":1.8799999999999999,"5min":2.1150000000000002,"15min":0.96899999999999997},"min":{"1min":0.192,"5min":0.192,"15min":0.192},"max":{"1min":17.812000000000001,"5min":42.994999999999997,"15min":42.994999999999997},"last":7.4850000000000003}]},{"osd":2,"last update":"Sat Sep 23 20:27:30 2023","interfaces":[{"interface":"back","average":{"1min":2.0299999999999998,"5min":2.1320000000000001,"15min":0.96499999999999997},"min":{"1min":0.30099999999999999,"5min":0.253,"15min":0.253},"max":{"1min":17.940000000000001,"5min":39.962000000000003,"15min":39.962000000000003},"last":0.59099999999999997},{"interface":"front","average":{"1min":2.3730000000000002,"5min":2.1579999999999999,"15min":1.087},"min":{"1min":0.26300000000000001,"5min":0.246,"15min":0.246},"max":{"1min":15.090999999999999,"5min":47.442,"15min":47.442},"last":8.7899999999999991}]},{"osd":3,"last update":"Sat Sep 23 20:28:02 2023","interfaces":[{"interface":"back","average":{"1min":1.361,"5min":1.5660000000000001,"15min":0.88800000000000001},"min":{"1min":0.32900000000000001,"5min":0.246,"15min":0.246},"max":{"1min":4.0030000000000001,"5min":9.0410000000000004,"15min":9.0410000000000004},"last":0.22500000000000001},{"interface":"front","average":{"1min":2.2000000000000002,"5min":1.8300000000000001,"15min":0.88200000000000001},"min":{"1min":0.248,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":17.853000000000002,"5min":17.853000000000002,"15min":17.853000000000002},"last":7.7439999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:27:55 2023","interfaces":[{"interface":"back","average":{"1min":1.359,"5min":1.7370000000000001,"15min":0.98999999999999999},"min":{"1min":0.33600000000000002,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":4.4550000000000001,"5min":15.353,"15min":15.353},"last":0.32300000000000001},{"interface":"front","average":{"1min":1.071,"5min":1.625,"15min":0.96399999999999997},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":3.6480000000000001,"5min":8.7899999999999991,"15min":8.7899999999999991},"last":0.42499999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:27:55 2023","interfaces":[{"interface":"back","average":{"1min":1.143,"5min":1.72,"15min":1.0009999999999999},"min":{"1min":0.32800000000000001,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":4.6920000000000002,"5min":12.590999999999999,"15min":12.590999999999999},"last":7.5460000000000003},{"interface":"front","average":{"1min":1.605,"5min":2.3250000000000002,"15min":1.2250000000000001},"min":{"1min":0.40100000000000002,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":4.5369999999999999,"5min":39.579000000000001,"15min":39.579000000000001},"last":0.35699999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:27:55 2023","interfaces":[{"interface":"back","average":{"1min":1.6379999999999999,"5min":2.198,"15min":1.175},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":9.0310000000000006,"5min":34.409999999999997,"15min":34.409999999999997},"last":7.5750000000000002},{"interface":"front","average":{"1min":1.6140000000000001,"5min":2.4220000000000002,"15min":1.232},"min":{"1min":0.48599999999999999,"5min":0.377,"15min":0.377},"max":{"1min":4.5860000000000003,"5min":35.554000000000002,"15min":35.554000000000002},"last":0.39000000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:27:55 2023","interfaces":[{"interface":"back","average":{"1min":1.617,"5min":2.044,"15min":1.115},"min":{"1min":0.505,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":4.6139999999999999,"5min":9.7439999999999998,"15min":9.7439999999999998},"last":4.1660000000000004},{"interface":"front","average":{"1min":1.3200000000000001,"5min":1.9079999999999999,"15min":1.089},"min":{"1min":0.41299999999999998,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":5.7439999999999998,"5min":9.2279999999999998,"15min":9.2279999999999998},"last":0.40100000000000002}]}]},{"osd":0,"up_from":94,"seq":403726925933,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1714060,"kb_used_data":1457156,"kb_used_omap":7,"kb_used_meta":256888,"kb_avail":92031092,"statfs":{"total":95995035648,"available":94239838208,"internally_reserved":0,"allocated":1492127744,"data_stored":1490901560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":263053329},"hb_peers":[2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:27:57 2023","interfaces":[{"interface":"back","average":{"1min":0.97099999999999997,"5min":1.4650000000000001,"15min":0.72599999999999998},"min":{"1min":0.26300000000000001,"5min":0.20100000000000001,"15min":0.17699999999999999},"max":{"1min":4.7060000000000004,"5min":24.247,"15min":24.247},"last":4.3479999999999999},{"interface":"front","average":{"1min":0.98899999999999999,"5min":1.4339999999999999,"15min":0.73599999999999999},"min":{"1min":0.27900000000000003,"5min":0.222,"15min":0.222},"max":{"1min":2.9289999999999998,"5min":15.994999999999999,"15min":15.994999999999999},"last":6.4279999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:28:25 2023","interfaces":[{"interface":"back","average":{"1min":2.8959999999999999,"5min":2.3940000000000001,"15min":1.1990000000000001},"min":{"1min":0.19800000000000001,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":9.4459999999999997,"5min":23.832999999999998,"15min":23.832999999999998},"last":0.39400000000000002},{"interface":"front","average":{"1min":2.9289999999999998,"5min":2.0430000000000001,"15min":1.028},"min":{"1min":0.20999999999999999,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":13.337,"5min":15.948,"15min":15.948},"last":1.3400000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:28:59 2023","interfaces":[{"interface":"back","average":{"1min":4.798,"5min":2.5550000000000002,"15min":1.2609999999999999},"min":{"1min":0.36399999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":15.558,"5min":23.748999999999999,"15min":23.748999999999999},"last":0.61399999999999999},{"interface":"front","average":{"1min":3.9820000000000002,"5min":2.3260000000000001,"15min":1.071},"min":{"1min":0.36699999999999999,"5min":0.28000000000000003,"15min":0.23899999999999999},"max":{"1min":15.061,"5min":24.274999999999999,"15min":24.274999999999999},"last":1.948}]},{"osd":4,"last update":"Sat Sep 23 20:28:25 2023","interfaces":[{"interface":"back","average":{"1min":2.5310000000000001,"5min":2.4750000000000001,"15min":1.238},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":9.5459999999999994,"5min":23.876999999999999,"15min":23.876999999999999},"last":0.43099999999999999},{"interface":"front","average":{"1min":1.514,"5min":1.944,"15min":1.0589999999999999},"min":{"1min":0.312,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":4.6609999999999996,"5min":23.681999999999999,"15min":23.681999999999999},"last":2.0089999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:28:25 2023","interfaces":[{"interface":"back","average":{"1min":1.867,"5min":2.7109999999999999,"15min":1.343},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":4.774,"5min":24.198,"15min":24.198},"last":2.077},{"interface":"front","average":{"1min":2.6739999999999999,"5min":2.254,"15min":1.202},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":6.694,"5min":11.167999999999999,"15min":11.167999999999999},"last":1.4219999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:28:25 2023","interfaces":[{"interface":"back","average":{"1min":2.0870000000000002,"5min":2.0470000000000002,"15min":1.1220000000000001},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":4.8360000000000003,"5min":24.408000000000001,"15min":24.408000000000001},"last":2.044},{"interface":"front","average":{"1min":2.4409999999999998,"5min":1.9650000000000001,"15min":1.1000000000000001},"min":{"1min":0.38800000000000001,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":6.3620000000000001,"5min":25.271000000000001,"15min":25.271000000000001},"last":1.4490000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:28:25 2023","interfaces":[{"interface":"back","average":{"1min":2.2639999999999998,"5min":2.133,"15min":1.163},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":7.9429999999999996,"5min":25.359000000000002,"15min":25.359000000000002},"last":24.614000000000001},{"interface":"front","average":{"1min":2.6669999999999998,"5min":2.7120000000000002,"15min":1.3600000000000001},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":14.509,"5min":23.905999999999999,"15min":23.905999999999999},"last":24.669}]}]},{"osd":2,"up_from":104,"seq":446676598881,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3577460,"kb_used_data":3230956,"kb_used_omap":11,"kb_used_meta":346484,"kb_avail":90167692,"statfs":{"total":95995035648,"available":92331716608,"internally_reserved":0,"allocated":3308498944,"data_stored":3307293754,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11878,"internal_metadata":354800026},"hb_peers":[0,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:28:33 2023","interfaces":[{"interface":"back","average":{"1min":2.7869999999999999,"5min":2.1619999999999999,"15min":1.012},"min":{"1min":0.19500000000000001,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":21.695,"5min":21.695,"15min":21.695},"last":0.39200000000000002},{"interface":"front","average":{"1min":2.109,"5min":1.5680000000000001,"15min":0.82599999999999996},"min":{"1min":0.25900000000000001,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":8.6340000000000003,"5min":15.746,"15min":15.746},"last":1.071}]},{"osd":1,"last update":"Sat Sep 23 20:27:30 2023","interfaces":[{"interface":"back","average":{"1min":0.93799999999999994,"5min":1.633,"15min":0.80700000000000005},"min":{"1min":0.29799999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":3.3410000000000002,"5min":14.135999999999999,"15min":14.135999999999999},"last":8.8049999999999997},{"interface":"front","average":{"1min":1.863,"5min":1.522,"15min":0.76200000000000001},"min":{"1min":0.27500000000000002,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":14.201000000000001,"5min":14.201000000000001,"15min":14.201000000000001},"last":2.1640000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:29:01 2023","interfaces":[{"interface":"back","average":{"1min":1.758,"5min":1.7050000000000001,"15min":0.86599999999999999},"min":{"1min":0.27000000000000002,"5min":0.26500000000000001,"15min":0.23699999999999999},"max":{"1min":15.17,"5min":15.319000000000001,"15min":15.319000000000001},"last":0.19800000000000001},{"interface":"front","average":{"1min":3.8639999999999999,"5min":2.3620000000000001,"15min":1.123},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":21.594000000000001,"5min":21.594000000000001,"15min":21.594000000000001},"last":0.35299999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:28:33 2023","interfaces":[{"interface":"back","average":{"1min":1.889,"5min":1.633,"15min":0.94499999999999995},"min":{"1min":0.44900000000000001,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":10.84,"5min":10.84,"15min":10.84},"last":1.1679999999999999},{"interface":"front","average":{"1min":2.7320000000000002,"5min":1.8,"15min":1.0029999999999999},"min":{"1min":0.28799999999999998,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":14.971,"5min":14.971,"15min":14.971},"last":0.69099999999999995}]},{"osd":5,"last update":"Sat Sep 23 20:28:33 2023","interfaces":[{"interface":"back","average":{"1min":1.5669999999999999,"5min":1.788,"15min":1.046},"min":{"1min":0.42299999999999999,"5min":0.309,"15min":0.309},"max":{"1min":5.399,"5min":11.237,"15min":11.237},"last":5.0449999999999999},{"interface":"front","average":{"1min":2.698,"5min":2.0249999999999999,"15min":1.133},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":15.057,"5min":15.057,"15min":15.057},"last":5.0570000000000004}]},{"osd":6,"last update":"Sat Sep 23 20:28:33 2023","interfaces":[{"interface":"back","average":{"1min":2.8130000000000002,"5min":1.728,"15min":1.0289999999999999},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":15.023999999999999,"5min":15.023999999999999,"15min":15.023999999999999},"last":1.9099999999999999},{"interface":"front","average":{"1min":3.1379999999999999,"5min":2.2080000000000002,"15min":1.1890000000000001},"min":{"1min":0.45600000000000002,"5min":0.377,"15min":0.377},"max":{"1min":21.663,"5min":21.663,"15min":21.663},"last":1.1870000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:28:33 2023","interfaces":[{"interface":"back","average":{"1min":3.2570000000000001,"5min":2.3180000000000001,"15min":1.226},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":21.637,"5min":21.637,"15min":21.637},"last":2.0369999999999999},{"interface":"front","average":{"1min":1.907,"5min":1.8700000000000001,"15min":1.085},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":6.0049999999999999,"5min":10.064,"15min":10.064},"last":1.2050000000000001}]}]},{"osd":3,"up_from":109,"seq":468151435354,"num_pgs":65,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3963908,"kb_used_data":3655292,"kb_used_omap":4,"kb_used_meta":308475,"kb_avail":89781244,"statfs":{"total":95995035648,"available":91935993856,"internally_reserved":0,"allocated":3743019008,"data_stored":3741798240,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4514,"internal_metadata":315879006},"hb_peers":[0,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,0,0,0,3],"upper_bound":2048},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:29:02 2023","interfaces":[{"interface":"back","average":{"1min":1.9770000000000001,"5min":2.6339999999999999,"15min":1.2150000000000001},"min":{"1min":0.186,"5min":0.186,"15min":0.186},"max":{"1min":9.0960000000000001,"5min":67.305999999999997,"15min":67.305999999999997},"last":0.25},{"interface":"front","average":{"1min":2.2839999999999998,"5min":2.2989999999999999,"15min":1.129},"min":{"1min":0.224,"5min":0.224,"15min":0.224},"max":{"1min":13.743,"5min":67.403999999999996,"15min":67.403999999999996},"last":0.33300000000000002}]},{"osd":1,"last update":"Sat Sep 23 20:27:59 2023","interfaces":[{"interface":"back","average":{"1min":1.026,"5min":1.633,"15min":0.80900000000000005},"min":{"1min":0.221,"5min":0.214,"15min":0.214},"max":{"1min":4.6390000000000002,"5min":19.675999999999998,"15min":19.675999999999998},"last":1.0169999999999999},{"interface":"front","average":{"1min":1.371,"5min":1.5600000000000001,"15min":0.81599999999999995},"min":{"1min":0.29099999999999998,"5min":0.245,"15min":0.245},"max":{"1min":7.0800000000000001,"5min":11.644,"15min":11.644},"last":3.2770000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:29:02 2023","interfaces":[{"interface":"back","average":{"1min":2.569,"5min":1.982,"15min":1.05},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":13.523999999999999,"5min":24.045000000000002,"15min":24.045000000000002},"last":0.30199999999999999},{"interface":"front","average":{"1min":2.2730000000000001,"5min":2.0219999999999998,"15min":1.0840000000000001},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":11.526999999999999,"5min":12.427,"15min":17.684000000000001},"last":0.26400000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:29:02 2023","interfaces":[{"interface":"back","average":{"1min":1.6970000000000001,"5min":2.1200000000000001,"15min":1.1299999999999999},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":8.8010000000000002,"5min":46.042999999999999,"15min":46.042999999999999},"last":0.72799999999999998},{"interface":"front","average":{"1min":2.0790000000000002,"5min":2.5249999999999999,"15min":1.27},"min":{"1min":0.33500000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":5.3220000000000001,"5min":46.579999999999998,"15min":46.579999999999998},"last":0.78700000000000003}]},{"osd":5,"last update":"Sat Sep 23 20:29:02 2023","interfaces":[{"interface":"back","average":{"1min":2.077,"5min":2.4820000000000002,"15min":1.29},"min":{"1min":0.36699999999999999,"5min":0.311,"15min":0.311},"max":{"1min":8.8819999999999997,"5min":45.292000000000002,"15min":45.292000000000002},"last":0.65600000000000003},{"interface":"front","average":{"1min":2.8719999999999999,"5min":2.0779999999999998,"15min":1.171},"min":{"1min":0.378,"5min":0.313,"15min":0.313},"max":{"1min":13.428000000000001,"5min":44.679000000000002,"15min":44.679000000000002},"last":0.53400000000000003}]},{"osd":6,"last update":"Sat Sep 23 20:29:02 2023","interfaces":[{"interface":"back","average":{"1min":2.8010000000000002,"5min":1.9850000000000001,"15min":1.141},"min":{"1min":0.42299999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":13.475,"5min":13.475,"15min":13.475},"last":0.57099999999999995},{"interface":"front","average":{"1min":2.1520000000000001,"5min":1.8899999999999999,"15min":1.0980000000000001},"min":{"1min":0.374,"5min":0.374,"15min":0.35499999999999998},"max":{"1min":5.9829999999999997,"5min":9.3179999999999996,"15min":9.3179999999999996},"last":0.71299999999999997}]},{"osd":7,"last update":"Sat Sep 23 20:29:02 2023","interfaces":[{"interface":"back","average":{"1min":2.3439999999999999,"5min":1.9770000000000001,"15min":1.1459999999999999},"min":{"1min":0.441,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":7.1859999999999999,"5min":9.2750000000000004,"15min":9.2750000000000004},"last":0.745},{"interface":"front","average":{"1min":2.0459999999999998,"5min":2.7029999999999998,"15min":1.3819999999999999},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":8.8460000000000001,"5min":45.387999999999998,"15min":45.387999999999998},"last":0.61599999999999999}]}]},{"osd":4,"up_from":30,"seq":128849019108,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1960556,"kb_used_data":1792228,"kb_used_omap":0,"kb_used_meta":168320,"kb_avail":91784596,"statfs":{"total":95995035648,"available":93987426304,"internally_reserved":0,"allocated":1835241472,"data_stored":1834450363,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":172359680},"hb_peers":[0,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:28:28 2023","interfaces":[{"interface":"back","average":{"1min":1.7070000000000001,"5min":1.9259999999999999,"15min":1.022},"min":{"1min":0.25600000000000001,"5min":0.222,"15min":0.222},"max":{"1min":8.2899999999999991,"5min":20.411999999999999,"15min":20.411999999999999},"last":0.41499999999999998},{"interface":"front","average":{"1min":1.778,"5min":2.004,"15min":1.091},"min":{"1min":0.307,"5min":0.253,"15min":0.253},"max":{"1min":6.899,"5min":20.454999999999998,"15min":20.454999999999998},"last":3.395}]},{"osd":1,"last update":"Sat Sep 23 20:28:01 2023","interfaces":[{"interface":"back","average":{"1min":1.7509999999999999,"5min":1.748,"15min":0.995},"min":{"1min":0.35699999999999998,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":8.7249999999999996,"5min":9.6660000000000004,"15min":9.6660000000000004},"last":3.4529999999999998},{"interface":"front","average":{"1min":0.70799999999999996,"5min":1.595,"15min":0.94399999999999995},"min":{"1min":0.32700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":2.2149999999999999,"5min":12.821999999999999,"15min":12.821999999999999},"last":0.30599999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:28:37 2023","interfaces":[{"interface":"back","average":{"1min":1.194,"5min":1.913,"15min":1.048},"min":{"1min":0.29899999999999999,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":4.7130000000000001,"5min":21.462,"15min":21.462},"last":0.44800000000000001},{"interface":"front","average":{"1min":2.2440000000000002,"5min":2.3490000000000002,"15min":1.202},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.26800000000000002},"max":{"1min":7.6619999999999999,"5min":22.593,"15min":22.593},"last":3.48}]},{"osd":3,"last update":"Sat Sep 23 20:29:01 2023","interfaces":[{"interface":"back","average":{"1min":3.2690000000000001,"5min":2.4249999999999998,"15min":1.2450000000000001},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":32.332999999999998,"5min":32.332999999999998,"15min":32.332999999999998},"last":0.49399999999999999},{"interface":"front","average":{"1min":4.165,"5min":2.6280000000000001,"15min":1.323},"min":{"1min":0.39500000000000002,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":32.399999999999999,"5min":32.399999999999999,"15min":32.399999999999999},"last":0.47099999999999997}]},{"osd":5,"last update":"Sat Sep 23 20:28:15 2023","interfaces":[{"interface":"back","average":{"1min":1.1479999999999999,"5min":2.0099999999999998,"15min":0.997},"min":{"1min":0.26900000000000002,"5min":0.247,"15min":0.247},"max":{"1min":4.149,"5min":17.704000000000001,"15min":17.704000000000001},"last":1.452},{"interface":"front","average":{"1min":1.3640000000000001,"5min":1.8100000000000001,"15min":0.94299999999999995},"min":{"1min":0.35899999999999999,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":8.1379999999999999,"5min":15.329000000000001,"15min":15.329000000000001},"last":3.5329999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:28:37 2023","interfaces":[{"interface":"back","average":{"1min":1.4179999999999999,"5min":2.2189999999999999,"15min":1.095},"min":{"1min":0.33100000000000002,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":3.8140000000000001,"5min":41.371000000000002,"15min":41.371000000000002},"last":3.4489999999999998},{"interface":"front","average":{"1min":1.341,"5min":1.9570000000000001,"15min":1.018},"min":{"1min":0.318,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":4.2160000000000002,"5min":41.140999999999998,"15min":41.140999999999998},"last":3.2010000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:28:54 2023","interfaces":[{"interface":"back","average":{"1min":1.637,"5min":2.1280000000000001,"15min":1.091},"min":{"1min":0.38200000000000001,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":4.0430000000000001,"5min":16.427,"15min":16.427},"last":3.5089999999999999},{"interface":"front","average":{"1min":1.7350000000000001,"5min":2.0289999999999999,"15min":1.0669999999999999},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":5.0999999999999996,"5min":19.018999999999998,"15min":19.018999999999998},"last":1.4950000000000001}]}]},{"osd":5,"up_from":39,"seq":167503724769,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1968492,"kb_used_data":1788260,"kb_used_omap":0,"kb_used_meta":180224,"kb_avail":91776660,"statfs":{"total":95995035648,"available":93979299840,"internally_reserved":0,"allocated":1831178240,"data_stored":1830383785,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":184549376},"hb_peers":[0,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,6],"upper_bound":128},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:28:29 2023","interfaces":[{"interface":"back","average":{"1min":1.488,"5min":1.843,"15min":1.0129999999999999},"min":{"1min":0.26100000000000001,"5min":0.24099999999999999,"15min":0.23499999999999999},"max":{"1min":6.6479999999999997,"5min":32.475000000000001,"15min":32.475000000000001},"last":0.318},{"interface":"front","average":{"1min":1.728,"5min":1.8600000000000001,"15min":1.0309999999999999},"min":{"1min":0.33600000000000002,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":8.4900000000000002,"5min":32.457000000000001,"15min":32.457000000000001},"last":0.36199999999999999}]},{"osd":1,"last update":"Sat Sep 23 20:28:01 2023","interfaces":[{"interface":"back","average":{"1min":1.486,"5min":2.0059999999999998,"15min":1.032},"min":{"1min":0.25600000000000001,"5min":0.245,"15min":0.245},"max":{"1min":11.958,"5min":32.243000000000002,"15min":32.243000000000002},"last":2.0009999999999999},{"interface":"front","average":{"1min":1.5509999999999999,"5min":1.548,"15min":0.88100000000000001},"min":{"1min":0.29599999999999999,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":12.901,"5min":32.231000000000002,"15min":32.231000000000002},"last":8.532}]},{"osd":2,"last update":"Sat Sep 23 20:28:29 2023","interfaces":[{"interface":"back","average":{"1min":1.079,"5min":2.1840000000000002,"15min":1.1160000000000001},"min":{"1min":0.314,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":3.5,"5min":34.514000000000003,"15min":34.514000000000003},"last":0.28199999999999997},{"interface":"front","average":{"1min":2.093,"5min":2.1150000000000002,"15min":1.0940000000000001},"min":{"1min":0.35999999999999999,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":8.6820000000000004,"5min":32.515000000000001,"15min":32.515000000000001},"last":0.40100000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:29:02 2023","interfaces":[{"interface":"back","average":{"1min":1.518,"5min":1.8600000000000001,"15min":1.024},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":3.7349999999999999,"5min":12.846,"15min":12.846},"last":0.65200000000000002},{"interface":"front","average":{"1min":1.51,"5min":1.8799999999999999,"15min":1.048},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":7.6299999999999999,"5min":12.005000000000001,"15min":12.005000000000001},"last":0.35099999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:28:23 2023","interfaces":[{"interface":"back","average":{"1min":0.93000000000000005,"5min":1.925,"15min":0.97399999999999998},"min":{"1min":0.31,"5min":0.26700000000000002,"15min":0.26400000000000001},"max":{"1min":3.726,"5min":14.218999999999999,"15min":14.218999999999999},"last":0.56399999999999995},{"interface":"front","average":{"1min":0.78100000000000003,"5min":1.536,"15min":0.86499999999999999},"min":{"1min":0.33700000000000002,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":1.9670000000000001,"5min":11.664,"15min":11.664},"last":0.38200000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:28:35 2023","interfaces":[{"interface":"back","average":{"1min":1.3720000000000001,"5min":1.9259999999999999,"15min":1.0009999999999999},"min":{"1min":0.32900000000000001,"5min":0.30499999999999999,"15min":0.29799999999999999},"max":{"1min":5.9450000000000003,"5min":12.890000000000001,"15min":12.890000000000001},"last":0.70899999999999996},{"interface":"front","average":{"1min":1.0109999999999999,"5min":1.708,"15min":0.97699999999999998},"min":{"1min":0.36399999999999999,"5min":0.35999999999999999,"15min":0.34499999999999997},"max":{"1min":5.8920000000000003,"5min":11.456,"15min":11.456},"last":0.69799999999999995}]},{"osd":7,"last update":"Sat Sep 23 20:29:02 2023","interfaces":[{"interface":"back","average":{"1min":0.78500000000000003,"5min":2.0249999999999999,"15min":1.0660000000000001},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":2.294,"5min":36.621000000000002,"15min":36.621000000000002},"last":2.847},{"interface":"front","average":{"1min":1.0049999999999999,"5min":1.7010000000000001,"15min":0.95899999999999996},"min":{"1min":0.376,"5min":0.316,"15min":0.315},"max":{"1min":4.8360000000000003,"5min":37.331000000000003,"15min":37.331000000000003},"last":0.68100000000000005}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4722,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1360,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6696,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7156,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3154,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1474957312,"data_stored":1474953240,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1455095808,"data_stored":1455095808,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":3289120768,"data_stored":3289120768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3733520384,"data_stored":3733520384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1827471360,"data_stored":1827471360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1819086848,"data_stored":1819082776,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1496055808,"data_stored":1496055808,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2529886208,"data_stored":2529886208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:29:12.819 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd pool set unique_pool_1 pgp_num 26 2023-09-23T20:29:12.949 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:29:13.225 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:29:13.231 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:13.231 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:29:13.332 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:29:13.384 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:13 smithi018 ceph-mon[184144]: pgmap v563: 119 pgs: 11 active+clean+remapped, 2 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+remapped, 22 active+undersized, 2 active+recovery_wait+degraded+remapped, 65 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 911 op/s; 31688/268578 objects degraded (11.798%); 4235/268578 objects misplaced (1.577%); 8.2 MiB/s, 143 objects/s recovering 2023-09-23T20:29:13.384 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:13 smithi018 ceph-mon[184144]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:29:13.384 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:13 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:29:13.384 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:13 smithi018 ceph-mon[184144]: osdmap e255: 8 total, 7 up, 8 in 2023-09-23T20:29:13.384 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:13 smithi018 ceph-mon[184144]: failed to encode map e255 with expected crc 2023-09-23T20:29:13.384 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:13 smithi018 ceph-mon[184144]: failed to encode map e255 with expected crc 2023-09-23T20:29:13.385 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:13 smithi018 ceph-mon[184144]: failed to encode map e255 with expected crc 2023-09-23T20:29:13.385 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:13 smithi018 ceph-mon[184144]: from='client.35439 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:29:13.385 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:13 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:29:13.385 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:13 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:29:13.385 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:13 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:29:13.385 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:13 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:29:13.385 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:13 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:29:13.385 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:13 smithi018 ceph-mon[179647]: pgmap v563: 119 pgs: 11 active+clean+remapped, 2 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+remapped, 22 active+undersized, 2 active+recovery_wait+degraded+remapped, 65 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 911 op/s; 31688/268578 objects degraded (11.798%); 4235/268578 objects misplaced (1.577%); 8.2 MiB/s, 143 objects/s recovering 2023-09-23T20:29:13.386 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:13 smithi018 ceph-mon[179647]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:29:13.386 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:13 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:29:13.386 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:13 smithi018 ceph-mon[179647]: osdmap e255: 8 total, 7 up, 8 in 2023-09-23T20:29:13.386 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:13 smithi018 ceph-mon[179647]: failed to encode map e255 with expected crc 2023-09-23T20:29:13.386 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:13 smithi018 ceph-mon[179647]: failed to encode map e255 with expected crc 2023-09-23T20:29:13.386 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:13 smithi018 ceph-mon[179647]: failed to encode map e255 with expected crc 2023-09-23T20:29:13.386 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:13 smithi018 ceph-mon[179647]: from='client.35439 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:29:13.386 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:13 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:29:13.386 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:13 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:29:13.386 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:13 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:29:13.387 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:13 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:29:13.387 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:13 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:29:13.387 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:13.444 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:29:13.449 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:13.449 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:29:13.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:13 smithi149 ceph-mon[170394]: pgmap v563: 119 pgs: 11 active+clean+remapped, 2 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+remapped, 22 active+undersized, 2 active+recovery_wait+degraded+remapped, 65 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 911 op/s; 31688/268578 objects degraded (11.798%); 4235/268578 objects misplaced (1.577%); 8.2 MiB/s, 143 objects/s recovering 2023-09-23T20:29:13.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:13 smithi149 ceph-mon[170394]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:29:13.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:13 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:29:13.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:13 smithi149 ceph-mon[170394]: osdmap e255: 8 total, 7 up, 8 in 2023-09-23T20:29:13.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:13 smithi149 ceph-mon[170394]: failed to encode map e255 with expected crc 2023-09-23T20:29:13.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:13 smithi149 ceph-mon[170394]: failed to encode map e255 with expected crc 2023-09-23T20:29:13.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:13 smithi149 ceph-mon[170394]: failed to encode map e255 with expected crc 2023-09-23T20:29:13.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:13 smithi149 ceph-mon[170394]: from='client.35439 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:29:13.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:13 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:29:13.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:13 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:29:13.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:13 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:29:13.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:13 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:29:13.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:13 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:29:13.550 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:29:13.598 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:13.620 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:29:13.635 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:29:13.744 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:29:13.760 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:13.761 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:29:13.862 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:29:13.911 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:13.964 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:29:13.971 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:13.971 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:29:14.072 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:29:14.150 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:14.227 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:14.227 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:29:14.228 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:29:14.328 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:29:14.405 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:14 smithi149 ceph-mon[170394]: 1.e scrub starts 2023-09-23T20:29:14.413 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:14.462 INFO:teuthology.orchestra.run.smithi149.stdout:{ 2023-09-23T20:29:14.462 INFO:teuthology.orchestra.run.smithi149.stdout: "ops": [ 2023-09-23T20:29:14.462 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:14.462 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_repop(client.45161.0:124182 6.c e255/250 6:341a7909:::benchmark_data_smithi018_270558_object19443:head v 255'7623, mlcod=232'4153)", 2023-09-23T20:29:14.462 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:29:14.447221+0000", 2023-09-23T20:29:14.463 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 0.014512493, 2023-09-23T20:29:14.463 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.014536874, 2023-09-23T20:29:14.463 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:29:14.463 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "started", 2023-09-23T20:29:14.463 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:29:14.463 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:14.463 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:29:14.463 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:14.447221+0000", 2023-09-23T20:29:14.463 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:29:14.463 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:29:14.463 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:14.464 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:29:14.464 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:14.447221+0000", 2023-09-23T20:29:14.464 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:29:14.464 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:29:14.464 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:14.464 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:29:14.464 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:14.447220+0000", 2023-09-23T20:29:14.464 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:29:14.464 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:29:14.464 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:14.464 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:29:14.464 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:14.447232+0000", 2023-09-23T20:29:14.465 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.2011000000000001e-05 2023-09-23T20:29:14.465 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:29:14.465 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:14.465 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:29:14.465 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:14.447246+0000", 2023-09-23T20:29:14.465 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.4099e-05 2023-09-23T20:29:14.465 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:29:14.465 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:14.465 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:29:14.465 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:14.447253+0000", 2023-09-23T20:29:14.466 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.0480000000000001e-06 2023-09-23T20:29:14.466 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:29:14.466 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:14.466 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:29:14.466 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:14.447305+0000", 2023-09-23T20:29:14.466 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.1388999999999999e-05 2023-09-23T20:29:14.466 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:29:14.466 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:14.466 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:29:14.466 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:14.447334+0000", 2023-09-23T20:29:14.466 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.9357e-05 2023-09-23T20:29:14.466 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:29:14.467 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:29:14.467 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:29:14.467 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:29:14.467 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:14.467 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.45161.0:124185 6.17 6:e9e18e45:::benchmark_data_smithi018_270558_object19446:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e255)", 2023-09-23T20:29:14.467 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:29:14.447288+0000", 2023-09-23T20:29:14.467 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 0.014446318, 2023-09-23T20:29:14.467 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.01451565, 2023-09-23T20:29:14.467 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:29:14.467 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:29:14.467 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:29:14.468 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.45161", 2023-09-23T20:29:14.468 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3795614987", 2023-09-23T20:29:14.468 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 124185 2023-09-23T20:29:14.468 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:29:14.468 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:29:14.468 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:14.468 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:29:14.468 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:14.447288+0000", 2023-09-23T20:29:14.468 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:29:14.468 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:29:14.468 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:14.468 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:29:14.469 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:14.447288+0000", 2023-09-23T20:29:14.469 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:29:14.469 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:29:14.469 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:14.469 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:29:14.469 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:14.447286+0000", 2023-09-23T20:29:14.469 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999981 2023-09-23T20:29:14.469 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:29:14.469 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:14.469 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:29:14.469 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:14.447291+0000", 2023-09-23T20:29:14.470 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.5799999999999999e-06 2023-09-23T20:29:14.470 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:29:14.470 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:14.470 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:29:14.470 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:14.447300+0000", 2023-09-23T20:29:14.470 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.1309999999999995e-06 2023-09-23T20:29:14.470 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:29:14.470 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:14.470 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:29:14.470 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:14.447306+0000", 2023-09-23T20:29:14.470 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.434e-06 2023-09-23T20:29:14.471 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:29:14.471 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:14.471 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:29:14.471 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:14.447400+0000", 2023-09-23T20:29:14.471 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.4473000000000002e-05 2023-09-23T20:29:14.471 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:29:14.471 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:14.471 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:29:14.471 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:14.447507+0000", 2023-09-23T20:29:14.471 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000106872 2023-09-23T20:29:14.471 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:29:14.472 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:14.472 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:29:14.472 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:14.447614+0000", 2023-09-23T20:29:14.472 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000106498 2023-09-23T20:29:14.472 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:29:14.472 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:14.472 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:29:14.472 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:14.461687+0000", 2023-09-23T20:29:14.472 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.014073242999999999 2023-09-23T20:29:14.472 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:29:14.473 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:29:14.473 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:29:14.473 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:29:14.473 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:14.473 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.45161.0:124186 6.17 6:ef60e828:::benchmark_data_smithi018_270558_object19447:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e255)", 2023-09-23T20:29:14.473 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:29:14.447324+0000", 2023-09-23T20:29:14.473 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 0.014409952, 2023-09-23T20:29:14.473 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.014530428999999999, 2023-09-23T20:29:14.473 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:29:14.473 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:29:14.473 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:29:14.474 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.45161", 2023-09-23T20:29:14.474 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3795614987", 2023-09-23T20:29:14.474 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 124186 2023-09-23T20:29:14.474 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:29:14.474 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:29:14.474 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:14.474 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:29:14.474 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:14.447324+0000", 2023-09-23T20:29:14.474 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:29:14.474 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:29:14.475 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:14.475 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:29:14.475 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:14.447324+0000", 2023-09-23T20:29:14.475 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:29:14.475 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:29:14.475 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:14.475 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:29:14.475 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:14.447324+0000", 2023-09-23T20:29:14.475 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:29:14.475 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:29:14.475 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:14.475 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:29:14.476 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:14.447327+0000", 2023-09-23T20:29:14.476 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.1939999999999998e-06 2023-09-23T20:29:14.476 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:29:14.476 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:14.476 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:29:14.476 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:14.447333+0000", 2023-09-23T20:29:14.476 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.9370000000000001e-06 2023-09-23T20:29:14.476 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:29:14.476 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:14.476 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:29:14.476 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:14.447337+0000", 2023-09-23T20:29:14.477 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.1409999999999998e-06 2023-09-23T20:29:14.477 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:29:14.477 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:14.477 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:29:14.477 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:14.449820+0000", 2023-09-23T20:29:14.477 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0024827349999999998 2023-09-23T20:29:14.477 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:29:14.477 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:14.477 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:29:14.477 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:14.449936+0000", 2023-09-23T20:29:14.477 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000116284 2023-09-23T20:29:14.477 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:29:14.478 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:14.478 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:29:14.478 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:14.450046+0000", 2023-09-23T20:29:14.478 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00010992700000000001 2023-09-23T20:29:14.478 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:29:14.478 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:29:14.478 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:29:14.478 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:29:14.478 INFO:teuthology.orchestra.run.smithi149.stdout: ], 2023-09-23T20:29:14.478 INFO:teuthology.orchestra.run.smithi149.stdout: "num_ops": 3 2023-09-23T20:29:14.478 INFO:teuthology.orchestra.run.smithi149.stdout:} 2023-09-23T20:29:14.536 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:29:14.542 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:14.542 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:29:14.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:14 smithi018 ceph-mon[184144]: 1.e scrub starts 2023-09-23T20:29:14.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:14 smithi018 ceph-mon[179647]: 1.e scrub starts 2023-09-23T20:29:14.643 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:29:14.693 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:14.908 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T20:29:15.008 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:29:15.012 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:15.013 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:29:15.060 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T20:29:15.114 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:29:15.153 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:15.227 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:29:14 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:29:14] "GET /metrics HTTP/1.1" 200 36552 "" "Prometheus/2.43.0" 2023-09-23T20:29:15.247 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:29:15.252 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:15.253 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:29:15.353 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:29:15.446 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:15.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:15 smithi018 ceph-mon[179647]: pgmap v565: 119 pgs: 11 active+clean+remapped, 2 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded, 1 active+recovery_wait+undersized+remapped, 1 active+recovering+undersized+remapped, 22 active+undersized, 2 active+recovery_wait+degraded+remapped, 65 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 728 op/s; 31369/264912 objects degraded (11.841%); 4235/264912 objects misplaced (1.599%); 3.5 MiB/s, 62 objects/s recovering 2023-09-23T20:29:15.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:15 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:29:15.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:15 smithi018 ceph-mon[184144]: pgmap v565: 119 pgs: 11 active+clean+remapped, 2 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded, 1 active+recovery_wait+undersized+remapped, 1 active+recovering+undersized+remapped, 22 active+undersized, 2 active+recovery_wait+degraded+remapped, 65 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 728 op/s; 31369/264912 objects degraded (11.841%); 4235/264912 objects misplaced (1.599%); 3.5 MiB/s, 62 objects/s recovering 2023-09-23T20:29:15.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:15 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:29:15.718 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:15 smithi149 ceph-mon[170394]: pgmap v565: 119 pgs: 11 active+clean+remapped, 2 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded, 1 active+recovery_wait+undersized+remapped, 1 active+recovering+undersized+remapped, 22 active+undersized, 2 active+recovery_wait+degraded+remapped, 65 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 728 op/s; 31369/264912 objects degraded (11.841%); 4235/264912 objects misplaced (1.599%); 3.5 MiB/s, 62 objects/s recovering 2023-09-23T20:29:15.719 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:15 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:29:15.832 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:29:15.971 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:29:15.974 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:15.975 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:29:16.075 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:29:16.167 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:16.238 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:29:16.243 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:16.244 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:29:16.345 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:29:16.417 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:16 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 31369/264912 objects degraded (11.841%), 19 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2023-09-23T20:29:16.418 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:16 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 31369/264912 objects degraded (11.841%), 19 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2023-09-23T20:29:16.441 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:16.575 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:29:16.652 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:16 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 31369/264912 objects degraded (11.841%), 19 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2023-09-23T20:29:16.665 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:29:16.675 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:29:16.686 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:16.686 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:29:16.703 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:29:16.729 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:29:16.756 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:29:16.787 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:29:16.788 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:29:16.827 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:29:16.865 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:29:16.865 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:29:16.865 INFO:teuthology.orchestra.run.smithi018.stderr:Error ENXIO: problem getting command descriptions from osd.1 2023-09-23T20:29:16.865 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:29:16.865 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:29:16.866 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:29:16.866 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:29:16.866 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:29:16.866 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:29:16.877 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:17.195 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:29:17.210 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:17.210 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:29:17.265 INFO:teuthology.orchestra.run.smithi018.stderr:set pool 6 pgp_num to 26 2023-09-23T20:29:17.294 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T20:29:17.295 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T20:29:17.311 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:29:17.393 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:17 smithi149 ceph-mon[170394]: pgmap v566: 119 pgs: 11 active+clean+remapped, 2 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded, 1 active+recovery_wait+undersized+remapped, 1 active+recovering+undersized+remapped, 22 active+undersized, 2 active+recovery_wait+degraded+remapped, 65 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 747 op/s; 31073/262254 objects degraded (11.848%); 4236/262254 objects misplaced (1.615%); 5.2 MiB/s, 89 objects/s recovering 2023-09-23T20:29:17.393 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:17 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2822787374' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "26"}]: dispatch 2023-09-23T20:29:17.393 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:17 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "26"}]: dispatch 2023-09-23T20:29:17.393 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:17 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1265422215' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:29:17.394 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:17.468 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:29:17.471 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:17.472 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:29:17.512 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:17 smithi018 ceph-mon[184144]: pgmap v566: 119 pgs: 11 active+clean+remapped, 2 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded, 1 active+recovery_wait+undersized+remapped, 1 active+recovering+undersized+remapped, 22 active+undersized, 2 active+recovery_wait+degraded+remapped, 65 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 747 op/s; 31073/262254 objects degraded (11.848%); 4236/262254 objects misplaced (1.615%); 5.2 MiB/s, 89 objects/s recovering 2023-09-23T20:29:17.512 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:17 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2822787374' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "26"}]: dispatch 2023-09-23T20:29:17.513 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:17 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "26"}]: dispatch 2023-09-23T20:29:17.513 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:17 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1265422215' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:29:17.513 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:17 smithi018 ceph-mon[179647]: pgmap v566: 119 pgs: 11 active+clean+remapped, 2 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded, 1 active+recovery_wait+undersized+remapped, 1 active+recovering+undersized+remapped, 22 active+undersized, 2 active+recovery_wait+degraded+remapped, 65 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 747 op/s; 31073/262254 objects degraded (11.848%); 4236/262254 objects misplaced (1.615%); 5.2 MiB/s, 89 objects/s recovering 2023-09-23T20:29:17.513 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:17 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2822787374' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "26"}]: dispatch 2023-09-23T20:29:17.513 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:17 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "26"}]: dispatch 2023-09-23T20:29:17.513 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:17 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1265422215' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:29:17.573 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:29:17.668 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:17.764 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:29:17.772 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:17.773 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:29:17.873 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:29:17.926 INFO:teuthology.orchestra.run.smithi149.stdout:ERROR: (22) Invalid argument 2023-09-23T20:29:17.926 INFO:teuthology.orchestra.run.smithi149.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:29:17.949 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:18.271 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:18 smithi018 ceph-mon[179647]: 6.2 scrub starts 2023-09-23T20:29:18.271 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:18 smithi018 ceph-mon[179647]: failed to encode map e256 with expected crc 2023-09-23T20:29:18.271 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:18 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "26"}]': finished 2023-09-23T20:29:18.271 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:18 smithi018 ceph-mon[179647]: osdmap e256: 8 total, 7 up, 8 in 2023-09-23T20:29:18.584 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:18 smithi149 ceph-mon[170394]: 6.2 scrub starts 2023-09-23T20:29:18.584 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:18 smithi149 ceph-mon[170394]: failed to encode map e256 with expected crc 2023-09-23T20:29:18.584 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:18 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "26"}]': finished 2023-09-23T20:29:18.584 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:18 smithi149 ceph-mon[170394]: osdmap e256: 8 total, 7 up, 8 in 2023-09-23T20:29:18.644 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:18 smithi018 ceph-mon[184144]: 6.2 scrub starts 2023-09-23T20:29:18.644 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:18 smithi018 ceph-mon[184144]: failed to encode map e256 with expected crc 2023-09-23T20:29:18.644 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:18 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "26"}]': finished 2023-09-23T20:29:18.644 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:18 smithi018 ceph-mon[184144]: osdmap e256: 8 total, 7 up, 8 in 2023-09-23T20:29:19.296 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:29:19.297 INFO:teuthology.orchestra.run.smithi018.stderr:nodeep-scrub is unset 2023-09-23T20:29:19.603 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:19 smithi149 ceph-mon[170394]: pgmap v568: 119 pgs: 11 active+clean+remapped, 2 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+remapped, 22 active+undersized, 2 active+recovery_wait+degraded+remapped, 65 active+clean; 5.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.07k op/s; 30226/253713 objects degraded (11.913%); 4067/253713 objects misplaced (1.603%); 7.4 MiB/s, 118 objects/s recovering 2023-09-23T20:29:19.603 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:19 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1807997152' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:29:19.603 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:19 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:29:19.604 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T20:29:19.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:19 smithi018 ceph-mon[184144]: pgmap v568: 119 pgs: 11 active+clean+remapped, 2 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+remapped, 22 active+undersized, 2 active+recovery_wait+degraded+remapped, 65 active+clean; 5.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.07k op/s; 30226/253713 objects degraded (11.913%); 4067/253713 objects misplaced (1.603%); 7.4 MiB/s, 118 objects/s recovering 2023-09-23T20:29:19.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:19 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1807997152' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:29:19.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:19 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:29:19.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:19 smithi018 ceph-mon[179647]: pgmap v568: 119 pgs: 11 active+clean+remapped, 2 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+remapped, 22 active+undersized, 2 active+recovery_wait+degraded+remapped, 65 active+clean; 5.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.07k op/s; 30226/253713 objects degraded (11.913%); 4067/253713 objects misplaced (1.603%); 7.4 MiB/s, 118 objects/s recovering 2023-09-23T20:29:19.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:19 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1807997152' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:29:19.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:19 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:29:19.830 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:29:19.838 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:19.839 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:29:19.939 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:29:19.974 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:20.021 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:29:20.024 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:20.024 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:29:20.125 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:29:20.145 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:29:20.177 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:20.239 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:29:20.243 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:20.243 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:29:20.344 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:29:20.440 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:20.534 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:29:20.539 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:20 smithi149 ceph-mon[170394]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:29:20.539 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:20 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:29:20.539 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:20 smithi149 ceph-mon[170394]: osdmap e257: 8 total, 7 up, 8 in 2023-09-23T20:29:20.539 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:20 smithi149 ceph-mon[170394]: failed to encode map e257 with expected crc 2023-09-23T20:29:20.539 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:20 smithi149 ceph-mon[170394]: failed to encode map e257 with expected crc 2023-09-23T20:29:20.539 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:20 smithi149 ceph-mon[170394]: failed to encode map e257 with expected crc 2023-09-23T20:29:20.539 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:20 smithi149 ceph-mon[170394]: failed to encode map e257 with expected crc 2023-09-23T20:29:20.540 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:20 smithi149 ceph-mon[170394]: failed to encode map e257 with expected crc 2023-09-23T20:29:20.540 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:20.540 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:29:20.617 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:29:20.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:20 smithi018 ceph-mon[184144]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:29:20.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:20 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:29:20.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:20 smithi018 ceph-mon[184144]: osdmap e257: 8 total, 7 up, 8 in 2023-09-23T20:29:20.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:20 smithi018 ceph-mon[184144]: failed to encode map e257 with expected crc 2023-09-23T20:29:20.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:20 smithi018 ceph-mon[184144]: failed to encode map e257 with expected crc 2023-09-23T20:29:20.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:20 smithi018 ceph-mon[184144]: failed to encode map e257 with expected crc 2023-09-23T20:29:20.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:20 smithi018 ceph-mon[184144]: failed to encode map e257 with expected crc 2023-09-23T20:29:20.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:20 smithi018 ceph-mon[184144]: failed to encode map e257 with expected crc 2023-09-23T20:29:20.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:20 smithi018 ceph-mon[179647]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:29:20.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:20 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:29:20.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:20 smithi018 ceph-mon[179647]: osdmap e257: 8 total, 7 up, 8 in 2023-09-23T20:29:20.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:20 smithi018 ceph-mon[179647]: failed to encode map e257 with expected crc 2023-09-23T20:29:20.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:20 smithi018 ceph-mon[179647]: failed to encode map e257 with expected crc 2023-09-23T20:29:20.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:20 smithi018 ceph-mon[179647]: failed to encode map e257 with expected crc 2023-09-23T20:29:20.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:20 smithi018 ceph-mon[179647]: failed to encode map e257 with expected crc 2023-09-23T20:29:20.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:20 smithi018 ceph-mon[179647]: failed to encode map e257 with expected crc 2023-09-23T20:29:20.643 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:29:20.680 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:21.178 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:29:21.182 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:21.182 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:29:21.283 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:29:21.369 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:21 smithi149 ceph-mon[170394]: pgmap v570: 119 pgs: 11 active+clean+remapped, 2 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded, 1 active+recovery_wait+undersized+remapped, 1 active+recovering+undersized+remapped, 22 active+undersized, 2 active+recovery_wait+degraded+remapped, 65 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 733 op/s; 29961/251262 objects degraded (11.924%); 4067/251262 objects misplaced (1.619%); 3.9 MiB/s, 62 objects/s recovering 2023-09-23T20:29:21.369 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:21 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 29961/251262 objects degraded (11.924%), 19 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2023-09-23T20:29:21.379 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:21.469 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:29:21.479 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:21.480 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:29:21.580 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:29:21.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:21 smithi018 ceph-mon[184144]: pgmap v570: 119 pgs: 11 active+clean+remapped, 2 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded, 1 active+recovery_wait+undersized+remapped, 1 active+recovering+undersized+remapped, 22 active+undersized, 2 active+recovery_wait+degraded+remapped, 65 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 733 op/s; 29961/251262 objects degraded (11.924%); 4067/251262 objects misplaced (1.619%); 3.9 MiB/s, 62 objects/s recovering 2023-09-23T20:29:21.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:21 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 29961/251262 objects degraded (11.924%), 19 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2023-09-23T20:29:21.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:21 smithi018 ceph-mon[179647]: pgmap v570: 119 pgs: 11 active+clean+remapped, 2 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded, 1 active+recovery_wait+undersized+remapped, 1 active+recovering+undersized+remapped, 22 active+undersized, 2 active+recovery_wait+degraded+remapped, 65 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 733 op/s; 29961/251262 objects degraded (11.924%); 4067/251262 objects misplaced (1.619%); 3.9 MiB/s, 62 objects/s recovering 2023-09-23T20:29:21.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:21 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 29961/251262 objects degraded (11.924%), 19 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2023-09-23T20:29:21.656 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:21.742 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T20:29:21.965 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:29:21.989 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:21.989 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:29:22.090 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:29:22.172 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:22.988 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:29:22.993 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:22.994 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:29:23.004 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 4, 5, 7, 3, 2, 6] out_osds: [] dead_osds: [1] live_osds: [0, 2, 3, 4, 5, 6, 7] 2023-09-23T20:29:23.004 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:29:23.004 INFO:tasks.thrashosds.thrasher:Removing osd 4, in_osds are: [0, 1, 4, 5, 7, 3, 2, 6] 2023-09-23T20:29:23.005 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd out 4 2023-09-23T20:29:23.095 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:29:23.175 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:23.232 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:29:23.235 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:23.236 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:29:23.282 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:23 smithi149 ceph-mon[170394]: pgmap v571: 119 pgs: 11 active+clean+remapped, 2 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+remapped, 22 active+undersized, 2 active+recovery_wait+degraded+remapped, 65 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 992 op/s; 28867/241083 objects degraded (11.974%); 3901/241083 objects misplaced (1.618%); 7.1 MiB/s, 112 objects/s recovering 2023-09-23T20:29:23.282 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:23 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:29:23.282 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:23 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:29:23.282 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:23 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:29:23.282 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:23 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:29:23.337 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:29:23.406 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:23.409 INFO:teuthology.orchestra.run.smithi149.stdout:ERROR: (22) Invalid argument 2023-09-23T20:29:23.409 INFO:teuthology.orchestra.run.smithi149.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:29:23.491 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:29:23.503 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:23.503 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:23 smithi018 ceph-mon[179647]: pgmap v571: 119 pgs: 11 active+clean+remapped, 2 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+remapped, 22 active+undersized, 2 active+recovery_wait+degraded+remapped, 65 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 992 op/s; 28867/241083 objects degraded (11.974%); 3901/241083 objects misplaced (1.618%); 7.1 MiB/s, 112 objects/s recovering 2023-09-23T20:29:23.503 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:23 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:29:23.503 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:23 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:29:23.504 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:23 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:29:23.504 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:23 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:29:23.504 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:23 smithi018 ceph-mon[184144]: pgmap v571: 119 pgs: 11 active+clean+remapped, 2 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+remapped, 22 active+undersized, 2 active+recovery_wait+degraded+remapped, 65 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 992 op/s; 28867/241083 objects degraded (11.974%); 3901/241083 objects misplaced (1.618%); 7.1 MiB/s, 112 objects/s recovering 2023-09-23T20:29:23.504 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:23 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:29:23.504 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:23 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:29:23.504 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:23 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:29:23.504 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:23 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:29:23.505 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:29:23.590 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:29:23.605 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:29:23.675 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:23.929 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T20:29:24.145 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:29:24.146 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:29:24.163 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:24.164 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:29:24.192 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:29:24.225 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:29:24.257 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:29:24.265 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:29:24.298 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:29:24.326 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:29:24.359 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:29:24.369 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:24.390 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:29:24.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:24 smithi018 ceph-mon[179647]: 6.12 scrub starts 2023-09-23T20:29:24.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:24 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2510377288' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:29:24.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:24 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:29:24.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:24 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:29:24.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:24 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:29:24.392 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:29:24.392 INFO:teuthology.orchestra.run.smithi018.stderr:Error ENXIO: problem getting command descriptions from osd.1 2023-09-23T20:29:24.392 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:29:24.392 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:29:24.392 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:29:24.392 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:29:24.392 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:29:24.392 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:29:24.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:24 smithi018 ceph-mon[184144]: 6.12 scrub starts 2023-09-23T20:29:24.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:24 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2510377288' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:29:24.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:24 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:29:24.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:24 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:29:24.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:24 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:29:24.464 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:29:24.471 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:24.471 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:29:24.572 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:29:24.664 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:24 smithi149 ceph-mon[170394]: 6.12 scrub starts 2023-09-23T20:29:24.664 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:24 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2510377288' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:29:24.664 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:24 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:29:24.665 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:24 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:29:24.665 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:24 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:29:24.673 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:24.749 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:29:24.772 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:29:24.787 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:24.787 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:29:24.888 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:29:24.943 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:25.038 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:29:24 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:29:24] "GET /metrics HTTP/1.1" 200 36552 "" "Prometheus/2.43.0" 2023-09-23T20:29:25.168 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T20:29:25.169 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T20:29:25.303 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:25 smithi018 ceph-mon[184144]: 1.5 scrub starts 2023-09-23T20:29:25.304 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:25 smithi018 ceph-mon[184144]: pgmap v572: 119 pgs: 11 active+clean+remapped, 2 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded, 1 active+recovery_wait+undersized+remapped, 1 active+recovering+undersized+remapped, 22 active+undersized, 2 active+recovery_wait+degraded+remapped, 65 active+clean; 4.8 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s; 28593/237867 objects degraded (12.021%); 3901/237867 objects misplaced (1.640%); 4.7 MiB/s, 75 objects/s recovering 2023-09-23T20:29:25.304 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:25 smithi018 ceph-mon[179647]: 1.5 scrub starts 2023-09-23T20:29:25.305 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:25 smithi018 ceph-mon[179647]: pgmap v572: 119 pgs: 11 active+clean+remapped, 2 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded, 1 active+recovery_wait+undersized+remapped, 1 active+recovering+undersized+remapped, 22 active+undersized, 2 active+recovery_wait+degraded+remapped, 65 active+clean; 4.8 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s; 28593/237867 objects degraded (12.021%); 3901/237867 objects misplaced (1.640%); 4.7 MiB/s, 75 objects/s recovering 2023-09-23T20:29:25.354 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:29:25.360 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:25.361 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:29:25.461 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:29:25.468 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:25 smithi149 ceph-mon[170394]: 1.5 scrub starts 2023-09-23T20:29:25.468 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:25 smithi149 ceph-mon[170394]: pgmap v572: 119 pgs: 11 active+clean+remapped, 2 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded, 1 active+recovery_wait+undersized+remapped, 1 active+recovering+undersized+remapped, 22 active+undersized, 2 active+recovery_wait+degraded+remapped, 65 active+clean; 4.8 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s; 28593/237867 objects degraded (12.021%); 3901/237867 objects misplaced (1.640%); 4.7 MiB/s, 75 objects/s recovering 2023-09-23T20:29:25.542 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:25.620 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:29:25.633 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:25.633 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:29:25.734 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:29:25.786 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:25.902 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:29:25.908 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:25.908 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:29:26.009 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:29:26.079 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:26.144 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:29:26.152 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:26.153 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:29:26.253 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:29:26.304 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:26.324 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:26 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mon-a[179623]: 2023-09-23T20:29:26.319+0000 7f160fef9700 -1 mon.a@0(leader).osd e257 definitely_dead 0 2023-09-23T20:29:26.448 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:29:26.457 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:26.457 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:29:26.558 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:29:26.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:26 smithi018 ceph-mon[184144]: pgmap v573: 119 pgs: 11 active+clean+remapped, 2 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded, 1 active+recovery_wait+undersized+remapped, 1 active+recovering+undersized+remapped, 22 active+undersized, 2 active+recovery_wait+degraded+remapped, 65 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 735 op/s; 28294/235212 objects degraded (12.029%); 3902/235212 objects misplaced (1.659%); 5.5 MiB/s, 87 objects/s recovering 2023-09-23T20:29:26.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:26 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 28593/237867 objects degraded (12.021%), 19 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2023-09-23T20:29:26.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:26 smithi018 ceph-mon[179647]: pgmap v573: 119 pgs: 11 active+clean+remapped, 2 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded, 1 active+recovery_wait+undersized+remapped, 1 active+recovering+undersized+remapped, 22 active+undersized, 2 active+recovery_wait+degraded+remapped, 65 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 735 op/s; 28294/235212 objects degraded (12.029%); 3902/235212 objects misplaced (1.659%); 5.5 MiB/s, 87 objects/s recovering 2023-09-23T20:29:26.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:26 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 28593/237867 objects degraded (12.021%), 19 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2023-09-23T20:29:26.650 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:26 smithi149 ceph-mon[170394]: pgmap v573: 119 pgs: 11 active+clean+remapped, 2 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded, 1 active+recovery_wait+undersized+remapped, 1 active+recovering+undersized+remapped, 22 active+undersized, 2 active+recovery_wait+degraded+remapped, 65 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 735 op/s; 28294/235212 objects degraded (12.029%); 3902/235212 objects misplaced (1.659%); 5.5 MiB/s, 87 objects/s recovering 2023-09-23T20:29:26.650 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:26 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 28593/237867 objects degraded (12.021%), 19 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2023-09-23T20:29:26.654 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:26.739 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:29:26.749 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:26.750 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:29:26.811 INFO:teuthology.orchestra.run.smithi149.stdout:{ 2023-09-23T20:29:26.811 INFO:teuthology.orchestra.run.smithi149.stdout: "ops": [ 2023-09-23T20:29:26.811 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:26.811 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_repop(client.45161.0:135526 6.c e257/250 6:381775c4:::benchmark_data_smithi018_270558_object30787:head v 257'8321, mlcod=232'4153)", 2023-09-23T20:29:26.812 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:29:26.793691+0000", 2023-09-23T20:29:26.812 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 0.014163808999999999, 2023-09-23T20:29:26.812 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.014194705, 2023-09-23T20:29:26.812 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:29:26.812 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "started", 2023-09-23T20:29:26.812 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:29:26.812 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:26.812 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:29:26.812 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:26.793691+0000", 2023-09-23T20:29:26.812 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:29:26.813 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:29:26.813 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:26.813 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:29:26.813 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:26.793691+0000", 2023-09-23T20:29:26.813 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:29:26.813 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:29:26.813 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:26.813 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:29:26.813 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:26.793690+0000", 2023-09-23T20:29:26.813 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:29:26.813 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:29:26.814 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:26.814 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:29:26.814 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:26.793697+0000", 2023-09-23T20:29:26.814 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.9650000000000002e-06 2023-09-23T20:29:26.814 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:29:26.814 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:26.814 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:29:26.814 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:26.793712+0000", 2023-09-23T20:29:26.814 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.4819e-05 2023-09-23T20:29:26.814 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:29:26.815 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:26.815 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:29:26.815 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:26.793718+0000", 2023-09-23T20:29:26.815 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.2160000000000003e-06 2023-09-23T20:29:26.815 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:29:26.815 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:26.815 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:29:26.815 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:26.793803+0000", 2023-09-23T20:29:26.815 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.4981000000000005e-05 2023-09-23T20:29:26.815 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:29:26.815 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:26.816 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:29:26.816 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:26.793830+0000", 2023-09-23T20:29:26.816 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.6916000000000001e-05 2023-09-23T20:29:26.816 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:29:26.816 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:29:26.816 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:29:26.816 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:29:26.816 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:26.816 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.45161.0:135525 6.11 6:8fc73b91:::benchmark_data_smithi018_270558_object30786:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e257)", 2023-09-23T20:29:26.816 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:29:26.795369+0000", 2023-09-23T20:29:26.816 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 0.012485817, 2023-09-23T20:29:26.817 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.012591000999999999, 2023-09-23T20:29:26.817 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:29:26.817 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:29:26.817 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:29:26.817 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.45161", 2023-09-23T20:29:26.817 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3795614987", 2023-09-23T20:29:26.817 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 135525 2023-09-23T20:29:26.817 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:29:26.818 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:29:26.818 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:26.818 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:29:26.818 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:26.795369+0000", 2023-09-23T20:29:26.818 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:29:26.818 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:29:26.818 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:26.818 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:29:26.818 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:26.795369+0000", 2023-09-23T20:29:26.819 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:29:26.819 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:29:26.819 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:26.819 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:29:26.819 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:26.795367+0000", 2023-09-23T20:29:26.819 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999976 2023-09-23T20:29:26.819 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:29:26.819 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:26.819 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:29:26.819 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:26.795377+0000", 2023-09-23T20:29:26.819 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0648e-05 2023-09-23T20:29:26.820 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:29:26.820 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:26.820 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:29:26.820 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:26.795394+0000", 2023-09-23T20:29:26.820 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.6470999999999999e-05 2023-09-23T20:29:26.820 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:29:26.820 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:26.820 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:29:26.820 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:26.795402+0000", 2023-09-23T20:29:26.820 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.2300000000000008e-06 2023-09-23T20:29:26.821 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:29:26.821 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:26.821 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:29:26.821 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:26.795465+0000", 2023-09-23T20:29:26.821 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.2941999999999995e-05 2023-09-23T20:29:26.821 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:29:26.821 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:26.821 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for degraded object", 2023-09-23T20:29:26.821 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:26.796354+0000", 2023-09-23T20:29:26.821 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000889521 2023-09-23T20:29:26.822 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:29:26.822 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:26.822 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:29:26.822 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:26.805595+0000", 2023-09-23T20:29:26.822 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0092406160000000001 2023-09-23T20:29:26.822 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:29:26.822 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:26.822 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:29:26.822 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:26.805658+0000", 2023-09-23T20:29:26.822 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.2680000000000003e-05 2023-09-23T20:29:26.822 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:29:26.822 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:26.823 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 2,7", 2023-09-23T20:29:26.823 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:26.805978+0000", 2023-09-23T20:29:26.823 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00032055300000000001 2023-09-23T20:29:26.823 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:29:26.823 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:29:26.823 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:29:26.823 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:29:26.823 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:26.823 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_repop(client.45161.0:135532 6.15 e257/251 6:ae193225:::benchmark_data_smithi018_270558_object30793:head v 257'6914, mlcod=229'4228)", 2023-09-23T20:29:26.823 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:29:26.797961+0000", 2023-09-23T20:29:26.823 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 0.0098942639999999998, 2023-09-23T20:29:26.824 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.010095002, 2023-09-23T20:29:26.824 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:29:26.824 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "started", 2023-09-23T20:29:26.824 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:29:26.824 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:26.824 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:29:26.824 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:26.797961+0000", 2023-09-23T20:29:26.824 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:29:26.824 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:29:26.824 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:26.824 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:29:26.824 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:26.797961+0000", 2023-09-23T20:29:26.825 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:29:26.825 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:29:26.825 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:26.825 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:29:26.825 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:26.797960+0000", 2023-09-23T20:29:26.825 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:29:26.825 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:29:26.825 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:26.825 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:29:26.825 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:26.797968+0000", 2023-09-23T20:29:26.825 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.6549999999999993e-06 2023-09-23T20:29:26.825 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:29:26.826 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:26.826 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:29:26.826 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:26.797978+0000", 2023-09-23T20:29:26.826 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0168e-05 2023-09-23T20:29:26.826 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:29:26.826 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:26.826 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:29:26.826 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:26.797984+0000", 2023-09-23T20:29:26.826 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.4970000000000001e-06 2023-09-23T20:29:26.826 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:29:26.826 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:26.827 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:29:26.827 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:26.798043+0000", 2023-09-23T20:29:26.827 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.8804000000000001e-05 2023-09-23T20:29:26.827 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:29:26.827 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:26.827 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:29:26.827 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:26.798066+0000", 2023-09-23T20:29:26.827 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.2906999999999999e-05 2023-09-23T20:29:26.827 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:29:26.827 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:29:26.827 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:29:26.827 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:29:26.827 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:26.828 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.45161.0:135530 6.11 6:8d360ca6:::benchmark_data_smithi018_270558_object30791:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e257)", 2023-09-23T20:29:26.828 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:29:26.799627+0000", 2023-09-23T20:29:26.828 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 0.0082273789999999999, 2023-09-23T20:29:26.828 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0084941099999999992, 2023-09-23T20:29:26.828 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:29:26.828 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "delayed", 2023-09-23T20:29:26.828 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:29:26.828 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.45161", 2023-09-23T20:29:26.828 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3795614987", 2023-09-23T20:29:26.828 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 135530 2023-09-23T20:29:26.828 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:29:26.829 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:29:26.829 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:26.829 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:29:26.829 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:26.799627+0000", 2023-09-23T20:29:26.829 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:29:26.829 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:29:26.829 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:26.829 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:29:26.829 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:26.799627+0000", 2023-09-23T20:29:26.830 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:29:26.830 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:29:26.830 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:26.830 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:29:26.830 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:26.799626+0000", 2023-09-23T20:29:26.830 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:29:26.830 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:29:26.830 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:26.830 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:29:26.830 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:26.799633+0000", 2023-09-23T20:29:26.830 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.1069999999999999e-06 2023-09-23T20:29:26.830 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:29:26.831 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:26.831 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:29:26.831 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:26.799646+0000", 2023-09-23T20:29:26.831 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.322e-05 2023-09-23T20:29:26.831 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:29:26.831 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:26.831 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:29:26.831 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:26.799653+0000", 2023-09-23T20:29:26.831 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.5529999999999998e-06 2023-09-23T20:29:26.831 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:29:26.831 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:26.831 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:29:26.832 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:26.799700+0000", 2023-09-23T20:29:26.832 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.6535000000000001e-05 2023-09-23T20:29:26.832 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:29:26.832 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:26.832 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for degraded object", 2023-09-23T20:29:26.832 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:26.800415+0000", 2023-09-23T20:29:26.832 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00071503699999999996 2023-09-23T20:29:26.832 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:29:26.832 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:29:26.832 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:29:26.832 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:29:26.832 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:26.833 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.45161.0:135534 6.1 6:84875756:::benchmark_data_smithi018_270558_object30795:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e257)", 2023-09-23T20:29:26.833 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:29:26.799663+0000", 2023-09-23T20:29:26.833 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 0.0081919379999999993, 2023-09-23T20:29:26.833 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0085290290000000005, 2023-09-23T20:29:26.833 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:29:26.833 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "delayed", 2023-09-23T20:29:26.833 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:29:26.833 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.45161", 2023-09-23T20:29:26.833 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3795614987", 2023-09-23T20:29:26.833 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 135534 2023-09-23T20:29:26.833 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:29:26.833 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:29:26.834 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:26.834 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:29:26.834 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:26.799663+0000", 2023-09-23T20:29:26.834 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:29:26.834 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:29:26.834 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:26.834 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:29:26.834 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:26.799663+0000", 2023-09-23T20:29:26.834 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:29:26.834 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:29:26.834 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:26.835 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:29:26.835 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:26.799663+0000", 2023-09-23T20:29:26.835 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:29:26.835 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:29:26.835 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:26.835 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:29:26.835 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:26.799665+0000", 2023-09-23T20:29:26.835 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.7199999999999998e-06 2023-09-23T20:29:26.835 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:29:26.835 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:26.835 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:29:26.836 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:26.799671+0000", 2023-09-23T20:29:26.836 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.519e-06 2023-09-23T20:29:26.836 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:29:26.836 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:26.836 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:29:26.836 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:26.799675+0000", 2023-09-23T20:29:26.836 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.8530000000000002e-06 2023-09-23T20:29:26.836 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:29:26.836 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:26.836 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:29:26.836 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:26.800081+0000", 2023-09-23T20:29:26.836 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00040653199999999998 2023-09-23T20:29:26.837 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:29:26.837 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:26.837 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for degraded object", 2023-09-23T20:29:26.837 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:26.800997+0000", 2023-09-23T20:29:26.837 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00091534199999999998 2023-09-23T20:29:26.837 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:29:26.837 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:29:26.837 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:29:26.837 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:29:26.837 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:26.837 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_repop(client.45161.0:135537 6.c e257/250 6:3e29e8fa:::benchmark_data_smithi018_270558_object30798:head v 257'8322, mlcod=232'4153)", 2023-09-23T20:29:26.837 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:29:26.802418+0000", 2023-09-23T20:29:26.837 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 0.0054365719999999998, 2023-09-23T20:29:26.838 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0058412020000000002, 2023-09-23T20:29:26.838 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:29:26.838 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "started", 2023-09-23T20:29:26.838 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:29:26.838 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:26.838 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:29:26.838 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:26.802418+0000", 2023-09-23T20:29:26.838 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:29:26.838 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:29:26.838 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:26.838 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:29:26.838 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:26.802418+0000", 2023-09-23T20:29:26.839 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:29:26.839 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:29:26.839 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:26.839 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:29:26.839 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:26.802417+0000", 2023-09-23T20:29:26.839 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:29:26.839 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:29:26.839 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:26.839 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:29:26.839 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:26.802424+0000", 2023-09-23T20:29:26.839 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.6109999999999999e-06 2023-09-23T20:29:26.840 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:29:26.840 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:26.840 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:29:26.840 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:26.802439+0000", 2023-09-23T20:29:26.840 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.5242000000000001e-05 2023-09-23T20:29:26.840 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:29:26.840 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:26.840 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:29:26.840 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:26.802446+0000", 2023-09-23T20:29:26.840 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.3199999999999996e-06 2023-09-23T20:29:26.840 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:29:26.840 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:26.841 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:29:26.841 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:26.802508+0000", 2023-09-23T20:29:26.841 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.2537000000000001e-05 2023-09-23T20:29:26.841 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:29:26.841 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:26.841 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:29:26.841 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:26.802530+0000", 2023-09-23T20:29:26.841 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.1829000000000001e-05 2023-09-23T20:29:26.841 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:29:26.841 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:29:26.841 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:29:26.841 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:29:26.842 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:26.842 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_repop(client.45161.0:135539 6.15 e257/251 6:a8cb997b:::benchmark_data_smithi018_270558_object30800:head v 257'6915, mlcod=229'4228)", 2023-09-23T20:29:26.842 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:29:26.808105+0000", 2023-09-23T20:29:26.842 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 4294967295.9997501, 2023-09-23T20:29:26.842 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00023125700000000001, 2023-09-23T20:29:26.842 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:29:26.842 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "started", 2023-09-23T20:29:26.842 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:29:26.842 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:26.842 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:29:26.842 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:26.808105+0000", 2023-09-23T20:29:26.843 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:29:26.843 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:29:26.843 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:26.843 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:29:26.843 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:26.808105+0000", 2023-09-23T20:29:26.843 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:29:26.843 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:29:26.843 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:26.843 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:29:26.843 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:26.808104+0000", 2023-09-23T20:29:26.843 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:29:26.844 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:29:26.844 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:26.844 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:29:26.844 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:26.808112+0000", 2023-09-23T20:29:26.844 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.9160000000000001e-06 2023-09-23T20:29:26.844 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:29:26.844 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:26.844 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:29:26.844 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:26.808128+0000", 2023-09-23T20:29:26.844 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.6033e-05 2023-09-23T20:29:26.844 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:29:26.844 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:26.845 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:29:26.845 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:26.808134+0000", 2023-09-23T20:29:26.845 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.2999999999999998e-06 2023-09-23T20:29:26.845 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:29:26.845 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:26.845 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:29:26.845 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:26.808194+0000", 2023-09-23T20:29:26.845 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.94e-05 2023-09-23T20:29:26.845 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:29:26.845 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:26.845 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:29:26.845 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:26.808218+0000", 2023-09-23T20:29:26.846 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.4861000000000001e-05 2023-09-23T20:29:26.846 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:29:26.846 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:29:26.846 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:29:26.846 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:29:26.846 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:26.846 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_repop_reply(client.45161.0:135525 6.11 e257/251)", 2023-09-23T20:29:26.846 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:29:26.808228+0000", 2023-09-23T20:29:26.846 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 4294967295.9996266, 2023-09-23T20:29:26.846 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000167663, 2023-09-23T20:29:26.846 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:29:26.846 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "queued for pg", 2023-09-23T20:29:26.847 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:29:26.847 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:26.847 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:29:26.847 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:26.808228+0000", 2023-09-23T20:29:26.847 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:29:26.847 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:29:26.847 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:26.847 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:29:26.847 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:26.808228+0000", 2023-09-23T20:29:26.847 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:29:26.847 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:29:26.848 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:26.848 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:29:26.848 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:26.808227+0000", 2023-09-23T20:29:26.848 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:29:26.848 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:29:26.848 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:26.848 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:29:26.848 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:26.808233+0000", 2023-09-23T20:29:26.848 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.2800000000000003e-06 2023-09-23T20:29:26.848 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:29:26.848 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:26.848 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:29:26.849 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:26.808243+0000", 2023-09-23T20:29:26.849 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.8989999999999996e-06 2023-09-23T20:29:26.849 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:29:26.849 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:26.849 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:29:26.849 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:26.808248+0000", 2023-09-23T20:29:26.849 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.2329999999999997e-06 2023-09-23T20:29:26.849 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:29:26.849 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:29:26.849 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:29:26.849 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:29:26.849 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:26.850 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.45161.0:135520 6.11 6:8b88bbe9:::benchmark_data_smithi018_270558_object30781:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e257)", 2023-09-23T20:29:26.850 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:29:26.783630+0000", 2023-09-23T20:29:26.850 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 0.024224994, 2023-09-23T20:29:26.850 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.024817908999999999, 2023-09-23T20:29:26.850 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:29:26.850 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:29:26.850 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:29:26.850 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.45161", 2023-09-23T20:29:26.850 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3795614987", 2023-09-23T20:29:26.850 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 135520 2023-09-23T20:29:26.850 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:29:26.850 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:29:26.851 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:26.851 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:29:26.851 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:26.783630+0000", 2023-09-23T20:29:26.851 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:29:26.851 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:29:26.851 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:26.851 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:29:26.851 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:26.783630+0000", 2023-09-23T20:29:26.851 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:29:26.851 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:29:26.851 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:26.851 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:29:26.852 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:26.783628+0000", 2023-09-23T20:29:26.852 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999976 2023-09-23T20:29:26.852 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:29:26.852 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:26.852 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:29:26.852 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:26.783635+0000", 2023-09-23T20:29:26.852 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.3529999999999997e-06 2023-09-23T20:29:26.852 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:29:26.852 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:26.852 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:29:26.852 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:26.783645+0000", 2023-09-23T20:29:26.853 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0196e-05 2023-09-23T20:29:26.853 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:29:26.853 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:26.853 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:29:26.853 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:26.783651+0000", 2023-09-23T20:29:26.853 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.5960000000000003e-06 2023-09-23T20:29:26.853 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:29:26.853 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:26.853 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:29:26.853 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:26.783690+0000", 2023-09-23T20:29:26.853 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.9684999999999997e-05 2023-09-23T20:29:26.853 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:29:26.854 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:26.854 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for degraded object", 2023-09-23T20:29:26.854 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:26.784410+0000", 2023-09-23T20:29:26.854 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00071969899999999997 2023-09-23T20:29:26.854 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:29:26.854 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:26.854 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:29:26.854 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:26.797510+0000", 2023-09-23T20:29:26.854 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.013100272 2023-09-23T20:29:26.854 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:29:26.854 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:26.854 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:29:26.855 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:26.797601+0000", 2023-09-23T20:29:26.855 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.0265000000000004e-05 2023-09-23T20:29:26.855 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:29:26.855 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:26.855 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 2,7", 2023-09-23T20:29:26.855 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:26.797998+0000", 2023-09-23T20:29:26.855 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00039758099999999999 2023-09-23T20:29:26.855 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:29:26.855 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:26.855 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:29:26.855 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:26.801683+0000", 2023-09-23T20:29:26.856 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0036852220000000002 2023-09-23T20:29:26.856 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:29:26.856 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:29:26.856 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:29:26.856 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:29:26.856 INFO:teuthology.orchestra.run.smithi149.stdout: ], 2023-09-23T20:29:26.856 INFO:teuthology.orchestra.run.smithi149.stdout: "num_ops": 9 2023-09-23T20:29:26.856 INFO:teuthology.orchestra.run.smithi149.stdout:} 2023-09-23T20:29:26.856 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:29:26.931 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:27.170 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:29:27.202 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:29:27.209 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:27.209 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:29:27.263 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T20:29:27.310 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:29:27.331 INFO:teuthology.orchestra.run.smithi018.stderr:marked out osd.4. 2023-09-23T20:29:27.356 INFO:teuthology.orchestra.run.smithi018.stderr:noscrub is set 2023-09-23T20:29:27.369 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:27.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:27 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1465893026' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["4"]}]: dispatch 2023-09-23T20:29:27.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:27 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["4"]}]: dispatch 2023-09-23T20:29:27.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:27 smithi149 ceph-mon[170394]: Client client.admin marked osd.4 out, while it was still marked up 2023-09-23T20:29:27.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:27 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2995746899' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:29:27.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:27 smithi149 ceph-mon[170394]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:29:27.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:27 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["4"]}]': finished 2023-09-23T20:29:27.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:27 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2995746899' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:29:27.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:27 smithi149 ceph-mon[170394]: osdmap e258: 8 total, 7 up, 7 in 2023-09-23T20:29:27.475 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:27 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1465893026' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["4"]}]: dispatch 2023-09-23T20:29:27.476 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:27 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["4"]}]: dispatch 2023-09-23T20:29:27.476 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:27 smithi018 ceph-mon[184144]: Client client.admin marked osd.4 out, while it was still marked up 2023-09-23T20:29:27.476 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:27 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2995746899' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:29:27.476 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:27 smithi018 ceph-mon[184144]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:29:27.476 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:27 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["4"]}]': finished 2023-09-23T20:29:27.476 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:27 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2995746899' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:29:27.476 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:27 smithi018 ceph-mon[184144]: osdmap e258: 8 total, 7 up, 7 in 2023-09-23T20:29:27.476 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:27 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1465893026' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["4"]}]: dispatch 2023-09-23T20:29:27.476 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:27 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["4"]}]: dispatch 2023-09-23T20:29:27.477 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:27 smithi018 ceph-mon[179647]: Client client.admin marked osd.4 out, while it was still marked up 2023-09-23T20:29:27.477 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:27 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2995746899' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:29:27.477 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:27 smithi018 ceph-mon[179647]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:29:27.477 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:27 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["4"]}]': finished 2023-09-23T20:29:27.477 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:27 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2995746899' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:29:27.477 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:27 smithi018 ceph-mon[179647]: osdmap e258: 8 total, 7 up, 7 in 2023-09-23T20:29:27.870 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:29:27.873 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:27.873 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:29:27.952 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:29:27.974 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:29:28.005 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:28.038 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:29:28.040 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:28.041 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:29:28.114 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:29:28.142 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:29:28.169 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:28.361 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:29:28.364 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:28.365 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:29:28.448 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:28 smithi018 ceph-mon[184144]: 1.e scrub starts 2023-09-23T20:29:28.449 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:28 smithi018 ceph-mon[184144]: failed to encode map e258 with expected crc 2023-09-23T20:29:28.449 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:28 smithi018 ceph-mon[184144]: pgmap v575: 119 pgs: 11 active+clean+remapped, 2 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+remapped, 22 active+undersized, 2 active+recovery_wait+degraded+remapped, 65 active+clean; 4.6 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 991 op/s; 27427/226392 objects degraded (12.115%); 3703/226392 objects misplaced (1.636%); 7.0 MiB/s, 111 objects/s recovering 2023-09-23T20:29:28.449 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:28 smithi018 ceph-mon[179647]: 1.e scrub starts 2023-09-23T20:29:28.449 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:28 smithi018 ceph-mon[179647]: failed to encode map e258 with expected crc 2023-09-23T20:29:28.449 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:28 smithi018 ceph-mon[179647]: pgmap v575: 119 pgs: 11 active+clean+remapped, 2 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+remapped, 22 active+undersized, 2 active+recovery_wait+degraded+remapped, 65 active+clean; 4.6 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 991 op/s; 27427/226392 objects degraded (12.115%); 3703/226392 objects misplaced (1.636%); 7.0 MiB/s, 111 objects/s recovering 2023-09-23T20:29:28.466 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:29:28.503 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:28.609 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:28 smithi149 ceph-mon[170394]: 1.e scrub starts 2023-09-23T20:29:28.609 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:28 smithi149 ceph-mon[170394]: failed to encode map e258 with expected crc 2023-09-23T20:29:28.609 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:28 smithi149 ceph-mon[170394]: pgmap v575: 119 pgs: 11 active+clean+remapped, 2 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+remapped, 22 active+undersized, 2 active+recovery_wait+degraded+remapped, 65 active+clean; 4.6 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 991 op/s; 27427/226392 objects degraded (12.115%); 3703/226392 objects misplaced (1.636%); 7.0 MiB/s, 111 objects/s recovering 2023-09-23T20:29:28.641 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:29:28.644 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:28.645 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:29:28.745 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:29:28.774 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:28.922 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:29:28.925 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:28.925 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:29:29.026 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:29:29.056 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:29.157 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:29:29.159 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:29.160 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:29:29.260 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:29:29.291 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:29.327 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:29:29.330 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:29.330 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:29:29.352 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:29 smithi149 ceph-mon[170394]: failed to encode map e259 with expected crc 2023-09-23T20:29:29.352 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:29 smithi149 ceph-mon[170394]: failed to encode map e259 with expected crc 2023-09-23T20:29:29.352 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:29 smithi149 ceph-mon[170394]: failed to encode map e259 with expected crc 2023-09-23T20:29:29.353 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:29 smithi149 ceph-mon[170394]: failed to encode map e259 with expected crc 2023-09-23T20:29:29.353 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:29 smithi149 ceph-mon[170394]: osdmap e259: 8 total, 7 up, 7 in 2023-09-23T20:29:29.353 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:29 smithi149 ceph-mon[170394]: failed to encode map e259 with expected crc 2023-09-23T20:29:29.353 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:29 smithi149 ceph-mon[170394]: failed to encode map e259 with expected crc 2023-09-23T20:29:29.353 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:29 smithi149 ceph-mon[170394]: failed to encode map e259 with expected crc 2023-09-23T20:29:29.353 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:29 smithi149 ceph-mon[170394]: failed to encode map e259 with expected crc 2023-09-23T20:29:29.353 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:29 smithi149 ceph-mon[170394]: failed to encode map e259 with expected crc 2023-09-23T20:29:29.353 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:29 smithi149 ceph-mon[170394]: failed to encode map e259 with expected crc 2023-09-23T20:29:29.353 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:29 smithi149 ceph-mon[170394]: failed to encode map e259 with expected crc 2023-09-23T20:29:29.353 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:29 smithi149 ceph-mon[170394]: failed to encode map e259 with expected crc 2023-09-23T20:29:29.431 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:29:29.496 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:29.529 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:29:29.559 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:29:29.590 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:29:29.621 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:29:29.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:29 smithi018 ceph-mon[184144]: failed to encode map e259 with expected crc 2023-09-23T20:29:29.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:29 smithi018 ceph-mon[184144]: failed to encode map e259 with expected crc 2023-09-23T20:29:29.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:29 smithi018 ceph-mon[184144]: failed to encode map e259 with expected crc 2023-09-23T20:29:29.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:29 smithi018 ceph-mon[184144]: failed to encode map e259 with expected crc 2023-09-23T20:29:29.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:29 smithi018 ceph-mon[184144]: osdmap e259: 8 total, 7 up, 7 in 2023-09-23T20:29:29.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:29 smithi018 ceph-mon[184144]: failed to encode map e259 with expected crc 2023-09-23T20:29:29.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:29 smithi018 ceph-mon[184144]: failed to encode map e259 with expected crc 2023-09-23T20:29:29.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:29 smithi018 ceph-mon[184144]: failed to encode map e259 with expected crc 2023-09-23T20:29:29.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:29 smithi018 ceph-mon[184144]: failed to encode map e259 with expected crc 2023-09-23T20:29:29.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:29 smithi018 ceph-mon[184144]: failed to encode map e259 with expected crc 2023-09-23T20:29:29.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:29 smithi018 ceph-mon[184144]: failed to encode map e259 with expected crc 2023-09-23T20:29:29.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:29 smithi018 ceph-mon[184144]: failed to encode map e259 with expected crc 2023-09-23T20:29:29.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:29 smithi018 ceph-mon[184144]: failed to encode map e259 with expected crc 2023-09-23T20:29:29.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:29 smithi018 ceph-mon[179647]: failed to encode map e259 with expected crc 2023-09-23T20:29:29.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:29 smithi018 ceph-mon[179647]: failed to encode map e259 with expected crc 2023-09-23T20:29:29.644 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:29 smithi018 ceph-mon[179647]: failed to encode map e259 with expected crc 2023-09-23T20:29:29.644 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:29 smithi018 ceph-mon[179647]: failed to encode map e259 with expected crc 2023-09-23T20:29:29.644 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:29 smithi018 ceph-mon[179647]: osdmap e259: 8 total, 7 up, 7 in 2023-09-23T20:29:29.644 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:29 smithi018 ceph-mon[179647]: failed to encode map e259 with expected crc 2023-09-23T20:29:29.644 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:29 smithi018 ceph-mon[179647]: failed to encode map e259 with expected crc 2023-09-23T20:29:29.644 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:29 smithi018 ceph-mon[179647]: failed to encode map e259 with expected crc 2023-09-23T20:29:29.644 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:29 smithi018 ceph-mon[179647]: failed to encode map e259 with expected crc 2023-09-23T20:29:29.644 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:29 smithi018 ceph-mon[179647]: failed to encode map e259 with expected crc 2023-09-23T20:29:29.644 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:29 smithi018 ceph-mon[179647]: failed to encode map e259 with expected crc 2023-09-23T20:29:29.644 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:29 smithi018 ceph-mon[179647]: failed to encode map e259 with expected crc 2023-09-23T20:29:29.644 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:29 smithi018 ceph-mon[179647]: failed to encode map e259 with expected crc 2023-09-23T20:29:29.648 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:29:29.679 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:29:29.742 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:29:29.742 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:29:29.742 INFO:teuthology.orchestra.run.smithi018.stderr:Error ENXIO: problem getting command descriptions from osd.1 2023-09-23T20:29:29.742 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:29:29.742 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:29:29.742 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:29:29.743 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:29:29.743 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:29:29.743 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:29:29.922 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T20:29:30.193 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T20:29:30.194 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T20:29:30.563 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:29:30.569 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:30.570 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:29:30.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:30 smithi018 ceph-mon[179647]: failed to encode map e260 with expected crc 2023-09-23T20:29:30.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:30 smithi018 ceph-mon[179647]: osdmap e260: 8 total, 7 up, 7 in 2023-09-23T20:29:30.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:30 smithi018 ceph-mon[179647]: failed to encode map e260 with expected crc 2023-09-23T20:29:30.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:30 smithi018 ceph-mon[179647]: failed to encode map e260 with expected crc 2023-09-23T20:29:30.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:30 smithi018 ceph-mon[179647]: failed to encode map e260 with expected crc 2023-09-23T20:29:30.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:30 smithi018 ceph-mon[179647]: 6.e starting backfill to osd.5 from (0'0,0'0] MAX to 257'8467 2023-09-23T20:29:30.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:30 smithi018 ceph-mon[179647]: failed to encode map e260 with expected crc 2023-09-23T20:29:30.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:30 smithi018 ceph-mon[179647]: failed to encode map e260 with expected crc 2023-09-23T20:29:30.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:30 smithi018 ceph-mon[179647]: 6.f starting backfill to osd.6 from (0'0,0'0] MAX to 257'8608 2023-09-23T20:29:30.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:30 smithi018 ceph-mon[179647]: failed to encode map e260 with expected crc 2023-09-23T20:29:30.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:30 smithi018 ceph-mon[179647]: 6.16 starting backfill to osd.6 from (0'0,0'0] MAX to 257'6930 2023-09-23T20:29:30.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:30 smithi018 ceph-mon[179647]: 6.6 starting backfill to osd.6 from (0'0,0'0] MAX to 257'6942 2023-09-23T20:29:30.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:30 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:29:30.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:30 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2407862249' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:29:30.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:30 smithi018 ceph-mon[179647]: pgmap v578: 119 pgs: 6 unknown, 1 remapped+peering, 7 peering, 11 active+clean+remapped, 2 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded, 1 active+recovering+undersized+remapped, 16 active+undersized, 2 active+recovery_wait+degraded+remapped, 58 active+clean; 4.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 780 op/s; 27182/223812 objects degraded (12.145%); 3696/223812 objects misplaced (1.651%); 5.2 MiB/s, 82 objects/s recovering 2023-09-23T20:29:30.643 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:30 smithi018 ceph-mon[184144]: failed to encode map e260 with expected crc 2023-09-23T20:29:30.643 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:30 smithi018 ceph-mon[184144]: osdmap e260: 8 total, 7 up, 7 in 2023-09-23T20:29:30.644 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:30 smithi018 ceph-mon[184144]: failed to encode map e260 with expected crc 2023-09-23T20:29:30.644 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:30 smithi018 ceph-mon[184144]: failed to encode map e260 with expected crc 2023-09-23T20:29:30.644 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:30 smithi018 ceph-mon[184144]: failed to encode map e260 with expected crc 2023-09-23T20:29:30.644 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:30 smithi018 ceph-mon[184144]: 6.e starting backfill to osd.5 from (0'0,0'0] MAX to 257'8467 2023-09-23T20:29:30.644 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:30 smithi018 ceph-mon[184144]: failed to encode map e260 with expected crc 2023-09-23T20:29:30.644 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:30 smithi018 ceph-mon[184144]: failed to encode map e260 with expected crc 2023-09-23T20:29:30.644 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:30 smithi018 ceph-mon[184144]: 6.f starting backfill to osd.6 from (0'0,0'0] MAX to 257'8608 2023-09-23T20:29:30.644 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:30 smithi018 ceph-mon[184144]: failed to encode map e260 with expected crc 2023-09-23T20:29:30.644 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:30 smithi018 ceph-mon[184144]: 6.16 starting backfill to osd.6 from (0'0,0'0] MAX to 257'6930 2023-09-23T20:29:30.644 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:30 smithi018 ceph-mon[184144]: 6.6 starting backfill to osd.6 from (0'0,0'0] MAX to 257'6942 2023-09-23T20:29:30.644 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:30 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:29:30.644 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:30 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2407862249' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:29:30.645 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:30 smithi018 ceph-mon[184144]: pgmap v578: 119 pgs: 6 unknown, 1 remapped+peering, 7 peering, 11 active+clean+remapped, 2 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded, 1 active+recovering+undersized+remapped, 16 active+undersized, 2 active+recovery_wait+degraded+remapped, 58 active+clean; 4.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 780 op/s; 27182/223812 objects degraded (12.145%); 3696/223812 objects misplaced (1.651%); 5.2 MiB/s, 82 objects/s recovering 2023-09-23T20:29:30.671 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:29:30.687 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:30 smithi149 ceph-mon[170394]: failed to encode map e260 with expected crc 2023-09-23T20:29:30.688 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:30 smithi149 ceph-mon[170394]: osdmap e260: 8 total, 7 up, 7 in 2023-09-23T20:29:30.688 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:30 smithi149 ceph-mon[170394]: failed to encode map e260 with expected crc 2023-09-23T20:29:30.688 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:30 smithi149 ceph-mon[170394]: failed to encode map e260 with expected crc 2023-09-23T20:29:30.688 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:30 smithi149 ceph-mon[170394]: failed to encode map e260 with expected crc 2023-09-23T20:29:30.688 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:30 smithi149 ceph-mon[170394]: 6.e starting backfill to osd.5 from (0'0,0'0] MAX to 257'8467 2023-09-23T20:29:30.688 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:30 smithi149 ceph-mon[170394]: failed to encode map e260 with expected crc 2023-09-23T20:29:30.688 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:30 smithi149 ceph-mon[170394]: failed to encode map e260 with expected crc 2023-09-23T20:29:30.688 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:30 smithi149 ceph-mon[170394]: 6.f starting backfill to osd.6 from (0'0,0'0] MAX to 257'8608 2023-09-23T20:29:30.688 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:30 smithi149 ceph-mon[170394]: failed to encode map e260 with expected crc 2023-09-23T20:29:30.688 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:30 smithi149 ceph-mon[170394]: 6.16 starting backfill to osd.6 from (0'0,0'0] MAX to 257'6930 2023-09-23T20:29:30.688 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:30 smithi149 ceph-mon[170394]: 6.6 starting backfill to osd.6 from (0'0,0'0] MAX to 257'6942 2023-09-23T20:29:30.689 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:30 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:29:30.689 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:30 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2407862249' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:29:30.689 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:30 smithi149 ceph-mon[170394]: pgmap v578: 119 pgs: 6 unknown, 1 remapped+peering, 7 peering, 11 active+clean+remapped, 2 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded, 1 active+recovering+undersized+remapped, 16 active+undersized, 2 active+recovery_wait+degraded+remapped, 58 active+clean; 4.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 780 op/s; 27182/223812 objects degraded (12.145%); 3696/223812 objects misplaced (1.651%); 5.2 MiB/s, 82 objects/s recovering 2023-09-23T20:29:30.689 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:29:30.768 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:31.485 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:31 smithi018 ceph-mon[179647]: Health check failed: Reduced data availability: 3 pgs inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T20:29:31.485 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:31 smithi018 ceph-mon[179647]: failed to encode map e261 with expected crc 2023-09-23T20:29:31.485 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:31 smithi018 ceph-mon[179647]: osdmap e261: 8 total, 7 up, 7 in 2023-09-23T20:29:31.486 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:31 smithi018 ceph-mon[179647]: failed to encode map e261 with expected crc 2023-09-23T20:29:31.486 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:31 smithi018 ceph-mon[179647]: failed to encode map e261 with expected crc 2023-09-23T20:29:31.486 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:31 smithi018 ceph-mon[179647]: failed to encode map e261 with expected crc 2023-09-23T20:29:31.486 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:31 smithi018 ceph-mon[179647]: failed to encode map e261 with expected crc 2023-09-23T20:29:31.486 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:31 smithi018 ceph-mon[179647]: failed to encode map e261 with expected crc 2023-09-23T20:29:31.486 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:31 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 27182/223812 objects degraded (12.145%), 19 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2023-09-23T20:29:31.487 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:31 smithi018 ceph-mon[184144]: Health check failed: Reduced data availability: 3 pgs inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T20:29:31.487 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:31 smithi018 ceph-mon[184144]: failed to encode map e261 with expected crc 2023-09-23T20:29:31.487 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:31 smithi018 ceph-mon[184144]: osdmap e261: 8 total, 7 up, 7 in 2023-09-23T20:29:31.487 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:31 smithi018 ceph-mon[184144]: failed to encode map e261 with expected crc 2023-09-23T20:29:31.487 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:31 smithi018 ceph-mon[184144]: failed to encode map e261 with expected crc 2023-09-23T20:29:31.487 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:31 smithi018 ceph-mon[184144]: failed to encode map e261 with expected crc 2023-09-23T20:29:31.487 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:31 smithi018 ceph-mon[184144]: failed to encode map e261 with expected crc 2023-09-23T20:29:31.487 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:31 smithi018 ceph-mon[184144]: failed to encode map e261 with expected crc 2023-09-23T20:29:31.488 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:31 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 27182/223812 objects degraded (12.145%), 19 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2023-09-23T20:29:31.607 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:29:31.611 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:31.611 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:29:31.712 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:29:31.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:31 smithi149 ceph-mon[170394]: Health check failed: Reduced data availability: 3 pgs inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T20:29:31.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:31 smithi149 ceph-mon[170394]: failed to encode map e261 with expected crc 2023-09-23T20:29:31.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:31 smithi149 ceph-mon[170394]: osdmap e261: 8 total, 7 up, 7 in 2023-09-23T20:29:31.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:31 smithi149 ceph-mon[170394]: failed to encode map e261 with expected crc 2023-09-23T20:29:31.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:31 smithi149 ceph-mon[170394]: failed to encode map e261 with expected crc 2023-09-23T20:29:31.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:31 smithi149 ceph-mon[170394]: failed to encode map e261 with expected crc 2023-09-23T20:29:31.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:31 smithi149 ceph-mon[170394]: failed to encode map e261 with expected crc 2023-09-23T20:29:31.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:31 smithi149 ceph-mon[170394]: failed to encode map e261 with expected crc 2023-09-23T20:29:31.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:31 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 27182/223812 objects degraded (12.145%), 19 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2023-09-23T20:29:31.739 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:31.772 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:29:31.775 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:31.775 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:29:31.876 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:29:31.946 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:32.032 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:29:32.043 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:32.043 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:29:32.144 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:29:32.194 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:29:32.215 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:32.291 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:29:32.299 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:32.299 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:29:32.400 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:29:32.419 INFO:teuthology.orchestra.run.smithi018.stderr:nodeep-scrub is set 2023-09-23T20:29:32.455 INFO:teuthology.orchestra.run.smithi149.stdout:ERROR: (22) Invalid argument 2023-09-23T20:29:32.455 INFO:teuthology.orchestra.run.smithi149.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:29:32.514 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:32.559 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:32 smithi018 ceph-mon[179647]: osdmap e262: 8 total, 7 up, 7 in 2023-09-23T20:29:32.559 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:32 smithi018 ceph-mon[179647]: failed to encode map e262 with expected crc 2023-09-23T20:29:32.559 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:32 smithi018 ceph-mon[179647]: failed to encode map e262 with expected crc 2023-09-23T20:29:32.559 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:32 smithi018 ceph-mon[179647]: failed to encode map e262 with expected crc 2023-09-23T20:29:32.559 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:32 smithi018 ceph-mon[179647]: failed to encode map e262 with expected crc 2023-09-23T20:29:32.559 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:32 smithi018 ceph-mon[179647]: failed to encode map e262 with expected crc 2023-09-23T20:29:32.559 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:32 smithi018 ceph-mon[179647]: failed to encode map e262 with expected crc 2023-09-23T20:29:32.559 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:32 smithi018 ceph-mon[179647]: failed to encode map e262 with expected crc 2023-09-23T20:29:32.559 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:32 smithi018 ceph-mon[179647]: failed to encode map e262 with expected crc 2023-09-23T20:29:32.559 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:32 smithi018 ceph-mon[179647]: pgmap v581: 119 pgs: 6 unknown, 1 remapped+peering, 9 peering, 20 active+clean+remapped, 2 active+recovery_wait+remapped, 5 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded, 1 active+recovering+undersized+remapped, 13 active+undersized, 4 active+recovery_wait+degraded+remapped, 46 active+clean; 4.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 634 op/s; 26683/218772 objects degraded (12.197%); 7671/218772 objects misplaced (3.506%); 10 MiB/s, 163 objects/s recovering 2023-09-23T20:29:32.560 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:32 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3948996734' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:29:32.560 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:32 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:29:32.560 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:32 smithi018 ceph-mon[184144]: osdmap e262: 8 total, 7 up, 7 in 2023-09-23T20:29:32.560 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:32 smithi018 ceph-mon[184144]: failed to encode map e262 with expected crc 2023-09-23T20:29:32.560 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:32 smithi018 ceph-mon[184144]: failed to encode map e262 with expected crc 2023-09-23T20:29:32.560 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:32 smithi018 ceph-mon[184144]: failed to encode map e262 with expected crc 2023-09-23T20:29:32.560 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:32 smithi018 ceph-mon[184144]: failed to encode map e262 with expected crc 2023-09-23T20:29:32.560 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:32 smithi018 ceph-mon[184144]: failed to encode map e262 with expected crc 2023-09-23T20:29:32.561 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:32 smithi018 ceph-mon[184144]: failed to encode map e262 with expected crc 2023-09-23T20:29:32.561 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:32 smithi018 ceph-mon[184144]: failed to encode map e262 with expected crc 2023-09-23T20:29:32.561 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:32 smithi018 ceph-mon[184144]: failed to encode map e262 with expected crc 2023-09-23T20:29:32.561 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:32 smithi018 ceph-mon[184144]: pgmap v581: 119 pgs: 6 unknown, 1 remapped+peering, 9 peering, 20 active+clean+remapped, 2 active+recovery_wait+remapped, 5 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded, 1 active+recovering+undersized+remapped, 13 active+undersized, 4 active+recovery_wait+degraded+remapped, 46 active+clean; 4.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 634 op/s; 26683/218772 objects degraded (12.197%); 7671/218772 objects misplaced (3.506%); 10 MiB/s, 163 objects/s recovering 2023-09-23T20:29:32.561 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:32 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3948996734' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:29:32.561 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:32 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:29:32.592 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:32 smithi149 ceph-mon[170394]: osdmap e262: 8 total, 7 up, 7 in 2023-09-23T20:29:32.592 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:32 smithi149 ceph-mon[170394]: failed to encode map e262 with expected crc 2023-09-23T20:29:32.592 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:32 smithi149 ceph-mon[170394]: failed to encode map e262 with expected crc 2023-09-23T20:29:32.592 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:32 smithi149 ceph-mon[170394]: failed to encode map e262 with expected crc 2023-09-23T20:29:32.593 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:32 smithi149 ceph-mon[170394]: failed to encode map e262 with expected crc 2023-09-23T20:29:32.593 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:32 smithi149 ceph-mon[170394]: failed to encode map e262 with expected crc 2023-09-23T20:29:32.593 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:32 smithi149 ceph-mon[170394]: failed to encode map e262 with expected crc 2023-09-23T20:29:32.593 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:32 smithi149 ceph-mon[170394]: failed to encode map e262 with expected crc 2023-09-23T20:29:32.593 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:32 smithi149 ceph-mon[170394]: failed to encode map e262 with expected crc 2023-09-23T20:29:32.593 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:32 smithi149 ceph-mon[170394]: pgmap v581: 119 pgs: 6 unknown, 1 remapped+peering, 9 peering, 20 active+clean+remapped, 2 active+recovery_wait+remapped, 5 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded, 1 active+recovering+undersized+remapped, 13 active+undersized, 4 active+recovery_wait+degraded+remapped, 46 active+clean; 4.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 634 op/s; 26683/218772 objects degraded (12.197%); 7671/218772 objects misplaced (3.506%); 10 MiB/s, 163 objects/s recovering 2023-09-23T20:29:32.593 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:32 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3948996734' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:29:32.593 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:32 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:29:32.601 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:29:32.608 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:32.609 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:29:32.710 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:29:32.753 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:32.804 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:29:32.817 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:32.817 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:29:32.845 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T20:29:32.917 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 5, 7, 3, 2, 6] out_osds: [4] dead_osds: [1] live_osds: [0, 2, 3, 4, 5, 6, 7] 2023-09-23T20:29:32.917 INFO:tasks.thrashosds.thrasher:Reviving osd 1 2023-09-23T20:29:32.918 INFO:tasks.cephadm.osd.1:Restarting osd.1 (starting--it wasn't running)... 2023-09-23T20:29:32.918 DEBUG:teuthology.orchestra.run.smithi018:osd.1> sudo journalctl -f -n 0 -u ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7@osd.1.service 2023-09-23T20:29:32.918 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:29:32.920 DEBUG:teuthology.orchestra.run.smithi018:> sudo systemctl start ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7@osd.1 2023-09-23T20:29:32.961 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:32.963 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:29:32.965 INFO:journalctl@ceph.osd.1.smithi018.stdout:-- Logs begin at Sat 2023-09-23 19:50:10 UTC. -- 2023-09-23T20:29:33.027 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:29:33.035 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:33.036 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:29:33.136 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:29:33.142 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 23 20:29:33 smithi018 systemd[1]: Starting Ceph osd.1 for a3b3b17e-5a4c-11ee-8db4-212e2dc638e7... 2023-09-23T20:29:33.220 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:33.311 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:29:33.319 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:33.319 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:29:33.420 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:29:33.471 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:33 smithi149 ceph-mon[170394]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:29:33.471 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:33 smithi149 ceph-mon[170394]: failed to encode map e263 with expected crc 2023-09-23T20:29:33.471 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:33 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:29:33.471 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:33 smithi149 ceph-mon[170394]: osdmap e263: 8 total, 7 up, 7 in 2023-09-23T20:29:33.471 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:33 smithi149 ceph-mon[170394]: failed to encode map e263 with expected crc 2023-09-23T20:29:33.472 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:33 smithi149 ceph-mon[170394]: failed to encode map e263 with expected crc 2023-09-23T20:29:33.472 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:33 smithi149 ceph-mon[170394]: failed to encode map e263 with expected crc 2023-09-23T20:29:33.481 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:33.549 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:29:33.558 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:33.559 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:29:33.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:33 smithi018 ceph-mon[179647]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:29:33.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:33 smithi018 ceph-mon[179647]: failed to encode map e263 with expected crc 2023-09-23T20:29:33.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:33 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:29:33.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:33 smithi018 ceph-mon[179647]: osdmap e263: 8 total, 7 up, 7 in 2023-09-23T20:29:33.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:33 smithi018 ceph-mon[179647]: failed to encode map e263 with expected crc 2023-09-23T20:29:33.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:33 smithi018 ceph-mon[179647]: failed to encode map e263 with expected crc 2023-09-23T20:29:33.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:33 smithi018 ceph-mon[179647]: failed to encode map e263 with expected crc 2023-09-23T20:29:33.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:33 smithi018 ceph-mon[184144]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:29:33.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:33 smithi018 ceph-mon[184144]: failed to encode map e263 with expected crc 2023-09-23T20:29:33.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:33 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:29:33.643 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:33 smithi018 ceph-mon[184144]: osdmap e263: 8 total, 7 up, 7 in 2023-09-23T20:29:33.643 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:33 smithi018 ceph-mon[184144]: failed to encode map e263 with expected crc 2023-09-23T20:29:33.643 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:33 smithi018 ceph-mon[184144]: failed to encode map e263 with expected crc 2023-09-23T20:29:33.643 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:33 smithi018 ceph-mon[184144]: failed to encode map e263 with expected crc 2023-09-23T20:29:33.659 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:29:33.716 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:33.767 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:29:33.924 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:29:33.931 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:33.931 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:29:34.032 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:29:34.081 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:34.678 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:34 smithi149 ceph-mon[170394]: failed to encode map e263 with expected crc 2023-09-23T20:29:34.678 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:34 smithi149 ceph-mon[170394]: pgmap v583: 119 pgs: 1 remapped+peering, 2 peering, 31 active+clean+remapped, 2 active+recovery_wait+remapped, 5 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 12 active+undersized, 4 active+recovery_wait+degraded+remapped, 44 active+clean; 4.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 9.5 KiB/s rd, 26 KiB/s wr, 669 op/s; 32902/215271 objects degraded (15.284%); 21045/215271 objects misplaced (9.776%); 9.6 MiB/s, 152 objects/s recovering 2023-09-23T20:29:34.837 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T20:29:34.872 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:34 smithi018 ceph-mon[184144]: failed to encode map e263 with expected crc 2023-09-23T20:29:34.873 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:34 smithi018 ceph-mon[184144]: pgmap v583: 119 pgs: 1 remapped+peering, 2 peering, 31 active+clean+remapped, 2 active+recovery_wait+remapped, 5 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 12 active+undersized, 4 active+recovery_wait+degraded+remapped, 44 active+clean; 4.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 9.5 KiB/s rd, 26 KiB/s wr, 669 op/s; 32902/215271 objects degraded (15.284%); 21045/215271 objects misplaced (9.776%); 9.6 MiB/s, 152 objects/s recovering 2023-09-23T20:29:34.873 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:34 smithi018 ceph-mon[179647]: failed to encode map e263 with expected crc 2023-09-23T20:29:34.873 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:34 smithi018 ceph-mon[179647]: pgmap v583: 119 pgs: 1 remapped+peering, 2 peering, 31 active+clean+remapped, 2 active+recovery_wait+remapped, 5 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 12 active+undersized, 4 active+recovery_wait+degraded+remapped, 44 active+clean; 4.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 9.5 KiB/s rd, 26 KiB/s wr, 669 op/s; 32902/215271 objects degraded (15.284%); 21045/215271 objects misplaced (9.776%); 9.6 MiB/s, 152 objects/s recovering 2023-09-23T20:29:35.143 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:29:34 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:29:34] "GET /metrics HTTP/1.1" 200 36547 "" "Prometheus/2.43.0" 2023-09-23T20:29:35.348 INFO:teuthology.orchestra.run.smithi149.stdout:ERROR: (22) Invalid argument 2023-09-23T20:29:35.348 INFO:teuthology.orchestra.run.smithi149.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:29:35.725 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:35 smithi149 ceph-mon[170394]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 1 pg peering) 2023-09-23T20:29:35.790 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T20:29:35.811 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:29:35.816 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:35.817 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:29:35.817 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:35 smithi018 ceph-mon[179647]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 1 pg peering) 2023-09-23T20:29:35.817 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:35 smithi018 ceph-mon[184144]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 1 pg peering) 2023-09-23T20:29:35.918 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:29:35.979 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:36.136 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:29:36.145 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:36.145 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:29:36.246 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:29:36.347 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:36.412 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:29:36.422 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:36.422 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:29:36.523 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:29:36.580 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:36 smithi149 ceph-mon[170394]: pgmap v584: 119 pgs: 1 remapped+peering, 2 peering, 31 active+clean+remapped, 2 active+recovery_wait+remapped, 5 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 12 active+undersized, 4 active+recovery_wait+degraded+remapped, 44 active+clean; 4.3 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 6.8 KiB/s rd, 18 KiB/s wr, 620 op/s; 32620/212733 objects degraded (15.334%); 21046/212733 objects misplaced (9.893%); 9.8 MiB/s, 156 objects/s recovering 2023-09-23T20:29:36.580 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:36 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 32902/215271 objects degraded (15.284%), 23 pgs degraded, 15 pgs undersized (PG_DEGRADED) 2023-09-23T20:29:36.582 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:36.609 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:29:36.667 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:29:36.672 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:36.673 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:29:36.734 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:29:36.759 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:29:36.763 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:29:36.774 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:29:36.792 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:29:36.818 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:36 smithi018 ceph-mon[184144]: pgmap v584: 119 pgs: 1 remapped+peering, 2 peering, 31 active+clean+remapped, 2 active+recovery_wait+remapped, 5 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 12 active+undersized, 4 active+recovery_wait+degraded+remapped, 44 active+clean; 4.3 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 6.8 KiB/s rd, 18 KiB/s wr, 620 op/s; 32620/212733 objects degraded (15.334%); 21046/212733 objects misplaced (9.893%); 9.8 MiB/s, 156 objects/s recovering 2023-09-23T20:29:36.818 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:36 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 32902/215271 objects degraded (15.284%), 23 pgs degraded, 15 pgs undersized (PG_DEGRADED) 2023-09-23T20:29:36.818 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:36 smithi018 ceph-mon[179647]: pgmap v584: 119 pgs: 1 remapped+peering, 2 peering, 31 active+clean+remapped, 2 active+recovery_wait+remapped, 5 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 12 active+undersized, 4 active+recovery_wait+degraded+remapped, 44 active+clean; 4.3 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 6.8 KiB/s rd, 18 KiB/s wr, 620 op/s; 32620/212733 objects degraded (15.334%); 21046/212733 objects misplaced (9.893%); 9.8 MiB/s, 156 objects/s recovering 2023-09-23T20:29:36.819 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:36 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 32902/215271 objects degraded (15.284%), 23 pgs degraded, 15 pgs undersized (PG_DEGRADED) 2023-09-23T20:29:36.829 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:36.843 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:29:36.883 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:29:36.899 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:29:36.902 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:36.903 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:29:36.922 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:29:36.965 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:29:36.966 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:29:36.966 INFO:teuthology.orchestra.run.smithi018.stderr:Error ENXIO: problem getting command descriptions from osd.1 2023-09-23T20:29:36.966 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:29:36.966 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:29:36.966 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:29:36.966 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:29:36.966 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:29:36.967 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:29:37.004 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:29:37.088 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:37.141 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 23 20:29:36 smithi018 bash[288326]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-09-23T20:29:37.141 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 23 20:29:36 smithi018 bash[288326]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2023-09-23T20:29:37.141 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 23 20:29:36 smithi018 bash[288326]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_3 --path /var/lib/ceph/osd/ceph-1 --no-mon-config 2023-09-23T20:29:37.141 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 23 20:29:36 smithi018 bash[288326]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_3 /var/lib/ceph/osd/ceph-1/block 2023-09-23T20:29:37.141 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 23 20:29:36 smithi018 bash[288326]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-1/block 2023-09-23T20:29:37.141 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 23 20:29:36 smithi018 bash[288326]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2023-09-23T20:29:37.141 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 23 20:29:36 smithi018 bash[288326]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2023-09-23T20:29:37.142 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 23 20:29:36 smithi018 bash[288326]: --> ceph-volume lvm activate successful for osd ID: 1 2023-09-23T20:29:37.161 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:29:37.177 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:37.177 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:29:37.278 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:29:37.381 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:37.511 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:37 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2126565142' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:29:37.512 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:37 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2126565142' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:29:37.589 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T20:29:37.589 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T20:29:37.723 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:37 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2126565142' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:29:37.764 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:29:37.768 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:37.769 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:29:37.870 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:29:37.934 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:38.003 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:29:38.019 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:38.020 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:29:38.120 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:29:38.199 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:38.391 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 23 20:29:38 smithi018 podman[288965]: 2023-09-23T20:29:38.450 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:29:38.459 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:38.459 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:29:38.560 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:29:38.600 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T20:29:38.630 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:38.722 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:38 smithi149 ceph-mon[170394]: pgmap v585: 119 pgs: 32 active+clean+remapped, 2 active+recovery_wait+remapped, 5 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 12 active+undersized, 4 active+recovery_wait+degraded+remapped, 46 active+clean; 4.2 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 5.6 KiB/s rd, 15 KiB/s wr, 810 op/s; 31788/205125 objects degraded (15.497%); 19985/205125 objects misplaced (9.743%); 9.9 MiB/s, 157 objects/s recovering 2023-09-23T20:29:38.848 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 23 20:29:38 smithi018 ceph-osd[289080]: -- 172.21.15.18:0/3954544253 <== mon.0 v2:172.21.15.18:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55829851a4e0 con 0x5582984eac00 2023-09-23T20:29:38.848 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 23 20:29:38 smithi018 bash[288965]: 2555194d2504f8a6d5d0680950c509826747a553a9515e43d8115f6319826ab6 2023-09-23T20:29:38.848 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 23 20:29:38 smithi018 systemd[1]: Started Ceph osd.1 for a3b3b17e-5a4c-11ee-8db4-212e2dc638e7. 2023-09-23T20:29:38.849 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:38 smithi018 ceph-mon[184144]: pgmap v585: 119 pgs: 32 active+clean+remapped, 2 active+recovery_wait+remapped, 5 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 12 active+undersized, 4 active+recovery_wait+degraded+remapped, 46 active+clean; 4.2 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 5.6 KiB/s rd, 15 KiB/s wr, 810 op/s; 31788/205125 objects degraded (15.497%); 19985/205125 objects misplaced (9.743%); 9.9 MiB/s, 157 objects/s recovering 2023-09-23T20:29:38.851 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:38 smithi018 ceph-mon[179647]: pgmap v585: 119 pgs: 32 active+clean+remapped, 2 active+recovery_wait+remapped, 5 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 12 active+undersized, 4 active+recovery_wait+degraded+remapped, 46 active+clean; 4.2 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 5.6 KiB/s rd, 15 KiB/s wr, 810 op/s; 31788/205125 objects degraded (15.497%); 19985/205125 objects misplaced (9.743%); 9.9 MiB/s, 157 objects/s recovering 2023-09-23T20:29:38.873 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:29:38.878 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:38.879 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:29:38.979 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:29:39.034 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:39.085 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:29:39.089 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:39.089 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:29:39.144 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 23 20:29:38 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-1[289056]: 2023-09-23T20:29:38.843+0000 7f73deddd640 -1 Falling back to public interface 2023-09-23T20:29:39.190 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:29:39.248 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:39.301 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:29:39.305 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:39.306 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:29:39.407 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:29:39.449 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:29:39.473 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:39.478 INFO:teuthology.orchestra.run.smithi018.stderr:noscrub is unset 2023-09-23T20:29:39.504 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:39 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2873690516' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:29:39.504 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:39 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2873690516' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:29:39.591 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:29:39.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:39 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2873690516' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:29:39.808 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:29:39.811 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:39.811 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:29:39.912 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:29:39.973 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:40.101 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:29:40.109 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:40.109 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:29:40.210 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:29:40.271 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:40.330 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:29:40.339 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:40.340 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:29:40.363 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:29:40.435 INFO:teuthology.orchestra.run.smithi149.stdout:{ 2023-09-23T20:29:40.435 INFO:teuthology.orchestra.run.smithi149.stdout: "ops": [ 2023-09-23T20:29:40.435 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:40.435 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.45161.0:145869 6.18 6:1966129b:::benchmark_data_smithi018_270558_object41130:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e264)", 2023-09-23T20:29:40.435 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:29:40.427995+0000", 2023-09-23T20:29:40.435 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 0.0067476089999999999, 2023-09-23T20:29:40.436 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0067766429999999997, 2023-09-23T20:29:40.436 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:29:40.436 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:29:40.436 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:29:40.436 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.45161", 2023-09-23T20:29:40.436 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3795614987", 2023-09-23T20:29:40.436 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 145869 2023-09-23T20:29:40.436 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:29:40.436 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:29:40.436 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:40.436 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:29:40.436 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:40.427995+0000", 2023-09-23T20:29:40.437 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:29:40.437 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:29:40.437 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:40.437 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:29:40.437 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:40.427995+0000", 2023-09-23T20:29:40.437 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:29:40.437 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:29:40.437 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:40.437 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:29:40.437 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:40.427994+0000", 2023-09-23T20:29:40.437 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:29:40.437 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:29:40.438 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:40.438 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:29:40.438 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:40.428001+0000", 2023-09-23T20:29:40.438 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.4019999999999996e-06 2023-09-23T20:29:40.438 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:29:40.438 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:40.438 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:29:40.438 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:40.428017+0000", 2023-09-23T20:29:40.438 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.6027000000000001e-05 2023-09-23T20:29:40.438 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:29:40.438 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:40.438 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:29:40.439 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:40.428024+0000", 2023-09-23T20:29:40.439 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.5719999999999999e-06 2023-09-23T20:29:40.439 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:29:40.439 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:40.439 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:29:40.439 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:40.428082+0000", 2023-09-23T20:29:40.439 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.7930000000000003e-05 2023-09-23T20:29:40.439 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:29:40.439 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:40.439 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:29:40.439 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:40.428192+0000", 2023-09-23T20:29:40.439 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00011034100000000001 2023-09-23T20:29:40.440 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:29:40.440 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:40.440 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 2,3", 2023-09-23T20:29:40.440 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:40.428596+0000", 2023-09-23T20:29:40.440 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00040422800000000002 2023-09-23T20:29:40.440 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:29:40.440 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:40.440 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:29:40.440 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:40.430655+0000", 2023-09-23T20:29:40.440 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.002058341 2023-09-23T20:29:40.440 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:29:40.440 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:29:40.441 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:29:40.441 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:40.432598+0000", 2023-09-23T20:29:40.441 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.001943497 2023-09-23T20:29:40.441 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:29:40.441 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:29:40.441 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:29:40.441 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:29:40.441 INFO:teuthology.orchestra.run.smithi149.stdout: ], 2023-09-23T20:29:40.441 INFO:teuthology.orchestra.run.smithi149.stdout: "num_ops": 1 2023-09-23T20:29:40.441 INFO:teuthology.orchestra.run.smithi149.stdout:} 2023-09-23T20:29:40.442 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:29:40.550 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:40.719 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:40 smithi149 ceph-mon[170394]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:29:40.719 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:40 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2873690516' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:29:40.719 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:40 smithi149 ceph-mon[170394]: osdmap e264: 8 total, 7 up, 7 in 2023-09-23T20:29:40.719 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:40 smithi149 ceph-mon[170394]: failed to encode map e264 with expected crc 2023-09-23T20:29:40.720 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:40 smithi149 ceph-mon[170394]: failed to encode map e264 with expected crc 2023-09-23T20:29:40.720 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:40 smithi149 ceph-mon[170394]: failed to encode map e264 with expected crc 2023-09-23T20:29:40.720 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:40 smithi149 ceph-mon[170394]: pgmap v587: 119 pgs: 32 active+clean+remapped, 2 active+recovery_wait+remapped, 5 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 12 active+undersized, 4 active+recovery_wait+degraded+remapped, 46 active+clean; 4.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 5.1 KiB/s rd, 14 KiB/s wr, 716 op/s; 31498/201666 objects degraded (15.619%); 19683/201666 objects misplaced (9.760%); 3.9 MiB/s, 61 objects/s recovering 2023-09-23T20:29:40.809 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:29:40.821 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:40.821 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:29:40.822 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:40 smithi018 ceph-mon[184144]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:29:40.822 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:40 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2873690516' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:29:40.822 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:40 smithi018 ceph-mon[184144]: osdmap e264: 8 total, 7 up, 7 in 2023-09-23T20:29:40.823 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:40 smithi018 ceph-mon[184144]: failed to encode map e264 with expected crc 2023-09-23T20:29:40.823 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:40 smithi018 ceph-mon[184144]: failed to encode map e264 with expected crc 2023-09-23T20:29:40.823 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:40 smithi018 ceph-mon[184144]: failed to encode map e264 with expected crc 2023-09-23T20:29:40.823 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:40 smithi018 ceph-mon[184144]: pgmap v587: 119 pgs: 32 active+clean+remapped, 2 active+recovery_wait+remapped, 5 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 12 active+undersized, 4 active+recovery_wait+degraded+remapped, 46 active+clean; 4.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 5.1 KiB/s rd, 14 KiB/s wr, 716 op/s; 31498/201666 objects degraded (15.619%); 19683/201666 objects misplaced (9.760%); 3.9 MiB/s, 61 objects/s recovering 2023-09-23T20:29:40.823 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:40 smithi018 ceph-mon[179647]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:29:40.823 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:40 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2873690516' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:29:40.823 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:40 smithi018 ceph-mon[179647]: osdmap e264: 8 total, 7 up, 7 in 2023-09-23T20:29:40.823 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:40 smithi018 ceph-mon[179647]: failed to encode map e264 with expected crc 2023-09-23T20:29:40.823 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:40 smithi018 ceph-mon[179647]: failed to encode map e264 with expected crc 2023-09-23T20:29:40.824 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:40 smithi018 ceph-mon[179647]: failed to encode map e264 with expected crc 2023-09-23T20:29:40.824 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:40 smithi018 ceph-mon[179647]: pgmap v587: 119 pgs: 32 active+clean+remapped, 2 active+recovery_wait+remapped, 5 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 12 active+undersized, 4 active+recovery_wait+degraded+remapped, 46 active+clean; 4.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 5.1 KiB/s rd, 14 KiB/s wr, 716 op/s; 31498/201666 objects degraded (15.619%); 19683/201666 objects misplaced (9.760%); 3.9 MiB/s, 61 objects/s recovering 2023-09-23T20:29:40.892 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T20:29:40.923 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:29:40.995 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:41.065 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:29:41.069 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:41.070 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:29:41.153 INFO:teuthology.orchestra.run.smithi018.stderr:no valid command found; 10 closest matches: 2023-09-23T20:29:41.153 INFO:teuthology.orchestra.run.smithi018.stderr:0 2023-09-23T20:29:41.153 INFO:teuthology.orchestra.run.smithi018.stderr:1 2023-09-23T20:29:41.153 INFO:teuthology.orchestra.run.smithi018.stderr:2 2023-09-23T20:29:41.154 INFO:teuthology.orchestra.run.smithi018.stderr:abort 2023-09-23T20:29:41.154 INFO:teuthology.orchestra.run.smithi018.stderr:assert 2023-09-23T20:29:41.154 INFO:teuthology.orchestra.run.smithi018.stderr:bluestore allocator dump block 2023-09-23T20:29:41.154 INFO:teuthology.orchestra.run.smithi018.stderr:bluestore allocator fragmentation block 2023-09-23T20:29:41.154 INFO:teuthology.orchestra.run.smithi018.stderr:bluestore allocator score block 2023-09-23T20:29:41.154 INFO:teuthology.orchestra.run.smithi018.stderr:config diff 2023-09-23T20:29:41.154 INFO:teuthology.orchestra.run.smithi018.stderr:config diff get 2023-09-23T20:29:41.154 INFO:teuthology.orchestra.run.smithi018.stderr:admin_socket: invalid command 2023-09-23T20:29:41.171 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:29:41.215 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:41.423 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:29:41.428 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:41.429 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:29:41.529 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:29:41.582 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:41.718 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:41 smithi149 ceph-mon[170394]: failed to encode map e264 with expected crc 2023-09-23T20:29:41.718 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:41 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 31498/201666 objects degraded (15.619%), 23 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2023-09-23T20:29:41.727 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:41 smithi018 ceph-mon[179647]: failed to encode map e264 with expected crc 2023-09-23T20:29:41.727 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:41 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 31498/201666 objects degraded (15.619%), 23 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2023-09-23T20:29:41.727 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:41 smithi018 ceph-mon[184144]: failed to encode map e264 with expected crc 2023-09-23T20:29:41.727 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:41 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 31498/201666 objects degraded (15.619%), 23 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2023-09-23T20:29:41.810 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:29:42.112 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T20:29:42.464 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:42 smithi018 ceph-mon[184144]: pgmap v588: 119 pgs: 32 active+clean+remapped, 2 active+recovery_wait+remapped, 5 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 12 active+undersized, 4 active+recovery_wait+degraded+remapped, 46 active+clean; 3.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 870 op/s; 30659/193401 objects degraded (15.853%); 19126/193401 objects misplaced (9.889%); 7.1 MiB/s, 113 objects/s recovering 2023-09-23T20:29:42.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:42 smithi149 ceph-mon[170394]: pgmap v588: 119 pgs: 32 active+clean+remapped, 2 active+recovery_wait+remapped, 5 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 12 active+undersized, 4 active+recovery_wait+degraded+remapped, 46 active+clean; 3.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 870 op/s; 30659/193401 objects degraded (15.853%); 19126/193401 objects misplaced (9.889%); 7.1 MiB/s, 113 objects/s recovering 2023-09-23T20:29:42.781 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:42 smithi018 ceph-mon[179647]: pgmap v588: 119 pgs: 32 active+clean+remapped, 2 active+recovery_wait+remapped, 5 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 12 active+undersized, 4 active+recovery_wait+degraded+remapped, 46 active+clean; 3.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 870 op/s; 30659/193401 objects degraded (15.853%); 19126/193401 objects misplaced (9.889%); 7.1 MiB/s, 113 objects/s recovering 2023-09-23T20:29:42.783 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 23 20:29:42 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-1[289056]: 2023-09-23T20:29:42.647+0000 7f73deddd640 -1 osd.1 236 log_to_monitors true 2023-09-23T20:29:42.891 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:29:42.898 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:42.899 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:29:42.958 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-09-23T20:29:42.959 INFO:tasks.cephadm.ceph_manager.ceph:waiting on admin_socket for osd-1, ['dump_ops_in_flight'] 2023-09-23T20:29:42.999 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:29:43.073 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:43.200 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:29:43.229 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:29:43.239 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:43.239 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:29:43.340 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:29:43.400 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:43.472 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:29:43.479 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:43.479 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:29:43.555 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:29:43.581 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:29:43.586 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:29:43.613 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:29:43.647 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:43 smithi149 ceph-mon[170394]: from='osd.1 [v2:172.21.15.18:6810/190056949,v1:172.21.15.18:6811/190056949]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T20:29:43.650 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:43.656 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:29:43.691 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:29:43.727 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 23 20:29:43 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-1[289056]: 2023-09-23T20:29:43.517+0000 7f73cc242700 -1 osd.1 236 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T20:29:43.727 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:43 smithi018 ceph-mon[179647]: from='osd.1 [v2:172.21.15.18:6810/190056949,v1:172.21.15.18:6811/190056949]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T20:29:43.728 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:43 smithi018 ceph-mon[184144]: from='osd.1 [v2:172.21.15.18:6810/190056949,v1:172.21.15.18:6811/190056949]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T20:29:43.750 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:29:43.781 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:29:43.781 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:29:43.781 INFO:teuthology.orchestra.run.smithi018.stderr:Error ENXIO: problem getting command descriptions from osd.1 2023-09-23T20:29:43.781 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:29:43.781 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:29:43.781 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:29:43.781 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:29:43.781 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:29:43.781 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:29:44.446 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T20:29:44.447 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T20:29:44.469 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:29:44.472 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:44.473 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:29:44.573 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:29:44.602 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:44.645 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:29:44.649 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:44.649 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:29:44.750 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:29:44.778 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:44 smithi018 ceph-mon[184144]: from='osd.1 [v2:172.21.15.18:6810/190056949,v1:172.21.15.18:6811/190056949]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-09-23T20:29:44.779 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:44 smithi018 ceph-mon[184144]: osdmap e265: 8 total, 7 up, 7 in 2023-09-23T20:29:44.779 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:44 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/4281257440' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:29:44.779 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:44 smithi018 ceph-mon[184144]: failed to encode map e265 with expected crc 2023-09-23T20:29:44.779 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:44 smithi018 ceph-mon[184144]: from='osd.1 [v2:172.21.15.18:6810/190056949,v1:172.21.15.18:6811/190056949]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi018", "root=default"]}]: dispatch 2023-09-23T20:29:44.779 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:44 smithi018 ceph-mon[184144]: pgmap v590: 119 pgs: 32 active+clean+remapped, 2 active+recovery_wait+remapped, 5 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 12 active+undersized, 4 active+recovery_wait+degraded+remapped, 46 active+clean; 3.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 1.02k op/s; 30101/188211 objects degraded (15.993%); 18265/188211 objects misplaced (9.705%); 6.0 MiB/s, 95 objects/s recovering 2023-09-23T20:29:44.779 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:44 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:29:44.779 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:44 smithi018 ceph-mon[179647]: from='osd.1 [v2:172.21.15.18:6810/190056949,v1:172.21.15.18:6811/190056949]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-09-23T20:29:44.779 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:44 smithi018 ceph-mon[179647]: osdmap e265: 8 total, 7 up, 7 in 2023-09-23T20:29:44.780 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:44 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/4281257440' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:29:44.780 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:44 smithi018 ceph-mon[179647]: failed to encode map e265 with expected crc 2023-09-23T20:29:44.780 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:44 smithi018 ceph-mon[179647]: from='osd.1 [v2:172.21.15.18:6810/190056949,v1:172.21.15.18:6811/190056949]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi018", "root=default"]}]: dispatch 2023-09-23T20:29:44.780 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:44 smithi018 ceph-mon[179647]: pgmap v590: 119 pgs: 32 active+clean+remapped, 2 active+recovery_wait+remapped, 5 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 12 active+undersized, 4 active+recovery_wait+degraded+remapped, 46 active+clean; 3.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 1.02k op/s; 30101/188211 objects degraded (15.993%); 18265/188211 objects misplaced (9.705%); 6.0 MiB/s, 95 objects/s recovering 2023-09-23T20:29:44.780 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:44 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:29:44.783 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:44.818 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:29:44.821 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:44.821 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:29:44.922 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:29:44.949 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:44.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:44 smithi149 ceph-mon[170394]: from='osd.1 [v2:172.21.15.18:6810/190056949,v1:172.21.15.18:6811/190056949]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-09-23T20:29:44.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:44 smithi149 ceph-mon[170394]: osdmap e265: 8 total, 7 up, 7 in 2023-09-23T20:29:44.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:44 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/4281257440' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:29:44.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:44 smithi149 ceph-mon[170394]: failed to encode map e265 with expected crc 2023-09-23T20:29:44.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:44 smithi149 ceph-mon[170394]: from='osd.1 [v2:172.21.15.18:6810/190056949,v1:172.21.15.18:6811/190056949]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi018", "root=default"]}]: dispatch 2023-09-23T20:29:44.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:44 smithi149 ceph-mon[170394]: pgmap v590: 119 pgs: 32 active+clean+remapped, 2 active+recovery_wait+remapped, 5 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 12 active+undersized, 4 active+recovery_wait+degraded+remapped, 46 active+clean; 3.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 1.02k op/s; 30101/188211 objects degraded (15.993%); 18265/188211 objects misplaced (9.705%); 6.0 MiB/s, 95 objects/s recovering 2023-09-23T20:29:44.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:44 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:29:44.968 INFO:teuthology.orchestra.run.smithi149.stdout:ERROR: (22) Invalid argument 2023-09-23T20:29:44.968 INFO:teuthology.orchestra.run.smithi149.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:29:44.984 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:29:44.987 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:44.987 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:29:45.088 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:29:45.119 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:45.141 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:29:44 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:29:44] "GET /metrics HTTP/1.1" 200 36410 "" "Prometheus/2.43.0" 2023-09-23T20:29:45.519 INFO:teuthology.orchestra.run.smithi018.stderr:nodeep-scrub is unset 2023-09-23T20:29:45.875 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:45 smithi018 ceph-mon[184144]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T20:29:45.875 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:45 smithi018 ceph-mon[184144]: failed to encode map e266 with expected crc 2023-09-23T20:29:45.875 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:45 smithi018 ceph-mon[184144]: osd.1 [v2:172.21.15.18:6810/190056949,v1:172.21.15.18:6811/190056949] boot 2023-09-23T20:29:45.875 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:45 smithi018 ceph-mon[184144]: osdmap e266: 8 total, 8 up, 7 in 2023-09-23T20:29:45.876 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:45 smithi018 ceph-mon[184144]: failed to encode map e266 with expected crc 2023-09-23T20:29:45.876 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:45 smithi018 ceph-mon[184144]: failed to encode map e266 with expected crc 2023-09-23T20:29:45.876 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:45 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T20:29:45.876 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:45 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3787337587' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:29:45.876 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:45 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:29:45.876 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:45 smithi018 ceph-mon[184144]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:29:45.876 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:45 smithi018 ceph-mon[179647]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T20:29:45.876 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:45 smithi018 ceph-mon[179647]: failed to encode map e266 with expected crc 2023-09-23T20:29:45.876 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:45 smithi018 ceph-mon[179647]: osd.1 [v2:172.21.15.18:6810/190056949,v1:172.21.15.18:6811/190056949] boot 2023-09-23T20:29:45.877 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:45 smithi018 ceph-mon[179647]: osdmap e266: 8 total, 8 up, 7 in 2023-09-23T20:29:45.877 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:45 smithi018 ceph-mon[179647]: failed to encode map e266 with expected crc 2023-09-23T20:29:45.877 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:45 smithi018 ceph-mon[179647]: failed to encode map e266 with expected crc 2023-09-23T20:29:45.877 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:45 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T20:29:45.877 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:45 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3787337587' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:29:45.877 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:45 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:29:45.877 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:45 smithi018 ceph-mon[179647]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:29:45.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:45 smithi149 ceph-mon[170394]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T20:29:45.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:45 smithi149 ceph-mon[170394]: failed to encode map e266 with expected crc 2023-09-23T20:29:45.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:45 smithi149 ceph-mon[170394]: osd.1 [v2:172.21.15.18:6810/190056949,v1:172.21.15.18:6811/190056949] boot 2023-09-23T20:29:45.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:45 smithi149 ceph-mon[170394]: osdmap e266: 8 total, 8 up, 7 in 2023-09-23T20:29:45.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:45 smithi149 ceph-mon[170394]: failed to encode map e266 with expected crc 2023-09-23T20:29:45.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:45 smithi149 ceph-mon[170394]: failed to encode map e266 with expected crc 2023-09-23T20:29:45.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:45 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T20:29:45.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:45 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3787337587' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:29:45.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:45 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:29:45.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:45 smithi149 ceph-mon[170394]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:29:46.083 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:29:46.095 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:46.095 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:29:46.159 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T20:29:46.197 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:29:46.247 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:46.296 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:29:46.299 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:46.300 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:29:46.400 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:29:46.448 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:29:46.472 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:46.514 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:29:46.516 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:46 smithi018 ceph-mon[179647]: failed to encode map e267 with expected crc 2023-09-23T20:29:46.516 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:46 smithi018 ceph-mon[179647]: failed to encode map e267 with expected crc 2023-09-23T20:29:46.516 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:46 smithi018 ceph-mon[179647]: failed to encode map e267 with expected crc 2023-09-23T20:29:46.516 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:46 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:29:46.516 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:46 smithi018 ceph-mon[179647]: osdmap e267: 8 total, 8 up, 7 in 2023-09-23T20:29:46.517 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:46 smithi018 ceph-mon[179647]: failed to encode map e267 with expected crc 2023-09-23T20:29:46.517 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:46 smithi018 ceph-mon[179647]: failed to encode map e267 with expected crc 2023-09-23T20:29:46.517 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:46 smithi018 ceph-mon[179647]: failed to encode map e267 with expected crc 2023-09-23T20:29:46.517 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:46 smithi018 ceph-mon[179647]: failed to encode map e267 with expected crc 2023-09-23T20:29:46.517 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:46 smithi018 ceph-mon[179647]: failed to encode map e267 with expected crc 2023-09-23T20:29:46.517 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:46 smithi018 ceph-mon[179647]: failed to encode map e267 with expected crc 2023-09-23T20:29:46.517 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:46 smithi018 ceph-mon[179647]: 6.e continuing backfill to osd.5 from (223'2772,265'9299] MIN to 265'9299 2023-09-23T20:29:46.517 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:46 smithi018 ceph-mon[179647]: pgmap v593: 119 pgs: 7 peering, 6 remapped+peering, 22 active+clean+remapped, 2 active+recovery_wait+remapped, 5 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 11 active+undersized, 2 active+recovery_wait+degraded+remapped, 46 active+clean; 3.8 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 874 op/s; 28388/185925 objects degraded (15.269%); 16046/185925 objects misplaced (8.630%); 8.3 MiB/s, 132 objects/s recovering 2023-09-23T20:29:46.517 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:46 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 30101/188211 objects degraded (15.993%), 23 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2023-09-23T20:29:46.517 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:46 smithi018 ceph-mon[179647]: osdmap e268: 8 total, 8 up, 7 in 2023-09-23T20:29:46.518 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:46 smithi018 ceph-mon[179647]: failed to encode map e268 with expected crc 2023-09-23T20:29:46.518 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:46 smithi018 ceph-mon[179647]: failed to encode map e268 with expected crc 2023-09-23T20:29:46.518 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:46 smithi018 ceph-mon[179647]: failed to encode map e268 with expected crc 2023-09-23T20:29:46.518 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:46 smithi018 ceph-mon[179647]: failed to encode map e268 with expected crc 2023-09-23T20:29:46.518 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:46 smithi018 ceph-mon[184144]: failed to encode map e267 with expected crc 2023-09-23T20:29:46.518 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:46 smithi018 ceph-mon[184144]: failed to encode map e267 with expected crc 2023-09-23T20:29:46.518 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:46 smithi018 ceph-mon[184144]: failed to encode map e267 with expected crc 2023-09-23T20:29:46.518 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:46 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:29:46.518 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:46 smithi018 ceph-mon[184144]: osdmap e267: 8 total, 8 up, 7 in 2023-09-23T20:29:46.518 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:46 smithi018 ceph-mon[184144]: failed to encode map e267 with expected crc 2023-09-23T20:29:46.519 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:46 smithi018 ceph-mon[184144]: failed to encode map e267 with expected crc 2023-09-23T20:29:46.519 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:46 smithi018 ceph-mon[184144]: failed to encode map e267 with expected crc 2023-09-23T20:29:46.520 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:46.520 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:29:46.621 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:29:46.653 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:46.689 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:29:46.692 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:46.693 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:29:46.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:46 smithi149 ceph-mon[170394]: failed to encode map e267 with expected crc 2023-09-23T20:29:46.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:46 smithi149 ceph-mon[170394]: failed to encode map e267 with expected crc 2023-09-23T20:29:46.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:46 smithi149 ceph-mon[170394]: failed to encode map e267 with expected crc 2023-09-23T20:29:46.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:46 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:29:46.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:46 smithi149 ceph-mon[170394]: osdmap e267: 8 total, 8 up, 7 in 2023-09-23T20:29:46.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:46 smithi149 ceph-mon[170394]: failed to encode map e267 with expected crc 2023-09-23T20:29:46.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:46 smithi149 ceph-mon[170394]: failed to encode map e267 with expected crc 2023-09-23T20:29:46.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:46 smithi149 ceph-mon[170394]: failed to encode map e267 with expected crc 2023-09-23T20:29:46.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:46 smithi149 ceph-mon[170394]: failed to encode map e267 with expected crc 2023-09-23T20:29:46.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:46 smithi149 ceph-mon[170394]: failed to encode map e267 with expected crc 2023-09-23T20:29:46.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:46 smithi149 ceph-mon[170394]: failed to encode map e267 with expected crc 2023-09-23T20:29:46.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:46 smithi149 ceph-mon[170394]: 6.e continuing backfill to osd.5 from (223'2772,265'9299] MIN to 265'9299 2023-09-23T20:29:46.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:46 smithi149 ceph-mon[170394]: pgmap v593: 119 pgs: 7 peering, 6 remapped+peering, 22 active+clean+remapped, 2 active+recovery_wait+remapped, 5 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 11 active+undersized, 2 active+recovery_wait+degraded+remapped, 46 active+clean; 3.8 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 874 op/s; 28388/185925 objects degraded (15.269%); 16046/185925 objects misplaced (8.630%); 8.3 MiB/s, 132 objects/s recovering 2023-09-23T20:29:46.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:46 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 30101/188211 objects degraded (15.993%), 23 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2023-09-23T20:29:46.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:46 smithi149 ceph-mon[170394]: osdmap e268: 8 total, 8 up, 7 in 2023-09-23T20:29:46.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:46 smithi149 ceph-mon[170394]: failed to encode map e268 with expected crc 2023-09-23T20:29:46.718 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:46 smithi149 ceph-mon[170394]: failed to encode map e268 with expected crc 2023-09-23T20:29:46.718 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:46 smithi149 ceph-mon[170394]: failed to encode map e268 with expected crc 2023-09-23T20:29:46.718 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:46 smithi149 ceph-mon[170394]: failed to encode map e268 with expected crc 2023-09-23T20:29:46.793 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:29:46.821 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:46.861 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:29:46.863 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:46.863 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:29:46.891 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:29:46.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:46 smithi018 ceph-mon[184144]: failed to encode map e267 with expected crc 2023-09-23T20:29:46.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:46 smithi018 ceph-mon[184144]: failed to encode map e267 with expected crc 2023-09-23T20:29:46.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:46 smithi018 ceph-mon[184144]: failed to encode map e267 with expected crc 2023-09-23T20:29:46.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:46 smithi018 ceph-mon[184144]: 6.e continuing backfill to osd.5 from (223'2772,265'9299] MIN to 265'9299 2023-09-23T20:29:46.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:46 smithi018 ceph-mon[184144]: pgmap v593: 119 pgs: 7 peering, 6 remapped+peering, 22 active+clean+remapped, 2 active+recovery_wait+remapped, 5 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 11 active+undersized, 2 active+recovery_wait+degraded+remapped, 46 active+clean; 3.8 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 874 op/s; 28388/185925 objects degraded (15.269%); 16046/185925 objects misplaced (8.630%); 8.3 MiB/s, 132 objects/s recovering 2023-09-23T20:29:46.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:46 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 30101/188211 objects degraded (15.993%), 23 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2023-09-23T20:29:46.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:46 smithi018 ceph-mon[184144]: osdmap e268: 8 total, 8 up, 7 in 2023-09-23T20:29:46.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:46 smithi018 ceph-mon[184144]: failed to encode map e268 with expected crc 2023-09-23T20:29:46.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:46 smithi018 ceph-mon[184144]: failed to encode map e268 with expected crc 2023-09-23T20:29:46.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:46 smithi018 ceph-mon[184144]: failed to encode map e268 with expected crc 2023-09-23T20:29:46.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:46 smithi018 ceph-mon[184144]: failed to encode map e268 with expected crc 2023-09-23T20:29:46.964 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:29:46.993 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:47.033 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:29:47.035 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:47.036 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:29:47.121 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:29:47.137 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:29:47.207 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:47.276 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:29:47.285 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:47.285 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:29:47.386 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:29:47.458 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:47.521 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:29:47.535 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:47.536 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:29:47.636 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:29:47.646 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:47 smithi018 ceph-mon[179647]: failed to encode map e268 with expected crc 2023-09-23T20:29:47.646 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:47 smithi018 ceph-mon[179647]: failed to encode map e268 with expected crc 2023-09-23T20:29:47.646 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:47 smithi018 ceph-mon[179647]: osdmap e269: 8 total, 8 up, 7 in 2023-09-23T20:29:47.646 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:47 smithi018 ceph-mon[179647]: failed to encode map e269 with expected crc 2023-09-23T20:29:47.648 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:47 smithi018 ceph-mon[184144]: failed to encode map e268 with expected crc 2023-09-23T20:29:47.648 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:47 smithi018 ceph-mon[184144]: failed to encode map e268 with expected crc 2023-09-23T20:29:47.648 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:47 smithi018 ceph-mon[184144]: osdmap e269: 8 total, 8 up, 7 in 2023-09-23T20:29:47.648 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:47 smithi018 ceph-mon[184144]: failed to encode map e269 with expected crc 2023-09-23T20:29:47.691 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:47.855 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:47 smithi149 ceph-mon[170394]: failed to encode map e268 with expected crc 2023-09-23T20:29:47.856 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:47 smithi149 ceph-mon[170394]: failed to encode map e268 with expected crc 2023-09-23T20:29:47.856 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:47 smithi149 ceph-mon[170394]: osdmap e269: 8 total, 8 up, 7 in 2023-09-23T20:29:47.856 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:47 smithi149 ceph-mon[170394]: failed to encode map e269 with expected crc 2023-09-23T20:29:47.878 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T20:29:47.959 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T20:29:48.871 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:29:48.874 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:48.874 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:29:48.975 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:29:49.056 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:49.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:49 smithi018 ceph-mon[179647]: pgmap v596: 119 pgs: 15 peering, 12 remapped+peering, 14 active+clean+remapped, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+degraded, 1 active+recovering+undersized+remapped, 8 active+undersized, 50 active+clean; 3.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 584 op/s; 24038/181194 objects degraded (13.266%); 12359/181194 objects misplaced (6.821%); 11 MiB/s, 176 objects/s recovering 2023-09-23T20:29:49.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:49 smithi018 ceph-mon[179647]: osdmap e270: 8 total, 8 up, 7 in 2023-09-23T20:29:49.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:49 smithi018 ceph-mon[179647]: failed to encode map e270 with expected crc 2023-09-23T20:29:49.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:49 smithi018 ceph-mon[179647]: failed to encode map e270 with expected crc 2023-09-23T20:29:49.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:49 smithi018 ceph-mon[179647]: failed to encode map e270 with expected crc 2023-09-23T20:29:49.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:49 smithi018 ceph-mon[179647]: failed to encode map e270 with expected crc 2023-09-23T20:29:49.144 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:49 smithi018 ceph-mon[184144]: pgmap v596: 119 pgs: 15 peering, 12 remapped+peering, 14 active+clean+remapped, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+degraded, 1 active+recovering+undersized+remapped, 8 active+undersized, 50 active+clean; 3.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 584 op/s; 24038/181194 objects degraded (13.266%); 12359/181194 objects misplaced (6.821%); 11 MiB/s, 176 objects/s recovering 2023-09-23T20:29:49.144 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:49 smithi018 ceph-mon[184144]: osdmap e270: 8 total, 8 up, 7 in 2023-09-23T20:29:49.144 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:49 smithi018 ceph-mon[184144]: failed to encode map e270 with expected crc 2023-09-23T20:29:49.144 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:49 smithi018 ceph-mon[184144]: failed to encode map e270 with expected crc 2023-09-23T20:29:49.144 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:49 smithi018 ceph-mon[184144]: failed to encode map e270 with expected crc 2023-09-23T20:29:49.144 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:49 smithi018 ceph-mon[184144]: failed to encode map e270 with expected crc 2023-09-23T20:29:49.218 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:49 smithi149 ceph-mon[170394]: pgmap v596: 119 pgs: 15 peering, 12 remapped+peering, 14 active+clean+remapped, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+degraded, 1 active+recovering+undersized+remapped, 8 active+undersized, 50 active+clean; 3.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 584 op/s; 24038/181194 objects degraded (13.266%); 12359/181194 objects misplaced (6.821%); 11 MiB/s, 176 objects/s recovering 2023-09-23T20:29:49.218 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:49 smithi149 ceph-mon[170394]: osdmap e270: 8 total, 8 up, 7 in 2023-09-23T20:29:49.218 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:49 smithi149 ceph-mon[170394]: failed to encode map e270 with expected crc 2023-09-23T20:29:49.218 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:49 smithi149 ceph-mon[170394]: failed to encode map e270 with expected crc 2023-09-23T20:29:49.219 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:49 smithi149 ceph-mon[170394]: failed to encode map e270 with expected crc 2023-09-23T20:29:49.219 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:49 smithi149 ceph-mon[170394]: failed to encode map e270 with expected crc 2023-09-23T20:29:49.984 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:29:49.989 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:49.989 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:29:50.090 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:29:50.185 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:50.206 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:29:50.207 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:29:50.220 INFO:teuthology.orchestra.run.smithi149.stdout:ERROR: (22) Invalid argument 2023-09-23T20:29:50.220 INFO:teuthology.orchestra.run.smithi149.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:29:50.240 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:29:50.248 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:50.249 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:29:50.349 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:29:50.389 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:50.820 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:29:50.824 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:50.824 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:29:50.828 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T20:29:50.925 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:29:50.954 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:50.992 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:29:50.995 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:50.995 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:29:51.051 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:29:51.080 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:29:51.094 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:51 smithi149 ceph-mon[170394]: pgmap v598: 119 pgs: 15 peering, 12 remapped+peering, 5 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 72 active+clean; 3.6 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 632 op/s; 25885/178770 objects degraded (14.479%); 12224/178770 objects misplaced (6.838%); 11 MiB/s, 170 objects/s recovering 2023-09-23T20:29:51.094 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:51 smithi149 ceph-mon[170394]: osdmap e271: 8 total, 8 up, 7 in 2023-09-23T20:29:51.096 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:29:51.104 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:29:51.127 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:29:51.134 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:51.152 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:29:51.174 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:29:51.176 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:29:51.179 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:51.179 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:29:51.196 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:29:51.215 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:29:51.216 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:29:51.216 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:29:51.216 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:29:51.216 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:29:51.217 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:29:51.217 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:29:51.217 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:29:51.217 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:29:51.279 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:29:51.305 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:51 smithi018 ceph-mon[179647]: pgmap v598: 119 pgs: 15 peering, 12 remapped+peering, 5 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 72 active+clean; 3.6 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 632 op/s; 25885/178770 objects degraded (14.479%); 12224/178770 objects misplaced (6.838%); 11 MiB/s, 170 objects/s recovering 2023-09-23T20:29:51.305 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:51 smithi018 ceph-mon[179647]: osdmap e271: 8 total, 8 up, 7 in 2023-09-23T20:29:51.305 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:51 smithi018 ceph-mon[179647]: failed to encode map e271 with expected crc 2023-09-23T20:29:51.305 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:51 smithi018 ceph-mon[179647]: failed to encode map e271 with expected crc 2023-09-23T20:29:51.305 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:51 smithi018 ceph-mon[179647]: failed to encode map e271 with expected crc 2023-09-23T20:29:51.305 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:51 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/998079376' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:29:51.305 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:51 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 25885/178770 objects degraded (14.479%), 16 pgs degraded (PG_DEGRADED) 2023-09-23T20:29:51.305 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:51 smithi018 ceph-mon[184144]: pgmap v598: 119 pgs: 15 peering, 12 remapped+peering, 5 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 72 active+clean; 3.6 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 632 op/s; 25885/178770 objects degraded (14.479%); 12224/178770 objects misplaced (6.838%); 11 MiB/s, 170 objects/s recovering 2023-09-23T20:29:51.306 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:51 smithi018 ceph-mon[184144]: osdmap e271: 8 total, 8 up, 7 in 2023-09-23T20:29:51.306 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:51 smithi018 ceph-mon[184144]: failed to encode map e271 with expected crc 2023-09-23T20:29:51.306 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:51 smithi018 ceph-mon[184144]: failed to encode map e271 with expected crc 2023-09-23T20:29:51.306 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:51 smithi018 ceph-mon[184144]: failed to encode map e271 with expected crc 2023-09-23T20:29:51.306 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:51 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/998079376' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:29:51.306 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:51 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 25885/178770 objects degraded (14.479%), 16 pgs degraded (PG_DEGRADED) 2023-09-23T20:29:51.321 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:51.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:51 smithi149 ceph-mon[170394]: failed to encode map e271 with expected crc 2023-09-23T20:29:51.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:51 smithi149 ceph-mon[170394]: failed to encode map e271 with expected crc 2023-09-23T20:29:51.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:51 smithi149 ceph-mon[170394]: failed to encode map e271 with expected crc 2023-09-23T20:29:51.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:51 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/998079376' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:29:51.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:51 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 25885/178770 objects degraded (14.479%), 16 pgs degraded (PG_DEGRADED) 2023-09-23T20:29:52.156 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:29:52.165 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:52.165 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:29:52.204 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:52 smithi018 ceph-mon[179647]: failed to encode map e272 with expected crc 2023-09-23T20:29:52.204 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:52 smithi018 ceph-mon[179647]: osdmap e272: 8 total, 8 up, 7 in 2023-09-23T20:29:52.204 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:52 smithi018 ceph-mon[179647]: failed to encode map e272 with expected crc 2023-09-23T20:29:52.204 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:52 smithi018 ceph-mon[184144]: failed to encode map e272 with expected crc 2023-09-23T20:29:52.204 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:52 smithi018 ceph-mon[184144]: osdmap e272: 8 total, 8 up, 7 in 2023-09-23T20:29:52.204 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:52 smithi018 ceph-mon[184144]: failed to encode map e272 with expected crc 2023-09-23T20:29:52.266 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:29:52.319 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:29:52.324 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:52.376 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:29:52.381 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:52 smithi149 ceph-mon[170394]: failed to encode map e272 with expected crc 2023-09-23T20:29:52.381 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:52 smithi149 ceph-mon[170394]: osdmap e272: 8 total, 8 up, 7 in 2023-09-23T20:29:52.381 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:52 smithi149 ceph-mon[170394]: failed to encode map e272 with expected crc 2023-09-23T20:29:52.382 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:52.382 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:29:52.483 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:29:52.546 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:52.609 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:29:52.615 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:52.616 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:29:52.716 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:29:52.795 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:52.875 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:29:52.887 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:52.887 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:29:52.988 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:29:53.088 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:53.144 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:53 smithi018 ceph-mon[179647]: pgmap v601: 119 pgs: 2 peering, 11 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 86 active+clean; 3.6 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 536 op/s; 38319/174651 objects degraded (21.940%); 14173/174651 objects misplaced (8.115%); 5.8 MiB/s, 239 objects/s recovering 2023-09-23T20:29:53.144 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:53 smithi018 ceph-mon[184144]: pgmap v601: 119 pgs: 2 peering, 11 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 86 active+clean; 3.6 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 536 op/s; 38319/174651 objects degraded (21.940%); 14173/174651 objects misplaced (8.115%); 5.8 MiB/s, 239 objects/s recovering 2023-09-23T20:29:53.200 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:29:53.215 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:53.216 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:29:53.317 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:29:53.325 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2023-09-23T20:29:53.325 INFO:teuthology.orchestra.run.smithi018.stdout: "ops": [ 2023-09-23T20:29:53.325 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:29:53.325 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_repop(client.45161.0:153421 6.b e272/267 6:df027892:::benchmark_data_smithi018_270558_object48682:head v 272'9383, mlcod=235'4440)", 2023-09-23T20:29:53.325 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:29:53.304501+0000", 2023-09-23T20:29:53.326 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.019333564000000001, 2023-09-23T20:29:53.326 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.019373301999999998, 2023-09-23T20:29:53.326 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:29:53.326 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "started", 2023-09-23T20:29:53.326 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:29:53.326 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:29:53.326 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:29:53.326 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:53.304501+0000", 2023-09-23T20:29:53.327 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:29:53.327 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:29:53.327 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:29:53.327 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:29:53.327 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:53.304501+0000", 2023-09-23T20:29:53.327 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:29:53.327 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:29:53.327 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:29:53.327 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:29:53.328 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:53.304501+0000", 2023-09-23T20:29:53.328 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:29:53.328 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:29:53.328 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:29:53.328 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:29:53.328 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:53.304507+0000", 2023-09-23T20:29:53.328 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.3419999999999996e-06 2023-09-23T20:29:53.328 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:29:53.328 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:29:53.329 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:29:53.329 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:53.304515+0000", 2023-09-23T20:29:53.329 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.028e-06 2023-09-23T20:29:53.329 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:29:53.329 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:29:53.329 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:29:53.329 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:53.304521+0000", 2023-09-23T20:29:53.329 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.3879999999999999e-06 2023-09-23T20:29:53.330 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:29:53.330 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:29:53.330 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:29:53.330 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:53.304593+0000", 2023-09-23T20:29:53.330 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.2361999999999996e-05 2023-09-23T20:29:53.330 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:29:53.330 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:29:53.330 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:29:53.330 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:53.304613+0000", 2023-09-23T20:29:53.331 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.0151000000000001e-05 2023-09-23T20:29:53.331 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:29:53.331 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:29:53.331 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:29:53.331 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:29:53.331 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:29:53.331 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_repop(client.45161.0:153423 6.b e272/267 6:d2b99075:::benchmark_data_smithi018_270558_object48684:head v 272'9384, mlcod=235'4440)", 2023-09-23T20:29:53.331 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:29:53.304557+0000", 2023-09-23T20:29:53.331 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.019277869, 2023-09-23T20:29:53.331 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.019390706000000001, 2023-09-23T20:29:53.331 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:29:53.331 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "started", 2023-09-23T20:29:53.332 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:29:53.332 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:29:53.332 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:29:53.332 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:53.304557+0000", 2023-09-23T20:29:53.332 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:29:53.332 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:29:53.332 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:29:53.332 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:29:53.332 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:53.304557+0000", 2023-09-23T20:29:53.332 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:29:53.332 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:29:53.333 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:29:53.333 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:29:53.333 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:53.304557+0000", 2023-09-23T20:29:53.333 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967296 2023-09-23T20:29:53.333 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:29:53.333 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:29:53.333 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:29:53.333 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:53.304564+0000", 2023-09-23T20:29:53.333 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.0679999999999999e-06 2023-09-23T20:29:53.334 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:29:53.334 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:29:53.334 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:29:53.334 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:53.304569+0000", 2023-09-23T20:29:53.334 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.1959999999999999e-06 2023-09-23T20:29:53.334 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:29:53.334 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:29:53.334 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:29:53.334 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:53.304573+0000", 2023-09-23T20:29:53.335 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.9700000000000001e-06 2023-09-23T20:29:53.335 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:29:53.335 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:29:53.335 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:29:53.335 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:53.310528+0000", 2023-09-23T20:29:53.335 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0059551910000000003 2023-09-23T20:29:53.335 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:29:53.335 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:29:53.335 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:29:53.336 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:53.310556+0000", 2023-09-23T20:29:53.336 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.7636000000000001e-05 2023-09-23T20:29:53.336 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:29:53.336 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:29:53.336 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:29:53.336 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:29:53.336 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:29:53.336 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_repop(client.45161.0:153425 6.b e272/267 6:d768a571:::benchmark_data_smithi018_270558_object48686:head v 272'9385, mlcod=235'4440)", 2023-09-23T20:29:53.336 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:29:53.304590+0000", 2023-09-23T20:29:53.337 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.019244936000000001, 2023-09-23T20:29:53.337 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.019422708, 2023-09-23T20:29:53.337 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:29:53.337 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "started", 2023-09-23T20:29:53.337 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:29:53.337 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:29:53.337 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:29:53.337 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:53.304590+0000", 2023-09-23T20:29:53.337 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:29:53.338 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:29:53.338 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:29:53.338 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:29:53.338 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:53.304590+0000", 2023-09-23T20:29:53.338 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:29:53.338 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:29:53.338 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:29:53.338 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:29:53.339 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:53.304590+0000", 2023-09-23T20:29:53.339 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967296 2023-09-23T20:29:53.339 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:29:53.339 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:29:53.339 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:29:53.339 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:53.304593+0000", 2023-09-23T20:29:53.339 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.2100000000000002e-06 2023-09-23T20:29:53.339 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:29:53.339 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:29:53.340 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:29:53.340 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:53.304598+0000", 2023-09-23T20:29:53.340 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.1139999999999997e-06 2023-09-23T20:29:53.340 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:29:53.340 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:29:53.340 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:29:53.340 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:53.304602+0000", 2023-09-23T20:29:53.340 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.9859999999999996e-06 2023-09-23T20:29:53.341 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:29:53.341 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:29:53.341 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:29:53.341 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:53.316805+0000", 2023-09-23T20:29:53.341 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.012202737 2023-09-23T20:29:53.341 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:29:53.341 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:29:53.341 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:29:53.341 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:53.316832+0000", 2023-09-23T20:29:53.342 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.6903999999999999e-05 2023-09-23T20:29:53.342 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:29:53.342 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:29:53.342 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:29:53.342 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:29:53.342 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:29:53.342 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_repop(client.45161.0:153428 6.e e272/266 6:7f4d5972:::benchmark_data_smithi018_270558_object48689:head v 272'9544, mlcod=235'4522)", 2023-09-23T20:29:53.342 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:29:53.308866+0000", 2023-09-23T20:29:53.342 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.014968911, 2023-09-23T20:29:53.343 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.015213908999999999, 2023-09-23T20:29:53.343 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:29:53.343 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "started", 2023-09-23T20:29:53.343 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:29:53.343 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:29:53.343 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:29:53.343 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:53.308866+0000", 2023-09-23T20:29:53.343 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:29:53.343 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:29:53.343 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:29:53.343 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:29:53.344 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:53.308866+0000", 2023-09-23T20:29:53.344 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:29:53.344 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:29:53.344 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:29:53.344 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:29:53.344 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:53.308865+0000", 2023-09-23T20:29:53.344 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.999999 2023-09-23T20:29:53.344 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:29:53.344 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:29:53.344 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:29:53.344 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:53.308875+0000", 2023-09-23T20:29:53.344 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.5529999999999998e-06 2023-09-23T20:29:53.345 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:29:53.345 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:29:53.345 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:29:53.345 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:53.308886+0000", 2023-09-23T20:29:53.345 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1505000000000001e-05 2023-09-23T20:29:53.345 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:29:53.345 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:29:53.345 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:29:53.345 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:53.308894+0000", 2023-09-23T20:29:53.345 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.6499999999999996e-06 2023-09-23T20:29:53.345 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:29:53.346 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:29:53.346 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:29:53.346 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:53.316348+0000", 2023-09-23T20:29:53.346 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.007454189 2023-09-23T20:29:53.346 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:29:53.346 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:29:53.346 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:29:53.346 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:53.316377+0000", 2023-09-23T20:29:53.346 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.9110000000000001e-05 2023-09-23T20:29:53.346 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:29:53.346 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:29:53.347 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:29:53.347 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:29:53.347 INFO:teuthology.orchestra.run.smithi018.stdout: ], 2023-09-23T20:29:53.347 INFO:teuthology.orchestra.run.smithi018.stdout: "num_ops": 4 2023-09-23T20:29:53.347 INFO:teuthology.orchestra.run.smithi018.stdout:} 2023-09-23T20:29:53.362 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:53 smithi149 ceph-mon[170394]: pgmap v601: 119 pgs: 2 peering, 11 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 86 active+clean; 3.6 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 536 op/s; 38319/174651 objects degraded (21.940%); 14173/174651 objects misplaced (8.115%); 5.8 MiB/s, 239 objects/s recovering 2023-09-23T20:29:53.373 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:53.429 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:29:53.434 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:53.434 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:29:53.535 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:29:53.604 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:53.665 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:29:53.668 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:53.668 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:29:53.769 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:29:53.823 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:53.887 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:29:53.900 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:53.900 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:29:54.001 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:29:54.054 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd reweight 0 1 2023-09-23T20:29:54.145 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:54.172 INFO:teuthology.orchestra.run.smithi018.stderr:noscrub is set 2023-09-23T20:29:54.251 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:29:54.308 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:54 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/735500676' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:29:54.308 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:54 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:29:54.308 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:54 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/735500676' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:29:54.308 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:54 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:29:54.334 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:29:54.340 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:54.341 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:29:54.442 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:29:54.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:54 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/735500676' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:29:54.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:54 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:29:54.595 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:54.688 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:29:54.696 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:54.696 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:29:54.763 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2023-09-23T20:29:54.763 INFO:teuthology.orchestra.run.smithi018.stdout: "ops": [ 2023-09-23T20:29:54.764 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:29:54.764 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.45161.0:154398 6.0 6:04becb00:::benchmark_data_smithi018_270558_object49659:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e273)", 2023-09-23T20:29:54.764 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:29:54.754873+0000", 2023-09-23T20:29:54.764 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.0085971399999999996, 2023-09-23T20:29:54.764 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0086234109999999992, 2023-09-23T20:29:54.764 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:29:54.764 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:29:54.764 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:29:54.764 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.45161", 2023-09-23T20:29:54.764 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3795614987", 2023-09-23T20:29:54.764 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 154398 2023-09-23T20:29:54.765 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:29:54.765 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:29:54.765 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:29:54.765 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:29:54.765 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:54.754873+0000", 2023-09-23T20:29:54.765 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:29:54.765 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:29:54.765 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:29:54.765 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:29:54.765 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:54.754873+0000", 2023-09-23T20:29:54.765 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:29:54.765 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:29:54.766 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:29:54.766 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:29:54.766 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:54.754871+0000", 2023-09-23T20:29:54.766 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:29:54.766 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:29:54.766 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:29:54.766 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:29:54.766 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:54.754880+0000", 2023-09-23T20:29:54.766 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.8040000000000007e-06 2023-09-23T20:29:54.766 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:29:54.766 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:29:54.767 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:29:54.767 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:54.754901+0000", 2023-09-23T20:29:54.767 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.1189999999999999e-05 2023-09-23T20:29:54.767 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:29:54.767 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:29:54.767 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:29:54.767 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:54.754911+0000", 2023-09-23T20:29:54.767 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.7699999999999996e-06 2023-09-23T20:29:54.767 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:29:54.767 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:29:54.767 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:29:54.767 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:54.754963+0000", 2023-09-23T20:29:54.768 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.1734999999999998e-05 2023-09-23T20:29:54.768 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:29:54.768 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:29:54.768 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:29:54.768 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:54.755090+0000", 2023-09-23T20:29:54.768 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00012697500000000001 2023-09-23T20:29:54.768 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:29:54.768 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:29:54.768 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 2,3", 2023-09-23T20:29:54.768 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:54.755596+0000", 2023-09-23T20:29:54.768 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00050586399999999997 2023-09-23T20:29:54.769 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:29:54.769 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:29:54.769 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:29:54.769 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:54.762778+0000", 2023-09-23T20:29:54.769 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0071827430000000001 2023-09-23T20:29:54.769 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:29:54.769 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:29:54.769 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:29:54.769 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:29:54.769 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:29:54.769 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_repop(client.45161.0:154397 6.d e273/267 6:b7969dc9:::benchmark_data_smithi018_270558_object49658:head v 273'9803, mlcod=235'4594)", 2023-09-23T20:29:54.769 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:29:54.760957+0000", 2023-09-23T20:29:54.769 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.0025126520000000002, 2023-09-23T20:29:54.770 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0026056579999999998, 2023-09-23T20:29:54.770 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:29:54.770 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "started", 2023-09-23T20:29:54.770 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:29:54.770 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:29:54.770 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:29:54.770 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:54.760957+0000", 2023-09-23T20:29:54.770 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:29:54.770 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:29:54.770 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:29:54.770 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:29:54.770 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:54.760957+0000", 2023-09-23T20:29:54.771 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:29:54.771 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:29:54.771 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:29:54.771 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:29:54.771 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:54.760956+0000", 2023-09-23T20:29:54.771 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.999999 2023-09-23T20:29:54.771 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:29:54.771 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:29:54.771 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:29:54.771 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:54.760967+0000", 2023-09-23T20:29:54.771 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.084e-05 2023-09-23T20:29:54.771 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:29:54.771 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:29:54.772 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:29:54.772 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:54.760989+0000", 2023-09-23T20:29:54.772 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.1993000000000001e-05 2023-09-23T20:29:54.772 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:29:54.772 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:29:54.772 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:29:54.772 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:54.760999+0000", 2023-09-23T20:29:54.772 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0171e-05 2023-09-23T20:29:54.772 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:29:54.772 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:29:54.772 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:29:54.772 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:54.761067+0000", 2023-09-23T20:29:54.772 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.7884000000000003e-05 2023-09-23T20:29:54.773 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:29:54.773 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:29:54.773 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:29:54.773 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:54.761096+0000", 2023-09-23T20:29:54.773 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.8385e-05 2023-09-23T20:29:54.773 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:29:54.773 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:29:54.773 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:29:54.773 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:29:54.773 INFO:teuthology.orchestra.run.smithi018.stdout: ], 2023-09-23T20:29:54.773 INFO:teuthology.orchestra.run.smithi018.stdout: "num_ops": 2 2023-09-23T20:29:54.773 INFO:teuthology.orchestra.run.smithi018.stdout:} 2023-09-23T20:29:54.797 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:29:54.907 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:55.214 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:29:55.221 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:55 smithi018 ceph-mon[184144]: pgmap v602: 119 pgs: 1 peering, 10 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 88 active+clean; 3.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 565 op/s; 38141/171000 objects degraded (22.305%); 13812/171000 objects misplaced (8.077%); 4.5 MiB/s, 183 objects/s recovering 2023-09-23T20:29:55.221 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:55 smithi018 ceph-mon[184144]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:29:55.222 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:55 smithi018 ceph-mon[184144]: failed to encode map e273 with expected crc 2023-09-23T20:29:55.222 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:55 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:29:55.222 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:55 smithi018 ceph-mon[184144]: osdmap e273: 8 total, 8 up, 7 in 2023-09-23T20:29:55.222 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:29:54 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:29:54] "GET /metrics HTTP/1.1" 200 36410 "" "Prometheus/2.43.0" 2023-09-23T20:29:55.222 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:55 smithi018 ceph-mon[179647]: pgmap v602: 119 pgs: 1 peering, 10 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 88 active+clean; 3.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 565 op/s; 38141/171000 objects degraded (22.305%); 13812/171000 objects misplaced (8.077%); 4.5 MiB/s, 183 objects/s recovering 2023-09-23T20:29:55.222 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:55 smithi018 ceph-mon[179647]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:29:55.223 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:55 smithi018 ceph-mon[179647]: failed to encode map e273 with expected crc 2023-09-23T20:29:55.223 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:55 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:29:55.223 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:55 smithi018 ceph-mon[179647]: osdmap e273: 8 total, 8 up, 7 in 2023-09-23T20:29:55.231 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:55.231 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:29:55.332 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T20:29:55.332 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:29:55.431 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:55.468 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:55 smithi149 ceph-mon[170394]: pgmap v602: 119 pgs: 1 peering, 10 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 88 active+clean; 3.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 565 op/s; 38141/171000 objects degraded (22.305%); 13812/171000 objects misplaced (8.077%); 4.5 MiB/s, 183 objects/s recovering 2023-09-23T20:29:55.468 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:55 smithi149 ceph-mon[170394]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:29:55.468 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:55 smithi149 ceph-mon[170394]: failed to encode map e273 with expected crc 2023-09-23T20:29:55.468 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:55 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:29:55.468 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:55 smithi149 ceph-mon[170394]: osdmap e273: 8 total, 8 up, 7 in 2023-09-23T20:29:55.472 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:29:55.479 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:29:55.517 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:29:55.525 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:55.525 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:29:55.626 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:29:55.674 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:55.724 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:29:55.728 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:55.729 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:29:55.829 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:29:55.870 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:55.926 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:29:55.933 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:55.934 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:29:56.034 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:29:56.091 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:56.147 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:29:56.150 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:56.150 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:29:56.221 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:56 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 38141/171000 objects degraded (22.305%), 25 pgs degraded (PG_DEGRADED) 2023-09-23T20:29:56.252 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:29:56.304 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:56.358 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:29:56.370 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:56 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 38141/171000 objects degraded (22.305%), 25 pgs degraded (PG_DEGRADED) 2023-09-23T20:29:56.371 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:56 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 38141/171000 objects degraded (22.305%), 25 pgs degraded (PG_DEGRADED) 2023-09-23T20:29:56.373 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:56.373 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:29:56.474 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:29:56.551 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:56.661 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:29:56.662 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:29:56.676 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:56.676 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:29:56.759 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T20:29:56.777 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:29:56.886 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:56.979 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:29:56.989 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:56.989 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:29:57.090 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:29:57.219 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:57.297 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:57 smithi018 ceph-mon[179647]: pgmap v604: 119 pgs: 1 peering, 10 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 88 active+clean; 3.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 524 op/s; 38141/169323 objects degraded (22.526%); 13444/169323 objects misplaced (7.940%); 8.3 MiB/s, 244 objects/s recovering 2023-09-23T20:29:57.298 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:57 smithi018 ceph-mon[184144]: pgmap v604: 119 pgs: 1 peering, 10 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 88 active+clean; 3.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 524 op/s; 38141/169323 objects degraded (22.526%); 13444/169323 objects misplaced (7.940%); 8.3 MiB/s, 244 objects/s recovering 2023-09-23T20:29:57.353 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:29:57.362 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:57.363 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:29:57.463 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:29:57.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:57 smithi149 ceph-mon[170394]: pgmap v604: 119 pgs: 1 peering, 10 active+recovery_wait+degraded, 3 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 88 active+clean; 3.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 524 op/s; 38141/169323 objects degraded (22.526%); 13444/169323 objects misplaced (7.940%); 8.3 MiB/s, 244 objects/s recovering 2023-09-23T20:29:57.508 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:57.550 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:29:57.554 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:57.555 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:29:57.655 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:29:57.743 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:57.880 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:29:57.890 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:57.890 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:29:57.991 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:29:58.121 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:58.193 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:29:58.199 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:58.199 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:29:58.226 INFO:teuthology.orchestra.run.smithi018.stderr:reweighted osd.0 to 1 (10000) 2023-09-23T20:29:58.300 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:29:58.317 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:29:58.339 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:58.387 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:58 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1764862447' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T20:29:58.388 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:58 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1764862447' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T20:29:58.393 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:29:58.398 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:58.399 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:29:58.443 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:29:58.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:58 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1764862447' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T20:29:58.485 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:29:58.500 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:29:58.520 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:29:58.563 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:29:58.599 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:58.602 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:29:58.632 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:29:58.675 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:29:58.684 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:29:58.687 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:58.688 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:29:58.717 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:29:58.750 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:29:58.750 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:29:58.750 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:29:58.750 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:29:58.751 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:29:58.751 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:29:58.751 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:29:58.751 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:29:58.788 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:29:58.858 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:58.894 INFO:teuthology.orchestra.run.smithi018.stdout:ERROR: (22) Invalid argument 2023-09-23T20:29:58.894 INFO:teuthology.orchestra.run.smithi018.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:29:58.988 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd reweight 1 1 2023-09-23T20:29:59.003 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:29:59.005 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:59.006 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:29:59.106 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:29:59.147 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:59.214 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:29:59.218 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:59.219 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:29:59.310 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:59 smithi018 ceph-mon[179647]: pgmap v605: 119 pgs: 10 active+recovery_wait+degraded, 1 active+recovery_wait, 3 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 88 active+clean; 3.4 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 579 op/s; 37922/164742 objects degraded (23.019%); 12579/164742 objects misplaced (7.636%); 12 MiB/s, 282 objects/s recovering 2023-09-23T20:29:59.310 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:59 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1764862447' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T20:29:59.310 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:59 smithi018 ceph-mon[179647]: osdmap e274: 8 total, 8 up, 7 in 2023-09-23T20:29:59.310 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:59 smithi018 ceph-mon[179647]: failed to encode map e274 with expected crc 2023-09-23T20:29:59.310 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:59 smithi018 ceph-mon[179647]: failed to encode map e274 with expected crc 2023-09-23T20:29:59.310 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:29:59 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/452969679' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:29:59.311 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:59 smithi018 ceph-mon[184144]: pgmap v605: 119 pgs: 10 active+recovery_wait+degraded, 1 active+recovery_wait, 3 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 88 active+clean; 3.4 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 579 op/s; 37922/164742 objects degraded (23.019%); 12579/164742 objects misplaced (7.636%); 12 MiB/s, 282 objects/s recovering 2023-09-23T20:29:59.311 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:59 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1764862447' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T20:29:59.311 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:59 smithi018 ceph-mon[184144]: osdmap e274: 8 total, 8 up, 7 in 2023-09-23T20:29:59.311 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:59 smithi018 ceph-mon[184144]: failed to encode map e274 with expected crc 2023-09-23T20:29:59.311 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:59 smithi018 ceph-mon[184144]: failed to encode map e274 with expected crc 2023-09-23T20:29:59.311 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:29:59 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/452969679' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:29:59.319 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:29:59.450 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:59.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:59 smithi149 ceph-mon[170394]: pgmap v605: 119 pgs: 10 active+recovery_wait+degraded, 1 active+recovery_wait, 3 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 88 active+clean; 3.4 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 579 op/s; 37922/164742 objects degraded (23.019%); 12579/164742 objects misplaced (7.636%); 12 MiB/s, 282 objects/s recovering 2023-09-23T20:29:59.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:59 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1764862447' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T20:29:59.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:59 smithi149 ceph-mon[170394]: osdmap e274: 8 total, 8 up, 7 in 2023-09-23T20:29:59.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:59 smithi149 ceph-mon[170394]: failed to encode map e274 with expected crc 2023-09-23T20:29:59.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:59 smithi149 ceph-mon[170394]: failed to encode map e274 with expected crc 2023-09-23T20:29:59.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:29:59 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/452969679' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:29:59.684 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:29:59.693 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:59.694 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:29:59.795 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:29:59.805 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T20:29:59.832 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:29:59.874 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:29:59.880 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:29:59.880 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:29:59.981 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:30:00.114 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:00.196 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:30:00.204 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:00.204 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:30:00.305 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:30:00.335 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:00.374 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:30:00.377 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:00.378 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:30:00.469 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:00 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:30:00.469 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:00 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:30:00.469 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:00 smithi018 ceph-mon[179647]: pgmap v607: 119 pgs: 10 active+recovery_wait+degraded, 1 active+recovery_wait, 3 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 88 active+clean; 3.3 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 543 op/s; 37922/161604 objects degraded (23.466%); 12368/161604 objects misplaced (7.653%); 8.1 MiB/s, 129 objects/s recovering 2023-09-23T20:30:00.469 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:00 smithi018 ceph-mon[179647]: Health detail: HEALTH_WARN noscrub flag(s) set; Degraded data redundancy: 37922/164742 objects degraded (23.019%), 25 pgs degraded 2023-09-23T20:30:00.469 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:00 smithi018 ceph-mon[179647]: [WRN] OSDMAP_FLAGS: noscrub flag(s) set 2023-09-23T20:30:00.469 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:00 smithi018 ceph-mon[179647]: [WRN] PG_DEGRADED: Degraded data redundancy: 37922/164742 objects degraded (23.019%), 25 pgs degraded 2023-09-23T20:30:00.469 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:00 smithi018 ceph-mon[179647]: pg 1.c is active+recovery_wait+undersized+degraded+remapped, acting [3,0] 2023-09-23T20:30:00.469 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:00 smithi018 ceph-mon[179647]: pg 2.2 is active+recovery_wait+degraded, acting [5,1,6] 2023-09-23T20:30:00.469 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:00 smithi018 ceph-mon[179647]: pg 2.5 is active+recovery_wait+undersized+degraded+remapped, acting [7,0] 2023-09-23T20:30:00.470 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:00 smithi018 ceph-mon[179647]: pg 2.6 is active+recovery_wait+degraded, acting [1,6,3] 2023-09-23T20:30:00.470 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:00 smithi018 ceph-mon[179647]: pg 2.7 is active+recovery_wait+undersized+degraded+remapped, acting [2,7] 2023-09-23T20:30:00.470 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:00 smithi018 ceph-mon[179647]: pg 3.e is active+recovery_wait+degraded, acting [7,1,0] 2023-09-23T20:30:00.470 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:00 smithi018 ceph-mon[179647]: pg 3.f is active+recovery_wait+degraded, acting [7,0,1] 2023-09-23T20:30:00.470 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:00 smithi018 ceph-mon[179647]: pg 3.13 is active+recovery_wait+degraded, acting [7,2,6] 2023-09-23T20:30:00.470 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:00 smithi018 ceph-mon[179647]: pg 3.14 is active+recovery_wait+degraded, acting [7,6,2] 2023-09-23T20:30:00.470 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:00 smithi018 ceph-mon[179647]: pg 3.15 is active+recovery_wait+degraded, acting [7,3,0] 2023-09-23T20:30:00.470 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:00 smithi018 ceph-mon[179647]: pg 3.1c is active+recovery_wait+degraded, acting [5,1,6] 2023-09-23T20:30:00.470 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:00 smithi018 ceph-mon[179647]: pg 6.2 is active+recovery_wait+undersized+degraded+remapped, acting [3,2] 2023-09-23T20:30:00.470 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:00 smithi018 ceph-mon[179647]: pg 6.3 is active+recovery_wait+degraded, acting [7,6,2] 2023-09-23T20:30:00.470 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:00 smithi018 ceph-mon[179647]: pg 6.4 is active+recovery_wait+undersized+degraded+remapped, acting [3,5] 2023-09-23T20:30:00.470 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:00 smithi018 ceph-mon[179647]: pg 6.5 is active+recovery_wait+undersized+degraded+remapped, acting [7,3] 2023-09-23T20:30:00.471 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:00 smithi018 ceph-mon[179647]: pg 6.7 is active+recovery_wait+undersized+degraded+remapped, acting [5,3] 2023-09-23T20:30:00.471 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:00 smithi018 ceph-mon[179647]: pg 6.b is active+recovery_wait+undersized+degraded+remapped, acting [3,7] 2023-09-23T20:30:00.471 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:00 smithi018 ceph-mon[179647]: pg 6.c is active+recovery_wait+undersized+degraded+remapped, acting [3,5] 2023-09-23T20:30:00.471 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:00 smithi018 ceph-mon[179647]: pg 6.d is active+recovery_wait+undersized+degraded+remapped, acting [5,0] 2023-09-23T20:30:00.471 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:00 smithi018 ceph-mon[179647]: pg 6.e is active+recovery_wait+undersized+degraded+remapped, acting [2,4] 2023-09-23T20:30:00.471 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:00 smithi018 ceph-mon[179647]: pg 6.11 is active+recovery_wait+undersized+degraded+remapped, acting [6,2] 2023-09-23T20:30:00.471 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:00 smithi018 ceph-mon[179647]: pg 6.12 is active+recovery_wait+undersized+degraded+remapped, acting [3,2] 2023-09-23T20:30:00.471 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:00 smithi018 ceph-mon[179647]: pg 6.13 is active+recovery_wait+degraded, acting [7,6,2] 2023-09-23T20:30:00.471 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:00 smithi018 ceph-mon[179647]: pg 6.14 is active+recovery_wait+undersized+degraded+remapped, acting [3,5] 2023-09-23T20:30:00.471 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:00 smithi018 ceph-mon[179647]: pg 6.17 is active+recovery_wait+undersized+degraded+remapped, acting [5,3] 2023-09-23T20:30:00.472 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:00 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:30:00.472 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:00 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:30:00.472 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:00 smithi018 ceph-mon[184144]: pgmap v607: 119 pgs: 10 active+recovery_wait+degraded, 1 active+recovery_wait, 3 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 88 active+clean; 3.3 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 543 op/s; 37922/161604 objects degraded (23.466%); 12368/161604 objects misplaced (7.653%); 8.1 MiB/s, 129 objects/s recovering 2023-09-23T20:30:00.472 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:00 smithi018 ceph-mon[184144]: Health detail: HEALTH_WARN noscrub flag(s) set; Degraded data redundancy: 37922/164742 objects degraded (23.019%), 25 pgs degraded 2023-09-23T20:30:00.472 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:00 smithi018 ceph-mon[184144]: [WRN] OSDMAP_FLAGS: noscrub flag(s) set 2023-09-23T20:30:00.472 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:00 smithi018 ceph-mon[184144]: [WRN] PG_DEGRADED: Degraded data redundancy: 37922/164742 objects degraded (23.019%), 25 pgs degraded 2023-09-23T20:30:00.472 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:00 smithi018 ceph-mon[184144]: pg 1.c is active+recovery_wait+undersized+degraded+remapped, acting [3,0] 2023-09-23T20:30:00.472 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:00 smithi018 ceph-mon[184144]: pg 2.2 is active+recovery_wait+degraded, acting [5,1,6] 2023-09-23T20:30:00.472 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:00 smithi018 ceph-mon[184144]: pg 2.5 is active+recovery_wait+undersized+degraded+remapped, acting [7,0] 2023-09-23T20:30:00.472 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:00 smithi018 ceph-mon[184144]: pg 2.6 is active+recovery_wait+degraded, acting [1,6,3] 2023-09-23T20:30:00.472 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:00 smithi018 ceph-mon[184144]: pg 2.7 is active+recovery_wait+undersized+degraded+remapped, acting [2,7] 2023-09-23T20:30:00.473 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:00 smithi018 ceph-mon[184144]: pg 3.e is active+recovery_wait+degraded, acting [7,1,0] 2023-09-23T20:30:00.473 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:00 smithi018 ceph-mon[184144]: pg 3.f is active+recovery_wait+degraded, acting [7,0,1] 2023-09-23T20:30:00.473 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:00 smithi018 ceph-mon[184144]: pg 3.13 is active+recovery_wait+degraded, acting [7,2,6] 2023-09-23T20:30:00.473 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:00 smithi018 ceph-mon[184144]: pg 3.14 is active+recovery_wait+degraded, acting [7,6,2] 2023-09-23T20:30:00.473 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:00 smithi018 ceph-mon[184144]: pg 3.15 is active+recovery_wait+degraded, acting [7,3,0] 2023-09-23T20:30:00.473 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:00 smithi018 ceph-mon[184144]: pg 3.1c is active+recovery_wait+degraded, acting [5,1,6] 2023-09-23T20:30:00.473 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:00 smithi018 ceph-mon[184144]: pg 6.2 is active+recovery_wait+undersized+degraded+remapped, acting [3,2] 2023-09-23T20:30:00.473 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:00 smithi018 ceph-mon[184144]: pg 6.3 is active+recovery_wait+degraded, acting [7,6,2] 2023-09-23T20:30:00.473 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:00 smithi018 ceph-mon[184144]: pg 6.4 is active+recovery_wait+undersized+degraded+remapped, acting [3,5] 2023-09-23T20:30:00.473 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:00 smithi018 ceph-mon[184144]: pg 6.5 is active+recovery_wait+undersized+degraded+remapped, acting [7,3] 2023-09-23T20:30:00.473 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:00 smithi018 ceph-mon[184144]: pg 6.7 is active+recovery_wait+undersized+degraded+remapped, acting [5,3] 2023-09-23T20:30:00.474 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:00 smithi018 ceph-mon[184144]: pg 6.b is active+recovery_wait+undersized+degraded+remapped, acting [3,7] 2023-09-23T20:30:00.474 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:00 smithi018 ceph-mon[184144]: pg 6.c is active+recovery_wait+undersized+degraded+remapped, acting [3,5] 2023-09-23T20:30:00.474 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:00 smithi018 ceph-mon[184144]: pg 6.d is active+recovery_wait+undersized+degraded+remapped, acting [5,0] 2023-09-23T20:30:00.474 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:00 smithi018 ceph-mon[184144]: pg 6.e is active+recovery_wait+undersized+degraded+remapped, acting [2,4] 2023-09-23T20:30:00.474 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:00 smithi018 ceph-mon[184144]: pg 6.11 is active+recovery_wait+undersized+degraded+remapped, acting [6,2] 2023-09-23T20:30:00.474 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:00 smithi018 ceph-mon[184144]: pg 6.12 is active+recovery_wait+undersized+degraded+remapped, acting [3,2] 2023-09-23T20:30:00.474 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:00 smithi018 ceph-mon[184144]: pg 6.13 is active+recovery_wait+degraded, acting [7,6,2] 2023-09-23T20:30:00.474 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:00 smithi018 ceph-mon[184144]: pg 6.14 is active+recovery_wait+undersized+degraded+remapped, acting [3,5] 2023-09-23T20:30:00.474 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:00 smithi018 ceph-mon[184144]: pg 6.17 is active+recovery_wait+undersized+degraded+remapped, acting [5,3] 2023-09-23T20:30:00.478 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:30:00.478 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:30:00.525 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:00.580 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:30:00.584 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:00.585 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:30:00.585 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:00 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:30:00.585 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:00 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:30:00.585 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:00 smithi149 ceph-mon[170394]: pgmap v607: 119 pgs: 10 active+recovery_wait+degraded, 1 active+recovery_wait, 3 active+remapped+backfill_wait, 15 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 88 active+clean; 3.3 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 543 op/s; 37922/161604 objects degraded (23.466%); 12368/161604 objects misplaced (7.653%); 8.1 MiB/s, 129 objects/s recovering 2023-09-23T20:30:00.585 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:00 smithi149 ceph-mon[170394]: Health detail: HEALTH_WARN noscrub flag(s) set; Degraded data redundancy: 37922/164742 objects degraded (23.019%), 25 pgs degraded 2023-09-23T20:30:00.585 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:00 smithi149 ceph-mon[170394]: [WRN] OSDMAP_FLAGS: noscrub flag(s) set 2023-09-23T20:30:00.585 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:00 smithi149 ceph-mon[170394]: [WRN] PG_DEGRADED: Degraded data redundancy: 37922/164742 objects degraded (23.019%), 25 pgs degraded 2023-09-23T20:30:00.585 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:00 smithi149 ceph-mon[170394]: pg 1.c is active+recovery_wait+undersized+degraded+remapped, acting [3,0] 2023-09-23T20:30:00.586 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:00 smithi149 ceph-mon[170394]: pg 2.2 is active+recovery_wait+degraded, acting [5,1,6] 2023-09-23T20:30:00.586 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:00 smithi149 ceph-mon[170394]: pg 2.5 is active+recovery_wait+undersized+degraded+remapped, acting [7,0] 2023-09-23T20:30:00.586 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:00 smithi149 ceph-mon[170394]: pg 2.6 is active+recovery_wait+degraded, acting [1,6,3] 2023-09-23T20:30:00.586 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:00 smithi149 ceph-mon[170394]: pg 2.7 is active+recovery_wait+undersized+degraded+remapped, acting [2,7] 2023-09-23T20:30:00.586 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:00 smithi149 ceph-mon[170394]: pg 3.e is active+recovery_wait+degraded, acting [7,1,0] 2023-09-23T20:30:00.586 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:00 smithi149 ceph-mon[170394]: pg 3.f is active+recovery_wait+degraded, acting [7,0,1] 2023-09-23T20:30:00.586 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:00 smithi149 ceph-mon[170394]: pg 3.13 is active+recovery_wait+degraded, acting [7,2,6] 2023-09-23T20:30:00.586 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:00 smithi149 ceph-mon[170394]: pg 3.14 is active+recovery_wait+degraded, acting [7,6,2] 2023-09-23T20:30:00.586 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:00 smithi149 ceph-mon[170394]: pg 3.15 is active+recovery_wait+degraded, acting [7,3,0] 2023-09-23T20:30:00.586 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:00 smithi149 ceph-mon[170394]: pg 3.1c is active+recovery_wait+degraded, acting [5,1,6] 2023-09-23T20:30:00.586 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:00 smithi149 ceph-mon[170394]: pg 6.2 is active+recovery_wait+undersized+degraded+remapped, acting [3,2] 2023-09-23T20:30:00.587 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:00 smithi149 ceph-mon[170394]: pg 6.3 is active+recovery_wait+degraded, acting [7,6,2] 2023-09-23T20:30:00.587 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:00 smithi149 ceph-mon[170394]: pg 6.4 is active+recovery_wait+undersized+degraded+remapped, acting [3,5] 2023-09-23T20:30:00.587 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:00 smithi149 ceph-mon[170394]: pg 6.5 is active+recovery_wait+undersized+degraded+remapped, acting [7,3] 2023-09-23T20:30:00.587 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:00 smithi149 ceph-mon[170394]: pg 6.7 is active+recovery_wait+undersized+degraded+remapped, acting [5,3] 2023-09-23T20:30:00.587 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:00 smithi149 ceph-mon[170394]: pg 6.b is active+recovery_wait+undersized+degraded+remapped, acting [3,7] 2023-09-23T20:30:00.587 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:00 smithi149 ceph-mon[170394]: pg 6.c is active+recovery_wait+undersized+degraded+remapped, acting [3,5] 2023-09-23T20:30:00.587 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:00 smithi149 ceph-mon[170394]: pg 6.d is active+recovery_wait+undersized+degraded+remapped, acting [5,0] 2023-09-23T20:30:00.587 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:00 smithi149 ceph-mon[170394]: pg 6.e is active+recovery_wait+undersized+degraded+remapped, acting [2,4] 2023-09-23T20:30:00.587 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:00 smithi149 ceph-mon[170394]: pg 6.11 is active+recovery_wait+undersized+degraded+remapped, acting [6,2] 2023-09-23T20:30:00.587 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:00 smithi149 ceph-mon[170394]: pg 6.12 is active+recovery_wait+undersized+degraded+remapped, acting [3,2] 2023-09-23T20:30:00.587 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:00 smithi149 ceph-mon[170394]: pg 6.13 is active+recovery_wait+degraded, acting [7,6,2] 2023-09-23T20:30:00.588 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:00 smithi149 ceph-mon[170394]: pg 6.14 is active+recovery_wait+undersized+degraded+remapped, acting [3,5] 2023-09-23T20:30:00.588 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:00 smithi149 ceph-mon[170394]: pg 6.17 is active+recovery_wait+undersized+degraded+remapped, acting [5,3] 2023-09-23T20:30:00.685 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:30:00.718 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:00.764 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:30:00.768 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:00.769 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:30:00.869 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:30:00.925 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:30:00.955 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:01.025 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:30:01.036 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:01.037 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:30:01.138 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:30:01.167 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:01.208 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:30:01.211 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:01.212 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:30:01.313 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:30:01.313 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:30:01.379 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:01.494 INFO:teuthology.orchestra.run.smithi018.stderr:nodeep-scrub is set 2023-09-23T20:30:01.625 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:30:01.637 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:01.638 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:30:01.638 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:01 smithi018 ceph-mon[179647]: failed to encode map e275 with expected crc 2023-09-23T20:30:01.638 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:01 smithi018 ceph-mon[179647]: osdmap e275: 8 total, 8 up, 7 in 2023-09-23T20:30:01.638 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:01 smithi018 ceph-mon[179647]: failed to encode map e275 with expected crc 2023-09-23T20:30:01.638 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:01 smithi018 ceph-mon[179647]: failed to encode map e275 with expected crc 2023-09-23T20:30:01.638 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:01 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 37922/161604 objects degraded (23.466%), 25 pgs degraded (PG_DEGRADED) 2023-09-23T20:30:01.638 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:01 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3863539689' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:30:01.639 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:01 smithi018 ceph-mon[184144]: failed to encode map e275 with expected crc 2023-09-23T20:30:01.640 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:01 smithi018 ceph-mon[184144]: osdmap e275: 8 total, 8 up, 7 in 2023-09-23T20:30:01.640 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:01 smithi018 ceph-mon[184144]: failed to encode map e275 with expected crc 2023-09-23T20:30:01.640 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:01 smithi018 ceph-mon[184144]: failed to encode map e275 with expected crc 2023-09-23T20:30:01.640 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:01 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 37922/161604 objects degraded (23.466%), 25 pgs degraded (PG_DEGRADED) 2023-09-23T20:30:01.640 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:01 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3863539689' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:30:01.739 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:30:01.766 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:01 smithi149 ceph-mon[170394]: failed to encode map e275 with expected crc 2023-09-23T20:30:01.766 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:01 smithi149 ceph-mon[170394]: osdmap e275: 8 total, 8 up, 7 in 2023-09-23T20:30:01.767 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:01 smithi149 ceph-mon[170394]: failed to encode map e275 with expected crc 2023-09-23T20:30:01.767 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:01 smithi149 ceph-mon[170394]: failed to encode map e275 with expected crc 2023-09-23T20:30:01.767 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:01 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 37922/161604 objects degraded (23.466%), 25 pgs degraded (PG_DEGRADED) 2023-09-23T20:30:01.767 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:01 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3863539689' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:30:01.770 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:01.822 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:30:01.825 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:01.825 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:30:01.926 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:30:01.962 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:02.041 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:30:02.046 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:02.046 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:30:02.147 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:30:02.183 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:02.231 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:30:02.234 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:02.235 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:30:02.336 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:30:02.429 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:02.512 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:02 smithi018 ceph-mon[184144]: failed to encode map e275 with expected crc 2023-09-23T20:30:02.512 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:02 smithi018 ceph-mon[184144]: failed to encode map e275 with expected crc 2023-09-23T20:30:02.513 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:02 smithi018 ceph-mon[184144]: failed to encode map e275 with expected crc 2023-09-23T20:30:02.513 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:02 smithi018 ceph-mon[184144]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:30:02.513 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:02 smithi018 ceph-mon[184144]: failed to encode map e276 with expected crc 2023-09-23T20:30:02.513 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:02 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3863539689' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:30:02.513 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:02 smithi018 ceph-mon[184144]: failed to encode map e276 with expected crc 2023-09-23T20:30:02.513 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:02 smithi018 ceph-mon[184144]: osdmap e276: 8 total, 8 up, 7 in 2023-09-23T20:30:02.513 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:02 smithi018 ceph-mon[184144]: failed to encode map e276 with expected crc 2023-09-23T20:30:02.513 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:02 smithi018 ceph-mon[184144]: pgmap v610: 119 pgs: 1 peering, 4 active+recovery_wait+degraded, 1 active+recovery_wait, 3 active+remapped+backfill_wait, 16 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.2 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 745 op/s; 39639/155889 objects degraded (25.428%); 11443/155889 objects misplaced (7.340%); 13 MiB/s, 1 keys/s, 271 objects/s recovering 2023-09-23T20:30:02.550 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:30:02.561 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:02.561 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:30:02.662 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:30:02.697 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:02.744 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:30:02.748 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:02.748 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:30:02.812 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:02 smithi018 ceph-mon[179647]: failed to encode map e275 with expected crc 2023-09-23T20:30:02.812 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:02 smithi018 ceph-mon[179647]: failed to encode map e275 with expected crc 2023-09-23T20:30:02.812 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:02 smithi018 ceph-mon[179647]: failed to encode map e275 with expected crc 2023-09-23T20:30:02.812 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:02 smithi018 ceph-mon[179647]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:30:02.812 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:02 smithi018 ceph-mon[179647]: failed to encode map e276 with expected crc 2023-09-23T20:30:02.813 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:02 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3863539689' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:30:02.813 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:02 smithi018 ceph-mon[179647]: failed to encode map e276 with expected crc 2023-09-23T20:30:02.813 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:02 smithi018 ceph-mon[179647]: osdmap e276: 8 total, 8 up, 7 in 2023-09-23T20:30:02.813 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:02 smithi018 ceph-mon[179647]: failed to encode map e276 with expected crc 2023-09-23T20:30:02.813 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:02 smithi018 ceph-mon[179647]: pgmap v610: 119 pgs: 1 peering, 4 active+recovery_wait+degraded, 1 active+recovery_wait, 3 active+remapped+backfill_wait, 16 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.2 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 745 op/s; 39639/155889 objects degraded (25.428%); 11443/155889 objects misplaced (7.340%); 13 MiB/s, 1 keys/s, 271 objects/s recovering 2023-09-23T20:30:02.848 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:30:02.925 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:02.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:02 smithi149 ceph-mon[170394]: failed to encode map e275 with expected crc 2023-09-23T20:30:02.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:02 smithi149 ceph-mon[170394]: failed to encode map e275 with expected crc 2023-09-23T20:30:02.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:02 smithi149 ceph-mon[170394]: failed to encode map e275 with expected crc 2023-09-23T20:30:02.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:02 smithi149 ceph-mon[170394]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:30:02.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:02 smithi149 ceph-mon[170394]: failed to encode map e276 with expected crc 2023-09-23T20:30:02.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:02 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3863539689' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:30:02.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:02 smithi149 ceph-mon[170394]: failed to encode map e276 with expected crc 2023-09-23T20:30:02.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:02 smithi149 ceph-mon[170394]: osdmap e276: 8 total, 8 up, 7 in 2023-09-23T20:30:02.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:02 smithi149 ceph-mon[170394]: failed to encode map e276 with expected crc 2023-09-23T20:30:02.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:02 smithi149 ceph-mon[170394]: pgmap v610: 119 pgs: 1 peering, 4 active+recovery_wait+degraded, 1 active+recovery_wait, 3 active+remapped+backfill_wait, 16 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 3.2 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 745 op/s; 39639/155889 objects degraded (25.428%); 11443/155889 objects misplaced (7.340%); 13 MiB/s, 1 keys/s, 271 objects/s recovering 2023-09-23T20:30:03.148 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:30:03.156 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:03.156 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:30:03.182 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:30:03.257 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:30:03.291 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:03.332 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:30:03.338 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:03.338 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:30:03.439 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:30:03.473 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:03.508 INFO:teuthology.orchestra.run.smithi018.stdout:ERROR: (22) Invalid argument 2023-09-23T20:30:03.508 INFO:teuthology.orchestra.run.smithi018.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:30:03.517 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:30:03.523 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:03.523 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:30:03.624 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:30:03.750 INFO:teuthology.orchestra.run.smithi018.stderr:reweighted osd.1 to 1 (10000) 2023-09-23T20:30:03.755 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:03 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/172674257' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T20:30:03.755 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:03 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/172674257' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T20:30:03.775 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:03.994 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:30:04.010 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:04.010 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:30:04.111 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T20:30:04.112 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:30:04.196 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:04.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:03 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/172674257' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T20:30:04.498 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:30:04.511 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:04.512 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:30:04.606 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd reweight 5 1 2023-09-23T20:30:04.613 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:30:04.634 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T20:30:04.762 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:04 smithi018 ceph-mon[184144]: pgmap v611: 119 pgs: 1 peering, 4 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 3 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 3.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 706 op/s; 35628/152007 objects degraded (23.438%); 9672/152007 objects misplaced (6.363%); 9.1 MiB/s, 1 keys/s, 205 objects/s recovering 2023-09-23T20:30:04.762 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:04 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/172674257' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T20:30:04.762 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:04 smithi018 ceph-mon[184144]: osdmap e277: 8 total, 8 up, 7 in 2023-09-23T20:30:04.762 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:04 smithi018 ceph-mon[184144]: failed to encode map e277 with expected crc 2023-09-23T20:30:04.762 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:04 smithi018 ceph-mon[184144]: failed to encode map e277 with expected crc 2023-09-23T20:30:04.762 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:04 smithi018 ceph-mon[184144]: failed to encode map e277 with expected crc 2023-09-23T20:30:04.762 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:04 smithi018 ceph-mon[184144]: failed to encode map e277 with expected crc 2023-09-23T20:30:04.762 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:04 smithi018 ceph-mon[184144]: failed to encode map e277 with expected crc 2023-09-23T20:30:04.763 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:04 smithi018 ceph-mon[179647]: pgmap v611: 119 pgs: 1 peering, 4 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 3 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 3.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 706 op/s; 35628/152007 objects degraded (23.438%); 9672/152007 objects misplaced (6.363%); 9.1 MiB/s, 1 keys/s, 205 objects/s recovering 2023-09-23T20:30:04.763 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:04 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/172674257' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T20:30:04.763 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:04 smithi018 ceph-mon[179647]: osdmap e277: 8 total, 8 up, 7 in 2023-09-23T20:30:04.763 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:04 smithi018 ceph-mon[179647]: failed to encode map e277 with expected crc 2023-09-23T20:30:04.763 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:04 smithi018 ceph-mon[179647]: failed to encode map e277 with expected crc 2023-09-23T20:30:04.763 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:04 smithi018 ceph-mon[179647]: failed to encode map e277 with expected crc 2023-09-23T20:30:04.763 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:04 smithi018 ceph-mon[179647]: failed to encode map e277 with expected crc 2023-09-23T20:30:04.763 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:04 smithi018 ceph-mon[179647]: failed to encode map e277 with expected crc 2023-09-23T20:30:04.778 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:04.887 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:30:04.901 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:04.902 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:30:05.002 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:30:05.032 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:04 smithi149 ceph-mon[170394]: pgmap v611: 119 pgs: 1 peering, 4 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 3 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 3.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 706 op/s; 35628/152007 objects degraded (23.438%); 9672/152007 objects misplaced (6.363%); 9.1 MiB/s, 1 keys/s, 205 objects/s recovering 2023-09-23T20:30:05.032 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:04 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/172674257' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T20:30:05.032 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:04 smithi149 ceph-mon[170394]: osdmap e277: 8 total, 8 up, 7 in 2023-09-23T20:30:05.032 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:04 smithi149 ceph-mon[170394]: failed to encode map e277 with expected crc 2023-09-23T20:30:05.032 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:04 smithi149 ceph-mon[170394]: failed to encode map e277 with expected crc 2023-09-23T20:30:05.032 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:04 smithi149 ceph-mon[170394]: failed to encode map e277 with expected crc 2023-09-23T20:30:05.032 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:04 smithi149 ceph-mon[170394]: failed to encode map e277 with expected crc 2023-09-23T20:30:05.033 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:04 smithi149 ceph-mon[170394]: failed to encode map e277 with expected crc 2023-09-23T20:30:05.036 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:05.082 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:30:05.086 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:05.087 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:30:05.132 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:30:05.152 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:30:05 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:30:05] "GET /metrics HTTP/1.1" 200 36577 "" "Prometheus/2.43.0" 2023-09-23T20:30:05.187 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:30:05.281 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:05.426 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:30:05.432 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:05.432 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:30:05.533 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:30:05.636 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:30:05.649 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:05.651 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:30:05.688 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:30:05.733 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:30:05.739 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:05.740 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:30:05.748 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:30:05.760 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:30:05.803 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:30:05.831 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:30:05.841 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:30:05.862 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:30:05.882 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:05.914 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:30:05.932 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:30:05.935 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:05.935 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:30:05.970 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:30:05.974 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:30:05.974 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:30:05.974 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:30:05.974 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:30:05.974 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:30:05.974 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:30:05.974 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:30:05.974 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:30:06.023 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:05 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1125289251' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:30:06.025 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:05 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1125289251' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:30:06.035 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:30:06.073 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:05 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1125289251' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:30:06.075 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:06.127 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:30:06.130 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:06.130 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:30:06.235 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:30:06.273 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:06.326 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:30:06.330 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:06.330 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:30:06.431 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:30:06.471 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:06.536 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:30:06.541 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:06.541 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:30:06.642 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:30:06.781 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:06.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:06 smithi018 ceph-mon[179647]: pgmap v613: 119 pgs: 1 peering, 4 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 3 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 3.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 622 op/s; 35628/150384 objects degraded (23.691%); 9672/150384 objects misplaced (6.432%); 9.1 MiB/s, 1 keys/s, 205 objects/s recovering 2023-09-23T20:30:06.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:06 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 35628/150384 objects degraded (23.691%), 19 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:30:06.902 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:06 smithi018 ceph-mon[184144]: pgmap v613: 119 pgs: 1 peering, 4 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 3 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 3.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 622 op/s; 35628/150384 objects degraded (23.691%); 9672/150384 objects misplaced (6.432%); 9.1 MiB/s, 1 keys/s, 205 objects/s recovering 2023-09-23T20:30:06.902 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:06 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 35628/150384 objects degraded (23.691%), 19 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:30:07.069 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:06 smithi149 ceph-mon[170394]: pgmap v613: 119 pgs: 1 peering, 4 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 3 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 3.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 622 op/s; 35628/150384 objects degraded (23.691%); 9672/150384 objects misplaced (6.432%); 9.1 MiB/s, 1 keys/s, 205 objects/s recovering 2023-09-23T20:30:07.069 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:06 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 35628/150384 objects degraded (23.691%), 19 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:30:07.070 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:30:07 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[153650]: logger=sqlstore.transactions t=2023-09-23T20:30:07.064853806Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2023-09-23T20:30:08.673 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:30:09.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:08 smithi018 ceph-mon[179647]: pgmap v614: 119 pgs: 4 active+recovery_wait+degraded, 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 3.0 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 662 op/s; 35502/145938 objects degraded (24.327%); 9359/145938 objects misplaced (6.413%); 8.7 MiB/s, 1 keys/s, 189 objects/s recovering 2023-09-23T20:30:09.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:08 smithi018 ceph-mon[184144]: pgmap v614: 119 pgs: 4 active+recovery_wait+degraded, 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 3.0 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 662 op/s; 35502/145938 objects degraded (24.327%); 9359/145938 objects misplaced (6.413%); 8.7 MiB/s, 1 keys/s, 189 objects/s recovering 2023-09-23T20:30:09.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:08 smithi149 ceph-mon[170394]: pgmap v614: 119 pgs: 4 active+recovery_wait+degraded, 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 3.0 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 662 op/s; 35502/145938 objects degraded (24.327%); 9359/145938 objects misplaced (6.413%); 8.7 MiB/s, 1 keys/s, 189 objects/s recovering 2023-09-23T20:30:10.229 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:30:10.235 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:10.236 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:30:10.337 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:30:10.371 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:10.422 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:30:10.426 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:10.426 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:30:10.527 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:30:10.630 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:10.798 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:10 smithi018 ceph-mon[179647]: pgmap v615: 119 pgs: 4 active+recovery_wait+degraded, 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 2.9 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 520 op/s; 35502/143091 objects degraded (24.811%); 8895/143091 objects misplaced (6.216%); 5.0 MiB/s, 79 objects/s recovering 2023-09-23T20:30:10.799 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:10 smithi018 ceph-mon[184144]: pgmap v615: 119 pgs: 4 active+recovery_wait+degraded, 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 2.9 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 520 op/s; 35502/143091 objects degraded (24.811%); 8895/143091 objects misplaced (6.216%); 5.0 MiB/s, 79 objects/s recovering 2023-09-23T20:30:10.904 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:30:10.912 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:30:10.917 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:10.917 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:30:11.018 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:30:11.126 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:11.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:10 smithi149 ceph-mon[170394]: pgmap v615: 119 pgs: 4 active+recovery_wait+degraded, 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 2.9 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 520 op/s; 35502/143091 objects degraded (24.811%); 8895/143091 objects misplaced (6.216%); 5.0 MiB/s, 79 objects/s recovering 2023-09-23T20:30:11.417 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:30:11.427 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:11.427 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:30:11.528 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:30:11.671 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:11.707 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2023-09-23T20:30:11.708 INFO:teuthology.orchestra.run.smithi018.stdout: "ops": [ 2023-09-23T20:30:11.708 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:11.708 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_repop(client.45161.0:165316 6.0 e277/242)", 2023-09-23T20:30:11.708 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:30:11.685414+0000", 2023-09-23T20:30:11.708 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.0020131939999999998, 2023-09-23T20:30:11.708 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0020466009999999999, 2023-09-23T20:30:11.708 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:30:11.708 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "queued for pg", 2023-09-23T20:30:11.708 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:30:11.708 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:11.708 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:30:11.709 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:11.685414+0000", 2023-09-23T20:30:11.709 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:30:11.709 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:11.709 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:11.709 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:30:11.709 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:11.685414+0000", 2023-09-23T20:30:11.709 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:30:11.709 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:11.709 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:11.709 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:30:11.709 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:11.685414+0000", 2023-09-23T20:30:11.709 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:30:11.710 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:11.710 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:11.710 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:30:11.710 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:11.685419+0000", 2023-09-23T20:30:11.710 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.0980000000000001e-06 2023-09-23T20:30:11.710 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:11.710 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:11.710 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:30:11.710 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:11.685424+0000", 2023-09-23T20:30:11.710 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.327e-06 2023-09-23T20:30:11.710 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:11.711 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:11.711 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:30:11.711 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:11.685429+0000", 2023-09-23T20:30:11.711 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.4229999999999998e-06 2023-09-23T20:30:11.711 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:30:11.711 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:30:11.711 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:30:11.711 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:11.711 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:11.711 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_repop(client.45161.0:165313 6.2 e277/259 6:41397d9b:::benchmark_data_smithi018_270558_object60574:head v 277'8143, mlcod=222'2026)", 2023-09-23T20:30:11.711 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:30:11.685452+0000", 2023-09-23T20:30:11.711 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.001975376, 2023-09-23T20:30:11.712 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0020678189999999998, 2023-09-23T20:30:11.712 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:30:11.712 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "started", 2023-09-23T20:30:11.712 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:30:11.712 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:11.712 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:30:11.712 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:11.685452+0000", 2023-09-23T20:30:11.712 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:30:11.712 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:11.712 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:11.712 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:30:11.712 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:11.685452+0000", 2023-09-23T20:30:11.713 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:30:11.713 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:11.713 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:11.713 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:30:11.713 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:11.685452+0000", 2023-09-23T20:30:11.713 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:30:11.713 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:11.713 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:11.713 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:30:11.713 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:11.685457+0000", 2023-09-23T20:30:11.713 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.2850000000000002e-06 2023-09-23T20:30:11.713 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:11.714 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:11.714 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:30:11.714 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:11.685464+0000", 2023-09-23T20:30:11.714 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.6599999999999998e-06 2023-09-23T20:30:11.714 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:11.714 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:11.714 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:30:11.714 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:11.685468+0000", 2023-09-23T20:30:11.714 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.6190000000000002e-06 2023-09-23T20:30:11.714 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:11.714 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:11.714 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:30:11.715 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:11.685583+0000", 2023-09-23T20:30:11.715 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00011444500000000001 2023-09-23T20:30:11.715 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:11.715 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:11.715 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:30:11.715 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:11.685615+0000", 2023-09-23T20:30:11.715 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.2116999999999998e-05 2023-09-23T20:30:11.715 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:30:11.715 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:30:11.715 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:30:11.715 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:11.716 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:11.716 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.45161.0:165294 6.e 6:7b350999:::benchmark_data_smithi018_270558_object60555:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e277)", 2023-09-23T20:30:11.716 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:30:11.635486+0000", 2023-09-23T20:30:11.716 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.051941520999999997, 2023-09-23T20:30:11.716 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.052104167, 2023-09-23T20:30:11.716 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:30:11.716 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:30:11.716 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:30:11.716 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.45161", 2023-09-23T20:30:11.716 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3795614987", 2023-09-23T20:30:11.716 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 165294 2023-09-23T20:30:11.716 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:11.717 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:30:11.717 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:11.717 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:30:11.717 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:11.635486+0000", 2023-09-23T20:30:11.717 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:30:11.717 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:11.717 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:11.717 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:30:11.717 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:11.635486+0000", 2023-09-23T20:30:11.717 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:30:11.717 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:11.717 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:11.718 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:30:11.718 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:11.635484+0000", 2023-09-23T20:30:11.718 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:30:11.718 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:11.718 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:11.718 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:30:11.718 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:11.635495+0000", 2023-09-23T20:30:11.718 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0888e-05 2023-09-23T20:30:11.718 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:11.718 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:11.718 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:30:11.718 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:11.635519+0000", 2023-09-23T20:30:11.719 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.4247999999999999e-05 2023-09-23T20:30:11.719 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:11.719 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:11.719 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:30:11.719 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:11.635529+0000", 2023-09-23T20:30:11.719 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0553e-05 2023-09-23T20:30:11.719 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:11.719 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:11.719 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:30:11.719 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:11.655039+0000", 2023-09-23T20:30:11.719 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.019509347 2023-09-23T20:30:11.719 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:11.720 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:11.720 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:30:11.720 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:11.655213+0000", 2023-09-23T20:30:11.720 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000174661 2023-09-23T20:30:11.720 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:11.720 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:11.720 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,4,5", 2023-09-23T20:30:11.720 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:11.655374+0000", 2023-09-23T20:30:11.720 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000160295 2023-09-23T20:30:11.720 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:11.720 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:11.720 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:30:11.721 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:11.676932+0000", 2023-09-23T20:30:11.721 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.021558399999999998 2023-09-23T20:30:11.721 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:11.721 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:11.721 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:30:11.721 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:11.676967+0000", 2023-09-23T20:30:11.721 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.4413999999999999e-05 2023-09-23T20:30:11.721 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:11.721 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:11.721 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:30:11.721 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:11.677487+0000", 2023-09-23T20:30:11.721 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00052088899999999997 2023-09-23T20:30:11.721 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:30:11.722 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:30:11.722 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:30:11.722 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:11.722 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:11.722 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.45161.0:165299 6.e 6:7f9f4dca:::benchmark_data_smithi018_270558_object60560:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e277)", 2023-09-23T20:30:11.722 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:30:11.649802+0000", 2023-09-23T20:30:11.722 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.037624971, 2023-09-23T20:30:11.722 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.037886114999999998, 2023-09-23T20:30:11.722 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:30:11.722 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:30:11.722 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:30:11.722 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.45161", 2023-09-23T20:30:11.723 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3795614987", 2023-09-23T20:30:11.723 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 165299 2023-09-23T20:30:11.723 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:11.723 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:30:11.723 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:11.723 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:30:11.723 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:11.649802+0000", 2023-09-23T20:30:11.723 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:30:11.723 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:11.723 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:11.723 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:30:11.723 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:11.649802+0000", 2023-09-23T20:30:11.724 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:30:11.724 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:11.724 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:11.724 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:30:11.724 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:11.649800+0000", 2023-09-23T20:30:11.724 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:30:11.724 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:11.724 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:11.724 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:30:11.724 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:11.649810+0000", 2023-09-23T20:30:11.724 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0033e-05 2023-09-23T20:30:11.724 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:11.725 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:11.725 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:30:11.725 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:11.649831+0000", 2023-09-23T20:30:11.725 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.1027999999999999e-05 2023-09-23T20:30:11.725 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:11.725 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:11.725 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:30:11.725 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:11.649842+0000", 2023-09-23T20:30:11.725 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1001e-05 2023-09-23T20:30:11.725 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:11.725 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:11.725 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:30:11.726 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:11.663219+0000", 2023-09-23T20:30:11.726 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.013376618 2023-09-23T20:30:11.726 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:11.726 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:11.726 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:30:11.726 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:11.663380+0000", 2023-09-23T20:30:11.726 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000161273 2023-09-23T20:30:11.726 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:11.726 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:11.726 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,4,5", 2023-09-23T20:30:11.726 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:11.663536+0000", 2023-09-23T20:30:11.726 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000155859 2023-09-23T20:30:11.727 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:11.727 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:11.727 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:30:11.727 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:11.676998+0000", 2023-09-23T20:30:11.727 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.013461673 2023-09-23T20:30:11.727 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:11.727 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:11.727 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:30:11.727 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:11.677035+0000", 2023-09-23T20:30:11.727 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.7381e-05 2023-09-23T20:30:11.727 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:30:11.728 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:30:11.728 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:30:11.728 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:11.728 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:11.728 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_repop(client.45161.0:165301 6.2 e277/259 6:4177015b:::benchmark_data_smithi018_270558_object60562:head v 277'8141, mlcod=222'2026)", 2023-09-23T20:30:11.728 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:30:11.665332+0000", 2023-09-23T20:30:11.728 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.022094935999999999, 2023-09-23T20:30:11.728 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.022445328000000001, 2023-09-23T20:30:11.728 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:30:11.728 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "started", 2023-09-23T20:30:11.728 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:30:11.728 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:11.729 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:30:11.729 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:11.665332+0000", 2023-09-23T20:30:11.729 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:30:11.729 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:11.729 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:11.729 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:30:11.729 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:11.665332+0000", 2023-09-23T20:30:11.729 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:30:11.729 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:11.729 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:11.729 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:30:11.729 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:11.665331+0000", 2023-09-23T20:30:11.730 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.999999 2023-09-23T20:30:11.730 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:11.730 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:11.730 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:30:11.730 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:11.665341+0000", 2023-09-23T20:30:11.730 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.6849999999999996e-06 2023-09-23T20:30:11.730 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:11.730 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:11.730 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:30:11.730 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:11.665355+0000", 2023-09-23T20:30:11.730 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.4433999999999999e-05 2023-09-23T20:30:11.730 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:11.731 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:11.731 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:30:11.731 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:11.665366+0000", 2023-09-23T20:30:11.731 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0385e-05 2023-09-23T20:30:11.731 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:11.731 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:11.731 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:30:11.731 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:11.665420+0000", 2023-09-23T20:30:11.731 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.3740999999999999e-05 2023-09-23T20:30:11.731 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:11.732 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:11.732 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:30:11.732 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:11.665456+0000", 2023-09-23T20:30:11.732 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.6417e-05 2023-09-23T20:30:11.732 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:30:11.732 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:30:11.732 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:30:11.732 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:11.732 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:11.732 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_repop(client.45161.0:165303 6.12 e277/259 6:484f5662:::benchmark_data_smithi018_270558_object60564:head v 277'7985, mlcod=238'4683)", 2023-09-23T20:30:11.732 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:30:11.665387+0000", 2023-09-23T20:30:11.732 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.022039950999999999, 2023-09-23T20:30:11.733 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.022461497, 2023-09-23T20:30:11.733 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:30:11.733 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "started", 2023-09-23T20:30:11.733 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:30:11.733 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:11.733 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:30:11.733 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:11.665387+0000", 2023-09-23T20:30:11.733 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:30:11.733 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:11.733 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:11.733 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:30:11.733 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:11.665387+0000", 2023-09-23T20:30:11.733 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:30:11.734 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:11.734 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:11.734 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:30:11.734 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:11.665387+0000", 2023-09-23T20:30:11.734 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:30:11.734 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:11.734 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:11.734 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:30:11.734 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:11.665392+0000", 2023-09-23T20:30:11.734 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.1780000000000002e-06 2023-09-23T20:30:11.734 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:11.735 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:11.735 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:30:11.735 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:11.665400+0000", 2023-09-23T20:30:11.735 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.2970000000000004e-06 2023-09-23T20:30:11.735 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:11.735 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:11.735 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:30:11.735 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:11.665406+0000", 2023-09-23T20:30:11.735 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.322e-06 2023-09-23T20:30:11.735 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:11.735 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:11.735 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:30:11.736 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:11.665483+0000", 2023-09-23T20:30:11.736 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.6778000000000002e-05 2023-09-23T20:30:11.736 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:11.736 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:11.736 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:30:11.736 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:11.665509+0000", 2023-09-23T20:30:11.736 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.6931e-05 2023-09-23T20:30:11.736 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:30:11.736 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:30:11.736 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:30:11.737 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:11.737 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:11.737 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.45161.0:165306 6.f 6:f2282c62:::benchmark_data_smithi018_270558_object60567:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e277)", 2023-09-23T20:30:11.737 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:30:11.674051+0000", 2023-09-23T20:30:11.737 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.013376486999999999, 2023-09-23T20:30:11.737 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.013862789, 2023-09-23T20:30:11.737 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:30:11.737 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:30:11.737 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:30:11.737 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.45161", 2023-09-23T20:30:11.737 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3795614987", 2023-09-23T20:30:11.737 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 165306 2023-09-23T20:30:11.738 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:11.738 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:30:11.738 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:11.738 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:30:11.738 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:11.674051+0000", 2023-09-23T20:30:11.738 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:30:11.738 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:11.738 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:11.738 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:30:11.738 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:11.674051+0000", 2023-09-23T20:30:11.738 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:30:11.738 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:11.739 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:11.739 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:30:11.739 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:11.674049+0000", 2023-09-23T20:30:11.739 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:30:11.739 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:11.739 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:11.739 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:30:11.739 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:11.674057+0000", 2023-09-23T20:30:11.739 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.1249999999999993e-06 2023-09-23T20:30:11.739 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:11.739 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:11.739 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:30:11.740 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:11.674071+0000", 2023-09-23T20:30:11.740 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.4262e-05 2023-09-23T20:30:11.740 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:11.740 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:11.740 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:30:11.740 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:11.674082+0000", 2023-09-23T20:30:11.740 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0735e-05 2023-09-23T20:30:11.740 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:11.740 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:11.740 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:30:11.740 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:11.674659+0000", 2023-09-23T20:30:11.740 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00057662800000000004 2023-09-23T20:30:11.741 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:11.741 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:11.741 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:30:11.741 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:11.674850+0000", 2023-09-23T20:30:11.741 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000191564 2023-09-23T20:30:11.741 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:11.741 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:11.741 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 3,4,6", 2023-09-23T20:30:11.741 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:11.675007+0000", 2023-09-23T20:30:11.741 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00015680300000000001 2023-09-23T20:30:11.741 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:11.742 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:11.742 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:30:11.742 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:11.682802+0000", 2023-09-23T20:30:11.742 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0077944609999999999 2023-09-23T20:30:11.742 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:11.742 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:11.742 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:30:11.742 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:11.682874+0000", 2023-09-23T20:30:11.742 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.2422000000000006e-05 2023-09-23T20:30:11.742 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:30:11.743 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:30:11.743 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:30:11.743 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:11.743 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:11.743 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.45161.0:165307 6.f 6:fac2292c:::benchmark_data_smithi018_270558_object60568:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e277)", 2023-09-23T20:30:11.743 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:30:11.674599+0000", 2023-09-23T20:30:11.743 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.012828625999999999, 2023-09-23T20:30:11.743 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.013405728, 2023-09-23T20:30:11.743 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:30:11.743 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:30:11.743 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:30:11.743 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.45161", 2023-09-23T20:30:11.744 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3795614987", 2023-09-23T20:30:11.744 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 165307 2023-09-23T20:30:11.744 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:11.744 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:30:11.744 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:11.744 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:30:11.744 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:11.674599+0000", 2023-09-23T20:30:11.744 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:30:11.744 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:11.744 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:11.744 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:30:11.744 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:11.674599+0000", 2023-09-23T20:30:11.744 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:30:11.745 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:11.745 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:11.745 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:30:11.745 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:11.674598+0000", 2023-09-23T20:30:11.745 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:30:11.745 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:11.745 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:11.745 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:30:11.745 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:11.674603+0000", 2023-09-23T20:30:11.745 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.6249999999999998e-06 2023-09-23T20:30:11.745 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:11.745 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:11.745 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:30:11.746 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:11.674615+0000", 2023-09-23T20:30:11.746 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1973999999999999e-05 2023-09-23T20:30:11.746 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:11.746 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:11.746 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:30:11.746 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:11.674622+0000", 2023-09-23T20:30:11.746 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.7360000000000001e-06 2023-09-23T20:30:11.746 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:11.746 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:11.746 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:30:11.746 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:11.678715+0000", 2023-09-23T20:30:11.747 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0040923779999999998 2023-09-23T20:30:11.747 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:11.747 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:11.747 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:30:11.747 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:11.678866+0000", 2023-09-23T20:30:11.747 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00015085599999999999 2023-09-23T20:30:11.747 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:11.747 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:11.747 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 3,4,6", 2023-09-23T20:30:11.747 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:11.679025+0000", 2023-09-23T20:30:11.747 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000158901 2023-09-23T20:30:11.747 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:11.748 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:11.748 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:30:11.748 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:11.683219+0000", 2023-09-23T20:30:11.748 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0041944920000000002 2023-09-23T20:30:11.748 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:30:11.748 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:30:11.748 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:30:11.748 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:11.748 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:11.748 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_repop(client.45161.0:165309 6.2 e277/259 6:4476ad61:::benchmark_data_smithi018_270558_object60570:head v 277'8142, mlcod=222'2026)", 2023-09-23T20:30:11.748 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:30:11.675783+0000", 2023-09-23T20:30:11.748 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.011643908, 2023-09-23T20:30:11.748 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.012304799999999999, 2023-09-23T20:30:11.749 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:30:11.749 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "started", 2023-09-23T20:30:11.749 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:30:11.749 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:11.749 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:30:11.749 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:11.675783+0000", 2023-09-23T20:30:11.749 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:30:11.749 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:11.749 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:11.749 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:30:11.749 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:11.675783+0000", 2023-09-23T20:30:11.749 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:30:11.749 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:11.750 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:11.750 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:30:11.750 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:11.675783+0000", 2023-09-23T20:30:11.750 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.999999 2023-09-23T20:30:11.750 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:11.750 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:11.750 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:30:11.750 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:11.675795+0000", 2023-09-23T20:30:11.750 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.254e-05 2023-09-23T20:30:11.750 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:11.750 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:11.751 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:30:11.751 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:11.675805+0000", 2023-09-23T20:30:11.751 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.9729999999999992e-06 2023-09-23T20:30:11.751 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:11.751 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:11.751 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:30:11.751 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:11.675812+0000", 2023-09-23T20:30:11.751 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.8519999999999998e-06 2023-09-23T20:30:11.751 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:11.751 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:11.751 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:30:11.751 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:11.675971+0000", 2023-09-23T20:30:11.752 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000159364 2023-09-23T20:30:11.752 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:11.752 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:11.752 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:30:11.752 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:11.675997+0000", 2023-09-23T20:30:11.752 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.5324999999999999e-05 2023-09-23T20:30:11.752 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:30:11.752 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:30:11.752 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:30:11.752 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:11.752 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:11.752 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_repop(client.45161.0:165315 6.13 e277/268 6:c9c7e265:::benchmark_data_smithi018_270558_object60576:head v 277'7884, mlcod=229'4253)", 2023-09-23T20:30:11.753 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:30:11.680440+0000", 2023-09-23T20:30:11.753 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.0069868819999999998, 2023-09-23T20:30:11.753 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0077103099999999997, 2023-09-23T20:30:11.753 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:30:11.753 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "started", 2023-09-23T20:30:11.753 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:30:11.753 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:11.753 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:30:11.753 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:11.680440+0000", 2023-09-23T20:30:11.753 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:30:11.753 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:11.753 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:11.754 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:30:11.754 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:11.680440+0000", 2023-09-23T20:30:11.754 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:30:11.754 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:11.754 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:11.754 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:30:11.754 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:11.680440+0000", 2023-09-23T20:30:11.754 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:30:11.754 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:11.754 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:11.754 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:30:11.754 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:11.680444+0000", 2023-09-23T20:30:11.755 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.3730000000000003e-06 2023-09-23T20:30:11.755 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:11.755 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:11.755 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:30:11.755 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:11.680449+0000", 2023-09-23T20:30:11.755 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.9779999999999996e-06 2023-09-23T20:30:11.755 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:11.755 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:11.755 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:30:11.755 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:11.680453+0000", 2023-09-23T20:30:11.755 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.3739999999999998e-06 2023-09-23T20:30:11.755 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:11.756 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:11.756 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:30:11.756 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:11.680493+0000", 2023-09-23T20:30:11.756 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.0114999999999998e-05 2023-09-23T20:30:11.756 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:11.756 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:11.756 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:30:11.756 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:11.680522+0000", 2023-09-23T20:30:11.756 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.8969999999999999e-05 2023-09-23T20:30:11.756 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:30:11.756 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:30:11.756 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:30:11.757 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:11.757 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:11.757 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_repop_reply(client.45161.0:165294 6.e e277/266)", 2023-09-23T20:30:11.757 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:30:11.682921+0000", 2023-09-23T20:30:11.757 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.0045063819999999997, 2023-09-23T20:30:11.757 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0052893009999999997, 2023-09-23T20:30:11.757 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:30:11.757 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "queued for pg", 2023-09-23T20:30:11.757 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:30:11.757 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:11.757 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:30:11.757 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:11.682921+0000", 2023-09-23T20:30:11.758 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:30:11.758 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:11.758 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:11.758 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:30:11.758 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:11.682921+0000", 2023-09-23T20:30:11.758 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:30:11.758 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:11.758 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:11.758 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:30:11.758 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:11.682920+0000", 2023-09-23T20:30:11.758 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.999999 2023-09-23T20:30:11.758 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:11.759 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:11.759 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:30:11.759 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:11.682927+0000", 2023-09-23T20:30:11.759 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.6660000000000008e-06 2023-09-23T20:30:11.759 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:11.759 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:11.759 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:30:11.759 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:11.682943+0000", 2023-09-23T20:30:11.759 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.5831000000000001e-05 2023-09-23T20:30:11.759 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:11.759 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:11.759 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:30:11.759 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:11.682951+0000", 2023-09-23T20:30:11.760 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.9629999999999998e-06 2023-09-23T20:30:11.760 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:30:11.760 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:30:11.760 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:30:11.760 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:11.760 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:11.760 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_repop(client.45161.0:165312 6.0 e277/242)", 2023-09-23T20:30:11.760 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:30:11.685368+0000", 2023-09-23T20:30:11.760 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.0020591110000000002, 2023-09-23T20:30:11.760 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0028890930000000001, 2023-09-23T20:30:11.761 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:30:11.761 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "queued for pg", 2023-09-23T20:30:11.761 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:30:11.761 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:11.761 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:30:11.761 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:11.685368+0000", 2023-09-23T20:30:11.761 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:30:11.761 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:11.761 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:11.761 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:30:11.761 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:11.685368+0000", 2023-09-23T20:30:11.761 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:30:11.762 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:11.762 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:11.762 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:30:11.762 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:11.685367+0000", 2023-09-23T20:30:11.762 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.999999 2023-09-23T20:30:11.762 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:11.762 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:11.762 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:30:11.762 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:11.685376+0000", 2023-09-23T20:30:11.762 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.6109999999999994e-06 2023-09-23T20:30:11.762 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:11.762 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:11.763 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:30:11.763 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:11.685389+0000", 2023-09-23T20:30:11.763 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3481e-05 2023-09-23T20:30:11.763 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:11.763 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:11.763 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:30:11.763 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:11.685398+0000", 2023-09-23T20:30:11.763 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.7520000000000002e-06 2023-09-23T20:30:11.763 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:30:11.763 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:30:11.763 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:30:11.763 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:30:11.764 INFO:teuthology.orchestra.run.smithi018.stdout: ], 2023-09-23T20:30:11.764 INFO:teuthology.orchestra.run.smithi018.stdout: "num_ops": 12 2023-09-23T20:30:11.764 INFO:teuthology.orchestra.run.smithi018.stdout:} 2023-09-23T20:30:12.088 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:11 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 35502/143091 objects degraded (24.811%), 19 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:30:12.089 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:11 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 35502/143091 objects degraded (24.811%), 19 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:30:12.121 INFO:teuthology.orchestra.run.smithi018.stderr:noscrub is unset 2023-09-23T20:30:12.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:11 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 35502/143091 objects degraded (24.811%), 19 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:30:12.309 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T20:30:12.687 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:30:12.696 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:12.696 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:30:12.797 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:30:12.837 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:12.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:12 smithi018 ceph-mon[184144]: pgmap v616: 119 pgs: 4 active+recovery_wait+degraded, 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 2.8 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 628 op/s; 35502/137025 objects degraded (25.909%); 8895/137025 objects misplaced (6.492%); 4.1 MiB/s, 65 objects/s recovering 2023-09-23T20:30:12.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:12 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3620392071' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:30:12.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:12 smithi018 ceph-mon[184144]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:30:12.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:12 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3620392071' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:30:12.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:12 smithi018 ceph-mon[184144]: osdmap e278: 8 total, 8 up, 7 in 2023-09-23T20:30:12.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:12 smithi018 ceph-mon[184144]: failed to encode map e278 with expected crc 2023-09-23T20:30:12.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:12 smithi018 ceph-mon[184144]: failed to encode map e278 with expected crc 2023-09-23T20:30:12.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:12 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1603127116' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T20:30:12.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:12 smithi018 ceph-mon[179647]: pgmap v616: 119 pgs: 4 active+recovery_wait+degraded, 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 2.8 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 628 op/s; 35502/137025 objects degraded (25.909%); 8895/137025 objects misplaced (6.492%); 4.1 MiB/s, 65 objects/s recovering 2023-09-23T20:30:12.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:12 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3620392071' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:30:12.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:12 smithi018 ceph-mon[179647]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:30:12.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:12 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3620392071' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:30:12.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:12 smithi018 ceph-mon[179647]: osdmap e278: 8 total, 8 up, 7 in 2023-09-23T20:30:12.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:12 smithi018 ceph-mon[179647]: failed to encode map e278 with expected crc 2023-09-23T20:30:12.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:12 smithi018 ceph-mon[179647]: failed to encode map e278 with expected crc 2023-09-23T20:30:12.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:12 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1603127116' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T20:30:12.897 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:30:12.900 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:12.901 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:30:13.002 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:30:13.033 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:13.086 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:30:13.089 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:13.089 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:30:13.091 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:12 smithi149 ceph-mon[170394]: pgmap v616: 119 pgs: 4 active+recovery_wait+degraded, 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 2.8 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 628 op/s; 35502/137025 objects degraded (25.909%); 8895/137025 objects misplaced (6.492%); 4.1 MiB/s, 65 objects/s recovering 2023-09-23T20:30:13.091 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:12 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3620392071' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:30:13.091 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:12 smithi149 ceph-mon[170394]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:30:13.091 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:12 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3620392071' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:30:13.091 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:12 smithi149 ceph-mon[170394]: osdmap e278: 8 total, 8 up, 7 in 2023-09-23T20:30:13.091 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:12 smithi149 ceph-mon[170394]: failed to encode map e278 with expected crc 2023-09-23T20:30:13.091 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:12 smithi149 ceph-mon[170394]: failed to encode map e278 with expected crc 2023-09-23T20:30:13.091 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:12 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1603127116' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T20:30:13.130 INFO:teuthology.orchestra.run.smithi018.stderr:reweighted osd.5 to 1 (10000) 2023-09-23T20:30:13.190 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:30:13.229 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:13.273 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:30:13.280 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:13.280 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:30:13.381 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:30:13.426 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:13.479 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:30:13.482 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:13.483 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:30:13.584 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:30:13.697 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd reweight 7 1 2023-09-23T20:30:13.733 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:30:13.733 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:13.850 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:30:13.872 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:13.872 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:30:13.973 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:30:14.008 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:14.054 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:30:14.058 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:14.059 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:30:14.121 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:30:14.134 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:13 smithi018 ceph-mon[179647]: failed to encode map e278 with expected crc 2023-09-23T20:30:14.134 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:13 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1603127116' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T20:30:14.134 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:13 smithi018 ceph-mon[179647]: osdmap e279: 8 total, 8 up, 7 in 2023-09-23T20:30:14.134 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:13 smithi018 ceph-mon[179647]: failed to encode map e279 with expected crc 2023-09-23T20:30:14.134 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:13 smithi018 ceph-mon[179647]: failed to encode map e279 with expected crc 2023-09-23T20:30:14.134 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:13 smithi018 ceph-mon[179647]: failed to encode map e279 with expected crc 2023-09-23T20:30:14.136 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:13 smithi018 ceph-mon[184144]: failed to encode map e278 with expected crc 2023-09-23T20:30:14.136 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:13 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1603127116' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T20:30:14.136 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:13 smithi018 ceph-mon[184144]: osdmap e279: 8 total, 8 up, 7 in 2023-09-23T20:30:14.136 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:13 smithi018 ceph-mon[184144]: failed to encode map e279 with expected crc 2023-09-23T20:30:14.136 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:13 smithi018 ceph-mon[184144]: failed to encode map e279 with expected crc 2023-09-23T20:30:14.137 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:13 smithi018 ceph-mon[184144]: failed to encode map e279 with expected crc 2023-09-23T20:30:14.160 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:30:14.167 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:30:14.191 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:13 smithi149 ceph-mon[170394]: failed to encode map e278 with expected crc 2023-09-23T20:30:14.191 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:13 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1603127116' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T20:30:14.191 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:13 smithi149 ceph-mon[170394]: osdmap e279: 8 total, 8 up, 7 in 2023-09-23T20:30:14.191 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:13 smithi149 ceph-mon[170394]: failed to encode map e279 with expected crc 2023-09-23T20:30:14.191 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:13 smithi149 ceph-mon[170394]: failed to encode map e279 with expected crc 2023-09-23T20:30:14.191 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:13 smithi149 ceph-mon[170394]: failed to encode map e279 with expected crc 2023-09-23T20:30:14.194 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:14.221 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:30:14.238 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:30:14.241 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:14.241 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:30:14.286 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:30:14.341 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:30:14.342 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:30:14.381 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:30:14.382 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:14.427 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:30:14.430 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:14.431 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:30:14.446 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:30:14.478 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:30:14.494 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:30:14.494 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:30:14.494 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:30:14.494 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:30:14.494 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:30:14.494 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:30:14.494 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:30:14.494 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:30:14.532 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:30:14.617 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:14.770 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:30:14.775 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:14.776 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:30:14.779 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T20:30:14.877 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:30:14.908 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:14.941 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:30:14.946 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:14.946 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:30:15.047 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:30:15.076 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:15.130 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:30:15.133 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:14 smithi149 ceph-mon[170394]: failed to encode map e279 with expected crc 2023-09-23T20:30:15.133 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:14 smithi149 ceph-mon[170394]: failed to encode map e279 with expected crc 2023-09-23T20:30:15.133 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:14 smithi149 ceph-mon[170394]: pgmap v619: 119 pgs: 4 active+recovery_wait+degraded, 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 2.7 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 736 op/s; 35369/132684 objects degraded (26.657%); 8066/132684 objects misplaced (6.079%); 6.2 MiB/s, 98 objects/s recovering 2023-09-23T20:30:15.133 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:14 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2221155048' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:30:15.134 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:14 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:30:15.134 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:15.135 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:30:15.146 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:14 smithi018 ceph-mon[179647]: failed to encode map e279 with expected crc 2023-09-23T20:30:15.146 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:14 smithi018 ceph-mon[179647]: failed to encode map e279 with expected crc 2023-09-23T20:30:15.146 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:14 smithi018 ceph-mon[179647]: pgmap v619: 119 pgs: 4 active+recovery_wait+degraded, 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 2.7 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 736 op/s; 35369/132684 objects degraded (26.657%); 8066/132684 objects misplaced (6.079%); 6.2 MiB/s, 98 objects/s recovering 2023-09-23T20:30:15.147 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:14 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2221155048' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:30:15.147 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:14 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:30:15.147 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:14 smithi018 ceph-mon[184144]: failed to encode map e279 with expected crc 2023-09-23T20:30:15.147 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:14 smithi018 ceph-mon[184144]: failed to encode map e279 with expected crc 2023-09-23T20:30:15.147 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:14 smithi018 ceph-mon[184144]: pgmap v619: 119 pgs: 4 active+recovery_wait+degraded, 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 2.7 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 736 op/s; 35369/132684 objects degraded (26.657%); 8066/132684 objects misplaced (6.079%); 6.2 MiB/s, 98 objects/s recovering 2023-09-23T20:30:15.147 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:14 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2221155048' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:30:15.147 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:14 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:30:15.148 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:30:14 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:30:14] "GET /metrics HTTP/1.1" 200 36570 "" "Prometheus/2.43.0" 2023-09-23T20:30:15.235 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:30:15.325 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:17.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:16 smithi018 ceph-mon[179647]: pgmap v620: 119 pgs: 4 active+recovery_wait+degraded, 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 2.7 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 626 op/s; 35369/130893 objects degraded (27.021%); 8066/130893 objects misplaced (6.162%); 5.2 MiB/s, 83 objects/s recovering 2023-09-23T20:30:17.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:16 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 35369/130893 objects degraded (27.021%), 19 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:30:17.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:16 smithi018 ceph-mon[184144]: pgmap v620: 119 pgs: 4 active+recovery_wait+degraded, 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 2.7 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 626 op/s; 35369/130893 objects degraded (27.021%); 8066/130893 objects misplaced (6.162%); 5.2 MiB/s, 83 objects/s recovering 2023-09-23T20:30:17.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:16 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 35369/130893 objects degraded (27.021%), 19 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:30:17.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:16 smithi149 ceph-mon[170394]: pgmap v620: 119 pgs: 4 active+recovery_wait+degraded, 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 2.7 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 626 op/s; 35369/130893 objects degraded (27.021%); 8066/130893 objects misplaced (6.162%); 5.2 MiB/s, 83 objects/s recovering 2023-09-23T20:30:17.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:16 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 35369/130893 objects degraded (27.021%), 19 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:30:18.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:17 smithi018 ceph-mon[184144]: 1.11 scrub starts 2023-09-23T20:30:18.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:17 smithi018 ceph-mon[179647]: 1.11 scrub starts 2023-09-23T20:30:18.219 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:17 smithi149 ceph-mon[170394]: 1.11 scrub starts 2023-09-23T20:30:19.028 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:30:19.035 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:19.036 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:30:19.137 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:30:19.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:18 smithi018 ceph-mon[184144]: pgmap v621: 119 pgs: 4 active+recovery_wait+degraded, 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 2.6 GiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 703 op/s; 35216/126186 objects degraded (27.908%); 7702/126186 objects misplaced (6.104%); 4.4 MiB/s, 70 objects/s recovering 2023-09-23T20:30:19.144 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:18 smithi018 ceph-mon[179647]: pgmap v621: 119 pgs: 4 active+recovery_wait+degraded, 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 2.6 GiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 703 op/s; 35216/126186 objects degraded (27.908%); 7702/126186 objects misplaced (6.104%); 4.4 MiB/s, 70 objects/s recovering 2023-09-23T20:30:19.177 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:18 smithi149 ceph-mon[170394]: pgmap v621: 119 pgs: 4 active+recovery_wait+degraded, 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 2.6 GiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 703 op/s; 35216/126186 objects degraded (27.908%); 7702/126186 objects misplaced (6.104%); 4.4 MiB/s, 70 objects/s recovering 2023-09-23T20:30:19.181 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:19.231 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:30:19.235 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:19.236 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:30:19.336 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:30:19.433 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:19.888 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:30:19.898 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:19.899 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:30:19.994 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:30:20.000 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:30:20.032 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:20.075 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:30:20.081 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:20.081 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:30:20.182 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:30:20.223 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:20.273 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:30:20.278 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:20.279 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:30:20.379 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:30:20.471 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:21.038 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:30:21.043 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:21.043 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:30:21.046 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:30:21.144 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:30:21.209 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:20 smithi018 ceph-mon[184144]: pgmap v622: 119 pgs: 4 active+recovery_wait+degraded, 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 2.5 GiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 581 op/s; 35216/123051 objects degraded (28.619%); 7207/123051 objects misplaced (5.857%); 6.7 MiB/s, 106 objects/s recovering 2023-09-23T20:30:21.210 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:20 smithi018 ceph-mon[179647]: pgmap v622: 119 pgs: 4 active+recovery_wait+degraded, 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 2.5 GiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 581 op/s; 35216/123051 objects degraded (28.619%); 7207/123051 objects misplaced (5.857%); 6.7 MiB/s, 106 objects/s recovering 2023-09-23T20:30:21.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:20 smithi149 ceph-mon[170394]: pgmap v622: 119 pgs: 4 active+recovery_wait+degraded, 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 2.5 GiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 581 op/s; 35216/123051 objects degraded (28.619%); 7207/123051 objects misplaced (5.857%); 6.7 MiB/s, 106 objects/s recovering 2023-09-23T20:30:21.217 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:21.669 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:30:21.946 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:30:21.957 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:21.957 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:30:22.026 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:21 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 35216/123051 objects degraded (28.619%), 19 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:30:22.026 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:21 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 35216/123051 objects degraded (28.619%), 19 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:30:22.059 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:30:22.093 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:22.142 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:30:22.146 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:22.146 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:30:22.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:21 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 35216/123051 objects degraded (28.619%), 19 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:30:22.248 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:30:22.293 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:22.345 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:30:22.351 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:22.352 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:30:22.452 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:30:22.537 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:22.575 INFO:teuthology.orchestra.run.smithi018.stdout:ERROR: (22) Invalid argument 2023-09-23T20:30:22.575 INFO:teuthology.orchestra.run.smithi018.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:30:22.680 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:30:22.708 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:22.708 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:30:22.809 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:30:22.841 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:22.889 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:30:22.895 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:22.895 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:30:22.996 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:30:23.065 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:23 smithi018 ceph-mon[179647]: pgmap v623: 119 pgs: 1 active+clean+scrubbing, 4 active+recovery_wait+degraded, 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 2.4 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 691 op/s; 35216/117096 objects degraded (30.074%); 7207/117096 objects misplaced (6.155%); 5.6 MiB/s, 88 objects/s recovering 2023-09-23T20:30:23.088 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:23.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:23 smithi149 ceph-mon[170394]: pgmap v623: 119 pgs: 1 active+clean+scrubbing, 4 active+recovery_wait+degraded, 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 2.4 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 691 op/s; 35216/117096 objects degraded (30.074%); 7207/117096 objects misplaced (6.155%); 5.6 MiB/s, 88 objects/s recovering 2023-09-23T20:30:23.255 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:30:23.267 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:23.267 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:30:23.359 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:23 smithi018 ceph-mon[184144]: pgmap v623: 119 pgs: 1 active+clean+scrubbing, 4 active+recovery_wait+degraded, 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 2.4 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 691 op/s; 35216/117096 objects degraded (30.074%); 7207/117096 objects misplaced (6.155%); 5.6 MiB/s, 88 objects/s recovering 2023-09-23T20:30:23.360 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T20:30:23.368 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:30:23.411 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:23.471 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:30:23.474 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:23.475 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:30:23.575 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:30:23.726 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:24.000 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:30:24.021 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:24.021 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:30:24.098 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:30:24.122 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:30:24.233 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:24.290 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:24 smithi018 ceph-mon[179647]: 6.9 scrub starts 2023-09-23T20:30:24.291 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:24 smithi018 ceph-mon[184144]: 6.9 scrub starts 2023-09-23T20:30:24.359 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:30:24.371 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:24.371 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:30:24.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:24 smithi149 ceph-mon[170394]: 6.9 scrub starts 2023-09-23T20:30:24.471 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:30:24.624 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:24.647 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:30:24.733 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:30:24.762 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:24.763 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:30:24.864 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:30:24.898 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:24.939 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:30:24.947 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:24.948 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:30:25.048 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:30:25.086 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:25.189 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:30:25.197 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:25.198 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:25 smithi149 ceph-mon[170394]: pgmap v624: 119 pgs: 1 active+clean+scrubbing, 4 active+recovery_wait+degraded, 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 2.3 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 728 op/s; 35072/112542 objects degraded (31.163%); 6281/112542 objects misplaced (5.581%); 7.0 MiB/s, 112 objects/s recovering 2023-09-23T20:30:25.198 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:25 smithi149 ceph-mon[170394]: 6.0 scrub starts 2023-09-23T20:30:25.198 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:25 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:30:25.198 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:30:25.277 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:30:24 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:30:24] "GET /metrics HTTP/1.1" 200 36570 "" "Prometheus/2.43.0" 2023-09-23T20:30:25.281 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:25 smithi018 ceph-mon[184144]: pgmap v624: 119 pgs: 1 active+clean+scrubbing, 4 active+recovery_wait+degraded, 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 2.3 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 728 op/s; 35072/112542 objects degraded (31.163%); 6281/112542 objects misplaced (5.581%); 7.0 MiB/s, 112 objects/s recovering 2023-09-23T20:30:25.281 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:25 smithi018 ceph-mon[184144]: 6.0 scrub starts 2023-09-23T20:30:25.281 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:25 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:30:25.282 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:25 smithi018 ceph-mon[179647]: pgmap v624: 119 pgs: 1 active+clean+scrubbing, 4 active+recovery_wait+degraded, 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 2.3 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 728 op/s; 35072/112542 objects degraded (31.163%); 6281/112542 objects misplaced (5.581%); 7.0 MiB/s, 112 objects/s recovering 2023-09-23T20:30:25.282 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:25 smithi018 ceph-mon[179647]: 6.0 scrub starts 2023-09-23T20:30:25.282 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:25 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:30:25.299 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:30:25.446 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:25.691 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:30:25.714 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:25.715 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:30:25.816 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:30:25.889 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:26.131 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:30:26.141 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:26.142 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:30:26.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:26 smithi018 ceph-mon[179647]: 6.10 scrub starts 2023-09-23T20:30:26.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:26 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/400676728' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:30:26.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:26 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:30:26.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:26 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1949513284' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T20:30:26.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:26 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T20:30:26.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:26 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 35072/112542 objects degraded (31.163%), 19 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:30:26.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:26 smithi018 ceph-mon[184144]: 6.10 scrub starts 2023-09-23T20:30:26.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:26 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/400676728' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:30:26.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:26 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:30:26.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:26 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1949513284' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T20:30:26.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:26 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T20:30:26.144 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:26 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 35072/112542 objects degraded (31.163%), 19 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:30:26.147 INFO:teuthology.orchestra.run.smithi018.stderr:reweighted osd.7 to 1 (10000) 2023-09-23T20:30:26.162 INFO:teuthology.orchestra.run.smithi018.stderr:nodeep-scrub is unset 2023-09-23T20:30:26.243 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:30:26.295 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:26.460 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:30:26.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:26 smithi149 ceph-mon[170394]: 6.10 scrub starts 2023-09-23T20:30:26.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:26 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/400676728' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:30:26.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:26 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:30:26.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:26 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1949513284' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T20:30:26.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:26 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T20:30:26.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:26 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 35072/112542 objects degraded (31.163%), 19 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:30:26.502 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:30:26.552 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:30:26.610 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:30:26.652 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:30:26.710 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:30:26.727 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:30:26.782 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:30:26.792 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:30:26.799 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:26.799 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:30:26.815 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:30:26.815 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:30:26.815 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:30:26.815 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:30:26.815 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:30:26.815 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:30:26.816 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:30:26.816 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:30:26.900 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:30:26.933 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:26.954 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd reweight 3 1 2023-09-23T20:30:26.973 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:30:26.977 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:26.977 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:30:27.078 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:30:27.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:27 smithi018 ceph-mon[184144]: pgmap v625: 119 pgs: 4 active+recovery_wait+degraded, 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 2.3 GiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 609 op/s; 35072/110733 objects degraded (31.673%); 6281/110733 objects misplaced (5.672%); 4.7 MiB/s, 75 objects/s recovering 2023-09-23T20:30:27.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:27 smithi018 ceph-mon[184144]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:30:27.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:27 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:30:27.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:27 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T20:30:27.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:27 smithi018 ceph-mon[184144]: osdmap e280: 8 total, 8 up, 7 in 2023-09-23T20:30:27.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:27 smithi018 ceph-mon[184144]: failed to encode map e280 with expected crc 2023-09-23T20:30:27.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:27 smithi018 ceph-mon[184144]: failed to encode map e280 with expected crc 2023-09-23T20:30:27.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:27 smithi018 ceph-mon[184144]: failed to encode map e280 with expected crc 2023-09-23T20:30:27.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:27 smithi018 ceph-mon[184144]: failed to encode map e280 with expected crc 2023-09-23T20:30:27.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:27 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3473143920' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:30:27.278 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:27.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:27 smithi018 ceph-mon[179647]: pgmap v625: 119 pgs: 4 active+recovery_wait+degraded, 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 2.3 GiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 609 op/s; 35072/110733 objects degraded (31.673%); 6281/110733 objects misplaced (5.672%); 4.7 MiB/s, 75 objects/s recovering 2023-09-23T20:30:27.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:27 smithi018 ceph-mon[179647]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:30:27.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:27 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:30:27.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:27 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T20:30:27.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:27 smithi018 ceph-mon[179647]: osdmap e280: 8 total, 8 up, 7 in 2023-09-23T20:30:27.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:27 smithi018 ceph-mon[179647]: failed to encode map e280 with expected crc 2023-09-23T20:30:27.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:27 smithi018 ceph-mon[179647]: failed to encode map e280 with expected crc 2023-09-23T20:30:27.394 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:27 smithi018 ceph-mon[179647]: failed to encode map e280 with expected crc 2023-09-23T20:30:27.394 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:27 smithi018 ceph-mon[179647]: failed to encode map e280 with expected crc 2023-09-23T20:30:27.394 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:27 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3473143920' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:30:27.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:27 smithi149 ceph-mon[170394]: pgmap v625: 119 pgs: 4 active+recovery_wait+degraded, 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 2.3 GiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 609 op/s; 35072/110733 objects degraded (31.673%); 6281/110733 objects misplaced (5.672%); 4.7 MiB/s, 75 objects/s recovering 2023-09-23T20:30:27.475 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:27 smithi149 ceph-mon[170394]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:30:27.475 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:27 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:30:27.476 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:27 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T20:30:27.476 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:27 smithi149 ceph-mon[170394]: osdmap e280: 8 total, 8 up, 7 in 2023-09-23T20:30:27.476 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:27 smithi149 ceph-mon[170394]: failed to encode map e280 with expected crc 2023-09-23T20:30:27.476 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:27 smithi149 ceph-mon[170394]: failed to encode map e280 with expected crc 2023-09-23T20:30:27.476 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:27 smithi149 ceph-mon[170394]: failed to encode map e280 with expected crc 2023-09-23T20:30:27.476 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:27 smithi149 ceph-mon[170394]: failed to encode map e280 with expected crc 2023-09-23T20:30:27.476 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:27 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3473143920' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:30:27.847 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:30:27.855 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:27.855 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:30:27.957 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:30:28.008 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:28.068 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:30:28.075 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:28.075 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:30:28.176 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:30:28.228 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:28.297 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:30:28.306 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:28.306 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:30:28.407 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:30:28.484 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2023-09-23T20:30:28.484 INFO:teuthology.orchestra.run.smithi018.stdout: "size": 20, 2023-09-23T20:30:28.484 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 600, 2023-09-23T20:30:28.484 INFO:teuthology.orchestra.run.smithi018.stdout: "ops": [ 2023-09-23T20:30:28.484 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.484 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.34791.0:28891 5.8 5:116f3062:::benchmark_data_smithi018_228451_object28890:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e118)", 2023-09-23T20:30:28.485 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:23:09.968465+0000", 2023-09-23T20:30:28.485 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 437.39049598399998, 2023-09-23T20:30:28.485 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1002673000000001, 2023-09-23T20:30:28.485 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:30:28.485 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:30:28.485 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:30:28.485 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.34791", 2023-09-23T20:30:28.485 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:30:28.485 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 28891 2023-09-23T20:30:28.485 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.485 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:30:28.485 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.486 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:30:28.486 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.968465+0000", 2023-09-23T20:30:28.486 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:30:28.486 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.486 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.486 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:30:28.486 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.968465+0000", 2023-09-23T20:30:28.486 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:30:28.486 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.486 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.486 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:30:28.487 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.968464+0000", 2023-09-23T20:30:28.487 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:30:28.487 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.487 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.487 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:30:28.487 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.968506+0000", 2023-09-23T20:30:28.487 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.2608999999999998e-05 2023-09-23T20:30:28.487 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.487 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.487 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:30:28.487 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.968529+0000", 2023-09-23T20:30:28.488 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.2875999999999999e-05 2023-09-23T20:30:28.488 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.488 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.488 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:30:28.488 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.968543+0000", 2023-09-23T20:30:28.488 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.4151000000000001e-05 2023-09-23T20:30:28.488 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.488 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.488 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:30:28.488 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.968601+0000", 2023-09-23T20:30:28.488 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.8199000000000003e-05 2023-09-23T20:30:28.488 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.489 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.489 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op must wait for map", 2023-09-23T20:30:28.489 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.968609+0000", 2023-09-23T20:30:28.489 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.1509999999999998e-06 2023-09-23T20:30:28.489 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.489 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.489 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:30:28.489 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:10.024810+0000", 2023-09-23T20:30:28.489 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.056201444000000003 2023-09-23T20:30:28.489 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.489 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.490 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:30:28.490 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:10.024819+0000", 2023-09-23T20:30:28.490 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.2210000000000003e-06 2023-09-23T20:30:28.490 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.490 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.490 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:30:28.490 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.068626+0000", 2023-09-23T20:30:28.490 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0438065649999999 2023-09-23T20:30:28.491 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.491 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.491 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:30:28.491 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.068733+0000", 2023-09-23T20:30:28.491 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000106909 2023-09-23T20:30:28.491 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:30:28.491 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:30:28.491 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:30:28.491 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.491 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.491 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.34791.0:28895 5.8 5:1248f566:::benchmark_data_smithi018_228451_object28894:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e118)", 2023-09-23T20:30:28.492 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:23:09.968563+0000", 2023-09-23T20:30:28.492 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 437.39039889100002, 2023-09-23T20:30:28.492 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.106749932, 2023-09-23T20:30:28.492 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:30:28.492 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:30:28.492 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:30:28.492 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.34791", 2023-09-23T20:30:28.492 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:30:28.492 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 28895 2023-09-23T20:30:28.492 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.492 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:30:28.492 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.493 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:30:28.493 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.968563+0000", 2023-09-23T20:30:28.493 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:30:28.493 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.493 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.493 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:30:28.493 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.968563+0000", 2023-09-23T20:30:28.493 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:30:28.493 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.493 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.493 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:30:28.494 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.968562+0000", 2023-09-23T20:30:28.494 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.999999 2023-09-23T20:30:28.494 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.494 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.494 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:30:28.494 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.968606+0000", 2023-09-23T20:30:28.494 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.4425999999999998e-05 2023-09-23T20:30:28.494 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.494 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.494 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:30:28.495 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.968616+0000", 2023-09-23T20:30:28.495 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.8810000000000008e-06 2023-09-23T20:30:28.495 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.495 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.495 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:30:28.495 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.968623+0000", 2023-09-23T20:30:28.495 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.652e-06 2023-09-23T20:30:28.495 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.495 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.495 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:30:28.495 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.968652+0000", 2023-09-23T20:30:28.495 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.9105e-05 2023-09-23T20:30:28.496 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.496 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.496 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting_for_map not empty", 2023-09-23T20:30:28.496 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.968656+0000", 2023-09-23T20:30:28.496 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.4340000000000002e-06 2023-09-23T20:30:28.496 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.496 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.496 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:30:28.496 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:10.024864+0000", 2023-09-23T20:30:28.496 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.056207319999999998 2023-09-23T20:30:28.496 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.496 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.497 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:30:28.497 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:10.024873+0000", 2023-09-23T20:30:28.497 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.0820000000000005e-06 2023-09-23T20:30:28.497 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.497 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.497 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:30:28.497 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.068764+0000", 2023-09-23T20:30:28.497 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.043891471 2023-09-23T20:30:28.497 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.497 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.497 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for degraded object", 2023-09-23T20:30:28.497 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.069685+0000", 2023-09-23T20:30:28.498 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00092115299999999997 2023-09-23T20:30:28.498 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.498 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.498 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:30:28.498 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.075213+0000", 2023-09-23T20:30:28.498 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0055280440000000002 2023-09-23T20:30:28.498 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.498 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.498 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:30:28.498 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.075312+0000", 2023-09-23T20:30:28.498 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.9123000000000006e-05 2023-09-23T20:30:28.499 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:30:28.499 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:30:28.499 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:30:28.499 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.499 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.499 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.34791.0:28897 5.18 5:1dddaaad:::benchmark_data_smithi018_228451_object28896:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e118)", 2023-09-23T20:30:28.499 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:23:09.968628+0000", 2023-09-23T20:30:28.499 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 437.39033326100002, 2023-09-23T20:30:28.499 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1136079919999999, 2023-09-23T20:30:28.499 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:30:28.499 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:30:28.499 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:30:28.500 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.34791", 2023-09-23T20:30:28.500 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:30:28.500 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 28897 2023-09-23T20:30:28.500 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.500 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:30:28.500 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.500 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:30:28.500 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.968628+0000", 2023-09-23T20:30:28.500 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:30:28.501 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.501 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.501 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:30:28.501 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.968628+0000", 2023-09-23T20:30:28.501 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:30:28.501 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.501 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.501 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:30:28.501 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.968628+0000", 2023-09-23T20:30:28.501 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:30:28.501 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.502 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.502 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:30:28.502 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.968656+0000", 2023-09-23T20:30:28.502 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.7968999999999999e-05 2023-09-23T20:30:28.502 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.502 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.502 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:30:28.502 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.968664+0000", 2023-09-23T20:30:28.502 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.4509999999999992e-06 2023-09-23T20:30:28.502 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.502 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.502 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:30:28.503 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.968670+0000", 2023-09-23T20:30:28.503 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.5589999999999996e-06 2023-09-23T20:30:28.503 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.503 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.503 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:30:28.503 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:10.024730+0000", 2023-09-23T20:30:28.503 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.056060655000000001 2023-09-23T20:30:28.503 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.503 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.503 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:30:28.503 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:10.024738+0000", 2023-09-23T20:30:28.504 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.1130000000000001e-06 2023-09-23T20:30:28.504 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.504 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.504 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:30:28.504 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.078174+0000", 2023-09-23T20:30:28.504 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.053435809 2023-09-23T20:30:28.504 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.504 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.504 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for degraded object", 2023-09-23T20:30:28.504 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.078877+0000", 2023-09-23T20:30:28.504 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000702913 2023-09-23T20:30:28.505 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.505 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.505 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:30:28.505 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.082148+0000", 2023-09-23T20:30:28.505 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0032704880000000002 2023-09-23T20:30:28.505 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.505 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.505 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:30:28.505 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.082236+0000", 2023-09-23T20:30:28.505 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.8488999999999999e-05 2023-09-23T20:30:28.505 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:30:28.505 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:30:28.506 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:30:28.506 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.506 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.506 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.34791.0:28898 5.16 5:6da0b3ba:::benchmark_data_smithi018_228451_object28897:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e118)", 2023-09-23T20:30:28.506 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:23:09.969024+0000", 2023-09-23T20:30:28.506 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 437.38993788800002, 2023-09-23T20:30:28.506 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.102100836, 2023-09-23T20:30:28.506 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:30:28.506 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:30:28.506 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:30:28.506 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.34791", 2023-09-23T20:30:28.506 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:30:28.507 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 28898 2023-09-23T20:30:28.507 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.507 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:30:28.507 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.507 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:30:28.507 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.969024+0000", 2023-09-23T20:30:28.507 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:30:28.507 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.507 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.507 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:30:28.507 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.969024+0000", 2023-09-23T20:30:28.508 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:30:28.508 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.508 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.508 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:30:28.508 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.969023+0000", 2023-09-23T20:30:28.508 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:30:28.508 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.508 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.508 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:30:28.508 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.969054+0000", 2023-09-23T20:30:28.508 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.1010000000000003e-05 2023-09-23T20:30:28.508 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.509 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.509 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:30:28.509 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.969066+0000", 2023-09-23T20:30:28.509 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1771e-05 2023-09-23T20:30:28.509 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.509 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.509 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:30:28.509 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.969075+0000", 2023-09-23T20:30:28.509 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.5040000000000008e-06 2023-09-23T20:30:28.509 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.509 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.509 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:30:28.510 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:10.010679+0000", 2023-09-23T20:30:28.510 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.041603581000000001 2023-09-23T20:30:28.510 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.510 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.510 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:30:28.510 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:10.010687+0000", 2023-09-23T20:30:28.510 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.6880000000000005e-06 2023-09-23T20:30:28.510 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.510 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.510 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:30:28.510 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.071051+0000", 2023-09-23T20:30:28.510 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.060363755 2023-09-23T20:30:28.511 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.511 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.511 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:30:28.511 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.071124+0000", 2023-09-23T20:30:28.511 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.3145000000000005e-05 2023-09-23T20:30:28.511 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:30:28.511 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:30:28.511 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:30:28.512 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.512 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.512 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.34791.0:28899 5.8 5:162cc4a5:::benchmark_data_smithi018_228451_object28898:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e118)", 2023-09-23T20:30:28.512 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:23:09.969091+0000", 2023-09-23T20:30:28.512 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 437.38986997799998, 2023-09-23T20:30:28.512 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.107737296, 2023-09-23T20:30:28.512 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:30:28.512 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:30:28.512 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:30:28.512 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.34791", 2023-09-23T20:30:28.512 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:30:28.512 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 28899 2023-09-23T20:30:28.513 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.513 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:30:28.513 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.513 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:30:28.513 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.969091+0000", 2023-09-23T20:30:28.513 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:30:28.513 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.513 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.513 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:30:28.513 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.969091+0000", 2023-09-23T20:30:28.513 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:30:28.513 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.514 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.514 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:30:28.514 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.969091+0000", 2023-09-23T20:30:28.514 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:30:28.514 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.514 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.514 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:30:28.514 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.969122+0000", 2023-09-23T20:30:28.514 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.1510000000000002e-05 2023-09-23T20:30:28.515 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.515 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.515 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:30:28.515 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.969130+0000", 2023-09-23T20:30:28.515 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.1980000000000001e-06 2023-09-23T20:30:28.515 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.515 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.515 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:30:28.515 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.969137+0000", 2023-09-23T20:30:28.515 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.9120000000000001e-06 2023-09-23T20:30:28.515 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.516 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.516 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:30:28.516 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.969293+0000", 2023-09-23T20:30:28.516 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000155568 2023-09-23T20:30:28.516 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.516 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.516 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting_for_map not empty", 2023-09-23T20:30:28.516 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.969302+0000", 2023-09-23T20:30:28.517 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.2180000000000005e-06 2023-09-23T20:30:28.517 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.517 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.517 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:30:28.517 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:10.024903+0000", 2023-09-23T20:30:28.517 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.055601105999999997 2023-09-23T20:30:28.517 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.517 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.517 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:30:28.518 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:10.024911+0000", 2023-09-23T20:30:28.518 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.6499999999999996e-06 2023-09-23T20:30:28.518 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.518 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.518 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:30:28.518 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.069774+0000", 2023-09-23T20:30:28.518 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.044863152 2023-09-23T20:30:28.518 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.518 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.518 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for degraded object", 2023-09-23T20:30:28.519 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.070421+0000", 2023-09-23T20:30:28.519 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00064713200000000002 2023-09-23T20:30:28.519 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.519 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.519 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:30:28.519 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.076740+0000", 2023-09-23T20:30:28.519 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0063188300000000001 2023-09-23T20:30:28.519 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.519 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.519 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:30:28.519 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.076829+0000", 2023-09-23T20:30:28.520 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.8709999999999996e-05 2023-09-23T20:30:28.520 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:30:28.520 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:30:28.520 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:30:28.520 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.520 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.520 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.34791.0:28902 5.6 5:650457bb:::benchmark_data_smithi018_228451_object28901:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e118)", 2023-09-23T20:30:28.520 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:23:09.969142+0000", 2023-09-23T20:30:28.520 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 437.38981904000002, 2023-09-23T20:30:28.520 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0945083250000001, 2023-09-23T20:30:28.520 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:30:28.520 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:30:28.521 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:30:28.521 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.34791", 2023-09-23T20:30:28.521 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:30:28.521 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 28902 2023-09-23T20:30:28.521 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.521 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:30:28.521 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.522 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:30:28.522 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.969142+0000", 2023-09-23T20:30:28.522 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:30:28.522 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.522 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.522 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:30:28.522 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.969142+0000", 2023-09-23T20:30:28.522 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:30:28.522 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.522 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.522 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:30:28.523 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.969142+0000", 2023-09-23T20:30:28.523 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:30:28.523 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.523 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.523 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:30:28.523 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.969171+0000", 2023-09-23T20:30:28.523 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.9196999999999998e-05 2023-09-23T20:30:28.523 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.523 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.523 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:30:28.524 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.969178+0000", 2023-09-23T20:30:28.524 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.2350000000000002e-06 2023-09-23T20:30:28.524 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.524 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.524 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:30:28.524 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.969184+0000", 2023-09-23T20:30:28.524 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.0329999999999997e-06 2023-09-23T20:30:28.524 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.524 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.524 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:30:28.524 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.970393+0000", 2023-09-23T20:30:28.525 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.001209183 2023-09-23T20:30:28.525 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.525 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.525 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op must wait for map", 2023-09-23T20:30:28.525 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.970400+0000", 2023-09-23T20:30:28.525 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.2720000000000003e-06 2023-09-23T20:30:28.525 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.525 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.525 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:30:28.525 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:10.010509+0000", 2023-09-23T20:30:28.525 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.04010975 2023-09-23T20:30:28.525 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.526 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.526 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:30:28.526 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:10.010524+0000", 2023-09-23T20:30:28.526 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.4620999999999999e-05 2023-09-23T20:30:28.526 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.526 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.526 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:30:28.526 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.063562+0000", 2023-09-23T20:30:28.526 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0530376589999999 2023-09-23T20:30:28.526 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.527 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.527 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:30:28.527 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.063651+0000", 2023-09-23T20:30:28.527 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.8992000000000004e-05 2023-09-23T20:30:28.527 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:30:28.527 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:30:28.527 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:30:28.527 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.527 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.527 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.34791.0:115711 5.b 5:d0314ec1:::benchmark_data_smithi018_228451_object8650:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e158)", 2023-09-23T20:30:28.527 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:24:44.792263+0000", 2023-09-23T20:30:28.528 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 342.56669837999999, 2023-09-23T20:30:28.528 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.17426969, 2023-09-23T20:30:28.528 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:30:28.528 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:30:28.528 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:30:28.528 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.34791", 2023-09-23T20:30:28.528 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:30:28.529 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 115711 2023-09-23T20:30:28.529 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.529 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:30:28.529 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.529 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:30:28.529 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:44.792263+0000", 2023-09-23T20:30:28.529 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:30:28.529 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.529 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.529 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:30:28.529 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:44.792263+0000", 2023-09-23T20:30:28.529 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:30:28.530 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.530 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.530 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:30:28.530 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:44.792262+0000", 2023-09-23T20:30:28.530 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.999999 2023-09-23T20:30:28.530 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.530 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.530 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:30:28.530 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:44.792268+0000", 2023-09-23T20:30:28.530 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.9580000000000004e-06 2023-09-23T20:30:28.531 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.531 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.531 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:30:28.531 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:44.792280+0000", 2023-09-23T20:30:28.531 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2616000000000001e-05 2023-09-23T20:30:28.531 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.531 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.531 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:30:28.531 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:44.792288+0000", 2023-09-23T20:30:28.532 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.6240000000000002e-06 2023-09-23T20:30:28.532 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.532 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.532 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:30:28.532 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:44.792332+0000", 2023-09-23T20:30:28.532 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.3705999999999998e-05 2023-09-23T20:30:28.532 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.532 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.533 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:30:28.533 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:44.792457+0000", 2023-09-23T20:30:28.533 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00012493399999999999 2023-09-23T20:30:28.533 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.533 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.533 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:30:28.533 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:44.792568+0000", 2023-09-23T20:30:28.533 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000111208 2023-09-23T20:30:28.533 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.533 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.533 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:30:28.533 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:44.796427+0000", 2023-09-23T20:30:28.534 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0038591530000000001 2023-09-23T20:30:28.534 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.534 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.534 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:30:28.534 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:44.798952+0000", 2023-09-23T20:30:28.534 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0025252249999999999 2023-09-23T20:30:28.534 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.534 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.534 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:30:28.534 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:44.966445+0000", 2023-09-23T20:30:28.534 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.167493109 2023-09-23T20:30:28.535 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.535 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.535 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:30:28.535 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:44.966499+0000", 2023-09-23T20:30:28.535 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.3919000000000003e-05 2023-09-23T20:30:28.535 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.535 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.535 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:30:28.535 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:44.966533+0000", 2023-09-23T20:30:28.535 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.3426999999999997e-05 2023-09-23T20:30:28.535 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:30:28.536 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:30:28.536 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:30:28.536 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.536 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.536 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.34791.0:122477 5.a 5:5f61046a:::benchmark_data_smithi018_228451_object15416:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e159)", 2023-09-23T20:30:28.536 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:24:52.040682+0000", 2023-09-23T20:30:28.536 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 335.31827958500003, 2023-09-23T20:30:28.536 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0382268649999999, 2023-09-23T20:30:28.536 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:30:28.536 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:30:28.536 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:30:28.537 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.34791", 2023-09-23T20:30:28.537 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:30:28.537 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 122477 2023-09-23T20:30:28.537 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.537 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:30:28.537 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.537 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:30:28.537 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:52.040682+0000", 2023-09-23T20:30:28.537 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:30:28.538 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.538 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.538 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:30:28.538 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:52.040682+0000", 2023-09-23T20:30:28.538 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:30:28.538 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.538 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.538 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:30:28.538 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:52.040680+0000", 2023-09-23T20:30:28.538 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:30:28.539 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.539 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.539 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:30:28.539 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:52.040685+0000", 2023-09-23T20:30:28.539 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.9540000000000003e-06 2023-09-23T20:30:28.539 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.539 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.539 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:30:28.539 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:52.040695+0000", 2023-09-23T20:30:28.539 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.8600000000000005e-06 2023-09-23T20:30:28.539 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.539 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.539 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:30:28.540 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:52.040702+0000", 2023-09-23T20:30:28.540 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.0809999999999996e-06 2023-09-23T20:30:28.540 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.540 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.540 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:30:28.540 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:52.046630+0000", 2023-09-23T20:30:28.540 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0059274710000000001 2023-09-23T20:30:28.540 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.540 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.540 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:30:28.540 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:52.046727+0000", 2023-09-23T20:30:28.540 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.7386999999999994e-05 2023-09-23T20:30:28.540 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.541 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.541 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 0,4", 2023-09-23T20:30:28.541 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:52.046829+0000", 2023-09-23T20:30:28.541 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000101801 2023-09-23T20:30:28.541 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.541 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.541 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:30:28.541 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:52.055043+0000", 2023-09-23T20:30:28.541 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0082142259999999998 2023-09-23T20:30:28.541 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.541 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.541 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:30:28.542 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:52.063458+0000", 2023-09-23T20:30:28.542 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0084143670000000007 2023-09-23T20:30:28.542 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.542 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.542 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:30:28.542 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:52.090685+0000", 2023-09-23T20:30:28.542 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.027227429000000001 2023-09-23T20:30:28.542 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.542 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.543 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:30:28.543 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:52.090696+0000", 2023-09-23T20:30:28.543 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0993e-05 2023-09-23T20:30:28.543 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.543 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.543 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:30:28.543 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:53.078892+0000", 2023-09-23T20:30:28.543 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.988195822 2023-09-23T20:30:28.543 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.543 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.544 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:30:28.544 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:53.078909+0000", 2023-09-23T20:30:28.544 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.6815000000000001e-05 2023-09-23T20:30:28.544 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:30:28.544 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:30:28.544 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:30:28.544 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.544 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.544 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.34791.0:122487 5.a 5:53c53dfe:::benchmark_data_smithi018_228451_object15426:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e159)", 2023-09-23T20:30:28.544 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:24:52.046007+0000", 2023-09-23T20:30:28.544 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 335.31295460899997, 2023-09-23T20:30:28.545 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0329430020000001, 2023-09-23T20:30:28.545 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:30:28.545 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:30:28.545 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:30:28.545 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.34791", 2023-09-23T20:30:28.545 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:30:28.545 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 122487 2023-09-23T20:30:28.545 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.545 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:30:28.545 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.545 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:30:28.545 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:52.046007+0000", 2023-09-23T20:30:28.546 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:30:28.546 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.546 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.546 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:30:28.546 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:52.046007+0000", 2023-09-23T20:30:28.546 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:30:28.546 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.546 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.546 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:30:28.546 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:52.046006+0000", 2023-09-23T20:30:28.546 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.999999 2023-09-23T20:30:28.547 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.547 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.547 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:30:28.547 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:52.046011+0000", 2023-09-23T20:30:28.547 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.6459999999999998e-06 2023-09-23T20:30:28.547 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.547 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.547 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:30:28.547 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:52.046021+0000", 2023-09-23T20:30:28.547 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.3589999999999997e-06 2023-09-23T20:30:28.547 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.547 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.548 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:30:28.548 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:52.046028+0000", 2023-09-23T20:30:28.548 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.9290000000000001e-06 2023-09-23T20:30:28.548 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.548 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.548 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:30:28.548 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:52.050156+0000", 2023-09-23T20:30:28.548 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0041279200000000002 2023-09-23T20:30:28.548 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.548 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.548 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:30:28.548 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:52.050339+0000", 2023-09-23T20:30:28.548 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00018302999999999999 2023-09-23T20:30:28.549 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.549 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.549 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 0,4", 2023-09-23T20:30:28.549 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:52.050509+0000", 2023-09-23T20:30:28.549 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00017038000000000001 2023-09-23T20:30:28.549 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.549 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.549 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:30:28.549 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:52.058430+0000", 2023-09-23T20:30:28.549 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0079205839999999996 2023-09-23T20:30:28.549 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.549 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.550 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:30:28.550 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:52.063549+0000", 2023-09-23T20:30:28.550 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0051190029999999996 2023-09-23T20:30:28.550 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.550 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.550 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:30:28.550 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:52.090721+0000", 2023-09-23T20:30:28.550 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.027172321999999999 2023-09-23T20:30:28.550 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.550 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.550 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:30:28.551 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:52.090729+0000", 2023-09-23T20:30:28.551 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.695e-06 2023-09-23T20:30:28.551 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.551 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.551 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:30:28.551 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:53.078939+0000", 2023-09-23T20:30:28.551 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.98820984999999995 2023-09-23T20:30:28.551 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.551 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.551 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:30:28.551 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:53.078950+0000", 2023-09-23T20:30:28.551 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1259e-05 2023-09-23T20:30:28.552 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:30:28.552 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:30:28.552 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:30:28.552 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.552 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.552 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.34791.0:122497 5.a 5:52cb959d:::benchmark_data_smithi018_228451_object15436:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e159)", 2023-09-23T20:30:28.552 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:24:52.063924+0000", 2023-09-23T20:30:28.552 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 335.295037546, 2023-09-23T20:30:28.552 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0150647070000001, 2023-09-23T20:30:28.552 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:30:28.552 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:30:28.552 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:30:28.552 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.34791", 2023-09-23T20:30:28.553 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:30:28.553 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 122497 2023-09-23T20:30:28.553 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.553 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:30:28.553 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.553 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:30:28.553 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:52.063924+0000", 2023-09-23T20:30:28.553 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:30:28.554 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.554 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.554 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:30:28.554 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:52.063924+0000", 2023-09-23T20:30:28.554 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:30:28.554 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.554 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.554 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:30:28.554 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:52.063922+0000", 2023-09-23T20:30:28.554 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:30:28.555 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.555 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.555 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:30:28.555 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:52.063929+0000", 2023-09-23T20:30:28.555 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.109e-06 2023-09-23T20:30:28.555 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.555 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.555 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:30:28.555 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:52.063945+0000", 2023-09-23T20:30:28.555 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.5875999999999998e-05 2023-09-23T20:30:28.555 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.555 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.555 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:30:28.556 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:52.063954+0000", 2023-09-23T20:30:28.556 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.4659999999999999e-06 2023-09-23T20:30:28.556 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.556 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.556 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:30:28.556 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:52.064052+0000", 2023-09-23T20:30:28.556 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.8398000000000002e-05 2023-09-23T20:30:28.556 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.556 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.556 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:30:28.556 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:52.064239+0000", 2023-09-23T20:30:28.557 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00018665399999999999 2023-09-23T20:30:28.557 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.557 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.557 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 0,4", 2023-09-23T20:30:28.557 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:52.064381+0000", 2023-09-23T20:30:28.557 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000142464 2023-09-23T20:30:28.557 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.557 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.557 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:30:28.557 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:52.090753+0000", 2023-09-23T20:30:28.557 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.026371896999999998 2023-09-23T20:30:28.557 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.557 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.558 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:30:28.558 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:52.090761+0000", 2023-09-23T20:30:28.558 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.6909999999999997e-06 2023-09-23T20:30:28.558 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.558 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.558 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:30:28.558 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:53.078976+0000", 2023-09-23T20:30:28.558 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.98821585199999995 2023-09-23T20:30:28.558 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.558 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.558 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:30:28.559 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:53.078989+0000", 2023-09-23T20:30:28.559 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2106000000000001e-05 2023-09-23T20:30:28.559 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:30:28.559 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:30:28.559 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:30:28.559 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.559 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.559 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.34791.0:122499 5.a 5.db9883aa (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e159)", 2023-09-23T20:30:28.559 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:24:52.063982+0000", 2023-09-23T20:30:28.559 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 335.29497970400001, 2023-09-23T20:30:28.559 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.01503842, 2023-09-23T20:30:28.560 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:30:28.560 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:30:28.560 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:30:28.560 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.34791", 2023-09-23T20:30:28.560 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:30:28.560 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 122499 2023-09-23T20:30:28.560 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.560 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:30:28.560 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.560 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:30:28.560 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:52.063982+0000", 2023-09-23T20:30:28.560 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:30:28.560 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.561 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.561 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:30:28.561 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:52.063982+0000", 2023-09-23T20:30:28.561 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:30:28.561 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.561 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.561 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:30:28.561 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:52.063981+0000", 2023-09-23T20:30:28.562 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:30:28.562 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.562 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.562 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:30:28.562 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:52.063986+0000", 2023-09-23T20:30:28.562 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.8600000000000001e-06 2023-09-23T20:30:28.563 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.563 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.563 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:30:28.563 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:52.063997+0000", 2023-09-23T20:30:28.563 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1418e-05 2023-09-23T20:30:28.564 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.564 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.564 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:30:28.564 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:52.064004+0000", 2023-09-23T20:30:28.564 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.0219999999999999e-06 2023-09-23T20:30:28.564 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.564 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.565 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:30:28.565 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:52.090781+0000", 2023-09-23T20:30:28.565 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.026776423000000001 2023-09-23T20:30:28.565 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.565 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.565 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:30:28.566 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:52.090797+0000", 2023-09-23T20:30:28.566 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.6223999999999999e-05 2023-09-23T20:30:28.566 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.566 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.566 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:30:28.566 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:53.079010+0000", 2023-09-23T20:30:28.566 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.98821315700000001 2023-09-23T20:30:28.566 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.566 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.567 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:30:28.567 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:53.079020+0000", 2023-09-23T20:30:28.567 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0013999999999999e-05 2023-09-23T20:30:28.567 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:30:28.567 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:30:28.567 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:30:28.567 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.567 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.567 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.34791.0:122509 5.a 5.9a5288fa (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e159)", 2023-09-23T20:30:28.567 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:24:52.093777+0000", 2023-09-23T20:30:28.567 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 335.26518414600002, 2023-09-23T20:30:28.568 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.98527409700000002, 2023-09-23T20:30:28.568 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:30:28.568 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:30:28.568 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:30:28.568 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.34791", 2023-09-23T20:30:28.568 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:30:28.568 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 122509 2023-09-23T20:30:28.568 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.568 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:30:28.568 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.568 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:30:28.568 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:52.093777+0000", 2023-09-23T20:30:28.569 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:30:28.569 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.569 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.569 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:30:28.569 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:52.093777+0000", 2023-09-23T20:30:28.569 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:30:28.569 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.569 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.569 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:30:28.569 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:52.093776+0000", 2023-09-23T20:30:28.569 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:30:28.570 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.570 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.570 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:30:28.570 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:52.093782+0000", 2023-09-23T20:30:28.570 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.9599999999999997e-06 2023-09-23T20:30:28.570 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.570 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.570 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:30:28.570 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:52.093800+0000", 2023-09-23T20:30:28.570 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.7878000000000001e-05 2023-09-23T20:30:28.570 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.571 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.571 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:30:28.571 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:52.093819+0000", 2023-09-23T20:30:28.571 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.9227000000000001e-05 2023-09-23T20:30:28.571 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.571 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.571 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:30:28.571 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:52.094412+0000", 2023-09-23T20:30:28.571 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00059324900000000001 2023-09-23T20:30:28.571 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.571 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.571 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:30:28.571 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:52.094423+0000", 2023-09-23T20:30:28.572 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0997e-05 2023-09-23T20:30:28.572 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.572 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.572 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:30:28.572 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:53.079042+0000", 2023-09-23T20:30:28.572 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.98461890399999996 2023-09-23T20:30:28.572 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.572 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.572 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:30:28.572 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:53.079051+0000", 2023-09-23T20:30:28.572 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.4509999999999998e-06 2023-09-23T20:30:28.573 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:30:28.573 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:30:28.573 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:30:28.573 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.573 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.573 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.34791.0:122516 5.a 5.76cbecaa (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e160)", 2023-09-23T20:30:28.573 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:24:52.104509+0000", 2023-09-23T20:30:28.573 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 335.25445228699999, 2023-09-23T20:30:28.573 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.974572522, 2023-09-23T20:30:28.573 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:30:28.573 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:30:28.573 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:30:28.574 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.34791", 2023-09-23T20:30:28.574 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:30:28.574 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 122516 2023-09-23T20:30:28.574 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.574 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:30:28.574 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.574 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:30:28.574 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:52.104509+0000", 2023-09-23T20:30:28.574 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:30:28.574 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.574 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.575 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:30:28.575 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:52.104509+0000", 2023-09-23T20:30:28.575 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:30:28.575 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.575 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.575 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:30:28.575 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:52.104507+0000", 2023-09-23T20:30:28.575 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:30:28.575 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.575 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.575 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:30:28.575 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:52.104516+0000", 2023-09-23T20:30:28.576 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.0939999999999997e-06 2023-09-23T20:30:28.576 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.576 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.576 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:30:28.576 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:52.104531+0000", 2023-09-23T20:30:28.576 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.4579999999999999e-05 2023-09-23T20:30:28.576 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.576 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.576 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:30:28.576 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:52.104540+0000", 2023-09-23T20:30:28.576 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.1910000000000006e-06 2023-09-23T20:30:28.576 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.576 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.577 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:30:28.577 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:52.104587+0000", 2023-09-23T20:30:28.577 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.6489999999999997e-05 2023-09-23T20:30:28.577 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.577 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.577 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:30:28.577 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:52.104596+0000", 2023-09-23T20:30:28.577 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.3009999999999995e-06 2023-09-23T20:30:28.577 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.578 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.578 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:30:28.578 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:53.079072+0000", 2023-09-23T20:30:28.578 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.97447639500000005 2023-09-23T20:30:28.578 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.578 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.578 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:30:28.578 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:24:53.079082+0000", 2023-09-23T20:30:28.579 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.4150000000000005e-06 2023-09-23T20:30:28.579 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:30:28.579 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:30:28.579 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:30:28.579 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.579 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.579 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.45161.0:82233 6.f 6:f2ac9d30:::benchmark_data_smithi018_270558_object82232:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e239)", 2023-09-23T20:30:28.579 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:28:26.128298+0000", 2023-09-23T20:30:28.579 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 121.230663009, 2023-09-23T20:30:28.579 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.13991342900000001, 2023-09-23T20:30:28.579 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:30:28.579 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:30:28.580 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:30:28.580 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.45161", 2023-09-23T20:30:28.580 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3795614987", 2023-09-23T20:30:28.580 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 82233 2023-09-23T20:30:28.580 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.580 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:30:28.580 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.580 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:30:28.580 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:26.128298+0000", 2023-09-23T20:30:28.580 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:30:28.580 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.580 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.580 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:30:28.581 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:26.128298+0000", 2023-09-23T20:30:28.581 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:30:28.581 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.581 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.581 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:30:28.581 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:26.128296+0000", 2023-09-23T20:30:28.581 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:30:28.581 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.581 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.581 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:30:28.581 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:26.128321+0000", 2023-09-23T20:30:28.581 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.4227e-05 2023-09-23T20:30:28.581 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.582 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.582 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:30:28.582 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:26.128334+0000", 2023-09-23T20:30:28.582 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3112000000000001e-05 2023-09-23T20:30:28.582 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.582 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.582 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:30:28.582 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:26.128343+0000", 2023-09-23T20:30:28.582 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.7849999999999997e-06 2023-09-23T20:30:28.583 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.583 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.583 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:30:28.583 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:26.128405+0000", 2023-09-23T20:30:28.583 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.2830999999999995e-05 2023-09-23T20:30:28.583 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.584 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.584 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:30:28.584 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:26.128596+0000", 2023-09-23T20:30:28.584 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00019013 2023-09-23T20:30:28.584 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.584 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.584 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:30:28.584 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:26.128706+0000", 2023-09-23T20:30:28.585 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000110388 2023-09-23T20:30:28.585 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.585 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.585 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:30:28.585 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:26.135625+0000", 2023-09-23T20:30:28.585 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.006919494 2023-09-23T20:30:28.585 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.585 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.585 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:30:28.585 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:26.135805+0000", 2023-09-23T20:30:28.585 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00017975399999999999 2023-09-23T20:30:28.585 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.586 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.586 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:30:28.586 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:26.268129+0000", 2023-09-23T20:30:28.586 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.13232397000000001 2023-09-23T20:30:28.586 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.586 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.586 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:30:28.586 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:26.268177+0000", 2023-09-23T20:30:28.586 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.7707000000000002e-05 2023-09-23T20:30:28.586 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.586 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.586 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:30:28.587 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:26.268212+0000", 2023-09-23T20:30:28.587 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.4941999999999999e-05 2023-09-23T20:30:28.587 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:30:28.587 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:30:28.587 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:30:28.587 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.587 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.587 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.45161.0:136049 6.f 6:f1f0f499:::benchmark_data_smithi018_270558_object31310:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e258)", 2023-09-23T20:30:28.587 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:29:27.360919+0000", 2023-09-23T20:30:28.587 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 59.998042368999997, 2023-09-23T20:30:28.587 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.1770357090000001, 2023-09-23T20:30:28.587 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:30:28.588 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:30:28.588 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:30:28.588 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.45161", 2023-09-23T20:30:28.588 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3795614987", 2023-09-23T20:30:28.588 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 136049 2023-09-23T20:30:28.588 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.588 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:30:28.588 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.588 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:30:28.588 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.360919+0000", 2023-09-23T20:30:28.588 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:30:28.589 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.589 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.589 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:30:28.589 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.360919+0000", 2023-09-23T20:30:28.589 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:30:28.589 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.589 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.589 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:30:28.589 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.360917+0000", 2023-09-23T20:30:28.589 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:30:28.589 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.589 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.589 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:30:28.590 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.360925+0000", 2023-09-23T20:30:28.590 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.8480000000000001e-06 2023-09-23T20:30:28.590 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.590 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.590 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:30:28.590 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.360944+0000", 2023-09-23T20:30:28.590 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.8439000000000001e-05 2023-09-23T20:30:28.590 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.590 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.590 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:30:28.590 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.360954+0000", 2023-09-23T20:30:28.591 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0327e-05 2023-09-23T20:30:28.591 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.591 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.591 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:30:28.591 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.363434+0000", 2023-09-23T20:30:28.591 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0024801100000000002 2023-09-23T20:30:28.591 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.591 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.591 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:30:28.591 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.363450+0000", 2023-09-23T20:30:28.591 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.5359999999999999e-05 2023-09-23T20:30:28.591 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.592 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.592 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:30:28.592 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:28.342983+0000", 2023-09-23T20:30:28.592 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.97953384099999996 2023-09-23T20:30:28.592 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.592 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.592 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:30:28.592 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:28.342997+0000", 2023-09-23T20:30:28.592 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3641e-05 2023-09-23T20:30:28.592 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.592 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.592 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:30:28.593 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.518718+0000", 2023-09-23T20:30:28.593 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.175721432 2023-09-23T20:30:28.593 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.593 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.593 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:30:28.593 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.518815+0000", 2023-09-23T20:30:28.593 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.6764999999999999e-05 2023-09-23T20:30:28.593 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.593 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.593 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 3,4,6", 2023-09-23T20:30:28.593 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.519272+0000", 2023-09-23T20:30:28.593 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00045647899999999998 2023-09-23T20:30:28.594 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.594 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.594 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:30:28.594 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.537608+0000", 2023-09-23T20:30:28.594 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.018336363000000001 2023-09-23T20:30:28.594 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.594 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.594 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:30:28.594 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.537675+0000", 2023-09-23T20:30:28.594 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.7039999999999995e-05 2023-09-23T20:30:28.594 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.595 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.595 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:30:28.595 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.537777+0000", 2023-09-23T20:30:28.595 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000102147 2023-09-23T20:30:28.595 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.595 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.595 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:30:28.595 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.537821+0000", 2023-09-23T20:30:28.595 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.3547999999999997e-05 2023-09-23T20:30:28.595 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.595 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.595 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:30:28.596 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.537858+0000", 2023-09-23T20:30:28.596 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.7132999999999998e-05 2023-09-23T20:30:28.596 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.596 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.596 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:30:28.596 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.537955+0000", 2023-09-23T20:30:28.596 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.6804000000000004e-05 2023-09-23T20:30:28.596 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:30:28.596 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:30:28.596 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:30:28.596 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.596 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.596 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.45161.0:136051 6.f 6:fcaf49a8:::benchmark_data_smithi018_270558_object31312:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e258)", 2023-09-23T20:30:28.597 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:29:27.375493+0000", 2023-09-23T20:30:28.597 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 59.983467996000002, 2023-09-23T20:30:28.597 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.174356832, 2023-09-23T20:30:28.597 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:30:28.597 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:30:28.597 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:30:28.597 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.45161", 2023-09-23T20:30:28.597 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3795614987", 2023-09-23T20:30:28.597 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 136051 2023-09-23T20:30:28.597 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.597 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:30:28.598 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.598 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:30:28.598 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.375493+0000", 2023-09-23T20:30:28.598 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:30:28.598 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.598 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.598 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:30:28.598 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.375493+0000", 2023-09-23T20:30:28.598 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:30:28.598 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.598 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.598 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:30:28.599 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.375491+0000", 2023-09-23T20:30:28.599 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:30:28.599 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.599 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.599 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:30:28.599 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.375501+0000", 2023-09-23T20:30:28.599 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.149e-06 2023-09-23T20:30:28.599 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.599 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.599 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:30:28.599 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.375516+0000", 2023-09-23T20:30:28.599 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.5463000000000001e-05 2023-09-23T20:30:28.600 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.600 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.600 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:30:28.600 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.375528+0000", 2023-09-23T20:30:28.600 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1800999999999999e-05 2023-09-23T20:30:28.600 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.600 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.600 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:30:28.600 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.375579+0000", 2023-09-23T20:30:28.600 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.096e-05 2023-09-23T20:30:28.600 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.600 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.601 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:30:28.601 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.375592+0000", 2023-09-23T20:30:28.601 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.343e-05 2023-09-23T20:30:28.601 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.601 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.601 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:30:28.601 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:28.343022+0000", 2023-09-23T20:30:28.601 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.96742986500000006 2023-09-23T20:30:28.601 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.601 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.601 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:30:28.601 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:28.343030+0000", 2023-09-23T20:30:28.601 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.2400000000000007e-06 2023-09-23T20:30:28.602 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.602 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.602 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:30:28.602 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.530153+0000", 2023-09-23T20:30:28.602 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.187122427 2023-09-23T20:30:28.602 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.602 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.602 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:30:28.602 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.530266+0000", 2023-09-23T20:30:28.602 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00011287500000000001 2023-09-23T20:30:28.602 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.602 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.603 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 3,4,6", 2023-09-23T20:30:28.603 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.530349+0000", 2023-09-23T20:30:28.603 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.3065999999999994e-05 2023-09-23T20:30:28.603 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.603 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.603 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:30:28.603 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.537739+0000", 2023-09-23T20:30:28.603 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0073898840000000002 2023-09-23T20:30:28.603 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.603 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.603 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:30:28.603 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.537745+0000", 2023-09-23T20:30:28.604 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.5860000000000001e-06 2023-09-23T20:30:28.604 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.604 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.604 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:30:28.604 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.538012+0000", 2023-09-23T20:30:28.604 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000267018 2023-09-23T20:30:28.604 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.604 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.604 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:30:28.604 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.549701+0000", 2023-09-23T20:30:28.604 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.011688426 2023-09-23T20:30:28.604 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.605 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.605 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:30:28.605 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.549735+0000", 2023-09-23T20:30:28.605 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.4161e-05 2023-09-23T20:30:28.605 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.605 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.605 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:30:28.605 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.549850+0000", 2023-09-23T20:30:28.605 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00011539099999999999 2023-09-23T20:30:28.605 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:30:28.605 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:30:28.606 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:30:28.606 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.606 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.606 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.45161.0:136053 6.f 6:f2209241:::benchmark_data_smithi018_270558_object31314:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e258)", 2023-09-23T20:30:28.606 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:29:27.379666+0000", 2023-09-23T20:30:28.606 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 59.979295467999997, 2023-09-23T20:30:28.606 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.1768631190000001, 2023-09-23T20:30:28.606 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:30:28.606 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:30:28.606 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:30:28.606 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.45161", 2023-09-23T20:30:28.607 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3795614987", 2023-09-23T20:30:28.607 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 136053 2023-09-23T20:30:28.607 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.607 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:30:28.607 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.607 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:30:28.607 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.379666+0000", 2023-09-23T20:30:28.607 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:30:28.607 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.607 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.607 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:30:28.607 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.379666+0000", 2023-09-23T20:30:28.607 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:30:28.608 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.608 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.608 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:30:28.608 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.379664+0000", 2023-09-23T20:30:28.608 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999976 2023-09-23T20:30:28.608 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.608 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.608 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:30:28.608 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.379674+0000", 2023-09-23T20:30:28.608 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0271999999999999e-05 2023-09-23T20:30:28.608 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.608 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.609 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:30:28.609 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.379698+0000", 2023-09-23T20:30:28.609 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.3941999999999999e-05 2023-09-23T20:30:28.609 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.609 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.609 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:30:28.609 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.379711+0000", 2023-09-23T20:30:28.609 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2743e-05 2023-09-23T20:30:28.609 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.609 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.609 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:30:28.609 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.379865+0000", 2023-09-23T20:30:28.610 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00015445199999999999 2023-09-23T20:30:28.610 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.610 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.610 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:30:28.610 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.379881+0000", 2023-09-23T20:30:28.610 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.5423000000000001e-05 2023-09-23T20:30:28.610 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.610 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.610 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:30:28.610 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:28.343053+0000", 2023-09-23T20:30:28.610 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.963171993 2023-09-23T20:30:28.611 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.611 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.611 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:30:28.611 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:28.343061+0000", 2023-09-23T20:30:28.611 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.9480000000000008e-06 2023-09-23T20:30:28.611 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.611 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.611 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:30:28.611 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.534004+0000", 2023-09-23T20:30:28.611 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1909433570000001 2023-09-23T20:30:28.611 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.612 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.612 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:30:28.612 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.534089+0000", 2023-09-23T20:30:28.612 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.5394000000000003e-05 2023-09-23T20:30:28.612 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.612 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.612 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 3,4,6", 2023-09-23T20:30:28.612 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.534172+0000", 2023-09-23T20:30:28.612 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.2553999999999996e-05 2023-09-23T20:30:28.612 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.612 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.613 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:30:28.613 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.537964+0000", 2023-09-23T20:30:28.613 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0037922590000000001 2023-09-23T20:30:28.613 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.613 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.613 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:30:28.613 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.538056+0000", 2023-09-23T20:30:28.613 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.1833e-05 2023-09-23T20:30:28.613 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.613 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.613 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:30:28.613 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.538092+0000", 2023-09-23T20:30:28.614 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.6528e-05 2023-09-23T20:30:28.614 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.614 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.614 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:30:28.614 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.556365+0000", 2023-09-23T20:30:28.614 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.018272594 2023-09-23T20:30:28.614 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.614 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.614 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:30:28.614 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.556403+0000", 2023-09-23T20:30:28.615 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.8034999999999998e-05 2023-09-23T20:30:28.615 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.615 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.615 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:30:28.615 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.556529+0000", 2023-09-23T20:30:28.615 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00012598199999999999 2023-09-23T20:30:28.615 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:30:28.615 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:30:28.615 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:30:28.615 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.616 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.616 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.45161.0:136065 6.e 6:7df88343:::benchmark_data_smithi018_270558_object31326:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e258)", 2023-09-23T20:30:28.616 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:29:27.389482+0000", 2023-09-23T20:30:28.616 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 59.969479903999996, 2023-09-23T20:30:28.616 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.1285085910000001, 2023-09-23T20:30:28.616 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:30:28.616 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:30:28.616 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:30:28.616 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.45161", 2023-09-23T20:30:28.616 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3795614987", 2023-09-23T20:30:28.617 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 136065 2023-09-23T20:30:28.617 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.617 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:30:28.617 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.617 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:30:28.617 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.389482+0000", 2023-09-23T20:30:28.617 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:30:28.617 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.617 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.617 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:30:28.618 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.389482+0000", 2023-09-23T20:30:28.618 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:30:28.618 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.618 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.618 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:30:28.618 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.389480+0000", 2023-09-23T20:30:28.618 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:30:28.618 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.618 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.618 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:30:28.618 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.389488+0000", 2023-09-23T20:30:28.619 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.6120000000000001e-06 2023-09-23T20:30:28.619 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.619 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.619 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:30:28.619 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.389503+0000", 2023-09-23T20:30:28.619 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.5339999999999999e-05 2023-09-23T20:30:28.619 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.619 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.619 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:30:28.619 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.389516+0000", 2023-09-23T20:30:28.619 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3467e-05 2023-09-23T20:30:28.619 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.620 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.620 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:30:28.620 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.389586+0000", 2023-09-23T20:30:28.620 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.9797999999999998e-05 2023-09-23T20:30:28.620 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.620 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.620 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:30:28.620 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.389597+0000", 2023-09-23T20:30:28.620 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0961999999999999e-05 2023-09-23T20:30:28.620 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.620 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.620 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:30:28.620 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:28.343523+0000", 2023-09-23T20:30:28.621 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.95392566499999998 2023-09-23T20:30:28.621 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.621 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.621 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:30:28.621 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:28.343532+0000", 2023-09-23T20:30:28.621 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.8070000000000006e-06 2023-09-23T20:30:28.621 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.621 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.621 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:30:28.621 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.508816+0000", 2023-09-23T20:30:28.621 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1652842480000001 2023-09-23T20:30:28.621 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.622 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.622 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:30:28.622 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.511009+0000", 2023-09-23T20:30:28.622 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0021932660000000001 2023-09-23T20:30:28.622 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.622 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.622 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 4,5", 2023-09-23T20:30:28.622 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.511162+0000", 2023-09-23T20:30:28.622 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00015258099999999999 2023-09-23T20:30:28.622 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.622 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.623 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:30:28.623 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.516451+0000", 2023-09-23T20:30:28.623 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.005289261 2023-09-23T20:30:28.623 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.623 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.623 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:30:28.623 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.516483+0000", 2023-09-23T20:30:28.623 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.2236000000000001e-05 2023-09-23T20:30:28.623 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.623 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.623 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:30:28.624 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.517808+0000", 2023-09-23T20:30:28.624 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0013247910000000001 2023-09-23T20:30:28.624 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.624 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.624 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:30:28.624 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.517866+0000", 2023-09-23T20:30:28.624 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.7624000000000003e-05 2023-09-23T20:30:28.624 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.624 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.624 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:30:28.624 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.517990+0000", 2023-09-23T20:30:28.624 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000124411 2023-09-23T20:30:28.625 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:30:28.625 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:30:28.625 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:30:28.625 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.625 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.625 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.45161.0:136068 6.f 6:f6357533:::benchmark_data_smithi018_270558_object31329:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e258)", 2023-09-23T20:30:28.625 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:29:27.391642+0000", 2023-09-23T20:30:28.625 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 59.967318943999999, 2023-09-23T20:30:28.625 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.1651141979999999, 2023-09-23T20:30:28.625 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:30:28.625 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:30:28.625 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:30:28.625 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.45161", 2023-09-23T20:30:28.626 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3795614987", 2023-09-23T20:30:28.626 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 136068 2023-09-23T20:30:28.626 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.626 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:30:28.626 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.626 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:30:28.626 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.391642+0000", 2023-09-23T20:30:28.626 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:30:28.627 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.627 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.627 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:30:28.627 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.391642+0000", 2023-09-23T20:30:28.627 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:30:28.627 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.627 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.627 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:30:28.627 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.391641+0000", 2023-09-23T20:30:28.627 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.999999 2023-09-23T20:30:28.627 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.627 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.628 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:30:28.628 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.391648+0000", 2023-09-23T20:30:28.628 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.0350000000000004e-06 2023-09-23T20:30:28.628 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.628 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.628 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:30:28.628 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.391663+0000", 2023-09-23T20:30:28.628 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.4722000000000001e-05 2023-09-23T20:30:28.628 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.628 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.628 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:30:28.628 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.391672+0000", 2023-09-23T20:30:28.629 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.8359999999999998e-06 2023-09-23T20:30:28.629 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.629 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.629 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:30:28.629 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.391746+0000", 2023-09-23T20:30:28.629 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.3660999999999998e-05 2023-09-23T20:30:28.629 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.629 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.629 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:30:28.629 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.391758+0000", 2023-09-23T20:30:28.629 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.236e-05 2023-09-23T20:30:28.630 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.630 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.630 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:30:28.630 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:28.343081+0000", 2023-09-23T20:30:28.630 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.95132316299999997 2023-09-23T20:30:28.630 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.630 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.630 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:30:28.630 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:28.343089+0000", 2023-09-23T20:30:28.630 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.0809999999999994e-06 2023-09-23T20:30:28.630 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.630 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.631 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:30:28.631 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.538120+0000", 2023-09-23T20:30:28.631 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.195030437 2023-09-23T20:30:28.631 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.631 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.631 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:30:28.631 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.538205+0000", 2023-09-23T20:30:28.631 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.5142999999999994e-05 2023-09-23T20:30:28.631 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.631 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.632 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 3,4,6", 2023-09-23T20:30:28.632 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.538283+0000", 2023-09-23T20:30:28.632 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.8652999999999994e-05 2023-09-23T20:30:28.632 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.632 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.632 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:30:28.632 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.549604+0000", 2023-09-23T20:30:28.632 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.011320436 2023-09-23T20:30:28.632 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.632 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.632 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:30:28.632 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.549613+0000", 2023-09-23T20:30:28.633 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.0680000000000003e-06 2023-09-23T20:30:28.633 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.633 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.633 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:30:28.633 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.549680+0000", 2023-09-23T20:30:28.633 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.6829999999999995e-05 2023-09-23T20:30:28.633 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.633 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.633 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:30:28.633 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.556587+0000", 2023-09-23T20:30:28.633 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0069069190000000001 2023-09-23T20:30:28.634 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.634 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.634 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:30:28.634 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.556645+0000", 2023-09-23T20:30:28.634 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.8740999999999999e-05 2023-09-23T20:30:28.634 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.634 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.634 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:30:28.634 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.556757+0000", 2023-09-23T20:30:28.635 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000111208 2023-09-23T20:30:28.635 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:30:28.635 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:30:28.635 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:30:28.635 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.635 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.635 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.45161.0:136069 6.e 6:7343e225:::benchmark_data_smithi018_270558_object31330:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e258)", 2023-09-23T20:30:28.635 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:29:27.396169+0000", 2023-09-23T20:30:28.635 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 59.962792108000002, 2023-09-23T20:30:28.635 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.122030412, 2023-09-23T20:30:28.636 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:30:28.636 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:30:28.636 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:30:28.636 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.45161", 2023-09-23T20:30:28.636 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3795614987", 2023-09-23T20:30:28.636 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 136069 2023-09-23T20:30:28.636 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.636 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:30:28.636 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.636 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:30:28.637 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.396169+0000", 2023-09-23T20:30:28.637 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:30:28.637 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.637 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.637 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:30:28.637 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.396169+0000", 2023-09-23T20:30:28.638 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:30:28.638 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.638 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.638 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:30:28.638 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.396168+0000", 2023-09-23T20:30:28.638 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:30:28.638 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.638 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.638 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:30:28.639 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.396175+0000", 2023-09-23T20:30:28.639 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.8310000000000003e-06 2023-09-23T20:30:28.639 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.639 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.639 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:30:28.639 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.396191+0000", 2023-09-23T20:30:28.639 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.5736999999999999e-05 2023-09-23T20:30:28.639 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.639 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.639 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:30:28.639 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.396199+0000", 2023-09-23T20:30:28.639 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.1839999999999999e-06 2023-09-23T20:30:28.639 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.640 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.640 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:30:28.640 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.396257+0000", 2023-09-23T20:30:28.640 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.8400999999999999e-05 2023-09-23T20:30:28.640 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.640 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.640 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:30:28.640 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.396268+0000", 2023-09-23T20:30:28.640 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1099e-05 2023-09-23T20:30:28.640 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.640 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.640 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:30:28.641 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:28.345135+0000", 2023-09-23T20:30:28.641 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.948866459 2023-09-23T20:30:28.641 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.641 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.641 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:30:28.641 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:28.345145+0000", 2023-09-23T20:30:28.641 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.9210000000000004e-06 2023-09-23T20:30:28.641 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.641 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.641 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:30:28.641 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.514448+0000", 2023-09-23T20:30:28.641 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1693032249999999 2023-09-23T20:30:28.641 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.642 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.642 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:30:28.642 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.514537+0000", 2023-09-23T20:30:28.642 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.9529000000000003e-05 2023-09-23T20:30:28.642 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.642 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.642 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 4,5", 2023-09-23T20:30:28.642 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.514611+0000", 2023-09-23T20:30:28.642 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.3763999999999994e-05 2023-09-23T20:30:28.642 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.642 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.643 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:30:28.643 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.516564+0000", 2023-09-23T20:30:28.643 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.001952992 2023-09-23T20:30:28.643 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.643 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.643 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:30:28.643 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.516815+0000", 2023-09-23T20:30:28.643 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00025055699999999999 2023-09-23T20:30:28.643 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.643 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.643 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:30:28.644 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.518040+0000", 2023-09-23T20:30:28.644 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0012252529999999999 2023-09-23T20:30:28.644 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.644 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.644 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:30:28.644 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.518094+0000", 2023-09-23T20:30:28.644 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.4104000000000002e-05 2023-09-23T20:30:28.644 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:28.644 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:28.644 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:30:28.644 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.518200+0000", 2023-09-23T20:30:28.644 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000105715 2023-09-23T20:30:28.645 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:30:28.645 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:30:28.645 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:30:28.645 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:30:28.645 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:30:28.645 INFO:teuthology.orchestra.run.smithi018.stdout:} 2023-09-23T20:30:28.645 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:28.697 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:30:28.703 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:28.703 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:30:28.804 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:30:28.891 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:29.008 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:30:29.014 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:29.014 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:30:29.115 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:30:29.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:29 smithi018 ceph-mon[184144]: pgmap v627: 119 pgs: 4 active+recovery_wait+degraded, 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 2.2 GiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 671 op/s; 34840/106038 objects degraded (32.856%); 5967/106038 objects misplaced (5.627%); 6.2 MiB/s, 98 objects/s recovering 2023-09-23T20:30:29.144 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:29 smithi018 ceph-mon[179647]: pgmap v627: 119 pgs: 4 active+recovery_wait+degraded, 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 2.2 GiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 671 op/s; 34840/106038 objects degraded (32.856%); 5967/106038 objects misplaced (5.627%); 6.2 MiB/s, 98 objects/s recovering 2023-09-23T20:30:29.238 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:29.301 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:30:29.305 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:30:29.309 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:29.309 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:30:29.410 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:30:29.427 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T20:30:29.450 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:29 smithi149 ceph-mon[170394]: pgmap v627: 119 pgs: 4 active+recovery_wait+degraded, 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 2.2 GiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 671 op/s; 34840/106038 objects degraded (32.856%); 5967/106038 objects misplaced (5.627%); 6.2 MiB/s, 98 objects/s recovering 2023-09-23T20:30:29.459 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:29.566 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:30:29.573 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:29.573 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:30:29.674 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:30:29.775 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:29.857 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:30:29.864 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:29.865 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:30:29.942 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T20:30:29.966 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:30:30.004 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:30.100 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:30:30.106 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:30.106 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:30:30.207 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:30:30.340 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:30.478 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:30:30.485 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:30 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:30:30.485 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:30 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:30:30.485 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:30 smithi018 ceph-mon[179647]: pgmap v628: 119 pgs: 4 active+recovery_wait+degraded, 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 2.1 GiB data, 8.3 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 656 op/s; 34840/103332 objects degraded (33.717%); 5522/103332 objects misplaced (5.344%); 6.1 MiB/s, 103 objects/s recovering 2023-09-23T20:30:30.485 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:30 smithi018 ceph-mon[179647]: 6.19 scrub starts 2023-09-23T20:30:30.486 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:30 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:30:30.486 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:30 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:30:30.486 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:30 smithi018 ceph-mon[184144]: pgmap v628: 119 pgs: 4 active+recovery_wait+degraded, 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 2.1 GiB data, 8.3 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 656 op/s; 34840/103332 objects degraded (33.717%); 5522/103332 objects misplaced (5.344%); 6.1 MiB/s, 103 objects/s recovering 2023-09-23T20:30:30.486 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:30 smithi018 ceph-mon[184144]: 6.19 scrub starts 2023-09-23T20:30:30.486 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:30.487 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:30:30.537 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:30:30.588 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:30:30.626 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:30.681 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:30:30.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:30 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:30:30.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:30 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:30:30.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:30 smithi149 ceph-mon[170394]: pgmap v628: 119 pgs: 4 active+recovery_wait+degraded, 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 2.1 GiB data, 8.3 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 656 op/s; 34840/103332 objects degraded (33.717%); 5522/103332 objects misplaced (5.344%); 6.1 MiB/s, 103 objects/s recovering 2023-09-23T20:30:30.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:30 smithi149 ceph-mon[170394]: 6.19 scrub starts 2023-09-23T20:30:30.717 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:30:30.721 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:30.721 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:30:30.822 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:30:30.866 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:30.919 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:30:30.922 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:30.922 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:30:31.023 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:30:31.139 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:31.243 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:30:31.691 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:30:31.705 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:31.706 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:30:31.725 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:31 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 34840/103332 objects degraded (33.717%), 19 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T20:30:31.725 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:31 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:30:31.725 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:31 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:30:31.727 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:31 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 34840/103332 objects degraded (33.717%), 19 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T20:30:31.727 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:31 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:30:31.727 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:31 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:30:31.807 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:30:31.833 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:31 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 34840/103332 objects degraded (33.717%), 19 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T20:30:31.833 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:31 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:30:31.833 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:31 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:30:31.836 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:31.885 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:30:31.891 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:31.891 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:30:31.992 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:30:32.029 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:32.069 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:30:32.073 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:32.073 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:30:32.174 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:30:32.204 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:32.252 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:30:32.259 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:32.260 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:30:32.360 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:30:32.456 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:32.531 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:30:32.537 INFO:teuthology.orchestra.run.smithi018.stderr:reweighted osd.3 to 1 (10000) 2023-09-23T20:30:32.538 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:32.538 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:30:32.639 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:30:32.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:32 smithi018 ceph-mon[184144]: pgmap v629: 119 pgs: 2 active+recovery_wait+degraded, 3 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 2.0 GiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 619 op/s; 34671/98508 objects degraded (35.196%); 5522/98508 objects misplaced (5.606%); 7.1 MiB/s, 120 objects/s recovering 2023-09-23T20:30:32.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:32 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1598699310' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T20:30:32.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:32 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T20:30:32.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:32 smithi018 ceph-mon[179647]: pgmap v629: 119 pgs: 2 active+recovery_wait+degraded, 3 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 2.0 GiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 619 op/s; 34671/98508 objects degraded (35.196%); 5522/98508 objects misplaced (5.606%); 7.1 MiB/s, 120 objects/s recovering 2023-09-23T20:30:32.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:32 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1598699310' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T20:30:32.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:32 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T20:30:32.669 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:32.730 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:30:32.737 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:32.737 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:30:32.737 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:32 smithi149 ceph-mon[170394]: pgmap v629: 119 pgs: 2 active+recovery_wait+degraded, 3 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 2.0 GiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 619 op/s; 34671/98508 objects degraded (35.196%); 5522/98508 objects misplaced (5.606%); 7.1 MiB/s, 120 objects/s recovering 2023-09-23T20:30:32.737 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:32 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1598699310' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T20:30:32.738 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:32 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T20:30:32.838 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:30:32.944 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:33.766 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:30:33.777 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:33 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T20:30:33.777 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:33 smithi018 ceph-mon[184144]: osdmap e281: 8 total, 8 up, 7 in 2023-09-23T20:30:33.777 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:33 smithi018 ceph-mon[184144]: failed to encode map e281 with expected crc 2023-09-23T20:30:33.778 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:33 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T20:30:33.778 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:33 smithi018 ceph-mon[179647]: osdmap e281: 8 total, 8 up, 7 in 2023-09-23T20:30:33.778 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:33 smithi018 ceph-mon[179647]: failed to encode map e281 with expected crc 2023-09-23T20:30:33.786 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:33.787 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:30:33.887 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:30:33.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:33 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T20:30:33.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:33 smithi149 ceph-mon[170394]: osdmap e281: 8 total, 8 up, 7 in 2023-09-23T20:30:33.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:33 smithi149 ceph-mon[170394]: failed to encode map e281 with expected crc 2023-09-23T20:30:33.984 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:34.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:34 smithi018 ceph-mon[179647]: pgmap v631: 119 pgs: 2 active+recovery_wait+degraded, 3 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 1.9 GiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 669 op/s; 34506/94665 objects degraded (36.451%); 4982/94665 objects misplaced (5.263%); 6.5 MiB/s, 164 objects/s recovering 2023-09-23T20:30:34.643 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:34 smithi018 ceph-mon[184144]: pgmap v631: 119 pgs: 2 active+recovery_wait+degraded, 3 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 1.9 GiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 669 op/s; 34506/94665 objects degraded (36.451%); 4982/94665 objects misplaced (5.263%); 6.5 MiB/s, 164 objects/s recovering 2023-09-23T20:30:34.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:34 smithi149 ceph-mon[170394]: pgmap v631: 119 pgs: 2 active+recovery_wait+degraded, 3 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 1.9 GiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 669 op/s; 34506/94665 objects degraded (36.451%); 4982/94665 objects misplaced (5.263%); 6.5 MiB/s, 164 objects/s recovering 2023-09-23T20:30:34.978 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:30:34.985 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd reweight 2 1 2023-09-23T20:30:35.162 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:30:35.174 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:35.174 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:30:35.180 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2023-09-23T20:30:35.180 INFO:teuthology.orchestra.run.smithi018.stdout: "ops": [ 2023-09-23T20:30:35.180 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:35.181 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.45161.0:180150 6.c 6:34aee11a:::benchmark_data_smithi018_270558_object75411:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e281)", 2023-09-23T20:30:35.181 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:30:35.160678+0000", 2023-09-23T20:30:35.181 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.016021385999999999, 2023-09-23T20:30:35.181 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.016055860000000002, 2023-09-23T20:30:35.181 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:30:35.181 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:30:35.181 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:30:35.181 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.45161", 2023-09-23T20:30:35.181 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3795614987", 2023-09-23T20:30:35.181 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 180150 2023-09-23T20:30:35.181 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:35.181 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:30:35.182 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:35.182 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:30:35.182 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:35.160678+0000", 2023-09-23T20:30:35.182 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:30:35.182 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:35.182 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:35.182 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:30:35.182 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:35.160678+0000", 2023-09-23T20:30:35.182 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:30:35.182 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:35.182 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:35.183 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:30:35.183 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:35.160676+0000", 2023-09-23T20:30:35.183 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:30:35.183 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:35.183 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:35.183 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:30:35.183 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:35.160685+0000", 2023-09-23T20:30:35.183 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.4299999999999995e-06 2023-09-23T20:30:35.183 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:35.183 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:35.183 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:30:35.183 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:35.160704+0000", 2023-09-23T20:30:35.184 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.8919999999999998e-05 2023-09-23T20:30:35.184 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:35.184 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:35.184 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:30:35.184 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:35.160715+0000", 2023-09-23T20:30:35.184 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0193e-05 2023-09-23T20:30:35.184 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:35.184 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:35.184 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:30:35.184 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:35.160968+0000", 2023-09-23T20:30:35.184 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000253401 2023-09-23T20:30:35.184 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:35.185 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:35.185 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:30:35.185 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:35.161126+0000", 2023-09-23T20:30:35.185 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00015829200000000001 2023-09-23T20:30:35.185 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:35.185 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:35.185 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 5,6", 2023-09-23T20:30:35.185 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:35.161292+0000", 2023-09-23T20:30:35.185 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000165699 2023-09-23T20:30:35.185 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:35.185 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:35.185 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:30:35.186 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:35.168259+0000", 2023-09-23T20:30:35.186 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.006967249 2023-09-23T20:30:35.186 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:30:35.186 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:30:35.186 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:30:35.186 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:35.186 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:35.186 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.45161.0:180154 6.2 6:4069ed46:::benchmark_data_smithi018_270558_object75415:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e281)", 2023-09-23T20:30:35.186 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:30:35.160734+0000", 2023-09-23T20:30:35.186 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.015964877999999998, 2023-09-23T20:30:35.186 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.016078421999999998, 2023-09-23T20:30:35.186 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:30:35.187 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:30:35.187 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:30:35.187 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.45161", 2023-09-23T20:30:35.187 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3795614987", 2023-09-23T20:30:35.187 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 180154 2023-09-23T20:30:35.187 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:35.187 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:30:35.187 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:35.187 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:30:35.187 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:35.160734+0000", 2023-09-23T20:30:35.187 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:30:35.187 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:35.188 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:35.188 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:30:35.188 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:35.160734+0000", 2023-09-23T20:30:35.188 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:30:35.188 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:35.188 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:35.188 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:30:35.188 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:35.160734+0000", 2023-09-23T20:30:35.188 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:30:35.188 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:35.188 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:35.188 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:30:35.189 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:35.160738+0000", 2023-09-23T20:30:35.189 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.4100000000000001e-06 2023-09-23T20:30:35.189 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:35.189 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:35.189 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:30:35.189 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:35.160750+0000", 2023-09-23T20:30:35.189 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1921e-05 2023-09-23T20:30:35.189 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:35.189 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:35.189 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:30:35.189 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:35.160757+0000", 2023-09-23T20:30:35.189 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.5980000000000002e-06 2023-09-23T20:30:35.189 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:35.190 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:35.190 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:30:35.190 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:35.160836+0000", 2023-09-23T20:30:35.190 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.9859999999999998e-05 2023-09-23T20:30:35.190 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:35.190 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:35.190 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:30:35.190 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:35.161001+0000", 2023-09-23T20:30:35.190 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00016414500000000001 2023-09-23T20:30:35.190 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:35.190 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:35.191 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 2,6", 2023-09-23T20:30:35.191 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:35.161487+0000", 2023-09-23T20:30:35.191 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00048646699999999998 2023-09-23T20:30:35.191 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:35.191 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:35.191 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:30:35.191 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:35.171656+0000", 2023-09-23T20:30:35.191 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.010169104999999999 2023-09-23T20:30:35.191 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:35.191 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:35.191 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:30:35.191 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:35.174256+0000", 2023-09-23T20:30:35.192 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0025998250000000001 2023-09-23T20:30:35.192 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:30:35.192 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:30:35.192 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:30:35.192 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:35.192 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:35.192 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.45161.0:180157 6.6 6:64d47ceb:::benchmark_data_smithi018_270558_object75418:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e281)", 2023-09-23T20:30:35.192 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:30:35.164303+0000", 2023-09-23T20:30:35.192 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.012396312, 2023-09-23T20:30:35.192 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.012626729, 2023-09-23T20:30:35.192 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:30:35.192 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:30:35.193 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:30:35.193 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.45161", 2023-09-23T20:30:35.193 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3795614987", 2023-09-23T20:30:35.193 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 180157 2023-09-23T20:30:35.193 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:35.193 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:30:35.193 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:35.193 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:30:35.193 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:35.164303+0000", 2023-09-23T20:30:35.193 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:30:35.193 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:35.194 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:35.194 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:30:35.194 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:35.164303+0000", 2023-09-23T20:30:35.194 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:30:35.194 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:35.194 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:35.194 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:30:35.194 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:35.164301+0000", 2023-09-23T20:30:35.194 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:30:35.194 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:35.194 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:35.194 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:30:35.195 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:35.164308+0000", 2023-09-23T20:30:35.195 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.7259999999999996e-06 2023-09-23T20:30:35.195 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:35.195 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:35.195 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:30:35.195 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:35.164320+0000", 2023-09-23T20:30:35.195 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2482e-05 2023-09-23T20:30:35.195 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:35.195 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:35.195 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:30:35.195 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:35.164329+0000", 2023-09-23T20:30:35.195 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.4670000000000004e-06 2023-09-23T20:30:35.195 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:35.196 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:35.196 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:30:35.196 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:35.164412+0000", 2023-09-23T20:30:35.196 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.3591000000000007e-05 2023-09-23T20:30:35.196 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:35.196 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:35.196 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:30:35.196 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:35.164555+0000", 2023-09-23T20:30:35.196 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00014232000000000001 2023-09-23T20:30:35.196 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:35.196 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:35.196 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 4,6,7", 2023-09-23T20:30:35.197 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:35.165009+0000", 2023-09-23T20:30:35.197 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00045445199999999999 2023-09-23T20:30:35.197 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:35.197 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:35.197 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:30:35.197 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:35.172118+0000", 2023-09-23T20:30:35.197 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0071088849999999997 2023-09-23T20:30:35.197 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:35.197 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:35.197 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:30:35.197 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:35.174784+0000", 2023-09-23T20:30:35.197 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0026655899999999998 2023-09-23T20:30:35.198 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:30:35.198 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:30:35.198 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:30:35.198 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:35.198 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:35.198 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.45161.0:180159 6.c 6:3ef333c4:::benchmark_data_smithi018_270558_object75420:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e281)", 2023-09-23T20:30:35.198 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:30:35.164347+0000", 2023-09-23T20:30:35.198 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.012352429, 2023-09-23T20:30:35.198 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.012678178999999999, 2023-09-23T20:30:35.198 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:30:35.198 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:30:35.199 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:30:35.199 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.45161", 2023-09-23T20:30:35.199 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3795614987", 2023-09-23T20:30:35.199 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 180159 2023-09-23T20:30:35.199 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:35.199 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:30:35.199 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:35.199 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:30:35.199 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:35.164347+0000", 2023-09-23T20:30:35.199 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:30:35.199 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:35.199 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:35.200 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:30:35.200 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:35.164347+0000", 2023-09-23T20:30:35.200 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:30:35.200 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:35.200 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:35.200 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:30:35.200 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:35.164346+0000", 2023-09-23T20:30:35.200 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:30:35.200 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:35.200 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:35.200 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:30:35.200 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:35.164350+0000", 2023-09-23T20:30:35.201 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.3670000000000001e-06 2023-09-23T20:30:35.201 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:35.201 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:35.201 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:30:35.201 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:35.164356+0000", 2023-09-23T20:30:35.201 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.703e-06 2023-09-23T20:30:35.201 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:35.201 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:35.202 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:30:35.202 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:35.164362+0000", 2023-09-23T20:30:35.202 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.8189999999999997e-06 2023-09-23T20:30:35.202 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:35.202 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:35.202 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:30:35.202 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:35.168284+0000", 2023-09-23T20:30:35.202 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0039223900000000004 2023-09-23T20:30:35.202 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:35.202 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:35.202 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:30:35.202 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:35.168430+0000", 2023-09-23T20:30:35.203 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0001459 2023-09-23T20:30:35.203 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:35.203 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:35.203 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 5,6", 2023-09-23T20:30:35.203 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:35.168576+0000", 2023-09-23T20:30:35.203 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000146098 2023-09-23T20:30:35.203 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:30:35.203 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:30:35.203 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:30:35.203 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:35.203 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:35.204 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.45161.0:180161 6.c 6:30969fac:::benchmark_data_smithi018_270558_object75422:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e281)", 2023-09-23T20:30:35.204 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:30:35.171993+0000", 2023-09-23T20:30:35.204 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.0047057840000000002, 2023-09-23T20:30:35.204 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0051168979999999999, 2023-09-23T20:30:35.204 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:30:35.204 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:30:35.204 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:30:35.204 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.45161", 2023-09-23T20:30:35.204 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3795614987", 2023-09-23T20:30:35.204 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 180161 2023-09-23T20:30:35.204 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:35.204 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:30:35.205 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:35.205 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:30:35.205 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:35.171993+0000", 2023-09-23T20:30:35.205 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:30:35.205 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:35.205 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:35.205 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:30:35.205 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:35.171993+0000", 2023-09-23T20:30:35.205 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:30:35.205 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:35.205 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:35.206 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:30:35.206 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:35.171991+0000", 2023-09-23T20:30:35.206 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:30:35.206 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:35.206 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:35.206 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:30:35.206 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:35.171999+0000", 2023-09-23T20:30:35.206 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.2189999999999998e-06 2023-09-23T20:30:35.206 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:35.206 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:35.206 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:30:35.206 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:35.172015+0000", 2023-09-23T20:30:35.207 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.6616000000000002e-05 2023-09-23T20:30:35.207 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:35.207 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:35.207 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:30:35.207 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:35.172024+0000", 2023-09-23T20:30:35.207 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.5920000000000001e-06 2023-09-23T20:30:35.207 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:35.207 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:35.207 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:30:35.207 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:35.172092+0000", 2023-09-23T20:30:35.207 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.8641999999999998e-05 2023-09-23T20:30:35.207 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:35.208 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:35.208 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:30:35.208 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:35.172265+0000", 2023-09-23T20:30:35.208 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000172127 2023-09-23T20:30:35.208 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:35.208 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:35.208 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 5,6", 2023-09-23T20:30:35.208 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:35.172382+0000", 2023-09-23T20:30:35.208 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000117414 2023-09-23T20:30:35.208 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:30:35.208 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:30:35.208 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:30:35.209 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:35.209 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:35.209 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_repop(client.45161.0:180158 6.f e281/259 6:f6a6612b:::benchmark_data_smithi018_270558_object75419:head v 281'11394, mlcod=281'11390)", 2023-09-23T20:30:35.209 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:30:35.174132+0000", 2023-09-23T20:30:35.209 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.0025675210000000001, 2023-09-23T20:30:35.209 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0030304419999999999, 2023-09-23T20:30:35.209 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:30:35.209 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "started", 2023-09-23T20:30:35.209 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:30:35.209 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:35.209 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:30:35.210 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:35.174132+0000", 2023-09-23T20:30:35.210 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:30:35.210 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:35.210 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:35.210 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:30:35.210 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:35.174132+0000", 2023-09-23T20:30:35.210 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:30:35.210 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:35.210 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:35.210 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:30:35.210 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:35.174131+0000", 2023-09-23T20:30:35.210 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:30:35.211 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:35.211 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:35.211 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:30:35.211 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:35.174137+0000", 2023-09-23T20:30:35.211 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.852e-06 2023-09-23T20:30:35.211 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:35.211 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:35.211 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:30:35.211 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:35.174147+0000", 2023-09-23T20:30:35.211 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.8099999999999992e-06 2023-09-23T20:30:35.211 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:35.211 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:35.211 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:30:35.212 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:35.174153+0000", 2023-09-23T20:30:35.212 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.1290000000000001e-06 2023-09-23T20:30:35.212 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:35.212 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:35.212 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:30:35.212 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:35.174866+0000", 2023-09-23T20:30:35.212 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00071259199999999996 2023-09-23T20:30:35.212 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:30:35.212 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:30:35.212 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:30:35.212 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:30:35.174890+0000", 2023-09-23T20:30:35.212 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.4752e-05 2023-09-23T20:30:35.213 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:30:35.213 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:30:35.213 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:30:35.213 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:30:35.213 INFO:teuthology.orchestra.run.smithi018.stdout: ], 2023-09-23T20:30:35.213 INFO:teuthology.orchestra.run.smithi018.stdout: "num_ops": 6 2023-09-23T20:30:35.213 INFO:teuthology.orchestra.run.smithi018.stdout:} 2023-09-23T20:30:35.276 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:30:35.299 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:30:34 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:30:34] "GET /metrics HTTP/1.1" 200 36570 "" "Prometheus/2.43.0" 2023-09-23T20:30:35.407 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:35.517 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:30:35.522 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:35.523 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:30:35.624 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:30:35.721 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:35.894 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T20:30:35.935 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:30:35.947 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:35.948 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:30:36.048 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:30:36.080 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:36.128 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:30:36.134 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:36.134 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:30:36.235 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:30:36.346 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:36.491 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:30:36.497 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:36.498 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:30:36.519 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:30:36.599 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:30:36.695 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:36.807 INFO:teuthology.orchestra.run.smithi018.stderr:noscrub is set 2023-09-23T20:30:36.834 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:36 smithi018 ceph-mon[184144]: pgmap v632: 119 pgs: 2 active+recovery_wait+degraded, 3 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 1.9 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 494 op/s; 34506/92934 objects degraded (37.130%); 4982/92934 objects misplaced (5.361%); 4.8 MiB/s, 128 objects/s recovering 2023-09-23T20:30:36.834 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:36 smithi018 ceph-mon[184144]: 1.11 deep-scrub starts 2023-09-23T20:30:36.834 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:36 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1360126321' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:30:36.834 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:36 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:30:36.834 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:36 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 34506/92934 objects degraded (37.130%), 17 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T20:30:36.839 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:36 smithi018 ceph-mon[179647]: pgmap v632: 119 pgs: 2 active+recovery_wait+degraded, 3 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 1.9 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 494 op/s; 34506/92934 objects degraded (37.130%); 4982/92934 objects misplaced (5.361%); 4.8 MiB/s, 128 objects/s recovering 2023-09-23T20:30:36.840 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:36 smithi018 ceph-mon[179647]: 1.11 deep-scrub starts 2023-09-23T20:30:36.840 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:36 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1360126321' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:30:36.840 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:36 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:30:36.840 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:36 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 34506/92934 objects degraded (37.130%), 17 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T20:30:37.072 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:30:37.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:36 smithi149 ceph-mon[170394]: pgmap v632: 119 pgs: 2 active+recovery_wait+degraded, 3 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 1.9 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 494 op/s; 34506/92934 objects degraded (37.130%); 4982/92934 objects misplaced (5.361%); 4.8 MiB/s, 128 objects/s recovering 2023-09-23T20:30:37.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:36 smithi149 ceph-mon[170394]: 1.11 deep-scrub starts 2023-09-23T20:30:37.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:36 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1360126321' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:30:37.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:36 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:30:37.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:36 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 34506/92934 objects degraded (37.130%), 17 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T20:30:38.144 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:37 smithi018 ceph-mon[184144]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:30:38.145 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:37 smithi018 ceph-mon[184144]: failed to encode map e282 with expected crc 2023-09-23T20:30:38.145 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:37 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:30:38.145 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:37 smithi018 ceph-mon[184144]: osdmap e282: 8 total, 8 up, 7 in 2023-09-23T20:30:38.149 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:37 smithi018 ceph-mon[179647]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:30:38.149 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:37 smithi018 ceph-mon[179647]: failed to encode map e282 with expected crc 2023-09-23T20:30:38.149 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:37 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:30:38.149 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:37 smithi018 ceph-mon[179647]: osdmap e282: 8 total, 8 up, 7 in 2023-09-23T20:30:38.218 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:37 smithi149 ceph-mon[170394]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:30:38.218 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:37 smithi149 ceph-mon[170394]: failed to encode map e282 with expected crc 2023-09-23T20:30:38.218 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:37 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:30:38.218 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:37 smithi149 ceph-mon[170394]: osdmap e282: 8 total, 8 up, 7 in 2023-09-23T20:30:38.758 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:30:38.764 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:38.765 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:30:38.866 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:30:39.013 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:39.015 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:38 smithi018 ceph-mon[184144]: pgmap v634: 119 pgs: 2 active+recovery_wait+degraded, 3 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 1.8 GiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 620 op/s; 34349/88431 objects degraded (38.843%); 4614/88431 objects misplaced (5.218%); 3.8 MiB/s, 113 objects/s recovering 2023-09-23T20:30:39.015 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:38 smithi018 ceph-mon[179647]: pgmap v634: 119 pgs: 2 active+recovery_wait+degraded, 3 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 1.8 GiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 620 op/s; 34349/88431 objects degraded (38.843%); 4614/88431 objects misplaced (5.218%); 3.8 MiB/s, 113 objects/s recovering 2023-09-23T20:30:39.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:38 smithi149 ceph-mon[170394]: pgmap v634: 119 pgs: 2 active+recovery_wait+degraded, 3 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 1.8 GiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 620 op/s; 34349/88431 objects degraded (38.843%); 4614/88431 objects misplaced (5.218%); 3.8 MiB/s, 113 objects/s recovering 2023-09-23T20:30:40.706 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:30:40.720 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:40.720 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:30:40.821 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:30:40.872 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:40.884 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:40 smithi018 ceph-mon[179647]: pgmap v635: 119 pgs: 2 active+recovery_wait+degraded, 3 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 1.7 GiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 538 op/s; 34349/85590 objects degraded (40.132%); 4421/85590 objects misplaced (5.165%); 2.5 MiB/s, 145 objects/s recovering 2023-09-23T20:30:40.884 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:40 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:30:40.884 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:40 smithi018 ceph-mon[184144]: pgmap v635: 119 pgs: 2 active+recovery_wait+degraded, 3 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 1.7 GiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 538 op/s; 34349/85590 objects degraded (40.132%); 4421/85590 objects misplaced (5.165%); 2.5 MiB/s, 145 objects/s recovering 2023-09-23T20:30:40.884 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:40 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:30:40.913 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:30:40.916 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:40.917 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:30:41.017 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:30:41.134 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:41.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:40 smithi149 ceph-mon[170394]: pgmap v635: 119 pgs: 2 active+recovery_wait+degraded, 3 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 1.7 GiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 538 op/s; 34349/85590 objects degraded (40.132%); 4421/85590 objects misplaced (5.165%); 2.5 MiB/s, 145 objects/s recovering 2023-09-23T20:30:41.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:40 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:30:41.224 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:30:41.296 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:30:41.317 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:30:41.372 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:30:41.414 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:30:41.458 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:30:41.518 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:30:41.562 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:30:41.571 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:30:41.571 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:30:41.572 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:30:41.572 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:30:41.572 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:30:41.572 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:30:41.572 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:30:41.572 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:30:41.869 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:30:41.874 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:41.874 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:30:41.874 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:41 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:30:41.875 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:41 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 34349/85590 objects degraded (40.132%), 17 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T20:30:41.875 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:41 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1576494491' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:30:41.875 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:41 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:30:41.875 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:41 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 34349/85590 objects degraded (40.132%), 17 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T20:30:41.875 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:41 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1576494491' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:30:41.975 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:30:42.128 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:42.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:41 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:30:42.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:41 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 34349/85590 objects degraded (40.132%), 17 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T20:30:42.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:41 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1576494491' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:30:42.285 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:30:42.291 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:42.292 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:30:42.393 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:30:42.424 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:42.474 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:30:42.477 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:42.478 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:30:42.578 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:30:42.693 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:42.806 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:30:42.812 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:42.813 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:30:42.820 INFO:teuthology.orchestra.run.smithi018.stdout:ERROR: (22) Invalid argument 2023-09-23T20:30:42.820 INFO:teuthology.orchestra.run.smithi018.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:30:42.830 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T20:30:42.913 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:30:42.994 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:42 smithi018 ceph-mon[179647]: pgmap v636: 119 pgs: 2 active+recovery_wait+degraded, 3 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 1.6 GiB data, 7.3 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 619 op/s; 34349/80298 objects degraded (42.777%); 4421/80298 objects misplaced (5.506%); 1.1 MiB/s, 108 objects/s recovering 2023-09-23T20:30:42.994 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:42 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:30:42.994 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:42 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:30:42.994 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:42 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:30:42.994 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:42 smithi018 ceph-mon[184144]: pgmap v636: 119 pgs: 2 active+recovery_wait+degraded, 3 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 1.6 GiB data, 7.3 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 619 op/s; 34349/80298 objects degraded (42.777%); 4421/80298 objects misplaced (5.506%); 1.1 MiB/s, 108 objects/s recovering 2023-09-23T20:30:42.995 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:42 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:30:42.995 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:42 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:30:42.995 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:42 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:30:43.059 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:43.215 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:30:43.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:42 smithi149 ceph-mon[170394]: pgmap v636: 119 pgs: 2 active+recovery_wait+degraded, 3 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 1.6 GiB data, 7.3 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 619 op/s; 34349/80298 objects degraded (42.777%); 4421/80298 objects misplaced (5.506%); 1.1 MiB/s, 108 objects/s recovering 2023-09-23T20:30:43.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:42 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:30:43.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:42 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:30:43.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:42 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:30:43.230 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:43.230 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:30:43.292 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T20:30:43.331 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:30:43.361 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:43.408 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:30:43.414 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:43.415 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:30:43.515 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:30:43.567 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:43.618 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:30:43.628 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:43.629 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:30:43.665 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:30:43.730 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:30:43.826 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:43.921 INFO:teuthology.orchestra.run.smithi018.stderr:reweighted osd.2 to 1 (10000) 2023-09-23T20:30:44.074 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:30:44.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:43 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2361380465' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T20:30:44.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:43 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T20:30:44.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:43 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2361380465' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T20:30:44.144 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:43 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T20:30:44.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:43 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2361380465' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T20:30:44.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:43 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T20:30:44.448 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:30:44.458 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:44.458 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:30:44.559 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:30:44.611 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:44.653 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:30:44.660 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:44.660 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:30:44.761 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:30:44.801 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:44.854 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:30:44.862 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:44.863 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:30:44.963 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:30:45.038 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:45.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:44 smithi149 ceph-mon[170394]: pgmap v637: 119 pgs: 2 active+recovery_wait+degraded, 3 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 1.6 GiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 608 op/s; 33715/76395 objects degraded (44.132%); 3872/76395 objects misplaced (5.068%); 2.0 MiB/s, 127 objects/s recovering 2023-09-23T20:30:45.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:44 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T20:30:45.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:44 smithi149 ceph-mon[170394]: osdmap e283: 8 total, 8 up, 7 in 2023-09-23T20:30:45.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:44 smithi149 ceph-mon[170394]: failed to encode map e283 with expected crc 2023-09-23T20:30:45.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:44 smithi149 ceph-mon[170394]: failed to encode map e283 with expected crc 2023-09-23T20:30:45.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:44 smithi149 ceph-mon[170394]: failed to encode map e283 with expected crc 2023-09-23T20:30:45.218 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:44 smithi149 ceph-mon[170394]: failed to encode map e283 with expected crc 2023-09-23T20:30:45.218 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:44 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:30:45.218 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:44 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:30:45.226 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:44 smithi018 ceph-mon[179647]: pgmap v637: 119 pgs: 2 active+recovery_wait+degraded, 3 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 1.6 GiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 608 op/s; 33715/76395 objects degraded (44.132%); 3872/76395 objects misplaced (5.068%); 2.0 MiB/s, 127 objects/s recovering 2023-09-23T20:30:45.226 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:44 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T20:30:45.227 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:44 smithi018 ceph-mon[179647]: osdmap e283: 8 total, 8 up, 7 in 2023-09-23T20:30:45.227 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:44 smithi018 ceph-mon[179647]: failed to encode map e283 with expected crc 2023-09-23T20:30:45.227 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:44 smithi018 ceph-mon[179647]: failed to encode map e283 with expected crc 2023-09-23T20:30:45.227 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:44 smithi018 ceph-mon[179647]: failed to encode map e283 with expected crc 2023-09-23T20:30:45.227 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:44 smithi018 ceph-mon[179647]: failed to encode map e283 with expected crc 2023-09-23T20:30:45.227 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:44 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:30:45.227 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:44 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:30:45.227 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:30:44 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:30:44] "GET /metrics HTTP/1.1" 200 36570 "" "Prometheus/2.43.0" 2023-09-23T20:30:45.228 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:44 smithi018 ceph-mon[184144]: pgmap v637: 119 pgs: 2 active+recovery_wait+degraded, 3 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 13 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 1.6 GiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 608 op/s; 33715/76395 objects degraded (44.132%); 3872/76395 objects misplaced (5.068%); 2.0 MiB/s, 127 objects/s recovering 2023-09-23T20:30:45.228 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:44 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T20:30:45.228 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:44 smithi018 ceph-mon[184144]: osdmap e283: 8 total, 8 up, 7 in 2023-09-23T20:30:45.228 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:44 smithi018 ceph-mon[184144]: failed to encode map e283 with expected crc 2023-09-23T20:30:45.228 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:44 smithi018 ceph-mon[184144]: failed to encode map e283 with expected crc 2023-09-23T20:30:45.228 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:44 smithi018 ceph-mon[184144]: failed to encode map e283 with expected crc 2023-09-23T20:30:45.228 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:44 smithi018 ceph-mon[184144]: failed to encode map e283 with expected crc 2023-09-23T20:30:45.228 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:44 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:30:45.228 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:44 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:30:46.831 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:30:46.896 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:30:46.905 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:46.906 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:30:46.929 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd reweight 6 1 2023-09-23T20:30:47.007 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:30:47.062 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:47.108 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:30:47.115 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:47.115 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:30:47.174 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:46 smithi018 ceph-mon[179647]: pgmap v639: 119 pgs: 2 active+recovery_wait+degraded, 3 active+recovery_wait, 3 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 12 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 1.5 GiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 559 op/s; 31655/74649 objects degraded (42.405%); 4521/74649 objects misplaced (6.056%); 2.6 MiB/s, 146 objects/s recovering 2023-09-23T20:30:47.174 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:46 smithi018 ceph-mon[179647]: osdmap e284: 8 total, 8 up, 7 in 2023-09-23T20:30:47.174 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:46 smithi018 ceph-mon[179647]: failed to encode map e284 with expected crc 2023-09-23T20:30:47.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:46 smithi018 ceph-mon[179647]: failed to encode map e284 with expected crc 2023-09-23T20:30:47.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:46 smithi018 ceph-mon[179647]: failed to encode map e284 with expected crc 2023-09-23T20:30:47.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:46 smithi018 ceph-mon[179647]: failed to encode map e284 with expected crc 2023-09-23T20:30:47.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:46 smithi018 ceph-mon[179647]: failed to encode map e284 with expected crc 2023-09-23T20:30:47.175 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:46 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 31655/74649 objects degraded (42.405%), 17 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T20:30:47.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:46 smithi018 ceph-mon[184144]: pgmap v639: 119 pgs: 2 active+recovery_wait+degraded, 3 active+recovery_wait, 3 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 12 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 1.5 GiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 559 op/s; 31655/74649 objects degraded (42.405%); 4521/74649 objects misplaced (6.056%); 2.6 MiB/s, 146 objects/s recovering 2023-09-23T20:30:47.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:46 smithi018 ceph-mon[184144]: osdmap e284: 8 total, 8 up, 7 in 2023-09-23T20:30:47.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:46 smithi018 ceph-mon[184144]: failed to encode map e284 with expected crc 2023-09-23T20:30:47.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:46 smithi018 ceph-mon[184144]: failed to encode map e284 with expected crc 2023-09-23T20:30:47.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:46 smithi018 ceph-mon[184144]: failed to encode map e284 with expected crc 2023-09-23T20:30:47.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:46 smithi018 ceph-mon[184144]: failed to encode map e284 with expected crc 2023-09-23T20:30:47.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:46 smithi018 ceph-mon[184144]: failed to encode map e284 with expected crc 2023-09-23T20:30:47.176 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:46 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 31655/74649 objects degraded (42.405%), 17 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T20:30:47.216 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:30:47.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:46 smithi149 ceph-mon[170394]: pgmap v639: 119 pgs: 2 active+recovery_wait+degraded, 3 active+recovery_wait, 3 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 12 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 1.5 GiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 559 op/s; 31655/74649 objects degraded (42.405%); 4521/74649 objects misplaced (6.056%); 2.6 MiB/s, 146 objects/s recovering 2023-09-23T20:30:47.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:46 smithi149 ceph-mon[170394]: osdmap e284: 8 total, 8 up, 7 in 2023-09-23T20:30:47.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:46 smithi149 ceph-mon[170394]: failed to encode map e284 with expected crc 2023-09-23T20:30:47.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:46 smithi149 ceph-mon[170394]: failed to encode map e284 with expected crc 2023-09-23T20:30:47.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:46 smithi149 ceph-mon[170394]: failed to encode map e284 with expected crc 2023-09-23T20:30:47.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:46 smithi149 ceph-mon[170394]: failed to encode map e284 with expected crc 2023-09-23T20:30:47.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:46 smithi149 ceph-mon[170394]: failed to encode map e284 with expected crc 2023-09-23T20:30:47.218 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:46 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 31655/74649 objects degraded (42.405%), 17 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T20:30:47.342 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:47.419 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:30:47.441 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:47.442 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:30:47.543 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:30:47.583 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:47.631 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:30:47.635 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:47.635 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:30:47.736 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:30:47.791 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:47.844 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:30:47.852 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:47.853 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:30:47.953 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:30:48.002 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:48.051 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:30:48.071 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:30:48.075 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:48.076 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:30:48.144 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:47 smithi018 ceph-mon[179647]: osdmap e285: 8 total, 8 up, 7 in 2023-09-23T20:30:48.144 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:47 smithi018 ceph-mon[179647]: failed to encode map e285 with expected crc 2023-09-23T20:30:48.144 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:47 smithi018 ceph-mon[179647]: failed to encode map e285 with expected crc 2023-09-23T20:30:48.144 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:47 smithi018 ceph-mon[179647]: failed to encode map e285 with expected crc 2023-09-23T20:30:48.144 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:47 smithi018 ceph-mon[179647]: failed to encode map e285 with expected crc 2023-09-23T20:30:48.144 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:47 smithi018 ceph-mon[179647]: failed to encode map e285 with expected crc 2023-09-23T20:30:48.144 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:47 smithi018 ceph-mon[179647]: failed to encode map e285 with expected crc 2023-09-23T20:30:48.146 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:47 smithi018 ceph-mon[184144]: osdmap e285: 8 total, 8 up, 7 in 2023-09-23T20:30:48.146 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:47 smithi018 ceph-mon[184144]: failed to encode map e285 with expected crc 2023-09-23T20:30:48.146 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:47 smithi018 ceph-mon[184144]: failed to encode map e285 with expected crc 2023-09-23T20:30:48.147 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:47 smithi018 ceph-mon[184144]: failed to encode map e285 with expected crc 2023-09-23T20:30:48.147 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:47 smithi018 ceph-mon[184144]: failed to encode map e285 with expected crc 2023-09-23T20:30:48.147 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:47 smithi018 ceph-mon[184144]: failed to encode map e285 with expected crc 2023-09-23T20:30:48.147 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:47 smithi018 ceph-mon[184144]: failed to encode map e285 with expected crc 2023-09-23T20:30:48.176 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:30:48.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:47 smithi149 ceph-mon[170394]: osdmap e285: 8 total, 8 up, 7 in 2023-09-23T20:30:48.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:47 smithi149 ceph-mon[170394]: failed to encode map e285 with expected crc 2023-09-23T20:30:48.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:47 smithi149 ceph-mon[170394]: failed to encode map e285 with expected crc 2023-09-23T20:30:48.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:47 smithi149 ceph-mon[170394]: failed to encode map e285 with expected crc 2023-09-23T20:30:48.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:47 smithi149 ceph-mon[170394]: failed to encode map e285 with expected crc 2023-09-23T20:30:48.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:47 smithi149 ceph-mon[170394]: failed to encode map e285 with expected crc 2023-09-23T20:30:48.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:47 smithi149 ceph-mon[170394]: failed to encode map e285 with expected crc 2023-09-23T20:30:48.250 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:48.413 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:30:48.419 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:30:48.425 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:48.426 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:30:48.526 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:30:48.582 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:48.633 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:30:48.637 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:48.637 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:30:48.738 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:30:48.837 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:48.983 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:30:48.989 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:48.989 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:30:49.018 INFO:teuthology.orchestra.run.smithi018.stderr:nodeep-scrub is set 2023-09-23T20:30:49.090 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:30:49.221 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:49.291 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:30:49.295 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:49 smithi149 ceph-mon[170394]: pgmap v642: 119 pgs: 2 active+recovery_wait+degraded, 3 active+recovery_wait, 4 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 1.4 GiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 583 op/s; 30399/69792 objects degraded (43.557%); 4831/69792 objects misplaced (6.922%); 4.7 MiB/s, 162 objects/s recovering 2023-09-23T20:30:49.296 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:49 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2214416459' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:30:49.296 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:49 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:30:49.296 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:49.297 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:30:49.321 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:49 smithi018 ceph-mon[184144]: pgmap v642: 119 pgs: 2 active+recovery_wait+degraded, 3 active+recovery_wait, 4 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 1.4 GiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 583 op/s; 30399/69792 objects degraded (43.557%); 4831/69792 objects misplaced (6.922%); 4.7 MiB/s, 162 objects/s recovering 2023-09-23T20:30:49.322 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:49 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2214416459' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:30:49.322 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:49 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:30:49.324 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:49 smithi018 ceph-mon[179647]: pgmap v642: 119 pgs: 2 active+recovery_wait+degraded, 3 active+recovery_wait, 4 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 1.4 GiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 583 op/s; 30399/69792 objects degraded (43.557%); 4831/69792 objects misplaced (6.922%); 4.7 MiB/s, 162 objects/s recovering 2023-09-23T20:30:49.324 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:49 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2214416459' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:30:49.324 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:49 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:30:49.398 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:30:49.513 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:49.574 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:30:49.579 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:49.580 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:30:49.681 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:30:49.783 INFO:teuthology.orchestra.run.smithi018.stdout:ERROR: (22) Invalid argument 2023-09-23T20:30:49.783 INFO:teuthology.orchestra.run.smithi018.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:30:49.833 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:50.023 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:30:50.038 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:50.038 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:30:50.139 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:30:50.208 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:50.283 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:50 smithi018 ceph-mon[179647]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:30:50.284 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:50 smithi018 ceph-mon[179647]: failed to encode map e286 with expected crc 2023-09-23T20:30:50.284 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:50 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:30:50.284 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:50 smithi018 ceph-mon[179647]: osdmap e286: 8 total, 8 up, 7 in 2023-09-23T20:30:50.286 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:50 smithi018 ceph-mon[184144]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:30:50.286 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:50 smithi018 ceph-mon[184144]: failed to encode map e286 with expected crc 2023-09-23T20:30:50.286 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:50 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:30:50.287 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:50 smithi018 ceph-mon[184144]: osdmap e286: 8 total, 8 up, 7 in 2023-09-23T20:30:50.339 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:30:50.346 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:50.346 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:30:50.447 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:30:50.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:50 smithi149 ceph-mon[170394]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:30:50.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:50 smithi149 ceph-mon[170394]: failed to encode map e286 with expected crc 2023-09-23T20:30:50.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:50 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:30:50.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:50 smithi149 ceph-mon[170394]: osdmap e286: 8 total, 8 up, 7 in 2023-09-23T20:30:50.548 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:50.597 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:30:50.605 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:50.605 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:30:50.641 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T20:30:50.706 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:30:50.827 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:50.969 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:30:50.975 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:50.976 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:30:51.077 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:30:51.145 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:30:51.176 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:30:51.213 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:30:51.226 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:51.254 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:30:51.288 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:30:51.298 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:51.298 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:30:51.298 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:51 smithi149 ceph-mon[170394]: pgmap v644: 119 pgs: 1 active+recovery_wait+degraded, 3 active+recovery_wait, 3 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 1.4 GiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 555 op/s; 30188/66717 objects degraded (45.248%); 4661/66717 objects misplaced (6.986%); 3.1 MiB/s, 85 objects/s recovering 2023-09-23T20:30:51.308 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:30:51.349 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:30:51.394 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:51 smithi018 ceph-mon[179647]: pgmap v644: 119 pgs: 1 active+recovery_wait+degraded, 3 active+recovery_wait, 3 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 1.4 GiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 555 op/s; 30188/66717 objects degraded (45.248%); 4661/66717 objects misplaced (6.986%); 3.1 MiB/s, 85 objects/s recovering 2023-09-23T20:30:51.394 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:51 smithi018 ceph-mon[184144]: pgmap v644: 119 pgs: 1 active+recovery_wait+degraded, 3 active+recovery_wait, 3 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 1.4 GiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 555 op/s; 30188/66717 objects degraded (45.248%); 4661/66717 objects misplaced (6.986%); 3.1 MiB/s, 85 objects/s recovering 2023-09-23T20:30:51.398 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:30:51.403 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:30:51.453 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:30:51.459 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:30:51.459 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:30:51.459 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:30:51.459 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:30:51.459 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:30:51.460 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:30:51.460 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:30:51.460 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:30:51.462 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:30:51.487 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:51.592 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:30:51.598 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:51.598 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:30:51.680 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T20:30:51.699 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:30:51.780 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:52.054 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:52 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 30188/66717 objects degraded (45.248%), 15 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2023-09-23T20:30:52.054 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:52 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/372972248' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:30:52.054 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:52 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/4038645023' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T20:30:52.058 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:52 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 30188/66717 objects degraded (45.248%), 15 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2023-09-23T20:30:52.058 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:52 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/372972248' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:30:52.058 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:52 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/4038645023' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T20:30:52.128 INFO:teuthology.orchestra.run.smithi018.stderr:reweighted osd.6 to 1 (10000) 2023-09-23T20:30:52.372 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:30:52.376 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:52.377 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:30:52.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:52 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 30188/66717 objects degraded (45.248%), 15 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2023-09-23T20:30:52.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:52 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/372972248' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:30:52.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:52 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/4038645023' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T20:30:52.478 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:30:52.586 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:53.203 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:30:53.210 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:53.211 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:30:53.312 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:30:53.371 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:53 smithi149 ceph-mon[170394]: pgmap v645: 119 pgs: 1 active+recovery_wait+degraded, 3 active+recovery_wait, 3 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 1.3 GiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 731 op/s; 30188/61455 objects degraded (49.122%); 3747/61455 objects misplaced (6.097%); 12 MiB/s, 228 objects/s recovering 2023-09-23T20:30:53.371 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:53 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/4038645023' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T20:30:53.371 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:53 smithi149 ceph-mon[170394]: osdmap e287: 8 total, 8 up, 7 in 2023-09-23T20:30:53.371 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:53 smithi149 ceph-mon[170394]: failed to encode map e287 with expected crc 2023-09-23T20:30:53.382 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:53.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:53 smithi018 ceph-mon[184144]: pgmap v645: 119 pgs: 1 active+recovery_wait+degraded, 3 active+recovery_wait, 3 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 1.3 GiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 731 op/s; 30188/61455 objects degraded (49.122%); 3747/61455 objects misplaced (6.097%); 12 MiB/s, 228 objects/s recovering 2023-09-23T20:30:53.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:53 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/4038645023' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T20:30:53.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:53 smithi018 ceph-mon[184144]: osdmap e287: 8 total, 8 up, 7 in 2023-09-23T20:30:53.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:53 smithi018 ceph-mon[184144]: failed to encode map e287 with expected crc 2023-09-23T20:30:53.394 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:53 smithi018 ceph-mon[179647]: pgmap v645: 119 pgs: 1 active+recovery_wait+degraded, 3 active+recovery_wait, 3 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 1.3 GiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 731 op/s; 30188/61455 objects degraded (49.122%); 3747/61455 objects misplaced (6.097%); 12 MiB/s, 228 objects/s recovering 2023-09-23T20:30:53.394 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:53 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/4038645023' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T20:30:53.394 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:53 smithi018 ceph-mon[179647]: osdmap e287: 8 total, 8 up, 7 in 2023-09-23T20:30:53.394 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:53 smithi018 ceph-mon[179647]: failed to encode map e287 with expected crc 2023-09-23T20:30:53.503 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:30:53.507 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:53.508 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:30:53.609 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:30:53.699 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:54.481 INFO:teuthology.orchestra.run.smithi149.stdout:{ 2023-09-23T20:30:54.481 INFO:teuthology.orchestra.run.smithi149.stdout: "ops": [ 2023-09-23T20:30:54.481 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:30:54.481 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_repop(client.45161.0:192465 6.f e287/259 6:f9ee98c1:::benchmark_data_smithi018_270558_object87726:head v 287'12157, mlcod=287'12154)", 2023-09-23T20:30:54.481 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:30:54.477366+0000", 2023-09-23T20:30:54.481 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 0.0028379830000000001, 2023-09-23T20:30:54.482 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0028630600000000002, 2023-09-23T20:30:54.482 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:30:54.482 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "started", 2023-09-23T20:30:54.482 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:30:54.482 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:30:54.482 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:30:54.482 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:54.477366+0000", 2023-09-23T20:30:54.483 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:30:54.483 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:30:54.483 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:30:54.483 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:30:54.483 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:54.477366+0000", 2023-09-23T20:30:54.483 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:30:54.483 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:30:54.484 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:30:54.484 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:30:54.484 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:54.477365+0000", 2023-09-23T20:30:54.484 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:30:54.484 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:30:54.484 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:30:54.484 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:30:54.485 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:54.477372+0000", 2023-09-23T20:30:54.485 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.584e-06 2023-09-23T20:30:54.485 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:30:54.485 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:30:54.485 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:30:54.485 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:54.477383+0000", 2023-09-23T20:30:54.485 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1491000000000001e-05 2023-09-23T20:30:54.486 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:30:54.486 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:30:54.486 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:30:54.486 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:54.477388+0000", 2023-09-23T20:30:54.486 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.8500000000000002e-06 2023-09-23T20:30:54.486 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:30:54.486 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:30:54.487 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:30:54.487 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:54.477424+0000", 2023-09-23T20:30:54.487 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.5361000000000002e-05 2023-09-23T20:30:54.487 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:30:54.487 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:30:54.487 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:30:54.487 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:54.477443+0000", 2023-09-23T20:30:54.488 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.9565999999999999e-05 2023-09-23T20:30:54.488 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:30:54.488 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:30:54.488 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:30:54.488 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:30:54.488 INFO:teuthology.orchestra.run.smithi149.stdout: ], 2023-09-23T20:30:54.488 INFO:teuthology.orchestra.run.smithi149.stdout: "num_ops": 1 2023-09-23T20:30:54.489 INFO:teuthology.orchestra.run.smithi149.stdout:} 2023-09-23T20:30:55.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:55 smithi149 ceph-mon[170394]: pgmap v647: 119 pgs: 1 active+recovery_wait+degraded, 3 active+recovery_wait, 3 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 1.2 GiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 686 op/s; 30001/57186 objects degraded (52.462%); 3158/57186 objects misplaced (5.522%); 12 MiB/s, 220 objects/s recovering 2023-09-23T20:30:55.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:55 smithi018 ceph-mon[179647]: pgmap v647: 119 pgs: 1 active+recovery_wait+degraded, 3 active+recovery_wait, 3 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 1.2 GiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 686 op/s; 30001/57186 objects degraded (52.462%); 3158/57186 objects misplaced (5.522%); 12 MiB/s, 220 objects/s recovering 2023-09-23T20:30:55.394 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:55 smithi018 ceph-mon[184144]: pgmap v647: 119 pgs: 1 active+recovery_wait+degraded, 3 active+recovery_wait, 3 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 1.2 GiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 686 op/s; 30001/57186 objects degraded (52.462%); 3158/57186 objects misplaced (5.522%); 12 MiB/s, 220 objects/s recovering 2023-09-23T20:30:55.394 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:30:54 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:30:54] "GET /metrics HTTP/1.1" 200 36570 "" "Prometheus/2.43.0" 2023-09-23T20:30:55.561 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-09-23T20:30:55.562 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:30:55.768 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:30:55.774 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:55.775 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:30:55.876 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:30:56.001 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:56.100 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:30:56.107 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:56 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 30001/57186 objects degraded (52.462%), 15 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2023-09-23T20:30:56.107 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:56 smithi018 ceph-mon[179647]: osdmap e288: 8 total, 8 up, 7 in 2023-09-23T20:30:56.114 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:30:56.122 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T20:30:56.130 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:56.131 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:30:56.232 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:30:56.297 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:56.345 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:30:56.348 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:56.349 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:30:56.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:56 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 30001/57186 objects degraded (52.462%), 15 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2023-09-23T20:30:56.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:56 smithi018 ceph-mon[184144]: osdmap e288: 8 total, 8 up, 7 in 2023-09-23T20:30:56.450 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:30:56.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:56 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 30001/57186 objects degraded (52.462%), 15 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2023-09-23T20:30:56.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:56 smithi149 ceph-mon[170394]: osdmap e288: 8 total, 8 up, 7 in 2023-09-23T20:30:56.484 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:56.830 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:30:57.037 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:30:57.039 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:57.040 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:30:57.141 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:30:57.193 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:57.248 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:30:57.252 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:57.253 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:30:57.270 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:30:57.301 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:57 smithi149 ceph-mon[170394]: pgmap v648: 119 pgs: 1 active+recovery_wait+degraded, 3 active+recovery_wait, 3 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 1.1 GiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 606 op/s; 30001/55206 objects degraded (54.344%); 2851/55206 objects misplaced (5.164%); 11 MiB/s, 202 objects/s recovering 2023-09-23T20:30:57.301 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:57 smithi149 ceph-mon[170394]: failed to encode map e288 with expected crc 2023-09-23T20:30:57.301 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:57 smithi149 ceph-mon[170394]: failed to encode map e288 with expected crc 2023-09-23T20:30:57.302 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:57 smithi149 ceph-mon[170394]: failed to encode map e288 with expected crc 2023-09-23T20:30:57.302 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:57 smithi149 ceph-mon[170394]: osdmap e289: 8 total, 8 up, 7 in 2023-09-23T20:30:57.353 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:30:57.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:57 smithi018 ceph-mon[184144]: pgmap v648: 119 pgs: 1 active+recovery_wait+degraded, 3 active+recovery_wait, 3 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 1.1 GiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 606 op/s; 30001/55206 objects degraded (54.344%); 2851/55206 objects misplaced (5.164%); 11 MiB/s, 202 objects/s recovering 2023-09-23T20:30:57.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:57 smithi018 ceph-mon[184144]: failed to encode map e288 with expected crc 2023-09-23T20:30:57.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:57 smithi018 ceph-mon[184144]: failed to encode map e288 with expected crc 2023-09-23T20:30:57.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:57 smithi018 ceph-mon[184144]: failed to encode map e288 with expected crc 2023-09-23T20:30:57.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:57 smithi018 ceph-mon[184144]: osdmap e289: 8 total, 8 up, 7 in 2023-09-23T20:30:57.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:57 smithi018 ceph-mon[179647]: pgmap v648: 119 pgs: 1 active+recovery_wait+degraded, 3 active+recovery_wait, 3 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 1.1 GiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 606 op/s; 30001/55206 objects degraded (54.344%); 2851/55206 objects misplaced (5.164%); 11 MiB/s, 202 objects/s recovering 2023-09-23T20:30:57.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:57 smithi018 ceph-mon[179647]: failed to encode map e288 with expected crc 2023-09-23T20:30:57.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:57 smithi018 ceph-mon[179647]: failed to encode map e288 with expected crc 2023-09-23T20:30:57.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:57 smithi018 ceph-mon[179647]: failed to encode map e288 with expected crc 2023-09-23T20:30:57.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:57 smithi018 ceph-mon[179647]: osdmap e289: 8 total, 8 up, 7 in 2023-09-23T20:30:57.418 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:57.597 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:30:57.601 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:57.601 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:30:57.702 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:30:57.773 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:57.863 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:30:57.901 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:30:57.910 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:57.910 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:30:58.011 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:30:58.069 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:58.182 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:30:58.223 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:58 smithi149 ceph-mon[170394]: failed to encode map e289 with expected crc 2023-09-23T20:30:58.224 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:58.224 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:30:58.265 INFO:teuthology.orchestra.run.smithi149.stdout:{ 2023-09-23T20:30:58.266 INFO:teuthology.orchestra.run.smithi149.stdout: "ops": [], 2023-09-23T20:30:58.266 INFO:teuthology.orchestra.run.smithi149.stdout: "complaint_time": 30, 2023-09-23T20:30:58.266 INFO:teuthology.orchestra.run.smithi149.stdout: "num_blocked_ops": 0 2023-09-23T20:30:58.266 INFO:teuthology.orchestra.run.smithi149.stdout:} 2023-09-23T20:30:58.326 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:30:58.344 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:58 smithi018 ceph-mon[184144]: failed to encode map e289 with expected crc 2023-09-23T20:30:58.344 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:58 smithi018 ceph-mon[179647]: failed to encode map e289 with expected crc 2023-09-23T20:30:58.371 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:58.657 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T20:30:58.689 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:30:58.693 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:58.694 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:30:58.794 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:30:58.835 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:58.908 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:30:58.916 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:58.916 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:30:59.017 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:30:59.055 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:30:59.063 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:59.106 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:30:59.111 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:59.112 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:30:59.212 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:30:59.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:30:59 smithi149 ceph-mon[170394]: pgmap v651: 119 pgs: 1 active+recovery_wait+degraded, 3 active+recovery_wait, 3 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 1.0 GiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 638 op/s; 29365/49950 objects degraded (58.789%); 2273/49950 objects misplaced (4.551%); 9.2 MiB/s, 255 objects/s recovering 2023-09-23T20:30:59.326 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:59.359 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:30:59 smithi018 ceph-mon[184144]: pgmap v651: 119 pgs: 1 active+recovery_wait+degraded, 3 active+recovery_wait, 3 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 1.0 GiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 638 op/s; 29365/49950 objects degraded (58.789%); 2273/49950 objects misplaced (4.551%); 9.2 MiB/s, 255 objects/s recovering 2023-09-23T20:30:59.361 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:30:59 smithi018 ceph-mon[179647]: pgmap v651: 119 pgs: 1 active+recovery_wait+degraded, 3 active+recovery_wait, 3 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 1.0 GiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 638 op/s; 29365/49950 objects degraded (58.789%); 2273/49950 objects misplaced (4.551%); 9.2 MiB/s, 255 objects/s recovering 2023-09-23T20:30:59.392 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:30:59.489 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:30:59.498 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:59.499 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:30:59.600 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:30:59.640 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:30:59.685 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:30:59.688 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:30:59.689 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:30:59.789 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:30:59.884 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:00.096 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:31:00.103 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:00.104 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:31:00.205 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:31:00.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:00 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:31:00.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:00 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1783567631' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:31:00.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:00 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:31:00.228 INFO:teuthology.orchestra.run.smithi018.stderr:noscrub is unset 2023-09-23T20:31:00.295 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:00.368 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:31:00.379 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:00.379 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:31:00.480 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:31:00.498 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:31:00.504 INFO:teuthology.orchestra.run.smithi018.stderr:dumped all 2023-09-23T20:31:00.532 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:00 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:31:00.532 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:00 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1783567631' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:31:00.532 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:00 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:31:00.536 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:00 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:31:00.536 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:00 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1783567631' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:31:00.537 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:00 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:31:00.538 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:00.605 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:31:00.609 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:00.609 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:31:00.710 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:31:00.806 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:00.846 INFO:teuthology.orchestra.run.smithi149.stdout:{ 2023-09-23T20:31:00.846 INFO:teuthology.orchestra.run.smithi149.stdout: "size": 20, 2023-09-23T20:31:00.846 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 600, 2023-09-23T20:31:00.846 INFO:teuthology.orchestra.run.smithi149.stdout: "ops": [ 2023-09-23T20:31:00.846 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.846 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:28909 5.1 5:825aa99e:::benchmark_data_smithi018_228451_object28908:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e118)", 2023-09-23T20:31:00.846 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:09.981257+0000", 2023-09-23T20:31:00.846 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 470.86183286099998, 2023-09-23T20:31:00.847 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0826966440000001, 2023-09-23T20:31:00.847 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:31:00.847 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:31:00.847 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:31:00.847 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:31:00.847 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:31:00.847 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 28909 2023-09-23T20:31:00.847 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.847 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:31:00.847 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.847 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:31:00.847 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:09.981257+0000", 2023-09-23T20:31:00.848 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:31:00.848 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.848 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.848 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:31:00.848 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:09.981257+0000", 2023-09-23T20:31:00.848 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:31:00.848 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.848 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.848 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:31:00.848 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:09.981255+0000", 2023-09-23T20:31:00.848 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999981 2023-09-23T20:31:00.848 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.849 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.849 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:31:00.849 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:09.981345+0000", 2023-09-23T20:31:00.849 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.9339999999999995e-05 2023-09-23T20:31:00.849 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.849 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.849 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:31:00.849 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:09.981360+0000", 2023-09-23T20:31:00.849 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.5591999999999999e-05 2023-09-23T20:31:00.849 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.849 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.849 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:31:00.850 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:09.981370+0000", 2023-09-23T20:31:00.850 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.6069999999999997e-06 2023-09-23T20:31:00.850 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.850 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.850 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:31:00.850 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:10.015283+0000", 2023-09-23T20:31:00.850 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.033912963999999997 2023-09-23T20:31:00.850 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.850 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.850 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:31:00.850 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:10.015297+0000", 2023-09-23T20:31:00.850 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.4477e-05 2023-09-23T20:31:00.851 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.851 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.851 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:31:00.851 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.054705+0000", 2023-09-23T20:31:00.851 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.03940805 2023-09-23T20:31:00.851 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.851 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.851 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:31:00.851 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.054798+0000", 2023-09-23T20:31:00.851 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.2872000000000001e-05 2023-09-23T20:31:00.851 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.851 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.851 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 3,7", 2023-09-23T20:31:00.852 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.054907+0000", 2023-09-23T20:31:00.852 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000108952 2023-09-23T20:31:00.852 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.852 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.852 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:31:00.852 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.056066+0000", 2023-09-23T20:31:00.852 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0011585460000000001 2023-09-23T20:31:00.852 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.852 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.853 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:31:00.853 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.056406+0000", 2023-09-23T20:31:00.853 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000340064 2023-09-23T20:31:00.853 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.853 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.853 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:31:00.853 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.063876+0000", 2023-09-23T20:31:00.853 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.007470661 2023-09-23T20:31:00.853 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.853 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.854 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:31:00.854 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.063917+0000", 2023-09-23T20:31:00.854 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.1151e-05 2023-09-23T20:31:00.854 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.854 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.854 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:31:00.854 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.063954+0000", 2023-09-23T20:31:00.854 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.6075999999999998e-05 2023-09-23T20:31:00.854 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:31:00.854 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:31:00.854 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:31:00.854 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.855 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.855 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:76669 5.18 5.af594e58 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e147)", 2023-09-23T20:31:00.855 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:58.071853+0000", 2023-09-23T20:31:00.855 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 422.77123625899998, 2023-09-23T20:31:00.855 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.86360499599999996, 2023-09-23T20:31:00.855 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:31:00.855 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:31:00.855 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:31:00.855 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:31:00.855 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:31:00.855 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 76669 2023-09-23T20:31:00.855 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.856 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:31:00.856 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.856 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:31:00.856 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:58.071853+0000", 2023-09-23T20:31:00.856 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:31:00.856 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.856 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.856 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:31:00.856 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:58.071853+0000", 2023-09-23T20:31:00.856 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:31:00.856 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.856 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.857 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:31:00.857 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:58.071853+0000", 2023-09-23T20:31:00.857 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:31:00.857 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.857 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.857 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:31:00.857 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:58.071910+0000", 2023-09-23T20:31:00.857 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.7513000000000003e-05 2023-09-23T20:31:00.857 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.857 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.857 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:31:00.858 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:58.071922+0000", 2023-09-23T20:31:00.858 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.2041e-05 2023-09-23T20:31:00.858 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.858 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.858 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:31:00.858 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:58.071926+0000", 2023-09-23T20:31:00.858 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.2660000000000003e-06 2023-09-23T20:31:00.858 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.858 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.858 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:31:00.859 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:58.071963+0000", 2023-09-23T20:31:00.859 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.6186999999999998e-05 2023-09-23T20:31:00.859 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.859 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.859 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:31:00.859 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:58.071968+0000", 2023-09-23T20:31:00.859 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.9200000000000003e-06 2023-09-23T20:31:00.859 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.859 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.859 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:31:00.859 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:58.935445+0000", 2023-09-23T20:31:00.859 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.86347702999999998 2023-09-23T20:31:00.860 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.860 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.860 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:31:00.860 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:58.935458+0000", 2023-09-23T20:31:00.860 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.3918e-05 2023-09-23T20:31:00.860 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:31:00.860 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:31:00.860 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:31:00.860 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.860 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.860 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:76679 5.18 5.48e67dd8 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e147)", 2023-09-23T20:31:00.861 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:58.103169+0000", 2023-09-23T20:31:00.861 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 422.73992116599999, 2023-09-23T20:31:00.861 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.83523706399999997, 2023-09-23T20:31:00.861 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:31:00.861 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:31:00.861 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:31:00.861 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:31:00.861 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:31:00.861 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 76679 2023-09-23T20:31:00.861 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.861 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:31:00.861 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.862 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:31:00.862 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:58.103169+0000", 2023-09-23T20:31:00.862 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:31:00.862 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.862 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.862 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:31:00.862 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:58.103169+0000", 2023-09-23T20:31:00.862 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:31:00.862 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.862 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.862 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:31:00.862 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:58.103167+0000", 2023-09-23T20:31:00.863 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:31:00.863 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.863 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.863 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:31:00.863 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:58.103270+0000", 2023-09-23T20:31:00.863 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00010306000000000001 2023-09-23T20:31:00.863 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.863 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.863 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:31:00.863 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:58.103287+0000", 2023-09-23T20:31:00.863 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.7147e-05 2023-09-23T20:31:00.864 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.864 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.864 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:31:00.864 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:58.103294+0000", 2023-09-23T20:31:00.864 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.8090000000000004e-06 2023-09-23T20:31:00.864 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.864 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.865 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:31:00.865 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:58.103352+0000", 2023-09-23T20:31:00.865 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.7491000000000002e-05 2023-09-23T20:31:00.866 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.866 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.866 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:31:00.866 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:58.103359+0000", 2023-09-23T20:31:00.866 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.1999999999999997e-06 2023-09-23T20:31:00.866 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.866 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.866 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:31:00.866 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:58.938390+0000", 2023-09-23T20:31:00.866 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.83503123000000001 2023-09-23T20:31:00.867 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.867 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.867 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:31:00.867 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:58.938406+0000", 2023-09-23T20:31:00.867 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.5665e-05 2023-09-23T20:31:00.867 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:31:00.867 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:31:00.867 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:31:00.867 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.867 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.867 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:76873 5.18 5.9a8cb918 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e147)", 2023-09-23T20:31:00.867 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:58.246134+0000", 2023-09-23T20:31:00.868 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 422.59695564100002, 2023-09-23T20:31:00.868 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.69232187000000001, 2023-09-23T20:31:00.868 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:31:00.868 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:31:00.868 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:31:00.868 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:31:00.868 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:31:00.868 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 76873 2023-09-23T20:31:00.868 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.868 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:31:00.868 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.868 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:31:00.869 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:58.246134+0000", 2023-09-23T20:31:00.869 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:31:00.869 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.869 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.869 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:31:00.869 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:58.246134+0000", 2023-09-23T20:31:00.869 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:31:00.869 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.869 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.869 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:31:00.869 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:58.246133+0000", 2023-09-23T20:31:00.870 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:31:00.870 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.870 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.870 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:31:00.870 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:58.246193+0000", 2023-09-23T20:31:00.870 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.0713e-05 2023-09-23T20:31:00.870 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.870 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.870 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:31:00.870 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:58.246209+0000", 2023-09-23T20:31:00.870 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.6067e-05 2023-09-23T20:31:00.871 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.871 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.871 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:31:00.871 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:58.246216+0000", 2023-09-23T20:31:00.871 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.2790000000000004e-06 2023-09-23T20:31:00.871 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.871 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.871 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:31:00.871 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:58.246271+0000", 2023-09-23T20:31:00.871 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.499e-05 2023-09-23T20:31:00.871 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.872 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.872 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:31:00.872 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:58.246280+0000", 2023-09-23T20:31:00.872 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.3619999999999995e-06 2023-09-23T20:31:00.872 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.872 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.872 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:31:00.873 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:58.938442+0000", 2023-09-23T20:31:00.873 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.69216180900000002 2023-09-23T20:31:00.873 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.873 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.873 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:31:00.873 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:58.938456+0000", 2023-09-23T20:31:00.873 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.412e-05 2023-09-23T20:31:00.873 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:31:00.873 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:31:00.873 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:31:00.873 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.873 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.874 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:126867 5.a 5:50925f4d:::benchmark_data_smithi018_228451_object19806:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e164)", 2023-09-23T20:31:00.874 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:25:01.833816+0000", 2023-09-23T20:31:00.874 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 359.00927330100001, 2023-09-23T20:31:00.874 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.442573232, 2023-09-23T20:31:00.874 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:31:00.874 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:31:00.874 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:31:00.874 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:31:00.874 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:31:00.874 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 126867 2023-09-23T20:31:00.874 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.875 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:31:00.875 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.875 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:31:00.875 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.833816+0000", 2023-09-23T20:31:00.875 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:31:00.875 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.875 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.875 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:31:00.875 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.833816+0000", 2023-09-23T20:31:00.875 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:31:00.875 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.876 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.876 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:31:00.876 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.833815+0000", 2023-09-23T20:31:00.876 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:31:00.876 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.876 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.876 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:31:00.876 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.833821+0000", 2023-09-23T20:31:00.876 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.122e-06 2023-09-23T20:31:00.876 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.876 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.877 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:31:00.877 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.833832+0000", 2023-09-23T20:31:00.877 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1338999999999999e-05 2023-09-23T20:31:00.877 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.877 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.877 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:31:00.877 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.833839+0000", 2023-09-23T20:31:00.877 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.0419999999999999e-06 2023-09-23T20:31:00.877 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.877 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.877 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:31:00.878 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.898829+0000", 2023-09-23T20:31:00.878 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.064990508000000002 2023-09-23T20:31:00.879 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.879 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.879 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for degraded object", 2023-09-23T20:31:00.879 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.899653+0000", 2023-09-23T20:31:00.879 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00082397299999999998 2023-09-23T20:31:00.879 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.879 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.879 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:31:00.879 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.945495+0000", 2023-09-23T20:31:00.879 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.045841848999999997 2023-09-23T20:31:00.879 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.880 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.880 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:31:00.880 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.945501+0000", 2023-09-23T20:31:00.880 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.3770000000000001e-06 2023-09-23T20:31:00.880 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.880 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.880 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:31:00.880 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.124574+0000", 2023-09-23T20:31:00.880 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1790724560000001 2023-09-23T20:31:00.880 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.880 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.880 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for degraded object", 2023-09-23T20:31:00.880 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.125110+0000", 2023-09-23T20:31:00.881 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00053609000000000003 2023-09-23T20:31:00.881 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.881 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.881 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:31:00.881 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.258824+0000", 2023-09-23T20:31:00.881 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.133714534 2023-09-23T20:31:00.881 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.881 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.881 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:31:00.881 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.258902+0000", 2023-09-23T20:31:00.881 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.7299000000000006e-05 2023-09-23T20:31:00.881 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.882 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.882 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:31:00.882 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.259000+0000", 2023-09-23T20:31:00.882 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.8595e-05 2023-09-23T20:31:00.882 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.882 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.882 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:31:00.882 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.269120+0000", 2023-09-23T20:31:00.882 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.010120013000000001 2023-09-23T20:31:00.882 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.882 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.882 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:31:00.883 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.272674+0000", 2023-09-23T20:31:00.883 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0035536449999999998 2023-09-23T20:31:00.883 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.883 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.883 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:31:00.883 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.272702+0000", 2023-09-23T20:31:00.883 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.8435000000000001e-05 2023-09-23T20:31:00.883 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.883 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.883 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:31:00.883 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.272739+0000", 2023-09-23T20:31:00.884 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.6727999999999998e-05 2023-09-23T20:31:00.884 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.884 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.884 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:31:00.884 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.276390+0000", 2023-09-23T20:31:00.884 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0036506640000000001 2023-09-23T20:31:00.884 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:31:00.884 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:31:00.884 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:31:00.884 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.884 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.884 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:126894 5.a 5:5259ce05:::benchmark_data_smithi018_228451_object19833:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e164)", 2023-09-23T20:31:00.885 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:25:01.880498+0000", 2023-09-23T20:31:00.885 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 358.96259184500002, 2023-09-23T20:31:00.885 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.3921141399999999, 2023-09-23T20:31:00.885 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:31:00.885 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:31:00.885 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:31:00.885 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:31:00.885 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:31:00.885 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 126894 2023-09-23T20:31:00.885 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.885 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:31:00.886 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.886 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:31:00.886 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.880498+0000", 2023-09-23T20:31:00.886 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:31:00.886 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.886 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.886 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:31:00.886 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.880498+0000", 2023-09-23T20:31:00.886 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:31:00.886 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.886 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.886 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:31:00.887 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.880496+0000", 2023-09-23T20:31:00.887 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:31:00.887 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.887 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.887 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:31:00.887 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.880501+0000", 2023-09-23T20:31:00.887 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.904e-06 2023-09-23T20:31:00.887 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.887 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.887 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:31:00.887 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.880511+0000", 2023-09-23T20:31:00.887 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.5380000000000008e-06 2023-09-23T20:31:00.888 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.888 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.888 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:31:00.888 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.880516+0000", 2023-09-23T20:31:00.888 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.9980000000000003e-06 2023-09-23T20:31:00.888 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.888 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.888 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:31:00.888 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.899727+0000", 2023-09-23T20:31:00.888 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.019211236999999999 2023-09-23T20:31:00.888 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.888 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.889 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for degraded object", 2023-09-23T20:31:00.889 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.900501+0000", 2023-09-23T20:31:00.889 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00077373899999999996 2023-09-23T20:31:00.889 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.889 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.889 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:31:00.889 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.945461+0000", 2023-09-23T20:31:00.889 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.044960444000000002 2023-09-23T20:31:00.889 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.889 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.889 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:31:00.889 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.945472+0000", 2023-09-23T20:31:00.889 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0738e-05 2023-09-23T20:31:00.890 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.890 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.890 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:31:00.890 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.124180+0000", 2023-09-23T20:31:00.890 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1787084809999999 2023-09-23T20:31:00.890 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.890 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.890 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for degraded object", 2023-09-23T20:31:00.890 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.124532+0000", 2023-09-23T20:31:00.890 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00035113800000000003 2023-09-23T20:31:00.891 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.891 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.891 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:31:00.891 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.250481+0000", 2023-09-23T20:31:00.891 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.12594944799999999 2023-09-23T20:31:00.891 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.891 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.891 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:31:00.891 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.250553+0000", 2023-09-23T20:31:00.891 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.1903000000000006e-05 2023-09-23T20:31:00.891 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.891 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.892 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:31:00.892 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.250646+0000", 2023-09-23T20:31:00.892 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.3293000000000003e-05 2023-09-23T20:31:00.892 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.892 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.892 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:31:00.892 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.269045+0000", 2023-09-23T20:31:00.892 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.018399174000000001 2023-09-23T20:31:00.892 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.892 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.892 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:31:00.892 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.269114+0000", 2023-09-23T20:31:00.893 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.8677000000000007e-05 2023-09-23T20:31:00.893 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.893 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.893 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:31:00.893 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.269161+0000", 2023-09-23T20:31:00.893 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.7015999999999998e-05 2023-09-23T20:31:00.893 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.893 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.893 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:31:00.893 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.269214+0000", 2023-09-23T20:31:00.893 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.3128999999999999e-05 2023-09-23T20:31:00.893 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.894 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.894 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:31:00.894 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.272612+0000", 2023-09-23T20:31:00.894 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0033978659999999998 2023-09-23T20:31:00.894 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:31:00.894 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:31:00.894 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:31:00.894 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.894 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.894 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:126925 5.a 5:5e6040ba:::benchmark_data_smithi018_228451_object19864:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e165)", 2023-09-23T20:31:00.894 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:25:01.937985+0000", 2023-09-23T20:31:00.895 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 358.90510471900001, 2023-09-23T20:31:00.895 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.341994892, 2023-09-23T20:31:00.895 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:31:00.895 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:31:00.895 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:31:00.895 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:31:00.895 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:31:00.895 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 126925 2023-09-23T20:31:00.895 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.895 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:31:00.895 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.895 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:31:00.895 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.937985+0000", 2023-09-23T20:31:00.896 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:31:00.896 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.896 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.896 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:31:00.896 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.937985+0000", 2023-09-23T20:31:00.896 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:31:00.896 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.896 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.896 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:31:00.896 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.937983+0000", 2023-09-23T20:31:00.896 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999981 2023-09-23T20:31:00.896 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.897 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.897 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:31:00.897 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.937989+0000", 2023-09-23T20:31:00.897 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.1920000000000002e-06 2023-09-23T20:31:00.897 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.897 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.897 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:31:00.897 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.937998+0000", 2023-09-23T20:31:00.897 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.5250000000000005e-06 2023-09-23T20:31:00.897 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.897 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.897 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:31:00.898 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.938003+0000", 2023-09-23T20:31:00.898 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.2129999999999999e-06 2023-09-23T20:31:00.898 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.898 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.898 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:31:00.898 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.947727+0000", 2023-09-23T20:31:00.898 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0097239900000000001 2023-09-23T20:31:00.898 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.898 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.898 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:31:00.898 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.947739+0000", 2023-09-23T20:31:00.899 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1355000000000001e-05 2023-09-23T20:31:00.899 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.899 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.899 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:31:00.899 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.125146+0000", 2023-09-23T20:31:00.899 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1774076529999999 2023-09-23T20:31:00.899 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.899 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.899 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for degraded object", 2023-09-23T20:31:00.899 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.125724+0000", 2023-09-23T20:31:00.899 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00057758600000000005 2023-09-23T20:31:00.899 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.900 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.900 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:31:00.900 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.266528+0000", 2023-09-23T20:31:00.900 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.14080421800000001 2023-09-23T20:31:00.900 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.900 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.900 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:31:00.900 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.266606+0000", 2023-09-23T20:31:00.900 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.7725999999999994e-05 2023-09-23T20:31:00.900 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.900 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.900 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:31:00.901 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.266716+0000", 2023-09-23T20:31:00.901 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00011035999999999999 2023-09-23T20:31:00.901 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.901 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.901 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:31:00.901 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.276441+0000", 2023-09-23T20:31:00.901 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0097248620000000008 2023-09-23T20:31:00.901 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.901 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.901 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:31:00.901 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.276449+0000", 2023-09-23T20:31:00.901 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.0299999999999994e-06 2023-09-23T20:31:00.902 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.902 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.902 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:31:00.902 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.276482+0000", 2023-09-23T20:31:00.902 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.2796999999999998e-05 2023-09-23T20:31:00.902 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.902 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.902 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:31:00.902 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.276533+0000", 2023-09-23T20:31:00.902 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.1218000000000003e-05 2023-09-23T20:31:00.902 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.902 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.903 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:31:00.903 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.279980+0000", 2023-09-23T20:31:00.903 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0034469639999999998 2023-09-23T20:31:00.903 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:31:00.903 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:31:00.903 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:31:00.903 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.903 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.903 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:126935 5.1 5:8421d65c:::benchmark_data_smithi018_228451_object19874:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e165)", 2023-09-23T20:31:00.903 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:25:01.964401+0000", 2023-09-23T20:31:00.903 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 358.878689227, 2023-09-23T20:31:00.903 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0449979250000001, 2023-09-23T20:31:00.904 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:31:00.904 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:31:00.904 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:31:00.904 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:31:00.904 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:31:00.904 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 126935 2023-09-23T20:31:00.904 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.904 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:31:00.904 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.904 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:31:00.905 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.964401+0000", 2023-09-23T20:31:00.905 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:31:00.905 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.905 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.905 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:31:00.905 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.964401+0000", 2023-09-23T20:31:00.905 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:31:00.905 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.905 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.905 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:31:00.905 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.964398+0000", 2023-09-23T20:31:00.905 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999976 2023-09-23T20:31:00.905 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.906 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.906 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:31:00.906 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.964405+0000", 2023-09-23T20:31:00.906 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.8800000000000002e-06 2023-09-23T20:31:00.906 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.906 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.906 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:31:00.906 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.964423+0000", 2023-09-23T20:31:00.906 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.7864e-05 2023-09-23T20:31:00.906 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.906 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.907 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:31:00.907 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.964429+0000", 2023-09-23T20:31:00.907 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.3910000000000003e-06 2023-09-23T20:31:00.907 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.907 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.907 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:31:00.907 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.970829+0000", 2023-09-23T20:31:00.907 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0063990360000000003 2023-09-23T20:31:00.907 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.907 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.907 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:31:00.907 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.970843+0000", 2023-09-23T20:31:00.908 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.49e-05 2023-09-23T20:31:00.908 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.908 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.908 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:31:00.908 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.002512+0000", 2023-09-23T20:31:00.908 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.031668544 2023-09-23T20:31:00.908 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.908 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.908 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:31:00.908 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.002617+0000", 2023-09-23T20:31:00.908 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00010461 2023-09-23T20:31:00.908 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.909 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.909 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 3,7", 2023-09-23T20:31:00.909 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.002730+0000", 2023-09-23T20:31:00.909 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00011356700000000001 2023-09-23T20:31:00.909 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.909 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.909 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:31:00.909 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.007261+0000", 2023-09-23T20:31:00.909 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0045309240000000004 2023-09-23T20:31:00.909 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.909 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.909 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:31:00.910 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.008570+0000", 2023-09-23T20:31:00.910 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.001309225 2023-09-23T20:31:00.910 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.910 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.910 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:31:00.910 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.009210+0000", 2023-09-23T20:31:00.910 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00063932400000000003 2023-09-23T20:31:00.910 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.910 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.910 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:31:00.910 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.009257+0000", 2023-09-23T20:31:00.911 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.7692999999999999e-05 2023-09-23T20:31:00.911 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.911 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.911 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:31:00.911 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.009398+0000", 2023-09-23T20:31:00.911 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000141123 2023-09-23T20:31:00.911 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:31:00.911 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:31:00.911 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:31:00.911 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.911 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.912 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:126945 5.1 5:856ddac8:::benchmark_data_smithi018_228451_object19884:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e165)", 2023-09-23T20:31:00.912 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:25:01.991896+0000", 2023-09-23T20:31:00.912 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 358.85119413299998, 2023-09-23T20:31:00.912 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0216826400000001, 2023-09-23T20:31:00.912 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:31:00.912 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:31:00.912 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:31:00.912 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:31:00.912 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:31:00.912 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 126945 2023-09-23T20:31:00.912 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.912 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:31:00.913 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.913 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:31:00.913 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.991896+0000", 2023-09-23T20:31:00.913 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:31:00.913 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.913 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.913 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:31:00.913 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.991896+0000", 2023-09-23T20:31:00.913 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:31:00.913 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.913 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.913 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:31:00.914 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.991894+0000", 2023-09-23T20:31:00.914 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:31:00.914 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.914 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.914 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:31:00.914 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.991900+0000", 2023-09-23T20:31:00.914 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.2790000000000004e-06 2023-09-23T20:31:00.914 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.914 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.914 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:31:00.914 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.991914+0000", 2023-09-23T20:31:00.915 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.3226e-05 2023-09-23T20:31:00.915 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.915 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.915 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:31:00.915 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.991920+0000", 2023-09-23T20:31:00.915 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.7710000000000001e-06 2023-09-23T20:31:00.915 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.915 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.915 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:31:00.915 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.991975+0000", 2023-09-23T20:31:00.915 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.5021000000000001e-05 2023-09-23T20:31:00.916 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.916 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.916 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:31:00.916 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.991985+0000", 2023-09-23T20:31:00.916 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.5999999999999996e-06 2023-09-23T20:31:00.916 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.916 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.916 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:31:00.916 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.004845+0000", 2023-09-23T20:31:00.916 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0128602600000001 2023-09-23T20:31:00.916 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.916 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.917 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:31:00.917 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.004955+0000", 2023-09-23T20:31:00.917 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000109419 2023-09-23T20:31:00.917 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.917 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.917 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 3,7", 2023-09-23T20:31:00.917 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.005108+0000", 2023-09-23T20:31:00.917 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00015359099999999999 2023-09-23T20:31:00.917 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.917 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.917 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:31:00.917 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.007360+0000", 2023-09-23T20:31:00.918 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0022521389999999998 2023-09-23T20:31:00.918 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.918 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.918 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:31:00.918 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.008577+0000", 2023-09-23T20:31:00.918 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00121667 2023-09-23T20:31:00.918 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.918 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.918 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:31:00.918 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.013397+0000", 2023-09-23T20:31:00.918 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0048202760000000001 2023-09-23T20:31:00.919 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.919 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.919 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:31:00.919 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.013444+0000", 2023-09-23T20:31:00.919 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.6585000000000002e-05 2023-09-23T20:31:00.919 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.919 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.919 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:31:00.919 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.013578+0000", 2023-09-23T20:31:00.919 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00013424899999999999 2023-09-23T20:31:00.919 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:31:00.919 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:31:00.920 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:31:00.920 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.920 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.920 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:134103 5.18 5:1940be66:::benchmark_data_smithi018_228451_object27042:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e169)", 2023-09-23T20:31:00.920 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:25:14.071252+0000", 2023-09-23T20:31:00.920 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 346.77183765500001, 2023-09-23T20:31:00.920 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0302794790000001, 2023-09-23T20:31:00.920 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:31:00.920 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:31:00.920 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:31:00.920 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:31:00.920 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:31:00.921 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 134103 2023-09-23T20:31:00.921 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.921 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:31:00.921 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.921 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:31:00.921 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.071252+0000", 2023-09-23T20:31:00.921 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:31:00.921 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.921 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.921 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:31:00.921 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.071252+0000", 2023-09-23T20:31:00.921 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:31:00.921 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.922 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.922 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:31:00.922 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.071251+0000", 2023-09-23T20:31:00.922 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:31:00.922 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.922 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.922 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:31:00.922 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.071255+0000", 2023-09-23T20:31:00.922 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.335e-06 2023-09-23T20:31:00.922 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.923 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.923 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:31:00.923 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.071266+0000", 2023-09-23T20:31:00.923 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0402e-05 2023-09-23T20:31:00.923 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.923 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.923 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:31:00.923 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.071272+0000", 2023-09-23T20:31:00.923 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.3609999999999997e-06 2023-09-23T20:31:00.923 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.923 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.923 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:31:00.924 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.071501+0000", 2023-09-23T20:31:00.924 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00022861499999999999 2023-09-23T20:31:00.924 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.924 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.924 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:31:00.924 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.071508+0000", 2023-09-23T20:31:00.924 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.8290000000000008e-06 2023-09-23T20:31:00.924 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.924 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.924 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:31:00.924 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.073848+0000", 2023-09-23T20:31:00.924 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0023397380000001 2023-09-23T20:31:00.925 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.925 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.925 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:31:00.925 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.073991+0000", 2023-09-23T20:31:00.925 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00014334999999999999 2023-09-23T20:31:00.925 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.925 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.925 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 1,6", 2023-09-23T20:31:00.925 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.074073+0000", 2023-09-23T20:31:00.925 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.1357e-05 2023-09-23T20:31:00.925 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.925 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.926 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:31:00.926 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.086054+0000", 2023-09-23T20:31:00.926 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.011981103 2023-09-23T20:31:00.926 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.926 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.926 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:31:00.926 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.101412+0000", 2023-09-23T20:31:00.926 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.015358449999999999 2023-09-23T20:31:00.926 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.926 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.926 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:31:00.927 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.101441+0000", 2023-09-23T20:31:00.927 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.8416999999999999e-05 2023-09-23T20:31:00.927 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.927 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.927 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:31:00.927 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.101492+0000", 2023-09-23T20:31:00.927 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.1372000000000002e-05 2023-09-23T20:31:00.927 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.927 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.927 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:31:00.927 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.101532+0000", 2023-09-23T20:31:00.927 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.9385999999999999e-05 2023-09-23T20:31:00.927 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:31:00.928 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:31:00.928 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:31:00.928 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.928 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.928 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:134131 5.18 5:1c0b9433:::benchmark_data_smithi018_228451_object27070:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e169)", 2023-09-23T20:31:00.928 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:25:14.103862+0000", 2023-09-23T20:31:00.928 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 346.73922814899998, 2023-09-23T20:31:00.928 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.99783719299999996, 2023-09-23T20:31:00.928 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:31:00.928 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:31:00.928 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:31:00.928 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:31:00.928 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:31:00.929 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 134131 2023-09-23T20:31:00.929 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.929 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:31:00.929 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.929 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:31:00.929 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.103862+0000", 2023-09-23T20:31:00.929 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:31:00.929 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.929 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.929 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:31:00.929 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.103862+0000", 2023-09-23T20:31:00.929 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:31:00.930 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.930 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.930 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:31:00.930 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.103861+0000", 2023-09-23T20:31:00.930 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:31:00.930 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.930 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.930 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:31:00.930 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.103864+0000", 2023-09-23T20:31:00.930 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.4109999999999999e-06 2023-09-23T20:31:00.930 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.931 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.931 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:31:00.931 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.103870+0000", 2023-09-23T20:31:00.931 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.0190000000000003e-06 2023-09-23T20:31:00.931 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.931 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.931 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:31:00.931 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.103873+0000", 2023-09-23T20:31:00.931 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.6049999999999998e-06 2023-09-23T20:31:00.931 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.931 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.932 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:31:00.932 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.103936+0000", 2023-09-23T20:31:00.932 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.2351e-05 2023-09-23T20:31:00.932 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.932 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.932 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:31:00.932 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.103942+0000", 2023-09-23T20:31:00.932 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.7009999999999999e-06 2023-09-23T20:31:00.932 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.932 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.932 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:31:00.932 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.076330+0000", 2023-09-23T20:31:00.932 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.97238725299999995 2023-09-23T20:31:00.933 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.933 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.933 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:31:00.933 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.076434+0000", 2023-09-23T20:31:00.933 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00010414 2023-09-23T20:31:00.933 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.933 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.933 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 1,6", 2023-09-23T20:31:00.933 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.076538+0000", 2023-09-23T20:31:00.933 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000104041 2023-09-23T20:31:00.933 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.934 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.934 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:31:00.934 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.086064+0000", 2023-09-23T20:31:00.934 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0095265880000000008 2023-09-23T20:31:00.934 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.934 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.934 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:31:00.934 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.101562+0000", 2023-09-23T20:31:00.934 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.015497858 2023-09-23T20:31:00.934 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.934 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.935 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:31:00.935 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.101613+0000", 2023-09-23T20:31:00.935 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.0283e-05 2023-09-23T20:31:00.935 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.935 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.935 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:31:00.935 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.101660+0000", 2023-09-23T20:31:00.935 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.7633999999999998e-05 2023-09-23T20:31:00.935 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.935 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.935 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:31:00.935 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.101699+0000", 2023-09-23T20:31:00.935 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.8646999999999998e-05 2023-09-23T20:31:00.936 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:31:00.936 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:31:00.936 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:31:00.936 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.936 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.936 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:134133 5.18 5:1bf4905b:::benchmark_data_smithi018_228451_object27072:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e169)", 2023-09-23T20:31:00.936 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:25:14.109028+0000", 2023-09-23T20:31:00.936 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 346.73406214599999, 2023-09-23T20:31:00.936 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.99277708499999995, 2023-09-23T20:31:00.936 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:31:00.936 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:31:00.936 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:31:00.937 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:31:00.937 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:31:00.937 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 134133 2023-09-23T20:31:00.937 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.937 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:31:00.937 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.937 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:31:00.937 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.109028+0000", 2023-09-23T20:31:00.937 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:31:00.937 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.937 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.937 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:31:00.938 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.109028+0000", 2023-09-23T20:31:00.938 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:31:00.938 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.938 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.938 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:31:00.938 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.109027+0000", 2023-09-23T20:31:00.938 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:31:00.938 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.938 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.938 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:31:00.938 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.109031+0000", 2023-09-23T20:31:00.938 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.4290000000000003e-06 2023-09-23T20:31:00.939 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.939 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.939 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:31:00.939 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.109047+0000", 2023-09-23T20:31:00.939 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.6314999999999999e-05 2023-09-23T20:31:00.939 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.939 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.939 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:31:00.939 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.109054+0000", 2023-09-23T20:31:00.939 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.6370000000000002e-06 2023-09-23T20:31:00.939 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.939 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.940 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:31:00.940 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.109142+0000", 2023-09-23T20:31:00.940 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.7954999999999994e-05 2023-09-23T20:31:00.940 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.940 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.940 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:31:00.940 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.109150+0000", 2023-09-23T20:31:00.940 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.4999999999999999e-06 2023-09-23T20:31:00.940 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.940 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.940 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:31:00.941 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.078060+0000", 2023-09-23T20:31:00.941 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.96890956900000003 2023-09-23T20:31:00.941 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.941 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.941 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:31:00.941 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.078173+0000", 2023-09-23T20:31:00.941 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000113341 2023-09-23T20:31:00.941 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.941 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.941 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 1,6", 2023-09-23T20:31:00.941 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.078295+0000", 2023-09-23T20:31:00.942 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000121894 2023-09-23T20:31:00.942 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.942 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.942 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:31:00.942 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.086074+0000", 2023-09-23T20:31:00.942 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0077791140000000002 2023-09-23T20:31:00.942 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.942 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.942 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:31:00.942 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.101588+0000", 2023-09-23T20:31:00.942 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.015513968 2023-09-23T20:31:00.943 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.943 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.943 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:31:00.943 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.101732+0000", 2023-09-23T20:31:00.943 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00014366499999999999 2023-09-23T20:31:00.943 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.943 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.943 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:31:00.943 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.101772+0000", 2023-09-23T20:31:00.943 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.0204999999999999e-05 2023-09-23T20:31:00.943 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.943 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.944 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:31:00.944 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.101805+0000", 2023-09-23T20:31:00.944 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.2564000000000002e-05 2023-09-23T20:31:00.944 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:31:00.944 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:31:00.944 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:31:00.944 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.944 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.944 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:134135 5.18 5:1df619a8:::benchmark_data_smithi018_228451_object27074:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e169)", 2023-09-23T20:31:00.944 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:25:14.109193+0000", 2023-09-23T20:31:00.944 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 346.73389691699998, 2023-09-23T20:31:00.945 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.99274740800000005, 2023-09-23T20:31:00.945 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:31:00.945 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:31:00.945 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:31:00.945 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:31:00.945 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:31:00.945 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 134135 2023-09-23T20:31:00.945 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.945 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:31:00.945 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.945 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:31:00.945 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.109193+0000", 2023-09-23T20:31:00.946 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:31:00.946 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.946 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.946 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:31:00.946 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.109193+0000", 2023-09-23T20:31:00.946 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:31:00.946 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.946 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.946 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:31:00.946 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.109192+0000", 2023-09-23T20:31:00.946 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:31:00.947 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.947 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.947 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:31:00.947 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.109196+0000", 2023-09-23T20:31:00.947 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.2740000000000001e-06 2023-09-23T20:31:00.947 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.947 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.947 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:31:00.947 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.109205+0000", 2023-09-23T20:31:00.947 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.0970000000000006e-06 2023-09-23T20:31:00.947 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.947 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.948 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:31:00.948 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.109211+0000", 2023-09-23T20:31:00.948 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.5400000000000001e-06 2023-09-23T20:31:00.948 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.948 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.948 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:31:00.948 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.109256+0000", 2023-09-23T20:31:00.948 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.5410000000000001e-05 2023-09-23T20:31:00.948 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.948 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.948 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:31:00.948 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.109267+0000", 2023-09-23T20:31:00.949 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.004e-05 2023-09-23T20:31:00.949 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.949 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.949 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:31:00.949 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.079350+0000", 2023-09-23T20:31:00.949 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.970083841 2023-09-23T20:31:00.949 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.949 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.949 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:31:00.949 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.079462+0000", 2023-09-23T20:31:00.949 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00011156 2023-09-23T20:31:00.949 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.949 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.950 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 1,6", 2023-09-23T20:31:00.950 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.079535+0000", 2023-09-23T20:31:00.950 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.2638000000000005e-05 2023-09-23T20:31:00.950 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.950 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.950 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:31:00.950 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.086084+0000", 2023-09-23T20:31:00.950 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.006549109 2023-09-23T20:31:00.950 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.950 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.951 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:31:00.951 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.101837+0000", 2023-09-23T20:31:00.951 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.015753775000000001 2023-09-23T20:31:00.951 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.951 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.951 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:31:00.951 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.101896+0000", 2023-09-23T20:31:00.951 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.8525e-05 2023-09-23T20:31:00.951 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.951 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.951 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:31:00.951 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.101921+0000", 2023-09-23T20:31:00.951 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.4905e-05 2023-09-23T20:31:00.952 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.952 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.952 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:31:00.952 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.101940+0000", 2023-09-23T20:31:00.952 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.9368000000000002e-05 2023-09-23T20:31:00.952 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:31:00.952 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:31:00.952 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:31:00.952 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.952 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.952 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:134176 5.18 5:1f2720a7:::benchmark_data_smithi018_228451_object27115:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e169)", 2023-09-23T20:31:00.953 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:25:14.171790+0000", 2023-09-23T20:31:00.953 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 346.67129975799998, 2023-09-23T20:31:00.953 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.93025216899999996, 2023-09-23T20:31:00.953 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:31:00.953 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:31:00.953 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:31:00.953 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:31:00.953 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:31:00.953 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 134176 2023-09-23T20:31:00.953 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.953 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:31:00.953 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.954 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:31:00.954 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.171790+0000", 2023-09-23T20:31:00.954 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:31:00.954 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.954 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.954 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:31:00.954 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.171790+0000", 2023-09-23T20:31:00.954 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:31:00.954 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.954 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.954 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:31:00.954 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.171790+0000", 2023-09-23T20:31:00.955 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:31:00.955 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.955 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.955 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:31:00.955 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.171792+0000", 2023-09-23T20:31:00.955 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.5330000000000001e-06 2023-09-23T20:31:00.955 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.955 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.955 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:31:00.955 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.171797+0000", 2023-09-23T20:31:00.955 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.2109999999999997e-06 2023-09-23T20:31:00.955 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.956 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.956 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:31:00.956 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.171801+0000", 2023-09-23T20:31:00.956 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.709e-06 2023-09-23T20:31:00.956 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.956 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.956 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:31:00.956 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.172312+0000", 2023-09-23T20:31:00.956 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00051119399999999997 2023-09-23T20:31:00.956 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.956 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.957 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:31:00.957 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.172327+0000", 2023-09-23T20:31:00.957 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.4389000000000001e-05 2023-09-23T20:31:00.957 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.957 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.957 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:31:00.957 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.080465+0000", 2023-09-23T20:31:00.957 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.90813792800000004 2023-09-23T20:31:00.957 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.957 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.958 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:31:00.958 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.080593+0000", 2023-09-23T20:31:00.958 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00012820799999999999 2023-09-23T20:31:00.958 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.958 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.958 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 1,6", 2023-09-23T20:31:00.958 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.080723+0000", 2023-09-23T20:31:00.958 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000130362 2023-09-23T20:31:00.958 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.958 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.958 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:31:00.959 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.090073+0000", 2023-09-23T20:31:00.959 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0093501950000000004 2023-09-23T20:31:00.959 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.959 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.959 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:31:00.959 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.101864+0000", 2023-09-23T20:31:00.959 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.011790429 2023-09-23T20:31:00.959 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.959 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.959 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:31:00.959 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.102002+0000", 2023-09-23T20:31:00.959 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00013855900000000001 2023-09-23T20:31:00.960 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.960 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.960 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:31:00.960 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.102024+0000", 2023-09-23T20:31:00.960 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.1926e-05 2023-09-23T20:31:00.960 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.960 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.960 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:31:00.960 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.102042+0000", 2023-09-23T20:31:00.960 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.7946000000000001e-05 2023-09-23T20:31:00.960 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:31:00.960 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:31:00.961 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:31:00.961 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.961 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.961 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:134225 5.18 5:18f2ddba:::benchmark_data_smithi018_228451_object27164:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e169)", 2023-09-23T20:31:00.961 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:25:14.238878+0000", 2023-09-23T20:31:00.961 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 346.60421162599999, 2023-09-23T20:31:00.961 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.86321889900000004, 2023-09-23T20:31:00.961 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:31:00.961 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:31:00.961 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:31:00.961 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:31:00.961 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:31:00.962 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 134225 2023-09-23T20:31:00.962 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.962 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:31:00.962 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.962 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:31:00.962 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.238878+0000", 2023-09-23T20:31:00.962 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:31:00.962 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.962 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.962 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:31:00.962 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.238878+0000", 2023-09-23T20:31:00.962 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:31:00.963 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.963 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.963 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:31:00.963 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.238877+0000", 2023-09-23T20:31:00.963 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:31:00.963 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.963 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.963 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:31:00.963 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.238881+0000", 2023-09-23T20:31:00.963 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.42e-06 2023-09-23T20:31:00.963 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.963 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.964 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:31:00.964 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.238893+0000", 2023-09-23T20:31:00.964 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1618e-05 2023-09-23T20:31:00.964 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.964 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.964 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:31:00.964 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.238898+0000", 2023-09-23T20:31:00.964 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.2800000000000003e-06 2023-09-23T20:31:00.964 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.964 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.964 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:31:00.965 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.238954+0000", 2023-09-23T20:31:00.965 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.5578999999999998e-05 2023-09-23T20:31:00.965 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.965 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.965 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:31:00.965 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.238965+0000", 2023-09-23T20:31:00.965 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0740999999999999e-05 2023-09-23T20:31:00.965 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.965 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.965 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:31:00.965 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.082298+0000", 2023-09-23T20:31:00.965 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.843333361 2023-09-23T20:31:00.965 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.966 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.966 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:31:00.966 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.082458+0000", 2023-09-23T20:31:00.966 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000159937 2023-09-23T20:31:00.966 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.966 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.966 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 1,6", 2023-09-23T20:31:00.966 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.082595+0000", 2023-09-23T20:31:00.966 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00013719000000000001 2023-09-23T20:31:00.966 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.966 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.966 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:31:00.967 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.090083+0000", 2023-09-23T20:31:00.967 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0074878849999999997 2023-09-23T20:31:00.967 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.967 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.967 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:31:00.967 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.101879+0000", 2023-09-23T20:31:00.967 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.011796152000000001 2023-09-23T20:31:00.967 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.967 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.967 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:31:00.967 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.102058+0000", 2023-09-23T20:31:00.967 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00017879199999999999 2023-09-23T20:31:00.967 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.968 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.968 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:31:00.968 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.102079+0000", 2023-09-23T20:31:00.968 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.1396e-05 2023-09-23T20:31:00.968 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.968 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.968 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:31:00.968 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.102097+0000", 2023-09-23T20:31:00.968 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.7615999999999999e-05 2023-09-23T20:31:00.968 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:31:00.968 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:31:00.969 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:31:00.969 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.969 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.969 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:134291 5.18 5:1e60ddf2:::benchmark_data_smithi018_228451_object27230:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e169)", 2023-09-23T20:31:00.969 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:25:14.332140+0000", 2023-09-23T20:31:00.969 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 346.51094995, 2023-09-23T20:31:00.969 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.77005607600000003, 2023-09-23T20:31:00.969 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:31:00.969 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:31:00.969 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:31:00.969 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:31:00.970 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:31:00.970 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 134291 2023-09-23T20:31:00.970 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.970 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:31:00.970 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.970 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:31:00.970 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.332140+0000", 2023-09-23T20:31:00.970 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:31:00.970 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.970 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.970 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:31:00.970 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.332140+0000", 2023-09-23T20:31:00.971 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:31:00.971 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.971 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.971 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:31:00.971 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.332139+0000", 2023-09-23T20:31:00.971 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:31:00.971 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.971 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.971 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:31:00.971 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.332143+0000", 2023-09-23T20:31:00.971 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.8080000000000004e-06 2023-09-23T20:31:00.971 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.971 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.972 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:31:00.972 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.332155+0000", 2023-09-23T20:31:00.972 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1216999999999999e-05 2023-09-23T20:31:00.972 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.972 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.972 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:31:00.972 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.332160+0000", 2023-09-23T20:31:00.972 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.4450000000000004e-06 2023-09-23T20:31:00.972 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.972 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.972 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:31:00.973 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.332209+0000", 2023-09-23T20:31:00.973 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.8822000000000001e-05 2023-09-23T20:31:00.973 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.973 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.973 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:31:00.973 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.332216+0000", 2023-09-23T20:31:00.973 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.0809999999999996e-06 2023-09-23T20:31:00.973 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.973 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.973 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:31:00.973 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.084295+0000", 2023-09-23T20:31:00.974 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.75207854200000002 2023-09-23T20:31:00.974 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.974 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.974 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:31:00.974 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.084424+0000", 2023-09-23T20:31:00.974 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00012909099999999999 2023-09-23T20:31:00.974 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.974 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.974 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 1,6", 2023-09-23T20:31:00.974 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.084534+0000", 2023-09-23T20:31:00.974 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000110449 2023-09-23T20:31:00.975 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.975 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.975 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:31:00.975 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.093553+0000", 2023-09-23T20:31:00.975 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0090184629999999991 2023-09-23T20:31:00.975 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.975 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.975 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:31:00.975 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.101957+0000", 2023-09-23T20:31:00.975 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0084048639999999997 2023-09-23T20:31:00.975 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.975 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.975 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:31:00.976 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.102157+0000", 2023-09-23T20:31:00.976 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00019960400000000001 2023-09-23T20:31:00.976 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.976 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.976 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:31:00.976 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.102178+0000", 2023-09-23T20:31:00.976 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.1035000000000001e-05 2023-09-23T20:31:00.976 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.976 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.976 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:31:00.976 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.102196+0000", 2023-09-23T20:31:00.976 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.7788e-05 2023-09-23T20:31:00.977 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:31:00.977 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:31:00.977 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:31:00.977 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.977 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.977 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:134367 5.18 5:1c202e23:::benchmark_data_smithi018_228451_object27306:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e169)", 2023-09-23T20:31:00.977 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:25:14.443008+0000", 2023-09-23T20:31:00.977 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 346.40008140800001, 2023-09-23T20:31:00.977 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.65924484999999999, 2023-09-23T20:31:00.977 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:31:00.977 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:31:00.977 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:31:00.978 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:31:00.978 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:31:00.978 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 134367 2023-09-23T20:31:00.978 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.978 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:31:00.978 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.978 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:31:00.978 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.443008+0000", 2023-09-23T20:31:00.978 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:31:00.978 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.978 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.978 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:31:00.979 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.443008+0000", 2023-09-23T20:31:00.979 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:31:00.979 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.979 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.979 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:31:00.979 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.443007+0000", 2023-09-23T20:31:00.979 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:31:00.979 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.979 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.979 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:31:00.979 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.443013+0000", 2023-09-23T20:31:00.980 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.0000000000000002e-06 2023-09-23T20:31:00.980 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.980 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.980 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:31:00.980 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.443030+0000", 2023-09-23T20:31:00.980 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.7093000000000001e-05 2023-09-23T20:31:00.980 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.980 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.980 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:31:00.980 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.443037+0000", 2023-09-23T20:31:00.980 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.3679999999999996e-06 2023-09-23T20:31:00.981 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.981 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.981 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:31:00.981 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.443107+0000", 2023-09-23T20:31:00.981 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.9842e-05 2023-09-23T20:31:00.981 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.981 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.981 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:31:00.981 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.443118+0000", 2023-09-23T20:31:00.981 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0825999999999999e-05 2023-09-23T20:31:00.981 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.981 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.982 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:31:00.982 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.086130+0000", 2023-09-23T20:31:00.982 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.64301198999999998 2023-09-23T20:31:00.982 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.982 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.982 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:31:00.982 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.086234+0000", 2023-09-23T20:31:00.982 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000104379 2023-09-23T20:31:00.982 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.982 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.982 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 1,6", 2023-09-23T20:31:00.983 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.086307+0000", 2023-09-23T20:31:00.983 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.2948999999999996e-05 2023-09-23T20:31:00.983 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.983 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.983 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:31:00.983 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.093560+0000", 2023-09-23T20:31:00.983 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.007252975 2023-09-23T20:31:00.983 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.983 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.983 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:31:00.983 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.101973+0000", 2023-09-23T20:31:00.983 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.008412915 2023-09-23T20:31:00.984 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.984 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.984 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:31:00.984 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.102213+0000", 2023-09-23T20:31:00.984 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000239822 2023-09-23T20:31:00.984 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.984 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.984 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:31:00.984 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.102234+0000", 2023-09-23T20:31:00.984 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.0934e-05 2023-09-23T20:31:00.984 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.985 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.985 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:31:00.985 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.102253+0000", 2023-09-23T20:31:00.985 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.9196999999999999e-05 2023-09-23T20:31:00.985 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:31:00.985 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:31:00.985 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:31:00.985 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.985 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.985 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:150491 5.1 5:83c4f1b1:::benchmark_data_smithi018_228451_object43430:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e178)", 2023-09-23T20:31:00.985 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:25:40.788685+0000", 2023-09-23T20:31:00.985 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 320.05440429399999, 2023-09-23T20:31:00.986 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.86834999599999996, 2023-09-23T20:31:00.986 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:31:00.986 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:31:00.986 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:31:00.986 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:31:00.986 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:31:00.986 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 150491 2023-09-23T20:31:00.986 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.986 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:31:00.986 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.987 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:31:00.987 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:40.788685+0000", 2023-09-23T20:31:00.987 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:31:00.987 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.987 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.987 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:31:00.987 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:40.788685+0000", 2023-09-23T20:31:00.987 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:31:00.987 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.987 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.987 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:31:00.987 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:40.788684+0000", 2023-09-23T20:31:00.988 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:31:00.988 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.988 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.988 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:31:00.988 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:40.788689+0000", 2023-09-23T20:31:00.988 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.9599999999999999e-06 2023-09-23T20:31:00.988 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.988 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.988 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:31:00.988 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:40.788696+0000", 2023-09-23T20:31:00.988 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.3899999999999998e-06 2023-09-23T20:31:00.989 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.989 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.989 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:31:00.989 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:40.788703+0000", 2023-09-23T20:31:00.989 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.4189999999999996e-06 2023-09-23T20:31:00.989 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.989 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.989 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:31:00.989 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:40.788750+0000", 2023-09-23T20:31:00.989 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.7205e-05 2023-09-23T20:31:00.989 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.989 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.990 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:31:00.990 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:40.788756+0000", 2023-09-23T20:31:00.990 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.003e-06 2023-09-23T20:31:00.990 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.990 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.990 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:31:00.990 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:41.630396+0000", 2023-09-23T20:31:00.990 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.84163923299999999 2023-09-23T20:31:00.990 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.990 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.990 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:31:00.991 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:41.630529+0000", 2023-09-23T20:31:00.991 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00013375400000000001 2023-09-23T20:31:00.991 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.991 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.991 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 3,7", 2023-09-23T20:31:00.991 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:41.630649+0000", 2023-09-23T20:31:00.991 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000119683 2023-09-23T20:31:00.991 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.991 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.991 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:31:00.991 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:41.640197+0000", 2023-09-23T20:31:00.992 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0095476759999999997 2023-09-23T20:31:00.992 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.992 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.992 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:31:00.992 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:41.656929+0000", 2023-09-23T20:31:00.992 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.016732059000000001 2023-09-23T20:31:00.992 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.992 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.992 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:31:00.992 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:41.656958+0000", 2023-09-23T20:31:00.992 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.8915e-05 2023-09-23T20:31:00.992 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.993 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.993 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:31:00.993 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:41.657000+0000", 2023-09-23T20:31:00.993 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.2722999999999997e-05 2023-09-23T20:31:00.993 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.993 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.993 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:31:00.993 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:41.657035+0000", 2023-09-23T20:31:00.993 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.5107000000000001e-05 2023-09-23T20:31:00.993 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:31:00.993 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:31:00.994 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:31:00.994 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.994 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.994 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:150495 5.1 5:803dfbf3:::benchmark_data_smithi018_228451_object43434:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e178)", 2023-09-23T20:31:00.994 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:25:40.845643+0000", 2023-09-23T20:31:00.994 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 319.99744666200002, 2023-09-23T20:31:00.994 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.81154586399999995, 2023-09-23T20:31:00.994 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:31:00.994 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:31:00.994 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:31:00.994 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:31:00.994 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:31:00.995 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 150495 2023-09-23T20:31:00.995 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.995 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:31:00.995 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.995 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:31:00.995 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:40.845643+0000", 2023-09-23T20:31:00.995 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:31:00.995 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.995 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.995 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:31:00.995 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:40.845643+0000", 2023-09-23T20:31:00.995 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:31:00.996 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.996 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.996 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:31:00.996 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:40.845642+0000", 2023-09-23T20:31:00.996 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:31:00.996 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.996 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.996 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:31:00.996 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:40.845650+0000", 2023-09-23T20:31:00.996 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.2900000000000002e-06 2023-09-23T20:31:00.996 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.996 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.997 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:31:00.997 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:40.845664+0000", 2023-09-23T20:31:00.997 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.4066000000000001e-05 2023-09-23T20:31:00.997 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.997 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.997 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:31:00.997 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:40.845670+0000", 2023-09-23T20:31:00.997 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.1229999999999997e-06 2023-09-23T20:31:00.997 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.997 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.997 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:31:00.997 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:40.845722+0000", 2023-09-23T20:31:00.998 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.1867999999999999e-05 2023-09-23T20:31:00.998 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.998 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.998 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:31:00.998 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:40.845731+0000", 2023-09-23T20:31:00.998 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.551e-06 2023-09-23T20:31:00.998 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.998 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.998 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:31:00.998 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:41.633144+0000", 2023-09-23T20:31:00.998 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.78741325200000001 2023-09-23T20:31:00.999 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.999 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.999 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:31:00.999 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:41.633277+0000", 2023-09-23T20:31:00.999 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00013279700000000001 2023-09-23T20:31:00.999 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.999 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:00.999 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 3,7", 2023-09-23T20:31:00.999 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:41.633463+0000", 2023-09-23T20:31:00.999 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00018569499999999999 2023-09-23T20:31:00.999 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:00.999 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:01.000 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:31:01.000 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:41.643776+0000", 2023-09-23T20:31:01.000 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.010313387 2023-09-23T20:31:01.000 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:01.000 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:01.000 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:31:01.000 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:41.657067+0000", 2023-09-23T20:31:01.000 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.013291321 2023-09-23T20:31:01.000 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:01.000 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:01.000 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:31:01.001 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:41.657121+0000", 2023-09-23T20:31:01.001 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.3393999999999997e-05 2023-09-23T20:31:01.001 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:01.001 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:01.001 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:31:01.001 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:41.657161+0000", 2023-09-23T20:31:01.001 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.0726000000000003e-05 2023-09-23T20:31:01.001 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:01.001 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:01.001 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:31:01.001 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:41.657189+0000", 2023-09-23T20:31:01.002 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.7532999999999999e-05 2023-09-23T20:31:01.002 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:31:01.002 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:31:01.002 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:31:01.002 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:01.002 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:01.002 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:150532 5.1 5:8454aca9:::benchmark_data_smithi018_228451_object43471:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e178)", 2023-09-23T20:31:01.002 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:25:40.905728+0000", 2023-09-23T20:31:01.002 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 319.937361915, 2023-09-23T20:31:01.002 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.75163135700000006, 2023-09-23T20:31:01.002 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:31:01.003 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:31:01.003 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:31:01.003 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:31:01.003 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:31:01.003 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 150532 2023-09-23T20:31:01.003 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:01.003 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:31:01.003 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:01.003 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:31:01.003 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:40.905728+0000", 2023-09-23T20:31:01.003 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:31:01.003 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:01.004 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:01.004 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:31:01.004 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:40.905728+0000", 2023-09-23T20:31:01.004 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:31:01.004 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:01.004 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:01.004 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:31:01.004 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:40.905726+0000", 2023-09-23T20:31:01.004 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999981 2023-09-23T20:31:01.004 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:01.004 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:01.005 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:31:01.005 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:40.905736+0000", 2023-09-23T20:31:01.005 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.025e-05 2023-09-23T20:31:01.005 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:01.005 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:01.005 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:31:01.005 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:40.905747+0000", 2023-09-23T20:31:01.005 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0652999999999999e-05 2023-09-23T20:31:01.005 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:01.005 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:01.005 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:31:01.005 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:40.905752+0000", 2023-09-23T20:31:01.006 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.057e-06 2023-09-23T20:31:01.006 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:01.006 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:01.006 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:31:01.006 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:40.905817+0000", 2023-09-23T20:31:01.006 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.5423e-05 2023-09-23T20:31:01.006 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:01.006 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:01.006 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:31:01.006 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:40.905830+0000", 2023-09-23T20:31:01.007 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.2489e-05 2023-09-23T20:31:01.007 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:01.007 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:01.007 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:31:01.007 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:41.635829+0000", 2023-09-23T20:31:01.007 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.72999899199999996 2023-09-23T20:31:01.007 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:01.007 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:01.007 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:31:01.007 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:41.635942+0000", 2023-09-23T20:31:01.007 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000112639 2023-09-23T20:31:01.007 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:01.008 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:01.008 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 3,7", 2023-09-23T20:31:01.008 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:41.636014+0000", 2023-09-23T20:31:01.008 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.2274999999999995e-05 2023-09-23T20:31:01.008 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:01.008 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:01.008 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:31:01.008 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:41.643782+0000", 2023-09-23T20:31:01.008 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0077686179999999997 2023-09-23T20:31:01.008 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:01.008 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:01.008 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:31:01.008 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:41.657094+0000", 2023-09-23T20:31:01.009 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.013311922 2023-09-23T20:31:01.009 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:01.009 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:01.009 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:31:01.009 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:41.657283+0000", 2023-09-23T20:31:01.009 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000188588 2023-09-23T20:31:01.009 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:01.009 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:01.009 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:31:01.009 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:41.657325+0000", 2023-09-23T20:31:01.010 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.1868e-05 2023-09-23T20:31:01.010 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:01.010 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:01.010 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:31:01.010 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:41.657359+0000", 2023-09-23T20:31:01.010 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.4325000000000001e-05 2023-09-23T20:31:01.010 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:31:01.010 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:31:01.010 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:31:01.010 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:31:01.010 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:31:01.011 INFO:teuthology.orchestra.run.smithi149.stdout:} 2023-09-23T20:31:01.133 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:31:01.147 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:01.148 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:31:01.178 INFO:teuthology.orchestra.run.smithi018.stdout:{"pg_ready":true,"pg_map":{"version":653,"stamp":"2023-09-23T20:31:00.180923+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1006002798,"num_objects":15306,"num_object_clones":0,"num_object_copies":45918,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":29365,"num_objects_misplaced":2047,"num_objects_unfound":0,"num_objects_dirty":15306,"num_whiteouts":0,"num_read":6210,"num_read_kb":13614,"num_write":195211,"num_write_kb":6716893,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20802,"num_bytes_recovered":1142946016,"num_keys_recovered":30,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":191926,"ondisk_log_size":191926,"up":357,"acting":343,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":371,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":5983304,"kb_used_data":4134992,"kb_used_omap":32,"kb_used_meta":1847775,"kb_avail":650232760,"statfs":{"total":671965249536,"available":665838346240,"internally_reserved":0,"allocated":4234231808,"data_stored":4225688052,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32773,"internal_metadata":1892122619},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,4,8,3,7,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":9,"apply_latency_ms":9,"commit_latency_ns":9000000,"apply_latency_ns":9000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-246153216,"num_objects":-3756,"num_object_clones":0,"num_object_copies":-11268,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-636,"num_objects_misplaced":-1111,"num_objects_unfound":0,"num_objects_dirty":-3756,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3756,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1348,"num_bytes_recovered":45940736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.483650"},"pg_stats":[{"pgid":"3.1a","version":"69'1","reported_seq":34,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-09-23T20:30:56.128182+0000","last_change":"2023-09-23T20:29:46.050662+0000","last_active":"2023-09-23T20:30:56.128182+0000","last_peered":"2023-09-23T20:30:56.128182+0000","last_clean":"2023-09-23T20:30:56.128182+0000","last_became_active":"2023-09-23T20:29:46.050281+0000","last_became_peered":"2023-09-23T20:29:46.050281+0000","last_unstale":"2023-09-23T20:30:56.128182+0000","last_undegraded":"2023-09-23T20:30:56.128182+0000","last_fullsized":"2023-09-23T20:30:56.128182+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:55.227008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":162,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-09-23T20:30:56.088207+0000","last_change":"2023-09-23T20:29:46.067085+0000","last_active":"2023-09-23T20:30:56.088207+0000","last_peered":"2023-09-23T20:30:56.088207+0000","last_clean":"2023-09-23T20:30:56.088207+0000","last_became_active":"2023-09-23T20:29:46.066738+0000","last_became_peered":"2023-09-23T20:29:46.066738+0000","last_unstale":"2023-09-23T20:30:56.088207+0000","last_undegraded":"2023-09-23T20:30:56.088207+0000","last_fullsized":"2023-09-23T20:30:56.088207+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:33:14.679402+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":226,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-09-23T20:30:56.129716+0000","last_change":"2023-09-23T20:29:46.178684+0000","last_active":"2023-09-23T20:30:56.129716+0000","last_peered":"2023-09-23T20:30:56.129716+0000","last_clean":"2023-09-23T20:30:56.129716+0000","last_became_active":"2023-09-23T20:29:46.178249+0000","last_became_peered":"2023-09-23T20:29:46.178249+0000","last_unstale":"2023-09-23T20:30:56.129716+0000","last_undegraded":"2023-09-23T20:30:56.129716+0000","last_fullsized":"2023-09-23T20:30:56.129716+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:58:58.530687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":254,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-09-23T20:30:56.113857+0000","last_change":"2023-09-23T20:25:02.957185+0000","last_active":"2023-09-23T20:30:56.113857+0000","last_peered":"2023-09-23T20:30:56.113857+0000","last_clean":"2023-09-23T20:30:56.113857+0000","last_became_active":"2023-09-23T20:25:02.956689+0000","last_became_peered":"2023-09-23T20:25:02.956689+0000","last_unstale":"2023-09-23T20:30:56.113857+0000","last_undegraded":"2023-09-23T20:30:56.113857+0000","last_fullsized":"2023-09-23T20:30:56.113857+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:04:28.524015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":151,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-09-23T20:30:56.087729+0000","last_change":"2023-09-23T20:29:45.529762+0000","last_active":"2023-09-23T20:30:56.087729+0000","last_peered":"2023-09-23T20:30:56.087729+0000","last_clean":"2023-09-23T20:30:56.087729+0000","last_became_active":"2023-09-23T20:29:45.529283+0000","last_became_peered":"2023-09-23T20:29:45.529283+0000","last_unstale":"2023-09-23T20:30:56.087729+0000","last_undegraded":"2023-09-23T20:30:56.087729+0000","last_fullsized":"2023-09-23T20:30:56.087729+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:54:36.720075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":60,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T20:30:49.044298+0000","last_change":"2023-09-23T20:29:46.112977+0000","last_active":"2023-09-23T20:30:49.044298+0000","last_peered":"2023-09-23T20:30:49.044298+0000","last_clean":"2023-09-23T20:30:49.044298+0000","last_became_active":"2023-09-23T20:29:46.112600+0000","last_became_peered":"2023-09-23T20:29:46.112600+0000","last_unstale":"2023-09-23T20:30:49.044298+0000","last_undegraded":"2023-09-23T20:30:49.044298+0000","last_fullsized":"2023-09-23T20:30:49.044298+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:06:34.132247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.12","version":"289'8909","reported_seq":15500,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:30:58.307206+0000","last_change":"2023-09-23T20:30:46.961845+0000","last_active":"2023-09-23T20:30:58.307206+0000","last_peered":"2023-09-23T20:30:58.307206+0000","last_clean":"2023-09-23T20:30:58.307206+0000","last_became_active":"2023-09-23T20:30:46.939470+0000","last_became_peered":"2023-09-23T20:30:46.939470+0000","last_unstale":"2023-09-23T20:30:58.307206+0000","last_undegraded":"2023-09-23T20:30:58.307206+0000","last_fullsized":"2023-09-23T20:30:58.307206+0000","mapping_epoch":284,"log_start":"223'2761","ondisk_log_start":"223'2761","created":242,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":6148,"log_dups_size":0,"ondisk_log_size":6148,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":36962304,"num_objects":564,"num_object_clones":0,"num_object_copies":1692,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":564,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6098,"num_write_kb":213184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3241,"num_bytes_recovered":105185280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":252,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-09-23T20:30:56.114253+0000","last_change":"2023-09-23T20:25:02.969432+0000","last_active":"2023-09-23T20:30:56.114253+0000","last_peered":"2023-09-23T20:30:56.114253+0000","last_clean":"2023-09-23T20:30:56.114253+0000","last_became_active":"2023-09-23T20:25:02.969118+0000","last_became_peered":"2023-09-23T20:25:02.969118+0000","last_unstale":"2023-09-23T20:30:56.114253+0000","last_undegraded":"2023-09-23T20:30:56.114253+0000","last_fullsized":"2023-09-23T20:30:56.114253+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:02:22.453831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":171,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-09-23T20:30:56.087799+0000","last_change":"2023-09-23T20:29:46.069832+0000","last_active":"2023-09-23T20:30:56.087799+0000","last_peered":"2023-09-23T20:30:56.087799+0000","last_clean":"2023-09-23T20:30:56.087799+0000","last_became_active":"2023-09-23T20:29:46.069451+0000","last_became_peered":"2023-09-23T20:29:46.069451+0000","last_unstale":"2023-09-23T20:30:56.087799+0000","last_undegraded":"2023-09-23T20:30:56.087799+0000","last_fullsized":"2023-09-23T20:30:56.087799+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:06:06.104030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.13","version":"287'8752","reported_seq":9114,"reported_epoch":287,"state":"active+recovery_wait","last_fresh":"2023-09-23T20:30:55.447329+0000","last_change":"2023-09-23T20:30:30.361943+0000","last_active":"2023-09-23T20:30:55.447329+0000","last_peered":"2023-09-23T20:30:55.447329+0000","last_clean":"2023-09-23T20:28:04.010682+0000","last_became_active":"2023-09-23T20:29:47.054756+0000","last_became_peered":"2023-09-23T20:29:47.054756+0000","last_unstale":"2023-09-23T20:30:55.447329+0000","last_undegraded":"2023-09-23T20:30:55.447329+0000","last_fullsized":"2023-09-23T20:30:55.447329+0000","mapping_epoch":268,"log_start":"221'966","ondisk_log_start":"221'966","created":242,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":7786,"log_dups_size":0,"ondisk_log_size":7786,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":39026688,"num_objects":596,"num_object_clones":0,"num_object_copies":1788,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":596,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5984,"num_write_kb":210528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":117,"num_bytes_recovered":7503872,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":["7","2","6"],"object_location_counts":[{"shards":"2,6,7","objects":596}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.14","version":"90'161","reported_seq":559,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-09-23T20:30:56.102695+0000","last_change":"2023-09-23T20:29:48.092787+0000","last_active":"2023-09-23T20:30:56.102695+0000","last_peered":"2023-09-23T20:30:56.102695+0000","last_clean":"2023-09-23T20:30:56.102695+0000","last_became_active":"2023-09-23T20:29:48.092460+0000","last_became_peered":"2023-09-23T20:29:48.092460+0000","last_unstale":"2023-09-23T20:30:56.102695+0000","last_undegraded":"2023-09-23T20:30:56.102695+0000","last_fullsized":"2023-09-23T20:30:56.102695+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:13.244651+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.244651+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0019955509999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109227,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":263,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T20:30:49.033924+0000","last_change":"2023-09-23T20:29:45.523808+0000","last_active":"2023-09-23T20:30:49.033924+0000","last_peered":"2023-09-23T20:30:49.033924+0000","last_clean":"2023-09-23T20:30:49.033924+0000","last_became_active":"2023-09-23T20:29:45.523508+0000","last_became_peered":"2023-09-23T20:29:45.523508+0000","last_unstale":"2023-09-23T20:30:49.033924+0000","last_undegraded":"2023-09-23T20:30:49.033924+0000","last_fullsized":"2023-09-23T20:30:49.033924+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:50:55.565352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":154,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-09-23T20:30:56.089860+0000","last_change":"2023-09-23T20:29:04.130512+0000","last_active":"2023-09-23T20:30:56.089860+0000","last_peered":"2023-09-23T20:30:56.089860+0000","last_clean":"2023-09-23T20:30:56.089860+0000","last_became_active":"2023-09-23T20:29:04.130135+0000","last_became_peered":"2023-09-23T20:29:04.130135+0000","last_unstale":"2023-09-23T20:30:56.089860+0000","last_undegraded":"2023-09-23T20:30:56.089860+0000","last_fullsized":"2023-09-23T20:30:56.089860+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:53:59.385893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.10","version":"289'8971","reported_seq":9064,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:30:59.674709+0000","last_change":"2023-09-23T20:30:24.864928+0000","last_active":"2023-09-23T20:30:59.674709+0000","last_peered":"2023-09-23T20:30:59.674709+0000","last_clean":"2023-09-23T20:30:59.674709+0000","last_became_active":"2023-09-23T20:28:35.582165+0000","last_became_peered":"2023-09-23T20:28:35.582165+0000","last_unstale":"2023-09-23T20:30:59.674709+0000","last_undegraded":"2023-09-23T20:30:59.674709+0000","last_fullsized":"2023-09-23T20:30:59.674709+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":242,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":8971,"log_dups_size":0,"ondisk_log_size":8971,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":27951104,"num_objects":427,"num_object_clones":0,"num_object_copies":1281,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":427,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6153,"num_write_kb":210528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":298,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T20:30:49.021771+0000","last_change":"2023-09-23T20:29:59.501260+0000","last_active":"2023-09-23T20:30:49.021771+0000","last_peered":"2023-09-23T20:30:49.021771+0000","last_clean":"2023-09-23T20:30:49.021771+0000","last_became_active":"2023-09-23T20:29:28.362070+0000","last_became_peered":"2023-09-23T20:29:28.362070+0000","last_unstale":"2023-09-23T20:30:49.021771+0000","last_undegraded":"2023-09-23T20:30:49.021771+0000","last_fullsized":"2023-09-23T20:30:49.021771+0000","mapping_epoch":258,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:32:55.983696+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":236,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-09-23T20:30:56.128468+0000","last_change":"2023-09-23T20:29:46.080766+0000","last_active":"2023-09-23T20:30:56.128468+0000","last_peered":"2023-09-23T20:30:56.128468+0000","last_clean":"2023-09-23T20:30:56.128468+0000","last_became_active":"2023-09-23T20:29:46.080356+0000","last_became_peered":"2023-09-23T20:29:46.080356+0000","last_unstale":"2023-09-23T20:30:56.128468+0000","last_undegraded":"2023-09-23T20:30:56.128468+0000","last_fullsized":"2023-09-23T20:30:56.128468+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:08:01.862270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.11","version":"287'8738","reported_seq":14403,"reported_epoch":287,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T20:30:55.591591+0000","last_change":"2023-09-23T20:30:44.984034+0000","last_active":"2023-09-23T20:30:55.591591+0000","last_peered":"2023-09-23T20:30:55.591591+0000","last_clean":"2023-09-23T20:28:04.011715+0000","last_became_active":"2023-09-23T20:29:46.191939+0000","last_became_peered":"2023-09-23T20:29:46.191939+0000","last_unstale":"2023-09-23T20:30:55.591591+0000","last_undegraded":"2023-09-23T20:29:49.867530+0000","last_fullsized":"2023-09-23T20:29:46.041211+0000","mapping_epoch":267,"log_start":"221'1065","ondisk_log_start":"221'1065","created":242,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":7673,"log_dups_size":0,"ondisk_log_size":7673,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":31719424,"num_objects":484,"num_object_clones":0,"num_object_copies":1452,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":47,"num_objects_misplaced":33,"num_objects_unfound":0,"num_objects_dirty":484,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5984,"num_write_kb":206976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2423,"num_bytes_recovered":158793728,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[6,2],"avail_no_missing":["6","2"],"object_location_counts":[{"shards":"2,6","objects":404},{"shards":"2,6,7","objects":33}],"blocked_by":[],"up_primary":1,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":104,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T20:30:49.020236+0000","last_change":"2023-09-23T20:29:59.529392+0000","last_active":"2023-09-23T20:30:49.020236+0000","last_peered":"2023-09-23T20:30:49.020236+0000","last_clean":"2023-09-23T20:30:49.020236+0000","last_became_active":"2023-09-23T20:29:28.365579+0000","last_became_peered":"2023-09-23T20:29:28.365579+0000","last_unstale":"2023-09-23T20:30:49.020236+0000","last_undegraded":"2023-09-23T20:30:49.020236+0000","last_fullsized":"2023-09-23T20:30:49.020236+0000","mapping_epoch":258,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:31:11.546083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":282,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T20:30:49.043530+0000","last_change":"2023-09-23T20:29:46.091575+0000","last_active":"2023-09-23T20:30:49.043530+0000","last_peered":"2023-09-23T20:30:49.043530+0000","last_clean":"2023-09-23T20:30:49.043530+0000","last_became_active":"2023-09-23T20:29:46.091271+0000","last_became_peered":"2023-09-23T20:29:46.091271+0000","last_unstale":"2023-09-23T20:30:49.043530+0000","last_undegraded":"2023-09-23T20:30:49.043530+0000","last_fullsized":"2023-09-23T20:30:49.043530+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:08:01.594175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.16","version":"289'8753","reported_seq":8853,"reported_epoch":289,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T20:30:58.346950+0000","last_change":"2023-09-23T20:29:29.535020+0000","last_active":"2023-09-23T20:30:58.346950+0000","last_peered":"2023-09-23T20:30:58.346950+0000","last_clean":"2023-09-23T20:29:27.303063+0000","last_became_active":"2023-09-23T20:29:29.522829+0000","last_became_peered":"2023-09-23T20:29:29.522829+0000","last_unstale":"2023-09-23T20:30:58.346950+0000","last_undegraded":"2023-09-23T20:30:58.346950+0000","last_fullsized":"2023-09-23T20:30:58.346950+0000","mapping_epoch":259,"log_start":"223'2617","ondisk_log_start":"223'2617","created":242,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":6136,"log_dups_size":0,"ondisk_log_size":6136,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":32571392,"num_objects":497,"num_object_clones":0,"num_object_copies":1491,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":497,"num_objects_unfound":0,"num_objects_dirty":497,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6011,"num_write_kb":208256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,4],"avail_no_missing":["3","4","7"],"object_location_counts":[{"shards":"3,4,7","objects":497}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.11","version":"90'161","reported_seq":505,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-09-23T20:30:56.111544+0000","last_change":"2023-09-23T20:30:36.766139+0000","last_active":"2023-09-23T20:30:56.111544+0000","last_peered":"2023-09-23T20:30:56.111544+0000","last_clean":"2023-09-23T20:30:56.111544+0000","last_became_active":"2023-09-23T20:28:05.991304+0000","last_became_peered":"2023-09-23T20:28:05.991304+0000","last_unstale":"2023-09-23T20:30:56.111544+0000","last_undegraded":"2023-09-23T20:30:56.111544+0000","last_fullsized":"2023-09-23T20:30:56.111544+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:34.821789+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:34.821789+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":297,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T20:30:49.021687+0000","last_change":"2023-09-23T20:29:59.544701+0000","last_active":"2023-09-23T20:30:49.021687+0000","last_peered":"2023-09-23T20:30:49.021687+0000","last_clean":"2023-09-23T20:30:49.021687+0000","last_became_active":"2023-09-23T20:29:28.367679+0000","last_became_peered":"2023-09-23T20:29:28.367679+0000","last_unstale":"2023-09-23T20:30:49.021687+0000","last_undegraded":"2023-09-23T20:30:49.021687+0000","last_fullsized":"2023-09-23T20:30:49.021687+0000","mapping_epoch":258,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:36:19.089724+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":102,"num_keys_recovered":30,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":151,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-09-23T20:30:56.094413+0000","last_change":"2023-09-23T20:29:45.531343+0000","last_active":"2023-09-23T20:30:56.094413+0000","last_peered":"2023-09-23T20:30:56.094413+0000","last_clean":"2023-09-23T20:30:56.094413+0000","last_became_active":"2023-09-23T20:29:45.530713+0000","last_became_peered":"2023-09-23T20:29:45.530713+0000","last_unstale":"2023-09-23T20:30:56.094413+0000","last_undegraded":"2023-09-23T20:30:56.094413+0000","last_fullsized":"2023-09-23T20:30:56.094413+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:28:00.245911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.17","version":"287'8965","reported_seq":9075,"reported_epoch":287,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:30:55.662471+0000","last_change":"2023-09-23T20:29:46.291295+0000","last_active":"2023-09-23T20:30:55.662471+0000","last_peered":"2023-09-23T20:30:55.662471+0000","last_clean":"2023-09-23T20:29:44.520411+0000","last_became_active":"2023-09-23T20:29:46.269706+0000","last_became_peered":"2023-09-23T20:29:46.269706+0000","last_unstale":"2023-09-23T20:30:55.662471+0000","last_undegraded":"2023-09-23T20:29:46.075283+0000","last_fullsized":"2023-09-23T20:29:46.032679+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":242,"last_epoch_clean":260,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":8965,"log_dups_size":0,"ondisk_log_size":8965,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":33062912,"num_objects":505,"num_object_clones":0,"num_object_copies":1515,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3310,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":505,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6137,"num_write_kb":212512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3],"avail_no_missing":["5","3","4"],"object_location_counts":[{"shards":"3,4,5","objects":494},{"shards":"3,5","objects":-1329}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.10","version":"90'161","reported_seq":507,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T20:30:49.031092+0000","last_change":"2023-09-23T20:29:07.045212+0000","last_active":"2023-09-23T20:30:49.031092+0000","last_peered":"2023-09-23T20:30:49.031092+0000","last_clean":"2023-09-23T20:30:49.031092+0000","last_became_active":"2023-09-23T20:29:07.044892+0000","last_became_peered":"2023-09-23T20:29:07.044892+0000","last_unstale":"2023-09-23T20:30:49.031092+0000","last_undegraded":"2023-09-23T20:30:49.031092+0000","last_fullsized":"2023-09-23T20:30:49.031092+0000","mapping_epoch":253,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":254,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:32.801168+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:32.801168+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":252,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-09-23T20:30:56.114942+0000","last_change":"2023-09-23T20:25:02.953993+0000","last_active":"2023-09-23T20:30:56.114942+0000","last_peered":"2023-09-23T20:30:56.114942+0000","last_clean":"2023-09-23T20:30:56.114942+0000","last_became_active":"2023-09-23T20:25:02.953672+0000","last_became_peered":"2023-09-23T20:25:02.953672+0000","last_unstale":"2023-09-23T20:30:56.114942+0000","last_undegraded":"2023-09-23T20:30:56.114942+0000","last_fullsized":"2023-09-23T20:30:56.114942+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:26:08.603577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":265,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T20:30:49.034075+0000","last_change":"2023-09-23T20:25:02.949584+0000","last_active":"2023-09-23T20:30:49.034075+0000","last_peered":"2023-09-23T20:30:49.034075+0000","last_clean":"2023-09-23T20:30:49.034075+0000","last_became_active":"2023-09-23T20:25:02.949132+0000","last_became_peered":"2023-09-23T20:25:02.949132+0000","last_unstale":"2023-09-23T20:30:49.034075+0000","last_undegraded":"2023-09-23T20:30:49.034075+0000","last_fullsized":"2023-09-23T20:30:49.034075+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:34:20.509823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.14","version":"289'8881","reported_seq":8984,"reported_epoch":289,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:30:58.243172+0000","last_change":"2023-09-23T20:29:46.199482+0000","last_active":"2023-09-23T20:30:58.243172+0000","last_peered":"2023-09-23T20:30:58.243172+0000","last_clean":"2023-09-23T20:28:12.083147+0000","last_became_active":"2023-09-23T20:29:46.185251+0000","last_became_peered":"2023-09-23T20:29:46.185251+0000","last_unstale":"2023-09-23T20:30:58.243172+0000","last_undegraded":"2023-09-23T20:29:46.057476+0000","last_fullsized":"2023-09-23T20:29:46.043200+0000","mapping_epoch":267,"log_start":"224'2907","ondisk_log_start":"224'2907","created":242,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":5974,"log_dups_size":0,"ondisk_log_size":5974,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":29556736,"num_objects":451,"num_object_clones":0,"num_object_copies":1353,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2460,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":451,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6117,"num_write_kb":210176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[3,5],"avail_no_missing":["3","5"],"object_location_counts":[{"shards":"3,5","objects":-622}],"blocked_by":[],"up_primary":1,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.13","version":"90'161","reported_seq":515,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-09-23T20:30:56.133795+0000","last_change":"2023-09-23T20:29:46.067221+0000","last_active":"2023-09-23T20:30:56.133795+0000","last_peered":"2023-09-23T20:30:56.133795+0000","last_clean":"2023-09-23T20:30:56.133795+0000","last_became_active":"2023-09-23T20:29:46.066622+0000","last_became_peered":"2023-09-23T20:29:46.066622+0000","last_unstale":"2023-09-23T20:30:56.133795+0000","last_undegraded":"2023-09-23T20:30:56.133795+0000","last_fullsized":"2023-09-23T20:30:56.133795+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:12.386098+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:12.386098+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0019500940000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":260,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T20:30:49.020781+0000","last_change":"2023-09-23T20:29:28.364113+0000","last_active":"2023-09-23T20:30:49.020781+0000","last_peered":"2023-09-23T20:30:49.020781+0000","last_clean":"2023-09-23T20:30:49.020781+0000","last_became_active":"2023-09-23T20:29:28.363788+0000","last_became_peered":"2023-09-23T20:29:28.363788+0000","last_unstale":"2023-09-23T20:30:49.020781+0000","last_undegraded":"2023-09-23T20:30:49.020781+0000","last_fullsized":"2023-09-23T20:30:49.020781+0000","mapping_epoch":258,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":259,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:39:49.658803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":262,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-09-23T20:30:56.113783+0000","last_change":"2023-09-23T20:25:02.957010+0000","last_active":"2023-09-23T20:30:56.113783+0000","last_peered":"2023-09-23T20:30:56.113783+0000","last_clean":"2023-09-23T20:30:56.113783+0000","last_became_active":"2023-09-23T20:25:02.956267+0000","last_became_peered":"2023-09-23T20:25:02.956267+0000","last_unstale":"2023-09-23T20:30:56.113783+0000","last_undegraded":"2023-09-23T20:30:56.113783+0000","last_fullsized":"2023-09-23T20:30:56.113783+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:10:55.299422+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.15","version":"287'8713","reported_seq":12506,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-09-23T20:30:55.718599+0000","last_change":"2023-09-23T20:30:01.520686+0000","last_active":"2023-09-23T20:30:55.718599+0000","last_peered":"2023-09-23T20:30:55.718599+0000","last_clean":"2023-09-23T20:30:55.718599+0000","last_became_active":"2023-09-23T20:30:01.486115+0000","last_became_peered":"2023-09-23T20:30:01.486115+0000","last_unstale":"2023-09-23T20:30:55.718599+0000","last_undegraded":"2023-09-23T20:30:55.718599+0000","last_fullsized":"2023-09-23T20:30:55.718599+0000","mapping_epoch":275,"log_start":"221'1061","ondisk_log_start":"221'1061","created":242,"last_epoch_clean":276,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":7652,"log_dups_size":0,"ondisk_log_size":7652,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":28606464,"num_objects":437,"num_object_clones":0,"num_object_copies":1311,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":437,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5973,"num_write_kb":205088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1838,"num_bytes_recovered":97189888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.12","version":"90'161","reported_seq":526,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T20:30:49.033476+0000","last_change":"2023-09-23T20:29:46.079370+0000","last_active":"2023-09-23T20:30:49.033476+0000","last_peered":"2023-09-23T20:30:49.033476+0000","last_clean":"2023-09-23T20:30:49.033476+0000","last_became_active":"2023-09-23T20:29:46.078873+0000","last_became_peered":"2023-09-23T20:29:46.078873+0000","last_unstale":"2023-09-23T20:30:49.033476+0000","last_undegraded":"2023-09-23T20:30:49.033476+0000","last_fullsized":"2023-09-23T20:30:49.033476+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:35.734584+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:35.734584+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":56,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T20:30:49.043486+0000","last_change":"2023-09-23T20:29:04.179171+0000","last_active":"2023-09-23T20:30:49.043486+0000","last_peered":"2023-09-23T20:30:49.043486+0000","last_clean":"2023-09-23T20:30:49.043486+0000","last_became_active":"2023-09-23T20:29:04.128672+0000","last_became_peered":"2023-09-23T20:29:04.128672+0000","last_unstale":"2023-09-23T20:30:49.043486+0000","last_undegraded":"2023-09-23T20:30:49.043486+0000","last_fullsized":"2023-09-23T20:30:49.043486+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:49:15.506106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":151,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-09-23T20:30:56.089461+0000","last_change":"2023-09-23T20:29:45.528782+0000","last_active":"2023-09-23T20:30:56.089461+0000","last_peered":"2023-09-23T20:30:56.089461+0000","last_clean":"2023-09-23T20:30:56.089461+0000","last_became_active":"2023-09-23T20:29:45.528389+0000","last_became_peered":"2023-09-23T20:29:45.528389+0000","last_unstale":"2023-09-23T20:30:56.089461+0000","last_undegraded":"2023-09-23T20:30:56.089461+0000","last_fullsized":"2023-09-23T20:30:56.089461+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:13:41.453499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.d","version":"90'161","reported_seq":526,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T20:30:49.021004+0000","last_change":"2023-09-23T20:29:46.061373+0000","last_active":"2023-09-23T20:30:49.021004+0000","last_peered":"2023-09-23T20:30:49.021004+0000","last_clean":"2023-09-23T20:30:49.021004+0000","last_became_active":"2023-09-23T20:29:46.061003+0000","last_became_peered":"2023-09-23T20:29:46.061003+0000","last_unstale":"2023-09-23T20:30:49.021004+0000","last_undegraded":"2023-09-23T20:30:49.021004+0000","last_fullsized":"2023-09-23T20:30:49.021004+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:12.809933+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:12.809933+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.a","version":"287'12212","reported_seq":23233,"reported_epoch":287,"state":"active+recovery_wait","last_fresh":"2023-09-23T20:30:55.748097+0000","last_change":"2023-09-23T20:29:51.518803+0000","last_active":"2023-09-23T20:30:55.748097+0000","last_peered":"2023-09-23T20:30:55.748097+0000","last_clean":"2023-09-23T20:28:04.001969+0000","last_became_active":"2023-09-23T20:29:51.450026+0000","last_became_peered":"2023-09-23T20:29:51.450026+0000","last_unstale":"2023-09-23T20:30:55.748097+0000","last_undegraded":"2023-09-23T20:30:55.748097+0000","last_fullsized":"2023-09-23T20:30:55.748097+0000","mapping_epoch":271,"log_start":"240'5471","ondisk_log_start":"240'5471","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":6741,"log_dups_size":0,"ondisk_log_size":6741,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:55:12.986375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":62783488,"num_objects":958,"num_object_clones":0,"num_object_copies":2874,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":958,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12212,"num_write_kb":421440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5457,"num_bytes_recovered":313655296,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":["5","0","6"],"object_location_counts":[{"shards":"0,5,6","objects":958}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":276,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T20:30:49.020433+0000","last_change":"2023-09-23T20:29:59.586041+0000","last_active":"2023-09-23T20:30:49.020433+0000","last_peered":"2023-09-23T20:30:49.020433+0000","last_clean":"2023-09-23T20:30:49.020433+0000","last_became_active":"2023-09-23T20:29:46.059906+0000","last_became_peered":"2023-09-23T20:29:46.059906+0000","last_unstale":"2023-09-23T20:30:49.020433+0000","last_undegraded":"2023-09-23T20:30:49.020433+0000","last_fullsized":"2023-09-23T20:30:49.020433+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:26:12.412320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":248,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T20:30:49.031309+0000","last_change":"2023-09-23T20:29:04.128604+0000","last_active":"2023-09-23T20:30:49.031309+0000","last_peered":"2023-09-23T20:30:49.031309+0000","last_clean":"2023-09-23T20:30:49.031309+0000","last_became_active":"2023-09-23T20:29:04.128003+0000","last_became_peered":"2023-09-23T20:29:04.128003+0000","last_unstale":"2023-09-23T20:30:49.031309+0000","last_undegraded":"2023-09-23T20:30:49.031309+0000","last_fullsized":"2023-09-23T20:30:49.031309+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:50:49.385599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"90'161","reported_seq":572,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-09-23T20:30:56.090215+0000","last_change":"2023-09-23T20:30:01.493499+0000","last_active":"2023-09-23T20:30:56.090215+0000","last_peered":"2023-09-23T20:30:56.090215+0000","last_clean":"2023-09-23T20:30:56.090215+0000","last_became_active":"2023-09-23T20:30:01.493025+0000","last_became_peered":"2023-09-23T20:30:01.493025+0000","last_unstale":"2023-09-23T20:30:56.090215+0000","last_undegraded":"2023-09-23T20:30:56.090215+0000","last_fullsized":"2023-09-23T20:30:56.090215+0000","mapping_epoch":275,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":276,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:13.244651+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.244651+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0019955509999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109227,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":655360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.b","version":"289'11898","reported_seq":11992,"reported_epoch":289,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:30:58.374424+0000","last_change":"2023-09-23T20:29:46.285273+0000","last_active":"2023-09-23T20:30:58.374424+0000","last_peered":"2023-09-23T20:30:58.374424+0000","last_clean":"2023-09-23T20:28:12.089537+0000","last_became_active":"2023-09-23T20:29:46.255023+0000","last_became_peered":"2023-09-23T20:29:46.255023+0000","last_unstale":"2023-09-23T20:30:58.374424+0000","last_undegraded":"2023-09-23T20:29:46.058377+0000","last_fullsized":"2023-09-23T20:29:46.031666+0000","mapping_epoch":267,"log_start":"253'7129","ondisk_log_start":"253'7129","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":4769,"log_dups_size":0,"ondisk_log_size":4769,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:37:55.158131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":58851328,"num_objects":898,"num_object_clones":0,"num_object_copies":2694,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4695,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":898,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11898,"num_write_kb":409472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":-1060}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":283,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T20:30:49.020593+0000","last_change":"2023-09-23T20:29:59.567502+0000","last_active":"2023-09-23T20:30:49.020593+0000","last_peered":"2023-09-23T20:30:49.020593+0000","last_clean":"2023-09-23T20:30:49.020593+0000","last_became_active":"2023-09-23T20:29:46.055041+0000","last_became_peered":"2023-09-23T20:29:46.055041+0000","last_unstale":"2023-09-23T20:30:49.020593+0000","last_undegraded":"2023-09-23T20:30:49.020593+0000","last_fullsized":"2023-09-23T20:30:49.020593+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:47:13.874812+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":186,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-09-23T20:30:56.129570+0000","last_change":"2023-09-23T20:29:46.178516+0000","last_active":"2023-09-23T20:30:56.129570+0000","last_peered":"2023-09-23T20:30:56.129570+0000","last_clean":"2023-09-23T20:30:56.129570+0000","last_became_active":"2023-09-23T20:29:46.177858+0000","last_became_peered":"2023-09-23T20:29:46.177858+0000","last_unstale":"2023-09-23T20:30:56.129570+0000","last_undegraded":"2023-09-23T20:30:56.129570+0000","last_fullsized":"2023-09-23T20:30:56.129570+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:09:14.906568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.f","version":"90'161","reported_seq":479,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T20:30:49.044215+0000","last_change":"2023-09-23T20:29:28.374177+0000","last_active":"2023-09-23T20:30:49.044215+0000","last_peered":"2023-09-23T20:30:49.044215+0000","last_clean":"2023-09-23T20:30:49.044215+0000","last_became_active":"2023-09-23T20:29:28.373750+0000","last_became_peered":"2023-09-23T20:29:28.373750+0000","last_unstale":"2023-09-23T20:30:49.044215+0000","last_undegraded":"2023-09-23T20:30:49.044215+0000","last_fullsized":"2023-09-23T20:30:49.044215+0000","mapping_epoch":258,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":259,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:32.746340+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:32.746340+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,3],"acting":[6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.8","version":"287'8813","reported_seq":8902,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-09-23T20:30:55.681998+0000","last_change":"2023-09-23T20:28:35.581921+0000","last_active":"2023-09-23T20:30:55.681998+0000","last_peered":"2023-09-23T20:30:55.681998+0000","last_clean":"2023-09-23T20:30:55.681998+0000","last_became_active":"2023-09-23T20:28:35.576360+0000","last_became_peered":"2023-09-23T20:28:35.576360+0000","last_unstale":"2023-09-23T20:30:55.681998+0000","last_undegraded":"2023-09-23T20:30:55.681998+0000","last_fullsized":"2023-09-23T20:30:55.681998+0000","mapping_epoch":217,"log_start":"221'1087","ondisk_log_start":"221'1087","created":217,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":7726,"log_dups_size":0,"ondisk_log_size":7726,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":33226752,"num_objects":507,"num_object_clones":0,"num_object_copies":1521,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":507,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6027,"num_write_kb":209088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":259,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T20:30:49.021663+0000","last_change":"2023-09-23T20:29:04.184893+0000","last_active":"2023-09-23T20:30:49.021663+0000","last_peered":"2023-09-23T20:30:49.021663+0000","last_clean":"2023-09-23T20:30:49.021663+0000","last_became_active":"2023-09-23T20:29:04.152723+0000","last_became_peered":"2023-09-23T20:29:04.152723+0000","last_unstale":"2023-09-23T20:30:49.021663+0000","last_undegraded":"2023-09-23T20:30:49.021663+0000","last_fullsized":"2023-09-23T20:30:49.021663+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:51:27.017151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":60,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T20:30:49.045706+0000","last_change":"2023-09-23T20:29:46.086912+0000","last_active":"2023-09-23T20:30:49.045706+0000","last_peered":"2023-09-23T20:30:49.045706+0000","last_clean":"2023-09-23T20:30:49.045706+0000","last_became_active":"2023-09-23T20:29:46.085471+0000","last_became_peered":"2023-09-23T20:29:46.085471+0000","last_unstale":"2023-09-23T20:30:49.045706+0000","last_undegraded":"2023-09-23T20:30:49.045706+0000","last_fullsized":"2023-09-23T20:30:49.045706+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:04:55.751660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.e","version":"90'161","reported_seq":466,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T20:30:49.020471+0000","last_change":"2023-09-23T20:29:31.394412+0000","last_active":"2023-09-23T20:30:49.020471+0000","last_peered":"2023-09-23T20:30:49.020471+0000","last_clean":"2023-09-23T20:30:49.020471+0000","last_became_active":"2023-09-23T20:29:31.394053+0000","last_became_peered":"2023-09-23T20:29:31.394053+0000","last_unstale":"2023-09-23T20:30:49.020471+0000","last_undegraded":"2023-09-23T20:30:49.020471+0000","last_fullsized":"2023-09-23T20:30:49.020471+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:31.769333+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:31.769333+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.9","version":"289'9019","reported_seq":9111,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:30:59.946142+0000","last_change":"2023-09-23T20:30:22.842563+0000","last_active":"2023-09-23T20:30:59.946142+0000","last_peered":"2023-09-23T20:30:59.946142+0000","last_clean":"2023-09-23T20:30:59.946142+0000","last_became_active":"2023-09-23T20:28:35.536824+0000","last_became_peered":"2023-09-23T20:28:35.536824+0000","last_unstale":"2023-09-23T20:30:59.946142+0000","last_undegraded":"2023-09-23T20:30:59.946142+0000","last_fullsized":"2023-09-23T20:30:59.946142+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":9019,"log_dups_size":0,"ondisk_log_size":9019,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":21561344,"num_objects":329,"num_object_clones":0,"num_object_copies":987,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":329,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6223,"num_write_kb":209664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":293,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T20:30:49.032941+0000","last_change":"2023-09-23T20:29:04.138284+0000","last_active":"2023-09-23T20:30:49.032941+0000","last_peered":"2023-09-23T20:30:49.032941+0000","last_clean":"2023-09-23T20:30:49.032941+0000","last_became_active":"2023-09-23T20:29:04.137960+0000","last_became_peered":"2023-09-23T20:29:04.137960+0000","last_unstale":"2023-09-23T20:30:49.032941+0000","last_undegraded":"2023-09-23T20:30:49.032941+0000","last_fullsized":"2023-09-23T20:30:49.032941+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:18:59.548311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":261,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-09-23T20:30:56.113187+0000","last_change":"2023-09-23T20:29:46.061136+0000","last_active":"2023-09-23T20:30:56.113187+0000","last_peered":"2023-09-23T20:30:56.113187+0000","last_clean":"2023-09-23T20:30:56.113187+0000","last_became_active":"2023-09-23T20:29:46.060777+0000","last_became_peered":"2023-09-23T20:29:46.060777+0000","last_unstale":"2023-09-23T20:30:56.113187+0000","last_undegraded":"2023-09-23T20:30:56.113187+0000","last_fullsized":"2023-09-23T20:30:56.113187+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:08:43.673870+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"289'12107","reported_seq":16072,"reported_epoch":289,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T20:30:57.332458+0000","last_change":"2023-09-23T20:29:59.598466+0000","last_active":"2023-09-23T20:30:57.332458+0000","last_peered":"2023-09-23T20:30:57.332458+0000","last_clean":"2023-09-23T20:28:12.106078+0000","last_became_active":"2023-09-23T20:29:45.690954+0000","last_became_peered":"2023-09-23T20:29:45.690954+0000","last_unstale":"2023-09-23T20:30:57.332458+0000","last_undegraded":"2023-09-23T20:29:45.686952+0000","last_fullsized":"2023-09-23T20:29:45.501064+0000","mapping_epoch":266,"log_start":"245'5572","ondisk_log_start":"245'5572","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":6535,"log_dups_size":0,"ondisk_log_size":6535,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:52:59.514150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":65208320,"num_objects":995,"num_object_clones":0,"num_object_copies":2985,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":98,"num_objects_misplaced":98,"num_objects_unfound":0,"num_objects_dirty":995,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12107,"num_write_kb":419264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1933,"num_bytes_recovered":126550016,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[2,4],"avail_no_missing":["2","4"],"object_location_counts":[{"shards":"2,4","objects":897}],"blocked_by":[],"up_primary":1,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":669,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T20:30:49.020735+0000","last_change":"2023-09-23T20:28:05.872149+0000","last_active":"2023-09-23T20:30:49.020735+0000","last_peered":"2023-09-23T20:30:49.020735+0000","last_clean":"2023-09-23T20:30:49.020735+0000","last_became_active":"2023-09-23T20:27:11.961310+0000","last_became_peered":"2023-09-23T20:27:11.961310+0000","last_unstale":"2023-09-23T20:30:49.020735+0000","last_undegraded":"2023-09-23T20:30:49.020735+0000","last_fullsized":"2023-09-23T20:30:49.020735+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:28:05.872074+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:28:05.872074+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:52:32.495955+0000","scrub_duration":0.0029215980000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":164,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-09-23T20:30:56.089033+0000","last_change":"2023-09-23T20:29:49.873018+0000","last_active":"2023-09-23T20:30:56.089033+0000","last_peered":"2023-09-23T20:30:56.089033+0000","last_clean":"2023-09-23T20:30:56.089033+0000","last_became_active":"2023-09-23T20:29:46.070369+0000","last_became_peered":"2023-09-23T20:29:46.070369+0000","last_unstale":"2023-09-23T20:30:56.089033+0000","last_undegraded":"2023-09-23T20:30:56.089033+0000","last_fullsized":"2023-09-23T20:30:56.089033+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:24:13.119725+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":300,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-09-23T20:30:56.094547+0000","last_change":"2023-09-23T20:29:46.173195+0000","last_active":"2023-09-23T20:30:56.094547+0000","last_peered":"2023-09-23T20:30:56.094547+0000","last_clean":"2023-09-23T20:30:56.094547+0000","last_became_active":"2023-09-23T20:29:46.172808+0000","last_became_peered":"2023-09-23T20:29:46.172808+0000","last_unstale":"2023-09-23T20:30:56.094547+0000","last_undegraded":"2023-09-23T20:30:56.094547+0000","last_fullsized":"2023-09-23T20:30:56.094547+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:41:58.927658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.f","version":"289'12244","reported_seq":12335,"reported_epoch":289,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T20:30:57.229227+0000","last_change":"2023-09-23T20:29:29.530059+0000","last_active":"2023-09-23T20:30:57.229227+0000","last_peered":"2023-09-23T20:30:57.229227+0000","last_clean":"2023-09-23T20:29:27.332951+0000","last_became_active":"2023-09-23T20:29:29.505420+0000","last_became_peered":"2023-09-23T20:29:29.505420+0000","last_unstale":"2023-09-23T20:30:57.229227+0000","last_undegraded":"2023-09-23T20:30:57.229227+0000","last_fullsized":"2023-09-23T20:30:57.229227+0000","mapping_epoch":259,"log_start":"245'5687","ondisk_log_start":"245'5687","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":6557,"log_dups_size":0,"ondisk_log_size":6557,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:18:15.383066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":65929216,"num_objects":1006,"num_object_clones":0,"num_object_copies":3018,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1006,"num_objects_unfound":0,"num_objects_dirty":1006,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12244,"num_write_kb":424000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,4],"avail_no_missing":["2","3","4"],"object_location_counts":[{"shards":"2,3,4","objects":1006}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":511,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T20:30:49.031176+0000","last_change":"2023-09-23T20:27:31.757060+0000","last_active":"2023-09-23T20:30:49.031176+0000","last_peered":"2023-09-23T20:30:49.031176+0000","last_clean":"2023-09-23T20:30:49.031176+0000","last_became_active":"2023-09-23T20:27:11.959516+0000","last_became_peered":"2023-09-23T20:27:11.959516+0000","last_unstale":"2023-09-23T20:30:49.031176+0000","last_undegraded":"2023-09-23T20:30:49.031176+0000","last_fullsized":"2023-09-23T20:30:49.031176+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:31.756974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:31.756974+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:35:11.939335+0000","scrub_duration":0.0031802670000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":72,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T20:30:49.045805+0000","last_change":"2023-09-23T20:29:46.087097+0000","last_active":"2023-09-23T20:30:49.045805+0000","last_peered":"2023-09-23T20:30:49.045805+0000","last_clean":"2023-09-23T20:30:49.045805+0000","last_became_active":"2023-09-23T20:29:46.085908+0000","last_became_peered":"2023-09-23T20:29:46.085908+0000","last_unstale":"2023-09-23T20:30:49.045805+0000","last_undegraded":"2023-09-23T20:30:49.045805+0000","last_fullsized":"2023-09-23T20:30:49.045805+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:38:46.001432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":198,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-09-23T20:30:56.103151+0000","last_change":"2023-09-23T20:29:28.370476+0000","last_active":"2023-09-23T20:30:56.103151+0000","last_peered":"2023-09-23T20:30:56.103151+0000","last_clean":"2023-09-23T20:30:56.103151+0000","last_became_active":"2023-09-23T20:29:28.370158+0000","last_became_peered":"2023-09-23T20:29:28.370158+0000","last_unstale":"2023-09-23T20:30:56.103151+0000","last_undegraded":"2023-09-23T20:30:56.103151+0000","last_fullsized":"2023-09-23T20:30:56.103151+0000","mapping_epoch":258,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":259,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:09:47.335404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.b","version":"90'161","reported_seq":517,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-09-23T20:30:56.133839+0000","last_change":"2023-09-23T20:29:45.534549+0000","last_active":"2023-09-23T20:30:56.133839+0000","last_peered":"2023-09-23T20:30:56.133839+0000","last_clean":"2023-09-23T20:30:56.133839+0000","last_became_active":"2023-09-23T20:29:45.534070+0000","last_became_peered":"2023-09-23T20:29:45.534070+0000","last_unstale":"2023-09-23T20:30:56.133839+0000","last_undegraded":"2023-09-23T20:30:56.133839+0000","last_fullsized":"2023-09-23T20:30:56.133839+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:12.386098+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:12.386098+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0019500940000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.c","version":"289'11941","reported_seq":15435,"reported_epoch":289,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:30:58.255290+0000","last_change":"2023-09-23T20:29:59.529817+0000","last_active":"2023-09-23T20:30:58.255290+0000","last_peered":"2023-09-23T20:30:58.255290+0000","last_clean":"2023-09-23T20:28:03.999709+0000","last_became_active":"2023-09-23T20:29:04.247715+0000","last_became_peered":"2023-09-23T20:29:04.247715+0000","last_unstale":"2023-09-23T20:30:58.255290+0000","last_undegraded":"2023-09-23T20:29:59.527865+0000","last_fullsized":"2023-09-23T20:29:04.105278+0000","mapping_epoch":250,"log_start":"253'7171","ondisk_log_start":"253'7171","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":4770,"log_dups_size":0,"ondisk_log_size":4770,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:01:13.556577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":63504384,"num_objects":969,"num_object_clones":0,"num_object_copies":2907,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2952,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":969,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11941,"num_write_kb":413120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1694,"num_bytes_recovered":110886912,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,5],"avail_no_missing":["3","5"],"object_location_counts":[{"shards":"3,5","objects":-1331},{"shards":"3,5,7","objects":969}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":200,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-09-23T20:30:56.105861+0000","last_change":"2023-09-23T20:29:46.061620+0000","last_active":"2023-09-23T20:30:56.105861+0000","last_peered":"2023-09-23T20:30:56.105861+0000","last_clean":"2023-09-23T20:30:56.105861+0000","last_became_active":"2023-09-23T20:29:46.061271+0000","last_became_peered":"2023-09-23T20:29:46.061271+0000","last_unstale":"2023-09-23T20:30:56.105861+0000","last_undegraded":"2023-09-23T20:30:56.105861+0000","last_fullsized":"2023-09-23T20:30:56.105861+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:24:51.409521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":282,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T20:30:49.043761+0000","last_change":"2023-09-23T20:29:46.085962+0000","last_active":"2023-09-23T20:30:49.043761+0000","last_peered":"2023-09-23T20:30:49.043761+0000","last_clean":"2023-09-23T20:30:49.043761+0000","last_became_active":"2023-09-23T20:29:46.082826+0000","last_became_peered":"2023-09-23T20:29:46.082826+0000","last_unstale":"2023-09-23T20:30:49.043761+0000","last_undegraded":"2023-09-23T20:30:49.043761+0000","last_fullsized":"2023-09-23T20:30:49.043761+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:26:17.599051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.d","version":"287'12181","reported_seq":12287,"reported_epoch":287,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:30:55.732079+0000","last_change":"2023-09-23T20:29:59.564467+0000","last_active":"2023-09-23T20:30:55.732079+0000","last_peered":"2023-09-23T20:30:55.732079+0000","last_clean":"2023-09-23T20:28:12.085747+0000","last_became_active":"2023-09-23T20:29:46.294362+0000","last_became_peered":"2023-09-23T20:29:46.294362+0000","last_unstale":"2023-09-23T20:30:55.732079+0000","last_undegraded":"2023-09-23T20:29:46.239165+0000","last_fullsized":"2023-09-23T20:29:46.032265+0000","mapping_epoch":267,"log_start":"240'5459","ondisk_log_start":"240'5459","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":6722,"log_dups_size":0,"ondisk_log_size":6722,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:38:13.504714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":68157464,"num_objects":1041,"num_object_clones":0,"num_object_copies":3123,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4915,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1041,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":12181,"num_write_kb":423041,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,0],"avail_no_missing":["5","0"],"object_location_counts":[{"shards":"0,5","objects":-975}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":551,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T20:30:49.033220+0000","last_change":"2023-09-23T20:29:07.053561+0000","last_active":"2023-09-23T20:30:49.033220+0000","last_peered":"2023-09-23T20:30:49.033220+0000","last_clean":"2023-09-23T20:30:49.033220+0000","last_became_active":"2023-09-23T20:29:07.053177+0000","last_became_peered":"2023-09-23T20:29:07.053177+0000","last_unstale":"2023-09-23T20:30:49.033220+0000","last_undegraded":"2023-09-23T20:30:49.033220+0000","last_fullsized":"2023-09-23T20:30:49.033220+0000","mapping_epoch":253,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":254,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:30.727267+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:30.727267+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:04:32.661761+0000","scrub_duration":0.0052560289999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":151,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-09-23T20:30:56.087663+0000","last_change":"2023-09-23T20:29:45.529576+0000","last_active":"2023-09-23T20:30:56.087663+0000","last_peered":"2023-09-23T20:30:56.087663+0000","last_clean":"2023-09-23T20:30:56.087663+0000","last_became_active":"2023-09-23T20:29:45.528858+0000","last_became_peered":"2023-09-23T20:29:45.528858+0000","last_unstale":"2023-09-23T20:30:56.087663+0000","last_undegraded":"2023-09-23T20:30:56.087663+0000","last_fullsized":"2023-09-23T20:30:56.087663+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:42:15.589821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":99,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-09-23T20:30:56.132992+0000","last_change":"2023-09-23T20:29:46.064515+0000","last_active":"2023-09-23T20:30:56.132992+0000","last_peered":"2023-09-23T20:30:56.132992+0000","last_clean":"2023-09-23T20:30:56.132992+0000","last_became_active":"2023-09-23T20:29:46.064253+0000","last_became_peered":"2023-09-23T20:29:46.064253+0000","last_unstale":"2023-09-23T20:30:56.132992+0000","last_undegraded":"2023-09-23T20:30:56.132992+0000","last_fullsized":"2023-09-23T20:30:56.132992+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:21:20.039440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"289'9059","reported_seq":9152,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:30:59.842045+0000","last_change":"2023-09-23T20:30:23.834705+0000","last_active":"2023-09-23T20:30:59.842045+0000","last_peered":"2023-09-23T20:30:59.842045+0000","last_clean":"2023-09-23T20:30:59.842045+0000","last_became_active":"2023-09-23T20:28:35.532298+0000","last_became_peered":"2023-09-23T20:28:35.532298+0000","last_unstale":"2023-09-23T20:30:59.842045+0000","last_undegraded":"2023-09-23T20:30:59.842045+0000","last_fullsized":"2023-09-23T20:30:59.842045+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":9059,"log_dups_size":0,"ondisk_log_size":9059,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":27688960,"num_objects":422,"num_object_clones":0,"num_object_copies":1266,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":422,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6240,"num_write_kb":213216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":506,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T20:30:49.044259+0000","last_change":"2023-09-23T20:29:07.115982+0000","last_active":"2023-09-23T20:30:49.044259+0000","last_peered":"2023-09-23T20:30:49.044259+0000","last_clean":"2023-09-23T20:30:49.044259+0000","last_became_active":"2023-09-23T20:29:07.115647+0000","last_became_peered":"2023-09-23T20:29:07.115647+0000","last_unstale":"2023-09-23T20:30:49.044259+0000","last_undegraded":"2023-09-23T20:30:49.044259+0000","last_fullsized":"2023-09-23T20:30:49.044259+0000","mapping_epoch":253,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":254,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:32.746340+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:32.746340+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"86'119","reported_seq":592,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-09-23T20:30:56.134633+0000","last_change":"2023-09-23T20:29:45.521257+0000","last_active":"2023-09-23T20:30:56.134633+0000","last_peered":"2023-09-23T20:30:56.134633+0000","last_clean":"2023-09-23T20:30:56.134633+0000","last_became_active":"2023-09-23T20:29:45.520546+0000","last_became_peered":"2023-09-23T20:29:45.520546+0000","last_unstale":"2023-09-23T20:30:56.134633+0000","last_undegraded":"2023-09-23T20:30:56.134633+0000","last_fullsized":"2023-09-23T20:30:56.134633+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":119,"log_dups_size":0,"ondisk_log_size":119,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:58:56.085777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032705499999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":523,"num_read_kb":3503,"num_write":136,"num_write_kb":3128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":303,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T20:30:49.034006+0000","last_change":"2023-09-23T20:25:39.637160+0000","last_active":"2023-09-23T20:30:49.034006+0000","last_peered":"2023-09-23T20:30:49.034006+0000","last_clean":"2023-09-23T20:30:49.034006+0000","last_became_active":"2023-09-23T20:25:39.636860+0000","last_became_peered":"2023-09-23T20:25:39.636860+0000","last_unstale":"2023-09-23T20:30:49.034006+0000","last_undegraded":"2023-09-23T20:30:49.034006+0000","last_fullsized":"2023-09-23T20:30:49.034006+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:21:47.860063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":194,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-09-23T20:30:56.128087+0000","last_change":"2023-09-23T20:29:46.090683+0000","last_active":"2023-09-23T20:30:56.128087+0000","last_peered":"2023-09-23T20:30:56.128087+0000","last_clean":"2023-09-23T20:30:56.128087+0000","last_became_active":"2023-09-23T20:29:46.090131+0000","last_became_peered":"2023-09-23T20:29:46.090131+0000","last_unstale":"2023-09-23T20:30:56.128087+0000","last_undegraded":"2023-09-23T20:30:56.128087+0000","last_fullsized":"2023-09-23T20:30:56.128087+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:36:35.754684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.6","version":"289'8811","reported_seq":8912,"reported_epoch":289,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T20:30:58.351635+0000","last_change":"2023-09-23T20:29:29.535022+0000","last_active":"2023-09-23T20:30:58.351635+0000","last_peered":"2023-09-23T20:30:58.351635+0000","last_clean":"2023-09-23T20:29:27.350688+0000","last_became_active":"2023-09-23T20:29:29.522984+0000","last_became_peered":"2023-09-23T20:29:29.522984+0000","last_unstale":"2023-09-23T20:30:58.351635+0000","last_undegraded":"2023-09-23T20:30:58.351635+0000","last_fullsized":"2023-09-23T20:30:58.351635+0000","mapping_epoch":259,"log_start":"224'2803","ondisk_log_start":"224'2803","created":217,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":6008,"log_dups_size":0,"ondisk_log_size":6008,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":27066368,"num_objects":413,"num_object_clones":0,"num_object_copies":1239,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":413,"num_objects_unfound":0,"num_objects_dirty":413,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6069,"num_write_kb":207424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,4],"avail_no_missing":["3","4","7"],"object_location_counts":[{"shards":"3,4,7","objects":413}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":521,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T20:30:49.032645+0000","last_change":"2023-09-23T20:27:55.213707+0000","last_active":"2023-09-23T20:30:49.032645+0000","last_peered":"2023-09-23T20:30:49.032645+0000","last_clean":"2023-09-23T20:30:49.032645+0000","last_became_active":"2023-09-23T20:27:55.213394+0000","last_became_peered":"2023-09-23T20:27:55.213394+0000","last_unstale":"2023-09-23T20:30:49.032645+0000","last_undegraded":"2023-09-23T20:30:49.032645+0000","last_fullsized":"2023-09-23T20:30:49.032645+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:34.821789+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:34.821789+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0042962870000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"86'106","reported_seq":981,"reported_epoch":286,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T20:30:49.032960+0000","last_change":"2023-09-23T20:29:46.078905+0000","last_active":"2023-09-23T20:30:49.032960+0000","last_peered":"2023-09-23T20:30:49.032960+0000","last_clean":"2023-09-23T20:28:10.123134+0000","last_became_active":"2023-09-23T20:29:46.078501+0000","last_became_peered":"2023-09-23T20:29:46.078501+0000","last_unstale":"2023-09-23T20:30:49.032960+0000","last_undegraded":"2023-09-23T20:29:46.032390+0000","last_fullsized":"2023-09-23T20:30:49.032960+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":106,"log_dups_size":0,"ondisk_log_size":106,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:37.015974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00099302599999999993,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1105,"num_read_kb":1689,"num_write":100,"num_write_kb":404,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":["5","0","1"],"object_location_counts":[{"shards":"0,1,5","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":281,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-09-23T20:30:56.113469+0000","last_change":"2023-09-23T20:29:46.059640+0000","last_active":"2023-09-23T20:30:56.113469+0000","last_peered":"2023-09-23T20:30:56.113469+0000","last_clean":"2023-09-23T20:30:56.113469+0000","last_became_active":"2023-09-23T20:29:46.059279+0000","last_became_peered":"2023-09-23T20:29:46.059279+0000","last_unstale":"2023-09-23T20:30:56.113469+0000","last_undegraded":"2023-09-23T20:30:56.113469+0000","last_fullsized":"2023-09-23T20:30:56.113469+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:03:31.536847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":245,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-09-23T20:30:56.134555+0000","last_change":"2023-09-23T20:29:45.533406+0000","last_active":"2023-09-23T20:30:56.134555+0000","last_peered":"2023-09-23T20:30:56.134555+0000","last_clean":"2023-09-23T20:30:56.134555+0000","last_became_active":"2023-09-23T20:29:45.526001+0000","last_became_peered":"2023-09-23T20:29:45.526001+0000","last_unstale":"2023-09-23T20:30:56.134555+0000","last_undegraded":"2023-09-23T20:30:56.134555+0000","last_fullsized":"2023-09-23T20:30:56.134555+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:43.787251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.5","version":"287'8713","reported_seq":12261,"reported_epoch":287,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T20:30:55.719563+0000","last_change":"2023-09-23T20:30:45.014679+0000","last_active":"2023-09-23T20:30:55.719563+0000","last_peered":"2023-09-23T20:30:55.719563+0000","last_clean":"2023-09-23T20:28:04.012743+0000","last_became_active":"2023-09-23T20:29:05.259417+0000","last_became_peered":"2023-09-23T20:29:05.259417+0000","last_unstale":"2023-09-23T20:30:55.719563+0000","last_undegraded":"2023-09-23T20:29:05.129039+0000","last_fullsized":"2023-09-23T20:29:05.113332+0000","mapping_epoch":251,"log_start":"221'1015","ondisk_log_start":"221'1015","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":7698,"log_dups_size":0,"ondisk_log_size":7698,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":36995072,"num_objects":564,"num_object_clones":0,"num_object_copies":1692,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":13,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":564,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5972,"num_write_kb":209184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1720,"num_bytes_recovered":70320128,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,3],"avail_no_missing":["7","3"],"object_location_counts":[{"shards":"3,7","objects":551}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":699,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T20:30:49.019964+0000","last_change":"2023-09-23T20:29:31.389645+0000","last_active":"2023-09-23T20:30:49.019964+0000","last_peered":"2023-09-23T20:30:49.019964+0000","last_clean":"2023-09-23T20:30:49.019964+0000","last_became_active":"2023-09-23T20:29:31.389288+0000","last_became_peered":"2023-09-23T20:29:31.389288+0000","last_unstale":"2023-09-23T20:30:49.019964+0000","last_undegraded":"2023-09-23T20:30:49.019964+0000","last_fullsized":"2023-09-23T20:30:49.019964+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:35.734584+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:35.734584+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0035251380000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,2],"acting":[7,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"86'130","reported_seq":600,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-09-23T20:30:56.105758+0000","last_change":"2023-09-23T20:25:39.637458+0000","last_active":"2023-09-23T20:30:56.105758+0000","last_peered":"2023-09-23T20:30:56.105758+0000","last_clean":"2023-09-23T20:30:56.105758+0000","last_became_active":"2023-09-23T20:25:39.636773+0000","last_became_peered":"2023-09-23T20:25:39.636773+0000","last_unstale":"2023-09-23T20:30:56.105758+0000","last_undegraded":"2023-09-23T20:30:56.105758+0000","last_fullsized":"2023-09-23T20:30:56.105758+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:00:32.810564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011300049999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":512,"num_read_kb":1820,"num_write":133,"num_write_kb":1347,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":235,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-09-23T20:30:56.122227+0000","last_change":"2023-09-23T20:29:46.073394+0000","last_active":"2023-09-23T20:30:56.122227+0000","last_peered":"2023-09-23T20:30:56.122227+0000","last_clean":"2023-09-23T20:30:56.122227+0000","last_became_active":"2023-09-23T20:29:46.072067+0000","last_became_peered":"2023-09-23T20:29:46.072067+0000","last_unstale":"2023-09-23T20:30:56.122227+0000","last_undegraded":"2023-09-23T20:30:56.122227+0000","last_fullsized":"2023-09-23T20:30:56.122227+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:01:56.944614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":227,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-09-23T20:30:56.132432+0000","last_change":"2023-09-23T20:29:45.533185+0000","last_active":"2023-09-23T20:30:56.132432+0000","last_peered":"2023-09-23T20:30:56.132432+0000","last_clean":"2023-09-23T20:30:56.132432+0000","last_became_active":"2023-09-23T20:29:45.526765+0000","last_became_peered":"2023-09-23T20:29:45.526765+0000","last_unstale":"2023-09-23T20:30:56.132432+0000","last_undegraded":"2023-09-23T20:30:56.132432+0000","last_fullsized":"2023-09-23T20:30:56.132432+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:18:33.063215+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"287'8956","reported_seq":9064,"reported_epoch":287,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:30:55.791141+0000","last_change":"2023-09-23T20:29:46.386541+0000","last_active":"2023-09-23T20:30:55.791141+0000","last_peered":"2023-09-23T20:30:55.791141+0000","last_clean":"2023-09-23T20:29:27.161343+0000","last_became_active":"2023-09-23T20:29:46.365588+0000","last_became_peered":"2023-09-23T20:29:46.365588+0000","last_unstale":"2023-09-23T20:30:55.791141+0000","last_undegraded":"2023-09-23T20:29:46.092299+0000","last_fullsized":"2023-09-23T20:29:46.032978+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":8956,"log_dups_size":0,"ondisk_log_size":8956,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":29589504,"num_objects":451,"num_object_clones":0,"num_object_copies":1353,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3300,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":451,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6127,"num_write_kb":210528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3],"avail_no_missing":["5","3","4"],"object_location_counts":[{"shards":"3,4,5","objects":462},{"shards":"3,5","objects":-1399}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"86'194","reported_seq":935,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T20:30:49.033500+0000","last_change":"2023-09-23T20:25:40.618665+0000","last_active":"2023-09-23T20:30:49.033500+0000","last_peered":"2023-09-23T20:30:49.033500+0000","last_clean":"2023-09-23T20:30:49.033500+0000","last_became_active":"2023-09-23T20:25:40.618087+0000","last_became_peered":"2023-09-23T20:25:40.618087+0000","last_unstale":"2023-09-23T20:30:49.033500+0000","last_undegraded":"2023-09-23T20:30:49.033500+0000","last_fullsized":"2023-09-23T20:30:49.033500+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":194,"log_dups_size":0,"ondisk_log_size":194,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:07:16.114968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00037995999999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1025,"num_read_kb":1201,"num_write":173,"num_write_kb":1276,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":710,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T20:30:49.019792+0000","last_change":"2023-09-23T20:29:07.042758+0000","last_active":"2023-09-23T20:30:49.019792+0000","last_peered":"2023-09-23T20:30:49.019792+0000","last_clean":"2023-09-23T20:30:49.019792+0000","last_became_active":"2023-09-23T20:29:07.042407+0000","last_became_peered":"2023-09-23T20:29:07.042407+0000","last_unstale":"2023-09-23T20:30:49.019792+0000","last_undegraded":"2023-09-23T20:30:49.019792+0000","last_fullsized":"2023-09-23T20:30:49.019792+0000","mapping_epoch":253,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":254,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:32.801168+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:32.801168+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0096333110000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":154,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-09-23T20:30:56.088394+0000","last_change":"2023-09-23T20:29:04.216400+0000","last_active":"2023-09-23T20:30:56.088394+0000","last_peered":"2023-09-23T20:30:56.088394+0000","last_clean":"2023-09-23T20:30:56.088394+0000","last_became_active":"2023-09-23T20:29:04.216069+0000","last_became_peered":"2023-09-23T20:29:04.216069+0000","last_unstale":"2023-09-23T20:30:56.088394+0000","last_undegraded":"2023-09-23T20:30:56.088394+0000","last_fullsized":"2023-09-23T20:30:56.088394+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:27:37.433035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":56,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T20:30:49.045970+0000","last_change":"2023-09-23T20:29:28.376036+0000","last_active":"2023-09-23T20:30:49.045970+0000","last_peered":"2023-09-23T20:30:49.045970+0000","last_clean":"2023-09-23T20:30:49.045970+0000","last_became_active":"2023-09-23T20:29:28.375712+0000","last_became_peered":"2023-09-23T20:29:28.375712+0000","last_unstale":"2023-09-23T20:30:49.045970+0000","last_undegraded":"2023-09-23T20:30:49.045970+0000","last_fullsized":"2023-09-23T20:30:49.045970+0000","mapping_epoch":258,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":259,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:23:50.722625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.4","version":"289'8847","reported_seq":8947,"reported_epoch":289,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:30:58.350934+0000","last_change":"2023-09-23T20:29:46.186959+0000","last_active":"2023-09-23T20:30:58.350934+0000","last_peered":"2023-09-23T20:30:58.350934+0000","last_clean":"2023-09-23T20:28:12.083147+0000","last_became_active":"2023-09-23T20:29:46.162334+0000","last_became_peered":"2023-09-23T20:29:46.162334+0000","last_unstale":"2023-09-23T20:30:58.350934+0000","last_undegraded":"2023-09-23T20:29:46.054138+0000","last_fullsized":"2023-09-23T20:29:46.043728+0000","mapping_epoch":267,"log_start":"223'2726","ondisk_log_start":"223'2726","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":6121,"log_dups_size":0,"ondisk_log_size":6121,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":35586048,"num_objects":543,"num_object_clones":0,"num_object_copies":1629,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2427,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":543,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6083,"num_write_kb":212032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[3,5],"avail_no_missing":["3","5"],"object_location_counts":[{"shards":"3,5","objects":-500}],"blocked_by":[],"up_primary":1,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":517,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-09-23T20:30:56.133749+0000","last_change":"2023-09-23T20:29:46.072303+0000","last_active":"2023-09-23T20:30:56.133749+0000","last_peered":"2023-09-23T20:30:56.133749+0000","last_clean":"2023-09-23T20:30:56.133749+0000","last_became_active":"2023-09-23T20:29:46.071938+0000","last_became_peered":"2023-09-23T20:29:46.071938+0000","last_unstale":"2023-09-23T20:30:56.133749+0000","last_undegraded":"2023-09-23T20:30:56.133749+0000","last_fullsized":"2023-09-23T20:30:56.133749+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:12.386098+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:12.386098+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,5],"acting":[1,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"86'80","reported_seq":550,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T20:30:49.019704+0000","last_change":"2023-09-23T20:29:45.536796+0000","last_active":"2023-09-23T20:30:49.019704+0000","last_peered":"2023-09-23T20:30:49.019704+0000","last_clean":"2023-09-23T20:30:49.019704+0000","last_became_active":"2023-09-23T20:29:45.536439+0000","last_became_peered":"2023-09-23T20:29:45.536439+0000","last_unstale":"2023-09-23T20:30:49.019704+0000","last_undegraded":"2023-09-23T20:30:49.019704+0000","last_fullsized":"2023-09-23T20:30:49.019704+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":80,"log_dups_size":0,"ondisk_log_size":80,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:30:57.793132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035115,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":395,"num_read_kb":1370,"num_write":66,"num_write_kb":1061,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":262,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-09-23T20:30:56.111822+0000","last_change":"2023-09-23T20:29:28.360680+0000","last_active":"2023-09-23T20:30:56.111822+0000","last_peered":"2023-09-23T20:30:56.111822+0000","last_clean":"2023-09-23T20:30:56.111822+0000","last_became_active":"2023-09-23T20:29:28.360366+0000","last_became_peered":"2023-09-23T20:29:28.360366+0000","last_unstale":"2023-09-23T20:30:56.111822+0000","last_undegraded":"2023-09-23T20:30:56.111822+0000","last_fullsized":"2023-09-23T20:30:56.111822+0000","mapping_epoch":258,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":259,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:02:40.594828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":284,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:30:57.091622+0000","last_change":"2023-09-23T20:29:45.690815+0000","last_active":"2023-09-23T20:30:57.091622+0000","last_peered":"2023-09-23T20:30:57.091622+0000","last_clean":"2023-09-23T20:30:57.091622+0000","last_became_active":"2023-09-23T20:29:45.690478+0000","last_became_peered":"2023-09-23T20:29:45.690478+0000","last_unstale":"2023-09-23T20:30:57.091622+0000","last_undegraded":"2023-09-23T20:30:57.091622+0000","last_fullsized":"2023-09-23T20:30:57.091622+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:38.288193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"287'8728","reported_seq":13866,"reported_epoch":287,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:30:55.437311+0000","last_change":"2023-09-23T20:29:59.519844+0000","last_active":"2023-09-23T20:30:55.437311+0000","last_peered":"2023-09-23T20:30:55.437311+0000","last_clean":"2023-09-23T20:28:04.011715+0000","last_became_active":"2023-09-23T20:29:46.206627+0000","last_became_peered":"2023-09-23T20:29:46.206627+0000","last_unstale":"2023-09-23T20:30:55.437311+0000","last_undegraded":"2023-09-23T20:29:59.519468+0000","last_fullsized":"2023-09-23T20:29:46.046256+0000","mapping_epoch":267,"log_start":"221'1029","ondisk_log_start":"221'1029","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":7699,"log_dups_size":0,"ondisk_log_size":7699,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":31457280,"num_objects":480,"num_object_clones":0,"num_object_copies":1440,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1729,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":480,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5974,"num_write_kb":206528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2160,"num_bytes_recovered":141557760,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[6,2],"avail_no_missing":["6","2"],"object_location_counts":[{"shards":"2,6","objects":152},{"shards":"2,6,7","objects":328}],"blocked_by":[],"up_primary":1,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":499,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T20:30:49.043904+0000","last_change":"2023-09-23T20:29:07.113747+0000","last_active":"2023-09-23T20:30:49.043904+0000","last_peered":"2023-09-23T20:30:49.043904+0000","last_clean":"2023-09-23T20:30:49.043904+0000","last_became_active":"2023-09-23T20:29:07.113428+0000","last_became_peered":"2023-09-23T20:29:07.113428+0000","last_unstale":"2023-09-23T20:30:49.043904+0000","last_undegraded":"2023-09-23T20:30:49.043904+0000","last_fullsized":"2023-09-23T20:30:49.043904+0000","mapping_epoch":253,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":254,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:31.769333+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:31.769333+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"86'102","reported_seq":815,"reported_epoch":286,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:30:49.019593+0000","last_change":"2023-09-23T20:29:46.060111+0000","last_active":"2023-09-23T20:30:49.019593+0000","last_peered":"2023-09-23T20:30:49.019593+0000","last_clean":"2023-09-23T20:29:43.509457+0000","last_became_active":"2023-09-23T20:29:46.059787+0000","last_became_peered":"2023-09-23T20:29:46.059787+0000","last_unstale":"2023-09-23T20:30:49.019593+0000","last_undegraded":"2023-09-23T20:29:46.049994+0000","last_fullsized":"2023-09-23T20:29:46.048495+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":260,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":102,"log_dups_size":0,"ondisk_log_size":102,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:53:45.933578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030656500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":963,"num_read_kb":1234,"num_write":60,"num_write_kb":225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0],"avail_no_missing":["7","0","4"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":211,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-09-23T20:30:56.134717+0000","last_change":"2023-09-23T20:29:45.538283+0000","last_active":"2023-09-23T20:30:56.134717+0000","last_peered":"2023-09-23T20:30:56.134717+0000","last_clean":"2023-09-23T20:30:56.134717+0000","last_became_active":"2023-09-23T20:29:45.537922+0000","last_became_peered":"2023-09-23T20:29:45.537922+0000","last_unstale":"2023-09-23T20:30:56.134717+0000","last_undegraded":"2023-09-23T20:30:56.134717+0000","last_fullsized":"2023-09-23T20:30:56.134717+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:30:52.397836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":221,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-09-23T20:30:56.113398+0000","last_change":"2023-09-23T20:20:14.888874+0000","last_active":"2023-09-23T20:30:56.113398+0000","last_peered":"2023-09-23T20:30:56.113398+0000","last_clean":"2023-09-23T20:30:56.113398+0000","last_became_active":"2023-09-23T20:20:14.887724+0000","last_became_peered":"2023-09-23T20:20:14.887724+0000","last_unstale":"2023-09-23T20:30:56.113398+0000","last_undegraded":"2023-09-23T20:30:56.113398+0000","last_fullsized":"2023-09-23T20:30:56.113398+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:09:24.510326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"289'9058","reported_seq":9162,"reported_epoch":289,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:30:58.305526+0000","last_change":"2023-09-23T20:29:29.642590+0000","last_active":"2023-09-23T20:30:58.305526+0000","last_peered":"2023-09-23T20:30:58.305526+0000","last_clean":"2023-09-23T20:29:27.208118+0000","last_became_active":"2023-09-23T20:29:29.628868+0000","last_became_peered":"2023-09-23T20:29:29.628868+0000","last_unstale":"2023-09-23T20:30:58.305526+0000","last_undegraded":"2023-09-23T20:29:29.393326+0000","last_fullsized":"2023-09-23T20:29:29.348340+0000","mapping_epoch":259,"log_start":"224'3171","ondisk_log_start":"224'3171","created":217,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":5887,"log_dups_size":0,"ondisk_log_size":5887,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":23527424,"num_objects":359,"num_object_clones":0,"num_object_copies":1077,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3393,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":359,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6247,"num_write_kb":211392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2],"avail_no_missing":["3","2","4"],"object_location_counts":[{"shards":"2,3","objects":-1974},{"shards":"2,3,4","objects":449}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":458,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-09-23T20:30:56.103057+0000","last_change":"2023-09-23T20:29:28.372908+0000","last_active":"2023-09-23T20:30:56.103057+0000","last_peered":"2023-09-23T20:30:56.103057+0000","last_clean":"2023-09-23T20:30:56.103057+0000","last_became_active":"2023-09-23T20:29:28.372652+0000","last_became_peered":"2023-09-23T20:29:28.372652+0000","last_unstale":"2023-09-23T20:30:56.103057+0000","last_undegraded":"2023-09-23T20:30:56.103057+0000","last_fullsized":"2023-09-23T20:30:56.103057+0000","mapping_epoch":258,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":259,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:12.809933+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:12.809933+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,5],"acting":[2,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":140,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-09-23T20:30:56.089383+0000","last_change":"2023-09-23T20:25:03.007736+0000","last_active":"2023-09-23T20:30:56.089383+0000","last_peered":"2023-09-23T20:30:56.089383+0000","last_clean":"2023-09-23T20:30:56.089383+0000","last_became_active":"2023-09-23T20:25:02.973689+0000","last_became_peered":"2023-09-23T20:25:02.973689+0000","last_unstale":"2023-09-23T20:30:56.089383+0000","last_undegraded":"2023-09-23T20:30:56.089383+0000","last_fullsized":"2023-09-23T20:30:56.089383+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:37:25.643774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"86'128","reported_seq":644,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-09-23T20:30:56.132069+0000","last_change":"2023-09-23T20:30:45.008506+0000","last_active":"2023-09-23T20:30:56.132069+0000","last_peered":"2023-09-23T20:30:56.132069+0000","last_clean":"2023-09-23T20:30:56.132069+0000","last_became_active":"2023-09-23T20:29:46.071848+0000","last_became_peered":"2023-09-23T20:29:46.071848+0000","last_unstale":"2023-09-23T20:30:56.132069+0000","last_undegraded":"2023-09-23T20:30:56.132069+0000","last_fullsized":"2023-09-23T20:30:56.132069+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":283,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:44:18.031530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000293202,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":579,"num_read_kb":1473,"num_write":96,"num_write_kb":1062,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":136,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-09-23T20:30:56.087525+0000","last_change":"2023-09-23T20:25:02.971740+0000","last_active":"2023-09-23T20:30:56.087525+0000","last_peered":"2023-09-23T20:30:56.087525+0000","last_clean":"2023-09-23T20:30:56.087525+0000","last_became_active":"2023-09-23T20:25:02.970412+0000","last_became_peered":"2023-09-23T20:25:02.970412+0000","last_unstale":"2023-09-23T20:30:56.087525+0000","last_undegraded":"2023-09-23T20:30:56.087525+0000","last_fullsized":"2023-09-23T20:30:56.087525+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:53:17.247216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"287'8846","reported_seq":9273,"reported_epoch":287,"state":"active+recovery_wait","last_fresh":"2023-09-23T20:30:55.579544+0000","last_change":"2023-09-23T20:30:30.444650+0000","last_active":"2023-09-23T20:30:55.579544+0000","last_peered":"2023-09-23T20:30:55.579544+0000","last_clean":"2023-09-23T20:28:04.010682+0000","last_became_active":"2023-09-23T20:29:47.069776+0000","last_became_peered":"2023-09-23T20:29:47.069776+0000","last_unstale":"2023-09-23T20:30:55.579544+0000","last_undegraded":"2023-09-23T20:30:55.579544+0000","last_fullsized":"2023-09-23T20:30:55.579544+0000","mapping_epoch":268,"log_start":"222'1312","ondisk_log_start":"222'1312","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":7534,"log_dups_size":0,"ondisk_log_size":7534,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26836992,"num_objects":409,"num_object_clones":0,"num_object_copies":1227,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":409,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6077,"num_write_kb":207584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":149,"num_bytes_recovered":9797632,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":["7","2","6"],"object_location_counts":[{"shards":"2,6,7","objects":409}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.4","version":"90'161","reported_seq":536,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-09-23T20:30:56.094261+0000","last_change":"2023-09-23T20:28:03.553294+0000","last_active":"2023-09-23T20:30:56.094261+0000","last_peered":"2023-09-23T20:30:56.094261+0000","last_clean":"2023-09-23T20:30:56.094261+0000","last_became_active":"2023-09-23T20:27:55.212201+0000","last_became_peered":"2023-09-23T20:27:55.212201+0000","last_unstale":"2023-09-23T20:30:56.094261+0000","last_undegraded":"2023-09-23T20:30:56.094261+0000","last_fullsized":"2023-09-23T20:30:56.094261+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:28:03.553106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:28:03.553106+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:24:06.706940+0000","scrub_duration":0.011176864,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"86'117","reported_seq":602,"reported_epoch":288,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:30:56.103158+0000","last_change":"2023-09-23T20:29:59.542691+0000","last_active":"2023-09-23T20:30:56.103158+0000","last_peered":"2023-09-23T20:30:56.103158+0000","last_clean":"2023-09-23T20:13:53.392235+0000","last_became_active":"2023-09-23T20:29:05.138184+0000","last_became_peered":"2023-09-23T20:29:05.138184+0000","last_unstale":"2023-09-23T20:30:56.103158+0000","last_undegraded":"2023-09-23T20:29:59.542326+0000","last_fullsized":"2023-09-23T20:29:05.108660+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:50:02.080933+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046891400000000002,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":893,"num_read_kb":922,"num_write":82,"num_write_kb":4193,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[2,7],"avail_no_missing":["2","3","7"],"object_location_counts":[{"shards":"2,3,7","objects":1}],"blocked_by":[],"up_primary":6,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":271,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:30:57.091561+0000","last_change":"2023-09-23T20:29:46.053198+0000","last_active":"2023-09-23T20:30:57.091561+0000","last_peered":"2023-09-23T20:30:57.091561+0000","last_clean":"2023-09-23T20:30:57.091561+0000","last_became_active":"2023-09-23T20:29:46.052866+0000","last_became_peered":"2023-09-23T20:29:46.052866+0000","last_unstale":"2023-09-23T20:30:57.091561+0000","last_undegraded":"2023-09-23T20:30:57.091561+0000","last_fullsized":"2023-09-23T20:30:57.091561+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:30:18.857140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":244,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T20:30:49.032510+0000","last_change":"2023-09-23T20:29:28.367722+0000","last_active":"2023-09-23T20:30:49.032510+0000","last_peered":"2023-09-23T20:30:49.032510+0000","last_clean":"2023-09-23T20:30:49.032510+0000","last_became_active":"2023-09-23T20:29:28.367404+0000","last_became_peered":"2023-09-23T20:29:28.367404+0000","last_unstale":"2023-09-23T20:30:49.032510+0000","last_undegraded":"2023-09-23T20:30:49.032510+0000","last_fullsized":"2023-09-23T20:30:49.032510+0000","mapping_epoch":258,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":259,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:02:51.886571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":243,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-09-23T20:30:56.113329+0000","last_change":"2023-09-23T20:29:46.061960+0000","last_active":"2023-09-23T20:30:56.113329+0000","last_peered":"2023-09-23T20:30:56.113329+0000","last_clean":"2023-09-23T20:30:56.113329+0000","last_became_active":"2023-09-23T20:29:46.061601+0000","last_became_peered":"2023-09-23T20:29:46.061601+0000","last_unstale":"2023-09-23T20:30:56.113329+0000","last_undegraded":"2023-09-23T20:30:56.113329+0000","last_fullsized":"2023-09-23T20:30:56.113329+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:15:52.620692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":140,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-09-23T20:30:56.103295+0000","last_change":"2023-09-23T20:29:45.538314+0000","last_active":"2023-09-23T20:30:56.103295+0000","last_peered":"2023-09-23T20:30:56.103295+0000","last_clean":"2023-09-23T20:30:56.103295+0000","last_became_active":"2023-09-23T20:29:45.536847+0000","last_became_peered":"2023-09-23T20:29:45.536847+0000","last_unstale":"2023-09-23T20:30:56.103295+0000","last_undegraded":"2023-09-23T20:30:56.103295+0000","last_fullsized":"2023-09-23T20:30:56.103295+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:51.948721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"6.19","version":"289'8844","reported_seq":8935,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:30:59.961203+0000","last_change":"2023-09-23T20:30:29.820384+0000","last_active":"2023-09-23T20:30:59.961203+0000","last_peered":"2023-09-23T20:30:59.961203+0000","last_clean":"2023-09-23T20:30:59.961203+0000","last_became_active":"2023-09-23T20:28:35.567626+0000","last_became_peered":"2023-09-23T20:28:35.567626+0000","last_unstale":"2023-09-23T20:30:59.961203+0000","last_undegraded":"2023-09-23T20:30:59.961203+0000","last_fullsized":"2023-09-23T20:30:59.961203+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":242,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":8844,"log_dups_size":0,"ondisk_log_size":8844,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":33423360,"num_objects":510,"num_object_clones":0,"num_object_copies":1530,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":510,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6048,"num_write_kb":209856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":282,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T20:30:49.032879+0000","last_change":"2023-09-23T20:29:59.578082+0000","last_active":"2023-09-23T20:30:49.032879+0000","last_peered":"2023-09-23T20:30:49.032879+0000","last_clean":"2023-09-23T20:30:49.032879+0000","last_became_active":"2023-09-23T20:29:46.072626+0000","last_became_peered":"2023-09-23T20:29:46.072626+0000","last_unstale":"2023-09-23T20:30:49.032879+0000","last_undegraded":"2023-09-23T20:30:49.032879+0000","last_fullsized":"2023-09-23T20:30:49.032879+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:24:14.704823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":296,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-09-23T20:30:56.088892+0000","last_change":"2023-09-23T20:29:46.070732+0000","last_active":"2023-09-23T20:30:56.088892+0000","last_peered":"2023-09-23T20:30:56.088892+0000","last_clean":"2023-09-23T20:30:56.088892+0000","last_became_active":"2023-09-23T20:29:46.069861+0000","last_became_peered":"2023-09-23T20:29:46.069861+0000","last_unstale":"2023-09-23T20:30:56.088892+0000","last_undegraded":"2023-09-23T20:30:56.088892+0000","last_fullsized":"2023-09-23T20:30:56.088892+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:33:00.813261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.18","version":"287'8857","reported_seq":8945,"reported_epoch":287,"state":"active+clean","last_fresh":"2023-09-23T20:30:55.583780+0000","last_change":"2023-09-23T20:28:35.587755+0000","last_active":"2023-09-23T20:30:55.583780+0000","last_peered":"2023-09-23T20:30:55.583780+0000","last_clean":"2023-09-23T20:30:55.583780+0000","last_became_active":"2023-09-23T20:28:35.585502+0000","last_became_peered":"2023-09-23T20:28:35.585502+0000","last_unstale":"2023-09-23T20:30:55.583780+0000","last_undegraded":"2023-09-23T20:30:55.583780+0000","last_fullsized":"2023-09-23T20:30:55.583780+0000","mapping_epoch":242,"log_start":"222'1293","ondisk_log_start":"222'1293","created":242,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":7564,"log_dups_size":0,"ondisk_log_size":7564,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":30212096,"num_objects":461,"num_object_clones":0,"num_object_copies":1383,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":461,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6071,"num_write_kb":209024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":290,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T20:30:49.033872+0000","last_change":"2023-09-23T20:29:46.242242+0000","last_active":"2023-09-23T20:30:49.033872+0000","last_peered":"2023-09-23T20:30:49.033872+0000","last_clean":"2023-09-23T20:30:49.033872+0000","last_became_active":"2023-09-23T20:29:46.241216+0000","last_became_peered":"2023-09-23T20:29:46.241216+0000","last_unstale":"2023-09-23T20:30:49.033872+0000","last_undegraded":"2023-09-23T20:30:49.033872+0000","last_fullsized":"2023-09-23T20:30:49.033872+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:12:48.530720+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":204,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-09-23T20:30:56.088538+0000","last_change":"2023-09-23T20:29:28.353700+0000","last_active":"2023-09-23T20:30:56.088538+0000","last_peered":"2023-09-23T20:30:56.088538+0000","last_clean":"2023-09-23T20:30:56.088538+0000","last_became_active":"2023-09-23T20:29:28.353357+0000","last_became_peered":"2023-09-23T20:29:28.353357+0000","last_unstale":"2023-09-23T20:30:56.088538+0000","last_undegraded":"2023-09-23T20:30:56.088538+0000","last_fullsized":"2023-09-23T20:30:56.088538+0000","mapping_epoch":258,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":259,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:12.077050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":169,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-09-23T20:30:56.088550+0000","last_change":"2023-09-23T20:29:04.138540+0000","last_active":"2023-09-23T20:30:56.088550+0000","last_peered":"2023-09-23T20:30:56.088550+0000","last_clean":"2023-09-23T20:30:56.088550+0000","last_became_active":"2023-09-23T20:29:04.138182+0000","last_became_peered":"2023-09-23T20:29:04.138182+0000","last_unstale":"2023-09-23T20:30:56.088550+0000","last_undegraded":"2023-09-23T20:30:56.088550+0000","last_fullsized":"2023-09-23T20:30:56.088550+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:26.098284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":145,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-09-23T20:30:56.089825+0000","last_change":"2023-09-23T20:25:39.622819+0000","last_active":"2023-09-23T20:30:56.089825+0000","last_peered":"2023-09-23T20:30:56.089825+0000","last_clean":"2023-09-23T20:30:56.089825+0000","last_became_active":"2023-09-23T20:25:39.622484+0000","last_became_peered":"2023-09-23T20:25:39.622484+0000","last_unstale":"2023-09-23T20:30:56.089825+0000","last_undegraded":"2023-09-23T20:30:56.089825+0000","last_fullsized":"2023-09-23T20:30:56.089825+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:51:53.973261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":269,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-09-23T20:30:56.114331+0000","last_change":"2023-09-23T20:25:39.668882+0000","last_active":"2023-09-23T20:30:56.114331+0000","last_peered":"2023-09-23T20:30:56.114331+0000","last_clean":"2023-09-23T20:30:56.114331+0000","last_became_active":"2023-09-23T20:25:39.637128+0000","last_became_peered":"2023-09-23T20:25:39.637128+0000","last_unstale":"2023-09-23T20:30:56.114331+0000","last_undegraded":"2023-09-23T20:30:56.114331+0000","last_fullsized":"2023-09-23T20:30:56.114331+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:56.406298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":302,"reported_epoch":286,"state":"active+clean","last_fresh":"2023-09-23T20:30:49.043231+0000","last_change":"2023-09-23T20:29:46.082412+0000","last_active":"2023-09-23T20:30:49.043231+0000","last_peered":"2023-09-23T20:30:49.043231+0000","last_clean":"2023-09-23T20:30:49.043231+0000","last_became_active":"2023-09-23T20:29:46.082028+0000","last_became_peered":"2023-09-23T20:29:46.082028+0000","last_unstale":"2023-09-23T20:30:49.043231+0000","last_undegraded":"2023-09-23T20:30:49.043231+0000","last_fullsized":"2023-09-23T20:30:49.043231+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:48:07.017969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":26,"stat_sum":{"num_bytes":1001062424,"num_objects":15276,"num_object_clones":0,"num_object_copies":45828,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":29339,"num_objects_misplaced":2047,"num_objects_unfound":0,"num_objects_dirty":15276,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":194198,"num_write_kb":6703105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20739,"num_bytes_recovered":1141506048,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":4904656896,"data_stored":4904648752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":187514,"ondisk_log_size":187514,"up":78,"acting":66,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":6376,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":122880,"data_stored":21392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26691,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":740547,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":214,"num_read_kb":401,"num_write":106,"num_write_kb":1063,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1433573,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2437120,"data_stored":2426464,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3381,"ondisk_log_size":3381,"up":63,"acting":63,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":26,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":17,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6082,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":22,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201517,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":970048,"kb_used_data":747000,"kb_used_omap":0,"kb_used_meta":223040,"kb_avail":92775104,"statfs":{"total":95995035648,"available":95001706496,"internally_reserved":0,"allocated":764928000,"data_stored":764016722,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":228392960},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:30:34 2023","interfaces":[{"interface":"back","average":{"1min":1.1459999999999999,"5min":1.96,"15min":1.163},"min":{"1min":0.25800000000000001,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":3.4790000000000001,"5min":48.798999999999999,"15min":48.798999999999999},"last":4.8289999999999997},{"interface":"front","average":{"1min":2.617,"5min":2.4209999999999998,"15min":1.3300000000000001},"min":{"1min":0.308,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":7.8179999999999996,"5min":48.247,"15min":48.247},"last":2.6200000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:30:47 2023","interfaces":[{"interface":"back","average":{"1min":1.762,"5min":1.762,"15min":1.762},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":9.7370000000000001,"5min":9.7370000000000001,"15min":9.7370000000000001},"last":14.026},{"interface":"front","average":{"1min":1.9810000000000001,"5min":1.9810000000000001,"15min":1.9810000000000001},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":5.7679999999999998,"5min":5.7679999999999998,"15min":5.7679999999999998},"last":2.4700000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:30:38 2023","interfaces":[{"interface":"back","average":{"1min":2.9300000000000002,"5min":2.0249999999999999,"15min":1.2210000000000001},"min":{"1min":0.35899999999999999,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":8.2509999999999994,"5min":11.372999999999999,"15min":11.372999999999999},"last":1.8420000000000001},{"interface":"front","average":{"1min":2.9729999999999999,"5min":2.2709999999999999,"15min":1.349},"min":{"1min":0.33500000000000002,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":14.993,"5min":14.993,"15min":14.993},"last":10.513}]},{"osd":3,"last update":"Sat Sep 23 20:30:01 2023","interfaces":[{"interface":"back","average":{"1min":2.3210000000000002,"5min":2.2429999999999999,"15min":1.254},"min":{"1min":0.29199999999999998,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":10.397,"5min":10.397,"15min":10.397},"last":1.5209999999999999},{"interface":"front","average":{"1min":1.0649999999999999,"5min":2.0449999999999999,"15min":1.159},"min":{"1min":0.33100000000000002,"5min":0.32500000000000001,"15min":0.27600000000000002},"max":{"1min":2.8940000000000001,"5min":9.7840000000000007,"15min":9.7840000000000007},"last":1.484}]},{"osd":4,"last update":"Sat Sep 23 20:30:01 2023","interfaces":[{"interface":"back","average":{"1min":3.5870000000000002,"5min":1.7490000000000001,"15min":0.98899999999999999},"min":{"1min":0.28399999999999997,"5min":0.27100000000000002,"15min":0.27000000000000002},"max":{"1min":15.177,"5min":15.177,"15min":15.177},"last":0.35299999999999998},{"interface":"front","average":{"1min":2.0430000000000001,"5min":2.0390000000000001,"15min":1.1060000000000001},"min":{"1min":0.36599999999999999,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":9.3740000000000006,"5min":11.122,"15min":11.122},"last":2.0470000000000002}]},{"osd":5,"last update":"Sat Sep 23 20:30:01 2023","interfaces":[{"interface":"back","average":{"1min":1.7509999999999999,"5min":1.899,"15min":1.0880000000000001},"min":{"1min":0.33000000000000002,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":6.8300000000000001,"5min":13.241,"15min":13.241},"last":1.9099999999999999},{"interface":"front","average":{"1min":2.9249999999999998,"5min":2.1150000000000002,"15min":1.1910000000000001},"min":{"1min":0.34999999999999998,"5min":0.315,"15min":0.315},"max":{"1min":18.831,"5min":18.831,"15min":18.831},"last":0.41699999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:30:01 2023","interfaces":[{"interface":"back","average":{"1min":3.0550000000000002,"5min":1.873,"15min":1.087},"min":{"1min":0.41899999999999998,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":12.298999999999999,"5min":12.298999999999999,"15min":13.185},"last":0.39500000000000002},{"interface":"front","average":{"1min":1.6619999999999999,"5min":1.7010000000000001,"15min":1.022},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.29899999999999999},"max":{"1min":8.0069999999999997,"5min":9.9120000000000008,"15min":9.9120000000000008},"last":2.1640000000000001}]}]},{"osd":6,"up_from":47,"seq":201863463153,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":532920,"kb_used_data":335600,"kb_used_omap":0,"kb_used_meta":197312,"kb_avail":93212232,"statfs":{"total":95995035648,"available":95449325568,"internally_reserved":0,"allocated":343654400,"data_stored":342751202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":202047488},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1,0,0,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:30:34 2023","interfaces":[{"interface":"back","average":{"1min":1.446,"5min":1.413,"15min":0.997},"min":{"1min":0.23300000000000001,"5min":0.221,"15min":0.221},"max":{"1min":7.609,"5min":8.4209999999999994,"15min":8.4209999999999994},"last":5.9420000000000002},{"interface":"front","average":{"1min":2.1619999999999999,"5min":1.7949999999999999,"15min":1.2310000000000001},"min":{"1min":0.28499999999999998,"5min":0.25,"15min":0.25},"max":{"1min":8.6229999999999993,"5min":8.6760000000000002,"15min":12.512},"last":2.0350000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:30:46 2023","interfaces":[{"interface":"back","average":{"1min":2.7509999999999999,"5min":2.7509999999999999,"15min":2.7509999999999999},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":14.1,"5min":14.1,"15min":14.1},"last":0.317},{"interface":"front","average":{"1min":3.4870000000000001,"5min":3.4870000000000001,"15min":3.4870000000000001},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":14.266999999999999,"5min":14.266999999999999,"15min":14.266999999999999},"last":4.3029999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:30:34 2023","interfaces":[{"interface":"back","average":{"1min":2.7519999999999998,"5min":2.2440000000000002,"15min":1.304},"min":{"1min":0.35099999999999998,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":7.8700000000000001,"5min":10.631,"15min":13.856},"last":1.4199999999999999},{"interface":"front","average":{"1min":2.5110000000000001,"5min":2.056,"15min":1.2190000000000001},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":7.891,"5min":8.3729999999999993,"15min":8.3729999999999993},"last":4.1849999999999996}]},{"osd":3,"last update":"Sat Sep 23 20:30:08 2023","interfaces":[{"interface":"back","average":{"1min":1.9390000000000001,"5min":2.0590000000000002,"15min":1.2310000000000001},"min":{"1min":0.34799999999999998,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":6.8979999999999997,"5min":9.4239999999999995,"15min":9.4239999999999995},"last":4.8079999999999998},{"interface":"front","average":{"1min":2.0550000000000002,"5min":2.1179999999999999,"15min":1.272},"min":{"1min":0.437,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":6.6539999999999999,"5min":8.3770000000000007,"15min":8.6050000000000004},"last":4.6479999999999997}]},{"osd":4,"last update":"Sat Sep 23 20:30:53 2023","interfaces":[{"interface":"back","average":{"1min":0.44,"5min":1.0349999999999999,"15min":0.85299999999999998},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":1.599,"5min":10.173,"15min":10.173},"last":0.33300000000000002},{"interface":"front","average":{"1min":1.032,"5min":1.3779999999999999,"15min":0.98199999999999998},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":3.363,"5min":8.3249999999999993,"15min":8.3249999999999993},"last":0.69899999999999995}]},{"osd":5,"last update":"Sat Sep 23 20:30:53 2023","interfaces":[{"interface":"back","average":{"1min":0.46400000000000002,"5min":0.91100000000000003,"15min":0.89800000000000002},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":1.2010000000000001,"5min":8.4499999999999993,"15min":14.085000000000001},"last":0.46999999999999997},{"interface":"front","average":{"1min":0.98099999999999998,"5min":1.76,"15min":1.236},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.29499999999999998},"max":{"1min":3.294,"5min":18.192,"15min":18.192},"last":0.624}]},{"osd":7,"last update":"Sat Sep 23 20:30:08 2023","interfaces":[{"interface":"back","average":{"1min":2.0449999999999999,"5min":1.7829999999999999,"15min":1.032},"min":{"1min":0.35399999999999998,"5min":0.316,"15min":0.316},"max":{"1min":8.5060000000000002,"5min":10.217000000000001,"15min":10.217000000000001},"last":0.505},{"interface":"front","average":{"1min":2.2559999999999998,"5min":1.9490000000000001,"15min":1.125},"min":{"1min":0.38400000000000001,"5min":0.36499999999999999,"15min":0.33500000000000002},"max":{"1min":7.1289999999999996,"5min":9.7360000000000007,"15min":9.7360000000000007},"last":0.66800000000000004}]}]},{"osd":1,"up_from":266,"seq":1142461300752,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1097748,"kb_used_data":985868,"kb_used_omap":8,"kb_used_meta":111799,"kb_avail":92647404,"statfs":{"total":95995035648,"available":94870941696,"internally_reserved":0,"allocated":1009528832,"data_stored":1007973896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8206,"internal_metadata":114483186},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:30:44 2023","interfaces":[{"interface":"back","average":{"1min":2.3300000000000001,"5min":2.3300000000000001,"15min":2.3300000000000001},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":9.5470000000000006,"5min":9.5470000000000006,"15min":9.5470000000000006},"last":0.40699999999999997},{"interface":"front","average":{"1min":3.1960000000000002,"5min":3.1960000000000002,"15min":3.1960000000000002},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":9.7129999999999992,"5min":9.7129999999999992,"15min":9.7129999999999992},"last":0.34000000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:30:44 2023","interfaces":[{"interface":"back","average":{"1min":5.125,"5min":5.125,"15min":5.125},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":16.151,"5min":16.151,"15min":16.151},"last":0.316},{"interface":"front","average":{"1min":2.0939999999999999,"5min":2.0939999999999999,"15min":2.0939999999999999},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":9.4649999999999999,"5min":9.4649999999999999,"15min":9.4649999999999999},"last":0.36199999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:30:44 2023","interfaces":[{"interface":"back","average":{"1min":3.867,"5min":3.867,"15min":3.867},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":9.5739999999999998,"5min":9.5739999999999998,"15min":9.5739999999999998},"last":0.42799999999999999},{"interface":"front","average":{"1min":4.8460000000000001,"5min":4.8460000000000001,"15min":4.8460000000000001},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":12.012,"5min":12.012,"15min":12.012},"last":0.38200000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:30:44 2023","interfaces":[{"interface":"back","average":{"1min":2.0790000000000002,"5min":2.0790000000000002,"15min":2.0790000000000002},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":9.4429999999999996,"5min":9.4429999999999996,"15min":9.4429999999999996},"last":0.39300000000000002},{"interface":"front","average":{"1min":2.024,"5min":2.024,"15min":2.024},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":9.5039999999999996,"5min":9.5039999999999996,"15min":9.5039999999999996},"last":0.46500000000000002}]},{"osd":5,"last update":"Sat Sep 23 20:30:44 2023","interfaces":[{"interface":"back","average":{"1min":2.802,"5min":2.802,"15min":2.802},"min":{"1min":0.44,"5min":0.44,"15min":0.44},"max":{"1min":9.6479999999999997,"5min":9.6479999999999997,"15min":9.6479999999999997},"last":0.45000000000000001},{"interface":"front","average":{"1min":2.0899999999999999,"5min":2.0899999999999999,"15min":2.0899999999999999},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":9.4700000000000006,"5min":9.4700000000000006,"15min":9.4700000000000006},"last":0.41699999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:30:44 2023","interfaces":[{"interface":"back","average":{"1min":7.4489999999999998,"5min":7.4489999999999998,"15min":7.4489999999999998},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":72.682000000000002,"5min":72.682000000000002,"15min":72.682000000000002},"last":0.55500000000000005},{"interface":"front","average":{"1min":6.4400000000000004,"5min":6.4400000000000004,"15min":6.4400000000000004},"min":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":72.563999999999993,"5min":72.563999999999993,"15min":72.563999999999993},"last":0.50800000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:30:44 2023","interfaces":[{"interface":"back","average":{"1min":3.2829999999999999,"5min":3.2829999999999999,"15min":3.2829999999999999},"min":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"max":{"1min":9.6850000000000005,"5min":9.6850000000000005,"15min":9.6850000000000005},"last":0.60599999999999998},{"interface":"front","average":{"1min":2.3599999999999999,"5min":2.3599999999999999,"15min":2.3599999999999999},"min":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"max":{"1min":9.3949999999999996,"5min":9.3949999999999996,"15min":9.3949999999999996},"last":0.53900000000000003}]}]},{"osd":0,"up_from":94,"seq":403726925955,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":557848,"kb_used_data":224656,"kb_used_omap":7,"kb_used_meta":333176,"kb_avail":93187304,"statfs":{"total":95995035648,"available":95423799296,"internally_reserved":0,"allocated":230047744,"data_stored":228606976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":341172241},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:30:48 2023","interfaces":[{"interface":"back","average":{"1min":3.887,"5min":3.887,"15min":3.887},"min":{"1min":0.17999999999999999,"5min":0.17999999999999999,"15min":0.17999999999999999},"max":{"1min":15.542,"5min":15.542,"15min":15.542},"last":2.9660000000000002},{"interface":"front","average":{"1min":4.4569999999999999,"5min":4.4569999999999999,"15min":4.4569999999999999},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":10.49,"5min":10.49,"15min":10.49},"last":2.9409999999999998}]},{"osd":2,"last update":"Sat Sep 23 20:30:27 2023","interfaces":[{"interface":"back","average":{"1min":5.8789999999999996,"5min":3.3180000000000001,"15min":1.6619999999999999},"min":{"1min":0.185,"5min":0.185,"15min":0.185},"max":{"1min":77.448999999999998,"5min":77.448999999999998,"15min":77.448999999999998},"last":5.9850000000000003},{"interface":"front","average":{"1min":6.0869999999999997,"5min":2.9449999999999998,"15min":1.5069999999999999},"min":{"1min":0.24299999999999999,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":81.724999999999994,"5min":81.724999999999994,"15min":81.724999999999994},"last":0.41899999999999998}]},{"osd":3,"last update":"Sat Sep 23 20:30:01 2023","interfaces":[{"interface":"back","average":{"1min":2.0430000000000001,"5min":2.5640000000000001,"15min":1.3580000000000001},"min":{"1min":0.245,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":8.5530000000000008,"5min":23.748999999999999,"15min":23.748999999999999},"last":5.8789999999999996},{"interface":"front","average":{"1min":1.8819999999999999,"5min":2.4609999999999999,"15min":1.169},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":8.1790000000000003,"5min":24.274999999999999,"15min":24.274999999999999},"last":10.276999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:30:27 2023","interfaces":[{"interface":"back","average":{"1min":2.093,"5min":2.4990000000000001,"15min":1.409},"min":{"1min":0.315,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":8.3209999999999997,"5min":23.876999999999999,"15min":23.876999999999999},"last":5.9160000000000004},{"interface":"front","average":{"1min":2.0009999999999999,"5min":1.9690000000000001,"15min":1.2390000000000001},"min":{"1min":0.34499999999999997,"5min":0.312,"15min":0.29299999999999998},"max":{"1min":10.414,"5min":23.681999999999999,"15min":23.681999999999999},"last":2.867}]},{"osd":5,"last update":"Sat Sep 23 20:30:27 2023","interfaces":[{"interface":"back","average":{"1min":2.645,"5min":2.657,"15min":1.6040000000000001},"min":{"1min":0.377,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":10.368,"5min":24.198,"15min":24.198},"last":2.9060000000000001},{"interface":"front","average":{"1min":2.2130000000000001,"5min":2.1869999999999998,"15min":1.45},"min":{"1min":0.38100000000000001,"5min":0.314,"15min":0.314},"max":{"1min":8.7780000000000005,"5min":14.938000000000001,"15min":14.938000000000001},"last":0.57699999999999996}]},{"osd":6,"last update":"Sat Sep 23 20:30:27 2023","interfaces":[{"interface":"back","average":{"1min":2.2450000000000001,"5min":2.3420000000000001,"15min":1.319},"min":{"1min":0.40100000000000002,"5min":0.318,"15min":0.318},"max":{"1min":10.441000000000001,"5min":24.408000000000001,"15min":24.408000000000001},"last":4.5620000000000003},{"interface":"front","average":{"1min":1.764,"5min":2.1709999999999998,"15min":1.2629999999999999},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":6.9480000000000004,"5min":25.271000000000001,"15min":25.271000000000001},"last":6.109}]},{"osd":7,"last update":"Sat Sep 23 20:30:27 2023","interfaces":[{"interface":"back","average":{"1min":2.024,"5min":2.3839999999999999,"15min":1.4179999999999999},"min":{"1min":0.42199999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":7.4470000000000001,"5min":25.359000000000002,"15min":25.359000000000002},"last":0.60599999999999998},{"interface":"front","average":{"1min":2.5550000000000002,"5min":3.1200000000000001,"15min":1.6799999999999999},"min":{"1min":0.46500000000000002,"5min":0.378,"15min":0.378},"max":{"1min":8.2729999999999997,"5min":24.669,"15min":24.669},"last":5.9550000000000001}]}]},{"osd":2,"up_from":104,"seq":446676598902,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1290408,"kb_used_data":911904,"kb_used_omap":11,"kb_used_meta":378484,"kb_avail":92454744,"statfs":{"total":95995035648,"available":94673657856,"internally_reserved":0,"allocated":933789696,"data_stored":932387910,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11878,"internal_metadata":387568026},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,3,1,4],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:30:37 2023","interfaces":[{"interface":"back","average":{"1min":3.3199999999999998,"5min":2.2509999999999999,"15min":1.282},"min":{"1min":0.193,"5min":0.17999999999999999,"15min":0.17999999999999999},"max":{"1min":30.765999999999998,"5min":30.765999999999998,"15min":30.765999999999998},"last":2.9609999999999999},{"interface":"front","average":{"1min":5.1280000000000001,"5min":2.3199999999999998,"15min":1.2150000000000001},"min":{"1min":0.252,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":16.824000000000002,"5min":16.824000000000002,"15min":16.824000000000002},"last":5.9260000000000002}]},{"osd":1,"last update":"Sat Sep 23 20:30:45 2023","interfaces":[{"interface":"back","average":{"1min":4.851,"5min":4.851,"15min":4.851},"min":{"1min":0.221,"5min":0.221,"15min":0.221},"max":{"1min":16.280000000000001,"5min":16.280000000000001,"15min":16.280000000000001},"last":0.48699999999999999},{"interface":"front","average":{"1min":2.988,"5min":2.988,"15min":2.988},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":30.565000000000001,"5min":30.565000000000001,"15min":30.565000000000001},"last":0.248}]},{"osd":3,"last update":"Sat Sep 23 20:30:01 2023","interfaces":[{"interface":"back","average":{"1min":3.0019999999999998,"5min":1.865,"15min":1.042},"min":{"1min":0.19800000000000001,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":16.331,"5min":16.331,"15min":16.331},"last":0.52700000000000002},{"interface":"front","average":{"1min":1.5920000000000001,"5min":2.3969999999999998,"15min":1.2030000000000001},"min":{"1min":0.249,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":7.7709999999999999,"5min":21.594000000000001,"15min":21.594000000000001},"last":0.27300000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:30:37 2023","interfaces":[{"interface":"back","average":{"1min":3.98,"5min":2.1669999999999998,"15min":1.234},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.28100000000000003},"max":{"1min":16.797999999999998,"5min":16.797999999999998,"15min":16.797999999999998},"last":0.41899999999999998},{"interface":"front","average":{"1min":2.7309999999999999,"5min":2.1589999999999998,"15min":1.2609999999999999},"min":{"1min":0.309,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":10.518000000000001,"5min":14.971,"15min":14.971},"last":0.55800000000000005}]},{"osd":5,"last update":"Sat Sep 23 20:30:37 2023","interfaces":[{"interface":"back","average":{"1min":4.5359999999999996,"5min":3.512,"15min":1.77},"min":{"1min":0.35299999999999998,"5min":0.309,"15min":0.309},"max":{"1min":16.766999999999999,"5min":95.393000000000001,"15min":95.393000000000001},"last":0.38400000000000001},{"interface":"front","average":{"1min":3.4790000000000001,"5min":3.6190000000000002,"15min":1.8},"min":{"1min":0.34799999999999998,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":10.474,"5min":95.414000000000001,"15min":95.414000000000001},"last":0.58599999999999997}]},{"osd":6,"last update":"Sat Sep 23 20:30:37 2023","interfaces":[{"interface":"back","average":{"1min":3.258,"5min":2.29,"15min":1.323},"min":{"1min":0.33600000000000002,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":10.544,"5min":15.023999999999999,"15min":15.023999999999999},"last":0.69999999999999996},{"interface":"front","average":{"1min":2.7930000000000001,"5min":2.3719999999999999,"15min":1.405},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":30.670999999999999,"5min":30.670999999999999,"15min":30.670999999999999},"last":1.3819999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:30:37 2023","interfaces":[{"interface":"back","average":{"1min":3.4609999999999999,"5min":2.5529999999999999,"15min":1.4730000000000001},"min":{"1min":0.41299999999999998,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":30.741,"5min":30.741,"15min":30.741},"last":0.505},{"interface":"front","average":{"1min":4.8650000000000002,"5min":2.52,"15min":1.4550000000000001},"min":{"1min":0.40899999999999997,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":16.719000000000001,"5min":16.719000000000001,"15min":16.719000000000001},"last":0.44800000000000001}]}]},{"osd":3,"up_from":109,"seq":468151435376,"num_pgs":64,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":992004,"kb_used_data":601084,"kb_used_omap":4,"kb_used_meta":390523,"kb_avail":92753148,"statfs":{"total":95995035648,"available":94979223552,"internally_reserved":0,"allocated":615510016,"data_stored":614085051,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4514,"internal_metadata":399896158},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,4,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:30:05 2023","interfaces":[{"interface":"back","average":{"1min":1.8340000000000001,"5min":2.657,"15min":1.3129999999999999},"min":{"1min":0.19600000000000001,"5min":0.186,"15min":0.186},"max":{"1min":6.9470000000000001,"5min":67.305999999999997,"15min":67.305999999999997},"last":2.9009999999999998},{"interface":"front","average":{"1min":1.879,"5min":2.3250000000000002,"15min":1.2290000000000001},"min":{"1min":0.26000000000000001,"5min":0.224,"15min":0.224},"max":{"1min":5.3220000000000001,"5min":67.403999999999996,"15min":67.403999999999996},"last":2.2669999999999999}]},{"osd":1,"last update":"Sat Sep 23 20:30:47 2023","interfaces":[{"interface":"back","average":{"1min":3.0880000000000001,"5min":3.0880000000000001,"15min":3.0880000000000001},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":8.9199999999999999,"5min":8.9199999999999999,"15min":8.9199999999999999},"last":4.0449999999999999},{"interface":"front","average":{"1min":3.2530000000000001,"5min":3.2530000000000001,"15min":3.2530000000000001},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":10.967000000000001,"5min":10.967000000000001,"15min":10.967000000000001},"last":2.492}]},{"osd":2,"last update":"Sat Sep 23 20:30:05 2023","interfaces":[{"interface":"back","average":{"1min":2.7549999999999999,"5min":2.1739999999999999,"15min":1.21},"min":{"1min":0.29699999999999999,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":11.499000000000001,"5min":24.045000000000002,"15min":24.045000000000002},"last":2.1949999999999998},{"interface":"front","average":{"1min":3.1539999999999999,"5min":2.298,"15min":1.2629999999999999},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":16.530000000000001,"5min":16.530000000000001,"15min":17.684000000000001},"last":3.0369999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:30:05 2023","interfaces":[{"interface":"back","average":{"1min":2.2789999999999999,"5min":2.2970000000000002,"15min":1.2410000000000001},"min":{"1min":0.34399999999999997,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":6.6440000000000001,"5min":46.042999999999999,"15min":46.042999999999999},"last":3.028},{"interface":"front","average":{"1min":2.3420000000000001,"5min":2.7890000000000001,"15min":1.385},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":13.462999999999999,"5min":46.579999999999998,"15min":46.579999999999998},"last":3.0880000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:30:05 2023","interfaces":[{"interface":"back","average":{"1min":2.323,"5min":2.6160000000000001,"15min":1.405},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.311},"max":{"1min":7.0469999999999997,"5min":45.292000000000002,"15min":45.292000000000002},"last":1.885},{"interface":"front","average":{"1min":2.0790000000000002,"5min":2.302,"15min":1.268},"min":{"1min":0.39800000000000002,"5min":0.313,"15min":0.313},"max":{"1min":5.5140000000000002,"5min":44.679000000000002,"15min":44.679000000000002},"last":2.7000000000000002}]},{"osd":6,"last update":"Sat Sep 23 20:30:05 2023","interfaces":[{"interface":"back","average":{"1min":2.2240000000000002,"5min":2.2309999999999999,"15min":1.2470000000000001},"min":{"1min":0.41299999999999998,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":4.202,"5min":13.475,"15min":13.475},"last":2.2360000000000002},{"interface":"front","average":{"1min":2.613,"5min":2.1920000000000002,"15min":1.2270000000000001},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.35499999999999998},"max":{"1min":13.506,"5min":13.506,"15min":13.506},"last":4.7619999999999996}]},{"osd":7,"last update":"Sat Sep 23 20:30:05 2023","interfaces":[{"interface":"back","average":{"1min":3.3730000000000002,"5min":2.3820000000000001,"15min":1.3240000000000001},"min":{"1min":0.46899999999999997,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":13.532999999999999,"5min":13.532999999999999,"15min":13.532999999999999},"last":4.7939999999999996},{"interface":"front","average":{"1min":2.3460000000000001,"5min":2.7919999999999998,"15min":1.4950000000000001},"min":{"1min":0.44500000000000001,"5min":0.371,"15min":0.371},"max":{"1min":7.0899999999999999,"5min":45.387999999999998,"15min":45.387999999999998},"last":0.54200000000000004}]}]},{"osd":4,"up_from":0,"seq":128849019129,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":5,"up_from":39,"seq":167503724790,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":542328,"kb_used_data":328880,"kb_used_omap":0,"kb_used_meta":213440,"kb_avail":93202824,"statfs":{"total":95995035648,"available":95439691776,"internally_reserved":0,"allocated":336773120,"data_stored":335866295,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":218562560},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,3,0,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:30:31 2023","interfaces":[{"interface":"back","average":{"1min":2.8109999999999999,"5min":1.7529999999999999,"15min":1.181},"min":{"1min":0.28499999999999998,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":8.1609999999999996,"5min":11.346,"15min":32.475000000000001},"last":4.673},{"interface":"front","average":{"1min":2.5950000000000002,"5min":1.681,"15min":1.2030000000000001},"min":{"1min":0.29199999999999998,"5min":0.245,"15min":0.20599999999999999},"max":{"1min":8.1859999999999999,"5min":8.4900000000000002,"15min":32.457000000000001},"last":1.857}]},{"osd":1,"last update":"Sat Sep 23 20:30:45 2023","interfaces":[{"interface":"back","average":{"1min":2.0550000000000002,"5min":2.0550000000000002,"15min":2.0550000000000002},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":8.5839999999999996,"5min":8.5839999999999996,"15min":8.5839999999999996},"last":3.919},{"interface":"front","average":{"1min":1.5609999999999999,"5min":1.5609999999999999,"15min":1.5609999999999999},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":6.7969999999999997,"5min":6.7969999999999997,"15min":6.7969999999999997},"last":4.7039999999999997}]},{"osd":2,"last update":"Sat Sep 23 20:30:31 2023","interfaces":[{"interface":"back","average":{"1min":3.5259999999999998,"5min":1.823,"15min":1.325},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.26700000000000002},"max":{"1min":18.472999999999999,"5min":18.472999999999999,"15min":34.514000000000003},"last":4.6230000000000002},{"interface":"front","average":{"1min":1.8919999999999999,"5min":1.7869999999999999,"15min":1.256},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":8.016,"5min":11.574999999999999,"15min":32.515000000000001},"last":0.5}]},{"osd":3,"last update":"Sat Sep 23 20:30:04 2023","interfaces":[{"interface":"back","average":{"1min":2.4670000000000001,"5min":2.0059999999999998,"15min":1.151},"min":{"1min":0.318,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":12.51,"5min":12.846,"15min":12.846},"last":0.61799999999999999},{"interface":"front","average":{"1min":1.6339999999999999,"5min":1.851,"15min":1.1160000000000001},"min":{"1min":0.35099999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":5.2670000000000003,"5min":12.005000000000001,"15min":12.005000000000001},"last":1.097}]},{"osd":4,"last update":"Sat Sep 23 20:30:25 2023","interfaces":[{"interface":"back","average":{"1min":3.1560000000000001,"5min":2.359,"15min":1.2230000000000001},"min":{"1min":0.32500000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":28.015999999999998,"5min":28.015999999999998,"15min":28.015999999999998},"last":0.64700000000000002},{"interface":"front","average":{"1min":2.0339999999999998,"5min":1.6459999999999999,"15min":1.0149999999999999},"min":{"1min":0.33800000000000002,"5min":0.32600000000000001,"15min":0.29099999999999998},"max":{"1min":12.300000000000001,"5min":12.300000000000001,"15min":12.300000000000001},"last":1.0589999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:30:39 2023","interfaces":[{"interface":"back","average":{"1min":1.1220000000000001,"5min":1.796,"15min":1.1499999999999999},"min":{"1min":0.309,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":4.9829999999999997,"5min":12.890000000000001,"15min":12.890000000000001},"last":0.70499999999999996},{"interface":"front","average":{"1min":0.92500000000000004,"5min":1.5389999999999999,"15min":1.137},"min":{"1min":0.371,"5min":0.36399999999999999,"15min":0.34499999999999997},"max":{"1min":2.4710000000000001,"5min":16.265999999999998,"15min":16.265999999999998},"last":1.125}]},{"osd":7,"last update":"Sat Sep 23 20:30:04 2023","interfaces":[{"interface":"back","average":{"1min":2.0710000000000002,"5min":2.1989999999999998,"15min":1.1719999999999999},"min":{"1min":0.36199999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":7.343,"5min":36.621000000000002,"15min":36.621000000000002},"last":1.296},{"interface":"front","average":{"1min":2.2170000000000001,"5min":1.8169999999999999,"15min":1.075},"min":{"1min":0.35199999999999998,"5min":0.316,"15min":0.315},"max":{"1min":13.122,"5min":37.331000000000003,"15min":37.331000000000003},"last":1.952}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4722,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1360,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8206,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7156,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3154,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":224661504,"data_stored":224657432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1003290624,"data_stored":1003290624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":921436160,"data_stored":921436160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":605814784,"data_stored":605814784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":727318528,"data_stored":727318528,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":332337152,"data_stored":332333080,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":335740928,"data_stored":335740928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":754057216,"data_stored":754057216,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:31:01.186 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:31:01.249 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:31:01.305 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T20:31:01.318 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:01.372 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:31:01.376 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:01.376 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:31:01.402 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:01 smithi018 ceph-mon[179647]: pgmap v652: 119 pgs: 1 active+recovery_wait+degraded, 3 active+recovery_wait, 3 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 981 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 635 op/s; 29365/46932 objects degraded (62.569%); 2047/46932 objects misplaced (4.362%); 7.3 MiB/s, 201 objects/s recovering 2023-09-23T20:31:01.402 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:01 smithi018 ceph-mon[179647]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:31:01.402 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:01 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:31:01.402 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:01 smithi018 ceph-mon[179647]: osdmap e290: 8 total, 8 up, 7 in 2023-09-23T20:31:01.402 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:01 smithi018 ceph-mon[179647]: failed to encode map e290 with expected crc 2023-09-23T20:31:01.402 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:01 smithi018 ceph-mon[179647]: failed to encode map e290 with expected crc 2023-09-23T20:31:01.402 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:01 smithi018 ceph-mon[179647]: failed to encode map e290 with expected crc 2023-09-23T20:31:01.403 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:01 smithi018 ceph-mon[179647]: failed to encode map e290 with expected crc 2023-09-23T20:31:01.403 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:01 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 29365/46932 objects degraded (62.569%), 15 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2023-09-23T20:31:01.403 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:01 smithi018 ceph-mon[184144]: pgmap v652: 119 pgs: 1 active+recovery_wait+degraded, 3 active+recovery_wait, 3 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 981 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 635 op/s; 29365/46932 objects degraded (62.569%); 2047/46932 objects misplaced (4.362%); 7.3 MiB/s, 201 objects/s recovering 2023-09-23T20:31:01.403 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:01 smithi018 ceph-mon[184144]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:31:01.403 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:01 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:31:01.403 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:01 smithi018 ceph-mon[184144]: osdmap e290: 8 total, 8 up, 7 in 2023-09-23T20:31:01.403 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:01 smithi018 ceph-mon[184144]: failed to encode map e290 with expected crc 2023-09-23T20:31:01.403 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:01 smithi018 ceph-mon[184144]: failed to encode map e290 with expected crc 2023-09-23T20:31:01.403 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:01 smithi018 ceph-mon[184144]: failed to encode map e290 with expected crc 2023-09-23T20:31:01.403 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:01 smithi018 ceph-mon[184144]: failed to encode map e290 with expected crc 2023-09-23T20:31:01.404 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:01 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 29365/46932 objects degraded (62.569%), 15 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2023-09-23T20:31:01.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:01 smithi149 ceph-mon[170394]: pgmap v652: 119 pgs: 1 active+recovery_wait+degraded, 3 active+recovery_wait, 3 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 981 MiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 635 op/s; 29365/46932 objects degraded (62.569%); 2047/46932 objects misplaced (4.362%); 7.3 MiB/s, 201 objects/s recovering 2023-09-23T20:31:01.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:01 smithi149 ceph-mon[170394]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:31:01.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:01 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:31:01.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:01 smithi149 ceph-mon[170394]: osdmap e290: 8 total, 8 up, 7 in 2023-09-23T20:31:01.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:01 smithi149 ceph-mon[170394]: failed to encode map e290 with expected crc 2023-09-23T20:31:01.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:01 smithi149 ceph-mon[170394]: failed to encode map e290 with expected crc 2023-09-23T20:31:01.468 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:01 smithi149 ceph-mon[170394]: failed to encode map e290 with expected crc 2023-09-23T20:31:01.468 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:01 smithi149 ceph-mon[170394]: failed to encode map e290 with expected crc 2023-09-23T20:31:01.468 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:01 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 29365/46932 objects degraded (62.569%), 15 pgs degraded, 14 pgs undersized (PG_DEGRADED) 2023-09-23T20:31:01.477 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:31:01.586 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:31:01.620 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:01.661 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:31:01.716 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:31:01.717 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:31:01.724 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:01.724 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:31:01.770 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:31:01.813 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:31:01.825 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:31:01.850 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:31:01.858 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:01.890 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:31:01.916 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:31:01.924 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:01.924 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:31:01.935 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:31:01.966 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:31:01.966 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:31:01.966 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:31:01.966 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:31:01.966 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:31:01.966 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:31:01.966 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:31:01.966 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:31:02.025 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:31:02.065 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:31:02.142 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:31:02.142 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:02.234 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:31:02.243 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:02.243 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:31:02.279 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:02 smithi018 ceph-mon[179647]: from='client.35577 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:31:02.279 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:02 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/922820275' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:31:02.282 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:02 smithi018 ceph-mon[184144]: from='client.35577 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:31:02.282 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:02 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/922820275' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:31:02.344 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:31:02.420 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:02.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:02 smithi149 ceph-mon[170394]: from='client.35577 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:31:02.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:02 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/922820275' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:31:02.638 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:31:02.649 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:02.650 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:31:02.750 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:31:02.824 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T20:31:02.844 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:02.939 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:31:02.949 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:02.949 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:31:03.050 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:31:03.203 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:03.427 INFO:teuthology.orchestra.run.smithi149.stdout:ERROR: (22) Invalid argument 2023-09-23T20:31:03.427 INFO:teuthology.orchestra.run.smithi149.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:31:03.458 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:03 smithi149 ceph-mon[170394]: pgmap v654: 119 pgs: 1 active+recovery_wait+degraded, 3 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 889 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 703 op/s; 29352/42537 objects degraded (69.003%); 2014/42537 objects misplaced (4.735%); 4.4 MiB/s, 254 objects/s recovering 2023-09-23T20:31:03.644 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:03 smithi018 ceph-mon[179647]: pgmap v654: 119 pgs: 1 active+recovery_wait+degraded, 3 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 889 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 703 op/s; 29352/42537 objects degraded (69.003%); 2014/42537 objects misplaced (4.735%); 4.4 MiB/s, 254 objects/s recovering 2023-09-23T20:31:03.644 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:03 smithi018 ceph-mon[184144]: pgmap v654: 119 pgs: 1 active+recovery_wait+degraded, 3 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 889 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 703 op/s; 29352/42537 objects degraded (69.003%); 2014/42537 objects misplaced (4.735%); 4.4 MiB/s, 254 objects/s recovering 2023-09-23T20:31:03.808 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T20:31:04.023 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:31:04.032 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:04.032 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:31:04.133 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:31:04.174 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:04.222 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:31:04.227 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:04.228 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:31:04.328 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:31:04.418 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:04.431 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:31:04.497 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:31:04.513 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:04.513 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:31:04.537 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:31:04.614 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:31:04.663 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:04.697 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:31:04.769 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:31:04.773 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:04.774 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:31:04.874 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:31:05.058 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:05.137 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:31:04 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:31:04] "GET /metrics HTTP/1.1" 200 36569 "" "Prometheus/2.43.0" 2023-09-23T20:31:05.195 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:31:05.203 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:05.204 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:31:05.250 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:31:05.285 INFO:teuthology.orchestra.run.smithi018.stderr:dumped all 2023-09-23T20:31:05.305 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:31:05.389 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:05 smithi018 ceph-mon[184144]: pgmap v655: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 3 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 800 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 558 op/s; 29254/38271 objects degraded (76.439%); 1505/38271 objects misplaced (3.932%); 3.5 MiB/s, 129 objects/s recovering 2023-09-23T20:31:05.390 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:05 smithi018 ceph-mon[179647]: pgmap v655: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 3 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 800 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 558 op/s; 29254/38271 objects degraded (76.439%); 1505/38271 objects misplaced (3.932%); 3.5 MiB/s, 129 objects/s recovering 2023-09-23T20:31:05.410 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:05.572 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:31:05.579 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:05.579 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:31:05.592 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:05 smithi149 ceph-mon[170394]: pgmap v655: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 3 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 800 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 558 op/s; 29254/38271 objects degraded (76.439%); 1505/38271 objects misplaced (3.932%); 3.5 MiB/s, 129 objects/s recovering 2023-09-23T20:31:05.680 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:31:05.770 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:05.838 INFO:teuthology.orchestra.run.smithi149.stdout:ERROR: (22) Invalid argument 2023-09-23T20:31:05.839 INFO:teuthology.orchestra.run.smithi149.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:31:06.011 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:31:06.021 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:06.022 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:31:06.122 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:31:06.169 INFO:teuthology.orchestra.run.smithi018.stdout:{"pg_ready":true,"pg_map":{"version":655,"stamp":"2023-09-23T20:31:03.710591+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":838951534,"num_objects":12757,"num_object_clones":0,"num_object_copies":38271,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":29254,"num_objects_misplaced":1505,"num_objects_unfound":0,"num_objects_dirty":12757,"num_whiteouts":0,"num_read":6210,"num_read_kb":13614,"num_write":197760,"num_write_kb":6716893,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21495,"num_bytes_recovered":1151531232,"num_keys_recovered":30,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":190863,"ondisk_log_size":190863,"up":357,"acting":344,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":371,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":5568164,"kb_used_data":3703404,"kb_used_omap":32,"kb_used_meta":1864543,"kb_avail":650647900,"statfs":{"total":671965249536,"available":666263449600,"internally_reserved":0,"allocated":3792285696,"data_stored":3783685255,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32773,"internal_metadata":1909293051},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,0,2,1,1,1,0,1,14],"upper_bound":1024},"perf_stat":{"commit_latency_ms":9,"apply_latency_ms":9,"commit_latency_ns":9000000,"apply_latency_ns":9000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-279248896,"num_objects":-4261,"num_object_clones":0,"num_object_copies":-12783,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-408,"num_objects_misplaced":-1249,"num_objects_unfound":0,"num_objects_dirty":-4261,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4261,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":990,"num_bytes_recovered":28049408,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"7.628370"},"pg_stats":[{"pgid":"3.1a","version":"69'1","reported_seq":35,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:30:57.101310+0000","last_change":"2023-09-23T20:29:46.050662+0000","last_active":"2023-09-23T20:30:57.101310+0000","last_peered":"2023-09-23T20:30:57.101310+0000","last_clean":"2023-09-23T20:30:57.101310+0000","last_became_active":"2023-09-23T20:29:46.050281+0000","last_became_peered":"2023-09-23T20:29:46.050281+0000","last_unstale":"2023-09-23T20:30:57.101310+0000","last_undegraded":"2023-09-23T20:30:57.101310+0000","last_fullsized":"2023-09-23T20:30:57.101310+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:55.227008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":163,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:30:57.096053+0000","last_change":"2023-09-23T20:29:46.067085+0000","last_active":"2023-09-23T20:30:57.096053+0000","last_peered":"2023-09-23T20:30:57.096053+0000","last_clean":"2023-09-23T20:30:57.096053+0000","last_became_active":"2023-09-23T20:29:46.066738+0000","last_became_peered":"2023-09-23T20:29:46.066738+0000","last_unstale":"2023-09-23T20:30:57.096053+0000","last_undegraded":"2023-09-23T20:30:57.096053+0000","last_fullsized":"2023-09-23T20:30:57.096053+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:33:14.679402+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":227,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:30:57.101857+0000","last_change":"2023-09-23T20:29:46.178684+0000","last_active":"2023-09-23T20:30:57.101857+0000","last_peered":"2023-09-23T20:30:57.101857+0000","last_clean":"2023-09-23T20:30:57.101857+0000","last_became_active":"2023-09-23T20:29:46.178249+0000","last_became_peered":"2023-09-23T20:29:46.178249+0000","last_unstale":"2023-09-23T20:30:57.101857+0000","last_undegraded":"2023-09-23T20:30:57.101857+0000","last_fullsized":"2023-09-23T20:30:57.101857+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:58:58.530687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":254,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-09-23T20:30:56.113857+0000","last_change":"2023-09-23T20:25:02.957185+0000","last_active":"2023-09-23T20:30:56.113857+0000","last_peered":"2023-09-23T20:30:56.113857+0000","last_clean":"2023-09-23T20:30:56.113857+0000","last_became_active":"2023-09-23T20:25:02.956689+0000","last_became_peered":"2023-09-23T20:25:02.956689+0000","last_unstale":"2023-09-23T20:30:56.113857+0000","last_undegraded":"2023-09-23T20:30:56.113857+0000","last_fullsized":"2023-09-23T20:30:56.113857+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:04:28.524015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":152,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:30:57.095754+0000","last_change":"2023-09-23T20:29:45.529762+0000","last_active":"2023-09-23T20:30:57.095754+0000","last_peered":"2023-09-23T20:30:57.095754+0000","last_clean":"2023-09-23T20:30:57.095754+0000","last_became_active":"2023-09-23T20:29:45.529283+0000","last_became_peered":"2023-09-23T20:29:45.529283+0000","last_unstale":"2023-09-23T20:30:57.095754+0000","last_undegraded":"2023-09-23T20:30:57.095754+0000","last_fullsized":"2023-09-23T20:30:57.095754+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:54:36.720075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":63,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:30:57.096585+0000","last_change":"2023-09-23T20:29:46.112977+0000","last_active":"2023-09-23T20:30:57.096585+0000","last_peered":"2023-09-23T20:30:57.096585+0000","last_clean":"2023-09-23T20:30:57.096585+0000","last_became_active":"2023-09-23T20:29:46.112600+0000","last_became_peered":"2023-09-23T20:29:46.112600+0000","last_unstale":"2023-09-23T20:30:57.096585+0000","last_undegraded":"2023-09-23T20:30:57.096585+0000","last_fullsized":"2023-09-23T20:30:57.096585+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:06:34.132247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.12","version":"290'9006","reported_seq":15598,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T20:31:03.358908+0000","last_change":"2023-09-23T20:30:46.961845+0000","last_active":"2023-09-23T20:31:03.358908+0000","last_peered":"2023-09-23T20:31:03.358908+0000","last_clean":"2023-09-23T20:31:03.358908+0000","last_became_active":"2023-09-23T20:30:46.939470+0000","last_became_peered":"2023-09-23T20:30:46.939470+0000","last_unstale":"2023-09-23T20:31:03.358908+0000","last_undegraded":"2023-09-23T20:31:03.358908+0000","last_fullsized":"2023-09-23T20:31:03.358908+0000","mapping_epoch":284,"log_start":"223'2946","ondisk_log_start":"223'2946","created":242,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":6060,"log_dups_size":0,"ondisk_log_size":6060,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":30605312,"num_objects":467,"num_object_clones":0,"num_object_copies":1401,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":467,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6195,"num_write_kb":213184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3241,"num_bytes_recovered":105185280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":252,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-09-23T20:30:56.114253+0000","last_change":"2023-09-23T20:25:02.969432+0000","last_active":"2023-09-23T20:30:56.114253+0000","last_peered":"2023-09-23T20:30:56.114253+0000","last_clean":"2023-09-23T20:30:56.114253+0000","last_became_active":"2023-09-23T20:25:02.969118+0000","last_became_peered":"2023-09-23T20:25:02.969118+0000","last_unstale":"2023-09-23T20:30:56.114253+0000","last_undegraded":"2023-09-23T20:30:56.114253+0000","last_fullsized":"2023-09-23T20:30:56.114253+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:02:22.453831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":172,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:30:57.095798+0000","last_change":"2023-09-23T20:29:46.069832+0000","last_active":"2023-09-23T20:30:57.095798+0000","last_peered":"2023-09-23T20:30:57.095798+0000","last_clean":"2023-09-23T20:30:57.095798+0000","last_became_active":"2023-09-23T20:29:46.069451+0000","last_became_peered":"2023-09-23T20:29:46.069451+0000","last_unstale":"2023-09-23T20:30:57.095798+0000","last_undegraded":"2023-09-23T20:30:57.095798+0000","last_fullsized":"2023-09-23T20:30:57.095798+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:06:06.104030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.13","version":"290'8839","reported_seq":9204,"reported_epoch":290,"state":"active+recovery_wait","last_fresh":"2023-09-23T20:31:00.682288+0000","last_change":"2023-09-23T20:30:30.361943+0000","last_active":"2023-09-23T20:31:00.682288+0000","last_peered":"2023-09-23T20:31:00.682288+0000","last_clean":"2023-09-23T20:28:04.010682+0000","last_became_active":"2023-09-23T20:29:47.054756+0000","last_became_peered":"2023-09-23T20:29:47.054756+0000","last_unstale":"2023-09-23T20:31:00.682288+0000","last_undegraded":"2023-09-23T20:31:00.682288+0000","last_fullsized":"2023-09-23T20:31:00.682288+0000","mapping_epoch":268,"log_start":"221'1140","ondisk_log_start":"221'1140","created":242,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":7699,"log_dups_size":0,"ondisk_log_size":7699,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":33325056,"num_objects":509,"num_object_clones":0,"num_object_copies":1527,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":509,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6071,"num_write_kb":210528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":117,"num_bytes_recovered":7503872,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":["7","2","6"],"object_location_counts":[{"shards":"2,6,7","objects":509}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.14","version":"90'161","reported_seq":560,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:30:57.109673+0000","last_change":"2023-09-23T20:29:48.092787+0000","last_active":"2023-09-23T20:30:57.109673+0000","last_peered":"2023-09-23T20:30:57.109673+0000","last_clean":"2023-09-23T20:30:57.109673+0000","last_became_active":"2023-09-23T20:29:48.092460+0000","last_became_peered":"2023-09-23T20:29:48.092460+0000","last_unstale":"2023-09-23T20:30:57.109673+0000","last_undegraded":"2023-09-23T20:30:57.109673+0000","last_fullsized":"2023-09-23T20:30:57.109673+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:13.244651+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.244651+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0019955509999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109227,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":266,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:30:57.111152+0000","last_change":"2023-09-23T20:29:45.523808+0000","last_active":"2023-09-23T20:30:57.111152+0000","last_peered":"2023-09-23T20:30:57.111152+0000","last_clean":"2023-09-23T20:30:57.111152+0000","last_became_active":"2023-09-23T20:29:45.523508+0000","last_became_peered":"2023-09-23T20:29:45.523508+0000","last_unstale":"2023-09-23T20:30:57.111152+0000","last_undegraded":"2023-09-23T20:30:57.111152+0000","last_fullsized":"2023-09-23T20:30:57.111152+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:50:55.565352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":155,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:30:57.095820+0000","last_change":"2023-09-23T20:29:04.130512+0000","last_active":"2023-09-23T20:30:57.095820+0000","last_peered":"2023-09-23T20:30:57.095820+0000","last_clean":"2023-09-23T20:30:57.095820+0000","last_became_active":"2023-09-23T20:29:04.130135+0000","last_became_peered":"2023-09-23T20:29:04.130135+0000","last_unstale":"2023-09-23T20:30:57.095820+0000","last_undegraded":"2023-09-23T20:30:57.095820+0000","last_fullsized":"2023-09-23T20:30:57.095820+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:53:59.385893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.10","version":"289'8971","reported_seq":9064,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:30:59.674709+0000","last_change":"2023-09-23T20:30:24.864928+0000","last_active":"2023-09-23T20:30:59.674709+0000","last_peered":"2023-09-23T20:30:59.674709+0000","last_clean":"2023-09-23T20:30:59.674709+0000","last_became_active":"2023-09-23T20:28:35.582165+0000","last_became_peered":"2023-09-23T20:28:35.582165+0000","last_unstale":"2023-09-23T20:30:59.674709+0000","last_undegraded":"2023-09-23T20:30:59.674709+0000","last_fullsized":"2023-09-23T20:30:59.674709+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":242,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":8971,"log_dups_size":0,"ondisk_log_size":8971,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":27951104,"num_objects":427,"num_object_clones":0,"num_object_copies":1281,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":427,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6153,"num_write_kb":210528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":301,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:30:57.094067+0000","last_change":"2023-09-23T20:29:59.501260+0000","last_active":"2023-09-23T20:30:57.094067+0000","last_peered":"2023-09-23T20:30:57.094067+0000","last_clean":"2023-09-23T20:30:57.094067+0000","last_became_active":"2023-09-23T20:29:28.362070+0000","last_became_peered":"2023-09-23T20:29:28.362070+0000","last_unstale":"2023-09-23T20:30:57.094067+0000","last_undegraded":"2023-09-23T20:30:57.094067+0000","last_fullsized":"2023-09-23T20:30:57.094067+0000","mapping_epoch":258,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:32:55.983696+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":237,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:30:57.101489+0000","last_change":"2023-09-23T20:29:46.080766+0000","last_active":"2023-09-23T20:30:57.101489+0000","last_peered":"2023-09-23T20:30:57.101489+0000","last_clean":"2023-09-23T20:30:57.101489+0000","last_became_active":"2023-09-23T20:29:46.080356+0000","last_became_peered":"2023-09-23T20:29:46.080356+0000","last_unstale":"2023-09-23T20:30:57.101489+0000","last_undegraded":"2023-09-23T20:30:57.101489+0000","last_fullsized":"2023-09-23T20:30:57.101489+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:08:01.862270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.11","version":"290'8827","reported_seq":15439,"reported_epoch":290,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T20:31:00.597152+0000","last_change":"2023-09-23T20:30:44.984034+0000","last_active":"2023-09-23T20:31:00.597152+0000","last_peered":"2023-09-23T20:31:00.597152+0000","last_clean":"2023-09-23T20:28:04.011715+0000","last_became_active":"2023-09-23T20:29:46.191939+0000","last_became_peered":"2023-09-23T20:29:46.191939+0000","last_unstale":"2023-09-23T20:31:00.597152+0000","last_undegraded":"2023-09-23T20:29:49.867530+0000","last_fullsized":"2023-09-23T20:29:46.041211+0000","mapping_epoch":267,"log_start":"222'1209","ondisk_log_start":"222'1209","created":242,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":7618,"log_dups_size":0,"ondisk_log_size":7618,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":25886720,"num_objects":395,"num_object_clones":0,"num_object_copies":1185,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":47,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":395,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6073,"num_write_kb":206976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2895,"num_bytes_recovered":160956416,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[6,2],"avail_no_missing":["6","2"],"object_location_counts":[{"shards":"2,6","objects":348}],"blocked_by":[],"up_primary":1,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":107,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:30:57.094440+0000","last_change":"2023-09-23T20:29:59.529392+0000","last_active":"2023-09-23T20:30:57.094440+0000","last_peered":"2023-09-23T20:30:57.094440+0000","last_clean":"2023-09-23T20:30:57.094440+0000","last_became_active":"2023-09-23T20:29:28.365579+0000","last_became_peered":"2023-09-23T20:29:28.365579+0000","last_unstale":"2023-09-23T20:30:57.094440+0000","last_undegraded":"2023-09-23T20:30:57.094440+0000","last_fullsized":"2023-09-23T20:30:57.094440+0000","mapping_epoch":258,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:31:11.546083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":285,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:30:57.095874+0000","last_change":"2023-09-23T20:29:46.091575+0000","last_active":"2023-09-23T20:30:57.095874+0000","last_peered":"2023-09-23T20:30:57.095874+0000","last_clean":"2023-09-23T20:30:57.095874+0000","last_became_active":"2023-09-23T20:29:46.091271+0000","last_became_peered":"2023-09-23T20:29:46.091271+0000","last_unstale":"2023-09-23T20:30:57.095874+0000","last_undegraded":"2023-09-23T20:30:57.095874+0000","last_fullsized":"2023-09-23T20:30:57.095874+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:08:01.594175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.16","version":"290'8852","reported_seq":8954,"reported_epoch":290,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T20:31:03.349423+0000","last_change":"2023-09-23T20:29:29.535020+0000","last_active":"2023-09-23T20:31:03.349423+0000","last_peered":"2023-09-23T20:31:03.349423+0000","last_clean":"2023-09-23T20:29:27.303063+0000","last_became_active":"2023-09-23T20:29:29.522829+0000","last_became_peered":"2023-09-23T20:29:29.522829+0000","last_unstale":"2023-09-23T20:31:03.349423+0000","last_undegraded":"2023-09-23T20:31:03.349423+0000","last_fullsized":"2023-09-23T20:31:03.349423+0000","mapping_epoch":259,"log_start":"224'2801","ondisk_log_start":"224'2801","created":242,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":6051,"log_dups_size":0,"ondisk_log_size":6051,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26083328,"num_objects":398,"num_object_clones":0,"num_object_copies":1194,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":398,"num_objects_unfound":0,"num_objects_dirty":398,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6110,"num_write_kb":208256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,4],"avail_no_missing":["3","4","7"],"object_location_counts":[{"shards":"3,4,7","objects":398}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.11","version":"90'161","reported_seq":505,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-09-23T20:30:56.111544+0000","last_change":"2023-09-23T20:30:36.766139+0000","last_active":"2023-09-23T20:30:56.111544+0000","last_peered":"2023-09-23T20:30:56.111544+0000","last_clean":"2023-09-23T20:30:56.111544+0000","last_became_active":"2023-09-23T20:28:05.991304+0000","last_became_peered":"2023-09-23T20:28:05.991304+0000","last_unstale":"2023-09-23T20:30:56.111544+0000","last_undegraded":"2023-09-23T20:30:56.111544+0000","last_fullsized":"2023-09-23T20:30:56.111544+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:34.821789+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:34.821789+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":300,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:30:57.093842+0000","last_change":"2023-09-23T20:29:59.544701+0000","last_active":"2023-09-23T20:30:57.093842+0000","last_peered":"2023-09-23T20:30:57.093842+0000","last_clean":"2023-09-23T20:30:57.093842+0000","last_became_active":"2023-09-23T20:29:28.367679+0000","last_became_peered":"2023-09-23T20:29:28.367679+0000","last_unstale":"2023-09-23T20:30:57.093842+0000","last_undegraded":"2023-09-23T20:30:57.093842+0000","last_fullsized":"2023-09-23T20:30:57.093842+0000","mapping_epoch":258,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:36:19.089724+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":102,"num_keys_recovered":30,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":152,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:30:57.098468+0000","last_change":"2023-09-23T20:29:45.531343+0000","last_active":"2023-09-23T20:30:57.098468+0000","last_peered":"2023-09-23T20:30:57.098468+0000","last_clean":"2023-09-23T20:30:57.098468+0000","last_became_active":"2023-09-23T20:29:45.530713+0000","last_became_peered":"2023-09-23T20:29:45.530713+0000","last_unstale":"2023-09-23T20:30:57.098468+0000","last_undegraded":"2023-09-23T20:30:57.098468+0000","last_fullsized":"2023-09-23T20:30:57.098468+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:28:00.245911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.17","version":"290'9049","reported_seq":9162,"reported_epoch":290,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:31:00.257043+0000","last_change":"2023-09-23T20:29:46.291295+0000","last_active":"2023-09-23T20:31:00.257043+0000","last_peered":"2023-09-23T20:31:00.257043+0000","last_clean":"2023-09-23T20:29:44.520411+0000","last_became_active":"2023-09-23T20:29:46.269706+0000","last_became_peered":"2023-09-23T20:29:46.269706+0000","last_unstale":"2023-09-23T20:31:00.257043+0000","last_undegraded":"2023-09-23T20:29:46.075283+0000","last_fullsized":"2023-09-23T20:29:46.032679+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":242,"last_epoch_clean":260,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":9049,"log_dups_size":0,"ondisk_log_size":9049,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":27557888,"num_objects":421,"num_object_clones":0,"num_object_copies":1263,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3310,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":421,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6221,"num_write_kb":212512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3],"avail_no_missing":["5","3","4"],"object_location_counts":[{"shards":"3,4,5","objects":410},{"shards":"3,5","objects":-1413}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.10","version":"90'161","reported_seq":510,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:30:57.110151+0000","last_change":"2023-09-23T20:29:07.045212+0000","last_active":"2023-09-23T20:30:57.110151+0000","last_peered":"2023-09-23T20:30:57.110151+0000","last_clean":"2023-09-23T20:30:57.110151+0000","last_became_active":"2023-09-23T20:29:07.044892+0000","last_became_peered":"2023-09-23T20:29:07.044892+0000","last_unstale":"2023-09-23T20:30:57.110151+0000","last_undegraded":"2023-09-23T20:30:57.110151+0000","last_fullsized":"2023-09-23T20:30:57.110151+0000","mapping_epoch":253,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":254,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:32.801168+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:32.801168+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":252,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-09-23T20:30:56.114942+0000","last_change":"2023-09-23T20:25:02.953993+0000","last_active":"2023-09-23T20:30:56.114942+0000","last_peered":"2023-09-23T20:30:56.114942+0000","last_clean":"2023-09-23T20:30:56.114942+0000","last_became_active":"2023-09-23T20:25:02.953672+0000","last_became_peered":"2023-09-23T20:25:02.953672+0000","last_unstale":"2023-09-23T20:30:56.114942+0000","last_undegraded":"2023-09-23T20:30:56.114942+0000","last_fullsized":"2023-09-23T20:30:56.114942+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:26:08.603577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":268,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:30:57.113066+0000","last_change":"2023-09-23T20:25:02.949584+0000","last_active":"2023-09-23T20:30:57.113066+0000","last_peered":"2023-09-23T20:30:57.113066+0000","last_clean":"2023-09-23T20:30:57.113066+0000","last_became_active":"2023-09-23T20:25:02.949132+0000","last_became_peered":"2023-09-23T20:25:02.949132+0000","last_unstale":"2023-09-23T20:30:57.113066+0000","last_undegraded":"2023-09-23T20:30:57.113066+0000","last_fullsized":"2023-09-23T20:30:57.113066+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:34:20.509823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.14","version":"290'8988","reported_seq":9092,"reported_epoch":290,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:31:03.347122+0000","last_change":"2023-09-23T20:29:46.199482+0000","last_active":"2023-09-23T20:31:03.347122+0000","last_peered":"2023-09-23T20:31:03.347122+0000","last_clean":"2023-09-23T20:28:12.083147+0000","last_became_active":"2023-09-23T20:29:46.185251+0000","last_became_peered":"2023-09-23T20:29:46.185251+0000","last_unstale":"2023-09-23T20:31:03.347122+0000","last_undegraded":"2023-09-23T20:29:46.057476+0000","last_fullsized":"2023-09-23T20:29:46.043200+0000","mapping_epoch":267,"log_start":"224'3113","ondisk_log_start":"224'3113","created":242,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":5875,"log_dups_size":0,"ondisk_log_size":5875,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22544384,"num_objects":344,"num_object_clones":0,"num_object_copies":1032,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2460,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":344,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6224,"num_write_kb":210176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[3,5],"avail_no_missing":["3","5"],"object_location_counts":[{"shards":"3,5","objects":-729}],"blocked_by":[],"up_primary":1,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.13","version":"90'161","reported_seq":516,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:30:57.101879+0000","last_change":"2023-09-23T20:29:46.067221+0000","last_active":"2023-09-23T20:30:57.101879+0000","last_peered":"2023-09-23T20:30:57.101879+0000","last_clean":"2023-09-23T20:30:57.101879+0000","last_became_active":"2023-09-23T20:29:46.066622+0000","last_became_peered":"2023-09-23T20:29:46.066622+0000","last_unstale":"2023-09-23T20:30:57.101879+0000","last_undegraded":"2023-09-23T20:30:57.101879+0000","last_fullsized":"2023-09-23T20:30:57.101879+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:12.386098+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:12.386098+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0019500940000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":263,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:30:57.093371+0000","last_change":"2023-09-23T20:29:28.364113+0000","last_active":"2023-09-23T20:30:57.093371+0000","last_peered":"2023-09-23T20:30:57.093371+0000","last_clean":"2023-09-23T20:30:57.093371+0000","last_became_active":"2023-09-23T20:29:28.363788+0000","last_became_peered":"2023-09-23T20:29:28.363788+0000","last_unstale":"2023-09-23T20:30:57.093371+0000","last_undegraded":"2023-09-23T20:30:57.093371+0000","last_fullsized":"2023-09-23T20:30:57.093371+0000","mapping_epoch":258,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":259,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:39:49.658803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":262,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-09-23T20:30:56.113783+0000","last_change":"2023-09-23T20:25:02.957010+0000","last_active":"2023-09-23T20:30:56.113783+0000","last_peered":"2023-09-23T20:30:56.113783+0000","last_clean":"2023-09-23T20:30:56.113783+0000","last_became_active":"2023-09-23T20:25:02.956267+0000","last_became_peered":"2023-09-23T20:25:02.956267+0000","last_unstale":"2023-09-23T20:30:56.113783+0000","last_undegraded":"2023-09-23T20:30:56.113783+0000","last_fullsized":"2023-09-23T20:30:56.113783+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:10:55.299422+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.15","version":"290'8793","reported_seq":12589,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T20:31:00.533775+0000","last_change":"2023-09-23T20:30:01.520686+0000","last_active":"2023-09-23T20:31:00.533775+0000","last_peered":"2023-09-23T20:31:00.533775+0000","last_clean":"2023-09-23T20:31:00.533775+0000","last_became_active":"2023-09-23T20:30:01.486115+0000","last_became_peered":"2023-09-23T20:30:01.486115+0000","last_unstale":"2023-09-23T20:31:00.533775+0000","last_undegraded":"2023-09-23T20:31:00.533775+0000","last_fullsized":"2023-09-23T20:31:00.533775+0000","mapping_epoch":275,"log_start":"221'1228","ondisk_log_start":"221'1228","created":242,"last_epoch_clean":276,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":7565,"log_dups_size":0,"ondisk_log_size":7565,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":23363584,"num_objects":357,"num_object_clones":0,"num_object_copies":1071,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":357,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6053,"num_write_kb":205088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1838,"num_bytes_recovered":97189888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.12","version":"90'161","reported_seq":529,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:30:57.110605+0000","last_change":"2023-09-23T20:29:46.079370+0000","last_active":"2023-09-23T20:30:57.110605+0000","last_peered":"2023-09-23T20:30:57.110605+0000","last_clean":"2023-09-23T20:30:57.110605+0000","last_became_active":"2023-09-23T20:29:46.078873+0000","last_became_peered":"2023-09-23T20:29:46.078873+0000","last_unstale":"2023-09-23T20:30:57.110605+0000","last_undegraded":"2023-09-23T20:30:57.110605+0000","last_fullsized":"2023-09-23T20:30:57.110605+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:35.734584+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:35.734584+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":59,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:30:57.096288+0000","last_change":"2023-09-23T20:29:04.179171+0000","last_active":"2023-09-23T20:30:57.096288+0000","last_peered":"2023-09-23T20:30:57.096288+0000","last_clean":"2023-09-23T20:30:57.096288+0000","last_became_active":"2023-09-23T20:29:04.128672+0000","last_became_peered":"2023-09-23T20:29:04.128672+0000","last_unstale":"2023-09-23T20:30:57.096288+0000","last_undegraded":"2023-09-23T20:30:57.096288+0000","last_fullsized":"2023-09-23T20:30:57.096288+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:49:15.506106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":152,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:30:57.097866+0000","last_change":"2023-09-23T20:29:45.528782+0000","last_active":"2023-09-23T20:30:57.097866+0000","last_peered":"2023-09-23T20:30:57.097866+0000","last_clean":"2023-09-23T20:30:57.097866+0000","last_became_active":"2023-09-23T20:29:45.528389+0000","last_became_peered":"2023-09-23T20:29:45.528389+0000","last_unstale":"2023-09-23T20:30:57.097866+0000","last_undegraded":"2023-09-23T20:30:57.097866+0000","last_fullsized":"2023-09-23T20:30:57.097866+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:13:41.453499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.d","version":"90'161","reported_seq":529,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:30:57.093710+0000","last_change":"2023-09-23T20:29:46.061373+0000","last_active":"2023-09-23T20:30:57.093710+0000","last_peered":"2023-09-23T20:30:57.093710+0000","last_clean":"2023-09-23T20:30:57.093710+0000","last_became_active":"2023-09-23T20:29:46.061003+0000","last_became_peered":"2023-09-23T20:29:46.061003+0000","last_unstale":"2023-09-23T20:30:57.093710+0000","last_undegraded":"2023-09-23T20:30:57.093710+0000","last_fullsized":"2023-09-23T20:30:57.093710+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:12.809933+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:12.809933+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.a","version":"290'12346","reported_seq":23370,"reported_epoch":290,"state":"active+recovery_wait","last_fresh":"2023-09-23T20:31:00.316496+0000","last_change":"2023-09-23T20:29:51.518803+0000","last_active":"2023-09-23T20:31:00.316496+0000","last_peered":"2023-09-23T20:31:00.316496+0000","last_clean":"2023-09-23T20:28:04.001969+0000","last_became_active":"2023-09-23T20:29:51.450026+0000","last_became_peered":"2023-09-23T20:29:51.450026+0000","last_unstale":"2023-09-23T20:31:00.316496+0000","last_undegraded":"2023-09-23T20:31:00.316496+0000","last_fullsized":"2023-09-23T20:31:00.316496+0000","mapping_epoch":271,"log_start":"245'5671","ondisk_log_start":"245'5671","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":6675,"log_dups_size":0,"ondisk_log_size":6675,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:55:12.986375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":54001664,"num_objects":824,"num_object_clones":0,"num_object_copies":2472,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":824,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12346,"num_write_kb":421440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5457,"num_bytes_recovered":313655296,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":["5","0","6"],"object_location_counts":[{"shards":"0,5,6","objects":824}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":279,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:30:57.093979+0000","last_change":"2023-09-23T20:29:59.586041+0000","last_active":"2023-09-23T20:30:57.093979+0000","last_peered":"2023-09-23T20:30:57.093979+0000","last_clean":"2023-09-23T20:30:57.093979+0000","last_became_active":"2023-09-23T20:29:46.059906+0000","last_became_peered":"2023-09-23T20:29:46.059906+0000","last_unstale":"2023-09-23T20:30:57.093979+0000","last_undegraded":"2023-09-23T20:30:57.093979+0000","last_fullsized":"2023-09-23T20:30:57.093979+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:26:12.412320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":251,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:30:57.110280+0000","last_change":"2023-09-23T20:29:04.128604+0000","last_active":"2023-09-23T20:30:57.110280+0000","last_peered":"2023-09-23T20:30:57.110280+0000","last_clean":"2023-09-23T20:30:57.110280+0000","last_became_active":"2023-09-23T20:29:04.128003+0000","last_became_peered":"2023-09-23T20:29:04.128003+0000","last_unstale":"2023-09-23T20:30:57.110280+0000","last_undegraded":"2023-09-23T20:30:57.110280+0000","last_fullsized":"2023-09-23T20:30:57.110280+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:50:49.385599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"90'161","reported_seq":573,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:30:57.097257+0000","last_change":"2023-09-23T20:30:01.493499+0000","last_active":"2023-09-23T20:30:57.097257+0000","last_peered":"2023-09-23T20:30:57.097257+0000","last_clean":"2023-09-23T20:30:57.097257+0000","last_became_active":"2023-09-23T20:30:01.493025+0000","last_became_peered":"2023-09-23T20:30:01.493025+0000","last_unstale":"2023-09-23T20:30:57.097257+0000","last_undegraded":"2023-09-23T20:30:57.097257+0000","last_fullsized":"2023-09-23T20:30:57.097257+0000","mapping_epoch":275,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":276,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:13.244651+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.244651+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0019955509999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109227,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":655360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.b","version":"290'12115","reported_seq":12209,"reported_epoch":290,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:31:03.359258+0000","last_change":"2023-09-23T20:29:46.285273+0000","last_active":"2023-09-23T20:31:03.359258+0000","last_peered":"2023-09-23T20:31:03.359258+0000","last_clean":"2023-09-23T20:28:12.089537+0000","last_became_active":"2023-09-23T20:29:46.255023+0000","last_became_peered":"2023-09-23T20:29:46.255023+0000","last_unstale":"2023-09-23T20:31:03.359258+0000","last_undegraded":"2023-09-23T20:29:46.058377+0000","last_fullsized":"2023-09-23T20:29:46.031666+0000","mapping_epoch":267,"log_start":"254'7329","ondisk_log_start":"254'7329","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":4786,"log_dups_size":0,"ondisk_log_size":4786,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:37:55.158131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":44630016,"num_objects":681,"num_object_clones":0,"num_object_copies":2043,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4695,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":681,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12115,"num_write_kb":409472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":-1277}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":286,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:30:57.094154+0000","last_change":"2023-09-23T20:29:59.567502+0000","last_active":"2023-09-23T20:30:57.094154+0000","last_peered":"2023-09-23T20:30:57.094154+0000","last_clean":"2023-09-23T20:30:57.094154+0000","last_became_active":"2023-09-23T20:29:46.055041+0000","last_became_peered":"2023-09-23T20:29:46.055041+0000","last_unstale":"2023-09-23T20:30:57.094154+0000","last_undegraded":"2023-09-23T20:30:57.094154+0000","last_fullsized":"2023-09-23T20:30:57.094154+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:47:13.874812+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":187,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:30:57.101721+0000","last_change":"2023-09-23T20:29:46.178516+0000","last_active":"2023-09-23T20:30:57.101721+0000","last_peered":"2023-09-23T20:30:57.101721+0000","last_clean":"2023-09-23T20:30:57.101721+0000","last_became_active":"2023-09-23T20:29:46.177858+0000","last_became_peered":"2023-09-23T20:29:46.177858+0000","last_unstale":"2023-09-23T20:30:57.101721+0000","last_undegraded":"2023-09-23T20:30:57.101721+0000","last_fullsized":"2023-09-23T20:30:57.101721+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:09:14.906568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.f","version":"90'161","reported_seq":482,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:30:57.096457+0000","last_change":"2023-09-23T20:29:28.374177+0000","last_active":"2023-09-23T20:30:57.096457+0000","last_peered":"2023-09-23T20:30:57.096457+0000","last_clean":"2023-09-23T20:30:57.096457+0000","last_became_active":"2023-09-23T20:29:28.373750+0000","last_became_peered":"2023-09-23T20:29:28.373750+0000","last_unstale":"2023-09-23T20:30:57.096457+0000","last_undegraded":"2023-09-23T20:30:57.096457+0000","last_fullsized":"2023-09-23T20:30:57.096457+0000","mapping_epoch":258,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":259,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:32.746340+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:32.746340+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,3],"acting":[6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.8","version":"290'8898","reported_seq":8990,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T20:31:00.696255+0000","last_change":"2023-09-23T20:28:35.581921+0000","last_active":"2023-09-23T20:31:00.696255+0000","last_peered":"2023-09-23T20:31:00.696255+0000","last_clean":"2023-09-23T20:31:00.696255+0000","last_became_active":"2023-09-23T20:28:35.576360+0000","last_became_peered":"2023-09-23T20:28:35.576360+0000","last_unstale":"2023-09-23T20:31:00.696255+0000","last_undegraded":"2023-09-23T20:31:00.696255+0000","last_fullsized":"2023-09-23T20:31:00.696255+0000","mapping_epoch":217,"log_start":"222'1258","ondisk_log_start":"222'1258","created":217,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":7640,"log_dups_size":0,"ondisk_log_size":7640,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":27656192,"num_objects":422,"num_object_clones":0,"num_object_copies":1266,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":422,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6112,"num_write_kb":209088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":262,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:30:57.093803+0000","last_change":"2023-09-23T20:29:04.184893+0000","last_active":"2023-09-23T20:30:57.093803+0000","last_peered":"2023-09-23T20:30:57.093803+0000","last_clean":"2023-09-23T20:30:57.093803+0000","last_became_active":"2023-09-23T20:29:04.152723+0000","last_became_peered":"2023-09-23T20:29:04.152723+0000","last_unstale":"2023-09-23T20:30:57.093803+0000","last_undegraded":"2023-09-23T20:30:57.093803+0000","last_fullsized":"2023-09-23T20:30:57.093803+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:51:27.017151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":63,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:30:57.096001+0000","last_change":"2023-09-23T20:29:46.086912+0000","last_active":"2023-09-23T20:30:57.096001+0000","last_peered":"2023-09-23T20:30:57.096001+0000","last_clean":"2023-09-23T20:30:57.096001+0000","last_became_active":"2023-09-23T20:29:46.085471+0000","last_became_peered":"2023-09-23T20:29:46.085471+0000","last_unstale":"2023-09-23T20:30:57.096001+0000","last_undegraded":"2023-09-23T20:30:57.096001+0000","last_fullsized":"2023-09-23T20:30:57.096001+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:04:55.751660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.e","version":"90'161","reported_seq":469,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:30:57.093945+0000","last_change":"2023-09-23T20:29:31.394412+0000","last_active":"2023-09-23T20:30:57.093945+0000","last_peered":"2023-09-23T20:30:57.093945+0000","last_clean":"2023-09-23T20:30:57.093945+0000","last_became_active":"2023-09-23T20:29:31.394053+0000","last_became_peered":"2023-09-23T20:29:31.394053+0000","last_unstale":"2023-09-23T20:30:57.093945+0000","last_undegraded":"2023-09-23T20:30:57.093945+0000","last_fullsized":"2023-09-23T20:30:57.093945+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:31.769333+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:31.769333+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.9","version":"289'9019","reported_seq":9111,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:30:59.946142+0000","last_change":"2023-09-23T20:30:22.842563+0000","last_active":"2023-09-23T20:30:59.946142+0000","last_peered":"2023-09-23T20:30:59.946142+0000","last_clean":"2023-09-23T20:30:59.946142+0000","last_became_active":"2023-09-23T20:28:35.536824+0000","last_became_peered":"2023-09-23T20:28:35.536824+0000","last_unstale":"2023-09-23T20:30:59.946142+0000","last_undegraded":"2023-09-23T20:30:59.946142+0000","last_fullsized":"2023-09-23T20:30:59.946142+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":9019,"log_dups_size":0,"ondisk_log_size":9019,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":21561344,"num_objects":329,"num_object_clones":0,"num_object_copies":987,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":329,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6223,"num_write_kb":209664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":296,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:30:57.110978+0000","last_change":"2023-09-23T20:29:04.138284+0000","last_active":"2023-09-23T20:30:57.110978+0000","last_peered":"2023-09-23T20:30:57.110978+0000","last_clean":"2023-09-23T20:30:57.110978+0000","last_became_active":"2023-09-23T20:29:04.137960+0000","last_became_peered":"2023-09-23T20:29:04.137960+0000","last_unstale":"2023-09-23T20:30:57.110978+0000","last_undegraded":"2023-09-23T20:30:57.110978+0000","last_fullsized":"2023-09-23T20:30:57.110978+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:18:59.548311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":261,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-09-23T20:30:56.113187+0000","last_change":"2023-09-23T20:29:46.061136+0000","last_active":"2023-09-23T20:30:56.113187+0000","last_peered":"2023-09-23T20:30:56.113187+0000","last_clean":"2023-09-23T20:30:56.113187+0000","last_became_active":"2023-09-23T20:29:46.060777+0000","last_became_peered":"2023-09-23T20:29:46.060777+0000","last_unstale":"2023-09-23T20:30:56.113187+0000","last_undegraded":"2023-09-23T20:30:56.113187+0000","last_fullsized":"2023-09-23T20:30:56.113187+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:08:43.673870+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"290'12286","reported_seq":16667,"reported_epoch":290,"state":"active+recovering+undersized+remapped","last_fresh":"2023-09-23T20:31:02.321303+0000","last_change":"2023-09-23T20:30:59.555825+0000","last_active":"2023-09-23T20:31:02.321303+0000","last_peered":"2023-09-23T20:31:02.321303+0000","last_clean":"2023-09-23T20:28:12.106078+0000","last_became_active":"2023-09-23T20:29:45.690954+0000","last_became_peered":"2023-09-23T20:29:45.690954+0000","last_unstale":"2023-09-23T20:31:02.321303+0000","last_undegraded":"2023-09-23T20:31:02.321303+0000","last_fullsized":"2023-09-23T20:29:45.501064+0000","mapping_epoch":266,"log_start":"245'5672","ondisk_log_start":"245'5672","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":6614,"log_dups_size":0,"ondisk_log_size":6614,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:52:59.514150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53477376,"num_objects":816,"num_object_clones":0,"num_object_copies":2448,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":816,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12286,"num_write_kb":419264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2141,"num_bytes_recovered":132972544,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[2,4],"avail_no_missing":["2","4"],"object_location_counts":[{"shards":"2,4","objects":816}],"blocked_by":[],"up_primary":1,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":672,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:30:57.093268+0000","last_change":"2023-09-23T20:28:05.872149+0000","last_active":"2023-09-23T20:30:57.093268+0000","last_peered":"2023-09-23T20:30:57.093268+0000","last_clean":"2023-09-23T20:30:57.093268+0000","last_became_active":"2023-09-23T20:27:11.961310+0000","last_became_peered":"2023-09-23T20:27:11.961310+0000","last_unstale":"2023-09-23T20:30:57.093268+0000","last_undegraded":"2023-09-23T20:30:57.093268+0000","last_fullsized":"2023-09-23T20:30:57.093268+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:28:05.872074+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:28:05.872074+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:52:32.495955+0000","scrub_duration":0.0029215980000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":165,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:30:57.097032+0000","last_change":"2023-09-23T20:29:49.873018+0000","last_active":"2023-09-23T20:30:57.097032+0000","last_peered":"2023-09-23T20:30:57.097032+0000","last_clean":"2023-09-23T20:30:57.097032+0000","last_became_active":"2023-09-23T20:29:46.070369+0000","last_became_peered":"2023-09-23T20:29:46.070369+0000","last_unstale":"2023-09-23T20:30:57.097032+0000","last_undegraded":"2023-09-23T20:30:57.097032+0000","last_fullsized":"2023-09-23T20:30:57.097032+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:24:13.119725+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":301,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:30:57.098586+0000","last_change":"2023-09-23T20:29:46.173195+0000","last_active":"2023-09-23T20:30:57.098586+0000","last_peered":"2023-09-23T20:30:57.098586+0000","last_clean":"2023-09-23T20:30:57.098586+0000","last_became_active":"2023-09-23T20:29:46.172808+0000","last_became_peered":"2023-09-23T20:29:46.172808+0000","last_unstale":"2023-09-23T20:30:57.098586+0000","last_undegraded":"2023-09-23T20:30:57.098586+0000","last_fullsized":"2023-09-23T20:30:57.098586+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:41:58.927658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.f","version":"290'12453","reported_seq":12545,"reported_epoch":290,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T20:31:02.291258+0000","last_change":"2023-09-23T20:29:29.530059+0000","last_active":"2023-09-23T20:31:02.291258+0000","last_peered":"2023-09-23T20:31:02.291258+0000","last_clean":"2023-09-23T20:29:27.332951+0000","last_became_active":"2023-09-23T20:29:29.505420+0000","last_became_peered":"2023-09-23T20:29:29.505420+0000","last_unstale":"2023-09-23T20:31:02.291258+0000","last_undegraded":"2023-09-23T20:31:02.291258+0000","last_fullsized":"2023-09-23T20:31:02.291258+0000","mapping_epoch":259,"log_start":"246'5887","ondisk_log_start":"246'5887","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":6566,"log_dups_size":0,"ondisk_log_size":6566,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:18:15.383066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":52232192,"num_objects":797,"num_object_clones":0,"num_object_copies":2391,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":797,"num_objects_unfound":0,"num_objects_dirty":797,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12453,"num_write_kb":424000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,4],"avail_no_missing":["2","3","4"],"object_location_counts":[{"shards":"2,3,4","objects":797}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":514,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:30:57.110192+0000","last_change":"2023-09-23T20:27:31.757060+0000","last_active":"2023-09-23T20:30:57.110192+0000","last_peered":"2023-09-23T20:30:57.110192+0000","last_clean":"2023-09-23T20:30:57.110192+0000","last_became_active":"2023-09-23T20:27:11.959516+0000","last_became_peered":"2023-09-23T20:27:11.959516+0000","last_unstale":"2023-09-23T20:30:57.110192+0000","last_undegraded":"2023-09-23T20:30:57.110192+0000","last_fullsized":"2023-09-23T20:30:57.110192+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:31.756974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:31.756974+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:35:11.939335+0000","scrub_duration":0.0031802670000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":75,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:30:57.096025+0000","last_change":"2023-09-23T20:29:46.087097+0000","last_active":"2023-09-23T20:30:57.096025+0000","last_peered":"2023-09-23T20:30:57.096025+0000","last_clean":"2023-09-23T20:30:57.096025+0000","last_became_active":"2023-09-23T20:29:46.085908+0000","last_became_peered":"2023-09-23T20:29:46.085908+0000","last_unstale":"2023-09-23T20:30:57.096025+0000","last_undegraded":"2023-09-23T20:30:57.096025+0000","last_fullsized":"2023-09-23T20:30:57.096025+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:38:46.001432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":199,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:30:57.110159+0000","last_change":"2023-09-23T20:29:28.370476+0000","last_active":"2023-09-23T20:30:57.110159+0000","last_peered":"2023-09-23T20:30:57.110159+0000","last_clean":"2023-09-23T20:30:57.110159+0000","last_became_active":"2023-09-23T20:29:28.370158+0000","last_became_peered":"2023-09-23T20:29:28.370158+0000","last_unstale":"2023-09-23T20:30:57.110159+0000","last_undegraded":"2023-09-23T20:30:57.110159+0000","last_fullsized":"2023-09-23T20:30:57.110159+0000","mapping_epoch":258,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":259,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:09:47.335404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.b","version":"90'161","reported_seq":518,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:30:57.101923+0000","last_change":"2023-09-23T20:29:45.534549+0000","last_active":"2023-09-23T20:30:57.101923+0000","last_peered":"2023-09-23T20:30:57.101923+0000","last_clean":"2023-09-23T20:30:57.101923+0000","last_became_active":"2023-09-23T20:29:45.534070+0000","last_became_peered":"2023-09-23T20:29:45.534070+0000","last_unstale":"2023-09-23T20:30:57.101923+0000","last_undegraded":"2023-09-23T20:30:57.101923+0000","last_fullsized":"2023-09-23T20:30:57.101923+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:12.386098+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:12.386098+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0019500940000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.c","version":"290'12144","reported_seq":15639,"reported_epoch":290,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:31:03.338212+0000","last_change":"2023-09-23T20:29:59.529817+0000","last_active":"2023-09-23T20:31:03.338212+0000","last_peered":"2023-09-23T20:31:03.338212+0000","last_clean":"2023-09-23T20:28:03.999709+0000","last_became_active":"2023-09-23T20:29:04.247715+0000","last_became_peered":"2023-09-23T20:29:04.247715+0000","last_unstale":"2023-09-23T20:31:03.338212+0000","last_undegraded":"2023-09-23T20:29:59.527865+0000","last_fullsized":"2023-09-23T20:29:04.105278+0000","mapping_epoch":250,"log_start":"254'7371","ondisk_log_start":"254'7371","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":4773,"log_dups_size":0,"ondisk_log_size":4773,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:01:13.556577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":50200576,"num_objects":766,"num_object_clones":0,"num_object_copies":2298,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2952,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":766,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12144,"num_write_kb":413120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1694,"num_bytes_recovered":110886912,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,5],"avail_no_missing":["3","5"],"object_location_counts":[{"shards":"3,5","objects":-1534},{"shards":"3,5,7","objects":766}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":201,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:30:57.108843+0000","last_change":"2023-09-23T20:29:46.061620+0000","last_active":"2023-09-23T20:30:57.108843+0000","last_peered":"2023-09-23T20:30:57.108843+0000","last_clean":"2023-09-23T20:30:57.108843+0000","last_became_active":"2023-09-23T20:29:46.061271+0000","last_became_peered":"2023-09-23T20:29:46.061271+0000","last_unstale":"2023-09-23T20:30:57.108843+0000","last_undegraded":"2023-09-23T20:30:57.108843+0000","last_fullsized":"2023-09-23T20:30:57.108843+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:24:51.409521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":285,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:30:57.096452+0000","last_change":"2023-09-23T20:29:46.085962+0000","last_active":"2023-09-23T20:30:57.096452+0000","last_peered":"2023-09-23T20:30:57.096452+0000","last_clean":"2023-09-23T20:30:57.096452+0000","last_became_active":"2023-09-23T20:29:46.082826+0000","last_became_peered":"2023-09-23T20:29:46.082826+0000","last_unstale":"2023-09-23T20:30:57.096452+0000","last_undegraded":"2023-09-23T20:30:57.096452+0000","last_fullsized":"2023-09-23T20:30:57.096452+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:26:17.599051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.d","version":"290'12323","reported_seq":12432,"reported_epoch":290,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:31:00.285740+0000","last_change":"2023-09-23T20:29:59.564467+0000","last_active":"2023-09-23T20:31:00.285740+0000","last_peered":"2023-09-23T20:31:00.285740+0000","last_clean":"2023-09-23T20:28:12.085747+0000","last_became_active":"2023-09-23T20:29:46.294362+0000","last_became_peered":"2023-09-23T20:29:46.294362+0000","last_unstale":"2023-09-23T20:31:00.285740+0000","last_undegraded":"2023-09-23T20:29:46.239165+0000","last_fullsized":"2023-09-23T20:29:46.032265+0000","mapping_epoch":267,"log_start":"241'5559","ondisk_log_start":"241'5559","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":6764,"log_dups_size":0,"ondisk_log_size":6764,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:38:13.504714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":58851352,"num_objects":899,"num_object_clones":0,"num_object_copies":2697,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4915,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":899,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":12323,"num_write_kb":423041,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,0],"avail_no_missing":["5","0"],"object_location_counts":[{"shards":"0,5","objects":-1117}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":554,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:30:57.110431+0000","last_change":"2023-09-23T20:29:07.053561+0000","last_active":"2023-09-23T20:30:57.110431+0000","last_peered":"2023-09-23T20:30:57.110431+0000","last_clean":"2023-09-23T20:30:57.110431+0000","last_became_active":"2023-09-23T20:29:07.053177+0000","last_became_peered":"2023-09-23T20:29:07.053177+0000","last_unstale":"2023-09-23T20:30:57.110431+0000","last_undegraded":"2023-09-23T20:30:57.110431+0000","last_fullsized":"2023-09-23T20:30:57.110431+0000","mapping_epoch":253,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":254,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:30.727267+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:30.727267+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:04:32.661761+0000","scrub_duration":0.0052560289999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":152,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:30:57.095711+0000","last_change":"2023-09-23T20:29:45.529576+0000","last_active":"2023-09-23T20:30:57.095711+0000","last_peered":"2023-09-23T20:30:57.095711+0000","last_clean":"2023-09-23T20:30:57.095711+0000","last_became_active":"2023-09-23T20:29:45.528858+0000","last_became_peered":"2023-09-23T20:29:45.528858+0000","last_unstale":"2023-09-23T20:30:57.095711+0000","last_undegraded":"2023-09-23T20:30:57.095711+0000","last_fullsized":"2023-09-23T20:30:57.095711+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:42:15.589821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":100,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:30:57.102501+0000","last_change":"2023-09-23T20:29:46.064515+0000","last_active":"2023-09-23T20:30:57.102501+0000","last_peered":"2023-09-23T20:30:57.102501+0000","last_clean":"2023-09-23T20:30:57.102501+0000","last_became_active":"2023-09-23T20:29:46.064253+0000","last_became_peered":"2023-09-23T20:29:46.064253+0000","last_unstale":"2023-09-23T20:30:57.102501+0000","last_undegraded":"2023-09-23T20:30:57.102501+0000","last_fullsized":"2023-09-23T20:30:57.102501+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:21:20.039440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"289'9059","reported_seq":9152,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:30:59.842045+0000","last_change":"2023-09-23T20:30:23.834705+0000","last_active":"2023-09-23T20:30:59.842045+0000","last_peered":"2023-09-23T20:30:59.842045+0000","last_clean":"2023-09-23T20:30:59.842045+0000","last_became_active":"2023-09-23T20:28:35.532298+0000","last_became_peered":"2023-09-23T20:28:35.532298+0000","last_unstale":"2023-09-23T20:30:59.842045+0000","last_undegraded":"2023-09-23T20:30:59.842045+0000","last_fullsized":"2023-09-23T20:30:59.842045+0000","mapping_epoch":217,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":9059,"log_dups_size":0,"ondisk_log_size":9059,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":27688960,"num_objects":422,"num_object_clones":0,"num_object_copies":1266,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":422,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6240,"num_write_kb":213216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":509,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:30:57.096488+0000","last_change":"2023-09-23T20:29:07.115982+0000","last_active":"2023-09-23T20:30:57.096488+0000","last_peered":"2023-09-23T20:30:57.096488+0000","last_clean":"2023-09-23T20:30:57.096488+0000","last_became_active":"2023-09-23T20:29:07.115647+0000","last_became_peered":"2023-09-23T20:29:07.115647+0000","last_unstale":"2023-09-23T20:30:57.096488+0000","last_undegraded":"2023-09-23T20:30:57.096488+0000","last_fullsized":"2023-09-23T20:30:57.096488+0000","mapping_epoch":253,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":254,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:32.746340+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:32.746340+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"86'119","reported_seq":593,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:30:57.102264+0000","last_change":"2023-09-23T20:29:45.521257+0000","last_active":"2023-09-23T20:30:57.102264+0000","last_peered":"2023-09-23T20:30:57.102264+0000","last_clean":"2023-09-23T20:30:57.102264+0000","last_became_active":"2023-09-23T20:29:45.520546+0000","last_became_peered":"2023-09-23T20:29:45.520546+0000","last_unstale":"2023-09-23T20:30:57.102264+0000","last_undegraded":"2023-09-23T20:30:57.102264+0000","last_fullsized":"2023-09-23T20:30:57.102264+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":119,"log_dups_size":0,"ondisk_log_size":119,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:58:56.085777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032705499999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":523,"num_read_kb":3503,"num_write":136,"num_write_kb":3128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":306,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:30:57.113005+0000","last_change":"2023-09-23T20:25:39.637160+0000","last_active":"2023-09-23T20:30:57.113005+0000","last_peered":"2023-09-23T20:30:57.113005+0000","last_clean":"2023-09-23T20:30:57.113005+0000","last_became_active":"2023-09-23T20:25:39.636860+0000","last_became_peered":"2023-09-23T20:25:39.636860+0000","last_unstale":"2023-09-23T20:30:57.113005+0000","last_undegraded":"2023-09-23T20:30:57.113005+0000","last_fullsized":"2023-09-23T20:30:57.113005+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:21:47.860063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":195,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:30:57.101269+0000","last_change":"2023-09-23T20:29:46.090683+0000","last_active":"2023-09-23T20:30:57.101269+0000","last_peered":"2023-09-23T20:30:57.101269+0000","last_clean":"2023-09-23T20:30:57.101269+0000","last_became_active":"2023-09-23T20:29:46.090131+0000","last_became_peered":"2023-09-23T20:29:46.090131+0000","last_unstale":"2023-09-23T20:30:57.101269+0000","last_undegraded":"2023-09-23T20:30:57.101269+0000","last_fullsized":"2023-09-23T20:30:57.101269+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:36:35.754684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.6","version":"290'8914","reported_seq":9016,"reported_epoch":290,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T20:31:03.259960+0000","last_change":"2023-09-23T20:29:29.535022+0000","last_active":"2023-09-23T20:31:03.259960+0000","last_peered":"2023-09-23T20:31:03.259960+0000","last_clean":"2023-09-23T20:29:27.350688+0000","last_became_active":"2023-09-23T20:29:29.522984+0000","last_became_peered":"2023-09-23T20:29:29.522984+0000","last_unstale":"2023-09-23T20:31:03.259960+0000","last_undegraded":"2023-09-23T20:31:03.259960+0000","last_fullsized":"2023-09-23T20:31:03.259960+0000","mapping_epoch":259,"log_start":"224'3004","ondisk_log_start":"224'3004","created":217,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":5910,"log_dups_size":0,"ondisk_log_size":5910,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":20316160,"num_objects":310,"num_object_clones":0,"num_object_copies":930,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":310,"num_objects_unfound":0,"num_objects_dirty":310,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6172,"num_write_kb":207424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,4],"avail_no_missing":["3","4","7"],"object_location_counts":[{"shards":"3,4,7","objects":310}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":524,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:30:57.110444+0000","last_change":"2023-09-23T20:27:55.213707+0000","last_active":"2023-09-23T20:30:57.110444+0000","last_peered":"2023-09-23T20:30:57.110444+0000","last_clean":"2023-09-23T20:30:57.110444+0000","last_became_active":"2023-09-23T20:27:55.213394+0000","last_became_peered":"2023-09-23T20:27:55.213394+0000","last_unstale":"2023-09-23T20:30:57.110444+0000","last_undegraded":"2023-09-23T20:30:57.110444+0000","last_fullsized":"2023-09-23T20:30:57.110444+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:34.821789+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:34.821789+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0042962870000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"86'106","reported_seq":984,"reported_epoch":289,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T20:30:57.110372+0000","last_change":"2023-09-23T20:29:46.078905+0000","last_active":"2023-09-23T20:30:57.110372+0000","last_peered":"2023-09-23T20:30:57.110372+0000","last_clean":"2023-09-23T20:28:10.123134+0000","last_became_active":"2023-09-23T20:29:46.078501+0000","last_became_peered":"2023-09-23T20:29:46.078501+0000","last_unstale":"2023-09-23T20:30:57.110372+0000","last_undegraded":"2023-09-23T20:29:46.032390+0000","last_fullsized":"2023-09-23T20:30:57.110372+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":106,"log_dups_size":0,"ondisk_log_size":106,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:37.015974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00099302599999999993,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1105,"num_read_kb":1689,"num_write":100,"num_write_kb":404,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":["5","0","1"],"object_location_counts":[{"shards":"0,1,5","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":281,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-09-23T20:30:56.113469+0000","last_change":"2023-09-23T20:29:46.059640+0000","last_active":"2023-09-23T20:30:56.113469+0000","last_peered":"2023-09-23T20:30:56.113469+0000","last_clean":"2023-09-23T20:30:56.113469+0000","last_became_active":"2023-09-23T20:29:46.059279+0000","last_became_peered":"2023-09-23T20:29:46.059279+0000","last_unstale":"2023-09-23T20:30:56.113469+0000","last_undegraded":"2023-09-23T20:30:56.113469+0000","last_fullsized":"2023-09-23T20:30:56.113469+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:03:31.536847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":246,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:30:57.102209+0000","last_change":"2023-09-23T20:29:45.533406+0000","last_active":"2023-09-23T20:30:57.102209+0000","last_peered":"2023-09-23T20:30:57.102209+0000","last_clean":"2023-09-23T20:30:57.102209+0000","last_became_active":"2023-09-23T20:29:45.526001+0000","last_became_peered":"2023-09-23T20:29:45.526001+0000","last_unstale":"2023-09-23T20:30:57.102209+0000","last_undegraded":"2023-09-23T20:30:57.102209+0000","last_fullsized":"2023-09-23T20:30:57.102209+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:43.787251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.5","version":"290'8831","reported_seq":12420,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T20:31:00.680517+0000","last_change":"2023-09-23T20:30:57.151375+0000","last_active":"2023-09-23T20:31:00.680517+0000","last_peered":"2023-09-23T20:31:00.680517+0000","last_clean":"2023-09-23T20:31:00.680517+0000","last_became_active":"2023-09-23T20:30:57.098276+0000","last_became_peered":"2023-09-23T20:30:57.098276+0000","last_unstale":"2023-09-23T20:31:00.680517+0000","last_undegraded":"2023-09-23T20:31:00.680517+0000","last_fullsized":"2023-09-23T20:31:00.680517+0000","mapping_epoch":288,"log_start":"221'1237","ondisk_log_start":"221'1237","created":217,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":7594,"log_dups_size":0,"ondisk_log_size":7594,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":29261824,"num_objects":446,"num_object_clones":0,"num_object_copies":1338,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":446,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6090,"num_write_kb":209184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1733,"num_bytes_recovered":70320128,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":702,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:30:57.093457+0000","last_change":"2023-09-23T20:29:31.389645+0000","last_active":"2023-09-23T20:30:57.093457+0000","last_peered":"2023-09-23T20:30:57.093457+0000","last_clean":"2023-09-23T20:30:57.093457+0000","last_became_active":"2023-09-23T20:29:31.389288+0000","last_became_peered":"2023-09-23T20:29:31.389288+0000","last_unstale":"2023-09-23T20:30:57.093457+0000","last_undegraded":"2023-09-23T20:30:57.093457+0000","last_fullsized":"2023-09-23T20:30:57.093457+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:35.734584+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:35.734584+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0035251380000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,2],"acting":[7,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"86'130","reported_seq":601,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:30:57.108741+0000","last_change":"2023-09-23T20:25:39.637458+0000","last_active":"2023-09-23T20:30:57.108741+0000","last_peered":"2023-09-23T20:30:57.108741+0000","last_clean":"2023-09-23T20:30:57.108741+0000","last_became_active":"2023-09-23T20:25:39.636773+0000","last_became_peered":"2023-09-23T20:25:39.636773+0000","last_unstale":"2023-09-23T20:30:57.108741+0000","last_undegraded":"2023-09-23T20:30:57.108741+0000","last_fullsized":"2023-09-23T20:30:57.108741+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:00:32.810564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011300049999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":512,"num_read_kb":1820,"num_write":133,"num_write_kb":1347,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":236,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:30:57.101190+0000","last_change":"2023-09-23T20:29:46.073394+0000","last_active":"2023-09-23T20:30:57.101190+0000","last_peered":"2023-09-23T20:30:57.101190+0000","last_clean":"2023-09-23T20:30:57.101190+0000","last_became_active":"2023-09-23T20:29:46.072067+0000","last_became_peered":"2023-09-23T20:29:46.072067+0000","last_unstale":"2023-09-23T20:30:57.101190+0000","last_undegraded":"2023-09-23T20:30:57.101190+0000","last_fullsized":"2023-09-23T20:30:57.101190+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:01:56.944614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":228,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:30:57.102460+0000","last_change":"2023-09-23T20:29:45.533185+0000","last_active":"2023-09-23T20:30:57.102460+0000","last_peered":"2023-09-23T20:30:57.102460+0000","last_clean":"2023-09-23T20:30:57.102460+0000","last_became_active":"2023-09-23T20:29:45.526765+0000","last_became_peered":"2023-09-23T20:29:45.526765+0000","last_unstale":"2023-09-23T20:30:57.102460+0000","last_undegraded":"2023-09-23T20:30:57.102460+0000","last_fullsized":"2023-09-23T20:30:57.102460+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:18:33.063215+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"290'9017","reported_seq":9131,"reported_epoch":290,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:31:00.224242+0000","last_change":"2023-09-23T20:29:46.386541+0000","last_active":"2023-09-23T20:31:00.224242+0000","last_peered":"2023-09-23T20:31:00.224242+0000","last_clean":"2023-09-23T20:29:27.161343+0000","last_became_active":"2023-09-23T20:29:46.365588+0000","last_became_peered":"2023-09-23T20:29:46.365588+0000","last_unstale":"2023-09-23T20:31:00.224242+0000","last_undegraded":"2023-09-23T20:29:46.092299+0000","last_fullsized":"2023-09-23T20:29:46.032978+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":9017,"log_dups_size":0,"ondisk_log_size":9017,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":25591808,"num_objects":390,"num_object_clones":0,"num_object_copies":1170,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3300,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":390,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6188,"num_write_kb":210528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3],"avail_no_missing":["5","3","4"],"object_location_counts":[{"shards":"3,4,5","objects":401},{"shards":"3,5","objects":-1460}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"86'194","reported_seq":938,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:30:57.110706+0000","last_change":"2023-09-23T20:25:40.618665+0000","last_active":"2023-09-23T20:30:57.110706+0000","last_peered":"2023-09-23T20:30:57.110706+0000","last_clean":"2023-09-23T20:30:57.110706+0000","last_became_active":"2023-09-23T20:25:40.618087+0000","last_became_peered":"2023-09-23T20:25:40.618087+0000","last_unstale":"2023-09-23T20:30:57.110706+0000","last_undegraded":"2023-09-23T20:30:57.110706+0000","last_fullsized":"2023-09-23T20:30:57.110706+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":194,"log_dups_size":0,"ondisk_log_size":194,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:07:16.114968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00037995999999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1025,"num_read_kb":1201,"num_write":173,"num_write_kb":1276,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":713,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:30:57.093184+0000","last_change":"2023-09-23T20:29:07.042758+0000","last_active":"2023-09-23T20:30:57.093184+0000","last_peered":"2023-09-23T20:30:57.093184+0000","last_clean":"2023-09-23T20:30:57.093184+0000","last_became_active":"2023-09-23T20:29:07.042407+0000","last_became_peered":"2023-09-23T20:29:07.042407+0000","last_unstale":"2023-09-23T20:30:57.093184+0000","last_undegraded":"2023-09-23T20:30:57.093184+0000","last_fullsized":"2023-09-23T20:30:57.093184+0000","mapping_epoch":253,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":254,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:32.801168+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:32.801168+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0096333110000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":155,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:30:57.097039+0000","last_change":"2023-09-23T20:29:04.216400+0000","last_active":"2023-09-23T20:30:57.097039+0000","last_peered":"2023-09-23T20:30:57.097039+0000","last_clean":"2023-09-23T20:30:57.097039+0000","last_became_active":"2023-09-23T20:29:04.216069+0000","last_became_peered":"2023-09-23T20:29:04.216069+0000","last_unstale":"2023-09-23T20:30:57.097039+0000","last_undegraded":"2023-09-23T20:30:57.097039+0000","last_fullsized":"2023-09-23T20:30:57.097039+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:27:37.433035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":59,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:30:57.096569+0000","last_change":"2023-09-23T20:29:28.376036+0000","last_active":"2023-09-23T20:30:57.096569+0000","last_peered":"2023-09-23T20:30:57.096569+0000","last_clean":"2023-09-23T20:30:57.096569+0000","last_became_active":"2023-09-23T20:29:28.375712+0000","last_became_peered":"2023-09-23T20:29:28.375712+0000","last_unstale":"2023-09-23T20:30:57.096569+0000","last_undegraded":"2023-09-23T20:30:57.096569+0000","last_fullsized":"2023-09-23T20:30:57.096569+0000","mapping_epoch":258,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":259,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:23:50.722625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.4","version":"290'8961","reported_seq":9062,"reported_epoch":290,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:31:03.357175+0000","last_change":"2023-09-23T20:29:46.186959+0000","last_active":"2023-09-23T20:31:03.357175+0000","last_peered":"2023-09-23T20:31:03.357175+0000","last_clean":"2023-09-23T20:28:12.083147+0000","last_became_active":"2023-09-23T20:29:46.162334+0000","last_became_peered":"2023-09-23T20:29:46.162334+0000","last_unstale":"2023-09-23T20:31:03.357175+0000","last_undegraded":"2023-09-23T20:29:46.054138+0000","last_fullsized":"2023-09-23T20:29:46.043728+0000","mapping_epoch":267,"log_start":"224'2960","ondisk_log_start":"224'2960","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":6001,"log_dups_size":0,"ondisk_log_size":6001,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":28114944,"num_objects":429,"num_object_clones":0,"num_object_copies":1287,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2427,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":429,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6197,"num_write_kb":212032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[3,5],"avail_no_missing":["3","5"],"object_location_counts":[{"shards":"3,5","objects":-614}],"blocked_by":[],"up_primary":1,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":518,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:30:57.101830+0000","last_change":"2023-09-23T20:29:46.072303+0000","last_active":"2023-09-23T20:30:57.101830+0000","last_peered":"2023-09-23T20:30:57.101830+0000","last_clean":"2023-09-23T20:30:57.101830+0000","last_became_active":"2023-09-23T20:29:46.071938+0000","last_became_peered":"2023-09-23T20:29:46.071938+0000","last_unstale":"2023-09-23T20:30:57.101830+0000","last_undegraded":"2023-09-23T20:30:57.101830+0000","last_fullsized":"2023-09-23T20:30:57.101830+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:12.386098+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:12.386098+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,5],"acting":[1,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"86'80","reported_seq":553,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:30:57.093124+0000","last_change":"2023-09-23T20:29:45.536796+0000","last_active":"2023-09-23T20:30:57.093124+0000","last_peered":"2023-09-23T20:30:57.093124+0000","last_clean":"2023-09-23T20:30:57.093124+0000","last_became_active":"2023-09-23T20:29:45.536439+0000","last_became_peered":"2023-09-23T20:29:45.536439+0000","last_unstale":"2023-09-23T20:30:57.093124+0000","last_undegraded":"2023-09-23T20:30:57.093124+0000","last_fullsized":"2023-09-23T20:30:57.093124+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":80,"log_dups_size":0,"ondisk_log_size":80,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:30:57.793132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035115,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":395,"num_read_kb":1370,"num_write":66,"num_write_kb":1061,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":262,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-09-23T20:30:56.111822+0000","last_change":"2023-09-23T20:29:28.360680+0000","last_active":"2023-09-23T20:30:56.111822+0000","last_peered":"2023-09-23T20:30:56.111822+0000","last_clean":"2023-09-23T20:30:56.111822+0000","last_became_active":"2023-09-23T20:29:28.360366+0000","last_became_peered":"2023-09-23T20:29:28.360366+0000","last_unstale":"2023-09-23T20:30:56.111822+0000","last_undegraded":"2023-09-23T20:30:56.111822+0000","last_fullsized":"2023-09-23T20:30:56.111822+0000","mapping_epoch":258,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":259,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:02:40.594828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":284,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:30:57.091622+0000","last_change":"2023-09-23T20:29:45.690815+0000","last_active":"2023-09-23T20:30:57.091622+0000","last_peered":"2023-09-23T20:30:57.091622+0000","last_clean":"2023-09-23T20:30:57.091622+0000","last_became_active":"2023-09-23T20:29:45.690478+0000","last_became_peered":"2023-09-23T20:29:45.690478+0000","last_unstale":"2023-09-23T20:30:57.091622+0000","last_undegraded":"2023-09-23T20:30:57.091622+0000","last_fullsized":"2023-09-23T20:30:57.091622+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:38.288193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"290'8823","reported_seq":13964,"reported_epoch":290,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:31:00.373094+0000","last_change":"2023-09-23T20:29:59.519844+0000","last_active":"2023-09-23T20:31:00.373094+0000","last_peered":"2023-09-23T20:31:00.373094+0000","last_clean":"2023-09-23T20:28:04.011715+0000","last_became_active":"2023-09-23T20:29:46.206627+0000","last_became_peered":"2023-09-23T20:29:46.206627+0000","last_unstale":"2023-09-23T20:31:00.373094+0000","last_undegraded":"2023-09-23T20:29:59.519468+0000","last_fullsized":"2023-09-23T20:29:46.046256+0000","mapping_epoch":267,"log_start":"222'1250","ondisk_log_start":"222'1250","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":7573,"log_dups_size":0,"ondisk_log_size":7573,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":25231360,"num_objects":385,"num_object_clones":0,"num_object_copies":1155,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1729,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":385,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6069,"num_write_kb":206528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2160,"num_bytes_recovered":141557760,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[6,2],"avail_no_missing":["6","2"],"object_location_counts":[{"shards":"2,6","objects":57},{"shards":"2,6,7","objects":328}],"blocked_by":[],"up_primary":1,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":502,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:30:57.096687+0000","last_change":"2023-09-23T20:29:07.113747+0000","last_active":"2023-09-23T20:30:57.096687+0000","last_peered":"2023-09-23T20:30:57.096687+0000","last_clean":"2023-09-23T20:30:57.096687+0000","last_became_active":"2023-09-23T20:29:07.113428+0000","last_became_peered":"2023-09-23T20:29:07.113428+0000","last_unstale":"2023-09-23T20:30:57.096687+0000","last_undegraded":"2023-09-23T20:30:57.096687+0000","last_fullsized":"2023-09-23T20:30:57.096687+0000","mapping_epoch":253,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":254,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:31.769333+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:31.769333+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"86'102","reported_seq":818,"reported_epoch":289,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:30:57.093913+0000","last_change":"2023-09-23T20:29:46.060111+0000","last_active":"2023-09-23T20:30:57.093913+0000","last_peered":"2023-09-23T20:30:57.093913+0000","last_clean":"2023-09-23T20:29:43.509457+0000","last_became_active":"2023-09-23T20:29:46.059787+0000","last_became_peered":"2023-09-23T20:29:46.059787+0000","last_unstale":"2023-09-23T20:30:57.093913+0000","last_undegraded":"2023-09-23T20:29:46.049994+0000","last_fullsized":"2023-09-23T20:29:46.048495+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":260,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":102,"log_dups_size":0,"ondisk_log_size":102,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:53:45.933578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030656500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":963,"num_read_kb":1234,"num_write":60,"num_write_kb":225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0],"avail_no_missing":["7","0","4"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":212,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:30:57.102298+0000","last_change":"2023-09-23T20:29:45.538283+0000","last_active":"2023-09-23T20:30:57.102298+0000","last_peered":"2023-09-23T20:30:57.102298+0000","last_clean":"2023-09-23T20:30:57.102298+0000","last_became_active":"2023-09-23T20:29:45.537922+0000","last_became_peered":"2023-09-23T20:29:45.537922+0000","last_unstale":"2023-09-23T20:30:57.102298+0000","last_undegraded":"2023-09-23T20:30:57.102298+0000","last_fullsized":"2023-09-23T20:30:57.102298+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:30:52.397836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":221,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-09-23T20:30:56.113398+0000","last_change":"2023-09-23T20:20:14.888874+0000","last_active":"2023-09-23T20:30:56.113398+0000","last_peered":"2023-09-23T20:30:56.113398+0000","last_clean":"2023-09-23T20:30:56.113398+0000","last_became_active":"2023-09-23T20:20:14.887724+0000","last_became_peered":"2023-09-23T20:20:14.887724+0000","last_unstale":"2023-09-23T20:30:56.113398+0000","last_undegraded":"2023-09-23T20:30:56.113398+0000","last_fullsized":"2023-09-23T20:30:56.113398+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:09:24.510326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"290'9152","reported_seq":9257,"reported_epoch":290,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:31:03.338766+0000","last_change":"2023-09-23T20:29:29.642590+0000","last_active":"2023-09-23T20:31:03.338766+0000","last_peered":"2023-09-23T20:31:03.338766+0000","last_clean":"2023-09-23T20:29:27.208118+0000","last_became_active":"2023-09-23T20:29:29.628868+0000","last_became_peered":"2023-09-23T20:29:29.628868+0000","last_unstale":"2023-09-23T20:31:03.338766+0000","last_undegraded":"2023-09-23T20:29:29.393326+0000","last_fullsized":"2023-09-23T20:29:29.348340+0000","mapping_epoch":259,"log_start":"224'3332","ondisk_log_start":"224'3332","created":217,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":5820,"log_dups_size":0,"ondisk_log_size":5820,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17367040,"num_objects":265,"num_object_clones":0,"num_object_copies":795,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3393,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":265,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6341,"num_write_kb":211392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2],"avail_no_missing":["3","2","4"],"object_location_counts":[{"shards":"2,3","objects":-2068},{"shards":"2,3,4","objects":355}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":459,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:30:57.110113+0000","last_change":"2023-09-23T20:29:28.372908+0000","last_active":"2023-09-23T20:30:57.110113+0000","last_peered":"2023-09-23T20:30:57.110113+0000","last_clean":"2023-09-23T20:30:57.110113+0000","last_became_active":"2023-09-23T20:29:28.372652+0000","last_became_peered":"2023-09-23T20:29:28.372652+0000","last_unstale":"2023-09-23T20:30:57.110113+0000","last_undegraded":"2023-09-23T20:30:57.110113+0000","last_fullsized":"2023-09-23T20:30:57.110113+0000","mapping_epoch":258,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":259,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:12.809933+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:12.809933+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,5],"acting":[2,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":141,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:30:57.097840+0000","last_change":"2023-09-23T20:25:03.007736+0000","last_active":"2023-09-23T20:30:57.097840+0000","last_peered":"2023-09-23T20:30:57.097840+0000","last_clean":"2023-09-23T20:30:57.097840+0000","last_became_active":"2023-09-23T20:25:02.973689+0000","last_became_peered":"2023-09-23T20:25:02.973689+0000","last_unstale":"2023-09-23T20:30:57.097840+0000","last_undegraded":"2023-09-23T20:30:57.097840+0000","last_fullsized":"2023-09-23T20:30:57.097840+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:37:25.643774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"86'128","reported_seq":645,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:30:57.102350+0000","last_change":"2023-09-23T20:30:45.008506+0000","last_active":"2023-09-23T20:30:57.102350+0000","last_peered":"2023-09-23T20:30:57.102350+0000","last_clean":"2023-09-23T20:30:57.102350+0000","last_became_active":"2023-09-23T20:29:46.071848+0000","last_became_peered":"2023-09-23T20:29:46.071848+0000","last_unstale":"2023-09-23T20:30:57.102350+0000","last_undegraded":"2023-09-23T20:30:57.102350+0000","last_fullsized":"2023-09-23T20:30:57.102350+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":283,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:44:18.031530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000293202,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":579,"num_read_kb":1473,"num_write":96,"num_write_kb":1062,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":138,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T20:31:00.222015+0000","last_change":"2023-09-23T20:25:02.971740+0000","last_active":"2023-09-23T20:31:00.222015+0000","last_peered":"2023-09-23T20:31:00.222015+0000","last_clean":"2023-09-23T20:31:00.222015+0000","last_became_active":"2023-09-23T20:25:02.970412+0000","last_became_peered":"2023-09-23T20:25:02.970412+0000","last_unstale":"2023-09-23T20:31:00.222015+0000","last_undegraded":"2023-09-23T20:31:00.222015+0000","last_fullsized":"2023-09-23T20:31:00.222015+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:53:17.247216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"290'8928","reported_seq":9358,"reported_epoch":290,"state":"active+recovery_wait","last_fresh":"2023-09-23T20:31:00.688999+0000","last_change":"2023-09-23T20:30:30.444650+0000","last_active":"2023-09-23T20:31:00.688999+0000","last_peered":"2023-09-23T20:31:00.688999+0000","last_clean":"2023-09-23T20:28:04.010682+0000","last_became_active":"2023-09-23T20:29:47.069776+0000","last_became_peered":"2023-09-23T20:29:47.069776+0000","last_unstale":"2023-09-23T20:31:00.688999+0000","last_undegraded":"2023-09-23T20:31:00.688999+0000","last_fullsized":"2023-09-23T20:31:00.688999+0000","mapping_epoch":268,"log_start":"222'1518","ondisk_log_start":"222'1518","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":7410,"log_dups_size":0,"ondisk_log_size":7410,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":21463040,"num_objects":327,"num_object_clones":0,"num_object_copies":981,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":327,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6159,"num_write_kb":207584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":149,"num_bytes_recovered":9797632,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":["7","2","6"],"object_location_counts":[{"shards":"2,6,7","objects":327}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.4","version":"90'161","reported_seq":537,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:30:57.098354+0000","last_change":"2023-09-23T20:28:03.553294+0000","last_active":"2023-09-23T20:30:57.098354+0000","last_peered":"2023-09-23T20:30:57.098354+0000","last_clean":"2023-09-23T20:30:57.098354+0000","last_became_active":"2023-09-23T20:27:55.212201+0000","last_became_peered":"2023-09-23T20:27:55.212201+0000","last_unstale":"2023-09-23T20:30:57.098354+0000","last_undegraded":"2023-09-23T20:30:57.098354+0000","last_fullsized":"2023-09-23T20:30:57.098354+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:28:03.553106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:28:03.553106+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:24:06.706940+0000","scrub_duration":0.011176864,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"86'117","reported_seq":603,"reported_epoch":289,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:30:57.109486+0000","last_change":"2023-09-23T20:29:59.542691+0000","last_active":"2023-09-23T20:30:57.109486+0000","last_peered":"2023-09-23T20:30:57.109486+0000","last_clean":"2023-09-23T20:13:53.392235+0000","last_became_active":"2023-09-23T20:29:05.138184+0000","last_became_peered":"2023-09-23T20:29:05.138184+0000","last_unstale":"2023-09-23T20:30:57.109486+0000","last_undegraded":"2023-09-23T20:29:59.542326+0000","last_fullsized":"2023-09-23T20:29:05.108660+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:50:02.080933+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046891400000000002,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":893,"num_read_kb":922,"num_write":82,"num_write_kb":4193,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[2,7],"avail_no_missing":["2","3","7"],"object_location_counts":[{"shards":"2,3,7","objects":1}],"blocked_by":[],"up_primary":6,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":271,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:30:57.091561+0000","last_change":"2023-09-23T20:29:46.053198+0000","last_active":"2023-09-23T20:30:57.091561+0000","last_peered":"2023-09-23T20:30:57.091561+0000","last_clean":"2023-09-23T20:30:57.091561+0000","last_became_active":"2023-09-23T20:29:46.052866+0000","last_became_peered":"2023-09-23T20:29:46.052866+0000","last_unstale":"2023-09-23T20:30:57.091561+0000","last_undegraded":"2023-09-23T20:30:57.091561+0000","last_fullsized":"2023-09-23T20:30:57.091561+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:30:18.857140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":247,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:30:57.110503+0000","last_change":"2023-09-23T20:29:28.367722+0000","last_active":"2023-09-23T20:30:57.110503+0000","last_peered":"2023-09-23T20:30:57.110503+0000","last_clean":"2023-09-23T20:30:57.110503+0000","last_became_active":"2023-09-23T20:29:28.367404+0000","last_became_peered":"2023-09-23T20:29:28.367404+0000","last_unstale":"2023-09-23T20:30:57.110503+0000","last_undegraded":"2023-09-23T20:30:57.110503+0000","last_fullsized":"2023-09-23T20:30:57.110503+0000","mapping_epoch":258,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":259,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:02:51.886571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":243,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-09-23T20:30:56.113329+0000","last_change":"2023-09-23T20:29:46.061960+0000","last_active":"2023-09-23T20:30:56.113329+0000","last_peered":"2023-09-23T20:30:56.113329+0000","last_clean":"2023-09-23T20:30:56.113329+0000","last_became_active":"2023-09-23T20:29:46.061601+0000","last_became_peered":"2023-09-23T20:29:46.061601+0000","last_unstale":"2023-09-23T20:30:56.113329+0000","last_undegraded":"2023-09-23T20:30:56.113329+0000","last_fullsized":"2023-09-23T20:30:56.113329+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:15:52.620692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":141,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:30:57.109902+0000","last_change":"2023-09-23T20:29:45.538314+0000","last_active":"2023-09-23T20:30:57.109902+0000","last_peered":"2023-09-23T20:30:57.109902+0000","last_clean":"2023-09-23T20:30:57.109902+0000","last_became_active":"2023-09-23T20:29:45.536847+0000","last_became_peered":"2023-09-23T20:29:45.536847+0000","last_unstale":"2023-09-23T20:30:57.109902+0000","last_undegraded":"2023-09-23T20:30:57.109902+0000","last_fullsized":"2023-09-23T20:30:57.109902+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:51.948721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"6.19","version":"289'8844","reported_seq":8935,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:30:59.961203+0000","last_change":"2023-09-23T20:30:29.820384+0000","last_active":"2023-09-23T20:30:59.961203+0000","last_peered":"2023-09-23T20:30:59.961203+0000","last_clean":"2023-09-23T20:30:59.961203+0000","last_became_active":"2023-09-23T20:28:35.567626+0000","last_became_peered":"2023-09-23T20:28:35.567626+0000","last_unstale":"2023-09-23T20:30:59.961203+0000","last_undegraded":"2023-09-23T20:30:59.961203+0000","last_fullsized":"2023-09-23T20:30:59.961203+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":242,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":8844,"log_dups_size":0,"ondisk_log_size":8844,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":33423360,"num_objects":510,"num_object_clones":0,"num_object_copies":1530,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":510,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6048,"num_write_kb":209856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":285,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:30:57.110901+0000","last_change":"2023-09-23T20:29:59.578082+0000","last_active":"2023-09-23T20:30:57.110901+0000","last_peered":"2023-09-23T20:30:57.110901+0000","last_clean":"2023-09-23T20:30:57.110901+0000","last_became_active":"2023-09-23T20:29:46.072626+0000","last_became_peered":"2023-09-23T20:29:46.072626+0000","last_unstale":"2023-09-23T20:30:57.110901+0000","last_undegraded":"2023-09-23T20:30:57.110901+0000","last_fullsized":"2023-09-23T20:30:57.110901+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:24:14.704823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":297,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:30:57.096885+0000","last_change":"2023-09-23T20:29:46.070732+0000","last_active":"2023-09-23T20:30:57.096885+0000","last_peered":"2023-09-23T20:30:57.096885+0000","last_clean":"2023-09-23T20:30:57.096885+0000","last_became_active":"2023-09-23T20:29:46.069861+0000","last_became_peered":"2023-09-23T20:29:46.069861+0000","last_unstale":"2023-09-23T20:30:57.096885+0000","last_undegraded":"2023-09-23T20:30:57.096885+0000","last_fullsized":"2023-09-23T20:30:57.096885+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:33:00.813261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.18","version":"290'8927","reported_seq":9018,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T20:31:00.600423+0000","last_change":"2023-09-23T20:28:35.587755+0000","last_active":"2023-09-23T20:31:00.600423+0000","last_peered":"2023-09-23T20:31:00.600423+0000","last_clean":"2023-09-23T20:31:00.600423+0000","last_became_active":"2023-09-23T20:28:35.585502+0000","last_became_peered":"2023-09-23T20:28:35.585502+0000","last_unstale":"2023-09-23T20:31:00.600423+0000","last_undegraded":"2023-09-23T20:31:00.600423+0000","last_fullsized":"2023-09-23T20:31:00.600423+0000","mapping_epoch":242,"log_start":"222'1429","ondisk_log_start":"222'1429","created":242,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":7498,"log_dups_size":0,"ondisk_log_size":7498,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":25624576,"num_objects":391,"num_object_clones":0,"num_object_copies":1173,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":391,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6141,"num_write_kb":209024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":293,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:30:57.112877+0000","last_change":"2023-09-23T20:29:46.242242+0000","last_active":"2023-09-23T20:30:57.112877+0000","last_peered":"2023-09-23T20:30:57.112877+0000","last_clean":"2023-09-23T20:30:57.112877+0000","last_became_active":"2023-09-23T20:29:46.241216+0000","last_became_peered":"2023-09-23T20:29:46.241216+0000","last_unstale":"2023-09-23T20:30:57.112877+0000","last_undegraded":"2023-09-23T20:30:57.112877+0000","last_fullsized":"2023-09-23T20:30:57.112877+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:12:48.530720+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":205,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:30:57.097116+0000","last_change":"2023-09-23T20:29:28.353700+0000","last_active":"2023-09-23T20:30:57.097116+0000","last_peered":"2023-09-23T20:30:57.097116+0000","last_clean":"2023-09-23T20:30:57.097116+0000","last_became_active":"2023-09-23T20:29:28.353357+0000","last_became_peered":"2023-09-23T20:29:28.353357+0000","last_unstale":"2023-09-23T20:30:57.097116+0000","last_undegraded":"2023-09-23T20:30:57.097116+0000","last_fullsized":"2023-09-23T20:30:57.097116+0000","mapping_epoch":258,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":259,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:12.077050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":171,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T20:31:00.222887+0000","last_change":"2023-09-23T20:29:04.138540+0000","last_active":"2023-09-23T20:31:00.222887+0000","last_peered":"2023-09-23T20:31:00.222887+0000","last_clean":"2023-09-23T20:31:00.222887+0000","last_became_active":"2023-09-23T20:29:04.138182+0000","last_became_peered":"2023-09-23T20:29:04.138182+0000","last_unstale":"2023-09-23T20:31:00.222887+0000","last_undegraded":"2023-09-23T20:31:00.222887+0000","last_fullsized":"2023-09-23T20:31:00.222887+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:26.098284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":146,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:30:57.095721+0000","last_change":"2023-09-23T20:25:39.622819+0000","last_active":"2023-09-23T20:30:57.095721+0000","last_peered":"2023-09-23T20:30:57.095721+0000","last_clean":"2023-09-23T20:30:57.095721+0000","last_became_active":"2023-09-23T20:25:39.622484+0000","last_became_peered":"2023-09-23T20:25:39.622484+0000","last_unstale":"2023-09-23T20:30:57.095721+0000","last_undegraded":"2023-09-23T20:30:57.095721+0000","last_fullsized":"2023-09-23T20:30:57.095721+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:51:53.973261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":269,"reported_epoch":288,"state":"active+clean","last_fresh":"2023-09-23T20:30:56.114331+0000","last_change":"2023-09-23T20:25:39.668882+0000","last_active":"2023-09-23T20:30:56.114331+0000","last_peered":"2023-09-23T20:30:56.114331+0000","last_clean":"2023-09-23T20:30:56.114331+0000","last_became_active":"2023-09-23T20:25:39.637128+0000","last_became_peered":"2023-09-23T20:25:39.637128+0000","last_unstale":"2023-09-23T20:30:56.114331+0000","last_undegraded":"2023-09-23T20:30:56.114331+0000","last_fullsized":"2023-09-23T20:30:56.114331+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:56.406298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":305,"reported_epoch":289,"state":"active+clean","last_fresh":"2023-09-23T20:30:57.096279+0000","last_change":"2023-09-23T20:29:46.082412+0000","last_active":"2023-09-23T20:30:57.096279+0000","last_peered":"2023-09-23T20:30:57.096279+0000","last_clean":"2023-09-23T20:30:57.096279+0000","last_became_active":"2023-09-23T20:29:46.082028+0000","last_became_peered":"2023-09-23T20:29:46.082028+0000","last_unstale":"2023-09-23T20:30:57.096279+0000","last_undegraded":"2023-09-23T20:30:57.096279+0000","last_fullsized":"2023-09-23T20:30:57.096279+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:48:07.017969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":26,"stat_sum":{"num_bytes":834011160,"num_objects":12727,"num_object_clones":0,"num_object_copies":38181,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":29228,"num_objects_misplaced":1505,"num_objects_unfound":0,"num_objects_dirty":12727,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":196747,"num_write_kb":6703105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21432,"num_bytes_recovered":1150091264,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":4401078272,"data_stored":4401070128,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":186451,"ondisk_log_size":186451,"up":78,"acting":67,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":6376,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":122880,"data_stored":21392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26691,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":740547,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":214,"num_read_kb":401,"num_write":106,"num_write_kb":1063,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1433573,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2437120,"data_stored":2426464,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3381,"ondisk_log_size":3381,"up":63,"acting":63,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":26,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":17,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6082,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":22,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201518,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":908580,"kb_used_data":680092,"kb_used_omap":0,"kb_used_meta":228480,"kb_avail":92836572,"statfs":{"total":95995035648,"available":95064649728,"internally_reserved":0,"allocated":696414208,"data_stored":695491626,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":233963520},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,1,0,1,14],"upper_bound":1024},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:30:34 2023","interfaces":[{"interface":"back","average":{"1min":1.1459999999999999,"5min":1.96,"15min":1.163},"min":{"1min":0.25800000000000001,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":3.4790000000000001,"5min":48.798999999999999,"15min":48.798999999999999},"last":0.39900000000000002},{"interface":"front","average":{"1min":2.617,"5min":2.4209999999999998,"15min":1.3300000000000001},"min":{"1min":0.308,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":7.8179999999999996,"5min":48.247,"15min":48.247},"last":0.41799999999999998}]},{"osd":1,"last update":"Sat Sep 23 20:30:47 2023","interfaces":[{"interface":"back","average":{"1min":1.762,"5min":1.762,"15min":1.762},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":9.7370000000000001,"5min":9.7370000000000001,"15min":9.7370000000000001},"last":0.44},{"interface":"front","average":{"1min":1.9810000000000001,"5min":1.9810000000000001,"15min":1.9810000000000001},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":5.7679999999999998,"5min":5.7679999999999998,"15min":5.7679999999999998},"last":0.48099999999999998}]},{"osd":2,"last update":"Sat Sep 23 20:30:38 2023","interfaces":[{"interface":"back","average":{"1min":2.9300000000000002,"5min":2.0249999999999999,"15min":1.2210000000000001},"min":{"1min":0.35899999999999999,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":8.2509999999999994,"5min":11.372999999999999,"15min":11.372999999999999},"last":0.53000000000000003},{"interface":"front","average":{"1min":2.9729999999999999,"5min":2.2709999999999999,"15min":1.349},"min":{"1min":0.33500000000000002,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":14.993,"5min":14.993,"15min":14.993},"last":0.48999999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:30:01 2023","interfaces":[{"interface":"back","average":{"1min":2.3210000000000002,"5min":2.2429999999999999,"15min":1.254},"min":{"1min":0.29199999999999998,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":10.397,"5min":10.397,"15min":10.397},"last":0.54100000000000004},{"interface":"front","average":{"1min":1.0649999999999999,"5min":2.0449999999999999,"15min":1.159},"min":{"1min":0.33100000000000002,"5min":0.32500000000000001,"15min":0.27600000000000002},"max":{"1min":2.8940000000000001,"5min":9.7840000000000007,"15min":9.7840000000000007},"last":0.60599999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:30:01 2023","interfaces":[{"interface":"back","average":{"1min":3.5870000000000002,"5min":1.7490000000000001,"15min":0.98899999999999999},"min":{"1min":0.28399999999999997,"5min":0.27100000000000002,"15min":0.27000000000000002},"max":{"1min":15.177,"5min":15.177,"15min":15.177},"last":0.51700000000000002},{"interface":"front","average":{"1min":2.0430000000000001,"5min":2.0390000000000001,"15min":1.1060000000000001},"min":{"1min":0.36599999999999999,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":9.3740000000000006,"5min":11.122,"15min":11.122},"last":0.56100000000000005}]},{"osd":5,"last update":"Sat Sep 23 20:30:01 2023","interfaces":[{"interface":"back","average":{"1min":1.7509999999999999,"5min":1.899,"15min":1.0880000000000001},"min":{"1min":0.33000000000000002,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":6.8300000000000001,"5min":13.241,"15min":13.241},"last":0.61699999999999999},{"interface":"front","average":{"1min":2.9249999999999998,"5min":2.1150000000000002,"15min":1.1910000000000001},"min":{"1min":0.34999999999999998,"5min":0.315,"15min":0.315},"max":{"1min":18.831,"5min":18.831,"15min":18.831},"last":0.64800000000000002}]},{"osd":6,"last update":"Sat Sep 23 20:30:01 2023","interfaces":[{"interface":"back","average":{"1min":3.0550000000000002,"5min":1.873,"15min":1.087},"min":{"1min":0.41899999999999998,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":12.298999999999999,"5min":12.298999999999999,"15min":13.185},"last":0.59199999999999997},{"interface":"front","average":{"1min":1.6619999999999999,"5min":1.7010000000000001,"15min":1.022},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.29899999999999999},"max":{"1min":8.0069999999999997,"5min":9.9120000000000008,"15min":9.9120000000000008},"last":0.63500000000000001}]}]},{"osd":6,"up_from":47,"seq":201863463154,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":474908,"kb_used_data":283604,"kb_used_omap":0,"kb_used_meta":191296,"kb_avail":93270244,"statfs":{"total":95995035648,"available":95508729856,"internally_reserved":0,"allocated":290410496,"data_stored":289495994,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":195887104},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:30:34 2023","interfaces":[{"interface":"back","average":{"1min":1.446,"5min":1.413,"15min":0.997},"min":{"1min":0.23300000000000001,"5min":0.221,"15min":0.221},"max":{"1min":7.609,"5min":8.4209999999999994,"15min":8.4209999999999994},"last":0.32300000000000001},{"interface":"front","average":{"1min":2.1619999999999999,"5min":1.7949999999999999,"15min":1.2310000000000001},"min":{"1min":0.28499999999999998,"5min":0.25,"15min":0.25},"max":{"1min":8.6229999999999993,"5min":8.6760000000000002,"15min":12.512},"last":0.44800000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:30:46 2023","interfaces":[{"interface":"back","average":{"1min":2.7509999999999999,"5min":2.7509999999999999,"15min":2.7509999999999999},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":14.1,"5min":14.1,"15min":14.1},"last":1.1819999999999999},{"interface":"front","average":{"1min":3.4870000000000001,"5min":3.4870000000000001,"15min":3.4870000000000001},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":14.266999999999999,"5min":14.266999999999999,"15min":14.266999999999999},"last":0.501}]},{"osd":2,"last update":"Sat Sep 23 20:30:34 2023","interfaces":[{"interface":"back","average":{"1min":2.7519999999999998,"5min":2.2440000000000002,"15min":1.304},"min":{"1min":0.35099999999999998,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":7.8700000000000001,"5min":10.631,"15min":13.856},"last":0.33600000000000002},{"interface":"front","average":{"1min":2.5110000000000001,"5min":2.056,"15min":1.2190000000000001},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":7.891,"5min":8.3729999999999993,"15min":8.3729999999999993},"last":0.438}]},{"osd":3,"last update":"Sat Sep 23 20:30:08 2023","interfaces":[{"interface":"back","average":{"1min":1.9390000000000001,"5min":2.0590000000000002,"15min":1.2310000000000001},"min":{"1min":0.34799999999999998,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":6.8979999999999997,"5min":9.4239999999999995,"15min":9.4239999999999995},"last":2.3490000000000002},{"interface":"front","average":{"1min":2.0550000000000002,"5min":2.1179999999999999,"15min":1.272},"min":{"1min":0.437,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":6.6539999999999999,"5min":8.3770000000000007,"15min":8.6050000000000004},"last":4.7990000000000004}]},{"osd":4,"last update":"Sat Sep 23 20:30:53 2023","interfaces":[{"interface":"back","average":{"1min":0.44,"5min":1.0349999999999999,"15min":0.85299999999999998},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":1.599,"5min":10.173,"15min":10.173},"last":0.36599999999999999},{"interface":"front","average":{"1min":1.032,"5min":1.3779999999999999,"15min":0.98199999999999998},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":3.363,"5min":8.3249999999999993,"15min":8.3249999999999993},"last":0.35099999999999998}]},{"osd":5,"last update":"Sat Sep 23 20:30:53 2023","interfaces":[{"interface":"back","average":{"1min":0.46400000000000002,"5min":0.91100000000000003,"15min":0.89800000000000002},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":1.2010000000000001,"5min":8.4499999999999993,"15min":14.085000000000001},"last":0.40000000000000002},{"interface":"front","average":{"1min":0.98099999999999998,"5min":1.76,"15min":1.236},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.29499999999999998},"max":{"1min":3.294,"5min":18.192,"15min":18.192},"last":0.38900000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:30:08 2023","interfaces":[{"interface":"back","average":{"1min":2.0449999999999999,"5min":1.7829999999999999,"15min":1.032},"min":{"1min":0.35399999999999998,"5min":0.316,"15min":0.316},"max":{"1min":8.5060000000000002,"5min":10.217000000000001,"15min":10.217000000000001},"last":0.41999999999999998},{"interface":"front","average":{"1min":2.2559999999999998,"5min":1.9490000000000001,"15min":1.125},"min":{"1min":0.38400000000000001,"5min":0.36499999999999999,"15min":0.33500000000000002},"max":{"1min":7.1289999999999996,"5min":9.7360000000000007,"15min":9.7360000000000007},"last":0.52900000000000003}]}]},{"osd":1,"up_from":266,"seq":1142461300753,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1051492,"kb_used_data":935580,"kb_used_omap":8,"kb_used_meta":115895,"kb_avail":92693660,"statfs":{"total":95995035648,"available":94918307840,"internally_reserved":0,"allocated":958033920,"data_stored":956471393,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8206,"internal_metadata":118677490},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:30:44 2023","interfaces":[{"interface":"back","average":{"1min":2.3300000000000001,"5min":2.3300000000000001,"15min":2.3300000000000001},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":9.5470000000000006,"5min":9.5470000000000006,"15min":9.5470000000000006},"last":10.789},{"interface":"front","average":{"1min":3.1960000000000002,"5min":3.1960000000000002,"15min":3.1960000000000002},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":9.7129999999999992,"5min":9.7129999999999992,"15min":9.7129999999999992},"last":7.6310000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:30:44 2023","interfaces":[{"interface":"back","average":{"1min":5.125,"5min":5.125,"15min":5.125},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":16.151,"5min":16.151,"15min":16.151},"last":0.371},{"interface":"front","average":{"1min":2.0939999999999999,"5min":2.0939999999999999,"15min":2.0939999999999999},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":9.4649999999999999,"5min":9.4649999999999999,"15min":9.4649999999999999},"last":10.712999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:30:44 2023","interfaces":[{"interface":"back","average":{"1min":3.867,"5min":3.867,"15min":3.867},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":9.5739999999999998,"5min":9.5739999999999998,"15min":9.5739999999999998},"last":10.811},{"interface":"front","average":{"1min":4.8460000000000001,"5min":4.8460000000000001,"15min":4.8460000000000001},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":12.012,"5min":12.012,"15min":12.012},"last":7.665}]},{"osd":4,"last update":"Sat Sep 23 20:30:44 2023","interfaces":[{"interface":"back","average":{"1min":2.0790000000000002,"5min":2.0790000000000002,"15min":2.0790000000000002},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":9.4429999999999996,"5min":9.4429999999999996,"15min":9.4429999999999996},"last":0.42899999999999999},{"interface":"front","average":{"1min":2.024,"5min":2.024,"15min":2.024},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":9.5039999999999996,"5min":9.5039999999999996,"15min":9.5039999999999996},"last":10.760999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:30:44 2023","interfaces":[{"interface":"back","average":{"1min":2.802,"5min":2.802,"15min":2.802},"min":{"1min":0.44,"5min":0.44,"15min":0.44},"max":{"1min":9.6479999999999997,"5min":9.6479999999999997,"15min":9.6479999999999997},"last":4.6379999999999999},{"interface":"front","average":{"1min":2.0899999999999999,"5min":2.0899999999999999,"15min":2.0899999999999999},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":9.4700000000000006,"5min":9.4700000000000006,"15min":9.4700000000000006},"last":2.29}]},{"osd":6,"last update":"Sat Sep 23 20:30:44 2023","interfaces":[{"interface":"back","average":{"1min":7.4489999999999998,"5min":7.4489999999999998,"15min":7.4489999999999998},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":72.682000000000002,"5min":72.682000000000002,"15min":72.682000000000002},"last":7.5679999999999996},{"interface":"front","average":{"1min":6.4400000000000004,"5min":6.4400000000000004,"15min":6.4400000000000004},"min":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":72.563999999999993,"5min":72.563999999999993,"15min":72.563999999999993},"last":0.46999999999999997}]},{"osd":7,"last update":"Sat Sep 23 20:30:44 2023","interfaces":[{"interface":"back","average":{"1min":3.2829999999999999,"5min":3.2829999999999999,"15min":3.2829999999999999},"min":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"max":{"1min":9.6850000000000005,"5min":9.6850000000000005,"15min":9.6850000000000005},"last":4.577},{"interface":"front","average":{"1min":2.3599999999999999,"5min":2.3599999999999999,"15min":2.3599999999999999},"min":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"max":{"1min":9.3949999999999996,"5min":9.3949999999999996,"15min":9.3949999999999996},"last":2.1619999999999999}]}]},{"osd":0,"up_from":94,"seq":403726925955,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":557848,"kb_used_data":224656,"kb_used_omap":7,"kb_used_meta":333176,"kb_avail":93187304,"statfs":{"total":95995035648,"available":95423799296,"internally_reserved":0,"allocated":230047744,"data_stored":228606976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":341172241},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:30:48 2023","interfaces":[{"interface":"back","average":{"1min":3.887,"5min":3.887,"15min":3.887},"min":{"1min":0.17999999999999999,"5min":0.17999999999999999,"15min":0.17999999999999999},"max":{"1min":15.542,"5min":15.542,"15min":15.542},"last":2.9660000000000002},{"interface":"front","average":{"1min":4.4569999999999999,"5min":4.4569999999999999,"15min":4.4569999999999999},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":10.49,"5min":10.49,"15min":10.49},"last":2.9409999999999998}]},{"osd":2,"last update":"Sat Sep 23 20:30:27 2023","interfaces":[{"interface":"back","average":{"1min":5.8789999999999996,"5min":3.3180000000000001,"15min":1.6619999999999999},"min":{"1min":0.185,"5min":0.185,"15min":0.185},"max":{"1min":77.448999999999998,"5min":77.448999999999998,"15min":77.448999999999998},"last":5.9850000000000003},{"interface":"front","average":{"1min":6.0869999999999997,"5min":2.9449999999999998,"15min":1.5069999999999999},"min":{"1min":0.24299999999999999,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":81.724999999999994,"5min":81.724999999999994,"15min":81.724999999999994},"last":0.41899999999999998}]},{"osd":3,"last update":"Sat Sep 23 20:30:01 2023","interfaces":[{"interface":"back","average":{"1min":2.0430000000000001,"5min":2.5640000000000001,"15min":1.3580000000000001},"min":{"1min":0.245,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":8.5530000000000008,"5min":23.748999999999999,"15min":23.748999999999999},"last":5.8789999999999996},{"interface":"front","average":{"1min":1.8819999999999999,"5min":2.4609999999999999,"15min":1.169},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":8.1790000000000003,"5min":24.274999999999999,"15min":24.274999999999999},"last":10.276999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:30:27 2023","interfaces":[{"interface":"back","average":{"1min":2.093,"5min":2.4990000000000001,"15min":1.409},"min":{"1min":0.315,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":8.3209999999999997,"5min":23.876999999999999,"15min":23.876999999999999},"last":5.9160000000000004},{"interface":"front","average":{"1min":2.0009999999999999,"5min":1.9690000000000001,"15min":1.2390000000000001},"min":{"1min":0.34499999999999997,"5min":0.312,"15min":0.29299999999999998},"max":{"1min":10.414,"5min":23.681999999999999,"15min":23.681999999999999},"last":2.867}]},{"osd":5,"last update":"Sat Sep 23 20:30:27 2023","interfaces":[{"interface":"back","average":{"1min":2.645,"5min":2.657,"15min":1.6040000000000001},"min":{"1min":0.377,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":10.368,"5min":24.198,"15min":24.198},"last":2.9060000000000001},{"interface":"front","average":{"1min":2.2130000000000001,"5min":2.1869999999999998,"15min":1.45},"min":{"1min":0.38100000000000001,"5min":0.314,"15min":0.314},"max":{"1min":8.7780000000000005,"5min":14.938000000000001,"15min":14.938000000000001},"last":0.57699999999999996}]},{"osd":6,"last update":"Sat Sep 23 20:30:27 2023","interfaces":[{"interface":"back","average":{"1min":2.2450000000000001,"5min":2.3420000000000001,"15min":1.319},"min":{"1min":0.40100000000000002,"5min":0.318,"15min":0.318},"max":{"1min":10.441000000000001,"5min":24.408000000000001,"15min":24.408000000000001},"last":4.5620000000000003},{"interface":"front","average":{"1min":1.764,"5min":2.1709999999999998,"15min":1.2629999999999999},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":6.9480000000000004,"5min":25.271000000000001,"15min":25.271000000000001},"last":6.109}]},{"osd":7,"last update":"Sat Sep 23 20:30:27 2023","interfaces":[{"interface":"back","average":{"1min":2.024,"5min":2.3839999999999999,"15min":1.4179999999999999},"min":{"1min":0.42199999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":7.4470000000000001,"5min":25.359000000000002,"15min":25.359000000000002},"last":0.60599999999999998},{"interface":"front","average":{"1min":2.5550000000000002,"5min":3.1200000000000001,"15min":1.6799999999999999},"min":{"1min":0.46500000000000002,"5min":0.378,"15min":0.378},"max":{"1min":8.2729999999999997,"5min":24.669,"15min":24.669},"last":5.9550000000000001}]}]},{"osd":2,"up_from":104,"seq":446676598903,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1205060,"kb_used_data":822332,"kb_used_omap":11,"kb_used_meta":382580,"kb_avail":92540092,"statfs":{"total":95995035648,"available":94761054208,"internally_reserved":0,"allocated":842067968,"data_stored":840654823,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11878,"internal_metadata":391762330},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:30:37 2023","interfaces":[{"interface":"back","average":{"1min":3.3199999999999998,"5min":2.2509999999999999,"15min":1.282},"min":{"1min":0.193,"5min":0.17999999999999999,"15min":0.17999999999999999},"max":{"1min":30.765999999999998,"5min":30.765999999999998,"15min":30.765999999999998},"last":0.434},{"interface":"front","average":{"1min":5.1280000000000001,"5min":2.3199999999999998,"15min":1.2150000000000001},"min":{"1min":0.252,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":16.824000000000002,"5min":16.824000000000002,"15min":16.824000000000002},"last":2.0790000000000002}]},{"osd":1,"last update":"Sat Sep 23 20:30:45 2023","interfaces":[{"interface":"back","average":{"1min":4.851,"5min":4.851,"15min":4.851},"min":{"1min":0.221,"5min":0.221,"15min":0.221},"max":{"1min":16.280000000000001,"5min":16.280000000000001,"15min":16.280000000000001},"last":3.6680000000000001},{"interface":"front","average":{"1min":2.988,"5min":2.988,"15min":2.988},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":30.565000000000001,"5min":30.565000000000001,"15min":30.565000000000001},"last":0.48099999999999998}]},{"osd":3,"last update":"Sat Sep 23 20:30:01 2023","interfaces":[{"interface":"back","average":{"1min":3.0019999999999998,"5min":1.865,"15min":1.042},"min":{"1min":0.19800000000000001,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":16.331,"5min":16.331,"15min":16.331},"last":3.6240000000000001},{"interface":"front","average":{"1min":1.5920000000000001,"5min":2.3969999999999998,"15min":1.2030000000000001},"min":{"1min":0.249,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":7.7709999999999999,"5min":21.594000000000001,"15min":21.594000000000001},"last":1.786}]},{"osd":4,"last update":"Sat Sep 23 20:30:37 2023","interfaces":[{"interface":"back","average":{"1min":3.98,"5min":2.1669999999999998,"15min":1.234},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.28100000000000003},"max":{"1min":16.797999999999998,"5min":16.797999999999998,"15min":16.797999999999998},"last":2.1509999999999998},{"interface":"front","average":{"1min":2.7309999999999999,"5min":2.1589999999999998,"15min":1.2609999999999999},"min":{"1min":0.309,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":10.518000000000001,"5min":14.971,"15min":14.971},"last":0.41199999999999998}]},{"osd":5,"last update":"Sat Sep 23 20:30:37 2023","interfaces":[{"interface":"back","average":{"1min":4.5359999999999996,"5min":3.512,"15min":1.77},"min":{"1min":0.35299999999999998,"5min":0.309,"15min":0.309},"max":{"1min":16.766999999999999,"5min":95.393000000000001,"15min":95.393000000000001},"last":2.1179999999999999},{"interface":"front","average":{"1min":3.4790000000000001,"5min":3.6190000000000002,"15min":1.8},"min":{"1min":0.34799999999999998,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":10.474,"5min":95.414000000000001,"15min":95.414000000000001},"last":0.38}]},{"osd":6,"last update":"Sat Sep 23 20:30:37 2023","interfaces":[{"interface":"back","average":{"1min":3.258,"5min":2.29,"15min":1.323},"min":{"1min":0.33600000000000002,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":10.544,"5min":15.023999999999999,"15min":15.023999999999999},"last":0.51600000000000001},{"interface":"front","average":{"1min":2.7930000000000001,"5min":2.3719999999999999,"15min":1.405},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":30.670999999999999,"5min":30.670999999999999,"15min":30.670999999999999},"last":0.64200000000000002}]},{"osd":7,"last update":"Sat Sep 23 20:30:37 2023","interfaces":[{"interface":"back","average":{"1min":3.4609999999999999,"5min":2.5529999999999999,"15min":1.4730000000000001},"min":{"1min":0.41299999999999998,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":30.741,"5min":30.741,"15min":30.741},"last":0.68600000000000005},{"interface":"front","average":{"1min":4.8650000000000002,"5min":2.52,"15min":1.4550000000000001},"min":{"1min":0.40899999999999997,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":16.719000000000001,"5min":16.719000000000001,"15min":16.719000000000001},"last":2.1800000000000002}]}]},{"osd":3,"up_from":109,"seq":468151435377,"num_pgs":64,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":867796,"kb_used_data":469068,"kb_used_omap":4,"kb_used_meta":398715,"kb_avail":92877356,"statfs":{"total":95995035648,"available":95106412544,"internally_reserved":0,"allocated":480325632,"data_stored":478893076,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4514,"internal_metadata":408284766},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:30:05 2023","interfaces":[{"interface":"back","average":{"1min":1.8340000000000001,"5min":2.657,"15min":1.3129999999999999},"min":{"1min":0.19600000000000001,"5min":0.186,"15min":0.186},"max":{"1min":6.9470000000000001,"5min":67.305999999999997,"15min":67.305999999999997},"last":0.66500000000000004},{"interface":"front","average":{"1min":1.879,"5min":2.3250000000000002,"15min":1.2290000000000001},"min":{"1min":0.26000000000000001,"5min":0.224,"15min":0.224},"max":{"1min":5.3220000000000001,"5min":67.403999999999996,"15min":67.403999999999996},"last":1.3799999999999999}]},{"osd":1,"last update":"Sat Sep 23 20:30:47 2023","interfaces":[{"interface":"back","average":{"1min":3.0880000000000001,"5min":3.0880000000000001,"15min":3.0880000000000001},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":8.9199999999999999,"5min":8.9199999999999999,"15min":8.9199999999999999},"last":1.129},{"interface":"front","average":{"1min":3.2530000000000001,"5min":3.2530000000000001,"15min":3.2530000000000001},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":10.967000000000001,"5min":10.967000000000001,"15min":10.967000000000001},"last":1.4339999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:30:05 2023","interfaces":[{"interface":"back","average":{"1min":2.7549999999999999,"5min":2.1739999999999999,"15min":1.21},"min":{"1min":0.29699999999999999,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":11.499000000000001,"5min":24.045000000000002,"15min":24.045000000000002},"last":1.409},{"interface":"front","average":{"1min":3.1539999999999999,"5min":2.298,"15min":1.2629999999999999},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":16.530000000000001,"5min":16.530000000000001,"15min":17.684000000000001},"last":3.3050000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:30:05 2023","interfaces":[{"interface":"back","average":{"1min":2.2789999999999999,"5min":2.2970000000000002,"15min":1.2410000000000001},"min":{"1min":0.34399999999999997,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":6.6440000000000001,"5min":46.042999999999999,"15min":46.042999999999999},"last":0.46300000000000002},{"interface":"front","average":{"1min":2.3420000000000001,"5min":2.7890000000000001,"15min":1.385},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":13.462999999999999,"5min":46.579999999999998,"15min":46.579999999999998},"last":1.2010000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:30:05 2023","interfaces":[{"interface":"back","average":{"1min":2.323,"5min":2.6160000000000001,"15min":1.405},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.311},"max":{"1min":7.0469999999999997,"5min":45.292000000000002,"15min":45.292000000000002},"last":0.48899999999999999},{"interface":"front","average":{"1min":2.0790000000000002,"5min":2.302,"15min":1.268},"min":{"1min":0.39800000000000002,"5min":0.313,"15min":0.313},"max":{"1min":5.5140000000000002,"5min":44.679000000000002,"15min":44.679000000000002},"last":1.3089999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:30:05 2023","interfaces":[{"interface":"back","average":{"1min":2.2240000000000002,"5min":2.2309999999999999,"15min":1.2470000000000001},"min":{"1min":0.41299999999999998,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":4.202,"5min":13.475,"15min":13.475},"last":1.3440000000000001},{"interface":"front","average":{"1min":2.613,"5min":2.1920000000000002,"15min":1.2270000000000001},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.35499999999999998},"max":{"1min":13.506,"5min":13.506,"15min":13.506},"last":1.2669999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:30:05 2023","interfaces":[{"interface":"back","average":{"1min":3.3730000000000002,"5min":2.3820000000000001,"15min":1.3240000000000001},"min":{"1min":0.46899999999999997,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":13.532999999999999,"5min":13.532999999999999,"15min":13.532999999999999},"last":1.236},{"interface":"front","average":{"1min":2.3460000000000001,"5min":2.7919999999999998,"15min":1.4950000000000001},"min":{"1min":0.44500000000000001,"5min":0.371,"15min":0.371},"max":{"1min":7.0899999999999999,"5min":45.387999999999998,"15min":45.387999999999998},"last":0.42999999999999999}]}]},{"osd":4,"up_from":0,"seq":128849019130,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":5,"up_from":39,"seq":167503724791,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":502480,"kb_used_data":288072,"kb_used_omap":0,"kb_used_meta":214400,"kb_avail":93242672,"statfs":{"total":95995035648,"available":95480496128,"internally_reserved":0,"allocated":294985728,"data_stored":294071367,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":219545600},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:30:31 2023","interfaces":[{"interface":"back","average":{"1min":2.8109999999999999,"5min":1.7529999999999999,"15min":1.181},"min":{"1min":0.28499999999999998,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":8.1609999999999996,"5min":11.346,"15min":32.475000000000001},"last":0.32600000000000001},{"interface":"front","average":{"1min":2.5950000000000002,"5min":1.681,"15min":1.2030000000000001},"min":{"1min":0.29199999999999998,"5min":0.245,"15min":0.20599999999999999},"max":{"1min":8.1859999999999999,"5min":8.4900000000000002,"15min":32.457000000000001},"last":0.41499999999999998}]},{"osd":1,"last update":"Sat Sep 23 20:30:45 2023","interfaces":[{"interface":"back","average":{"1min":2.0550000000000002,"5min":2.0550000000000002,"15min":2.0550000000000002},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":8.5839999999999996,"5min":8.5839999999999996,"15min":8.5839999999999996},"last":0.437},{"interface":"front","average":{"1min":1.5609999999999999,"5min":1.5609999999999999,"15min":1.5609999999999999},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":6.7969999999999997,"5min":6.7969999999999997,"15min":6.7969999999999997},"last":2.323}]},{"osd":2,"last update":"Sat Sep 23 20:30:31 2023","interfaces":[{"interface":"back","average":{"1min":3.5259999999999998,"5min":1.823,"15min":1.325},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.26700000000000002},"max":{"1min":18.472999999999999,"5min":18.472999999999999,"15min":34.514000000000003},"last":0.35899999999999999},{"interface":"front","average":{"1min":1.8919999999999999,"5min":1.7869999999999999,"15min":1.256},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":8.016,"5min":11.574999999999999,"15min":32.515000000000001},"last":0.89800000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:30:04 2023","interfaces":[{"interface":"back","average":{"1min":2.4670000000000001,"5min":2.0059999999999998,"15min":1.151},"min":{"1min":0.318,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":12.51,"5min":12.846,"15min":12.846},"last":3.5699999999999998},{"interface":"front","average":{"1min":1.6339999999999999,"5min":1.851,"15min":1.1160000000000001},"min":{"1min":0.35099999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":5.2670000000000003,"5min":12.005000000000001,"15min":12.005000000000001},"last":0.51600000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:30:25 2023","interfaces":[{"interface":"back","average":{"1min":3.1560000000000001,"5min":2.359,"15min":1.2230000000000001},"min":{"1min":0.32500000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":28.015999999999998,"5min":28.015999999999998,"15min":28.015999999999998},"last":0.61399999999999999},{"interface":"front","average":{"1min":2.0339999999999998,"5min":1.6459999999999999,"15min":1.0149999999999999},"min":{"1min":0.33800000000000002,"5min":0.32600000000000001,"15min":0.29099999999999998},"max":{"1min":12.300000000000001,"5min":12.300000000000001,"15min":12.300000000000001},"last":0.40300000000000002}]},{"osd":6,"last update":"Sat Sep 23 20:30:39 2023","interfaces":[{"interface":"back","average":{"1min":1.1220000000000001,"5min":1.796,"15min":1.1499999999999999},"min":{"1min":0.309,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":4.9829999999999997,"5min":12.890000000000001,"15min":12.890000000000001},"last":0.34799999999999998},{"interface":"front","average":{"1min":0.92500000000000004,"5min":1.5389999999999999,"15min":1.137},"min":{"1min":0.371,"5min":0.36399999999999999,"15min":0.34499999999999997},"max":{"1min":2.4710000000000001,"5min":16.265999999999998,"15min":16.265999999999998},"last":0.54700000000000004}]},{"osd":7,"last update":"Sat Sep 23 20:30:04 2023","interfaces":[{"interface":"back","average":{"1min":2.0710000000000002,"5min":2.1989999999999998,"15min":1.1719999999999999},"min":{"1min":0.36199999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":7.343,"5min":36.621000000000002,"15min":36.621000000000002},"last":0.48299999999999998},{"interface":"front","average":{"1min":2.2170000000000001,"5min":1.8169999999999999,"15min":1.075},"min":{"1min":0.35199999999999998,"5min":0.316,"15min":0.315},"max":{"1min":13.122,"5min":37.331000000000003,"15min":37.331000000000003},"last":0.46000000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4722,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1360,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8206,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7156,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3154,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":224661504,"data_stored":224657432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":948699136,"data_stored":948699136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":818741248,"data_stored":818741248,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":471269376,"data_stored":471269376,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":686817280,"data_stored":686817280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":284692480,"data_stored":284688408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":283181056,"data_stored":283181056,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":683016192,"data_stored":683016192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:31:06.177 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-23T20:31:06.227 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:06.247 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T20:31:06.332 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:31:06.338 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:06.338 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:31:06.439 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:31:06.469 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:06 smithi149 ceph-mon[170394]: from='client.55282 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:31:06.469 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:06 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 29254/38271 objects degraded (76.439%), 13 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2023-09-23T20:31:06.555 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:06 smithi018 ceph-mon[184144]: from='client.55282 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:31:06.555 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:06 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 29254/38271 objects degraded (76.439%), 13 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2023-09-23T20:31:06.558 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:06 smithi018 ceph-mon[179647]: from='client.55282 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:31:06.558 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:06 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 29254/38271 objects degraded (76.439%), 13 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2023-09-23T20:31:06.560 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:06.574 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:31:06.665 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:31:06.674 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:06.674 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:31:06.775 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:31:06.830 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:06.885 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:31:06.888 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:06.889 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:31:06.956 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:31:06.990 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:31:07.022 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:07.082 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:31:07.089 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:07.090 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:31:07.192 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:31:07.252 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:07.311 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:31:07.314 INFO:teuthology.orchestra.run.smithi018.stderr:nodeep-scrub is unset 2023-09-23T20:31:07.319 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:07.320 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:31:07.421 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:31:07.447 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:07 smithi149 ceph-mon[170394]: pgmap v656: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 725 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 633 op/s; 29254/34659 objects degraded (84.405%); 1505/34659 objects misplaced (4.342%); 3.1 MiB/s, 152 objects/s recovering 2023-09-23T20:31:07.447 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:07 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1902129965' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:31:07.458 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:07.512 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:31:07.518 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:07.518 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:31:07.584 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:07 smithi018 ceph-mon[179647]: pgmap v656: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 725 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 633 op/s; 29254/34659 objects degraded (84.405%); 1505/34659 objects misplaced (4.342%); 3.1 MiB/s, 152 objects/s recovering 2023-09-23T20:31:07.585 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:07 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1902129965' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:31:07.585 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:07 smithi018 ceph-mon[184144]: pgmap v656: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 725 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 633 op/s; 29254/34659 objects degraded (84.405%); 1505/34659 objects misplaced (4.342%); 3.1 MiB/s, 152 objects/s recovering 2023-09-23T20:31:07.585 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:07 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1902129965' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:31:07.619 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:31:07.684 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:07.925 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:31:07.930 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:07.931 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:31:07.943 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:31:08.031 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:31:08.072 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:08.127 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:31:08.134 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:08.135 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:31:08.236 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:31:08.276 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:08.333 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:31:08.340 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:08.340 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:31:08.386 INFO:teuthology.orchestra.run.smithi149.stdout:ERROR: (22) Invalid argument 2023-09-23T20:31:08.386 INFO:teuthology.orchestra.run.smithi149.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:31:08.394 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:08 smithi018 ceph-mon[179647]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:31:08.394 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:08 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1902129965' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:31:08.394 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:08 smithi018 ceph-mon[179647]: osdmap e291: 8 total, 8 up, 7 in 2023-09-23T20:31:08.394 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:08 smithi018 ceph-mon[179647]: failed to encode map e291 with expected crc 2023-09-23T20:31:08.394 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:08 smithi018 ceph-mon[179647]: failed to encode map e291 with expected crc 2023-09-23T20:31:08.394 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:08 smithi018 ceph-mon[184144]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:31:08.394 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:08 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1902129965' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:31:08.394 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:08 smithi018 ceph-mon[184144]: osdmap e291: 8 total, 8 up, 7 in 2023-09-23T20:31:08.395 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:08 smithi018 ceph-mon[184144]: failed to encode map e291 with expected crc 2023-09-23T20:31:08.395 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:08 smithi018 ceph-mon[184144]: failed to encode map e291 with expected crc 2023-09-23T20:31:08.417 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:08 smithi149 ceph-mon[170394]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:31:08.417 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:08 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1902129965' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:31:08.417 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:08 smithi149 ceph-mon[170394]: osdmap e291: 8 total, 8 up, 7 in 2023-09-23T20:31:08.417 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:08 smithi149 ceph-mon[170394]: failed to encode map e291 with expected crc 2023-09-23T20:31:08.417 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:08 smithi149 ceph-mon[170394]: failed to encode map e291 with expected crc 2023-09-23T20:31:08.441 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:31:08.446 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:31:08.491 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:31:08.504 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:08.525 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:31:08.562 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:31:08.602 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:31:08.658 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:31:08.706 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:31:08.753 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:31:08.763 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:31:08.764 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:31:08.764 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:31:08.764 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:31:08.764 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:31:08.764 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:31:08.764 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:31:08.764 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:31:08.781 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:31:08.787 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:08.787 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:31:08.873 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T20:31:08.888 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:31:08.961 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:09.050 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:31:09.068 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:09.069 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:31:09.170 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:31:09.271 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:09.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:09 smithi149 ceph-mon[170394]: pgmap v658: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 657 MiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 646 op/s; 29254/31404 objects degraded (93.154%); 1277/31404 objects misplaced (4.066%); 1.0 MiB/s, 152 objects/s recovering 2023-09-23T20:31:09.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:09 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2381663127' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:31:09.612 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:31:09.644 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:09 smithi018 ceph-mon[184144]: pgmap v658: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 657 MiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 646 op/s; 29254/31404 objects degraded (93.154%); 1277/31404 objects misplaced (4.066%); 1.0 MiB/s, 152 objects/s recovering 2023-09-23T20:31:09.644 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:09 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2381663127' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:31:09.647 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:09 smithi018 ceph-mon[179647]: pgmap v658: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 657 MiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 646 op/s; 29254/31404 objects degraded (93.154%); 1277/31404 objects misplaced (4.066%); 1.0 MiB/s, 152 objects/s recovering 2023-09-23T20:31:09.647 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:09 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2381663127' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:31:09.973 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T20:31:10.704 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:31:10.721 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:10.722 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:31:10.823 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:31:10.879 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:10.977 INFO:teuthology.orchestra.run.smithi149.stdout:{ 2023-09-23T20:31:10.977 INFO:teuthology.orchestra.run.smithi149.stdout: "ops": [ 2023-09-23T20:31:10.977 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:10.978 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "MOSDPGRecoveryDeleteReply(6.11 e291,267 6:8e57c059:::benchmark_data_smithi018_270558_object38168:head,263'7154)", 2023-09-23T20:31:10.978 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:31:10.962324+0000", 2023-09-23T20:31:10.978 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 0.0072432260000000002, 2023-09-23T20:31:10.978 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0072774629999999996, 2023-09-23T20:31:10.978 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:31:10.978 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "queued for pg", 2023-09-23T20:31:10.978 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:31:10.978 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:10.978 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:31:10.978 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:31:10.962324+0000", 2023-09-23T20:31:10.979 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:31:10.979 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:10.979 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:10.979 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:31:10.979 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:31:10.962324+0000", 2023-09-23T20:31:10.979 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:31:10.979 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:10.979 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:10.979 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:31:10.979 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:31:10.962324+0000", 2023-09-23T20:31:10.980 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:31:10.980 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:10.980 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:10.980 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:31:10.980 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:31:10.962328+0000", 2023-09-23T20:31:10.980 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.8809999999999998e-06 2023-09-23T20:31:10.980 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:10.980 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:10.980 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:31:10.980 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:31:10.962335+0000", 2023-09-23T20:31:10.980 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.7700000000000004e-06 2023-09-23T20:31:10.981 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:10.981 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:10.981 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:31:10.981 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:31:10.962339+0000", 2023-09-23T20:31:10.981 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.3970000000000004e-06 2023-09-23T20:31:10.981 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:31:10.981 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:31:10.981 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:31:10.981 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:10.981 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:10.981 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.45161.0:203043 6.11 6.aea0a111 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e291)", 2023-09-23T20:31:10.982 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:31:10.967754+0000", 2023-09-23T20:31:10.982 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 0.0018137839999999999, 2023-09-23T20:31:10.982 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0019062490000000001, 2023-09-23T20:31:10.982 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:31:10.982 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "queued for pg", 2023-09-23T20:31:10.982 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:31:10.982 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.45161", 2023-09-23T20:31:10.982 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3795614987", 2023-09-23T20:31:10.982 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 203043 2023-09-23T20:31:10.982 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:10.982 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:31:10.983 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:10.983 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:31:10.983 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:31:10.967754+0000", 2023-09-23T20:31:10.983 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:31:10.983 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:10.983 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:10.983 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:31:10.983 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:31:10.967754+0000", 2023-09-23T20:31:10.983 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:31:10.983 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:10.983 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:10.983 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:31:10.984 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:31:10.967752+0000", 2023-09-23T20:31:10.984 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999981 2023-09-23T20:31:10.984 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:10.984 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:10.984 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:31:10.984 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:31:10.967762+0000", 2023-09-23T20:31:10.984 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0301e-05 2023-09-23T20:31:10.984 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:10.984 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:10.984 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:31:10.984 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:31:10.967784+0000", 2023-09-23T20:31:10.985 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.1651e-05 2023-09-23T20:31:10.985 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:10.985 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:10.985 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:31:10.985 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:31:10.967793+0000", 2023-09-23T20:31:10.985 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.0089999999999996e-06 2023-09-23T20:31:10.985 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:31:10.985 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:31:10.985 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:31:10.985 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:31:10.985 INFO:teuthology.orchestra.run.smithi149.stdout: ], 2023-09-23T20:31:10.985 INFO:teuthology.orchestra.run.smithi149.stdout: "num_ops": 2 2023-09-23T20:31:10.986 INFO:teuthology.orchestra.run.smithi149.stdout:} 2023-09-23T20:31:11.345 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T20:31:11.569 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:31:11.582 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:11.583 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:31:11.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:11 smithi018 ceph-mon[179647]: pgmap v659: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 593 MiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 615 op/s; 29254/28308 objects degraded (103.342%); 1089/28308 objects misplaced (3.847%); 879 KiB/s, 128 objects/s recovering 2023-09-23T20:31:11.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:11 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 29254/28308 objects degraded (103.342%), 13 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2023-09-23T20:31:11.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:11 smithi018 ceph-mon[184144]: pgmap v659: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 593 MiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 615 op/s; 29254/28308 objects degraded (103.342%); 1089/28308 objects misplaced (3.847%); 879 KiB/s, 128 objects/s recovering 2023-09-23T20:31:11.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:11 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 29254/28308 objects degraded (103.342%), 13 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2023-09-23T20:31:11.684 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:31:11.719 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:11 smithi149 ceph-mon[170394]: pgmap v659: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 593 MiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 615 op/s; 29254/28308 objects degraded (103.342%); 1089/28308 objects misplaced (3.847%); 879 KiB/s, 128 objects/s recovering 2023-09-23T20:31:11.719 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:11 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 29254/28308 objects degraded (103.342%), 13 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2023-09-23T20:31:11.728 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:11.782 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:31:11.789 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:11.789 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:31:11.890 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:31:11.967 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:12.092 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:31:12.292 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:31:12.306 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:12.306 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:31:12.407 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:31:12.461 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:12.593 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:31:12.597 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:12.597 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:31:12.602 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:31:12.698 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:31:12.753 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:31:12.775 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:12.906 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:31:12.917 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:12.917 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:31:13.018 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:31:13.056 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:13.153 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:31:13.161 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:13.162 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:31:13.262 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:31:13.269 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:31:13.269 INFO:teuthology.orchestra.run.smithi018.stdout:{"fsid":"a3b3b17e-5a4c-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 29254/22707 objects degraded (128.833%), 13 pgs degraded, 13 pgs undersized","count":26},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":749,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":291,"num_osds":8,"num_up_osds":8,"osd_up_since":1695500984,"num_in_osds":7,"osd_in_since":1695500967,"num_remapped_pgs":16},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":99},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":11},{"state_name":"active+remapped+backfill_wait","count":3},{"state_name":"active+recovery_wait","count":3},{"state_name":"active+recovering+undersized+degraded+remapped","count":1},{"state_name":"active+recovering+undersized+remapped","count":1},{"state_name":"active+recovery_wait+degraded","count":1}],"num_pgs":119,"num_pools":5,"num_objects":7569,"data_bytes":498950766,"bytes_used":4906688512,"bytes_avail":667058561024,"bytes_total":671965249536,"degraded_objects":29254,"degraded_total":22707,"degraded_ratio":1.288325186065971,"misplaced_objects":1089,"misplaced_total":22707,"misplaced_ratio":0.047958779231074118,"recovering_objects_per_sec":109,"recovering_bytes_per_sec":641709,"recovering_keys_per_sec":0,"num_objects_recovered":1100,"num_bytes_recovered":6422528,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":660},"fsmap":{"epoch":9,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi018.hduiol","status":"up:active","gid":24566}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.18:8443/","prometheus":"http://172.21.15.18:9283/"}},"servicemap":{"epoch":21,"modified":"2023-09-23T20:30:43.686281+0000","services":{}},"progress_events":{"8e7eff18-0e45-4130-a6e1-662b3902f0d2":{"message":"Upgrade to 18.2.0-460-gbb0daeff (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true},"a3000690-5684-42b5-9dec-3c8689315562":{"message":"Global Recovery Event (3m)\n [=======================.....] (remaining: 37s)","progress":0.83193278312683105,"add_to_ceph_s":true}}} 2023-09-23T20:31:13.381 INFO:teuthology.orchestra.run.smithi149.stdout:{ 2023-09-23T20:31:13.381 INFO:teuthology.orchestra.run.smithi149.stdout: "ops": [], 2023-09-23T20:31:13.381 INFO:teuthology.orchestra.run.smithi149.stdout: "complaint_time": 30, 2023-09-23T20:31:13.381 INFO:teuthology.orchestra.run.smithi149.stdout: "num_blocked_ops": 0 2023-09-23T20:31:13.381 INFO:teuthology.orchestra.run.smithi149.stdout:} 2023-09-23T20:31:13.393 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:13.405 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:13 smithi149 ceph-mon[170394]: pgmap v660: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 476 MiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 660 op/s; 29254/22707 objects degraded (128.833%); 1089/22707 objects misplaced (4.796%); 627 KiB/s, 109 objects/s recovering 2023-09-23T20:31:13.405 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:13 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/347881159' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T20:31:13.470 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:31:13.477 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:13.477 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:31:13.578 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:31:13.630 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:13.645 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:13 smithi018 ceph-mon[179647]: pgmap v660: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 476 MiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 660 op/s; 29254/22707 objects degraded (128.833%); 1089/22707 objects misplaced (4.796%); 627 KiB/s, 109 objects/s recovering 2023-09-23T20:31:13.645 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:13 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/347881159' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T20:31:13.647 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:13 smithi018 ceph-mon[184144]: pgmap v660: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 476 MiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 660 op/s; 29254/22707 objects degraded (128.833%); 1089/22707 objects misplaced (4.796%); 627 KiB/s, 109 objects/s recovering 2023-09-23T20:31:13.647 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:13 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/347881159' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T20:31:13.749 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:31:13.763 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:13.764 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:31:13.793 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T20:31:13.865 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:31:13.982 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:14.116 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:31:14.122 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:14.122 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:31:14.164 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-09-23T20:31:14.165 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:31:14.224 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:31:14.265 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:14.315 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:31:14.319 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:14.319 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:31:14.387 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:31:14.420 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:31:14.549 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:31:14.552 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:14.766 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:31:14.770 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:14.771 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:31:14.872 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:31:14.915 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:15.059 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:31:15.062 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:15.063 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:31:15.163 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:31:15.198 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:15.319 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:31:15.322 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:15.323 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:31:15.325 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:31:14 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:31:14] "GET /metrics HTTP/1.1" 200 36567 "" "Prometheus/2.43.0" 2023-09-23T20:31:15.392 INFO:teuthology.orchestra.run.smithi018.stderr:noscrub is set 2023-09-23T20:31:15.405 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:31:15.424 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:31:15.538 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:15.584 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:15 smithi018 ceph-mon[179647]: pgmap v661: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 378 MiB data, 4.3 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 675 op/s; 29254/17994 objects degraded (162.576%); 653/17994 objects misplaced (3.629%); 0 B/s, 112 objects/s recovering 2023-09-23T20:31:15.584 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:15 smithi018 ceph-mon[179647]: osdmap e292: 8 total, 8 up, 7 in 2023-09-23T20:31:15.585 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:15 smithi018 ceph-mon[179647]: failed to encode map e292 with expected crc 2023-09-23T20:31:15.585 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:15 smithi018 ceph-mon[179647]: failed to encode map e292 with expected crc 2023-09-23T20:31:15.585 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:15 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:31:15.585 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:15 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:31:15.585 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:15 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/805526381' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:31:15.585 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:15 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:31:15.585 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:15 smithi018 ceph-mon[184144]: pgmap v661: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 378 MiB data, 4.3 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 675 op/s; 29254/17994 objects degraded (162.576%); 653/17994 objects misplaced (3.629%); 0 B/s, 112 objects/s recovering 2023-09-23T20:31:15.585 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:15 smithi018 ceph-mon[184144]: osdmap e292: 8 total, 8 up, 7 in 2023-09-23T20:31:15.586 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:15 smithi018 ceph-mon[184144]: failed to encode map e292 with expected crc 2023-09-23T20:31:15.586 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:15 smithi018 ceph-mon[184144]: failed to encode map e292 with expected crc 2023-09-23T20:31:15.586 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:15 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:31:15.586 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:15 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:31:15.586 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:15 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/805526381' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:31:15.586 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:15 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:31:15.607 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:15 smithi149 ceph-mon[170394]: pgmap v661: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 378 MiB data, 4.3 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 675 op/s; 29254/17994 objects degraded (162.576%); 653/17994 objects misplaced (3.629%); 0 B/s, 112 objects/s recovering 2023-09-23T20:31:15.607 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:15 smithi149 ceph-mon[170394]: osdmap e292: 8 total, 8 up, 7 in 2023-09-23T20:31:15.607 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:15 smithi149 ceph-mon[170394]: failed to encode map e292 with expected crc 2023-09-23T20:31:15.607 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:15 smithi149 ceph-mon[170394]: failed to encode map e292 with expected crc 2023-09-23T20:31:15.607 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:15 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:31:15.608 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:15 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:31:15.608 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:15 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/805526381' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:31:15.608 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:15 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:31:15.880 INFO:teuthology.orchestra.run.smithi149.stdout:{ 2023-09-23T20:31:15.880 INFO:teuthology.orchestra.run.smithi149.stdout: "size": 20, 2023-09-23T20:31:15.880 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 600, 2023-09-23T20:31:15.881 INFO:teuthology.orchestra.run.smithi149.stdout: "ops": [ 2023-09-23T20:31:15.881 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.881 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:28907 5.12 5:4860c7d8:::benchmark_data_smithi018_228451_object28906:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e118)", 2023-09-23T20:31:15.881 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:09.975817+0000", 2023-09-23T20:31:15.881 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 485.90114861000001, 2023-09-23T20:31:15.881 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1014516590000001, 2023-09-23T20:31:15.881 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:31:15.881 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:31:15.881 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:31:15.881 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:31:15.882 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:31:15.882 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 28907 2023-09-23T20:31:15.882 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.882 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:31:15.882 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.882 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:31:15.882 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:09.975817+0000", 2023-09-23T20:31:15.882 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:31:15.882 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.882 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.882 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:31:15.883 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:09.975817+0000", 2023-09-23T20:31:15.883 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:31:15.883 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.883 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.883 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:31:15.883 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:09.975813+0000", 2023-09-23T20:31:15.883 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999952 2023-09-23T20:31:15.883 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.883 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.883 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:31:15.883 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:09.975911+0000", 2023-09-23T20:31:15.884 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.7844999999999996e-05 2023-09-23T20:31:15.884 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.884 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.884 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:31:15.884 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:09.975954+0000", 2023-09-23T20:31:15.884 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.3825999999999997e-05 2023-09-23T20:31:15.884 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.884 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.884 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:31:15.884 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:09.975970+0000", 2023-09-23T20:31:15.884 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.5252999999999999e-05 2023-09-23T20:31:15.884 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.885 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.885 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:31:15.885 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:10.068105+0000", 2023-09-23T20:31:15.885 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.092135898999999993 2023-09-23T20:31:15.885 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.885 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.885 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:31:15.885 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:10.068116+0000", 2023-09-23T20:31:15.885 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0644e-05 2023-09-23T20:31:15.885 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.885 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.886 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:31:15.886 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.073358+0000", 2023-09-23T20:31:15.886 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.00524149 2023-09-23T20:31:15.886 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.886 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.886 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:31:15.886 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.073492+0000", 2023-09-23T20:31:15.886 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00013435199999999999 2023-09-23T20:31:15.886 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.886 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.886 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:31:15.887 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.073626+0000", 2023-09-23T20:31:15.887 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000133654 2023-09-23T20:31:15.887 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.887 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.887 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:31:15.887 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.074639+0000", 2023-09-23T20:31:15.887 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.001013634 2023-09-23T20:31:15.887 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.887 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.887 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:31:15.887 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.074705+0000", 2023-09-23T20:31:15.887 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.6208e-05 2023-09-23T20:31:15.888 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.888 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.888 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:31:15.888 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.077176+0000", 2023-09-23T20:31:15.888 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0024700550000000001 2023-09-23T20:31:15.888 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.888 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.888 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:31:15.888 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.077228+0000", 2023-09-23T20:31:15.888 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.2320000000000001e-05 2023-09-23T20:31:15.888 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.889 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.889 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:31:15.889 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.077269+0000", 2023-09-23T20:31:15.889 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.1258999999999999e-05 2023-09-23T20:31:15.889 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:31:15.889 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:31:15.889 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:31:15.889 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.889 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.889 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:39722 5.11 5.6eed25d1 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e122)", 2023-09-23T20:31:15.889 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:18.137113+0000", 2023-09-23T20:31:15.890 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 477.73985345699998, 2023-09-23T20:31:15.890 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.91218893999999995, 2023-09-23T20:31:15.890 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:31:15.890 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:31:15.890 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:31:15.890 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:31:15.890 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:31:15.890 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 39722 2023-09-23T20:31:15.890 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.890 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:31:15.890 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.891 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:31:15.891 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.137113+0000", 2023-09-23T20:31:15.891 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:31:15.891 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.891 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.891 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:31:15.891 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.137113+0000", 2023-09-23T20:31:15.891 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:31:15.891 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.891 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.892 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:31:15.892 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.137112+0000", 2023-09-23T20:31:15.892 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:31:15.892 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.892 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.892 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:31:15.892 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.137161+0000", 2023-09-23T20:31:15.892 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.9493999999999997e-05 2023-09-23T20:31:15.892 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.892 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.893 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:31:15.893 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.137181+0000", 2023-09-23T20:31:15.893 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.0123999999999999e-05 2023-09-23T20:31:15.893 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.893 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.893 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:31:15.893 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.137186+0000", 2023-09-23T20:31:15.893 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.9629999999999997e-06 2023-09-23T20:31:15.893 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.893 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.894 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:31:15.894 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.137243+0000", 2023-09-23T20:31:15.894 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.698e-05 2023-09-23T20:31:15.894 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.894 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.894 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:31:15.894 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.137256+0000", 2023-09-23T20:31:15.894 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.27e-05 2023-09-23T20:31:15.894 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.894 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.894 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:31:15.895 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:19.049292+0000", 2023-09-23T20:31:15.895 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.91203635400000005 2023-09-23T20:31:15.895 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.895 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.895 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:31:15.895 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:19.049302+0000", 2023-09-23T20:31:15.895 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.3160000000000003e-06 2023-09-23T20:31:15.895 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:31:15.895 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:31:15.895 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:31:15.895 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.896 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.896 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:39726 5.11 5.e6874e31 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e122)", 2023-09-23T20:31:15.896 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:18.137208+0000", 2023-09-23T20:31:15.896 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 477.73975821400001, 2023-09-23T20:31:15.896 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.91213475899999996, 2023-09-23T20:31:15.896 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:31:15.896 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:31:15.896 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:31:15.896 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:31:15.896 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:31:15.896 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 39726 2023-09-23T20:31:15.896 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.897 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:31:15.897 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.897 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:31:15.897 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.137208+0000", 2023-09-23T20:31:15.897 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:31:15.897 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.897 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.897 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:31:15.897 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.137208+0000", 2023-09-23T20:31:15.897 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:31:15.897 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.898 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.898 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:31:15.898 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.137207+0000", 2023-09-23T20:31:15.898 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:31:15.898 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.898 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.898 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:31:15.898 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.137242+0000", 2023-09-23T20:31:15.898 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.4764999999999997e-05 2023-09-23T20:31:15.898 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.898 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.899 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:31:15.899 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.137253+0000", 2023-09-23T20:31:15.899 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0495e-05 2023-09-23T20:31:15.899 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.899 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.899 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:31:15.899 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.137261+0000", 2023-09-23T20:31:15.899 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.9500000000000001e-06 2023-09-23T20:31:15.899 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.899 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.899 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:31:15.899 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.137292+0000", 2023-09-23T20:31:15.899 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.1035999999999999e-05 2023-09-23T20:31:15.900 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.900 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.900 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:31:15.900 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.137296+0000", 2023-09-23T20:31:15.900 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.4900000000000002e-06 2023-09-23T20:31:15.900 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.900 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.900 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:31:15.900 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:19.049335+0000", 2023-09-23T20:31:15.900 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.91203900500000001 2023-09-23T20:31:15.900 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.900 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.901 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:31:15.901 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:19.049343+0000", 2023-09-23T20:31:15.901 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.5290000000000003e-06 2023-09-23T20:31:15.901 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:31:15.901 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:31:15.901 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:31:15.901 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.901 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.901 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:39760 5.11 5.824f07d1 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e122)", 2023-09-23T20:31:15.901 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:18.156298+0000", 2023-09-23T20:31:15.901 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 477.72066799300001, 2023-09-23T20:31:15.901 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.89306234699999998, 2023-09-23T20:31:15.902 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:31:15.902 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:31:15.902 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:31:15.902 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:31:15.902 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:31:15.902 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 39760 2023-09-23T20:31:15.902 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.902 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:31:15.902 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.902 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:31:15.902 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.156298+0000", 2023-09-23T20:31:15.902 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:31:15.903 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.903 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.903 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:31:15.903 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.156298+0000", 2023-09-23T20:31:15.903 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:31:15.903 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.903 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.903 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:31:15.903 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.156296+0000", 2023-09-23T20:31:15.903 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999981 2023-09-23T20:31:15.903 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.903 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.903 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:31:15.904 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.156367+0000", 2023-09-23T20:31:15.904 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.0866999999999998e-05 2023-09-23T20:31:15.904 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.904 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.904 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:31:15.904 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.156392+0000", 2023-09-23T20:31:15.904 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.5219999999999999e-05 2023-09-23T20:31:15.904 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.904 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.904 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:31:15.904 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.156400+0000", 2023-09-23T20:31:15.904 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.3259999999999996e-06 2023-09-23T20:31:15.905 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.905 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.905 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:31:15.905 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.156461+0000", 2023-09-23T20:31:15.905 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.0371999999999997e-05 2023-09-23T20:31:15.905 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.905 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.905 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:31:15.905 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.156469+0000", 2023-09-23T20:31:15.905 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.4959999999999996e-06 2023-09-23T20:31:15.905 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.905 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.906 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:31:15.906 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:19.049355+0000", 2023-09-23T20:31:15.906 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.89288598100000005 2023-09-23T20:31:15.906 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.906 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.906 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:31:15.906 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:19.049360+0000", 2023-09-23T20:31:15.906 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.1449999999999999e-06 2023-09-23T20:31:15.906 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:31:15.906 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:31:15.906 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:31:15.906 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.907 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.907 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:39802 5.11 5.6adc9311 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e122)", 2023-09-23T20:31:15.907 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:18.185993+0000", 2023-09-23T20:31:15.907 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 477.69097270499998, 2023-09-23T20:31:15.907 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.86338408, 2023-09-23T20:31:15.907 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:31:15.907 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:31:15.907 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:31:15.907 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:31:15.907 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:31:15.907 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 39802 2023-09-23T20:31:15.907 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.907 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:31:15.908 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.908 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:31:15.908 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.185993+0000", 2023-09-23T20:31:15.908 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:31:15.908 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.908 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.908 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:31:15.908 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.185993+0000", 2023-09-23T20:31:15.908 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:31:15.908 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.908 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.908 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:31:15.909 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.185991+0000", 2023-09-23T20:31:15.909 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999981 2023-09-23T20:31:15.909 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.909 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.909 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:31:15.909 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.186111+0000", 2023-09-23T20:31:15.909 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000119867 2023-09-23T20:31:15.909 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.909 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.909 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:31:15.909 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.186134+0000", 2023-09-23T20:31:15.909 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.2836e-05 2023-09-23T20:31:15.910 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.910 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.910 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:31:15.910 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.186143+0000", 2023-09-23T20:31:15.910 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.2890000000000003e-06 2023-09-23T20:31:15.910 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.910 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.910 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:31:15.910 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.186204+0000", 2023-09-23T20:31:15.910 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.0875000000000002e-05 2023-09-23T20:31:15.910 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.910 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.911 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:31:15.911 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.186214+0000", 2023-09-23T20:31:15.911 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.3700000000000001e-06 2023-09-23T20:31:15.911 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.911 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.911 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:31:15.911 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:19.049371+0000", 2023-09-23T20:31:15.911 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.86315763099999998 2023-09-23T20:31:15.911 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.911 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.911 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:31:15.911 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:19.049377+0000", 2023-09-23T20:31:15.912 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.1469999999999998e-06 2023-09-23T20:31:15.912 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:31:15.912 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:31:15.912 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:31:15.912 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.912 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.912 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:39822 5.11 5.72f66af1 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e122)", 2023-09-23T20:31:15.912 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:18.195831+0000", 2023-09-23T20:31:15.912 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 477.681135488, 2023-09-23T20:31:15.912 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.85358087500000002, 2023-09-23T20:31:15.912 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:31:15.912 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:31:15.913 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:31:15.913 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:31:15.913 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:31:15.913 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 39822 2023-09-23T20:31:15.913 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.913 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:31:15.913 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.913 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:31:15.913 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.195831+0000", 2023-09-23T20:31:15.913 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:31:15.913 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.913 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.914 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:31:15.914 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.195831+0000", 2023-09-23T20:31:15.914 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:31:15.914 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.914 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.914 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:31:15.914 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.195829+0000", 2023-09-23T20:31:15.914 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:31:15.914 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.914 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.914 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:31:15.915 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.195955+0000", 2023-09-23T20:31:15.915 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000125561 2023-09-23T20:31:15.915 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.915 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.915 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:31:15.915 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.195970+0000", 2023-09-23T20:31:15.915 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.4769999999999999e-05 2023-09-23T20:31:15.915 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.915 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.915 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:31:15.915 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.195977+0000", 2023-09-23T20:31:15.916 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.8800000000000002e-06 2023-09-23T20:31:15.916 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.916 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.916 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:31:15.916 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.196021+0000", 2023-09-23T20:31:15.916 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.4672000000000002e-05 2023-09-23T20:31:15.916 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.916 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.916 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:31:15.916 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.196028+0000", 2023-09-23T20:31:15.916 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.2119999999999997e-06 2023-09-23T20:31:15.917 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.917 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.917 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:31:15.917 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:19.049401+0000", 2023-09-23T20:31:15.917 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.85337292600000003 2023-09-23T20:31:15.917 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.917 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.917 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:31:15.917 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:19.049411+0000", 2023-09-23T20:31:15.917 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0093e-05 2023-09-23T20:31:15.917 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:31:15.917 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:31:15.918 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:31:15.918 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.918 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.918 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:39831 5.11 5.3f234b1 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e122)", 2023-09-23T20:31:15.918 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:18.200476+0000", 2023-09-23T20:31:15.918 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 477.67648961600003, 2023-09-23T20:31:15.918 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.84895043999999997, 2023-09-23T20:31:15.918 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:31:15.918 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:31:15.918 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:31:15.918 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:31:15.919 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:31:15.919 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 39831 2023-09-23T20:31:15.919 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.919 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:31:15.919 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.919 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:31:15.919 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.200476+0000", 2023-09-23T20:31:15.919 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:31:15.919 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.919 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.919 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:31:15.920 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.200476+0000", 2023-09-23T20:31:15.920 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:31:15.920 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.920 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.920 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:31:15.920 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.200476+0000", 2023-09-23T20:31:15.920 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:31:15.920 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.920 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.920 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:31:15.920 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.200552+0000", 2023-09-23T20:31:15.921 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.6395000000000003e-05 2023-09-23T20:31:15.921 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.921 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.921 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:31:15.921 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.200561+0000", 2023-09-23T20:31:15.921 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.2099999999999993e-06 2023-09-23T20:31:15.921 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.921 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.921 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:31:15.921 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.200567+0000", 2023-09-23T20:31:15.921 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.3330000000000002e-06 2023-09-23T20:31:15.921 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.922 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.922 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:31:15.922 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.200619+0000", 2023-09-23T20:31:15.922 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.2435000000000002e-05 2023-09-23T20:31:15.922 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.922 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.922 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:31:15.922 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.200626+0000", 2023-09-23T20:31:15.922 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.1099999999999997e-06 2023-09-23T20:31:15.922 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.922 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.923 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:31:15.923 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:19.049420+0000", 2023-09-23T20:31:15.923 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.84879393000000003 2023-09-23T20:31:15.923 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.923 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.923 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:31:15.923 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:19.049427+0000", 2023-09-23T20:31:15.923 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.5180000000000001e-06 2023-09-23T20:31:15.923 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:31:15.923 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:31:15.923 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:31:15.923 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.924 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.924 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:39856 5.11 5.d501d031 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e122)", 2023-09-23T20:31:15.924 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:18.218506+0000", 2023-09-23T20:31:15.924 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 477.65846039199999, 2023-09-23T20:31:15.924 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.83093710200000004, 2023-09-23T20:31:15.924 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:31:15.924 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:31:15.924 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:31:15.926 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:31:15.926 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:31:15.926 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 39856 2023-09-23T20:31:15.926 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.926 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:31:15.926 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.926 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:31:15.926 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.218506+0000", 2023-09-23T20:31:15.927 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:31:15.927 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.927 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.927 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:31:15.927 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.218506+0000", 2023-09-23T20:31:15.927 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:31:15.927 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.927 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.927 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:31:15.927 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.218504+0000", 2023-09-23T20:31:15.927 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:31:15.927 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.928 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.928 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:31:15.928 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.218539+0000", 2023-09-23T20:31:15.928 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.5040999999999999e-05 2023-09-23T20:31:15.928 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.928 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.928 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:31:15.928 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.218558+0000", 2023-09-23T20:31:15.928 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.8372e-05 2023-09-23T20:31:15.928 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.928 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.928 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:31:15.929 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.218566+0000", 2023-09-23T20:31:15.929 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.9149999999999996e-06 2023-09-23T20:31:15.929 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.929 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.929 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:31:15.929 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.218615+0000", 2023-09-23T20:31:15.929 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.9008000000000002e-05 2023-09-23T20:31:15.929 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.929 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.929 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:31:15.929 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.218623+0000", 2023-09-23T20:31:15.929 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.9840000000000001e-06 2023-09-23T20:31:15.930 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.930 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.930 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:31:15.930 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:19.049437+0000", 2023-09-23T20:31:15.930 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.83081455500000001 2023-09-23T20:31:15.930 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.930 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.930 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:31:15.930 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:19.049443+0000", 2023-09-23T20:31:15.930 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.6790000000000002e-06 2023-09-23T20:31:15.930 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:31:15.931 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:31:15.931 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:31:15.931 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.931 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.931 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:39890 5.11 5.15c21ed1 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e122)", 2023-09-23T20:31:15.931 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:18.248436+0000", 2023-09-23T20:31:15.931 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 477.62853039300001, 2023-09-23T20:31:15.931 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.80101856299999996, 2023-09-23T20:31:15.931 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:31:15.931 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:31:15.931 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:31:15.931 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:31:15.932 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:31:15.932 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 39890 2023-09-23T20:31:15.932 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.932 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:31:15.932 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.932 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:31:15.932 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.248436+0000", 2023-09-23T20:31:15.932 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:31:15.932 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.932 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.932 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:31:15.932 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.248436+0000", 2023-09-23T20:31:15.933 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:31:15.933 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.933 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.933 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:31:15.933 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.248434+0000", 2023-09-23T20:31:15.933 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:31:15.933 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.933 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.933 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:31:15.933 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.248491+0000", 2023-09-23T20:31:15.933 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.6669000000000002e-05 2023-09-23T20:31:15.934 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.934 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.934 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:31:15.934 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.248508+0000", 2023-09-23T20:31:15.934 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.6878e-05 2023-09-23T20:31:15.934 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.934 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.934 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:31:15.934 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.248514+0000", 2023-09-23T20:31:15.934 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.0909999999999998e-06 2023-09-23T20:31:15.934 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.934 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.935 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:31:15.935 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.248558+0000", 2023-09-23T20:31:15.935 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.4320999999999998e-05 2023-09-23T20:31:15.935 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.935 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.935 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:31:15.935 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.248566+0000", 2023-09-23T20:31:15.935 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.6879999999999999e-06 2023-09-23T20:31:15.935 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.935 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.935 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:31:15.935 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:19.049450+0000", 2023-09-23T20:31:15.936 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.80088458600000001 2023-09-23T20:31:15.936 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.936 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.936 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:31:15.936 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:19.049454+0000", 2023-09-23T20:31:15.936 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.783e-06 2023-09-23T20:31:15.936 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:31:15.936 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:31:15.936 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:31:15.936 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.936 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.936 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:39893 5.11 5.4eed3c91 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e122)", 2023-09-23T20:31:15.937 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:18.251010+0000", 2023-09-23T20:31:15.937 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 477.62595627000002, 2023-09-23T20:31:15.937 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.79845648700000005, 2023-09-23T20:31:15.937 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:31:15.937 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:31:15.937 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:31:15.937 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:31:15.937 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:31:15.937 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 39893 2023-09-23T20:31:15.937 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.937 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:31:15.937 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.938 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:31:15.938 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.251010+0000", 2023-09-23T20:31:15.938 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:31:15.938 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.938 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.938 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:31:15.938 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.251010+0000", 2023-09-23T20:31:15.938 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:31:15.938 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.938 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.938 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:31:15.938 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.251009+0000", 2023-09-23T20:31:15.939 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:31:15.939 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.939 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.939 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:31:15.939 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.251097+0000", 2023-09-23T20:31:15.939 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.7786000000000003e-05 2023-09-23T20:31:15.939 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.939 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.939 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:31:15.939 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.251109+0000", 2023-09-23T20:31:15.939 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.2044e-05 2023-09-23T20:31:15.939 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.940 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.940 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:31:15.940 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.251115+0000", 2023-09-23T20:31:15.940 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.4420000000000003e-06 2023-09-23T20:31:15.940 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.940 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.940 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:31:15.940 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.251152+0000", 2023-09-23T20:31:15.940 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.6545999999999999e-05 2023-09-23T20:31:15.940 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.940 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.940 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:31:15.941 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:18.251159+0000", 2023-09-23T20:31:15.941 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.8909999999999998e-06 2023-09-23T20:31:15.941 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.941 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.941 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:31:15.941 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:19.049463+0000", 2023-09-23T20:31:15.941 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.79830458699999995 2023-09-23T20:31:15.941 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.941 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.941 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:31:15.941 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:19.049466+0000", 2023-09-23T20:31:15.941 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.145e-06 2023-09-23T20:31:15.942 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:31:15.942 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:31:15.942 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:31:15.942 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.942 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.942 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:49668 5.11 5:8ffafbda:::benchmark_data_smithi018_228451_object49667:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e130)", 2023-09-23T20:31:15.942 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:30.144126+0000", 2023-09-23T20:31:15.942 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 465.73284016999997, 2023-09-23T20:31:15.942 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0173537960000001, 2023-09-23T20:31:15.942 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:31:15.942 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:31:15.942 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:31:15.942 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:31:15.943 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:31:15.943 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 49668 2023-09-23T20:31:15.943 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.943 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:31:15.943 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.943 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:31:15.943 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.144126+0000", 2023-09-23T20:31:15.943 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:31:15.943 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.943 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.943 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:31:15.943 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.144126+0000", 2023-09-23T20:31:15.944 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:31:15.944 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.944 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.944 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:31:15.944 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.144124+0000", 2023-09-23T20:31:15.944 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:31:15.944 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.944 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.944 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:31:15.944 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.144171+0000", 2023-09-23T20:31:15.944 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.7029e-05 2023-09-23T20:31:15.945 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.945 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.945 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:31:15.945 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.144188+0000", 2023-09-23T20:31:15.945 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.721e-05 2023-09-23T20:31:15.945 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.945 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.945 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:31:15.945 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.144196+0000", 2023-09-23T20:31:15.945 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.498e-06 2023-09-23T20:31:15.945 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.945 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.946 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:31:15.946 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.144252+0000", 2023-09-23T20:31:15.946 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.5965999999999999e-05 2023-09-23T20:31:15.946 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.946 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.946 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op must wait for map", 2023-09-23T20:31:15.946 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.144260+0000", 2023-09-23T20:31:15.946 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.1920000000000005e-06 2023-09-23T20:31:15.946 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.946 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.946 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:31:15.946 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.149093+0000", 2023-09-23T20:31:15.947 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.004832622 2023-09-23T20:31:15.947 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.947 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.947 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:31:15.947 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.149103+0000", 2023-09-23T20:31:15.947 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.8479999999999996e-06 2023-09-23T20:31:15.947 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.947 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.947 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:31:15.947 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.147686+0000", 2023-09-23T20:31:15.947 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.99858325400000003 2023-09-23T20:31:15.947 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.947 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.948 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:31:15.948 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.147822+0000", 2023-09-23T20:31:15.948 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00013592199999999999 2023-09-23T20:31:15.948 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.948 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.948 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 4,7", 2023-09-23T20:31:15.948 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.148002+0000", 2023-09-23T20:31:15.948 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000180185 2023-09-23T20:31:15.948 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.948 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.948 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:31:15.948 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.152001+0000", 2023-09-23T20:31:15.949 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0039989049999999996 2023-09-23T20:31:15.949 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.949 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.949 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:31:15.949 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.161379+0000", 2023-09-23T20:31:15.949 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0093781179999999995 2023-09-23T20:31:15.949 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.949 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.949 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:31:15.949 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.161407+0000", 2023-09-23T20:31:15.949 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.7951999999999999e-05 2023-09-23T20:31:15.949 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.950 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.950 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:31:15.950 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.161445+0000", 2023-09-23T20:31:15.950 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.7999e-05 2023-09-23T20:31:15.950 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.950 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.950 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:31:15.950 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.161480+0000", 2023-09-23T20:31:15.950 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.4752999999999997e-05 2023-09-23T20:31:15.950 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:31:15.950 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:31:15.950 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:31:15.951 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.951 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.951 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:49677 5.11 5:8cd3bece:::benchmark_data_smithi018_228451_object49676:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-09-23T20:31:15.951 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:30.145601+0000", 2023-09-23T20:31:15.951 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 465.73136530599999, 2023-09-23T20:31:15.951 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0160527699999999, 2023-09-23T20:31:15.951 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:31:15.951 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:31:15.951 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:31:15.951 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:31:15.951 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:31:15.951 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 49677 2023-09-23T20:31:15.951 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.952 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:31:15.952 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.952 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:31:15.952 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.145601+0000", 2023-09-23T20:31:15.952 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:31:15.952 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.952 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.952 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:31:15.952 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.145601+0000", 2023-09-23T20:31:15.952 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:31:15.952 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.952 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.953 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:31:15.953 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.145600+0000", 2023-09-23T20:31:15.953 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:31:15.953 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.953 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.953 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:31:15.953 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.145714+0000", 2023-09-23T20:31:15.953 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000114341 2023-09-23T20:31:15.953 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.953 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.953 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:31:15.953 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.145729+0000", 2023-09-23T20:31:15.954 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.5061e-05 2023-09-23T20:31:15.954 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.954 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.954 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:31:15.954 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.145736+0000", 2023-09-23T20:31:15.954 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.9800000000000001e-06 2023-09-23T20:31:15.954 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.954 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.954 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:31:15.954 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.146231+0000", 2023-09-23T20:31:15.954 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00049503600000000002 2023-09-23T20:31:15.954 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.955 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.955 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting_for_map not empty", 2023-09-23T20:31:15.955 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.146242+0000", 2023-09-23T20:31:15.955 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1114999999999999e-05 2023-09-23T20:31:15.955 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.955 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.955 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:31:15.955 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.149132+0000", 2023-09-23T20:31:15.955 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0028893930000000001 2023-09-23T20:31:15.955 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.955 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.955 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:31:15.956 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.149139+0000", 2023-09-23T20:31:15.956 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.0319999999999998e-06 2023-09-23T20:31:15.956 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.956 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.956 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:31:15.956 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.148678+0000", 2023-09-23T20:31:15.956 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.999538861 2023-09-23T20:31:15.956 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.956 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.956 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:31:15.956 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.148809+0000", 2023-09-23T20:31:15.957 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000130968 2023-09-23T20:31:15.957 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.957 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.957 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 4,7", 2023-09-23T20:31:15.957 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.148965+0000", 2023-09-23T20:31:15.957 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00015674800000000001 2023-09-23T20:31:15.957 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.957 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.957 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:31:15.957 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.152012+0000", 2023-09-23T20:31:15.957 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0030466339999999999 2023-09-23T20:31:15.957 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.957 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.958 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:31:15.958 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.161507+0000", 2023-09-23T20:31:15.958 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0094949679999999995 2023-09-23T20:31:15.958 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.958 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.958 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:31:15.958 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.161588+0000", 2023-09-23T20:31:15.958 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.1589999999999996e-05 2023-09-23T20:31:15.958 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.958 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.958 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:31:15.958 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.161617+0000", 2023-09-23T20:31:15.959 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.8099e-05 2023-09-23T20:31:15.959 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.959 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.959 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:31:15.959 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.161654+0000", 2023-09-23T20:31:15.959 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.6927000000000001e-05 2023-09-23T20:31:15.959 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:31:15.959 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:31:15.959 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:31:15.959 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.959 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.959 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:49755 5.11 5:8b8671c4:::benchmark_data_smithi018_228451_object49754:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-09-23T20:31:15.960 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:30.216172+0000", 2023-09-23T20:31:15.960 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 465.660793598, 2023-09-23T20:31:15.960 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.94558546499999996, 2023-09-23T20:31:15.960 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:31:15.960 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:31:15.960 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:31:15.960 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:31:15.960 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:31:15.960 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 49755 2023-09-23T20:31:15.960 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.960 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:31:15.960 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.961 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:31:15.961 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.216172+0000", 2023-09-23T20:31:15.961 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:31:15.961 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.961 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.961 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:31:15.961 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.216172+0000", 2023-09-23T20:31:15.961 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:31:15.961 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.961 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.961 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:31:15.961 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.216170+0000", 2023-09-23T20:31:15.961 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999976 2023-09-23T20:31:15.962 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.962 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.962 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:31:15.962 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.216305+0000", 2023-09-23T20:31:15.962 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00013507699999999999 2023-09-23T20:31:15.962 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.962 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.962 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:31:15.962 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.216341+0000", 2023-09-23T20:31:15.962 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.6109999999999998e-05 2023-09-23T20:31:15.962 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.962 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.963 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:31:15.963 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.216349+0000", 2023-09-23T20:31:15.963 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.123e-06 2023-09-23T20:31:15.963 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.963 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.963 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:31:15.963 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.216426+0000", 2023-09-23T20:31:15.963 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.6604e-05 2023-09-23T20:31:15.963 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.963 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.963 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:31:15.964 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.216440+0000", 2023-09-23T20:31:15.964 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.3787e-05 2023-09-23T20:31:15.964 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.964 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.964 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:31:15.964 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.149653+0000", 2023-09-23T20:31:15.964 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.93321383099999999 2023-09-23T20:31:15.964 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.964 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.964 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:31:15.964 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.149779+0000", 2023-09-23T20:31:15.965 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00012566300000000001 2023-09-23T20:31:15.965 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.965 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.965 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 4,7", 2023-09-23T20:31:15.965 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.149925+0000", 2023-09-23T20:31:15.965 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00014582199999999999 2023-09-23T20:31:15.965 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.965 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.965 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:31:15.965 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.153584+0000", 2023-09-23T20:31:15.965 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0036591560000000002 2023-09-23T20:31:15.965 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.966 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.966 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:31:15.966 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.161679+0000", 2023-09-23T20:31:15.966 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0080945359999999994 2023-09-23T20:31:15.966 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.966 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.966 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:31:15.966 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.161695+0000", 2023-09-23T20:31:15.966 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.6311e-05 2023-09-23T20:31:15.966 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.966 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.967 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:31:15.967 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.161729+0000", 2023-09-23T20:31:15.967 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.4637000000000001e-05 2023-09-23T20:31:15.967 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.967 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.967 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:31:15.967 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.161758+0000", 2023-09-23T20:31:15.967 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.8422999999999999e-05 2023-09-23T20:31:15.967 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:31:15.967 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:31:15.967 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:31:15.967 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.968 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.968 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:49769 5.11 5:8ef0bf42:::benchmark_data_smithi018_228451_object49768:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-09-23T20:31:15.968 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:30.222066+0000", 2023-09-23T20:31:15.968 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 465.65489963300001, 2023-09-23T20:31:15.968 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.93977777699999998, 2023-09-23T20:31:15.968 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:31:15.968 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:31:15.968 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:31:15.968 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:31:15.968 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:31:15.968 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 49769 2023-09-23T20:31:15.969 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.969 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:31:15.969 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.969 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:31:15.969 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.222066+0000", 2023-09-23T20:31:15.969 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:31:15.969 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.969 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.969 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:31:15.969 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.222066+0000", 2023-09-23T20:31:15.969 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:31:15.969 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.970 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.970 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:31:15.970 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.222065+0000", 2023-09-23T20:31:15.970 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:31:15.970 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.970 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.970 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:31:15.970 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.222094+0000", 2023-09-23T20:31:15.970 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.9128999999999998e-05 2023-09-23T20:31:15.970 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.970 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.971 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:31:15.971 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.222112+0000", 2023-09-23T20:31:15.971 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.7099999999999999e-05 2023-09-23T20:31:15.971 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.971 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.971 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:31:15.971 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.222119+0000", 2023-09-23T20:31:15.971 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.1389999999999998e-06 2023-09-23T20:31:15.971 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.971 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.971 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:31:15.971 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.222180+0000", 2023-09-23T20:31:15.972 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.1301000000000001e-05 2023-09-23T20:31:15.972 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.972 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.972 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:31:15.972 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.222192+0000", 2023-09-23T20:31:15.972 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1661999999999999e-05 2023-09-23T20:31:15.972 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.972 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.972 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:31:15.972 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.150500+0000", 2023-09-23T20:31:15.972 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.92830877700000003 2023-09-23T20:31:15.973 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.973 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.973 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:31:15.973 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.150616+0000", 2023-09-23T20:31:15.973 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00011518099999999999 2023-09-23T20:31:15.973 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.973 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.973 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 4,7", 2023-09-23T20:31:15.973 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.150742+0000", 2023-09-23T20:31:15.973 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00012607699999999999 2023-09-23T20:31:15.973 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.973 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.974 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:31:15.974 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.153592+0000", 2023-09-23T20:31:15.974 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0028501569999999999 2023-09-23T20:31:15.974 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.974 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.974 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:31:15.974 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.161779+0000", 2023-09-23T20:31:15.974 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0081868459999999994 2023-09-23T20:31:15.974 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.974 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.974 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:31:15.974 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.161794+0000", 2023-09-23T20:31:15.975 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.5384999999999999e-05 2023-09-23T20:31:15.975 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.975 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.975 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:31:15.975 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.161826+0000", 2023-09-23T20:31:15.975 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.1483999999999999e-05 2023-09-23T20:31:15.975 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.975 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.975 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:31:15.975 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.161844+0000", 2023-09-23T20:31:15.975 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.8658999999999999e-05 2023-09-23T20:31:15.976 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:31:15.976 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:31:15.976 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:31:15.976 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.976 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.976 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:49810 5.11 5:8d760048:::benchmark_data_smithi018_228451_object49809:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-09-23T20:31:15.976 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:30.257088+0000", 2023-09-23T20:31:15.976 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 465.619878341, 2023-09-23T20:31:15.976 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.90484562899999998, 2023-09-23T20:31:15.976 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:31:15.976 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:31:15.976 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:31:15.977 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:31:15.977 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:31:15.977 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 49810 2023-09-23T20:31:15.977 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.977 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:31:15.977 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.977 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:31:15.977 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.257088+0000", 2023-09-23T20:31:15.977 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:31:15.977 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.977 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.978 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:31:15.978 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.257088+0000", 2023-09-23T20:31:15.978 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:31:15.978 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.978 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.978 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:31:15.978 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.257086+0000", 2023-09-23T20:31:15.978 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:31:15.978 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.978 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.978 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:31:15.978 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.257121+0000", 2023-09-23T20:31:15.979 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.4424000000000001e-05 2023-09-23T20:31:15.979 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.979 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.979 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:31:15.979 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.257134+0000", 2023-09-23T20:31:15.979 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.3409e-05 2023-09-23T20:31:15.979 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.979 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.979 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:31:15.979 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.257140+0000", 2023-09-23T20:31:15.979 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.9089999999999997e-06 2023-09-23T20:31:15.979 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.980 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.980 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:31:15.980 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.257208+0000", 2023-09-23T20:31:15.980 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.7716e-05 2023-09-23T20:31:15.980 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.980 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.980 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:31:15.980 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.257219+0000", 2023-09-23T20:31:15.980 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1382999999999999e-05 2023-09-23T20:31:15.980 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.980 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.980 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:31:15.981 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.151334+0000", 2023-09-23T20:31:15.981 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.89411527000000002 2023-09-23T20:31:15.981 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.981 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.981 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:31:15.981 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.151410+0000", 2023-09-23T20:31:15.981 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.5368000000000001e-05 2023-09-23T20:31:15.981 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.981 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.981 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 4,7", 2023-09-23T20:31:15.981 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.151519+0000", 2023-09-23T20:31:15.982 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000109481 2023-09-23T20:31:15.982 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.982 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.982 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:31:15.982 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.155912+0000", 2023-09-23T20:31:15.982 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0043927510000000003 2023-09-23T20:31:15.982 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.982 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.982 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:31:15.982 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.161867+0000", 2023-09-23T20:31:15.982 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0059548459999999998 2023-09-23T20:31:15.982 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.983 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.983 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:31:15.983 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.161882+0000", 2023-09-23T20:31:15.983 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.5053e-05 2023-09-23T20:31:15.983 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.983 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.983 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:31:15.983 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.161908+0000", 2023-09-23T20:31:15.983 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.6160999999999999e-05 2023-09-23T20:31:15.983 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.983 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.983 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:31:15.984 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.161933+0000", 2023-09-23T20:31:15.984 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.5448000000000001e-05 2023-09-23T20:31:15.984 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:31:15.984 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:31:15.984 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:31:15.984 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.984 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.984 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:49846 5.11 5:8a7c8f0f:::benchmark_data_smithi018_228451_object49845:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-09-23T20:31:15.984 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:30.283871+0000", 2023-09-23T20:31:15.984 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 465.593095017, 2023-09-23T20:31:15.984 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.87818426900000002, 2023-09-23T20:31:15.985 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:31:15.985 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:31:15.985 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:31:15.985 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:31:15.985 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:31:15.985 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 49846 2023-09-23T20:31:15.985 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.985 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:31:15.985 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.985 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:31:15.985 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.283871+0000", 2023-09-23T20:31:15.985 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:31:15.986 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.986 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.986 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:31:15.986 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.283871+0000", 2023-09-23T20:31:15.986 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:31:15.986 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.986 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.986 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:31:15.986 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.283870+0000", 2023-09-23T20:31:15.986 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:31:15.986 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.986 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.987 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:31:15.987 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.283939+0000", 2023-09-23T20:31:15.987 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.9799e-05 2023-09-23T20:31:15.987 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.987 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.987 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:31:15.987 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.283960+0000", 2023-09-23T20:31:15.987 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.0613999999999999e-05 2023-09-23T20:31:15.987 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.987 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.987 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:31:15.987 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.283968+0000", 2023-09-23T20:31:15.988 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.6429999999999995e-06 2023-09-23T20:31:15.988 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.988 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.988 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:31:15.988 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.284026+0000", 2023-09-23T20:31:15.988 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.8850000000000001e-05 2023-09-23T20:31:15.988 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.988 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.988 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:31:15.988 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.284033+0000", 2023-09-23T20:31:15.988 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.9249999999999998e-06 2023-09-23T20:31:15.988 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.989 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.989 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:31:15.989 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.152057+0000", 2023-09-23T20:31:15.989 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.86802386899999995 2023-09-23T20:31:15.989 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.989 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.989 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:31:15.989 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.152189+0000", 2023-09-23T20:31:15.989 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00013166600000000001 2023-09-23T20:31:15.989 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.989 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.989 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 4,7", 2023-09-23T20:31:15.990 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.152356+0000", 2023-09-23T20:31:15.990 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00016740599999999999 2023-09-23T20:31:15.990 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.990 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.990 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:31:15.990 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.156658+0000", 2023-09-23T20:31:15.990 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0043016720000000003 2023-09-23T20:31:15.990 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.990 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.990 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:31:15.990 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.161959+0000", 2023-09-23T20:31:15.990 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0053010699999999997 2023-09-23T20:31:15.991 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.991 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.991 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:31:15.991 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.161975+0000", 2023-09-23T20:31:15.991 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.6016e-05 2023-09-23T20:31:15.991 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.991 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.991 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:31:15.991 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.162015+0000", 2023-09-23T20:31:15.991 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.0240000000000001e-05 2023-09-23T20:31:15.991 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.991 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.992 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:31:15.992 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.162055+0000", 2023-09-23T20:31:15.992 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.9984000000000002e-05 2023-09-23T20:31:15.992 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:31:15.992 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:31:15.992 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:31:15.992 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.992 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.992 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:49862 5.11 5:8acf6c38:::benchmark_data_smithi018_228451_object49861:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-09-23T20:31:15.992 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:30.291752+0000", 2023-09-23T20:31:15.992 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 465.58521364699999, 2023-09-23T20:31:15.992 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.87046705199999996, 2023-09-23T20:31:15.992 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:31:15.993 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:31:15.993 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:31:15.993 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:31:15.993 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:31:15.993 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 49862 2023-09-23T20:31:15.993 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.993 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:31:15.993 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.993 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:31:15.993 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.291752+0000", 2023-09-23T20:31:15.993 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:31:15.993 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.994 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.994 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:31:15.994 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.291752+0000", 2023-09-23T20:31:15.994 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:31:15.994 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.994 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.994 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:31:15.994 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.291751+0000", 2023-09-23T20:31:15.994 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:31:15.994 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.994 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.994 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:31:15.995 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.291796+0000", 2023-09-23T20:31:15.995 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.4452000000000001e-05 2023-09-23T20:31:15.995 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.995 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.995 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:31:15.995 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.291814+0000", 2023-09-23T20:31:15.995 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.7846000000000002e-05 2023-09-23T20:31:15.995 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.995 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.995 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:31:15.995 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.291821+0000", 2023-09-23T20:31:15.995 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.4549999999999998e-06 2023-09-23T20:31:15.996 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.996 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.996 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:31:15.996 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.291932+0000", 2023-09-23T20:31:15.996 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000110945 2023-09-23T20:31:15.996 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.996 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.996 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:31:15.996 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.291942+0000", 2023-09-23T20:31:15.996 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0392e-05 2023-09-23T20:31:15.996 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.996 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.997 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:31:15.997 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.152826+0000", 2023-09-23T20:31:15.997 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.86088340100000005 2023-09-23T20:31:15.997 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.997 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.997 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:31:15.997 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.152946+0000", 2023-09-23T20:31:15.997 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000119963 2023-09-23T20:31:15.997 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.997 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.997 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 4,7", 2023-09-23T20:31:15.997 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.153060+0000", 2023-09-23T20:31:15.998 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00011451300000000001 2023-09-23T20:31:15.998 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.998 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.998 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:31:15.998 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.159094+0000", 2023-09-23T20:31:15.998 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0060340300000000001 2023-09-23T20:31:15.998 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.998 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.998 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:31:15.998 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.162100+0000", 2023-09-23T20:31:15.998 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0030057199999999999 2023-09-23T20:31:15.998 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.999 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.999 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:31:15.999 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.162137+0000", 2023-09-23T20:31:15.999 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.7002999999999998e-05 2023-09-23T20:31:15.999 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.999 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.999 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:31:15.999 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.162183+0000", 2023-09-23T20:31:15.999 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.6199000000000003e-05 2023-09-23T20:31:15.999 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:15.999 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:15.999 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:31:16.000 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.162219+0000", 2023-09-23T20:31:16.000 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.6186999999999998e-05 2023-09-23T20:31:16.000 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:31:16.000 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:31:16.000 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:31:16.000 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:16.000 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:16.000 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:49899 5.11 5:8d0362d8:::benchmark_data_smithi018_228451_object49898:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-09-23T20:31:16.000 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:30.316014+0000", 2023-09-23T20:31:16.000 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 465.56095202300003, 2023-09-23T20:31:16.000 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.84638897400000002, 2023-09-23T20:31:16.000 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:31:16.001 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:31:16.001 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:31:16.001 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:31:16.001 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:31:16.001 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 49899 2023-09-23T20:31:16.001 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:16.001 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:31:16.001 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:16.001 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:31:16.001 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.316014+0000", 2023-09-23T20:31:16.001 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:31:16.001 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:16.001 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:16.002 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:31:16.002 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.316014+0000", 2023-09-23T20:31:16.002 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:31:16.002 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:16.002 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:16.002 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:31:16.002 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.316007+0000", 2023-09-23T20:31:16.002 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999933 2023-09-23T20:31:16.002 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:16.002 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:16.002 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:31:16.003 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.316092+0000", 2023-09-23T20:31:16.003 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.4536999999999994e-05 2023-09-23T20:31:16.003 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:16.003 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:16.003 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:31:16.003 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.316110+0000", 2023-09-23T20:31:16.003 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.8204000000000001e-05 2023-09-23T20:31:16.003 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:16.003 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:16.003 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:31:16.003 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.316115+0000", 2023-09-23T20:31:16.003 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.2079999999999999e-06 2023-09-23T20:31:16.004 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:16.004 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:16.004 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:31:16.004 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.316190+0000", 2023-09-23T20:31:16.004 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.4400999999999995e-05 2023-09-23T20:31:16.004 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:16.004 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:16.004 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:31:16.004 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.316197+0000", 2023-09-23T20:31:16.004 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.0620000000000003e-06 2023-09-23T20:31:16.004 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:16.004 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:16.005 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:31:16.005 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.153639+0000", 2023-09-23T20:31:16.005 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.83744230799999997 2023-09-23T20:31:16.005 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:16.005 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:16.005 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:31:16.005 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.153769+0000", 2023-09-23T20:31:16.005 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00012973699999999999 2023-09-23T20:31:16.005 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:16.005 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:16.005 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 4,7", 2023-09-23T20:31:16.005 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.153882+0000", 2023-09-23T20:31:16.006 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00011381400000000001 2023-09-23T20:31:16.006 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:16.006 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:16.006 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:31:16.006 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.159107+0000", 2023-09-23T20:31:16.006 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0052248950000000002 2023-09-23T20:31:16.006 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:16.006 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:16.006 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:31:16.006 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.162256+0000", 2023-09-23T20:31:16.006 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0031487669999999998 2023-09-23T20:31:16.007 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:16.007 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:16.007 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:31:16.007 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.162314+0000", 2023-09-23T20:31:16.007 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.7707999999999997e-05 2023-09-23T20:31:16.007 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:16.007 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:16.007 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:31:16.007 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.162367+0000", 2023-09-23T20:31:16.007 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.3381000000000003e-05 2023-09-23T20:31:16.007 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:16.007 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:16.008 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:31:16.008 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.162403+0000", 2023-09-23T20:31:16.008 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.5754999999999999e-05 2023-09-23T20:31:16.008 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:31:16.008 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:31:16.008 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:31:16.008 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:16.008 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:16.008 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:49914 5.11 5:8de3f302:::benchmark_data_smithi018_228451_object49913:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-09-23T20:31:16.008 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:30.327710+0000", 2023-09-23T20:31:16.008 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 465.54925568499999, 2023-09-23T20:31:16.008 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.83480937700000002, 2023-09-23T20:31:16.009 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:31:16.009 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:31:16.009 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:31:16.009 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:31:16.009 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:31:16.009 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 49914 2023-09-23T20:31:16.009 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:16.009 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:31:16.009 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:16.009 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:31:16.009 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.327710+0000", 2023-09-23T20:31:16.009 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:31:16.009 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:16.010 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:16.010 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:31:16.010 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.327710+0000", 2023-09-23T20:31:16.010 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:31:16.010 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:16.010 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:16.010 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:31:16.010 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.327709+0000", 2023-09-23T20:31:16.010 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:31:16.010 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:16.010 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:16.011 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:31:16.011 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.327798+0000", 2023-09-23T20:31:16.011 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.9086999999999996e-05 2023-09-23T20:31:16.011 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:16.011 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:16.011 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:31:16.011 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.327814+0000", 2023-09-23T20:31:16.011 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.5974999999999999e-05 2023-09-23T20:31:16.011 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:16.011 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:16.011 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:31:16.011 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.327820+0000", 2023-09-23T20:31:16.012 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.614e-06 2023-09-23T20:31:16.012 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:16.012 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:16.012 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:31:16.012 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.327881+0000", 2023-09-23T20:31:16.012 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.1271000000000003e-05 2023-09-23T20:31:16.012 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:16.012 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:16.012 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:31:16.012 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.327897+0000", 2023-09-23T20:31:16.012 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.5962000000000001e-05 2023-09-23T20:31:16.012 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:16.013 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:16.013 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:31:16.013 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.154407+0000", 2023-09-23T20:31:16.013 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.82650979199999997 2023-09-23T20:31:16.013 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:16.013 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:16.013 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:31:16.013 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.154489+0000", 2023-09-23T20:31:16.013 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.1904999999999996e-05 2023-09-23T20:31:16.013 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:16.013 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:16.013 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 4,7", 2023-09-23T20:31:16.014 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.154587+0000", 2023-09-23T20:31:16.014 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.8391e-05 2023-09-23T20:31:16.014 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:16.014 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:16.014 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:31:16.014 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.159115+0000", 2023-09-23T20:31:16.014 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0045281030000000003 2023-09-23T20:31:16.014 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:16.014 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:16.014 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:31:16.014 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.162286+0000", 2023-09-23T20:31:16.014 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0031707110000000001 2023-09-23T20:31:16.015 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:16.015 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:16.015 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:31:16.015 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.162438+0000", 2023-09-23T20:31:16.015 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000152192 2023-09-23T20:31:16.015 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:16.015 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:16.015 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:31:16.015 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.162483+0000", 2023-09-23T20:31:16.015 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.4950000000000002e-05 2023-09-23T20:31:16.015 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:16.016 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:16.016 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:31:16.016 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.162520+0000", 2023-09-23T20:31:16.016 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.6643999999999997e-05 2023-09-23T20:31:16.016 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:31:16.016 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:31:16.016 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:31:16.016 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:16.016 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:16.016 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:49943 5.11 5:88faa156:::benchmark_data_smithi018_228451_object49942:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-09-23T20:31:16.016 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:30.349163+0000", 2023-09-23T20:31:16.016 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 465.52780298499999, 2023-09-23T20:31:16.017 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.81349829299999998, 2023-09-23T20:31:16.017 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:31:16.017 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:31:16.017 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:31:16.017 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:31:16.017 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:31:16.017 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 49943 2023-09-23T20:31:16.017 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:16.017 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:31:16.017 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:16.017 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:31:16.017 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.349163+0000", 2023-09-23T20:31:16.018 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:31:16.018 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:16.018 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:16.018 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:31:16.018 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.349163+0000", 2023-09-23T20:31:16.018 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:31:16.018 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:16.018 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:16.018 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:31:16.018 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.349162+0000", 2023-09-23T20:31:16.018 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:31:16.018 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:16.019 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:16.019 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:31:16.019 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.349210+0000", 2023-09-23T20:31:16.019 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.8068999999999997e-05 2023-09-23T20:31:16.019 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:16.019 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:16.019 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:31:16.019 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.349222+0000", 2023-09-23T20:31:16.019 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.2434e-05 2023-09-23T20:31:16.019 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:16.019 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:16.019 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:31:16.020 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.349228+0000", 2023-09-23T20:31:16.020 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.0170000000000004e-06 2023-09-23T20:31:16.020 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:16.020 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:16.020 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:31:16.020 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.349276+0000", 2023-09-23T20:31:16.020 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.8976e-05 2023-09-23T20:31:16.020 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:16.020 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:16.020 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:31:16.020 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.349285+0000", 2023-09-23T20:31:16.020 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.4009999999999997e-06 2023-09-23T20:31:16.021 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:16.021 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:16.021 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:31:16.021 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.155211+0000", 2023-09-23T20:31:16.021 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.80592635199999996 2023-09-23T20:31:16.021 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:16.021 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:16.021 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:31:16.021 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.155298+0000", 2023-09-23T20:31:16.021 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.6293999999999997e-05 2023-09-23T20:31:16.021 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:16.021 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:16.022 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 4,7", 2023-09-23T20:31:16.022 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.155431+0000", 2023-09-23T20:31:16.022 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000133416 2023-09-23T20:31:16.022 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:16.022 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:16.022 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:31:16.022 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.159126+0000", 2023-09-23T20:31:16.022 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0036948879999999999 2023-09-23T20:31:16.022 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:16.022 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:16.022 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:31:16.022 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.162565+0000", 2023-09-23T20:31:16.022 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.003438853 2023-09-23T20:31:16.023 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:16.023 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:16.023 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:31:16.023 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.162594+0000", 2023-09-23T20:31:16.023 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.9000999999999999e-05 2023-09-23T20:31:16.023 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:16.023 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:16.023 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:31:16.023 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.162632+0000", 2023-09-23T20:31:16.023 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.8451000000000002e-05 2023-09-23T20:31:16.023 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:31:16.023 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:31:16.024 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:31:16.024 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.162661+0000", 2023-09-23T20:31:16.024 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.9220000000000001e-05 2023-09-23T20:31:16.024 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:31:16.024 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:31:16.024 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:31:16.024 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:31:16.024 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:31:16.024 INFO:teuthology.orchestra.run.smithi149.stdout:} 2023-09-23T20:31:16.025 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:16.025 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:31:16.026 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:31:16.127 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:31:16.234 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T20:31:16.245 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:16.334 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:31:16.339 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:16.339 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:31:16.440 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:31:16.502 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:16 smithi149 ceph-mon[170394]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:31:16.502 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:16 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:31:16.502 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:16 smithi149 ceph-mon[170394]: osdmap e293: 8 total, 8 up, 7 in 2023-09-23T20:31:16.502 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:16 smithi149 ceph-mon[170394]: failed to encode map e293 with expected crc 2023-09-23T20:31:16.502 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:16 smithi149 ceph-mon[170394]: failed to encode map e293 with expected crc 2023-09-23T20:31:16.502 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:16 smithi149 ceph-mon[170394]: failed to encode map e293 with expected crc 2023-09-23T20:31:16.502 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:16 smithi149 ceph-mon[170394]: failed to encode map e293 with expected crc 2023-09-23T20:31:16.503 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:16 smithi149 ceph-mon[170394]: failed to encode map e293 with expected crc 2023-09-23T20:31:16.503 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:16 smithi149 ceph-mon[170394]: failed to encode map e293 with expected crc 2023-09-23T20:31:16.503 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:16 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 29254/17994 objects degraded (162.576%), 13 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2023-09-23T20:31:16.503 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:16.549 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:31:16.552 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:16.552 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:31:16.581 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:16 smithi018 ceph-mon[179647]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:31:16.581 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:16 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:31:16.581 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:16 smithi018 ceph-mon[179647]: osdmap e293: 8 total, 8 up, 7 in 2023-09-23T20:31:16.581 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:16 smithi018 ceph-mon[179647]: failed to encode map e293 with expected crc 2023-09-23T20:31:16.581 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:16 smithi018 ceph-mon[179647]: failed to encode map e293 with expected crc 2023-09-23T20:31:16.581 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:16 smithi018 ceph-mon[179647]: failed to encode map e293 with expected crc 2023-09-23T20:31:16.582 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:16 smithi018 ceph-mon[179647]: failed to encode map e293 with expected crc 2023-09-23T20:31:16.582 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:16 smithi018 ceph-mon[179647]: failed to encode map e293 with expected crc 2023-09-23T20:31:16.582 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:16 smithi018 ceph-mon[179647]: failed to encode map e293 with expected crc 2023-09-23T20:31:16.582 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:16 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 29254/17994 objects degraded (162.576%), 13 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2023-09-23T20:31:16.582 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:16 smithi018 ceph-mon[184144]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:31:16.582 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:16 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:31:16.582 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:16 smithi018 ceph-mon[184144]: osdmap e293: 8 total, 8 up, 7 in 2023-09-23T20:31:16.582 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:16 smithi018 ceph-mon[184144]: failed to encode map e293 with expected crc 2023-09-23T20:31:16.582 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:16 smithi018 ceph-mon[184144]: failed to encode map e293 with expected crc 2023-09-23T20:31:16.583 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:16 smithi018 ceph-mon[184144]: failed to encode map e293 with expected crc 2023-09-23T20:31:16.583 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:16 smithi018 ceph-mon[184144]: failed to encode map e293 with expected crc 2023-09-23T20:31:16.583 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:16 smithi018 ceph-mon[184144]: failed to encode map e293 with expected crc 2023-09-23T20:31:16.583 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:16 smithi018 ceph-mon[184144]: failed to encode map e293 with expected crc 2023-09-23T20:31:16.583 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:16 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 29254/17994 objects degraded (162.576%), 13 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2023-09-23T20:31:16.653 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:31:16.757 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:16.758 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:31:16.818 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:31:16.840 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:31:16.853 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:16.853 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:31:16.868 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:31:16.922 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:31:16.945 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:31:16.954 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:31:16.963 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:31:17.000 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:17.013 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:31:17.069 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:31:17.104 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:31:17.124 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:31:17.124 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:31:17.124 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:31:17.125 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:31:17.125 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:31:17.125 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:31:17.125 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:31:17.125 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:31:17.438 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:17 smithi018 ceph-mon[179647]: pgmap v664: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 311 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 692 op/s; 29254/14787 objects degraded (197.836%); 653/14787 objects misplaced (4.416%); 0 B/s, 75 objects/s recovering 2023-09-23T20:31:17.439 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:17 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3616563480' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:31:17.439 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:17 smithi018 ceph-mon[184144]: pgmap v664: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 311 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 692 op/s; 29254/14787 objects degraded (197.836%); 653/14787 objects misplaced (4.416%); 0 B/s, 75 objects/s recovering 2023-09-23T20:31:17.439 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:17 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3616563480' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:31:17.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:17 smithi149 ceph-mon[170394]: pgmap v664: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 311 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 692 op/s; 29254/14787 objects degraded (197.836%); 653/14787 objects misplaced (4.416%); 0 B/s, 75 objects/s recovering 2023-09-23T20:31:17.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:17 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3616563480' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:31:18.027 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T20:31:18.216 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:31:18.220 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:18.220 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:31:18.321 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:31:18.404 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:18.515 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:31:18.535 INFO:teuthology.orchestra.run.smithi018.stderr:dumped all 2023-09-23T20:31:18.693 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:18 smithi018 ceph-mon[179647]: pgmap v665: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 248 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 687 op/s; 29254/11790 objects degraded (248.126%); 462/11790 objects misplaced (3.919%); 0 B/s, 105 objects/s recovering 2023-09-23T20:31:18.693 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:18 smithi018 ceph-mon[184144]: pgmap v665: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 248 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 687 op/s; 29254/11790 objects degraded (248.126%); 462/11790 objects misplaced (3.919%); 0 B/s, 105 objects/s recovering 2023-09-23T20:31:18.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:18 smithi149 ceph-mon[170394]: pgmap v665: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait, 3 active+remapped+backfill_wait, 11 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 248 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 687 op/s; 29254/11790 objects degraded (248.126%); 462/11790 objects misplaced (3.919%); 0 B/s, 105 objects/s recovering 2023-09-23T20:31:18.874 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:31:18.877 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:18.877 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:31:18.955 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:31:18.965 INFO:teuthology.orchestra.run.smithi018.stdout:{"pg_ready":true,"pg_map":{"version":665,"stamp":"2023-09-23T20:31:17.718669+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":260465262,"num_objects":3930,"num_object_clones":0,"num_object_copies":11790,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":29254,"num_objects_misplaced":462,"num_objects_unfound":0,"num_objects_dirty":3930,"num_whiteouts":0,"num_read":6210,"num_read_kb":13614,"num_write":206587,"num_write_kb":6716893,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22864,"num_bytes_recovered":1151531232,"num_keys_recovered":30,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":188104,"ondisk_log_size":188104,"up":357,"acting":344,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":371,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":4143216,"kb_used_data":2260472,"kb_used_omap":32,"kb_used_meta":1882591,"kb_avail":652072848,"statfs":{"total":671965249536,"available":667722596352,"internally_reserved":0,"allocated":2314723328,"data_stored":2306050935,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32773,"internal_metadata":1927774203},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[4,0,5,4,6],"upper_bound":32},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-360841216,"num_objects":-5506,"num_object_clones":0,"num_object_copies":-16518,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-627,"num_objects_unfound":0,"num_objects_dirty":-5506,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5506,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":841,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.003132"},"pg_stats":[{"pgid":"3.1a","version":"69'1","reported_seq":36,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T20:31:00.194697+0000","last_change":"2023-09-23T20:29:46.050662+0000","last_active":"2023-09-23T20:31:00.194697+0000","last_peered":"2023-09-23T20:31:00.194697+0000","last_clean":"2023-09-23T20:31:00.194697+0000","last_became_active":"2023-09-23T20:29:46.050281+0000","last_became_peered":"2023-09-23T20:29:46.050281+0000","last_unstale":"2023-09-23T20:31:00.194697+0000","last_undegraded":"2023-09-23T20:31:00.194697+0000","last_fullsized":"2023-09-23T20:31:00.194697+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:55.227008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":164,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T20:31:00.222214+0000","last_change":"2023-09-23T20:29:46.067085+0000","last_active":"2023-09-23T20:31:00.222214+0000","last_peered":"2023-09-23T20:31:00.222214+0000","last_clean":"2023-09-23T20:31:00.222214+0000","last_became_active":"2023-09-23T20:29:46.066738+0000","last_became_peered":"2023-09-23T20:29:46.066738+0000","last_unstale":"2023-09-23T20:31:00.222214+0000","last_undegraded":"2023-09-23T20:31:00.222214+0000","last_fullsized":"2023-09-23T20:31:00.222214+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:33:14.679402+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":228,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T20:31:00.196002+0000","last_change":"2023-09-23T20:29:46.178684+0000","last_active":"2023-09-23T20:31:00.196002+0000","last_peered":"2023-09-23T20:31:00.196002+0000","last_clean":"2023-09-23T20:31:00.196002+0000","last_became_active":"2023-09-23T20:29:46.178249+0000","last_became_peered":"2023-09-23T20:29:46.178249+0000","last_unstale":"2023-09-23T20:31:00.196002+0000","last_undegraded":"2023-09-23T20:31:00.196002+0000","last_fullsized":"2023-09-23T20:31:00.196002+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:58:58.530687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":257,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T20:31:07.300242+0000","last_change":"2023-09-23T20:25:02.957185+0000","last_active":"2023-09-23T20:31:07.300242+0000","last_peered":"2023-09-23T20:31:07.300242+0000","last_clean":"2023-09-23T20:31:07.300242+0000","last_became_active":"2023-09-23T20:25:02.956689+0000","last_became_peered":"2023-09-23T20:25:02.956689+0000","last_unstale":"2023-09-23T20:31:07.300242+0000","last_undegraded":"2023-09-23T20:31:07.300242+0000","last_fullsized":"2023-09-23T20:31:07.300242+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:04:28.524015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":153,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T20:31:00.222173+0000","last_change":"2023-09-23T20:29:45.529762+0000","last_active":"2023-09-23T20:31:00.222173+0000","last_peered":"2023-09-23T20:31:00.222173+0000","last_clean":"2023-09-23T20:31:00.222173+0000","last_became_active":"2023-09-23T20:29:45.529283+0000","last_became_peered":"2023-09-23T20:29:45.529283+0000","last_unstale":"2023-09-23T20:31:00.222173+0000","last_undegraded":"2023-09-23T20:31:00.222173+0000","last_fullsized":"2023-09-23T20:31:00.222173+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:54:36.720075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":66,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-09-23T20:31:14.394757+0000","last_change":"2023-09-23T20:29:46.112977+0000","last_active":"2023-09-23T20:31:14.394757+0000","last_peered":"2023-09-23T20:31:14.394757+0000","last_clean":"2023-09-23T20:31:14.394757+0000","last_became_active":"2023-09-23T20:29:46.112600+0000","last_became_peered":"2023-09-23T20:29:46.112600+0000","last_unstale":"2023-09-23T20:31:14.394757+0000","last_undegraded":"2023-09-23T20:31:14.394757+0000","last_fullsized":"2023-09-23T20:31:14.394757+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:06:34.132247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.12","version":"291'9238","reported_seq":15831,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T20:31:13.362120+0000","last_change":"2023-09-23T20:30:46.961845+0000","last_active":"2023-09-23T20:31:13.362120+0000","last_peered":"2023-09-23T20:31:13.362120+0000","last_clean":"2023-09-23T20:31:13.362120+0000","last_became_active":"2023-09-23T20:30:46.939470+0000","last_became_peered":"2023-09-23T20:30:46.939470+0000","last_unstale":"2023-09-23T20:31:13.362120+0000","last_undegraded":"2023-09-23T20:31:13.362120+0000","last_fullsized":"2023-09-23T20:31:13.362120+0000","mapping_epoch":284,"log_start":"224'3457","ondisk_log_start":"224'3457","created":242,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":5781,"log_dups_size":0,"ondisk_log_size":5781,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":15400960,"num_objects":235,"num_object_clones":0,"num_object_copies":705,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":235,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6427,"num_write_kb":213184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3241,"num_bytes_recovered":105185280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":255,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T20:31:07.300575+0000","last_change":"2023-09-23T20:25:02.969432+0000","last_active":"2023-09-23T20:31:07.300575+0000","last_peered":"2023-09-23T20:31:07.300575+0000","last_clean":"2023-09-23T20:31:07.300575+0000","last_became_active":"2023-09-23T20:25:02.969118+0000","last_became_peered":"2023-09-23T20:25:02.969118+0000","last_unstale":"2023-09-23T20:31:07.300575+0000","last_undegraded":"2023-09-23T20:31:07.300575+0000","last_fullsized":"2023-09-23T20:31:07.300575+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:02:22.453831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":173,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T20:31:00.222242+0000","last_change":"2023-09-23T20:29:46.069832+0000","last_active":"2023-09-23T20:31:00.222242+0000","last_peered":"2023-09-23T20:31:00.222242+0000","last_clean":"2023-09-23T20:31:00.222242+0000","last_became_active":"2023-09-23T20:29:46.069451+0000","last_became_peered":"2023-09-23T20:29:46.069451+0000","last_unstale":"2023-09-23T20:31:00.222242+0000","last_undegraded":"2023-09-23T20:31:00.222242+0000","last_fullsized":"2023-09-23T20:31:00.222242+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:06:06.104030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.13","version":"293'9153","reported_seq":9521,"reported_epoch":293,"state":"active+recovery_wait","last_fresh":"2023-09-23T20:31:15.704242+0000","last_change":"2023-09-23T20:30:30.361943+0000","last_active":"2023-09-23T20:31:15.704242+0000","last_peered":"2023-09-23T20:31:15.704242+0000","last_clean":"2023-09-23T20:28:04.010682+0000","last_became_active":"2023-09-23T20:29:47.054756+0000","last_became_peered":"2023-09-23T20:29:47.054756+0000","last_unstale":"2023-09-23T20:31:15.704242+0000","last_undegraded":"2023-09-23T20:31:15.704242+0000","last_fullsized":"2023-09-23T20:31:15.704242+0000","mapping_epoch":268,"log_start":"222'1746","ondisk_log_start":"222'1746","created":242,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":7407,"log_dups_size":0,"ondisk_log_size":7407,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":12746752,"num_objects":195,"num_object_clones":0,"num_object_copies":585,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":195,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6385,"num_write_kb":210528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":117,"num_bytes_recovered":7503872,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":["7","2","6"],"object_location_counts":[{"shards":"2,6,7","objects":195}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.14","version":"90'161","reported_seq":563,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-09-23T20:31:14.402412+0000","last_change":"2023-09-23T20:29:48.092787+0000","last_active":"2023-09-23T20:31:14.402412+0000","last_peered":"2023-09-23T20:31:14.402412+0000","last_clean":"2023-09-23T20:31:14.402412+0000","last_became_active":"2023-09-23T20:29:48.092460+0000","last_became_peered":"2023-09-23T20:29:48.092460+0000","last_unstale":"2023-09-23T20:31:14.402412+0000","last_undegraded":"2023-09-23T20:31:14.402412+0000","last_fullsized":"2023-09-23T20:31:14.402412+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:13.244651+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.244651+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0019955509999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109227,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":268,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T20:31:07.307051+0000","last_change":"2023-09-23T20:29:45.523808+0000","last_active":"2023-09-23T20:31:07.307051+0000","last_peered":"2023-09-23T20:31:07.307051+0000","last_clean":"2023-09-23T20:31:07.307051+0000","last_became_active":"2023-09-23T20:29:45.523508+0000","last_became_peered":"2023-09-23T20:29:45.523508+0000","last_unstale":"2023-09-23T20:31:07.307051+0000","last_undegraded":"2023-09-23T20:31:07.307051+0000","last_fullsized":"2023-09-23T20:31:07.307051+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:50:55.565352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":156,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T20:31:00.221996+0000","last_change":"2023-09-23T20:29:04.130512+0000","last_active":"2023-09-23T20:31:00.221996+0000","last_peered":"2023-09-23T20:31:00.221996+0000","last_clean":"2023-09-23T20:31:00.221996+0000","last_became_active":"2023-09-23T20:29:04.130135+0000","last_became_peered":"2023-09-23T20:29:04.130135+0000","last_unstale":"2023-09-23T20:31:00.221996+0000","last_undegraded":"2023-09-23T20:31:00.221996+0000","last_fullsized":"2023-09-23T20:31:00.221996+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:53:59.385893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.10","version":"292'9258","reported_seq":9354,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-09-23T20:31:14.958168+0000","last_change":"2023-09-23T20:30:24.864928+0000","last_active":"2023-09-23T20:31:14.958168+0000","last_peered":"2023-09-23T20:31:14.958168+0000","last_clean":"2023-09-23T20:31:14.958168+0000","last_became_active":"2023-09-23T20:28:35.582165+0000","last_became_peered":"2023-09-23T20:28:35.582165+0000","last_unstale":"2023-09-23T20:31:14.958168+0000","last_undegraded":"2023-09-23T20:31:14.958168+0000","last_fullsized":"2023-09-23T20:31:14.958168+0000","mapping_epoch":242,"log_start":"221'72","ondisk_log_start":"221'72","created":242,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":9186,"log_dups_size":0,"ondisk_log_size":9186,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":9142272,"num_objects":140,"num_object_clones":0,"num_object_copies":420,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":140,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6440,"num_write_kb":210528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":304,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-09-23T20:31:14.390129+0000","last_change":"2023-09-23T20:29:59.501260+0000","last_active":"2023-09-23T20:31:14.390129+0000","last_peered":"2023-09-23T20:31:14.390129+0000","last_clean":"2023-09-23T20:31:14.390129+0000","last_became_active":"2023-09-23T20:29:28.362070+0000","last_became_peered":"2023-09-23T20:29:28.362070+0000","last_unstale":"2023-09-23T20:31:14.390129+0000","last_undegraded":"2023-09-23T20:31:14.390129+0000","last_fullsized":"2023-09-23T20:31:14.390129+0000","mapping_epoch":258,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:32:55.983696+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":238,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T20:31:00.195010+0000","last_change":"2023-09-23T20:29:46.080766+0000","last_active":"2023-09-23T20:31:00.195010+0000","last_peered":"2023-09-23T20:31:00.195010+0000","last_clean":"2023-09-23T20:31:00.195010+0000","last_became_active":"2023-09-23T20:29:46.080356+0000","last_became_peered":"2023-09-23T20:29:46.080356+0000","last_unstale":"2023-09-23T20:31:00.195010+0000","last_undegraded":"2023-09-23T20:31:00.195010+0000","last_fullsized":"2023-09-23T20:31:00.195010+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:08:01.862270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.11","version":"291'9026","reported_seq":17023,"reported_epoch":291,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T20:31:10.591402+0000","last_change":"2023-09-23T20:30:44.984034+0000","last_active":"2023-09-23T20:31:10.591402+0000","last_peered":"2023-09-23T20:31:10.591402+0000","last_clean":"2023-09-23T20:28:04.011715+0000","last_became_active":"2023-09-23T20:29:46.191939+0000","last_became_peered":"2023-09-23T20:29:46.191939+0000","last_unstale":"2023-09-23T20:31:10.591402+0000","last_undegraded":"2023-09-23T20:29:49.867530+0000","last_fullsized":"2023-09-23T20:29:46.041211+0000","mapping_epoch":267,"log_start":"222'1590","ondisk_log_start":"222'1590","created":242,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":7436,"log_dups_size":0,"ondisk_log_size":7436,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":12845056,"num_objects":196,"num_object_clones":0,"num_object_copies":588,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":47,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":196,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6272,"num_write_kb":206976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3587,"num_bytes_recovered":160956416,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[6,2],"avail_no_missing":["6","2"],"object_location_counts":[{"shards":"2,6","objects":149}],"blocked_by":[],"up_primary":1,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":110,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-09-23T20:31:14.389520+0000","last_change":"2023-09-23T20:29:59.529392+0000","last_active":"2023-09-23T20:31:14.389520+0000","last_peered":"2023-09-23T20:31:14.389520+0000","last_clean":"2023-09-23T20:31:14.389520+0000","last_became_active":"2023-09-23T20:29:28.365579+0000","last_became_peered":"2023-09-23T20:29:28.365579+0000","last_unstale":"2023-09-23T20:31:14.389520+0000","last_undegraded":"2023-09-23T20:31:14.389520+0000","last_fullsized":"2023-09-23T20:31:14.389520+0000","mapping_epoch":258,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:31:11.546083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":288,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-09-23T20:31:14.395009+0000","last_change":"2023-09-23T20:29:46.091575+0000","last_active":"2023-09-23T20:31:14.395009+0000","last_peered":"2023-09-23T20:31:14.395009+0000","last_clean":"2023-09-23T20:31:14.395009+0000","last_became_active":"2023-09-23T20:29:46.091271+0000","last_became_peered":"2023-09-23T20:29:46.091271+0000","last_unstale":"2023-09-23T20:31:14.395009+0000","last_undegraded":"2023-09-23T20:31:14.395009+0000","last_fullsized":"2023-09-23T20:31:14.395009+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:08:01.594175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.16","version":"291'9059","reported_seq":9162,"reported_epoch":291,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T20:31:13.279024+0000","last_change":"2023-09-23T20:29:29.535020+0000","last_active":"2023-09-23T20:31:13.279024+0000","last_peered":"2023-09-23T20:31:13.279024+0000","last_clean":"2023-09-23T20:29:27.303063+0000","last_became_active":"2023-09-23T20:29:29.522829+0000","last_became_peered":"2023-09-23T20:29:29.522829+0000","last_unstale":"2023-09-23T20:31:13.279024+0000","last_undegraded":"2023-09-23T20:31:13.279024+0000","last_fullsized":"2023-09-23T20:31:13.279024+0000","mapping_epoch":259,"log_start":"224'3213","ondisk_log_start":"224'3213","created":242,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":5846,"log_dups_size":0,"ondisk_log_size":5846,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":12517376,"num_objects":191,"num_object_clones":0,"num_object_copies":573,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":191,"num_objects_unfound":0,"num_objects_dirty":191,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6317,"num_write_kb":208256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,4],"avail_no_missing":["3","4","7"],"object_location_counts":[{"shards":"3,4,7","objects":191}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.11","version":"90'161","reported_seq":508,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T20:31:07.301451+0000","last_change":"2023-09-23T20:30:36.766139+0000","last_active":"2023-09-23T20:31:07.301451+0000","last_peered":"2023-09-23T20:31:07.301451+0000","last_clean":"2023-09-23T20:31:07.301451+0000","last_became_active":"2023-09-23T20:28:05.991304+0000","last_became_peered":"2023-09-23T20:28:05.991304+0000","last_unstale":"2023-09-23T20:31:07.301451+0000","last_undegraded":"2023-09-23T20:31:07.301451+0000","last_fullsized":"2023-09-23T20:31:07.301451+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:34.821789+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:34.821789+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":303,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-09-23T20:31:14.389822+0000","last_change":"2023-09-23T20:29:59.544701+0000","last_active":"2023-09-23T20:31:14.389822+0000","last_peered":"2023-09-23T20:31:14.389822+0000","last_clean":"2023-09-23T20:31:14.389822+0000","last_became_active":"2023-09-23T20:29:28.367679+0000","last_became_peered":"2023-09-23T20:29:28.367679+0000","last_unstale":"2023-09-23T20:31:14.389822+0000","last_undegraded":"2023-09-23T20:31:14.389822+0000","last_fullsized":"2023-09-23T20:31:14.389822+0000","mapping_epoch":258,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:36:19.089724+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":102,"num_keys_recovered":30,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":153,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T20:31:00.228048+0000","last_change":"2023-09-23T20:29:45.531343+0000","last_active":"2023-09-23T20:31:00.228048+0000","last_peered":"2023-09-23T20:31:00.228048+0000","last_clean":"2023-09-23T20:31:00.228048+0000","last_became_active":"2023-09-23T20:29:45.530713+0000","last_became_peered":"2023-09-23T20:29:45.530713+0000","last_unstale":"2023-09-23T20:31:00.228048+0000","last_undegraded":"2023-09-23T20:31:00.228048+0000","last_fullsized":"2023-09-23T20:31:00.228048+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:28:00.245911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.17","version":"292'9356","reported_seq":9471,"reported_epoch":292,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:31:15.219059+0000","last_change":"2023-09-23T20:29:46.291295+0000","last_active":"2023-09-23T20:31:15.219059+0000","last_peered":"2023-09-23T20:31:15.219059+0000","last_clean":"2023-09-23T20:29:44.520411+0000","last_became_active":"2023-09-23T20:29:46.269706+0000","last_became_peered":"2023-09-23T20:29:46.269706+0000","last_unstale":"2023-09-23T20:31:15.219059+0000","last_undegraded":"2023-09-23T20:29:46.075283+0000","last_fullsized":"2023-09-23T20:29:46.032679+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":242,"last_epoch_clean":260,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":9356,"log_dups_size":0,"ondisk_log_size":9356,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":7438336,"num_objects":114,"num_object_clones":0,"num_object_copies":342,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3310,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":114,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6528,"num_write_kb":212512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3],"avail_no_missing":["5","3","4"],"object_location_counts":[{"shards":"3,4,5","objects":103},{"shards":"3,5","objects":-1720}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.10","version":"90'161","reported_seq":512,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T20:31:07.306553+0000","last_change":"2023-09-23T20:29:07.045212+0000","last_active":"2023-09-23T20:31:07.306553+0000","last_peered":"2023-09-23T20:31:07.306553+0000","last_clean":"2023-09-23T20:31:07.306553+0000","last_became_active":"2023-09-23T20:29:07.044892+0000","last_became_peered":"2023-09-23T20:29:07.044892+0000","last_unstale":"2023-09-23T20:31:07.306553+0000","last_undegraded":"2023-09-23T20:31:07.306553+0000","last_fullsized":"2023-09-23T20:31:07.306553+0000","mapping_epoch":253,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":254,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:32.801168+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:32.801168+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":255,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T20:31:07.301106+0000","last_change":"2023-09-23T20:25:02.953993+0000","last_active":"2023-09-23T20:31:07.301106+0000","last_peered":"2023-09-23T20:31:07.301106+0000","last_clean":"2023-09-23T20:31:07.301106+0000","last_became_active":"2023-09-23T20:25:02.953672+0000","last_became_peered":"2023-09-23T20:25:02.953672+0000","last_unstale":"2023-09-23T20:31:07.301106+0000","last_undegraded":"2023-09-23T20:31:07.301106+0000","last_fullsized":"2023-09-23T20:31:07.301106+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:26:08.603577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":270,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T20:31:07.309072+0000","last_change":"2023-09-23T20:25:02.949584+0000","last_active":"2023-09-23T20:31:07.309072+0000","last_peered":"2023-09-23T20:31:07.309072+0000","last_clean":"2023-09-23T20:31:07.309072+0000","last_became_active":"2023-09-23T20:25:02.949132+0000","last_became_peered":"2023-09-23T20:25:02.949132+0000","last_unstale":"2023-09-23T20:31:07.309072+0000","last_undegraded":"2023-09-23T20:31:07.309072+0000","last_fullsized":"2023-09-23T20:31:07.309072+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:34:20.509823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.14","version":"291'9198","reported_seq":9303,"reported_epoch":291,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:31:13.386288+0000","last_change":"2023-09-23T20:29:46.199482+0000","last_active":"2023-09-23T20:31:13.386288+0000","last_peered":"2023-09-23T20:31:13.386288+0000","last_clean":"2023-09-23T20:28:12.083147+0000","last_became_active":"2023-09-23T20:29:46.185251+0000","last_became_peered":"2023-09-23T20:29:46.185251+0000","last_unstale":"2023-09-23T20:31:13.386288+0000","last_undegraded":"2023-09-23T20:29:46.057476+0000","last_fullsized":"2023-09-23T20:29:46.043200+0000","mapping_epoch":267,"log_start":"226'3514","ondisk_log_start":"226'3514","created":242,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":5684,"log_dups_size":0,"ondisk_log_size":5684,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8781824,"num_objects":134,"num_object_clones":0,"num_object_copies":402,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2460,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":134,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6434,"num_write_kb":210176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[3,5],"avail_no_missing":["3","5"],"object_location_counts":[{"shards":"3,5","objects":-939}],"blocked_by":[],"up_primary":1,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.13","version":"90'161","reported_seq":517,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T20:31:00.203724+0000","last_change":"2023-09-23T20:29:46.067221+0000","last_active":"2023-09-23T20:31:00.203724+0000","last_peered":"2023-09-23T20:31:00.203724+0000","last_clean":"2023-09-23T20:31:00.203724+0000","last_became_active":"2023-09-23T20:29:46.066622+0000","last_became_peered":"2023-09-23T20:29:46.066622+0000","last_unstale":"2023-09-23T20:31:00.203724+0000","last_undegraded":"2023-09-23T20:31:00.203724+0000","last_fullsized":"2023-09-23T20:31:00.203724+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:12.386098+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:12.386098+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0019500940000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":266,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-09-23T20:31:14.390095+0000","last_change":"2023-09-23T20:29:28.364113+0000","last_active":"2023-09-23T20:31:14.390095+0000","last_peered":"2023-09-23T20:31:14.390095+0000","last_clean":"2023-09-23T20:31:14.390095+0000","last_became_active":"2023-09-23T20:29:28.363788+0000","last_became_peered":"2023-09-23T20:29:28.363788+0000","last_unstale":"2023-09-23T20:31:14.390095+0000","last_undegraded":"2023-09-23T20:31:14.390095+0000","last_fullsized":"2023-09-23T20:31:14.390095+0000","mapping_epoch":258,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":259,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:39:49.658803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":265,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T20:31:07.300176+0000","last_change":"2023-09-23T20:25:02.957010+0000","last_active":"2023-09-23T20:31:07.300176+0000","last_peered":"2023-09-23T20:31:07.300176+0000","last_clean":"2023-09-23T20:31:07.300176+0000","last_became_active":"2023-09-23T20:25:02.956267+0000","last_became_peered":"2023-09-23T20:25:02.956267+0000","last_unstale":"2023-09-23T20:31:07.300176+0000","last_undegraded":"2023-09-23T20:31:07.300176+0000","last_fullsized":"2023-09-23T20:31:07.300176+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:10:55.299422+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.15","version":"293'9074","reported_seq":12873,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-09-23T20:31:15.597373+0000","last_change":"2023-09-23T20:30:01.520686+0000","last_active":"2023-09-23T20:31:15.597373+0000","last_peered":"2023-09-23T20:31:15.597373+0000","last_clean":"2023-09-23T20:31:15.597373+0000","last_became_active":"2023-09-23T20:30:01.486115+0000","last_became_peered":"2023-09-23T20:30:01.486115+0000","last_unstale":"2023-09-23T20:31:15.597373+0000","last_undegraded":"2023-09-23T20:31:15.597373+0000","last_fullsized":"2023-09-23T20:31:15.597373+0000","mapping_epoch":275,"log_start":"222'1801","ondisk_log_start":"222'1801","created":242,"last_epoch_clean":276,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":7273,"log_dups_size":0,"ondisk_log_size":7273,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4947968,"num_objects":76,"num_object_clones":0,"num_object_copies":228,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":76,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6334,"num_write_kb":205088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1838,"num_bytes_recovered":97189888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.12","version":"90'161","reported_seq":531,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T20:31:07.307040+0000","last_change":"2023-09-23T20:29:46.079370+0000","last_active":"2023-09-23T20:31:07.307040+0000","last_peered":"2023-09-23T20:31:07.307040+0000","last_clean":"2023-09-23T20:31:07.307040+0000","last_became_active":"2023-09-23T20:29:46.078873+0000","last_became_peered":"2023-09-23T20:29:46.078873+0000","last_unstale":"2023-09-23T20:31:07.307040+0000","last_undegraded":"2023-09-23T20:31:07.307040+0000","last_fullsized":"2023-09-23T20:31:07.307040+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:35.734584+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:35.734584+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":62,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-09-23T20:31:14.393976+0000","last_change":"2023-09-23T20:29:04.179171+0000","last_active":"2023-09-23T20:31:14.393976+0000","last_peered":"2023-09-23T20:31:14.393976+0000","last_clean":"2023-09-23T20:31:14.393976+0000","last_became_active":"2023-09-23T20:29:04.128672+0000","last_became_peered":"2023-09-23T20:29:04.128672+0000","last_unstale":"2023-09-23T20:31:14.393976+0000","last_undegraded":"2023-09-23T20:31:14.393976+0000","last_fullsized":"2023-09-23T20:31:14.393976+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:49:15.506106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":153,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T20:31:00.226303+0000","last_change":"2023-09-23T20:29:45.528782+0000","last_active":"2023-09-23T20:31:00.226303+0000","last_peered":"2023-09-23T20:31:00.226303+0000","last_clean":"2023-09-23T20:31:00.226303+0000","last_became_active":"2023-09-23T20:29:45.528389+0000","last_became_peered":"2023-09-23T20:29:45.528389+0000","last_unstale":"2023-09-23T20:31:00.226303+0000","last_undegraded":"2023-09-23T20:31:00.226303+0000","last_fullsized":"2023-09-23T20:31:00.226303+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:13:41.453499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.d","version":"90'161","reported_seq":532,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-09-23T20:31:14.389804+0000","last_change":"2023-09-23T20:29:46.061373+0000","last_active":"2023-09-23T20:31:14.389804+0000","last_peered":"2023-09-23T20:31:14.389804+0000","last_clean":"2023-09-23T20:31:14.389804+0000","last_became_active":"2023-09-23T20:29:46.061003+0000","last_became_peered":"2023-09-23T20:29:46.061003+0000","last_unstale":"2023-09-23T20:31:14.389804+0000","last_undegraded":"2023-09-23T20:31:14.389804+0000","last_fullsized":"2023-09-23T20:31:14.389804+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:12.809933+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:12.809933+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.a","version":"292'12940","reported_seq":23966,"reported_epoch":292,"state":"active+recovery_wait","last_fresh":"2023-09-23T20:31:15.250918+0000","last_change":"2023-09-23T20:29:51.518803+0000","last_active":"2023-09-23T20:31:15.250918+0000","last_peered":"2023-09-23T20:31:15.250918+0000","last_clean":"2023-09-23T20:28:04.001969+0000","last_became_active":"2023-09-23T20:29:51.450026+0000","last_became_peered":"2023-09-23T20:29:51.450026+0000","last_unstale":"2023-09-23T20:31:15.250918+0000","last_undegraded":"2023-09-23T20:31:15.250918+0000","last_fullsized":"2023-09-23T20:31:15.250918+0000","mapping_epoch":271,"log_start":"246'6271","ondisk_log_start":"246'6271","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":6669,"log_dups_size":0,"ondisk_log_size":6669,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:55:12.986375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":15073280,"num_objects":230,"num_object_clones":0,"num_object_copies":690,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":230,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12940,"num_write_kb":421440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5457,"num_bytes_recovered":313655296,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":["5","0","6"],"object_location_counts":[{"shards":"0,5,6","objects":230}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":282,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-09-23T20:31:14.390014+0000","last_change":"2023-09-23T20:29:59.586041+0000","last_active":"2023-09-23T20:31:14.390014+0000","last_peered":"2023-09-23T20:31:14.390014+0000","last_clean":"2023-09-23T20:31:14.390014+0000","last_became_active":"2023-09-23T20:29:46.059906+0000","last_became_peered":"2023-09-23T20:29:46.059906+0000","last_unstale":"2023-09-23T20:31:14.390014+0000","last_undegraded":"2023-09-23T20:31:14.390014+0000","last_fullsized":"2023-09-23T20:31:14.390014+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:26:12.412320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":253,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T20:31:07.306727+0000","last_change":"2023-09-23T20:29:04.128604+0000","last_active":"2023-09-23T20:31:07.306727+0000","last_peered":"2023-09-23T20:31:07.306727+0000","last_clean":"2023-09-23T20:31:07.306727+0000","last_became_active":"2023-09-23T20:29:04.128003+0000","last_became_peered":"2023-09-23T20:29:04.128003+0000","last_unstale":"2023-09-23T20:31:07.306727+0000","last_undegraded":"2023-09-23T20:31:07.306727+0000","last_fullsized":"2023-09-23T20:31:07.306727+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:50:49.385599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"90'161","reported_seq":574,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T20:31:00.223974+0000","last_change":"2023-09-23T20:30:01.493499+0000","last_active":"2023-09-23T20:31:00.223974+0000","last_peered":"2023-09-23T20:31:00.223974+0000","last_clean":"2023-09-23T20:31:00.223974+0000","last_became_active":"2023-09-23T20:30:01.493025+0000","last_became_peered":"2023-09-23T20:30:01.493025+0000","last_unstale":"2023-09-23T20:31:00.223974+0000","last_undegraded":"2023-09-23T20:31:00.223974+0000","last_fullsized":"2023-09-23T20:31:00.223974+0000","mapping_epoch":275,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":276,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:13.244651+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.244651+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0019955509999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109227,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":655360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.b","version":"291'12511","reported_seq":12607,"reported_epoch":291,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:31:13.351127+0000","last_change":"2023-09-23T20:29:46.285273+0000","last_active":"2023-09-23T20:31:13.351127+0000","last_peered":"2023-09-23T20:31:13.351127+0000","last_clean":"2023-09-23T20:28:12.089537+0000","last_became_active":"2023-09-23T20:29:46.255023+0000","last_became_peered":"2023-09-23T20:29:46.255023+0000","last_unstale":"2023-09-23T20:31:13.351127+0000","last_undegraded":"2023-09-23T20:29:46.058377+0000","last_fullsized":"2023-09-23T20:29:46.031666+0000","mapping_epoch":267,"log_start":"255'7729","ondisk_log_start":"255'7729","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":4782,"log_dups_size":0,"ondisk_log_size":4782,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:37:55.158131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":18677760,"num_objects":285,"num_object_clones":0,"num_object_copies":855,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4695,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":285,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12511,"num_write_kb":409472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":-1673}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":289,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-09-23T20:31:14.390468+0000","last_change":"2023-09-23T20:29:59.567502+0000","last_active":"2023-09-23T20:31:14.390468+0000","last_peered":"2023-09-23T20:31:14.390468+0000","last_clean":"2023-09-23T20:31:14.390468+0000","last_became_active":"2023-09-23T20:29:46.055041+0000","last_became_peered":"2023-09-23T20:29:46.055041+0000","last_unstale":"2023-09-23T20:31:14.390468+0000","last_undegraded":"2023-09-23T20:31:14.390468+0000","last_fullsized":"2023-09-23T20:31:14.390468+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:47:13.874812+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":188,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T20:31:00.195882+0000","last_change":"2023-09-23T20:29:46.178516+0000","last_active":"2023-09-23T20:31:00.195882+0000","last_peered":"2023-09-23T20:31:00.195882+0000","last_clean":"2023-09-23T20:31:00.195882+0000","last_became_active":"2023-09-23T20:29:46.177858+0000","last_became_peered":"2023-09-23T20:29:46.177858+0000","last_unstale":"2023-09-23T20:31:00.195882+0000","last_undegraded":"2023-09-23T20:31:00.195882+0000","last_fullsized":"2023-09-23T20:31:00.195882+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:09:14.906568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.f","version":"90'161","reported_seq":485,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-09-23T20:31:14.394612+0000","last_change":"2023-09-23T20:29:28.374177+0000","last_active":"2023-09-23T20:31:14.394612+0000","last_peered":"2023-09-23T20:31:14.394612+0000","last_clean":"2023-09-23T20:31:14.394612+0000","last_became_active":"2023-09-23T20:29:28.373750+0000","last_became_peered":"2023-09-23T20:29:28.373750+0000","last_unstale":"2023-09-23T20:31:14.394612+0000","last_undegraded":"2023-09-23T20:31:14.394612+0000","last_fullsized":"2023-09-23T20:31:14.394612+0000","mapping_epoch":258,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":259,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:32.746340+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:32.746340+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,3],"acting":[6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.8","version":"293'9217","reported_seq":9312,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-09-23T20:31:15.600436+0000","last_change":"2023-09-23T20:28:35.581921+0000","last_active":"2023-09-23T20:31:15.600436+0000","last_peered":"2023-09-23T20:31:15.600436+0000","last_clean":"2023-09-23T20:31:15.600436+0000","last_became_active":"2023-09-23T20:28:35.576360+0000","last_became_peered":"2023-09-23T20:28:35.576360+0000","last_unstale":"2023-09-23T20:31:15.600436+0000","last_undegraded":"2023-09-23T20:31:15.600436+0000","last_fullsized":"2023-09-23T20:31:15.600436+0000","mapping_epoch":217,"log_start":"222'1913","ondisk_log_start":"222'1913","created":217,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":7304,"log_dups_size":0,"ondisk_log_size":7304,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":6750208,"num_objects":103,"num_object_clones":0,"num_object_copies":309,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":103,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6431,"num_write_kb":209088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":265,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-09-23T20:31:14.389987+0000","last_change":"2023-09-23T20:29:04.184893+0000","last_active":"2023-09-23T20:31:14.389987+0000","last_peered":"2023-09-23T20:31:14.389987+0000","last_clean":"2023-09-23T20:31:14.389987+0000","last_became_active":"2023-09-23T20:29:04.152723+0000","last_became_peered":"2023-09-23T20:29:04.152723+0000","last_unstale":"2023-09-23T20:31:14.389987+0000","last_undegraded":"2023-09-23T20:31:14.389987+0000","last_fullsized":"2023-09-23T20:31:14.389987+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:51:27.017151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":66,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-09-23T20:31:14.394917+0000","last_change":"2023-09-23T20:29:46.086912+0000","last_active":"2023-09-23T20:31:14.394917+0000","last_peered":"2023-09-23T20:31:14.394917+0000","last_clean":"2023-09-23T20:31:14.394917+0000","last_became_active":"2023-09-23T20:29:46.085471+0000","last_became_peered":"2023-09-23T20:29:46.085471+0000","last_unstale":"2023-09-23T20:31:14.394917+0000","last_undegraded":"2023-09-23T20:31:14.394917+0000","last_fullsized":"2023-09-23T20:31:14.394917+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:04:55.751660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.e","version":"90'161","reported_seq":472,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-09-23T20:31:14.390354+0000","last_change":"2023-09-23T20:29:31.394412+0000","last_active":"2023-09-23T20:31:14.390354+0000","last_peered":"2023-09-23T20:31:14.390354+0000","last_clean":"2023-09-23T20:31:14.390354+0000","last_became_active":"2023-09-23T20:29:31.394053+0000","last_became_peered":"2023-09-23T20:29:31.394053+0000","last_unstale":"2023-09-23T20:31:14.390354+0000","last_undegraded":"2023-09-23T20:31:14.390354+0000","last_fullsized":"2023-09-23T20:31:14.390354+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:31.769333+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:31.769333+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.9","version":"292'9323","reported_seq":9418,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-09-23T20:31:14.770651+0000","last_change":"2023-09-23T20:30:22.842563+0000","last_active":"2023-09-23T20:31:14.770651+0000","last_peered":"2023-09-23T20:31:14.770651+0000","last_clean":"2023-09-23T20:31:14.770651+0000","last_became_active":"2023-09-23T20:28:35.536824+0000","last_became_peered":"2023-09-23T20:28:35.536824+0000","last_unstale":"2023-09-23T20:31:14.770651+0000","last_undegraded":"2023-09-23T20:31:14.770651+0000","last_fullsized":"2023-09-23T20:31:14.770651+0000","mapping_epoch":217,"log_start":"221'463","ondisk_log_start":"221'463","created":217,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":8860,"log_dups_size":0,"ondisk_log_size":8860,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1638400,"num_objects":25,"num_object_clones":0,"num_object_copies":75,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":25,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6527,"num_write_kb":209664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":298,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T20:31:07.308399+0000","last_change":"2023-09-23T20:29:04.138284+0000","last_active":"2023-09-23T20:31:07.308399+0000","last_peered":"2023-09-23T20:31:07.308399+0000","last_clean":"2023-09-23T20:31:07.308399+0000","last_became_active":"2023-09-23T20:29:04.137960+0000","last_became_peered":"2023-09-23T20:29:04.137960+0000","last_unstale":"2023-09-23T20:31:07.308399+0000","last_undegraded":"2023-09-23T20:31:07.308399+0000","last_fullsized":"2023-09-23T20:31:07.308399+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:18:59.548311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":264,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T20:31:07.299232+0000","last_change":"2023-09-23T20:29:46.061136+0000","last_active":"2023-09-23T20:31:07.299232+0000","last_peered":"2023-09-23T20:31:07.299232+0000","last_clean":"2023-09-23T20:31:07.299232+0000","last_became_active":"2023-09-23T20:29:46.060777+0000","last_became_peered":"2023-09-23T20:29:46.060777+0000","last_unstale":"2023-09-23T20:31:07.299232+0000","last_undegraded":"2023-09-23T20:31:07.299232+0000","last_fullsized":"2023-09-23T20:31:07.299232+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:08:43.673870+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"293'12948","reported_seq":18687,"reported_epoch":293,"state":"active+recovering+undersized+remapped","last_fresh":"2023-09-23T20:31:17.347232+0000","last_change":"2023-09-23T20:30:59.555825+0000","last_active":"2023-09-23T20:31:17.347232+0000","last_peered":"2023-09-23T20:31:17.347232+0000","last_clean":"2023-09-23T20:28:12.106078+0000","last_became_active":"2023-09-23T20:29:45.690954+0000","last_became_peered":"2023-09-23T20:29:45.690954+0000","last_unstale":"2023-09-23T20:31:17.347232+0000","last_undegraded":"2023-09-23T20:31:17.347232+0000","last_fullsized":"2023-09-23T20:29:45.501064+0000","mapping_epoch":266,"log_start":"247'6372","ondisk_log_start":"247'6372","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":6576,"log_dups_size":0,"ondisk_log_size":6576,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:52:59.514150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":10092544,"num_objects":154,"num_object_clones":0,"num_object_copies":462,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":154,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12948,"num_write_kb":419264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2818,"num_bytes_recovered":132972544,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[2,4],"avail_no_missing":["2","4"],"object_location_counts":[{"shards":"2,4","objects":154}],"blocked_by":[],"up_primary":1,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":675,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-09-23T20:31:14.389957+0000","last_change":"2023-09-23T20:28:05.872149+0000","last_active":"2023-09-23T20:31:14.389957+0000","last_peered":"2023-09-23T20:31:14.389957+0000","last_clean":"2023-09-23T20:31:14.389957+0000","last_became_active":"2023-09-23T20:27:11.961310+0000","last_became_peered":"2023-09-23T20:27:11.961310+0000","last_unstale":"2023-09-23T20:31:14.389957+0000","last_undegraded":"2023-09-23T20:31:14.389957+0000","last_fullsized":"2023-09-23T20:31:14.389957+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:28:05.872074+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:28:05.872074+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:52:32.495955+0000","scrub_duration":0.0029215980000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":166,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T20:31:00.224058+0000","last_change":"2023-09-23T20:29:49.873018+0000","last_active":"2023-09-23T20:31:00.224058+0000","last_peered":"2023-09-23T20:31:00.224058+0000","last_clean":"2023-09-23T20:31:00.224058+0000","last_became_active":"2023-09-23T20:29:46.070369+0000","last_became_peered":"2023-09-23T20:29:46.070369+0000","last_unstale":"2023-09-23T20:31:00.224058+0000","last_undegraded":"2023-09-23T20:31:00.224058+0000","last_fullsized":"2023-09-23T20:31:00.224058+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:24:13.119725+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":302,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T20:31:00.228160+0000","last_change":"2023-09-23T20:29:46.173195+0000","last_active":"2023-09-23T20:31:00.228160+0000","last_peered":"2023-09-23T20:31:00.228160+0000","last_clean":"2023-09-23T20:31:00.228160+0000","last_became_active":"2023-09-23T20:29:46.172808+0000","last_became_peered":"2023-09-23T20:29:46.172808+0000","last_unstale":"2023-09-23T20:31:00.228160+0000","last_undegraded":"2023-09-23T20:31:00.228160+0000","last_fullsized":"2023-09-23T20:31:00.228160+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:41:58.927658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.f","version":"293'13100","reported_seq":13195,"reported_epoch":293,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T20:31:17.318908+0000","last_change":"2023-09-23T20:29:29.530059+0000","last_active":"2023-09-23T20:31:17.318908+0000","last_peered":"2023-09-23T20:31:17.318908+0000","last_clean":"2023-09-23T20:29:27.332951+0000","last_became_active":"2023-09-23T20:29:29.505420+0000","last_became_peered":"2023-09-23T20:29:29.505420+0000","last_unstale":"2023-09-23T20:31:17.318908+0000","last_undegraded":"2023-09-23T20:31:17.318908+0000","last_fullsized":"2023-09-23T20:31:17.318908+0000","mapping_epoch":259,"log_start":"247'6487","ondisk_log_start":"247'6487","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":6613,"log_dups_size":0,"ondisk_log_size":6613,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:18:15.383066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":9830400,"num_objects":150,"num_object_clones":0,"num_object_copies":450,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":150,"num_objects_unfound":0,"num_objects_dirty":150,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13100,"num_write_kb":424000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,4],"avail_no_missing":["2","3","4"],"object_location_counts":[{"shards":"2,3,4","objects":150}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":516,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T20:31:07.306554+0000","last_change":"2023-09-23T20:27:31.757060+0000","last_active":"2023-09-23T20:31:07.306554+0000","last_peered":"2023-09-23T20:31:07.306554+0000","last_clean":"2023-09-23T20:31:07.306554+0000","last_became_active":"2023-09-23T20:27:11.959516+0000","last_became_peered":"2023-09-23T20:27:11.959516+0000","last_unstale":"2023-09-23T20:31:07.306554+0000","last_undegraded":"2023-09-23T20:31:07.306554+0000","last_fullsized":"2023-09-23T20:31:07.306554+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:31.756974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:31.756974+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:35:11.939335+0000","scrub_duration":0.0031802670000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":78,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-09-23T20:31:14.394997+0000","last_change":"2023-09-23T20:29:46.087097+0000","last_active":"2023-09-23T20:31:14.394997+0000","last_peered":"2023-09-23T20:31:14.394997+0000","last_clean":"2023-09-23T20:31:14.394997+0000","last_became_active":"2023-09-23T20:29:46.085908+0000","last_became_peered":"2023-09-23T20:29:46.085908+0000","last_unstale":"2023-09-23T20:31:14.394997+0000","last_undegraded":"2023-09-23T20:31:14.394997+0000","last_fullsized":"2023-09-23T20:31:14.394997+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:38:46.001432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":202,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-09-23T20:31:14.402821+0000","last_change":"2023-09-23T20:29:28.370476+0000","last_active":"2023-09-23T20:31:14.402821+0000","last_peered":"2023-09-23T20:31:14.402821+0000","last_clean":"2023-09-23T20:31:14.402821+0000","last_became_active":"2023-09-23T20:29:28.370158+0000","last_became_peered":"2023-09-23T20:29:28.370158+0000","last_unstale":"2023-09-23T20:31:14.402821+0000","last_undegraded":"2023-09-23T20:31:14.402821+0000","last_fullsized":"2023-09-23T20:31:14.402821+0000","mapping_epoch":258,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":259,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:09:47.335404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.b","version":"90'161","reported_seq":519,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T20:31:00.203821+0000","last_change":"2023-09-23T20:29:45.534549+0000","last_active":"2023-09-23T20:31:00.203821+0000","last_peered":"2023-09-23T20:31:00.203821+0000","last_clean":"2023-09-23T20:31:00.203821+0000","last_became_active":"2023-09-23T20:29:45.534070+0000","last_became_peered":"2023-09-23T20:29:45.534070+0000","last_unstale":"2023-09-23T20:31:00.203821+0000","last_undegraded":"2023-09-23T20:31:00.203821+0000","last_fullsized":"2023-09-23T20:31:00.203821+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:12.386098+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:12.386098+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0019500940000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.c","version":"291'12622","reported_seq":16118,"reported_epoch":291,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:31:13.369157+0000","last_change":"2023-09-23T20:29:59.529817+0000","last_active":"2023-09-23T20:31:13.369157+0000","last_peered":"2023-09-23T20:31:13.369157+0000","last_clean":"2023-09-23T20:28:03.999709+0000","last_became_active":"2023-09-23T20:29:04.247715+0000","last_became_peered":"2023-09-23T20:29:04.247715+0000","last_unstale":"2023-09-23T20:31:13.369157+0000","last_undegraded":"2023-09-23T20:29:59.527865+0000","last_fullsized":"2023-09-23T20:29:04.105278+0000","mapping_epoch":250,"log_start":"256'7871","ondisk_log_start":"256'7871","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":4751,"log_dups_size":0,"ondisk_log_size":4751,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:01:13.556577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":18874368,"num_objects":288,"num_object_clones":0,"num_object_copies":864,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2952,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":288,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12622,"num_write_kb":413120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1694,"num_bytes_recovered":110886912,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,5],"avail_no_missing":["3","5"],"object_location_counts":[{"shards":"3,5","objects":-2012},{"shards":"3,5,7","objects":288}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":204,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-09-23T20:31:14.408045+0000","last_change":"2023-09-23T20:29:46.061620+0000","last_active":"2023-09-23T20:31:14.408045+0000","last_peered":"2023-09-23T20:31:14.408045+0000","last_clean":"2023-09-23T20:31:14.408045+0000","last_became_active":"2023-09-23T20:29:46.061271+0000","last_became_peered":"2023-09-23T20:29:46.061271+0000","last_unstale":"2023-09-23T20:31:14.408045+0000","last_undegraded":"2023-09-23T20:31:14.408045+0000","last_fullsized":"2023-09-23T20:31:14.408045+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:24:51.409521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":288,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-09-23T20:31:14.394177+0000","last_change":"2023-09-23T20:29:46.085962+0000","last_active":"2023-09-23T20:31:14.394177+0000","last_peered":"2023-09-23T20:31:14.394177+0000","last_clean":"2023-09-23T20:31:14.394177+0000","last_became_active":"2023-09-23T20:29:46.082826+0000","last_became_peered":"2023-09-23T20:29:46.082826+0000","last_unstale":"2023-09-23T20:31:14.394177+0000","last_undegraded":"2023-09-23T20:31:14.394177+0000","last_fullsized":"2023-09-23T20:31:14.394177+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:26:17.599051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.d","version":"292'12986","reported_seq":13097,"reported_epoch":292,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:31:15.313504+0000","last_change":"2023-09-23T20:29:59.564467+0000","last_active":"2023-09-23T20:31:15.313504+0000","last_peered":"2023-09-23T20:31:15.313504+0000","last_clean":"2023-09-23T20:28:12.085747+0000","last_became_active":"2023-09-23T20:29:46.294362+0000","last_became_peered":"2023-09-23T20:29:46.294362+0000","last_unstale":"2023-09-23T20:31:15.313504+0000","last_undegraded":"2023-09-23T20:29:46.239165+0000","last_fullsized":"2023-09-23T20:29:46.032265+0000","mapping_epoch":267,"log_start":"246'6259","ondisk_log_start":"246'6259","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":6727,"log_dups_size":0,"ondisk_log_size":6727,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:38:13.504714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":15400984,"num_objects":236,"num_object_clones":0,"num_object_copies":708,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4915,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":236,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":12986,"num_write_kb":423041,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,0],"avail_no_missing":["5","0"],"object_location_counts":[{"shards":"0,5","objects":-1780}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":556,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T20:31:07.306951+0000","last_change":"2023-09-23T20:29:07.053561+0000","last_active":"2023-09-23T20:31:07.306951+0000","last_peered":"2023-09-23T20:31:07.306951+0000","last_clean":"2023-09-23T20:31:07.306951+0000","last_became_active":"2023-09-23T20:29:07.053177+0000","last_became_peered":"2023-09-23T20:29:07.053177+0000","last_unstale":"2023-09-23T20:31:07.306951+0000","last_undegraded":"2023-09-23T20:31:07.306951+0000","last_fullsized":"2023-09-23T20:31:07.306951+0000","mapping_epoch":253,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":254,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:30.727267+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:30.727267+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:04:32.661761+0000","scrub_duration":0.0052560289999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":153,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T20:31:00.222100+0000","last_change":"2023-09-23T20:29:45.529576+0000","last_active":"2023-09-23T20:31:00.222100+0000","last_peered":"2023-09-23T20:31:00.222100+0000","last_clean":"2023-09-23T20:31:00.222100+0000","last_became_active":"2023-09-23T20:29:45.528858+0000","last_became_peered":"2023-09-23T20:29:45.528858+0000","last_unstale":"2023-09-23T20:31:00.222100+0000","last_undegraded":"2023-09-23T20:31:00.222100+0000","last_fullsized":"2023-09-23T20:31:00.222100+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:42:15.589821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":101,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T20:31:00.195230+0000","last_change":"2023-09-23T20:29:46.064515+0000","last_active":"2023-09-23T20:31:00.195230+0000","last_peered":"2023-09-23T20:31:00.195230+0000","last_clean":"2023-09-23T20:31:00.195230+0000","last_became_active":"2023-09-23T20:29:46.064253+0000","last_became_peered":"2023-09-23T20:29:46.064253+0000","last_unstale":"2023-09-23T20:31:00.195230+0000","last_undegraded":"2023-09-23T20:31:00.195230+0000","last_fullsized":"2023-09-23T20:31:00.195230+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:21:20.039440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"292'9371","reported_seq":9467,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-09-23T20:31:15.030722+0000","last_change":"2023-09-23T20:30:23.834705+0000","last_active":"2023-09-23T20:31:15.030722+0000","last_peered":"2023-09-23T20:31:15.030722+0000","last_clean":"2023-09-23T20:31:15.030722+0000","last_became_active":"2023-09-23T20:28:35.532298+0000","last_became_peered":"2023-09-23T20:28:35.532298+0000","last_unstale":"2023-09-23T20:31:15.030722+0000","last_undegraded":"2023-09-23T20:31:15.030722+0000","last_fullsized":"2023-09-23T20:31:15.030722+0000","mapping_epoch":217,"log_start":"221'367","ondisk_log_start":"221'367","created":217,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":9004,"log_dups_size":0,"ondisk_log_size":9004,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":7241728,"num_objects":110,"num_object_clones":0,"num_object_copies":330,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":110,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6552,"num_write_kb":213216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":512,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-09-23T20:31:14.394688+0000","last_change":"2023-09-23T20:29:07.115982+0000","last_active":"2023-09-23T20:31:14.394688+0000","last_peered":"2023-09-23T20:31:14.394688+0000","last_clean":"2023-09-23T20:31:14.394688+0000","last_became_active":"2023-09-23T20:29:07.115647+0000","last_became_peered":"2023-09-23T20:29:07.115647+0000","last_unstale":"2023-09-23T20:31:14.394688+0000","last_undegraded":"2023-09-23T20:31:14.394688+0000","last_fullsized":"2023-09-23T20:31:14.394688+0000","mapping_epoch":253,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":254,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:32.746340+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:32.746340+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"86'119","reported_seq":594,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T20:31:00.197370+0000","last_change":"2023-09-23T20:29:45.521257+0000","last_active":"2023-09-23T20:31:00.197370+0000","last_peered":"2023-09-23T20:31:00.197370+0000","last_clean":"2023-09-23T20:31:00.197370+0000","last_became_active":"2023-09-23T20:29:45.520546+0000","last_became_peered":"2023-09-23T20:29:45.520546+0000","last_unstale":"2023-09-23T20:31:00.197370+0000","last_undegraded":"2023-09-23T20:31:00.197370+0000","last_fullsized":"2023-09-23T20:31:00.197370+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":119,"log_dups_size":0,"ondisk_log_size":119,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:58:56.085777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032705499999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":523,"num_read_kb":3503,"num_write":136,"num_write_kb":3128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":308,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T20:31:07.308997+0000","last_change":"2023-09-23T20:25:39.637160+0000","last_active":"2023-09-23T20:31:07.308997+0000","last_peered":"2023-09-23T20:31:07.308997+0000","last_clean":"2023-09-23T20:31:07.308997+0000","last_became_active":"2023-09-23T20:25:39.636860+0000","last_became_peered":"2023-09-23T20:25:39.636860+0000","last_unstale":"2023-09-23T20:31:07.308997+0000","last_undegraded":"2023-09-23T20:31:07.308997+0000","last_fullsized":"2023-09-23T20:31:07.308997+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:21:47.860063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":196,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T20:31:00.194605+0000","last_change":"2023-09-23T20:29:46.090683+0000","last_active":"2023-09-23T20:31:00.194605+0000","last_peered":"2023-09-23T20:31:00.194605+0000","last_clean":"2023-09-23T20:31:00.194605+0000","last_became_active":"2023-09-23T20:29:46.090131+0000","last_became_peered":"2023-09-23T20:29:46.090131+0000","last_unstale":"2023-09-23T20:31:00.194605+0000","last_undegraded":"2023-09-23T20:31:00.194605+0000","last_fullsized":"2023-09-23T20:31:00.194605+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:36:35.754684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.6","version":"291'9103","reported_seq":9206,"reported_epoch":291,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T20:31:13.278822+0000","last_change":"2023-09-23T20:29:29.535022+0000","last_active":"2023-09-23T20:31:13.278822+0000","last_peered":"2023-09-23T20:31:13.278822+0000","last_clean":"2023-09-23T20:29:27.350688+0000","last_became_active":"2023-09-23T20:29:29.522984+0000","last_became_peered":"2023-09-23T20:29:29.522984+0000","last_unstale":"2023-09-23T20:31:13.278822+0000","last_undegraded":"2023-09-23T20:31:13.278822+0000","last_fullsized":"2023-09-23T20:31:13.278822+0000","mapping_epoch":259,"log_start":"224'3391","ondisk_log_start":"224'3391","created":217,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":5712,"log_dups_size":0,"ondisk_log_size":5712,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":7929856,"num_objects":121,"num_object_clones":0,"num_object_copies":363,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":121,"num_objects_unfound":0,"num_objects_dirty":121,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6361,"num_write_kb":207424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,4],"avail_no_missing":["3","4","7"],"object_location_counts":[{"shards":"3,4,7","objects":121}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":526,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T20:31:07.306843+0000","last_change":"2023-09-23T20:27:55.213707+0000","last_active":"2023-09-23T20:31:07.306843+0000","last_peered":"2023-09-23T20:31:07.306843+0000","last_clean":"2023-09-23T20:31:07.306843+0000","last_became_active":"2023-09-23T20:27:55.213394+0000","last_became_peered":"2023-09-23T20:27:55.213394+0000","last_unstale":"2023-09-23T20:31:07.306843+0000","last_undegraded":"2023-09-23T20:31:07.306843+0000","last_fullsized":"2023-09-23T20:31:07.306843+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:34.821789+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:34.821789+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0042962870000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"86'106","reported_seq":986,"reported_epoch":291,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T20:31:07.306785+0000","last_change":"2023-09-23T20:29:46.078905+0000","last_active":"2023-09-23T20:31:07.306785+0000","last_peered":"2023-09-23T20:31:07.306785+0000","last_clean":"2023-09-23T20:28:10.123134+0000","last_became_active":"2023-09-23T20:29:46.078501+0000","last_became_peered":"2023-09-23T20:29:46.078501+0000","last_unstale":"2023-09-23T20:31:07.306785+0000","last_undegraded":"2023-09-23T20:29:46.032390+0000","last_fullsized":"2023-09-23T20:31:07.306785+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":106,"log_dups_size":0,"ondisk_log_size":106,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:37.015974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00099302599999999993,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1105,"num_read_kb":1689,"num_write":100,"num_write_kb":404,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":["5","0","1"],"object_location_counts":[{"shards":"0,1,5","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":284,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T20:31:07.299513+0000","last_change":"2023-09-23T20:29:46.059640+0000","last_active":"2023-09-23T20:31:07.299513+0000","last_peered":"2023-09-23T20:31:07.299513+0000","last_clean":"2023-09-23T20:31:07.299513+0000","last_became_active":"2023-09-23T20:29:46.059279+0000","last_became_peered":"2023-09-23T20:29:46.059279+0000","last_unstale":"2023-09-23T20:31:07.299513+0000","last_undegraded":"2023-09-23T20:31:07.299513+0000","last_fullsized":"2023-09-23T20:31:07.299513+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:03:31.536847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":247,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T20:31:00.197299+0000","last_change":"2023-09-23T20:29:45.533406+0000","last_active":"2023-09-23T20:31:00.197299+0000","last_peered":"2023-09-23T20:31:00.197299+0000","last_clean":"2023-09-23T20:31:00.197299+0000","last_became_active":"2023-09-23T20:29:45.526001+0000","last_became_peered":"2023-09-23T20:29:45.526001+0000","last_unstale":"2023-09-23T20:31:00.197299+0000","last_undegraded":"2023-09-23T20:31:00.197299+0000","last_fullsized":"2023-09-23T20:31:00.197299+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:43.787251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.5","version":"293'9150","reported_seq":12742,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-09-23T20:31:15.495829+0000","last_change":"2023-09-23T20:30:57.151375+0000","last_active":"2023-09-23T20:31:15.495829+0000","last_peered":"2023-09-23T20:31:15.495829+0000","last_clean":"2023-09-23T20:31:15.495829+0000","last_became_active":"2023-09-23T20:30:57.098276+0000","last_became_peered":"2023-09-23T20:30:57.098276+0000","last_unstale":"2023-09-23T20:31:15.495829+0000","last_undegraded":"2023-09-23T20:31:15.495829+0000","last_fullsized":"2023-09-23T20:31:15.495829+0000","mapping_epoch":288,"log_start":"222'1860","ondisk_log_start":"222'1860","created":217,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":7290,"log_dups_size":0,"ondisk_log_size":7290,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8355840,"num_objects":127,"num_object_clones":0,"num_object_copies":381,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":127,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6409,"num_write_kb":209184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1733,"num_bytes_recovered":70320128,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":705,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-09-23T20:31:14.389599+0000","last_change":"2023-09-23T20:29:31.389645+0000","last_active":"2023-09-23T20:31:14.389599+0000","last_peered":"2023-09-23T20:31:14.389599+0000","last_clean":"2023-09-23T20:31:14.389599+0000","last_became_active":"2023-09-23T20:29:31.389288+0000","last_became_peered":"2023-09-23T20:29:31.389288+0000","last_unstale":"2023-09-23T20:31:14.389599+0000","last_undegraded":"2023-09-23T20:31:14.389599+0000","last_fullsized":"2023-09-23T20:31:14.389599+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:35.734584+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:35.734584+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0035251380000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,2],"acting":[7,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"86'130","reported_seq":604,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-09-23T20:31:14.407957+0000","last_change":"2023-09-23T20:25:39.637458+0000","last_active":"2023-09-23T20:31:14.407957+0000","last_peered":"2023-09-23T20:31:14.407957+0000","last_clean":"2023-09-23T20:31:14.407957+0000","last_became_active":"2023-09-23T20:25:39.636773+0000","last_became_peered":"2023-09-23T20:25:39.636773+0000","last_unstale":"2023-09-23T20:31:14.407957+0000","last_undegraded":"2023-09-23T20:31:14.407957+0000","last_fullsized":"2023-09-23T20:31:14.407957+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:00:32.810564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011300049999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":512,"num_read_kb":1820,"num_write":133,"num_write_kb":1347,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":237,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T20:31:00.194274+0000","last_change":"2023-09-23T20:29:46.073394+0000","last_active":"2023-09-23T20:31:00.194274+0000","last_peered":"2023-09-23T20:31:00.194274+0000","last_clean":"2023-09-23T20:31:00.194274+0000","last_became_active":"2023-09-23T20:29:46.072067+0000","last_became_peered":"2023-09-23T20:29:46.072067+0000","last_unstale":"2023-09-23T20:31:00.194274+0000","last_undegraded":"2023-09-23T20:31:00.194274+0000","last_fullsized":"2023-09-23T20:31:00.194274+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:01:56.944614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":229,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T20:31:00.195163+0000","last_change":"2023-09-23T20:29:45.533185+0000","last_active":"2023-09-23T20:31:00.195163+0000","last_peered":"2023-09-23T20:31:00.195163+0000","last_clean":"2023-09-23T20:31:00.195163+0000","last_became_active":"2023-09-23T20:29:45.526765+0000","last_became_peered":"2023-09-23T20:29:45.526765+0000","last_unstale":"2023-09-23T20:31:00.195163+0000","last_undegraded":"2023-09-23T20:31:00.195163+0000","last_fullsized":"2023-09-23T20:31:00.195163+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:18:33.063215+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"292'9301","reported_seq":9417,"reported_epoch":292,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:31:15.049911+0000","last_change":"2023-09-23T20:29:46.386541+0000","last_active":"2023-09-23T20:31:15.049911+0000","last_peered":"2023-09-23T20:31:15.049911+0000","last_clean":"2023-09-23T20:29:27.161343+0000","last_became_active":"2023-09-23T20:29:46.365588+0000","last_became_peered":"2023-09-23T20:29:46.365588+0000","last_unstale":"2023-09-23T20:31:15.049911+0000","last_undegraded":"2023-09-23T20:29:46.092299+0000","last_fullsized":"2023-09-23T20:29:46.032978+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":9301,"log_dups_size":0,"ondisk_log_size":9301,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":6979584,"num_objects":106,"num_object_clones":0,"num_object_copies":318,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3300,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":106,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6472,"num_write_kb":210528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3],"avail_no_missing":["5","3","4"],"object_location_counts":[{"shards":"3,4,5","objects":117},{"shards":"3,5","objects":-1744}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"86'194","reported_seq":940,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T20:31:07.307728+0000","last_change":"2023-09-23T20:25:40.618665+0000","last_active":"2023-09-23T20:31:07.307728+0000","last_peered":"2023-09-23T20:31:07.307728+0000","last_clean":"2023-09-23T20:31:07.307728+0000","last_became_active":"2023-09-23T20:25:40.618087+0000","last_became_peered":"2023-09-23T20:25:40.618087+0000","last_unstale":"2023-09-23T20:31:07.307728+0000","last_undegraded":"2023-09-23T20:31:07.307728+0000","last_fullsized":"2023-09-23T20:31:07.307728+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":194,"log_dups_size":0,"ondisk_log_size":194,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:07:16.114968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00037995999999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1025,"num_read_kb":1201,"num_write":173,"num_write_kb":1276,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":716,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-09-23T20:31:14.389171+0000","last_change":"2023-09-23T20:29:07.042758+0000","last_active":"2023-09-23T20:31:14.389171+0000","last_peered":"2023-09-23T20:31:14.389171+0000","last_clean":"2023-09-23T20:31:14.389171+0000","last_became_active":"2023-09-23T20:29:07.042407+0000","last_became_peered":"2023-09-23T20:29:07.042407+0000","last_unstale":"2023-09-23T20:31:14.389171+0000","last_undegraded":"2023-09-23T20:31:14.389171+0000","last_fullsized":"2023-09-23T20:31:14.389171+0000","mapping_epoch":253,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":254,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:32.801168+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:32.801168+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0096333110000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":156,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T20:31:00.223492+0000","last_change":"2023-09-23T20:29:04.216400+0000","last_active":"2023-09-23T20:31:00.223492+0000","last_peered":"2023-09-23T20:31:00.223492+0000","last_clean":"2023-09-23T20:31:00.223492+0000","last_became_active":"2023-09-23T20:29:04.216069+0000","last_became_peered":"2023-09-23T20:29:04.216069+0000","last_unstale":"2023-09-23T20:31:00.223492+0000","last_undegraded":"2023-09-23T20:31:00.223492+0000","last_fullsized":"2023-09-23T20:31:00.223492+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:27:37.433035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":62,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-09-23T20:31:14.394736+0000","last_change":"2023-09-23T20:29:28.376036+0000","last_active":"2023-09-23T20:31:14.394736+0000","last_peered":"2023-09-23T20:31:14.394736+0000","last_clean":"2023-09-23T20:31:14.394736+0000","last_became_active":"2023-09-23T20:29:28.375712+0000","last_became_peered":"2023-09-23T20:29:28.375712+0000","last_unstale":"2023-09-23T20:31:14.394736+0000","last_undegraded":"2023-09-23T20:31:14.394736+0000","last_fullsized":"2023-09-23T20:31:14.394736+0000","mapping_epoch":258,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":259,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:23:50.722625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.4","version":"291'9199","reported_seq":9300,"reported_epoch":291,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:31:13.356461+0000","last_change":"2023-09-23T20:29:46.186959+0000","last_active":"2023-09-23T20:31:13.356461+0000","last_peered":"2023-09-23T20:31:13.356461+0000","last_clean":"2023-09-23T20:28:12.083147+0000","last_became_active":"2023-09-23T20:29:46.162334+0000","last_became_peered":"2023-09-23T20:29:46.162334+0000","last_unstale":"2023-09-23T20:31:13.356461+0000","last_undegraded":"2023-09-23T20:29:46.054138+0000","last_fullsized":"2023-09-23T20:29:46.043728+0000","mapping_epoch":267,"log_start":"225'3464","ondisk_log_start":"225'3464","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":5735,"log_dups_size":0,"ondisk_log_size":5735,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":12517376,"num_objects":191,"num_object_clones":0,"num_object_copies":573,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2427,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":191,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6435,"num_write_kb":212032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[3,5],"avail_no_missing":["3","5"],"object_location_counts":[{"shards":"3,5","objects":-852}],"blocked_by":[],"up_primary":1,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":519,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T20:31:00.203632+0000","last_change":"2023-09-23T20:29:46.072303+0000","last_active":"2023-09-23T20:31:00.203632+0000","last_peered":"2023-09-23T20:31:00.203632+0000","last_clean":"2023-09-23T20:31:00.203632+0000","last_became_active":"2023-09-23T20:29:46.071938+0000","last_became_peered":"2023-09-23T20:29:46.071938+0000","last_unstale":"2023-09-23T20:31:00.203632+0000","last_undegraded":"2023-09-23T20:31:00.203632+0000","last_fullsized":"2023-09-23T20:31:00.203632+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:12.386098+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:12.386098+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,5],"acting":[1,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"86'80","reported_seq":556,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-09-23T20:31:14.389493+0000","last_change":"2023-09-23T20:29:45.536796+0000","last_active":"2023-09-23T20:31:14.389493+0000","last_peered":"2023-09-23T20:31:14.389493+0000","last_clean":"2023-09-23T20:31:14.389493+0000","last_became_active":"2023-09-23T20:29:45.536439+0000","last_became_peered":"2023-09-23T20:29:45.536439+0000","last_unstale":"2023-09-23T20:31:14.389493+0000","last_undegraded":"2023-09-23T20:31:14.389493+0000","last_fullsized":"2023-09-23T20:31:14.389493+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":80,"log_dups_size":0,"ondisk_log_size":80,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:30:57.793132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035115,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":395,"num_read_kb":1370,"num_write":66,"num_write_kb":1061,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":265,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T20:31:07.301729+0000","last_change":"2023-09-23T20:29:28.360680+0000","last_active":"2023-09-23T20:31:07.301729+0000","last_peered":"2023-09-23T20:31:07.301729+0000","last_clean":"2023-09-23T20:31:07.301729+0000","last_became_active":"2023-09-23T20:29:28.360366+0000","last_became_peered":"2023-09-23T20:29:28.360366+0000","last_unstale":"2023-09-23T20:31:07.301729+0000","last_undegraded":"2023-09-23T20:31:07.301729+0000","last_fullsized":"2023-09-23T20:31:07.301729+0000","mapping_epoch":258,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":259,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:02:40.594828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":287,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T20:31:07.300384+0000","last_change":"2023-09-23T20:29:45.690815+0000","last_active":"2023-09-23T20:31:07.300384+0000","last_peered":"2023-09-23T20:31:07.300384+0000","last_clean":"2023-09-23T20:31:07.300384+0000","last_became_active":"2023-09-23T20:29:45.690478+0000","last_became_peered":"2023-09-23T20:29:45.690478+0000","last_unstale":"2023-09-23T20:31:07.300384+0000","last_undegraded":"2023-09-23T20:31:07.300384+0000","last_fullsized":"2023-09-23T20:31:07.300384+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:38.288193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"293'9108","reported_seq":14252,"reported_epoch":293,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:31:15.573678+0000","last_change":"2023-09-23T20:29:59.519844+0000","last_active":"2023-09-23T20:31:15.573678+0000","last_peered":"2023-09-23T20:31:15.573678+0000","last_clean":"2023-09-23T20:28:04.011715+0000","last_became_active":"2023-09-23T20:29:46.206627+0000","last_became_peered":"2023-09-23T20:29:46.206627+0000","last_unstale":"2023-09-23T20:31:15.573678+0000","last_undegraded":"2023-09-23T20:29:59.519468+0000","last_fullsized":"2023-09-23T20:29:46.046256+0000","mapping_epoch":267,"log_start":"222'1827","ondisk_log_start":"222'1827","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":7281,"log_dups_size":0,"ondisk_log_size":7281,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":6553600,"num_objects":100,"num_object_clones":0,"num_object_copies":300,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1729,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6354,"num_write_kb":206528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2160,"num_bytes_recovered":141557760,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[6,2],"avail_no_missing":["6","2"],"object_location_counts":[{"shards":"2,6","objects":-228},{"shards":"2,6,7","objects":102}],"blocked_by":[],"up_primary":1,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":505,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-09-23T20:31:14.394465+0000","last_change":"2023-09-23T20:29:07.113747+0000","last_active":"2023-09-23T20:31:14.394465+0000","last_peered":"2023-09-23T20:31:14.394465+0000","last_clean":"2023-09-23T20:31:14.394465+0000","last_became_active":"2023-09-23T20:29:07.113428+0000","last_became_peered":"2023-09-23T20:29:07.113428+0000","last_unstale":"2023-09-23T20:31:14.394465+0000","last_undegraded":"2023-09-23T20:31:14.394465+0000","last_fullsized":"2023-09-23T20:31:14.394465+0000","mapping_epoch":253,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":254,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:31.769333+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:31.769333+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"86'102","reported_seq":821,"reported_epoch":292,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:31:14.388906+0000","last_change":"2023-09-23T20:29:46.060111+0000","last_active":"2023-09-23T20:31:14.388906+0000","last_peered":"2023-09-23T20:31:14.388906+0000","last_clean":"2023-09-23T20:29:43.509457+0000","last_became_active":"2023-09-23T20:29:46.059787+0000","last_became_peered":"2023-09-23T20:29:46.059787+0000","last_unstale":"2023-09-23T20:31:14.388906+0000","last_undegraded":"2023-09-23T20:29:46.049994+0000","last_fullsized":"2023-09-23T20:29:46.048495+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":260,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":102,"log_dups_size":0,"ondisk_log_size":102,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:53:45.933578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030656500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":963,"num_read_kb":1234,"num_write":60,"num_write_kb":225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0],"avail_no_missing":["7","0","4"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":213,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T20:31:00.197439+0000","last_change":"2023-09-23T20:29:45.538283+0000","last_active":"2023-09-23T20:31:00.197439+0000","last_peered":"2023-09-23T20:31:00.197439+0000","last_clean":"2023-09-23T20:31:00.197439+0000","last_became_active":"2023-09-23T20:29:45.537922+0000","last_became_peered":"2023-09-23T20:29:45.537922+0000","last_unstale":"2023-09-23T20:31:00.197439+0000","last_undegraded":"2023-09-23T20:31:00.197439+0000","last_fullsized":"2023-09-23T20:31:00.197439+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:30:52.397836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":224,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T20:31:07.299441+0000","last_change":"2023-09-23T20:20:14.888874+0000","last_active":"2023-09-23T20:31:07.299441+0000","last_peered":"2023-09-23T20:31:07.299441+0000","last_clean":"2023-09-23T20:31:07.299441+0000","last_became_active":"2023-09-23T20:20:14.887724+0000","last_became_peered":"2023-09-23T20:20:14.887724+0000","last_unstale":"2023-09-23T20:31:07.299441+0000","last_undegraded":"2023-09-23T20:31:07.299441+0000","last_fullsized":"2023-09-23T20:31:07.299441+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:09:24.510326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"291'9357","reported_seq":9463,"reported_epoch":291,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:31:13.239502+0000","last_change":"2023-09-23T20:29:29.642590+0000","last_active":"2023-09-23T20:31:13.239502+0000","last_peered":"2023-09-23T20:31:13.239502+0000","last_clean":"2023-09-23T20:29:27.208118+0000","last_became_active":"2023-09-23T20:29:29.628868+0000","last_became_peered":"2023-09-23T20:29:29.628868+0000","last_unstale":"2023-09-23T20:31:13.239502+0000","last_undegraded":"2023-09-23T20:29:29.393326+0000","last_fullsized":"2023-09-23T20:29:29.348340+0000","mapping_epoch":259,"log_start":"226'3727","ondisk_log_start":"226'3727","created":217,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":5630,"log_dups_size":0,"ondisk_log_size":5630,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":3932160,"num_objects":60,"num_object_clones":0,"num_object_copies":180,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3393,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":60,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6546,"num_write_kb":211392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2],"avail_no_missing":["3","2","4"],"object_location_counts":[{"shards":"2,3","objects":-2273},{"shards":"2,3,4","objects":150}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":462,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-09-23T20:31:14.402735+0000","last_change":"2023-09-23T20:29:28.372908+0000","last_active":"2023-09-23T20:31:14.402735+0000","last_peered":"2023-09-23T20:31:14.402735+0000","last_clean":"2023-09-23T20:31:14.402735+0000","last_became_active":"2023-09-23T20:29:28.372652+0000","last_became_peered":"2023-09-23T20:29:28.372652+0000","last_unstale":"2023-09-23T20:31:14.402735+0000","last_undegraded":"2023-09-23T20:31:14.402735+0000","last_fullsized":"2023-09-23T20:31:14.402735+0000","mapping_epoch":258,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":259,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:12.809933+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:12.809933+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,5],"acting":[2,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":142,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T20:31:00.226066+0000","last_change":"2023-09-23T20:25:03.007736+0000","last_active":"2023-09-23T20:31:00.226066+0000","last_peered":"2023-09-23T20:31:00.226066+0000","last_clean":"2023-09-23T20:31:00.226066+0000","last_became_active":"2023-09-23T20:25:02.973689+0000","last_became_peered":"2023-09-23T20:25:02.973689+0000","last_unstale":"2023-09-23T20:31:00.226066+0000","last_undegraded":"2023-09-23T20:31:00.226066+0000","last_fullsized":"2023-09-23T20:31:00.226066+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:37:25.643774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"86'128","reported_seq":646,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T20:31:00.195518+0000","last_change":"2023-09-23T20:30:45.008506+0000","last_active":"2023-09-23T20:31:00.195518+0000","last_peered":"2023-09-23T20:31:00.195518+0000","last_clean":"2023-09-23T20:31:00.195518+0000","last_became_active":"2023-09-23T20:29:46.071848+0000","last_became_peered":"2023-09-23T20:29:46.071848+0000","last_unstale":"2023-09-23T20:31:00.195518+0000","last_undegraded":"2023-09-23T20:31:00.195518+0000","last_fullsized":"2023-09-23T20:31:00.195518+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":283,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:44:18.031530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000293202,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":579,"num_read_kb":1473,"num_write":96,"num_write_kb":1062,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":139,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T20:31:03.385175+0000","last_change":"2023-09-23T20:25:02.971740+0000","last_active":"2023-09-23T20:31:03.385175+0000","last_peered":"2023-09-23T20:31:03.385175+0000","last_clean":"2023-09-23T20:31:03.385175+0000","last_became_active":"2023-09-23T20:25:02.970412+0000","last_became_peered":"2023-09-23T20:25:02.970412+0000","last_unstale":"2023-09-23T20:31:03.385175+0000","last_undegraded":"2023-09-23T20:31:03.385175+0000","last_fullsized":"2023-09-23T20:31:03.385175+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:53:17.247216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"293'9214","reported_seq":9647,"reported_epoch":293,"state":"active+recovery_wait","last_fresh":"2023-09-23T20:31:15.610860+0000","last_change":"2023-09-23T20:30:30.444650+0000","last_active":"2023-09-23T20:31:15.610860+0000","last_peered":"2023-09-23T20:31:15.610860+0000","last_clean":"2023-09-23T20:28:04.010682+0000","last_became_active":"2023-09-23T20:29:47.069776+0000","last_became_peered":"2023-09-23T20:29:47.069776+0000","last_unstale":"2023-09-23T20:31:15.610860+0000","last_undegraded":"2023-09-23T20:31:15.610860+0000","last_fullsized":"2023-09-23T20:31:15.610860+0000","mapping_epoch":268,"log_start":"222'2084","ondisk_log_start":"222'2084","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":7130,"log_dups_size":0,"ondisk_log_size":7130,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":2719744,"num_objects":41,"num_object_clones":0,"num_object_copies":123,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":41,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6445,"num_write_kb":207584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":149,"num_bytes_recovered":9797632,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":["7","2","6"],"object_location_counts":[{"shards":"2,6,7","objects":41}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.4","version":"90'161","reported_seq":538,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T20:31:00.227925+0000","last_change":"2023-09-23T20:28:03.553294+0000","last_active":"2023-09-23T20:31:00.227925+0000","last_peered":"2023-09-23T20:31:00.227925+0000","last_clean":"2023-09-23T20:31:00.227925+0000","last_became_active":"2023-09-23T20:27:55.212201+0000","last_became_peered":"2023-09-23T20:27:55.212201+0000","last_unstale":"2023-09-23T20:31:00.227925+0000","last_undegraded":"2023-09-23T20:31:00.227925+0000","last_fullsized":"2023-09-23T20:31:00.227925+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:28:03.553106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:28:03.553106+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:24:06.706940+0000","scrub_duration":0.011176864,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"86'117","reported_seq":607,"reported_epoch":293,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:31:15.382448+0000","last_change":"2023-09-23T20:29:59.542691+0000","last_active":"2023-09-23T20:31:15.382448+0000","last_peered":"2023-09-23T20:31:15.382448+0000","last_clean":"2023-09-23T20:13:53.392235+0000","last_became_active":"2023-09-23T20:29:05.138184+0000","last_became_peered":"2023-09-23T20:29:05.138184+0000","last_unstale":"2023-09-23T20:31:15.382448+0000","last_undegraded":"2023-09-23T20:29:59.542326+0000","last_fullsized":"2023-09-23T20:29:05.108660+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:50:02.080933+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046891400000000002,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":893,"num_read_kb":922,"num_write":82,"num_write_kb":4193,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[2,7],"avail_no_missing":["2","3","7"],"object_location_counts":[{"shards":"2,3,7","objects":1}],"blocked_by":[],"up_primary":6,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":274,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T20:31:07.300317+0000","last_change":"2023-09-23T20:29:46.053198+0000","last_active":"2023-09-23T20:31:07.300317+0000","last_peered":"2023-09-23T20:31:07.300317+0000","last_clean":"2023-09-23T20:31:07.300317+0000","last_became_active":"2023-09-23T20:29:46.052866+0000","last_became_peered":"2023-09-23T20:29:46.052866+0000","last_unstale":"2023-09-23T20:31:07.300317+0000","last_undegraded":"2023-09-23T20:31:07.300317+0000","last_fullsized":"2023-09-23T20:31:07.300317+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:30:18.857140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":249,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T20:31:07.306902+0000","last_change":"2023-09-23T20:29:28.367722+0000","last_active":"2023-09-23T20:31:07.306902+0000","last_peered":"2023-09-23T20:31:07.306902+0000","last_clean":"2023-09-23T20:31:07.306902+0000","last_became_active":"2023-09-23T20:29:28.367404+0000","last_became_peered":"2023-09-23T20:29:28.367404+0000","last_unstale":"2023-09-23T20:31:07.306902+0000","last_undegraded":"2023-09-23T20:31:07.306902+0000","last_fullsized":"2023-09-23T20:31:07.306902+0000","mapping_epoch":258,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":259,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:02:51.886571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":246,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T20:31:07.299371+0000","last_change":"2023-09-23T20:29:46.061960+0000","last_active":"2023-09-23T20:31:07.299371+0000","last_peered":"2023-09-23T20:31:07.299371+0000","last_clean":"2023-09-23T20:31:07.299371+0000","last_became_active":"2023-09-23T20:29:46.061601+0000","last_became_peered":"2023-09-23T20:29:46.061601+0000","last_unstale":"2023-09-23T20:31:07.299371+0000","last_undegraded":"2023-09-23T20:31:07.299371+0000","last_fullsized":"2023-09-23T20:31:07.299371+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:15:52.620692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":144,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-09-23T20:31:14.402551+0000","last_change":"2023-09-23T20:29:45.538314+0000","last_active":"2023-09-23T20:31:14.402551+0000","last_peered":"2023-09-23T20:31:14.402551+0000","last_clean":"2023-09-23T20:31:14.402551+0000","last_became_active":"2023-09-23T20:29:45.536847+0000","last_became_peered":"2023-09-23T20:29:45.536847+0000","last_unstale":"2023-09-23T20:31:14.402551+0000","last_undegraded":"2023-09-23T20:31:14.402551+0000","last_fullsized":"2023-09-23T20:31:14.402551+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:51.948721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"6.19","version":"292'9140","reported_seq":9235,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-09-23T20:31:14.968966+0000","last_change":"2023-09-23T20:30:29.820384+0000","last_active":"2023-09-23T20:31:14.968966+0000","last_peered":"2023-09-23T20:31:14.968966+0000","last_clean":"2023-09-23T20:31:14.968966+0000","last_became_active":"2023-09-23T20:28:35.567626+0000","last_became_peered":"2023-09-23T20:28:35.567626+0000","last_unstale":"2023-09-23T20:31:14.968966+0000","last_undegraded":"2023-09-23T20:31:14.968966+0000","last_fullsized":"2023-09-23T20:31:14.968966+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":242,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":9140,"log_dups_size":0,"ondisk_log_size":9140,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":14024704,"num_objects":214,"num_object_clones":0,"num_object_copies":642,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":214,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6344,"num_write_kb":209856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":287,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T20:31:07.308299+0000","last_change":"2023-09-23T20:29:59.578082+0000","last_active":"2023-09-23T20:31:07.308299+0000","last_peered":"2023-09-23T20:31:07.308299+0000","last_clean":"2023-09-23T20:31:07.308299+0000","last_became_active":"2023-09-23T20:29:46.072626+0000","last_became_peered":"2023-09-23T20:29:46.072626+0000","last_unstale":"2023-09-23T20:31:07.308299+0000","last_undegraded":"2023-09-23T20:31:07.308299+0000","last_fullsized":"2023-09-23T20:31:07.308299+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:24:14.704823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":298,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T20:31:00.223934+0000","last_change":"2023-09-23T20:29:46.070732+0000","last_active":"2023-09-23T20:31:00.223934+0000","last_peered":"2023-09-23T20:31:00.223934+0000","last_clean":"2023-09-23T20:31:00.223934+0000","last_became_active":"2023-09-23T20:29:46.069861+0000","last_became_peered":"2023-09-23T20:29:46.069861+0000","last_unstale":"2023-09-23T20:31:00.223934+0000","last_undegraded":"2023-09-23T20:31:00.223934+0000","last_fullsized":"2023-09-23T20:31:00.223934+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:33:00.813261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.18","version":"293'9240","reported_seq":9334,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-09-23T20:31:15.674987+0000","last_change":"2023-09-23T20:28:35.587755+0000","last_active":"2023-09-23T20:31:15.674987+0000","last_peered":"2023-09-23T20:31:15.674987+0000","last_clean":"2023-09-23T20:31:15.674987+0000","last_became_active":"2023-09-23T20:28:35.585502+0000","last_became_peered":"2023-09-23T20:28:35.585502+0000","last_unstale":"2023-09-23T20:31:15.674987+0000","last_undegraded":"2023-09-23T20:31:15.674987+0000","last_fullsized":"2023-09-23T20:31:15.674987+0000","mapping_epoch":242,"log_start":"222'2022","ondisk_log_start":"222'2022","created":242,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":7218,"log_dups_size":0,"ondisk_log_size":7218,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":5111808,"num_objects":78,"num_object_clones":0,"num_object_copies":234,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":78,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6454,"num_write_kb":209024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":295,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T20:31:07.308856+0000","last_change":"2023-09-23T20:29:46.242242+0000","last_active":"2023-09-23T20:31:07.308856+0000","last_peered":"2023-09-23T20:31:07.308856+0000","last_clean":"2023-09-23T20:31:07.308856+0000","last_became_active":"2023-09-23T20:29:46.241216+0000","last_became_peered":"2023-09-23T20:29:46.241216+0000","last_unstale":"2023-09-23T20:31:07.308856+0000","last_undegraded":"2023-09-23T20:31:07.308856+0000","last_fullsized":"2023-09-23T20:31:07.308856+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:12:48.530720+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":206,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T20:31:00.223582+0000","last_change":"2023-09-23T20:29:28.353700+0000","last_active":"2023-09-23T20:31:00.223582+0000","last_peered":"2023-09-23T20:31:00.223582+0000","last_clean":"2023-09-23T20:31:00.223582+0000","last_became_active":"2023-09-23T20:29:28.353357+0000","last_became_peered":"2023-09-23T20:29:28.353357+0000","last_unstale":"2023-09-23T20:31:00.223582+0000","last_undegraded":"2023-09-23T20:31:00.223582+0000","last_fullsized":"2023-09-23T20:31:00.223582+0000","mapping_epoch":258,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":259,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:12.077050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":172,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T20:31:03.387395+0000","last_change":"2023-09-23T20:29:04.138540+0000","last_active":"2023-09-23T20:31:03.387395+0000","last_peered":"2023-09-23T20:31:03.387395+0000","last_clean":"2023-09-23T20:31:03.387395+0000","last_became_active":"2023-09-23T20:29:04.138182+0000","last_became_peered":"2023-09-23T20:29:04.138182+0000","last_unstale":"2023-09-23T20:31:03.387395+0000","last_undegraded":"2023-09-23T20:31:03.387395+0000","last_fullsized":"2023-09-23T20:31:03.387395+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:26.098284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":147,"reported_epoch":290,"state":"active+clean","last_fresh":"2023-09-23T20:31:00.221907+0000","last_change":"2023-09-23T20:25:39.622819+0000","last_active":"2023-09-23T20:31:00.221907+0000","last_peered":"2023-09-23T20:31:00.221907+0000","last_clean":"2023-09-23T20:31:00.221907+0000","last_became_active":"2023-09-23T20:25:39.622484+0000","last_became_peered":"2023-09-23T20:25:39.622484+0000","last_unstale":"2023-09-23T20:31:00.221907+0000","last_undegraded":"2023-09-23T20:31:00.221907+0000","last_fullsized":"2023-09-23T20:31:00.221907+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:51:53.973261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":272,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T20:31:07.300664+0000","last_change":"2023-09-23T20:25:39.668882+0000","last_active":"2023-09-23T20:31:07.300664+0000","last_peered":"2023-09-23T20:31:07.300664+0000","last_clean":"2023-09-23T20:31:07.300664+0000","last_became_active":"2023-09-23T20:25:39.637128+0000","last_became_peered":"2023-09-23T20:25:39.637128+0000","last_unstale":"2023-09-23T20:31:07.300664+0000","last_undegraded":"2023-09-23T20:31:07.300664+0000","last_fullsized":"2023-09-23T20:31:07.300664+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:56.406298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":308,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-09-23T20:31:14.393821+0000","last_change":"2023-09-23T20:29:46.082412+0000","last_active":"2023-09-23T20:31:14.393821+0000","last_peered":"2023-09-23T20:31:14.393821+0000","last_clean":"2023-09-23T20:31:14.393821+0000","last_became_active":"2023-09-23T20:29:46.082028+0000","last_became_peered":"2023-09-23T20:29:46.082028+0000","last_unstale":"2023-09-23T20:31:14.393821+0000","last_undegraded":"2023-09-23T20:31:14.393821+0000","last_fullsized":"2023-09-23T20:31:14.393821+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:48:07.017969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":26,"stat_sum":{"num_bytes":255524888,"num_objects":3900,"num_object_clones":0,"num_object_copies":11700,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":29228,"num_objects_misplaced":462,"num_objects_unfound":0,"num_objects_dirty":3900,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":205574,"num_write_kb":6703105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22801,"num_bytes_recovered":1150091264,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2751406080,"data_stored":2751397936,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":183692,"ondisk_log_size":183692,"up":78,"acting":67,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":6376,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":122880,"data_stored":21392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26691,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":740547,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":214,"num_read_kb":401,"num_write":106,"num_write_kb":1063,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1433573,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2437120,"data_stored":2426464,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3381,"ondisk_log_size":3381,"up":63,"acting":63,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":26,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":17,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6082,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":22,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201521,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":676616,"kb_used_data":447296,"kb_used_omap":0,"kb_used_meta":229248,"kb_avail":93068536,"statfs":{"total":95995035648,"available":95302180864,"internally_reserved":0,"allocated":458031104,"data_stored":457097160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":234749952},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:30:34 2023","interfaces":[{"interface":"back","average":{"1min":1.1459999999999999,"5min":1.96,"15min":1.163},"min":{"1min":0.25800000000000001,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":3.4790000000000001,"5min":48.798999999999999,"15min":48.798999999999999},"last":3.9750000000000001},{"interface":"front","average":{"1min":2.617,"5min":2.4209999999999998,"15min":1.3300000000000001},"min":{"1min":0.308,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":7.8179999999999996,"5min":48.247,"15min":48.247},"last":0.40200000000000002}]},{"osd":1,"last update":"Sat Sep 23 20:30:47 2023","interfaces":[{"interface":"back","average":{"1min":1.762,"5min":1.762,"15min":1.762},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":9.7370000000000001,"5min":9.7370000000000001,"15min":9.7370000000000001},"last":3.6240000000000001},{"interface":"front","average":{"1min":1.9810000000000001,"5min":1.9810000000000001,"15min":1.9810000000000001},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":5.7679999999999998,"5min":5.7679999999999998,"15min":5.7679999999999998},"last":1.8280000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:30:38 2023","interfaces":[{"interface":"back","average":{"1min":2.9300000000000002,"5min":2.0249999999999999,"15min":1.2210000000000001},"min":{"1min":0.35899999999999999,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":8.2509999999999994,"5min":11.372999999999999,"15min":11.372999999999999},"last":0.68100000000000005},{"interface":"front","average":{"1min":2.9729999999999999,"5min":2.2709999999999999,"15min":1.349},"min":{"1min":0.33500000000000002,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":14.993,"5min":14.993,"15min":14.993},"last":1.3200000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:31:02 2023","interfaces":[{"interface":"back","average":{"1min":2.5059999999999998,"5min":2.0710000000000002,"15min":1.377},"min":{"1min":0.34000000000000002,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":11.015000000000001,"5min":11.015000000000001,"15min":11.015000000000001},"last":0.38300000000000001},{"interface":"front","average":{"1min":2.7970000000000002,"5min":1.9510000000000001,"15min":1.3},"min":{"1min":0.38,"5min":0.33100000000000002,"15min":0.27600000000000002},"max":{"1min":11.031000000000001,"5min":11.031000000000001,"15min":11.031000000000001},"last":0.60199999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:31:02 2023","interfaces":[{"interface":"back","average":{"1min":0.73699999999999999,"5min":1.583,"15min":1.008},"min":{"1min":0.29399999999999998,"5min":0.27100000000000002,"15min":0.27000000000000002},"max":{"1min":5.7220000000000004,"5min":15.177,"15min":15.177},"last":0.32600000000000001},{"interface":"front","average":{"1min":0.79900000000000004,"5min":1.3560000000000001,"15min":1.127},"min":{"1min":0.28399999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":4.2800000000000002,"5min":11.122,"15min":11.122},"last":0.54600000000000004}]},{"osd":5,"last update":"Sat Sep 23 20:31:02 2023","interfaces":[{"interface":"back","average":{"1min":0.76200000000000001,"5min":1.4159999999999999,"15min":1.1040000000000001},"min":{"1min":0.30599999999999999,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":2.8730000000000002,"5min":13.241,"15min":13.241},"last":0.34499999999999997},{"interface":"front","average":{"1min":0.79400000000000004,"5min":1.7230000000000001,"15min":1.2070000000000001},"min":{"1min":0.32200000000000001,"5min":0.315,"15min":0.315},"max":{"1min":5.7869999999999999,"5min":18.831,"15min":18.831},"last":0.37}]},{"osd":6,"last update":"Sat Sep 23 20:31:02 2023","interfaces":[{"interface":"back","average":{"1min":0.82399999999999995,"5min":1.5580000000000001,"15min":1.1060000000000001},"min":{"1min":0.34399999999999997,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":5.7629999999999999,"5min":12.298999999999999,"15min":13.185},"last":0.41299999999999998},{"interface":"front","average":{"1min":0.93600000000000005,"5min":1.302,"15min":1.048},"min":{"1min":0.36799999999999999,"5min":0.35399999999999998,"15min":0.29899999999999999},"max":{"1min":4.4160000000000004,"5min":8.0069999999999997,"15min":9.9120000000000008},"last":0.58199999999999996}]}]},{"osd":6,"up_from":47,"seq":201863463157,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":313268,"kb_used_data":115116,"kb_used_omap":0,"kb_used_meta":198144,"kb_avail":93431884,"statfs":{"total":95995035648,"available":95674249216,"internally_reserved":0,"allocated":117878784,"data_stored":116956717,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":202899456},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:30:34 2023","interfaces":[{"interface":"back","average":{"1min":1.446,"5min":1.413,"15min":0.997},"min":{"1min":0.23300000000000001,"5min":0.221,"15min":0.221},"max":{"1min":7.609,"5min":8.4209999999999994,"15min":8.4209999999999994},"last":0.40300000000000002},{"interface":"front","average":{"1min":2.1619999999999999,"5min":1.7949999999999999,"15min":1.2310000000000001},"min":{"1min":0.28499999999999998,"5min":0.25,"15min":0.25},"max":{"1min":8.6229999999999993,"5min":8.6760000000000002,"15min":12.512},"last":0.28799999999999998}]},{"osd":1,"last update":"Sat Sep 23 20:30:46 2023","interfaces":[{"interface":"back","average":{"1min":2.7509999999999999,"5min":2.7509999999999999,"15min":2.7509999999999999},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":14.1,"5min":14.1,"15min":14.1},"last":1.536},{"interface":"front","average":{"1min":3.4870000000000001,"5min":3.4870000000000001,"15min":3.4870000000000001},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":14.266999999999999,"5min":14.266999999999999,"15min":14.266999999999999},"last":3.0790000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:30:34 2023","interfaces":[{"interface":"back","average":{"1min":2.7519999999999998,"5min":2.2440000000000002,"15min":1.304},"min":{"1min":0.35099999999999998,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":7.8700000000000001,"5min":10.631,"15min":13.856},"last":2.9199999999999999},{"interface":"front","average":{"1min":2.5110000000000001,"5min":2.056,"15min":1.2190000000000001},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":7.891,"5min":8.3729999999999993,"15min":8.3729999999999993},"last":2.8719999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:31:11 2023","interfaces":[{"interface":"back","average":{"1min":2.9420000000000002,"5min":2.0099999999999998,"15min":1.3859999999999999},"min":{"1min":0.51600000000000001,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":6.1829999999999998,"5min":8.2989999999999995,"15min":9.4239999999999995},"last":0.32800000000000001},{"interface":"front","average":{"1min":3.0910000000000002,"5min":2.21,"15min":1.4330000000000001},"min":{"1min":0.40600000000000003,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":7.6609999999999996,"5min":8.3770000000000007,"15min":8.6050000000000004},"last":0.39000000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:30:53 2023","interfaces":[{"interface":"back","average":{"1min":0.44,"5min":1.0349999999999999,"15min":0.85299999999999998},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":1.599,"5min":10.173,"15min":10.173},"last":1.6579999999999999},{"interface":"front","average":{"1min":1.032,"5min":1.3779999999999999,"15min":0.98199999999999998},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":3.363,"5min":8.3249999999999993,"15min":8.3249999999999993},"last":0.35899999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:30:53 2023","interfaces":[{"interface":"back","average":{"1min":0.46400000000000002,"5min":0.91100000000000003,"15min":0.89800000000000002},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":1.2010000000000001,"5min":8.4499999999999993,"15min":14.085000000000001},"last":0.55400000000000005},{"interface":"front","average":{"1min":0.98099999999999998,"5min":1.76,"15min":1.236},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.29499999999999998},"max":{"1min":3.294,"5min":18.192,"15min":18.192},"last":0.59799999999999998}]},{"osd":7,"last update":"Sat Sep 23 20:31:11 2023","interfaces":[{"interface":"back","average":{"1min":0.45300000000000001,"5min":1.27,"15min":1.0309999999999999},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.754,"5min":8.5060000000000002,"15min":10.217000000000001},"last":0.438},{"interface":"front","average":{"1min":1.0669999999999999,"5min":1.508,"15min":1.163},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":6.4160000000000004,"5min":8.407,"15min":9.7360000000000007},"last":0.56999999999999995}]}]},{"osd":1,"up_from":266,"seq":1142461300755,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":933808,"kb_used_data":817896,"kb_used_omap":8,"kb_used_meta":115895,"kb_avail":92811344,"statfs":{"total":95995035648,"available":95038816256,"internally_reserved":0,"allocated":837525504,"data_stored":835959209,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8206,"internal_metadata":118677490},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:30:44 2023","interfaces":[{"interface":"back","average":{"1min":2.3300000000000001,"5min":2.3300000000000001,"15min":2.3300000000000001},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":9.5470000000000006,"5min":9.5470000000000006,"15min":9.5470000000000006},"last":0.54400000000000004},{"interface":"front","average":{"1min":3.1960000000000002,"5min":3.1960000000000002,"15min":3.1960000000000002},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":9.7129999999999992,"5min":9.7129999999999992,"15min":9.7129999999999992},"last":0.30199999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:30:44 2023","interfaces":[{"interface":"back","average":{"1min":5.125,"5min":5.125,"15min":5.125},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":16.151,"5min":16.151,"15min":16.151},"last":2.0369999999999999},{"interface":"front","average":{"1min":2.0939999999999999,"5min":2.0939999999999999,"15min":2.0939999999999999},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":9.4649999999999999,"5min":9.4649999999999999,"15min":9.4649999999999999},"last":0.58599999999999997}]},{"osd":3,"last update":"Sat Sep 23 20:30:44 2023","interfaces":[{"interface":"back","average":{"1min":3.867,"5min":3.867,"15min":3.867},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":9.5739999999999998,"5min":9.5739999999999998,"15min":9.5739999999999998},"last":0.96899999999999997},{"interface":"front","average":{"1min":4.8460000000000001,"5min":4.8460000000000001,"15min":4.8460000000000001},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":12.012,"5min":12.012,"15min":12.012},"last":2.3250000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:30:44 2023","interfaces":[{"interface":"back","average":{"1min":2.0790000000000002,"5min":2.0790000000000002,"15min":2.0790000000000002},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":9.4429999999999996,"5min":9.4429999999999996,"15min":9.4429999999999996},"last":2.0760000000000001},{"interface":"front","average":{"1min":2.024,"5min":2.024,"15min":2.024},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":9.5039999999999996,"5min":9.5039999999999996,"15min":9.5039999999999996},"last":0.51400000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:30:44 2023","interfaces":[{"interface":"back","average":{"1min":2.802,"5min":2.802,"15min":2.802},"min":{"1min":0.44,"5min":0.44,"15min":0.44},"max":{"1min":9.6479999999999997,"5min":9.6479999999999997,"15min":9.6479999999999997},"last":0.71099999999999997},{"interface":"front","average":{"1min":2.0899999999999999,"5min":2.0899999999999999,"15min":2.0899999999999999},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":9.4700000000000006,"5min":9.4700000000000006,"15min":9.4700000000000006},"last":3.8359999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:30:44 2023","interfaces":[{"interface":"back","average":{"1min":7.4489999999999998,"5min":7.4489999999999998,"15min":7.4489999999999998},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":72.682000000000002,"5min":72.682000000000002,"15min":72.682000000000002},"last":0.67100000000000004},{"interface":"front","average":{"1min":6.4400000000000004,"5min":6.4400000000000004,"15min":6.4400000000000004},"min":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":72.563999999999993,"5min":72.563999999999993,"15min":72.563999999999993},"last":9.6460000000000008}]},{"osd":7,"last update":"Sat Sep 23 20:30:44 2023","interfaces":[{"interface":"back","average":{"1min":3.2829999999999999,"5min":3.2829999999999999,"15min":3.2829999999999999},"min":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"max":{"1min":9.6850000000000005,"5min":9.6850000000000005,"15min":9.6850000000000005},"last":0.749},{"interface":"front","average":{"1min":2.3599999999999999,"5min":2.3599999999999999,"15min":2.3599999999999999},"min":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"max":{"1min":9.3949999999999996,"5min":9.3949999999999996,"15min":9.3949999999999996},"last":2.6619999999999999}]}]},{"osd":0,"up_from":94,"seq":403726925958,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":411968,"kb_used_data":68920,"kb_used_omap":7,"kb_used_meta":343032,"kb_avail":93333184,"statfs":{"total":95995035648,"available":95573180416,"internally_reserved":0,"allocated":70574080,"data_stored":69118156,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":351264785},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:30:48 2023","interfaces":[{"interface":"back","average":{"1min":3.887,"5min":3.887,"15min":3.887},"min":{"1min":0.17999999999999999,"5min":0.17999999999999999,"15min":0.17999999999999999},"max":{"1min":15.542,"5min":15.542,"15min":15.542},"last":0.36899999999999999},{"interface":"front","average":{"1min":4.4569999999999999,"5min":4.4569999999999999,"15min":4.4569999999999999},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":10.49,"5min":10.49,"15min":10.49},"last":1.2050000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:30:27 2023","interfaces":[{"interface":"back","average":{"1min":5.8789999999999996,"5min":3.3180000000000001,"15min":1.6619999999999999},"min":{"1min":0.185,"5min":0.185,"15min":0.185},"max":{"1min":77.448999999999998,"5min":77.448999999999998,"15min":77.448999999999998},"last":0.84099999999999997},{"interface":"front","average":{"1min":6.0869999999999997,"5min":2.9449999999999998,"15min":1.5069999999999999},"min":{"1min":0.24299999999999999,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":81.724999999999994,"5min":81.724999999999994,"15min":81.724999999999994},"last":0.42199999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:31:03 2023","interfaces":[{"interface":"back","average":{"1min":7.0750000000000002,"5min":3.6080000000000001,"15min":1.79},"min":{"1min":0.25,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":81.400999999999996,"5min":81.400999999999996,"15min":81.400999999999996},"last":0.81599999999999995},{"interface":"front","average":{"1min":9.3719999999999999,"5min":4.0679999999999996,"15min":1.7649999999999999},"min":{"1min":0.34699999999999998,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":81.137,"5min":81.137,"15min":81.137},"last":1.1639999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:30:27 2023","interfaces":[{"interface":"back","average":{"1min":2.093,"5min":2.4990000000000001,"15min":1.409},"min":{"1min":0.315,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":8.3209999999999997,"5min":23.876999999999999,"15min":23.876999999999999},"last":0.73499999999999999},{"interface":"front","average":{"1min":2.0009999999999999,"5min":1.9690000000000001,"15min":1.2390000000000001},"min":{"1min":0.34499999999999997,"5min":0.312,"15min":0.29299999999999998},"max":{"1min":10.414,"5min":23.681999999999999,"15min":23.681999999999999},"last":0.51100000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:30:27 2023","interfaces":[{"interface":"back","average":{"1min":2.645,"5min":2.657,"15min":1.6040000000000001},"min":{"1min":0.377,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":10.368,"5min":24.198,"15min":24.198},"last":0.56399999999999995},{"interface":"front","average":{"1min":2.2130000000000001,"5min":2.1869999999999998,"15min":1.45},"min":{"1min":0.38100000000000001,"5min":0.314,"15min":0.314},"max":{"1min":8.7780000000000005,"5min":14.938000000000001,"15min":14.938000000000001},"last":0.47199999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:30:27 2023","interfaces":[{"interface":"back","average":{"1min":2.2450000000000001,"5min":2.3420000000000001,"15min":1.319},"min":{"1min":0.40100000000000002,"5min":0.318,"15min":0.318},"max":{"1min":10.441000000000001,"5min":24.408000000000001,"15min":24.408000000000001},"last":0.53200000000000003},{"interface":"front","average":{"1min":1.764,"5min":2.1709999999999998,"15min":1.2629999999999999},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":6.9480000000000004,"5min":25.271000000000001,"15min":25.271000000000001},"last":0.45000000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:30:27 2023","interfaces":[{"interface":"back","average":{"1min":2.024,"5min":2.3839999999999999,"15min":1.4179999999999999},"min":{"1min":0.42199999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":7.4470000000000001,"5min":25.359000000000002,"15min":25.359000000000002},"last":0.58399999999999996},{"interface":"front","average":{"1min":2.5550000000000002,"5min":3.1200000000000001,"15min":1.6799999999999999},"min":{"1min":0.46500000000000002,"5min":0.378,"15min":0.378},"max":{"1min":8.2729999999999997,"5min":24.669,"15min":24.669},"last":0.78300000000000003}]}]},{"osd":2,"up_from":104,"seq":446676598906,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":890032,"kb_used_data":507432,"kb_used_omap":11,"kb_used_meta":382580,"kb_avail":92855120,"statfs":{"total":95995035648,"available":95083642880,"internally_reserved":0,"allocated":519610368,"data_stored":518178223,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11878,"internal_metadata":391762330},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1,0,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:30:37 2023","interfaces":[{"interface":"back","average":{"1min":3.3199999999999998,"5min":2.2509999999999999,"15min":1.282},"min":{"1min":0.193,"5min":0.17999999999999999,"15min":0.17999999999999999},"max":{"1min":30.765999999999998,"5min":30.765999999999998,"15min":30.765999999999998},"last":0.41599999999999998},{"interface":"front","average":{"1min":5.1280000000000001,"5min":2.3199999999999998,"15min":1.2150000000000001},"min":{"1min":0.252,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":16.824000000000002,"5min":16.824000000000002,"15min":16.824000000000002},"last":4.79}]},{"osd":1,"last update":"Sat Sep 23 20:30:45 2023","interfaces":[{"interface":"back","average":{"1min":4.851,"5min":4.851,"15min":4.851},"min":{"1min":0.221,"5min":0.221,"15min":0.221},"max":{"1min":16.280000000000001,"5min":16.280000000000001,"15min":16.280000000000001},"last":2.3809999999999998},{"interface":"front","average":{"1min":2.988,"5min":2.988,"15min":2.988},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":30.565000000000001,"5min":30.565000000000001,"15min":30.565000000000001},"last":2.121}]},{"osd":3,"last update":"Sat Sep 23 20:31:02 2023","interfaces":[{"interface":"back","average":{"1min":2.8849999999999998,"5min":2.2959999999999998,"15min":1.21},"min":{"1min":0.27800000000000002,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":11.132,"5min":16.331,"15min":16.331},"last":2.4300000000000002},{"interface":"front","average":{"1min":4.2539999999999996,"5min":2.8109999999999999,"15min":1.46},"min":{"1min":0.27300000000000002,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":30.704999999999998,"5min":30.704999999999998,"15min":30.704999999999998},"last":4.2249999999999996}]},{"osd":4,"last update":"Sat Sep 23 20:30:37 2023","interfaces":[{"interface":"back","average":{"1min":3.98,"5min":2.1669999999999998,"15min":1.234},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.28100000000000003},"max":{"1min":16.797999999999998,"5min":16.797999999999998,"15min":16.797999999999998},"last":3.851},{"interface":"front","average":{"1min":2.7309999999999999,"5min":2.1589999999999998,"15min":1.2609999999999999},"min":{"1min":0.309,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":10.518000000000001,"5min":14.971,"15min":14.971},"last":2.516}]},{"osd":5,"last update":"Sat Sep 23 20:30:37 2023","interfaces":[{"interface":"back","average":{"1min":4.5359999999999996,"5min":3.512,"15min":1.77},"min":{"1min":0.35299999999999998,"5min":0.309,"15min":0.309},"max":{"1min":16.766999999999999,"5min":95.393000000000001,"15min":95.393000000000001},"last":3.8980000000000001},{"interface":"front","average":{"1min":3.4790000000000001,"5min":3.6190000000000002,"15min":1.8},"min":{"1min":0.34799999999999998,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":10.474,"5min":95.414000000000001,"15min":95.414000000000001},"last":2.4630000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:30:37 2023","interfaces":[{"interface":"back","average":{"1min":3.258,"5min":2.29,"15min":1.323},"min":{"1min":0.33600000000000002,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":10.544,"5min":15.023999999999999,"15min":15.023999999999999},"last":2.548},{"interface":"front","average":{"1min":2.7930000000000001,"5min":2.3719999999999999,"15min":1.405},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":30.670999999999999,"5min":30.670999999999999,"15min":30.670999999999999},"last":0.46700000000000003}]},{"osd":7,"last update":"Sat Sep 23 20:30:37 2023","interfaces":[{"interface":"back","average":{"1min":3.4609999999999999,"5min":2.5529999999999999,"15min":1.4730000000000001},"min":{"1min":0.41299999999999998,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":30.741,"5min":30.741,"15min":30.741},"last":0.499},{"interface":"front","average":{"1min":4.8650000000000002,"5min":2.52,"15min":1.4550000000000001},"min":{"1min":0.40899999999999997,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":16.719000000000001,"5min":16.719000000000001,"15min":16.719000000000001},"last":3.9279999999999999}]}]},{"osd":3,"up_from":109,"seq":468151435379,"num_pgs":64,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":612192,"kb_used_data":217560,"kb_used_omap":4,"kb_used_meta":394619,"kb_avail":93132960,"statfs":{"total":95995035648,"available":95368151040,"internally_reserved":0,"allocated":222781440,"data_stored":221345116,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4514,"internal_metadata":404090462},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,0,0,3,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:31:05 2023","interfaces":[{"interface":"back","average":{"1min":2.5600000000000001,"5min":2.2909999999999999,"15min":1.4610000000000001},"min":{"1min":0.22500000000000001,"5min":0.186,"15min":0.186},"max":{"1min":6.71,"5min":15.589,"15min":67.305999999999997},"last":12.061999999999999},{"interface":"front","average":{"1min":2.839,"5min":1.978,"15min":1.3919999999999999},"min":{"1min":0.19500000000000001,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":12.781000000000001,"5min":13.743,"15min":67.403999999999996},"last":4.274}]},{"osd":1,"last update":"Sat Sep 23 20:30:47 2023","interfaces":[{"interface":"back","average":{"1min":3.0880000000000001,"5min":3.0880000000000001,"15min":3.0880000000000001},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":8.9199999999999999,"5min":8.9199999999999999,"15min":8.9199999999999999},"last":12.632},{"interface":"front","average":{"1min":3.2530000000000001,"5min":3.2530000000000001,"15min":3.2530000000000001},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":10.967000000000001,"5min":10.967000000000001,"15min":10.967000000000001},"last":4.3860000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:31:05 2023","interfaces":[{"interface":"back","average":{"1min":3.1419999999999999,"5min":2.6739999999999999,"15min":1.3959999999999999},"min":{"1min":0.29799999999999999,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":11.253,"5min":24.045000000000002,"15min":24.045000000000002},"last":4.4509999999999996},{"interface":"front","average":{"1min":3.1709999999999998,"5min":2.649,"15min":1.4430000000000001},"min":{"1min":0.27100000000000002,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":19.603000000000002,"5min":19.603000000000002,"15min":19.603000000000002},"last":3.8580000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:31:05 2023","interfaces":[{"interface":"back","average":{"1min":2.331,"5min":2.46,"15min":1.3560000000000001},"min":{"1min":0.34200000000000003,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":6.5670000000000002,"5min":46.042999999999999,"15min":46.042999999999999},"last":11.802},{"interface":"front","average":{"1min":3.7120000000000002,"5min":2.9569999999999999,"15min":1.591},"min":{"1min":0.34599999999999997,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":19.635000000000002,"5min":46.579999999999998,"15min":46.579999999999998},"last":3.7440000000000002}]},{"osd":5,"last update":"Sat Sep 23 20:31:05 2023","interfaces":[{"interface":"back","average":{"1min":2.319,"5min":2.5960000000000001,"15min":1.5189999999999999},"min":{"1min":0.45700000000000002,"5min":0.31900000000000001,"15min":0.311},"max":{"1min":6.2370000000000001,"5min":45.292000000000002,"15min":45.292000000000002},"last":11.507999999999999},{"interface":"front","average":{"1min":2.6139999999999999,"5min":2.5129999999999999,"15min":1.3999999999999999},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.313},"max":{"1min":11.028,"5min":44.679000000000002,"15min":44.679000000000002},"last":4.2439999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:31:05 2023","interfaces":[{"interface":"back","average":{"1min":3.3519999999999999,"5min":2.3410000000000002,"15min":1.4279999999999999},"min":{"1min":0.38500000000000001,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":17.524999999999999,"5min":17.524999999999999,"15min":17.524999999999999},"last":4.2119999999999997},{"interface":"front","average":{"1min":3.0670000000000002,"5min":2.3300000000000001,"15min":1.387},"min":{"1min":0.439,"5min":0.36399999999999999,"15min":0.35499999999999998},"max":{"1min":19.524000000000001,"5min":19.524000000000001,"15min":19.524000000000001},"last":11.191000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:31:05 2023","interfaces":[{"interface":"back","average":{"1min":4.1749999999999998,"5min":2.8370000000000002,"15min":1.5549999999999999},"min":{"1min":0.40699999999999997,"5min":0.374,"15min":0.34999999999999998},"max":{"1min":19.571999999999999,"5min":19.571999999999999,"15min":19.571999999999999},"last":11.24},{"interface":"front","average":{"1min":2.3100000000000001,"5min":2.762,"15min":1.607},"min":{"1min":0.42999999999999999,"5min":0.371,"15min":0.371},"max":{"1min":6.407,"5min":45.387999999999998,"15min":45.387999999999998},"last":11.856999999999999}]}]},{"osd":4,"up_from":0,"seq":128849019133,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":5,"up_from":39,"seq":167503724794,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":305332,"kb_used_data":86252,"kb_used_omap":0,"kb_used_meta":219072,"kb_avail":93439820,"statfs":{"total":95995035648,"available":95682375680,"internally_reserved":0,"allocated":88322048,"data_stored":87396354,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":224329728},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:30:31 2023","interfaces":[{"interface":"back","average":{"1min":2.8109999999999999,"5min":1.7529999999999999,"15min":1.181},"min":{"1min":0.28499999999999998,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":8.1609999999999996,"5min":11.346,"15min":32.475000000000001},"last":0.83299999999999996},{"interface":"front","average":{"1min":2.5950000000000002,"5min":1.681,"15min":1.2030000000000001},"min":{"1min":0.29199999999999998,"5min":0.245,"15min":0.20599999999999999},"max":{"1min":8.1859999999999999,"5min":8.4900000000000002,"15min":32.457000000000001},"last":0.80500000000000005}]},{"osd":1,"last update":"Sat Sep 23 20:30:45 2023","interfaces":[{"interface":"back","average":{"1min":2.0550000000000002,"5min":2.0550000000000002,"15min":2.0550000000000002},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":8.5839999999999996,"5min":8.5839999999999996,"15min":8.5839999999999996},"last":1.571},{"interface":"front","average":{"1min":1.5609999999999999,"5min":1.5609999999999999,"15min":1.5609999999999999},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":6.7969999999999997,"5min":6.7969999999999997,"15min":6.7969999999999997},"last":0.94399999999999995}]},{"osd":2,"last update":"Sat Sep 23 20:30:31 2023","interfaces":[{"interface":"back","average":{"1min":3.5259999999999998,"5min":1.823,"15min":1.325},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.26700000000000002},"max":{"1min":18.472999999999999,"5min":18.472999999999999,"15min":34.514000000000003},"last":0.41899999999999998},{"interface":"front","average":{"1min":1.8919999999999999,"5min":1.7869999999999999,"15min":1.256},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":8.016,"5min":11.574999999999999,"15min":32.515000000000001},"last":0.73199999999999998}]},{"osd":3,"last update":"Sat Sep 23 20:31:05 2023","interfaces":[{"interface":"back","average":{"1min":3.1230000000000002,"5min":2.2109999999999999,"15min":1.3220000000000001},"min":{"1min":0.61799999999999999,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":10.952999999999999,"5min":12.846,"15min":12.846},"last":0.96599999999999997},{"interface":"front","average":{"1min":2.3479999999999999,"5min":1.754,"15min":1.2310000000000001},"min":{"1min":0.32300000000000001,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":7.2060000000000004,"5min":12.005000000000001,"15min":12.005000000000001},"last":2.4369999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:30:25 2023","interfaces":[{"interface":"back","average":{"1min":3.1560000000000001,"5min":2.359,"15min":1.2230000000000001},"min":{"1min":0.32500000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":28.015999999999998,"5min":28.015999999999998,"15min":28.015999999999998},"last":0.316},{"interface":"front","average":{"1min":2.0339999999999998,"5min":1.6459999999999999,"15min":1.0149999999999999},"min":{"1min":0.33800000000000002,"5min":0.32600000000000001,"15min":0.29099999999999998},"max":{"1min":12.300000000000001,"5min":12.300000000000001,"15min":12.300000000000001},"last":0.38200000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:30:39 2023","interfaces":[{"interface":"back","average":{"1min":1.1220000000000001,"5min":1.796,"15min":1.1499999999999999},"min":{"1min":0.309,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":4.9829999999999997,"5min":12.890000000000001,"15min":12.890000000000001},"last":0.85199999999999998},{"interface":"front","average":{"1min":0.92500000000000004,"5min":1.5389999999999999,"15min":1.137},"min":{"1min":0.371,"5min":0.36399999999999999,"15min":0.34499999999999997},"max":{"1min":2.4710000000000001,"5min":16.265999999999998,"15min":16.265999999999998},"last":0.78400000000000003}]},{"osd":7,"last update":"Sat Sep 23 20:31:05 2023","interfaces":[{"interface":"back","average":{"1min":1.01,"5min":1.8169999999999999,"15min":1.204},"min":{"1min":0.32400000000000001,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":2.77,"5min":36.621000000000002,"15min":36.621000000000002},"last":0.48999999999999999},{"interface":"front","average":{"1min":0.94299999999999995,"5min":1.6759999999999999,"15min":1.103},"min":{"1min":0.33500000000000002,"5min":0.316,"15min":0.315},"max":{"1min":3.6219999999999999,"5min":37.331000000000003,"15min":37.331000000000003},"last":0.69799999999999995}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4722,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1360,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8206,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7156,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3154,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":63836160,"data_stored":63832088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":829161472,"data_stored":829161472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":494403584,"data_stored":494403584,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":194248704,"data_stored":194248704,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":533069824,"data_stored":533069824,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":76746752,"data_stored":76742680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":112459776,"data_stored":112459776,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":447479808,"data_stored":447479808,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:31:18.978 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:31:19.019 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:19.226 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:31:19.230 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:19.230 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:31:19.331 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:31:19.455 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:19.571 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:19 smithi149 ceph-mon[170394]: from='client.45467 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:31:19.761 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:19 smithi018 ceph-mon[184144]: from='client.45467 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:31:19.765 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:19 smithi018 ceph-mon[179647]: from='client.45467 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:31:19.795 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:31:19.804 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:19.804 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:31:19.905 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:31:19.961 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:20.063 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:31:20.066 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:20.066 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:31:20.111 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:31:20.167 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:31:20.255 INFO:teuthology.orchestra.run.smithi149.stdout:ERROR: (22) Invalid argument 2023-09-23T20:31:20.255 INFO:teuthology.orchestra.run.smithi149.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:31:20.277 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:20.507 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:31:20.514 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:20.514 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:31:20.615 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:31:20.657 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:20 smithi149 ceph-mon[170394]: pgmap v666: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait, 3 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 182 MiB data, 3.8 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 589 op/s; 28009/8610 objects degraded (325.308%); 257/8610 objects misplaced (2.985%); 1.1 MiB/s, 127 objects/s recovering 2023-09-23T20:31:20.661 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:20.685 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T20:31:20.724 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:31:20.728 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:20.729 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:31:20.830 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:31:20.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:20 smithi018 ceph-mon[184144]: pgmap v666: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait, 3 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 182 MiB data, 3.8 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 589 op/s; 28009/8610 objects degraded (325.308%); 257/8610 objects misplaced (2.985%); 1.1 MiB/s, 127 objects/s recovering 2023-09-23T20:31:20.898 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:20 smithi018 ceph-mon[179647]: pgmap v666: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait, 3 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 182 MiB data, 3.8 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 589 op/s; 28009/8610 objects degraded (325.308%); 257/8610 objects misplaced (2.985%); 1.1 MiB/s, 127 objects/s recovering 2023-09-23T20:31:20.905 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:20.959 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Removed 104736 objects 2023-09-23T20:31:20.959 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Clean up completed and total clean up time :148.352 2023-09-23T20:31:21.386 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:31:21.678 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:31:21.682 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:21.683 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:31:21.758 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:21 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 28009/8610 objects degraded (325.308%), 12 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-09-23T20:31:21.784 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:31:21.810 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:21 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 28009/8610 objects degraded (325.308%), 12 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-09-23T20:31:21.811 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:21 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 28009/8610 objects degraded (325.308%), 12 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-09-23T20:31:21.813 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:21.938 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:31:21.941 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:21.942 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:31:21.942 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:31:21.976 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:31:22.043 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:31:22.073 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:22.466 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:31:22.469 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:22.470 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:31:22.570 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:31:22.603 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:22.629 INFO:teuthology.orchestra.run.smithi149.stdout:ERROR: (22) Invalid argument 2023-09-23T20:31:22.629 INFO:teuthology.orchestra.run.smithi149.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:31:22.655 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:22 smithi149 ceph-mon[170394]: pgmap v667: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait, 3 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 86 MiB data, 3.6 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 609 op/s; 28009/3984 objects degraded (703.037%); 257/3984 objects misplaced (6.451%); 1.1 MiB/s, 97 objects/s recovering 2023-09-23T20:31:22.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:22 smithi018 ceph-mon[179647]: pgmap v667: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait, 3 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 86 MiB data, 3.6 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 609 op/s; 28009/3984 objects degraded (703.037%); 257/3984 objects misplaced (6.451%); 1.1 MiB/s, 97 objects/s recovering 2023-09-23T20:31:22.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:22 smithi018 ceph-mon[184144]: pgmap v667: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait, 3 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 86 MiB data, 3.6 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 609 op/s; 28009/3984 objects degraded (703.037%); 257/3984 objects misplaced (6.451%); 1.1 MiB/s, 97 objects/s recovering 2023-09-23T20:31:23.031 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T20:31:23.250 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:31:23.252 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:23.253 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:31:23.354 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:31:23.383 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:23.417 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:31:23.420 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:23.420 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:31:23.511 INFO:teuthology.orchestra.run.smithi018.stderr:nodeep-scrub is set 2023-09-23T20:31:23.521 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:31:23.533 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:23 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3942365110' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:31:23.533 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:23 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:31:23.533 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:23 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3942365110' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:31:23.534 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:23 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:31:23.550 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:23.583 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:31:23.586 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:23.586 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:31:23.644 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:31:23.687 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:31:23.711 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:23 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3942365110' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:31:23.711 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:23 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:31:23.713 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:23.775 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:31:23.777 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:23.778 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:31:23.879 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:31:24.233 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:24.635 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:24 smithi149 ceph-mon[170394]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:31:24.636 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:24 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:31:24.636 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:24 smithi149 ceph-mon[170394]: osdmap e294: 8 total, 8 up, 7 in 2023-09-23T20:31:24.636 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:24 smithi149 ceph-mon[170394]: failed to encode map e294 with expected crc 2023-09-23T20:31:24.636 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:24 smithi149 ceph-mon[170394]: pgmap v669: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait, 3 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 40 MiB data, 3.4 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 570 op/s; 28009/1782 objects degraded (1571.773%); 35/1782 objects misplaced (1.964%); 1.0 MiB/s, 210 objects/s recovering 2023-09-23T20:31:24.809 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:24 smithi018 ceph-mon[179647]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:31:24.809 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:24 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:31:24.809 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:24 smithi018 ceph-mon[179647]: osdmap e294: 8 total, 8 up, 7 in 2023-09-23T20:31:24.809 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:24 smithi018 ceph-mon[179647]: failed to encode map e294 with expected crc 2023-09-23T20:31:24.810 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:24 smithi018 ceph-mon[179647]: pgmap v669: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait, 3 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 40 MiB data, 3.4 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 570 op/s; 28009/1782 objects degraded (1571.773%); 35/1782 objects misplaced (1.964%); 1.0 MiB/s, 210 objects/s recovering 2023-09-23T20:31:24.810 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:24 smithi018 ceph-mon[184144]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:31:24.810 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:24 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:31:24.810 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:24 smithi018 ceph-mon[184144]: osdmap e294: 8 total, 8 up, 7 in 2023-09-23T20:31:24.810 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:24 smithi018 ceph-mon[184144]: failed to encode map e294 with expected crc 2023-09-23T20:31:24.810 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:24 smithi018 ceph-mon[184144]: pgmap v669: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait, 3 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 40 MiB data, 3.4 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 570 op/s; 28009/1782 objects degraded (1571.773%); 35/1782 objects misplaced (1.964%); 1.0 MiB/s, 210 objects/s recovering 2023-09-23T20:31:24.885 INFO:teuthology.orchestra.run.smithi149.stdout:ERROR: (22) Invalid argument 2023-09-23T20:31:24.886 INFO:teuthology.orchestra.run.smithi149.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:31:25.125 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:31:24 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:31:24] "GET /metrics HTTP/1.1" 200 36567 "" "Prometheus/2.43.0" 2023-09-23T20:31:25.296 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T20:31:25.425 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:31:25.473 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:25.473 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:31:25.574 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:31:25.605 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:25.640 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:31:25.643 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:25.644 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:31:25.744 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:31:25.769 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:25 smithi149 ceph-mon[170394]: failed to encode map e294 with expected crc 2023-09-23T20:31:25.769 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:25 smithi149 ceph-mon[170394]: failed to encode map e294 with expected crc 2023-09-23T20:31:25.769 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:25 smithi149 ceph-mon[170394]: failed to encode map e294 with expected crc 2023-09-23T20:31:25.772 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:25.805 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:31:25.807 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:25.808 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:31:25.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:25 smithi018 ceph-mon[184144]: failed to encode map e294 with expected crc 2023-09-23T20:31:25.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:25 smithi018 ceph-mon[184144]: failed to encode map e294 with expected crc 2023-09-23T20:31:25.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:25 smithi018 ceph-mon[184144]: failed to encode map e294 with expected crc 2023-09-23T20:31:25.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:25 smithi018 ceph-mon[179647]: failed to encode map e294 with expected crc 2023-09-23T20:31:25.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:25 smithi018 ceph-mon[179647]: failed to encode map e294 with expected crc 2023-09-23T20:31:25.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:25 smithi018 ceph-mon[179647]: failed to encode map e294 with expected crc 2023-09-23T20:31:25.908 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:31:25.942 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:26.683 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_1 2023-09-23T20:31:26.684 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd pool rm unique_pool_1 unique_pool_1 --yes-i-really-really-mean-it 2023-09-23T20:31:26.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:26 smithi018 ceph-mon[184144]: pgmap v670: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait, 3 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 31 MiB data, 3.3 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 486 op/s; 28009/1332 objects degraded (2102.778%); 35/1332 objects misplaced (2.628%); 883 KiB/s, 175 objects/s recovering 2023-09-23T20:31:26.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:26 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 28009/1782 objects degraded (1571.773%), 12 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-09-23T20:31:26.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:26 smithi018 ceph-mon[179647]: pgmap v670: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait, 3 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 31 MiB data, 3.3 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 486 op/s; 28009/1332 objects degraded (2102.778%); 35/1332 objects misplaced (2.628%); 883 KiB/s, 175 objects/s recovering 2023-09-23T20:31:26.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:26 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 28009/1782 objects degraded (1571.773%), 12 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-09-23T20:31:26.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:26 smithi149 ceph-mon[170394]: pgmap v670: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait, 3 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 31 MiB data, 3.3 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 486 op/s; 28009/1332 objects degraded (2102.778%); 35/1332 objects misplaced (2.628%); 883 KiB/s, 175 objects/s recovering 2023-09-23T20:31:26.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:26 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 28009/1782 objects degraded (1571.773%), 12 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-09-23T20:31:28.416 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:31:28.419 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:28.419 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:31:28.520 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:31:28.549 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:28.781 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:28 smithi018 ceph-mon[184144]: pgmap v671: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait, 3 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 29 MiB data, 3.3 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 389 op/s; 28009/1278 objects degraded (2191.628%); 35/1278 objects misplaced (2.739%); 883 KiB/s, 269 objects/s recovering 2023-09-23T20:31:28.914 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:31:28.936 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:28.937 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:31:28.943 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:31:29.037 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:31:29.066 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:28 smithi018 ceph-mon[179647]: pgmap v671: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait, 3 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 29 MiB data, 3.3 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 389 op/s; 28009/1278 objects degraded (2191.628%); 35/1278 objects misplaced (2.739%); 883 KiB/s, 269 objects/s recovering 2023-09-23T20:31:29.070 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:29.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:28 smithi149 ceph-mon[170394]: pgmap v671: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait, 3 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 29 MiB data, 3.3 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 389 op/s; 28009/1278 objects degraded (2191.628%); 35/1278 objects misplaced (2.739%); 883 KiB/s, 269 objects/s recovering 2023-09-23T20:31:29.455 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T20:31:29.787 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:29 smithi018 ceph-mon[179647]: failed to encode map e295 with expected crc 2023-09-23T20:31:29.787 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:29 smithi018 ceph-mon[179647]: osdmap e295: 8 total, 8 up, 7 in 2023-09-23T20:31:29.787 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:29 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:31:29.787 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:29 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:31:29.790 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:31:29.793 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:29.794 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:31:29.894 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:31:29.922 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:30.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:29 smithi018 ceph-mon[184144]: failed to encode map e295 with expected crc 2023-09-23T20:31:30.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:29 smithi018 ceph-mon[184144]: osdmap e295: 8 total, 8 up, 7 in 2023-09-23T20:31:30.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:29 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:31:30.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:29 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:31:30.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:29 smithi149 ceph-mon[170394]: failed to encode map e295 with expected crc 2023-09-23T20:31:30.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:29 smithi149 ceph-mon[170394]: osdmap e295: 8 total, 8 up, 7 in 2023-09-23T20:31:30.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:29 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:31:30.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:29 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:31:30.233 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:31:30.242 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:30.243 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:31:30.343 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:31:30.370 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:30.402 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:31:30.405 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:30.405 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:31:30.506 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:31:30.535 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:31.052 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:31:31.055 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:30 smithi018 ceph-mon[179647]: pgmap v673: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait, 3 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 29 MiB data, 3.2 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 136 op/s; 26986/1278 objects degraded (2111.581%); 35/1278 objects misplaced (2.739%); 0 B/s, 445 objects/s recovering 2023-09-23T20:31:31.056 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:30 smithi018 ceph-mon[179647]: failed to encode map e296 with expected crc 2023-09-23T20:31:31.056 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:30 smithi018 ceph-mon[179647]: osdmap e296: 8 total, 8 up, 7 in 2023-09-23T20:31:31.056 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:30 smithi018 ceph-mon[179647]: 6.e continuing backfill to osd.5 from (248'6572,293'13102] MIN to 293'13102 2023-09-23T20:31:31.056 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:30 smithi018 ceph-mon[179647]: failed to encode map e296 with expected crc 2023-09-23T20:31:31.056 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:30 smithi018 ceph-mon[179647]: failed to encode map e296 with expected crc 2023-09-23T20:31:31.056 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:30 smithi018 ceph-mon[179647]: failed to encode map e296 with expected crc 2023-09-23T20:31:31.057 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:31.057 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:31:31.057 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:30 smithi018 ceph-mon[184144]: pgmap v673: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait, 3 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 29 MiB data, 3.2 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 136 op/s; 26986/1278 objects degraded (2111.581%); 35/1278 objects misplaced (2.739%); 0 B/s, 445 objects/s recovering 2023-09-23T20:31:31.057 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:30 smithi018 ceph-mon[184144]: failed to encode map e296 with expected crc 2023-09-23T20:31:31.058 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:30 smithi018 ceph-mon[184144]: osdmap e296: 8 total, 8 up, 7 in 2023-09-23T20:31:31.058 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:30 smithi018 ceph-mon[184144]: 6.e continuing backfill to osd.5 from (248'6572,293'13102] MIN to 293'13102 2023-09-23T20:31:31.058 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:30 smithi018 ceph-mon[184144]: failed to encode map e296 with expected crc 2023-09-23T20:31:31.058 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:30 smithi018 ceph-mon[184144]: failed to encode map e296 with expected crc 2023-09-23T20:31:31.058 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:30 smithi018 ceph-mon[184144]: failed to encode map e296 with expected crc 2023-09-23T20:31:31.158 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:31:31.185 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:31.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:30 smithi149 ceph-mon[170394]: pgmap v673: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait, 3 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 29 MiB data, 3.2 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 136 op/s; 26986/1278 objects degraded (2111.581%); 35/1278 objects misplaced (2.739%); 0 B/s, 445 objects/s recovering 2023-09-23T20:31:31.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:30 smithi149 ceph-mon[170394]: failed to encode map e296 with expected crc 2023-09-23T20:31:31.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:30 smithi149 ceph-mon[170394]: osdmap e296: 8 total, 8 up, 7 in 2023-09-23T20:31:31.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:30 smithi149 ceph-mon[170394]: 6.e continuing backfill to osd.5 from (248'6572,293'13102] MIN to 293'13102 2023-09-23T20:31:31.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:30 smithi149 ceph-mon[170394]: failed to encode map e296 with expected crc 2023-09-23T20:31:31.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:30 smithi149 ceph-mon[170394]: failed to encode map e296 with expected crc 2023-09-23T20:31:31.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:30 smithi149 ceph-mon[170394]: failed to encode map e296 with expected crc 2023-09-23T20:31:31.302 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:31:31.305 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:31.306 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:31:31.406 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:31:31.435 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:31.772 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:31:31.775 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:31.776 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:31:31.876 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:31:31.908 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:32.007 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:31:32.009 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:31 smithi018 ceph-mon[184144]: failed to encode map e297 with expected crc 2023-09-23T20:31:32.009 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:31 smithi018 ceph-mon[184144]: osdmap e297: 8 total, 8 up, 7 in 2023-09-23T20:31:32.009 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:31 smithi018 ceph-mon[184144]: failed to encode map e297 with expected crc 2023-09-23T20:31:32.009 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:31 smithi018 ceph-mon[184144]: failed to encode map e297 with expected crc 2023-09-23T20:31:32.010 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:31 smithi018 ceph-mon[184144]: failed to encode map e297 with expected crc 2023-09-23T20:31:32.010 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:31 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 26986/1278 objects degraded (2111.581%), 12 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-09-23T20:31:32.010 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:31 smithi018 ceph-mon[179647]: failed to encode map e297 with expected crc 2023-09-23T20:31:32.010 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:31 smithi018 ceph-mon[179647]: osdmap e297: 8 total, 8 up, 7 in 2023-09-23T20:31:32.010 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:31 smithi018 ceph-mon[179647]: failed to encode map e297 with expected crc 2023-09-23T20:31:32.010 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:31 smithi018 ceph-mon[179647]: failed to encode map e297 with expected crc 2023-09-23T20:31:32.010 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:31 smithi018 ceph-mon[179647]: failed to encode map e297 with expected crc 2023-09-23T20:31:32.010 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:31 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 26986/1278 objects degraded (2111.581%), 12 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-09-23T20:31:32.011 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:32.011 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:31:32.112 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:31:32.115 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:31:32.118 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:31:32.136 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:31 smithi149 ceph-mon[170394]: failed to encode map e297 with expected crc 2023-09-23T20:31:32.136 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:31 smithi149 ceph-mon[170394]: osdmap e297: 8 total, 8 up, 7 in 2023-09-23T20:31:32.136 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:31 smithi149 ceph-mon[170394]: failed to encode map e297 with expected crc 2023-09-23T20:31:32.136 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:31 smithi149 ceph-mon[170394]: failed to encode map e297 with expected crc 2023-09-23T20:31:32.136 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:31 smithi149 ceph-mon[170394]: failed to encode map e297 with expected crc 2023-09-23T20:31:32.136 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:31 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 26986/1278 objects degraded (2111.581%), 12 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-09-23T20:31:32.139 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:32.173 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:31:32.175 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:32.176 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:31:32.199 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:31:32.218 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:31:32.245 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:31:32.263 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:31:32.277 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:31:32.285 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:31:32.305 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:31:32.307 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:32.327 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:31:32.349 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:31:32.351 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:31:32.351 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:31:32.351 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:31:32.351 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:31:32.351 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:31:32.351 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:31:32.351 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:31:32.351 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:31:32.408 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:31:32.500 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:31:32.503 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:32.503 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:31:32.604 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:31:32.630 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:32.666 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:31:32.668 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:32.669 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:31:32.769 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:31:32.795 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:32.836 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:31:32.838 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:32.839 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:31:32.939 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:31:32.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:32 smithi149 ceph-mon[170394]: pgmap v676: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait, 3 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 29 MiB data, 3.2 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 5 op/s; 26986/1278 objects degraded (2111.581%); 35/1278 objects misplaced (2.739%); 0 B/s, 432 objects/s recovering 2023-09-23T20:31:32.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:32 smithi149 ceph-mon[170394]: failed to encode map e298 with expected crc 2023-09-23T20:31:32.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:32 smithi149 ceph-mon[170394]: osdmap e298: 8 total, 8 up, 7 in 2023-09-23T20:31:32.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:32 smithi149 ceph-mon[170394]: failed to encode map e298 with expected crc 2023-09-23T20:31:32.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:32 smithi149 ceph-mon[170394]: failed to encode map e298 with expected crc 2023-09-23T20:31:32.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:32 smithi149 ceph-mon[170394]: failed to encode map e298 with expected crc 2023-09-23T20:31:32.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:32 smithi149 ceph-mon[170394]: failed to encode map e298 with expected crc 2023-09-23T20:31:32.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:32 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1457574396' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:31:32.968 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:32 smithi018 ceph-mon[179647]: pgmap v676: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait, 3 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 29 MiB data, 3.2 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 5 op/s; 26986/1278 objects degraded (2111.581%); 35/1278 objects misplaced (2.739%); 0 B/s, 432 objects/s recovering 2023-09-23T20:31:32.968 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:32 smithi018 ceph-mon[179647]: failed to encode map e298 with expected crc 2023-09-23T20:31:32.968 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:32 smithi018 ceph-mon[179647]: osdmap e298: 8 total, 8 up, 7 in 2023-09-23T20:31:32.968 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:32 smithi018 ceph-mon[179647]: failed to encode map e298 with expected crc 2023-09-23T20:31:32.968 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:32 smithi018 ceph-mon[179647]: failed to encode map e298 with expected crc 2023-09-23T20:31:32.968 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:32 smithi018 ceph-mon[179647]: failed to encode map e298 with expected crc 2023-09-23T20:31:32.968 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:32 smithi018 ceph-mon[179647]: failed to encode map e298 with expected crc 2023-09-23T20:31:32.968 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:32 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1457574396' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:31:32.968 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:32 smithi018 ceph-mon[184144]: pgmap v676: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait, 3 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 29 MiB data, 3.2 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 5 op/s; 26986/1278 objects degraded (2111.581%); 35/1278 objects misplaced (2.739%); 0 B/s, 432 objects/s recovering 2023-09-23T20:31:32.968 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:32 smithi018 ceph-mon[184144]: failed to encode map e298 with expected crc 2023-09-23T20:31:32.969 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:32 smithi018 ceph-mon[184144]: osdmap e298: 8 total, 8 up, 7 in 2023-09-23T20:31:32.969 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:32 smithi018 ceph-mon[184144]: failed to encode map e298 with expected crc 2023-09-23T20:31:32.969 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:32 smithi018 ceph-mon[184144]: failed to encode map e298 with expected crc 2023-09-23T20:31:32.969 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:32 smithi018 ceph-mon[184144]: failed to encode map e298 with expected crc 2023-09-23T20:31:32.969 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:32 smithi018 ceph-mon[184144]: failed to encode map e298 with expected crc 2023-09-23T20:31:32.969 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:32 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1457574396' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:31:32.971 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:33.208 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:31:33.211 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:33.211 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:31:33.312 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:31:33.343 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:33.923 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:31:33.926 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:33.927 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:31:34.027 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:31:34.060 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:34.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:33 smithi018 ceph-mon[184144]: failed to encode map e299 with expected crc 2023-09-23T20:31:34.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:33 smithi018 ceph-mon[184144]: osdmap e299: 8 total, 8 up, 7 in 2023-09-23T20:31:34.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:33 smithi018 ceph-mon[184144]: failed to encode map e299 with expected crc 2023-09-23T20:31:34.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:33 smithi018 ceph-mon[184144]: failed to encode map e299 with expected crc 2023-09-23T20:31:34.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:33 smithi018 ceph-mon[184144]: failed to encode map e299 with expected crc 2023-09-23T20:31:34.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:33 smithi018 ceph-mon[184144]: failed to encode map e299 with expected crc 2023-09-23T20:31:34.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:33 smithi018 ceph-mon[184144]: failed to encode map e299 with expected crc 2023-09-23T20:31:34.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:33 smithi018 ceph-mon[184144]: failed to encode map e299 with expected crc 2023-09-23T20:31:34.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:33 smithi018 ceph-mon[184144]: failed to encode map e299 with expected crc 2023-09-23T20:31:34.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:33 smithi018 ceph-mon[179647]: failed to encode map e299 with expected crc 2023-09-23T20:31:34.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:33 smithi018 ceph-mon[179647]: osdmap e299: 8 total, 8 up, 7 in 2023-09-23T20:31:34.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:33 smithi018 ceph-mon[179647]: failed to encode map e299 with expected crc 2023-09-23T20:31:34.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:33 smithi018 ceph-mon[179647]: failed to encode map e299 with expected crc 2023-09-23T20:31:34.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:33 smithi018 ceph-mon[179647]: failed to encode map e299 with expected crc 2023-09-23T20:31:34.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:33 smithi018 ceph-mon[179647]: failed to encode map e299 with expected crc 2023-09-23T20:31:34.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:33 smithi018 ceph-mon[179647]: failed to encode map e299 with expected crc 2023-09-23T20:31:34.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:33 smithi018 ceph-mon[179647]: failed to encode map e299 with expected crc 2023-09-23T20:31:34.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:33 smithi018 ceph-mon[179647]: failed to encode map e299 with expected crc 2023-09-23T20:31:34.145 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:31:34.148 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:34.149 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:31:34.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:33 smithi149 ceph-mon[170394]: failed to encode map e299 with expected crc 2023-09-23T20:31:34.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:33 smithi149 ceph-mon[170394]: osdmap e299: 8 total, 8 up, 7 in 2023-09-23T20:31:34.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:33 smithi149 ceph-mon[170394]: failed to encode map e299 with expected crc 2023-09-23T20:31:34.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:33 smithi149 ceph-mon[170394]: failed to encode map e299 with expected crc 2023-09-23T20:31:34.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:33 smithi149 ceph-mon[170394]: failed to encode map e299 with expected crc 2023-09-23T20:31:34.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:33 smithi149 ceph-mon[170394]: failed to encode map e299 with expected crc 2023-09-23T20:31:34.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:33 smithi149 ceph-mon[170394]: failed to encode map e299 with expected crc 2023-09-23T20:31:34.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:33 smithi149 ceph-mon[170394]: failed to encode map e299 with expected crc 2023-09-23T20:31:34.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:33 smithi149 ceph-mon[170394]: failed to encode map e299 with expected crc 2023-09-23T20:31:34.249 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:31:34.283 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:34.806 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:31:34.808 INFO:teuthology.orchestra.run.smithi018.stderr:dumped all 2023-09-23T20:31:34.817 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:34 smithi018 ceph-mon[179647]: pgmap v679: 119 pgs: 1 peering, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 3 active+recovery_wait, 2 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 29 MiB data, 3.2 GiB used, 623 GiB / 626 GiB avail; 25814/1278 objects degraded (2019.875%); 35/1278 objects misplaced (2.739%); 0 B/s, 292 objects/s recovering 2023-09-23T20:31:35.034 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:31:35.068 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:34 smithi018 ceph-mon[184144]: pgmap v679: 119 pgs: 1 peering, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 3 active+recovery_wait, 2 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 29 MiB data, 3.2 GiB used, 623 GiB / 626 GiB avail; 25814/1278 objects degraded (2019.875%); 35/1278 objects misplaced (2.739%); 0 B/s, 292 objects/s recovering 2023-09-23T20:31:35.068 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:31:34 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:31:34] "GET /metrics HTTP/1.1" 200 36572 "" "Prometheus/2.43.0" 2023-09-23T20:31:35.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:34 smithi149 ceph-mon[170394]: pgmap v679: 119 pgs: 1 peering, 1 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 3 active+recovery_wait, 2 active+remapped+backfill_wait, 10 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 29 MiB data, 3.2 GiB used, 623 GiB / 626 GiB avail; 25814/1278 objects degraded (2019.875%); 35/1278 objects misplaced (2.739%); 0 B/s, 292 objects/s recovering 2023-09-23T20:31:36.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:35 smithi018 ceph-mon[184144]: from='client.35625 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:31:36.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:35 smithi018 ceph-mon[179647]: from='client.35625 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:31:36.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:35 smithi149 ceph-mon[170394]: from='client.35625 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:31:36.411 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:31:36.414 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:36.414 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:31:36.515 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:31:36.543 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:36.577 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:31:36.580 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:36.614 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:31:36.715 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:31:36.744 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:37.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:36 smithi018 ceph-mon[184144]: pgmap v680: 119 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 29 MiB data, 3.2 GiB used, 623 GiB / 626 GiB avail; 24395/1278 objects degraded (1908.842%); 35/1278 objects misplaced (2.739%); 689 KiB/s, 483 objects/s recovering 2023-09-23T20:31:37.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:36 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 24395/1278 objects degraded (1908.842%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T20:31:37.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:36 smithi018 ceph-mon[179647]: pgmap v680: 119 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 29 MiB data, 3.2 GiB used, 623 GiB / 626 GiB avail; 24395/1278 objects degraded (1908.842%); 35/1278 objects misplaced (2.739%); 689 KiB/s, 483 objects/s recovering 2023-09-23T20:31:37.144 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:36 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 24395/1278 objects degraded (1908.842%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T20:31:37.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:36 smithi149 ceph-mon[170394]: pgmap v680: 119 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait, 2 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 29 MiB data, 3.2 GiB used, 623 GiB / 626 GiB avail; 24395/1278 objects degraded (1908.842%); 35/1278 objects misplaced (2.739%); 689 KiB/s, 483 objects/s recovering 2023-09-23T20:31:37.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:36 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 24395/1278 objects degraded (1908.842%), 9 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T20:31:37.983 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:31:37.986 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:37.987 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:31:38.087 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:31:38.125 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:38.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:38 smithi018 ceph-mon[184144]: osdmap e300: 8 total, 8 up, 7 in 2023-09-23T20:31:38.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:38 smithi018 ceph-mon[184144]: failed to encode map e300 with expected crc 2023-09-23T20:31:38.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:38 smithi018 ceph-mon[184144]: failed to encode map e300 with expected crc 2023-09-23T20:31:38.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:38 smithi018 ceph-mon[184144]: failed to encode map e300 with expected crc 2023-09-23T20:31:38.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:38 smithi018 ceph-mon[184144]: failed to encode map e300 with expected crc 2023-09-23T20:31:38.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:38 smithi018 ceph-mon[179647]: osdmap e300: 8 total, 8 up, 7 in 2023-09-23T20:31:38.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:38 smithi018 ceph-mon[179647]: failed to encode map e300 with expected crc 2023-09-23T20:31:38.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:38 smithi018 ceph-mon[179647]: failed to encode map e300 with expected crc 2023-09-23T20:31:38.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:38 smithi018 ceph-mon[179647]: failed to encode map e300 with expected crc 2023-09-23T20:31:38.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:38 smithi018 ceph-mon[179647]: failed to encode map e300 with expected crc 2023-09-23T20:31:38.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:38 smithi149 ceph-mon[170394]: osdmap e300: 8 total, 8 up, 7 in 2023-09-23T20:31:38.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:38 smithi149 ceph-mon[170394]: failed to encode map e300 with expected crc 2023-09-23T20:31:38.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:38 smithi149 ceph-mon[170394]: failed to encode map e300 with expected crc 2023-09-23T20:31:38.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:38 smithi149 ceph-mon[170394]: failed to encode map e300 with expected crc 2023-09-23T20:31:38.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:38 smithi149 ceph-mon[170394]: failed to encode map e300 with expected crc 2023-09-23T20:31:39.166 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:31:39.168 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:39.169 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:31:39.269 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:31:39.300 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:39.382 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:39 smithi018 ceph-mon[184144]: pgmap v682: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 2 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 29 MiB data, 3.2 GiB used, 623 GiB / 626 GiB avail; 24387/1278 objects degraded (1908.216%); 35/1278 objects misplaced (2.739%); 682 KiB/s, 479 objects/s recovering 2023-09-23T20:31:39.382 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:39 smithi018 ceph-mon[184144]: osdmap e301: 8 total, 8 up, 7 in 2023-09-23T20:31:39.382 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:39 smithi018 ceph-mon[184144]: failed to encode map e301 with expected crc 2023-09-23T20:31:39.383 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:39 smithi018 ceph-mon[184144]: failed to encode map e301 with expected crc 2023-09-23T20:31:39.383 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:39 smithi018 ceph-mon[184144]: failed to encode map e301 with expected crc 2023-09-23T20:31:39.383 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:39 smithi018 ceph-mon[184144]: failed to encode map e301 with expected crc 2023-09-23T20:31:39.383 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:39 smithi018 ceph-mon[184144]: failed to encode map e301 with expected crc 2023-09-23T20:31:39.383 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:39 smithi018 ceph-mon[184144]: failed to encode map e301 with expected crc 2023-09-23T20:31:39.383 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:39 smithi018 ceph-mon[179647]: pgmap v682: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 2 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 29 MiB data, 3.2 GiB used, 623 GiB / 626 GiB avail; 24387/1278 objects degraded (1908.216%); 35/1278 objects misplaced (2.739%); 682 KiB/s, 479 objects/s recovering 2023-09-23T20:31:39.383 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:39 smithi018 ceph-mon[179647]: osdmap e301: 8 total, 8 up, 7 in 2023-09-23T20:31:39.383 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:39 smithi018 ceph-mon[179647]: failed to encode map e301 with expected crc 2023-09-23T20:31:39.383 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:39 smithi018 ceph-mon[179647]: failed to encode map e301 with expected crc 2023-09-23T20:31:39.384 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:39 smithi018 ceph-mon[179647]: failed to encode map e301 with expected crc 2023-09-23T20:31:39.384 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:39 smithi018 ceph-mon[179647]: failed to encode map e301 with expected crc 2023-09-23T20:31:39.384 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:39 smithi018 ceph-mon[179647]: failed to encode map e301 with expected crc 2023-09-23T20:31:39.384 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:39 smithi018 ceph-mon[179647]: failed to encode map e301 with expected crc 2023-09-23T20:31:39.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:39 smithi149 ceph-mon[170394]: pgmap v682: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 2 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 29 MiB data, 3.2 GiB used, 623 GiB / 626 GiB avail; 24387/1278 objects degraded (1908.216%); 35/1278 objects misplaced (2.739%); 682 KiB/s, 479 objects/s recovering 2023-09-23T20:31:39.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:39 smithi149 ceph-mon[170394]: osdmap e301: 8 total, 8 up, 7 in 2023-09-23T20:31:39.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:39 smithi149 ceph-mon[170394]: failed to encode map e301 with expected crc 2023-09-23T20:31:39.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:39 smithi149 ceph-mon[170394]: failed to encode map e301 with expected crc 2023-09-23T20:31:39.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:39 smithi149 ceph-mon[170394]: failed to encode map e301 with expected crc 2023-09-23T20:31:39.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:39 smithi149 ceph-mon[170394]: failed to encode map e301 with expected crc 2023-09-23T20:31:39.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:39 smithi149 ceph-mon[170394]: failed to encode map e301 with expected crc 2023-09-23T20:31:39.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:39 smithi149 ceph-mon[170394]: failed to encode map e301 with expected crc 2023-09-23T20:31:39.593 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:31:39.598 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:39.599 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:31:39.621 INFO:teuthology.orchestra.run.smithi018.stdout:{"pg_ready":true,"pg_map":{"version":679,"stamp":"2023-09-23T20:31:33.724053+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":30827094,"num_objects":426,"num_object_clones":0,"num_object_copies":1278,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":25814,"num_objects_misplaced":35,"num_objects_unfound":0,"num_objects_dirty":426,"num_whiteouts":0,"num_read":6210,"num_read_kb":13614,"num_write":210487,"num_write_kb":6716893,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28149,"num_bytes_recovered":1160575200,"num_keys_recovered":30,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":185350,"ondisk_log_size":185350,"up":357,"acting":346,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":371,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":3377740,"kb_used_data":1483540,"kb_used_omap":32,"kb_used_meta":1894111,"kb_avail":652838324,"statfs":{"total":671965249536,"available":668506443776,"internally_reserved":0,"allocated":1519144960,"data_stored":1510339983,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32773,"internal_metadata":1939570683},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-1172,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1172,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"4.000977"},"pg_stats":[{"pgid":"3.1a","version":"69'1","reported_seq":39,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-09-23T20:31:15.379897+0000","last_change":"2023-09-23T20:29:46.050662+0000","last_active":"2023-09-23T20:31:15.379897+0000","last_peered":"2023-09-23T20:31:15.379897+0000","last_clean":"2023-09-23T20:31:15.379897+0000","last_became_active":"2023-09-23T20:29:46.050281+0000","last_became_peered":"2023-09-23T20:29:46.050281+0000","last_unstale":"2023-09-23T20:31:15.379897+0000","last_undegraded":"2023-09-23T20:31:15.379897+0000","last_fullsized":"2023-09-23T20:31:15.379897+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:55.227008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":172,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-09-23T20:31:31.806973+0000","last_change":"2023-09-23T20:29:46.067085+0000","last_active":"2023-09-23T20:31:31.806973+0000","last_peered":"2023-09-23T20:31:31.806973+0000","last_clean":"2023-09-23T20:31:31.806973+0000","last_became_active":"2023-09-23T20:29:46.066738+0000","last_became_peered":"2023-09-23T20:29:46.066738+0000","last_unstale":"2023-09-23T20:31:31.806973+0000","last_undegraded":"2023-09-23T20:31:31.806973+0000","last_fullsized":"2023-09-23T20:31:31.806973+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:33:14.679402+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":231,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-09-23T20:31:15.381436+0000","last_change":"2023-09-23T20:29:46.178684+0000","last_active":"2023-09-23T20:31:15.381436+0000","last_peered":"2023-09-23T20:31:15.381436+0000","last_clean":"2023-09-23T20:31:15.381436+0000","last_became_active":"2023-09-23T20:29:46.178249+0000","last_became_peered":"2023-09-23T20:29:46.178249+0000","last_unstale":"2023-09-23T20:31:15.381436+0000","last_undegraded":"2023-09-23T20:31:15.381436+0000","last_fullsized":"2023-09-23T20:31:15.381436+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:58:58.530687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":260,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T20:31:23.586402+0000","last_change":"2023-09-23T20:25:02.957185+0000","last_active":"2023-09-23T20:31:23.586402+0000","last_peered":"2023-09-23T20:31:23.586402+0000","last_clean":"2023-09-23T20:31:23.586402+0000","last_became_active":"2023-09-23T20:25:02.956689+0000","last_became_peered":"2023-09-23T20:25:02.956689+0000","last_unstale":"2023-09-23T20:31:23.586402+0000","last_undegraded":"2023-09-23T20:31:23.586402+0000","last_fullsized":"2023-09-23T20:31:23.586402+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:04:28.524015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":161,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-09-23T20:31:31.805724+0000","last_change":"2023-09-23T20:29:45.529762+0000","last_active":"2023-09-23T20:31:31.805724+0000","last_peered":"2023-09-23T20:31:31.805724+0000","last_clean":"2023-09-23T20:31:31.805724+0000","last_became_active":"2023-09-23T20:29:45.529283+0000","last_became_peered":"2023-09-23T20:29:45.529283+0000","last_unstale":"2023-09-23T20:31:31.805724+0000","last_undegraded":"2023-09-23T20:31:31.805724+0000","last_fullsized":"2023-09-23T20:31:31.805724+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:54:36.720075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":69,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-09-23T20:31:28.933716+0000","last_change":"2023-09-23T20:29:46.112977+0000","last_active":"2023-09-23T20:31:28.933716+0000","last_peered":"2023-09-23T20:31:28.933716+0000","last_clean":"2023-09-23T20:31:28.933716+0000","last_became_active":"2023-09-23T20:29:46.112600+0000","last_became_peered":"2023-09-23T20:29:46.112600+0000","last_unstale":"2023-09-23T20:31:28.933716+0000","last_undegraded":"2023-09-23T20:31:28.933716+0000","last_fullsized":"2023-09-23T20:31:28.933716+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:06:34.132247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.12","version":"293'9383","reported_seq":15984,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-09-23T20:31:31.808417+0000","last_change":"2023-09-23T20:30:46.961845+0000","last_active":"2023-09-23T20:31:31.808417+0000","last_peered":"2023-09-23T20:31:31.808417+0000","last_clean":"2023-09-23T20:31:31.808417+0000","last_became_active":"2023-09-23T20:30:46.939470+0000","last_became_peered":"2023-09-23T20:30:46.939470+0000","last_unstale":"2023-09-23T20:31:31.808417+0000","last_undegraded":"2023-09-23T20:31:31.808417+0000","last_fullsized":"2023-09-23T20:31:31.808417+0000","mapping_epoch":284,"log_start":"227'3756","ondisk_log_start":"227'3756","created":242,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":5627,"log_dups_size":0,"ondisk_log_size":5627,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":5898240,"num_objects":90,"num_object_clones":0,"num_object_copies":270,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":90,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6572,"num_write_kb":213184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3241,"num_bytes_recovered":105185280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":258,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T20:31:23.586581+0000","last_change":"2023-09-23T20:25:02.969432+0000","last_active":"2023-09-23T20:31:23.586581+0000","last_peered":"2023-09-23T20:31:23.586581+0000","last_clean":"2023-09-23T20:31:23.586581+0000","last_became_active":"2023-09-23T20:25:02.969118+0000","last_became_peered":"2023-09-23T20:25:02.969118+0000","last_unstale":"2023-09-23T20:31:23.586581+0000","last_undegraded":"2023-09-23T20:31:23.586581+0000","last_fullsized":"2023-09-23T20:31:23.586581+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:02:22.453831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":181,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-09-23T20:31:31.805811+0000","last_change":"2023-09-23T20:29:46.069832+0000","last_active":"2023-09-23T20:31:31.805811+0000","last_peered":"2023-09-23T20:31:31.805811+0000","last_clean":"2023-09-23T20:31:31.805811+0000","last_became_active":"2023-09-23T20:29:46.069451+0000","last_became_peered":"2023-09-23T20:29:46.069451+0000","last_unstale":"2023-09-23T20:31:31.805811+0000","last_undegraded":"2023-09-23T20:31:31.805811+0000","last_fullsized":"2023-09-23T20:31:31.805811+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:06:06.104030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.13","version":"293'9279","reported_seq":9650,"reported_epoch":295,"state":"active+recovery_wait","last_fresh":"2023-09-23T20:31:28.784703+0000","last_change":"2023-09-23T20:30:30.361943+0000","last_active":"2023-09-23T20:31:28.784703+0000","last_peered":"2023-09-23T20:31:28.784703+0000","last_clean":"2023-09-23T20:28:04.010682+0000","last_became_active":"2023-09-23T20:29:47.054756+0000","last_became_peered":"2023-09-23T20:29:47.054756+0000","last_unstale":"2023-09-23T20:31:28.784703+0000","last_undegraded":"2023-09-23T20:31:28.784703+0000","last_fullsized":"2023-09-23T20:31:28.784703+0000","mapping_epoch":268,"log_start":"222'2005","ondisk_log_start":"222'2005","created":242,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":7274,"log_dups_size":0,"ondisk_log_size":7274,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4489216,"num_objects":69,"num_object_clones":0,"num_object_copies":207,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":69,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6511,"num_write_kb":210528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":117,"num_bytes_recovered":7503872,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":["7","2","6"],"object_location_counts":[{"shards":"2,6,7","objects":69}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.14","version":"90'161","reported_seq":568,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-09-23T20:31:30.798983+0000","last_change":"2023-09-23T20:29:48.092787+0000","last_active":"2023-09-23T20:31:30.798983+0000","last_peered":"2023-09-23T20:31:30.798983+0000","last_clean":"2023-09-23T20:31:30.798983+0000","last_became_active":"2023-09-23T20:29:48.092460+0000","last_became_peered":"2023-09-23T20:29:48.092460+0000","last_unstale":"2023-09-23T20:31:30.798983+0000","last_undegraded":"2023-09-23T20:31:30.798983+0000","last_fullsized":"2023-09-23T20:31:30.798983+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:13.244651+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.244651+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0019955509999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109227,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":272,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-09-23T20:31:28.779845+0000","last_change":"2023-09-23T20:29:45.523808+0000","last_active":"2023-09-23T20:31:28.779845+0000","last_peered":"2023-09-23T20:31:28.779845+0000","last_clean":"2023-09-23T20:31:28.779845+0000","last_became_active":"2023-09-23T20:29:45.523508+0000","last_became_peered":"2023-09-23T20:29:45.523508+0000","last_unstale":"2023-09-23T20:31:28.779845+0000","last_undegraded":"2023-09-23T20:31:28.779845+0000","last_fullsized":"2023-09-23T20:31:28.779845+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:50:55.565352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":164,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-09-23T20:31:31.805812+0000","last_change":"2023-09-23T20:29:04.130512+0000","last_active":"2023-09-23T20:31:31.805812+0000","last_peered":"2023-09-23T20:31:31.805812+0000","last_clean":"2023-09-23T20:31:31.805812+0000","last_became_active":"2023-09-23T20:29:04.130135+0000","last_became_peered":"2023-09-23T20:29:04.130135+0000","last_unstale":"2023-09-23T20:31:31.805812+0000","last_undegraded":"2023-09-23T20:31:31.805812+0000","last_fullsized":"2023-09-23T20:31:31.805812+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:53:59.385893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.10","version":"293'9384","reported_seq":9483,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T20:31:23.585893+0000","last_change":"2023-09-23T20:30:24.864928+0000","last_active":"2023-09-23T20:31:23.585893+0000","last_peered":"2023-09-23T20:31:23.585893+0000","last_clean":"2023-09-23T20:31:23.585893+0000","last_became_active":"2023-09-23T20:28:35.582165+0000","last_became_peered":"2023-09-23T20:28:35.582165+0000","last_unstale":"2023-09-23T20:31:23.585893+0000","last_undegraded":"2023-09-23T20:31:23.585893+0000","last_fullsized":"2023-09-23T20:31:23.585893+0000","mapping_epoch":242,"log_start":"221'337","ondisk_log_start":"221'337","created":242,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":9047,"log_dups_size":0,"ondisk_log_size":9047,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":884736,"num_objects":14,"num_object_clones":0,"num_object_copies":42,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":14,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6566,"num_write_kb":210528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":307,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-09-23T20:31:28.785253+0000","last_change":"2023-09-23T20:29:59.501260+0000","last_active":"2023-09-23T20:31:28.785253+0000","last_peered":"2023-09-23T20:31:28.785253+0000","last_clean":"2023-09-23T20:31:28.785253+0000","last_became_active":"2023-09-23T20:29:28.362070+0000","last_became_peered":"2023-09-23T20:29:28.362070+0000","last_unstale":"2023-09-23T20:31:28.785253+0000","last_undegraded":"2023-09-23T20:31:28.785253+0000","last_fullsized":"2023-09-23T20:31:28.785253+0000","mapping_epoch":258,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:32:55.983696+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":241,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-09-23T20:31:15.380235+0000","last_change":"2023-09-23T20:29:46.080766+0000","last_active":"2023-09-23T20:31:15.380235+0000","last_peered":"2023-09-23T20:31:15.380235+0000","last_clean":"2023-09-23T20:31:15.380235+0000","last_became_active":"2023-09-23T20:29:46.080356+0000","last_became_peered":"2023-09-23T20:29:46.080356+0000","last_unstale":"2023-09-23T20:31:15.380235+0000","last_undegraded":"2023-09-23T20:31:15.380235+0000","last_fullsized":"2023-09-23T20:31:15.380235+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:08:01.862270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.11","version":"293'9220","reported_seq":17810,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-09-23T20:31:20.886623+0000","last_change":"2023-09-23T20:31:15.416292+0000","last_active":"2023-09-23T20:31:20.886623+0000","last_peered":"2023-09-23T20:31:20.886623+0000","last_clean":"2023-09-23T20:31:20.886623+0000","last_became_active":"2023-09-23T20:31:15.389067+0000","last_became_peered":"2023-09-23T20:31:15.389067+0000","last_unstale":"2023-09-23T20:31:20.886623+0000","last_undegraded":"2023-09-23T20:31:20.886623+0000","last_fullsized":"2023-09-23T20:31:20.886623+0000","mapping_epoch":292,"log_start":"223'2784","ondisk_log_start":"223'2784","created":242,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":6436,"log_dups_size":0,"ondisk_log_size":6436,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131072,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6466,"num_write_kb":206976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3876,"num_bytes_recovered":160956416,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":113,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-09-23T20:31:28.785089+0000","last_change":"2023-09-23T20:29:59.529392+0000","last_active":"2023-09-23T20:31:28.785089+0000","last_peered":"2023-09-23T20:31:28.785089+0000","last_clean":"2023-09-23T20:31:28.785089+0000","last_became_active":"2023-09-23T20:29:28.365579+0000","last_became_peered":"2023-09-23T20:29:28.365579+0000","last_unstale":"2023-09-23T20:31:28.785089+0000","last_undegraded":"2023-09-23T20:31:28.785089+0000","last_fullsized":"2023-09-23T20:31:28.785089+0000","mapping_epoch":258,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:31:11.546083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":291,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-09-23T20:31:28.933172+0000","last_change":"2023-09-23T20:29:46.091575+0000","last_active":"2023-09-23T20:31:28.933172+0000","last_peered":"2023-09-23T20:31:28.933172+0000","last_clean":"2023-09-23T20:31:28.933172+0000","last_became_active":"2023-09-23T20:29:46.091271+0000","last_became_peered":"2023-09-23T20:29:46.091271+0000","last_unstale":"2023-09-23T20:31:28.933172+0000","last_undegraded":"2023-09-23T20:31:28.933172+0000","last_fullsized":"2023-09-23T20:31:28.933172+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:08:01.594175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.16","version":"293'9215","reported_seq":9326,"reported_epoch":298,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T20:31:31.808716+0000","last_change":"2023-09-23T20:29:29.535020+0000","last_active":"2023-09-23T20:31:31.808716+0000","last_peered":"2023-09-23T20:31:31.808716+0000","last_clean":"2023-09-23T20:29:27.303063+0000","last_became_active":"2023-09-23T20:29:29.522829+0000","last_became_peered":"2023-09-23T20:29:29.522829+0000","last_unstale":"2023-09-23T20:31:31.808716+0000","last_undegraded":"2023-09-23T20:31:31.808716+0000","last_fullsized":"2023-09-23T20:31:31.808716+0000","mapping_epoch":259,"log_start":"226'3525","ondisk_log_start":"226'3525","created":242,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":5690,"log_dups_size":0,"ondisk_log_size":5690,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":2293760,"num_objects":35,"num_object_clones":0,"num_object_copies":105,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":35,"num_objects_unfound":0,"num_objects_dirty":35,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6473,"num_write_kb":208256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,4],"avail_no_missing":["3","4","7"],"object_location_counts":[{"shards":"3,4,7","objects":35}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.11","version":"90'161","reported_seq":511,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T20:31:23.586066+0000","last_change":"2023-09-23T20:30:36.766139+0000","last_active":"2023-09-23T20:31:23.586066+0000","last_peered":"2023-09-23T20:31:23.586066+0000","last_clean":"2023-09-23T20:31:23.586066+0000","last_became_active":"2023-09-23T20:28:05.991304+0000","last_became_peered":"2023-09-23T20:28:05.991304+0000","last_unstale":"2023-09-23T20:31:23.586066+0000","last_undegraded":"2023-09-23T20:31:23.586066+0000","last_fullsized":"2023-09-23T20:31:23.586066+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:34.821789+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:34.821789+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":306,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-09-23T20:31:28.785142+0000","last_change":"2023-09-23T20:29:59.544701+0000","last_active":"2023-09-23T20:31:28.785142+0000","last_peered":"2023-09-23T20:31:28.785142+0000","last_clean":"2023-09-23T20:31:28.785142+0000","last_became_active":"2023-09-23T20:29:28.367679+0000","last_became_peered":"2023-09-23T20:29:28.367679+0000","last_unstale":"2023-09-23T20:31:28.785142+0000","last_undegraded":"2023-09-23T20:31:28.785142+0000","last_fullsized":"2023-09-23T20:31:28.785142+0000","mapping_epoch":258,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:36:19.089724+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":102,"num_keys_recovered":30,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":161,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-09-23T20:31:31.808051+0000","last_change":"2023-09-23T20:29:45.531343+0000","last_active":"2023-09-23T20:31:31.808051+0000","last_peered":"2023-09-23T20:31:31.808051+0000","last_clean":"2023-09-23T20:31:31.808051+0000","last_became_active":"2023-09-23T20:29:45.530713+0000","last_became_peered":"2023-09-23T20:29:45.530713+0000","last_unstale":"2023-09-23T20:31:31.808051+0000","last_undegraded":"2023-09-23T20:31:31.808051+0000","last_fullsized":"2023-09-23T20:31:31.808051+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:28:00.245911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.17","version":"293'9459","reported_seq":9578,"reported_epoch":295,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:31:28.780536+0000","last_change":"2023-09-23T20:29:46.291295+0000","last_active":"2023-09-23T20:31:28.780536+0000","last_peered":"2023-09-23T20:31:28.780536+0000","last_clean":"2023-09-23T20:29:44.520411+0000","last_became_active":"2023-09-23T20:29:46.269706+0000","last_became_peered":"2023-09-23T20:29:46.269706+0000","last_unstale":"2023-09-23T20:31:28.780536+0000","last_undegraded":"2023-09-23T20:29:46.075283+0000","last_fullsized":"2023-09-23T20:29:46.032679+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":242,"last_epoch_clean":260,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":9459,"log_dups_size":0,"ondisk_log_size":9459,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":688128,"num_objects":11,"num_object_clones":0,"num_object_copies":33,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3310,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":11,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6631,"num_write_kb":212512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3],"avail_no_missing":["5","3","4"],"object_location_counts":[{"shards":"3,5","objects":-1823}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.10","version":"90'161","reported_seq":516,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-09-23T20:31:28.779153+0000","last_change":"2023-09-23T20:29:07.045212+0000","last_active":"2023-09-23T20:31:28.779153+0000","last_peered":"2023-09-23T20:31:28.779153+0000","last_clean":"2023-09-23T20:31:28.779153+0000","last_became_active":"2023-09-23T20:29:07.044892+0000","last_became_peered":"2023-09-23T20:29:07.044892+0000","last_unstale":"2023-09-23T20:31:28.779153+0000","last_undegraded":"2023-09-23T20:31:28.779153+0000","last_fullsized":"2023-09-23T20:31:28.779153+0000","mapping_epoch":253,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":254,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:32.801168+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:32.801168+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":258,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T20:31:23.586842+0000","last_change":"2023-09-23T20:25:02.953993+0000","last_active":"2023-09-23T20:31:23.586842+0000","last_peered":"2023-09-23T20:31:23.586842+0000","last_clean":"2023-09-23T20:31:23.586842+0000","last_became_active":"2023-09-23T20:25:02.953672+0000","last_became_peered":"2023-09-23T20:25:02.953672+0000","last_unstale":"2023-09-23T20:31:23.586842+0000","last_undegraded":"2023-09-23T20:31:23.586842+0000","last_fullsized":"2023-09-23T20:31:23.586842+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:26:08.603577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":274,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-09-23T20:31:28.780024+0000","last_change":"2023-09-23T20:25:02.949584+0000","last_active":"2023-09-23T20:31:28.780024+0000","last_peered":"2023-09-23T20:31:28.780024+0000","last_clean":"2023-09-23T20:31:28.780024+0000","last_became_active":"2023-09-23T20:25:02.949132+0000","last_became_peered":"2023-09-23T20:25:02.949132+0000","last_unstale":"2023-09-23T20:31:28.780024+0000","last_undegraded":"2023-09-23T20:31:28.780024+0000","last_fullsized":"2023-09-23T20:31:28.780024+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:34:20.509823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.14","version":"293'9362","reported_seq":9475,"reported_epoch":298,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:31:31.806771+0000","last_change":"2023-09-23T20:29:46.199482+0000","last_active":"2023-09-23T20:31:31.806771+0000","last_peered":"2023-09-23T20:31:31.806771+0000","last_clean":"2023-09-23T20:28:12.083147+0000","last_became_active":"2023-09-23T20:29:46.185251+0000","last_became_peered":"2023-09-23T20:29:46.185251+0000","last_unstale":"2023-09-23T20:31:31.806771+0000","last_undegraded":"2023-09-23T20:29:46.057476+0000","last_fullsized":"2023-09-23T20:29:46.043200+0000","mapping_epoch":267,"log_start":"227'3862","ondisk_log_start":"227'3862","created":242,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":5500,"log_dups_size":0,"ondisk_log_size":5500,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2460,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6598,"num_write_kb":210176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[3,5],"avail_no_missing":["3","5"],"object_location_counts":[{"shards":"3,5","objects":-1073}],"blocked_by":[],"up_primary":1,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.13","version":"90'161","reported_seq":520,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-09-23T20:31:15.380407+0000","last_change":"2023-09-23T20:29:46.067221+0000","last_active":"2023-09-23T20:31:15.380407+0000","last_peered":"2023-09-23T20:31:15.380407+0000","last_clean":"2023-09-23T20:31:15.380407+0000","last_became_active":"2023-09-23T20:29:46.066622+0000","last_became_peered":"2023-09-23T20:29:46.066622+0000","last_unstale":"2023-09-23T20:31:15.380407+0000","last_undegraded":"2023-09-23T20:31:15.380407+0000","last_fullsized":"2023-09-23T20:31:15.380407+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:12.386098+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:12.386098+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0019500940000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":269,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-09-23T20:31:28.784787+0000","last_change":"2023-09-23T20:29:28.364113+0000","last_active":"2023-09-23T20:31:28.784787+0000","last_peered":"2023-09-23T20:31:28.784787+0000","last_clean":"2023-09-23T20:31:28.784787+0000","last_became_active":"2023-09-23T20:29:28.363788+0000","last_became_peered":"2023-09-23T20:29:28.363788+0000","last_unstale":"2023-09-23T20:31:28.784787+0000","last_undegraded":"2023-09-23T20:31:28.784787+0000","last_fullsized":"2023-09-23T20:31:28.784787+0000","mapping_epoch":258,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":259,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:39:49.658803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":268,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T20:31:23.586396+0000","last_change":"2023-09-23T20:25:02.957010+0000","last_active":"2023-09-23T20:31:23.586396+0000","last_peered":"2023-09-23T20:31:23.586396+0000","last_clean":"2023-09-23T20:31:23.586396+0000","last_became_active":"2023-09-23T20:25:02.956267+0000","last_became_peered":"2023-09-23T20:25:02.956267+0000","last_unstale":"2023-09-23T20:31:23.586396+0000","last_undegraded":"2023-09-23T20:31:23.586396+0000","last_fullsized":"2023-09-23T20:31:23.586396+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:10:55.299422+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.15","version":"293'9180","reported_seq":12982,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-09-23T20:31:28.785038+0000","last_change":"2023-09-23T20:30:01.520686+0000","last_active":"2023-09-23T20:31:28.785038+0000","last_peered":"2023-09-23T20:31:28.785038+0000","last_clean":"2023-09-23T20:31:28.785038+0000","last_became_active":"2023-09-23T20:30:01.486115+0000","last_became_peered":"2023-09-23T20:30:01.486115+0000","last_unstale":"2023-09-23T20:31:28.785038+0000","last_undegraded":"2023-09-23T20:31:28.785038+0000","last_fullsized":"2023-09-23T20:31:28.785038+0000","mapping_epoch":275,"log_start":"222'2004","ondisk_log_start":"222'2004","created":242,"last_epoch_clean":276,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":7176,"log_dups_size":0,"ondisk_log_size":7176,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6440,"num_write_kb":205088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1838,"num_bytes_recovered":97189888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.12","version":"90'161","reported_seq":535,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-09-23T20:31:28.779787+0000","last_change":"2023-09-23T20:29:46.079370+0000","last_active":"2023-09-23T20:31:28.779787+0000","last_peered":"2023-09-23T20:31:28.779787+0000","last_clean":"2023-09-23T20:31:28.779787+0000","last_became_active":"2023-09-23T20:29:46.078873+0000","last_became_peered":"2023-09-23T20:29:46.078873+0000","last_unstale":"2023-09-23T20:31:28.779787+0000","last_undegraded":"2023-09-23T20:31:28.779787+0000","last_fullsized":"2023-09-23T20:31:28.779787+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:35.734584+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:35.734584+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":65,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-09-23T20:31:28.933365+0000","last_change":"2023-09-23T20:29:04.179171+0000","last_active":"2023-09-23T20:31:28.933365+0000","last_peered":"2023-09-23T20:31:28.933365+0000","last_clean":"2023-09-23T20:31:28.933365+0000","last_became_active":"2023-09-23T20:29:04.128672+0000","last_became_peered":"2023-09-23T20:29:04.128672+0000","last_unstale":"2023-09-23T20:31:28.933365+0000","last_undegraded":"2023-09-23T20:31:28.933365+0000","last_fullsized":"2023-09-23T20:31:28.933365+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:49:15.506106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":161,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-09-23T20:31:31.810278+0000","last_change":"2023-09-23T20:29:45.528782+0000","last_active":"2023-09-23T20:31:31.810278+0000","last_peered":"2023-09-23T20:31:31.810278+0000","last_clean":"2023-09-23T20:31:31.810278+0000","last_became_active":"2023-09-23T20:29:45.528389+0000","last_became_peered":"2023-09-23T20:29:45.528389+0000","last_unstale":"2023-09-23T20:31:31.810278+0000","last_undegraded":"2023-09-23T20:31:31.810278+0000","last_fullsized":"2023-09-23T20:31:31.810278+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:13:41.453499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.d","version":"90'161","reported_seq":535,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-09-23T20:31:28.785069+0000","last_change":"2023-09-23T20:29:46.061373+0000","last_active":"2023-09-23T20:31:28.785069+0000","last_peered":"2023-09-23T20:31:28.785069+0000","last_clean":"2023-09-23T20:31:28.785069+0000","last_became_active":"2023-09-23T20:29:46.061003+0000","last_became_peered":"2023-09-23T20:29:46.061003+0000","last_unstale":"2023-09-23T20:31:28.785069+0000","last_undegraded":"2023-09-23T20:31:28.785069+0000","last_fullsized":"2023-09-23T20:31:28.785069+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:12.809933+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:12.809933+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.a","version":"293'13170","reported_seq":24200,"reported_epoch":295,"state":"active+recovery_wait","last_fresh":"2023-09-23T20:31:28.779601+0000","last_change":"2023-09-23T20:29:51.518803+0000","last_active":"2023-09-23T20:31:28.779601+0000","last_peered":"2023-09-23T20:31:28.779601+0000","last_clean":"2023-09-23T20:28:04.001969+0000","last_became_active":"2023-09-23T20:29:51.450026+0000","last_became_peered":"2023-09-23T20:29:51.450026+0000","last_unstale":"2023-09-23T20:31:28.779601+0000","last_undegraded":"2023-09-23T20:31:28.779601+0000","last_fullsized":"2023-09-23T20:31:28.779601+0000","mapping_epoch":271,"log_start":"247'6471","ondisk_log_start":"247'6471","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":6699,"log_dups_size":0,"ondisk_log_size":6699,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:55:12.986375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13170,"num_write_kb":421440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5457,"num_bytes_recovered":313655296,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":["5","0","6"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":285,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-09-23T20:31:28.785415+0000","last_change":"2023-09-23T20:29:59.586041+0000","last_active":"2023-09-23T20:31:28.785415+0000","last_peered":"2023-09-23T20:31:28.785415+0000","last_clean":"2023-09-23T20:31:28.785415+0000","last_became_active":"2023-09-23T20:29:46.059906+0000","last_became_peered":"2023-09-23T20:29:46.059906+0000","last_unstale":"2023-09-23T20:31:28.785415+0000","last_undegraded":"2023-09-23T20:31:28.785415+0000","last_fullsized":"2023-09-23T20:31:28.785415+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:26:12.412320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":257,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-09-23T20:31:28.779276+0000","last_change":"2023-09-23T20:29:04.128604+0000","last_active":"2023-09-23T20:31:28.779276+0000","last_peered":"2023-09-23T20:31:28.779276+0000","last_clean":"2023-09-23T20:31:28.779276+0000","last_became_active":"2023-09-23T20:29:04.128003+0000","last_became_peered":"2023-09-23T20:29:04.128003+0000","last_unstale":"2023-09-23T20:31:28.779276+0000","last_undegraded":"2023-09-23T20:31:28.779276+0000","last_fullsized":"2023-09-23T20:31:28.779276+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:50:49.385599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"90'161","reported_seq":582,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-09-23T20:31:31.807219+0000","last_change":"2023-09-23T20:30:01.493499+0000","last_active":"2023-09-23T20:31:31.807219+0000","last_peered":"2023-09-23T20:31:31.807219+0000","last_clean":"2023-09-23T20:31:31.807219+0000","last_became_active":"2023-09-23T20:30:01.493025+0000","last_became_peered":"2023-09-23T20:30:01.493025+0000","last_unstale":"2023-09-23T20:31:31.807219+0000","last_undegraded":"2023-09-23T20:31:31.807219+0000","last_fullsized":"2023-09-23T20:31:31.807219+0000","mapping_epoch":275,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":276,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:13.244651+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.244651+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0019955509999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109227,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":655360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.b","version":"293'12796","reported_seq":12900,"reported_epoch":298,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:31:31.807730+0000","last_change":"2023-09-23T20:29:46.285273+0000","last_active":"2023-09-23T20:31:31.807730+0000","last_peered":"2023-09-23T20:31:31.807730+0000","last_clean":"2023-09-23T20:28:12.089537+0000","last_became_active":"2023-09-23T20:29:46.255023+0000","last_became_peered":"2023-09-23T20:29:46.255023+0000","last_unstale":"2023-09-23T20:31:31.807730+0000","last_undegraded":"2023-09-23T20:29:46.058377+0000","last_fullsized":"2023-09-23T20:29:46.031666+0000","mapping_epoch":267,"log_start":"257'8029","ondisk_log_start":"257'8029","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":4767,"log_dups_size":0,"ondisk_log_size":4767,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:37:55.158131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4695,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12796,"num_write_kb":409472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":-1958}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":292,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-09-23T20:31:28.785364+0000","last_change":"2023-09-23T20:29:59.567502+0000","last_active":"2023-09-23T20:31:28.785364+0000","last_peered":"2023-09-23T20:31:28.785364+0000","last_clean":"2023-09-23T20:31:28.785364+0000","last_became_active":"2023-09-23T20:29:46.055041+0000","last_became_peered":"2023-09-23T20:29:46.055041+0000","last_unstale":"2023-09-23T20:31:28.785364+0000","last_undegraded":"2023-09-23T20:31:28.785364+0000","last_fullsized":"2023-09-23T20:31:28.785364+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:47:13.874812+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":191,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-09-23T20:31:15.381283+0000","last_change":"2023-09-23T20:29:46.178516+0000","last_active":"2023-09-23T20:31:15.381283+0000","last_peered":"2023-09-23T20:31:15.381283+0000","last_clean":"2023-09-23T20:31:15.381283+0000","last_became_active":"2023-09-23T20:29:46.177858+0000","last_became_peered":"2023-09-23T20:29:46.177858+0000","last_unstale":"2023-09-23T20:31:15.381283+0000","last_undegraded":"2023-09-23T20:31:15.381283+0000","last_fullsized":"2023-09-23T20:31:15.381283+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:09:14.906568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.f","version":"90'161","reported_seq":488,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-09-23T20:31:28.933632+0000","last_change":"2023-09-23T20:29:28.374177+0000","last_active":"2023-09-23T20:31:28.933632+0000","last_peered":"2023-09-23T20:31:28.933632+0000","last_clean":"2023-09-23T20:31:28.933632+0000","last_became_active":"2023-09-23T20:29:28.373750+0000","last_became_peered":"2023-09-23T20:29:28.373750+0000","last_unstale":"2023-09-23T20:31:28.933632+0000","last_undegraded":"2023-09-23T20:31:28.933632+0000","last_fullsized":"2023-09-23T20:31:28.933632+0000","mapping_epoch":258,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":259,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:32.746340+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:32.746340+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,3],"acting":[6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.8","version":"293'9299","reported_seq":9397,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-09-23T20:31:28.784524+0000","last_change":"2023-09-23T20:28:35.581921+0000","last_active":"2023-09-23T20:31:28.784524+0000","last_peered":"2023-09-23T20:31:28.784524+0000","last_clean":"2023-09-23T20:31:28.784524+0000","last_became_active":"2023-09-23T20:28:35.576360+0000","last_became_peered":"2023-09-23T20:28:35.576360+0000","last_unstale":"2023-09-23T20:31:28.784524+0000","last_undegraded":"2023-09-23T20:31:28.784524+0000","last_fullsized":"2023-09-23T20:31:28.784524+0000","mapping_epoch":217,"log_start":"222'2085","ondisk_log_start":"222'2085","created":217,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":7214,"log_dups_size":0,"ondisk_log_size":7214,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1376256,"num_objects":21,"num_object_clones":0,"num_object_copies":63,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":21,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6513,"num_write_kb":209088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":268,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-09-23T20:31:28.785166+0000","last_change":"2023-09-23T20:29:04.184893+0000","last_active":"2023-09-23T20:31:28.785166+0000","last_peered":"2023-09-23T20:31:28.785166+0000","last_clean":"2023-09-23T20:31:28.785166+0000","last_became_active":"2023-09-23T20:29:04.152723+0000","last_became_peered":"2023-09-23T20:29:04.152723+0000","last_unstale":"2023-09-23T20:31:28.785166+0000","last_undegraded":"2023-09-23T20:31:28.785166+0000","last_fullsized":"2023-09-23T20:31:28.785166+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:51:27.017151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":69,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-09-23T20:31:28.933326+0000","last_change":"2023-09-23T20:29:46.086912+0000","last_active":"2023-09-23T20:31:28.933326+0000","last_peered":"2023-09-23T20:31:28.933326+0000","last_clean":"2023-09-23T20:31:28.933326+0000","last_became_active":"2023-09-23T20:29:46.085471+0000","last_became_peered":"2023-09-23T20:29:46.085471+0000","last_unstale":"2023-09-23T20:31:28.933326+0000","last_undegraded":"2023-09-23T20:31:28.933326+0000","last_fullsized":"2023-09-23T20:31:28.933326+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:04:55.751660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.e","version":"90'161","reported_seq":475,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-09-23T20:31:28.785253+0000","last_change":"2023-09-23T20:29:31.394412+0000","last_active":"2023-09-23T20:31:28.785253+0000","last_peered":"2023-09-23T20:31:28.785253+0000","last_clean":"2023-09-23T20:31:28.785253+0000","last_became_active":"2023-09-23T20:29:31.394053+0000","last_became_peered":"2023-09-23T20:29:31.394053+0000","last_unstale":"2023-09-23T20:31:28.785253+0000","last_undegraded":"2023-09-23T20:31:28.785253+0000","last_fullsized":"2023-09-23T20:31:28.785253+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:31.769333+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:31.769333+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"6.9","version":"293'9442","reported_seq":9540,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T20:31:23.586134+0000","last_change":"2023-09-23T20:30:22.842563+0000","last_active":"2023-09-23T20:31:23.586134+0000","last_peered":"2023-09-23T20:31:23.586134+0000","last_clean":"2023-09-23T20:31:23.586134+0000","last_became_active":"2023-09-23T20:28:35.536824+0000","last_became_peered":"2023-09-23T20:28:35.536824+0000","last_unstale":"2023-09-23T20:31:23.586134+0000","last_undegraded":"2023-09-23T20:31:23.586134+0000","last_fullsized":"2023-09-23T20:31:23.586134+0000","mapping_epoch":217,"log_start":"221'690","ondisk_log_start":"221'690","created":217,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":8752,"log_dups_size":0,"ondisk_log_size":8752,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6646,"num_write_kb":209664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":302,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-09-23T20:31:28.779716+0000","last_change":"2023-09-23T20:29:04.138284+0000","last_active":"2023-09-23T20:31:28.779716+0000","last_peered":"2023-09-23T20:31:28.779716+0000","last_clean":"2023-09-23T20:31:28.779716+0000","last_became_active":"2023-09-23T20:29:04.137960+0000","last_became_peered":"2023-09-23T20:29:04.137960+0000","last_unstale":"2023-09-23T20:31:28.779716+0000","last_undegraded":"2023-09-23T20:31:28.779716+0000","last_fullsized":"2023-09-23T20:31:28.779716+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:18:59.548311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":268,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-09-23T20:31:28.933681+0000","last_change":"2023-09-23T20:29:46.061136+0000","last_active":"2023-09-23T20:31:28.933681+0000","last_peered":"2023-09-23T20:31:28.933681+0000","last_clean":"2023-09-23T20:31:28.933681+0000","last_became_active":"2023-09-23T20:29:46.060777+0000","last_became_peered":"2023-09-23T20:29:46.060777+0000","last_unstale":"2023-09-23T20:31:28.933681+0000","last_undegraded":"2023-09-23T20:31:28.933681+0000","last_fullsized":"2023-09-23T20:31:28.933681+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:08:43.673870+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"293'13102","reported_seq":22048,"reported_epoch":294,"state":"active+recovering+undersized+remapped","last_fresh":"2023-09-23T20:31:27.361448+0000","last_change":"2023-09-23T20:30:59.555825+0000","last_active":"2023-09-23T20:31:27.361448+0000","last_peered":"2023-09-23T20:31:27.361448+0000","last_clean":"2023-09-23T20:28:12.106078+0000","last_became_active":"2023-09-23T20:29:45.690954+0000","last_became_peered":"2023-09-23T20:29:45.690954+0000","last_unstale":"2023-09-23T20:31:27.361448+0000","last_undegraded":"2023-09-23T20:31:27.361448+0000","last_fullsized":"2023-09-23T20:29:45.501064+0000","mapping_epoch":266,"log_start":"248'6572","ondisk_log_start":"248'6572","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":6530,"log_dups_size":0,"ondisk_log_size":6530,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:52:59.514150+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13102,"num_write_kb":419264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4421,"num_bytes_recovered":132972544,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[2,4],"avail_no_missing":["2","4","5"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":678,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-09-23T20:31:28.784678+0000","last_change":"2023-09-23T20:28:05.872149+0000","last_active":"2023-09-23T20:31:28.784678+0000","last_peered":"2023-09-23T20:31:28.784678+0000","last_clean":"2023-09-23T20:31:28.784678+0000","last_became_active":"2023-09-23T20:27:11.961310+0000","last_became_peered":"2023-09-23T20:27:11.961310+0000","last_unstale":"2023-09-23T20:31:28.784678+0000","last_undegraded":"2023-09-23T20:31:28.784678+0000","last_fullsized":"2023-09-23T20:31:28.784678+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:28:05.872074+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:28:05.872074+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:52:32.495955+0000","scrub_duration":0.0029215980000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":174,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-09-23T20:31:31.807501+0000","last_change":"2023-09-23T20:29:49.873018+0000","last_active":"2023-09-23T20:31:31.807501+0000","last_peered":"2023-09-23T20:31:31.807501+0000","last_clean":"2023-09-23T20:31:31.807501+0000","last_became_active":"2023-09-23T20:29:46.070369+0000","last_became_peered":"2023-09-23T20:29:46.070369+0000","last_unstale":"2023-09-23T20:31:31.807501+0000","last_undegraded":"2023-09-23T20:31:31.807501+0000","last_fullsized":"2023-09-23T20:31:31.807501+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:24:13.119725+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":310,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-09-23T20:31:31.808337+0000","last_change":"2023-09-23T20:29:46.173195+0000","last_active":"2023-09-23T20:31:31.808337+0000","last_peered":"2023-09-23T20:31:31.808337+0000","last_clean":"2023-09-23T20:31:31.808337+0000","last_became_active":"2023-09-23T20:29:46.172808+0000","last_became_peered":"2023-09-23T20:29:46.172808+0000","last_unstale":"2023-09-23T20:31:31.808337+0000","last_undegraded":"2023-09-23T20:31:31.808337+0000","last_fullsized":"2023-09-23T20:31:31.808337+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:41:58.927658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.f","version":"293'13250","reported_seq":13356,"reported_epoch":298,"state":"peering","last_fresh":"2023-09-23T20:31:31.805812+0000","last_change":"2023-09-23T20:31:31.806027+0000","last_active":"2023-09-23T20:31:31.805812+0000","last_peered":"2023-09-23T20:31:31.536596+0000","last_clean":"2023-09-23T20:31:31.536596+0000","last_became_active":"2023-09-23T20:29:29.505420+0000","last_became_peered":"2023-09-23T20:29:29.505420+0000","last_unstale":"2023-09-23T20:31:31.805812+0000","last_undegraded":"2023-09-23T20:31:31.805812+0000","last_fullsized":"2023-09-23T20:31:31.805812+0000","mapping_epoch":298,"log_start":"248'6687","ondisk_log_start":"248'6687","created":217,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":6563,"log_dups_size":0,"ondisk_log_size":6563,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:03:55.743079+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13250,"num_write_kb":424000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[3,4,6],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":520,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-09-23T20:31:28.779153+0000","last_change":"2023-09-23T20:27:31.757060+0000","last_active":"2023-09-23T20:31:28.779153+0000","last_peered":"2023-09-23T20:31:28.779153+0000","last_clean":"2023-09-23T20:31:28.779153+0000","last_became_active":"2023-09-23T20:27:11.959516+0000","last_became_peered":"2023-09-23T20:27:11.959516+0000","last_unstale":"2023-09-23T20:31:28.779153+0000","last_undegraded":"2023-09-23T20:31:28.779153+0000","last_fullsized":"2023-09-23T20:31:28.779153+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:31.756974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:31.756974+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:35:11.939335+0000","scrub_duration":0.0031802670000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":81,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-09-23T20:31:28.933349+0000","last_change":"2023-09-23T20:29:46.087097+0000","last_active":"2023-09-23T20:31:28.933349+0000","last_peered":"2023-09-23T20:31:28.933349+0000","last_clean":"2023-09-23T20:31:28.933349+0000","last_became_active":"2023-09-23T20:29:46.085908+0000","last_became_peered":"2023-09-23T20:29:46.085908+0000","last_unstale":"2023-09-23T20:31:28.933349+0000","last_undegraded":"2023-09-23T20:31:28.933349+0000","last_fullsized":"2023-09-23T20:31:28.933349+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:38:46.001432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":207,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-09-23T20:31:30.799418+0000","last_change":"2023-09-23T20:29:28.370476+0000","last_active":"2023-09-23T20:31:30.799418+0000","last_peered":"2023-09-23T20:31:30.799418+0000","last_clean":"2023-09-23T20:31:30.799418+0000","last_became_active":"2023-09-23T20:29:28.370158+0000","last_became_peered":"2023-09-23T20:29:28.370158+0000","last_unstale":"2023-09-23T20:31:30.799418+0000","last_undegraded":"2023-09-23T20:31:30.799418+0000","last_fullsized":"2023-09-23T20:31:30.799418+0000","mapping_epoch":258,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":259,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:09:47.335404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.b","version":"90'161","reported_seq":522,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-09-23T20:31:15.380457+0000","last_change":"2023-09-23T20:29:45.534549+0000","last_active":"2023-09-23T20:31:15.380457+0000","last_peered":"2023-09-23T20:31:15.380457+0000","last_clean":"2023-09-23T20:31:15.380457+0000","last_became_active":"2023-09-23T20:29:45.534070+0000","last_became_peered":"2023-09-23T20:29:45.534070+0000","last_unstale":"2023-09-23T20:31:15.380457+0000","last_undegraded":"2023-09-23T20:31:15.380457+0000","last_fullsized":"2023-09-23T20:31:15.380457+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:12.386098+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:12.386098+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0019500940000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.c","version":"293'12910","reported_seq":16414,"reported_epoch":298,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:31:31.807614+0000","last_change":"2023-09-23T20:29:59.529817+0000","last_active":"2023-09-23T20:31:31.807614+0000","last_peered":"2023-09-23T20:31:31.807614+0000","last_clean":"2023-09-23T20:28:03.999709+0000","last_became_active":"2023-09-23T20:29:04.247715+0000","last_became_peered":"2023-09-23T20:29:04.247715+0000","last_unstale":"2023-09-23T20:31:31.807614+0000","last_undegraded":"2023-09-23T20:29:59.527865+0000","last_fullsized":"2023-09-23T20:29:04.105278+0000","mapping_epoch":250,"log_start":"257'8171","ondisk_log_start":"257'8171","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":4739,"log_dups_size":0,"ondisk_log_size":4739,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:01:13.556577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2952,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12910,"num_write_kb":413120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1694,"num_bytes_recovered":110886912,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,5],"avail_no_missing":["3","5"],"object_location_counts":[{"shards":"3,5","objects":-2300}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":209,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-09-23T20:31:30.798864+0000","last_change":"2023-09-23T20:29:46.061620+0000","last_active":"2023-09-23T20:31:30.798864+0000","last_peered":"2023-09-23T20:31:30.798864+0000","last_clean":"2023-09-23T20:31:30.798864+0000","last_became_active":"2023-09-23T20:29:46.061271+0000","last_became_peered":"2023-09-23T20:29:46.061271+0000","last_unstale":"2023-09-23T20:31:30.798864+0000","last_undegraded":"2023-09-23T20:31:30.798864+0000","last_fullsized":"2023-09-23T20:31:30.798864+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:24:51.409521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":291,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-09-23T20:31:28.933543+0000","last_change":"2023-09-23T20:29:46.085962+0000","last_active":"2023-09-23T20:31:28.933543+0000","last_peered":"2023-09-23T20:31:28.933543+0000","last_clean":"2023-09-23T20:31:28.933543+0000","last_became_active":"2023-09-23T20:29:46.082826+0000","last_became_peered":"2023-09-23T20:29:46.082826+0000","last_unstale":"2023-09-23T20:31:28.933543+0000","last_undegraded":"2023-09-23T20:31:28.933543+0000","last_fullsized":"2023-09-23T20:31:28.933543+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:26:17.599051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.d","version":"293'13222","reported_seq":13337,"reported_epoch":295,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:31:28.780283+0000","last_change":"2023-09-23T20:29:59.564467+0000","last_active":"2023-09-23T20:31:28.780283+0000","last_peered":"2023-09-23T20:31:28.780283+0000","last_clean":"2023-09-23T20:28:12.085747+0000","last_became_active":"2023-09-23T20:29:46.294362+0000","last_became_peered":"2023-09-23T20:29:46.294362+0000","last_unstale":"2023-09-23T20:31:28.780283+0000","last_undegraded":"2023-09-23T20:29:46.239165+0000","last_fullsized":"2023-09-23T20:29:46.032265+0000","mapping_epoch":267,"log_start":"247'6459","ondisk_log_start":"247'6459","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":6763,"log_dups_size":0,"ondisk_log_size":6763,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:38:13.504714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4915,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":13222,"num_write_kb":423041,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,0],"avail_no_missing":["5","0"],"object_location_counts":[{"shards":"0,5","objects":-2016}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":560,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-09-23T20:31:28.779576+0000","last_change":"2023-09-23T20:29:07.053561+0000","last_active":"2023-09-23T20:31:28.779576+0000","last_peered":"2023-09-23T20:31:28.779576+0000","last_clean":"2023-09-23T20:31:28.779576+0000","last_became_active":"2023-09-23T20:29:07.053177+0000","last_became_peered":"2023-09-23T20:29:07.053177+0000","last_unstale":"2023-09-23T20:31:28.779576+0000","last_undegraded":"2023-09-23T20:31:28.779576+0000","last_fullsized":"2023-09-23T20:31:28.779576+0000","mapping_epoch":253,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":254,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:30.727267+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:30.727267+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:04:32.661761+0000","scrub_duration":0.0052560289999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":162,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T20:31:32.803871+0000","last_change":"2023-09-23T20:29:45.529576+0000","last_active":"2023-09-23T20:31:32.803871+0000","last_peered":"2023-09-23T20:31:32.803871+0000","last_clean":"2023-09-23T20:31:32.803871+0000","last_became_active":"2023-09-23T20:29:45.528858+0000","last_became_peered":"2023-09-23T20:29:45.528858+0000","last_unstale":"2023-09-23T20:31:32.803871+0000","last_undegraded":"2023-09-23T20:31:32.803871+0000","last_fullsized":"2023-09-23T20:31:32.803871+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:42:15.589821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":104,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-09-23T20:31:15.380786+0000","last_change":"2023-09-23T20:29:46.064515+0000","last_active":"2023-09-23T20:31:15.380786+0000","last_peered":"2023-09-23T20:31:15.380786+0000","last_clean":"2023-09-23T20:31:15.380786+0000","last_became_active":"2023-09-23T20:29:46.064253+0000","last_became_peered":"2023-09-23T20:29:46.064253+0000","last_unstale":"2023-09-23T20:31:15.380786+0000","last_undegraded":"2023-09-23T20:31:15.380786+0000","last_fullsized":"2023-09-23T20:31:15.380786+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:21:20.039440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"293'9495","reported_seq":9595,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T20:31:23.586031+0000","last_change":"2023-09-23T20:30:23.834705+0000","last_active":"2023-09-23T20:31:23.586031+0000","last_peered":"2023-09-23T20:31:23.586031+0000","last_clean":"2023-09-23T20:31:23.586031+0000","last_became_active":"2023-09-23T20:28:35.532298+0000","last_became_peered":"2023-09-23T20:28:35.532298+0000","last_unstale":"2023-09-23T20:31:23.586031+0000","last_undegraded":"2023-09-23T20:31:23.586031+0000","last_fullsized":"2023-09-23T20:31:23.586031+0000","mapping_epoch":217,"log_start":"221'594","ondisk_log_start":"221'594","created":217,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":8901,"log_dups_size":0,"ondisk_log_size":8901,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6676,"num_write_kb":213216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":515,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-09-23T20:31:28.933660+0000","last_change":"2023-09-23T20:29:07.115982+0000","last_active":"2023-09-23T20:31:28.933660+0000","last_peered":"2023-09-23T20:31:28.933660+0000","last_clean":"2023-09-23T20:31:28.933660+0000","last_became_active":"2023-09-23T20:29:07.115647+0000","last_became_peered":"2023-09-23T20:29:07.115647+0000","last_unstale":"2023-09-23T20:31:28.933660+0000","last_undegraded":"2023-09-23T20:31:28.933660+0000","last_fullsized":"2023-09-23T20:31:28.933660+0000","mapping_epoch":253,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":254,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:32.746340+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:32.746340+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"86'119","reported_seq":597,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-09-23T20:31:15.380977+0000","last_change":"2023-09-23T20:29:45.521257+0000","last_active":"2023-09-23T20:31:15.380977+0000","last_peered":"2023-09-23T20:31:15.380977+0000","last_clean":"2023-09-23T20:31:15.380977+0000","last_became_active":"2023-09-23T20:29:45.520546+0000","last_became_peered":"2023-09-23T20:29:45.520546+0000","last_unstale":"2023-09-23T20:31:15.380977+0000","last_undegraded":"2023-09-23T20:31:15.380977+0000","last_fullsized":"2023-09-23T20:31:15.380977+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":119,"log_dups_size":0,"ondisk_log_size":119,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:58:56.085777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032705499999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":523,"num_read_kb":3503,"num_write":136,"num_write_kb":3128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":312,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-09-23T20:31:28.779983+0000","last_change":"2023-09-23T20:25:39.637160+0000","last_active":"2023-09-23T20:31:28.779983+0000","last_peered":"2023-09-23T20:31:28.779983+0000","last_clean":"2023-09-23T20:31:28.779983+0000","last_became_active":"2023-09-23T20:25:39.636860+0000","last_became_peered":"2023-09-23T20:25:39.636860+0000","last_unstale":"2023-09-23T20:31:28.779983+0000","last_undegraded":"2023-09-23T20:31:28.779983+0000","last_fullsized":"2023-09-23T20:31:28.779983+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:21:47.860063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":199,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-09-23T20:31:15.379807+0000","last_change":"2023-09-23T20:29:46.090683+0000","last_active":"2023-09-23T20:31:15.379807+0000","last_peered":"2023-09-23T20:31:15.379807+0000","last_clean":"2023-09-23T20:31:15.379807+0000","last_became_active":"2023-09-23T20:29:46.090131+0000","last_became_peered":"2023-09-23T20:29:46.090131+0000","last_unstale":"2023-09-23T20:31:15.379807+0000","last_undegraded":"2023-09-23T20:31:15.379807+0000","last_fullsized":"2023-09-23T20:31:15.379807+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:36:35.754684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.6","version":"293'9259","reported_seq":9370,"reported_epoch":298,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T20:31:31.809324+0000","last_change":"2023-09-23T20:29:29.535022+0000","last_active":"2023-09-23T20:31:31.809324+0000","last_peered":"2023-09-23T20:31:31.809324+0000","last_clean":"2023-09-23T20:29:27.350688+0000","last_became_active":"2023-09-23T20:29:29.522984+0000","last_became_peered":"2023-09-23T20:29:29.522984+0000","last_unstale":"2023-09-23T20:31:31.809324+0000","last_undegraded":"2023-09-23T20:31:31.809324+0000","last_fullsized":"2023-09-23T20:31:31.809324+0000","mapping_epoch":259,"log_start":"227'3708","ondisk_log_start":"227'3708","created":217,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":5551,"log_dups_size":0,"ondisk_log_size":5551,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6517,"num_write_kb":207424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,4],"avail_no_missing":["3","4","6","7"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":530,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-09-23T20:31:28.779274+0000","last_change":"2023-09-23T20:27:55.213707+0000","last_active":"2023-09-23T20:31:28.779274+0000","last_peered":"2023-09-23T20:31:28.779274+0000","last_clean":"2023-09-23T20:31:28.779274+0000","last_became_active":"2023-09-23T20:27:55.213394+0000","last_became_peered":"2023-09-23T20:27:55.213394+0000","last_unstale":"2023-09-23T20:31:28.779274+0000","last_undegraded":"2023-09-23T20:31:28.779274+0000","last_fullsized":"2023-09-23T20:31:28.779274+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:34.821789+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:34.821789+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0042962870000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"86'106","reported_seq":990,"reported_epoch":295,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T20:31:28.779353+0000","last_change":"2023-09-23T20:29:46.078905+0000","last_active":"2023-09-23T20:31:28.779353+0000","last_peered":"2023-09-23T20:31:28.779353+0000","last_clean":"2023-09-23T20:28:10.123134+0000","last_became_active":"2023-09-23T20:29:46.078501+0000","last_became_peered":"2023-09-23T20:29:46.078501+0000","last_unstale":"2023-09-23T20:31:28.779353+0000","last_undegraded":"2023-09-23T20:29:46.032390+0000","last_fullsized":"2023-09-23T20:31:28.779353+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":106,"log_dups_size":0,"ondisk_log_size":106,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:37.015974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00099302599999999993,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1105,"num_read_kb":1689,"num_write":100,"num_write_kb":404,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":["5","0","1"],"object_location_counts":[{"shards":"0,1,5","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":287,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T20:31:23.586726+0000","last_change":"2023-09-23T20:29:46.059640+0000","last_active":"2023-09-23T20:31:23.586726+0000","last_peered":"2023-09-23T20:31:23.586726+0000","last_clean":"2023-09-23T20:31:23.586726+0000","last_became_active":"2023-09-23T20:29:46.059279+0000","last_became_peered":"2023-09-23T20:29:46.059279+0000","last_unstale":"2023-09-23T20:31:23.586726+0000","last_undegraded":"2023-09-23T20:31:23.586726+0000","last_fullsized":"2023-09-23T20:31:23.586726+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:03:31.536847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":250,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-09-23T20:31:15.380892+0000","last_change":"2023-09-23T20:29:45.533406+0000","last_active":"2023-09-23T20:31:15.380892+0000","last_peered":"2023-09-23T20:31:15.380892+0000","last_clean":"2023-09-23T20:31:15.380892+0000","last_became_active":"2023-09-23T20:29:45.526001+0000","last_became_peered":"2023-09-23T20:29:45.526001+0000","last_unstale":"2023-09-23T20:31:15.380892+0000","last_undegraded":"2023-09-23T20:31:15.380892+0000","last_fullsized":"2023-09-23T20:31:15.380892+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:43.787251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.5","version":"293'9247","reported_seq":12842,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-09-23T20:31:28.785141+0000","last_change":"2023-09-23T20:30:57.151375+0000","last_active":"2023-09-23T20:31:28.785141+0000","last_peered":"2023-09-23T20:31:28.785141+0000","last_clean":"2023-09-23T20:31:28.785141+0000","last_became_active":"2023-09-23T20:30:57.098276+0000","last_became_peered":"2023-09-23T20:30:57.098276+0000","last_unstale":"2023-09-23T20:31:28.785141+0000","last_undegraded":"2023-09-23T20:31:28.785141+0000","last_fullsized":"2023-09-23T20:31:28.785141+0000","mapping_epoch":288,"log_start":"222'2060","ondisk_log_start":"222'2060","created":217,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":7187,"log_dups_size":0,"ondisk_log_size":7187,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1998848,"num_objects":30,"num_object_clones":0,"num_object_copies":90,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":30,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6506,"num_write_kb":209184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1733,"num_bytes_recovered":70320128,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":708,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-09-23T20:31:28.784742+0000","last_change":"2023-09-23T20:29:31.389645+0000","last_active":"2023-09-23T20:31:28.784742+0000","last_peered":"2023-09-23T20:31:28.784742+0000","last_clean":"2023-09-23T20:31:28.784742+0000","last_became_active":"2023-09-23T20:29:31.389288+0000","last_became_peered":"2023-09-23T20:29:31.389288+0000","last_unstale":"2023-09-23T20:31:28.784742+0000","last_undegraded":"2023-09-23T20:31:28.784742+0000","last_fullsized":"2023-09-23T20:31:28.784742+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:35.734584+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:35.734584+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0035251380000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,2],"acting":[7,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"86'130","reported_seq":609,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-09-23T20:31:30.798846+0000","last_change":"2023-09-23T20:25:39.637458+0000","last_active":"2023-09-23T20:31:30.798846+0000","last_peered":"2023-09-23T20:31:30.798846+0000","last_clean":"2023-09-23T20:31:30.798846+0000","last_became_active":"2023-09-23T20:25:39.636773+0000","last_became_peered":"2023-09-23T20:25:39.636773+0000","last_unstale":"2023-09-23T20:31:30.798846+0000","last_undegraded":"2023-09-23T20:31:30.798846+0000","last_fullsized":"2023-09-23T20:31:30.798846+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:00:32.810564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011300049999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":512,"num_read_kb":1820,"num_write":133,"num_write_kb":1347,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":240,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-09-23T20:31:15.378902+0000","last_change":"2023-09-23T20:29:46.073394+0000","last_active":"2023-09-23T20:31:15.378902+0000","last_peered":"2023-09-23T20:31:15.378902+0000","last_clean":"2023-09-23T20:31:15.378902+0000","last_became_active":"2023-09-23T20:29:46.072067+0000","last_became_peered":"2023-09-23T20:29:46.072067+0000","last_unstale":"2023-09-23T20:31:15.378902+0000","last_undegraded":"2023-09-23T20:31:15.378902+0000","last_fullsized":"2023-09-23T20:31:15.378902+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:01:56.944614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":232,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-09-23T20:31:15.380702+0000","last_change":"2023-09-23T20:29:45.533185+0000","last_active":"2023-09-23T20:31:15.380702+0000","last_peered":"2023-09-23T20:31:15.380702+0000","last_clean":"2023-09-23T20:31:15.380702+0000","last_became_active":"2023-09-23T20:29:45.526765+0000","last_became_peered":"2023-09-23T20:29:45.526765+0000","last_unstale":"2023-09-23T20:31:15.380702+0000","last_undegraded":"2023-09-23T20:31:15.380702+0000","last_fullsized":"2023-09-23T20:31:15.380702+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:18:33.063215+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"293'9418","reported_seq":9538,"reported_epoch":295,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:31:28.780688+0000","last_change":"2023-09-23T20:29:46.386541+0000","last_active":"2023-09-23T20:31:28.780688+0000","last_peered":"2023-09-23T20:31:28.780688+0000","last_clean":"2023-09-23T20:29:27.161343+0000","last_became_active":"2023-09-23T20:29:46.365588+0000","last_became_peered":"2023-09-23T20:29:46.365588+0000","last_unstale":"2023-09-23T20:31:28.780688+0000","last_undegraded":"2023-09-23T20:29:46.092299+0000","last_fullsized":"2023-09-23T20:29:46.032978+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":217,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":9418,"log_dups_size":0,"ondisk_log_size":9418,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3300,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6589,"num_write_kb":210528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3],"avail_no_missing":["5","3","4"],"object_location_counts":[{"shards":"3,5","objects":-1850}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"86'194","reported_seq":944,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-09-23T20:31:28.779568+0000","last_change":"2023-09-23T20:25:40.618665+0000","last_active":"2023-09-23T20:31:28.779568+0000","last_peered":"2023-09-23T20:31:28.779568+0000","last_clean":"2023-09-23T20:31:28.779568+0000","last_became_active":"2023-09-23T20:25:40.618087+0000","last_became_peered":"2023-09-23T20:25:40.618087+0000","last_unstale":"2023-09-23T20:31:28.779568+0000","last_undegraded":"2023-09-23T20:31:28.779568+0000","last_fullsized":"2023-09-23T20:31:28.779568+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":194,"log_dups_size":0,"ondisk_log_size":194,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:07:16.114968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00037995999999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1025,"num_read_kb":1201,"num_write":173,"num_write_kb":1276,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":719,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-09-23T20:31:28.784604+0000","last_change":"2023-09-23T20:29:07.042758+0000","last_active":"2023-09-23T20:31:28.784604+0000","last_peered":"2023-09-23T20:31:28.784604+0000","last_clean":"2023-09-23T20:31:28.784604+0000","last_became_active":"2023-09-23T20:29:07.042407+0000","last_became_peered":"2023-09-23T20:29:07.042407+0000","last_unstale":"2023-09-23T20:31:28.784604+0000","last_undegraded":"2023-09-23T20:31:28.784604+0000","last_fullsized":"2023-09-23T20:31:28.784604+0000","mapping_epoch":253,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":254,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:32.801168+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:32.801168+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0096333110000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":165,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-09-23T20:31:31.808594+0000","last_change":"2023-09-23T20:29:04.216400+0000","last_active":"2023-09-23T20:31:31.808594+0000","last_peered":"2023-09-23T20:31:31.808594+0000","last_clean":"2023-09-23T20:31:31.808594+0000","last_became_active":"2023-09-23T20:29:04.216069+0000","last_became_peered":"2023-09-23T20:29:04.216069+0000","last_unstale":"2023-09-23T20:31:31.808594+0000","last_undegraded":"2023-09-23T20:31:31.808594+0000","last_fullsized":"2023-09-23T20:31:31.808594+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:27:37.433035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":65,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-09-23T20:31:28.933612+0000","last_change":"2023-09-23T20:29:28.376036+0000","last_active":"2023-09-23T20:31:28.933612+0000","last_peered":"2023-09-23T20:31:28.933612+0000","last_clean":"2023-09-23T20:31:28.933612+0000","last_became_active":"2023-09-23T20:29:28.375712+0000","last_became_peered":"2023-09-23T20:29:28.375712+0000","last_unstale":"2023-09-23T20:31:28.933612+0000","last_undegraded":"2023-09-23T20:31:28.933612+0000","last_fullsized":"2023-09-23T20:31:28.933612+0000","mapping_epoch":258,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":259,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:23:50.722625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.4","version":"293'9360","reported_seq":9470,"reported_epoch":298,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:31:31.807514+0000","last_change":"2023-09-23T20:29:46.186959+0000","last_active":"2023-09-23T20:31:31.807514+0000","last_peered":"2023-09-23T20:31:31.807514+0000","last_clean":"2023-09-23T20:28:12.083147+0000","last_became_active":"2023-09-23T20:29:46.162334+0000","last_became_peered":"2023-09-23T20:29:46.162334+0000","last_unstale":"2023-09-23T20:31:31.807514+0000","last_undegraded":"2023-09-23T20:29:46.054138+0000","last_fullsized":"2023-09-23T20:29:46.043728+0000","mapping_epoch":267,"log_start":"227'3767","ondisk_log_start":"227'3767","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":5593,"log_dups_size":0,"ondisk_log_size":5593,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1966080,"num_objects":30,"num_object_clones":0,"num_object_copies":90,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2427,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":30,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6596,"num_write_kb":212032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[3,5],"avail_no_missing":["3","5"],"object_location_counts":[{"shards":"3,5","objects":-1013}],"blocked_by":[],"up_primary":1,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":522,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-09-23T20:31:15.380368+0000","last_change":"2023-09-23T20:29:46.072303+0000","last_active":"2023-09-23T20:31:15.380368+0000","last_peered":"2023-09-23T20:31:15.380368+0000","last_clean":"2023-09-23T20:31:15.380368+0000","last_became_active":"2023-09-23T20:29:46.071938+0000","last_became_peered":"2023-09-23T20:29:46.071938+0000","last_unstale":"2023-09-23T20:31:15.380368+0000","last_undegraded":"2023-09-23T20:31:15.380368+0000","last_fullsized":"2023-09-23T20:31:15.380368+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:12.386098+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:12.386098+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,5],"acting":[1,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"86'80","reported_seq":559,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-09-23T20:31:28.784554+0000","last_change":"2023-09-23T20:29:45.536796+0000","last_active":"2023-09-23T20:31:28.784554+0000","last_peered":"2023-09-23T20:31:28.784554+0000","last_clean":"2023-09-23T20:31:28.784554+0000","last_became_active":"2023-09-23T20:29:45.536439+0000","last_became_peered":"2023-09-23T20:29:45.536439+0000","last_unstale":"2023-09-23T20:31:28.784554+0000","last_undegraded":"2023-09-23T20:31:28.784554+0000","last_fullsized":"2023-09-23T20:31:28.784554+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":80,"log_dups_size":0,"ondisk_log_size":80,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:30:57.793132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035115,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":395,"num_read_kb":1370,"num_write":66,"num_write_kb":1061,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":268,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T20:31:23.586280+0000","last_change":"2023-09-23T20:29:28.360680+0000","last_active":"2023-09-23T20:31:23.586280+0000","last_peered":"2023-09-23T20:31:23.586280+0000","last_clean":"2023-09-23T20:31:23.586280+0000","last_became_active":"2023-09-23T20:29:28.360366+0000","last_became_peered":"2023-09-23T20:29:28.360366+0000","last_unstale":"2023-09-23T20:31:23.586280+0000","last_undegraded":"2023-09-23T20:31:23.586280+0000","last_fullsized":"2023-09-23T20:31:23.586280+0000","mapping_epoch":258,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":259,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:02:40.594828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":290,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T20:31:23.586587+0000","last_change":"2023-09-23T20:29:45.690815+0000","last_active":"2023-09-23T20:31:23.586587+0000","last_peered":"2023-09-23T20:31:23.586587+0000","last_clean":"2023-09-23T20:31:23.586587+0000","last_became_active":"2023-09-23T20:29:45.690478+0000","last_became_peered":"2023-09-23T20:29:45.690478+0000","last_unstale":"2023-09-23T20:31:23.586587+0000","last_undegraded":"2023-09-23T20:31:23.586587+0000","last_fullsized":"2023-09-23T20:31:23.586587+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:38.288193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"293'9210","reported_seq":14357,"reported_epoch":295,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:31:28.933403+0000","last_change":"2023-09-23T20:29:59.519844+0000","last_active":"2023-09-23T20:31:28.933403+0000","last_peered":"2023-09-23T20:31:28.933403+0000","last_clean":"2023-09-23T20:28:04.011715+0000","last_became_active":"2023-09-23T20:29:46.206627+0000","last_became_peered":"2023-09-23T20:29:46.206627+0000","last_unstale":"2023-09-23T20:31:28.933403+0000","last_undegraded":"2023-09-23T20:29:59.519468+0000","last_fullsized":"2023-09-23T20:29:46.046256+0000","mapping_epoch":267,"log_start":"222'2028","ondisk_log_start":"222'2028","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":7182,"log_dups_size":0,"ondisk_log_size":7182,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1729,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6456,"num_write_kb":206528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2160,"num_bytes_recovered":141557760,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[6,2],"avail_no_missing":["6","2"],"object_location_counts":[{"shards":"2,6","objects":-328}],"blocked_by":[],"up_primary":1,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":508,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-09-23T20:31:28.933737+0000","last_change":"2023-09-23T20:29:07.113747+0000","last_active":"2023-09-23T20:31:28.933737+0000","last_peered":"2023-09-23T20:31:28.933737+0000","last_clean":"2023-09-23T20:31:28.933737+0000","last_became_active":"2023-09-23T20:29:07.113428+0000","last_became_peered":"2023-09-23T20:29:07.113428+0000","last_unstale":"2023-09-23T20:31:28.933737+0000","last_undegraded":"2023-09-23T20:31:28.933737+0000","last_fullsized":"2023-09-23T20:31:28.933737+0000","mapping_epoch":253,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":254,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:31.769333+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:31.769333+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"86'102","reported_seq":824,"reported_epoch":295,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:31:28.784417+0000","last_change":"2023-09-23T20:29:46.060111+0000","last_active":"2023-09-23T20:31:28.784417+0000","last_peered":"2023-09-23T20:31:28.784417+0000","last_clean":"2023-09-23T20:29:43.509457+0000","last_became_active":"2023-09-23T20:29:46.059787+0000","last_became_peered":"2023-09-23T20:29:46.059787+0000","last_unstale":"2023-09-23T20:31:28.784417+0000","last_undegraded":"2023-09-23T20:29:46.049994+0000","last_fullsized":"2023-09-23T20:29:46.048495+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":260,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":102,"log_dups_size":0,"ondisk_log_size":102,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:53:45.933578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030656500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":963,"num_read_kb":1234,"num_write":60,"num_write_kb":225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0],"avail_no_missing":["7","0","4"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":216,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-09-23T20:31:15.381064+0000","last_change":"2023-09-23T20:29:45.538283+0000","last_active":"2023-09-23T20:31:15.381064+0000","last_peered":"2023-09-23T20:31:15.381064+0000","last_clean":"2023-09-23T20:31:15.381064+0000","last_became_active":"2023-09-23T20:29:45.537922+0000","last_became_peered":"2023-09-23T20:29:45.537922+0000","last_unstale":"2023-09-23T20:31:15.381064+0000","last_undegraded":"2023-09-23T20:31:15.381064+0000","last_fullsized":"2023-09-23T20:31:15.381064+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:30:52.397836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":227,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T20:31:23.586686+0000","last_change":"2023-09-23T20:20:14.888874+0000","last_active":"2023-09-23T20:31:23.586686+0000","last_peered":"2023-09-23T20:31:23.586686+0000","last_clean":"2023-09-23T20:31:23.586686+0000","last_became_active":"2023-09-23T20:20:14.887724+0000","last_became_peered":"2023-09-23T20:20:14.887724+0000","last_unstale":"2023-09-23T20:31:23.586686+0000","last_undegraded":"2023-09-23T20:31:23.586686+0000","last_fullsized":"2023-09-23T20:31:23.586686+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:09:24.510326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"293'9507","reported_seq":16419,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T20:31:32.841749+0000","last_change":"2023-09-23T20:31:32.841494+0000","last_active":"2023-09-23T20:31:32.841749+0000","last_peered":"2023-09-23T20:31:32.841749+0000","last_clean":"2023-09-23T20:31:32.841749+0000","last_became_active":"2023-09-23T20:31:32.814108+0000","last_became_peered":"2023-09-23T20:31:32.814108+0000","last_unstale":"2023-09-23T20:31:32.841749+0000","last_undegraded":"2023-09-23T20:31:32.841749+0000","last_fullsized":"2023-09-23T20:31:32.841749+0000","mapping_epoch":298,"log_start":"227'3998","ondisk_log_start":"227'3998","created":217,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":5509,"log_dups_size":0,"ondisk_log_size":5509,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6696,"num_write_kb":211392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3393,"num_bytes_recovered":9043968,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":467,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-09-23T20:31:30.799409+0000","last_change":"2023-09-23T20:29:28.372908+0000","last_active":"2023-09-23T20:31:30.799409+0000","last_peered":"2023-09-23T20:31:30.799409+0000","last_clean":"2023-09-23T20:31:30.799409+0000","last_became_active":"2023-09-23T20:29:28.372652+0000","last_became_peered":"2023-09-23T20:29:28.372652+0000","last_unstale":"2023-09-23T20:31:30.799409+0000","last_undegraded":"2023-09-23T20:31:30.799409+0000","last_fullsized":"2023-09-23T20:31:30.799409+0000","mapping_epoch":258,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":259,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:12.809933+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:12.809933+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,5],"acting":[2,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":151,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T20:31:32.806107+0000","last_change":"2023-09-23T20:25:03.007736+0000","last_active":"2023-09-23T20:31:32.806107+0000","last_peered":"2023-09-23T20:31:32.806107+0000","last_clean":"2023-09-23T20:31:32.806107+0000","last_became_active":"2023-09-23T20:25:02.973689+0000","last_became_peered":"2023-09-23T20:25:02.973689+0000","last_unstale":"2023-09-23T20:31:32.806107+0000","last_undegraded":"2023-09-23T20:31:32.806107+0000","last_fullsized":"2023-09-23T20:31:32.806107+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:37:25.643774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"86'128","reported_seq":649,"reported_epoch":293,"state":"active+clean","last_fresh":"2023-09-23T20:31:15.380939+0000","last_change":"2023-09-23T20:30:45.008506+0000","last_active":"2023-09-23T20:31:15.380939+0000","last_peered":"2023-09-23T20:31:15.380939+0000","last_clean":"2023-09-23T20:31:15.380939+0000","last_became_active":"2023-09-23T20:29:46.071848+0000","last_became_peered":"2023-09-23T20:29:46.071848+0000","last_unstale":"2023-09-23T20:31:15.380939+0000","last_undegraded":"2023-09-23T20:31:15.380939+0000","last_fullsized":"2023-09-23T20:31:15.380939+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":283,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:44:18.031530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000293202,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":579,"num_read_kb":1473,"num_write":96,"num_write_kb":1062,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":147,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-09-23T20:31:31.805504+0000","last_change":"2023-09-23T20:25:02.971740+0000","last_active":"2023-09-23T20:31:31.805504+0000","last_peered":"2023-09-23T20:31:31.805504+0000","last_clean":"2023-09-23T20:31:31.805504+0000","last_became_active":"2023-09-23T20:25:02.970412+0000","last_became_peered":"2023-09-23T20:25:02.970412+0000","last_unstale":"2023-09-23T20:31:31.805504+0000","last_undegraded":"2023-09-23T20:31:31.805504+0000","last_fullsized":"2023-09-23T20:31:31.805504+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:53:17.247216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"293'9324","reported_seq":9760,"reported_epoch":295,"state":"active+recovery_wait","last_fresh":"2023-09-23T20:31:28.784975+0000","last_change":"2023-09-23T20:30:30.444650+0000","last_active":"2023-09-23T20:31:28.784975+0000","last_peered":"2023-09-23T20:31:28.784975+0000","last_clean":"2023-09-23T20:28:04.010682+0000","last_became_active":"2023-09-23T20:29:47.069776+0000","last_became_peered":"2023-09-23T20:29:47.069776+0000","last_unstale":"2023-09-23T20:31:28.784975+0000","last_undegraded":"2023-09-23T20:31:28.784975+0000","last_fullsized":"2023-09-23T20:31:28.784975+0000","mapping_epoch":268,"log_start":"223'2316","ondisk_log_start":"223'2316","created":217,"last_epoch_clean":218,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":7008,"log_dups_size":0,"ondisk_log_size":7008,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6555,"num_write_kb":207584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":149,"num_bytes_recovered":9797632,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":["7","2","6"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.4","version":"90'161","reported_seq":547,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-09-23T20:31:31.807946+0000","last_change":"2023-09-23T20:28:03.553294+0000","last_active":"2023-09-23T20:31:31.807946+0000","last_peered":"2023-09-23T20:31:31.807946+0000","last_clean":"2023-09-23T20:31:31.807946+0000","last_became_active":"2023-09-23T20:27:55.212201+0000","last_became_peered":"2023-09-23T20:27:55.212201+0000","last_unstale":"2023-09-23T20:31:31.807946+0000","last_undegraded":"2023-09-23T20:31:31.807946+0000","last_fullsized":"2023-09-23T20:31:31.807946+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:28:03.553106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:28:03.553106+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:24:06.706940+0000","scrub_duration":0.011176864,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"86'117","reported_seq":608,"reported_epoch":293,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:31:17.360444+0000","last_change":"2023-09-23T20:29:59.542691+0000","last_active":"2023-09-23T20:31:17.360444+0000","last_peered":"2023-09-23T20:31:17.360444+0000","last_clean":"2023-09-23T20:13:53.392235+0000","last_became_active":"2023-09-23T20:29:05.138184+0000","last_became_peered":"2023-09-23T20:29:05.138184+0000","last_unstale":"2023-09-23T20:31:17.360444+0000","last_undegraded":"2023-09-23T20:29:59.542326+0000","last_fullsized":"2023-09-23T20:29:05.108660+0000","mapping_epoch":251,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:50:02.080933+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046891400000000002,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":893,"num_read_kb":922,"num_write":82,"num_write_kb":4193,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[2,7],"avail_no_missing":["2","3","7"],"object_location_counts":[{"shards":"2,3,7","objects":1}],"blocked_by":[],"up_primary":6,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":277,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T20:31:23.586504+0000","last_change":"2023-09-23T20:29:46.053198+0000","last_active":"2023-09-23T20:31:23.586504+0000","last_peered":"2023-09-23T20:31:23.586504+0000","last_clean":"2023-09-23T20:31:23.586504+0000","last_became_active":"2023-09-23T20:29:46.052866+0000","last_became_peered":"2023-09-23T20:29:46.052866+0000","last_unstale":"2023-09-23T20:31:23.586504+0000","last_undegraded":"2023-09-23T20:31:23.586504+0000","last_fullsized":"2023-09-23T20:31:23.586504+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:30:18.857140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":253,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-09-23T20:31:28.779280+0000","last_change":"2023-09-23T20:29:28.367722+0000","last_active":"2023-09-23T20:31:28.779280+0000","last_peered":"2023-09-23T20:31:28.779280+0000","last_clean":"2023-09-23T20:31:28.779280+0000","last_became_active":"2023-09-23T20:29:28.367404+0000","last_became_peered":"2023-09-23T20:29:28.367404+0000","last_unstale":"2023-09-23T20:31:28.779280+0000","last_undegraded":"2023-09-23T20:31:28.779280+0000","last_fullsized":"2023-09-23T20:31:28.779280+0000","mapping_epoch":258,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":259,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:02:51.886571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":249,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T20:31:23.586652+0000","last_change":"2023-09-23T20:29:46.061960+0000","last_active":"2023-09-23T20:31:23.586652+0000","last_peered":"2023-09-23T20:31:23.586652+0000","last_clean":"2023-09-23T20:31:23.586652+0000","last_became_active":"2023-09-23T20:29:46.061601+0000","last_became_peered":"2023-09-23T20:29:46.061601+0000","last_unstale":"2023-09-23T20:31:23.586652+0000","last_undegraded":"2023-09-23T20:31:23.586652+0000","last_fullsized":"2023-09-23T20:31:23.586652+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:15:52.620692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":149,"reported_epoch":297,"state":"active+clean","last_fresh":"2023-09-23T20:31:30.799213+0000","last_change":"2023-09-23T20:29:45.538314+0000","last_active":"2023-09-23T20:31:30.799213+0000","last_peered":"2023-09-23T20:31:30.799213+0000","last_clean":"2023-09-23T20:31:30.799213+0000","last_became_active":"2023-09-23T20:29:45.536847+0000","last_became_peered":"2023-09-23T20:29:45.536847+0000","last_unstale":"2023-09-23T20:31:30.799213+0000","last_undegraded":"2023-09-23T20:31:30.799213+0000","last_fullsized":"2023-09-23T20:31:30.799213+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:51.948721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"6.19","version":"293'9260","reported_seq":9358,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T20:31:23.586033+0000","last_change":"2023-09-23T20:30:29.820384+0000","last_active":"2023-09-23T20:31:23.586033+0000","last_peered":"2023-09-23T20:31:23.586033+0000","last_clean":"2023-09-23T20:31:23.586033+0000","last_became_active":"2023-09-23T20:28:35.567626+0000","last_became_peered":"2023-09-23T20:28:35.567626+0000","last_unstale":"2023-09-23T20:31:23.586033+0000","last_undegraded":"2023-09-23T20:31:23.586033+0000","last_fullsized":"2023-09-23T20:31:23.586033+0000","mapping_epoch":242,"log_start":"0'0","ondisk_log_start":"0'0","created":242,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":9260,"log_dups_size":0,"ondisk_log_size":9260,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":6160384,"num_objects":94,"num_object_clones":0,"num_object_copies":282,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":94,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6464,"num_write_kb":209856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":291,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-09-23T20:31:28.779705+0000","last_change":"2023-09-23T20:29:59.578082+0000","last_active":"2023-09-23T20:31:28.779705+0000","last_peered":"2023-09-23T20:31:28.779705+0000","last_clean":"2023-09-23T20:31:28.779705+0000","last_became_active":"2023-09-23T20:29:46.072626+0000","last_became_peered":"2023-09-23T20:29:46.072626+0000","last_unstale":"2023-09-23T20:31:28.779705+0000","last_undegraded":"2023-09-23T20:31:28.779705+0000","last_fullsized":"2023-09-23T20:31:28.779705+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:24:14.704823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":306,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-09-23T20:31:31.807387+0000","last_change":"2023-09-23T20:29:46.070732+0000","last_active":"2023-09-23T20:31:31.807387+0000","last_peered":"2023-09-23T20:31:31.807387+0000","last_clean":"2023-09-23T20:31:31.807387+0000","last_became_active":"2023-09-23T20:29:46.069861+0000","last_became_peered":"2023-09-23T20:29:46.069861+0000","last_unstale":"2023-09-23T20:31:31.807387+0000","last_undegraded":"2023-09-23T20:31:31.807387+0000","last_fullsized":"2023-09-23T20:31:31.807387+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:33:00.813261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.18","version":"293'9339","reported_seq":9436,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-09-23T20:31:28.784429+0000","last_change":"2023-09-23T20:28:35.587755+0000","last_active":"2023-09-23T20:31:28.784429+0000","last_peered":"2023-09-23T20:31:28.784429+0000","last_clean":"2023-09-23T20:31:28.784429+0000","last_became_active":"2023-09-23T20:28:35.585502+0000","last_became_peered":"2023-09-23T20:28:35.585502+0000","last_unstale":"2023-09-23T20:31:28.784429+0000","last_undegraded":"2023-09-23T20:31:28.784429+0000","last_fullsized":"2023-09-23T20:31:28.784429+0000","mapping_epoch":242,"log_start":"222'2246","ondisk_log_start":"222'2246","created":242,"last_epoch_clean":243,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:27:13.250180+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.250180+0000","objects_scrubbed":0,"log_size":7093,"log_dups_size":0,"ondisk_log_size":7093,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6553,"num_write_kb":209024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":299,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-09-23T20:31:28.779899+0000","last_change":"2023-09-23T20:29:46.242242+0000","last_active":"2023-09-23T20:31:28.779899+0000","last_peered":"2023-09-23T20:31:28.779899+0000","last_clean":"2023-09-23T20:31:28.779899+0000","last_became_active":"2023-09-23T20:29:46.241216+0000","last_became_peered":"2023-09-23T20:29:46.241216+0000","last_unstale":"2023-09-23T20:31:28.779899+0000","last_undegraded":"2023-09-23T20:31:28.779899+0000","last_fullsized":"2023-09-23T20:31:28.779899+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:12:48.530720+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":214,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-09-23T20:31:31.808723+0000","last_change":"2023-09-23T20:29:28.353700+0000","last_active":"2023-09-23T20:31:31.808723+0000","last_peered":"2023-09-23T20:31:31.808723+0000","last_clean":"2023-09-23T20:31:31.808723+0000","last_became_active":"2023-09-23T20:29:28.353357+0000","last_became_peered":"2023-09-23T20:29:28.353357+0000","last_unstale":"2023-09-23T20:31:31.808723+0000","last_undegraded":"2023-09-23T20:31:31.808723+0000","last_fullsized":"2023-09-23T20:31:31.808723+0000","mapping_epoch":258,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":259,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:12.077050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":180,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-09-23T20:31:31.809047+0000","last_change":"2023-09-23T20:29:04.138540+0000","last_active":"2023-09-23T20:31:31.809047+0000","last_peered":"2023-09-23T20:31:31.809047+0000","last_clean":"2023-09-23T20:31:31.809047+0000","last_became_active":"2023-09-23T20:29:04.138182+0000","last_became_peered":"2023-09-23T20:29:04.138182+0000","last_unstale":"2023-09-23T20:31:31.809047+0000","last_undegraded":"2023-09-23T20:31:31.809047+0000","last_fullsized":"2023-09-23T20:31:31.809047+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:26.098284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":155,"reported_epoch":298,"state":"active+clean","last_fresh":"2023-09-23T20:31:31.805735+0000","last_change":"2023-09-23T20:25:39.622819+0000","last_active":"2023-09-23T20:31:31.805735+0000","last_peered":"2023-09-23T20:31:31.805735+0000","last_clean":"2023-09-23T20:31:31.805735+0000","last_became_active":"2023-09-23T20:25:39.622484+0000","last_became_peered":"2023-09-23T20:25:39.622484+0000","last_unstale":"2023-09-23T20:31:31.805735+0000","last_undegraded":"2023-09-23T20:31:31.805735+0000","last_fullsized":"2023-09-23T20:31:31.805735+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:51:53.973261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":275,"reported_epoch":294,"state":"active+clean","last_fresh":"2023-09-23T20:31:23.586690+0000","last_change":"2023-09-23T20:25:39.668882+0000","last_active":"2023-09-23T20:31:23.586690+0000","last_peered":"2023-09-23T20:31:23.586690+0000","last_clean":"2023-09-23T20:31:23.586690+0000","last_became_active":"2023-09-23T20:25:39.637128+0000","last_became_peered":"2023-09-23T20:25:39.637128+0000","last_unstale":"2023-09-23T20:31:23.586690+0000","last_undegraded":"2023-09-23T20:31:23.586690+0000","last_fullsized":"2023-09-23T20:31:23.586690+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:56.406298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":311,"reported_epoch":295,"state":"active+clean","last_fresh":"2023-09-23T20:31:28.932931+0000","last_change":"2023-09-23T20:29:46.082412+0000","last_active":"2023-09-23T20:31:28.932931+0000","last_peered":"2023-09-23T20:31:28.932931+0000","last_clean":"2023-09-23T20:31:28.932931+0000","last_became_active":"2023-09-23T20:29:46.082028+0000","last_became_peered":"2023-09-23T20:29:46.082028+0000","last_unstale":"2023-09-23T20:31:28.932931+0000","last_undegraded":"2023-09-23T20:31:28.932931+0000","last_fullsized":"2023-09-23T20:31:28.932931+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:48:07.017969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":26,"stat_sum":{"num_bytes":25886720,"num_objects":396,"num_object_clones":0,"num_object_copies":1188,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":25788,"num_objects_misplaced":35,"num_objects_unfound":0,"num_objects_dirty":396,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":209474,"num_write_kb":6703105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28086,"num_bytes_recovered":1159135232,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1937178624,"data_stored":1937178624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":180938,"ondisk_log_size":180938,"up":78,"acting":69,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":6376,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":122880,"data_stored":21392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26691,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":740547,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":214,"num_read_kb":401,"num_write":106,"num_write_kb":1063,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1433573,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2437120,"data_stored":2426464,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3381,"ondisk_log_size":3381,"up":63,"acting":63,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":26,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":17,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6082,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":22,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201524,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":585580,"kb_used_data":362724,"kb_used_omap":0,"kb_used_meta":222848,"kb_avail":93159572,"statfs":{"total":95995035648,"available":95395401728,"internally_reserved":0,"allocated":371429376,"data_stored":370484049,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":228196352},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:30:34 2023","interfaces":[{"interface":"back","average":{"1min":1.1459999999999999,"5min":1.96,"15min":1.163},"min":{"1min":0.25800000000000001,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":3.4790000000000001,"5min":48.798999999999999,"15min":48.798999999999999},"last":0.25700000000000001},{"interface":"front","average":{"1min":2.617,"5min":2.4209999999999998,"15min":1.3300000000000001},"min":{"1min":0.308,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":7.8179999999999996,"5min":48.247,"15min":48.247},"last":0.27300000000000002}]},{"osd":1,"last update":"Sat Sep 23 20:30:47 2023","interfaces":[{"interface":"back","average":{"1min":1.762,"5min":1.762,"15min":1.762},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":9.7370000000000001,"5min":9.7370000000000001,"15min":9.7370000000000001},"last":0.28899999999999998},{"interface":"front","average":{"1min":1.9810000000000001,"5min":1.9810000000000001,"15min":1.9810000000000001},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":5.7679999999999998,"5min":5.7679999999999998,"15min":5.7679999999999998},"last":0.30599999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:30:38 2023","interfaces":[{"interface":"back","average":{"1min":2.9300000000000002,"5min":2.0249999999999999,"15min":1.2210000000000001},"min":{"1min":0.35899999999999999,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":8.2509999999999994,"5min":11.372999999999999,"15min":11.372999999999999},"last":0.33400000000000002},{"interface":"front","average":{"1min":2.9729999999999999,"5min":2.2709999999999999,"15min":1.349},"min":{"1min":0.33500000000000002,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":14.993,"5min":14.993,"15min":14.993},"last":0.40300000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:31:02 2023","interfaces":[{"interface":"back","average":{"1min":2.5059999999999998,"5min":2.0710000000000002,"15min":1.377},"min":{"1min":0.34000000000000002,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":11.015000000000001,"5min":11.015000000000001,"15min":11.015000000000001},"last":17.155999999999999},{"interface":"front","average":{"1min":2.7970000000000002,"5min":1.9510000000000001,"15min":1.3},"min":{"1min":0.38,"5min":0.33100000000000002,"15min":0.27600000000000002},"max":{"1min":11.031000000000001,"5min":11.031000000000001,"15min":11.031000000000001},"last":16.960000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:31:02 2023","interfaces":[{"interface":"back","average":{"1min":0.73699999999999999,"5min":1.583,"15min":1.008},"min":{"1min":0.29399999999999998,"5min":0.27100000000000002,"15min":0.27000000000000002},"max":{"1min":5.7220000000000004,"5min":15.177,"15min":15.177},"last":0.32300000000000001},{"interface":"front","average":{"1min":0.79900000000000004,"5min":1.3560000000000001,"15min":1.127},"min":{"1min":0.28399999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":4.2800000000000002,"5min":11.122,"15min":11.122},"last":0.379}]},{"osd":5,"last update":"Sat Sep 23 20:31:02 2023","interfaces":[{"interface":"back","average":{"1min":0.76200000000000001,"5min":1.4159999999999999,"15min":1.1040000000000001},"min":{"1min":0.30599999999999999,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":2.8730000000000002,"5min":13.241,"15min":13.241},"last":0.35999999999999999},{"interface":"front","average":{"1min":0.79400000000000004,"5min":1.7230000000000001,"15min":1.2070000000000001},"min":{"1min":0.32200000000000001,"5min":0.315,"15min":0.315},"max":{"1min":5.7869999999999999,"5min":18.831,"15min":18.831},"last":0.35199999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:31:02 2023","interfaces":[{"interface":"back","average":{"1min":0.82399999999999995,"5min":1.5580000000000001,"15min":1.1060000000000001},"min":{"1min":0.34399999999999997,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":5.7629999999999999,"5min":12.298999999999999,"15min":13.185},"last":0.39000000000000001},{"interface":"front","average":{"1min":0.93600000000000005,"5min":1.302,"15min":1.048},"min":{"1min":0.36799999999999999,"5min":0.35399999999999998,"15min":0.29899999999999999},"max":{"1min":4.4160000000000004,"5min":8.0069999999999997,"15min":9.9120000000000008},"last":0.46100000000000002}]}]},{"osd":6,"up_from":47,"seq":201863463160,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":245208,"kb_used_data":45072,"kb_used_omap":0,"kb_used_meta":200128,"kb_avail":93499944,"statfs":{"total":95995035648,"available":95743942656,"internally_reserved":0,"allocated":46153728,"data_stored":45220274,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":204931072},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:30:34 2023","interfaces":[{"interface":"back","average":{"1min":1.446,"5min":1.413,"15min":0.997},"min":{"1min":0.23300000000000001,"5min":0.221,"15min":0.221},"max":{"1min":7.609,"5min":8.4209999999999994,"15min":8.4209999999999994},"last":0.378},{"interface":"front","average":{"1min":2.1619999999999999,"5min":1.7949999999999999,"15min":1.2310000000000001},"min":{"1min":0.28499999999999998,"5min":0.25,"15min":0.25},"max":{"1min":8.6229999999999993,"5min":8.6760000000000002,"15min":12.512},"last":0.42899999999999999}]},{"osd":1,"last update":"Sat Sep 23 20:30:46 2023","interfaces":[{"interface":"back","average":{"1min":2.7509999999999999,"5min":2.7509999999999999,"15min":2.7509999999999999},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":14.1,"5min":14.1,"15min":14.1},"last":0.31},{"interface":"front","average":{"1min":3.4870000000000001,"5min":3.4870000000000001,"15min":3.4870000000000001},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":14.266999999999999,"5min":14.266999999999999,"15min":14.266999999999999},"last":0.55600000000000005}]},{"osd":2,"last update":"Sat Sep 23 20:30:34 2023","interfaces":[{"interface":"back","average":{"1min":2.7519999999999998,"5min":2.2440000000000002,"15min":1.304},"min":{"1min":0.35099999999999998,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":7.8700000000000001,"5min":10.631,"15min":13.856},"last":0.34799999999999998},{"interface":"front","average":{"1min":2.5110000000000001,"5min":2.056,"15min":1.2190000000000001},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":7.891,"5min":8.3729999999999993,"15min":8.3729999999999993},"last":0.48999999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:31:11 2023","interfaces":[{"interface":"back","average":{"1min":2.9420000000000002,"5min":2.0099999999999998,"15min":1.3859999999999999},"min":{"1min":0.51600000000000001,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":6.1829999999999998,"5min":8.2989999999999995,"15min":9.4239999999999995},"last":0.33000000000000002},{"interface":"front","average":{"1min":3.0910000000000002,"5min":2.21,"15min":1.4330000000000001},"min":{"1min":0.40600000000000003,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":7.6609999999999996,"5min":8.3770000000000007,"15min":8.6050000000000004},"last":0.51500000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:30:53 2023","interfaces":[{"interface":"back","average":{"1min":0.44,"5min":1.0349999999999999,"15min":0.85299999999999998},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":1.599,"5min":10.173,"15min":10.173},"last":0.44500000000000001},{"interface":"front","average":{"1min":1.032,"5min":1.3779999999999999,"15min":0.98199999999999998},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":3.363,"5min":8.3249999999999993,"15min":8.3249999999999993},"last":0.39300000000000002}]},{"osd":5,"last update":"Sat Sep 23 20:30:53 2023","interfaces":[{"interface":"back","average":{"1min":0.46400000000000002,"5min":0.91100000000000003,"15min":0.89800000000000002},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":1.2010000000000001,"5min":8.4499999999999993,"15min":14.085000000000001},"last":0.53600000000000003},{"interface":"front","average":{"1min":0.98099999999999998,"5min":1.76,"15min":1.236},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.29499999999999998},"max":{"1min":3.294,"5min":18.192,"15min":18.192},"last":0.45800000000000002}]},{"osd":7,"last update":"Sat Sep 23 20:31:11 2023","interfaces":[{"interface":"back","average":{"1min":0.45300000000000001,"5min":1.27,"15min":1.0309999999999999},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.754,"5min":8.5060000000000002,"15min":10.217000000000001},"last":0.41899999999999998},{"interface":"front","average":{"1min":1.0669999999999999,"5min":1.508,"15min":1.163},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":6.4160000000000004,"5min":8.407,"15min":9.7360000000000007},"last":0.47899999999999998}]}]},{"osd":1,"up_from":266,"seq":1142461300758,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":772444,"kb_used_data":648340,"kb_used_omap":8,"kb_used_meta":124087,"kb_avail":92972708,"statfs":{"total":95995035648,"available":95204052992,"internally_reserved":0,"allocated":663900160,"data_stored":662314836,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8206,"internal_metadata":127066098},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:30:44 2023","interfaces":[{"interface":"back","average":{"1min":2.3300000000000001,"5min":2.3300000000000001,"15min":2.3300000000000001},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":9.5470000000000006,"5min":9.5470000000000006,"15min":9.5470000000000006},"last":0.218},{"interface":"front","average":{"1min":3.1960000000000002,"5min":3.1960000000000002,"15min":3.1960000000000002},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":9.7129999999999992,"5min":9.7129999999999992,"15min":9.7129999999999992},"last":0.317}]},{"osd":2,"last update":"Sat Sep 23 20:30:44 2023","interfaces":[{"interface":"back","average":{"1min":5.125,"5min":5.125,"15min":5.125},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":16.151,"5min":16.151,"15min":16.151},"last":0.23599999999999999},{"interface":"front","average":{"1min":2.0939999999999999,"5min":2.0939999999999999,"15min":2.0939999999999999},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":9.4649999999999999,"5min":9.4649999999999999,"15min":9.4649999999999999},"last":0.30099999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:30:44 2023","interfaces":[{"interface":"back","average":{"1min":3.867,"5min":3.867,"15min":3.867},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":9.5739999999999998,"5min":9.5739999999999998,"15min":9.5739999999999998},"last":0.65300000000000002},{"interface":"front","average":{"1min":4.8460000000000001,"5min":4.8460000000000001,"15min":4.8460000000000001},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":12.012,"5min":12.012,"15min":12.012},"last":0.48499999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:30:44 2023","interfaces":[{"interface":"back","average":{"1min":2.0790000000000002,"5min":2.0790000000000002,"15min":2.0790000000000002},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":9.4429999999999996,"5min":9.4429999999999996,"15min":9.4429999999999996},"last":0.47299999999999998},{"interface":"front","average":{"1min":2.024,"5min":2.024,"15min":2.024},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":9.5039999999999996,"5min":9.5039999999999996,"15min":9.5039999999999996},"last":0.39800000000000002}]},{"osd":5,"last update":"Sat Sep 23 20:30:44 2023","interfaces":[{"interface":"back","average":{"1min":2.802,"5min":2.802,"15min":2.802},"min":{"1min":0.44,"5min":0.44,"15min":0.44},"max":{"1min":9.6479999999999997,"5min":9.6479999999999997,"15min":9.6479999999999997},"last":0.59099999999999997},{"interface":"front","average":{"1min":2.0899999999999999,"5min":2.0899999999999999,"15min":2.0899999999999999},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":9.4700000000000006,"5min":9.4700000000000006,"15min":9.4700000000000006},"last":0.52500000000000002}]},{"osd":6,"last update":"Sat Sep 23 20:30:44 2023","interfaces":[{"interface":"back","average":{"1min":7.4489999999999998,"5min":7.4489999999999998,"15min":7.4489999999999998},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":72.682000000000002,"5min":72.682000000000002,"15min":72.682000000000002},"last":0.53400000000000003},{"interface":"front","average":{"1min":6.4400000000000004,"5min":6.4400000000000004,"15min":6.4400000000000004},"min":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":72.563999999999993,"5min":72.563999999999993,"15min":72.563999999999993},"last":0.436}]},{"osd":7,"last update":"Sat Sep 23 20:30:44 2023","interfaces":[{"interface":"back","average":{"1min":3.2829999999999999,"5min":3.2829999999999999,"15min":3.2829999999999999},"min":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"max":{"1min":9.6850000000000005,"5min":9.6850000000000005,"15min":9.6850000000000005},"last":0.55900000000000005},{"interface":"front","average":{"1min":2.3599999999999999,"5min":2.3599999999999999,"15min":2.3599999999999999},"min":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"max":{"1min":9.3949999999999996,"5min":9.3949999999999996,"15min":9.3949999999999996},"last":0.505}]}]},{"osd":0,"up_from":94,"seq":403726925961,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":351080,"kb_used_data":3936,"kb_used_omap":7,"kb_used_meta":347128,"kb_avail":93394072,"statfs":{"total":95995035648,"available":95635529728,"internally_reserved":0,"allocated":4030464,"data_stored":2559587,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":355459089},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:30:48 2023","interfaces":[{"interface":"back","average":{"1min":3.887,"5min":3.887,"15min":3.887},"min":{"1min":0.17999999999999999,"5min":0.17999999999999999,"15min":0.17999999999999999},"max":{"1min":15.542,"5min":15.542,"15min":15.542},"last":0.28899999999999998},{"interface":"front","average":{"1min":4.4569999999999999,"5min":4.4569999999999999,"15min":4.4569999999999999},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":10.49,"5min":10.49,"15min":10.49},"last":0.45400000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:30:27 2023","interfaces":[{"interface":"back","average":{"1min":5.8789999999999996,"5min":3.3180000000000001,"15min":1.6619999999999999},"min":{"1min":0.185,"5min":0.185,"15min":0.185},"max":{"1min":77.448999999999998,"5min":77.448999999999998,"15min":77.448999999999998},"last":0.25900000000000001},{"interface":"front","average":{"1min":6.0869999999999997,"5min":2.9449999999999998,"15min":1.5069999999999999},"min":{"1min":0.24299999999999999,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":81.724999999999994,"5min":81.724999999999994,"15min":81.724999999999994},"last":0.36699999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:31:03 2023","interfaces":[{"interface":"back","average":{"1min":7.0750000000000002,"5min":3.6080000000000001,"15min":1.79},"min":{"1min":0.25,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":81.400999999999996,"5min":81.400999999999996,"15min":81.400999999999996},"last":0.42799999999999999},{"interface":"front","average":{"1min":9.3719999999999999,"5min":4.0679999999999996,"15min":1.7649999999999999},"min":{"1min":0.34699999999999998,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":81.137,"5min":81.137,"15min":81.137},"last":0.38700000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:30:27 2023","interfaces":[{"interface":"back","average":{"1min":2.093,"5min":2.4990000000000001,"15min":1.409},"min":{"1min":0.315,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":8.3209999999999997,"5min":23.876999999999999,"15min":23.876999999999999},"last":0.34200000000000003},{"interface":"front","average":{"1min":2.0009999999999999,"5min":1.9690000000000001,"15min":1.2390000000000001},"min":{"1min":0.34499999999999997,"5min":0.312,"15min":0.29299999999999998},"max":{"1min":10.414,"5min":23.681999999999999,"15min":23.681999999999999},"last":0.55000000000000004}]},{"osd":5,"last update":"Sat Sep 23 20:30:27 2023","interfaces":[{"interface":"back","average":{"1min":2.645,"5min":2.657,"15min":1.6040000000000001},"min":{"1min":0.377,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":10.368,"5min":24.198,"15min":24.198},"last":0.5},{"interface":"front","average":{"1min":2.2130000000000001,"5min":2.1869999999999998,"15min":1.45},"min":{"1min":0.38100000000000001,"5min":0.314,"15min":0.314},"max":{"1min":8.7780000000000005,"5min":14.938000000000001,"15min":14.938000000000001},"last":0.47199999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:30:27 2023","interfaces":[{"interface":"back","average":{"1min":2.2450000000000001,"5min":2.3420000000000001,"15min":1.319},"min":{"1min":0.40100000000000002,"5min":0.318,"15min":0.318},"max":{"1min":10.441000000000001,"5min":24.408000000000001,"15min":24.408000000000001},"last":0.58399999999999996},{"interface":"front","average":{"1min":1.764,"5min":2.1709999999999998,"15min":1.2629999999999999},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":6.9480000000000004,"5min":25.271000000000001,"15min":25.271000000000001},"last":0.40899999999999997}]},{"osd":7,"last update":"Sat Sep 23 20:30:27 2023","interfaces":[{"interface":"back","average":{"1min":2.024,"5min":2.3839999999999999,"15min":1.4179999999999999},"min":{"1min":0.42199999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":7.4470000000000001,"5min":25.359000000000002,"15min":25.359000000000002},"last":0.52500000000000002},{"interface":"front","average":{"1min":2.5550000000000002,"5min":3.1200000000000001,"15min":1.6799999999999999},"min":{"1min":0.46500000000000002,"5min":0.378,"15min":0.378},"max":{"1min":8.2729999999999997,"5min":24.669,"15min":24.669},"last":0.48499999999999999}]}]},{"osd":2,"up_from":104,"seq":446676598909,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":799920,"kb_used_data":411560,"kb_used_omap":11,"kb_used_meta":388340,"kb_avail":92945232,"statfs":{"total":95995035648,"available":95175917568,"internally_reserved":0,"allocated":421437440,"data_stored":419974872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11878,"internal_metadata":397660570},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:30:37 2023","interfaces":[{"interface":"back","average":{"1min":3.3199999999999998,"5min":2.2509999999999999,"15min":1.282},"min":{"1min":0.193,"5min":0.17999999999999999,"15min":0.17999999999999999},"max":{"1min":30.765999999999998,"5min":30.765999999999998,"15min":30.765999999999998},"last":0.186},{"interface":"front","average":{"1min":5.1280000000000001,"5min":2.3199999999999998,"15min":1.2150000000000001},"min":{"1min":0.252,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":16.824000000000002,"5min":16.824000000000002,"15min":16.824000000000002},"last":0.372}]},{"osd":1,"last update":"Sat Sep 23 20:30:45 2023","interfaces":[{"interface":"back","average":{"1min":4.851,"5min":4.851,"15min":4.851},"min":{"1min":0.221,"5min":0.221,"15min":0.221},"max":{"1min":16.280000000000001,"5min":16.280000000000001,"15min":16.280000000000001},"last":0.27700000000000002},{"interface":"front","average":{"1min":2.988,"5min":2.988,"15min":2.988},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":30.565000000000001,"5min":30.565000000000001,"15min":30.565000000000001},"last":0.34200000000000003}]},{"osd":3,"last update":"Sat Sep 23 20:31:02 2023","interfaces":[{"interface":"back","average":{"1min":2.8849999999999998,"5min":2.2959999999999998,"15min":1.21},"min":{"1min":0.27800000000000002,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":11.132,"5min":16.331,"15min":16.331},"last":0.315},{"interface":"front","average":{"1min":4.2539999999999996,"5min":2.8109999999999999,"15min":1.46},"min":{"1min":0.27300000000000002,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":30.704999999999998,"5min":30.704999999999998,"15min":30.704999999999998},"last":0.29899999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:30:37 2023","interfaces":[{"interface":"back","average":{"1min":3.98,"5min":2.1669999999999998,"15min":1.234},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.28100000000000003},"max":{"1min":16.797999999999998,"5min":16.797999999999998,"15min":16.797999999999998},"last":0.42999999999999999},{"interface":"front","average":{"1min":2.7309999999999999,"5min":2.1589999999999998,"15min":1.2609999999999999},"min":{"1min":0.309,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":10.518000000000001,"5min":14.971,"15min":14.971},"last":0.39000000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:30:37 2023","interfaces":[{"interface":"back","average":{"1min":4.5359999999999996,"5min":3.512,"15min":1.77},"min":{"1min":0.35299999999999998,"5min":0.309,"15min":0.309},"max":{"1min":16.766999999999999,"5min":95.393000000000001,"15min":95.393000000000001},"last":0.49199999999999999},{"interface":"front","average":{"1min":3.4790000000000001,"5min":3.6190000000000002,"15min":1.8},"min":{"1min":0.34799999999999998,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":10.474,"5min":95.414000000000001,"15min":95.414000000000001},"last":0.42099999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:30:37 2023","interfaces":[{"interface":"back","average":{"1min":3.258,"5min":2.29,"15min":1.323},"min":{"1min":0.33600000000000002,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":10.544,"5min":15.023999999999999,"15min":15.023999999999999},"last":0.35799999999999998},{"interface":"front","average":{"1min":2.7930000000000001,"5min":2.3719999999999999,"15min":1.405},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":30.670999999999999,"5min":30.670999999999999,"15min":30.670999999999999},"last":0.40200000000000002}]},{"osd":7,"last update":"Sat Sep 23 20:30:37 2023","interfaces":[{"interface":"back","average":{"1min":3.4609999999999999,"5min":2.5529999999999999,"15min":1.4730000000000001},"min":{"1min":0.41299999999999998,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":30.741,"5min":30.741,"15min":30.741},"last":0.44600000000000001},{"interface":"front","average":{"1min":4.8650000000000002,"5min":2.52,"15min":1.4550000000000001},"min":{"1min":0.40899999999999997,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":16.719000000000001,"5min":16.719000000000001,"15min":16.719000000000001},"last":0.46999999999999997}]}]},{"osd":3,"up_from":109,"seq":468151435383,"num_pgs":64,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":394752,"kb_used_data":8312,"kb_used_omap":4,"kb_used_meta":386427,"kb_avail":93350400,"statfs":{"total":95995035648,"available":95590809600,"internally_reserved":0,"allocated":8511488,"data_stored":7037070,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4514,"internal_metadata":395701854},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:31:05 2023","interfaces":[{"interface":"back","average":{"1min":2.5600000000000001,"5min":2.2909999999999999,"15min":1.4610000000000001},"min":{"1min":0.22500000000000001,"5min":0.186,"15min":0.186},"max":{"1min":6.71,"5min":15.589,"15min":67.305999999999997},"last":0.33200000000000002},{"interface":"front","average":{"1min":2.839,"5min":1.978,"15min":1.3919999999999999},"min":{"1min":0.19500000000000001,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":12.781000000000001,"5min":13.743,"15min":67.403999999999996},"last":0.26600000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:30:47 2023","interfaces":[{"interface":"back","average":{"1min":3.0880000000000001,"5min":3.0880000000000001,"15min":3.0880000000000001},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":8.9199999999999999,"5min":8.9199999999999999,"15min":8.9199999999999999},"last":0.28199999999999997},{"interface":"front","average":{"1min":3.2530000000000001,"5min":3.2530000000000001,"15min":3.2530000000000001},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":10.967000000000001,"5min":10.967000000000001,"15min":10.967000000000001},"last":0.35899999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:31:05 2023","interfaces":[{"interface":"back","average":{"1min":3.1419999999999999,"5min":2.6739999999999999,"15min":1.3959999999999999},"min":{"1min":0.29799999999999999,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":11.253,"5min":24.045000000000002,"15min":24.045000000000002},"last":0.30599999999999999},{"interface":"front","average":{"1min":3.1709999999999998,"5min":2.649,"15min":1.4430000000000001},"min":{"1min":0.27100000000000002,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":19.603000000000002,"5min":19.603000000000002,"15min":19.603000000000002},"last":0.38600000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:31:05 2023","interfaces":[{"interface":"back","average":{"1min":2.331,"5min":2.46,"15min":1.3560000000000001},"min":{"1min":0.34200000000000003,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":6.5670000000000002,"5min":46.042999999999999,"15min":46.042999999999999},"last":0.45200000000000001},{"interface":"front","average":{"1min":3.7120000000000002,"5min":2.9569999999999999,"15min":1.591},"min":{"1min":0.34599999999999997,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":19.635000000000002,"5min":46.579999999999998,"15min":46.579999999999998},"last":0.34100000000000003}]},{"osd":5,"last update":"Sat Sep 23 20:31:05 2023","interfaces":[{"interface":"back","average":{"1min":2.319,"5min":2.5960000000000001,"15min":1.5189999999999999},"min":{"1min":0.45700000000000002,"5min":0.31900000000000001,"15min":0.311},"max":{"1min":6.2370000000000001,"5min":45.292000000000002,"15min":45.292000000000002},"last":0.40699999999999997},{"interface":"front","average":{"1min":2.6139999999999999,"5min":2.5129999999999999,"15min":1.3999999999999999},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.313},"max":{"1min":11.028,"5min":44.679000000000002,"15min":44.679000000000002},"last":0.46300000000000002}]},{"osd":6,"last update":"Sat Sep 23 20:31:05 2023","interfaces":[{"interface":"back","average":{"1min":3.3519999999999999,"5min":2.3410000000000002,"15min":1.4279999999999999},"min":{"1min":0.38500000000000001,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":17.524999999999999,"5min":17.524999999999999,"15min":17.524999999999999},"last":0.42799999999999999},{"interface":"front","average":{"1min":3.0670000000000002,"5min":2.3300000000000001,"15min":1.387},"min":{"1min":0.439,"5min":0.36399999999999999,"15min":0.35499999999999998},"max":{"1min":19.524000000000001,"5min":19.524000000000001,"15min":19.524000000000001},"last":0.44}]},{"osd":7,"last update":"Sat Sep 23 20:31:05 2023","interfaces":[{"interface":"back","average":{"1min":4.1749999999999998,"5min":2.8370000000000002,"15min":1.5549999999999999},"min":{"1min":0.40699999999999997,"5min":0.374,"15min":0.34999999999999998},"max":{"1min":19.571999999999999,"5min":19.571999999999999,"15min":19.571999999999999},"last":0.47699999999999998},{"interface":"front","average":{"1min":2.3100000000000001,"5min":2.762,"15min":1.607},"min":{"1min":0.42999999999999999,"5min":0.371,"15min":0.371},"max":{"1min":6.407,"5min":45.387999999999998,"15min":45.387999999999998},"last":0.51500000000000001}]}]},{"osd":4,"up_from":0,"seq":128849019136,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":5,"up_from":39,"seq":167503724797,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":228756,"kb_used_data":3596,"kb_used_omap":0,"kb_used_meta":225152,"kb_avail":93516396,"statfs":{"total":95995035648,"available":95760789504,"internally_reserved":0,"allocated":3682304,"data_stored":2749295,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":230555648},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:30:31 2023","interfaces":[{"interface":"back","average":{"1min":2.8109999999999999,"5min":1.7529999999999999,"15min":1.181},"min":{"1min":0.28499999999999998,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":8.1609999999999996,"5min":11.346,"15min":32.475000000000001},"last":0.29499999999999998},{"interface":"front","average":{"1min":2.5950000000000002,"5min":1.681,"15min":1.2030000000000001},"min":{"1min":0.29199999999999998,"5min":0.245,"15min":0.20599999999999999},"max":{"1min":8.1859999999999999,"5min":8.4900000000000002,"15min":32.457000000000001},"last":0.42099999999999999}]},{"osd":1,"last update":"Sat Sep 23 20:30:45 2023","interfaces":[{"interface":"back","average":{"1min":2.0550000000000002,"5min":2.0550000000000002,"15min":2.0550000000000002},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":8.5839999999999996,"5min":8.5839999999999996,"15min":8.5839999999999996},"last":0.32500000000000001},{"interface":"front","average":{"1min":1.5609999999999999,"5min":1.5609999999999999,"15min":1.5609999999999999},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":6.7969999999999997,"5min":6.7969999999999997,"15min":6.7969999999999997},"last":0.35999999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:30:31 2023","interfaces":[{"interface":"back","average":{"1min":3.5259999999999998,"5min":1.823,"15min":1.325},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.26700000000000002},"max":{"1min":18.472999999999999,"5min":18.472999999999999,"15min":34.514000000000003},"last":0.48199999999999998},{"interface":"front","average":{"1min":1.8919999999999999,"5min":1.7869999999999999,"15min":1.256},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":8.016,"5min":11.574999999999999,"15min":32.515000000000001},"last":0.34000000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:31:05 2023","interfaces":[{"interface":"back","average":{"1min":3.1230000000000002,"5min":2.2109999999999999,"15min":1.3220000000000001},"min":{"1min":0.61799999999999999,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":10.952999999999999,"5min":12.846,"15min":12.846},"last":0.45000000000000001},{"interface":"front","average":{"1min":2.3479999999999999,"5min":1.754,"15min":1.2310000000000001},"min":{"1min":0.32300000000000001,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":7.2060000000000004,"5min":12.005000000000001,"15min":12.005000000000001},"last":0.52100000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:31:26 2023","interfaces":[{"interface":"back","average":{"1min":1.1899999999999999,"5min":1.919,"15min":1.276},"min":{"1min":0.27800000000000002,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":6.282,"5min":28.015999999999998,"15min":28.015999999999998},"last":0.379},{"interface":"front","average":{"1min":0.81799999999999995,"5min":1.2749999999999999,"15min":1.04},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.29099999999999998},"max":{"1min":2.427,"5min":12.300000000000001,"15min":12.300000000000001},"last":0.39800000000000002}]},{"osd":6,"last update":"Sat Sep 23 20:30:39 2023","interfaces":[{"interface":"back","average":{"1min":1.1220000000000001,"5min":1.796,"15min":1.1499999999999999},"min":{"1min":0.309,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":4.9829999999999997,"5min":12.890000000000001,"15min":12.890000000000001},"last":0.41299999999999998},{"interface":"front","average":{"1min":0.92500000000000004,"5min":1.5389999999999999,"15min":1.137},"min":{"1min":0.371,"5min":0.36399999999999999,"15min":0.34499999999999997},"max":{"1min":2.4710000000000001,"5min":16.265999999999998,"15min":16.265999999999998},"last":0.45800000000000002}]},{"osd":7,"last update":"Sat Sep 23 20:31:05 2023","interfaces":[{"interface":"back","average":{"1min":1.01,"5min":1.8169999999999999,"15min":1.204},"min":{"1min":0.32400000000000001,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":2.77,"5min":36.621000000000002,"15min":36.621000000000002},"last":0.5},{"interface":"front","average":{"1min":0.94299999999999995,"5min":1.6759999999999999,"15min":1.103},"min":{"1min":0.33500000000000002,"5min":0.316,"15min":0.315},"max":{"1min":3.6219999999999999,"5min":37.331000000000003,"15min":37.331000000000003},"last":0.39100000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4722,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1360,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8206,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7156,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3154,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":660078592,"data_stored":660078592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":413270016,"data_stored":413270016,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":456982528,"data_stored":456982528,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":42729472,"data_stored":42729472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":364118016,"data_stored":364118016,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:31:39.629 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-23T20:31:39.699 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:31:39.732 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:39.766 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:31:39.769 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:39.770 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:31:39.870 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:31:39.897 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:39.931 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:31:39.933 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:39.934 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:31:40.034 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:31:40.067 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:40.861 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:31:40.864 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:40.865 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:31:40.966 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:31:40.996 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:41.031 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:31:41.034 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:41.034 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:31:41.124 INFO:teuthology.orchestra.run.smithi018.stderr:noscrub is unset 2023-09-23T20:31:41.135 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:31:41.143 INFO:teuthology.orchestra.run.smithi018.stderr:pool 'unique_pool_1' removed 2023-09-23T20:31:41.176 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:41.317 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:31:41.325 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:41 smithi018 ceph-mon[184144]: pgmap v684: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 29 MiB data, 3.1 GiB used, 623 GiB / 626 GiB avail; 22675/1278 objects degraded (1774.257%); 35/1278 objects misplaced (2.739%); 591 KiB/s, 295 objects/s recovering 2023-09-23T20:31:41.325 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:41 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2412428246' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T20:31:41.325 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:41 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T20:31:41.326 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:41 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/123288943' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:31:41.326 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:41 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:31:41.326 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:41 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 22675/1278 objects degraded (1774.257%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T20:31:41.326 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:41 smithi018 ceph-mon[184144]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:31:41.326 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:41 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T20:31:41.326 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:41 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:31:41.326 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:41 smithi018 ceph-mon[184144]: osdmap e302: 8 total, 8 up, 7 in 2023-09-23T20:31:41.327 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:41.327 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:31:41.329 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:41 smithi018 ceph-mon[179647]: pgmap v684: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 29 MiB data, 3.1 GiB used, 623 GiB / 626 GiB avail; 22675/1278 objects degraded (1774.257%); 35/1278 objects misplaced (2.739%); 591 KiB/s, 295 objects/s recovering 2023-09-23T20:31:41.329 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:41 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2412428246' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T20:31:41.329 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:41 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T20:31:41.329 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:41 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/123288943' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:31:41.329 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:41 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:31:41.330 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:41 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 22675/1278 objects degraded (1774.257%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T20:31:41.330 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:41 smithi018 ceph-mon[179647]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:31:41.330 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:41 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T20:31:41.330 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:41 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:31:41.330 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:41 smithi018 ceph-mon[179647]: osdmap e302: 8 total, 8 up, 7 in 2023-09-23T20:31:41.380 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2023-09-23T20:31:41.380 INFO:teuthology.orchestra.run.smithi018.stdout: "ops": [], 2023-09-23T20:31:41.380 INFO:teuthology.orchestra.run.smithi018.stdout: "num_ops": 0 2023-09-23T20:31:41.380 INFO:teuthology.orchestra.run.smithi018.stdout:} 2023-09-23T20:31:41.428 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:31:41.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:41 smithi149 ceph-mon[170394]: pgmap v684: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait, 2 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 29 MiB data, 3.1 GiB used, 623 GiB / 626 GiB avail; 22675/1278 objects degraded (1774.257%); 35/1278 objects misplaced (2.739%); 591 KiB/s, 295 objects/s recovering 2023-09-23T20:31:41.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:41 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2412428246' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T20:31:41.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:41 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T20:31:41.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:41 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/123288943' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:31:41.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:41 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:31:41.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:41 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 22675/1278 objects degraded (1774.257%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T20:31:41.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:41 smithi149 ceph-mon[170394]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:31:41.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:41 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T20:31:41.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:41 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:31:41.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:41 smithi149 ceph-mon[170394]: osdmap e302: 8 total, 8 up, 7 in 2023-09-23T20:31:41.517 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:41.840 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:31:41.842 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:41.843 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:31:41.944 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:31:41.972 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:42.003 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-09-23T20:31:42.004 INFO:tasks.radosbench:Beginning radosbench... 2023-09-23T20:31:42.004 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_2 2023-09-23T20:31:42.004 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd pool create unique_pool_2 16 2023-09-23T20:31:42.192 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:31:42.194 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:42 smithi018 ceph-mon[179647]: failed to encode map e302 with expected crc 2023-09-23T20:31:42.194 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:42 smithi018 ceph-mon[179647]: failed to encode map e302 with expected crc 2023-09-23T20:31:42.194 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:42 smithi018 ceph-mon[179647]: failed to encode map e302 with expected crc 2023-09-23T20:31:42.194 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:42 smithi018 ceph-mon[179647]: 1.b scrub starts 2023-09-23T20:31:42.194 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:42 smithi018 ceph-mon[179647]: failed to encode map e302 with expected crc 2023-09-23T20:31:42.194 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:42 smithi018 ceph-mon[179647]: failed to encode map e302 with expected crc 2023-09-23T20:31:42.195 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:42 smithi018 ceph-mon[184144]: failed to encode map e302 with expected crc 2023-09-23T20:31:42.195 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:42 smithi018 ceph-mon[184144]: failed to encode map e302 with expected crc 2023-09-23T20:31:42.195 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:42 smithi018 ceph-mon[184144]: failed to encode map e302 with expected crc 2023-09-23T20:31:42.195 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:42 smithi018 ceph-mon[184144]: 1.b scrub starts 2023-09-23T20:31:42.195 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:42 smithi018 ceph-mon[184144]: failed to encode map e302 with expected crc 2023-09-23T20:31:42.195 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:42 smithi018 ceph-mon[184144]: failed to encode map e302 with expected crc 2023-09-23T20:31:42.195 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:42.196 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:31:42.207 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T20:31:42.229 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:31:42.247 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:31:42.296 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:31:42.327 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:42.386 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:31:42.391 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:42.392 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:31:42.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:42 smithi149 ceph-mon[170394]: failed to encode map e302 with expected crc 2023-09-23T20:31:42.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:42 smithi149 ceph-mon[170394]: failed to encode map e302 with expected crc 2023-09-23T20:31:42.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:42 smithi149 ceph-mon[170394]: failed to encode map e302 with expected crc 2023-09-23T20:31:42.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:42 smithi149 ceph-mon[170394]: 1.b scrub starts 2023-09-23T20:31:42.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:42 smithi149 ceph-mon[170394]: failed to encode map e302 with expected crc 2023-09-23T20:31:42.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:42 smithi149 ceph-mon[170394]: failed to encode map e302 with expected crc 2023-09-23T20:31:42.492 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:31:42.525 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:42.572 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:31:42.576 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:42.576 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:31:42.677 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:31:42.717 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:42.754 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:31:42.758 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:42.759 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:31:42.860 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:31:42.890 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:42.974 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:31:42.976 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:42.977 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:31:43.077 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:31:43.105 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:43.207 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:31:43.210 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:43.210 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:31:43.311 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:31:43.341 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:43.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:43 smithi018 ceph-mon[184144]: 1.c scrub starts 2023-09-23T20:31:43.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:43 smithi018 ceph-mon[184144]: 1.c scrub ok 2023-09-23T20:31:43.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:43 smithi018 ceph-mon[184144]: pgmap v686: 93 pgs: 93 active+clean; 4.7 MiB data, 3.1 GiB used, 623 GiB / 626 GiB avail 2023-09-23T20:31:43.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:43 smithi018 ceph-mon[184144]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 22675/1278 objects degraded (1774.257%), 7 pgs degraded, 7 pgs undersized) 2023-09-23T20:31:43.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:43 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:31:43.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:43 smithi018 ceph-mon[184144]: 1.13 scrub starts 2023-09-23T20:31:43.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:43 smithi018 ceph-mon[184144]: 1.13 scrub ok 2023-09-23T20:31:43.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:43 smithi018 ceph-mon[179647]: 1.c scrub starts 2023-09-23T20:31:43.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:43 smithi018 ceph-mon[179647]: 1.c scrub ok 2023-09-23T20:31:43.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:43 smithi018 ceph-mon[179647]: pgmap v686: 93 pgs: 93 active+clean; 4.7 MiB data, 3.1 GiB used, 623 GiB / 626 GiB avail 2023-09-23T20:31:43.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:43 smithi018 ceph-mon[179647]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 22675/1278 objects degraded (1774.257%), 7 pgs degraded, 7 pgs undersized) 2023-09-23T20:31:43.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:43 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:31:43.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:43 smithi018 ceph-mon[179647]: 1.13 scrub starts 2023-09-23T20:31:43.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:43 smithi018 ceph-mon[179647]: 1.13 scrub ok 2023-09-23T20:31:43.451 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:31:43.453 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:43.454 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:31:43.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:43 smithi149 ceph-mon[170394]: 1.c scrub starts 2023-09-23T20:31:43.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:43 smithi149 ceph-mon[170394]: 1.c scrub ok 2023-09-23T20:31:43.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:43 smithi149 ceph-mon[170394]: pgmap v686: 93 pgs: 93 active+clean; 4.7 MiB data, 3.1 GiB used, 623 GiB / 626 GiB avail 2023-09-23T20:31:43.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:43 smithi149 ceph-mon[170394]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 22675/1278 objects degraded (1774.257%), 7 pgs degraded, 7 pgs undersized) 2023-09-23T20:31:43.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:43 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:31:43.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:43 smithi149 ceph-mon[170394]: 1.13 scrub starts 2023-09-23T20:31:43.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:43 smithi149 ceph-mon[170394]: 1.13 scrub ok 2023-09-23T20:31:43.555 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:31:43.555 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:31:43.593 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:43.632 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:31:43.636 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:31:43.643 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:43.644 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:31:43.725 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T20:31:43.744 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:31:43.797 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:44.057 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:31:44.059 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:44.060 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:31:44.161 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:31:44.194 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:44.227 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:31:44.230 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:44.231 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:31:44.249 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:31:44.249 INFO:teuthology.orchestra.run.smithi018.stdout:{"fsid":"a3b3b17e-5a4c-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodeep-scrub flag(s) set","count":12},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":780,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":302,"num_osds":8,"num_up_osds":8,"osd_up_since":1695500984,"num_in_osds":7,"osd_in_since":1695500967,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":93}],"num_pgs":93,"num_pools":4,"num_objects":30,"data_bytes":5158827,"bytes_used":3302866944,"bytes_avail":668662382592,"bytes_total":671965249536},"fsmap":{"epoch":9,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi018.hduiol","status":"up:active","gid":24566}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.18:8443/","prometheus":"http://172.21.15.18:9283/"}},"servicemap":{"epoch":21,"modified":"2023-09-23T20:30:43.686281+0000","services":{}},"progress_events":{"8e7eff18-0e45-4130-a6e1-662b3902f0d2":{"message":"Upgrade to 18.2.0-460-gbb0daeff (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true},"a3000690-5684-42b5-9dec-3c8689315562":{"message":"Global Recovery Event (3m)\n [=======================.....] (remaining: 37s)","progress":0.84873950481414795,"add_to_ceph_s":true}}} 2023-09-23T20:31:44.266 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:44 smithi018 ceph-mon[179647]: 1.2 scrub starts 2023-09-23T20:31:44.266 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:44 smithi018 ceph-mon[179647]: 1.2 scrub ok 2023-09-23T20:31:44.266 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:44 smithi018 ceph-mon[179647]: 1.3 scrub starts 2023-09-23T20:31:44.266 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:44 smithi018 ceph-mon[179647]: 1.3 scrub ok 2023-09-23T20:31:44.266 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:44 smithi018 ceph-mon[184144]: 1.2 scrub starts 2023-09-23T20:31:44.266 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:44 smithi018 ceph-mon[184144]: 1.2 scrub ok 2023-09-23T20:31:44.267 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:44 smithi018 ceph-mon[184144]: 1.3 scrub starts 2023-09-23T20:31:44.267 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:44 smithi018 ceph-mon[184144]: 1.3 scrub ok 2023-09-23T20:31:44.331 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:31:44.358 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:44.361 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:31:44.383 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:31:44.391 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:31:44.393 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:44 smithi149 ceph-mon[170394]: 1.2 scrub starts 2023-09-23T20:31:44.393 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:44 smithi149 ceph-mon[170394]: 1.2 scrub ok 2023-09-23T20:31:44.393 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:44 smithi149 ceph-mon[170394]: 1.3 scrub starts 2023-09-23T20:31:44.393 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:44 smithi149 ceph-mon[170394]: 1.3 scrub ok 2023-09-23T20:31:44.394 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:44.394 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:31:44.403 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:31:44.423 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:31:44.448 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:31:44.468 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:31:44.489 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:31:44.494 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:31:44.517 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:31:44.519 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:31:44.519 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:31:44.519 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:31:44.519 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:31:44.519 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:31:44.519 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:31:44.519 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:31:44.519 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:31:44.537 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:45.143 INFO:tasks.cephadm.ceph_manager.ceph:no progress seen, keeping timeout for now 2023-09-23T20:31:45.144 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:31:45.298 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:31:44 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:31:44] "GET /metrics HTTP/1.1" 200 34167 "" "Prometheus/2.43.0" 2023-09-23T20:31:45.298 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:45 smithi018 ceph-mon[179647]: 1.11 scrub starts 2023-09-23T20:31:45.298 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:45 smithi018 ceph-mon[179647]: 1.11 scrub ok 2023-09-23T20:31:45.298 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:45 smithi018 ceph-mon[179647]: pgmap v687: 93 pgs: 93 active+clean; 4.9 MiB data, 3.1 GiB used, 623 GiB / 626 GiB avail 2023-09-23T20:31:45.298 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:45 smithi018 ceph-mon[179647]: 1.6 scrub starts 2023-09-23T20:31:45.299 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:45 smithi018 ceph-mon[179647]: 1.6 scrub ok 2023-09-23T20:31:45.299 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:45 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1181556320' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T20:31:45.299 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:45 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3734889379' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:31:45.299 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:45 smithi018 ceph-mon[179647]: 1.e scrub starts 2023-09-23T20:31:45.299 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:45 smithi018 ceph-mon[179647]: 1.e scrub ok 2023-09-23T20:31:45.299 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:45 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:31:45.299 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:45 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:31:45.299 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:45 smithi018 ceph-mon[184144]: 1.11 scrub starts 2023-09-23T20:31:45.299 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:45 smithi018 ceph-mon[184144]: 1.11 scrub ok 2023-09-23T20:31:45.299 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:45 smithi018 ceph-mon[184144]: pgmap v687: 93 pgs: 93 active+clean; 4.9 MiB data, 3.1 GiB used, 623 GiB / 626 GiB avail 2023-09-23T20:31:45.300 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:45 smithi018 ceph-mon[184144]: 1.6 scrub starts 2023-09-23T20:31:45.300 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:45 smithi018 ceph-mon[184144]: 1.6 scrub ok 2023-09-23T20:31:45.300 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:45 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1181556320' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T20:31:45.300 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:45 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3734889379' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:31:45.300 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:45 smithi018 ceph-mon[184144]: 1.e scrub starts 2023-09-23T20:31:45.300 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:45 smithi018 ceph-mon[184144]: 1.e scrub ok 2023-09-23T20:31:45.300 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:45 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:31:45.300 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:45 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:31:45.443 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:31:45.447 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:45.447 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:31:45.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:45 smithi149 ceph-mon[170394]: 1.11 scrub starts 2023-09-23T20:31:45.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:45 smithi149 ceph-mon[170394]: 1.11 scrub ok 2023-09-23T20:31:45.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:45 smithi149 ceph-mon[170394]: pgmap v687: 93 pgs: 93 active+clean; 4.9 MiB data, 3.1 GiB used, 623 GiB / 626 GiB avail 2023-09-23T20:31:45.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:45 smithi149 ceph-mon[170394]: 1.6 scrub starts 2023-09-23T20:31:45.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:45 smithi149 ceph-mon[170394]: 1.6 scrub ok 2023-09-23T20:31:45.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:45 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1181556320' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T20:31:45.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:45 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3734889379' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:31:45.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:45 smithi149 ceph-mon[170394]: 1.e scrub starts 2023-09-23T20:31:45.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:45 smithi149 ceph-mon[170394]: 1.e scrub ok 2023-09-23T20:31:45.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:45 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:31:45.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:45 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:31:45.547 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:31:45.577 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:45.710 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:31:45.713 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:45.714 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:31:45.796 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:31:45.815 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:31:45.850 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:45.930 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:31:45.933 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:45.933 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:31:46.034 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:31:46.061 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:46.100 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:31:46.135 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:46 smithi018 ceph-mon[184144]: 1.5 scrub starts 2023-09-23T20:31:46.135 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:46 smithi018 ceph-mon[184144]: 1.5 scrub ok 2023-09-23T20:31:46.135 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:46 smithi018 ceph-mon[184144]: 1.7 scrub starts 2023-09-23T20:31:46.135 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:46 smithi018 ceph-mon[184144]: 1.7 scrub ok 2023-09-23T20:31:46.136 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:46 smithi018 ceph-mon[184144]: 1.0 scrub starts 2023-09-23T20:31:46.136 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:46 smithi018 ceph-mon[184144]: 1.0 scrub ok 2023-09-23T20:31:46.136 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:46 smithi018 ceph-mon[184144]: 1.1 scrub starts 2023-09-23T20:31:46.136 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:46 smithi018 ceph-mon[184144]: 1.1 scrub ok 2023-09-23T20:31:46.145 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:31:46.148 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:46.149 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:31:46.249 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:31:46.282 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:46.319 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:31:46.323 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:46.323 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:31:46.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:46 smithi018 ceph-mon[179647]: 1.5 scrub starts 2023-09-23T20:31:46.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:46 smithi018 ceph-mon[179647]: 1.5 scrub ok 2023-09-23T20:31:46.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:46 smithi018 ceph-mon[179647]: 1.7 scrub starts 2023-09-23T20:31:46.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:46 smithi018 ceph-mon[179647]: 1.7 scrub ok 2023-09-23T20:31:46.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:46 smithi018 ceph-mon[179647]: 1.0 scrub starts 2023-09-23T20:31:46.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:46 smithi018 ceph-mon[179647]: 1.0 scrub ok 2023-09-23T20:31:46.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:46 smithi018 ceph-mon[179647]: 1.1 scrub starts 2023-09-23T20:31:46.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:46 smithi018 ceph-mon[179647]: 1.1 scrub ok 2023-09-23T20:31:46.424 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:31:46.450 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:46 smithi149 ceph-mon[170394]: 1.5 scrub starts 2023-09-23T20:31:46.450 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:46 smithi149 ceph-mon[170394]: 1.5 scrub ok 2023-09-23T20:31:46.450 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:46 smithi149 ceph-mon[170394]: 1.7 scrub starts 2023-09-23T20:31:46.450 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:46 smithi149 ceph-mon[170394]: 1.7 scrub ok 2023-09-23T20:31:46.450 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:46 smithi149 ceph-mon[170394]: 1.0 scrub starts 2023-09-23T20:31:46.450 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:46 smithi149 ceph-mon[170394]: 1.0 scrub ok 2023-09-23T20:31:46.450 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:46 smithi149 ceph-mon[170394]: 1.1 scrub starts 2023-09-23T20:31:46.450 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:46 smithi149 ceph-mon[170394]: 1.1 scrub ok 2023-09-23T20:31:46.453 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:46.489 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:31:46.492 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:46.492 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:31:46.533 INFO:teuthology.orchestra.run.smithi018.stdout:ERROR: (22) Invalid argument 2023-09-23T20:31:46.533 INFO:teuthology.orchestra.run.smithi018.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:31:46.593 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:31:46.628 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:46.764 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:31:46.767 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:46.767 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:31:46.868 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:31:46.896 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:46.930 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:31:46.933 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:46.933 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:31:47.034 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:31:47.060 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:47.094 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:31:47.096 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:47.097 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:31:47.165 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T20:31:47.198 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:31:47.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:47 smithi149 ceph-mon[170394]: pgmap v688: 93 pgs: 1 active+clean+scrubbing, 92 active+clean; 4.8 MiB data, 3.0 GiB used, 623 GiB / 626 GiB avail 2023-09-23T20:31:47.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:47 smithi149 ceph-mon[170394]: 1.f scrub starts 2023-09-23T20:31:47.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:47 smithi149 ceph-mon[170394]: 1.f scrub ok 2023-09-23T20:31:47.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:47 smithi149 ceph-mon[170394]: 1.d scrub starts 2023-09-23T20:31:47.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:47 smithi149 ceph-mon[170394]: 1.d scrub ok 2023-09-23T20:31:47.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:47 smithi149 ceph-mon[170394]: 1.b scrub starts 2023-09-23T20:31:47.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:47 smithi149 ceph-mon[170394]: 1.b scrub ok 2023-09-23T20:31:47.237 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:47.289 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:31:47.292 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:47.293 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:31:47.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:47 smithi018 ceph-mon[179647]: pgmap v688: 93 pgs: 1 active+clean+scrubbing, 92 active+clean; 4.8 MiB data, 3.0 GiB used, 623 GiB / 626 GiB avail 2023-09-23T20:31:47.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:47 smithi018 ceph-mon[179647]: 1.f scrub starts 2023-09-23T20:31:47.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:47 smithi018 ceph-mon[179647]: 1.f scrub ok 2023-09-23T20:31:47.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:47 smithi018 ceph-mon[179647]: 1.d scrub starts 2023-09-23T20:31:47.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:47 smithi018 ceph-mon[179647]: 1.d scrub ok 2023-09-23T20:31:47.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:47 smithi018 ceph-mon[179647]: 1.b scrub starts 2023-09-23T20:31:47.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:47 smithi018 ceph-mon[179647]: 1.b scrub ok 2023-09-23T20:31:47.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:47 smithi018 ceph-mon[184144]: pgmap v688: 93 pgs: 1 active+clean+scrubbing, 92 active+clean; 4.8 MiB data, 3.0 GiB used, 623 GiB / 626 GiB avail 2023-09-23T20:31:47.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:47 smithi018 ceph-mon[184144]: 1.f scrub starts 2023-09-23T20:31:47.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:47 smithi018 ceph-mon[184144]: 1.f scrub ok 2023-09-23T20:31:47.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:47 smithi018 ceph-mon[184144]: 1.d scrub starts 2023-09-23T20:31:47.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:47 smithi018 ceph-mon[184144]: 1.d scrub ok 2023-09-23T20:31:47.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:47 smithi018 ceph-mon[184144]: 1.b scrub starts 2023-09-23T20:31:47.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:47 smithi018 ceph-mon[184144]: 1.b scrub ok 2023-09-23T20:31:47.394 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:31:47.423 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:47.558 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:31:47.561 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:47.561 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:31:47.662 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:31:47.689 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:47.723 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:31:47.726 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:47.727 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:31:47.810 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:31:47.828 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:31:47.854 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:47.888 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:31:47.891 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:47.892 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:31:47.992 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:31:48.028 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:48.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:48 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/502817565' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-09-23T20:31:48.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:48 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:31:48.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:48 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:31:48.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:48 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/502817565' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-09-23T20:31:48.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:48 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:31:48.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:48 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:31:48.163 INFO:teuthology.orchestra.run.smithi018.stderr:pool 'unique_pool_2' created 2023-09-23T20:31:48.169 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:31:48.173 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:48.174 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:31:48.275 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:31:48.308 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:48.355 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:31:48.357 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:48.358 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:31:48.445 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:31:48.447 INFO:teuthology.orchestra.run.smithi018.stderr:dumped all 2023-09-23T20:31:48.459 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:31:48.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:48 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/502817565' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-09-23T20:31:48.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:48 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:31:48.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:48 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:31:48.485 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:48.519 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:31:48.522 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:48.522 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:31:48.623 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:31:48.651 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:48.781 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:31:48.784 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:48.784 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:31:48.813 INFO:teuthology.orchestra.run.smithi018.stdout:{"pg_ready":true,"pg_map":{"version":690,"stamp":"2023-09-23T20:31:48.141646+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5006762,"num_objects":29,"num_object_clones":0,"num_object_copies":87,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":29,"num_whiteouts":0,"num_read":6084,"num_read_kb":13379,"num_write":951,"num_write_kb":13165,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":85,"num_bytes_recovered":4522738,"num_keys_recovered":30,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4412,"ondisk_log_size":4412,"up":279,"acting":279,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":314,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":3108852,"kb_used_data":1164668,"kb_used_omap":32,"kb_used_meta":1944095,"kb_avail":653107212,"statfs":{"total":671965249536,"available":668781785088,"internally_reserved":0,"allocated":1192620032,"data_stored":1183557655,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32773,"internal_metadata":1990754299},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":59,"apply_latency_ms":59,"commit_latency_ns":59000000,"apply_latency_ns":59000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-25820332,"num_objects":-397,"num_object_clones":0,"num_object_copies":-1191,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-22675,"num_objects_misplaced":-35,"num_objects_unfound":0,"num_objects_dirty":-397,"num_whiteouts":0,"num_read":-126,"num_read_kb":-235,"num_write":-209536,"num_write_kb":-6703728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-28936,"num_bytes_recovered":-1160246785,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.415829"},"pg_stats":[{"pgid":"3.1a","version":"69'1","reported_seq":47,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-09-23T20:31:38.109487+0000","last_change":"2023-09-23T20:29:46.050662+0000","last_active":"2023-09-23T20:31:38.109487+0000","last_peered":"2023-09-23T20:31:38.109487+0000","last_clean":"2023-09-23T20:31:38.109487+0000","last_became_active":"2023-09-23T20:29:46.050281+0000","last_became_peered":"2023-09-23T20:29:46.050281+0000","last_unstale":"2023-09-23T20:31:38.109487+0000","last_undegraded":"2023-09-23T20:31:38.109487+0000","last_fullsized":"2023-09-23T20:31:38.109487+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:55.227008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":175,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-09-23T20:31:38.135504+0000","last_change":"2023-09-23T20:29:46.067085+0000","last_active":"2023-09-23T20:31:38.135504+0000","last_peered":"2023-09-23T20:31:38.135504+0000","last_clean":"2023-09-23T20:31:38.135504+0000","last_became_active":"2023-09-23T20:29:46.066738+0000","last_became_peered":"2023-09-23T20:29:46.066738+0000","last_unstale":"2023-09-23T20:31:38.135504+0000","last_undegraded":"2023-09-23T20:31:38.135504+0000","last_fullsized":"2023-09-23T20:31:38.135504+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:33:14.679402+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":239,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-09-23T20:31:38.109932+0000","last_change":"2023-09-23T20:29:46.178684+0000","last_active":"2023-09-23T20:31:38.109932+0000","last_peered":"2023-09-23T20:31:38.109932+0000","last_clean":"2023-09-23T20:31:38.109932+0000","last_became_active":"2023-09-23T20:29:46.178249+0000","last_became_peered":"2023-09-23T20:29:46.178249+0000","last_unstale":"2023-09-23T20:31:38.109932+0000","last_undegraded":"2023-09-23T20:31:38.109932+0000","last_fullsized":"2023-09-23T20:31:38.109932+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:58:58.530687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":267,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-09-23T20:31:38.107295+0000","last_change":"2023-09-23T20:25:02.957185+0000","last_active":"2023-09-23T20:31:38.107295+0000","last_peered":"2023-09-23T20:31:38.107295+0000","last_clean":"2023-09-23T20:31:38.107295+0000","last_became_active":"2023-09-23T20:25:02.956689+0000","last_became_peered":"2023-09-23T20:25:02.956689+0000","last_unstale":"2023-09-23T20:31:38.107295+0000","last_undegraded":"2023-09-23T20:31:38.107295+0000","last_fullsized":"2023-09-23T20:31:38.107295+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:04:28.524015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":164,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-09-23T20:31:38.136165+0000","last_change":"2023-09-23T20:29:45.529762+0000","last_active":"2023-09-23T20:31:38.136165+0000","last_peered":"2023-09-23T20:31:38.136165+0000","last_clean":"2023-09-23T20:31:38.136165+0000","last_became_active":"2023-09-23T20:29:45.529283+0000","last_became_peered":"2023-09-23T20:29:45.529283+0000","last_unstale":"2023-09-23T20:31:38.136165+0000","last_undegraded":"2023-09-23T20:31:38.136165+0000","last_fullsized":"2023-09-23T20:31:38.136165+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:54:36.720075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":75,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-09-23T20:31:38.114112+0000","last_change":"2023-09-23T20:29:46.112977+0000","last_active":"2023-09-23T20:31:38.114112+0000","last_peered":"2023-09-23T20:31:38.114112+0000","last_clean":"2023-09-23T20:31:38.114112+0000","last_became_active":"2023-09-23T20:29:46.112600+0000","last_became_peered":"2023-09-23T20:29:46.112600+0000","last_unstale":"2023-09-23T20:31:38.114112+0000","last_undegraded":"2023-09-23T20:31:38.114112+0000","last_fullsized":"2023-09-23T20:31:38.114112+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:06:34.132247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":265,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-09-23T20:31:38.107526+0000","last_change":"2023-09-23T20:25:02.969432+0000","last_active":"2023-09-23T20:31:38.107526+0000","last_peered":"2023-09-23T20:31:38.107526+0000","last_clean":"2023-09-23T20:31:38.107526+0000","last_became_active":"2023-09-23T20:25:02.969118+0000","last_became_peered":"2023-09-23T20:25:02.969118+0000","last_unstale":"2023-09-23T20:31:38.107526+0000","last_undegraded":"2023-09-23T20:31:38.107526+0000","last_fullsized":"2023-09-23T20:31:38.107526+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:02:22.453831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":184,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-09-23T20:31:38.136239+0000","last_change":"2023-09-23T20:29:46.069832+0000","last_active":"2023-09-23T20:31:38.136239+0000","last_peered":"2023-09-23T20:31:38.136239+0000","last_clean":"2023-09-23T20:31:38.136239+0000","last_became_active":"2023-09-23T20:29:46.069451+0000","last_became_peered":"2023-09-23T20:29:46.069451+0000","last_unstale":"2023-09-23T20:31:38.136239+0000","last_undegraded":"2023-09-23T20:31:38.136239+0000","last_fullsized":"2023-09-23T20:31:38.136239+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:06:06.104030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"90'161","reported_seq":572,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-09-23T20:31:38.111962+0000","last_change":"2023-09-23T20:29:48.092787+0000","last_active":"2023-09-23T20:31:38.111962+0000","last_peered":"2023-09-23T20:31:38.111962+0000","last_clean":"2023-09-23T20:31:38.111962+0000","last_became_active":"2023-09-23T20:29:48.092460+0000","last_became_peered":"2023-09-23T20:29:48.092460+0000","last_unstale":"2023-09-23T20:31:38.111962+0000","last_undegraded":"2023-09-23T20:31:38.111962+0000","last_fullsized":"2023-09-23T20:31:38.111962+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:13.244651+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:13.244651+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0019955509999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109227,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":278,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-09-23T20:31:38.116310+0000","last_change":"2023-09-23T20:29:45.523808+0000","last_active":"2023-09-23T20:31:38.116310+0000","last_peered":"2023-09-23T20:31:38.116310+0000","last_clean":"2023-09-23T20:31:38.116310+0000","last_became_active":"2023-09-23T20:29:45.523508+0000","last_became_peered":"2023-09-23T20:29:45.523508+0000","last_unstale":"2023-09-23T20:31:38.116310+0000","last_undegraded":"2023-09-23T20:31:38.116310+0000","last_fullsized":"2023-09-23T20:31:38.116310+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:50:55.565352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":167,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-09-23T20:31:38.136158+0000","last_change":"2023-09-23T20:29:04.130512+0000","last_active":"2023-09-23T20:31:38.136158+0000","last_peered":"2023-09-23T20:31:38.136158+0000","last_clean":"2023-09-23T20:31:38.136158+0000","last_became_active":"2023-09-23T20:29:04.130135+0000","last_became_peered":"2023-09-23T20:29:04.130135+0000","last_unstale":"2023-09-23T20:31:38.136158+0000","last_undegraded":"2023-09-23T20:31:38.136158+0000","last_fullsized":"2023-09-23T20:31:38.136158+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:53:59.385893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":313,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-09-23T20:31:38.167403+0000","last_change":"2023-09-23T20:29:59.501260+0000","last_active":"2023-09-23T20:31:38.167403+0000","last_peered":"2023-09-23T20:31:38.167403+0000","last_clean":"2023-09-23T20:31:38.167403+0000","last_became_active":"2023-09-23T20:29:28.362070+0000","last_became_peered":"2023-09-23T20:29:28.362070+0000","last_unstale":"2023-09-23T20:31:38.167403+0000","last_undegraded":"2023-09-23T20:31:38.167403+0000","last_fullsized":"2023-09-23T20:31:38.167403+0000","mapping_epoch":258,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:32:55.983696+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":249,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-09-23T20:31:38.109883+0000","last_change":"2023-09-23T20:29:46.080766+0000","last_active":"2023-09-23T20:31:38.109883+0000","last_peered":"2023-09-23T20:31:38.109883+0000","last_clean":"2023-09-23T20:31:38.109883+0000","last_became_active":"2023-09-23T20:29:46.080356+0000","last_became_peered":"2023-09-23T20:29:46.080356+0000","last_unstale":"2023-09-23T20:31:38.109883+0000","last_undegraded":"2023-09-23T20:31:38.109883+0000","last_fullsized":"2023-09-23T20:31:38.109883+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:08:01.862270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":119,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-09-23T20:31:38.167269+0000","last_change":"2023-09-23T20:29:59.529392+0000","last_active":"2023-09-23T20:31:38.167269+0000","last_peered":"2023-09-23T20:31:38.167269+0000","last_clean":"2023-09-23T20:31:38.167269+0000","last_became_active":"2023-09-23T20:29:28.365579+0000","last_became_peered":"2023-09-23T20:29:28.365579+0000","last_unstale":"2023-09-23T20:31:38.167269+0000","last_undegraded":"2023-09-23T20:31:38.167269+0000","last_fullsized":"2023-09-23T20:31:38.167269+0000","mapping_epoch":258,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:31:11.546083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":297,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-09-23T20:31:38.113561+0000","last_change":"2023-09-23T20:29:46.091575+0000","last_active":"2023-09-23T20:31:38.113561+0000","last_peered":"2023-09-23T20:31:38.113561+0000","last_clean":"2023-09-23T20:31:38.113561+0000","last_became_active":"2023-09-23T20:29:46.091271+0000","last_became_peered":"2023-09-23T20:29:46.091271+0000","last_unstale":"2023-09-23T20:31:38.113561+0000","last_undegraded":"2023-09-23T20:31:38.113561+0000","last_fullsized":"2023-09-23T20:31:38.113561+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:08:01.594175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.11","version":"90'161","reported_seq":530,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-09-23T20:31:43.578687+0000","last_change":"2023-09-23T20:31:43.578810+0000","last_active":"2023-09-23T20:31:43.578687+0000","last_peered":"2023-09-23T20:31:43.578687+0000","last_clean":"2023-09-23T20:31:43.578687+0000","last_became_active":"2023-09-23T20:28:05.991304+0000","last_became_peered":"2023-09-23T20:28:05.991304+0000","last_unstale":"2023-09-23T20:31:43.578687+0000","last_undegraded":"2023-09-23T20:31:43.578687+0000","last_fullsized":"2023-09-23T20:31:43.578687+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:43.578666+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:43.578666+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:46:18.999991+0000","scrub_duration":0.0021890379999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":312,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-09-23T20:31:38.166969+0000","last_change":"2023-09-23T20:29:59.544701+0000","last_active":"2023-09-23T20:31:38.166969+0000","last_peered":"2023-09-23T20:31:38.166969+0000","last_clean":"2023-09-23T20:31:38.166969+0000","last_became_active":"2023-09-23T20:29:28.367679+0000","last_became_peered":"2023-09-23T20:29:28.367679+0000","last_unstale":"2023-09-23T20:31:38.166969+0000","last_undegraded":"2023-09-23T20:31:38.166969+0000","last_fullsized":"2023-09-23T20:31:38.166969+0000","mapping_epoch":258,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:36:19.089724+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":102,"num_keys_recovered":30,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":164,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-09-23T20:31:38.140951+0000","last_change":"2023-09-23T20:29:45.531343+0000","last_active":"2023-09-23T20:31:38.140951+0000","last_peered":"2023-09-23T20:31:38.140951+0000","last_clean":"2023-09-23T20:31:38.140951+0000","last_became_active":"2023-09-23T20:29:45.530713+0000","last_became_peered":"2023-09-23T20:29:45.530713+0000","last_unstale":"2023-09-23T20:31:38.140951+0000","last_undegraded":"2023-09-23T20:31:38.140951+0000","last_fullsized":"2023-09-23T20:31:38.140951+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:28:00.245911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"90'161","reported_seq":522,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-09-23T20:31:38.115661+0000","last_change":"2023-09-23T20:29:07.045212+0000","last_active":"2023-09-23T20:31:38.115661+0000","last_peered":"2023-09-23T20:31:38.115661+0000","last_clean":"2023-09-23T20:31:38.115661+0000","last_became_active":"2023-09-23T20:29:07.044892+0000","last_became_peered":"2023-09-23T20:29:07.044892+0000","last_unstale":"2023-09-23T20:31:38.115661+0000","last_undegraded":"2023-09-23T20:31:38.115661+0000","last_fullsized":"2023-09-23T20:31:38.115661+0000","mapping_epoch":253,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":254,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:32.801168+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:32.801168+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":265,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-09-23T20:31:38.106612+0000","last_change":"2023-09-23T20:25:02.953993+0000","last_active":"2023-09-23T20:31:38.106612+0000","last_peered":"2023-09-23T20:31:38.106612+0000","last_clean":"2023-09-23T20:31:38.106612+0000","last_became_active":"2023-09-23T20:25:02.953672+0000","last_became_peered":"2023-09-23T20:25:02.953672+0000","last_unstale":"2023-09-23T20:31:38.106612+0000","last_undegraded":"2023-09-23T20:31:38.106612+0000","last_fullsized":"2023-09-23T20:31:38.106612+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:26:08.603577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":280,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-09-23T20:31:38.116538+0000","last_change":"2023-09-23T20:25:02.949584+0000","last_active":"2023-09-23T20:31:38.116538+0000","last_peered":"2023-09-23T20:31:38.116538+0000","last_clean":"2023-09-23T20:31:38.116538+0000","last_became_active":"2023-09-23T20:25:02.949132+0000","last_became_peered":"2023-09-23T20:25:02.949132+0000","last_unstale":"2023-09-23T20:31:38.116538+0000","last_undegraded":"2023-09-23T20:31:38.116538+0000","last_fullsized":"2023-09-23T20:31:38.116538+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:34:20.509823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"90'161","reported_seq":540,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-09-23T20:31:42.544333+0000","last_change":"2023-09-23T20:31:42.544488+0000","last_active":"2023-09-23T20:31:42.544333+0000","last_peered":"2023-09-23T20:31:42.544333+0000","last_clean":"2023-09-23T20:31:42.544333+0000","last_became_active":"2023-09-23T20:29:46.066622+0000","last_became_peered":"2023-09-23T20:29:46.066622+0000","last_unstale":"2023-09-23T20:31:42.544333+0000","last_undegraded":"2023-09-23T20:31:42.544333+0000","last_fullsized":"2023-09-23T20:31:42.544333+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:42.544308+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:42.544308+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:15:07.852825+0000","scrub_duration":0.0029958179999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":275,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-09-23T20:31:38.166959+0000","last_change":"2023-09-23T20:29:28.364113+0000","last_active":"2023-09-23T20:31:38.166959+0000","last_peered":"2023-09-23T20:31:38.166959+0000","last_clean":"2023-09-23T20:31:38.166959+0000","last_became_active":"2023-09-23T20:29:28.363788+0000","last_became_peered":"2023-09-23T20:29:28.363788+0000","last_unstale":"2023-09-23T20:31:38.166959+0000","last_undegraded":"2023-09-23T20:31:38.166959+0000","last_fullsized":"2023-09-23T20:31:38.166959+0000","mapping_epoch":258,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":259,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:39:49.658803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":275,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-09-23T20:31:38.106218+0000","last_change":"2023-09-23T20:25:02.957010+0000","last_active":"2023-09-23T20:31:38.106218+0000","last_peered":"2023-09-23T20:31:38.106218+0000","last_clean":"2023-09-23T20:31:38.106218+0000","last_became_active":"2023-09-23T20:25:02.956267+0000","last_became_peered":"2023-09-23T20:25:02.956267+0000","last_unstale":"2023-09-23T20:31:38.106218+0000","last_undegraded":"2023-09-23T20:31:38.106218+0000","last_fullsized":"2023-09-23T20:31:38.106218+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:10:55.299422+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"90'161","reported_seq":541,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-09-23T20:31:38.116051+0000","last_change":"2023-09-23T20:29:46.079370+0000","last_active":"2023-09-23T20:31:38.116051+0000","last_peered":"2023-09-23T20:31:38.116051+0000","last_clean":"2023-09-23T20:31:38.116051+0000","last_became_active":"2023-09-23T20:29:46.078873+0000","last_became_peered":"2023-09-23T20:29:46.078873+0000","last_unstale":"2023-09-23T20:31:38.116051+0000","last_undegraded":"2023-09-23T20:31:38.116051+0000","last_fullsized":"2023-09-23T20:31:38.116051+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:35.734584+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:35.734584+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":71,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-09-23T20:31:38.113417+0000","last_change":"2023-09-23T20:29:04.179171+0000","last_active":"2023-09-23T20:31:38.113417+0000","last_peered":"2023-09-23T20:31:38.113417+0000","last_clean":"2023-09-23T20:31:38.113417+0000","last_became_active":"2023-09-23T20:29:04.128672+0000","last_became_peered":"2023-09-23T20:29:04.128672+0000","last_unstale":"2023-09-23T20:31:38.113417+0000","last_undegraded":"2023-09-23T20:31:38.113417+0000","last_fullsized":"2023-09-23T20:31:38.113417+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:49:15.506106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":164,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-09-23T20:31:38.141144+0000","last_change":"2023-09-23T20:29:45.528782+0000","last_active":"2023-09-23T20:31:38.141144+0000","last_peered":"2023-09-23T20:31:38.141144+0000","last_clean":"2023-09-23T20:31:38.141144+0000","last_became_active":"2023-09-23T20:29:45.528389+0000","last_became_peered":"2023-09-23T20:29:45.528389+0000","last_unstale":"2023-09-23T20:31:38.141144+0000","last_undegraded":"2023-09-23T20:31:38.141144+0000","last_fullsized":"2023-09-23T20:31:38.141144+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:13:41.453499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.b","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T20:31:48.135169+0000","last_change":"2023-09-23T20:31:48.135169+0000","last_active":"2023-09-23T20:31:48.135169+0000","last_peered":"2023-09-23T20:31:48.135169+0000","last_clean":"2023-09-23T20:31:48.135169+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T20:31:48.135169+0000","last_undegraded":"2023-09-23T20:31:48.135169+0000","last_fullsized":"2023-09-23T20:31:48.135169+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.d","version":"90'161","reported_seq":541,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-09-23T20:31:38.167234+0000","last_change":"2023-09-23T20:29:46.061373+0000","last_active":"2023-09-23T20:31:38.167234+0000","last_peered":"2023-09-23T20:31:38.167234+0000","last_clean":"2023-09-23T20:31:38.167234+0000","last_became_active":"2023-09-23T20:29:46.061003+0000","last_became_peered":"2023-09-23T20:29:46.061003+0000","last_unstale":"2023-09-23T20:31:38.167234+0000","last_undegraded":"2023-09-23T20:31:38.167234+0000","last_fullsized":"2023-09-23T20:31:38.167234+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:12.809933+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:12.809933+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":291,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-09-23T20:31:38.167467+0000","last_change":"2023-09-23T20:29:59.586041+0000","last_active":"2023-09-23T20:31:38.167467+0000","last_peered":"2023-09-23T20:31:38.167467+0000","last_clean":"2023-09-23T20:31:38.167467+0000","last_became_active":"2023-09-23T20:29:46.059906+0000","last_became_peered":"2023-09-23T20:29:46.059906+0000","last_unstale":"2023-09-23T20:31:38.167467+0000","last_undegraded":"2023-09-23T20:31:38.167467+0000","last_fullsized":"2023-09-23T20:31:38.167467+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:26:12.412320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":263,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-09-23T20:31:38.115790+0000","last_change":"2023-09-23T20:29:04.128604+0000","last_active":"2023-09-23T20:31:38.115790+0000","last_peered":"2023-09-23T20:31:38.115790+0000","last_clean":"2023-09-23T20:31:38.115790+0000","last_became_active":"2023-09-23T20:29:04.128003+0000","last_became_peered":"2023-09-23T20:29:04.128003+0000","last_unstale":"2023-09-23T20:31:38.115790+0000","last_undegraded":"2023-09-23T20:31:38.115790+0000","last_fullsized":"2023-09-23T20:31:38.115790+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:50:49.385599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T20:31:48.135169+0000","last_change":"2023-09-23T20:31:48.135169+0000","last_active":"2023-09-23T20:31:48.135169+0000","last_peered":"2023-09-23T20:31:48.135169+0000","last_clean":"2023-09-23T20:31:48.135169+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T20:31:48.135169+0000","last_undegraded":"2023-09-23T20:31:48.135169+0000","last_fullsized":"2023-09-23T20:31:48.135169+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.c","version":"90'161","reported_seq":597,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-09-23T20:31:41.701187+0000","last_change":"2023-09-23T20:31:41.701273+0000","last_active":"2023-09-23T20:31:41.701187+0000","last_peered":"2023-09-23T20:31:41.701187+0000","last_clean":"2023-09-23T20:31:41.701187+0000","last_became_active":"2023-09-23T20:30:01.493025+0000","last_became_peered":"2023-09-23T20:30:01.493025+0000","last_unstale":"2023-09-23T20:31:41.701187+0000","last_undegraded":"2023-09-23T20:31:41.701187+0000","last_fullsized":"2023-09-23T20:31:41.701187+0000","mapping_epoch":275,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":276,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:41.701171+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:41.701171+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:54:32.156494+0000","scrub_duration":0.002180043,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":298,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-09-23T20:31:38.167573+0000","last_change":"2023-09-23T20:29:59.567502+0000","last_active":"2023-09-23T20:31:38.167573+0000","last_peered":"2023-09-23T20:31:38.167573+0000","last_clean":"2023-09-23T20:31:38.167573+0000","last_became_active":"2023-09-23T20:29:46.055041+0000","last_became_peered":"2023-09-23T20:29:46.055041+0000","last_unstale":"2023-09-23T20:31:38.167573+0000","last_undegraded":"2023-09-23T20:31:38.167573+0000","last_fullsized":"2023-09-23T20:31:38.167573+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:47:13.874812+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":199,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-09-23T20:31:38.109354+0000","last_change":"2023-09-23T20:29:46.178516+0000","last_active":"2023-09-23T20:31:38.109354+0000","last_peered":"2023-09-23T20:31:38.109354+0000","last_clean":"2023-09-23T20:31:38.109354+0000","last_became_active":"2023-09-23T20:29:46.177858+0000","last_became_peered":"2023-09-23T20:29:46.177858+0000","last_unstale":"2023-09-23T20:31:38.109354+0000","last_undegraded":"2023-09-23T20:31:38.109354+0000","last_fullsized":"2023-09-23T20:31:38.109354+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:09:14.906568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.9","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T20:31:48.135169+0000","last_change":"2023-09-23T20:31:48.135169+0000","last_active":"2023-09-23T20:31:48.135169+0000","last_peered":"2023-09-23T20:31:48.135169+0000","last_clean":"2023-09-23T20:31:48.135169+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T20:31:48.135169+0000","last_undegraded":"2023-09-23T20:31:48.135169+0000","last_fullsized":"2023-09-23T20:31:48.135169+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.f","version":"90'161","reported_seq":494,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-09-23T20:31:38.114055+0000","last_change":"2023-09-23T20:29:28.374177+0000","last_active":"2023-09-23T20:31:38.114055+0000","last_peered":"2023-09-23T20:31:38.114055+0000","last_clean":"2023-09-23T20:31:38.114055+0000","last_became_active":"2023-09-23T20:29:28.373750+0000","last_became_peered":"2023-09-23T20:29:28.373750+0000","last_unstale":"2023-09-23T20:31:38.114055+0000","last_undegraded":"2023-09-23T20:31:38.114055+0000","last_fullsized":"2023-09-23T20:31:38.114055+0000","mapping_epoch":258,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":259,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:32.746340+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:32.746340+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,3],"acting":[6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":274,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-09-23T20:31:38.167281+0000","last_change":"2023-09-23T20:29:04.184893+0000","last_active":"2023-09-23T20:31:38.167281+0000","last_peered":"2023-09-23T20:31:38.167281+0000","last_clean":"2023-09-23T20:31:38.167281+0000","last_became_active":"2023-09-23T20:29:04.152723+0000","last_became_peered":"2023-09-23T20:29:04.152723+0000","last_unstale":"2023-09-23T20:31:38.167281+0000","last_undegraded":"2023-09-23T20:31:38.167281+0000","last_fullsized":"2023-09-23T20:31:38.167281+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:51:27.017151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":75,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-09-23T20:31:38.113631+0000","last_change":"2023-09-23T20:29:46.086912+0000","last_active":"2023-09-23T20:31:38.113631+0000","last_peered":"2023-09-23T20:31:38.113631+0000","last_clean":"2023-09-23T20:31:38.113631+0000","last_became_active":"2023-09-23T20:29:46.085471+0000","last_became_peered":"2023-09-23T20:29:46.085471+0000","last_unstale":"2023-09-23T20:31:38.113631+0000","last_undegraded":"2023-09-23T20:31:38.113631+0000","last_fullsized":"2023-09-23T20:31:38.113631+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:04:55.751660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.8","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T20:31:48.135169+0000","last_change":"2023-09-23T20:31:48.135169+0000","last_active":"2023-09-23T20:31:48.135169+0000","last_peered":"2023-09-23T20:31:48.135169+0000","last_clean":"2023-09-23T20:31:48.135169+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T20:31:48.135169+0000","last_undegraded":"2023-09-23T20:31:48.135169+0000","last_fullsized":"2023-09-23T20:31:48.135169+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.e","version":"90'161","reported_seq":491,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-09-23T20:31:44.387173+0000","last_change":"2023-09-23T20:31:44.387138+0000","last_active":"2023-09-23T20:31:44.387173+0000","last_peered":"2023-09-23T20:31:44.387173+0000","last_clean":"2023-09-23T20:31:44.387173+0000","last_became_active":"2023-09-23T20:29:31.394053+0000","last_became_peered":"2023-09-23T20:29:31.394053+0000","last_unstale":"2023-09-23T20:31:44.387173+0000","last_undegraded":"2023-09-23T20:31:44.387173+0000","last_fullsized":"2023-09-23T20:31:44.387173+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:44.386120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:44.386120+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:01:15.591281+0000","scrub_duration":0.0015151330000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":308,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-09-23T20:31:38.116185+0000","last_change":"2023-09-23T20:29:04.138284+0000","last_active":"2023-09-23T20:31:38.116185+0000","last_peered":"2023-09-23T20:31:38.116185+0000","last_clean":"2023-09-23T20:31:38.116185+0000","last_became_active":"2023-09-23T20:29:04.137960+0000","last_became_peered":"2023-09-23T20:29:04.137960+0000","last_unstale":"2023-09-23T20:31:38.116185+0000","last_undegraded":"2023-09-23T20:31:38.116185+0000","last_fullsized":"2023-09-23T20:31:38.116185+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:18:59.548311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":275,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-09-23T20:31:38.106459+0000","last_change":"2023-09-23T20:29:46.061136+0000","last_active":"2023-09-23T20:31:38.106459+0000","last_peered":"2023-09-23T20:31:38.106459+0000","last_clean":"2023-09-23T20:31:38.106459+0000","last_became_active":"2023-09-23T20:29:46.060777+0000","last_became_peered":"2023-09-23T20:29:46.060777+0000","last_unstale":"2023-09-23T20:31:38.106459+0000","last_undegraded":"2023-09-23T20:31:38.106459+0000","last_fullsized":"2023-09-23T20:31:38.106459+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:08:43.673870+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T20:31:48.135169+0000","last_change":"2023-09-23T20:31:48.135169+0000","last_active":"2023-09-23T20:31:48.135169+0000","last_peered":"2023-09-23T20:31:48.135169+0000","last_clean":"2023-09-23T20:31:48.135169+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T20:31:48.135169+0000","last_undegraded":"2023-09-23T20:31:48.135169+0000","last_fullsized":"2023-09-23T20:31:48.135169+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":684,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-09-23T20:31:38.167127+0000","last_change":"2023-09-23T20:28:05.872149+0000","last_active":"2023-09-23T20:31:38.167127+0000","last_peered":"2023-09-23T20:31:38.167127+0000","last_clean":"2023-09-23T20:31:38.167127+0000","last_became_active":"2023-09-23T20:27:11.961310+0000","last_became_peered":"2023-09-23T20:27:11.961310+0000","last_unstale":"2023-09-23T20:31:38.167127+0000","last_undegraded":"2023-09-23T20:31:38.167127+0000","last_fullsized":"2023-09-23T20:31:38.167127+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:28:05.872074+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:28:05.872074+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:52:32.495955+0000","scrub_duration":0.0029215980000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":177,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-09-23T20:31:38.140164+0000","last_change":"2023-09-23T20:29:49.873018+0000","last_active":"2023-09-23T20:31:38.140164+0000","last_peered":"2023-09-23T20:31:38.140164+0000","last_clean":"2023-09-23T20:31:38.140164+0000","last_became_active":"2023-09-23T20:29:46.070369+0000","last_became_peered":"2023-09-23T20:29:46.070369+0000","last_unstale":"2023-09-23T20:31:38.140164+0000","last_undegraded":"2023-09-23T20:31:38.140164+0000","last_fullsized":"2023-09-23T20:31:38.140164+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:24:13.119725+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":314,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-09-23T20:31:41.314029+0000","last_change":"2023-09-23T20:29:46.173195+0000","last_active":"2023-09-23T20:31:41.314029+0000","last_peered":"2023-09-23T20:31:41.314029+0000","last_clean":"2023-09-23T20:31:41.314029+0000","last_became_active":"2023-09-23T20:29:46.172808+0000","last_became_peered":"2023-09-23T20:29:46.172808+0000","last_unstale":"2023-09-23T20:31:41.314029+0000","last_undegraded":"2023-09-23T20:31:41.314029+0000","last_fullsized":"2023-09-23T20:31:41.314029+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:41:58.927658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.e","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T20:31:48.135169+0000","last_change":"2023-09-23T20:31:48.135169+0000","last_active":"2023-09-23T20:31:48.135169+0000","last_peered":"2023-09-23T20:31:48.135169+0000","last_clean":"2023-09-23T20:31:48.135169+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T20:31:48.135169+0000","last_undegraded":"2023-09-23T20:31:48.135169+0000","last_fullsized":"2023-09-23T20:31:48.135169+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":526,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-09-23T20:31:38.115657+0000","last_change":"2023-09-23T20:27:31.757060+0000","last_active":"2023-09-23T20:31:38.115657+0000","last_peered":"2023-09-23T20:31:38.115657+0000","last_clean":"2023-09-23T20:31:38.115657+0000","last_became_active":"2023-09-23T20:27:11.959516+0000","last_became_peered":"2023-09-23T20:27:11.959516+0000","last_unstale":"2023-09-23T20:31:38.115657+0000","last_undegraded":"2023-09-23T20:31:38.115657+0000","last_fullsized":"2023-09-23T20:31:38.115657+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:31.756974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:31.756974+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:35:11.939335+0000","scrub_duration":0.0031802670000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":87,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-09-23T20:31:38.113700+0000","last_change":"2023-09-23T20:29:46.087097+0000","last_active":"2023-09-23T20:31:38.113700+0000","last_peered":"2023-09-23T20:31:38.113700+0000","last_clean":"2023-09-23T20:31:38.113700+0000","last_became_active":"2023-09-23T20:29:46.085908+0000","last_became_peered":"2023-09-23T20:29:46.085908+0000","last_unstale":"2023-09-23T20:31:38.113700+0000","last_undegraded":"2023-09-23T20:31:38.113700+0000","last_fullsized":"2023-09-23T20:31:38.113700+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:38:46.001432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":212,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-09-23T20:31:41.330356+0000","last_change":"2023-09-23T20:29:28.370476+0000","last_active":"2023-09-23T20:31:41.330356+0000","last_peered":"2023-09-23T20:31:41.330356+0000","last_clean":"2023-09-23T20:31:41.330356+0000","last_became_active":"2023-09-23T20:29:28.370158+0000","last_became_peered":"2023-09-23T20:29:28.370158+0000","last_unstale":"2023-09-23T20:31:41.330356+0000","last_undegraded":"2023-09-23T20:31:41.330356+0000","last_fullsized":"2023-09-23T20:31:41.330356+0000","mapping_epoch":258,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":259,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:09:47.335404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.d","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T20:31:48.135169+0000","last_change":"2023-09-23T20:31:48.135169+0000","last_active":"2023-09-23T20:31:48.135169+0000","last_peered":"2023-09-23T20:31:48.135169+0000","last_clean":"2023-09-23T20:31:48.135169+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T20:31:48.135169+0000","last_undegraded":"2023-09-23T20:31:48.135169+0000","last_fullsized":"2023-09-23T20:31:48.135169+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.b","version":"90'161","reported_seq":531,"reported_epoch":302,"state":"active+clean+scrubbing","last_fresh":"2023-09-23T20:31:41.325100+0000","last_change":"2023-09-23T20:31:41.540505+0000","last_active":"2023-09-23T20:31:41.325100+0000","last_peered":"2023-09-23T20:31:41.325100+0000","last_clean":"2023-09-23T20:31:41.325100+0000","last_became_active":"2023-09-23T20:29:45.534070+0000","last_became_peered":"2023-09-23T20:29:45.534070+0000","last_unstale":"2023-09-23T20:31:41.325100+0000","last_undegraded":"2023-09-23T20:31:41.325100+0000","last_fullsized":"2023-09-23T20:31:41.325100+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:12.386098+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:12.386098+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0019500940000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":214,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-09-23T20:31:41.324092+0000","last_change":"2023-09-23T20:29:46.061620+0000","last_active":"2023-09-23T20:31:41.324092+0000","last_peered":"2023-09-23T20:31:41.324092+0000","last_clean":"2023-09-23T20:31:41.324092+0000","last_became_active":"2023-09-23T20:29:46.061271+0000","last_became_peered":"2023-09-23T20:29:46.061271+0000","last_unstale":"2023-09-23T20:31:41.324092+0000","last_undegraded":"2023-09-23T20:31:41.324092+0000","last_fullsized":"2023-09-23T20:31:41.324092+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:24:51.409521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":297,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-09-23T20:31:38.113557+0000","last_change":"2023-09-23T20:29:46.085962+0000","last_active":"2023-09-23T20:31:38.113557+0000","last_peered":"2023-09-23T20:31:38.113557+0000","last_clean":"2023-09-23T20:31:38.113557+0000","last_became_active":"2023-09-23T20:29:46.082826+0000","last_became_peered":"2023-09-23T20:29:46.082826+0000","last_unstale":"2023-09-23T20:31:38.113557+0000","last_undegraded":"2023-09-23T20:31:38.113557+0000","last_fullsized":"2023-09-23T20:31:38.113557+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:26:17.599051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.c","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T20:31:48.135169+0000","last_change":"2023-09-23T20:31:48.135169+0000","last_active":"2023-09-23T20:31:48.135169+0000","last_peered":"2023-09-23T20:31:48.135169+0000","last_clean":"2023-09-23T20:31:48.135169+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T20:31:48.135169+0000","last_undegraded":"2023-09-23T20:31:48.135169+0000","last_fullsized":"2023-09-23T20:31:48.135169+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":566,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-09-23T20:31:38.115995+0000","last_change":"2023-09-23T20:29:07.053561+0000","last_active":"2023-09-23T20:31:38.115995+0000","last_peered":"2023-09-23T20:31:38.115995+0000","last_clean":"2023-09-23T20:31:38.115995+0000","last_became_active":"2023-09-23T20:29:07.053177+0000","last_became_peered":"2023-09-23T20:29:07.053177+0000","last_unstale":"2023-09-23T20:31:38.115995+0000","last_undegraded":"2023-09-23T20:31:38.115995+0000","last_fullsized":"2023-09-23T20:31:38.115995+0000","mapping_epoch":253,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":254,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:30.727267+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:30.727267+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:04:32.661761+0000","scrub_duration":0.0052560289999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":165,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-09-23T20:31:38.136085+0000","last_change":"2023-09-23T20:29:45.529576+0000","last_active":"2023-09-23T20:31:38.136085+0000","last_peered":"2023-09-23T20:31:38.136085+0000","last_clean":"2023-09-23T20:31:38.136085+0000","last_became_active":"2023-09-23T20:29:45.528858+0000","last_became_peered":"2023-09-23T20:29:45.528858+0000","last_unstale":"2023-09-23T20:31:38.136085+0000","last_undegraded":"2023-09-23T20:31:38.136085+0000","last_fullsized":"2023-09-23T20:31:38.136085+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:42:15.589821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":112,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-09-23T20:31:38.110566+0000","last_change":"2023-09-23T20:29:46.064515+0000","last_active":"2023-09-23T20:31:38.110566+0000","last_peered":"2023-09-23T20:31:38.110566+0000","last_clean":"2023-09-23T20:31:38.110566+0000","last_became_active":"2023-09-23T20:29:46.064253+0000","last_became_peered":"2023-09-23T20:29:46.064253+0000","last_unstale":"2023-09-23T20:31:38.110566+0000","last_undegraded":"2023-09-23T20:31:38.110566+0000","last_fullsized":"2023-09-23T20:31:38.110566+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:21:20.039440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T20:31:48.135169+0000","last_change":"2023-09-23T20:31:48.135169+0000","last_active":"2023-09-23T20:31:48.135169+0000","last_peered":"2023-09-23T20:31:48.135169+0000","last_clean":"2023-09-23T20:31:48.135169+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T20:31:48.135169+0000","last_undegraded":"2023-09-23T20:31:48.135169+0000","last_fullsized":"2023-09-23T20:31:48.135169+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":531,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-09-23T20:31:45.196388+0000","last_change":"2023-09-23T20:31:45.196356+0000","last_active":"2023-09-23T20:31:45.196388+0000","last_peered":"2023-09-23T20:31:45.196388+0000","last_clean":"2023-09-23T20:31:45.196388+0000","last_became_active":"2023-09-23T20:29:07.115647+0000","last_became_peered":"2023-09-23T20:29:07.115647+0000","last_unstale":"2023-09-23T20:31:45.196388+0000","last_undegraded":"2023-09-23T20:31:45.196388+0000","last_fullsized":"2023-09-23T20:31:45.196388+0000","mapping_epoch":253,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":254,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:45.196285+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:45.196285+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:33:06.666183+0000","scrub_duration":0.0014380860000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"86'119","reported_seq":605,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-09-23T20:31:38.110935+0000","last_change":"2023-09-23T20:29:45.521257+0000","last_active":"2023-09-23T20:31:38.110935+0000","last_peered":"2023-09-23T20:31:38.110935+0000","last_clean":"2023-09-23T20:31:38.110935+0000","last_became_active":"2023-09-23T20:29:45.520546+0000","last_became_peered":"2023-09-23T20:29:45.520546+0000","last_unstale":"2023-09-23T20:31:38.110935+0000","last_undegraded":"2023-09-23T20:31:38.110935+0000","last_fullsized":"2023-09-23T20:31:38.110935+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":119,"log_dups_size":0,"ondisk_log_size":119,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:58:56.085777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032705499999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":523,"num_read_kb":3503,"num_write":136,"num_write_kb":3128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":318,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-09-23T20:31:38.116489+0000","last_change":"2023-09-23T20:25:39.637160+0000","last_active":"2023-09-23T20:31:38.116489+0000","last_peered":"2023-09-23T20:31:38.116489+0000","last_clean":"2023-09-23T20:31:38.116489+0000","last_became_active":"2023-09-23T20:25:39.636860+0000","last_became_peered":"2023-09-23T20:25:39.636860+0000","last_unstale":"2023-09-23T20:31:38.116489+0000","last_undegraded":"2023-09-23T20:31:38.116489+0000","last_fullsized":"2023-09-23T20:31:38.116489+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:21:47.860063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":207,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-09-23T20:31:38.109381+0000","last_change":"2023-09-23T20:29:46.090683+0000","last_active":"2023-09-23T20:31:38.109381+0000","last_peered":"2023-09-23T20:31:38.109381+0000","last_clean":"2023-09-23T20:31:38.109381+0000","last_became_active":"2023-09-23T20:29:46.090131+0000","last_became_peered":"2023-09-23T20:29:46.090131+0000","last_unstale":"2023-09-23T20:31:38.109381+0000","last_undegraded":"2023-09-23T20:31:38.109381+0000","last_fullsized":"2023-09-23T20:31:38.109381+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:36:35.754684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T20:31:48.135169+0000","last_change":"2023-09-23T20:31:48.135169+0000","last_active":"2023-09-23T20:31:48.135169+0000","last_peered":"2023-09-23T20:31:48.135169+0000","last_clean":"2023-09-23T20:31:48.135169+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T20:31:48.135169+0000","last_undegraded":"2023-09-23T20:31:48.135169+0000","last_fullsized":"2023-09-23T20:31:48.135169+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":536,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-09-23T20:31:38.115825+0000","last_change":"2023-09-23T20:27:55.213707+0000","last_active":"2023-09-23T20:31:38.115825+0000","last_peered":"2023-09-23T20:31:38.115825+0000","last_clean":"2023-09-23T20:31:38.115825+0000","last_became_active":"2023-09-23T20:27:55.213394+0000","last_became_peered":"2023-09-23T20:27:55.213394+0000","last_unstale":"2023-09-23T20:31:38.115825+0000","last_undegraded":"2023-09-23T20:31:38.115825+0000","last_fullsized":"2023-09-23T20:31:38.115825+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:34.821789+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:34.821789+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0042962870000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"86'106","reported_seq":1020,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-09-23T20:31:38.115824+0000","last_change":"2023-09-23T20:31:31.531882+0000","last_active":"2023-09-23T20:31:38.115824+0000","last_peered":"2023-09-23T20:31:38.115824+0000","last_clean":"2023-09-23T20:31:38.115824+0000","last_became_active":"2023-09-23T20:29:46.078501+0000","last_became_peered":"2023-09-23T20:29:46.078501+0000","last_unstale":"2023-09-23T20:31:38.115824+0000","last_undegraded":"2023-09-23T20:31:38.115824+0000","last_fullsized":"2023-09-23T20:31:38.115824+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":106,"log_dups_size":0,"ondisk_log_size":106,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:37.015974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00099302599999999993,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1105,"num_read_kb":1689,"num_write":100,"num_write_kb":404,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":295,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-09-23T20:31:38.106710+0000","last_change":"2023-09-23T20:29:46.059640+0000","last_active":"2023-09-23T20:31:38.106710+0000","last_peered":"2023-09-23T20:31:38.106710+0000","last_clean":"2023-09-23T20:31:38.106710+0000","last_became_active":"2023-09-23T20:29:46.059279+0000","last_became_peered":"2023-09-23T20:29:46.059279+0000","last_unstale":"2023-09-23T20:31:38.106710+0000","last_undegraded":"2023-09-23T20:31:38.106710+0000","last_fullsized":"2023-09-23T20:31:38.106710+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:03:31.536847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":258,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-09-23T20:31:38.110883+0000","last_change":"2023-09-23T20:29:45.533406+0000","last_active":"2023-09-23T20:31:38.110883+0000","last_peered":"2023-09-23T20:31:38.110883+0000","last_clean":"2023-09-23T20:31:38.110883+0000","last_became_active":"2023-09-23T20:29:45.526001+0000","last_became_peered":"2023-09-23T20:29:45.526001+0000","last_unstale":"2023-09-23T20:31:38.110883+0000","last_undegraded":"2023-09-23T20:31:38.110883+0000","last_fullsized":"2023-09-23T20:31:38.110883+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:43.787251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.4","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T20:31:48.135169+0000","last_change":"2023-09-23T20:31:48.135169+0000","last_active":"2023-09-23T20:31:48.135169+0000","last_peered":"2023-09-23T20:31:48.135169+0000","last_clean":"2023-09-23T20:31:48.135169+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T20:31:48.135169+0000","last_undegraded":"2023-09-23T20:31:48.135169+0000","last_fullsized":"2023-09-23T20:31:48.135169+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":724,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-09-23T20:31:43.391063+0000","last_change":"2023-09-23T20:31:43.391012+0000","last_active":"2023-09-23T20:31:43.391063+0000","last_peered":"2023-09-23T20:31:43.391063+0000","last_clean":"2023-09-23T20:31:43.391063+0000","last_became_active":"2023-09-23T20:29:31.389288+0000","last_became_peered":"2023-09-23T20:29:31.389288+0000","last_unstale":"2023-09-23T20:31:43.391063+0000","last_undegraded":"2023-09-23T20:31:43.391063+0000","last_fullsized":"2023-09-23T20:31:43.391063+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:43.390908+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:43.390908+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:51:15.405092+0000","scrub_duration":0.0032721,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,2],"acting":[7,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"86'130","reported_seq":613,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-09-23T20:31:38.111455+0000","last_change":"2023-09-23T20:25:39.637458+0000","last_active":"2023-09-23T20:31:38.111455+0000","last_peered":"2023-09-23T20:31:38.111455+0000","last_clean":"2023-09-23T20:31:38.111455+0000","last_became_active":"2023-09-23T20:25:39.636773+0000","last_became_peered":"2023-09-23T20:25:39.636773+0000","last_unstale":"2023-09-23T20:31:38.111455+0000","last_undegraded":"2023-09-23T20:31:38.111455+0000","last_fullsized":"2023-09-23T20:31:38.111455+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:00:32.810564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011300049999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":512,"num_read_kb":1820,"num_write":133,"num_write_kb":1347,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":248,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-09-23T20:31:38.109187+0000","last_change":"2023-09-23T20:29:46.073394+0000","last_active":"2023-09-23T20:31:38.109187+0000","last_peered":"2023-09-23T20:31:38.109187+0000","last_clean":"2023-09-23T20:31:38.109187+0000","last_became_active":"2023-09-23T20:29:46.072067+0000","last_became_peered":"2023-09-23T20:29:46.072067+0000","last_unstale":"2023-09-23T20:31:38.109187+0000","last_undegraded":"2023-09-23T20:31:38.109187+0000","last_fullsized":"2023-09-23T20:31:38.109187+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:01:56.944614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":240,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-09-23T20:31:38.110493+0000","last_change":"2023-09-23T20:29:45.533185+0000","last_active":"2023-09-23T20:31:38.110493+0000","last_peered":"2023-09-23T20:31:38.110493+0000","last_clean":"2023-09-23T20:31:38.110493+0000","last_became_active":"2023-09-23T20:29:45.526765+0000","last_became_peered":"2023-09-23T20:29:45.526765+0000","last_unstale":"2023-09-23T20:31:38.110493+0000","last_undegraded":"2023-09-23T20:31:38.110493+0000","last_fullsized":"2023-09-23T20:31:38.110493+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:18:33.063215+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T20:31:48.135169+0000","last_change":"2023-09-23T20:31:48.135169+0000","last_active":"2023-09-23T20:31:48.135169+0000","last_peered":"2023-09-23T20:31:48.135169+0000","last_clean":"2023-09-23T20:31:48.135169+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T20:31:48.135169+0000","last_undegraded":"2023-09-23T20:31:48.135169+0000","last_fullsized":"2023-09-23T20:31:48.135169+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.3","version":"86'194","reported_seq":950,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-09-23T20:31:38.116041+0000","last_change":"2023-09-23T20:25:40.618665+0000","last_active":"2023-09-23T20:31:38.116041+0000","last_peered":"2023-09-23T20:31:38.116041+0000","last_clean":"2023-09-23T20:31:38.116041+0000","last_became_active":"2023-09-23T20:25:40.618087+0000","last_became_peered":"2023-09-23T20:25:40.618087+0000","last_unstale":"2023-09-23T20:31:38.116041+0000","last_undegraded":"2023-09-23T20:31:38.116041+0000","last_fullsized":"2023-09-23T20:31:38.116041+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":194,"log_dups_size":0,"ondisk_log_size":194,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:07:16.114968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00037995999999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1025,"num_read_kb":1201,"num_write":173,"num_write_kb":1276,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":735,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-09-23T20:31:45.343885+0000","last_change":"2023-09-23T20:31:45.343835+0000","last_active":"2023-09-23T20:31:45.343885+0000","last_peered":"2023-09-23T20:31:45.343885+0000","last_clean":"2023-09-23T20:31:45.343885+0000","last_became_active":"2023-09-23T20:29:07.042407+0000","last_became_peered":"2023-09-23T20:29:07.042407+0000","last_unstale":"2023-09-23T20:31:45.343885+0000","last_undegraded":"2023-09-23T20:31:45.343885+0000","last_fullsized":"2023-09-23T20:31:45.343885+0000","mapping_epoch":253,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":254,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:45.343742+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:45.343742+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:33:12.745366+0000","scrub_duration":0.001475802,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":168,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-09-23T20:31:38.136130+0000","last_change":"2023-09-23T20:29:04.216400+0000","last_active":"2023-09-23T20:31:38.136130+0000","last_peered":"2023-09-23T20:31:38.136130+0000","last_clean":"2023-09-23T20:31:38.136130+0000","last_became_active":"2023-09-23T20:29:04.216069+0000","last_became_peered":"2023-09-23T20:29:04.216069+0000","last_unstale":"2023-09-23T20:31:38.136130+0000","last_undegraded":"2023-09-23T20:31:38.136130+0000","last_fullsized":"2023-09-23T20:31:38.136130+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:27:37.433035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":71,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-09-23T20:31:38.113868+0000","last_change":"2023-09-23T20:29:28.376036+0000","last_active":"2023-09-23T20:31:38.113868+0000","last_peered":"2023-09-23T20:31:38.113868+0000","last_clean":"2023-09-23T20:31:38.113868+0000","last_became_active":"2023-09-23T20:29:28.375712+0000","last_became_peered":"2023-09-23T20:29:28.375712+0000","last_unstale":"2023-09-23T20:31:38.113868+0000","last_undegraded":"2023-09-23T20:31:38.113868+0000","last_fullsized":"2023-09-23T20:31:38.113868+0000","mapping_epoch":258,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":259,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:23:50.722625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T20:31:48.135169+0000","last_change":"2023-09-23T20:31:48.135169+0000","last_active":"2023-09-23T20:31:48.135169+0000","last_peered":"2023-09-23T20:31:48.135169+0000","last_clean":"2023-09-23T20:31:48.135169+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T20:31:48.135169+0000","last_undegraded":"2023-09-23T20:31:48.135169+0000","last_fullsized":"2023-09-23T20:31:48.135169+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":542,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-09-23T20:31:43.582190+0000","last_change":"2023-09-23T20:31:43.582312+0000","last_active":"2023-09-23T20:31:43.582190+0000","last_peered":"2023-09-23T20:31:43.582190+0000","last_clean":"2023-09-23T20:31:43.582190+0000","last_became_active":"2023-09-23T20:29:46.071938+0000","last_became_peered":"2023-09-23T20:29:46.071938+0000","last_unstale":"2023-09-23T20:31:43.582190+0000","last_undegraded":"2023-09-23T20:31:43.582190+0000","last_fullsized":"2023-09-23T20:31:43.582190+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:43.582146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:43.582146+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:55:14.837811+0000","scrub_duration":0.0016219730000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,5],"acting":[1,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"86'80","reported_seq":565,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-09-23T20:31:38.166690+0000","last_change":"2023-09-23T20:29:45.536796+0000","last_active":"2023-09-23T20:31:38.166690+0000","last_peered":"2023-09-23T20:31:38.166690+0000","last_clean":"2023-09-23T20:31:38.166690+0000","last_became_active":"2023-09-23T20:29:45.536439+0000","last_became_peered":"2023-09-23T20:29:45.536439+0000","last_unstale":"2023-09-23T20:31:38.166690+0000","last_undegraded":"2023-09-23T20:31:38.166690+0000","last_fullsized":"2023-09-23T20:31:38.166690+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":80,"log_dups_size":0,"ondisk_log_size":80,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:30:57.793132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035115,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":395,"num_read_kb":1370,"num_write":66,"num_write_kb":1061,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":275,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-09-23T20:31:38.106632+0000","last_change":"2023-09-23T20:29:28.360680+0000","last_active":"2023-09-23T20:31:38.106632+0000","last_peered":"2023-09-23T20:31:38.106632+0000","last_clean":"2023-09-23T20:31:38.106632+0000","last_became_active":"2023-09-23T20:29:28.360366+0000","last_became_peered":"2023-09-23T20:29:28.360366+0000","last_unstale":"2023-09-23T20:31:38.106632+0000","last_undegraded":"2023-09-23T20:31:38.106632+0000","last_fullsized":"2023-09-23T20:31:38.106632+0000","mapping_epoch":258,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":259,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:02:40.594828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":297,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-09-23T20:31:38.107405+0000","last_change":"2023-09-23T20:29:45.690815+0000","last_active":"2023-09-23T20:31:38.107405+0000","last_peered":"2023-09-23T20:31:38.107405+0000","last_clean":"2023-09-23T20:31:38.107405+0000","last_became_active":"2023-09-23T20:29:45.690478+0000","last_became_peered":"2023-09-23T20:29:45.690478+0000","last_unstale":"2023-09-23T20:31:38.107405+0000","last_undegraded":"2023-09-23T20:31:38.107405+0000","last_fullsized":"2023-09-23T20:31:38.107405+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:38.288193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.0","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T20:31:48.135169+0000","last_change":"2023-09-23T20:31:48.135169+0000","last_active":"2023-09-23T20:31:48.135169+0000","last_peered":"2023-09-23T20:31:48.135169+0000","last_clean":"2023-09-23T20:31:48.135169+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T20:31:48.135169+0000","last_undegraded":"2023-09-23T20:31:48.135169+0000","last_fullsized":"2023-09-23T20:31:48.135169+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":524,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-09-23T20:31:44.211384+0000","last_change":"2023-09-23T20:31:44.211308+0000","last_active":"2023-09-23T20:31:44.211384+0000","last_peered":"2023-09-23T20:31:44.211384+0000","last_clean":"2023-09-23T20:31:44.211384+0000","last_became_active":"2023-09-23T20:29:07.113428+0000","last_became_peered":"2023-09-23T20:29:07.113428+0000","last_unstale":"2023-09-23T20:31:44.211384+0000","last_undegraded":"2023-09-23T20:31:44.211384+0000","last_fullsized":"2023-09-23T20:31:44.211384+0000","mapping_epoch":253,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":254,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:44.211184+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:44.211184+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:53:50.053167+0000","scrub_duration":0.0017707720000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"86'102","reported_seq":858,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-09-23T20:31:38.167337+0000","last_change":"2023-09-23T20:31:32.809213+0000","last_active":"2023-09-23T20:31:38.167337+0000","last_peered":"2023-09-23T20:31:38.167337+0000","last_clean":"2023-09-23T20:31:38.167337+0000","last_became_active":"2023-09-23T20:31:32.808744+0000","last_became_peered":"2023-09-23T20:31:32.808744+0000","last_unstale":"2023-09-23T20:31:38.167337+0000","last_undegraded":"2023-09-23T20:31:38.167337+0000","last_fullsized":"2023-09-23T20:31:38.167337+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":102,"log_dups_size":0,"ondisk_log_size":102,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:17:25.506476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030656500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":963,"num_read_kb":1234,"num_write":60,"num_write_kb":225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":224,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-09-23T20:31:38.110996+0000","last_change":"2023-09-23T20:29:45.538283+0000","last_active":"2023-09-23T20:31:38.110996+0000","last_peered":"2023-09-23T20:31:38.110996+0000","last_clean":"2023-09-23T20:31:38.110996+0000","last_became_active":"2023-09-23T20:29:45.537922+0000","last_became_peered":"2023-09-23T20:29:45.537922+0000","last_unstale":"2023-09-23T20:31:38.110996+0000","last_undegraded":"2023-09-23T20:31:38.110996+0000","last_fullsized":"2023-09-23T20:31:38.110996+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:30:52.397836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":234,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-09-23T20:31:38.106664+0000","last_change":"2023-09-23T20:20:14.888874+0000","last_active":"2023-09-23T20:31:38.106664+0000","last_peered":"2023-09-23T20:31:38.106664+0000","last_clean":"2023-09-23T20:31:38.106664+0000","last_became_active":"2023-09-23T20:20:14.887724+0000","last_became_peered":"2023-09-23T20:20:14.887724+0000","last_unstale":"2023-09-23T20:31:38.106664+0000","last_undegraded":"2023-09-23T20:31:38.106664+0000","last_fullsized":"2023-09-23T20:31:38.106664+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:09:24.510326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T20:31:48.135169+0000","last_change":"2023-09-23T20:31:48.135169+0000","last_active":"2023-09-23T20:31:48.135169+0000","last_peered":"2023-09-23T20:31:48.135169+0000","last_clean":"2023-09-23T20:31:48.135169+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T20:31:48.135169+0000","last_undegraded":"2023-09-23T20:31:48.135169+0000","last_fullsized":"2023-09-23T20:31:48.135169+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":484,"reported_epoch":302,"state":"active+clean","last_fresh":"2023-09-23T20:31:44.509052+0000","last_change":"2023-09-23T20:31:44.509200+0000","last_active":"2023-09-23T20:31:44.509052+0000","last_peered":"2023-09-23T20:31:44.509052+0000","last_clean":"2023-09-23T20:31:44.509052+0000","last_became_active":"2023-09-23T20:29:28.372652+0000","last_became_peered":"2023-09-23T20:29:28.372652+0000","last_unstale":"2023-09-23T20:31:44.509052+0000","last_undegraded":"2023-09-23T20:31:44.509052+0000","last_fullsized":"2023-09-23T20:31:44.509052+0000","mapping_epoch":258,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":259,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:44.509012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:44.509012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:15:44.777003+0000","scrub_duration":0.0032186319999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,5],"acting":[2,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":154,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-09-23T20:31:38.141066+0000","last_change":"2023-09-23T20:25:03.007736+0000","last_active":"2023-09-23T20:31:38.141066+0000","last_peered":"2023-09-23T20:31:38.141066+0000","last_clean":"2023-09-23T20:31:38.141066+0000","last_became_active":"2023-09-23T20:25:02.973689+0000","last_became_peered":"2023-09-23T20:25:02.973689+0000","last_unstale":"2023-09-23T20:31:38.141066+0000","last_undegraded":"2023-09-23T20:31:38.141066+0000","last_fullsized":"2023-09-23T20:31:38.141066+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:37:25.643774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"86'128","reported_seq":657,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-09-23T20:31:38.110147+0000","last_change":"2023-09-23T20:30:45.008506+0000","last_active":"2023-09-23T20:31:38.110147+0000","last_peered":"2023-09-23T20:31:38.110147+0000","last_clean":"2023-09-23T20:31:38.110147+0000","last_became_active":"2023-09-23T20:29:46.071848+0000","last_became_peered":"2023-09-23T20:29:46.071848+0000","last_unstale":"2023-09-23T20:31:38.110147+0000","last_undegraded":"2023-09-23T20:31:38.110147+0000","last_fullsized":"2023-09-23T20:31:38.110147+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":283,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:44:18.031530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000293202,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":579,"num_read_kb":1473,"num_write":96,"num_write_kb":1062,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":150,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-09-23T20:31:38.135946+0000","last_change":"2023-09-23T20:25:02.971740+0000","last_active":"2023-09-23T20:31:38.135946+0000","last_peered":"2023-09-23T20:31:38.135946+0000","last_clean":"2023-09-23T20:31:38.135946+0000","last_became_active":"2023-09-23T20:25:02.970412+0000","last_became_peered":"2023-09-23T20:25:02.970412+0000","last_unstale":"2023-09-23T20:31:38.135946+0000","last_undegraded":"2023-09-23T20:31:38.135946+0000","last_fullsized":"2023-09-23T20:31:38.135946+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:53:17.247216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T20:31:48.135169+0000","last_change":"2023-09-23T20:31:48.135169+0000","last_active":"2023-09-23T20:31:48.135169+0000","last_peered":"2023-09-23T20:31:48.135169+0000","last_clean":"2023-09-23T20:31:48.135169+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T20:31:48.135169+0000","last_undegraded":"2023-09-23T20:31:48.135169+0000","last_fullsized":"2023-09-23T20:31:48.135169+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.4","version":"90'161","reported_seq":550,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-09-23T20:31:38.140818+0000","last_change":"2023-09-23T20:28:03.553294+0000","last_active":"2023-09-23T20:31:38.140818+0000","last_peered":"2023-09-23T20:31:38.140818+0000","last_clean":"2023-09-23T20:31:38.140818+0000","last_became_active":"2023-09-23T20:27:55.212201+0000","last_became_peered":"2023-09-23T20:27:55.212201+0000","last_unstale":"2023-09-23T20:31:38.140818+0000","last_undegraded":"2023-09-23T20:31:38.140818+0000","last_fullsized":"2023-09-23T20:31:38.140818+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:28:03.553106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:28:03.553106+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:24:06.706940+0000","scrub_duration":0.011176864,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"86'117","reported_seq":571,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-09-23T20:31:38.114162+0000","last_change":"2023-09-23T20:31:32.814392+0000","last_active":"2023-09-23T20:31:38.114162+0000","last_peered":"2023-09-23T20:31:38.114162+0000","last_clean":"2023-09-23T20:31:38.114162+0000","last_became_active":"2023-09-23T20:31:32.813857+0000","last_became_peered":"2023-09-23T20:31:32.813857+0000","last_unstale":"2023-09-23T20:31:38.114162+0000","last_undegraded":"2023-09-23T20:31:38.114162+0000","last_fullsized":"2023-09-23T20:31:38.114162+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:56:57.863005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046891400000000002,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":893,"num_read_kb":922,"num_write":82,"num_write_kb":4193,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":284,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-09-23T20:31:38.107335+0000","last_change":"2023-09-23T20:29:46.053198+0000","last_active":"2023-09-23T20:31:38.107335+0000","last_peered":"2023-09-23T20:31:38.107335+0000","last_clean":"2023-09-23T20:31:38.107335+0000","last_became_active":"2023-09-23T20:29:46.052866+0000","last_became_peered":"2023-09-23T20:29:46.052866+0000","last_unstale":"2023-09-23T20:31:38.107335+0000","last_undegraded":"2023-09-23T20:31:38.107335+0000","last_fullsized":"2023-09-23T20:31:38.107335+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:30:18.857140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":259,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-09-23T20:31:38.115828+0000","last_change":"2023-09-23T20:29:28.367722+0000","last_active":"2023-09-23T20:31:38.115828+0000","last_peered":"2023-09-23T20:31:38.115828+0000","last_clean":"2023-09-23T20:31:38.115828+0000","last_became_active":"2023-09-23T20:29:28.367404+0000","last_became_peered":"2023-09-23T20:29:28.367404+0000","last_unstale":"2023-09-23T20:31:38.115828+0000","last_undegraded":"2023-09-23T20:31:38.115828+0000","last_fullsized":"2023-09-23T20:31:38.115828+0000","mapping_epoch":258,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":259,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:02:51.886571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":256,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-09-23T20:31:38.106616+0000","last_change":"2023-09-23T20:29:46.061960+0000","last_active":"2023-09-23T20:31:38.106616+0000","last_peered":"2023-09-23T20:31:38.106616+0000","last_clean":"2023-09-23T20:31:38.106616+0000","last_became_active":"2023-09-23T20:29:46.061601+0000","last_became_peered":"2023-09-23T20:29:46.061601+0000","last_unstale":"2023-09-23T20:31:38.106616+0000","last_undegraded":"2023-09-23T20:31:38.106616+0000","last_fullsized":"2023-09-23T20:31:38.106616+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:15:52.620692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":153,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-09-23T20:31:38.112162+0000","last_change":"2023-09-23T20:29:45.538314+0000","last_active":"2023-09-23T20:31:38.112162+0000","last_peered":"2023-09-23T20:31:38.112162+0000","last_clean":"2023-09-23T20:31:38.112162+0000","last_became_active":"2023-09-23T20:29:45.536847+0000","last_became_peered":"2023-09-23T20:29:45.536847+0000","last_unstale":"2023-09-23T20:31:38.112162+0000","last_undegraded":"2023-09-23T20:31:38.112162+0000","last_fullsized":"2023-09-23T20:31:38.112162+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:51.948721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":297,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-09-23T20:31:38.116149+0000","last_change":"2023-09-23T20:29:59.578082+0000","last_active":"2023-09-23T20:31:38.116149+0000","last_peered":"2023-09-23T20:31:38.116149+0000","last_clean":"2023-09-23T20:31:38.116149+0000","last_became_active":"2023-09-23T20:29:46.072626+0000","last_became_peered":"2023-09-23T20:29:46.072626+0000","last_unstale":"2023-09-23T20:31:38.116149+0000","last_undegraded":"2023-09-23T20:31:38.116149+0000","last_fullsized":"2023-09-23T20:31:38.116149+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:24:14.704823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":309,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-09-23T20:31:38.139942+0000","last_change":"2023-09-23T20:29:46.070732+0000","last_active":"2023-09-23T20:31:38.139942+0000","last_peered":"2023-09-23T20:31:38.139942+0000","last_clean":"2023-09-23T20:31:38.139942+0000","last_became_active":"2023-09-23T20:29:46.069861+0000","last_became_peered":"2023-09-23T20:29:46.069861+0000","last_unstale":"2023-09-23T20:31:38.139942+0000","last_undegraded":"2023-09-23T20:31:38.139942+0000","last_fullsized":"2023-09-23T20:31:38.139942+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:33:00.813261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":305,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-09-23T20:31:38.116390+0000","last_change":"2023-09-23T20:29:46.242242+0000","last_active":"2023-09-23T20:31:38.116390+0000","last_peered":"2023-09-23T20:31:38.116390+0000","last_clean":"2023-09-23T20:31:38.116390+0000","last_became_active":"2023-09-23T20:29:46.241216+0000","last_became_peered":"2023-09-23T20:29:46.241216+0000","last_unstale":"2023-09-23T20:31:38.116390+0000","last_undegraded":"2023-09-23T20:31:38.116390+0000","last_fullsized":"2023-09-23T20:31:38.116390+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:12:48.530720+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":217,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-09-23T20:31:38.136289+0000","last_change":"2023-09-23T20:29:28.353700+0000","last_active":"2023-09-23T20:31:38.136289+0000","last_peered":"2023-09-23T20:31:38.136289+0000","last_clean":"2023-09-23T20:31:38.136289+0000","last_became_active":"2023-09-23T20:29:28.353357+0000","last_became_peered":"2023-09-23T20:29:28.353357+0000","last_unstale":"2023-09-23T20:31:38.136289+0000","last_undegraded":"2023-09-23T20:31:38.136289+0000","last_fullsized":"2023-09-23T20:31:38.136289+0000","mapping_epoch":258,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":259,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:12.077050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":183,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-09-23T20:31:38.140652+0000","last_change":"2023-09-23T20:29:04.138540+0000","last_active":"2023-09-23T20:31:38.140652+0000","last_peered":"2023-09-23T20:31:38.140652+0000","last_clean":"2023-09-23T20:31:38.140652+0000","last_became_active":"2023-09-23T20:29:04.138182+0000","last_became_peered":"2023-09-23T20:29:04.138182+0000","last_unstale":"2023-09-23T20:31:38.140652+0000","last_undegraded":"2023-09-23T20:31:38.140652+0000","last_fullsized":"2023-09-23T20:31:38.140652+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:26.098284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":158,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-09-23T20:31:38.136070+0000","last_change":"2023-09-23T20:25:39.622819+0000","last_active":"2023-09-23T20:31:38.136070+0000","last_peered":"2023-09-23T20:31:38.136070+0000","last_clean":"2023-09-23T20:31:38.136070+0000","last_became_active":"2023-09-23T20:25:39.622484+0000","last_became_peered":"2023-09-23T20:25:39.622484+0000","last_unstale":"2023-09-23T20:31:38.136070+0000","last_undegraded":"2023-09-23T20:31:38.136070+0000","last_fullsized":"2023-09-23T20:31:38.136070+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:51:53.973261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":282,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-09-23T20:31:38.107621+0000","last_change":"2023-09-23T20:25:39.668882+0000","last_active":"2023-09-23T20:31:38.107621+0000","last_peered":"2023-09-23T20:31:38.107621+0000","last_clean":"2023-09-23T20:31:38.107621+0000","last_became_active":"2023-09-23T20:25:39.637128+0000","last_became_peered":"2023-09-23T20:25:39.637128+0000","last_unstale":"2023-09-23T20:31:38.107621+0000","last_undegraded":"2023-09-23T20:31:38.107621+0000","last_fullsized":"2023-09-23T20:31:38.107621+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:56.406298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":317,"reported_epoch":301,"state":"active+clean","last_fresh":"2023-09-23T20:31:38.113029+0000","last_change":"2023-09-23T20:29:46.082412+0000","last_active":"2023-09-23T20:31:38.113029+0000","last_peered":"2023-09-23T20:31:38.113029+0000","last_clean":"2023-09-23T20:31:38.113029+0000","last_became_active":"2023-09-23T20:29:46.082028+0000","last_became_peered":"2023-09-23T20:29:46.082028+0000","last_unstale":"2023-09-23T20:31:38.113029+0000","last_undegraded":"2023-09-23T20:31:38.113029+0000","last_fullsized":"2023-09-23T20:31:38.113029+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:48:07.017969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":6376,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":122880,"data_stored":21392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26691,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":806935,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":89,"num_read_kb":167,"num_write":44,"num_write_kb":440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":322020,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2109440,"data_stored":2098784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3381,"ondisk_log_size":3381,"up":63,"acting":63,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":43,"num_bytes_recovered":4194342,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6082,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201527,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":552500,"kb_used_data":326636,"kb_used_omap":0,"kb_used_meta":225856,"kb_avail":93192652,"statfs":{"total":95995035648,"available":95429275648,"internally_reserved":0,"allocated":334475264,"data_stored":333506025,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":231276544},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":9,"apply_latency_ms":9,"commit_latency_ns":9000000,"apply_latency_ns":9000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:31:35 2023","interfaces":[{"interface":"back","average":{"1min":2.0049999999999999,"5min":1.988,"15min":1.2589999999999999},"min":{"1min":0.25700000000000001,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":8.0660000000000007,"5min":48.798999999999999,"15min":48.798999999999999},"last":0.38400000000000001},{"interface":"front","average":{"1min":2.0459999999999998,"5min":2.1230000000000002,"15min":1.4279999999999999},"min":{"1min":0.27300000000000002,"5min":0.251,"15min":0.24399999999999999},"max":{"1min":7.968,"5min":48.247,"15min":48.247},"last":0.34599999999999997}]},{"osd":1,"last update":"Sat Sep 23 20:30:47 2023","interfaces":[{"interface":"back","average":{"1min":1.762,"5min":1.762,"15min":1.762},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":9.7370000000000001,"5min":9.7370000000000001,"15min":9.7370000000000001},"last":0.499},{"interface":"front","average":{"1min":1.9810000000000001,"5min":1.9810000000000001,"15min":1.9810000000000001},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":5.7679999999999998,"5min":5.7679999999999998,"15min":5.7679999999999998},"last":0.47599999999999998}]},{"osd":2,"last update":"Sat Sep 23 20:31:38 2023","interfaces":[{"interface":"back","average":{"1min":1.478,"5min":1.669,"15min":1.282},"min":{"1min":0.29199999999999998,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":6.7220000000000004,"5min":11.372999999999999,"15min":11.372999999999999},"last":0.437},{"interface":"front","average":{"1min":2.7810000000000001,"5min":2.1899999999999999,"15min":1.4950000000000001},"min":{"1min":0.318,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":13.156000000000001,"5min":14.993,"15min":14.993},"last":0.41399999999999998}]},{"osd":3,"last update":"Sat Sep 23 20:31:02 2023","interfaces":[{"interface":"back","average":{"1min":2.5059999999999998,"5min":2.0710000000000002,"15min":1.377},"min":{"1min":0.34000000000000002,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":11.015000000000001,"5min":11.015000000000001,"15min":11.015000000000001},"last":0.45500000000000002},{"interface":"front","average":{"1min":2.7970000000000002,"5min":1.9510000000000001,"15min":1.3},"min":{"1min":0.38,"5min":0.33100000000000002,"15min":0.27600000000000002},"max":{"1min":11.031000000000001,"5min":11.031000000000001,"15min":11.031000000000001},"last":0.57099999999999995}]},{"osd":4,"last update":"Sat Sep 23 20:31:02 2023","interfaces":[{"interface":"back","average":{"1min":0.73699999999999999,"5min":1.583,"15min":1.008},"min":{"1min":0.29399999999999998,"5min":0.27100000000000002,"15min":0.27000000000000002},"max":{"1min":5.7220000000000004,"5min":15.177,"15min":15.177},"last":0.46700000000000003},{"interface":"front","average":{"1min":0.79900000000000004,"5min":1.3560000000000001,"15min":1.127},"min":{"1min":0.28399999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":4.2800000000000002,"5min":11.122,"15min":11.122},"last":0.51000000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:31:02 2023","interfaces":[{"interface":"back","average":{"1min":0.76200000000000001,"5min":1.4159999999999999,"15min":1.1040000000000001},"min":{"1min":0.30599999999999999,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":2.8730000000000002,"5min":13.241,"15min":13.241},"last":0.48799999999999999},{"interface":"front","average":{"1min":0.79400000000000004,"5min":1.7230000000000001,"15min":1.2070000000000001},"min":{"1min":0.32200000000000001,"5min":0.315,"15min":0.315},"max":{"1min":5.7869999999999999,"5min":18.831,"15min":18.831},"last":0.42499999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:31:02 2023","interfaces":[{"interface":"back","average":{"1min":0.82399999999999995,"5min":1.5580000000000001,"15min":1.1060000000000001},"min":{"1min":0.34399999999999997,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":5.7629999999999999,"5min":12.298999999999999,"15min":13.185},"last":0.52300000000000002},{"interface":"front","average":{"1min":0.93600000000000005,"5min":1.302,"15min":1.048},"min":{"1min":0.36799999999999999,"5min":0.35399999999999998,"15min":0.29899999999999999},"max":{"1min":4.4160000000000004,"5min":8.0069999999999997,"15min":9.9120000000000008},"last":0.54200000000000004}]}]},{"osd":6,"up_from":47,"seq":201863463163,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":249712,"kb_used_data":41640,"kb_used_omap":0,"kb_used_meta":208064,"kb_avail":93495440,"statfs":{"total":95995035648,"available":95739330560,"internally_reserved":0,"allocated":42639360,"data_stored":41674124,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":213057536},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":11,"apply_latency_ms":11,"commit_latency_ns":11000000,"apply_latency_ns":11000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:31:35 2023","interfaces":[{"interface":"back","average":{"1min":1.246,"5min":1.3169999999999999,"15min":1.0409999999999999},"min":{"1min":0.24099999999999999,"5min":0.23300000000000001,"15min":0.221},"max":{"1min":5.9420000000000002,"5min":8.4209999999999994,"15min":8.4209999999999994},"last":0.48299999999999998},{"interface":"front","average":{"1min":1.7090000000000001,"5min":1.8759999999999999,"15min":1.3080000000000001},"min":{"1min":0.26900000000000002,"5min":0.25,"15min":0.25},"max":{"1min":8.8729999999999993,"5min":8.8729999999999993,"15min":12.512},"last":0.39000000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:30:46 2023","interfaces":[{"interface":"back","average":{"1min":2.7509999999999999,"5min":2.7509999999999999,"15min":2.7509999999999999},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":14.1,"5min":14.1,"15min":14.1},"last":0.36499999999999999},{"interface":"front","average":{"1min":3.4870000000000001,"5min":3.4870000000000001,"15min":3.4870000000000001},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":14.266999999999999,"5min":14.266999999999999,"15min":14.266999999999999},"last":0.53500000000000003}]},{"osd":2,"last update":"Sat Sep 23 20:31:35 2023","interfaces":[{"interface":"back","average":{"1min":2.2160000000000002,"5min":2.2559999999999998,"15min":1.4119999999999999},"min":{"1min":0.33000000000000002,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":10.138999999999999,"5min":10.631,"15min":13.856},"last":0.42199999999999999},{"interface":"front","average":{"1min":2.4390000000000001,"5min":2.1299999999999999,"15min":1.341},"min":{"1min":0.41099999999999998,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":10.151999999999999,"5min":10.151999999999999,"15min":10.151999999999999},"last":0.55800000000000005}]},{"osd":3,"last update":"Sat Sep 23 20:31:11 2023","interfaces":[{"interface":"back","average":{"1min":2.9420000000000002,"5min":2.0099999999999998,"15min":1.3859999999999999},"min":{"1min":0.51600000000000001,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":6.1829999999999998,"5min":8.2989999999999995,"15min":9.4239999999999995},"last":0.56899999999999995},{"interface":"front","average":{"1min":3.0910000000000002,"5min":2.21,"15min":1.4330000000000001},"min":{"1min":0.40600000000000003,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":7.6609999999999996,"5min":8.3770000000000007,"15min":8.6050000000000004},"last":0.46600000000000003}]},{"osd":4,"last update":"Sat Sep 23 20:30:53 2023","interfaces":[{"interface":"back","average":{"1min":0.44,"5min":1.0349999999999999,"15min":0.85299999999999998},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":1.599,"5min":10.173,"15min":10.173},"last":0.58099999999999996},{"interface":"front","average":{"1min":1.032,"5min":1.3779999999999999,"15min":0.98199999999999998},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":3.363,"5min":8.3249999999999993,"15min":8.3249999999999993},"last":0.58999999999999997}]},{"osd":5,"last update":"Sat Sep 23 20:30:53 2023","interfaces":[{"interface":"back","average":{"1min":0.46400000000000002,"5min":0.91100000000000003,"15min":0.89800000000000002},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":1.2010000000000001,"5min":8.4499999999999993,"15min":14.085000000000001},"last":0.624},{"interface":"front","average":{"1min":0.98099999999999998,"5min":1.76,"15min":1.236},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.29499999999999998},"max":{"1min":3.294,"5min":18.192,"15min":18.192},"last":0.51200000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:31:11 2023","interfaces":[{"interface":"back","average":{"1min":0.45300000000000001,"5min":1.27,"15min":1.0309999999999999},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.754,"5min":8.5060000000000002,"15min":10.217000000000001},"last":0.60299999999999998},{"interface":"front","average":{"1min":1.0669999999999999,"5min":1.508,"15min":1.163},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":6.4160000000000004,"5min":8.407,"15min":9.7360000000000007},"last":0.54500000000000004}]}]},{"osd":1,"up_from":266,"seq":1142461300761,"num_pgs":58,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":570200,"kb_used_data":436240,"kb_used_omap":8,"kb_used_meta":133943,"kb_avail":93174952,"statfs":{"total":95995035648,"available":95411150848,"internally_reserved":0,"allocated":446709760,"data_stored":445066596,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8206,"internal_metadata":137158642},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:30:44 2023","interfaces":[{"interface":"back","average":{"1min":2.3300000000000001,"5min":2.3300000000000001,"15min":2.3300000000000001},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":9.5470000000000006,"5min":9.5470000000000006,"15min":9.5470000000000006},"last":0.27700000000000002},{"interface":"front","average":{"1min":3.1960000000000002,"5min":3.1960000000000002,"15min":3.1960000000000002},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":9.7129999999999992,"5min":9.7129999999999992,"15min":9.7129999999999992},"last":0.30199999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:30:44 2023","interfaces":[{"interface":"back","average":{"1min":5.125,"5min":5.125,"15min":5.125},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":16.151,"5min":16.151,"15min":16.151},"last":0.79400000000000004},{"interface":"front","average":{"1min":2.0939999999999999,"5min":2.0939999999999999,"15min":2.0939999999999999},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":9.4649999999999999,"5min":9.4649999999999999,"15min":9.4649999999999999},"last":0.41299999999999998}]},{"osd":3,"last update":"Sat Sep 23 20:30:44 2023","interfaces":[{"interface":"back","average":{"1min":3.867,"5min":3.867,"15min":3.867},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":9.5739999999999998,"5min":9.5739999999999998,"15min":9.5739999999999998},"last":0.443},{"interface":"front","average":{"1min":4.8460000000000001,"5min":4.8460000000000001,"15min":4.8460000000000001},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":12.012,"5min":12.012,"15min":12.012},"last":0.34799999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:30:44 2023","interfaces":[{"interface":"back","average":{"1min":2.0790000000000002,"5min":2.0790000000000002,"15min":2.0790000000000002},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":9.4429999999999996,"5min":9.4429999999999996,"15min":9.4429999999999996},"last":0.82999999999999996},{"interface":"front","average":{"1min":2.024,"5min":2.024,"15min":2.024},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":9.5039999999999996,"5min":9.5039999999999996,"15min":9.5039999999999996},"last":0.72299999999999998}]},{"osd":5,"last update":"Sat Sep 23 20:30:44 2023","interfaces":[{"interface":"back","average":{"1min":2.802,"5min":2.802,"15min":2.802},"min":{"1min":0.44,"5min":0.44,"15min":0.44},"max":{"1min":9.6479999999999997,"5min":9.6479999999999997,"15min":9.6479999999999997},"last":0.74399999999999999},{"interface":"front","average":{"1min":2.0899999999999999,"5min":2.0899999999999999,"15min":2.0899999999999999},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":9.4700000000000006,"5min":9.4700000000000006,"15min":9.4700000000000006},"last":0.84799999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:30:44 2023","interfaces":[{"interface":"back","average":{"1min":7.4489999999999998,"5min":7.4489999999999998,"15min":7.4489999999999998},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":72.682000000000002,"5min":72.682000000000002,"15min":72.682000000000002},"last":0.59899999999999998},{"interface":"front","average":{"1min":6.4400000000000004,"5min":6.4400000000000004,"15min":6.4400000000000004},"min":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":72.563999999999993,"5min":72.563999999999993,"15min":72.563999999999993},"last":0.86299999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:30:44 2023","interfaces":[{"interface":"back","average":{"1min":3.2829999999999999,"5min":3.2829999999999999,"15min":3.2829999999999999},"min":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"max":{"1min":9.6850000000000005,"5min":9.6850000000000005,"15min":9.6850000000000005},"last":0.58199999999999996},{"interface":"front","average":{"1min":2.3599999999999999,"5min":2.3599999999999999,"15min":2.3599999999999999},"min":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"max":{"1min":9.3949999999999996,"5min":9.3949999999999996,"15min":9.3949999999999996},"last":1.101}]}]},{"osd":0,"up_from":94,"seq":403726925964,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":351184,"kb_used_data":4040,"kb_used_omap":7,"kb_used_meta":347128,"kb_avail":93393968,"statfs":{"total":95995035648,"available":95635423232,"internally_reserved":0,"allocated":4136960,"data_stored":2619907,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":355459089},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:30:48 2023","interfaces":[{"interface":"back","average":{"1min":3.887,"5min":3.887,"15min":3.887},"min":{"1min":0.17999999999999999,"5min":0.17999999999999999,"15min":0.17999999999999999},"max":{"1min":15.542,"5min":15.542,"15min":15.542},"last":0.26400000000000001},{"interface":"front","average":{"1min":4.4569999999999999,"5min":4.4569999999999999,"15min":4.4569999999999999},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":10.49,"5min":10.49,"15min":10.49},"last":0.37}]},{"osd":2,"last update":"Sat Sep 23 20:31:30 2023","interfaces":[{"interface":"back","average":{"1min":2.8620000000000001,"5min":3.024,"15min":1.8129999999999999},"min":{"1min":0.25900000000000001,"5min":0.185,"15min":0.185},"max":{"1min":5.9850000000000003,"5min":77.448999999999998,"15min":77.448999999999998},"last":0.246},{"interface":"front","average":{"1min":1.6799999999999999,"5min":3.0049999999999999,"15min":1.5860000000000001},"min":{"1min":0.27200000000000002,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":9.1349999999999998,"5min":81.724999999999994,"15min":81.724999999999994},"last":0.22900000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:31:03 2023","interfaces":[{"interface":"back","average":{"1min":7.0750000000000002,"5min":3.6080000000000001,"15min":1.79},"min":{"1min":0.25,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":81.400999999999996,"5min":81.400999999999996,"15min":81.400999999999996},"last":0.29299999999999998},{"interface":"front","average":{"1min":9.3719999999999999,"5min":4.0679999999999996,"15min":1.7649999999999999},"min":{"1min":0.34699999999999998,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":81.137,"5min":81.137,"15min":81.137},"last":0.42399999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:31:30 2023","interfaces":[{"interface":"back","average":{"1min":1.6890000000000001,"5min":1.9039999999999999,"15min":1.482},"min":{"1min":0.30499999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":5.9160000000000004,"5min":9.9339999999999993,"15min":23.876999999999999},"last":0.51800000000000002},{"interface":"front","average":{"1min":1.3420000000000001,"5min":1.661,"15min":1.2869999999999999},"min":{"1min":0.442,"5min":0.312,"15min":0.29299999999999998},"max":{"1min":4.4029999999999996,"5min":10.414,"15min":23.681999999999999},"last":0.53400000000000003}]},{"osd":5,"last update":"Sat Sep 23 20:31:30 2023","interfaces":[{"interface":"back","average":{"1min":1.4610000000000001,"5min":2.0840000000000001,"15min":1.657},"min":{"1min":0.438,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":5.0780000000000003,"5min":14.289,"15min":24.198},"last":0.504},{"interface":"front","average":{"1min":1.3029999999999999,"5min":2.044,"15min":1.492},"min":{"1min":0.41799999999999998,"5min":0.314,"15min":0.314},"max":{"1min":5.29,"5min":14.938000000000001,"15min":14.938000000000001},"last":0.34599999999999997}]},{"osd":6,"last update":"Sat Sep 23 20:31:30 2023","interfaces":[{"interface":"back","average":{"1min":1.47,"5min":1.897,"15min":1.3740000000000001},"min":{"1min":0.52400000000000002,"5min":0.318,"15min":0.318},"max":{"1min":4.5620000000000003,"5min":10.441000000000001,"15min":24.408000000000001},"last":0.46400000000000002},{"interface":"front","average":{"1min":1.6180000000000001,"5min":1.79,"15min":1.3260000000000001},"min":{"1min":0.40899999999999997,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":6.109,"5min":9.6769999999999996,"15min":25.271000000000001},"last":0.41299999999999998}]},{"osd":7,"last update":"Sat Sep 23 20:31:30 2023","interfaces":[{"interface":"back","average":{"1min":1.954,"5min":2.1110000000000002,"15min":1.5029999999999999},"min":{"1min":0.52500000000000002,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":9.7309999999999999,"5min":24.614000000000001,"15min":25.359000000000002},"last":0.44400000000000001},{"interface":"front","average":{"1min":2.2829999999999999,"5min":2.5169999999999999,"15min":1.788},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":5.9550000000000001,"5min":24.669,"15min":24.669},"last":0.47999999999999998}]}]},{"osd":2,"up_from":104,"seq":446676598912,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":736832,"kb_used_data":348472,"kb_used_omap":11,"kb_used_meta":388340,"kb_avail":93008320,"statfs":{"total":95995035648,"available":95240519680,"internally_reserved":0,"allocated":356835328,"data_stored":355333941,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11878,"internal_metadata":397660570},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:31:39 2023","interfaces":[{"interface":"back","average":{"1min":2.3359999999999999,"5min":2.1560000000000001,"15min":1.413},"min":{"1min":0.186,"5min":0.17999999999999999,"15min":0.17999999999999999},"max":{"1min":18.876000000000001,"5min":30.765999999999998,"15min":30.765999999999998},"last":0.34899999999999998},{"interface":"front","average":{"1min":2.7130000000000001,"5min":2.4420000000000002,"15min":1.369},"min":{"1min":0.29099999999999998,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":18.684999999999999,"5min":18.684999999999999,"15min":18.684999999999999},"last":0.32600000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:30:45 2023","interfaces":[{"interface":"back","average":{"1min":4.851,"5min":4.851,"15min":4.851},"min":{"1min":0.221,"5min":0.221,"15min":0.221},"max":{"1min":16.280000000000001,"5min":16.280000000000001,"15min":16.280000000000001},"last":0.27700000000000002},{"interface":"front","average":{"1min":2.988,"5min":2.988,"15min":2.988},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":30.565000000000001,"5min":30.565000000000001,"15min":30.565000000000001},"last":0.39000000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:31:02 2023","interfaces":[{"interface":"back","average":{"1min":2.8849999999999998,"5min":2.2959999999999998,"15min":1.21},"min":{"1min":0.27800000000000002,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":11.132,"5min":16.331,"15min":16.331},"last":0.34000000000000002},{"interface":"front","average":{"1min":4.2539999999999996,"5min":2.8109999999999999,"15min":1.46},"min":{"1min":0.27300000000000002,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":30.704999999999998,"5min":30.704999999999998,"15min":30.704999999999998},"last":0.46899999999999997}]},{"osd":4,"last update":"Sat Sep 23 20:31:39 2023","interfaces":[{"interface":"back","average":{"1min":1.835,"5min":2.1349999999999998,"15min":1.3169999999999999},"min":{"1min":0.34999999999999998,"5min":0.32800000000000001,"15min":0.28100000000000003},"max":{"1min":5.0860000000000003,"5min":16.797999999999998,"15min":16.797999999999998},"last":0.60299999999999998},{"interface":"front","average":{"1min":1.5640000000000001,"5min":1.998,"15min":1.327},"min":{"1min":0.35399999999999998,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":7.4139999999999997,"5min":14.971,"15min":14.971},"last":0.54600000000000004}]},{"osd":5,"last update":"Sat Sep 23 20:31:39 2023","interfaces":[{"interface":"back","average":{"1min":1.7549999999999999,"5min":3.2970000000000002,"15min":1.8440000000000001},"min":{"1min":0.314,"5min":0.314,"15min":0.309},"max":{"1min":4.992,"5min":95.393000000000001,"15min":95.393000000000001},"last":0.44700000000000001},{"interface":"front","average":{"1min":1.9419999999999999,"5min":3.3170000000000002,"15min":1.8859999999999999},"min":{"1min":0.38,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":8.7509999999999994,"5min":95.414000000000001,"15min":95.414000000000001},"last":0.58899999999999997}]},{"osd":6,"last update":"Sat Sep 23 20:31:39 2023","interfaces":[{"interface":"back","average":{"1min":2.206,"5min":2.3279999999999998,"15min":1.4259999999999999},"min":{"1min":0.35799999999999998,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":13.154999999999999,"5min":15.023999999999999,"15min":15.023999999999999},"last":0.56899999999999995},{"interface":"front","average":{"1min":1.3779999999999999,"5min":2.1000000000000001,"15min":1.452},"min":{"1min":0.40200000000000002,"5min":0.316,"15min":0.316},"max":{"1min":4.7249999999999996,"5min":30.670999999999999,"15min":30.670999999999999},"last":0.65200000000000002}]},{"osd":7,"last update":"Sat Sep 23 20:31:39 2023","interfaces":[{"interface":"back","average":{"1min":1.2210000000000001,"5min":2.1739999999999999,"15min":1.5109999999999999},"min":{"1min":0.44600000000000001,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":4.8300000000000001,"5min":30.741,"15min":30.741},"last":0.68899999999999995},{"interface":"front","average":{"1min":2.0670000000000002,"5min":2.4079999999999999,"15min":1.5489999999999999},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":10.861000000000001,"5min":16.719000000000001,"15min":16.719000000000001},"last":0.58199999999999996}]}]},{"osd":3,"up_from":109,"seq":468151435385,"num_pgs":64,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":417792,"kb_used_data":4280,"kb_used_omap":4,"kb_used_meta":413499,"kb_avail":93327360,"statfs":{"total":95995035648,"available":95567216640,"internally_reserved":0,"allocated":4382720,"data_stored":2877136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4514,"internal_metadata":423423582},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":28,"apply_latency_ms":28,"commit_latency_ns":28000000,"apply_latency_ns":28000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:31:05 2023","interfaces":[{"interface":"back","average":{"1min":2.5600000000000001,"5min":2.2909999999999999,"15min":1.4610000000000001},"min":{"1min":0.22500000000000001,"5min":0.186,"15min":0.186},"max":{"1min":6.71,"5min":15.589,"15min":67.305999999999997},"last":0.38100000000000001},{"interface":"front","average":{"1min":2.839,"5min":1.978,"15min":1.3919999999999999},"min":{"1min":0.19500000000000001,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":12.781000000000001,"5min":13.743,"15min":67.403999999999996},"last":0.44800000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:30:47 2023","interfaces":[{"interface":"back","average":{"1min":3.0880000000000001,"5min":3.0880000000000001,"15min":3.0880000000000001},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":8.9199999999999999,"5min":8.9199999999999999,"15min":8.9199999999999999},"last":0.23999999999999999},{"interface":"front","average":{"1min":3.2530000000000001,"5min":3.2530000000000001,"15min":3.2530000000000001},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":10.967000000000001,"5min":10.967000000000001,"15min":10.967000000000001},"last":0.745}]},{"osd":2,"last update":"Sat Sep 23 20:31:05 2023","interfaces":[{"interface":"back","average":{"1min":3.1419999999999999,"5min":2.6739999999999999,"15min":1.3959999999999999},"min":{"1min":0.29799999999999999,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":11.253,"5min":24.045000000000002,"15min":24.045000000000002},"last":0.42799999999999999},{"interface":"front","average":{"1min":3.1709999999999998,"5min":2.649,"15min":1.4430000000000001},"min":{"1min":0.27100000000000002,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":19.603000000000002,"5min":19.603000000000002,"15min":19.603000000000002},"last":4.5069999999999997}]},{"osd":4,"last update":"Sat Sep 23 20:31:05 2023","interfaces":[{"interface":"back","average":{"1min":2.331,"5min":2.46,"15min":1.3560000000000001},"min":{"1min":0.34200000000000003,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":6.5670000000000002,"5min":46.042999999999999,"15min":46.042999999999999},"last":0.76500000000000001},{"interface":"front","average":{"1min":3.7120000000000002,"5min":2.9569999999999999,"15min":1.591},"min":{"1min":0.34599999999999997,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":19.635000000000002,"5min":46.579999999999998,"15min":46.579999999999998},"last":3.2440000000000002}]},{"osd":5,"last update":"Sat Sep 23 20:31:05 2023","interfaces":[{"interface":"back","average":{"1min":2.319,"5min":2.5960000000000001,"15min":1.5189999999999999},"min":{"1min":0.45700000000000002,"5min":0.31900000000000001,"15min":0.311},"max":{"1min":6.2370000000000001,"5min":45.292000000000002,"15min":45.292000000000002},"last":0.71399999999999997},{"interface":"front","average":{"1min":2.6139999999999999,"5min":2.5129999999999999,"15min":1.3999999999999999},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.313},"max":{"1min":11.028,"5min":44.679000000000002,"15min":44.679000000000002},"last":0.85699999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:31:05 2023","interfaces":[{"interface":"back","average":{"1min":3.3519999999999999,"5min":2.3410000000000002,"15min":1.4279999999999999},"min":{"1min":0.38500000000000001,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":17.524999999999999,"5min":17.524999999999999,"15min":17.524999999999999},"last":0.82699999999999996},{"interface":"front","average":{"1min":3.0670000000000002,"5min":2.3300000000000001,"15min":1.387},"min":{"1min":0.439,"5min":0.36399999999999999,"15min":0.35499999999999998},"max":{"1min":19.524000000000001,"5min":19.524000000000001,"15min":19.524000000000001},"last":3.298}]},{"osd":7,"last update":"Sat Sep 23 20:31:05 2023","interfaces":[{"interface":"back","average":{"1min":4.1749999999999998,"5min":2.8370000000000002,"15min":1.5549999999999999},"min":{"1min":0.40699999999999997,"5min":0.374,"15min":0.34999999999999998},"max":{"1min":19.571999999999999,"5min":19.571999999999999,"15min":19.571999999999999},"last":3.1469999999999998},{"interface":"front","average":{"1min":2.3100000000000001,"5min":2.762,"15min":1.607},"min":{"1min":0.42999999999999999,"5min":0.371,"15min":0.371},"max":{"1min":6.407,"5min":45.387999999999998,"15min":45.387999999999998},"last":0.66200000000000003}]}]},{"osd":4,"up_from":0,"seq":128849019139,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":5,"up_from":39,"seq":167503724800,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":230632,"kb_used_data":3360,"kb_used_omap":0,"kb_used_meta":227264,"kb_avail":93514520,"statfs":{"total":95995035648,"available":95758868480,"internally_reserved":0,"allocated":3440640,"data_stored":2479926,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":232718336},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:31:32 2023","interfaces":[{"interface":"back","average":{"1min":1.3600000000000001,"5min":1.5580000000000001,"15min":1.232},"min":{"1min":0.23799999999999999,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":5.782,"5min":8.1609999999999996,"15min":32.475000000000001},"last":0.54400000000000004},{"interface":"front","average":{"1min":0.93899999999999995,"5min":1.4750000000000001,"15min":1.2250000000000001},"min":{"1min":0.34000000000000002,"5min":0.248,"15min":0.20599999999999999},"max":{"1min":5.1900000000000004,"5min":8.4900000000000002,"15min":32.457000000000001},"last":0.42199999999999999}]},{"osd":1,"last update":"Sat Sep 23 20:30:45 2023","interfaces":[{"interface":"back","average":{"1min":2.0550000000000002,"5min":2.0550000000000002,"15min":2.0550000000000002},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":8.5839999999999996,"5min":8.5839999999999996,"15min":8.5839999999999996},"last":0.59299999999999997},{"interface":"front","average":{"1min":1.5609999999999999,"5min":1.5609999999999999,"15min":1.5609999999999999},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":6.7969999999999997,"5min":6.7969999999999997,"15min":6.7969999999999997},"last":0.46600000000000003}]},{"osd":2,"last update":"Sat Sep 23 20:31:32 2023","interfaces":[{"interface":"back","average":{"1min":2.2360000000000002,"5min":1.8700000000000001,"15min":1.4370000000000001},"min":{"1min":0.29699999999999999,"5min":0.27500000000000002,"15min":0.26700000000000002},"max":{"1min":8.3439999999999994,"5min":18.472999999999999,"15min":34.514000000000003},"last":0.443},{"interface":"front","average":{"1min":2.4590000000000001,"5min":1.8580000000000001,"15min":1.383},"min":{"1min":0.28799999999999998,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":8.3230000000000004,"5min":8.6820000000000004,"15min":32.515000000000001},"last":0.45400000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:31:05 2023","interfaces":[{"interface":"back","average":{"1min":3.1230000000000002,"5min":2.2109999999999999,"15min":1.3220000000000001},"min":{"1min":0.61799999999999999,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":10.952999999999999,"5min":12.846,"15min":12.846},"last":0.501},{"interface":"front","average":{"1min":2.3479999999999999,"5min":1.754,"15min":1.2310000000000001},"min":{"1min":0.32300000000000001,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":7.2060000000000004,"5min":12.005000000000001,"15min":12.005000000000001},"last":0.58199999999999996}]},{"osd":4,"last update":"Sat Sep 23 20:31:26 2023","interfaces":[{"interface":"back","average":{"1min":1.1899999999999999,"5min":1.919,"15min":1.276},"min":{"1min":0.27800000000000002,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":6.282,"5min":28.015999999999998,"15min":28.015999999999998},"last":0.55200000000000005},{"interface":"front","average":{"1min":0.81799999999999995,"5min":1.2749999999999999,"15min":1.04},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.29099999999999998},"max":{"1min":2.427,"5min":12.300000000000001,"15min":12.300000000000001},"last":0.40899999999999997}]},{"osd":6,"last update":"Sat Sep 23 20:31:41 2023","interfaces":[{"interface":"back","average":{"1min":0.877,"5min":1.4450000000000001,"15min":1.177},"min":{"1min":0.33600000000000002,"5min":0.30599999999999999,"15min":0.30499999999999999},"max":{"1min":4.3330000000000002,"5min":12.890000000000001,"15min":12.890000000000001},"last":0.63200000000000001},{"interface":"front","average":{"1min":0.84699999999999998,"5min":1.371,"15min":1.1579999999999999},"min":{"1min":0.39100000000000001,"5min":0.36399999999999999,"15min":0.34499999999999997},"max":{"1min":1.855,"5min":16.265999999999998,"15min":16.265999999999998},"last":0.56299999999999994}]},{"osd":7,"last update":"Sat Sep 23 20:31:05 2023","interfaces":[{"interface":"back","average":{"1min":1.01,"5min":1.8169999999999999,"15min":1.204},"min":{"1min":0.32400000000000001,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":2.77,"5min":36.621000000000002,"15min":36.621000000000002},"last":0.52000000000000002},{"interface":"front","average":{"1min":0.94299999999999995,"5min":1.6759999999999999,"15min":1.103},"min":{"1min":0.33500000000000002,"5min":0.316,"15min":0.315},"max":{"1min":3.6219999999999999,"5min":37.331000000000003,"15min":37.331000000000003},"last":0.48699999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4722,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1360,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8206,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7156,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3154,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:31:48.873 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:31:48.885 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:31:48.912 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:48.945 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:31:48.948 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:48.948 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:31:49.049 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:31:49.077 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:49.163 INFO:teuthology.orchestra.run.smithi018.stderr:nodeep-scrub is unset 2023-09-23T20:31:49.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:49 smithi149 ceph-mon[170394]: 1.14 scrub starts 2023-09-23T20:31:49.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:49 smithi149 ceph-mon[170394]: 1.14 scrub ok 2023-09-23T20:31:49.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:49 smithi149 ceph-mon[170394]: pgmap v689: 93 pgs: 1 active+clean+scrubbing, 92 active+clean; 4.8 MiB data, 3.0 GiB used, 623 GiB / 626 GiB avail 2023-09-23T20:31:49.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:49 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/502817565' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2023-09-23T20:31:49.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:49 smithi149 ceph-mon[170394]: osdmap e303: 8 total, 8 up, 7 in 2023-09-23T20:31:49.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:49 smithi149 ceph-mon[170394]: failed to encode map e303 with expected crc 2023-09-23T20:31:49.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:49 smithi149 ceph-mon[170394]: failed to encode map e303 with expected crc 2023-09-23T20:31:49.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:49 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3936898490' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:31:49.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:49 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:31:49.264 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:31:49.274 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:49.274 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:31:49.291 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd pool application enable unique_pool_2 rados --yes-i-really-mean-it || true 2023-09-23T20:31:49.359 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:49 smithi018 ceph-mon[184144]: 1.14 scrub starts 2023-09-23T20:31:49.359 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:49 smithi018 ceph-mon[184144]: 1.14 scrub ok 2023-09-23T20:31:49.359 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:49 smithi018 ceph-mon[184144]: pgmap v689: 93 pgs: 1 active+clean+scrubbing, 92 active+clean; 4.8 MiB data, 3.0 GiB used, 623 GiB / 626 GiB avail 2023-09-23T20:31:49.359 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:49 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/502817565' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2023-09-23T20:31:49.359 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:49 smithi018 ceph-mon[184144]: osdmap e303: 8 total, 8 up, 7 in 2023-09-23T20:31:49.359 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:49 smithi018 ceph-mon[184144]: failed to encode map e303 with expected crc 2023-09-23T20:31:49.359 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:49 smithi018 ceph-mon[184144]: failed to encode map e303 with expected crc 2023-09-23T20:31:49.360 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:49 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3936898490' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:31:49.360 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:49 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:31:49.360 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:49 smithi018 ceph-mon[179647]: 1.14 scrub starts 2023-09-23T20:31:49.360 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:49 smithi018 ceph-mon[179647]: 1.14 scrub ok 2023-09-23T20:31:49.360 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:49 smithi018 ceph-mon[179647]: pgmap v689: 93 pgs: 1 active+clean+scrubbing, 92 active+clean; 4.8 MiB data, 3.0 GiB used, 623 GiB / 626 GiB avail 2023-09-23T20:31:49.360 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:49 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/502817565' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2023-09-23T20:31:49.360 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:49 smithi018 ceph-mon[179647]: osdmap e303: 8 total, 8 up, 7 in 2023-09-23T20:31:49.360 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:49 smithi018 ceph-mon[179647]: failed to encode map e303 with expected crc 2023-09-23T20:31:49.360 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:49 smithi018 ceph-mon[179647]: failed to encode map e303 with expected crc 2023-09-23T20:31:49.360 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:49 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3936898490' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:31:49.361 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:49 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:31:49.375 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:31:49.402 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:49.435 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:31:49.438 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:49.439 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:31:49.540 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:31:49.566 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:49.599 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:31:49.602 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:49.602 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:31:49.703 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:31:49.730 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:49.761 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:31:49.764 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:49.765 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:31:49.865 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:31:49.895 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:50.029 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:31:50.032 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:50.032 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:31:50.042 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:31:50.133 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:31:50.153 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:50 smithi149 ceph-mon[170394]: from='client.45527 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:31:50.153 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:50 smithi149 ceph-mon[170394]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:31:50.154 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:50 smithi149 ceph-mon[170394]: Cluster is now healthy 2023-09-23T20:31:50.154 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:50 smithi149 ceph-mon[170394]: failed to encode map e304 with expected crc 2023-09-23T20:31:50.154 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:50 smithi149 ceph-mon[170394]: failed to encode map e304 with expected crc 2023-09-23T20:31:50.161 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:50.194 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:31:50.196 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:50.197 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:31:50.298 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:31:50.328 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:50.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:50 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:31:50.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:50 smithi149 ceph-mon[170394]: failed to encode map e304 with expected crc 2023-09-23T20:31:50.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:50 smithi149 ceph-mon[170394]: failed to encode map e304 with expected crc 2023-09-23T20:31:50.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:50 smithi149 ceph-mon[170394]: osdmap e304: 8 total, 8 up, 7 in 2023-09-23T20:31:50.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:50 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:31:50.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:50 smithi149 ceph-mon[170394]: 1.10 scrub starts 2023-09-23T20:31:50.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:50 smithi149 ceph-mon[170394]: 1.10 scrub ok 2023-09-23T20:31:50.536 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:50 smithi018 ceph-mon[184144]: from='client.45527 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:31:50.537 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:50 smithi018 ceph-mon[184144]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:31:50.537 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:50 smithi018 ceph-mon[184144]: Cluster is now healthy 2023-09-23T20:31:50.537 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:50 smithi018 ceph-mon[184144]: failed to encode map e304 with expected crc 2023-09-23T20:31:50.537 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:50 smithi018 ceph-mon[184144]: failed to encode map e304 with expected crc 2023-09-23T20:31:50.537 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:50 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:31:50.537 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:50 smithi018 ceph-mon[184144]: failed to encode map e304 with expected crc 2023-09-23T20:31:50.537 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:50 smithi018 ceph-mon[184144]: failed to encode map e304 with expected crc 2023-09-23T20:31:50.537 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:50 smithi018 ceph-mon[184144]: osdmap e304: 8 total, 8 up, 7 in 2023-09-23T20:31:50.537 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:50 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:31:50.537 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:50 smithi018 ceph-mon[184144]: 1.10 scrub starts 2023-09-23T20:31:50.538 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:50 smithi018 ceph-mon[184144]: 1.10 scrub ok 2023-09-23T20:31:50.538 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:50 smithi018 ceph-mon[179647]: from='client.45527 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:31:50.538 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:50 smithi018 ceph-mon[179647]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:31:50.538 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:50 smithi018 ceph-mon[179647]: Cluster is now healthy 2023-09-23T20:31:50.538 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:50 smithi018 ceph-mon[179647]: failed to encode map e304 with expected crc 2023-09-23T20:31:50.538 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:50 smithi018 ceph-mon[179647]: failed to encode map e304 with expected crc 2023-09-23T20:31:50.538 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:50 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:31:50.538 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:50 smithi018 ceph-mon[179647]: failed to encode map e304 with expected crc 2023-09-23T20:31:50.538 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:50 smithi018 ceph-mon[179647]: failed to encode map e304 with expected crc 2023-09-23T20:31:50.539 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:50 smithi018 ceph-mon[179647]: osdmap e304: 8 total, 8 up, 7 in 2023-09-23T20:31:50.539 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:50 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:31:50.539 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:50 smithi018 ceph-mon[179647]: 1.10 scrub starts 2023-09-23T20:31:50.539 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:50 smithi018 ceph-mon[179647]: 1.10 scrub ok 2023-09-23T20:31:50.549 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:31:50.551 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:50.552 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:31:50.593 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:31:50.653 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:31:50.701 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:50.938 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:31:50.942 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:50.942 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:31:51.043 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:31:51.073 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:51.108 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:31:51.111 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:51.112 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:31:51.212 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:31:51.238 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:51 smithi018 ceph-mon[184144]: failed to encode map e304 with expected crc 2023-09-23T20:31:51.238 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:51 smithi018 ceph-mon[184144]: pgmap v692: 109 pgs: 5 creating+peering, 11 unknown, 93 active+clean; 4.8 MiB data, 2.9 GiB used, 623 GiB / 626 GiB avail 2023-09-23T20:31:51.238 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:51 smithi018 ceph-mon[184144]: failed to encode map e305 with expected crc 2023-09-23T20:31:51.238 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:51 smithi018 ceph-mon[184144]: osdmap e305: 8 total, 8 up, 7 in 2023-09-23T20:31:51.238 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:51 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:31:51.238 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:51 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:31:51.239 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:51 smithi018 ceph-mon[179647]: failed to encode map e304 with expected crc 2023-09-23T20:31:51.239 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:51 smithi018 ceph-mon[179647]: pgmap v692: 109 pgs: 5 creating+peering, 11 unknown, 93 active+clean; 4.8 MiB data, 2.9 GiB used, 623 GiB / 626 GiB avail 2023-09-23T20:31:51.239 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:51 smithi018 ceph-mon[179647]: failed to encode map e305 with expected crc 2023-09-23T20:31:51.239 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:51 smithi018 ceph-mon[179647]: osdmap e305: 8 total, 8 up, 7 in 2023-09-23T20:31:51.239 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:51 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:31:51.239 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:51 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:31:51.241 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:51.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:51 smithi149 ceph-mon[170394]: failed to encode map e304 with expected crc 2023-09-23T20:31:51.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:51 smithi149 ceph-mon[170394]: pgmap v692: 109 pgs: 5 creating+peering, 11 unknown, 93 active+clean; 4.8 MiB data, 2.9 GiB used, 623 GiB / 626 GiB avail 2023-09-23T20:31:51.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:51 smithi149 ceph-mon[170394]: failed to encode map e305 with expected crc 2023-09-23T20:31:51.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:51 smithi149 ceph-mon[170394]: osdmap e305: 8 total, 8 up, 7 in 2023-09-23T20:31:51.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:51 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:31:51.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:51 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:31:51.821 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:31:51.884 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T20:31:52.002 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:31:52.005 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:52.006 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:31:52.107 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:31:52.139 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:52.174 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:31:52.176 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:52.177 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:31:52.278 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:31:52.307 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:52.826 INFO:teuthology.orchestra.run.smithi018.stdout:ERROR: (22) Invalid argument 2023-09-23T20:31:52.826 INFO:teuthology.orchestra.run.smithi018.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:31:53.360 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:53 smithi018 ceph-mon[179647]: pgmap v694: 109 pgs: 5 creating+peering, 2 unknown, 102 active+clean; 4.7 MiB data, 2.9 GiB used, 623 GiB / 626 GiB avail 2023-09-23T20:31:53.360 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:53 smithi018 ceph-mon[184144]: pgmap v694: 109 pgs: 5 creating+peering, 2 unknown, 102 active+clean; 4.7 MiB data, 2.9 GiB used, 623 GiB / 626 GiB avail 2023-09-23T20:31:53.405 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:31:53.431 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:31:53.450 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:31:53.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:53 smithi149 ceph-mon[170394]: pgmap v694: 109 pgs: 5 creating+peering, 2 unknown, 102 active+clean; 4.7 MiB data, 2.9 GiB used, 623 GiB / 626 GiB avail 2023-09-23T20:31:53.469 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:31:53.488 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:31:53.507 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:31:53.529 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:31:53.549 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:31:53.550 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:31:53.550 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:31:53.551 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:31:53.551 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:31:53.551 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:31:53.551 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:31:53.551 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:31:53.551 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:31:53.553 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:31:53.558 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:53.558 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:31:53.569 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T20:31:53.660 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:31:53.668 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:31:53.689 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:53.725 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:31:53.727 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:53.728 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:31:53.828 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:31:53.856 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:53.889 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:31:53.892 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:53.892 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:31:53.993 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:31:54.020 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:54.052 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:31:54.055 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:54.055 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:31:54.097 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:31:54.156 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:31:54.185 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:54.225 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:31:54.228 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:54.229 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:31:54.329 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:31:54.358 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:54.398 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:31:54.402 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:54.402 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:31:54.463 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:54 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:31:54.463 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:54 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:31:54.463 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:54 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:31:54.463 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:54 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1850350889' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:31:54.464 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:54 smithi018 ceph-mon[184144]: 1.12 scrub starts 2023-09-23T20:31:54.464 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:54 smithi018 ceph-mon[184144]: 1.12 scrub ok 2023-09-23T20:31:54.464 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:54 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:31:54.464 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:54 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:31:54.464 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:54 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:31:54.464 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:54 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1850350889' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:31:54.464 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:54 smithi018 ceph-mon[179647]: 1.12 scrub starts 2023-09-23T20:31:54.464 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:54 smithi018 ceph-mon[179647]: 1.12 scrub ok 2023-09-23T20:31:54.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:54 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:31:54.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:54 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:31:54.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:54 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:31:54.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:54 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1850350889' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:31:54.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:54 smithi149 ceph-mon[170394]: 1.12 scrub starts 2023-09-23T20:31:54.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:54 smithi149 ceph-mon[170394]: 1.12 scrub ok 2023-09-23T20:31:54.503 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:31:54.537 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:54.631 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:31:54.635 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:54.635 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:31:54.736 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:31:54.766 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:55.141 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:31:54 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:31:54] "GET /metrics HTTP/1.1" 200 34167 "" "Prometheus/2.43.0" 2023-09-23T20:31:55.176 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:31:55.179 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:55.180 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:31:55.281 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:31:55.284 INFO:teuthology.orchestra.run.smithi018.stderr:enabled application 'rados' on pool 'unique_pool_2' 2023-09-23T20:31:55.315 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:31:55.316 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:55.434 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:55 smithi018 ceph-mon[184144]: pgmap v695: 109 pgs: 1 creating+peering, 108 active+clean; 4.6 MiB data, 2.9 GiB used, 623 GiB / 626 GiB avail 2023-09-23T20:31:55.434 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:55 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1310196390' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T20:31:55.434 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:55 smithi018 ceph-mon[179647]: pgmap v695: 109 pgs: 1 creating+peering, 108 active+clean; 4.6 MiB data, 2.9 GiB used, 623 GiB / 626 GiB avail 2023-09-23T20:31:55.434 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:55 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1310196390' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T20:31:55.656 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:31:55.659 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:55.659 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:31:55.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:55 smithi149 ceph-mon[170394]: pgmap v695: 109 pgs: 1 creating+peering, 108 active+clean; 4.6 MiB data, 2.9 GiB used, 623 GiB / 626 GiB avail 2023-09-23T20:31:55.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:55 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1310196390' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T20:31:55.760 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:31:55.790 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:56.004 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:31:56.069 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:31:56.073 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:56.074 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:31:56.168 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:31:56.170 INFO:teuthology.orchestra.run.smithi018.stderr:dumped all 2023-09-23T20:31:56.175 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:31:56.206 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:56.251 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:31:56.255 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:56.255 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:31:56.356 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:31:56.387 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:56.422 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:31:56.425 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:56.425 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:31:56.460 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:56 smithi018 ceph-mon[179647]: failed to encode map e306 with expected crc 2023-09-23T20:31:56.461 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:56 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1310196390' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T20:31:56.461 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:56 smithi018 ceph-mon[179647]: osdmap e306: 8 total, 8 up, 7 in 2023-09-23T20:31:56.461 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:56 smithi018 ceph-mon[184144]: failed to encode map e306 with expected crc 2023-09-23T20:31:56.461 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:56 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1310196390' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T20:31:56.461 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:56 smithi018 ceph-mon[184144]: osdmap e306: 8 total, 8 up, 7 in 2023-09-23T20:31:56.526 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:31:56.551 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:56 smithi149 ceph-mon[170394]: failed to encode map e306 with expected crc 2023-09-23T20:31:56.551 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:56 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1310196390' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T20:31:56.551 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:56 smithi149 ceph-mon[170394]: osdmap e306: 8 total, 8 up, 7 in 2023-09-23T20:31:56.553 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:56.587 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:31:56.589 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:56.590 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:31:56.672 DEBUG:teuthology.orchestra.run.smithi018:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_2 bench 90 write' 2023-09-23T20:31:56.674 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-09-23T20:31:56.674 INFO:teuthology.orchestra.run:waiting for 3000 2023-09-23T20:31:56.690 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:31:56.717 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:56.749 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:31:56.752 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:56.752 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:31:56.761 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:hints = 1 2023-09-23T20:31:56.761 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-09-23T20:31:56.761 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Object prefix: benchmark_data_smithi018_308601 2023-09-23T20:31:56.762 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T20:31:56.762 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 0 0 0 0 0 0 - 0 2023-09-23T20:31:56.853 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:31:56.921 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:56.983 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:31:56.992 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:56.992 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:31:57.093 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:31:57.139 INFO:teuthology.orchestra.run.smithi018.stdout:{"pg_ready":true,"pg_map":{"version":697,"stamp":"2023-09-23T20:31:55.729940+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790195,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":907,"num_write_kb":12725,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":83,"num_bytes_recovered":4200718,"num_keys_recovered":30,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4412,"ondisk_log_size":4412,"up":327,"acting":327,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":331,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":2969120,"kb_used_data":1024680,"kb_used_omap":32,"kb_used_meta":1944351,"kb_avail":653246944,"statfs":{"total":671965249536,"available":668924870656,"internally_reserved":0,"allocated":1049272320,"data_stored":1040188485,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32773,"internal_metadata":1991016443},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-216567,"num_objects":-1,"num_object_clones":0,"num_object_copies":-3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-1,"num_whiteouts":0,"num_read":-89,"num_read_kb":-167,"num_write":-44,"num_write_kb":-440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-2,"num_bytes_recovered":-322020,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.588700"},"pg_stats":[{"pgid":"3.1a","version":"69'1","reported_seq":50,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T20:31:49.145223+0000","last_change":"2023-09-23T20:29:46.050662+0000","last_active":"2023-09-23T20:31:49.145223+0000","last_peered":"2023-09-23T20:31:49.145223+0000","last_clean":"2023-09-23T20:31:49.145223+0000","last_became_active":"2023-09-23T20:29:46.050281+0000","last_became_peered":"2023-09-23T20:29:46.050281+0000","last_unstale":"2023-09-23T20:31:49.145223+0000","last_undegraded":"2023-09-23T20:31:49.145223+0000","last_fullsized":"2023-09-23T20:31:49.145223+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:55.227008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":178,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T20:31:49.158167+0000","last_change":"2023-09-23T20:29:46.067085+0000","last_active":"2023-09-23T20:31:49.158167+0000","last_peered":"2023-09-23T20:31:49.158167+0000","last_clean":"2023-09-23T20:31:49.158167+0000","last_became_active":"2023-09-23T20:29:46.066738+0000","last_became_peered":"2023-09-23T20:29:46.066738+0000","last_unstale":"2023-09-23T20:31:49.158167+0000","last_undegraded":"2023-09-23T20:31:49.158167+0000","last_fullsized":"2023-09-23T20:31:49.158167+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:33:14.679402+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":242,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T20:31:49.146162+0000","last_change":"2023-09-23T20:29:46.178684+0000","last_active":"2023-09-23T20:31:49.146162+0000","last_peered":"2023-09-23T20:31:49.146162+0000","last_clean":"2023-09-23T20:31:49.146162+0000","last_became_active":"2023-09-23T20:29:46.178249+0000","last_became_peered":"2023-09-23T20:29:46.178249+0000","last_unstale":"2023-09-23T20:31:49.146162+0000","last_undegraded":"2023-09-23T20:31:49.146162+0000","last_fullsized":"2023-09-23T20:31:49.146162+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:58:58.530687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":270,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T20:31:49.145951+0000","last_change":"2023-09-23T20:25:02.957185+0000","last_active":"2023-09-23T20:31:49.145951+0000","last_peered":"2023-09-23T20:31:49.145951+0000","last_clean":"2023-09-23T20:31:49.145951+0000","last_became_active":"2023-09-23T20:25:02.956689+0000","last_became_peered":"2023-09-23T20:25:02.956689+0000","last_unstale":"2023-09-23T20:31:49.145951+0000","last_undegraded":"2023-09-23T20:31:49.145951+0000","last_fullsized":"2023-09-23T20:31:49.145951+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:04:28.524015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":167,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T20:31:49.151227+0000","last_change":"2023-09-23T20:29:45.529762+0000","last_active":"2023-09-23T20:31:49.151227+0000","last_peered":"2023-09-23T20:31:49.151227+0000","last_clean":"2023-09-23T20:31:49.151227+0000","last_became_active":"2023-09-23T20:29:45.529283+0000","last_became_peered":"2023-09-23T20:29:45.529283+0000","last_unstale":"2023-09-23T20:31:49.151227+0000","last_undegraded":"2023-09-23T20:31:49.151227+0000","last_fullsized":"2023-09-23T20:31:49.151227+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:54:36.720075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":78,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T20:31:49.150117+0000","last_change":"2023-09-23T20:29:46.112977+0000","last_active":"2023-09-23T20:31:49.150117+0000","last_peered":"2023-09-23T20:31:49.150117+0000","last_clean":"2023-09-23T20:31:49.150117+0000","last_became_active":"2023-09-23T20:29:46.112600+0000","last_became_peered":"2023-09-23T20:29:46.112600+0000","last_unstale":"2023-09-23T20:31:49.150117+0000","last_undegraded":"2023-09-23T20:31:49.150117+0000","last_fullsized":"2023-09-23T20:31:49.150117+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:06:34.132247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":268,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T20:31:49.145319+0000","last_change":"2023-09-23T20:25:02.969432+0000","last_active":"2023-09-23T20:31:49.145319+0000","last_peered":"2023-09-23T20:31:49.145319+0000","last_clean":"2023-09-23T20:31:49.145319+0000","last_became_active":"2023-09-23T20:25:02.969118+0000","last_became_peered":"2023-09-23T20:25:02.969118+0000","last_unstale":"2023-09-23T20:31:49.145319+0000","last_undegraded":"2023-09-23T20:31:49.145319+0000","last_fullsized":"2023-09-23T20:31:49.145319+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:02:22.453831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":187,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T20:31:49.151253+0000","last_change":"2023-09-23T20:29:46.069832+0000","last_active":"2023-09-23T20:31:49.151253+0000","last_peered":"2023-09-23T20:31:49.151253+0000","last_clean":"2023-09-23T20:31:49.151253+0000","last_became_active":"2023-09-23T20:29:46.069451+0000","last_became_peered":"2023-09-23T20:29:46.069451+0000","last_unstale":"2023-09-23T20:31:49.151253+0000","last_undegraded":"2023-09-23T20:31:49.151253+0000","last_fullsized":"2023-09-23T20:31:49.151253+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:06:06.104030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"90'161","reported_seq":587,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T20:31:49.147178+0000","last_change":"2023-09-23T20:31:47.479140+0000","last_active":"2023-09-23T20:31:49.147178+0000","last_peered":"2023-09-23T20:31:49.147178+0000","last_clean":"2023-09-23T20:31:49.147178+0000","last_became_active":"2023-09-23T20:29:48.092460+0000","last_became_peered":"2023-09-23T20:29:48.092460+0000","last_unstale":"2023-09-23T20:31:49.147178+0000","last_undegraded":"2023-09-23T20:31:49.147178+0000","last_fullsized":"2023-09-23T20:31:49.147178+0000","mapping_epoch":269,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:47.479012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:47.479012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:37:14.860791+0000","scrub_duration":0.0020136590000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":281,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T20:31:49.147703+0000","last_change":"2023-09-23T20:29:45.523808+0000","last_active":"2023-09-23T20:31:49.147703+0000","last_peered":"2023-09-23T20:31:49.147703+0000","last_clean":"2023-09-23T20:31:49.147703+0000","last_became_active":"2023-09-23T20:29:45.523508+0000","last_became_peered":"2023-09-23T20:29:45.523508+0000","last_unstale":"2023-09-23T20:31:49.147703+0000","last_undegraded":"2023-09-23T20:31:49.147703+0000","last_fullsized":"2023-09-23T20:31:49.147703+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:50:55.565352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":170,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T20:31:49.150617+0000","last_change":"2023-09-23T20:29:04.130512+0000","last_active":"2023-09-23T20:31:49.150617+0000","last_peered":"2023-09-23T20:31:49.150617+0000","last_clean":"2023-09-23T20:31:49.150617+0000","last_became_active":"2023-09-23T20:29:04.130135+0000","last_became_peered":"2023-09-23T20:29:04.130135+0000","last_unstale":"2023-09-23T20:31:49.150617+0000","last_undegraded":"2023-09-23T20:31:49.150617+0000","last_fullsized":"2023-09-23T20:31:49.150617+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:53:59.385893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":315,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-09-23T20:31:48.151460+0000","last_change":"2023-09-23T20:29:59.501260+0000","last_active":"2023-09-23T20:31:48.151460+0000","last_peered":"2023-09-23T20:31:48.151460+0000","last_clean":"2023-09-23T20:31:48.151460+0000","last_became_active":"2023-09-23T20:29:28.362070+0000","last_became_peered":"2023-09-23T20:29:28.362070+0000","last_unstale":"2023-09-23T20:31:48.151460+0000","last_undegraded":"2023-09-23T20:31:48.151460+0000","last_fullsized":"2023-09-23T20:31:48.151460+0000","mapping_epoch":258,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:32:55.983696+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":252,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T20:31:49.145541+0000","last_change":"2023-09-23T20:29:46.080766+0000","last_active":"2023-09-23T20:31:49.145541+0000","last_peered":"2023-09-23T20:31:49.145541+0000","last_clean":"2023-09-23T20:31:49.145541+0000","last_became_active":"2023-09-23T20:29:46.080356+0000","last_became_peered":"2023-09-23T20:29:46.080356+0000","last_unstale":"2023-09-23T20:31:49.145541+0000","last_undegraded":"2023-09-23T20:31:49.145541+0000","last_fullsized":"2023-09-23T20:31:49.145541+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:08:01.862270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":121,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-09-23T20:31:48.151402+0000","last_change":"2023-09-23T20:29:59.529392+0000","last_active":"2023-09-23T20:31:48.151402+0000","last_peered":"2023-09-23T20:31:48.151402+0000","last_clean":"2023-09-23T20:31:48.151402+0000","last_became_active":"2023-09-23T20:29:28.365579+0000","last_became_peered":"2023-09-23T20:29:28.365579+0000","last_unstale":"2023-09-23T20:31:48.151402+0000","last_undegraded":"2023-09-23T20:31:48.151402+0000","last_fullsized":"2023-09-23T20:31:48.151402+0000","mapping_epoch":258,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:31:11.546083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":300,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T20:31:49.149803+0000","last_change":"2023-09-23T20:29:46.091575+0000","last_active":"2023-09-23T20:31:49.149803+0000","last_peered":"2023-09-23T20:31:49.149803+0000","last_clean":"2023-09-23T20:31:49.149803+0000","last_became_active":"2023-09-23T20:29:46.091271+0000","last_became_peered":"2023-09-23T20:29:46.091271+0000","last_unstale":"2023-09-23T20:31:49.149803+0000","last_undegraded":"2023-09-23T20:31:49.149803+0000","last_fullsized":"2023-09-23T20:31:49.149803+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:08:01.594175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.11","version":"90'161","reported_seq":533,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T20:31:49.147099+0000","last_change":"2023-09-23T20:31:43.578810+0000","last_active":"2023-09-23T20:31:49.147099+0000","last_peered":"2023-09-23T20:31:49.147099+0000","last_clean":"2023-09-23T20:31:49.147099+0000","last_became_active":"2023-09-23T20:28:05.991304+0000","last_became_peered":"2023-09-23T20:28:05.991304+0000","last_unstale":"2023-09-23T20:31:49.147099+0000","last_undegraded":"2023-09-23T20:31:49.147099+0000","last_fullsized":"2023-09-23T20:31:49.147099+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:43.578666+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:43.578666+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:46:18.999991+0000","scrub_duration":0.0021890379999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":314,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-09-23T20:31:48.150980+0000","last_change":"2023-09-23T20:29:59.544701+0000","last_active":"2023-09-23T20:31:48.150980+0000","last_peered":"2023-09-23T20:31:48.150980+0000","last_clean":"2023-09-23T20:31:48.150980+0000","last_became_active":"2023-09-23T20:29:28.367679+0000","last_became_peered":"2023-09-23T20:29:28.367679+0000","last_unstale":"2023-09-23T20:31:48.150980+0000","last_undegraded":"2023-09-23T20:31:48.150980+0000","last_fullsized":"2023-09-23T20:31:48.150980+0000","mapping_epoch":258,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:36:19.089724+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":102,"num_keys_recovered":30,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":167,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T20:31:49.158723+0000","last_change":"2023-09-23T20:29:45.531343+0000","last_active":"2023-09-23T20:31:49.158723+0000","last_peered":"2023-09-23T20:31:49.158723+0000","last_clean":"2023-09-23T20:31:49.158723+0000","last_became_active":"2023-09-23T20:29:45.530713+0000","last_became_peered":"2023-09-23T20:29:45.530713+0000","last_unstale":"2023-09-23T20:31:49.158723+0000","last_undegraded":"2023-09-23T20:31:49.158723+0000","last_fullsized":"2023-09-23T20:31:49.158723+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:28:00.245911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"90'161","reported_seq":535,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T20:31:49.726289+0000","last_change":"2023-09-23T20:31:49.726214+0000","last_active":"2023-09-23T20:31:49.726289+0000","last_peered":"2023-09-23T20:31:49.726289+0000","last_clean":"2023-09-23T20:31:49.726289+0000","last_became_active":"2023-09-23T20:29:07.044892+0000","last_became_peered":"2023-09-23T20:29:07.044892+0000","last_unstale":"2023-09-23T20:31:49.726289+0000","last_undegraded":"2023-09-23T20:31:49.726289+0000","last_fullsized":"2023-09-23T20:31:49.726289+0000","mapping_epoch":253,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":254,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:49.726139+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:49.726139+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:59:11.810516+0000","scrub_duration":0.0015441470000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":269,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-09-23T20:31:50.181963+0000","last_change":"2023-09-23T20:25:02.953993+0000","last_active":"2023-09-23T20:31:50.181963+0000","last_peered":"2023-09-23T20:31:50.181963+0000","last_clean":"2023-09-23T20:31:50.181963+0000","last_became_active":"2023-09-23T20:25:02.953672+0000","last_became_peered":"2023-09-23T20:25:02.953672+0000","last_unstale":"2023-09-23T20:31:50.181963+0000","last_undegraded":"2023-09-23T20:31:50.181963+0000","last_fullsized":"2023-09-23T20:31:50.181963+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:26:08.603577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":283,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T20:31:49.147778+0000","last_change":"2023-09-23T20:25:02.949584+0000","last_active":"2023-09-23T20:31:49.147778+0000","last_peered":"2023-09-23T20:31:49.147778+0000","last_clean":"2023-09-23T20:31:49.147778+0000","last_became_active":"2023-09-23T20:25:02.949132+0000","last_became_peered":"2023-09-23T20:25:02.949132+0000","last_unstale":"2023-09-23T20:31:49.147778+0000","last_undegraded":"2023-09-23T20:31:49.147778+0000","last_fullsized":"2023-09-23T20:31:49.147778+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:34:20.509823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"90'161","reported_seq":543,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T20:31:49.147734+0000","last_change":"2023-09-23T20:31:42.544488+0000","last_active":"2023-09-23T20:31:49.147734+0000","last_peered":"2023-09-23T20:31:49.147734+0000","last_clean":"2023-09-23T20:31:49.147734+0000","last_became_active":"2023-09-23T20:29:46.066622+0000","last_became_peered":"2023-09-23T20:29:46.066622+0000","last_unstale":"2023-09-23T20:31:49.147734+0000","last_undegraded":"2023-09-23T20:31:49.147734+0000","last_fullsized":"2023-09-23T20:31:49.147734+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:42.544308+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:42.544308+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:15:07.852825+0000","scrub_duration":0.0029958179999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":277,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-09-23T20:31:48.150885+0000","last_change":"2023-09-23T20:29:28.364113+0000","last_active":"2023-09-23T20:31:48.150885+0000","last_peered":"2023-09-23T20:31:48.150885+0000","last_clean":"2023-09-23T20:31:48.150885+0000","last_became_active":"2023-09-23T20:29:28.363788+0000","last_became_peered":"2023-09-23T20:29:28.363788+0000","last_unstale":"2023-09-23T20:31:48.150885+0000","last_undegraded":"2023-09-23T20:31:48.150885+0000","last_fullsized":"2023-09-23T20:31:48.150885+0000","mapping_epoch":258,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":259,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:39:49.658803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":278,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T20:31:49.146041+0000","last_change":"2023-09-23T20:25:02.957010+0000","last_active":"2023-09-23T20:31:49.146041+0000","last_peered":"2023-09-23T20:31:49.146041+0000","last_clean":"2023-09-23T20:31:49.146041+0000","last_became_active":"2023-09-23T20:25:02.956267+0000","last_became_peered":"2023-09-23T20:25:02.956267+0000","last_unstale":"2023-09-23T20:31:49.146041+0000","last_undegraded":"2023-09-23T20:31:49.146041+0000","last_fullsized":"2023-09-23T20:31:49.146041+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:10:55.299422+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"90'161","reported_seq":554,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-09-23T20:31:53.707081+0000","last_change":"2023-09-23T20:31:53.707033+0000","last_active":"2023-09-23T20:31:53.707081+0000","last_peered":"2023-09-23T20:31:53.707081+0000","last_clean":"2023-09-23T20:31:53.707081+0000","last_became_active":"2023-09-23T20:29:46.078873+0000","last_became_peered":"2023-09-23T20:29:46.078873+0000","last_unstale":"2023-09-23T20:31:53.707081+0000","last_undegraded":"2023-09-23T20:31:53.707081+0000","last_fullsized":"2023-09-23T20:31:53.707081+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:53.706934+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:53.706934+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:51:58.039657+0000","scrub_duration":0.0016840900000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":74,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T20:31:49.149479+0000","last_change":"2023-09-23T20:29:04.179171+0000","last_active":"2023-09-23T20:31:49.149479+0000","last_peered":"2023-09-23T20:31:49.149479+0000","last_clean":"2023-09-23T20:31:49.149479+0000","last_became_active":"2023-09-23T20:29:04.128672+0000","last_became_peered":"2023-09-23T20:29:04.128672+0000","last_unstale":"2023-09-23T20:31:49.149479+0000","last_undegraded":"2023-09-23T20:31:49.149479+0000","last_fullsized":"2023-09-23T20:31:49.149479+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:49:15.506106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":167,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T20:31:49.159992+0000","last_change":"2023-09-23T20:29:45.528782+0000","last_active":"2023-09-23T20:31:49.159992+0000","last_peered":"2023-09-23T20:31:49.159992+0000","last_clean":"2023-09-23T20:31:49.159992+0000","last_became_active":"2023-09-23T20:29:45.528389+0000","last_became_peered":"2023-09-23T20:29:45.528389+0000","last_unstale":"2023-09-23T20:31:49.159992+0000","last_undegraded":"2023-09-23T20:31:49.159992+0000","last_fullsized":"2023-09-23T20:31:49.159992+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:13:41.453499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.b","version":"0'0","reported_seq":11,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T20:31:49.159809+0000","last_change":"2023-09-23T20:31:49.152343+0000","last_active":"2023-09-23T20:31:49.159809+0000","last_peered":"2023-09-23T20:31:49.159809+0000","last_clean":"2023-09-23T20:31:49.159809+0000","last_became_active":"2023-09-23T20:31:49.152042+0000","last_became_peered":"2023-09-23T20:31:49.152042+0000","last_unstale":"2023-09-23T20:31:49.159809+0000","last_undegraded":"2023-09-23T20:31:49.159809+0000","last_fullsized":"2023-09-23T20:31:49.159809+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:53:34.237468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.d","version":"90'161","reported_seq":553,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-09-23T20:31:48.151342+0000","last_change":"2023-09-23T20:31:46.388677+0000","last_active":"2023-09-23T20:31:48.151342+0000","last_peered":"2023-09-23T20:31:48.151342+0000","last_clean":"2023-09-23T20:31:48.151342+0000","last_became_active":"2023-09-23T20:29:46.061003+0000","last_became_peered":"2023-09-23T20:29:46.061003+0000","last_unstale":"2023-09-23T20:31:48.151342+0000","last_undegraded":"2023-09-23T20:31:48.151342+0000","last_fullsized":"2023-09-23T20:31:48.151342+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:46.388545+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:46.388545+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:07:46.116173+0000","scrub_duration":0.00168356,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":293,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-09-23T20:31:48.151650+0000","last_change":"2023-09-23T20:29:59.586041+0000","last_active":"2023-09-23T20:31:48.151650+0000","last_peered":"2023-09-23T20:31:48.151650+0000","last_clean":"2023-09-23T20:31:48.151650+0000","last_became_active":"2023-09-23T20:29:46.059906+0000","last_became_peered":"2023-09-23T20:29:46.059906+0000","last_unstale":"2023-09-23T20:31:48.151650+0000","last_undegraded":"2023-09-23T20:31:48.151650+0000","last_fullsized":"2023-09-23T20:31:48.151650+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:26:12.412320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":266,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T20:31:49.147375+0000","last_change":"2023-09-23T20:29:04.128604+0000","last_active":"2023-09-23T20:31:49.147375+0000","last_peered":"2023-09-23T20:31:49.147375+0000","last_clean":"2023-09-23T20:31:49.147375+0000","last_became_active":"2023-09-23T20:29:04.128003+0000","last_became_peered":"2023-09-23T20:29:04.128003+0000","last_unstale":"2023-09-23T20:31:49.147375+0000","last_undegraded":"2023-09-23T20:31:49.147375+0000","last_fullsized":"2023-09-23T20:31:49.147375+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:50:49.385599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"0'0","reported_seq":11,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T20:31:49.157811+0000","last_change":"2023-09-23T20:31:49.157410+0000","last_active":"2023-09-23T20:31:49.157811+0000","last_peered":"2023-09-23T20:31:49.157811+0000","last_clean":"2023-09-23T20:31:49.157811+0000","last_became_active":"2023-09-23T20:31:49.157114+0000","last_became_peered":"2023-09-23T20:31:49.157114+0000","last_unstale":"2023-09-23T20:31:49.157811+0000","last_undegraded":"2023-09-23T20:31:49.157811+0000","last_fullsized":"2023-09-23T20:31:49.157811+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:06:34.944202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.c","version":"90'161","reported_seq":600,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T20:31:49.158477+0000","last_change":"2023-09-23T20:31:41.701273+0000","last_active":"2023-09-23T20:31:49.158477+0000","last_peered":"2023-09-23T20:31:49.158477+0000","last_clean":"2023-09-23T20:31:49.158477+0000","last_became_active":"2023-09-23T20:30:01.493025+0000","last_became_peered":"2023-09-23T20:30:01.493025+0000","last_unstale":"2023-09-23T20:31:49.158477+0000","last_undegraded":"2023-09-23T20:31:49.158477+0000","last_fullsized":"2023-09-23T20:31:49.158477+0000","mapping_epoch":275,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":276,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:41.701171+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:41.701171+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:54:32.156494+0000","scrub_duration":0.002180043,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":300,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-09-23T20:31:48.151562+0000","last_change":"2023-09-23T20:29:59.567502+0000","last_active":"2023-09-23T20:31:48.151562+0000","last_peered":"2023-09-23T20:31:48.151562+0000","last_clean":"2023-09-23T20:31:48.151562+0000","last_became_active":"2023-09-23T20:29:46.055041+0000","last_became_peered":"2023-09-23T20:29:46.055041+0000","last_unstale":"2023-09-23T20:31:48.151562+0000","last_undegraded":"2023-09-23T20:31:48.151562+0000","last_fullsized":"2023-09-23T20:31:48.151562+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:47:13.874812+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":202,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T20:31:49.146088+0000","last_change":"2023-09-23T20:29:46.178516+0000","last_active":"2023-09-23T20:31:49.146088+0000","last_peered":"2023-09-23T20:31:49.146088+0000","last_clean":"2023-09-23T20:31:49.146088+0000","last_became_active":"2023-09-23T20:29:46.177858+0000","last_became_peered":"2023-09-23T20:29:46.177858+0000","last_unstale":"2023-09-23T20:31:49.146088+0000","last_undegraded":"2023-09-23T20:31:49.146088+0000","last_fullsized":"2023-09-23T20:31:49.146088+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:09:14.906568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.9","version":"0'0","reported_seq":11,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T20:31:50.040103+0000","last_change":"2023-09-23T20:31:49.158873+0000","last_active":"2023-09-23T20:31:50.040103+0000","last_peered":"2023-09-23T20:31:50.040103+0000","last_clean":"2023-09-23T20:31:50.040103+0000","last_became_active":"2023-09-23T20:31:49.157064+0000","last_became_peered":"2023-09-23T20:31:49.157064+0000","last_unstale":"2023-09-23T20:31:50.040103+0000","last_undegraded":"2023-09-23T20:31:50.040103+0000","last_fullsized":"2023-09-23T20:31:50.040103+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:35:34.905900+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"90'161","reported_seq":507,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T20:31:49.150034+0000","last_change":"2023-09-23T20:31:46.217574+0000","last_active":"2023-09-23T20:31:49.150034+0000","last_peered":"2023-09-23T20:31:49.150034+0000","last_clean":"2023-09-23T20:31:49.150034+0000","last_became_active":"2023-09-23T20:29:28.373750+0000","last_became_peered":"2023-09-23T20:29:28.373750+0000","last_unstale":"2023-09-23T20:31:49.150034+0000","last_undegraded":"2023-09-23T20:31:49.150034+0000","last_fullsized":"2023-09-23T20:31:49.150034+0000","mapping_epoch":258,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":259,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:46.217489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:46.217489+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:12:09.959810+0000","scrub_duration":0.001456429,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,3],"acting":[6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":276,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-09-23T20:31:48.151365+0000","last_change":"2023-09-23T20:29:04.184893+0000","last_active":"2023-09-23T20:31:48.151365+0000","last_peered":"2023-09-23T20:31:48.151365+0000","last_clean":"2023-09-23T20:31:48.151365+0000","last_became_active":"2023-09-23T20:29:04.152723+0000","last_became_peered":"2023-09-23T20:29:04.152723+0000","last_unstale":"2023-09-23T20:31:48.151365+0000","last_undegraded":"2023-09-23T20:31:48.151365+0000","last_fullsized":"2023-09-23T20:31:48.151365+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:51:27.017151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":78,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T20:31:49.150397+0000","last_change":"2023-09-23T20:29:46.086912+0000","last_active":"2023-09-23T20:31:49.150397+0000","last_peered":"2023-09-23T20:31:49.150397+0000","last_clean":"2023-09-23T20:31:49.150397+0000","last_became_active":"2023-09-23T20:29:46.085471+0000","last_became_peered":"2023-09-23T20:29:46.085471+0000","last_unstale":"2023-09-23T20:31:49.150397+0000","last_undegraded":"2023-09-23T20:31:49.150397+0000","last_fullsized":"2023-09-23T20:31:49.150397+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:04:55.751660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.8","version":"0'0","reported_seq":11,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-09-23T20:31:51.425688+0000","last_change":"2023-09-23T20:31:49.160706+0000","last_active":"2023-09-23T20:31:51.425688+0000","last_peered":"2023-09-23T20:31:51.425688+0000","last_clean":"2023-09-23T20:31:51.425688+0000","last_became_active":"2023-09-23T20:31:49.160477+0000","last_became_peered":"2023-09-23T20:31:49.160477+0000","last_unstale":"2023-09-23T20:31:51.425688+0000","last_undegraded":"2023-09-23T20:31:51.425688+0000","last_fullsized":"2023-09-23T20:31:51.425688+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:17:43.864776+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.e","version":"90'161","reported_seq":493,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-09-23T20:31:48.151432+0000","last_change":"2023-09-23T20:31:44.387138+0000","last_active":"2023-09-23T20:31:48.151432+0000","last_peered":"2023-09-23T20:31:48.151432+0000","last_clean":"2023-09-23T20:31:48.151432+0000","last_became_active":"2023-09-23T20:29:31.394053+0000","last_became_peered":"2023-09-23T20:29:31.394053+0000","last_unstale":"2023-09-23T20:31:48.151432+0000","last_undegraded":"2023-09-23T20:31:48.151432+0000","last_fullsized":"2023-09-23T20:31:48.151432+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:44.386120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:44.386120+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:01:15.591281+0000","scrub_duration":0.0015151330000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":311,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T20:31:49.147522+0000","last_change":"2023-09-23T20:29:04.138284+0000","last_active":"2023-09-23T20:31:49.147522+0000","last_peered":"2023-09-23T20:31:49.147522+0000","last_clean":"2023-09-23T20:31:49.147522+0000","last_became_active":"2023-09-23T20:29:04.137960+0000","last_became_peered":"2023-09-23T20:29:04.137960+0000","last_unstale":"2023-09-23T20:31:49.147522+0000","last_undegraded":"2023-09-23T20:31:49.147522+0000","last_fullsized":"2023-09-23T20:31:49.147522+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:18:59.548311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":278,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T20:31:49.145135+0000","last_change":"2023-09-23T20:29:46.061136+0000","last_active":"2023-09-23T20:31:49.145135+0000","last_peered":"2023-09-23T20:31:49.145135+0000","last_clean":"2023-09-23T20:31:49.145135+0000","last_became_active":"2023-09-23T20:29:46.060777+0000","last_became_peered":"2023-09-23T20:29:46.060777+0000","last_unstale":"2023-09-23T20:31:49.145135+0000","last_undegraded":"2023-09-23T20:31:49.145135+0000","last_fullsized":"2023-09-23T20:31:49.145135+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:08:43.673870+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"0'0","reported_seq":11,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T20:31:50.040348+0000","last_change":"2023-09-23T20:31:49.158461+0000","last_active":"2023-09-23T20:31:50.040348+0000","last_peered":"2023-09-23T20:31:50.040348+0000","last_clean":"2023-09-23T20:31:50.040348+0000","last_became_active":"2023-09-23T20:31:49.158188+0000","last_became_peered":"2023-09-23T20:31:49.158188+0000","last_unstale":"2023-09-23T20:31:50.040348+0000","last_undegraded":"2023-09-23T20:31:50.040348+0000","last_fullsized":"2023-09-23T20:31:50.040348+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:31:57.678090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":686,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-09-23T20:31:48.150784+0000","last_change":"2023-09-23T20:28:05.872149+0000","last_active":"2023-09-23T20:31:48.150784+0000","last_peered":"2023-09-23T20:31:48.150784+0000","last_clean":"2023-09-23T20:31:48.150784+0000","last_became_active":"2023-09-23T20:27:11.961310+0000","last_became_peered":"2023-09-23T20:27:11.961310+0000","last_unstale":"2023-09-23T20:31:48.150784+0000","last_undegraded":"2023-09-23T20:31:48.150784+0000","last_fullsized":"2023-09-23T20:31:48.150784+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:28:05.872074+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:28:05.872074+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:52:32.495955+0000","scrub_duration":0.0029215980000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":181,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T20:31:49.158050+0000","last_change":"2023-09-23T20:29:49.873018+0000","last_active":"2023-09-23T20:31:49.158050+0000","last_peered":"2023-09-23T20:31:49.158050+0000","last_clean":"2023-09-23T20:31:49.158050+0000","last_became_active":"2023-09-23T20:29:46.070369+0000","last_became_peered":"2023-09-23T20:29:46.070369+0000","last_unstale":"2023-09-23T20:31:49.158050+0000","last_undegraded":"2023-09-23T20:31:49.158050+0000","last_fullsized":"2023-09-23T20:31:49.158050+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:24:13.119725+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":317,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T20:31:49.158878+0000","last_change":"2023-09-23T20:29:46.173195+0000","last_active":"2023-09-23T20:31:49.158878+0000","last_peered":"2023-09-23T20:31:49.158878+0000","last_clean":"2023-09-23T20:31:49.158878+0000","last_became_active":"2023-09-23T20:29:46.172808+0000","last_became_peered":"2023-09-23T20:29:46.172808+0000","last_unstale":"2023-09-23T20:31:49.158878+0000","last_undegraded":"2023-09-23T20:31:49.158878+0000","last_fullsized":"2023-09-23T20:31:49.158878+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:41:58.927658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.e","version":"0'0","reported_seq":11,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-09-23T20:31:51.426095+0000","last_change":"2023-09-23T20:31:49.155754+0000","last_active":"2023-09-23T20:31:51.426095+0000","last_peered":"2023-09-23T20:31:51.426095+0000","last_clean":"2023-09-23T20:31:51.426095+0000","last_became_active":"2023-09-23T20:31:49.155363+0000","last_became_peered":"2023-09-23T20:31:49.155363+0000","last_unstale":"2023-09-23T20:31:51.426095+0000","last_undegraded":"2023-09-23T20:31:51.426095+0000","last_fullsized":"2023-09-23T20:31:51.426095+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:45:42.661641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":529,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T20:31:49.147200+0000","last_change":"2023-09-23T20:27:31.757060+0000","last_active":"2023-09-23T20:31:49.147200+0000","last_peered":"2023-09-23T20:31:49.147200+0000","last_clean":"2023-09-23T20:31:49.147200+0000","last_became_active":"2023-09-23T20:27:11.959516+0000","last_became_peered":"2023-09-23T20:27:11.959516+0000","last_unstale":"2023-09-23T20:31:49.147200+0000","last_undegraded":"2023-09-23T20:31:49.147200+0000","last_fullsized":"2023-09-23T20:31:49.147200+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:31.756974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:31.756974+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:35:11.939335+0000","scrub_duration":0.0031802670000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":90,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T20:31:49.150449+0000","last_change":"2023-09-23T20:29:46.087097+0000","last_active":"2023-09-23T20:31:49.150449+0000","last_peered":"2023-09-23T20:31:49.150449+0000","last_clean":"2023-09-23T20:31:49.150449+0000","last_became_active":"2023-09-23T20:29:46.085908+0000","last_became_peered":"2023-09-23T20:29:46.085908+0000","last_unstale":"2023-09-23T20:31:49.150449+0000","last_undegraded":"2023-09-23T20:31:49.150449+0000","last_fullsized":"2023-09-23T20:31:49.150449+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:38:46.001432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":215,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T20:31:49.147415+0000","last_change":"2023-09-23T20:29:28.370476+0000","last_active":"2023-09-23T20:31:49.147415+0000","last_peered":"2023-09-23T20:31:49.147415+0000","last_clean":"2023-09-23T20:31:49.147415+0000","last_became_active":"2023-09-23T20:29:28.370158+0000","last_became_peered":"2023-09-23T20:29:28.370158+0000","last_unstale":"2023-09-23T20:31:49.147415+0000","last_undegraded":"2023-09-23T20:31:49.147415+0000","last_fullsized":"2023-09-23T20:31:49.147415+0000","mapping_epoch":258,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":259,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:09:47.335404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.d","version":"0'0","reported_seq":10,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T20:31:49.158968+0000","last_change":"2023-09-23T20:31:49.158641+0000","last_active":"2023-09-23T20:31:49.158968+0000","last_peered":"2023-09-23T20:31:49.158968+0000","last_clean":"2023-09-23T20:31:49.158968+0000","last_became_active":"2023-09-23T20:31:49.157604+0000","last_became_peered":"2023-09-23T20:31:49.157604+0000","last_unstale":"2023-09-23T20:31:49.158968+0000","last_undegraded":"2023-09-23T20:31:49.158968+0000","last_fullsized":"2023-09-23T20:31:49.158968+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:54:04.726000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.b","version":"90'161","reported_seq":546,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T20:31:49.147814+0000","last_change":"2023-09-23T20:31:46.558396+0000","last_active":"2023-09-23T20:31:49.147814+0000","last_peered":"2023-09-23T20:31:49.147814+0000","last_clean":"2023-09-23T20:31:49.147814+0000","last_became_active":"2023-09-23T20:29:45.534070+0000","last_became_peered":"2023-09-23T20:29:45.534070+0000","last_unstale":"2023-09-23T20:31:49.147814+0000","last_undegraded":"2023-09-23T20:31:49.147814+0000","last_fullsized":"2023-09-23T20:31:49.147814+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:46.558292+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:46.558292+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:10:34.051169+0000","scrub_duration":0.0023048779999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":217,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T20:31:49.147854+0000","last_change":"2023-09-23T20:29:46.061620+0000","last_active":"2023-09-23T20:31:49.147854+0000","last_peered":"2023-09-23T20:31:49.147854+0000","last_clean":"2023-09-23T20:31:49.147854+0000","last_became_active":"2023-09-23T20:29:46.061271+0000","last_became_peered":"2023-09-23T20:29:46.061271+0000","last_unstale":"2023-09-23T20:31:49.147854+0000","last_undegraded":"2023-09-23T20:31:49.147854+0000","last_fullsized":"2023-09-23T20:31:49.147854+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:24:51.409521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":300,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T20:31:49.149866+0000","last_change":"2023-09-23T20:29:46.085962+0000","last_active":"2023-09-23T20:31:49.149866+0000","last_peered":"2023-09-23T20:31:49.149866+0000","last_clean":"2023-09-23T20:31:49.149866+0000","last_became_active":"2023-09-23T20:29:46.082826+0000","last_became_peered":"2023-09-23T20:29:46.082826+0000","last_unstale":"2023-09-23T20:31:49.149866+0000","last_undegraded":"2023-09-23T20:31:49.149866+0000","last_fullsized":"2023-09-23T20:31:49.149866+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:26:17.599051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.c","version":"0'0","reported_seq":10,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T20:31:49.157533+0000","last_change":"2023-09-23T20:31:49.157247+0000","last_active":"2023-09-23T20:31:49.157533+0000","last_peered":"2023-09-23T20:31:49.157533+0000","last_clean":"2023-09-23T20:31:49.157533+0000","last_became_active":"2023-09-23T20:31:49.156961+0000","last_became_peered":"2023-09-23T20:31:49.156961+0000","last_unstale":"2023-09-23T20:31:49.157533+0000","last_undegraded":"2023-09-23T20:31:49.157533+0000","last_fullsized":"2023-09-23T20:31:49.157533+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:49:17.346795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":569,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T20:31:49.147515+0000","last_change":"2023-09-23T20:29:07.053561+0000","last_active":"2023-09-23T20:31:49.147515+0000","last_peered":"2023-09-23T20:31:49.147515+0000","last_clean":"2023-09-23T20:31:49.147515+0000","last_became_active":"2023-09-23T20:29:07.053177+0000","last_became_peered":"2023-09-23T20:29:07.053177+0000","last_unstale":"2023-09-23T20:31:49.147515+0000","last_undegraded":"2023-09-23T20:31:49.147515+0000","last_fullsized":"2023-09-23T20:31:49.147515+0000","mapping_epoch":253,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":254,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:30.727267+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:30.727267+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:04:32.661761+0000","scrub_duration":0.0052560289999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":168,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T20:31:49.151140+0000","last_change":"2023-09-23T20:29:45.529576+0000","last_active":"2023-09-23T20:31:49.151140+0000","last_peered":"2023-09-23T20:31:49.151140+0000","last_clean":"2023-09-23T20:31:49.151140+0000","last_became_active":"2023-09-23T20:29:45.528858+0000","last_became_peered":"2023-09-23T20:29:45.528858+0000","last_unstale":"2023-09-23T20:31:49.151140+0000","last_undegraded":"2023-09-23T20:31:49.151140+0000","last_fullsized":"2023-09-23T20:31:49.151140+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:42:15.589821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":115,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T20:31:49.146337+0000","last_change":"2023-09-23T20:29:46.064515+0000","last_active":"2023-09-23T20:31:49.146337+0000","last_peered":"2023-09-23T20:31:49.146337+0000","last_clean":"2023-09-23T20:31:49.146337+0000","last_became_active":"2023-09-23T20:29:46.064253+0000","last_became_peered":"2023-09-23T20:29:46.064253+0000","last_unstale":"2023-09-23T20:31:49.146337+0000","last_undegraded":"2023-09-23T20:31:49.146337+0000","last_fullsized":"2023-09-23T20:31:49.146337+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:21:20.039440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"0'0","reported_seq":11,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T20:31:49.160139+0000","last_change":"2023-09-23T20:31:49.159680+0000","last_active":"2023-09-23T20:31:49.160139+0000","last_peered":"2023-09-23T20:31:49.160139+0000","last_clean":"2023-09-23T20:31:49.160139+0000","last_became_active":"2023-09-23T20:31:49.159058+0000","last_became_peered":"2023-09-23T20:31:49.159058+0000","last_unstale":"2023-09-23T20:31:49.160139+0000","last_undegraded":"2023-09-23T20:31:49.160139+0000","last_fullsized":"2023-09-23T20:31:49.160139+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:39:30.678742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":534,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T20:31:49.150117+0000","last_change":"2023-09-23T20:31:45.196356+0000","last_active":"2023-09-23T20:31:49.150117+0000","last_peered":"2023-09-23T20:31:49.150117+0000","last_clean":"2023-09-23T20:31:49.150117+0000","last_became_active":"2023-09-23T20:29:07.115647+0000","last_became_peered":"2023-09-23T20:29:07.115647+0000","last_unstale":"2023-09-23T20:31:49.150117+0000","last_undegraded":"2023-09-23T20:31:49.150117+0000","last_fullsized":"2023-09-23T20:31:49.150117+0000","mapping_epoch":253,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":254,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:45.196285+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:45.196285+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:33:06.666183+0000","scrub_duration":0.0014380860000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"86'119","reported_seq":609,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-09-23T20:31:51.859499+0000","last_change":"2023-09-23T20:29:45.521257+0000","last_active":"2023-09-23T20:31:51.859499+0000","last_peered":"2023-09-23T20:31:51.859499+0000","last_clean":"2023-09-23T20:31:51.859499+0000","last_became_active":"2023-09-23T20:29:45.520546+0000","last_became_peered":"2023-09-23T20:29:45.520546+0000","last_unstale":"2023-09-23T20:31:51.859499+0000","last_undegraded":"2023-09-23T20:31:51.859499+0000","last_fullsized":"2023-09-23T20:31:51.859499+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":119,"log_dups_size":0,"ondisk_log_size":119,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:58:56.085777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032705499999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":523,"num_read_kb":3503,"num_write":136,"num_write_kb":3128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":321,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T20:31:49.147741+0000","last_change":"2023-09-23T20:25:39.637160+0000","last_active":"2023-09-23T20:31:49.147741+0000","last_peered":"2023-09-23T20:31:49.147741+0000","last_clean":"2023-09-23T20:31:49.147741+0000","last_became_active":"2023-09-23T20:25:39.636860+0000","last_became_peered":"2023-09-23T20:25:39.636860+0000","last_unstale":"2023-09-23T20:31:49.147741+0000","last_undegraded":"2023-09-23T20:31:49.147741+0000","last_fullsized":"2023-09-23T20:31:49.147741+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:21:47.860063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":211,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-09-23T20:31:51.859030+0000","last_change":"2023-09-23T20:29:46.090683+0000","last_active":"2023-09-23T20:31:51.859030+0000","last_peered":"2023-09-23T20:31:51.859030+0000","last_clean":"2023-09-23T20:31:51.859030+0000","last_became_active":"2023-09-23T20:29:46.090131+0000","last_became_peered":"2023-09-23T20:29:46.090131+0000","last_unstale":"2023-09-23T20:31:51.859030+0000","last_undegraded":"2023-09-23T20:31:51.859030+0000","last_fullsized":"2023-09-23T20:31:51.859030+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:36:35.754684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"0'0","reported_seq":10,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T20:31:49.157726+0000","last_change":"2023-09-23T20:31:49.157301+0000","last_active":"2023-09-23T20:31:49.157726+0000","last_peered":"2023-09-23T20:31:49.157726+0000","last_clean":"2023-09-23T20:31:49.157726+0000","last_became_active":"2023-09-23T20:31:49.156992+0000","last_became_peered":"2023-09-23T20:31:49.156992+0000","last_unstale":"2023-09-23T20:31:49.157726+0000","last_undegraded":"2023-09-23T20:31:49.157726+0000","last_fullsized":"2023-09-23T20:31:49.157726+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:21:35.645838+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":549,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T20:31:49.147065+0000","last_change":"2023-09-23T20:31:45.785521+0000","last_active":"2023-09-23T20:31:49.147065+0000","last_peered":"2023-09-23T20:31:49.147065+0000","last_clean":"2023-09-23T20:31:49.147065+0000","last_became_active":"2023-09-23T20:27:55.213394+0000","last_became_peered":"2023-09-23T20:27:55.213394+0000","last_unstale":"2023-09-23T20:31:49.147065+0000","last_undegraded":"2023-09-23T20:31:49.147065+0000","last_fullsized":"2023-09-23T20:31:49.147065+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:45.785450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:45.785450+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:21:26.092578+0000","scrub_duration":0.001558306,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"86'106","reported_seq":1023,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T20:31:49.147294+0000","last_change":"2023-09-23T20:31:31.531882+0000","last_active":"2023-09-23T20:31:49.147294+0000","last_peered":"2023-09-23T20:31:49.147294+0000","last_clean":"2023-09-23T20:31:49.147294+0000","last_became_active":"2023-09-23T20:29:46.078501+0000","last_became_peered":"2023-09-23T20:29:46.078501+0000","last_unstale":"2023-09-23T20:31:49.147294+0000","last_undegraded":"2023-09-23T20:31:49.147294+0000","last_fullsized":"2023-09-23T20:31:49.147294+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":106,"log_dups_size":0,"ondisk_log_size":106,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:37.015974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00099302599999999993,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1105,"num_read_kb":1689,"num_write":100,"num_write_kb":404,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":298,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T20:31:49.145421+0000","last_change":"2023-09-23T20:29:46.059640+0000","last_active":"2023-09-23T20:31:49.145421+0000","last_peered":"2023-09-23T20:31:49.145421+0000","last_clean":"2023-09-23T20:31:49.145421+0000","last_became_active":"2023-09-23T20:29:46.059279+0000","last_became_peered":"2023-09-23T20:29:46.059279+0000","last_unstale":"2023-09-23T20:31:49.145421+0000","last_undegraded":"2023-09-23T20:31:49.145421+0000","last_fullsized":"2023-09-23T20:31:49.145421+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:03:31.536847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":262,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-09-23T20:31:51.859471+0000","last_change":"2023-09-23T20:29:45.533406+0000","last_active":"2023-09-23T20:31:51.859471+0000","last_peered":"2023-09-23T20:31:51.859471+0000","last_clean":"2023-09-23T20:31:51.859471+0000","last_became_active":"2023-09-23T20:29:45.526001+0000","last_became_peered":"2023-09-23T20:29:45.526001+0000","last_unstale":"2023-09-23T20:31:51.859471+0000","last_undegraded":"2023-09-23T20:31:51.859471+0000","last_fullsized":"2023-09-23T20:31:51.859471+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:43.787251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.4","version":"0'0","reported_seq":11,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T20:31:50.040237+0000","last_change":"2023-09-23T20:31:49.157381+0000","last_active":"2023-09-23T20:31:50.040237+0000","last_peered":"2023-09-23T20:31:50.040237+0000","last_clean":"2023-09-23T20:31:50.040237+0000","last_became_active":"2023-09-23T20:31:49.157056+0000","last_became_peered":"2023-09-23T20:31:49.157056+0000","last_unstale":"2023-09-23T20:31:50.040237+0000","last_undegraded":"2023-09-23T20:31:50.040237+0000","last_fullsized":"2023-09-23T20:31:50.040237+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:10:18.406106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,6],"acting":[0,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":726,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-09-23T20:31:48.150838+0000","last_change":"2023-09-23T20:31:43.391012+0000","last_active":"2023-09-23T20:31:48.150838+0000","last_peered":"2023-09-23T20:31:48.150838+0000","last_clean":"2023-09-23T20:31:48.150838+0000","last_became_active":"2023-09-23T20:29:31.389288+0000","last_became_peered":"2023-09-23T20:29:31.389288+0000","last_unstale":"2023-09-23T20:31:48.150838+0000","last_undegraded":"2023-09-23T20:31:48.150838+0000","last_fullsized":"2023-09-23T20:31:48.150838+0000","mapping_epoch":261,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:43.390908+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:43.390908+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:51:15.405092+0000","scrub_duration":0.0032721,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,2],"acting":[7,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"86'130","reported_seq":616,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T20:31:49.147760+0000","last_change":"2023-09-23T20:25:39.637458+0000","last_active":"2023-09-23T20:31:49.147760+0000","last_peered":"2023-09-23T20:31:49.147760+0000","last_clean":"2023-09-23T20:31:49.147760+0000","last_became_active":"2023-09-23T20:25:39.636773+0000","last_became_peered":"2023-09-23T20:25:39.636773+0000","last_unstale":"2023-09-23T20:31:49.147760+0000","last_undegraded":"2023-09-23T20:31:49.147760+0000","last_fullsized":"2023-09-23T20:31:49.147760+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:00:32.810564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011300049999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":512,"num_read_kb":1820,"num_write":133,"num_write_kb":1347,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":252,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T20:31:49.145696+0000","last_change":"2023-09-23T20:29:46.073394+0000","last_active":"2023-09-23T20:31:49.145696+0000","last_peered":"2023-09-23T20:31:49.145696+0000","last_clean":"2023-09-23T20:31:49.145696+0000","last_became_active":"2023-09-23T20:29:46.072067+0000","last_became_peered":"2023-09-23T20:29:46.072067+0000","last_unstale":"2023-09-23T20:31:49.145696+0000","last_undegraded":"2023-09-23T20:31:49.145696+0000","last_fullsized":"2023-09-23T20:31:49.145696+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:01:56.944614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":243,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T20:31:49.146263+0000","last_change":"2023-09-23T20:29:45.533185+0000","last_active":"2023-09-23T20:31:49.146263+0000","last_peered":"2023-09-23T20:31:49.146263+0000","last_clean":"2023-09-23T20:31:49.146263+0000","last_became_active":"2023-09-23T20:29:45.526765+0000","last_became_peered":"2023-09-23T20:29:45.526765+0000","last_unstale":"2023-09-23T20:31:49.146263+0000","last_undegraded":"2023-09-23T20:31:49.146263+0000","last_fullsized":"2023-09-23T20:31:49.146263+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:18:33.063215+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"0'0","reported_seq":11,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T20:31:50.040309+0000","last_change":"2023-09-23T20:31:49.158194+0000","last_active":"2023-09-23T20:31:50.040309+0000","last_peered":"2023-09-23T20:31:50.040309+0000","last_clean":"2023-09-23T20:31:50.040309+0000","last_became_active":"2023-09-23T20:31:49.157875+0000","last_became_peered":"2023-09-23T20:31:49.157875+0000","last_unstale":"2023-09-23T20:31:50.040309+0000","last_undegraded":"2023-09-23T20:31:50.040309+0000","last_fullsized":"2023-09-23T20:31:50.040309+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:04:42.411181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"86'194","reported_seq":953,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T20:31:49.147440+0000","last_change":"2023-09-23T20:25:40.618665+0000","last_active":"2023-09-23T20:31:49.147440+0000","last_peered":"2023-09-23T20:31:49.147440+0000","last_clean":"2023-09-23T20:31:49.147440+0000","last_became_active":"2023-09-23T20:25:40.618087+0000","last_became_peered":"2023-09-23T20:25:40.618087+0000","last_unstale":"2023-09-23T20:31:49.147440+0000","last_undegraded":"2023-09-23T20:31:49.147440+0000","last_fullsized":"2023-09-23T20:31:49.147440+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":194,"log_dups_size":0,"ondisk_log_size":194,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:07:16.114968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00037995999999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1025,"num_read_kb":1201,"num_write":173,"num_write_kb":1276,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":737,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-09-23T20:31:48.150716+0000","last_change":"2023-09-23T20:31:45.343835+0000","last_active":"2023-09-23T20:31:48.150716+0000","last_peered":"2023-09-23T20:31:48.150716+0000","last_clean":"2023-09-23T20:31:48.150716+0000","last_became_active":"2023-09-23T20:29:07.042407+0000","last_became_peered":"2023-09-23T20:29:07.042407+0000","last_unstale":"2023-09-23T20:31:48.150716+0000","last_undegraded":"2023-09-23T20:31:48.150716+0000","last_fullsized":"2023-09-23T20:31:48.150716+0000","mapping_epoch":253,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":254,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:45.343742+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:45.343742+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:33:12.745366+0000","scrub_duration":0.001475802,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":171,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T20:31:49.158662+0000","last_change":"2023-09-23T20:29:04.216400+0000","last_active":"2023-09-23T20:31:49.158662+0000","last_peered":"2023-09-23T20:31:49.158662+0000","last_clean":"2023-09-23T20:31:49.158662+0000","last_became_active":"2023-09-23T20:29:04.216069+0000","last_became_peered":"2023-09-23T20:29:04.216069+0000","last_unstale":"2023-09-23T20:31:49.158662+0000","last_undegraded":"2023-09-23T20:31:49.158662+0000","last_fullsized":"2023-09-23T20:31:49.158662+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:27:37.433035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":74,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T20:31:49.150036+0000","last_change":"2023-09-23T20:29:28.376036+0000","last_active":"2023-09-23T20:31:49.150036+0000","last_peered":"2023-09-23T20:31:49.150036+0000","last_clean":"2023-09-23T20:31:49.150036+0000","last_became_active":"2023-09-23T20:29:28.375712+0000","last_became_peered":"2023-09-23T20:29:28.375712+0000","last_unstale":"2023-09-23T20:31:49.150036+0000","last_undegraded":"2023-09-23T20:31:49.150036+0000","last_fullsized":"2023-09-23T20:31:49.150036+0000","mapping_epoch":258,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":259,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:23:50.722625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"0'0","reported_seq":9,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T20:31:49.154418+0000","last_change":"2023-09-23T20:31:49.154418+0000","last_active":"2023-09-23T20:31:49.154418+0000","last_peered":"2023-09-23T20:31:49.154418+0000","last_clean":"2023-09-23T20:31:49.154418+0000","last_became_active":"2023-09-23T20:31:49.154182+0000","last_became_peered":"2023-09-23T20:31:49.154182+0000","last_unstale":"2023-09-23T20:31:49.154418+0000","last_undegraded":"2023-09-23T20:31:49.154418+0000","last_fullsized":"2023-09-23T20:31:49.154418+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:44:10.087486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":546,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-09-23T20:31:51.859733+0000","last_change":"2023-09-23T20:31:43.582312+0000","last_active":"2023-09-23T20:31:51.859733+0000","last_peered":"2023-09-23T20:31:51.859733+0000","last_clean":"2023-09-23T20:31:51.859733+0000","last_became_active":"2023-09-23T20:29:46.071938+0000","last_became_peered":"2023-09-23T20:29:46.071938+0000","last_unstale":"2023-09-23T20:31:51.859733+0000","last_undegraded":"2023-09-23T20:31:51.859733+0000","last_fullsized":"2023-09-23T20:31:51.859733+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:43.582146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:43.582146+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:55:14.837811+0000","scrub_duration":0.0016219730000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,5],"acting":[1,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"86'80","reported_seq":567,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-09-23T20:31:48.150649+0000","last_change":"2023-09-23T20:29:45.536796+0000","last_active":"2023-09-23T20:31:48.150649+0000","last_peered":"2023-09-23T20:31:48.150649+0000","last_clean":"2023-09-23T20:31:48.150649+0000","last_became_active":"2023-09-23T20:29:45.536439+0000","last_became_peered":"2023-09-23T20:29:45.536439+0000","last_unstale":"2023-09-23T20:31:48.150649+0000","last_undegraded":"2023-09-23T20:31:48.150649+0000","last_fullsized":"2023-09-23T20:31:48.150649+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":80,"log_dups_size":0,"ondisk_log_size":80,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:30:57.793132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035115,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":395,"num_read_kb":1370,"num_write":66,"num_write_kb":1061,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":278,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T20:31:49.147301+0000","last_change":"2023-09-23T20:29:28.360680+0000","last_active":"2023-09-23T20:31:49.147301+0000","last_peered":"2023-09-23T20:31:49.147301+0000","last_clean":"2023-09-23T20:31:49.147301+0000","last_became_active":"2023-09-23T20:29:28.360366+0000","last_became_peered":"2023-09-23T20:29:28.360366+0000","last_unstale":"2023-09-23T20:31:49.147301+0000","last_undegraded":"2023-09-23T20:31:49.147301+0000","last_fullsized":"2023-09-23T20:31:49.147301+0000","mapping_epoch":258,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":259,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:02:40.594828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":300,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T20:31:49.145802+0000","last_change":"2023-09-23T20:29:45.690815+0000","last_active":"2023-09-23T20:31:49.145802+0000","last_peered":"2023-09-23T20:31:49.145802+0000","last_clean":"2023-09-23T20:31:49.145802+0000","last_became_active":"2023-09-23T20:29:45.690478+0000","last_became_peered":"2023-09-23T20:29:45.690478+0000","last_unstale":"2023-09-23T20:31:49.145802+0000","last_undegraded":"2023-09-23T20:31:49.145802+0000","last_fullsized":"2023-09-23T20:31:49.145802+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:38.288193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.0","version":"0'0","reported_seq":10,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T20:31:49.157420+0000","last_change":"2023-09-23T20:31:49.157205+0000","last_active":"2023-09-23T20:31:49.157420+0000","last_peered":"2023-09-23T20:31:49.157420+0000","last_clean":"2023-09-23T20:31:49.157420+0000","last_became_active":"2023-09-23T20:31:49.156922+0000","last_became_peered":"2023-09-23T20:31:49.156922+0000","last_unstale":"2023-09-23T20:31:49.157420+0000","last_undegraded":"2023-09-23T20:31:49.157420+0000","last_fullsized":"2023-09-23T20:31:49.157420+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:51:36.120059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":527,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T20:31:49.150079+0000","last_change":"2023-09-23T20:31:44.211308+0000","last_active":"2023-09-23T20:31:49.150079+0000","last_peered":"2023-09-23T20:31:49.150079+0000","last_clean":"2023-09-23T20:31:49.150079+0000","last_became_active":"2023-09-23T20:29:07.113428+0000","last_became_peered":"2023-09-23T20:29:07.113428+0000","last_unstale":"2023-09-23T20:31:49.150079+0000","last_undegraded":"2023-09-23T20:31:49.150079+0000","last_fullsized":"2023-09-23T20:31:49.150079+0000","mapping_epoch":253,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":254,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:44.211184+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:44.211184+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:53:50.053167+0000","scrub_duration":0.0017707720000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"86'102","reported_seq":860,"reported_epoch":303,"state":"active+clean","last_fresh":"2023-09-23T20:31:48.151439+0000","last_change":"2023-09-23T20:31:32.809213+0000","last_active":"2023-09-23T20:31:48.151439+0000","last_peered":"2023-09-23T20:31:48.151439+0000","last_clean":"2023-09-23T20:31:48.151439+0000","last_became_active":"2023-09-23T20:31:32.808744+0000","last_became_peered":"2023-09-23T20:31:32.808744+0000","last_unstale":"2023-09-23T20:31:48.151439+0000","last_undegraded":"2023-09-23T20:31:48.151439+0000","last_fullsized":"2023-09-23T20:31:48.151439+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":102,"log_dups_size":0,"ondisk_log_size":102,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:17:25.506476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030656500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":963,"num_read_kb":1234,"num_write":60,"num_write_kb":225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":228,"reported_epoch":305,"state":"active+clean","last_fresh":"2023-09-23T20:31:51.859567+0000","last_change":"2023-09-23T20:29:45.538283+0000","last_active":"2023-09-23T20:31:51.859567+0000","last_peered":"2023-09-23T20:31:51.859567+0000","last_clean":"2023-09-23T20:31:51.859567+0000","last_became_active":"2023-09-23T20:29:45.537922+0000","last_became_peered":"2023-09-23T20:29:45.537922+0000","last_unstale":"2023-09-23T20:31:51.859567+0000","last_undegraded":"2023-09-23T20:31:51.859567+0000","last_fullsized":"2023-09-23T20:31:51.859567+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:30:52.397836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":237,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T20:31:49.145387+0000","last_change":"2023-09-23T20:20:14.888874+0000","last_active":"2023-09-23T20:31:49.145387+0000","last_peered":"2023-09-23T20:31:49.145387+0000","last_clean":"2023-09-23T20:31:49.145387+0000","last_became_active":"2023-09-23T20:20:14.887724+0000","last_became_peered":"2023-09-23T20:20:14.887724+0000","last_unstale":"2023-09-23T20:31:49.145387+0000","last_undegraded":"2023-09-23T20:31:49.145387+0000","last_fullsized":"2023-09-23T20:31:49.145387+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:09:24.510326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"0'0","reported_seq":11,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T20:31:49.157727+0000","last_change":"2023-09-23T20:31:49.157359+0000","last_active":"2023-09-23T20:31:49.157727+0000","last_peered":"2023-09-23T20:31:49.157727+0000","last_clean":"2023-09-23T20:31:49.157727+0000","last_became_active":"2023-09-23T20:31:49.157061+0000","last_became_peered":"2023-09-23T20:31:49.157061+0000","last_unstale":"2023-09-23T20:31:49.157727+0000","last_undegraded":"2023-09-23T20:31:49.157727+0000","last_fullsized":"2023-09-23T20:31:49.157727+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:28.957094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":487,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T20:31:49.147477+0000","last_change":"2023-09-23T20:31:44.509200+0000","last_active":"2023-09-23T20:31:49.147477+0000","last_peered":"2023-09-23T20:31:49.147477+0000","last_clean":"2023-09-23T20:31:49.147477+0000","last_became_active":"2023-09-23T20:29:28.372652+0000","last_became_peered":"2023-09-23T20:29:28.372652+0000","last_unstale":"2023-09-23T20:31:49.147477+0000","last_undegraded":"2023-09-23T20:31:49.147477+0000","last_fullsized":"2023-09-23T20:31:49.147477+0000","mapping_epoch":258,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":259,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:44.509012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:44.509012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:15:44.777003+0000","scrub_duration":0.0032186319999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,5],"acting":[2,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":157,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T20:31:49.159921+0000","last_change":"2023-09-23T20:25:03.007736+0000","last_active":"2023-09-23T20:31:49.159921+0000","last_peered":"2023-09-23T20:31:49.159921+0000","last_clean":"2023-09-23T20:31:49.159921+0000","last_became_active":"2023-09-23T20:25:02.973689+0000","last_became_peered":"2023-09-23T20:25:02.973689+0000","last_unstale":"2023-09-23T20:31:49.159921+0000","last_undegraded":"2023-09-23T20:31:49.159921+0000","last_fullsized":"2023-09-23T20:31:49.159921+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:37:25.643774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"86'128","reported_seq":660,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T20:31:49.147847+0000","last_change":"2023-09-23T20:30:45.008506+0000","last_active":"2023-09-23T20:31:49.147847+0000","last_peered":"2023-09-23T20:31:49.147847+0000","last_clean":"2023-09-23T20:31:49.147847+0000","last_became_active":"2023-09-23T20:29:46.071848+0000","last_became_peered":"2023-09-23T20:29:46.071848+0000","last_unstale":"2023-09-23T20:31:49.147847+0000","last_undegraded":"2023-09-23T20:31:49.147847+0000","last_fullsized":"2023-09-23T20:31:49.147847+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":283,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:44:18.031530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000293202,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":579,"num_read_kb":1473,"num_write":96,"num_write_kb":1062,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":153,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T20:31:49.150960+0000","last_change":"2023-09-23T20:25:02.971740+0000","last_active":"2023-09-23T20:31:49.150960+0000","last_peered":"2023-09-23T20:31:49.150960+0000","last_clean":"2023-09-23T20:31:49.150960+0000","last_became_active":"2023-09-23T20:25:02.970412+0000","last_became_peered":"2023-09-23T20:25:02.970412+0000","last_unstale":"2023-09-23T20:31:49.150960+0000","last_undegraded":"2023-09-23T20:31:49.150960+0000","last_fullsized":"2023-09-23T20:31:49.150960+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:53:17.247216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"0'0","reported_seq":11,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T20:31:49.160155+0000","last_change":"2023-09-23T20:31:49.159750+0000","last_active":"2023-09-23T20:31:49.160155+0000","last_peered":"2023-09-23T20:31:49.160155+0000","last_clean":"2023-09-23T20:31:49.160155+0000","last_became_active":"2023-09-23T20:31:49.159158+0000","last_became_peered":"2023-09-23T20:31:49.159158+0000","last_unstale":"2023-09-23T20:31:49.160155+0000","last_undegraded":"2023-09-23T20:31:49.160155+0000","last_fullsized":"2023-09-23T20:31:49.160155+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:23:19.699225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"90'161","reported_seq":553,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T20:31:49.158563+0000","last_change":"2023-09-23T20:28:03.553294+0000","last_active":"2023-09-23T20:31:49.158563+0000","last_peered":"2023-09-23T20:31:49.158563+0000","last_clean":"2023-09-23T20:31:49.158563+0000","last_became_active":"2023-09-23T20:27:55.212201+0000","last_became_peered":"2023-09-23T20:27:55.212201+0000","last_unstale":"2023-09-23T20:31:49.158563+0000","last_undegraded":"2023-09-23T20:31:49.158563+0000","last_fullsized":"2023-09-23T20:31:49.158563+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:28:03.553106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:28:03.553106+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:24:06.706940+0000","scrub_duration":0.011176864,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"86'117","reported_seq":574,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T20:31:49.150196+0000","last_change":"2023-09-23T20:31:32.814392+0000","last_active":"2023-09-23T20:31:49.150196+0000","last_peered":"2023-09-23T20:31:49.150196+0000","last_clean":"2023-09-23T20:31:49.150196+0000","last_became_active":"2023-09-23T20:31:32.813857+0000","last_became_peered":"2023-09-23T20:31:32.813857+0000","last_unstale":"2023-09-23T20:31:49.150196+0000","last_undegraded":"2023-09-23T20:31:49.150196+0000","last_fullsized":"2023-09-23T20:31:49.150196+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:56:57.863005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046891400000000002,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":893,"num_read_kb":922,"num_write":82,"num_write_kb":4193,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":287,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T20:31:49.145886+0000","last_change":"2023-09-23T20:29:46.053198+0000","last_active":"2023-09-23T20:31:49.145886+0000","last_peered":"2023-09-23T20:31:49.145886+0000","last_clean":"2023-09-23T20:31:49.145886+0000","last_became_active":"2023-09-23T20:29:46.052866+0000","last_became_peered":"2023-09-23T20:29:46.052866+0000","last_unstale":"2023-09-23T20:31:49.145886+0000","last_undegraded":"2023-09-23T20:31:49.145886+0000","last_fullsized":"2023-09-23T20:31:49.145886+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:30:18.857140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":262,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T20:31:49.147152+0000","last_change":"2023-09-23T20:29:28.367722+0000","last_active":"2023-09-23T20:31:49.147152+0000","last_peered":"2023-09-23T20:31:49.147152+0000","last_clean":"2023-09-23T20:31:49.147152+0000","last_became_active":"2023-09-23T20:29:28.367404+0000","last_became_peered":"2023-09-23T20:29:28.367404+0000","last_unstale":"2023-09-23T20:31:49.147152+0000","last_undegraded":"2023-09-23T20:31:49.147152+0000","last_fullsized":"2023-09-23T20:31:49.147152+0000","mapping_epoch":258,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":259,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:02:51.886571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":259,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T20:31:49.145299+0000","last_change":"2023-09-23T20:29:46.061960+0000","last_active":"2023-09-23T20:31:49.145299+0000","last_peered":"2023-09-23T20:31:49.145299+0000","last_clean":"2023-09-23T20:31:49.145299+0000","last_became_active":"2023-09-23T20:29:46.061601+0000","last_became_peered":"2023-09-23T20:29:46.061601+0000","last_unstale":"2023-09-23T20:31:49.145299+0000","last_undegraded":"2023-09-23T20:31:49.145299+0000","last_fullsized":"2023-09-23T20:31:49.145299+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:15:52.620692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":156,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T20:31:49.147299+0000","last_change":"2023-09-23T20:29:45.538314+0000","last_active":"2023-09-23T20:31:49.147299+0000","last_peered":"2023-09-23T20:31:49.147299+0000","last_clean":"2023-09-23T20:31:49.147299+0000","last_became_active":"2023-09-23T20:29:45.536847+0000","last_became_peered":"2023-09-23T20:29:45.536847+0000","last_unstale":"2023-09-23T20:31:49.147299+0000","last_undegraded":"2023-09-23T20:31:49.147299+0000","last_fullsized":"2023-09-23T20:31:49.147299+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:51.948721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":300,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T20:31:49.147471+0000","last_change":"2023-09-23T20:29:59.578082+0000","last_active":"2023-09-23T20:31:49.147471+0000","last_peered":"2023-09-23T20:31:49.147471+0000","last_clean":"2023-09-23T20:31:49.147471+0000","last_became_active":"2023-09-23T20:29:46.072626+0000","last_became_peered":"2023-09-23T20:29:46.072626+0000","last_unstale":"2023-09-23T20:31:49.147471+0000","last_undegraded":"2023-09-23T20:31:49.147471+0000","last_fullsized":"2023-09-23T20:31:49.147471+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:24:14.704823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":312,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T20:31:49.157909+0000","last_change":"2023-09-23T20:29:46.070732+0000","last_active":"2023-09-23T20:31:49.157909+0000","last_peered":"2023-09-23T20:31:49.157909+0000","last_clean":"2023-09-23T20:31:49.157909+0000","last_became_active":"2023-09-23T20:29:46.069861+0000","last_became_peered":"2023-09-23T20:29:46.069861+0000","last_unstale":"2023-09-23T20:31:49.157909+0000","last_undegraded":"2023-09-23T20:31:49.157909+0000","last_fullsized":"2023-09-23T20:31:49.157909+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:33:00.813261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":308,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T20:31:49.147664+0000","last_change":"2023-09-23T20:29:46.242242+0000","last_active":"2023-09-23T20:31:49.147664+0000","last_peered":"2023-09-23T20:31:49.147664+0000","last_clean":"2023-09-23T20:31:49.147664+0000","last_became_active":"2023-09-23T20:29:46.241216+0000","last_became_peered":"2023-09-23T20:29:46.241216+0000","last_unstale":"2023-09-23T20:31:49.147664+0000","last_undegraded":"2023-09-23T20:31:49.147664+0000","last_fullsized":"2023-09-23T20:31:49.147664+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:12:48.530720+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":220,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T20:31:49.157771+0000","last_change":"2023-09-23T20:29:28.353700+0000","last_active":"2023-09-23T20:31:49.157771+0000","last_peered":"2023-09-23T20:31:49.157771+0000","last_clean":"2023-09-23T20:31:49.157771+0000","last_became_active":"2023-09-23T20:29:28.353357+0000","last_became_peered":"2023-09-23T20:29:28.353357+0000","last_unstale":"2023-09-23T20:31:49.157771+0000","last_undegraded":"2023-09-23T20:31:49.157771+0000","last_fullsized":"2023-09-23T20:31:49.157771+0000","mapping_epoch":258,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":259,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:12.077050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":186,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T20:31:49.158902+0000","last_change":"2023-09-23T20:29:04.138540+0000","last_active":"2023-09-23T20:31:49.158902+0000","last_peered":"2023-09-23T20:31:49.158902+0000","last_clean":"2023-09-23T20:31:49.158902+0000","last_became_active":"2023-09-23T20:29:04.138182+0000","last_became_peered":"2023-09-23T20:29:04.138182+0000","last_unstale":"2023-09-23T20:31:49.158902+0000","last_undegraded":"2023-09-23T20:31:49.158902+0000","last_fullsized":"2023-09-23T20:31:49.158902+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:26.098284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":161,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T20:31:49.150535+0000","last_change":"2023-09-23T20:25:39.622819+0000","last_active":"2023-09-23T20:31:49.150535+0000","last_peered":"2023-09-23T20:31:49.150535+0000","last_clean":"2023-09-23T20:31:49.150535+0000","last_became_active":"2023-09-23T20:25:39.622484+0000","last_became_peered":"2023-09-23T20:25:39.622484+0000","last_unstale":"2023-09-23T20:31:49.150535+0000","last_undegraded":"2023-09-23T20:31:49.150535+0000","last_fullsized":"2023-09-23T20:31:49.150535+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:51:53.973261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":285,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T20:31:49.145335+0000","last_change":"2023-09-23T20:25:39.668882+0000","last_active":"2023-09-23T20:31:49.145335+0000","last_peered":"2023-09-23T20:31:49.145335+0000","last_clean":"2023-09-23T20:31:49.145335+0000","last_became_active":"2023-09-23T20:25:39.637128+0000","last_became_peered":"2023-09-23T20:25:39.637128+0000","last_unstale":"2023-09-23T20:31:49.145335+0000","last_undegraded":"2023-09-23T20:31:49.145335+0000","last_fullsized":"2023-09-23T20:31:49.145335+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:56.406298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":320,"reported_epoch":304,"state":"active+clean","last_fresh":"2023-09-23T20:31:49.149228+0000","last_change":"2023-09-23T20:29:46.082412+0000","last_active":"2023-09-23T20:31:49.149228+0000","last_peered":"2023-09-23T20:31:49.149228+0000","last_clean":"2023-09-23T20:31:49.149228+0000","last_became_active":"2023-09-23T20:29:46.082028+0000","last_became_peered":"2023-09-23T20:29:46.082028+0000","last_unstale":"2023-09-23T20:31:49.149228+0000","last_undegraded":"2023-09-23T20:31:49.149228+0000","last_fullsized":"2023-09-23T20:31:49.149228+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:48:07.017969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":48,"acting":48,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":6376,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":122880,"data_stored":21392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26691,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2109440,"data_stored":2098784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3381,"ondisk_log_size":3381,"up":63,"acting":63,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":43,"num_bytes_recovered":4194342,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6082,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201528,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":518084,"kb_used_data":292092,"kb_used_omap":0,"kb_used_meta":225984,"kb_avail":93227068,"statfs":{"total":95995035648,"available":95464517632,"internally_reserved":0,"allocated":299102208,"data_stored":298132667,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":231407616},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:31:35 2023","interfaces":[{"interface":"back","average":{"1min":2.0049999999999999,"5min":1.988,"15min":1.2589999999999999},"min":{"1min":0.25700000000000001,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":8.0660000000000007,"5min":48.798999999999999,"15min":48.798999999999999},"last":0.38400000000000001},{"interface":"front","average":{"1min":2.0459999999999998,"5min":2.1230000000000002,"15min":1.4279999999999999},"min":{"1min":0.27300000000000002,"5min":0.251,"15min":0.24399999999999999},"max":{"1min":7.968,"5min":48.247,"15min":48.247},"last":0.34599999999999997}]},{"osd":1,"last update":"Sat Sep 23 20:30:47 2023","interfaces":[{"interface":"back","average":{"1min":1.762,"5min":1.762,"15min":1.762},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":9.7370000000000001,"5min":9.7370000000000001,"15min":9.7370000000000001},"last":0.499},{"interface":"front","average":{"1min":1.9810000000000001,"5min":1.9810000000000001,"15min":1.9810000000000001},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":5.7679999999999998,"5min":5.7679999999999998,"15min":5.7679999999999998},"last":0.47599999999999998}]},{"osd":2,"last update":"Sat Sep 23 20:31:38 2023","interfaces":[{"interface":"back","average":{"1min":1.478,"5min":1.669,"15min":1.282},"min":{"1min":0.29199999999999998,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":6.7220000000000004,"5min":11.372999999999999,"15min":11.372999999999999},"last":0.437},{"interface":"front","average":{"1min":2.7810000000000001,"5min":2.1899999999999999,"15min":1.4950000000000001},"min":{"1min":0.318,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":13.156000000000001,"5min":14.993,"15min":14.993},"last":0.41399999999999998}]},{"osd":3,"last update":"Sat Sep 23 20:31:02 2023","interfaces":[{"interface":"back","average":{"1min":2.5059999999999998,"5min":2.0710000000000002,"15min":1.377},"min":{"1min":0.34000000000000002,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":11.015000000000001,"5min":11.015000000000001,"15min":11.015000000000001},"last":0.45500000000000002},{"interface":"front","average":{"1min":2.7970000000000002,"5min":1.9510000000000001,"15min":1.3},"min":{"1min":0.38,"5min":0.33100000000000002,"15min":0.27600000000000002},"max":{"1min":11.031000000000001,"5min":11.031000000000001,"15min":11.031000000000001},"last":0.57099999999999995}]},{"osd":4,"last update":"Sat Sep 23 20:31:02 2023","interfaces":[{"interface":"back","average":{"1min":0.73699999999999999,"5min":1.583,"15min":1.008},"min":{"1min":0.29399999999999998,"5min":0.27100000000000002,"15min":0.27000000000000002},"max":{"1min":5.7220000000000004,"5min":15.177,"15min":15.177},"last":0.46700000000000003},{"interface":"front","average":{"1min":0.79900000000000004,"5min":1.3560000000000001,"15min":1.127},"min":{"1min":0.28399999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":4.2800000000000002,"5min":11.122,"15min":11.122},"last":0.51000000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:31:02 2023","interfaces":[{"interface":"back","average":{"1min":0.76200000000000001,"5min":1.4159999999999999,"15min":1.1040000000000001},"min":{"1min":0.30599999999999999,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":2.8730000000000002,"5min":13.241,"15min":13.241},"last":0.48799999999999999},{"interface":"front","average":{"1min":0.79400000000000004,"5min":1.7230000000000001,"15min":1.2070000000000001},"min":{"1min":0.32200000000000001,"5min":0.315,"15min":0.315},"max":{"1min":5.7869999999999999,"5min":18.831,"15min":18.831},"last":0.42499999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:31:02 2023","interfaces":[{"interface":"back","average":{"1min":0.82399999999999995,"5min":1.5580000000000001,"15min":1.1060000000000001},"min":{"1min":0.34399999999999997,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":5.7629999999999999,"5min":12.298999999999999,"15min":13.185},"last":0.52300000000000002},{"interface":"front","average":{"1min":0.93600000000000005,"5min":1.302,"15min":1.048},"min":{"1min":0.36799999999999999,"5min":0.35399999999999998,"15min":0.29899999999999999},"max":{"1min":4.4160000000000004,"5min":8.0069999999999997,"15min":9.9120000000000008},"last":0.54200000000000004}]}]},{"osd":6,"up_from":47,"seq":201863463165,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":230600,"kb_used_data":22464,"kb_used_omap":0,"kb_used_meta":208128,"kb_avail":93514552,"statfs":{"total":95995035648,"available":95758901248,"internally_reserved":0,"allocated":23003136,"data_stored":22037437,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":213123072},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:31:35 2023","interfaces":[{"interface":"back","average":{"1min":1.246,"5min":1.3169999999999999,"15min":1.0409999999999999},"min":{"1min":0.24099999999999999,"5min":0.23300000000000001,"15min":0.221},"max":{"1min":5.9420000000000002,"5min":8.4209999999999994,"15min":8.4209999999999994},"last":0.61699999999999999},{"interface":"front","average":{"1min":1.7090000000000001,"5min":1.8759999999999999,"15min":1.3080000000000001},"min":{"1min":0.26900000000000002,"5min":0.25,"15min":0.25},"max":{"1min":8.8729999999999993,"5min":8.8729999999999993,"15min":12.512},"last":0.42299999999999999}]},{"osd":1,"last update":"Sat Sep 23 20:31:48 2023","interfaces":[{"interface":"back","average":{"1min":1.8440000000000001,"5min":2.569,"15min":2.6899999999999999},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":9.1790000000000003,"5min":14.1,"15min":14.1},"last":0.40300000000000002},{"interface":"front","average":{"1min":1.627,"5min":3.1150000000000002,"15min":3.363},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":6.1159999999999997,"5min":14.266999999999999,"15min":14.266999999999999},"last":0.93899999999999995}]},{"osd":2,"last update":"Sat Sep 23 20:31:35 2023","interfaces":[{"interface":"back","average":{"1min":2.2160000000000002,"5min":2.2559999999999998,"15min":1.4119999999999999},"min":{"1min":0.33000000000000002,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":10.138999999999999,"5min":10.631,"15min":13.856},"last":0.46500000000000002},{"interface":"front","average":{"1min":2.4390000000000001,"5min":2.1299999999999999,"15min":1.341},"min":{"1min":0.41099999999999998,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":10.151999999999999,"5min":10.151999999999999,"15min":10.151999999999999},"last":0.91400000000000003}]},{"osd":3,"last update":"Sat Sep 23 20:31:11 2023","interfaces":[{"interface":"back","average":{"1min":2.9420000000000002,"5min":2.0099999999999998,"15min":1.3859999999999999},"min":{"1min":0.51600000000000001,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":6.1829999999999998,"5min":8.2989999999999995,"15min":9.4239999999999995},"last":0.55000000000000004},{"interface":"front","average":{"1min":3.0910000000000002,"5min":2.21,"15min":1.4330000000000001},"min":{"1min":0.40600000000000003,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":7.6609999999999996,"5min":8.3770000000000007,"15min":8.6050000000000004},"last":0.51200000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:30:53 2023","interfaces":[{"interface":"back","average":{"1min":0.44,"5min":1.0349999999999999,"15min":0.85299999999999998},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":1.599,"5min":10.173,"15min":10.173},"last":0.83899999999999997},{"interface":"front","average":{"1min":1.032,"5min":1.3779999999999999,"15min":0.98199999999999998},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":3.363,"5min":8.3249999999999993,"15min":8.3249999999999993},"last":0.59199999999999997}]},{"osd":5,"last update":"Sat Sep 23 20:30:53 2023","interfaces":[{"interface":"back","average":{"1min":0.46400000000000002,"5min":0.91100000000000003,"15min":0.89800000000000002},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":1.2010000000000001,"5min":8.4499999999999993,"15min":14.085000000000001},"last":0.80800000000000005},{"interface":"front","average":{"1min":0.98099999999999998,"5min":1.76,"15min":1.236},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.29499999999999998},"max":{"1min":3.294,"5min":18.192,"15min":18.192},"last":0.63600000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:31:11 2023","interfaces":[{"interface":"back","average":{"1min":0.45300000000000001,"5min":1.27,"15min":1.0309999999999999},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.754,"5min":8.5060000000000002,"15min":10.217000000000001},"last":0.88800000000000001},{"interface":"front","average":{"1min":1.0669999999999999,"5min":1.508,"15min":1.163},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":6.4160000000000004,"5min":8.407,"15min":9.7360000000000007},"last":0.65800000000000003}]}]},{"osd":1,"up_from":266,"seq":1142461300763,"num_pgs":58,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":512632,"kb_used_data":378672,"kb_used_omap":8,"kb_used_meta":133943,"kb_avail":93232520,"statfs":{"total":95995035648,"available":95470100480,"internally_reserved":0,"allocated":387760128,"data_stored":386109263,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8206,"internal_metadata":137158642},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:31:48 2023","interfaces":[{"interface":"back","average":{"1min":2.3359999999999999,"5min":2.331,"15min":2.3300000000000001},"min":{"1min":0.218,"5min":0.218,"15min":0.218},"max":{"1min":12.403,"5min":12.403,"15min":12.403},"last":0.45100000000000001},{"interface":"front","average":{"1min":1.5149999999999999,"5min":2.859,"15min":3.0830000000000002},"min":{"1min":0.217,"5min":0.217,"15min":0.217},"max":{"1min":8.0150000000000006,"5min":9.7129999999999992,"15min":9.7129999999999992},"last":0.28999999999999998}]},{"osd":2,"last update":"Sat Sep 23 20:31:48 2023","interfaces":[{"interface":"back","average":{"1min":2.4140000000000001,"5min":4.5819999999999999,"15min":4.944},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":11.066000000000001,"5min":16.151,"15min":16.151},"last":0.30399999999999999},{"interface":"front","average":{"1min":1.9590000000000001,"5min":2.0670000000000002,"15min":2.085},"min":{"1min":0.30099999999999999,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":12.356,"5min":12.356,"15min":12.356},"last":0.36799999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:31:48 2023","interfaces":[{"interface":"back","average":{"1min":2.1259999999999999,"5min":3.5179999999999998,"15min":3.75},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":10.811,"5min":10.811,"15min":10.811},"last":0.318},{"interface":"front","average":{"1min":1.538,"5min":4.1840000000000002,"15min":4.625},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":7.665,"5min":12.012,"15min":12.012},"last":0.38400000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:31:48 2023","interfaces":[{"interface":"back","average":{"1min":1.9710000000000001,"5min":2.0569999999999999,"15min":2.0710000000000002},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":6.6980000000000004,"5min":9.4429999999999996,"15min":9.4429999999999996},"last":0.66900000000000004},{"interface":"front","average":{"1min":1.2350000000000001,"5min":1.8660000000000001,"15min":1.9710000000000001},"min":{"1min":0.38200000000000001,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":10.760999999999999,"5min":10.760999999999999,"15min":10.760999999999999},"last":0.53200000000000003}]},{"osd":5,"last update":"Sat Sep 23 20:31:48 2023","interfaces":[{"interface":"back","average":{"1min":1.2370000000000001,"5min":2.4889999999999999,"15min":2.6970000000000001},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":6.7720000000000002,"5min":9.6479999999999997,"15min":9.6479999999999997},"last":0.51700000000000002},{"interface":"front","average":{"1min":2.4900000000000002,"5min":2.1699999999999999,"15min":2.1160000000000001},"min":{"1min":0.41699999999999998,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":9.1959999999999997,"5min":9.4700000000000006,"15min":9.4700000000000006},"last":0.63700000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:31:48 2023","interfaces":[{"interface":"back","average":{"1min":1.6259999999999999,"5min":6.2839999999999998,"15min":7.0599999999999996},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":7.5679999999999996,"5min":72.682000000000002,"15min":72.682000000000002},"last":0.55500000000000005},{"interface":"front","average":{"1min":2.468,"5min":5.6449999999999996,"15min":6.1749999999999998},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":9.6460000000000008,"5min":72.563999999999993,"15min":72.563999999999993},"last":0.57099999999999995}]},{"osd":7,"last update":"Sat Sep 23 20:31:48 2023","interfaces":[{"interface":"back","average":{"1min":1.157,"5min":2.8570000000000002,"15min":3.141},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":4.8380000000000001,"5min":9.6850000000000005,"15min":9.6850000000000005},"last":0.59299999999999997},{"interface":"front","average":{"1min":2.5129999999999999,"5min":2.3900000000000001,"15min":2.3700000000000001},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":9.2349999999999994,"5min":9.3949999999999996,"15min":9.3949999999999996},"last":0.60599999999999998}]}]},{"osd":0,"up_from":94,"seq":403726925966,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":351212,"kb_used_data":4068,"kb_used_omap":7,"kb_used_meta":347128,"kb_avail":93393940,"statfs":{"total":95995035648,"available":95635394560,"internally_reserved":0,"allocated":4165632,"data_stored":2644349,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":355459089},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:31:50 2023","interfaces":[{"interface":"back","average":{"1min":1.3919999999999999,"5min":3.3879999999999999,"15min":3.7200000000000002},"min":{"1min":0.22,"5min":0.17999999999999999,"15min":0.17999999999999999},"max":{"1min":9.0969999999999995,"5min":15.542,"15min":15.542},"last":0.52700000000000002},{"interface":"front","average":{"1min":1.2829999999999999,"5min":3.8220000000000001,"15min":4.2450000000000001},"min":{"1min":0.29399999999999998,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":9.1370000000000005,"5min":10.49,"15min":10.49},"last":0.59699999999999998}]},{"osd":2,"last update":"Sat Sep 23 20:31:30 2023","interfaces":[{"interface":"back","average":{"1min":2.8620000000000001,"5min":3.024,"15min":1.8129999999999999},"min":{"1min":0.25900000000000001,"5min":0.185,"15min":0.185},"max":{"1min":5.9850000000000003,"5min":77.448999999999998,"15min":77.448999999999998},"last":0.61799999999999999},{"interface":"front","average":{"1min":1.6799999999999999,"5min":3.0049999999999999,"15min":1.5860000000000001},"min":{"1min":0.27200000000000002,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":9.1349999999999998,"5min":81.724999999999994,"15min":81.724999999999994},"last":0.57899999999999996}]},{"osd":3,"last update":"Sat Sep 23 20:31:03 2023","interfaces":[{"interface":"back","average":{"1min":7.0750000000000002,"5min":3.6080000000000001,"15min":1.79},"min":{"1min":0.25,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":81.400999999999996,"5min":81.400999999999996,"15min":81.400999999999996},"last":0.67000000000000004},{"interface":"front","average":{"1min":9.3719999999999999,"5min":4.0679999999999996,"15min":1.7649999999999999},"min":{"1min":0.34699999999999998,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":81.137,"5min":81.137,"15min":81.137},"last":0.63900000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:31:30 2023","interfaces":[{"interface":"back","average":{"1min":1.6890000000000001,"5min":1.9039999999999999,"15min":1.482},"min":{"1min":0.30499999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":5.9160000000000004,"5min":9.9339999999999993,"15min":23.876999999999999},"last":0.84699999999999998},{"interface":"front","average":{"1min":1.3420000000000001,"5min":1.661,"15min":1.2869999999999999},"min":{"1min":0.442,"5min":0.312,"15min":0.29299999999999998},"max":{"1min":4.4029999999999996,"5min":10.414,"15min":23.681999999999999},"last":1.054}]},{"osd":5,"last update":"Sat Sep 23 20:31:30 2023","interfaces":[{"interface":"back","average":{"1min":1.4610000000000001,"5min":2.0840000000000001,"15min":1.657},"min":{"1min":0.438,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":5.0780000000000003,"5min":14.289,"15min":24.198},"last":1.087},{"interface":"front","average":{"1min":1.3029999999999999,"5min":2.044,"15min":1.492},"min":{"1min":0.41799999999999998,"5min":0.314,"15min":0.314},"max":{"1min":5.29,"5min":14.938000000000001,"15min":14.938000000000001},"last":1.038}]},{"osd":6,"last update":"Sat Sep 23 20:31:30 2023","interfaces":[{"interface":"back","average":{"1min":1.47,"5min":1.897,"15min":1.3740000000000001},"min":{"1min":0.52400000000000002,"5min":0.318,"15min":0.318},"max":{"1min":4.5620000000000003,"5min":10.441000000000001,"15min":24.408000000000001},"last":1.0029999999999999},{"interface":"front","average":{"1min":1.6180000000000001,"5min":1.79,"15min":1.3260000000000001},"min":{"1min":0.40899999999999997,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":6.109,"5min":9.6769999999999996,"15min":25.271000000000001},"last":0.83499999999999996}]},{"osd":7,"last update":"Sat Sep 23 20:31:30 2023","interfaces":[{"interface":"back","average":{"1min":1.954,"5min":2.1110000000000002,"15min":1.5029999999999999},"min":{"1min":0.52500000000000002,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":9.7309999999999999,"5min":24.614000000000001,"15min":25.359000000000002},"last":0.876},{"interface":"front","average":{"1min":2.2829999999999999,"5min":2.5169999999999999,"15min":1.788},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":5.9550000000000001,"5min":24.669,"15min":24.669},"last":1.0209999999999999}]}]},{"osd":2,"up_from":104,"seq":446676598913,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":708052,"kb_used_data":319692,"kb_used_omap":11,"kb_used_meta":388340,"kb_avail":93037100,"statfs":{"total":95995035648,"available":95269990400,"internally_reserved":0,"allocated":327364608,"data_stored":325859152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11878,"internal_metadata":397660570},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:31:39 2023","interfaces":[{"interface":"back","average":{"1min":2.3359999999999999,"5min":2.1560000000000001,"15min":1.413},"min":{"1min":0.186,"5min":0.17999999999999999,"15min":0.17999999999999999},"max":{"1min":18.876000000000001,"5min":30.765999999999998,"15min":30.765999999999998},"last":0.34899999999999998},{"interface":"front","average":{"1min":2.7130000000000001,"5min":2.4420000000000002,"15min":1.369},"min":{"1min":0.29099999999999998,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":18.684999999999999,"5min":18.684999999999999,"15min":18.684999999999999},"last":0.44900000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:31:49 2023","interfaces":[{"interface":"back","average":{"1min":1.5609999999999999,"5min":4.1929999999999996,"15min":4.6310000000000002},"min":{"1min":0.22900000000000001,"5min":0.221,"15min":0.221},"max":{"1min":7.3230000000000004,"5min":16.280000000000001,"15min":16.280000000000001},"last":0.28100000000000003},{"interface":"front","average":{"1min":1.339,"5min":2.6579999999999999,"15min":2.8780000000000001},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":4.9580000000000002,"5min":30.565000000000001,"15min":30.565000000000001},"last":0.30199999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:31:02 2023","interfaces":[{"interface":"back","average":{"1min":2.8849999999999998,"5min":2.2959999999999998,"15min":1.21},"min":{"1min":0.27800000000000002,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":11.132,"5min":16.331,"15min":16.331},"last":0.46700000000000003},{"interface":"front","average":{"1min":4.2539999999999996,"5min":2.8109999999999999,"15min":1.46},"min":{"1min":0.27300000000000002,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":30.704999999999998,"5min":30.704999999999998,"15min":30.704999999999998},"last":0.42899999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:31:39 2023","interfaces":[{"interface":"back","average":{"1min":1.835,"5min":2.1349999999999998,"15min":1.3169999999999999},"min":{"1min":0.34999999999999998,"5min":0.32800000000000001,"15min":0.28100000000000003},"max":{"1min":5.0860000000000003,"5min":16.797999999999998,"15min":16.797999999999998},"last":0.67400000000000004},{"interface":"front","average":{"1min":1.5640000000000001,"5min":1.998,"15min":1.327},"min":{"1min":0.35399999999999998,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":7.4139999999999997,"5min":14.971,"15min":14.971},"last":0.79200000000000004}]},{"osd":5,"last update":"Sat Sep 23 20:31:39 2023","interfaces":[{"interface":"back","average":{"1min":1.7549999999999999,"5min":3.2970000000000002,"15min":1.8440000000000001},"min":{"1min":0.314,"5min":0.314,"15min":0.309},"max":{"1min":4.992,"5min":95.393000000000001,"15min":95.393000000000001},"last":0.90500000000000003},{"interface":"front","average":{"1min":1.9419999999999999,"5min":3.3170000000000002,"15min":1.8859999999999999},"min":{"1min":0.38,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":8.7509999999999994,"5min":95.414000000000001,"15min":95.414000000000001},"last":0.89200000000000002}]},{"osd":6,"last update":"Sat Sep 23 20:31:39 2023","interfaces":[{"interface":"back","average":{"1min":2.206,"5min":2.3279999999999998,"15min":1.4259999999999999},"min":{"1min":0.35799999999999998,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":13.154999999999999,"5min":15.023999999999999,"15min":15.023999999999999},"last":0.82999999999999996},{"interface":"front","average":{"1min":1.3779999999999999,"5min":2.1000000000000001,"15min":1.452},"min":{"1min":0.40200000000000002,"5min":0.316,"15min":0.316},"max":{"1min":4.7249999999999996,"5min":30.670999999999999,"15min":30.670999999999999},"last":0.77100000000000002}]},{"osd":7,"last update":"Sat Sep 23 20:31:39 2023","interfaces":[{"interface":"back","average":{"1min":1.2210000000000001,"5min":2.1739999999999999,"15min":1.5109999999999999},"min":{"1min":0.44600000000000001,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":4.8300000000000001,"5min":30.741,"15min":30.741},"last":0.87},{"interface":"front","average":{"1min":2.0670000000000002,"5min":2.4079999999999999,"15min":1.5489999999999999},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":10.861000000000001,"5min":16.719000000000001,"15min":16.719000000000001},"last":0.77900000000000003}]}]},{"osd":3,"up_from":109,"seq":468151435387,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":417820,"kb_used_data":4308,"kb_used_omap":4,"kb_used_meta":413499,"kb_avail":93327332,"statfs":{"total":95995035648,"available":95567187968,"internally_reserved":0,"allocated":4411392,"data_stored":2901578,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4514,"internal_metadata":423423582},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:31:05 2023","interfaces":[{"interface":"back","average":{"1min":2.5600000000000001,"5min":2.2909999999999999,"15min":1.4610000000000001},"min":{"1min":0.22500000000000001,"5min":0.186,"15min":0.186},"max":{"1min":6.71,"5min":15.589,"15min":67.305999999999997},"last":0.435},{"interface":"front","average":{"1min":2.839,"5min":1.978,"15min":1.3919999999999999},"min":{"1min":0.19500000000000001,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":12.781000000000001,"5min":13.743,"15min":67.403999999999996},"last":0.41399999999999998}]},{"osd":1,"last update":"Sat Sep 23 20:31:48 2023","interfaces":[{"interface":"back","average":{"1min":2.4100000000000001,"5min":2.952,"15min":3.0419999999999998},"min":{"1min":0.222,"5min":0.222,"15min":0.222},"max":{"1min":13.263,"5min":13.263,"15min":13.263},"last":0.372},{"interface":"front","average":{"1min":1.7230000000000001,"5min":2.9470000000000001,"15min":3.1509999999999998},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":9.9260000000000002,"5min":10.967000000000001,"15min":10.967000000000001},"last":0.311}]},{"osd":2,"last update":"Sat Sep 23 20:31:05 2023","interfaces":[{"interface":"back","average":{"1min":3.1419999999999999,"5min":2.6739999999999999,"15min":1.3959999999999999},"min":{"1min":0.29799999999999999,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":11.253,"5min":24.045000000000002,"15min":24.045000000000002},"last":0.34899999999999998},{"interface":"front","average":{"1min":3.1709999999999998,"5min":2.649,"15min":1.4430000000000001},"min":{"1min":0.27100000000000002,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":19.603000000000002,"5min":19.603000000000002,"15min":19.603000000000002},"last":0.29499999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:31:05 2023","interfaces":[{"interface":"back","average":{"1min":2.331,"5min":2.46,"15min":1.3560000000000001},"min":{"1min":0.34200000000000003,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":6.5670000000000002,"5min":46.042999999999999,"15min":46.042999999999999},"last":0.51700000000000002},{"interface":"front","average":{"1min":3.7120000000000002,"5min":2.9569999999999999,"15min":1.591},"min":{"1min":0.34599999999999997,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":19.635000000000002,"5min":46.579999999999998,"15min":46.579999999999998},"last":0.51000000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:31:05 2023","interfaces":[{"interface":"back","average":{"1min":2.319,"5min":2.5960000000000001,"15min":1.5189999999999999},"min":{"1min":0.45700000000000002,"5min":0.31900000000000001,"15min":0.311},"max":{"1min":6.2370000000000001,"5min":45.292000000000002,"15min":45.292000000000002},"last":0.63300000000000001},{"interface":"front","average":{"1min":2.6139999999999999,"5min":2.5129999999999999,"15min":1.3999999999999999},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.313},"max":{"1min":11.028,"5min":44.679000000000002,"15min":44.679000000000002},"last":0.49299999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:31:05 2023","interfaces":[{"interface":"back","average":{"1min":3.3519999999999999,"5min":2.3410000000000002,"15min":1.4279999999999999},"min":{"1min":0.38500000000000001,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":17.524999999999999,"5min":17.524999999999999,"15min":17.524999999999999},"last":0.53900000000000003},{"interface":"front","average":{"1min":3.0670000000000002,"5min":2.3300000000000001,"15min":1.387},"min":{"1min":0.439,"5min":0.36399999999999999,"15min":0.35499999999999998},"max":{"1min":19.524000000000001,"5min":19.524000000000001,"15min":19.524000000000001},"last":0.45500000000000002}]},{"osd":7,"last update":"Sat Sep 23 20:31:05 2023","interfaces":[{"interface":"back","average":{"1min":4.1749999999999998,"5min":2.8370000000000002,"15min":1.5549999999999999},"min":{"1min":0.40699999999999997,"5min":0.374,"15min":0.34999999999999998},"max":{"1min":19.571999999999999,"5min":19.571999999999999,"15min":19.571999999999999},"last":0.48099999999999998},{"interface":"front","average":{"1min":2.3100000000000001,"5min":2.762,"15min":1.607},"min":{"1min":0.42999999999999999,"5min":0.371,"15min":0.371},"max":{"1min":6.407,"5min":45.387999999999998,"15min":45.387999999999998},"last":0.59299999999999997}]}]},{"osd":4,"up_from":0,"seq":128849019141,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":5,"up_from":39,"seq":167503724802,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":230720,"kb_used_data":3384,"kb_used_omap":0,"kb_used_meta":227328,"kb_avail":93514432,"statfs":{"total":95995035648,"available":95758778368,"internally_reserved":0,"allocated":3465216,"data_stored":2504039,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":232783872},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:31:32 2023","interfaces":[{"interface":"back","average":{"1min":1.3600000000000001,"5min":1.5580000000000001,"15min":1.232},"min":{"1min":0.23799999999999999,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":5.782,"5min":8.1609999999999996,"15min":32.475000000000001},"last":0.48099999999999998},{"interface":"front","average":{"1min":0.93899999999999995,"5min":1.4750000000000001,"15min":1.2250000000000001},"min":{"1min":0.34000000000000002,"5min":0.248,"15min":0.20599999999999999},"max":{"1min":5.1900000000000004,"5min":8.4900000000000002,"15min":32.457000000000001},"last":0.46600000000000003}]},{"osd":1,"last update":"Sat Sep 23 20:31:48 2023","interfaces":[{"interface":"back","average":{"1min":1.2,"5min":1.8839999999999999,"15min":1.998},"min":{"1min":0.30199999999999999,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":6.6310000000000002,"5min":8.5839999999999996,"15min":8.5839999999999996},"last":0.442},{"interface":"front","average":{"1min":0.97999999999999998,"5min":1.444,"15min":1.522},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":4.7039999999999997,"5min":6.7969999999999997,"15min":6.7969999999999997},"last":0.38500000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:31:32 2023","interfaces":[{"interface":"back","average":{"1min":2.2360000000000002,"5min":1.8700000000000001,"15min":1.4370000000000001},"min":{"1min":0.29699999999999999,"5min":0.27500000000000002,"15min":0.26700000000000002},"max":{"1min":8.3439999999999994,"5min":18.472999999999999,"15min":34.514000000000003},"last":0.42799999999999999},{"interface":"front","average":{"1min":2.4590000000000001,"5min":1.8580000000000001,"15min":1.383},"min":{"1min":0.28799999999999998,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":8.3230000000000004,"5min":8.6820000000000004,"15min":32.515000000000001},"last":0.497}]},{"osd":3,"last update":"Sat Sep 23 20:31:05 2023","interfaces":[{"interface":"back","average":{"1min":3.1230000000000002,"5min":2.2109999999999999,"15min":1.3220000000000001},"min":{"1min":0.61799999999999999,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":10.952999999999999,"5min":12.846,"15min":12.846},"last":0.36899999999999999},{"interface":"front","average":{"1min":2.3479999999999999,"5min":1.754,"15min":1.2310000000000001},"min":{"1min":0.32300000000000001,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":7.2060000000000004,"5min":12.005000000000001,"15min":12.005000000000001},"last":0.51200000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:31:26 2023","interfaces":[{"interface":"back","average":{"1min":1.1899999999999999,"5min":1.919,"15min":1.276},"min":{"1min":0.27800000000000002,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":6.282,"5min":28.015999999999998,"15min":28.015999999999998},"last":0.28599999999999998},{"interface":"front","average":{"1min":0.81799999999999995,"5min":1.2749999999999999,"15min":1.04},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.29099999999999998},"max":{"1min":2.427,"5min":12.300000000000001,"15min":12.300000000000001},"last":0.30399999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:31:41 2023","interfaces":[{"interface":"back","average":{"1min":0.877,"5min":1.4450000000000001,"15min":1.177},"min":{"1min":0.33600000000000002,"5min":0.30599999999999999,"15min":0.30499999999999999},"max":{"1min":4.3330000000000002,"5min":12.890000000000001,"15min":12.890000000000001},"last":0.33900000000000002},{"interface":"front","average":{"1min":0.84699999999999998,"5min":1.371,"15min":1.1579999999999999},"min":{"1min":0.39100000000000001,"5min":0.36399999999999999,"15min":0.34499999999999997},"max":{"1min":1.855,"5min":16.265999999999998,"15min":16.265999999999998},"last":0.45300000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:31:05 2023","interfaces":[{"interface":"back","average":{"1min":1.01,"5min":1.8169999999999999,"15min":1.204},"min":{"1min":0.32400000000000001,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":2.77,"5min":36.621000000000002,"15min":36.621000000000002},"last":0.35399999999999998},{"interface":"front","average":{"1min":0.94299999999999995,"5min":1.6759999999999999,"15min":1.103},"min":{"1min":0.33500000000000002,"5min":0.316,"15min":0.315},"max":{"1min":3.6219999999999999,"5min":37.331000000000003,"15min":37.331000000000003},"last":0.40699999999999997}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4722,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1360,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8206,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7156,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3154,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:31:57.146 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-09-23T20:31:57.146 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:31:57.146 INFO:tasks.thrashosds.thrasher:inject_pause on osd.1 2023-09-23T20:31:57.146 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T20:31:57.146 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T20:31:57.147 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 config set filestore_inject_stall 3 2023-09-23T20:31:57.190 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:57.204 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:31:57.258 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:31:57.266 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:57.266 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:31:57.333 INFO:teuthology.orchestra.run.smithi018.stderr:noscrub is set 2023-09-23T20:31:57.367 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:31:57.394 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:57 smithi018 ceph-mon[179647]: pgmap v697: 109 pgs: 109 active+clean; 4.6 MiB data, 2.8 GiB used, 623 GiB / 626 GiB avail 2023-09-23T20:31:57.395 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:57 smithi018 ceph-mon[179647]: from='client.45545 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:31:57.395 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:57 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1752923421' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:31:57.395 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:57 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:31:57.395 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:57 smithi018 ceph-mon[184144]: pgmap v697: 109 pgs: 109 active+clean; 4.6 MiB data, 2.8 GiB used, 623 GiB / 626 GiB avail 2023-09-23T20:31:57.395 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:57 smithi018 ceph-mon[184144]: from='client.45545 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:31:57.395 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:57 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1752923421' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:31:57.395 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:57 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:31:57.436 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:57.597 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2023-09-23T20:31:57.598 INFO:teuthology.orchestra.run.smithi018.stdout: "ops": [ 2023-09-23T20:31:57.598 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:31:57.598 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:1974 7.4 7:27b17a23:::benchmark_data_smithi018_308601_object1973:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e307)", 2023-09-23T20:31:57.598 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:31:57.592454+0000", 2023-09-23T20:31:57.598 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.0026264109999999999, 2023-09-23T20:31:57.598 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.002667563, 2023-09-23T20:31:57.598 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:31:57.598 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:31:57.598 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:31:57.598 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:31:57.598 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:31:57.599 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 1974 2023-09-23T20:31:57.599 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:31:57.599 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:31:57.599 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:31:57.599 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:31:57.599 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:57.592454+0000", 2023-09-23T20:31:57.599 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:31:57.599 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:31:57.599 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:31:57.599 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:31:57.600 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:57.592454+0000", 2023-09-23T20:31:57.600 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:31:57.600 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:31:57.600 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:31:57.600 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:31:57.600 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:57.592453+0000", 2023-09-23T20:31:57.600 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.999999 2023-09-23T20:31:57.600 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:31:57.600 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:31:57.600 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:31:57.600 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:57.592503+0000", 2023-09-23T20:31:57.601 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.9879999999999997e-05 2023-09-23T20:31:57.601 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:31:57.601 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:31:57.601 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:31:57.601 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:57.592513+0000", 2023-09-23T20:31:57.601 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0338e-05 2023-09-23T20:31:57.601 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:31:57.601 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:31:57.601 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:31:57.601 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:57.592521+0000", 2023-09-23T20:31:57.602 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.841e-06 2023-09-23T20:31:57.602 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:31:57.602 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:31:57.602 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:31:57.602 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:57.592584+0000", 2023-09-23T20:31:57.602 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.3227999999999998e-05 2023-09-23T20:31:57.602 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:31:57.602 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:31:57.602 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:31:57.602 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:57.592747+0000", 2023-09-23T20:31:57.603 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00016224999999999999 2023-09-23T20:31:57.603 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:31:57.603 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:31:57.603 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,6", 2023-09-23T20:31:57.603 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:57.592886+0000", 2023-09-23T20:31:57.603 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000139111 2023-09-23T20:31:57.603 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:31:57.603 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:31:57.603 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:31:57.603 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:57.593922+0000", 2023-09-23T20:31:57.603 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00103608 2023-09-23T20:31:57.603 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:31:57.603 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:31:57.604 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:31:57.604 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:57.594127+0000", 2023-09-23T20:31:57.604 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00020485699999999999 2023-09-23T20:31:57.604 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:31:57.604 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:31:57.604 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:31:57.604 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:31:57.604 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:31:57.604 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_repop(client.35700.0:1964 7.5 e307/303 7:a651f274:::benchmark_data_smithi018_308601_object1963:head v 307'111, mlcod=307'109)", 2023-09-23T20:31:57.604 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:31:57.594313+0000", 2023-09-23T20:31:57.604 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.00076741000000000005, 2023-09-23T20:31:57.604 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00091831900000000001, 2023-09-23T20:31:57.605 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:31:57.605 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "started", 2023-09-23T20:31:57.605 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:31:57.605 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:31:57.605 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:31:57.605 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:57.594313+0000", 2023-09-23T20:31:57.605 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:31:57.605 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:31:57.605 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:31:57.605 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:31:57.605 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:57.594313+0000", 2023-09-23T20:31:57.605 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:31:57.606 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:31:57.606 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:31:57.606 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:31:57.606 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:57.594312+0000", 2023-09-23T20:31:57.606 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.999999 2023-09-23T20:31:57.606 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:31:57.606 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:31:57.606 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:31:57.606 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:57.594357+0000", 2023-09-23T20:31:57.606 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.4313000000000001e-05 2023-09-23T20:31:57.606 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:31:57.606 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:31:57.607 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:31:57.607 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:57.594369+0000", 2023-09-23T20:31:57.607 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2554999999999999e-05 2023-09-23T20:31:57.607 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:31:57.607 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:31:57.607 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:31:57.607 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:57.594377+0000", 2023-09-23T20:31:57.607 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.4739999999999999e-06 2023-09-23T20:31:57.607 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:31:57.607 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:31:57.607 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:31:57.607 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:57.594442+0000", 2023-09-23T20:31:57.607 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.5077000000000001e-05 2023-09-23T20:31:57.608 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:31:57.608 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:31:57.608 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:31:57.608 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:57.594463+0000", 2023-09-23T20:31:57.608 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.1172e-05 2023-09-23T20:31:57.608 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:31:57.608 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:31:57.608 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:31:57.608 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:31:57.608 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:31:57.608 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:1977 7.6 7:6f863bcd:::benchmark_data_smithi018_308601_object1976:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e307)", 2023-09-23T20:31:57.608 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:31:57.594330+0000", 2023-09-23T20:31:57.609 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.00075008400000000004, 2023-09-23T20:31:57.609 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00097713399999999995, 2023-09-23T20:31:57.609 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:31:57.609 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:31:57.609 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:31:57.609 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:31:57.609 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:31:57.609 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 1977 2023-09-23T20:31:57.609 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:31:57.609 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:31:57.609 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:31:57.609 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:31:57.610 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:57.594330+0000", 2023-09-23T20:31:57.610 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:31:57.610 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:31:57.610 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:31:57.610 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:31:57.610 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:57.594330+0000", 2023-09-23T20:31:57.610 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:31:57.610 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:31:57.610 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:31:57.610 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:31:57.610 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:57.594329+0000", 2023-09-23T20:31:57.610 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.999999 2023-09-23T20:31:57.611 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:31:57.611 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:31:57.611 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:31:57.611 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:57.594355+0000", 2023-09-23T20:31:57.611 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.6145000000000001e-05 2023-09-23T20:31:57.611 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:31:57.611 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:31:57.611 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:31:57.611 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:57.594371+0000", 2023-09-23T20:31:57.611 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.5068999999999999e-05 2023-09-23T20:31:57.611 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:31:57.612 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:31:57.612 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:31:57.612 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:57.594380+0000", 2023-09-23T20:31:57.612 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.0659999999999993e-06 2023-09-23T20:31:57.612 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:31:57.612 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:31:57.612 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:31:57.612 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:57.594466+0000", 2023-09-23T20:31:57.612 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.6520999999999994e-05 2023-09-23T20:31:57.612 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:31:57.612 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:31:57.612 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:31:57.613 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:57.594597+0000", 2023-09-23T20:31:57.613 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00013056499999999999 2023-09-23T20:31:57.613 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:31:57.613 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:31:57.613 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 3,7", 2023-09-23T20:31:57.613 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:57.594738+0000", 2023-09-23T20:31:57.613 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000141595 2023-09-23T20:31:57.613 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:31:57.613 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:31:57.613 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:31:57.613 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:31:57.613 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:31:57.613 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_repop_reply(client.35700.0:1974 7.4 e307/303)", 2023-09-23T20:31:57.614 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:31:57.594410+0000", 2023-09-23T20:31:57.614 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.00067021400000000003, 2023-09-23T20:31:57.614 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.001008739, 2023-09-23T20:31:57.614 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:31:57.614 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "queued for pg", 2023-09-23T20:31:57.614 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:31:57.614 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:31:57.614 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:31:57.614 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:57.594410+0000", 2023-09-23T20:31:57.614 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:31:57.615 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:31:57.615 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:31:57.615 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:31:57.615 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:57.594410+0000", 2023-09-23T20:31:57.615 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:31:57.615 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:31:57.615 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:31:57.615 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:31:57.615 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:57.594410+0000", 2023-09-23T20:31:57.615 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:31:57.615 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:31:57.616 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:31:57.616 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:31:57.616 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:57.594413+0000", 2023-09-23T20:31:57.616 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.6650000000000001e-06 2023-09-23T20:31:57.616 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:31:57.616 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:31:57.616 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:31:57.616 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:57.594422+0000", 2023-09-23T20:31:57.616 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.2460000000000003e-06 2023-09-23T20:31:57.616 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:31:57.616 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:31:57.616 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:31:57.617 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:57.594427+0000", 2023-09-23T20:31:57.617 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.5079999999999996e-06 2023-09-23T20:31:57.617 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:31:57.617 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:31:57.617 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:31:57.617 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:31:57.617 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:31:57.617 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:1958 7.6 7:64fc2052:::benchmark_data_smithi018_308601_object1957:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e307)", 2023-09-23T20:31:57.617 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:31:57.586876+0000", 2023-09-23T20:31:57.617 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.0082049170000000008, 2023-09-23T20:31:57.617 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0086371959999999998, 2023-09-23T20:31:57.617 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:31:57.618 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:31:57.618 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:31:57.618 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:31:57.618 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:31:57.618 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 1958 2023-09-23T20:31:57.618 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:31:57.618 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:31:57.618 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:31:57.618 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:31:57.618 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:57.586876+0000", 2023-09-23T20:31:57.618 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:31:57.618 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:31:57.619 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:31:57.619 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:31:57.619 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:57.586876+0000", 2023-09-23T20:31:57.619 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:31:57.619 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:31:57.619 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:31:57.619 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:31:57.619 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:57.586875+0000", 2023-09-23T20:31:57.619 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:31:57.619 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:31:57.619 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:31:57.621 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:31:57.621 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:57.586904+0000", 2023-09-23T20:31:57.621 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.8941e-05 2023-09-23T20:31:57.621 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:31:57.621 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:31:57.621 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:31:57.621 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:57.586913+0000", 2023-09-23T20:31:57.621 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.8990000000000007e-06 2023-09-23T20:31:57.621 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:31:57.621 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:31:57.622 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:31:57.622 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:57.586919+0000", 2023-09-23T20:31:57.622 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.2899999999999999e-06 2023-09-23T20:31:57.622 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:31:57.622 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:31:57.622 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:31:57.622 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:57.586965+0000", 2023-09-23T20:31:57.622 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.6397000000000003e-05 2023-09-23T20:31:57.622 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:31:57.622 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:31:57.622 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:31:57.623 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:57.587093+0000", 2023-09-23T20:31:57.623 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000127859 2023-09-23T20:31:57.623 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:31:57.623 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:31:57.623 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 3,7", 2023-09-23T20:31:57.623 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:57.587233+0000", 2023-09-23T20:31:57.623 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00013972799999999999 2023-09-23T20:31:57.623 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:31:57.623 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:31:57.623 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:31:57.623 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:57.589007+0000", 2023-09-23T20:31:57.623 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0017736150000000001 2023-09-23T20:31:57.624 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:31:57.624 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:31:57.624 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:31:57.624 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:57.592365+0000", 2023-09-23T20:31:57.624 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0033581629999999999 2023-09-23T20:31:57.624 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:31:57.624 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:31:57.624 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:31:57.624 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:31:57.624 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:31:57.624 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:1965 7.6 7:65a91b5b:::benchmark_data_smithi018_308601_object1964:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e307)", 2023-09-23T20:31:57.624 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:31:57.589050+0000", 2023-09-23T20:31:57.625 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.006030227, 2023-09-23T20:31:57.625 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0065586139999999999, 2023-09-23T20:31:57.625 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:31:57.625 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:31:57.625 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:31:57.625 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:31:57.625 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:31:57.625 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 1965 2023-09-23T20:31:57.625 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:31:57.625 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:31:57.625 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:31:57.625 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:31:57.626 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:57.589050+0000", 2023-09-23T20:31:57.626 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:31:57.626 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:31:57.626 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:31:57.626 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:31:57.626 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:57.589050+0000", 2023-09-23T20:31:57.626 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:31:57.626 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:31:57.626 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:31:57.626 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:31:57.626 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:57.589050+0000", 2023-09-23T20:31:57.626 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:31:57.626 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:31:57.627 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:31:57.627 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:31:57.627 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:57.589073+0000", 2023-09-23T20:31:57.627 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.3419999999999999e-05 2023-09-23T20:31:57.627 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:31:57.627 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:31:57.627 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:31:57.627 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:57.589082+0000", 2023-09-23T20:31:57.627 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.6619999999999994e-06 2023-09-23T20:31:57.627 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:31:57.627 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:31:57.627 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:31:57.628 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:57.589088+0000", 2023-09-23T20:31:57.628 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.4910000000000002e-06 2023-09-23T20:31:57.628 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:31:57.628 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:31:57.628 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:31:57.628 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:57.589135+0000", 2023-09-23T20:31:57.628 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.6629999999999999e-05 2023-09-23T20:31:57.628 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:31:57.628 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:31:57.628 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:31:57.628 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:57.589265+0000", 2023-09-23T20:31:57.628 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00012995300000000001 2023-09-23T20:31:57.629 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:31:57.629 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:31:57.629 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 3,7", 2023-09-23T20:31:57.629 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:57.589398+0000", 2023-09-23T20:31:57.629 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00013282800000000001 2023-09-23T20:31:57.629 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:31:57.629 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:31:57.629 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:31:57.629 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:57.590371+0000", 2023-09-23T20:31:57.629 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00097293 2023-09-23T20:31:57.629 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:31:57.629 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:31:57.630 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:31:57.630 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:57.593524+0000", 2023-09-23T20:31:57.630 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0031530450000000002 2023-09-23T20:31:57.630 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:31:57.630 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:31:57.630 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:31:57.630 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:31:57.630 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:31:57.630 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:1967 7.6 7:67ae518d:::benchmark_data_smithi018_308601_object1966:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e307)", 2023-09-23T20:31:57.630 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:31:57.589101+0000", 2023-09-23T20:31:57.630 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.0059792079999999997, 2023-09-23T20:31:57.630 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0066244340000000002, 2023-09-23T20:31:57.631 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:31:57.631 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:31:57.631 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:31:57.631 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:31:57.631 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:31:57.631 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 1967 2023-09-23T20:31:57.631 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:31:57.631 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:31:57.631 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:31:57.631 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:31:57.631 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:57.589101+0000", 2023-09-23T20:31:57.631 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:31:57.632 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:31:57.632 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:31:57.632 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:31:57.632 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:57.589101+0000", 2023-09-23T20:31:57.632 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:31:57.632 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:31:57.632 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:31:57.632 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:31:57.632 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:57.589101+0000", 2023-09-23T20:31:57.632 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:31:57.632 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:31:57.632 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:31:57.633 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:31:57.633 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:57.589122+0000", 2023-09-23T20:31:57.633 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.1675000000000002e-05 2023-09-23T20:31:57.633 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:31:57.633 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:31:57.633 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:31:57.633 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:57.589130+0000", 2023-09-23T20:31:57.633 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.661e-06 2023-09-23T20:31:57.633 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:31:57.633 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:31:57.633 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:31:57.633 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:57.589136+0000", 2023-09-23T20:31:57.634 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.1009999999999997e-06 2023-09-23T20:31:57.634 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:31:57.634 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:31:57.634 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:31:57.634 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:57.589838+0000", 2023-09-23T20:31:57.634 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00070176600000000004 2023-09-23T20:31:57.634 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:31:57.634 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:31:57.634 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:31:57.634 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:57.589950+0000", 2023-09-23T20:31:57.634 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000111958 2023-09-23T20:31:57.634 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:31:57.635 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:31:57.635 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 3,7", 2023-09-23T20:31:57.635 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:57.590075+0000", 2023-09-23T20:31:57.635 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00012473599999999999 2023-09-23T20:31:57.635 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:31:57.635 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:31:57.635 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:31:57.635 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:57.590514+0000", 2023-09-23T20:31:57.635 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00043896000000000001 2023-09-23T20:31:57.635 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:31:57.635 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:31:57.635 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:31:57.636 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:57.593553+0000", 2023-09-23T20:31:57.636 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0030396389999999998 2023-09-23T20:31:57.636 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:31:57.636 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:31:57.636 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:31:57.636 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:31:57.636 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:31:57.636 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:1968 7.f 7:fc65255a:::benchmark_data_smithi018_308601_object1967:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e307)", 2023-09-23T20:31:57.636 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:31:57.589146+0000", 2023-09-23T20:31:57.636 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.0059348020000000003, 2023-09-23T20:31:57.636 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0067159389999999998, 2023-09-23T20:31:57.636 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:31:57.637 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:31:57.637 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:31:57.637 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:31:57.637 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:31:57.637 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 1968 2023-09-23T20:31:57.637 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:31:57.637 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:31:57.637 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:31:57.637 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:31:57.637 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:57.589146+0000", 2023-09-23T20:31:57.637 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:31:57.637 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:31:57.638 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:31:57.638 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:31:57.638 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:57.589146+0000", 2023-09-23T20:31:57.638 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:31:57.638 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:31:57.638 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:31:57.638 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:31:57.638 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:57.589145+0000", 2023-09-23T20:31:57.638 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:31:57.638 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:31:57.638 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:31:57.638 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:31:57.639 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:57.589198+0000", 2023-09-23T20:31:57.639 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.3161999999999997e-05 2023-09-23T20:31:57.639 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:31:57.639 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:31:57.639 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:31:57.639 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:57.589207+0000", 2023-09-23T20:31:57.639 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.2269999999999999e-06 2023-09-23T20:31:57.639 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:31:57.639 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:31:57.639 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:31:57.639 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:57.589214+0000", 2023-09-23T20:31:57.639 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.8839999999999997e-06 2023-09-23T20:31:57.639 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:31:57.640 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:31:57.640 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:31:57.640 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:57.589260+0000", 2023-09-23T20:31:57.640 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.5157999999999997e-05 2023-09-23T20:31:57.640 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:31:57.640 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:31:57.640 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:31:57.640 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:57.589379+0000", 2023-09-23T20:31:57.640 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000119065 2023-09-23T20:31:57.640 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:31:57.640 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:31:57.641 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 3,7", 2023-09-23T20:31:57.641 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:57.589507+0000", 2023-09-23T20:31:57.641 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00012878299999999999 2023-09-23T20:31:57.641 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:31:57.641 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:31:57.641 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:31:57.641 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:57.590507+0000", 2023-09-23T20:31:57.641 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0009992200000000001 2023-09-23T20:31:57.641 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:31:57.641 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:31:57.641 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:31:57.641 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:57.593342+0000", 2023-09-23T20:31:57.641 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0028349299999999998 2023-09-23T20:31:57.642 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:31:57.642 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:31:57.642 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:31:57.642 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:31:57.642 INFO:teuthology.orchestra.run.smithi018.stdout: ], 2023-09-23T20:31:57.642 INFO:teuthology.orchestra.run.smithi018.stdout: "num_ops": 8 2023-09-23T20:31:57.642 INFO:teuthology.orchestra.run.smithi018.stdout:} 2023-09-23T20:31:57.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:57 smithi149 ceph-mon[170394]: pgmap v697: 109 pgs: 109 active+clean; 4.6 MiB data, 2.8 GiB used, 623 GiB / 626 GiB avail 2023-09-23T20:31:57.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:57 smithi149 ceph-mon[170394]: from='client.45545 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:31:57.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:57 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1752923421' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:31:57.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:57 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:31:57.721 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:31:57.725 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:57.726 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:31:57.762 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 1 16 2409 2393 149.554 149.562 0.00552974 0.00666339 2023-09-23T20:31:57.827 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:31:57.858 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:57.899 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:31:57.902 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:31:57.903 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:31:58.003 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:31:58.073 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:31:58.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:58 smithi018 ceph-mon[179647]: failed to encode map e306 with expected crc 2023-09-23T20:31:58.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:58 smithi018 ceph-mon[179647]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:31:58.394 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:58 smithi018 ceph-mon[179647]: failed to encode map e307 with expected crc 2023-09-23T20:31:58.394 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:58 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:31:58.394 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:58 smithi018 ceph-mon[179647]: osdmap e307: 8 total, 8 up, 7 in 2023-09-23T20:31:58.394 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:58 smithi018 ceph-mon[179647]: failed to encode map e307 with expected crc 2023-09-23T20:31:58.394 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:58 smithi018 ceph-mon[184144]: failed to encode map e306 with expected crc 2023-09-23T20:31:58.394 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:58 smithi018 ceph-mon[184144]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:31:58.394 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:58 smithi018 ceph-mon[184144]: failed to encode map e307 with expected crc 2023-09-23T20:31:58.394 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:58 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:31:58.394 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:58 smithi018 ceph-mon[184144]: osdmap e307: 8 total, 8 up, 7 in 2023-09-23T20:31:58.395 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:58 smithi018 ceph-mon[184144]: failed to encode map e307 with expected crc 2023-09-23T20:31:58.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:58 smithi149 ceph-mon[170394]: failed to encode map e306 with expected crc 2023-09-23T20:31:58.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:58 smithi149 ceph-mon[170394]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:31:58.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:58 smithi149 ceph-mon[170394]: failed to encode map e307 with expected crc 2023-09-23T20:31:58.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:58 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:31:58.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:58 smithi149 ceph-mon[170394]: osdmap e307: 8 total, 8 up, 7 in 2023-09-23T20:31:58.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:58 smithi149 ceph-mon[170394]: failed to encode map e307 with expected crc 2023-09-23T20:31:58.764 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 2 16 5216 5200 162.331 175.438 0.00290543 0.00615122 2023-09-23T20:31:59.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:31:59 smithi018 ceph-mon[179647]: pgmap v699: 109 pgs: 109 active+clean; 17 MiB data, 2.8 GiB used, 623 GiB / 626 GiB avail; 1.5 MiB/s wr, 19 op/s 2023-09-23T20:31:59.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:31:59 smithi018 ceph-mon[184144]: pgmap v699: 109 pgs: 109 active+clean; 17 MiB data, 2.8 GiB used, 623 GiB / 626 GiB avail; 1.5 MiB/s wr, 19 op/s 2023-09-23T20:31:59.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:31:59 smithi149 ceph-mon[170394]: pgmap v699: 109 pgs: 109 active+clean; 17 MiB data, 2.8 GiB used, 623 GiB / 626 GiB avail; 1.5 MiB/s wr, 19 op/s 2023-09-23T20:31:59.764 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 3 16 8160 8144 169.543 184 0.00770358 0.00588936 2023-09-23T20:31:59.890 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T20:32:00.321 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:00 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]: dispatch 2023-09-23T20:32:00.322 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:00 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [3, 2]}]: dispatch 2023-09-23T20:32:00.322 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:00 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [3, 5]}]: dispatch 2023-09-23T20:32:00.322 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:00 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2023-09-23T20:32:00.322 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:00 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:32:00.322 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:00 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:32:00.474 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:32:00.478 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:00.479 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:32:00.515 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T20:32:00.580 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:32:00.581 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:00 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]: dispatch 2023-09-23T20:32:00.581 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:00 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [3, 2]}]: dispatch 2023-09-23T20:32:00.581 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:00 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [3, 5]}]: dispatch 2023-09-23T20:32:00.581 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:00 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2023-09-23T20:32:00.581 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:00 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:32:00.581 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:00 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:32:00.610 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:00.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:00 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]: dispatch 2023-09-23T20:32:00.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:00 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [3, 2]}]: dispatch 2023-09-23T20:32:00.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:00 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [3, 5]}]: dispatch 2023-09-23T20:32:00.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:00 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2023-09-23T20:32:00.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:00 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:32:00.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:00 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:32:00.753 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:32:00.760 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:00.761 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:32:00.766 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 4 16 10085 10069 157.167 120.312 0.00816257 0.00635471 2023-09-23T20:32:00.861 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:32:00.931 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:00.975 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:32:00.979 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:00.980 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:32:01.080 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:32:01.138 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:01.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:01 smithi018 ceph-mon[184144]: pgmap v700: 109 pgs: 109 active+clean; 101 MiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 12 MiB/s wr, 191 op/s 2023-09-23T20:32:01.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:01 smithi018 ceph-mon[184144]: failed to encode map e308 with expected crc 2023-09-23T20:32:01.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:01 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]': finished 2023-09-23T20:32:01.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:01 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [3, 2]}]': finished 2023-09-23T20:32:01.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:01 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [3, 5]}]': finished 2023-09-23T20:32:01.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:01 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]': finished 2023-09-23T20:32:01.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:01 smithi018 ceph-mon[184144]: osdmap e308: 8 total, 8 up, 7 in 2023-09-23T20:32:01.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:01 smithi018 ceph-mon[179647]: pgmap v700: 109 pgs: 109 active+clean; 101 MiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 12 MiB/s wr, 191 op/s 2023-09-23T20:32:01.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:01 smithi018 ceph-mon[179647]: failed to encode map e308 with expected crc 2023-09-23T20:32:01.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:01 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]': finished 2023-09-23T20:32:01.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:01 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [3, 2]}]': finished 2023-09-23T20:32:01.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:01 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [3, 5]}]': finished 2023-09-23T20:32:01.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:01 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]': finished 2023-09-23T20:32:01.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:01 smithi018 ceph-mon[179647]: osdmap e308: 8 total, 8 up, 7 in 2023-09-23T20:32:01.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:01 smithi149 ceph-mon[170394]: pgmap v700: 109 pgs: 109 active+clean; 101 MiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 12 MiB/s wr, 191 op/s 2023-09-23T20:32:01.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:01 smithi149 ceph-mon[170394]: failed to encode map e308 with expected crc 2023-09-23T20:32:01.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:01 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 2]}]': finished 2023-09-23T20:32:01.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:01 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [3, 2]}]': finished 2023-09-23T20:32:01.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:01 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [3, 5]}]': finished 2023-09-23T20:32:01.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:01 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]': finished 2023-09-23T20:32:01.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:01 smithi149 ceph-mon[170394]: osdmap e308: 8 total, 8 up, 7 in 2023-09-23T20:32:01.766 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 5 16 12571 12555 156.804 155.375 0.00799071 0.00636851 2023-09-23T20:32:02.541 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:32:02.545 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:02.545 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:32:02.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:02 smithi018 ceph-mon[179647]: failed to encode map e309 with expected crc 2023-09-23T20:32:02.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:02 smithi018 ceph-mon[179647]: osdmap e309: 8 total, 8 up, 7 in 2023-09-23T20:32:02.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:02 smithi018 ceph-mon[179647]: failed to encode map e309 with expected crc 2023-09-23T20:32:02.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:02 smithi018 ceph-mon[179647]: failed to encode map e309 with expected crc 2023-09-23T20:32:02.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:02 smithi018 ceph-mon[179647]: failed to encode map e309 with expected crc 2023-09-23T20:32:02.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:02 smithi018 ceph-mon[184144]: failed to encode map e309 with expected crc 2023-09-23T20:32:02.643 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:02 smithi018 ceph-mon[184144]: osdmap e309: 8 total, 8 up, 7 in 2023-09-23T20:32:02.643 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:02 smithi018 ceph-mon[184144]: failed to encode map e309 with expected crc 2023-09-23T20:32:02.643 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:02 smithi018 ceph-mon[184144]: failed to encode map e309 with expected crc 2023-09-23T20:32:02.643 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:02 smithi018 ceph-mon[184144]: failed to encode map e309 with expected crc 2023-09-23T20:32:02.646 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:32:02.676 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:02 smithi149 ceph-mon[170394]: failed to encode map e309 with expected crc 2023-09-23T20:32:02.676 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:02 smithi149 ceph-mon[170394]: osdmap e309: 8 total, 8 up, 7 in 2023-09-23T20:32:02.677 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:02 smithi149 ceph-mon[170394]: failed to encode map e309 with expected crc 2023-09-23T20:32:02.677 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:02 smithi149 ceph-mon[170394]: failed to encode map e309 with expected crc 2023-09-23T20:32:02.677 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:02 smithi149 ceph-mon[170394]: failed to encode map e309 with expected crc 2023-09-23T20:32:02.679 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:02.690 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:32:02.732 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:32:02.735 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:02.735 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:32:02.769 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 6 16 14763 14747 153.448 137 0.0287014 0.00650612 2023-09-23T20:32:02.836 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:32:02.871 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:02.918 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:32:02.921 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:02.922 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:32:03.023 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:32:03.058 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:03.104 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:32:03.108 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:03.109 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:32:03.209 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:32:03.249 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:03.285 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:32:03.288 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:03.288 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:32:03.389 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:32:03.443 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:03.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:03 smithi018 ceph-mon[179647]: pgmap v703: 109 pgs: 1 peering, 108 active+clean; 430 MiB data, 3.7 GiB used, 622 GiB / 626 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-09-23T20:32:03.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:03 smithi018 ceph-mon[179647]: failed to encode map e309 with expected crc 2023-09-23T20:32:03.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:03 smithi018 ceph-mon[184144]: pgmap v703: 109 pgs: 1 peering, 108 active+clean; 430 MiB data, 3.7 GiB used, 622 GiB / 626 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-09-23T20:32:03.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:03 smithi018 ceph-mon[184144]: failed to encode map e309 with expected crc 2023-09-23T20:32:03.660 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:32:03.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:03 smithi149 ceph-mon[170394]: pgmap v703: 109 pgs: 1 peering, 108 active+clean; 430 MiB data, 3.7 GiB used, 622 GiB / 626 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-09-23T20:32:03.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:03 smithi149 ceph-mon[170394]: failed to encode map e309 with expected crc 2023-09-23T20:32:03.768 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 7 15 16972 16957 151.26 138.125 0.00584371 0.00660398 2023-09-23T20:32:03.982 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:32:03.991 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:32:03.992 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:03.992 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:32:04.093 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:32:04.150 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:04.769 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 8 15 19136 19121 149.259 135.25 0.00492328 0.00669209 2023-09-23T20:32:05.350 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:32:05.355 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:32:04 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:32:04] "GET /metrics HTTP/1.1" 200 36492 "" "Prometheus/2.43.0" 2023-09-23T20:32:05.357 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:05.357 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:32:05.458 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:32:05.492 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:05.549 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:32:05.552 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:05.552 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:32:05.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:05 smithi018 ceph-mon[179647]: pgmap v704: 109 pgs: 1 peering, 108 active+clean; 715 MiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 110 MiB/s wr, 1.76k op/s 2023-09-23T20:32:05.643 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:05 smithi018 ceph-mon[184144]: pgmap v704: 109 pgs: 1 peering, 108 active+clean; 715 MiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 110 MiB/s wr, 1.76k op/s 2023-09-23T20:32:05.653 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:32:05.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:05 smithi149 ceph-mon[170394]: pgmap v704: 109 pgs: 1 peering, 108 active+clean; 715 MiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 110 MiB/s wr, 1.76k op/s 2023-09-23T20:32:05.722 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:05.769 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 9 16 21270 21254 147.487 133.312 0.00430258 0.00676309 2023-09-23T20:32:06.631 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:32:06.637 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:06.638 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:32:06.738 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:32:06.770 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 10 16 23008 22992 143.581 108.625 0.0189335 0.00695786 2023-09-23T20:32:06.780 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:06.825 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:32:06.828 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:06.828 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:32:06.929 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:32:06.963 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:07.010 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:32:07.014 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:07.014 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:32:07.115 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:32:07.188 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:07.313 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:32:07.321 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:07.322 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:32:07.374 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:07 smithi018 ceph-mon[179647]: pgmap v705: 109 pgs: 109 active+clean; 1.1 GiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 137 MiB/s wr, 2.19k op/s 2023-09-23T20:32:07.375 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:07 smithi018 ceph-mon[184144]: pgmap v705: 109 pgs: 109 active+clean; 1.1 GiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 137 MiB/s wr, 2.19k op/s 2023-09-23T20:32:07.423 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:32:07.462 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:07.505 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:32:07.512 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:07.513 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:32:07.532 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:32:07.567 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:32:07.632 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:32:07.633 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:32:07.658 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:32:07.702 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:32:07.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:07 smithi149 ceph-mon[170394]: pgmap v705: 109 pgs: 109 active+clean; 1.1 GiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 137 MiB/s wr, 2.19k op/s 2023-09-23T20:32:07.746 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:32:07.746 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:07.772 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 11 16 24609 24593 139.616 100.062 0.0152308 0.00715336 2023-09-23T20:32:07.787 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:32:07.790 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:07.791 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:32:07.797 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:32:07.836 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:32:07.843 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:32:07.843 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:32:07.843 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:32:07.843 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:32:07.843 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:32:07.843 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:32:07.843 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:32:07.843 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:32:07.892 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:32:07.927 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:07.967 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:32:07.971 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:07.971 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:32:08.072 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:32:08.153 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:08.575 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:32:08.585 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:08.586 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:32:08.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:08 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/4255443042' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:32:08.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:08 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/4255443042' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:32:08.686 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:32:08.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:08 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/4255443042' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:32:08.775 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 12 16 26245 26229 136.503 102.25 0.00908723 0.00731862 2023-09-23T20:32:08.775 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:08.913 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2023-09-23T20:32:08.913 INFO:teuthology.orchestra.run.smithi018.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-09-23T20:32:08.913 INFO:teuthology.orchestra.run.smithi018.stdout:} 2023-09-23T20:32:08.957 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:32:08.966 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:08.967 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:32:09.068 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:32:09.143 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:09.535 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:32:09.543 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:09.544 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:32:09.635 INFO:teuthology.orchestra.run.smithi018.stdout:ERROR: (22) Invalid argument 2023-09-23T20:32:09.636 INFO:teuthology.orchestra.run.smithi018.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:32:09.645 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:32:09.668 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:09 smithi018 ceph-mon[184144]: pgmap v706: 109 pgs: 109 active+clean; 1.2 GiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 141 MiB/s wr, 2.26k op/s 2023-09-23T20:32:09.668 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:09 smithi018 ceph-mon[179647]: pgmap v706: 109 pgs: 109 active+clean; 1.2 GiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 141 MiB/s wr, 2.26k op/s 2023-09-23T20:32:09.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:09 smithi149 ceph-mon[170394]: pgmap v706: 109 pgs: 109 active+clean; 1.2 GiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 141 MiB/s wr, 2.26k op/s 2023-09-23T20:32:09.716 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:09.775 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 13 16 27825 27809 133.59 98.75 0.0111163 0.00747742 2023-09-23T20:32:09.785 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:32:09.789 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:09.789 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:32:09.890 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:32:09.961 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:10.205 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:32:10.211 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:10.211 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:32:10.268 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T20:32:10.313 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:32:10.342 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:10.394 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:32:10.398 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:10.398 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:32:10.499 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:32:10.602 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:10 smithi018 ceph-mon[184144]: pgmap v707: 109 pgs: 109 active+clean; 1.4 GiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 125 MiB/s wr, 2.00k op/s 2023-09-23T20:32:10.602 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:10 smithi018 ceph-mon[179647]: pgmap v707: 109 pgs: 109 active+clean; 1.4 GiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 125 MiB/s wr, 2.00k op/s 2023-09-23T20:32:10.607 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:10.684 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:32:10.689 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:10.690 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:32:10.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:10 smithi149 ceph-mon[170394]: pgmap v707: 109 pgs: 109 active+clean; 1.4 GiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 125 MiB/s wr, 2.00k op/s 2023-09-23T20:32:10.768 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:32:10.772 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 14 16 29305 29289 130.656 92.5 0.00930236 0.00764791 2023-09-23T20:32:10.791 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:32:10.823 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:10.868 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:32:10.873 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:10.873 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:32:10.974 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:32:11.072 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:11.080 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:32:11.176 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:32:11.182 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:11.182 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:32:11.283 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:32:11.317 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:11.363 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:32:11.367 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:11.367 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:32:11.445 INFO:teuthology.orchestra.run.smithi018.stderr:nodeep-scrub is set 2023-09-23T20:32:11.468 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:32:11.546 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:11.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:11 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1417951683' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:32:11.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:11 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:32:11.772 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 15 16 30801 30785 128.18 93.5 0.00520982 0.00779487 2023-09-23T20:32:11.884 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:11 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1417951683' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:32:11.885 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:11 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:32:11.885 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:11 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1417951683' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:32:11.885 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:11 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:32:12.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:12 smithi149 ceph-mon[170394]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:32:12.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:12 smithi149 ceph-mon[170394]: failed to encode map e310 with expected crc 2023-09-23T20:32:12.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:12 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:32:12.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:12 smithi149 ceph-mon[170394]: osdmap e310: 8 total, 8 up, 7 in 2023-09-23T20:32:12.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:12 smithi149 ceph-mon[170394]: pgmap v709: 109 pgs: 109 active+clean; 1.7 GiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 128 MiB/s wr, 2.05k op/s 2023-09-23T20:32:12.774 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 16 16 32608 32592 127.214 112.938 0.00967634 0.0078552 2023-09-23T20:32:12.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:12 smithi018 ceph-mon[179647]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:32:12.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:12 smithi018 ceph-mon[179647]: failed to encode map e310 with expected crc 2023-09-23T20:32:12.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:12 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:32:12.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:12 smithi018 ceph-mon[179647]: osdmap e310: 8 total, 8 up, 7 in 2023-09-23T20:32:12.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:12 smithi018 ceph-mon[179647]: pgmap v709: 109 pgs: 109 active+clean; 1.7 GiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 128 MiB/s wr, 2.05k op/s 2023-09-23T20:32:12.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:12 smithi018 ceph-mon[184144]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:32:12.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:12 smithi018 ceph-mon[184144]: failed to encode map e310 with expected crc 2023-09-23T20:32:12.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:12 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:32:12.894 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:12 smithi018 ceph-mon[184144]: osdmap e310: 8 total, 8 up, 7 in 2023-09-23T20:32:12.894 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:12 smithi018 ceph-mon[184144]: pgmap v709: 109 pgs: 109 active+clean; 1.7 GiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 128 MiB/s wr, 2.05k op/s 2023-09-23T20:32:13.775 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 17 16 34277 34261 125.867 104.312 0.00998145 0.00793934 2023-09-23T20:32:14.578 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 5, 7, 3, 2, 6] out_osds: [4] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T20:32:14.578 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:32:14.578 INFO:tasks.thrashosds.thrasher:Adding osd 4 2023-09-23T20:32:14.579 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd in 4 2023-09-23T20:32:14.775 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 18 16 35790 35774 124.127 94.5625 0.0111922 0.00804877 2023-09-23T20:32:14.995 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:14 smithi018 ceph-mon[184144]: pgmap v710: 109 pgs: 109 active+clean; 1.8 GiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 118 MiB/s wr, 1.89k op/s 2023-09-23T20:32:14.995 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:14 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:32:14.996 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:14 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:32:14.996 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:14 smithi018 ceph-mon[179647]: pgmap v710: 109 pgs: 109 active+clean; 1.8 GiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 118 MiB/s wr, 1.89k op/s 2023-09-23T20:32:14.997 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:14 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:32:14.997 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:14 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:32:14.997 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:32:14 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:32:14] "GET /metrics HTTP/1.1" 200 36499 "" "Prometheus/2.43.0" 2023-09-23T20:32:15.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:14 smithi149 ceph-mon[170394]: pgmap v710: 109 pgs: 109 active+clean; 1.8 GiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 118 MiB/s wr, 1.89k op/s 2023-09-23T20:32:15.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:14 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:32:15.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:14 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:32:15.724 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:32:15.733 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:15.734 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:32:15.776 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 19 16 37404 37388 122.897 100.875 0.00537001 0.00813111 2023-09-23T20:32:15.834 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:32:15.937 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T20:32:15.958 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:16.434 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:32:16.439 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:16.439 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:32:16.439 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:32:16.540 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:32:16.621 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:16.776 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:2023-09-23T20:32:16.776563+0000 min lat: 0.00108768 max lat: 0.118694 avg lat: 0.0082705 2023-09-23T20:32:16.776 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T20:32:16.776 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 20 16 38708 38692 120.828 81.5 0.00738554 0.0082705 2023-09-23T20:32:17.395 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:17 smithi018 ceph-mon[179647]: pgmap v711: 109 pgs: 109 active+clean; 2.2 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 110 MiB/s wr, 1.75k op/s 2023-09-23T20:32:17.395 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:17 smithi018 ceph-mon[184144]: pgmap v711: 109 pgs: 109 active+clean; 2.2 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 110 MiB/s wr, 1.75k op/s 2023-09-23T20:32:17.418 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:32:17.425 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:17.425 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:32:17.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:17 smithi149 ceph-mon[170394]: pgmap v711: 109 pgs: 109 active+clean; 2.2 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 110 MiB/s wr, 1.75k op/s 2023-09-23T20:32:17.526 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:32:17.595 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:17.673 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:32:17.778 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 21 16 39964 39948 118.802 78.5 0.0141323 0.00841019 2023-09-23T20:32:18.512 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:32:18.517 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:18.518 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:32:18.525 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:32:18.619 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:32:18.649 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:18.778 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 22 16 41330 41314 117.283 85.375 0.00745465 0.00852125 2023-09-23T20:32:18.820 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:32:18.830 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:18.830 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:32:18.931 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:32:19.001 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:19.230 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:32:19.236 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:19.237 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:32:19.338 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:32:19.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:19 smithi018 ceph-mon[184144]: pgmap v712: 109 pgs: 109 active+clean; 2.2 GiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 103 MiB/s wr, 1.65k op/s 2023-09-23T20:32:19.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:19 smithi018 ceph-mon[179647]: pgmap v712: 109 pgs: 109 active+clean; 2.2 GiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 103 MiB/s wr, 1.65k op/s 2023-09-23T20:32:19.407 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:19.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:19 smithi149 ceph-mon[170394]: pgmap v712: 109 pgs: 109 active+clean; 2.2 GiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 103 MiB/s wr, 1.65k op/s 2023-09-23T20:32:19.720 INFO:teuthology.orchestra.run.smithi018.stdout:ERROR: (22) Invalid argument 2023-09-23T20:32:19.721 INFO:teuthology.orchestra.run.smithi018.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:32:19.778 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 23 16 42602 42586 115.64 79.5 0.0192742 0.0086416 2023-09-23T20:32:20.778 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 24 16 43950 43934 114.333 84.25 0.00676133 0.00874121 2023-09-23T20:32:21.395 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:21 smithi018 ceph-mon[179647]: pgmap v713: 109 pgs: 109 active+clean; 2.4 GiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 99 MiB/s wr, 1.59k op/s 2023-09-23T20:32:21.395 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:21 smithi018 ceph-mon[184144]: pgmap v713: 109 pgs: 109 active+clean; 2.4 GiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 99 MiB/s wr, 1.59k op/s 2023-09-23T20:32:21.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:21 smithi149 ceph-mon[170394]: pgmap v713: 109 pgs: 109 active+clean; 2.4 GiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 99 MiB/s wr, 1.59k op/s 2023-09-23T20:32:21.778 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 25 16 45256 45240 113.025 81.625 0.0121467 0.00884222 2023-09-23T20:32:22.781 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 26 16 46587 46571 111.872 83.1875 0.00758028 0.00893341 2023-09-23T20:32:22.789 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:32:22.794 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:22.794 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:32:22.895 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:32:22.942 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:23.064 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T20:32:23.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:23 smithi018 ceph-mon[184144]: pgmap v714: 109 pgs: 109 active+clean; 2.6 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 90 MiB/s wr, 1.44k op/s 2023-09-23T20:32:23.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:23 smithi018 ceph-mon[179647]: pgmap v714: 109 pgs: 109 active+clean; 2.6 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 90 MiB/s wr, 1.44k op/s 2023-09-23T20:32:23.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:23 smithi149 ceph-mon[170394]: pgmap v714: 109 pgs: 109 active+clean; 2.6 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 90 MiB/s wr, 1.44k op/s 2023-09-23T20:32:23.677 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:32:23.682 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:23.682 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:32:23.780 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 27 15 47843 47828 110.639 78.5625 0.00771266 0.00903331 2023-09-23T20:32:23.783 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:32:23.817 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:23.909 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:32:23.912 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:23.912 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:32:24.013 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:32:24.098 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:24.782 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 28 16 49180 49164 109.664 83.5 0.00926256 0.00911447 2023-09-23T20:32:25.394 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:25 smithi018 ceph-mon[179647]: pgmap v715: 109 pgs: 109 active+clean; 2.7 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 90 MiB/s wr, 1.44k op/s 2023-09-23T20:32:25.606 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:25 smithi018 ceph-mon[184144]: pgmap v715: 109 pgs: 109 active+clean; 2.7 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 90 MiB/s wr, 1.44k op/s 2023-09-23T20:32:25.607 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:32:24 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:32:24] "GET /metrics HTTP/1.1" 200 36499 "" "Prometheus/2.43.0" 2023-09-23T20:32:25.607 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:25 smithi149 ceph-mon[170394]: pgmap v715: 109 pgs: 109 active+clean; 2.7 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 90 MiB/s wr, 1.44k op/s 2023-09-23T20:32:25.784 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 29 16 50337 50321 108.367 72.3125 0.00650719 0.00922278 2023-09-23T20:32:26.768 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:32:26.773 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:26.774 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:32:26.784 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 30 16 51640 51624 107.469 81.4375 0.0140076 0.00929809 2023-09-23T20:32:26.875 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:32:26.906 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:26.947 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:32:26.949 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:26.950 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:32:27.051 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:32:27.086 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:27.136 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:32:27.141 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:27.141 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:32:27.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:27 smithi018 ceph-mon[179647]: pgmap v716: 109 pgs: 109 active+clean; 3.0 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 95 MiB/s wr, 1.53k op/s 2023-09-23T20:32:27.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:27 smithi149 ceph-mon[170394]: pgmap v716: 109 pgs: 109 active+clean; 3.0 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 95 MiB/s wr, 1.53k op/s 2023-09-23T20:32:27.243 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:32:27.312 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:27.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:27 smithi018 ceph-mon[184144]: pgmap v716: 109 pgs: 109 active+clean; 3.0 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 95 MiB/s wr, 1.53k op/s 2023-09-23T20:32:27.786 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 31 16 52906 52890 106.551 79.125 0.0197383 0.00938005 2023-09-23T20:32:28.786 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 32 16 54023 54007 105.403 69.8125 0.0121167 0.00948206 2023-09-23T20:32:29.031 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:32:29.034 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:29.035 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:32:29.135 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:32:29.219 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:29.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:29 smithi018 ceph-mon[179647]: pgmap v717: 109 pgs: 109 active+clean; 3.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 74 MiB/s wr, 1.18k op/s 2023-09-23T20:32:29.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:29 smithi018 ceph-mon[184144]: pgmap v717: 109 pgs: 109 active+clean; 3.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 74 MiB/s wr, 1.18k op/s 2023-09-23T20:32:29.468 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:29 smithi149 ceph-mon[170394]: pgmap v717: 109 pgs: 109 active+clean; 3.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 74 MiB/s wr, 1.18k op/s 2023-09-23T20:32:29.786 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 33 16 55248 55232 104.529 76.5625 0.012946 0.00956217 2023-09-23T20:32:29.810 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:32:29.816 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:29.816 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:32:29.917 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:32:29.943 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:30.020 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:32:30.025 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:30.025 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:32:30.126 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:32:30.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:30 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:32:30.220 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:30.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:30 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:32:30.395 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:30 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:32:30.414 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:32:30.422 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:32:30.426 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:30.427 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:32:30.528 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:32:30.628 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:30.786 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 34 16 56209 56193 103.222 60.0625 0.0152249 0.00968252 2023-09-23T20:32:30.882 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:32:30.893 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:30.894 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:32:30.995 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:32:31.083 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:31.139 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:32:31.211 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:32:31.211 INFO:teuthology.orchestra.run.smithi018.stderr:noscrub is unset 2023-09-23T20:32:31.262 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:32:31.332 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:32:31.372 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:32:31.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:31 smithi018 ceph-mon[179647]: pgmap v718: 109 pgs: 109 active+clean; 3.1 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 79 MiB/s wr, 1.26k op/s 2023-09-23T20:32:31.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:31 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3861532743' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:32:31.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:31 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:32:31.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:31 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/681489263' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:32:31.394 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:31 smithi018 ceph-mon[184144]: pgmap v718: 109 pgs: 109 active+clean; 3.1 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 79 MiB/s wr, 1.26k op/s 2023-09-23T20:32:31.394 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:31 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3861532743' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:32:31.394 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:31 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:32:31.394 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:31 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/681489263' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:32:31.434 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:32:31.459 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:32:31.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:31 smithi149 ceph-mon[170394]: pgmap v718: 109 pgs: 109 active+clean; 3.1 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 79 MiB/s wr, 1.26k op/s 2023-09-23T20:32:31.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:31 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3861532743' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:32:31.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:31 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:32:31.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:31 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/681489263' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:32:31.505 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:32:31.526 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:32:31.526 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:32:31.526 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:32:31.526 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:32:31.527 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:32:31.527 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:32:31.527 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:32:31.527 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:32:31.714 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:32:31.719 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:31 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mon-a[179623]: 2023-09-23T20:32:31.455+0000 7f160fef9700 -1 mon.a@0(leader).osd e311 definitely_dead 0 2023-09-23T20:32:31.725 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:31.726 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:32:31.786 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 35 16 57121 57105 101.902 57 0.0231664 0.00980737 2023-09-23T20:32:31.827 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:32:31.927 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:32.127 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:32:32.135 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:32.136 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:32:32.237 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:32:32.255 INFO:teuthology.orchestra.run.smithi018.stderr:marked in osd.4. 2023-09-23T20:32:32.270 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:32.306 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:32:32.308 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:32.309 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:32:32.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:32 smithi018 ceph-mon[184144]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:32:32.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:32 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:32:32.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:32 smithi018 ceph-mon[184144]: osdmap e311: 8 total, 8 up, 7 in 2023-09-23T20:32:32.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:32 smithi018 ceph-mon[184144]: failed to encode map e311 with expected crc 2023-09-23T20:32:32.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:32 smithi018 ceph-mon[184144]: failed to encode map e311 with expected crc 2023-09-23T20:32:32.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:32 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2557457065' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["4"]}]: dispatch 2023-09-23T20:32:32.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:32 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["4"]}]: dispatch 2023-09-23T20:32:32.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:32 smithi018 ceph-mon[179647]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:32:32.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:32 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:32:32.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:32 smithi018 ceph-mon[179647]: osdmap e311: 8 total, 8 up, 7 in 2023-09-23T20:32:32.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:32 smithi018 ceph-mon[179647]: failed to encode map e311 with expected crc 2023-09-23T20:32:32.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:32 smithi018 ceph-mon[179647]: failed to encode map e311 with expected crc 2023-09-23T20:32:32.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:32 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2557457065' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["4"]}]: dispatch 2023-09-23T20:32:32.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:32 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["4"]}]: dispatch 2023-09-23T20:32:32.409 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:32:32.436 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:32.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:32 smithi149 ceph-mon[170394]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:32:32.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:32 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:32:32.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:32 smithi149 ceph-mon[170394]: osdmap e311: 8 total, 8 up, 7 in 2023-09-23T20:32:32.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:32 smithi149 ceph-mon[170394]: failed to encode map e311 with expected crc 2023-09-23T20:32:32.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:32 smithi149 ceph-mon[170394]: failed to encode map e311 with expected crc 2023-09-23T20:32:32.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:32 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2557457065' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["4"]}]: dispatch 2023-09-23T20:32:32.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:32 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["4"]}]: dispatch 2023-09-23T20:32:32.468 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:32:32.470 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:32.470 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:32:32.554 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2023-09-23T20:32:32.554 INFO:teuthology.orchestra.run.smithi018.stdout: "ops": [ 2023-09-23T20:32:32.554 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:32.554 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:57620 7.8 7.1a1b638 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e312)", 2023-09-23T20:32:32.554 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:32:32.247325+0000", 2023-09-23T20:32:32.554 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.30639638200000002, 2023-09-23T20:32:32.554 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.30642044899999998, 2023-09-23T20:32:32.554 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:32:32.555 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "delayed", 2023-09-23T20:32:32.555 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:32:32.555 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:32:32.555 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:32:32.555 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 57620 2023-09-23T20:32:32.555 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:32.555 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:32:32.555 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:32.555 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:32:32.555 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.247325+0000", 2023-09-23T20:32:32.556 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:32:32.556 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:32.556 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:32.556 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:32:32.556 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.247325+0000", 2023-09-23T20:32:32.556 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:32:32.556 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:32.556 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:32.556 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:32:32.556 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.247323+0000", 2023-09-23T20:32:32.556 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:32:32.556 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:32.557 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:32.557 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:32:32.557 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.247348+0000", 2023-09-23T20:32:32.557 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.5418e-05 2023-09-23T20:32:32.557 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:32.557 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:32.557 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:32:32.557 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.247362+0000", 2023-09-23T20:32:32.557 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.4116e-05 2023-09-23T20:32:32.557 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:32.557 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:32.558 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:32:32.558 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.247370+0000", 2023-09-23T20:32:32.558 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.9459999999999998e-06 2023-09-23T20:32:32.558 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:32.558 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:32.558 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:32:32.558 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.247428+0000", 2023-09-23T20:32:32.558 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.7472999999999997e-05 2023-09-23T20:32:32.558 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:32.558 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:32.558 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:32:32.558 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.247439+0000", 2023-09-23T20:32:32.559 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0764000000000001e-05 2023-09-23T20:32:32.559 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:32:32.559 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:32:32.559 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:32:32.559 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:32:32.559 INFO:teuthology.orchestra.run.smithi018.stdout: ], 2023-09-23T20:32:32.559 INFO:teuthology.orchestra.run.smithi018.stdout: "num_ops": 1 2023-09-23T20:32:32.559 INFO:teuthology.orchestra.run.smithi018.stdout:} 2023-09-23T20:32:32.571 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:32:32.600 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:32.636 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:32:32.639 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:32.640 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:32:32.703 INFO:tasks.thrashosds.thrasher:Added osd 4 2023-09-23T20:32:32.741 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:32:32.767 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:32.786 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 36 16 57675 57659 100.034 34.625 0.00369816 0.00984671 2023-09-23T20:32:32.947 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:32:32.949 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:32.950 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:32:32.998 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T20:32:33.051 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:32:33.078 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:33.111 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:32:33.114 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:33.114 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:32:33.215 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:32:33.279 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:33.318 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:32:33.320 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:33.321 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:32:33.422 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:32:33.450 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:33.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:33 smithi149 ceph-mon[170394]: pgmap v720: 109 pgs: 109 active+clean; 3.3 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 79 MiB/s wr, 1.26k op/s 2023-09-23T20:32:33.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:33 smithi149 ceph-mon[170394]: failed to encode map e312 with expected crc 2023-09-23T20:32:33.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:33 smithi149 ceph-mon[170394]: failed to encode map e312 with expected crc 2023-09-23T20:32:33.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:33 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["4"]}]': finished 2023-09-23T20:32:33.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:33 smithi149 ceph-mon[170394]: osdmap e312: 8 total, 8 up, 8 in 2023-09-23T20:32:33.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:33 smithi149 ceph-mon[170394]: failed to encode map e312 with expected crc 2023-09-23T20:32:33.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:33 smithi149 ceph-mon[170394]: failed to encode map e312 with expected crc 2023-09-23T20:32:33.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:33 smithi149 ceph-mon[170394]: failed to encode map e312 with expected crc 2023-09-23T20:32:33.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:33 smithi149 ceph-mon[170394]: failed to encode map e312 with expected crc 2023-09-23T20:32:33.482 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:32:33.484 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:33 smithi018 ceph-mon[184144]: pgmap v720: 109 pgs: 109 active+clean; 3.3 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 79 MiB/s wr, 1.26k op/s 2023-09-23T20:32:33.484 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:33 smithi018 ceph-mon[184144]: failed to encode map e312 with expected crc 2023-09-23T20:32:33.484 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:33 smithi018 ceph-mon[184144]: failed to encode map e312 with expected crc 2023-09-23T20:32:33.484 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:33 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["4"]}]': finished 2023-09-23T20:32:33.485 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:33 smithi018 ceph-mon[184144]: osdmap e312: 8 total, 8 up, 8 in 2023-09-23T20:32:33.485 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:33 smithi018 ceph-mon[184144]: failed to encode map e312 with expected crc 2023-09-23T20:32:33.485 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:33 smithi018 ceph-mon[184144]: failed to encode map e312 with expected crc 2023-09-23T20:32:33.485 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:33 smithi018 ceph-mon[184144]: failed to encode map e312 with expected crc 2023-09-23T20:32:33.485 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:33 smithi018 ceph-mon[184144]: failed to encode map e312 with expected crc 2023-09-23T20:32:33.485 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:33 smithi018 ceph-mon[179647]: pgmap v720: 109 pgs: 109 active+clean; 3.3 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 79 MiB/s wr, 1.26k op/s 2023-09-23T20:32:33.485 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:33 smithi018 ceph-mon[179647]: failed to encode map e312 with expected crc 2023-09-23T20:32:33.485 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:33 smithi018 ceph-mon[179647]: failed to encode map e312 with expected crc 2023-09-23T20:32:33.486 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:33 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["4"]}]': finished 2023-09-23T20:32:33.486 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:33 smithi018 ceph-mon[179647]: osdmap e312: 8 total, 8 up, 8 in 2023-09-23T20:32:33.486 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:33 smithi018 ceph-mon[179647]: failed to encode map e312 with expected crc 2023-09-23T20:32:33.486 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:33 smithi018 ceph-mon[179647]: failed to encode map e312 with expected crc 2023-09-23T20:32:33.486 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:33 smithi018 ceph-mon[179647]: failed to encode map e312 with expected crc 2023-09-23T20:32:33.486 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:33 smithi018 ceph-mon[179647]: failed to encode map e312 with expected crc 2023-09-23T20:32:33.486 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:33.487 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:32:33.540 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:32:33.588 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:32:33.615 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:33.646 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:32:33.649 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:33.649 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:32:33.750 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:32:33.787 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:33.787 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 37 16 57675 57659 97.332 0 - 0.00984671 2023-09-23T20:32:33.821 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:32:33.825 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:33.825 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:32:33.891 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T20:32:33.925 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:32:33.953 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:33.986 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:32:33.988 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:33.989 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:32:34.090 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:32:34.116 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:34.149 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:32:34.151 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:34.152 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:32:34.250 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:32:34.253 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:32:34.278 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:34.310 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:32:34.313 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:34.313 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:32:34.324 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:34 smithi018 ceph-mon[184144]: failed to encode map e313 with expected crc 2023-09-23T20:32:34.324 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:34 smithi018 ceph-mon[184144]: failed to encode map e313 with expected crc 2023-09-23T20:32:34.324 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:34 smithi018 ceph-mon[184144]: osdmap e313: 8 total, 8 up, 8 in 2023-09-23T20:32:34.325 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:34 smithi018 ceph-mon[184144]: failed to encode map e313 with expected crc 2023-09-23T20:32:34.325 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:34 smithi018 ceph-mon[184144]: failed to encode map e313 with expected crc 2023-09-23T20:32:34.325 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:34 smithi018 ceph-mon[184144]: failed to encode map e313 with expected crc 2023-09-23T20:32:34.325 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:34 smithi018 ceph-mon[184144]: failed to encode map e313 with expected crc 2023-09-23T20:32:34.325 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:34 smithi018 ceph-mon[184144]: failed to encode map e313 with expected crc 2023-09-23T20:32:34.325 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:34 smithi018 ceph-mon[184144]: osdmap e314: 8 total, 8 up, 8 in 2023-09-23T20:32:34.325 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:34 smithi018 ceph-mon[179647]: failed to encode map e313 with expected crc 2023-09-23T20:32:34.325 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:34 smithi018 ceph-mon[179647]: failed to encode map e313 with expected crc 2023-09-23T20:32:34.325 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:34 smithi018 ceph-mon[179647]: osdmap e313: 8 total, 8 up, 8 in 2023-09-23T20:32:34.325 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:34 smithi018 ceph-mon[179647]: failed to encode map e313 with expected crc 2023-09-23T20:32:34.325 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:34 smithi018 ceph-mon[179647]: failed to encode map e313 with expected crc 2023-09-23T20:32:34.326 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:34 smithi018 ceph-mon[179647]: failed to encode map e313 with expected crc 2023-09-23T20:32:34.326 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:34 smithi018 ceph-mon[179647]: failed to encode map e313 with expected crc 2023-09-23T20:32:34.326 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:34 smithi018 ceph-mon[179647]: failed to encode map e313 with expected crc 2023-09-23T20:32:34.326 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:34 smithi018 ceph-mon[179647]: osdmap e314: 8 total, 8 up, 8 in 2023-09-23T20:32:34.414 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:32:34.447 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:34.603 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:32:34.608 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:34.609 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:32:34.609 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:32:34.710 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:32:34.721 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:34 smithi149 ceph-mon[170394]: failed to encode map e313 with expected crc 2023-09-23T20:32:34.721 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:34 smithi149 ceph-mon[170394]: failed to encode map e313 with expected crc 2023-09-23T20:32:34.721 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:34 smithi149 ceph-mon[170394]: osdmap e313: 8 total, 8 up, 8 in 2023-09-23T20:32:34.721 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:34 smithi149 ceph-mon[170394]: failed to encode map e313 with expected crc 2023-09-23T20:32:34.721 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:34 smithi149 ceph-mon[170394]: failed to encode map e313 with expected crc 2023-09-23T20:32:34.721 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:34 smithi149 ceph-mon[170394]: failed to encode map e313 with expected crc 2023-09-23T20:32:34.721 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:34 smithi149 ceph-mon[170394]: failed to encode map e313 with expected crc 2023-09-23T20:32:34.721 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:34 smithi149 ceph-mon[170394]: failed to encode map e313 with expected crc 2023-09-23T20:32:34.721 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:34 smithi149 ceph-mon[170394]: osdmap e314: 8 total, 8 up, 8 in 2023-09-23T20:32:34.746 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:34.787 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 38 16 57802 57786 94.9808 3.96875 0.0210697 0.0105217 2023-09-23T20:32:34.807 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:32:34.810 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:34.810 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:32:34.911 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:32:34.979 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2023-09-23T20:32:34.979 INFO:teuthology.orchestra.run.smithi018.stdout: "ops": [], 2023-09-23T20:32:34.979 INFO:teuthology.orchestra.run.smithi018.stdout: "complaint_time": 30, 2023-09-23T20:32:34.979 INFO:teuthology.orchestra.run.smithi018.stdout: "num_blocked_ops": 0 2023-09-23T20:32:34.980 INFO:teuthology.orchestra.run.smithi018.stdout:} 2023-09-23T20:32:35.033 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:35.344 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:35 smithi018 ceph-mon[179647]: pgmap v723: 109 pgs: 1 remapped+peering, 2 peering, 106 active+clean; 3.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 71 MiB/s wr, 1.13k op/s; 0 B/s, 0 objects/s recovering 2023-09-23T20:32:35.344 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:35 smithi018 ceph-mon[179647]: failed to encode map e314 with expected crc 2023-09-23T20:32:35.344 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:35 smithi018 ceph-mon[179647]: failed to encode map e314 with expected crc 2023-09-23T20:32:35.344 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:35 smithi018 ceph-mon[179647]: failed to encode map e314 with expected crc 2023-09-23T20:32:35.345 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:35 smithi018 ceph-mon[179647]: failed to encode map e314 with expected crc 2023-09-23T20:32:35.345 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:35 smithi018 ceph-mon[179647]: failed to encode map e314 with expected crc 2023-09-23T20:32:35.345 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:32:34 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:32:34] "GET /metrics HTTP/1.1" 200 36505 "" "Prometheus/2.43.0" 2023-09-23T20:32:35.345 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:35 smithi018 ceph-mon[184144]: pgmap v723: 109 pgs: 1 remapped+peering, 2 peering, 106 active+clean; 3.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 71 MiB/s wr, 1.13k op/s; 0 B/s, 0 objects/s recovering 2023-09-23T20:32:35.345 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:35 smithi018 ceph-mon[184144]: failed to encode map e314 with expected crc 2023-09-23T20:32:35.345 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:35 smithi018 ceph-mon[184144]: failed to encode map e314 with expected crc 2023-09-23T20:32:35.345 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:35 smithi018 ceph-mon[184144]: failed to encode map e314 with expected crc 2023-09-23T20:32:35.346 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:35 smithi018 ceph-mon[184144]: failed to encode map e314 with expected crc 2023-09-23T20:32:35.346 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:35 smithi018 ceph-mon[184144]: failed to encode map e314 with expected crc 2023-09-23T20:32:35.418 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:32:35.423 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:35.423 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:32:35.524 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:32:35.564 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:35 smithi149 ceph-mon[170394]: pgmap v723: 109 pgs: 1 remapped+peering, 2 peering, 106 active+clean; 3.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 71 MiB/s wr, 1.13k op/s; 0 B/s, 0 objects/s recovering 2023-09-23T20:32:35.565 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:35 smithi149 ceph-mon[170394]: failed to encode map e314 with expected crc 2023-09-23T20:32:35.565 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:35 smithi149 ceph-mon[170394]: failed to encode map e314 with expected crc 2023-09-23T20:32:35.565 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:35 smithi149 ceph-mon[170394]: failed to encode map e314 with expected crc 2023-09-23T20:32:35.565 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:35 smithi149 ceph-mon[170394]: failed to encode map e314 with expected crc 2023-09-23T20:32:35.565 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:35 smithi149 ceph-mon[170394]: failed to encode map e314 with expected crc 2023-09-23T20:32:35.567 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:35.588 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T20:32:35.618 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:32:35.628 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:35.628 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:32:35.729 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:32:35.770 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:35.787 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 39 16 58629 58613 93.8712 51.6875 0.0170505 0.0106457 2023-09-23T20:32:35.812 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:32:35.818 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:35.818 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:32:35.834 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:32:35.918 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:32:36.027 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:36.090 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:32:36.100 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:36.100 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:32:36.201 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:32:36.289 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:36 smithi018 ceph-mon[179647]: osdmap e315: 8 total, 8 up, 8 in 2023-09-23T20:32:36.289 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:36 smithi018 ceph-mon[179647]: failed to encode map e315 with expected crc 2023-09-23T20:32:36.289 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:36 smithi018 ceph-mon[179647]: failed to encode map e315 with expected crc 2023-09-23T20:32:36.289 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:36 smithi018 ceph-mon[179647]: failed to encode map e315 with expected crc 2023-09-23T20:32:36.335 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:36.405 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:32:36.420 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:36.421 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:32:36.522 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:32:36.564 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:36 smithi149 ceph-mon[170394]: osdmap e315: 8 total, 8 up, 8 in 2023-09-23T20:32:36.565 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:36 smithi149 ceph-mon[170394]: failed to encode map e315 with expected crc 2023-09-23T20:32:36.565 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:36 smithi149 ceph-mon[170394]: failed to encode map e315 with expected crc 2023-09-23T20:32:36.565 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:36 smithi149 ceph-mon[170394]: failed to encode map e315 with expected crc 2023-09-23T20:32:36.567 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:36.612 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:32:36.615 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:36.616 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:32:36.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:36 smithi018 ceph-mon[184144]: osdmap e315: 8 total, 8 up, 8 in 2023-09-23T20:32:36.643 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:36 smithi018 ceph-mon[184144]: failed to encode map e315 with expected crc 2023-09-23T20:32:36.643 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:36 smithi018 ceph-mon[184144]: failed to encode map e315 with expected crc 2023-09-23T20:32:36.643 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:36 smithi018 ceph-mon[184144]: failed to encode map e315 with expected crc 2023-09-23T20:32:36.716 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:32:36.787 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:2023-09-23T20:32:36.787594+0000 min lat: 0.00108768 max lat: 2.45012 avg lat: 0.0107806 2023-09-23T20:32:36.787 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T20:32:36.787 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 40 16 59387 59371 92.7093 47.375 0.0185422 0.0107806 2023-09-23T20:32:36.837 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:36.939 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:32:36.952 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:36.952 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:32:37.053 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:32:37.063 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:32:37.154 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:37.253 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:32:37.261 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:37.262 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:32:37.338 INFO:teuthology.orchestra.run.smithi018.stderr:nodeep-scrub is unset 2023-09-23T20:32:37.363 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:32:37.436 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:37 smithi018 ceph-mon[184144]: pgmap v726: 109 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 5 remapped+peering, 8 peering, 94 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 10 KiB/s rd, 51 MiB/s wr, 825 op/s; 7135/174144 objects degraded (4.097%); 36 KiB/s, 2 keys/s, 4 objects/s recovering 2023-09-23T20:32:37.437 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:37 smithi018 ceph-mon[184144]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T20:32:37.437 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:37 smithi018 ceph-mon[184144]: Health check failed: Degraded data redundancy: 7135/174144 objects degraded (4.097%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T20:32:37.437 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:37 smithi018 ceph-mon[184144]: failed to encode map e316 with expected crc 2023-09-23T20:32:37.437 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:37 smithi018 ceph-mon[184144]: osdmap e316: 8 total, 8 up, 8 in 2023-09-23T20:32:37.437 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:37 smithi018 ceph-mon[184144]: failed to encode map e316 with expected crc 2023-09-23T20:32:37.437 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:37 smithi018 ceph-mon[184144]: failed to encode map e316 with expected crc 2023-09-23T20:32:37.437 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:37 smithi018 ceph-mon[184144]: failed to encode map e316 with expected crc 2023-09-23T20:32:37.437 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:37 smithi018 ceph-mon[184144]: failed to encode map e316 with expected crc 2023-09-23T20:32:37.437 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:37 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1782696098' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:32:37.437 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:37 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:32:37.438 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:37 smithi018 ceph-mon[179647]: pgmap v726: 109 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 5 remapped+peering, 8 peering, 94 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 10 KiB/s rd, 51 MiB/s wr, 825 op/s; 7135/174144 objects degraded (4.097%); 36 KiB/s, 2 keys/s, 4 objects/s recovering 2023-09-23T20:32:37.438 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:37 smithi018 ceph-mon[179647]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T20:32:37.438 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:37 smithi018 ceph-mon[179647]: Health check failed: Degraded data redundancy: 7135/174144 objects degraded (4.097%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T20:32:37.438 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:37 smithi018 ceph-mon[179647]: failed to encode map e316 with expected crc 2023-09-23T20:32:37.438 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:37 smithi018 ceph-mon[179647]: osdmap e316: 8 total, 8 up, 8 in 2023-09-23T20:32:37.438 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:37 smithi018 ceph-mon[179647]: failed to encode map e316 with expected crc 2023-09-23T20:32:37.438 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:37 smithi018 ceph-mon[179647]: failed to encode map e316 with expected crc 2023-09-23T20:32:37.438 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:37 smithi018 ceph-mon[179647]: failed to encode map e316 with expected crc 2023-09-23T20:32:37.438 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:37 smithi018 ceph-mon[179647]: failed to encode map e316 with expected crc 2023-09-23T20:32:37.438 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:37 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1782696098' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:32:37.438 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:37 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:32:37.440 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:37.679 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:32:37.687 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:37.687 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:32:37.704 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 5, 7, 3, 2, 6, 4] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T20:32:37.704 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:32:37.704 INFO:tasks.thrashosds.thrasher:inject_pause on osd.6 2023-09-23T20:32:37.704 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T20:32:37.705 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T20:32:37.705 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 config set filestore_inject_stall 3 2023-09-23T20:32:37.705 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:32:37.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:37 smithi149 ceph-mon[170394]: pgmap v726: 109 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 5 remapped+peering, 8 peering, 94 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 10 KiB/s rd, 51 MiB/s wr, 825 op/s; 7135/174144 objects degraded (4.097%); 36 KiB/s, 2 keys/s, 4 objects/s recovering 2023-09-23T20:32:37.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:37 smithi149 ceph-mon[170394]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T20:32:37.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:37 smithi149 ceph-mon[170394]: Health check failed: Degraded data redundancy: 7135/174144 objects degraded (4.097%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T20:32:37.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:37 smithi149 ceph-mon[170394]: failed to encode map e316 with expected crc 2023-09-23T20:32:37.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:37 smithi149 ceph-mon[170394]: osdmap e316: 8 total, 8 up, 8 in 2023-09-23T20:32:37.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:37 smithi149 ceph-mon[170394]: failed to encode map e316 with expected crc 2023-09-23T20:32:37.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:37 smithi149 ceph-mon[170394]: failed to encode map e316 with expected crc 2023-09-23T20:32:37.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:37 smithi149 ceph-mon[170394]: failed to encode map e316 with expected crc 2023-09-23T20:32:37.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:37 smithi149 ceph-mon[170394]: failed to encode map e316 with expected crc 2023-09-23T20:32:37.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:37 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1782696098' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:32:37.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:37 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:32:37.758 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:32:37.787 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 41 16 60105 60089 91.5431 44.875 0.0354394 0.0109181 2023-09-23T20:32:37.787 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:32:37.788 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:32:37.828 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:32:37.828 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:37.862 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:32:37.885 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:32:37.894 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:37.894 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:32:37.906 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:32:37.950 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:32:37.995 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:32:38.008 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:32:38.024 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:32:38.024 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:32:38.024 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:32:38.024 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:32:38.024 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:32:38.025 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:32:38.025 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:32:38.025 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:32:38.065 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:38.188 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:32:38.202 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:38.203 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:32:38.303 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:32:38.344 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:32:38.352 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:38 smithi018 ceph-mon[184144]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:32:38.352 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:38 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:32:38.352 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:38 smithi018 ceph-mon[184144]: osdmap e317: 8 total, 8 up, 8 in 2023-09-23T20:32:38.352 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:38 smithi018 ceph-mon[184144]: failed to encode map e317 with expected crc 2023-09-23T20:32:38.352 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:38 smithi018 ceph-mon[184144]: failed to encode map e317 with expected crc 2023-09-23T20:32:38.353 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:38 smithi018 ceph-mon[184144]: failed to encode map e317 with expected crc 2023-09-23T20:32:38.353 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:38 smithi018 ceph-mon[184144]: failed to encode map e317 with expected crc 2023-09-23T20:32:38.353 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:38 smithi018 ceph-mon[184144]: failed to encode map e317 with expected crc 2023-09-23T20:32:38.353 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:38 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3463800521' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:32:38.353 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:38 smithi018 ceph-mon[179647]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:32:38.353 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:38 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:32:38.353 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:38 smithi018 ceph-mon[179647]: osdmap e317: 8 total, 8 up, 8 in 2023-09-23T20:32:38.353 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:38 smithi018 ceph-mon[179647]: failed to encode map e317 with expected crc 2023-09-23T20:32:38.353 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:38 smithi018 ceph-mon[179647]: failed to encode map e317 with expected crc 2023-09-23T20:32:38.353 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:38 smithi018 ceph-mon[179647]: failed to encode map e317 with expected crc 2023-09-23T20:32:38.354 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:38 smithi018 ceph-mon[179647]: failed to encode map e317 with expected crc 2023-09-23T20:32:38.354 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:38 smithi018 ceph-mon[179647]: failed to encode map e317 with expected crc 2023-09-23T20:32:38.354 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:38 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3463800521' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:32:38.357 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:38.499 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:32:38.504 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:38.504 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:32:38.560 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:38 smithi149 ceph-mon[170394]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:32:38.561 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:38 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:32:38.561 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:38 smithi149 ceph-mon[170394]: osdmap e317: 8 total, 8 up, 8 in 2023-09-23T20:32:38.561 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:38 smithi149 ceph-mon[170394]: failed to encode map e317 with expected crc 2023-09-23T20:32:38.561 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:38 smithi149 ceph-mon[170394]: failed to encode map e317 with expected crc 2023-09-23T20:32:38.561 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:38 smithi149 ceph-mon[170394]: failed to encode map e317 with expected crc 2023-09-23T20:32:38.561 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:38 smithi149 ceph-mon[170394]: failed to encode map e317 with expected crc 2023-09-23T20:32:38.561 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:38 smithi149 ceph-mon[170394]: failed to encode map e317 with expected crc 2023-09-23T20:32:38.561 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:38 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3463800521' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:32:38.605 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:32:38.666 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:38.752 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:32:38.758 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:38.759 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:32:38.787 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 42 16 60938 60922 90.6034 52.0625 0.017614 0.0110314 2023-09-23T20:32:38.860 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:32:38.912 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:38.973 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:32:38.979 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:38.979 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:32:39.080 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:32:39.157 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:39.231 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:32:39.237 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:39.237 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:32:39.331 INFO:teuthology.orchestra.run.smithi018.stdout:ERROR: (22) Invalid argument 2023-09-23T20:32:39.332 INFO:teuthology.orchestra.run.smithi018.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:32:39.338 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:32:39.394 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:39.439 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:32:39.442 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:39.442 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:32:39.543 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:32:39.546 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:39 smithi149 ceph-mon[170394]: pgmap v729: 109 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 5 remapped+peering, 6 peering, 95 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 10 KiB/s rd, 33 MiB/s wr, 533 op/s; 10773/174972 objects degraded (6.157%); 36 KiB/s, 2 keys/s, 4 objects/s recovering 2023-09-23T20:32:39.597 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:39.644 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:39 smithi018 ceph-mon[184144]: pgmap v729: 109 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 5 remapped+peering, 6 peering, 95 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 10 KiB/s rd, 33 MiB/s wr, 533 op/s; 10773/174972 objects degraded (6.157%); 36 KiB/s, 2 keys/s, 4 objects/s recovering 2023-09-23T20:32:39.645 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:39 smithi018 ceph-mon[179647]: pgmap v729: 109 pgs: 3 active+recovery_wait+undersized+degraded+remapped, 5 remapped+peering, 6 peering, 95 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 10 KiB/s rd, 33 MiB/s wr, 533 op/s; 10773/174972 objects degraded (6.157%); 36 KiB/s, 2 keys/s, 4 objects/s recovering 2023-09-23T20:32:39.660 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:32:39.667 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:39.667 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:32:39.768 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:32:39.770 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T20:32:39.787 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 43 16 61813 61797 89.7685 54.6875 0.0135839 0.011134 2023-09-23T20:32:39.823 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:39.885 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:32:39.889 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:39.890 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:32:39.925 INFO:teuthology.orchestra.run.smithi149.stdout:{ 2023-09-23T20:32:39.925 INFO:teuthology.orchestra.run.smithi149.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-09-23T20:32:39.926 INFO:teuthology.orchestra.run.smithi149.stdout:} 2023-09-23T20:32:39.991 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:32:40.039 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:40.180 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T20:32:40.347 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:32:40.357 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:40.358 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:32:40.459 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:32:40.569 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:40.607 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:32:40.665 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:32:40.675 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:40.675 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:32:40.776 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:32:40.787 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 44 16 62645 62629 88.9104 52 0.0341065 0.011242 2023-09-23T20:32:40.801 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:32:40.839 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:40.928 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:32:40.940 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:40.941 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:32:41.042 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:32:41.168 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:41.555 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:41 smithi018 ceph-mon[184144]: pgmap v730: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 6 peering, 98 active+clean; 3.6 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 7.6 KiB/s rd, 29 MiB/s wr, 476 op/s; 10773/177861 objects degraded (6.057%); 6994/177861 objects misplaced (3.932%); 1.7 MiB/s, 1 keys/s, 31 objects/s recovering 2023-09-23T20:32:41.563 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:41 smithi018 ceph-mon[179647]: pgmap v730: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 6 peering, 98 active+clean; 3.6 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 7.6 KiB/s rd, 29 MiB/s wr, 476 op/s; 10773/177861 objects degraded (6.057%); 6994/177861 objects misplaced (3.932%); 1.7 MiB/s, 1 keys/s, 31 objects/s recovering 2023-09-23T20:32:41.591 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:32:41.641 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:32:41.650 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:41.650 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:32:41.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:41 smithi149 ceph-mon[170394]: pgmap v730: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 6 peering, 98 active+clean; 3.6 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 7.6 KiB/s rd, 29 MiB/s wr, 476 op/s; 10773/177861 objects degraded (6.057%); 6994/177861 objects misplaced (3.932%); 1.7 MiB/s, 1 keys/s, 31 objects/s recovering 2023-09-23T20:32:41.751 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:32:41.792 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 45 16 63377 63361 87.9516 45.75 0.0164608 0.0113634 2023-09-23T20:32:41.862 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:41.971 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:32:41.978 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:41.978 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:32:42.079 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:32:42.128 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:42.160 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:32:42.174 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:32:42.178 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:42.178 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:32:42.279 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:32:42.346 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:42.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:42 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 10773/184776 objects degraded (5.830%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T20:32:42.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:42 smithi018 ceph-mon[184144]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T20:32:42.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:42 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 10773/184776 objects degraded (5.830%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T20:32:42.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:42 smithi018 ceph-mon[179647]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T20:32:42.664 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:32:42.675 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:42.676 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:32:42.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:42 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 10773/184776 objects degraded (5.830%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T20:32:42.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:42 smithi149 ceph-mon[170394]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T20:32:42.725 INFO:teuthology.orchestra.run.smithi018.stdout:ERROR: (22) Invalid argument 2023-09-23T20:32:42.725 INFO:teuthology.orchestra.run.smithi018.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:32:42.777 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:32:42.787 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 46 16 64181 64165 87.1323 50.25 0.0217313 0.01147 2023-09-23T20:32:42.929 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:43.260 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:32:43.273 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:43.273 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:32:43.316 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T20:32:43.374 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:32:43.407 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:43.462 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:32:43.465 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:43.466 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:32:43.530 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:43 smithi018 ceph-mon[179647]: pgmap v731: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 6.5 KiB/s rd, 41 MiB/s wr, 665 op/s; 10773/184776 objects degraded (5.830%); 6994/184776 objects misplaced (3.785%); 1.4 MiB/s, 1 keys/s, 26 objects/s recovering 2023-09-23T20:32:43.530 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:43 smithi018 ceph-mon[184144]: pgmap v731: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 6.5 KiB/s rd, 41 MiB/s wr, 665 op/s; 10773/184776 objects degraded (5.830%); 6994/184776 objects misplaced (3.785%); 1.4 MiB/s, 1 keys/s, 26 objects/s recovering 2023-09-23T20:32:43.567 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:32:43.606 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:43.653 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:32:43.655 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:43 smithi149 ceph-mon[170394]: pgmap v731: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 6.5 KiB/s rd, 41 MiB/s wr, 665 op/s; 10773/184776 objects degraded (5.830%); 6994/184776 objects misplaced (3.785%); 1.4 MiB/s, 1 keys/s, 26 objects/s recovering 2023-09-23T20:32:43.656 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:43.656 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:32:43.757 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:32:43.787 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 47 16 64970 64954 86.3279 49.3125 0.0135433 0.0115769 2023-09-23T20:32:43.795 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:43.843 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:32:43.845 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:43.846 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:32:43.947 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:32:43.982 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:44.027 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:32:44.034 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:44.034 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:32:44.135 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:32:44.167 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:44.217 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:32:44.218 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:44.219 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:32:44.319 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:32:44.350 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:44.396 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:32:44.399 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:44.400 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:32:44.430 INFO:teuthology.orchestra.run.smithi018.stderr:noscrub is set 2023-09-23T20:32:44.500 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:32:44.527 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:32:44.593 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:44 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3418419913' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:32:44.594 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:44 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:32:44.594 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:44 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3418419913' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:32:44.594 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:44 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:32:44.603 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:44.671 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:32:44.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:44 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3418419913' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:32:44.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:44 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:32:44.720 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:32:44.767 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:32:44.792 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 48 16 65660 65644 85.4273 43.125 0.0349132 0.0117002 2023-09-23T20:32:44.811 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:32:44.823 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:32:44.830 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:44.831 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:32:44.853 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:32:44.897 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:32:44.932 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:32:44.933 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:32:44.989 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:32:44.999 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:32:44.999 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:32:44.999 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:32:44.999 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:32:44.999 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:32:45.000 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:32:45.000 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:32:45.000 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:32:45.019 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:45.148 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:32:44 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:32:44] "GET /metrics HTTP/1.1" 200 36539 "" "Prometheus/2.43.0" 2023-09-23T20:32:45.316 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:32:45.325 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:45.325 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:32:45.365 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 5, 7, 3, 2, 6, 4] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T20:32:45.366 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:32:45.366 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-09-23T20:32:45.366 INFO:tasks.thrashosds.thrasher:Setting osd 3 primary_affinity to 0.000000 2023-09-23T20:32:45.366 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 3 0 2023-09-23T20:32:45.426 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:32:45.463 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:45.513 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:32:45.515 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:45.515 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:32:45.616 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:32:45.643 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:45 smithi018 ceph-mon[184144]: pgmap v732: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 41 MiB/s wr, 659 op/s; 10773/190008 objects degraded (5.670%); 6801/190008 objects misplaced (3.579%); 2.7 MiB/s, 43 objects/s recovering 2023-09-23T20:32:45.643 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:45 smithi018 ceph-mon[184144]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:32:45.643 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:45 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:32:45.644 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:45 smithi018 ceph-mon[184144]: osdmap e318: 8 total, 8 up, 8 in 2023-09-23T20:32:45.644 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:45 smithi018 ceph-mon[184144]: failed to encode map e318 with expected crc 2023-09-23T20:32:45.644 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:45 smithi018 ceph-mon[184144]: failed to encode map e318 with expected crc 2023-09-23T20:32:45.644 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:45 smithi018 ceph-mon[184144]: failed to encode map e318 with expected crc 2023-09-23T20:32:45.644 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:45 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:32:45.644 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:45 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:32:45.644 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:45 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/339603423' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:32:45.644 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:45 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:32:45.644 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:45 smithi018 ceph-mon[179647]: pgmap v732: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 41 MiB/s wr, 659 op/s; 10773/190008 objects degraded (5.670%); 6801/190008 objects misplaced (3.579%); 2.7 MiB/s, 43 objects/s recovering 2023-09-23T20:32:45.644 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:45 smithi018 ceph-mon[179647]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:32:45.645 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:45 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:32:45.645 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:45 smithi018 ceph-mon[179647]: osdmap e318: 8 total, 8 up, 8 in 2023-09-23T20:32:45.645 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:45 smithi018 ceph-mon[179647]: failed to encode map e318 with expected crc 2023-09-23T20:32:45.645 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:45 smithi018 ceph-mon[179647]: failed to encode map e318 with expected crc 2023-09-23T20:32:45.645 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:45 smithi018 ceph-mon[179647]: failed to encode map e318 with expected crc 2023-09-23T20:32:45.645 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:45 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:32:45.645 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:45 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:32:45.645 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:45 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/339603423' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:32:45.645 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:45 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:32:45.714 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:45.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:45 smithi149 ceph-mon[170394]: pgmap v732: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 41 MiB/s wr, 659 op/s; 10773/190008 objects degraded (5.670%); 6801/190008 objects misplaced (3.579%); 2.7 MiB/s, 43 objects/s recovering 2023-09-23T20:32:45.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:45 smithi149 ceph-mon[170394]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:32:45.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:45 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:32:45.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:45 smithi149 ceph-mon[170394]: osdmap e318: 8 total, 8 up, 8 in 2023-09-23T20:32:45.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:45 smithi149 ceph-mon[170394]: failed to encode map e318 with expected crc 2023-09-23T20:32:45.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:45 smithi149 ceph-mon[170394]: failed to encode map e318 with expected crc 2023-09-23T20:32:45.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:45 smithi149 ceph-mon[170394]: failed to encode map e318 with expected crc 2023-09-23T20:32:45.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:45 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:32:45.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:45 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:32:45.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:45 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/339603423' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:32:45.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:45 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:32:45.789 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 49 16 66436 66420 84.6731 48.5 0.0248429 0.011804 2023-09-23T20:32:46.073 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:32:46.081 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:46.081 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:32:46.182 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:32:46.218 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:46.266 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:32:46.268 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:46.269 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:32:46.369 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:32:46.440 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:46.555 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:32:46.586 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:32:46.595 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:46.596 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:32:46.697 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:32:46.734 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:46.790 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:32:46.791 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 50 16 67252 67236 83.9967 51 0.0103488 0.0118984 2023-09-23T20:32:46.799 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:46.799 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:32:46.900 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:32:46.940 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2023-09-23T20:32:46.940 INFO:teuthology.orchestra.run.smithi018.stdout: "ops": [], 2023-09-23T20:32:46.940 INFO:teuthology.orchestra.run.smithi018.stdout: "complaint_time": 30, 2023-09-23T20:32:46.940 INFO:teuthology.orchestra.run.smithi018.stdout: "num_blocked_ops": 0 2023-09-23T20:32:46.940 INFO:teuthology.orchestra.run.smithi018.stdout:} 2023-09-23T20:32:46.953 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T20:32:46.987 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:47.070 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:32:47.077 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:47.078 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:32:47.178 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:32:47.314 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:47.465 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:32:47.484 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:47.484 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:32:47.514 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T20:32:47.550 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:47 smithi018 ceph-mon[184144]: pgmap v734: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 4.0 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 55 MiB/s wr, 879 op/s; 10773/196470 objects degraded (5.483%); 6801/196470 objects misplaced (3.462%); 2.5 MiB/s, 41 objects/s recovering 2023-09-23T20:32:47.551 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:47 smithi018 ceph-mon[179647]: pgmap v734: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 4.0 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 55 MiB/s wr, 879 op/s; 10773/196470 objects degraded (5.483%); 6801/196470 objects misplaced (3.462%); 2.5 MiB/s, 41 objects/s recovering 2023-09-23T20:32:47.585 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:32:47.619 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:47.692 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:32:47.694 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:47 smithi149 ceph-mon[170394]: pgmap v734: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 4.0 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 55 MiB/s wr, 879 op/s; 10773/196470 objects degraded (5.483%); 6801/196470 objects misplaced (3.462%); 2.5 MiB/s, 41 objects/s recovering 2023-09-23T20:32:47.695 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:47.696 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:32:47.795 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 51 16 67949 67933 83.1981 43.5625 0.0260102 0.0120121 2023-09-23T20:32:47.797 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:32:47.832 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:47.876 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:32:47.882 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:47.882 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:32:47.983 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:32:48.075 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:48.109 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:32:48.126 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:32:48.245 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:32:48.251 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:48.251 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:32:48.352 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:32:48.420 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:48.469 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:32:48.473 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:48.474 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:32:48.574 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:32:48.604 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:48.642 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:32:48.645 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:48.645 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:32:48.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:48 smithi149 ceph-mon[170394]: pgmap v735: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 48 MiB/s wr, 762 op/s; 10773/197883 objects degraded (5.444%); 6801/197883 objects misplaced (3.437%); 2.1 MiB/s, 34 objects/s recovering 2023-09-23T20:32:48.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:48 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 10773/197883 objects degraded (5.444%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T20:32:48.746 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:32:48.786 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:48.798 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 52 16 68632 68616 82.418 42.6875 0.0160406 0.0121264 2023-09-23T20:32:48.833 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:48 smithi018 ceph-mon[179647]: pgmap v735: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 48 MiB/s wr, 762 op/s; 10773/197883 objects degraded (5.444%); 6801/197883 objects misplaced (3.437%); 2.1 MiB/s, 34 objects/s recovering 2023-09-23T20:32:48.833 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:48 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 10773/197883 objects degraded (5.444%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T20:32:48.833 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:48 smithi018 ceph-mon[184144]: pgmap v735: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 48 MiB/s wr, 762 op/s; 10773/197883 objects degraded (5.444%); 6801/197883 objects misplaced (3.437%); 2.1 MiB/s, 34 objects/s recovering 2023-09-23T20:32:48.833 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:48 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 10773/197883 objects degraded (5.444%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T20:32:48.835 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:32:48.839 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:48.840 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:32:48.940 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:32:48.960 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:32:48.970 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:49.009 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:32:49.014 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:49.014 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:32:49.115 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:32:49.158 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:49.212 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:32:49.215 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:49.215 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:32:49.316 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:32:49.353 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:32:49.422 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:49.540 INFO:teuthology.orchestra.run.smithi018.stderr:set osd.3 primary-affinity to 0 (802) 2023-09-23T20:32:49.608 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:32:49.613 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:49.613 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:49 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/4184683505' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.0}]: dispatch 2023-09-23T20:32:49.613 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:49 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.0}]: dispatch 2023-09-23T20:32:49.613 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:49 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/4184683505' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.0}]: dispatch 2023-09-23T20:32:49.613 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:49 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.0}]: dispatch 2023-09-23T20:32:49.614 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:32:49.715 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:32:49.740 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:49.773 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:32:49.775 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:49 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/4184683505' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.0}]: dispatch 2023-09-23T20:32:49.775 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:49 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.0}]: dispatch 2023-09-23T20:32:49.775 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:49.776 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:32:49.796 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 53 16 69223 69207 81.5597 36.9375 0.00392907 0.0122039 2023-09-23T20:32:49.877 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:32:49.929 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:50.344 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:32:50.348 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:50.349 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:32:50.449 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:32:50.485 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:50.593 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:32:50.613 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:50.613 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:32:50.714 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:32:50.733 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:50 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.0}]': finished 2023-09-23T20:32:50.734 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:50 smithi018 ceph-mon[184144]: osdmap e319: 8 total, 8 up, 8 in 2023-09-23T20:32:50.734 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:50 smithi018 ceph-mon[184144]: failed to encode map e319 with expected crc 2023-09-23T20:32:50.734 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:50 smithi018 ceph-mon[184144]: failed to encode map e319 with expected crc 2023-09-23T20:32:50.734 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:50 smithi018 ceph-mon[184144]: failed to encode map e319 with expected crc 2023-09-23T20:32:50.734 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:50 smithi018 ceph-mon[184144]: failed to encode map e319 with expected crc 2023-09-23T20:32:50.734 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:50 smithi018 ceph-mon[184144]: failed to encode map e319 with expected crc 2023-09-23T20:32:50.734 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:50 smithi018 ceph-mon[184144]: failed to encode map e319 with expected crc 2023-09-23T20:32:50.734 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:50 smithi018 ceph-mon[184144]: failed to encode map e319 with expected crc 2023-09-23T20:32:50.734 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:50 smithi018 ceph-mon[184144]: failed to encode map e319 with expected crc 2023-09-23T20:32:50.734 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:50 smithi018 ceph-mon[184144]: failed to encode map e319 with expected crc 2023-09-23T20:32:50.734 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:50 smithi018 ceph-mon[184144]: failed to encode map e319 with expected crc 2023-09-23T20:32:50.735 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:50 smithi018 ceph-mon[184144]: failed to encode map e319 with expected crc 2023-09-23T20:32:50.735 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:50 smithi018 ceph-mon[184144]: pgmap v737: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 43 MiB/s wr, 684 op/s; 10773/201207 objects degraded (5.354%); 6596/201207 objects misplaced (3.278%); 3.1 MiB/s, 49 objects/s recovering 2023-09-23T20:32:50.735 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:50 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.0}]': finished 2023-09-23T20:32:50.735 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:50 smithi018 ceph-mon[179647]: osdmap e319: 8 total, 8 up, 8 in 2023-09-23T20:32:50.735 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:50 smithi018 ceph-mon[179647]: failed to encode map e319 with expected crc 2023-09-23T20:32:50.735 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:50 smithi018 ceph-mon[179647]: failed to encode map e319 with expected crc 2023-09-23T20:32:50.735 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:50 smithi018 ceph-mon[179647]: failed to encode map e319 with expected crc 2023-09-23T20:32:50.736 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:50 smithi018 ceph-mon[179647]: failed to encode map e319 with expected crc 2023-09-23T20:32:50.736 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:50 smithi018 ceph-mon[179647]: failed to encode map e319 with expected crc 2023-09-23T20:32:50.736 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:50 smithi018 ceph-mon[179647]: failed to encode map e319 with expected crc 2023-09-23T20:32:50.736 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:50 smithi018 ceph-mon[179647]: failed to encode map e319 with expected crc 2023-09-23T20:32:50.736 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:50 smithi018 ceph-mon[179647]: failed to encode map e319 with expected crc 2023-09-23T20:32:50.736 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:50 smithi018 ceph-mon[179647]: failed to encode map e319 with expected crc 2023-09-23T20:32:50.736 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:50 smithi018 ceph-mon[179647]: failed to encode map e319 with expected crc 2023-09-23T20:32:50.736 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:50 smithi018 ceph-mon[179647]: failed to encode map e319 with expected crc 2023-09-23T20:32:50.736 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:50 smithi018 ceph-mon[179647]: pgmap v737: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 43 MiB/s wr, 684 op/s; 10773/201207 objects degraded (5.354%); 6596/201207 objects misplaced (3.278%); 3.1 MiB/s, 49 objects/s recovering 2023-09-23T20:32:50.779 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:50.796 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 54 16 69369 69353 80.2191 9.125 0.00535026 0.0123612 2023-09-23T20:32:50.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:50 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.0}]': finished 2023-09-23T20:32:50.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:50 smithi149 ceph-mon[170394]: osdmap e319: 8 total, 8 up, 8 in 2023-09-23T20:32:50.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:50 smithi149 ceph-mon[170394]: failed to encode map e319 with expected crc 2023-09-23T20:32:50.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:50 smithi149 ceph-mon[170394]: failed to encode map e319 with expected crc 2023-09-23T20:32:50.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:50 smithi149 ceph-mon[170394]: failed to encode map e319 with expected crc 2023-09-23T20:32:50.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:50 smithi149 ceph-mon[170394]: failed to encode map e319 with expected crc 2023-09-23T20:32:50.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:50 smithi149 ceph-mon[170394]: failed to encode map e319 with expected crc 2023-09-23T20:32:50.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:50 smithi149 ceph-mon[170394]: failed to encode map e319 with expected crc 2023-09-23T20:32:50.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:50 smithi149 ceph-mon[170394]: failed to encode map e319 with expected crc 2023-09-23T20:32:50.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:50 smithi149 ceph-mon[170394]: failed to encode map e319 with expected crc 2023-09-23T20:32:50.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:50 smithi149 ceph-mon[170394]: failed to encode map e319 with expected crc 2023-09-23T20:32:50.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:50 smithi149 ceph-mon[170394]: failed to encode map e319 with expected crc 2023-09-23T20:32:50.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:50 smithi149 ceph-mon[170394]: failed to encode map e319 with expected crc 2023-09-23T20:32:50.968 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:50 smithi149 ceph-mon[170394]: pgmap v737: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 43 MiB/s wr, 684 op/s; 10773/201207 objects degraded (5.354%); 6596/201207 objects misplaced (3.278%); 3.1 MiB/s, 49 objects/s recovering 2023-09-23T20:32:50.968 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:32:50.970 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:50.971 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:32:51.072 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:32:51.098 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:51.134 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:32:51.137 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:51.137 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:32:51.238 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:32:51.265 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2023-09-23T20:32:51.265 INFO:teuthology.orchestra.run.smithi018.stdout: "size": 20, 2023-09-23T20:32:51.265 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 600, 2023-09-23T20:32:51.265 INFO:teuthology.orchestra.run.smithi018.stdout: "ops": [ 2023-09-23T20:32:51.265 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.265 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.34791.0:28889 5.10 5:0f2ad13f:::benchmark_data_smithi018_228451_object28888:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e118)", 2023-09-23T20:32:51.266 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:23:09.972687+0000", 2023-09-23T20:32:51.266 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 581.28949894799996, 2023-09-23T20:32:51.266 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0965630740000001, 2023-09-23T20:32:51.266 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:32:51.266 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:32:51.266 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:32:51.266 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.34791", 2023-09-23T20:32:51.266 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:32:51.267 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 28889 2023-09-23T20:32:51.267 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.267 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:32:51.267 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.267 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:32:51.267 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.972687+0000", 2023-09-23T20:32:51.267 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:32:51.267 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.267 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.268 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:32:51.268 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.972687+0000", 2023-09-23T20:32:51.268 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:32:51.268 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.268 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.268 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:32:51.268 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.972685+0000", 2023-09-23T20:32:51.268 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:32:51.268 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.268 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.269 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:32:51.269 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.972723+0000", 2023-09-23T20:32:51.269 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.7648999999999998e-05 2023-09-23T20:32:51.269 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.269 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.269 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:32:51.269 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.972743+0000", 2023-09-23T20:32:51.269 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.9748000000000001e-05 2023-09-23T20:32:51.269 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.269 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.269 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:32:51.270 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.972754+0000", 2023-09-23T20:32:51.270 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1235e-05 2023-09-23T20:32:51.270 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.270 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.270 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:32:51.270 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:10.007585+0000", 2023-09-23T20:32:51.270 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.034831612999999997 2023-09-23T20:32:51.270 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.270 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.270 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:32:51.270 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:10.007595+0000", 2023-09-23T20:32:51.270 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.5759999999999995e-06 2023-09-23T20:32:51.271 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.271 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.271 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:32:51.271 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.069163+0000", 2023-09-23T20:32:51.271 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0615679520000001 2023-09-23T20:32:51.271 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.271 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.271 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:32:51.271 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.069250+0000", 2023-09-23T20:32:51.271 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.7239999999999998e-05 2023-09-23T20:32:51.271 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:32:51.272 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:32:51.272 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:32:51.272 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.272 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.272 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.34791.0:28905 5.0 5:069f0235:::benchmark_data_smithi018_228451_object28904:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e118)", 2023-09-23T20:32:51.272 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:23:09.973354+0000", 2023-09-23T20:32:51.272 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 581.288831974, 2023-09-23T20:32:51.272 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0987403600000001, 2023-09-23T20:32:51.272 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:32:51.272 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:32:51.272 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:32:51.272 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.34791", 2023-09-23T20:32:51.273 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:32:51.273 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 28905 2023-09-23T20:32:51.273 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.273 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:32:51.273 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.273 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:32:51.273 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.973354+0000", 2023-09-23T20:32:51.273 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:32:51.273 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.273 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.274 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:32:51.274 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.973354+0000", 2023-09-23T20:32:51.274 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:32:51.274 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.274 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.274 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:32:51.274 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.973353+0000", 2023-09-23T20:32:51.274 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:32:51.274 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.274 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.274 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:32:51.275 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.973390+0000", 2023-09-23T20:32:51.275 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.7347e-05 2023-09-23T20:32:51.275 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.275 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.275 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:32:51.275 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.973402+0000", 2023-09-23T20:32:51.275 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2091e-05 2023-09-23T20:32:51.275 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.275 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.275 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:32:51.275 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:09.973411+0000", 2023-09-23T20:32:51.276 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.5140000000000001e-06 2023-09-23T20:32:51.276 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.276 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.276 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:32:51.276 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:10.008100+0000", 2023-09-23T20:32:51.276 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.034689603999999999 2023-09-23T20:32:51.276 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.276 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.276 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:32:51.276 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:10.008109+0000", 2023-09-23T20:32:51.277 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.0499999999999997e-06 2023-09-23T20:32:51.277 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.277 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.277 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:32:51.277 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.066580+0000", 2023-09-23T20:32:51.277 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0584706420000001 2023-09-23T20:32:51.277 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.277 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.277 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:32:51.277 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.066831+0000", 2023-09-23T20:32:51.277 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00025123499999999999 2023-09-23T20:32:51.277 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.278 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.278 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,4", 2023-09-23T20:32:51.278 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.066984+0000", 2023-09-23T20:32:51.278 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00015263599999999999 2023-09-23T20:32:51.278 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.278 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.278 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:32:51.278 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.068869+0000", 2023-09-23T20:32:51.278 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0018853699999999999 2023-09-23T20:32:51.279 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.279 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.279 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:32:51.279 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.071781+0000", 2023-09-23T20:32:51.279 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.002912235 2023-09-23T20:32:51.279 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.279 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.279 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:32:51.279 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.072001+0000", 2023-09-23T20:32:51.279 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000219118 2023-09-23T20:32:51.279 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.280 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.280 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:32:51.280 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.072053+0000", 2023-09-23T20:32:51.280 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.2621000000000003e-05 2023-09-23T20:32:51.280 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.280 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.280 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:32:51.280 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.072094+0000", 2023-09-23T20:32:51.280 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.1186999999999997e-05 2023-09-23T20:32:51.280 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:32:51.280 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:32:51.281 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:32:51.281 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.281 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.281 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.34791.0:28913 5.10 5:08948f05:::benchmark_data_smithi018_228451_object28912:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e118)", 2023-09-23T20:32:51.281 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:23:10.068876+0000", 2023-09-23T20:32:51.281 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 581.19330965200004, 2023-09-23T20:32:51.281 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.00850116, 2023-09-23T20:32:51.281 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:32:51.281 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:32:51.281 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:32:51.281 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.34791", 2023-09-23T20:32:51.282 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:32:51.282 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 28913 2023-09-23T20:32:51.282 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.282 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:32:51.282 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.282 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:32:51.282 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:10.068876+0000", 2023-09-23T20:32:51.282 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:32:51.282 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.282 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.282 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:32:51.283 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:10.068876+0000", 2023-09-23T20:32:51.283 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:32:51.283 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.283 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.283 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:32:51.283 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:10.068875+0000", 2023-09-23T20:32:51.283 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:32:51.283 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.283 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.283 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:32:51.283 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:10.068903+0000", 2023-09-23T20:32:51.283 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.8710000000000001e-05 2023-09-23T20:32:51.284 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.284 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.284 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:32:51.284 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:10.068925+0000", 2023-09-23T20:32:51.284 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.128e-05 2023-09-23T20:32:51.284 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.284 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.284 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:32:51.284 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:10.068936+0000", 2023-09-23T20:32:51.284 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1348999999999999e-05 2023-09-23T20:32:51.284 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.284 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.285 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:32:51.285 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:10.069003+0000", 2023-09-23T20:32:51.285 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.6523e-05 2023-09-23T20:32:51.285 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.285 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.285 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:32:51.285 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:10.069017+0000", 2023-09-23T20:32:51.285 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.4270000000000001e-05 2023-09-23T20:32:51.285 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.285 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.286 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:32:51.286 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.069308+0000", 2023-09-23T20:32:51.286 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0002914000000001 2023-09-23T20:32:51.286 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.286 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.286 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:32:51.286 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.069443+0000", 2023-09-23T20:32:51.286 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000135038 2023-09-23T20:32:51.286 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.286 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.286 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,4", 2023-09-23T20:32:51.286 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.069691+0000", 2023-09-23T20:32:51.287 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00024818099999999999 2023-09-23T20:32:51.287 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.287 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.287 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:32:51.287 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.072098+0000", 2023-09-23T20:32:51.287 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.002407003 2023-09-23T20:32:51.287 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.287 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.287 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:32:51.287 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.072544+0000", 2023-09-23T20:32:51.287 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00044530200000000001 2023-09-23T20:32:51.288 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.288 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.288 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:32:51.288 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.077159+0000", 2023-09-23T20:32:51.288 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0046152789999999999 2023-09-23T20:32:51.288 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.288 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.288 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:32:51.288 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.077338+0000", 2023-09-23T20:32:51.288 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00017922200000000001 2023-09-23T20:32:51.288 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.288 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.289 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:32:51.289 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:23:11.077378+0000", 2023-09-23T20:32:51.289 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.9271999999999999e-05 2023-09-23T20:32:51.289 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:32:51.289 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:32:51.289 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:32:51.289 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.289 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.289 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.34791.0:126924 5.17 5.44e46a57 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e165)", 2023-09-23T20:32:51.289 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:25:01.938216+0000", 2023-09-23T20:32:51.289 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 469.32396968, 2023-09-23T20:32:51.289 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.97596713099999999, 2023-09-23T20:32:51.290 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:32:51.290 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:32:51.290 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:32:51.290 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.34791", 2023-09-23T20:32:51.290 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:32:51.290 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 126924 2023-09-23T20:32:51.290 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.290 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:32:51.290 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.291 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:32:51.291 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:25:01.938216+0000", 2023-09-23T20:32:51.291 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:32:51.291 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.291 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.291 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:32:51.291 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:25:01.938216+0000", 2023-09-23T20:32:51.291 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:32:51.291 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.291 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.291 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:32:51.291 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:25:01.938215+0000", 2023-09-23T20:32:51.292 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:32:51.292 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.292 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.292 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:32:51.292 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:25:01.938221+0000", 2023-09-23T20:32:51.292 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.8599999999999998e-06 2023-09-23T20:32:51.292 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.292 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.292 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:32:51.292 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:25:01.938230+0000", 2023-09-23T20:32:51.292 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.6910000000000003e-06 2023-09-23T20:32:51.292 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.293 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.294 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:32:51.294 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:25:01.938238+0000", 2023-09-23T20:32:51.294 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.3950000000000001e-06 2023-09-23T20:32:51.294 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.294 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.294 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:32:51.294 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:25:01.938299+0000", 2023-09-23T20:32:51.295 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.0990999999999999e-05 2023-09-23T20:32:51.295 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.295 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.295 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:32:51.295 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:25:01.938312+0000", 2023-09-23T20:32:51.295 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2652e-05 2023-09-23T20:32:51.295 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.295 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.295 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:32:51.296 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:25:02.914169+0000", 2023-09-23T20:32:51.296 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.975857631 2023-09-23T20:32:51.296 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.296 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.296 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:32:51.296 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:25:02.914183+0000", 2023-09-23T20:32:51.296 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.4209000000000001e-05 2023-09-23T20:32:51.296 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:32:51.296 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:32:51.296 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:32:51.296 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.296 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.297 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.45161.0:68466 6.c 6:320f8bfb:::benchmark_data_smithi018_270558_object68465:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e230)", 2023-09-23T20:32:51.297 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:28:04.042968+0000", 2023-09-23T20:32:51.297 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 287.21921758600001, 2023-09-23T20:32:51.297 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.1849001010000002, 2023-09-23T20:32:51.297 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:32:51.297 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:32:51.297 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:32:51.297 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.45161", 2023-09-23T20:32:51.297 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3795614987", 2023-09-23T20:32:51.298 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 68466 2023-09-23T20:32:51.298 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.298 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:32:51.298 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.298 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:32:51.298 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:04.042968+0000", 2023-09-23T20:32:51.298 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:32:51.298 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.298 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.298 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:32:51.298 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:04.042968+0000", 2023-09-23T20:32:51.299 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:32:51.299 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.299 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.299 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:32:51.299 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:04.042967+0000", 2023-09-23T20:32:51.299 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:32:51.299 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.299 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.299 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:32:51.299 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:04.043005+0000", 2023-09-23T20:32:51.300 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.7880999999999999e-05 2023-09-23T20:32:51.300 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.300 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.300 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:32:51.300 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:04.043021+0000", 2023-09-23T20:32:51.300 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.5979999999999999e-05 2023-09-23T20:32:51.300 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.300 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.300 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:32:51.300 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:04.043030+0000", 2023-09-23T20:32:51.300 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.5910000000000002e-06 2023-09-23T20:32:51.301 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.301 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.301 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:32:51.301 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:04.043170+0000", 2023-09-23T20:32:51.301 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00014017599999999999 2023-09-23T20:32:51.301 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.301 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.301 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:32:51.301 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:04.043185+0000", 2023-09-23T20:32:51.301 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.4789e-05 2023-09-23T20:32:51.302 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.302 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.302 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:32:51.302 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:05.022058+0000", 2023-09-23T20:32:51.302 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.97887254400000001 2023-09-23T20:32:51.302 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.302 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.302 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:32:51.302 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:05.022071+0000", 2023-09-23T20:32:51.302 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3016999999999999e-05 2023-09-23T20:32:51.302 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.303 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.303 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:32:51.303 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:06.207745+0000", 2023-09-23T20:32:51.303 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1856746280000001 2023-09-23T20:32:51.303 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.303 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.303 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:32:51.303 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:06.207958+0000", 2023-09-23T20:32:51.303 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000212408 2023-09-23T20:32:51.303 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.304 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.304 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 5,7", 2023-09-23T20:32:51.304 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:06.208139+0000", 2023-09-23T20:32:51.304 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00018073899999999999 2023-09-23T20:32:51.304 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.304 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.304 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:32:51.304 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:06.215444+0000", 2023-09-23T20:32:51.304 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0073050779999999996 2023-09-23T20:32:51.304 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.304 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.305 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:32:51.305 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:06.215482+0000", 2023-09-23T20:32:51.305 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.7985999999999998e-05 2023-09-23T20:32:51.305 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.305 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.305 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:32:51.305 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:06.225399+0000", 2023-09-23T20:32:51.305 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0099177599999999994 2023-09-23T20:32:51.305 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.305 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.305 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:32:51.306 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:06.225470+0000", 2023-09-23T20:32:51.306 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.0081999999999997e-05 2023-09-23T20:32:51.306 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.307 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.307 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:32:51.307 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:06.227869+0000", 2023-09-23T20:32:51.307 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0023990259999999998 2023-09-23T20:32:51.307 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:32:51.307 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:32:51.307 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:32:51.307 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.307 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.307 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.45161.0:68467 6.c 6:33c0ab26:::benchmark_data_smithi018_270558_object68466:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e230)", 2023-09-23T20:32:51.307 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:28:04.043047+0000", 2023-09-23T20:32:51.308 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 287.21913875600001, 2023-09-23T20:32:51.308 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.1959208280000002, 2023-09-23T20:32:51.308 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:32:51.308 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:32:51.308 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:32:51.308 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.45161", 2023-09-23T20:32:51.308 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3795614987", 2023-09-23T20:32:51.308 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 68467 2023-09-23T20:32:51.308 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.308 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:32:51.308 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.308 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:32:51.309 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:04.043047+0000", 2023-09-23T20:32:51.309 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:32:51.309 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.309 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.309 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:32:51.309 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:04.043047+0000", 2023-09-23T20:32:51.309 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:32:51.309 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.309 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.309 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:32:51.309 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:04.043046+0000", 2023-09-23T20:32:51.310 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.999999 2023-09-23T20:32:51.310 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.310 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.310 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:32:51.310 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:04.043101+0000", 2023-09-23T20:32:51.310 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.5018000000000001e-05 2023-09-23T20:32:51.310 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.310 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.310 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:32:51.310 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:04.043111+0000", 2023-09-23T20:32:51.311 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.4590000000000004e-06 2023-09-23T20:32:51.311 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.311 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.311 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:32:51.311 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:04.043118+0000", 2023-09-23T20:32:51.311 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.6340000000000003e-06 2023-09-23T20:32:51.311 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.311 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.312 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:32:51.312 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:04.043212+0000", 2023-09-23T20:32:51.312 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.399e-05 2023-09-23T20:32:51.312 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.312 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.312 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:32:51.312 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:04.043221+0000", 2023-09-23T20:32:51.312 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.6579999999999997e-06 2023-09-23T20:32:51.312 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.312 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.312 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:32:51.312 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:05.022095+0000", 2023-09-23T20:32:51.313 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.97887364200000004 2023-09-23T20:32:51.313 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.313 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.313 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:32:51.313 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:05.022103+0000", 2023-09-23T20:32:51.313 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.0090000000000007e-06 2023-09-23T20:32:51.313 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.313 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.314 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:32:51.314 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:06.213032+0000", 2023-09-23T20:32:51.314 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1909290669999999 2023-09-23T20:32:51.314 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.314 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.314 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:32:51.314 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:06.213177+0000", 2023-09-23T20:32:51.314 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00014537999999999999 2023-09-23T20:32:51.314 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.314 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.314 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 5,7", 2023-09-23T20:32:51.315 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:06.213344+0000", 2023-09-23T20:32:51.315 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00016685100000000001 2023-09-23T20:32:51.315 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.315 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.315 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:32:51.315 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:06.225333+0000", 2023-09-23T20:32:51.315 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.011988403999999999 2023-09-23T20:32:51.315 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.316 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.316 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:32:51.316 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:06.227963+0000", 2023-09-23T20:32:51.316 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0026302629999999999 2023-09-23T20:32:51.316 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.316 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.316 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:32:51.316 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:06.235942+0000", 2023-09-23T20:32:51.316 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0079790829999999997 2023-09-23T20:32:51.316 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.316 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.317 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:32:51.317 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:06.236003+0000", 2023-09-23T20:32:51.317 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.0649000000000002e-05 2023-09-23T20:32:51.317 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.317 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.317 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:32:51.317 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:06.238968+0000", 2023-09-23T20:32:51.318 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0029655139999999998 2023-09-23T20:32:51.318 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:32:51.318 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:32:51.318 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:32:51.318 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.318 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.318 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.45161.0:68472 6.c 6:3a94d71c:::benchmark_data_smithi018_270558_object68471:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e230)", 2023-09-23T20:32:51.318 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:28:04.058107+0000", 2023-09-23T20:32:51.318 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 287.20407934500003, 2023-09-23T20:32:51.318 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.1836921600000001, 2023-09-23T20:32:51.319 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:32:51.319 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:32:51.319 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:32:51.319 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.45161", 2023-09-23T20:32:51.319 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3795614987", 2023-09-23T20:32:51.319 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 68472 2023-09-23T20:32:51.319 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.319 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:32:51.320 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.320 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:32:51.320 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:04.058107+0000", 2023-09-23T20:32:51.320 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:32:51.320 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.320 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.320 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:32:51.320 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:04.058107+0000", 2023-09-23T20:32:51.321 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:32:51.321 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.321 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.321 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:32:51.321 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:04.058106+0000", 2023-09-23T20:32:51.321 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.999999 2023-09-23T20:32:51.321 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.321 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.321 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:32:51.322 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:04.058127+0000", 2023-09-23T20:32:51.322 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.143e-05 2023-09-23T20:32:51.322 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.322 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.322 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:32:51.322 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:04.058147+0000", 2023-09-23T20:32:51.322 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.0217999999999999e-05 2023-09-23T20:32:51.322 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.322 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.322 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:32:51.322 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:04.058154+0000", 2023-09-23T20:32:51.323 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.8229999999999997e-06 2023-09-23T20:32:51.323 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.323 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.323 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:32:51.323 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:04.058206+0000", 2023-09-23T20:32:51.323 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.2494000000000003e-05 2023-09-23T20:32:51.323 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.323 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.323 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:32:51.323 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:04.058216+0000", 2023-09-23T20:32:51.323 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.5189999999999998e-06 2023-09-23T20:32:51.324 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.324 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.324 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:32:51.324 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:05.022124+0000", 2023-09-23T20:32:51.324 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.96390836099999999 2023-09-23T20:32:51.324 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.324 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.324 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:32:51.324 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:05.022132+0000", 2023-09-23T20:32:51.324 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.0099999999999995e-06 2023-09-23T20:32:51.324 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.325 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.325 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:32:51.325 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:06.215585+0000", 2023-09-23T20:32:51.325 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1934526110000001 2023-09-23T20:32:51.325 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.325 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.325 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:32:51.326 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:06.215718+0000", 2023-09-23T20:32:51.326 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000132941 2023-09-23T20:32:51.326 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.326 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.326 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 5,7", 2023-09-23T20:32:51.326 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:06.215888+0000", 2023-09-23T20:32:51.326 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000169908 2023-09-23T20:32:51.326 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.326 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.327 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:32:51.327 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:06.225349+0000", 2023-09-23T20:32:51.327 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0094613979999999993 2023-09-23T20:32:51.327 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.327 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.327 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:32:51.327 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:06.228020+0000", 2023-09-23T20:32:51.327 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0026708610000000001 2023-09-23T20:32:51.327 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.327 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.327 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:32:51.327 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:06.239066+0000", 2023-09-23T20:32:51.328 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.011046344 2023-09-23T20:32:51.328 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.328 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.328 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:32:51.328 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:06.239126+0000", 2023-09-23T20:32:51.328 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.9778999999999998e-05 2023-09-23T20:32:51.328 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.328 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.328 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:32:51.328 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:06.241799+0000", 2023-09-23T20:32:51.328 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0026726219999999999 2023-09-23T20:32:51.328 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:32:51.329 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:32:51.329 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:32:51.329 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.329 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.329 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.45161.0:68480 6.c 6:3eaf7db7:::benchmark_data_smithi018_270558_object68479:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e230)", 2023-09-23T20:32:51.329 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:28:04.068993+0000", 2023-09-23T20:32:51.329 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 287.19319261700002, 2023-09-23T20:32:51.329 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.1783830000000002, 2023-09-23T20:32:51.329 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:32:51.329 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:32:51.329 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:32:51.329 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.45161", 2023-09-23T20:32:51.330 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3795614987", 2023-09-23T20:32:51.330 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 68480 2023-09-23T20:32:51.330 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.330 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:32:51.330 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.330 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:32:51.330 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:04.068993+0000", 2023-09-23T20:32:51.330 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:32:51.330 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.330 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.330 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:32:51.331 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:04.068993+0000", 2023-09-23T20:32:51.331 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:32:51.331 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.331 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.331 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:32:51.331 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:04.068986+0000", 2023-09-23T20:32:51.331 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999924 2023-09-23T20:32:51.331 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.331 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.331 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:32:51.331 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:04.069020+0000", 2023-09-23T20:32:51.331 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.4041000000000002e-05 2023-09-23T20:32:51.332 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.332 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.332 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:32:51.332 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:04.069034+0000", 2023-09-23T20:32:51.332 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3719e-05 2023-09-23T20:32:51.332 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.332 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.332 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:32:51.332 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:04.069043+0000", 2023-09-23T20:32:51.332 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.8429999999999999e-06 2023-09-23T20:32:51.332 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.332 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.333 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:32:51.333 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:04.069107+0000", 2023-09-23T20:32:51.333 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.3947000000000003e-05 2023-09-23T20:32:51.333 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.333 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.333 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:32:51.333 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:04.069119+0000", 2023-09-23T20:32:51.333 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2758999999999999e-05 2023-09-23T20:32:51.334 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.334 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.334 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:32:51.334 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:05.022156+0000", 2023-09-23T20:32:51.334 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.95303622600000004 2023-09-23T20:32:51.334 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.334 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.334 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:32:51.334 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:05.022164+0000", 2023-09-23T20:32:51.334 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.977e-06 2023-09-23T20:32:51.334 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.335 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.335 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:32:51.335 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:06.217835+0000", 2023-09-23T20:32:51.335 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.195671851 2023-09-23T20:32:51.335 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.335 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.335 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:32:51.335 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:06.217933+0000", 2023-09-23T20:32:51.335 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.7876999999999998e-05 2023-09-23T20:32:51.335 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.336 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.336 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 5,7", 2023-09-23T20:32:51.336 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:06.218034+0000", 2023-09-23T20:32:51.336 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00010027900000000001 2023-09-23T20:32:51.336 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.336 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.336 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:32:51.336 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:06.227890+0000", 2023-09-23T20:32:51.336 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0098562379999999998 2023-09-23T20:32:51.336 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.337 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.337 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:32:51.337 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:06.228062+0000", 2023-09-23T20:32:51.337 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00017186299999999999 2023-09-23T20:32:51.337 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.337 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.337 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:32:51.337 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:06.245432+0000", 2023-09-23T20:32:51.337 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.017370072 2023-09-23T20:32:51.337 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.337 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.338 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:32:51.338 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:06.245492+0000", 2023-09-23T20:32:51.338 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.0316999999999999e-05 2023-09-23T20:32:51.338 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.338 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.338 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:32:51.338 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:06.247376+0000", 2023-09-23T20:32:51.338 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0018844000000000001 2023-09-23T20:32:51.338 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:32:51.339 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:32:51.339 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:32:51.339 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.339 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.339 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.45161.0:68499 6.c 6:3ab32f48:::benchmark_data_smithi018_270558_object68498:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e230)", 2023-09-23T20:32:51.339 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:28:04.092063+0000", 2023-09-23T20:32:51.339 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 287.17012253899998, 2023-09-23T20:32:51.339 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.1571343239999998, 2023-09-23T20:32:51.339 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:32:51.339 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:32:51.340 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:32:51.340 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.45161", 2023-09-23T20:32:51.340 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3795614987", 2023-09-23T20:32:51.340 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 68499 2023-09-23T20:32:51.340 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.340 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:32:51.340 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.340 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:32:51.340 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:04.092063+0000", 2023-09-23T20:32:51.340 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:32:51.341 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.341 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.341 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:32:51.341 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:04.092063+0000", 2023-09-23T20:32:51.341 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:32:51.341 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.341 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.341 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:32:51.341 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:04.092062+0000", 2023-09-23T20:32:51.342 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:32:51.342 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.342 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.342 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:32:51.342 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:04.092088+0000", 2023-09-23T20:32:51.342 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.5961000000000001e-05 2023-09-23T20:32:51.342 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.342 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.342 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:32:51.343 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:04.092105+0000", 2023-09-23T20:32:51.343 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.6957999999999999e-05 2023-09-23T20:32:51.343 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.343 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.343 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:32:51.343 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:04.092113+0000", 2023-09-23T20:32:51.343 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.8110000000000003e-06 2023-09-23T20:32:51.343 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.343 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.343 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:32:51.343 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:04.092184+0000", 2023-09-23T20:32:51.343 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.1218999999999997e-05 2023-09-23T20:32:51.344 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.344 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.344 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:32:51.344 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:04.092199+0000", 2023-09-23T20:32:51.344 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.5506999999999999e-05 2023-09-23T20:32:51.344 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.344 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.344 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:32:51.344 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:05.022185+0000", 2023-09-23T20:32:51.344 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.92998589899999995 2023-09-23T20:32:51.344 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.344 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.345 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:32:51.345 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:05.022193+0000", 2023-09-23T20:32:51.345 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.8650000000000001e-06 2023-09-23T20:32:51.345 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.345 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.345 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:32:51.345 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:06.220364+0000", 2023-09-23T20:32:51.345 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1981710510000001 2023-09-23T20:32:51.345 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.345 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.345 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:32:51.345 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:06.220506+0000", 2023-09-23T20:32:51.346 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000142046 2023-09-23T20:32:51.346 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.346 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.346 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 5,7", 2023-09-23T20:32:51.346 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:06.220664+0000", 2023-09-23T20:32:51.346 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00015750000000000001 2023-09-23T20:32:51.346 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.346 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.347 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:32:51.347 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:06.227899+0000", 2023-09-23T20:32:51.347 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0072355859999999996 2023-09-23T20:32:51.347 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.347 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.347 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:32:51.347 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:06.228196+0000", 2023-09-23T20:32:51.347 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00029671400000000001 2023-09-23T20:32:51.347 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.347 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.348 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:32:51.348 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:06.247435+0000", 2023-09-23T20:32:51.348 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.019239136 2023-09-23T20:32:51.348 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.348 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.348 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:32:51.348 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:06.247483+0000", 2023-09-23T20:32:51.348 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.8130000000000002e-05 2023-09-23T20:32:51.348 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.348 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.348 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:32:51.349 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:06.249198+0000", 2023-09-23T20:32:51.349 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0017144510000000001 2023-09-23T20:32:51.349 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:32:51.349 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:32:51.349 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:32:51.349 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.349 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.349 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.45161.0:136070 6.16 6:6af4a005:::benchmark_data_smithi018_270558_object31331:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e258)", 2023-09-23T20:32:51.349 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:29:27.396634+0000", 2023-09-23T20:32:51.349 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 203.865551727, 2023-09-23T20:32:51.350 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.1461166540000001, 2023-09-23T20:32:51.350 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:32:51.350 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:32:51.350 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:32:51.350 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.45161", 2023-09-23T20:32:51.350 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3795614987", 2023-09-23T20:32:51.350 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 136070 2023-09-23T20:32:51.350 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.350 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:32:51.350 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.350 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:32:51.351 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.396634+0000", 2023-09-23T20:32:51.351 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:32:51.351 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.351 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.351 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:32:51.351 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.396634+0000", 2023-09-23T20:32:51.351 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:32:51.351 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.352 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.352 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:32:51.352 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.396633+0000", 2023-09-23T20:32:51.352 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.999999 2023-09-23T20:32:51.352 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.352 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.352 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:32:51.352 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.396639+0000", 2023-09-23T20:32:51.352 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.6099999999999997e-06 2023-09-23T20:32:51.352 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.352 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.353 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:32:51.353 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.396654+0000", 2023-09-23T20:32:51.353 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.5432999999999999e-05 2023-09-23T20:32:51.353 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.353 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.353 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:32:51.353 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.396661+0000", 2023-09-23T20:32:51.353 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.4799999999999998e-06 2023-09-23T20:32:51.353 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.354 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.354 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:32:51.354 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.396728+0000", 2023-09-23T20:32:51.354 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.6884999999999994e-05 2023-09-23T20:32:51.354 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.354 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.354 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:32:51.354 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.396741+0000", 2023-09-23T20:32:51.354 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3933e-05 2023-09-23T20:32:51.354 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.355 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.355 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:32:51.355 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:28.349683+0000", 2023-09-23T20:32:51.355 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.95294156299999999 2023-09-23T20:32:51.355 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.355 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.355 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:32:51.355 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:28.349696+0000", 2023-09-23T20:32:51.355 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3191e-05 2023-09-23T20:32:51.356 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.356 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.356 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:32:51.356 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.535166+0000", 2023-09-23T20:32:51.356 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1854697679999999 2023-09-23T20:32:51.356 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.356 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.356 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:32:51.356 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.535254+0000", 2023-09-23T20:32:51.356 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.7965000000000002e-05 2023-09-23T20:32:51.357 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.357 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.357 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 4,6,7", 2023-09-23T20:32:51.357 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.535550+0000", 2023-09-23T20:32:51.357 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00029616100000000003 2023-09-23T20:32:51.357 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.357 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.357 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:32:51.357 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.538940+0000", 2023-09-23T20:32:51.358 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.003389651 2023-09-23T20:32:51.358 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.358 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.358 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:32:51.358 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.538996+0000", 2023-09-23T20:32:51.358 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.6674e-05 2023-09-23T20:32:51.358 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.358 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.358 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:32:51.358 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.540647+0000", 2023-09-23T20:32:51.359 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0016507220000000001 2023-09-23T20:32:51.359 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.359 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.359 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:32:51.359 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.542633+0000", 2023-09-23T20:32:51.359 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0019862809999999999 2023-09-23T20:32:51.359 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.359 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.360 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:32:51.360 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.542666+0000", 2023-09-23T20:32:51.360 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.2740999999999997e-05 2023-09-23T20:32:51.360 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.360 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.360 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:32:51.360 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.542751+0000", 2023-09-23T20:32:51.360 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.4754999999999997e-05 2023-09-23T20:32:51.360 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:32:51.360 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:32:51.360 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:32:51.360 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.361 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.361 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.45161.0:136077 6.16 6:6b87af5f:::benchmark_data_smithi018_270558_object31338:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e258)", 2023-09-23T20:32:51.361 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:29:27.409208+0000", 2023-09-23T20:32:51.361 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 203.85297753099999, 2023-09-23T20:32:51.361 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.1337409109999999, 2023-09-23T20:32:51.361 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:32:51.361 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:32:51.361 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:32:51.361 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.45161", 2023-09-23T20:32:51.361 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3795614987", 2023-09-23T20:32:51.361 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 136077 2023-09-23T20:32:51.362 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.362 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:32:51.362 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.362 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:32:51.362 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.409208+0000", 2023-09-23T20:32:51.362 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:32:51.362 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.362 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.362 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:32:51.362 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.409208+0000", 2023-09-23T20:32:51.362 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:32:51.362 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.363 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.363 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:32:51.363 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.409207+0000", 2023-09-23T20:32:51.363 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:32:51.363 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.363 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.363 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:32:51.363 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.409214+0000", 2023-09-23T20:32:51.363 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.3930000000000002e-06 2023-09-23T20:32:51.363 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.363 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.363 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:32:51.363 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.409229+0000", 2023-09-23T20:32:51.364 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.4535000000000001e-05 2023-09-23T20:32:51.364 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.364 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.364 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:32:51.364 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.409238+0000", 2023-09-23T20:32:51.364 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.5899999999999997e-06 2023-09-23T20:32:51.364 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.364 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.364 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:32:51.364 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.409307+0000", 2023-09-23T20:32:51.364 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.9023000000000006e-05 2023-09-23T20:32:51.364 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.365 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.365 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:32:51.365 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.409318+0000", 2023-09-23T20:32:51.365 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0387999999999999e-05 2023-09-23T20:32:51.365 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.365 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.365 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:32:51.365 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:28.349722+0000", 2023-09-23T20:32:51.365 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.94040487299999997 2023-09-23T20:32:51.365 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.365 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.366 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:32:51.366 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:28.349733+0000", 2023-09-23T20:32:51.366 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0152e-05 2023-09-23T20:32:51.366 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.366 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.366 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:32:51.366 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.536807+0000", 2023-09-23T20:32:51.366 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.187074006 2023-09-23T20:32:51.366 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.366 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.366 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:32:51.367 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.536968+0000", 2023-09-23T20:32:51.367 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00016101399999999999 2023-09-23T20:32:51.367 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.367 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.367 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 4,6,7", 2023-09-23T20:32:51.367 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.537271+0000", 2023-09-23T20:32:51.367 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000302887 2023-09-23T20:32:51.367 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.367 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.367 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:32:51.367 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.539134+0000", 2023-09-23T20:32:51.367 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.001863611 2023-09-23T20:32:51.368 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.368 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.368 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:32:51.368 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.542232+0000", 2023-09-23T20:32:51.368 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.003097533 2023-09-23T20:32:51.368 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.368 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.368 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:32:51.368 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.542819+0000", 2023-09-23T20:32:51.368 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00058755999999999999 2023-09-23T20:32:51.368 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.368 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.369 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:32:51.369 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.542846+0000", 2023-09-23T20:32:51.369 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.6855e-05 2023-09-23T20:32:51.369 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.369 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.369 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:32:51.369 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.542874+0000", 2023-09-23T20:32:51.369 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.7798e-05 2023-09-23T20:32:51.369 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.369 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.369 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:32:51.369 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.542949+0000", 2023-09-23T20:32:51.370 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.5531999999999994e-05 2023-09-23T20:32:51.370 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:32:51.370 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:32:51.370 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:32:51.370 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.370 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.370 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:57605 7.3 7:c29712e0:::benchmark_data_smithi018_308601_object57604:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e311)", 2023-09-23T20:32:51.370 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:32:32.219481+0000", 2023-09-23T20:32:51.370 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 19.042704949000001, 2023-09-23T20:32:51.370 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.3860234459999998, 2023-09-23T20:32:51.370 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:32:51.371 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:32:51.371 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:32:51.371 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:32:51.371 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:32:51.371 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 57605 2023-09-23T20:32:51.371 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.371 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:32:51.371 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.371 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:32:51.371 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.219481+0000", 2023-09-23T20:32:51.371 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:32:51.371 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.372 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.372 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:32:51.372 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.219481+0000", 2023-09-23T20:32:51.372 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:32:51.372 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.372 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.372 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:32:51.372 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.219480+0000", 2023-09-23T20:32:51.372 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:32:51.372 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.372 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.372 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:32:51.373 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.219513+0000", 2023-09-23T20:32:51.373 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.3324999999999997e-05 2023-09-23T20:32:51.373 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.373 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.373 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:32:51.373 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.219526+0000", 2023-09-23T20:32:51.373 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3145999999999999e-05 2023-09-23T20:32:51.373 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.373 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.373 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:32:51.373 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.219534+0000", 2023-09-23T20:32:51.373 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.729e-06 2023-09-23T20:32:51.373 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.374 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.374 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:32:51.374 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.231964+0000", 2023-09-23T20:32:51.374 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.012430274999999999 2023-09-23T20:32:51.374 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.374 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.374 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:32:51.374 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.231974+0000", 2023-09-23T20:32:51.374 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.4939999999999992e-06 2023-09-23T20:32:51.374 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.374 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.375 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:32:51.375 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.215331+0000", 2023-09-23T20:32:51.375 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.983357276 2023-09-23T20:32:51.375 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.375 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.375 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:32:51.375 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.215344+0000", 2023-09-23T20:32:51.375 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3061000000000001e-05 2023-09-23T20:32:51.375 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.375 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.375 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:32:51.376 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.573779+0000", 2023-09-23T20:32:51.376 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.358434581 2023-09-23T20:32:51.376 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.376 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.376 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:32:51.376 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.573953+0000", 2023-09-23T20:32:51.376 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00017463999999999999 2023-09-23T20:32:51.376 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.376 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.376 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 2,4", 2023-09-23T20:32:51.376 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.574063+0000", 2023-09-23T20:32:51.376 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000110059 2023-09-23T20:32:51.377 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.377 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.377 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:32:51.377 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.578839+0000", 2023-09-23T20:32:51.377 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0047753709999999996 2023-09-23T20:32:51.377 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.377 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.377 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:32:51.377 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.578852+0000", 2023-09-23T20:32:51.377 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3360000000000001e-05 2023-09-23T20:32:51.377 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.377 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.378 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:32:51.378 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.602935+0000", 2023-09-23T20:32:51.378 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.024082883999999999 2023-09-23T20:32:51.378 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.378 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.378 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:32:51.378 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.602997+0000", 2023-09-23T20:32:51.378 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.2113000000000007e-05 2023-09-23T20:32:51.378 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.378 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.378 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:32:51.379 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.605505+0000", 2023-09-23T20:32:51.379 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.002507493 2023-09-23T20:32:51.379 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:32:51.379 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:32:51.379 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:32:51.379 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.379 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.379 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:57610 7.3 7:cbaa52d6:::benchmark_data_smithi018_308601_object57609:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e311)", 2023-09-23T20:32:51.379 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:32:32.219584+0000", 2023-09-23T20:32:51.379 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 19.042602314, 2023-09-23T20:32:51.379 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.38833787, 2023-09-23T20:32:51.379 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:32:51.379 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:32:51.380 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:32:51.380 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:32:51.380 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:32:51.380 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 57610 2023-09-23T20:32:51.380 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.380 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:32:51.380 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.381 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:32:51.381 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.219584+0000", 2023-09-23T20:32:51.381 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:32:51.381 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.381 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.381 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:32:51.381 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.219584+0000", 2023-09-23T20:32:51.381 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:32:51.381 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.381 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.381 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:32:51.381 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.219583+0000", 2023-09-23T20:32:51.382 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:32:51.382 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.382 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.382 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:32:51.382 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.219607+0000", 2023-09-23T20:32:51.382 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.4170000000000001e-05 2023-09-23T20:32:51.382 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.382 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.382 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:32:51.382 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.219614+0000", 2023-09-23T20:32:51.383 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.4790000000000001e-06 2023-09-23T20:32:51.383 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.383 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.383 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:32:51.383 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.219619+0000", 2023-09-23T20:32:51.383 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.079e-06 2023-09-23T20:32:51.383 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.383 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.383 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:32:51.383 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.231994+0000", 2023-09-23T20:32:51.384 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.012374484999999999 2023-09-23T20:32:51.384 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.384 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.384 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:32:51.384 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.232000+0000", 2023-09-23T20:32:51.384 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.4790000000000001e-06 2023-09-23T20:32:51.384 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.384 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.384 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:32:51.384 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.215369+0000", 2023-09-23T20:32:51.384 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.98336916699999999 2023-09-23T20:32:51.385 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.385 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.385 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:32:51.385 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.215378+0000", 2023-09-23T20:32:51.385 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.6479999999999992e-06 2023-09-23T20:32:51.385 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.385 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.385 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:32:51.385 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.577198+0000", 2023-09-23T20:32:51.385 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.36182066 2023-09-23T20:32:51.386 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.386 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.386 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:32:51.386 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.577312+0000", 2023-09-23T20:32:51.386 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000113257 2023-09-23T20:32:51.386 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.386 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.386 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 2,4", 2023-09-23T20:32:51.386 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.577424+0000", 2023-09-23T20:32:51.386 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00011234999999999999 2023-09-23T20:32:51.387 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.387 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.387 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:32:51.387 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.580718+0000", 2023-09-23T20:32:51.387 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0032940090000000001 2023-09-23T20:32:51.387 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.387 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.387 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:32:51.387 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.582068+0000", 2023-09-23T20:32:51.387 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0013497990000000001 2023-09-23T20:32:51.388 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.388 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.388 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:32:51.388 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.605587+0000", 2023-09-23T20:32:51.388 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.023518779 2023-09-23T20:32:51.388 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.388 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.388 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:32:51.388 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.605666+0000", 2023-09-23T20:32:51.389 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.9777999999999994e-05 2023-09-23T20:32:51.389 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.389 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.389 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:32:51.389 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.607922+0000", 2023-09-23T20:32:51.389 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00225515 2023-09-23T20:32:51.389 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:32:51.389 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:32:51.389 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:32:51.389 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.389 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.390 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:57645 7.3 7:cb6be1ad:::benchmark_data_smithi018_308601_object57644:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e312)", 2023-09-23T20:32:51.390 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:32:32.275903+0000", 2023-09-23T20:32:51.390 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 18.98628296, 2023-09-23T20:32:51.390 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.3348626370000001, 2023-09-23T20:32:51.390 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:32:51.390 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:32:51.390 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:32:51.390 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:32:51.390 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:32:51.391 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 57645 2023-09-23T20:32:51.391 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.391 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:32:51.391 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.391 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:32:51.391 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.275903+0000", 2023-09-23T20:32:51.391 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:32:51.391 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.391 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.392 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:32:51.392 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.275903+0000", 2023-09-23T20:32:51.392 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:32:51.392 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.392 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.392 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:32:51.392 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.275902+0000", 2023-09-23T20:32:51.392 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:32:51.392 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.393 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.393 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:32:51.393 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.275926+0000", 2023-09-23T20:32:51.393 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.4209000000000001e-05 2023-09-23T20:32:51.393 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.393 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.393 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:32:51.393 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.275938+0000", 2023-09-23T20:32:51.393 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1992e-05 2023-09-23T20:32:51.394 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.394 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.394 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:32:51.394 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.275946+0000", 2023-09-23T20:32:51.394 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.8839999999999994e-06 2023-09-23T20:32:51.394 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.394 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.394 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:32:51.394 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.276851+0000", 2023-09-23T20:32:51.394 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00090492800000000005 2023-09-23T20:32:51.394 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.394 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.395 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:32:51.395 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.276861+0000", 2023-09-23T20:32:51.395 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0208e-05 2023-09-23T20:32:51.395 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.395 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.395 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:32:51.395 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.215401+0000", 2023-09-23T20:32:51.395 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.93854029800000005 2023-09-23T20:32:51.395 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.395 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.395 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:32:51.395 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.215410+0000", 2023-09-23T20:32:51.396 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.3510000000000002e-06 2023-09-23T20:32:51.396 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.396 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.396 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:32:51.396 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.578900+0000", 2023-09-23T20:32:51.396 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.363490173 2023-09-23T20:32:51.396 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.396 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.396 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:32:51.396 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.579070+0000", 2023-09-23T20:32:51.396 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00017048899999999999 2023-09-23T20:32:51.396 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.397 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.397 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 2,4", 2023-09-23T20:32:51.397 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.579260+0000", 2023-09-23T20:32:51.397 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000189667 2023-09-23T20:32:51.397 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.397 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.397 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:32:51.397 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.583356+0000", 2023-09-23T20:32:51.397 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0040960520000000002 2023-09-23T20:32:51.397 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.397 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.397 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:32:51.398 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.583406+0000", 2023-09-23T20:32:51.398 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.9449e-05 2023-09-23T20:32:51.398 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.398 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.398 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:32:51.398 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.607983+0000", 2023-09-23T20:32:51.398 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.024577576 2023-09-23T20:32:51.398 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.398 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.398 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:32:51.398 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.608038+0000", 2023-09-23T20:32:51.398 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.4787000000000002e-05 2023-09-23T20:32:51.399 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.399 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.399 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:32:51.399 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.610766+0000", 2023-09-23T20:32:51.399 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0027278319999999999 2023-09-23T20:32:51.399 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:32:51.399 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:32:51.399 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:32:51.399 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.399 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.399 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:57662 7.3 7:ce017a4a:::benchmark_data_smithi018_308601_object57661:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e312)", 2023-09-23T20:32:51.399 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:32:32.295077+0000", 2023-09-23T20:32:51.400 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 18.967108883000002, 2023-09-23T20:32:51.400 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.3221079059999998, 2023-09-23T20:32:51.400 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:32:51.400 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:32:51.400 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:32:51.400 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:32:51.400 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:32:51.400 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 57662 2023-09-23T20:32:51.400 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.400 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:32:51.400 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.400 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:32:51.401 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.295077+0000", 2023-09-23T20:32:51.401 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:32:51.401 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.401 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.401 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:32:51.401 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.295077+0000", 2023-09-23T20:32:51.401 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:32:51.401 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.401 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.401 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:32:51.401 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.295075+0000", 2023-09-23T20:32:51.401 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:32:51.402 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.402 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.402 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:32:51.402 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.295104+0000", 2023-09-23T20:32:51.402 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.8374999999999999e-05 2023-09-23T20:32:51.402 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.402 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.402 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:32:51.402 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.295117+0000", 2023-09-23T20:32:51.402 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2943e-05 2023-09-23T20:32:51.402 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.403 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.403 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:32:51.403 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.295123+0000", 2023-09-23T20:32:51.403 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.5989999999999998e-06 2023-09-23T20:32:51.403 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.403 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.403 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:32:51.403 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.295189+0000", 2023-09-23T20:32:51.403 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.5988000000000006e-05 2023-09-23T20:32:51.403 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.403 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.403 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:32:51.404 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.295205+0000", 2023-09-23T20:32:51.404 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.5434000000000002e-05 2023-09-23T20:32:51.404 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.404 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.404 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:32:51.404 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.215432+0000", 2023-09-23T20:32:51.404 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.92022715499999996 2023-09-23T20:32:51.404 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.404 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.404 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:32:51.404 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.215440+0000", 2023-09-23T20:32:51.405 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.2519999999999999e-06 2023-09-23T20:32:51.405 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.405 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.405 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:32:51.405 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.580753+0000", 2023-09-23T20:32:51.405 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.365312673 2023-09-23T20:32:51.405 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.405 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.405 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:32:51.405 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.580896+0000", 2023-09-23T20:32:51.405 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00014352000000000001 2023-09-23T20:32:51.405 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.406 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.406 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 2,4", 2023-09-23T20:32:51.406 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.581023+0000", 2023-09-23T20:32:51.406 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000127112 2023-09-23T20:32:51.406 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.406 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.406 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:32:51.406 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.583459+0000", 2023-09-23T20:32:51.406 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0024356450000000002 2023-09-23T20:32:51.406 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.406 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.406 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:32:51.407 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.583468+0000", 2023-09-23T20:32:51.407 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.4840000000000004e-06 2023-09-23T20:32:51.407 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.407 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.407 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:32:51.407 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.613992+0000", 2023-09-23T20:32:51.407 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.030524193000000002 2023-09-23T20:32:51.407 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.407 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.407 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:32:51.407 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.614070+0000", 2023-09-23T20:32:51.407 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.8054999999999997e-05 2023-09-23T20:32:51.408 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.408 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.408 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:32:51.408 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.617185+0000", 2023-09-23T20:32:51.408 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0031151690000000001 2023-09-23T20:32:51.408 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:32:51.408 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:32:51.408 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:32:51.408 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.408 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.408 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:57667 7.3 7:cbc699af:::benchmark_data_smithi018_308601_object57666:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e312)", 2023-09-23T20:32:51.409 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:32:32.300970+0000", 2023-09-23T20:32:51.409 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 18.961215930000002, 2023-09-23T20:32:51.409 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.3193818780000002, 2023-09-23T20:32:51.409 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:32:51.409 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:32:51.409 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:32:51.409 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:32:51.409 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:32:51.409 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 57667 2023-09-23T20:32:51.409 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.409 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:32:51.409 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.410 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:32:51.410 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.300970+0000", 2023-09-23T20:32:51.410 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:32:51.410 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.410 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.410 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:32:51.410 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.300970+0000", 2023-09-23T20:32:51.410 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:32:51.410 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.410 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.410 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:32:51.411 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.300969+0000", 2023-09-23T20:32:51.411 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.999999 2023-09-23T20:32:51.411 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.411 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.411 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:32:51.411 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.300992+0000", 2023-09-23T20:32:51.411 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.2997e-05 2023-09-23T20:32:51.411 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.411 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.411 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:32:51.411 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.301006+0000", 2023-09-23T20:32:51.411 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3509000000000001e-05 2023-09-23T20:32:51.412 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.412 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.412 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:32:51.412 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.301014+0000", 2023-09-23T20:32:51.412 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.9189999999999999e-06 2023-09-23T20:32:51.412 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.412 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.412 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:32:51.412 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.301060+0000", 2023-09-23T20:32:51.412 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.6597999999999997e-05 2023-09-23T20:32:51.412 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.412 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.413 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:32:51.413 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.301067+0000", 2023-09-23T20:32:51.413 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.6039999999999998e-06 2023-09-23T20:32:51.413 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.413 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.413 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:32:51.413 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.215463+0000", 2023-09-23T20:32:51.413 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.914396019 2023-09-23T20:32:51.413 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.413 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.413 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:32:51.413 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.215471+0000", 2023-09-23T20:32:51.414 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.1880000000000002e-06 2023-09-23T20:32:51.414 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.414 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.414 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:32:51.414 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.582113+0000", 2023-09-23T20:32:51.414 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.366641642 2023-09-23T20:32:51.414 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.414 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.414 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:32:51.414 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.582227+0000", 2023-09-23T20:32:51.414 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000114545 2023-09-23T20:32:51.415 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.415 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.415 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 2,4", 2023-09-23T20:32:51.415 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.582342+0000", 2023-09-23T20:32:51.415 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000114294 2023-09-23T20:32:51.415 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.415 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.415 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:32:51.415 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.584646+0000", 2023-09-23T20:32:51.415 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.002304038 2023-09-23T20:32:51.415 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.415 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.416 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:32:51.416 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.584666+0000", 2023-09-23T20:32:51.416 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.0596000000000001e-05 2023-09-23T20:32:51.416 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.416 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.416 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:32:51.416 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.617277+0000", 2023-09-23T20:32:51.416 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.032610293999999998 2023-09-23T20:32:51.416 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.416 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.417 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:32:51.417 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.617329+0000", 2023-09-23T20:32:51.417 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.2527e-05 2023-09-23T20:32:51.417 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.417 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.417 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:32:51.417 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.620352+0000", 2023-09-23T20:32:51.417 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0030229179999999999 2023-09-23T20:32:51.417 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:32:51.417 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:32:51.418 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:32:51.418 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.418 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.418 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:57675 7.3 7:c7bba963:::benchmark_data_smithi018_308601_object57674:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e312)", 2023-09-23T20:32:51.418 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:32:32.327987+0000", 2023-09-23T20:32:51.418 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 18.934199406000001, 2023-09-23T20:32:51.418 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.2956588660000001, 2023-09-23T20:32:51.418 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:32:51.418 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:32:51.418 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:32:51.418 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:32:51.419 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:32:51.419 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 57675 2023-09-23T20:32:51.419 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.419 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:32:51.419 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.419 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:32:51.419 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.327987+0000", 2023-09-23T20:32:51.419 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:32:51.420 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.420 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.420 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:32:51.420 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.327987+0000", 2023-09-23T20:32:51.420 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:32:51.420 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.420 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.420 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:32:51.420 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.327985+0000", 2023-09-23T20:32:51.420 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:32:51.420 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.420 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.421 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:32:51.421 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.328012+0000", 2023-09-23T20:32:51.421 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.6673e-05 2023-09-23T20:32:51.421 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.421 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.421 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:32:51.421 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.328033+0000", 2023-09-23T20:32:51.421 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.1865000000000002e-05 2023-09-23T20:32:51.422 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.422 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.422 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:32:51.422 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.328046+0000", 2023-09-23T20:32:51.422 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2138e-05 2023-09-23T20:32:51.422 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.422 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.422 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:32:51.422 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.328112+0000", 2023-09-23T20:32:51.422 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.6155999999999995e-05 2023-09-23T20:32:51.422 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.423 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.423 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:32:51.423 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.328123+0000", 2023-09-23T20:32:51.423 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1323e-05 2023-09-23T20:32:51.423 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.425 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.425 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:32:51.425 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.215494+0000", 2023-09-23T20:32:51.425 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.88737101299999999 2023-09-23T20:32:51.425 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.425 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.426 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:32:51.426 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.215502+0000", 2023-09-23T20:32:51.426 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.3140000000000004e-06 2023-09-23T20:32:51.426 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.426 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.426 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:32:51.426 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.583495+0000", 2023-09-23T20:32:51.426 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.367992519 2023-09-23T20:32:51.426 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.427 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.427 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:32:51.427 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.583598+0000", 2023-09-23T20:32:51.427 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00010268399999999999 2023-09-23T20:32:51.427 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.427 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.427 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 2,4", 2023-09-23T20:32:51.427 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.583704+0000", 2023-09-23T20:32:51.427 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000106096 2023-09-23T20:32:51.428 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.428 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.428 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:32:51.428 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.584776+0000", 2023-09-23T20:32:51.428 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00107215 2023-09-23T20:32:51.428 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.428 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.428 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:32:51.428 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.585126+0000", 2023-09-23T20:32:51.429 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00035052599999999998 2023-09-23T20:32:51.429 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.429 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.429 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:32:51.429 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.620444+0000", 2023-09-23T20:32:51.429 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.035318081000000001 2023-09-23T20:32:51.429 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.429 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.429 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:32:51.430 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.620519+0000", 2023-09-23T20:32:51.430 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.4976999999999997e-05 2023-09-23T20:32:51.430 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.430 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.430 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:32:51.430 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.623645+0000", 2023-09-23T20:32:51.430 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.003126017 2023-09-23T20:32:51.430 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:32:51.430 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:32:51.430 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:32:51.431 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.431 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.431 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:69129 7.3 7:cf914298:::benchmark_data_smithi018_308601_object69128:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e318)", 2023-09-23T20:32:51.431 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:32:49.475881+0000", 2023-09-23T20:32:51.431 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 1.786305018, 2023-09-23T20:32:51.431 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.048825159, 2023-09-23T20:32:51.431 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:32:51.431 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:32:51.431 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:32:51.431 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:32:51.431 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:32:51.431 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 69129 2023-09-23T20:32:51.432 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.432 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:32:51.432 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.432 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:32:51.432 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.475881+0000", 2023-09-23T20:32:51.432 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:32:51.432 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.432 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.432 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:32:51.432 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.475881+0000", 2023-09-23T20:32:51.432 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:32:51.432 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.432 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.433 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:32:51.433 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.475881+0000", 2023-09-23T20:32:51.433 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:32:51.433 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.433 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.433 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:32:51.433 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.475925+0000", 2023-09-23T20:32:51.433 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.4274000000000003e-05 2023-09-23T20:32:51.433 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.433 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.433 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:32:51.433 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.475932+0000", 2023-09-23T20:32:51.434 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.2300000000000002e-06 2023-09-23T20:32:51.434 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.434 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.434 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:32:51.434 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.475937+0000", 2023-09-23T20:32:51.434 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.4600000000000002e-06 2023-09-23T20:32:51.434 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.434 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.434 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:32:51.434 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.480114+0000", 2023-09-23T20:32:51.434 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0041766099999999999 2023-09-23T20:32:51.434 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.435 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.435 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:32:51.435 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.480293+0000", 2023-09-23T20:32:51.435 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00017894300000000001 2023-09-23T20:32:51.435 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.435 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.435 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 2,4", 2023-09-23T20:32:51.435 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.480478+0000", 2023-09-23T20:32:51.435 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00018538100000000001 2023-09-23T20:32:51.435 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.435 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.435 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:32:51.436 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.520416+0000", 2023-09-23T20:32:51.436 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.039937451999999998 2023-09-23T20:32:51.436 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.436 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.436 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:32:51.436 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.523198+0000", 2023-09-23T20:32:51.436 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0027818999999999999 2023-09-23T20:32:51.436 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.436 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.436 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:32:51.436 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.528277+0000", 2023-09-23T20:32:51.436 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0050791120000000002 2023-09-23T20:32:51.436 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.437 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.437 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:32:51.437 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.528303+0000", 2023-09-23T20:32:51.437 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.5732e-05 2023-09-23T20:32:51.437 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.437 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.437 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:32:51.437 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:50.524690+0000", 2023-09-23T20:32:51.437 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.99638690299999999 2023-09-23T20:32:51.437 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.437 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.437 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:32:51.438 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:50.524706+0000", 2023-09-23T20:32:51.438 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.6634e-05 2023-09-23T20:32:51.438 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:32:51.438 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:32:51.438 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:32:51.438 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.438 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.438 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:69152 7.3 7.f242f463 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e318)", 2023-09-23T20:32:51.438 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:32:49.520371+0000", 2023-09-23T20:32:51.438 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 1.741815331, 2023-09-23T20:32:51.438 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0043691610000001, 2023-09-23T20:32:51.438 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:32:51.439 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:32:51.439 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:32:51.439 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:32:51.439 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:32:51.439 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 69152 2023-09-23T20:32:51.439 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.439 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:32:51.439 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.439 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:32:51.439 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.520371+0000", 2023-09-23T20:32:51.439 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:32:51.439 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.440 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.440 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:32:51.440 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.520371+0000", 2023-09-23T20:32:51.440 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:32:51.440 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.440 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.440 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:32:51.440 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.520369+0000", 2023-09-23T20:32:51.440 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:32:51.440 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.440 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.441 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:32:51.441 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.520395+0000", 2023-09-23T20:32:51.441 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.5843999999999999e-05 2023-09-23T20:32:51.441 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.441 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.441 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:32:51.441 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.520408+0000", 2023-09-23T20:32:51.441 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3189e-05 2023-09-23T20:32:51.441 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.441 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.441 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:32:51.441 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.520417+0000", 2023-09-23T20:32:51.442 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.6899999999999998e-06 2023-09-23T20:32:51.442 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.442 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.442 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:32:51.442 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.529310+0000", 2023-09-23T20:32:51.442 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0088929609999999996 2023-09-23T20:32:51.442 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.442 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.442 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:32:51.442 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.529330+0000", 2023-09-23T20:32:51.442 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.9673e-05 2023-09-23T20:32:51.442 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.443 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.443 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:32:51.443 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:50.524730+0000", 2023-09-23T20:32:51.443 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.99540061999999996 2023-09-23T20:32:51.443 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.443 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.443 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:32:51.443 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:50.524740+0000", 2023-09-23T20:32:51.443 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.6139999999999998e-06 2023-09-23T20:32:51.443 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:32:51.443 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:32:51.443 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:32:51.443 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.444 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.444 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:69159 7.3 7.de3ad3c3 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e318)", 2023-09-23T20:32:51.444 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:32:49.534755+0000", 2023-09-23T20:32:51.444 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 1.7274305999999999, 2023-09-23T20:32:51.444 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.99001591200000005, 2023-09-23T20:32:51.444 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:32:51.444 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:32:51.444 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:32:51.444 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:32:51.444 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:32:51.444 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 69159 2023-09-23T20:32:51.444 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.445 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:32:51.445 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.445 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:32:51.445 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.534755+0000", 2023-09-23T20:32:51.445 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:32:51.445 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.445 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.445 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:32:51.445 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.534755+0000", 2023-09-23T20:32:51.445 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:32:51.445 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.446 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.446 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:32:51.446 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.534754+0000", 2023-09-23T20:32:51.446 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:32:51.446 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.446 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.446 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:32:51.446 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.534782+0000", 2023-09-23T20:32:51.446 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.8348999999999999e-05 2023-09-23T20:32:51.446 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.446 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.446 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:32:51.447 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.534808+0000", 2023-09-23T20:32:51.447 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.5941000000000001e-05 2023-09-23T20:32:51.447 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.447 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.447 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:32:51.447 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.534818+0000", 2023-09-23T20:32:51.447 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0132999999999999e-05 2023-09-23T20:32:51.447 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.447 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.447 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:32:51.447 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.535076+0000", 2023-09-23T20:32:51.447 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00025796400000000001 2023-09-23T20:32:51.448 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.448 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.448 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:32:51.448 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.535090+0000", 2023-09-23T20:32:51.448 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3505e-05 2023-09-23T20:32:51.448 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.448 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.448 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:32:51.448 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:50.524762+0000", 2023-09-23T20:32:51.448 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.98967230399999995 2023-09-23T20:32:51.448 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:32:51.448 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:32:51.449 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:32:51.449 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:50.524771+0000", 2023-09-23T20:32:51.449 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.5070000000000006e-06 2023-09-23T20:32:51.449 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:32:51.449 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:32:51.449 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:32:51.449 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:32:51.449 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:32:51.449 INFO:teuthology.orchestra.run.smithi018.stdout:} 2023-09-23T20:32:51.450 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:51.450 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:51.451 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:32:51.451 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:32:51.526 INFO:teuthology.orchestra.run.smithi018.stderr:nodeep-scrub is set 2023-09-23T20:32:51.548 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:51 smithi018 ceph-mon[184144]: failed to encode map e320 with expected crc 2023-09-23T20:32:51.549 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:51 smithi018 ceph-mon[184144]: failed to encode map e320 with expected crc 2023-09-23T20:32:51.549 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:51 smithi018 ceph-mon[184144]: failed to encode map e320 with expected crc 2023-09-23T20:32:51.549 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:51 smithi018 ceph-mon[184144]: osdmap e320: 8 total, 8 up, 8 in 2023-09-23T20:32:51.549 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:51 smithi018 ceph-mon[184144]: failed to encode map e320 with expected crc 2023-09-23T20:32:51.549 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:51 smithi018 ceph-mon[184144]: failed to encode map e320 with expected crc 2023-09-23T20:32:51.549 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:51 smithi018 ceph-mon[184144]: failed to encode map e320 with expected crc 2023-09-23T20:32:51.549 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:51 smithi018 ceph-mon[184144]: failed to encode map e320 with expected crc 2023-09-23T20:32:51.549 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:51 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2359148631' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:32:51.549 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:51 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:32:51.550 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:51 smithi018 ceph-mon[179647]: failed to encode map e320 with expected crc 2023-09-23T20:32:51.550 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:51 smithi018 ceph-mon[179647]: failed to encode map e320 with expected crc 2023-09-23T20:32:51.550 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:51 smithi018 ceph-mon[179647]: failed to encode map e320 with expected crc 2023-09-23T20:32:51.550 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:51 smithi018 ceph-mon[179647]: osdmap e320: 8 total, 8 up, 8 in 2023-09-23T20:32:51.550 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:51 smithi018 ceph-mon[179647]: failed to encode map e320 with expected crc 2023-09-23T20:32:51.550 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:51 smithi018 ceph-mon[179647]: failed to encode map e320 with expected crc 2023-09-23T20:32:51.550 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:51 smithi018 ceph-mon[179647]: failed to encode map e320 with expected crc 2023-09-23T20:32:51.550 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:51 smithi018 ceph-mon[179647]: failed to encode map e320 with expected crc 2023-09-23T20:32:51.550 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:51 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2359148631' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:32:51.551 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:51 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:32:51.552 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:32:51.582 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:51.626 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:32:51.632 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:51.633 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:32:51.734 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:32:51.796 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 55 16 69641 69625 79.0702 17 0.0220974 0.0126413 2023-09-23T20:32:51.824 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:51.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:51 smithi149 ceph-mon[170394]: failed to encode map e320 with expected crc 2023-09-23T20:32:51.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:51 smithi149 ceph-mon[170394]: failed to encode map e320 with expected crc 2023-09-23T20:32:51.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:51 smithi149 ceph-mon[170394]: failed to encode map e320 with expected crc 2023-09-23T20:32:51.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:51 smithi149 ceph-mon[170394]: osdmap e320: 8 total, 8 up, 8 in 2023-09-23T20:32:51.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:51 smithi149 ceph-mon[170394]: failed to encode map e320 with expected crc 2023-09-23T20:32:51.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:51 smithi149 ceph-mon[170394]: failed to encode map e320 with expected crc 2023-09-23T20:32:51.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:51 smithi149 ceph-mon[170394]: failed to encode map e320 with expected crc 2023-09-23T20:32:51.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:51 smithi149 ceph-mon[170394]: failed to encode map e320 with expected crc 2023-09-23T20:32:51.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:51 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2359148631' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:32:51.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:51 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:32:52.800 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 56 16 70422 70406 78.5297 48.8125 0.0121276 0.0127285 2023-09-23T20:32:52.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:52 smithi018 ceph-mon[179647]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:32:52.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:52 smithi018 ceph-mon[179647]: failed to encode map e321 with expected crc 2023-09-23T20:32:52.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:52 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:32:52.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:52 smithi018 ceph-mon[179647]: osdmap e321: 8 total, 8 up, 8 in 2023-09-23T20:32:52.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:52 smithi018 ceph-mon[179647]: failed to encode map e321 with expected crc 2023-09-23T20:32:52.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:52 smithi018 ceph-mon[179647]: failed to encode map e321 with expected crc 2023-09-23T20:32:52.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:52 smithi018 ceph-mon[179647]: failed to encode map e321 with expected crc 2023-09-23T20:32:52.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:52 smithi018 ceph-mon[179647]: pgmap v740: 109 pgs: 5 peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 4.5 KiB/s rd, 35 MiB/s wr, 557 op/s; 10773/206439 objects degraded (5.218%); 6596/206439 objects misplaced (3.195%); 2.1 MiB/s, 33 objects/s recovering 2023-09-23T20:32:52.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:52 smithi018 ceph-mon[184144]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:32:52.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:52 smithi018 ceph-mon[184144]: failed to encode map e321 with expected crc 2023-09-23T20:32:52.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:52 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:32:52.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:52 smithi018 ceph-mon[184144]: osdmap e321: 8 total, 8 up, 8 in 2023-09-23T20:32:52.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:52 smithi018 ceph-mon[184144]: failed to encode map e321 with expected crc 2023-09-23T20:32:52.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:52 smithi018 ceph-mon[184144]: failed to encode map e321 with expected crc 2023-09-23T20:32:52.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:52 smithi018 ceph-mon[184144]: failed to encode map e321 with expected crc 2023-09-23T20:32:52.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:52 smithi018 ceph-mon[184144]: pgmap v740: 109 pgs: 5 peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 4.5 KiB/s rd, 35 MiB/s wr, 557 op/s; 10773/206439 objects degraded (5.218%); 6596/206439 objects misplaced (3.195%); 2.1 MiB/s, 33 objects/s recovering 2023-09-23T20:32:52.969 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:52 smithi149 ceph-mon[170394]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:32:52.969 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:52 smithi149 ceph-mon[170394]: failed to encode map e321 with expected crc 2023-09-23T20:32:52.970 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:52 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:32:52.970 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:52 smithi149 ceph-mon[170394]: osdmap e321: 8 total, 8 up, 8 in 2023-09-23T20:32:52.970 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:52 smithi149 ceph-mon[170394]: failed to encode map e321 with expected crc 2023-09-23T20:32:52.970 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:52 smithi149 ceph-mon[170394]: failed to encode map e321 with expected crc 2023-09-23T20:32:52.970 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:52 smithi149 ceph-mon[170394]: failed to encode map e321 with expected crc 2023-09-23T20:32:52.970 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:52 smithi149 ceph-mon[170394]: pgmap v740: 109 pgs: 5 peering, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 4.5 KiB/s rd, 35 MiB/s wr, 557 op/s; 10773/206439 objects degraded (5.218%); 6596/206439 objects misplaced (3.195%); 2.1 MiB/s, 33 objects/s recovering 2023-09-23T20:32:53.798 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 57 16 71275 71259 78.0856 53.3125 0.0180976 0.0128001 2023-09-23T20:32:53.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:53 smithi018 ceph-mon[184144]: Health check failed: Reduced data availability: 4 pgs inactive, 5 pgs peering (PG_AVAILABILITY) 2023-09-23T20:32:53.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:53 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:32:53.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:53 smithi018 ceph-mon[179647]: Health check failed: Reduced data availability: 4 pgs inactive, 5 pgs peering (PG_AVAILABILITY) 2023-09-23T20:32:53.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:53 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:32:53.974 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:53 smithi149 ceph-mon[170394]: Health check failed: Reduced data availability: 4 pgs inactive, 5 pgs peering (PG_AVAILABILITY) 2023-09-23T20:32:53.974 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:53 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:32:53.999 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:32:54.005 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:54.006 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:32:54.107 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:32:54.208 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:54.798 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 58 16 72134 72118 77.665 53.6875 0.0215498 0.0128695 2023-09-23T20:32:54.861 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:54 smithi018 ceph-mon[184144]: pgmap v741: 109 pgs: 5 peering, 1 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 4.5 KiB/s rd, 35 MiB/s wr, 565 op/s; 13730/207999 objects degraded (6.601%); 3597/207999 objects misplaced (1.729%); 2.6 MiB/s, 40 objects/s recovering 2023-09-23T20:32:54.862 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:54 smithi018 ceph-mon[179647]: pgmap v741: 109 pgs: 5 peering, 1 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 4.5 KiB/s rd, 35 MiB/s wr, 565 op/s; 13730/207999 objects degraded (6.601%); 3597/207999 objects misplaced (1.729%); 2.6 MiB/s, 40 objects/s recovering 2023-09-23T20:32:54.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:54 smithi149 ceph-mon[170394]: pgmap v741: 109 pgs: 5 peering, 1 active+recovery_wait+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 4.5 KiB/s rd, 35 MiB/s wr, 565 op/s; 13730/207999 objects degraded (6.601%); 3597/207999 objects misplaced (1.729%); 2.6 MiB/s, 40 objects/s recovering 2023-09-23T20:32:55.122 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:32:55.127 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:55.127 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:32:55.132 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:32:54 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:32:54] "GET /metrics HTTP/1.1" 200 36539 "" "Prometheus/2.43.0" 2023-09-23T20:32:55.218 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T20:32:55.228 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:32:55.331 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:55.419 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 5, 7, 3, 2, 6, 4] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T20:32:55.419 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:32:55.419 INFO:tasks.thrashosds.thrasher:Reweighting osd 4 to 0.6412024695125249 2023-09-23T20:32:55.420 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd reweight 4 0.6412024695125249 2023-09-23T20:32:55.430 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:32:55.467 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:32:55.481 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:32:55.490 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:55.490 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:32:55.525 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:32:55.576 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:32:55.591 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:32:55.622 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:55.655 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:32:55.682 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:32:55.688 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:55.689 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:32:55.701 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:32:55.757 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:32:55.790 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:32:55.798 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 59 16 72858 72842 77.1159 45.25 0.0290319 0.0129615 2023-09-23T20:32:55.814 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:32:55.841 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:32:55.841 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:32:55.841 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:32:55.841 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:32:55.841 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:32:55.841 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:32:55.842 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:32:55.842 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:32:55.871 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:55 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 13730/207999 objects degraded (6.601%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T20:32:55.871 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:55 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/678302605' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:32:55.872 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:55 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 13730/207999 objects degraded (6.601%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T20:32:55.872 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:55 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/678302605' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:32:55.901 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:55.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:55 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 13730/207999 objects degraded (6.601%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T20:32:55.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:55 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/678302605' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:32:56.013 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:32:56.025 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:56.025 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:32:56.051 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:32:56.126 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:32:56.194 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:56.353 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:32:56.360 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:56.361 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:32:56.461 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:32:56.506 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:32:56.539 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:56.798 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:2023-09-23T20:32:56.798891+0000 min lat: 0.00108768 max lat: 2.45012 avg lat: 0.0130462 2023-09-23T20:32:56.798 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T20:32:56.798 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 60 16 73609 73593 76.6131 46.9375 0.0107888 0.0130462 2023-09-23T20:32:56.939 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:32:56.944 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:56.945 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:32:57.046 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:32:57.130 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:57.199 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:32:57.200 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:57.200 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:32:57.253 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:57 smithi149 ceph-mon[170394]: pgmap v742: 109 pgs: 1 active+recovering+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 4.3 KiB/s rd, 47 MiB/s wr, 748 op/s; 13730/215220 objects degraded (6.380%); 3296/215220 objects misplaced (1.531%); 3.4 MiB/s, 53 objects/s recovering 2023-09-23T20:32:57.254 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:57 smithi149 ceph-mon[170394]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 5 pgs peering) 2023-09-23T20:32:57.301 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:32:57.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:57 smithi018 ceph-mon[184144]: pgmap v742: 109 pgs: 1 active+recovering+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 4.3 KiB/s rd, 47 MiB/s wr, 748 op/s; 13730/215220 objects degraded (6.380%); 3296/215220 objects misplaced (1.531%); 3.4 MiB/s, 53 objects/s recovering 2023-09-23T20:32:57.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:57 smithi018 ceph-mon[184144]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 5 pgs peering) 2023-09-23T20:32:57.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:57 smithi018 ceph-mon[179647]: pgmap v742: 109 pgs: 1 active+recovering+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 4.3 KiB/s rd, 47 MiB/s wr, 748 op/s; 13730/215220 objects degraded (6.380%); 3296/215220 objects misplaced (1.531%); 3.4 MiB/s, 53 objects/s recovering 2023-09-23T20:32:57.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:57 smithi018 ceph-mon[179647]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 5 pgs peering) 2023-09-23T20:32:57.418 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:57.522 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T20:32:57.606 INFO:teuthology.orchestra.run.smithi149.stdout:ERROR: (22) Invalid argument 2023-09-23T20:32:57.606 INFO:teuthology.orchestra.run.smithi149.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:32:57.708 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:32:57.713 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:57.713 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:32:57.798 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 61 16 74326 74310 76.0919 44.8125 0.0386283 0.0131345 2023-09-23T20:32:57.814 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:32:57.862 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:58.353 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:32:58.659 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:32:58.710 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:32:58.714 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:58.715 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:32:58.798 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 62 16 74962 74946 75.506 39.75 0.0340164 0.0132335 2023-09-23T20:32:58.811 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T20:32:58.815 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:32:58.907 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:59.002 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:32:59.011 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:59.013 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:32:59.114 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:32:59.149 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:59.173 INFO:teuthology.orchestra.run.smithi018.stderr:reweighted osd.4 to 0.641202 (a425) 2023-09-23T20:32:59.402 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:32:59.404 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:59 smithi149 ceph-mon[170394]: pgmap v743: 109 pgs: 1 active+recovering+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 3.4 KiB/s rd, 44 MiB/s wr, 711 op/s; 13730/218235 objects degraded (6.291%); 3296/218235 objects misplaced (1.510%); 2.7 MiB/s, 42 objects/s recovering 2023-09-23T20:32:59.404 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:59 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3339453468' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 0.6412024695125249}]: dispatch 2023-09-23T20:32:59.404 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:32:59 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 0.6412024695125249}]: dispatch 2023-09-23T20:32:59.405 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:59.405 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:32:59.506 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:32:59.535 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:59.574 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:32:59.576 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:59.577 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:32:59.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:59 smithi018 ceph-mon[184144]: pgmap v743: 109 pgs: 1 active+recovering+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 3.4 KiB/s rd, 44 MiB/s wr, 711 op/s; 13730/218235 objects degraded (6.291%); 3296/218235 objects misplaced (1.510%); 2.7 MiB/s, 42 objects/s recovering 2023-09-23T20:32:59.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:59 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3339453468' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 0.6412024695125249}]: dispatch 2023-09-23T20:32:59.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:32:59 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 0.6412024695125249}]: dispatch 2023-09-23T20:32:59.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:59 smithi018 ceph-mon[179647]: pgmap v743: 109 pgs: 1 active+recovering+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 3.4 KiB/s rd, 44 MiB/s wr, 711 op/s; 13730/218235 objects degraded (6.291%); 3296/218235 objects misplaced (1.510%); 2.7 MiB/s, 42 objects/s recovering 2023-09-23T20:32:59.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:59 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3339453468' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 0.6412024695125249}]: dispatch 2023-09-23T20:32:59.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:32:59 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 0.6412024695125249}]: dispatch 2023-09-23T20:32:59.678 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:32:59.706 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:32:59.739 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:32:59.742 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:32:59.742 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:32:59.798 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 63 16 75262 75246 74.6055 18.75 0.00403572 0.013278 2023-09-23T20:32:59.843 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:32:59.873 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:00.008 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:33:00.010 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:00.011 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:33:00.095 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:33:00.112 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:33:00.142 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:00.202 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:33:00.205 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:00.206 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:33:00.238 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:33:00.297 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:00 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 0.6412024695125249}]': finished 2023-09-23T20:33:00.297 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:00 smithi149 ceph-mon[170394]: osdmap e322: 8 total, 8 up, 8 in 2023-09-23T20:33:00.297 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:00 smithi149 ceph-mon[170394]: failed to encode map e322 with expected crc 2023-09-23T20:33:00.297 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:00 smithi149 ceph-mon[170394]: failed to encode map e322 with expected crc 2023-09-23T20:33:00.298 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:00 smithi149 ceph-mon[170394]: failed to encode map e322 with expected crc 2023-09-23T20:33:00.298 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:00 smithi149 ceph-mon[170394]: failed to encode map e322 with expected crc 2023-09-23T20:33:00.298 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:00 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:33:00.298 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:00 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:33:00.298 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:00 smithi149 ceph-mon[170394]: osdmap e323: 8 total, 8 up, 8 in 2023-09-23T20:33:00.307 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:33:00.315 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:00 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 0.6412024695125249}]': finished 2023-09-23T20:33:00.315 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:00 smithi018 ceph-mon[184144]: osdmap e322: 8 total, 8 up, 8 in 2023-09-23T20:33:00.316 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:00 smithi018 ceph-mon[184144]: failed to encode map e322 with expected crc 2023-09-23T20:33:00.316 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:00 smithi018 ceph-mon[184144]: failed to encode map e322 with expected crc 2023-09-23T20:33:00.316 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:00 smithi018 ceph-mon[184144]: failed to encode map e322 with expected crc 2023-09-23T20:33:00.316 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:00 smithi018 ceph-mon[184144]: failed to encode map e322 with expected crc 2023-09-23T20:33:00.316 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:00 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:33:00.316 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:00 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:33:00.316 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:00 smithi018 ceph-mon[184144]: osdmap e323: 8 total, 8 up, 8 in 2023-09-23T20:33:00.317 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:00 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 0.6412024695125249}]': finished 2023-09-23T20:33:00.317 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:00 smithi018 ceph-mon[179647]: osdmap e322: 8 total, 8 up, 8 in 2023-09-23T20:33:00.317 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:00 smithi018 ceph-mon[179647]: failed to encode map e322 with expected crc 2023-09-23T20:33:00.317 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:00 smithi018 ceph-mon[179647]: failed to encode map e322 with expected crc 2023-09-23T20:33:00.317 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:00 smithi018 ceph-mon[179647]: failed to encode map e322 with expected crc 2023-09-23T20:33:00.318 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:00 smithi018 ceph-mon[179647]: failed to encode map e322 with expected crc 2023-09-23T20:33:00.318 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:00 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:33:00.318 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:00 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:33:00.318 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:00 smithi018 ceph-mon[179647]: osdmap e323: 8 total, 8 up, 8 in 2023-09-23T20:33:00.347 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:00.398 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:33:00.402 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:00.403 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:33:00.503 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:33:00.534 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:00.799 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 64 16 75467 75451 73.6404 12.8125 0.0071281 0.0134456 2023-09-23T20:33:00.880 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:33:00.882 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:00.883 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:33:00.984 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:33:01.013 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:01.150 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:33:01.153 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:01.154 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:33:01.174 INFO:teuthology.orchestra.run.smithi018.stderr:noscrub is unset 2023-09-23T20:33:01.255 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:33:01.313 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:01 smithi018 ceph-mon[179647]: pgmap v745: 109 pgs: 1 active+recovering+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 35 MiB/s wr, 562 op/s; 13730/220377 objects degraded (6.230%); 3096/220377 objects misplaced (1.405%); 4.1 MiB/s, 65 objects/s recovering 2023-09-23T20:33:01.313 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:01 smithi018 ceph-mon[179647]: failed to encode map e323 with expected crc 2023-09-23T20:33:01.313 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:01 smithi018 ceph-mon[179647]: failed to encode map e323 with expected crc 2023-09-23T20:33:01.313 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:01 smithi018 ceph-mon[179647]: failed to encode map e323 with expected crc 2023-09-23T20:33:01.313 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:01 smithi018 ceph-mon[179647]: failed to encode map e323 with expected crc 2023-09-23T20:33:01.313 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:01 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 13730/220377 objects degraded (6.230%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T20:33:01.313 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:01 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3575832416' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:33:01.314 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:01 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:33:01.314 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:01 smithi018 ceph-mon[179647]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:33:01.314 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:01 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:33:01.314 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:01 smithi018 ceph-mon[179647]: osdmap e324: 8 total, 8 up, 8 in 2023-09-23T20:33:01.314 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:01 smithi018 ceph-mon[184144]: pgmap v745: 109 pgs: 1 active+recovering+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 35 MiB/s wr, 562 op/s; 13730/220377 objects degraded (6.230%); 3096/220377 objects misplaced (1.405%); 4.1 MiB/s, 65 objects/s recovering 2023-09-23T20:33:01.314 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:01 smithi018 ceph-mon[184144]: failed to encode map e323 with expected crc 2023-09-23T20:33:01.314 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:01 smithi018 ceph-mon[184144]: failed to encode map e323 with expected crc 2023-09-23T20:33:01.314 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:01 smithi018 ceph-mon[184144]: failed to encode map e323 with expected crc 2023-09-23T20:33:01.314 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:01 smithi018 ceph-mon[184144]: failed to encode map e323 with expected crc 2023-09-23T20:33:01.315 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:01 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 13730/220377 objects degraded (6.230%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T20:33:01.315 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:01 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3575832416' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:33:01.315 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:01 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:33:01.315 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:01 smithi018 ceph-mon[184144]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:33:01.315 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:01 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:33:01.315 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:01 smithi018 ceph-mon[184144]: osdmap e324: 8 total, 8 up, 8 in 2023-09-23T20:33:01.333 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:01.428 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:01 smithi149 ceph-mon[170394]: pgmap v745: 109 pgs: 1 active+recovering+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 35 MiB/s wr, 562 op/s; 13730/220377 objects degraded (6.230%); 3096/220377 objects misplaced (1.405%); 4.1 MiB/s, 65 objects/s recovering 2023-09-23T20:33:01.428 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:01 smithi149 ceph-mon[170394]: failed to encode map e323 with expected crc 2023-09-23T20:33:01.428 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:01 smithi149 ceph-mon[170394]: failed to encode map e323 with expected crc 2023-09-23T20:33:01.428 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:01 smithi149 ceph-mon[170394]: failed to encode map e323 with expected crc 2023-09-23T20:33:01.428 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:01 smithi149 ceph-mon[170394]: failed to encode map e323 with expected crc 2023-09-23T20:33:01.428 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:01 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 13730/220377 objects degraded (6.230%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T20:33:01.428 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:01 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3575832416' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:33:01.428 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:01 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:33:01.428 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:01 smithi149 ceph-mon[170394]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:33:01.429 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:01 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:33:01.429 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:01 smithi149 ceph-mon[170394]: osdmap e324: 8 total, 8 up, 8 in 2023-09-23T20:33:01.711 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:33:01.726 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:01.726 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:33:01.737 INFO:teuthology.orchestra.run.smithi149.stdout:ERROR: (22) Invalid argument 2023-09-23T20:33:01.737 INFO:teuthology.orchestra.run.smithi149.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:33:01.800 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 65 16 75861 75845 72.8846 24.625 0.0197364 0.0137155 2023-09-23T20:33:01.827 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:33:01.881 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:01.936 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:33:01.941 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:01.942 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:33:02.043 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:33:02.085 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:02.202 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:33:02.205 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:02.206 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:33:02.274 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T20:33:02.307 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:33:02.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:02 smithi018 ceph-mon[179647]: failed to encode map e324 with expected crc 2023-09-23T20:33:02.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:02 smithi018 ceph-mon[179647]: failed to encode map e324 with expected crc 2023-09-23T20:33:02.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:02 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:33:02.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:02 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:33:02.396 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:02 smithi018 ceph-mon[184144]: failed to encode map e324 with expected crc 2023-09-23T20:33:02.396 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:02 smithi018 ceph-mon[184144]: failed to encode map e324 with expected crc 2023-09-23T20:33:02.396 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:02 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:33:02.396 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:02 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:33:02.413 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:02.453 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:33:02.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:02 smithi149 ceph-mon[170394]: failed to encode map e324 with expected crc 2023-09-23T20:33:02.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:02 smithi149 ceph-mon[170394]: failed to encode map e324 with expected crc 2023-09-23T20:33:02.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:02 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:33:02.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:02 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:33:02.480 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:33:02.486 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:02.487 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:33:02.500 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:33:02.548 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:33:02.588 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:33:02.607 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:33:02.650 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:33:02.678 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:02.706 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:33:02.758 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:33:02.772 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:02.772 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:33:02.773 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:33:02.804 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 66 16 76564 76548 72.4421 43.9375 0.012352 0.0137979 2023-09-23T20:33:02.805 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:33:02.812 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:33:02.812 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:33:02.812 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:33:02.812 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:33:02.812 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:33:02.812 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:33:02.812 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:33:02.812 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:33:02.874 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:33:02.904 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:02.945 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:33:02.948 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:02.949 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:33:02.967 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:33:03.050 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:33:03.110 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:03.214 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:03 smithi149 ceph-mon[170394]: pgmap v748: 109 pgs: 1 active+undersized+degraded+remapped, 1 peering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 34 MiB/s wr, 539 op/s; 7706/224970 objects degraded (3.425%); 1052/224970 objects misplaced (0.468%); 2.3 MiB/s, 36 objects/s recovering 2023-09-23T20:33:03.215 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:03 smithi149 ceph-mon[170394]: failed to encode map e325 with expected crc 2023-09-23T20:33:03.215 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:03 smithi149 ceph-mon[170394]: osdmap e325: 8 total, 8 up, 8 in 2023-09-23T20:33:03.215 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:03 smithi149 ceph-mon[170394]: failed to encode map e325 with expected crc 2023-09-23T20:33:03.215 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:03 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:33:03.215 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:03 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:33:03.215 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:03 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2656023985' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:33:03.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:03 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:33:03.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:03 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:33:03.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:03 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:33:03.238 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:33:03.250 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:03.251 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:33:03.254 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:03 smithi018 ceph-mon[184144]: pgmap v748: 109 pgs: 1 active+undersized+degraded+remapped, 1 peering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 34 MiB/s wr, 539 op/s; 7706/224970 objects degraded (3.425%); 1052/224970 objects misplaced (0.468%); 2.3 MiB/s, 36 objects/s recovering 2023-09-23T20:33:03.254 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:03 smithi018 ceph-mon[184144]: failed to encode map e325 with expected crc 2023-09-23T20:33:03.254 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:03 smithi018 ceph-mon[184144]: osdmap e325: 8 total, 8 up, 8 in 2023-09-23T20:33:03.254 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:03 smithi018 ceph-mon[184144]: failed to encode map e325 with expected crc 2023-09-23T20:33:03.254 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:03 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:33:03.255 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:03 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:33:03.255 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:03 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2656023985' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:33:03.255 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:03 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:33:03.255 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:03 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:33:03.255 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:03 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:33:03.261 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:03 smithi018 ceph-mon[179647]: pgmap v748: 109 pgs: 1 active+undersized+degraded+remapped, 1 peering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 34 MiB/s wr, 539 op/s; 7706/224970 objects degraded (3.425%); 1052/224970 objects misplaced (0.468%); 2.3 MiB/s, 36 objects/s recovering 2023-09-23T20:33:03.261 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:03 smithi018 ceph-mon[179647]: failed to encode map e325 with expected crc 2023-09-23T20:33:03.261 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:03 smithi018 ceph-mon[179647]: osdmap e325: 8 total, 8 up, 8 in 2023-09-23T20:33:03.261 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:03 smithi018 ceph-mon[179647]: failed to encode map e325 with expected crc 2023-09-23T20:33:03.261 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:03 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:33:03.261 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:03 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:33:03.261 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:03 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2656023985' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:33:03.261 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:03 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:33:03.261 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:03 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:33:03.262 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:03 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:33:03.352 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:33:03.427 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:03.497 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:33:03.511 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:03.512 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:33:03.612 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:33:03.634 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T20:33:03.649 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:03.746 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:33:03.752 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:03.753 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:33:03.804 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 67 16 77358 77342 72.1016 49.625 0.0119651 0.0138642 2023-09-23T20:33:03.854 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:33:03.897 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:03.941 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:33:03.946 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:03.946 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:33:04.047 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:33:04.109 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:04.215 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:33:04.218 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:04.219 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:33:04.297 INFO:teuthology.orchestra.run.smithi149.stdout:{ 2023-09-23T20:33:04.297 INFO:teuthology.orchestra.run.smithi149.stdout: "size": 20, 2023-09-23T20:33:04.297 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 600, 2023-09-23T20:33:04.298 INFO:teuthology.orchestra.run.smithi149.stdout: "ops": [ 2023-09-23T20:33:04.298 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.298 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:28909 5.1 5:825aa99e:::benchmark_data_smithi018_228451_object28908:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e118)", 2023-09-23T20:33:04.298 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:09.981257+0000", 2023-09-23T20:33:04.298 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 594.31290527900001, 2023-09-23T20:33:04.298 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0826966440000001, 2023-09-23T20:33:04.298 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:33:04.298 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:33:04.298 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:33:04.298 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:33:04.299 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:33:04.299 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 28909 2023-09-23T20:33:04.299 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.299 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:33:04.299 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.299 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:33:04.299 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:09.981257+0000", 2023-09-23T20:33:04.299 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:33:04.299 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.299 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.299 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:33:04.299 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:09.981257+0000", 2023-09-23T20:33:04.300 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:33:04.300 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.300 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.300 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:33:04.300 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:09.981255+0000", 2023-09-23T20:33:04.300 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999981 2023-09-23T20:33:04.300 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.300 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.300 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:33:04.300 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:09.981345+0000", 2023-09-23T20:33:04.300 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.9339999999999995e-05 2023-09-23T20:33:04.301 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.301 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.301 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:33:04.301 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:09.981360+0000", 2023-09-23T20:33:04.301 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.5591999999999999e-05 2023-09-23T20:33:04.301 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.301 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.301 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:33:04.301 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:09.981370+0000", 2023-09-23T20:33:04.301 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.6069999999999997e-06 2023-09-23T20:33:04.301 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.302 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.302 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:33:04.302 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:10.015283+0000", 2023-09-23T20:33:04.302 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.033912963999999997 2023-09-23T20:33:04.302 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.302 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.302 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:33:04.302 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:10.015297+0000", 2023-09-23T20:33:04.302 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.4477e-05 2023-09-23T20:33:04.302 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.302 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.303 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:33:04.303 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.054705+0000", 2023-09-23T20:33:04.303 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.03940805 2023-09-23T20:33:04.303 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.303 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.303 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:33:04.303 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.054798+0000", 2023-09-23T20:33:04.303 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.2872000000000001e-05 2023-09-23T20:33:04.303 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.303 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.303 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 3,7", 2023-09-23T20:33:04.304 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.054907+0000", 2023-09-23T20:33:04.304 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000108952 2023-09-23T20:33:04.304 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.304 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.304 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:33:04.304 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.056066+0000", 2023-09-23T20:33:04.304 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0011585460000000001 2023-09-23T20:33:04.304 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.304 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.304 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:33:04.305 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.056406+0000", 2023-09-23T20:33:04.305 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000340064 2023-09-23T20:33:04.305 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.305 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.305 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:33:04.305 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.063876+0000", 2023-09-23T20:33:04.305 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.007470661 2023-09-23T20:33:04.305 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.305 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.305 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:33:04.305 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.063917+0000", 2023-09-23T20:33:04.305 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.1151e-05 2023-09-23T20:33:04.306 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.306 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.306 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:33:04.306 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:11.063954+0000", 2023-09-23T20:33:04.306 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.6075999999999998e-05 2023-09-23T20:33:04.306 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:33:04.306 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:33:04.306 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:33:04.306 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.306 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.307 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:76669 5.18 5.af594e58 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e147)", 2023-09-23T20:33:04.307 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:58.071853+0000", 2023-09-23T20:33:04.307 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 546.222308677, 2023-09-23T20:33:04.307 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.86360499599999996, 2023-09-23T20:33:04.307 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:33:04.307 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:33:04.307 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:33:04.307 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:33:04.307 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:33:04.307 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 76669 2023-09-23T20:33:04.307 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.308 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:33:04.308 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.308 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:33:04.308 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:58.071853+0000", 2023-09-23T20:33:04.308 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:33:04.308 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.308 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.308 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:33:04.308 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:58.071853+0000", 2023-09-23T20:33:04.308 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:33:04.309 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.309 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.309 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:33:04.309 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:58.071853+0000", 2023-09-23T20:33:04.309 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:33:04.309 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.309 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.309 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:33:04.309 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:58.071910+0000", 2023-09-23T20:33:04.309 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.7513000000000003e-05 2023-09-23T20:33:04.310 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.310 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.310 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:33:04.310 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:58.071922+0000", 2023-09-23T20:33:04.310 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.2041e-05 2023-09-23T20:33:04.310 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.310 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.310 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:33:04.310 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:58.071926+0000", 2023-09-23T20:33:04.311 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.2660000000000003e-06 2023-09-23T20:33:04.311 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.311 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.311 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:33:04.311 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:58.071963+0000", 2023-09-23T20:33:04.311 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.6186999999999998e-05 2023-09-23T20:33:04.311 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.311 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.311 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:33:04.311 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:58.071968+0000", 2023-09-23T20:33:04.311 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.9200000000000003e-06 2023-09-23T20:33:04.312 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.312 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.312 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:33:04.312 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:58.935445+0000", 2023-09-23T20:33:04.312 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.86347702999999998 2023-09-23T20:33:04.312 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.312 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.312 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:33:04.312 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:58.935458+0000", 2023-09-23T20:33:04.312 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.3918e-05 2023-09-23T20:33:04.312 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:33:04.312 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:33:04.313 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:33:04.313 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.313 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.313 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:76679 5.18 5.48e67dd8 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e147)", 2023-09-23T20:33:04.313 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:58.103169+0000", 2023-09-23T20:33:04.313 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 546.19099358400001, 2023-09-23T20:33:04.313 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.83523706399999997, 2023-09-23T20:33:04.313 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:33:04.313 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:33:04.313 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:33:04.313 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:33:04.314 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:33:04.314 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 76679 2023-09-23T20:33:04.314 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.314 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:33:04.314 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.314 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:33:04.314 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:58.103169+0000", 2023-09-23T20:33:04.314 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:33:04.314 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.314 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.315 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:33:04.315 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:58.103169+0000", 2023-09-23T20:33:04.315 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:33:04.315 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.315 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.315 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:33:04.315 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:58.103167+0000", 2023-09-23T20:33:04.315 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:33:04.315 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.315 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.315 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:33:04.316 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:58.103270+0000", 2023-09-23T20:33:04.316 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00010306000000000001 2023-09-23T20:33:04.316 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.316 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.316 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:33:04.316 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:58.103287+0000", 2023-09-23T20:33:04.316 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.7147e-05 2023-09-23T20:33:04.317 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.317 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.317 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:33:04.317 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:58.103294+0000", 2023-09-23T20:33:04.317 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.8090000000000004e-06 2023-09-23T20:33:04.318 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.318 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.318 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:33:04.318 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:58.103352+0000", 2023-09-23T20:33:04.318 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.7491000000000002e-05 2023-09-23T20:33:04.318 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.318 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.319 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:33:04.319 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:58.103359+0000", 2023-09-23T20:33:04.319 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.1999999999999997e-06 2023-09-23T20:33:04.319 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.319 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.319 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:33:04.319 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:58.938390+0000", 2023-09-23T20:33:04.319 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.83503123000000001 2023-09-23T20:33:04.320 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.320 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.320 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:33:04.320 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:58.938406+0000", 2023-09-23T20:33:04.320 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.5665e-05 2023-09-23T20:33:04.320 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:33:04.320 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:33:04.321 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:33:04.321 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.321 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.321 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:126867 5.a 5:50925f4d:::benchmark_data_smithi018_228451_object19806:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e164)", 2023-09-23T20:33:04.321 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:25:01.833816+0000", 2023-09-23T20:33:04.321 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 482.46034571899997, 2023-09-23T20:33:04.321 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.442573232, 2023-09-23T20:33:04.322 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:33:04.322 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:33:04.322 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:33:04.322 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:33:04.322 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:33:04.322 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 126867 2023-09-23T20:33:04.322 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.323 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:33:04.323 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.323 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:33:04.323 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.833816+0000", 2023-09-23T20:33:04.323 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:33:04.323 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.323 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.323 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:33:04.323 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.833816+0000", 2023-09-23T20:33:04.323 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:33:04.323 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.324 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.324 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:33:04.324 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.833815+0000", 2023-09-23T20:33:04.324 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:33:04.324 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.324 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.324 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:33:04.324 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.833821+0000", 2023-09-23T20:33:04.324 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.122e-06 2023-09-23T20:33:04.324 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.324 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.324 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:33:04.325 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.833832+0000", 2023-09-23T20:33:04.325 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1338999999999999e-05 2023-09-23T20:33:04.325 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.325 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.325 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:33:04.325 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.833839+0000", 2023-09-23T20:33:04.325 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.0419999999999999e-06 2023-09-23T20:33:04.325 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.325 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.325 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:33:04.325 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.898829+0000", 2023-09-23T20:33:04.325 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.064990508000000002 2023-09-23T20:33:04.326 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.326 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.326 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for degraded object", 2023-09-23T20:33:04.326 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.899653+0000", 2023-09-23T20:33:04.326 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00082397299999999998 2023-09-23T20:33:04.326 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.326 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.326 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:33:04.326 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.945495+0000", 2023-09-23T20:33:04.326 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.045841848999999997 2023-09-23T20:33:04.326 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.327 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.327 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:33:04.327 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.945501+0000", 2023-09-23T20:33:04.327 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.3770000000000001e-06 2023-09-23T20:33:04.327 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.327 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.327 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:33:04.327 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.124574+0000", 2023-09-23T20:33:04.327 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1790724560000001 2023-09-23T20:33:04.327 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.327 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.328 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for degraded object", 2023-09-23T20:33:04.328 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.125110+0000", 2023-09-23T20:33:04.328 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00053609000000000003 2023-09-23T20:33:04.328 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.328 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.328 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:33:04.328 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.258824+0000", 2023-09-23T20:33:04.328 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.133714534 2023-09-23T20:33:04.328 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.328 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.328 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:33:04.328 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.258902+0000", 2023-09-23T20:33:04.329 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.7299000000000006e-05 2023-09-23T20:33:04.329 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.329 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.329 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:33:04.329 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.259000+0000", 2023-09-23T20:33:04.329 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.8595e-05 2023-09-23T20:33:04.329 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.329 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.329 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:33:04.329 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.269120+0000", 2023-09-23T20:33:04.329 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.010120013000000001 2023-09-23T20:33:04.330 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.330 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.330 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:33:04.330 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.272674+0000", 2023-09-23T20:33:04.330 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0035536449999999998 2023-09-23T20:33:04.330 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.330 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.330 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:33:04.330 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.272702+0000", 2023-09-23T20:33:04.330 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.8435000000000001e-05 2023-09-23T20:33:04.330 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.331 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.331 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:33:04.331 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.272739+0000", 2023-09-23T20:33:04.331 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.6727999999999998e-05 2023-09-23T20:33:04.331 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.331 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.331 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:33:04.331 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.276390+0000", 2023-09-23T20:33:04.331 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0036506640000000001 2023-09-23T20:33:04.331 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:33:04.331 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:33:04.331 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:33:04.331 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.332 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.332 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:126894 5.a 5:5259ce05:::benchmark_data_smithi018_228451_object19833:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e164)", 2023-09-23T20:33:04.332 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:25:01.880498+0000", 2023-09-23T20:33:04.332 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 482.41366426299999, 2023-09-23T20:33:04.332 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.3921141399999999, 2023-09-23T20:33:04.332 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:33:04.332 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:33:04.332 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:33:04.332 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:33:04.332 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:33:04.332 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 126894 2023-09-23T20:33:04.333 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.333 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:33:04.333 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.333 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:33:04.333 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.880498+0000", 2023-09-23T20:33:04.333 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:33:04.333 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.333 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.333 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:33:04.333 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.880498+0000", 2023-09-23T20:33:04.333 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:33:04.333 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.334 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.334 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:33:04.334 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.880496+0000", 2023-09-23T20:33:04.334 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:33:04.334 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.334 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.334 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:33:04.334 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.880501+0000", 2023-09-23T20:33:04.334 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.904e-06 2023-09-23T20:33:04.334 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.334 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.335 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:33:04.335 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.880511+0000", 2023-09-23T20:33:04.335 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.5380000000000008e-06 2023-09-23T20:33:04.335 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.335 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.335 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:33:04.335 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.880516+0000", 2023-09-23T20:33:04.335 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.9980000000000003e-06 2023-09-23T20:33:04.335 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.335 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.335 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:33:04.336 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.899727+0000", 2023-09-23T20:33:04.336 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.019211236999999999 2023-09-23T20:33:04.336 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.336 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.336 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for degraded object", 2023-09-23T20:33:04.336 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.900501+0000", 2023-09-23T20:33:04.336 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00077373899999999996 2023-09-23T20:33:04.336 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.336 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.336 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:33:04.336 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.945461+0000", 2023-09-23T20:33:04.336 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.044960444000000002 2023-09-23T20:33:04.337 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.337 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.337 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:33:04.337 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.945472+0000", 2023-09-23T20:33:04.337 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0738e-05 2023-09-23T20:33:04.337 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.337 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.337 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:33:04.337 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.124180+0000", 2023-09-23T20:33:04.337 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1787084809999999 2023-09-23T20:33:04.337 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.338 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.338 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for degraded object", 2023-09-23T20:33:04.338 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.124532+0000", 2023-09-23T20:33:04.338 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00035113800000000003 2023-09-23T20:33:04.338 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.338 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.338 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:33:04.338 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.250481+0000", 2023-09-23T20:33:04.338 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.12594944799999999 2023-09-23T20:33:04.338 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.338 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.339 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:33:04.339 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.250553+0000", 2023-09-23T20:33:04.339 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.1903000000000006e-05 2023-09-23T20:33:04.339 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.339 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.339 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:33:04.339 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.250646+0000", 2023-09-23T20:33:04.339 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.3293000000000003e-05 2023-09-23T20:33:04.339 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.339 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.339 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:33:04.339 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.269045+0000", 2023-09-23T20:33:04.340 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.018399174000000001 2023-09-23T20:33:04.340 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.340 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.340 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:33:04.340 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.269114+0000", 2023-09-23T20:33:04.340 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.8677000000000007e-05 2023-09-23T20:33:04.340 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.340 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.340 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:33:04.340 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.269161+0000", 2023-09-23T20:33:04.340 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.7015999999999998e-05 2023-09-23T20:33:04.340 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.341 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.341 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:33:04.341 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.269214+0000", 2023-09-23T20:33:04.341 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.3128999999999999e-05 2023-09-23T20:33:04.341 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.341 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.341 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:33:04.341 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.272612+0000", 2023-09-23T20:33:04.341 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0033978659999999998 2023-09-23T20:33:04.341 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:33:04.341 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:33:04.342 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:33:04.342 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.342 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.342 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:126925 5.a 5:5e6040ba:::benchmark_data_smithi018_228451_object19864:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e165)", 2023-09-23T20:33:04.342 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:25:01.937985+0000", 2023-09-23T20:33:04.342 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 482.35617713699997, 2023-09-23T20:33:04.342 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.341994892, 2023-09-23T20:33:04.343 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:33:04.343 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:33:04.343 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:33:04.343 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:33:04.343 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:33:04.343 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 126925 2023-09-23T20:33:04.343 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.344 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:33:04.344 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.344 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:33:04.344 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.937985+0000", 2023-09-23T20:33:04.344 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:33:04.344 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.345 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.345 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:33:04.345 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.937985+0000", 2023-09-23T20:33:04.345 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:33:04.345 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.345 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.346 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:33:04.346 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.937983+0000", 2023-09-23T20:33:04.346 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999981 2023-09-23T20:33:04.346 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.346 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.346 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:33:04.346 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.937989+0000", 2023-09-23T20:33:04.347 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.1920000000000002e-06 2023-09-23T20:33:04.347 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.347 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.347 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:33:04.347 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.937998+0000", 2023-09-23T20:33:04.347 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.5250000000000005e-06 2023-09-23T20:33:04.347 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.348 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.348 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:33:04.348 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.938003+0000", 2023-09-23T20:33:04.348 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.2129999999999999e-06 2023-09-23T20:33:04.348 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.348 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.348 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:33:04.348 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.947727+0000", 2023-09-23T20:33:04.348 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0097239900000000001 2023-09-23T20:33:04.349 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.349 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.349 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:33:04.349 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.947739+0000", 2023-09-23T20:33:04.349 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1355000000000001e-05 2023-09-23T20:33:04.349 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.349 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.349 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:33:04.349 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.125146+0000", 2023-09-23T20:33:04.349 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1774076529999999 2023-09-23T20:33:04.349 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.349 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.350 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for degraded object", 2023-09-23T20:33:04.350 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.125724+0000", 2023-09-23T20:33:04.350 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00057758600000000005 2023-09-23T20:33:04.350 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.350 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.350 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:33:04.350 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.266528+0000", 2023-09-23T20:33:04.350 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.14080421800000001 2023-09-23T20:33:04.350 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.350 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.350 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:33:04.351 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.266606+0000", 2023-09-23T20:33:04.351 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.7725999999999994e-05 2023-09-23T20:33:04.351 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.351 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.351 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:33:04.351 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.266716+0000", 2023-09-23T20:33:04.351 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00011035999999999999 2023-09-23T20:33:04.351 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.351 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.351 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:33:04.351 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.276441+0000", 2023-09-23T20:33:04.351 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0097248620000000008 2023-09-23T20:33:04.352 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.352 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.352 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:33:04.352 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.276449+0000", 2023-09-23T20:33:04.352 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.0299999999999994e-06 2023-09-23T20:33:04.352 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.352 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.352 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:33:04.352 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.276482+0000", 2023-09-23T20:33:04.352 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.2796999999999998e-05 2023-09-23T20:33:04.352 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.353 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.353 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:33:04.353 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.276533+0000", 2023-09-23T20:33:04.353 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.1218000000000003e-05 2023-09-23T20:33:04.353 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.353 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.353 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:33:04.353 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.279980+0000", 2023-09-23T20:33:04.353 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0034469639999999998 2023-09-23T20:33:04.353 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:33:04.353 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:33:04.353 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:33:04.354 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.354 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.354 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:126935 5.1 5:8421d65c:::benchmark_data_smithi018_228451_object19874:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e165)", 2023-09-23T20:33:04.354 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:25:01.964401+0000", 2023-09-23T20:33:04.354 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 482.32976164500002, 2023-09-23T20:33:04.354 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0449979250000001, 2023-09-23T20:33:04.354 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:33:04.354 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:33:04.354 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:33:04.354 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:33:04.355 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:33:04.355 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 126935 2023-09-23T20:33:04.355 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.355 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:33:04.355 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.355 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:33:04.355 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.964401+0000", 2023-09-23T20:33:04.355 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:33:04.355 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.355 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.355 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:33:04.356 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.964401+0000", 2023-09-23T20:33:04.356 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:33:04.356 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.356 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.356 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:33:04.356 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.964398+0000", 2023-09-23T20:33:04.356 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999976 2023-09-23T20:33:04.356 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.356 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.356 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:33:04.356 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.964405+0000", 2023-09-23T20:33:04.356 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.8800000000000002e-06 2023-09-23T20:33:04.357 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.357 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.357 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:33:04.357 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.964423+0000", 2023-09-23T20:33:04.357 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.7864e-05 2023-09-23T20:33:04.357 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.357 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.357 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:33:04.357 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.964429+0000", 2023-09-23T20:33:04.357 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.3910000000000003e-06 2023-09-23T20:33:04.357 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.357 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.358 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:33:04.358 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.970829+0000", 2023-09-23T20:33:04.358 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0063990360000000003 2023-09-23T20:33:04.358 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.358 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.358 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:33:04.358 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.970843+0000", 2023-09-23T20:33:04.358 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.49e-05 2023-09-23T20:33:04.358 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.358 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.358 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:33:04.359 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.002512+0000", 2023-09-23T20:33:04.359 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.031668544 2023-09-23T20:33:04.359 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.359 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.359 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:33:04.359 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.002617+0000", 2023-09-23T20:33:04.359 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00010461 2023-09-23T20:33:04.359 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.359 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.359 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 3,7", 2023-09-23T20:33:04.359 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.002730+0000", 2023-09-23T20:33:04.359 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00011356700000000001 2023-09-23T20:33:04.360 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.360 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.360 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:33:04.360 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.007261+0000", 2023-09-23T20:33:04.360 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0045309240000000004 2023-09-23T20:33:04.360 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.360 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.360 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:33:04.360 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.008570+0000", 2023-09-23T20:33:04.360 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.001309225 2023-09-23T20:33:04.360 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.361 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.361 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:33:04.361 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.009210+0000", 2023-09-23T20:33:04.361 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00063932400000000003 2023-09-23T20:33:04.361 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.361 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.361 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:33:04.361 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.009257+0000", 2023-09-23T20:33:04.361 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.7692999999999999e-05 2023-09-23T20:33:04.361 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.361 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.361 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:33:04.362 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.009398+0000", 2023-09-23T20:33:04.362 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000141123 2023-09-23T20:33:04.362 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:33:04.362 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:33:04.362 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:33:04.362 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.362 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.362 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:126945 5.1 5:856ddac8:::benchmark_data_smithi018_228451_object19884:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e165)", 2023-09-23T20:33:04.362 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:25:01.991896+0000", 2023-09-23T20:33:04.362 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 482.302266551, 2023-09-23T20:33:04.362 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0216826400000001, 2023-09-23T20:33:04.363 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:33:04.363 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:33:04.363 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:33:04.363 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:33:04.363 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:33:04.363 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 126945 2023-09-23T20:33:04.363 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.363 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:33:04.363 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.363 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:33:04.363 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.991896+0000", 2023-09-23T20:33:04.363 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:33:04.364 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.364 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.364 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:33:04.364 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.991896+0000", 2023-09-23T20:33:04.364 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:33:04.364 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.364 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.364 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:33:04.364 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.991894+0000", 2023-09-23T20:33:04.364 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:33:04.364 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.365 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.365 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:33:04.365 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.991900+0000", 2023-09-23T20:33:04.365 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.2790000000000004e-06 2023-09-23T20:33:04.365 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.365 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.365 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:33:04.365 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.991914+0000", 2023-09-23T20:33:04.365 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.3226e-05 2023-09-23T20:33:04.365 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.365 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.365 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:33:04.366 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.991920+0000", 2023-09-23T20:33:04.366 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.7710000000000001e-06 2023-09-23T20:33:04.366 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.366 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.366 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:33:04.366 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.991975+0000", 2023-09-23T20:33:04.366 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.5021000000000001e-05 2023-09-23T20:33:04.366 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.366 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.366 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:33:04.366 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.991985+0000", 2023-09-23T20:33:04.367 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.5999999999999996e-06 2023-09-23T20:33:04.367 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.367 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.367 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:33:04.367 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.004845+0000", 2023-09-23T20:33:04.367 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0128602600000001 2023-09-23T20:33:04.367 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.367 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.367 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:33:04.367 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.004955+0000", 2023-09-23T20:33:04.367 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000109419 2023-09-23T20:33:04.368 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.368 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.368 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 3,7", 2023-09-23T20:33:04.368 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.005108+0000", 2023-09-23T20:33:04.368 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00015359099999999999 2023-09-23T20:33:04.368 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.368 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.368 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:33:04.368 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.007360+0000", 2023-09-23T20:33:04.368 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0022521389999999998 2023-09-23T20:33:04.368 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.368 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.369 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:33:04.369 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.008577+0000", 2023-09-23T20:33:04.369 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00121667 2023-09-23T20:33:04.369 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.369 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.369 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:33:04.369 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.013397+0000", 2023-09-23T20:33:04.369 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0048202760000000001 2023-09-23T20:33:04.369 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.369 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.369 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:33:04.370 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.013444+0000", 2023-09-23T20:33:04.370 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.6585000000000002e-05 2023-09-23T20:33:04.370 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.370 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.370 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:33:04.370 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.013578+0000", 2023-09-23T20:33:04.370 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00013424899999999999 2023-09-23T20:33:04.370 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:33:04.370 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:33:04.370 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:33:04.371 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.371 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.371 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:134103 5.18 5:1940be66:::benchmark_data_smithi018_228451_object27042:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e169)", 2023-09-23T20:33:04.371 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:25:14.071252+0000", 2023-09-23T20:33:04.371 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 470.22291007299998, 2023-09-23T20:33:04.371 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0302794790000001, 2023-09-23T20:33:04.371 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:33:04.371 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:33:04.371 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:33:04.371 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:33:04.371 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:33:04.371 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 134103 2023-09-23T20:33:04.372 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.372 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:33:04.372 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.372 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:33:04.372 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.071252+0000", 2023-09-23T20:33:04.372 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:33:04.372 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.372 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.372 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:33:04.372 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.071252+0000", 2023-09-23T20:33:04.372 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:33:04.373 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.373 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.373 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:33:04.373 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.071251+0000", 2023-09-23T20:33:04.373 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:33:04.373 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.373 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.373 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:33:04.373 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.071255+0000", 2023-09-23T20:33:04.373 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.335e-06 2023-09-23T20:33:04.373 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.373 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.374 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:33:04.374 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.071266+0000", 2023-09-23T20:33:04.374 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0402e-05 2023-09-23T20:33:04.374 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.374 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.374 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:33:04.374 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.071272+0000", 2023-09-23T20:33:04.374 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.3609999999999997e-06 2023-09-23T20:33:04.374 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.374 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.374 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:33:04.375 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.071501+0000", 2023-09-23T20:33:04.375 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00022861499999999999 2023-09-23T20:33:04.375 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.375 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.375 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:33:04.375 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.071508+0000", 2023-09-23T20:33:04.375 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.8290000000000008e-06 2023-09-23T20:33:04.375 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.375 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.375 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:33:04.375 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.073848+0000", 2023-09-23T20:33:04.375 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0023397380000001 2023-09-23T20:33:04.376 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.376 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.376 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:33:04.376 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.073991+0000", 2023-09-23T20:33:04.376 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00014334999999999999 2023-09-23T20:33:04.376 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.376 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.376 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 1,6", 2023-09-23T20:33:04.376 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.074073+0000", 2023-09-23T20:33:04.376 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.1357e-05 2023-09-23T20:33:04.376 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.376 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.377 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:33:04.377 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.086054+0000", 2023-09-23T20:33:04.377 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.011981103 2023-09-23T20:33:04.377 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.377 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.377 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:33:04.377 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.101412+0000", 2023-09-23T20:33:04.377 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.015358449999999999 2023-09-23T20:33:04.377 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.377 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.377 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:33:04.377 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.101441+0000", 2023-09-23T20:33:04.377 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.8416999999999999e-05 2023-09-23T20:33:04.378 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.378 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.378 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:33:04.378 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.101492+0000", 2023-09-23T20:33:04.378 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.1372000000000002e-05 2023-09-23T20:33:04.378 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.378 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.378 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:33:04.378 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.101532+0000", 2023-09-23T20:33:04.378 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.9385999999999999e-05 2023-09-23T20:33:04.379 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:33:04.379 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:33:04.379 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:33:04.379 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.379 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.379 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:134131 5.18 5:1c0b9433:::benchmark_data_smithi018_228451_object27070:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e169)", 2023-09-23T20:33:04.379 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:25:14.103862+0000", 2023-09-23T20:33:04.379 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 470.19030056700001, 2023-09-23T20:33:04.380 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.99783719299999996, 2023-09-23T20:33:04.380 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:33:04.380 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:33:04.380 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:33:04.380 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:33:04.380 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:33:04.380 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 134131 2023-09-23T20:33:04.380 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.380 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:33:04.380 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.380 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:33:04.381 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.103862+0000", 2023-09-23T20:33:04.381 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:33:04.381 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.381 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.381 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:33:04.381 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.103862+0000", 2023-09-23T20:33:04.381 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:33:04.381 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.381 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.381 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:33:04.381 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.103861+0000", 2023-09-23T20:33:04.382 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:33:04.382 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.382 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.382 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:33:04.382 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.103864+0000", 2023-09-23T20:33:04.382 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.4109999999999999e-06 2023-09-23T20:33:04.382 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.382 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.382 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:33:04.383 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.103870+0000", 2023-09-23T20:33:04.383 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.0190000000000003e-06 2023-09-23T20:33:04.383 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.383 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.383 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:33:04.383 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.103873+0000", 2023-09-23T20:33:04.383 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.6049999999999998e-06 2023-09-23T20:33:04.383 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.383 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.383 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:33:04.383 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.103936+0000", 2023-09-23T20:33:04.384 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.2351e-05 2023-09-23T20:33:04.384 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.384 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.384 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:33:04.384 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.103942+0000", 2023-09-23T20:33:04.384 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.7009999999999999e-06 2023-09-23T20:33:04.384 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.384 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.384 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:33:04.384 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.076330+0000", 2023-09-23T20:33:04.384 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.97238725299999995 2023-09-23T20:33:04.385 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.385 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.385 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:33:04.385 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.076434+0000", 2023-09-23T20:33:04.385 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00010414 2023-09-23T20:33:04.385 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.385 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.385 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 1,6", 2023-09-23T20:33:04.385 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.076538+0000", 2023-09-23T20:33:04.385 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000104041 2023-09-23T20:33:04.385 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.386 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.386 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:33:04.386 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.086064+0000", 2023-09-23T20:33:04.386 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0095265880000000008 2023-09-23T20:33:04.386 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.386 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.386 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:33:04.386 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.101562+0000", 2023-09-23T20:33:04.386 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.015497858 2023-09-23T20:33:04.386 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.386 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.387 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:33:04.387 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.101613+0000", 2023-09-23T20:33:04.387 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.0283e-05 2023-09-23T20:33:04.387 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.387 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.387 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:33:04.387 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.101660+0000", 2023-09-23T20:33:04.387 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.7633999999999998e-05 2023-09-23T20:33:04.387 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.387 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.387 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:33:04.388 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.101699+0000", 2023-09-23T20:33:04.388 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.8646999999999998e-05 2023-09-23T20:33:04.388 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:33:04.388 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:33:04.388 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:33:04.388 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.388 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.388 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:134133 5.18 5:1bf4905b:::benchmark_data_smithi018_228451_object27072:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e169)", 2023-09-23T20:33:04.388 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:25:14.109028+0000", 2023-09-23T20:33:04.388 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 470.18513456400001, 2023-09-23T20:33:04.388 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.99277708499999995, 2023-09-23T20:33:04.389 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:33:04.389 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:33:04.389 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:33:04.389 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:33:04.389 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:33:04.389 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 134133 2023-09-23T20:33:04.389 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.389 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:33:04.389 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.390 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:33:04.390 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.109028+0000", 2023-09-23T20:33:04.390 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:33:04.390 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.390 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.390 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:33:04.390 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.109028+0000", 2023-09-23T20:33:04.390 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:33:04.390 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.390 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.391 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:33:04.391 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.109027+0000", 2023-09-23T20:33:04.391 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:33:04.391 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.391 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.391 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:33:04.391 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.109031+0000", 2023-09-23T20:33:04.391 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.4290000000000003e-06 2023-09-23T20:33:04.392 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.392 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.392 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:33:04.392 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.109047+0000", 2023-09-23T20:33:04.392 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.6314999999999999e-05 2023-09-23T20:33:04.392 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.392 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.392 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:33:04.392 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.109054+0000", 2023-09-23T20:33:04.392 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.6370000000000002e-06 2023-09-23T20:33:04.392 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.393 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.393 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:33:04.393 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.109142+0000", 2023-09-23T20:33:04.393 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.7954999999999994e-05 2023-09-23T20:33:04.393 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.393 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.393 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:33:04.393 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.109150+0000", 2023-09-23T20:33:04.393 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.4999999999999999e-06 2023-09-23T20:33:04.393 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.393 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.393 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:33:04.394 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.078060+0000", 2023-09-23T20:33:04.394 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.96890956900000003 2023-09-23T20:33:04.394 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.394 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.394 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:33:04.394 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.078173+0000", 2023-09-23T20:33:04.394 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000113341 2023-09-23T20:33:04.394 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.394 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.394 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 1,6", 2023-09-23T20:33:04.395 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.078295+0000", 2023-09-23T20:33:04.395 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000121894 2023-09-23T20:33:04.395 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.395 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.395 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:33:04.395 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.086074+0000", 2023-09-23T20:33:04.395 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0077791140000000002 2023-09-23T20:33:04.395 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.395 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.395 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:33:04.395 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.101588+0000", 2023-09-23T20:33:04.395 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.015513968 2023-09-23T20:33:04.396 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.396 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.396 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:33:04.396 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.101732+0000", 2023-09-23T20:33:04.396 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00014366499999999999 2023-09-23T20:33:04.396 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.396 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.396 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:33:04.396 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.101772+0000", 2023-09-23T20:33:04.396 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.0204999999999999e-05 2023-09-23T20:33:04.396 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.396 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.397 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:33:04.397 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.101805+0000", 2023-09-23T20:33:04.397 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.2564000000000002e-05 2023-09-23T20:33:04.397 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:33:04.397 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:33:04.397 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:33:04.397 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.397 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.397 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:134135 5.18 5:1df619a8:::benchmark_data_smithi018_228451_object27074:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e169)", 2023-09-23T20:33:04.397 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:25:14.109193+0000", 2023-09-23T20:33:04.397 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 470.18496933500001, 2023-09-23T20:33:04.397 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.99274740800000005, 2023-09-23T20:33:04.398 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:33:04.398 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:33:04.398 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:33:04.398 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:33:04.398 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:33:04.398 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 134135 2023-09-23T20:33:04.398 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.398 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:33:04.398 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.398 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:33:04.398 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.109193+0000", 2023-09-23T20:33:04.399 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:33:04.399 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.399 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.399 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:33:04.399 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.109193+0000", 2023-09-23T20:33:04.399 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:33:04.399 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.399 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.399 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:33:04.399 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.109192+0000", 2023-09-23T20:33:04.399 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:33:04.399 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.400 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.400 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:33:04.400 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.109196+0000", 2023-09-23T20:33:04.400 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.2740000000000001e-06 2023-09-23T20:33:04.400 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.400 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.400 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:33:04.400 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.109205+0000", 2023-09-23T20:33:04.400 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.0970000000000006e-06 2023-09-23T20:33:04.400 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.400 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.400 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:33:04.401 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.109211+0000", 2023-09-23T20:33:04.401 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.5400000000000001e-06 2023-09-23T20:33:04.401 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.401 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.401 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:33:04.401 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.109256+0000", 2023-09-23T20:33:04.401 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.5410000000000001e-05 2023-09-23T20:33:04.401 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.401 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.401 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:33:04.401 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.109267+0000", 2023-09-23T20:33:04.402 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.004e-05 2023-09-23T20:33:04.402 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.402 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.402 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:33:04.402 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.079350+0000", 2023-09-23T20:33:04.402 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.970083841 2023-09-23T20:33:04.402 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.402 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.402 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:33:04.402 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.079462+0000", 2023-09-23T20:33:04.403 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00011156 2023-09-23T20:33:04.403 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.403 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.403 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 1,6", 2023-09-23T20:33:04.403 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.079535+0000", 2023-09-23T20:33:04.403 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.2638000000000005e-05 2023-09-23T20:33:04.403 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.403 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.403 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:33:04.403 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.086084+0000", 2023-09-23T20:33:04.403 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.006549109 2023-09-23T20:33:04.403 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.404 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.404 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:33:04.404 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.101837+0000", 2023-09-23T20:33:04.404 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.015753775000000001 2023-09-23T20:33:04.404 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.404 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.404 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:33:04.404 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.101896+0000", 2023-09-23T20:33:04.404 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.8525e-05 2023-09-23T20:33:04.404 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.404 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.404 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:33:04.404 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.101921+0000", 2023-09-23T20:33:04.405 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.4905e-05 2023-09-23T20:33:04.405 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.405 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.405 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:33:04.405 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.101940+0000", 2023-09-23T20:33:04.405 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.9368000000000002e-05 2023-09-23T20:33:04.405 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:33:04.405 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:33:04.405 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:33:04.405 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.405 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.405 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:134176 5.18 5:1f2720a7:::benchmark_data_smithi018_228451_object27115:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e169)", 2023-09-23T20:33:04.406 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:25:14.171790+0000", 2023-09-23T20:33:04.406 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 470.122372176, 2023-09-23T20:33:04.406 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.93025216899999996, 2023-09-23T20:33:04.406 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:33:04.406 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:33:04.406 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:33:04.406 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:33:04.406 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:33:04.406 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 134176 2023-09-23T20:33:04.406 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.406 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:33:04.407 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.407 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:33:04.407 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.171790+0000", 2023-09-23T20:33:04.407 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:33:04.407 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.407 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.407 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:33:04.407 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.171790+0000", 2023-09-23T20:33:04.407 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:33:04.407 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.407 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.408 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:33:04.408 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.171790+0000", 2023-09-23T20:33:04.408 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:33:04.408 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.408 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.408 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:33:04.408 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.171792+0000", 2023-09-23T20:33:04.408 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.5330000000000001e-06 2023-09-23T20:33:04.408 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.408 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.409 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:33:04.409 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.171797+0000", 2023-09-23T20:33:04.409 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.2109999999999997e-06 2023-09-23T20:33:04.409 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.409 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.409 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:33:04.409 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.171801+0000", 2023-09-23T20:33:04.409 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.709e-06 2023-09-23T20:33:04.409 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.409 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.410 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:33:04.410 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.172312+0000", 2023-09-23T20:33:04.410 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00051119399999999997 2023-09-23T20:33:04.410 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.410 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.410 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:33:04.410 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.172327+0000", 2023-09-23T20:33:04.410 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.4389000000000001e-05 2023-09-23T20:33:04.410 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.410 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.410 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:33:04.411 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.080465+0000", 2023-09-23T20:33:04.411 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.90813792800000004 2023-09-23T20:33:04.411 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.411 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.411 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:33:04.411 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.080593+0000", 2023-09-23T20:33:04.411 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00012820799999999999 2023-09-23T20:33:04.411 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.411 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.411 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 1,6", 2023-09-23T20:33:04.411 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.080723+0000", 2023-09-23T20:33:04.411 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000130362 2023-09-23T20:33:04.412 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.412 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.412 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:33:04.412 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.090073+0000", 2023-09-23T20:33:04.412 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0093501950000000004 2023-09-23T20:33:04.412 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.412 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.412 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:33:04.412 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.101864+0000", 2023-09-23T20:33:04.412 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.011790429 2023-09-23T20:33:04.412 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.413 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.413 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:33:04.413 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.102002+0000", 2023-09-23T20:33:04.413 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00013855900000000001 2023-09-23T20:33:04.413 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.413 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.413 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:33:04.413 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.102024+0000", 2023-09-23T20:33:04.413 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.1926e-05 2023-09-23T20:33:04.413 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.413 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.413 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:33:04.414 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.102042+0000", 2023-09-23T20:33:04.414 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.7946000000000001e-05 2023-09-23T20:33:04.414 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:33:04.414 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:33:04.414 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:33:04.414 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.414 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.415 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:134225 5.18 5:18f2ddba:::benchmark_data_smithi018_228451_object27164:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e169)", 2023-09-23T20:33:04.415 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:25:14.238878+0000", 2023-09-23T20:33:04.415 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 470.05528404400002, 2023-09-23T20:33:04.415 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.86321889900000004, 2023-09-23T20:33:04.415 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:33:04.415 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:33:04.415 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:33:04.415 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:33:04.415 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:33:04.415 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 134225 2023-09-23T20:33:04.415 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.416 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:33:04.416 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.416 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:33:04.416 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.238878+0000", 2023-09-23T20:33:04.416 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:33:04.416 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.416 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.416 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:33:04.416 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.238878+0000", 2023-09-23T20:33:04.416 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:33:04.416 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.417 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.417 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:33:04.417 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.238877+0000", 2023-09-23T20:33:04.417 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:33:04.417 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.417 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.417 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:33:04.417 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.238881+0000", 2023-09-23T20:33:04.417 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.42e-06 2023-09-23T20:33:04.417 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.417 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.418 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:33:04.418 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.238893+0000", 2023-09-23T20:33:04.418 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1618e-05 2023-09-23T20:33:04.418 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.418 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.418 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:33:04.418 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.238898+0000", 2023-09-23T20:33:04.418 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.2800000000000003e-06 2023-09-23T20:33:04.418 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.419 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.419 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:33:04.419 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.238954+0000", 2023-09-23T20:33:04.419 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.5578999999999998e-05 2023-09-23T20:33:04.419 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.419 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.419 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:33:04.419 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:14.238965+0000", 2023-09-23T20:33:04.419 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0740999999999999e-05 2023-09-23T20:33:04.420 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.420 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.420 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:33:04.420 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.082298+0000", 2023-09-23T20:33:04.420 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.843333361 2023-09-23T20:33:04.420 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.420 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.420 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:33:04.420 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.082458+0000", 2023-09-23T20:33:04.420 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000159937 2023-09-23T20:33:04.420 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.420 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.421 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 1,6", 2023-09-23T20:33:04.421 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.082595+0000", 2023-09-23T20:33:04.421 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00013719000000000001 2023-09-23T20:33:04.421 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.421 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.421 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:33:04.421 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.090083+0000", 2023-09-23T20:33:04.421 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0074878849999999997 2023-09-23T20:33:04.421 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.421 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.421 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:33:04.421 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.101879+0000", 2023-09-23T20:33:04.422 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.011796152000000001 2023-09-23T20:33:04.422 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.422 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.422 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:33:04.422 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.102058+0000", 2023-09-23T20:33:04.422 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00017879199999999999 2023-09-23T20:33:04.422 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.422 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.422 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:33:04.422 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.102079+0000", 2023-09-23T20:33:04.422 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.1396e-05 2023-09-23T20:33:04.423 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.423 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.423 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:33:04.423 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:15.102097+0000", 2023-09-23T20:33:04.423 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.7615999999999999e-05 2023-09-23T20:33:04.423 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:33:04.423 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:33:04.423 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:33:04.423 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.423 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.423 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:150491 5.1 5:83c4f1b1:::benchmark_data_smithi018_228451_object43430:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e178)", 2023-09-23T20:33:04.424 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:25:40.788685+0000", 2023-09-23T20:33:04.424 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 443.50547671200002, 2023-09-23T20:33:04.424 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.86834999599999996, 2023-09-23T20:33:04.424 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:33:04.424 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:33:04.424 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:33:04.424 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:33:04.424 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:33:04.424 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 150491 2023-09-23T20:33:04.424 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.424 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:33:04.425 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.425 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:33:04.425 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:40.788685+0000", 2023-09-23T20:33:04.425 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:33:04.425 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.425 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.425 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:33:04.425 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:40.788685+0000", 2023-09-23T20:33:04.425 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:33:04.425 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.425 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.425 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:33:04.426 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:40.788684+0000", 2023-09-23T20:33:04.426 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:33:04.426 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.426 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.426 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:33:04.426 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:40.788689+0000", 2023-09-23T20:33:04.426 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.9599999999999999e-06 2023-09-23T20:33:04.426 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.426 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.426 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:33:04.426 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:40.788696+0000", 2023-09-23T20:33:04.427 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.3899999999999998e-06 2023-09-23T20:33:04.427 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.427 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.427 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:33:04.427 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:40.788703+0000", 2023-09-23T20:33:04.427 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.4189999999999996e-06 2023-09-23T20:33:04.427 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.427 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.427 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:33:04.427 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:40.788750+0000", 2023-09-23T20:33:04.427 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.7205e-05 2023-09-23T20:33:04.428 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.428 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.428 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:33:04.428 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:40.788756+0000", 2023-09-23T20:33:04.428 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.003e-06 2023-09-23T20:33:04.428 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.428 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.428 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:33:04.428 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:41.630396+0000", 2023-09-23T20:33:04.428 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.84163923299999999 2023-09-23T20:33:04.428 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.428 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.429 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:33:04.429 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:41.630529+0000", 2023-09-23T20:33:04.429 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00013375400000000001 2023-09-23T20:33:04.429 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.429 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.429 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 3,7", 2023-09-23T20:33:04.429 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:41.630649+0000", 2023-09-23T20:33:04.429 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000119683 2023-09-23T20:33:04.429 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.429 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.429 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:33:04.429 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:41.640197+0000", 2023-09-23T20:33:04.429 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0095476759999999997 2023-09-23T20:33:04.430 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.430 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.430 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:33:04.430 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:41.656929+0000", 2023-09-23T20:33:04.430 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.016732059000000001 2023-09-23T20:33:04.430 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.430 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.430 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:33:04.430 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:41.656958+0000", 2023-09-23T20:33:04.430 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.8915e-05 2023-09-23T20:33:04.430 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.431 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.431 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:33:04.431 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:41.657000+0000", 2023-09-23T20:33:04.431 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.2722999999999997e-05 2023-09-23T20:33:04.431 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.431 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.431 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:33:04.431 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:41.657035+0000", 2023-09-23T20:33:04.431 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.5107000000000001e-05 2023-09-23T20:33:04.431 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:33:04.431 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:33:04.432 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:33:04.432 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.432 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.432 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35700.0:57591 7.1 7.7a2f2aa1 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e312)", 2023-09-23T20:33:04.432 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:32:32.227723+0000", 2023-09-23T20:33:04.432 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 32.066438832000003, 2023-09-23T20:33:04.432 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.99202805599999999, 2023-09-23T20:33:04.432 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:33:04.432 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:33:04.432 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:33:04.432 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35700", 2023-09-23T20:33:04.432 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:33:04.433 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 57591 2023-09-23T20:33:04.433 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.433 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:33:04.433 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.433 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:33:04.433 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:32:32.227723+0000", 2023-09-23T20:33:04.433 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:33:04.433 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.433 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.433 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:33:04.433 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:32:32.227723+0000", 2023-09-23T20:33:04.433 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:33:04.434 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.434 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.434 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:33:04.434 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:32:32.227722+0000", 2023-09-23T20:33:04.434 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:33:04.434 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.434 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.434 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:33:04.434 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:32:32.228185+0000", 2023-09-23T20:33:04.434 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00046322699999999997 2023-09-23T20:33:04.434 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.435 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.435 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:33:04.435 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:32:32.228205+0000", 2023-09-23T20:33:04.435 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.9927000000000001e-05 2023-09-23T20:33:04.435 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.435 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.435 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:33:04.435 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:32:32.228213+0000", 2023-09-23T20:33:04.435 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.6359999999999994e-06 2023-09-23T20:33:04.435 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.435 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.435 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:33:04.436 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:32:32.228446+0000", 2023-09-23T20:33:04.436 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00023344300000000001 2023-09-23T20:33:04.436 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.436 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.436 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:33:04.436 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:32:32.228454+0000", 2023-09-23T20:33:04.436 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.2029999999999992e-06 2023-09-23T20:33:04.436 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.436 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.437 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:33:04.437 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:32:33.219740+0000", 2023-09-23T20:33:04.437 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.99128563800000002 2023-09-23T20:33:04.437 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.437 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.437 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:33:04.437 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:32:33.219751+0000", 2023-09-23T20:33:04.437 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1409e-05 2023-09-23T20:33:04.437 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:33:04.437 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:33:04.437 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:33:04.437 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.438 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.438 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35700.0:57595 7.1 7.19a18ad1 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e312)", 2023-09-23T20:33:04.438 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:32:32.228233+0000", 2023-09-23T20:33:04.438 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 32.065929044000001, 2023-09-23T20:33:04.438 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.99154681200000006, 2023-09-23T20:33:04.438 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:33:04.438 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:33:04.438 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:33:04.438 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35700", 2023-09-23T20:33:04.439 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:33:04.439 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 57595 2023-09-23T20:33:04.439 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.439 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:33:04.439 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.439 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:33:04.439 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:32:32.228233+0000", 2023-09-23T20:33:04.440 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:33:04.440 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.440 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.440 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:33:04.440 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:32:32.228233+0000", 2023-09-23T20:33:04.440 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:33:04.441 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.441 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.441 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:33:04.441 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:32:32.228233+0000", 2023-09-23T20:33:04.441 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:33:04.441 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.442 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.442 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:33:04.442 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:32:32.228272+0000", 2023-09-23T20:33:04.442 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.9016e-05 2023-09-23T20:33:04.442 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.443 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.443 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:33:04.443 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:32:32.228283+0000", 2023-09-23T20:33:04.443 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1591e-05 2023-09-23T20:33:04.443 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.443 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.444 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:33:04.444 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:32:32.228289+0000", 2023-09-23T20:33:04.444 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.9989999999999997e-06 2023-09-23T20:33:04.444 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.444 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.444 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:33:04.445 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:32:32.228476+0000", 2023-09-23T20:33:04.445 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00018691899999999999 2023-09-23T20:33:04.445 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.445 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.445 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:33:04.446 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:32:32.228481+0000", 2023-09-23T20:33:04.446 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.6469999999999997e-06 2023-09-23T20:33:04.446 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.446 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.446 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:33:04.446 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:32:33.219773+0000", 2023-09-23T20:33:04.447 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.99129225700000001 2023-09-23T20:33:04.447 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.447 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.447 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:33:04.447 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:32:33.219780+0000", 2023-09-23T20:33:04.447 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.9800000000000001e-06 2023-09-23T20:33:04.447 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:33:04.448 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:33:04.448 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:33:04.448 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.448 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.448 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35700.0:57641 7.1 7.be520741 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e312)", 2023-09-23T20:33:04.448 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:32:32.269916+0000", 2023-09-23T20:33:04.449 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 32.024246437000002, 2023-09-23T20:33:04.449 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.94988938700000003, 2023-09-23T20:33:04.449 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:33:04.449 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:33:04.449 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:33:04.449 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35700", 2023-09-23T20:33:04.450 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:33:04.450 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 57641 2023-09-23T20:33:04.450 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.450 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:33:04.450 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.451 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:33:04.451 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:32:32.269916+0000", 2023-09-23T20:33:04.451 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:33:04.451 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.451 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.451 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:33:04.452 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:32:32.269916+0000", 2023-09-23T20:33:04.452 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:33:04.452 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.452 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.452 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:33:04.452 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:32:32.269913+0000", 2023-09-23T20:33:04.453 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999976 2023-09-23T20:33:04.453 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.453 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.453 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:33:04.453 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:32:32.269961+0000", 2023-09-23T20:33:04.454 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.7499e-05 2023-09-23T20:33:04.454 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.454 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.454 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:33:04.454 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:32:32.269987+0000", 2023-09-23T20:33:04.455 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.5522999999999999e-05 2023-09-23T20:33:04.455 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.455 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.455 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:33:04.455 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:32:32.269996+0000", 2023-09-23T20:33:04.455 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.4879999999999996e-06 2023-09-23T20:33:04.456 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.456 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.456 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:33:04.456 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:32:32.270066+0000", 2023-09-23T20:33:04.456 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.9770000000000005e-05 2023-09-23T20:33:04.457 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.457 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.457 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:33:04.457 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:32:32.270074+0000", 2023-09-23T20:33:04.457 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.8550000000000002e-06 2023-09-23T20:33:04.457 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.458 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.458 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:33:04.458 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:32:33.219798+0000", 2023-09-23T20:33:04.458 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.94972451999999996 2023-09-23T20:33:04.458 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.459 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.459 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:33:04.459 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:32:33.219805+0000", 2023-09-23T20:33:04.459 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.9630000000000001e-06 2023-09-23T20:33:04.459 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:33:04.459 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:33:04.460 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:33:04.460 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.460 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.460 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35700.0:57647 7.1 7.b3786c71 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e312)", 2023-09-23T20:33:04.460 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:32:32.275658+0000", 2023-09-23T20:33:04.460 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 32.018504299999996, 2023-09-23T20:33:04.460 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.94417131399999998, 2023-09-23T20:33:04.460 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:33:04.461 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:33:04.461 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:33:04.461 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35700", 2023-09-23T20:33:04.461 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:33:04.461 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 57647 2023-09-23T20:33:04.461 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.461 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:33:04.462 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.462 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:33:04.462 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:32:32.275658+0000", 2023-09-23T20:33:04.462 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:33:04.462 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.462 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.462 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:33:04.463 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:32:32.275658+0000", 2023-09-23T20:33:04.463 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:33:04.463 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.463 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.463 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:33:04.463 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:32:32.275655+0000", 2023-09-23T20:33:04.463 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999976 2023-09-23T20:33:04.464 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.464 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.464 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:33:04.464 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:32:32.275743+0000", 2023-09-23T20:33:04.464 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.7860000000000002e-05 2023-09-23T20:33:04.464 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.464 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.464 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:33:04.464 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:32:32.275769+0000", 2023-09-23T20:33:04.464 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.5711000000000002e-05 2023-09-23T20:33:04.464 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.464 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.465 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:33:04.465 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:32:32.275779+0000", 2023-09-23T20:33:04.465 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0327e-05 2023-09-23T20:33:04.465 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.465 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.465 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:33:04.465 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:32:32.275851+0000", 2023-09-23T20:33:04.465 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.1360999999999997e-05 2023-09-23T20:33:04.465 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.465 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.465 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:33:04.466 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:32:32.275862+0000", 2023-09-23T20:33:04.466 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1173e-05 2023-09-23T20:33:04.466 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.466 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.466 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:33:04.466 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:32:33.219822+0000", 2023-09-23T20:33:04.466 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.94396049000000004 2023-09-23T20:33:04.466 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.466 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.466 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:33:04.466 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:32:33.219829+0000", 2023-09-23T20:33:04.467 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.8399999999999997e-06 2023-09-23T20:33:04.467 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:33:04.467 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:33:04.467 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:33:04.467 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.467 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.467 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35700.0:57652 7.1 7.95417c01 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e312)", 2023-09-23T20:33:04.467 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:32:32.281114+0000", 2023-09-23T20:33:04.467 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 32.013048546, 2023-09-23T20:33:04.467 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.93874086199999995, 2023-09-23T20:33:04.467 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:33:04.467 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:33:04.468 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:33:04.468 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35700", 2023-09-23T20:33:04.468 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:33:04.468 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 57652 2023-09-23T20:33:04.468 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.468 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:33:04.468 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.468 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:33:04.468 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:32:32.281114+0000", 2023-09-23T20:33:04.468 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:33:04.468 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.468 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.469 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:33:04.469 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:32:32.281114+0000", 2023-09-23T20:33:04.469 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:33:04.469 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.469 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.469 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:33:04.469 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:32:32.281112+0000", 2023-09-23T20:33:04.469 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999981 2023-09-23T20:33:04.469 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.469 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.469 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:33:04.469 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:32:32.281163+0000", 2023-09-23T20:33:04.470 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.1251000000000001e-05 2023-09-23T20:33:04.470 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.470 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.470 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:33:04.470 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:32:32.281186+0000", 2023-09-23T20:33:04.470 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.2416999999999999e-05 2023-09-23T20:33:04.470 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.470 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.470 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:33:04.470 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:32:32.281195+0000", 2023-09-23T20:33:04.471 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.6180000000000001e-06 2023-09-23T20:33:04.471 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.471 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.471 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:33:04.471 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:32:32.281254+0000", 2023-09-23T20:33:04.471 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.8884000000000001e-05 2023-09-23T20:33:04.471 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.471 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.471 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:33:04.471 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:32:32.281265+0000", 2023-09-23T20:33:04.471 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.111e-05 2023-09-23T20:33:04.472 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.472 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.472 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:33:04.472 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:32:33.219848+0000", 2023-09-23T20:33:04.472 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.93858254200000002 2023-09-23T20:33:04.472 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:04.472 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:04.472 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:33:04.472 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:32:33.219854+0000", 2023-09-23T20:33:04.472 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.8190000000000003e-06 2023-09-23T20:33:04.472 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:33:04.472 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:33:04.473 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:33:04.473 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:33:04.473 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:33:04.473 INFO:teuthology.orchestra.run.smithi149.stdout:} 2023-09-23T20:33:04.473 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:33:04.505 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:04 smithi149 ceph-mon[170394]: 1.4 scrub starts 2023-09-23T20:33:04.505 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:04 smithi149 ceph-mon[170394]: osdmap e326: 8 total, 8 up, 8 in 2023-09-23T20:33:04.506 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:04 smithi149 ceph-mon[170394]: failed to encode map e326 with expected crc 2023-09-23T20:33:04.506 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:04 smithi149 ceph-mon[170394]: failed to encode map e326 with expected crc 2023-09-23T20:33:04.506 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:04 smithi149 ceph-mon[170394]: failed to encode map e326 with expected crc 2023-09-23T20:33:04.506 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:04 smithi149 ceph-mon[170394]: failed to encode map e326 with expected crc 2023-09-23T20:33:04.506 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:04 smithi149 ceph-mon[170394]: failed to encode map e326 with expected crc 2023-09-23T20:33:04.510 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:04.548 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:33:04.645 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:04 smithi018 ceph-mon[179647]: 1.4 scrub starts 2023-09-23T20:33:04.645 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:04 smithi018 ceph-mon[179647]: osdmap e326: 8 total, 8 up, 8 in 2023-09-23T20:33:04.645 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:04 smithi018 ceph-mon[179647]: failed to encode map e326 with expected crc 2023-09-23T20:33:04.646 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:04 smithi018 ceph-mon[179647]: failed to encode map e326 with expected crc 2023-09-23T20:33:04.646 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:04 smithi018 ceph-mon[179647]: failed to encode map e326 with expected crc 2023-09-23T20:33:04.646 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:04 smithi018 ceph-mon[179647]: failed to encode map e326 with expected crc 2023-09-23T20:33:04.646 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:04 smithi018 ceph-mon[179647]: failed to encode map e326 with expected crc 2023-09-23T20:33:04.646 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:04 smithi018 ceph-mon[184144]: 1.4 scrub starts 2023-09-23T20:33:04.646 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:04 smithi018 ceph-mon[184144]: osdmap e326: 8 total, 8 up, 8 in 2023-09-23T20:33:04.646 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:04 smithi018 ceph-mon[184144]: failed to encode map e326 with expected crc 2023-09-23T20:33:04.646 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:04 smithi018 ceph-mon[184144]: failed to encode map e326 with expected crc 2023-09-23T20:33:04.646 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:04 smithi018 ceph-mon[184144]: failed to encode map e326 with expected crc 2023-09-23T20:33:04.647 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:04 smithi018 ceph-mon[184144]: failed to encode map e326 with expected crc 2023-09-23T20:33:04.647 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:04 smithi018 ceph-mon[184144]: failed to encode map e326 with expected crc 2023-09-23T20:33:04.661 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:33:04.668 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:04.669 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:33:04.760 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T20:33:04.770 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:33:04.804 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 68 16 78205 78189 71.8199 52.9375 0.0234606 0.0139185 2023-09-23T20:33:04.841 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:04.909 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:33:04.914 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:04.914 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:33:05.015 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:33:05.072 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:05.128 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:33:05.131 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:05.132 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:33:05.145 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:33:05 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:33:04] "GET /metrics HTTP/1.1" 200 36554 "" "Prometheus/2.43.0" 2023-09-23T20:33:05.233 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:33:05.269 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:33:05.314 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:05.318 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 5, 7, 3, 2, 6, 4] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T20:33:05.318 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:33:05.318 INFO:tasks.thrashosds.thrasher:fixing pg num pool .mgr 2023-09-23T20:33:05.318 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:33:05.399 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:33:05.410 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:05.410 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:33:05.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:05 smithi149 ceph-mon[170394]: pgmap v751: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped, 1 peering, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 4.6 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 32 MiB/s wr, 517 op/s; 4068/226623 objects degraded (1.795%); 4456/226623 objects misplaced (1.966%); 3.9 MiB/s, 63 objects/s recovering 2023-09-23T20:33:05.499 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:33:05.511 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:33:05.544 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:05.637 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:33:05.640 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:05.641 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:33:05.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:05 smithi018 ceph-mon[179647]: pgmap v751: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped, 1 peering, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 4.6 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 32 MiB/s wr, 517 op/s; 4068/226623 objects degraded (1.795%); 4456/226623 objects misplaced (1.966%); 3.9 MiB/s, 63 objects/s recovering 2023-09-23T20:33:05.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:05 smithi018 ceph-mon[184144]: pgmap v751: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped, 1 peering, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 4.6 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 32 MiB/s wr, 517 op/s; 4068/226623 objects degraded (1.795%); 4456/226623 objects misplaced (1.966%); 3.9 MiB/s, 63 objects/s recovering 2023-09-23T20:33:05.742 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:33:05.778 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:05.805 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 69 16 78938 78922 71.4431 45.8125 0.0274457 0.0139918 2023-09-23T20:33:05.841 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:33:05.847 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:05.847 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:33:05.948 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:33:05.992 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:06.167 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:33:06.171 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:06.172 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:33:06.272 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:33:06.386 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:06.432 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:06 smithi149 ceph-mon[170394]: 1.4 scrub starts 2023-09-23T20:33:06.432 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:06 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 4068/226623 objects degraded (1.795%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T20:33:06.478 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:33:06.488 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:06.488 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:33:06.489 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:06 smithi018 ceph-mon[179647]: 1.4 scrub starts 2023-09-23T20:33:06.489 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:06 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 4068/226623 objects degraded (1.795%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T20:33:06.495 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:06 smithi018 ceph-mon[184144]: 1.4 scrub starts 2023-09-23T20:33:06.496 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:06 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 4068/226623 objects degraded (1.795%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T20:33:06.589 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:33:06.629 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:06.679 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:33:06.683 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:06.683 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:33:06.726 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:33:06.770 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:33:06.784 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:33:06.810 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 70 16 79707 79691 71.1037 48.0625 0.0157693 0.0140573 2023-09-23T20:33:06.820 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:06.873 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:33:06.879 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:06.879 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:33:06.919 INFO:teuthology.orchestra.run.smithi149.stdout:{ 2023-09-23T20:33:06.919 INFO:teuthology.orchestra.run.smithi149.stdout: "ops": [ 2023-09-23T20:33:06.919 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:06.919 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35700.0:79763 7.0 7:0a8301a3:::benchmark_data_smithi018_308601_object79762:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e326)", 2023-09-23T20:33:06.919 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:33:06.897139+0000", 2023-09-23T20:33:06.920 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 0.021330064999999999, 2023-09-23T20:33:06.920 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.021368248999999999, 2023-09-23T20:33:06.920 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:33:06.920 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:33:06.920 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:33:06.920 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35700", 2023-09-23T20:33:06.920 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:33:06.920 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 79763 2023-09-23T20:33:06.920 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:06.920 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:33:06.920 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:06.920 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:33:06.921 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:06.897139+0000", 2023-09-23T20:33:06.921 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:33:06.921 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:06.921 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:06.921 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:33:06.921 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:06.897139+0000", 2023-09-23T20:33:06.921 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:33:06.921 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:06.921 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:06.921 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:33:06.921 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:06.897137+0000", 2023-09-23T20:33:06.921 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999976 2023-09-23T20:33:06.922 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:06.922 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:06.922 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:33:06.922 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:06.897189+0000", 2023-09-23T20:33:06.922 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.2720999999999999e-05 2023-09-23T20:33:06.922 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:06.922 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:06.922 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:33:06.922 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:06.897210+0000", 2023-09-23T20:33:06.922 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.0559000000000001e-05 2023-09-23T20:33:06.922 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:06.922 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:06.923 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:33:06.923 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:06.897218+0000", 2023-09-23T20:33:06.923 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.3780000000000001e-06 2023-09-23T20:33:06.923 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:06.923 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:06.923 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:33:06.923 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:06.897287+0000", 2023-09-23T20:33:06.923 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.9029999999999995e-05 2023-09-23T20:33:06.923 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:06.923 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:06.923 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:33:06.923 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:06.897471+0000", 2023-09-23T20:33:06.924 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00018329699999999999 2023-09-23T20:33:06.924 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:06.924 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:06.924 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:33:06.924 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:06.897628+0000", 2023-09-23T20:33:06.924 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00015748300000000001 2023-09-23T20:33:06.924 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:06.924 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:06.924 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:33:06.924 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:06.900747+0000", 2023-09-23T20:33:06.924 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0031189109999999998 2023-09-23T20:33:06.924 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:06.925 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:06.925 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:33:06.925 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:06.904569+0000", 2023-09-23T20:33:06.925 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0038221290000000001 2023-09-23T20:33:06.925 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:33:06.925 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:33:06.925 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:33:06.925 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:06.925 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:06.925 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35700.0:79773 7.0 7:0ac921ee:::benchmark_data_smithi018_308601_object79772:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e326)", 2023-09-23T20:33:06.926 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:33:06.911346+0000", 2023-09-23T20:33:06.926 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 0.0071232860000000004, 2023-09-23T20:33:06.926 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0072246410000000004, 2023-09-23T20:33:06.926 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:33:06.926 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:33:06.926 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:33:06.926 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35700", 2023-09-23T20:33:06.926 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:33:06.926 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 79773 2023-09-23T20:33:06.927 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:06.927 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:33:06.927 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:06.927 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:33:06.927 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:06.911346+0000", 2023-09-23T20:33:06.927 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:33:06.927 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:06.927 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:06.927 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:33:06.927 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:06.911346+0000", 2023-09-23T20:33:06.927 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:33:06.928 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:06.928 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:06.928 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:33:06.928 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:06.911345+0000", 2023-09-23T20:33:06.928 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:33:06.928 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:06.929 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:06.929 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:33:06.929 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:06.911391+0000", 2023-09-23T20:33:06.929 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.5725e-05 2023-09-23T20:33:06.929 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:06.929 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:06.929 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:33:06.929 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:06.911403+0000", 2023-09-23T20:33:06.929 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.2592000000000001e-05 2023-09-23T20:33:06.929 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:06.929 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:06.930 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:33:06.930 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:06.911408+0000", 2023-09-23T20:33:06.930 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.8509999999999998e-06 2023-09-23T20:33:06.930 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:06.930 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:06.930 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:33:06.930 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:06.911463+0000", 2023-09-23T20:33:06.930 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.5337000000000002e-05 2023-09-23T20:33:06.931 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:06.931 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:06.931 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:33:06.931 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:06.911632+0000", 2023-09-23T20:33:06.931 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00016851 2023-09-23T20:33:06.931 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:06.931 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:06.931 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:33:06.931 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:06.911808+0000", 2023-09-23T20:33:06.931 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00017629599999999999 2023-09-23T20:33:06.931 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:06.931 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:06.932 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:33:06.932 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:06.914958+0000", 2023-09-23T20:33:06.932 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0031501530000000002 2023-09-23T20:33:06.932 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:33:06.932 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:33:06.932 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:33:06.932 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:33:06.932 INFO:teuthology.orchestra.run.smithi149.stdout: ], 2023-09-23T20:33:06.933 INFO:teuthology.orchestra.run.smithi149.stdout: "num_ops": 2 2023-09-23T20:33:06.933 INFO:teuthology.orchestra.run.smithi149.stdout:} 2023-09-23T20:33:06.980 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:33:07.080 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:07.202 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:33:07.208 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:07.209 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:33:07.296 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T20:33:07.310 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:33:07.330 INFO:teuthology.orchestra.run.smithi018.stderr:nodeep-scrub is unset 2023-09-23T20:33:07.363 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:07 smithi018 ceph-mon[179647]: pgmap v752: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 51 MiB/s wr, 809 op/s; 5117/234048 objects degraded (2.186%); 3404/234048 objects misplaced (1.454%); 2.8 MiB/s, 45 objects/s recovering 2023-09-23T20:33:07.363 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:07 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3646123815' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:33:07.364 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:07 smithi018 ceph-mon[184144]: pgmap v752: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 51 MiB/s wr, 809 op/s; 5117/234048 objects degraded (2.186%); 3404/234048 objects misplaced (1.454%); 2.8 MiB/s, 45 objects/s recovering 2023-09-23T20:33:07.364 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:07 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3646123815' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:33:07.431 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:07.561 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:33:07.570 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:07.570 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:33:07.671 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:33:07.704 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:07 smithi149 ceph-mon[170394]: pgmap v752: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 51 MiB/s wr, 809 op/s; 5117/234048 objects degraded (2.186%); 3404/234048 objects misplaced (1.454%); 2.8 MiB/s, 45 objects/s recovering 2023-09-23T20:33:07.704 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:07 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3646123815' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:33:07.707 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:07.766 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:33:07.773 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:07.774 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:33:07.813 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 71 16 80426 80410 70.7328 44.9375 0.0156923 0.0141326 2023-09-23T20:33:07.874 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:33:07.945 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:08.027 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:33:08.277 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:33:08.286 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:08.287 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:33:08.388 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:33:08.460 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:08 smithi018 ceph-mon[184144]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:33:08.460 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:08 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3646123815' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:33:08.461 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:08 smithi018 ceph-mon[184144]: osdmap e327: 8 total, 8 up, 8 in 2023-09-23T20:33:08.461 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:08 smithi018 ceph-mon[184144]: failed to encode map e327 with expected crc 2023-09-23T20:33:08.461 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:08 smithi018 ceph-mon[184144]: failed to encode map e327 with expected crc 2023-09-23T20:33:08.461 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:08 smithi018 ceph-mon[184144]: failed to encode map e327 with expected crc 2023-09-23T20:33:08.461 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:08 smithi018 ceph-mon[184144]: failed to encode map e327 with expected crc 2023-09-23T20:33:08.464 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:08 smithi018 ceph-mon[179647]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:33:08.464 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:08 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3646123815' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:33:08.464 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:08 smithi018 ceph-mon[179647]: osdmap e327: 8 total, 8 up, 8 in 2023-09-23T20:33:08.464 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:08 smithi018 ceph-mon[179647]: failed to encode map e327 with expected crc 2023-09-23T20:33:08.464 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:08 smithi018 ceph-mon[179647]: failed to encode map e327 with expected crc 2023-09-23T20:33:08.465 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:08 smithi018 ceph-mon[179647]: failed to encode map e327 with expected crc 2023-09-23T20:33:08.465 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:08 smithi018 ceph-mon[179647]: failed to encode map e327 with expected crc 2023-09-23T20:33:08.478 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:08.574 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:08 smithi149 ceph-mon[170394]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:33:08.574 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:08 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3646123815' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:33:08.574 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:08 smithi149 ceph-mon[170394]: osdmap e327: 8 total, 8 up, 8 in 2023-09-23T20:33:08.574 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:08 smithi149 ceph-mon[170394]: failed to encode map e327 with expected crc 2023-09-23T20:33:08.574 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:08 smithi149 ceph-mon[170394]: failed to encode map e327 with expected crc 2023-09-23T20:33:08.574 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:08 smithi149 ceph-mon[170394]: failed to encode map e327 with expected crc 2023-09-23T20:33:08.574 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:08 smithi149 ceph-mon[170394]: failed to encode map e327 with expected crc 2023-09-23T20:33:08.607 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:33:08.626 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:08.627 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:33:08.727 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:33:08.813 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 72 16 81233 81217 70.451 50.4375 0.0156672 0.014188 2023-09-23T20:33:08.822 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:08.907 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:33:08.915 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:08.916 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:33:09.017 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:33:09.047 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:33:09.061 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:09.081 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:33:09.107 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:33:09.112 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:09.113 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:33:09.120 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:33:09.162 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:33:09.204 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:33:09.214 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:33:09.248 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:33:09.292 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:33:09.311 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:09.342 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:33:09.343 INFO:teuthology.orchestra.run.smithi149.stdout:ERROR: (22) Invalid argument 2023-09-23T20:33:09.343 INFO:teuthology.orchestra.run.smithi149.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:33:09.373 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:09 smithi149 ceph-mon[170394]: pgmap v754: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 41 MiB/s wr, 662 op/s; 5117/236901 objects degraded (2.160%); 3213/236901 objects misplaced (1.356%); 4.5 MiB/s, 71 objects/s recovering 2023-09-23T20:33:09.373 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:09 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2761356491' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:33:09.377 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:33:09.377 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:33:09.378 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:33:09.378 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:33:09.378 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:33:09.378 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:33:09.378 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:33:09.378 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:33:09.393 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:33:09.404 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:09.405 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:33:09.505 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:33:09.582 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:09 smithi018 ceph-mon[179647]: pgmap v754: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 41 MiB/s wr, 662 op/s; 5117/236901 objects degraded (2.160%); 3213/236901 objects misplaced (1.356%); 4.5 MiB/s, 71 objects/s recovering 2023-09-23T20:33:09.582 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:09 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2761356491' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:33:09.582 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:09 smithi018 ceph-mon[184144]: pgmap v754: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 41 MiB/s wr, 662 op/s; 5117/236901 objects degraded (2.160%); 3213/236901 objects misplaced (1.356%); 4.5 MiB/s, 71 objects/s recovering 2023-09-23T20:33:09.582 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:09 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2761356491' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:33:09.597 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:09.729 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T20:33:09.813 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 73 16 81953 81937 70.1026 45 0.0175925 0.01426 2023-09-23T20:33:09.959 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:33:09.968 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:09.968 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:33:10.069 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:33:10.160 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:10.224 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:33:10.229 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:33:10.231 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T20:33:10.233 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:10.234 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:33:10.267 INFO:teuthology.orchestra.run.smithi018.stderr:dumped all 2023-09-23T20:33:10.335 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:33:10.440 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:33:10.491 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:10 smithi018 ceph-mon[179647]: 1.4 scrub starts 2023-09-23T20:33:10.492 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:10 smithi018 ceph-mon[184144]: 1.4 scrub starts 2023-09-23T20:33:10.516 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:10.652 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:10 smithi149 ceph-mon[170394]: 1.4 scrub starts 2023-09-23T20:33:10.813 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 74 16 82705 82689 69.7905 47 0.0217889 0.0143237 2023-09-23T20:33:11.566 INFO:teuthology.orchestra.run.smithi018.stdout:{"pg_ready":true,"pg_map":{"version":755,"stamp":"2023-09-23T20:33:09.779814+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5207492672,"num_objects":79418,"num_object_clones":0,"num_object_copies":238254,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5117,"num_objects_misplaced":3213,"num_objects_unfound":0,"num_objects_dirty":79418,"num_whiteouts":0,"num_read":6009,"num_read_kb":13239,"num_write":80304,"num_write_kb":5093758,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1642,"num_bytes_recovered":104917830,"num_keys_recovered":40,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":83802,"ondisk_log_size":83802,"up":327,"acting":323,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":337,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":18068252,"kb_used_data":15600028,"kb_used_omap":32,"kb_used_meta":2468127,"kb_avail":731892964,"statfs":{"total":767960285184,"available":749458395136,"internally_reserved":0,"allocated":15974428672,"data_stored":15963742728,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32773,"internal_metadata":2527363067},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,3,7,8,5,2,3],"upper_bound":128},"perf_stat":{"commit_latency_ms":10,"apply_latency_ms":10,"commit_latency_ns":10000000,"apply_latency_ns":10000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":290193408,"num_objects":4428,"num_object_clones":0,"num_object_copies":13284,"num_objects_missing_on_primary":0,"num_objects_missing":-1,"num_objects_degraded":-2589,"num_objects_misplaced":2161,"num_objects_unfound":0,"num_objects_dirty":4428,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4428,"num_write_kb":283392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":428,"num_bytes_recovered":28049408,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"7.551511"},"pg_stats":[{"pgid":"3.1a","version":"69'1","reported_seq":23,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T20:33:02.243927+0000","last_change":"2023-09-23T20:32:33.233853+0000","last_active":"2023-09-23T20:33:02.243927+0000","last_peered":"2023-09-23T20:33:02.243927+0000","last_clean":"2023-09-23T20:33:02.243927+0000","last_became_active":"2023-09-23T20:32:33.233546+0000","last_became_peered":"2023-09-23T20:32:33.233546+0000","last_unstale":"2023-09-23T20:33:02.243927+0000","last_undegraded":"2023-09-23T20:33:02.243927+0000","last_fullsized":"2023-09-23T20:33:02.243927+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:50:27.130808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":205,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-09-23T20:33:03.266290+0000","last_change":"2023-09-23T20:32:50.536495+0000","last_active":"2023-09-23T20:33:03.266290+0000","last_peered":"2023-09-23T20:33:03.266290+0000","last_clean":"2023-09-23T20:33:03.266290+0000","last_became_active":"2023-09-23T20:32:50.536180+0000","last_became_peered":"2023-09-23T20:32:50.536180+0000","last_unstale":"2023-09-23T20:33:03.266290+0000","last_undegraded":"2023-09-23T20:33:03.266290+0000","last_fullsized":"2023-09-23T20:33:03.266290+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:01:05.524308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":283,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-09-23T20:33:03.277970+0000","last_change":"2023-09-23T20:33:00.181677+0000","last_active":"2023-09-23T20:33:03.277970+0000","last_peered":"2023-09-23T20:33:03.277970+0000","last_clean":"2023-09-23T20:33:03.277970+0000","last_became_active":"2023-09-23T20:33:00.181400+0000","last_became_peered":"2023-09-23T20:33:00.181400+0000","last_unstale":"2023-09-23T20:33:03.277970+0000","last_undegraded":"2023-09-23T20:33:03.277970+0000","last_fullsized":"2023-09-23T20:33:03.277970+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:52:14.600448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":292,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T20:33:02.261345+0000","last_change":"2023-09-23T20:25:02.957185+0000","last_active":"2023-09-23T20:33:02.261345+0000","last_peered":"2023-09-23T20:33:02.261345+0000","last_clean":"2023-09-23T20:33:02.261345+0000","last_became_active":"2023-09-23T20:25:02.956689+0000","last_became_peered":"2023-09-23T20:25:02.956689+0000","last_unstale":"2023-09-23T20:33:02.261345+0000","last_undegraded":"2023-09-23T20:33:02.261345+0000","last_fullsized":"2023-09-23T20:33:02.261345+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:04:28.524015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":137,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T20:33:02.253894+0000","last_change":"2023-09-23T20:32:50.546428+0000","last_active":"2023-09-23T20:33:02.253894+0000","last_peered":"2023-09-23T20:33:02.253894+0000","last_clean":"2023-09-23T20:33:02.253894+0000","last_became_active":"2023-09-23T20:32:50.546120+0000","last_became_peered":"2023-09-23T20:32:50.546120+0000","last_unstale":"2023-09-23T20:33:02.253894+0000","last_undegraded":"2023-09-23T20:33:02.253894+0000","last_fullsized":"2023-09-23T20:33:02.253894+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:00:10.792410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":99,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T20:33:02.251424+0000","last_change":"2023-09-23T20:29:46.112977+0000","last_active":"2023-09-23T20:33:02.251424+0000","last_peered":"2023-09-23T20:33:02.251424+0000","last_clean":"2023-09-23T20:33:02.251424+0000","last_became_active":"2023-09-23T20:29:46.112600+0000","last_became_peered":"2023-09-23T20:29:46.112600+0000","last_unstale":"2023-09-23T20:33:02.251424+0000","last_undegraded":"2023-09-23T20:33:02.251424+0000","last_fullsized":"2023-09-23T20:33:02.251424+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:06:34.132247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":290,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T20:33:02.264512+0000","last_change":"2023-09-23T20:25:02.969432+0000","last_active":"2023-09-23T20:33:02.264512+0000","last_peered":"2023-09-23T20:33:02.264512+0000","last_clean":"2023-09-23T20:33:02.264512+0000","last_became_active":"2023-09-23T20:25:02.969118+0000","last_became_peered":"2023-09-23T20:25:02.969118+0000","last_unstale":"2023-09-23T20:33:02.264512+0000","last_undegraded":"2023-09-23T20:33:02.264512+0000","last_fullsized":"2023-09-23T20:33:02.264512+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:02:22.453831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":29,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-09-23T20:33:03.288519+0000","last_change":"2023-09-23T20:32:01.360359+0000","last_active":"2023-09-23T20:33:03.288519+0000","last_peered":"2023-09-23T20:33:03.288519+0000","last_clean":"2023-09-23T20:33:03.288519+0000","last_became_active":"2023-09-23T20:32:01.360008+0000","last_became_peered":"2023-09-23T20:32:01.360008+0000","last_unstale":"2023-09-23T20:33:03.288519+0000","last_undegraded":"2023-09-23T20:33:03.288519+0000","last_fullsized":"2023-09-23T20:33:03.288519+0000","mapping_epoch":308,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:33:43.135001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.14","version":"90'161","reported_seq":636,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-09-23T20:33:03.286663+0000","last_change":"2023-09-23T20:33:00.177495+0000","last_active":"2023-09-23T20:33:03.286663+0000","last_peered":"2023-09-23T20:33:03.286663+0000","last_clean":"2023-09-23T20:33:03.286663+0000","last_became_active":"2023-09-23T20:33:00.177086+0000","last_became_peered":"2023-09-23T20:33:00.177086+0000","last_unstale":"2023-09-23T20:33:03.286663+0000","last_undegraded":"2023-09-23T20:33:03.286663+0000","last_fullsized":"2023-09-23T20:33:03.286663+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:47.479012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:47.479012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:09:38.114270+0000","scrub_duration":0.0020136590000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":302,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T20:33:02.234149+0000","last_change":"2023-09-23T20:29:45.523808+0000","last_active":"2023-09-23T20:33:02.234149+0000","last_peered":"2023-09-23T20:33:02.234149+0000","last_clean":"2023-09-23T20:33:02.234149+0000","last_became_active":"2023-09-23T20:29:45.523508+0000","last_became_peered":"2023-09-23T20:29:45.523508+0000","last_unstale":"2023-09-23T20:33:02.234149+0000","last_undegraded":"2023-09-23T20:33:02.234149+0000","last_fullsized":"2023-09-23T20:33:02.234149+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:50:55.565352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":125,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T20:33:02.255335+0000","last_change":"2023-09-23T20:32:50.539003+0000","last_active":"2023-09-23T20:33:02.255335+0000","last_peered":"2023-09-23T20:33:02.255335+0000","last_clean":"2023-09-23T20:33:02.255335+0000","last_became_active":"2023-09-23T20:32:50.538596+0000","last_became_peered":"2023-09-23T20:32:50.538596+0000","last_unstale":"2023-09-23T20:33:02.255335+0000","last_undegraded":"2023-09-23T20:33:02.255335+0000","last_fullsized":"2023-09-23T20:33:02.255335+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:56:03.314375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":353,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T20:33:02.253559+0000","last_change":"2023-09-23T20:32:33.270617+0000","last_active":"2023-09-23T20:33:02.253559+0000","last_peered":"2023-09-23T20:33:02.253559+0000","last_clean":"2023-09-23T20:33:02.253559+0000","last_became_active":"2023-09-23T20:32:33.227902+0000","last_became_peered":"2023-09-23T20:32:33.227902+0000","last_unstale":"2023-09-23T20:33:02.253559+0000","last_undegraded":"2023-09-23T20:33:02.253559+0000","last_fullsized":"2023-09-23T20:33:02.253559+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:30:26.847831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":360,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":275,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-09-23T20:33:03.266047+0000","last_change":"2023-09-23T20:29:46.080766+0000","last_active":"2023-09-23T20:33:03.266047+0000","last_peered":"2023-09-23T20:33:03.266047+0000","last_clean":"2023-09-23T20:33:03.266047+0000","last_became_active":"2023-09-23T20:29:46.080356+0000","last_became_peered":"2023-09-23T20:29:46.080356+0000","last_unstale":"2023-09-23T20:33:03.266047+0000","last_undegraded":"2023-09-23T20:33:03.266047+0000","last_fullsized":"2023-09-23T20:33:03.266047+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:08:01.862270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":33,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T20:33:02.244173+0000","last_change":"2023-09-23T20:32:33.266787+0000","last_active":"2023-09-23T20:33:02.244173+0000","last_peered":"2023-09-23T20:33:02.244173+0000","last_clean":"2023-09-23T20:33:02.244173+0000","last_became_active":"2023-09-23T20:32:33.228482+0000","last_became_peered":"2023-09-23T20:32:33.228482+0000","last_unstale":"2023-09-23T20:33:02.244173+0000","last_undegraded":"2023-09-23T20:33:02.244173+0000","last_fullsized":"2023-09-23T20:33:02.244173+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:13:01.812073+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":23,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T20:33:02.244796+0000","last_change":"2023-09-23T20:32:33.228929+0000","last_active":"2023-09-23T20:33:02.244796+0000","last_peered":"2023-09-23T20:33:02.244796+0000","last_clean":"2023-09-23T20:33:02.244796+0000","last_became_active":"2023-09-23T20:32:33.228595+0000","last_became_peered":"2023-09-23T20:32:33.228595+0000","last_unstale":"2023-09-23T20:33:02.244796+0000","last_undegraded":"2023-09-23T20:33:02.244796+0000","last_fullsized":"2023-09-23T20:33:02.244796+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:17:53.053808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"90'161","reported_seq":555,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T20:33:02.265143+0000","last_change":"2023-09-23T20:31:43.578810+0000","last_active":"2023-09-23T20:33:02.265143+0000","last_peered":"2023-09-23T20:33:02.265143+0000","last_clean":"2023-09-23T20:33:02.265143+0000","last_became_active":"2023-09-23T20:28:05.991304+0000","last_became_peered":"2023-09-23T20:28:05.991304+0000","last_unstale":"2023-09-23T20:33:02.265143+0000","last_undegraded":"2023-09-23T20:33:02.265143+0000","last_fullsized":"2023-09-23T20:33:02.265143+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:43.578666+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:43.578666+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:46:18.999991+0000","scrub_duration":0.0021890379999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":352,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T20:33:02.253004+0000","last_change":"2023-09-23T20:32:33.242943+0000","last_active":"2023-09-23T20:33:02.253004+0000","last_peered":"2023-09-23T20:33:02.253004+0000","last_clean":"2023-09-23T20:33:02.253004+0000","last_became_active":"2023-09-23T20:32:33.225468+0000","last_became_peered":"2023-09-23T20:32:33.225468+0000","last_unstale":"2023-09-23T20:33:02.253004+0000","last_undegraded":"2023-09-23T20:33:02.253004+0000","last_fullsized":"2023-09-23T20:33:02.253004+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:40:47.867573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":27,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T20:33:02.233905+0000","last_change":"2023-09-23T20:32:01.358590+0000","last_active":"2023-09-23T20:33:02.233905+0000","last_peered":"2023-09-23T20:33:02.233905+0000","last_clean":"2023-09-23T20:33:02.233905+0000","last_became_active":"2023-09-23T20:32:01.358345+0000","last_became_peered":"2023-09-23T20:32:01.358345+0000","last_unstale":"2023-09-23T20:33:02.233905+0000","last_undegraded":"2023-09-23T20:33:02.233905+0000","last_fullsized":"2023-09-23T20:33:02.233905+0000","mapping_epoch":308,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:22:21.159708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.10","version":"90'161","reported_seq":556,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T20:33:02.233719+0000","last_change":"2023-09-23T20:31:49.726214+0000","last_active":"2023-09-23T20:33:02.233719+0000","last_peered":"2023-09-23T20:33:02.233719+0000","last_clean":"2023-09-23T20:33:02.233719+0000","last_became_active":"2023-09-23T20:29:07.044892+0000","last_became_peered":"2023-09-23T20:29:07.044892+0000","last_unstale":"2023-09-23T20:33:02.233719+0000","last_undegraded":"2023-09-23T20:33:02.233719+0000","last_fullsized":"2023-09-23T20:33:02.233719+0000","mapping_epoch":253,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":254,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:49.726139+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:49.726139+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:59:11.810516+0000","scrub_duration":0.0015441470000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":290,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T20:33:02.261209+0000","last_change":"2023-09-23T20:25:02.953993+0000","last_active":"2023-09-23T20:33:02.261209+0000","last_peered":"2023-09-23T20:33:02.261209+0000","last_clean":"2023-09-23T20:33:02.261209+0000","last_became_active":"2023-09-23T20:25:02.953672+0000","last_became_peered":"2023-09-23T20:25:02.953672+0000","last_unstale":"2023-09-23T20:33:02.261209+0000","last_undegraded":"2023-09-23T20:33:02.261209+0000","last_fullsized":"2023-09-23T20:33:02.261209+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:26:08.603577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":304,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T20:33:02.234125+0000","last_change":"2023-09-23T20:25:02.949584+0000","last_active":"2023-09-23T20:33:02.234125+0000","last_peered":"2023-09-23T20:33:02.234125+0000","last_clean":"2023-09-23T20:33:02.234125+0000","last_became_active":"2023-09-23T20:25:02.949132+0000","last_became_peered":"2023-09-23T20:25:02.949132+0000","last_unstale":"2023-09-23T20:33:02.234125+0000","last_undegraded":"2023-09-23T20:33:02.234125+0000","last_fullsized":"2023-09-23T20:33:02.234125+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:34:20.509823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"90'161","reported_seq":592,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-09-23T20:33:03.266434+0000","last_change":"2023-09-23T20:33:00.179977+0000","last_active":"2023-09-23T20:33:03.266434+0000","last_peered":"2023-09-23T20:33:03.266434+0000","last_clean":"2023-09-23T20:33:03.266434+0000","last_became_active":"2023-09-23T20:33:00.179654+0000","last_became_peered":"2023-09-23T20:33:00.179654+0000","last_unstale":"2023-09-23T20:33:03.266434+0000","last_undegraded":"2023-09-23T20:33:03.266434+0000","last_fullsized":"2023-09-23T20:33:03.266434+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:42.544308+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:42.544308+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:24:16.853678+0000","scrub_duration":0.0029958179999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":309,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T20:33:02.255260+0000","last_change":"2023-09-23T20:32:33.225148+0000","last_active":"2023-09-23T20:33:02.255260+0000","last_peered":"2023-09-23T20:33:02.255260+0000","last_clean":"2023-09-23T20:33:02.255260+0000","last_became_active":"2023-09-23T20:32:33.224805+0000","last_became_peered":"2023-09-23T20:32:33.224805+0000","last_unstale":"2023-09-23T20:33:02.255260+0000","last_undegraded":"2023-09-23T20:33:02.255260+0000","last_fullsized":"2023-09-23T20:33:02.255260+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:55:30.891398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":300,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T20:33:02.261509+0000","last_change":"2023-09-23T20:25:02.957010+0000","last_active":"2023-09-23T20:33:02.261509+0000","last_peered":"2023-09-23T20:33:02.261509+0000","last_clean":"2023-09-23T20:33:02.261509+0000","last_became_active":"2023-09-23T20:25:02.956267+0000","last_became_peered":"2023-09-23T20:25:02.956267+0000","last_unstale":"2023-09-23T20:33:02.261509+0000","last_undegraded":"2023-09-23T20:33:02.261509+0000","last_fullsized":"2023-09-23T20:33:02.261509+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:10:55.299422+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"90'161","reported_seq":575,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T20:33:02.234661+0000","last_change":"2023-09-23T20:31:53.707033+0000","last_active":"2023-09-23T20:33:02.234661+0000","last_peered":"2023-09-23T20:33:02.234661+0000","last_clean":"2023-09-23T20:33:02.234661+0000","last_became_active":"2023-09-23T20:29:46.078873+0000","last_became_peered":"2023-09-23T20:29:46.078873+0000","last_unstale":"2023-09-23T20:33:02.234661+0000","last_undegraded":"2023-09-23T20:33:02.234661+0000","last_fullsized":"2023-09-23T20:33:02.234661+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:53.706934+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:53.706934+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:51:58.039657+0000","scrub_duration":0.0016840900000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":95,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T20:33:02.249793+0000","last_change":"2023-09-23T20:29:04.179171+0000","last_active":"2023-09-23T20:33:02.249793+0000","last_peered":"2023-09-23T20:33:02.249793+0000","last_clean":"2023-09-23T20:33:02.249793+0000","last_became_active":"2023-09-23T20:29:04.128672+0000","last_became_peered":"2023-09-23T20:29:04.128672+0000","last_unstale":"2023-09-23T20:33:02.249793+0000","last_undegraded":"2023-09-23T20:33:02.249793+0000","last_fullsized":"2023-09-23T20:33:02.249793+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:49:15.506106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":138,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T20:33:02.255800+0000","last_change":"2023-09-23T20:32:50.539157+0000","last_active":"2023-09-23T20:33:02.255800+0000","last_peered":"2023-09-23T20:33:02.255800+0000","last_clean":"2023-09-23T20:33:02.255800+0000","last_became_active":"2023-09-23T20:32:50.538889+0000","last_became_peered":"2023-09-23T20:32:50.538889+0000","last_unstale":"2023-09-23T20:33:02.255800+0000","last_undegraded":"2023-09-23T20:33:02.255800+0000","last_fullsized":"2023-09-23T20:33:02.255800+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:28:28.857256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.b","version":"327'5086","reported_seq":5119,"reported_epoch":327,"state":"active+clean","last_fresh":"2023-09-23T20:33:08.782515+0000","last_change":"2023-09-23T20:31:49.152343+0000","last_active":"2023-09-23T20:33:08.782515+0000","last_peered":"2023-09-23T20:33:08.782515+0000","last_clean":"2023-09-23T20:33:08.782515+0000","last_became_active":"2023-09-23T20:31:49.152042+0000","last_became_peered":"2023-09-23T20:31:49.152042+0000","last_unstale":"2023-09-23T20:33:08.782515+0000","last_undegraded":"2023-09-23T20:33:08.782515+0000","last_fullsized":"2023-09-23T20:33:08.782515+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":5086,"log_dups_size":0,"ondisk_log_size":5086,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:53:34.237468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":333316096,"num_objects":5086,"num_object_clones":0,"num_object_copies":15258,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5086,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5086,"num_write_kb":325504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.d","version":"90'161","reported_seq":595,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T20:33:02.253332+0000","last_change":"2023-09-23T20:32:36.304873+0000","last_active":"2023-09-23T20:33:02.253332+0000","last_peered":"2023-09-23T20:33:02.253332+0000","last_clean":"2023-09-23T20:33:02.253332+0000","last_became_active":"2023-09-23T20:32:36.304524+0000","last_became_peered":"2023-09-23T20:32:36.304524+0000","last_unstale":"2023-09-23T20:33:02.253332+0000","last_undegraded":"2023-09-23T20:33:02.253332+0000","last_fullsized":"2023-09-23T20:33:02.253332+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:46.388545+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:46.388545+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:29:22.195276+0000","scrub_duration":0.00168356,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":329,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T20:33:02.255719+0000","last_change":"2023-09-23T20:32:33.283752+0000","last_active":"2023-09-23T20:33:02.255719+0000","last_peered":"2023-09-23T20:33:02.255719+0000","last_clean":"2023-09-23T20:33:02.255719+0000","last_became_active":"2023-09-23T20:32:33.230503+0000","last_became_peered":"2023-09-23T20:32:33.230503+0000","last_unstale":"2023-09-23T20:33:02.255719+0000","last_undegraded":"2023-09-23T20:33:02.255719+0000","last_fullsized":"2023-09-23T20:33:02.255719+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:29.084410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":287,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T20:33:02.233869+0000","last_change":"2023-09-23T20:29:04.128604+0000","last_active":"2023-09-23T20:33:02.233869+0000","last_peered":"2023-09-23T20:33:02.233869+0000","last_clean":"2023-09-23T20:33:02.233869+0000","last_became_active":"2023-09-23T20:29:04.128003+0000","last_became_peered":"2023-09-23T20:29:04.128003+0000","last_unstale":"2023-09-23T20:33:02.233869+0000","last_undegraded":"2023-09-23T20:33:02.233869+0000","last_fullsized":"2023-09-23T20:33:02.233869+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:50:49.385599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"327'5017","reported_seq":5056,"reported_epoch":327,"state":"active+clean","last_fresh":"2023-09-23T20:33:07.397732+0000","last_change":"2023-09-23T20:32:50.543873+0000","last_active":"2023-09-23T20:33:07.397732+0000","last_peered":"2023-09-23T20:33:07.397732+0000","last_clean":"2023-09-23T20:33:07.397732+0000","last_became_active":"2023-09-23T20:32:50.537253+0000","last_became_peered":"2023-09-23T20:32:50.537253+0000","last_unstale":"2023-09-23T20:33:07.397732+0000","last_undegraded":"2023-09-23T20:33:07.397732+0000","last_fullsized":"2023-09-23T20:33:07.397732+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":5017,"log_dups_size":0,"ondisk_log_size":5017,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:38:02.452291+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":328794112,"num_objects":5017,"num_object_clones":0,"num_object_copies":15051,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5017,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5017,"num_write_kb":321088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,5],"acting":[2,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.c","version":"90'161","reported_seq":486,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T20:33:02.263168+0000","last_change":"2023-09-23T20:32:50.534420+0000","last_active":"2023-09-23T20:33:02.263168+0000","last_peered":"2023-09-23T20:33:02.263168+0000","last_clean":"2023-09-23T20:33:02.263168+0000","last_became_active":"2023-09-23T20:32:50.533984+0000","last_became_peered":"2023-09-23T20:32:50.533984+0000","last_unstale":"2023-09-23T20:33:02.263168+0000","last_undegraded":"2023-09-23T20:33:02.263168+0000","last_fullsized":"2023-09-23T20:33:02.263168+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:41.701171+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:41.701171+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0019955509999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109227,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":350,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T20:33:02.255824+0000","last_change":"2023-09-23T20:33:00.196461+0000","last_active":"2023-09-23T20:33:02.255824+0000","last_peered":"2023-09-23T20:33:02.255824+0000","last_clean":"2023-09-23T20:33:02.255824+0000","last_became_active":"2023-09-23T20:33:00.178428+0000","last_became_peered":"2023-09-23T20:33:00.178428+0000","last_unstale":"2023-09-23T20:33:02.255824+0000","last_undegraded":"2023-09-23T20:33:02.255824+0000","last_fullsized":"2023-09-23T20:33:02.255824+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:48:19.958144+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":23,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T20:33:02.243899+0000","last_change":"2023-09-23T20:32:33.232142+0000","last_active":"2023-09-23T20:33:02.243899+0000","last_peered":"2023-09-23T20:33:02.243899+0000","last_clean":"2023-09-23T20:33:02.243899+0000","last_became_active":"2023-09-23T20:32:33.231683+0000","last_became_peered":"2023-09-23T20:32:33.231683+0000","last_unstale":"2023-09-23T20:33:02.243899+0000","last_undegraded":"2023-09-23T20:33:02.243899+0000","last_fullsized":"2023-09-23T20:33:02.243899+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:02:58.336348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"326'4863","reported_seq":4911,"reported_epoch":326,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:33:05.246389+0000","last_change":"2023-09-23T20:33:00.304857+0000","last_active":"2023-09-23T20:33:05.246389+0000","last_peered":"2023-09-23T20:33:05.246389+0000","last_clean":"2023-09-23T20:32:32.204699+0000","last_became_active":"2023-09-23T20:33:00.297863+0000","last_became_peered":"2023-09-23T20:33:00.297863+0000","last_unstale":"2023-09-23T20:33:05.246389+0000","last_undegraded":"2023-09-23T20:33:00.218246+0000","last_fullsized":"2023-09-23T20:33:00.168368+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":4863,"log_dups_size":0,"ondisk_log_size":4863,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:31.378123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":318701568,"num_objects":4863,"num_object_clones":0,"num_object_copies":14589,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1108,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4863,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4863,"num_write_kb":311232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1],"avail_no_missing":["0","1"],"object_location_counts":[{"shards":"0,1","objects":3755},{"shards":"0,1,4","objects":1106}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"90'161","reported_seq":548,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T20:33:02.251377+0000","last_change":"2023-09-23T20:32:36.302809+0000","last_active":"2023-09-23T20:33:02.251377+0000","last_peered":"2023-09-23T20:33:02.251377+0000","last_clean":"2023-09-23T20:33:02.251377+0000","last_became_active":"2023-09-23T20:32:36.302465+0000","last_became_peered":"2023-09-23T20:32:36.302465+0000","last_unstale":"2023-09-23T20:33:02.251377+0000","last_undegraded":"2023-09-23T20:33:02.251377+0000","last_fullsized":"2023-09-23T20:33:02.251377+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:46.217489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:46.217489+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:35:40.488427+0000","scrub_duration":0.001456429,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":298,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T20:33:02.253409+0000","last_change":"2023-09-23T20:29:04.184893+0000","last_active":"2023-09-23T20:33:02.253409+0000","last_peered":"2023-09-23T20:33:02.253409+0000","last_clean":"2023-09-23T20:33:02.253409+0000","last_became_active":"2023-09-23T20:29:04.152723+0000","last_became_peered":"2023-09-23T20:29:04.152723+0000","last_unstale":"2023-09-23T20:33:02.253409+0000","last_undegraded":"2023-09-23T20:33:02.253409+0000","last_fullsized":"2023-09-23T20:33:02.253409+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:51:27.017151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":99,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T20:33:02.249927+0000","last_change":"2023-09-23T20:29:46.086912+0000","last_active":"2023-09-23T20:33:02.249927+0000","last_peered":"2023-09-23T20:33:02.249927+0000","last_clean":"2023-09-23T20:33:02.249927+0000","last_became_active":"2023-09-23T20:29:46.085471+0000","last_became_peered":"2023-09-23T20:29:46.085471+0000","last_unstale":"2023-09-23T20:33:02.249927+0000","last_undegraded":"2023-09-23T20:33:02.249927+0000","last_fullsized":"2023-09-23T20:33:02.249927+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:04:55.751660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.8","version":"327'5051","reported_seq":5942,"reported_epoch":327,"state":"active+recovering+undersized+remapped","last_fresh":"2023-09-23T20:33:07.440754+0000","last_change":"2023-09-23T20:32:59.221315+0000","last_active":"2023-09-23T20:33:07.440754+0000","last_peered":"2023-09-23T20:33:07.440754+0000","last_clean":"2023-09-23T20:32:32.212521+0000","last_became_active":"2023-09-23T20:32:34.616677+0000","last_became_peered":"2023-09-23T20:32:34.616677+0000","last_unstale":"2023-09-23T20:33:07.440754+0000","last_undegraded":"2023-09-23T20:33:07.440754+0000","last_fullsized":"2023-09-23T20:32:34.212267+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":5051,"log_dups_size":0,"ondisk_log_size":5051,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:24:37.173047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":331022336,"num_objects":5051,"num_object_clones":0,"num_object_copies":15153,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3213,"num_objects_unfound":0,"num_objects_dirty":5051,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5051,"num_write_kb":323264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":425,"num_bytes_recovered":27852800,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7],"avail_no_missing":["2","3","7"],"object_location_counts":[{"shards":"2,3,7","objects":3213},{"shards":"2,7","objects":1838}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.e","version":"90'161","reported_seq":515,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T20:33:02.255679+0000","last_change":"2023-09-23T20:33:00.180075+0000","last_active":"2023-09-23T20:33:02.255679+0000","last_peered":"2023-09-23T20:33:02.255679+0000","last_clean":"2023-09-23T20:33:02.255679+0000","last_became_active":"2023-09-23T20:33:00.179783+0000","last_became_peered":"2023-09-23T20:33:00.179783+0000","last_unstale":"2023-09-23T20:33:02.255679+0000","last_undegraded":"2023-09-23T20:33:02.255679+0000","last_fullsized":"2023-09-23T20:33:02.255679+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:44.386120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:44.386120+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:16:54.770768+0000","scrub_duration":0.0015151330000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":332,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T20:33:02.233961+0000","last_change":"2023-09-23T20:29:04.138284+0000","last_active":"2023-09-23T20:33:02.233961+0000","last_peered":"2023-09-23T20:33:02.233961+0000","last_clean":"2023-09-23T20:33:02.233961+0000","last_became_active":"2023-09-23T20:29:04.137960+0000","last_became_peered":"2023-09-23T20:29:04.137960+0000","last_unstale":"2023-09-23T20:33:02.233961+0000","last_undegraded":"2023-09-23T20:33:02.233961+0000","last_fullsized":"2023-09-23T20:33:02.233961+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:18:59.548311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":308,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T20:33:02.261521+0000","last_change":"2023-09-23T20:32:33.236122+0000","last_active":"2023-09-23T20:33:02.261521+0000","last_peered":"2023-09-23T20:33:02.261521+0000","last_clean":"2023-09-23T20:33:02.261521+0000","last_became_active":"2023-09-23T20:32:33.235384+0000","last_became_peered":"2023-09-23T20:32:33.235384+0000","last_unstale":"2023-09-23T20:33:02.261521+0000","last_undegraded":"2023-09-23T20:33:02.261521+0000","last_fullsized":"2023-09-23T20:33:02.261521+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:56.843733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"326'4943","reported_seq":4975,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-09-23T20:33:05.291550+0000","last_change":"2023-09-23T20:31:49.158461+0000","last_active":"2023-09-23T20:33:05.291550+0000","last_peered":"2023-09-23T20:33:05.291550+0000","last_clean":"2023-09-23T20:33:05.291550+0000","last_became_active":"2023-09-23T20:31:49.158188+0000","last_became_peered":"2023-09-23T20:31:49.158188+0000","last_unstale":"2023-09-23T20:33:05.291550+0000","last_undegraded":"2023-09-23T20:33:05.291550+0000","last_fullsized":"2023-09-23T20:33:05.291550+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":4943,"log_dups_size":0,"ondisk_log_size":4943,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:31:57.678090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":323944448,"num_objects":4943,"num_object_clones":0,"num_object_copies":14829,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4943,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4943,"num_write_kb":316352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":708,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T20:33:02.255120+0000","last_change":"2023-09-23T20:28:05.872149+0000","last_active":"2023-09-23T20:33:02.255120+0000","last_peered":"2023-09-23T20:33:02.255120+0000","last_clean":"2023-09-23T20:33:02.255120+0000","last_became_active":"2023-09-23T20:27:11.961310+0000","last_became_peered":"2023-09-23T20:27:11.961310+0000","last_unstale":"2023-09-23T20:33:02.255120+0000","last_undegraded":"2023-09-23T20:33:02.255120+0000","last_fullsized":"2023-09-23T20:33:02.255120+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:28:05.872074+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:28:05.872074+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:52:32.495955+0000","scrub_duration":0.0029215980000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":60,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T20:33:02.261618+0000","last_change":"2023-09-23T20:32:50.538500+0000","last_active":"2023-09-23T20:33:02.261618+0000","last_peered":"2023-09-23T20:33:02.261618+0000","last_clean":"2023-09-23T20:33:02.261618+0000","last_became_active":"2023-09-23T20:32:50.538152+0000","last_became_peered":"2023-09-23T20:32:50.538152+0000","last_unstale":"2023-09-23T20:33:02.261618+0000","last_undegraded":"2023-09-23T20:33:02.261618+0000","last_fullsized":"2023-09-23T20:33:02.261618+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:00:41.103508+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":23,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T20:33:02.244246+0000","last_change":"2023-09-23T20:32:33.231614+0000","last_active":"2023-09-23T20:33:02.244246+0000","last_peered":"2023-09-23T20:33:02.244246+0000","last_clean":"2023-09-23T20:33:02.244246+0000","last_became_active":"2023-09-23T20:32:33.231207+0000","last_became_peered":"2023-09-23T20:32:33.231207+0000","last_unstale":"2023-09-23T20:33:02.244246+0000","last_undegraded":"2023-09-23T20:33:02.244246+0000","last_fullsized":"2023-09-23T20:33:02.244246+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:46:09.780829+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"327'5012","reported_seq":5045,"reported_epoch":327,"state":"active+clean","last_fresh":"2023-09-23T20:33:07.458411+0000","last_change":"2023-09-23T20:31:49.155754+0000","last_active":"2023-09-23T20:33:07.458411+0000","last_peered":"2023-09-23T20:33:07.458411+0000","last_clean":"2023-09-23T20:33:07.458411+0000","last_became_active":"2023-09-23T20:31:49.155363+0000","last_became_peered":"2023-09-23T20:31:49.155363+0000","last_unstale":"2023-09-23T20:33:07.458411+0000","last_undegraded":"2023-09-23T20:33:07.458411+0000","last_fullsized":"2023-09-23T20:33:07.458411+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":5012,"log_dups_size":0,"ondisk_log_size":5012,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:45:42.661641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":328466432,"num_objects":5012,"num_object_clones":0,"num_object_copies":15036,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5012,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5012,"num_write_kb":320768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":550,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T20:33:02.233807+0000","last_change":"2023-09-23T20:27:31.757060+0000","last_active":"2023-09-23T20:33:02.233807+0000","last_peered":"2023-09-23T20:33:02.233807+0000","last_clean":"2023-09-23T20:33:02.233807+0000","last_became_active":"2023-09-23T20:27:11.959516+0000","last_became_peered":"2023-09-23T20:27:11.959516+0000","last_unstale":"2023-09-23T20:33:02.233807+0000","last_undegraded":"2023-09-23T20:33:02.233807+0000","last_fullsized":"2023-09-23T20:33:02.233807+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:31.756974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:31.756974+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:35:11.939335+0000","scrub_duration":0.0031802670000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":121,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T20:33:02.249971+0000","last_change":"2023-09-23T20:32:33.229700+0000","last_active":"2023-09-23T20:33:02.249971+0000","last_peered":"2023-09-23T20:33:02.249971+0000","last_clean":"2023-09-23T20:33:02.249971+0000","last_became_active":"2023-09-23T20:32:33.229372+0000","last_became_peered":"2023-09-23T20:32:33.229372+0000","last_unstale":"2023-09-23T20:33:02.249971+0000","last_undegraded":"2023-09-23T20:33:02.249971+0000","last_fullsized":"2023-09-23T20:33:02.249971+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:18:10.958227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":235,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-09-23T20:33:03.286646+0000","last_change":"2023-09-23T20:33:00.188031+0000","last_active":"2023-09-23T20:33:03.286646+0000","last_peered":"2023-09-23T20:33:03.286646+0000","last_clean":"2023-09-23T20:33:03.286646+0000","last_became_active":"2023-09-23T20:33:00.187675+0000","last_became_peered":"2023-09-23T20:33:00.187675+0000","last_unstale":"2023-09-23T20:33:03.286646+0000","last_undegraded":"2023-09-23T20:33:03.286646+0000","last_fullsized":"2023-09-23T20:33:03.286646+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:42:28.967989+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.d","version":"326'4969","reported_seq":5000,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-09-23T20:33:05.593169+0000","last_change":"2023-09-23T20:31:49.158641+0000","last_active":"2023-09-23T20:33:05.593169+0000","last_peered":"2023-09-23T20:33:05.593169+0000","last_clean":"2023-09-23T20:33:05.593169+0000","last_became_active":"2023-09-23T20:31:49.157604+0000","last_became_peered":"2023-09-23T20:31:49.157604+0000","last_unstale":"2023-09-23T20:33:05.593169+0000","last_undegraded":"2023-09-23T20:33:05.593169+0000","last_fullsized":"2023-09-23T20:33:05.593169+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":4969,"log_dups_size":0,"ondisk_log_size":4969,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:54:04.726000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":325648384,"num_objects":4969,"num_object_clones":0,"num_object_copies":14907,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4969,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4969,"num_write_kb":318016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.b","version":"90'161","reported_seq":569,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-09-23T20:33:03.266521+0000","last_change":"2023-09-23T20:31:46.558396+0000","last_active":"2023-09-23T20:33:03.266521+0000","last_peered":"2023-09-23T20:33:03.266521+0000","last_clean":"2023-09-23T20:33:03.266521+0000","last_became_active":"2023-09-23T20:29:45.534070+0000","last_became_peered":"2023-09-23T20:29:45.534070+0000","last_unstale":"2023-09-23T20:33:03.266521+0000","last_undegraded":"2023-09-23T20:33:03.266521+0000","last_fullsized":"2023-09-23T20:33:03.266521+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:46.558292+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:46.558292+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:10:34.051169+0000","scrub_duration":0.0023048779999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":237,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-09-23T20:33:03.286493+0000","last_change":"2023-09-23T20:33:00.185583+0000","last_active":"2023-09-23T20:33:03.286493+0000","last_peered":"2023-09-23T20:33:03.286493+0000","last_clean":"2023-09-23T20:33:03.286493+0000","last_became_active":"2023-09-23T20:33:00.185273+0000","last_became_peered":"2023-09-23T20:33:00.185273+0000","last_unstale":"2023-09-23T20:33:03.286493+0000","last_undegraded":"2023-09-23T20:33:03.286493+0000","last_fullsized":"2023-09-23T20:33:03.286493+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:00:48.877233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":23,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T20:33:02.244691+0000","last_change":"2023-09-23T20:32:33.224963+0000","last_active":"2023-09-23T20:33:02.244691+0000","last_peered":"2023-09-23T20:33:02.244691+0000","last_clean":"2023-09-23T20:33:02.244691+0000","last_became_active":"2023-09-23T20:32:33.224634+0000","last_became_peered":"2023-09-23T20:32:33.224634+0000","last_unstale":"2023-09-23T20:33:02.244691+0000","last_undegraded":"2023-09-23T20:33:02.244691+0000","last_fullsized":"2023-09-23T20:33:02.244691+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:41:40.314140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"326'4938","reported_seq":4969,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-09-23T20:33:05.609774+0000","last_change":"2023-09-23T20:31:49.157247+0000","last_active":"2023-09-23T20:33:05.609774+0000","last_peered":"2023-09-23T20:33:05.609774+0000","last_clean":"2023-09-23T20:33:05.609774+0000","last_became_active":"2023-09-23T20:31:49.156961+0000","last_became_peered":"2023-09-23T20:31:49.156961+0000","last_unstale":"2023-09-23T20:33:05.609774+0000","last_undegraded":"2023-09-23T20:33:05.609774+0000","last_fullsized":"2023-09-23T20:33:05.609774+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":4938,"log_dups_size":0,"ondisk_log_size":4938,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:49:17.346795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":323616768,"num_objects":4938,"num_object_clones":0,"num_object_copies":14814,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4938,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4938,"num_write_kb":316032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":590,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T20:33:02.234651+0000","last_change":"2023-09-23T20:29:07.053561+0000","last_active":"2023-09-23T20:33:02.234651+0000","last_peered":"2023-09-23T20:33:02.234651+0000","last_clean":"2023-09-23T20:33:02.234651+0000","last_became_active":"2023-09-23T20:29:07.053177+0000","last_became_peered":"2023-09-23T20:29:07.053177+0000","last_unstale":"2023-09-23T20:33:02.234651+0000","last_undegraded":"2023-09-23T20:33:02.234651+0000","last_fullsized":"2023-09-23T20:33:02.234651+0000","mapping_epoch":253,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":254,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:30.727267+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:30.727267+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:04:32.661761+0000","scrub_duration":0.0052560289999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":139,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-09-23T20:33:03.263531+0000","last_change":"2023-09-23T20:32:50.530690+0000","last_active":"2023-09-23T20:33:03.263531+0000","last_peered":"2023-09-23T20:33:03.263531+0000","last_clean":"2023-09-23T20:33:03.263531+0000","last_became_active":"2023-09-23T20:32:50.530376+0000","last_became_peered":"2023-09-23T20:32:50.530376+0000","last_unstale":"2023-09-23T20:33:03.263531+0000","last_undegraded":"2023-09-23T20:33:03.263531+0000","last_fullsized":"2023-09-23T20:33:03.263531+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:24:53.184959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":147,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-09-23T20:33:03.266683+0000","last_change":"2023-09-23T20:32:33.230582+0000","last_active":"2023-09-23T20:33:03.266683+0000","last_peered":"2023-09-23T20:33:03.266683+0000","last_clean":"2023-09-23T20:33:03.266683+0000","last_became_active":"2023-09-23T20:32:33.230257+0000","last_became_peered":"2023-09-23T20:32:33.230257+0000","last_unstale":"2023-09-23T20:33:03.266683+0000","last_undegraded":"2023-09-23T20:33:03.266683+0000","last_fullsized":"2023-09-23T20:33:03.266683+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:11:45.333096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"326'4874","reported_seq":5961,"reported_epoch":326,"state":"active+recovery_wait","last_fresh":"2023-09-23T20:33:05.566084+0000","last_change":"2023-09-23T20:33:01.313842+0000","last_active":"2023-09-23T20:33:05.566084+0000","last_peered":"2023-09-23T20:33:05.566084+0000","last_clean":"2023-09-23T20:32:32.204597+0000","last_became_active":"2023-09-23T20:33:01.258982+0000","last_became_peered":"2023-09-23T20:33:01.258982+0000","last_unstale":"2023-09-23T20:33:05.566084+0000","last_undegraded":"2023-09-23T20:33:05.566084+0000","last_fullsized":"2023-09-23T20:33:05.566084+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":4874,"log_dups_size":0,"ondisk_log_size":4874,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:38:23.266319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":319422464,"num_objects":4874,"num_object_clones":0,"num_object_copies":14622,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4874,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4874,"num_write_kb":311936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":521,"num_bytes_recovered":34144256,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":["6","1","3"],"object_location_counts":[{"shards":"1,3,6","objects":4874}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":555,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T20:33:02.251390+0000","last_change":"2023-09-23T20:31:45.196356+0000","last_active":"2023-09-23T20:33:02.251390+0000","last_peered":"2023-09-23T20:33:02.251390+0000","last_clean":"2023-09-23T20:33:02.251390+0000","last_became_active":"2023-09-23T20:29:07.115647+0000","last_became_peered":"2023-09-23T20:29:07.115647+0000","last_unstale":"2023-09-23T20:33:02.251390+0000","last_undegraded":"2023-09-23T20:33:02.251390+0000","last_fullsized":"2023-09-23T20:33:02.251390+0000","mapping_epoch":253,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":254,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:45.196285+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:45.196285+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:33:06.666183+0000","scrub_duration":0.0014380860000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"86'119","reported_seq":631,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-09-23T20:33:03.267012+0000","last_change":"2023-09-23T20:29:45.521257+0000","last_active":"2023-09-23T20:33:03.267012+0000","last_peered":"2023-09-23T20:33:03.267012+0000","last_clean":"2023-09-23T20:33:03.267012+0000","last_became_active":"2023-09-23T20:29:45.520546+0000","last_became_peered":"2023-09-23T20:29:45.520546+0000","last_unstale":"2023-09-23T20:33:03.267012+0000","last_undegraded":"2023-09-23T20:33:03.267012+0000","last_fullsized":"2023-09-23T20:33:03.267012+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":119,"log_dups_size":0,"ondisk_log_size":119,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:58:56.085777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032705499999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":523,"num_read_kb":3503,"num_write":136,"num_write_kb":3128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":342,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T20:33:02.234071+0000","last_change":"2023-09-23T20:25:39.637160+0000","last_active":"2023-09-23T20:33:02.234071+0000","last_peered":"2023-09-23T20:33:02.234071+0000","last_clean":"2023-09-23T20:33:02.234071+0000","last_became_active":"2023-09-23T20:25:39.636860+0000","last_became_peered":"2023-09-23T20:25:39.636860+0000","last_unstale":"2023-09-23T20:33:02.234071+0000","last_undegraded":"2023-09-23T20:33:02.234071+0000","last_fullsized":"2023-09-23T20:33:02.234071+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:21:47.860063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":251,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-09-23T20:33:03.265477+0000","last_change":"2023-09-23T20:33:00.190831+0000","last_active":"2023-09-23T20:33:03.265477+0000","last_peered":"2023-09-23T20:33:03.265477+0000","last_clean":"2023-09-23T20:33:03.265477+0000","last_became_active":"2023-09-23T20:33:00.190494+0000","last_became_peered":"2023-09-23T20:33:00.190494+0000","last_unstale":"2023-09-23T20:33:03.265477+0000","last_undegraded":"2023-09-23T20:33:03.265477+0000","last_fullsized":"2023-09-23T20:33:03.265477+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:19:01.109316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"326'4956","reported_seq":4987,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-09-23T20:33:05.576859+0000","last_change":"2023-09-23T20:31:49.157301+0000","last_active":"2023-09-23T20:33:05.576859+0000","last_peered":"2023-09-23T20:33:05.576859+0000","last_clean":"2023-09-23T20:33:05.576859+0000","last_became_active":"2023-09-23T20:31:49.156992+0000","last_became_peered":"2023-09-23T20:31:49.156992+0000","last_unstale":"2023-09-23T20:33:05.576859+0000","last_undegraded":"2023-09-23T20:33:05.576859+0000","last_fullsized":"2023-09-23T20:33:05.576859+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":4956,"log_dups_size":0,"ondisk_log_size":4956,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:21:35.645838+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":324796416,"num_objects":4956,"num_object_clones":0,"num_object_copies":14868,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4956,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4956,"num_write_kb":317184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":570,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T20:33:02.233840+0000","last_change":"2023-09-23T20:31:45.785521+0000","last_active":"2023-09-23T20:33:02.233840+0000","last_peered":"2023-09-23T20:33:02.233840+0000","last_clean":"2023-09-23T20:33:02.233840+0000","last_became_active":"2023-09-23T20:27:55.213394+0000","last_became_peered":"2023-09-23T20:27:55.213394+0000","last_unstale":"2023-09-23T20:33:02.233840+0000","last_undegraded":"2023-09-23T20:33:02.233840+0000","last_fullsized":"2023-09-23T20:33:02.233840+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:45.785450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:45.785450+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:21:26.092578+0000","scrub_duration":0.001558306,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"86'106","reported_seq":1044,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T20:33:02.234470+0000","last_change":"2023-09-23T20:31:31.531882+0000","last_active":"2023-09-23T20:33:02.234470+0000","last_peered":"2023-09-23T20:33:02.234470+0000","last_clean":"2023-09-23T20:33:02.234470+0000","last_became_active":"2023-09-23T20:29:46.078501+0000","last_became_peered":"2023-09-23T20:29:46.078501+0000","last_unstale":"2023-09-23T20:33:02.234470+0000","last_undegraded":"2023-09-23T20:33:02.234470+0000","last_fullsized":"2023-09-23T20:33:02.234470+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":106,"log_dups_size":0,"ondisk_log_size":106,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:37.015974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00099302599999999993,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1105,"num_read_kb":1689,"num_write":100,"num_write_kb":404,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":23,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T20:33:02.244670+0000","last_change":"2023-09-23T20:32:33.230242+0000","last_active":"2023-09-23T20:33:02.244670+0000","last_peered":"2023-09-23T20:33:02.244670+0000","last_clean":"2023-09-23T20:33:02.244670+0000","last_became_active":"2023-09-23T20:32:33.229732+0000","last_became_peered":"2023-09-23T20:32:33.229732+0000","last_unstale":"2023-09-23T20:33:02.244670+0000","last_undegraded":"2023-09-23T20:33:02.244670+0000","last_fullsized":"2023-09-23T20:33:02.244670+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:27:55.538429+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":284,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-09-23T20:33:03.266923+0000","last_change":"2023-09-23T20:29:45.533406+0000","last_active":"2023-09-23T20:33:03.266923+0000","last_peered":"2023-09-23T20:33:03.266923+0000","last_clean":"2023-09-23T20:33:03.266923+0000","last_became_active":"2023-09-23T20:29:45.526001+0000","last_became_peered":"2023-09-23T20:29:45.526001+0000","last_unstale":"2023-09-23T20:33:03.266923+0000","last_undegraded":"2023-09-23T20:33:03.266923+0000","last_fullsized":"2023-09-23T20:33:03.266923+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:43.787251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.4","version":"326'4873","reported_seq":4932,"reported_epoch":326,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:33:05.270485+0000","last_change":"2023-09-23T20:33:00.311161+0000","last_active":"2023-09-23T20:33:05.270485+0000","last_peered":"2023-09-23T20:33:05.270485+0000","last_clean":"2023-09-23T20:32:32.192526+0000","last_became_active":"2023-09-23T20:33:00.256639+0000","last_became_peered":"2023-09-23T20:33:00.256639+0000","last_unstale":"2023-09-23T20:33:05.270485+0000","last_undegraded":"2023-09-23T20:33:00.185190+0000","last_fullsized":"2023-09-23T20:33:00.168423+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":4873,"log_dups_size":0,"ondisk_log_size":4873,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:59:02.098340+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":319356928,"num_objects":4873,"num_object_clones":0,"num_object_copies":14619,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1052,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4873,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4873,"num_write_kb":311872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":262144,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,6],"acting":[0,1],"avail_no_missing":["0","1"],"object_location_counts":[{"shards":"0,1","objects":3821},{"shards":"0,1,4","objects":1051}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":778,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T20:33:02.252900+0000","last_change":"2023-09-23T20:33:00.189173+0000","last_active":"2023-09-23T20:33:02.252900+0000","last_peered":"2023-09-23T20:33:02.252900+0000","last_clean":"2023-09-23T20:33:02.252900+0000","last_became_active":"2023-09-23T20:33:00.188795+0000","last_became_peered":"2023-09-23T20:33:00.188795+0000","last_unstale":"2023-09-23T20:33:02.252900+0000","last_undegraded":"2023-09-23T20:33:02.252900+0000","last_fullsized":"2023-09-23T20:33:02.252900+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:43.390908+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:43.390908+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:44:16.658526+0000","scrub_duration":0.0032721,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,2],"acting":[7,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"86'130","reported_seq":639,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-09-23T20:33:03.286425+0000","last_change":"2023-09-23T20:25:39.637458+0000","last_active":"2023-09-23T20:33:03.286425+0000","last_peered":"2023-09-23T20:33:03.286425+0000","last_clean":"2023-09-23T20:33:03.286425+0000","last_became_active":"2023-09-23T20:25:39.636773+0000","last_became_peered":"2023-09-23T20:25:39.636773+0000","last_unstale":"2023-09-23T20:33:03.286425+0000","last_undegraded":"2023-09-23T20:33:03.286425+0000","last_fullsized":"2023-09-23T20:33:03.286425+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:00:32.810564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011300049999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":512,"num_read_kb":1820,"num_write":133,"num_write_kb":1347,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":274,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-09-23T20:33:03.263399+0000","last_change":"2023-09-23T20:29:46.073394+0000","last_active":"2023-09-23T20:33:03.263399+0000","last_peered":"2023-09-23T20:33:03.263399+0000","last_clean":"2023-09-23T20:33:03.263399+0000","last_became_active":"2023-09-23T20:29:46.072067+0000","last_became_peered":"2023-09-23T20:29:46.072067+0000","last_unstale":"2023-09-23T20:33:03.263399+0000","last_undegraded":"2023-09-23T20:33:03.263399+0000","last_fullsized":"2023-09-23T20:33:03.263399+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:01:56.944614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":266,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-09-23T20:33:03.266611+0000","last_change":"2023-09-23T20:29:45.533185+0000","last_active":"2023-09-23T20:33:03.266611+0000","last_peered":"2023-09-23T20:33:03.266611+0000","last_clean":"2023-09-23T20:33:03.266611+0000","last_became_active":"2023-09-23T20:29:45.526765+0000","last_became_peered":"2023-09-23T20:29:45.526765+0000","last_unstale":"2023-09-23T20:33:03.266611+0000","last_undegraded":"2023-09-23T20:33:03.266611+0000","last_fullsized":"2023-09-23T20:33:03.266611+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:18:33.063215+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"326'4944","reported_seq":4976,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-09-23T20:33:05.282931+0000","last_change":"2023-09-23T20:31:49.158194+0000","last_active":"2023-09-23T20:33:05.282931+0000","last_peered":"2023-09-23T20:33:05.282931+0000","last_clean":"2023-09-23T20:33:05.282931+0000","last_became_active":"2023-09-23T20:31:49.157875+0000","last_became_peered":"2023-09-23T20:31:49.157875+0000","last_unstale":"2023-09-23T20:33:05.282931+0000","last_undegraded":"2023-09-23T20:33:05.282931+0000","last_fullsized":"2023-09-23T20:33:05.282931+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":4944,"log_dups_size":0,"ondisk_log_size":4944,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:04:42.411181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":324009984,"num_objects":4944,"num_object_clones":0,"num_object_copies":14832,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4944,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4944,"num_write_kb":316416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"86'194","reported_seq":974,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T20:33:02.234651+0000","last_change":"2023-09-23T20:25:40.618665+0000","last_active":"2023-09-23T20:33:02.234651+0000","last_peered":"2023-09-23T20:33:02.234651+0000","last_clean":"2023-09-23T20:33:02.234651+0000","last_became_active":"2023-09-23T20:25:40.618087+0000","last_became_peered":"2023-09-23T20:25:40.618087+0000","last_unstale":"2023-09-23T20:33:02.234651+0000","last_undegraded":"2023-09-23T20:33:02.234651+0000","last_fullsized":"2023-09-23T20:33:02.234651+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":194,"log_dups_size":0,"ondisk_log_size":194,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:07:16.114968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00037995999999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1025,"num_read_kb":1201,"num_write":173,"num_write_kb":1276,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":759,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T20:33:02.255054+0000","last_change":"2023-09-23T20:31:45.343835+0000","last_active":"2023-09-23T20:33:02.255054+0000","last_peered":"2023-09-23T20:33:02.255054+0000","last_clean":"2023-09-23T20:33:02.255054+0000","last_became_active":"2023-09-23T20:29:07.042407+0000","last_became_peered":"2023-09-23T20:29:07.042407+0000","last_unstale":"2023-09-23T20:33:02.255054+0000","last_undegraded":"2023-09-23T20:33:02.255054+0000","last_fullsized":"2023-09-23T20:33:02.255054+0000","mapping_epoch":253,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":254,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:45.343742+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:45.343742+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:33:12.745366+0000","scrub_duration":0.001475802,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":125,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T20:33:02.234567+0000","last_change":"2023-09-23T20:32:50.534315+0000","last_active":"2023-09-23T20:33:02.234567+0000","last_peered":"2023-09-23T20:33:02.234567+0000","last_clean":"2023-09-23T20:33:02.234567+0000","last_became_active":"2023-09-23T20:32:50.533896+0000","last_became_peered":"2023-09-23T20:32:50.533896+0000","last_unstale":"2023-09-23T20:33:02.234567+0000","last_undegraded":"2023-09-23T20:33:02.234567+0000","last_fullsized":"2023-09-23T20:33:02.234567+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:06:53.464923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":105,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T20:33:02.251418+0000","last_change":"2023-09-23T20:32:33.225012+0000","last_active":"2023-09-23T20:33:02.251418+0000","last_peered":"2023-09-23T20:33:02.251418+0000","last_clean":"2023-09-23T20:33:02.251418+0000","last_became_active":"2023-09-23T20:32:33.224699+0000","last_became_peered":"2023-09-23T20:32:33.224699+0000","last_unstale":"2023-09-23T20:33:02.251418+0000","last_undegraded":"2023-09-23T20:33:02.251418+0000","last_fullsized":"2023-09-23T20:33:02.251418+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:10:06.267878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"326'4977","reported_seq":5008,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-09-23T20:33:05.729199+0000","last_change":"2023-09-23T20:31:49.154418+0000","last_active":"2023-09-23T20:33:05.729199+0000","last_peered":"2023-09-23T20:33:05.729199+0000","last_clean":"2023-09-23T20:33:05.729199+0000","last_became_active":"2023-09-23T20:31:49.154182+0000","last_became_peered":"2023-09-23T20:31:49.154182+0000","last_unstale":"2023-09-23T20:33:05.729199+0000","last_undegraded":"2023-09-23T20:33:05.729199+0000","last_fullsized":"2023-09-23T20:33:05.729199+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":4977,"log_dups_size":0,"ondisk_log_size":4977,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:44:10.087486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":326172672,"num_objects":4977,"num_object_clones":0,"num_object_copies":14931,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4977,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4977,"num_write_kb":318528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":602,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-09-23T20:33:03.280343+0000","last_change":"2023-09-23T20:33:03.279925+0000","last_active":"2023-09-23T20:33:03.280343+0000","last_peered":"2023-09-23T20:33:03.280343+0000","last_clean":"2023-09-23T20:33:03.280343+0000","last_became_active":"2023-09-23T20:33:03.279531+0000","last_became_peered":"2023-09-23T20:33:03.279531+0000","last_unstale":"2023-09-23T20:33:03.280343+0000","last_undegraded":"2023-09-23T20:33:03.280343+0000","last_fullsized":"2023-09-23T20:33:03.280343+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:43.582146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:43.582146+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:06:12.587774+0000","scrub_duration":0.0016219730000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,5],"acting":[1,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"86'80","reported_seq":589,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T20:33:02.255397+0000","last_change":"2023-09-23T20:29:45.536796+0000","last_active":"2023-09-23T20:33:02.255397+0000","last_peered":"2023-09-23T20:33:02.255397+0000","last_clean":"2023-09-23T20:33:02.255397+0000","last_became_active":"2023-09-23T20:29:45.536439+0000","last_became_peered":"2023-09-23T20:29:45.536439+0000","last_unstale":"2023-09-23T20:33:02.255397+0000","last_undegraded":"2023-09-23T20:33:02.255397+0000","last_fullsized":"2023-09-23T20:33:02.255397+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":80,"log_dups_size":0,"ondisk_log_size":80,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:30:57.793132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035115,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":395,"num_read_kb":1370,"num_write":66,"num_write_kb":1061,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":318,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T20:33:02.265367+0000","last_change":"2023-09-23T20:33:00.220984+0000","last_active":"2023-09-23T20:33:02.265367+0000","last_peered":"2023-09-23T20:33:02.265367+0000","last_clean":"2023-09-23T20:33:02.265367+0000","last_became_active":"2023-09-23T20:33:00.220642+0000","last_became_peered":"2023-09-23T20:33:00.220642+0000","last_unstale":"2023-09-23T20:33:02.265367+0000","last_undegraded":"2023-09-23T20:33:02.265367+0000","last_fullsized":"2023-09-23T20:33:02.265367+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:52:14.218890+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":321,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T20:33:02.261950+0000","last_change":"2023-09-23T20:29:45.690815+0000","last_active":"2023-09-23T20:33:02.261950+0000","last_peered":"2023-09-23T20:33:02.261950+0000","last_clean":"2023-09-23T20:33:02.261950+0000","last_became_active":"2023-09-23T20:29:45.690478+0000","last_became_peered":"2023-09-23T20:29:45.690478+0000","last_unstale":"2023-09-23T20:33:02.261950+0000","last_undegraded":"2023-09-23T20:33:02.261950+0000","last_fullsized":"2023-09-23T20:33:02.261950+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:38.288193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.0","version":"326'4980","reported_seq":5011,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-09-23T20:33:05.353871+0000","last_change":"2023-09-23T20:31:49.157205+0000","last_active":"2023-09-23T20:33:05.353871+0000","last_peered":"2023-09-23T20:33:05.353871+0000","last_clean":"2023-09-23T20:33:05.353871+0000","last_became_active":"2023-09-23T20:31:49.156922+0000","last_became_peered":"2023-09-23T20:31:49.156922+0000","last_unstale":"2023-09-23T20:33:05.353871+0000","last_undegraded":"2023-09-23T20:33:05.353871+0000","last_fullsized":"2023-09-23T20:33:05.353871+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":4980,"log_dups_size":0,"ondisk_log_size":4980,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:51:36.120059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":326369280,"num_objects":4980,"num_object_clones":0,"num_object_copies":14940,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4980,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4980,"num_write_kb":318720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":548,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T20:33:02.251284+0000","last_change":"2023-09-23T20:31:44.211308+0000","last_active":"2023-09-23T20:33:02.251284+0000","last_peered":"2023-09-23T20:33:02.251284+0000","last_clean":"2023-09-23T20:33:02.251284+0000","last_became_active":"2023-09-23T20:29:07.113428+0000","last_became_peered":"2023-09-23T20:29:07.113428+0000","last_unstale":"2023-09-23T20:33:02.251284+0000","last_undegraded":"2023-09-23T20:33:02.251284+0000","last_fullsized":"2023-09-23T20:33:02.251284+0000","mapping_epoch":253,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":254,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:44.211184+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:44.211184+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:53:50.053167+0000","scrub_duration":0.0017707720000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"86'102","reported_seq":920,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T20:33:02.253479+0000","last_change":"2023-09-23T20:32:36.314672+0000","last_active":"2023-09-23T20:33:02.253479+0000","last_peered":"2023-09-23T20:33:02.253479+0000","last_clean":"2023-09-23T20:33:02.253479+0000","last_became_active":"2023-09-23T20:32:36.314231+0000","last_became_peered":"2023-09-23T20:32:36.314231+0000","last_unstale":"2023-09-23T20:33:02.253479+0000","last_undegraded":"2023-09-23T20:33:02.253479+0000","last_fullsized":"2023-09-23T20:33:02.253479+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":102,"log_dups_size":0,"ondisk_log_size":102,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:11:40.514709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030656500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":963,"num_read_kb":1234,"num_write":60,"num_write_kb":225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":250,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-09-23T20:33:03.267096+0000","last_change":"2023-09-23T20:29:45.538283+0000","last_active":"2023-09-23T20:33:03.267096+0000","last_peered":"2023-09-23T20:33:03.267096+0000","last_clean":"2023-09-23T20:33:03.267096+0000","last_became_active":"2023-09-23T20:29:45.537922+0000","last_became_peered":"2023-09-23T20:29:45.537922+0000","last_unstale":"2023-09-23T20:33:03.267096+0000","last_undegraded":"2023-09-23T20:33:03.267096+0000","last_fullsized":"2023-09-23T20:33:03.267096+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:30:52.397836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":259,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T20:33:02.261799+0000","last_change":"2023-09-23T20:20:14.888874+0000","last_active":"2023-09-23T20:33:02.261799+0000","last_peered":"2023-09-23T20:33:02.261799+0000","last_clean":"2023-09-23T20:33:02.261799+0000","last_became_active":"2023-09-23T20:20:14.887724+0000","last_became_peered":"2023-09-23T20:20:14.887724+0000","last_unstale":"2023-09-23T20:33:02.261799+0000","last_undegraded":"2023-09-23T20:33:02.261799+0000","last_fullsized":"2023-09-23T20:33:02.261799+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:09:24.510326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"327'4868","reported_seq":6085,"reported_epoch":327,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:33:07.405995+0000","last_change":"2023-09-23T20:32:51.571341+0000","last_active":"2023-09-23T20:33:07.405995+0000","last_peered":"2023-09-23T20:33:07.405995+0000","last_clean":"2023-09-23T20:32:32.204529+0000","last_became_active":"2023-09-23T20:32:51.533262+0000","last_became_peered":"2023-09-23T20:32:51.533262+0000","last_unstale":"2023-09-23T20:33:07.405995+0000","last_undegraded":"2023-09-23T20:32:51.523866+0000","last_fullsized":"2023-09-23T20:32:51.518152+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":4868,"log_dups_size":0,"ondisk_log_size":4868,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:50:09.051328+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":319029248,"num_objects":4868,"num_object_clones":0,"num_object_copies":14604,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2957,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4868,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4868,"num_write_kb":311552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":585,"num_bytes_recovered":38338560,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3],"avail_no_missing":["2","3","6"],"object_location_counts":[{"shards":"2,3","objects":1911},{"shards":"2,3,6","objects":2957}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":518,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T20:33:02.244446+0000","last_change":"2023-09-23T20:32:36.307867+0000","last_active":"2023-09-23T20:33:02.244446+0000","last_peered":"2023-09-23T20:33:02.244446+0000","last_clean":"2023-09-23T20:33:02.244446+0000","last_became_active":"2023-09-23T20:32:36.307498+0000","last_became_peered":"2023-09-23T20:32:36.307498+0000","last_unstale":"2023-09-23T20:33:02.244446+0000","last_undegraded":"2023-09-23T20:33:02.244446+0000","last_fullsized":"2023-09-23T20:33:02.244446+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:44.509012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:44.509012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:14:40.400935+0000","scrub_duration":0.0032186319999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":63,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T20:33:02.255897+0000","last_change":"2023-09-23T20:32:50.549695+0000","last_active":"2023-09-23T20:33:02.255897+0000","last_peered":"2023-09-23T20:33:02.255897+0000","last_clean":"2023-09-23T20:33:02.255897+0000","last_became_active":"2023-09-23T20:32:50.549431+0000","last_became_peered":"2023-09-23T20:32:50.549431+0000","last_unstale":"2023-09-23T20:33:02.255897+0000","last_undegraded":"2023-09-23T20:33:02.255897+0000","last_fullsized":"2023-09-23T20:33:02.255897+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:45:53.666569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"86'128","reported_seq":716,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-09-23T20:33:03.265945+0000","last_change":"2023-09-23T20:32:36.310656+0000","last_active":"2023-09-23T20:33:03.265945+0000","last_peered":"2023-09-23T20:33:03.265945+0000","last_clean":"2023-09-23T20:33:03.265945+0000","last_became_active":"2023-09-23T20:32:36.310155+0000","last_became_peered":"2023-09-23T20:32:36.310155+0000","last_unstale":"2023-09-23T20:33:03.265945+0000","last_undegraded":"2023-09-23T20:33:03.265945+0000","last_fullsized":"2023-09-23T20:33:03.265945+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:58:14.217466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000293202,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":579,"num_read_kb":1473,"num_write":96,"num_write_kb":1062,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":96,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T20:33:02.255254+0000","last_change":"2023-09-23T20:32:50.539051+0000","last_active":"2023-09-23T20:33:02.255254+0000","last_peered":"2023-09-23T20:33:02.255254+0000","last_clean":"2023-09-23T20:33:02.255254+0000","last_became_active":"2023-09-23T20:32:50.538755+0000","last_became_peered":"2023-09-23T20:32:50.538755+0000","last_unstale":"2023-09-23T20:33:02.255254+0000","last_undegraded":"2023-09-23T20:33:02.255254+0000","last_fullsized":"2023-09-23T20:33:02.255254+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:16.656613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.2","version":"327'5039","reported_seq":5079,"reported_epoch":327,"state":"active+clean","last_fresh":"2023-09-23T20:33:08.759045+0000","last_change":"2023-09-23T20:32:50.557165+0000","last_active":"2023-09-23T20:33:08.759045+0000","last_peered":"2023-09-23T20:33:08.759045+0000","last_clean":"2023-09-23T20:33:08.759045+0000","last_became_active":"2023-09-23T20:32:50.553409+0000","last_became_peered":"2023-09-23T20:32:50.553409+0000","last_unstale":"2023-09-23T20:33:08.759045+0000","last_undegraded":"2023-09-23T20:33:08.759045+0000","last_fullsized":"2023-09-23T20:33:08.759045+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":5039,"log_dups_size":0,"ondisk_log_size":5039,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:55:13.578195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":330235904,"num_objects":5039,"num_object_clones":0,"num_object_copies":15117,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5039,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5039,"num_write_kb":322496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"90'161","reported_seq":438,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T20:33:02.901920+0000","last_change":"2023-09-23T20:33:02.901898+0000","last_active":"2023-09-23T20:33:02.901920+0000","last_peered":"2023-09-23T20:33:02.901920+0000","last_clean":"2023-09-23T20:33:02.901920+0000","last_became_active":"2023-09-23T20:32:50.533470+0000","last_became_peered":"2023-09-23T20:32:50.533470+0000","last_unstale":"2023-09-23T20:33:02.901920+0000","last_undegraded":"2023-09-23T20:33:02.901920+0000","last_fullsized":"2023-09-23T20:33:02.901920+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:28:03.553106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:28:03.553106+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.7","version":"86'117","reported_seq":595,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T20:33:02.251466+0000","last_change":"2023-09-23T20:31:32.814392+0000","last_active":"2023-09-23T20:33:02.251466+0000","last_peered":"2023-09-23T20:33:02.251466+0000","last_clean":"2023-09-23T20:33:02.251466+0000","last_became_active":"2023-09-23T20:31:32.813857+0000","last_became_peered":"2023-09-23T20:31:32.813857+0000","last_unstale":"2023-09-23T20:33:02.251466+0000","last_undegraded":"2023-09-23T20:33:02.251466+0000","last_fullsized":"2023-09-23T20:33:02.251466+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:56:57.863005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046891400000000002,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":893,"num_read_kb":922,"num_write":82,"num_write_kb":4193,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":321,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T20:33:02.261213+0000","last_change":"2023-09-23T20:32:33.277827+0000","last_active":"2023-09-23T20:33:02.261213+0000","last_peered":"2023-09-23T20:33:02.261213+0000","last_clean":"2023-09-23T20:33:02.261213+0000","last_became_active":"2023-09-23T20:32:33.236087+0000","last_became_peered":"2023-09-23T20:32:33.236087+0000","last_unstale":"2023-09-23T20:33:02.261213+0000","last_undegraded":"2023-09-23T20:33:02.261213+0000","last_fullsized":"2023-09-23T20:33:02.261213+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:20:58.176854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":23,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T20:33:02.243889+0000","last_change":"2023-09-23T20:32:33.238290+0000","last_active":"2023-09-23T20:33:02.243889+0000","last_peered":"2023-09-23T20:33:02.243889+0000","last_clean":"2023-09-23T20:33:02.243889+0000","last_became_active":"2023-09-23T20:32:33.237984+0000","last_became_peered":"2023-09-23T20:32:33.237984+0000","last_unstale":"2023-09-23T20:33:02.243889+0000","last_undegraded":"2023-09-23T20:33:02.243889+0000","last_fullsized":"2023-09-23T20:33:02.243889+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:49:02.089307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":290,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T20:33:02.261711+0000","last_change":"2023-09-23T20:32:33.237234+0000","last_active":"2023-09-23T20:33:02.261711+0000","last_peered":"2023-09-23T20:33:02.261711+0000","last_clean":"2023-09-23T20:33:02.261711+0000","last_became_active":"2023-09-23T20:32:33.236881+0000","last_became_peered":"2023-09-23T20:32:33.236881+0000","last_unstale":"2023-09-23T20:33:02.261711+0000","last_undegraded":"2023-09-23T20:33:02.261711+0000","last_fullsized":"2023-09-23T20:33:02.261711+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:56:17.384831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":178,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-09-23T20:33:03.286895+0000","last_change":"2023-09-23T20:29:45.538314+0000","last_active":"2023-09-23T20:33:03.286895+0000","last_peered":"2023-09-23T20:33:03.286895+0000","last_clean":"2023-09-23T20:33:03.286895+0000","last_became_active":"2023-09-23T20:29:45.536847+0000","last_became_peered":"2023-09-23T20:29:45.536847+0000","last_unstale":"2023-09-23T20:33:03.286895+0000","last_undegraded":"2023-09-23T20:33:03.286895+0000","last_fullsized":"2023-09-23T20:33:03.286895+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:51.948721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":335,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T20:33:02.233970+0000","last_change":"2023-09-23T20:32:33.235987+0000","last_active":"2023-09-23T20:33:02.233970+0000","last_peered":"2023-09-23T20:33:02.233970+0000","last_clean":"2023-09-23T20:33:02.233970+0000","last_became_active":"2023-09-23T20:32:33.226544+0000","last_became_peered":"2023-09-23T20:32:33.226544+0000","last_unstale":"2023-09-23T20:33:02.233970+0000","last_undegraded":"2023-09-23T20:33:02.233970+0000","last_fullsized":"2023-09-23T20:33:02.233970+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:53:22.794856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":282,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-09-23T20:33:03.786657+0000","last_change":"2023-09-23T20:33:00.177976+0000","last_active":"2023-09-23T20:33:03.786657+0000","last_peered":"2023-09-23T20:33:03.786657+0000","last_clean":"2023-09-23T20:33:03.786657+0000","last_became_active":"2023-09-23T20:33:00.177665+0000","last_became_peered":"2023-09-23T20:33:00.177665+0000","last_unstale":"2023-09-23T20:33:03.786657+0000","last_undegraded":"2023-09-23T20:33:03.786657+0000","last_fullsized":"2023-09-23T20:33:03.786657+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:20:25.273907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":343,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T20:33:02.234024+0000","last_change":"2023-09-23T20:32:33.268260+0000","last_active":"2023-09-23T20:33:02.234024+0000","last_peered":"2023-09-23T20:33:02.234024+0000","last_clean":"2023-09-23T20:33:02.234024+0000","last_became_active":"2023-09-23T20:32:33.233695+0000","last_became_peered":"2023-09-23T20:32:33.233695+0000","last_unstale":"2023-09-23T20:33:02.234024+0000","last_undegraded":"2023-09-23T20:33:02.234024+0000","last_fullsized":"2023-09-23T20:33:02.234024+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:19:29.299623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":180,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T20:33:02.261293+0000","last_change":"2023-09-23T20:33:00.179961+0000","last_active":"2023-09-23T20:33:02.261293+0000","last_peered":"2023-09-23T20:33:02.261293+0000","last_clean":"2023-09-23T20:33:02.261293+0000","last_became_active":"2023-09-23T20:33:00.178429+0000","last_became_peered":"2023-09-23T20:33:00.178429+0000","last_unstale":"2023-09-23T20:33:02.261293+0000","last_undegraded":"2023-09-23T20:33:02.261293+0000","last_fullsized":"2023-09-23T20:33:02.261293+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:28:20.964989+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":141,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T20:33:02.251199+0000","last_change":"2023-09-23T20:32:50.537447+0000","last_active":"2023-09-23T20:33:02.251199+0000","last_peered":"2023-09-23T20:33:02.251199+0000","last_clean":"2023-09-23T20:33:02.251199+0000","last_became_active":"2023-09-23T20:32:50.537136+0000","last_became_peered":"2023-09-23T20:32:50.537136+0000","last_unstale":"2023-09-23T20:33:02.251199+0000","last_undegraded":"2023-09-23T20:33:02.251199+0000","last_fullsized":"2023-09-23T20:33:02.251199+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:46:51.599492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":41,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-09-23T20:33:03.286511+0000","last_change":"2023-09-23T20:32:50.537450+0000","last_active":"2023-09-23T20:33:03.286511+0000","last_peered":"2023-09-23T20:33:03.286511+0000","last_clean":"2023-09-23T20:33:03.286511+0000","last_became_active":"2023-09-23T20:32:50.537128+0000","last_became_peered":"2023-09-23T20:32:50.537128+0000","last_unstale":"2023-09-23T20:33:03.286511+0000","last_undegraded":"2023-09-23T20:33:03.286511+0000","last_fullsized":"2023-09-23T20:33:03.286511+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:37:15.889029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":307,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T20:33:02.264596+0000","last_change":"2023-09-23T20:25:39.668882+0000","last_active":"2023-09-23T20:33:02.264596+0000","last_peered":"2023-09-23T20:33:02.264596+0000","last_clean":"2023-09-23T20:33:02.264596+0000","last_became_active":"2023-09-23T20:25:39.637128+0000","last_became_peered":"2023-09-23T20:25:39.637128+0000","last_unstale":"2023-09-23T20:33:02.264596+0000","last_undegraded":"2023-09-23T20:33:02.264596+0000","last_fullsized":"2023-09-23T20:33:02.264596+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:56.406298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":23,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T20:33:02.243799+0000","last_change":"2023-09-23T20:32:33.232281+0000","last_active":"2023-09-23T20:33:02.243799+0000","last_peered":"2023-09-23T20:33:02.243799+0000","last_clean":"2023-09-23T20:33:02.243799+0000","last_became_active":"2023-09-23T20:32:33.232003+0000","last_became_peered":"2023-09-23T20:32:33.232003+0000","last_unstale":"2023-09-23T20:33:02.243799+0000","last_undegraded":"2023-09-23T20:33:02.243799+0000","last_fullsized":"2023-09-23T20:33:02.243799+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:58:27.314355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":5202903040,"num_objects":79390,"num_object_clones":0,"num_object_copies":238170,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5117,"num_objects_misplaced":3213,"num_objects_unfound":0,"num_objects_dirty":79390,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":79390,"num_write_kb":5080960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1536,"num_bytes_recovered":100663296,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":15834021888,"data_stored":15834021888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":79390,"ondisk_log_size":79390,"up":48,"acting":44,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":48,"num_bytes_recovered":6522,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26691,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":389805,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3381,"ondisk_log_size":3381,"up":63,"acting":63,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":58,"num_bytes_recovered":4194342,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6082,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201543,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2152172,"kb_used_data":1883236,"kb_used_omap":0,"kb_used_meta":268928,"kb_avail":91592980,"statfs":{"total":95995035648,"available":93791211520,"internally_reserved":0,"allocated":1928433664,"data_stored":1927408314,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":275382272},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:32:36 2023","interfaces":[{"interface":"back","average":{"1min":0.91500000000000004,"5min":1.3740000000000001,"15min":1.282},"min":{"1min":0.23599999999999999,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":3.9119999999999999,"5min":8.0660000000000007,"15min":48.798999999999999},"last":1.4390000000000001},{"interface":"front","average":{"1min":0.94799999999999995,"5min":1.522,"15min":1.4530000000000001},"min":{"1min":0.30299999999999999,"5min":0.27300000000000002,"15min":0.24399999999999999},"max":{"1min":2.46,"5min":7.968,"15min":48.247},"last":1.4550000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:32:53 2023","interfaces":[{"interface":"back","average":{"1min":1.405,"5min":1.8859999999999999,"15min":1.8029999999999999},"min":{"1min":0.31900000000000001,"5min":0.254,"15min":0.254},"max":{"1min":4.7309999999999999,"5min":14.026,"15min":14.026},"last":1.7869999999999999},{"interface":"front","average":{"1min":1.3700000000000001,"5min":1.6699999999999999,"15min":1.877},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":5.2750000000000004,"5min":5.7679999999999998,"15min":5.7679999999999998},"last":2.1339999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:32:42 2023","interfaces":[{"interface":"back","average":{"1min":2.1219999999999999,"5min":1.8959999999999999,"15min":1.385},"min":{"1min":0.29099999999999998,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":13.173999999999999,"5min":13.173999999999999,"15min":13.173999999999999},"last":0.28599999999999998},{"interface":"front","average":{"1min":1.4139999999999999,"5min":2.1709999999999998,"15min":1.55},"min":{"1min":0.40400000000000003,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":6.9379999999999997,"5min":14.993,"15min":14.993},"last":5.6980000000000004}]},{"osd":3,"last update":"Sat Sep 23 20:33:05 2023","interfaces":[{"interface":"back","average":{"1min":2.903,"5min":2.2229999999999999,"15min":1.5880000000000001},"min":{"1min":0.375,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":7.9850000000000003,"5min":17.155999999999999,"15min":17.155999999999999},"last":2.036},{"interface":"front","average":{"1min":2.7829999999999999,"5min":2.0630000000000002,"15min":1.5049999999999999},"min":{"1min":0.47299999999999998,"5min":0.33100000000000002,"15min":0.27600000000000002},"max":{"1min":6.6379999999999999,"5min":16.960000000000001,"15min":16.960000000000001},"last":2}]},{"osd":4,"last update":"Sat Sep 23 20:33:05 2023","interfaces":[{"interface":"back","average":{"1min":0.65900000000000003,"5min":1.381,"15min":1.034},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.27000000000000002},"max":{"1min":2.3370000000000002,"5min":15.177,"15min":15.177},"last":0.371},{"interface":"front","average":{"1min":0.92500000000000004,"5min":1.1619999999999999,"15min":1.155},"min":{"1min":0.33900000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":5.2190000000000003,"5min":11.122,"15min":11.122},"last":0.38900000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:33:05 2023","interfaces":[{"interface":"back","average":{"1min":0.77100000000000002,"5min":0.97599999999999998,"15min":1.1120000000000001},"min":{"1min":0.317,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":3.0339999999999998,"5min":6.8300000000000001,"15min":13.241},"last":0.44900000000000001},{"interface":"front","average":{"1min":0.55100000000000005,"5min":1.272,"15min":1.2},"min":{"1min":0.35899999999999999,"5min":0.32200000000000001,"15min":0.315},"max":{"1min":1.3400000000000001,"5min":18.831,"15min":18.831},"last":0.41099999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:33:05 2023","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":1.2270000000000001,"15min":1.0960000000000001},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33400000000000002},"max":{"1min":1.0940000000000001,"5min":12.298999999999999,"15min":13.185},"last":0.53300000000000003},{"interface":"front","average":{"1min":0.93400000000000005,"5min":1.069,"15min":1.0649999999999999},"min":{"1min":0.37,"5min":0.318,"15min":0.29899999999999999},"max":{"1min":5.1059999999999999,"5min":8.0069999999999997,"15min":9.9120000000000008},"last":0.433}]}]},{"osd":6,"up_from":47,"seq":201863463179,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2303140,"kb_used_data":2053276,"kb_used_omap":0,"kb_used_meta":249856,"kb_avail":91442012,"statfs":{"total":95995035648,"available":93636620288,"internally_reserved":0,"allocated":2102554624,"data_stored":2101537339,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":255852544},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,3,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:32:36 2023","interfaces":[{"interface":"back","average":{"1min":3.9550000000000001,"5min":1.891,"15min":1.2669999999999999},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.221},"max":{"1min":53.701999999999998,"5min":53.701999999999998,"15min":53.701999999999998},"last":0.29599999999999999},{"interface":"front","average":{"1min":4.5439999999999996,"5min":2.4550000000000001,"15min":1.573},"min":{"1min":0.35599999999999998,"5min":0.251,"15min":0.25},"max":{"1min":53.875999999999998,"5min":53.875999999999998,"15min":53.875999999999998},"last":3.7829999999999999}]},{"osd":1,"last update":"Sat Sep 23 20:32:49 2023","interfaces":[{"interface":"back","average":{"1min":1.216,"5min":2.262,"15min":2.5880000000000001},"min":{"1min":0.35399999999999998,"5min":0.249,"15min":0.249},"max":{"1min":3.6019999999999999,"5min":14.1,"15min":14.1},"last":1.2470000000000001},{"interface":"front","average":{"1min":1.4390000000000001,"5min":2.7050000000000001,"15min":3.226},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":4.9199999999999999,"5min":14.266999999999999,"15min":14.266999999999999},"last":2.0590000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:32:36 2023","interfaces":[{"interface":"back","average":{"1min":1.0900000000000001,"5min":2.1859999999999999,"15min":1.4450000000000001},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":3.617,"5min":10.631,"15min":13.856},"last":2.153},{"interface":"front","average":{"1min":1.147,"5min":2.0939999999999999,"15min":1.377},"min":{"1min":0.38200000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":3.629,"5min":10.151999999999999,"15min":10.151999999999999},"last":2.218}]},{"osd":3,"last update":"Sat Sep 23 20:32:17 2023","interfaces":[{"interface":"back","average":{"1min":0.88300000000000001,"5min":1.766,"15min":1.4039999999999999},"min":{"1min":0.32200000000000001,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":5.1440000000000001,"5min":7.7130000000000001,"15min":9.4239999999999995},"last":4.0069999999999997},{"interface":"front","average":{"1min":0.92400000000000004,"5min":1.8440000000000001,"15min":1.45},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.36399999999999999},"max":{"1min":5.1500000000000004,"5min":7.7409999999999997,"15min":8.6050000000000004},"last":3.9820000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:33:00 2023","interfaces":[{"interface":"back","average":{"1min":0.49399999999999999,"5min":0.78800000000000003,"15min":0.85899999999999999},"min":{"1min":0.253,"5min":0.251,"15min":0.251},"max":{"1min":2.9020000000000001,"5min":8.4749999999999996,"15min":10.173},"last":0.52000000000000002},{"interface":"front","average":{"1min":0.83499999999999996,"5min":1.1259999999999999,"15min":1.0049999999999999},"min":{"1min":0.28199999999999997,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":7.7629999999999999,"5min":7.7629999999999999,"15min":8.3249999999999993},"last":3.8159999999999998}]},{"osd":5,"last update":"Sat Sep 23 20:33:00 2023","interfaces":[{"interface":"back","average":{"1min":0.88300000000000001,"5min":0.83099999999999996,"15min":0.92300000000000004},"min":{"1min":0.311,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":6.8310000000000004,"5min":8.4499999999999993,"15min":14.085000000000001},"last":0.45500000000000002},{"interface":"front","average":{"1min":0.90800000000000003,"5min":1.258,"15min":1.262},"min":{"1min":0.32100000000000001,"5min":0.30599999999999999,"15min":0.29499999999999998},"max":{"1min":7.7300000000000004,"5min":9.9299999999999997,"15min":18.192},"last":3.9510000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:32:17 2023","interfaces":[{"interface":"back","average":{"1min":0.52000000000000002,"5min":0.98399999999999999,"15min":1.032},"min":{"1min":0.34699999999999998,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.88800000000000001,"5min":8.5060000000000002,"15min":10.217000000000001},"last":0.49299999999999999},{"interface":"front","average":{"1min":0.55300000000000005,"5min":1.2889999999999999,"15min":1.1639999999999999},"min":{"1min":0.36399999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":1.175,"5min":7.8120000000000003,"15min":9.7360000000000007},"last":3.8420000000000001}]}]},{"osd":1,"up_from":266,"seq":1142461300778,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2467232,"kb_used_data":2287320,"kb_used_omap":8,"kb_used_meta":179895,"kb_avail":91277920,"statfs":{"total":95995035648,"available":93468590080,"internally_reserved":0,"allocated":2342215680,"data_stored":2340452511,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8206,"internal_metadata":184213490},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,3],"upper_bound":16},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:32:49 2023","interfaces":[{"interface":"back","average":{"1min":4.5289999999999999,"5min":2.7709999999999999,"15min":2.4769999999999999},"min":{"1min":0.224,"5min":0.218,"15min":0.218},"max":{"1min":52.689999999999998,"5min":52.689999999999998,"15min":52.689999999999998},"last":0.34000000000000002},{"interface":"front","average":{"1min":4.5279999999999996,"5min":3.1259999999999999,"15min":3.1720000000000002},"min":{"1min":0.218,"5min":0.217,"15min":0.217},"max":{"1min":52.662999999999997,"5min":52.662999999999997,"15min":52.662999999999997},"last":3.1859999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:32:49 2023","interfaces":[{"interface":"back","average":{"1min":1.371,"5min":3.8319999999999999,"15min":4.694},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":4.8689999999999998,"5min":16.151,"15min":16.151},"last":12.887},{"interface":"front","average":{"1min":1.7749999999999999,"5min":2.0030000000000001,"15min":2.0630000000000002},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":8.077,"5min":12.356,"15min":12.356},"last":0.72899999999999998}]},{"osd":3,"last update":"Sat Sep 23 20:32:49 2023","interfaces":[{"interface":"back","average":{"1min":8.2010000000000005,"5min":4.3849999999999998,"15min":4.0389999999999997},"min":{"1min":0.311,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":80.745999999999995,"5min":80.745999999999995,"15min":80.745999999999995},"last":7.9180000000000001},{"interface":"front","average":{"1min":7.6909999999999998,"5min":4.7530000000000001,"15min":4.8150000000000004},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":82.988,"5min":82.988,"15min":82.988},"last":0.60499999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:32:49 2023","interfaces":[{"interface":"back","average":{"1min":1.0700000000000001,"5min":1.855,"15min":2.004},"min":{"1min":0.40699999999999997,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":2.863,"5min":9.4429999999999996,"15min":9.4429999999999996},"last":7.6399999999999997},{"interface":"front","average":{"1min":1.272,"5min":1.7150000000000001,"15min":1.921},"min":{"1min":0.372,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":4.0730000000000004,"5min":10.760999999999999,"15min":10.760999999999999},"last":0.58499999999999996}]},{"osd":5,"last update":"Sat Sep 23 20:32:49 2023","interfaces":[{"interface":"back","average":{"1min":0.95799999999999996,"5min":2.1200000000000001,"15min":2.5739999999999998},"min":{"1min":0.33400000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":2.649,"5min":9.6479999999999997,"15min":9.6479999999999997},"last":0.42399999999999999},{"interface":"front","average":{"1min":1.0669999999999999,"5min":1.9650000000000001,"15min":2.048},"min":{"1min":0.36899999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":2.9700000000000002,"5min":9.4700000000000006,"15min":9.4700000000000006},"last":7.6669999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:32:49 2023","interfaces":[{"interface":"back","average":{"1min":1.018,"5min":4.9980000000000002,"15min":6.6319999999999997},"min":{"1min":0.40300000000000002,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":2.714,"5min":72.682000000000002,"15min":72.682000000000002},"last":0.35899999999999999},{"interface":"front","average":{"1min":1.0740000000000001,"5min":4.5720000000000001,"15min":5.8170000000000002},"min":{"1min":0.438,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":2.9319999999999999,"5min":72.563999999999993,"15min":72.563999999999993},"last":7.5960000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:32:49 2023","interfaces":[{"interface":"back","average":{"1min":1.085,"5min":2.4180000000000001,"15min":2.9940000000000002},"min":{"1min":0.39900000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":2.6880000000000002,"5min":9.6850000000000005,"15min":9.6850000000000005},"last":0.39900000000000002},{"interface":"front","average":{"1min":1.147,"5min":2.1480000000000001,"15min":2.2890000000000001},"min":{"1min":0.51200000000000001,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":2.8980000000000001,"5min":9.3949999999999996,"15min":9.3949999999999996},"last":7.6929999999999996}]}]},{"osd":0,"up_from":94,"seq":403726925980,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2909688,"kb_used_data":2512496,"kb_used_omap":7,"kb_used_meta":397176,"kb_avail":90835464,"statfs":{"total":95995035648,"available":93015515136,"internally_reserved":0,"allocated":2572795904,"data_stored":2571180302,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":406708241},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:32:55 2023","interfaces":[{"interface":"back","average":{"1min":1.825,"5min":2.9750000000000001,"15min":3.5830000000000002},"min":{"1min":0.245,"5min":0.17999999999999999,"15min":0.17999999999999999},"max":{"1min":8.0630000000000006,"5min":15.542,"15min":15.542},"last":12.750999999999999},{"interface":"front","average":{"1min":1.873,"5min":3.3050000000000002,"15min":4.0730000000000004},"min":{"1min":0.29899999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":5.3410000000000002,"5min":10.49,"15min":10.49},"last":12.566000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:32:32 2023","interfaces":[{"interface":"back","average":{"1min":1.5249999999999999,"5min":3.085,"15min":1.875},"min":{"1min":0.246,"5min":0.185,"15min":0.185},"max":{"1min":6.5890000000000004,"5min":77.448999999999998,"15min":77.448999999999998},"last":12.487},{"interface":"front","average":{"1min":1.0269999999999999,"5min":2.7610000000000001,"15min":1.621},"min":{"1min":0.188,"5min":0.188,"15min":0.188},"max":{"1min":3.7130000000000001,"5min":81.724999999999994,"15min":81.724999999999994},"last":12.904999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:32:04 2023","interfaces":[{"interface":"back","average":{"1min":1.659,"5min":3.278,"15min":1.8600000000000001},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":15.458,"5min":81.400999999999996,"15min":81.400999999999996},"last":13.51},{"interface":"front","average":{"1min":0.93400000000000005,"5min":3.4580000000000002,"15min":1.8},"min":{"1min":0.34100000000000003,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":4.5220000000000002,"5min":81.137,"15min":81.137},"last":12.651999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:32:32 2023","interfaces":[{"interface":"back","average":{"1min":0.92800000000000005,"5min":1.78,"15min":1.504},"min":{"1min":0.34999999999999998,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":2.4550000000000001,"5min":9.9339999999999993,"15min":23.876999999999999},"last":12.587999999999999},{"interface":"front","average":{"1min":0.997,"5min":1.554,"15min":1.3129999999999999},"min":{"1min":0.39000000000000001,"5min":0.312,"15min":0.29299999999999998},"max":{"1min":3.8340000000000001,"5min":10.414,"15min":23.681999999999999},"last":12.608000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:32:32 2023","interfaces":[{"interface":"back","average":{"1min":1.0049999999999999,"5min":1.913,"15min":1.6799999999999999},"min":{"1min":0.35399999999999998,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":3.7160000000000002,"5min":14.289,"15min":24.198},"last":12.688000000000001},{"interface":"front","average":{"1min":0.94799999999999995,"5min":2.0030000000000001,"15min":1.51},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":2.0779999999999998,"5min":14.938000000000001,"15min":14.938000000000001},"last":12.634}]},{"osd":6,"last update":"Sat Sep 23 20:32:32 2023","interfaces":[{"interface":"back","average":{"1min":0.99199999999999999,"5min":1.7589999999999999,"15min":1.397},"min":{"1min":0.432,"5min":0.318,"15min":0.318},"max":{"1min":3.7789999999999999,"5min":10.441000000000001,"15min":24.408000000000001},"last":12.726000000000001},{"interface":"front","average":{"1min":0.95699999999999996,"5min":1.7569999999999999,"15min":1.3460000000000001},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":2.1539999999999999,"5min":9.6769999999999996,"15min":25.271000000000001},"last":12.673999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:32:32 2023","interfaces":[{"interface":"back","average":{"1min":0.96699999999999997,"5min":2.0720000000000001,"15min":1.522},"min":{"1min":0.32200000000000001,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":2.1259999999999999,"5min":24.614000000000001,"15min":25.359000000000002},"last":12.709},{"interface":"front","average":{"1min":1.0429999999999999,"5min":2.4220000000000002,"15min":1.8129999999999999},"min":{"1min":0.44,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":3.0550000000000002,"5min":24.669,"15min":24.669},"last":12.548999999999999}]}]},{"osd":2,"up_from":104,"seq":446676598928,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2459312,"kb_used_data":2016168,"kb_used_omap":11,"kb_used_meta":443124,"kb_avail":91285840,"statfs":{"total":95995035648,"available":93476700160,"internally_reserved":0,"allocated":2064556032,"data_stored":2062945474,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11878,"internal_metadata":453759386},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,3,0,2,2,3],"upper_bound":128},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:32:39 2023","interfaces":[{"interface":"back","average":{"1min":5.2270000000000003,"5min":3.032,"15min":1.736},"min":{"1min":0.185,"5min":0.17999999999999999,"15min":0.17999999999999999},"max":{"1min":82.424999999999997,"5min":82.424999999999997,"15min":82.424999999999997},"last":0.86099999999999999},{"interface":"front","average":{"1min":5.2329999999999997,"5min":3.3359999999999999,"15min":1.6919999999999999},"min":{"1min":0.27400000000000002,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":82.524000000000001,"5min":82.524000000000001,"15min":82.524000000000001},"last":0.78200000000000003}]},{"osd":1,"last update":"Sat Sep 23 20:32:49 2023","interfaces":[{"interface":"back","average":{"1min":1.782,"5min":3.5790000000000002,"15min":4.4269999999999996},"min":{"1min":0.245,"5min":0.221,"15min":0.221},"max":{"1min":6.6589999999999998,"5min":16.280000000000001,"15min":16.280000000000001},"last":3.1659999999999999},{"interface":"front","average":{"1min":1.696,"5min":2.399,"15min":2.7909999999999999},"min":{"1min":0.28000000000000003,"5min":0.248,"15min":0.248},"max":{"1min":5.1790000000000003,"5min":30.565000000000001,"15min":30.565000000000001},"last":0.53200000000000003}]},{"osd":3,"last update":"Sat Sep 23 20:33:04 2023","interfaces":[{"interface":"back","average":{"1min":2.6699999999999999,"5min":2.4129999999999998,"15min":1.456},"min":{"1min":0.31900000000000001,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":11.766999999999999,"5min":16.331,"15min":16.331},"last":3.2050000000000001},{"interface":"front","average":{"1min":2.859,"5min":2.786,"15min":1.6879999999999999},"min":{"1min":0.28999999999999998,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":15.266,"5min":30.704999999999998,"15min":30.704999999999998},"last":3.0139999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:32:39 2023","interfaces":[{"interface":"back","average":{"1min":1.5920000000000001,"5min":2.1629999999999998,"15min":1.385},"min":{"1min":0.36399999999999999,"5min":0.32800000000000001,"15min":0.28100000000000003},"max":{"1min":4.2549999999999999,"5min":16.797999999999998,"15min":16.797999999999998},"last":0.61199999999999999},{"interface":"front","average":{"1min":1.2250000000000001,"5min":2.1139999999999999,"15min":1.369},"min":{"1min":0.39200000000000002,"5min":0.28799999999999998,"15min":0.27600000000000002},"max":{"1min":3.8300000000000001,"5min":14.971,"15min":14.971},"last":3.3530000000000002}]},{"osd":5,"last update":"Sat Sep 23 20:32:39 2023","interfaces":[{"interface":"back","average":{"1min":1.496,"5min":3.3999999999999999,"15min":1.8999999999999999},"min":{"1min":0.32200000000000001,"5min":0.314,"15min":0.309},"max":{"1min":4.3289999999999997,"5min":95.393000000000001,"15min":95.393000000000001},"last":0.58999999999999997},{"interface":"front","average":{"1min":1.413,"5min":3.4740000000000002,"15min":1.9359999999999999},"min":{"1min":0.48099999999999998,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":6.0389999999999997,"5min":95.414000000000001,"15min":95.414000000000001},"last":3.3919999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:32:39 2023","interfaces":[{"interface":"back","average":{"1min":1.238,"5min":2.3959999999999999,"15min":1.4650000000000001},"min":{"1min":0.42899999999999999,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":3.8540000000000001,"5min":15.023999999999999,"15min":15.023999999999999},"last":3.4220000000000002},{"interface":"front","average":{"1min":1.3460000000000001,"5min":2.0910000000000002,"15min":1.4970000000000001},"min":{"1min":0.40000000000000002,"5min":0.316,"15min":0.316},"max":{"1min":6,"5min":30.670999999999999,"15min":30.670999999999999},"last":0.50700000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:32:39 2023","interfaces":[{"interface":"back","average":{"1min":1.2969999999999999,"5min":2.1539999999999999,"15min":1.554},"min":{"1min":0.43099999999999999,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":5.9649999999999999,"5min":30.741,"15min":30.741},"last":0.47199999999999998},{"interface":"front","average":{"1min":1.651,"5min":2.4980000000000002,"15min":1.6160000000000001},"min":{"1min":0.42799999999999999,"5min":0.38,"15min":0.38},"max":{"1min":4.2999999999999998,"5min":16.719000000000001,"15min":16.719000000000001},"last":0.56299999999999994}]}]},{"osd":3,"up_from":109,"seq":468151435402,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3595328,"kb_used_data":3127672,"kb_used_omap":4,"kb_used_meta":467643,"kb_avail":90149824,"statfs":{"total":95995035648,"available":92313419776,"internally_reserved":0,"allocated":3202736128,"data_stored":3201116524,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4514,"internal_metadata":478867038},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,4,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:32:08 2023","interfaces":[{"interface":"back","average":{"1min":1.694,"5min":1.9790000000000001,"15min":1.55},"min":{"1min":0.254,"5min":0.186,"15min":0.186},"max":{"1min":12.061999999999999,"5min":12.061999999999999,"15min":67.305999999999997},"last":14.808999999999999},{"interface":"front","average":{"1min":1.298,"5min":1.9410000000000001,"15min":1.4530000000000001},"min":{"1min":0.26600000000000001,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":9.9619999999999997,"5min":13.743,"15min":67.403999999999996},"last":7.9880000000000004}]},{"osd":1,"last update":"Sat Sep 23 20:32:51 2023","interfaces":[{"interface":"back","average":{"1min":1.617,"5min":2.6579999999999999,"15min":2.944},"min":{"1min":0.27100000000000002,"5min":0.222,"15min":0.222},"max":{"1min":5.8380000000000001,"5min":13.263,"15min":13.263},"last":0.23799999999999999},{"interface":"front","average":{"1min":3.4620000000000002,"5min":2.988,"15min":3.1640000000000001},"min":{"1min":0.26400000000000001,"5min":0.252,"15min":0.252},"max":{"1min":32.603999999999999,"5min":32.603999999999999,"15min":32.603999999999999},"last":0.40200000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:32:08 2023","interfaces":[{"interface":"back","average":{"1min":1.2609999999999999,"5min":2.2389999999999999,"15min":1.4570000000000001},"min":{"1min":0.24399999999999999,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":9.8179999999999996,"5min":13.523999999999999,"15min":24.045000000000002},"last":1.528},{"interface":"front","average":{"1min":1.887,"5min":2.3439999999999999,"15min":1.538},"min":{"1min":0.246,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":12.244999999999999,"5min":19.603000000000002,"15min":19.603000000000002},"last":13.568}]},{"osd":4,"last update":"Sat Sep 23 20:32:08 2023","interfaces":[{"interface":"back","average":{"1min":1.474,"5min":2.1800000000000002,"15min":1.4139999999999999},"min":{"1min":0.39100000000000001,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":11.802,"5min":46.042999999999999,"15min":46.042999999999999},"last":0.44900000000000001},{"interface":"front","average":{"1min":1.7250000000000001,"5min":2.5619999999999998,"15min":1.665},"min":{"1min":0.34100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":8.6579999999999995,"5min":46.579999999999998,"15min":46.579999999999998},"last":0.38400000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:32:08 2023","interfaces":[{"interface":"back","average":{"1min":1.903,"5min":2.3530000000000002,"15min":1.605},"min":{"1min":0.35599999999999998,"5min":0.31900000000000001,"15min":0.311},"max":{"1min":11.507999999999999,"5min":45.292000000000002,"15min":45.292000000000002},"last":0.48999999999999999},{"interface":"front","average":{"1min":1.669,"5min":2.4820000000000002,"15min":1.4690000000000001},"min":{"1min":0.41899999999999998,"5min":0.35099999999999998,"15min":0.313},"max":{"1min":9.891,"5min":44.679000000000002,"15min":44.679000000000002},"last":3.3519999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:32:08 2023","interfaces":[{"interface":"back","average":{"1min":1.53,"5min":2.2349999999999999,"15min":1.488},"min":{"1min":0.39400000000000002,"5min":0.38500000000000001,"15min":0.36899999999999999},"max":{"1min":9.859,"5min":17.524999999999999,"15min":17.524999999999999},"last":0.47099999999999997},{"interface":"front","average":{"1min":2.1059999999999999,"5min":2.202,"15min":1.4830000000000001},"min":{"1min":0.374,"5min":0.36399999999999999,"15min":0.35499999999999998},"max":{"1min":11.191000000000001,"5min":19.524000000000001,"15min":19.524000000000001},"last":0.42499999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:32:08 2023","interfaces":[{"interface":"back","average":{"1min":2.23,"5min":2.6829999999999998,"15min":1.657},"min":{"1min":0.441,"5min":0.374,"15min":0.34999999999999998},"max":{"1min":11.24,"5min":19.571999999999999,"15min":19.571999999999999},"last":0.56499999999999995},{"interface":"front","average":{"1min":1.4870000000000001,"5min":2.3260000000000001,"15min":1.663},"min":{"1min":0.42999999999999999,"5min":0.371,"15min":0.371},"max":{"1min":11.856999999999999,"5min":45.387999999999998,"15min":45.387999999999998},"last":1.6739999999999999}]}]},{"osd":4,"up_from":30,"seq":128849019155,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":676324,"kb_used_data":457756,"kb_used_omap":0,"kb_used_meta":218560,"kb_avail":93068828,"statfs":{"total":95995035648,"available":95302479872,"internally_reserved":0,"allocated":468742144,"data_stored":467724593,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":223805440},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:32:36 2023","interfaces":[{"interface":"back","average":{"1min":0.73099999999999998,"5min":1.3899999999999999,"15min":1.22},"min":{"1min":0.33600000000000002,"5min":0.24199999999999999,"15min":0.222},"max":{"1min":1.7450000000000001,"5min":8.984,"15min":20.411999999999999},"last":1.496},{"interface":"front","average":{"1min":0.90400000000000003,"5min":1.5880000000000001,"15min":1.331},"min":{"1min":0.36399999999999999,"5min":0.25800000000000001,"15min":0.253},"max":{"1min":3.0259999999999998,"5min":11.868,"15min":20.454999999999998},"last":0.42199999999999999}]},{"osd":1,"last update":"Sat Sep 23 20:32:52 2023","interfaces":[{"interface":"back","average":{"1min":1.1319999999999999,"5min":1.7170000000000001,"15min":1.911},"min":{"1min":0.29099999999999998,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":5.3940000000000001,"5min":8.6799999999999997,"15min":8.6799999999999997},"last":0.47099999999999997},{"interface":"front","average":{"1min":0.94299999999999995,"5min":1.6910000000000001,"15min":1.907},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":2.4870000000000001,"5min":8.7219999999999995,"15min":8.7219999999999995},"last":1.5369999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:32:41 2023","interfaces":[{"interface":"back","average":{"1min":1.258,"5min":1.994,"15min":1.4710000000000001},"min":{"1min":0.36599999999999999,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":4.1500000000000004,"5min":32.363,"15min":32.363},"last":1.637},{"interface":"front","average":{"1min":1.093,"5min":2.2170000000000001,"15min":1.635},"min":{"1min":0.375,"5min":0.27600000000000002,"15min":0.26800000000000002},"max":{"1min":3.782,"5min":34.042999999999999,"15min":34.042999999999999},"last":1.786}]},{"osd":3,"last update":"Sat Sep 23 20:33:05 2023","interfaces":[{"interface":"back","average":{"1min":2.004,"5min":2.4199999999999999,"15min":1.663},"min":{"1min":0.34799999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":6.5090000000000003,"5min":32.332999999999998,"15min":32.332999999999998},"last":1.605},{"interface":"front","average":{"1min":1.5629999999999999,"5min":2.4119999999999999,"15min":1.673},"min":{"1min":0.36299999999999999,"5min":0.318,"15min":0.30499999999999999},"max":{"1min":4.968,"5min":32.399999999999999,"15min":32.399999999999999},"last":1.571}]},{"osd":5,"last update":"Sat Sep 23 20:32:25 2023","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":2.0609999999999999,"15min":1.484},"min":{"1min":0.31900000000000001,"5min":0.25800000000000001,"15min":0.247},"max":{"1min":1.095,"5min":82.866,"15min":82.866},"last":0.438},{"interface":"front","average":{"1min":0.48299999999999998,"5min":1.9370000000000001,"15min":1.3700000000000001},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.27800000000000002},"max":{"1min":0.83099999999999996,"5min":82.546000000000006,"15min":82.546000000000006},"last":0.48599999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:32:41 2023","interfaces":[{"interface":"back","average":{"1min":0.68500000000000005,"5min":1.139,"15min":1.242},"min":{"1min":0.33300000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":2.9820000000000002,"5min":6.3620000000000001,"15min":41.371000000000002},"last":0.58399999999999996},{"interface":"front","average":{"1min":0.52600000000000002,"5min":1.175,"15min":1.1839999999999999},"min":{"1min":0.35099999999999998,"5min":0.318,"15min":0.27200000000000002},"max":{"1min":0.96099999999999997,"5min":11.199,"15min":41.140999999999998},"last":0.63}]},{"osd":7,"last update":"Sat Sep 23 20:33:05 2023","interfaces":[{"interface":"back","average":{"1min":1.024,"5min":1.4650000000000001,"15min":1.325},"min":{"1min":0.371,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":4.4349999999999996,"5min":15.667999999999999,"15min":16.427},"last":0.61599999999999999},{"interface":"front","average":{"1min":0.876,"5min":1.347,"15min":1.2549999999999999},"min":{"1min":0.38300000000000001,"5min":0.307,"15min":0.307},"max":{"1min":5.4889999999999999,"5min":15.58,"15min":19.018999999999998},"last":0.69999999999999996}]}]},{"osd":5,"up_from":39,"seq":167503724816,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1505056,"kb_used_data":1262104,"kb_used_omap":0,"kb_used_meta":242944,"kb_avail":92240096,"statfs":{"total":95995035648,"available":94453858304,"internally_reserved":0,"allocated":1292394496,"data_stored":1291377671,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":248774656},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:32:36 2023","interfaces":[{"interface":"back","average":{"1min":0.746,"5min":1.4610000000000001,"15min":1.2430000000000001},"min":{"1min":0.254,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":2.129,"5min":8.1609999999999996,"15min":32.475000000000001},"last":0.20699999999999999},{"interface":"front","average":{"1min":0.80300000000000005,"5min":1.4490000000000001,"15min":1.2390000000000001},"min":{"1min":0.30199999999999999,"5min":0.29199999999999998,"15min":0.20599999999999999},"max":{"1min":2.4569999999999999,"5min":8.4900000000000002,"15min":32.457000000000001},"last":0.26300000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:32:51 2023","interfaces":[{"interface":"back","average":{"1min":1.1459999999999999,"5min":1.702,"15min":1.9370000000000001},"min":{"1min":0.32900000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":5.2709999999999999,"5min":8.5839999999999996,"15min":8.5839999999999996},"last":0.52000000000000002},{"interface":"front","average":{"1min":0.878,"5min":1.3080000000000001,"15min":1.476},"min":{"1min":0.30399999999999999,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":4.577,"5min":6.7969999999999997,"15min":6.7969999999999997},"last":0.65000000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:32:36 2023","interfaces":[{"interface":"back","average":{"1min":0.96099999999999997,"5min":1.7,"15min":1.464},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.26700000000000002},"max":{"1min":3.609,"5min":18.472999999999999,"15min":34.514000000000003},"last":0.28199999999999997},{"interface":"front","average":{"1min":1.0840000000000001,"5min":1.8360000000000001,"15min":1.4179999999999999},"min":{"1min":0.34499999999999997,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":5.5309999999999997,"5min":8.6820000000000004,"15min":32.515000000000001},"last":0.69699999999999995}]},{"osd":3,"last update":"Sat Sep 23 20:32:08 2023","interfaces":[{"interface":"back","average":{"1min":1.6870000000000001,"5min":2.0449999999999999,"15min":1.3979999999999999},"min":{"1min":0.34200000000000003,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":8.3819999999999997,"5min":12.51,"15min":12.846},"last":1.508},{"interface":"front","average":{"1min":1.1679999999999999,"5min":1.635,"15min":1.268},"min":{"1min":0.436,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":8.3729999999999993,"5min":12.005000000000001,"15min":12.005000000000001},"last":0.73599999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:32:27 2023","interfaces":[{"interface":"back","average":{"1min":0.41699999999999998,"5min":1.4339999999999999,"15min":1.2729999999999999},"min":{"1min":0.28599999999999998,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.66300000000000003,"5min":28.015999999999998,"15min":28.015999999999998},"last":0.48899999999999999},{"interface":"front","average":{"1min":0.49099999999999999,"5min":1.0609999999999999,"15min":1.0409999999999999},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.29099999999999998},"max":{"1min":0.96599999999999997,"5min":12.300000000000001,"15min":12.300000000000001},"last":0.30599999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:32:41 2023","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":1.2090000000000001,"15min":1.179},"min":{"1min":0.32200000000000001,"5min":0.30599999999999999,"15min":0.30499999999999999},"max":{"1min":1.6080000000000001,"5min":7.7519999999999998,"15min":12.890000000000001},"last":0.53600000000000003},{"interface":"front","average":{"1min":0.57499999999999996,"5min":1.1850000000000001,"15min":1.1559999999999999},"min":{"1min":0.42099999999999999,"5min":0.36399999999999999,"15min":0.34499999999999997},"max":{"1min":1.105,"5min":16.265999999999998,"15min":16.265999999999998},"last":0.54000000000000004}]},{"osd":7,"last update":"Sat Sep 23 20:32:08 2023","interfaces":[{"interface":"back","average":{"1min":0.64300000000000002,"5min":1.4870000000000001,"15min":1.2130000000000001},"min":{"1min":0.35399999999999998,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":2.419,"5min":36.621000000000002,"15min":36.621000000000002},"last":0.36899999999999999},{"interface":"front","average":{"1min":0.48399999999999999,"5min":1.5289999999999999,"15min":1.1000000000000001},"min":{"1min":0.35399999999999998,"5min":0.33500000000000002,"15min":0.315},"max":{"1min":0.71199999999999997,"5min":37.331000000000003,"15min":37.331000000000003},"last":0.35299999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4722,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1360,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8206,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7156,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3154,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2581463040,"data_stored":2581463040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2325217280,"data_stored":2325217280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1963458560,"data_stored":1963458560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3220439040,"data_stored":3220439040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":473104384,"data_stored":473104384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1295253504,"data_stored":1295253504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2101870592,"data_stored":2101870592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1873215488,"data_stored":1873215488,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:33:11.573 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd pool set .mgr pgp_num 21 2023-09-23T20:33:11.595 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:33:11.604 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:11.604 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:33:11.638 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:11 smithi018 ceph-mon[179647]: pgmap v755: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 37 MiB/s wr, 586 op/s; 5117/238254 objects degraded (2.148%); 3213/238254 objects misplaced (1.349%); 3.5 MiB/s, 56 objects/s recovering 2023-09-23T20:33:11.639 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:11 smithi018 ceph-mon[179647]: from='client.55447 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:33:11.639 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:11 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 5117/238254 objects degraded (2.148%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T20:33:11.639 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:11 smithi018 ceph-mon[184144]: pgmap v755: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 37 MiB/s wr, 586 op/s; 5117/238254 objects degraded (2.148%); 3213/238254 objects misplaced (1.349%); 3.5 MiB/s, 56 objects/s recovering 2023-09-23T20:33:11.639 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:11 smithi018 ceph-mon[184144]: from='client.55447 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:33:11.639 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:11 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 5117/238254 objects degraded (2.148%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T20:33:11.705 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:33:11.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:11 smithi149 ceph-mon[170394]: pgmap v755: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 37 MiB/s wr, 586 op/s; 5117/238254 objects degraded (2.148%); 3213/238254 objects misplaced (1.349%); 3.5 MiB/s, 56 objects/s recovering 2023-09-23T20:33:11.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:11 smithi149 ceph-mon[170394]: from='client.55447 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:33:11.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:11 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 5117/238254 objects degraded (2.148%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T20:33:11.752 INFO:teuthology.orchestra.run.smithi149.stdout:ERROR: (22) Invalid argument 2023-09-23T20:33:11.752 INFO:teuthology.orchestra.run.smithi149.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:33:11.814 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 75 16 83524 83508 69.5416 51.1875 0.0132794 0.0143737 2023-09-23T20:33:11.835 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:11.853 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:33:12.048 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:33:12.167 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T20:33:12.814 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 76 16 84322 84306 69.2829 49.875 0.0120468 0.0144284 2023-09-23T20:33:12.888 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:33:13.170 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:33:13.185 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:13.186 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:33:13.287 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:33:13.372 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:13.394 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:13 smithi149 ceph-mon[170394]: pgmap v756: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 5.0 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 46 MiB/s wr, 732 op/s; 5117/245343 objects degraded (2.086%); 3213/245343 objects misplaced (1.310%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T20:33:13.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:13 smithi018 ceph-mon[179647]: pgmap v756: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 5.0 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 46 MiB/s wr, 732 op/s; 5117/245343 objects degraded (2.086%); 3213/245343 objects misplaced (1.310%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T20:33:13.643 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:13 smithi018 ceph-mon[184144]: pgmap v756: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 5.0 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 46 MiB/s wr, 732 op/s; 5117/245343 objects degraded (2.086%); 3213/245343 objects misplaced (1.310%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T20:33:13.815 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 77 16 85173 85157 69.0733 53.1875 0.0206778 0.014473 2023-09-23T20:33:14.119 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:33:14.124 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:14.125 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:33:14.226 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:33:14.309 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:14.329 INFO:teuthology.orchestra.run.smithi149.stdout:ERROR: (22) Invalid argument 2023-09-23T20:33:14.330 INFO:teuthology.orchestra.run.smithi149.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:33:14.389 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:33:14.398 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:14.399 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:33:14.500 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:33:14.533 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:14.708 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:33:14.712 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:14.712 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:14 smithi149 ceph-mon[170394]: 1.4 scrub starts 2023-09-23T20:33:14.713 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:33:14.805 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:14 smithi018 ceph-mon[179647]: 1.4 scrub starts 2023-09-23T20:33:14.807 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:14 smithi018 ceph-mon[184144]: 1.4 scrub starts 2023-09-23T20:33:14.812 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T20:33:14.813 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:33:14.816 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 78 16 85921 85905 68.7872 46.75 0.0196049 0.0145314 2023-09-23T20:33:14.822 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:33:14.905 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:15.022 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:33:15.023 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:33:15.042 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:15.043 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:33:15.138 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:33:14 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:33:14] "GET /metrics HTTP/1.1" 200 36552 "" "Prometheus/2.43.0" 2023-09-23T20:33:15.143 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:33:15.185 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:15.242 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:33:15.246 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:15.247 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:33:15.348 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:33:15.430 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:15.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:15 smithi149 ceph-mon[170394]: pgmap v757: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 45 MiB/s wr, 719 op/s; 5117/248208 objects degraded (2.062%); 3010/248208 objects misplaced (1.213%); 2.5 MiB/s, 39 objects/s recovering 2023-09-23T20:33:15.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:15 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:33:15.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:15 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:33:15.520 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:33:15.648 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:33:15.662 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:15 smithi018 ceph-mon[179647]: pgmap v757: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 45 MiB/s wr, 719 op/s; 5117/248208 objects degraded (2.062%); 3010/248208 objects misplaced (1.213%); 2.5 MiB/s, 39 objects/s recovering 2023-09-23T20:33:15.662 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:15 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:33:15.662 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:15 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:33:15.663 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:15.664 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:33:15.664 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:15 smithi018 ceph-mon[184144]: pgmap v757: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 45 MiB/s wr, 719 op/s; 5117/248208 objects degraded (2.062%); 3010/248208 objects misplaced (1.213%); 2.5 MiB/s, 39 objects/s recovering 2023-09-23T20:33:15.664 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:15 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:33:15.664 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:15 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:33:15.765 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:33:15.815 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 79 16 86627 86611 68.4749 44.125 0.0198336 0.0145983 2023-09-23T20:33:15.866 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:16.233 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:33:16.239 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:16.240 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:33:16.340 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:33:16.409 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:16.428 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:16 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 5117/248208 objects degraded (2.062%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T20:33:16.586 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:33:16.596 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:16.597 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:33:16.698 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:33:16.736 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:16 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 5117/248208 objects degraded (2.062%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T20:33:16.736 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:16 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 5117/248208 objects degraded (2.062%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T20:33:16.826 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:2023-09-23T20:33:16.820315+0000 min lat: 0.00108768 max lat: 2.45012 avg lat: 0.0146653 2023-09-23T20:33:16.827 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T20:33:16.827 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 80 16 87329 87313 68.1641 43.875 0.0318309 0.0146653 2023-09-23T20:33:16.827 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:16.915 INFO:teuthology.orchestra.run.smithi149.stdout:ERROR: (22) Invalid argument 2023-09-23T20:33:16.915 INFO:teuthology.orchestra.run.smithi149.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:33:17.068 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:33:17.076 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:17.077 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:33:17.178 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:33:17.289 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:17.309 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T20:33:17.368 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:33:17.378 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:17.378 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:33:17.462 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:33:17.479 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:33:17.497 INFO:teuthology.orchestra.run.smithi018.stderr:noscrub is set 2023-09-23T20:33:17.504 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:33:17.565 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:33:17.571 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:17.603 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:33:17.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:17 smithi018 ceph-mon[184144]: pgmap v758: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 48 MiB/s wr, 768 op/s; 5117/257121 objects degraded (1.990%); 3010/257121 objects misplaced (1.171%); 2.5 MiB/s, 39 objects/s recovering 2023-09-23T20:33:17.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:17 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2255608510' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:33:17.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:17 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:33:17.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:17 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2868803330' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:33:17.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:17 smithi018 ceph-mon[179647]: pgmap v758: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 48 MiB/s wr, 768 op/s; 5117/257121 objects degraded (1.990%); 3010/257121 objects misplaced (1.171%); 2.5 MiB/s, 39 objects/s recovering 2023-09-23T20:33:17.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:17 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2255608510' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:33:17.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:17 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:33:17.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:17 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2868803330' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:33:17.655 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:33:17.692 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:33:17.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:17 smithi149 ceph-mon[170394]: pgmap v758: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 48 MiB/s wr, 768 op/s; 5117/257121 objects degraded (1.990%); 3010/257121 objects misplaced (1.171%); 2.5 MiB/s, 39 objects/s recovering 2023-09-23T20:33:17.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:17 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2255608510' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:33:17.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:17 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:33:17.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:17 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2868803330' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:33:17.724 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:33:17.763 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:33:17.779 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:33:17.779 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:33:17.780 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:33:17.780 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:33:17.780 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:33:17.780 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:33:17.780 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:33:17.780 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:33:17.822 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 81 16 87973 87957 67.8177 40.25 0.0121171 0.0147394 2023-09-23T20:33:18.014 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:33:18.192 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:33:18.198 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:18.198 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:33:18.299 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:33:18.386 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:18.465 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:33:18.471 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:18.472 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:33:18.493 INFO:teuthology.orchestra.run.smithi018.stderr:set pool 1 pgp_num to 21 2023-09-23T20:33:18.572 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:33:18.620 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:18 smithi149 ceph-mon[170394]: 1.4 scrub starts 2023-09-23T20:33:18.620 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:18 smithi149 ceph-mon[170394]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:33:18.620 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:18 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:33:18.621 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:18 smithi149 ceph-mon[170394]: osdmap e328: 8 total, 8 up, 8 in 2023-09-23T20:33:18.621 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:18 smithi149 ceph-mon[170394]: failed to encode map e328 with expected crc 2023-09-23T20:33:18.621 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:18 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/87085685' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "21"}]: dispatch 2023-09-23T20:33:18.621 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:18 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "21"}]: dispatch 2023-09-23T20:33:18.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:18 smithi018 ceph-mon[179647]: 1.4 scrub starts 2023-09-23T20:33:18.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:18 smithi018 ceph-mon[179647]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:33:18.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:18 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:33:18.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:18 smithi018 ceph-mon[179647]: osdmap e328: 8 total, 8 up, 8 in 2023-09-23T20:33:18.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:18 smithi018 ceph-mon[179647]: failed to encode map e328 with expected crc 2023-09-23T20:33:18.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:18 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/87085685' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "21"}]: dispatch 2023-09-23T20:33:18.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:18 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "21"}]: dispatch 2023-09-23T20:33:18.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:18 smithi018 ceph-mon[184144]: 1.4 scrub starts 2023-09-23T20:33:18.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:18 smithi018 ceph-mon[184144]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:33:18.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:18 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:33:18.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:18 smithi018 ceph-mon[184144]: osdmap e328: 8 total, 8 up, 8 in 2023-09-23T20:33:18.643 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:18 smithi018 ceph-mon[184144]: failed to encode map e328 with expected crc 2023-09-23T20:33:18.643 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:18 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/87085685' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "21"}]: dispatch 2023-09-23T20:33:18.643 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:18 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "21"}]: dispatch 2023-09-23T20:33:18.643 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:18.823 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 82 16 88755 88739 67.5867 48.875 0.0209924 0.0147909 2023-09-23T20:33:18.864 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:33:18.871 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:18.872 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:33:18.973 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:33:19.043 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:19.102 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:33:19.105 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:19.105 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:33:19.206 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:33:19.252 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:19.409 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:33:19.419 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:19.419 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:33:19.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:19 smithi149 ceph-mon[170394]: pgmap v760: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 48 MiB/s wr, 762 op/s; 5117/259797 objects degraded (1.970%); 2806/259797 objects misplaced (1.080%); 2.5 MiB/s, 40 objects/s recovering 2023-09-23T20:33:19.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:19 smithi149 ceph-mon[170394]: failed to encode map e329 with expected crc 2023-09-23T20:33:19.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:19 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "21"}]': finished 2023-09-23T20:33:19.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:19 smithi149 ceph-mon[170394]: osdmap e329: 8 total, 8 up, 8 in 2023-09-23T20:33:19.520 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:33:19.526 INFO:teuthology.orchestra.run.smithi149.stdout:{ 2023-09-23T20:33:19.526 INFO:teuthology.orchestra.run.smithi149.stdout: "size": 20, 2023-09-23T20:33:19.526 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 600, 2023-09-23T20:33:19.526 INFO:teuthology.orchestra.run.smithi149.stdout: "ops": [ 2023-09-23T20:33:19.527 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.527 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:49668 5.11 5:8ffafbda:::benchmark_data_smithi018_228451_object49667:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e130)", 2023-09-23T20:33:19.527 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:30.144126+0000", 2023-09-23T20:33:19.527 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 589.37863364299994, 2023-09-23T20:33:19.527 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0173537960000001, 2023-09-23T20:33:19.527 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:33:19.527 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:33:19.527 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:33:19.527 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:33:19.527 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:33:19.527 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 49668 2023-09-23T20:33:19.528 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.528 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:33:19.528 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.528 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:33:19.528 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.144126+0000", 2023-09-23T20:33:19.528 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:33:19.528 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.528 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.528 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:33:19.528 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.144126+0000", 2023-09-23T20:33:19.528 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:33:19.528 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.529 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.529 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:33:19.529 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.144124+0000", 2023-09-23T20:33:19.529 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:33:19.529 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.529 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.529 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:33:19.529 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.144171+0000", 2023-09-23T20:33:19.529 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.7029e-05 2023-09-23T20:33:19.529 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.529 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.529 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:33:19.530 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.144188+0000", 2023-09-23T20:33:19.530 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.721e-05 2023-09-23T20:33:19.530 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.530 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.530 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:33:19.530 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.144196+0000", 2023-09-23T20:33:19.530 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.498e-06 2023-09-23T20:33:19.530 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.530 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.530 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:33:19.530 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.144252+0000", 2023-09-23T20:33:19.530 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.5965999999999999e-05 2023-09-23T20:33:19.531 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.531 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.531 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op must wait for map", 2023-09-23T20:33:19.531 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.144260+0000", 2023-09-23T20:33:19.531 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.1920000000000005e-06 2023-09-23T20:33:19.531 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.531 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.531 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:33:19.531 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.149093+0000", 2023-09-23T20:33:19.531 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.004832622 2023-09-23T20:33:19.531 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.531 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.532 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:33:19.532 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.149103+0000", 2023-09-23T20:33:19.532 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.8479999999999996e-06 2023-09-23T20:33:19.532 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.532 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.532 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:33:19.532 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.147686+0000", 2023-09-23T20:33:19.532 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.99858325400000003 2023-09-23T20:33:19.532 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.532 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.532 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:33:19.532 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.147822+0000", 2023-09-23T20:33:19.532 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00013592199999999999 2023-09-23T20:33:19.533 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.533 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.533 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 4,7", 2023-09-23T20:33:19.533 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.148002+0000", 2023-09-23T20:33:19.533 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000180185 2023-09-23T20:33:19.533 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.533 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.533 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:33:19.533 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.152001+0000", 2023-09-23T20:33:19.533 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0039989049999999996 2023-09-23T20:33:19.533 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.533 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.534 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:33:19.534 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.161379+0000", 2023-09-23T20:33:19.534 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0093781179999999995 2023-09-23T20:33:19.534 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.534 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.534 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:33:19.534 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.161407+0000", 2023-09-23T20:33:19.534 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.7951999999999999e-05 2023-09-23T20:33:19.534 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.534 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.534 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:33:19.534 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.161445+0000", 2023-09-23T20:33:19.535 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.7999e-05 2023-09-23T20:33:19.535 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.535 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.535 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:33:19.535 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.161480+0000", 2023-09-23T20:33:19.535 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.4752999999999997e-05 2023-09-23T20:33:19.535 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:33:19.535 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:33:19.535 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:33:19.535 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.535 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.535 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:49677 5.11 5:8cd3bece:::benchmark_data_smithi018_228451_object49676:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-09-23T20:33:19.536 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:30.145601+0000", 2023-09-23T20:33:19.536 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 589.37715877899996, 2023-09-23T20:33:19.536 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0160527699999999, 2023-09-23T20:33:19.536 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:33:19.536 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:33:19.536 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:33:19.536 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:33:19.536 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:33:19.536 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 49677 2023-09-23T20:33:19.536 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.536 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:33:19.536 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.536 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:33:19.537 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.145601+0000", 2023-09-23T20:33:19.537 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:33:19.537 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.537 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.537 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:33:19.537 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.145601+0000", 2023-09-23T20:33:19.537 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:33:19.537 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.537 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.537 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:33:19.537 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.145600+0000", 2023-09-23T20:33:19.537 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:33:19.538 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.538 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.538 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:33:19.538 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.145714+0000", 2023-09-23T20:33:19.538 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000114341 2023-09-23T20:33:19.538 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.538 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.538 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:33:19.538 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.145729+0000", 2023-09-23T20:33:19.538 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.5061e-05 2023-09-23T20:33:19.538 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.539 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.539 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:33:19.539 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.145736+0000", 2023-09-23T20:33:19.539 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.9800000000000001e-06 2023-09-23T20:33:19.539 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.539 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.539 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:33:19.539 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.146231+0000", 2023-09-23T20:33:19.539 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00049503600000000002 2023-09-23T20:33:19.539 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.539 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.540 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting_for_map not empty", 2023-09-23T20:33:19.540 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.146242+0000", 2023-09-23T20:33:19.540 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1114999999999999e-05 2023-09-23T20:33:19.540 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.540 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.540 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:33:19.540 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.149132+0000", 2023-09-23T20:33:19.540 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0028893930000000001 2023-09-23T20:33:19.540 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.540 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.540 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:33:19.541 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.149139+0000", 2023-09-23T20:33:19.541 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.0319999999999998e-06 2023-09-23T20:33:19.541 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.541 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.541 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:33:19.541 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.148678+0000", 2023-09-23T20:33:19.541 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.999538861 2023-09-23T20:33:19.541 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.541 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.541 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:33:19.541 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.148809+0000", 2023-09-23T20:33:19.541 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000130968 2023-09-23T20:33:19.542 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.542 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.542 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 4,7", 2023-09-23T20:33:19.542 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.148965+0000", 2023-09-23T20:33:19.542 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00015674800000000001 2023-09-23T20:33:19.542 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.542 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.542 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:33:19.542 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.152012+0000", 2023-09-23T20:33:19.542 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0030466339999999999 2023-09-23T20:33:19.543 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.543 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.543 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:33:19.543 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.161507+0000", 2023-09-23T20:33:19.543 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0094949679999999995 2023-09-23T20:33:19.543 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.543 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.543 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:33:19.543 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.161588+0000", 2023-09-23T20:33:19.543 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.1589999999999996e-05 2023-09-23T20:33:19.543 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.543 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.544 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:33:19.544 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.161617+0000", 2023-09-23T20:33:19.544 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.8099e-05 2023-09-23T20:33:19.544 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.544 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.544 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:33:19.544 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.161654+0000", 2023-09-23T20:33:19.544 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.6927000000000001e-05 2023-09-23T20:33:19.544 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:33:19.544 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:33:19.544 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:33:19.545 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.545 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.545 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:49755 5.11 5:8b8671c4:::benchmark_data_smithi018_228451_object49754:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-09-23T20:33:19.545 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:30.216172+0000", 2023-09-23T20:33:19.545 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 589.30658707099997, 2023-09-23T20:33:19.545 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.94558546499999996, 2023-09-23T20:33:19.545 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:33:19.545 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:33:19.545 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:33:19.545 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:33:19.545 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:33:19.546 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 49755 2023-09-23T20:33:19.546 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.546 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:33:19.546 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.546 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:33:19.546 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.216172+0000", 2023-09-23T20:33:19.546 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:33:19.546 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.546 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.546 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:33:19.546 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.216172+0000", 2023-09-23T20:33:19.547 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:33:19.547 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.547 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.547 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:33:19.547 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.216170+0000", 2023-09-23T20:33:19.547 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999976 2023-09-23T20:33:19.547 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.547 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.547 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:33:19.547 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.216305+0000", 2023-09-23T20:33:19.547 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00013507699999999999 2023-09-23T20:33:19.548 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.548 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.548 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:33:19.548 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.216341+0000", 2023-09-23T20:33:19.548 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.6109999999999998e-05 2023-09-23T20:33:19.548 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.548 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.548 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:33:19.548 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.216349+0000", 2023-09-23T20:33:19.548 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.123e-06 2023-09-23T20:33:19.548 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.548 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.549 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:33:19.549 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.216426+0000", 2023-09-23T20:33:19.549 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.6604e-05 2023-09-23T20:33:19.549 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.549 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.549 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:33:19.549 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.216440+0000", 2023-09-23T20:33:19.549 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.3787e-05 2023-09-23T20:33:19.549 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.549 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.549 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:33:19.549 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.149653+0000", 2023-09-23T20:33:19.550 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.93321383099999999 2023-09-23T20:33:19.550 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.550 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.550 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:33:19.550 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.149779+0000", 2023-09-23T20:33:19.550 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00012566300000000001 2023-09-23T20:33:19.550 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.550 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.550 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 4,7", 2023-09-23T20:33:19.550 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.149925+0000", 2023-09-23T20:33:19.550 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00014582199999999999 2023-09-23T20:33:19.551 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.551 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.551 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:33:19.551 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.153584+0000", 2023-09-23T20:33:19.551 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0036591560000000002 2023-09-23T20:33:19.551 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.551 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.551 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:33:19.551 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.161679+0000", 2023-09-23T20:33:19.551 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0080945359999999994 2023-09-23T20:33:19.551 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.551 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.552 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:33:19.552 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.161695+0000", 2023-09-23T20:33:19.552 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.6311e-05 2023-09-23T20:33:19.552 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.552 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.552 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:33:19.552 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.161729+0000", 2023-09-23T20:33:19.552 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.4637000000000001e-05 2023-09-23T20:33:19.552 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.552 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.552 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:33:19.552 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.161758+0000", 2023-09-23T20:33:19.553 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.8422999999999999e-05 2023-09-23T20:33:19.553 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:33:19.553 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:33:19.553 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:33:19.553 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.553 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.553 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:49769 5.11 5:8ef0bf42:::benchmark_data_smithi018_228451_object49768:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-09-23T20:33:19.553 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:30.222066+0000", 2023-09-23T20:33:19.553 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 589.30069310600004, 2023-09-23T20:33:19.553 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.93977777699999998, 2023-09-23T20:33:19.553 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:33:19.554 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:33:19.554 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:33:19.554 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:33:19.554 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:33:19.554 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 49769 2023-09-23T20:33:19.554 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.554 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:33:19.554 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.554 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:33:19.554 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.222066+0000", 2023-09-23T20:33:19.554 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:33:19.554 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.555 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.555 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:33:19.555 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.222066+0000", 2023-09-23T20:33:19.555 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:33:19.555 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.555 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.555 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:33:19.555 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.222065+0000", 2023-09-23T20:33:19.555 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:33:19.555 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.555 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.555 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:33:19.556 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.222094+0000", 2023-09-23T20:33:19.556 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.9128999999999998e-05 2023-09-23T20:33:19.556 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.556 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.556 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:33:19.556 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.222112+0000", 2023-09-23T20:33:19.556 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.7099999999999999e-05 2023-09-23T20:33:19.556 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.556 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.556 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:33:19.556 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.222119+0000", 2023-09-23T20:33:19.556 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.1389999999999998e-06 2023-09-23T20:33:19.557 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.557 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.557 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:33:19.557 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.222180+0000", 2023-09-23T20:33:19.557 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.1301000000000001e-05 2023-09-23T20:33:19.557 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.557 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.557 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:33:19.557 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.222192+0000", 2023-09-23T20:33:19.557 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1661999999999999e-05 2023-09-23T20:33:19.557 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.557 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.558 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:33:19.558 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.150500+0000", 2023-09-23T20:33:19.558 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.92830877700000003 2023-09-23T20:33:19.558 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.558 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.558 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:33:19.558 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.150616+0000", 2023-09-23T20:33:19.558 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00011518099999999999 2023-09-23T20:33:19.558 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.558 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.558 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 4,7", 2023-09-23T20:33:19.559 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.150742+0000", 2023-09-23T20:33:19.559 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00012607699999999999 2023-09-23T20:33:19.559 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.559 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.559 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:33:19.559 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.153592+0000", 2023-09-23T20:33:19.559 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0028501569999999999 2023-09-23T20:33:19.559 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.559 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.559 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:33:19.559 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.161779+0000", 2023-09-23T20:33:19.559 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0081868459999999994 2023-09-23T20:33:19.560 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.560 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.560 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:33:19.560 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.161794+0000", 2023-09-23T20:33:19.560 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.5384999999999999e-05 2023-09-23T20:33:19.560 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.560 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.560 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:33:19.560 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.161826+0000", 2023-09-23T20:33:19.560 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.1483999999999999e-05 2023-09-23T20:33:19.560 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.560 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.561 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:33:19.561 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.161844+0000", 2023-09-23T20:33:19.561 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.8658999999999999e-05 2023-09-23T20:33:19.561 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:33:19.561 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:33:19.561 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:33:19.561 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.561 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.561 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:49810 5.11 5:8d760048:::benchmark_data_smithi018_228451_object49809:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-09-23T20:33:19.561 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:30.257088+0000", 2023-09-23T20:33:19.561 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 589.26567181400003, 2023-09-23T20:33:19.562 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.90484562899999998, 2023-09-23T20:33:19.562 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:33:19.562 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:33:19.562 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:33:19.562 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:33:19.562 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:33:19.562 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 49810 2023-09-23T20:33:19.562 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.562 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:33:19.562 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.562 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:33:19.562 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.257088+0000", 2023-09-23T20:33:19.563 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:33:19.563 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.563 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.563 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:33:19.563 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.257088+0000", 2023-09-23T20:33:19.563 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:33:19.563 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.563 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.563 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:33:19.563 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.257086+0000", 2023-09-23T20:33:19.563 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:33:19.563 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.564 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.564 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:33:19.564 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.257121+0000", 2023-09-23T20:33:19.564 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.4424000000000001e-05 2023-09-23T20:33:19.564 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.564 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.564 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:33:19.564 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.257134+0000", 2023-09-23T20:33:19.564 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.3409e-05 2023-09-23T20:33:19.564 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.564 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.564 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:33:19.565 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.257140+0000", 2023-09-23T20:33:19.565 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.9089999999999997e-06 2023-09-23T20:33:19.565 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.565 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.565 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:33:19.565 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.257208+0000", 2023-09-23T20:33:19.565 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.7716e-05 2023-09-23T20:33:19.565 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.565 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.565 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:33:19.565 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.257219+0000", 2023-09-23T20:33:19.565 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1382999999999999e-05 2023-09-23T20:33:19.566 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.566 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.566 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:33:19.566 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.151334+0000", 2023-09-23T20:33:19.566 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.89411527000000002 2023-09-23T20:33:19.566 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.566 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.566 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:33:19.566 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.151410+0000", 2023-09-23T20:33:19.566 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.5368000000000001e-05 2023-09-23T20:33:19.566 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.566 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.567 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 4,7", 2023-09-23T20:33:19.567 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.151519+0000", 2023-09-23T20:33:19.567 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000109481 2023-09-23T20:33:19.567 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.567 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.567 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:33:19.567 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.155912+0000", 2023-09-23T20:33:19.567 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0043927510000000003 2023-09-23T20:33:19.567 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.567 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.567 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:33:19.567 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.161867+0000", 2023-09-23T20:33:19.568 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0059548459999999998 2023-09-23T20:33:19.568 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.568 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.568 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:33:19.568 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.161882+0000", 2023-09-23T20:33:19.568 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.5053e-05 2023-09-23T20:33:19.568 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.568 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.568 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:33:19.568 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.161908+0000", 2023-09-23T20:33:19.568 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.6160999999999999e-05 2023-09-23T20:33:19.568 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.569 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.569 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:33:19.569 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.161933+0000", 2023-09-23T20:33:19.569 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.5448000000000001e-05 2023-09-23T20:33:19.569 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:33:19.569 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:33:19.569 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:33:19.569 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.569 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.569 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:49846 5.11 5:8a7c8f0f:::benchmark_data_smithi018_228451_object49845:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-09-23T20:33:19.569 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:30.283871+0000", 2023-09-23T20:33:19.569 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 589.23888849000002, 2023-09-23T20:33:19.570 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.87818426900000002, 2023-09-23T20:33:19.570 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:33:19.570 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:33:19.570 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:33:19.570 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:33:19.570 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:33:19.570 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 49846 2023-09-23T20:33:19.570 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.570 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:33:19.570 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.570 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:33:19.571 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.283871+0000", 2023-09-23T20:33:19.571 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:33:19.571 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.571 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.571 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:33:19.571 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.283871+0000", 2023-09-23T20:33:19.571 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:33:19.571 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.571 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.571 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:33:19.571 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.283870+0000", 2023-09-23T20:33:19.572 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:33:19.572 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.572 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.572 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:33:19.572 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.283939+0000", 2023-09-23T20:33:19.572 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.9799e-05 2023-09-23T20:33:19.572 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.572 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.572 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:33:19.572 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.283960+0000", 2023-09-23T20:33:19.572 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.0613999999999999e-05 2023-09-23T20:33:19.572 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.573 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.573 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:33:19.573 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.283968+0000", 2023-09-23T20:33:19.573 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.6429999999999995e-06 2023-09-23T20:33:19.573 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.573 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.573 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:33:19.573 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.284026+0000", 2023-09-23T20:33:19.573 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.8850000000000001e-05 2023-09-23T20:33:19.573 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.573 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.573 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:33:19.574 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.284033+0000", 2023-09-23T20:33:19.574 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.9249999999999998e-06 2023-09-23T20:33:19.574 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.574 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.574 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:33:19.574 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.152057+0000", 2023-09-23T20:33:19.574 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.86802386899999995 2023-09-23T20:33:19.574 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.574 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.574 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:33:19.574 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.152189+0000", 2023-09-23T20:33:19.575 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00013166600000000001 2023-09-23T20:33:19.575 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.575 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.575 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 4,7", 2023-09-23T20:33:19.575 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.152356+0000", 2023-09-23T20:33:19.575 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00016740599999999999 2023-09-23T20:33:19.575 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.575 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.575 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:33:19.575 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.156658+0000", 2023-09-23T20:33:19.575 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0043016720000000003 2023-09-23T20:33:19.575 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.576 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.576 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:33:19.576 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.161959+0000", 2023-09-23T20:33:19.576 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0053010699999999997 2023-09-23T20:33:19.576 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.576 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.576 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:33:19.576 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.161975+0000", 2023-09-23T20:33:19.576 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.6016e-05 2023-09-23T20:33:19.576 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.576 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.576 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:33:19.576 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.162015+0000", 2023-09-23T20:33:19.577 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.0240000000000001e-05 2023-09-23T20:33:19.577 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.577 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.577 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:33:19.577 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.162055+0000", 2023-09-23T20:33:19.577 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.9984000000000002e-05 2023-09-23T20:33:19.577 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:33:19.577 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:33:19.577 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:33:19.577 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.577 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.577 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:49862 5.11 5:8acf6c38:::benchmark_data_smithi018_228451_object49861:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-09-23T20:33:19.578 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:30.291752+0000", 2023-09-23T20:33:19.578 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 589.23100711999996, 2023-09-23T20:33:19.578 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.87046705199999996, 2023-09-23T20:33:19.578 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:33:19.578 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:33:19.578 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:33:19.578 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:33:19.578 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:33:19.578 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 49862 2023-09-23T20:33:19.578 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.578 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:33:19.578 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.579 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:33:19.579 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.291752+0000", 2023-09-23T20:33:19.579 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:33:19.579 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.579 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.579 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:33:19.579 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.291752+0000", 2023-09-23T20:33:19.579 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:33:19.579 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.579 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.579 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:33:19.579 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.291751+0000", 2023-09-23T20:33:19.579 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:33:19.580 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.580 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.580 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:33:19.580 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.291796+0000", 2023-09-23T20:33:19.580 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.4452000000000001e-05 2023-09-23T20:33:19.580 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.580 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.580 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:33:19.580 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.291814+0000", 2023-09-23T20:33:19.580 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.7846000000000002e-05 2023-09-23T20:33:19.580 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.580 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.581 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:33:19.581 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.291821+0000", 2023-09-23T20:33:19.581 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.4549999999999998e-06 2023-09-23T20:33:19.581 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.581 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.581 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:33:19.581 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.291932+0000", 2023-09-23T20:33:19.581 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000110945 2023-09-23T20:33:19.581 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.581 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.581 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:33:19.581 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.291942+0000", 2023-09-23T20:33:19.582 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0392e-05 2023-09-23T20:33:19.582 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.582 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.582 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:33:19.582 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.152826+0000", 2023-09-23T20:33:19.582 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.86088340100000005 2023-09-23T20:33:19.582 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.582 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.582 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:33:19.582 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.152946+0000", 2023-09-23T20:33:19.582 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000119963 2023-09-23T20:33:19.582 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.583 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.583 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 4,7", 2023-09-23T20:33:19.583 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.153060+0000", 2023-09-23T20:33:19.583 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00011451300000000001 2023-09-23T20:33:19.583 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.583 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.583 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:33:19.583 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.159094+0000", 2023-09-23T20:33:19.583 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0060340300000000001 2023-09-23T20:33:19.583 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.583 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.583 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:33:19.584 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.162100+0000", 2023-09-23T20:33:19.584 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0030057199999999999 2023-09-23T20:33:19.584 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.584 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.584 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:33:19.584 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.162137+0000", 2023-09-23T20:33:19.584 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.7002999999999998e-05 2023-09-23T20:33:19.584 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.584 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.584 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:33:19.584 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.162183+0000", 2023-09-23T20:33:19.584 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.6199000000000003e-05 2023-09-23T20:33:19.584 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.585 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.585 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:33:19.585 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.162219+0000", 2023-09-23T20:33:19.585 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.6186999999999998e-05 2023-09-23T20:33:19.585 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:33:19.585 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:33:19.585 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:33:19.585 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.585 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.585 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:49899 5.11 5:8d0362d8:::benchmark_data_smithi018_228451_object49898:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-09-23T20:33:19.585 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:30.316014+0000", 2023-09-23T20:33:19.585 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 589.20674549600005, 2023-09-23T20:33:19.586 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.84638897400000002, 2023-09-23T20:33:19.586 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:33:19.586 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:33:19.586 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:33:19.586 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:33:19.586 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:33:19.586 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 49899 2023-09-23T20:33:19.586 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.586 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:33:19.586 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.586 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:33:19.586 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.316014+0000", 2023-09-23T20:33:19.587 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:33:19.587 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.587 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.587 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:33:19.587 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.316014+0000", 2023-09-23T20:33:19.587 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:33:19.587 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.587 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.587 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:33:19.587 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.316007+0000", 2023-09-23T20:33:19.587 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999933 2023-09-23T20:33:19.587 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.588 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.588 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:33:19.588 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.316092+0000", 2023-09-23T20:33:19.588 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.4536999999999994e-05 2023-09-23T20:33:19.588 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.588 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.588 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:33:19.588 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.316110+0000", 2023-09-23T20:33:19.588 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.8204000000000001e-05 2023-09-23T20:33:19.588 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.588 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.588 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:33:19.589 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.316115+0000", 2023-09-23T20:33:19.589 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.2079999999999999e-06 2023-09-23T20:33:19.589 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.589 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.589 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:33:19.589 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.316190+0000", 2023-09-23T20:33:19.589 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.4400999999999995e-05 2023-09-23T20:33:19.589 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.589 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.589 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:33:19.589 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.316197+0000", 2023-09-23T20:33:19.589 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.0620000000000003e-06 2023-09-23T20:33:19.590 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.590 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.590 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:33:19.590 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.153639+0000", 2023-09-23T20:33:19.590 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.83744230799999997 2023-09-23T20:33:19.590 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.590 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.590 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:33:19.590 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.153769+0000", 2023-09-23T20:33:19.590 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00012973699999999999 2023-09-23T20:33:19.590 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.591 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.591 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 4,7", 2023-09-23T20:33:19.591 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.153882+0000", 2023-09-23T20:33:19.591 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00011381400000000001 2023-09-23T20:33:19.591 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.591 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.591 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:33:19.591 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.159107+0000", 2023-09-23T20:33:19.591 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0052248950000000002 2023-09-23T20:33:19.591 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.591 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.591 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:33:19.592 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.162256+0000", 2023-09-23T20:33:19.592 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0031487669999999998 2023-09-23T20:33:19.592 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.592 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.592 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:33:19.592 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.162314+0000", 2023-09-23T20:33:19.592 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.7707999999999997e-05 2023-09-23T20:33:19.592 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.592 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.592 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:33:19.592 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.162367+0000", 2023-09-23T20:33:19.592 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.3381000000000003e-05 2023-09-23T20:33:19.593 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.593 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.593 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:33:19.593 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.162403+0000", 2023-09-23T20:33:19.593 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.5754999999999999e-05 2023-09-23T20:33:19.593 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:33:19.593 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:33:19.593 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:33:19.593 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.593 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.593 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:49914 5.11 5:8de3f302:::benchmark_data_smithi018_228451_object49913:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-09-23T20:33:19.593 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:30.327710+0000", 2023-09-23T20:33:19.594 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 589.19504915799996, 2023-09-23T20:33:19.594 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.83480937700000002, 2023-09-23T20:33:19.594 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:33:19.594 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:33:19.594 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:33:19.594 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:33:19.594 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:33:19.594 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 49914 2023-09-23T20:33:19.594 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.594 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:33:19.594 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.594 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:33:19.595 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.327710+0000", 2023-09-23T20:33:19.595 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:33:19.595 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.595 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.595 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:33:19.595 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.327710+0000", 2023-09-23T20:33:19.595 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:33:19.595 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.595 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.595 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:33:19.595 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.327709+0000", 2023-09-23T20:33:19.595 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:33:19.595 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.596 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.596 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:33:19.596 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.327798+0000", 2023-09-23T20:33:19.596 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.9086999999999996e-05 2023-09-23T20:33:19.596 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.596 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.596 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:33:19.596 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.327814+0000", 2023-09-23T20:33:19.596 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.5974999999999999e-05 2023-09-23T20:33:19.596 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.596 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.596 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:33:19.597 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.327820+0000", 2023-09-23T20:33:19.597 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.614e-06 2023-09-23T20:33:19.597 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.597 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.597 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:33:19.597 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.327881+0000", 2023-09-23T20:33:19.597 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.1271000000000003e-05 2023-09-23T20:33:19.597 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.597 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.597 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:33:19.597 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.327897+0000", 2023-09-23T20:33:19.597 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.5962000000000001e-05 2023-09-23T20:33:19.597 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.598 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.598 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:33:19.598 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.154407+0000", 2023-09-23T20:33:19.598 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.82650979199999997 2023-09-23T20:33:19.598 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.598 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.598 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:33:19.598 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.154489+0000", 2023-09-23T20:33:19.598 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.1904999999999996e-05 2023-09-23T20:33:19.598 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.598 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.598 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 4,7", 2023-09-23T20:33:19.599 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.154587+0000", 2023-09-23T20:33:19.599 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.8391e-05 2023-09-23T20:33:19.599 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.599 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.599 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:33:19.599 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.159115+0000", 2023-09-23T20:33:19.599 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0045281030000000003 2023-09-23T20:33:19.599 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.599 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.599 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:33:19.599 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.162286+0000", 2023-09-23T20:33:19.599 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0031707110000000001 2023-09-23T20:33:19.600 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.600 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.600 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:33:19.600 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.162438+0000", 2023-09-23T20:33:19.600 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000152192 2023-09-23T20:33:19.600 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.600 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.600 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:33:19.600 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.162483+0000", 2023-09-23T20:33:19.600 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.4950000000000002e-05 2023-09-23T20:33:19.600 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.600 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.601 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:33:19.601 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.162520+0000", 2023-09-23T20:33:19.601 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.6643999999999997e-05 2023-09-23T20:33:19.601 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:33:19.601 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:33:19.601 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:33:19.601 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.601 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.601 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:49943 5.11 5:88faa156:::benchmark_data_smithi018_228451_object49942:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e130)", 2023-09-23T20:33:19.601 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:23:30.349163+0000", 2023-09-23T20:33:19.601 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 589.17359645800002, 2023-09-23T20:33:19.601 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.81349829299999998, 2023-09-23T20:33:19.601 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:33:19.602 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:33:19.602 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:33:19.602 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:33:19.602 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:33:19.602 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 49943 2023-09-23T20:33:19.602 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.602 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:33:19.602 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.602 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:33:19.602 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.349163+0000", 2023-09-23T20:33:19.602 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:33:19.602 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.603 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.603 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:33:19.603 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.349163+0000", 2023-09-23T20:33:19.603 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:33:19.603 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.603 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.603 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:33:19.603 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.349162+0000", 2023-09-23T20:33:19.603 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:33:19.603 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.603 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.603 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:33:19.604 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.349210+0000", 2023-09-23T20:33:19.604 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.8068999999999997e-05 2023-09-23T20:33:19.604 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.604 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.604 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:33:19.604 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.349222+0000", 2023-09-23T20:33:19.604 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.2434e-05 2023-09-23T20:33:19.604 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.604 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.604 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:33:19.604 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.349228+0000", 2023-09-23T20:33:19.604 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.0170000000000004e-06 2023-09-23T20:33:19.605 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.605 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.605 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:33:19.605 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.349276+0000", 2023-09-23T20:33:19.605 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.8976e-05 2023-09-23T20:33:19.605 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.605 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.605 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:33:19.605 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:30.349285+0000", 2023-09-23T20:33:19.605 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.4009999999999997e-06 2023-09-23T20:33:19.605 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.605 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.606 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:33:19.606 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.155211+0000", 2023-09-23T20:33:19.606 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.80592635199999996 2023-09-23T20:33:19.606 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.606 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.606 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:33:19.606 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.155298+0000", 2023-09-23T20:33:19.606 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.6293999999999997e-05 2023-09-23T20:33:19.606 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.606 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.606 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 4,7", 2023-09-23T20:33:19.607 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.155431+0000", 2023-09-23T20:33:19.607 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000133416 2023-09-23T20:33:19.607 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.607 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.607 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:33:19.607 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.159126+0000", 2023-09-23T20:33:19.607 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0036948879999999999 2023-09-23T20:33:19.607 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.607 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.607 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:33:19.607 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.162565+0000", 2023-09-23T20:33:19.607 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.003438853 2023-09-23T20:33:19.608 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.608 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.608 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:33:19.608 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.162594+0000", 2023-09-23T20:33:19.608 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.9000999999999999e-05 2023-09-23T20:33:19.608 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.608 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.608 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:33:19.608 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.162632+0000", 2023-09-23T20:33:19.608 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.8451000000000002e-05 2023-09-23T20:33:19.608 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.608 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.609 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:33:19.609 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:23:31.162661+0000", 2023-09-23T20:33:19.609 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.9220000000000001e-05 2023-09-23T20:33:19.609 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:33:19.609 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:33:19.609 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:33:19.609 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.609 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.609 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35700.0:88072 7.7 7:ec9f0cc2:::benchmark_data_smithi018_308601_object88071:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e328)", 2023-09-23T20:33:19.609 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:33:17.936528+0000", 2023-09-23T20:33:19.609 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 1.586231626, 2023-09-23T20:33:19.609 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.028614133999999999, 2023-09-23T20:33:19.609 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:33:19.610 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:33:19.610 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:33:19.610 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35700", 2023-09-23T20:33:19.610 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:33:19.610 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 88072 2023-09-23T20:33:19.610 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.610 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:33:19.610 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.610 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:33:19.610 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:17.936528+0000", 2023-09-23T20:33:19.610 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:33:19.610 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.611 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.611 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:33:19.611 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:17.936528+0000", 2023-09-23T20:33:19.611 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:33:19.611 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.611 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.611 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:33:19.611 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:17.936527+0000", 2023-09-23T20:33:19.611 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:33:19.611 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.611 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.612 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:33:19.612 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:17.936574+0000", 2023-09-23T20:33:19.612 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.6817000000000003e-05 2023-09-23T20:33:19.612 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.612 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.612 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:33:19.612 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:17.936588+0000", 2023-09-23T20:33:19.612 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.4229999999999999e-05 2023-09-23T20:33:19.612 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.612 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.612 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:33:19.612 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:17.936593+0000", 2023-09-23T20:33:19.613 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.1270000000000002e-06 2023-09-23T20:33:19.613 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.613 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.613 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:33:19.613 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:17.936641+0000", 2023-09-23T20:33:19.613 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.8155000000000002e-05 2023-09-23T20:33:19.613 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.613 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.613 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:33:19.613 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:17.936775+0000", 2023-09-23T20:33:19.613 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00013327799999999999 2023-09-23T20:33:19.613 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.614 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.614 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:33:19.614 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:17.936936+0000", 2023-09-23T20:33:19.614 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00016150600000000001 2023-09-23T20:33:19.617 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.617 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.617 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:33:19.617 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:17.939595+0000", 2023-09-23T20:33:19.617 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0026592619999999999 2023-09-23T20:33:19.617 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.617 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.617 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:33:19.618 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:17.943063+0000", 2023-09-23T20:33:19.618 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0034672409999999998 2023-09-23T20:33:19.618 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.618 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.618 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:33:19.618 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:17.965059+0000", 2023-09-23T20:33:19.618 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.021996156999999999 2023-09-23T20:33:19.618 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.618 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.618 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:33:19.618 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:17.965110+0000", 2023-09-23T20:33:19.619 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.1538e-05 2023-09-23T20:33:19.619 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.619 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.619 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:33:19.619 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:17.965142+0000", 2023-09-23T20:33:19.619 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.1780999999999999e-05 2023-09-23T20:33:19.619 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:33:19.619 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:33:19.619 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:33:19.619 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.619 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.619 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35700.0:88222 7.7 7:e2567f43:::benchmark_data_smithi018_308601_object88221:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e328)", 2023-09-23T20:33:19.620 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:33:18.120786+0000", 2023-09-23T20:33:19.620 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 1.401973084, 2023-09-23T20:33:19.620 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.027380524, 2023-09-23T20:33:19.620 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:33:19.620 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:33:19.620 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:33:19.620 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35700", 2023-09-23T20:33:19.620 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:33:19.620 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 88222 2023-09-23T20:33:19.620 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.620 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:33:19.621 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.621 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:33:19.621 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:18.120786+0000", 2023-09-23T20:33:19.621 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:33:19.621 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.621 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.621 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:33:19.621 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:18.120786+0000", 2023-09-23T20:33:19.621 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:33:19.621 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.621 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.622 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:33:19.622 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:18.120786+0000", 2023-09-23T20:33:19.622 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:33:19.622 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.622 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.622 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:33:19.622 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:18.120900+0000", 2023-09-23T20:33:19.622 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000114073 2023-09-23T20:33:19.622 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.622 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.622 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:33:19.622 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:18.120910+0000", 2023-09-23T20:33:19.623 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0164e-05 2023-09-23T20:33:19.623 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.623 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.623 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:33:19.623 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:18.120915+0000", 2023-09-23T20:33:19.623 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.7010000000000004e-06 2023-09-23T20:33:19.623 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.623 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.623 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:33:19.623 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:18.120952+0000", 2023-09-23T20:33:19.623 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.6903000000000002e-05 2023-09-23T20:33:19.623 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.623 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.624 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:33:19.624 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:18.121075+0000", 2023-09-23T20:33:19.624 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000123122 2023-09-23T20:33:19.624 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.624 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.624 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:33:19.624 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:18.121241+0000", 2023-09-23T20:33:19.624 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00016613899999999999 2023-09-23T20:33:19.624 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.624 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.624 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:33:19.625 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:18.124555+0000", 2023-09-23T20:33:19.625 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.003313959 2023-09-23T20:33:19.625 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.625 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.625 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:33:19.625 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:18.143599+0000", 2023-09-23T20:33:19.625 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.019044704999999999 2023-09-23T20:33:19.625 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.625 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.625 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:33:19.625 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:18.148088+0000", 2023-09-23T20:33:19.625 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.004488077 2023-09-23T20:33:19.626 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.626 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.626 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:33:19.626 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:18.148126+0000", 2023-09-23T20:33:19.626 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.8213000000000002e-05 2023-09-23T20:33:19.626 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.626 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.626 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:33:19.626 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:18.148167+0000", 2023-09-23T20:33:19.626 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.1207e-05 2023-09-23T20:33:19.626 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:33:19.626 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:33:19.627 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:33:19.627 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.627 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.627 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35700.0:88255 7.7 7:e0ae9182:::benchmark_data_smithi018_308601_object88254:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e328)", 2023-09-23T20:33:19.627 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:33:18.166566+0000", 2023-09-23T20:33:19.627 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 1.356193384, 2023-09-23T20:33:19.627 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.033275987, 2023-09-23T20:33:19.627 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:33:19.627 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:33:19.627 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:33:19.627 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35700", 2023-09-23T20:33:19.627 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:33:19.628 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 88255 2023-09-23T20:33:19.628 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.628 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:33:19.628 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.628 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:33:19.628 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:18.166566+0000", 2023-09-23T20:33:19.628 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:33:19.628 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.628 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.628 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:33:19.628 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:18.166566+0000", 2023-09-23T20:33:19.628 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:33:19.629 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.629 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.629 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:33:19.629 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:18.166565+0000", 2023-09-23T20:33:19.629 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:33:19.629 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.629 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.629 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:33:19.629 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:18.166616+0000", 2023-09-23T20:33:19.629 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.1091000000000003e-05 2023-09-23T20:33:19.629 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.629 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.629 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:33:19.630 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:18.166628+0000", 2023-09-23T20:33:19.630 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1615e-05 2023-09-23T20:33:19.630 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.630 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.630 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:33:19.630 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:18.166635+0000", 2023-09-23T20:33:19.630 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.8260000000000004e-06 2023-09-23T20:33:19.630 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.630 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.630 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:33:19.630 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:18.166696+0000", 2023-09-23T20:33:19.630 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.1435999999999999e-05 2023-09-23T20:33:19.631 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.631 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.631 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:33:19.631 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:18.166825+0000", 2023-09-23T20:33:19.631 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0001293 2023-09-23T20:33:19.631 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.631 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.631 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:33:19.631 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:18.166978+0000", 2023-09-23T20:33:19.631 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00015306099999999999 2023-09-23T20:33:19.631 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.631 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.632 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:33:19.632 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:18.170086+0000", 2023-09-23T20:33:19.632 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.003107364 2023-09-23T20:33:19.632 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.632 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.632 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:33:19.632 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:18.178087+0000", 2023-09-23T20:33:19.632 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0080017349999999994 2023-09-23T20:33:19.632 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.632 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.632 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:33:19.632 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:18.199767+0000", 2023-09-23T20:33:19.633 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.021679932999999998 2023-09-23T20:33:19.633 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.633 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.633 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:33:19.633 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:18.199802+0000", 2023-09-23T20:33:19.633 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.4904999999999999e-05 2023-09-23T20:33:19.633 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.633 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.633 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:33:19.633 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:18.199842+0000", 2023-09-23T20:33:19.633 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.9833999999999998e-05 2023-09-23T20:33:19.633 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:33:19.634 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:33:19.634 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:33:19.634 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.634 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.634 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35700.0:88304 7.1 7:85b7f1b0:::benchmark_data_smithi018_308601_object88303:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e328)", 2023-09-23T20:33:19.634 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:33:18.224905+0000", 2023-09-23T20:33:19.634 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 1.2978541029999999, 2023-09-23T20:33:19.634 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.031081711000000001, 2023-09-23T20:33:19.634 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:33:19.634 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:33:19.634 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:33:19.634 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35700", 2023-09-23T20:33:19.635 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:33:19.635 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 88304 2023-09-23T20:33:19.635 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.635 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:33:19.635 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.635 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:33:19.635 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:18.224905+0000", 2023-09-23T20:33:19.635 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:33:19.635 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.635 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.635 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:33:19.635 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:18.224905+0000", 2023-09-23T20:33:19.635 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:33:19.636 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.636 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.636 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:33:19.636 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:18.224904+0000", 2023-09-23T20:33:19.636 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:33:19.636 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.636 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.636 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:33:19.636 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:18.224999+0000", 2023-09-23T20:33:19.636 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.5365999999999995e-05 2023-09-23T20:33:19.636 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.636 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.637 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:33:19.637 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:18.225027+0000", 2023-09-23T20:33:19.637 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.8195999999999999e-05 2023-09-23T20:33:19.637 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.637 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.637 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:33:19.637 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:18.225035+0000", 2023-09-23T20:33:19.637 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.5939999999999996e-06 2023-09-23T20:33:19.637 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.637 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.637 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:33:19.637 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:18.225093+0000", 2023-09-23T20:33:19.637 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.8244e-05 2023-09-23T20:33:19.638 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.638 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.638 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:33:19.638 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:18.225254+0000", 2023-09-23T20:33:19.638 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00016070200000000001 2023-09-23T20:33:19.638 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.638 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.638 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:33:19.638 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:18.225421+0000", 2023-09-23T20:33:19.638 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00016741099999999999 2023-09-23T20:33:19.638 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.638 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.639 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:33:19.639 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:18.228002+0000", 2023-09-23T20:33:19.639 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0025804840000000001 2023-09-23T20:33:19.639 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.639 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.639 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:33:19.639 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:18.235872+0000", 2023-09-23T20:33:19.639 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0078697920000000005 2023-09-23T20:33:19.639 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.639 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.639 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:33:19.639 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:18.255802+0000", 2023-09-23T20:33:19.640 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.019929946 2023-09-23T20:33:19.640 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.640 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.640 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:33:19.640 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:18.255872+0000", 2023-09-23T20:33:19.640 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.0009e-05 2023-09-23T20:33:19.640 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.640 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.640 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:33:19.640 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:18.255987+0000", 2023-09-23T20:33:19.640 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00011560700000000001 2023-09-23T20:33:19.640 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:33:19.641 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:33:19.641 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:33:19.641 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.641 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.641 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35700.0:88575 7.d 7:b1c6af72:::benchmark_data_smithi018_308601_object88574:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e329)", 2023-09-23T20:33:19.641 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:33:18.598393+0000", 2023-09-23T20:33:19.641 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 0.92436643399999996, 2023-09-23T20:33:19.641 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.029553843999999999, 2023-09-23T20:33:19.641 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:33:19.641 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:33:19.641 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:33:19.641 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35700", 2023-09-23T20:33:19.641 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:33:19.642 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 88575 2023-09-23T20:33:19.642 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.642 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:33:19.642 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.642 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:33:19.642 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:18.598393+0000", 2023-09-23T20:33:19.642 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:33:19.642 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.642 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.642 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:33:19.642 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:18.598393+0000", 2023-09-23T20:33:19.643 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:33:19.643 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.643 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.643 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:33:19.643 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:18.598392+0000", 2023-09-23T20:33:19.643 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:33:19.643 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.643 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.643 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:33:19.643 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:18.598438+0000", 2023-09-23T20:33:19.643 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.5179000000000002e-05 2023-09-23T20:33:19.643 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.643 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.644 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:33:19.644 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:18.598443+0000", 2023-09-23T20:33:19.644 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.9009999999999999e-06 2023-09-23T20:33:19.644 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.644 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.644 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:33:19.644 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:18.598447+0000", 2023-09-23T20:33:19.644 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.985e-06 2023-09-23T20:33:19.644 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.644 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.644 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:33:19.644 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:18.601820+0000", 2023-09-23T20:33:19.645 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0033724919999999999 2023-09-23T20:33:19.645 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.645 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.645 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:33:19.645 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:18.601969+0000", 2023-09-23T20:33:19.645 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00014886500000000001 2023-09-23T20:33:19.645 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.645 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.645 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,1", 2023-09-23T20:33:19.645 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:18.602127+0000", 2023-09-23T20:33:19.645 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00015816199999999999 2023-09-23T20:33:19.645 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.646 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.646 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:33:19.646 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:18.604782+0000", 2023-09-23T20:33:19.646 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0026553409999999999 2023-09-23T20:33:19.646 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.646 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.646 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:33:19.646 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:18.618617+0000", 2023-09-23T20:33:19.646 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.013835048000000001 2023-09-23T20:33:19.646 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.646 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.646 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:33:19.647 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:18.627848+0000", 2023-09-23T20:33:19.647 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.009230505 2023-09-23T20:33:19.647 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.647 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.647 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:33:19.647 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:18.627904+0000", 2023-09-23T20:33:19.647 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.6320000000000003e-05 2023-09-23T20:33:19.647 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.647 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.647 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:33:19.647 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:18.627947+0000", 2023-09-23T20:33:19.647 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.2752e-05 2023-09-23T20:33:19.648 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:33:19.648 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:33:19.648 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:33:19.648 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.648 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.648 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35700.0:88748 7.d 7:be6368a1:::benchmark_data_smithi018_308601_object88747:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e329)", 2023-09-23T20:33:19.648 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:33:18.816737+0000", 2023-09-23T20:33:19.648 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 0.70602294300000001, 2023-09-23T20:33:19.648 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.032862818000000002, 2023-09-23T20:33:19.648 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:33:19.648 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:33:19.648 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:33:19.648 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35700", 2023-09-23T20:33:19.649 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:33:19.649 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 88748 2023-09-23T20:33:19.649 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.649 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:33:19.649 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.649 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:33:19.649 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:18.816737+0000", 2023-09-23T20:33:19.649 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:33:19.649 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.649 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.649 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:33:19.649 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:18.816737+0000", 2023-09-23T20:33:19.650 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:33:19.650 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.650 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.650 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:33:19.650 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:18.816735+0000", 2023-09-23T20:33:19.650 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:33:19.650 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.650 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.650 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:33:19.650 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:18.816799+0000", 2023-09-23T20:33:19.650 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.3596000000000006e-05 2023-09-23T20:33:19.650 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.651 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.651 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:33:19.651 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:18.816812+0000", 2023-09-23T20:33:19.651 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.3608000000000001e-05 2023-09-23T20:33:19.651 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.651 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.651 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:33:19.651 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:18.816821+0000", 2023-09-23T20:33:19.651 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.2719999999999997e-06 2023-09-23T20:33:19.651 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.651 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.651 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:33:19.652 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:18.816869+0000", 2023-09-23T20:33:19.652 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.7981e-05 2023-09-23T20:33:19.652 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.652 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.652 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:33:19.652 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:18.817003+0000", 2023-09-23T20:33:19.652 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00013443600000000001 2023-09-23T20:33:19.652 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.652 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.652 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,1", 2023-09-23T20:33:19.652 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:18.817163+0000", 2023-09-23T20:33:19.653 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00015996900000000001 2023-09-23T20:33:19.653 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.653 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.653 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:33:19.653 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:18.820412+0000", 2023-09-23T20:33:19.653 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0032491220000000001 2023-09-23T20:33:19.653 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.653 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.653 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:33:19.653 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:18.824821+0000", 2023-09-23T20:33:19.653 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0044092109999999997 2023-09-23T20:33:19.653 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.653 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.654 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:33:19.654 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:18.849518+0000", 2023-09-23T20:33:19.654 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.024697014999999999 2023-09-23T20:33:19.654 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.654 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.654 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:33:19.654 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:18.849566+0000", 2023-09-23T20:33:19.654 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.7673000000000002e-05 2023-09-23T20:33:19.654 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.654 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.654 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:33:19.655 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:18.849599+0000", 2023-09-23T20:33:19.655 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.3345e-05 2023-09-23T20:33:19.655 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:33:19.655 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:33:19.655 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:33:19.655 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.655 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.655 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35700.0:88749 7.7 7:ea81d780:::benchmark_data_smithi018_308601_object88748:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e329)", 2023-09-23T20:33:19.655 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:33:18.816839+0000", 2023-09-23T20:33:19.655 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 0.70592064499999996, 2023-09-23T20:33:19.655 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.027697741000000001, 2023-09-23T20:33:19.655 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:33:19.656 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:33:19.656 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:33:19.656 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35700", 2023-09-23T20:33:19.656 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:33:19.656 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 88749 2023-09-23T20:33:19.656 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.656 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:33:19.656 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.656 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:33:19.656 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:18.816839+0000", 2023-09-23T20:33:19.656 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:33:19.656 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.657 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.657 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:33:19.657 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:18.816839+0000", 2023-09-23T20:33:19.657 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:33:19.657 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.657 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.657 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:33:19.657 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:18.816838+0000", 2023-09-23T20:33:19.657 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:33:19.657 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.657 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.657 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:33:19.658 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:18.816870+0000", 2023-09-23T20:33:19.658 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.1075999999999999e-05 2023-09-23T20:33:19.658 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.658 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.658 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:33:19.658 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:18.816877+0000", 2023-09-23T20:33:19.658 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.9179999999999994e-06 2023-09-23T20:33:19.658 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.658 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.658 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:33:19.658 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:18.816883+0000", 2023-09-23T20:33:19.658 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.2449999999999998e-06 2023-09-23T20:33:19.659 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.659 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.659 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:33:19.659 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:18.816942+0000", 2023-09-23T20:33:19.659 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.9329000000000001e-05 2023-09-23T20:33:19.659 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.659 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.659 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:33:19.659 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:18.817041+0000", 2023-09-23T20:33:19.659 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.8855000000000001e-05 2023-09-23T20:33:19.659 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.659 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.660 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:33:19.660 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:18.817162+0000", 2023-09-23T20:33:19.660 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00012094100000000001 2023-09-23T20:33:19.660 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.660 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.660 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:33:19.660 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:18.819978+0000", 2023-09-23T20:33:19.660 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0028165030000000001 2023-09-23T20:33:19.660 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.660 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.660 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:33:19.660 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:18.821976+0000", 2023-09-23T20:33:19.661 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0019976669999999998 2023-09-23T20:33:19.661 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.661 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.661 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:33:19.661 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:18.844470+0000", 2023-09-23T20:33:19.661 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.022494466000000001 2023-09-23T20:33:19.661 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.661 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.661 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:33:19.661 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:18.844507+0000", 2023-09-23T20:33:19.661 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.6891000000000003e-05 2023-09-23T20:33:19.661 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.662 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.662 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:33:19.662 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:18.844537+0000", 2023-09-23T20:33:19.662 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.9237000000000001e-05 2023-09-23T20:33:19.662 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:33:19.662 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:33:19.662 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:33:19.662 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.662 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.662 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35700.0:88920 7.d 7:b84bc94e:::benchmark_data_smithi018_308601_object88919:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e329)", 2023-09-23T20:33:19.662 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:33:19.033437+0000", 2023-09-23T20:33:19.662 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 0.489322586, 2023-09-23T20:33:19.663 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.035926954999999997, 2023-09-23T20:33:19.663 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:33:19.663 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:33:19.663 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:33:19.663 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35700", 2023-09-23T20:33:19.663 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:33:19.663 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 88920 2023-09-23T20:33:19.663 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.663 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:33:19.663 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.663 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:33:19.663 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:19.033437+0000", 2023-09-23T20:33:19.663 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:33:19.664 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.664 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.664 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:33:19.664 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:19.033437+0000", 2023-09-23T20:33:19.664 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:33:19.664 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.664 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.664 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:33:19.664 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:19.033436+0000", 2023-09-23T20:33:19.664 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:33:19.664 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.664 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.665 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:33:19.665 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:19.033494+0000", 2023-09-23T20:33:19.665 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.7342000000000001e-05 2023-09-23T20:33:19.665 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.665 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.665 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:33:19.665 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:19.033503+0000", 2023-09-23T20:33:19.665 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.8710000000000003e-06 2023-09-23T20:33:19.665 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.665 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.665 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:33:19.665 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:19.033508+0000", 2023-09-23T20:33:19.665 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.4419999999999997e-06 2023-09-23T20:33:19.666 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.666 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.666 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:33:19.666 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:19.033561+0000", 2023-09-23T20:33:19.666 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.3099000000000001e-05 2023-09-23T20:33:19.666 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.666 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.666 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:33:19.666 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:19.033685+0000", 2023-09-23T20:33:19.666 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000123714 2023-09-23T20:33:19.666 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.666 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.667 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,1", 2023-09-23T20:33:19.667 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:19.033845+0000", 2023-09-23T20:33:19.667 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00016002099999999999 2023-09-23T20:33:19.667 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.667 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.667 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:33:19.667 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:19.037400+0000", 2023-09-23T20:33:19.667 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0035550920000000001 2023-09-23T20:33:19.667 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.667 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.667 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:33:19.668 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:19.054882+0000", 2023-09-23T20:33:19.668 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.017481875000000001 2023-09-23T20:33:19.668 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.668 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.668 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:33:19.668 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:19.069204+0000", 2023-09-23T20:33:19.668 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.014321835999999999 2023-09-23T20:33:19.668 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.668 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.668 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:33:19.668 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:19.069272+0000", 2023-09-23T20:33:19.668 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.8029999999999997e-05 2023-09-23T20:33:19.669 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.669 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.669 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:33:19.669 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:19.069364+0000", 2023-09-23T20:33:19.669 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.2187000000000004e-05 2023-09-23T20:33:19.669 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:33:19.669 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:33:19.669 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:33:19.669 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.669 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.669 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35700.0:88933 7.d 7:b98365fe:::benchmark_data_smithi018_308601_object88932:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e329)", 2023-09-23T20:33:19.669 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:33:19.047984+0000", 2023-09-23T20:33:19.670 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 0.47477538600000002, 2023-09-23T20:33:19.670 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.030297010999999999, 2023-09-23T20:33:19.670 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:33:19.670 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:33:19.670 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:33:19.670 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35700", 2023-09-23T20:33:19.670 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:33:19.670 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 88933 2023-09-23T20:33:19.670 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.670 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:33:19.670 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.671 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:33:19.671 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:19.047984+0000", 2023-09-23T20:33:19.671 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:33:19.671 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.671 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.671 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:33:19.671 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:19.047984+0000", 2023-09-23T20:33:19.671 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:33:19.671 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.671 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.671 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:33:19.671 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:19.047984+0000", 2023-09-23T20:33:19.671 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:33:19.672 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.672 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.672 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:33:19.672 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:19.048027+0000", 2023-09-23T20:33:19.672 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.3408999999999997e-05 2023-09-23T20:33:19.672 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.672 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.672 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:33:19.672 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:19.048034+0000", 2023-09-23T20:33:19.672 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.8569999999999997e-06 2023-09-23T20:33:19.672 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.672 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.673 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:33:19.673 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:19.048055+0000", 2023-09-23T20:33:19.673 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.0890000000000002e-05 2023-09-23T20:33:19.673 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.673 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.673 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:33:19.673 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:19.051519+0000", 2023-09-23T20:33:19.673 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0034637940000000001 2023-09-23T20:33:19.673 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.673 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.673 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:33:19.673 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:19.051691+0000", 2023-09-23T20:33:19.674 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00017267500000000001 2023-09-23T20:33:19.674 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.674 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.674 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,1", 2023-09-23T20:33:19.674 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:19.051843+0000", 2023-09-23T20:33:19.674 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000151349 2023-09-23T20:33:19.674 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.674 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.674 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:33:19.674 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:19.055090+0000", 2023-09-23T20:33:19.674 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0032476369999999998 2023-09-23T20:33:19.674 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.675 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.675 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:33:19.675 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:19.055645+0000", 2023-09-23T20:33:19.675 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00055461400000000004 2023-09-23T20:33:19.675 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.675 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.675 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:33:19.675 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:19.078163+0000", 2023-09-23T20:33:19.675 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.022517951000000001 2023-09-23T20:33:19.675 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.675 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.675 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:33:19.676 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:19.078231+0000", 2023-09-23T20:33:19.676 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.8112000000000001e-05 2023-09-23T20:33:19.676 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.676 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.676 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:33:19.676 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:19.078281+0000", 2023-09-23T20:33:19.676 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.0210000000000002e-05 2023-09-23T20:33:19.676 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:33:19.676 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:33:19.676 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:33:19.676 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.677 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.677 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35700.0:89030 7.1 7:80aafdac:::benchmark_data_smithi018_308601_object89029:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e329)", 2023-09-23T20:33:19.677 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:33:19.175253+0000", 2023-09-23T20:33:19.677 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 0.34750655800000002, 2023-09-23T20:33:19.677 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.038369660999999999, 2023-09-23T20:33:19.677 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:33:19.677 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:33:19.677 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:33:19.677 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35700", 2023-09-23T20:33:19.677 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:33:19.677 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 89030 2023-09-23T20:33:19.677 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.678 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:33:19.678 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.678 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:33:19.678 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:19.175253+0000", 2023-09-23T20:33:19.678 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:33:19.678 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.678 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.678 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:33:19.678 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:19.175253+0000", 2023-09-23T20:33:19.678 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:33:19.678 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.678 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.679 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:33:19.679 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:19.175252+0000", 2023-09-23T20:33:19.679 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:33:19.679 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.679 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.679 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:33:19.679 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:19.175284+0000", 2023-09-23T20:33:19.679 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.1953999999999999e-05 2023-09-23T20:33:19.679 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.679 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.679 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:33:19.679 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:19.175296+0000", 2023-09-23T20:33:19.680 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1782e-05 2023-09-23T20:33:19.680 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.680 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.680 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:33:19.680 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:19.175301+0000", 2023-09-23T20:33:19.680 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.7769999999999999e-06 2023-09-23T20:33:19.680 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.680 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.680 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:33:19.680 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:19.175360+0000", 2023-09-23T20:33:19.680 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.8943999999999997e-05 2023-09-23T20:33:19.680 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.681 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.681 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:33:19.681 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:19.175506+0000", 2023-09-23T20:33:19.681 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00014551500000000001 2023-09-23T20:33:19.681 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.681 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.681 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:33:19.681 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:19.175657+0000", 2023-09-23T20:33:19.681 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00015074899999999999 2023-09-23T20:33:19.681 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.681 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.682 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:33:19.682 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:19.178335+0000", 2023-09-23T20:33:19.682 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0026783129999999999 2023-09-23T20:33:19.682 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.682 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.682 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:33:19.682 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:19.185708+0000", 2023-09-23T20:33:19.682 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.007372539 2023-09-23T20:33:19.682 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.682 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.682 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:33:19.683 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:19.213509+0000", 2023-09-23T20:33:19.683 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.027801148000000001 2023-09-23T20:33:19.683 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.683 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.683 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:33:19.683 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:19.213554+0000", 2023-09-23T20:33:19.683 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.5389000000000002e-05 2023-09-23T20:33:19.683 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:19.683 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:19.683 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:33:19.683 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:19.213623+0000", 2023-09-23T20:33:19.683 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.8539000000000002e-05 2023-09-23T20:33:19.683 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:33:19.684 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:33:19.684 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:33:19.684 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:33:19.684 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:33:19.684 INFO:teuthology.orchestra.run.smithi149.stdout:} 2023-09-23T20:33:19.684 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:19.694 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:33:19.712 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:19.713 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:33:19.713 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:19 smithi018 ceph-mon[179647]: pgmap v760: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 48 MiB/s wr, 762 op/s; 5117/259797 objects degraded (1.970%); 2806/259797 objects misplaced (1.080%); 2.5 MiB/s, 40 objects/s recovering 2023-09-23T20:33:19.713 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:19 smithi018 ceph-mon[179647]: failed to encode map e329 with expected crc 2023-09-23T20:33:19.713 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:19 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "21"}]': finished 2023-09-23T20:33:19.713 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:19 smithi018 ceph-mon[179647]: osdmap e329: 8 total, 8 up, 8 in 2023-09-23T20:33:19.714 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:19 smithi018 ceph-mon[184144]: pgmap v760: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 48 MiB/s wr, 762 op/s; 5117/259797 objects degraded (1.970%); 2806/259797 objects misplaced (1.080%); 2.5 MiB/s, 40 objects/s recovering 2023-09-23T20:33:19.714 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:19 smithi018 ceph-mon[184144]: failed to encode map e329 with expected crc 2023-09-23T20:33:19.714 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:19 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "21"}]': finished 2023-09-23T20:33:19.714 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:19 smithi018 ceph-mon[184144]: osdmap e329: 8 total, 8 up, 8 in 2023-09-23T20:33:19.814 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:33:19.822 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 83 16 89563 89547 67.3809 50.5 0.0207976 0.0148357 2023-09-23T20:33:19.879 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:19.924 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T20:33:19.953 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:33:19.962 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:19.963 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:33:20.063 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:33:20.101 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:20.170 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:33:20.178 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:20.179 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:33:20.280 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:33:20.334 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T20:33:20.369 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:33:20.372 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:20.457 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:33:20.461 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:20.461 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:33:20.562 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:33:20.581 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:33:20.605 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:20.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:20 smithi018 ceph-mon[179647]: pgmap v762: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 5.3 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 41 MiB/s wr, 660 op/s; 5117/261189 objects degraded (1.959%); 2806/261189 objects misplaced (1.074%); 3.2 MiB/s, 50 objects/s recovering 2023-09-23T20:33:20.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:20 smithi018 ceph-mon[184144]: pgmap v762: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 5.3 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 41 MiB/s wr, 660 op/s; 5117/261189 objects degraded (1.959%); 2806/261189 objects misplaced (1.074%); 3.2 MiB/s, 50 objects/s recovering 2023-09-23T20:33:20.822 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 84 16 90285 90269 67.1161 45.125 0.0239092 0.0148935 2023-09-23T20:33:20.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:20 smithi149 ceph-mon[170394]: pgmap v762: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 5.3 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 41 MiB/s wr, 660 op/s; 5117/261189 objects degraded (1.959%); 2806/261189 objects misplaced (1.074%); 3.2 MiB/s, 50 objects/s recovering 2023-09-23T20:33:21.182 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:33:21.390 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:33:21.744 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:21 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 5117/261189 objects degraded (1.959%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T20:33:21.747 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:21 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 5117/261189 objects degraded (1.959%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T20:33:21.795 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:33:21.797 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:21 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 5117/261189 objects degraded (1.959%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T20:33:21.798 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:21.798 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:33:21.826 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 85 16 91037 91021 66.8767 47 0.0131762 0.0149478 2023-09-23T20:33:21.899 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:33:22.005 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:22.083 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:33:22.093 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:22.094 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:33:22.195 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:33:22.296 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:22.549 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:33:22.555 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:22.555 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:33:22.656 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:33:22.751 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:22 smithi018 ceph-mon[179647]: pgmap v763: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 52 MiB/s wr, 824 op/s; 5117/268005 objects degraded (1.909%); 2806/268005 objects misplaced (1.047%); 1.6 MiB/s, 25 objects/s recovering 2023-09-23T20:33:22.752 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:22 smithi018 ceph-mon[184144]: pgmap v763: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 52 MiB/s wr, 824 op/s; 5117/268005 objects degraded (1.909%); 2806/268005 objects misplaced (1.047%); 1.6 MiB/s, 25 objects/s recovering 2023-09-23T20:33:22.761 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:22.814 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:22 smithi149 ceph-mon[170394]: pgmap v763: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 52 MiB/s wr, 824 op/s; 5117/268005 objects degraded (1.909%); 2806/268005 objects misplaced (1.047%); 1.6 MiB/s, 25 objects/s recovering 2023-09-23T20:33:22.830 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 86 16 91778 91762 66.6349 46.3125 0.0311067 0.0150013 2023-09-23T20:33:22.932 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:33:22.937 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:22.938 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:33:23.038 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:33:23.120 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:23.266 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:33:23.277 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:23.278 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:33:23.379 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:33:23.417 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:23.467 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:33:23.475 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:23.475 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:33:23.560 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:33:23.567 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:23 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/4180923736' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:33:23.573 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:23 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/4180923736' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:33:23.576 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:33:23.609 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:23.612 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:33:23.661 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:33:23.672 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:33:23.678 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:23 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/4180923736' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:33:23.679 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:23.680 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:33:23.697 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:33:23.753 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:33:23.776 INFO:teuthology.orchestra.run.smithi149.stdout:{ 2023-09-23T20:33:23.776 INFO:teuthology.orchestra.run.smithi149.stdout: "ops": [ 2023-09-23T20:33:23.776 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:23.776 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_repop(client.35700.0:92411 7.8 e329/313 7:161637f9:::benchmark_data_smithi018_308601_object92410:head v 329'5781, mlcod=309'1103)", 2023-09-23T20:33:23.777 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:33:23.772126+0000", 2023-09-23T20:33:23.777 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 0.0035398029999999998, 2023-09-23T20:33:23.777 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0035686530000000002, 2023-09-23T20:33:23.777 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:33:23.777 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:33:23.777 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:33:23.777 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:23.777 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:33:23.777 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:23.772126+0000", 2023-09-23T20:33:23.777 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:33:23.777 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:23.778 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:23.778 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:33:23.778 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:23.772126+0000", 2023-09-23T20:33:23.778 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:33:23.778 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:23.778 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:23.778 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:33:23.778 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:23.772125+0000", 2023-09-23T20:33:23.778 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:33:23.778 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:23.778 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:23.778 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:33:23.779 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:23.772180+0000", 2023-09-23T20:33:23.779 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.4880999999999999e-05 2023-09-23T20:33:23.779 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:23.779 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:23.779 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:33:23.779 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:23.772198+0000", 2023-09-23T20:33:23.779 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.8122999999999999e-05 2023-09-23T20:33:23.779 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:23.779 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:23.779 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:33:23.779 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:23.772205+0000", 2023-09-23T20:33:23.779 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.0759999999999997e-06 2023-09-23T20:33:23.780 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:23.780 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:23.780 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:33:23.780 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:23.772251+0000", 2023-09-23T20:33:23.780 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.5677999999999999e-05 2023-09-23T20:33:23.780 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:23.780 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:23.780 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:33:23.780 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:23.772276+0000", 2023-09-23T20:33:23.780 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.5772999999999999e-05 2023-09-23T20:33:23.780 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:33:23.780 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:33:23.780 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:33:23.781 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:33:23.775682+0000", 2023-09-23T20:33:23.781 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0034051580000000001 2023-09-23T20:33:23.781 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:33:23.781 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:33:23.781 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:33:23.781 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:33:23.781 INFO:teuthology.orchestra.run.smithi149.stdout: ], 2023-09-23T20:33:23.781 INFO:teuthology.orchestra.run.smithi149.stdout: "num_ops": 1 2023-09-23T20:33:23.781 INFO:teuthology.orchestra.run.smithi149.stdout:} 2023-09-23T20:33:23.781 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:33:23.793 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:33:23.828 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:23.828 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:33:23.830 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 87 16 92460 92444 66.3591 42.625 0.023114 0.015064 2023-09-23T20:33:23.870 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:33:23.872 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:33:23.873 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:33:23.873 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:33:23.873 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:33:23.873 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:33:23.873 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:33:23.873 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:33:23.873 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:33:23.885 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:33:23.890 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:23.891 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:33:23.992 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:33:24.064 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:24.076 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 5, 7, 3, 2, 6, 4] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T20:33:24.076 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:33:24.076 INFO:tasks.thrashosds.thrasher:Growing pool .mgr 2023-09-23T20:33:24.076 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:33:24.181 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T20:33:24.275 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:33:24.297 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:24.298 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:33:24.399 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:33:24.458 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:24.528 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:33:24.534 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:24.535 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:33:24.587 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:24 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/4088704699' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:33:24.587 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:24 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:33:24.588 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:24 smithi018 ceph-mon[184144]: pgmap v764: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 36 MiB/s wr, 571 op/s; 5117/270852 objects degraded (1.889%); 2587/270852 objects misplaced (0.955%); 3.3 MiB/s, 52 objects/s recovering 2023-09-23T20:33:24.590 INFO:teuthology.orchestra.run.smithi018.stderr:nodeep-scrub is set 2023-09-23T20:33:24.590 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:24 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/4088704699' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:33:24.590 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:24 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:33:24.591 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:24 smithi018 ceph-mon[179647]: pgmap v764: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 36 MiB/s wr, 571 op/s; 5117/270852 objects degraded (1.889%); 2587/270852 objects misplaced (0.955%); 3.3 MiB/s, 52 objects/s recovering 2023-09-23T20:33:24.635 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:33:24.682 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:24.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:24 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/4088704699' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:33:24.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:24 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:33:24.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:24 smithi149 ceph-mon[170394]: pgmap v764: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 5.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 36 MiB/s wr, 571 op/s; 5117/270852 objects degraded (1.889%); 2587/270852 objects misplaced (0.955%); 3.3 MiB/s, 52 objects/s recovering 2023-09-23T20:33:24.731 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:33:24.736 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:24.737 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:33:24.793 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:33:24.830 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 88 16 93161 93145 66.103 43.8125 0.014446 0.0151223 2023-09-23T20:33:24.838 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:33:24.906 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:33:24.930 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:25.320 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:33:25.327 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:33:24 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:33:24] "GET /metrics HTTP/1.1" 200 36552 "" "Prometheus/2.43.0" 2023-09-23T20:33:25.328 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:25.328 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:33:25.429 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:33:25.477 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:25.604 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:25 smithi018 ceph-mon[184144]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:33:25.605 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:25 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:33:25.605 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:25 smithi018 ceph-mon[184144]: osdmap e330: 8 total, 8 up, 8 in 2023-09-23T20:33:25.605 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:25 smithi018 ceph-mon[184144]: failed to encode map e330 with expected crc 2023-09-23T20:33:25.605 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:25 smithi018 ceph-mon[184144]: failed to encode map e330 with expected crc 2023-09-23T20:33:25.605 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:25 smithi018 ceph-mon[179647]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:33:25.605 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:25 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:33:25.605 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:25 smithi018 ceph-mon[179647]: osdmap e330: 8 total, 8 up, 8 in 2023-09-23T20:33:25.605 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:25 smithi018 ceph-mon[179647]: failed to encode map e330 with expected crc 2023-09-23T20:33:25.605 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:25 smithi018 ceph-mon[179647]: failed to encode map e330 with expected crc 2023-09-23T20:33:25.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:25 smithi149 ceph-mon[170394]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:33:25.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:25 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:33:25.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:25 smithi149 ceph-mon[170394]: osdmap e330: 8 total, 8 up, 8 in 2023-09-23T20:33:25.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:25 smithi149 ceph-mon[170394]: failed to encode map e330 with expected crc 2023-09-23T20:33:25.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:25 smithi149 ceph-mon[170394]: failed to encode map e330 with expected crc 2023-09-23T20:33:25.744 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:33:25.748 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:25.749 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:33:25.830 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 89 16 93895 93879 65.8758 45.875 0.013824 0.0151746 2023-09-23T20:33:25.850 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:33:25.948 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:26.033 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:33:26.042 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:26.043 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:33:26.144 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:33:26.256 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:26.352 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:33:26.361 INFO:teuthology.orchestra.run.smithi149.stdout:ERROR: (22) Invalid argument 2023-09-23T20:33:26.361 INFO:teuthology.orchestra.run.smithi149.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:33:26.363 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:33:26.368 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:26.369 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:33:26.470 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:33:26.611 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:26.762 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:33:26.767 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:26.768 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:33:26.768 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:26 smithi018 ceph-mon[179647]: pgmap v766: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 5.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 50 MiB/s wr, 803 op/s; 5117/279096 objects degraded (1.833%); 2587/279096 objects misplaced (0.927%); 1.7 MiB/s, 27 objects/s recovering 2023-09-23T20:33:26.769 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:26 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 5117/270852 objects degraded (1.889%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T20:33:26.769 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:26 smithi018 ceph-mon[184144]: pgmap v766: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 5.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 50 MiB/s wr, 803 op/s; 5117/279096 objects degraded (1.833%); 2587/279096 objects misplaced (0.927%); 1.7 MiB/s, 27 objects/s recovering 2023-09-23T20:33:26.769 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:26 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 5117/270852 objects degraded (1.889%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T20:33:26.820 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T20:33:26.830 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Total time run: 90.0102 2023-09-23T20:33:26.830 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Total writes made: 94549 2023-09-23T20:33:26.830 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Write size: 65536 2023-09-23T20:33:26.830 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Object size: 65536 2023-09-23T20:33:26.830 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Bandwidth (MB/sec): 65.6516 2023-09-23T20:33:26.830 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Stddev Bandwidth: 36.5227 2023-09-23T20:33:26.831 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Max bandwidth (MB/sec): 184 2023-09-23T20:33:26.831 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Min bandwidth (MB/sec): 0 2023-09-23T20:33:26.831 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Average IOPS: 1050 2023-09-23T20:33:26.831 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Stddev IOPS: 584.373 2023-09-23T20:33:26.831 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Max IOPS: 2944 2023-09-23T20:33:26.831 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Min IOPS: 0 2023-09-23T20:33:26.831 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Average Latency(s): 0.0152282 2023-09-23T20:33:26.831 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Stddev Latency(s): 0.0431639 2023-09-23T20:33:26.831 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Max latency(s): 2.45012 2023-09-23T20:33:26.831 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Min latency(s): 0.00108768 2023-09-23T20:33:26.835 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Cleaning up (deleting benchmark objects) 2023-09-23T20:33:26.869 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:33:26.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:26 smithi149 ceph-mon[170394]: pgmap v766: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 5.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 50 MiB/s wr, 803 op/s; 5117/279096 objects degraded (1.833%); 2587/279096 objects misplaced (0.927%); 1.7 MiB/s, 27 objects/s recovering 2023-09-23T20:33:26.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:26 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 5117/270852 objects degraded (1.889%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T20:33:26.966 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:27.030 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:33:27.045 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:27.045 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:33:27.146 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:33:27.177 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:27.215 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:33:27.225 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:27.225 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:33:27.232 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:33:27.242 INFO:teuthology.orchestra.run.smithi018.stderr:dumped all 2023-09-23T20:33:27.296 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:33:27.326 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:33:27.360 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:27.413 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:33:27.417 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:27.418 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:33:27.518 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:33:27.539 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:33:27.622 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:27.675 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T20:33:27.686 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:27 smithi018 ceph-mon[179647]: from='client.45644 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:33:27.688 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:27 smithi018 ceph-mon[184144]: from='client.45644 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:33:27.773 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:33:27.779 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:27.780 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:33:27.815 INFO:teuthology.orchestra.run.smithi018.stdout:{"pg_ready":true,"pg_map":{"version":766,"stamp":"2023-09-23T20:33:25.787500+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":6099699776,"num_objects":93032,"num_object_clones":0,"num_object_copies":279096,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5117,"num_objects_misplaced":2587,"num_objects_unfound":0,"num_objects_dirty":93032,"num_whiteouts":0,"num_read":6009,"num_read_kb":13239,"num_write":93918,"num_write_kb":5965054,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2268,"num_bytes_recovered":145943366,"num_keys_recovered":40,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":97416,"ondisk_log_size":97416,"up":327,"acting":323,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":335,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":20649564,"kb_used_data":18113116,"kb_used_omap":32,"kb_used_meta":2536351,"kb_avail":729311652,"statfs":{"total":767960285184,"available":746815131648,"internally_reserved":0,"allocated":18547830784,"data_stored":18537033114,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32773,"internal_metadata":2597224443},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,3,1,5,5,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":10,"apply_latency_ms":10,"commit_latency_ns":10000000,"apply_latency_ns":10000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":421593088,"num_objects":6433,"num_object_clones":0,"num_object_copies":19299,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-219,"num_objects_unfound":0,"num_objects_dirty":6433,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6433,"num_write_kb":411712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":219,"num_bytes_recovered":14352384,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.004770"},"pg_stats":[{"pgid":"3.1a","version":"69'1","reported_seq":27,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T20:33:18.476495+0000","last_change":"2023-09-23T20:32:33.233853+0000","last_active":"2023-09-23T20:33:18.476495+0000","last_peered":"2023-09-23T20:33:18.476495+0000","last_clean":"2023-09-23T20:33:18.476495+0000","last_became_active":"2023-09-23T20:32:33.233546+0000","last_became_peered":"2023-09-23T20:32:33.233546+0000","last_unstale":"2023-09-23T20:33:18.476495+0000","last_undegraded":"2023-09-23T20:33:18.476495+0000","last_fullsized":"2023-09-23T20:33:18.476495+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:50:27.130808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":207,"reported_epoch":328,"state":"active+clean","last_fresh":"2023-09-23T20:33:17.469390+0000","last_change":"2023-09-23T20:32:50.536495+0000","last_active":"2023-09-23T20:33:17.469390+0000","last_peered":"2023-09-23T20:33:17.469390+0000","last_clean":"2023-09-23T20:33:17.469390+0000","last_became_active":"2023-09-23T20:32:50.536180+0000","last_became_peered":"2023-09-23T20:32:50.536180+0000","last_unstale":"2023-09-23T20:33:17.469390+0000","last_undegraded":"2023-09-23T20:33:17.469390+0000","last_fullsized":"2023-09-23T20:33:17.469390+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:01:05.524308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":285,"reported_epoch":328,"state":"active+clean","last_fresh":"2023-09-23T20:33:17.468624+0000","last_change":"2023-09-23T20:33:00.181677+0000","last_active":"2023-09-23T20:33:17.468624+0000","last_peered":"2023-09-23T20:33:17.468624+0000","last_clean":"2023-09-23T20:33:17.468624+0000","last_became_active":"2023-09-23T20:33:00.181400+0000","last_became_peered":"2023-09-23T20:33:00.181400+0000","last_unstale":"2023-09-23T20:33:17.468624+0000","last_undegraded":"2023-09-23T20:33:17.468624+0000","last_fullsized":"2023-09-23T20:33:17.468624+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:52:14.600448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":296,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T20:33:18.497525+0000","last_change":"2023-09-23T20:25:02.957185+0000","last_active":"2023-09-23T20:33:18.497525+0000","last_peered":"2023-09-23T20:33:18.497525+0000","last_clean":"2023-09-23T20:33:18.497525+0000","last_became_active":"2023-09-23T20:25:02.956689+0000","last_became_peered":"2023-09-23T20:25:02.956689+0000","last_unstale":"2023-09-23T20:33:18.497525+0000","last_undegraded":"2023-09-23T20:33:18.497525+0000","last_fullsized":"2023-09-23T20:33:18.497525+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:04:28.524015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":141,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T20:33:18.483858+0000","last_change":"2023-09-23T20:32:50.546428+0000","last_active":"2023-09-23T20:33:18.483858+0000","last_peered":"2023-09-23T20:33:18.483858+0000","last_clean":"2023-09-23T20:33:18.483858+0000","last_became_active":"2023-09-23T20:32:50.546120+0000","last_became_peered":"2023-09-23T20:32:50.546120+0000","last_unstale":"2023-09-23T20:33:18.483858+0000","last_undegraded":"2023-09-23T20:33:18.483858+0000","last_fullsized":"2023-09-23T20:33:18.483858+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:00:10.792410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":103,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T20:33:18.479378+0000","last_change":"2023-09-23T20:29:46.112977+0000","last_active":"2023-09-23T20:33:18.479378+0000","last_peered":"2023-09-23T20:33:18.479378+0000","last_clean":"2023-09-23T20:33:18.479378+0000","last_became_active":"2023-09-23T20:29:46.112600+0000","last_became_peered":"2023-09-23T20:29:46.112600+0000","last_unstale":"2023-09-23T20:33:18.479378+0000","last_undegraded":"2023-09-23T20:33:18.479378+0000","last_fullsized":"2023-09-23T20:33:18.479378+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:06:34.132247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":294,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T20:33:18.484224+0000","last_change":"2023-09-23T20:25:02.969432+0000","last_active":"2023-09-23T20:33:18.484224+0000","last_peered":"2023-09-23T20:33:18.484224+0000","last_clean":"2023-09-23T20:33:18.484224+0000","last_became_active":"2023-09-23T20:25:02.969118+0000","last_became_peered":"2023-09-23T20:25:02.969118+0000","last_unstale":"2023-09-23T20:33:18.484224+0000","last_undegraded":"2023-09-23T20:33:18.484224+0000","last_fullsized":"2023-09-23T20:33:18.484224+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:02:22.453831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":31,"reported_epoch":328,"state":"active+clean","last_fresh":"2023-09-23T20:33:17.500466+0000","last_change":"2023-09-23T20:32:01.360359+0000","last_active":"2023-09-23T20:33:17.500466+0000","last_peered":"2023-09-23T20:33:17.500466+0000","last_clean":"2023-09-23T20:33:17.500466+0000","last_became_active":"2023-09-23T20:32:01.360008+0000","last_became_peered":"2023-09-23T20:32:01.360008+0000","last_unstale":"2023-09-23T20:33:17.500466+0000","last_undegraded":"2023-09-23T20:33:17.500466+0000","last_fullsized":"2023-09-23T20:33:17.500466+0000","mapping_epoch":308,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:33:43.135001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.14","version":"90'161","reported_seq":638,"reported_epoch":328,"state":"active+clean","last_fresh":"2023-09-23T20:33:17.504296+0000","last_change":"2023-09-23T20:33:00.177495+0000","last_active":"2023-09-23T20:33:17.504296+0000","last_peered":"2023-09-23T20:33:17.504296+0000","last_clean":"2023-09-23T20:33:17.504296+0000","last_became_active":"2023-09-23T20:33:00.177086+0000","last_became_peered":"2023-09-23T20:33:00.177086+0000","last_unstale":"2023-09-23T20:33:17.504296+0000","last_undegraded":"2023-09-23T20:33:17.504296+0000","last_fullsized":"2023-09-23T20:33:17.504296+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:47.479012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:47.479012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:09:38.114270+0000","scrub_duration":0.0020136590000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":306,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T20:33:18.497296+0000","last_change":"2023-09-23T20:29:45.523808+0000","last_active":"2023-09-23T20:33:18.497296+0000","last_peered":"2023-09-23T20:33:18.497296+0000","last_clean":"2023-09-23T20:33:18.497296+0000","last_became_active":"2023-09-23T20:29:45.523508+0000","last_became_peered":"2023-09-23T20:29:45.523508+0000","last_unstale":"2023-09-23T20:33:18.497296+0000","last_undegraded":"2023-09-23T20:33:18.497296+0000","last_fullsized":"2023-09-23T20:33:18.497296+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:50:55.565352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":129,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T20:33:18.492390+0000","last_change":"2023-09-23T20:32:50.539003+0000","last_active":"2023-09-23T20:33:18.492390+0000","last_peered":"2023-09-23T20:33:18.492390+0000","last_clean":"2023-09-23T20:33:18.492390+0000","last_became_active":"2023-09-23T20:32:50.538596+0000","last_became_peered":"2023-09-23T20:32:50.538596+0000","last_unstale":"2023-09-23T20:33:18.492390+0000","last_undegraded":"2023-09-23T20:33:18.492390+0000","last_fullsized":"2023-09-23T20:33:18.492390+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:56:03.314375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":357,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T20:33:18.490741+0000","last_change":"2023-09-23T20:32:33.270617+0000","last_active":"2023-09-23T20:33:18.490741+0000","last_peered":"2023-09-23T20:33:18.490741+0000","last_clean":"2023-09-23T20:33:18.490741+0000","last_became_active":"2023-09-23T20:32:33.227902+0000","last_became_peered":"2023-09-23T20:32:33.227902+0000","last_unstale":"2023-09-23T20:33:18.490741+0000","last_undegraded":"2023-09-23T20:33:18.490741+0000","last_fullsized":"2023-09-23T20:33:18.490741+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:30:26.847831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":360,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":277,"reported_epoch":328,"state":"active+clean","last_fresh":"2023-09-23T20:33:17.469193+0000","last_change":"2023-09-23T20:29:46.080766+0000","last_active":"2023-09-23T20:33:17.469193+0000","last_peered":"2023-09-23T20:33:17.469193+0000","last_clean":"2023-09-23T20:33:17.469193+0000","last_became_active":"2023-09-23T20:29:46.080356+0000","last_became_peered":"2023-09-23T20:29:46.080356+0000","last_unstale":"2023-09-23T20:33:17.469193+0000","last_undegraded":"2023-09-23T20:33:17.469193+0000","last_fullsized":"2023-09-23T20:33:17.469193+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:08:01.862270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":37,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T20:33:18.476847+0000","last_change":"2023-09-23T20:32:33.266787+0000","last_active":"2023-09-23T20:33:18.476847+0000","last_peered":"2023-09-23T20:33:18.476847+0000","last_clean":"2023-09-23T20:33:18.476847+0000","last_became_active":"2023-09-23T20:32:33.228482+0000","last_became_peered":"2023-09-23T20:32:33.228482+0000","last_unstale":"2023-09-23T20:33:18.476847+0000","last_undegraded":"2023-09-23T20:33:18.476847+0000","last_fullsized":"2023-09-23T20:33:18.476847+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:13:01.812073+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":27,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T20:33:18.476954+0000","last_change":"2023-09-23T20:32:33.228929+0000","last_active":"2023-09-23T20:33:18.476954+0000","last_peered":"2023-09-23T20:33:18.476954+0000","last_clean":"2023-09-23T20:33:18.476954+0000","last_became_active":"2023-09-23T20:32:33.228595+0000","last_became_peered":"2023-09-23T20:32:33.228595+0000","last_unstale":"2023-09-23T20:33:18.476954+0000","last_undegraded":"2023-09-23T20:33:18.476954+0000","last_fullsized":"2023-09-23T20:33:18.476954+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:17:53.053808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"90'161","reported_seq":559,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T20:33:18.486013+0000","last_change":"2023-09-23T20:31:43.578810+0000","last_active":"2023-09-23T20:33:18.486013+0000","last_peered":"2023-09-23T20:33:18.486013+0000","last_clean":"2023-09-23T20:33:18.486013+0000","last_became_active":"2023-09-23T20:28:05.991304+0000","last_became_peered":"2023-09-23T20:28:05.991304+0000","last_unstale":"2023-09-23T20:33:18.486013+0000","last_undegraded":"2023-09-23T20:33:18.486013+0000","last_fullsized":"2023-09-23T20:33:18.486013+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:43.578666+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:43.578666+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:46:18.999991+0000","scrub_duration":0.0021890379999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":356,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T20:33:18.489779+0000","last_change":"2023-09-23T20:32:33.242943+0000","last_active":"2023-09-23T20:33:18.489779+0000","last_peered":"2023-09-23T20:33:18.489779+0000","last_clean":"2023-09-23T20:33:18.489779+0000","last_became_active":"2023-09-23T20:32:33.225468+0000","last_became_peered":"2023-09-23T20:32:33.225468+0000","last_unstale":"2023-09-23T20:33:18.489779+0000","last_undegraded":"2023-09-23T20:33:18.489779+0000","last_fullsized":"2023-09-23T20:33:18.489779+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:40:47.867573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":31,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T20:33:18.495391+0000","last_change":"2023-09-23T20:32:01.358590+0000","last_active":"2023-09-23T20:33:18.495391+0000","last_peered":"2023-09-23T20:33:18.495391+0000","last_clean":"2023-09-23T20:33:18.495391+0000","last_became_active":"2023-09-23T20:32:01.358345+0000","last_became_peered":"2023-09-23T20:32:01.358345+0000","last_unstale":"2023-09-23T20:33:18.495391+0000","last_undegraded":"2023-09-23T20:33:18.495391+0000","last_fullsized":"2023-09-23T20:33:18.495391+0000","mapping_epoch":308,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:22:21.159708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.10","version":"90'161","reported_seq":560,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T20:33:18.495199+0000","last_change":"2023-09-23T20:31:49.726214+0000","last_active":"2023-09-23T20:33:18.495199+0000","last_peered":"2023-09-23T20:33:18.495199+0000","last_clean":"2023-09-23T20:33:18.495199+0000","last_became_active":"2023-09-23T20:29:07.044892+0000","last_became_peered":"2023-09-23T20:29:07.044892+0000","last_unstale":"2023-09-23T20:33:18.495199+0000","last_undegraded":"2023-09-23T20:33:18.495199+0000","last_fullsized":"2023-09-23T20:33:18.495199+0000","mapping_epoch":253,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":254,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:49.726139+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:49.726139+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:59:11.810516+0000","scrub_duration":0.0015441470000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":294,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T20:33:18.486278+0000","last_change":"2023-09-23T20:25:02.953993+0000","last_active":"2023-09-23T20:33:18.486278+0000","last_peered":"2023-09-23T20:33:18.486278+0000","last_clean":"2023-09-23T20:33:18.486278+0000","last_became_active":"2023-09-23T20:25:02.953672+0000","last_became_peered":"2023-09-23T20:25:02.953672+0000","last_unstale":"2023-09-23T20:33:18.486278+0000","last_undegraded":"2023-09-23T20:33:18.486278+0000","last_fullsized":"2023-09-23T20:33:18.486278+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:26:08.603577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":308,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T20:33:18.496748+0000","last_change":"2023-09-23T20:25:02.949584+0000","last_active":"2023-09-23T20:33:18.496748+0000","last_peered":"2023-09-23T20:33:18.496748+0000","last_clean":"2023-09-23T20:33:18.496748+0000","last_became_active":"2023-09-23T20:25:02.949132+0000","last_became_peered":"2023-09-23T20:25:02.949132+0000","last_unstale":"2023-09-23T20:33:18.496748+0000","last_undegraded":"2023-09-23T20:33:18.496748+0000","last_fullsized":"2023-09-23T20:33:18.496748+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:34:20.509823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"90'161","reported_seq":594,"reported_epoch":328,"state":"active+clean","last_fresh":"2023-09-23T20:33:17.470271+0000","last_change":"2023-09-23T20:33:00.179977+0000","last_active":"2023-09-23T20:33:17.470271+0000","last_peered":"2023-09-23T20:33:17.470271+0000","last_clean":"2023-09-23T20:33:17.470271+0000","last_became_active":"2023-09-23T20:33:00.179654+0000","last_became_peered":"2023-09-23T20:33:00.179654+0000","last_unstale":"2023-09-23T20:33:17.470271+0000","last_undegraded":"2023-09-23T20:33:17.470271+0000","last_fullsized":"2023-09-23T20:33:17.470271+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:42.544308+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:42.544308+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:24:16.853678+0000","scrub_duration":0.0029958179999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":313,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T20:33:18.492391+0000","last_change":"2023-09-23T20:32:33.225148+0000","last_active":"2023-09-23T20:33:18.492391+0000","last_peered":"2023-09-23T20:33:18.492391+0000","last_clean":"2023-09-23T20:33:18.492391+0000","last_became_active":"2023-09-23T20:32:33.224805+0000","last_became_peered":"2023-09-23T20:32:33.224805+0000","last_unstale":"2023-09-23T20:33:18.492391+0000","last_undegraded":"2023-09-23T20:33:18.492391+0000","last_fullsized":"2023-09-23T20:33:18.492391+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:55:30.891398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":304,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T20:33:18.497726+0000","last_change":"2023-09-23T20:25:02.957010+0000","last_active":"2023-09-23T20:33:18.497726+0000","last_peered":"2023-09-23T20:33:18.497726+0000","last_clean":"2023-09-23T20:33:18.497726+0000","last_became_active":"2023-09-23T20:25:02.956267+0000","last_became_peered":"2023-09-23T20:25:02.956267+0000","last_unstale":"2023-09-23T20:33:18.497726+0000","last_undegraded":"2023-09-23T20:33:18.497726+0000","last_fullsized":"2023-09-23T20:33:18.497726+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:10:55.299422+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"90'161","reported_seq":579,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T20:33:18.496168+0000","last_change":"2023-09-23T20:31:53.707033+0000","last_active":"2023-09-23T20:33:18.496168+0000","last_peered":"2023-09-23T20:33:18.496168+0000","last_clean":"2023-09-23T20:33:18.496168+0000","last_became_active":"2023-09-23T20:29:46.078873+0000","last_became_peered":"2023-09-23T20:29:46.078873+0000","last_unstale":"2023-09-23T20:33:18.496168+0000","last_undegraded":"2023-09-23T20:33:18.496168+0000","last_fullsized":"2023-09-23T20:33:18.496168+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:53.706934+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:53.706934+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:51:58.039657+0000","scrub_duration":0.0016840900000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":99,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T20:33:18.477912+0000","last_change":"2023-09-23T20:29:04.179171+0000","last_active":"2023-09-23T20:33:18.477912+0000","last_peered":"2023-09-23T20:33:18.477912+0000","last_clean":"2023-09-23T20:33:18.477912+0000","last_became_active":"2023-09-23T20:29:04.128672+0000","last_became_peered":"2023-09-23T20:29:04.128672+0000","last_unstale":"2023-09-23T20:33:18.477912+0000","last_undegraded":"2023-09-23T20:33:18.477912+0000","last_fullsized":"2023-09-23T20:33:18.477912+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:49:15.506106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":142,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T20:33:18.490784+0000","last_change":"2023-09-23T20:32:50.539157+0000","last_active":"2023-09-23T20:33:18.490784+0000","last_peered":"2023-09-23T20:33:18.490784+0000","last_clean":"2023-09-23T20:33:18.490784+0000","last_became_active":"2023-09-23T20:32:50.538889+0000","last_became_peered":"2023-09-23T20:32:50.538889+0000","last_unstale":"2023-09-23T20:33:18.490784+0000","last_undegraded":"2023-09-23T20:33:18.490784+0000","last_fullsized":"2023-09-23T20:33:18.490784+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:28:28.857256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.b","version":"329'5799","reported_seq":5834,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T20:33:23.805657+0000","last_change":"2023-09-23T20:31:49.152343+0000","last_active":"2023-09-23T20:33:23.805657+0000","last_peered":"2023-09-23T20:33:23.805657+0000","last_clean":"2023-09-23T20:33:23.805657+0000","last_became_active":"2023-09-23T20:31:49.152042+0000","last_became_peered":"2023-09-23T20:31:49.152042+0000","last_unstale":"2023-09-23T20:33:23.805657+0000","last_undegraded":"2023-09-23T20:33:23.805657+0000","last_fullsized":"2023-09-23T20:33:23.805657+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":5799,"log_dups_size":0,"ondisk_log_size":5799,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:53:34.237468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":380043264,"num_objects":5799,"num_object_clones":0,"num_object_copies":17397,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5799,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5799,"num_write_kb":371136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.d","version":"90'161","reported_seq":599,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T20:33:18.490494+0000","last_change":"2023-09-23T20:32:36.304873+0000","last_active":"2023-09-23T20:33:18.490494+0000","last_peered":"2023-09-23T20:33:18.490494+0000","last_clean":"2023-09-23T20:33:18.490494+0000","last_became_active":"2023-09-23T20:32:36.304524+0000","last_became_peered":"2023-09-23T20:32:36.304524+0000","last_unstale":"2023-09-23T20:33:18.490494+0000","last_undegraded":"2023-09-23T20:33:18.490494+0000","last_fullsized":"2023-09-23T20:33:18.490494+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:46.388545+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:46.388545+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:29:22.195276+0000","scrub_duration":0.00168356,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":333,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T20:33:18.490666+0000","last_change":"2023-09-23T20:32:33.283752+0000","last_active":"2023-09-23T20:33:18.490666+0000","last_peered":"2023-09-23T20:33:18.490666+0000","last_clean":"2023-09-23T20:33:18.490666+0000","last_became_active":"2023-09-23T20:32:33.230503+0000","last_became_peered":"2023-09-23T20:32:33.230503+0000","last_unstale":"2023-09-23T20:33:18.490666+0000","last_undegraded":"2023-09-23T20:33:18.490666+0000","last_fullsized":"2023-09-23T20:33:18.490666+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:29.084410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":291,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T20:33:18.495967+0000","last_change":"2023-09-23T20:29:04.128604+0000","last_active":"2023-09-23T20:33:18.495967+0000","last_peered":"2023-09-23T20:33:18.495967+0000","last_clean":"2023-09-23T20:33:18.495967+0000","last_became_active":"2023-09-23T20:29:04.128003+0000","last_became_peered":"2023-09-23T20:29:04.128003+0000","last_unstale":"2023-09-23T20:33:18.495967+0000","last_undegraded":"2023-09-23T20:33:18.495967+0000","last_fullsized":"2023-09-23T20:33:18.495967+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:50:49.385599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"329'5747","reported_seq":5789,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T20:33:22.458573+0000","last_change":"2023-09-23T20:32:50.543873+0000","last_active":"2023-09-23T20:33:22.458573+0000","last_peered":"2023-09-23T20:33:22.458573+0000","last_clean":"2023-09-23T20:33:22.458573+0000","last_became_active":"2023-09-23T20:32:50.537253+0000","last_became_peered":"2023-09-23T20:32:50.537253+0000","last_unstale":"2023-09-23T20:33:22.458573+0000","last_undegraded":"2023-09-23T20:33:22.458573+0000","last_fullsized":"2023-09-23T20:33:22.458573+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":5747,"log_dups_size":0,"ondisk_log_size":5747,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:38:02.452291+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":376635392,"num_objects":5747,"num_object_clones":0,"num_object_copies":17241,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5747,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5747,"num_write_kb":367808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,5],"acting":[2,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.c","version":"90'161","reported_seq":490,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T20:33:18.487403+0000","last_change":"2023-09-23T20:32:50.534420+0000","last_active":"2023-09-23T20:33:18.487403+0000","last_peered":"2023-09-23T20:33:18.487403+0000","last_clean":"2023-09-23T20:33:18.487403+0000","last_became_active":"2023-09-23T20:32:50.533984+0000","last_became_peered":"2023-09-23T20:32:50.533984+0000","last_unstale":"2023-09-23T20:33:18.487403+0000","last_undegraded":"2023-09-23T20:33:18.487403+0000","last_fullsized":"2023-09-23T20:33:18.487403+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:41.701171+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:41.701171+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0019955509999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109227,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":354,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T20:33:18.490359+0000","last_change":"2023-09-23T20:33:00.196461+0000","last_active":"2023-09-23T20:33:18.490359+0000","last_peered":"2023-09-23T20:33:18.490359+0000","last_clean":"2023-09-23T20:33:18.490359+0000","last_became_active":"2023-09-23T20:33:00.178428+0000","last_became_peered":"2023-09-23T20:33:00.178428+0000","last_unstale":"2023-09-23T20:33:18.490359+0000","last_undegraded":"2023-09-23T20:33:18.490359+0000","last_fullsized":"2023-09-23T20:33:18.490359+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:48:19.958144+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":27,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T20:33:18.476514+0000","last_change":"2023-09-23T20:32:33.232142+0000","last_active":"2023-09-23T20:33:18.476514+0000","last_peered":"2023-09-23T20:33:18.476514+0000","last_clean":"2023-09-23T20:33:18.476514+0000","last_became_active":"2023-09-23T20:32:33.231683+0000","last_became_peered":"2023-09-23T20:32:33.231683+0000","last_unstale":"2023-09-23T20:33:18.476514+0000","last_undegraded":"2023-09-23T20:33:18.476514+0000","last_fullsized":"2023-09-23T20:33:18.476514+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:02:58.336348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"330'5797","reported_seq":5849,"reported_epoch":330,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:33:25.310142+0000","last_change":"2023-09-23T20:33:00.304857+0000","last_active":"2023-09-23T20:33:25.310142+0000","last_peered":"2023-09-23T20:33:25.310142+0000","last_clean":"2023-09-23T20:32:32.204699+0000","last_became_active":"2023-09-23T20:33:00.297863+0000","last_became_peered":"2023-09-23T20:33:00.297863+0000","last_unstale":"2023-09-23T20:33:25.310142+0000","last_undegraded":"2023-09-23T20:33:00.218246+0000","last_fullsized":"2023-09-23T20:33:00.168368+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":5797,"log_dups_size":0,"ondisk_log_size":5797,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:31.378123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":379912192,"num_objects":5797,"num_object_clones":0,"num_object_copies":17391,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1108,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5797,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5797,"num_write_kb":371008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1],"avail_no_missing":["0","1"],"object_location_counts":[{"shards":"0,1","objects":4689},{"shards":"0,1,4","objects":1106}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"90'161","reported_seq":552,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T20:33:18.479180+0000","last_change":"2023-09-23T20:32:36.302809+0000","last_active":"2023-09-23T20:33:18.479180+0000","last_peered":"2023-09-23T20:33:18.479180+0000","last_clean":"2023-09-23T20:33:18.479180+0000","last_became_active":"2023-09-23T20:32:36.302465+0000","last_became_peered":"2023-09-23T20:32:36.302465+0000","last_unstale":"2023-09-23T20:33:18.479180+0000","last_undegraded":"2023-09-23T20:33:18.479180+0000","last_fullsized":"2023-09-23T20:33:18.479180+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:46.217489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:46.217489+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:35:40.488427+0000","scrub_duration":0.001456429,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":302,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T20:33:18.490578+0000","last_change":"2023-09-23T20:29:04.184893+0000","last_active":"2023-09-23T20:33:18.490578+0000","last_peered":"2023-09-23T20:33:18.490578+0000","last_clean":"2023-09-23T20:33:18.490578+0000","last_became_active":"2023-09-23T20:29:04.152723+0000","last_became_peered":"2023-09-23T20:29:04.152723+0000","last_unstale":"2023-09-23T20:33:18.490578+0000","last_undegraded":"2023-09-23T20:33:18.490578+0000","last_fullsized":"2023-09-23T20:33:18.490578+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:51:27.017151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":103,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T20:33:18.478026+0000","last_change":"2023-09-23T20:29:46.086912+0000","last_active":"2023-09-23T20:33:18.478026+0000","last_peered":"2023-09-23T20:33:18.478026+0000","last_clean":"2023-09-23T20:33:18.478026+0000","last_became_active":"2023-09-23T20:29:46.085471+0000","last_became_peered":"2023-09-23T20:29:46.085471+0000","last_unstale":"2023-09-23T20:33:18.478026+0000","last_undegraded":"2023-09-23T20:33:18.478026+0000","last_fullsized":"2023-09-23T20:33:18.478026+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:04:55.751660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.8","version":"329'5730","reported_seq":7876,"reported_epoch":329,"state":"active+recovering+undersized+remapped","last_fresh":"2023-09-23T20:33:22.507395+0000","last_change":"2023-09-23T20:32:59.221315+0000","last_active":"2023-09-23T20:33:22.507395+0000","last_peered":"2023-09-23T20:33:22.507395+0000","last_clean":"2023-09-23T20:32:32.212521+0000","last_became_active":"2023-09-23T20:32:34.616677+0000","last_became_peered":"2023-09-23T20:32:34.616677+0000","last_unstale":"2023-09-23T20:33:22.507395+0000","last_undegraded":"2023-09-23T20:33:22.507395+0000","last_fullsized":"2023-09-23T20:32:34.212267+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":5730,"log_dups_size":0,"ondisk_log_size":5730,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:24:37.173047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":375521280,"num_objects":5730,"num_object_clones":0,"num_object_copies":17190,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2587,"num_objects_unfound":0,"num_objects_dirty":5730,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5730,"num_write_kb":366720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1051,"num_bytes_recovered":68878336,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7],"avail_no_missing":["2","3","7"],"object_location_counts":[{"shards":"2,3,7","objects":2587},{"shards":"2,7","objects":3143}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.e","version":"90'161","reported_seq":519,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T20:33:18.490205+0000","last_change":"2023-09-23T20:33:00.180075+0000","last_active":"2023-09-23T20:33:18.490205+0000","last_peered":"2023-09-23T20:33:18.490205+0000","last_clean":"2023-09-23T20:33:18.490205+0000","last_became_active":"2023-09-23T20:33:00.179783+0000","last_became_peered":"2023-09-23T20:33:00.179783+0000","last_unstale":"2023-09-23T20:33:18.490205+0000","last_undegraded":"2023-09-23T20:33:18.490205+0000","last_fullsized":"2023-09-23T20:33:18.490205+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:44.386120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:44.386120+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:16:54.770768+0000","scrub_duration":0.0015151330000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":336,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T20:33:18.495597+0000","last_change":"2023-09-23T20:29:04.138284+0000","last_active":"2023-09-23T20:33:18.495597+0000","last_peered":"2023-09-23T20:33:18.495597+0000","last_clean":"2023-09-23T20:33:18.495597+0000","last_became_active":"2023-09-23T20:29:04.137960+0000","last_became_peered":"2023-09-23T20:29:04.137960+0000","last_unstale":"2023-09-23T20:33:18.495597+0000","last_undegraded":"2023-09-23T20:33:18.495597+0000","last_fullsized":"2023-09-23T20:33:18.495597+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:18:59.548311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":312,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T20:33:18.484055+0000","last_change":"2023-09-23T20:32:33.236122+0000","last_active":"2023-09-23T20:33:18.484055+0000","last_peered":"2023-09-23T20:33:18.484055+0000","last_clean":"2023-09-23T20:33:18.484055+0000","last_became_active":"2023-09-23T20:32:33.235384+0000","last_became_peered":"2023-09-23T20:32:33.235384+0000","last_unstale":"2023-09-23T20:33:18.484055+0000","last_undegraded":"2023-09-23T20:33:18.484055+0000","last_fullsized":"2023-09-23T20:33:18.484055+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:56.843733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"330'5906","reported_seq":5942,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-09-23T20:33:25.296150+0000","last_change":"2023-09-23T20:31:49.158461+0000","last_active":"2023-09-23T20:33:25.296150+0000","last_peered":"2023-09-23T20:33:25.296150+0000","last_clean":"2023-09-23T20:33:25.296150+0000","last_became_active":"2023-09-23T20:31:49.158188+0000","last_became_peered":"2023-09-23T20:31:49.158188+0000","last_unstale":"2023-09-23T20:33:25.296150+0000","last_undegraded":"2023-09-23T20:33:25.296150+0000","last_fullsized":"2023-09-23T20:33:25.296150+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":5906,"log_dups_size":0,"ondisk_log_size":5906,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:31:57.678090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":387055616,"num_objects":5906,"num_object_clones":0,"num_object_copies":17718,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5906,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5906,"num_write_kb":377984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":712,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T20:33:18.492281+0000","last_change":"2023-09-23T20:28:05.872149+0000","last_active":"2023-09-23T20:33:18.492281+0000","last_peered":"2023-09-23T20:33:18.492281+0000","last_clean":"2023-09-23T20:33:18.492281+0000","last_became_active":"2023-09-23T20:27:11.961310+0000","last_became_peered":"2023-09-23T20:27:11.961310+0000","last_unstale":"2023-09-23T20:33:18.492281+0000","last_undegraded":"2023-09-23T20:33:18.492281+0000","last_fullsized":"2023-09-23T20:33:18.492281+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:28:05.872074+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:28:05.872074+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:52:32.495955+0000","scrub_duration":0.0029215980000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":64,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T20:33:18.484149+0000","last_change":"2023-09-23T20:32:50.538500+0000","last_active":"2023-09-23T20:33:18.484149+0000","last_peered":"2023-09-23T20:33:18.484149+0000","last_clean":"2023-09-23T20:33:18.484149+0000","last_became_active":"2023-09-23T20:32:50.538152+0000","last_became_peered":"2023-09-23T20:32:50.538152+0000","last_unstale":"2023-09-23T20:33:18.484149+0000","last_undegraded":"2023-09-23T20:33:18.484149+0000","last_fullsized":"2023-09-23T20:33:18.484149+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:00:41.103508+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":27,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T20:33:18.476918+0000","last_change":"2023-09-23T20:32:33.231614+0000","last_active":"2023-09-23T20:33:18.476918+0000","last_peered":"2023-09-23T20:33:18.476918+0000","last_clean":"2023-09-23T20:33:18.476918+0000","last_became_active":"2023-09-23T20:32:33.231207+0000","last_became_peered":"2023-09-23T20:32:33.231207+0000","last_unstale":"2023-09-23T20:33:18.476918+0000","last_undegraded":"2023-09-23T20:33:18.476918+0000","last_fullsized":"2023-09-23T20:33:18.476918+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:46:09.780829+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"329'5729","reported_seq":5764,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T20:33:22.425817+0000","last_change":"2023-09-23T20:31:49.155754+0000","last_active":"2023-09-23T20:33:22.425817+0000","last_peered":"2023-09-23T20:33:22.425817+0000","last_clean":"2023-09-23T20:33:22.425817+0000","last_became_active":"2023-09-23T20:31:49.155363+0000","last_became_peered":"2023-09-23T20:31:49.155363+0000","last_unstale":"2023-09-23T20:33:22.425817+0000","last_undegraded":"2023-09-23T20:33:22.425817+0000","last_fullsized":"2023-09-23T20:33:22.425817+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":5729,"log_dups_size":0,"ondisk_log_size":5729,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:45:42.661641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":375455744,"num_objects":5729,"num_object_clones":0,"num_object_copies":17187,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5729,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5729,"num_write_kb":366656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":554,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T20:33:18.495254+0000","last_change":"2023-09-23T20:27:31.757060+0000","last_active":"2023-09-23T20:33:18.495254+0000","last_peered":"2023-09-23T20:33:18.495254+0000","last_clean":"2023-09-23T20:33:18.495254+0000","last_became_active":"2023-09-23T20:27:11.959516+0000","last_became_peered":"2023-09-23T20:27:11.959516+0000","last_unstale":"2023-09-23T20:33:18.495254+0000","last_undegraded":"2023-09-23T20:33:18.495254+0000","last_fullsized":"2023-09-23T20:33:18.495254+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:31.756974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:31.756974+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:35:11.939335+0000","scrub_duration":0.0031802670000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":125,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T20:33:18.478086+0000","last_change":"2023-09-23T20:32:33.229700+0000","last_active":"2023-09-23T20:33:18.478086+0000","last_peered":"2023-09-23T20:33:18.478086+0000","last_clean":"2023-09-23T20:33:18.478086+0000","last_became_active":"2023-09-23T20:32:33.229372+0000","last_became_peered":"2023-09-23T20:32:33.229372+0000","last_unstale":"2023-09-23T20:33:18.478086+0000","last_undegraded":"2023-09-23T20:33:18.478086+0000","last_fullsized":"2023-09-23T20:33:18.478086+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:18:10.958227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":237,"reported_epoch":328,"state":"active+clean","last_fresh":"2023-09-23T20:33:17.504631+0000","last_change":"2023-09-23T20:33:00.188031+0000","last_active":"2023-09-23T20:33:17.504631+0000","last_peered":"2023-09-23T20:33:17.504631+0000","last_clean":"2023-09-23T20:33:17.504631+0000","last_became_active":"2023-09-23T20:33:00.187675+0000","last_became_peered":"2023-09-23T20:33:00.187675+0000","last_unstale":"2023-09-23T20:33:17.504631+0000","last_undegraded":"2023-09-23T20:33:17.504631+0000","last_fullsized":"2023-09-23T20:33:17.504631+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:42:28.967989+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.d","version":"330'5935","reported_seq":5970,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-09-23T20:33:25.622274+0000","last_change":"2023-09-23T20:31:49.158641+0000","last_active":"2023-09-23T20:33:25.622274+0000","last_peered":"2023-09-23T20:33:25.622274+0000","last_clean":"2023-09-23T20:33:25.622274+0000","last_became_active":"2023-09-23T20:31:49.157604+0000","last_became_peered":"2023-09-23T20:31:49.157604+0000","last_unstale":"2023-09-23T20:33:25.622274+0000","last_undegraded":"2023-09-23T20:33:25.622274+0000","last_fullsized":"2023-09-23T20:33:25.622274+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":5935,"log_dups_size":0,"ondisk_log_size":5935,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:54:04.726000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":388956160,"num_objects":5935,"num_object_clones":0,"num_object_copies":17805,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5935,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5935,"num_write_kb":379840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.b","version":"90'161","reported_seq":571,"reported_epoch":328,"state":"active+clean","last_fresh":"2023-09-23T20:33:17.470357+0000","last_change":"2023-09-23T20:31:46.558396+0000","last_active":"2023-09-23T20:33:17.470357+0000","last_peered":"2023-09-23T20:33:17.470357+0000","last_clean":"2023-09-23T20:33:17.470357+0000","last_became_active":"2023-09-23T20:29:45.534070+0000","last_became_peered":"2023-09-23T20:29:45.534070+0000","last_unstale":"2023-09-23T20:33:17.470357+0000","last_undegraded":"2023-09-23T20:33:17.470357+0000","last_fullsized":"2023-09-23T20:33:17.470357+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:46.558292+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:46.558292+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:10:34.051169+0000","scrub_duration":0.0023048779999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":239,"reported_epoch":328,"state":"active+clean","last_fresh":"2023-09-23T20:33:17.491532+0000","last_change":"2023-09-23T20:33:00.185583+0000","last_active":"2023-09-23T20:33:17.491532+0000","last_peered":"2023-09-23T20:33:17.491532+0000","last_clean":"2023-09-23T20:33:17.491532+0000","last_became_active":"2023-09-23T20:33:00.185273+0000","last_became_peered":"2023-09-23T20:33:00.185273+0000","last_unstale":"2023-09-23T20:33:17.491532+0000","last_undegraded":"2023-09-23T20:33:17.491532+0000","last_fullsized":"2023-09-23T20:33:17.491532+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:00:48.877233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":27,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T20:33:18.478365+0000","last_change":"2023-09-23T20:32:33.224963+0000","last_active":"2023-09-23T20:33:18.478365+0000","last_peered":"2023-09-23T20:33:18.478365+0000","last_clean":"2023-09-23T20:33:18.478365+0000","last_became_active":"2023-09-23T20:32:33.224634+0000","last_became_peered":"2023-09-23T20:32:33.224634+0000","last_unstale":"2023-09-23T20:33:18.478365+0000","last_undegraded":"2023-09-23T20:33:18.478365+0000","last_fullsized":"2023-09-23T20:33:18.478365+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:41:40.314140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"330'5873","reported_seq":5908,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-09-23T20:33:25.591038+0000","last_change":"2023-09-23T20:31:49.157247+0000","last_active":"2023-09-23T20:33:25.591038+0000","last_peered":"2023-09-23T20:33:25.591038+0000","last_clean":"2023-09-23T20:33:25.591038+0000","last_became_active":"2023-09-23T20:31:49.156961+0000","last_became_peered":"2023-09-23T20:31:49.156961+0000","last_unstale":"2023-09-23T20:33:25.591038+0000","last_undegraded":"2023-09-23T20:33:25.591038+0000","last_fullsized":"2023-09-23T20:33:25.591038+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":5873,"log_dups_size":0,"ondisk_log_size":5873,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:49:17.346795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":384892928,"num_objects":5873,"num_object_clones":0,"num_object_copies":17619,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5873,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5873,"num_write_kb":375872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":594,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T20:33:18.496119+0000","last_change":"2023-09-23T20:29:07.053561+0000","last_active":"2023-09-23T20:33:18.496119+0000","last_peered":"2023-09-23T20:33:18.496119+0000","last_clean":"2023-09-23T20:33:18.496119+0000","last_became_active":"2023-09-23T20:29:07.053177+0000","last_became_peered":"2023-09-23T20:29:07.053177+0000","last_unstale":"2023-09-23T20:33:18.496119+0000","last_undegraded":"2023-09-23T20:33:18.496119+0000","last_fullsized":"2023-09-23T20:33:18.496119+0000","mapping_epoch":253,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":254,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:30.727267+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:30.727267+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:04:32.661761+0000","scrub_duration":0.0052560289999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":141,"reported_epoch":328,"state":"active+clean","last_fresh":"2023-09-23T20:33:17.469858+0000","last_change":"2023-09-23T20:32:50.530690+0000","last_active":"2023-09-23T20:33:17.469858+0000","last_peered":"2023-09-23T20:33:17.469858+0000","last_clean":"2023-09-23T20:33:17.469858+0000","last_became_active":"2023-09-23T20:32:50.530376+0000","last_became_peered":"2023-09-23T20:32:50.530376+0000","last_unstale":"2023-09-23T20:33:17.469858+0000","last_undegraded":"2023-09-23T20:33:17.469858+0000","last_fullsized":"2023-09-23T20:33:17.469858+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:24:53.184959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":149,"reported_epoch":328,"state":"active+clean","last_fresh":"2023-09-23T20:33:17.468985+0000","last_change":"2023-09-23T20:32:33.230582+0000","last_active":"2023-09-23T20:33:17.468985+0000","last_peered":"2023-09-23T20:33:17.468985+0000","last_clean":"2023-09-23T20:33:17.468985+0000","last_became_active":"2023-09-23T20:32:33.230257+0000","last_became_peered":"2023-09-23T20:32:33.230257+0000","last_unstale":"2023-09-23T20:33:17.468985+0000","last_undegraded":"2023-09-23T20:33:17.468985+0000","last_fullsized":"2023-09-23T20:33:17.468985+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:11:45.333096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"330'5828","reported_seq":6919,"reported_epoch":330,"state":"active+recovery_wait","last_fresh":"2023-09-23T20:33:25.595939+0000","last_change":"2023-09-23T20:33:01.313842+0000","last_active":"2023-09-23T20:33:25.595939+0000","last_peered":"2023-09-23T20:33:25.595939+0000","last_clean":"2023-09-23T20:32:32.204597+0000","last_became_active":"2023-09-23T20:33:01.258982+0000","last_became_peered":"2023-09-23T20:33:01.258982+0000","last_unstale":"2023-09-23T20:33:25.595939+0000","last_undegraded":"2023-09-23T20:33:25.595939+0000","last_fullsized":"2023-09-23T20:33:25.595939+0000","mapping_epoch":323,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":5828,"log_dups_size":0,"ondisk_log_size":5828,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:38:23.266319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":381943808,"num_objects":5828,"num_object_clones":0,"num_object_copies":17484,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5828,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5828,"num_write_kb":372992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":521,"num_bytes_recovered":34144256,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":["6","1","3"],"object_location_counts":[{"shards":"1,3,6","objects":5828}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":559,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T20:33:18.479286+0000","last_change":"2023-09-23T20:31:45.196356+0000","last_active":"2023-09-23T20:33:18.479286+0000","last_peered":"2023-09-23T20:33:18.479286+0000","last_clean":"2023-09-23T20:33:18.479286+0000","last_became_active":"2023-09-23T20:29:07.115647+0000","last_became_peered":"2023-09-23T20:29:07.115647+0000","last_unstale":"2023-09-23T20:33:18.479286+0000","last_undegraded":"2023-09-23T20:33:18.479286+0000","last_fullsized":"2023-09-23T20:33:18.479286+0000","mapping_epoch":253,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":254,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:45.196285+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:45.196285+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:33:06.666183+0000","scrub_duration":0.0014380860000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"86'119","reported_seq":633,"reported_epoch":328,"state":"active+clean","last_fresh":"2023-09-23T20:33:17.470307+0000","last_change":"2023-09-23T20:29:45.521257+0000","last_active":"2023-09-23T20:33:17.470307+0000","last_peered":"2023-09-23T20:33:17.470307+0000","last_clean":"2023-09-23T20:33:17.470307+0000","last_became_active":"2023-09-23T20:29:45.520546+0000","last_became_peered":"2023-09-23T20:29:45.520546+0000","last_unstale":"2023-09-23T20:33:17.470307+0000","last_undegraded":"2023-09-23T20:33:17.470307+0000","last_fullsized":"2023-09-23T20:33:17.470307+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":119,"log_dups_size":0,"ondisk_log_size":119,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:58:56.085777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032705499999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":523,"num_read_kb":3503,"num_write":136,"num_write_kb":3128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":346,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T20:33:18.496675+0000","last_change":"2023-09-23T20:25:39.637160+0000","last_active":"2023-09-23T20:33:18.496675+0000","last_peered":"2023-09-23T20:33:18.496675+0000","last_clean":"2023-09-23T20:33:18.496675+0000","last_became_active":"2023-09-23T20:25:39.636860+0000","last_became_peered":"2023-09-23T20:25:39.636860+0000","last_unstale":"2023-09-23T20:33:18.496675+0000","last_undegraded":"2023-09-23T20:33:18.496675+0000","last_fullsized":"2023-09-23T20:33:18.496675+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:21:47.860063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":253,"reported_epoch":328,"state":"active+clean","last_fresh":"2023-09-23T20:33:17.468717+0000","last_change":"2023-09-23T20:33:00.190831+0000","last_active":"2023-09-23T20:33:17.468717+0000","last_peered":"2023-09-23T20:33:17.468717+0000","last_clean":"2023-09-23T20:33:17.468717+0000","last_became_active":"2023-09-23T20:33:00.190494+0000","last_became_peered":"2023-09-23T20:33:00.190494+0000","last_unstale":"2023-09-23T20:33:17.468717+0000","last_undegraded":"2023-09-23T20:33:17.468717+0000","last_fullsized":"2023-09-23T20:33:17.468717+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:19:01.109316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"330'5904","reported_seq":5939,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-09-23T20:33:25.570694+0000","last_change":"2023-09-23T20:31:49.157301+0000","last_active":"2023-09-23T20:33:25.570694+0000","last_peered":"2023-09-23T20:33:25.570694+0000","last_clean":"2023-09-23T20:33:25.570694+0000","last_became_active":"2023-09-23T20:31:49.156992+0000","last_became_peered":"2023-09-23T20:31:49.156992+0000","last_unstale":"2023-09-23T20:33:25.570694+0000","last_undegraded":"2023-09-23T20:33:25.570694+0000","last_fullsized":"2023-09-23T20:33:25.570694+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":5904,"log_dups_size":0,"ondisk_log_size":5904,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:21:35.645838+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":386924544,"num_objects":5904,"num_object_clones":0,"num_object_copies":17712,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5904,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5904,"num_write_kb":377856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":574,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T20:33:18.495861+0000","last_change":"2023-09-23T20:31:45.785521+0000","last_active":"2023-09-23T20:33:18.495861+0000","last_peered":"2023-09-23T20:33:18.495861+0000","last_clean":"2023-09-23T20:33:18.495861+0000","last_became_active":"2023-09-23T20:27:55.213394+0000","last_became_peered":"2023-09-23T20:27:55.213394+0000","last_unstale":"2023-09-23T20:33:18.495861+0000","last_undegraded":"2023-09-23T20:33:18.495861+0000","last_fullsized":"2023-09-23T20:33:18.495861+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:45.785450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:45.785450+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:21:26.092578+0000","scrub_duration":0.001558306,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"86'106","reported_seq":1048,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T20:33:18.495981+0000","last_change":"2023-09-23T20:31:31.531882+0000","last_active":"2023-09-23T20:33:18.495981+0000","last_peered":"2023-09-23T20:33:18.495981+0000","last_clean":"2023-09-23T20:33:18.495981+0000","last_became_active":"2023-09-23T20:29:46.078501+0000","last_became_peered":"2023-09-23T20:29:46.078501+0000","last_unstale":"2023-09-23T20:33:18.495981+0000","last_undegraded":"2023-09-23T20:33:18.495981+0000","last_fullsized":"2023-09-23T20:33:18.495981+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":106,"log_dups_size":0,"ondisk_log_size":106,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:37.015974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00099302599999999993,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1105,"num_read_kb":1689,"num_write":100,"num_write_kb":404,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":27,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T20:33:18.476750+0000","last_change":"2023-09-23T20:32:33.230242+0000","last_active":"2023-09-23T20:33:18.476750+0000","last_peered":"2023-09-23T20:33:18.476750+0000","last_clean":"2023-09-23T20:33:18.476750+0000","last_became_active":"2023-09-23T20:32:33.229732+0000","last_became_peered":"2023-09-23T20:32:33.229732+0000","last_unstale":"2023-09-23T20:33:18.476750+0000","last_undegraded":"2023-09-23T20:33:18.476750+0000","last_fullsized":"2023-09-23T20:33:18.476750+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:27:55.538429+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":286,"reported_epoch":328,"state":"active+clean","last_fresh":"2023-09-23T20:33:17.470229+0000","last_change":"2023-09-23T20:29:45.533406+0000","last_active":"2023-09-23T20:33:17.470229+0000","last_peered":"2023-09-23T20:33:17.470229+0000","last_clean":"2023-09-23T20:33:17.470229+0000","last_became_active":"2023-09-23T20:29:45.526001+0000","last_became_peered":"2023-09-23T20:29:45.526001+0000","last_unstale":"2023-09-23T20:33:17.470229+0000","last_undegraded":"2023-09-23T20:33:17.470229+0000","last_fullsized":"2023-09-23T20:33:17.470229+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:43.787251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.4","version":"330'5815","reported_seq":5878,"reported_epoch":330,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:33:25.290033+0000","last_change":"2023-09-23T20:33:00.311161+0000","last_active":"2023-09-23T20:33:25.290033+0000","last_peered":"2023-09-23T20:33:25.290033+0000","last_clean":"2023-09-23T20:32:32.192526+0000","last_became_active":"2023-09-23T20:33:00.256639+0000","last_became_peered":"2023-09-23T20:33:00.256639+0000","last_unstale":"2023-09-23T20:33:25.290033+0000","last_undegraded":"2023-09-23T20:33:00.185190+0000","last_fullsized":"2023-09-23T20:33:00.168423+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":5815,"log_dups_size":0,"ondisk_log_size":5815,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:59:02.098340+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":381091840,"num_objects":5815,"num_object_clones":0,"num_object_copies":17445,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1052,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5815,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5815,"num_write_kb":372160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":262144,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,6],"acting":[0,1],"avail_no_missing":["0","1"],"object_location_counts":[{"shards":"0,1","objects":4763},{"shards":"0,1,4","objects":1051}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":782,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T20:33:18.489896+0000","last_change":"2023-09-23T20:33:00.189173+0000","last_active":"2023-09-23T20:33:18.489896+0000","last_peered":"2023-09-23T20:33:18.489896+0000","last_clean":"2023-09-23T20:33:18.489896+0000","last_became_active":"2023-09-23T20:33:00.188795+0000","last_became_peered":"2023-09-23T20:33:00.188795+0000","last_unstale":"2023-09-23T20:33:18.489896+0000","last_undegraded":"2023-09-23T20:33:18.489896+0000","last_fullsized":"2023-09-23T20:33:18.489896+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:43.390908+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:43.390908+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:44:16.658526+0000","scrub_duration":0.0032721,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,2],"acting":[7,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"86'130","reported_seq":641,"reported_epoch":328,"state":"active+clean","last_fresh":"2023-09-23T20:33:17.491441+0000","last_change":"2023-09-23T20:25:39.637458+0000","last_active":"2023-09-23T20:33:17.491441+0000","last_peered":"2023-09-23T20:33:17.491441+0000","last_clean":"2023-09-23T20:33:17.491441+0000","last_became_active":"2023-09-23T20:25:39.636773+0000","last_became_peered":"2023-09-23T20:25:39.636773+0000","last_unstale":"2023-09-23T20:33:17.491441+0000","last_undegraded":"2023-09-23T20:33:17.491441+0000","last_fullsized":"2023-09-23T20:33:17.491441+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:00:32.810564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011300049999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":512,"num_read_kb":1820,"num_write":133,"num_write_kb":1347,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":276,"reported_epoch":328,"state":"active+clean","last_fresh":"2023-09-23T20:33:17.469717+0000","last_change":"2023-09-23T20:29:46.073394+0000","last_active":"2023-09-23T20:33:17.469717+0000","last_peered":"2023-09-23T20:33:17.469717+0000","last_clean":"2023-09-23T20:33:17.469717+0000","last_became_active":"2023-09-23T20:29:46.072067+0000","last_became_peered":"2023-09-23T20:29:46.072067+0000","last_unstale":"2023-09-23T20:33:17.469717+0000","last_undegraded":"2023-09-23T20:33:17.469717+0000","last_fullsized":"2023-09-23T20:33:17.469717+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:01:56.944614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":268,"reported_epoch":328,"state":"active+clean","last_fresh":"2023-09-23T20:33:17.468908+0000","last_change":"2023-09-23T20:29:45.533185+0000","last_active":"2023-09-23T20:33:17.468908+0000","last_peered":"2023-09-23T20:33:17.468908+0000","last_clean":"2023-09-23T20:33:17.468908+0000","last_became_active":"2023-09-23T20:29:45.526765+0000","last_became_peered":"2023-09-23T20:29:45.526765+0000","last_unstale":"2023-09-23T20:33:17.468908+0000","last_undegraded":"2023-09-23T20:33:17.468908+0000","last_fullsized":"2023-09-23T20:33:17.468908+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:18:33.063215+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"330'5863","reported_seq":5897,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-09-23T20:33:25.299069+0000","last_change":"2023-09-23T20:31:49.158194+0000","last_active":"2023-09-23T20:33:25.299069+0000","last_peered":"2023-09-23T20:33:25.299069+0000","last_clean":"2023-09-23T20:33:25.299069+0000","last_became_active":"2023-09-23T20:31:49.157875+0000","last_became_peered":"2023-09-23T20:31:49.157875+0000","last_unstale":"2023-09-23T20:33:25.299069+0000","last_undegraded":"2023-09-23T20:33:25.299069+0000","last_fullsized":"2023-09-23T20:33:25.299069+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":5863,"log_dups_size":0,"ondisk_log_size":5863,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:04:42.411181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":384237568,"num_objects":5863,"num_object_clones":0,"num_object_copies":17589,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5863,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5863,"num_write_kb":375232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"86'194","reported_seq":978,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T20:33:18.495864+0000","last_change":"2023-09-23T20:25:40.618665+0000","last_active":"2023-09-23T20:33:18.495864+0000","last_peered":"2023-09-23T20:33:18.495864+0000","last_clean":"2023-09-23T20:33:18.495864+0000","last_became_active":"2023-09-23T20:25:40.618087+0000","last_became_peered":"2023-09-23T20:25:40.618087+0000","last_unstale":"2023-09-23T20:33:18.495864+0000","last_undegraded":"2023-09-23T20:33:18.495864+0000","last_fullsized":"2023-09-23T20:33:18.495864+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":194,"log_dups_size":0,"ondisk_log_size":194,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:07:16.114968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00037995999999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1025,"num_read_kb":1201,"num_write":173,"num_write_kb":1276,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":763,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T20:33:18.489908+0000","last_change":"2023-09-23T20:31:45.343835+0000","last_active":"2023-09-23T20:33:18.489908+0000","last_peered":"2023-09-23T20:33:18.489908+0000","last_clean":"2023-09-23T20:33:18.489908+0000","last_became_active":"2023-09-23T20:29:07.042407+0000","last_became_peered":"2023-09-23T20:29:07.042407+0000","last_unstale":"2023-09-23T20:33:18.489908+0000","last_undegraded":"2023-09-23T20:33:18.489908+0000","last_fullsized":"2023-09-23T20:33:18.489908+0000","mapping_epoch":253,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":254,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:45.343742+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:45.343742+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:33:12.745366+0000","scrub_duration":0.001475802,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":129,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T20:33:18.496034+0000","last_change":"2023-09-23T20:32:50.534315+0000","last_active":"2023-09-23T20:33:18.496034+0000","last_peered":"2023-09-23T20:33:18.496034+0000","last_clean":"2023-09-23T20:33:18.496034+0000","last_became_active":"2023-09-23T20:32:50.533896+0000","last_became_peered":"2023-09-23T20:32:50.533896+0000","last_unstale":"2023-09-23T20:33:18.496034+0000","last_undegraded":"2023-09-23T20:33:18.496034+0000","last_fullsized":"2023-09-23T20:33:18.496034+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:06:53.464923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":109,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T20:33:18.479643+0000","last_change":"2023-09-23T20:32:33.225012+0000","last_active":"2023-09-23T20:33:18.479643+0000","last_peered":"2023-09-23T20:33:18.479643+0000","last_clean":"2023-09-23T20:33:18.479643+0000","last_became_active":"2023-09-23T20:32:33.224699+0000","last_became_peered":"2023-09-23T20:32:33.224699+0000","last_unstale":"2023-09-23T20:33:18.479643+0000","last_undegraded":"2023-09-23T20:33:18.479643+0000","last_fullsized":"2023-09-23T20:33:18.479643+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:10:06.267878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"330'5917","reported_seq":5952,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-09-23T20:33:25.687554+0000","last_change":"2023-09-23T20:31:49.154418+0000","last_active":"2023-09-23T20:33:25.687554+0000","last_peered":"2023-09-23T20:33:25.687554+0000","last_clean":"2023-09-23T20:33:25.687554+0000","last_became_active":"2023-09-23T20:31:49.154182+0000","last_became_peered":"2023-09-23T20:31:49.154182+0000","last_unstale":"2023-09-23T20:33:25.687554+0000","last_undegraded":"2023-09-23T20:33:25.687554+0000","last_fullsized":"2023-09-23T20:33:25.687554+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":5917,"log_dups_size":0,"ondisk_log_size":5917,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:44:10.087486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":387776512,"num_objects":5917,"num_object_clones":0,"num_object_copies":17751,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5917,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5917,"num_write_kb":378688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":604,"reported_epoch":328,"state":"active+clean","last_fresh":"2023-09-23T20:33:17.470186+0000","last_change":"2023-09-23T20:33:03.279925+0000","last_active":"2023-09-23T20:33:17.470186+0000","last_peered":"2023-09-23T20:33:17.470186+0000","last_clean":"2023-09-23T20:33:17.470186+0000","last_became_active":"2023-09-23T20:33:03.279531+0000","last_became_peered":"2023-09-23T20:33:03.279531+0000","last_unstale":"2023-09-23T20:33:17.470186+0000","last_undegraded":"2023-09-23T20:33:17.470186+0000","last_fullsized":"2023-09-23T20:33:17.470186+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:43.582146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:43.582146+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:06:12.587774+0000","scrub_duration":0.0016219730000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,5],"acting":[1,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"86'80","reported_seq":593,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T20:33:18.489818+0000","last_change":"2023-09-23T20:29:45.536796+0000","last_active":"2023-09-23T20:33:18.489818+0000","last_peered":"2023-09-23T20:33:18.489818+0000","last_clean":"2023-09-23T20:33:18.489818+0000","last_became_active":"2023-09-23T20:29:45.536439+0000","last_became_peered":"2023-09-23T20:29:45.536439+0000","last_unstale":"2023-09-23T20:33:18.489818+0000","last_undegraded":"2023-09-23T20:33:18.489818+0000","last_fullsized":"2023-09-23T20:33:18.489818+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":80,"log_dups_size":0,"ondisk_log_size":80,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:30:57.793132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035115,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":395,"num_read_kb":1370,"num_write":66,"num_write_kb":1061,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":322,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T20:33:18.486197+0000","last_change":"2023-09-23T20:33:00.220984+0000","last_active":"2023-09-23T20:33:18.486197+0000","last_peered":"2023-09-23T20:33:18.486197+0000","last_clean":"2023-09-23T20:33:18.486197+0000","last_became_active":"2023-09-23T20:33:00.220642+0000","last_became_peered":"2023-09-23T20:33:00.220642+0000","last_unstale":"2023-09-23T20:33:18.486197+0000","last_undegraded":"2023-09-23T20:33:18.486197+0000","last_fullsized":"2023-09-23T20:33:18.486197+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:52:14.218890+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":326,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T20:33:18.497281+0000","last_change":"2023-09-23T20:29:45.690815+0000","last_active":"2023-09-23T20:33:18.497281+0000","last_peered":"2023-09-23T20:33:18.497281+0000","last_clean":"2023-09-23T20:33:18.497281+0000","last_became_active":"2023-09-23T20:29:45.690478+0000","last_became_peered":"2023-09-23T20:29:45.690478+0000","last_unstale":"2023-09-23T20:33:18.497281+0000","last_undegraded":"2023-09-23T20:33:18.497281+0000","last_fullsized":"2023-09-23T20:33:18.497281+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:38.288193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.0","version":"330'5892","reported_seq":5927,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-09-23T20:33:25.272793+0000","last_change":"2023-09-23T20:31:49.157205+0000","last_active":"2023-09-23T20:33:25.272793+0000","last_peered":"2023-09-23T20:33:25.272793+0000","last_clean":"2023-09-23T20:33:25.272793+0000","last_became_active":"2023-09-23T20:31:49.156922+0000","last_became_peered":"2023-09-23T20:31:49.156922+0000","last_unstale":"2023-09-23T20:33:25.272793+0000","last_undegraded":"2023-09-23T20:33:25.272793+0000","last_fullsized":"2023-09-23T20:33:25.272793+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":5892,"log_dups_size":0,"ondisk_log_size":5892,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:51:36.120059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":386138112,"num_objects":5892,"num_object_clones":0,"num_object_copies":17676,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5892,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5892,"num_write_kb":377088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":552,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T20:33:18.478920+0000","last_change":"2023-09-23T20:31:44.211308+0000","last_active":"2023-09-23T20:33:18.478920+0000","last_peered":"2023-09-23T20:33:18.478920+0000","last_clean":"2023-09-23T20:33:18.478920+0000","last_became_active":"2023-09-23T20:29:07.113428+0000","last_became_peered":"2023-09-23T20:29:07.113428+0000","last_unstale":"2023-09-23T20:33:18.478920+0000","last_undegraded":"2023-09-23T20:33:18.478920+0000","last_fullsized":"2023-09-23T20:33:18.478920+0000","mapping_epoch":253,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":254,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:44.211184+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:44.211184+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:53:50.053167+0000","scrub_duration":0.0017707720000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"86'102","reported_seq":924,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T20:33:18.490652+0000","last_change":"2023-09-23T20:32:36.314672+0000","last_active":"2023-09-23T20:33:18.490652+0000","last_peered":"2023-09-23T20:33:18.490652+0000","last_clean":"2023-09-23T20:33:18.490652+0000","last_became_active":"2023-09-23T20:32:36.314231+0000","last_became_peered":"2023-09-23T20:32:36.314231+0000","last_unstale":"2023-09-23T20:33:18.490652+0000","last_undegraded":"2023-09-23T20:33:18.490652+0000","last_fullsized":"2023-09-23T20:33:18.490652+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":102,"log_dups_size":0,"ondisk_log_size":102,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:11:40.514709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030656500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":963,"num_read_kb":1234,"num_write":60,"num_write_kb":225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":252,"reported_epoch":328,"state":"active+clean","last_fresh":"2023-09-23T20:33:17.470384+0000","last_change":"2023-09-23T20:29:45.538283+0000","last_active":"2023-09-23T20:33:17.470384+0000","last_peered":"2023-09-23T20:33:17.470384+0000","last_clean":"2023-09-23T20:33:17.470384+0000","last_became_active":"2023-09-23T20:29:45.537922+0000","last_became_peered":"2023-09-23T20:29:45.537922+0000","last_unstale":"2023-09-23T20:33:17.470384+0000","last_undegraded":"2023-09-23T20:33:17.470384+0000","last_fullsized":"2023-09-23T20:33:17.470384+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:30:52.397836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":263,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T20:33:18.484346+0000","last_change":"2023-09-23T20:20:14.888874+0000","last_active":"2023-09-23T20:33:18.484346+0000","last_peered":"2023-09-23T20:33:18.484346+0000","last_clean":"2023-09-23T20:33:18.484346+0000","last_became_active":"2023-09-23T20:20:14.887724+0000","last_became_peered":"2023-09-23T20:20:14.887724+0000","last_unstale":"2023-09-23T20:33:18.484346+0000","last_undegraded":"2023-09-23T20:33:18.484346+0000","last_fullsized":"2023-09-23T20:33:18.484346+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:09:24.510326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"329'5538","reported_seq":6757,"reported_epoch":329,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:33:22.504507+0000","last_change":"2023-09-23T20:32:51.571341+0000","last_active":"2023-09-23T20:33:22.504507+0000","last_peered":"2023-09-23T20:33:22.504507+0000","last_clean":"2023-09-23T20:32:32.204529+0000","last_became_active":"2023-09-23T20:32:51.533262+0000","last_became_peered":"2023-09-23T20:32:51.533262+0000","last_unstale":"2023-09-23T20:33:22.504507+0000","last_undegraded":"2023-09-23T20:32:51.523866+0000","last_fullsized":"2023-09-23T20:32:51.518152+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":5538,"log_dups_size":0,"ondisk_log_size":5538,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:50:09.051328+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":362938368,"num_objects":5538,"num_object_clones":0,"num_object_copies":16614,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2957,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5538,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5538,"num_write_kb":354432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":585,"num_bytes_recovered":38338560,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3],"avail_no_missing":["2","3","6"],"object_location_counts":[{"shards":"2,3","objects":2581},{"shards":"2,3,6","objects":2957}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":522,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T20:33:18.478894+0000","last_change":"2023-09-23T20:32:36.307867+0000","last_active":"2023-09-23T20:33:18.478894+0000","last_peered":"2023-09-23T20:33:18.478894+0000","last_clean":"2023-09-23T20:33:18.478894+0000","last_became_active":"2023-09-23T20:32:36.307498+0000","last_became_peered":"2023-09-23T20:32:36.307498+0000","last_unstale":"2023-09-23T20:33:18.478894+0000","last_undegraded":"2023-09-23T20:33:18.478894+0000","last_fullsized":"2023-09-23T20:33:18.478894+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:44.509012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:44.509012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:14:40.400935+0000","scrub_duration":0.0032186319999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":67,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T20:33:18.490859+0000","last_change":"2023-09-23T20:32:50.549695+0000","last_active":"2023-09-23T20:33:18.490859+0000","last_peered":"2023-09-23T20:33:18.490859+0000","last_clean":"2023-09-23T20:33:18.490859+0000","last_became_active":"2023-09-23T20:32:50.549431+0000","last_became_peered":"2023-09-23T20:32:50.549431+0000","last_unstale":"2023-09-23T20:33:18.490859+0000","last_undegraded":"2023-09-23T20:33:18.490859+0000","last_fullsized":"2023-09-23T20:33:18.490859+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:45:53.666569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"86'128","reported_seq":718,"reported_epoch":328,"state":"active+clean","last_fresh":"2023-09-23T20:33:17.468637+0000","last_change":"2023-09-23T20:32:36.310656+0000","last_active":"2023-09-23T20:33:17.468637+0000","last_peered":"2023-09-23T20:33:17.468637+0000","last_clean":"2023-09-23T20:33:17.468637+0000","last_became_active":"2023-09-23T20:32:36.310155+0000","last_became_peered":"2023-09-23T20:32:36.310155+0000","last_unstale":"2023-09-23T20:33:17.468637+0000","last_undegraded":"2023-09-23T20:33:17.468637+0000","last_fullsized":"2023-09-23T20:33:17.468637+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:58:14.217466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000293202,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":579,"num_read_kb":1473,"num_write":96,"num_write_kb":1062,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":100,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T20:33:18.490074+0000","last_change":"2023-09-23T20:32:50.539051+0000","last_active":"2023-09-23T20:33:18.490074+0000","last_peered":"2023-09-23T20:33:18.490074+0000","last_clean":"2023-09-23T20:33:18.490074+0000","last_became_active":"2023-09-23T20:32:50.538755+0000","last_became_peered":"2023-09-23T20:32:50.538755+0000","last_unstale":"2023-09-23T20:33:18.490074+0000","last_undegraded":"2023-09-23T20:33:18.490074+0000","last_fullsized":"2023-09-23T20:33:18.490074+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:16.656613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.2","version":"329'5731","reported_seq":5773,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T20:33:23.784437+0000","last_change":"2023-09-23T20:32:50.557165+0000","last_active":"2023-09-23T20:33:23.784437+0000","last_peered":"2023-09-23T20:33:23.784437+0000","last_clean":"2023-09-23T20:33:23.784437+0000","last_became_active":"2023-09-23T20:32:50.553409+0000","last_became_peered":"2023-09-23T20:32:50.553409+0000","last_unstale":"2023-09-23T20:33:23.784437+0000","last_undegraded":"2023-09-23T20:33:23.784437+0000","last_fullsized":"2023-09-23T20:33:23.784437+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":5731,"log_dups_size":0,"ondisk_log_size":5731,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:55:13.578195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":375586816,"num_objects":5731,"num_object_clones":0,"num_object_copies":17193,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5731,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5731,"num_write_kb":366784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"90'161","reported_seq":454,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T20:33:18.495693+0000","last_change":"2023-09-23T20:33:17.033023+0000","last_active":"2023-09-23T20:33:18.495693+0000","last_peered":"2023-09-23T20:33:18.495693+0000","last_clean":"2023-09-23T20:33:18.495693+0000","last_became_active":"2023-09-23T20:32:50.533470+0000","last_became_peered":"2023-09-23T20:32:50.533470+0000","last_unstale":"2023-09-23T20:33:18.495693+0000","last_undegraded":"2023-09-23T20:33:18.495693+0000","last_fullsized":"2023-09-23T20:33:18.495693+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:28:03.553106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:28:03.553106+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.7","version":"86'117","reported_seq":599,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T20:33:18.479458+0000","last_change":"2023-09-23T20:31:32.814392+0000","last_active":"2023-09-23T20:33:18.479458+0000","last_peered":"2023-09-23T20:33:18.479458+0000","last_clean":"2023-09-23T20:33:18.479458+0000","last_became_active":"2023-09-23T20:31:32.813857+0000","last_became_peered":"2023-09-23T20:31:32.813857+0000","last_unstale":"2023-09-23T20:33:18.479458+0000","last_undegraded":"2023-09-23T20:33:18.479458+0000","last_fullsized":"2023-09-23T20:33:18.479458+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:56:57.863005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046891400000000002,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":893,"num_read_kb":922,"num_write":82,"num_write_kb":4193,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":326,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T20:33:18.497375+0000","last_change":"2023-09-23T20:32:33.277827+0000","last_active":"2023-09-23T20:33:18.497375+0000","last_peered":"2023-09-23T20:33:18.497375+0000","last_clean":"2023-09-23T20:33:18.497375+0000","last_became_active":"2023-09-23T20:32:33.236087+0000","last_became_peered":"2023-09-23T20:32:33.236087+0000","last_unstale":"2023-09-23T20:33:18.497375+0000","last_undegraded":"2023-09-23T20:33:18.497375+0000","last_fullsized":"2023-09-23T20:33:18.497375+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:20:58.176854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":27,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T20:33:18.476510+0000","last_change":"2023-09-23T20:32:33.238290+0000","last_active":"2023-09-23T20:33:18.476510+0000","last_peered":"2023-09-23T20:33:18.476510+0000","last_clean":"2023-09-23T20:33:18.476510+0000","last_became_active":"2023-09-23T20:32:33.237984+0000","last_became_peered":"2023-09-23T20:32:33.237984+0000","last_unstale":"2023-09-23T20:33:18.476510+0000","last_undegraded":"2023-09-23T20:33:18.476510+0000","last_fullsized":"2023-09-23T20:33:18.476510+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:49:02.089307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":294,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T20:33:18.484242+0000","last_change":"2023-09-23T20:32:33.237234+0000","last_active":"2023-09-23T20:33:18.484242+0000","last_peered":"2023-09-23T20:33:18.484242+0000","last_clean":"2023-09-23T20:33:18.484242+0000","last_became_active":"2023-09-23T20:32:33.236881+0000","last_became_peered":"2023-09-23T20:32:33.236881+0000","last_unstale":"2023-09-23T20:33:18.484242+0000","last_undegraded":"2023-09-23T20:33:18.484242+0000","last_fullsized":"2023-09-23T20:33:18.484242+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:56:17.384831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":181,"reported_epoch":328,"state":"active+clean","last_fresh":"2023-09-23T20:33:17.504486+0000","last_change":"2023-09-23T20:29:45.538314+0000","last_active":"2023-09-23T20:33:17.504486+0000","last_peered":"2023-09-23T20:33:17.504486+0000","last_clean":"2023-09-23T20:33:17.504486+0000","last_became_active":"2023-09-23T20:29:45.536847+0000","last_became_peered":"2023-09-23T20:29:45.536847+0000","last_unstale":"2023-09-23T20:33:17.504486+0000","last_undegraded":"2023-09-23T20:33:17.504486+0000","last_fullsized":"2023-09-23T20:33:17.504486+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:51.948721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":339,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T20:33:18.495502+0000","last_change":"2023-09-23T20:32:33.235987+0000","last_active":"2023-09-23T20:33:18.495502+0000","last_peered":"2023-09-23T20:33:18.495502+0000","last_clean":"2023-09-23T20:33:18.495502+0000","last_became_active":"2023-09-23T20:32:33.226544+0000","last_became_peered":"2023-09-23T20:32:33.226544+0000","last_unstale":"2023-09-23T20:33:18.495502+0000","last_undegraded":"2023-09-23T20:33:18.495502+0000","last_fullsized":"2023-09-23T20:33:18.495502+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:53:22.794856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":284,"reported_epoch":328,"state":"active+clean","last_fresh":"2023-09-23T20:33:17.470000+0000","last_change":"2023-09-23T20:33:00.177976+0000","last_active":"2023-09-23T20:33:17.470000+0000","last_peered":"2023-09-23T20:33:17.470000+0000","last_clean":"2023-09-23T20:33:17.470000+0000","last_became_active":"2023-09-23T20:33:00.177665+0000","last_became_peered":"2023-09-23T20:33:00.177665+0000","last_unstale":"2023-09-23T20:33:17.470000+0000","last_undegraded":"2023-09-23T20:33:17.470000+0000","last_fullsized":"2023-09-23T20:33:17.470000+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:20:25.273907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":347,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T20:33:18.496525+0000","last_change":"2023-09-23T20:32:33.268260+0000","last_active":"2023-09-23T20:33:18.496525+0000","last_peered":"2023-09-23T20:33:18.496525+0000","last_clean":"2023-09-23T20:33:18.496525+0000","last_became_active":"2023-09-23T20:32:33.233695+0000","last_became_peered":"2023-09-23T20:32:33.233695+0000","last_unstale":"2023-09-23T20:33:18.496525+0000","last_undegraded":"2023-09-23T20:33:18.496525+0000","last_fullsized":"2023-09-23T20:33:18.496525+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:19:29.299623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":184,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T20:33:18.486351+0000","last_change":"2023-09-23T20:33:00.179961+0000","last_active":"2023-09-23T20:33:18.486351+0000","last_peered":"2023-09-23T20:33:18.486351+0000","last_clean":"2023-09-23T20:33:18.486351+0000","last_became_active":"2023-09-23T20:33:00.178429+0000","last_became_peered":"2023-09-23T20:33:00.178429+0000","last_unstale":"2023-09-23T20:33:18.486351+0000","last_undegraded":"2023-09-23T20:33:18.486351+0000","last_fullsized":"2023-09-23T20:33:18.486351+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:28:20.964989+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":145,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T20:33:18.478764+0000","last_change":"2023-09-23T20:32:50.537447+0000","last_active":"2023-09-23T20:33:18.478764+0000","last_peered":"2023-09-23T20:33:18.478764+0000","last_clean":"2023-09-23T20:33:18.478764+0000","last_became_active":"2023-09-23T20:32:50.537136+0000","last_became_peered":"2023-09-23T20:32:50.537136+0000","last_unstale":"2023-09-23T20:33:18.478764+0000","last_undegraded":"2023-09-23T20:33:18.478764+0000","last_fullsized":"2023-09-23T20:33:18.478764+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:46:51.599492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":43,"reported_epoch":328,"state":"active+clean","last_fresh":"2023-09-23T20:33:17.491604+0000","last_change":"2023-09-23T20:32:50.537450+0000","last_active":"2023-09-23T20:33:17.491604+0000","last_peered":"2023-09-23T20:33:17.491604+0000","last_clean":"2023-09-23T20:33:17.491604+0000","last_became_active":"2023-09-23T20:32:50.537128+0000","last_became_peered":"2023-09-23T20:32:50.537128+0000","last_unstale":"2023-09-23T20:33:17.491604+0000","last_undegraded":"2023-09-23T20:33:17.491604+0000","last_fullsized":"2023-09-23T20:33:17.491604+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:37:15.889029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":311,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T20:33:18.484311+0000","last_change":"2023-09-23T20:25:39.668882+0000","last_active":"2023-09-23T20:33:18.484311+0000","last_peered":"2023-09-23T20:33:18.484311+0000","last_clean":"2023-09-23T20:33:18.484311+0000","last_became_active":"2023-09-23T20:25:39.637128+0000","last_became_peered":"2023-09-23T20:25:39.637128+0000","last_unstale":"2023-09-23T20:33:18.484311+0000","last_undegraded":"2023-09-23T20:33:18.484311+0000","last_fullsized":"2023-09-23T20:33:18.484311+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:56.406298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":27,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T20:33:18.482960+0000","last_change":"2023-09-23T20:32:33.232281+0000","last_active":"2023-09-23T20:33:18.482960+0000","last_peered":"2023-09-23T20:33:18.482960+0000","last_clean":"2023-09-23T20:33:18.482960+0000","last_became_active":"2023-09-23T20:32:33.232003+0000","last_became_peered":"2023-09-23T20:32:33.232003+0000","last_unstale":"2023-09-23T20:33:18.482960+0000","last_undegraded":"2023-09-23T20:33:18.482960+0000","last_fullsized":"2023-09-23T20:33:18.482960+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:58:27.314355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":6095110144,"num_objects":93004,"num_object_clones":0,"num_object_copies":279012,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5117,"num_objects_misplaced":2587,"num_objects_unfound":0,"num_objects_dirty":93004,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":93004,"num_write_kb":5952256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2162,"num_bytes_recovered":141688832,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":18495766528,"data_stored":18495766528,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":93004,"ondisk_log_size":93004,"up":48,"acting":44,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":48,"num_bytes_recovered":6522,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26691,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":389805,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3381,"ondisk_log_size":3381,"up":63,"acting":63,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":58,"num_bytes_recovered":4194342,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6082,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201547,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2464412,"kb_used_data":2201172,"kb_used_omap":0,"kb_used_meta":263232,"kb_avail":91280740,"statfs":{"total":95995035648,"available":93471477760,"internally_reserved":0,"allocated":2254000128,"data_stored":2252958730,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":269549568},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:32:36 2023","interfaces":[{"interface":"back","average":{"1min":0.91500000000000004,"5min":1.3740000000000001,"15min":1.282},"min":{"1min":0.23599999999999999,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":3.9119999999999999,"5min":8.0660000000000007,"15min":48.798999999999999},"last":0.255},{"interface":"front","average":{"1min":0.94799999999999995,"5min":1.522,"15min":1.4530000000000001},"min":{"1min":0.30299999999999999,"5min":0.27300000000000002,"15min":0.24399999999999999},"max":{"1min":2.46,"5min":7.968,"15min":48.247},"last":0.39800000000000002}]},{"osd":1,"last update":"Sat Sep 23 20:32:53 2023","interfaces":[{"interface":"back","average":{"1min":1.405,"5min":1.8859999999999999,"15min":1.8029999999999999},"min":{"1min":0.31900000000000001,"5min":0.254,"15min":0.254},"max":{"1min":4.7309999999999999,"5min":14.026,"15min":14.026},"last":0.30199999999999999},{"interface":"front","average":{"1min":1.3700000000000001,"5min":1.6699999999999999,"15min":1.877},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":5.2750000000000004,"5min":5.7679999999999998,"15min":5.7679999999999998},"last":1.482}]},{"osd":2,"last update":"Sat Sep 23 20:32:42 2023","interfaces":[{"interface":"back","average":{"1min":2.1219999999999999,"5min":1.8959999999999999,"15min":1.385},"min":{"1min":0.29099999999999998,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":13.173999999999999,"5min":13.173999999999999,"15min":13.173999999999999},"last":0.32000000000000001},{"interface":"front","average":{"1min":1.4139999999999999,"5min":2.1709999999999998,"15min":1.55},"min":{"1min":0.40400000000000003,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":6.9379999999999997,"5min":14.993,"15min":14.993},"last":0.27400000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:33:05 2023","interfaces":[{"interface":"back","average":{"1min":2.903,"5min":2.2229999999999999,"15min":1.5880000000000001},"min":{"1min":0.375,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":7.9850000000000003,"5min":17.155999999999999,"15min":17.155999999999999},"last":7.202},{"interface":"front","average":{"1min":2.7829999999999999,"5min":2.0630000000000002,"15min":1.5049999999999999},"min":{"1min":0.47299999999999998,"5min":0.33100000000000002,"15min":0.27600000000000002},"max":{"1min":6.6379999999999999,"5min":16.960000000000001,"15min":16.960000000000001},"last":3.8130000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:33:05 2023","interfaces":[{"interface":"back","average":{"1min":0.65900000000000003,"5min":1.381,"15min":1.034},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.27000000000000002},"max":{"1min":2.3370000000000002,"5min":15.177,"15min":15.177},"last":0.33700000000000002},{"interface":"front","average":{"1min":0.92500000000000004,"5min":1.1619999999999999,"15min":1.155},"min":{"1min":0.33900000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":5.2190000000000003,"5min":11.122,"15min":11.122},"last":0.36899999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:33:05 2023","interfaces":[{"interface":"back","average":{"1min":0.77100000000000002,"5min":0.97599999999999998,"15min":1.1120000000000001},"min":{"1min":0.317,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":3.0339999999999998,"5min":6.8300000000000001,"15min":13.241},"last":0.308},{"interface":"front","average":{"1min":0.55100000000000005,"5min":1.272,"15min":1.2},"min":{"1min":0.35899999999999999,"5min":0.32200000000000001,"15min":0.315},"max":{"1min":1.3400000000000001,"5min":18.831,"15min":18.831},"last":0.378}]},{"osd":6,"last update":"Sat Sep 23 20:33:05 2023","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":1.2270000000000001,"15min":1.0960000000000001},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33400000000000002},"max":{"1min":1.0940000000000001,"5min":12.298999999999999,"15min":13.185},"last":0.48099999999999998},{"interface":"front","average":{"1min":0.93400000000000005,"5min":1.069,"15min":1.0649999999999999},"min":{"1min":0.37,"5min":0.318,"15min":0.29899999999999999},"max":{"1min":5.1059999999999999,"5min":8.0069999999999997,"15min":9.9120000000000008},"last":0.42999999999999999}]}]},{"osd":6,"up_from":47,"seq":201863463183,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2661204,"kb_used_data":2416908,"kb_used_omap":0,"kb_used_meta":244288,"kb_avail":91083948,"statfs":{"total":95995035648,"available":93269962752,"internally_reserved":0,"allocated":2474913792,"data_stored":2473880459,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":250150912},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:32:36 2023","interfaces":[{"interface":"back","average":{"1min":3.9550000000000001,"5min":1.891,"15min":1.2669999999999999},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.221},"max":{"1min":53.701999999999998,"5min":53.701999999999998,"15min":53.701999999999998},"last":8.7509999999999994},{"interface":"front","average":{"1min":4.5439999999999996,"5min":2.4550000000000001,"15min":1.573},"min":{"1min":0.35599999999999998,"5min":0.251,"15min":0.25},"max":{"1min":53.875999999999998,"5min":53.875999999999998,"15min":53.875999999999998},"last":3.1200000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:32:49 2023","interfaces":[{"interface":"back","average":{"1min":1.216,"5min":2.262,"15min":2.5880000000000001},"min":{"1min":0.35399999999999998,"5min":0.249,"15min":0.249},"max":{"1min":3.6019999999999999,"5min":14.1,"15min":14.1},"last":1.913},{"interface":"front","average":{"1min":1.4390000000000001,"5min":2.7050000000000001,"15min":3.226},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":4.9199999999999999,"5min":14.266999999999999,"15min":14.266999999999999},"last":2.5649999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:32:36 2023","interfaces":[{"interface":"back","average":{"1min":1.0900000000000001,"5min":2.1859999999999999,"15min":1.4450000000000001},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":3.617,"5min":10.631,"15min":13.856},"last":6.0439999999999996},{"interface":"front","average":{"1min":1.147,"5min":2.0939999999999999,"15min":1.377},"min":{"1min":0.38200000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":3.629,"5min":10.151999999999999,"15min":10.151999999999999},"last":8.6449999999999996}]},{"osd":3,"last update":"Sat Sep 23 20:33:20 2023","interfaces":[{"interface":"back","average":{"1min":2.1110000000000002,"5min":1.9670000000000001,"15min":1.504},"min":{"1min":0.30499999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":7.5620000000000003,"5min":7.7130000000000001,"15min":9.4239999999999995},"last":0.75700000000000001},{"interface":"front","average":{"1min":2.5310000000000001,"5min":2.1739999999999999,"15min":1.573},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":10.805,"5min":10.805,"15min":10.805},"last":0.70799999999999996}]},{"osd":4,"last update":"Sat Sep 23 20:33:00 2023","interfaces":[{"interface":"back","average":{"1min":0.49399999999999999,"5min":0.78800000000000003,"15min":0.85899999999999999},"min":{"1min":0.253,"5min":0.251,"15min":0.251},"max":{"1min":2.9020000000000001,"5min":8.4749999999999996,"15min":10.173},"last":0.67100000000000004},{"interface":"front","average":{"1min":0.83499999999999996,"5min":1.1259999999999999,"15min":1.0049999999999999},"min":{"1min":0.28199999999999997,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":7.7629999999999999,"5min":7.7629999999999999,"15min":8.3249999999999993},"last":0.36399999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:33:00 2023","interfaces":[{"interface":"back","average":{"1min":0.88300000000000001,"5min":0.83099999999999996,"15min":0.92300000000000004},"min":{"1min":0.311,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":6.8310000000000004,"5min":8.4499999999999993,"15min":14.085000000000001},"last":0.52800000000000002},{"interface":"front","average":{"1min":0.90800000000000003,"5min":1.258,"15min":1.262},"min":{"1min":0.32100000000000001,"5min":0.30599999999999999,"15min":0.29499999999999998},"max":{"1min":7.7300000000000004,"5min":9.9299999999999997,"15min":18.192},"last":0.58499999999999996}]},{"osd":7,"last update":"Sat Sep 23 20:33:20 2023","interfaces":[{"interface":"back","average":{"1min":1.0960000000000001,"5min":1.0920000000000001,"15min":1.0649999999999999},"min":{"1min":0.34499999999999997,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":5.0350000000000001,"5min":8.5060000000000002,"15min":10.217000000000001},"last":0.41999999999999998},{"interface":"front","average":{"1min":1.4239999999999999,"5min":1.409,"15min":1.218},"min":{"1min":0.375,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":7.601,"5min":7.8120000000000003,"15min":9.7360000000000007},"last":0.40400000000000003}]}]},{"osd":1,"up_from":266,"seq":1142461300781,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2790840,"kb_used_data":2579696,"kb_used_omap":8,"kb_used_meta":211127,"kb_avail":90954312,"statfs":{"total":95995035648,"available":93137215488,"internally_reserved":0,"allocated":2641608704,"data_stored":2639837511,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8206,"internal_metadata":216195058},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:32:49 2023","interfaces":[{"interface":"back","average":{"1min":4.5289999999999999,"5min":2.7709999999999999,"15min":2.4769999999999999},"min":{"1min":0.224,"5min":0.218,"15min":0.218},"max":{"1min":52.689999999999998,"5min":52.689999999999998,"15min":52.689999999999998},"last":1.1060000000000001},{"interface":"front","average":{"1min":4.5279999999999996,"5min":3.1259999999999999,"15min":3.1720000000000002},"min":{"1min":0.218,"5min":0.217,"15min":0.217},"max":{"1min":52.662999999999997,"5min":52.662999999999997,"15min":52.662999999999997},"last":0.41399999999999998}]},{"osd":2,"last update":"Sat Sep 23 20:32:49 2023","interfaces":[{"interface":"back","average":{"1min":1.371,"5min":3.8319999999999999,"15min":4.694},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":4.8689999999999998,"5min":16.151,"15min":16.151},"last":5.1429999999999998},{"interface":"front","average":{"1min":1.7749999999999999,"5min":2.0030000000000001,"15min":2.0630000000000002},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":8.077,"5min":12.356,"15min":12.356},"last":2.1629999999999998}]},{"osd":3,"last update":"Sat Sep 23 20:32:49 2023","interfaces":[{"interface":"back","average":{"1min":8.2010000000000005,"5min":4.3849999999999998,"15min":4.0389999999999997},"min":{"1min":0.311,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":80.745999999999995,"5min":80.745999999999995,"15min":80.745999999999995},"last":4.2679999999999998},{"interface":"front","average":{"1min":7.6909999999999998,"5min":4.7530000000000001,"15min":4.8150000000000004},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":82.988,"5min":82.988,"15min":82.988},"last":5.1790000000000003}]},{"osd":4,"last update":"Sat Sep 23 20:32:49 2023","interfaces":[{"interface":"back","average":{"1min":1.0700000000000001,"5min":1.855,"15min":2.004},"min":{"1min":0.40699999999999997,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":2.863,"5min":9.4429999999999996,"15min":9.4429999999999996},"last":2.7280000000000002},{"interface":"front","average":{"1min":1.272,"5min":1.7150000000000001,"15min":1.921},"min":{"1min":0.372,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":4.0730000000000004,"5min":10.760999999999999,"15min":10.760999999999999},"last":1.161}]},{"osd":5,"last update":"Sat Sep 23 20:32:49 2023","interfaces":[{"interface":"back","average":{"1min":0.95799999999999996,"5min":2.1200000000000001,"15min":2.5739999999999998},"min":{"1min":0.33400000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":2.649,"5min":9.6479999999999997,"15min":9.6479999999999997},"last":0.438},{"interface":"front","average":{"1min":1.0669999999999999,"5min":1.9650000000000001,"15min":2.048},"min":{"1min":0.36899999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":2.9700000000000002,"5min":9.4700000000000006,"15min":9.4700000000000006},"last":1.8320000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:32:49 2023","interfaces":[{"interface":"back","average":{"1min":1.018,"5min":4.9980000000000002,"15min":6.6319999999999997},"min":{"1min":0.40300000000000002,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":2.714,"5min":72.682000000000002,"15min":72.682000000000002},"last":0.38},{"interface":"front","average":{"1min":1.0740000000000001,"5min":4.5720000000000001,"15min":5.8170000000000002},"min":{"1min":0.438,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":2.9319999999999999,"5min":72.563999999999993,"15min":72.563999999999993},"last":1.8700000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:32:49 2023","interfaces":[{"interface":"back","average":{"1min":1.085,"5min":2.4180000000000001,"15min":2.9940000000000002},"min":{"1min":0.39900000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":2.6880000000000002,"5min":9.6850000000000005,"15min":9.6850000000000005},"last":0.46000000000000002},{"interface":"front","average":{"1min":1.147,"5min":2.1480000000000001,"15min":2.2890000000000001},"min":{"1min":0.51200000000000001,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":2.8980000000000001,"5min":9.3949999999999996,"15min":9.3949999999999996},"last":2.202}]}]},{"osd":0,"up_from":94,"seq":403726925984,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3400172,"kb_used_data":2994788,"kb_used_omap":7,"kb_used_meta":405368,"kb_avail":90344980,"statfs":{"total":95995035648,"available":92513259520,"internally_reserved":0,"allocated":3066662912,"data_stored":3065027555,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":415096849},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,0,0,2,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:32:55 2023","interfaces":[{"interface":"back","average":{"1min":1.825,"5min":2.9750000000000001,"15min":3.5830000000000002},"min":{"1min":0.245,"5min":0.17999999999999999,"15min":0.17999999999999999},"max":{"1min":8.0630000000000006,"5min":15.542,"15min":15.542},"last":2.1320000000000001},{"interface":"front","average":{"1min":1.873,"5min":3.3050000000000002,"15min":4.0730000000000004},"min":{"1min":0.29899999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":5.3410000000000002,"5min":10.49,"15min":10.49},"last":0.86499999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:32:32 2023","interfaces":[{"interface":"back","average":{"1min":1.5249999999999999,"5min":3.085,"15min":1.875},"min":{"1min":0.246,"5min":0.185,"15min":0.185},"max":{"1min":6.5890000000000004,"5min":77.448999999999998,"15min":77.448999999999998},"last":0.32300000000000001},{"interface":"front","average":{"1min":1.0269999999999999,"5min":2.7610000000000001,"15min":1.621},"min":{"1min":0.188,"5min":0.188,"15min":0.188},"max":{"1min":3.7130000000000001,"5min":81.724999999999994,"15min":81.724999999999994},"last":9.7769999999999992}]},{"osd":3,"last update":"Sat Sep 23 20:33:05 2023","interfaces":[{"interface":"back","average":{"1min":3.0019999999999998,"5min":3.7149999999999999,"15min":2.0209999999999999},"min":{"1min":0.35799999999999998,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":13.51,"5min":81.400999999999996,"15min":81.400999999999996},"last":2.109},{"interface":"front","average":{"1min":3.113,"5min":3.8559999999999999,"15min":1.9790000000000001},"min":{"1min":0.35599999999999998,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":12.651999999999999,"5min":81.137,"15min":81.137},"last":1.464}]},{"osd":4,"last update":"Sat Sep 23 20:32:32 2023","interfaces":[{"interface":"back","average":{"1min":0.92800000000000005,"5min":1.78,"15min":1.504},"min":{"1min":0.34999999999999998,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":2.4550000000000001,"5min":9.9339999999999993,"15min":23.876999999999999},"last":0.35899999999999999},{"interface":"front","average":{"1min":0.997,"5min":1.554,"15min":1.3129999999999999},"min":{"1min":0.39000000000000001,"5min":0.312,"15min":0.29299999999999998},"max":{"1min":3.8340000000000001,"5min":10.414,"15min":23.681999999999999},"last":0.45800000000000002}]},{"osd":5,"last update":"Sat Sep 23 20:32:32 2023","interfaces":[{"interface":"back","average":{"1min":1.0049999999999999,"5min":1.913,"15min":1.6799999999999999},"min":{"1min":0.35399999999999998,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":3.7160000000000002,"5min":14.289,"15min":24.198},"last":2.5430000000000001},{"interface":"front","average":{"1min":0.94799999999999995,"5min":2.0030000000000001,"15min":1.51},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":2.0779999999999998,"5min":14.938000000000001,"15min":14.938000000000001},"last":9.8219999999999992}]},{"osd":6,"last update":"Sat Sep 23 20:32:32 2023","interfaces":[{"interface":"back","average":{"1min":0.99199999999999999,"5min":1.7589999999999999,"15min":1.397},"min":{"1min":0.432,"5min":0.318,"15min":0.318},"max":{"1min":3.7789999999999999,"5min":10.441000000000001,"15min":24.408000000000001},"last":6.5739999999999998},{"interface":"front","average":{"1min":0.95699999999999996,"5min":1.7569999999999999,"15min":1.3460000000000001},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":2.1539999999999999,"5min":9.6769999999999996,"15min":25.271000000000001},"last":9.8719999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:32:32 2023","interfaces":[{"interface":"back","average":{"1min":0.96699999999999997,"5min":2.0720000000000001,"15min":1.522},"min":{"1min":0.32200000000000001,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":2.1259999999999999,"5min":24.614000000000001,"15min":25.359000000000002},"last":9.8480000000000008},{"interface":"front","average":{"1min":1.0429999999999999,"5min":2.4220000000000002,"15min":1.8129999999999999},"min":{"1min":0.44,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":3.0550000000000002,"5min":24.669,"15min":24.669},"last":2.984}]}]},{"osd":2,"up_from":104,"seq":446676598931,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2714584,"kb_used_data":2257488,"kb_used_omap":11,"kb_used_meta":457076,"kb_avail":91030568,"statfs":{"total":95995035648,"available":93215301632,"internally_reserved":0,"allocated":2311667712,"data_stored":2310041411,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11878,"internal_metadata":468046234},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1,2,3],"upper_bound":32},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:32:39 2023","interfaces":[{"interface":"back","average":{"1min":5.2270000000000003,"5min":3.032,"15min":1.736},"min":{"1min":0.185,"5min":0.17999999999999999,"15min":0.17999999999999999},"max":{"1min":82.424999999999997,"5min":82.424999999999997,"15min":82.424999999999997},"last":6.218},{"interface":"front","average":{"1min":5.2329999999999997,"5min":3.3359999999999999,"15min":1.6919999999999999},"min":{"1min":0.27400000000000002,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":82.524000000000001,"5min":82.524000000000001,"15min":82.524000000000001},"last":0.36199999999999999}]},{"osd":1,"last update":"Sat Sep 23 20:32:49 2023","interfaces":[{"interface":"back","average":{"1min":1.782,"5min":3.5790000000000002,"15min":4.4269999999999996},"min":{"1min":0.245,"5min":0.221,"15min":0.221},"max":{"1min":6.6589999999999998,"5min":16.280000000000001,"15min":16.280000000000001},"last":0.28799999999999998},{"interface":"front","average":{"1min":1.696,"5min":2.399,"15min":2.7909999999999999},"min":{"1min":0.28000000000000003,"5min":0.248,"15min":0.248},"max":{"1min":5.1790000000000003,"5min":30.565000000000001,"15min":30.565000000000001},"last":6.181}]},{"osd":3,"last update":"Sat Sep 23 20:33:04 2023","interfaces":[{"interface":"back","average":{"1min":2.6699999999999999,"5min":2.4129999999999998,"15min":1.456},"min":{"1min":0.31900000000000001,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":11.766999999999999,"5min":16.331,"15min":16.331},"last":0.317},{"interface":"front","average":{"1min":2.859,"5min":2.786,"15min":1.6879999999999999},"min":{"1min":0.28999999999999998,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":15.266,"5min":30.704999999999998,"15min":30.704999999999998},"last":6.1310000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:32:39 2023","interfaces":[{"interface":"back","average":{"1min":1.5920000000000001,"5min":2.1629999999999998,"15min":1.385},"min":{"1min":0.36399999999999999,"5min":0.32800000000000001,"15min":0.28100000000000003},"max":{"1min":4.2549999999999999,"5min":16.797999999999998,"15min":16.797999999999998},"last":0.432},{"interface":"front","average":{"1min":1.2250000000000001,"5min":2.1139999999999999,"15min":1.369},"min":{"1min":0.39200000000000002,"5min":0.28799999999999998,"15min":0.27600000000000002},"max":{"1min":3.8300000000000001,"5min":14.971,"15min":14.971},"last":0.57799999999999996}]},{"osd":5,"last update":"Sat Sep 23 20:32:39 2023","interfaces":[{"interface":"back","average":{"1min":1.496,"5min":3.3999999999999999,"15min":1.8999999999999999},"min":{"1min":0.32200000000000001,"5min":0.314,"15min":0.309},"max":{"1min":4.3289999999999997,"5min":95.393000000000001,"15min":95.393000000000001},"last":1.2949999999999999},{"interface":"front","average":{"1min":1.413,"5min":3.4740000000000002,"15min":1.9359999999999999},"min":{"1min":0.48099999999999998,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":6.0389999999999997,"5min":95.414000000000001,"15min":95.414000000000001},"last":1.216}]},{"osd":6,"last update":"Sat Sep 23 20:32:39 2023","interfaces":[{"interface":"back","average":{"1min":1.238,"5min":2.3959999999999999,"15min":1.4650000000000001},"min":{"1min":0.42899999999999999,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":3.8540000000000001,"5min":15.023999999999999,"15min":15.023999999999999},"last":0.54400000000000004},{"interface":"front","average":{"1min":1.3460000000000001,"5min":2.0910000000000002,"15min":1.4970000000000001},"min":{"1min":0.40000000000000002,"5min":0.316,"15min":0.316},"max":{"1min":6,"5min":30.670999999999999,"15min":30.670999999999999},"last":7.742}]},{"osd":7,"last update":"Sat Sep 23 20:32:39 2023","interfaces":[{"interface":"back","average":{"1min":1.2969999999999999,"5min":2.1539999999999999,"15min":1.554},"min":{"1min":0.43099999999999999,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":5.9649999999999999,"5min":30.741,"15min":30.741},"last":12.734999999999999},{"interface":"front","average":{"1min":1.651,"5min":2.4980000000000002,"15min":1.6160000000000001},"min":{"1min":0.42799999999999999,"5min":0.38,"15min":0.38},"max":{"1min":4.2999999999999998,"5min":16.719000000000001,"15min":16.719000000000001},"last":0.46800000000000003}]}]},{"osd":3,"up_from":109,"seq":468151435405,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4025880,"kb_used_data":3536080,"kb_used_omap":4,"kb_used_meta":489787,"kb_avail":89719272,"statfs":{"total":95995035648,"available":91872534528,"internally_reserved":0,"allocated":3620945920,"data_stored":3619318292,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4514,"internal_metadata":501542494},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,3],"upper_bound":16},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:33:09 2023","interfaces":[{"interface":"back","average":{"1min":3.3149999999999999,"5min":2.2759999999999998,"15min":1.748},"min":{"1min":0.191,"5min":0.186,"15min":0.186},"max":{"1min":14.808999999999999,"5min":14.808999999999999,"15min":67.305999999999997},"last":6.9580000000000002},{"interface":"front","average":{"1min":3.016,"5min":2.2629999999999999,"15min":1.629},"min":{"1min":0.22600000000000001,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":7.9880000000000004,"5min":13.743,"15min":67.403999999999996},"last":6.5979999999999999}]},{"osd":1,"last update":"Sat Sep 23 20:32:51 2023","interfaces":[{"interface":"back","average":{"1min":1.617,"5min":2.6579999999999999,"15min":2.944},"min":{"1min":0.27100000000000002,"5min":0.222,"15min":0.222},"max":{"1min":5.8380000000000001,"5min":13.263,"15min":13.263},"last":0.25},{"interface":"front","average":{"1min":3.4620000000000002,"5min":2.988,"15min":3.1640000000000001},"min":{"1min":0.26400000000000001,"5min":0.252,"15min":0.252},"max":{"1min":32.603999999999999,"5min":32.603999999999999,"15min":32.603999999999999},"last":0.52800000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:33:09 2023","interfaces":[{"interface":"back","average":{"1min":4.6150000000000002,"5min":2.8679999999999999,"15min":1.7410000000000001},"min":{"1min":0.33200000000000002,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":32.857999999999997,"5min":32.857999999999997,"15min":32.857999999999997},"last":0.621},{"interface":"front","average":{"1min":2.6280000000000001,"5min":2.6219999999999999,"15min":1.6830000000000001},"min":{"1min":0.26900000000000002,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":13.568,"5min":19.603000000000002,"15min":19.603000000000002},"last":4.6660000000000004}]},{"osd":4,"last update":"Sat Sep 23 20:33:09 2023","interfaces":[{"interface":"back","average":{"1min":1.524,"5min":1.861,"15min":1.476},"min":{"1min":0.39100000000000001,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":5.3029999999999999,"5min":11.802,"15min":46.042999999999999},"last":0.66000000000000003},{"interface":"front","average":{"1min":1.6040000000000001,"5min":2.2919999999999998,"15min":1.7310000000000001},"min":{"1min":0.36399999999999999,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":7.9409999999999998,"5min":19.635000000000002,"15min":46.579999999999998},"last":4.2380000000000004}]},{"osd":5,"last update":"Sat Sep 23 20:33:09 2023","interfaces":[{"interface":"back","average":{"1min":1.581,"5min":2.04,"15min":1.6699999999999999},"min":{"1min":0.32200000000000001,"5min":0.31900000000000001,"15min":0.311},"max":{"1min":5.3280000000000003,"5min":11.507999999999999,"15min":45.292000000000002},"last":0.63600000000000001},{"interface":"front","average":{"1min":2.2120000000000002,"5min":2.2890000000000001,"15min":1.5740000000000001},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.313},"max":{"1min":6.9790000000000001,"5min":13.428000000000001,"15min":44.679000000000002},"last":0.59699999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:33:09 2023","interfaces":[{"interface":"back","average":{"1min":2.1920000000000002,"5min":2.419,"15min":1.591},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":6.9240000000000004,"5min":17.524999999999999,"15min":17.524999999999999},"last":0.56499999999999995},{"interface":"front","average":{"1min":1.2270000000000001,"5min":2.2330000000000001,"15min":1.52},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":6.7800000000000002,"5min":19.524000000000001,"15min":19.524000000000001},"last":4.1950000000000003}]},{"osd":7,"last update":"Sat Sep 23 20:33:09 2023","interfaces":[{"interface":"back","average":{"1min":1.24,"5min":2.6720000000000002,"15min":1.6930000000000001},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.34999999999999998},"max":{"1min":6.8079999999999998,"5min":19.571999999999999,"15min":19.571999999999999},"last":4.2699999999999996},{"interface":"front","average":{"1min":1.508,"5min":1.9390000000000001,"15min":1.7210000000000001},"min":{"1min":0.42599999999999999,"5min":0.371,"15min":0.371},"max":{"1min":5.2690000000000001,"5min":11.856999999999999,"15min":45.387999999999998},"last":0.68000000000000005}]}]},{"osd":4,"up_from":30,"seq":128849019159,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":844744,"kb_used_data":625920,"kb_used_omap":0,"kb_used_meta":218816,"kb_avail":92900408,"statfs":{"total":95995035648,"available":95130017792,"internally_reserved":0,"allocated":640942080,"data_stored":639912493,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":224067584},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:32:36 2023","interfaces":[{"interface":"back","average":{"1min":0.73099999999999998,"5min":1.3899999999999999,"15min":1.22},"min":{"1min":0.33600000000000002,"5min":0.24199999999999999,"15min":0.222},"max":{"1min":1.7450000000000001,"5min":8.984,"15min":20.411999999999999},"last":3.4079999999999999},{"interface":"front","average":{"1min":0.90400000000000003,"5min":1.5880000000000001,"15min":1.331},"min":{"1min":0.36399999999999999,"5min":0.25800000000000001,"15min":0.253},"max":{"1min":3.0259999999999998,"5min":11.868,"15min":20.454999999999998},"last":3.4209999999999998}]},{"osd":1,"last update":"Sat Sep 23 20:32:52 2023","interfaces":[{"interface":"back","average":{"1min":1.1319999999999999,"5min":1.7170000000000001,"15min":1.911},"min":{"1min":0.29099999999999998,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":5.3940000000000001,"5min":8.6799999999999997,"15min":8.6799999999999997},"last":0.45200000000000001},{"interface":"front","average":{"1min":0.94299999999999995,"5min":1.6910000000000001,"15min":1.907},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":2.4870000000000001,"5min":8.7219999999999995,"15min":8.7219999999999995},"last":0.35499999999999998}]},{"osd":2,"last update":"Sat Sep 23 20:32:41 2023","interfaces":[{"interface":"back","average":{"1min":1.258,"5min":1.994,"15min":1.4710000000000001},"min":{"1min":0.36599999999999999,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":4.1500000000000004,"5min":32.363,"15min":32.363},"last":4.2839999999999998},{"interface":"front","average":{"1min":1.093,"5min":2.2170000000000001,"15min":1.635},"min":{"1min":0.375,"5min":0.27600000000000002,"15min":0.26800000000000002},"max":{"1min":3.782,"5min":34.042999999999999,"15min":34.042999999999999},"last":7.8250000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:33:05 2023","interfaces":[{"interface":"back","average":{"1min":2.004,"5min":2.4199999999999999,"15min":1.663},"min":{"1min":0.34799999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":6.5090000000000003,"5min":32.332999999999998,"15min":32.332999999999998},"last":0.42799999999999999},{"interface":"front","average":{"1min":1.5629999999999999,"5min":2.4119999999999999,"15min":1.673},"min":{"1min":0.36299999999999999,"5min":0.318,"15min":0.30499999999999999},"max":{"1min":4.968,"5min":32.399999999999999,"15min":32.399999999999999},"last":7.5019999999999998}]},{"osd":5,"last update":"Sat Sep 23 20:32:25 2023","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":2.0609999999999999,"15min":1.484},"min":{"1min":0.31900000000000001,"5min":0.25800000000000001,"15min":0.247},"max":{"1min":1.095,"5min":82.866,"15min":82.866},"last":0.39600000000000002},{"interface":"front","average":{"1min":0.48299999999999998,"5min":1.9370000000000001,"15min":1.3700000000000001},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.27800000000000002},"max":{"1min":0.83099999999999996,"5min":82.546000000000006,"15min":82.546000000000006},"last":0.46700000000000003}]},{"osd":6,"last update":"Sat Sep 23 20:32:41 2023","interfaces":[{"interface":"back","average":{"1min":0.68500000000000005,"5min":1.139,"15min":1.242},"min":{"1min":0.33300000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":2.9820000000000002,"5min":6.3620000000000001,"15min":41.371000000000002},"last":0.38300000000000001},{"interface":"front","average":{"1min":0.52600000000000002,"5min":1.175,"15min":1.1839999999999999},"min":{"1min":0.35099999999999998,"5min":0.318,"15min":0.27200000000000002},"max":{"1min":0.96099999999999997,"5min":11.199,"15min":41.140999999999998},"last":0.47799999999999998}]},{"osd":7,"last update":"Sat Sep 23 20:33:05 2023","interfaces":[{"interface":"back","average":{"1min":1.024,"5min":1.4650000000000001,"15min":1.325},"min":{"1min":0.371,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":4.4349999999999996,"5min":15.667999999999999,"15min":16.427},"last":0.44},{"interface":"front","average":{"1min":0.876,"5min":1.347,"15min":1.2549999999999999},"min":{"1min":0.38300000000000001,"5min":0.307,"15min":0.307},"max":{"1min":5.4889999999999999,"5min":15.58,"15min":19.018999999999998},"last":0.497}]}]},{"osd":5,"up_from":39,"seq":167503724820,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1747728,"kb_used_data":1501064,"kb_used_omap":0,"kb_used_meta":246656,"kb_avail":91997424,"statfs":{"total":95995035648,"available":94205362176,"internally_reserved":0,"allocated":1537089536,"data_stored":1536056663,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":252575744},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:32:36 2023","interfaces":[{"interface":"back","average":{"1min":0.746,"5min":1.4610000000000001,"15min":1.2430000000000001},"min":{"1min":0.254,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":2.129,"5min":8.1609999999999996,"15min":32.475000000000001},"last":0.71499999999999997},{"interface":"front","average":{"1min":0.80300000000000005,"5min":1.4490000000000001,"15min":1.2390000000000001},"min":{"1min":0.30199999999999999,"5min":0.29199999999999998,"15min":0.20599999999999999},"max":{"1min":2.4569999999999999,"5min":8.4900000000000002,"15min":32.457000000000001},"last":0.76000000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:32:51 2023","interfaces":[{"interface":"back","average":{"1min":1.1459999999999999,"5min":1.702,"15min":1.9370000000000001},"min":{"1min":0.32900000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":5.2709999999999999,"5min":8.5839999999999996,"15min":8.5839999999999996},"last":1.3160000000000001},{"interface":"front","average":{"1min":0.878,"5min":1.3080000000000001,"15min":1.476},"min":{"1min":0.30399999999999999,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":4.577,"5min":6.7969999999999997,"15min":6.7969999999999997},"last":0.28999999999999998}]},{"osd":2,"last update":"Sat Sep 23 20:32:36 2023","interfaces":[{"interface":"back","average":{"1min":0.96099999999999997,"5min":1.7,"15min":1.464},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.26700000000000002},"max":{"1min":3.609,"5min":18.472999999999999,"15min":34.514000000000003},"last":3.169},{"interface":"front","average":{"1min":1.0840000000000001,"5min":1.8360000000000001,"15min":1.4179999999999999},"min":{"1min":0.34499999999999997,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":5.5309999999999997,"5min":8.6820000000000004,"15min":32.515000000000001},"last":13.654}]},{"osd":3,"last update":"Sat Sep 23 20:33:08 2023","interfaces":[{"interface":"back","average":{"1min":1.6419999999999999,"5min":2.0870000000000002,"15min":1.47},"min":{"1min":0.29599999999999999,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":6.0099999999999998,"5min":12.51,"15min":12.846},"last":0.73199999999999998},{"interface":"front","average":{"1min":1.4670000000000001,"5min":1.625,"15min":1.325},"min":{"1min":0.36799999999999999,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":5.7430000000000003,"5min":8.3729999999999993,"15min":12.005000000000001},"last":0.42699999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:32:27 2023","interfaces":[{"interface":"back","average":{"1min":0.41699999999999998,"5min":1.4339999999999999,"15min":1.2729999999999999},"min":{"1min":0.28599999999999998,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.66300000000000003,"5min":28.015999999999998,"15min":28.015999999999998},"last":0.34899999999999998},{"interface":"front","average":{"1min":0.49099999999999999,"5min":1.0609999999999999,"15min":1.0409999999999999},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.29099999999999998},"max":{"1min":0.96599999999999997,"5min":12.300000000000001,"15min":12.300000000000001},"last":0.39700000000000002}]},{"osd":6,"last update":"Sat Sep 23 20:32:41 2023","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":1.2090000000000001,"15min":1.179},"min":{"1min":0.32200000000000001,"5min":0.30599999999999999,"15min":0.30499999999999999},"max":{"1min":1.6080000000000001,"5min":7.7519999999999998,"15min":12.890000000000001},"last":0.38600000000000001},{"interface":"front","average":{"1min":0.57499999999999996,"5min":1.1850000000000001,"15min":1.1559999999999999},"min":{"1min":0.42099999999999999,"5min":0.36399999999999999,"15min":0.34499999999999997},"max":{"1min":1.105,"5min":16.265999999999998,"15min":16.265999999999998},"last":0.45300000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:33:08 2023","interfaces":[{"interface":"back","average":{"1min":0.621,"5min":1.026,"15min":1.2170000000000001},"min":{"1min":0.36899999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":2.6520000000000001,"5min":7.343,"15min":36.621000000000002},"last":0.36399999999999999},{"interface":"front","average":{"1min":0.67100000000000004,"5min":1.0640000000000001,"15min":1.107},"min":{"1min":0.35299999999999998,"5min":0.33500000000000002,"15min":0.315},"max":{"1min":2.415,"5min":13.122,"15min":37.331000000000003},"last":0.437}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4722,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1360,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8206,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7156,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3154,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":3076653056,"data_stored":3076653056,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2649227264,"data_stored":2649227264,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2241003520,"data_stored":2241003520,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3633840128,"data_stored":3633840128,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":644874240,"data_stored":644874240,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1537671168,"data_stored":1537671168,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2472804352,"data_stored":2472804352,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2239692800,"data_stored":2239692800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:33:27.823 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-09-23T20:33:27.824 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd pool set .mgr pg_num 31 2023-09-23T20:33:27.881 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:33:27.915 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:27 smithi149 ceph-mon[170394]: from='client.45644 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:33:27.921 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:27.963 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:33:27.967 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:27.967 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:33:28.068 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:33:28.111 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:28.348 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:33:28.352 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:28.352 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:33:28.453 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:33:28.532 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:28.662 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:28.663 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:33:28.663 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:33:28.764 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:33:28.801 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:28.845 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:33:28.906 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:33:28.910 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:28.910 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:33:28.963 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:28 smithi018 ceph-mon[179647]: pgmap v767: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 48 MiB/s wr, 780 op/s; 5117/280959 objects degraded (1.821%); 2360/280959 objects misplaced (0.840%); 3.0 MiB/s, 47 objects/s recovering 2023-09-23T20:33:28.963 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:28 smithi018 ceph-mon[184144]: pgmap v767: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 48 MiB/s wr, 780 op/s; 5117/280959 objects degraded (1.821%); 2360/280959 objects misplaced (0.840%); 3.0 MiB/s, 47 objects/s recovering 2023-09-23T20:33:28.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:28 smithi149 ceph-mon[170394]: pgmap v767: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 48 MiB/s wr, 780 op/s; 5117/280959 objects degraded (1.821%); 2360/280959 objects misplaced (0.840%); 3.0 MiB/s, 47 objects/s recovering 2023-09-23T20:33:28.978 INFO:teuthology.orchestra.run.smithi149.stdout:ERROR: (22) Invalid argument 2023-09-23T20:33:28.978 INFO:teuthology.orchestra.run.smithi149.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:33:29.011 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:33:29.053 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:29.104 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:33:29.109 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:29.109 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:33:29.210 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:33:29.275 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:33:29.312 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:29.394 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:33:29.398 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:29.399 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:33:29.404 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:33:29.441 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:33:29.488 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T20:33:29.493 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:33:29.499 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:33:29.536 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:29.551 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:33:29.582 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:33:29.586 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:33:29.587 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:29.587 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:33:29.627 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:33:29.662 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:33:29.688 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:33:29.704 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:33:29.706 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:33:29.707 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:33:29.707 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:33:29.707 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:33:29.707 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:33:29.707 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:33:29.707 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:33:29.707 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:33:29.719 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:29.762 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:33:29.766 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:29.766 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:33:29.867 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:33:29.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:29 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3355020524' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:33:29.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:29 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:33:29.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:29 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:33:29.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:29 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3355020524' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:33:29.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:29 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:33:29.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:29 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:33:29.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:29 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3355020524' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:33:29.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:29 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:33:29.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:29 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:33:29.968 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:30.479 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:33:30.486 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:30.487 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:33:30.587 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:33:30.616 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:30.662 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:33:30.665 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:30.665 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:33:30.766 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:33:30.849 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:30.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:30 smithi149 ceph-mon[170394]: pgmap v768: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 43 MiB/s wr, 722 op/s; 5117/281193 objects degraded (1.820%); 2360/281193 objects misplaced (0.839%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T20:33:31.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:30 smithi018 ceph-mon[179647]: pgmap v768: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 43 MiB/s wr, 722 op/s; 5117/281193 objects degraded (1.820%); 2360/281193 objects misplaced (0.839%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T20:33:31.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:30 smithi018 ceph-mon[184144]: pgmap v768: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 43 MiB/s wr, 722 op/s; 5117/281193 objects degraded (1.820%); 2360/281193 objects misplaced (0.839%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T20:33:32.146 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:31 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 5117/281193 objects degraded (1.820%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T20:33:32.147 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:31 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 5117/281193 objects degraded (1.820%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T20:33:32.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:31 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 5117/281193 objects degraded (1.820%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T20:33:32.372 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:33:32.476 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:33:32.496 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:32.497 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:33:32.598 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:33:32.626 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:32.669 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:33:32.673 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:32.673 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:33:32.744 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:32 smithi018 ceph-mon[179647]: pgmap v769: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 102 B/s rd, 33 MiB/s wr, 710 op/s; 5117/278142 objects degraded (1.840%); 2360/278142 objects misplaced (0.848%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T20:33:32.774 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:33:32.844 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:32.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:32 smithi149 ceph-mon[170394]: pgmap v769: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 102 B/s rd, 33 MiB/s wr, 710 op/s; 5117/278142 objects degraded (1.840%); 2360/278142 objects misplaced (0.848%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T20:33:33.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:32 smithi018 ceph-mon[184144]: pgmap v769: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 5.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 102 B/s rd, 33 MiB/s wr, 710 op/s; 5117/278142 objects degraded (1.840%); 2360/278142 objects misplaced (0.848%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T20:33:33.262 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:33:33.529 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:33:33.534 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:33.535 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:33:33.635 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:33:33.715 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:34.721 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:33:34.730 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:34.731 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:33:34.832 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:33:34.943 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:34 smithi018 ceph-mon[179647]: pgmap v770: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 5.6 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 102 B/s rd, 27 MiB/s wr, 707 op/s; 5117/275385 objects degraded (1.858%); 2121/275385 objects misplaced (0.770%); 2.9 MiB/s, 46 objects/s recovering 2023-09-23T20:33:34.943 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:34 smithi018 ceph-mon[184144]: pgmap v770: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 5.6 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 102 B/s rd, 27 MiB/s wr, 707 op/s; 5117/275385 objects degraded (1.858%); 2121/275385 objects misplaced (0.770%); 2.9 MiB/s, 46 objects/s recovering 2023-09-23T20:33:34.944 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:35.039 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:33:35.057 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:35.058 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:33:35.158 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:33:35.211 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:33:34 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:33:34] "GET /metrics HTTP/1.1" 200 36550 "" "Prometheus/2.43.0" 2023-09-23T20:33:35.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:34 smithi149 ceph-mon[170394]: pgmap v770: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 5.6 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 102 B/s rd, 27 MiB/s wr, 707 op/s; 5117/275385 objects degraded (1.858%); 2121/275385 objects misplaced (0.770%); 2.9 MiB/s, 46 objects/s recovering 2023-09-23T20:33:35.255 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:35.463 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:33:35.472 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:35.472 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:33:35.532 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:33:35.574 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:33:35.662 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:35.835 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:33:35.842 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:35.843 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:33:35.898 INFO:teuthology.orchestra.run.smithi018.stderr:set pool 1 pg_num to 31 2023-09-23T20:33:35.901 INFO:teuthology.orchestra.run.smithi018.stderr:noscrub is unset 2023-09-23T20:33:35.944 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:33:36.065 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:36.083 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:35 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3933454514' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:33:36.083 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:35 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1610754676' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"}]: dispatch 2023-09-23T20:33:36.083 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:35 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"}]: dispatch 2023-09-23T20:33:36.089 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:35 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3933454514' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:33:36.089 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:35 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1610754676' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"}]: dispatch 2023-09-23T20:33:36.089 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:35 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"}]: dispatch 2023-09-23T20:33:36.153 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:33:36.158 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:36.158 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:33:36.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:35 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3933454514' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:33:36.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:35 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1610754676' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"}]: dispatch 2023-09-23T20:33:36.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:35 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"}]: dispatch 2023-09-23T20:33:36.260 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:33:36.298 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:36.363 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:33:36.369 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:36.369 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:33:36.470 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:33:36.499 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:36.577 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:33:36.583 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:36.584 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:33:36.685 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:33:36.775 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:36.886 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2023-09-23T20:33:36.887 INFO:teuthology.orchestra.run.smithi018.stdout: "ops": [ 2023-09-23T20:33:36.887 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:33:36.887 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:101743 7.2 7:4202eed7:::benchmark_data_smithi018_308601_object7191:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e331)", 2023-09-23T20:33:36.887 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:33:36.851435+0000", 2023-09-23T20:33:36.887 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.033427802, 2023-09-23T20:33:36.887 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.033455374000000003, 2023-09-23T20:33:36.887 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:33:36.887 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:33:36.887 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:33:36.887 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:33:36.888 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:33:36.888 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 101743 2023-09-23T20:33:36.888 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:33:36.888 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:33:36.888 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:33:36.888 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:33:36.888 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:33:36.851435+0000", 2023-09-23T20:33:36.888 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:33:36.888 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:33:36.888 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:33:36.888 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:33:36.889 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:33:36.851435+0000", 2023-09-23T20:33:36.889 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:33:36.889 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:33:36.889 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:33:36.889 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:33:36.889 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:33:36.851433+0000", 2023-09-23T20:33:36.889 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:33:36.889 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:33:36.889 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:33:36.889 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:33:36.889 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:33:36.851442+0000", 2023-09-23T20:33:36.890 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.3200000000000006e-06 2023-09-23T20:33:36.890 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:33:36.890 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:33:36.890 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:33:36.890 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:33:36.851463+0000", 2023-09-23T20:33:36.890 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.1027999999999999e-05 2023-09-23T20:33:36.890 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:33:36.890 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:33:36.890 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:33:36.890 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:33:36.851474+0000", 2023-09-23T20:33:36.890 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0644e-05 2023-09-23T20:33:36.891 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:33:36.891 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:33:36.891 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:33:36.891 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:33:36.851550+0000", 2023-09-23T20:33:36.891 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.6787999999999997e-05 2023-09-23T20:33:36.891 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:33:36.891 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:33:36.891 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:33:36.891 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:33:36.851678+0000", 2023-09-23T20:33:36.891 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00012793400000000001 2023-09-23T20:33:36.891 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:33:36.892 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:33:36.892 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 2,3", 2023-09-23T20:33:36.892 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:33:36.851787+0000", 2023-09-23T20:33:36.892 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00010821500000000001 2023-09-23T20:33:36.892 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:33:36.892 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:33:36.892 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:33:36.892 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:33:36.859373+0000", 2023-09-23T20:33:36.892 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0075861160000000004 2023-09-23T20:33:36.892 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:33:36.892 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:33:36.892 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:33:36.893 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:33:36.893 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:33:36.893 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_repop(client.35700.0:101756 7.d e331/303 7:b45791b4:::benchmark_data_smithi018_308601_object7204:head v 331'6460, mlcod=331'6458)", 2023-09-23T20:33:36.893 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:33:36.871692+0000", 2023-09-23T20:33:36.893 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.013170668999999999, 2023-09-23T20:33:36.893 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.013254774, 2023-09-23T20:33:36.893 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:33:36.893 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "started", 2023-09-23T20:33:36.893 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:33:36.893 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:33:36.893 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:33:36.894 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:33:36.871692+0000", 2023-09-23T20:33:36.894 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:33:36.894 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:33:36.894 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:33:36.894 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:33:36.894 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:33:36.871692+0000", 2023-09-23T20:33:36.894 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:33:36.894 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:33:36.894 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:33:36.894 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:33:36.894 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:33:36.871690+0000", 2023-09-23T20:33:36.895 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:33:36.895 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:33:36.895 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:33:36.895 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:33:36.895 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:33:36.871704+0000", 2023-09-23T20:33:36.895 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3834e-05 2023-09-23T20:33:36.895 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:33:36.895 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:33:36.895 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:33:36.895 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:33:36.871725+0000", 2023-09-23T20:33:36.895 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.1440000000000001e-05 2023-09-23T20:33:36.895 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:33:36.896 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:33:36.896 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:33:36.896 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:33:36.871740+0000", 2023-09-23T20:33:36.896 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.5204999999999999e-05 2023-09-23T20:33:36.896 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:33:36.896 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:33:36.896 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:33:36.896 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:33:36.877217+0000", 2023-09-23T20:33:36.896 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0054766720000000001 2023-09-23T20:33:36.896 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:33:36.896 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:33:36.897 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:33:36.897 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:33:36.877242+0000", 2023-09-23T20:33:36.897 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.4944999999999999e-05 2023-09-23T20:33:36.897 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:33:36.897 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:33:36.897 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:33:36.897 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:33:36.897 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:33:36.897 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_repop(client.35700.0:101753 7.1 e331/323 7:855d7dd1:::benchmark_data_smithi018_308601_object7201:head v 331'6303, mlcod=331'6301)", 2023-09-23T20:33:36.897 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:33:36.871768+0000", 2023-09-23T20:33:36.897 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.013094722, 2023-09-23T20:33:36.897 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.013217833999999999, 2023-09-23T20:33:36.898 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:33:36.898 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "started", 2023-09-23T20:33:36.898 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:33:36.898 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:33:36.898 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:33:36.898 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:33:36.871768+0000", 2023-09-23T20:33:36.898 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:33:36.898 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:33:36.898 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:33:36.898 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:33:36.898 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:33:36.871768+0000", 2023-09-23T20:33:36.899 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:33:36.899 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:33:36.899 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:33:36.899 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:33:36.899 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:33:36.871767+0000", 2023-09-23T20:33:36.899 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:33:36.899 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:33:36.899 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:33:36.899 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:33:36.899 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:33:36.871772+0000", 2023-09-23T20:33:36.899 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.843e-06 2023-09-23T20:33:36.899 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:33:36.900 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:33:36.900 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:33:36.900 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:33:36.871782+0000", 2023-09-23T20:33:36.900 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.4700000000000008e-06 2023-09-23T20:33:36.900 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:33:36.900 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:33:36.900 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:33:36.900 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:33:36.871803+0000", 2023-09-23T20:33:36.900 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.1352000000000002e-05 2023-09-23T20:33:36.900 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:33:36.900 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:33:36.901 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:33:36.901 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:33:36.871872+0000", 2023-09-23T20:33:36.901 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.9004000000000005e-05 2023-09-23T20:33:36.901 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:33:36.901 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:33:36.901 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:33:36.901 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:33:36.871907+0000", 2023-09-23T20:33:36.901 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.5518000000000001e-05 2023-09-23T20:33:36.901 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:33:36.901 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:33:36.901 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:33:36.901 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:33:36.902 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:33:36.902 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_repop(client.35700.0:101758 7.4 e331/322 7:2756b79b:::benchmark_data_smithi018_308601_object7206:head v 331'6261, mlcod=314'3606)", 2023-09-23T20:33:36.902 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:33:36.877111+0000", 2023-09-23T20:33:36.902 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.0077517929999999999, 2023-09-23T20:33:36.902 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0079111010000000002, 2023-09-23T20:33:36.902 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:33:36.902 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "started", 2023-09-23T20:33:36.902 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:33:36.902 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:33:36.902 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:33:36.902 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:33:36.877111+0000", 2023-09-23T20:33:36.903 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:33:36.903 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:33:36.903 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:33:36.903 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:33:36.903 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:33:36.877111+0000", 2023-09-23T20:33:36.903 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:33:36.903 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:33:36.903 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:33:36.903 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:33:36.903 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:33:36.877109+0000", 2023-09-23T20:33:36.903 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:33:36.903 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:33:36.904 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:33:36.904 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:33:36.904 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:33:36.877122+0000", 2023-09-23T20:33:36.904 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2748000000000001e-05 2023-09-23T20:33:36.904 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:33:36.904 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:33:36.904 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:33:36.904 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:33:36.877138+0000", 2023-09-23T20:33:36.904 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.5812e-05 2023-09-23T20:33:36.904 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:33:36.904 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:33:36.905 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:33:36.905 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:33:36.877148+0000", 2023-09-23T20:33:36.905 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0322999999999999e-05 2023-09-23T20:33:36.905 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:33:36.905 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:33:36.905 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:33:36.905 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:33:36.878471+0000", 2023-09-23T20:33:36.905 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0013226970000000001 2023-09-23T20:33:36.905 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:33:36.905 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:33:36.905 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:33:36.905 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:33:36.878500+0000", 2023-09-23T20:33:36.906 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.951e-05 2023-09-23T20:33:36.906 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:33:36.906 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:33:36.906 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:33:36.906 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:33:36.906 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:33:36.906 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_repop(client.35700.0:101750 7.9 e331/322)", 2023-09-23T20:33:36.906 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:33:36.878332+0000", 2023-09-23T20:33:36.906 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.0065309290000000004, 2023-09-23T20:33:36.906 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0067304629999999999, 2023-09-23T20:33:36.906 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:33:36.907 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "queued for pg", 2023-09-23T20:33:36.907 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:33:36.907 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:33:36.907 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:33:36.907 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:33:36.878332+0000", 2023-09-23T20:33:36.907 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:33:36.907 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:33:36.907 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:33:36.907 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:33:36.907 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:33:36.878332+0000", 2023-09-23T20:33:36.907 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:33:36.907 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:33:36.908 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:33:36.908 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:33:36.908 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:33:36.878330+0000", 2023-09-23T20:33:36.908 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:33:36.908 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:33:36.908 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:33:36.908 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:33:36.908 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:33:36.878342+0000", 2023-09-23T20:33:36.908 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.119e-05 2023-09-23T20:33:36.908 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:33:36.908 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:33:36.909 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:33:36.909 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:33:36.878357+0000", 2023-09-23T20:33:36.909 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.5133000000000001e-05 2023-09-23T20:33:36.909 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:33:36.909 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:33:36.909 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:33:36.909 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:33:36.878366+0000", 2023-09-23T20:33:36.909 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.8130000000000007e-06 2023-09-23T20:33:36.909 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:33:36.909 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:33:36.909 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:33:36.911 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:33:36.911 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:33:36.911 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_repop(client.35700.0:101749 7.0 e331/303 7:00307c20:::benchmark_data_smithi018_308601_object7197:head v 331'6393, mlcod=331'6391)", 2023-09-23T20:33:36.911 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:33:36.879873+0000", 2023-09-23T20:33:36.911 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.0049899139999999998, 2023-09-23T20:33:36.911 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0052197459999999999, 2023-09-23T20:33:36.911 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:33:36.911 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "started", 2023-09-23T20:33:36.911 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:33:36.911 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:33:36.911 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:33:36.912 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:33:36.879873+0000", 2023-09-23T20:33:36.912 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:33:36.912 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:33:36.912 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:33:36.912 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:33:36.912 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:33:36.879873+0000", 2023-09-23T20:33:36.912 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:33:36.912 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:33:36.912 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:33:36.912 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:33:36.913 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:33:36.879872+0000", 2023-09-23T20:33:36.913 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.999999 2023-09-23T20:33:36.913 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:33:36.913 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:33:36.913 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:33:36.913 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:33:36.879881+0000", 2023-09-23T20:33:36.913 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.9220000000000003e-06 2023-09-23T20:33:36.913 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:33:36.913 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:33:36.913 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:33:36.914 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:33:36.879893+0000", 2023-09-23T20:33:36.914 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2638e-05 2023-09-23T20:33:36.914 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:33:36.914 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:33:36.914 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:33:36.914 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:33:36.879902+0000", 2023-09-23T20:33:36.914 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.2169999999999994e-06 2023-09-23T20:33:36.914 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:33:36.914 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:33:36.914 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:33:36.915 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:33:36.879979+0000", 2023-09-23T20:33:36.915 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.7526000000000003e-05 2023-09-23T20:33:36.915 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:33:36.915 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:33:36.915 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:33:36.915 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:33:36.880000+0000", 2023-09-23T20:33:36.915 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.0825000000000001e-05 2023-09-23T20:33:36.915 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:33:36.915 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:33:36.916 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:33:36.916 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:33:36.916 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:33:36.916 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_repop(client.35700.0:101754 7.0 e331/303 7:0edf4a56:::benchmark_data_smithi018_308601_object7202:head v 331'6394, mlcod=331'6391)", 2023-09-23T20:33:36.916 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:33:36.879916+0000", 2023-09-23T20:33:36.916 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.0049468760000000002, 2023-09-23T20:33:36.916 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0052121800000000003, 2023-09-23T20:33:36.916 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:33:36.916 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "started", 2023-09-23T20:33:36.916 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:33:36.916 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:33:36.917 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:33:36.917 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:33:36.879916+0000", 2023-09-23T20:33:36.917 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:33:36.917 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:33:36.917 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:33:36.917 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:33:36.917 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:33:36.879916+0000", 2023-09-23T20:33:36.917 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:33:36.917 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:33:36.917 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:33:36.917 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:33:36.917 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:33:36.879915+0000", 2023-09-23T20:33:36.918 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:33:36.918 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:33:36.918 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:33:36.918 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:33:36.918 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:33:36.879919+0000", 2023-09-23T20:33:36.918 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.9500000000000003e-06 2023-09-23T20:33:36.918 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:33:36.918 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:33:36.918 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:33:36.918 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:33:36.879925+0000", 2023-09-23T20:33:36.918 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.4999999999999999e-06 2023-09-23T20:33:36.919 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:33:36.919 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:33:36.919 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:33:36.919 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:33:36.879929+0000", 2023-09-23T20:33:36.919 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.1119999999999998e-06 2023-09-23T20:33:36.919 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:33:36.919 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:33:36.919 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:33:36.919 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:33:36.883105+0000", 2023-09-23T20:33:36.919 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0031760429999999999 2023-09-23T20:33:36.919 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:33:36.920 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:33:36.920 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:33:36.920 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:33:36.883126+0000", 2023-09-23T20:33:36.920 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.1203999999999999e-05 2023-09-23T20:33:36.920 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:33:36.920 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:33:36.920 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:33:36.920 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:33:36.920 INFO:teuthology.orchestra.run.smithi018.stdout: ], 2023-09-23T20:33:36.920 INFO:teuthology.orchestra.run.smithi018.stdout: "num_ops": 7 2023-09-23T20:33:36.920 INFO:teuthology.orchestra.run.smithi018.stdout:} 2023-09-23T20:33:37.147 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:36 smithi018 ceph-mon[179647]: pgmap v771: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 91 B/s rd, 21 MiB/s wr, 838 op/s; 5117/267021 objects degraded (1.916%); 2121/267021 objects misplaced (0.794%); 2.6 MiB/s, 41 objects/s recovering 2023-09-23T20:33:37.147 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:36 smithi018 ceph-mon[179647]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:33:37.147 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:36 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3933454514' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:33:37.147 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:36 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"}]': finished 2023-09-23T20:33:37.147 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:36 smithi018 ceph-mon[179647]: osdmap e331: 8 total, 8 up, 8 in 2023-09-23T20:33:37.147 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:36 smithi018 ceph-mon[179647]: failed to encode map e331 with expected crc 2023-09-23T20:33:37.147 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:36 smithi018 ceph-mon[179647]: failed to encode map e331 with expected crc 2023-09-23T20:33:37.147 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:36 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 5117/267021 objects degraded (1.916%), 3 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T20:33:37.148 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:36 smithi018 ceph-mon[184144]: pgmap v771: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 91 B/s rd, 21 MiB/s wr, 838 op/s; 5117/267021 objects degraded (1.916%); 2121/267021 objects misplaced (0.794%); 2.6 MiB/s, 41 objects/s recovering 2023-09-23T20:33:37.148 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:36 smithi018 ceph-mon[184144]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:33:37.148 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:36 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3933454514' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:33:37.148 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:36 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"}]': finished 2023-09-23T20:33:37.148 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:36 smithi018 ceph-mon[184144]: osdmap e331: 8 total, 8 up, 8 in 2023-09-23T20:33:37.148 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:36 smithi018 ceph-mon[184144]: failed to encode map e331 with expected crc 2023-09-23T20:33:37.148 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:36 smithi018 ceph-mon[184144]: failed to encode map e331 with expected crc 2023-09-23T20:33:37.148 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:36 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 5117/267021 objects degraded (1.916%), 3 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T20:33:37.187 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:33:37.195 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:37.196 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:33:37.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:36 smithi149 ceph-mon[170394]: pgmap v771: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 91 B/s rd, 21 MiB/s wr, 838 op/s; 5117/267021 objects degraded (1.916%); 2121/267021 objects misplaced (0.794%); 2.6 MiB/s, 41 objects/s recovering 2023-09-23T20:33:37.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:36 smithi149 ceph-mon[170394]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:33:37.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:36 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3933454514' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:33:37.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:36 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"}]': finished 2023-09-23T20:33:37.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:36 smithi149 ceph-mon[170394]: osdmap e331: 8 total, 8 up, 8 in 2023-09-23T20:33:37.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:36 smithi149 ceph-mon[170394]: failed to encode map e331 with expected crc 2023-09-23T20:33:37.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:36 smithi149 ceph-mon[170394]: failed to encode map e331 with expected crc 2023-09-23T20:33:37.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:36 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 5117/267021 objects degraded (1.916%), 3 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T20:33:37.297 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:33:37.444 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:37.467 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T20:33:37.557 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:33:37.566 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:37.567 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:33:37.667 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:33:37.700 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:37.748 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:33:37.752 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:37.752 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:33:37.853 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:33:37.887 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:37.946 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:33:37.948 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:37 smithi149 ceph-mon[170394]: failed to encode map e331 with expected crc 2023-09-23T20:33:37.948 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:37 smithi149 ceph-mon[170394]: 1.4 scrub starts 2023-09-23T20:33:37.948 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:37 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"}]: dispatch 2023-09-23T20:33:37.949 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:37.950 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:33:38.050 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:33:38.094 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:33:38.144 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:37 smithi018 ceph-mon[179647]: failed to encode map e331 with expected crc 2023-09-23T20:33:38.144 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:37 smithi018 ceph-mon[179647]: 1.4 scrub starts 2023-09-23T20:33:38.144 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:37 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"}]: dispatch 2023-09-23T20:33:38.145 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:37 smithi018 ceph-mon[184144]: failed to encode map e331 with expected crc 2023-09-23T20:33:38.145 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:37 smithi018 ceph-mon[184144]: 1.4 scrub starts 2023-09-23T20:33:38.145 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:37 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"}]: dispatch 2023-09-23T20:33:38.155 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:33:38.195 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:33:38.200 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:38.256 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:33:38.301 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:33:38.306 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:33:38.309 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:38.310 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:33:38.352 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:33:38.380 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:33:38.392 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:33:38.410 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:33:38.429 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:33:38.440 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:33:38.440 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:33:38.440 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:33:38.440 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:33:38.440 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:33:38.440 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:33:38.441 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:33:38.441 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:33:38.507 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T20:33:38.513 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:38.608 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:33:38.621 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:38.622 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:33:38.722 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:33:38.830 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:38.993 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:33:39.004 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:39.004 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:33:39.105 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:33:39.136 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:39.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:38 smithi018 ceph-mon[184144]: 1.4 scrub starts 2023-09-23T20:33:39.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:38 smithi018 ceph-mon[184144]: pgmap v773: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 102 B/s rd, 5.1 MiB/s wr, 715 op/s; 5117/264345 objects degraded (1.936%); 1872/264345 objects misplaced (0.708%); 3.0 MiB/s, 48 objects/s recovering 2023-09-23T20:33:39.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:38 smithi018 ceph-mon[184144]: failed to encode map e332 with expected crc 2023-09-23T20:33:39.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:38 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"}]': finished 2023-09-23T20:33:39.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:38 smithi018 ceph-mon[184144]: osdmap e332: 8 total, 8 up, 8 in 2023-09-23T20:33:39.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:38 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2249052260' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:33:39.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:38 smithi018 ceph-mon[179647]: 1.4 scrub starts 2023-09-23T20:33:39.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:38 smithi018 ceph-mon[179647]: pgmap v773: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 102 B/s rd, 5.1 MiB/s wr, 715 op/s; 5117/264345 objects degraded (1.936%); 1872/264345 objects misplaced (0.708%); 3.0 MiB/s, 48 objects/s recovering 2023-09-23T20:33:39.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:38 smithi018 ceph-mon[179647]: failed to encode map e332 with expected crc 2023-09-23T20:33:39.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:38 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"}]': finished 2023-09-23T20:33:39.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:38 smithi018 ceph-mon[179647]: osdmap e332: 8 total, 8 up, 8 in 2023-09-23T20:33:39.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:38 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2249052260' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:33:39.180 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:33:39.186 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:39.186 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:33:39.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:38 smithi149 ceph-mon[170394]: 1.4 scrub starts 2023-09-23T20:33:39.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:38 smithi149 ceph-mon[170394]: pgmap v773: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 102 B/s rd, 5.1 MiB/s wr, 715 op/s; 5117/264345 objects degraded (1.936%); 1872/264345 objects misplaced (0.708%); 3.0 MiB/s, 48 objects/s recovering 2023-09-23T20:33:39.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:38 smithi149 ceph-mon[170394]: failed to encode map e332 with expected crc 2023-09-23T20:33:39.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:38 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"}]': finished 2023-09-23T20:33:39.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:38 smithi149 ceph-mon[170394]: osdmap e332: 8 total, 8 up, 8 in 2023-09-23T20:33:39.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:38 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2249052260' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:33:39.287 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:33:39.322 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:39.362 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:33:39.365 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:39.366 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:33:39.466 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:33:39.505 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:39.550 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:33:39.553 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:39.554 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:33:39.654 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:33:39.665 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:33:39.694 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:39.740 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:33:39.744 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:39.745 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:33:39.846 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:33:39.886 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:39.931 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:33:39.936 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:39.936 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:33:40.037 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:33:40.068 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:40.110 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:33:40.114 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:40.114 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:33:40.215 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:33:40.268 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:39 smithi018 ceph-mon[179647]: osdmap e333: 8 total, 8 up, 8 in 2023-09-23T20:33:40.268 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:39 smithi018 ceph-mon[179647]: failed to encode map e333 with expected crc 2023-09-23T20:33:40.268 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:39 smithi018 ceph-mon[179647]: failed to encode map e333 with expected crc 2023-09-23T20:33:40.269 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:39 smithi018 ceph-mon[179647]: failed to encode map e333 with expected crc 2023-09-23T20:33:40.269 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:39 smithi018 ceph-mon[179647]: failed to encode map e333 with expected crc 2023-09-23T20:33:40.269 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:39 smithi018 ceph-mon[179647]: failed to encode map e333 with expected crc 2023-09-23T20:33:40.269 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:39 smithi018 ceph-mon[179647]: failed to encode map e333 with expected crc 2023-09-23T20:33:40.269 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:39 smithi018 ceph-mon[179647]: failed to encode map e333 with expected crc 2023-09-23T20:33:40.269 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:39 smithi018 ceph-mon[179647]: 1.a scrub starts 2023-09-23T20:33:40.271 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:39 smithi018 ceph-mon[184144]: osdmap e333: 8 total, 8 up, 8 in 2023-09-23T20:33:40.271 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:39 smithi018 ceph-mon[184144]: failed to encode map e333 with expected crc 2023-09-23T20:33:40.271 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:39 smithi018 ceph-mon[184144]: failed to encode map e333 with expected crc 2023-09-23T20:33:40.271 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:39 smithi018 ceph-mon[184144]: failed to encode map e333 with expected crc 2023-09-23T20:33:40.271 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:39 smithi018 ceph-mon[184144]: failed to encode map e333 with expected crc 2023-09-23T20:33:40.272 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:39 smithi018 ceph-mon[184144]: failed to encode map e333 with expected crc 2023-09-23T20:33:40.272 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:39 smithi018 ceph-mon[184144]: failed to encode map e333 with expected crc 2023-09-23T20:33:40.272 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:39 smithi018 ceph-mon[184144]: failed to encode map e333 with expected crc 2023-09-23T20:33:40.272 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:39 smithi018 ceph-mon[184144]: 1.a scrub starts 2023-09-23T20:33:40.335 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:40.412 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:33:40.423 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:40.423 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:33:40.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:39 smithi149 ceph-mon[170394]: osdmap e333: 8 total, 8 up, 8 in 2023-09-23T20:33:40.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:39 smithi149 ceph-mon[170394]: failed to encode map e333 with expected crc 2023-09-23T20:33:40.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:39 smithi149 ceph-mon[170394]: failed to encode map e333 with expected crc 2023-09-23T20:33:40.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:39 smithi149 ceph-mon[170394]: failed to encode map e333 with expected crc 2023-09-23T20:33:40.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:39 smithi149 ceph-mon[170394]: failed to encode map e333 with expected crc 2023-09-23T20:33:40.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:39 smithi149 ceph-mon[170394]: failed to encode map e333 with expected crc 2023-09-23T20:33:40.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:39 smithi149 ceph-mon[170394]: failed to encode map e333 with expected crc 2023-09-23T20:33:40.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:39 smithi149 ceph-mon[170394]: failed to encode map e333 with expected crc 2023-09-23T20:33:40.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:39 smithi149 ceph-mon[170394]: 1.a scrub starts 2023-09-23T20:33:40.524 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:33:40.599 INFO:teuthology.orchestra.run.smithi018.stdout:ERROR: (22) Invalid argument 2023-09-23T20:33:40.599 INFO:teuthology.orchestra.run.smithi018.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:33:40.646 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:40.999 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:33:41.009 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:41.009 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:33:41.043 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:33:41.110 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:33:41.133 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T20:33:41.140 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:41.181 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:33:41.186 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:41.186 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:33:41.238 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:41 smithi018 ceph-mon[184144]: pgmap v776: 119 pgs: 1 peering, 10 unknown, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 682 op/s; 5117/263088 objects degraded (1.945%); 1872/263088 objects misplaced (0.712%); 2.6 MiB/s, 41 objects/s recovering 2023-09-23T20:33:41.239 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:41 smithi018 ceph-mon[179647]: pgmap v776: 119 pgs: 1 peering, 10 unknown, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 682 op/s; 5117/263088 objects degraded (1.945%); 1872/263088 objects misplaced (0.712%); 2.6 MiB/s, 41 objects/s recovering 2023-09-23T20:33:41.287 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:33:41.410 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:41.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:41 smithi149 ceph-mon[170394]: pgmap v776: 119 pgs: 1 peering, 10 unknown, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 682 op/s; 5117/263088 objects degraded (1.945%); 1872/263088 objects misplaced (0.712%); 2.6 MiB/s, 41 objects/s recovering 2023-09-23T20:33:41.532 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:33:41.543 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:41.543 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:33:41.644 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:33:41.759 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:41.844 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:33:41.850 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:41.851 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:33:41.880 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 5, 7, 3, 2, 6, 4] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T20:33:41.881 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:33:41.881 INFO:tasks.thrashosds.thrasher:fixing pg num pool .mgr 2023-09-23T20:33:41.881 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:33:41.952 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:33:42.027 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:33:42.048 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:42.082 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:33:42.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:42 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 5117/263088 objects degraded (1.945%), 3 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T20:33:42.150 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:42 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 5117/263088 objects degraded (1.945%), 3 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T20:33:42.185 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:33:42.196 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:42.197 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:33:42.298 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:33:42.322 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:42 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 5117/263088 objects degraded (1.945%), 3 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T20:33:42.325 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:42.368 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:33:42.374 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:42.374 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:33:42.475 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:33:42.529 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:42.576 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:33:42.579 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:42.580 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:33:42.681 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:33:42.718 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:42.768 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:33:42.772 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:42.772 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:33:42.873 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:33:42.903 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:42.947 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:33:42.950 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:42.951 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:33:43.051 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:33:43.116 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:43.185 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:43 smithi149 ceph-mon[170394]: pgmap v777: 119 pgs: 1 peering, 1 unknown, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 5.2 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 578 op/s; 5117/256596 objects degraded (1.994%); 1872/256596 objects misplaced (0.730%); 2.6 MiB/s, 41 objects/s recovering 2023-09-23T20:33:43.186 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:43 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2854485356' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:33:43.186 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:43 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:33:43.235 INFO:teuthology.orchestra.run.smithi018.stderr:nodeep-scrub is unset 2023-09-23T20:33:43.263 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:33:43.279 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:43.280 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:33:43.381 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:33:43.537 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:43 smithi018 ceph-mon[179647]: pgmap v777: 119 pgs: 1 peering, 1 unknown, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 5.2 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 578 op/s; 5117/256596 objects degraded (1.994%); 1872/256596 objects misplaced (0.730%); 2.6 MiB/s, 41 objects/s recovering 2023-09-23T20:33:43.538 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:43 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2854485356' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:33:43.538 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:43 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:33:43.538 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:43 smithi018 ceph-mon[184144]: pgmap v777: 119 pgs: 1 peering, 1 unknown, 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 5.2 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 578 op/s; 5117/256596 objects degraded (1.994%); 1872/256596 objects misplaced (0.730%); 2.6 MiB/s, 41 objects/s recovering 2023-09-23T20:33:43.538 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:43 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2854485356' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:33:43.538 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:43 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:33:43.575 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:43.628 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:33:43.632 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:43.633 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:33:43.734 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:33:43.815 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:43.867 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:33:43.871 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:43.871 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:33:43.972 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:33:44.054 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:44.126 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:33:44.176 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:33:44.180 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:44.181 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:33:44.281 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:33:44.317 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:44.358 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:33:44.364 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:44.364 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:33:44.398 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:44 smithi018 ceph-mon[179647]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:33:44.398 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:44 smithi018 ceph-mon[179647]: failed to encode map e334 with expected crc 2023-09-23T20:33:44.398 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:44 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:33:44.398 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:44 smithi018 ceph-mon[179647]: osdmap e334: 8 total, 8 up, 8 in 2023-09-23T20:33:44.398 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:44 smithi018 ceph-mon[179647]: failed to encode map e334 with expected crc 2023-09-23T20:33:44.398 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:44 smithi018 ceph-mon[184144]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:33:44.398 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:44 smithi018 ceph-mon[184144]: failed to encode map e334 with expected crc 2023-09-23T20:33:44.399 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:44 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:33:44.399 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:44 smithi018 ceph-mon[184144]: osdmap e334: 8 total, 8 up, 8 in 2023-09-23T20:33:44.399 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:44 smithi018 ceph-mon[184144]: failed to encode map e334 with expected crc 2023-09-23T20:33:44.465 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:33:44.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:44 smithi149 ceph-mon[170394]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:33:44.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:44 smithi149 ceph-mon[170394]: failed to encode map e334 with expected crc 2023-09-23T20:33:44.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:44 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:33:44.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:44 smithi149 ceph-mon[170394]: osdmap e334: 8 total, 8 up, 8 in 2023-09-23T20:33:44.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:44 smithi149 ceph-mon[170394]: failed to encode map e334 with expected crc 2023-09-23T20:33:44.502 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:44.548 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:33:44.550 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:44.551 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:33:44.651 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:33:44.739 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:44.845 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:33:44.853 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:44.854 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:33:44.955 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:33:44.993 INFO:teuthology.orchestra.run.smithi018.stdout:ERROR: (22) Invalid argument 2023-09-23T20:33:44.993 INFO:teuthology.orchestra.run.smithi018.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:33:45.028 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:45.141 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:33:44 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:33:44] "GET /metrics HTTP/1.1" 200 36556 "" "Prometheus/2.43.0" 2023-09-23T20:33:45.240 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:33:45.249 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:45.249 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:33:45.350 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:33:45.445 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:45.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:45 smithi018 ceph-mon[184144]: pgmap v779: 119 pgs: 1 active+recovering+undersized+remapped, 1 peering, 1 unknown, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 563 op/s; 5117/254190 objects degraded (2.013%); 1577/254190 objects misplaced (0.620%); 3.1 MiB/s, 49 objects/s recovering 2023-09-23T20:33:45.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:45 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:33:45.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:45 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:33:45.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:45 smithi018 ceph-mon[179647]: pgmap v779: 119 pgs: 1 active+recovering+undersized+remapped, 1 peering, 1 unknown, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 563 op/s; 5117/254190 objects degraded (2.013%); 1577/254190 objects misplaced (0.620%); 3.1 MiB/s, 49 objects/s recovering 2023-09-23T20:33:45.645 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:45 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:33:45.645 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:45 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:33:45.674 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:33:45.678 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:45.678 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:33:45.691 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T20:33:45.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:45 smithi149 ceph-mon[170394]: pgmap v779: 119 pgs: 1 active+recovering+undersized+remapped, 1 peering, 1 unknown, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 563 op/s; 5117/254190 objects degraded (2.013%); 1577/254190 objects misplaced (0.620%); 3.1 MiB/s, 49 objects/s recovering 2023-09-23T20:33:45.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:45 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:33:45.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:45 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:33:45.780 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:33:45.812 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:33:45.822 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T20:33:45.852 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:45.862 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:33:45.912 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:33:45.961 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:33:46.014 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:33:46.068 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:33:46.074 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:33:46.085 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:46.085 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:33:46.121 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:33:46.162 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:33:46.186 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:33:46.209 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:33:46.209 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:33:46.209 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:33:46.210 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:33:46.210 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:33:46.210 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:33:46.210 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:33:46.210 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:33:46.220 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:46.262 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:33:46.268 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:46.268 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:33:46.369 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:33:46.398 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:46 smithi018 ceph-mon[184144]: 1.8 scrub starts 2023-09-23T20:33:46.398 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:46 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3594581898' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:33:46.398 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:46 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 5117/254190 objects degraded (2.013%), 3 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T20:33:46.399 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:46 smithi018 ceph-mon[179647]: 1.8 scrub starts 2023-09-23T20:33:46.399 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:46 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3594581898' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:33:46.399 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:46 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 5117/254190 objects degraded (2.013%), 3 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T20:33:46.459 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:46.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:46 smithi149 ceph-mon[170394]: 1.8 scrub starts 2023-09-23T20:33:46.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:46 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3594581898' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:33:46.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:46 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 5117/254190 objects degraded (2.013%), 3 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T20:33:46.835 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:33:46.847 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:46.848 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:33:46.949 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:33:46.999 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:47.047 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:33:47.051 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:47.051 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:33:47.116 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:33:47.152 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:33:47.195 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:47.214 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:33:47.236 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:33:47.239 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:47.240 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:33:47.340 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:33:47.376 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:47.424 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:33:47.428 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:47.428 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:33:47.476 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:33:47.492 INFO:teuthology.orchestra.run.smithi018.stderr:dumped all 2023-09-23T20:33:47.523 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:47 smithi018 ceph-mon[179647]: pgmap v780: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 748 op/s; 5117/246627 objects degraded (2.075%); 1577/246627 objects misplaced (0.639%); 2.3 MiB/s, 37 objects/s recovering 2023-09-23T20:33:47.523 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:47 smithi018 ceph-mon[179647]: 1.18 scrub starts 2023-09-23T20:33:47.524 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:47 smithi018 ceph-mon[184144]: pgmap v780: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 748 op/s; 5117/246627 objects degraded (2.075%); 1577/246627 objects misplaced (0.639%); 2.3 MiB/s, 37 objects/s recovering 2023-09-23T20:33:47.524 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:47 smithi018 ceph-mon[184144]: 1.18 scrub starts 2023-09-23T20:33:47.529 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:33:47.566 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:47 smithi149 ceph-mon[170394]: pgmap v780: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 748 op/s; 5117/246627 objects degraded (2.075%); 1577/246627 objects misplaced (0.639%); 2.3 MiB/s, 37 objects/s recovering 2023-09-23T20:33:47.566 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:47 smithi149 ceph-mon[170394]: 1.18 scrub starts 2023-09-23T20:33:47.568 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:47.624 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:33:47.628 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:47.629 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:33:47.730 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:33:47.774 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:47.821 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:33:47.829 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:47.830 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:33:47.931 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:33:48.037 INFO:teuthology.orchestra.run.smithi018.stdout:{"pg_ready":true,"pg_map":{"version":780,"stamp":"2023-09-23T20:33:45.799414+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5390338136,"num_objects":82209,"num_object_clones":0,"num_object_copies":246627,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5117,"num_objects_misplaced":1577,"num_objects_unfound":0,"num_objects_dirty":82209,"num_whiteouts":0,"num_read":6010,"num_read_kb":13240,"num_write":107833,"num_write_kb":6063935,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3278,"num_bytes_recovered":212134726,"num_keys_recovered":40,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":108333,"ondisk_log_size":108333,"up":357,"acting":353,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":365,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":18803736,"kb_used_data":16204760,"kb_used_omap":32,"kb_used_meta":2598495,"kb_avail":731157480,"statfs":{"total":767960285184,"available":748705259520,"internally_reserved":0,"allocated":16593674240,"data_stored":16582706282,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32773,"internal_metadata":2660859899},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":10,"apply_latency_ms":10,"commit_latency_ns":10000000,"apply_latency_ns":10000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-387055616,"num_objects":-5906,"num_object_clones":0,"num_object_copies":-17718,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-295,"num_objects_unfound":0,"num_objects_dirty":-5906,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5906,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":295,"num_bytes_recovered":19333120,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"7.890313"},"pg_stats":[{"pgid":"1.18","version":"90'161","reported_seq":568,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:33:39.039966+0000","last_change":"2023-09-23T20:33:39.039716+0000","last_active":"2023-09-23T20:33:39.039966+0000","last_peered":"2023-09-23T20:33:39.039966+0000","last_clean":"2023-09-23T20:33:39.039966+0000","last_became_active":"2023-09-23T20:33:39.039296+0000","last_became_peered":"2023-09-23T20:33:39.039296+0000","last_unstale":"2023-09-23T20:33:39.039966+0000","last_undegraded":"2023-09-23T20:33:39.039966+0000","last_fullsized":"2023-09-23T20:33:39.039966+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:31.756974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:31.756974+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0031802670000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131344,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"69'1","reported_seq":31,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:33:38.967725+0000","last_change":"2023-09-23T20:32:33.233853+0000","last_active":"2023-09-23T20:33:38.967725+0000","last_peered":"2023-09-23T20:33:38.967725+0000","last_clean":"2023-09-23T20:33:38.967725+0000","last_became_active":"2023-09-23T20:32:33.233546+0000","last_became_peered":"2023-09-23T20:32:33.233546+0000","last_unstale":"2023-09-23T20:33:38.967725+0000","last_undegraded":"2023-09-23T20:33:38.967725+0000","last_fullsized":"2023-09-23T20:33:38.967725+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:50:27.130808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":212,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:33:38.959328+0000","last_change":"2023-09-23T20:32:50.536495+0000","last_active":"2023-09-23T20:33:38.959328+0000","last_peered":"2023-09-23T20:33:38.959328+0000","last_clean":"2023-09-23T20:33:38.959328+0000","last_became_active":"2023-09-23T20:32:50.536180+0000","last_became_peered":"2023-09-23T20:32:50.536180+0000","last_unstale":"2023-09-23T20:33:38.959328+0000","last_undegraded":"2023-09-23T20:33:38.959328+0000","last_fullsized":"2023-09-23T20:33:38.959328+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:01:05.524308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1b","version":"90'161","reported_seq":585,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:33:39.016262+0000","last_change":"2023-09-23T20:33:39.013809+0000","last_active":"2023-09-23T20:33:39.016262+0000","last_peered":"2023-09-23T20:33:39.016262+0000","last_clean":"2023-09-23T20:33:39.016262+0000","last_became_active":"2023-09-23T20:33:39.013467+0000","last_became_peered":"2023-09-23T20:33:39.013467+0000","last_unstale":"2023-09-23T20:33:39.016262+0000","last_undegraded":"2023-09-23T20:33:39.016262+0000","last_fullsized":"2023-09-23T20:33:39.016262+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:46.558292+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:46.558292+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0023048779999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":290,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:33:38.960133+0000","last_change":"2023-09-23T20:33:00.181677+0000","last_active":"2023-09-23T20:33:38.960133+0000","last_peered":"2023-09-23T20:33:38.960133+0000","last_clean":"2023-09-23T20:33:38.960133+0000","last_became_active":"2023-09-23T20:33:00.181400+0000","last_became_peered":"2023-09-23T20:33:00.181400+0000","last_unstale":"2023-09-23T20:33:38.960133+0000","last_undegraded":"2023-09-23T20:33:38.960133+0000","last_fullsized":"2023-09-23T20:33:38.960133+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:52:14.600448+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":300,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:33:38.975299+0000","last_change":"2023-09-23T20:25:02.957185+0000","last_active":"2023-09-23T20:33:38.975299+0000","last_peered":"2023-09-23T20:33:38.975299+0000","last_clean":"2023-09-23T20:33:38.975299+0000","last_became_active":"2023-09-23T20:25:02.956689+0000","last_became_peered":"2023-09-23T20:25:02.956689+0000","last_unstale":"2023-09-23T20:33:38.975299+0000","last_undegraded":"2023-09-23T20:33:38.975299+0000","last_fullsized":"2023-09-23T20:33:38.975299+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:04:28.524015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"90'161","reported_seq":607,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:33:39.001380+0000","last_change":"2023-09-23T20:33:39.000925+0000","last_active":"2023-09-23T20:33:39.001380+0000","last_peered":"2023-09-23T20:33:39.001380+0000","last_clean":"2023-09-23T20:33:39.001380+0000","last_became_active":"2023-09-23T20:33:39.000572+0000","last_became_peered":"2023-09-23T20:33:39.000572+0000","last_unstale":"2023-09-23T20:33:39.001380+0000","last_undegraded":"2023-09-23T20:33:39.001380+0000","last_fullsized":"2023-09-23T20:33:39.001380+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:30.727267+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:30.727267+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0052560289999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":145,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:33:38.969513+0000","last_change":"2023-09-23T20:32:50.546428+0000","last_active":"2023-09-23T20:33:38.969513+0000","last_peered":"2023-09-23T20:33:38.969513+0000","last_clean":"2023-09-23T20:33:38.969513+0000","last_became_active":"2023-09-23T20:32:50.546120+0000","last_became_peered":"2023-09-23T20:32:50.546120+0000","last_unstale":"2023-09-23T20:33:38.969513+0000","last_undegraded":"2023-09-23T20:33:38.969513+0000","last_fullsized":"2023-09-23T20:33:38.969513+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:00:10.792410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":107,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:33:38.969956+0000","last_change":"2023-09-23T20:29:46.112977+0000","last_active":"2023-09-23T20:33:38.969956+0000","last_peered":"2023-09-23T20:33:38.969956+0000","last_clean":"2023-09-23T20:33:38.969956+0000","last_became_active":"2023-09-23T20:29:46.112600+0000","last_became_peered":"2023-09-23T20:29:46.112600+0000","last_unstale":"2023-09-23T20:33:38.969956+0000","last_undegraded":"2023-09-23T20:33:38.969956+0000","last_fullsized":"2023-09-23T20:33:38.969956+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:06:34.132247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"90'161","reported_seq":535,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:33:38.988775+0000","last_change":"2023-09-23T20:33:38.988536+0000","last_active":"2023-09-23T20:33:38.988775+0000","last_peered":"2023-09-23T20:33:38.988775+0000","last_clean":"2023-09-23T20:33:38.988775+0000","last_became_active":"2023-09-23T20:33:38.988198+0000","last_became_peered":"2023-09-23T20:33:38.988198+0000","last_unstale":"2023-09-23T20:33:38.988775+0000","last_undegraded":"2023-09-23T20:33:38.988775+0000","last_fullsized":"2023-09-23T20:33:38.988775+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:44.509012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:44.509012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0032186319999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":298,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:33:38.975243+0000","last_change":"2023-09-23T20:25:02.969432+0000","last_active":"2023-09-23T20:33:38.975243+0000","last_peered":"2023-09-23T20:33:38.975243+0000","last_clean":"2023-09-23T20:33:38.975243+0000","last_became_active":"2023-09-23T20:25:02.969118+0000","last_became_peered":"2023-09-23T20:25:02.969118+0000","last_unstale":"2023-09-23T20:33:38.975243+0000","last_undegraded":"2023-09-23T20:33:38.975243+0000","last_fullsized":"2023-09-23T20:33:38.975243+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:02:22.453831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":35,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T20:33:37.932082+0000","last_change":"2023-09-23T20:32:01.360359+0000","last_active":"2023-09-23T20:33:37.932082+0000","last_peered":"2023-09-23T20:33:37.932082+0000","last_clean":"2023-09-23T20:33:37.932082+0000","last_became_active":"2023-09-23T20:32:01.360008+0000","last_became_peered":"2023-09-23T20:32:01.360008+0000","last_unstale":"2023-09-23T20:33:37.932082+0000","last_undegraded":"2023-09-23T20:33:37.932082+0000","last_fullsized":"2023-09-23T20:33:37.932082+0000","mapping_epoch":308,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:33:43.135001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.14","version":"90'161","reported_seq":642,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T20:33:37.931277+0000","last_change":"2023-09-23T20:33:00.177495+0000","last_active":"2023-09-23T20:33:37.931277+0000","last_peered":"2023-09-23T20:33:37.931277+0000","last_clean":"2023-09-23T20:33:37.931277+0000","last_became_active":"2023-09-23T20:33:00.177086+0000","last_became_peered":"2023-09-23T20:33:00.177086+0000","last_unstale":"2023-09-23T20:33:37.931277+0000","last_undegraded":"2023-09-23T20:33:37.931277+0000","last_fullsized":"2023-09-23T20:33:37.931277+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:47.479012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:47.479012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:09:38.114270+0000","scrub_duration":0.0020136590000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":310,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:33:38.966036+0000","last_change":"2023-09-23T20:29:45.523808+0000","last_active":"2023-09-23T20:33:38.966036+0000","last_peered":"2023-09-23T20:33:38.966036+0000","last_clean":"2023-09-23T20:33:38.966036+0000","last_became_active":"2023-09-23T20:29:45.523508+0000","last_became_peered":"2023-09-23T20:29:45.523508+0000","last_unstale":"2023-09-23T20:33:38.966036+0000","last_undegraded":"2023-09-23T20:33:38.966036+0000","last_fullsized":"2023-09-23T20:33:38.966036+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:50:55.565352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":133,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:33:38.965613+0000","last_change":"2023-09-23T20:32:50.539003+0000","last_active":"2023-09-23T20:33:38.965613+0000","last_peered":"2023-09-23T20:33:38.965613+0000","last_clean":"2023-09-23T20:33:38.965613+0000","last_became_active":"2023-09-23T20:32:50.538596+0000","last_became_peered":"2023-09-23T20:32:50.538596+0000","last_unstale":"2023-09-23T20:33:38.965613+0000","last_undegraded":"2023-09-23T20:33:38.965613+0000","last_fullsized":"2023-09-23T20:33:38.965613+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:56:03.314375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.17","version":"90'161","reported_seq":572,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:33:38.991136+0000","last_change":"2023-09-23T20:33:38.990504+0000","last_active":"2023-09-23T20:33:38.991136+0000","last_peered":"2023-09-23T20:33:38.991136+0000","last_clean":"2023-09-23T20:33:38.991136+0000","last_became_active":"2023-09-23T20:33:38.990149+0000","last_became_peered":"2023-09-23T20:33:38.990149+0000","last_unstale":"2023-09-23T20:33:38.991136+0000","last_undegraded":"2023-09-23T20:33:38.991136+0000","last_fullsized":"2023-09-23T20:33:38.991136+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:45.196285+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:45.196285+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0014380860000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":361,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:33:38.969480+0000","last_change":"2023-09-23T20:32:33.270617+0000","last_active":"2023-09-23T20:33:38.969480+0000","last_peered":"2023-09-23T20:33:38.969480+0000","last_clean":"2023-09-23T20:33:38.969480+0000","last_became_active":"2023-09-23T20:32:33.227902+0000","last_became_peered":"2023-09-23T20:32:33.227902+0000","last_unstale":"2023-09-23T20:33:38.969480+0000","last_undegraded":"2023-09-23T20:33:38.969480+0000","last_fullsized":"2023-09-23T20:33:38.969480+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:30:26.847831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":360,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":282,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:33:38.963627+0000","last_change":"2023-09-23T20:29:46.080766+0000","last_active":"2023-09-23T20:33:38.963627+0000","last_peered":"2023-09-23T20:33:38.963627+0000","last_clean":"2023-09-23T20:33:38.963627+0000","last_became_active":"2023-09-23T20:29:46.080356+0000","last_became_peered":"2023-09-23T20:29:46.080356+0000","last_unstale":"2023-09-23T20:33:38.963627+0000","last_undegraded":"2023-09-23T20:33:38.963627+0000","last_fullsized":"2023-09-23T20:33:38.963627+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:08:01.862270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"90'161","reported_seq":565,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:33:38.991699+0000","last_change":"2023-09-23T20:33:38.991452+0000","last_active":"2023-09-23T20:33:38.991699+0000","last_peered":"2023-09-23T20:33:38.991699+0000","last_clean":"2023-09-23T20:33:38.991699+0000","last_became_active":"2023-09-23T20:33:38.991153+0000","last_became_peered":"2023-09-23T20:33:38.991153+0000","last_unstale":"2023-09-23T20:33:38.991699+0000","last_undegraded":"2023-09-23T20:33:38.991699+0000","last_fullsized":"2023-09-23T20:33:38.991699+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:44.211184+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:44.211184+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0017707720000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":41,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:33:38.967930+0000","last_change":"2023-09-23T20:32:33.266787+0000","last_active":"2023-09-23T20:33:38.967930+0000","last_peered":"2023-09-23T20:33:38.967930+0000","last_clean":"2023-09-23T20:33:38.967930+0000","last_became_active":"2023-09-23T20:32:33.228482+0000","last_became_peered":"2023-09-23T20:32:33.228482+0000","last_unstale":"2023-09-23T20:33:38.967930+0000","last_undegraded":"2023-09-23T20:33:38.967930+0000","last_fullsized":"2023-09-23T20:33:38.967930+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:13:01.812073+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":31,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:33:38.968530+0000","last_change":"2023-09-23T20:32:33.228929+0000","last_active":"2023-09-23T20:33:38.968530+0000","last_peered":"2023-09-23T20:33:38.968530+0000","last_clean":"2023-09-23T20:33:38.968530+0000","last_became_active":"2023-09-23T20:32:33.228595+0000","last_became_peered":"2023-09-23T20:32:33.228595+0000","last_unstale":"2023-09-23T20:33:38.968530+0000","last_undegraded":"2023-09-23T20:33:38.968530+0000","last_fullsized":"2023-09-23T20:33:38.968530+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:17:53.053808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"90'161","reported_seq":563,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:33:38.974246+0000","last_change":"2023-09-23T20:31:43.578810+0000","last_active":"2023-09-23T20:33:38.974246+0000","last_peered":"2023-09-23T20:33:38.974246+0000","last_clean":"2023-09-23T20:33:38.974246+0000","last_became_active":"2023-09-23T20:28:05.991304+0000","last_became_peered":"2023-09-23T20:28:05.991304+0000","last_unstale":"2023-09-23T20:33:38.974246+0000","last_undegraded":"2023-09-23T20:33:38.974246+0000","last_fullsized":"2023-09-23T20:33:38.974246+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:43.578666+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:43.578666+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:46:18.999991+0000","scrub_duration":0.0021890379999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":360,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:33:38.964085+0000","last_change":"2023-09-23T20:32:33.242943+0000","last_active":"2023-09-23T20:33:38.964085+0000","last_peered":"2023-09-23T20:33:38.964085+0000","last_clean":"2023-09-23T20:33:38.964085+0000","last_became_active":"2023-09-23T20:32:33.225468+0000","last_became_peered":"2023-09-23T20:32:33.225468+0000","last_unstale":"2023-09-23T20:33:38.964085+0000","last_undegraded":"2023-09-23T20:33:38.964085+0000","last_fullsized":"2023-09-23T20:33:38.964085+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:40:47.867573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":35,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:33:38.964692+0000","last_change":"2023-09-23T20:32:01.358590+0000","last_active":"2023-09-23T20:33:38.964692+0000","last_peered":"2023-09-23T20:33:38.964692+0000","last_clean":"2023-09-23T20:33:38.964692+0000","last_became_active":"2023-09-23T20:32:01.358345+0000","last_became_peered":"2023-09-23T20:32:01.358345+0000","last_unstale":"2023-09-23T20:33:38.964692+0000","last_undegraded":"2023-09-23T20:33:38.964692+0000","last_fullsized":"2023-09-23T20:33:38.964692+0000","mapping_epoch":308,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:22:21.159708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.10","version":"90'161","reported_seq":564,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:33:38.964586+0000","last_change":"2023-09-23T20:31:49.726214+0000","last_active":"2023-09-23T20:33:38.964586+0000","last_peered":"2023-09-23T20:33:38.964586+0000","last_clean":"2023-09-23T20:33:38.964586+0000","last_became_active":"2023-09-23T20:29:07.044892+0000","last_became_peered":"2023-09-23T20:29:07.044892+0000","last_unstale":"2023-09-23T20:33:38.964586+0000","last_undegraded":"2023-09-23T20:33:38.964586+0000","last_fullsized":"2023-09-23T20:33:38.964586+0000","mapping_epoch":253,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":254,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:49.726139+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:49.726139+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:59:11.810516+0000","scrub_duration":0.0015441470000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":298,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:33:38.969725+0000","last_change":"2023-09-23T20:25:02.953993+0000","last_active":"2023-09-23T20:33:38.969725+0000","last_peered":"2023-09-23T20:33:38.969725+0000","last_clean":"2023-09-23T20:33:38.969725+0000","last_became_active":"2023-09-23T20:25:02.953672+0000","last_became_peered":"2023-09-23T20:25:02.953672+0000","last_unstale":"2023-09-23T20:33:38.969725+0000","last_undegraded":"2023-09-23T20:33:38.969725+0000","last_fullsized":"2023-09-23T20:33:38.969725+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:26:08.603577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":312,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:33:38.965409+0000","last_change":"2023-09-23T20:25:02.949584+0000","last_active":"2023-09-23T20:33:38.965409+0000","last_peered":"2023-09-23T20:33:38.965409+0000","last_clean":"2023-09-23T20:33:38.965409+0000","last_became_active":"2023-09-23T20:25:02.949132+0000","last_became_peered":"2023-09-23T20:25:02.949132+0000","last_unstale":"2023-09-23T20:33:38.965409+0000","last_undegraded":"2023-09-23T20:33:38.965409+0000","last_fullsized":"2023-09-23T20:33:38.965409+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:34:20.509823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"90'161","reported_seq":599,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:33:38.960435+0000","last_change":"2023-09-23T20:33:00.179977+0000","last_active":"2023-09-23T20:33:38.960435+0000","last_peered":"2023-09-23T20:33:38.960435+0000","last_clean":"2023-09-23T20:33:38.960435+0000","last_became_active":"2023-09-23T20:33:00.179654+0000","last_became_peered":"2023-09-23T20:33:00.179654+0000","last_unstale":"2023-09-23T20:33:38.960435+0000","last_undegraded":"2023-09-23T20:33:38.960435+0000","last_fullsized":"2023-09-23T20:33:38.960435+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:42.544308+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:42.544308+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:24:16.853678+0000","scrub_duration":0.0029958179999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":317,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:33:38.965538+0000","last_change":"2023-09-23T20:32:33.225148+0000","last_active":"2023-09-23T20:33:38.965538+0000","last_peered":"2023-09-23T20:33:38.965538+0000","last_clean":"2023-09-23T20:33:38.965538+0000","last_became_active":"2023-09-23T20:32:33.224805+0000","last_became_peered":"2023-09-23T20:32:33.224805+0000","last_unstale":"2023-09-23T20:33:38.965538+0000","last_undegraded":"2023-09-23T20:33:38.965538+0000","last_fullsized":"2023-09-23T20:33:38.965538+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:55:30.891398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":308,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:33:38.975462+0000","last_change":"2023-09-23T20:25:02.957010+0000","last_active":"2023-09-23T20:33:38.975462+0000","last_peered":"2023-09-23T20:33:38.975462+0000","last_clean":"2023-09-23T20:33:38.975462+0000","last_became_active":"2023-09-23T20:25:02.956267+0000","last_became_peered":"2023-09-23T20:25:02.956267+0000","last_unstale":"2023-09-23T20:33:38.975462+0000","last_undegraded":"2023-09-23T20:33:38.975462+0000","last_fullsized":"2023-09-23T20:33:38.975462+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:10:55.299422+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"90'161","reported_seq":583,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:33:38.964877+0000","last_change":"2023-09-23T20:31:53.707033+0000","last_active":"2023-09-23T20:33:38.964877+0000","last_peered":"2023-09-23T20:33:38.964877+0000","last_clean":"2023-09-23T20:33:38.964877+0000","last_became_active":"2023-09-23T20:29:46.078873+0000","last_became_peered":"2023-09-23T20:29:46.078873+0000","last_unstale":"2023-09-23T20:33:38.964877+0000","last_undegraded":"2023-09-23T20:33:38.964877+0000","last_fullsized":"2023-09-23T20:33:38.964877+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:53.706934+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:53.706934+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:51:58.039657+0000","scrub_duration":0.0016840900000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":103,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:33:38.968825+0000","last_change":"2023-09-23T20:29:04.179171+0000","last_active":"2023-09-23T20:33:38.968825+0000","last_peered":"2023-09-23T20:33:38.968825+0000","last_clean":"2023-09-23T20:33:38.968825+0000","last_became_active":"2023-09-23T20:29:04.128672+0000","last_became_peered":"2023-09-23T20:29:04.128672+0000","last_unstale":"2023-09-23T20:33:38.968825+0000","last_undegraded":"2023-09-23T20:33:38.968825+0000","last_fullsized":"2023-09-23T20:33:38.968825+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:49:15.506106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":146,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:33:38.966576+0000","last_change":"2023-09-23T20:32:50.539157+0000","last_active":"2023-09-23T20:33:38.966576+0000","last_peered":"2023-09-23T20:33:38.966576+0000","last_clean":"2023-09-23T20:33:38.966576+0000","last_became_active":"2023-09-23T20:32:50.538889+0000","last_became_peered":"2023-09-23T20:32:50.538889+0000","last_unstale":"2023-09-23T20:33:38.966576+0000","last_undegraded":"2023-09-23T20:33:38.966576+0000","last_fullsized":"2023-09-23T20:33:38.966576+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:28:28.857256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.b","version":"334'6698","reported_seq":6738,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-23T20:33:43.772234+0000","last_change":"2023-09-23T20:31:49.152343+0000","last_active":"2023-09-23T20:33:43.772234+0000","last_peered":"2023-09-23T20:33:43.772234+0000","last_clean":"2023-09-23T20:33:43.772234+0000","last_became_active":"2023-09-23T20:31:49.152042+0000","last_became_peered":"2023-09-23T20:31:49.152042+0000","last_unstale":"2023-09-23T20:33:43.772234+0000","last_undegraded":"2023-09-23T20:33:43.772234+0000","last_fullsized":"2023-09-23T20:33:43.772234+0000","mapping_epoch":303,"log_start":"307'390","ondisk_log_start":"307'390","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":6308,"log_dups_size":0,"ondisk_log_size":6308,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:53:34.237468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":338034688,"num_objects":5158,"num_object_clones":0,"num_object_copies":15474,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5158,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6698,"num_write_kb":379392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.d","version":"90'161","reported_seq":613,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:33:38.971578+0000","last_change":"2023-09-23T20:33:38.970866+0000","last_active":"2023-09-23T20:33:38.971578+0000","last_peered":"2023-09-23T20:33:38.971578+0000","last_clean":"2023-09-23T20:33:38.971578+0000","last_became_active":"2023-09-23T20:33:38.970527+0000","last_became_peered":"2023-09-23T20:33:38.970527+0000","last_unstale":"2023-09-23T20:33:38.971578+0000","last_undegraded":"2023-09-23T20:33:38.971578+0000","last_fullsized":"2023-09-23T20:33:38.971578+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:46.388545+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:46.388545+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.00168356,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":337,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:33:38.966324+0000","last_change":"2023-09-23T20:32:33.283752+0000","last_active":"2023-09-23T20:33:38.966324+0000","last_peered":"2023-09-23T20:33:38.966324+0000","last_clean":"2023-09-23T20:33:38.966324+0000","last_became_active":"2023-09-23T20:32:33.230503+0000","last_became_peered":"2023-09-23T20:32:33.230503+0000","last_unstale":"2023-09-23T20:33:38.966324+0000","last_undegraded":"2023-09-23T20:33:38.966324+0000","last_fullsized":"2023-09-23T20:33:38.966324+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:29.084410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":295,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:33:38.965214+0000","last_change":"2023-09-23T20:29:04.128604+0000","last_active":"2023-09-23T20:33:38.965214+0000","last_peered":"2023-09-23T20:33:38.965214+0000","last_clean":"2023-09-23T20:33:38.965214+0000","last_became_active":"2023-09-23T20:29:04.128003+0000","last_became_peered":"2023-09-23T20:29:04.128003+0000","last_unstale":"2023-09-23T20:33:38.965214+0000","last_undegraded":"2023-09-23T20:33:38.965214+0000","last_fullsized":"2023-09-23T20:33:38.965214+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:50:49.385599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"333'6542","reported_seq":6588,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:33:42.531851+0000","last_change":"2023-09-23T20:32:50.543873+0000","last_active":"2023-09-23T20:33:42.531851+0000","last_peered":"2023-09-23T20:33:42.531851+0000","last_clean":"2023-09-23T20:33:42.531851+0000","last_became_active":"2023-09-23T20:32:50.537253+0000","last_became_peered":"2023-09-23T20:32:50.537253+0000","last_unstale":"2023-09-23T20:33:42.531851+0000","last_undegraded":"2023-09-23T20:33:42.531851+0000","last_fullsized":"2023-09-23T20:33:42.531851+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":6542,"log_dups_size":0,"ondisk_log_size":6542,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:38:02.452291+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":345374720,"num_objects":5270,"num_object_clones":0,"num_object_copies":15810,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5270,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6542,"num_write_kb":377984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,5],"acting":[2,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.c","version":"90'161","reported_seq":504,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:33:38.995035+0000","last_change":"2023-09-23T20:33:38.989595+0000","last_active":"2023-09-23T20:33:38.995035+0000","last_peered":"2023-09-23T20:33:38.995035+0000","last_clean":"2023-09-23T20:33:38.995035+0000","last_became_active":"2023-09-23T20:33:38.989255+0000","last_became_peered":"2023-09-23T20:33:38.989255+0000","last_unstale":"2023-09-23T20:33:38.995035+0000","last_undegraded":"2023-09-23T20:33:38.995035+0000","last_fullsized":"2023-09-23T20:33:38.995035+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:41.701171+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:41.701171+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0019955509999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":54613,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":358,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:33:38.966659+0000","last_change":"2023-09-23T20:33:00.196461+0000","last_active":"2023-09-23T20:33:38.966659+0000","last_peered":"2023-09-23T20:33:38.966659+0000","last_clean":"2023-09-23T20:33:38.966659+0000","last_became_active":"2023-09-23T20:33:00.178428+0000","last_became_peered":"2023-09-23T20:33:00.178428+0000","last_unstale":"2023-09-23T20:33:38.966659+0000","last_undegraded":"2023-09-23T20:33:38.966659+0000","last_fullsized":"2023-09-23T20:33:38.966659+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:48:19.958144+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":31,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:33:38.967346+0000","last_change":"2023-09-23T20:32:33.232142+0000","last_active":"2023-09-23T20:33:38.967346+0000","last_peered":"2023-09-23T20:33:38.967346+0000","last_clean":"2023-09-23T20:33:38.967346+0000","last_became_active":"2023-09-23T20:32:33.231683+0000","last_became_peered":"2023-09-23T20:32:33.231683+0000","last_unstale":"2023-09-23T20:33:38.967346+0000","last_undegraded":"2023-09-23T20:33:38.967346+0000","last_fullsized":"2023-09-23T20:33:38.967346+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:02:58.336348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"334'6644","reported_seq":6698,"reported_epoch":334,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:33:45.320477+0000","last_change":"2023-09-23T20:33:00.304857+0000","last_active":"2023-09-23T20:33:45.320477+0000","last_peered":"2023-09-23T20:33:45.320477+0000","last_clean":"2023-09-23T20:32:32.204699+0000","last_became_active":"2023-09-23T20:33:00.297863+0000","last_became_peered":"2023-09-23T20:33:00.297863+0000","last_unstale":"2023-09-23T20:33:45.320477+0000","last_undegraded":"2023-09-23T20:33:00.218246+0000","last_fullsized":"2023-09-23T20:33:00.168368+0000","mapping_epoch":322,"log_start":"307'100","ondisk_log_start":"307'100","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":6544,"log_dups_size":0,"ondisk_log_size":6544,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:31.378123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":333185024,"num_objects":5084,"num_object_clones":0,"num_object_copies":15252,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1108,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5084,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6644,"num_write_kb":375296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1],"avail_no_missing":["0","1"],"object_location_counts":[{"shards":"0,1","objects":3976},{"shards":"0,1,4","objects":1106}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"90'161","reported_seq":556,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:33:38.969410+0000","last_change":"2023-09-23T20:32:36.302809+0000","last_active":"2023-09-23T20:33:38.969410+0000","last_peered":"2023-09-23T20:33:38.969410+0000","last_clean":"2023-09-23T20:33:38.969410+0000","last_became_active":"2023-09-23T20:32:36.302465+0000","last_became_peered":"2023-09-23T20:32:36.302465+0000","last_unstale":"2023-09-23T20:33:38.969410+0000","last_undegraded":"2023-09-23T20:33:38.969410+0000","last_fullsized":"2023-09-23T20:33:38.969410+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:46.217489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:46.217489+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:35:40.488427+0000","scrub_duration":0.001456429,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":306,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:33:38.969261+0000","last_change":"2023-09-23T20:29:04.184893+0000","last_active":"2023-09-23T20:33:38.969261+0000","last_peered":"2023-09-23T20:33:38.969261+0000","last_clean":"2023-09-23T20:33:38.969261+0000","last_became_active":"2023-09-23T20:29:04.152723+0000","last_became_peered":"2023-09-23T20:29:04.152723+0000","last_unstale":"2023-09-23T20:33:38.969261+0000","last_undegraded":"2023-09-23T20:33:38.969261+0000","last_fullsized":"2023-09-23T20:33:38.969261+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:51:27.017151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":107,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:33:38.968763+0000","last_change":"2023-09-23T20:29:46.086912+0000","last_active":"2023-09-23T20:33:38.968763+0000","last_peered":"2023-09-23T20:33:38.968763+0000","last_clean":"2023-09-23T20:33:38.968763+0000","last_became_active":"2023-09-23T20:29:46.085471+0000","last_became_peered":"2023-09-23T20:29:46.085471+0000","last_unstale":"2023-09-23T20:33:38.968763+0000","last_undegraded":"2023-09-23T20:33:38.968763+0000","last_fullsized":"2023-09-23T20:33:38.968763+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:04:55.751660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.8","version":"333'6633","reported_seq":10803,"reported_epoch":333,"state":"active+recovering+undersized+remapped","last_fresh":"2023-09-23T20:33:42.524972+0000","last_change":"2023-09-23T20:32:59.221315+0000","last_active":"2023-09-23T20:33:42.524972+0000","last_peered":"2023-09-23T20:33:42.524972+0000","last_clean":"2023-09-23T20:32:32.212521+0000","last_became_active":"2023-09-23T20:32:34.616677+0000","last_became_peered":"2023-09-23T20:32:34.616677+0000","last_unstale":"2023-09-23T20:33:42.524972+0000","last_undegraded":"2023-09-23T20:33:42.524972+0000","last_fullsized":"2023-09-23T20:32:34.212267+0000","mapping_epoch":313,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":6633,"log_dups_size":0,"ondisk_log_size":6633,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:24:37.173047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":338755584,"num_objects":5169,"num_object_clones":0,"num_object_copies":15507,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1577,"num_objects_unfound":0,"num_objects_dirty":5169,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6633,"num_write_kb":377664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2061,"num_bytes_recovered":135069696,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7],"avail_no_missing":["2","3","7"],"object_location_counts":[{"shards":"2,3,7","objects":1577},{"shards":"2,7","objects":3592}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.e","version":"90'161","reported_seq":533,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:33:38.976209+0000","last_change":"2023-09-23T20:33:38.975850+0000","last_active":"2023-09-23T20:33:38.976209+0000","last_peered":"2023-09-23T20:33:38.976209+0000","last_clean":"2023-09-23T20:33:38.976209+0000","last_became_active":"2023-09-23T20:33:38.975554+0000","last_became_peered":"2023-09-23T20:33:38.975554+0000","last_unstale":"2023-09-23T20:33:38.976209+0000","last_undegraded":"2023-09-23T20:33:38.976209+0000","last_fullsized":"2023-09-23T20:33:38.976209+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:44.386120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:44.386120+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0015151330000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":340,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:33:38.964894+0000","last_change":"2023-09-23T20:29:04.138284+0000","last_active":"2023-09-23T20:33:38.964894+0000","last_peered":"2023-09-23T20:33:38.964894+0000","last_clean":"2023-09-23T20:33:38.964894+0000","last_became_active":"2023-09-23T20:29:04.137960+0000","last_became_peered":"2023-09-23T20:29:04.137960+0000","last_unstale":"2023-09-23T20:33:38.964894+0000","last_undegraded":"2023-09-23T20:33:38.964894+0000","last_fullsized":"2023-09-23T20:33:38.964894+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:18:59.548311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":317,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-23T20:33:43.218920+0000","last_change":"2023-09-23T20:32:33.236122+0000","last_active":"2023-09-23T20:33:43.218920+0000","last_peered":"2023-09-23T20:33:43.218920+0000","last_clean":"2023-09-23T20:33:43.218920+0000","last_became_active":"2023-09-23T20:32:33.235384+0000","last_became_peered":"2023-09-23T20:32:33.235384+0000","last_unstale":"2023-09-23T20:33:43.218920+0000","last_undegraded":"2023-09-23T20:33:43.218920+0000","last_fullsized":"2023-09-23T20:33:43.218920+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:56.843733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"334'6790","reported_seq":6830,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-23T20:33:45.286679+0000","last_change":"2023-09-23T20:31:49.158461+0000","last_active":"2023-09-23T20:33:45.286679+0000","last_peered":"2023-09-23T20:33:45.286679+0000","last_clean":"2023-09-23T20:33:45.286679+0000","last_became_active":"2023-09-23T20:31:49.158188+0000","last_became_peered":"2023-09-23T20:31:49.158188+0000","last_unstale":"2023-09-23T20:33:45.286679+0000","last_undegraded":"2023-09-23T20:33:45.286679+0000","last_fullsized":"2023-09-23T20:33:45.286679+0000","mapping_epoch":303,"log_start":"307'200","ondisk_log_start":"307'200","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":6590,"log_dups_size":0,"ondisk_log_size":6590,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:31:57.678090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":338558976,"num_objects":5166,"num_object_clones":0,"num_object_copies":15498,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5166,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6790,"num_write_kb":382592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":726,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:33:38.980372+0000","last_change":"2023-09-23T20:33:38.980195+0000","last_active":"2023-09-23T20:33:38.980372+0000","last_peered":"2023-09-23T20:33:38.980372+0000","last_clean":"2023-09-23T20:33:38.980372+0000","last_became_active":"2023-09-23T20:33:38.979989+0000","last_became_peered":"2023-09-23T20:33:38.979989+0000","last_unstale":"2023-09-23T20:33:38.980372+0000","last_undegraded":"2023-09-23T20:33:38.980372+0000","last_fullsized":"2023-09-23T20:33:38.980372+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:28:05.872074+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:28:05.872074+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0029215980000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":68,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:33:38.970114+0000","last_change":"2023-09-23T20:32:50.538500+0000","last_active":"2023-09-23T20:33:38.970114+0000","last_peered":"2023-09-23T20:33:38.970114+0000","last_clean":"2023-09-23T20:33:38.970114+0000","last_became_active":"2023-09-23T20:32:50.538152+0000","last_became_peered":"2023-09-23T20:32:50.538152+0000","last_unstale":"2023-09-23T20:33:38.970114+0000","last_undegraded":"2023-09-23T20:33:38.970114+0000","last_fullsized":"2023-09-23T20:33:38.970114+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:00:41.103508+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":31,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:33:38.968000+0000","last_change":"2023-09-23T20:32:33.231614+0000","last_active":"2023-09-23T20:33:38.968000+0000","last_peered":"2023-09-23T20:33:38.968000+0000","last_clean":"2023-09-23T20:33:38.968000+0000","last_became_active":"2023-09-23T20:32:33.231207+0000","last_became_peered":"2023-09-23T20:32:33.231207+0000","last_unstale":"2023-09-23T20:33:38.968000+0000","last_undegraded":"2023-09-23T20:33:38.968000+0000","last_fullsized":"2023-09-23T20:33:38.968000+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:46:09.780829+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"333'6621","reported_seq":6659,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:33:42.520423+0000","last_change":"2023-09-23T20:31:49.155754+0000","last_active":"2023-09-23T20:33:42.520423+0000","last_peered":"2023-09-23T20:33:42.520423+0000","last_clean":"2023-09-23T20:33:42.520423+0000","last_became_active":"2023-09-23T20:31:49.155363+0000","last_became_peered":"2023-09-23T20:31:49.155363+0000","last_unstale":"2023-09-23T20:33:42.520423+0000","last_undegraded":"2023-09-23T20:33:42.520423+0000","last_fullsized":"2023-09-23T20:33:42.520423+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":6621,"log_dups_size":0,"ondisk_log_size":6621,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:45:42.661641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":342556672,"num_objects":5227,"num_object_clones":0,"num_object_copies":15681,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5227,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6621,"num_write_kb":379136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":571,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-23T20:33:45.058820+0000","last_change":"2023-09-23T20:33:45.058820+0000","last_active":"2023-09-23T20:33:45.058820+0000","last_peered":"2023-09-23T20:33:45.058820+0000","last_clean":"2023-09-23T20:33:45.058820+0000","last_became_active":"2023-09-23T20:33:38.996273+0000","last_became_peered":"2023-09-23T20:33:38.996273+0000","last_unstale":"2023-09-23T20:33:45.058820+0000","last_undegraded":"2023-09-23T20:33:45.058820+0000","last_fullsized":"2023-09-23T20:33:45.058820+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:31.756974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:31.756974+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0031802670000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131344,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":129,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:33:38.968838+0000","last_change":"2023-09-23T20:32:33.229700+0000","last_active":"2023-09-23T20:33:38.968838+0000","last_peered":"2023-09-23T20:33:38.968838+0000","last_clean":"2023-09-23T20:33:38.968838+0000","last_became_active":"2023-09-23T20:32:33.229372+0000","last_became_peered":"2023-09-23T20:32:33.229372+0000","last_unstale":"2023-09-23T20:33:38.968838+0000","last_undegraded":"2023-09-23T20:33:38.968838+0000","last_fullsized":"2023-09-23T20:33:38.968838+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:18:10.958227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":241,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T20:33:37.940167+0000","last_change":"2023-09-23T20:33:00.188031+0000","last_active":"2023-09-23T20:33:37.940167+0000","last_peered":"2023-09-23T20:33:37.940167+0000","last_clean":"2023-09-23T20:33:37.940167+0000","last_became_active":"2023-09-23T20:33:00.187675+0000","last_became_peered":"2023-09-23T20:33:00.187675+0000","last_unstale":"2023-09-23T20:33:37.940167+0000","last_undegraded":"2023-09-23T20:33:37.940167+0000","last_fullsized":"2023-09-23T20:33:37.940167+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:42:28.967989+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.d","version":"334'6787","reported_seq":6827,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-23T20:33:45.599614+0000","last_change":"2023-09-23T20:31:49.158641+0000","last_active":"2023-09-23T20:33:45.599614+0000","last_peered":"2023-09-23T20:33:45.599614+0000","last_clean":"2023-09-23T20:33:45.599614+0000","last_became_active":"2023-09-23T20:31:49.157604+0000","last_became_peered":"2023-09-23T20:31:49.157604+0000","last_unstale":"2023-09-23T20:33:45.599614+0000","last_undegraded":"2023-09-23T20:33:45.599614+0000","last_fullsized":"2023-09-23T20:33:45.599614+0000","mapping_epoch":303,"log_start":"308'695","ondisk_log_start":"308'695","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":6092,"log_dups_size":0,"ondisk_log_size":6092,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:54:04.726000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":341049368,"num_objects":5205,"num_object_clones":0,"num_object_copies":15615,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5205,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":6787,"num_write_kb":383681,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.b","version":"90'161","reported_seq":585,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:33:39.005207+0000","last_change":"2023-09-23T20:33:38.992494+0000","last_active":"2023-09-23T20:33:39.005207+0000","last_peered":"2023-09-23T20:33:39.005207+0000","last_clean":"2023-09-23T20:33:39.005207+0000","last_became_active":"2023-09-23T20:33:38.991910+0000","last_became_peered":"2023-09-23T20:33:38.991910+0000","last_unstale":"2023-09-23T20:33:39.005207+0000","last_undegraded":"2023-09-23T20:33:39.005207+0000","last_fullsized":"2023-09-23T20:33:39.005207+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:46.558292+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:46.558292+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0023048779999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":243,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T20:33:37.929998+0000","last_change":"2023-09-23T20:33:00.185583+0000","last_active":"2023-09-23T20:33:37.929998+0000","last_peered":"2023-09-23T20:33:37.929998+0000","last_clean":"2023-09-23T20:33:37.929998+0000","last_became_active":"2023-09-23T20:33:00.185273+0000","last_became_peered":"2023-09-23T20:33:00.185273+0000","last_unstale":"2023-09-23T20:33:37.929998+0000","last_undegraded":"2023-09-23T20:33:37.929998+0000","last_fullsized":"2023-09-23T20:33:37.929998+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:00:48.877233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":31,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:33:38.970627+0000","last_change":"2023-09-23T20:32:33.224963+0000","last_active":"2023-09-23T20:33:38.970627+0000","last_peered":"2023-09-23T20:33:38.970627+0000","last_clean":"2023-09-23T20:33:38.970627+0000","last_became_active":"2023-09-23T20:32:33.224634+0000","last_became_peered":"2023-09-23T20:32:33.224634+0000","last_unstale":"2023-09-23T20:33:38.970627+0000","last_undegraded":"2023-09-23T20:33:38.970627+0000","last_fullsized":"2023-09-23T20:33:38.970627+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:41:40.314140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"334'6727","reported_seq":6766,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-23T20:33:45.610975+0000","last_change":"2023-09-23T20:31:49.157247+0000","last_active":"2023-09-23T20:33:45.610975+0000","last_peered":"2023-09-23T20:33:45.610975+0000","last_clean":"2023-09-23T20:33:45.610975+0000","last_became_active":"2023-09-23T20:31:49.156961+0000","last_became_peered":"2023-09-23T20:31:49.156961+0000","last_unstale":"2023-09-23T20:33:45.610975+0000","last_undegraded":"2023-09-23T20:33:45.610975+0000","last_fullsized":"2023-09-23T20:33:45.610975+0000","mapping_epoch":303,"log_start":"309'701","ondisk_log_start":"309'701","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":6026,"log_dups_size":0,"ondisk_log_size":6026,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:49:17.346795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":336396288,"num_objects":5133,"num_object_clones":0,"num_object_copies":15399,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5133,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6727,"num_write_kb":379520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":611,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:33:39.020374+0000","last_change":"2023-09-23T20:33:39.020352+0000","last_active":"2023-09-23T20:33:39.020374+0000","last_peered":"2023-09-23T20:33:39.020374+0000","last_clean":"2023-09-23T20:33:39.020374+0000","last_became_active":"2023-09-23T20:33:38.988954+0000","last_became_peered":"2023-09-23T20:33:38.988954+0000","last_unstale":"2023-09-23T20:33:39.020374+0000","last_undegraded":"2023-09-23T20:33:39.020374+0000","last_fullsized":"2023-09-23T20:33:39.020374+0000","mapping_epoch":253,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:30.727267+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:30.727267+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0052560289999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":146,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:33:38.958518+0000","last_change":"2023-09-23T20:32:50.530690+0000","last_active":"2023-09-23T20:33:38.958518+0000","last_peered":"2023-09-23T20:33:38.958518+0000","last_clean":"2023-09-23T20:33:38.958518+0000","last_became_active":"2023-09-23T20:32:50.530376+0000","last_became_peered":"2023-09-23T20:32:50.530376+0000","last_unstale":"2023-09-23T20:33:38.958518+0000","last_undegraded":"2023-09-23T20:33:38.958518+0000","last_fullsized":"2023-09-23T20:33:38.958518+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:24:53.184959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":154,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:33:38.958907+0000","last_change":"2023-09-23T20:32:33.230582+0000","last_active":"2023-09-23T20:33:38.958907+0000","last_peered":"2023-09-23T20:33:38.958907+0000","last_clean":"2023-09-23T20:33:38.958907+0000","last_became_active":"2023-09-23T20:32:33.230257+0000","last_became_peered":"2023-09-23T20:32:33.230257+0000","last_unstale":"2023-09-23T20:33:38.958907+0000","last_undegraded":"2023-09-23T20:33:38.958907+0000","last_fullsized":"2023-09-23T20:33:38.958907+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:11:45.333096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"334'6683","reported_seq":7778,"reported_epoch":334,"state":"active+recovery_wait","last_fresh":"2023-09-23T20:33:45.512046+0000","last_change":"2023-09-23T20:33:01.313842+0000","last_active":"2023-09-23T20:33:45.512046+0000","last_peered":"2023-09-23T20:33:45.512046+0000","last_clean":"2023-09-23T20:32:32.204597+0000","last_became_active":"2023-09-23T20:33:01.258982+0000","last_became_peered":"2023-09-23T20:33:01.258982+0000","last_unstale":"2023-09-23T20:33:45.512046+0000","last_undegraded":"2023-09-23T20:33:45.512046+0000","last_fullsized":"2023-09-23T20:33:45.512046+0000","mapping_epoch":323,"log_start":"308'647","ondisk_log_start":"308'647","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":6036,"log_dups_size":0,"ondisk_log_size":6036,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:38:23.266319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":331546624,"num_objects":5059,"num_object_clones":0,"num_object_copies":15177,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5059,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6683,"num_write_kb":375744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":521,"num_bytes_recovered":34144256,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":["6","1","3"],"object_location_counts":[{"shards":"1,3,6","objects":5059}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":573,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:33:38.979472+0000","last_change":"2023-09-23T20:33:38.978909+0000","last_active":"2023-09-23T20:33:38.979472+0000","last_peered":"2023-09-23T20:33:38.979472+0000","last_clean":"2023-09-23T20:33:38.979472+0000","last_became_active":"2023-09-23T20:33:38.978580+0000","last_became_peered":"2023-09-23T20:33:38.978580+0000","last_unstale":"2023-09-23T20:33:38.979472+0000","last_undegraded":"2023-09-23T20:33:38.979472+0000","last_fullsized":"2023-09-23T20:33:38.979472+0000","mapping_epoch":253,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:45.196285+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:45.196285+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0014380860000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"86'119","reported_seq":638,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:33:38.958730+0000","last_change":"2023-09-23T20:29:45.521257+0000","last_active":"2023-09-23T20:33:38.958730+0000","last_peered":"2023-09-23T20:33:38.958730+0000","last_clean":"2023-09-23T20:33:38.958730+0000","last_became_active":"2023-09-23T20:29:45.520546+0000","last_became_peered":"2023-09-23T20:29:45.520546+0000","last_unstale":"2023-09-23T20:33:38.958730+0000","last_undegraded":"2023-09-23T20:33:38.958730+0000","last_fullsized":"2023-09-23T20:33:38.958730+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":119,"log_dups_size":0,"ondisk_log_size":119,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:58:56.085777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032705499999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":523,"num_read_kb":3503,"num_write":136,"num_write_kb":3128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":350,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:33:38.965334+0000","last_change":"2023-09-23T20:25:39.637160+0000","last_active":"2023-09-23T20:33:38.965334+0000","last_peered":"2023-09-23T20:33:38.965334+0000","last_clean":"2023-09-23T20:33:38.965334+0000","last_became_active":"2023-09-23T20:25:39.636860+0000","last_became_peered":"2023-09-23T20:25:39.636860+0000","last_unstale":"2023-09-23T20:33:38.965334+0000","last_undegraded":"2023-09-23T20:33:38.965334+0000","last_fullsized":"2023-09-23T20:33:38.965334+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:21:47.860063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":258,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:33:38.963364+0000","last_change":"2023-09-23T20:33:00.190831+0000","last_active":"2023-09-23T20:33:38.963364+0000","last_peered":"2023-09-23T20:33:38.963364+0000","last_clean":"2023-09-23T20:33:38.963364+0000","last_became_active":"2023-09-23T20:33:00.190494+0000","last_became_peered":"2023-09-23T20:33:00.190494+0000","last_unstale":"2023-09-23T20:33:38.963364+0000","last_undegraded":"2023-09-23T20:33:38.963364+0000","last_fullsized":"2023-09-23T20:33:38.963364+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:19:01.109316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"334'6799","reported_seq":6838,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-23T20:33:45.598687+0000","last_change":"2023-09-23T20:31:49.157301+0000","last_active":"2023-09-23T20:33:45.598687+0000","last_peered":"2023-09-23T20:33:45.598687+0000","last_clean":"2023-09-23T20:33:45.598687+0000","last_became_active":"2023-09-23T20:31:49.156992+0000","last_became_peered":"2023-09-23T20:31:49.156992+0000","last_unstale":"2023-09-23T20:33:45.598687+0000","last_undegraded":"2023-09-23T20:33:45.598687+0000","last_fullsized":"2023-09-23T20:33:45.598687+0000","mapping_epoch":303,"log_start":"309'776","ondisk_log_start":"309'776","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":6023,"log_dups_size":0,"ondisk_log_size":6023,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:21:35.645838+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":334561280,"num_objects":5105,"num_object_clones":0,"num_object_copies":15315,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5105,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6799,"num_write_kb":380928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":578,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:33:38.964380+0000","last_change":"2023-09-23T20:31:45.785521+0000","last_active":"2023-09-23T20:33:38.964380+0000","last_peered":"2023-09-23T20:33:38.964380+0000","last_clean":"2023-09-23T20:33:38.964380+0000","last_became_active":"2023-09-23T20:27:55.213394+0000","last_became_peered":"2023-09-23T20:27:55.213394+0000","last_unstale":"2023-09-23T20:33:38.964380+0000","last_undegraded":"2023-09-23T20:33:38.964380+0000","last_fullsized":"2023-09-23T20:33:38.964380+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:45.785450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:45.785450+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:21:26.092578+0000","scrub_duration":0.001558306,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"86'106","reported_seq":1052,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:33:38.964698+0000","last_change":"2023-09-23T20:31:31.531882+0000","last_active":"2023-09-23T20:33:38.964698+0000","last_peered":"2023-09-23T20:33:38.964698+0000","last_clean":"2023-09-23T20:33:38.964698+0000","last_became_active":"2023-09-23T20:29:46.078501+0000","last_became_peered":"2023-09-23T20:29:46.078501+0000","last_unstale":"2023-09-23T20:33:38.964698+0000","last_undegraded":"2023-09-23T20:33:38.964698+0000","last_fullsized":"2023-09-23T20:33:38.964698+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":106,"log_dups_size":0,"ondisk_log_size":106,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:37.015974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00099302599999999993,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1105,"num_read_kb":1689,"num_write":100,"num_write_kb":404,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":31,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:33:38.968351+0000","last_change":"2023-09-23T20:32:33.230242+0000","last_active":"2023-09-23T20:33:38.968351+0000","last_peered":"2023-09-23T20:33:38.968351+0000","last_clean":"2023-09-23T20:33:38.968351+0000","last_became_active":"2023-09-23T20:32:33.229732+0000","last_became_peered":"2023-09-23T20:32:33.229732+0000","last_unstale":"2023-09-23T20:33:38.968351+0000","last_undegraded":"2023-09-23T20:33:38.968351+0000","last_fullsized":"2023-09-23T20:33:38.968351+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:27:55.538429+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":291,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:33:38.958702+0000","last_change":"2023-09-23T20:29:45.533406+0000","last_active":"2023-09-23T20:33:38.958702+0000","last_peered":"2023-09-23T20:33:38.958702+0000","last_clean":"2023-09-23T20:33:38.958702+0000","last_became_active":"2023-09-23T20:29:45.526001+0000","last_became_peered":"2023-09-23T20:29:45.526001+0000","last_unstale":"2023-09-23T20:33:38.958702+0000","last_undegraded":"2023-09-23T20:33:38.958702+0000","last_fullsized":"2023-09-23T20:33:38.958702+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:43.787251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.4","version":"334'6649","reported_seq":6716,"reported_epoch":334,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:33:45.347946+0000","last_change":"2023-09-23T20:33:00.311161+0000","last_active":"2023-09-23T20:33:45.347946+0000","last_peered":"2023-09-23T20:33:45.347946+0000","last_clean":"2023-09-23T20:32:32.192526+0000","last_became_active":"2023-09-23T20:33:00.256639+0000","last_became_peered":"2023-09-23T20:33:00.256639+0000","last_unstale":"2023-09-23T20:33:45.347946+0000","last_undegraded":"2023-09-23T20:33:00.185190+0000","last_fullsized":"2023-09-23T20:33:00.168423+0000","mapping_epoch":322,"log_start":"307'100","ondisk_log_start":"307'100","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":6549,"log_dups_size":0,"ondisk_log_size":6549,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:59:02.098340+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":332464128,"num_objects":5073,"num_object_clones":0,"num_object_copies":15219,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1052,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5073,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6649,"num_write_kb":375104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":262144,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,6],"acting":[0,1],"avail_no_missing":["0","1"],"object_location_counts":[{"shards":"0,1","objects":4021},{"shards":"0,1,4","objects":1051}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":786,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:33:38.964070+0000","last_change":"2023-09-23T20:33:00.189173+0000","last_active":"2023-09-23T20:33:38.964070+0000","last_peered":"2023-09-23T20:33:38.964070+0000","last_clean":"2023-09-23T20:33:38.964070+0000","last_became_active":"2023-09-23T20:33:00.188795+0000","last_became_peered":"2023-09-23T20:33:00.188795+0000","last_unstale":"2023-09-23T20:33:38.964070+0000","last_undegraded":"2023-09-23T20:33:38.964070+0000","last_fullsized":"2023-09-23T20:33:38.964070+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:43.390908+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:43.390908+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:44:16.658526+0000","scrub_duration":0.0032721,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,2],"acting":[7,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"86'130","reported_seq":645,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T20:33:37.929890+0000","last_change":"2023-09-23T20:25:39.637458+0000","last_active":"2023-09-23T20:33:37.929890+0000","last_peered":"2023-09-23T20:33:37.929890+0000","last_clean":"2023-09-23T20:33:37.929890+0000","last_became_active":"2023-09-23T20:25:39.636773+0000","last_became_peered":"2023-09-23T20:25:39.636773+0000","last_unstale":"2023-09-23T20:33:37.929890+0000","last_undegraded":"2023-09-23T20:33:37.929890+0000","last_fullsized":"2023-09-23T20:33:37.929890+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:00:32.810564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011300049999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":512,"num_read_kb":1820,"num_write":133,"num_write_kb":1347,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":281,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:33:38.958428+0000","last_change":"2023-09-23T20:29:46.073394+0000","last_active":"2023-09-23T20:33:38.958428+0000","last_peered":"2023-09-23T20:33:38.958428+0000","last_clean":"2023-09-23T20:33:38.958428+0000","last_became_active":"2023-09-23T20:29:46.072067+0000","last_became_peered":"2023-09-23T20:29:46.072067+0000","last_unstale":"2023-09-23T20:33:38.958428+0000","last_undegraded":"2023-09-23T20:33:38.958428+0000","last_fullsized":"2023-09-23T20:33:38.958428+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:01:56.944614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":273,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:33:38.958839+0000","last_change":"2023-09-23T20:29:45.533185+0000","last_active":"2023-09-23T20:33:38.958839+0000","last_peered":"2023-09-23T20:33:38.958839+0000","last_clean":"2023-09-23T20:33:38.958839+0000","last_became_active":"2023-09-23T20:29:45.526765+0000","last_became_peered":"2023-09-23T20:29:45.526765+0000","last_unstale":"2023-09-23T20:33:38.958839+0000","last_undegraded":"2023-09-23T20:33:38.958839+0000","last_fullsized":"2023-09-23T20:33:38.958839+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:18:33.063215+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"334'6782","reported_seq":6822,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-23T20:33:45.360945+0000","last_change":"2023-09-23T20:31:49.158194+0000","last_active":"2023-09-23T20:33:45.360945+0000","last_peered":"2023-09-23T20:33:45.360945+0000","last_clean":"2023-09-23T20:33:45.360945+0000","last_became_active":"2023-09-23T20:31:49.157875+0000","last_became_peered":"2023-09-23T20:31:49.157875+0000","last_unstale":"2023-09-23T20:33:45.360945+0000","last_undegraded":"2023-09-23T20:33:45.360945+0000","last_fullsized":"2023-09-23T20:33:45.360945+0000","mapping_epoch":303,"log_start":"307'200","ondisk_log_start":"307'200","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":6582,"log_dups_size":0,"ondisk_log_size":6582,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:04:42.411181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":333316096,"num_objects":5086,"num_object_clones":0,"num_object_copies":15258,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5086,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6782,"num_write_kb":379776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"86'194","reported_seq":982,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:33:38.966342+0000","last_change":"2023-09-23T20:25:40.618665+0000","last_active":"2023-09-23T20:33:38.966342+0000","last_peered":"2023-09-23T20:33:38.966342+0000","last_clean":"2023-09-23T20:33:38.966342+0000","last_became_active":"2023-09-23T20:25:40.618087+0000","last_became_peered":"2023-09-23T20:25:40.618087+0000","last_unstale":"2023-09-23T20:33:38.966342+0000","last_undegraded":"2023-09-23T20:33:38.966342+0000","last_fullsized":"2023-09-23T20:33:38.966342+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":194,"log_dups_size":0,"ondisk_log_size":194,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:07:16.114968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00037995999999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1025,"num_read_kb":1201,"num_write":173,"num_write_kb":1276,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":767,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:33:38.964351+0000","last_change":"2023-09-23T20:31:45.343835+0000","last_active":"2023-09-23T20:33:38.964351+0000","last_peered":"2023-09-23T20:33:38.964351+0000","last_clean":"2023-09-23T20:33:38.964351+0000","last_became_active":"2023-09-23T20:29:07.042407+0000","last_became_peered":"2023-09-23T20:29:07.042407+0000","last_unstale":"2023-09-23T20:33:38.964351+0000","last_undegraded":"2023-09-23T20:33:38.964351+0000","last_fullsized":"2023-09-23T20:33:38.964351+0000","mapping_epoch":253,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":254,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:45.343742+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:45.343742+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:33:12.745366+0000","scrub_duration":0.001475802,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":133,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:33:38.964698+0000","last_change":"2023-09-23T20:32:50.534315+0000","last_active":"2023-09-23T20:33:38.964698+0000","last_peered":"2023-09-23T20:33:38.964698+0000","last_clean":"2023-09-23T20:33:38.964698+0000","last_became_active":"2023-09-23T20:32:50.533896+0000","last_became_peered":"2023-09-23T20:32:50.533896+0000","last_unstale":"2023-09-23T20:33:38.964698+0000","last_undegraded":"2023-09-23T20:33:38.964698+0000","last_fullsized":"2023-09-23T20:33:38.964698+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:06:53.464923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":113,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:33:38.969789+0000","last_change":"2023-09-23T20:32:33.225012+0000","last_active":"2023-09-23T20:33:38.969789+0000","last_peered":"2023-09-23T20:33:38.969789+0000","last_clean":"2023-09-23T20:33:38.969789+0000","last_became_active":"2023-09-23T20:32:33.224699+0000","last_became_peered":"2023-09-23T20:32:33.224699+0000","last_unstale":"2023-09-23T20:33:38.969789+0000","last_undegraded":"2023-09-23T20:33:38.969789+0000","last_fullsized":"2023-09-23T20:33:38.969789+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:10:06.267878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"334'6792","reported_seq":6831,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-23T20:33:45.727099+0000","last_change":"2023-09-23T20:31:49.154418+0000","last_active":"2023-09-23T20:33:45.727099+0000","last_peered":"2023-09-23T20:33:45.727099+0000","last_clean":"2023-09-23T20:33:45.727099+0000","last_became_active":"2023-09-23T20:31:49.154182+0000","last_became_peered":"2023-09-23T20:31:49.154182+0000","last_unstale":"2023-09-23T20:33:45.727099+0000","last_undegraded":"2023-09-23T20:33:45.727099+0000","last_fullsized":"2023-09-23T20:33:45.727099+0000","mapping_epoch":303,"log_start":"307'498","ondisk_log_start":"307'498","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":6294,"log_dups_size":0,"ondisk_log_size":6294,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:44:10.087486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":336461824,"num_objects":5134,"num_object_clones":0,"num_object_copies":15402,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5134,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6792,"num_write_kb":381632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":609,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:33:38.960357+0000","last_change":"2023-09-23T20:33:03.279925+0000","last_active":"2023-09-23T20:33:38.960357+0000","last_peered":"2023-09-23T20:33:38.960357+0000","last_clean":"2023-09-23T20:33:38.960357+0000","last_became_active":"2023-09-23T20:33:03.279531+0000","last_became_peered":"2023-09-23T20:33:03.279531+0000","last_unstale":"2023-09-23T20:33:38.960357+0000","last_undegraded":"2023-09-23T20:33:38.960357+0000","last_fullsized":"2023-09-23T20:33:38.960357+0000","mapping_epoch":325,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:43.582146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:43.582146+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:06:12.587774+0000","scrub_duration":0.0016219730000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,5],"acting":[1,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"86'80","reported_seq":597,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:33:38.964260+0000","last_change":"2023-09-23T20:29:45.536796+0000","last_active":"2023-09-23T20:33:38.964260+0000","last_peered":"2023-09-23T20:33:38.964260+0000","last_clean":"2023-09-23T20:33:38.964260+0000","last_became_active":"2023-09-23T20:29:45.536439+0000","last_became_peered":"2023-09-23T20:29:45.536439+0000","last_unstale":"2023-09-23T20:33:38.964260+0000","last_undegraded":"2023-09-23T20:33:38.964260+0000","last_fullsized":"2023-09-23T20:33:38.964260+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":80,"log_dups_size":0,"ondisk_log_size":80,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:30:57.793132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035115,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":395,"num_read_kb":1370,"num_write":66,"num_write_kb":1061,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":326,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:33:38.974439+0000","last_change":"2023-09-23T20:33:00.220984+0000","last_active":"2023-09-23T20:33:38.974439+0000","last_peered":"2023-09-23T20:33:38.974439+0000","last_clean":"2023-09-23T20:33:38.974439+0000","last_became_active":"2023-09-23T20:33:00.220642+0000","last_became_peered":"2023-09-23T20:33:00.220642+0000","last_unstale":"2023-09-23T20:33:38.974439+0000","last_undegraded":"2023-09-23T20:33:38.974439+0000","last_fullsized":"2023-09-23T20:33:38.974439+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:52:14.218890+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":330,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:33:38.975071+0000","last_change":"2023-09-23T20:29:45.690815+0000","last_active":"2023-09-23T20:33:38.975071+0000","last_peered":"2023-09-23T20:33:38.975071+0000","last_clean":"2023-09-23T20:33:38.975071+0000","last_became_active":"2023-09-23T20:29:45.690478+0000","last_became_peered":"2023-09-23T20:29:45.690478+0000","last_unstale":"2023-09-23T20:33:38.975071+0000","last_undegraded":"2023-09-23T20:33:38.975071+0000","last_fullsized":"2023-09-23T20:33:38.975071+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:38.288193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.0","version":"334'6790","reported_seq":6829,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-23T20:33:45.358695+0000","last_change":"2023-09-23T20:31:49.157205+0000","last_active":"2023-09-23T20:33:45.358695+0000","last_peered":"2023-09-23T20:33:45.358695+0000","last_clean":"2023-09-23T20:33:45.358695+0000","last_became_active":"2023-09-23T20:31:49.156922+0000","last_became_peered":"2023-09-23T20:31:49.156922+0000","last_unstale":"2023-09-23T20:33:45.358695+0000","last_undegraded":"2023-09-23T20:33:45.358695+0000","last_fullsized":"2023-09-23T20:33:45.358695+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":6790,"log_dups_size":0,"ondisk_log_size":6790,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:51:36.120059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":334757888,"num_objects":5108,"num_object_clones":0,"num_object_copies":15324,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5108,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6790,"num_write_kb":380736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":566,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:33:38.991688+0000","last_change":"2023-09-23T20:33:38.991361+0000","last_active":"2023-09-23T20:33:38.991688+0000","last_peered":"2023-09-23T20:33:38.991688+0000","last_clean":"2023-09-23T20:33:38.991688+0000","last_became_active":"2023-09-23T20:33:38.991030+0000","last_became_peered":"2023-09-23T20:33:38.991030+0000","last_unstale":"2023-09-23T20:33:38.991688+0000","last_undegraded":"2023-09-23T20:33:38.991688+0000","last_fullsized":"2023-09-23T20:33:38.991688+0000","mapping_epoch":253,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:44.211184+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:44.211184+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0017707720000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"86'102","reported_seq":928,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:33:38.969381+0000","last_change":"2023-09-23T20:32:36.314672+0000","last_active":"2023-09-23T20:33:38.969381+0000","last_peered":"2023-09-23T20:33:38.969381+0000","last_clean":"2023-09-23T20:33:38.969381+0000","last_became_active":"2023-09-23T20:32:36.314231+0000","last_became_peered":"2023-09-23T20:32:36.314231+0000","last_unstale":"2023-09-23T20:33:38.969381+0000","last_undegraded":"2023-09-23T20:33:38.969381+0000","last_fullsized":"2023-09-23T20:33:38.969381+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":102,"log_dups_size":0,"ondisk_log_size":102,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:11:40.514709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030656500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":963,"num_read_kb":1234,"num_write":60,"num_write_kb":225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":257,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:33:38.958803+0000","last_change":"2023-09-23T20:29:45.538283+0000","last_active":"2023-09-23T20:33:38.958803+0000","last_peered":"2023-09-23T20:33:38.958803+0000","last_clean":"2023-09-23T20:33:38.958803+0000","last_became_active":"2023-09-23T20:29:45.537922+0000","last_became_peered":"2023-09-23T20:29:45.537922+0000","last_unstale":"2023-09-23T20:33:38.958803+0000","last_undegraded":"2023-09-23T20:33:38.958803+0000","last_fullsized":"2023-09-23T20:33:38.958803+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:30:52.397836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":267,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:33:38.970211+0000","last_change":"2023-09-23T20:20:14.888874+0000","last_active":"2023-09-23T20:33:38.970211+0000","last_peered":"2023-09-23T20:33:38.970211+0000","last_clean":"2023-09-23T20:33:38.970211+0000","last_became_active":"2023-09-23T20:20:14.887724+0000","last_became_peered":"2023-09-23T20:20:14.887724+0000","last_unstale":"2023-09-23T20:33:38.970211+0000","last_undegraded":"2023-09-23T20:33:38.970211+0000","last_fullsized":"2023-09-23T20:33:38.970211+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:09:24.510326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"333'6408","reported_seq":7631,"reported_epoch":333,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:33:42.495998+0000","last_change":"2023-09-23T20:32:51.571341+0000","last_active":"2023-09-23T20:33:42.495998+0000","last_peered":"2023-09-23T20:33:42.495998+0000","last_clean":"2023-09-23T20:32:32.204529+0000","last_became_active":"2023-09-23T20:32:51.533262+0000","last_became_peered":"2023-09-23T20:32:51.533262+0000","last_unstale":"2023-09-23T20:33:42.495998+0000","last_undegraded":"2023-09-23T20:32:51.523866+0000","last_fullsized":"2023-09-23T20:32:51.518152+0000","mapping_epoch":320,"log_start":"0'0","ondisk_log_start":"0'0","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":6408,"log_dups_size":0,"ondisk_log_size":6408,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:50:09.051328+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":333316096,"num_objects":5086,"num_object_clones":0,"num_object_copies":15258,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2957,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5086,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6408,"num_write_kb":367808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":585,"num_bytes_recovered":38338560,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3],"avail_no_missing":["2","3","6"],"object_location_counts":[{"shards":"2,3","objects":2129},{"shards":"2,3,6","objects":2883}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":536,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:33:38.985911+0000","last_change":"2023-09-23T20:33:38.985620+0000","last_active":"2023-09-23T20:33:38.985911+0000","last_peered":"2023-09-23T20:33:38.985911+0000","last_clean":"2023-09-23T20:33:38.985911+0000","last_became_active":"2023-09-23T20:33:38.985300+0000","last_became_peered":"2023-09-23T20:33:38.985300+0000","last_unstale":"2023-09-23T20:33:38.985911+0000","last_undegraded":"2023-09-23T20:33:38.985911+0000","last_fullsized":"2023-09-23T20:33:38.985911+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:44.509012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:44.509012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0032186319999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":71,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:33:38.966660+0000","last_change":"2023-09-23T20:32:50.549695+0000","last_active":"2023-09-23T20:33:38.966660+0000","last_peered":"2023-09-23T20:33:38.966660+0000","last_clean":"2023-09-23T20:33:38.966660+0000","last_became_active":"2023-09-23T20:32:50.549431+0000","last_became_peered":"2023-09-23T20:32:50.549431+0000","last_unstale":"2023-09-23T20:33:38.966660+0000","last_undegraded":"2023-09-23T20:33:38.966660+0000","last_fullsized":"2023-09-23T20:33:38.966660+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:45:53.666569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"86'128","reported_seq":723,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:33:38.962176+0000","last_change":"2023-09-23T20:32:36.310656+0000","last_active":"2023-09-23T20:33:38.962176+0000","last_peered":"2023-09-23T20:33:38.962176+0000","last_clean":"2023-09-23T20:33:38.962176+0000","last_became_active":"2023-09-23T20:32:36.310155+0000","last_became_peered":"2023-09-23T20:32:36.310155+0000","last_unstale":"2023-09-23T20:33:38.962176+0000","last_undegraded":"2023-09-23T20:33:38.962176+0000","last_fullsized":"2023-09-23T20:33:38.962176+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:58:14.217466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000293202,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":579,"num_read_kb":1473,"num_write":96,"num_write_kb":1062,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":104,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:33:38.964587+0000","last_change":"2023-09-23T20:32:50.539051+0000","last_active":"2023-09-23T20:33:38.964587+0000","last_peered":"2023-09-23T20:33:38.964587+0000","last_clean":"2023-09-23T20:33:38.964587+0000","last_became_active":"2023-09-23T20:32:50.538755+0000","last_became_peered":"2023-09-23T20:32:50.538755+0000","last_unstale":"2023-09-23T20:33:38.964587+0000","last_undegraded":"2023-09-23T20:33:38.964587+0000","last_fullsized":"2023-09-23T20:33:38.964587+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:16.656613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.2","version":"334'6574","reported_seq":6621,"reported_epoch":334,"state":"active+clean","last_fresh":"2023-09-23T20:33:43.732093+0000","last_change":"2023-09-23T20:32:50.557165+0000","last_active":"2023-09-23T20:33:43.732093+0000","last_peered":"2023-09-23T20:33:43.732093+0000","last_clean":"2023-09-23T20:33:43.732093+0000","last_became_active":"2023-09-23T20:32:50.553409+0000","last_became_peered":"2023-09-23T20:32:50.553409+0000","last_unstale":"2023-09-23T20:33:43.732093+0000","last_undegraded":"2023-09-23T20:33:43.732093+0000","last_fullsized":"2023-09-23T20:33:43.732093+0000","mapping_epoch":319,"log_start":"307'301","ondisk_log_start":"307'301","created":303,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":6273,"log_dups_size":0,"ondisk_log_size":6273,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:55:13.578195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":335413248,"num_objects":5118,"num_object_clones":0,"num_object_copies":15354,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5118,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6574,"num_write_kb":374144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"90'161","reported_seq":464,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:33:38.965086+0000","last_change":"2023-09-23T20:33:37.002856+0000","last_active":"2023-09-23T20:33:38.965086+0000","last_peered":"2023-09-23T20:33:38.965086+0000","last_clean":"2023-09-23T20:33:38.965086+0000","last_became_active":"2023-09-23T20:32:50.533470+0000","last_became_peered":"2023-09-23T20:32:50.533470+0000","last_unstale":"2023-09-23T20:33:38.965086+0000","last_undegraded":"2023-09-23T20:33:38.965086+0000","last_fullsized":"2023-09-23T20:33:38.965086+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:28:03.553106+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:28:03.553106+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.7","version":"86'117","reported_seq":603,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:33:38.970393+0000","last_change":"2023-09-23T20:31:32.814392+0000","last_active":"2023-09-23T20:33:38.970393+0000","last_peered":"2023-09-23T20:33:38.970393+0000","last_clean":"2023-09-23T20:33:38.970393+0000","last_became_active":"2023-09-23T20:31:32.813857+0000","last_became_peered":"2023-09-23T20:31:32.813857+0000","last_unstale":"2023-09-23T20:33:38.970393+0000","last_undegraded":"2023-09-23T20:33:38.970393+0000","last_fullsized":"2023-09-23T20:33:38.970393+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:56:57.863005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046891400000000002,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":893,"num_read_kb":922,"num_write":82,"num_write_kb":4193,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":330,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:33:38.975157+0000","last_change":"2023-09-23T20:32:33.277827+0000","last_active":"2023-09-23T20:33:38.975157+0000","last_peered":"2023-09-23T20:33:38.975157+0000","last_clean":"2023-09-23T20:33:38.975157+0000","last_became_active":"2023-09-23T20:32:33.236087+0000","last_became_peered":"2023-09-23T20:32:33.236087+0000","last_unstale":"2023-09-23T20:33:38.975157+0000","last_undegraded":"2023-09-23T20:33:38.975157+0000","last_fullsized":"2023-09-23T20:33:38.975157+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:20:58.176854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":31,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:33:38.967613+0000","last_change":"2023-09-23T20:32:33.238290+0000","last_active":"2023-09-23T20:33:38.967613+0000","last_peered":"2023-09-23T20:33:38.967613+0000","last_clean":"2023-09-23T20:33:38.967613+0000","last_became_active":"2023-09-23T20:32:33.237984+0000","last_became_peered":"2023-09-23T20:32:33.237984+0000","last_unstale":"2023-09-23T20:33:38.967613+0000","last_undegraded":"2023-09-23T20:33:38.967613+0000","last_fullsized":"2023-09-23T20:33:38.967613+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:49:02.089307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"90'161","reported_seq":725,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:33:38.991477+0000","last_change":"2023-09-23T20:33:38.990264+0000","last_active":"2023-09-23T20:33:38.991477+0000","last_peered":"2023-09-23T20:33:38.991477+0000","last_clean":"2023-09-23T20:33:38.991477+0000","last_became_active":"2023-09-23T20:33:38.989881+0000","last_became_peered":"2023-09-23T20:33:38.989881+0000","last_unstale":"2023-09-23T20:33:38.991477+0000","last_undegraded":"2023-09-23T20:33:38.991477+0000","last_fullsized":"2023-09-23T20:33:38.991477+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:28:05.872074+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:28:05.872074+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0029215980000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":298,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:33:38.970123+0000","last_change":"2023-09-23T20:32:33.237234+0000","last_active":"2023-09-23T20:33:38.970123+0000","last_peered":"2023-09-23T20:33:38.970123+0000","last_clean":"2023-09-23T20:33:38.970123+0000","last_became_active":"2023-09-23T20:32:33.236881+0000","last_became_peered":"2023-09-23T20:32:33.236881+0000","last_unstale":"2023-09-23T20:33:38.970123+0000","last_undegraded":"2023-09-23T20:33:38.970123+0000","last_fullsized":"2023-09-23T20:33:38.970123+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:56:17.384831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":185,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T20:33:37.931479+0000","last_change":"2023-09-23T20:29:45.538314+0000","last_active":"2023-09-23T20:33:37.931479+0000","last_peered":"2023-09-23T20:33:37.931479+0000","last_clean":"2023-09-23T20:33:37.931479+0000","last_became_active":"2023-09-23T20:29:45.536847+0000","last_became_peered":"2023-09-23T20:29:45.536847+0000","last_unstale":"2023-09-23T20:33:37.931479+0000","last_undegraded":"2023-09-23T20:33:37.931479+0000","last_fullsized":"2023-09-23T20:33:37.931479+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:51.948721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"90'161","reported_seq":532,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:33:38.976364+0000","last_change":"2023-09-23T20:33:38.975985+0000","last_active":"2023-09-23T20:33:38.976364+0000","last_peered":"2023-09-23T20:33:38.976364+0000","last_clean":"2023-09-23T20:33:38.976364+0000","last_became_active":"2023-09-23T20:33:38.975688+0000","last_became_peered":"2023-09-23T20:33:38.975688+0000","last_unstale":"2023-09-23T20:33:38.976364+0000","last_undegraded":"2023-09-23T20:33:38.976364+0000","last_fullsized":"2023-09-23T20:33:38.976364+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:44.386120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:44.386120+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0015151330000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":343,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:33:38.964795+0000","last_change":"2023-09-23T20:32:33.235987+0000","last_active":"2023-09-23T20:33:38.964795+0000","last_peered":"2023-09-23T20:33:38.964795+0000","last_clean":"2023-09-23T20:33:38.964795+0000","last_became_active":"2023-09-23T20:32:33.226544+0000","last_became_peered":"2023-09-23T20:32:33.226544+0000","last_unstale":"2023-09-23T20:33:38.964795+0000","last_undegraded":"2023-09-23T20:33:38.964795+0000","last_fullsized":"2023-09-23T20:33:38.964795+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:53:22.794856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":289,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:33:38.960185+0000","last_change":"2023-09-23T20:33:00.177976+0000","last_active":"2023-09-23T20:33:38.960185+0000","last_peered":"2023-09-23T20:33:38.960185+0000","last_clean":"2023-09-23T20:33:38.960185+0000","last_became_active":"2023-09-23T20:33:00.177665+0000","last_became_peered":"2023-09-23T20:33:00.177665+0000","last_unstale":"2023-09-23T20:33:38.960185+0000","last_undegraded":"2023-09-23T20:33:38.960185+0000","last_fullsized":"2023-09-23T20:33:38.960185+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:20:25.273907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":351,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:33:38.965186+0000","last_change":"2023-09-23T20:32:33.268260+0000","last_active":"2023-09-23T20:33:38.965186+0000","last_peered":"2023-09-23T20:33:38.965186+0000","last_clean":"2023-09-23T20:33:38.965186+0000","last_became_active":"2023-09-23T20:32:33.233695+0000","last_became_peered":"2023-09-23T20:32:33.233695+0000","last_unstale":"2023-09-23T20:33:38.965186+0000","last_undegraded":"2023-09-23T20:33:38.965186+0000","last_fullsized":"2023-09-23T20:33:38.965186+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:19:29.299623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":188,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:33:38.969825+0000","last_change":"2023-09-23T20:33:00.179961+0000","last_active":"2023-09-23T20:33:38.969825+0000","last_peered":"2023-09-23T20:33:38.969825+0000","last_clean":"2023-09-23T20:33:38.969825+0000","last_became_active":"2023-09-23T20:33:00.178429+0000","last_became_peered":"2023-09-23T20:33:00.178429+0000","last_unstale":"2023-09-23T20:33:38.969825+0000","last_undegraded":"2023-09-23T20:33:38.969825+0000","last_fullsized":"2023-09-23T20:33:38.969825+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:28:20.964989+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1c","version":"90'161","reported_seq":503,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:33:40.373823+0000","last_change":"2023-09-23T20:33:38.990525+0000","last_active":"2023-09-23T20:33:40.373823+0000","last_peered":"2023-09-23T20:33:40.373823+0000","last_clean":"2023-09-23T20:33:40.373823+0000","last_became_active":"2023-09-23T20:33:38.990173+0000","last_became_peered":"2023-09-23T20:33:38.990173+0000","last_unstale":"2023-09-23T20:33:40.373823+0000","last_undegraded":"2023-09-23T20:33:40.373823+0000","last_fullsized":"2023-09-23T20:33:40.373823+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:41.701171+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:41.701171+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0019955509999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":54614,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":149,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:33:38.969392+0000","last_change":"2023-09-23T20:32:50.537447+0000","last_active":"2023-09-23T20:33:38.969392+0000","last_peered":"2023-09-23T20:33:38.969392+0000","last_clean":"2023-09-23T20:33:38.969392+0000","last_became_active":"2023-09-23T20:32:50.537136+0000","last_became_peered":"2023-09-23T20:32:50.537136+0000","last_unstale":"2023-09-23T20:33:38.969392+0000","last_undegraded":"2023-09-23T20:33:38.969392+0000","last_fullsized":"2023-09-23T20:33:38.969392+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:46:51.599492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":47,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T20:33:37.930080+0000","last_change":"2023-09-23T20:32:50.537450+0000","last_active":"2023-09-23T20:33:37.930080+0000","last_peered":"2023-09-23T20:33:37.930080+0000","last_clean":"2023-09-23T20:33:37.930080+0000","last_became_active":"2023-09-23T20:32:50.537128+0000","last_became_peered":"2023-09-23T20:32:50.537128+0000","last_unstale":"2023-09-23T20:33:37.930080+0000","last_undegraded":"2023-09-23T20:33:37.930080+0000","last_fullsized":"2023-09-23T20:33:37.930080+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:37:15.889029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1d","version":"90'161","reported_seq":612,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:33:38.972651+0000","last_change":"2023-09-23T20:33:38.972393+0000","last_active":"2023-09-23T20:33:38.972651+0000","last_peered":"2023-09-23T20:33:38.972651+0000","last_clean":"2023-09-23T20:33:38.972651+0000","last_became_active":"2023-09-23T20:33:38.972109+0000","last_became_peered":"2023-09-23T20:33:38.972109+0000","last_unstale":"2023-09-23T20:33:38.972651+0000","last_undegraded":"2023-09-23T20:33:38.972651+0000","last_fullsized":"2023-09-23T20:33:38.972651+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:46.388545+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:46.388545+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.00168356,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":315,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:33:38.975331+0000","last_change":"2023-09-23T20:25:39.668882+0000","last_active":"2023-09-23T20:33:38.975331+0000","last_peered":"2023-09-23T20:33:38.975331+0000","last_clean":"2023-09-23T20:33:38.975331+0000","last_became_active":"2023-09-23T20:25:39.637128+0000","last_became_peered":"2023-09-23T20:25:39.637128+0000","last_unstale":"2023-09-23T20:33:38.975331+0000","last_undegraded":"2023-09-23T20:33:38.975331+0000","last_fullsized":"2023-09-23T20:33:38.975331+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:56.406298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":31,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T20:33:38.967149+0000","last_change":"2023-09-23T20:32:33.232281+0000","last_active":"2023-09-23T20:33:38.967149+0000","last_peered":"2023-09-23T20:33:38.967149+0000","last_clean":"2023-09-23T20:33:38.967149+0000","last_became_active":"2023-09-23T20:32:33.232003+0000","last_became_peered":"2023-09-23T20:32:33.232003+0000","last_unstale":"2023-09-23T20:33:38.967149+0000","last_undegraded":"2023-09-23T20:33:38.967149+0000","last_fullsized":"2023-09-23T20:33:38.967149+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:58:27.314355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":5385748504,"num_objects":82181,"num_object_clones":0,"num_object_copies":246543,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5117,"num_objects_misplaced":1577,"num_objects_unfound":0,"num_objects_dirty":82181,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":106919,"num_write_kb":6051137,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3172,"num_bytes_recovered":207880192,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16459902976,"data_stored":16459890760,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":102311,"ondisk_log_size":102311,"up":48,"acting":44,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":48,"num_bytes_recovered":6522,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26691,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":389805,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":58,"num_bytes_recovered":4194342,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6082,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201551,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2170380,"kb_used_data":1898372,"kb_used_omap":0,"kb_used_meta":272000,"kb_avail":91574772,"statfs":{"total":95995035648,"available":93772566528,"internally_reserved":0,"allocated":1943932928,"data_stored":1942875490,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":278528000},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:33:41 2023","interfaces":[{"interface":"back","average":{"1min":3.8500000000000001,"5min":1.929,"15min":1.5},"min":{"1min":0.255,"5min":0.23599999999999999,"15min":0.20799999999999999},"max":{"1min":14.66,"5min":14.66,"15min":48.798999999999999},"last":6.1959999999999997},{"interface":"front","average":{"1min":1.996,"5min":1.7090000000000001,"15min":1.5489999999999999},"min":{"1min":0.31,"5min":0.27300000000000002,"15min":0.24399999999999999},"max":{"1min":6.6230000000000002,"5min":7.968,"15min":48.247},"last":3.7320000000000002}]},{"osd":1,"last update":"Sat Sep 23 20:32:53 2023","interfaces":[{"interface":"back","average":{"1min":1.405,"5min":1.8859999999999999,"15min":1.8029999999999999},"min":{"1min":0.31900000000000001,"5min":0.254,"15min":0.254},"max":{"1min":4.7309999999999999,"5min":14.026,"15min":14.026},"last":0.68700000000000006},{"interface":"front","average":{"1min":1.3700000000000001,"5min":1.6699999999999999,"15min":1.877},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":5.2750000000000004,"5min":5.7679999999999998,"15min":5.7679999999999998},"last":0.92200000000000004}]},{"osd":2,"last update":"Sat Sep 23 20:33:43 2023","interfaces":[{"interface":"back","average":{"1min":2.879,"5min":2.085,"15min":1.5389999999999999},"min":{"1min":0.28599999999999998,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":12.196999999999999,"5min":13.173999999999999,"15min":13.173999999999999},"last":0.89200000000000002},{"interface":"front","average":{"1min":3.8639999999999999,"5min":2.6200000000000001,"15min":1.768},"min":{"1min":0.27400000000000002,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":23.494,"5min":23.494,"15min":23.494},"last":0.36899999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:33:05 2023","interfaces":[{"interface":"back","average":{"1min":2.903,"5min":2.2229999999999999,"15min":1.5880000000000001},"min":{"1min":0.375,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":7.9850000000000003,"5min":17.155999999999999,"15min":17.155999999999999},"last":0.40100000000000002},{"interface":"front","average":{"1min":2.7829999999999999,"5min":2.0630000000000002,"15min":1.5049999999999999},"min":{"1min":0.47299999999999998,"5min":0.33100000000000002,"15min":0.27600000000000002},"max":{"1min":6.6379999999999999,"5min":16.960000000000001,"15min":16.960000000000001},"last":2.089}]},{"osd":4,"last update":"Sat Sep 23 20:33:05 2023","interfaces":[{"interface":"back","average":{"1min":0.65900000000000003,"5min":1.381,"15min":1.034},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.27000000000000002},"max":{"1min":2.3370000000000002,"5min":15.177,"15min":15.177},"last":0.77600000000000002},{"interface":"front","average":{"1min":0.92500000000000004,"5min":1.1619999999999999,"15min":1.155},"min":{"1min":0.33900000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":5.2190000000000003,"5min":11.122,"15min":11.122},"last":1.129}]},{"osd":5,"last update":"Sat Sep 23 20:33:05 2023","interfaces":[{"interface":"back","average":{"1min":0.77100000000000002,"5min":0.97599999999999998,"15min":1.1120000000000001},"min":{"1min":0.317,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":3.0339999999999998,"5min":6.8300000000000001,"15min":13.241},"last":0.33200000000000002},{"interface":"front","average":{"1min":0.55100000000000005,"5min":1.272,"15min":1.2},"min":{"1min":0.35899999999999999,"5min":0.32200000000000001,"15min":0.315},"max":{"1min":1.3400000000000001,"5min":18.831,"15min":18.831},"last":0.747}]},{"osd":6,"last update":"Sat Sep 23 20:33:05 2023","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":1.2270000000000001,"15min":1.0960000000000001},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33400000000000002},"max":{"1min":1.0940000000000001,"5min":12.298999999999999,"15min":13.185},"last":0.72099999999999997},{"interface":"front","average":{"1min":0.93400000000000005,"5min":1.069,"15min":1.0649999999999999},"min":{"1min":0.37,"5min":0.318,"15min":0.29899999999999999},"max":{"1min":5.1059999999999999,"5min":8.0069999999999997,"15min":9.9120000000000008},"last":1.1599999999999999}]}]},{"osd":6,"up_from":47,"seq":201863463187,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2386888,"kb_used_data":2134144,"kb_used_omap":0,"kb_used_meta":252736,"kb_avail":91358264,"statfs":{"total":95995035648,"available":93550862336,"internally_reserved":0,"allocated":2185363456,"data_stored":2184310011,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":258801664},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:33:41 2023","interfaces":[{"interface":"back","average":{"1min":2.0110000000000001,"5min":2.0859999999999999,"15min":1.3620000000000001},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.221},"max":{"1min":8.7509999999999994,"5min":53.701999999999998,"15min":53.701999999999998},"last":1.2290000000000001},{"interface":"front","average":{"1min":2.407,"5min":2.5579999999999998,"15min":1.696},"min":{"1min":0.25800000000000001,"5min":0.251,"15min":0.25},"max":{"1min":15.452999999999999,"5min":53.875999999999998,"15min":53.875999999999998},"last":1.3220000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:32:49 2023","interfaces":[{"interface":"back","average":{"1min":1.216,"5min":2.262,"15min":2.5880000000000001},"min":{"1min":0.35399999999999998,"5min":0.249,"15min":0.249},"max":{"1min":3.6019999999999999,"5min":14.1,"15min":14.1},"last":1.48},{"interface":"front","average":{"1min":1.4390000000000001,"5min":2.7050000000000001,"15min":3.226},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":4.9199999999999999,"5min":14.266999999999999,"15min":14.266999999999999},"last":0.41899999999999998}]},{"osd":2,"last update":"Sat Sep 23 20:33:41 2023","interfaces":[{"interface":"back","average":{"1min":2.7530000000000001,"5min":2.355,"15min":1.589},"min":{"1min":0.33900000000000002,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":8.5860000000000003,"5min":10.631,"15min":13.856},"last":2.5939999999999999},{"interface":"front","average":{"1min":2.4590000000000001,"5min":2.2879999999999998,"15min":1.5},"min":{"1min":0.33000000000000002,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":8.6449999999999996,"5min":10.151999999999999,"15min":10.151999999999999},"last":2.6070000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:33:20 2023","interfaces":[{"interface":"back","average":{"1min":2.1110000000000002,"5min":1.9670000000000001,"15min":1.504},"min":{"1min":0.30499999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":7.5620000000000003,"5min":7.7130000000000001,"15min":9.4239999999999995},"last":2.8119999999999998},{"interface":"front","average":{"1min":2.5310000000000001,"5min":2.1739999999999999,"15min":1.573},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":10.805,"5min":10.805,"15min":10.805},"last":2.6960000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:33:00 2023","interfaces":[{"interface":"back","average":{"1min":0.49399999999999999,"5min":0.78800000000000003,"15min":0.85899999999999999},"min":{"1min":0.253,"5min":0.251,"15min":0.251},"max":{"1min":2.9020000000000001,"5min":8.4749999999999996,"15min":10.173},"last":0.501},{"interface":"front","average":{"1min":0.83499999999999996,"5min":1.1259999999999999,"15min":1.0049999999999999},"min":{"1min":0.28199999999999997,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":7.7629999999999999,"5min":7.7629999999999999,"15min":8.3249999999999993},"last":0.38900000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:33:00 2023","interfaces":[{"interface":"back","average":{"1min":0.88300000000000001,"5min":0.83099999999999996,"15min":0.92300000000000004},"min":{"1min":0.311,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":6.8310000000000004,"5min":8.4499999999999993,"15min":14.085000000000001},"last":0.53600000000000003},{"interface":"front","average":{"1min":0.90800000000000003,"5min":1.258,"15min":1.262},"min":{"1min":0.32100000000000001,"5min":0.30599999999999999,"15min":0.29499999999999998},"max":{"1min":7.7300000000000004,"5min":9.9299999999999997,"15min":18.192},"last":0.432}]},{"osd":7,"last update":"Sat Sep 23 20:33:20 2023","interfaces":[{"interface":"back","average":{"1min":1.0960000000000001,"5min":1.0920000000000001,"15min":1.0649999999999999},"min":{"1min":0.34499999999999997,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":5.0350000000000001,"5min":8.5060000000000002,"15min":10.217000000000001},"last":0.46899999999999997},{"interface":"front","average":{"1min":1.4239999999999999,"5min":1.409,"15min":1.218},"min":{"1min":0.375,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":7.601,"5min":7.8120000000000003,"15min":9.7360000000000007},"last":0.45800000000000002}]}]},{"osd":1,"up_from":266,"seq":1142461300785,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2546240,"kb_used_data":2330744,"kb_used_omap":8,"kb_used_meta":215223,"kb_avail":91198912,"statfs":{"total":95995035648,"available":93387685888,"internally_reserved":0,"allocated":2386681856,"data_stored":2384879173,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8206,"internal_metadata":220389362},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:32:49 2023","interfaces":[{"interface":"back","average":{"1min":4.5289999999999999,"5min":2.7709999999999999,"15min":2.4769999999999999},"min":{"1min":0.224,"5min":0.218,"15min":0.218},"max":{"1min":52.689999999999998,"5min":52.689999999999998,"15min":52.689999999999998},"last":2.423},{"interface":"front","average":{"1min":4.5279999999999996,"5min":3.1259999999999999,"15min":3.1720000000000002},"min":{"1min":0.218,"5min":0.217,"15min":0.217},"max":{"1min":52.662999999999997,"5min":52.662999999999997,"15min":52.662999999999997},"last":0.997}]},{"osd":2,"last update":"Sat Sep 23 20:32:49 2023","interfaces":[{"interface":"back","average":{"1min":1.371,"5min":3.8319999999999999,"15min":4.694},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":4.8689999999999998,"5min":16.151,"15min":16.151},"last":4.798},{"interface":"front","average":{"1min":1.7749999999999999,"5min":2.0030000000000001,"15min":2.0630000000000002},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":8.077,"5min":12.356,"15min":12.356},"last":0.34899999999999998}]},{"osd":3,"last update":"Sat Sep 23 20:32:49 2023","interfaces":[{"interface":"back","average":{"1min":8.2010000000000005,"5min":4.3849999999999998,"15min":4.0389999999999997},"min":{"1min":0.311,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":80.745999999999995,"5min":80.745999999999995,"15min":80.745999999999995},"last":0.313},{"interface":"front","average":{"1min":7.6909999999999998,"5min":4.7530000000000001,"15min":4.8150000000000004},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":82.988,"5min":82.988,"15min":82.988},"last":8.6600000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:32:49 2023","interfaces":[{"interface":"back","average":{"1min":1.0700000000000001,"5min":1.855,"15min":2.004},"min":{"1min":0.40699999999999997,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":2.863,"5min":9.4429999999999996,"15min":9.4429999999999996},"last":0.77400000000000002},{"interface":"front","average":{"1min":1.272,"5min":1.7150000000000001,"15min":1.921},"min":{"1min":0.372,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":4.0730000000000004,"5min":10.760999999999999,"15min":10.760999999999999},"last":0.372}]},{"osd":5,"last update":"Sat Sep 23 20:32:49 2023","interfaces":[{"interface":"back","average":{"1min":0.95799999999999996,"5min":2.1200000000000001,"15min":2.5739999999999998},"min":{"1min":0.33400000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":2.649,"5min":9.6479999999999997,"15min":9.6479999999999997},"last":0.58299999999999996},{"interface":"front","average":{"1min":1.0669999999999999,"5min":1.9650000000000001,"15min":2.048},"min":{"1min":0.36899999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":2.9700000000000002,"5min":9.4700000000000006,"15min":9.4700000000000006},"last":0.74199999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:32:49 2023","interfaces":[{"interface":"back","average":{"1min":1.018,"5min":4.9980000000000002,"15min":6.6319999999999997},"min":{"1min":0.40300000000000002,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":2.714,"5min":72.682000000000002,"15min":72.682000000000002},"last":0.63100000000000001},{"interface":"front","average":{"1min":1.0740000000000001,"5min":4.5720000000000001,"15min":5.8170000000000002},"min":{"1min":0.438,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":2.9319999999999999,"5min":72.563999999999993,"15min":72.563999999999993},"last":0.82699999999999996}]},{"osd":7,"last update":"Sat Sep 23 20:32:49 2023","interfaces":[{"interface":"back","average":{"1min":1.085,"5min":2.4180000000000001,"15min":2.9940000000000002},"min":{"1min":0.39900000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":2.6880000000000002,"5min":9.6850000000000005,"15min":9.6850000000000005},"last":0.66300000000000003},{"interface":"front","average":{"1min":1.147,"5min":2.1480000000000001,"15min":2.2890000000000001},"min":{"1min":0.51200000000000001,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":2.8980000000000001,"5min":9.3949999999999996,"15min":9.3949999999999996},"last":0.80100000000000005}]}]},{"osd":0,"up_from":94,"seq":403726925988,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3065700,"kb_used_data":2638172,"kb_used_omap":7,"kb_used_meta":427512,"kb_avail":90679452,"statfs":{"total":95995035648,"available":92855758848,"internally_reserved":0,"allocated":2701488128,"data_stored":2699829000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":437772305},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:32:55 2023","interfaces":[{"interface":"back","average":{"1min":1.825,"5min":2.9750000000000001,"15min":3.5830000000000002},"min":{"1min":0.245,"5min":0.17999999999999999,"15min":0.17999999999999999},"max":{"1min":8.0630000000000006,"5min":15.542,"15min":15.542},"last":4.274},{"interface":"front","average":{"1min":1.873,"5min":3.3050000000000002,"15min":4.0730000000000004},"min":{"1min":0.29899999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":5.3410000000000002,"5min":10.49,"15min":10.49},"last":4.2480000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:33:35 2023","interfaces":[{"interface":"back","average":{"1min":3.698,"5min":3.2450000000000001,"15min":2.0819999999999999},"min":{"1min":0.193,"5min":0.185,"15min":0.185},"max":{"1min":22.460999999999999,"5min":77.448999999999998,"15min":77.448999999999998},"last":4.0430000000000001},{"interface":"front","average":{"1min":5.0039999999999996,"5min":3.1760000000000002,"15min":1.9219999999999999},"min":{"1min":0.317,"5min":0.188,"15min":0.188},"max":{"1min":22.260000000000002,"5min":81.724999999999994,"15min":81.724999999999994},"last":1.052}]},{"osd":3,"last update":"Sat Sep 23 20:33:05 2023","interfaces":[{"interface":"back","average":{"1min":3.0019999999999998,"5min":3.7149999999999999,"15min":2.0209999999999999},"min":{"1min":0.35799999999999998,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":13.51,"5min":81.400999999999996,"15min":81.400999999999996},"last":3.923},{"interface":"front","average":{"1min":3.113,"5min":3.8559999999999999,"15min":1.9790000000000001},"min":{"1min":0.35599999999999998,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":12.651999999999999,"5min":81.137,"15min":81.137},"last":4.2249999999999996}]},{"osd":4,"last update":"Sat Sep 23 20:33:35 2023","interfaces":[{"interface":"back","average":{"1min":2.7599999999999998,"5min":1.825,"15min":1.6479999999999999},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":12.587999999999999,"5min":12.587999999999999,"15min":23.876999999999999},"last":3.8929999999999998},{"interface":"front","average":{"1min":3.6019999999999999,"5min":1.972,"15min":1.512},"min":{"1min":0.32700000000000001,"5min":0.31900000000000001,"15min":0.29299999999999998},"max":{"1min":23.001000000000001,"5min":23.001000000000001,"15min":23.681999999999999},"last":4.1180000000000003}]},{"osd":5,"last update":"Sat Sep 23 20:33:35 2023","interfaces":[{"interface":"back","average":{"1min":3.0579999999999998,"5min":2.1509999999999998,"15min":1.8400000000000001},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34200000000000003},"max":{"1min":22.969999999999999,"5min":22.969999999999999,"15min":24.198},"last":4.1589999999999998},{"interface":"front","average":{"1min":4.4900000000000002,"5min":2.3660000000000001,"15min":1.764},"min":{"1min":0.376,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":22.236000000000001,"5min":22.236000000000001,"15min":22.236000000000001},"last":0.97699999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:33:35 2023","interfaces":[{"interface":"back","average":{"1min":3.3719999999999999,"5min":2.016,"15min":1.5780000000000001},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.318},"max":{"1min":22.934999999999999,"5min":22.934999999999999,"15min":24.408000000000001},"last":4.1929999999999996},{"interface":"front","average":{"1min":4.585,"5min":2.1850000000000001,"15min":1.607},"min":{"1min":0.34100000000000003,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":22.164999999999999,"5min":22.164999999999999,"15min":25.271000000000001},"last":1.0089999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:33:35 2023","interfaces":[{"interface":"back","average":{"1min":4.6559999999999997,"5min":2.5499999999999998,"15min":1.7869999999999999},"min":{"1min":0.39600000000000002,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":22.206,"5min":24.614000000000001,"15min":25.359000000000002},"last":1.0309999999999999},{"interface":"front","average":{"1min":3.5960000000000001,"5min":2.6080000000000001,"15min":2.008},"min":{"1min":0.39100000000000001,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":22.422999999999998,"5min":24.669,"15min":24.669},"last":3.9500000000000002}]}]},{"osd":2,"up_from":104,"seq":446676598935,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2500556,"kb_used_data":2043396,"kb_used_omap":11,"kb_used_meta":457076,"kb_avail":91244596,"statfs":{"total":95995035648,"available":93434466304,"internally_reserved":0,"allocated":2092437504,"data_stored":2090791826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11878,"internal_metadata":468046234},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:33:40 2023","interfaces":[{"interface":"back","average":{"1min":2.802,"5min":3.0350000000000001,"15min":1.8979999999999999},"min":{"1min":0.249,"5min":0.17999999999999999,"15min":0.17999999999999999},"max":{"1min":11.599,"5min":82.424999999999997,"15min":82.424999999999997},"last":11.599},{"interface":"front","average":{"1min":2.121,"5min":3.339,"15min":1.8069999999999999},"min":{"1min":0.29799999999999999,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":5.6440000000000001,"5min":82.524000000000001,"15min":82.524000000000001},"last":1.018}]},{"osd":1,"last update":"Sat Sep 23 20:32:49 2023","interfaces":[{"interface":"back","average":{"1min":1.782,"5min":3.5790000000000002,"15min":4.4269999999999996},"min":{"1min":0.245,"5min":0.221,"15min":0.221},"max":{"1min":6.6589999999999998,"5min":16.280000000000001,"15min":16.280000000000001},"last":15.711},{"interface":"front","average":{"1min":1.696,"5min":2.399,"15min":2.7909999999999999},"min":{"1min":0.28000000000000003,"5min":0.248,"15min":0.248},"max":{"1min":5.1790000000000003,"5min":30.565000000000001,"15min":30.565000000000001},"last":14.375}]},{"osd":3,"last update":"Sat Sep 23 20:33:04 2023","interfaces":[{"interface":"back","average":{"1min":2.6699999999999999,"5min":2.4129999999999998,"15min":1.456},"min":{"1min":0.31900000000000001,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":11.766999999999999,"5min":16.331,"15min":16.331},"last":5.569},{"interface":"front","average":{"1min":2.859,"5min":2.786,"15min":1.6879999999999999},"min":{"1min":0.28999999999999998,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":15.266,"5min":30.704999999999998,"15min":30.704999999999998},"last":15.173}]},{"osd":4,"last update":"Sat Sep 23 20:33:40 2023","interfaces":[{"interface":"back","average":{"1min":1.7689999999999999,"5min":2.1389999999999998,"15min":1.464},"min":{"1min":0.311,"5min":0.311,"15min":0.28100000000000003},"max":{"1min":4.601,"5min":16.797999999999998,"15min":16.797999999999998},"last":1.095},{"interface":"front","average":{"1min":3.472,"5min":2.262,"15min":1.5620000000000001},"min":{"1min":0.35899999999999999,"5min":0.309,"15min":0.27600000000000002},"max":{"1min":11.009,"5min":11.009,"15min":14.971},"last":5.6630000000000003}]},{"osd":5,"last update":"Sat Sep 23 20:33:40 2023","interfaces":[{"interface":"back","average":{"1min":1.891,"5min":3.4649999999999999,"15min":1.982},"min":{"1min":0.34799999999999998,"5min":0.314,"15min":0.309},"max":{"1min":4.6230000000000002,"5min":95.393000000000001,"15min":95.393000000000001},"last":1.0629999999999999},{"interface":"front","average":{"1min":3.512,"5min":3.637,"15min":2.1269999999999998},"min":{"1min":0.38800000000000001,"5min":0.34799999999999998,"15min":0.32300000000000001},"max":{"1min":10.922000000000001,"5min":95.414000000000001,"15min":95.414000000000001},"last":5.6150000000000002}]},{"osd":6,"last update":"Sat Sep 23 20:33:40 2023","interfaces":[{"interface":"back","average":{"1min":3.6560000000000001,"5min":2.5640000000000001,"15min":1.665},"min":{"1min":0.41499999999999998,"5min":0.33600000000000002,"15min":0.32100000000000001},"max":{"1min":12.629,"5min":13.154999999999999,"15min":15.023999999999999},"last":5.6920000000000002},{"interface":"front","average":{"1min":2.4780000000000002,"5min":1.9590000000000001,"15min":1.6180000000000001},"min":{"1min":0.318,"5min":0.316,"15min":0.316},"max":{"1min":11.523,"5min":30.670999999999999,"15min":30.670999999999999},"last":11.523}]},{"osd":7,"last update":"Sat Sep 23 20:33:40 2023","interfaces":[{"interface":"back","average":{"1min":2.8530000000000002,"5min":2.073,"15min":1.7010000000000001},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":12.734999999999999,"5min":30.741,"15min":30.741},"last":11.567},{"interface":"front","average":{"1min":1.885,"5min":2.4929999999999999,"15min":1.6970000000000001},"min":{"1min":0.39800000000000002,"5min":0.38,"15min":0.38},"max":{"1min":4.6280000000000001,"5min":16.719000000000001,"15min":16.719000000000001},"last":1.1479999999999999}]}]},{"osd":3,"up_from":109,"seq":468151435409,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3723792,"kb_used_data":3229832,"kb_used_omap":4,"kb_used_meta":493883,"kb_avail":90021360,"statfs":{"total":95995035648,"available":92181872640,"internally_reserved":0,"allocated":3307347968,"data_stored":3305696934,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4514,"internal_metadata":505736798},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:33:09 2023","interfaces":[{"interface":"back","average":{"1min":3.3149999999999999,"5min":2.2759999999999998,"15min":1.748},"min":{"1min":0.191,"5min":0.186,"15min":0.186},"max":{"1min":14.808999999999999,"5min":14.808999999999999,"15min":67.305999999999997},"last":3.0910000000000002},{"interface":"front","average":{"1min":3.016,"5min":2.2629999999999999,"15min":1.629},"min":{"1min":0.22600000000000001,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":7.9880000000000004,"5min":13.743,"15min":67.403999999999996},"last":5.2469999999999999}]},{"osd":1,"last update":"Sat Sep 23 20:32:51 2023","interfaces":[{"interface":"back","average":{"1min":1.617,"5min":2.6579999999999999,"15min":2.944},"min":{"1min":0.27100000000000002,"5min":0.222,"15min":0.222},"max":{"1min":5.8380000000000001,"5min":13.263,"15min":13.263},"last":0.34200000000000003},{"interface":"front","average":{"1min":3.4620000000000002,"5min":2.988,"15min":3.1640000000000001},"min":{"1min":0.26400000000000001,"5min":0.252,"15min":0.252},"max":{"1min":32.603999999999999,"5min":32.603999999999999,"15min":32.603999999999999},"last":5.6790000000000003}]},{"osd":2,"last update":"Sat Sep 23 20:33:09 2023","interfaces":[{"interface":"back","average":{"1min":4.6150000000000002,"5min":2.8679999999999999,"15min":1.7410000000000001},"min":{"1min":0.33200000000000002,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":32.857999999999997,"5min":32.857999999999997,"15min":32.857999999999997},"last":5.7229999999999999},{"interface":"front","average":{"1min":2.6280000000000001,"5min":2.6219999999999999,"15min":1.6830000000000001},"min":{"1min":0.26900000000000002,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":13.568,"5min":19.603000000000002,"15min":19.603000000000002},"last":5.6040000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:33:09 2023","interfaces":[{"interface":"back","average":{"1min":1.524,"5min":1.861,"15min":1.476},"min":{"1min":0.39100000000000001,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":5.3029999999999999,"5min":11.802,"15min":46.042999999999999},"last":0.41499999999999998},{"interface":"front","average":{"1min":1.6040000000000001,"5min":2.2919999999999998,"15min":1.7310000000000001},"min":{"1min":0.36399999999999999,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":7.9409999999999998,"5min":19.635000000000002,"15min":46.579999999999998},"last":5.3940000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:33:09 2023","interfaces":[{"interface":"back","average":{"1min":1.581,"5min":2.04,"15min":1.6699999999999999},"min":{"1min":0.32200000000000001,"5min":0.31900000000000001,"15min":0.311},"max":{"1min":5.3280000000000003,"5min":11.507999999999999,"15min":45.292000000000002},"last":0.64000000000000001},{"interface":"front","average":{"1min":2.2120000000000002,"5min":2.2890000000000001,"15min":1.5740000000000001},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.313},"max":{"1min":6.9790000000000001,"5min":13.428000000000001,"15min":44.679000000000002},"last":5.3070000000000004}]},{"osd":6,"last update":"Sat Sep 23 20:33:09 2023","interfaces":[{"interface":"back","average":{"1min":2.1920000000000002,"5min":2.419,"15min":1.591},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":6.9240000000000004,"5min":17.524999999999999,"15min":17.524999999999999},"last":5.351},{"interface":"front","average":{"1min":1.2270000000000001,"5min":2.2330000000000001,"15min":1.52},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":6.7800000000000002,"5min":19.524000000000001,"15min":19.524000000000001},"last":5.4370000000000003}]},{"osd":7,"last update":"Sat Sep 23 20:33:09 2023","interfaces":[{"interface":"back","average":{"1min":1.24,"5min":2.6720000000000002,"15min":1.6930000000000001},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.34999999999999998},"max":{"1min":6.8079999999999998,"5min":19.571999999999999,"15min":19.571999999999999},"last":5.4749999999999996},{"interface":"front","average":{"1min":1.508,"5min":1.9390000000000001,"15min":1.7210000000000001},"min":{"1min":0.42599999999999999,"5min":0.371,"15min":0.371},"max":{"1min":5.2690000000000001,"5min":11.856999999999999,"15min":45.387999999999998},"last":0.67800000000000005}]}]},{"osd":4,"up_from":30,"seq":128849019163,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":850116,"kb_used_data":614908,"kb_used_omap":0,"kb_used_meta":235200,"kb_avail":92895036,"statfs":{"total":95995035648,"available":95124516864,"internally_reserved":0,"allocated":629665792,"data_stored":628616153,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":240844800},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:33:36 2023","interfaces":[{"interface":"back","average":{"1min":1.7869999999999999,"5min":1.4059999999999999,"15min":1.3009999999999999},"min":{"1min":0.255,"5min":0.24199999999999999,"15min":0.222},"max":{"1min":4.3019999999999996,"5min":8.984,"15min":20.411999999999999},"last":2.4209999999999998},{"interface":"front","average":{"1min":2.4580000000000002,"5min":1.724,"15min":1.452},"min":{"1min":0.34000000000000002,"5min":0.25800000000000001,"15min":0.253},"max":{"1min":10.675000000000001,"5min":11.868,"15min":20.454999999999998},"last":2.4340000000000002}]},{"osd":1,"last update":"Sat Sep 23 20:32:52 2023","interfaces":[{"interface":"back","average":{"1min":1.1319999999999999,"5min":1.7170000000000001,"15min":1.911},"min":{"1min":0.29099999999999998,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":5.3940000000000001,"5min":8.6799999999999997,"15min":8.6799999999999997},"last":4.0860000000000003},{"interface":"front","average":{"1min":0.94299999999999995,"5min":1.6910000000000001,"15min":1.907},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":2.4870000000000001,"5min":8.7219999999999995,"15min":8.7219999999999995},"last":4.0679999999999996}]},{"osd":2,"last update":"Sat Sep 23 20:32:41 2023","interfaces":[{"interface":"back","average":{"1min":1.258,"5min":1.994,"15min":1.4710000000000001},"min":{"1min":0.36599999999999999,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":4.1500000000000004,"5min":32.363,"15min":32.363},"last":1.3180000000000001},{"interface":"front","average":{"1min":1.093,"5min":2.2170000000000001,"15min":1.635},"min":{"1min":0.375,"5min":0.27600000000000002,"15min":0.26800000000000002},"max":{"1min":3.782,"5min":34.042999999999999,"15min":34.042999999999999},"last":3.093}]},{"osd":3,"last update":"Sat Sep 23 20:33:05 2023","interfaces":[{"interface":"back","average":{"1min":2.004,"5min":2.4199999999999999,"15min":1.663},"min":{"1min":0.34799999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":6.5090000000000003,"5min":32.332999999999998,"15min":32.332999999999998},"last":2.7080000000000002},{"interface":"front","average":{"1min":1.5629999999999999,"5min":2.4119999999999999,"15min":1.673},"min":{"1min":0.36299999999999999,"5min":0.318,"15min":0.30499999999999999},"max":{"1min":4.968,"5min":32.399999999999999,"15min":32.399999999999999},"last":2.5369999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:33:29 2023","interfaces":[{"interface":"back","average":{"1min":0.96499999999999997,"5min":2.024,"15min":1.5149999999999999},"min":{"1min":0.29999999999999999,"5min":0.25800000000000001,"15min":0.247},"max":{"1min":5.4340000000000002,"5min":82.866,"15min":82.866},"last":0.32500000000000001},{"interface":"front","average":{"1min":1.236,"5min":1.911,"15min":1.415},"min":{"1min":0.317,"5min":0.30299999999999999,"15min":0.27800000000000002},"max":{"1min":6.8520000000000003,"5min":82.546000000000006,"15min":82.546000000000006},"last":1.3320000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:32:41 2023","interfaces":[{"interface":"back","average":{"1min":0.68500000000000005,"5min":1.139,"15min":1.242},"min":{"1min":0.33300000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":2.9820000000000002,"5min":6.3620000000000001,"15min":41.371000000000002},"last":1.3759999999999999},{"interface":"front","average":{"1min":0.52600000000000002,"5min":1.175,"15min":1.1839999999999999},"min":{"1min":0.35099999999999998,"5min":0.318,"15min":0.27200000000000002},"max":{"1min":0.96099999999999997,"5min":11.199,"15min":41.140999999999998},"last":0.46800000000000003}]},{"osd":7,"last update":"Sat Sep 23 20:33:05 2023","interfaces":[{"interface":"back","average":{"1min":1.024,"5min":1.4650000000000001,"15min":1.325},"min":{"1min":0.371,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":4.4349999999999996,"5min":15.667999999999999,"15min":16.427},"last":1.359},{"interface":"front","average":{"1min":0.876,"5min":1.347,"15min":1.2549999999999999},"min":{"1min":0.38300000000000001,"5min":0.307,"15min":0.307},"max":{"1min":5.4889999999999999,"5min":15.58,"15min":19.018999999999998},"last":1.4610000000000001}]}]},{"osd":5,"up_from":39,"seq":167503724824,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1560064,"kb_used_data":1315192,"kb_used_omap":0,"kb_used_meta":244864,"kb_avail":92185088,"statfs":{"total":95995035648,"available":94397530112,"internally_reserved":0,"allocated":1346756608,"data_stored":1345707695,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":250740736},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:33:36 2023","interfaces":[{"interface":"back","average":{"1min":1.7649999999999999,"5min":1.516,"15min":1.321},"min":{"1min":0.20699999999999999,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":5.7030000000000003,"5min":8.1609999999999996,"15min":32.475000000000001},"last":4.9390000000000001},{"interface":"front","average":{"1min":1.6599999999999999,"5min":1.4359999999999999,"15min":1.3100000000000001},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.20599999999999999},"max":{"1min":5.7220000000000004,"5min":8.1859999999999999,"15min":32.457000000000001},"last":4.9610000000000003}]},{"osd":1,"last update":"Sat Sep 23 20:32:51 2023","interfaces":[{"interface":"back","average":{"1min":1.1459999999999999,"5min":1.702,"15min":1.9370000000000001},"min":{"1min":0.32900000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":5.2709999999999999,"5min":8.5839999999999996,"15min":8.5839999999999996},"last":5.1699999999999999},{"interface":"front","average":{"1min":0.878,"5min":1.3080000000000001,"15min":1.476},"min":{"1min":0.30399999999999999,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":4.577,"5min":6.7969999999999997,"15min":6.7969999999999997},"last":4.9779999999999998}]},{"osd":2,"last update":"Sat Sep 23 20:33:36 2023","interfaces":[{"interface":"back","average":{"1min":1.169,"5min":1.718,"15min":1.506},"min":{"1min":0.28199999999999997,"5min":0.27400000000000002,"15min":0.26700000000000002},"max":{"1min":3.6880000000000002,"5min":18.472999999999999,"15min":34.514000000000003},"last":8.9120000000000008},{"interface":"front","average":{"1min":2.3479999999999999,"5min":1.887,"15min":1.5369999999999999},"min":{"1min":0.30399999999999999,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":13.654,"5min":13.654,"15min":32.515000000000001},"last":6.226}]},{"osd":3,"last update":"Sat Sep 23 20:33:08 2023","interfaces":[{"interface":"back","average":{"1min":1.6419999999999999,"5min":2.0870000000000002,"15min":1.47},"min":{"1min":0.29599999999999999,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":6.0099999999999998,"5min":12.51,"15min":12.846},"last":1.4379999999999999},{"interface":"front","average":{"1min":1.4670000000000001,"5min":1.625,"15min":1.325},"min":{"1min":0.36799999999999999,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":5.7430000000000003,"5min":8.3729999999999993,"15min":12.005000000000001},"last":1.0620000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:33:30 2023","interfaces":[{"interface":"back","average":{"1min":0.68899999999999995,"5min":1.3859999999999999,"15min":1.288},"min":{"1min":0.29999999999999999,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":3.3010000000000002,"5min":28.015999999999998,"15min":28.015999999999998},"last":0.29199999999999998},{"interface":"front","average":{"1min":0.65500000000000003,"5min":1.036,"15min":1.052},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29099999999999998},"max":{"1min":2.512,"5min":12.300000000000001,"15min":12.300000000000001},"last":0.33900000000000002}]},{"osd":6,"last update":"Sat Sep 23 20:33:43 2023","interfaces":[{"interface":"back","average":{"1min":0.69099999999999995,"5min":1.073,"15min":1.1930000000000001},"min":{"1min":0.35999999999999999,"5min":0.30599999999999999,"15min":0.30499999999999999},"max":{"1min":3.3969999999999998,"5min":7.7519999999999998,"15min":12.890000000000001},"last":0.39000000000000001},{"interface":"front","average":{"1min":0.66100000000000003,"5min":1.115,"15min":1.1599999999999999},"min":{"1min":0.40699999999999997,"5min":0.371,"15min":0.34499999999999997},"max":{"1min":2.5409999999999999,"5min":16.265999999999998,"15min":16.265999999999998},"last":0.42799999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:33:08 2023","interfaces":[{"interface":"back","average":{"1min":0.621,"5min":1.026,"15min":1.2170000000000001},"min":{"1min":0.36899999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":2.6520000000000001,"5min":7.343,"15min":36.621000000000002},"last":0.39900000000000002},{"interface":"front","average":{"1min":0.67100000000000004,"5min":1.0640000000000001,"15min":1.107},"min":{"1min":0.35299999999999998,"5min":0.33500000000000002,"15min":0.315},"max":{"1min":2.415,"5min":13.122,"15min":37.331000000000003},"last":0.379}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4722,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1360,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8206,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7156,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3154,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2685865984,"data_stored":2685861912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2370637824,"data_stored":2370633752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2044592128,"data_stored":2044592128,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3290693632,"data_stored":3290693632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":626982912,"data_stored":626982912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1339490304,"data_stored":1339490304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2171277312,"data_stored":2171273240,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1930362880,"data_stored":1930362880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:33:48.047 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd pool set .mgr pgp_num 31 2023-09-23T20:33:48.068 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:48.289 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:48 smithi018 ceph-mon[179647]: 1.1a scrub starts 2023-09-23T20:33:48.290 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:48 smithi018 ceph-mon[179647]: from='client.55495 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:33:48.290 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:48 smithi018 ceph-mon[184144]: 1.1a scrub starts 2023-09-23T20:33:48.290 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:48 smithi018 ceph-mon[184144]: from='client.55495 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:33:48.411 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:33:48.422 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:48.422 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:33:48.523 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:33:48.559 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:48 smithi149 ceph-mon[170394]: 1.1a scrub starts 2023-09-23T20:33:48.560 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:48 smithi149 ceph-mon[170394]: from='client.55495 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:33:48.563 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:48.612 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:33:48.618 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:48.618 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:33:48.719 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:33:48.760 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:48.800 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:33:48.802 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:48.803 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:33:48.841 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:33:48.903 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:33:49.095 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:49.394 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:49 smithi018 ceph-mon[179647]: pgmap v781: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 711 op/s; 5117/244200 objects degraded (2.095%); 1274/244200 objects misplaced (0.522%); 4.2 MiB/s, 67 objects/s recovering 2023-09-23T20:33:49.394 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:49 smithi018 ceph-mon[184144]: pgmap v781: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 711 op/s; 5117/244200 objects degraded (2.095%); 1274/244200 objects misplaced (0.522%); 4.2 MiB/s, 67 objects/s recovering 2023-09-23T20:33:49.394 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:33:49.401 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:49.402 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:33:49.456 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:33:49.502 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:33:49.514 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2023-09-23T20:33:49.515 INFO:teuthology.orchestra.run.smithi018.stdout: "ops": [ 2023-09-23T20:33:49.515 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:33:49.515 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_repop(client.35700.0:110184 7.7 e334/303 7:e03505ed:::benchmark_data_smithi018_308601_object15632:head v 334'6968, mlcod=334'6966)", 2023-09-23T20:33:49.515 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:33:49.485123+0000", 2023-09-23T20:33:49.515 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.021196144, 2023-09-23T20:33:49.515 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.021226458, 2023-09-23T20:33:49.515 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:33:49.515 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "started", 2023-09-23T20:33:49.515 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:33:49.515 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:33:49.515 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:33:49.516 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:33:49.485123+0000", 2023-09-23T20:33:49.516 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:33:49.516 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:33:49.516 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:33:49.516 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:33:49.516 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:33:49.485123+0000", 2023-09-23T20:33:49.516 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:33:49.516 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:33:49.516 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:33:49.516 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:33:49.516 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:33:49.485123+0000", 2023-09-23T20:33:49.516 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:33:49.517 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:33:49.517 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:33:49.517 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:33:49.517 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:33:49.485128+0000", 2023-09-23T20:33:49.517 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.2429999999999996e-06 2023-09-23T20:33:49.517 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:33:49.517 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:33:49.517 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:33:49.517 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:33:49.485136+0000", 2023-09-23T20:33:49.517 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.909e-06 2023-09-23T20:33:49.517 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:33:49.518 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:33:49.518 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:33:49.518 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:33:49.485141+0000", 2023-09-23T20:33:49.518 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.7759999999999997e-06 2023-09-23T20:33:49.518 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:33:49.518 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:33:49.518 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:33:49.518 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:33:49.503304+0000", 2023-09-23T20:33:49.518 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.018163136999999999 2023-09-23T20:33:49.518 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:33:49.518 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:33:49.519 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:33:49.519 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:33:49.503331+0000", 2023-09-23T20:33:49.519 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.7698000000000001e-05 2023-09-23T20:33:49.519 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:33:49.519 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:33:49.519 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:33:49.519 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:33:49.519 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:33:49.519 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_repop_reply(client.35700.0:110170 7.f e334/303)", 2023-09-23T20:33:49.519 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:33:49.487888+0000", 2023-09-23T20:33:49.519 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.018431486, 2023-09-23T20:33:49.519 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.018532553, 2023-09-23T20:33:49.520 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:33:49.520 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "queued for pg", 2023-09-23T20:33:49.520 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:33:49.520 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:33:49.520 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:33:49.520 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:33:49.487888+0000", 2023-09-23T20:33:49.520 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:33:49.520 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:33:49.520 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:33:49.520 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:33:49.520 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:33:49.487888+0000", 2023-09-23T20:33:49.520 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:33:49.521 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:33:49.521 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:33:49.521 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:33:49.521 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:33:49.487886+0000", 2023-09-23T20:33:49.521 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:33:49.521 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:33:49.521 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:33:49.521 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:33:49.521 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:33:49.487893+0000", 2023-09-23T20:33:49.521 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.2940000000000002e-06 2023-09-23T20:33:49.521 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:33:49.521 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:33:49.522 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:33:49.522 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:33:49.487907+0000", 2023-09-23T20:33:49.522 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.4819e-05 2023-09-23T20:33:49.522 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:33:49.522 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:33:49.522 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:33:49.522 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:33:49.487917+0000", 2023-09-23T20:33:49.522 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.5659999999999996e-06 2023-09-23T20:33:49.522 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:33:49.522 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:33:49.522 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:33:49.523 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:33:49.523 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:33:49.523 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:110192 7.6 7:61373bdd:::benchmark_data_smithi018_308601_object15640:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e334)", 2023-09-23T20:33:49.523 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:33:49.497178+0000", 2023-09-23T20:33:49.523 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.0091409119999999993, 2023-09-23T20:33:49.523 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0092968340000000003, 2023-09-23T20:33:49.523 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:33:49.523 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:33:49.523 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:33:49.523 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:33:49.523 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:33:49.523 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 110192 2023-09-23T20:33:49.524 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:33:49.524 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:33:49.524 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:33:49.524 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:33:49.524 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:33:49.497178+0000", 2023-09-23T20:33:49.524 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:33:49.524 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:33:49.524 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:33:49.524 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:33:49.524 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:33:49.497178+0000", 2023-09-23T20:33:49.524 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:33:49.524 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:33:49.525 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:33:49.525 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:33:49.525 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:33:49.497178+0000", 2023-09-23T20:33:49.525 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:33:49.525 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:33:49.525 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:33:49.525 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:33:49.525 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:33:49.497181+0000", 2023-09-23T20:33:49.525 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.6449999999999998e-06 2023-09-23T20:33:49.525 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:33:49.525 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:33:49.525 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:33:49.526 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:33:49.497190+0000", 2023-09-23T20:33:49.526 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.3459999999999994e-06 2023-09-23T20:33:49.526 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:33:49.526 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:33:49.526 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:33:49.526 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:33:49.497198+0000", 2023-09-23T20:33:49.526 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.9279999999999993e-06 2023-09-23T20:33:49.526 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:33:49.526 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:33:49.526 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:33:49.526 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:33:49.501063+0000", 2023-09-23T20:33:49.527 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0038650540000000001 2023-09-23T20:33:49.527 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:33:49.527 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:33:49.527 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:33:49.527 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:33:49.501181+0000", 2023-09-23T20:33:49.527 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000118528 2023-09-23T20:33:49.527 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:33:49.527 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:33:49.527 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 3,7", 2023-09-23T20:33:49.527 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:33:49.501294+0000", 2023-09-23T20:33:49.527 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000112939 2023-09-23T20:33:49.527 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:33:49.528 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:33:49.528 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:33:49.528 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:33:49.505358+0000", 2023-09-23T20:33:49.528 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0040636049999999996 2023-09-23T20:33:49.528 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:33:49.528 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:33:49.528 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:33:49.528 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:33:49.528 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:33:49.528 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:110198 7.f 7.f22e36af (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e334)", 2023-09-23T20:33:49.528 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:33:49.505423+0000", 2023-09-23T20:33:49.528 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.00089641099999999995, 2023-09-23T20:33:49.529 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0011362830000000001, 2023-09-23T20:33:49.529 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:33:49.529 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "queued for pg", 2023-09-23T20:33:49.529 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:33:49.529 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:33:49.529 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:33:49.529 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 110198 2023-09-23T20:33:49.529 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:33:49.529 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:33:49.529 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:33:49.529 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:33:49.529 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:33:49.505423+0000", 2023-09-23T20:33:49.530 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:33:49.530 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:33:49.530 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:33:49.530 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:33:49.530 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:33:49.505423+0000", 2023-09-23T20:33:49.530 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:33:49.530 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:33:49.530 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:33:49.530 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:33:49.530 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:33:49.505422+0000", 2023-09-23T20:33:49.530 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.999999 2023-09-23T20:33:49.531 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:33:49.531 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:33:49.531 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:33:49.531 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:33:49.505429+0000", 2023-09-23T20:33:49.531 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.058e-06 2023-09-23T20:33:49.531 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:33:49.531 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:33:49.531 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:33:49.531 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:33:49.505440+0000", 2023-09-23T20:33:49.531 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1415e-05 2023-09-23T20:33:49.531 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:33:49.531 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:33:49.532 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:33:49.532 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:33:49.505447+0000", 2023-09-23T20:33:49.532 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.4239999999999998e-06 2023-09-23T20:33:49.532 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:33:49.532 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:33:49.532 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:33:49.532 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:33:49.532 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:33:49.532 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_repop(client.35700.0:110197 7.7 e334/303)", 2023-09-23T20:33:49.532 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:33:49.505549+0000", 2023-09-23T20:33:49.532 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.00076994699999999999, 2023-09-23T20:33:49.532 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.001065498, 2023-09-23T20:33:49.533 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:33:49.533 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "queued for pg", 2023-09-23T20:33:49.533 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:33:49.533 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:33:49.533 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:33:49.533 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:33:49.505549+0000", 2023-09-23T20:33:49.533 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:33:49.533 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:33:49.533 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:33:49.533 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:33:49.533 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:33:49.505549+0000", 2023-09-23T20:33:49.534 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:33:49.534 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:33:49.534 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:33:49.534 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:33:49.534 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:33:49.505549+0000", 2023-09-23T20:33:49.534 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:33:49.534 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:33:49.534 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:33:49.534 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:33:49.534 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:33:49.505554+0000", 2023-09-23T20:33:49.534 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.8239999999999999e-06 2023-09-23T20:33:49.534 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:33:49.535 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:33:49.535 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:33:49.535 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:33:49.505560+0000", 2023-09-23T20:33:49.535 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.6939999999999998e-06 2023-09-23T20:33:49.535 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:33:49.535 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:33:49.535 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:33:49.535 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:33:49.505564+0000", 2023-09-23T20:33:49.535 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.0260000000000001e-06 2023-09-23T20:33:49.535 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:33:49.535 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:33:49.535 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:33:49.536 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:33:49.536 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:33:49.536 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:110170 7.f 7:f7fc92cd:::benchmark_data_smithi018_308601_object15618:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e334)", 2023-09-23T20:33:49.536 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:33:49.459285+0000", 2023-09-23T20:33:49.536 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.047034395999999999, 2023-09-23T20:33:49.536 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.047380028999999997, 2023-09-23T20:33:49.536 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:33:49.536 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:33:49.536 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:33:49.536 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:33:49.536 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:33:49.536 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 110170 2023-09-23T20:33:49.537 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:33:49.537 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:33:49.537 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:33:49.537 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:33:49.537 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:33:49.459285+0000", 2023-09-23T20:33:49.537 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:33:49.537 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:33:49.537 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:33:49.537 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:33:49.537 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:33:49.459285+0000", 2023-09-23T20:33:49.537 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:33:49.537 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:33:49.538 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:33:49.538 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:33:49.538 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:33:49.459283+0000", 2023-09-23T20:33:49.538 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:33:49.538 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:33:49.538 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:33:49.538 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:33:49.538 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:33:49.459294+0000", 2023-09-23T20:33:49.538 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1303e-05 2023-09-23T20:33:49.538 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:33:49.538 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:33:49.539 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:33:49.539 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:33:49.459318+0000", 2023-09-23T20:33:49.539 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.3975e-05 2023-09-23T20:33:49.539 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:33:49.539 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:33:49.539 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:33:49.539 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:33:49.459330+0000", 2023-09-23T20:33:49.539 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2204999999999999e-05 2023-09-23T20:33:49.539 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:33:49.539 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:33:49.539 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:33:49.539 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:33:49.468076+0000", 2023-09-23T20:33:49.540 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0087455929999999994 2023-09-23T20:33:49.540 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:33:49.540 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:33:49.540 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:33:49.540 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:33:49.468205+0000", 2023-09-23T20:33:49.540 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00012949599999999999 2023-09-23T20:33:49.540 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:33:49.540 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:33:49.540 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 3,7", 2023-09-23T20:33:49.540 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:33:49.468327+0000", 2023-09-23T20:33:49.540 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00012155300000000001 2023-09-23T20:33:49.540 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:33:49.541 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:33:49.541 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:33:49.541 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:33:49.473015+0000", 2023-09-23T20:33:49.541 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0046878589999999999 2023-09-23T20:33:49.541 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:33:49.541 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:33:49.541 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:33:49.541 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:33:49.477409+0000", 2023-09-23T20:33:49.541 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0043937910000000002 2023-09-23T20:33:49.541 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:33:49.541 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:33:49.541 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:33:49.542 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:33:49.506585+0000", 2023-09-23T20:33:49.542 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.029176634 2023-09-23T20:33:49.542 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:33:49.542 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:33:49.542 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:33:49.542 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:33:49.506631+0000", 2023-09-23T20:33:49.542 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.5352000000000002e-05 2023-09-23T20:33:49.542 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:33:49.542 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:33:49.542 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:33:49.542 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:33:49.506665+0000", 2023-09-23T20:33:49.542 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.4316000000000001e-05 2023-09-23T20:33:49.543 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:33:49.543 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:33:49.543 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:33:49.543 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:33:49.543 INFO:teuthology.orchestra.run.smithi018.stdout: ], 2023-09-23T20:33:49.543 INFO:teuthology.orchestra.run.smithi018.stdout: "num_ops": 6 2023-09-23T20:33:49.543 INFO:teuthology.orchestra.run.smithi018.stdout:} 2023-09-23T20:33:49.630 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:49.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:49 smithi149 ceph-mon[170394]: pgmap v781: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 711 op/s; 5117/244200 objects degraded (2.095%); 1274/244200 objects misplaced (0.522%); 4.2 MiB/s, 67 objects/s recovering 2023-09-23T20:33:50.088 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T20:33:50.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:50 smithi018 ceph-mon[184144]: 1.4 deep-scrub starts 2023-09-23T20:33:50.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:50 smithi018 ceph-mon[179647]: 1.4 deep-scrub starts 2023-09-23T20:33:50.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:50 smithi149 ceph-mon[170394]: 1.4 deep-scrub starts 2023-09-23T20:33:51.358 INFO:teuthology.orchestra.run.smithi018.stderr:noscrub is set 2023-09-23T20:33:51.467 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:33:51.477 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:51.478 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:33:51.556 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:33:51.579 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:33:51.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:51 smithi018 ceph-mon[179647]: pgmap v782: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 4.9 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 670 op/s; 5117/242952 objects degraded (2.106%); 1274/242952 objects misplaced (0.524%); 3.7 MiB/s, 59 objects/s recovering 2023-09-23T20:33:51.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:51 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3132398926' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:33:51.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:51 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 5117/242952 objects degraded (2.106%), 3 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T20:33:51.645 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:51 smithi018 ceph-mon[184144]: pgmap v782: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 4.9 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 670 op/s; 5117/242952 objects degraded (2.106%); 1274/242952 objects misplaced (0.524%); 3.7 MiB/s, 59 objects/s recovering 2023-09-23T20:33:51.645 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:51 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3132398926' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:33:51.645 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:51 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 5117/242952 objects degraded (2.106%), 3 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T20:33:51.687 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:51.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:51 smithi149 ceph-mon[170394]: pgmap v782: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 4.9 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 670 op/s; 5117/242952 objects degraded (2.106%); 1274/242952 objects misplaced (0.524%); 3.7 MiB/s, 59 objects/s recovering 2023-09-23T20:33:51.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:51 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3132398926' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:33:51.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:51 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 5117/242952 objects degraded (2.106%), 3 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T20:33:52.322 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:52 smithi018 ceph-mon[179647]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:33:52.322 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:52 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3132398926' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:33:52.322 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:52 smithi018 ceph-mon[179647]: osdmap e335: 8 total, 8 up, 8 in 2023-09-23T20:33:52.322 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:52 smithi018 ceph-mon[179647]: failed to encode map e335 with expected crc 2023-09-23T20:33:52.323 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:52 smithi018 ceph-mon[179647]: failed to encode map e335 with expected crc 2023-09-23T20:33:52.323 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:52 smithi018 ceph-mon[179647]: failed to encode map e335 with expected crc 2023-09-23T20:33:52.644 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:52 smithi018 ceph-mon[184144]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:33:52.644 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:52 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3132398926' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:33:52.644 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:52 smithi018 ceph-mon[184144]: osdmap e335: 8 total, 8 up, 8 in 2023-09-23T20:33:52.644 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:52 smithi018 ceph-mon[184144]: failed to encode map e335 with expected crc 2023-09-23T20:33:52.644 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:52 smithi018 ceph-mon[184144]: failed to encode map e335 with expected crc 2023-09-23T20:33:52.644 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:52 smithi018 ceph-mon[184144]: failed to encode map e335 with expected crc 2023-09-23T20:33:52.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:52 smithi149 ceph-mon[170394]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:33:52.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:52 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3132398926' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:33:52.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:52 smithi149 ceph-mon[170394]: osdmap e335: 8 total, 8 up, 8 in 2023-09-23T20:33:52.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:52 smithi149 ceph-mon[170394]: failed to encode map e335 with expected crc 2023-09-23T20:33:52.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:52 smithi149 ceph-mon[170394]: failed to encode map e335 with expected crc 2023-09-23T20:33:52.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:52 smithi149 ceph-mon[170394]: failed to encode map e335 with expected crc 2023-09-23T20:33:53.034 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:33:53.044 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:53.045 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:33:53.146 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:33:53.190 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:53.243 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:33:53.248 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:53.249 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:33:53.332 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:53 smithi018 ceph-mon[179647]: pgmap v784: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 676 op/s; 5117/236727 objects degraded (2.162%); 1274/236727 objects misplaced (0.538%); 2.2 MiB/s, 35 objects/s recovering 2023-09-23T20:33:53.350 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:33:53.385 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:53.434 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:33:53.437 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:53 smithi149 ceph-mon[170394]: pgmap v784: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 676 op/s; 5117/236727 objects degraded (2.162%); 1274/236727 objects misplaced (0.538%); 2.2 MiB/s, 35 objects/s recovering 2023-09-23T20:33:53.437 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:53.438 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:33:53.489 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:33:53.538 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:33:53.574 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:53.589 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:53 smithi018 ceph-mon[184144]: pgmap v784: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 676 op/s; 5117/236727 objects degraded (2.162%); 1274/236727 objects misplaced (0.538%); 2.2 MiB/s, 35 objects/s recovering 2023-09-23T20:33:53.616 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:33:53.619 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:53.619 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:33:53.720 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:33:53.783 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:54.409 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:33:54.412 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:54.413 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:33:54.514 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:33:54.542 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:54.582 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:33:54.586 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:54.586 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:33:54.687 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:33:54.724 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:54.771 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:33:54.776 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:54.776 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:33:54.877 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:33:54.978 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:55.036 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:33:55.045 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:55.046 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:33:55.147 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:33:55.230 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:33:54 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:33:54] "GET /metrics HTTP/1.1" 200 36556 "" "Prometheus/2.43.0" 2023-09-23T20:33:55.236 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:55.324 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:33:55.346 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:55.347 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:33:55.408 INFO:teuthology.orchestra.run.smithi018.stderr:set pool 1 pgp_num to 31 2023-09-23T20:33:55.447 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:33:55.477 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:55.482 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:33:55.526 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:33:55.529 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:55.529 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:33:55.535 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T20:33:55.535 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:33:55.573 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:33:55.618 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:55 smithi018 ceph-mon[179647]: pgmap v785: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 669 op/s; 5117/234102 objects degraded (2.186%); 926/234102 objects misplaced (0.396%); 4.1 MiB/s, 65 objects/s recovering 2023-09-23T20:33:55.618 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:55 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1265657788' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "31"}]: dispatch 2023-09-23T20:33:55.619 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:55 smithi018 ceph-mon[184144]: pgmap v785: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 669 op/s; 5117/234102 objects degraded (2.186%); 926/234102 objects misplaced (0.396%); 4.1 MiB/s, 65 objects/s recovering 2023-09-23T20:33:55.619 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:55 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1265657788' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "31"}]: dispatch 2023-09-23T20:33:55.630 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:33:55.631 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:33:55.676 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:33:55.712 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:33:55.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:55 smithi149 ceph-mon[170394]: pgmap v785: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 669 op/s; 5117/234102 objects degraded (2.186%); 926/234102 objects misplaced (0.396%); 4.1 MiB/s, 65 objects/s recovering 2023-09-23T20:33:55.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:55 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1265657788' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "31"}]: dispatch 2023-09-23T20:33:55.728 INFO:teuthology.orchestra.run.smithi018.stdout:ERROR: (22) Invalid argument 2023-09-23T20:33:55.728 INFO:teuthology.orchestra.run.smithi018.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:33:55.765 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:33:55.781 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:55.816 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:33:55.880 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:33:55.880 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:33:55.880 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:33:55.880 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:33:55.880 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:33:55.880 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:33:55.880 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:33:55.880 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:33:56.082 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:33:56.095 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:56.096 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:33:56.196 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:33:56.327 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:56.579 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:33:56.593 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:56.594 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:33:56.595 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:56 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1654561226' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:33:56.595 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:56 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1265657788' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "31"}]': finished 2023-09-23T20:33:56.595 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:56 smithi018 ceph-mon[184144]: osdmap e336: 8 total, 8 up, 8 in 2023-09-23T20:33:56.595 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:56 smithi018 ceph-mon[184144]: failed to encode map e336 with expected crc 2023-09-23T20:33:56.595 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:56 smithi018 ceph-mon[184144]: failed to encode map e336 with expected crc 2023-09-23T20:33:56.595 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:56 smithi018 ceph-mon[184144]: failed to encode map e336 with expected crc 2023-09-23T20:33:56.596 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:56 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 5117/234102 objects degraded (2.186%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T20:33:56.596 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:56 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1654561226' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:33:56.596 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:56 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1265657788' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "31"}]': finished 2023-09-23T20:33:56.596 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:56 smithi018 ceph-mon[179647]: osdmap e336: 8 total, 8 up, 8 in 2023-09-23T20:33:56.596 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:56 smithi018 ceph-mon[179647]: failed to encode map e336 with expected crc 2023-09-23T20:33:56.596 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:56 smithi018 ceph-mon[179647]: failed to encode map e336 with expected crc 2023-09-23T20:33:56.596 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:56 smithi018 ceph-mon[179647]: failed to encode map e336 with expected crc 2023-09-23T20:33:56.596 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:56 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 5117/234102 objects degraded (2.186%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T20:33:56.602 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T20:33:56.695 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:33:56.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:56 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1654561226' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:33:56.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:56 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1265657788' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "31"}]': finished 2023-09-23T20:33:56.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:56 smithi149 ceph-mon[170394]: osdmap e336: 8 total, 8 up, 8 in 2023-09-23T20:33:56.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:56 smithi149 ceph-mon[170394]: failed to encode map e336 with expected crc 2023-09-23T20:33:56.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:56 smithi149 ceph-mon[170394]: failed to encode map e336 with expected crc 2023-09-23T20:33:56.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:56 smithi149 ceph-mon[170394]: failed to encode map e336 with expected crc 2023-09-23T20:33:56.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:56 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 5117/234102 objects degraded (2.186%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T20:33:56.728 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:56.770 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:33:56.775 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:56.776 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:33:56.877 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:33:56.902 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:33:56.908 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:56.954 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:33:56.958 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:56.959 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:33:57.059 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:33:57.092 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:57.137 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:33:57.142 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:57.142 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:33:57.243 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:33:57.320 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:57.410 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:33:57.420 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:57 smithi149 ceph-mon[170394]: pgmap v787: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 767 op/s; 5117/225756 objects degraded (2.267%); 926/225756 objects misplaced (0.410%); 2.7 MiB/s, 43 objects/s recovering 2023-09-23T20:33:57.427 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:57.428 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:33:57.529 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:33:57.569 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:57.607 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:33:57.610 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:57.611 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:33:57.644 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:57 smithi018 ceph-mon[184144]: pgmap v787: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 767 op/s; 5117/225756 objects degraded (2.267%); 926/225756 objects misplaced (0.410%); 2.7 MiB/s, 43 objects/s recovering 2023-09-23T20:33:57.645 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:57 smithi018 ceph-mon[179647]: pgmap v787: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 767 op/s; 5117/225756 objects degraded (2.267%); 926/225756 objects misplaced (0.410%); 2.7 MiB/s, 43 objects/s recovering 2023-09-23T20:33:57.711 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:33:57.743 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:57.795 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:33:57.799 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:57.799 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:33:57.900 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:33:57.923 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:33:58.029 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:58.240 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:33:58.248 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:58.249 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:33:58.350 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:33:58.415 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:33:58.439 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:33:59.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:33:59 smithi149 ceph-mon[170394]: pgmap v788: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 4.5 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 817 op/s; 5117/223308 objects degraded (2.291%); 539/223308 objects misplaced (0.241%); 5.7 MiB/s, 91 objects/s recovering 2023-09-23T20:33:59.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:33:59 smithi018 ceph-mon[184144]: pgmap v788: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 4.5 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 817 op/s; 5117/223308 objects degraded (2.291%); 539/223308 objects misplaced (0.241%); 5.7 MiB/s, 91 objects/s recovering 2023-09-23T20:33:59.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:33:59 smithi018 ceph-mon[179647]: pgmap v788: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 4.5 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 817 op/s; 5117/223308 objects degraded (2.291%); 539/223308 objects misplaced (0.241%); 5.7 MiB/s, 91 objects/s recovering 2023-09-23T20:33:59.976 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:33:59.987 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:33:59.987 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:34:00.088 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:34:00.122 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:00.175 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:34:00.181 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:00.181 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:34:00.282 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:34:00.362 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:00.429 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:34:00.487 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:34:00.495 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:00 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:34:00.495 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:00 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:34:00.503 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:00.504 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:34:00.605 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:34:00.639 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:00.695 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:34:00.700 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:00.700 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:34:00.751 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:00 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:34:00.751 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:00 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:34:00.801 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:34:00.838 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:00 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:34:00.838 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:00 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:34:00.838 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:00.887 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:34:00.891 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:00.891 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:34:00.993 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:34:01.070 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 5, 7, 3, 2, 6, 4] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T20:34:01.070 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd reweight 0 1 2023-09-23T20:34:01.094 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:01.223 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:34:01.237 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:01.237 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:34:01.338 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:34:01.366 INFO:teuthology.orchestra.run.smithi018.stdout:ERROR: (22) Invalid argument 2023-09-23T20:34:01.366 INFO:teuthology.orchestra.run.smithi018.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:34:01.483 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:01.545 INFO:teuthology.orchestra.run.smithi018.stderr:nodeep-scrub is set 2023-09-23T20:34:01.709 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:01 smithi018 ceph-mon[184144]: pgmap v789: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 575 op/s; 5117/222084 objects degraded (2.304%); 539/222084 objects misplaced (0.243%); 5.4 MiB/s, 86 objects/s recovering 2023-09-23T20:34:01.709 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:01 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2224054556' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:34:01.709 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:01 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 5117/222084 objects degraded (2.304%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T20:34:01.709 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:01 smithi018 ceph-mon[179647]: pgmap v789: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 575 op/s; 5117/222084 objects degraded (2.304%); 539/222084 objects misplaced (0.243%); 5.4 MiB/s, 86 objects/s recovering 2023-09-23T20:34:01.709 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:01 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2224054556' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:34:01.709 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:01 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 5117/222084 objects degraded (2.304%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T20:34:01.968 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:01 smithi149 ceph-mon[170394]: pgmap v789: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 575 op/s; 5117/222084 objects degraded (2.304%); 539/222084 objects misplaced (0.243%); 5.4 MiB/s, 86 objects/s recovering 2023-09-23T20:34:01.968 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:01 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2224054556' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:34:01.968 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:01 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 5117/222084 objects degraded (2.304%), 3 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T20:34:02.026 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:34:02.039 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:02.039 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:34:02.101 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T20:34:02.140 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:34:02.257 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:02.372 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:34:02.380 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:02.381 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:34:02.482 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:34:02.586 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:02.616 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:02 smithi018 ceph-mon[184144]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:34:02.616 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:02 smithi018 ceph-mon[184144]: failed to encode map e337 with expected crc 2023-09-23T20:34:02.616 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:02 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2224054556' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:34:02.617 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:02 smithi018 ceph-mon[184144]: osdmap e337: 8 total, 8 up, 8 in 2023-09-23T20:34:02.617 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:02 smithi018 ceph-mon[184144]: failed to encode map e337 with expected crc 2023-09-23T20:34:02.617 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:02 smithi018 ceph-mon[184144]: pgmap v791: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 780 op/s; 5117/215343 objects degraded (2.376%); 539/215343 objects misplaced (0.250%); 3.0 MiB/s, 48 objects/s recovering 2023-09-23T20:34:02.617 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:02 smithi018 ceph-mon[179647]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:34:02.617 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:02 smithi018 ceph-mon[179647]: failed to encode map e337 with expected crc 2023-09-23T20:34:02.617 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:02 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2224054556' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:34:02.617 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:02 smithi018 ceph-mon[179647]: osdmap e337: 8 total, 8 up, 8 in 2023-09-23T20:34:02.617 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:02 smithi018 ceph-mon[179647]: failed to encode map e337 with expected crc 2023-09-23T20:34:02.617 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:02 smithi018 ceph-mon[179647]: pgmap v791: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 780 op/s; 5117/215343 objects degraded (2.376%); 539/215343 objects misplaced (0.250%); 3.0 MiB/s, 48 objects/s recovering 2023-09-23T20:34:02.667 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:34:02.675 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:02.675 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:34:02.708 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:34:02.777 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:34:02.925 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:02.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:02 smithi149 ceph-mon[170394]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:34:02.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:02 smithi149 ceph-mon[170394]: failed to encode map e337 with expected crc 2023-09-23T20:34:02.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:02 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2224054556' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:34:02.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:02 smithi149 ceph-mon[170394]: osdmap e337: 8 total, 8 up, 8 in 2023-09-23T20:34:02.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:02 smithi149 ceph-mon[170394]: failed to encode map e337 with expected crc 2023-09-23T20:34:02.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:02 smithi149 ceph-mon[170394]: pgmap v791: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 780 op/s; 5117/215343 objects degraded (2.376%); 539/215343 objects misplaced (0.250%); 3.0 MiB/s, 48 objects/s recovering 2023-09-23T20:34:03.023 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:34:03.051 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:34:03.062 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:34:03.066 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:03.067 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:34:03.100 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:34:03.162 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:34:03.167 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:34:03.202 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:03.216 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:34:03.246 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:34:03.250 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:03.251 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:34:03.253 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:34:03.290 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:34:03.302 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:34:03.329 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:34:03.351 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:34:03.360 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:34:03.361 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:34:03.361 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:34:03.361 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:34:03.362 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:34:03.362 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:34:03.362 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:34:03.362 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:34:03.470 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:03.527 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:03 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/359355415' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:34:03.527 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:03 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:34:03.778 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:03 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/359355415' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:34:03.778 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:03 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:34:03.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:03 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/359355415' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:34:03.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:03 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:34:04.075 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:34:04.083 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:04.084 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:34:04.185 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:34:04.221 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:04.264 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:34:04.297 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:04.298 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:34:04.399 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:34:04.426 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T20:34:04.508 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:04.616 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:34:04.627 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:04.628 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:34:04.644 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:04 smithi018 ceph-mon[184144]: pgmap v792: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 758 op/s; 5117/212733 objects degraded (2.405%); 94/212733 objects misplaced (0.044%); 6.2 MiB/s, 98 objects/s recovering 2023-09-23T20:34:04.647 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:04 smithi018 ceph-mon[179647]: pgmap v792: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 758 op/s; 5117/212733 objects degraded (2.405%); 94/212733 objects misplaced (0.044%); 6.2 MiB/s, 98 objects/s recovering 2023-09-23T20:34:04.729 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:34:04.841 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:04.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:04 smithi149 ceph-mon[170394]: pgmap v792: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 4.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 758 op/s; 5117/212733 objects degraded (2.405%); 94/212733 objects misplaced (0.044%); 6.2 MiB/s, 98 objects/s recovering 2023-09-23T20:34:04.977 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:34:04.990 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:04.990 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:34:05.092 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:34:05.141 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:34:05 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:34:04] "GET /metrics HTTP/1.1" 200 36551 "" "Prometheus/2.43.0" 2023-09-23T20:34:05.178 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:05.251 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:34:05.258 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:05.259 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:34:05.359 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:34:05.396 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:05.444 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:34:05.448 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:05.448 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:34:05.520 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:34:05.521 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2023-09-23T20:34:05.521 INFO:teuthology.orchestra.run.smithi018.stdout: "ops": [ 2023-09-23T20:34:05.522 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.522 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:120828 7.8 7.966f4218 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e338)", 2023-09-23T20:34:05.522 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:34:04.911638+0000", 2023-09-23T20:34:05.522 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.60836679900000001, 2023-09-23T20:34:05.522 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.60838896499999995, 2023-09-23T20:34:05.522 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:34:05.522 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "delayed", 2023-09-23T20:34:05.522 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:34:05.522 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:34:05.522 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:34:05.523 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 120828 2023-09-23T20:34:05.523 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.523 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:34:05.523 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.523 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:34:05.523 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.911638+0000", 2023-09-23T20:34:05.523 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:34:05.523 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.523 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.523 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:34:05.524 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.911638+0000", 2023-09-23T20:34:05.524 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:34:05.524 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.524 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.524 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:34:05.524 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.911636+0000", 2023-09-23T20:34:05.524 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:34:05.524 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.524 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.524 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:34:05.524 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.911651+0000", 2023-09-23T20:34:05.525 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.5067e-05 2023-09-23T20:34:05.525 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.525 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.525 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:34:05.525 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.911668+0000", 2023-09-23T20:34:05.525 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.6957999999999999e-05 2023-09-23T20:34:05.525 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.525 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.525 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:34:05.525 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.911677+0000", 2023-09-23T20:34:05.525 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.0909999999999999e-06 2023-09-23T20:34:05.526 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.526 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.526 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:34:05.526 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.911744+0000", 2023-09-23T20:34:05.526 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.6674000000000006e-05 2023-09-23T20:34:05.526 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.526 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.526 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:34:05.527 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.911757+0000", 2023-09-23T20:34:05.527 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2653999999999999e-05 2023-09-23T20:34:05.527 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:34:05.528 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:34:05.528 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:34:05.528 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.528 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.529 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:120832 7.8 7.5c86ef78 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e338)", 2023-09-23T20:34:05.529 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:34:04.932921+0000", 2023-09-23T20:34:05.529 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.58708326099999997, 2023-09-23T20:34:05.529 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.58715363799999998, 2023-09-23T20:34:05.530 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:34:05.530 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "delayed", 2023-09-23T20:34:05.530 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:34:05.530 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:34:05.531 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:34:05.531 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 120832 2023-09-23T20:34:05.531 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.531 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:34:05.532 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.532 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:34:05.532 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.932921+0000", 2023-09-23T20:34:05.533 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:34:05.533 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.533 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.534 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:34:05.534 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.932921+0000", 2023-09-23T20:34:05.534 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:34:05.535 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.535 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.535 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:34:05.535 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.932920+0000", 2023-09-23T20:34:05.535 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:34:05.535 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.535 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.535 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:34:05.535 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.932926+0000", 2023-09-23T20:34:05.535 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.7409999999999998e-06 2023-09-23T20:34:05.535 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.536 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.536 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:34:05.536 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.932938+0000", 2023-09-23T20:34:05.536 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2281e-05 2023-09-23T20:34:05.536 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.536 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.536 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:34:05.536 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.932946+0000", 2023-09-23T20:34:05.536 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.6850000000000001e-06 2023-09-23T20:34:05.536 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.536 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.536 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:34:05.537 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.936712+0000", 2023-09-23T20:34:05.537 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0037664959999999998 2023-09-23T20:34:05.537 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.537 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.537 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:34:05.537 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.936727+0000", 2023-09-23T20:34:05.537 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.4932999999999999e-05 2023-09-23T20:34:05.537 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:34:05.537 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:34:05.537 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:34:05.537 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.537 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.538 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:120921 7.8 7.78d30808 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e338)", 2023-09-23T20:34:05.538 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:34:05.151346+0000", 2023-09-23T20:34:05.538 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.36865853999999998, 2023-09-23T20:34:05.538 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.368769613, 2023-09-23T20:34:05.538 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:34:05.538 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "delayed", 2023-09-23T20:34:05.538 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:34:05.538 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:34:05.538 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:34:05.538 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 120921 2023-09-23T20:34:05.538 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.539 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:34:05.539 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.539 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:34:05.539 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.151346+0000", 2023-09-23T20:34:05.539 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:34:05.539 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.539 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.539 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:34:05.539 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.151346+0000", 2023-09-23T20:34:05.539 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:34:05.539 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.539 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.540 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:34:05.540 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.151345+0000", 2023-09-23T20:34:05.540 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:34:05.540 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.540 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.540 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:34:05.540 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.151353+0000", 2023-09-23T20:34:05.540 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.5779999999999999e-06 2023-09-23T20:34:05.540 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.540 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.540 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:34:05.540 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.151367+0000", 2023-09-23T20:34:05.541 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3893000000000001e-05 2023-09-23T20:34:05.541 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.541 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.541 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:34:05.541 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.151376+0000", 2023-09-23T20:34:05.541 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.3740000000000004e-06 2023-09-23T20:34:05.541 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.541 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.541 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:34:05.541 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.152848+0000", 2023-09-23T20:34:05.541 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.001471499 2023-09-23T20:34:05.541 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.542 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.542 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:34:05.542 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.152864+0000", 2023-09-23T20:34:05.542 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.5617999999999999e-05 2023-09-23T20:34:05.542 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:34:05.542 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:34:05.542 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:34:05.542 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.542 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.542 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:120978 7.8 7.9ec36298 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e338)", 2023-09-23T20:34:05.542 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:34:05.291932+0000", 2023-09-23T20:34:05.542 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.22807271800000001, 2023-09-23T20:34:05.543 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.22822271599999999, 2023-09-23T20:34:05.543 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:34:05.543 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "delayed", 2023-09-23T20:34:05.543 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:34:05.543 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:34:05.543 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:34:05.543 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 120978 2023-09-23T20:34:05.543 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.543 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:34:05.543 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.543 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:34:05.543 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.291932+0000", 2023-09-23T20:34:05.544 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:34:05.544 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.544 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.544 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:34:05.544 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.291932+0000", 2023-09-23T20:34:05.544 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:34:05.544 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.544 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.544 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:34:05.544 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.291930+0000", 2023-09-23T20:34:05.544 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:34:05.545 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.545 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.545 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:34:05.545 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.291940+0000", 2023-09-23T20:34:05.545 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.5489999999999995e-06 2023-09-23T20:34:05.545 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.545 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.545 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:34:05.545 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.291958+0000", 2023-09-23T20:34:05.545 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.7869000000000001e-05 2023-09-23T20:34:05.545 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.545 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.546 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:34:05.546 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.291968+0000", 2023-09-23T20:34:05.546 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0091e-05 2023-09-23T20:34:05.546 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.546 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.546 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:34:05.546 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.292066+0000", 2023-09-23T20:34:05.546 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.8709999999999995e-05 2023-09-23T20:34:05.546 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.546 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.546 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:34:05.547 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.292081+0000", 2023-09-23T20:34:05.547 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.4207e-05 2023-09-23T20:34:05.547 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:34:05.547 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:34:05.547 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:34:05.547 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.547 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.547 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:120780 7.8 7.873123c8 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e338)", 2023-09-23T20:34:05.547 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:34:04.802248+0000", 2023-09-23T20:34:05.547 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.71775705000000001, 2023-09-23T20:34:05.547 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.71796248500000004, 2023-09-23T20:34:05.548 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:34:05.548 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "delayed", 2023-09-23T20:34:05.548 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:34:05.548 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:34:05.548 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:34:05.548 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 120780 2023-09-23T20:34:05.548 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.548 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:34:05.548 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.548 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:34:05.548 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.802248+0000", 2023-09-23T20:34:05.548 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:34:05.549 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.549 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.549 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:34:05.549 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.802248+0000", 2023-09-23T20:34:05.549 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:34:05.549 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.549 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.549 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:34:05.549 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.802247+0000", 2023-09-23T20:34:05.549 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.999999 2023-09-23T20:34:05.549 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.549 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.550 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:34:05.550 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.802252+0000", 2023-09-23T20:34:05.550 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.541e-06 2023-09-23T20:34:05.550 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.550 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.550 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:34:05.550 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.802264+0000", 2023-09-23T20:34:05.550 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2283000000000001e-05 2023-09-23T20:34:05.550 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.550 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.550 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:34:05.550 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.802273+0000", 2023-09-23T20:34:05.551 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.49e-06 2023-09-23T20:34:05.551 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.551 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.551 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:34:05.551 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.802382+0000", 2023-09-23T20:34:05.551 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00010888 2023-09-23T20:34:05.551 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.551 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.551 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:34:05.551 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.802390+0000", 2023-09-23T20:34:05.551 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.9559999999999997e-06 2023-09-23T20:34:05.551 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:34:05.552 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:34:05.552 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:34:05.552 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.552 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.552 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:120781 7.8 7.e2eddfb8 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e338)", 2023-09-23T20:34:05.552 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:34:04.802289+0000", 2023-09-23T20:34:05.552 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.71771573300000002, 2023-09-23T20:34:05.552 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.71799228900000001, 2023-09-23T20:34:05.552 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:34:05.552 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "delayed", 2023-09-23T20:34:05.552 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:34:05.552 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:34:05.553 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:34:05.553 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 120781 2023-09-23T20:34:05.553 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.553 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:34:05.553 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.553 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:34:05.553 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.802289+0000", 2023-09-23T20:34:05.553 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:34:05.553 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.553 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.553 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:34:05.553 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.802289+0000", 2023-09-23T20:34:05.554 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:34:05.554 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.554 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.554 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:34:05.554 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.802288+0000", 2023-09-23T20:34:05.554 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:34:05.554 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.554 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.554 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:34:05.554 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.802292+0000", 2023-09-23T20:34:05.554 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.4769999999999999e-06 2023-09-23T20:34:05.554 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.555 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.555 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:34:05.555 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.802304+0000", 2023-09-23T20:34:05.555 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1613e-05 2023-09-23T20:34:05.555 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.555 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.555 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:34:05.555 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.802310+0000", 2023-09-23T20:34:05.555 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.3820000000000001e-06 2023-09-23T20:34:05.555 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.555 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.556 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:34:05.556 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.802404+0000", 2023-09-23T20:34:05.556 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.3603000000000005e-05 2023-09-23T20:34:05.556 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.556 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.556 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:34:05.556 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.802408+0000", 2023-09-23T20:34:05.556 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.5240000000000002e-06 2023-09-23T20:34:05.556 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:34:05.556 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:34:05.556 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:34:05.556 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.557 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.557 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:120961 7.8 7.b3685e28 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e338)", 2023-09-23T20:34:05.557 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:34:05.236474+0000", 2023-09-23T20:34:05.557 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.28353099700000001, 2023-09-23T20:34:05.557 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.28387596900000001, 2023-09-23T20:34:05.557 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:34:05.557 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "delayed", 2023-09-23T20:34:05.557 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:34:05.557 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:34:05.557 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:34:05.557 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 120961 2023-09-23T20:34:05.557 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.558 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:34:05.558 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.558 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:34:05.558 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.236474+0000", 2023-09-23T20:34:05.558 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:34:05.558 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.558 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.559 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:34:05.559 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.236474+0000", 2023-09-23T20:34:05.559 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:34:05.559 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.559 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.559 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:34:05.559 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.236473+0000", 2023-09-23T20:34:05.559 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.999999 2023-09-23T20:34:05.559 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.559 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.559 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:34:05.559 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.236479+0000", 2023-09-23T20:34:05.560 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.9780000000000002e-06 2023-09-23T20:34:05.560 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.560 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.560 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:34:05.560 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.236491+0000", 2023-09-23T20:34:05.560 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2012e-05 2023-09-23T20:34:05.560 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.560 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.560 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:34:05.560 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.236497+0000", 2023-09-23T20:34:05.560 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.1199999999999999e-06 2023-09-23T20:34:05.560 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.561 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.561 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:34:05.561 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.236560+0000", 2023-09-23T20:34:05.561 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.3167e-05 2023-09-23T20:34:05.561 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.561 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.561 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:34:05.561 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.236569+0000", 2023-09-23T20:34:05.561 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.4239999999999999e-06 2023-09-23T20:34:05.561 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:34:05.561 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:34:05.561 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:34:05.562 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.562 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.562 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:120746 7.8 7.818b25c8 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e338)", 2023-09-23T20:34:05.562 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:34:04.743456+0000", 2023-09-23T20:34:05.562 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.77654837099999996, 2023-09-23T20:34:05.562 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.77696335100000002, 2023-09-23T20:34:05.562 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:34:05.562 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "delayed", 2023-09-23T20:34:05.562 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:34:05.562 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:34:05.562 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:34:05.563 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 120746 2023-09-23T20:34:05.563 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.563 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:34:05.563 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.563 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:34:05.563 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.743456+0000", 2023-09-23T20:34:05.563 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:34:05.563 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.563 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.563 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:34:05.563 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.743456+0000", 2023-09-23T20:34:05.563 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:34:05.564 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.564 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.564 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:34:05.564 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.743455+0000", 2023-09-23T20:34:05.564 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:34:05.564 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.564 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.564 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:34:05.564 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.743462+0000", 2023-09-23T20:34:05.564 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.3270000000000003e-06 2023-09-23T20:34:05.564 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.564 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.565 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:34:05.565 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.743480+0000", 2023-09-23T20:34:05.565 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.7609999999999999e-05 2023-09-23T20:34:05.565 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.565 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.565 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:34:05.565 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.743488+0000", 2023-09-23T20:34:05.565 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.7100000000000007e-06 2023-09-23T20:34:05.565 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.565 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.565 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:34:05.565 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.743543+0000", 2023-09-23T20:34:05.566 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.5819000000000002e-05 2023-09-23T20:34:05.566 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.566 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.566 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:34:05.566 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.743558+0000", 2023-09-23T20:34:05.566 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.4195e-05 2023-09-23T20:34:05.566 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:34:05.566 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:34:05.566 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:34:05.566 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.566 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.566 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:120962 7.8 7.20a4c3a8 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e338)", 2023-09-23T20:34:05.567 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:34:05.236507+0000", 2023-09-23T20:34:05.567 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.28349784500000003, 2023-09-23T20:34:05.567 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.28398372799999999, 2023-09-23T20:34:05.567 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:34:05.567 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "delayed", 2023-09-23T20:34:05.567 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:34:05.567 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:34:05.567 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:34:05.567 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 120962 2023-09-23T20:34:05.567 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.567 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:34:05.568 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.568 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:34:05.568 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.236507+0000", 2023-09-23T20:34:05.568 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:34:05.568 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.568 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.568 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:34:05.568 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.236507+0000", 2023-09-23T20:34:05.568 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:34:05.568 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.568 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.568 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:34:05.569 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.236506+0000", 2023-09-23T20:34:05.569 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:34:05.569 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.569 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.569 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:34:05.569 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.236509+0000", 2023-09-23T20:34:05.569 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.6919999999999998e-06 2023-09-23T20:34:05.569 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.569 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.569 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:34:05.569 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.236522+0000", 2023-09-23T20:34:05.569 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3036e-05 2023-09-23T20:34:05.570 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.570 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.570 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:34:05.570 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.236530+0000", 2023-09-23T20:34:05.570 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.5310000000000004e-06 2023-09-23T20:34:05.570 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.570 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.570 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:34:05.570 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.236593+0000", 2023-09-23T20:34:05.570 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.3416000000000004e-05 2023-09-23T20:34:05.570 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.570 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.571 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:34:05.571 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.236601+0000", 2023-09-23T20:34:05.571 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.61e-06 2023-09-23T20:34:05.571 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:34:05.571 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:34:05.571 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:34:05.571 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.571 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.571 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:120851 7.8 7.75801e18 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e338)", 2023-09-23T20:34:05.571 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:34:04.975252+0000", 2023-09-23T20:34:05.571 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.54475264700000003, 2023-09-23T20:34:05.571 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.545311765, 2023-09-23T20:34:05.572 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:34:05.572 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "delayed", 2023-09-23T20:34:05.572 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:34:05.572 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:34:05.572 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:34:05.572 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 120851 2023-09-23T20:34:05.572 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.572 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:34:05.572 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.572 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:34:05.572 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.975252+0000", 2023-09-23T20:34:05.572 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:34:05.573 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.573 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.573 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:34:05.573 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.975252+0000", 2023-09-23T20:34:05.573 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:34:05.573 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.573 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.573 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:34:05.573 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.975250+0000", 2023-09-23T20:34:05.573 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:34:05.573 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.573 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.574 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:34:05.574 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.975257+0000", 2023-09-23T20:34:05.574 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.6159999999999998e-06 2023-09-23T20:34:05.574 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.574 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.574 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:34:05.574 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.975287+0000", 2023-09-23T20:34:05.574 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.0151999999999999e-05 2023-09-23T20:34:05.574 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.574 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.574 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:34:05.574 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.975297+0000", 2023-09-23T20:34:05.575 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.7289999999999995e-06 2023-09-23T20:34:05.575 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.575 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.575 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:34:05.575 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.975373+0000", 2023-09-23T20:34:05.575 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.6670999999999998e-05 2023-09-23T20:34:05.575 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.575 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.575 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:34:05.575 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.975389+0000", 2023-09-23T20:34:05.575 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.5588999999999999e-05 2023-09-23T20:34:05.575 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:34:05.576 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:34:05.576 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:34:05.576 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.576 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.576 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:120680 7.8 7.164c9198 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e337)", 2023-09-23T20:34:05.576 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:34:04.594933+0000", 2023-09-23T20:34:05.576 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.92507128100000002, 2023-09-23T20:34:05.576 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.92570044200000001, 2023-09-23T20:34:05.576 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:34:05.576 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "delayed", 2023-09-23T20:34:05.576 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:34:05.576 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:34:05.577 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:34:05.577 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 120680 2023-09-23T20:34:05.577 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.577 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:34:05.577 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.577 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:34:05.577 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.594933+0000", 2023-09-23T20:34:05.577 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:34:05.577 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.577 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.577 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:34:05.577 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.594933+0000", 2023-09-23T20:34:05.578 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:34:05.578 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.578 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.578 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:34:05.578 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.594933+0000", 2023-09-23T20:34:05.578 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:34:05.578 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.578 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.578 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:34:05.578 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.594936+0000", 2023-09-23T20:34:05.578 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.5360000000000001e-06 2023-09-23T20:34:05.579 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.579 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.579 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:34:05.579 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.594943+0000", 2023-09-23T20:34:05.579 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.4300000000000003e-06 2023-09-23T20:34:05.579 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.579 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.579 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:34:05.579 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.594949+0000", 2023-09-23T20:34:05.579 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.4099999999999996e-06 2023-09-23T20:34:05.579 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.579 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.580 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:34:05.580 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.598565+0000", 2023-09-23T20:34:05.580 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0036157450000000001 2023-09-23T20:34:05.580 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.580 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.580 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:34:05.580 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.598581+0000", 2023-09-23T20:34:05.580 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.5749000000000001e-05 2023-09-23T20:34:05.580 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:34:05.580 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:34:05.580 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:34:05.580 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.581 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.581 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:120991 7.8 7.7e99dce8 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e338)", 2023-09-23T20:34:05.581 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:34:05.362063+0000", 2023-09-23T20:34:05.581 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.15794154299999999, 2023-09-23T20:34:05.581 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.15864022999999999, 2023-09-23T20:34:05.581 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:34:05.581 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "delayed", 2023-09-23T20:34:05.581 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:34:05.581 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:34:05.581 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:34:05.581 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 120991 2023-09-23T20:34:05.581 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.582 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:34:05.582 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.582 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:34:05.582 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.362063+0000", 2023-09-23T20:34:05.582 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:34:05.582 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.582 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.582 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:34:05.582 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.362063+0000", 2023-09-23T20:34:05.582 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:34:05.582 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.583 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.583 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:34:05.583 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.362061+0000", 2023-09-23T20:34:05.583 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999976 2023-09-23T20:34:05.583 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.583 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.583 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:34:05.583 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.362071+0000", 2023-09-23T20:34:05.583 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0603e-05 2023-09-23T20:34:05.583 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.583 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.583 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:34:05.584 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.362092+0000", 2023-09-23T20:34:05.584 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.0378999999999999e-05 2023-09-23T20:34:05.584 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.584 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.584 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:34:05.584 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.362103+0000", 2023-09-23T20:34:05.584 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1364e-05 2023-09-23T20:34:05.584 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.584 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.584 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:34:05.584 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.362177+0000", 2023-09-23T20:34:05.584 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.4264999999999995e-05 2023-09-23T20:34:05.584 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.585 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.585 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:34:05.585 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.362196+0000", 2023-09-23T20:34:05.585 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.8206000000000002e-05 2023-09-23T20:34:05.585 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:34:05.585 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:34:05.585 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:34:05.585 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.585 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.585 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:120969 7.8 7.9d0343e8 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e338)", 2023-09-23T20:34:05.585 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:34:05.258832+0000", 2023-09-23T20:34:05.585 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.26117285499999998, 2023-09-23T20:34:05.586 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.26194088700000001, 2023-09-23T20:34:05.586 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:34:05.586 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "delayed", 2023-09-23T20:34:05.586 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:34:05.586 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:34:05.586 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:34:05.586 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 120969 2023-09-23T20:34:05.586 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.586 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:34:05.586 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.586 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:34:05.587 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.258832+0000", 2023-09-23T20:34:05.587 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:34:05.587 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.587 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.587 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:34:05.587 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.258832+0000", 2023-09-23T20:34:05.587 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:34:05.587 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.587 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.587 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:34:05.587 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.258830+0000", 2023-09-23T20:34:05.587 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:34:05.588 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.588 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.588 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:34:05.588 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.258840+0000", 2023-09-23T20:34:05.588 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.5259999999999999e-06 2023-09-23T20:34:05.588 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.588 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.588 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:34:05.588 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.258855+0000", 2023-09-23T20:34:05.588 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.5693999999999999e-05 2023-09-23T20:34:05.588 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.588 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.589 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:34:05.589 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.258862+0000", 2023-09-23T20:34:05.589 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.0280000000000003e-06 2023-09-23T20:34:05.589 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.589 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.589 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:34:05.589 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.258923+0000", 2023-09-23T20:34:05.589 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.0167999999999997e-05 2023-09-23T20:34:05.589 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.589 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.589 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:34:05.589 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.258932+0000", 2023-09-23T20:34:05.590 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.5699999999999999e-06 2023-09-23T20:34:05.590 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:34:05.590 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:34:05.590 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:34:05.590 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.590 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.590 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:120761 7.8 7.e8601c48 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e338)", 2023-09-23T20:34:05.590 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:34:04.772652+0000", 2023-09-23T20:34:05.590 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.74735309299999997, 2023-09-23T20:34:05.590 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.74819834900000004, 2023-09-23T20:34:05.590 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:34:05.590 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "delayed", 2023-09-23T20:34:05.591 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:34:05.591 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:34:05.591 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:34:05.591 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 120761 2023-09-23T20:34:05.591 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.591 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:34:05.591 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.591 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:34:05.591 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.772652+0000", 2023-09-23T20:34:05.591 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:34:05.591 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.591 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.592 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:34:05.592 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.772652+0000", 2023-09-23T20:34:05.592 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:34:05.592 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.592 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.592 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:34:05.592 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.772650+0000", 2023-09-23T20:34:05.592 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:34:05.592 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.592 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.592 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:34:05.592 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.772660+0000", 2023-09-23T20:34:05.593 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0069e-05 2023-09-23T20:34:05.593 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.593 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.593 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:34:05.593 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.772684+0000", 2023-09-23T20:34:05.593 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.3852000000000001e-05 2023-09-23T20:34:05.593 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.593 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.594 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:34:05.594 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.772697+0000", 2023-09-23T20:34:05.594 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3505e-05 2023-09-23T20:34:05.594 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.594 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.594 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:34:05.594 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.772764+0000", 2023-09-23T20:34:05.594 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.7130000000000003e-05 2023-09-23T20:34:05.594 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.594 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.594 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:34:05.594 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.772778+0000", 2023-09-23T20:34:05.595 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.4035e-05 2023-09-23T20:34:05.595 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:34:05.595 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:34:05.595 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:34:05.595 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.595 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.595 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:120907 7.8 7.ff3d5aa8 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e338)", 2023-09-23T20:34:05.595 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:34:05.124351+0000", 2023-09-23T20:34:05.595 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.39565371399999999, 2023-09-23T20:34:05.595 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.39658109600000002, 2023-09-23T20:34:05.595 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:34:05.596 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "delayed", 2023-09-23T20:34:05.596 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:34:05.596 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:34:05.596 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:34:05.596 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 120907 2023-09-23T20:34:05.596 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.596 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:34:05.596 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.596 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:34:05.596 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.124351+0000", 2023-09-23T20:34:05.596 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:34:05.596 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.597 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.597 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:34:05.597 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.124351+0000", 2023-09-23T20:34:05.597 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:34:05.597 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.597 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.597 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:34:05.597 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.124349+0000", 2023-09-23T20:34:05.597 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:34:05.597 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.597 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.597 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:34:05.598 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.124357+0000", 2023-09-23T20:34:05.598 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.8669999999999994e-06 2023-09-23T20:34:05.598 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.598 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.598 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:34:05.598 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.124371+0000", 2023-09-23T20:34:05.598 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3994e-05 2023-09-23T20:34:05.598 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.598 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.598 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:34:05.598 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.124381+0000", 2023-09-23T20:34:05.598 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.5990000000000008e-06 2023-09-23T20:34:05.599 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.599 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.599 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:34:05.599 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.124458+0000", 2023-09-23T20:34:05.599 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.6977000000000005e-05 2023-09-23T20:34:05.599 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.599 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.599 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:34:05.599 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.124472+0000", 2023-09-23T20:34:05.599 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.4284000000000001e-05 2023-09-23T20:34:05.599 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:34:05.599 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:34:05.600 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:34:05.600 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.600 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.600 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:120936 7.8 7.9f63b188 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e338)", 2023-09-23T20:34:05.600 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:34:05.190420+0000", 2023-09-23T20:34:05.600 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.32958499000000002, 2023-09-23T20:34:05.600 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.33058259899999998, 2023-09-23T20:34:05.600 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:34:05.600 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "delayed", 2023-09-23T20:34:05.600 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:34:05.600 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:34:05.601 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:34:05.601 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 120936 2023-09-23T20:34:05.601 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.601 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:34:05.601 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.601 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:34:05.601 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.190420+0000", 2023-09-23T20:34:05.601 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:34:05.601 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.601 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.601 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:34:05.601 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.190420+0000", 2023-09-23T20:34:05.602 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:34:05.602 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.602 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.602 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:34:05.602 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.190418+0000", 2023-09-23T20:34:05.602 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:34:05.602 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.602 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.602 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:34:05.602 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.190425+0000", 2023-09-23T20:34:05.602 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.2810000000000003e-06 2023-09-23T20:34:05.602 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.603 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.603 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:34:05.603 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.190443+0000", 2023-09-23T20:34:05.603 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.7161999999999999e-05 2023-09-23T20:34:05.603 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.603 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.603 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:34:05.603 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.190451+0000", 2023-09-23T20:34:05.603 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.6389999999999998e-06 2023-09-23T20:34:05.603 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.603 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.603 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:34:05.604 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.190513+0000", 2023-09-23T20:34:05.604 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.1506000000000004e-05 2023-09-23T20:34:05.604 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:05.604 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:05.604 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:34:05.604 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.190524+0000", 2023-09-23T20:34:05.604 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1591e-05 2023-09-23T20:34:05.604 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:34:05.604 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:34:05.604 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:34:05.604 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:34:05.604 INFO:teuthology.orchestra.run.smithi018.stdout: ], 2023-09-23T20:34:05.605 INFO:teuthology.orchestra.run.smithi018.stdout: "num_ops": 16 2023-09-23T20:34:05.605 INFO:teuthology.orchestra.run.smithi018.stdout:} 2023-09-23T20:34:05.605 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:34:05.606 INFO:teuthology.orchestra.run.smithi018.stderr:reweighted osd.0 to 1 (10000) 2023-09-23T20:34:05.688 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:05.751 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:34:05.754 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:05.755 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:34:05.855 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:34:05.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:05 smithi018 ceph-mon[179647]: failed to encode map e338 with expected crc 2023-09-23T20:34:05.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:05 smithi018 ceph-mon[179647]: osdmap e338: 8 total, 8 up, 8 in 2023-09-23T20:34:05.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:05 smithi018 ceph-mon[179647]: failed to encode map e338 with expected crc 2023-09-23T20:34:05.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:05 smithi018 ceph-mon[179647]: failed to encode map e338 with expected crc 2023-09-23T20:34:05.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:05 smithi018 ceph-mon[179647]: failed to encode map e338 with expected crc 2023-09-23T20:34:05.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:05 smithi018 ceph-mon[179647]: failed to encode map e338 with expected crc 2023-09-23T20:34:05.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:05 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1843564062' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T20:34:05.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:05 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T20:34:05.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:05 smithi018 ceph-mon[184144]: failed to encode map e338 with expected crc 2023-09-23T20:34:05.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:05 smithi018 ceph-mon[184144]: osdmap e338: 8 total, 8 up, 8 in 2023-09-23T20:34:05.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:05 smithi018 ceph-mon[184144]: failed to encode map e338 with expected crc 2023-09-23T20:34:05.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:05 smithi018 ceph-mon[184144]: failed to encode map e338 with expected crc 2023-09-23T20:34:05.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:05 smithi018 ceph-mon[184144]: failed to encode map e338 with expected crc 2023-09-23T20:34:05.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:05 smithi018 ceph-mon[184144]: failed to encode map e338 with expected crc 2023-09-23T20:34:05.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:05 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1843564062' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T20:34:05.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:05 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T20:34:05.910 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:05 smithi149 ceph-mon[170394]: failed to encode map e338 with expected crc 2023-09-23T20:34:05.910 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:05 smithi149 ceph-mon[170394]: osdmap e338: 8 total, 8 up, 8 in 2023-09-23T20:34:05.910 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:05 smithi149 ceph-mon[170394]: failed to encode map e338 with expected crc 2023-09-23T20:34:05.911 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:05 smithi149 ceph-mon[170394]: failed to encode map e338 with expected crc 2023-09-23T20:34:05.911 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:05 smithi149 ceph-mon[170394]: failed to encode map e338 with expected crc 2023-09-23T20:34:05.911 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:05 smithi149 ceph-mon[170394]: failed to encode map e338 with expected crc 2023-09-23T20:34:05.911 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:05 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1843564062' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T20:34:05.911 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:05 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T20:34:05.913 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:06.055 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:34:06.061 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:06.061 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:34:06.162 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:34:06.238 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:34:06.254 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:06.599 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:06 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T20:34:06.599 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:06 smithi018 ceph-mon[179647]: osdmap e339: 8 total, 8 up, 8 in 2023-09-23T20:34:06.600 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:06 smithi018 ceph-mon[179647]: failed to encode map e339 with expected crc 2023-09-23T20:34:06.600 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:06 smithi018 ceph-mon[179647]: failed to encode map e339 with expected crc 2023-09-23T20:34:06.600 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:06 smithi018 ceph-mon[179647]: failed to encode map e339 with expected crc 2023-09-23T20:34:06.600 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:06 smithi018 ceph-mon[179647]: pgmap v795: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 4.2 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 903 op/s; 4011/205818 objects degraded (1.949%); 1068/205818 objects misplaced (0.519%); 6.0 MiB/s, 96 objects/s recovering 2023-09-23T20:34:06.601 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:06 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 5117/212733 objects degraded (2.405%), 3 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T20:34:06.818 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:34:06.820 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:06.821 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:34:06.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:06 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T20:34:06.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:06 smithi018 ceph-mon[184144]: osdmap e339: 8 total, 8 up, 8 in 2023-09-23T20:34:06.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:06 smithi018 ceph-mon[184144]: failed to encode map e339 with expected crc 2023-09-23T20:34:06.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:06 smithi018 ceph-mon[184144]: failed to encode map e339 with expected crc 2023-09-23T20:34:06.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:06 smithi018 ceph-mon[184144]: failed to encode map e339 with expected crc 2023-09-23T20:34:06.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:06 smithi018 ceph-mon[184144]: pgmap v795: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 4.2 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 903 op/s; 4011/205818 objects degraded (1.949%); 1068/205818 objects misplaced (0.519%); 6.0 MiB/s, 96 objects/s recovering 2023-09-23T20:34:06.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:06 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 5117/212733 objects degraded (2.405%), 3 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T20:34:06.921 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:34:06.957 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:06 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T20:34:06.958 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:06 smithi149 ceph-mon[170394]: osdmap e339: 8 total, 8 up, 8 in 2023-09-23T20:34:06.958 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:06 smithi149 ceph-mon[170394]: failed to encode map e339 with expected crc 2023-09-23T20:34:06.958 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:06 smithi149 ceph-mon[170394]: failed to encode map e339 with expected crc 2023-09-23T20:34:06.959 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:06 smithi149 ceph-mon[170394]: failed to encode map e339 with expected crc 2023-09-23T20:34:06.959 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:06 smithi149 ceph-mon[170394]: pgmap v795: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 4.2 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 903 op/s; 4011/205818 objects degraded (1.949%); 1068/205818 objects misplaced (0.519%); 6.0 MiB/s, 96 objects/s recovering 2023-09-23T20:34:06.959 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:06 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 5117/212733 objects degraded (2.405%), 3 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T20:34:06.961 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:07.061 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:34:07.066 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:07.067 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:34:07.168 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:34:07.225 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:07.391 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:34:07.399 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:07.399 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:34:07.500 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:34:07.592 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:09.094 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd reweight 1 1 2023-09-23T20:34:09.139 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:08 smithi018 ceph-mon[184144]: pgmap v796: 119 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 4.1 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 619 op/s; 1997/203622 objects degraded (0.981%); 2797/203622 objects misplaced (1.374%); 8.5 MiB/s, 136 objects/s recovering 2023-09-23T20:34:09.140 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:08 smithi018 ceph-mon[179647]: pgmap v796: 119 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 4.1 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 619 op/s; 1997/203622 objects degraded (0.981%); 2797/203622 objects misplaced (1.374%); 8.5 MiB/s, 136 objects/s recovering 2023-09-23T20:34:09.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:08 smithi149 ceph-mon[170394]: pgmap v796: 119 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 4.1 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 619 op/s; 1997/203622 objects degraded (0.981%); 2797/203622 objects misplaced (1.374%); 8.5 MiB/s, 136 objects/s recovering 2023-09-23T20:34:09.463 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:34:09.473 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:09.473 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:34:09.574 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:34:09.606 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:09.660 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:34:09.663 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:09.664 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:34:09.670 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T20:34:09.765 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:34:09.880 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:09.994 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:34:10.063 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:34:10.069 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:10.069 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:34:10.170 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:34:10.264 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:10.358 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:34:10.367 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:10.368 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:34:10.388 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:34:10.469 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:34:10.503 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:10.546 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:34:10.550 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:10.550 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:34:10.643 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:10 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:34:10.643 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:10 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:34:10.644 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:10 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:34:10.644 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:10 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:34:10.650 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:34:10.677 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:10 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:34:10.677 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:10 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:34:10.680 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:10.720 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:34:10.725 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:10.726 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:34:10.827 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:34:10.974 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:11.162 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:34:11.178 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:11.179 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:34:11.279 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:34:11.313 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:11.365 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:34:11.367 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:11.367 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:34:11.468 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:34:11.497 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:11.548 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:34:11.552 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:11.553 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:34:11.616 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:34:11.631 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:11 smithi018 ceph-mon[179647]: pgmap v797: 119 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 536 op/s; 1997/202452 objects degraded (0.986%); 2797/202452 objects misplaced (1.382%); 6.7 MiB/s, 107 objects/s recovering 2023-09-23T20:34:11.631 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:11 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 1997/202452 objects degraded (0.986%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:34:11.632 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:11 smithi018 ceph-mon[184144]: pgmap v797: 119 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 536 op/s; 1997/202452 objects degraded (0.986%); 2797/202452 objects misplaced (1.382%); 6.7 MiB/s, 107 objects/s recovering 2023-09-23T20:34:11.632 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:11 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 1997/202452 objects degraded (0.986%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:34:11.653 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:34:11.681 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:11 smithi149 ceph-mon[170394]: pgmap v797: 119 pgs: 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 536 op/s; 1997/202452 objects degraded (0.986%); 2797/202452 objects misplaced (1.382%); 6.7 MiB/s, 107 objects/s recovering 2023-09-23T20:34:11.682 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:11 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 1997/202452 objects degraded (0.986%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:34:11.685 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:11.727 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:34:11.734 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:11.735 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:34:11.836 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:34:11.941 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:12.246 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:34:12.251 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:12.251 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:34:12.353 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:34:12.426 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:12.460 INFO:teuthology.orchestra.run.smithi018.stderr:noscrub is unset 2023-09-23T20:34:12.496 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:12 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2763955693' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:34:12.496 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:12 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:34:12.497 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:12 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2763955693' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:34:12.497 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:12 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:34:12.562 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:34:12.570 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:12.570 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:34:12.671 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:34:12.698 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:12 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2763955693' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:34:12.698 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:12 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:34:12.701 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:12.754 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:34:12.757 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:12.758 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:34:12.820 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:34:12.858 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:34:12.882 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:34:12.938 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:34:12.966 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:12.991 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:34:13.042 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:34:13.077 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:34:13.104 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:34:13.339 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:34:13.345 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:13.346 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:34:13.369 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:34:13.403 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:34:13.403 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:34:13.403 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:34:13.403 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:34:13.403 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:34:13.403 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:34:13.403 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:34:13.404 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:34:13.447 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:34:13.493 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:13.545 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:34:13.548 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:13.548 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:34:13.643 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:13 smithi018 ceph-mon[184144]: pgmap v798: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 115 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 690 op/s; 1997/196128 objects degraded (1.018%); 2453/196128 objects misplaced (1.251%); 5.9 MiB/s, 94 objects/s recovering 2023-09-23T20:34:13.643 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:13 smithi018 ceph-mon[184144]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:34:13.643 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:13 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:34:13.643 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:13 smithi018 ceph-mon[184144]: osdmap e340: 8 total, 8 up, 8 in 2023-09-23T20:34:13.643 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:13 smithi018 ceph-mon[184144]: failed to encode map e340 with expected crc 2023-09-23T20:34:13.643 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:13 smithi018 ceph-mon[184144]: failed to encode map e340 with expected crc 2023-09-23T20:34:13.643 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:13 smithi018 ceph-mon[184144]: failed to encode map e340 with expected crc 2023-09-23T20:34:13.643 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:13 smithi018 ceph-mon[184144]: failed to encode map e340 with expected crc 2023-09-23T20:34:13.643 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:13 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3177632564' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:34:13.644 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:13 smithi018 ceph-mon[179647]: pgmap v798: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 115 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 690 op/s; 1997/196128 objects degraded (1.018%); 2453/196128 objects misplaced (1.251%); 5.9 MiB/s, 94 objects/s recovering 2023-09-23T20:34:13.644 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:13 smithi018 ceph-mon[179647]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:34:13.644 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:13 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:34:13.644 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:13 smithi018 ceph-mon[179647]: osdmap e340: 8 total, 8 up, 8 in 2023-09-23T20:34:13.644 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:13 smithi018 ceph-mon[179647]: failed to encode map e340 with expected crc 2023-09-23T20:34:13.644 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:13 smithi018 ceph-mon[179647]: failed to encode map e340 with expected crc 2023-09-23T20:34:13.644 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:13 smithi018 ceph-mon[179647]: failed to encode map e340 with expected crc 2023-09-23T20:34:13.644 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:13 smithi018 ceph-mon[179647]: failed to encode map e340 with expected crc 2023-09-23T20:34:13.644 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:13 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3177632564' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:34:13.649 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:34:13.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:13 smithi149 ceph-mon[170394]: pgmap v798: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 115 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 690 op/s; 1997/196128 objects degraded (1.018%); 2453/196128 objects misplaced (1.251%); 5.9 MiB/s, 94 objects/s recovering 2023-09-23T20:34:13.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:13 smithi149 ceph-mon[170394]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:34:13.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:13 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:34:13.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:13 smithi149 ceph-mon[170394]: osdmap e340: 8 total, 8 up, 8 in 2023-09-23T20:34:13.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:13 smithi149 ceph-mon[170394]: failed to encode map e340 with expected crc 2023-09-23T20:34:13.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:13 smithi149 ceph-mon[170394]: failed to encode map e340 with expected crc 2023-09-23T20:34:13.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:13 smithi149 ceph-mon[170394]: failed to encode map e340 with expected crc 2023-09-23T20:34:13.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:13 smithi149 ceph-mon[170394]: failed to encode map e340 with expected crc 2023-09-23T20:34:13.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:13 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3177632564' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:34:13.734 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:13.958 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:34:13.965 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:13.966 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:34:14.066 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:34:14.180 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:14.573 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:34:14.582 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:14.583 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:34:14.684 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:34:14.768 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:15.052 INFO:teuthology.orchestra.run.smithi018.stdout:ERROR: (22) Invalid argument 2023-09-23T20:34:15.053 INFO:teuthology.orchestra.run.smithi018.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:34:15.092 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:34:15.101 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:15.102 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:34:15.203 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:34:15.235 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:15.282 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:34:15.286 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:15.286 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:34:15.342 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:34:14 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:34:14] "GET /metrics HTTP/1.1" 200 36555 "" "Prometheus/2.43.0" 2023-09-23T20:34:15.387 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:34:15.457 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T20:34:15.484 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:15.622 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:34:15.626 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:15 smithi018 ceph-mon[179647]: pgmap v800: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 115 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 596 op/s; 1997/193743 objects degraded (1.031%); 2227/193743 objects misplaced (1.149%); 6.4 MiB/s, 103 objects/s recovering 2023-09-23T20:34:15.627 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:15 smithi018 ceph-mon[179647]: 1.4 scrub starts 2023-09-23T20:34:15.627 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:15 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:34:15.627 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:15 smithi018 ceph-mon[184144]: pgmap v800: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 115 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 596 op/s; 1997/193743 objects degraded (1.031%); 2227/193743 objects misplaced (1.149%); 6.4 MiB/s, 103 objects/s recovering 2023-09-23T20:34:15.627 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:15 smithi018 ceph-mon[184144]: 1.4 scrub starts 2023-09-23T20:34:15.627 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:15 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:34:15.628 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:15.629 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:34:15.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:15 smithi149 ceph-mon[170394]: pgmap v800: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 115 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 596 op/s; 1997/193743 objects degraded (1.031%); 2227/193743 objects misplaced (1.149%); 6.4 MiB/s, 103 objects/s recovering 2023-09-23T20:34:15.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:15 smithi149 ceph-mon[170394]: 1.4 scrub starts 2023-09-23T20:34:15.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:15 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:34:15.719 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T20:34:15.730 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:34:15.759 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:15.801 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:34:15.804 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:15.804 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:34:15.905 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:34:15.938 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:15.978 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:34:15.981 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:15.982 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:34:16.038 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:34:16.082 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:34:16.211 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:16.410 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:34:16.434 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:16.434 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:34:16.535 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:34:16.541 INFO:teuthology.orchestra.run.smithi018.stderr:reweighted osd.1 to 1 (10000) 2023-09-23T20:34:16.566 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:16.604 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:34:16.609 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:16.609 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:34:16.627 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:34:16.710 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:34:16.737 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:16 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/272342401' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T20:34:16.737 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:16 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 1997/193743 objects degraded (1.031%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:34:16.740 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:16.745 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:16 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/272342401' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T20:34:16.745 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:16 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 1997/193743 objects degraded (1.031%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:34:16.745 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:16 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/272342401' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T20:34:16.745 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:16 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 1997/193743 objects degraded (1.031%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:34:16.785 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:34:16.790 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:16.790 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:34:16.891 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:34:16.983 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:17.060 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:34:17.092 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd reweight 5 1 2023-09-23T20:34:17.239 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:34:17.245 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:17.246 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:34:17.346 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:34:17.538 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:17 smithi018 ceph-mon[179647]: pgmap v801: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 115 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 622 op/s; 1997/187116 objects degraded (1.067%); 1862/187116 objects misplaced (0.995%); 7.6 MiB/s, 121 objects/s recovering 2023-09-23T20:34:17.538 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:17 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/272342401' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T20:34:17.538 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:17 smithi018 ceph-mon[179647]: osdmap e341: 8 total, 8 up, 8 in 2023-09-23T20:34:17.538 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:17 smithi018 ceph-mon[179647]: failed to encode map e341 with expected crc 2023-09-23T20:34:17.539 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:17 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:34:17.539 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:17 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:34:17.539 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:17 smithi018 ceph-mon[184144]: pgmap v801: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 115 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 622 op/s; 1997/187116 objects degraded (1.067%); 1862/187116 objects misplaced (0.995%); 7.6 MiB/s, 121 objects/s recovering 2023-09-23T20:34:17.539 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:17 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/272342401' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T20:34:17.539 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:17 smithi018 ceph-mon[184144]: osdmap e341: 8 total, 8 up, 8 in 2023-09-23T20:34:17.539 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:17 smithi018 ceph-mon[184144]: failed to encode map e341 with expected crc 2023-09-23T20:34:17.539 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:17 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:34:17.539 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:17 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:34:17.546 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:17.663 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:34:17.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:17 smithi149 ceph-mon[170394]: pgmap v801: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 115 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 622 op/s; 1997/187116 objects degraded (1.067%); 1862/187116 objects misplaced (0.995%); 7.6 MiB/s, 121 objects/s recovering 2023-09-23T20:34:17.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:17 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/272342401' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T20:34:17.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:17 smithi149 ceph-mon[170394]: osdmap e341: 8 total, 8 up, 8 in 2023-09-23T20:34:17.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:17 smithi149 ceph-mon[170394]: failed to encode map e341 with expected crc 2023-09-23T20:34:17.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:17 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:34:17.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:17 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:34:18.134 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:34:18.141 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:18.142 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:34:18.243 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:34:18.318 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:18.789 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:34:18.791 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:18 smithi018 ceph-mon[184144]: 1.a scrub starts 2023-09-23T20:34:18.791 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:18 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:34:18.791 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:18 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:34:18.792 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:18 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:34:18.792 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:18.793 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:34:18.793 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:18 smithi018 ceph-mon[179647]: 1.a scrub starts 2023-09-23T20:34:18.793 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:18 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:34:18.794 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:18 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:34:18.794 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:18 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:34:18.893 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:34:18.918 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:18 smithi149 ceph-mon[170394]: 1.a scrub starts 2023-09-23T20:34:18.918 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:18 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:34:18.918 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:18 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:34:18.918 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:18 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:34:18.920 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:18.955 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:34:18.957 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:18.958 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:34:19.058 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:34:19.085 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:19.117 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:34:19.120 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:19.120 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:34:19.221 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:34:19.247 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:19.280 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:34:19.283 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:19.283 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:34:19.384 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:34:19.420 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:19.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:19 smithi018 ceph-mon[179647]: pgmap v803: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 115 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 728 op/s; 1997/184947 objects degraded (1.080%); 1600/184947 objects misplaced (0.865%); 9.3 MiB/s, 149 objects/s recovering 2023-09-23T20:34:19.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:19 smithi018 ceph-mon[179647]: osdmap e342: 8 total, 8 up, 8 in 2023-09-23T20:34:19.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:19 smithi018 ceph-mon[179647]: failed to encode map e342 with expected crc 2023-09-23T20:34:19.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:19 smithi018 ceph-mon[179647]: failed to encode map e342 with expected crc 2023-09-23T20:34:19.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:19 smithi018 ceph-mon[184144]: pgmap v803: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 115 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 728 op/s; 1997/184947 objects degraded (1.080%); 1600/184947 objects misplaced (0.865%); 9.3 MiB/s, 149 objects/s recovering 2023-09-23T20:34:19.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:19 smithi018 ceph-mon[184144]: osdmap e342: 8 total, 8 up, 8 in 2023-09-23T20:34:19.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:19 smithi018 ceph-mon[184144]: failed to encode map e342 with expected crc 2023-09-23T20:34:19.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:19 smithi018 ceph-mon[184144]: failed to encode map e342 with expected crc 2023-09-23T20:34:19.801 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:34:19.892 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:19.892 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:34:19.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:19 smithi149 ceph-mon[170394]: pgmap v803: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 115 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 728 op/s; 1997/184947 objects degraded (1.080%); 1600/184947 objects misplaced (0.865%); 9.3 MiB/s, 149 objects/s recovering 2023-09-23T20:34:19.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:19 smithi149 ceph-mon[170394]: osdmap e342: 8 total, 8 up, 8 in 2023-09-23T20:34:19.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:19 smithi149 ceph-mon[170394]: failed to encode map e342 with expected crc 2023-09-23T20:34:19.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:19 smithi149 ceph-mon[170394]: failed to encode map e342 with expected crc 2023-09-23T20:34:19.994 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:34:20.026 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:20.075 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:34:20.076 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:20.077 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:34:20.177 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:34:20.215 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:20.252 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:34:20.255 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:20.256 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:34:20.356 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:34:20.387 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:20.418 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:34:20.421 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:20.421 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:34:20.522 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:34:20.558 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:20.596 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:34:20.605 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:20.606 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:34:20.670 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:34:20.707 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:34:20.719 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:20 smithi149 ceph-mon[170394]: failed to encode map e342 with expected crc 2023-09-23T20:34:20.719 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:20 smithi149 ceph-mon[170394]: failed to encode map e342 with expected crc 2023-09-23T20:34:20.720 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:20 smithi149 ceph-mon[170394]: 1.e scrub starts 2023-09-23T20:34:20.720 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:20 smithi149 ceph-mon[170394]: failed to encode map e343 with expected crc 2023-09-23T20:34:20.720 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:20 smithi149 ceph-mon[170394]: osdmap e343: 8 total, 8 up, 8 in 2023-09-23T20:34:20.790 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:20 smithi018 ceph-mon[184144]: failed to encode map e342 with expected crc 2023-09-23T20:34:20.790 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:20 smithi018 ceph-mon[184144]: failed to encode map e342 with expected crc 2023-09-23T20:34:20.790 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:20 smithi018 ceph-mon[184144]: 1.e scrub starts 2023-09-23T20:34:20.790 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:20 smithi018 ceph-mon[184144]: failed to encode map e343 with expected crc 2023-09-23T20:34:20.790 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:20 smithi018 ceph-mon[184144]: osdmap e343: 8 total, 8 up, 8 in 2023-09-23T20:34:20.791 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:20 smithi018 ceph-mon[179647]: failed to encode map e342 with expected crc 2023-09-23T20:34:20.791 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:20 smithi018 ceph-mon[179647]: failed to encode map e342 with expected crc 2023-09-23T20:34:20.791 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:20 smithi018 ceph-mon[179647]: 1.e scrub starts 2023-09-23T20:34:20.791 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:20 smithi018 ceph-mon[179647]: failed to encode map e343 with expected crc 2023-09-23T20:34:20.791 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:20 smithi018 ceph-mon[179647]: osdmap e343: 8 total, 8 up, 8 in 2023-09-23T20:34:20.802 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:21.517 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:21 smithi018 ceph-mon[179647]: failed to encode map e343 with expected crc 2023-09-23T20:34:21.518 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:21 smithi018 ceph-mon[179647]: pgmap v806: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 115 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 555 op/s; 1997/183744 objects degraded (1.087%); 1600/183744 objects misplaced (0.871%); 6.5 MiB/s, 104 objects/s recovering 2023-09-23T20:34:21.518 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:21 smithi018 ceph-mon[179647]: 1.d scrub starts 2023-09-23T20:34:21.518 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:21 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/4279655715' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:34:21.518 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:21 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 1997/183744 objects degraded (1.087%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:34:21.629 INFO:teuthology.orchestra.run.smithi018.stderr:nodeep-scrub is unset 2023-09-23T20:34:21.781 INFO:teuthology.orchestra.run.smithi018.stdout:ERROR: (22) Invalid argument 2023-09-23T20:34:21.781 INFO:teuthology.orchestra.run.smithi018.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:34:21.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:21 smithi018 ceph-mon[184144]: failed to encode map e343 with expected crc 2023-09-23T20:34:21.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:21 smithi018 ceph-mon[184144]: pgmap v806: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 115 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 555 op/s; 1997/183744 objects degraded (1.087%); 1600/183744 objects misplaced (0.871%); 6.5 MiB/s, 104 objects/s recovering 2023-09-23T20:34:21.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:21 smithi018 ceph-mon[184144]: 1.d scrub starts 2023-09-23T20:34:21.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:21 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/4279655715' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:34:21.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:21 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 1997/183744 objects degraded (1.087%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:34:21.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:21 smithi149 ceph-mon[170394]: failed to encode map e343 with expected crc 2023-09-23T20:34:21.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:21 smithi149 ceph-mon[170394]: pgmap v806: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 2 active+recovering+undersized+degraded+remapped, 115 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 555 op/s; 1997/183744 objects degraded (1.087%); 1600/183744 objects misplaced (0.871%); 6.5 MiB/s, 104 objects/s recovering 2023-09-23T20:34:21.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:21 smithi149 ceph-mon[170394]: 1.d scrub starts 2023-09-23T20:34:21.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:21 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/4279655715' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:34:21.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:21 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 1997/183744 objects degraded (1.087%), 3 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T20:34:22.894 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:22 smithi018 ceph-mon[184144]: 1.9 scrub starts 2023-09-23T20:34:22.894 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:22 smithi018 ceph-mon[184144]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:34:22.894 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:22 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/4279655715' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:34:22.894 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:22 smithi018 ceph-mon[184144]: osdmap e344: 8 total, 8 up, 8 in 2023-09-23T20:34:22.894 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:22 smithi018 ceph-mon[184144]: failed to encode map e344 with expected crc 2023-09-23T20:34:22.894 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:22 smithi018 ceph-mon[184144]: failed to encode map e344 with expected crc 2023-09-23T20:34:22.894 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:22 smithi018 ceph-mon[184144]: failed to encode map e344 with expected crc 2023-09-23T20:34:22.895 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:22 smithi018 ceph-mon[184144]: failed to encode map e344 with expected crc 2023-09-23T20:34:22.895 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:22 smithi018 ceph-mon[179647]: 1.9 scrub starts 2023-09-23T20:34:22.895 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:22 smithi018 ceph-mon[179647]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:34:22.895 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:22 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/4279655715' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:34:22.895 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:22 smithi018 ceph-mon[179647]: osdmap e344: 8 total, 8 up, 8 in 2023-09-23T20:34:22.895 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:22 smithi018 ceph-mon[179647]: failed to encode map e344 with expected crc 2023-09-23T20:34:22.895 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:22 smithi018 ceph-mon[179647]: failed to encode map e344 with expected crc 2023-09-23T20:34:22.895 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:22 smithi018 ceph-mon[179647]: failed to encode map e344 with expected crc 2023-09-23T20:34:22.896 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:22 smithi018 ceph-mon[179647]: failed to encode map e344 with expected crc 2023-09-23T20:34:22.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:22 smithi149 ceph-mon[170394]: 1.9 scrub starts 2023-09-23T20:34:22.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:22 smithi149 ceph-mon[170394]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:34:22.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:22 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/4279655715' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:34:22.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:22 smithi149 ceph-mon[170394]: osdmap e344: 8 total, 8 up, 8 in 2023-09-23T20:34:22.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:22 smithi149 ceph-mon[170394]: failed to encode map e344 with expected crc 2023-09-23T20:34:22.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:22 smithi149 ceph-mon[170394]: failed to encode map e344 with expected crc 2023-09-23T20:34:22.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:22 smithi149 ceph-mon[170394]: failed to encode map e344 with expected crc 2023-09-23T20:34:22.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:22 smithi149 ceph-mon[170394]: failed to encode map e344 with expected crc 2023-09-23T20:34:23.566 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:34:23.582 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:23.583 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:34:23.684 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:34:23.788 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:23 smithi018 ceph-mon[184144]: failed to encode map e344 with expected crc 2023-09-23T20:34:23.788 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:23 smithi018 ceph-mon[184144]: pgmap v808: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 116 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 513 op/s; 1995/178872 objects degraded (1.115%); 1335/178872 objects misplaced (0.746%); 6.2 MiB/s, 98 objects/s recovering 2023-09-23T20:34:23.788 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:23 smithi018 ceph-mon[179647]: failed to encode map e344 with expected crc 2023-09-23T20:34:23.788 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:23 smithi018 ceph-mon[179647]: pgmap v808: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 116 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 513 op/s; 1995/178872 objects degraded (1.115%); 1335/178872 objects misplaced (0.746%); 6.2 MiB/s, 98 objects/s recovering 2023-09-23T20:34:23.807 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:23.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:23 smithi149 ceph-mon[170394]: failed to encode map e344 with expected crc 2023-09-23T20:34:23.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:23 smithi149 ceph-mon[170394]: pgmap v808: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 116 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 513 op/s; 1995/178872 objects degraded (1.115%); 1335/178872 objects misplaced (0.746%); 6.2 MiB/s, 98 objects/s recovering 2023-09-23T20:34:24.353 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:34:24.359 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:24.360 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:34:24.460 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:34:24.473 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T20:34:24.615 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:24.630 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:34:24.699 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:34:24.704 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:34:24.718 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:24.719 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:34:24.747 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:34:24.808 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:34:24.820 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:34:24.848 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:24 smithi149 ceph-mon[170394]: pgmap v809: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 116 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 464 op/s; 1995/176571 objects degraded (1.130%); 993/176571 objects misplaced (0.562%); 6.3 MiB/s, 101 objects/s recovering 2023-09-23T20:34:24.849 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:24 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3135569084' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:34:24.851 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:34:24.851 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:24.864 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:24 smithi018 ceph-mon[184144]: pgmap v809: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 116 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 464 op/s; 1995/176571 objects degraded (1.130%); 993/176571 objects misplaced (0.562%); 6.3 MiB/s, 101 objects/s recovering 2023-09-23T20:34:24.864 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:24 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3135569084' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:34:24.865 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:24 smithi018 ceph-mon[179647]: pgmap v809: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 116 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 464 op/s; 1995/176571 objects degraded (1.130%); 993/176571 objects misplaced (0.562%); 6.3 MiB/s, 101 objects/s recovering 2023-09-23T20:34:24.865 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:24 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3135569084' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:34:24.891 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:34:24.895 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:24.895 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:34:24.906 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:34:24.959 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:34:24.996 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:34:25.014 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:34:25.037 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:34:25.037 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:34:25.037 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:34:25.037 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:34:25.037 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:34:25.037 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:34:25.037 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:34:25.037 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:34:25.083 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:25.146 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:34:24 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:34:24] "GET /metrics HTTP/1.1" 200 36555 "" "Prometheus/2.43.0" 2023-09-23T20:34:25.169 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:34:25.175 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:25.175 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:34:25.276 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:34:25.309 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:25.351 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:34:25.354 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:25.355 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:34:25.455 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:34:25.490 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:25.531 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:34:25.534 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:25.534 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:34:25.635 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:34:25.719 INFO:teuthology.orchestra.run.smithi018.stderr:reweighted osd.5 to 1 (10000) 2023-09-23T20:34:25.746 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:34:25.765 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:25.859 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:34:25.870 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:25.870 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:25 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3623953184' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T20:34:25.871 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:25 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T20:34:25.871 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:25 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3623953184' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T20:34:25.871 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:25 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T20:34:25.871 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:34:25.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:25 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3623953184' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T20:34:25.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:25 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T20:34:25.972 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:34:26.053 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:26.150 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:34:26.152 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:26.153 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:34:26.253 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:34:26.285 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:26.330 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:34:26.334 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:26.334 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:34:26.435 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:34:26.469 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:26.511 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:34:26.514 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:26.515 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:34:26.571 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd reweight 7 1 2023-09-23T20:34:26.599 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T20:34:26.615 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:34:26.705 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:26 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T20:34:26.705 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:26 smithi018 ceph-mon[179647]: osdmap e345: 8 total, 8 up, 8 in 2023-09-23T20:34:26.706 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:26 smithi018 ceph-mon[179647]: failed to encode map e345 with expected crc 2023-09-23T20:34:26.706 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:26 smithi018 ceph-mon[179647]: pgmap v811: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 116 active+clean; 3.4 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 804 op/s; 1995/168594 objects degraded (1.183%); 993/168594 objects misplaced (0.589%); 6.1 MiB/s, 97 objects/s recovering 2023-09-23T20:34:26.706 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:26 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 1995/176571 objects degraded (1.130%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T20:34:26.763 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:26.883 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:34:26.902 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:26.903 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:34:26.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:26 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T20:34:26.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:26 smithi149 ceph-mon[170394]: osdmap e345: 8 total, 8 up, 8 in 2023-09-23T20:34:26.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:26 smithi149 ceph-mon[170394]: failed to encode map e345 with expected crc 2023-09-23T20:34:26.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:26 smithi149 ceph-mon[170394]: pgmap v811: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 116 active+clean; 3.4 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 804 op/s; 1995/168594 objects degraded (1.183%); 993/168594 objects misplaced (0.589%); 6.1 MiB/s, 97 objects/s recovering 2023-09-23T20:34:26.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:26 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 1995/176571 objects degraded (1.130%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T20:34:27.004 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:34:27.078 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:26 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T20:34:27.078 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:26 smithi018 ceph-mon[184144]: osdmap e345: 8 total, 8 up, 8 in 2023-09-23T20:34:27.078 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:26 smithi018 ceph-mon[184144]: failed to encode map e345 with expected crc 2023-09-23T20:34:27.078 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:26 smithi018 ceph-mon[184144]: pgmap v811: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 116 active+clean; 3.4 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 804 op/s; 1995/168594 objects degraded (1.183%); 993/168594 objects misplaced (0.589%); 6.1 MiB/s, 97 objects/s recovering 2023-09-23T20:34:27.078 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:26 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 1995/176571 objects degraded (1.130%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T20:34:27.079 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:27.303 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:34:27.313 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:27.314 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:34:27.415 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:34:27.496 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:27.580 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:34:27.587 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:27.588 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:34:27.588 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:34:27.608 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:34:27.679 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2023-09-23T20:34:27.680 INFO:teuthology.orchestra.run.smithi018.stdout: "ops": [ 2023-09-23T20:34:27.680 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:27.680 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_repop(client.35700.0:134978 7.1 e345/323 7:8ef85698:::benchmark_data_smithi018_308601_object40426:head v 345'8396, mlcod=345'8393)", 2023-09-23T20:34:27.680 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:34:27.669928+0000", 2023-09-23T20:34:27.681 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.0065149170000000003, 2023-09-23T20:34:27.681 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0065512469999999996, 2023-09-23T20:34:27.681 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:34:27.681 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "started", 2023-09-23T20:34:27.681 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:34:27.682 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:27.682 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:34:27.682 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:27.669928+0000", 2023-09-23T20:34:27.682 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:34:27.683 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:27.683 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:27.683 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:34:27.683 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:27.669928+0000", 2023-09-23T20:34:27.683 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:34:27.684 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:27.684 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:27.684 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:34:27.684 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:27.669927+0000", 2023-09-23T20:34:27.684 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.999999 2023-09-23T20:34:27.685 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:27.685 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:27.685 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:34:27.685 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:27.669935+0000", 2023-09-23T20:34:27.685 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.0260000000000007e-06 2023-09-23T20:34:27.685 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:27.686 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:27.686 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:34:27.686 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:27.669952+0000", 2023-09-23T20:34:27.686 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.7073000000000001e-05 2023-09-23T20:34:27.686 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:27.686 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:27.686 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:34:27.687 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:27.669961+0000", 2023-09-23T20:34:27.687 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.0480000000000004e-06 2023-09-23T20:34:27.687 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:27.687 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:27.687 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:34:27.688 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:27.671887+0000", 2023-09-23T20:34:27.688 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.001926281 2023-09-23T20:34:27.688 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:27.688 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:27.688 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:34:27.688 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:27.671912+0000", 2023-09-23T20:34:27.689 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.5113000000000002e-05 2023-09-23T20:34:27.689 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:34:27.689 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:34:27.689 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:34:27.689 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:27.689 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:27.689 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_repop(client.35700.0:134981 7.1 e345/323 7:844f9b76:::benchmark_data_smithi018_308601_object40429:head v 345'8397, mlcod=345'8393)", 2023-09-23T20:34:27.690 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:34:27.675280+0000", 2023-09-23T20:34:27.690 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.001162522, 2023-09-23T20:34:27.690 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0012695709999999999, 2023-09-23T20:34:27.690 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:34:27.690 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "started", 2023-09-23T20:34:27.690 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:34:27.691 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:27.691 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:34:27.691 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:27.675280+0000", 2023-09-23T20:34:27.691 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:34:27.691 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:27.691 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:27.691 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:34:27.692 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:27.675280+0000", 2023-09-23T20:34:27.692 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:34:27.692 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:27.692 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:27.692 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:34:27.692 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:27.675279+0000", 2023-09-23T20:34:27.692 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.999999 2023-09-23T20:34:27.693 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:27.693 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:27.693 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:34:27.693 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:27.675287+0000", 2023-09-23T20:34:27.693 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.4030000000000001e-06 2023-09-23T20:34:27.693 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:27.694 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:27.694 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:34:27.694 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:27.675295+0000", 2023-09-23T20:34:27.694 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.2670000000000006e-06 2023-09-23T20:34:27.694 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:27.694 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:27.695 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:34:27.695 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:27.675302+0000", 2023-09-23T20:34:27.695 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.8020000000000003e-06 2023-09-23T20:34:27.695 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:27.695 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:27.696 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:34:27.696 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:27.676397+0000", 2023-09-23T20:34:27.696 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.001095265 2023-09-23T20:34:27.696 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:34:27.696 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:34:27.697 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:34:27.697 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:27.676423+0000", 2023-09-23T20:34:27.697 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.6242999999999999e-05 2023-09-23T20:34:27.697 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:34:27.697 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:34:27.698 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:34:27.698 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:34:27.698 INFO:teuthology.orchestra.run.smithi018.stdout: ], 2023-09-23T20:34:27.698 INFO:teuthology.orchestra.run.smithi018.stdout: "num_ops": 2 2023-09-23T20:34:27.698 INFO:teuthology.orchestra.run.smithi018.stdout:} 2023-09-23T20:34:27.699 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:34:27.774 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:27.872 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:34:28.135 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:34:28.140 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:28.140 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:34:28.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:27 smithi018 ceph-mon[179647]: 1.8 scrub starts 2023-09-23T20:34:28.145 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:27 smithi018 ceph-mon[184144]: 1.8 scrub starts 2023-09-23T20:34:28.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:27 smithi149 ceph-mon[170394]: 1.8 scrub starts 2023-09-23T20:34:28.242 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:34:28.269 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:28.282 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T20:34:28.319 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:34:28.327 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:28.327 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:34:28.428 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:34:28.500 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:28.742 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:28 smithi018 ceph-mon[179647]: 1.18 scrub starts 2023-09-23T20:34:28.742 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:28 smithi018 ceph-mon[179647]: pgmap v812: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 116 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 735 op/s; 1995/166089 objects degraded (1.201%); 700/166089 objects misplaced (0.421%); 7.0 MiB/s, 112 objects/s recovering 2023-09-23T20:34:29.145 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:28 smithi018 ceph-mon[184144]: 1.18 scrub starts 2023-09-23T20:34:29.145 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:28 smithi018 ceph-mon[184144]: pgmap v812: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 116 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 735 op/s; 1995/166089 objects degraded (1.201%); 700/166089 objects misplaced (0.421%); 7.0 MiB/s, 112 objects/s recovering 2023-09-23T20:34:29.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:28 smithi149 ceph-mon[170394]: 1.18 scrub starts 2023-09-23T20:34:29.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:28 smithi149 ceph-mon[170394]: pgmap v812: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 116 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 735 op/s; 1995/166089 objects degraded (1.201%); 700/166089 objects misplaced (0.421%); 7.0 MiB/s, 112 objects/s recovering 2023-09-23T20:34:29.635 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:34:29.640 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:29.641 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:34:29.742 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:34:29.821 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:30.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:30 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:34:30.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:30 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:34:30.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:30 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:34:30.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:30 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:34:30.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:30 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:34:30.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:30 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:34:31.334 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:34:31.336 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:31.336 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:34:31.437 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:34:31.472 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:31.519 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:34:31.522 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:31.522 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:34:31.623 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:34:31.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:31 smithi018 ceph-mon[184144]: 1.1e scrub starts 2023-09-23T20:34:31.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:31 smithi018 ceph-mon[184144]: pgmap v813: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 116 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 568 op/s; 1995/164808 objects degraded (1.210%); 700/164808 objects misplaced (0.425%); 4.8 MiB/s, 77 objects/s recovering 2023-09-23T20:34:31.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:31 smithi018 ceph-mon[184144]: 1.1d scrub starts 2023-09-23T20:34:31.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:31 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 1995/164808 objects degraded (1.210%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T20:34:31.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:31 smithi018 ceph-mon[179647]: 1.1e scrub starts 2023-09-23T20:34:31.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:31 smithi018 ceph-mon[179647]: pgmap v813: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 116 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 568 op/s; 1995/164808 objects degraded (1.210%); 700/164808 objects misplaced (0.425%); 4.8 MiB/s, 77 objects/s recovering 2023-09-23T20:34:31.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:31 smithi018 ceph-mon[179647]: 1.1d scrub starts 2023-09-23T20:34:31.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:31 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 1995/164808 objects degraded (1.210%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T20:34:31.649 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:34:31.655 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:31.695 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:34:31.698 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:31.698 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:34:31.799 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:34:31.827 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:31 smithi149 ceph-mon[170394]: 1.1e scrub starts 2023-09-23T20:34:31.827 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:31 smithi149 ceph-mon[170394]: pgmap v813: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 116 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 568 op/s; 1995/164808 objects degraded (1.210%); 700/164808 objects misplaced (0.425%); 4.8 MiB/s, 77 objects/s recovering 2023-09-23T20:34:31.827 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:31 smithi149 ceph-mon[170394]: 1.1d scrub starts 2023-09-23T20:34:31.827 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:31 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 1995/164808 objects degraded (1.210%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T20:34:31.833 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:31.875 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:34:31.881 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:31.882 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:34:31.983 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:34:32.079 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:32.148 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:34:32.250 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:34:32.264 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:32.265 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:34:32.366 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:34:32.398 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:32.442 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:34:32.447 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:32.447 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:34:32.548 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:34:32.583 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:32.625 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:34:32.630 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:32.631 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:34:32.645 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:32 smithi018 ceph-mon[184144]: 1.1a scrub starts 2023-09-23T20:34:32.646 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:32 smithi018 ceph-mon[179647]: 1.1a scrub starts 2023-09-23T20:34:32.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:32 smithi149 ceph-mon[170394]: 1.1a scrub starts 2023-09-23T20:34:32.732 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:34:32.766 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:32.811 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:34:32.816 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:32.817 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:34:32.918 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:34:32.946 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:32.991 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:34:32.994 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:32.994 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:34:33.095 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:34:33.246 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:33.502 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:34:33.506 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:33.507 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:34:33.579 INFO:teuthology.orchestra.run.smithi018.stderr:reweighted osd.7 to 1 (10000) 2023-09-23T20:34:33.590 INFO:teuthology.orchestra.run.smithi018.stderr:noscrub is set 2023-09-23T20:34:33.608 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:34:33.671 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:33.776 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:34:33.783 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:33 smithi018 ceph-mon[179647]: pgmap v814: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 116 active+clean; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 681 op/s; 1995/158418 objects degraded (1.259%); 700/158418 objects misplaced (0.442%); 4.0 MiB/s, 63 objects/s recovering 2023-09-23T20:34:33.783 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:33 smithi018 ceph-mon[179647]: 1.4 scrub starts 2023-09-23T20:34:33.783 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:33 smithi018 ceph-mon[179647]: 1.19 scrub starts 2023-09-23T20:34:33.783 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:33 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/4122362490' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:34:33.783 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:33 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:34:33.783 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:33 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1768619167' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T20:34:33.784 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:33 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T20:34:33.784 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:33 smithi018 ceph-mon[184144]: pgmap v814: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 116 active+clean; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 681 op/s; 1995/158418 objects degraded (1.259%); 700/158418 objects misplaced (0.442%); 4.0 MiB/s, 63 objects/s recovering 2023-09-23T20:34:33.784 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:33 smithi018 ceph-mon[184144]: 1.4 scrub starts 2023-09-23T20:34:33.784 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:33 smithi018 ceph-mon[184144]: 1.19 scrub starts 2023-09-23T20:34:33.784 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:33 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/4122362490' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:34:33.784 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:33 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:34:33.785 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:33 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1768619167' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T20:34:33.785 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:33 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T20:34:33.785 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:33.786 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:34:33.887 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:34:33.921 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:33 smithi149 ceph-mon[170394]: pgmap v814: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 116 active+clean; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 681 op/s; 1995/158418 objects degraded (1.259%); 700/158418 objects misplaced (0.442%); 4.0 MiB/s, 63 objects/s recovering 2023-09-23T20:34:33.921 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:33 smithi149 ceph-mon[170394]: 1.4 scrub starts 2023-09-23T20:34:33.921 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:33 smithi149 ceph-mon[170394]: 1.19 scrub starts 2023-09-23T20:34:33.921 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:33 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/4122362490' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:34:33.921 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:33 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:34:33.921 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:33 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1768619167' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T20:34:33.921 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:33 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T20:34:33.923 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:33.970 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:34:33.973 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:33.973 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:34:34.074 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:34:34.105 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:34.155 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:34:34.163 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:34.163 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:34:34.175 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:34:34.215 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:34:34.265 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:34:34.291 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:34:34.298 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:34.342 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:34:34.351 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:34:34.354 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:34.354 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:34:34.378 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:34:34.414 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:34:34.455 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:34:34.456 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:34:34.492 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:34.511 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:34:34.523 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:34:34.523 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:34:34.523 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:34:34.523 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:34:34.523 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:34:34.523 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:34:34.523 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:34:34.523 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:34:34.538 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:34:34.541 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:34.541 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:34:34.618 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd reweight 3 1 2023-09-23T20:34:34.642 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:34:34.672 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:34.712 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:34:34.717 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:34.718 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:34:34.818 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:34:34.854 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:34 smithi149 ceph-mon[170394]: 1.a scrub starts 2023-09-23T20:34:34.854 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:34 smithi149 ceph-mon[170394]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:34:34.854 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:34 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:34:34.854 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:34 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T20:34:34.855 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:34 smithi149 ceph-mon[170394]: osdmap e346: 8 total, 8 up, 8 in 2023-09-23T20:34:34.855 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:34 smithi149 ceph-mon[170394]: failed to encode map e346 with expected crc 2023-09-23T20:34:34.855 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:34 smithi149 ceph-mon[170394]: failed to encode map e346 with expected crc 2023-09-23T20:34:34.855 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:34 smithi149 ceph-mon[170394]: failed to encode map e346 with expected crc 2023-09-23T20:34:34.855 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:34 smithi149 ceph-mon[170394]: failed to encode map e346 with expected crc 2023-09-23T20:34:34.855 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:34 smithi149 ceph-mon[170394]: failed to encode map e346 with expected crc 2023-09-23T20:34:34.855 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:34 smithi149 ceph-mon[170394]: failed to encode map e346 with expected crc 2023-09-23T20:34:34.855 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:34 smithi149 ceph-mon[170394]: pgmap v816: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 116 active+clean; 3.2 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 554 op/s; 1995/155718 objects degraded (1.281%); 396/155718 objects misplaced (0.254%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T20:34:34.855 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:34 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3961050510' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:34:34.856 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:34.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:34 smithi018 ceph-mon[184144]: 1.a scrub starts 2023-09-23T20:34:34.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:34 smithi018 ceph-mon[184144]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:34:34.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:34 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:34:34.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:34 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T20:34:34.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:34 smithi018 ceph-mon[184144]: osdmap e346: 8 total, 8 up, 8 in 2023-09-23T20:34:34.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:34 smithi018 ceph-mon[184144]: failed to encode map e346 with expected crc 2023-09-23T20:34:34.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:34 smithi018 ceph-mon[184144]: failed to encode map e346 with expected crc 2023-09-23T20:34:34.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:34 smithi018 ceph-mon[184144]: failed to encode map e346 with expected crc 2023-09-23T20:34:34.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:34 smithi018 ceph-mon[184144]: failed to encode map e346 with expected crc 2023-09-23T20:34:34.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:34 smithi018 ceph-mon[184144]: failed to encode map e346 with expected crc 2023-09-23T20:34:34.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:34 smithi018 ceph-mon[184144]: failed to encode map e346 with expected crc 2023-09-23T20:34:34.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:34 smithi018 ceph-mon[184144]: pgmap v816: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 116 active+clean; 3.2 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 554 op/s; 1995/155718 objects degraded (1.281%); 396/155718 objects misplaced (0.254%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T20:34:34.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:34 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3961050510' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:34:34.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:34 smithi018 ceph-mon[179647]: 1.a scrub starts 2023-09-23T20:34:34.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:34 smithi018 ceph-mon[179647]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:34:34.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:34 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:34:34.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:34 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T20:34:34.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:34 smithi018 ceph-mon[179647]: osdmap e346: 8 total, 8 up, 8 in 2023-09-23T20:34:34.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:34 smithi018 ceph-mon[179647]: failed to encode map e346 with expected crc 2023-09-23T20:34:34.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:34 smithi018 ceph-mon[179647]: failed to encode map e346 with expected crc 2023-09-23T20:34:34.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:34 smithi018 ceph-mon[179647]: failed to encode map e346 with expected crc 2023-09-23T20:34:34.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:34 smithi018 ceph-mon[179647]: failed to encode map e346 with expected crc 2023-09-23T20:34:34.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:34 smithi018 ceph-mon[179647]: failed to encode map e346 with expected crc 2023-09-23T20:34:34.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:34 smithi018 ceph-mon[179647]: failed to encode map e346 with expected crc 2023-09-23T20:34:34.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:34 smithi018 ceph-mon[179647]: pgmap v816: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 116 active+clean; 3.2 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 554 op/s; 1995/155718 objects degraded (1.281%); 396/155718 objects misplaced (0.254%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T20:34:34.895 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:34 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3961050510' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:34:34.901 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:34:34.903 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:34.904 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:34:35.005 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:34:35.118 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:35.201 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:34:35.203 INFO:teuthology.orchestra.run.smithi018.stdout:ERROR: (22) Invalid argument 2023-09-23T20:34:35.203 INFO:teuthology.orchestra.run.smithi018.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:34:35.209 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:35.209 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:34:35.310 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:34:35.342 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:35.380 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:34:35.386 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:35.386 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:34:35.391 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:34:34 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:34:34] "GET /metrics HTTP/1.1" 200 36554 "" "Prometheus/2.43.0" 2023-09-23T20:34:35.487 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:34:35.574 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:35.836 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:34:35.848 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:35.849 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:34:35.890 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:34:35.900 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T20:34:35.950 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:34:36.022 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:36.233 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:34:36.242 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:36.242 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:34:36.254 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T20:34:36.343 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:34:36.374 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:36.415 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:34:36.417 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:36.418 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:34:36.519 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:34:36.552 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:36.594 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:34:36.596 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:36.597 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:34:36.697 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:34:36.727 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:36.771 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:34:36.774 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:36.775 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:34:36.875 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:34:36.950 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:37.130 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:34:37.209 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:36 smithi018 ceph-mon[179647]: pgmap v817: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 116 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 688 op/s; 1995/147918 objects degraded (1.349%); 396/147918 objects misplaced (0.268%); 3.7 MiB/s, 59 objects/s recovering 2023-09-23T20:34:37.210 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:36 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 1995/147918 objects degraded (1.349%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T20:34:37.210 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:36 smithi018 ceph-mon[184144]: pgmap v817: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 116 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 688 op/s; 1995/147918 objects degraded (1.349%); 396/147918 objects misplaced (0.268%); 3.7 MiB/s, 59 objects/s recovering 2023-09-23T20:34:37.210 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:36 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 1995/147918 objects degraded (1.349%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T20:34:37.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:36 smithi149 ceph-mon[170394]: pgmap v817: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 116 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 688 op/s; 1995/147918 objects degraded (1.349%); 396/147918 objects misplaced (0.268%); 3.7 MiB/s, 59 objects/s recovering 2023-09-23T20:34:37.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:36 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 1995/147918 objects degraded (1.349%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T20:34:38.726 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:34:38.735 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:38.735 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:34:38.737 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:34:38.836 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:34:38.954 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:38 smithi018 ceph-mon[184144]: pgmap v818: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 116 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 694 op/s; 1995/145230 objects degraded (1.374%); 44/145230 objects misplaced (0.030%); 4.1 MiB/s, 65 objects/s recovering 2023-09-23T20:34:38.957 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:38 smithi018 ceph-mon[179647]: pgmap v818: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 116 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 694 op/s; 1995/145230 objects degraded (1.374%); 44/145230 objects misplaced (0.030%); 4.1 MiB/s, 65 objects/s recovering 2023-09-23T20:34:38.972 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:39.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:38 smithi149 ceph-mon[170394]: pgmap v818: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 116 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 694 op/s; 1995/145230 objects degraded (1.374%); 44/145230 objects misplaced (0.030%); 4.1 MiB/s, 65 objects/s recovering 2023-09-23T20:34:39.700 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:34:39.708 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:39.708 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:34:39.809 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:34:39.844 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:39.906 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:34:39.910 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:39.910 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:34:39.961 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:34:39.961 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:34:40.011 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:34:40.042 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:40.088 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:34:40.094 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:40.095 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:34:40.195 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:34:40.272 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:40.823 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:34:40.826 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:40.826 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:34:40.927 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:34:41.033 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:41.098 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:40 smithi018 ceph-mon[184144]: pgmap v819: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 116 active+clean; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 697 op/s; 1995/143847 objects degraded (1.387%); 44/143847 objects misplaced (0.031%); 4.1 MiB/s, 65 objects/s recovering 2023-09-23T20:34:41.100 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:40 smithi018 ceph-mon[179647]: pgmap v819: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 116 active+clean; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 697 op/s; 1995/143847 objects degraded (1.387%); 44/143847 objects misplaced (0.031%); 4.1 MiB/s, 65 objects/s recovering 2023-09-23T20:34:41.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:40 smithi149 ceph-mon[170394]: pgmap v819: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 116 active+clean; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 697 op/s; 1995/143847 objects degraded (1.387%); 44/143847 objects misplaced (0.031%); 4.1 MiB/s, 65 objects/s recovering 2023-09-23T20:34:41.216 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:34:41.219 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:41.219 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:34:41.320 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:34:41.367 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:41.423 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:34:41.430 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:41.431 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:34:41.532 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:34:41.634 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:41.913 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:34:41.917 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:41.917 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:34:41.963 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:41 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 1995/143847 objects degraded (1.387%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T20:34:42.018 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:34:42.110 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:42.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:41 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 1995/143847 objects degraded (1.387%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T20:34:42.287 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:41 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 1995/143847 objects degraded (1.387%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T20:34:42.496 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:34:42.514 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:42.515 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:34:42.616 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:34:42.650 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:42.695 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:34:42.698 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:42.698 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:34:42.799 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:34:42.834 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:42.879 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:34:42.883 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:42.883 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:34:42.933 INFO:teuthology.orchestra.run.smithi018.stdout:ERROR: (22) Invalid argument 2023-09-23T20:34:42.933 INFO:teuthology.orchestra.run.smithi018.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:34:42.984 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:34:43.014 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:43.058 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:34:43.064 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:43.066 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:34:43.166 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:34:43.250 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:43.255 INFO:teuthology.orchestra.run.smithi018.stderr:reweighted osd.3 to 1 (10000) 2023-09-23T20:34:43.468 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:43 smithi149 ceph-mon[170394]: pgmap v820: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 116 active+clean; 2.8 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 722 op/s; 1995/136716 objects degraded (1.459%); 44/136716 objects misplaced (0.032%); 4.1 MiB/s, 65 objects/s recovering 2023-09-23T20:34:43.468 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:43 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3381771054' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T20:34:43.468 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:43 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T20:34:43.526 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:34:43.577 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:34:43.610 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:43 smithi018 ceph-mon[179647]: pgmap v820: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 116 active+clean; 2.8 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 722 op/s; 1995/136716 objects degraded (1.459%); 44/136716 objects misplaced (0.032%); 4.1 MiB/s, 65 objects/s recovering 2023-09-23T20:34:43.610 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:43 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3381771054' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T20:34:43.610 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:43 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T20:34:43.611 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T20:34:43.612 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:43 smithi018 ceph-mon[184144]: pgmap v820: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 116 active+clean; 2.8 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 722 op/s; 1995/136716 objects degraded (1.459%); 44/136716 objects misplaced (0.032%); 4.1 MiB/s, 65 objects/s recovering 2023-09-23T20:34:43.612 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:43 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3381771054' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T20:34:43.612 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:43 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T20:34:43.614 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:34:43.667 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:34:43.668 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:34:43.678 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:43.679 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:34:43.709 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:34:43.763 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:34:43.780 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:34:43.800 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:34:43.822 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:43.846 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:34:43.869 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:34:43.869 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:34:43.869 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:34:43.869 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:34:43.870 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:34:43.870 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:34:43.870 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:34:43.870 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:34:43.894 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:34:43.901 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:43.902 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:34:44.002 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:34:44.037 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:44.083 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:34:44.087 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:44.088 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:34:44.189 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:34:44.220 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:44.236 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:34:44.239 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd reweight 2 1 2023-09-23T20:34:44.266 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:34:44.274 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:44.274 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:34:44.375 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:34:44.412 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:44.457 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:34:44.462 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:44.462 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:34:44.471 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:44 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T20:34:44.471 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:44 smithi149 ceph-mon[170394]: osdmap e347: 8 total, 8 up, 8 in 2023-09-23T20:34:44.472 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:44 smithi149 ceph-mon[170394]: failed to encode map e347 with expected crc 2023-09-23T20:34:44.472 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:44 smithi149 ceph-mon[170394]: failed to encode map e347 with expected crc 2023-09-23T20:34:44.472 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:44 smithi149 ceph-mon[170394]: failed to encode map e347 with expected crc 2023-09-23T20:34:44.472 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:44 smithi149 ceph-mon[170394]: failed to encode map e347 with expected crc 2023-09-23T20:34:44.472 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:44 smithi149 ceph-mon[170394]: failed to encode map e347 with expected crc 2023-09-23T20:34:44.472 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:44 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3483355722' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:34:44.521 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:44 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T20:34:44.521 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:44 smithi018 ceph-mon[184144]: osdmap e347: 8 total, 8 up, 8 in 2023-09-23T20:34:44.521 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:44 smithi018 ceph-mon[184144]: failed to encode map e347 with expected crc 2023-09-23T20:34:44.521 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:44 smithi018 ceph-mon[184144]: failed to encode map e347 with expected crc 2023-09-23T20:34:44.521 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:44 smithi018 ceph-mon[184144]: failed to encode map e347 with expected crc 2023-09-23T20:34:44.521 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:44 smithi018 ceph-mon[184144]: failed to encode map e347 with expected crc 2023-09-23T20:34:44.521 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:44 smithi018 ceph-mon[184144]: failed to encode map e347 with expected crc 2023-09-23T20:34:44.521 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:44 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3483355722' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:34:44.522 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:44 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T20:34:44.522 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:44 smithi018 ceph-mon[179647]: osdmap e347: 8 total, 8 up, 8 in 2023-09-23T20:34:44.522 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:44 smithi018 ceph-mon[179647]: failed to encode map e347 with expected crc 2023-09-23T20:34:44.522 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:44 smithi018 ceph-mon[179647]: failed to encode map e347 with expected crc 2023-09-23T20:34:44.522 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:44 smithi018 ceph-mon[179647]: failed to encode map e347 with expected crc 2023-09-23T20:34:44.522 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:44 smithi018 ceph-mon[179647]: failed to encode map e347 with expected crc 2023-09-23T20:34:44.522 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:44 smithi018 ceph-mon[179647]: failed to encode map e347 with expected crc 2023-09-23T20:34:44.522 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:44 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3483355722' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:34:44.563 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:34:44.598 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:44.639 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:34:44.645 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:44.645 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:34:44.746 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:34:44.793 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:45.092 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:34:45.096 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:45.096 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:34:45.197 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:34:45.250 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:45.291 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:34:45.301 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:34:45.306 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:45.306 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:34:45.391 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:34:44 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:34:44] "GET /metrics HTTP/1.1" 200 36549 "" "Prometheus/2.43.0" 2023-09-23T20:34:45.407 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:34:45.462 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:45.505 INFO:teuthology.orchestra.run.smithi018.stderr:nodeep-scrub is set 2023-09-23T20:34:45.514 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:34:45.517 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:45.518 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:34:45.619 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:34:45.671 INFO:teuthology.orchestra.run.smithi149.stdout:{ 2023-09-23T20:34:45.671 INFO:teuthology.orchestra.run.smithi149.stdout: "ops": [ 2023-09-23T20:34:45.671 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:45.672 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_repop(client.35700.0:147916 7.8 e348/338 7:17f4e81d:::benchmark_data_smithi018_308601_object53364:head v 348'9258, mlcod=348'9256)", 2023-09-23T20:34:45.672 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:34:45.662982+0000", 2023-09-23T20:34:45.672 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 0.0081215230000000003, 2023-09-23T20:34:45.672 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0081493150000000007, 2023-09-23T20:34:45.672 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:34:45.672 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:34:45.672 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:34:45.672 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:45.672 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:34:45.672 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:45.662982+0000", 2023-09-23T20:34:45.672 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:34:45.673 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:45.673 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:45.673 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:34:45.673 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:45.662982+0000", 2023-09-23T20:34:45.673 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:34:45.673 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:45.673 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:45.673 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:34:45.673 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:45.662981+0000", 2023-09-23T20:34:45.673 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:34:45.673 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:45.674 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:45.674 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:34:45.674 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:45.662991+0000", 2023-09-23T20:34:45.674 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.006e-05 2023-09-23T20:34:45.674 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:45.674 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:45.674 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:34:45.674 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:45.663013+0000", 2023-09-23T20:34:45.674 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.1576000000000001e-05 2023-09-23T20:34:45.674 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:45.674 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:45.674 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:34:45.675 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:45.663021+0000", 2023-09-23T20:34:45.675 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.1210000000000007e-06 2023-09-23T20:34:45.675 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:45.675 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:45.675 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:34:45.675 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:45.663070+0000", 2023-09-23T20:34:45.675 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.8652000000000001e-05 2023-09-23T20:34:45.675 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:45.675 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:45.675 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:34:45.675 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:45.663093+0000", 2023-09-23T20:34:45.675 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.2986999999999999e-05 2023-09-23T20:34:45.676 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:45.676 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:45.676 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:34:45.676 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:45.671092+0000", 2023-09-23T20:34:45.676 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0079994139999999998 2023-09-23T20:34:45.676 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:34:45.676 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:34:45.676 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:34:45.676 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:45.676 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:45.676 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_repop(client.35700.0:147917 7.3 e348/320 7:c7dab29c:::benchmark_data_smithi018_308601_object53365:head v 348'9019, mlcod=336'6983)", 2023-09-23T20:34:45.676 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:34:45.668924+0000", 2023-09-23T20:34:45.677 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 0.0021798529999999998, 2023-09-23T20:34:45.677 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0022723109999999999, 2023-09-23T20:34:45.677 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:34:45.677 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "started", 2023-09-23T20:34:45.677 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:34:45.677 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:45.677 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:34:45.677 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:45.668924+0000", 2023-09-23T20:34:45.677 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:34:45.677 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:45.677 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:45.677 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:34:45.678 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:45.668924+0000", 2023-09-23T20:34:45.678 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:34:45.678 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:45.678 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:45.678 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:34:45.678 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:45.668923+0000", 2023-09-23T20:34:45.678 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:34:45.678 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:45.678 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:45.678 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:34:45.678 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:45.668933+0000", 2023-09-23T20:34:45.679 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0456e-05 2023-09-23T20:34:45.679 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:45.679 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:45.679 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:34:45.679 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:45.668950+0000", 2023-09-23T20:34:45.679 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.6719999999999999e-05 2023-09-23T20:34:45.679 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:45.679 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:45.679 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:34:45.679 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:45.668957+0000", 2023-09-23T20:34:45.679 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.6529999999999997e-06 2023-09-23T20:34:45.679 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:45.680 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:45.680 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:34:45.680 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:45.669013+0000", 2023-09-23T20:34:45.680 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.6382000000000003e-05 2023-09-23T20:34:45.680 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:45.680 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:45.680 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:34:45.680 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:45.669036+0000", 2023-09-23T20:34:45.680 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.3220000000000001e-05 2023-09-23T20:34:45.680 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:34:45.680 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:34:45.680 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:34:45.680 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:34:45.681 INFO:teuthology.orchestra.run.smithi149.stdout: ], 2023-09-23T20:34:45.681 INFO:teuthology.orchestra.run.smithi149.stdout: "num_ops": 2 2023-09-23T20:34:45.681 INFO:teuthology.orchestra.run.smithi149.stdout:} 2023-09-23T20:34:45.693 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:45 smithi149 ceph-mon[170394]: pgmap v822: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 116 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 726 op/s; 1592/133887 objects degraded (1.189%); 2.5 MiB/s, 79 objects/s recovering 2023-09-23T20:34:45.693 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:45 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:34:45.693 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:45 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:34:45.694 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:45 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/376195885' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:34:45.727 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:45.896 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:45 smithi018 ceph-mon[179647]: pgmap v822: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 116 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 726 op/s; 1592/133887 objects degraded (1.189%); 2.5 MiB/s, 79 objects/s recovering 2023-09-23T20:34:45.896 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:45 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:34:45.897 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:45 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:34:45.897 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:45 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/376195885' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:34:45.897 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:45 smithi018 ceph-mon[184144]: pgmap v822: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 116 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 726 op/s; 1592/133887 objects degraded (1.189%); 2.5 MiB/s, 79 objects/s recovering 2023-09-23T20:34:45.897 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:45 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:34:45.897 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:45 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:34:45.897 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:45 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/376195885' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:34:46.142 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T20:34:46.546 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:34:46.570 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:46 smithi018 ceph-mon[179647]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:34:46.570 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:46 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/376195885' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:34:46.571 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:46 smithi018 ceph-mon[179647]: osdmap e348: 8 total, 8 up, 8 in 2023-09-23T20:34:46.571 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:46 smithi018 ceph-mon[179647]: failed to encode map e348 with expected crc 2023-09-23T20:34:46.571 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:46 smithi018 ceph-mon[179647]: failed to encode map e348 with expected crc 2023-09-23T20:34:46.571 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:46 smithi018 ceph-mon[179647]: failed to encode map e348 with expected crc 2023-09-23T20:34:46.571 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:46 smithi018 ceph-mon[179647]: failed to encode map e348 with expected crc 2023-09-23T20:34:46.571 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:46 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 1592/133887 objects degraded (1.189%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T20:34:46.571 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:46 smithi018 ceph-mon[184144]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:34:46.571 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:46 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/376195885' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:34:46.571 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:46 smithi018 ceph-mon[184144]: osdmap e348: 8 total, 8 up, 8 in 2023-09-23T20:34:46.572 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:46 smithi018 ceph-mon[184144]: failed to encode map e348 with expected crc 2023-09-23T20:34:46.572 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:46 smithi018 ceph-mon[184144]: failed to encode map e348 with expected crc 2023-09-23T20:34:46.572 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:46 smithi018 ceph-mon[184144]: failed to encode map e348 with expected crc 2023-09-23T20:34:46.572 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:46 smithi018 ceph-mon[184144]: failed to encode map e348 with expected crc 2023-09-23T20:34:46.572 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:46 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 1592/133887 objects degraded (1.189%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T20:34:46.850 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:34:46.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:46 smithi149 ceph-mon[170394]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:34:46.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:46 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/376195885' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:34:46.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:46 smithi149 ceph-mon[170394]: osdmap e348: 8 total, 8 up, 8 in 2023-09-23T20:34:46.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:46 smithi149 ceph-mon[170394]: failed to encode map e348 with expected crc 2023-09-23T20:34:46.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:46 smithi149 ceph-mon[170394]: failed to encode map e348 with expected crc 2023-09-23T20:34:46.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:46 smithi149 ceph-mon[170394]: failed to encode map e348 with expected crc 2023-09-23T20:34:46.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:46 smithi149 ceph-mon[170394]: failed to encode map e348 with expected crc 2023-09-23T20:34:46.968 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:46 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 1592/133887 objects degraded (1.189%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T20:34:47.039 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:34:47.049 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:47.049 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:34:47.150 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:34:47.220 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:47.298 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:34:47.306 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:47.306 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:34:47.407 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:34:47.442 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:47.626 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:34:47.632 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:47.632 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:34:47.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:47 smithi018 ceph-mon[179647]: pgmap v824: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 116 active+clean; 2.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 807 op/s; 1592/125844 objects degraded (1.265%); 352 KiB/s, 55 objects/s recovering 2023-09-23T20:34:47.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:47 smithi018 ceph-mon[184144]: pgmap v824: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 116 active+clean; 2.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 807 op/s; 1592/125844 objects degraded (1.265%); 352 KiB/s, 55 objects/s recovering 2023-09-23T20:34:47.718 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:47 smithi149 ceph-mon[170394]: pgmap v824: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 116 active+clean; 2.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 807 op/s; 1592/125844 objects degraded (1.265%); 352 KiB/s, 55 objects/s recovering 2023-09-23T20:34:47.733 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:34:47.826 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:48.011 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:34:48.017 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:48.018 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:34:48.077 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:34:48.119 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:34:48.175 INFO:teuthology.orchestra.run.smithi149.stdout:{ 2023-09-23T20:34:48.175 INFO:teuthology.orchestra.run.smithi149.stdout: "ops": [], 2023-09-23T20:34:48.176 INFO:teuthology.orchestra.run.smithi149.stdout: "complaint_time": 30, 2023-09-23T20:34:48.176 INFO:teuthology.orchestra.run.smithi149.stdout: "num_blocked_ops": 0 2023-09-23T20:34:48.177 INFO:teuthology.orchestra.run.smithi149.stdout:} 2023-09-23T20:34:48.197 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:48.320 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:34:48.336 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:48.336 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:34:48.437 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:34:48.477 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:48.570 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T20:34:48.597 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:34:48.604 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:48.604 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:34:48.669 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T20:34:48.706 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:34:48.807 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:49.041 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:34:49.058 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:49.059 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:34:49.159 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:34:49.192 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:49.247 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:34:49.252 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:49.253 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:34:49.354 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:34:49.378 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:34:49.445 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:49.459 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:34:49.548 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:34:49.563 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:49.564 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:34:49.568 INFO:teuthology.orchestra.run.smithi018.stderr:reweighted osd.2 to 1 (10000) 2023-09-23T20:34:49.665 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:34:49.744 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:49 smithi018 ceph-mon[184144]: pgmap v825: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 116 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 865 op/s; 1155/123042 objects degraded (0.939%); 352 KiB/s, 110 objects/s recovering 2023-09-23T20:34:49.744 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:49 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2701479907' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T20:34:49.744 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:49 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T20:34:49.745 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:49 smithi018 ceph-mon[179647]: pgmap v825: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 116 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 865 op/s; 1155/123042 objects degraded (0.939%); 352 KiB/s, 110 objects/s recovering 2023-09-23T20:34:49.745 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:49 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2701479907' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T20:34:49.745 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:49 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T20:34:49.754 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:49.839 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:49 smithi149 ceph-mon[170394]: pgmap v825: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 116 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 865 op/s; 1155/123042 objects degraded (0.939%); 352 KiB/s, 110 objects/s recovering 2023-09-23T20:34:49.839 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:49 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2701479907' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T20:34:49.839 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:49 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T20:34:50.005 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:34:50.005 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:34:50.005 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:34:50.005 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:34:50.006 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:34:50.009 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:34:50.028 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:34:50.049 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:34:50.051 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:34:50.051 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:34:50.051 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:34:50.051 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:34:50.051 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:34:50.051 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:34:50.051 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:34:50.052 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:34:50.147 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd reweight 6 1 2023-09-23T20:34:50.214 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:34:50.221 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:50.222 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:34:50.322 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:34:50.352 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:50.439 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:34:50.442 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:50.443 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:34:50.544 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:34:50.581 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:50.646 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:50 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T20:34:50.646 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:50 smithi018 ceph-mon[184144]: osdmap e349: 8 total, 8 up, 8 in 2023-09-23T20:34:50.647 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:50 smithi018 ceph-mon[184144]: failed to encode map e349 with expected crc 2023-09-23T20:34:50.647 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:50 smithi018 ceph-mon[184144]: failed to encode map e349 with expected crc 2023-09-23T20:34:50.647 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:50 smithi018 ceph-mon[184144]: failed to encode map e349 with expected crc 2023-09-23T20:34:50.647 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:50 smithi018 ceph-mon[184144]: failed to encode map e349 with expected crc 2023-09-23T20:34:50.647 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:50 smithi018 ceph-mon[184144]: failed to encode map e349 with expected crc 2023-09-23T20:34:50.648 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:50 smithi018 ceph-mon[184144]: failed to encode map e349 with expected crc 2023-09-23T20:34:50.648 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:50 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1216689430' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:34:50.649 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:50 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T20:34:50.649 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:50 smithi018 ceph-mon[179647]: osdmap e349: 8 total, 8 up, 8 in 2023-09-23T20:34:50.649 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:50 smithi018 ceph-mon[179647]: failed to encode map e349 with expected crc 2023-09-23T20:34:50.649 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:50 smithi018 ceph-mon[179647]: failed to encode map e349 with expected crc 2023-09-23T20:34:50.650 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:50 smithi018 ceph-mon[179647]: failed to encode map e349 with expected crc 2023-09-23T20:34:50.650 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:50 smithi018 ceph-mon[179647]: failed to encode map e349 with expected crc 2023-09-23T20:34:50.650 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:50 smithi018 ceph-mon[179647]: failed to encode map e349 with expected crc 2023-09-23T20:34:50.650 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:50 smithi018 ceph-mon[179647]: failed to encode map e349 with expected crc 2023-09-23T20:34:50.650 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:50 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1216689430' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:34:50.651 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:34:50.656 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:50.657 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:34:50.671 INFO:teuthology.orchestra.run.smithi149.stdout:ERROR: (22) Invalid argument 2023-09-23T20:34:50.671 INFO:teuthology.orchestra.run.smithi149.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:34:50.695 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:50 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T20:34:50.695 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:50 smithi149 ceph-mon[170394]: osdmap e349: 8 total, 8 up, 8 in 2023-09-23T20:34:50.695 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:50 smithi149 ceph-mon[170394]: failed to encode map e349 with expected crc 2023-09-23T20:34:50.695 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:50 smithi149 ceph-mon[170394]: failed to encode map e349 with expected crc 2023-09-23T20:34:50.695 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:50 smithi149 ceph-mon[170394]: failed to encode map e349 with expected crc 2023-09-23T20:34:50.696 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:50 smithi149 ceph-mon[170394]: failed to encode map e349 with expected crc 2023-09-23T20:34:50.696 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:50 smithi149 ceph-mon[170394]: failed to encode map e349 with expected crc 2023-09-23T20:34:50.696 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:50 smithi149 ceph-mon[170394]: failed to encode map e349 with expected crc 2023-09-23T20:34:50.696 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:50 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1216689430' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:34:50.758 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:34:50.888 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:51.051 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T20:34:51.105 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:34:51.117 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:51.117 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:34:51.122 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:34:51.218 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:34:51.256 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:51.312 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:34:51.320 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:51.321 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:34:51.422 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:34:51.456 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:51.562 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:34:51.565 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:51.565 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:34:51.666 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:34:51.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:51 smithi149 ceph-mon[170394]: pgmap v827: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 116 active+clean; 2.5 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 612 op/s; 1155/121704 objects degraded (0.949%); 0 B/s, 65 objects/s recovering 2023-09-23T20:34:51.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:51 smithi149 ceph-mon[170394]: osdmap e350: 8 total, 8 up, 8 in 2023-09-23T20:34:51.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:51 smithi149 ceph-mon[170394]: failed to encode map e350 with expected crc 2023-09-23T20:34:51.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:51 smithi149 ceph-mon[170394]: failed to encode map e350 with expected crc 2023-09-23T20:34:51.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:51 smithi149 ceph-mon[170394]: failed to encode map e350 with expected crc 2023-09-23T20:34:51.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:51 smithi149 ceph-mon[170394]: failed to encode map e350 with expected crc 2023-09-23T20:34:51.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:51 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 1155/121704 objects degraded (0.949%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T20:34:51.771 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:51.778 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:34:51.874 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:34:51.883 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:51.883 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:34:51.884 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:51 smithi018 ceph-mon[184144]: pgmap v827: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 116 active+clean; 2.5 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 612 op/s; 1155/121704 objects degraded (0.949%); 0 B/s, 65 objects/s recovering 2023-09-23T20:34:51.884 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:51 smithi018 ceph-mon[184144]: osdmap e350: 8 total, 8 up, 8 in 2023-09-23T20:34:51.884 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:51 smithi018 ceph-mon[184144]: failed to encode map e350 with expected crc 2023-09-23T20:34:51.884 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:51 smithi018 ceph-mon[184144]: failed to encode map e350 with expected crc 2023-09-23T20:34:51.884 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:51 smithi018 ceph-mon[184144]: failed to encode map e350 with expected crc 2023-09-23T20:34:51.884 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:51 smithi018 ceph-mon[184144]: failed to encode map e350 with expected crc 2023-09-23T20:34:51.884 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:51 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 1155/121704 objects degraded (0.949%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T20:34:51.887 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:51 smithi018 ceph-mon[179647]: pgmap v827: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 116 active+clean; 2.5 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 612 op/s; 1155/121704 objects degraded (0.949%); 0 B/s, 65 objects/s recovering 2023-09-23T20:34:51.888 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:51 smithi018 ceph-mon[179647]: osdmap e350: 8 total, 8 up, 8 in 2023-09-23T20:34:51.888 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:51 smithi018 ceph-mon[179647]: failed to encode map e350 with expected crc 2023-09-23T20:34:51.888 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:51 smithi018 ceph-mon[179647]: failed to encode map e350 with expected crc 2023-09-23T20:34:51.888 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:51 smithi018 ceph-mon[179647]: failed to encode map e350 with expected crc 2023-09-23T20:34:51.888 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:51 smithi018 ceph-mon[179647]: failed to encode map e350 with expected crc 2023-09-23T20:34:51.888 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:51 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 1155/121704 objects degraded (0.949%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T20:34:51.984 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:34:52.075 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:52.144 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:34:52.149 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:52.150 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:34:52.251 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:34:52.304 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:52.526 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:34:52.530 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:52.531 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:34:52.601 INFO:teuthology.orchestra.run.smithi018.stderr:noscrub is unset 2023-09-23T20:34:52.631 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:34:52.694 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:52.704 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:34:52.771 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:34:52.792 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:52.793 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:34:52.829 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:52 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3485211321' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:34:52.830 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:52 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:34:52.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:52 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3485211321' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:34:52.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:52 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:34:52.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:52 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3485211321' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:34:52.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:52 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:34:52.893 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:34:53.013 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:53.087 INFO:teuthology.orchestra.run.smithi149.stdout:ERROR: (22) Invalid argument 2023-09-23T20:34:53.088 INFO:teuthology.orchestra.run.smithi149.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:34:53.144 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:34:53.161 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:53.162 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:34:53.262 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:34:53.299 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:53.489 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:34:53.492 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:53.492 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:34:53.579 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T20:34:53.593 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:34:53.661 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:53.737 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:34:53.744 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:53.744 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:53 smithi018 ceph-mon[179647]: pgmap v829: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 117 active+clean; 2.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 828 op/s; 104/115959 objects degraded (0.090%); 836/115959 objects misplaced (0.721%); 2.1 MiB/s, 101 objects/s recovering 2023-09-23T20:34:53.745 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:53 smithi018 ceph-mon[179647]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:34:53.745 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:53 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:34:53.745 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:53 smithi018 ceph-mon[179647]: osdmap e351: 8 total, 8 up, 8 in 2023-09-23T20:34:53.745 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:53 smithi018 ceph-mon[179647]: failed to encode map e351 with expected crc 2023-09-23T20:34:53.745 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:53 smithi018 ceph-mon[179647]: failed to encode map e351 with expected crc 2023-09-23T20:34:53.745 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:53 smithi018 ceph-mon[179647]: failed to encode map e351 with expected crc 2023-09-23T20:34:53.745 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:53 smithi018 ceph-mon[179647]: failed to encode map e351 with expected crc 2023-09-23T20:34:53.745 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:53 smithi018 ceph-mon[179647]: failed to encode map e351 with expected crc 2023-09-23T20:34:53.745 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:53 smithi018 ceph-mon[179647]: failed to encode map e351 with expected crc 2023-09-23T20:34:53.745 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:53 smithi018 ceph-mon[179647]: 1.5 scrub starts 2023-09-23T20:34:53.746 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:53 smithi018 ceph-mon[184144]: pgmap v829: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 117 active+clean; 2.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 828 op/s; 104/115959 objects degraded (0.090%); 836/115959 objects misplaced (0.721%); 2.1 MiB/s, 101 objects/s recovering 2023-09-23T20:34:53.746 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:53 smithi018 ceph-mon[184144]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:34:53.746 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:53 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:34:53.746 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:53 smithi018 ceph-mon[184144]: osdmap e351: 8 total, 8 up, 8 in 2023-09-23T20:34:53.746 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:53 smithi018 ceph-mon[184144]: failed to encode map e351 with expected crc 2023-09-23T20:34:53.746 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:53 smithi018 ceph-mon[184144]: failed to encode map e351 with expected crc 2023-09-23T20:34:53.747 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:53 smithi018 ceph-mon[184144]: failed to encode map e351 with expected crc 2023-09-23T20:34:53.747 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:53 smithi018 ceph-mon[184144]: failed to encode map e351 with expected crc 2023-09-23T20:34:53.747 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:53 smithi018 ceph-mon[184144]: failed to encode map e351 with expected crc 2023-09-23T20:34:53.747 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:53 smithi018 ceph-mon[184144]: failed to encode map e351 with expected crc 2023-09-23T20:34:53.747 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:53 smithi018 ceph-mon[184144]: 1.5 scrub starts 2023-09-23T20:34:53.748 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:34:53.788 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:34:53.849 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:34:53.882 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:53 smithi149 ceph-mon[170394]: pgmap v829: 119 pgs: 2 active+recovering+undersized+degraded+remapped, 117 active+clean; 2.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 828 op/s; 104/115959 objects degraded (0.090%); 836/115959 objects misplaced (0.721%); 2.1 MiB/s, 101 objects/s recovering 2023-09-23T20:34:53.882 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:53 smithi149 ceph-mon[170394]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:34:53.883 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:53 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:34:53.883 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:53 smithi149 ceph-mon[170394]: osdmap e351: 8 total, 8 up, 8 in 2023-09-23T20:34:53.883 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:53 smithi149 ceph-mon[170394]: failed to encode map e351 with expected crc 2023-09-23T20:34:53.883 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:53 smithi149 ceph-mon[170394]: failed to encode map e351 with expected crc 2023-09-23T20:34:53.883 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:53 smithi149 ceph-mon[170394]: failed to encode map e351 with expected crc 2023-09-23T20:34:53.883 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:53 smithi149 ceph-mon[170394]: failed to encode map e351 with expected crc 2023-09-23T20:34:53.883 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:53 smithi149 ceph-mon[170394]: failed to encode map e351 with expected crc 2023-09-23T20:34:53.883 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:53 smithi149 ceph-mon[170394]: failed to encode map e351 with expected crc 2023-09-23T20:34:53.883 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:53 smithi149 ceph-mon[170394]: 1.5 scrub starts 2023-09-23T20:34:53.886 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:53.933 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:34:53.938 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:53.938 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:34:54.039 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:34:54.079 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:54.128 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:34:54.136 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:54.137 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:34:54.237 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:34:54.272 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:54.283 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:34:54.332 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:34:54.338 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:54.338 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:34:54.439 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:34:54.524 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:54.648 INFO:teuthology.orchestra.run.smithi018.stderr:reweighted osd.6 to 1 (10000) 2023-09-23T20:34:54.651 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:34:54.662 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:54.662 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:34:54.763 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:34:54.822 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:54 smithi018 ceph-mon[179647]: 1.15 scrub starts 2023-09-23T20:34:54.823 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:54 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/940806561' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T20:34:54.823 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:54 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T20:34:54.823 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:54 smithi018 ceph-mon[179647]: pgmap v831: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 118 active+clean; 2.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 517 op/s; 1/113721 objects degraded (0.001%); 836/113721 objects misplaced (0.735%); 2.2 MiB/s, 52 objects/s recovering 2023-09-23T20:34:54.823 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:54 smithi018 ceph-mon[184144]: 1.15 scrub starts 2023-09-23T20:34:54.823 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:54 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/940806561' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T20:34:54.823 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:54 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T20:34:54.823 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:54 smithi018 ceph-mon[184144]: pgmap v831: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 118 active+clean; 2.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 517 op/s; 1/113721 objects degraded (0.001%); 836/113721 objects misplaced (0.735%); 2.2 MiB/s, 52 objects/s recovering 2023-09-23T20:34:54.844 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:54.890 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:54 smithi149 ceph-mon[170394]: 1.15 scrub starts 2023-09-23T20:34:54.890 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:54 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/940806561' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T20:34:54.890 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:54 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T20:34:54.890 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:54 smithi149 ceph-mon[170394]: pgmap v831: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 118 active+clean; 2.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 517 op/s; 1/113721 objects degraded (0.001%); 836/113721 objects misplaced (0.735%); 2.2 MiB/s, 52 objects/s recovering 2023-09-23T20:34:55.047 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:34:55.063 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:55.064 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:34:55.097 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd reweight 4 1 2023-09-23T20:34:55.098 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:34:54 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:34:54] "GET /metrics HTTP/1.1" 200 36549 "" "Prometheus/2.43.0" 2023-09-23T20:34:55.165 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:34:55.263 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:55.370 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:34:55.377 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:55.377 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:34:55.478 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:34:55.517 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:55.519 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T20:34:55.565 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:34:55.568 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:55.568 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:34:55.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:55 smithi018 ceph-mon[179647]: 1.4 scrub starts 2023-09-23T20:34:55.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:55 smithi018 ceph-mon[179647]: 1.4 scrub ok 2023-09-23T20:34:55.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:55 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T20:34:55.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:55 smithi018 ceph-mon[179647]: osdmap e352: 8 total, 8 up, 8 in 2023-09-23T20:34:55.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:55 smithi018 ceph-mon[179647]: failed to encode map e352 with expected crc 2023-09-23T20:34:55.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:55 smithi018 ceph-mon[179647]: failed to encode map e352 with expected crc 2023-09-23T20:34:55.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:55 smithi018 ceph-mon[179647]: failed to encode map e352 with expected crc 2023-09-23T20:34:55.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:55 smithi018 ceph-mon[179647]: failed to encode map e352 with expected crc 2023-09-23T20:34:55.644 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:55 smithi018 ceph-mon[184144]: 1.4 scrub starts 2023-09-23T20:34:55.644 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:55 smithi018 ceph-mon[184144]: 1.4 scrub ok 2023-09-23T20:34:55.644 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:55 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T20:34:55.644 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:55 smithi018 ceph-mon[184144]: osdmap e352: 8 total, 8 up, 8 in 2023-09-23T20:34:55.644 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:55 smithi018 ceph-mon[184144]: failed to encode map e352 with expected crc 2023-09-23T20:34:55.644 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:55 smithi018 ceph-mon[184144]: failed to encode map e352 with expected crc 2023-09-23T20:34:55.644 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:55 smithi018 ceph-mon[184144]: failed to encode map e352 with expected crc 2023-09-23T20:34:55.644 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:55 smithi018 ceph-mon[184144]: failed to encode map e352 with expected crc 2023-09-23T20:34:55.669 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:34:55.687 INFO:teuthology.orchestra.run.smithi149.stdout:ERROR: (22) Invalid argument 2023-09-23T20:34:55.687 INFO:teuthology.orchestra.run.smithi149.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:34:55.713 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:55 smithi149 ceph-mon[170394]: 1.4 scrub starts 2023-09-23T20:34:55.713 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:55 smithi149 ceph-mon[170394]: 1.4 scrub ok 2023-09-23T20:34:55.713 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:55 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T20:34:55.713 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:55 smithi149 ceph-mon[170394]: osdmap e352: 8 total, 8 up, 8 in 2023-09-23T20:34:55.713 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:55 smithi149 ceph-mon[170394]: failed to encode map e352 with expected crc 2023-09-23T20:34:55.713 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:55 smithi149 ceph-mon[170394]: failed to encode map e352 with expected crc 2023-09-23T20:34:55.713 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:55 smithi149 ceph-mon[170394]: failed to encode map e352 with expected crc 2023-09-23T20:34:55.713 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:55 smithi149 ceph-mon[170394]: failed to encode map e352 with expected crc 2023-09-23T20:34:55.754 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:34:55.770 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:55.801 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:34:55.852 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:34:55.858 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:55.858 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:34:55.874 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:34:55.941 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:34:55.959 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:34:55.983 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:34:55.999 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:56.031 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:34:56.050 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:34:56.051 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:34:56.056 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:56.056 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:34:56.078 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:34:56.083 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T20:34:56.129 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:34:56.129 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:34:56.129 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:34:56.129 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:34:56.129 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:34:56.129 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:34:56.130 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:34:56.130 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:34:56.130 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:34:56.157 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:34:56.212 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:56.263 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:34:56.268 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:56.269 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:34:56.370 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:34:56.461 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:56.554 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:34:56.558 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:56.559 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:34:56.659 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:34:56.728 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:56 smithi018 ceph-mon[179647]: failed to encode map e352 with expected crc 2023-09-23T20:34:56.728 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:56 smithi018 ceph-mon[179647]: 1.5 scrub starts 2023-09-23T20:34:56.729 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:56 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2225924837' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:34:56.729 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:56 smithi018 ceph-mon[179647]: pgmap v833: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 118 active+clean; 2.2 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 871 op/s; 1/105972 objects degraded (0.001%); 460/105972 objects misplaced (0.434%); 6.1 MiB/s, 115 objects/s recovering 2023-09-23T20:34:56.729 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:56 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 1/113721 objects degraded (0.001%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T20:34:56.735 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:56 smithi018 ceph-mon[184144]: failed to encode map e352 with expected crc 2023-09-23T20:34:56.735 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:56 smithi018 ceph-mon[184144]: 1.5 scrub starts 2023-09-23T20:34:56.736 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:56 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2225924837' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:34:56.736 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:56 smithi018 ceph-mon[184144]: pgmap v833: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 118 active+clean; 2.2 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 871 op/s; 1/105972 objects degraded (0.001%); 460/105972 objects misplaced (0.434%); 6.1 MiB/s, 115 objects/s recovering 2023-09-23T20:34:56.736 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:56 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 1/113721 objects degraded (0.001%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T20:34:56.741 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:56.842 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:34:56.870 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:34:56.881 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:56.881 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:34:56.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:56 smithi149 ceph-mon[170394]: failed to encode map e352 with expected crc 2023-09-23T20:34:56.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:56 smithi149 ceph-mon[170394]: 1.5 scrub starts 2023-09-23T20:34:56.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:56 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2225924837' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:34:56.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:56 smithi149 ceph-mon[170394]: pgmap v833: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 118 active+clean; 2.2 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 871 op/s; 1/105972 objects degraded (0.001%); 460/105972 objects misplaced (0.434%); 6.1 MiB/s, 115 objects/s recovering 2023-09-23T20:34:56.968 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:56 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 1/113721 objects degraded (0.001%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T20:34:56.986 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:34:57.066 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:34:57.077 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:57.219 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:34:57.221 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:57.221 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:34:57.322 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:34:57.366 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:57.661 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:34:57.664 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:57.665 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:34:57.766 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:34:57.841 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:57.934 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:34:57.939 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:57.940 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:34:58.042 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:34:58.132 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:58.247 INFO:teuthology.orchestra.run.smithi149.stdout:{ 2023-09-23T20:34:58.248 INFO:teuthology.orchestra.run.smithi149.stdout: "size": 20, 2023-09-23T20:34:58.248 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 600, 2023-09-23T20:34:58.248 INFO:teuthology.orchestra.run.smithi149.stdout: "ops": [ 2023-09-23T20:34:58.248 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.248 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:126911 5.16 5:6dbf396b:::benchmark_data_smithi018_228451_object19850:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e164)", 2023-09-23T20:34:58.248 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:25:01.899768+0000", 2023-09-23T20:34:58.248 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 596.34481097599996, 2023-09-23T20:34:58.248 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.135701898, 2023-09-23T20:34:58.249 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:34:58.249 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:34:58.249 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:34:58.249 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:34:58.249 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:34:58.249 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 126911 2023-09-23T20:34:58.249 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.249 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:34:58.249 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.249 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:34:58.249 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.899768+0000", 2023-09-23T20:34:58.249 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:34:58.250 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.250 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.250 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:34:58.250 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.899768+0000", 2023-09-23T20:34:58.250 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:34:58.250 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.250 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.250 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:34:58.250 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.899766+0000", 2023-09-23T20:34:58.250 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999981 2023-09-23T20:34:58.250 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.251 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.251 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:34:58.251 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.899777+0000", 2023-09-23T20:34:58.251 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1313e-05 2023-09-23T20:34:58.251 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.251 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.251 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:34:58.251 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.899791+0000", 2023-09-23T20:34:58.251 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.3606e-05 2023-09-23T20:34:58.251 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.251 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.252 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:34:58.252 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.899799+0000", 2023-09-23T20:34:58.252 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.2349999999999999e-06 2023-09-23T20:34:58.252 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.252 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.252 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:34:58.252 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.900660+0000", 2023-09-23T20:34:58.252 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00086041100000000005 2023-09-23T20:34:58.252 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.252 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.252 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:34:58.252 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.900793+0000", 2023-09-23T20:34:58.253 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00013361699999999999 2023-09-23T20:34:58.253 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.253 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.253 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 1,6", 2023-09-23T20:34:58.253 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.900938+0000", 2023-09-23T20:34:58.253 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000144666 2023-09-23T20:34:58.253 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.253 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.253 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:34:58.253 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.914887+0000", 2023-09-23T20:34:58.253 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.01394951 2023-09-23T20:34:58.254 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.254 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.254 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:34:58.254 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.924042+0000", 2023-09-23T20:34:58.254 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0091545440000000006 2023-09-23T20:34:58.254 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.254 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.254 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:34:58.254 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.924051+0000", 2023-09-23T20:34:58.254 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.7609999999999996e-06 2023-09-23T20:34:58.254 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.255 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.255 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:34:58.255 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.027985+0000", 2023-09-23T20:34:58.255 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.103934398 2023-09-23T20:34:58.255 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.255 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.255 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for missing object", 2023-09-23T20:34:58.255 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.028127+0000", 2023-09-23T20:34:58.255 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000141865 2023-09-23T20:34:58.255 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.255 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.255 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:34:58.256 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.035390+0000", 2023-09-23T20:34:58.256 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0072634400000000003 2023-09-23T20:34:58.256 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.256 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.256 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:34:58.256 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.035470+0000", 2023-09-23T20:34:58.256 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.9461000000000004e-05 2023-09-23T20:34:58.256 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:34:58.256 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:34:58.256 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:34:58.256 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.256 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.257 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.34791.0:126916 5.16 5:6b393819:::benchmark_data_smithi018_228451_object19855:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e164)", 2023-09-23T20:34:58.257 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:25:01.916468+0000", 2023-09-23T20:34:58.257 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 596.32811065400006, 2023-09-23T20:34:58.257 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.11937182, 2023-09-23T20:34:58.257 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:34:58.257 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:34:58.257 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:34:58.257 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.34791", 2023-09-23T20:34:58.257 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/279244081", 2023-09-23T20:34:58.257 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 126916 2023-09-23T20:34:58.257 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.258 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:34:58.258 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.258 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:34:58.258 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.916468+0000", 2023-09-23T20:34:58.258 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:34:58.258 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.258 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.258 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:34:58.258 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.916468+0000", 2023-09-23T20:34:58.258 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:34:58.258 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.258 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.259 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:34:58.259 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.916467+0000", 2023-09-23T20:34:58.259 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:34:58.259 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.259 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.259 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:34:58.259 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.916472+0000", 2023-09-23T20:34:58.259 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.8099999999999997e-06 2023-09-23T20:34:58.259 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.260 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.260 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:34:58.260 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.916483+0000", 2023-09-23T20:34:58.260 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1178e-05 2023-09-23T20:34:58.260 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.260 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.260 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:34:58.260 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.916490+0000", 2023-09-23T20:34:58.260 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.939e-06 2023-09-23T20:34:58.260 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.260 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.261 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:34:58.261 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.916534+0000", 2023-09-23T20:34:58.261 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.4317000000000003e-05 2023-09-23T20:34:58.261 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.261 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.261 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:34:58.261 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.916710+0000", 2023-09-23T20:34:58.261 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00017583099999999999 2023-09-23T20:34:58.261 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.261 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.261 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 1,6", 2023-09-23T20:34:58.261 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.916891+0000", 2023-09-23T20:34:58.262 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00018042199999999999 2023-09-23T20:34:58.262 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.262 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.262 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:34:58.262 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.927117+0000", 2023-09-23T20:34:58.262 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.010226625 2023-09-23T20:34:58.262 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.262 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.262 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:34:58.262 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:01.927128+0000", 2023-09-23T20:34:58.262 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0468999999999999e-05 2023-09-23T20:34:58.263 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.263 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.263 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:34:58.263 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.028427+0000", 2023-09-23T20:34:58.263 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.101299356 2023-09-23T20:34:58.263 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.263 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.263 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for missing object", 2023-09-23T20:34:58.263 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.028528+0000", 2023-09-23T20:34:58.263 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000101366 2023-09-23T20:34:58.263 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.263 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.264 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:34:58.264 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.035800+0000", 2023-09-23T20:34:58.264 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0072718260000000003 2023-09-23T20:34:58.264 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.264 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.264 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:34:58.264 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:25:03.035840+0000", 2023-09-23T20:34:58.264 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.9988999999999999e-05 2023-09-23T20:34:58.264 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:34:58.264 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:34:58.264 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:34:58.265 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.265 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.265 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.45161.0:68483 6.a 6:5eb5ac9f:::benchmark_data_smithi018_270558_object68482:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e230)", 2023-09-23T20:34:58.265 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:28:04.072353+0000", 2023-09-23T20:34:58.265 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 414.17222563799999, 2023-09-23T20:34:58.265 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.1210222989999998, 2023-09-23T20:34:58.265 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:34:58.265 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:34:58.265 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:34:58.265 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.45161", 2023-09-23T20:34:58.265 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3795614987", 2023-09-23T20:34:58.266 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 68483 2023-09-23T20:34:58.266 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.266 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:34:58.266 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.266 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:34:58.266 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.072353+0000", 2023-09-23T20:34:58.266 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:34:58.266 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.266 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.266 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:34:58.266 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.072353+0000", 2023-09-23T20:34:58.267 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:34:58.267 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.267 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.267 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:34:58.267 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.072352+0000", 2023-09-23T20:34:58.267 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:34:58.267 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.267 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.267 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:34:58.267 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.072388+0000", 2023-09-23T20:34:58.267 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.6284e-05 2023-09-23T20:34:58.267 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.268 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.268 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:34:58.268 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.072401+0000", 2023-09-23T20:34:58.268 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.2706e-05 2023-09-23T20:34:58.268 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.268 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.268 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:34:58.268 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.072407+0000", 2023-09-23T20:34:58.268 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.7679999999999997e-06 2023-09-23T20:34:58.268 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.268 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.269 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:34:58.269 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.072536+0000", 2023-09-23T20:34:58.269 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000129694 2023-09-23T20:34:58.269 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.269 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.269 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:34:58.269 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.072547+0000", 2023-09-23T20:34:58.269 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0692e-05 2023-09-23T20:34:58.269 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.269 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.269 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:34:58.269 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:05.030894+0000", 2023-09-23T20:34:58.270 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.95834700699999997 2023-09-23T20:34:58.270 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.270 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.270 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:34:58.270 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:05.030907+0000", 2023-09-23T20:34:58.270 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.3186e-05 2023-09-23T20:34:58.270 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.270 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.270 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:34:58.270 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.177110+0000", 2023-09-23T20:34:58.270 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1462023859999999 2023-09-23T20:34:58.271 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.271 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.271 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:34:58.271 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.177301+0000", 2023-09-23T20:34:58.271 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00019089000000000001 2023-09-23T20:34:58.271 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.271 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.271 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,2", 2023-09-23T20:34:58.271 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.177511+0000", 2023-09-23T20:34:58.271 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00021060299999999999 2023-09-23T20:34:58.271 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.271 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.272 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:34:58.272 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.187865+0000", 2023-09-23T20:34:58.272 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.010353431 2023-09-23T20:34:58.272 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.272 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.272 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:34:58.272 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.188200+0000", 2023-09-23T20:34:58.272 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00033555299999999999 2023-09-23T20:34:58.272 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.272 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.272 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:34:58.273 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.189703+0000", 2023-09-23T20:34:58.273 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.001502499 2023-09-23T20:34:58.273 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.273 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.273 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:34:58.273 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.189754+0000", 2023-09-23T20:34:58.273 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.1598999999999998e-05 2023-09-23T20:34:58.273 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.273 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.273 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:34:58.273 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.193376+0000", 2023-09-23T20:34:58.273 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0036214429999999998 2023-09-23T20:34:58.274 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:34:58.274 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:34:58.274 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:34:58.274 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.274 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.274 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.45161.0:68495 6.a 6:5fb49a37:::benchmark_data_smithi018_270558_object68494:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e230)", 2023-09-23T20:34:58.274 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:28:04.084817+0000", 2023-09-23T20:34:58.274 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 414.15976162800001, 2023-09-23T20:34:58.274 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.110988222, 2023-09-23T20:34:58.274 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:34:58.274 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:34:58.275 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:34:58.275 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.45161", 2023-09-23T20:34:58.275 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3795614987", 2023-09-23T20:34:58.275 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 68495 2023-09-23T20:34:58.275 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.275 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:34:58.275 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.275 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:34:58.275 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.084817+0000", 2023-09-23T20:34:58.275 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:34:58.275 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.275 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.276 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:34:58.276 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.084817+0000", 2023-09-23T20:34:58.276 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:34:58.276 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.276 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.276 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:34:58.276 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.084815+0000", 2023-09-23T20:34:58.276 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999981 2023-09-23T20:34:58.276 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.276 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.276 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:34:58.277 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.084859+0000", 2023-09-23T20:34:58.277 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.3095e-05 2023-09-23T20:34:58.277 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.277 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.277 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:34:58.277 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.084875+0000", 2023-09-23T20:34:58.277 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.6739e-05 2023-09-23T20:34:58.277 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.277 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.277 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:34:58.277 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.084883+0000", 2023-09-23T20:34:58.277 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.0010000000000001e-06 2023-09-23T20:34:58.278 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.278 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.278 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:34:58.278 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.084956+0000", 2023-09-23T20:34:58.278 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.2482000000000002e-05 2023-09-23T20:34:58.278 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.278 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.278 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:34:58.278 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.084970+0000", 2023-09-23T20:34:58.278 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.4154999999999999e-05 2023-09-23T20:34:58.278 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.279 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.279 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:34:58.279 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:05.030931+0000", 2023-09-23T20:34:58.279 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.94596105600000002 2023-09-23T20:34:58.279 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.279 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.279 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:34:58.279 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:05.030940+0000", 2023-09-23T20:34:58.279 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.6419999999999996e-06 2023-09-23T20:34:58.279 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.279 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.280 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:34:58.280 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.180341+0000", 2023-09-23T20:34:58.280 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1494014969999999 2023-09-23T20:34:58.280 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.280 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.280 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:34:58.280 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.180523+0000", 2023-09-23T20:34:58.280 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00018138299999999999 2023-09-23T20:34:58.280 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.280 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.280 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,2", 2023-09-23T20:34:58.280 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.180726+0000", 2023-09-23T20:34:58.281 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00020395100000000001 2023-09-23T20:34:58.281 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.281 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.281 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:34:58.281 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.188188+0000", 2023-09-23T20:34:58.281 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0074612180000000004 2023-09-23T20:34:58.281 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.281 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.281 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:34:58.281 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.188341+0000", 2023-09-23T20:34:58.281 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000153776 2023-09-23T20:34:58.282 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.282 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.282 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:34:58.282 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.193438+0000", 2023-09-23T20:34:58.282 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0050964640000000002 2023-09-23T20:34:58.282 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.282 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.282 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:34:58.282 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.193481+0000", 2023-09-23T20:34:58.282 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.2864999999999997e-05 2023-09-23T20:34:58.282 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.283 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.283 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:34:58.283 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.195806+0000", 2023-09-23T20:34:58.283 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0023247419999999999 2023-09-23T20:34:58.283 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:34:58.283 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:34:58.283 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:34:58.283 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.283 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.283 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.45161.0:136033 6.7 6:e12c7d04:::benchmark_data_smithi018_270558_object31294:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e257)", 2023-09-23T20:34:58.283 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:29:27.327366+0000", 2023-09-23T20:34:58.283 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 330.91721332399999, 2023-09-23T20:34:58.284 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.0930456, 2023-09-23T20:34:58.284 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:34:58.284 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:34:58.284 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:34:58.284 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.45161", 2023-09-23T20:34:58.284 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3795614987", 2023-09-23T20:34:58.284 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 136033 2023-09-23T20:34:58.284 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.284 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:34:58.284 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.284 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:34:58.285 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:27.327366+0000", 2023-09-23T20:34:58.285 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:34:58.285 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.285 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.285 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:34:58.285 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:27.327366+0000", 2023-09-23T20:34:58.285 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:34:58.285 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.285 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.285 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:34:58.285 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:27.327363+0000", 2023-09-23T20:34:58.285 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999976 2023-09-23T20:34:58.286 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.286 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.286 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:34:58.286 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:27.327372+0000", 2023-09-23T20:34:58.286 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.6750000000000008e-06 2023-09-23T20:34:58.286 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.286 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.286 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:34:58.286 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:27.327386+0000", 2023-09-23T20:34:58.286 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.4151999999999999e-05 2023-09-23T20:34:58.286 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.287 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.287 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:34:58.287 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:27.327393+0000", 2023-09-23T20:34:58.287 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.9210000000000003e-06 2023-09-23T20:34:58.287 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.287 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.287 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:34:58.287 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:27.327440+0000", 2023-09-23T20:34:58.287 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.7227e-05 2023-09-23T20:34:58.287 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.287 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.287 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:34:58.288 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:27.327599+0000", 2023-09-23T20:34:58.288 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00015890000000000001 2023-09-23T20:34:58.288 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.288 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.288 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:34:58.288 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:27.328068+0000", 2023-09-23T20:34:58.288 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00046883200000000003 2023-09-23T20:34:58.288 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.288 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.288 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:34:58.288 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:27.330709+0000", 2023-09-23T20:34:58.289 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0026409200000000002 2023-09-23T20:34:58.289 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.289 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.289 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:34:58.289 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:27.344609+0000", 2023-09-23T20:34:58.289 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.013900341 2023-09-23T20:34:58.289 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.289 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.289 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:34:58.289 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:27.369483+0000", 2023-09-23T20:34:58.289 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.024873704 2023-09-23T20:34:58.289 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.290 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.290 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:34:58.290 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:27.369498+0000", 2023-09-23T20:34:58.290 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.4545999999999999e-05 2023-09-23T20:34:58.290 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.290 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.290 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:34:58.290 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:28.355543+0000", 2023-09-23T20:34:58.290 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.98604489900000003 2023-09-23T20:34:58.290 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.290 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.290 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:34:58.291 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:28.355555+0000", 2023-09-23T20:34:58.291 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.2612000000000001e-05 2023-09-23T20:34:58.291 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.291 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.291 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:34:58.291 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:29.391776+0000", 2023-09-23T20:34:58.291 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0362208989999999 2023-09-23T20:34:58.291 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.291 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.291 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for missing object", 2023-09-23T20:34:58.291 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:29.391962+0000", 2023-09-23T20:34:58.291 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00018626499999999999 2023-09-23T20:34:58.292 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.292 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.292 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:34:58.292 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:29.420342+0000", 2023-09-23T20:34:58.292 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.028379945 2023-09-23T20:34:58.292 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.292 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.292 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:34:58.292 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:29.420411+0000", 2023-09-23T20:34:58.292 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.8942000000000005e-05 2023-09-23T20:34:58.292 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:34:58.293 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:34:58.293 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:34:58.293 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.293 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.293 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.45161.0:136047 6.7 6:e530c251:::benchmark_data_smithi018_270558_object31308:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e257)", 2023-09-23T20:34:58.293 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:29:27.363510+0000", 2023-09-23T20:34:58.293 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 330.88106886200001, 2023-09-23T20:34:58.293 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.0564319210000002, 2023-09-23T20:34:58.293 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:34:58.293 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:34:58.293 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:34:58.293 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.45161", 2023-09-23T20:34:58.294 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3795614987", 2023-09-23T20:34:58.294 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 136047 2023-09-23T20:34:58.294 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.294 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:34:58.294 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.294 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:34:58.294 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:27.363510+0000", 2023-09-23T20:34:58.294 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:34:58.294 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.294 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.294 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:34:58.295 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:27.363510+0000", 2023-09-23T20:34:58.295 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:34:58.295 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.295 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.295 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:34:58.295 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:27.363508+0000", 2023-09-23T20:34:58.295 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999981 2023-09-23T20:34:58.295 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.295 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.295 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:34:58.295 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:27.363514+0000", 2023-09-23T20:34:58.295 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.3580000000000002e-06 2023-09-23T20:34:58.296 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.296 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.296 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:34:58.296 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:27.363524+0000", 2023-09-23T20:34:58.296 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0543000000000001e-05 2023-09-23T20:34:58.296 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.296 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.296 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:34:58.296 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:27.363535+0000", 2023-09-23T20:34:58.296 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.044e-05 2023-09-23T20:34:58.296 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.296 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.297 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:34:58.297 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:27.369525+0000", 2023-09-23T20:34:58.297 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0059905039999999998 2023-09-23T20:34:58.297 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.297 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.297 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:34:58.297 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:27.369532+0000", 2023-09-23T20:34:58.297 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.6619999999999999e-06 2023-09-23T20:34:58.297 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.297 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.297 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:34:58.298 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:28.355582+0000", 2023-09-23T20:34:58.298 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.98605059900000003 2023-09-23T20:34:58.298 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.298 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.298 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:34:58.298 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:28.355589+0000", 2023-09-23T20:34:58.298 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.8929999999999999e-06 2023-09-23T20:34:58.298 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.298 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.298 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:34:58.298 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:29.392324+0000", 2023-09-23T20:34:58.299 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0367343600000001 2023-09-23T20:34:58.299 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.299 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.299 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:34:58.299 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:29.392475+0000", 2023-09-23T20:34:58.299 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00015086799999999999 2023-09-23T20:34:58.299 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.299 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.299 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:34:58.299 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:29.393040+0000", 2023-09-23T20:34:58.299 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00056566799999999997 2023-09-23T20:34:58.299 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.300 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.300 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:34:58.300 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:29.399625+0000", 2023-09-23T20:34:58.300 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0065850880000000002 2023-09-23T20:34:58.300 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.300 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.300 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:34:58.300 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:29.399866+0000", 2023-09-23T20:34:58.300 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00024026000000000001 2023-09-23T20:34:58.300 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.300 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.300 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:34:58.301 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:29.419769+0000", 2023-09-23T20:34:58.301 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.019903745 2023-09-23T20:34:58.301 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.301 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.301 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:34:58.301 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:29.419830+0000", 2023-09-23T20:34:58.301 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.0566000000000003e-05 2023-09-23T20:34:58.301 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.301 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.301 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:34:58.301 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:29.419942+0000", 2023-09-23T20:34:58.301 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000112093 2023-09-23T20:34:58.302 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:34:58.302 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:34:58.302 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:34:58.302 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.302 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.302 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.45161.0:136059 6.17 6:ed3cc3ed:::benchmark_data_smithi018_270558_object31320:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e258)", 2023-09-23T20:34:58.302 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:29:27.380270+0000", 2023-09-23T20:34:58.302 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 330.86430917199999, 2023-09-23T20:34:58.302 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.019361118, 2023-09-23T20:34:58.302 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:34:58.302 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:34:58.302 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:34:58.303 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.45161", 2023-09-23T20:34:58.303 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3795614987", 2023-09-23T20:34:58.303 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 136059 2023-09-23T20:34:58.303 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.303 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:34:58.303 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.303 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:34:58.303 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:27.380270+0000", 2023-09-23T20:34:58.303 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:34:58.303 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.303 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.304 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:34:58.304 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:27.380270+0000", 2023-09-23T20:34:58.304 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:34:58.304 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.304 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.304 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:34:58.304 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:27.380269+0000", 2023-09-23T20:34:58.304 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:34:58.304 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.304 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.304 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:34:58.304 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:27.380274+0000", 2023-09-23T20:34:58.305 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.169e-06 2023-09-23T20:34:58.305 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.305 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.305 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:34:58.305 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:27.380284+0000", 2023-09-23T20:34:58.305 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.7580000000000004e-06 2023-09-23T20:34:58.305 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.305 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.305 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:34:58.305 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:27.380290+0000", 2023-09-23T20:34:58.305 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.8370000000000002e-06 2023-09-23T20:34:58.305 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.306 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.306 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:34:58.306 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:27.380359+0000", 2023-09-23T20:34:58.306 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.9441000000000001e-05 2023-09-23T20:34:58.306 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.306 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.306 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:34:58.306 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:27.380368+0000", 2023-09-23T20:34:58.306 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.5879999999999998e-06 2023-09-23T20:34:58.306 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.306 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.306 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:34:58.307 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:28.355613+0000", 2023-09-23T20:34:58.307 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.97524513400000001 2023-09-23T20:34:58.307 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.307 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.307 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:34:58.307 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:28.355621+0000", 2023-09-23T20:34:58.307 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.0209999999999999e-06 2023-09-23T20:34:58.307 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.307 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.307 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:34:58.307 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:29.376357+0000", 2023-09-23T20:34:58.307 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.020736445 2023-09-23T20:34:58.308 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.308 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.308 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:34:58.308 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:29.376497+0000", 2023-09-23T20:34:58.308 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00014008400000000001 2023-09-23T20:34:58.308 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.308 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.308 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:34:58.308 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:29.376961+0000", 2023-09-23T20:34:58.308 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00046314499999999998 2023-09-23T20:34:58.308 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.309 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.309 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:34:58.309 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:29.392111+0000", 2023-09-23T20:34:58.309 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.015150788 2023-09-23T20:34:58.309 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.309 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.309 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:34:58.309 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:29.392146+0000", 2023-09-23T20:34:58.309 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.4604999999999998e-05 2023-09-23T20:34:58.309 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.309 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.309 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:34:58.310 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:29.399462+0000", 2023-09-23T20:34:58.310 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0073164119999999996 2023-09-23T20:34:58.310 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.310 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.310 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:34:58.310 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:29.399517+0000", 2023-09-23T20:34:58.310 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.4987000000000001e-05 2023-09-23T20:34:58.310 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.310 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.310 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:34:58.310 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:29.399631+0000", 2023-09-23T20:34:58.311 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00011351 2023-09-23T20:34:58.311 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:34:58.311 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:34:58.311 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:34:58.311 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.311 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.311 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.45161.0:136060 6.7 6:e35290e5:::benchmark_data_smithi018_270558_object31321:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e258)", 2023-09-23T20:34:58.311 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:29:27.380303+0000", 2023-09-23T20:34:58.311 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 330.86427598900002, 2023-09-23T20:34:58.311 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.0397914629999998, 2023-09-23T20:34:58.311 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:34:58.311 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:34:58.312 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:34:58.312 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.45161", 2023-09-23T20:34:58.312 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3795614987", 2023-09-23T20:34:58.312 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 136060 2023-09-23T20:34:58.312 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.312 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:34:58.312 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.312 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:34:58.312 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:27.380303+0000", 2023-09-23T20:34:58.312 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:34:58.312 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.313 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.313 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:34:58.313 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:27.380303+0000", 2023-09-23T20:34:58.313 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:34:58.313 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.313 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.313 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:34:58.313 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:27.380303+0000", 2023-09-23T20:34:58.313 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:34:58.313 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.313 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.313 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:34:58.314 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:27.380306+0000", 2023-09-23T20:34:58.314 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.0070000000000002e-06 2023-09-23T20:34:58.314 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.314 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.314 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:34:58.314 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:27.380312+0000", 2023-09-23T20:34:58.314 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.6200000000000001e-06 2023-09-23T20:34:58.314 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.314 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.314 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:34:58.314 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:27.380321+0000", 2023-09-23T20:34:58.315 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.3200000000000006e-06 2023-09-23T20:34:58.315 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.315 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.315 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:34:58.315 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:27.380389+0000", 2023-09-23T20:34:58.315 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.7699000000000004e-05 2023-09-23T20:34:58.315 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.315 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.315 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:34:58.315 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:27.380395+0000", 2023-09-23T20:34:58.315 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.139e-06 2023-09-23T20:34:58.316 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.316 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.316 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:34:58.316 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:28.355749+0000", 2023-09-23T20:34:58.316 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.97535333199999996 2023-09-23T20:34:58.316 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.316 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.316 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:34:58.316 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:28.355757+0000", 2023-09-23T20:34:58.316 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.0430000000000008e-06 2023-09-23T20:34:58.316 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.316 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.317 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:34:58.317 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:29.396046+0000", 2023-09-23T20:34:58.317 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0402891299999999 2023-09-23T20:34:58.317 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.317 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.317 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:34:58.317 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:29.396225+0000", 2023-09-23T20:34:58.317 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00017960099999999999 2023-09-23T20:34:58.317 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.317 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.317 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:34:58.318 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:29.396790+0000", 2023-09-23T20:34:58.318 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00056465699999999998 2023-09-23T20:34:58.318 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.318 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.318 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:34:58.318 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:29.399660+0000", 2023-09-23T20:34:58.318 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.002869969 2023-09-23T20:34:58.318 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.318 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.318 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:34:58.318 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:29.400003+0000", 2023-09-23T20:34:58.318 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00034285899999999999 2023-09-23T20:34:58.319 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.319 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.319 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:34:58.319 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:29.419979+0000", 2023-09-23T20:34:58.319 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.019976224000000001 2023-09-23T20:34:58.319 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.319 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.319 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:34:58.319 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:29.420015+0000", 2023-09-23T20:34:58.319 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.6282999999999998e-05 2023-09-23T20:34:58.319 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.320 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.320 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:34:58.320 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:29.420094+0000", 2023-09-23T20:34:58.320 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.9040000000000002e-05 2023-09-23T20:34:58.320 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:34:58.320 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:34:58.320 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:34:58.320 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.320 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.320 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35700.0:123655 7.0 7:0e4c1787:::benchmark_data_smithi018_308601_object29103:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e339)", 2023-09-23T20:34:58.320 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:34:09.323022+0000", 2023-09-23T20:34:58.320 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 48.921556561000003, 2023-09-23T20:34:58.321 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.049133461000000003, 2023-09-23T20:34:58.321 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:34:58.321 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:34:58.321 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:34:58.321 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35700", 2023-09-23T20:34:58.321 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:34:58.321 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 123655 2023-09-23T20:34:58.321 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.321 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:34:58.321 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.321 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:34:58.321 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:09.323022+0000", 2023-09-23T20:34:58.322 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:34:58.322 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.322 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.322 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:34:58.322 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:09.323022+0000", 2023-09-23T20:34:58.322 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:34:58.322 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.322 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.322 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:34:58.322 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:09.323021+0000", 2023-09-23T20:34:58.322 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:34:58.323 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.323 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.323 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:34:58.323 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:09.323028+0000", 2023-09-23T20:34:58.323 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.1590000000000004e-06 2023-09-23T20:34:58.323 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.323 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.323 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:34:58.323 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:09.323045+0000", 2023-09-23T20:34:58.323 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.6776e-05 2023-09-23T20:34:58.323 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.323 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.324 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:34:58.324 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:09.323068+0000", 2023-09-23T20:34:58.324 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.3291000000000001e-05 2023-09-23T20:34:58.324 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.324 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.324 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:34:58.324 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:09.323117+0000", 2023-09-23T20:34:58.324 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.8235999999999997e-05 2023-09-23T20:34:58.324 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.324 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.324 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:34:58.324 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:09.323223+0000", 2023-09-23T20:34:58.325 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000106889 2023-09-23T20:34:58.325 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.325 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.325 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:34:58.325 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:09.323308+0000", 2023-09-23T20:34:58.325 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.488e-05 2023-09-23T20:34:58.325 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.325 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.325 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:34:58.325 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:09.326772+0000", 2023-09-23T20:34:58.325 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0034636670000000001 2023-09-23T20:34:58.326 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.326 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.326 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:34:58.326 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:09.329355+0000", 2023-09-23T20:34:58.326 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0025828299999999999 2023-09-23T20:34:58.326 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.326 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.326 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:34:58.326 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:09.372067+0000", 2023-09-23T20:34:58.326 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.042712245000000003 2023-09-23T20:34:58.326 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.326 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.327 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:34:58.327 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:09.372116+0000", 2023-09-23T20:34:58.327 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.8498999999999998e-05 2023-09-23T20:34:58.327 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.327 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.327 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:34:58.327 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:09.372156+0000", 2023-09-23T20:34:58.327 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.0351e-05 2023-09-23T20:34:58.327 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:34:58.327 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:34:58.327 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:34:58.328 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.328 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.328 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35700.0:124033 7.0 7:053df0f3:::benchmark_data_smithi018_308601_object29481:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e339)", 2023-09-23T20:34:58.328 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:34:09.967189+0000", 2023-09-23T20:34:58.328 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 48.277390007000001, 2023-09-23T20:34:58.328 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.051331518, 2023-09-23T20:34:58.328 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:34:58.328 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:34:58.328 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:34:58.328 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35700", 2023-09-23T20:34:58.328 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:34:58.328 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 124033 2023-09-23T20:34:58.329 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.329 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:34:58.329 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.329 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:34:58.329 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:09.967189+0000", 2023-09-23T20:34:58.329 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:34:58.329 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.329 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.329 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:34:58.329 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:09.967189+0000", 2023-09-23T20:34:58.329 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:34:58.329 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.330 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.330 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:34:58.330 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:09.967187+0000", 2023-09-23T20:34:58.330 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:34:58.330 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.330 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.330 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:34:58.330 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:09.967195+0000", 2023-09-23T20:34:58.330 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.2840000000000001e-06 2023-09-23T20:34:58.330 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.331 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.331 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:34:58.331 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:09.967220+0000", 2023-09-23T20:34:58.331 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.4936999999999999e-05 2023-09-23T20:34:58.331 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.331 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.331 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:34:58.331 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:09.967228+0000", 2023-09-23T20:34:58.331 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.0989999999999999e-06 2023-09-23T20:34:58.331 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.331 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.332 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:34:58.332 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:09.967280+0000", 2023-09-23T20:34:58.332 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.2357e-05 2023-09-23T20:34:58.332 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.332 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.332 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:34:58.332 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:09.967411+0000", 2023-09-23T20:34:58.332 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00013086599999999999 2023-09-23T20:34:58.332 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.332 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.332 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:34:58.333 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:09.967529+0000", 2023-09-23T20:34:58.333 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00011797400000000001 2023-09-23T20:34:58.333 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.333 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.333 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:34:58.333 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:09.971799+0000", 2023-09-23T20:34:58.333 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0042696080000000003 2023-09-23T20:34:58.333 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.333 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.333 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:34:58.333 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:10.007164+0000", 2023-09-23T20:34:58.333 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.035365959000000002 2023-09-23T20:34:58.334 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.334 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.334 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:34:58.334 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:10.018446+0000", 2023-09-23T20:34:58.334 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.011281447 2023-09-23T20:34:58.334 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.334 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.334 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:34:58.334 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:10.018481+0000", 2023-09-23T20:34:58.334 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.4820000000000002e-05 2023-09-23T20:34:58.334 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.335 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.335 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:34:58.335 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:10.018520+0000", 2023-09-23T20:34:58.335 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.9740000000000002e-05 2023-09-23T20:34:58.335 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:34:58.335 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:34:58.335 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:34:58.335 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.335 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.335 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35700.0:124632 7.0 7:03ed5fcb:::benchmark_data_smithi018_308601_object30080:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e339)", 2023-09-23T20:34:58.335 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:34:11.013771+0000", 2023-09-23T20:34:58.335 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 47.230808027999998, 2023-09-23T20:34:58.336 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.043378516999999998, 2023-09-23T20:34:58.336 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:34:58.336 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:34:58.336 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:34:58.336 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35700", 2023-09-23T20:34:58.336 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:34:58.336 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 124632 2023-09-23T20:34:58.336 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.336 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:34:58.336 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.336 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:34:58.337 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:11.013771+0000", 2023-09-23T20:34:58.337 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:34:58.337 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.337 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.337 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:34:58.337 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:11.013771+0000", 2023-09-23T20:34:58.337 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:34:58.337 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.337 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.337 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:34:58.337 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:11.013769+0000", 2023-09-23T20:34:58.337 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:34:58.338 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.338 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.338 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:34:58.338 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:11.013776+0000", 2023-09-23T20:34:58.338 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.5089999999999999e-06 2023-09-23T20:34:58.338 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.338 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.338 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:34:58.338 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:11.013791+0000", 2023-09-23T20:34:58.338 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.4915e-05 2023-09-23T20:34:58.339 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.339 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.339 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:34:58.339 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:11.013798+0000", 2023-09-23T20:34:58.339 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.2280000000000001e-06 2023-09-23T20:34:58.339 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.339 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.339 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:34:58.339 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:11.013858+0000", 2023-09-23T20:34:58.339 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.0015000000000001e-05 2023-09-23T20:34:58.339 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.339 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.340 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:34:58.340 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:11.013981+0000", 2023-09-23T20:34:58.340 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00012298699999999999 2023-09-23T20:34:58.340 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.340 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.340 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:34:58.340 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:11.014108+0000", 2023-09-23T20:34:58.340 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00012668900000000001 2023-09-23T20:34:58.340 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.340 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.340 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:34:58.341 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:11.020597+0000", 2023-09-23T20:34:58.341 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.006489234 2023-09-23T20:34:58.341 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.341 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.341 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:34:58.341 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:11.030140+0000", 2023-09-23T20:34:58.341 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0095430970000000004 2023-09-23T20:34:58.341 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.341 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.341 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:34:58.341 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:11.057056+0000", 2023-09-23T20:34:58.342 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.026916414999999999 2023-09-23T20:34:58.342 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.342 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.342 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:34:58.342 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:11.057107+0000", 2023-09-23T20:34:58.342 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.0535999999999999e-05 2023-09-23T20:34:58.342 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.342 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.342 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:34:58.342 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:11.057149+0000", 2023-09-23T20:34:58.342 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.2425999999999997e-05 2023-09-23T20:34:58.343 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:34:58.343 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:34:58.343 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:34:58.343 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.343 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.343 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35700.0:124633 7.0 7:0db22370:::benchmark_data_smithi018_308601_object30081:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e339)", 2023-09-23T20:34:58.343 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:34:11.013815+0000", 2023-09-23T20:34:58.343 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 47.230764413000003, 2023-09-23T20:34:58.343 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.043465786999999999, 2023-09-23T20:34:58.343 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:34:58.343 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:34:58.344 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:34:58.344 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35700", 2023-09-23T20:34:58.344 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:34:58.344 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 124633 2023-09-23T20:34:58.344 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.344 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:34:58.344 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.344 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:34:58.344 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:11.013815+0000", 2023-09-23T20:34:58.344 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:34:58.344 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.345 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.345 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:34:58.345 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:11.013815+0000", 2023-09-23T20:34:58.345 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:34:58.345 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.345 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.345 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:34:58.345 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:11.013814+0000", 2023-09-23T20:34:58.345 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:34:58.345 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.345 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.346 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:34:58.346 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:11.013817+0000", 2023-09-23T20:34:58.346 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.01e-06 2023-09-23T20:34:58.346 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.346 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.346 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:34:58.346 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:11.013832+0000", 2023-09-23T20:34:58.346 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.4721e-05 2023-09-23T20:34:58.346 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.346 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.346 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:34:58.346 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:11.013837+0000", 2023-09-23T20:34:58.347 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.1490000000000002e-06 2023-09-23T20:34:58.347 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.347 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.347 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:34:58.347 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:11.017003+0000", 2023-09-23T20:34:58.347 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0031662729999999998 2023-09-23T20:34:58.347 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.347 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.347 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:34:58.347 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:11.017086+0000", 2023-09-23T20:34:58.347 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.3028999999999994e-05 2023-09-23T20:34:58.348 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.348 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.348 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:34:58.348 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:11.017155+0000", 2023-09-23T20:34:58.348 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.8534999999999993e-05 2023-09-23T20:34:58.348 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.348 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.348 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:34:58.348 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:11.020794+0000", 2023-09-23T20:34:58.348 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.003639644 2023-09-23T20:34:58.348 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.349 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.349 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:34:58.349 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:11.030414+0000", 2023-09-23T20:34:58.349 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0096198440000000007 2023-09-23T20:34:58.349 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.349 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.349 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:34:58.349 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:11.057180+0000", 2023-09-23T20:34:58.349 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.026766109999999999 2023-09-23T20:34:58.349 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.349 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.349 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:34:58.350 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:11.057245+0000", 2023-09-23T20:34:58.350 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.4675e-05 2023-09-23T20:34:58.350 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.350 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.350 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:34:58.350 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:11.057280+0000", 2023-09-23T20:34:58.350 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.5277000000000001e-05 2023-09-23T20:34:58.350 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:34:58.350 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:34:58.350 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:34:58.350 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.351 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.351 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35700.0:124686 7.0 7:0f4d34bb:::benchmark_data_smithi018_308601_object30134:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e339)", 2023-09-23T20:34:58.351 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:34:11.097013+0000", 2023-09-23T20:34:58.351 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 47.147565534000002, 2023-09-23T20:34:58.351 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.052954094, 2023-09-23T20:34:58.351 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:34:58.351 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:34:58.351 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:34:58.351 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35700", 2023-09-23T20:34:58.351 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:34:58.351 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 124686 2023-09-23T20:34:58.351 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.352 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:34:58.352 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.352 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:34:58.352 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:11.097013+0000", 2023-09-23T20:34:58.352 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:34:58.352 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.352 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.352 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:34:58.352 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:11.097013+0000", 2023-09-23T20:34:58.352 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:34:58.352 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.353 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.353 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:34:58.353 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:11.097013+0000", 2023-09-23T20:34:58.353 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967296 2023-09-23T20:34:58.353 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.353 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.353 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:34:58.353 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:11.097015+0000", 2023-09-23T20:34:58.353 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.6500000000000001e-06 2023-09-23T20:34:58.353 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.353 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.353 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:34:58.354 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:11.097018+0000", 2023-09-23T20:34:58.354 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.3749999999999999e-06 2023-09-23T20:34:58.354 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.354 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.354 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:34:58.354 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:11.097021+0000", 2023-09-23T20:34:58.354 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.6299999999999998e-06 2023-09-23T20:34:58.354 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.354 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.354 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:34:58.354 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:11.108297+0000", 2023-09-23T20:34:58.354 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.011276262 2023-09-23T20:34:58.355 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.355 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.355 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:34:58.355 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:11.108428+0000", 2023-09-23T20:34:58.355 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00013134 2023-09-23T20:34:58.355 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.355 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.355 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:34:58.355 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:11.108555+0000", 2023-09-23T20:34:58.355 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000126236 2023-09-23T20:34:58.355 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.356 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.356 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:34:58.356 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:11.113808+0000", 2023-09-23T20:34:58.356 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0052536309999999999 2023-09-23T20:34:58.356 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.356 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.356 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:34:58.356 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:11.137975+0000", 2023-09-23T20:34:58.356 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.024166324999999999 2023-09-23T20:34:58.356 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.356 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.356 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:34:58.357 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:11.149873+0000", 2023-09-23T20:34:58.357 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.011898136 2023-09-23T20:34:58.357 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.357 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.357 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:34:58.357 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:11.149922+0000", 2023-09-23T20:34:58.357 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.8769e-05 2023-09-23T20:34:58.357 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.357 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.357 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:34:58.357 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:11.149968+0000", 2023-09-23T20:34:58.358 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.5954000000000001e-05 2023-09-23T20:34:58.358 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:34:58.358 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:34:58.358 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:34:58.358 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.358 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.358 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35700.0:124690 7.0 7:0e963b80:::benchmark_data_smithi018_308601_object30138:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e339)", 2023-09-23T20:34:58.358 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:34:11.097024+0000", 2023-09-23T20:34:58.358 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 47.147555365000002, 2023-09-23T20:34:58.358 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.053104429000000002, 2023-09-23T20:34:58.358 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:34:58.358 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:34:58.359 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:34:58.359 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35700", 2023-09-23T20:34:58.359 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:34:58.359 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 124690 2023-09-23T20:34:58.359 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.359 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:34:58.359 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.359 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:34:58.359 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:11.097024+0000", 2023-09-23T20:34:58.359 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:34:58.359 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.360 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.360 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:34:58.360 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:11.097024+0000", 2023-09-23T20:34:58.360 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:34:58.360 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.360 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.360 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:34:58.360 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:11.097023+0000", 2023-09-23T20:34:58.360 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967296 2023-09-23T20:34:58.360 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.360 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.361 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:34:58.361 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:11.097025+0000", 2023-09-23T20:34:58.361 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.9920000000000002e-06 2023-09-23T20:34:58.361 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.361 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.361 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:34:58.361 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:11.097028+0000", 2023-09-23T20:34:58.361 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.9340000000000002e-06 2023-09-23T20:34:58.361 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.361 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.361 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:34:58.361 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:11.097030+0000", 2023-09-23T20:34:58.362 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.1739999999999999e-06 2023-09-23T20:34:58.362 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.362 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.362 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:34:58.362 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:11.111194+0000", 2023-09-23T20:34:58.362 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.014163283 2023-09-23T20:34:58.362 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.362 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.362 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:34:58.362 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:11.111287+0000", 2023-09-23T20:34:58.362 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.3016000000000005e-05 2023-09-23T20:34:58.363 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.363 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.363 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:34:58.363 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:11.111374+0000", 2023-09-23T20:34:58.363 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.7150000000000004e-05 2023-09-23T20:34:58.363 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.363 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.363 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:34:58.363 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:11.113990+0000", 2023-09-23T20:34:58.363 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0026160100000000002 2023-09-23T20:34:58.363 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.363 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.364 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:34:58.364 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:11.140637+0000", 2023-09-23T20:34:58.364 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.026646633999999999 2023-09-23T20:34:58.364 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.364 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.364 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:34:58.364 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:11.150033+0000", 2023-09-23T20:34:58.364 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0093966459999999998 2023-09-23T20:34:58.364 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.364 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.364 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:34:58.365 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:11.150087+0000", 2023-09-23T20:34:58.365 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.3715000000000003e-05 2023-09-23T20:34:58.365 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.365 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.365 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:34:58.365 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:11.150128+0000", 2023-09-23T20:34:58.365 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.108e-05 2023-09-23T20:34:58.365 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:34:58.365 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:34:58.365 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:34:58.365 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.366 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.366 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35700.0:135148 7.0 7:046605a2:::benchmark_data_smithi018_308601_object40596:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e345)", 2023-09-23T20:34:58.366 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:34:27.949814+0000", 2023-09-23T20:34:58.366 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 30.294765341000002, 2023-09-23T20:34:58.366 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.062452647, 2023-09-23T20:34:58.366 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:34:58.366 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:34:58.366 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:34:58.366 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35700", 2023-09-23T20:34:58.366 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:34:58.366 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 135148 2023-09-23T20:34:58.367 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.367 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:34:58.367 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.367 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:34:58.367 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:27.949814+0000", 2023-09-23T20:34:58.367 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:34:58.367 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.367 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.367 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:34:58.367 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:27.949814+0000", 2023-09-23T20:34:58.367 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:34:58.368 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.368 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.368 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:34:58.368 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:27.949812+0000", 2023-09-23T20:34:58.368 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:34:58.368 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.368 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.368 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:34:58.368 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:27.949819+0000", 2023-09-23T20:34:58.368 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.781e-06 2023-09-23T20:34:58.368 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.368 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.369 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:34:58.369 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:27.949834+0000", 2023-09-23T20:34:58.369 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.4765e-05 2023-09-23T20:34:58.369 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.369 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.369 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:34:58.369 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:27.949840+0000", 2023-09-23T20:34:58.369 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.4529999999999999e-06 2023-09-23T20:34:58.369 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.369 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.369 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:34:58.369 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:27.949906+0000", 2023-09-23T20:34:58.370 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.6033999999999998e-05 2023-09-23T20:34:58.370 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.370 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.370 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:34:58.370 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:27.950040+0000", 2023-09-23T20:34:58.370 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00013335899999999999 2023-09-23T20:34:58.370 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.370 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.370 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:34:58.370 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:27.950186+0000", 2023-09-23T20:34:58.370 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00014578400000000001 2023-09-23T20:34:58.371 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.371 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.371 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:34:58.371 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:27.953394+0000", 2023-09-23T20:34:58.371 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0032083179999999999 2023-09-23T20:34:58.371 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.371 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.371 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:34:58.371 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:27.972433+0000", 2023-09-23T20:34:58.371 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.019039378999999999 2023-09-23T20:34:58.371 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.371 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.372 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:34:58.372 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:28.012154+0000", 2023-09-23T20:34:58.372 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.039721168000000001 2023-09-23T20:34:58.372 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.372 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.372 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:34:58.372 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:28.012212+0000", 2023-09-23T20:34:58.372 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.7459e-05 2023-09-23T20:34:58.372 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.372 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.372 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:34:58.372 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:28.012266+0000", 2023-09-23T20:34:58.373 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.4407000000000003e-05 2023-09-23T20:34:58.373 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:34:58.373 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:34:58.373 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:34:58.373 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.373 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.373 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35700.0:135155 7.0 7:0eee6c75:::benchmark_data_smithi018_308601_object40603:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e345)", 2023-09-23T20:34:58.373 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:34:27.959808+0000", 2023-09-23T20:34:58.373 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 30.284771267, 2023-09-23T20:34:58.373 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.052585896, 2023-09-23T20:34:58.373 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:34:58.374 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:34:58.374 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:34:58.374 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35700", 2023-09-23T20:34:58.374 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:34:58.374 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 135155 2023-09-23T20:34:58.374 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.374 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:34:58.374 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.374 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:34:58.374 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:27.959808+0000", 2023-09-23T20:34:58.374 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:34:58.375 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.375 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.375 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:34:58.375 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:27.959808+0000", 2023-09-23T20:34:58.375 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:34:58.375 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.375 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.375 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:34:58.375 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:27.959806+0000", 2023-09-23T20:34:58.375 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:34:58.375 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.375 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.376 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:34:58.376 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:27.959814+0000", 2023-09-23T20:34:58.376 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.7549999999999994e-06 2023-09-23T20:34:58.376 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.376 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.376 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:34:58.376 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:27.959832+0000", 2023-09-23T20:34:58.376 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.7952999999999999e-05 2023-09-23T20:34:58.376 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.376 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.376 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:34:58.377 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:27.959840+0000", 2023-09-23T20:34:58.377 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.0660000000000004e-06 2023-09-23T20:34:58.377 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.377 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.377 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:34:58.377 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:27.959894+0000", 2023-09-23T20:34:58.377 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.4324999999999999e-05 2023-09-23T20:34:58.377 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.377 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.377 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:34:58.377 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:27.960020+0000", 2023-09-23T20:34:58.377 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00012545899999999999 2023-09-23T20:34:58.378 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.378 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.378 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:34:58.378 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:27.960136+0000", 2023-09-23T20:34:58.378 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000116054 2023-09-23T20:34:58.378 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.378 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.378 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:34:58.378 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:27.963687+0000", 2023-09-23T20:34:58.378 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.003551177 2023-09-23T20:34:58.378 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.379 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.379 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:34:58.379 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:27.981268+0000", 2023-09-23T20:34:58.379 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.017581374 2023-09-23T20:34:58.379 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.379 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.379 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:34:58.379 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:28.012325+0000", 2023-09-23T20:34:58.379 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.031056818 2023-09-23T20:34:58.379 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.379 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.379 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:34:58.380 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:28.012357+0000", 2023-09-23T20:34:58.380 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.2196000000000001e-05 2023-09-23T20:34:58.380 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.380 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.380 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:34:58.380 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:28.012394+0000", 2023-09-23T20:34:58.380 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.6164000000000001e-05 2023-09-23T20:34:58.380 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:34:58.380 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:34:58.380 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:34:58.380 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.380 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.381 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35700.0:135158 7.0 7:09a3cf1b:::benchmark_data_smithi018_308601_object40606:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e345)", 2023-09-23T20:34:58.381 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:34:27.964429+0000", 2023-09-23T20:34:58.381 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 30.280149829999999, 2023-09-23T20:34:58.381 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.048095366000000001, 2023-09-23T20:34:58.381 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:34:58.381 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:34:58.381 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:34:58.381 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35700", 2023-09-23T20:34:58.381 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:34:58.381 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 135158 2023-09-23T20:34:58.381 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.382 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:34:58.382 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.382 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:34:58.382 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:27.964429+0000", 2023-09-23T20:34:58.382 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:34:58.382 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.382 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.382 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:34:58.382 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:27.964429+0000", 2023-09-23T20:34:58.382 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:34:58.382 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.383 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.383 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:34:58.383 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:27.964428+0000", 2023-09-23T20:34:58.383 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:34:58.383 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.383 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.383 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:34:58.383 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:27.964433+0000", 2023-09-23T20:34:58.383 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.3770000000000004e-06 2023-09-23T20:34:58.383 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.383 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.383 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:34:58.384 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:27.964445+0000", 2023-09-23T20:34:58.384 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1575e-05 2023-09-23T20:34:58.384 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.384 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.384 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:34:58.384 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:27.964450+0000", 2023-09-23T20:34:58.384 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.0050000000000004e-06 2023-09-23T20:34:58.384 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.384 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.384 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:34:58.384 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:27.964495+0000", 2023-09-23T20:34:58.384 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.4684000000000001e-05 2023-09-23T20:34:58.385 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.385 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.385 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:34:58.385 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:27.964626+0000", 2023-09-23T20:34:58.385 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00013076299999999999 2023-09-23T20:34:58.385 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.385 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.385 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:34:58.385 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:27.964744+0000", 2023-09-23T20:34:58.385 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000118002 2023-09-23T20:34:58.385 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.386 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.386 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:34:58.386 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:27.968292+0000", 2023-09-23T20:34:58.386 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0035486200000000002 2023-09-23T20:34:58.386 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.386 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.386 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:34:58.386 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:27.981305+0000", 2023-09-23T20:34:58.386 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.013012786 2023-09-23T20:34:58.386 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.386 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.387 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:34:58.387 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:28.012442+0000", 2023-09-23T20:34:58.387 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.031137578999999999 2023-09-23T20:34:58.387 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.387 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.387 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:34:58.387 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:28.012486+0000", 2023-09-23T20:34:58.387 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.3832000000000003e-05 2023-09-23T20:34:58.387 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.387 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.387 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:34:58.387 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:28.012525+0000", 2023-09-23T20:34:58.388 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.8175e-05 2023-09-23T20:34:58.388 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:34:58.388 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:34:58.388 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:34:58.388 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.388 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.388 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35700.0:137859 7.0 7:0e3ef151:::benchmark_data_smithi018_308601_object43307:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e345)", 2023-09-23T20:34:58.388 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:34:31.650683+0000", 2023-09-23T20:34:58.388 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 26.593895736, 2023-09-23T20:34:58.388 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.046408575000000001, 2023-09-23T20:34:58.388 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:34:58.388 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:34:58.389 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:34:58.389 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35700", 2023-09-23T20:34:58.389 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:34:58.389 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 137859 2023-09-23T20:34:58.389 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.389 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:34:58.389 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.389 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:34:58.389 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:31.650683+0000", 2023-09-23T20:34:58.389 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:34:58.389 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.390 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.390 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:34:58.390 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:31.650683+0000", 2023-09-23T20:34:58.390 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:34:58.390 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.390 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.390 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:34:58.390 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:31.650681+0000", 2023-09-23T20:34:58.390 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999981 2023-09-23T20:34:58.390 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.390 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.390 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:34:58.391 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:31.650690+0000", 2023-09-23T20:34:58.391 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.6889999999999993e-06 2023-09-23T20:34:58.391 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.391 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.391 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:34:58.391 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:31.650710+0000", 2023-09-23T20:34:58.391 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.9488999999999999e-05 2023-09-23T20:34:58.391 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.391 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.391 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:34:58.391 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:31.650717+0000", 2023-09-23T20:34:58.392 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.5270000000000001e-06 2023-09-23T20:34:58.392 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.392 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.392 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:34:58.392 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:31.650776+0000", 2023-09-23T20:34:58.392 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.8943999999999997e-05 2023-09-23T20:34:58.392 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.392 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.392 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:34:58.392 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:31.650886+0000", 2023-09-23T20:34:58.392 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00011011 2023-09-23T20:34:58.392 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.393 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.393 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:34:58.393 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:31.650960+0000", 2023-09-23T20:34:58.393 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.3611999999999999e-05 2023-09-23T20:34:58.393 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.393 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.393 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:34:58.393 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:31.654661+0000", 2023-09-23T20:34:58.393 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0037009159999999998 2023-09-23T20:34:58.393 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.393 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.394 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:34:58.394 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:31.667092+0000", 2023-09-23T20:34:58.394 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.012431513 2023-09-23T20:34:58.394 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.394 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.394 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:34:58.394 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:31.697021+0000", 2023-09-23T20:34:58.394 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.029929155999999998 2023-09-23T20:34:58.394 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.394 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.394 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:34:58.394 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:31.697062+0000", 2023-09-23T20:34:58.395 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.0864e-05 2023-09-23T20:34:58.395 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.395 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.395 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:34:58.395 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:31.697092+0000", 2023-09-23T20:34:58.395 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.9556999999999998e-05 2023-09-23T20:34:58.395 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:34:58.395 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:34:58.395 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:34:58.395 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.395 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.396 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35700.0:147077 7.0 7:05e07395:::benchmark_data_smithi018_308601_object52525:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e347)", 2023-09-23T20:34:58.396 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:34:44.506612+0000", 2023-09-23T20:34:58.396 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 13.73796673, 2023-09-23T20:34:58.396 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.047901842, 2023-09-23T20:34:58.396 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:34:58.396 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:34:58.396 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:34:58.396 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35700", 2023-09-23T20:34:58.396 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:34:58.396 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 147077 2023-09-23T20:34:58.396 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.396 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:34:58.397 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.397 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:34:58.397 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:44.506612+0000", 2023-09-23T20:34:58.397 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:34:58.397 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.397 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.397 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:34:58.397 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:44.506612+0000", 2023-09-23T20:34:58.397 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:34:58.397 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.397 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.398 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:34:58.398 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:44.506610+0000", 2023-09-23T20:34:58.398 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999976 2023-09-23T20:34:58.398 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.398 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.398 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:34:58.398 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:44.506620+0000", 2023-09-23T20:34:58.398 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0443e-05 2023-09-23T20:34:58.398 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.398 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.398 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:34:58.399 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:44.506642+0000", 2023-09-23T20:34:58.399 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.2019e-05 2023-09-23T20:34:58.399 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.399 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.399 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:34:58.399 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:44.506655+0000", 2023-09-23T20:34:58.399 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.2274e-05 2023-09-23T20:34:58.399 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.399 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.399 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:34:58.399 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:44.506714+0000", 2023-09-23T20:34:58.399 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.9694000000000001e-05 2023-09-23T20:34:58.400 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.400 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.400 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:34:58.400 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:44.506861+0000", 2023-09-23T20:34:58.400 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00014658600000000001 2023-09-23T20:34:58.400 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.400 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.400 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:34:58.400 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:44.507001+0000", 2023-09-23T20:34:58.400 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00014041800000000001 2023-09-23T20:34:58.400 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.400 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.401 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:34:58.401 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:44.514357+0000", 2023-09-23T20:34:58.401 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0073551650000000003 2023-09-23T20:34:58.401 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.401 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.401 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:34:58.401 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:44.534206+0000", 2023-09-23T20:34:58.401 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.019849585999999999 2023-09-23T20:34:58.401 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.401 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.401 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:34:58.402 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:44.554459+0000", 2023-09-23T20:34:58.402 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.020252288 2023-09-23T20:34:58.402 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.402 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.402 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:34:58.402 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:44.554489+0000", 2023-09-23T20:34:58.402 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.0009e-05 2023-09-23T20:34:58.402 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.402 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.402 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:34:58.402 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:44.554514+0000", 2023-09-23T20:34:58.402 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.5570000000000001e-05 2023-09-23T20:34:58.403 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:34:58.403 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:34:58.403 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:34:58.403 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.403 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.403 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35700.0:147078 7.0 7:03803315:::benchmark_data_smithi018_308601_object52526:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e347)", 2023-09-23T20:34:58.403 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:34:44.506674+0000", 2023-09-23T20:34:58.403 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 13.737904582000001, 2023-09-23T20:34:58.403 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.047969833000000003, 2023-09-23T20:34:58.403 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:34:58.403 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:34:58.404 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:34:58.404 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35700", 2023-09-23T20:34:58.404 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:34:58.404 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 147078 2023-09-23T20:34:58.404 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.404 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:34:58.404 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.404 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:34:58.404 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:44.506674+0000", 2023-09-23T20:34:58.404 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:34:58.404 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.404 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.405 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:34:58.405 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:44.506674+0000", 2023-09-23T20:34:58.405 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:34:58.405 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.405 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.405 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:34:58.405 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:44.506674+0000", 2023-09-23T20:34:58.405 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:34:58.405 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.405 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.405 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:34:58.406 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:44.506679+0000", 2023-09-23T20:34:58.406 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.7489999999999998e-06 2023-09-23T20:34:58.406 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.406 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.406 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:34:58.406 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:44.506686+0000", 2023-09-23T20:34:58.406 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.1280000000000002e-06 2023-09-23T20:34:58.406 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.406 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.406 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:34:58.406 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:44.506690+0000", 2023-09-23T20:34:58.406 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.7129999999999996e-06 2023-09-23T20:34:58.407 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.407 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.407 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:34:58.407 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:44.510760+0000", 2023-09-23T20:34:58.407 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0040694800000000003 2023-09-23T20:34:58.407 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.407 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.407 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:34:58.407 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:44.510922+0000", 2023-09-23T20:34:58.407 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000161628 2023-09-23T20:34:58.407 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.408 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.408 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:34:58.408 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:44.511069+0000", 2023-09-23T20:34:58.408 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00014759500000000001 2023-09-23T20:34:58.408 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.408 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.408 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:34:58.408 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:44.514510+0000", 2023-09-23T20:34:58.408 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0034409060000000001 2023-09-23T20:34:58.408 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.408 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.408 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:34:58.409 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:44.534238+0000", 2023-09-23T20:34:58.409 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.019727740000000001 2023-09-23T20:34:58.409 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.409 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.409 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:34:58.409 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:44.554553+0000", 2023-09-23T20:34:58.409 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.020314843999999999 2023-09-23T20:34:58.409 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.409 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.409 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:34:58.409 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:44.554603+0000", 2023-09-23T20:34:58.410 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.9897e-05 2023-09-23T20:34:58.410 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:34:58.410 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:34:58.410 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:34:58.410 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:44.554644+0000", 2023-09-23T20:34:58.410 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.1681999999999998e-05 2023-09-23T20:34:58.410 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:34:58.410 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:34:58.410 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:34:58.410 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:34:58.410 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:34:58.410 INFO:teuthology.orchestra.run.smithi149.stdout:} 2023-09-23T20:34:58.652 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T20:34:58.676 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:34:59.044 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:58 smithi018 ceph-mon[179647]: pgmap v834: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 118 active+clean; 2.1 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 705 op/s; 1/103335 objects degraded (0.001%); 460/103335 objects misplaced (0.445%); 3.2 MiB/s, 65 objects/s recovering 2023-09-23T20:34:59.045 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:58 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1057533816' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T20:34:59.045 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:58 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T20:34:59.045 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:58 smithi018 ceph-mon[184144]: pgmap v834: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 118 active+clean; 2.1 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 705 op/s; 1/103335 objects degraded (0.001%); 460/103335 objects misplaced (0.445%); 3.2 MiB/s, 65 objects/s recovering 2023-09-23T20:34:59.045 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:58 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1057533816' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T20:34:59.046 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:58 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T20:34:59.104 INFO:teuthology.orchestra.run.smithi018.stderr:reweighted osd.4 to 1 (10000) 2023-09-23T20:34:59.126 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:34:59.128 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:59.129 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:34:59.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:58 smithi149 ceph-mon[170394]: pgmap v834: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 118 active+clean; 2.1 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 705 op/s; 1/103335 objects degraded (0.001%); 460/103335 objects misplaced (0.445%); 3.2 MiB/s, 65 objects/s recovering 2023-09-23T20:34:59.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:58 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1057533816' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T20:34:59.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:58 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T20:34:59.230 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:34:59.264 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:34:59.298 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:34:59.311 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:34:59.312 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:34:59.348 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:34:59.413 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:34:59.443 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:35:00.075 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:35:00.076 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:35:00.076 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:35:00.178 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:35:00.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:59 smithi149 ceph-mon[170394]: failed to encode map e353 with expected crc 2023-09-23T20:35:00.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:59 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T20:35:00.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:59 smithi149 ceph-mon[170394]: osdmap e353: 8 total, 8 up, 8 in 2023-09-23T20:35:00.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:59 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:35:00.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:59 smithi149 ceph-mon[170394]: 1.a scrub starts 2023-09-23T20:35:00.218 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:59 smithi149 ceph-mon[170394]: 1.a scrub ok 2023-09-23T20:35:00.218 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:59 smithi149 ceph-mon[170394]: 1.d scrub starts 2023-09-23T20:35:00.218 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:34:59 smithi149 ceph-mon[170394]: 1.d scrub ok 2023-09-23T20:35:00.226 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:35:00.265 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:35:00.269 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:35:00.270 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:35:00.290 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:59 smithi018 ceph-mon[179647]: failed to encode map e353 with expected crc 2023-09-23T20:35:00.290 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:59 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T20:35:00.291 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:59 smithi018 ceph-mon[179647]: osdmap e353: 8 total, 8 up, 8 in 2023-09-23T20:35:00.291 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:59 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:35:00.291 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:59 smithi018 ceph-mon[179647]: 1.a scrub starts 2023-09-23T20:35:00.291 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:59 smithi018 ceph-mon[179647]: 1.a scrub ok 2023-09-23T20:35:00.291 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:59 smithi018 ceph-mon[179647]: 1.d scrub starts 2023-09-23T20:35:00.292 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:34:59 smithi018 ceph-mon[179647]: 1.d scrub ok 2023-09-23T20:35:00.292 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:59 smithi018 ceph-mon[184144]: failed to encode map e353 with expected crc 2023-09-23T20:35:00.292 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:59 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T20:35:00.292 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:59 smithi018 ceph-mon[184144]: osdmap e353: 8 total, 8 up, 8 in 2023-09-23T20:35:00.292 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:59 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:35:00.293 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:59 smithi018 ceph-mon[184144]: 1.a scrub starts 2023-09-23T20:35:00.293 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:59 smithi018 ceph-mon[184144]: 1.a scrub ok 2023-09-23T20:35:00.293 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:59 smithi018 ceph-mon[184144]: 1.d scrub starts 2023-09-23T20:35:00.293 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:34:59 smithi018 ceph-mon[184144]: 1.d scrub ok 2023-09-23T20:35:00.371 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:35:00.399 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:35:00.537 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:35:00.542 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:35:00.543 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:35:00.644 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:35:00.673 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:35:00.710 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:35:00.713 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:35:00.714 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:35:00.725 INFO:teuthology.orchestra.run.smithi149.stdout:{ 2023-09-23T20:35:00.725 INFO:teuthology.orchestra.run.smithi149.stdout: "ops": [], 2023-09-23T20:35:00.725 INFO:teuthology.orchestra.run.smithi149.stdout: "num_ops": 0 2023-09-23T20:35:00.725 INFO:teuthology.orchestra.run.smithi149.stdout:} 2023-09-23T20:35:00.815 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:35:00.847 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:35:01.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:00 smithi018 ceph-mon[179647]: pgmap v836: 119 pgs: 4 peering, 1 active+recovering+undersized+degraded+remapped, 114 active+clean; 2.1 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 633 op/s; 1/102048 objects degraded (0.001%); 460/102048 objects misplaced (0.451%); 3.2 MiB/s, 65 objects/s recovering 2023-09-23T20:35:01.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:00 smithi018 ceph-mon[179647]: failed to encode map e354 with expected crc 2023-09-23T20:35:01.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:00 smithi018 ceph-mon[179647]: failed to encode map e354 with expected crc 2023-09-23T20:35:01.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:00 smithi018 ceph-mon[179647]: failed to encode map e354 with expected crc 2023-09-23T20:35:01.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:00 smithi018 ceph-mon[179647]: failed to encode map e354 with expected crc 2023-09-23T20:35:01.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:00 smithi018 ceph-mon[179647]: osdmap e354: 8 total, 8 up, 8 in 2023-09-23T20:35:01.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:00 smithi018 ceph-mon[179647]: failed to encode map e354 with expected crc 2023-09-23T20:35:01.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:00 smithi018 ceph-mon[179647]: failed to encode map e354 with expected crc 2023-09-23T20:35:01.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:00 smithi018 ceph-mon[179647]: failed to encode map e354 with expected crc 2023-09-23T20:35:01.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:00 smithi018 ceph-mon[179647]: failed to encode map e354 with expected crc 2023-09-23T20:35:01.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:00 smithi018 ceph-mon[179647]: 1.6 scrub starts 2023-09-23T20:35:01.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:00 smithi018 ceph-mon[179647]: 1.6 scrub ok 2023-09-23T20:35:01.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:00 smithi018 ceph-mon[179647]: 1.8 scrub starts 2023-09-23T20:35:01.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:00 smithi018 ceph-mon[179647]: 1.8 scrub ok 2023-09-23T20:35:01.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:00 smithi018 ceph-mon[184144]: pgmap v836: 119 pgs: 4 peering, 1 active+recovering+undersized+degraded+remapped, 114 active+clean; 2.1 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 633 op/s; 1/102048 objects degraded (0.001%); 460/102048 objects misplaced (0.451%); 3.2 MiB/s, 65 objects/s recovering 2023-09-23T20:35:01.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:00 smithi018 ceph-mon[184144]: failed to encode map e354 with expected crc 2023-09-23T20:35:01.144 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:00 smithi018 ceph-mon[184144]: failed to encode map e354 with expected crc 2023-09-23T20:35:01.144 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:00 smithi018 ceph-mon[184144]: failed to encode map e354 with expected crc 2023-09-23T20:35:01.144 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:00 smithi018 ceph-mon[184144]: failed to encode map e354 with expected crc 2023-09-23T20:35:01.144 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:00 smithi018 ceph-mon[184144]: osdmap e354: 8 total, 8 up, 8 in 2023-09-23T20:35:01.144 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:00 smithi018 ceph-mon[184144]: failed to encode map e354 with expected crc 2023-09-23T20:35:01.144 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:00 smithi018 ceph-mon[184144]: failed to encode map e354 with expected crc 2023-09-23T20:35:01.144 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:00 smithi018 ceph-mon[184144]: failed to encode map e354 with expected crc 2023-09-23T20:35:01.144 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:00 smithi018 ceph-mon[184144]: failed to encode map e354 with expected crc 2023-09-23T20:35:01.144 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:00 smithi018 ceph-mon[184144]: 1.6 scrub starts 2023-09-23T20:35:01.144 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:00 smithi018 ceph-mon[184144]: 1.6 scrub ok 2023-09-23T20:35:01.144 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:00 smithi018 ceph-mon[184144]: 1.8 scrub starts 2023-09-23T20:35:01.145 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:00 smithi018 ceph-mon[184144]: 1.8 scrub ok 2023-09-23T20:35:01.181 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:00 smithi149 ceph-mon[170394]: pgmap v836: 119 pgs: 4 peering, 1 active+recovering+undersized+degraded+remapped, 114 active+clean; 2.1 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 633 op/s; 1/102048 objects degraded (0.001%); 460/102048 objects misplaced (0.451%); 3.2 MiB/s, 65 objects/s recovering 2023-09-23T20:35:01.182 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:00 smithi149 ceph-mon[170394]: failed to encode map e354 with expected crc 2023-09-23T20:35:01.182 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:00 smithi149 ceph-mon[170394]: failed to encode map e354 with expected crc 2023-09-23T20:35:01.182 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:00 smithi149 ceph-mon[170394]: failed to encode map e354 with expected crc 2023-09-23T20:35:01.182 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:00 smithi149 ceph-mon[170394]: failed to encode map e354 with expected crc 2023-09-23T20:35:01.183 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:00 smithi149 ceph-mon[170394]: osdmap e354: 8 total, 8 up, 8 in 2023-09-23T20:35:01.183 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:00 smithi149 ceph-mon[170394]: failed to encode map e354 with expected crc 2023-09-23T20:35:01.183 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:00 smithi149 ceph-mon[170394]: failed to encode map e354 with expected crc 2023-09-23T20:35:01.183 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:00 smithi149 ceph-mon[170394]: failed to encode map e354 with expected crc 2023-09-23T20:35:01.183 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:00 smithi149 ceph-mon[170394]: failed to encode map e354 with expected crc 2023-09-23T20:35:01.184 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:00 smithi149 ceph-mon[170394]: 1.6 scrub starts 2023-09-23T20:35:01.184 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:00 smithi149 ceph-mon[170394]: 1.6 scrub ok 2023-09-23T20:35:01.184 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:00 smithi149 ceph-mon[170394]: 1.8 scrub starts 2023-09-23T20:35:01.184 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:00 smithi149 ceph-mon[170394]: 1.8 scrub ok 2023-09-23T20:35:01.239 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T20:35:01.548 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:35:01.574 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:35:01.575 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:35:01.604 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-09-23T20:35:01.604 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:35:01.676 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:35:01.720 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:35:01.767 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:35:01.771 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:35:01.774 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:35:01.775 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:35:01.876 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:35:01.909 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:35:01.949 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:35:01.955 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:35:01.956 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:35:01.978 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:01 smithi149 ceph-mon[170394]: failed to encode map e355 with expected crc 2023-09-23T20:35:01.978 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:01 smithi149 ceph-mon[170394]: osdmap e355: 8 total, 8 up, 8 in 2023-09-23T20:35:01.978 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:01 smithi149 ceph-mon[170394]: failed to encode map e355 with expected crc 2023-09-23T20:35:01.978 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:01 smithi149 ceph-mon[170394]: 7.1 starting backfill to osd.4 from (0'0,321'4660] MIN to 352'9714 2023-09-23T20:35:01.979 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:01 smithi149 ceph-mon[170394]: 7.9 starting backfill to osd.4 from (0'0,0'0] MAX to 352'9697 2023-09-23T20:35:01.979 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:01 smithi149 ceph-mon[170394]: failed to encode map e355 with expected crc 2023-09-23T20:35:01.979 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:01 smithi149 ceph-mon[170394]: failed to encode map e355 with expected crc 2023-09-23T20:35:01.979 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:01 smithi149 ceph-mon[170394]: failed to encode map e355 with expected crc 2023-09-23T20:35:01.979 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:01 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 1/102048 objects degraded (0.001%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T20:35:01.979 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:01 smithi149 ceph-mon[170394]: osdmap e356: 8 total, 8 up, 8 in 2023-09-23T20:35:01.979 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:01 smithi149 ceph-mon[170394]: failed to encode map e356 with expected crc 2023-09-23T20:35:01.979 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:01 smithi149 ceph-mon[170394]: failed to encode map e356 with expected crc 2023-09-23T20:35:01.979 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:01 smithi149 ceph-mon[170394]: failed to encode map e356 with expected crc 2023-09-23T20:35:01.992 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:35:02.057 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:35:02.090 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:35:02.136 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:35:02.139 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:35:02.140 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:35:02.241 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:35:02.247 INFO:teuthology.orchestra.run.smithi018.stderr:nodeep-scrub is unset 2023-09-23T20:35:02.285 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:01 smithi018 ceph-mon[184144]: failed to encode map e355 with expected crc 2023-09-23T20:35:02.286 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:01 smithi018 ceph-mon[184144]: osdmap e355: 8 total, 8 up, 8 in 2023-09-23T20:35:02.286 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:01 smithi018 ceph-mon[184144]: failed to encode map e355 with expected crc 2023-09-23T20:35:02.286 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:01 smithi018 ceph-mon[184144]: 7.1 starting backfill to osd.4 from (0'0,321'4660] MIN to 352'9714 2023-09-23T20:35:02.286 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:01 smithi018 ceph-mon[184144]: 7.9 starting backfill to osd.4 from (0'0,0'0] MAX to 352'9697 2023-09-23T20:35:02.286 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:01 smithi018 ceph-mon[184144]: failed to encode map e355 with expected crc 2023-09-23T20:35:02.286 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:01 smithi018 ceph-mon[184144]: failed to encode map e355 with expected crc 2023-09-23T20:35:02.286 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:01 smithi018 ceph-mon[184144]: failed to encode map e355 with expected crc 2023-09-23T20:35:02.287 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:01 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 1/102048 objects degraded (0.001%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T20:35:02.287 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:01 smithi018 ceph-mon[184144]: osdmap e356: 8 total, 8 up, 8 in 2023-09-23T20:35:02.287 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:01 smithi018 ceph-mon[184144]: failed to encode map e356 with expected crc 2023-09-23T20:35:02.287 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:01 smithi018 ceph-mon[184144]: failed to encode map e356 with expected crc 2023-09-23T20:35:02.287 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:01 smithi018 ceph-mon[184144]: failed to encode map e356 with expected crc 2023-09-23T20:35:02.287 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:01 smithi018 ceph-mon[179647]: failed to encode map e355 with expected crc 2023-09-23T20:35:02.287 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:01 smithi018 ceph-mon[179647]: osdmap e355: 8 total, 8 up, 8 in 2023-09-23T20:35:02.287 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:01 smithi018 ceph-mon[179647]: failed to encode map e355 with expected crc 2023-09-23T20:35:02.287 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:01 smithi018 ceph-mon[179647]: 7.1 starting backfill to osd.4 from (0'0,321'4660] MIN to 352'9714 2023-09-23T20:35:02.288 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:01 smithi018 ceph-mon[179647]: 7.9 starting backfill to osd.4 from (0'0,0'0] MAX to 352'9697 2023-09-23T20:35:02.288 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:01 smithi018 ceph-mon[179647]: failed to encode map e355 with expected crc 2023-09-23T20:35:02.288 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:01 smithi018 ceph-mon[179647]: failed to encode map e355 with expected crc 2023-09-23T20:35:02.288 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:01 smithi018 ceph-mon[179647]: failed to encode map e355 with expected crc 2023-09-23T20:35:02.288 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:01 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 1/102048 objects degraded (0.001%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T20:35:02.288 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:01 smithi018 ceph-mon[179647]: osdmap e356: 8 total, 8 up, 8 in 2023-09-23T20:35:02.288 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:01 smithi018 ceph-mon[179647]: failed to encode map e356 with expected crc 2023-09-23T20:35:02.288 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:01 smithi018 ceph-mon[179647]: failed to encode map e356 with expected crc 2023-09-23T20:35:02.288 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:01 smithi018 ceph-mon[179647]: failed to encode map e356 with expected crc 2023-09-23T20:35:02.344 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:35:02.782 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:35:02.799 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:35:02.799 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:35:02.900 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:35:02.998 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:35:03.078 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:02 smithi018 ceph-mon[179647]: pgmap v840: 119 pgs: 1 active+recovering+undersized+remapped, 2 remapped+peering, 4 peering, 112 active+clean; 2.0 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 472 op/s; 1179/97662 objects misplaced (1.207%); 4.9 MiB/s, 78 objects/s recovering 2023-09-23T20:35:03.078 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:02 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/958067415' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:35:03.078 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:02 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:35:03.078 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:02 smithi018 ceph-mon[179647]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:35:03.078 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:02 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:35:03.079 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:02 smithi018 ceph-mon[179647]: osdmap e357: 8 total, 8 up, 8 in 2023-09-23T20:35:03.079 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:02 smithi018 ceph-mon[179647]: failed to encode map e357 with expected crc 2023-09-23T20:35:03.079 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:02 smithi018 ceph-mon[179647]: failed to encode map e357 with expected crc 2023-09-23T20:35:03.079 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:02 smithi018 ceph-mon[179647]: failed to encode map e357 with expected crc 2023-09-23T20:35:03.079 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:02 smithi018 ceph-mon[179647]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1/102048 objects degraded (0.001%), 1 pg degraded, 1 pg undersized) 2023-09-23T20:35:03.079 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:02 smithi018 ceph-mon[179647]: Cluster is now healthy 2023-09-23T20:35:03.079 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:02 smithi018 ceph-mon[179647]: failed to encode map e357 with expected crc 2023-09-23T20:35:03.079 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:02 smithi018 ceph-mon[179647]: 1.9 deep-scrub starts 2023-09-23T20:35:03.080 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:35:03.081 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:02 smithi018 ceph-mon[184144]: pgmap v840: 119 pgs: 1 active+recovering+undersized+remapped, 2 remapped+peering, 4 peering, 112 active+clean; 2.0 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 472 op/s; 1179/97662 objects misplaced (1.207%); 4.9 MiB/s, 78 objects/s recovering 2023-09-23T20:35:03.081 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:02 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/958067415' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:35:03.081 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:02 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:35:03.081 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:02 smithi018 ceph-mon[184144]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:35:03.081 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:02 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:35:03.081 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:02 smithi018 ceph-mon[184144]: osdmap e357: 8 total, 8 up, 8 in 2023-09-23T20:35:03.081 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:02 smithi018 ceph-mon[184144]: failed to encode map e357 with expected crc 2023-09-23T20:35:03.082 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:02 smithi018 ceph-mon[184144]: failed to encode map e357 with expected crc 2023-09-23T20:35:03.082 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:02 smithi018 ceph-mon[184144]: failed to encode map e357 with expected crc 2023-09-23T20:35:03.082 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:02 smithi018 ceph-mon[184144]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1/102048 objects degraded (0.001%), 1 pg degraded, 1 pg undersized) 2023-09-23T20:35:03.082 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:02 smithi018 ceph-mon[184144]: Cluster is now healthy 2023-09-23T20:35:03.082 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:02 smithi018 ceph-mon[184144]: failed to encode map e357 with expected crc 2023-09-23T20:35:03.082 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:02 smithi018 ceph-mon[184144]: 1.9 deep-scrub starts 2023-09-23T20:35:03.083 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:35:03.084 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:35:03.185 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:35:03.205 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:35:03.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:02 smithi149 ceph-mon[170394]: pgmap v840: 119 pgs: 1 active+recovering+undersized+remapped, 2 remapped+peering, 4 peering, 112 active+clean; 2.0 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 472 op/s; 1179/97662 objects misplaced (1.207%); 4.9 MiB/s, 78 objects/s recovering 2023-09-23T20:35:03.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:02 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/958067415' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:35:03.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:02 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:35:03.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:02 smithi149 ceph-mon[170394]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:35:03.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:02 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:35:03.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:02 smithi149 ceph-mon[170394]: osdmap e357: 8 total, 8 up, 8 in 2023-09-23T20:35:03.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:02 smithi149 ceph-mon[170394]: failed to encode map e357 with expected crc 2023-09-23T20:35:03.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:02 smithi149 ceph-mon[170394]: failed to encode map e357 with expected crc 2023-09-23T20:35:03.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:02 smithi149 ceph-mon[170394]: failed to encode map e357 with expected crc 2023-09-23T20:35:03.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:02 smithi149 ceph-mon[170394]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1/102048 objects degraded (0.001%), 1 pg degraded, 1 pg undersized) 2023-09-23T20:35:03.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:02 smithi149 ceph-mon[170394]: Cluster is now healthy 2023-09-23T20:35:03.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:02 smithi149 ceph-mon[170394]: failed to encode map e357 with expected crc 2023-09-23T20:35:03.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:02 smithi149 ceph-mon[170394]: 1.9 deep-scrub starts 2023-09-23T20:35:03.230 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:35:03.282 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:35:03.285 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:35:03.286 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:35:03.387 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:35:03.486 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:35:03.637 INFO:teuthology.orchestra.run.smithi149.stdout:{ 2023-09-23T20:35:03.637 INFO:teuthology.orchestra.run.smithi149.stdout: "ops": [], 2023-09-23T20:35:03.637 INFO:teuthology.orchestra.run.smithi149.stdout: "complaint_time": 30, 2023-09-23T20:35:03.637 INFO:teuthology.orchestra.run.smithi149.stdout: "num_blocked_ops": 0 2023-09-23T20:35:03.637 INFO:teuthology.orchestra.run.smithi149.stdout:} 2023-09-23T20:35:03.647 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:35:03.660 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:35:03.660 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:35:03.671 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:35:03.726 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:35:03.762 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:35:03.777 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:35:03.803 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:35:03.816 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:35:03.867 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:35:03.917 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:35:03.957 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:35:04.004 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:35:04.024 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:35:04.024 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:35:04.024 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:35:04.024 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:35:04.024 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:35:04.024 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:35:04.025 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:35:04.025 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:35:04.042 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:35:04.045 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:35:04.045 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:35:04.097 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T20:35:04.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:04 smithi018 ceph-mon[184144]: 1.9 deep-scrub ok 2023-09-23T20:35:04.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:04 smithi018 ceph-mon[184144]: 1.18 scrub starts 2023-09-23T20:35:04.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:04 smithi018 ceph-mon[184144]: 1.18 scrub ok 2023-09-23T20:35:04.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:04 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3064605142' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:35:04.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:04 smithi018 ceph-mon[179647]: 1.9 deep-scrub ok 2023-09-23T20:35:04.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:04 smithi018 ceph-mon[179647]: 1.18 scrub starts 2023-09-23T20:35:04.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:04 smithi018 ceph-mon[179647]: 1.18 scrub ok 2023-09-23T20:35:04.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:04 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3064605142' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:35:04.146 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:35:04.203 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:35:04.262 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:35:04.267 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:35:04.268 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:35:04.369 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:35:04.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:04 smithi149 ceph-mon[170394]: 1.9 deep-scrub ok 2023-09-23T20:35:04.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:04 smithi149 ceph-mon[170394]: 1.18 scrub starts 2023-09-23T20:35:04.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:04 smithi149 ceph-mon[170394]: 1.18 scrub ok 2023-09-23T20:35:04.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:04 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3064605142' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:35:04.517 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:35:04.628 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:35:04.637 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:35:04.637 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:35:04.738 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:35:04.775 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:35:04.794 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:35:04.837 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:35:04.847 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:35:04.848 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:35:04.948 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:35:04.993 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:35:05.028 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:05 smithi018 ceph-mon[179647]: pgmap v842: 119 pgs: 1 active+recovering+undersized+remapped, 2 remapped+peering, 4 peering, 112 active+clean; 2.0 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 505 op/s; 1179/95982 objects misplaced (1.228%); 4.9 MiB/s, 78 objects/s recovering 2023-09-23T20:35:05.028 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:05 smithi018 ceph-mon[179647]: 1.7 scrub starts 2023-09-23T20:35:05.028 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:05 smithi018 ceph-mon[179647]: 1.7 scrub ok 2023-09-23T20:35:05.028 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:35:04 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:35:04] "GET /metrics HTTP/1.1" 200 36524 "" "Prometheus/2.43.0" 2023-09-23T20:35:05.032 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:05 smithi149 ceph-mon[170394]: pgmap v842: 119 pgs: 1 active+recovering+undersized+remapped, 2 remapped+peering, 4 peering, 112 active+clean; 2.0 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 505 op/s; 1179/95982 objects misplaced (1.228%); 4.9 MiB/s, 78 objects/s recovering 2023-09-23T20:35:05.032 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:05 smithi149 ceph-mon[170394]: 1.7 scrub starts 2023-09-23T20:35:05.032 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:05 smithi149 ceph-mon[170394]: 1.7 scrub ok 2023-09-23T20:35:05.049 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:35:05.056 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:35:05.056 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:35:05.118 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T20:35:05.156 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:35:05.191 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:35:05.389 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:35:05.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:05 smithi018 ceph-mon[184144]: pgmap v842: 119 pgs: 1 active+recovering+undersized+remapped, 2 remapped+peering, 4 peering, 112 active+clean; 2.0 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 505 op/s; 1179/95982 objects misplaced (1.228%); 4.9 MiB/s, 78 objects/s recovering 2023-09-23T20:35:05.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:05 smithi018 ceph-mon[184144]: 1.7 scrub starts 2023-09-23T20:35:05.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:05 smithi018 ceph-mon[184144]: 1.7 scrub ok 2023-09-23T20:35:05.392 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:35:05.393 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:35:05.494 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:35:05.504 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:35:05.523 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:35:05.562 INFO:teuthology.orchestra.run.smithi018.stderr:dumped all 2023-09-23T20:35:05.630 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:35:05.635 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:35:05.635 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:35:05.727 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:35:05.736 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:35:05.783 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:35:05.864 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:35:05.867 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:35:05.868 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:35:05.969 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:35:06.042 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:35:06.147 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:35:06.154 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:35:06.155 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:35:06.250 INFO:teuthology.orchestra.run.smithi149.stdout:ERROR: (22) Invalid argument 2023-09-23T20:35:06.250 INFO:teuthology.orchestra.run.smithi149.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:35:06.255 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:35:06.285 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:06 smithi149 ceph-mon[170394]: from='client.45752 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:35:06.304 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:35:06.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:06 smithi018 ceph-mon[184144]: from='client.45752 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:35:06.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:06 smithi018 ceph-mon[179647]: from='client.45752 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:35:06.405 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:35:06.408 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:35:06.409 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:35:06.510 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:35:06.620 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:35:06.677 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:35:06.808 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T20:35:07.513 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:35:07.531 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:07 smithi018 ceph-mon[179647]: pgmap v843: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 116 active+clean; 1.8 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 7.1 KiB/s rd, 19 KiB/s wr, 707 op/s; 4709/89589 objects misplaced (5.256%); 5.4 MiB/s, 86 objects/s recovering 2023-09-23T20:35:07.531 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:07 smithi018 ceph-mon[179647]: 1.16 deep-scrub starts 2023-09-23T20:35:07.531 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:07 smithi018 ceph-mon[179647]: 1.1a scrub starts 2023-09-23T20:35:07.532 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:07 smithi018 ceph-mon[179647]: 1.1a scrub ok 2023-09-23T20:35:07.532 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:07 smithi018 ceph-mon[184144]: pgmap v843: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 116 active+clean; 1.8 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 7.1 KiB/s rd, 19 KiB/s wr, 707 op/s; 4709/89589 objects misplaced (5.256%); 5.4 MiB/s, 86 objects/s recovering 2023-09-23T20:35:07.532 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:07 smithi018 ceph-mon[184144]: 1.16 deep-scrub starts 2023-09-23T20:35:07.532 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:07 smithi018 ceph-mon[184144]: 1.1a scrub starts 2023-09-23T20:35:07.532 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:07 smithi018 ceph-mon[184144]: 1.1a scrub ok 2023-09-23T20:35:07.719 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:07 smithi149 ceph-mon[170394]: pgmap v843: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 116 active+clean; 1.8 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 7.1 KiB/s rd, 19 KiB/s wr, 707 op/s; 4709/89589 objects misplaced (5.256%); 5.4 MiB/s, 86 objects/s recovering 2023-09-23T20:35:07.720 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:07 smithi149 ceph-mon[170394]: 1.16 deep-scrub starts 2023-09-23T20:35:07.720 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:07 smithi149 ceph-mon[170394]: 1.1a scrub starts 2023-09-23T20:35:07.720 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:07 smithi149 ceph-mon[170394]: 1.1a scrub ok 2023-09-23T20:35:08.219 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:35:07 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[153650]: logger=cleanup t=2023-09-23T20:35:07.962641416Z level=info msg="Completed cleanup jobs" duration=48.656817ms 2023-09-23T20:35:08.334 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:35:08.356 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:35:08.357 INFO:teuthology.orchestra.run.smithi018.stdout:{"pg_ready":true,"pg_map":{"version":842,"stamp":"2023-09-23T20:35:03.884732+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2099430006,"num_objects":31994,"num_object_clones":0,"num_object_copies":95982,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1179,"num_objects_unfound":0,"num_objects_dirty":31994,"num_whiteouts":0,"num_read":5996,"num_read_kb":13213,"num_write":158041,"num_write_kb":6063862,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9825,"num_bytes_recovered":579148192,"num_keys_recovered":40,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":112748,"ondisk_log_size":112748,"up":357,"acting":355,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":360,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":9414408,"kb_used_data":6631496,"kb_used_omap":32,"kb_used_meta":2782623,"kb_avail":740546808,"statfs":{"total":767960285184,"available":758319931392,"internally_reserved":0,"allocated":6790651904,"data_stored":6778970516,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32773,"internal_metadata":2849406971},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,6,5,4,1,5,5],"upper_bound":512},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-132513792,"num_objects":-2022,"num_object_clones":0,"num_object_copies":-6066,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-1,"num_objects_misplaced":719,"num_objects_unfound":0,"num_objects_dirty":-2022,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2022,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":314,"num_bytes_recovered":20381696,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"4.003547"},"pg_stats":[{"pgid":"1.18","version":"90'161","reported_seq":594,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T20:34:58.968314+0000","last_change":"2023-09-23T20:34:27.305980+0000","last_active":"2023-09-23T20:34:58.968314+0000","last_peered":"2023-09-23T20:34:58.968314+0000","last_clean":"2023-09-23T20:34:58.968314+0000","last_became_active":"2023-09-23T20:33:39.039296+0000","last_became_peered":"2023-09-23T20:33:39.039296+0000","last_unstale":"2023-09-23T20:34:58.968314+0000","last_undegraded":"2023-09-23T20:34:58.968314+0000","last_fullsized":"2023-09-23T20:34:58.968314+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:31.756974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:31.756974+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0031802670000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131344,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"69'1","reported_seq":51,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T20:34:58.961183+0000","last_change":"2023-09-23T20:32:33.233853+0000","last_active":"2023-09-23T20:34:58.961183+0000","last_peered":"2023-09-23T20:34:58.961183+0000","last_clean":"2023-09-23T20:34:58.961183+0000","last_became_active":"2023-09-23T20:32:33.233546+0000","last_became_peered":"2023-09-23T20:32:33.233546+0000","last_unstale":"2023-09-23T20:34:58.961183+0000","last_undegraded":"2023-09-23T20:34:58.961183+0000","last_fullsized":"2023-09-23T20:34:58.961183+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:50:27.130808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":231,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-09-23T20:34:54.610866+0000","last_change":"2023-09-23T20:32:50.536495+0000","last_active":"2023-09-23T20:34:54.610866+0000","last_peered":"2023-09-23T20:34:54.610866+0000","last_clean":"2023-09-23T20:34:54.610866+0000","last_became_active":"2023-09-23T20:32:50.536180+0000","last_became_peered":"2023-09-23T20:32:50.536180+0000","last_unstale":"2023-09-23T20:34:54.610866+0000","last_undegraded":"2023-09-23T20:34:54.610866+0000","last_fullsized":"2023-09-23T20:34:54.610866+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:01:05.524308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1b","version":"90'161","reported_seq":604,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-09-23T20:34:54.613674+0000","last_change":"2023-09-23T20:33:39.013809+0000","last_active":"2023-09-23T20:34:54.613674+0000","last_peered":"2023-09-23T20:34:54.613674+0000","last_clean":"2023-09-23T20:34:54.613674+0000","last_became_active":"2023-09-23T20:33:39.013467+0000","last_became_peered":"2023-09-23T20:33:39.013467+0000","last_unstale":"2023-09-23T20:34:54.613674+0000","last_undegraded":"2023-09-23T20:34:54.613674+0000","last_fullsized":"2023-09-23T20:34:54.613674+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:46.558292+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:46.558292+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0023048779999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":311,"reported_epoch":353,"state":"peering","last_fresh":"2023-09-23T20:34:58.973285+0000","last_change":"2023-09-23T20:34:58.973442+0000","last_active":"2023-09-23T20:34:58.973285+0000","last_peered":"2023-09-23T20:34:54.602650+0000","last_clean":"2023-09-23T20:34:54.602650+0000","last_became_active":"2023-09-23T20:33:00.181400+0000","last_became_peered":"2023-09-23T20:33:00.181400+0000","last_unstale":"2023-09-23T20:34:58.973285+0000","last_undegraded":"2023-09-23T20:34:58.973285+0000","last_fullsized":"2023-09-23T20:34:58.973285+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:46:48.144113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[3,4,6],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":321,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T20:34:58.974463+0000","last_change":"2023-09-23T20:25:02.957185+0000","last_active":"2023-09-23T20:34:58.974463+0000","last_peered":"2023-09-23T20:34:58.974463+0000","last_clean":"2023-09-23T20:34:58.974463+0000","last_became_active":"2023-09-23T20:25:02.956689+0000","last_became_peered":"2023-09-23T20:25:02.956689+0000","last_unstale":"2023-09-23T20:34:58.974463+0000","last_undegraded":"2023-09-23T20:34:58.974463+0000","last_fullsized":"2023-09-23T20:34:58.974463+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:04:28.524015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"90'161","reported_seq":633,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T20:34:58.968727+0000","last_change":"2023-09-23T20:34:31.273248+0000","last_active":"2023-09-23T20:34:58.968727+0000","last_peered":"2023-09-23T20:34:58.968727+0000","last_clean":"2023-09-23T20:34:58.968727+0000","last_became_active":"2023-09-23T20:33:39.000572+0000","last_became_peered":"2023-09-23T20:33:39.000572+0000","last_unstale":"2023-09-23T20:34:58.968727+0000","last_undegraded":"2023-09-23T20:34:58.968727+0000","last_fullsized":"2023-09-23T20:34:58.968727+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:30.727267+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:30.727267+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0052560289999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":165,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T20:34:58.969988+0000","last_change":"2023-09-23T20:32:50.546428+0000","last_active":"2023-09-23T20:34:58.969988+0000","last_peered":"2023-09-23T20:34:58.969988+0000","last_clean":"2023-09-23T20:34:58.969988+0000","last_became_active":"2023-09-23T20:32:50.546120+0000","last_became_peered":"2023-09-23T20:32:50.546120+0000","last_unstale":"2023-09-23T20:34:58.969988+0000","last_undegraded":"2023-09-23T20:34:58.969988+0000","last_fullsized":"2023-09-23T20:34:58.969988+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:00:10.792410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":127,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T20:34:58.966724+0000","last_change":"2023-09-23T20:29:46.112977+0000","last_active":"2023-09-23T20:34:58.966724+0000","last_peered":"2023-09-23T20:34:58.966724+0000","last_clean":"2023-09-23T20:34:58.966724+0000","last_became_active":"2023-09-23T20:29:46.112600+0000","last_became_peered":"2023-09-23T20:29:46.112600+0000","last_unstale":"2023-09-23T20:34:58.966724+0000","last_undegraded":"2023-09-23T20:34:58.966724+0000","last_fullsized":"2023-09-23T20:34:58.966724+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:06:34.132247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"90'161","reported_seq":558,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T20:34:58.961182+0000","last_change":"2023-09-23T20:34:53.557349+0000","last_active":"2023-09-23T20:34:58.961182+0000","last_peered":"2023-09-23T20:34:58.961182+0000","last_clean":"2023-09-23T20:34:58.961182+0000","last_became_active":"2023-09-23T20:33:38.988198+0000","last_became_peered":"2023-09-23T20:33:38.988198+0000","last_unstale":"2023-09-23T20:34:58.961182+0000","last_undegraded":"2023-09-23T20:34:58.961182+0000","last_fullsized":"2023-09-23T20:34:58.961182+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:44.509012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:44.509012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0032186319999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":319,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T20:34:58.974472+0000","last_change":"2023-09-23T20:25:02.969432+0000","last_active":"2023-09-23T20:34:58.974472+0000","last_peered":"2023-09-23T20:34:58.974472+0000","last_clean":"2023-09-23T20:34:58.974472+0000","last_became_active":"2023-09-23T20:25:02.969118+0000","last_became_peered":"2023-09-23T20:25:02.969118+0000","last_unstale":"2023-09-23T20:34:58.974472+0000","last_undegraded":"2023-09-23T20:34:58.974472+0000","last_fullsized":"2023-09-23T20:34:58.974472+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:02:22.453831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":60,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-09-23T20:35:01.219087+0000","last_change":"2023-09-23T20:32:01.360359+0000","last_active":"2023-09-23T20:35:01.219087+0000","last_peered":"2023-09-23T20:35:01.219087+0000","last_clean":"2023-09-23T20:35:01.219087+0000","last_became_active":"2023-09-23T20:32:01.360008+0000","last_became_peered":"2023-09-23T20:32:01.360008+0000","last_unstale":"2023-09-23T20:35:01.219087+0000","last_undegraded":"2023-09-23T20:35:01.219087+0000","last_fullsized":"2023-09-23T20:35:01.219087+0000","mapping_epoch":308,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:33:43.135001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.14","version":"90'161","reported_seq":684,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T20:35:02.241040+0000","last_change":"2023-09-23T20:35:02.236398+0000","last_active":"2023-09-23T20:35:02.241040+0000","last_peered":"2023-09-23T20:35:02.241040+0000","last_clean":"2023-09-23T20:35:02.241040+0000","last_became_active":"2023-09-23T20:35:02.235988+0000","last_became_peered":"2023-09-23T20:35:02.235988+0000","last_unstale":"2023-09-23T20:35:02.241040+0000","last_undegraded":"2023-09-23T20:35:02.241040+0000","last_fullsized":"2023-09-23T20:35:02.241040+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:47.479012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:47.479012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:13.519648+0000","scrub_duration":0.0020136590000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":330,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T20:34:58.968928+0000","last_change":"2023-09-23T20:29:45.523808+0000","last_active":"2023-09-23T20:34:58.968928+0000","last_peered":"2023-09-23T20:34:58.968928+0000","last_clean":"2023-09-23T20:34:58.968928+0000","last_became_active":"2023-09-23T20:29:45.523508+0000","last_became_peered":"2023-09-23T20:29:45.523508+0000","last_unstale":"2023-09-23T20:34:58.968928+0000","last_undegraded":"2023-09-23T20:34:58.968928+0000","last_fullsized":"2023-09-23T20:34:58.968928+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:50:55.565352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":153,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T20:34:58.982513+0000","last_change":"2023-09-23T20:32:50.539003+0000","last_active":"2023-09-23T20:34:58.982513+0000","last_peered":"2023-09-23T20:34:58.982513+0000","last_clean":"2023-09-23T20:34:58.982513+0000","last_became_active":"2023-09-23T20:32:50.538596+0000","last_became_peered":"2023-09-23T20:32:50.538596+0000","last_unstale":"2023-09-23T20:34:58.982513+0000","last_undegraded":"2023-09-23T20:34:58.982513+0000","last_fullsized":"2023-09-23T20:34:58.982513+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:56:03.314375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.17","version":"90'161","reported_seq":592,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T20:34:58.966807+0000","last_change":"2023-09-23T20:33:38.990504+0000","last_active":"2023-09-23T20:34:58.966807+0000","last_peered":"2023-09-23T20:34:58.966807+0000","last_clean":"2023-09-23T20:34:58.966807+0000","last_became_active":"2023-09-23T20:33:38.990149+0000","last_became_peered":"2023-09-23T20:33:38.990149+0000","last_unstale":"2023-09-23T20:34:58.966807+0000","last_undegraded":"2023-09-23T20:34:58.966807+0000","last_fullsized":"2023-09-23T20:34:58.966807+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:45.196285+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:45.196285+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0014380860000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":381,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T20:34:58.983306+0000","last_change":"2023-09-23T20:32:33.270617+0000","last_active":"2023-09-23T20:34:58.983306+0000","last_peered":"2023-09-23T20:34:58.983306+0000","last_clean":"2023-09-23T20:34:58.983306+0000","last_became_active":"2023-09-23T20:32:33.227902+0000","last_became_peered":"2023-09-23T20:32:33.227902+0000","last_unstale":"2023-09-23T20:34:58.983306+0000","last_undegraded":"2023-09-23T20:34:58.983306+0000","last_fullsized":"2023-09-23T20:34:58.983306+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:30:26.847831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":360,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":302,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-09-23T20:34:54.602310+0000","last_change":"2023-09-23T20:29:46.080766+0000","last_active":"2023-09-23T20:34:54.602310+0000","last_peered":"2023-09-23T20:34:54.602310+0000","last_clean":"2023-09-23T20:34:54.602310+0000","last_became_active":"2023-09-23T20:29:46.080356+0000","last_became_peered":"2023-09-23T20:29:46.080356+0000","last_unstale":"2023-09-23T20:34:54.602310+0000","last_undegraded":"2023-09-23T20:34:54.602310+0000","last_fullsized":"2023-09-23T20:34:54.602310+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:08:01.862270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"90'161","reported_seq":585,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T20:34:58.965523+0000","last_change":"2023-09-23T20:33:38.991452+0000","last_active":"2023-09-23T20:34:58.965523+0000","last_peered":"2023-09-23T20:34:58.965523+0000","last_clean":"2023-09-23T20:34:58.965523+0000","last_became_active":"2023-09-23T20:33:38.991153+0000","last_became_peered":"2023-09-23T20:33:38.991153+0000","last_unstale":"2023-09-23T20:34:58.965523+0000","last_undegraded":"2023-09-23T20:34:58.965523+0000","last_fullsized":"2023-09-23T20:34:58.965523+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:44.211184+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:44.211184+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0017707720000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":61,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T20:34:58.961355+0000","last_change":"2023-09-23T20:32:33.266787+0000","last_active":"2023-09-23T20:34:58.961355+0000","last_peered":"2023-09-23T20:34:58.961355+0000","last_clean":"2023-09-23T20:34:58.961355+0000","last_became_active":"2023-09-23T20:32:33.228482+0000","last_became_peered":"2023-09-23T20:32:33.228482+0000","last_unstale":"2023-09-23T20:34:58.961355+0000","last_undegraded":"2023-09-23T20:34:58.961355+0000","last_fullsized":"2023-09-23T20:34:58.961355+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:13:01.812073+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":51,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T20:34:58.961765+0000","last_change":"2023-09-23T20:32:33.228929+0000","last_active":"2023-09-23T20:34:58.961765+0000","last_peered":"2023-09-23T20:34:58.961765+0000","last_clean":"2023-09-23T20:34:58.961765+0000","last_became_active":"2023-09-23T20:32:33.228595+0000","last_became_peered":"2023-09-23T20:32:33.228595+0000","last_unstale":"2023-09-23T20:34:58.961765+0000","last_undegraded":"2023-09-23T20:34:58.961765+0000","last_fullsized":"2023-09-23T20:34:58.961765+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:17:53.053808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"90'161","reported_seq":583,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T20:34:58.971162+0000","last_change":"2023-09-23T20:31:43.578810+0000","last_active":"2023-09-23T20:34:58.971162+0000","last_peered":"2023-09-23T20:34:58.971162+0000","last_clean":"2023-09-23T20:34:58.971162+0000","last_became_active":"2023-09-23T20:28:05.991304+0000","last_became_peered":"2023-09-23T20:28:05.991304+0000","last_unstale":"2023-09-23T20:34:58.971162+0000","last_undegraded":"2023-09-23T20:34:58.971162+0000","last_fullsized":"2023-09-23T20:34:58.971162+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:43.578666+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:43.578666+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:46:18.999991+0000","scrub_duration":0.0021890379999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":380,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T20:34:58.981630+0000","last_change":"2023-09-23T20:32:33.242943+0000","last_active":"2023-09-23T20:34:58.981630+0000","last_peered":"2023-09-23T20:34:58.981630+0000","last_clean":"2023-09-23T20:34:58.981630+0000","last_became_active":"2023-09-23T20:32:33.225468+0000","last_became_peered":"2023-09-23T20:32:33.225468+0000","last_unstale":"2023-09-23T20:34:58.981630+0000","last_undegraded":"2023-09-23T20:34:58.981630+0000","last_fullsized":"2023-09-23T20:34:58.981630+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:40:47.867573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":55,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T20:34:58.968582+0000","last_change":"2023-09-23T20:32:01.358590+0000","last_active":"2023-09-23T20:34:58.968582+0000","last_peered":"2023-09-23T20:34:58.968582+0000","last_clean":"2023-09-23T20:34:58.968582+0000","last_became_active":"2023-09-23T20:32:01.358345+0000","last_became_peered":"2023-09-23T20:32:01.358345+0000","last_unstale":"2023-09-23T20:34:58.968582+0000","last_undegraded":"2023-09-23T20:34:58.968582+0000","last_fullsized":"2023-09-23T20:34:58.968582+0000","mapping_epoch":308,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:22:21.159708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.10","version":"90'161","reported_seq":584,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T20:34:58.968424+0000","last_change":"2023-09-23T20:31:49.726214+0000","last_active":"2023-09-23T20:34:58.968424+0000","last_peered":"2023-09-23T20:34:58.968424+0000","last_clean":"2023-09-23T20:34:58.968424+0000","last_became_active":"2023-09-23T20:29:07.044892+0000","last_became_peered":"2023-09-23T20:29:07.044892+0000","last_unstale":"2023-09-23T20:34:58.968424+0000","last_undegraded":"2023-09-23T20:34:58.968424+0000","last_fullsized":"2023-09-23T20:34:58.968424+0000","mapping_epoch":253,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":254,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:49.726139+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:49.726139+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:59:11.810516+0000","scrub_duration":0.0015441470000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":319,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T20:34:58.971257+0000","last_change":"2023-09-23T20:25:02.953993+0000","last_active":"2023-09-23T20:34:58.971257+0000","last_peered":"2023-09-23T20:34:58.971257+0000","last_clean":"2023-09-23T20:34:58.971257+0000","last_became_active":"2023-09-23T20:25:02.953672+0000","last_became_peered":"2023-09-23T20:25:02.953672+0000","last_unstale":"2023-09-23T20:34:58.971257+0000","last_undegraded":"2023-09-23T20:34:58.971257+0000","last_fullsized":"2023-09-23T20:34:58.971257+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:26:08.603577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":332,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T20:34:58.968953+0000","last_change":"2023-09-23T20:25:02.949584+0000","last_active":"2023-09-23T20:34:58.968953+0000","last_peered":"2023-09-23T20:34:58.968953+0000","last_clean":"2023-09-23T20:34:58.968953+0000","last_became_active":"2023-09-23T20:25:02.949132+0000","last_became_peered":"2023-09-23T20:25:02.949132+0000","last_unstale":"2023-09-23T20:34:58.968953+0000","last_undegraded":"2023-09-23T20:34:58.968953+0000","last_fullsized":"2023-09-23T20:34:58.968953+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:34:20.509823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"90'161","reported_seq":620,"reported_epoch":353,"state":"peering","last_fresh":"2023-09-23T20:34:58.976225+0000","last_change":"2023-09-23T20:34:58.976312+0000","last_active":"2023-09-23T20:34:58.976225+0000","last_peered":"2023-09-23T20:34:54.614102+0000","last_clean":"2023-09-23T20:34:54.614102+0000","last_became_active":"2023-09-23T20:33:00.179654+0000","last_became_peered":"2023-09-23T20:33:00.179654+0000","last_unstale":"2023-09-23T20:34:58.976225+0000","last_undegraded":"2023-09-23T20:34:58.976225+0000","last_fullsized":"2023-09-23T20:34:58.976225+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:42.544308+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:42.544308+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:29:09.451901+0000","scrub_duration":0.0029958179999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[0,3,4],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":337,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T20:34:58.982438+0000","last_change":"2023-09-23T20:32:33.225148+0000","last_active":"2023-09-23T20:34:58.982438+0000","last_peered":"2023-09-23T20:34:58.982438+0000","last_clean":"2023-09-23T20:34:58.982438+0000","last_became_active":"2023-09-23T20:32:33.224805+0000","last_became_peered":"2023-09-23T20:32:33.224805+0000","last_unstale":"2023-09-23T20:34:58.982438+0000","last_undegraded":"2023-09-23T20:34:58.982438+0000","last_fullsized":"2023-09-23T20:34:58.982438+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:55:30.891398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":329,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T20:34:58.974608+0000","last_change":"2023-09-23T20:25:02.957010+0000","last_active":"2023-09-23T20:34:58.974608+0000","last_peered":"2023-09-23T20:34:58.974608+0000","last_clean":"2023-09-23T20:34:58.974608+0000","last_became_active":"2023-09-23T20:25:02.956267+0000","last_became_peered":"2023-09-23T20:25:02.956267+0000","last_unstale":"2023-09-23T20:34:58.974608+0000","last_undegraded":"2023-09-23T20:34:58.974608+0000","last_fullsized":"2023-09-23T20:34:58.974608+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:10:55.299422+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"90'161","reported_seq":603,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T20:34:58.969479+0000","last_change":"2023-09-23T20:31:53.707033+0000","last_active":"2023-09-23T20:34:58.969479+0000","last_peered":"2023-09-23T20:34:58.969479+0000","last_clean":"2023-09-23T20:34:58.969479+0000","last_became_active":"2023-09-23T20:29:46.078873+0000","last_became_peered":"2023-09-23T20:29:46.078873+0000","last_unstale":"2023-09-23T20:34:58.969479+0000","last_undegraded":"2023-09-23T20:34:58.969479+0000","last_fullsized":"2023-09-23T20:34:58.969479+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:53.706934+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:53.706934+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:51:58.039657+0000","scrub_duration":0.0016840900000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":123,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T20:34:58.963591+0000","last_change":"2023-09-23T20:29:04.179171+0000","last_active":"2023-09-23T20:34:58.963591+0000","last_peered":"2023-09-23T20:34:58.963591+0000","last_clean":"2023-09-23T20:34:58.963591+0000","last_became_active":"2023-09-23T20:29:04.128672+0000","last_became_peered":"2023-09-23T20:29:04.128672+0000","last_unstale":"2023-09-23T20:34:58.963591+0000","last_undegraded":"2023-09-23T20:34:58.963591+0000","last_fullsized":"2023-09-23T20:34:58.963591+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:49:15.506106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":166,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T20:34:58.983667+0000","last_change":"2023-09-23T20:32:50.539157+0000","last_active":"2023-09-23T20:34:58.983667+0000","last_peered":"2023-09-23T20:34:58.983667+0000","last_clean":"2023-09-23T20:34:58.983667+0000","last_became_active":"2023-09-23T20:32:50.538889+0000","last_became_peered":"2023-09-23T20:32:50.538889+0000","last_unstale":"2023-09-23T20:34:58.983667+0000","last_undegraded":"2023-09-23T20:34:58.983667+0000","last_fullsized":"2023-09-23T20:34:58.983667+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:28:28.857256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.b","version":"353'9881","reported_seq":9941,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T20:34:59.068761+0000","last_change":"2023-09-23T20:31:49.152343+0000","last_active":"2023-09-23T20:34:59.068761+0000","last_peered":"2023-09-23T20:34:59.068761+0000","last_clean":"2023-09-23T20:34:59.068761+0000","last_became_active":"2023-09-23T20:31:49.152042+0000","last_became_peered":"2023-09-23T20:31:49.152042+0000","last_unstale":"2023-09-23T20:34:59.068761+0000","last_undegraded":"2023-09-23T20:34:59.068761+0000","last_fullsized":"2023-09-23T20:34:59.068761+0000","mapping_epoch":303,"log_start":"311'3590","ondisk_log_start":"311'3590","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":6291,"log_dups_size":0,"ondisk_log_size":6291,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:53:34.237468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":129433600,"num_objects":1975,"num_object_clones":0,"num_object_copies":5925,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1975,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9881,"num_write_kb":379392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.d","version":"90'161","reported_seq":646,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T20:34:59.496478+0000","last_change":"2023-09-23T20:34:59.496438+0000","last_active":"2023-09-23T20:34:59.496478+0000","last_peered":"2023-09-23T20:34:59.496478+0000","last_clean":"2023-09-23T20:34:59.496478+0000","last_became_active":"2023-09-23T20:33:38.970527+0000","last_became_peered":"2023-09-23T20:33:38.970527+0000","last_unstale":"2023-09-23T20:34:59.496478+0000","last_undegraded":"2023-09-23T20:34:59.496478+0000","last_fullsized":"2023-09-23T20:34:59.496478+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:34:59.496375+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:34:59.496375+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:46:18.244734+0000","scrub_duration":0.0017843189999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":357,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T20:34:58.983613+0000","last_change":"2023-09-23T20:32:33.283752+0000","last_active":"2023-09-23T20:34:58.983613+0000","last_peered":"2023-09-23T20:34:58.983613+0000","last_clean":"2023-09-23T20:34:58.983613+0000","last_became_active":"2023-09-23T20:32:33.230503+0000","last_became_peered":"2023-09-23T20:32:33.230503+0000","last_unstale":"2023-09-23T20:34:58.983613+0000","last_undegraded":"2023-09-23T20:34:58.983613+0000","last_fullsized":"2023-09-23T20:34:58.983613+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:29.084410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":315,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T20:34:58.968576+0000","last_change":"2023-09-23T20:29:04.128604+0000","last_active":"2023-09-23T20:34:58.968576+0000","last_peered":"2023-09-23T20:34:58.968576+0000","last_clean":"2023-09-23T20:34:58.968576+0000","last_became_active":"2023-09-23T20:29:04.128003+0000","last_became_peered":"2023-09-23T20:29:04.128003+0000","last_unstale":"2023-09-23T20:34:58.968576+0000","last_undegraded":"2023-09-23T20:34:58.968576+0000","last_fullsized":"2023-09-23T20:34:58.968576+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:50:49.385599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"357'9885","reported_seq":9956,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T20:35:02.696285+0000","last_change":"2023-09-23T20:32:50.543873+0000","last_active":"2023-09-23T20:35:02.696285+0000","last_peered":"2023-09-23T20:35:02.696285+0000","last_clean":"2023-09-23T20:35:02.696285+0000","last_became_active":"2023-09-23T20:32:50.537253+0000","last_became_peered":"2023-09-23T20:32:50.537253+0000","last_unstale":"2023-09-23T20:35:02.696285+0000","last_undegraded":"2023-09-23T20:35:02.696285+0000","last_fullsized":"2023-09-23T20:35:02.696285+0000","mapping_epoch":319,"log_start":"310'2400","ondisk_log_start":"310'2400","created":303,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":7485,"log_dups_size":0,"ondisk_log_size":7485,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:38:02.452291+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":126287872,"num_objects":1927,"num_object_clones":0,"num_object_copies":5781,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1927,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9885,"num_write_kb":377984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,5],"acting":[2,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.c","version":"90'161","reported_seq":524,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T20:34:58.982073+0000","last_change":"2023-09-23T20:33:38.989595+0000","last_active":"2023-09-23T20:34:58.982073+0000","last_peered":"2023-09-23T20:34:58.982073+0000","last_clean":"2023-09-23T20:34:58.982073+0000","last_became_active":"2023-09-23T20:33:38.989255+0000","last_became_peered":"2023-09-23T20:33:38.989255+0000","last_unstale":"2023-09-23T20:34:58.982073+0000","last_undegraded":"2023-09-23T20:34:58.982073+0000","last_fullsized":"2023-09-23T20:34:58.982073+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:41.701171+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:41.701171+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0019955509999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":54613,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":392,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-09-23T20:34:59.987981+0000","last_change":"2023-09-23T20:34:59.987981+0000","last_active":"2023-09-23T20:34:59.987981+0000","last_peered":"2023-09-23T20:34:59.987981+0000","last_clean":"2023-09-23T20:34:59.987981+0000","last_became_active":"2023-09-23T20:34:59.985368+0000","last_became_peered":"2023-09-23T20:34:59.985368+0000","last_unstale":"2023-09-23T20:34:59.987981+0000","last_undegraded":"2023-09-23T20:34:59.987981+0000","last_fullsized":"2023-09-23T20:34:59.987981+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:12:43.082493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":51,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T20:34:58.961128+0000","last_change":"2023-09-23T20:32:33.232142+0000","last_active":"2023-09-23T20:34:58.961128+0000","last_peered":"2023-09-23T20:34:58.961128+0000","last_clean":"2023-09-23T20:34:58.961128+0000","last_became_active":"2023-09-23T20:32:33.231683+0000","last_became_peered":"2023-09-23T20:32:33.231683+0000","last_unstale":"2023-09-23T20:34:58.961128+0000","last_undegraded":"2023-09-23T20:34:58.961128+0000","last_fullsized":"2023-09-23T20:34:58.961128+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:02:58.336348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"352'9697","reported_seq":12003,"reported_epoch":354,"state":"remapped+peering","last_fresh":"2023-09-23T20:34:59.969089+0000","last_change":"2023-09-23T20:34:59.969254+0000","last_active":"2023-09-23T20:34:58.975215+0000","last_peered":"2023-09-23T20:34:58.962913+0000","last_clean":"2023-09-23T20:34:58.962913+0000","last_became_active":"2023-09-23T20:34:19.577489+0000","last_became_peered":"2023-09-23T20:34:19.577489+0000","last_unstale":"2023-09-23T20:34:59.969089+0000","last_undegraded":"2023-09-23T20:34:59.969089+0000","last_fullsized":"2023-09-23T20:34:59.969089+0000","mapping_epoch":354,"log_start":"310'3100","ondisk_log_start":"310'3100","created":303,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":6597,"log_dups_size":0,"ondisk_log_size":6597,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:41:43.581871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":133103616,"num_objects":2031,"num_object_clones":0,"num_object_copies":6093,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2031,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9697,"num_write_kb":375296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1109,"num_bytes_recovered":72679424,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[1,4,7],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"90'161","reported_seq":576,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T20:34:58.966556+0000","last_change":"2023-09-23T20:32:36.302809+0000","last_active":"2023-09-23T20:34:58.966556+0000","last_peered":"2023-09-23T20:34:58.966556+0000","last_clean":"2023-09-23T20:34:58.966556+0000","last_became_active":"2023-09-23T20:32:36.302465+0000","last_became_peered":"2023-09-23T20:32:36.302465+0000","last_unstale":"2023-09-23T20:34:58.966556+0000","last_undegraded":"2023-09-23T20:34:58.966556+0000","last_fullsized":"2023-09-23T20:34:58.966556+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:46.217489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:46.217489+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:35:40.488427+0000","scrub_duration":0.001456429,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":326,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T20:34:58.983249+0000","last_change":"2023-09-23T20:29:04.184893+0000","last_active":"2023-09-23T20:34:58.983249+0000","last_peered":"2023-09-23T20:34:58.983249+0000","last_clean":"2023-09-23T20:34:58.983249+0000","last_became_active":"2023-09-23T20:29:04.152723+0000","last_became_peered":"2023-09-23T20:29:04.152723+0000","last_unstale":"2023-09-23T20:34:58.983249+0000","last_undegraded":"2023-09-23T20:34:58.983249+0000","last_fullsized":"2023-09-23T20:34:58.983249+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:51:27.017151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":127,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T20:34:58.963734+0000","last_change":"2023-09-23T20:29:46.086912+0000","last_active":"2023-09-23T20:34:58.963734+0000","last_peered":"2023-09-23T20:34:58.963734+0000","last_clean":"2023-09-23T20:34:58.963734+0000","last_became_active":"2023-09-23T20:29:46.085471+0000","last_became_peered":"2023-09-23T20:29:46.085471+0000","last_unstale":"2023-09-23T20:34:58.963734+0000","last_undegraded":"2023-09-23T20:34:58.963734+0000","last_fullsized":"2023-09-23T20:34:58.963734+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:04:55.751660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.8","version":"357'9899","reported_seq":17258,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T20:35:02.681681+0000","last_change":"2023-09-23T20:34:05.581246+0000","last_active":"2023-09-23T20:35:02.681681+0000","last_peered":"2023-09-23T20:35:02.681681+0000","last_clean":"2023-09-23T20:35:02.681681+0000","last_became_active":"2023-09-23T20:34:05.560057+0000","last_became_peered":"2023-09-23T20:34:05.560057+0000","last_unstale":"2023-09-23T20:35:02.681681+0000","last_undegraded":"2023-09-23T20:35:02.681681+0000","last_fullsized":"2023-09-23T20:35:02.681681+0000","mapping_epoch":338,"log_start":"310'2400","ondisk_log_start":"310'2400","created":303,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":7499,"log_dups_size":0,"ondisk_log_size":7499,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:08:18.482746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":124715008,"num_objects":1903,"num_object_clones":0,"num_object_copies":5709,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1903,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9899,"num_write_kb":377664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3638,"num_bytes_recovered":238419968,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.e","version":"90'161","reported_seq":554,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-09-23T20:34:52.598987+0000","last_change":"2023-09-23T20:34:19.365282+0000","last_active":"2023-09-23T20:34:52.598987+0000","last_peered":"2023-09-23T20:34:52.598987+0000","last_clean":"2023-09-23T20:34:52.598987+0000","last_became_active":"2023-09-23T20:33:38.975554+0000","last_became_peered":"2023-09-23T20:33:38.975554+0000","last_unstale":"2023-09-23T20:34:52.598987+0000","last_undegraded":"2023-09-23T20:34:52.598987+0000","last_fullsized":"2023-09-23T20:34:52.598987+0000","mapping_epoch":322,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:44.386120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:44.386120+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0015151330000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":360,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T20:34:58.968695+0000","last_change":"2023-09-23T20:29:04.138284+0000","last_active":"2023-09-23T20:34:58.968695+0000","last_peered":"2023-09-23T20:34:58.968695+0000","last_clean":"2023-09-23T20:34:58.968695+0000","last_became_active":"2023-09-23T20:29:04.137960+0000","last_became_peered":"2023-09-23T20:29:04.137960+0000","last_unstale":"2023-09-23T20:34:58.968695+0000","last_undegraded":"2023-09-23T20:34:58.968695+0000","last_fullsized":"2023-09-23T20:34:58.968695+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:18:59.548311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":337,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T20:34:58.972443+0000","last_change":"2023-09-23T20:32:33.236122+0000","last_active":"2023-09-23T20:34:58.972443+0000","last_peered":"2023-09-23T20:34:58.972443+0000","last_clean":"2023-09-23T20:34:58.972443+0000","last_became_active":"2023-09-23T20:32:33.235384+0000","last_became_peered":"2023-09-23T20:32:33.235384+0000","last_unstale":"2023-09-23T20:34:58.972443+0000","last_undegraded":"2023-09-23T20:34:58.972443+0000","last_fullsized":"2023-09-23T20:34:58.972443+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:56.843733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"354'9868","reported_seq":9929,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-09-23T20:35:00.224465+0000","last_change":"2023-09-23T20:31:49.158461+0000","last_active":"2023-09-23T20:35:00.224465+0000","last_peered":"2023-09-23T20:35:00.224465+0000","last_clean":"2023-09-23T20:35:00.224465+0000","last_became_active":"2023-09-23T20:31:49.158188+0000","last_became_peered":"2023-09-23T20:31:49.158188+0000","last_unstale":"2023-09-23T20:35:00.224465+0000","last_undegraded":"2023-09-23T20:35:00.224465+0000","last_fullsized":"2023-09-23T20:35:00.224465+0000","mapping_epoch":303,"log_start":"310'3300","ondisk_log_start":"310'3300","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":6568,"log_dups_size":0,"ondisk_log_size":6568,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:31:57.678090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":136839168,"num_objects":2088,"num_object_clones":0,"num_object_copies":6264,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2088,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9868,"num_write_kb":382592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":749,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T20:34:58.982325+0000","last_change":"2023-09-23T20:34:21.452828+0000","last_active":"2023-09-23T20:34:58.982325+0000","last_peered":"2023-09-23T20:34:58.982325+0000","last_clean":"2023-09-23T20:34:58.982325+0000","last_became_active":"2023-09-23T20:33:38.979989+0000","last_became_peered":"2023-09-23T20:33:38.979989+0000","last_unstale":"2023-09-23T20:34:58.982325+0000","last_undegraded":"2023-09-23T20:34:58.982325+0000","last_fullsized":"2023-09-23T20:34:58.982325+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:28:05.872074+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:28:05.872074+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0029215980000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":88,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T20:34:58.972541+0000","last_change":"2023-09-23T20:32:50.538500+0000","last_active":"2023-09-23T20:34:58.972541+0000","last_peered":"2023-09-23T20:34:58.972541+0000","last_clean":"2023-09-23T20:34:58.972541+0000","last_became_active":"2023-09-23T20:32:50.538152+0000","last_became_peered":"2023-09-23T20:32:50.538152+0000","last_unstale":"2023-09-23T20:34:58.972541+0000","last_undegraded":"2023-09-23T20:34:58.972541+0000","last_fullsized":"2023-09-23T20:34:58.972541+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:00:41.103508+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":51,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T20:34:58.961439+0000","last_change":"2023-09-23T20:32:33.231614+0000","last_active":"2023-09-23T20:34:58.961439+0000","last_peered":"2023-09-23T20:34:58.961439+0000","last_clean":"2023-09-23T20:34:58.961439+0000","last_became_active":"2023-09-23T20:32:33.231207+0000","last_became_peered":"2023-09-23T20:32:33.231207+0000","last_unstale":"2023-09-23T20:34:58.961439+0000","last_undegraded":"2023-09-23T20:34:58.961439+0000","last_fullsized":"2023-09-23T20:34:58.961439+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:46:09.780829+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"357'9871","reported_seq":9935,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T20:35:02.680852+0000","last_change":"2023-09-23T20:31:49.155754+0000","last_active":"2023-09-23T20:35:02.680852+0000","last_peered":"2023-09-23T20:35:02.680852+0000","last_clean":"2023-09-23T20:35:02.680852+0000","last_became_active":"2023-09-23T20:31:49.155363+0000","last_became_peered":"2023-09-23T20:31:49.155363+0000","last_unstale":"2023-09-23T20:35:02.680852+0000","last_undegraded":"2023-09-23T20:35:02.680852+0000","last_fullsized":"2023-09-23T20:35:02.680852+0000","mapping_epoch":303,"log_start":"310'2400","ondisk_log_start":"310'2400","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":7471,"log_dups_size":0,"ondisk_log_size":7471,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:45:42.661641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":129564672,"num_objects":1977,"num_object_clones":0,"num_object_copies":5931,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1977,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9871,"num_write_kb":379136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":594,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T20:34:58.968480+0000","last_change":"2023-09-23T20:34:26.269676+0000","last_active":"2023-09-23T20:34:58.968480+0000","last_peered":"2023-09-23T20:34:58.968480+0000","last_clean":"2023-09-23T20:34:58.968480+0000","last_became_active":"2023-09-23T20:33:38.996273+0000","last_became_peered":"2023-09-23T20:33:38.996273+0000","last_unstale":"2023-09-23T20:34:58.968480+0000","last_undegraded":"2023-09-23T20:34:58.968480+0000","last_fullsized":"2023-09-23T20:34:58.968480+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:27:31.756974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:27:31.756974+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0031802670000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131344,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":149,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T20:34:58.963791+0000","last_change":"2023-09-23T20:32:33.229700+0000","last_active":"2023-09-23T20:34:58.963791+0000","last_peered":"2023-09-23T20:34:58.963791+0000","last_clean":"2023-09-23T20:34:58.963791+0000","last_became_active":"2023-09-23T20:32:33.229372+0000","last_became_peered":"2023-09-23T20:32:33.229372+0000","last_unstale":"2023-09-23T20:34:58.963791+0000","last_undegraded":"2023-09-23T20:34:58.963791+0000","last_fullsized":"2023-09-23T20:34:58.963791+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:18:10.958227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":10,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-09-23T20:34:59.985545+0000","last_change":"2023-09-23T20:34:59.985545+0000","last_active":"2023-09-23T20:34:59.985545+0000","last_peered":"2023-09-23T20:34:59.985545+0000","last_clean":"2023-09-23T20:34:59.985545+0000","last_became_active":"2023-09-23T20:34:59.985090+0000","last_became_peered":"2023-09-23T20:34:59.985090+0000","last_unstale":"2023-09-23T20:34:59.985545+0000","last_undegraded":"2023-09-23T20:34:59.985545+0000","last_fullsized":"2023-09-23T20:34:59.985545+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:21:40.825065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"354'9935","reported_seq":9995,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-09-23T20:34:59.984428+0000","last_change":"2023-09-23T20:31:49.158641+0000","last_active":"2023-09-23T20:34:59.984428+0000","last_peered":"2023-09-23T20:34:59.984428+0000","last_clean":"2023-09-23T20:34:59.984428+0000","last_became_active":"2023-09-23T20:31:49.157604+0000","last_became_peered":"2023-09-23T20:31:49.157604+0000","last_unstale":"2023-09-23T20:34:59.984428+0000","last_undegraded":"2023-09-23T20:34:59.984428+0000","last_fullsized":"2023-09-23T20:34:59.984428+0000","mapping_epoch":303,"log_start":"317'3895","ondisk_log_start":"317'3895","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":6040,"log_dups_size":0,"ondisk_log_size":6040,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:54:04.726000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":134742040,"num_objects":2057,"num_object_clones":0,"num_object_copies":6171,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2057,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9935,"num_write_kb":383681,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.b","version":"90'161","reported_seq":605,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-09-23T20:34:54.614176+0000","last_change":"2023-09-23T20:33:38.992494+0000","last_active":"2023-09-23T20:34:54.614176+0000","last_peered":"2023-09-23T20:34:54.614176+0000","last_clean":"2023-09-23T20:34:54.614176+0000","last_became_active":"2023-09-23T20:33:38.991910+0000","last_became_peered":"2023-09-23T20:33:38.991910+0000","last_unstale":"2023-09-23T20:34:54.614176+0000","last_undegraded":"2023-09-23T20:34:54.614176+0000","last_fullsized":"2023-09-23T20:34:54.614176+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:46.558292+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:46.558292+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0023048779999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":10,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-09-23T20:34:59.983122+0000","last_change":"2023-09-23T20:34:59.983122+0000","last_active":"2023-09-23T20:34:59.983122+0000","last_peered":"2023-09-23T20:34:59.983122+0000","last_clean":"2023-09-23T20:34:59.983122+0000","last_became_active":"2023-09-23T20:34:59.982809+0000","last_became_peered":"2023-09-23T20:34:59.982809+0000","last_unstale":"2023-09-23T20:34:59.983122+0000","last_undegraded":"2023-09-23T20:34:59.983122+0000","last_fullsized":"2023-09-23T20:34:59.983122+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:49:40.839723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":51,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T20:34:58.962156+0000","last_change":"2023-09-23T20:32:33.224963+0000","last_active":"2023-09-23T20:34:58.962156+0000","last_peered":"2023-09-23T20:34:58.962156+0000","last_clean":"2023-09-23T20:34:58.962156+0000","last_became_active":"2023-09-23T20:32:33.224634+0000","last_became_peered":"2023-09-23T20:32:33.224634+0000","last_unstale":"2023-09-23T20:34:58.962156+0000","last_undegraded":"2023-09-23T20:34:58.962156+0000","last_fullsized":"2023-09-23T20:34:58.962156+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:41:40.314140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"353'9857","reported_seq":9916,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T20:34:59.126952+0000","last_change":"2023-09-23T20:31:49.157247+0000","last_active":"2023-09-23T20:34:59.126952+0000","last_peered":"2023-09-23T20:34:59.126952+0000","last_clean":"2023-09-23T20:34:59.126952+0000","last_became_active":"2023-09-23T20:31:49.156961+0000","last_became_peered":"2023-09-23T20:31:49.156961+0000","last_unstale":"2023-09-23T20:34:59.126952+0000","last_undegraded":"2023-09-23T20:34:59.126952+0000","last_fullsized":"2023-09-23T20:34:59.126952+0000","mapping_epoch":303,"log_start":"317'3801","ondisk_log_start":"317'3801","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":6056,"log_dups_size":0,"ondisk_log_size":6056,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:49:17.346795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131268608,"num_objects":2003,"num_object_clones":0,"num_object_copies":6009,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2003,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9857,"num_write_kb":379520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":647,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T20:34:59.481559+0000","last_change":"2023-09-23T20:34:59.481511+0000","last_active":"2023-09-23T20:34:59.481559+0000","last_peered":"2023-09-23T20:34:59.481559+0000","last_clean":"2023-09-23T20:34:59.481559+0000","last_became_active":"2023-09-23T20:33:38.988954+0000","last_became_peered":"2023-09-23T20:33:38.988954+0000","last_unstale":"2023-09-23T20:34:59.481559+0000","last_undegraded":"2023-09-23T20:34:59.481559+0000","last_fullsized":"2023-09-23T20:34:59.481559+0000","mapping_epoch":253,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:34:59.481446+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:34:59.481446+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:53:42.768648+0000","scrub_duration":0.001554773,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":165,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-09-23T20:34:54.601656+0000","last_change":"2023-09-23T20:32:50.530690+0000","last_active":"2023-09-23T20:34:54.601656+0000","last_peered":"2023-09-23T20:34:54.601656+0000","last_clean":"2023-09-23T20:34:54.601656+0000","last_became_active":"2023-09-23T20:32:50.530376+0000","last_became_peered":"2023-09-23T20:32:50.530376+0000","last_unstale":"2023-09-23T20:34:54.601656+0000","last_undegraded":"2023-09-23T20:34:54.601656+0000","last_fullsized":"2023-09-23T20:34:54.601656+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:24:53.184959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":174,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-09-23T20:34:54.610126+0000","last_change":"2023-09-23T20:32:33.230582+0000","last_active":"2023-09-23T20:34:54.610126+0000","last_peered":"2023-09-23T20:34:54.610126+0000","last_clean":"2023-09-23T20:34:54.610126+0000","last_became_active":"2023-09-23T20:32:33.230257+0000","last_became_peered":"2023-09-23T20:32:33.230257+0000","last_unstale":"2023-09-23T20:34:54.610126+0000","last_undegraded":"2023-09-23T20:34:54.610126+0000","last_fullsized":"2023-09-23T20:34:54.610126+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:11:45.333096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"352'9597","reported_seq":10714,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-09-23T20:34:55.626979+0000","last_change":"2023-09-23T20:34:48.691440+0000","last_active":"2023-09-23T20:34:55.626979+0000","last_peered":"2023-09-23T20:34:55.626979+0000","last_clean":"2023-09-23T20:34:55.626979+0000","last_became_active":"2023-09-23T20:33:01.258982+0000","last_became_peered":"2023-09-23T20:33:01.258982+0000","last_unstale":"2023-09-23T20:34:55.626979+0000","last_undegraded":"2023-09-23T20:34:55.626979+0000","last_fullsized":"2023-09-23T20:34:55.626979+0000","mapping_epoch":323,"log_start":"311'3547","ondisk_log_start":"311'3547","created":303,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":6050,"log_dups_size":0,"ondisk_log_size":6050,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:38:23.266319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":140574720,"num_objects":2145,"num_object_clones":0,"num_object_copies":6435,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2145,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9597,"num_write_kb":375744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":521,"num_bytes_recovered":34144256,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":593,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T20:34:58.966640+0000","last_change":"2023-09-23T20:33:38.978909+0000","last_active":"2023-09-23T20:34:58.966640+0000","last_peered":"2023-09-23T20:34:58.966640+0000","last_clean":"2023-09-23T20:34:58.966640+0000","last_became_active":"2023-09-23T20:33:38.978580+0000","last_became_peered":"2023-09-23T20:33:38.978580+0000","last_unstale":"2023-09-23T20:34:58.966640+0000","last_undegraded":"2023-09-23T20:34:58.966640+0000","last_fullsized":"2023-09-23T20:34:58.966640+0000","mapping_epoch":253,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:45.196285+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:45.196285+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0014380860000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"86'119","reported_seq":658,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-09-23T20:34:54.612734+0000","last_change":"2023-09-23T20:29:45.521257+0000","last_active":"2023-09-23T20:34:54.612734+0000","last_peered":"2023-09-23T20:34:54.612734+0000","last_clean":"2023-09-23T20:34:54.612734+0000","last_became_active":"2023-09-23T20:29:45.520546+0000","last_became_peered":"2023-09-23T20:29:45.520546+0000","last_unstale":"2023-09-23T20:34:54.612734+0000","last_undegraded":"2023-09-23T20:34:54.612734+0000","last_fullsized":"2023-09-23T20:34:54.612734+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":119,"log_dups_size":0,"ondisk_log_size":119,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:58:56.085777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032705499999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":523,"num_read_kb":3503,"num_write":136,"num_write_kb":3128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":370,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T20:34:58.968928+0000","last_change":"2023-09-23T20:25:39.637160+0000","last_active":"2023-09-23T20:34:58.968928+0000","last_peered":"2023-09-23T20:34:58.968928+0000","last_clean":"2023-09-23T20:34:58.968928+0000","last_became_active":"2023-09-23T20:25:39.636860+0000","last_became_peered":"2023-09-23T20:25:39.636860+0000","last_unstale":"2023-09-23T20:34:58.968928+0000","last_undegraded":"2023-09-23T20:34:58.968928+0000","last_fullsized":"2023-09-23T20:34:58.968928+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:21:47.860063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":279,"reported_epoch":353,"state":"peering","last_fresh":"2023-09-23T20:34:58.971893+0000","last_change":"2023-09-23T20:34:58.972050+0000","last_active":"2023-09-23T20:34:58.971893+0000","last_peered":"2023-09-23T20:34:54.601769+0000","last_clean":"2023-09-23T20:34:54.601769+0000","last_became_active":"2023-09-23T20:33:00.190494+0000","last_became_peered":"2023-09-23T20:33:00.190494+0000","last_unstale":"2023-09-23T20:34:58.971893+0000","last_undegraded":"2023-09-23T20:34:58.971893+0000","last_fullsized":"2023-09-23T20:34:58.971893+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:11:35.369137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[0,4,5],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"354'9902","reported_seq":9961,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-09-23T20:35:00.210646+0000","last_change":"2023-09-23T20:31:49.157301+0000","last_active":"2023-09-23T20:35:00.210646+0000","last_peered":"2023-09-23T20:35:00.210646+0000","last_clean":"2023-09-23T20:35:00.210646+0000","last_became_active":"2023-09-23T20:31:49.156992+0000","last_became_peered":"2023-09-23T20:31:49.156992+0000","last_unstale":"2023-09-23T20:35:00.210646+0000","last_undegraded":"2023-09-23T20:35:00.210646+0000","last_fullsized":"2023-09-23T20:35:00.210646+0000","mapping_epoch":303,"log_start":"317'3876","ondisk_log_start":"317'3876","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":6026,"log_dups_size":0,"ondisk_log_size":6026,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:21:35.645838+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131203072,"num_objects":2002,"num_object_clones":0,"num_object_copies":6006,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2002,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9902,"num_write_kb":380928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":598,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T20:34:58.968512+0000","last_change":"2023-09-23T20:31:45.785521+0000","last_active":"2023-09-23T20:34:58.968512+0000","last_peered":"2023-09-23T20:34:58.968512+0000","last_clean":"2023-09-23T20:34:58.968512+0000","last_became_active":"2023-09-23T20:27:55.213394+0000","last_became_peered":"2023-09-23T20:27:55.213394+0000","last_unstale":"2023-09-23T20:34:58.968512+0000","last_undegraded":"2023-09-23T20:34:58.968512+0000","last_fullsized":"2023-09-23T20:34:58.968512+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:45.785450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:45.785450+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:21:26.092578+0000","scrub_duration":0.001558306,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"86'106","reported_seq":1072,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T20:34:58.969131+0000","last_change":"2023-09-23T20:31:31.531882+0000","last_active":"2023-09-23T20:34:58.969131+0000","last_peered":"2023-09-23T20:34:58.969131+0000","last_clean":"2023-09-23T20:34:58.969131+0000","last_became_active":"2023-09-23T20:29:46.078501+0000","last_became_peered":"2023-09-23T20:29:46.078501+0000","last_unstale":"2023-09-23T20:34:58.969131+0000","last_undegraded":"2023-09-23T20:34:58.969131+0000","last_fullsized":"2023-09-23T20:34:58.969131+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":106,"log_dups_size":0,"ondisk_log_size":106,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:37.015974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00099302599999999993,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1105,"num_read_kb":1689,"num_write":100,"num_write_kb":404,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":51,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T20:34:58.961625+0000","last_change":"2023-09-23T20:32:33.230242+0000","last_active":"2023-09-23T20:34:58.961625+0000","last_peered":"2023-09-23T20:34:58.961625+0000","last_clean":"2023-09-23T20:34:58.961625+0000","last_became_active":"2023-09-23T20:32:33.229732+0000","last_became_peered":"2023-09-23T20:32:33.229732+0000","last_unstale":"2023-09-23T20:34:58.961625+0000","last_undegraded":"2023-09-23T20:34:58.961625+0000","last_fullsized":"2023-09-23T20:34:58.961625+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:27:55.538429+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":311,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-09-23T20:34:54.612656+0000","last_change":"2023-09-23T20:29:45.533406+0000","last_active":"2023-09-23T20:34:54.612656+0000","last_peered":"2023-09-23T20:34:54.612656+0000","last_clean":"2023-09-23T20:34:54.612656+0000","last_became_active":"2023-09-23T20:29:45.526001+0000","last_became_peered":"2023-09-23T20:29:45.526001+0000","last_unstale":"2023-09-23T20:34:54.612656+0000","last_undegraded":"2023-09-23T20:34:54.612656+0000","last_fullsized":"2023-09-23T20:34:54.612656+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:43.787251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.4","version":"354'9764","reported_seq":11669,"reported_epoch":354,"state":"active+recovering+undersized+remapped","last_fresh":"2023-09-23T20:35:00.501770+0000","last_change":"2023-09-23T20:35:00.204596+0000","last_active":"2023-09-23T20:35:00.501770+0000","last_peered":"2023-09-23T20:35:00.501770+0000","last_clean":"2023-09-23T20:32:32.192526+0000","last_became_active":"2023-09-23T20:35:00.124743+0000","last_became_peered":"2023-09-23T20:35:00.124743+0000","last_unstale":"2023-09-23T20:35:00.501770+0000","last_undegraded":"2023-09-23T20:35:00.501770+0000","last_fullsized":"2023-09-23T20:34:59.968003+0000","mapping_epoch":353,"log_start":"310'3200","ondisk_log_start":"310'3200","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":6564,"log_dups_size":0,"ondisk_log_size":6564,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:40:39.030291+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":128319488,"num_objects":1958,"num_object_clones":0,"num_object_copies":5874,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1179,"num_objects_unfound":0,"num_objects_dirty":1958,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9764,"num_write_kb":375104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":908,"num_bytes_recovered":59375616,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,1],"avail_no_missing":["0","1"],"object_location_counts":[{"shards":"0,1","objects":779},{"shards":"0,1,6","objects":1179}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":810,"reported_epoch":354,"state":"remapped+peering","last_fresh":"2023-09-23T20:34:59.977260+0000","last_change":"2023-09-23T20:34:59.977519+0000","last_active":"2023-09-23T20:34:58.982057+0000","last_peered":"2023-09-23T20:34:54.616041+0000","last_clean":"2023-09-23T20:34:54.616041+0000","last_became_active":"2023-09-23T20:33:00.188795+0000","last_became_peered":"2023-09-23T20:33:00.188795+0000","last_unstale":"2023-09-23T20:34:59.977260+0000","last_undegraded":"2023-09-23T20:34:59.977260+0000","last_fullsized":"2023-09-23T20:34:59.977260+0000","mapping_epoch":354,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":323,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:43.390908+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:43.390908+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:48:19.107567+0000","scrub_duration":0.0032721,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[0,2,4],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"86'130","reported_seq":670,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-09-23T20:35:01.216449+0000","last_change":"2023-09-23T20:25:39.637458+0000","last_active":"2023-09-23T20:35:01.216449+0000","last_peered":"2023-09-23T20:35:01.216449+0000","last_clean":"2023-09-23T20:35:01.216449+0000","last_became_active":"2023-09-23T20:25:39.636773+0000","last_became_peered":"2023-09-23T20:25:39.636773+0000","last_unstale":"2023-09-23T20:35:01.216449+0000","last_undegraded":"2023-09-23T20:35:01.216449+0000","last_fullsized":"2023-09-23T20:35:01.216449+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:00:32.810564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011300049999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":512,"num_read_kb":1820,"num_write":133,"num_write_kb":1347,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":301,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-09-23T20:34:54.601456+0000","last_change":"2023-09-23T20:29:46.073394+0000","last_active":"2023-09-23T20:34:54.601456+0000","last_peered":"2023-09-23T20:34:54.601456+0000","last_clean":"2023-09-23T20:34:54.601456+0000","last_became_active":"2023-09-23T20:29:46.072067+0000","last_became_peered":"2023-09-23T20:29:46.072067+0000","last_unstale":"2023-09-23T20:34:54.601456+0000","last_undegraded":"2023-09-23T20:34:54.601456+0000","last_fullsized":"2023-09-23T20:34:54.601456+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:01:56.944614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":293,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-09-23T20:34:54.610042+0000","last_change":"2023-09-23T20:29:45.533185+0000","last_active":"2023-09-23T20:34:54.610042+0000","last_peered":"2023-09-23T20:34:54.610042+0000","last_clean":"2023-09-23T20:34:54.610042+0000","last_became_active":"2023-09-23T20:29:45.526765+0000","last_became_peered":"2023-09-23T20:29:45.526765+0000","last_unstale":"2023-09-23T20:34:54.610042+0000","last_undegraded":"2023-09-23T20:34:54.610042+0000","last_fullsized":"2023-09-23T20:34:54.610042+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:18:33.063215+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"354'9847","reported_seq":9908,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-09-23T20:35:00.214967+0000","last_change":"2023-09-23T20:31:49.158194+0000","last_active":"2023-09-23T20:35:00.214967+0000","last_peered":"2023-09-23T20:35:00.214967+0000","last_clean":"2023-09-23T20:35:00.214967+0000","last_became_active":"2023-09-23T20:31:49.157875+0000","last_became_peered":"2023-09-23T20:31:49.157875+0000","last_unstale":"2023-09-23T20:35:00.214967+0000","last_undegraded":"2023-09-23T20:35:00.214967+0000","last_fullsized":"2023-09-23T20:35:00.214967+0000","mapping_epoch":303,"log_start":"310'3300","ondisk_log_start":"310'3300","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":6547,"log_dups_size":0,"ondisk_log_size":6547,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:04:42.411181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":132448256,"num_objects":2021,"num_object_clones":0,"num_object_copies":6063,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2021,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9847,"num_write_kb":379776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"86'194","reported_seq":1002,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T20:34:58.969633+0000","last_change":"2023-09-23T20:25:40.618665+0000","last_active":"2023-09-23T20:34:58.969633+0000","last_peered":"2023-09-23T20:34:58.969633+0000","last_clean":"2023-09-23T20:34:58.969633+0000","last_became_active":"2023-09-23T20:25:40.618087+0000","last_became_peered":"2023-09-23T20:25:40.618087+0000","last_unstale":"2023-09-23T20:34:58.969633+0000","last_undegraded":"2023-09-23T20:34:58.969633+0000","last_fullsized":"2023-09-23T20:34:58.969633+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":194,"log_dups_size":0,"ondisk_log_size":194,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:07:16.114968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00037995999999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1025,"num_read_kb":1201,"num_write":173,"num_write_kb":1276,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":787,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T20:34:58.981213+0000","last_change":"2023-09-23T20:31:45.343835+0000","last_active":"2023-09-23T20:34:58.981213+0000","last_peered":"2023-09-23T20:34:58.981213+0000","last_clean":"2023-09-23T20:34:58.981213+0000","last_became_active":"2023-09-23T20:29:07.042407+0000","last_became_peered":"2023-09-23T20:29:07.042407+0000","last_unstale":"2023-09-23T20:34:58.981213+0000","last_undegraded":"2023-09-23T20:34:58.981213+0000","last_fullsized":"2023-09-23T20:34:58.981213+0000","mapping_epoch":253,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":254,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:45.343742+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:45.343742+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:33:12.745366+0000","scrub_duration":0.001475802,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":153,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T20:34:58.969191+0000","last_change":"2023-09-23T20:32:50.534315+0000","last_active":"2023-09-23T20:34:58.969191+0000","last_peered":"2023-09-23T20:34:58.969191+0000","last_clean":"2023-09-23T20:34:58.969191+0000","last_became_active":"2023-09-23T20:32:50.533896+0000","last_became_peered":"2023-09-23T20:32:50.533896+0000","last_unstale":"2023-09-23T20:34:58.969191+0000","last_undegraded":"2023-09-23T20:34:58.969191+0000","last_fullsized":"2023-09-23T20:34:58.969191+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:06:53.464923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":133,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T20:34:58.966757+0000","last_change":"2023-09-23T20:32:33.225012+0000","last_active":"2023-09-23T20:34:58.966757+0000","last_peered":"2023-09-23T20:34:58.966757+0000","last_clean":"2023-09-23T20:34:58.966757+0000","last_became_active":"2023-09-23T20:32:33.224699+0000","last_became_peered":"2023-09-23T20:32:33.224699+0000","last_unstale":"2023-09-23T20:34:58.966757+0000","last_undegraded":"2023-09-23T20:34:58.966757+0000","last_fullsized":"2023-09-23T20:34:58.966757+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:10:06.267878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"354'9909","reported_seq":9968,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-09-23T20:35:00.197092+0000","last_change":"2023-09-23T20:31:49.154418+0000","last_active":"2023-09-23T20:35:00.197092+0000","last_peered":"2023-09-23T20:35:00.197092+0000","last_clean":"2023-09-23T20:35:00.197092+0000","last_became_active":"2023-09-23T20:31:49.154182+0000","last_became_peered":"2023-09-23T20:31:49.154182+0000","last_unstale":"2023-09-23T20:35:00.197092+0000","last_undegraded":"2023-09-23T20:35:00.197092+0000","last_fullsized":"2023-09-23T20:35:00.197092+0000","mapping_epoch":303,"log_start":"311'3598","ondisk_log_start":"311'3598","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":6311,"log_dups_size":0,"ondisk_log_size":6311,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:44:10.087486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":132186112,"num_objects":2017,"num_object_clones":0,"num_object_copies":6051,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2017,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9909,"num_write_kb":381632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":630,"reported_epoch":353,"state":"peering","last_fresh":"2023-09-23T20:34:58.974687+0000","last_change":"2023-09-23T20:34:58.974793+0000","last_active":"2023-09-23T20:34:58.974687+0000","last_peered":"2023-09-23T20:34:54.614025+0000","last_clean":"2023-09-23T20:34:54.614025+0000","last_became_active":"2023-09-23T20:33:03.279531+0000","last_became_peered":"2023-09-23T20:33:03.279531+0000","last_unstale":"2023-09-23T20:34:58.974687+0000","last_undegraded":"2023-09-23T20:34:58.974687+0000","last_fullsized":"2023-09-23T20:34:58.974687+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:43.582146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:43.582146+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:50:52.546729+0000","scrub_duration":0.0016219730000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[4,5,6],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"86'80","reported_seq":617,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T20:34:58.981126+0000","last_change":"2023-09-23T20:29:45.536796+0000","last_active":"2023-09-23T20:34:58.981126+0000","last_peered":"2023-09-23T20:34:58.981126+0000","last_clean":"2023-09-23T20:34:58.981126+0000","last_became_active":"2023-09-23T20:29:45.536439+0000","last_became_peered":"2023-09-23T20:29:45.536439+0000","last_unstale":"2023-09-23T20:34:58.981126+0000","last_undegraded":"2023-09-23T20:34:58.981126+0000","last_fullsized":"2023-09-23T20:34:58.981126+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":80,"log_dups_size":0,"ondisk_log_size":80,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:30:57.793132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035115,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":395,"num_read_kb":1370,"num_write":66,"num_write_kb":1061,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":356,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-09-23T20:34:59.984286+0000","last_change":"2023-09-23T20:34:59.984286+0000","last_active":"2023-09-23T20:34:59.984286+0000","last_peered":"2023-09-23T20:34:59.984286+0000","last_clean":"2023-09-23T20:34:59.984286+0000","last_became_active":"2023-09-23T20:34:59.983950+0000","last_became_peered":"2023-09-23T20:34:59.983950+0000","last_unstale":"2023-09-23T20:34:59.984286+0000","last_undegraded":"2023-09-23T20:34:59.984286+0000","last_fullsized":"2023-09-23T20:34:59.984286+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:31:53.557035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":350,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T20:34:58.974242+0000","last_change":"2023-09-23T20:29:45.690815+0000","last_active":"2023-09-23T20:34:58.974242+0000","last_peered":"2023-09-23T20:34:58.974242+0000","last_clean":"2023-09-23T20:34:58.974242+0000","last_became_active":"2023-09-23T20:29:45.690478+0000","last_became_peered":"2023-09-23T20:29:45.690478+0000","last_unstale":"2023-09-23T20:34:58.974242+0000","last_undegraded":"2023-09-23T20:34:58.974242+0000","last_fullsized":"2023-09-23T20:34:58.974242+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:38.288193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.0","version":"354'9903","reported_seq":9962,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-09-23T20:35:00.184861+0000","last_change":"2023-09-23T20:31:49.157205+0000","last_active":"2023-09-23T20:35:00.184861+0000","last_peered":"2023-09-23T20:35:00.184861+0000","last_clean":"2023-09-23T20:35:00.184861+0000","last_became_active":"2023-09-23T20:31:49.156922+0000","last_became_peered":"2023-09-23T20:31:49.156922+0000","last_unstale":"2023-09-23T20:35:00.184861+0000","last_undegraded":"2023-09-23T20:35:00.184861+0000","last_fullsized":"2023-09-23T20:35:00.184861+0000","mapping_epoch":303,"log_start":"310'2400","ondisk_log_start":"310'2400","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":7503,"log_dups_size":0,"ondisk_log_size":7503,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:51:36.120059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":130744320,"num_objects":1995,"num_object_clones":0,"num_object_copies":5985,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1995,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9903,"num_write_kb":380736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":596,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-09-23T20:35:00.151879+0000","last_change":"2023-09-23T20:35:00.151823+0000","last_active":"2023-09-23T20:35:00.151879+0000","last_peered":"2023-09-23T20:35:00.151879+0000","last_clean":"2023-09-23T20:35:00.151879+0000","last_became_active":"2023-09-23T20:33:38.991030+0000","last_became_peered":"2023-09-23T20:33:38.991030+0000","last_unstale":"2023-09-23T20:35:00.151879+0000","last_undegraded":"2023-09-23T20:35:00.151879+0000","last_fullsized":"2023-09-23T20:35:00.151879+0000","mapping_epoch":253,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:35:00.151695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:35:00.151695+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:09:55.183927+0000","scrub_duration":0.0016030090000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"86'102","reported_seq":948,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T20:34:58.983308+0000","last_change":"2023-09-23T20:32:36.314672+0000","last_active":"2023-09-23T20:34:58.983308+0000","last_peered":"2023-09-23T20:34:58.983308+0000","last_clean":"2023-09-23T20:34:58.983308+0000","last_became_active":"2023-09-23T20:32:36.314231+0000","last_became_peered":"2023-09-23T20:32:36.314231+0000","last_unstale":"2023-09-23T20:34:58.983308+0000","last_undegraded":"2023-09-23T20:34:58.983308+0000","last_fullsized":"2023-09-23T20:34:58.983308+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":102,"log_dups_size":0,"ondisk_log_size":102,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:11:40.514709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030656500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":963,"num_read_kb":1234,"num_write":60,"num_write_kb":225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":277,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-09-23T20:34:54.612820+0000","last_change":"2023-09-23T20:29:45.538283+0000","last_active":"2023-09-23T20:34:54.612820+0000","last_peered":"2023-09-23T20:34:54.612820+0000","last_clean":"2023-09-23T20:34:54.612820+0000","last_became_active":"2023-09-23T20:29:45.537922+0000","last_became_peered":"2023-09-23T20:29:45.537922+0000","last_unstale":"2023-09-23T20:34:54.612820+0000","last_undegraded":"2023-09-23T20:34:54.612820+0000","last_fullsized":"2023-09-23T20:34:54.612820+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:30:52.397836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":288,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T20:34:58.972722+0000","last_change":"2023-09-23T20:20:14.888874+0000","last_active":"2023-09-23T20:34:58.972722+0000","last_peered":"2023-09-23T20:34:58.972722+0000","last_clean":"2023-09-23T20:34:58.972722+0000","last_became_active":"2023-09-23T20:20:14.887724+0000","last_became_peered":"2023-09-23T20:20:14.887724+0000","last_unstale":"2023-09-23T20:34:58.972722+0000","last_undegraded":"2023-09-23T20:34:58.972722+0000","last_fullsized":"2023-09-23T20:34:58.972722+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:09:24.510326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"357'9629","reported_seq":16800,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T20:35:02.675524+0000","last_change":"2023-09-23T20:34:50.587971+0000","last_active":"2023-09-23T20:35:02.675524+0000","last_peered":"2023-09-23T20:35:02.675524+0000","last_clean":"2023-09-23T20:35:02.675524+0000","last_became_active":"2023-09-23T20:34:50.555328+0000","last_became_peered":"2023-09-23T20:34:50.555328+0000","last_unstale":"2023-09-23T20:35:02.675524+0000","last_undegraded":"2023-09-23T20:35:02.675524+0000","last_fullsized":"2023-09-23T20:35:02.675524+0000","mapping_epoch":349,"log_start":"310'2200","ondisk_log_start":"310'2200","created":303,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":7429,"log_dups_size":0,"ondisk_log_size":7429,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:20:26.250987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":122224640,"num_objects":1865,"num_object_clones":0,"num_object_copies":5595,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1865,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9629,"num_write_kb":367808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3542,"num_bytes_recovered":170328064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":562,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T20:34:58.962615+0000","last_change":"2023-09-23T20:34:55.534723+0000","last_active":"2023-09-23T20:34:58.962615+0000","last_peered":"2023-09-23T20:34:58.962615+0000","last_clean":"2023-09-23T20:34:58.962615+0000","last_became_active":"2023-09-23T20:33:38.985300+0000","last_became_peered":"2023-09-23T20:33:38.985300+0000","last_unstale":"2023-09-23T20:34:58.962615+0000","last_undegraded":"2023-09-23T20:34:58.962615+0000","last_fullsized":"2023-09-23T20:34:58.962615+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:44.509012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:44.509012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0032186319999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":91,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T20:34:58.983754+0000","last_change":"2023-09-23T20:32:50.549695+0000","last_active":"2023-09-23T20:34:58.983754+0000","last_peered":"2023-09-23T20:34:58.983754+0000","last_clean":"2023-09-23T20:34:58.983754+0000","last_became_active":"2023-09-23T20:32:50.549431+0000","last_became_peered":"2023-09-23T20:32:50.549431+0000","last_unstale":"2023-09-23T20:34:58.983754+0000","last_undegraded":"2023-09-23T20:34:58.983754+0000","last_fullsized":"2023-09-23T20:34:58.983754+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:45:53.666569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"86'128","reported_seq":743,"reported_epoch":352,"state":"active+clean","last_fresh":"2023-09-23T20:34:54.610184+0000","last_change":"2023-09-23T20:32:36.310656+0000","last_active":"2023-09-23T20:34:54.610184+0000","last_peered":"2023-09-23T20:34:54.610184+0000","last_clean":"2023-09-23T20:34:54.610184+0000","last_became_active":"2023-09-23T20:32:36.310155+0000","last_became_peered":"2023-09-23T20:32:36.310155+0000","last_unstale":"2023-09-23T20:34:54.610184+0000","last_undegraded":"2023-09-23T20:34:54.610184+0000","last_fullsized":"2023-09-23T20:34:54.610184+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:58:14.217466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000293202,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":579,"num_read_kb":1473,"num_write":96,"num_write_kb":1062,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":124,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T20:34:58.981416+0000","last_change":"2023-09-23T20:32:50.539051+0000","last_active":"2023-09-23T20:34:58.981416+0000","last_peered":"2023-09-23T20:34:58.981416+0000","last_clean":"2023-09-23T20:34:58.981416+0000","last_became_active":"2023-09-23T20:32:50.538755+0000","last_became_peered":"2023-09-23T20:32:50.538755+0000","last_unstale":"2023-09-23T20:34:58.981416+0000","last_undegraded":"2023-09-23T20:34:58.981416+0000","last_fullsized":"2023-09-23T20:34:58.981416+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:16.656613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.2","version":"353'9690","reported_seq":9757,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T20:34:59.131352+0000","last_change":"2023-09-23T20:32:50.557165+0000","last_active":"2023-09-23T20:34:59.131352+0000","last_peered":"2023-09-23T20:34:59.131352+0000","last_clean":"2023-09-23T20:34:59.131352+0000","last_became_active":"2023-09-23T20:32:50.553409+0000","last_became_peered":"2023-09-23T20:32:50.553409+0000","last_unstale":"2023-09-23T20:34:59.131352+0000","last_undegraded":"2023-09-23T20:34:59.131352+0000","last_fullsized":"2023-09-23T20:34:59.131352+0000","mapping_epoch":319,"log_start":"310'3401","ondisk_log_start":"310'3401","created":303,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":6289,"log_dups_size":0,"ondisk_log_size":6289,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:55:13.578195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131203072,"num_objects":2002,"num_object_clones":0,"num_object_copies":6006,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2002,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9690,"num_write_kb":374144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"90'161","reported_seq":503,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T20:34:58.969112+0000","last_change":"2023-09-23T20:34:54.504652+0000","last_active":"2023-09-23T20:34:58.969112+0000","last_peered":"2023-09-23T20:34:58.969112+0000","last_clean":"2023-09-23T20:34:58.969112+0000","last_became_active":"2023-09-23T20:32:50.533470+0000","last_became_peered":"2023-09-23T20:32:50.533470+0000","last_unstale":"2023-09-23T20:34:58.969112+0000","last_undegraded":"2023-09-23T20:34:58.969112+0000","last_fullsized":"2023-09-23T20:34:58.969112+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:34:54.504554+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:34:54.504554+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:17:03.108888+0000","scrub_duration":0.013031021,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.7","version":"86'117","reported_seq":623,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T20:34:58.966887+0000","last_change":"2023-09-23T20:31:32.814392+0000","last_active":"2023-09-23T20:34:58.966887+0000","last_peered":"2023-09-23T20:34:58.966887+0000","last_clean":"2023-09-23T20:34:58.966887+0000","last_became_active":"2023-09-23T20:31:32.813857+0000","last_became_peered":"2023-09-23T20:31:32.813857+0000","last_unstale":"2023-09-23T20:34:58.966887+0000","last_undegraded":"2023-09-23T20:34:58.966887+0000","last_fullsized":"2023-09-23T20:34:58.966887+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:56:57.863005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046891400000000002,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":893,"num_read_kb":922,"num_write":82,"num_write_kb":4193,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":350,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T20:34:58.974326+0000","last_change":"2023-09-23T20:32:33.277827+0000","last_active":"2023-09-23T20:34:58.974326+0000","last_peered":"2023-09-23T20:34:58.974326+0000","last_clean":"2023-09-23T20:34:58.974326+0000","last_became_active":"2023-09-23T20:32:33.236087+0000","last_became_peered":"2023-09-23T20:32:33.236087+0000","last_unstale":"2023-09-23T20:34:58.974326+0000","last_undegraded":"2023-09-23T20:34:58.974326+0000","last_fullsized":"2023-09-23T20:34:58.974326+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:20:58.176854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":51,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T20:34:58.961055+0000","last_change":"2023-09-23T20:32:33.238290+0000","last_active":"2023-09-23T20:34:58.961055+0000","last_peered":"2023-09-23T20:34:58.961055+0000","last_clean":"2023-09-23T20:34:58.961055+0000","last_became_active":"2023-09-23T20:32:33.237984+0000","last_became_peered":"2023-09-23T20:32:33.237984+0000","last_unstale":"2023-09-23T20:34:58.961055+0000","last_undegraded":"2023-09-23T20:34:58.961055+0000","last_fullsized":"2023-09-23T20:34:58.961055+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:49:02.089307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"90'161","reported_seq":748,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T20:34:58.981563+0000","last_change":"2023-09-23T20:34:32.517512+0000","last_active":"2023-09-23T20:34:58.981563+0000","last_peered":"2023-09-23T20:34:58.981563+0000","last_clean":"2023-09-23T20:34:58.981563+0000","last_became_active":"2023-09-23T20:33:38.989881+0000","last_became_peered":"2023-09-23T20:33:38.989881+0000","last_unstale":"2023-09-23T20:34:58.981563+0000","last_undegraded":"2023-09-23T20:34:58.981563+0000","last_fullsized":"2023-09-23T20:34:58.981563+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:28:05.872074+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:28:05.872074+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0029215980000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":318,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T20:34:58.972634+0000","last_change":"2023-09-23T20:32:33.237234+0000","last_active":"2023-09-23T20:34:58.972634+0000","last_peered":"2023-09-23T20:34:58.972634+0000","last_clean":"2023-09-23T20:34:58.972634+0000","last_became_active":"2023-09-23T20:32:33.236881+0000","last_became_peered":"2023-09-23T20:32:33.236881+0000","last_unstale":"2023-09-23T20:34:58.972634+0000","last_undegraded":"2023-09-23T20:34:58.972634+0000","last_fullsized":"2023-09-23T20:34:58.972634+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:56:17.384831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":209,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-09-23T20:35:01.219489+0000","last_change":"2023-09-23T20:29:45.538314+0000","last_active":"2023-09-23T20:35:01.219489+0000","last_peered":"2023-09-23T20:35:01.219489+0000","last_clean":"2023-09-23T20:35:01.219489+0000","last_became_active":"2023-09-23T20:29:45.536847+0000","last_became_peered":"2023-09-23T20:29:45.536847+0000","last_unstale":"2023-09-23T20:35:01.219489+0000","last_undegraded":"2023-09-23T20:35:01.219489+0000","last_fullsized":"2023-09-23T20:35:01.219489+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:51.948721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"90'161","reported_seq":553,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-09-23T20:34:52.598499+0000","last_change":"2023-09-23T20:34:29.525104+0000","last_active":"2023-09-23T20:34:52.598499+0000","last_peered":"2023-09-23T20:34:52.598499+0000","last_clean":"2023-09-23T20:34:52.598499+0000","last_became_active":"2023-09-23T20:33:38.975688+0000","last_became_peered":"2023-09-23T20:33:38.975688+0000","last_unstale":"2023-09-23T20:34:52.598499+0000","last_undegraded":"2023-09-23T20:34:52.598499+0000","last_fullsized":"2023-09-23T20:34:52.598499+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:44.386120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:44.386120+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0015151330000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":363,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T20:34:58.968694+0000","last_change":"2023-09-23T20:32:33.235987+0000","last_active":"2023-09-23T20:34:58.968694+0000","last_peered":"2023-09-23T20:34:58.968694+0000","last_clean":"2023-09-23T20:34:58.968694+0000","last_became_active":"2023-09-23T20:32:33.226544+0000","last_became_peered":"2023-09-23T20:32:33.226544+0000","last_unstale":"2023-09-23T20:34:58.968694+0000","last_undegraded":"2023-09-23T20:34:58.968694+0000","last_fullsized":"2023-09-23T20:34:58.968694+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:53:22.794856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":10,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-09-23T20:34:59.987575+0000","last_change":"2023-09-23T20:34:59.987575+0000","last_active":"2023-09-23T20:34:59.987575+0000","last_peered":"2023-09-23T20:34:59.987575+0000","last_clean":"2023-09-23T20:34:59.987575+0000","last_became_active":"2023-09-23T20:34:59.987229+0000","last_became_peered":"2023-09-23T20:34:59.987229+0000","last_unstale":"2023-09-23T20:34:59.987575+0000","last_undegraded":"2023-09-23T20:34:59.987575+0000","last_fullsized":"2023-09-23T20:34:59.987575+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:34:01.820159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":371,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T20:34:58.968731+0000","last_change":"2023-09-23T20:32:33.268260+0000","last_active":"2023-09-23T20:34:58.968731+0000","last_peered":"2023-09-23T20:34:58.968731+0000","last_clean":"2023-09-23T20:34:58.968731+0000","last_became_active":"2023-09-23T20:32:33.233695+0000","last_became_peered":"2023-09-23T20:32:33.233695+0000","last_unstale":"2023-09-23T20:34:58.968731+0000","last_undegraded":"2023-09-23T20:34:58.968731+0000","last_fullsized":"2023-09-23T20:34:58.968731+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:19:29.299623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":10,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-09-23T20:34:59.987416+0000","last_change":"2023-09-23T20:34:59.987416+0000","last_active":"2023-09-23T20:34:59.987416+0000","last_peered":"2023-09-23T20:34:59.987416+0000","last_clean":"2023-09-23T20:34:59.987416+0000","last_became_active":"2023-09-23T20:34:59.987117+0000","last_became_peered":"2023-09-23T20:34:59.987117+0000","last_unstale":"2023-09-23T20:34:59.987416+0000","last_undegraded":"2023-09-23T20:34:59.987416+0000","last_fullsized":"2023-09-23T20:34:59.987416+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:36:16.318623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"90'161","reported_seq":523,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T20:34:58.981860+0000","last_change":"2023-09-23T20:33:38.990525+0000","last_active":"2023-09-23T20:34:58.981860+0000","last_peered":"2023-09-23T20:34:58.981860+0000","last_clean":"2023-09-23T20:34:58.981860+0000","last_became_active":"2023-09-23T20:33:38.990173+0000","last_became_peered":"2023-09-23T20:33:38.990173+0000","last_unstale":"2023-09-23T20:34:58.981860+0000","last_undegraded":"2023-09-23T20:34:58.981860+0000","last_fullsized":"2023-09-23T20:34:58.981860+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:41.701171+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:41.701171+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0019955509999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":54614,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":169,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T20:34:58.965675+0000","last_change":"2023-09-23T20:32:50.537447+0000","last_active":"2023-09-23T20:34:58.965675+0000","last_peered":"2023-09-23T20:34:58.965675+0000","last_clean":"2023-09-23T20:34:58.965675+0000","last_became_active":"2023-09-23T20:32:50.537136+0000","last_became_peered":"2023-09-23T20:32:50.537136+0000","last_unstale":"2023-09-23T20:34:58.965675+0000","last_undegraded":"2023-09-23T20:34:58.965675+0000","last_fullsized":"2023-09-23T20:34:58.965675+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:46:51.599492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":71,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-09-23T20:35:01.216598+0000","last_change":"2023-09-23T20:32:50.537450+0000","last_active":"2023-09-23T20:35:01.216598+0000","last_peered":"2023-09-23T20:35:01.216598+0000","last_clean":"2023-09-23T20:35:01.216598+0000","last_became_active":"2023-09-23T20:32:50.537128+0000","last_became_peered":"2023-09-23T20:32:50.537128+0000","last_unstale":"2023-09-23T20:35:01.216598+0000","last_undegraded":"2023-09-23T20:35:01.216598+0000","last_fullsized":"2023-09-23T20:35:01.216598+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:37:15.889029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1d","version":"90'161","reported_seq":635,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T20:34:58.981030+0000","last_change":"2023-09-23T20:34:30.499622+0000","last_active":"2023-09-23T20:34:58.981030+0000","last_peered":"2023-09-23T20:34:58.981030+0000","last_clean":"2023-09-23T20:34:58.981030+0000","last_became_active":"2023-09-23T20:33:38.972109+0000","last_became_peered":"2023-09-23T20:33:38.972109+0000","last_unstale":"2023-09-23T20:34:58.981030+0000","last_undegraded":"2023-09-23T20:34:58.981030+0000","last_fullsized":"2023-09-23T20:34:58.981030+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:46.388545+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:46.388545+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.00168356,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":336,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T20:34:58.974567+0000","last_change":"2023-09-23T20:25:39.668882+0000","last_active":"2023-09-23T20:34:58.974567+0000","last_peered":"2023-09-23T20:34:58.974567+0000","last_clean":"2023-09-23T20:34:58.974567+0000","last_became_active":"2023-09-23T20:25:39.637128+0000","last_became_peered":"2023-09-23T20:25:39.637128+0000","last_unstale":"2023-09-23T20:34:58.974567+0000","last_undegraded":"2023-09-23T20:34:58.974567+0000","last_fullsized":"2023-09-23T20:34:58.974567+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:56.406298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":51,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T20:34:58.961858+0000","last_change":"2023-09-23T20:32:33.232281+0000","last_active":"2023-09-23T20:34:58.961858+0000","last_peered":"2023-09-23T20:34:58.961858+0000","last_clean":"2023-09-23T20:34:58.961858+0000","last_became_active":"2023-09-23T20:32:33.232003+0000","last_became_peered":"2023-09-23T20:32:33.232003+0000","last_unstale":"2023-09-23T20:34:58.961858+0000","last_undegraded":"2023-09-23T20:34:58.961858+0000","last_fullsized":"2023-09-23T20:34:58.961858+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:58:27.314355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":2094858264,"num_objects":31966,"num_object_clones":0,"num_object_copies":95898,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1179,"num_objects_unfound":0,"num_objects_dirty":31966,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":157134,"num_write_kb":6051137,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9718,"num_bytes_recovered":574947328,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":6704803840,"data_stored":6704791624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":106726,"ondisk_log_size":106726,"up":48,"acting":47,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":49,"num_bytes_recovered":6522,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26691,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":371915,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":92,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":58,"num_bytes_recovered":4194342,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6082,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201566,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1068008,"kb_used_data":784800,"kb_used_omap":0,"kb_used_meta":283200,"kb_avail":92677144,"statfs":{"total":95995035648,"available":94901395456,"internally_reserved":0,"allocated":803635200,"data_stored":802517206,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":289996800},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:34:43 2023","interfaces":[{"interface":"back","average":{"1min":2.323,"5min":2.0470000000000002,"15min":1.617},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.20799999999999999},"max":{"1min":6.1959999999999997,"5min":14.66,"15min":48.798999999999999},"last":0.496},{"interface":"front","average":{"1min":2.871,"5min":2.0950000000000002,"15min":1.702},"min":{"1min":0.25,"5min":0.25,"15min":0.24399999999999999},"max":{"1min":14.605,"5min":14.605,"15min":48.247},"last":0.53200000000000003}]},{"osd":1,"last update":"Sat Sep 23 20:34:54 2023","interfaces":[{"interface":"back","average":{"1min":2.3319999999999999,"5min":1.984,"15min":1.8360000000000001},"min":{"1min":0.29999999999999999,"5min":0.254,"15min":0.254},"max":{"1min":5.5030000000000001,"5min":14.026,"15min":14.026},"last":0.54900000000000004},{"interface":"front","average":{"1min":2.5139999999999998,"5min":1.8340000000000001,"15min":1.9319999999999999},"min":{"1min":0.32000000000000001,"5min":0.248,"15min":0.248},"max":{"1min":6.7220000000000004,"5min":6.7220000000000004,"15min":6.7220000000000004},"last":0.47699999999999998}]},{"osd":2,"last update":"Sat Sep 23 20:34:43 2023","interfaces":[{"interface":"back","average":{"1min":2.6739999999999999,"5min":2.4159999999999999,"15min":1.6799999999999999},"min":{"1min":0.34100000000000003,"5min":0.28599999999999998,"15min":0.27600000000000002},"max":{"1min":7.9029999999999996,"5min":13.173999999999999,"15min":13.173999999999999},"last":0.438},{"interface":"front","average":{"1min":2.6280000000000001,"5min":2.7320000000000002,"15min":1.9039999999999999},"min":{"1min":0.36899999999999999,"5min":0.27400000000000002,"15min":0.26600000000000001},"max":{"1min":9.7230000000000008,"5min":23.494,"15min":23.494},"last":0.67600000000000005}]},{"osd":3,"last update":"Sat Sep 23 20:34:07 2023","interfaces":[{"interface":"back","average":{"1min":2.4380000000000002,"5min":2.3479999999999999,"15min":1.7070000000000001},"min":{"1min":0.35199999999999998,"5min":0.29199999999999998,"15min":0.26200000000000001},"max":{"1min":7.202,"5min":17.155999999999999,"15min":17.155999999999999},"last":0.56299999999999994},{"interface":"front","average":{"1min":2.7269999999999999,"5min":2.2010000000000001,"15min":1.6419999999999999},"min":{"1min":0.45300000000000001,"5min":0.33100000000000002,"15min":0.27600000000000002},"max":{"1min":6.6589999999999998,"5min":16.960000000000001,"15min":16.960000000000001},"last":0.60299999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:34:07 2023","interfaces":[{"interface":"back","average":{"1min":0.67100000000000004,"5min":1.2849999999999999,"15min":1.05},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":2.2370000000000001,"5min":15.177,"15min":15.177},"last":0.33000000000000002},{"interface":"front","average":{"1min":0.82999999999999996,"5min":1.048,"15min":1.1759999999999999},"min":{"1min":0.33000000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":2.9790000000000001,"5min":9.3740000000000006,"15min":11.122},"last":0.39100000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:34:07 2023","interfaces":[{"interface":"back","average":{"1min":0.78400000000000003,"5min":0.91100000000000003,"15min":1.1319999999999999},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":2.7730000000000001,"5min":6.8300000000000001,"15min":13.241},"last":0.40999999999999998},{"interface":"front","average":{"1min":0.47699999999999998,"5min":1.0660000000000001,"15min":1.1950000000000001},"min":{"1min":0.34399999999999997,"5min":0.32200000000000001,"15min":0.315},"max":{"1min":0.747,"5min":18.831,"15min":18.831},"last":0.46100000000000002}]},{"osd":6,"last update":"Sat Sep 23 20:34:07 2023","interfaces":[{"interface":"back","average":{"1min":0.59799999999999998,"5min":1.1180000000000001,"15min":1.0980000000000001},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":2.177,"5min":12.298999999999999,"15min":13.185},"last":0.42199999999999999},{"interface":"front","average":{"1min":0.97299999999999998,"5min":1.022,"15min":1.0920000000000001},"min":{"1min":0.36899999999999999,"5min":0.318,"15min":0.29899999999999999},"max":{"1min":3.0190000000000001,"5min":8.0069999999999997,"15min":9.9120000000000008},"last":0.50700000000000001}]}]},{"osd":6,"up_from":47,"seq":201863463202,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1252772,"kb_used_data":974812,"kb_used_omap":0,"kb_used_meta":277952,"kb_avail":92492380,"statfs":{"total":95995035648,"available":94712197120,"internally_reserved":0,"allocated":998207488,"data_stored":997093487,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":284622848},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,0,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:34:41 2023","interfaces":[{"interface":"back","average":{"1min":3.9790000000000001,"5min":2.5270000000000001,"15min":1.589},"min":{"1min":0.24299999999999999,"5min":0.22800000000000001,"15min":0.221},"max":{"1min":34.143999999999998,"5min":53.701999999999998,"15min":53.701999999999998},"last":0.42799999999999999},{"interface":"front","average":{"1min":4.202,"5min":3.004,"15min":1.9390000000000001},"min":{"1min":0.253,"5min":0.253,"15min":0.25},"max":{"1min":34.197000000000003,"5min":53.875999999999998,"15min":53.875999999999998},"last":6.093}]},{"osd":1,"last update":"Sat Sep 23 20:34:57 2023","interfaces":[{"interface":"back","average":{"1min":3.4510000000000001,"5min":2.2719999999999998,"15min":2.5910000000000002},"min":{"1min":0.25600000000000001,"5min":0.249,"15min":0.249},"max":{"1min":34.173999999999999,"5min":34.173999999999999,"15min":34.173999999999999},"last":8.3420000000000005},{"interface":"front","average":{"1min":4.29,"5min":2.569,"15min":3.181},"min":{"1min":0.32900000000000001,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":34.490000000000002,"5min":34.490000000000002,"15min":34.490000000000002},"last":10.263999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:34:41 2023","interfaces":[{"interface":"back","average":{"1min":3.879,"5min":2.5379999999999998,"15min":1.8080000000000001},"min":{"1min":0.29599999999999999,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":34.317999999999998,"5min":34.317999999999998,"15min":34.317999999999998},"last":2.1190000000000002},{"interface":"front","average":{"1min":4.1390000000000002,"5min":2.5390000000000001,"15min":1.7350000000000001},"min":{"1min":0.35399999999999998,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":34.292999999999999,"5min":34.292999999999999,"15min":34.292999999999999},"last":2.133}]},{"osd":3,"last update":"Sat Sep 23 20:34:23 2023","interfaces":[{"interface":"back","average":{"1min":2.242,"5min":2.0230000000000001,"15min":1.6120000000000001},"min":{"1min":0.31900000000000001,"5min":0.30499999999999999,"15min":0.29599999999999999},"max":{"1min":6.2089999999999996,"5min":7.5620000000000003,"15min":9.4239999999999995},"last":5.6230000000000002},{"interface":"front","average":{"1min":2.6960000000000002,"5min":2.2589999999999999,"15min":1.708},"min":{"1min":0.443,"5min":0.317,"15min":0.317},"max":{"1min":12.875999999999999,"5min":12.875999999999999,"15min":12.875999999999999},"last":8.5600000000000005}]},{"osd":4,"last update":"Sat Sep 23 20:34:01 2023","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":0.71299999999999997,"15min":0.86499999999999999},"min":{"1min":0.29999999999999999,"5min":0.251,"15min":0.251},"max":{"1min":2.4060000000000001,"5min":8.4749999999999996,"15min":10.173},"last":0.60399999999999998},{"interface":"front","average":{"1min":1.095,"5min":1.1779999999999999,"15min":1.0409999999999999},"min":{"1min":0.31900000000000001,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":6.5309999999999997,"5min":7.7629999999999999,"15min":8.3249999999999993},"last":0.318}]},{"osd":5,"last update":"Sat Sep 23 20:34:01 2023","interfaces":[{"interface":"back","average":{"1min":0.79500000000000004,"5min":0.874,"15min":0.93799999999999994},"min":{"1min":0.34999999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":2.847,"5min":8.4499999999999993,"15min":14.085000000000001},"last":0.65300000000000002},{"interface":"front","average":{"1min":1.1359999999999999,"5min":1.2370000000000001,"15min":1.298},"min":{"1min":0.34999999999999998,"5min":0.30599999999999999,"15min":0.29499999999999998},"max":{"1min":6.4909999999999997,"5min":9.9299999999999997,"15min":18.192},"last":0.371}]},{"osd":7,"last update":"Sat Sep 23 20:34:23 2023","interfaces":[{"interface":"back","average":{"1min":0.59599999999999997,"5min":0.94199999999999995,"15min":1.0660000000000001},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":2.9470000000000001,"5min":8.5060000000000002,"15min":10.217000000000001},"last":0.68799999999999994},{"interface":"front","average":{"1min":0.72299999999999998,"5min":1.204,"15min":1.2250000000000001},"min":{"1min":0.34499999999999997,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":3.0920000000000001,"5min":7.601,"15min":9.7360000000000007},"last":0.46000000000000002}]}]},{"osd":1,"up_from":266,"seq":1142461300800,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1173116,"kb_used_data":913588,"kb_used_omap":8,"kb_used_meta":259511,"kb_avail":92572036,"statfs":{"total":95995035648,"available":94793764864,"internally_reserved":0,"allocated":935514112,"data_stored":933607012,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8206,"internal_metadata":265740274},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,2,0,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:34:54 2023","interfaces":[{"interface":"back","average":{"1min":1.9470000000000001,"5min":2.9750000000000001,"15min":2.5449999999999999},"min":{"1min":0.221,"5min":0.218,"15min":0.218},"max":{"1min":8.6080000000000005,"5min":52.689999999999998,"15min":52.689999999999998},"last":4.4820000000000002},{"interface":"front","average":{"1min":5.3540000000000001,"5min":3.4809999999999999,"15min":3.2909999999999999},"min":{"1min":0.20599999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":26.588000000000001,"5min":52.662999999999997,"15min":52.662999999999997},"last":7.9580000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:34:54 2023","interfaces":[{"interface":"back","average":{"1min":7.2110000000000003,"5min":4.069,"15min":4.7729999999999997},"min":{"1min":0.312,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":28.359000000000002,"5min":28.359000000000002,"15min":28.359000000000002},"last":2.9700000000000002},{"interface":"front","average":{"1min":3.4550000000000001,"5min":2.6179999999999999,"15min":2.2679999999999998},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":12.801,"5min":12.801,"15min":12.801},"last":4.5140000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:34:54 2023","interfaces":[{"interface":"back","average":{"1min":4.1239999999999997,"5min":4.3799999999999999,"15min":4.0380000000000003},"min":{"1min":0.31,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":15.247999999999999,"5min":80.745999999999995,"15min":80.745999999999995},"last":0.81100000000000005},{"interface":"front","average":{"1min":5.3380000000000001,"5min":4.6689999999999996,"15min":4.7869999999999999},"min":{"1min":0.35799999999999998,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":17.893999999999998,"5min":82.988,"15min":82.988},"last":2.218}]},{"osd":4,"last update":"Sat Sep 23 20:34:54 2023","interfaces":[{"interface":"back","average":{"1min":4.1449999999999996,"5min":2.2989999999999999,"15min":2.1520000000000001},"min":{"1min":0.34200000000000003,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":24.327999999999999,"5min":24.327999999999999,"15min":24.327999999999999},"last":2.085},{"interface":"front","average":{"1min":1.2889999999999999,"5min":1.6759999999999999,"15min":1.9079999999999999},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":4.7889999999999997,"5min":10.760999999999999,"15min":10.760999999999999},"last":0.84999999999999998}]},{"osd":5,"last update":"Sat Sep 23 20:34:54 2023","interfaces":[{"interface":"back","average":{"1min":3.3170000000000002,"5min":2.1760000000000002,"15min":2.593},"min":{"1min":0.45000000000000001,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":7.6909999999999998,"5min":9.6479999999999997,"15min":9.6479999999999997},"last":4.5819999999999999},{"interface":"front","average":{"1min":4.1059999999999999,"5min":2.4809999999999999,"15min":2.2200000000000002},"min":{"1min":0.38400000000000001,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":24.404,"5min":24.404,"15min":24.404},"last":2.9390000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:34:54 2023","interfaces":[{"interface":"back","average":{"1min":3.3250000000000002,"5min":3.1059999999999999,"15min":6.0010000000000003},"min":{"1min":0.51200000000000001,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":7.7199999999999998,"5min":72.682000000000002,"15min":72.682000000000002},"last":2.5209999999999999},{"interface":"front","average":{"1min":4.2489999999999997,"5min":3.4180000000000001,"15min":5.4320000000000004},"min":{"1min":0.41099999999999998,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":24.376000000000001,"5min":72.563999999999993,"15min":72.563999999999993},"last":1.5760000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:34:54 2023","interfaces":[{"interface":"back","average":{"1min":3.4489999999999998,"5min":2.3759999999999999,"15min":2.98},"min":{"1min":0.54400000000000004,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":9.0830000000000002,"5min":9.6850000000000005,"15min":9.6850000000000005},"last":2.2549999999999999},{"interface":"front","average":{"1min":4.2519999999999998,"5min":2.6230000000000002,"15min":2.4470000000000001},"min":{"1min":0.441,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":24.524000000000001,"5min":24.524000000000001,"15min":24.524000000000001},"last":2.2069999999999999}]}]},{"osd":0,"up_from":94,"seq":403726926003,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1506536,"kb_used_data":1044256,"kb_used_omap":7,"kb_used_meta":462264,"kb_avail":92238616,"statfs":{"total":95995035648,"available":94452342784,"internally_reserved":0,"allocated":1069318144,"data_stored":1067554625,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":473358353},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,1,5,5],"upper_bound":512},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:34:00 2023","interfaces":[{"interface":"back","average":{"1min":6.6790000000000003,"5min":3.5339999999999998,"15min":3.7690000000000001},"min":{"1min":0.23000000000000001,"5min":0.17999999999999999,"15min":0.17999999999999999},"max":{"1min":42.000999999999998,"5min":42.000999999999998,"15min":42.000999999999998},"last":0.21299999999999999},{"interface":"front","average":{"1min":5.7359999999999998,"5min":3.5609999999999999,"15min":4.1580000000000004},"min":{"1min":0.26400000000000001,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":23.036000000000001,"5min":23.036000000000001,"15min":23.036000000000001},"last":0.33400000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:34:35 2023","interfaces":[{"interface":"back","average":{"1min":3.2410000000000001,"5min":3.4409999999999998,"15min":2.258},"min":{"1min":0.26400000000000001,"5min":0.185,"15min":0.185},"max":{"1min":9.3160000000000007,"5min":77.448999999999998,"15min":77.448999999999998},"last":0.24299999999999999},{"interface":"front","average":{"1min":3.1269999999999998,"5min":3.3849999999999998,"15min":2.097},"min":{"1min":0.32300000000000001,"5min":0.188,"15min":0.188},"max":{"1min":13.961,"5min":81.724999999999994,"15min":81.724999999999994},"last":0.25700000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:34:07 2023","interfaces":[{"interface":"back","average":{"1min":3.5720000000000001,"5min":3.4700000000000002,"15min":2.2189999999999999},"min":{"1min":0.24099999999999999,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":22.486000000000001,"5min":81.400999999999996,"15min":81.400999999999996},"last":0.314},{"interface":"front","average":{"1min":5.3799999999999999,"5min":4.1360000000000001,"15min":2.3100000000000001},"min":{"1min":0.25700000000000001,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":22.867999999999999,"5min":81.137,"15min":81.137},"last":0.30299999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:34:35 2023","interfaces":[{"interface":"back","average":{"1min":2.1680000000000001,"5min":1.927,"15min":1.752},"min":{"1min":0.30599999999999999,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":9.407,"5min":12.587999999999999,"15min":23.876999999999999},"last":0.47299999999999998},{"interface":"front","average":{"1min":3.0289999999999999,"5min":2.194,"15min":1.673},"min":{"1min":0.308,"5min":0.308,"15min":0.29299999999999998},"max":{"1min":10.380000000000001,"5min":23.001000000000001,"15min":23.681999999999999},"last":0.63800000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:34:35 2023","interfaces":[{"interface":"back","average":{"1min":3.0609999999999999,"5min":2.246,"15min":2},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34200000000000003},"max":{"1min":10.449,"5min":22.969999999999999,"15min":24.198},"last":0.55200000000000005},{"interface":"front","average":{"1min":2.0430000000000001,"5min":2.1989999999999998,"15min":1.855},"min":{"1min":0.307,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":7.3520000000000003,"5min":22.236000000000001,"15min":22.236000000000001},"last":0.64900000000000002}]},{"osd":6,"last update":"Sat Sep 23 20:34:35 2023","interfaces":[{"interface":"back","average":{"1min":3.3010000000000002,"5min":2.2759999999999998,"15min":1.7549999999999999},"min":{"1min":0.372,"5min":0.32300000000000001,"15min":0.318},"max":{"1min":10.423,"5min":22.934999999999999,"15min":24.408000000000001},"last":0.60999999999999999},{"interface":"front","average":{"1min":2.0369999999999999,"5min":2.1920000000000002,"15min":1.6990000000000001},"min":{"1min":0.38400000000000001,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":7.3979999999999997,"5min":22.164999999999999,"15min":25.271000000000001},"last":0.57799999999999996}]},{"osd":7,"last update":"Sat Sep 23 20:34:35 2023","interfaces":[{"interface":"back","average":{"1min":2.052,"5min":2.3300000000000001,"15min":1.8779999999999999},"min":{"1min":0.36099999999999999,"5min":0.32200000000000001,"15min":0.30599999999999999},"max":{"1min":7.4320000000000004,"5min":22.206,"15min":25.359000000000002},"last":0.625},{"interface":"front","average":{"1min":2.2189999999999999,"5min":2.339,"15min":2.1120000000000001},"min":{"1min":0.38100000000000001,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":9.3710000000000004,"5min":22.422999999999998,"15min":24.669},"last":0.60099999999999998}]}]},{"osd":2,"up_from":104,"seq":446676598951,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1226588,"kb_used_data":751444,"kb_used_omap":11,"kb_used_meta":475124,"kb_avail":92518564,"statfs":{"total":95995035648,"available":94739009536,"internally_reserved":0,"allocated":769478656,"data_stored":767694119,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11878,"internal_metadata":486527386},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,4],"upper_bound":16},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:34:44 2023","interfaces":[{"interface":"back","average":{"1min":7.0650000000000004,"5min":4.1500000000000004,"15min":2.3439999999999999},"min":{"1min":0.252,"5min":0.185,"15min":0.17999999999999999},"max":{"1min":28.234999999999999,"5min":82.424999999999997,"15min":82.424999999999997},"last":14.124000000000001},{"interface":"front","average":{"1min":4.3920000000000003,"5min":3.9169999999999998,"15min":2.073},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":9.6370000000000005,"5min":82.524000000000001,"15min":82.524000000000001},"last":14.055}]},{"osd":1,"last update":"Sat Sep 23 20:34:52 2023","interfaces":[{"interface":"back","average":{"1min":2.7080000000000002,"5min":3.0699999999999998,"15min":4.2569999999999997},"min":{"1min":0.27100000000000002,"5min":0.221,"15min":0.221},"max":{"1min":10.205,"5min":16.280000000000001,"15min":16.280000000000001},"last":4.117},{"interface":"front","average":{"1min":4.1100000000000003,"5min":2.8029999999999999,"15min":2.9260000000000002},"min":{"1min":0.36699999999999999,"5min":0.248,"15min":0.248},"max":{"1min":13.130000000000001,"5min":30.565000000000001,"15min":30.565000000000001},"last":14.032999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:34:09 2023","interfaces":[{"interface":"back","average":{"1min":4.8079999999999998,"5min":3.0230000000000001,"15min":1.7529999999999999},"min":{"1min":0.317,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":15.804,"5min":16.331,"15min":16.331},"last":4.0709999999999997},{"interface":"front","average":{"1min":4.5199999999999996,"5min":2.9169999999999998,"15min":1.9630000000000001},"min":{"1min":0.36899999999999999,"5min":0.249,"15min":0.23899999999999999},"max":{"1min":15.173,"5min":30.704999999999998,"15min":30.704999999999998},"last":10.300000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:34:44 2023","interfaces":[{"interface":"back","average":{"1min":2.1549999999999998,"5min":2.266,"15min":1.5680000000000001},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.28100000000000003},"max":{"1min":7.6879999999999997,"5min":16.797999999999998,"15min":16.797999999999998},"last":7.4560000000000004},{"interface":"front","average":{"1min":3.1139999999999999,"5min":2.4209999999999998,"15min":1.73},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.27600000000000002},"max":{"1min":15.853999999999999,"5min":15.853999999999999,"15min":15.853999999999999},"last":3.9809999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:34:44 2023","interfaces":[{"interface":"back","average":{"1min":2.5270000000000001,"5min":2.4409999999999998,"15min":2.1059999999999999},"min":{"1min":0.46800000000000003,"5min":0.314,"15min":0.309},"max":{"1min":7.7190000000000003,"5min":16.766999999999999,"15min":95.393000000000001},"last":7.4969999999999999},{"interface":"front","average":{"1min":3.7930000000000001,"5min":2.827,"15min":2.3359999999999999},"min":{"1min":0.36799999999999999,"5min":0.34799999999999998,"15min":0.32300000000000001},"max":{"1min":15.885,"5min":15.885,"15min":95.414000000000001},"last":4.0170000000000003}]},{"osd":6,"last update":"Sat Sep 23 20:34:44 2023","interfaces":[{"interface":"back","average":{"1min":3.1120000000000001,"5min":2.694,"15min":1.829},"min":{"1min":0.36599999999999999,"5min":0.33600000000000002,"15min":0.32100000000000001},"max":{"1min":15.914,"5min":15.914,"15min":15.914},"last":4.0460000000000003},{"interface":"front","average":{"1min":3.6179999999999999,"5min":2.3220000000000001,"15min":1.8140000000000001},"min":{"1min":0.40400000000000003,"5min":0.316,"15min":0.316},"max":{"1min":9.2370000000000001,"5min":30.670999999999999,"15min":30.670999999999999},"last":10.271000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:34:44 2023","interfaces":[{"interface":"back","average":{"1min":3.3879999999999999,"5min":2.444,"15min":1.8839999999999999},"min":{"1min":0.372,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":9.2759999999999998,"5min":30.741,"15min":30.741},"last":10.231},{"interface":"front","average":{"1min":2.758,"5min":2.645,"15min":1.8380000000000001},"min":{"1min":0.495,"5min":0.38,"15min":0.38},"max":{"1min":7.7549999999999999,"5min":16.719000000000001,"15min":16.719000000000001},"last":8.6489999999999991}]}]},{"osd":3,"up_from":109,"seq":468151435425,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1769752,"kb_used_data":1243664,"kb_used_omap":4,"kb_used_meta":525883,"kb_avail":91975400,"statfs":{"total":95995035648,"available":94182809600,"internally_reserved":0,"allocated":1273511936,"data_stored":1271729661,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4514,"internal_metadata":538504798},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,3,3],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:34:10 2023","interfaces":[{"interface":"back","average":{"1min":4.9000000000000004,"5min":2.8599999999999999,"15min":2.0510000000000002},"min":{"1min":0.63200000000000001,"5min":0.191,"15min":0.186},"max":{"1min":17.129000000000001,"5min":17.129000000000001,"15min":67.305999999999997},"last":0.41999999999999998},{"interface":"front","average":{"1min":4.8099999999999996,"5min":2.7679999999999998,"15min":1.9239999999999999},"min":{"1min":0.26200000000000001,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":12.502000000000001,"5min":12.781000000000001,"15min":67.403999999999996},"last":2.6379999999999999}]},{"osd":1,"last update":"Sat Sep 23 20:34:55 2023","interfaces":[{"interface":"back","average":{"1min":2.7589999999999999,"5min":2.7189999999999999,"15min":2.9649999999999999},"min":{"1min":0.28799999999999998,"5min":0.222,"15min":0.222},"max":{"1min":14.641,"5min":16.189,"15min":16.189},"last":0.25700000000000001},{"interface":"front","average":{"1min":2.1139999999999999,"5min":2.6299999999999999,"15min":3.0449999999999999},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":10.811,"5min":32.603999999999999,"15min":32.603999999999999},"last":2.5990000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:34:10 2023","interfaces":[{"interface":"back","average":{"1min":5.5899999999999999,"5min":3.472,"15min":2.0899999999999999},"min":{"1min":0.36399999999999999,"5min":0.24399999999999999,"15min":0.22800000000000001},"max":{"1min":20.405000000000001,"5min":32.857999999999997,"15min":32.857999999999997},"last":3.6549999999999998},{"interface":"front","average":{"1min":4.7270000000000003,"5min":3.113,"15min":1.9670000000000001},"min":{"1min":0.25700000000000001,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":24.146999999999998,"5min":24.146999999999998,"15min":24.146999999999998},"last":1.304}]},{"osd":4,"last update":"Sat Sep 23 20:34:10 2023","interfaces":[{"interface":"back","average":{"1min":2.9790000000000001,"5min":2.117,"15min":1.6339999999999999},"min":{"1min":0.41499999999999998,"5min":0.34200000000000003,"15min":0.29799999999999999},"max":{"1min":12.151,"5min":12.151,"15min":46.042999999999999},"last":0.55000000000000004},{"interface":"front","average":{"1min":3.4350000000000001,"5min":2.5630000000000002,"15min":1.919},"min":{"1min":0.29399999999999998,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":12.664,"5min":19.635000000000002,"15min":46.579999999999998},"last":1.3999999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:34:10 2023","interfaces":[{"interface":"back","average":{"1min":3.2050000000000001,"5min":2.266,"15min":1.843},"min":{"1min":0.48999999999999999,"5min":0.31900000000000001,"15min":0.311},"max":{"1min":12.096,"5min":12.096,"15min":45.292000000000002},"last":0.57599999999999996},{"interface":"front","average":{"1min":2.4529999999999998,"5min":2.2050000000000001,"15min":1.696},"min":{"1min":0.47499999999999998,"5min":0.32300000000000001,"15min":0.313},"max":{"1min":10.300000000000001,"5min":11.028,"15min":44.679000000000002},"last":2.6720000000000002}]},{"osd":6,"last update":"Sat Sep 23 20:34:10 2023","interfaces":[{"interface":"back","average":{"1min":2.7970000000000002,"5min":2.419,"15min":1.7350000000000001},"min":{"1min":0.45300000000000001,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":12.464,"5min":17.524999999999999,"15min":17.524999999999999},"last":2.8500000000000001},{"interface":"front","average":{"1min":2.5779999999999998,"5min":2.3180000000000001,"15min":1.647},"min":{"1min":0.371,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":5.8710000000000004,"5min":19.524000000000001,"15min":19.524000000000001},"last":8.327}]},{"osd":7,"last update":"Sat Sep 23 20:34:10 2023","interfaces":[{"interface":"back","average":{"1min":2.4969999999999999,"5min":2.7029999999999998,"15min":1.8129999999999999},"min":{"1min":0.46000000000000002,"5min":0.36899999999999999,"15min":0.34999999999999998},"max":{"1min":5.9530000000000003,"5min":19.571999999999999,"15min":19.571999999999999},"last":1.4419999999999999},{"interface":"front","average":{"1min":3.427,"5min":2.2149999999999999,"15min":1.9059999999999999},"min":{"1min":0.58999999999999997,"5min":0.42599999999999999,"15min":0.371},"max":{"1min":12.186,"5min":12.186,"15min":45.387999999999998},"last":0.51900000000000002}]}]},{"osd":4,"up_from":30,"seq":128849019178,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":647252,"kb_used_data":401484,"kb_used_omap":0,"kb_used_meta":245760,"kb_avail":93097900,"statfs":{"total":95995035648,"available":95332249600,"internally_reserved":0,"allocated":411119616,"data_stored":410013272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":251658240},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:34:37 2023","interfaces":[{"interface":"back","average":{"1min":2.1859999999999999,"5min":1.51,"15min":1.409},"min":{"1min":0.26200000000000001,"5min":0.255,"15min":0.222},"max":{"1min":8.0800000000000001,"5min":8.984,"15min":20.411999999999999},"last":3.3170000000000002},{"interface":"front","average":{"1min":2.2229999999999999,"5min":1.726,"15min":1.5580000000000001},"min":{"1min":0.29499999999999998,"5min":0.25800000000000001,"15min":0.253},"max":{"1min":8.0990000000000002,"5min":10.675000000000001,"15min":20.454999999999998},"last":3.3340000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:34:56 2023","interfaces":[{"interface":"back","average":{"1min":1.4350000000000001,"5min":1.885,"15min":1.9670000000000001},"min":{"1min":0.27600000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":6.0869999999999997,"5min":11.303000000000001,"15min":11.303000000000001},"last":1.212},{"interface":"front","average":{"1min":1.2110000000000001,"5min":1.6299999999999999,"15min":1.8859999999999999},"min":{"1min":0.30499999999999999,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":3.9169999999999998,"5min":11.287000000000001,"15min":11.287000000000001},"last":1.2030000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:34:48 2023","interfaces":[{"interface":"back","average":{"1min":1.536,"5min":1.952,"15min":1.6579999999999999},"min":{"1min":0.314,"5min":0.311,"15min":0.26600000000000001},"max":{"1min":3.3140000000000001,"5min":10.275,"15min":32.363},"last":2.0870000000000002},{"interface":"front","average":{"1min":2.5990000000000002,"5min":2.0470000000000002,"15min":1.9370000000000001},"min":{"1min":0.36899999999999999,"5min":0.28999999999999998,"15min":0.26800000000000002},"max":{"1min":5.9009999999999998,"5min":14.614000000000001,"15min":34.042999999999999},"last":6.2629999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:34:08 2023","interfaces":[{"interface":"back","average":{"1min":2,"5min":2.1659999999999999,"15min":1.7529999999999999},"min":{"1min":0.35299999999999998,"5min":0.312,"15min":0.29099999999999998},"max":{"1min":4.5140000000000002,"5min":14.515000000000001,"15min":32.332999999999998},"last":2.206},{"interface":"front","average":{"1min":2.79,"5min":2.137,"15min":1.8140000000000001},"min":{"1min":0.33700000000000002,"5min":0.318,"15min":0.30499999999999999},"max":{"1min":14.384,"5min":15.991,"15min":32.399999999999999},"last":1.782}]},{"osd":5,"last update":"Sat Sep 23 20:34:32 2023","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":0.98599999999999999,"15min":1.5269999999999999},"min":{"1min":0.28000000000000003,"5min":0.25800000000000001,"15min":0.247},"max":{"1min":3.0920000000000001,"5min":7.3719999999999999,"15min":82.866},"last":0.32400000000000001},{"interface":"front","average":{"1min":0.70499999999999996,"5min":1.0009999999999999,"15min":1.4279999999999999},"min":{"1min":0.36699999999999999,"5min":0.30299999999999999,"15min":0.27800000000000002},"max":{"1min":3.0699999999999998,"5min":6.8520000000000003,"15min":82.546000000000006},"last":0.33700000000000002}]},{"osd":6,"last update":"Sat Sep 23 20:34:48 2023","interfaces":[{"interface":"back","average":{"1min":1.3460000000000001,"5min":1.0640000000000001,"15min":1.3560000000000001},"min":{"1min":0.33300000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":10.178000000000001,"5min":10.178000000000001,"15min":41.371000000000002},"last":2.012},{"interface":"front","average":{"1min":0.63900000000000001,"5min":0.84699999999999998,"15min":1.2410000000000001},"min":{"1min":0.39600000000000002,"5min":0.34000000000000002,"15min":0.27200000000000002},"max":{"1min":2.3900000000000001,"5min":5.4619999999999997,"15min":41.140999999999998},"last":0.35699999999999998}]},{"osd":7,"last update":"Sat Sep 23 20:34:08 2023","interfaces":[{"interface":"back","average":{"1min":1.754,"5min":1.488,"15min":1.405},"min":{"1min":0.39100000000000001,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":6.8849999999999998,"5min":15.667999999999999,"15min":16.427},"last":0.39800000000000002},{"interface":"front","average":{"1min":1.1850000000000001,"5min":1.2370000000000001,"15min":1.2949999999999999},"min":{"1min":0.379,"5min":0.307,"15min":0.307},"max":{"1min":6.3970000000000002,"5min":15.58,"15min":19.018999999999998},"last":0.38800000000000001}]}]},{"osd":5,"up_from":39,"seq":167503724839,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":770384,"kb_used_data":517448,"kb_used_omap":0,"kb_used_meta":252928,"kb_avail":92974768,"statfs":{"total":95995035648,"available":95206162432,"internally_reserved":0,"allocated":529866752,"data_stored":528761134,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":258998272},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:34:40 2023","interfaces":[{"interface":"back","average":{"1min":1.9930000000000001,"5min":1.7350000000000001,"15min":1.4139999999999999},"min":{"1min":0.23400000000000001,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":7.6280000000000001,"5min":8.1609999999999996,"15min":32.475000000000001},"last":0.63200000000000001},{"interface":"front","average":{"1min":2.4060000000000001,"5min":1.6799999999999999,"15min":1.4299999999999999},"min":{"1min":0.29599999999999999,"5min":0.26300000000000001,"15min":0.20599999999999999},"max":{"1min":8.4459999999999997,"5min":8.4459999999999997,"15min":32.457000000000001},"last":0.56399999999999995}]},{"osd":1,"last update":"Sat Sep 23 20:34:59 2023","interfaces":[{"interface":"back","average":{"1min":2.1640000000000001,"5min":1.694,"15min":1.9339999999999999},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":9.3539999999999992,"5min":9.3539999999999992,"15min":9.3539999999999992},"last":0.57399999999999995},{"interface":"front","average":{"1min":2.1070000000000002,"5min":1.516,"15min":1.546},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":9.3900000000000006,"5min":9.3900000000000006,"15min":9.3900000000000006},"last":0.503}]},{"osd":2,"last update":"Sat Sep 23 20:34:40 2023","interfaces":[{"interface":"back","average":{"1min":3.7709999999999999,"5min":2.3319999999999999,"15min":1.72},"min":{"1min":0.28000000000000003,"5min":0.27400000000000002,"15min":0.26700000000000002},"max":{"1min":27.603999999999999,"5min":27.603999999999999,"15min":34.514000000000003},"last":0.437},{"interface":"front","average":{"1min":3.7229999999999999,"5min":2.3010000000000002,"15min":1.748},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":27.626999999999999,"5min":27.626999999999999,"15min":32.515000000000001},"last":0.48399999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:34:08 2023","interfaces":[{"interface":"back","average":{"1min":4.085,"5min":2.6000000000000001,"15min":1.7050000000000001},"min":{"1min":0.33700000000000002,"5min":0.29599999999999999,"15min":0.25700000000000001},"max":{"1min":16.189,"5min":16.189,"15min":16.189},"last":0.54100000000000004},{"interface":"front","average":{"1min":2.6549999999999998,"5min":1.8540000000000001,"15min":1.4610000000000001},"min":{"1min":0.42699999999999999,"5min":0.32300000000000001,"15min":0.28799999999999998},"max":{"1min":10.577999999999999,"5min":10.577999999999999,"15min":12.005000000000001},"last":0.55000000000000004}]},{"osd":4,"last update":"Sat Sep 23 20:34:30 2023","interfaces":[{"interface":"back","average":{"1min":0.51700000000000002,"5min":1.1930000000000001,"15min":1.292},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.25800000000000001},"max":{"1min":1.8979999999999999,"5min":28.015999999999998,"15min":28.015999999999998},"last":0.42499999999999999},{"interface":"front","average":{"1min":0.68200000000000005,"5min":0.93600000000000005,"15min":1.0649999999999999},"min":{"1min":0.318,"5min":0.29799999999999999,"15min":0.29099999999999998},"max":{"1min":2.9119999999999999,"5min":12.300000000000001,"15min":12.300000000000001},"last":0.61399999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:34:46 2023","interfaces":[{"interface":"back","average":{"1min":0.64000000000000001,"5min":0.77600000000000002,"15min":1.202},"min":{"1min":0.35399999999999998,"5min":0.309,"15min":0.30499999999999999},"max":{"1min":1.841,"5min":4.9829999999999997,"15min":12.890000000000001},"last":0.51400000000000001},{"interface":"front","average":{"1min":0.77000000000000002,"5min":0.755,"15min":1.1719999999999999},"min":{"1min":0.39900000000000002,"5min":0.371,"15min":0.34499999999999997},"max":{"1min":5.1680000000000001,"5min":5.1680000000000001,"15min":16.265999999999998},"last":0.65000000000000002}]},{"osd":7,"last update":"Sat Sep 23 20:34:08 2023","interfaces":[{"interface":"back","average":{"1min":0.60499999999999998,"5min":0.98999999999999999,"15min":1.222},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.29699999999999999},"max":{"1min":2.262,"5min":7.343,"15min":36.621000000000002},"last":0.58999999999999997},{"interface":"front","average":{"1min":0.78500000000000003,"5min":1.02,"15min":1.1220000000000001},"min":{"1min":0.379,"5min":0.33500000000000002,"15min":0.315},"max":{"1min":2.1789999999999998,"5min":13.122,"15min":37.331000000000003},"last":0.52500000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4722,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1360,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8206,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7156,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3154,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1062932480,"data_stored":1062928408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":921178112,"data_stored":921174040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":757792768,"data_stored":757792768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1247150080,"data_stored":1247150080,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":407371776,"data_stored":407371776,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":525205504,"data_stored":525205504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":987500544,"data_stored":987496472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":795672576,"data_stored":795672576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:35:08.361 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:35:08.367 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:35:08.469 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:35:08.599 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:35:08.717 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:35:08.736 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:35:08.737 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:35:08.838 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:35:08.892 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:35:08.970 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:35:09.400 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:09 smithi018 ceph-mon[179647]: pgmap v844: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 116 active+clean; 1.8 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 6.1 KiB/s rd, 16 KiB/s wr, 515 op/s; 4709/87057 objects misplaced (5.409%); 1.8 MiB/s, 29 objects/s recovering 2023-09-23T20:35:09.401 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:09 smithi018 ceph-mon[184144]: pgmap v844: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 116 active+clean; 1.8 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 6.1 KiB/s rd, 16 KiB/s wr, 515 op/s; 4709/87057 objects misplaced (5.409%); 1.8 MiB/s, 29 objects/s recovering 2023-09-23T20:35:09.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:09 smithi149 ceph-mon[170394]: pgmap v844: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 116 active+clean; 1.8 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 6.1 KiB/s rd, 16 KiB/s wr, 515 op/s; 4709/87057 objects misplaced (5.409%); 1.8 MiB/s, 29 objects/s recovering 2023-09-23T20:35:10.207 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:35:10.217 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:35:10.217 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:35:10.294 INFO:teuthology.orchestra.run.smithi018.stderr:noscrub is set 2023-09-23T20:35:10.318 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:35:10.391 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:35:10.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:10 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1687197000' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:35:10.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:10 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1687197000' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:35:10.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:10 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1687197000' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:35:11.234 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:35:11.242 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:35:11.243 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:35:11.297 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:11 smithi018 ceph-mon[179647]: pgmap v845: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 116 active+clean; 1.7 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 4.8 KiB/s rd, 13 KiB/s wr, 484 op/s; 4488/85149 objects misplaced (5.271%); 1.4 MiB/s, 23 objects/s recovering 2023-09-23T20:35:11.297 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:11 smithi018 ceph-mon[179647]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:35:11.298 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:11 smithi018 ceph-mon[179647]: failed to encode map e358 with expected crc 2023-09-23T20:35:11.298 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:11 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1687197000' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:35:11.298 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:11 smithi018 ceph-mon[179647]: osdmap e358: 8 total, 8 up, 8 in 2023-09-23T20:35:11.298 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:11 smithi018 ceph-mon[184144]: pgmap v845: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 116 active+clean; 1.7 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 4.8 KiB/s rd, 13 KiB/s wr, 484 op/s; 4488/85149 objects misplaced (5.271%); 1.4 MiB/s, 23 objects/s recovering 2023-09-23T20:35:11.298 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:11 smithi018 ceph-mon[184144]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:35:11.298 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:11 smithi018 ceph-mon[184144]: failed to encode map e358 with expected crc 2023-09-23T20:35:11.299 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:11 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1687197000' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:35:11.299 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:11 smithi018 ceph-mon[184144]: osdmap e358: 8 total, 8 up, 8 in 2023-09-23T20:35:11.343 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:35:11.438 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:35:11.510 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:35:11.517 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:35:11.518 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:35:11.551 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:35:11.619 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:35:11.646 INFO:teuthology.orchestra.run.smithi149.stdout:ERROR: (22) Invalid argument 2023-09-23T20:35:11.646 INFO:teuthology.orchestra.run.smithi149.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:35:11.684 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:11 smithi149 ceph-mon[170394]: pgmap v845: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 116 active+clean; 1.7 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 4.8 KiB/s rd, 13 KiB/s wr, 484 op/s; 4488/85149 objects misplaced (5.271%); 1.4 MiB/s, 23 objects/s recovering 2023-09-23T20:35:11.684 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:11 smithi149 ceph-mon[170394]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:35:11.684 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:11 smithi149 ceph-mon[170394]: failed to encode map e358 with expected crc 2023-09-23T20:35:11.684 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:11 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1687197000' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:35:11.684 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:11 smithi149 ceph-mon[170394]: osdmap e358: 8 total, 8 up, 8 in 2023-09-23T20:35:11.757 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:35:11.857 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:35:11.863 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:35:11.864 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:35:11.965 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:35:12.051 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:35:12.055 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T20:35:12.153 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:35:12.159 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:35:12.161 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:35:12.246 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:35:12.262 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:35:12.296 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:35:12.321 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:35:12.351 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:35:12.354 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:35:12.364 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:35:12.364 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:35:12.399 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:35:12.439 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:35:12.465 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:35:12.476 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:35:12.496 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:35:12.518 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:35:12.561 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:35:12.561 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:12 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1850788201' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:35:12.562 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:12 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1850788201' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:35:12.562 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:35:12.563 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:12 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1850788201' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:35:12.564 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:35:12.564 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:35:12.586 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:35:12.586 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:35:12.586 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:35:12.586 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:35:12.586 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:35:12.586 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:35:12.586 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:35:12.587 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:35:12.665 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:35:12.698 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:35:12.743 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:35:12.747 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:35:12.747 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:35:12.795 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:35:12.848 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:35:12.942 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:35:13.078 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:35:13.085 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:35:13.086 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:35:13.187 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:35:13.227 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:35:13.329 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:35:13.331 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:13 smithi149 ceph-mon[170394]: pgmap v847: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 116 active+clean; 1.6 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 4.3 KiB/s rd, 11 KiB/s wr, 637 op/s; 4095/79248 objects misplaced (5.167%); 2.3 MiB/s, 37 objects/s recovering 2023-09-23T20:35:13.332 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:35:13.333 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:35:13.433 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:35:13.518 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:13 smithi018 ceph-mon[184144]: pgmap v847: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 116 active+clean; 1.6 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 4.3 KiB/s rd, 11 KiB/s wr, 637 op/s; 4095/79248 objects misplaced (5.167%); 2.3 MiB/s, 37 objects/s recovering 2023-09-23T20:35:13.526 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:13 smithi018 ceph-mon[179647]: pgmap v847: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 116 active+clean; 1.6 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 4.3 KiB/s rd, 11 KiB/s wr, 637 op/s; 4095/79248 objects misplaced (5.167%); 2.3 MiB/s, 37 objects/s recovering 2023-09-23T20:35:13.535 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:35:13.559 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T20:35:13.631 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:35:13.643 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:35:13.644 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:35:13.745 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:35:13.875 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:35:13.904 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:35:13.922 INFO:teuthology.orchestra.run.smithi018.stderr:dumped all 2023-09-23T20:35:13.991 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:35:13.998 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:35:13.998 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:35:14.099 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:35:14.455 INFO:teuthology.orchestra.run.smithi018.stdout:{"pg_ready":true,"pg_map":{"version":847,"stamp":"2023-09-23T20:35:11.892894+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1733897033,"num_objects":26416,"num_object_clones":0,"num_object_copies":79248,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4095,"num_objects_unfound":0,"num_objects_dirty":26416,"num_whiteouts":0,"num_read":6018,"num_read_kb":13255,"num_write":163630,"num_write_kb":6063972,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10188,"num_bytes_recovered":602952729,"num_keys_recovered":40,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":116926,"ondisk_log_size":116926,"up":357,"acting":356,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":363,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":8433148,"kb_used_data":5637628,"kb_used_omap":32,"kb_used_meta":2795231,"kb_avail":741528068,"statfs":{"total":767960285184,"available":759324741632,"internally_reserved":0,"allocated":5772931072,"data_stored":5761137322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32773,"internal_metadata":2862317563},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-402233133,"num_objects":-6138,"num_object_clones":0,"num_object_copies":-18414,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2916,"num_objects_unfound":0,"num_objects_dirty":-6138,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":6149,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":363,"num_bytes_recovered":23804537,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.678998"},"pg_stats":[{"pgid":"1.18","version":"90'161","reported_seq":608,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T20:35:03.551362+0000","last_change":"2023-09-23T20:35:03.551257+0000","last_active":"2023-09-23T20:35:03.551362+0000","last_peered":"2023-09-23T20:35:03.551362+0000","last_clean":"2023-09-23T20:35:03.551362+0000","last_became_active":"2023-09-23T20:33:39.039296+0000","last_became_peered":"2023-09-23T20:33:39.039296+0000","last_unstale":"2023-09-23T20:35:03.551362+0000","last_undegraded":"2023-09-23T20:35:03.551362+0000","last_fullsized":"2023-09-23T20:35:03.551362+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:35:03.551143+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:35:03.551143+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:58:22.741595+0000","scrub_duration":0.033122453000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"69'1","reported_seq":55,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T20:35:02.229175+0000","last_change":"2023-09-23T20:32:33.233853+0000","last_active":"2023-09-23T20:35:02.229175+0000","last_peered":"2023-09-23T20:35:02.229175+0000","last_clean":"2023-09-23T20:35:02.229175+0000","last_became_active":"2023-09-23T20:32:33.233546+0000","last_became_peered":"2023-09-23T20:32:33.233546+0000","last_unstale":"2023-09-23T20:35:02.229175+0000","last_undegraded":"2023-09-23T20:35:02.229175+0000","last_fullsized":"2023-09-23T20:35:02.229175+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:50:27.130808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":235,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-09-23T20:35:01.226990+0000","last_change":"2023-09-23T20:32:50.536495+0000","last_active":"2023-09-23T20:35:01.226990+0000","last_peered":"2023-09-23T20:35:01.226990+0000","last_clean":"2023-09-23T20:35:01.226990+0000","last_became_active":"2023-09-23T20:32:50.536180+0000","last_became_peered":"2023-09-23T20:32:50.536180+0000","last_unstale":"2023-09-23T20:35:01.226990+0000","last_undegraded":"2023-09-23T20:35:01.226990+0000","last_fullsized":"2023-09-23T20:35:01.226990+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:01:05.524308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1b","version":"90'161","reported_seq":608,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-09-23T20:35:01.224509+0000","last_change":"2023-09-23T20:33:39.013809+0000","last_active":"2023-09-23T20:35:01.224509+0000","last_peered":"2023-09-23T20:35:01.224509+0000","last_clean":"2023-09-23T20:35:01.224509+0000","last_became_active":"2023-09-23T20:33:39.013467+0000","last_became_peered":"2023-09-23T20:33:39.013467+0000","last_unstale":"2023-09-23T20:35:01.224509+0000","last_undegraded":"2023-09-23T20:35:01.224509+0000","last_fullsized":"2023-09-23T20:35:01.224509+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:46.558292+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:46.558292+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0023048779999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":323,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-09-23T20:35:01.221691+0000","last_change":"2023-09-23T20:34:59.983035+0000","last_active":"2023-09-23T20:35:01.221691+0000","last_peered":"2023-09-23T20:35:01.221691+0000","last_clean":"2023-09-23T20:35:01.221691+0000","last_became_active":"2023-09-23T20:34:59.982459+0000","last_became_peered":"2023-09-23T20:34:59.982459+0000","last_unstale":"2023-09-23T20:35:01.221691+0000","last_undegraded":"2023-09-23T20:35:01.221691+0000","last_fullsized":"2023-09-23T20:35:01.221691+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:16:48.510783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":325,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T20:35:02.255596+0000","last_change":"2023-09-23T20:25:02.957185+0000","last_active":"2023-09-23T20:35:02.255596+0000","last_peered":"2023-09-23T20:35:02.255596+0000","last_clean":"2023-09-23T20:35:02.255596+0000","last_became_active":"2023-09-23T20:25:02.956689+0000","last_became_peered":"2023-09-23T20:25:02.956689+0000","last_unstale":"2023-09-23T20:35:02.255596+0000","last_undegraded":"2023-09-23T20:35:02.255596+0000","last_fullsized":"2023-09-23T20:35:02.255596+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:04:28.524015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"90'161","reported_seq":647,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T20:35:06.490310+0000","last_change":"2023-09-23T20:35:06.490239+0000","last_active":"2023-09-23T20:35:06.490310+0000","last_peered":"2023-09-23T20:35:06.490310+0000","last_clean":"2023-09-23T20:35:06.490310+0000","last_became_active":"2023-09-23T20:33:39.000572+0000","last_became_peered":"2023-09-23T20:33:39.000572+0000","last_unstale":"2023-09-23T20:35:06.490310+0000","last_undegraded":"2023-09-23T20:35:06.490310+0000","last_fullsized":"2023-09-23T20:35:06.490310+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:35:06.490149+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:35:06.490149+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:38:46.764412+0000","scrub_duration":0.011140908999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":169,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T20:35:02.249261+0000","last_change":"2023-09-23T20:32:50.546428+0000","last_active":"2023-09-23T20:35:02.249261+0000","last_peered":"2023-09-23T20:35:02.249261+0000","last_clean":"2023-09-23T20:35:02.249261+0000","last_became_active":"2023-09-23T20:32:50.546120+0000","last_became_peered":"2023-09-23T20:32:50.546120+0000","last_unstale":"2023-09-23T20:35:02.249261+0000","last_undegraded":"2023-09-23T20:35:02.249261+0000","last_fullsized":"2023-09-23T20:35:02.249261+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:00:10.792410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":131,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T20:35:02.230986+0000","last_change":"2023-09-23T20:29:46.112977+0000","last_active":"2023-09-23T20:35:02.230986+0000","last_peered":"2023-09-23T20:35:02.230986+0000","last_clean":"2023-09-23T20:35:02.230986+0000","last_became_active":"2023-09-23T20:29:46.112600+0000","last_became_peered":"2023-09-23T20:29:46.112600+0000","last_unstale":"2023-09-23T20:35:02.230986+0000","last_undegraded":"2023-09-23T20:35:02.230986+0000","last_fullsized":"2023-09-23T20:35:02.230986+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:06:34.132247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"90'161","reported_seq":562,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T20:35:02.230043+0000","last_change":"2023-09-23T20:34:53.557349+0000","last_active":"2023-09-23T20:35:02.230043+0000","last_peered":"2023-09-23T20:35:02.230043+0000","last_clean":"2023-09-23T20:35:02.230043+0000","last_became_active":"2023-09-23T20:33:38.988198+0000","last_became_peered":"2023-09-23T20:33:38.988198+0000","last_unstale":"2023-09-23T20:35:02.230043+0000","last_undegraded":"2023-09-23T20:35:02.230043+0000","last_fullsized":"2023-09-23T20:35:02.230043+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:44.509012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:44.509012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0032186319999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":323,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T20:35:02.255387+0000","last_change":"2023-09-23T20:25:02.969432+0000","last_active":"2023-09-23T20:35:02.255387+0000","last_peered":"2023-09-23T20:35:02.255387+0000","last_clean":"2023-09-23T20:35:02.255387+0000","last_became_active":"2023-09-23T20:25:02.969118+0000","last_became_peered":"2023-09-23T20:25:02.969118+0000","last_unstale":"2023-09-23T20:35:02.255387+0000","last_undegraded":"2023-09-23T20:35:02.255387+0000","last_fullsized":"2023-09-23T20:35:02.255387+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:02:22.453831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":60,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-09-23T20:35:01.219087+0000","last_change":"2023-09-23T20:32:01.360359+0000","last_active":"2023-09-23T20:35:01.219087+0000","last_peered":"2023-09-23T20:35:01.219087+0000","last_clean":"2023-09-23T20:35:01.219087+0000","last_became_active":"2023-09-23T20:32:01.360008+0000","last_became_peered":"2023-09-23T20:32:01.360008+0000","last_unstale":"2023-09-23T20:35:01.219087+0000","last_undegraded":"2023-09-23T20:35:01.219087+0000","last_fullsized":"2023-09-23T20:35:01.219087+0000","mapping_epoch":308,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:33:43.135001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.14","version":"90'161","reported_seq":684,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T20:35:02.241040+0000","last_change":"2023-09-23T20:35:02.236398+0000","last_active":"2023-09-23T20:35:02.241040+0000","last_peered":"2023-09-23T20:35:02.241040+0000","last_clean":"2023-09-23T20:35:02.241040+0000","last_became_active":"2023-09-23T20:35:02.235988+0000","last_became_peered":"2023-09-23T20:35:02.235988+0000","last_unstale":"2023-09-23T20:35:02.241040+0000","last_undegraded":"2023-09-23T20:35:02.241040+0000","last_fullsized":"2023-09-23T20:35:02.241040+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:47.479012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:47.479012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:13.519648+0000","scrub_duration":0.0020136590000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":334,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T20:35:02.242144+0000","last_change":"2023-09-23T20:29:45.523808+0000","last_active":"2023-09-23T20:35:02.242144+0000","last_peered":"2023-09-23T20:35:02.242144+0000","last_clean":"2023-09-23T20:35:02.242144+0000","last_became_active":"2023-09-23T20:29:45.523508+0000","last_became_peered":"2023-09-23T20:29:45.523508+0000","last_unstale":"2023-09-23T20:35:02.242144+0000","last_undegraded":"2023-09-23T20:35:02.242144+0000","last_fullsized":"2023-09-23T20:35:02.242144+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:50:55.565352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":157,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T20:35:02.222800+0000","last_change":"2023-09-23T20:32:50.539003+0000","last_active":"2023-09-23T20:35:02.222800+0000","last_peered":"2023-09-23T20:35:02.222800+0000","last_clean":"2023-09-23T20:35:02.222800+0000","last_became_active":"2023-09-23T20:32:50.538596+0000","last_became_peered":"2023-09-23T20:32:50.538596+0000","last_unstale":"2023-09-23T20:35:02.222800+0000","last_undegraded":"2023-09-23T20:35:02.222800+0000","last_fullsized":"2023-09-23T20:35:02.222800+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:56:03.314375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.17","version":"90'161","reported_seq":596,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T20:35:02.230998+0000","last_change":"2023-09-23T20:33:38.990504+0000","last_active":"2023-09-23T20:35:02.230998+0000","last_peered":"2023-09-23T20:35:02.230998+0000","last_clean":"2023-09-23T20:35:02.230998+0000","last_became_active":"2023-09-23T20:33:38.990149+0000","last_became_peered":"2023-09-23T20:33:38.990149+0000","last_unstale":"2023-09-23T20:35:02.230998+0000","last_undegraded":"2023-09-23T20:35:02.230998+0000","last_fullsized":"2023-09-23T20:35:02.230998+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:45.196285+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:45.196285+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0014380860000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":385,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T20:35:02.221888+0000","last_change":"2023-09-23T20:32:33.270617+0000","last_active":"2023-09-23T20:35:02.221888+0000","last_peered":"2023-09-23T20:35:02.221888+0000","last_clean":"2023-09-23T20:35:02.221888+0000","last_became_active":"2023-09-23T20:32:33.227902+0000","last_became_peered":"2023-09-23T20:32:33.227902+0000","last_unstale":"2023-09-23T20:35:02.221888+0000","last_undegraded":"2023-09-23T20:35:02.221888+0000","last_fullsized":"2023-09-23T20:35:02.221888+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:30:26.847831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":360,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":306,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-09-23T20:35:01.222217+0000","last_change":"2023-09-23T20:29:46.080766+0000","last_active":"2023-09-23T20:35:01.222217+0000","last_peered":"2023-09-23T20:35:01.222217+0000","last_clean":"2023-09-23T20:35:01.222217+0000","last_became_active":"2023-09-23T20:29:46.080356+0000","last_became_peered":"2023-09-23T20:29:46.080356+0000","last_unstale":"2023-09-23T20:35:01.222217+0000","last_undegraded":"2023-09-23T20:35:01.222217+0000","last_fullsized":"2023-09-23T20:35:01.222217+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:08:01.862270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"90'161","reported_seq":592,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T20:35:06.045785+0000","last_change":"2023-09-23T20:35:06.045751+0000","last_active":"2023-09-23T20:35:06.045785+0000","last_peered":"2023-09-23T20:35:06.045785+0000","last_clean":"2023-09-23T20:35:06.045785+0000","last_became_active":"2023-09-23T20:33:38.991153+0000","last_became_peered":"2023-09-23T20:33:38.991153+0000","last_unstale":"2023-09-23T20:35:06.045785+0000","last_undegraded":"2023-09-23T20:35:06.045785+0000","last_fullsized":"2023-09-23T20:35:06.045785+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:44.211184+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:44.211184+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0017707720000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":65,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T20:35:02.229671+0000","last_change":"2023-09-23T20:32:33.266787+0000","last_active":"2023-09-23T20:35:02.229671+0000","last_peered":"2023-09-23T20:35:02.229671+0000","last_clean":"2023-09-23T20:35:02.229671+0000","last_became_active":"2023-09-23T20:32:33.228482+0000","last_became_peered":"2023-09-23T20:32:33.228482+0000","last_unstale":"2023-09-23T20:35:02.229671+0000","last_undegraded":"2023-09-23T20:35:02.229671+0000","last_fullsized":"2023-09-23T20:35:02.229671+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:13:01.812073+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":55,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T20:35:02.229748+0000","last_change":"2023-09-23T20:32:33.228929+0000","last_active":"2023-09-23T20:35:02.229748+0000","last_peered":"2023-09-23T20:35:02.229748+0000","last_clean":"2023-09-23T20:35:02.229748+0000","last_became_active":"2023-09-23T20:32:33.228595+0000","last_became_peered":"2023-09-23T20:32:33.228595+0000","last_unstale":"2023-09-23T20:35:02.229748+0000","last_undegraded":"2023-09-23T20:35:02.229748+0000","last_fullsized":"2023-09-23T20:35:02.229748+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:17:53.053808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"90'161","reported_seq":588,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T20:35:05.520068+0000","last_change":"2023-09-23T20:31:43.578810+0000","last_active":"2023-09-23T20:35:05.520068+0000","last_peered":"2023-09-23T20:35:05.520068+0000","last_clean":"2023-09-23T20:35:05.520068+0000","last_became_active":"2023-09-23T20:28:05.991304+0000","last_became_peered":"2023-09-23T20:28:05.991304+0000","last_unstale":"2023-09-23T20:35:05.520068+0000","last_undegraded":"2023-09-23T20:35:05.520068+0000","last_fullsized":"2023-09-23T20:35:05.520068+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:43.578666+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:43.578666+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:46:18.999991+0000","scrub_duration":0.0021890379999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":384,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T20:35:02.222538+0000","last_change":"2023-09-23T20:32:33.242943+0000","last_active":"2023-09-23T20:35:02.222538+0000","last_peered":"2023-09-23T20:35:02.222538+0000","last_clean":"2023-09-23T20:35:02.222538+0000","last_became_active":"2023-09-23T20:32:33.225468+0000","last_became_peered":"2023-09-23T20:32:33.225468+0000","last_unstale":"2023-09-23T20:35:02.222538+0000","last_undegraded":"2023-09-23T20:35:02.222538+0000","last_fullsized":"2023-09-23T20:35:02.222538+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:40:47.867573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":59,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T20:35:02.241974+0000","last_change":"2023-09-23T20:32:01.358590+0000","last_active":"2023-09-23T20:35:02.241974+0000","last_peered":"2023-09-23T20:35:02.241974+0000","last_clean":"2023-09-23T20:35:02.241974+0000","last_became_active":"2023-09-23T20:32:01.358345+0000","last_became_peered":"2023-09-23T20:32:01.358345+0000","last_unstale":"2023-09-23T20:35:02.241974+0000","last_undegraded":"2023-09-23T20:35:02.241974+0000","last_fullsized":"2023-09-23T20:35:02.241974+0000","mapping_epoch":308,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:22:21.159708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.10","version":"90'161","reported_seq":588,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T20:35:02.241747+0000","last_change":"2023-09-23T20:31:49.726214+0000","last_active":"2023-09-23T20:35:02.241747+0000","last_peered":"2023-09-23T20:35:02.241747+0000","last_clean":"2023-09-23T20:35:02.241747+0000","last_became_active":"2023-09-23T20:29:07.044892+0000","last_became_peered":"2023-09-23T20:29:07.044892+0000","last_unstale":"2023-09-23T20:35:02.241747+0000","last_undegraded":"2023-09-23T20:35:02.241747+0000","last_fullsized":"2023-09-23T20:35:02.241747+0000","mapping_epoch":253,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":254,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:49.726139+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:49.726139+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:59:11.810516+0000","scrub_duration":0.0015441470000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":323,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T20:35:02.254610+0000","last_change":"2023-09-23T20:25:02.953993+0000","last_active":"2023-09-23T20:35:02.254610+0000","last_peered":"2023-09-23T20:35:02.254610+0000","last_clean":"2023-09-23T20:35:02.254610+0000","last_became_active":"2023-09-23T20:25:02.953672+0000","last_became_peered":"2023-09-23T20:25:02.953672+0000","last_unstale":"2023-09-23T20:35:02.254610+0000","last_undegraded":"2023-09-23T20:35:02.254610+0000","last_fullsized":"2023-09-23T20:35:02.254610+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:26:08.603577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":336,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T20:35:02.242157+0000","last_change":"2023-09-23T20:25:02.949584+0000","last_active":"2023-09-23T20:35:02.242157+0000","last_peered":"2023-09-23T20:35:02.242157+0000","last_clean":"2023-09-23T20:35:02.242157+0000","last_became_active":"2023-09-23T20:25:02.949132+0000","last_became_peered":"2023-09-23T20:25:02.949132+0000","last_unstale":"2023-09-23T20:35:02.242157+0000","last_undegraded":"2023-09-23T20:35:02.242157+0000","last_fullsized":"2023-09-23T20:35:02.242157+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:34:20.509823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"90'161","reported_seq":640,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T20:35:02.238923+0000","last_change":"2023-09-23T20:35:02.238923+0000","last_active":"2023-09-23T20:35:02.238923+0000","last_peered":"2023-09-23T20:35:02.238923+0000","last_clean":"2023-09-23T20:35:02.238923+0000","last_became_active":"2023-09-23T20:35:02.238293+0000","last_became_peered":"2023-09-23T20:35:02.238293+0000","last_unstale":"2023-09-23T20:35:02.238923+0000","last_undegraded":"2023-09-23T20:35:02.238923+0000","last_fullsized":"2023-09-23T20:35:02.238923+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:42.544308+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:42.544308+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:51:22.281932+0000","scrub_duration":0.0029958179999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":341,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T20:35:02.222732+0000","last_change":"2023-09-23T20:32:33.225148+0000","last_active":"2023-09-23T20:35:02.222732+0000","last_peered":"2023-09-23T20:35:02.222732+0000","last_clean":"2023-09-23T20:35:02.222732+0000","last_became_active":"2023-09-23T20:32:33.224805+0000","last_became_peered":"2023-09-23T20:32:33.224805+0000","last_unstale":"2023-09-23T20:35:02.222732+0000","last_undegraded":"2023-09-23T20:35:02.222732+0000","last_fullsized":"2023-09-23T20:35:02.222732+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:55:30.891398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":333,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T20:35:02.255258+0000","last_change":"2023-09-23T20:25:02.957010+0000","last_active":"2023-09-23T20:35:02.255258+0000","last_peered":"2023-09-23T20:35:02.255258+0000","last_clean":"2023-09-23T20:35:02.255258+0000","last_became_active":"2023-09-23T20:25:02.956267+0000","last_became_peered":"2023-09-23T20:25:02.956267+0000","last_unstale":"2023-09-23T20:35:02.255258+0000","last_undegraded":"2023-09-23T20:35:02.255258+0000","last_fullsized":"2023-09-23T20:35:02.255258+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:10:55.299422+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"90'161","reported_seq":607,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T20:35:02.242121+0000","last_change":"2023-09-23T20:31:53.707033+0000","last_active":"2023-09-23T20:35:02.242121+0000","last_peered":"2023-09-23T20:35:02.242121+0000","last_clean":"2023-09-23T20:35:02.242121+0000","last_became_active":"2023-09-23T20:29:46.078873+0000","last_became_peered":"2023-09-23T20:29:46.078873+0000","last_unstale":"2023-09-23T20:35:02.242121+0000","last_undegraded":"2023-09-23T20:35:02.242121+0000","last_fullsized":"2023-09-23T20:35:02.242121+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:53.706934+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:53.706934+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:51:58.039657+0000","scrub_duration":0.0016840900000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":127,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T20:35:02.229629+0000","last_change":"2023-09-23T20:29:04.179171+0000","last_active":"2023-09-23T20:35:02.229629+0000","last_peered":"2023-09-23T20:35:02.229629+0000","last_clean":"2023-09-23T20:35:02.229629+0000","last_became_active":"2023-09-23T20:29:04.128672+0000","last_became_peered":"2023-09-23T20:29:04.128672+0000","last_unstale":"2023-09-23T20:35:02.229629+0000","last_undegraded":"2023-09-23T20:35:02.229629+0000","last_fullsized":"2023-09-23T20:35:02.229629+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:49:15.506106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":170,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T20:35:02.224422+0000","last_change":"2023-09-23T20:32:50.539157+0000","last_active":"2023-09-23T20:35:02.224422+0000","last_peered":"2023-09-23T20:35:02.224422+0000","last_clean":"2023-09-23T20:35:02.224422+0000","last_became_active":"2023-09-23T20:32:50.538889+0000","last_became_peered":"2023-09-23T20:32:50.538889+0000","last_unstale":"2023-09-23T20:35:02.224422+0000","last_undegraded":"2023-09-23T20:35:02.224422+0000","last_fullsized":"2023-09-23T20:35:02.224422+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:28:28.857256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.b","version":"357'10191","reported_seq":10255,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T20:35:09.136496+0000","last_change":"2023-09-23T20:31:49.152343+0000","last_active":"2023-09-23T20:35:09.136496+0000","last_peered":"2023-09-23T20:35:09.136496+0000","last_clean":"2023-09-23T20:35:09.136496+0000","last_became_active":"2023-09-23T20:31:49.152042+0000","last_became_peered":"2023-09-23T20:31:49.152042+0000","last_unstale":"2023-09-23T20:35:09.136496+0000","last_undegraded":"2023-09-23T20:35:09.136496+0000","last_fullsized":"2023-09-23T20:35:09.136496+0000","mapping_epoch":303,"log_start":"311'3590","ondisk_log_start":"311'3590","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":6601,"log_dups_size":0,"ondisk_log_size":6601,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:53:34.237468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109117440,"num_objects":1665,"num_object_clones":0,"num_object_copies":4995,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1665,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10191,"num_write_kb":379392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.d","version":"90'161","reported_seq":650,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T20:35:02.221699+0000","last_change":"2023-09-23T20:34:59.496438+0000","last_active":"2023-09-23T20:35:02.221699+0000","last_peered":"2023-09-23T20:35:02.221699+0000","last_clean":"2023-09-23T20:35:02.221699+0000","last_became_active":"2023-09-23T20:33:38.970527+0000","last_became_peered":"2023-09-23T20:33:38.970527+0000","last_unstale":"2023-09-23T20:35:02.221699+0000","last_undegraded":"2023-09-23T20:35:02.221699+0000","last_fullsized":"2023-09-23T20:35:02.221699+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:34:59.496375+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:34:59.496375+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:46:18.244734+0000","scrub_duration":0.0017843189999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":361,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T20:35:02.224204+0000","last_change":"2023-09-23T20:32:33.283752+0000","last_active":"2023-09-23T20:35:02.224204+0000","last_peered":"2023-09-23T20:35:02.224204+0000","last_clean":"2023-09-23T20:35:02.224204+0000","last_became_active":"2023-09-23T20:32:33.230503+0000","last_became_peered":"2023-09-23T20:32:33.230503+0000","last_unstale":"2023-09-23T20:35:02.224204+0000","last_undegraded":"2023-09-23T20:35:02.224204+0000","last_fullsized":"2023-09-23T20:35:02.224204+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:29.084410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":319,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T20:35:02.241980+0000","last_change":"2023-09-23T20:29:04.128604+0000","last_active":"2023-09-23T20:35:02.241980+0000","last_peered":"2023-09-23T20:35:02.241980+0000","last_clean":"2023-09-23T20:35:02.241980+0000","last_became_active":"2023-09-23T20:29:04.128003+0000","last_became_peered":"2023-09-23T20:29:04.128003+0000","last_unstale":"2023-09-23T20:35:02.241980+0000","last_undegraded":"2023-09-23T20:35:02.241980+0000","last_fullsized":"2023-09-23T20:35:02.241980+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:50:49.385599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"357'10102","reported_seq":10173,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T20:35:07.659650+0000","last_change":"2023-09-23T20:32:50.543873+0000","last_active":"2023-09-23T20:35:07.659650+0000","last_peered":"2023-09-23T20:35:07.659650+0000","last_clean":"2023-09-23T20:35:07.659650+0000","last_became_active":"2023-09-23T20:32:50.537253+0000","last_became_peered":"2023-09-23T20:32:50.537253+0000","last_unstale":"2023-09-23T20:35:07.659650+0000","last_undegraded":"2023-09-23T20:35:07.659650+0000","last_fullsized":"2023-09-23T20:35:07.659650+0000","mapping_epoch":319,"log_start":"310'2400","ondisk_log_start":"310'2400","created":303,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":7702,"log_dups_size":0,"ondisk_log_size":7702,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:38:02.452291+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112066560,"num_objects":1710,"num_object_clones":0,"num_object_copies":5130,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1710,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10102,"num_write_kb":377984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,5],"acting":[2,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.c","version":"90'161","reported_seq":529,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T20:35:05.520169+0000","last_change":"2023-09-23T20:33:38.989595+0000","last_active":"2023-09-23T20:35:05.520169+0000","last_peered":"2023-09-23T20:35:05.520169+0000","last_clean":"2023-09-23T20:35:05.520169+0000","last_became_active":"2023-09-23T20:33:38.989255+0000","last_became_peered":"2023-09-23T20:33:38.989255+0000","last_unstale":"2023-09-23T20:35:05.520169+0000","last_undegraded":"2023-09-23T20:35:05.520169+0000","last_fullsized":"2023-09-23T20:35:05.520169+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:41.701171+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:41.701171+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0019955509999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":54613,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":396,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T20:35:02.223009+0000","last_change":"2023-09-23T20:34:59.987981+0000","last_active":"2023-09-23T20:35:02.223009+0000","last_peered":"2023-09-23T20:35:02.223009+0000","last_clean":"2023-09-23T20:35:02.223009+0000","last_became_active":"2023-09-23T20:34:59.985368+0000","last_became_peered":"2023-09-23T20:34:59.985368+0000","last_unstale":"2023-09-23T20:35:02.223009+0000","last_undegraded":"2023-09-23T20:35:02.223009+0000","last_fullsized":"2023-09-23T20:35:02.223009+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:12:43.082493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":55,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T20:35:02.229790+0000","last_change":"2023-09-23T20:32:33.232142+0000","last_active":"2023-09-23T20:35:02.229790+0000","last_peered":"2023-09-23T20:35:02.229790+0000","last_clean":"2023-09-23T20:35:02.229790+0000","last_became_active":"2023-09-23T20:32:33.231683+0000","last_became_peered":"2023-09-23T20:32:33.231683+0000","last_unstale":"2023-09-23T20:35:02.229790+0000","last_undegraded":"2023-09-23T20:35:02.229790+0000","last_fullsized":"2023-09-23T20:35:02.229790+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:02:58.336348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"358'10115","reported_seq":12432,"reported_epoch":358,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T20:35:10.507219+0000","last_change":"2023-09-23T20:35:01.111484+0000","last_active":"2023-09-23T20:35:10.507219+0000","last_peered":"2023-09-23T20:35:10.507219+0000","last_clean":"2023-09-23T20:34:58.962913+0000","last_became_active":"2023-09-23T20:35:01.087993+0000","last_became_peered":"2023-09-23T20:35:01.087993+0000","last_unstale":"2023-09-23T20:35:10.507219+0000","last_undegraded":"2023-09-23T20:35:10.507219+0000","last_fullsized":"2023-09-23T20:35:10.507219+0000","mapping_epoch":354,"log_start":"310'3200","ondisk_log_start":"310'3200","created":303,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":6915,"log_dups_size":0,"ondisk_log_size":6915,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:09:02.548463+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":105709568,"num_objects":1613,"num_object_clones":0,"num_object_copies":4839,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1613,"num_objects_unfound":0,"num_objects_dirty":1613,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10115,"num_write_kb":375296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1109,"num_bytes_recovered":72679424,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,7],"avail_no_missing":["0","1","7"],"object_location_counts":[{"shards":"0,1,7","objects":1613}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"90'161","reported_seq":580,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T20:35:02.230871+0000","last_change":"2023-09-23T20:32:36.302809+0000","last_active":"2023-09-23T20:35:02.230871+0000","last_peered":"2023-09-23T20:35:02.230871+0000","last_clean":"2023-09-23T20:35:02.230871+0000","last_became_active":"2023-09-23T20:32:36.302465+0000","last_became_peered":"2023-09-23T20:32:36.302465+0000","last_unstale":"2023-09-23T20:35:02.230871+0000","last_undegraded":"2023-09-23T20:35:02.230871+0000","last_fullsized":"2023-09-23T20:35:02.230871+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:46.217489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:46.217489+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:35:40.488427+0000","scrub_duration":0.001456429,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":330,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T20:35:02.221764+0000","last_change":"2023-09-23T20:29:04.184893+0000","last_active":"2023-09-23T20:35:02.221764+0000","last_peered":"2023-09-23T20:35:02.221764+0000","last_clean":"2023-09-23T20:35:02.221764+0000","last_became_active":"2023-09-23T20:29:04.152723+0000","last_became_peered":"2023-09-23T20:29:04.152723+0000","last_unstale":"2023-09-23T20:35:02.221764+0000","last_undegraded":"2023-09-23T20:35:02.221764+0000","last_fullsized":"2023-09-23T20:35:02.221764+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:51:27.017151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":131,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T20:35:02.229127+0000","last_change":"2023-09-23T20:29:46.086912+0000","last_active":"2023-09-23T20:35:02.229127+0000","last_peered":"2023-09-23T20:35:02.229127+0000","last_clean":"2023-09-23T20:35:02.229127+0000","last_became_active":"2023-09-23T20:29:46.085471+0000","last_became_peered":"2023-09-23T20:29:46.085471+0000","last_unstale":"2023-09-23T20:35:02.229127+0000","last_undegraded":"2023-09-23T20:35:02.229127+0000","last_fullsized":"2023-09-23T20:35:02.229127+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:04:55.751660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.8","version":"357'10113","reported_seq":17472,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T20:35:07.660325+0000","last_change":"2023-09-23T20:34:05.581246+0000","last_active":"2023-09-23T20:35:07.660325+0000","last_peered":"2023-09-23T20:35:07.660325+0000","last_clean":"2023-09-23T20:35:07.660325+0000","last_became_active":"2023-09-23T20:34:05.560057+0000","last_became_peered":"2023-09-23T20:34:05.560057+0000","last_unstale":"2023-09-23T20:35:07.660325+0000","last_undegraded":"2023-09-23T20:35:07.660325+0000","last_fullsized":"2023-09-23T20:35:07.660325+0000","mapping_epoch":338,"log_start":"310'2400","ondisk_log_start":"310'2400","created":303,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":7713,"log_dups_size":0,"ondisk_log_size":7713,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:08:18.482746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":110690304,"num_objects":1689,"num_object_clones":0,"num_object_copies":5067,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1689,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10113,"num_write_kb":377664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3638,"num_bytes_recovered":238419968,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.e","version":"90'161","reported_seq":448,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T20:35:02.234312+0000","last_change":"2023-09-23T20:35:02.233909+0000","last_active":"2023-09-23T20:35:02.234312+0000","last_peered":"2023-09-23T20:35:02.234312+0000","last_clean":"2023-09-23T20:35:02.234312+0000","last_became_active":"2023-09-23T20:35:02.233536+0000","last_became_peered":"2023-09-23T20:35:02.233536+0000","last_unstale":"2023-09-23T20:35:02.234312+0000","last_undegraded":"2023-09-23T20:35:02.234312+0000","last_fullsized":"2023-09-23T20:35:02.234312+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:44.386120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:44.386120+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":364,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T20:35:02.242238+0000","last_change":"2023-09-23T20:29:04.138284+0000","last_active":"2023-09-23T20:35:02.242238+0000","last_peered":"2023-09-23T20:35:02.242238+0000","last_clean":"2023-09-23T20:35:02.242238+0000","last_became_active":"2023-09-23T20:29:04.137960+0000","last_became_peered":"2023-09-23T20:29:04.137960+0000","last_unstale":"2023-09-23T20:35:02.242238+0000","last_undegraded":"2023-09-23T20:35:02.242238+0000","last_fullsized":"2023-09-23T20:35:02.242238+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:18:59.548311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":341,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T20:35:02.249118+0000","last_change":"2023-09-23T20:32:33.236122+0000","last_active":"2023-09-23T20:35:02.249118+0000","last_peered":"2023-09-23T20:35:02.249118+0000","last_clean":"2023-09-23T20:35:02.249118+0000","last_became_active":"2023-09-23T20:32:33.235384+0000","last_became_peered":"2023-09-23T20:32:33.235384+0000","last_unstale":"2023-09-23T20:35:02.249118+0000","last_undegraded":"2023-09-23T20:35:02.249118+0000","last_fullsized":"2023-09-23T20:35:02.249118+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:56.843733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"358'10285","reported_seq":10350,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-09-23T20:35:10.499946+0000","last_change":"2023-09-23T20:31:49.158461+0000","last_active":"2023-09-23T20:35:10.499946+0000","last_peered":"2023-09-23T20:35:10.499946+0000","last_clean":"2023-09-23T20:35:10.499946+0000","last_became_active":"2023-09-23T20:31:49.158188+0000","last_became_peered":"2023-09-23T20:31:49.158188+0000","last_unstale":"2023-09-23T20:35:10.499946+0000","last_undegraded":"2023-09-23T20:35:10.499946+0000","last_fullsized":"2023-09-23T20:35:10.499946+0000","mapping_epoch":303,"log_start":"310'3400","ondisk_log_start":"310'3400","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":6885,"log_dups_size":0,"ondisk_log_size":6885,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:31:57.678090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109510656,"num_objects":1671,"num_object_clones":0,"num_object_copies":5013,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1671,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10285,"num_write_kb":382592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":763,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T20:35:02.523393+0000","last_change":"2023-09-23T20:35:02.523308+0000","last_active":"2023-09-23T20:35:02.523393+0000","last_peered":"2023-09-23T20:35:02.523393+0000","last_clean":"2023-09-23T20:35:02.523393+0000","last_became_active":"2023-09-23T20:33:38.979989+0000","last_became_peered":"2023-09-23T20:33:38.979989+0000","last_unstale":"2023-09-23T20:35:02.523393+0000","last_undegraded":"2023-09-23T20:35:02.523393+0000","last_fullsized":"2023-09-23T20:35:02.523393+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:35:02.523203+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:35:02.523203+0000","last_clean_scrub_stamp":"2023-09-23T20:35:02.523203+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:02:41.472958+0000","scrub_duration":0.0085927989999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":93,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T20:35:05.520113+0000","last_change":"2023-09-23T20:32:50.538500+0000","last_active":"2023-09-23T20:35:05.520113+0000","last_peered":"2023-09-23T20:35:05.520113+0000","last_clean":"2023-09-23T20:35:05.520113+0000","last_became_active":"2023-09-23T20:32:50.538152+0000","last_became_peered":"2023-09-23T20:32:50.538152+0000","last_unstale":"2023-09-23T20:35:05.520113+0000","last_undegraded":"2023-09-23T20:35:05.520113+0000","last_fullsized":"2023-09-23T20:35:05.520113+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:00:41.103508+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":55,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T20:35:02.229752+0000","last_change":"2023-09-23T20:32:33.231614+0000","last_active":"2023-09-23T20:35:02.229752+0000","last_peered":"2023-09-23T20:35:02.229752+0000","last_clean":"2023-09-23T20:35:02.229752+0000","last_became_active":"2023-09-23T20:32:33.231207+0000","last_became_peered":"2023-09-23T20:32:33.231207+0000","last_unstale":"2023-09-23T20:35:02.229752+0000","last_undegraded":"2023-09-23T20:35:02.229752+0000","last_fullsized":"2023-09-23T20:35:02.229752+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:46:09.780829+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"357'10091","reported_seq":10155,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T20:35:07.587240+0000","last_change":"2023-09-23T20:31:49.155754+0000","last_active":"2023-09-23T20:35:07.587240+0000","last_peered":"2023-09-23T20:35:07.587240+0000","last_clean":"2023-09-23T20:35:07.587240+0000","last_became_active":"2023-09-23T20:31:49.155363+0000","last_became_peered":"2023-09-23T20:31:49.155363+0000","last_unstale":"2023-09-23T20:35:07.587240+0000","last_undegraded":"2023-09-23T20:35:07.587240+0000","last_fullsized":"2023-09-23T20:35:07.587240+0000","mapping_epoch":303,"log_start":"310'2400","ondisk_log_start":"310'2400","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":7691,"log_dups_size":0,"ondisk_log_size":7691,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:45:42.661641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":115146752,"num_objects":1757,"num_object_clones":0,"num_object_copies":5271,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1757,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10091,"num_write_kb":379136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":608,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T20:35:02.241839+0000","last_change":"2023-09-23T20:35:00.496271+0000","last_active":"2023-09-23T20:35:02.241839+0000","last_peered":"2023-09-23T20:35:02.241839+0000","last_clean":"2023-09-23T20:35:02.241839+0000","last_became_active":"2023-09-23T20:33:38.996273+0000","last_became_peered":"2023-09-23T20:33:38.996273+0000","last_unstale":"2023-09-23T20:35:02.241839+0000","last_undegraded":"2023-09-23T20:35:02.241839+0000","last_fullsized":"2023-09-23T20:35:02.241839+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:35:00.496197+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:35:00.496197+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:30:15.813493+0000","scrub_duration":0.0016339779999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":153,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T20:35:02.229207+0000","last_change":"2023-09-23T20:32:33.229700+0000","last_active":"2023-09-23T20:35:02.229207+0000","last_peered":"2023-09-23T20:35:02.229207+0000","last_clean":"2023-09-23T20:35:02.229207+0000","last_became_active":"2023-09-23T20:32:33.229372+0000","last_became_peered":"2023-09-23T20:32:33.229372+0000","last_unstale":"2023-09-23T20:35:02.229207+0000","last_undegraded":"2023-09-23T20:35:02.229207+0000","last_fullsized":"2023-09-23T20:35:02.229207+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:18:10.958227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":14,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T20:35:02.229467+0000","last_change":"2023-09-23T20:34:59.985545+0000","last_active":"2023-09-23T20:35:02.229467+0000","last_peered":"2023-09-23T20:35:02.229467+0000","last_clean":"2023-09-23T20:35:02.229467+0000","last_became_active":"2023-09-23T20:34:59.985090+0000","last_became_peered":"2023-09-23T20:34:59.985090+0000","last_unstale":"2023-09-23T20:35:02.229467+0000","last_undegraded":"2023-09-23T20:35:02.229467+0000","last_fullsized":"2023-09-23T20:35:02.229467+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:21:40.825065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"358'10323","reported_seq":10387,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-09-23T20:35:10.523167+0000","last_change":"2023-09-23T20:31:49.158641+0000","last_active":"2023-09-23T20:35:10.523167+0000","last_peered":"2023-09-23T20:35:10.523167+0000","last_clean":"2023-09-23T20:35:10.523167+0000","last_became_active":"2023-09-23T20:31:49.157604+0000","last_became_peered":"2023-09-23T20:31:49.157604+0000","last_unstale":"2023-09-23T20:35:10.523167+0000","last_undegraded":"2023-09-23T20:35:10.523167+0000","last_fullsized":"2023-09-23T20:35:10.523167+0000","mapping_epoch":303,"log_start":"317'3995","ondisk_log_start":"317'3995","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":6328,"log_dups_size":0,"ondisk_log_size":6328,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:54:04.726000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109314072,"num_objects":1669,"num_object_clones":0,"num_object_copies":5007,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1669,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10323,"num_write_kb":383681,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.b","version":"90'161","reported_seq":609,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-09-23T20:35:01.226341+0000","last_change":"2023-09-23T20:33:38.992494+0000","last_active":"2023-09-23T20:35:01.226341+0000","last_peered":"2023-09-23T20:35:01.226341+0000","last_clean":"2023-09-23T20:35:01.226341+0000","last_became_active":"2023-09-23T20:33:38.991910+0000","last_became_peered":"2023-09-23T20:33:38.991910+0000","last_unstale":"2023-09-23T20:35:01.226341+0000","last_undegraded":"2023-09-23T20:35:01.226341+0000","last_fullsized":"2023-09-23T20:35:01.226341+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:46.558292+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:46.558292+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0023048779999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":14,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T20:35:02.229011+0000","last_change":"2023-09-23T20:34:59.983122+0000","last_active":"2023-09-23T20:35:02.229011+0000","last_peered":"2023-09-23T20:35:02.229011+0000","last_clean":"2023-09-23T20:35:02.229011+0000","last_became_active":"2023-09-23T20:34:59.982809+0000","last_became_peered":"2023-09-23T20:34:59.982809+0000","last_unstale":"2023-09-23T20:35:02.229011+0000","last_undegraded":"2023-09-23T20:35:02.229011+0000","last_fullsized":"2023-09-23T20:35:02.229011+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:49:40.839723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":55,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T20:35:02.231831+0000","last_change":"2023-09-23T20:32:33.224963+0000","last_active":"2023-09-23T20:35:02.231831+0000","last_peered":"2023-09-23T20:35:02.231831+0000","last_clean":"2023-09-23T20:35:02.231831+0000","last_became_active":"2023-09-23T20:32:33.224634+0000","last_became_peered":"2023-09-23T20:32:33.224634+0000","last_unstale":"2023-09-23T20:35:02.231831+0000","last_undegraded":"2023-09-23T20:35:02.231831+0000","last_fullsized":"2023-09-23T20:35:02.231831+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:41:40.314140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"358'10257","reported_seq":10318,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-09-23T20:35:10.459967+0000","last_change":"2023-09-23T20:31:49.157247+0000","last_active":"2023-09-23T20:35:10.459967+0000","last_peered":"2023-09-23T20:35:10.459967+0000","last_clean":"2023-09-23T20:35:10.459967+0000","last_became_active":"2023-09-23T20:31:49.156961+0000","last_became_peered":"2023-09-23T20:31:49.156961+0000","last_unstale":"2023-09-23T20:35:10.459967+0000","last_undegraded":"2023-09-23T20:35:10.459967+0000","last_fullsized":"2023-09-23T20:35:10.459967+0000","mapping_epoch":303,"log_start":"317'4001","ondisk_log_start":"317'4001","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":6256,"log_dups_size":0,"ondisk_log_size":6256,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:49:17.346795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":105054208,"num_objects":1603,"num_object_clones":0,"num_object_copies":4809,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1603,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10257,"num_write_kb":379520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":651,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T20:35:02.242086+0000","last_change":"2023-09-23T20:34:59.481511+0000","last_active":"2023-09-23T20:35:02.242086+0000","last_peered":"2023-09-23T20:35:02.242086+0000","last_clean":"2023-09-23T20:35:02.242086+0000","last_became_active":"2023-09-23T20:33:38.988954+0000","last_became_peered":"2023-09-23T20:33:38.988954+0000","last_unstale":"2023-09-23T20:35:02.242086+0000","last_undegraded":"2023-09-23T20:35:02.242086+0000","last_fullsized":"2023-09-23T20:35:02.242086+0000","mapping_epoch":253,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:34:59.481446+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:34:59.481446+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:53:42.768648+0000","scrub_duration":0.001554773,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":170,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T20:35:02.224923+0000","last_change":"2023-09-23T20:32:50.530690+0000","last_active":"2023-09-23T20:35:02.224923+0000","last_peered":"2023-09-23T20:35:02.224923+0000","last_clean":"2023-09-23T20:35:02.224923+0000","last_became_active":"2023-09-23T20:32:50.530376+0000","last_became_peered":"2023-09-23T20:32:50.530376+0000","last_unstale":"2023-09-23T20:35:02.224923+0000","last_undegraded":"2023-09-23T20:35:02.224923+0000","last_fullsized":"2023-09-23T20:35:02.224923+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:24:53.184959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":178,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-09-23T20:35:01.221971+0000","last_change":"2023-09-23T20:32:33.230582+0000","last_active":"2023-09-23T20:35:01.221971+0000","last_peered":"2023-09-23T20:35:01.221971+0000","last_clean":"2023-09-23T20:35:01.221971+0000","last_became_active":"2023-09-23T20:32:33.230257+0000","last_became_peered":"2023-09-23T20:32:33.230257+0000","last_unstale":"2023-09-23T20:35:01.221971+0000","last_undegraded":"2023-09-23T20:35:01.221971+0000","last_fullsized":"2023-09-23T20:35:01.221971+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:11:45.333096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"357'10077","reported_seq":11206,"reported_epoch":357,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T20:35:09.124799+0000","last_change":"2023-09-23T20:35:01.122419+0000","last_active":"2023-09-23T20:35:09.124799+0000","last_peered":"2023-09-23T20:35:09.124799+0000","last_clean":"2023-09-23T20:34:58.841504+0000","last_became_active":"2023-09-23T20:35:01.087671+0000","last_became_peered":"2023-09-23T20:35:01.087671+0000","last_unstale":"2023-09-23T20:35:09.124799+0000","last_undegraded":"2023-09-23T20:35:09.124799+0000","last_fullsized":"2023-09-23T20:35:09.124799+0000","mapping_epoch":354,"log_start":"315'3647","ondisk_log_start":"315'3647","created":303,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":6430,"log_dups_size":0,"ondisk_log_size":6430,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:29:52.175555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109117440,"num_objects":1665,"num_object_clones":0,"num_object_copies":4995,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1665,"num_objects_unfound":0,"num_objects_dirty":1665,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10077,"num_write_kb":375744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":521,"num_bytes_recovered":34144256,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[1,3,6],"avail_no_missing":["1","3","6"],"object_location_counts":[{"shards":"1,3,6","objects":1665}],"blocked_by":[],"up_primary":4,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":607,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T20:35:04.102151+0000","last_change":"2023-09-23T20:35:04.102089+0000","last_active":"2023-09-23T20:35:04.102151+0000","last_peered":"2023-09-23T20:35:04.102151+0000","last_clean":"2023-09-23T20:35:04.102151+0000","last_became_active":"2023-09-23T20:33:38.978580+0000","last_became_peered":"2023-09-23T20:33:38.978580+0000","last_unstale":"2023-09-23T20:35:04.102151+0000","last_undegraded":"2023-09-23T20:35:04.102151+0000","last_fullsized":"2023-09-23T20:35:04.102151+0000","mapping_epoch":253,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:35:04.101972+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:35:04.101972+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:44:38.790944+0000","scrub_duration":0.0063459650000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"86'119","reported_seq":662,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-09-23T20:35:01.221774+0000","last_change":"2023-09-23T20:29:45.521257+0000","last_active":"2023-09-23T20:35:01.221774+0000","last_peered":"2023-09-23T20:35:01.221774+0000","last_clean":"2023-09-23T20:35:01.221774+0000","last_became_active":"2023-09-23T20:29:45.520546+0000","last_became_peered":"2023-09-23T20:29:45.520546+0000","last_unstale":"2023-09-23T20:35:01.221774+0000","last_undegraded":"2023-09-23T20:35:01.221774+0000","last_fullsized":"2023-09-23T20:35:01.221774+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":119,"log_dups_size":0,"ondisk_log_size":119,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:58:56.085777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032705499999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":523,"num_read_kb":3503,"num_write":136,"num_write_kb":3128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":374,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T20:35:02.242121+0000","last_change":"2023-09-23T20:25:39.637160+0000","last_active":"2023-09-23T20:35:02.242121+0000","last_peered":"2023-09-23T20:35:02.242121+0000","last_clean":"2023-09-23T20:35:02.242121+0000","last_became_active":"2023-09-23T20:25:39.636860+0000","last_became_peered":"2023-09-23T20:25:39.636860+0000","last_unstale":"2023-09-23T20:35:02.242121+0000","last_undegraded":"2023-09-23T20:35:02.242121+0000","last_fullsized":"2023-09-23T20:35:02.242121+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:21:47.860063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":291,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-09-23T20:35:01.221469+0000","last_change":"2023-09-23T20:34:59.985572+0000","last_active":"2023-09-23T20:35:01.221469+0000","last_peered":"2023-09-23T20:35:01.221469+0000","last_clean":"2023-09-23T20:35:01.221469+0000","last_became_active":"2023-09-23T20:34:59.985241+0000","last_became_peered":"2023-09-23T20:34:59.985241+0000","last_unstale":"2023-09-23T20:35:01.221469+0000","last_undegraded":"2023-09-23T20:35:01.221469+0000","last_fullsized":"2023-09-23T20:35:01.221469+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:54:57.081334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"358'10280","reported_seq":10343,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-09-23T20:35:10.586719+0000","last_change":"2023-09-23T20:31:49.157301+0000","last_active":"2023-09-23T20:35:10.586719+0000","last_peered":"2023-09-23T20:35:10.586719+0000","last_clean":"2023-09-23T20:35:10.586719+0000","last_became_active":"2023-09-23T20:31:49.156992+0000","last_became_peered":"2023-09-23T20:31:49.156992+0000","last_unstale":"2023-09-23T20:35:10.586719+0000","last_undegraded":"2023-09-23T20:35:10.586719+0000","last_fullsized":"2023-09-23T20:35:10.586719+0000","mapping_epoch":303,"log_start":"317'3976","ondisk_log_start":"317'3976","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":6304,"log_dups_size":0,"ondisk_log_size":6304,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:21:35.645838+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":106430464,"num_objects":1624,"num_object_clones":0,"num_object_copies":4872,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1624,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10280,"num_write_kb":380928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":602,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T20:35:02.241813+0000","last_change":"2023-09-23T20:31:45.785521+0000","last_active":"2023-09-23T20:35:02.241813+0000","last_peered":"2023-09-23T20:35:02.241813+0000","last_clean":"2023-09-23T20:35:02.241813+0000","last_became_active":"2023-09-23T20:27:55.213394+0000","last_became_peered":"2023-09-23T20:27:55.213394+0000","last_unstale":"2023-09-23T20:35:02.241813+0000","last_undegraded":"2023-09-23T20:35:02.241813+0000","last_fullsized":"2023-09-23T20:35:02.241813+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:45.785450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:45.785450+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:21:26.092578+0000","scrub_duration":0.001558306,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"86'106","reported_seq":1076,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T20:35:02.241976+0000","last_change":"2023-09-23T20:31:31.531882+0000","last_active":"2023-09-23T20:35:02.241976+0000","last_peered":"2023-09-23T20:35:02.241976+0000","last_clean":"2023-09-23T20:35:02.241976+0000","last_became_active":"2023-09-23T20:29:46.078501+0000","last_became_peered":"2023-09-23T20:29:46.078501+0000","last_unstale":"2023-09-23T20:35:02.241976+0000","last_undegraded":"2023-09-23T20:35:02.241976+0000","last_fullsized":"2023-09-23T20:35:02.241976+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":106,"log_dups_size":0,"ondisk_log_size":106,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:37.015974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00099302599999999993,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1105,"num_read_kb":1689,"num_write":100,"num_write_kb":404,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":55,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T20:35:02.229546+0000","last_change":"2023-09-23T20:32:33.230242+0000","last_active":"2023-09-23T20:35:02.229546+0000","last_peered":"2023-09-23T20:35:02.229546+0000","last_clean":"2023-09-23T20:35:02.229546+0000","last_became_active":"2023-09-23T20:32:33.229732+0000","last_became_peered":"2023-09-23T20:32:33.229732+0000","last_unstale":"2023-09-23T20:35:02.229546+0000","last_undegraded":"2023-09-23T20:35:02.229546+0000","last_fullsized":"2023-09-23T20:35:02.229546+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:27:55.538429+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":315,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-09-23T20:35:01.221692+0000","last_change":"2023-09-23T20:29:45.533406+0000","last_active":"2023-09-23T20:35:01.221692+0000","last_peered":"2023-09-23T20:35:01.221692+0000","last_clean":"2023-09-23T20:35:01.221692+0000","last_became_active":"2023-09-23T20:29:45.526001+0000","last_became_peered":"2023-09-23T20:29:45.526001+0000","last_unstale":"2023-09-23T20:35:01.221692+0000","last_undegraded":"2023-09-23T20:35:01.221692+0000","last_fullsized":"2023-09-23T20:35:01.221692+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:43.787251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.4","version":"358'10144","reported_seq":12777,"reported_epoch":358,"state":"active+recovering+undersized+remapped","last_fresh":"2023-09-23T20:35:10.523929+0000","last_change":"2023-09-23T20:35:00.204596+0000","last_active":"2023-09-23T20:35:10.523929+0000","last_peered":"2023-09-23T20:35:10.523929+0000","last_clean":"2023-09-23T20:32:32.192526+0000","last_became_active":"2023-09-23T20:35:00.124743+0000","last_became_peered":"2023-09-23T20:35:00.124743+0000","last_unstale":"2023-09-23T20:35:10.523929+0000","last_undegraded":"2023-09-23T20:35:10.523929+0000","last_fullsized":"2023-09-23T20:34:59.968003+0000","mapping_epoch":353,"log_start":"310'3300","ondisk_log_start":"310'3300","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":6844,"log_dups_size":0,"ondisk_log_size":6844,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:40:39.030291+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":103415808,"num_objects":1578,"num_object_clones":0,"num_object_copies":4734,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":817,"num_objects_unfound":0,"num_objects_dirty":1578,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10144,"num_write_kb":375104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1270,"num_bytes_recovered":83099648,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,1],"avail_no_missing":["0","1"],"object_location_counts":[{"shards":"0,1","objects":761},{"shards":"0,1,6","objects":817}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":830,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T20:35:02.260940+0000","last_change":"2023-09-23T20:35:02.260708+0000","last_active":"2023-09-23T20:35:02.260940+0000","last_peered":"2023-09-23T20:35:02.260940+0000","last_clean":"2023-09-23T20:35:02.260940+0000","last_became_active":"2023-09-23T20:35:02.260348+0000","last_became_peered":"2023-09-23T20:35:02.260348+0000","last_unstale":"2023-09-23T20:35:02.260940+0000","last_undegraded":"2023-09-23T20:35:02.260940+0000","last_fullsized":"2023-09-23T20:35:02.260940+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:43.390908+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:43.390908+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:09:14.144070+0000","scrub_duration":0.0032721,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"86'130","reported_seq":670,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-09-23T20:35:01.216449+0000","last_change":"2023-09-23T20:25:39.637458+0000","last_active":"2023-09-23T20:35:01.216449+0000","last_peered":"2023-09-23T20:35:01.216449+0000","last_clean":"2023-09-23T20:35:01.216449+0000","last_became_active":"2023-09-23T20:25:39.636773+0000","last_became_peered":"2023-09-23T20:25:39.636773+0000","last_unstale":"2023-09-23T20:35:01.216449+0000","last_undegraded":"2023-09-23T20:35:01.216449+0000","last_fullsized":"2023-09-23T20:35:01.216449+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:00:32.810564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011300049999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":512,"num_read_kb":1820,"num_write":133,"num_write_kb":1347,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":305,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-09-23T20:35:01.223277+0000","last_change":"2023-09-23T20:29:46.073394+0000","last_active":"2023-09-23T20:35:01.223277+0000","last_peered":"2023-09-23T20:35:01.223277+0000","last_clean":"2023-09-23T20:35:01.223277+0000","last_became_active":"2023-09-23T20:29:46.072067+0000","last_became_peered":"2023-09-23T20:29:46.072067+0000","last_unstale":"2023-09-23T20:35:01.223277+0000","last_undegraded":"2023-09-23T20:35:01.223277+0000","last_fullsized":"2023-09-23T20:35:01.223277+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:01:56.944614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":297,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-09-23T20:35:01.221888+0000","last_change":"2023-09-23T20:29:45.533185+0000","last_active":"2023-09-23T20:35:01.221888+0000","last_peered":"2023-09-23T20:35:01.221888+0000","last_clean":"2023-09-23T20:35:01.221888+0000","last_became_active":"2023-09-23T20:29:45.526765+0000","last_became_peered":"2023-09-23T20:29:45.526765+0000","last_unstale":"2023-09-23T20:35:01.221888+0000","last_undegraded":"2023-09-23T20:35:01.221888+0000","last_fullsized":"2023-09-23T20:35:01.221888+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:18:33.063215+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"358'10252","reported_seq":10317,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-09-23T20:35:10.471927+0000","last_change":"2023-09-23T20:31:49.158194+0000","last_active":"2023-09-23T20:35:10.471927+0000","last_peered":"2023-09-23T20:35:10.471927+0000","last_clean":"2023-09-23T20:35:10.471927+0000","last_became_active":"2023-09-23T20:31:49.157875+0000","last_became_peered":"2023-09-23T20:31:49.157875+0000","last_unstale":"2023-09-23T20:35:10.471927+0000","last_undegraded":"2023-09-23T20:35:10.471927+0000","last_fullsized":"2023-09-23T20:35:10.471927+0000","mapping_epoch":303,"log_start":"310'3400","ondisk_log_start":"310'3400","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":6852,"log_dups_size":0,"ondisk_log_size":6852,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:04:42.411181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":105906176,"num_objects":1616,"num_object_clones":0,"num_object_copies":4848,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1616,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10252,"num_write_kb":379776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"86'194","reported_seq":1006,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T20:35:02.242710+0000","last_change":"2023-09-23T20:25:40.618665+0000","last_active":"2023-09-23T20:35:02.242710+0000","last_peered":"2023-09-23T20:35:02.242710+0000","last_clean":"2023-09-23T20:35:02.242710+0000","last_became_active":"2023-09-23T20:25:40.618087+0000","last_became_peered":"2023-09-23T20:25:40.618087+0000","last_unstale":"2023-09-23T20:35:02.242710+0000","last_undegraded":"2023-09-23T20:35:02.242710+0000","last_fullsized":"2023-09-23T20:35:02.242710+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":194,"log_dups_size":0,"ondisk_log_size":194,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:07:16.114968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00037995999999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1025,"num_read_kb":1201,"num_write":173,"num_write_kb":1276,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":791,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T20:35:02.223070+0000","last_change":"2023-09-23T20:31:45.343835+0000","last_active":"2023-09-23T20:35:02.223070+0000","last_peered":"2023-09-23T20:35:02.223070+0000","last_clean":"2023-09-23T20:35:02.223070+0000","last_became_active":"2023-09-23T20:29:07.042407+0000","last_became_peered":"2023-09-23T20:29:07.042407+0000","last_unstale":"2023-09-23T20:35:02.223070+0000","last_undegraded":"2023-09-23T20:35:02.223070+0000","last_fullsized":"2023-09-23T20:35:02.223070+0000","mapping_epoch":253,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":254,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:45.343742+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:45.343742+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:33:12.745366+0000","scrub_duration":0.001475802,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":157,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T20:35:02.242001+0000","last_change":"2023-09-23T20:32:50.534315+0000","last_active":"2023-09-23T20:35:02.242001+0000","last_peered":"2023-09-23T20:35:02.242001+0000","last_clean":"2023-09-23T20:35:02.242001+0000","last_became_active":"2023-09-23T20:32:50.533896+0000","last_became_peered":"2023-09-23T20:32:50.533896+0000","last_unstale":"2023-09-23T20:35:02.242001+0000","last_undegraded":"2023-09-23T20:35:02.242001+0000","last_fullsized":"2023-09-23T20:35:02.242001+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:06:53.464923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":137,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T20:35:02.230693+0000","last_change":"2023-09-23T20:32:33.225012+0000","last_active":"2023-09-23T20:35:02.230693+0000","last_peered":"2023-09-23T20:35:02.230693+0000","last_clean":"2023-09-23T20:35:02.230693+0000","last_became_active":"2023-09-23T20:32:33.224699+0000","last_became_peered":"2023-09-23T20:32:33.224699+0000","last_unstale":"2023-09-23T20:35:02.230693+0000","last_undegraded":"2023-09-23T20:35:02.230693+0000","last_fullsized":"2023-09-23T20:35:02.230693+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:10:06.267878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"358'10329","reported_seq":10392,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-09-23T20:35:10.490312+0000","last_change":"2023-09-23T20:31:49.154418+0000","last_active":"2023-09-23T20:35:10.490312+0000","last_peered":"2023-09-23T20:35:10.490312+0000","last_clean":"2023-09-23T20:35:10.490312+0000","last_became_active":"2023-09-23T20:31:49.154182+0000","last_became_peered":"2023-09-23T20:31:49.154182+0000","last_unstale":"2023-09-23T20:35:10.490312+0000","last_undegraded":"2023-09-23T20:35:10.490312+0000","last_fullsized":"2023-09-23T20:35:10.490312+0000","mapping_epoch":303,"log_start":"317'3998","ondisk_log_start":"317'3998","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":6331,"log_dups_size":0,"ondisk_log_size":6331,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:44:10.087486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":104660992,"num_objects":1597,"num_object_clones":0,"num_object_copies":4791,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1597,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10329,"num_write_kb":381632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":650,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T20:35:02.238632+0000","last_change":"2023-09-23T20:35:02.238632+0000","last_active":"2023-09-23T20:35:02.238632+0000","last_peered":"2023-09-23T20:35:02.238632+0000","last_clean":"2023-09-23T20:35:02.238632+0000","last_became_active":"2023-09-23T20:35:02.237812+0000","last_became_peered":"2023-09-23T20:35:02.237812+0000","last_unstale":"2023-09-23T20:35:02.238632+0000","last_undegraded":"2023-09-23T20:35:02.238632+0000","last_fullsized":"2023-09-23T20:35:02.238632+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:43.582146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:43.582146+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:25:04.582696+0000","scrub_duration":0.0016219730000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"86'80","reported_seq":621,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T20:35:02.222989+0000","last_change":"2023-09-23T20:29:45.536796+0000","last_active":"2023-09-23T20:35:02.222989+0000","last_peered":"2023-09-23T20:35:02.222989+0000","last_clean":"2023-09-23T20:35:02.222989+0000","last_became_active":"2023-09-23T20:29:45.536439+0000","last_became_peered":"2023-09-23T20:29:45.536439+0000","last_unstale":"2023-09-23T20:35:02.222989+0000","last_undegraded":"2023-09-23T20:35:02.222989+0000","last_fullsized":"2023-09-23T20:35:02.222989+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":80,"log_dups_size":0,"ondisk_log_size":80,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:30:57.793132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035115,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":395,"num_read_kb":1370,"num_write":66,"num_write_kb":1061,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":360,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T20:35:02.250103+0000","last_change":"2023-09-23T20:34:59.984286+0000","last_active":"2023-09-23T20:35:02.250103+0000","last_peered":"2023-09-23T20:35:02.250103+0000","last_clean":"2023-09-23T20:35:02.250103+0000","last_became_active":"2023-09-23T20:34:59.983950+0000","last_became_peered":"2023-09-23T20:34:59.983950+0000","last_unstale":"2023-09-23T20:35:02.250103+0000","last_undegraded":"2023-09-23T20:35:02.250103+0000","last_fullsized":"2023-09-23T20:35:02.250103+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:31:53.557035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":354,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T20:35:02.254761+0000","last_change":"2023-09-23T20:29:45.690815+0000","last_active":"2023-09-23T20:35:02.254761+0000","last_peered":"2023-09-23T20:35:02.254761+0000","last_clean":"2023-09-23T20:35:02.254761+0000","last_became_active":"2023-09-23T20:29:45.690478+0000","last_became_peered":"2023-09-23T20:29:45.690478+0000","last_unstale":"2023-09-23T20:35:02.254761+0000","last_undegraded":"2023-09-23T20:35:02.254761+0000","last_fullsized":"2023-09-23T20:35:02.254761+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:38.288193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.0","version":"358'10299","reported_seq":10361,"reported_epoch":358,"state":"active+clean","last_fresh":"2023-09-23T20:35:10.366924+0000","last_change":"2023-09-23T20:31:49.157205+0000","last_active":"2023-09-23T20:35:10.366924+0000","last_peered":"2023-09-23T20:35:10.366924+0000","last_clean":"2023-09-23T20:35:10.366924+0000","last_became_active":"2023-09-23T20:31:49.156922+0000","last_became_peered":"2023-09-23T20:31:49.156922+0000","last_unstale":"2023-09-23T20:35:10.366924+0000","last_undegraded":"2023-09-23T20:35:10.366924+0000","last_fullsized":"2023-09-23T20:35:10.366924+0000","mapping_epoch":303,"log_start":"310'2400","ondisk_log_start":"310'2400","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":7899,"log_dups_size":0,"ondisk_log_size":7899,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:51:36.120059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":104792064,"num_objects":1599,"num_object_clones":0,"num_object_copies":4797,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1599,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10299,"num_write_kb":380736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":600,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T20:35:02.230705+0000","last_change":"2023-09-23T20:35:00.151823+0000","last_active":"2023-09-23T20:35:02.230705+0000","last_peered":"2023-09-23T20:35:02.230705+0000","last_clean":"2023-09-23T20:35:02.230705+0000","last_became_active":"2023-09-23T20:33:38.991030+0000","last_became_peered":"2023-09-23T20:33:38.991030+0000","last_unstale":"2023-09-23T20:35:02.230705+0000","last_undegraded":"2023-09-23T20:35:02.230705+0000","last_fullsized":"2023-09-23T20:35:02.230705+0000","mapping_epoch":253,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:35:00.151695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:35:00.151695+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:09:55.183927+0000","scrub_duration":0.0016030090000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"86'102","reported_seq":952,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T20:35:02.221812+0000","last_change":"2023-09-23T20:32:36.314672+0000","last_active":"2023-09-23T20:35:02.221812+0000","last_peered":"2023-09-23T20:35:02.221812+0000","last_clean":"2023-09-23T20:35:02.221812+0000","last_became_active":"2023-09-23T20:32:36.314231+0000","last_became_peered":"2023-09-23T20:32:36.314231+0000","last_unstale":"2023-09-23T20:35:02.221812+0000","last_undegraded":"2023-09-23T20:35:02.221812+0000","last_fullsized":"2023-09-23T20:35:02.221812+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":102,"log_dups_size":0,"ondisk_log_size":102,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:11:40.514709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030656500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":963,"num_read_kb":1234,"num_write":60,"num_write_kb":225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":281,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-09-23T20:35:01.221865+0000","last_change":"2023-09-23T20:29:45.538283+0000","last_active":"2023-09-23T20:35:01.221865+0000","last_peered":"2023-09-23T20:35:01.221865+0000","last_clean":"2023-09-23T20:35:01.221865+0000","last_became_active":"2023-09-23T20:29:45.537922+0000","last_became_peered":"2023-09-23T20:29:45.537922+0000","last_unstale":"2023-09-23T20:35:01.221865+0000","last_undegraded":"2023-09-23T20:35:01.221865+0000","last_fullsized":"2023-09-23T20:35:01.221865+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:30:52.397836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":292,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T20:35:02.249383+0000","last_change":"2023-09-23T20:20:14.888874+0000","last_active":"2023-09-23T20:35:02.249383+0000","last_peered":"2023-09-23T20:35:02.249383+0000","last_clean":"2023-09-23T20:35:02.249383+0000","last_became_active":"2023-09-23T20:20:14.887724+0000","last_became_peered":"2023-09-23T20:20:14.887724+0000","last_unstale":"2023-09-23T20:35:02.249383+0000","last_undegraded":"2023-09-23T20:35:02.249383+0000","last_fullsized":"2023-09-23T20:35:02.249383+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:09:24.510326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"357'9822","reported_seq":16992,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T20:35:07.665455+0000","last_change":"2023-09-23T20:34:50.587971+0000","last_active":"2023-09-23T20:35:07.665455+0000","last_peered":"2023-09-23T20:35:07.665455+0000","last_clean":"2023-09-23T20:35:07.665455+0000","last_became_active":"2023-09-23T20:34:50.555328+0000","last_became_peered":"2023-09-23T20:34:50.555328+0000","last_unstale":"2023-09-23T20:35:07.665455+0000","last_undegraded":"2023-09-23T20:35:07.665455+0000","last_fullsized":"2023-09-23T20:35:07.665455+0000","mapping_epoch":349,"log_start":"310'2200","ondisk_log_start":"310'2200","created":303,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":7622,"log_dups_size":0,"ondisk_log_size":7622,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:20:26.250987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109576192,"num_objects":1672,"num_object_clones":0,"num_object_copies":5016,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1672,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9822,"num_write_kb":367808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3542,"num_bytes_recovered":170328064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":566,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T20:35:02.229910+0000","last_change":"2023-09-23T20:34:55.534723+0000","last_active":"2023-09-23T20:35:02.229910+0000","last_peered":"2023-09-23T20:35:02.229910+0000","last_clean":"2023-09-23T20:35:02.229910+0000","last_became_active":"2023-09-23T20:33:38.985300+0000","last_became_peered":"2023-09-23T20:33:38.985300+0000","last_unstale":"2023-09-23T20:35:02.229910+0000","last_undegraded":"2023-09-23T20:35:02.229910+0000","last_fullsized":"2023-09-23T20:35:02.229910+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:44.509012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:44.509012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0032186319999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":95,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T20:35:02.224498+0000","last_change":"2023-09-23T20:32:50.549695+0000","last_active":"2023-09-23T20:35:02.224498+0000","last_peered":"2023-09-23T20:35:02.224498+0000","last_clean":"2023-09-23T20:35:02.224498+0000","last_became_active":"2023-09-23T20:32:50.549431+0000","last_became_peered":"2023-09-23T20:32:50.549431+0000","last_unstale":"2023-09-23T20:35:02.224498+0000","last_undegraded":"2023-09-23T20:35:02.224498+0000","last_fullsized":"2023-09-23T20:35:02.224498+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:45:53.666569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"86'128","reported_seq":747,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-09-23T20:35:01.221762+0000","last_change":"2023-09-23T20:32:36.310656+0000","last_active":"2023-09-23T20:35:01.221762+0000","last_peered":"2023-09-23T20:35:01.221762+0000","last_clean":"2023-09-23T20:35:01.221762+0000","last_became_active":"2023-09-23T20:32:36.310155+0000","last_became_peered":"2023-09-23T20:32:36.310155+0000","last_unstale":"2023-09-23T20:35:01.221762+0000","last_undegraded":"2023-09-23T20:35:01.221762+0000","last_fullsized":"2023-09-23T20:35:01.221762+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:58:14.217466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000293202,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":579,"num_read_kb":1473,"num_write":96,"num_write_kb":1062,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":128,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T20:35:02.224349+0000","last_change":"2023-09-23T20:32:50.539051+0000","last_active":"2023-09-23T20:35:02.224349+0000","last_peered":"2023-09-23T20:35:02.224349+0000","last_clean":"2023-09-23T20:35:02.224349+0000","last_became_active":"2023-09-23T20:32:50.538755+0000","last_became_peered":"2023-09-23T20:32:50.538755+0000","last_unstale":"2023-09-23T20:35:02.224349+0000","last_undegraded":"2023-09-23T20:35:02.224349+0000","last_fullsized":"2023-09-23T20:35:02.224349+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:16.656613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.2","version":"357'10032","reported_seq":10103,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T20:35:09.124757+0000","last_change":"2023-09-23T20:32:50.557165+0000","last_active":"2023-09-23T20:35:09.124757+0000","last_peered":"2023-09-23T20:35:09.124757+0000","last_clean":"2023-09-23T20:35:09.124757+0000","last_became_active":"2023-09-23T20:32:50.553409+0000","last_became_peered":"2023-09-23T20:32:50.553409+0000","last_unstale":"2023-09-23T20:35:09.124757+0000","last_undegraded":"2023-09-23T20:35:09.124757+0000","last_fullsized":"2023-09-23T20:35:09.124757+0000","mapping_epoch":319,"log_start":"310'3501","ondisk_log_start":"310'3501","created":303,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":6531,"log_dups_size":0,"ondisk_log_size":6531,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:55:13.578195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":108789760,"num_objects":1660,"num_object_clones":0,"num_object_copies":4980,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1660,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10032,"num_write_kb":374144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"90'161","reported_seq":507,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T20:35:02.242352+0000","last_change":"2023-09-23T20:34:54.504652+0000","last_active":"2023-09-23T20:35:02.242352+0000","last_peered":"2023-09-23T20:35:02.242352+0000","last_clean":"2023-09-23T20:35:02.242352+0000","last_became_active":"2023-09-23T20:32:50.533470+0000","last_became_peered":"2023-09-23T20:32:50.533470+0000","last_unstale":"2023-09-23T20:35:02.242352+0000","last_undegraded":"2023-09-23T20:35:02.242352+0000","last_fullsized":"2023-09-23T20:35:02.242352+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:34:54.504554+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:34:54.504554+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:17:03.108888+0000","scrub_duration":0.013031021,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.7","version":"86'117","reported_seq":627,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T20:35:02.231036+0000","last_change":"2023-09-23T20:31:32.814392+0000","last_active":"2023-09-23T20:35:02.231036+0000","last_peered":"2023-09-23T20:35:02.231036+0000","last_clean":"2023-09-23T20:35:02.231036+0000","last_became_active":"2023-09-23T20:31:32.813857+0000","last_became_peered":"2023-09-23T20:31:32.813857+0000","last_unstale":"2023-09-23T20:35:02.231036+0000","last_undegraded":"2023-09-23T20:35:02.231036+0000","last_fullsized":"2023-09-23T20:35:02.231036+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:56:57.863005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046891400000000002,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":893,"num_read_kb":922,"num_write":82,"num_write_kb":4193,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":354,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T20:35:02.254871+0000","last_change":"2023-09-23T20:32:33.277827+0000","last_active":"2023-09-23T20:35:02.254871+0000","last_peered":"2023-09-23T20:35:02.254871+0000","last_clean":"2023-09-23T20:35:02.254871+0000","last_became_active":"2023-09-23T20:32:33.236087+0000","last_became_peered":"2023-09-23T20:32:33.236087+0000","last_unstale":"2023-09-23T20:35:02.254871+0000","last_undegraded":"2023-09-23T20:35:02.254871+0000","last_fullsized":"2023-09-23T20:35:02.254871+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:20:58.176854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":55,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T20:35:02.229268+0000","last_change":"2023-09-23T20:32:33.238290+0000","last_active":"2023-09-23T20:35:02.229268+0000","last_peered":"2023-09-23T20:35:02.229268+0000","last_clean":"2023-09-23T20:35:02.229268+0000","last_became_active":"2023-09-23T20:32:33.237984+0000","last_became_peered":"2023-09-23T20:32:33.237984+0000","last_unstale":"2023-09-23T20:35:02.229268+0000","last_undegraded":"2023-09-23T20:35:02.229268+0000","last_fullsized":"2023-09-23T20:35:02.229268+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:49:02.089307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"90'161","reported_seq":752,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T20:35:02.222459+0000","last_change":"2023-09-23T20:34:32.517512+0000","last_active":"2023-09-23T20:35:02.222459+0000","last_peered":"2023-09-23T20:35:02.222459+0000","last_clean":"2023-09-23T20:35:02.222459+0000","last_became_active":"2023-09-23T20:33:38.989881+0000","last_became_peered":"2023-09-23T20:33:38.989881+0000","last_unstale":"2023-09-23T20:35:02.222459+0000","last_undegraded":"2023-09-23T20:35:02.222459+0000","last_fullsized":"2023-09-23T20:35:02.222459+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:28:05.872074+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:28:05.872074+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0029215980000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":322,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T20:35:02.249286+0000","last_change":"2023-09-23T20:32:33.237234+0000","last_active":"2023-09-23T20:35:02.249286+0000","last_peered":"2023-09-23T20:35:02.249286+0000","last_clean":"2023-09-23T20:35:02.249286+0000","last_became_active":"2023-09-23T20:32:33.236881+0000","last_became_peered":"2023-09-23T20:32:33.236881+0000","last_unstale":"2023-09-23T20:35:02.249286+0000","last_undegraded":"2023-09-23T20:35:02.249286+0000","last_fullsized":"2023-09-23T20:35:02.249286+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:56:17.384831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":209,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-09-23T20:35:01.219489+0000","last_change":"2023-09-23T20:29:45.538314+0000","last_active":"2023-09-23T20:35:01.219489+0000","last_peered":"2023-09-23T20:35:01.219489+0000","last_clean":"2023-09-23T20:35:01.219489+0000","last_became_active":"2023-09-23T20:29:45.536847+0000","last_became_peered":"2023-09-23T20:29:45.536847+0000","last_unstale":"2023-09-23T20:35:01.219489+0000","last_undegraded":"2023-09-23T20:35:01.219489+0000","last_fullsized":"2023-09-23T20:35:01.219489+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:51.948721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"90'161","reported_seq":448,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T20:35:02.234431+0000","last_change":"2023-09-23T20:35:02.234117+0000","last_active":"2023-09-23T20:35:02.234431+0000","last_peered":"2023-09-23T20:35:02.234431+0000","last_clean":"2023-09-23T20:35:02.234431+0000","last_became_active":"2023-09-23T20:35:02.233642+0000","last_became_peered":"2023-09-23T20:35:02.233642+0000","last_unstale":"2023-09-23T20:35:02.234431+0000","last_undegraded":"2023-09-23T20:35:02.234431+0000","last_fullsized":"2023-09-23T20:35:02.234431+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:44.386120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:44.386120+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":40253,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":367,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T20:35:02.242189+0000","last_change":"2023-09-23T20:32:33.235987+0000","last_active":"2023-09-23T20:35:02.242189+0000","last_peered":"2023-09-23T20:35:02.242189+0000","last_clean":"2023-09-23T20:35:02.242189+0000","last_became_active":"2023-09-23T20:32:33.226544+0000","last_became_peered":"2023-09-23T20:32:33.226544+0000","last_unstale":"2023-09-23T20:35:02.242189+0000","last_undegraded":"2023-09-23T20:35:02.242189+0000","last_fullsized":"2023-09-23T20:35:02.242189+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:53:22.794856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":14,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T20:35:02.229279+0000","last_change":"2023-09-23T20:34:59.987575+0000","last_active":"2023-09-23T20:35:02.229279+0000","last_peered":"2023-09-23T20:35:02.229279+0000","last_clean":"2023-09-23T20:35:02.229279+0000","last_became_active":"2023-09-23T20:34:59.987229+0000","last_became_peered":"2023-09-23T20:34:59.987229+0000","last_unstale":"2023-09-23T20:35:02.229279+0000","last_undegraded":"2023-09-23T20:35:02.229279+0000","last_fullsized":"2023-09-23T20:35:02.229279+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:34:01.820159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":375,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T20:35:02.242005+0000","last_change":"2023-09-23T20:32:33.268260+0000","last_active":"2023-09-23T20:35:02.242005+0000","last_peered":"2023-09-23T20:35:02.242005+0000","last_clean":"2023-09-23T20:35:02.242005+0000","last_became_active":"2023-09-23T20:32:33.233695+0000","last_became_peered":"2023-09-23T20:32:33.233695+0000","last_unstale":"2023-09-23T20:35:02.242005+0000","last_undegraded":"2023-09-23T20:35:02.242005+0000","last_fullsized":"2023-09-23T20:35:02.242005+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:19:29.299623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":14,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T20:35:02.229041+0000","last_change":"2023-09-23T20:34:59.987416+0000","last_active":"2023-09-23T20:35:02.229041+0000","last_peered":"2023-09-23T20:35:02.229041+0000","last_clean":"2023-09-23T20:35:02.229041+0000","last_became_active":"2023-09-23T20:34:59.987117+0000","last_became_peered":"2023-09-23T20:34:59.987117+0000","last_unstale":"2023-09-23T20:35:02.229041+0000","last_undegraded":"2023-09-23T20:35:02.229041+0000","last_fullsized":"2023-09-23T20:35:02.229041+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:36:16.318623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"90'161","reported_seq":527,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T20:35:02.249447+0000","last_change":"2023-09-23T20:33:38.990525+0000","last_active":"2023-09-23T20:35:02.249447+0000","last_peered":"2023-09-23T20:35:02.249447+0000","last_clean":"2023-09-23T20:35:02.249447+0000","last_became_active":"2023-09-23T20:33:38.990173+0000","last_became_peered":"2023-09-23T20:33:38.990173+0000","last_unstale":"2023-09-23T20:35:02.249447+0000","last_undegraded":"2023-09-23T20:35:02.249447+0000","last_fullsized":"2023-09-23T20:35:02.249447+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:41.701171+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:41.701171+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0019955509999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":54614,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":173,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T20:35:02.230535+0000","last_change":"2023-09-23T20:32:50.537447+0000","last_active":"2023-09-23T20:35:02.230535+0000","last_peered":"2023-09-23T20:35:02.230535+0000","last_clean":"2023-09-23T20:35:02.230535+0000","last_became_active":"2023-09-23T20:32:50.537136+0000","last_became_peered":"2023-09-23T20:32:50.537136+0000","last_unstale":"2023-09-23T20:35:02.230535+0000","last_undegraded":"2023-09-23T20:35:02.230535+0000","last_fullsized":"2023-09-23T20:35:02.230535+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:46:51.599492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":71,"reported_epoch":356,"state":"active+clean","last_fresh":"2023-09-23T20:35:01.216598+0000","last_change":"2023-09-23T20:32:50.537450+0000","last_active":"2023-09-23T20:35:01.216598+0000","last_peered":"2023-09-23T20:35:01.216598+0000","last_clean":"2023-09-23T20:35:01.216598+0000","last_became_active":"2023-09-23T20:32:50.537128+0000","last_became_peered":"2023-09-23T20:32:50.537128+0000","last_unstale":"2023-09-23T20:35:01.216598+0000","last_undegraded":"2023-09-23T20:35:01.216598+0000","last_fullsized":"2023-09-23T20:35:01.216598+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:37:15.889029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1d","version":"90'161","reported_seq":639,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T20:35:02.221534+0000","last_change":"2023-09-23T20:34:30.499622+0000","last_active":"2023-09-23T20:35:02.221534+0000","last_peered":"2023-09-23T20:35:02.221534+0000","last_clean":"2023-09-23T20:35:02.221534+0000","last_became_active":"2023-09-23T20:33:38.972109+0000","last_became_peered":"2023-09-23T20:33:38.972109+0000","last_unstale":"2023-09-23T20:35:02.221534+0000","last_undegraded":"2023-09-23T20:35:02.221534+0000","last_fullsized":"2023-09-23T20:35:02.221534+0000","mapping_epoch":332,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:46.388545+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:46.388545+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.00168356,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":340,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T20:35:02.255473+0000","last_change":"2023-09-23T20:25:39.668882+0000","last_active":"2023-09-23T20:35:02.255473+0000","last_peered":"2023-09-23T20:35:02.255473+0000","last_clean":"2023-09-23T20:35:02.255473+0000","last_became_active":"2023-09-23T20:25:39.637128+0000","last_became_peered":"2023-09-23T20:25:39.637128+0000","last_unstale":"2023-09-23T20:35:02.255473+0000","last_undegraded":"2023-09-23T20:35:02.255473+0000","last_fullsized":"2023-09-23T20:35:02.255473+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:56.406298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":55,"reported_epoch":357,"state":"active+clean","last_fresh":"2023-09-23T20:35:02.228826+0000","last_change":"2023-09-23T20:32:33.232281+0000","last_active":"2023-09-23T20:35:02.228826+0000","last_peered":"2023-09-23T20:35:02.228826+0000","last_clean":"2023-09-23T20:35:02.228826+0000","last_became_active":"2023-09-23T20:32:33.232003+0000","last_became_peered":"2023-09-23T20:32:33.232003+0000","last_unstale":"2023-09-23T20:35:02.228826+0000","last_undegraded":"2023-09-23T20:35:02.228826+0000","last_fullsized":"2023-09-23T20:35:02.228826+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:58:27.314355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":1729298456,"num_objects":26388,"num_object_clones":0,"num_object_copies":79164,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4095,"num_objects_unfound":0,"num_objects_dirty":26388,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":162712,"num_write_kb":6051137,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10080,"num_bytes_recovered":598671360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":5646200832,"data_stored":5646188616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":110904,"ondisk_log_size":110904,"up":48,"acting":47,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":49,"num_bytes_recovered":6522,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26691,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":398750,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":58,"num_bytes_recovered":4194342,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6082,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201568,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":919896,"kb_used_data":626640,"kb_used_omap":0,"kb_used_meta":293248,"kb_avail":92825256,"statfs":{"total":95995035648,"available":95053062144,"internally_reserved":0,"allocated":641679360,"data_stored":640545416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":300285952},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:34:43 2023","interfaces":[{"interface":"back","average":{"1min":2.323,"5min":2.0470000000000002,"15min":1.617},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.20799999999999999},"max":{"1min":6.1959999999999997,"5min":14.66,"15min":48.798999999999999},"last":1.986},{"interface":"front","average":{"1min":2.871,"5min":2.0950000000000002,"15min":1.702},"min":{"1min":0.25,"5min":0.25,"15min":0.24399999999999999},"max":{"1min":14.605,"5min":14.605,"15min":48.247},"last":1.772}]},{"osd":1,"last update":"Sat Sep 23 20:34:54 2023","interfaces":[{"interface":"back","average":{"1min":2.3319999999999999,"5min":1.984,"15min":1.8360000000000001},"min":{"1min":0.29999999999999999,"5min":0.254,"15min":0.254},"max":{"1min":5.5030000000000001,"5min":14.026,"15min":14.026},"last":0.35799999999999998},{"interface":"front","average":{"1min":2.5139999999999998,"5min":1.8340000000000001,"15min":1.9319999999999999},"min":{"1min":0.32000000000000001,"5min":0.248,"15min":0.248},"max":{"1min":6.7220000000000004,"5min":6.7220000000000004,"15min":6.7220000000000004},"last":1.226}]},{"osd":2,"last update":"Sat Sep 23 20:34:43 2023","interfaces":[{"interface":"back","average":{"1min":2.6739999999999999,"5min":2.4159999999999999,"15min":1.6799999999999999},"min":{"1min":0.34100000000000003,"5min":0.28599999999999998,"15min":0.27600000000000002},"max":{"1min":7.9029999999999996,"5min":13.173999999999999,"15min":13.173999999999999},"last":4.3840000000000003},{"interface":"front","average":{"1min":2.6280000000000001,"5min":2.7320000000000002,"15min":1.9039999999999999},"min":{"1min":0.36899999999999999,"5min":0.27400000000000002,"15min":0.26600000000000001},"max":{"1min":9.7230000000000008,"5min":23.494,"15min":23.494},"last":2.125}]},{"osd":3,"last update":"Sat Sep 23 20:34:07 2023","interfaces":[{"interface":"back","average":{"1min":2.4380000000000002,"5min":2.3479999999999999,"15min":1.7070000000000001},"min":{"1min":0.35199999999999998,"5min":0.29199999999999998,"15min":0.26200000000000001},"max":{"1min":7.202,"5min":17.155999999999999,"15min":17.155999999999999},"last":1.605},{"interface":"front","average":{"1min":2.7269999999999999,"5min":2.2010000000000001,"15min":1.6419999999999999},"min":{"1min":0.45300000000000001,"5min":0.33100000000000002,"15min":0.27600000000000002},"max":{"1min":6.6589999999999998,"5min":16.960000000000001,"15min":16.960000000000001},"last":1.6180000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:34:07 2023","interfaces":[{"interface":"back","average":{"1min":0.67100000000000004,"5min":1.2849999999999999,"15min":1.05},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":2.2370000000000001,"5min":15.177,"15min":15.177},"last":0.32800000000000001},{"interface":"front","average":{"1min":0.82999999999999996,"5min":1.048,"15min":1.1759999999999999},"min":{"1min":0.33000000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":2.9790000000000001,"5min":9.3740000000000006,"15min":11.122},"last":0.47399999999999998}]},{"osd":5,"last update":"Sat Sep 23 20:34:07 2023","interfaces":[{"interface":"back","average":{"1min":0.78400000000000003,"5min":0.91100000000000003,"15min":1.1319999999999999},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":2.7730000000000001,"5min":6.8300000000000001,"15min":13.241},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.47699999999999998,"5min":1.0660000000000001,"15min":1.1950000000000001},"min":{"1min":0.34399999999999997,"5min":0.32200000000000001,"15min":0.315},"max":{"1min":0.747,"5min":18.831,"15min":18.831},"last":0.38}]},{"osd":6,"last update":"Sat Sep 23 20:34:07 2023","interfaces":[{"interface":"back","average":{"1min":0.59799999999999998,"5min":1.1180000000000001,"15min":1.0980000000000001},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":2.177,"5min":12.298999999999999,"15min":13.185},"last":0.40400000000000003},{"interface":"front","average":{"1min":0.97299999999999998,"5min":1.022,"15min":1.0920000000000001},"min":{"1min":0.36899999999999999,"5min":0.318,"15min":0.29899999999999999},"max":{"1min":3.0190000000000001,"5min":8.0069999999999997,"15min":9.9120000000000008},"last":0.42199999999999999}]}]},{"osd":6,"up_from":47,"seq":201863463204,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1109000,"kb_used_data":837824,"kb_used_omap":0,"kb_used_meta":271168,"kb_avail":92636152,"statfs":{"total":95995035648,"available":94859419648,"internally_reserved":0,"allocated":857931776,"data_stored":856805858,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":277676032},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:34:41 2023","interfaces":[{"interface":"back","average":{"1min":3.9790000000000001,"5min":2.5270000000000001,"15min":1.589},"min":{"1min":0.24299999999999999,"5min":0.22800000000000001,"15min":0.221},"max":{"1min":34.143999999999998,"5min":53.701999999999998,"15min":53.701999999999998},"last":2.9820000000000002},{"interface":"front","average":{"1min":4.202,"5min":3.004,"15min":1.9390000000000001},"min":{"1min":0.253,"5min":0.253,"15min":0.25},"max":{"1min":34.197000000000003,"5min":53.875999999999998,"15min":53.875999999999998},"last":0.40200000000000002}]},{"osd":1,"last update":"Sat Sep 23 20:34:57 2023","interfaces":[{"interface":"back","average":{"1min":3.4510000000000001,"5min":2.2719999999999998,"15min":2.5910000000000002},"min":{"1min":0.25600000000000001,"5min":0.249,"15min":0.249},"max":{"1min":34.173999999999999,"5min":34.173999999999999,"15min":34.173999999999999},"last":2.8660000000000001},{"interface":"front","average":{"1min":4.29,"5min":2.569,"15min":3.181},"min":{"1min":0.32900000000000001,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":34.490000000000002,"5min":34.490000000000002,"15min":34.490000000000002},"last":3.1659999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:34:41 2023","interfaces":[{"interface":"back","average":{"1min":3.879,"5min":2.5379999999999998,"15min":1.8080000000000001},"min":{"1min":0.29599999999999999,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":34.317999999999998,"5min":34.317999999999998,"15min":34.317999999999998},"last":1.0900000000000001},{"interface":"front","average":{"1min":4.1390000000000002,"5min":2.5390000000000001,"15min":1.7350000000000001},"min":{"1min":0.35399999999999998,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":34.292999999999999,"5min":34.292999999999999,"15min":34.292999999999999},"last":0.45500000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:34:23 2023","interfaces":[{"interface":"back","average":{"1min":2.242,"5min":2.0230000000000001,"15min":1.6120000000000001},"min":{"1min":0.31900000000000001,"5min":0.30499999999999999,"15min":0.29599999999999999},"max":{"1min":6.2089999999999996,"5min":7.5620000000000003,"15min":9.4239999999999995},"last":3.6000000000000001},{"interface":"front","average":{"1min":2.6960000000000002,"5min":2.2589999999999999,"15min":1.708},"min":{"1min":0.443,"5min":0.317,"15min":0.317},"max":{"1min":12.875999999999999,"5min":12.875999999999999,"15min":12.875999999999999},"last":2.5259999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:35:02 2023","interfaces":[{"interface":"back","average":{"1min":0.89200000000000002,"5min":0.58599999999999997,"15min":0.89500000000000002},"min":{"1min":0.27400000000000002,"5min":0.251,"15min":0.251},"max":{"1min":4.9279999999999999,"5min":4.9279999999999999,"15min":10.173},"last":0.34100000000000003},{"interface":"front","average":{"1min":0.77200000000000002,"5min":0.84699999999999998,"15min":1.0620000000000001},"min":{"1min":0.29099999999999998,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":3.1139999999999999,"5min":7.7629999999999999,"15min":8.3249999999999993},"last":0.74399999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:35:02 2023","interfaces":[{"interface":"back","average":{"1min":0.70599999999999996,"5min":0.66400000000000003,"15min":0.95199999999999996},"min":{"1min":0.31,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":4.8259999999999996,"5min":6.8310000000000004,"15min":14.085000000000001},"last":0.59399999999999997},{"interface":"front","average":{"1min":0.69199999999999995,"5min":0.84199999999999997,"15min":1.3100000000000001},"min":{"1min":0.32000000000000001,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":3.0920000000000001,"5min":7.7300000000000004,"15min":18.192},"last":0.438}]},{"osd":7,"last update":"Sat Sep 23 20:34:23 2023","interfaces":[{"interface":"back","average":{"1min":0.59599999999999997,"5min":0.94199999999999995,"15min":1.0660000000000001},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":2.9470000000000001,"5min":8.5060000000000002,"15min":10.217000000000001},"last":0.55800000000000005},{"interface":"front","average":{"1min":0.72299999999999998,"5min":1.204,"15min":1.2250000000000001},"min":{"1min":0.34499999999999997,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":3.0920000000000001,"5min":7.601,"15min":9.7360000000000007},"last":0.53800000000000003}]}]},{"osd":1,"up_from":266,"seq":1142461300802,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1012804,"kb_used_data":761340,"kb_used_omap":8,"kb_used_meta":251319,"kb_avail":92732348,"statfs":{"total":95995035648,"available":94957924352,"internally_reserved":0,"allocated":779612160,"data_stored":777674963,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8206,"internal_metadata":257351666},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:34:54 2023","interfaces":[{"interface":"back","average":{"1min":1.9470000000000001,"5min":2.9750000000000001,"15min":2.5449999999999999},"min":{"1min":0.221,"5min":0.218,"15min":0.218},"max":{"1min":8.6080000000000005,"5min":52.689999999999998,"15min":52.689999999999998},"last":3.7589999999999999},{"interface":"front","average":{"1min":5.3540000000000001,"5min":3.4809999999999999,"15min":3.2909999999999999},"min":{"1min":0.20599999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":26.588000000000001,"5min":52.662999999999997,"15min":52.662999999999997},"last":5.383}]},{"osd":2,"last update":"Sat Sep 23 20:34:54 2023","interfaces":[{"interface":"back","average":{"1min":7.2110000000000003,"5min":4.069,"15min":4.7729999999999997},"min":{"1min":0.312,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":28.359000000000002,"5min":28.359000000000002,"15min":28.359000000000002},"last":9.3960000000000008},{"interface":"front","average":{"1min":3.4550000000000001,"5min":2.6179999999999999,"15min":2.2679999999999998},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":12.801,"5min":12.801,"15min":12.801},"last":3.6520000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:34:54 2023","interfaces":[{"interface":"back","average":{"1min":4.1239999999999997,"5min":4.3799999999999999,"15min":4.0380000000000003},"min":{"1min":0.31,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":15.247999999999999,"5min":80.745999999999995,"15min":80.745999999999995},"last":5.069},{"interface":"front","average":{"1min":5.3380000000000001,"5min":4.6689999999999996,"15min":4.7869999999999999},"min":{"1min":0.35799999999999998,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":17.893999999999998,"5min":82.988,"15min":82.988},"last":3.2240000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:34:54 2023","interfaces":[{"interface":"back","average":{"1min":4.1449999999999996,"5min":2.2989999999999999,"15min":2.1520000000000001},"min":{"1min":0.34200000000000003,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":24.327999999999999,"5min":24.327999999999999,"15min":24.327999999999999},"last":9.4510000000000005},{"interface":"front","average":{"1min":1.2889999999999999,"5min":1.6759999999999999,"15min":1.9079999999999999},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":4.7889999999999997,"5min":10.760999999999999,"15min":10.760999999999999},"last":4.3179999999999996}]},{"osd":5,"last update":"Sat Sep 23 20:34:54 2023","interfaces":[{"interface":"back","average":{"1min":3.3170000000000002,"5min":2.1760000000000002,"15min":2.593},"min":{"1min":0.45000000000000001,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":7.6909999999999998,"5min":9.6479999999999997,"15min":9.6479999999999997},"last":3.4580000000000002},{"interface":"front","average":{"1min":4.1059999999999999,"5min":2.4809999999999999,"15min":2.2200000000000002},"min":{"1min":0.38400000000000001,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":24.404,"5min":24.404,"15min":24.404},"last":9.5150000000000006}]},{"osd":6,"last update":"Sat Sep 23 20:34:54 2023","interfaces":[{"interface":"back","average":{"1min":3.3250000000000002,"5min":3.1059999999999999,"15min":6.0010000000000003},"min":{"1min":0.51200000000000001,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":7.7199999999999998,"5min":72.682000000000002,"15min":72.682000000000002},"last":3.4889999999999999},{"interface":"front","average":{"1min":4.2489999999999997,"5min":3.4180000000000001,"15min":5.4320000000000004},"min":{"1min":0.41099999999999998,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":24.376000000000001,"5min":72.563999999999993,"15min":72.563999999999993},"last":9.4849999999999994}]},{"osd":7,"last update":"Sat Sep 23 20:34:54 2023","interfaces":[{"interface":"back","average":{"1min":3.4489999999999998,"5min":2.3759999999999999,"15min":2.98},"min":{"1min":0.54400000000000004,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":9.0830000000000002,"5min":9.6850000000000005,"15min":9.6850000000000005},"last":18.437000000000001},{"interface":"front","average":{"1min":4.2519999999999998,"5min":2.6230000000000002,"15min":2.4470000000000001},"min":{"1min":0.441,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":24.524000000000001,"5min":24.524000000000001,"15min":24.524000000000001},"last":12.021000000000001}]}]},{"osd":0,"up_from":94,"seq":403726926005,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1310496,"kb_used_data":856408,"kb_used_omap":7,"kb_used_meta":454072,"kb_avail":92434656,"statfs":{"total":95995035648,"available":94653087744,"internally_reserved":0,"allocated":876961792,"data_stored":875175411,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":464969745},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:35:02 2023","interfaces":[{"interface":"back","average":{"1min":3.8130000000000002,"5min":3.5190000000000001,"15min":3.7639999999999998},"min":{"1min":0.183,"5min":0.17999999999999999,"15min":0.17999999999999999},"max":{"1min":12.301,"5min":42.000999999999998,"15min":42.000999999999998},"last":4.7880000000000003},{"interface":"front","average":{"1min":4.5780000000000003,"5min":3.585,"15min":4.1660000000000004},"min":{"1min":0.247,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":18.355,"5min":23.036000000000001,"15min":23.036000000000001},"last":3.5979999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:34:35 2023","interfaces":[{"interface":"back","average":{"1min":3.2410000000000001,"5min":3.4409999999999998,"15min":2.258},"min":{"1min":0.26400000000000001,"5min":0.185,"15min":0.185},"max":{"1min":9.3160000000000007,"5min":77.448999999999998,"15min":77.448999999999998},"last":4.548},{"interface":"front","average":{"1min":3.1269999999999998,"5min":3.3849999999999998,"15min":2.097},"min":{"1min":0.32300000000000001,"5min":0.188,"15min":0.188},"max":{"1min":13.961,"5min":81.724999999999994,"15min":81.724999999999994},"last":5.8029999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:35:07 2023","interfaces":[{"interface":"back","average":{"1min":5.1449999999999996,"5min":4.0899999999999999,"15min":2.5219999999999998},"min":{"1min":0.27500000000000002,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":17.692,"5min":81.400999999999996,"15min":81.400999999999996},"last":2.532},{"interface":"front","average":{"1min":5.085,"5min":4.7759999999999998,"15min":2.6200000000000001},"min":{"1min":0.30299999999999999,"5min":0.25700000000000001,"15min":0.23699999999999999},"max":{"1min":12.262,"5min":81.137,"15min":81.137},"last":3.077}]},{"osd":4,"last update":"Sat Sep 23 20:34:35 2023","interfaces":[{"interface":"back","average":{"1min":2.1680000000000001,"5min":1.927,"15min":1.752},"min":{"1min":0.30599999999999999,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":9.407,"5min":12.587999999999999,"15min":23.876999999999999},"last":2.5720000000000001},{"interface":"front","average":{"1min":3.0289999999999999,"5min":2.194,"15min":1.673},"min":{"1min":0.308,"5min":0.308,"15min":0.29299999999999998},"max":{"1min":10.380000000000001,"5min":23.001000000000001,"15min":23.681999999999999},"last":3.149}]},{"osd":5,"last update":"Sat Sep 23 20:34:35 2023","interfaces":[{"interface":"back","average":{"1min":3.0609999999999999,"5min":2.246,"15min":2},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34200000000000003},"max":{"1min":10.449,"5min":22.969999999999999,"15min":24.198},"last":3.129},{"interface":"front","average":{"1min":2.0430000000000001,"5min":2.1989999999999998,"15min":1.855},"min":{"1min":0.307,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":7.3520000000000003,"5min":22.236000000000001,"15min":22.236000000000001},"last":0.41199999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:34:35 2023","interfaces":[{"interface":"back","average":{"1min":3.3010000000000002,"5min":2.2759999999999998,"15min":1.7549999999999999},"min":{"1min":0.372,"5min":0.32300000000000001,"15min":0.318},"max":{"1min":10.423,"5min":22.934999999999999,"15min":24.408000000000001},"last":3.7189999999999999},{"interface":"front","average":{"1min":2.0369999999999999,"5min":2.1920000000000002,"15min":1.6990000000000001},"min":{"1min":0.38400000000000001,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":7.3979999999999997,"5min":22.164999999999999,"15min":25.271000000000001},"last":0.443}]},{"osd":7,"last update":"Sat Sep 23 20:34:35 2023","interfaces":[{"interface":"back","average":{"1min":2.052,"5min":2.3300000000000001,"15min":1.8779999999999999},"min":{"1min":0.36099999999999999,"5min":0.32200000000000001,"15min":0.30599999999999999},"max":{"1min":7.4320000000000004,"5min":22.206,"15min":25.359000000000002},"last":0.52200000000000002},{"interface":"front","average":{"1min":2.2189999999999999,"5min":2.339,"15min":2.1120000000000001},"min":{"1min":0.38100000000000001,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":9.3710000000000004,"5min":22.422999999999998,"15min":24.669},"last":2.5960000000000001}]}]},{"osd":2,"up_from":104,"seq":446676598952,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1150748,"kb_used_data":667348,"kb_used_omap":11,"kb_used_meta":483316,"kb_avail":92594404,"statfs":{"total":95995035648,"available":94816669696,"internally_reserved":0,"allocated":683364352,"data_stored":681579815,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11878,"internal_metadata":494915994},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:34:44 2023","interfaces":[{"interface":"back","average":{"1min":7.0650000000000004,"5min":4.1500000000000004,"15min":2.3439999999999999},"min":{"1min":0.252,"5min":0.185,"15min":0.17999999999999999},"max":{"1min":28.234999999999999,"5min":82.424999999999997,"15min":82.424999999999997},"last":0.52300000000000002},{"interface":"front","average":{"1min":4.3920000000000003,"5min":3.9169999999999998,"15min":2.073},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":9.6370000000000005,"5min":82.524000000000001,"15min":82.524000000000001},"last":1.462}]},{"osd":1,"last update":"Sat Sep 23 20:34:52 2023","interfaces":[{"interface":"back","average":{"1min":2.7080000000000002,"5min":3.0699999999999998,"15min":4.2569999999999997},"min":{"1min":0.27100000000000002,"5min":0.221,"15min":0.221},"max":{"1min":10.205,"5min":16.280000000000001,"15min":16.280000000000001},"last":1.3120000000000001},{"interface":"front","average":{"1min":4.1100000000000003,"5min":2.8029999999999999,"15min":2.9260000000000002},"min":{"1min":0.36699999999999999,"5min":0.248,"15min":0.248},"max":{"1min":13.130000000000001,"5min":30.565000000000001,"15min":30.565000000000001},"last":9.0489999999999995}]},{"osd":3,"last update":"Sat Sep 23 20:34:09 2023","interfaces":[{"interface":"back","average":{"1min":4.8079999999999998,"5min":3.0230000000000001,"15min":1.7529999999999999},"min":{"1min":0.317,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":15.804,"5min":16.331,"15min":16.331},"last":2.3340000000000001},{"interface":"front","average":{"1min":4.5199999999999996,"5min":2.9169999999999998,"15min":1.9630000000000001},"min":{"1min":0.36899999999999999,"5min":0.249,"15min":0.23899999999999999},"max":{"1min":15.173,"5min":30.704999999999998,"15min":30.704999999999998},"last":0.68899999999999995}]},{"osd":4,"last update":"Sat Sep 23 20:34:44 2023","interfaces":[{"interface":"back","average":{"1min":2.1549999999999998,"5min":2.266,"15min":1.5680000000000001},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.28100000000000003},"max":{"1min":7.6879999999999997,"5min":16.797999999999998,"15min":16.797999999999998},"last":1.512},{"interface":"front","average":{"1min":3.1139999999999999,"5min":2.4209999999999998,"15min":1.73},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.27600000000000002},"max":{"1min":15.853999999999999,"5min":15.853999999999999,"15min":15.853999999999999},"last":1.411}]},{"osd":5,"last update":"Sat Sep 23 20:34:44 2023","interfaces":[{"interface":"back","average":{"1min":2.5270000000000001,"5min":2.4409999999999998,"15min":2.1059999999999999},"min":{"1min":0.46800000000000003,"5min":0.314,"15min":0.309},"max":{"1min":7.7190000000000003,"5min":16.766999999999999,"15min":95.393000000000001},"last":1.5509999999999999},{"interface":"front","average":{"1min":3.7930000000000001,"5min":2.827,"15min":2.3359999999999999},"min":{"1min":0.36799999999999999,"5min":0.34799999999999998,"15min":0.32300000000000001},"max":{"1min":15.885,"5min":15.885,"15min":95.414000000000001},"last":1.379}]},{"osd":6,"last update":"Sat Sep 23 20:34:44 2023","interfaces":[{"interface":"back","average":{"1min":3.1120000000000001,"5min":2.694,"15min":1.829},"min":{"1min":0.36599999999999999,"5min":0.33600000000000002,"15min":0.32100000000000001},"max":{"1min":15.914,"5min":15.914,"15min":15.914},"last":1.4370000000000001},{"interface":"front","average":{"1min":3.6179999999999999,"5min":2.3220000000000001,"15min":1.8140000000000001},"min":{"1min":0.40400000000000003,"5min":0.316,"15min":0.316},"max":{"1min":9.2370000000000001,"5min":30.670999999999999,"15min":30.670999999999999},"last":0.79300000000000004}]},{"osd":7,"last update":"Sat Sep 23 20:34:44 2023","interfaces":[{"interface":"back","average":{"1min":3.3879999999999999,"5min":2.444,"15min":1.8839999999999999},"min":{"1min":0.372,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":9.2759999999999998,"5min":30.741,"15min":30.741},"last":0.82199999999999995},{"interface":"front","average":{"1min":2.758,"5min":2.645,"15min":1.8380000000000001},"min":{"1min":0.495,"5min":0.38,"15min":0.38},"max":{"1min":7.7549999999999999,"5min":16.719000000000001,"15min":16.719000000000001},"last":1.7470000000000001}]}]},{"osd":3,"up_from":109,"seq":468151435426,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1640408,"kb_used_data":1106320,"kb_used_omap":4,"kb_used_meta":534075,"kb_avail":92104744,"statfs":{"total":95995035648,"available":94315257856,"internally_reserved":0,"allocated":1132871680,"data_stored":1131089405,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4514,"internal_metadata":546893406},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:34:10 2023","interfaces":[{"interface":"back","average":{"1min":4.9000000000000004,"5min":2.8599999999999999,"15min":2.0510000000000002},"min":{"1min":0.63200000000000001,"5min":0.191,"15min":0.186},"max":{"1min":17.129000000000001,"5min":17.129000000000001,"15min":67.305999999999997},"last":0.77700000000000002},{"interface":"front","average":{"1min":4.8099999999999996,"5min":2.7679999999999998,"15min":1.9239999999999999},"min":{"1min":0.26200000000000001,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":12.502000000000001,"5min":12.781000000000001,"15min":67.403999999999996},"last":0.68899999999999995}]},{"osd":1,"last update":"Sat Sep 23 20:34:55 2023","interfaces":[{"interface":"back","average":{"1min":2.7589999999999999,"5min":2.7189999999999999,"15min":2.9649999999999999},"min":{"1min":0.28799999999999998,"5min":0.222,"15min":0.222},"max":{"1min":14.641,"5min":16.189,"15min":16.189},"last":0.318},{"interface":"front","average":{"1min":2.1139999999999999,"5min":2.6299999999999999,"15min":3.0449999999999999},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":10.811,"5min":32.603999999999999,"15min":32.603999999999999},"last":3.887}]},{"osd":2,"last update":"Sat Sep 23 20:34:10 2023","interfaces":[{"interface":"back","average":{"1min":5.5899999999999999,"5min":3.472,"15min":2.0899999999999999},"min":{"1min":0.36399999999999999,"5min":0.24399999999999999,"15min":0.22800000000000001},"max":{"1min":20.405000000000001,"5min":32.857999999999997,"15min":32.857999999999997},"last":3.3119999999999998},{"interface":"front","average":{"1min":4.7270000000000003,"5min":3.113,"15min":1.9670000000000001},"min":{"1min":0.25700000000000001,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":24.146999999999998,"5min":24.146999999999998,"15min":24.146999999999998},"last":0.51100000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:34:10 2023","interfaces":[{"interface":"back","average":{"1min":2.9790000000000001,"5min":2.117,"15min":1.6339999999999999},"min":{"1min":0.41499999999999998,"5min":0.34200000000000003,"15min":0.29799999999999999},"max":{"1min":12.151,"5min":12.151,"15min":46.042999999999999},"last":0.35999999999999999},{"interface":"front","average":{"1min":3.4350000000000001,"5min":2.5630000000000002,"15min":1.919},"min":{"1min":0.29399999999999998,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":12.664,"5min":19.635000000000002,"15min":46.579999999999998},"last":1.1879999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:34:10 2023","interfaces":[{"interface":"back","average":{"1min":3.2050000000000001,"5min":2.266,"15min":1.843},"min":{"1min":0.48999999999999999,"5min":0.31900000000000001,"15min":0.311},"max":{"1min":12.096,"5min":12.096,"15min":45.292000000000002},"last":0.41599999999999998},{"interface":"front","average":{"1min":2.4529999999999998,"5min":2.2050000000000001,"15min":1.696},"min":{"1min":0.47499999999999998,"5min":0.32300000000000001,"15min":0.313},"max":{"1min":10.300000000000001,"5min":11.028,"15min":44.679000000000002},"last":0.438}]},{"osd":6,"last update":"Sat Sep 23 20:34:10 2023","interfaces":[{"interface":"back","average":{"1min":2.7970000000000002,"5min":2.419,"15min":1.7350000000000001},"min":{"1min":0.45300000000000001,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":12.464,"5min":17.524999999999999,"15min":17.524999999999999},"last":0.46800000000000003},{"interface":"front","average":{"1min":2.5779999999999998,"5min":2.3180000000000001,"15min":1.647},"min":{"1min":0.371,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":5.8710000000000004,"5min":19.524000000000001,"15min":19.524000000000001},"last":1.282}]},{"osd":7,"last update":"Sat Sep 23 20:34:10 2023","interfaces":[{"interface":"back","average":{"1min":2.4969999999999999,"5min":2.7029999999999998,"15min":1.8129999999999999},"min":{"1min":0.46000000000000002,"5min":0.36899999999999999,"15min":0.34999999999999998},"max":{"1min":5.9530000000000003,"5min":19.571999999999999,"15min":19.571999999999999},"last":5.0380000000000003},{"interface":"front","average":{"1min":3.427,"5min":2.2149999999999999,"15min":1.9059999999999999},"min":{"1min":0.58999999999999997,"5min":0.42599999999999999,"15min":0.371},"max":{"1min":12.186,"5min":12.186,"15min":45.387999999999998},"last":0.73999999999999999}]}]},{"osd":4,"up_from":30,"seq":128849019180,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":601476,"kb_used_data":355452,"kb_used_omap":0,"kb_used_meta":246016,"kb_avail":93143676,"statfs":{"total":95995035648,"available":95379124224,"internally_reserved":0,"allocated":363982848,"data_stored":362860736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":251920384},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:34:37 2023","interfaces":[{"interface":"back","average":{"1min":2.1859999999999999,"5min":1.51,"15min":1.409},"min":{"1min":0.26200000000000001,"5min":0.255,"15min":0.222},"max":{"1min":8.0800000000000001,"5min":8.984,"15min":20.411999999999999},"last":1.454},{"interface":"front","average":{"1min":2.2229999999999999,"5min":1.726,"15min":1.5580000000000001},"min":{"1min":0.29499999999999998,"5min":0.25800000000000001,"15min":0.253},"max":{"1min":8.0990000000000002,"5min":10.675000000000001,"15min":20.454999999999998},"last":1.468}]},{"osd":1,"last update":"Sat Sep 23 20:34:56 2023","interfaces":[{"interface":"back","average":{"1min":1.4350000000000001,"5min":1.885,"15min":1.9670000000000001},"min":{"1min":0.27600000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":6.0869999999999997,"5min":11.303000000000001,"15min":11.303000000000001},"last":0.32000000000000001},{"interface":"front","average":{"1min":1.2110000000000001,"5min":1.6299999999999999,"15min":1.8859999999999999},"min":{"1min":0.30499999999999999,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":3.9169999999999998,"5min":11.287000000000001,"15min":11.287000000000001},"last":0.309}]},{"osd":2,"last update":"Sat Sep 23 20:34:48 2023","interfaces":[{"interface":"back","average":{"1min":1.536,"5min":1.952,"15min":1.6579999999999999},"min":{"1min":0.314,"5min":0.311,"15min":0.26600000000000001},"max":{"1min":3.3140000000000001,"5min":10.275,"15min":32.363},"last":0.91100000000000003},{"interface":"front","average":{"1min":2.5990000000000002,"5min":2.0470000000000002,"15min":1.9370000000000001},"min":{"1min":0.36899999999999999,"5min":0.28999999999999998,"15min":0.26800000000000002},"max":{"1min":5.9009999999999998,"5min":14.614000000000001,"15min":34.042999999999999},"last":1.3899999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:35:09 2023","interfaces":[{"interface":"back","average":{"1min":2.085,"5min":2.2839999999999998,"15min":1.849},"min":{"1min":0.34899999999999998,"5min":0.34799999999999998,"15min":0.29099999999999998},"max":{"1min":7.2060000000000004,"5min":14.515000000000001,"15min":32.332999999999998},"last":0.877},{"interface":"front","average":{"1min":2.5150000000000001,"5min":2.2599999999999998,"15min":1.9379999999999999},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":7.9210000000000003,"5min":15.991,"15min":32.399999999999999},"last":0.39600000000000002}]},{"osd":5,"last update":"Sat Sep 23 20:34:32 2023","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":0.98599999999999999,"15min":1.5269999999999999},"min":{"1min":0.28000000000000003,"5min":0.25800000000000001,"15min":0.247},"max":{"1min":3.0920000000000001,"5min":7.3719999999999999,"15min":82.866},"last":0.38400000000000001},{"interface":"front","average":{"1min":0.70499999999999996,"5min":1.0009999999999999,"15min":1.4279999999999999},"min":{"1min":0.36699999999999999,"5min":0.30299999999999999,"15min":0.27800000000000002},"max":{"1min":3.0699999999999998,"5min":6.8520000000000003,"15min":82.546000000000006},"last":0.436}]},{"osd":6,"last update":"Sat Sep 23 20:34:48 2023","interfaces":[{"interface":"back","average":{"1min":1.3460000000000001,"5min":1.0640000000000001,"15min":1.3560000000000001},"min":{"1min":0.33300000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":10.178000000000001,"5min":10.178000000000001,"15min":41.371000000000002},"last":0.35799999999999998},{"interface":"front","average":{"1min":0.63900000000000001,"5min":0.84699999999999998,"15min":1.2410000000000001},"min":{"1min":0.39600000000000002,"5min":0.34000000000000002,"15min":0.27200000000000002},"max":{"1min":2.3900000000000001,"5min":5.4619999999999997,"15min":41.140999999999998},"last":0.48899999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:35:09 2023","interfaces":[{"interface":"back","average":{"1min":0.86499999999999999,"5min":1.0880000000000001,"15min":1.425},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.33600000000000002},"max":{"1min":3.2970000000000002,"5min":6.8849999999999998,"15min":16.427},"last":0.40899999999999997},{"interface":"front","average":{"1min":0.65200000000000002,"5min":0.81599999999999995,"15min":1.2989999999999999},"min":{"1min":0.36399999999999999,"5min":0.33000000000000002,"15min":0.307},"max":{"1min":1.4490000000000001,"5min":6.3970000000000002,"15min":19.018999999999998},"last":0.46000000000000002}]}]},{"osd":5,"up_from":39,"seq":167503724841,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":688320,"kb_used_data":426296,"kb_used_omap":0,"kb_used_meta":262016,"kb_avail":93056832,"statfs":{"total":95995035648,"available":95290195968,"internally_reserved":0,"allocated":436527104,"data_stored":435405718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":268304384},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:34:40 2023","interfaces":[{"interface":"back","average":{"1min":1.9930000000000001,"5min":1.7350000000000001,"15min":1.4139999999999999},"min":{"1min":0.23400000000000001,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":7.6280000000000001,"5min":8.1609999999999996,"15min":32.475000000000001},"last":0.53800000000000003},{"interface":"front","average":{"1min":2.4060000000000001,"5min":1.6799999999999999,"15min":1.4299999999999999},"min":{"1min":0.29599999999999999,"5min":0.26300000000000001,"15min":0.20599999999999999},"max":{"1min":8.4459999999999997,"5min":8.4459999999999997,"15min":32.457000000000001},"last":0.44900000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:34:59 2023","interfaces":[{"interface":"back","average":{"1min":2.1640000000000001,"5min":1.694,"15min":1.9339999999999999},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":9.3539999999999992,"5min":9.3539999999999992,"15min":9.3539999999999992},"last":1.2889999999999999},{"interface":"front","average":{"1min":2.1070000000000002,"5min":1.516,"15min":1.546},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":9.3900000000000006,"5min":9.3900000000000006,"15min":9.3900000000000006},"last":0.46200000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:34:40 2023","interfaces":[{"interface":"back","average":{"1min":3.7709999999999999,"5min":2.3319999999999999,"15min":1.72},"min":{"1min":0.28000000000000003,"5min":0.27400000000000002,"15min":0.26700000000000002},"max":{"1min":27.603999999999999,"5min":27.603999999999999,"15min":34.514000000000003},"last":1.8280000000000001},{"interface":"front","average":{"1min":3.7229999999999999,"5min":2.3010000000000002,"15min":1.748},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":27.626999999999999,"5min":27.626999999999999,"15min":32.515000000000001},"last":0.40899999999999997}]},{"osd":3,"last update":"Sat Sep 23 20:35:09 2023","interfaces":[{"interface":"back","average":{"1min":2.2130000000000001,"5min":2.5499999999999998,"15min":1.8149999999999999},"min":{"1min":0.314,"5min":0.29599999999999999,"15min":0.25700000000000001},"max":{"1min":9.4770000000000003,"5min":16.189,"15min":16.189},"last":0.48299999999999998},{"interface":"front","average":{"1min":3.7400000000000002,"5min":2.2749999999999999,"15min":1.669},"min":{"1min":0.36099999999999999,"5min":0.32300000000000001,"15min":0.28799999999999998},"max":{"1min":10.191000000000001,"5min":10.577999999999999,"15min":12.005000000000001},"last":0.50600000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:34:30 2023","interfaces":[{"interface":"back","average":{"1min":0.51700000000000002,"5min":1.1930000000000001,"15min":1.292},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.25800000000000001},"max":{"1min":1.8979999999999999,"5min":28.015999999999998,"15min":28.015999999999998},"last":0.36199999999999999},{"interface":"front","average":{"1min":0.68200000000000005,"5min":0.93600000000000005,"15min":1.0649999999999999},"min":{"1min":0.318,"5min":0.29799999999999999,"15min":0.29099999999999998},"max":{"1min":2.9119999999999999,"5min":12.300000000000001,"15min":12.300000000000001},"last":0.57899999999999996}]},{"osd":6,"last update":"Sat Sep 23 20:34:46 2023","interfaces":[{"interface":"back","average":{"1min":0.64000000000000001,"5min":0.77600000000000002,"15min":1.202},"min":{"1min":0.35399999999999998,"5min":0.309,"15min":0.30499999999999999},"max":{"1min":1.841,"5min":4.9829999999999997,"15min":12.890000000000001},"last":0.39600000000000002},{"interface":"front","average":{"1min":0.77000000000000002,"5min":0.755,"15min":1.1719999999999999},"min":{"1min":0.39900000000000002,"5min":0.371,"15min":0.34499999999999997},"max":{"1min":5.1680000000000001,"5min":5.1680000000000001,"15min":16.265999999999998},"last":0.60699999999999998}]},{"osd":7,"last update":"Sat Sep 23 20:35:09 2023","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.68999999999999995,"15min":1.226},"min":{"1min":0.36399999999999999,"5min":0.32100000000000001,"15min":0.29699999999999999},"max":{"1min":1.946,"5min":2.77,"15min":36.621000000000002},"last":0.63700000000000001},{"interface":"front","average":{"1min":0.76100000000000001,"5min":0.72799999999999998,"15min":1.1359999999999999},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":5.093,"5min":5.093,"15min":37.331000000000003},"last":0.629}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4722,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1360,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8206,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7156,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3154,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":851644416,"data_stored":851640344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":765661184,"data_stored":765657112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":672595968,"data_stored":672595968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1107034112,"data_stored":1107034112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":356909056,"data_stored":356909056,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":421593088,"data_stored":421593088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":839782400,"data_stored":839778328,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":630980608,"data_stored":630980608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:35:14.465 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-23T20:35:14.466 INFO:teuthology.orchestra.run.smithi149.stdout:{ 2023-09-23T20:35:14.466 INFO:teuthology.orchestra.run.smithi149.stdout: "ops": [], 2023-09-23T20:35:14.466 INFO:teuthology.orchestra.run.smithi149.stdout: "complaint_time": 30, 2023-09-23T20:35:14.466 INFO:teuthology.orchestra.run.smithi149.stdout: "num_blocked_ops": 0 2023-09-23T20:35:14.466 INFO:teuthology.orchestra.run.smithi149.stdout:} 2023-09-23T20:35:14.558 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:35:14.598 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:35:14.701 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T20:35:14.777 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:35:14.780 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:35:14.781 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:35:14.882 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:35:14.920 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:35:14.975 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:35:14.981 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:35:14.981 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:35:15.082 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:35:15.143 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:35:14 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:35:14] "GET /metrics HTTP/1.1" 200 36537 "" "Prometheus/2.43.0" 2023-09-23T20:35:15.156 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:35:15.239 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:35:15.244 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:35:15.244 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:35:15.259 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:35:15.346 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:35:15.375 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:35:15.416 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:35:15.425 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:35:15.431 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:35:15.432 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:35:15.464 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:15 smithi018 ceph-mon[179647]: from='client.55615 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:35:15.464 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:15 smithi018 ceph-mon[179647]: pgmap v848: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 116 active+clean; 1.6 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 4.2 KiB/s rd, 11 KiB/s wr, 641 op/s; 4095/76821 objects misplaced (5.331%); 2.3 MiB/s, 36 objects/s recovering 2023-09-23T20:35:15.464 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:15 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:35:15.464 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:15 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:35:15.465 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:15 smithi018 ceph-mon[184144]: from='client.55615 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:35:15.465 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:15 smithi018 ceph-mon[184144]: pgmap v848: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 116 active+clean; 1.6 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 4.2 KiB/s rd, 11 KiB/s wr, 641 op/s; 4095/76821 objects misplaced (5.331%); 2.3 MiB/s, 36 objects/s recovering 2023-09-23T20:35:15.465 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:15 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:35:15.465 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:15 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:35:15.533 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:35:15.630 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:35:15.655 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:15 smithi149 ceph-mon[170394]: from='client.55615 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:35:15.656 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:15 smithi149 ceph-mon[170394]: pgmap v848: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 116 active+clean; 1.6 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 4.2 KiB/s rd, 11 KiB/s wr, 641 op/s; 4095/76821 objects misplaced (5.331%); 2.3 MiB/s, 36 objects/s recovering 2023-09-23T20:35:15.656 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:15 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:35:15.656 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:15 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:35:16.959 INFO:teuthology.orchestra.run.smithi149.stdout:{ 2023-09-23T20:35:16.959 INFO:teuthology.orchestra.run.smithi149.stdout: "size": 20, 2023-09-23T20:35:16.959 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 600, 2023-09-23T20:35:16.959 INFO:teuthology.orchestra.run.smithi149.stdout: "ops": [ 2023-09-23T20:35:16.959 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:16.959 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.45161.0:68446 6.5 6:aa6b0392:::benchmark_data_smithi018_270558_object68445:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e229)", 2023-09-23T20:35:16.960 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:28:04.006452+0000", 2023-09-23T20:35:16.960 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 432.94811366800002, 2023-09-23T20:35:16.960 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.1775093220000001, 2023-09-23T20:35:16.960 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:35:16.960 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:35:16.960 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:35:16.960 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.45161", 2023-09-23T20:35:16.960 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3795614987", 2023-09-23T20:35:16.960 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 68446 2023-09-23T20:35:16.960 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:16.960 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:35:16.961 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:16.961 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:35:16.961 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.006452+0000", 2023-09-23T20:35:16.961 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:35:16.961 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:16.961 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:16.961 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:35:16.961 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.006452+0000", 2023-09-23T20:35:16.961 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:35:16.961 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:16.961 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:16.962 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:35:16.962 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.006451+0000", 2023-09-23T20:35:16.962 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:35:16.962 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:16.962 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:16.962 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:35:16.962 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.006483+0000", 2023-09-23T20:35:16.962 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.1628999999999998e-05 2023-09-23T20:35:16.962 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:16.962 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:16.962 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:35:16.963 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.006492+0000", 2023-09-23T20:35:16.963 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.85e-06 2023-09-23T20:35:16.963 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:16.963 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:16.963 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:35:16.963 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.006496+0000", 2023-09-23T20:35:16.963 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.6800000000000001e-06 2023-09-23T20:35:16.963 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:16.963 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:16.963 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:35:16.963 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.009246+0000", 2023-09-23T20:35:16.964 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.002750106 2023-09-23T20:35:16.964 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:16.964 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:16.964 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:35:16.964 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.009403+0000", 2023-09-23T20:35:16.964 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00015714200000000001 2023-09-23T20:35:16.964 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:16.964 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:16.964 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 3,6", 2023-09-23T20:35:16.964 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.009573+0000", 2023-09-23T20:35:16.964 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00016996700000000001 2023-09-23T20:35:16.965 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:16.965 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:16.965 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:35:16.965 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.012816+0000", 2023-09-23T20:35:16.965 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0032425520000000001 2023-09-23T20:35:16.965 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:16.965 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:16.965 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:35:16.965 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.039131+0000", 2023-09-23T20:35:16.965 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.026315412 2023-09-23T20:35:16.966 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:16.966 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:16.966 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:35:16.966 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.051095+0000", 2023-09-23T20:35:16.966 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.011963381 2023-09-23T20:35:16.966 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:16.966 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:16.966 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:35:16.966 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.051104+0000", 2023-09-23T20:35:16.966 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.3260000000000002e-06 2023-09-23T20:35:16.966 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:16.967 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:16.967 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:35:16.967 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:05.014602+0000", 2023-09-23T20:35:16.967 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.96349793900000003 2023-09-23T20:35:16.967 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:16.967 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:16.967 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:35:16.967 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:05.014610+0000", 2023-09-23T20:35:16.967 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.1079999999999993e-06 2023-09-23T20:35:16.967 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:16.967 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:16.968 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:35:16.968 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.183859+0000", 2023-09-23T20:35:16.968 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.169248861 2023-09-23T20:35:16.968 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:16.968 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:16.968 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:35:16.968 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.183961+0000", 2023-09-23T20:35:16.968 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00010183099999999999 2023-09-23T20:35:16.968 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:35:16.968 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:35:16.968 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:35:16.968 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:16.969 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:16.969 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.45161.0:68457 6.3 6:c96b4b0b:::benchmark_data_smithi018_270558_object68456:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e229)", 2023-09-23T20:35:16.969 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:28:04.017421+0000", 2023-09-23T20:35:16.969 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 432.93714412700001, 2023-09-23T20:35:16.969 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.1984804609999999, 2023-09-23T20:35:16.969 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:35:16.969 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:35:16.969 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:35:16.969 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.45161", 2023-09-23T20:35:16.969 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3795614987", 2023-09-23T20:35:16.969 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 68457 2023-09-23T20:35:16.970 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:16.970 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:35:16.970 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:16.970 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:35:16.970 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.017421+0000", 2023-09-23T20:35:16.970 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:35:16.970 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:16.970 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:16.970 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:35:16.970 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.017421+0000", 2023-09-23T20:35:16.970 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:35:16.970 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:16.971 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:16.971 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:35:16.971 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.017419+0000", 2023-09-23T20:35:16.971 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999976 2023-09-23T20:35:16.971 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:16.971 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:16.971 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:35:16.971 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.017470+0000", 2023-09-23T20:35:16.971 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.1082999999999999e-05 2023-09-23T20:35:16.971 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:16.971 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:16.971 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:35:16.972 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.017490+0000", 2023-09-23T20:35:16.972 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.9848999999999999e-05 2023-09-23T20:35:16.972 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:16.972 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:16.972 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:35:16.972 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.017500+0000", 2023-09-23T20:35:16.972 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0094e-05 2023-09-23T20:35:16.972 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:16.972 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:16.972 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:35:16.972 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.021025+0000", 2023-09-23T20:35:16.973 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0035254679999999999 2023-09-23T20:35:16.973 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:16.973 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:16.973 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:35:16.973 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.021234+0000", 2023-09-23T20:35:16.973 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000208805 2023-09-23T20:35:16.973 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:16.973 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:16.973 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 2,6", 2023-09-23T20:35:16.973 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.021415+0000", 2023-09-23T20:35:16.973 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00018162199999999999 2023-09-23T20:35:16.973 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:16.974 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:16.974 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:35:16.974 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.039193+0000", 2023-09-23T20:35:16.974 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.017777167999999999 2023-09-23T20:35:16.974 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:16.974 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:16.974 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:35:16.974 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.058121+0000", 2023-09-23T20:35:16.974 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.018928479000000002 2023-09-23T20:35:16.974 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:16.974 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:16.974 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:35:16.975 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.058131+0000", 2023-09-23T20:35:16.975 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0193999999999999e-05 2023-09-23T20:35:16.975 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:16.975 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:16.975 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:35:16.975 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:05.013114+0000", 2023-09-23T20:35:16.975 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.95498276500000001 2023-09-23T20:35:16.975 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:16.975 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:16.975 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:35:16.975 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:05.013126+0000", 2023-09-23T20:35:16.975 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.2140999999999999e-05 2023-09-23T20:35:16.976 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:16.976 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:16.976 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:35:16.976 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.163597+0000", 2023-09-23T20:35:16.976 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1504710300000001 2023-09-23T20:35:16.976 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:16.976 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:16.976 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for degraded object", 2023-09-23T20:35:16.976 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.164537+0000", 2023-09-23T20:35:16.976 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00094005900000000003 2023-09-23T20:35:16.976 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:16.976 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:16.977 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:35:16.977 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.215805+0000", 2023-09-23T20:35:16.977 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.05126737 2023-09-23T20:35:16.977 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:16.977 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:16.977 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:35:16.977 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.215902+0000", 2023-09-23T20:35:16.977 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.6806999999999997e-05 2023-09-23T20:35:16.977 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:35:16.977 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:35:16.977 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:35:16.977 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:16.978 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:16.978 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.45161.0:68471 6.5 6:a85ec5ad:::benchmark_data_smithi018_270558_object68470:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e230)", 2023-09-23T20:35:16.978 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:28:04.058014+0000", 2023-09-23T20:35:16.978 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 432.89655093699997, 2023-09-23T20:35:16.978 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.1399790890000001, 2023-09-23T20:35:16.978 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:35:16.978 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:35:16.978 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:35:16.978 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.45161", 2023-09-23T20:35:16.978 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3795614987", 2023-09-23T20:35:16.978 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 68471 2023-09-23T20:35:16.979 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:16.979 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:35:16.979 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:16.979 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:35:16.979 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.058014+0000", 2023-09-23T20:35:16.979 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:35:16.979 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:16.979 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:16.979 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:35:16.979 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.058014+0000", 2023-09-23T20:35:16.979 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:35:16.979 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:16.980 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:16.980 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:35:16.980 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.058013+0000", 2023-09-23T20:35:16.980 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:35:16.980 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:16.980 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:16.980 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:35:16.980 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.058072+0000", 2023-09-23T20:35:16.980 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.8891999999999998e-05 2023-09-23T20:35:16.980 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:16.980 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:16.980 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:35:16.981 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.058083+0000", 2023-09-23T20:35:16.981 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1775e-05 2023-09-23T20:35:16.981 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:16.981 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:16.981 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:35:16.981 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.058091+0000", 2023-09-23T20:35:16.981 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.6680000000000001e-06 2023-09-23T20:35:16.981 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:16.981 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:16.981 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:35:16.981 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.058348+0000", 2023-09-23T20:35:16.981 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00025689700000000002 2023-09-23T20:35:16.982 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:16.982 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:16.982 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:35:16.982 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.058358+0000", 2023-09-23T20:35:16.982 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.023e-05 2023-09-23T20:35:16.982 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:16.982 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:16.982 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:35:16.982 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:05.014631+0000", 2023-09-23T20:35:16.982 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.95627271700000005 2023-09-23T20:35:16.982 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:16.983 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:16.983 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:35:16.983 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:05.014638+0000", 2023-09-23T20:35:16.983 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.5449999999999998e-06 2023-09-23T20:35:16.983 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:16.983 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:16.983 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:35:16.983 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.183989+0000", 2023-09-23T20:35:16.983 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1693501770000001 2023-09-23T20:35:16.983 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:16.983 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:16.983 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:35:16.984 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.184136+0000", 2023-09-23T20:35:16.984 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00014772 2023-09-23T20:35:16.984 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:16.984 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:16.984 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 2,3", 2023-09-23T20:35:16.984 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.184314+0000", 2023-09-23T20:35:16.984 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00017729399999999999 2023-09-23T20:35:16.984 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:16.984 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:16.984 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:35:16.984 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.193577+0000", 2023-09-23T20:35:16.985 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0092632889999999992 2023-09-23T20:35:16.985 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:16.985 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:16.985 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:35:16.985 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.193688+0000", 2023-09-23T20:35:16.985 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00011158900000000001 2023-09-23T20:35:16.985 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:16.985 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:16.985 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:35:16.985 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.194841+0000", 2023-09-23T20:35:16.985 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0011521319999999999 2023-09-23T20:35:16.986 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:16.986 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:16.986 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:35:16.986 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.194879+0000", 2023-09-23T20:35:16.986 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.8649999999999998e-05 2023-09-23T20:35:16.986 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:16.986 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:16.986 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:35:16.986 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.197993+0000", 2023-09-23T20:35:16.986 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0031141210000000001 2023-09-23T20:35:16.986 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:35:16.987 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:35:16.987 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:35:16.987 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:16.987 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:16.987 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.45161.0:68482 6.5 6:a8dc35b6:::benchmark_data_smithi018_270558_object68481:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e230)", 2023-09-23T20:35:16.987 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:28:04.068966+0000", 2023-09-23T20:35:16.987 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 432.88559909100002, 2023-09-23T20:35:16.987 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.1433483820000001, 2023-09-23T20:35:16.987 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:35:16.987 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:35:16.987 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:35:16.987 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.45161", 2023-09-23T20:35:16.988 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3795614987", 2023-09-23T20:35:16.988 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 68482 2023-09-23T20:35:16.988 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:16.988 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:35:16.988 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:16.988 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:35:16.988 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.068966+0000", 2023-09-23T20:35:16.988 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:35:16.988 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:16.988 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:16.988 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:35:16.989 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.068966+0000", 2023-09-23T20:35:16.989 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:35:16.989 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:16.989 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:16.989 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:35:16.989 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.068965+0000", 2023-09-23T20:35:16.989 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:35:16.989 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:16.989 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:16.989 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:35:16.989 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.069008+0000", 2023-09-23T20:35:16.989 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.3155000000000003e-05 2023-09-23T20:35:16.990 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:16.990 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:16.990 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:35:16.990 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.069020+0000", 2023-09-23T20:35:16.990 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.2804e-05 2023-09-23T20:35:16.990 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:16.990 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:16.990 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:35:16.990 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.069027+0000", 2023-09-23T20:35:16.990 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.5080000000000002e-06 2023-09-23T20:35:16.990 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:16.991 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:16.991 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:35:16.991 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.069101+0000", 2023-09-23T20:35:16.991 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.4017999999999995e-05 2023-09-23T20:35:16.991 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:16.991 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:16.991 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:35:16.991 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.069111+0000", 2023-09-23T20:35:16.991 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0426e-05 2023-09-23T20:35:16.991 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:16.991 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:16.991 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:35:16.992 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:05.014659+0000", 2023-09-23T20:35:16.992 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.94554741200000003 2023-09-23T20:35:16.992 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:16.992 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:16.992 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:35:16.992 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:05.014666+0000", 2023-09-23T20:35:16.992 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.3170000000000004e-06 2023-09-23T20:35:16.992 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:16.992 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:16.992 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:35:16.992 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.187136+0000", 2023-09-23T20:35:16.992 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.172469896 2023-09-23T20:35:16.993 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:16.993 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:16.993 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:35:16.993 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.187291+0000", 2023-09-23T20:35:16.993 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000155437 2023-09-23T20:35:16.993 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:16.993 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:16.993 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 2,3", 2023-09-23T20:35:16.993 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.187476+0000", 2023-09-23T20:35:16.993 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00018494599999999999 2023-09-23T20:35:16.993 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:16.994 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:16.994 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:35:16.994 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.193695+0000", 2023-09-23T20:35:16.994 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0062182280000000001 2023-09-23T20:35:16.994 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:16.994 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:16.994 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:35:16.994 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.198072+0000", 2023-09-23T20:35:16.994 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0043777499999999997 2023-09-23T20:35:16.994 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:16.994 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:16.994 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:35:16.995 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.208878+0000", 2023-09-23T20:35:16.995 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.010805189999999999 2023-09-23T20:35:16.995 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:16.995 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:16.995 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:35:16.995 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.208936+0000", 2023-09-23T20:35:16.995 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.7957000000000001e-05 2023-09-23T20:35:16.995 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:16.995 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:16.995 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:35:16.995 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.212314+0000", 2023-09-23T20:35:16.995 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.003378902 2023-09-23T20:35:16.996 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:35:16.996 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:35:16.996 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:35:16.996 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:16.996 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:16.996 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.45161.0:68484 6.3 6:c8f6e5db:::benchmark_data_smithi018_270558_object68483:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e230)", 2023-09-23T20:35:16.996 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:28:04.071814+0000", 2023-09-23T20:35:16.996 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 432.88275162299999, 2023-09-23T20:35:16.996 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.1657179320000002, 2023-09-23T20:35:16.996 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:35:16.996 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:35:16.997 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:35:16.997 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.45161", 2023-09-23T20:35:16.997 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3795614987", 2023-09-23T20:35:16.997 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 68484 2023-09-23T20:35:16.997 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:16.997 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:35:16.997 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:16.997 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:35:16.997 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.071814+0000", 2023-09-23T20:35:16.997 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:35:16.997 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:16.998 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:16.998 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:35:16.998 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.071814+0000", 2023-09-23T20:35:16.998 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:35:16.998 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:16.998 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:16.998 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:35:16.998 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.071811+0000", 2023-09-23T20:35:16.999 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999976 2023-09-23T20:35:16.999 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:16.999 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:16.999 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:35:16.999 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.071871+0000", 2023-09-23T20:35:16.999 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.9790000000000002e-05 2023-09-23T20:35:16.999 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:16.999 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:16.999 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:35:17.000 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.071901+0000", 2023-09-23T20:35:17.000 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.9560000000000002e-05 2023-09-23T20:35:17.000 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.000 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.000 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:35:17.000 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.071909+0000", 2023-09-23T20:35:17.000 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.3890000000000005e-06 2023-09-23T20:35:17.001 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.001 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.001 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:35:17.001 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.072396+0000", 2023-09-23T20:35:17.001 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00048719099999999999 2023-09-23T20:35:17.001 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.001 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.002 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:35:17.002 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.072410+0000", 2023-09-23T20:35:17.002 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.3373000000000001e-05 2023-09-23T20:35:17.002 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.002 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.002 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:35:17.002 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:05.013152+0000", 2023-09-23T20:35:17.002 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.94074252199999997 2023-09-23T20:35:17.003 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.003 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.003 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:35:17.003 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:05.013160+0000", 2023-09-23T20:35:17.003 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.1990000000000006e-06 2023-09-23T20:35:17.003 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.003 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.003 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:35:17.003 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.215926+0000", 2023-09-23T20:35:17.003 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.2027656010000001 2023-09-23T20:35:17.004 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.004 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.004 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:35:17.004 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.216074+0000", 2023-09-23T20:35:17.004 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000148264 2023-09-23T20:35:17.004 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.004 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.004 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 1,2", 2023-09-23T20:35:17.004 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.216322+0000", 2023-09-23T20:35:17.005 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00024790499999999998 2023-09-23T20:35:17.005 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.005 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.005 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:35:17.005 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.228067+0000", 2023-09-23T20:35:17.005 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.011744747 2023-09-23T20:35:17.005 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.005 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.005 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:35:17.005 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.228080+0000", 2023-09-23T20:35:17.005 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.3380000000000001e-05 2023-09-23T20:35:17.006 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.006 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.006 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:35:17.006 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.234399+0000", 2023-09-23T20:35:17.006 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0063193010000000003 2023-09-23T20:35:17.006 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.006 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.006 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:35:17.006 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.234457+0000", 2023-09-23T20:35:17.006 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.7456e-05 2023-09-23T20:35:17.006 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.007 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.007 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:35:17.007 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.237531+0000", 2023-09-23T20:35:17.007 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0030745450000000001 2023-09-23T20:35:17.007 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:35:17.007 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:35:17.007 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:35:17.007 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.007 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.007 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.45161.0:68485 6.3 6:ceb1ac5f:::benchmark_data_smithi018_270558_object68484:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e230)", 2023-09-23T20:35:17.007 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:28:04.071924+0000", 2023-09-23T20:35:17.008 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 432.882641224, 2023-09-23T20:35:17.008 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.1713503310000002, 2023-09-23T20:35:17.008 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:35:17.008 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:35:17.008 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:35:17.008 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.45161", 2023-09-23T20:35:17.008 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3795614987", 2023-09-23T20:35:17.008 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 68485 2023-09-23T20:35:17.008 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.008 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:35:17.008 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.009 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:35:17.009 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.071924+0000", 2023-09-23T20:35:17.009 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:35:17.009 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.009 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.009 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:35:17.009 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.071924+0000", 2023-09-23T20:35:17.009 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:35:17.009 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.009 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.009 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:35:17.009 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.071923+0000", 2023-09-23T20:35:17.010 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:35:17.010 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.010 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.010 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:35:17.010 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.071978+0000", 2023-09-23T20:35:17.010 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.4419999999999997e-05 2023-09-23T20:35:17.010 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.010 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.010 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:35:17.010 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.071985+0000", 2023-09-23T20:35:17.010 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.0520000000000004e-06 2023-09-23T20:35:17.011 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.011 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.011 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:35:17.011 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.071989+0000", 2023-09-23T20:35:17.011 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.4850000000000003e-06 2023-09-23T20:35:17.011 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.011 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.011 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:35:17.011 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.072433+0000", 2023-09-23T20:35:17.011 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00044325099999999997 2023-09-23T20:35:17.011 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.012 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.012 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:35:17.012 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.072440+0000", 2023-09-23T20:35:17.012 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.3989999999999998e-06 2023-09-23T20:35:17.012 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.012 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.012 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:35:17.012 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:05.013184+0000", 2023-09-23T20:35:17.012 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.940744213 2023-09-23T20:35:17.012 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.012 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.012 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:35:17.013 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:05.013193+0000", 2023-09-23T20:35:17.013 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.6340000000000007e-06 2023-09-23T20:35:17.013 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.013 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.013 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:35:17.013 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.218351+0000", 2023-09-23T20:35:17.013 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.205158497 2023-09-23T20:35:17.013 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.013 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.013 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:35:17.013 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.218571+0000", 2023-09-23T20:35:17.013 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00022005200000000001 2023-09-23T20:35:17.014 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.014 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.014 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 1,2", 2023-09-23T20:35:17.014 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.218740+0000", 2023-09-23T20:35:17.014 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00016879699999999999 2023-09-23T20:35:17.014 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.014 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.014 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:35:17.014 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.228087+0000", 2023-09-23T20:35:17.014 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0093472799999999995 2023-09-23T20:35:17.014 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.015 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.015 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:35:17.015 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.230676+0000", 2023-09-23T20:35:17.015 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0025880819999999998 2023-09-23T20:35:17.015 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.015 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.015 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:35:17.015 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.240032+0000", 2023-09-23T20:35:17.015 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0093564019999999998 2023-09-23T20:35:17.015 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.015 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.015 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:35:17.016 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.240107+0000", 2023-09-23T20:35:17.016 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.5167999999999996e-05 2023-09-23T20:35:17.016 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.016 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.016 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:35:17.016 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.243274+0000", 2023-09-23T20:35:17.016 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.003167128 2023-09-23T20:35:17.016 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:35:17.016 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:35:17.016 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:35:17.016 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.017 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.017 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.45161.0:68486 6.3 6:c4625da8:::benchmark_data_smithi018_270558_object68485:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e230)", 2023-09-23T20:35:17.017 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:28:04.072143+0000", 2023-09-23T20:35:17.017 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 432.88242255, 2023-09-23T20:35:17.017 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.173711715, 2023-09-23T20:35:17.017 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:35:17.017 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:35:17.017 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:35:17.017 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.45161", 2023-09-23T20:35:17.017 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3795614987", 2023-09-23T20:35:17.017 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 68486 2023-09-23T20:35:17.017 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.018 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:35:17.018 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.018 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:35:17.018 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.072143+0000", 2023-09-23T20:35:17.018 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:35:17.018 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.018 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.018 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:35:17.018 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.072143+0000", 2023-09-23T20:35:17.018 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:35:17.018 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.018 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.019 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:35:17.019 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.072142+0000", 2023-09-23T20:35:17.019 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:35:17.019 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.019 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.019 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:35:17.019 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.072222+0000", 2023-09-23T20:35:17.019 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.0106999999999997e-05 2023-09-23T20:35:17.019 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.019 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.019 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:35:17.020 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.072236+0000", 2023-09-23T20:35:17.020 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.3760999999999999e-05 2023-09-23T20:35:17.020 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.020 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.020 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:35:17.020 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.072242+0000", 2023-09-23T20:35:17.020 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.7749999999999998e-06 2023-09-23T20:35:17.020 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.020 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.021 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:35:17.021 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.072471+0000", 2023-09-23T20:35:17.021 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000229779 2023-09-23T20:35:17.021 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.021 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.021 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:35:17.022 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.072479+0000", 2023-09-23T20:35:17.022 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.452e-06 2023-09-23T20:35:17.022 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.022 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.022 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:35:17.022 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:05.013217+0000", 2023-09-23T20:35:17.022 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.94073840099999995 2023-09-23T20:35:17.022 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.022 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.022 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:35:17.023 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:05.013224+0000", 2023-09-23T20:35:17.023 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.2919999999999998e-06 2023-09-23T20:35:17.023 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.023 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.023 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:35:17.023 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.221302+0000", 2023-09-23T20:35:17.023 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.208077751 2023-09-23T20:35:17.023 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.023 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.023 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:35:17.023 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.221476+0000", 2023-09-23T20:35:17.024 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00017371800000000001 2023-09-23T20:35:17.024 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.024 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.024 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 1,2", 2023-09-23T20:35:17.024 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.221657+0000", 2023-09-23T20:35:17.024 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00018101599999999999 2023-09-23T20:35:17.024 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.024 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.024 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:35:17.025 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.228092+0000", 2023-09-23T20:35:17.025 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0064346450000000001 2023-09-23T20:35:17.025 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.025 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.025 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:35:17.025 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.230712+0000", 2023-09-23T20:35:17.025 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0026200059999999998 2023-09-23T20:35:17.025 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.025 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.025 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:35:17.025 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.243356+0000", 2023-09-23T20:35:17.026 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.012643972 2023-09-23T20:35:17.026 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.026 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.026 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:35:17.026 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.243422+0000", 2023-09-23T20:35:17.026 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.6008999999999998e-05 2023-09-23T20:35:17.026 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.026 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.026 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:35:17.026 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.245854+0000", 2023-09-23T20:35:17.026 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0024327659999999998 2023-09-23T20:35:17.027 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:35:17.027 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:35:17.027 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:35:17.027 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.027 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.027 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.45161.0:68497 6.3 6:c484dd05:::benchmark_data_smithi018_270558_object68496:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e230)", 2023-09-23T20:35:17.027 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:28:04.088615+0000", 2023-09-23T20:35:17.027 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 432.865949844, 2023-09-23T20:35:17.028 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.1605494919999999, 2023-09-23T20:35:17.028 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:35:17.028 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:35:17.028 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:35:17.028 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.45161", 2023-09-23T20:35:17.028 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3795614987", 2023-09-23T20:35:17.028 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 68497 2023-09-23T20:35:17.028 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.028 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:35:17.028 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.028 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:35:17.029 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.088615+0000", 2023-09-23T20:35:17.029 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:35:17.029 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.029 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.029 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:35:17.029 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.088615+0000", 2023-09-23T20:35:17.029 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:35:17.029 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.030 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.030 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:35:17.030 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.088613+0000", 2023-09-23T20:35:17.030 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999981 2023-09-23T20:35:17.030 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.030 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.030 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:35:17.030 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.088682+0000", 2023-09-23T20:35:17.030 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.8730000000000001e-05 2023-09-23T20:35:17.031 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.031 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.031 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:35:17.031 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.088704+0000", 2023-09-23T20:35:17.031 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.2246999999999999e-05 2023-09-23T20:35:17.031 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.031 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.031 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:35:17.031 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.088714+0000", 2023-09-23T20:35:17.031 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.7280000000000007e-06 2023-09-23T20:35:17.032 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.032 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.032 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:35:17.032 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.088771+0000", 2023-09-23T20:35:17.032 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.6453000000000003e-05 2023-09-23T20:35:17.032 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.032 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.032 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:35:17.032 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.088779+0000", 2023-09-23T20:35:17.033 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.7050000000000005e-06 2023-09-23T20:35:17.033 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.033 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.033 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:35:17.033 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:05.013542+0000", 2023-09-23T20:35:17.033 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.92476320499999998 2023-09-23T20:35:17.033 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.033 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.033 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:35:17.034 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:05.013553+0000", 2023-09-23T20:35:17.034 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0118e-05 2023-09-23T20:35:17.034 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.034 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.034 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:35:17.034 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.223980+0000", 2023-09-23T20:35:17.034 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.2104271390000001 2023-09-23T20:35:17.034 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.035 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.035 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:35:17.035 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.224129+0000", 2023-09-23T20:35:17.035 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000149438 2023-09-23T20:35:17.035 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.035 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.035 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 1,2", 2023-09-23T20:35:17.035 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.224312+0000", 2023-09-23T20:35:17.036 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00018292499999999999 2023-09-23T20:35:17.036 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.036 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.036 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:35:17.036 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.228096+0000", 2023-09-23T20:35:17.036 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0037838569999999998 2023-09-23T20:35:17.036 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.036 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.037 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:35:17.037 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.230803+0000", 2023-09-23T20:35:17.037 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0027065489999999999 2023-09-23T20:35:17.037 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.037 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.037 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:35:17.037 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.245942+0000", 2023-09-23T20:35:17.037 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.015139832000000001 2023-09-23T20:35:17.037 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.037 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.038 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:35:17.038 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.245993+0000", 2023-09-23T20:35:17.038 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.0849000000000001e-05 2023-09-23T20:35:17.038 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.038 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.038 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:35:17.038 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.249165+0000", 2023-09-23T20:35:17.038 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0031716169999999998 2023-09-23T20:35:17.038 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:35:17.038 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:35:17.038 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:35:17.039 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.039 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.039 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.45161.0:193673 6.5 6:a4b932b5:::benchmark_data_smithi018_270558_object88934:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e288)", 2023-09-23T20:35:17.039 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:30:56.146817+0000", 2023-09-23T20:35:17.039 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 260.80774790300001, 2023-09-23T20:35:17.039 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0049539489999999, 2023-09-23T20:35:17.039 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:35:17.039 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:35:17.039 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:35:17.039 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.45161", 2023-09-23T20:35:17.039 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3795614987", 2023-09-23T20:35:17.039 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 193673 2023-09-23T20:35:17.040 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.040 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:35:17.040 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.040 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:35:17.040 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:56.146817+0000", 2023-09-23T20:35:17.040 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:35:17.040 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.040 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.040 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:35:17.040 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:56.146817+0000", 2023-09-23T20:35:17.040 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:35:17.040 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.041 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.041 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:35:17.041 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:56.146817+0000", 2023-09-23T20:35:17.041 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:35:17.041 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.041 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.041 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:35:17.041 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:56.146820+0000", 2023-09-23T20:35:17.041 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.5710000000000002e-06 2023-09-23T20:35:17.041 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.041 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.042 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:35:17.042 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:56.146828+0000", 2023-09-23T20:35:17.042 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.2960000000000001e-06 2023-09-23T20:35:17.042 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.042 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.042 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:35:17.042 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:56.146832+0000", 2023-09-23T20:35:17.042 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.8350000000000003e-06 2023-09-23T20:35:17.042 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.042 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.042 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:35:17.043 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:56.146935+0000", 2023-09-23T20:35:17.043 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000102823 2023-09-23T20:35:17.043 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.043 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.043 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:35:17.043 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:56.146944+0000", 2023-09-23T20:35:17.043 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.1330000000000005e-06 2023-09-23T20:35:17.043 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.043 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.043 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:35:17.043 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:57.114740+0000", 2023-09-23T20:35:17.044 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.96779585899999998 2023-09-23T20:35:17.044 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.044 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.044 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:35:17.044 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:57.114847+0000", 2023-09-23T20:35:17.044 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00010665799999999999 2023-09-23T20:35:17.044 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.044 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.044 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 3,6", 2023-09-23T20:35:17.044 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:57.115266+0000", 2023-09-23T20:35:17.044 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00041946699999999997 2023-09-23T20:35:17.044 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.045 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.045 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:35:17.045 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:57.126835+0000", 2023-09-23T20:35:17.045 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.011568983 2023-09-23T20:35:17.045 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.045 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.045 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:35:17.045 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:57.151518+0000", 2023-09-23T20:35:17.045 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.024682470000000001 2023-09-23T20:35:17.045 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.045 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.045 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:35:17.046 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:57.151681+0000", 2023-09-23T20:35:17.046 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00016317600000000001 2023-09-23T20:35:17.046 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.046 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.046 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:35:17.046 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:57.151739+0000", 2023-09-23T20:35:17.046 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.8257999999999997e-05 2023-09-23T20:35:17.046 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.046 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.046 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:35:17.046 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:57.151771+0000", 2023-09-23T20:35:17.047 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.1924000000000001e-05 2023-09-23T20:35:17.047 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:35:17.047 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:35:17.047 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:35:17.047 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.047 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.047 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.45161.0:193696 6.5 6:a5a4014d:::benchmark_data_smithi018_270558_object88957:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e288)", 2023-09-23T20:35:17.047 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:30:56.192672+0000", 2023-09-23T20:35:17.047 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 260.76189267699999, 2023-09-23T20:35:17.047 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.95918431800000004, 2023-09-23T20:35:17.047 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:35:17.048 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:35:17.048 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:35:17.048 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.45161", 2023-09-23T20:35:17.048 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3795614987", 2023-09-23T20:35:17.048 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 193696 2023-09-23T20:35:17.048 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.048 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:35:17.048 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.048 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:35:17.048 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:56.192672+0000", 2023-09-23T20:35:17.048 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:35:17.049 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.049 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.049 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:35:17.049 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:56.192672+0000", 2023-09-23T20:35:17.049 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:35:17.049 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.049 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.049 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:35:17.049 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:56.192672+0000", 2023-09-23T20:35:17.049 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:35:17.049 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.049 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.050 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:35:17.050 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:56.192676+0000", 2023-09-23T20:35:17.050 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.5480000000000002e-06 2023-09-23T20:35:17.050 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.050 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.050 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:35:17.050 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:56.192686+0000", 2023-09-23T20:35:17.050 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0149999999999999e-05 2023-09-23T20:35:17.050 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.050 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.050 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:35:17.051 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:56.192689+0000", 2023-09-23T20:35:17.051 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.0910000000000001e-06 2023-09-23T20:35:17.051 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.051 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.051 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:35:17.051 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:56.192739+0000", 2023-09-23T20:35:17.051 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.0253000000000001e-05 2023-09-23T20:35:17.051 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.051 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.051 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:35:17.051 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:56.192746+0000", 2023-09-23T20:35:17.052 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.0010000000000004e-06 2023-09-23T20:35:17.052 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.052 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.052 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:35:17.052 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:57.117501+0000", 2023-09-23T20:35:17.052 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.92475526299999999 2023-09-23T20:35:17.052 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.052 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.052 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:35:17.052 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:57.117635+0000", 2023-09-23T20:35:17.052 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00013371699999999999 2023-09-23T20:35:17.052 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.053 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.053 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 3,6", 2023-09-23T20:35:17.053 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:57.117987+0000", 2023-09-23T20:35:17.053 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00035175700000000001 2023-09-23T20:35:17.053 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.053 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.053 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:35:17.053 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:57.126842+0000", 2023-09-23T20:35:17.053 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0088546649999999994 2023-09-23T20:35:17.053 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.053 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.054 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:35:17.054 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:57.151620+0000", 2023-09-23T20:35:17.054 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.024778901999999998 2023-09-23T20:35:17.054 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.054 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.054 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:35:17.054 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:57.151810+0000", 2023-09-23T20:35:17.054 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00018969399999999999 2023-09-23T20:35:17.054 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.054 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.054 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:35:17.054 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:57.151837+0000", 2023-09-23T20:35:17.055 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.6838999999999998e-05 2023-09-23T20:35:17.055 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.055 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.055 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:35:17.055 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:57.151857+0000", 2023-09-23T20:35:17.055 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.9848e-05 2023-09-23T20:35:17.055 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:35:17.055 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:35:17.055 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:35:17.055 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.055 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.056 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.45161.0:193704 6.5 6:a27b446f:::benchmark_data_smithi018_270558_object88965:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e288)", 2023-09-23T20:35:17.056 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:30:56.201495+0000", 2023-09-23T20:35:17.056 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 260.753070347, 2023-09-23T20:35:17.056 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.95056226799999999, 2023-09-23T20:35:17.056 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:35:17.056 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:35:17.056 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:35:17.056 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.45161", 2023-09-23T20:35:17.056 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3795614987", 2023-09-23T20:35:17.056 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 193704 2023-09-23T20:35:17.056 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.056 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:35:17.057 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.057 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:35:17.057 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:56.201495+0000", 2023-09-23T20:35:17.057 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:35:17.057 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.057 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.057 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:35:17.057 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:56.201495+0000", 2023-09-23T20:35:17.057 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:35:17.057 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.058 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.058 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:35:17.058 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:56.201494+0000", 2023-09-23T20:35:17.058 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:35:17.058 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.058 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.058 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:35:17.058 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:56.201499+0000", 2023-09-23T20:35:17.058 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.1819999999999997e-06 2023-09-23T20:35:17.058 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.059 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.059 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:35:17.059 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:56.201508+0000", 2023-09-23T20:35:17.059 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.7440000000000002e-06 2023-09-23T20:35:17.059 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.059 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.059 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:35:17.059 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:56.201514+0000", 2023-09-23T20:35:17.059 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.8019999999999997e-06 2023-09-23T20:35:17.059 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.061 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.061 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:35:17.061 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:56.201581+0000", 2023-09-23T20:35:17.062 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.7176999999999997e-05 2023-09-23T20:35:17.062 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.062 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.062 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:35:17.062 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:56.201591+0000", 2023-09-23T20:35:17.062 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.2050000000000008e-06 2023-09-23T20:35:17.062 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.063 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.063 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:35:17.063 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:57.120171+0000", 2023-09-23T20:35:17.063 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.91858044599999999 2023-09-23T20:35:17.063 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.063 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.063 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:35:17.064 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:57.120254+0000", 2023-09-23T20:35:17.064 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.2799999999999993e-05 2023-09-23T20:35:17.064 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.064 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.064 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 3,6", 2023-09-23T20:35:17.064 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:57.120540+0000", 2023-09-23T20:35:17.064 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00028637999999999998 2023-09-23T20:35:17.064 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.064 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.064 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:35:17.064 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:57.129589+0000", 2023-09-23T20:35:17.065 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0090484699999999994 2023-09-23T20:35:17.065 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.065 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.065 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:35:17.065 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:57.151887+0000", 2023-09-23T20:35:17.065 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.022298370000000001 2023-09-23T20:35:17.065 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.065 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.065 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:35:17.066 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:57.151987+0000", 2023-09-23T20:35:17.066 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.9631999999999997e-05 2023-09-23T20:35:17.066 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.066 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.066 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:35:17.066 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:57.152031+0000", 2023-09-23T20:35:17.066 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.4570999999999997e-05 2023-09-23T20:35:17.066 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.066 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.066 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:35:17.067 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:57.152057+0000", 2023-09-23T20:35:17.067 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.5738999999999999e-05 2023-09-23T20:35:17.067 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:35:17.067 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:35:17.067 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:35:17.067 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.067 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.067 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.45161.0:193713 6.5 6:a53a13e1:::benchmark_data_smithi018_270558_object88974:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e288)", 2023-09-23T20:35:17.067 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:30:56.213097+0000", 2023-09-23T20:35:17.068 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 260.74146816799998, 2023-09-23T20:35:17.068 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.93902963399999995, 2023-09-23T20:35:17.068 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:35:17.068 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:35:17.068 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:35:17.068 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.45161", 2023-09-23T20:35:17.068 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3795614987", 2023-09-23T20:35:17.068 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 193713 2023-09-23T20:35:17.068 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.069 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:35:17.069 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.069 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:35:17.069 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:56.213097+0000", 2023-09-23T20:35:17.069 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:35:17.069 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.069 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.069 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:35:17.069 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:56.213097+0000", 2023-09-23T20:35:17.070 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:35:17.070 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.070 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.070 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:35:17.070 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:56.213096+0000", 2023-09-23T20:35:17.070 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:35:17.070 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.070 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.071 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:35:17.071 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:56.213102+0000", 2023-09-23T20:35:17.071 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.6579999999999999e-06 2023-09-23T20:35:17.071 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.071 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.071 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:35:17.071 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:56.213114+0000", 2023-09-23T20:35:17.071 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.2734e-05 2023-09-23T20:35:17.071 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.071 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.072 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:35:17.072 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:56.213120+0000", 2023-09-23T20:35:17.072 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.5899999999999998e-06 2023-09-23T20:35:17.072 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.072 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.072 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:35:17.072 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:56.213171+0000", 2023-09-23T20:35:17.072 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.1e-05 2023-09-23T20:35:17.072 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.072 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.072 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:35:17.072 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:56.213180+0000", 2023-09-23T20:35:17.073 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.8089999999999999e-06 2023-09-23T20:35:17.073 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.073 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.073 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:35:17.073 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:57.122356+0000", 2023-09-23T20:35:17.073 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.90917587 2023-09-23T20:35:17.073 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.073 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.073 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:35:17.073 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:57.122485+0000", 2023-09-23T20:35:17.073 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000129178 2023-09-23T20:35:17.073 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.074 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.074 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 3,6", 2023-09-23T20:35:17.074 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:57.122803+0000", 2023-09-23T20:35:17.074 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00031803799999999998 2023-09-23T20:35:17.074 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.074 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.074 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:35:17.074 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:57.129595+0000", 2023-09-23T20:35:17.074 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0067919410000000001 2023-09-23T20:35:17.074 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.074 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.074 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:35:17.075 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:57.151961+0000", 2023-09-23T20:35:17.075 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.022366456 2023-09-23T20:35:17.075 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.075 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.075 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:35:17.075 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:57.152083+0000", 2023-09-23T20:35:17.075 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000121366 2023-09-23T20:35:17.075 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.075 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.075 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:35:17.075 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:57.152108+0000", 2023-09-23T20:35:17.076 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.5058999999999998e-05 2023-09-23T20:35:17.076 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.076 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.076 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:35:17.076 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:57.152127+0000", 2023-09-23T20:35:17.076 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.9037000000000001e-05 2023-09-23T20:35:17.076 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:35:17.076 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:35:17.076 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:35:17.076 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.076 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.076 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.45161.0:193722 6.5 6:a673ba13:::benchmark_data_smithi018_270558_object88983:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e288)", 2023-09-23T20:35:17.077 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:30:56.236163+0000", 2023-09-23T20:35:17.077 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 260.718402035, 2023-09-23T20:35:17.077 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.91627206999999999, 2023-09-23T20:35:17.077 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:35:17.077 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:35:17.077 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:35:17.077 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.45161", 2023-09-23T20:35:17.077 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3795614987", 2023-09-23T20:35:17.077 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 193722 2023-09-23T20:35:17.077 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.077 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:35:17.078 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.078 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:35:17.078 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:56.236163+0000", 2023-09-23T20:35:17.078 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:35:17.078 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.078 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.078 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:35:17.078 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:56.236163+0000", 2023-09-23T20:35:17.078 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:35:17.078 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.078 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.078 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:35:17.079 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:56.236161+0000", 2023-09-23T20:35:17.079 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:35:17.079 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.079 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.079 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:35:17.079 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:56.236170+0000", 2023-09-23T20:35:17.079 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.0819999999999999e-06 2023-09-23T20:35:17.079 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.079 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.079 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:35:17.079 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:56.236188+0000", 2023-09-23T20:35:17.079 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.8587e-05 2023-09-23T20:35:17.080 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.080 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.080 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:35:17.080 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:56.236196+0000", 2023-09-23T20:35:17.080 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.9929999999999995e-06 2023-09-23T20:35:17.080 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.080 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.080 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:35:17.080 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:56.236267+0000", 2023-09-23T20:35:17.080 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.0878999999999997e-05 2023-09-23T20:35:17.080 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.081 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.081 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:35:17.081 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:56.236279+0000", 2023-09-23T20:35:17.081 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.2215000000000001e-05 2023-09-23T20:35:17.081 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.081 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.081 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:35:17.081 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:57.124823+0000", 2023-09-23T20:35:17.081 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.88854340200000004 2023-09-23T20:35:17.081 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.081 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.081 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:35:17.082 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:57.124900+0000", 2023-09-23T20:35:17.082 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.7144000000000005e-05 2023-09-23T20:35:17.082 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.082 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.082 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 3,6", 2023-09-23T20:35:17.082 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:57.125132+0000", 2023-09-23T20:35:17.082 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000232232 2023-09-23T20:35:17.082 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.082 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.082 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:35:17.082 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:57.131669+0000", 2023-09-23T20:35:17.083 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0065368040000000002 2023-09-23T20:35:17.083 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.083 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.083 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:35:17.083 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:57.152154+0000", 2023-09-23T20:35:17.083 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.020485493 2023-09-23T20:35:17.083 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.083 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.083 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:35:17.083 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:57.152360+0000", 2023-09-23T20:35:17.083 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00020602399999999999 2023-09-23T20:35:17.083 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.084 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.084 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:35:17.084 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:57.152403+0000", 2023-09-23T20:35:17.084 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.2268000000000003e-05 2023-09-23T20:35:17.084 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.084 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.084 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:35:17.084 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:57.152435+0000", 2023-09-23T20:35:17.084 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.2604999999999997e-05 2023-09-23T20:35:17.084 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:35:17.084 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:35:17.085 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:35:17.085 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.085 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.085 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_repop(client.35700.0:125805 7.9 e340/322 7:97d108be:::benchmark_data_smithi018_308601_object31253:head v 340'7797, mlcod=318'4185)", 2023-09-23T20:35:17.085 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:34:12.963312+0000", 2023-09-23T20:35:17.085 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 63.991253301, 2023-09-23T20:35:17.085 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.37645642600000001, 2023-09-23T20:35:17.085 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:35:17.085 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:35:17.085 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:35:17.086 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.086 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:35:17.086 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:12.963312+0000", 2023-09-23T20:35:17.086 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:35:17.086 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.086 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.086 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:35:17.086 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:12.963312+0000", 2023-09-23T20:35:17.086 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:35:17.086 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.086 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.086 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:35:17.087 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:12.963310+0000", 2023-09-23T20:35:17.087 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:35:17.087 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.087 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.087 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:35:17.087 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:12.963341+0000", 2023-09-23T20:35:17.087 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.0323000000000001e-05 2023-09-23T20:35:17.087 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.087 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.087 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:35:17.087 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:12.963363+0000", 2023-09-23T20:35:17.087 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.2563000000000001e-05 2023-09-23T20:35:17.088 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.088 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.088 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:35:17.088 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:12.963374+0000", 2023-09-23T20:35:17.088 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.111e-05 2023-09-23T20:35:17.088 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.088 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.088 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:35:17.088 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:12.963470+0000", 2023-09-23T20:35:17.088 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.5302000000000003e-05 2023-09-23T20:35:17.088 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.088 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.089 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:35:17.089 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:12.963511+0000", 2023-09-23T20:35:17.089 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.1306000000000001e-05 2023-09-23T20:35:17.089 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.089 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.089 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:35:17.089 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:13.339739+0000", 2023-09-23T20:35:17.089 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.37622825100000001 2023-09-23T20:35:17.089 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.089 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.089 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:35:17.089 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:13.339768+0000", 2023-09-23T20:35:17.090 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.9235e-05 2023-09-23T20:35:17.090 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:35:17.090 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:35:17.090 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:35:17.090 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.090 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.090 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "MOSDPGPush(7.9 340/322 [PushOp(7:975f1708:::benchmark_data_smithi018_308601_object68059:head, version: 318'4196, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(7:975f1708:::benchmark_data_smithi018_308601_object68059:head@318'4196, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-09-23T20:35:17.090 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:34:12.966007+0000", 2023-09-23T20:35:17.090 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 63.988558320999999, 2023-09-23T20:35:17.090 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.36607237799999998, 2023-09-23T20:35:17.090 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:35:17.090 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "started", 2023-09-23T20:35:17.091 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:35:17.091 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.091 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:35:17.091 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:12.966007+0000", 2023-09-23T20:35:17.091 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:35:17.091 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.091 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.091 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:35:17.091 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:12.966007+0000", 2023-09-23T20:35:17.091 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:35:17.091 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.092 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.092 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:35:17.092 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:12.966006+0000", 2023-09-23T20:35:17.092 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:35:17.092 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.092 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.092 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:35:17.092 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:12.966038+0000", 2023-09-23T20:35:17.093 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.1473000000000002e-05 2023-09-23T20:35:17.093 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.093 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.093 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:35:17.093 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:13.325876+0000", 2023-09-23T20:35:17.093 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.35983845800000003 2023-09-23T20:35:17.093 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.093 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.093 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:35:17.093 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:13.325895+0000", 2023-09-23T20:35:17.093 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.9037000000000001e-05 2023-09-23T20:35:17.093 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.094 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.094 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:35:17.094 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:13.329519+0000", 2023-09-23T20:35:17.094 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0036242380000000001 2023-09-23T20:35:17.094 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.094 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.094 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:35:17.094 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:13.329528+0000", 2023-09-23T20:35:17.094 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.4859999999999997e-06 2023-09-23T20:35:17.094 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.095 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.095 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:35:17.095 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:13.332079+0000", 2023-09-23T20:35:17.095 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0025512830000000001 2023-09-23T20:35:17.095 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:35:17.095 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:35:17.095 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:35:17.095 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.095 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.096 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35700.0:158071 7.5 7:a4acb4dc:::benchmark_data_smithi018_308601_object63519:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e357)", 2023-09-23T20:35:17.096 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:35:02.774332+0000", 2023-09-23T20:35:17.096 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 14.180233558999999, 2023-09-23T20:35:17.096 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.041081909, 2023-09-23T20:35:17.096 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:35:17.096 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:35:17.096 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:35:17.096 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35700", 2023-09-23T20:35:17.097 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:35:17.097 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 158071 2023-09-23T20:35:17.097 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.097 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:35:17.097 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.097 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:35:17.097 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:35:02.774332+0000", 2023-09-23T20:35:17.097 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:35:17.098 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.098 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.098 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:35:17.098 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:35:02.774332+0000", 2023-09-23T20:35:17.098 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:35:17.098 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.098 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.098 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:35:17.099 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:35:02.774331+0000", 2023-09-23T20:35:17.099 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:35:17.099 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.099 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.099 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:35:17.099 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:35:02.774338+0000", 2023-09-23T20:35:17.099 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.0099999999999998e-06 2023-09-23T20:35:17.099 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.099 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.100 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:35:17.100 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:35:02.774350+0000", 2023-09-23T20:35:17.100 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1914e-05 2023-09-23T20:35:17.100 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.100 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.100 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:35:17.100 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:35:02.774356+0000", 2023-09-23T20:35:17.100 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.3960000000000003e-06 2023-09-23T20:35:17.100 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.100 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.101 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:35:17.101 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:35:02.776395+0000", 2023-09-23T20:35:17.101 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0020391659999999998 2023-09-23T20:35:17.101 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.101 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.101 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:35:17.101 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:35:02.776559+0000", 2023-09-23T20:35:17.102 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00016357599999999999 2023-09-23T20:35:17.102 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.102 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.102 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,2", 2023-09-23T20:35:17.102 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:35:02.776700+0000", 2023-09-23T20:35:17.102 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000140791 2023-09-23T20:35:17.102 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.102 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.102 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:35:17.102 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:35:02.779749+0000", 2023-09-23T20:35:17.103 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.003048785 2023-09-23T20:35:17.103 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.103 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.103 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:35:17.103 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:35:02.803230+0000", 2023-09-23T20:35:17.103 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.023481096 2023-09-23T20:35:17.103 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.103 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.103 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:35:17.104 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:35:02.815342+0000", 2023-09-23T20:35:17.104 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.012112075 2023-09-23T20:35:17.104 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.104 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.104 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:35:17.104 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:35:02.815388+0000", 2023-09-23T20:35:17.104 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.6008999999999999e-05 2023-09-23T20:35:17.104 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.105 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.105 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:35:17.105 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:35:02.815414+0000", 2023-09-23T20:35:17.105 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.5795e-05 2023-09-23T20:35:17.105 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:35:17.105 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:35:17.105 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:35:17.105 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.105 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.106 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35700.0:164751 7.5 7:a7ea69d6:::benchmark_data_smithi018_308601_object70199:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e358)", 2023-09-23T20:35:17.106 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:35:12.674360+0000", 2023-09-23T20:35:17.106 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 4.2802053730000003, 2023-09-23T20:35:17.106 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.049059140000000001, 2023-09-23T20:35:17.106 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:35:17.106 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:35:17.106 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:35:17.106 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35700", 2023-09-23T20:35:17.106 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:35:17.107 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 164751 2023-09-23T20:35:17.107 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.107 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:35:17.107 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.107 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:35:17.107 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:35:12.674360+0000", 2023-09-23T20:35:17.107 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:35:17.107 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.107 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.108 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:35:17.108 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:35:12.674360+0000", 2023-09-23T20:35:17.108 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:35:17.108 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.108 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.108 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:35:17.108 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:35:12.674357+0000", 2023-09-23T20:35:17.108 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999976 2023-09-23T20:35:17.108 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.108 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.108 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:35:17.108 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:35:12.674370+0000", 2023-09-23T20:35:17.109 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.2384999999999999e-05 2023-09-23T20:35:17.109 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.109 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.109 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:35:17.109 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:35:12.674394+0000", 2023-09-23T20:35:17.109 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.4199999999999999e-05 2023-09-23T20:35:17.109 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.109 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.109 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:35:17.109 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:35:12.674404+0000", 2023-09-23T20:35:17.109 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.7669999999999998e-06 2023-09-23T20:35:17.109 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.110 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.110 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:35:17.110 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:35:12.674459+0000", 2023-09-23T20:35:17.110 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.5167000000000002e-05 2023-09-23T20:35:17.110 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.110 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.110 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:35:17.110 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:35:12.674622+0000", 2023-09-23T20:35:17.110 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000163239 2023-09-23T20:35:17.110 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.110 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.110 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,2", 2023-09-23T20:35:17.111 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:35:12.674767+0000", 2023-09-23T20:35:17.111 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00014512699999999999 2023-09-23T20:35:17.111 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.111 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.111 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:35:17.111 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:35:12.678540+0000", 2023-09-23T20:35:17.111 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.003772999 2023-09-23T20:35:17.111 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.111 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.111 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:35:17.111 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:35:12.682286+0000", 2023-09-23T20:35:17.112 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0037460169999999999 2023-09-23T20:35:17.112 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.112 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.112 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:35:17.112 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:35:12.723314+0000", 2023-09-23T20:35:17.112 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.041027646000000001 2023-09-23T20:35:17.112 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.112 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.112 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:35:17.112 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:35:12.723377+0000", 2023-09-23T20:35:17.112 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.2885000000000005e-05 2023-09-23T20:35:17.113 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.113 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.113 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:35:17.113 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:35:12.723419+0000", 2023-09-23T20:35:17.113 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.206e-05 2023-09-23T20:35:17.113 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:35:17.113 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:35:17.113 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:35:17.113 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.113 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.113 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35700.0:164760 7.5 7:a3f5fa94:::benchmark_data_smithi018_308601_object70208:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e358)", 2023-09-23T20:35:17.113 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:35:12.678569+0000", 2023-09-23T20:35:17.114 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 4.275996009, 2023-09-23T20:35:17.114 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0500086, 2023-09-23T20:35:17.114 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:35:17.114 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:35:17.114 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:35:17.114 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35700", 2023-09-23T20:35:17.114 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:35:17.114 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 164760 2023-09-23T20:35:17.114 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.114 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:35:17.114 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.115 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:35:17.115 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:35:12.678569+0000", 2023-09-23T20:35:17.115 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:35:17.115 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.115 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.115 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:35:17.115 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:35:12.678569+0000", 2023-09-23T20:35:17.115 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:35:17.115 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.115 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.115 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:35:17.115 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:35:12.678568+0000", 2023-09-23T20:35:17.116 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:35:17.116 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.116 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.116 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:35:17.116 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:35:12.678575+0000", 2023-09-23T20:35:17.116 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.6620000000000005e-06 2023-09-23T20:35:17.116 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.116 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.116 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:35:17.116 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:35:12.678597+0000", 2023-09-23T20:35:17.117 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.2048e-05 2023-09-23T20:35:17.117 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.117 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.117 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:35:17.117 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:35:12.678604+0000", 2023-09-23T20:35:17.117 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.3600000000000001e-06 2023-09-23T20:35:17.117 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.117 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.117 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:35:17.117 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:35:12.678653+0000", 2023-09-23T20:35:17.118 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.9141999999999998e-05 2023-09-23T20:35:17.118 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.118 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.118 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:35:17.118 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:35:12.678782+0000", 2023-09-23T20:35:17.118 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000129493 2023-09-23T20:35:17.118 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.118 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.118 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,2", 2023-09-23T20:35:17.118 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:35:12.678912+0000", 2023-09-23T20:35:17.119 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00012976400000000001 2023-09-23T20:35:17.119 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.119 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.119 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:35:17.119 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:35:12.683106+0000", 2023-09-23T20:35:17.119 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0041943420000000002 2023-09-23T20:35:17.119 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.119 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.119 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:35:17.120 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:35:12.683600+0000", 2023-09-23T20:35:17.120 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000493244 2023-09-23T20:35:17.120 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.120 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.120 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:35:17.120 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:35:12.728468+0000", 2023-09-23T20:35:17.120 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.044868406 2023-09-23T20:35:17.120 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.120 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.121 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:35:17.121 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:35:12.728528+0000", 2023-09-23T20:35:17.121 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.9548e-05 2023-09-23T20:35:17.121 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.121 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.121 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:35:17.121 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:35:12.728578+0000", 2023-09-23T20:35:17.121 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.0148999999999997e-05 2023-09-23T20:35:17.121 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:35:17.121 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:35:17.122 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:35:17.122 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.122 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.122 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35700.0:165938 7.5 7:abc74fce:::benchmark_data_smithi018_308601_object71386:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e358)", 2023-09-23T20:35:17.122 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:35:14.526889+0000", 2023-09-23T20:35:17.122 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 2.4276762339999998, 2023-09-23T20:35:17.122 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.040748961, 2023-09-23T20:35:17.122 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:35:17.122 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:35:17.122 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:35:17.123 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35700", 2023-09-23T20:35:17.123 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:35:17.123 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 165938 2023-09-23T20:35:17.123 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.123 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:35:17.123 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.123 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:35:17.123 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:35:14.526889+0000", 2023-09-23T20:35:17.123 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:35:17.123 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.123 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.124 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:35:17.124 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:35:14.526889+0000", 2023-09-23T20:35:17.124 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:35:17.124 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.124 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.124 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:35:17.124 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:35:14.526887+0000", 2023-09-23T20:35:17.124 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999981 2023-09-23T20:35:17.124 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.124 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.124 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:35:17.124 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:35:14.526894+0000", 2023-09-23T20:35:17.125 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.3599999999999998e-06 2023-09-23T20:35:17.125 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.125 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.125 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:35:17.125 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:35:14.526910+0000", 2023-09-23T20:35:17.125 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.5607000000000002e-05 2023-09-23T20:35:17.125 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.125 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.125 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:35:17.125 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:35:14.526917+0000", 2023-09-23T20:35:17.126 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.1629999999999999e-06 2023-09-23T20:35:17.126 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.126 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.126 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:35:17.126 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:35:14.526975+0000", 2023-09-23T20:35:17.126 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.7652999999999998e-05 2023-09-23T20:35:17.126 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.126 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.126 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:35:17.127 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:35:14.527119+0000", 2023-09-23T20:35:17.127 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00014407299999999999 2023-09-23T20:35:17.127 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.127 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.127 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,2", 2023-09-23T20:35:17.127 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:35:14.527256+0000", 2023-09-23T20:35:17.127 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00013687500000000001 2023-09-23T20:35:17.127 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.127 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.128 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:35:17.128 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:35:14.530017+0000", 2023-09-23T20:35:17.128 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0027609890000000002 2023-09-23T20:35:17.128 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.128 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.128 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:35:17.128 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:35:14.540110+0000", 2023-09-23T20:35:17.128 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.010093421 2023-09-23T20:35:17.128 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.129 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.129 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:35:17.129 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:35:14.567546+0000", 2023-09-23T20:35:17.129 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.02743545 2023-09-23T20:35:17.129 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.129 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.129 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:35:17.129 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:35:14.567602+0000", 2023-09-23T20:35:17.129 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.6668e-05 2023-09-23T20:35:17.130 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.130 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.130 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:35:17.130 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:35:14.567638+0000", 2023-09-23T20:35:17.130 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.5701000000000003e-05 2023-09-23T20:35:17.130 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:35:17.131 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:35:17.131 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:35:17.131 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.131 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.131 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35700.0:165946 7.5 7:a56a910b:::benchmark_data_smithi018_308601_object71394:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e358)", 2023-09-23T20:35:17.131 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:35:14.541242+0000", 2023-09-23T20:35:17.131 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 2.4133227430000002, 2023-09-23T20:35:17.131 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.048167926999999999, 2023-09-23T20:35:17.131 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:35:17.132 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:35:17.132 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:35:17.132 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35700", 2023-09-23T20:35:17.132 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:35:17.132 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 165946 2023-09-23T20:35:17.132 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.132 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:35:17.133 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.133 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:35:17.133 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:35:14.541242+0000", 2023-09-23T20:35:17.133 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:35:17.133 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.133 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.133 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:35:17.133 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:35:14.541242+0000", 2023-09-23T20:35:17.133 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:35:17.133 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.133 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.134 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:35:17.134 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:35:14.541241+0000", 2023-09-23T20:35:17.134 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:35:17.134 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.134 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.134 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:35:17.134 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:35:14.541247+0000", 2023-09-23T20:35:17.134 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.8449999999999999e-06 2023-09-23T20:35:17.135 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.135 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.135 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:35:17.135 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:35:14.541265+0000", 2023-09-23T20:35:17.135 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.8116000000000001e-05 2023-09-23T20:35:17.135 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.135 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.135 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:35:17.135 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:35:14.541275+0000", 2023-09-23T20:35:17.135 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.3400000000000004e-06 2023-09-23T20:35:17.136 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.136 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.136 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:35:17.136 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:35:14.544524+0000", 2023-09-23T20:35:17.136 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0032494899999999998 2023-09-23T20:35:17.136 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.136 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.136 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:35:17.136 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:35:14.544612+0000", 2023-09-23T20:35:17.136 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.7606000000000001e-05 2023-09-23T20:35:17.137 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.137 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.137 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,2", 2023-09-23T20:35:17.137 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:35:14.544690+0000", 2023-09-23T20:35:17.137 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.8603999999999995e-05 2023-09-23T20:35:17.137 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.137 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.137 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:35:17.138 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:35:14.547630+0000", 2023-09-23T20:35:17.138 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0029397759999999998 2023-09-23T20:35:17.138 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.138 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.138 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:35:17.138 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:35:14.552239+0000", 2023-09-23T20:35:17.138 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0046087189999999998 2023-09-23T20:35:17.139 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.139 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.139 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:35:17.139 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:35:14.589320+0000", 2023-09-23T20:35:17.139 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.037081472999999997 2023-09-23T20:35:17.139 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.139 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.140 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:35:17.140 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:35:14.589371+0000", 2023-09-23T20:35:17.140 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.0701999999999997e-05 2023-09-23T20:35:17.140 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:35:17.140 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:35:17.140 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:35:17.141 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:35:14.589410+0000", 2023-09-23T20:35:17.141 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.9217000000000001e-05 2023-09-23T20:35:17.141 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:35:17.142 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:35:17.142 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:35:17.142 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:35:17.142 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:35:17.142 INFO:teuthology.orchestra.run.smithi149.stdout:} 2023-09-23T20:35:17.439 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T20:35:17.582 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:35:17.592 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:35:17.593 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:35:17.646 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:35:17.694 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:35:17.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:17 smithi149 ceph-mon[170394]: pgmap v849: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 116 active+clean; 1.4 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 664 op/s; 3509/69627 objects misplaced (5.040%); 2.2 MiB/s, 35 objects/s recovering 2023-09-23T20:35:17.749 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:35:17.794 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:35:17.798 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:35:17.798 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:35:17.896 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:17 smithi018 ceph-mon[184144]: pgmap v849: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 116 active+clean; 1.4 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 664 op/s; 3509/69627 objects misplaced (5.040%); 2.2 MiB/s, 35 objects/s recovering 2023-09-23T20:35:17.896 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:17 smithi018 ceph-mon[179647]: pgmap v849: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 116 active+clean; 1.4 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 664 op/s; 3509/69627 objects misplaced (5.040%); 2.2 MiB/s, 35 objects/s recovering 2023-09-23T20:35:17.900 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:35:17.941 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:35:17.998 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:35:18.006 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:35:18.006 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:35:18.107 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:35:18.204 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:35:18.509 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:18 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:35:18.531 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:35:18.540 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:35:18.540 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:35:18.641 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:35:18.763 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:18 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:35:18.763 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:35:18.793 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:35:18.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:18 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:35:19.354 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:35:19.359 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:35:19.360 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:35:19.461 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:35:19.558 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:35:19.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:19 smithi018 ceph-mon[179647]: pgmap v850: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 116 active+clean; 1.4 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 668 op/s; 3509/66996 objects misplaced (5.238%); 2.2 MiB/s, 35 objects/s recovering 2023-09-23T20:35:19.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:19 smithi018 ceph-mon[184144]: pgmap v850: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 116 active+clean; 1.4 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 668 op/s; 3509/66996 objects misplaced (5.238%); 2.2 MiB/s, 35 objects/s recovering 2023-09-23T20:35:19.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:19 smithi149 ceph-mon[170394]: pgmap v850: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 116 active+clean; 1.4 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 668 op/s; 3509/66996 objects misplaced (5.238%); 2.2 MiB/s, 35 objects/s recovering 2023-09-23T20:35:20.000 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:35:20.021 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:35:20.022 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:35:20.123 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:35:20.223 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:35:20.712 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:35:20.719 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:35:20.720 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:35:20.821 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:35:20.875 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:35:20.927 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:35:20.934 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:35:20.935 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:35:21.037 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:35:21.123 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:35:21.544 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:35:21.557 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:35:21.557 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:35:21.658 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:35:21.703 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:35:21.759 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:35:21.765 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:35:21.765 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:35:21.866 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:35:21.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:21 smithi018 ceph-mon[184144]: pgmap v851: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 116 active+clean; 1.3 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 671 op/s; 3293/64983 objects misplaced (5.067%); 2.2 MiB/s, 35 objects/s recovering 2023-09-23T20:35:21.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:21 smithi018 ceph-mon[179647]: pgmap v851: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 116 active+clean; 1.3 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 671 op/s; 3293/64983 objects misplaced (5.067%); 2.2 MiB/s, 35 objects/s recovering 2023-09-23T20:35:21.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:21 smithi149 ceph-mon[170394]: pgmap v851: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 116 active+clean; 1.3 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 671 op/s; 3293/64983 objects misplaced (5.067%); 2.2 MiB/s, 35 objects/s recovering 2023-09-23T20:35:21.976 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:35:22.076 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:35:22.087 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:35:22.088 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:35:22.189 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:35:22.314 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:35:22.572 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:22 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3843358376' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:35:22.573 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:22 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:35:22.573 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:22 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"}]: dispatch 2023-09-23T20:35:22.573 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:22 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3843358376' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:35:22.573 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:22 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:35:22.573 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:22 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"}]: dispatch 2023-09-23T20:35:22.627 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:35:22.631 INFO:teuthology.orchestra.run.smithi018.stderr:nodeep-scrub is set 2023-09-23T20:35:22.781 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:35:22.789 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:35:22.790 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:35:22.891 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:35:22.938 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:22 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3843358376' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:35:22.939 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:22 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:35:22.939 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:22 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"}]: dispatch 2023-09-23T20:35:22.949 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:35:23.010 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:35:23.017 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:35:23.017 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:35:23.118 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:35:23.255 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:35:23.571 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:35:23.585 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:35:23.586 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:35:23.687 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:35:23.775 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:35:23.849 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:23 smithi018 ceph-mon[179647]: pgmap v852: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 116 active+clean; 1.2 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 748 op/s; 2873/59031 objects misplaced (4.867%); 3.0 MiB/s, 48 objects/s recovering 2023-09-23T20:35:23.849 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:23 smithi018 ceph-mon[179647]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:35:23.849 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:23 smithi018 ceph-mon[179647]: failed to encode map e359 with expected crc 2023-09-23T20:35:23.849 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:23 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:35:23.850 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:23 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"}]': finished 2023-09-23T20:35:23.850 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:23 smithi018 ceph-mon[179647]: osdmap e359: 8 total, 8 up, 8 in 2023-09-23T20:35:23.850 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:23 smithi018 ceph-mon[184144]: pgmap v852: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 116 active+clean; 1.2 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 748 op/s; 2873/59031 objects misplaced (4.867%); 3.0 MiB/s, 48 objects/s recovering 2023-09-23T20:35:23.850 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:23 smithi018 ceph-mon[184144]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:35:23.850 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:23 smithi018 ceph-mon[184144]: failed to encode map e359 with expected crc 2023-09-23T20:35:23.850 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:23 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:35:23.850 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:23 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"}]': finished 2023-09-23T20:35:23.850 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:23 smithi018 ceph-mon[184144]: osdmap e359: 8 total, 8 up, 8 in 2023-09-23T20:35:23.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:23 smithi149 ceph-mon[170394]: pgmap v852: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 116 active+clean; 1.2 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 748 op/s; 2873/59031 objects misplaced (4.867%); 3.0 MiB/s, 48 objects/s recovering 2023-09-23T20:35:23.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:23 smithi149 ceph-mon[170394]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:35:23.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:23 smithi149 ceph-mon[170394]: failed to encode map e359 with expected crc 2023-09-23T20:35:23.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:23 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:35:23.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:23 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"}]': finished 2023-09-23T20:35:23.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:23 smithi149 ceph-mon[170394]: osdmap e359: 8 total, 8 up, 8 in 2023-09-23T20:35:24.246 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:35:24.282 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:35:24.288 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:35:24.289 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:35:24.299 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:35:24.363 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:35:24.391 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:35:24.415 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:35:24.460 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:35:24.464 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:35:24.506 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:35:24.563 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:35:24.629 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:35:24.666 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:35:24.667 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:35:24.667 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:35:24.667 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:35:24.667 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:35:24.667 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:35:24.667 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:35:24.667 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:35:24.703 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:35:24.711 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:35:24.712 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:35:24.813 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:35:24.842 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:35:24.843 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:24 smithi018 ceph-mon[179647]: osdmap e360: 8 total, 8 up, 8 in 2023-09-23T20:35:24.843 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:24 smithi018 ceph-mon[179647]: failed to encode map e360 with expected crc 2023-09-23T20:35:24.843 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:24 smithi018 ceph-mon[179647]: failed to encode map e360 with expected crc 2023-09-23T20:35:24.843 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:24 smithi018 ceph-mon[179647]: failed to encode map e360 with expected crc 2023-09-23T20:35:24.843 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:24 smithi018 ceph-mon[179647]: failed to encode map e360 with expected crc 2023-09-23T20:35:24.843 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:24 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:35:24.843 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:24 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:35:24.843 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:24 smithi018 ceph-mon[179647]: pgmap v855: 119 pgs: 2 remapped+peering, 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 114 active+clean; 1.2 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 2.6 KiB/s rd, 6.9 KiB/s wr, 546 op/s; 2873/56559 objects misplaced (5.080%); 1.6 MiB/s, 25 objects/s recovering 2023-09-23T20:35:24.844 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:24 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2138328450' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:35:24.849 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:24 smithi018 ceph-mon[184144]: osdmap e360: 8 total, 8 up, 8 in 2023-09-23T20:35:24.850 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:24 smithi018 ceph-mon[184144]: failed to encode map e360 with expected crc 2023-09-23T20:35:24.850 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:24 smithi018 ceph-mon[184144]: failed to encode map e360 with expected crc 2023-09-23T20:35:24.850 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:24 smithi018 ceph-mon[184144]: failed to encode map e360 with expected crc 2023-09-23T20:35:24.850 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:24 smithi018 ceph-mon[184144]: failed to encode map e360 with expected crc 2023-09-23T20:35:24.850 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:24 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:35:24.850 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:24 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:35:24.850 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:24 smithi018 ceph-mon[184144]: pgmap v855: 119 pgs: 2 remapped+peering, 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 114 active+clean; 1.2 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 2.6 KiB/s rd, 6.9 KiB/s wr, 546 op/s; 2873/56559 objects misplaced (5.080%); 1.6 MiB/s, 25 objects/s recovering 2023-09-23T20:35:24.850 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:24 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2138328450' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:35:24.870 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:35:24.870 INFO:teuthology.orchestra.run.smithi018.stdout:{"fsid":"a3b3b17e-5a4c-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub,nodeep-scrub flag(s) set","count":20},"muted":false},"PG_AVAILABILITY":{"severity":"HEALTH_WARN","summary":{"message":"Reduced data availability: 1 pg inactive, 2 pgs peering","count":3},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1001,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":361,"num_osds":8,"num_up_osds":8,"osd_up_since":1695500984,"num_in_osds":8,"osd_in_since":1695501152,"num_remapped_pgs":13},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":114},{"state_name":"active+remapped+backfill_wait","count":2},{"state_name":"remapped+peering","count":2},{"state_name":"active+recovering+undersized+remapped","count":1}],"num_pgs":119,"num_pools":5,"num_objects":18853,"data_bytes":1238261683,"bytes_used":7138209792,"bytes_avail":760822075392,"bytes_total":767960285184,"inactive_pgs_ratio":0.016806723549962044,"misplaced_objects":2873,"misplaced_total":56559,"misplaced_ratio":0.050796513375413285,"recovering_objects_per_sec":25,"recovering_bytes_per_sec":1651031,"recovering_keys_per_sec":0,"num_objects_recovered":201,"num_bytes_recovered":13212988,"num_keys_recovered":0,"read_bytes_sec":2687,"write_bytes_sec":7037,"read_op_per_sec":1,"write_op_per_sec":545},"fsmap":{"epoch":9,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi018.hduiol","status":"up:active","gid":24566}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.18:8443/","prometheus":"http://172.21.15.18:9283/"}},"servicemap":{"epoch":23,"modified":"2023-09-23T20:34:43.862209+0000","services":{}},"progress_events":{"09b28c4a-46d3-4634-bc08-b7c059c5a8c4":{"message":"Global Recovery Event (2m)\n [===========================.] (remaining: 3s)","progress":0.97478991746902466,"add_to_ceph_s":true},"8e7eff18-0e45-4130-a6e1-662b3902f0d2":{"message":"Upgrade to 18.2.0-460-gbb0daeff (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2023-09-23T20:35:24.885 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:35:24.890 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:35:24.891 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:35:24.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:24 smithi149 ceph-mon[170394]: osdmap e360: 8 total, 8 up, 8 in 2023-09-23T20:35:24.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:24 smithi149 ceph-mon[170394]: failed to encode map e360 with expected crc 2023-09-23T20:35:24.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:24 smithi149 ceph-mon[170394]: failed to encode map e360 with expected crc 2023-09-23T20:35:24.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:24 smithi149 ceph-mon[170394]: failed to encode map e360 with expected crc 2023-09-23T20:35:24.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:24 smithi149 ceph-mon[170394]: failed to encode map e360 with expected crc 2023-09-23T20:35:24.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:24 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:35:24.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:24 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:35:24.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:24 smithi149 ceph-mon[170394]: pgmap v855: 119 pgs: 2 remapped+peering, 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 114 active+clean; 1.2 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 2.6 KiB/s rd, 6.9 KiB/s wr, 546 op/s; 2873/56559 objects misplaced (5.080%); 1.6 MiB/s, 25 objects/s recovering 2023-09-23T20:35:24.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:24 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2138328450' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:35:24.991 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:35:25.029 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:35:25.084 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:35:25.089 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:35:25.089 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:35:25.142 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:35:24 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:35:24] "GET /metrics HTTP/1.1" 200 36537 "" "Prometheus/2.43.0" 2023-09-23T20:35:25.190 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:35:25.228 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:35:25.280 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:35:25.284 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:35:25.284 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:35:25.385 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:35:25.455 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:35:25.495 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-09-23T20:35:25.495 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:35:25.561 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:35:25.572 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:35:25.572 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:35:25.673 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:35:25.749 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:25 smithi018 ceph-mon[184144]: osdmap e361: 8 total, 8 up, 8 in 2023-09-23T20:35:25.749 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:25 smithi018 ceph-mon[184144]: failed to encode map e361 with expected crc 2023-09-23T20:35:25.749 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:25 smithi018 ceph-mon[184144]: failed to encode map e361 with expected crc 2023-09-23T20:35:25.750 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:25 smithi018 ceph-mon[184144]: failed to encode map e361 with expected crc 2023-09-23T20:35:25.750 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:25 smithi018 ceph-mon[184144]: failed to encode map e361 with expected crc 2023-09-23T20:35:25.750 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:25 smithi018 ceph-mon[184144]: failed to encode map e361 with expected crc 2023-09-23T20:35:25.750 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:25 smithi018 ceph-mon[184144]: failed to encode map e361 with expected crc 2023-09-23T20:35:25.750 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:25 smithi018 ceph-mon[184144]: Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2023-09-23T20:35:25.750 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:25 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2055957061' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T20:35:25.750 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:25 smithi018 ceph-mon[184144]: osdmap e362: 8 total, 8 up, 8 in 2023-09-23T20:35:25.750 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:25 smithi018 ceph-mon[184144]: failed to encode map e362 with expected crc 2023-09-23T20:35:25.750 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:25 smithi018 ceph-mon[179647]: osdmap e361: 8 total, 8 up, 8 in 2023-09-23T20:35:25.751 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:25 smithi018 ceph-mon[179647]: failed to encode map e361 with expected crc 2023-09-23T20:35:25.751 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:25 smithi018 ceph-mon[179647]: failed to encode map e361 with expected crc 2023-09-23T20:35:25.751 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:25 smithi018 ceph-mon[179647]: failed to encode map e361 with expected crc 2023-09-23T20:35:25.751 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:25 smithi018 ceph-mon[179647]: failed to encode map e361 with expected crc 2023-09-23T20:35:25.751 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:25 smithi018 ceph-mon[179647]: failed to encode map e361 with expected crc 2023-09-23T20:35:25.751 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:25 smithi018 ceph-mon[179647]: failed to encode map e361 with expected crc 2023-09-23T20:35:25.751 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:25 smithi018 ceph-mon[179647]: Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2023-09-23T20:35:25.751 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:25 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2055957061' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T20:35:25.751 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:25 smithi018 ceph-mon[179647]: osdmap e362: 8 total, 8 up, 8 in 2023-09-23T20:35:25.751 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:25 smithi018 ceph-mon[179647]: failed to encode map e362 with expected crc 2023-09-23T20:35:25.757 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T20:35:25.776 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:35:25.885 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:35:25.903 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:35:25.904 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:35:25.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:25 smithi149 ceph-mon[170394]: osdmap e361: 8 total, 8 up, 8 in 2023-09-23T20:35:25.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:25 smithi149 ceph-mon[170394]: failed to encode map e361 with expected crc 2023-09-23T20:35:25.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:25 smithi149 ceph-mon[170394]: failed to encode map e361 with expected crc 2023-09-23T20:35:25.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:25 smithi149 ceph-mon[170394]: failed to encode map e361 with expected crc 2023-09-23T20:35:25.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:25 smithi149 ceph-mon[170394]: failed to encode map e361 with expected crc 2023-09-23T20:35:25.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:25 smithi149 ceph-mon[170394]: failed to encode map e361 with expected crc 2023-09-23T20:35:25.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:25 smithi149 ceph-mon[170394]: failed to encode map e361 with expected crc 2023-09-23T20:35:25.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:25 smithi149 ceph-mon[170394]: Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2023-09-23T20:35:25.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:25 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2055957061' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T20:35:25.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:25 smithi149 ceph-mon[170394]: osdmap e362: 8 total, 8 up, 8 in 2023-09-23T20:35:25.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:25 smithi149 ceph-mon[170394]: failed to encode map e362 with expected crc 2023-09-23T20:35:26.004 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:35:26.036 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:35:26.046 INFO:teuthology.orchestra.run.smithi018.stdout:ERROR: (22) Invalid argument 2023-09-23T20:35:26.046 INFO:teuthology.orchestra.run.smithi018.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:35:26.072 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:35:26.075 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:35:26.076 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:35:26.176 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:35:26.286 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:35:26.381 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:35:26.391 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:35:26.391 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:35:26.492 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:35:26.564 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:35:26.674 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:35:26.747 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T20:35:26.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:26 smithi149 ceph-mon[170394]: failed to encode map e362 with expected crc 2023-09-23T20:35:26.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:26 smithi149 ceph-mon[170394]: pgmap v858: 119 pgs: 1 active+recovering+undersized+remapped, 3 peering, 2 remapped+peering, 3 active+undersized+remapped, 2 active+remapped+backfill_wait, 108 active+clean; 1.0 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 16 KiB/s rd, 41 KiB/s wr, 811 op/s; 2270/49443 objects misplaced (4.591%); 3.4 MiB/s, 53 objects/s recovering 2023-09-23T20:35:26.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:26 smithi149 ceph-mon[170394]: failed to encode map e363 with expected crc 2023-09-23T20:35:26.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:26 smithi149 ceph-mon[170394]: osdmap e363: 8 total, 8 up, 8 in 2023-09-23T20:35:26.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:26 smithi149 ceph-mon[170394]: failed to encode map e363 with expected crc 2023-09-23T20:35:26.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:26 smithi149 ceph-mon[170394]: failed to encode map e363 with expected crc 2023-09-23T20:35:27.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:26 smithi018 ceph-mon[179647]: failed to encode map e362 with expected crc 2023-09-23T20:35:27.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:26 smithi018 ceph-mon[179647]: pgmap v858: 119 pgs: 1 active+recovering+undersized+remapped, 3 peering, 2 remapped+peering, 3 active+undersized+remapped, 2 active+remapped+backfill_wait, 108 active+clean; 1.0 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 16 KiB/s rd, 41 KiB/s wr, 811 op/s; 2270/49443 objects misplaced (4.591%); 3.4 MiB/s, 53 objects/s recovering 2023-09-23T20:35:27.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:26 smithi018 ceph-mon[179647]: failed to encode map e363 with expected crc 2023-09-23T20:35:27.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:26 smithi018 ceph-mon[179647]: osdmap e363: 8 total, 8 up, 8 in 2023-09-23T20:35:27.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:26 smithi018 ceph-mon[179647]: failed to encode map e363 with expected crc 2023-09-23T20:35:27.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:26 smithi018 ceph-mon[179647]: failed to encode map e363 with expected crc 2023-09-23T20:35:27.144 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:26 smithi018 ceph-mon[184144]: failed to encode map e362 with expected crc 2023-09-23T20:35:27.144 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:26 smithi018 ceph-mon[184144]: pgmap v858: 119 pgs: 1 active+recovering+undersized+remapped, 3 peering, 2 remapped+peering, 3 active+undersized+remapped, 2 active+remapped+backfill_wait, 108 active+clean; 1.0 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 16 KiB/s rd, 41 KiB/s wr, 811 op/s; 2270/49443 objects misplaced (4.591%); 3.4 MiB/s, 53 objects/s recovering 2023-09-23T20:35:27.144 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:26 smithi018 ceph-mon[184144]: failed to encode map e363 with expected crc 2023-09-23T20:35:27.144 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:26 smithi018 ceph-mon[184144]: osdmap e363: 8 total, 8 up, 8 in 2023-09-23T20:35:27.144 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:26 smithi018 ceph-mon[184144]: failed to encode map e363 with expected crc 2023-09-23T20:35:27.144 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:26 smithi018 ceph-mon[184144]: failed to encode map e363 with expected crc 2023-09-23T20:35:27.512 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:35:27.557 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:35:27.560 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:35:27.560 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:35:27.661 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:35:27.813 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:27 smithi018 ceph-mon[184144]: failed to encode map e363 with expected crc 2023-09-23T20:35:27.813 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:27 smithi018 ceph-mon[184144]: failed to encode map e363 with expected crc 2023-09-23T20:35:27.813 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:27 smithi018 ceph-mon[184144]: failed to encode map e363 with expected crc 2023-09-23T20:35:27.813 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:27 smithi018 ceph-mon[184144]: failed to encode map e363 with expected crc 2023-09-23T20:35:27.813 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:27 smithi018 ceph-mon[184144]: failed to encode map e363 with expected crc 2023-09-23T20:35:27.814 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:35:27.815 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:27 smithi018 ceph-mon[179647]: failed to encode map e363 with expected crc 2023-09-23T20:35:27.815 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:27 smithi018 ceph-mon[179647]: failed to encode map e363 with expected crc 2023-09-23T20:35:27.815 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:27 smithi018 ceph-mon[179647]: failed to encode map e363 with expected crc 2023-09-23T20:35:27.815 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:27 smithi018 ceph-mon[179647]: failed to encode map e363 with expected crc 2023-09-23T20:35:27.815 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:27 smithi018 ceph-mon[179647]: failed to encode map e363 with expected crc 2023-09-23T20:35:27.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:27 smithi149 ceph-mon[170394]: failed to encode map e363 with expected crc 2023-09-23T20:35:27.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:27 smithi149 ceph-mon[170394]: failed to encode map e363 with expected crc 2023-09-23T20:35:27.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:27 smithi149 ceph-mon[170394]: failed to encode map e363 with expected crc 2023-09-23T20:35:27.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:27 smithi149 ceph-mon[170394]: failed to encode map e363 with expected crc 2023-09-23T20:35:27.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:27 smithi149 ceph-mon[170394]: failed to encode map e363 with expected crc 2023-09-23T20:35:28.399 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:35:28.410 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:35:28.411 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:35:28.511 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:35:28.649 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:35:28.731 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:28 smithi018 ceph-mon[179647]: pgmap v860: 119 pgs: 1 active+recovering+undersized+remapped, 3 peering, 2 remapped+peering, 3 active+undersized+remapped, 2 active+remapped+backfill_wait, 108 active+clean; 987 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 15 KiB/s rd, 38 KiB/s wr, 732 op/s; 2270/47262 objects misplaced (4.803%); 3.2 MiB/s, 50 objects/s recovering 2023-09-23T20:35:29.085 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:28 smithi018 ceph-mon[184144]: pgmap v860: 119 pgs: 1 active+recovering+undersized+remapped, 3 peering, 2 remapped+peering, 3 active+undersized+remapped, 2 active+remapped+backfill_wait, 108 active+clean; 987 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 15 KiB/s rd, 38 KiB/s wr, 732 op/s; 2270/47262 objects misplaced (4.803%); 3.2 MiB/s, 50 objects/s recovering 2023-09-23T20:35:29.148 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:35:29.160 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:35:29.160 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:35:29.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:28 smithi149 ceph-mon[170394]: pgmap v860: 119 pgs: 1 active+recovering+undersized+remapped, 3 peering, 2 remapped+peering, 3 active+undersized+remapped, 2 active+remapped+backfill_wait, 108 active+clean; 987 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 15 KiB/s rd, 38 KiB/s wr, 732 op/s; 2270/47262 objects misplaced (4.803%); 3.2 MiB/s, 50 objects/s recovering 2023-09-23T20:35:29.260 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:35:29.351 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:35:29.425 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:35:29.435 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:35:29.436 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:35:29.536 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:35:29.614 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:35:29.762 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:35:29.768 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:35:29.769 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:35:29.869 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:35:30.049 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:35:30.250 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:35:30.254 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:35:30.263 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:35:30.264 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:35:30.365 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:35:30.376 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:35:30.455 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:35:30.459 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:35:30.626 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:35:30.643 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:35:30.643 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:30 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:35:30.644 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:30 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:35:30.644 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:35:30.646 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:30 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:35:30.646 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:30 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:35:30.745 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:35:30.779 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:30 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:35:30.779 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:30 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:35:30.782 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:35:30.834 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:35:30.844 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:35:30.844 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:35:30.945 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:35:30.980 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:35:31.029 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:35:31.033 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:35:31.033 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:35:31.134 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:35:31.255 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:35:31.342 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:35:31.356 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:35:31.356 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:35:31.457 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:35:31.507 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:31 smithi018 ceph-mon[179647]: pgmap v861: 119 pgs: 1 active+recovering+undersized+remapped, 3 peering, 1 remapped+peering, 2 active+undersized+remapped, 2 active+remapped+backfill_wait, 110 active+clean; 952 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 7.0 KiB/s rd, 18 KiB/s wr, 616 op/s; 2068/45564 objects misplaced (4.539%); 2.3 MiB/s, 35 objects/s recovering 2023-09-23T20:35:31.507 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:31 smithi018 ceph-mon[179647]: Health check update: Reduced data availability: 1 pg inactive, 3 pgs peering (PG_AVAILABILITY) 2023-09-23T20:35:31.538 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:35:31.821 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:31 smithi018 ceph-mon[184144]: pgmap v861: 119 pgs: 1 active+recovering+undersized+remapped, 3 peering, 1 remapped+peering, 2 active+undersized+remapped, 2 active+remapped+backfill_wait, 110 active+clean; 952 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 7.0 KiB/s rd, 18 KiB/s wr, 616 op/s; 2068/45564 objects misplaced (4.539%); 2.3 MiB/s, 35 objects/s recovering 2023-09-23T20:35:31.821 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:31 smithi018 ceph-mon[184144]: Health check update: Reduced data availability: 1 pg inactive, 3 pgs peering (PG_AVAILABILITY) 2023-09-23T20:35:31.973 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:31 smithi149 ceph-mon[170394]: pgmap v861: 119 pgs: 1 active+recovering+undersized+remapped, 3 peering, 1 remapped+peering, 2 active+undersized+remapped, 2 active+remapped+backfill_wait, 110 active+clean; 952 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 7.0 KiB/s rd, 18 KiB/s wr, 616 op/s; 2068/45564 objects misplaced (4.539%); 2.3 MiB/s, 35 objects/s recovering 2023-09-23T20:35:31.973 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:31 smithi149 ceph-mon[170394]: Health check update: Reduced data availability: 1 pg inactive, 3 pgs peering (PG_AVAILABILITY) 2023-09-23T20:35:32.131 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:35:32.142 INFO:teuthology.orchestra.run.smithi018.stderr:dumped all 2023-09-23T20:35:32.329 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:35:32.338 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:35:32.338 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:35:32.439 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:35:32.476 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:35:32.525 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:35:32.528 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:35:32.529 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:35:32.559 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:32 smithi018 ceph-mon[184144]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 3 pgs peering) 2023-09-23T20:35:32.559 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:32 smithi018 ceph-mon[179647]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 3 pgs peering) 2023-09-23T20:35:32.630 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:35:32.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:32 smithi149 ceph-mon[170394]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 3 pgs peering) 2023-09-23T20:35:32.988 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:35:33.190 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:35:33.196 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:35:33.197 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:35:33.297 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:35:33.389 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:35:33.646 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:33 smithi018 ceph-mon[179647]: pgmap v862: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 116 active+clean; 837 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 14 KiB/s rd, 38 KiB/s wr, 684 op/s; 1654/40047 objects misplaced (4.130%); 3.6 MiB/s, 59 objects/s recovering 2023-09-23T20:35:33.646 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:33 smithi018 ceph-mon[179647]: from='client.35919 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:35:33.646 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:33 smithi018 ceph-mon[184144]: pgmap v862: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 116 active+clean; 837 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 14 KiB/s rd, 38 KiB/s wr, 684 op/s; 1654/40047 objects misplaced (4.130%); 3.6 MiB/s, 59 objects/s recovering 2023-09-23T20:35:33.646 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:33 smithi018 ceph-mon[184144]: from='client.35919 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:35:33.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:33 smithi149 ceph-mon[170394]: pgmap v862: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 116 active+clean; 837 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 14 KiB/s rd, 38 KiB/s wr, 684 op/s; 1654/40047 objects misplaced (4.130%); 3.6 MiB/s, 59 objects/s recovering 2023-09-23T20:35:33.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:33 smithi149 ceph-mon[170394]: from='client.35919 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:35:35.141 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:35:34 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:35:34] "GET /metrics HTTP/1.1" 200 36540 "" "Prometheus/2.43.0" 2023-09-23T20:35:35.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:35 smithi018 ceph-mon[184144]: pgmap v863: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 116 active+clean; 787 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 13 KiB/s rd, 33 KiB/s wr, 678 op/s; 1654/37635 objects misplaced (4.395%); 3.2 MiB/s, 52 objects/s recovering 2023-09-23T20:35:35.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:35 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:35:35.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:35 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:35:35.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:35 smithi018 ceph-mon[179647]: pgmap v863: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 116 active+clean; 787 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 13 KiB/s rd, 33 KiB/s wr, 678 op/s; 1654/37635 objects misplaced (4.395%); 3.2 MiB/s, 52 objects/s recovering 2023-09-23T20:35:35.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:35 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:35:35.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:35 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:35:35.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:35 smithi149 ceph-mon[170394]: pgmap v863: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 116 active+clean; 787 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 13 KiB/s rd, 33 KiB/s wr, 678 op/s; 1654/37635 objects misplaced (4.395%); 3.2 MiB/s, 52 objects/s recovering 2023-09-23T20:35:35.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:35 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:35:35.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:35 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:35:37.355 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:35:37.364 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:35:37.365 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:35:37.465 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:35:37.536 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:35:37.645 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:37 smithi018 ceph-mon[184144]: pgmap v864: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 116 active+clean; 625 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 8.4 KiB/s rd, 22 KiB/s wr, 658 op/s; 1235/29874 objects misplaced (4.134%); 1.3 MiB/s, 48 objects/s recovering 2023-09-23T20:35:37.645 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:37 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:35:37.645 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:37 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:35:37.645 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:37 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:35:37.646 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:37 smithi018 ceph-mon[179647]: pgmap v864: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 116 active+clean; 625 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 8.4 KiB/s rd, 22 KiB/s wr, 658 op/s; 1235/29874 objects misplaced (4.134%); 1.3 MiB/s, 48 objects/s recovering 2023-09-23T20:35:37.646 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:37 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:35:37.646 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:37 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:35:37.647 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:37 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:35:37.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:37 smithi149 ceph-mon[170394]: pgmap v864: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 116 active+clean; 625 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 8.4 KiB/s rd, 22 KiB/s wr, 658 op/s; 1235/29874 objects misplaced (4.134%); 1.3 MiB/s, 48 objects/s recovering 2023-09-23T20:35:37.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:37 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:35:37.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:37 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:35:37.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:37 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:35:38.364 INFO:teuthology.orchestra.run.smithi018.stdout:{"pg_ready":true,"pg_map":{"version":862,"stamp":"2023-09-23T20:35:31.899542+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":877632045,"num_objects":13349,"num_object_clones":0,"num_object_copies":40047,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1654,"num_objects_unfound":0,"num_objects_dirty":13349,"num_whiteouts":0,"num_read":6096,"num_read_kb":13402,"num_write":176737,"num_write_kb":6064357,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11024,"num_bytes_recovered":657040096,"num_keys_recovered":40,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":116944,"ondisk_log_size":116944,"up":357,"acting":356,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":367,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":5935272,"kb_used_data":3089256,"kb_used_omap":32,"kb_used_meta":2845727,"kb_avail":744025944,"statfs":{"total":767960285184,"available":761882566656,"internally_reserved":0,"allocated":3163398144,"data_stored":3151369613,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32773,"internal_metadata":2914025467},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":11,"apply_latency_ms":11,"commit_latency_ns":11000000,"apply_latency_ns":11000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-321780208,"num_objects":-4911,"num_object_clones":0,"num_object_copies":-14733,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-1033,"num_objects_unfound":0,"num_objects_dirty":-4911,"num_whiteouts":0,"num_read":56,"num_read_kb":105,"num_write":4939,"num_write_kb":275,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":436,"num_bytes_recovered":27857998,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"7.296665"},"pg_stats":[{"pgid":"1.18","version":"90'161","reported_seq":637,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T20:35:26.644897+0000","last_change":"2023-09-23T20:35:26.644897+0000","last_active":"2023-09-23T20:35:26.644897+0000","last_peered":"2023-09-23T20:35:26.644897+0000","last_clean":"2023-09-23T20:35:26.644897+0000","last_became_active":"2023-09-23T20:35:26.644564+0000","last_became_peered":"2023-09-23T20:35:26.644564+0000","last_unstale":"2023-09-23T20:35:26.644897+0000","last_undegraded":"2023-09-23T20:35:26.644897+0000","last_fullsized":"2023-09-23T20:35:26.644897+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:35:03.551143+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:35:03.551143+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:55:55.872961+0000","scrub_duration":0.033122453000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"69'1","reported_seq":60,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-23T20:35:25.621251+0000","last_change":"2023-09-23T20:32:33.233853+0000","last_active":"2023-09-23T20:35:25.621251+0000","last_peered":"2023-09-23T20:35:25.621251+0000","last_clean":"2023-09-23T20:35:25.621251+0000","last_became_active":"2023-09-23T20:32:33.233546+0000","last_became_peered":"2023-09-23T20:32:33.233546+0000","last_unstale":"2023-09-23T20:35:25.621251+0000","last_undegraded":"2023-09-23T20:35:25.621251+0000","last_fullsized":"2023-09-23T20:35:25.621251+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:50:27.130808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":242,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-23T20:35:25.647775+0000","last_change":"2023-09-23T20:32:50.536495+0000","last_active":"2023-09-23T20:35:25.647775+0000","last_peered":"2023-09-23T20:35:25.647775+0000","last_clean":"2023-09-23T20:35:25.647775+0000","last_became_active":"2023-09-23T20:32:50.536180+0000","last_became_peered":"2023-09-23T20:32:50.536180+0000","last_unstale":"2023-09-23T20:35:25.647775+0000","last_undegraded":"2023-09-23T20:35:25.647775+0000","last_fullsized":"2023-09-23T20:35:25.647775+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:01:05.524308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1b","version":"90'161","reported_seq":491,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T20:35:26.647597+0000","last_change":"2023-09-23T20:35:26.647597+0000","last_active":"2023-09-23T20:35:26.647597+0000","last_peered":"2023-09-23T20:35:26.647597+0000","last_clean":"2023-09-23T20:35:26.647597+0000","last_became_active":"2023-09-23T20:35:26.647261+0000","last_became_peered":"2023-09-23T20:35:26.647261+0000","last_unstale":"2023-09-23T20:35:26.647597+0000","last_undegraded":"2023-09-23T20:35:26.647597+0000","last_fullsized":"2023-09-23T20:35:26.647597+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:46.558292+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:46.558292+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0019500940000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":329,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-23T20:35:25.642611+0000","last_change":"2023-09-23T20:34:59.983035+0000","last_active":"2023-09-23T20:35:25.642611+0000","last_peered":"2023-09-23T20:35:25.642611+0000","last_clean":"2023-09-23T20:35:25.642611+0000","last_became_active":"2023-09-23T20:34:59.982459+0000","last_became_peered":"2023-09-23T20:34:59.982459+0000","last_unstale":"2023-09-23T20:35:25.642611+0000","last_undegraded":"2023-09-23T20:35:25.642611+0000","last_fullsized":"2023-09-23T20:35:25.642611+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:16:48.510783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":330,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-23T20:35:25.629626+0000","last_change":"2023-09-23T20:25:02.957185+0000","last_active":"2023-09-23T20:35:25.629626+0000","last_peered":"2023-09-23T20:35:25.629626+0000","last_clean":"2023-09-23T20:35:25.629626+0000","last_became_active":"2023-09-23T20:25:02.956689+0000","last_became_peered":"2023-09-23T20:25:02.956689+0000","last_unstale":"2023-09-23T20:35:25.629626+0000","last_undegraded":"2023-09-23T20:35:25.629626+0000","last_fullsized":"2023-09-23T20:35:25.629626+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:04:28.524015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"90'161","reported_seq":432,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T20:35:26.640363+0000","last_change":"2023-09-23T20:35:26.640363+0000","last_active":"2023-09-23T20:35:26.640363+0000","last_peered":"2023-09-23T20:35:26.640363+0000","last_clean":"2023-09-23T20:35:26.640363+0000","last_became_active":"2023-09-23T20:35:26.639912+0000","last_became_peered":"2023-09-23T20:35:26.639912+0000","last_unstale":"2023-09-23T20:35:26.640363+0000","last_undegraded":"2023-09-23T20:35:26.640363+0000","last_fullsized":"2023-09-23T20:35:26.640363+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:35:06.490149+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:35:06.490149+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":175,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-23T20:35:25.627256+0000","last_change":"2023-09-23T20:32:50.546428+0000","last_active":"2023-09-23T20:35:25.627256+0000","last_peered":"2023-09-23T20:35:25.627256+0000","last_clean":"2023-09-23T20:35:25.627256+0000","last_became_active":"2023-09-23T20:32:50.546120+0000","last_became_peered":"2023-09-23T20:32:50.546120+0000","last_unstale":"2023-09-23T20:35:25.627256+0000","last_undegraded":"2023-09-23T20:35:25.627256+0000","last_fullsized":"2023-09-23T20:35:25.627256+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:00:10.792410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":136,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-23T20:35:25.624904+0000","last_change":"2023-09-23T20:29:46.112977+0000","last_active":"2023-09-23T20:35:25.624904+0000","last_peered":"2023-09-23T20:35:25.624904+0000","last_clean":"2023-09-23T20:35:25.624904+0000","last_became_active":"2023-09-23T20:29:46.112600+0000","last_became_peered":"2023-09-23T20:29:46.112600+0000","last_unstale":"2023-09-23T20:35:25.624904+0000","last_undegraded":"2023-09-23T20:35:25.624904+0000","last_fullsized":"2023-09-23T20:35:25.624904+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:06:34.132247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"90'161","reported_seq":516,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T20:35:26.642558+0000","last_change":"2023-09-23T20:35:26.642558+0000","last_active":"2023-09-23T20:35:26.642558+0000","last_peered":"2023-09-23T20:35:26.642558+0000","last_clean":"2023-09-23T20:35:26.642558+0000","last_became_active":"2023-09-23T20:35:26.642130+0000","last_became_peered":"2023-09-23T20:35:26.642130+0000","last_unstale":"2023-09-23T20:35:26.642558+0000","last_undegraded":"2023-09-23T20:35:26.642558+0000","last_fullsized":"2023-09-23T20:35:26.642558+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:44.509012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:44.509012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0032186319999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":328,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-23T20:35:25.628696+0000","last_change":"2023-09-23T20:25:02.969432+0000","last_active":"2023-09-23T20:35:25.628696+0000","last_peered":"2023-09-23T20:35:25.628696+0000","last_clean":"2023-09-23T20:35:25.628696+0000","last_became_active":"2023-09-23T20:25:02.969118+0000","last_became_peered":"2023-09-23T20:25:02.969118+0000","last_unstale":"2023-09-23T20:35:25.628696+0000","last_undegraded":"2023-09-23T20:35:25.628696+0000","last_fullsized":"2023-09-23T20:35:25.628696+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:02:22.453831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":66,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-23T20:35:25.621320+0000","last_change":"2023-09-23T20:32:01.360359+0000","last_active":"2023-09-23T20:35:25.621320+0000","last_peered":"2023-09-23T20:35:25.621320+0000","last_clean":"2023-09-23T20:35:25.621320+0000","last_became_active":"2023-09-23T20:32:01.360008+0000","last_became_peered":"2023-09-23T20:32:01.360008+0000","last_unstale":"2023-09-23T20:35:25.621320+0000","last_undegraded":"2023-09-23T20:35:25.621320+0000","last_fullsized":"2023-09-23T20:35:25.621320+0000","mapping_epoch":308,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:33:43.135001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.14","version":"90'161","reported_seq":690,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-23T20:35:25.621950+0000","last_change":"2023-09-23T20:35:02.236398+0000","last_active":"2023-09-23T20:35:25.621950+0000","last_peered":"2023-09-23T20:35:25.621950+0000","last_clean":"2023-09-23T20:35:25.621950+0000","last_became_active":"2023-09-23T20:35:02.235988+0000","last_became_peered":"2023-09-23T20:35:02.235988+0000","last_unstale":"2023-09-23T20:35:25.621950+0000","last_undegraded":"2023-09-23T20:35:25.621950+0000","last_fullsized":"2023-09-23T20:35:25.621950+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:47.479012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:47.479012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:13.519648+0000","scrub_duration":0.0020136590000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":339,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-23T20:35:25.625259+0000","last_change":"2023-09-23T20:29:45.523808+0000","last_active":"2023-09-23T20:35:25.625259+0000","last_peered":"2023-09-23T20:35:25.625259+0000","last_clean":"2023-09-23T20:35:25.625259+0000","last_became_active":"2023-09-23T20:29:45.523508+0000","last_became_peered":"2023-09-23T20:29:45.523508+0000","last_unstale":"2023-09-23T20:35:25.625259+0000","last_undegraded":"2023-09-23T20:35:25.625259+0000","last_fullsized":"2023-09-23T20:35:25.625259+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:50:55.565352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":162,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-23T20:35:25.634155+0000","last_change":"2023-09-23T20:32:50.539003+0000","last_active":"2023-09-23T20:35:25.634155+0000","last_peered":"2023-09-23T20:35:25.634155+0000","last_clean":"2023-09-23T20:35:25.634155+0000","last_became_active":"2023-09-23T20:32:50.538596+0000","last_became_peered":"2023-09-23T20:32:50.538596+0000","last_unstale":"2023-09-23T20:35:25.634155+0000","last_undegraded":"2023-09-23T20:35:25.634155+0000","last_fullsized":"2023-09-23T20:35:25.634155+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:56:03.314375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.17","version":"90'161","reported_seq":482,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T20:35:26.653008+0000","last_change":"2023-09-23T20:35:26.653008+0000","last_active":"2023-09-23T20:35:26.653008+0000","last_peered":"2023-09-23T20:35:26.653008+0000","last_clean":"2023-09-23T20:35:26.653008+0000","last_became_active":"2023-09-23T20:35:26.652669+0000","last_became_peered":"2023-09-23T20:35:26.652669+0000","last_unstale":"2023-09-23T20:35:26.653008+0000","last_undegraded":"2023-09-23T20:35:26.653008+0000","last_fullsized":"2023-09-23T20:35:26.653008+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:45.196285+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:45.196285+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":390,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-23T20:35:25.632293+0000","last_change":"2023-09-23T20:32:33.270617+0000","last_active":"2023-09-23T20:35:25.632293+0000","last_peered":"2023-09-23T20:35:25.632293+0000","last_clean":"2023-09-23T20:35:25.632293+0000","last_became_active":"2023-09-23T20:32:33.227902+0000","last_became_peered":"2023-09-23T20:32:33.227902+0000","last_unstale":"2023-09-23T20:35:25.632293+0000","last_undegraded":"2023-09-23T20:35:25.632293+0000","last_fullsized":"2023-09-23T20:35:25.632293+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:30:26.847831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":360,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":312,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-23T20:35:25.645325+0000","last_change":"2023-09-23T20:29:46.080766+0000","last_active":"2023-09-23T20:35:25.645325+0000","last_peered":"2023-09-23T20:35:25.645325+0000","last_clean":"2023-09-23T20:35:25.645325+0000","last_became_active":"2023-09-23T20:29:46.080356+0000","last_became_peered":"2023-09-23T20:29:46.080356+0000","last_unstale":"2023-09-23T20:35:25.645325+0000","last_undegraded":"2023-09-23T20:35:25.645325+0000","last_fullsized":"2023-09-23T20:35:25.645325+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:08:01.862270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"90'161","reported_seq":473,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T20:35:26.652066+0000","last_change":"2023-09-23T20:35:26.645192+0000","last_active":"2023-09-23T20:35:26.652066+0000","last_peered":"2023-09-23T20:35:26.652066+0000","last_clean":"2023-09-23T20:35:26.652066+0000","last_became_active":"2023-09-23T20:35:26.644787+0000","last_became_peered":"2023-09-23T20:35:26.644787+0000","last_unstale":"2023-09-23T20:35:26.652066+0000","last_undegraded":"2023-09-23T20:35:26.652066+0000","last_fullsized":"2023-09-23T20:35:26.652066+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:44.211184+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:44.211184+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":70,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-23T20:35:25.621969+0000","last_change":"2023-09-23T20:32:33.266787+0000","last_active":"2023-09-23T20:35:25.621969+0000","last_peered":"2023-09-23T20:35:25.621969+0000","last_clean":"2023-09-23T20:35:25.621969+0000","last_became_active":"2023-09-23T20:32:33.228482+0000","last_became_peered":"2023-09-23T20:32:33.228482+0000","last_unstale":"2023-09-23T20:35:25.621969+0000","last_undegraded":"2023-09-23T20:35:25.621969+0000","last_fullsized":"2023-09-23T20:35:25.621969+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:13:01.812073+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":60,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-23T20:35:25.621905+0000","last_change":"2023-09-23T20:32:33.228929+0000","last_active":"2023-09-23T20:35:25.621905+0000","last_peered":"2023-09-23T20:35:25.621905+0000","last_clean":"2023-09-23T20:35:25.621905+0000","last_became_active":"2023-09-23T20:32:33.228595+0000","last_became_peered":"2023-09-23T20:32:33.228595+0000","last_unstale":"2023-09-23T20:35:25.621905+0000","last_undegraded":"2023-09-23T20:35:25.621905+0000","last_fullsized":"2023-09-23T20:35:25.621905+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:17:53.053808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"90'161","reported_seq":593,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-23T20:35:25.629114+0000","last_change":"2023-09-23T20:31:43.578810+0000","last_active":"2023-09-23T20:35:25.629114+0000","last_peered":"2023-09-23T20:35:25.629114+0000","last_clean":"2023-09-23T20:35:25.629114+0000","last_became_active":"2023-09-23T20:28:05.991304+0000","last_became_peered":"2023-09-23T20:28:05.991304+0000","last_unstale":"2023-09-23T20:35:25.629114+0000","last_undegraded":"2023-09-23T20:35:25.629114+0000","last_fullsized":"2023-09-23T20:35:25.629114+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:43.578666+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:43.578666+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:46:18.999991+0000","scrub_duration":0.0021890379999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":389,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-23T20:35:25.630992+0000","last_change":"2023-09-23T20:32:33.242943+0000","last_active":"2023-09-23T20:35:25.630992+0000","last_peered":"2023-09-23T20:35:25.630992+0000","last_clean":"2023-09-23T20:35:25.630992+0000","last_became_active":"2023-09-23T20:32:33.225468+0000","last_became_peered":"2023-09-23T20:32:33.225468+0000","last_unstale":"2023-09-23T20:35:25.630992+0000","last_undegraded":"2023-09-23T20:35:25.630992+0000","last_fullsized":"2023-09-23T20:35:25.630992+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:40:47.867573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":64,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-23T20:35:25.623934+0000","last_change":"2023-09-23T20:32:01.358590+0000","last_active":"2023-09-23T20:35:25.623934+0000","last_peered":"2023-09-23T20:35:25.623934+0000","last_clean":"2023-09-23T20:35:25.623934+0000","last_became_active":"2023-09-23T20:32:01.358345+0000","last_became_peered":"2023-09-23T20:32:01.358345+0000","last_unstale":"2023-09-23T20:35:25.623934+0000","last_undegraded":"2023-09-23T20:35:25.623934+0000","last_fullsized":"2023-09-23T20:35:25.623934+0000","mapping_epoch":308,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:22:21.159708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.10","version":"90'161","reported_seq":593,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-23T20:35:25.626275+0000","last_change":"2023-09-23T20:31:49.726214+0000","last_active":"2023-09-23T20:35:25.626275+0000","last_peered":"2023-09-23T20:35:25.626275+0000","last_clean":"2023-09-23T20:35:25.626275+0000","last_became_active":"2023-09-23T20:29:07.044892+0000","last_became_peered":"2023-09-23T20:29:07.044892+0000","last_unstale":"2023-09-23T20:35:25.626275+0000","last_undegraded":"2023-09-23T20:35:25.626275+0000","last_fullsized":"2023-09-23T20:35:25.626275+0000","mapping_epoch":253,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":254,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:49.726139+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:49.726139+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:59:11.810516+0000","scrub_duration":0.0015441470000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":328,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-23T20:35:25.627403+0000","last_change":"2023-09-23T20:25:02.953993+0000","last_active":"2023-09-23T20:35:25.627403+0000","last_peered":"2023-09-23T20:35:25.627403+0000","last_clean":"2023-09-23T20:35:25.627403+0000","last_became_active":"2023-09-23T20:25:02.953672+0000","last_became_peered":"2023-09-23T20:25:02.953672+0000","last_unstale":"2023-09-23T20:35:25.627403+0000","last_undegraded":"2023-09-23T20:35:25.627403+0000","last_fullsized":"2023-09-23T20:35:25.627403+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:26:08.603577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":341,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-23T20:35:25.626255+0000","last_change":"2023-09-23T20:25:02.949584+0000","last_active":"2023-09-23T20:35:25.626255+0000","last_peered":"2023-09-23T20:35:25.626255+0000","last_clean":"2023-09-23T20:35:25.626255+0000","last_became_active":"2023-09-23T20:25:02.949132+0000","last_became_peered":"2023-09-23T20:25:02.949132+0000","last_unstale":"2023-09-23T20:35:25.626255+0000","last_undegraded":"2023-09-23T20:35:25.626255+0000","last_fullsized":"2023-09-23T20:35:25.626255+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:34:20.509823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"90'161","reported_seq":646,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-23T20:35:25.645959+0000","last_change":"2023-09-23T20:35:02.238923+0000","last_active":"2023-09-23T20:35:25.645959+0000","last_peered":"2023-09-23T20:35:25.645959+0000","last_clean":"2023-09-23T20:35:25.645959+0000","last_became_active":"2023-09-23T20:35:02.238293+0000","last_became_peered":"2023-09-23T20:35:02.238293+0000","last_unstale":"2023-09-23T20:35:25.645959+0000","last_undegraded":"2023-09-23T20:35:25.645959+0000","last_fullsized":"2023-09-23T20:35:25.645959+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:42.544308+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:42.544308+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:51:22.281932+0000","scrub_duration":0.0029958179999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":346,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-23T20:35:25.634114+0000","last_change":"2023-09-23T20:32:33.225148+0000","last_active":"2023-09-23T20:35:25.634114+0000","last_peered":"2023-09-23T20:35:25.634114+0000","last_clean":"2023-09-23T20:35:25.634114+0000","last_became_active":"2023-09-23T20:32:33.224805+0000","last_became_peered":"2023-09-23T20:32:33.224805+0000","last_unstale":"2023-09-23T20:35:25.634114+0000","last_undegraded":"2023-09-23T20:35:25.634114+0000","last_fullsized":"2023-09-23T20:35:25.634114+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:55:30.891398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":338,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-23T20:35:25.630840+0000","last_change":"2023-09-23T20:25:02.957010+0000","last_active":"2023-09-23T20:35:25.630840+0000","last_peered":"2023-09-23T20:35:25.630840+0000","last_clean":"2023-09-23T20:35:25.630840+0000","last_became_active":"2023-09-23T20:25:02.956267+0000","last_became_peered":"2023-09-23T20:25:02.956267+0000","last_unstale":"2023-09-23T20:35:25.630840+0000","last_undegraded":"2023-09-23T20:35:25.630840+0000","last_fullsized":"2023-09-23T20:35:25.630840+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:10:55.299422+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"90'161","reported_seq":612,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-23T20:35:25.624465+0000","last_change":"2023-09-23T20:31:53.707033+0000","last_active":"2023-09-23T20:35:25.624465+0000","last_peered":"2023-09-23T20:35:25.624465+0000","last_clean":"2023-09-23T20:35:25.624465+0000","last_became_active":"2023-09-23T20:29:46.078873+0000","last_became_peered":"2023-09-23T20:29:46.078873+0000","last_unstale":"2023-09-23T20:35:25.624465+0000","last_undegraded":"2023-09-23T20:35:25.624465+0000","last_fullsized":"2023-09-23T20:35:25.624465+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:53.706934+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:53.706934+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:51:58.039657+0000","scrub_duration":0.0016840900000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":132,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-23T20:35:25.624535+0000","last_change":"2023-09-23T20:29:04.179171+0000","last_active":"2023-09-23T20:35:25.624535+0000","last_peered":"2023-09-23T20:35:25.624535+0000","last_clean":"2023-09-23T20:35:25.624535+0000","last_became_active":"2023-09-23T20:29:04.128672+0000","last_became_peered":"2023-09-23T20:29:04.128672+0000","last_unstale":"2023-09-23T20:35:25.624535+0000","last_undegraded":"2023-09-23T20:35:25.624535+0000","last_fullsized":"2023-09-23T20:35:25.624535+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:49:15.506106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":175,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-23T20:35:25.634122+0000","last_change":"2023-09-23T20:32:50.539157+0000","last_active":"2023-09-23T20:35:25.634122+0000","last_peered":"2023-09-23T20:35:25.634122+0000","last_clean":"2023-09-23T20:35:25.634122+0000","last_became_active":"2023-09-23T20:32:50.538889+0000","last_became_peered":"2023-09-23T20:32:50.538889+0000","last_unstale":"2023-09-23T20:35:25.634122+0000","last_undegraded":"2023-09-23T20:35:25.634122+0000","last_fullsized":"2023-09-23T20:35:25.634122+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:28:28.857256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.b","version":"363'10996","reported_seq":11066,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T20:35:29.138173+0000","last_change":"2023-09-23T20:31:49.152343+0000","last_active":"2023-09-23T20:35:29.138173+0000","last_peered":"2023-09-23T20:35:29.138173+0000","last_clean":"2023-09-23T20:35:29.138173+0000","last_became_active":"2023-09-23T20:31:49.152042+0000","last_became_peered":"2023-09-23T20:31:49.152042+0000","last_unstale":"2023-09-23T20:35:29.138173+0000","last_undegraded":"2023-09-23T20:35:29.138173+0000","last_fullsized":"2023-09-23T20:35:29.138173+0000","mapping_epoch":303,"log_start":"325'4839","ondisk_log_start":"325'4839","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":6157,"log_dups_size":0,"ondisk_log_size":6157,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:53:34.237468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":56360960,"num_objects":860,"num_object_clones":0,"num_object_copies":2580,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":860,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10996,"num_write_kb":379392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.d","version":"90'161","reported_seq":655,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-23T20:35:25.632056+0000","last_change":"2023-09-23T20:34:59.496438+0000","last_active":"2023-09-23T20:35:25.632056+0000","last_peered":"2023-09-23T20:35:25.632056+0000","last_clean":"2023-09-23T20:35:25.632056+0000","last_became_active":"2023-09-23T20:33:38.970527+0000","last_became_peered":"2023-09-23T20:33:38.970527+0000","last_unstale":"2023-09-23T20:35:25.632056+0000","last_undegraded":"2023-09-23T20:35:25.632056+0000","last_fullsized":"2023-09-23T20:35:25.632056+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:34:59.496375+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:34:59.496375+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:46:18.244734+0000","scrub_duration":0.0017843189999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":366,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-23T20:35:25.634027+0000","last_change":"2023-09-23T20:32:33.283752+0000","last_active":"2023-09-23T20:35:25.634027+0000","last_peered":"2023-09-23T20:35:25.634027+0000","last_clean":"2023-09-23T20:35:25.634027+0000","last_became_active":"2023-09-23T20:32:33.230503+0000","last_became_peered":"2023-09-23T20:32:33.230503+0000","last_unstale":"2023-09-23T20:35:25.634027+0000","last_undegraded":"2023-09-23T20:35:25.634027+0000","last_fullsized":"2023-09-23T20:35:25.634027+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:29.084410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":324,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-23T20:35:25.626429+0000","last_change":"2023-09-23T20:29:04.128604+0000","last_active":"2023-09-23T20:35:25.626429+0000","last_peered":"2023-09-23T20:35:25.626429+0000","last_clean":"2023-09-23T20:35:25.626429+0000","last_became_active":"2023-09-23T20:29:04.128003+0000","last_became_peered":"2023-09-23T20:29:04.128003+0000","last_unstale":"2023-09-23T20:35:25.626429+0000","last_undegraded":"2023-09-23T20:35:25.626429+0000","last_fullsized":"2023-09-23T20:35:25.626429+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:50:49.385599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"363'10912","reported_seq":10989,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T20:35:27.665663+0000","last_change":"2023-09-23T20:32:50.543873+0000","last_active":"2023-09-23T20:35:27.665663+0000","last_peered":"2023-09-23T20:35:27.665663+0000","last_clean":"2023-09-23T20:35:27.665663+0000","last_became_active":"2023-09-23T20:32:50.537253+0000","last_became_peered":"2023-09-23T20:32:50.537253+0000","last_unstale":"2023-09-23T20:35:27.665663+0000","last_undegraded":"2023-09-23T20:35:27.665663+0000","last_fullsized":"2023-09-23T20:35:27.665663+0000","mapping_epoch":319,"log_start":"310'2500","ondisk_log_start":"310'2500","created":303,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":8412,"log_dups_size":0,"ondisk_log_size":8412,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:38:02.452291+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":58982400,"num_objects":900,"num_object_clones":0,"num_object_copies":2700,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":900,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10912,"num_write_kb":377984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,5],"acting":[2,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.c","version":"90'161","reported_seq":534,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-23T20:35:25.627963+0000","last_change":"2023-09-23T20:33:38.989595+0000","last_active":"2023-09-23T20:35:25.627963+0000","last_peered":"2023-09-23T20:35:25.627963+0000","last_clean":"2023-09-23T20:35:25.627963+0000","last_became_active":"2023-09-23T20:33:38.989255+0000","last_became_peered":"2023-09-23T20:33:38.989255+0000","last_unstale":"2023-09-23T20:35:25.627963+0000","last_undegraded":"2023-09-23T20:35:25.627963+0000","last_fullsized":"2023-09-23T20:35:25.627963+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:41.701171+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:41.701171+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0019955509999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":54613,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":401,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-23T20:35:25.632799+0000","last_change":"2023-09-23T20:34:59.987981+0000","last_active":"2023-09-23T20:35:25.632799+0000","last_peered":"2023-09-23T20:35:25.632799+0000","last_clean":"2023-09-23T20:35:25.632799+0000","last_became_active":"2023-09-23T20:34:59.985368+0000","last_became_peered":"2023-09-23T20:34:59.985368+0000","last_unstale":"2023-09-23T20:35:25.632799+0000","last_undegraded":"2023-09-23T20:35:25.632799+0000","last_fullsized":"2023-09-23T20:35:25.632799+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:12:43.082493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":60,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-23T20:35:25.621041+0000","last_change":"2023-09-23T20:32:33.232142+0000","last_active":"2023-09-23T20:35:25.621041+0000","last_peered":"2023-09-23T20:35:25.621041+0000","last_clean":"2023-09-23T20:35:25.621041+0000","last_became_active":"2023-09-23T20:32:33.231683+0000","last_became_peered":"2023-09-23T20:32:33.231683+0000","last_unstale":"2023-09-23T20:35:25.621041+0000","last_undegraded":"2023-09-23T20:35:25.621041+0000","last_fullsized":"2023-09-23T20:35:25.621041+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:02:58.336348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"363'10937","reported_seq":13259,"reported_epoch":363,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T20:35:30.589245+0000","last_change":"2023-09-23T20:35:01.111484+0000","last_active":"2023-09-23T20:35:30.589245+0000","last_peered":"2023-09-23T20:35:30.589245+0000","last_clean":"2023-09-23T20:34:58.962913+0000","last_became_active":"2023-09-23T20:35:01.087993+0000","last_became_peered":"2023-09-23T20:35:01.087993+0000","last_unstale":"2023-09-23T20:35:30.589245+0000","last_undegraded":"2023-09-23T20:35:30.589245+0000","last_fullsized":"2023-09-23T20:35:30.589245+0000","mapping_epoch":354,"log_start":"321'4377","ondisk_log_start":"321'4377","created":303,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":6560,"log_dups_size":0,"ondisk_log_size":6560,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:09:02.548463+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":51838976,"num_objects":791,"num_object_clones":0,"num_object_copies":2373,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":791,"num_objects_unfound":0,"num_objects_dirty":791,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10937,"num_write_kb":375296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1109,"num_bytes_recovered":72679424,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,7],"avail_no_missing":["0","1","7"],"object_location_counts":[{"shards":"0,1,7","objects":791}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"90'161","reported_seq":585,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-23T20:35:25.624402+0000","last_change":"2023-09-23T20:32:36.302809+0000","last_active":"2023-09-23T20:35:25.624402+0000","last_peered":"2023-09-23T20:35:25.624402+0000","last_clean":"2023-09-23T20:35:25.624402+0000","last_became_active":"2023-09-23T20:32:36.302465+0000","last_became_peered":"2023-09-23T20:32:36.302465+0000","last_unstale":"2023-09-23T20:35:25.624402+0000","last_undegraded":"2023-09-23T20:35:25.624402+0000","last_fullsized":"2023-09-23T20:35:25.624402+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:46.217489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:46.217489+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:35:40.488427+0000","scrub_duration":0.001456429,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":335,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-23T20:35:25.632137+0000","last_change":"2023-09-23T20:29:04.184893+0000","last_active":"2023-09-23T20:35:25.632137+0000","last_peered":"2023-09-23T20:35:25.632137+0000","last_clean":"2023-09-23T20:35:25.632137+0000","last_became_active":"2023-09-23T20:29:04.152723+0000","last_became_peered":"2023-09-23T20:29:04.152723+0000","last_unstale":"2023-09-23T20:35:25.632137+0000","last_undegraded":"2023-09-23T20:35:25.632137+0000","last_fullsized":"2023-09-23T20:35:25.632137+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:51:27.017151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":136,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-23T20:35:25.624353+0000","last_change":"2023-09-23T20:29:46.086912+0000","last_active":"2023-09-23T20:35:25.624353+0000","last_peered":"2023-09-23T20:35:25.624353+0000","last_clean":"2023-09-23T20:35:25.624353+0000","last_became_active":"2023-09-23T20:29:46.085471+0000","last_became_peered":"2023-09-23T20:29:46.085471+0000","last_unstale":"2023-09-23T20:35:25.624353+0000","last_undegraded":"2023-09-23T20:35:25.624353+0000","last_fullsized":"2023-09-23T20:35:25.624353+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:04:55.751660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.8","version":"363'10940","reported_seq":18304,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T20:35:27.725348+0000","last_change":"2023-09-23T20:34:05.581246+0000","last_active":"2023-09-23T20:35:27.725348+0000","last_peered":"2023-09-23T20:35:27.725348+0000","last_clean":"2023-09-23T20:35:27.725348+0000","last_became_active":"2023-09-23T20:34:05.560057+0000","last_became_peered":"2023-09-23T20:34:05.560057+0000","last_unstale":"2023-09-23T20:35:27.725348+0000","last_undegraded":"2023-09-23T20:35:27.725348+0000","last_fullsized":"2023-09-23T20:35:27.725348+0000","mapping_epoch":338,"log_start":"310'2500","ondisk_log_start":"310'2500","created":303,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":8440,"log_dups_size":0,"ondisk_log_size":8440,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:08:18.482746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":56492032,"num_objects":862,"num_object_clones":0,"num_object_copies":2586,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":862,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10940,"num_write_kb":377664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3638,"num_bytes_recovered":238419968,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.e","version":"90'161","reported_seq":453,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-23T20:35:25.622660+0000","last_change":"2023-09-23T20:35:02.233909+0000","last_active":"2023-09-23T20:35:25.622660+0000","last_peered":"2023-09-23T20:35:25.622660+0000","last_clean":"2023-09-23T20:35:25.622660+0000","last_became_active":"2023-09-23T20:35:02.233536+0000","last_became_peered":"2023-09-23T20:35:02.233536+0000","last_unstale":"2023-09-23T20:35:25.622660+0000","last_undegraded":"2023-09-23T20:35:25.622660+0000","last_fullsized":"2023-09-23T20:35:25.622660+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:44.386120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:44.386120+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":369,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-23T20:35:25.624104+0000","last_change":"2023-09-23T20:29:04.138284+0000","last_active":"2023-09-23T20:35:25.624104+0000","last_peered":"2023-09-23T20:35:25.624104+0000","last_clean":"2023-09-23T20:35:25.624104+0000","last_became_active":"2023-09-23T20:29:04.137960+0000","last_became_peered":"2023-09-23T20:29:04.137960+0000","last_unstale":"2023-09-23T20:35:25.624104+0000","last_undegraded":"2023-09-23T20:35:25.624104+0000","last_fullsized":"2023-09-23T20:35:25.624104+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:18:59.548311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":346,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-23T20:35:25.627925+0000","last_change":"2023-09-23T20:32:33.236122+0000","last_active":"2023-09-23T20:35:25.627925+0000","last_peered":"2023-09-23T20:35:25.627925+0000","last_clean":"2023-09-23T20:35:25.627925+0000","last_became_active":"2023-09-23T20:32:33.235384+0000","last_became_peered":"2023-09-23T20:32:33.235384+0000","last_unstale":"2023-09-23T20:35:25.627925+0000","last_undegraded":"2023-09-23T20:35:25.627925+0000","last_fullsized":"2023-09-23T20:35:25.627925+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:56.843733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"363'11141","reported_seq":11211,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T20:35:30.459688+0000","last_change":"2023-09-23T20:31:49.158461+0000","last_active":"2023-09-23T20:35:30.459688+0000","last_peered":"2023-09-23T20:35:30.459688+0000","last_clean":"2023-09-23T20:35:30.459688+0000","last_became_active":"2023-09-23T20:31:49.158188+0000","last_became_peered":"2023-09-23T20:31:49.158188+0000","last_unstale":"2023-09-23T20:35:30.459688+0000","last_undegraded":"2023-09-23T20:35:30.459688+0000","last_fullsized":"2023-09-23T20:35:30.459688+0000","mapping_epoch":303,"log_start":"321'4562","ondisk_log_start":"321'4562","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":6579,"log_dups_size":0,"ondisk_log_size":6579,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:31:57.678090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53411840,"num_objects":815,"num_object_clones":0,"num_object_copies":2445,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":815,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11141,"num_write_kb":382592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":768,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-23T20:35:25.634031+0000","last_change":"2023-09-23T20:35:02.523308+0000","last_active":"2023-09-23T20:35:25.634031+0000","last_peered":"2023-09-23T20:35:25.634031+0000","last_clean":"2023-09-23T20:35:25.634031+0000","last_became_active":"2023-09-23T20:33:38.979989+0000","last_became_peered":"2023-09-23T20:33:38.979989+0000","last_unstale":"2023-09-23T20:35:25.634031+0000","last_undegraded":"2023-09-23T20:35:25.634031+0000","last_fullsized":"2023-09-23T20:35:25.634031+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:35:02.523203+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:35:02.523203+0000","last_clean_scrub_stamp":"2023-09-23T20:35:02.523203+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:02:41.472958+0000","scrub_duration":0.0085927989999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":98,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-23T20:35:25.628031+0000","last_change":"2023-09-23T20:32:50.538500+0000","last_active":"2023-09-23T20:35:25.628031+0000","last_peered":"2023-09-23T20:35:25.628031+0000","last_clean":"2023-09-23T20:35:25.628031+0000","last_became_active":"2023-09-23T20:32:50.538152+0000","last_became_peered":"2023-09-23T20:32:50.538152+0000","last_unstale":"2023-09-23T20:35:25.628031+0000","last_undegraded":"2023-09-23T20:35:25.628031+0000","last_fullsized":"2023-09-23T20:35:25.628031+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:00:41.103508+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":60,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-23T20:35:25.621970+0000","last_change":"2023-09-23T20:32:33.231614+0000","last_active":"2023-09-23T20:35:25.621970+0000","last_peered":"2023-09-23T20:35:25.621970+0000","last_clean":"2023-09-23T20:35:25.621970+0000","last_became_active":"2023-09-23T20:32:33.231207+0000","last_became_peered":"2023-09-23T20:32:33.231207+0000","last_unstale":"2023-09-23T20:35:25.621970+0000","last_undegraded":"2023-09-23T20:35:25.621970+0000","last_fullsized":"2023-09-23T20:35:25.621970+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:46:09.780829+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"363'10913","reported_seq":10983,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T20:35:27.745964+0000","last_change":"2023-09-23T20:31:49.155754+0000","last_active":"2023-09-23T20:35:27.745964+0000","last_peered":"2023-09-23T20:35:27.745964+0000","last_clean":"2023-09-23T20:35:27.745964+0000","last_became_active":"2023-09-23T20:31:49.155363+0000","last_became_peered":"2023-09-23T20:31:49.155363+0000","last_unstale":"2023-09-23T20:35:27.745964+0000","last_undegraded":"2023-09-23T20:35:27.745964+0000","last_fullsized":"2023-09-23T20:35:27.745964+0000","mapping_epoch":303,"log_start":"310'2500","ondisk_log_start":"310'2500","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":8413,"log_dups_size":0,"ondisk_log_size":8413,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:45:42.661641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":61276160,"num_objects":935,"num_object_clones":0,"num_object_copies":2805,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":935,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10913,"num_write_kb":379136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":613,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-23T20:35:25.626353+0000","last_change":"2023-09-23T20:35:00.496271+0000","last_active":"2023-09-23T20:35:25.626353+0000","last_peered":"2023-09-23T20:35:25.626353+0000","last_clean":"2023-09-23T20:35:25.626353+0000","last_became_active":"2023-09-23T20:33:38.996273+0000","last_became_peered":"2023-09-23T20:33:38.996273+0000","last_unstale":"2023-09-23T20:35:25.626353+0000","last_undegraded":"2023-09-23T20:35:25.626353+0000","last_fullsized":"2023-09-23T20:35:25.626353+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:35:00.496197+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:35:00.496197+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:30:15.813493+0000","scrub_duration":0.0016339779999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":158,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-23T20:35:25.624436+0000","last_change":"2023-09-23T20:32:33.229700+0000","last_active":"2023-09-23T20:35:25.624436+0000","last_peered":"2023-09-23T20:35:25.624436+0000","last_clean":"2023-09-23T20:35:25.624436+0000","last_became_active":"2023-09-23T20:32:33.229372+0000","last_became_peered":"2023-09-23T20:32:33.229372+0000","last_unstale":"2023-09-23T20:35:25.624436+0000","last_undegraded":"2023-09-23T20:35:25.624436+0000","last_fullsized":"2023-09-23T20:35:25.624436+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:18:10.958227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":19,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-23T20:35:25.620899+0000","last_change":"2023-09-23T20:34:59.985545+0000","last_active":"2023-09-23T20:35:25.620899+0000","last_peered":"2023-09-23T20:35:25.620899+0000","last_clean":"2023-09-23T20:35:25.620899+0000","last_became_active":"2023-09-23T20:34:59.985090+0000","last_became_peered":"2023-09-23T20:34:59.985090+0000","last_unstale":"2023-09-23T20:35:25.620899+0000","last_undegraded":"2023-09-23T20:35:25.620899+0000","last_fullsized":"2023-09-23T20:35:25.620899+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:21:40.825065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"363'11168","reported_seq":11237,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T20:35:30.598756+0000","last_change":"2023-09-23T20:31:49.158641+0000","last_active":"2023-09-23T20:35:30.598756+0000","last_peered":"2023-09-23T20:35:30.598756+0000","last_clean":"2023-09-23T20:35:30.598756+0000","last_became_active":"2023-09-23T20:31:49.157604+0000","last_became_peered":"2023-09-23T20:31:49.157604+0000","last_unstale":"2023-09-23T20:35:30.598756+0000","last_undegraded":"2023-09-23T20:35:30.598756+0000","last_fullsized":"2023-09-23T20:35:30.598756+0000","mapping_epoch":303,"log_start":"326'4912","ondisk_log_start":"326'4912","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":6256,"log_dups_size":0,"ondisk_log_size":6256,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:54:04.726000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53936152,"num_objects":824,"num_object_clones":0,"num_object_copies":2472,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":824,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":11168,"num_write_kb":383681,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.b","version":"90'161","reported_seq":615,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-23T20:35:25.646052+0000","last_change":"2023-09-23T20:33:38.992494+0000","last_active":"2023-09-23T20:35:25.646052+0000","last_peered":"2023-09-23T20:35:25.646052+0000","last_clean":"2023-09-23T20:35:25.646052+0000","last_became_active":"2023-09-23T20:33:38.991910+0000","last_became_peered":"2023-09-23T20:33:38.991910+0000","last_unstale":"2023-09-23T20:35:25.646052+0000","last_undegraded":"2023-09-23T20:35:25.646052+0000","last_fullsized":"2023-09-23T20:35:25.646052+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:46.558292+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:46.558292+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0023048779999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":19,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-23T20:35:25.620757+0000","last_change":"2023-09-23T20:34:59.983122+0000","last_active":"2023-09-23T20:35:25.620757+0000","last_peered":"2023-09-23T20:35:25.620757+0000","last_clean":"2023-09-23T20:35:25.620757+0000","last_became_active":"2023-09-23T20:34:59.982809+0000","last_became_peered":"2023-09-23T20:34:59.982809+0000","last_unstale":"2023-09-23T20:35:25.620757+0000","last_undegraded":"2023-09-23T20:35:25.620757+0000","last_fullsized":"2023-09-23T20:35:25.620757+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:49:40.839723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":60,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-23T20:35:25.622809+0000","last_change":"2023-09-23T20:32:33.224963+0000","last_active":"2023-09-23T20:35:25.622809+0000","last_peered":"2023-09-23T20:35:25.622809+0000","last_clean":"2023-09-23T20:35:25.622809+0000","last_became_active":"2023-09-23T20:32:33.224634+0000","last_became_peered":"2023-09-23T20:32:33.224634+0000","last_unstale":"2023-09-23T20:35:25.622809+0000","last_undegraded":"2023-09-23T20:35:25.622809+0000","last_fullsized":"2023-09-23T20:35:25.622809+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:41:40.314140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"363'11057","reported_seq":11125,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T20:35:30.560012+0000","last_change":"2023-09-23T20:31:49.157247+0000","last_active":"2023-09-23T20:35:30.560012+0000","last_peered":"2023-09-23T20:35:30.560012+0000","last_clean":"2023-09-23T20:35:30.560012+0000","last_became_active":"2023-09-23T20:31:49.156961+0000","last_became_peered":"2023-09-23T20:31:49.156961+0000","last_unstale":"2023-09-23T20:35:30.560012+0000","last_undegraded":"2023-09-23T20:35:30.560012+0000","last_fullsized":"2023-09-23T20:35:30.560012+0000","mapping_epoch":303,"log_start":"326'4815","ondisk_log_start":"326'4815","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":6242,"log_dups_size":0,"ondisk_log_size":6242,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:49:17.346795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":52625408,"num_objects":803,"num_object_clones":0,"num_object_copies":2409,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":803,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11057,"num_write_kb":379520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":656,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-23T20:35:25.624362+0000","last_change":"2023-09-23T20:34:59.481511+0000","last_active":"2023-09-23T20:35:25.624362+0000","last_peered":"2023-09-23T20:35:25.624362+0000","last_clean":"2023-09-23T20:35:25.624362+0000","last_became_active":"2023-09-23T20:33:38.988954+0000","last_became_peered":"2023-09-23T20:33:38.988954+0000","last_unstale":"2023-09-23T20:35:25.624362+0000","last_undegraded":"2023-09-23T20:35:25.624362+0000","last_fullsized":"2023-09-23T20:35:25.624362+0000","mapping_epoch":253,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:34:59.481446+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:34:59.481446+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:53:42.768648+0000","scrub_duration":0.001554773,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":176,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-23T20:35:25.643918+0000","last_change":"2023-09-23T20:32:50.530690+0000","last_active":"2023-09-23T20:35:25.643918+0000","last_peered":"2023-09-23T20:35:25.643918+0000","last_clean":"2023-09-23T20:35:25.643918+0000","last_became_active":"2023-09-23T20:32:50.530376+0000","last_became_peered":"2023-09-23T20:32:50.530376+0000","last_unstale":"2023-09-23T20:35:25.643918+0000","last_undegraded":"2023-09-23T20:35:25.643918+0000","last_fullsized":"2023-09-23T20:35:25.643918+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:24:53.184959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":184,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-23T20:35:25.645721+0000","last_change":"2023-09-23T20:32:33.230582+0000","last_active":"2023-09-23T20:35:25.645721+0000","last_peered":"2023-09-23T20:35:25.645721+0000","last_clean":"2023-09-23T20:35:25.645721+0000","last_became_active":"2023-09-23T20:32:33.230257+0000","last_became_peered":"2023-09-23T20:32:33.230257+0000","last_unstale":"2023-09-23T20:35:25.645721+0000","last_undegraded":"2023-09-23T20:35:25.645721+0000","last_fullsized":"2023-09-23T20:35:25.645721+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:11:45.333096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"363'10879","reported_seq":12014,"reported_epoch":363,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T20:35:29.166340+0000","last_change":"2023-09-23T20:35:01.122419+0000","last_active":"2023-09-23T20:35:29.166340+0000","last_peered":"2023-09-23T20:35:29.166340+0000","last_clean":"2023-09-23T20:34:58.841504+0000","last_became_active":"2023-09-23T20:35:01.087671+0000","last_became_peered":"2023-09-23T20:35:01.087671+0000","last_unstale":"2023-09-23T20:35:29.166340+0000","last_undegraded":"2023-09-23T20:35:29.166340+0000","last_fullsized":"2023-09-23T20:35:29.166340+0000","mapping_epoch":354,"log_start":"324'4691","ondisk_log_start":"324'4691","created":303,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":6188,"log_dups_size":0,"ondisk_log_size":6188,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:29:52.175555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":56557568,"num_objects":863,"num_object_clones":0,"num_object_copies":2589,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":863,"num_objects_unfound":0,"num_objects_dirty":863,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10879,"num_write_kb":375744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":521,"num_bytes_recovered":34144256,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[1,3,6],"avail_no_missing":["1","3","6"],"object_location_counts":[{"shards":"1,3,6","objects":863}],"blocked_by":[],"up_primary":4,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":612,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-23T20:35:25.624816+0000","last_change":"2023-09-23T20:35:04.102089+0000","last_active":"2023-09-23T20:35:25.624816+0000","last_peered":"2023-09-23T20:35:25.624816+0000","last_clean":"2023-09-23T20:35:25.624816+0000","last_became_active":"2023-09-23T20:33:38.978580+0000","last_became_peered":"2023-09-23T20:33:38.978580+0000","last_unstale":"2023-09-23T20:35:25.624816+0000","last_undegraded":"2023-09-23T20:35:25.624816+0000","last_fullsized":"2023-09-23T20:35:25.624816+0000","mapping_epoch":253,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:35:04.101972+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:35:04.101972+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:44:38.790944+0000","scrub_duration":0.0063459650000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"86'119","reported_seq":668,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-23T20:35:25.650664+0000","last_change":"2023-09-23T20:29:45.521257+0000","last_active":"2023-09-23T20:35:25.650664+0000","last_peered":"2023-09-23T20:35:25.650664+0000","last_clean":"2023-09-23T20:35:25.650664+0000","last_became_active":"2023-09-23T20:29:45.520546+0000","last_became_peered":"2023-09-23T20:29:45.520546+0000","last_unstale":"2023-09-23T20:35:25.650664+0000","last_undegraded":"2023-09-23T20:35:25.650664+0000","last_fullsized":"2023-09-23T20:35:25.650664+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":119,"log_dups_size":0,"ondisk_log_size":119,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:58:56.085777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032705499999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":523,"num_read_kb":3503,"num_write":136,"num_write_kb":3128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":379,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-23T20:35:25.626151+0000","last_change":"2023-09-23T20:25:39.637160+0000","last_active":"2023-09-23T20:35:25.626151+0000","last_peered":"2023-09-23T20:35:25.626151+0000","last_clean":"2023-09-23T20:35:25.626151+0000","last_became_active":"2023-09-23T20:25:39.636860+0000","last_became_peered":"2023-09-23T20:25:39.636860+0000","last_unstale":"2023-09-23T20:35:25.626151+0000","last_undegraded":"2023-09-23T20:35:25.626151+0000","last_fullsized":"2023-09-23T20:35:25.626151+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:21:47.860063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":297,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-23T20:35:25.644812+0000","last_change":"2023-09-23T20:34:59.985572+0000","last_active":"2023-09-23T20:35:25.644812+0000","last_peered":"2023-09-23T20:35:25.644812+0000","last_clean":"2023-09-23T20:35:25.644812+0000","last_became_active":"2023-09-23T20:34:59.985241+0000","last_became_peered":"2023-09-23T20:34:59.985241+0000","last_unstale":"2023-09-23T20:35:25.644812+0000","last_undegraded":"2023-09-23T20:35:25.644812+0000","last_fullsized":"2023-09-23T20:35:25.644812+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:54:57.081334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"363'11110","reported_seq":11178,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T20:35:30.618282+0000","last_change":"2023-09-23T20:31:49.157301+0000","last_active":"2023-09-23T20:35:30.618282+0000","last_peered":"2023-09-23T20:35:30.618282+0000","last_clean":"2023-09-23T20:35:30.618282+0000","last_became_active":"2023-09-23T20:31:49.156992+0000","last_became_peered":"2023-09-23T20:31:49.156992+0000","last_unstale":"2023-09-23T20:35:30.618282+0000","last_undegraded":"2023-09-23T20:35:30.618282+0000","last_fullsized":"2023-09-23T20:35:30.618282+0000","mapping_epoch":303,"log_start":"326'4866","ondisk_log_start":"326'4866","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":6244,"log_dups_size":0,"ondisk_log_size":6244,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:21:35.645838+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":52035584,"num_objects":794,"num_object_clones":0,"num_object_copies":2382,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":794,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11110,"num_write_kb":380928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":607,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-23T20:35:25.623785+0000","last_change":"2023-09-23T20:31:45.785521+0000","last_active":"2023-09-23T20:35:25.623785+0000","last_peered":"2023-09-23T20:35:25.623785+0000","last_clean":"2023-09-23T20:35:25.623785+0000","last_became_active":"2023-09-23T20:27:55.213394+0000","last_became_peered":"2023-09-23T20:27:55.213394+0000","last_unstale":"2023-09-23T20:35:25.623785+0000","last_undegraded":"2023-09-23T20:35:25.623785+0000","last_fullsized":"2023-09-23T20:35:25.623785+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:45.785450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:45.785450+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:21:26.092578+0000","scrub_duration":0.001558306,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"86'106","reported_seq":1081,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-23T20:35:25.624807+0000","last_change":"2023-09-23T20:31:31.531882+0000","last_active":"2023-09-23T20:35:25.624807+0000","last_peered":"2023-09-23T20:35:25.624807+0000","last_clean":"2023-09-23T20:35:25.624807+0000","last_became_active":"2023-09-23T20:29:46.078501+0000","last_became_peered":"2023-09-23T20:29:46.078501+0000","last_unstale":"2023-09-23T20:35:25.624807+0000","last_undegraded":"2023-09-23T20:35:25.624807+0000","last_fullsized":"2023-09-23T20:35:25.624807+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":106,"log_dups_size":0,"ondisk_log_size":106,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:37.015974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00099302599999999993,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1105,"num_read_kb":1689,"num_write":100,"num_write_kb":404,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":60,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-23T20:35:25.621712+0000","last_change":"2023-09-23T20:32:33.230242+0000","last_active":"2023-09-23T20:35:25.621712+0000","last_peered":"2023-09-23T20:35:25.621712+0000","last_clean":"2023-09-23T20:35:25.621712+0000","last_became_active":"2023-09-23T20:32:33.229732+0000","last_became_peered":"2023-09-23T20:32:33.229732+0000","last_unstale":"2023-09-23T20:35:25.621712+0000","last_undegraded":"2023-09-23T20:35:25.621712+0000","last_fullsized":"2023-09-23T20:35:25.621712+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:27:55.538429+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":321,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-23T20:35:25.650579+0000","last_change":"2023-09-23T20:29:45.533406+0000","last_active":"2023-09-23T20:35:25.650579+0000","last_peered":"2023-09-23T20:35:25.650579+0000","last_clean":"2023-09-23T20:35:25.650579+0000","last_became_active":"2023-09-23T20:29:45.526001+0000","last_became_peered":"2023-09-23T20:29:45.526001+0000","last_unstale":"2023-09-23T20:35:25.650579+0000","last_undegraded":"2023-09-23T20:35:25.650579+0000","last_fullsized":"2023-09-23T20:35:25.650579+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:43.787251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.4","version":"363'10936","reported_seq":15242,"reported_epoch":363,"state":"active+recovering+undersized+remapped","last_fresh":"2023-09-23T20:35:30.617138+0000","last_change":"2023-09-23T20:35:00.204596+0000","last_active":"2023-09-23T20:35:30.617138+0000","last_peered":"2023-09-23T20:35:30.617138+0000","last_clean":"2023-09-23T20:32:32.192526+0000","last_became_active":"2023-09-23T20:35:00.124743+0000","last_became_peered":"2023-09-23T20:35:00.124743+0000","last_unstale":"2023-09-23T20:35:30.617138+0000","last_undegraded":"2023-09-23T20:35:30.617138+0000","last_fullsized":"2023-09-23T20:34:59.968003+0000","mapping_epoch":353,"log_start":"321'4384","ondisk_log_start":"321'4384","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":6552,"log_dups_size":0,"ondisk_log_size":6552,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:40:39.030291+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":51511296,"num_objects":786,"num_object_clones":0,"num_object_copies":2358,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":786,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10936,"num_write_kb":375104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2104,"num_bytes_recovered":136642560,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,1],"avail_no_missing":["0","1"],"object_location_counts":[{"shards":"0,1","objects":786}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":835,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-23T20:35:25.631007+0000","last_change":"2023-09-23T20:35:02.260708+0000","last_active":"2023-09-23T20:35:25.631007+0000","last_peered":"2023-09-23T20:35:25.631007+0000","last_clean":"2023-09-23T20:35:25.631007+0000","last_became_active":"2023-09-23T20:35:02.260348+0000","last_became_peered":"2023-09-23T20:35:02.260348+0000","last_unstale":"2023-09-23T20:35:25.631007+0000","last_undegraded":"2023-09-23T20:35:25.631007+0000","last_fullsized":"2023-09-23T20:35:25.631007+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:43.390908+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:43.390908+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:09:14.144070+0000","scrub_duration":0.0032721,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"86'130","reported_seq":676,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-23T20:35:25.621406+0000","last_change":"2023-09-23T20:25:39.637458+0000","last_active":"2023-09-23T20:35:25.621406+0000","last_peered":"2023-09-23T20:35:25.621406+0000","last_clean":"2023-09-23T20:35:25.621406+0000","last_became_active":"2023-09-23T20:25:39.636773+0000","last_became_peered":"2023-09-23T20:25:39.636773+0000","last_unstale":"2023-09-23T20:35:25.621406+0000","last_undegraded":"2023-09-23T20:35:25.621406+0000","last_fullsized":"2023-09-23T20:35:25.621406+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:00:32.810564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011300049999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":512,"num_read_kb":1820,"num_write":133,"num_write_kb":1347,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":311,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-23T20:35:25.643702+0000","last_change":"2023-09-23T20:29:46.073394+0000","last_active":"2023-09-23T20:35:25.643702+0000","last_peered":"2023-09-23T20:35:25.643702+0000","last_clean":"2023-09-23T20:35:25.643702+0000","last_became_active":"2023-09-23T20:29:46.072067+0000","last_became_peered":"2023-09-23T20:29:46.072067+0000","last_unstale":"2023-09-23T20:35:25.643702+0000","last_undegraded":"2023-09-23T20:35:25.643702+0000","last_fullsized":"2023-09-23T20:35:25.643702+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:01:56.944614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":303,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-23T20:35:25.645635+0000","last_change":"2023-09-23T20:29:45.533185+0000","last_active":"2023-09-23T20:35:25.645635+0000","last_peered":"2023-09-23T20:35:25.645635+0000","last_clean":"2023-09-23T20:35:25.645635+0000","last_became_active":"2023-09-23T20:29:45.526765+0000","last_became_peered":"2023-09-23T20:29:45.526765+0000","last_unstale":"2023-09-23T20:35:25.645635+0000","last_undegraded":"2023-09-23T20:35:25.645635+0000","last_fullsized":"2023-09-23T20:35:25.645635+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:18:33.063215+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"363'11066","reported_seq":11136,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T20:35:30.549823+0000","last_change":"2023-09-23T20:31:49.158194+0000","last_active":"2023-09-23T20:35:30.549823+0000","last_peered":"2023-09-23T20:35:30.549823+0000","last_clean":"2023-09-23T20:35:30.549823+0000","last_became_active":"2023-09-23T20:31:49.157875+0000","last_became_peered":"2023-09-23T20:31:49.157875+0000","last_unstale":"2023-09-23T20:35:30.549823+0000","last_undegraded":"2023-09-23T20:35:30.549823+0000","last_fullsized":"2023-09-23T20:35:30.549823+0000","mapping_epoch":303,"log_start":"321'4513","ondisk_log_start":"321'4513","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":6553,"log_dups_size":0,"ondisk_log_size":6553,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:04:42.411181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":52559872,"num_objects":802,"num_object_clones":0,"num_object_copies":2406,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":802,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11066,"num_write_kb":379776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"86'194","reported_seq":1011,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-23T20:35:25.626674+0000","last_change":"2023-09-23T20:25:40.618665+0000","last_active":"2023-09-23T20:35:25.626674+0000","last_peered":"2023-09-23T20:35:25.626674+0000","last_clean":"2023-09-23T20:35:25.626674+0000","last_became_active":"2023-09-23T20:25:40.618087+0000","last_became_peered":"2023-09-23T20:25:40.618087+0000","last_unstale":"2023-09-23T20:35:25.626674+0000","last_undegraded":"2023-09-23T20:35:25.626674+0000","last_fullsized":"2023-09-23T20:35:25.626674+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":194,"log_dups_size":0,"ondisk_log_size":194,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:07:16.114968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00037995999999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1025,"num_read_kb":1201,"num_write":173,"num_write_kb":1276,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":796,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-23T20:35:25.631440+0000","last_change":"2023-09-23T20:31:45.343835+0000","last_active":"2023-09-23T20:35:25.631440+0000","last_peered":"2023-09-23T20:35:25.631440+0000","last_clean":"2023-09-23T20:35:25.631440+0000","last_became_active":"2023-09-23T20:29:07.042407+0000","last_became_peered":"2023-09-23T20:29:07.042407+0000","last_unstale":"2023-09-23T20:35:25.631440+0000","last_undegraded":"2023-09-23T20:35:25.631440+0000","last_fullsized":"2023-09-23T20:35:25.631440+0000","mapping_epoch":253,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":254,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:45.343742+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:45.343742+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:33:12.745366+0000","scrub_duration":0.001475802,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":162,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-23T20:35:25.624222+0000","last_change":"2023-09-23T20:32:50.534315+0000","last_active":"2023-09-23T20:35:25.624222+0000","last_peered":"2023-09-23T20:35:25.624222+0000","last_clean":"2023-09-23T20:35:25.624222+0000","last_became_active":"2023-09-23T20:32:50.533896+0000","last_became_peered":"2023-09-23T20:32:50.533896+0000","last_unstale":"2023-09-23T20:35:25.624222+0000","last_undegraded":"2023-09-23T20:35:25.624222+0000","last_fullsized":"2023-09-23T20:35:25.624222+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:06:53.464923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":142,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-23T20:35:25.625520+0000","last_change":"2023-09-23T20:32:33.225012+0000","last_active":"2023-09-23T20:35:25.625520+0000","last_peered":"2023-09-23T20:35:25.625520+0000","last_clean":"2023-09-23T20:35:25.625520+0000","last_became_active":"2023-09-23T20:32:33.224699+0000","last_became_peered":"2023-09-23T20:32:33.224699+0000","last_unstale":"2023-09-23T20:35:25.625520+0000","last_undegraded":"2023-09-23T20:35:25.625520+0000","last_fullsized":"2023-09-23T20:35:25.625520+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:10:06.267878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"363'11133","reported_seq":11201,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T20:35:30.767136+0000","last_change":"2023-09-23T20:31:49.154418+0000","last_active":"2023-09-23T20:35:30.767136+0000","last_peered":"2023-09-23T20:35:30.767136+0000","last_clean":"2023-09-23T20:35:30.767136+0000","last_became_active":"2023-09-23T20:31:49.154182+0000","last_became_peered":"2023-09-23T20:31:49.154182+0000","last_unstale":"2023-09-23T20:35:30.767136+0000","last_undegraded":"2023-09-23T20:35:30.767136+0000","last_fullsized":"2023-09-23T20:35:30.767136+0000","mapping_epoch":303,"log_start":"327'5118","ondisk_log_start":"327'5118","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":6015,"log_dups_size":0,"ondisk_log_size":6015,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:44:10.087486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":51970048,"num_objects":793,"num_object_clones":0,"num_object_copies":2379,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":793,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11133,"num_write_kb":381632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":656,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-23T20:35:25.645875+0000","last_change":"2023-09-23T20:35:02.238632+0000","last_active":"2023-09-23T20:35:25.645875+0000","last_peered":"2023-09-23T20:35:25.645875+0000","last_clean":"2023-09-23T20:35:25.645875+0000","last_became_active":"2023-09-23T20:35:02.237812+0000","last_became_peered":"2023-09-23T20:35:02.237812+0000","last_unstale":"2023-09-23T20:35:25.645875+0000","last_undegraded":"2023-09-23T20:35:25.645875+0000","last_fullsized":"2023-09-23T20:35:25.645875+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:43.582146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:43.582146+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:25:04.582696+0000","scrub_duration":0.0016219730000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"86'80","reported_seq":626,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-23T20:35:25.631357+0000","last_change":"2023-09-23T20:29:45.536796+0000","last_active":"2023-09-23T20:35:25.631357+0000","last_peered":"2023-09-23T20:35:25.631357+0000","last_clean":"2023-09-23T20:35:25.631357+0000","last_became_active":"2023-09-23T20:29:45.536439+0000","last_became_peered":"2023-09-23T20:29:45.536439+0000","last_unstale":"2023-09-23T20:35:25.631357+0000","last_undegraded":"2023-09-23T20:35:25.631357+0000","last_fullsized":"2023-09-23T20:35:25.631357+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":80,"log_dups_size":0,"ondisk_log_size":80,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:30:57.793132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035115,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":395,"num_read_kb":1370,"num_write":66,"num_write_kb":1061,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":365,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-23T20:35:25.629586+0000","last_change":"2023-09-23T20:34:59.984286+0000","last_active":"2023-09-23T20:35:25.629586+0000","last_peered":"2023-09-23T20:35:25.629586+0000","last_clean":"2023-09-23T20:35:25.629586+0000","last_became_active":"2023-09-23T20:34:59.983950+0000","last_became_peered":"2023-09-23T20:34:59.983950+0000","last_unstale":"2023-09-23T20:35:25.629586+0000","last_undegraded":"2023-09-23T20:35:25.629586+0000","last_fullsized":"2023-09-23T20:35:25.629586+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:31:53.557035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":360,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-23T20:35:25.629376+0000","last_change":"2023-09-23T20:29:45.690815+0000","last_active":"2023-09-23T20:35:25.629376+0000","last_peered":"2023-09-23T20:35:25.629376+0000","last_clean":"2023-09-23T20:35:25.629376+0000","last_became_active":"2023-09-23T20:29:45.690478+0000","last_became_peered":"2023-09-23T20:29:45.690478+0000","last_unstale":"2023-09-23T20:35:25.629376+0000","last_undegraded":"2023-09-23T20:35:25.629376+0000","last_fullsized":"2023-09-23T20:35:25.629376+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:38.288193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.0","version":"363'11118","reported_seq":11185,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T20:35:30.348752+0000","last_change":"2023-09-23T20:31:49.157205+0000","last_active":"2023-09-23T20:35:30.348752+0000","last_peered":"2023-09-23T20:35:30.348752+0000","last_clean":"2023-09-23T20:35:30.348752+0000","last_became_active":"2023-09-23T20:31:49.156922+0000","last_became_peered":"2023-09-23T20:31:49.156922+0000","last_unstale":"2023-09-23T20:35:30.348752+0000","last_undegraded":"2023-09-23T20:35:30.348752+0000","last_fullsized":"2023-09-23T20:35:30.348752+0000","mapping_epoch":303,"log_start":"310'3387","ondisk_log_start":"310'3387","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":7731,"log_dups_size":0,"ondisk_log_size":7731,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:51:36.120059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":51118080,"num_objects":780,"num_object_clones":0,"num_object_copies":2340,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":780,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11118,"num_write_kb":380736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":605,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-23T20:35:25.626835+0000","last_change":"2023-09-23T20:35:00.151823+0000","last_active":"2023-09-23T20:35:25.626835+0000","last_peered":"2023-09-23T20:35:25.626835+0000","last_clean":"2023-09-23T20:35:25.626835+0000","last_became_active":"2023-09-23T20:33:38.991030+0000","last_became_peered":"2023-09-23T20:33:38.991030+0000","last_unstale":"2023-09-23T20:35:25.626835+0000","last_undegraded":"2023-09-23T20:35:25.626835+0000","last_fullsized":"2023-09-23T20:35:25.626835+0000","mapping_epoch":253,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:35:00.151695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:35:00.151695+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:09:55.183927+0000","scrub_duration":0.0016030090000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"86'102","reported_seq":957,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-23T20:35:25.632210+0000","last_change":"2023-09-23T20:32:36.314672+0000","last_active":"2023-09-23T20:35:25.632210+0000","last_peered":"2023-09-23T20:35:25.632210+0000","last_clean":"2023-09-23T20:35:25.632210+0000","last_became_active":"2023-09-23T20:32:36.314231+0000","last_became_peered":"2023-09-23T20:32:36.314231+0000","last_unstale":"2023-09-23T20:35:25.632210+0000","last_undegraded":"2023-09-23T20:35:25.632210+0000","last_fullsized":"2023-09-23T20:35:25.632210+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":102,"log_dups_size":0,"ondisk_log_size":102,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:11:40.514709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030656500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":963,"num_read_kb":1234,"num_write":60,"num_write_kb":225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":287,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-23T20:35:25.650752+0000","last_change":"2023-09-23T20:29:45.538283+0000","last_active":"2023-09-23T20:35:25.650752+0000","last_peered":"2023-09-23T20:35:25.650752+0000","last_clean":"2023-09-23T20:35:25.650752+0000","last_became_active":"2023-09-23T20:29:45.537922+0000","last_became_peered":"2023-09-23T20:29:45.537922+0000","last_unstale":"2023-09-23T20:35:25.650752+0000","last_undegraded":"2023-09-23T20:35:25.650752+0000","last_fullsized":"2023-09-23T20:35:25.650752+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:30:52.397836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":297,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-23T20:35:25.628141+0000","last_change":"2023-09-23T20:20:14.888874+0000","last_active":"2023-09-23T20:35:25.628141+0000","last_peered":"2023-09-23T20:35:25.628141+0000","last_clean":"2023-09-23T20:35:25.628141+0000","last_became_active":"2023-09-23T20:20:14.887724+0000","last_became_peered":"2023-09-23T20:20:14.887724+0000","last_unstale":"2023-09-23T20:35:25.628141+0000","last_undegraded":"2023-09-23T20:35:25.628141+0000","last_fullsized":"2023-09-23T20:35:25.628141+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:09:24.510326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"363'10600","reported_seq":17777,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T20:35:27.705190+0000","last_change":"2023-09-23T20:34:50.587971+0000","last_active":"2023-09-23T20:35:27.705190+0000","last_peered":"2023-09-23T20:35:27.705190+0000","last_clean":"2023-09-23T20:35:27.705190+0000","last_became_active":"2023-09-23T20:34:50.555328+0000","last_became_peered":"2023-09-23T20:34:50.555328+0000","last_unstale":"2023-09-23T20:35:27.705190+0000","last_undegraded":"2023-09-23T20:35:27.705190+0000","last_fullsized":"2023-09-23T20:35:27.705190+0000","mapping_epoch":349,"log_start":"310'2200","ondisk_log_start":"310'2200","created":303,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":8400,"log_dups_size":0,"ondisk_log_size":8400,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:20:26.250987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":58589184,"num_objects":894,"num_object_clones":0,"num_object_copies":2682,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":894,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10600,"num_write_kb":367808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3542,"num_bytes_recovered":170328064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":571,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-23T20:35:25.622487+0000","last_change":"2023-09-23T20:34:55.534723+0000","last_active":"2023-09-23T20:35:25.622487+0000","last_peered":"2023-09-23T20:35:25.622487+0000","last_clean":"2023-09-23T20:35:25.622487+0000","last_became_active":"2023-09-23T20:33:38.985300+0000","last_became_peered":"2023-09-23T20:33:38.985300+0000","last_unstale":"2023-09-23T20:35:25.622487+0000","last_undegraded":"2023-09-23T20:35:25.622487+0000","last_fullsized":"2023-09-23T20:35:25.622487+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:44.509012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:44.509012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0032186319999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":100,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-23T20:35:25.634146+0000","last_change":"2023-09-23T20:32:50.549695+0000","last_active":"2023-09-23T20:35:25.634146+0000","last_peered":"2023-09-23T20:35:25.634146+0000","last_clean":"2023-09-23T20:35:25.634146+0000","last_became_active":"2023-09-23T20:32:50.549431+0000","last_became_peered":"2023-09-23T20:32:50.549431+0000","last_unstale":"2023-09-23T20:35:25.634146+0000","last_undegraded":"2023-09-23T20:35:25.634146+0000","last_fullsized":"2023-09-23T20:35:25.634146+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:45:53.666569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"86'128","reported_seq":753,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-23T20:35:25.646026+0000","last_change":"2023-09-23T20:32:36.310656+0000","last_active":"2023-09-23T20:35:25.646026+0000","last_peered":"2023-09-23T20:35:25.646026+0000","last_clean":"2023-09-23T20:35:25.646026+0000","last_became_active":"2023-09-23T20:32:36.310155+0000","last_became_peered":"2023-09-23T20:32:36.310155+0000","last_unstale":"2023-09-23T20:35:25.646026+0000","last_undegraded":"2023-09-23T20:35:25.646026+0000","last_fullsized":"2023-09-23T20:35:25.646026+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:58:14.217466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000293202,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":579,"num_read_kb":1473,"num_write":96,"num_write_kb":1062,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":133,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-23T20:35:25.631605+0000","last_change":"2023-09-23T20:32:50.539051+0000","last_active":"2023-09-23T20:35:25.631605+0000","last_peered":"2023-09-23T20:35:25.631605+0000","last_clean":"2023-09-23T20:35:25.631605+0000","last_became_active":"2023-09-23T20:32:50.538755+0000","last_became_peered":"2023-09-23T20:32:50.538755+0000","last_unstale":"2023-09-23T20:35:25.631605+0000","last_undegraded":"2023-09-23T20:35:25.631605+0000","last_fullsized":"2023-09-23T20:35:25.631605+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:16.656613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.2","version":"363'10873","reported_seq":10950,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T20:35:29.123653+0000","last_change":"2023-09-23T20:32:50.557165+0000","last_active":"2023-09-23T20:35:29.123653+0000","last_peered":"2023-09-23T20:35:29.123653+0000","last_clean":"2023-09-23T20:35:29.123653+0000","last_became_active":"2023-09-23T20:32:50.553409+0000","last_became_peered":"2023-09-23T20:32:50.553409+0000","last_unstale":"2023-09-23T20:35:29.123653+0000","last_undegraded":"2023-09-23T20:35:29.123653+0000","last_fullsized":"2023-09-23T20:35:29.123653+0000","mapping_epoch":319,"log_start":"324'4693","ondisk_log_start":"324'4693","created":303,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":6180,"log_dups_size":0,"ondisk_log_size":6180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:55:13.578195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53673984,"num_objects":819,"num_object_clones":0,"num_object_copies":2457,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":819,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10873,"num_write_kb":374144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"90'161","reported_seq":512,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-23T20:35:25.625894+0000","last_change":"2023-09-23T20:34:54.504652+0000","last_active":"2023-09-23T20:35:25.625894+0000","last_peered":"2023-09-23T20:35:25.625894+0000","last_clean":"2023-09-23T20:35:25.625894+0000","last_became_active":"2023-09-23T20:32:50.533470+0000","last_became_peered":"2023-09-23T20:32:50.533470+0000","last_unstale":"2023-09-23T20:35:25.625894+0000","last_undegraded":"2023-09-23T20:35:25.625894+0000","last_fullsized":"2023-09-23T20:35:25.625894+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:34:54.504554+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:34:54.504554+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:17:03.108888+0000","scrub_duration":0.013031021,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.7","version":"86'117","reported_seq":632,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-23T20:35:25.625543+0000","last_change":"2023-09-23T20:31:32.814392+0000","last_active":"2023-09-23T20:35:25.625543+0000","last_peered":"2023-09-23T20:35:25.625543+0000","last_clean":"2023-09-23T20:35:25.625543+0000","last_became_active":"2023-09-23T20:31:32.813857+0000","last_became_peered":"2023-09-23T20:31:32.813857+0000","last_unstale":"2023-09-23T20:35:25.625543+0000","last_undegraded":"2023-09-23T20:35:25.625543+0000","last_fullsized":"2023-09-23T20:35:25.625543+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:56:57.863005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046891400000000002,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":893,"num_read_kb":922,"num_write":82,"num_write_kb":4193,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":360,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-23T20:35:25.629474+0000","last_change":"2023-09-23T20:32:33.277827+0000","last_active":"2023-09-23T20:35:25.629474+0000","last_peered":"2023-09-23T20:35:25.629474+0000","last_clean":"2023-09-23T20:35:25.629474+0000","last_became_active":"2023-09-23T20:32:33.236087+0000","last_became_peered":"2023-09-23T20:32:33.236087+0000","last_unstale":"2023-09-23T20:35:25.629474+0000","last_undegraded":"2023-09-23T20:35:25.629474+0000","last_fullsized":"2023-09-23T20:35:25.629474+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:20:58.176854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":60,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-23T20:35:25.620964+0000","last_change":"2023-09-23T20:32:33.238290+0000","last_active":"2023-09-23T20:35:25.620964+0000","last_peered":"2023-09-23T20:35:25.620964+0000","last_clean":"2023-09-23T20:35:25.620964+0000","last_became_active":"2023-09-23T20:32:33.237984+0000","last_became_peered":"2023-09-23T20:32:33.237984+0000","last_unstale":"2023-09-23T20:35:25.620964+0000","last_undegraded":"2023-09-23T20:35:25.620964+0000","last_fullsized":"2023-09-23T20:35:25.620964+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:49:02.089307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"90'161","reported_seq":584,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T20:35:26.659280+0000","last_change":"2023-09-23T20:35:26.651952+0000","last_active":"2023-09-23T20:35:26.659280+0000","last_peered":"2023-09-23T20:35:26.659280+0000","last_clean":"2023-09-23T20:35:26.659280+0000","last_became_active":"2023-09-23T20:35:26.651601+0000","last_became_peered":"2023-09-23T20:35:26.651601+0000","last_unstale":"2023-09-23T20:35:26.659280+0000","last_undegraded":"2023-09-23T20:35:26.659280+0000","last_fullsized":"2023-09-23T20:35:26.659280+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:28:05.872074+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:28:05.872074+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":328,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-23T20:35:25.628127+0000","last_change":"2023-09-23T20:32:33.237234+0000","last_active":"2023-09-23T20:35:25.628127+0000","last_peered":"2023-09-23T20:35:25.628127+0000","last_clean":"2023-09-23T20:35:25.628127+0000","last_became_active":"2023-09-23T20:32:33.236881+0000","last_became_peered":"2023-09-23T20:32:33.236881+0000","last_unstale":"2023-09-23T20:35:25.628127+0000","last_undegraded":"2023-09-23T20:35:25.628127+0000","last_fullsized":"2023-09-23T20:35:25.628127+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:56:17.384831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":215,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-23T20:35:25.622114+0000","last_change":"2023-09-23T20:29:45.538314+0000","last_active":"2023-09-23T20:35:25.622114+0000","last_peered":"2023-09-23T20:35:25.622114+0000","last_clean":"2023-09-23T20:35:25.622114+0000","last_became_active":"2023-09-23T20:29:45.536847+0000","last_became_peered":"2023-09-23T20:29:45.536847+0000","last_unstale":"2023-09-23T20:35:25.622114+0000","last_undegraded":"2023-09-23T20:35:25.622114+0000","last_fullsized":"2023-09-23T20:35:25.622114+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:51.948721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"90'161","reported_seq":437,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T20:35:26.665921+0000","last_change":"2023-09-23T20:35:26.665921+0000","last_active":"2023-09-23T20:35:26.665921+0000","last_peered":"2023-09-23T20:35:26.665921+0000","last_clean":"2023-09-23T20:35:26.665921+0000","last_became_active":"2023-09-23T20:35:26.665630+0000","last_became_peered":"2023-09-23T20:35:26.665630+0000","last_unstale":"2023-09-23T20:35:26.665921+0000","last_undegraded":"2023-09-23T20:35:26.665921+0000","last_fullsized":"2023-09-23T20:35:26.665921+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:44.386120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:44.386120+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":40253,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":372,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-23T20:35:25.624028+0000","last_change":"2023-09-23T20:32:33.235987+0000","last_active":"2023-09-23T20:35:25.624028+0000","last_peered":"2023-09-23T20:35:25.624028+0000","last_clean":"2023-09-23T20:35:25.624028+0000","last_became_active":"2023-09-23T20:32:33.226544+0000","last_became_peered":"2023-09-23T20:32:33.226544+0000","last_unstale":"2023-09-23T20:35:25.624028+0000","last_undegraded":"2023-09-23T20:35:25.624028+0000","last_fullsized":"2023-09-23T20:35:25.624028+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:53:22.794856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":19,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-23T20:35:25.621390+0000","last_change":"2023-09-23T20:34:59.987575+0000","last_active":"2023-09-23T20:35:25.621390+0000","last_peered":"2023-09-23T20:35:25.621390+0000","last_clean":"2023-09-23T20:35:25.621390+0000","last_became_active":"2023-09-23T20:34:59.987229+0000","last_became_peered":"2023-09-23T20:34:59.987229+0000","last_unstale":"2023-09-23T20:35:25.621390+0000","last_undegraded":"2023-09-23T20:35:25.621390+0000","last_fullsized":"2023-09-23T20:35:25.621390+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:34:01.820159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":380,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-23T20:35:25.625988+0000","last_change":"2023-09-23T20:32:33.268260+0000","last_active":"2023-09-23T20:35:25.625988+0000","last_peered":"2023-09-23T20:35:25.625988+0000","last_clean":"2023-09-23T20:35:25.625988+0000","last_became_active":"2023-09-23T20:32:33.233695+0000","last_became_peered":"2023-09-23T20:32:33.233695+0000","last_unstale":"2023-09-23T20:35:25.625988+0000","last_undegraded":"2023-09-23T20:35:25.625988+0000","last_fullsized":"2023-09-23T20:35:25.625988+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:19:29.299623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":19,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-23T20:35:25.621075+0000","last_change":"2023-09-23T20:34:59.987416+0000","last_active":"2023-09-23T20:35:25.621075+0000","last_peered":"2023-09-23T20:35:25.621075+0000","last_clean":"2023-09-23T20:35:25.621075+0000","last_became_active":"2023-09-23T20:34:59.987117+0000","last_became_peered":"2023-09-23T20:34:59.987117+0000","last_unstale":"2023-09-23T20:35:25.621075+0000","last_undegraded":"2023-09-23T20:35:25.621075+0000","last_fullsized":"2023-09-23T20:35:25.621075+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:36:16.318623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"90'161","reported_seq":542,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T20:35:26.643586+0000","last_change":"2023-09-23T20:35:26.643586+0000","last_active":"2023-09-23T20:35:26.643586+0000","last_peered":"2023-09-23T20:35:26.643586+0000","last_clean":"2023-09-23T20:35:26.643586+0000","last_became_active":"2023-09-23T20:35:26.643160+0000","last_became_peered":"2023-09-23T20:35:26.643160+0000","last_unstale":"2023-09-23T20:35:26.643586+0000","last_undegraded":"2023-09-23T20:35:26.643586+0000","last_fullsized":"2023-09-23T20:35:26.643586+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:41.701171+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:41.701171+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0019955509999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":54614,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":178,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-23T20:35:25.626771+0000","last_change":"2023-09-23T20:32:50.537447+0000","last_active":"2023-09-23T20:35:25.626771+0000","last_peered":"2023-09-23T20:35:25.626771+0000","last_clean":"2023-09-23T20:35:25.626771+0000","last_became_active":"2023-09-23T20:32:50.537136+0000","last_became_peered":"2023-09-23T20:32:50.537136+0000","last_unstale":"2023-09-23T20:35:25.626771+0000","last_undegraded":"2023-09-23T20:35:25.626771+0000","last_fullsized":"2023-09-23T20:35:25.626771+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:46:51.599492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":78,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-23T20:35:25.621534+0000","last_change":"2023-09-23T20:32:50.537450+0000","last_active":"2023-09-23T20:35:25.621534+0000","last_peered":"2023-09-23T20:35:25.621534+0000","last_clean":"2023-09-23T20:35:25.621534+0000","last_became_active":"2023-09-23T20:32:50.537128+0000","last_became_peered":"2023-09-23T20:32:50.537128+0000","last_unstale":"2023-09-23T20:35:25.621534+0000","last_undegraded":"2023-09-23T20:35:25.621534+0000","last_fullsized":"2023-09-23T20:35:25.621534+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:37:15.889029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1d","version":"90'161","reported_seq":441,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T20:35:26.645398+0000","last_change":"2023-09-23T20:35:26.645398+0000","last_active":"2023-09-23T20:35:26.645398+0000","last_peered":"2023-09-23T20:35:26.645398+0000","last_clean":"2023-09-23T20:35:26.645398+0000","last_became_active":"2023-09-23T20:35:26.644707+0000","last_became_peered":"2023-09-23T20:35:26.644707+0000","last_unstale":"2023-09-23T20:35:26.645398+0000","last_undegraded":"2023-09-23T20:35:26.645398+0000","last_fullsized":"2023-09-23T20:35:26.645398+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:46.388545+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:46.388545+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":40253,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":345,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-23T20:35:25.628781+0000","last_change":"2023-09-23T20:25:39.668882+0000","last_active":"2023-09-23T20:35:25.628781+0000","last_peered":"2023-09-23T20:35:25.628781+0000","last_clean":"2023-09-23T20:35:25.628781+0000","last_became_active":"2023-09-23T20:25:39.637128+0000","last_became_peered":"2023-09-23T20:25:39.637128+0000","last_unstale":"2023-09-23T20:35:25.628781+0000","last_undegraded":"2023-09-23T20:35:25.628781+0000","last_fullsized":"2023-09-23T20:35:25.628781+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:56.406298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":60,"reported_epoch":362,"state":"active+clean","last_fresh":"2023-09-23T20:35:25.620583+0000","last_change":"2023-09-23T20:32:33.232281+0000","last_active":"2023-09-23T20:35:25.620583+0000","last_peered":"2023-09-23T20:35:25.620583+0000","last_clean":"2023-09-23T20:35:25.620583+0000","last_became_active":"2023-09-23T20:32:33.232003+0000","last_became_peered":"2023-09-23T20:32:33.232003+0000","last_unstale":"2023-09-23T20:35:25.620583+0000","last_undegraded":"2023-09-23T20:35:25.620583+0000","last_fullsized":"2023-09-23T20:35:25.620583+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:58:27.314355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":872939544,"num_objects":13321,"num_object_clones":0,"num_object_copies":39963,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1654,"num_objects_unfound":0,"num_objects_dirty":13321,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":175779,"num_write_kb":6051137,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10914,"num_bytes_recovered":652214272,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3063427072,"data_stored":3063414856,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":110922,"ondisk_log_size":110922,"up":48,"acting":47,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":49,"num_bytes_recovered":6522,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26691,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":492674,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":100,"num_read_kb":189,"num_write":51,"num_write_kb":495,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":624960,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":58,"num_bytes_recovered":4194342,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6082,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201572,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":615768,"kb_used_data":314704,"kb_used_omap":0,"kb_used_meta":301056,"kb_avail":93129384,"statfs":{"total":95995035648,"available":95364489216,"internally_reserved":0,"allocated":322256896,"data_stored":321099815,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308281344},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:34:43 2023","interfaces":[{"interface":"back","average":{"1min":2.323,"5min":2.0470000000000002,"15min":1.617},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.20799999999999999},"max":{"1min":6.1959999999999997,"5min":14.66,"15min":48.798999999999999},"last":0.35899999999999999},{"interface":"front","average":{"1min":2.871,"5min":2.0950000000000002,"15min":1.702},"min":{"1min":0.25,"5min":0.25,"15min":0.24399999999999999},"max":{"1min":14.605,"5min":14.605,"15min":48.247},"last":0.45000000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:34:54 2023","interfaces":[{"interface":"back","average":{"1min":2.3319999999999999,"5min":1.984,"15min":1.8360000000000001},"min":{"1min":0.29999999999999999,"5min":0.254,"15min":0.254},"max":{"1min":5.5030000000000001,"5min":14.026,"15min":14.026},"last":0.47399999999999998},{"interface":"front","average":{"1min":2.5139999999999998,"5min":1.8340000000000001,"15min":1.9319999999999999},"min":{"1min":0.32000000000000001,"5min":0.248,"15min":0.248},"max":{"1min":6.7220000000000004,"5min":6.7220000000000004,"15min":6.7220000000000004},"last":0.46800000000000003}]},{"osd":2,"last update":"Sat Sep 23 20:34:43 2023","interfaces":[{"interface":"back","average":{"1min":2.6739999999999999,"5min":2.4159999999999999,"15min":1.6799999999999999},"min":{"1min":0.34100000000000003,"5min":0.28599999999999998,"15min":0.27600000000000002},"max":{"1min":7.9029999999999996,"5min":13.173999999999999,"15min":13.173999999999999},"last":4.1879999999999997},{"interface":"front","average":{"1min":2.6280000000000001,"5min":2.7320000000000002,"15min":1.9039999999999999},"min":{"1min":0.36899999999999999,"5min":0.27400000000000002,"15min":0.26600000000000001},"max":{"1min":9.7230000000000008,"5min":23.494,"15min":23.494},"last":2.7519999999999998}]},{"osd":3,"last update":"Sat Sep 23 20:35:11 2023","interfaces":[{"interface":"back","average":{"1min":4.8280000000000003,"5min":2.8500000000000001,"15min":1.9850000000000001},"min":{"1min":0.375,"5min":0.33700000000000002,"15min":0.26200000000000001},"max":{"1min":14.789,"5min":17.155999999999999,"15min":17.155999999999999},"last":7.3630000000000004},{"interface":"front","average":{"1min":4.819,"5min":2.952,"15min":1.9179999999999999},"min":{"1min":0.46200000000000002,"5min":0.34799999999999998,"15min":0.27600000000000002},"max":{"1min":14.805,"5min":16.960000000000001,"15min":16.960000000000001},"last":7.3719999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:35:11 2023","interfaces":[{"interface":"back","average":{"1min":1.248,"5min":0.81699999999999995,"15min":1.101},"min":{"1min":0.254,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":5.0460000000000003,"5min":5.8819999999999997,"15min":15.177},"last":0.27500000000000002},{"interface":"front","average":{"1min":1.389,"5min":0.91700000000000004,"15min":1.2350000000000001},"min":{"1min":0.32400000000000001,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":6.1779999999999999,"5min":6.1779999999999999,"15min":11.122},"last":0.374}]},{"osd":5,"last update":"Sat Sep 23 20:35:11 2023","interfaces":[{"interface":"back","average":{"1min":0.71299999999999997,"5min":0.70299999999999996,"15min":1.145},"min":{"1min":0.27600000000000002,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":4.133,"5min":4.133,"15min":13.241},"last":0.32200000000000001},{"interface":"front","average":{"1min":0.90200000000000002,"5min":0.66100000000000003,"15min":1.2170000000000001},"min":{"1min":0.32400000000000001,"5min":0.32200000000000001,"15min":0.315},"max":{"1min":4.7590000000000003,"5min":5.7869999999999999,"15min":18.831},"last":0.438}]},{"osd":6,"last update":"Sat Sep 23 20:35:11 2023","interfaces":[{"interface":"back","average":{"1min":0.85299999999999998,"5min":0.67800000000000005,"15min":1.1160000000000001},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":4.819,"5min":5.7629999999999999,"15min":13.185},"last":0.40100000000000002},{"interface":"front","average":{"1min":1.2649999999999999,"5min":0.94299999999999995,"15min":1.137},"min":{"1min":0.35299999999999998,"5min":0.318,"15min":0.29899999999999999},"max":{"1min":6.2770000000000001,"5min":6.2770000000000001,"15min":9.9120000000000008},"last":0.42499999999999999}]}]},{"osd":6,"up_from":47,"seq":201863463208,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":816144,"kb_used_data":541768,"kb_used_omap":0,"kb_used_meta":274368,"kb_avail":92929008,"statfs":{"total":95995035648,"available":95159304192,"internally_reserved":0,"allocated":554770432,"data_stored":553620896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":280952832},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:34:41 2023","interfaces":[{"interface":"back","average":{"1min":3.9790000000000001,"5min":2.5270000000000001,"15min":1.589},"min":{"1min":0.24299999999999999,"5min":0.22800000000000001,"15min":0.221},"max":{"1min":34.143999999999998,"5min":53.701999999999998,"15min":53.701999999999998},"last":4.8959999999999999},{"interface":"front","average":{"1min":4.202,"5min":3.004,"15min":1.9390000000000001},"min":{"1min":0.253,"5min":0.253,"15min":0.25},"max":{"1min":34.197000000000003,"5min":53.875999999999998,"15min":53.875999999999998},"last":0.29399999999999998}]},{"osd":1,"last update":"Sat Sep 23 20:34:57 2023","interfaces":[{"interface":"back","average":{"1min":3.4510000000000001,"5min":2.2719999999999998,"15min":2.5910000000000002},"min":{"1min":0.25600000000000001,"5min":0.249,"15min":0.249},"max":{"1min":34.173999999999999,"5min":34.173999999999999,"15min":34.173999999999999},"last":0.309},{"interface":"front","average":{"1min":4.29,"5min":2.569,"15min":3.181},"min":{"1min":0.32900000000000001,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":34.490000000000002,"5min":34.490000000000002,"15min":34.490000000000002},"last":4.0949999999999998}]},{"osd":2,"last update":"Sat Sep 23 20:34:41 2023","interfaces":[{"interface":"back","average":{"1min":3.879,"5min":2.5379999999999998,"15min":1.8080000000000001},"min":{"1min":0.29599999999999999,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":34.317999999999998,"5min":34.317999999999998,"15min":34.317999999999998},"last":0.54400000000000004},{"interface":"front","average":{"1min":4.1390000000000002,"5min":2.5390000000000001,"15min":1.7350000000000001},"min":{"1min":0.35399999999999998,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":34.292999999999999,"5min":34.292999999999999,"15min":34.292999999999999},"last":0.52300000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:35:25 2023","interfaces":[{"interface":"back","average":{"1min":2.0099999999999998,"5min":2.0369999999999999,"15min":1.7050000000000001},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29599999999999999},"max":{"1min":6.46,"5min":7.5620000000000003,"15min":9.4239999999999995},"last":4.2960000000000003},{"interface":"front","average":{"1min":2.8290000000000002,"5min":2.4140000000000001,"15min":1.851},"min":{"1min":0.39400000000000002,"5min":0.317,"15min":0.317},"max":{"1min":8.5600000000000005,"5min":12.875999999999999,"15min":12.875999999999999},"last":0.434}]},{"osd":4,"last update":"Sat Sep 23 20:35:02 2023","interfaces":[{"interface":"back","average":{"1min":0.89200000000000002,"5min":0.58599999999999997,"15min":0.89500000000000002},"min":{"1min":0.27400000000000002,"5min":0.251,"15min":0.251},"max":{"1min":4.9279999999999999,"5min":4.9279999999999999,"15min":10.173},"last":0.32500000000000001},{"interface":"front","average":{"1min":0.77200000000000002,"5min":0.84699999999999998,"15min":1.0620000000000001},"min":{"1min":0.29099999999999998,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":3.1139999999999999,"5min":7.7629999999999999,"15min":8.3249999999999993},"last":0.60399999999999998}]},{"osd":5,"last update":"Sat Sep 23 20:35:02 2023","interfaces":[{"interface":"back","average":{"1min":0.70599999999999996,"5min":0.66400000000000003,"15min":0.95199999999999996},"min":{"1min":0.31,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":4.8259999999999996,"5min":6.8310000000000004,"15min":14.085000000000001},"last":1.1040000000000001},{"interface":"front","average":{"1min":0.69199999999999995,"5min":0.84199999999999997,"15min":1.3100000000000001},"min":{"1min":0.32000000000000001,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":3.0920000000000001,"5min":7.7300000000000004,"15min":18.192},"last":0.40100000000000002}]},{"osd":7,"last update":"Sat Sep 23 20:35:25 2023","interfaces":[{"interface":"back","average":{"1min":0.82599999999999996,"5min":0.69799999999999995,"15min":1.085},"min":{"1min":0.36599999999999999,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":4.851,"5min":5.0350000000000001,"15min":10.217000000000001},"last":0.49299999999999999},{"interface":"front","average":{"1min":0.93500000000000005,"5min":0.93999999999999995,"15min":1.2509999999999999},"min":{"1min":0.35999999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":5.6820000000000004,"5min":7.601,"15min":9.7360000000000007},"last":1.2649999999999999}]}]},{"osd":1,"up_from":266,"seq":1142461300806,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":649500,"kb_used_data":398036,"kb_used_omap":8,"kb_used_meta":251319,"kb_avail":93095652,"statfs":{"total":95995035648,"available":95329947648,"internally_reserved":0,"allocated":407588864,"data_stored":405613897,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8206,"internal_metadata":257351666},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:34:54 2023","interfaces":[{"interface":"back","average":{"1min":1.9470000000000001,"5min":2.9750000000000001,"15min":2.5449999999999999},"min":{"1min":0.221,"5min":0.218,"15min":0.218},"max":{"1min":8.6080000000000005,"5min":52.689999999999998,"15min":52.689999999999998},"last":0.498},{"interface":"front","average":{"1min":5.3540000000000001,"5min":3.4809999999999999,"15min":3.2909999999999999},"min":{"1min":0.20599999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":26.588000000000001,"5min":52.662999999999997,"15min":52.662999999999997},"last":0.19900000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:34:54 2023","interfaces":[{"interface":"back","average":{"1min":7.2110000000000003,"5min":4.069,"15min":4.7729999999999997},"min":{"1min":0.312,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":28.359000000000002,"5min":28.359000000000002,"15min":28.359000000000002},"last":0.71099999999999997},{"interface":"front","average":{"1min":3.4550000000000001,"5min":2.6179999999999999,"15min":2.2679999999999998},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":12.801,"5min":12.801,"15min":12.801},"last":0.44}]},{"osd":3,"last update":"Sat Sep 23 20:34:54 2023","interfaces":[{"interface":"back","average":{"1min":4.1239999999999997,"5min":4.3799999999999999,"15min":4.0380000000000003},"min":{"1min":0.31,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":15.247999999999999,"5min":80.745999999999995,"15min":80.745999999999995},"last":0.47299999999999998},{"interface":"front","average":{"1min":5.3380000000000001,"5min":4.6689999999999996,"15min":4.7869999999999999},"min":{"1min":0.35799999999999998,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":17.893999999999998,"5min":82.988,"15min":82.988},"last":2.1829999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:34:54 2023","interfaces":[{"interface":"back","average":{"1min":4.1449999999999996,"5min":2.2989999999999999,"15min":2.1520000000000001},"min":{"1min":0.34200000000000003,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":24.327999999999999,"5min":24.327999999999999,"15min":24.327999999999999},"last":0.52800000000000002},{"interface":"front","average":{"1min":1.2889999999999999,"5min":1.6759999999999999,"15min":1.9079999999999999},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":4.7889999999999997,"5min":10.760999999999999,"15min":10.760999999999999},"last":0.66000000000000003}]},{"osd":5,"last update":"Sat Sep 23 20:34:54 2023","interfaces":[{"interface":"back","average":{"1min":3.3170000000000002,"5min":2.1760000000000002,"15min":2.593},"min":{"1min":0.45000000000000001,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":7.6909999999999998,"5min":9.6479999999999997,"15min":9.6479999999999997},"last":2.1240000000000001},{"interface":"front","average":{"1min":4.1059999999999999,"5min":2.4809999999999999,"15min":2.2200000000000002},"min":{"1min":0.38400000000000001,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":24.404,"5min":24.404,"15min":24.404},"last":0.56899999999999995}]},{"osd":6,"last update":"Sat Sep 23 20:34:54 2023","interfaces":[{"interface":"back","average":{"1min":3.3250000000000002,"5min":3.1059999999999999,"15min":6.0010000000000003},"min":{"1min":0.51200000000000001,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":7.7199999999999998,"5min":72.682000000000002,"15min":72.682000000000002},"last":2.0840000000000001},{"interface":"front","average":{"1min":4.2489999999999997,"5min":3.4180000000000001,"15min":5.4320000000000004},"min":{"1min":0.41099999999999998,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":24.376000000000001,"5min":72.563999999999993,"15min":72.563999999999993},"last":0.64600000000000002}]},{"osd":7,"last update":"Sat Sep 23 20:34:54 2023","interfaces":[{"interface":"back","average":{"1min":3.4489999999999998,"5min":2.3759999999999999,"15min":2.98},"min":{"1min":0.54400000000000004,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":9.0830000000000002,"5min":9.6850000000000005,"15min":9.6850000000000005},"last":2.1509999999999998},{"interface":"front","average":{"1min":4.2519999999999998,"5min":2.6230000000000002,"15min":2.4470000000000001},"min":{"1min":0.441,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":24.524000000000001,"5min":24.524000000000001,"15min":24.524000000000001},"last":0.59999999999999998}]}]},{"osd":0,"up_from":94,"seq":403726926009,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":878520,"kb_used_data":420272,"kb_used_omap":7,"kb_used_meta":458168,"kb_avail":92866632,"statfs":{"total":95995035648,"available":95095431168,"internally_reserved":0,"allocated":430358528,"data_stored":428534377,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":469164049},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:35:02 2023","interfaces":[{"interface":"back","average":{"1min":3.8130000000000002,"5min":3.5190000000000001,"15min":3.7639999999999998},"min":{"1min":0.183,"5min":0.17999999999999999,"15min":0.17999999999999999},"max":{"1min":12.301,"5min":42.000999999999998,"15min":42.000999999999998},"last":0.39700000000000002},{"interface":"front","average":{"1min":4.5780000000000003,"5min":3.585,"15min":4.1660000000000004},"min":{"1min":0.247,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":18.355,"5min":23.036000000000001,"15min":23.036000000000001},"last":0.432}]},{"osd":2,"last update":"Sat Sep 23 20:34:35 2023","interfaces":[{"interface":"back","average":{"1min":3.2410000000000001,"5min":3.4409999999999998,"15min":2.258},"min":{"1min":0.26400000000000001,"5min":0.185,"15min":0.185},"max":{"1min":9.3160000000000007,"5min":77.448999999999998,"15min":77.448999999999998},"last":1.488},{"interface":"front","average":{"1min":3.1269999999999998,"5min":3.3849999999999998,"15min":2.097},"min":{"1min":0.32300000000000001,"5min":0.188,"15min":0.188},"max":{"1min":13.961,"5min":81.724999999999994,"15min":81.724999999999994},"last":0.26600000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:35:07 2023","interfaces":[{"interface":"back","average":{"1min":5.1449999999999996,"5min":4.0899999999999999,"15min":2.5219999999999998},"min":{"1min":0.27500000000000002,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":17.692,"5min":81.400999999999996,"15min":81.400999999999996},"last":1.913},{"interface":"front","average":{"1min":5.085,"5min":4.7759999999999998,"15min":2.6200000000000001},"min":{"1min":0.30299999999999999,"5min":0.25700000000000001,"15min":0.23699999999999999},"max":{"1min":12.262,"5min":81.137,"15min":81.137},"last":0.60199999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:34:35 2023","interfaces":[{"interface":"back","average":{"1min":2.1680000000000001,"5min":1.927,"15min":1.752},"min":{"1min":0.30599999999999999,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":9.407,"5min":12.587999999999999,"15min":23.876999999999999},"last":1.5609999999999999},{"interface":"front","average":{"1min":3.0289999999999999,"5min":2.194,"15min":1.673},"min":{"1min":0.308,"5min":0.308,"15min":0.29299999999999998},"max":{"1min":10.380000000000001,"5min":23.001000000000001,"15min":23.681999999999999},"last":0.53600000000000003}]},{"osd":5,"last update":"Sat Sep 23 20:34:35 2023","interfaces":[{"interface":"back","average":{"1min":3.0609999999999999,"5min":2.246,"15min":2},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34200000000000003},"max":{"1min":10.449,"5min":22.969999999999999,"15min":24.198},"last":0.5},{"interface":"front","average":{"1min":2.0430000000000001,"5min":2.1989999999999998,"15min":1.855},"min":{"1min":0.307,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":7.3520000000000003,"5min":22.236000000000001,"15min":22.236000000000001},"last":0.47699999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:34:35 2023","interfaces":[{"interface":"back","average":{"1min":3.3010000000000002,"5min":2.2759999999999998,"15min":1.7549999999999999},"min":{"1min":0.372,"5min":0.32300000000000001,"15min":0.318},"max":{"1min":10.423,"5min":22.934999999999999,"15min":24.408000000000001},"last":0.57999999999999996},{"interface":"front","average":{"1min":2.0369999999999999,"5min":2.1920000000000002,"15min":1.6990000000000001},"min":{"1min":0.38400000000000001,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":7.3979999999999997,"5min":22.164999999999999,"15min":25.271000000000001},"last":0.51700000000000002}]},{"osd":7,"last update":"Sat Sep 23 20:34:35 2023","interfaces":[{"interface":"back","average":{"1min":2.052,"5min":2.3300000000000001,"15min":1.8779999999999999},"min":{"1min":0.36099999999999999,"5min":0.32200000000000001,"15min":0.30599999999999999},"max":{"1min":7.4320000000000004,"5min":22.206,"15min":25.359000000000002},"last":0.54400000000000004},{"interface":"front","average":{"1min":2.2189999999999999,"5min":2.339,"15min":2.1120000000000001},"min":{"1min":0.38100000000000001,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":9.3710000000000004,"5min":22.422999999999998,"15min":24.669},"last":1.5289999999999999}]}]},{"osd":2,"up_from":104,"seq":446676598956,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":859248,"kb_used_data":357864,"kb_used_omap":11,"kb_used_meta":501364,"kb_avail":92885904,"statfs":{"total":95995035648,"available":95115165696,"internally_reserved":0,"allocated":366452736,"data_stored":364633798,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11878,"internal_metadata":513397146},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:34:44 2023","interfaces":[{"interface":"back","average":{"1min":7.0650000000000004,"5min":4.1500000000000004,"15min":2.3439999999999999},"min":{"1min":0.252,"5min":0.185,"15min":0.17999999999999999},"max":{"1min":28.234999999999999,"5min":82.424999999999997,"15min":82.424999999999997},"last":1.47},{"interface":"front","average":{"1min":4.3920000000000003,"5min":3.9169999999999998,"15min":2.073},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":9.6370000000000005,"5min":82.524000000000001,"15min":82.524000000000001},"last":4.5190000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:34:52 2023","interfaces":[{"interface":"back","average":{"1min":2.7080000000000002,"5min":3.0699999999999998,"15min":4.2569999999999997},"min":{"1min":0.27100000000000002,"5min":0.221,"15min":0.221},"max":{"1min":10.205,"5min":16.280000000000001,"15min":16.280000000000001},"last":0.29299999999999998},{"interface":"front","average":{"1min":4.1100000000000003,"5min":2.8029999999999999,"15min":2.9260000000000002},"min":{"1min":0.36699999999999999,"5min":0.248,"15min":0.248},"max":{"1min":13.130000000000001,"5min":30.565000000000001,"15min":30.565000000000001},"last":0.80600000000000005}]},{"osd":3,"last update":"Sat Sep 23 20:35:13 2023","interfaces":[{"interface":"back","average":{"1min":2.6520000000000001,"5min":2.9529999999999998,"15min":1.905},"min":{"1min":0.32700000000000001,"5min":0.27800000000000002,"15min":0.19800000000000001},"max":{"1min":12.664,"5min":15.804,"15min":16.331},"last":0.32700000000000001},{"interface":"front","average":{"1min":4.577,"5min":3.5139999999999998,"15min":2.242},"min":{"1min":0.33400000000000002,"5min":0.27300000000000002,"15min":0.23899999999999999},"max":{"1min":13.157,"5min":30.704999999999998,"15min":30.704999999999998},"last":9.8729999999999993}]},{"osd":4,"last update":"Sat Sep 23 20:34:44 2023","interfaces":[{"interface":"back","average":{"1min":2.1549999999999998,"5min":2.266,"15min":1.5680000000000001},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.28100000000000003},"max":{"1min":7.6879999999999997,"5min":16.797999999999998,"15min":16.797999999999998},"last":0.42299999999999999},{"interface":"front","average":{"1min":3.1139999999999999,"5min":2.4209999999999998,"15min":1.73},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.27600000000000002},"max":{"1min":15.853999999999999,"5min":15.853999999999999,"15min":15.853999999999999},"last":0.38300000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:34:44 2023","interfaces":[{"interface":"back","average":{"1min":2.5270000000000001,"5min":2.4409999999999998,"15min":2.1059999999999999},"min":{"1min":0.46800000000000003,"5min":0.314,"15min":0.309},"max":{"1min":7.7190000000000003,"5min":16.766999999999999,"15min":95.393000000000001},"last":0.36599999999999999},{"interface":"front","average":{"1min":3.7930000000000001,"5min":2.827,"15min":2.3359999999999999},"min":{"1min":0.36799999999999999,"5min":0.34799999999999998,"15min":0.32300000000000001},"max":{"1min":15.885,"5min":15.885,"15min":95.414000000000001},"last":0.35299999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:34:44 2023","interfaces":[{"interface":"back","average":{"1min":3.1120000000000001,"5min":2.694,"15min":1.829},"min":{"1min":0.36599999999999999,"5min":0.33600000000000002,"15min":0.32100000000000001},"max":{"1min":15.914,"5min":15.914,"15min":15.914},"last":0.40999999999999998},{"interface":"front","average":{"1min":3.6179999999999999,"5min":2.3220000000000001,"15min":1.8140000000000001},"min":{"1min":0.40400000000000003,"5min":0.316,"15min":0.316},"max":{"1min":9.2370000000000001,"5min":30.670999999999999,"15min":30.670999999999999},"last":0.84599999999999997}]},{"osd":7,"last update":"Sat Sep 23 20:34:44 2023","interfaces":[{"interface":"back","average":{"1min":3.3879999999999999,"5min":2.444,"15min":1.8839999999999999},"min":{"1min":0.372,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":9.2759999999999998,"5min":30.741,"15min":30.741},"last":0.875},{"interface":"front","average":{"1min":2.758,"5min":2.645,"15min":1.8380000000000001},"min":{"1min":0.495,"5min":0.38,"15min":0.38},"max":{"1min":7.7549999999999999,"5min":16.719000000000001,"15min":16.719000000000001},"last":0.46600000000000003}]}]},{"osd":3,"up_from":109,"seq":468151435430,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1133096,"kb_used_data":589152,"kb_used_omap":4,"kb_used_meta":543931,"kb_avail":92612056,"statfs":{"total":95995035648,"available":94834745344,"internally_reserved":0,"allocated":603291648,"data_stored":601478524,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4514,"internal_metadata":556985950},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:35:11 2023","interfaces":[{"interface":"back","average":{"1min":3.742,"5min":3.242,"15min":2.2770000000000001},"min":{"1min":0.215,"5min":0.191,"15min":0.186},"max":{"1min":17.007000000000001,"5min":17.129000000000001,"15min":67.305999999999997},"last":5.7759999999999998},{"interface":"front","average":{"1min":3.3140000000000001,"5min":3.0550000000000002,"15min":2.1190000000000002},"min":{"1min":0.23699999999999999,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":9.5380000000000003,"5min":12.781000000000001,"15min":67.403999999999996},"last":2.6509999999999998}]},{"osd":1,"last update":"Sat Sep 23 20:34:55 2023","interfaces":[{"interface":"back","average":{"1min":2.7589999999999999,"5min":2.7189999999999999,"15min":2.9649999999999999},"min":{"1min":0.28799999999999998,"5min":0.222,"15min":0.222},"max":{"1min":14.641,"5min":16.189,"15min":16.189},"last":1.216},{"interface":"front","average":{"1min":2.1139999999999999,"5min":2.6299999999999999,"15min":3.0449999999999999},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":10.811,"5min":32.603999999999999,"15min":32.603999999999999},"last":0.441}]},{"osd":2,"last update":"Sat Sep 23 20:35:11 2023","interfaces":[{"interface":"back","average":{"1min":2.883,"5min":3.4980000000000002,"15min":2.2589999999999999},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.22800000000000001},"max":{"1min":9.5879999999999992,"5min":32.857999999999997,"15min":32.857999999999997},"last":0.48099999999999998},{"interface":"front","average":{"1min":3.4369999999999998,"5min":3.1699999999999999,"15min":2.165},"min":{"1min":0.251,"5min":0.246,"15min":0.20899999999999999},"max":{"1min":14.492000000000001,"5min":24.146999999999998,"15min":24.146999999999998},"last":1.96}]},{"osd":4,"last update":"Sat Sep 23 20:35:11 2023","interfaces":[{"interface":"back","average":{"1min":1.5669999999999999,"5min":1.9750000000000001,"15min":1.698},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.29799999999999999},"max":{"1min":5.3929999999999998,"5min":12.151,"15min":46.042999999999999},"last":0.46000000000000002},{"interface":"front","average":{"1min":2.6949999999999998,"5min":2.6339999999999999,"15min":2.0579999999999998},"min":{"1min":0.308,"5min":0.29399999999999998,"15min":0.28100000000000003},"max":{"1min":14.422000000000001,"5min":19.635000000000002,"15min":46.579999999999998},"last":2.0750000000000002}]},{"osd":5,"last update":"Sat Sep 23 20:35:11 2023","interfaces":[{"interface":"back","average":{"1min":2.056,"5min":2.2120000000000002,"15min":1.9399999999999999},"min":{"1min":0.36699999999999999,"5min":0.32200000000000001,"15min":0.311},"max":{"1min":13.228999999999999,"5min":13.228999999999999,"15min":45.292000000000002},"last":0.5},{"interface":"front","average":{"1min":1.647,"5min":2.1190000000000002,"15min":1.764},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.313},"max":{"1min":8.782,"5min":11.028,"15min":44.679000000000002},"last":0.55700000000000005}]},{"osd":6,"last update":"Sat Sep 23 20:35:11 2023","interfaces":[{"interface":"back","average":{"1min":1.4590000000000001,"5min":2.266,"15min":1.79},"min":{"1min":0.40500000000000003,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":5.2640000000000002,"5min":17.524999999999999,"15min":17.524999999999999},"last":0.51000000000000001},{"interface":"front","average":{"1min":3.1150000000000002,"5min":2.4180000000000001,"15min":1.8100000000000001},"min":{"1min":0.41299999999999998,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":14.464,"5min":19.524000000000001,"15min":19.524000000000001},"last":2.004}]},{"osd":7,"last update":"Sat Sep 23 20:35:11 2023","interfaces":[{"interface":"back","average":{"1min":2.9740000000000002,"5min":2.6230000000000002,"15min":1.964},"min":{"1min":0.44700000000000001,"5min":0.36899999999999999,"15min":0.34999999999999998},"max":{"1min":14.521000000000001,"5min":19.571999999999999,"15min":19.571999999999999},"last":2.0379999999999998},{"interface":"front","average":{"1min":2.016,"5min":2.149,"15min":1.998},"min":{"1min":0.44600000000000001,"5min":0.42599999999999999,"15min":0.371},"max":{"1min":8.3360000000000003,"5min":12.186,"15min":45.387999999999998},"last":0.53900000000000003}]}]},{"osd":4,"up_from":30,"seq":128849019184,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":513036,"kb_used_data":257604,"kb_used_omap":0,"kb_used_meta":255424,"kb_avail":93232116,"statfs":{"total":95995035648,"available":95469686784,"internally_reserved":0,"allocated":263786496,"data_stored":262640766,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":261554176},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:34:37 2023","interfaces":[{"interface":"back","average":{"1min":2.1859999999999999,"5min":1.51,"15min":1.409},"min":{"1min":0.26200000000000001,"5min":0.255,"15min":0.222},"max":{"1min":8.0800000000000001,"5min":8.984,"15min":20.411999999999999},"last":0.27300000000000002},{"interface":"front","average":{"1min":2.2229999999999999,"5min":1.726,"15min":1.5580000000000001},"min":{"1min":0.29499999999999998,"5min":0.25800000000000001,"15min":0.253},"max":{"1min":8.0990000000000002,"5min":10.675000000000001,"15min":20.454999999999998},"last":0.39700000000000002}]},{"osd":1,"last update":"Sat Sep 23 20:34:56 2023","interfaces":[{"interface":"back","average":{"1min":1.4350000000000001,"5min":1.885,"15min":1.9670000000000001},"min":{"1min":0.27600000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":6.0869999999999997,"5min":11.303000000000001,"15min":11.303000000000001},"last":3.484},{"interface":"front","average":{"1min":1.2110000000000001,"5min":1.6299999999999999,"15min":1.8859999999999999},"min":{"1min":0.30499999999999999,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":3.9169999999999998,"5min":11.287000000000001,"15min":11.287000000000001},"last":3.4950000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:34:48 2023","interfaces":[{"interface":"back","average":{"1min":1.536,"5min":1.952,"15min":1.6579999999999999},"min":{"1min":0.314,"5min":0.311,"15min":0.26600000000000001},"max":{"1min":3.3140000000000001,"5min":10.275,"15min":32.363},"last":1.458},{"interface":"front","average":{"1min":2.5990000000000002,"5min":2.0470000000000002,"15min":1.9370000000000001},"min":{"1min":0.36899999999999999,"5min":0.28999999999999998,"15min":0.26800000000000002},"max":{"1min":5.9009999999999998,"5min":14.614000000000001,"15min":34.042999999999999},"last":8.0239999999999991}]},{"osd":3,"last update":"Sat Sep 23 20:35:09 2023","interfaces":[{"interface":"back","average":{"1min":2.085,"5min":2.2839999999999998,"15min":1.849},"min":{"1min":0.34899999999999998,"5min":0.34799999999999998,"15min":0.29099999999999998},"max":{"1min":7.2060000000000004,"5min":14.515000000000001,"15min":32.332999999999998},"last":9.5239999999999991},{"interface":"front","average":{"1min":2.5150000000000001,"5min":2.2599999999999998,"15min":1.9379999999999999},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":7.9210000000000003,"5min":15.991,"15min":32.399999999999999},"last":8.2240000000000002}]},{"osd":5,"last update":"Sat Sep 23 20:34:32 2023","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":0.98599999999999999,"15min":1.5269999999999999},"min":{"1min":0.28000000000000003,"5min":0.25800000000000001,"15min":0.247},"max":{"1min":3.0920000000000001,"5min":7.3719999999999999,"15min":82.866},"last":0.497},{"interface":"front","average":{"1min":0.70499999999999996,"5min":1.0009999999999999,"15min":1.4279999999999999},"min":{"1min":0.36699999999999999,"5min":0.30299999999999999,"15min":0.27800000000000002},"max":{"1min":3.0699999999999998,"5min":6.8520000000000003,"15min":82.546000000000006},"last":0.35899999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:34:48 2023","interfaces":[{"interface":"back","average":{"1min":1.3460000000000001,"5min":1.0640000000000001,"15min":1.3560000000000001},"min":{"1min":0.33300000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":10.178000000000001,"5min":10.178000000000001,"15min":41.371000000000002},"last":0.434},{"interface":"front","average":{"1min":0.63900000000000001,"5min":0.84699999999999998,"15min":1.2410000000000001},"min":{"1min":0.39600000000000002,"5min":0.34000000000000002,"15min":0.27200000000000002},"max":{"1min":2.3900000000000001,"5min":5.4619999999999997,"15min":41.140999999999998},"last":0.44400000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:35:09 2023","interfaces":[{"interface":"back","average":{"1min":0.86499999999999999,"5min":1.0880000000000001,"15min":1.425},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.33600000000000002},"max":{"1min":3.2970000000000002,"5min":6.8849999999999998,"15min":16.427},"last":0.46200000000000002},{"interface":"front","average":{"1min":0.65200000000000002,"5min":0.81599999999999995,"15min":1.2989999999999999},"min":{"1min":0.36399999999999999,"5min":0.33000000000000002,"15min":0.307},"max":{"1min":1.4490000000000001,"5min":6.3970000000000002,"15min":19.018999999999998},"last":0.53300000000000003}]}]},{"osd":5,"up_from":39,"seq":167503724845,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":469960,"kb_used_data":209856,"kb_used_omap":0,"kb_used_meta":260096,"kb_avail":93275192,"statfs":{"total":95995035648,"available":95513796608,"internally_reserved":0,"allocated":214892544,"data_stored":213747540,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":266338304},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:34:40 2023","interfaces":[{"interface":"back","average":{"1min":1.9930000000000001,"5min":1.7350000000000001,"15min":1.4139999999999999},"min":{"1min":0.23400000000000001,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":7.6280000000000001,"5min":8.1609999999999996,"15min":32.475000000000001},"last":6.5419999999999998},{"interface":"front","average":{"1min":2.4060000000000001,"5min":1.6799999999999999,"15min":1.4299999999999999},"min":{"1min":0.29599999999999999,"5min":0.26300000000000001,"15min":0.20599999999999999},"max":{"1min":8.4459999999999997,"5min":8.4459999999999997,"15min":32.457000000000001},"last":6.556}]},{"osd":1,"last update":"Sat Sep 23 20:34:59 2023","interfaces":[{"interface":"back","average":{"1min":2.1640000000000001,"5min":1.694,"15min":1.9339999999999999},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":9.3539999999999992,"5min":9.3539999999999992,"15min":9.3539999999999992},"last":1.091},{"interface":"front","average":{"1min":2.1070000000000002,"5min":1.516,"15min":1.546},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":9.3900000000000006,"5min":9.3900000000000006,"15min":9.3900000000000006},"last":1.004}]},{"osd":2,"last update":"Sat Sep 23 20:34:40 2023","interfaces":[{"interface":"back","average":{"1min":3.7709999999999999,"5min":2.3319999999999999,"15min":1.72},"min":{"1min":0.28000000000000003,"5min":0.27400000000000002,"15min":0.26700000000000002},"max":{"1min":27.603999999999999,"5min":27.603999999999999,"15min":34.514000000000003},"last":5.7839999999999998},{"interface":"front","average":{"1min":3.7229999999999999,"5min":2.3010000000000002,"15min":1.748},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":27.626999999999999,"5min":27.626999999999999,"15min":32.515000000000001},"last":0.60199999999999998}]},{"osd":3,"last update":"Sat Sep 23 20:35:09 2023","interfaces":[{"interface":"back","average":{"1min":2.2130000000000001,"5min":2.5499999999999998,"15min":1.8149999999999999},"min":{"1min":0.314,"5min":0.29599999999999999,"15min":0.25700000000000001},"max":{"1min":9.4770000000000003,"5min":16.189,"15min":16.189},"last":5.5369999999999999},{"interface":"front","average":{"1min":3.7400000000000002,"5min":2.2749999999999999,"15min":1.669},"min":{"1min":0.36099999999999999,"5min":0.32300000000000001,"15min":0.28799999999999998},"max":{"1min":10.191000000000001,"5min":10.577999999999999,"15min":12.005000000000001},"last":1.2}]},{"osd":4,"last update":"Sat Sep 23 20:34:30 2023","interfaces":[{"interface":"back","average":{"1min":0.51700000000000002,"5min":1.1930000000000001,"15min":1.292},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.25800000000000001},"max":{"1min":1.8979999999999999,"5min":28.015999999999998,"15min":28.015999999999998},"last":0.94899999999999995},{"interface":"front","average":{"1min":0.68200000000000005,"5min":0.93600000000000005,"15min":1.0649999999999999},"min":{"1min":0.318,"5min":0.29799999999999999,"15min":0.29099999999999998},"max":{"1min":2.9119999999999999,"5min":12.300000000000001,"15min":12.300000000000001},"last":0.312}]},{"osd":6,"last update":"Sat Sep 23 20:34:46 2023","interfaces":[{"interface":"back","average":{"1min":0.64000000000000001,"5min":0.77600000000000002,"15min":1.202},"min":{"1min":0.35399999999999998,"5min":0.309,"15min":0.30499999999999999},"max":{"1min":1.841,"5min":4.9829999999999997,"15min":12.890000000000001},"last":0.91500000000000004},{"interface":"front","average":{"1min":0.77000000000000002,"5min":0.755,"15min":1.1719999999999999},"min":{"1min":0.39900000000000002,"5min":0.371,"15min":0.34499999999999997},"max":{"1min":5.1680000000000001,"5min":5.1680000000000001,"15min":16.265999999999998},"last":0.36499999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:35:09 2023","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.68999999999999995,"15min":1.226},"min":{"1min":0.36399999999999999,"5min":0.32100000000000001,"15min":0.29699999999999999},"max":{"1min":1.946,"5min":2.77,"15min":36.621000000000002},"last":0.34399999999999997},{"interface":"front","average":{"1min":0.76100000000000001,"5min":0.72799999999999998,"15min":1.1359999999999999},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":5.093,"5min":5.093,"15min":37.331000000000003},"last":0.45300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4722,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1360,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8206,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7156,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3154,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":421072896,"data_stored":421068824,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":389353472,"data_stored":389349400,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":354811904,"data_stored":354811904,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":589234176,"data_stored":589234176,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":254869504,"data_stored":254869504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":210567168,"data_stored":210567168,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":533925888,"data_stored":533921816,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":309592064,"data_stored":309592064,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:35:39.181 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:35:39.193 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:35:39.194 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:35:39.295 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:35:39.355 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:35:39.411 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:35:39.414 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:35:39.414 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:35:39.515 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:35:39.596 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:35:39.718 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:39 smithi149 ceph-mon[170394]: pgmap v865: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 116 active+clean; 568 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 7.4 KiB/s rd, 19 KiB/s wr, 663 op/s; 1235/27153 objects misplaced (4.548%); 1.1 MiB/s, 43 objects/s recovering 2023-09-23T20:35:39.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:39 smithi018 ceph-mon[184144]: pgmap v865: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 116 active+clean; 568 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 7.4 KiB/s rd, 19 KiB/s wr, 663 op/s; 1235/27153 objects misplaced (4.548%); 1.1 MiB/s, 43 objects/s recovering 2023-09-23T20:35:39.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:39 smithi018 ceph-mon[179647]: pgmap v865: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 116 active+clean; 568 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 7.4 KiB/s rd, 19 KiB/s wr, 663 op/s; 1235/27153 objects misplaced (4.548%); 1.1 MiB/s, 43 objects/s recovering 2023-09-23T20:35:40.637 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:35:40.643 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:35:40.643 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:35:40.744 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:35:40.831 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:35:41.049 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:35:41.052 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:35:41.053 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:35:41.153 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:35:41.266 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:35:41.366 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:35:41.366 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:35:41.374 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:35:41.377 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:35:41.378 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:35:41.410 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:35:41.460 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:35:41.478 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:35:41.510 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:35:41.514 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:35:41.514 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2023-09-23T20:35:41.514 INFO:teuthology.orchestra.run.smithi018.stdout: "ops": [], 2023-09-23T20:35:41.514 INFO:teuthology.orchestra.run.smithi018.stdout: "complaint_time": 30, 2023-09-23T20:35:41.514 INFO:teuthology.orchestra.run.smithi018.stdout: "num_blocked_ops": 0 2023-09-23T20:35:41.515 INFO:teuthology.orchestra.run.smithi018.stdout:} 2023-09-23T20:35:41.553 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:35:41.555 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:35:41.559 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:35:41.559 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:35:41.594 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:35:41.648 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:41 smithi018 ceph-mon[179647]: pgmap v866: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 116 active+clean; 526 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 7.0 KiB/s rd, 18 KiB/s wr, 619 op/s; 999/25128 objects misplaced (3.976%); 1.1 MiB/s, 40 objects/s recovering 2023-09-23T20:35:41.648 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:41 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3130133248' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:35:41.648 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:41 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:35:41.649 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:41 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3155926304' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:35:41.649 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:41 smithi018 ceph-mon[184144]: pgmap v866: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 116 active+clean; 526 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 7.0 KiB/s rd, 18 KiB/s wr, 619 op/s; 999/25128 objects misplaced (3.976%); 1.1 MiB/s, 40 objects/s recovering 2023-09-23T20:35:41.649 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:41 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3130133248' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:35:41.649 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:41 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:35:41.649 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:41 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3155926304' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:35:41.650 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:35:41.660 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:35:41.711 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:35:41.724 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:35:41.724 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:35:41.724 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:35:41.724 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:35:41.725 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:35:41.725 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:35:41.725 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:35:41.725 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:35:41.825 INFO:teuthology.orchestra.run.smithi018.stderr:noscrub is unset 2023-09-23T20:35:41.867 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:35:41.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:41 smithi149 ceph-mon[170394]: pgmap v866: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 116 active+clean; 526 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 7.0 KiB/s rd, 18 KiB/s wr, 619 op/s; 999/25128 objects misplaced (3.976%); 1.1 MiB/s, 40 objects/s recovering 2023-09-23T20:35:41.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:41 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3130133248' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:35:41.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:41 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:35:41.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:41 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3155926304' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:35:42.149 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:35:42.172 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:35:42.173 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:35:42.273 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:35:42.335 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:35:42.619 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:42 smithi018 ceph-mon[179647]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:35:42.619 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:42 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:35:42.620 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:42 smithi018 ceph-mon[179647]: osdmap e364: 8 total, 8 up, 8 in 2023-09-23T20:35:42.620 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:42 smithi018 ceph-mon[179647]: failed to encode map e364 with expected crc 2023-09-23T20:35:42.620 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:42 smithi018 ceph-mon[179647]: failed to encode map e364 with expected crc 2023-09-23T20:35:42.620 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:42 smithi018 ceph-mon[179647]: pgmap v868: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 116 active+clean; 407 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 686 op/s; 779/19416 objects misplaced (4.012%); 0 B/s, 58 objects/s recovering 2023-09-23T20:35:42.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:42 smithi018 ceph-mon[184144]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:35:42.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:42 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:35:42.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:42 smithi018 ceph-mon[184144]: osdmap e364: 8 total, 8 up, 8 in 2023-09-23T20:35:42.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:42 smithi018 ceph-mon[184144]: failed to encode map e364 with expected crc 2023-09-23T20:35:42.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:42 smithi018 ceph-mon[184144]: failed to encode map e364 with expected crc 2023-09-23T20:35:42.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:42 smithi018 ceph-mon[184144]: pgmap v868: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 116 active+clean; 407 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 686 op/s; 779/19416 objects misplaced (4.012%); 0 B/s, 58 objects/s recovering 2023-09-23T20:35:42.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:42 smithi149 ceph-mon[170394]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:35:42.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:42 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:35:42.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:42 smithi149 ceph-mon[170394]: osdmap e364: 8 total, 8 up, 8 in 2023-09-23T20:35:42.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:42 smithi149 ceph-mon[170394]: failed to encode map e364 with expected crc 2023-09-23T20:35:42.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:42 smithi149 ceph-mon[170394]: failed to encode map e364 with expected crc 2023-09-23T20:35:42.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:42 smithi149 ceph-mon[170394]: pgmap v868: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 116 active+clean; 407 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 686 op/s; 779/19416 objects misplaced (4.012%); 0 B/s, 58 objects/s recovering 2023-09-23T20:35:43.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:43 smithi018 ceph-mon[184144]: failed to encode map e364 with expected crc 2023-09-23T20:35:43.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:43 smithi018 ceph-mon[184144]: 1.17 scrub starts 2023-09-23T20:35:43.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:43 smithi018 ceph-mon[184144]: 1.1e scrub starts 2023-09-23T20:35:43.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:43 smithi018 ceph-mon[184144]: 1.1e scrub ok 2023-09-23T20:35:43.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:43 smithi018 ceph-mon[184144]: 1.15 scrub starts 2023-09-23T20:35:43.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:43 smithi018 ceph-mon[179647]: failed to encode map e364 with expected crc 2023-09-23T20:35:43.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:43 smithi018 ceph-mon[179647]: 1.17 scrub starts 2023-09-23T20:35:43.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:43 smithi018 ceph-mon[179647]: 1.1e scrub starts 2023-09-23T20:35:43.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:43 smithi018 ceph-mon[179647]: 1.1e scrub ok 2023-09-23T20:35:43.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:43 smithi018 ceph-mon[179647]: 1.15 scrub starts 2023-09-23T20:35:43.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:43 smithi149 ceph-mon[170394]: failed to encode map e364 with expected crc 2023-09-23T20:35:43.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:43 smithi149 ceph-mon[170394]: 1.17 scrub starts 2023-09-23T20:35:43.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:43 smithi149 ceph-mon[170394]: 1.1e scrub starts 2023-09-23T20:35:43.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:43 smithi149 ceph-mon[170394]: 1.1e scrub ok 2023-09-23T20:35:43.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:43 smithi149 ceph-mon[170394]: 1.15 scrub starts 2023-09-23T20:35:44.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:44 smithi149 ceph-mon[170394]: 1.1b scrub starts 2023-09-23T20:35:44.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:44 smithi149 ceph-mon[170394]: pgmap v869: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 116 active+clean; 359 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 682 op/s; 779/17118 objects misplaced (4.551%); 0 B/s, 58 objects/s recovering 2023-09-23T20:35:44.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:44 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:35:44.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:44 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:35:44.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:44 smithi149 ceph-mon[170394]: 1.1c scrub starts 2023-09-23T20:35:44.995 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:44 smithi018 ceph-mon[179647]: 1.1b scrub starts 2023-09-23T20:35:44.996 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:44 smithi018 ceph-mon[179647]: pgmap v869: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 116 active+clean; 359 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 682 op/s; 779/17118 objects misplaced (4.551%); 0 B/s, 58 objects/s recovering 2023-09-23T20:35:44.996 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:44 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:35:44.996 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:44 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:35:44.996 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:44 smithi018 ceph-mon[179647]: 1.1c scrub starts 2023-09-23T20:35:44.999 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:35:44 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:35:44] "GET /metrics HTTP/1.1" 200 36517 "" "Prometheus/2.43.0" 2023-09-23T20:35:44.999 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:44 smithi018 ceph-mon[184144]: 1.1b scrub starts 2023-09-23T20:35:44.999 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:44 smithi018 ceph-mon[184144]: pgmap v869: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 116 active+clean; 359 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 682 op/s; 779/17118 objects misplaced (4.551%); 0 B/s, 58 objects/s recovering 2023-09-23T20:35:44.999 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:44 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:35:45.000 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:44 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:35:45.000 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:44 smithi018 ceph-mon[184144]: 1.1c scrub starts 2023-09-23T20:35:45.079 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:35:45.086 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:35:45.087 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:35:45.187 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:35:45.230 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:35:45.280 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:35:45.286 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:35:45.287 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:35:45.388 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:35:45.462 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:35:45.652 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:45 smithi149 ceph-mon[170394]: 1.15 scrub starts 2023-09-23T20:35:45.668 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T20:35:45.733 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:45 smithi018 ceph-mon[184144]: 1.15 scrub starts 2023-09-23T20:35:45.733 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:45 smithi018 ceph-mon[179647]: 1.15 scrub starts 2023-09-23T20:35:46.769 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:35:46.775 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:35:46.776 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:35:46.876 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:35:46.911 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:35:46.959 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:35:46.961 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:35:46.961 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:46 smithi149 ceph-mon[170394]: 1.17 scrub starts 2023-09-23T20:35:46.962 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:46 smithi149 ceph-mon[170394]: pgmap v870: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 116 active+clean; 197 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 683 op/s; 361/9312 objects misplaced (3.877%); 0 B/s, 60 objects/s recovering 2023-09-23T20:35:46.962 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:35:47.063 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:35:47.098 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:35:47.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:46 smithi018 ceph-mon[179647]: 1.17 scrub starts 2023-09-23T20:35:47.144 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:46 smithi018 ceph-mon[179647]: pgmap v870: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 116 active+clean; 197 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 683 op/s; 361/9312 objects misplaced (3.877%); 0 B/s, 60 objects/s recovering 2023-09-23T20:35:47.144 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:46 smithi018 ceph-mon[184144]: 1.17 scrub starts 2023-09-23T20:35:47.144 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:46 smithi018 ceph-mon[184144]: pgmap v870: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 116 active+clean; 197 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 683 op/s; 361/9312 objects misplaced (3.877%); 0 B/s, 60 objects/s recovering 2023-09-23T20:35:47.156 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:35:47.163 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:35:47.163 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:35:47.264 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:35:47.307 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:35:47.367 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:35:47.379 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:35:47.380 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:35:47.480 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:35:47.513 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:35:47.558 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:35:47.562 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:35:47.562 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:35:47.663 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:35:47.699 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:35:47.718 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:35:47.748 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:35:47.753 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:35:47.753 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:35:47.756 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:35:47.854 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:35:47.884 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:47 smithi149 ceph-mon[170394]: 1.1b scrub starts 2023-09-23T20:35:47.887 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:35:47.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:47 smithi018 ceph-mon[179647]: 1.1b scrub starts 2023-09-23T20:35:47.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:47 smithi018 ceph-mon[184144]: 1.1b scrub starts 2023-09-23T20:35:47.932 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T20:35:47.938 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:35:47.943 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:35:47.943 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:35:48.044 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:35:48.085 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:35:48.128 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:35:48.139 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:35:48.139 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:35:48.240 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:35:48.335 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:35:48.456 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:35:48.463 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:35:48.463 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:35:48.564 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:35:48.602 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:35:48.645 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:35:48.654 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:35:48.655 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:35:48.755 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:35:48.821 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:35:48.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:48 smithi149 ceph-mon[170394]: pgmap v871: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 116 active+clean; 140 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 683 op/s; 361/6612 objects misplaced (5.460%); 0 B/s, 60 objects/s recovering 2023-09-23T20:35:48.973 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Removed 94549 objects 2023-09-23T20:35:48.973 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Clean up completed and total clean up time :142.137 2023-09-23T20:35:49.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:48 smithi018 ceph-mon[184144]: pgmap v871: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 116 active+clean; 140 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 683 op/s; 361/6612 objects misplaced (5.460%); 0 B/s, 60 objects/s recovering 2023-09-23T20:35:49.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:48 smithi018 ceph-mon[179647]: pgmap v871: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 116 active+clean; 140 MiB data, 3.7 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 683 op/s; 361/6612 objects misplaced (5.460%); 0 B/s, 60 objects/s recovering 2023-09-23T20:35:49.483 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:35:49.899 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:35:49.902 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:35:49.903 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:35:49.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:49 smithi149 ceph-mon[170394]: 1.17 scrub starts 2023-09-23T20:35:50.004 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:35:50.005 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:49 smithi018 ceph-mon[184144]: 1.17 scrub starts 2023-09-23T20:35:50.006 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:49 smithi018 ceph-mon[179647]: 1.17 scrub starts 2023-09-23T20:35:50.033 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:35:50.092 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:35:50.096 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:35:50.097 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:35:50.198 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:35:50.223 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:35:50.259 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:35:50.261 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:35:50.262 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:35:50.362 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:35:50.389 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:35:50.425 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:35:50.427 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:35:50.428 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:35:50.529 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:35:50.554 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:35:50.588 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:35:50.590 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:35:50.591 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:35:50.691 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:35:50.718 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:35:50.754 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:35:50.756 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:35:50.757 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:35:50.858 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:35:50.888 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:35:50.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:50 smithi149 ceph-mon[170394]: pgmap v872: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 116 active+clean; 101 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 678 op/s; 149/4734 objects misplaced (3.147%); 0 B/s, 60 objects/s recovering 2023-09-23T20:35:51.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:50 smithi018 ceph-mon[184144]: pgmap v872: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 116 active+clean; 101 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 678 op/s; 149/4734 objects misplaced (3.147%); 0 B/s, 60 objects/s recovering 2023-09-23T20:35:51.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:50 smithi018 ceph-mon[179647]: pgmap v872: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 116 active+clean; 101 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 678 op/s; 149/4734 objects misplaced (3.147%); 0 B/s, 60 objects/s recovering 2023-09-23T20:35:52.082 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:35:52.085 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:35:52.085 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:35:52.186 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:35:52.213 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:35:52.247 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:35:52.250 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:35:52.250 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:35:52.351 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:35:52.377 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:35:52.410 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:35:52.413 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:35:52.414 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:35:52.514 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:35:52.541 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:35:52.576 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:35:52.578 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:35:52.579 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:35:52.679 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:35:52.707 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:35:53.540 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:35:53.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:53 smithi018 ceph-mon[184144]: pgmap v873: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 116 active+clean; 18 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 604 op/s; 0 B/s, 80 objects/s recovering 2023-09-23T20:35:53.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:53 smithi018 ceph-mon[184144]: 1.1b scrub starts 2023-09-23T20:35:53.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:53 smithi018 ceph-mon[179647]: pgmap v873: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 116 active+clean; 18 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 604 op/s; 0 B/s, 80 objects/s recovering 2023-09-23T20:35:53.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:53 smithi018 ceph-mon[179647]: 1.1b scrub starts 2023-09-23T20:35:53.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:53 smithi149 ceph-mon[170394]: pgmap v873: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 116 active+clean; 18 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 604 op/s; 0 B/s, 80 objects/s recovering 2023-09-23T20:35:53.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:53 smithi149 ceph-mon[170394]: 1.1b scrub starts 2023-09-23T20:35:53.788 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:35:53.791 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:35:53.792 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:35:53.892 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:35:53.918 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:35:53.952 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:35:53.955 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:35:53.955 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:35:54.056 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:35:54.082 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:35:54.115 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:35:54.117 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:35:54.118 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:35:54.219 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:35:54.250 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:35:54.573 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:35:54.576 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:35:54.577 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:35:54.624 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_2 2023-09-23T20:35:54.625 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd pool rm unique_pool_2 unique_pool_2 --yes-i-really-really-mean-it 2023-09-23T20:35:54.677 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:35:54.708 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:35:54.881 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:35:54.885 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:35:54.885 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:35:54.986 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:35:55.012 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:35:55.048 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:35:55.050 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:35:55.051 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:35:55.141 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:35:54 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:35:54] "GET /metrics HTTP/1.1" 200 36517 "" "Prometheus/2.43.0" 2023-09-23T20:35:55.151 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:35:55.180 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:35:55.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:55 smithi149 ceph-mon[170394]: pgmap v874: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 116 active+clean; 4.5 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 536 op/s; 0 B/s, 69 objects/s recovering 2023-09-23T20:35:55.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:55 smithi149 ceph-mon[170394]: 1.17 scrub starts 2023-09-23T20:35:55.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:55 smithi149 ceph-mon[170394]: 1.1c scrub starts 2023-09-23T20:35:55.614 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:35:55.646 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:55 smithi018 ceph-mon[179647]: pgmap v874: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 116 active+clean; 4.5 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 536 op/s; 0 B/s, 69 objects/s recovering 2023-09-23T20:35:55.646 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:55 smithi018 ceph-mon[179647]: 1.17 scrub starts 2023-09-23T20:35:55.646 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:55 smithi018 ceph-mon[179647]: 1.1c scrub starts 2023-09-23T20:35:55.646 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:55 smithi018 ceph-mon[184144]: pgmap v874: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 116 active+clean; 4.5 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 536 op/s; 0 B/s, 69 objects/s recovering 2023-09-23T20:35:55.647 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:55 smithi018 ceph-mon[184144]: 1.17 scrub starts 2023-09-23T20:35:55.647 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:55 smithi018 ceph-mon[184144]: 1.1c scrub starts 2023-09-23T20:35:55.912 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:35:55.916 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:35:55.917 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:35:56.017 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:35:56.044 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:35:56.076 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:35:56.078 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:35:56.079 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:35:56.179 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:35:56.213 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:35:56.919 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:35:56.921 INFO:teuthology.orchestra.run.smithi018.stderr:dumped all 2023-09-23T20:35:57.244 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:57 smithi018 ceph-mon[184144]: pgmap v875: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 116 active+clean; 4.5 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 472 op/s; 0 B/s, 170 objects/s recovering 2023-09-23T20:35:57.244 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:57 smithi018 ceph-mon[184144]: 1.1b scrub starts 2023-09-23T20:35:57.244 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:57 smithi018 ceph-mon[179647]: pgmap v875: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 116 active+clean; 4.5 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 472 op/s; 0 B/s, 170 objects/s recovering 2023-09-23T20:35:57.244 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:57 smithi018 ceph-mon[179647]: 1.1b scrub starts 2023-09-23T20:35:57.348 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2023-09-23T20:35:57.348 INFO:teuthology.orchestra.run.smithi018.stdout: "size": 20, 2023-09-23T20:35:57.348 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 600, 2023-09-23T20:35:57.348 INFO:teuthology.orchestra.run.smithi018.stdout: "ops": [ 2023-09-23T20:35:57.348 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.349 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.45161.0:149500 6.14 6.2a5df714 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e266)", 2023-09-23T20:35:57.349 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:29:44.520246+0000", 2023-09-23T20:35:57.349 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 372.73695486000003, 2023-09-23T20:35:57.349 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.99822174500000005, 2023-09-23T20:35:57.349 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:35:57.349 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:35:57.350 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:35:57.350 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.45161", 2023-09-23T20:35:57.350 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3795614987", 2023-09-23T20:35:57.350 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 149500 2023-09-23T20:35:57.350 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.350 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:35:57.350 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.351 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:35:57.351 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:44.520246+0000", 2023-09-23T20:35:57.351 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:35:57.351 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.351 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.351 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:35:57.351 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:44.520246+0000", 2023-09-23T20:35:57.351 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:35:57.351 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.351 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.351 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:35:57.352 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:44.520244+0000", 2023-09-23T20:35:57.352 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:35:57.352 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.352 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.352 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:35:57.352 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:44.520250+0000", 2023-09-23T20:35:57.352 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.2379999999999997e-06 2023-09-23T20:35:57.352 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.353 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.353 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:35:57.353 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:44.520258+0000", 2023-09-23T20:35:57.353 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.7120000000000006e-06 2023-09-23T20:35:57.353 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.353 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.353 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:35:57.353 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:44.520267+0000", 2023-09-23T20:35:57.354 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.7690000000000002e-06 2023-09-23T20:35:57.354 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.354 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.354 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:35:57.354 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:44.520940+0000", 2023-09-23T20:35:57.354 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00067323100000000005 2023-09-23T20:35:57.354 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.354 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.355 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:35:57.355 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:44.520960+0000", 2023-09-23T20:35:57.355 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.9707999999999998e-05 2023-09-23T20:35:57.355 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.355 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.355 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:35:57.355 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:45.518455+0000", 2023-09-23T20:35:57.355 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.99749544300000004 2023-09-23T20:35:57.356 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.356 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.356 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:35:57.356 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:45.518468+0000", 2023-09-23T20:35:57.357 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2133000000000001e-05 2023-09-23T20:35:57.357 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:35:57.357 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:35:57.357 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:35:57.357 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.357 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.357 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.45161.0:149503 6.14 6.8f8a1754 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e266)", 2023-09-23T20:35:57.357 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:29:44.520275+0000", 2023-09-23T20:35:57.357 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 372.73692584999998, 2023-09-23T20:35:57.358 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.99822535599999995, 2023-09-23T20:35:57.358 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:35:57.358 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:35:57.358 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:35:57.358 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.45161", 2023-09-23T20:35:57.358 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3795614987", 2023-09-23T20:35:57.358 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 149503 2023-09-23T20:35:57.359 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.359 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:35:57.359 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.359 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:35:57.359 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:44.520275+0000", 2023-09-23T20:35:57.359 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:35:57.359 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.359 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.359 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:35:57.359 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:44.520275+0000", 2023-09-23T20:35:57.360 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:35:57.360 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.360 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.360 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:35:57.360 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:44.520275+0000", 2023-09-23T20:35:57.360 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:35:57.360 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.360 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.360 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:35:57.360 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:44.520277+0000", 2023-09-23T20:35:57.361 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.2180000000000002e-06 2023-09-23T20:35:57.361 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.361 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.361 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:35:57.361 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:44.520283+0000", 2023-09-23T20:35:57.361 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.7969999999999997e-06 2023-09-23T20:35:57.362 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.362 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.362 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:35:57.362 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:44.520286+0000", 2023-09-23T20:35:57.362 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.2339999999999999e-06 2023-09-23T20:35:57.362 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.362 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.363 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:35:57.363 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:44.520986+0000", 2023-09-23T20:35:57.363 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00069974100000000003 2023-09-23T20:35:57.363 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.363 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.363 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:35:57.363 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:44.520992+0000", 2023-09-23T20:35:57.364 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.6490000000000002e-06 2023-09-23T20:35:57.364 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.364 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.364 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:35:57.364 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:45.518491+0000", 2023-09-23T20:35:57.364 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.99749888399999997 2023-09-23T20:35:57.365 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.365 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.365 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:35:57.365 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:45.518500+0000", 2023-09-23T20:35:57.365 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.1530000000000003e-06 2023-09-23T20:35:57.366 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:35:57.366 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:35:57.367 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:35:57.367 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.367 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.367 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.45161.0:149511 6.14 6.2b03bf34 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e266)", 2023-09-23T20:35:57.367 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:29:44.520289+0000", 2023-09-23T20:35:57.367 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 372.73691198199998, 2023-09-23T20:35:57.367 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.99824324600000003, 2023-09-23T20:35:57.368 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:35:57.368 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:35:57.368 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:35:57.368 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.45161", 2023-09-23T20:35:57.368 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3795614987", 2023-09-23T20:35:57.368 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 149511 2023-09-23T20:35:57.369 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.369 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:35:57.369 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.369 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:35:57.369 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:44.520289+0000", 2023-09-23T20:35:57.369 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:35:57.369 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.370 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.370 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:35:57.370 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:44.520289+0000", 2023-09-23T20:35:57.370 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:35:57.370 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.370 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.371 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:35:57.371 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:44.520288+0000", 2023-09-23T20:35:57.371 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967296 2023-09-23T20:35:57.371 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.371 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.371 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:35:57.372 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:44.520290+0000", 2023-09-23T20:35:57.372 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3859999999999999e-06 2023-09-23T20:35:57.372 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.372 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.372 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:35:57.372 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:44.520293+0000", 2023-09-23T20:35:57.372 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.0410000000000002e-06 2023-09-23T20:35:57.373 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.373 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.373 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:35:57.373 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:44.520295+0000", 2023-09-23T20:35:57.373 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.5380000000000001e-06 2023-09-23T20:35:57.373 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.373 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.373 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:35:57.373 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:44.521013+0000", 2023-09-23T20:35:57.374 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00071736200000000003 2023-09-23T20:35:57.374 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.374 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.374 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:35:57.374 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:44.521019+0000", 2023-09-23T20:35:57.374 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.3799999999999999e-06 2023-09-23T20:35:57.374 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.375 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.375 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:35:57.375 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:45.518522+0000", 2023-09-23T20:35:57.375 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.99750267599999998 2023-09-23T20:35:57.375 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.375 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.376 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:35:57.376 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:45.518532+0000", 2023-09-23T20:35:57.376 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0101e-05 2023-09-23T20:35:57.376 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:35:57.376 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:35:57.376 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:35:57.377 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.377 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.377 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.45161.0:149543 6.14 6.21fbcb54 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e266)", 2023-09-23T20:35:57.377 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:29:44.521258+0000", 2023-09-23T20:35:57.377 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 372.735942255, 2023-09-23T20:35:57.377 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.99730052000000002, 2023-09-23T20:35:57.378 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:35:57.378 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:35:57.378 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:35:57.378 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.45161", 2023-09-23T20:35:57.378 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3795614987", 2023-09-23T20:35:57.378 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 149543 2023-09-23T20:35:57.378 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.378 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:35:57.379 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.379 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:35:57.379 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:44.521258+0000", 2023-09-23T20:35:57.379 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:35:57.379 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.379 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.379 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:35:57.379 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:44.521258+0000", 2023-09-23T20:35:57.379 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:35:57.379 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.379 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.380 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:35:57.380 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:44.521257+0000", 2023-09-23T20:35:57.380 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.999999 2023-09-23T20:35:57.380 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.380 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.380 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:35:57.380 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:44.521263+0000", 2023-09-23T20:35:57.380 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.7429999999999999e-06 2023-09-23T20:35:57.380 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.380 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.380 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:35:57.380 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:44.521274+0000", 2023-09-23T20:35:57.381 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1385e-05 2023-09-23T20:35:57.381 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.381 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.381 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:35:57.381 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:44.521284+0000", 2023-09-23T20:35:57.381 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.0480000000000004e-06 2023-09-23T20:35:57.381 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.381 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.381 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:35:57.381 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:44.521327+0000", 2023-09-23T20:35:57.381 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.3689999999999997e-05 2023-09-23T20:35:57.381 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.382 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.382 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:35:57.382 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:44.521336+0000", 2023-09-23T20:35:57.382 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.3930000000000008e-06 2023-09-23T20:35:57.382 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.383 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.383 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:35:57.383 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:45.518553+0000", 2023-09-23T20:35:57.383 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.99721703500000003 2023-09-23T20:35:57.383 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.383 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.384 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:35:57.384 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:45.518559+0000", 2023-09-23T20:35:57.384 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.353e-06 2023-09-23T20:35:57.384 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:35:57.384 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:35:57.384 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:35:57.384 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.384 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.385 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.45161.0:205351 6.11 6:8940b4e2:::benchmark_data_smithi018_270558_object100612:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e292)", 2023-09-23T20:35:57.385 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:31:14.403632+0000", 2023-09-23T20:35:57.385 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 282.85356889000002, 2023-09-23T20:35:57.385 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.030579779, 2023-09-23T20:35:57.385 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:35:57.385 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:35:57.385 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:35:57.385 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.45161", 2023-09-23T20:35:57.385 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3795614987", 2023-09-23T20:35:57.385 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 205351 2023-09-23T20:35:57.385 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.385 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:35:57.386 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.386 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:35:57.386 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:14.403632+0000", 2023-09-23T20:35:57.386 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:35:57.386 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.386 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.386 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:35:57.386 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:14.403632+0000", 2023-09-23T20:35:57.386 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:35:57.386 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.387 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.387 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:35:57.387 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:14.403630+0000", 2023-09-23T20:35:57.387 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:35:57.387 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.387 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.387 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:35:57.387 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:14.403641+0000", 2023-09-23T20:35:57.388 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1354e-05 2023-09-23T20:35:57.388 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.388 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.388 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:35:57.388 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:14.403656+0000", 2023-09-23T20:35:57.388 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.435e-05 2023-09-23T20:35:57.388 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.388 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.388 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:35:57.389 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:14.403665+0000", 2023-09-23T20:35:57.389 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.5519999999999993e-06 2023-09-23T20:35:57.389 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.389 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.389 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:35:57.389 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:14.403729+0000", 2023-09-23T20:35:57.390 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.3886999999999993e-05 2023-09-23T20:35:57.390 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.390 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.390 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:35:57.390 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:14.403741+0000", 2023-09-23T20:35:57.390 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1620999999999999e-05 2023-09-23T20:35:57.391 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.391 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.391 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:35:57.391 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:15.410699+0000", 2023-09-23T20:35:57.391 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0069581620000001 2023-09-23T20:35:57.391 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.391 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.391 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:35:57.391 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:15.410843+0000", 2023-09-23T20:35:57.392 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00014438 2023-09-23T20:35:57.392 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.392 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.392 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 2,6", 2023-09-23T20:35:57.392 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:15.411260+0000", 2023-09-23T20:35:57.392 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00041685600000000001 2023-09-23T20:35:57.392 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.392 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.392 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:35:57.393 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:15.416401+0000", 2023-09-23T20:35:57.393 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0051409209999999997 2023-09-23T20:35:57.393 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.393 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.393 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:35:57.393 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:15.422105+0000", 2023-09-23T20:35:57.393 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0057035250000000001 2023-09-23T20:35:57.393 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.393 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.393 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:35:57.394 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:15.434115+0000", 2023-09-23T20:35:57.394 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.012010758 2023-09-23T20:35:57.394 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.394 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.394 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:35:57.394 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:15.434165+0000", 2023-09-23T20:35:57.394 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.9317999999999998e-05 2023-09-23T20:35:57.394 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.394 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.394 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:35:57.394 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:15.434212+0000", 2023-09-23T20:35:57.395 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.6968000000000002e-05 2023-09-23T20:35:57.395 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:35:57.395 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:35:57.395 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:35:57.395 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.395 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.395 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.45161.0:205365 6.11 6:8afc9a6e:::benchmark_data_smithi018_270558_object100626:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e292)", 2023-09-23T20:35:57.395 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:31:14.428052+0000", 2023-09-23T20:35:57.395 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 282.829148369, 2023-09-23T20:35:57.395 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0435979980000001, 2023-09-23T20:35:57.395 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:35:57.395 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:35:57.396 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:35:57.396 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.45161", 2023-09-23T20:35:57.396 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3795614987", 2023-09-23T20:35:57.396 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 205365 2023-09-23T20:35:57.396 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.396 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:35:57.396 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.396 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:35:57.396 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:14.428052+0000", 2023-09-23T20:35:57.396 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:35:57.396 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.396 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.397 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:35:57.397 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:14.428052+0000", 2023-09-23T20:35:57.397 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:35:57.397 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.397 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.397 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:35:57.397 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:14.428050+0000", 2023-09-23T20:35:57.397 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999971 2023-09-23T20:35:57.398 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.398 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.398 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:35:57.398 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:14.428065+0000", 2023-09-23T20:35:57.398 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.5152e-05 2023-09-23T20:35:57.399 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.399 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.399 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:35:57.399 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:14.428084+0000", 2023-09-23T20:35:57.399 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.9346000000000001e-05 2023-09-23T20:35:57.399 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.399 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.399 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:35:57.399 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:14.428097+0000", 2023-09-23T20:35:57.400 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2559e-05 2023-09-23T20:35:57.400 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.400 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.400 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:35:57.400 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:14.435340+0000", 2023-09-23T20:35:57.400 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0072437339999999999 2023-09-23T20:35:57.400 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.401 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.401 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:35:57.401 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:14.435360+0000", 2023-09-23T20:35:57.401 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.9942e-05 2023-09-23T20:35:57.401 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.401 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.401 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:35:57.401 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:15.416434+0000", 2023-09-23T20:35:57.402 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.98107409499999998 2023-09-23T20:35:57.402 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.402 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.402 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:35:57.402 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:15.416578+0000", 2023-09-23T20:35:57.402 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00014369799999999999 2023-09-23T20:35:57.402 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.402 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.402 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 2,6", 2023-09-23T20:35:57.402 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:15.417093+0000", 2023-09-23T20:35:57.403 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00051506499999999999 2023-09-23T20:35:57.403 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.403 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.403 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:35:57.403 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:15.424390+0000", 2023-09-23T20:35:57.403 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0072970149999999996 2023-09-23T20:35:57.403 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.403 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.403 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:35:57.403 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:15.424416+0000", 2023-09-23T20:35:57.404 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.6003000000000002e-05 2023-09-23T20:35:57.404 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.404 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.404 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:35:57.404 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:15.471556+0000", 2023-09-23T20:35:57.404 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.047139595999999999 2023-09-23T20:35:57.404 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.404 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.404 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:35:57.404 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:15.471611+0000", 2023-09-23T20:35:57.404 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.5606999999999998e-05 2023-09-23T20:35:57.405 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.405 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.405 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:35:57.405 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:15.471650+0000", 2023-09-23T20:35:57.405 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.8862999999999997e-05 2023-09-23T20:35:57.405 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:35:57.405 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:35:57.405 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:35:57.405 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.405 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.405 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:57591 7.1 7:8554f45e:::benchmark_data_smithi018_308601_object57590:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e313)", 2023-09-23T20:35:57.406 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:32:33.220481+0000", 2023-09-23T20:35:57.406 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 204.036719874, 2023-09-23T20:35:57.406 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3899836859999999, 2023-09-23T20:35:57.406 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:35:57.406 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:35:57.406 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:35:57.406 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:35:57.406 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:35:57.406 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 57591 2023-09-23T20:35:57.406 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.406 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:35:57.406 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.407 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:35:57.407 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.220481+0000", 2023-09-23T20:35:57.407 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:35:57.407 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.407 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.407 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:35:57.407 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.220481+0000", 2023-09-23T20:35:57.407 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:35:57.407 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.407 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.407 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:35:57.408 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.220479+0000", 2023-09-23T20:35:57.408 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:35:57.408 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.408 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.408 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:35:57.408 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.220511+0000", 2023-09-23T20:35:57.408 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.1970000000000001e-05 2023-09-23T20:35:57.408 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.408 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.408 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:35:57.409 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.220523+0000", 2023-09-23T20:35:57.409 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2105e-05 2023-09-23T20:35:57.409 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.409 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.409 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:35:57.409 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.220535+0000", 2023-09-23T20:35:57.409 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1235e-05 2023-09-23T20:35:57.409 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.409 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.409 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:35:57.409 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.221636+0000", 2023-09-23T20:35:57.410 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0011011759999999999 2023-09-23T20:35:57.410 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.410 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.410 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:35:57.410 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.221643+0000", 2023-09-23T20:35:57.410 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.5610000000000001e-06 2023-09-23T20:35:57.410 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.410 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.410 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:35:57.411 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.609130+0000", 2023-09-23T20:35:57.411 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.387486223 2023-09-23T20:35:57.411 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.411 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.411 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:35:57.411 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.610465+0000", 2023-09-23T20:35:57.411 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0013348940000000001 2023-09-23T20:35:57.411 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:35:57.411 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:35:57.411 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:35:57.411 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.412 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.412 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:57595 7.1 7:8b518598:::benchmark_data_smithi018_308601_object57594:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e313)", 2023-09-23T20:35:57.412 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:32:33.220549+0000", 2023-09-23T20:35:57.412 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 204.036651662, 2023-09-23T20:35:57.412 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.4131732800000001, 2023-09-23T20:35:57.412 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:35:57.412 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:35:57.412 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:35:57.412 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:35:57.412 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:35:57.412 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 57595 2023-09-23T20:35:57.412 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.413 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:35:57.413 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.413 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:35:57.413 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.220549+0000", 2023-09-23T20:35:57.413 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:35:57.413 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.413 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.413 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:35:57.413 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.220549+0000", 2023-09-23T20:35:57.413 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:35:57.413 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.413 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.414 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:35:57.414 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.220549+0000", 2023-09-23T20:35:57.414 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:35:57.414 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.414 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.414 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:35:57.414 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.220574+0000", 2023-09-23T20:35:57.414 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.5612000000000001e-05 2023-09-23T20:35:57.414 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.414 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.414 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:35:57.414 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.220582+0000", 2023-09-23T20:35:57.415 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.8269999999999998e-06 2023-09-23T20:35:57.415 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.415 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.415 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:35:57.415 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.220588+0000", 2023-09-23T20:35:57.415 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.1349999999999997e-06 2023-09-23T20:35:57.415 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.415 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.416 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:35:57.416 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.221656+0000", 2023-09-23T20:35:57.416 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.001068098 2023-09-23T20:35:57.416 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.416 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.416 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:35:57.416 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.221660+0000", 2023-09-23T20:35:57.417 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.58e-06 2023-09-23T20:35:57.417 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.417 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.417 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:35:57.417 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.613460+0000", 2023-09-23T20:35:57.417 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3918001280000001 2023-09-23T20:35:57.418 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.418 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.418 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for degraded object", 2023-09-23T20:35:57.418 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.614021+0000", 2023-09-23T20:35:57.418 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00056140200000000002 2023-09-23T20:35:57.419 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.419 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.419 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:35:57.419 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.633574+0000", 2023-09-23T20:35:57.419 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.019552403999999999 2023-09-23T20:35:57.419 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.419 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.420 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:35:57.420 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.633722+0000", 2023-09-23T20:35:57.420 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00014861 2023-09-23T20:35:57.420 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:35:57.420 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:35:57.420 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:35:57.420 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.420 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.421 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:57641 7.1 7:82e04a7d:::benchmark_data_smithi018_308601_object57640:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e313)", 2023-09-23T20:35:57.421 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:32:33.220594+0000", 2023-09-23T20:35:57.421 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 204.036607199, 2023-09-23T20:35:57.421 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.41538217, 2023-09-23T20:35:57.421 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:35:57.421 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:35:57.422 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:35:57.422 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:35:57.422 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:35:57.422 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 57641 2023-09-23T20:35:57.422 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.422 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:35:57.422 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.422 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:35:57.422 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.220594+0000", 2023-09-23T20:35:57.422 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:35:57.423 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.423 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.423 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:35:57.423 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.220594+0000", 2023-09-23T20:35:57.423 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:35:57.423 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.423 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.423 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:35:57.423 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.220593+0000", 2023-09-23T20:35:57.423 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:35:57.423 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.423 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.424 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:35:57.424 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.220619+0000", 2023-09-23T20:35:57.424 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.6431000000000002e-05 2023-09-23T20:35:57.424 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.424 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.424 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:35:57.424 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.220626+0000", 2023-09-23T20:35:57.424 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.6259999999999997e-06 2023-09-23T20:35:57.424 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.424 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.424 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:35:57.425 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.220631+0000", 2023-09-23T20:35:57.425 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.215e-06 2023-09-23T20:35:57.425 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.425 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.425 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:35:57.425 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.221678+0000", 2023-09-23T20:35:57.425 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.001046892 2023-09-23T20:35:57.425 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.425 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.426 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:35:57.426 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.221682+0000", 2023-09-23T20:35:57.426 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.0300000000000004e-06 2023-09-23T20:35:57.426 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.426 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.426 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:35:57.426 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.614054+0000", 2023-09-23T20:35:57.426 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3923719809999999 2023-09-23T20:35:57.426 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.426 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.426 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:35:57.426 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.614542+0000", 2023-09-23T20:35:57.427 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00048730699999999997 2023-09-23T20:35:57.427 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.427 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.427 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 3,4,6", 2023-09-23T20:35:57.427 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.614721+0000", 2023-09-23T20:35:57.427 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00017952699999999999 2023-09-23T20:35:57.427 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.427 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.427 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:35:57.427 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.625753+0000", 2023-09-23T20:35:57.427 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.011032036 2023-09-23T20:35:57.427 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.428 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.428 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:35:57.428 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.625939+0000", 2023-09-23T20:35:57.428 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00018612199999999999 2023-09-23T20:35:57.428 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.428 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.428 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:35:57.428 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.628311+0000", 2023-09-23T20:35:57.428 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0023717389999999999 2023-09-23T20:35:57.428 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.428 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.429 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:35:57.429 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.633817+0000", 2023-09-23T20:35:57.429 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0055056690000000004 2023-09-23T20:35:57.429 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.429 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.429 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:35:57.429 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.633889+0000", 2023-09-23T20:35:57.429 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.2037000000000002e-05 2023-09-23T20:35:57.429 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.429 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.430 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:35:57.430 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.635976+0000", 2023-09-23T20:35:57.430 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0020870390000000002 2023-09-23T20:35:57.430 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:35:57.430 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:35:57.430 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:35:57.430 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.430 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.430 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:57647 7.1 7:8e361ecd:::benchmark_data_smithi018_308601_object57646:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e313)", 2023-09-23T20:35:57.430 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:32:33.220636+0000", 2023-09-23T20:35:57.431 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 204.03656484699999, 2023-09-23T20:35:57.431 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.417278721, 2023-09-23T20:35:57.431 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:35:57.431 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:35:57.431 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:35:57.431 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:35:57.431 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:35:57.431 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 57647 2023-09-23T20:35:57.431 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.432 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:35:57.432 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.432 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:35:57.432 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.220636+0000", 2023-09-23T20:35:57.432 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:35:57.432 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.432 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.432 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:35:57.432 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.220636+0000", 2023-09-23T20:35:57.432 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:35:57.432 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.433 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.433 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:35:57.433 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.220635+0000", 2023-09-23T20:35:57.433 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:35:57.433 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.433 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.433 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:35:57.433 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.220659+0000", 2023-09-23T20:35:57.433 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.3969999999999999e-05 2023-09-23T20:35:57.433 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.433 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.433 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:35:57.434 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.220666+0000", 2023-09-23T20:35:57.434 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.2299999999999996e-06 2023-09-23T20:35:57.434 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.434 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.434 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:35:57.434 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.220671+0000", 2023-09-23T20:35:57.434 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.9540000000000003e-06 2023-09-23T20:35:57.434 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.434 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.434 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:35:57.434 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.221699+0000", 2023-09-23T20:35:57.435 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0010287359999999999 2023-09-23T20:35:57.435 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.435 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.435 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:35:57.435 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.221703+0000", 2023-09-23T20:35:57.435 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.765e-06 2023-09-23T20:35:57.435 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.435 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.435 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:35:57.435 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.616911+0000", 2023-09-23T20:35:57.435 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3952080200000001 2023-09-23T20:35:57.435 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.436 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.436 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:35:57.436 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.617068+0000", 2023-09-23T20:35:57.436 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000156484 2023-09-23T20:35:57.436 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.436 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.436 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 3,4,6", 2023-09-23T20:35:57.436 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.617232+0000", 2023-09-23T20:35:57.436 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000163974 2023-09-23T20:35:57.436 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.436 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.436 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:35:57.437 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.625797+0000", 2023-09-23T20:35:57.437 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0085653410000000006 2023-09-23T20:35:57.437 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.437 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.437 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:35:57.437 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.625946+0000", 2023-09-23T20:35:57.437 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000148947 2023-09-23T20:35:57.437 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.437 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.437 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:35:57.438 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.629107+0000", 2023-09-23T20:35:57.438 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0031612279999999999 2023-09-23T20:35:57.438 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.438 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.438 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:35:57.438 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.636021+0000", 2023-09-23T20:35:57.438 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0069139689999999998 2023-09-23T20:35:57.438 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.438 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.439 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:35:57.439 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.636062+0000", 2023-09-23T20:35:57.439 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.0567e-05 2023-09-23T20:35:57.439 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.439 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.439 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:35:57.439 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.637915+0000", 2023-09-23T20:35:57.439 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.001852958 2023-09-23T20:35:57.440 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:35:57.440 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:35:57.440 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:35:57.440 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.440 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.440 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:57652 7.1 7:803e82a9:::benchmark_data_smithi018_308601_object57651:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e313)", 2023-09-23T20:35:57.440 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:32:33.220675+0000", 2023-09-23T20:35:57.440 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 204.03652539800001, 2023-09-23T20:35:57.440 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.4243310039999999, 2023-09-23T20:35:57.440 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:35:57.440 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:35:57.440 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:35:57.441 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:35:57.441 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:35:57.441 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 57652 2023-09-23T20:35:57.441 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.441 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:35:57.441 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.441 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:35:57.441 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.220675+0000", 2023-09-23T20:35:57.442 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:35:57.442 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.442 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.442 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:35:57.442 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.220675+0000", 2023-09-23T20:35:57.442 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:35:57.442 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.442 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.443 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:35:57.443 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.220675+0000", 2023-09-23T20:35:57.443 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:35:57.443 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.443 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.443 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:35:57.443 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.220708+0000", 2023-09-23T20:35:57.444 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.3080000000000002e-05 2023-09-23T20:35:57.444 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.444 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.444 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:35:57.444 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.220716+0000", 2023-09-23T20:35:57.444 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.7600000000000002e-06 2023-09-23T20:35:57.444 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.444 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.444 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:35:57.444 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.220721+0000", 2023-09-23T20:35:57.444 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.2510000000000002e-06 2023-09-23T20:35:57.445 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.445 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.445 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:35:57.445 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.221715+0000", 2023-09-23T20:35:57.445 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000993774 2023-09-23T20:35:57.445 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.445 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.445 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:35:57.445 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.221718+0000", 2023-09-23T20:35:57.445 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.5090000000000002e-06 2023-09-23T20:35:57.445 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.446 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.446 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:35:57.446 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.620216+0000", 2023-09-23T20:35:57.446 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.398498002 2023-09-23T20:35:57.446 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.446 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.446 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:35:57.446 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.620417+0000", 2023-09-23T20:35:57.446 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00020063700000000001 2023-09-23T20:35:57.446 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.446 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.446 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 3,4,6", 2023-09-23T20:35:57.447 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.620597+0000", 2023-09-23T20:35:57.447 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000180306 2023-09-23T20:35:57.447 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.447 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.447 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:35:57.447 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.625927+0000", 2023-09-23T20:35:57.447 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.005329828 2023-09-23T20:35:57.447 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.447 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.447 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:35:57.447 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.628080+0000", 2023-09-23T20:35:57.447 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0021524550000000002 2023-09-23T20:35:57.448 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.448 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.448 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:35:57.448 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.629309+0000", 2023-09-23T20:35:57.448 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.001229162 2023-09-23T20:35:57.448 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.448 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.448 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:35:57.448 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.641922+0000", 2023-09-23T20:35:57.448 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.012613177999999999 2023-09-23T20:35:57.448 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.448 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.449 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:35:57.449 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.641958+0000", 2023-09-23T20:35:57.449 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.5973000000000003e-05 2023-09-23T20:35:57.449 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.449 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.449 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:35:57.449 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.645006+0000", 2023-09-23T20:35:57.449 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0030484900000000001 2023-09-23T20:35:57.449 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:35:57.449 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:35:57.449 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:35:57.449 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.450 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.450 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:69139 7.2 7:47e596b6:::benchmark_data_smithi018_308601_object69138:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e319)", 2023-09-23T20:35:57.450 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:32:49.543356+0000", 2023-09-23T20:35:57.450 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 187.71384457900001, 2023-09-23T20:35:57.450 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0152868230000001, 2023-09-23T20:35:57.450 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:35:57.450 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:35:57.450 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:35:57.450 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:35:57.450 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:35:57.450 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 69139 2023-09-23T20:35:57.451 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.451 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:35:57.451 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.451 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:35:57.451 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.543356+0000", 2023-09-23T20:35:57.451 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:35:57.451 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.451 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.451 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:35:57.451 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.543356+0000", 2023-09-23T20:35:57.451 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:35:57.451 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.451 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.452 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:35:57.452 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.543356+0000", 2023-09-23T20:35:57.452 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:35:57.452 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.452 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.452 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:35:57.452 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.543376+0000", 2023-09-23T20:35:57.452 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.0636999999999999e-05 2023-09-23T20:35:57.452 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.453 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.453 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:35:57.453 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.543383+0000", 2023-09-23T20:35:57.453 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.1689999999999997e-06 2023-09-23T20:35:57.453 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.453 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.453 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:35:57.453 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.543388+0000", 2023-09-23T20:35:57.453 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.0420000000000002e-06 2023-09-23T20:35:57.453 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.454 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.454 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:35:57.454 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.548487+0000", 2023-09-23T20:35:57.454 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0050989529999999998 2023-09-23T20:35:57.454 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.454 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.454 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:35:57.454 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.548499+0000", 2023-09-23T20:35:57.454 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2714999999999999e-05 2023-09-23T20:35:57.454 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.454 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.455 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:35:57.455 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:50.557250+0000", 2023-09-23T20:35:57.455 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0087509569999999 2023-09-23T20:35:57.455 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.455 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.455 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:35:57.455 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:50.558643+0000", 2023-09-23T20:35:57.455 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.001392702 2023-09-23T20:35:57.455 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:35:57.455 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:35:57.455 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:35:57.455 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.456 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.456 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:69147 7.2 7:41693550:::benchmark_data_smithi018_308601_object69146:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e319)", 2023-09-23T20:35:57.456 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:32:49.543394+0000", 2023-09-23T20:35:57.456 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 187.71380661200001, 2023-09-23T20:35:57.456 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.015333775, 2023-09-23T20:35:57.456 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:35:57.456 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:35:57.456 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:35:57.456 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:35:57.456 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:35:57.456 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 69147 2023-09-23T20:35:57.457 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.457 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:35:57.457 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.457 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:35:57.457 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.543394+0000", 2023-09-23T20:35:57.457 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:35:57.457 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.457 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.457 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:35:57.457 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.543394+0000", 2023-09-23T20:35:57.457 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:35:57.457 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.458 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.458 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:35:57.458 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.543394+0000", 2023-09-23T20:35:57.458 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:35:57.458 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.458 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.458 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:35:57.458 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.543417+0000", 2023-09-23T20:35:57.458 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.3669e-05 2023-09-23T20:35:57.458 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.458 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.458 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:35:57.459 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.543423+0000", 2023-09-23T20:35:57.459 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.7740000000000001e-06 2023-09-23T20:35:57.459 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.459 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.459 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:35:57.459 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.543428+0000", 2023-09-23T20:35:57.459 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.5549999999999996e-06 2023-09-23T20:35:57.459 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.459 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.459 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:35:57.459 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.548533+0000", 2023-09-23T20:35:57.459 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0051051550000000001 2023-09-23T20:35:57.460 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.460 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.460 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:35:57.460 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.548542+0000", 2023-09-23T20:35:57.460 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.0790000000000007e-06 2023-09-23T20:35:57.460 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.460 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.461 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:35:57.461 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:50.558668+0000", 2023-09-23T20:35:57.461 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.010125878 2023-09-23T20:35:57.461 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.461 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.461 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:35:57.462 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:50.558728+0000", 2023-09-23T20:35:57.462 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.0145000000000001e-05 2023-09-23T20:35:57.462 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:35:57.462 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:35:57.462 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:35:57.462 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.462 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.462 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:69173 7.2 7:4de96838:::benchmark_data_smithi018_308601_object69172:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e319)", 2023-09-23T20:35:57.462 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:32:49.560482+0000", 2023-09-23T20:35:57.463 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 187.69671896099999, 2023-09-23T20:35:57.463 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0188592169999999, 2023-09-23T20:35:57.463 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:35:57.463 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:35:57.463 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:35:57.463 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:35:57.463 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:35:57.463 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 69173 2023-09-23T20:35:57.463 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.464 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:35:57.464 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.464 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:35:57.464 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.560482+0000", 2023-09-23T20:35:57.464 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:35:57.464 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.464 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.464 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:35:57.464 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.560482+0000", 2023-09-23T20:35:57.464 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:35:57.464 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.464 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.464 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:35:57.465 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.560480+0000", 2023-09-23T20:35:57.465 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:35:57.465 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.465 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.465 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:35:57.465 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.560502+0000", 2023-09-23T20:35:57.465 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.1999000000000001e-05 2023-09-23T20:35:57.465 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.465 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.465 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:35:57.465 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.560519+0000", 2023-09-23T20:35:57.465 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.7299000000000002e-05 2023-09-23T20:35:57.466 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.466 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.466 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:35:57.466 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.560530+0000", 2023-09-23T20:35:57.466 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0743e-05 2023-09-23T20:35:57.466 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.466 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.466 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:35:57.466 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.560599+0000", 2023-09-23T20:35:57.466 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.8956999999999997e-05 2023-09-23T20:35:57.466 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.467 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.467 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:35:57.467 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.560614+0000", 2023-09-23T20:35:57.467 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.5192e-05 2023-09-23T20:35:57.467 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.467 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.467 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:35:57.467 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:50.558780+0000", 2023-09-23T20:35:57.467 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.99816563800000002 2023-09-23T20:35:57.467 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.467 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.467 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:35:57.468 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:50.559169+0000", 2023-09-23T20:35:57.468 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00038896699999999999 2023-09-23T20:35:57.468 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.468 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.468 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 2,3", 2023-09-23T20:35:57.468 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:50.559324+0000", 2023-09-23T20:35:57.468 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00015468799999999999 2023-09-23T20:35:57.468 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.468 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.468 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:35:57.468 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:50.571153+0000", 2023-09-23T20:35:57.468 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.011829746 2023-09-23T20:35:57.469 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.469 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.469 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:35:57.469 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:50.571200+0000", 2023-09-23T20:35:57.469 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.6563000000000001e-05 2023-09-23T20:35:57.469 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.470 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.470 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:35:57.470 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:50.579242+0000", 2023-09-23T20:35:57.470 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0080423160000000007 2023-09-23T20:35:57.470 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.470 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.470 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:35:57.470 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:50.579297+0000", 2023-09-23T20:35:57.470 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.4564000000000001e-05 2023-09-23T20:35:57.470 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.471 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.471 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:35:57.471 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:50.579341+0000", 2023-09-23T20:35:57.471 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.4129999999999999e-05 2023-09-23T20:35:57.471 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:35:57.471 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:35:57.471 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:35:57.472 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.472 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.472 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:69191 7.2 7:4efd35cd:::benchmark_data_smithi018_308601_object69190:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e319)", 2023-09-23T20:35:57.472 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:32:49.583897+0000", 2023-09-23T20:35:57.472 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 187.673303382, 2023-09-23T20:35:57.472 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.99687193600000001, 2023-09-23T20:35:57.472 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:35:57.472 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:35:57.472 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:35:57.472 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:35:57.472 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:35:57.472 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 69191 2023-09-23T20:35:57.473 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.473 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:35:57.473 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.473 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:35:57.473 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.583897+0000", 2023-09-23T20:35:57.473 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:35:57.473 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.473 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.473 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:35:57.474 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.583897+0000", 2023-09-23T20:35:57.474 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:35:57.474 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.474 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.474 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:35:57.474 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.583897+0000", 2023-09-23T20:35:57.474 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:35:57.475 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.475 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.475 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:35:57.475 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.583915+0000", 2023-09-23T20:35:57.475 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.8610999999999999e-05 2023-09-23T20:35:57.475 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.475 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.475 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:35:57.475 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.583923+0000", 2023-09-23T20:35:57.475 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.6340000000000001e-06 2023-09-23T20:35:57.475 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.475 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.476 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:35:57.476 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.583929+0000", 2023-09-23T20:35:57.476 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.1770000000000003e-06 2023-09-23T20:35:57.476 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.476 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.476 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:35:57.476 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.584056+0000", 2023-09-23T20:35:57.476 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000126415 2023-09-23T20:35:57.476 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.476 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.476 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:35:57.476 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.584065+0000", 2023-09-23T20:35:57.477 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.8479999999999996e-06 2023-09-23T20:35:57.477 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.477 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.477 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:35:57.477 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:50.561535+0000", 2023-09-23T20:35:57.477 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.97746987799999996 2023-09-23T20:35:57.477 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.477 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.477 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:35:57.477 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:50.561670+0000", 2023-09-23T20:35:57.477 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000134957 2023-09-23T20:35:57.477 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.477 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.478 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 2,3", 2023-09-23T20:35:57.478 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:50.561819+0000", 2023-09-23T20:35:57.478 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00014892700000000001 2023-09-23T20:35:57.478 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.478 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.478 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:35:57.479 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:50.571205+0000", 2023-09-23T20:35:57.479 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0093862909999999997 2023-09-23T20:35:57.479 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.479 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.479 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:35:57.479 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:50.579391+0000", 2023-09-23T20:35:57.479 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0081852620000000004 2023-09-23T20:35:57.480 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.480 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.480 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:35:57.480 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:50.580690+0000", 2023-09-23T20:35:57.480 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0012987890000000001 2023-09-23T20:35:57.480 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.480 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.481 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:35:57.481 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:50.580734+0000", 2023-09-23T20:35:57.481 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.4913999999999997e-05 2023-09-23T20:35:57.481 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.481 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.481 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:35:57.481 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:50.580769+0000", 2023-09-23T20:35:57.481 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.4814999999999998e-05 2023-09-23T20:35:57.481 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:35:57.481 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:35:57.481 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:35:57.481 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.481 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.482 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:156899 7.1 7:894832cf:::benchmark_data_smithi018_308601_object62347:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e354)", 2023-09-23T20:35:57.482 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:34:59.972856+0000", 2023-09-23T20:35:57.482 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 57.284344957000002, 2023-09-23T20:35:57.482 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1521833800000001, 2023-09-23T20:35:57.482 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:35:57.482 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:35:57.482 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:35:57.482 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:35:57.482 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:35:57.482 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 156899 2023-09-23T20:35:57.482 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.483 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:35:57.483 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.483 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:35:57.483 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.972856+0000", 2023-09-23T20:35:57.483 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:35:57.483 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.483 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.483 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:35:57.483 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.972856+0000", 2023-09-23T20:35:57.483 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:35:57.483 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.484 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.484 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:35:57.484 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.972854+0000", 2023-09-23T20:35:57.484 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:35:57.484 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.484 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.484 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:35:57.484 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.972863+0000", 2023-09-23T20:35:57.484 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.738e-06 2023-09-23T20:35:57.484 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.485 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.485 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:35:57.485 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.972876+0000", 2023-09-23T20:35:57.485 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3362e-05 2023-09-23T20:35:57.485 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.485 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.485 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:35:57.485 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.972888+0000", 2023-09-23T20:35:57.485 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1856e-05 2023-09-23T20:35:57.486 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.486 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.486 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:35:57.486 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.975112+0000", 2023-09-23T20:35:57.486 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0022238980000000002 2023-09-23T20:35:57.486 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.487 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.487 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:35:57.487 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.975128+0000", 2023-09-23T20:35:57.487 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.6325999999999999e-05 2023-09-23T20:35:57.487 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.487 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.487 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:35:57.487 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.115938+0000", 2023-09-23T20:35:57.487 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1408093079999999 2023-09-23T20:35:57.487 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.487 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.488 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:35:57.488 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.116085+0000", 2023-09-23T20:35:57.488 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00014754199999999999 2023-09-23T20:35:57.488 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.488 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.488 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 3,4,6", 2023-09-23T20:35:57.488 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.116220+0000", 2023-09-23T20:35:57.489 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000135308 2023-09-23T20:35:57.489 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.489 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.489 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:35:57.489 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.122330+0000", 2023-09-23T20:35:57.489 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.006109377 2023-09-23T20:35:57.489 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.490 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.490 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:35:57.490 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.124884+0000", 2023-09-23T20:35:57.490 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0025545559999999999 2023-09-23T20:35:57.490 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.490 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.491 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:35:57.491 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.124904+0000", 2023-09-23T20:35:57.491 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.9967999999999999e-05 2023-09-23T20:35:57.491 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.491 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.491 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:35:57.491 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.124921+0000", 2023-09-23T20:35:57.491 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.7166000000000001e-05 2023-09-23T20:35:57.491 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.492 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.492 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:35:57.492 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.124957+0000", 2023-09-23T20:35:57.492 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.5417000000000003e-05 2023-09-23T20:35:57.492 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.492 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.492 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:35:57.492 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.125039+0000", 2023-09-23T20:35:57.492 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.2295999999999999e-05 2023-09-23T20:35:57.492 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:35:57.493 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:35:57.493 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:35:57.493 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.493 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.493 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:156906 7.1 7:8761197e:::benchmark_data_smithi018_308601_object62354:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e354)", 2023-09-23T20:35:57.493 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:34:59.972906+0000", 2023-09-23T20:35:57.493 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 57.284294449999997, 2023-09-23T20:35:57.493 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.204579289, 2023-09-23T20:35:57.493 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:35:57.493 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:35:57.494 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:35:57.494 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:35:57.494 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:35:57.494 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 156906 2023-09-23T20:35:57.494 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.494 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:35:57.494 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.494 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:35:57.494 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.972906+0000", 2023-09-23T20:35:57.494 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:35:57.495 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.495 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.495 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:35:57.495 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.972906+0000", 2023-09-23T20:35:57.495 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:35:57.495 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.495 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.495 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:35:57.495 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.972906+0000", 2023-09-23T20:35:57.495 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:35:57.495 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.495 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.496 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:35:57.496 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.972909+0000", 2023-09-23T20:35:57.496 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.6569999999999999e-06 2023-09-23T20:35:57.496 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.496 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.496 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:35:57.496 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.972917+0000", 2023-09-23T20:35:57.496 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.6389999999999992e-06 2023-09-23T20:35:57.496 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.496 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.496 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:35:57.496 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.972923+0000", 2023-09-23T20:35:57.497 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.6810000000000003e-06 2023-09-23T20:35:57.497 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.497 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.497 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:35:57.497 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.975155+0000", 2023-09-23T20:35:57.497 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.002232445 2023-09-23T20:35:57.497 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.497 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.497 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:35:57.497 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.975164+0000", 2023-09-23T20:35:57.497 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.6309999999999992e-06 2023-09-23T20:35:57.497 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.498 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.498 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:35:57.498 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.146302+0000", 2023-09-23T20:35:57.498 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.171138427 2023-09-23T20:35:57.498 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.498 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.498 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:35:57.498 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.146426+0000", 2023-09-23T20:35:57.498 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00012366800000000001 2023-09-23T20:35:57.499 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.499 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.499 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 3,4,6", 2023-09-23T20:35:57.499 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.146558+0000", 2023-09-23T20:35:57.499 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00013182499999999999 2023-09-23T20:35:57.499 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.499 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.499 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:35:57.499 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.150022+0000", 2023-09-23T20:35:57.499 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0034646070000000002 2023-09-23T20:35:57.499 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.500 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.500 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:35:57.500 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.150058+0000", 2023-09-23T20:35:57.500 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.5277000000000001e-05 2023-09-23T20:35:57.500 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.500 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.500 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:35:57.500 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.166990+0000", 2023-09-23T20:35:57.500 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.016932365000000001 2023-09-23T20:35:57.500 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.500 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.500 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:35:57.501 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.177360+0000", 2023-09-23T20:35:57.501 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.010369682 2023-09-23T20:35:57.501 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.501 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.501 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:35:57.501 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.177412+0000", 2023-09-23T20:35:57.501 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.2039000000000001e-05 2023-09-23T20:35:57.501 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.501 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.501 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:35:57.501 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.177486+0000", 2023-09-23T20:35:57.501 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.3830000000000003e-05 2023-09-23T20:35:57.502 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:35:57.502 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:35:57.502 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:35:57.502 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.502 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.502 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:156911 7.1 7:8bb3fc92:::benchmark_data_smithi018_308601_object62359:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e354)", 2023-09-23T20:35:57.502 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:34:59.972928+0000", 2023-09-23T20:35:57.502 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 57.284272631, 2023-09-23T20:35:57.502 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.204718468, 2023-09-23T20:35:57.502 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:35:57.502 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:35:57.503 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:35:57.503 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:35:57.503 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:35:57.503 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 156911 2023-09-23T20:35:57.503 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.503 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:35:57.503 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.503 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:35:57.503 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.972928+0000", 2023-09-23T20:35:57.503 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:35:57.503 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.503 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.504 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:35:57.504 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.972928+0000", 2023-09-23T20:35:57.504 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:35:57.504 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.504 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.504 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:35:57.504 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.972928+0000", 2023-09-23T20:35:57.504 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:35:57.505 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.505 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.505 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:35:57.505 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.972930+0000", 2023-09-23T20:35:57.505 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.5359999999999999e-06 2023-09-23T20:35:57.505 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.505 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.506 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:35:57.506 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.972936+0000", 2023-09-23T20:35:57.506 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.1190000000000002e-06 2023-09-23T20:35:57.506 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.506 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.506 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:35:57.506 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.972942+0000", 2023-09-23T20:35:57.507 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.5369999999999997e-06 2023-09-23T20:35:57.507 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.507 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.507 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:35:57.507 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.975188+0000", 2023-09-23T20:35:57.507 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.002246521 2023-09-23T20:35:57.507 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.508 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.508 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:35:57.508 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.975196+0000", 2023-09-23T20:35:57.508 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.0900000000000005e-06 2023-09-23T20:35:57.508 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.508 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.508 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:35:57.509 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.150087+0000", 2023-09-23T20:35:57.509 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.174890802 2023-09-23T20:35:57.509 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.509 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.509 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:35:57.509 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.150236+0000", 2023-09-23T20:35:57.509 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00014853699999999999 2023-09-23T20:35:57.509 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.509 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.509 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 3,4,6", 2023-09-23T20:35:57.509 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.150378+0000", 2023-09-23T20:35:57.510 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00014263000000000001 2023-09-23T20:35:57.510 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.510 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.510 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:35:57.510 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.157143+0000", 2023-09-23T20:35:57.510 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0067646909999999998 2023-09-23T20:35:57.510 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.510 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.510 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:35:57.510 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.157179+0000", 2023-09-23T20:35:57.511 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.6099000000000001e-05 2023-09-23T20:35:57.511 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.511 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.511 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:35:57.511 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.167022+0000", 2023-09-23T20:35:57.511 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0098431739999999997 2023-09-23T20:35:57.511 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.511 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.511 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:35:57.511 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.177564+0000", 2023-09-23T20:35:57.512 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.010541867999999999 2023-09-23T20:35:57.512 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.512 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.512 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:35:57.512 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.177590+0000", 2023-09-23T20:35:57.512 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.62e-05 2023-09-23T20:35:57.512 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.512 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.513 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:35:57.513 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.177647+0000", 2023-09-23T20:35:57.513 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.6063999999999997e-05 2023-09-23T20:35:57.513 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:35:57.513 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:35:57.513 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:35:57.514 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.514 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.514 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:156944 7.1 7:848d83ce:::benchmark_data_smithi018_308601_object62392:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e354)", 2023-09-23T20:35:57.514 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:34:59.972947+0000", 2023-09-23T20:35:57.514 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 57.284253941000003, 2023-09-23T20:35:57.514 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.207883992, 2023-09-23T20:35:57.515 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:35:57.515 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:35:57.515 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:35:57.515 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:35:57.515 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:35:57.515 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 156944 2023-09-23T20:35:57.516 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.516 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:35:57.516 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.516 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:35:57.516 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.972947+0000", 2023-09-23T20:35:57.516 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:35:57.517 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.517 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.517 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:35:57.517 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.972947+0000", 2023-09-23T20:35:57.517 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:35:57.517 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.518 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.518 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:35:57.518 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.972946+0000", 2023-09-23T20:35:57.518 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:35:57.518 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.518 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.519 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:35:57.519 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.972949+0000", 2023-09-23T20:35:57.519 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.503e-06 2023-09-23T20:35:57.519 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.519 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.519 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:35:57.520 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.972956+0000", 2023-09-23T20:35:57.520 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.9870000000000002e-06 2023-09-23T20:35:57.520 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.520 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.520 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:35:57.520 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.972962+0000", 2023-09-23T20:35:57.521 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.7339999999999996e-06 2023-09-23T20:35:57.521 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.521 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.521 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:35:57.521 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.975220+0000", 2023-09-23T20:35:57.522 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0022582489999999999 2023-09-23T20:35:57.522 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.522 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.522 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:35:57.522 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.975229+0000", 2023-09-23T20:35:57.522 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.3540000000000006e-06 2023-09-23T20:35:57.523 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.523 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.523 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:35:57.523 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.153614+0000", 2023-09-23T20:35:57.523 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1783843549999999 2023-09-23T20:35:57.523 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.524 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.524 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:35:57.524 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.153771+0000", 2023-09-23T20:35:57.524 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000156935 2023-09-23T20:35:57.524 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.524 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.525 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 3,4,6", 2023-09-23T20:35:57.525 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.153909+0000", 2023-09-23T20:35:57.525 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00013803099999999999 2023-09-23T20:35:57.525 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.525 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.525 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:35:57.526 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.166810+0000", 2023-09-23T20:35:57.526 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.012901347000000001 2023-09-23T20:35:57.526 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.526 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.526 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:35:57.526 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.166863+0000", 2023-09-23T20:35:57.527 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.3046e-05 2023-09-23T20:35:57.527 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.527 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.527 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:35:57.527 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.177657+0000", 2023-09-23T20:35:57.527 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.010794419 2023-09-23T20:35:57.528 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.528 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.528 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:35:57.528 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.180631+0000", 2023-09-23T20:35:57.528 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0029738070000000002 2023-09-23T20:35:57.528 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.529 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.529 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:35:57.529 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.180693+0000", 2023-09-23T20:35:57.529 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.1752000000000001e-05 2023-09-23T20:35:57.529 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.529 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.530 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:35:57.530 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.180831+0000", 2023-09-23T20:35:57.530 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00013783600000000001 2023-09-23T20:35:57.530 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:35:57.530 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:35:57.531 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:35:57.531 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.531 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.531 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:156954 7.1 7:8fbb61c7:::benchmark_data_smithi018_308601_object62402:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e354)", 2023-09-23T20:35:57.531 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:34:59.972967+0000", 2023-09-23T20:35:57.531 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 57.284234103999999, 2023-09-23T20:35:57.532 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.208111494, 2023-09-23T20:35:57.532 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:35:57.532 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:35:57.532 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:35:57.532 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:35:57.532 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:35:57.533 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 156954 2023-09-23T20:35:57.533 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.533 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:35:57.533 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.533 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:35:57.533 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.972967+0000", 2023-09-23T20:35:57.534 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:35:57.534 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.534 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.534 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:35:57.534 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.972967+0000", 2023-09-23T20:35:57.534 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:35:57.535 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.535 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.535 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:35:57.535 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.972966+0000", 2023-09-23T20:35:57.535 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:35:57.535 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.535 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.536 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:35:57.536 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.972969+0000", 2023-09-23T20:35:57.536 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.548e-06 2023-09-23T20:35:57.536 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.536 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.536 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:35:57.537 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.972974+0000", 2023-09-23T20:35:57.537 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.6389999999999997e-06 2023-09-23T20:35:57.537 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.537 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.537 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:35:57.537 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.972979+0000", 2023-09-23T20:35:57.538 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.2639999999999999e-06 2023-09-23T20:35:57.538 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.538 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.538 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:35:57.538 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.975256+0000", 2023-09-23T20:35:57.538 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0022768879999999999 2023-09-23T20:35:57.539 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.539 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.539 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:35:57.539 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.975264+0000", 2023-09-23T20:35:57.539 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.0260000000000007e-06 2023-09-23T20:35:57.539 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.540 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.540 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:35:57.540 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.157204+0000", 2023-09-23T20:35:57.540 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.181939297 2023-09-23T20:35:57.540 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.541 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.541 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:35:57.541 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.157336+0000", 2023-09-23T20:35:57.541 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00013253599999999999 2023-09-23T20:35:57.541 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.541 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.542 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 3,4,6", 2023-09-23T20:35:57.542 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.157473+0000", 2023-09-23T20:35:57.542 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000137219 2023-09-23T20:35:57.542 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.542 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.542 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:35:57.543 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.166957+0000", 2023-09-23T20:35:57.543 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0094833369999999997 2023-09-23T20:35:57.543 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.543 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.543 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:35:57.543 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.174176+0000", 2023-09-23T20:35:57.544 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.007219306 2023-09-23T20:35:57.544 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.544 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.544 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:35:57.544 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.177664+0000", 2023-09-23T20:35:57.545 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0034882519999999998 2023-09-23T20:35:57.545 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.545 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.545 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:35:57.545 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.180887+0000", 2023-09-23T20:35:57.545 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0032224430000000002 2023-09-23T20:35:57.546 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.546 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.546 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:35:57.546 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.180953+0000", 2023-09-23T20:35:57.546 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.5775e-05 2023-09-23T20:35:57.547 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:35:57.547 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:35:57.547 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:35:57.547 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.181078+0000", 2023-09-23T20:35:57.547 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00012554600000000001 2023-09-23T20:35:57.547 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:35:57.547 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:35:57.547 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:35:57.547 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:35:57.548 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:35:57.548 INFO:teuthology.orchestra.run.smithi018.stdout:} 2023-09-23T20:35:57.549 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:35:57.549 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:35:57.549 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:35:57.650 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:35:57.682 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:35:57.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:57 smithi149 ceph-mon[170394]: pgmap v875: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 116 active+clean; 4.5 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 472 op/s; 0 B/s, 170 objects/s recovering 2023-09-23T20:35:57.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:57 smithi149 ceph-mon[170394]: 1.1b scrub starts 2023-09-23T20:35:58.288 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:35:58.292 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:35:58.292 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:35:58.293 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:35:58.393 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:35:58.419 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:35:58.453 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:35:58.455 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:35:58.456 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:35:58.557 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:35:58.581 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:58 smithi149 ceph-mon[170394]: from='client.45788 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:35:58.583 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:35:58.617 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:35:58.620 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:35:58.621 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:35:58.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:58 smithi018 ceph-mon[179647]: from='client.45788 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:35:58.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:58 smithi018 ceph-mon[184144]: from='client.45788 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:35:58.721 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:35:58.750 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:35:59.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:35:59 smithi018 ceph-mon[184144]: pgmap v876: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 116 active+clean; 4.5 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 256 op/s; 0 B/s, 146 objects/s recovering 2023-09-23T20:35:59.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:35:59 smithi018 ceph-mon[179647]: pgmap v876: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 116 active+clean; 4.5 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 256 op/s; 0 B/s, 146 objects/s recovering 2023-09-23T20:35:59.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:35:59 smithi149 ceph-mon[170394]: pgmap v876: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 116 active+clean; 4.5 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 256 op/s; 0 B/s, 146 objects/s recovering 2023-09-23T20:36:00.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:00 smithi018 ceph-mon[184144]: 1.15 scrub starts 2023-09-23T20:36:00.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:00 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:36:00.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:00 smithi018 ceph-mon[184144]: 1.b scrub starts 2023-09-23T20:36:00.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:00 smithi018 ceph-mon[179647]: 1.15 scrub starts 2023-09-23T20:36:00.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:00 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:36:00.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:00 smithi018 ceph-mon[179647]: 1.b scrub starts 2023-09-23T20:36:00.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:00 smithi149 ceph-mon[170394]: 1.15 scrub starts 2023-09-23T20:36:00.783 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:00 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:36:00.783 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:00 smithi149 ceph-mon[170394]: 1.b scrub starts 2023-09-23T20:36:01.605 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:36:01.608 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:01 smithi018 ceph-mon[179647]: pgmap v877: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 116 active+clean; 4.5 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 181 op/s; 0 B/s, 146 objects/s recovering 2023-09-23T20:36:01.608 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:01 smithi018 ceph-mon[179647]: failed to encode map e365 with expected crc 2023-09-23T20:36:01.608 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:01 smithi018 ceph-mon[179647]: osdmap e365: 8 total, 8 up, 8 in 2023-09-23T20:36:01.608 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:01 smithi018 ceph-mon[179647]: failed to encode map e365 with expected crc 2023-09-23T20:36:01.608 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:01 smithi018 ceph-mon[179647]: 1.e scrub starts 2023-09-23T20:36:01.608 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:01 smithi018 ceph-mon[179647]: 1.e scrub ok 2023-09-23T20:36:01.608 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:01 smithi018 ceph-mon[179647]: failed to encode map e365 with expected crc 2023-09-23T20:36:01.609 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:01 smithi018 ceph-mon[184144]: pgmap v877: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 116 active+clean; 4.5 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 181 op/s; 0 B/s, 146 objects/s recovering 2023-09-23T20:36:01.609 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:01 smithi018 ceph-mon[184144]: failed to encode map e365 with expected crc 2023-09-23T20:36:01.609 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:01 smithi018 ceph-mon[184144]: osdmap e365: 8 total, 8 up, 8 in 2023-09-23T20:36:01.609 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:01 smithi018 ceph-mon[184144]: failed to encode map e365 with expected crc 2023-09-23T20:36:01.609 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:01 smithi018 ceph-mon[184144]: 1.e scrub starts 2023-09-23T20:36:01.609 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:01 smithi018 ceph-mon[184144]: 1.e scrub ok 2023-09-23T20:36:01.609 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:01 smithi018 ceph-mon[184144]: failed to encode map e365 with expected crc 2023-09-23T20:36:01.610 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:01.610 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:36:01.711 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:36:01.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:01 smithi149 ceph-mon[170394]: pgmap v877: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+remapped+backfill_wait, 116 active+clean; 4.5 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 181 op/s; 0 B/s, 146 objects/s recovering 2023-09-23T20:36:01.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:01 smithi149 ceph-mon[170394]: failed to encode map e365 with expected crc 2023-09-23T20:36:01.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:01 smithi149 ceph-mon[170394]: osdmap e365: 8 total, 8 up, 8 in 2023-09-23T20:36:01.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:01 smithi149 ceph-mon[170394]: failed to encode map e365 with expected crc 2023-09-23T20:36:01.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:01 smithi149 ceph-mon[170394]: 1.e scrub starts 2023-09-23T20:36:01.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:01 smithi149 ceph-mon[170394]: 1.e scrub ok 2023-09-23T20:36:01.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:01 smithi149 ceph-mon[170394]: failed to encode map e365 with expected crc 2023-09-23T20:36:01.739 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:01.774 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:36:01.777 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:01.777 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:36:01.878 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:36:01.912 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:02.033 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T20:36:02.155 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:36:02.159 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:02.159 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:36:02.193 INFO:teuthology.orchestra.run.smithi018.stdout:{"pg_ready":true,"pg_map":{"version":875,"stamp":"2023-09-23T20:35:55.923752+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4679083,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6085,"num_read_kb":13381,"num_write":190052,"num_write_kb":6064302,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":13658,"num_bytes_recovered":656999843,"num_keys_recovered":40,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":120565,"ondisk_log_size":120565,"up":357,"acting":356,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":364,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":3290372,"kb_used_data":388740,"kb_used_omap":32,"kb_used_meta":2901535,"kb_avail":746670844,"statfs":{"total":767960285184,"available":764590944256,"internally_reserved":0,"allocated":398069760,"data_stored":386013767,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32773,"internal_metadata":2971172859},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,0,0,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-372061314,"num_objects":-5678,"num_object_clones":0,"num_object_copies":-17034,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-779,"num_objects_unfound":0,"num_objects_dirty":-5678,"num_whiteouts":0,"num_read":-11,"num_read_kb":-21,"num_write":5672,"num_write_kb":-55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2050,"num_bytes_recovered":-40253,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003842"},"pg_stats":[{"pgid":"1.18","version":"90'161","reported_seq":638,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T20:35:26.648454+0000","last_change":"2023-09-23T20:35:26.644897+0000","last_active":"2023-09-23T20:35:26.648454+0000","last_peered":"2023-09-23T20:35:26.648454+0000","last_clean":"2023-09-23T20:35:26.648454+0000","last_became_active":"2023-09-23T20:35:26.644564+0000","last_became_peered":"2023-09-23T20:35:26.644564+0000","last_unstale":"2023-09-23T20:35:26.648454+0000","last_undegraded":"2023-09-23T20:35:26.648454+0000","last_fullsized":"2023-09-23T20:35:26.648454+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:35:03.551143+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:35:03.551143+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:55:55.872961+0000","scrub_duration":0.033122453000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"69'1","reported_seq":61,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T20:35:26.633063+0000","last_change":"2023-09-23T20:32:33.233853+0000","last_active":"2023-09-23T20:35:26.633063+0000","last_peered":"2023-09-23T20:35:26.633063+0000","last_clean":"2023-09-23T20:35:26.633063+0000","last_became_active":"2023-09-23T20:32:33.233546+0000","last_became_peered":"2023-09-23T20:32:33.233546+0000","last_unstale":"2023-09-23T20:35:26.633063+0000","last_undegraded":"2023-09-23T20:35:26.633063+0000","last_fullsized":"2023-09-23T20:35:26.633063+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:50:27.130808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":243,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T20:35:26.618987+0000","last_change":"2023-09-23T20:32:50.536495+0000","last_active":"2023-09-23T20:35:26.618987+0000","last_peered":"2023-09-23T20:35:26.618987+0000","last_clean":"2023-09-23T20:35:26.618987+0000","last_became_active":"2023-09-23T20:32:50.536180+0000","last_became_peered":"2023-09-23T20:32:50.536180+0000","last_unstale":"2023-09-23T20:35:26.618987+0000","last_undegraded":"2023-09-23T20:35:26.618987+0000","last_fullsized":"2023-09-23T20:35:26.618987+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:01:05.524308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1b","version":"90'161","reported_seq":501,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T20:35:52.426781+0000","last_change":"2023-09-23T20:35:52.426781+0000","last_active":"2023-09-23T20:35:52.426781+0000","last_peered":"2023-09-23T20:35:52.426781+0000","last_clean":"2023-09-23T20:35:52.426781+0000","last_became_active":"2023-09-23T20:35:26.647261+0000","last_became_peered":"2023-09-23T20:35:26.647261+0000","last_unstale":"2023-09-23T20:35:52.426781+0000","last_undegraded":"2023-09-23T20:35:52.426781+0000","last_fullsized":"2023-09-23T20:35:52.426781+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:46.558292+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:46.558292+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0019500940000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":330,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T20:35:26.618255+0000","last_change":"2023-09-23T20:34:59.983035+0000","last_active":"2023-09-23T20:35:26.618255+0000","last_peered":"2023-09-23T20:35:26.618255+0000","last_clean":"2023-09-23T20:35:26.618255+0000","last_became_active":"2023-09-23T20:34:59.982459+0000","last_became_peered":"2023-09-23T20:34:59.982459+0000","last_unstale":"2023-09-23T20:35:26.618255+0000","last_undegraded":"2023-09-23T20:35:26.618255+0000","last_fullsized":"2023-09-23T20:35:26.618255+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:16:48.510783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":331,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T20:35:26.636824+0000","last_change":"2023-09-23T20:25:02.957185+0000","last_active":"2023-09-23T20:35:26.636824+0000","last_peered":"2023-09-23T20:35:26.636824+0000","last_clean":"2023-09-23T20:35:26.636824+0000","last_became_active":"2023-09-23T20:25:02.956689+0000","last_became_peered":"2023-09-23T20:25:02.956689+0000","last_unstale":"2023-09-23T20:35:26.636824+0000","last_undegraded":"2023-09-23T20:35:26.636824+0000","last_fullsized":"2023-09-23T20:35:26.636824+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:04:28.524015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"90'161","reported_seq":433,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T20:35:26.640777+0000","last_change":"2023-09-23T20:35:26.640363+0000","last_active":"2023-09-23T20:35:26.640777+0000","last_peered":"2023-09-23T20:35:26.640777+0000","last_clean":"2023-09-23T20:35:26.640777+0000","last_became_active":"2023-09-23T20:35:26.639912+0000","last_became_peered":"2023-09-23T20:35:26.639912+0000","last_unstale":"2023-09-23T20:35:26.640777+0000","last_undegraded":"2023-09-23T20:35:26.640777+0000","last_fullsized":"2023-09-23T20:35:26.640777+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:35:06.490149+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:35:06.490149+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":176,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T20:35:26.636627+0000","last_change":"2023-09-23T20:32:50.546428+0000","last_active":"2023-09-23T20:35:26.636627+0000","last_peered":"2023-09-23T20:35:26.636627+0000","last_clean":"2023-09-23T20:35:26.636627+0000","last_became_active":"2023-09-23T20:32:50.546120+0000","last_became_peered":"2023-09-23T20:32:50.546120+0000","last_unstale":"2023-09-23T20:35:26.636627+0000","last_undegraded":"2023-09-23T20:35:26.636627+0000","last_fullsized":"2023-09-23T20:35:26.636627+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:00:10.792410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":137,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T20:35:26.636528+0000","last_change":"2023-09-23T20:29:46.112977+0000","last_active":"2023-09-23T20:35:26.636528+0000","last_peered":"2023-09-23T20:35:26.636528+0000","last_clean":"2023-09-23T20:35:26.636528+0000","last_became_active":"2023-09-23T20:29:46.112600+0000","last_became_peered":"2023-09-23T20:29:46.112600+0000","last_unstale":"2023-09-23T20:35:26.636528+0000","last_undegraded":"2023-09-23T20:35:26.636528+0000","last_fullsized":"2023-09-23T20:35:26.636528+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:06:34.132247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"90'161","reported_seq":523,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T20:35:45.530255+0000","last_change":"2023-09-23T20:35:45.530255+0000","last_active":"2023-09-23T20:35:45.530255+0000","last_peered":"2023-09-23T20:35:45.530255+0000","last_clean":"2023-09-23T20:35:45.530255+0000","last_became_active":"2023-09-23T20:35:26.642130+0000","last_became_peered":"2023-09-23T20:35:26.642130+0000","last_unstale":"2023-09-23T20:35:45.530255+0000","last_undegraded":"2023-09-23T20:35:45.530255+0000","last_fullsized":"2023-09-23T20:35:45.530255+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:44.509012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:44.509012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0032186319999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":329,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T20:35:26.642225+0000","last_change":"2023-09-23T20:25:02.969432+0000","last_active":"2023-09-23T20:35:26.642225+0000","last_peered":"2023-09-23T20:35:26.642225+0000","last_clean":"2023-09-23T20:35:26.642225+0000","last_became_active":"2023-09-23T20:25:02.969118+0000","last_became_peered":"2023-09-23T20:25:02.969118+0000","last_unstale":"2023-09-23T20:35:26.642225+0000","last_undegraded":"2023-09-23T20:35:26.642225+0000","last_fullsized":"2023-09-23T20:35:26.642225+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:02:22.453831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":67,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T20:35:26.635375+0000","last_change":"2023-09-23T20:32:01.360359+0000","last_active":"2023-09-23T20:35:26.635375+0000","last_peered":"2023-09-23T20:35:26.635375+0000","last_clean":"2023-09-23T20:35:26.635375+0000","last_became_active":"2023-09-23T20:32:01.360008+0000","last_became_peered":"2023-09-23T20:32:01.360008+0000","last_unstale":"2023-09-23T20:35:26.635375+0000","last_undegraded":"2023-09-23T20:35:26.635375+0000","last_fullsized":"2023-09-23T20:35:26.635375+0000","mapping_epoch":308,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:33:43.135001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.14","version":"90'161","reported_seq":691,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T20:35:26.634760+0000","last_change":"2023-09-23T20:35:02.236398+0000","last_active":"2023-09-23T20:35:26.634760+0000","last_peered":"2023-09-23T20:35:26.634760+0000","last_clean":"2023-09-23T20:35:26.634760+0000","last_became_active":"2023-09-23T20:35:02.235988+0000","last_became_peered":"2023-09-23T20:35:02.235988+0000","last_unstale":"2023-09-23T20:35:26.634760+0000","last_undegraded":"2023-09-23T20:35:26.634760+0000","last_fullsized":"2023-09-23T20:35:26.634760+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:47.479012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:47.479012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:13.519648+0000","scrub_duration":0.0020136590000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":340,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T20:35:26.637461+0000","last_change":"2023-09-23T20:29:45.523808+0000","last_active":"2023-09-23T20:35:26.637461+0000","last_peered":"2023-09-23T20:35:26.637461+0000","last_clean":"2023-09-23T20:35:26.637461+0000","last_became_active":"2023-09-23T20:29:45.523508+0000","last_became_peered":"2023-09-23T20:29:45.523508+0000","last_unstale":"2023-09-23T20:35:26.637461+0000","last_undegraded":"2023-09-23T20:35:26.637461+0000","last_fullsized":"2023-09-23T20:35:26.637461+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:50:55.565352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":163,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T20:35:26.635984+0000","last_change":"2023-09-23T20:32:50.539003+0000","last_active":"2023-09-23T20:35:26.635984+0000","last_peered":"2023-09-23T20:35:26.635984+0000","last_clean":"2023-09-23T20:35:26.635984+0000","last_became_active":"2023-09-23T20:32:50.538596+0000","last_became_peered":"2023-09-23T20:32:50.538596+0000","last_unstale":"2023-09-23T20:35:26.635984+0000","last_undegraded":"2023-09-23T20:35:26.635984+0000","last_fullsized":"2023-09-23T20:35:26.635984+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:56:03.314375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.17","version":"90'161","reported_seq":495,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T20:35:54.439713+0000","last_change":"2023-09-23T20:35:54.439713+0000","last_active":"2023-09-23T20:35:54.439713+0000","last_peered":"2023-09-23T20:35:54.439713+0000","last_clean":"2023-09-23T20:35:54.439713+0000","last_became_active":"2023-09-23T20:35:26.652669+0000","last_became_peered":"2023-09-23T20:35:26.652669+0000","last_unstale":"2023-09-23T20:35:54.439713+0000","last_undegraded":"2023-09-23T20:35:54.439713+0000","last_fullsized":"2023-09-23T20:35:54.439713+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:45.196285+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:45.196285+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":391,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T20:35:26.636379+0000","last_change":"2023-09-23T20:32:33.270617+0000","last_active":"2023-09-23T20:35:26.636379+0000","last_peered":"2023-09-23T20:35:26.636379+0000","last_clean":"2023-09-23T20:35:26.636379+0000","last_became_active":"2023-09-23T20:32:33.227902+0000","last_became_peered":"2023-09-23T20:32:33.227902+0000","last_unstale":"2023-09-23T20:35:26.636379+0000","last_undegraded":"2023-09-23T20:35:26.636379+0000","last_fullsized":"2023-09-23T20:35:26.636379+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:30:26.847831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":360,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":313,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T20:35:26.618865+0000","last_change":"2023-09-23T20:29:46.080766+0000","last_active":"2023-09-23T20:35:26.618865+0000","last_peered":"2023-09-23T20:35:26.618865+0000","last_clean":"2023-09-23T20:35:26.618865+0000","last_became_active":"2023-09-23T20:29:46.080356+0000","last_became_peered":"2023-09-23T20:29:46.080356+0000","last_unstale":"2023-09-23T20:35:26.618865+0000","last_undegraded":"2023-09-23T20:35:26.618865+0000","last_fullsized":"2023-09-23T20:35:26.618865+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:08:01.862270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"90'161","reported_seq":474,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T20:35:29.189072+0000","last_change":"2023-09-23T20:35:26.645192+0000","last_active":"2023-09-23T20:35:29.189072+0000","last_peered":"2023-09-23T20:35:29.189072+0000","last_clean":"2023-09-23T20:35:29.189072+0000","last_became_active":"2023-09-23T20:35:26.644787+0000","last_became_peered":"2023-09-23T20:35:26.644787+0000","last_unstale":"2023-09-23T20:35:29.189072+0000","last_undegraded":"2023-09-23T20:35:29.189072+0000","last_fullsized":"2023-09-23T20:35:29.189072+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:44.211184+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:44.211184+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":71,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T20:35:26.632492+0000","last_change":"2023-09-23T20:32:33.266787+0000","last_active":"2023-09-23T20:35:26.632492+0000","last_peered":"2023-09-23T20:35:26.632492+0000","last_clean":"2023-09-23T20:35:26.632492+0000","last_became_active":"2023-09-23T20:32:33.228482+0000","last_became_peered":"2023-09-23T20:32:33.228482+0000","last_unstale":"2023-09-23T20:35:26.632492+0000","last_undegraded":"2023-09-23T20:35:26.632492+0000","last_fullsized":"2023-09-23T20:35:26.632492+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:13:01.812073+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":61,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T20:35:26.634331+0000","last_change":"2023-09-23T20:32:33.228929+0000","last_active":"2023-09-23T20:35:26.634331+0000","last_peered":"2023-09-23T20:35:26.634331+0000","last_clean":"2023-09-23T20:35:26.634331+0000","last_became_active":"2023-09-23T20:32:33.228595+0000","last_became_peered":"2023-09-23T20:32:33.228595+0000","last_unstale":"2023-09-23T20:35:26.634331+0000","last_undegraded":"2023-09-23T20:35:26.634331+0000","last_fullsized":"2023-09-23T20:35:26.634331+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:17:53.053808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"90'161","reported_seq":594,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T20:35:26.637580+0000","last_change":"2023-09-23T20:31:43.578810+0000","last_active":"2023-09-23T20:35:26.637580+0000","last_peered":"2023-09-23T20:35:26.637580+0000","last_clean":"2023-09-23T20:35:26.637580+0000","last_became_active":"2023-09-23T20:28:05.991304+0000","last_became_peered":"2023-09-23T20:28:05.991304+0000","last_unstale":"2023-09-23T20:35:26.637580+0000","last_undegraded":"2023-09-23T20:35:26.637580+0000","last_fullsized":"2023-09-23T20:35:26.637580+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:43.578666+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:43.578666+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:46:18.999991+0000","scrub_duration":0.0021890379999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":390,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T20:35:26.635176+0000","last_change":"2023-09-23T20:32:33.242943+0000","last_active":"2023-09-23T20:35:26.635176+0000","last_peered":"2023-09-23T20:35:26.635176+0000","last_clean":"2023-09-23T20:35:26.635176+0000","last_became_active":"2023-09-23T20:32:33.225468+0000","last_became_peered":"2023-09-23T20:32:33.225468+0000","last_unstale":"2023-09-23T20:35:26.635176+0000","last_undegraded":"2023-09-23T20:35:26.635176+0000","last_fullsized":"2023-09-23T20:35:26.635176+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:40:47.867573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":65,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T20:35:26.637171+0000","last_change":"2023-09-23T20:32:01.358590+0000","last_active":"2023-09-23T20:35:26.637171+0000","last_peered":"2023-09-23T20:35:26.637171+0000","last_clean":"2023-09-23T20:35:26.637171+0000","last_became_active":"2023-09-23T20:32:01.358345+0000","last_became_peered":"2023-09-23T20:32:01.358345+0000","last_unstale":"2023-09-23T20:35:26.637171+0000","last_undegraded":"2023-09-23T20:35:26.637171+0000","last_fullsized":"2023-09-23T20:35:26.637171+0000","mapping_epoch":308,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:22:21.159708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.10","version":"90'161","reported_seq":594,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T20:35:26.637821+0000","last_change":"2023-09-23T20:31:49.726214+0000","last_active":"2023-09-23T20:35:26.637821+0000","last_peered":"2023-09-23T20:35:26.637821+0000","last_clean":"2023-09-23T20:35:26.637821+0000","last_became_active":"2023-09-23T20:29:07.044892+0000","last_became_peered":"2023-09-23T20:29:07.044892+0000","last_unstale":"2023-09-23T20:35:26.637821+0000","last_undegraded":"2023-09-23T20:35:26.637821+0000","last_fullsized":"2023-09-23T20:35:26.637821+0000","mapping_epoch":253,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":254,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:49.726139+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:49.726139+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:59:11.810516+0000","scrub_duration":0.0015441470000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":329,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T20:35:26.640901+0000","last_change":"2023-09-23T20:25:02.953993+0000","last_active":"2023-09-23T20:35:26.640901+0000","last_peered":"2023-09-23T20:35:26.640901+0000","last_clean":"2023-09-23T20:35:26.640901+0000","last_became_active":"2023-09-23T20:25:02.953672+0000","last_became_peered":"2023-09-23T20:25:02.953672+0000","last_unstale":"2023-09-23T20:35:26.640901+0000","last_undegraded":"2023-09-23T20:35:26.640901+0000","last_fullsized":"2023-09-23T20:35:26.640901+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:26:08.603577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":342,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T20:35:26.637395+0000","last_change":"2023-09-23T20:25:02.949584+0000","last_active":"2023-09-23T20:35:26.637395+0000","last_peered":"2023-09-23T20:35:26.637395+0000","last_clean":"2023-09-23T20:35:26.637395+0000","last_became_active":"2023-09-23T20:25:02.949132+0000","last_became_peered":"2023-09-23T20:25:02.949132+0000","last_unstale":"2023-09-23T20:35:26.637395+0000","last_undegraded":"2023-09-23T20:35:26.637395+0000","last_fullsized":"2023-09-23T20:35:26.637395+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:34:20.509823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"90'161","reported_seq":647,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T20:35:26.620398+0000","last_change":"2023-09-23T20:35:02.238923+0000","last_active":"2023-09-23T20:35:26.620398+0000","last_peered":"2023-09-23T20:35:26.620398+0000","last_clean":"2023-09-23T20:35:26.620398+0000","last_became_active":"2023-09-23T20:35:02.238293+0000","last_became_peered":"2023-09-23T20:35:02.238293+0000","last_unstale":"2023-09-23T20:35:26.620398+0000","last_undegraded":"2023-09-23T20:35:26.620398+0000","last_fullsized":"2023-09-23T20:35:26.620398+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:42.544308+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:42.544308+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:51:22.281932+0000","scrub_duration":0.0029958179999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":347,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T20:35:26.635994+0000","last_change":"2023-09-23T20:32:33.225148+0000","last_active":"2023-09-23T20:35:26.635994+0000","last_peered":"2023-09-23T20:35:26.635994+0000","last_clean":"2023-09-23T20:35:26.635994+0000","last_became_active":"2023-09-23T20:32:33.224805+0000","last_became_peered":"2023-09-23T20:32:33.224805+0000","last_unstale":"2023-09-23T20:35:26.635994+0000","last_undegraded":"2023-09-23T20:35:26.635994+0000","last_fullsized":"2023-09-23T20:35:26.635994+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:55:30.891398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":339,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T20:35:26.636907+0000","last_change":"2023-09-23T20:25:02.957010+0000","last_active":"2023-09-23T20:35:26.636907+0000","last_peered":"2023-09-23T20:35:26.636907+0000","last_clean":"2023-09-23T20:35:26.636907+0000","last_became_active":"2023-09-23T20:25:02.956267+0000","last_became_peered":"2023-09-23T20:25:02.956267+0000","last_unstale":"2023-09-23T20:35:26.636907+0000","last_undegraded":"2023-09-23T20:35:26.636907+0000","last_fullsized":"2023-09-23T20:35:26.636907+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:10:55.299422+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"90'161","reported_seq":613,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T20:35:26.637739+0000","last_change":"2023-09-23T20:31:53.707033+0000","last_active":"2023-09-23T20:35:26.637739+0000","last_peered":"2023-09-23T20:35:26.637739+0000","last_clean":"2023-09-23T20:35:26.637739+0000","last_became_active":"2023-09-23T20:29:46.078873+0000","last_became_peered":"2023-09-23T20:29:46.078873+0000","last_unstale":"2023-09-23T20:35:26.637739+0000","last_undegraded":"2023-09-23T20:35:26.637739+0000","last_fullsized":"2023-09-23T20:35:26.637739+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:53.706934+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:53.706934+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:51:58.039657+0000","scrub_duration":0.0016840900000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":133,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T20:35:26.634559+0000","last_change":"2023-09-23T20:29:04.179171+0000","last_active":"2023-09-23T20:35:26.634559+0000","last_peered":"2023-09-23T20:35:26.634559+0000","last_clean":"2023-09-23T20:35:26.634559+0000","last_became_active":"2023-09-23T20:29:04.128672+0000","last_became_peered":"2023-09-23T20:29:04.128672+0000","last_unstale":"2023-09-23T20:35:26.634559+0000","last_undegraded":"2023-09-23T20:35:26.634559+0000","last_fullsized":"2023-09-23T20:35:26.634559+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:49:15.506106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":176,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T20:35:26.635588+0000","last_change":"2023-09-23T20:32:50.539157+0000","last_active":"2023-09-23T20:35:26.635588+0000","last_peered":"2023-09-23T20:35:26.635588+0000","last_clean":"2023-09-23T20:35:26.635588+0000","last_became_active":"2023-09-23T20:32:50.538889+0000","last_became_peered":"2023-09-23T20:32:50.538889+0000","last_unstale":"2023-09-23T20:35:26.635588+0000","last_undegraded":"2023-09-23T20:35:26.635588+0000","last_fullsized":"2023-09-23T20:35:26.635588+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:28:28.857256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.b","version":"364'11856","reported_seq":11927,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T20:35:48.953056+0000","last_change":"2023-09-23T20:31:49.152343+0000","last_active":"2023-09-23T20:35:48.953056+0000","last_peered":"2023-09-23T20:35:48.953056+0000","last_clean":"2023-09-23T20:35:48.953056+0000","last_became_active":"2023-09-23T20:31:49.152042+0000","last_became_peered":"2023-09-23T20:31:49.152042+0000","last_unstale":"2023-09-23T20:35:48.953056+0000","last_undegraded":"2023-09-23T20:35:48.953056+0000","last_fullsized":"2023-09-23T20:35:48.953056+0000","mapping_epoch":303,"log_start":"329'5639","ondisk_log_start":"329'5639","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":6217,"log_dups_size":0,"ondisk_log_size":6217,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:53:34.237468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11856,"num_write_kb":379392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.d","version":"90'161","reported_seq":656,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T20:35:26.636149+0000","last_change":"2023-09-23T20:34:59.496438+0000","last_active":"2023-09-23T20:35:26.636149+0000","last_peered":"2023-09-23T20:35:26.636149+0000","last_clean":"2023-09-23T20:35:26.636149+0000","last_became_active":"2023-09-23T20:33:38.970527+0000","last_became_peered":"2023-09-23T20:33:38.970527+0000","last_unstale":"2023-09-23T20:35:26.636149+0000","last_undegraded":"2023-09-23T20:35:26.636149+0000","last_fullsized":"2023-09-23T20:35:26.636149+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:34:59.496375+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:34:59.496375+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:46:18.244734+0000","scrub_duration":0.0017843189999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":367,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T20:35:26.635341+0000","last_change":"2023-09-23T20:32:33.283752+0000","last_active":"2023-09-23T20:35:26.635341+0000","last_peered":"2023-09-23T20:35:26.635341+0000","last_clean":"2023-09-23T20:35:26.635341+0000","last_became_active":"2023-09-23T20:32:33.230503+0000","last_became_peered":"2023-09-23T20:32:33.230503+0000","last_unstale":"2023-09-23T20:35:26.635341+0000","last_undegraded":"2023-09-23T20:35:26.635341+0000","last_fullsized":"2023-09-23T20:35:26.635341+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:29.084410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":325,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T20:35:26.637878+0000","last_change":"2023-09-23T20:29:04.128604+0000","last_active":"2023-09-23T20:35:26.637878+0000","last_peered":"2023-09-23T20:35:26.637878+0000","last_clean":"2023-09-23T20:35:26.637878+0000","last_became_active":"2023-09-23T20:29:04.128003+0000","last_became_peered":"2023-09-23T20:29:04.128003+0000","last_unstale":"2023-09-23T20:35:26.637878+0000","last_undegraded":"2023-09-23T20:35:26.637878+0000","last_fullsized":"2023-09-23T20:35:26.637878+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:50:49.385599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"364'11812","reported_seq":11890,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T20:35:48.928717+0000","last_change":"2023-09-23T20:32:50.543873+0000","last_active":"2023-09-23T20:35:48.928717+0000","last_peered":"2023-09-23T20:35:48.928717+0000","last_clean":"2023-09-23T20:35:48.928717+0000","last_became_active":"2023-09-23T20:32:50.537253+0000","last_became_peered":"2023-09-23T20:32:50.537253+0000","last_unstale":"2023-09-23T20:35:48.928717+0000","last_undegraded":"2023-09-23T20:35:48.928717+0000","last_fullsized":"2023-09-23T20:35:48.928717+0000","mapping_epoch":319,"log_start":"310'2700","ondisk_log_start":"310'2700","created":303,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":9112,"log_dups_size":0,"ondisk_log_size":9112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:38:02.452291+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11812,"num_write_kb":377984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,5],"acting":[2,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.c","version":"90'161","reported_seq":535,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T20:35:26.640488+0000","last_change":"2023-09-23T20:33:38.989595+0000","last_active":"2023-09-23T20:35:26.640488+0000","last_peered":"2023-09-23T20:35:26.640488+0000","last_clean":"2023-09-23T20:35:26.640488+0000","last_became_active":"2023-09-23T20:33:38.989255+0000","last_became_peered":"2023-09-23T20:33:38.989255+0000","last_unstale":"2023-09-23T20:35:26.640488+0000","last_undegraded":"2023-09-23T20:35:26.640488+0000","last_fullsized":"2023-09-23T20:35:26.640488+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:41.701171+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:41.701171+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0019955509999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":54613,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":402,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T20:35:26.635714+0000","last_change":"2023-09-23T20:34:59.987981+0000","last_active":"2023-09-23T20:35:26.635714+0000","last_peered":"2023-09-23T20:35:26.635714+0000","last_clean":"2023-09-23T20:35:26.635714+0000","last_became_active":"2023-09-23T20:34:59.985368+0000","last_became_peered":"2023-09-23T20:34:59.985368+0000","last_unstale":"2023-09-23T20:35:26.635714+0000","last_undegraded":"2023-09-23T20:35:26.635714+0000","last_fullsized":"2023-09-23T20:35:26.635714+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:12:43.082493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":61,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T20:35:26.632641+0000","last_change":"2023-09-23T20:32:33.232142+0000","last_active":"2023-09-23T20:35:26.632641+0000","last_peered":"2023-09-23T20:35:26.632641+0000","last_clean":"2023-09-23T20:35:26.632641+0000","last_became_active":"2023-09-23T20:32:33.231683+0000","last_became_peered":"2023-09-23T20:32:33.231683+0000","last_unstale":"2023-09-23T20:35:26.632641+0000","last_undegraded":"2023-09-23T20:35:26.632641+0000","last_fullsized":"2023-09-23T20:35:26.632641+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:02:58.336348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"364'11728","reported_seq":14051,"reported_epoch":364,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T20:35:48.951612+0000","last_change":"2023-09-23T20:35:01.111484+0000","last_active":"2023-09-23T20:35:48.951612+0000","last_peered":"2023-09-23T20:35:48.951612+0000","last_clean":"2023-09-23T20:34:58.962913+0000","last_became_active":"2023-09-23T20:35:01.087993+0000","last_became_peered":"2023-09-23T20:35:01.087993+0000","last_unstale":"2023-09-23T20:35:48.951612+0000","last_undegraded":"2023-09-23T20:35:48.951612+0000","last_fullsized":"2023-09-23T20:35:48.951612+0000","mapping_epoch":354,"log_start":"327'5177","ondisk_log_start":"327'5177","created":303,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":6551,"log_dups_size":0,"ondisk_log_size":6551,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:09:02.548463+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11728,"num_write_kb":375296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1109,"num_bytes_recovered":72679424,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,7],"avail_no_missing":["0","1","4","7"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"90'161","reported_seq":586,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T20:35:26.636404+0000","last_change":"2023-09-23T20:32:36.302809+0000","last_active":"2023-09-23T20:35:26.636404+0000","last_peered":"2023-09-23T20:35:26.636404+0000","last_clean":"2023-09-23T20:35:26.636404+0000","last_became_active":"2023-09-23T20:32:36.302465+0000","last_became_peered":"2023-09-23T20:32:36.302465+0000","last_unstale":"2023-09-23T20:35:26.636404+0000","last_undegraded":"2023-09-23T20:35:26.636404+0000","last_fullsized":"2023-09-23T20:35:26.636404+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:46.217489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:46.217489+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:35:40.488427+0000","scrub_duration":0.001456429,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":336,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T20:35:26.636207+0000","last_change":"2023-09-23T20:29:04.184893+0000","last_active":"2023-09-23T20:35:26.636207+0000","last_peered":"2023-09-23T20:35:26.636207+0000","last_clean":"2023-09-23T20:35:26.636207+0000","last_became_active":"2023-09-23T20:29:04.152723+0000","last_became_peered":"2023-09-23T20:29:04.152723+0000","last_unstale":"2023-09-23T20:35:26.636207+0000","last_undegraded":"2023-09-23T20:35:26.636207+0000","last_fullsized":"2023-09-23T20:35:26.636207+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:51:27.017151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":137,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T20:35:26.632635+0000","last_change":"2023-09-23T20:29:46.086912+0000","last_active":"2023-09-23T20:35:26.632635+0000","last_peered":"2023-09-23T20:35:26.632635+0000","last_clean":"2023-09-23T20:35:26.632635+0000","last_became_active":"2023-09-23T20:29:46.085471+0000","last_became_peered":"2023-09-23T20:29:46.085471+0000","last_unstale":"2023-09-23T20:35:26.632635+0000","last_undegraded":"2023-09-23T20:35:26.632635+0000","last_fullsized":"2023-09-23T20:35:26.632635+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:04:55.751660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.8","version":"364'11802","reported_seq":19168,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T20:35:48.947945+0000","last_change":"2023-09-23T20:34:05.581246+0000","last_active":"2023-09-23T20:35:48.947945+0000","last_peered":"2023-09-23T20:35:48.947945+0000","last_clean":"2023-09-23T20:35:48.947945+0000","last_became_active":"2023-09-23T20:34:05.560057+0000","last_became_peered":"2023-09-23T20:34:05.560057+0000","last_unstale":"2023-09-23T20:35:48.947945+0000","last_undegraded":"2023-09-23T20:35:48.947945+0000","last_fullsized":"2023-09-23T20:35:48.947945+0000","mapping_epoch":338,"log_start":"310'2700","ondisk_log_start":"310'2700","created":303,"last_epoch_clean":339,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":9102,"log_dups_size":0,"ondisk_log_size":9102,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:08:18.482746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11802,"num_write_kb":377664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3638,"num_bytes_recovered":238419968,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.e","version":"90'161","reported_seq":454,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T20:35:26.633484+0000","last_change":"2023-09-23T20:35:02.233909+0000","last_active":"2023-09-23T20:35:26.633484+0000","last_peered":"2023-09-23T20:35:26.633484+0000","last_clean":"2023-09-23T20:35:26.633484+0000","last_became_active":"2023-09-23T20:35:02.233536+0000","last_became_peered":"2023-09-23T20:35:02.233536+0000","last_unstale":"2023-09-23T20:35:26.633484+0000","last_undegraded":"2023-09-23T20:35:26.633484+0000","last_fullsized":"2023-09-23T20:35:26.633484+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:44.386120+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:44.386120+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":370,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T20:35:26.637259+0000","last_change":"2023-09-23T20:29:04.138284+0000","last_active":"2023-09-23T20:35:26.637259+0000","last_peered":"2023-09-23T20:35:26.637259+0000","last_clean":"2023-09-23T20:35:26.637259+0000","last_became_active":"2023-09-23T20:29:04.137960+0000","last_became_peered":"2023-09-23T20:29:04.137960+0000","last_unstale":"2023-09-23T20:35:26.637259+0000","last_undegraded":"2023-09-23T20:35:26.637259+0000","last_fullsized":"2023-09-23T20:35:26.637259+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:18:59.548311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":347,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T20:35:26.636962+0000","last_change":"2023-09-23T20:32:33.236122+0000","last_active":"2023-09-23T20:35:26.636962+0000","last_peered":"2023-09-23T20:35:26.636962+0000","last_clean":"2023-09-23T20:35:26.636962+0000","last_became_active":"2023-09-23T20:32:33.235384+0000","last_became_peered":"2023-09-23T20:32:33.235384+0000","last_unstale":"2023-09-23T20:35:26.636962+0000","last_undegraded":"2023-09-23T20:35:26.636962+0000","last_fullsized":"2023-09-23T20:35:26.636962+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:56.843733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"364'11956","reported_seq":12027,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T20:35:48.963976+0000","last_change":"2023-09-23T20:31:49.158461+0000","last_active":"2023-09-23T20:35:48.963976+0000","last_peered":"2023-09-23T20:35:48.963976+0000","last_clean":"2023-09-23T20:35:48.963976+0000","last_became_active":"2023-09-23T20:31:49.158188+0000","last_became_peered":"2023-09-23T20:31:49.158188+0000","last_unstale":"2023-09-23T20:35:48.963976+0000","last_undegraded":"2023-09-23T20:35:48.963976+0000","last_fullsized":"2023-09-23T20:35:48.963976+0000","mapping_epoch":303,"log_start":"327'5362","ondisk_log_start":"327'5362","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":6594,"log_dups_size":0,"ondisk_log_size":6594,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:31:57.678090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11956,"num_write_kb":382592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":769,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T20:35:26.635059+0000","last_change":"2023-09-23T20:35:02.523308+0000","last_active":"2023-09-23T20:35:26.635059+0000","last_peered":"2023-09-23T20:35:26.635059+0000","last_clean":"2023-09-23T20:35:26.635059+0000","last_became_active":"2023-09-23T20:33:38.979989+0000","last_became_peered":"2023-09-23T20:33:38.979989+0000","last_unstale":"2023-09-23T20:35:26.635059+0000","last_undegraded":"2023-09-23T20:35:26.635059+0000","last_fullsized":"2023-09-23T20:35:26.635059+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:35:02.523203+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:35:02.523203+0000","last_clean_scrub_stamp":"2023-09-23T20:35:02.523203+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:02:41.472958+0000","scrub_duration":0.0085927989999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":99,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T20:35:26.637043+0000","last_change":"2023-09-23T20:32:50.538500+0000","last_active":"2023-09-23T20:35:26.637043+0000","last_peered":"2023-09-23T20:35:26.637043+0000","last_clean":"2023-09-23T20:35:26.637043+0000","last_became_active":"2023-09-23T20:32:50.538152+0000","last_became_peered":"2023-09-23T20:32:50.538152+0000","last_unstale":"2023-09-23T20:35:26.637043+0000","last_undegraded":"2023-09-23T20:35:26.637043+0000","last_fullsized":"2023-09-23T20:35:26.637043+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:00:41.103508+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":61,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T20:35:26.632573+0000","last_change":"2023-09-23T20:32:33.231614+0000","last_active":"2023-09-23T20:35:26.632573+0000","last_peered":"2023-09-23T20:35:26.632573+0000","last_clean":"2023-09-23T20:35:26.632573+0000","last_became_active":"2023-09-23T20:32:33.231207+0000","last_became_peered":"2023-09-23T20:32:33.231207+0000","last_unstale":"2023-09-23T20:35:26.632573+0000","last_undegraded":"2023-09-23T20:35:26.632573+0000","last_fullsized":"2023-09-23T20:35:26.632573+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:46:09.780829+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"364'11848","reported_seq":11919,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T20:35:48.946158+0000","last_change":"2023-09-23T20:31:49.155754+0000","last_active":"2023-09-23T20:35:48.946158+0000","last_peered":"2023-09-23T20:35:48.946158+0000","last_clean":"2023-09-23T20:35:48.946158+0000","last_became_active":"2023-09-23T20:31:49.155363+0000","last_became_peered":"2023-09-23T20:31:49.155363+0000","last_unstale":"2023-09-23T20:35:48.946158+0000","last_undegraded":"2023-09-23T20:35:48.946158+0000","last_fullsized":"2023-09-23T20:35:48.946158+0000","mapping_epoch":303,"log_start":"310'2700","ondisk_log_start":"310'2700","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":9148,"log_dups_size":0,"ondisk_log_size":9148,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:45:42.661641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11848,"num_write_kb":379136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":614,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T20:35:26.637739+0000","last_change":"2023-09-23T20:35:00.496271+0000","last_active":"2023-09-23T20:35:26.637739+0000","last_peered":"2023-09-23T20:35:26.637739+0000","last_clean":"2023-09-23T20:35:26.637739+0000","last_became_active":"2023-09-23T20:33:38.996273+0000","last_became_peered":"2023-09-23T20:33:38.996273+0000","last_unstale":"2023-09-23T20:35:26.637739+0000","last_undegraded":"2023-09-23T20:35:26.637739+0000","last_fullsized":"2023-09-23T20:35:26.637739+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:35:00.496197+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:35:00.496197+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:30:15.813493+0000","scrub_duration":0.0016339779999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":159,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T20:35:26.632715+0000","last_change":"2023-09-23T20:32:33.229700+0000","last_active":"2023-09-23T20:35:26.632715+0000","last_peered":"2023-09-23T20:35:26.632715+0000","last_clean":"2023-09-23T20:35:26.632715+0000","last_became_active":"2023-09-23T20:32:33.229372+0000","last_became_peered":"2023-09-23T20:32:33.229372+0000","last_unstale":"2023-09-23T20:35:26.632715+0000","last_undegraded":"2023-09-23T20:35:26.632715+0000","last_fullsized":"2023-09-23T20:35:26.632715+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:18:10.958227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":20,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T20:35:26.633363+0000","last_change":"2023-09-23T20:34:59.985545+0000","last_active":"2023-09-23T20:35:26.633363+0000","last_peered":"2023-09-23T20:35:26.633363+0000","last_clean":"2023-09-23T20:35:26.633363+0000","last_became_active":"2023-09-23T20:34:59.985090+0000","last_became_peered":"2023-09-23T20:34:59.985090+0000","last_unstale":"2023-09-23T20:35:26.633363+0000","last_undegraded":"2023-09-23T20:35:26.633363+0000","last_fullsized":"2023-09-23T20:35:26.633363+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:21:40.825065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"364'11992","reported_seq":12062,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T20:35:48.965369+0000","last_change":"2023-09-23T20:31:49.158641+0000","last_active":"2023-09-23T20:35:48.965369+0000","last_peered":"2023-09-23T20:35:48.965369+0000","last_clean":"2023-09-23T20:35:48.965369+0000","last_became_active":"2023-09-23T20:31:49.157604+0000","last_became_peered":"2023-09-23T20:31:49.157604+0000","last_unstale":"2023-09-23T20:35:48.965369+0000","last_undegraded":"2023-09-23T20:35:48.965369+0000","last_fullsized":"2023-09-23T20:35:48.965369+0000","mapping_epoch":303,"log_start":"327'5512","ondisk_log_start":"327'5512","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":6480,"log_dups_size":0,"ondisk_log_size":6480,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:54:04.726000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":11992,"num_write_kb":383681,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.b","version":"90'161","reported_seq":616,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T20:35:26.620480+0000","last_change":"2023-09-23T20:33:38.992494+0000","last_active":"2023-09-23T20:35:26.620480+0000","last_peered":"2023-09-23T20:35:26.620480+0000","last_clean":"2023-09-23T20:35:26.620480+0000","last_became_active":"2023-09-23T20:33:38.991910+0000","last_became_peered":"2023-09-23T20:33:38.991910+0000","last_unstale":"2023-09-23T20:35:26.620480+0000","last_undegraded":"2023-09-23T20:35:26.620480+0000","last_fullsized":"2023-09-23T20:35:26.620480+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:46.558292+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:46.558292+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0023048779999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":20,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T20:35:26.632405+0000","last_change":"2023-09-23T20:34:59.983122+0000","last_active":"2023-09-23T20:35:26.632405+0000","last_peered":"2023-09-23T20:35:26.632405+0000","last_clean":"2023-09-23T20:35:26.632405+0000","last_became_active":"2023-09-23T20:34:59.982809+0000","last_became_peered":"2023-09-23T20:34:59.982809+0000","last_unstale":"2023-09-23T20:35:26.632405+0000","last_undegraded":"2023-09-23T20:35:26.632405+0000","last_fullsized":"2023-09-23T20:35:26.632405+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:49:40.839723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":61,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T20:35:26.633661+0000","last_change":"2023-09-23T20:32:33.224963+0000","last_active":"2023-09-23T20:35:26.633661+0000","last_peered":"2023-09-23T20:35:26.633661+0000","last_clean":"2023-09-23T20:35:26.633661+0000","last_became_active":"2023-09-23T20:32:33.224634+0000","last_became_peered":"2023-09-23T20:32:33.224634+0000","last_unstale":"2023-09-23T20:35:26.633661+0000","last_undegraded":"2023-09-23T20:35:26.633661+0000","last_fullsized":"2023-09-23T20:35:26.633661+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:41:40.314140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"364'11860","reported_seq":11929,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T20:35:48.936084+0000","last_change":"2023-09-23T20:31:49.157247+0000","last_active":"2023-09-23T20:35:48.936084+0000","last_peered":"2023-09-23T20:35:48.936084+0000","last_clean":"2023-09-23T20:35:48.936084+0000","last_became_active":"2023-09-23T20:31:49.156961+0000","last_became_peered":"2023-09-23T20:31:49.156961+0000","last_unstale":"2023-09-23T20:35:48.936084+0000","last_undegraded":"2023-09-23T20:35:48.936084+0000","last_fullsized":"2023-09-23T20:35:48.936084+0000","mapping_epoch":303,"log_start":"327'5415","ondisk_log_start":"327'5415","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":6445,"log_dups_size":0,"ondisk_log_size":6445,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:49:17.346795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11860,"num_write_kb":379520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":657,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T20:35:26.637709+0000","last_change":"2023-09-23T20:34:59.481511+0000","last_active":"2023-09-23T20:35:26.637709+0000","last_peered":"2023-09-23T20:35:26.637709+0000","last_clean":"2023-09-23T20:35:26.637709+0000","last_became_active":"2023-09-23T20:33:38.988954+0000","last_became_peered":"2023-09-23T20:33:38.988954+0000","last_unstale":"2023-09-23T20:35:26.637709+0000","last_undegraded":"2023-09-23T20:35:26.637709+0000","last_fullsized":"2023-09-23T20:35:26.637709+0000","mapping_epoch":253,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:34:59.481446+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:34:59.481446+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:53:42.768648+0000","scrub_duration":0.001554773,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":177,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T20:35:26.618233+0000","last_change":"2023-09-23T20:32:50.530690+0000","last_active":"2023-09-23T20:35:26.618233+0000","last_peered":"2023-09-23T20:35:26.618233+0000","last_clean":"2023-09-23T20:35:26.618233+0000","last_became_active":"2023-09-23T20:32:50.530376+0000","last_became_peered":"2023-09-23T20:32:50.530376+0000","last_unstale":"2023-09-23T20:35:26.618233+0000","last_undegraded":"2023-09-23T20:35:26.618233+0000","last_fullsized":"2023-09-23T20:35:26.618233+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:24:53.184959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":185,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T20:35:26.631016+0000","last_change":"2023-09-23T20:32:33.230582+0000","last_active":"2023-09-23T20:35:26.631016+0000","last_peered":"2023-09-23T20:35:26.631016+0000","last_clean":"2023-09-23T20:35:26.631016+0000","last_became_active":"2023-09-23T20:32:33.230257+0000","last_became_peered":"2023-09-23T20:32:33.230257+0000","last_unstale":"2023-09-23T20:35:26.631016+0000","last_undegraded":"2023-09-23T20:35:26.631016+0000","last_fullsized":"2023-09-23T20:35:26.631016+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:11:45.333096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"364'11742","reported_seq":12878,"reported_epoch":364,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T20:35:48.972815+0000","last_change":"2023-09-23T20:35:01.122419+0000","last_active":"2023-09-23T20:35:48.972815+0000","last_peered":"2023-09-23T20:35:48.972815+0000","last_clean":"2023-09-23T20:34:58.841504+0000","last_became_active":"2023-09-23T20:35:01.087671+0000","last_became_peered":"2023-09-23T20:35:01.087671+0000","last_unstale":"2023-09-23T20:35:48.972815+0000","last_undegraded":"2023-09-23T20:35:48.972815+0000","last_fullsized":"2023-09-23T20:35:48.972815+0000","mapping_epoch":354,"log_start":"329'5591","ondisk_log_start":"329'5591","created":303,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":6151,"log_dups_size":0,"ondisk_log_size":6151,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:29:52.175555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11742,"num_write_kb":375744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":521,"num_bytes_recovered":34144256,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[1,3,6],"avail_no_missing":["1","3","4","6"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":613,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T20:35:26.636481+0000","last_change":"2023-09-23T20:35:04.102089+0000","last_active":"2023-09-23T20:35:26.636481+0000","last_peered":"2023-09-23T20:35:26.636481+0000","last_clean":"2023-09-23T20:35:26.636481+0000","last_became_active":"2023-09-23T20:33:38.978580+0000","last_became_peered":"2023-09-23T20:33:38.978580+0000","last_unstale":"2023-09-23T20:35:26.636481+0000","last_undegraded":"2023-09-23T20:35:26.636481+0000","last_fullsized":"2023-09-23T20:35:26.636481+0000","mapping_epoch":253,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:35:04.101972+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:35:04.101972+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:44:38.790944+0000","scrub_duration":0.0063459650000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"86'119","reported_seq":669,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T20:35:26.621022+0000","last_change":"2023-09-23T20:29:45.521257+0000","last_active":"2023-09-23T20:35:26.621022+0000","last_peered":"2023-09-23T20:35:26.621022+0000","last_clean":"2023-09-23T20:35:26.621022+0000","last_became_active":"2023-09-23T20:29:45.520546+0000","last_became_peered":"2023-09-23T20:29:45.520546+0000","last_unstale":"2023-09-23T20:35:26.621022+0000","last_undegraded":"2023-09-23T20:35:26.621022+0000","last_fullsized":"2023-09-23T20:35:26.621022+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":119,"log_dups_size":0,"ondisk_log_size":119,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:58:56.085777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032705499999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":523,"num_read_kb":3503,"num_write":136,"num_write_kb":3128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":380,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T20:35:26.637299+0000","last_change":"2023-09-23T20:25:39.637160+0000","last_active":"2023-09-23T20:35:26.637299+0000","last_peered":"2023-09-23T20:35:26.637299+0000","last_clean":"2023-09-23T20:35:26.637299+0000","last_became_active":"2023-09-23T20:25:39.636860+0000","last_became_peered":"2023-09-23T20:25:39.636860+0000","last_unstale":"2023-09-23T20:35:26.637299+0000","last_undegraded":"2023-09-23T20:35:26.637299+0000","last_fullsized":"2023-09-23T20:35:26.637299+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:21:47.860063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":298,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T20:35:26.618378+0000","last_change":"2023-09-23T20:34:59.985572+0000","last_active":"2023-09-23T20:35:26.618378+0000","last_peered":"2023-09-23T20:35:26.618378+0000","last_clean":"2023-09-23T20:35:26.618378+0000","last_became_active":"2023-09-23T20:34:59.985241+0000","last_became_peered":"2023-09-23T20:34:59.985241+0000","last_unstale":"2023-09-23T20:35:26.618378+0000","last_undegraded":"2023-09-23T20:35:26.618378+0000","last_fullsized":"2023-09-23T20:35:26.618378+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:54:57.081334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"364'11904","reported_seq":11973,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T20:35:48.939879+0000","last_change":"2023-09-23T20:31:49.157301+0000","last_active":"2023-09-23T20:35:48.939879+0000","last_peered":"2023-09-23T20:35:48.939879+0000","last_clean":"2023-09-23T20:35:48.939879+0000","last_became_active":"2023-09-23T20:31:49.156992+0000","last_became_peered":"2023-09-23T20:31:49.156992+0000","last_unstale":"2023-09-23T20:35:48.939879+0000","last_undegraded":"2023-09-23T20:35:48.939879+0000","last_fullsized":"2023-09-23T20:35:48.939879+0000","mapping_epoch":303,"log_start":"327'5466","ondisk_log_start":"327'5466","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":6438,"log_dups_size":0,"ondisk_log_size":6438,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:21:35.645838+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11904,"num_write_kb":380928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":608,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T20:35:26.637230+0000","last_change":"2023-09-23T20:31:45.785521+0000","last_active":"2023-09-23T20:35:26.637230+0000","last_peered":"2023-09-23T20:35:26.637230+0000","last_clean":"2023-09-23T20:35:26.637230+0000","last_became_active":"2023-09-23T20:27:55.213394+0000","last_became_peered":"2023-09-23T20:27:55.213394+0000","last_unstale":"2023-09-23T20:35:26.637230+0000","last_undegraded":"2023-09-23T20:35:26.637230+0000","last_fullsized":"2023-09-23T20:35:26.637230+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:45.785450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:45.785450+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:21:26.092578+0000","scrub_duration":0.001558306,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"86'106","reported_seq":1082,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T20:35:26.637575+0000","last_change":"2023-09-23T20:31:31.531882+0000","last_active":"2023-09-23T20:35:26.637575+0000","last_peered":"2023-09-23T20:35:26.637575+0000","last_clean":"2023-09-23T20:35:26.637575+0000","last_became_active":"2023-09-23T20:29:46.078501+0000","last_became_peered":"2023-09-23T20:29:46.078501+0000","last_unstale":"2023-09-23T20:35:26.637575+0000","last_undegraded":"2023-09-23T20:35:26.637575+0000","last_fullsized":"2023-09-23T20:35:26.637575+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":106,"log_dups_size":0,"ondisk_log_size":106,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:37.015974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00099302599999999993,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1105,"num_read_kb":1689,"num_write":100,"num_write_kb":404,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":61,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T20:35:26.634197+0000","last_change":"2023-09-23T20:32:33.230242+0000","last_active":"2023-09-23T20:35:26.634197+0000","last_peered":"2023-09-23T20:35:26.634197+0000","last_clean":"2023-09-23T20:35:26.634197+0000","last_became_active":"2023-09-23T20:32:33.229732+0000","last_became_peered":"2023-09-23T20:32:33.229732+0000","last_unstale":"2023-09-23T20:35:26.634197+0000","last_undegraded":"2023-09-23T20:35:26.634197+0000","last_fullsized":"2023-09-23T20:35:26.634197+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:27:55.538429+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":322,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T20:35:26.620936+0000","last_change":"2023-09-23T20:29:45.533406+0000","last_active":"2023-09-23T20:35:26.620936+0000","last_peered":"2023-09-23T20:35:26.620936+0000","last_clean":"2023-09-23T20:35:26.620936+0000","last_became_active":"2023-09-23T20:29:45.526001+0000","last_became_peered":"2023-09-23T20:29:45.526001+0000","last_unstale":"2023-09-23T20:35:26.620936+0000","last_undegraded":"2023-09-23T20:35:26.620936+0000","last_fullsized":"2023-09-23T20:35:26.620936+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:43.787251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.4","version":"364'11722","reported_seq":21299,"reported_epoch":364,"state":"active+recovering+undersized+remapped","last_fresh":"2023-09-23T20:35:55.669358+0000","last_change":"2023-09-23T20:35:00.204596+0000","last_active":"2023-09-23T20:35:55.669358+0000","last_peered":"2023-09-23T20:35:55.669358+0000","last_clean":"2023-09-23T20:32:32.192526+0000","last_became_active":"2023-09-23T20:35:00.124743+0000","last_became_peered":"2023-09-23T20:35:00.124743+0000","last_unstale":"2023-09-23T20:35:55.669358+0000","last_undegraded":"2023-09-23T20:35:55.669358+0000","last_fullsized":"2023-09-23T20:34:59.968003+0000","mapping_epoch":353,"log_start":"327'5184","ondisk_log_start":"327'5184","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":6538,"log_dups_size":0,"ondisk_log_size":6538,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:40:39.030291+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11722,"num_write_kb":375104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4739,"num_bytes_recovered":136642560,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,1],"avail_no_missing":["0","1"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":836,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T20:35:26.635076+0000","last_change":"2023-09-23T20:35:02.260708+0000","last_active":"2023-09-23T20:35:26.635076+0000","last_peered":"2023-09-23T20:35:26.635076+0000","last_clean":"2023-09-23T20:35:26.635076+0000","last_became_active":"2023-09-23T20:35:02.260348+0000","last_became_peered":"2023-09-23T20:35:02.260348+0000","last_unstale":"2023-09-23T20:35:26.635076+0000","last_undegraded":"2023-09-23T20:35:26.635076+0000","last_fullsized":"2023-09-23T20:35:26.635076+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:43.390908+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:43.390908+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:09:14.144070+0000","scrub_duration":0.0032721,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"86'130","reported_seq":677,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T20:35:26.635094+0000","last_change":"2023-09-23T20:25:39.637458+0000","last_active":"2023-09-23T20:35:26.635094+0000","last_peered":"2023-09-23T20:35:26.635094+0000","last_clean":"2023-09-23T20:35:26.635094+0000","last_became_active":"2023-09-23T20:25:39.636773+0000","last_became_peered":"2023-09-23T20:25:39.636773+0000","last_unstale":"2023-09-23T20:35:26.635094+0000","last_undegraded":"2023-09-23T20:35:26.635094+0000","last_fullsized":"2023-09-23T20:35:26.635094+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:00:32.810564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011300049999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":512,"num_read_kb":1820,"num_write":133,"num_write_kb":1347,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":313,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T20:35:41.651466+0000","last_change":"2023-09-23T20:29:46.073394+0000","last_active":"2023-09-23T20:35:41.651466+0000","last_peered":"2023-09-23T20:35:41.651466+0000","last_clean":"2023-09-23T20:35:41.651466+0000","last_became_active":"2023-09-23T20:29:46.072067+0000","last_became_peered":"2023-09-23T20:29:46.072067+0000","last_unstale":"2023-09-23T20:35:41.651466+0000","last_undegraded":"2023-09-23T20:35:41.651466+0000","last_fullsized":"2023-09-23T20:35:41.651466+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:01:56.944614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":304,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T20:35:26.630931+0000","last_change":"2023-09-23T20:29:45.533185+0000","last_active":"2023-09-23T20:35:26.630931+0000","last_peered":"2023-09-23T20:35:26.630931+0000","last_clean":"2023-09-23T20:35:26.630931+0000","last_became_active":"2023-09-23T20:29:45.526765+0000","last_became_peered":"2023-09-23T20:29:45.526765+0000","last_unstale":"2023-09-23T20:35:26.630931+0000","last_undegraded":"2023-09-23T20:35:26.630931+0000","last_fullsized":"2023-09-23T20:35:26.630931+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:18:33.063215+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"364'11868","reported_seq":11939,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T20:35:48.848065+0000","last_change":"2023-09-23T20:31:49.158194+0000","last_active":"2023-09-23T20:35:48.848065+0000","last_peered":"2023-09-23T20:35:48.848065+0000","last_clean":"2023-09-23T20:35:48.848065+0000","last_became_active":"2023-09-23T20:31:49.157875+0000","last_became_peered":"2023-09-23T20:31:49.157875+0000","last_unstale":"2023-09-23T20:35:48.848065+0000","last_undegraded":"2023-09-23T20:35:48.848065+0000","last_fullsized":"2023-09-23T20:35:48.848065+0000","mapping_epoch":303,"log_start":"327'5313","ondisk_log_start":"327'5313","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":6555,"log_dups_size":0,"ondisk_log_size":6555,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:04:42.411181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11868,"num_write_kb":379776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"86'194","reported_seq":1012,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T20:35:26.637820+0000","last_change":"2023-09-23T20:25:40.618665+0000","last_active":"2023-09-23T20:35:26.637820+0000","last_peered":"2023-09-23T20:35:26.637820+0000","last_clean":"2023-09-23T20:35:26.637820+0000","last_became_active":"2023-09-23T20:25:40.618087+0000","last_became_peered":"2023-09-23T20:25:40.618087+0000","last_unstale":"2023-09-23T20:35:26.637820+0000","last_undegraded":"2023-09-23T20:35:26.637820+0000","last_fullsized":"2023-09-23T20:35:26.637820+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":194,"log_dups_size":0,"ondisk_log_size":194,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:07:16.114968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00037995999999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1025,"num_read_kb":1201,"num_write":173,"num_write_kb":1276,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":797,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T20:35:26.635521+0000","last_change":"2023-09-23T20:31:45.343835+0000","last_active":"2023-09-23T20:35:26.635521+0000","last_peered":"2023-09-23T20:35:26.635521+0000","last_clean":"2023-09-23T20:35:26.635521+0000","last_became_active":"2023-09-23T20:29:07.042407+0000","last_became_peered":"2023-09-23T20:29:07.042407+0000","last_unstale":"2023-09-23T20:35:26.635521+0000","last_undegraded":"2023-09-23T20:35:26.635521+0000","last_fullsized":"2023-09-23T20:35:26.635521+0000","mapping_epoch":253,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":254,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:45.343742+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:45.343742+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:33:12.745366+0000","scrub_duration":0.001475802,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":163,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T20:35:26.637624+0000","last_change":"2023-09-23T20:32:50.534315+0000","last_active":"2023-09-23T20:35:26.637624+0000","last_peered":"2023-09-23T20:35:26.637624+0000","last_clean":"2023-09-23T20:35:26.637624+0000","last_became_active":"2023-09-23T20:32:50.533896+0000","last_became_peered":"2023-09-23T20:32:50.533896+0000","last_unstale":"2023-09-23T20:35:26.637624+0000","last_undegraded":"2023-09-23T20:35:26.637624+0000","last_fullsized":"2023-09-23T20:35:26.637624+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:06:53.464923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":143,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T20:35:26.636485+0000","last_change":"2023-09-23T20:32:33.225012+0000","last_active":"2023-09-23T20:35:26.636485+0000","last_peered":"2023-09-23T20:35:26.636485+0000","last_clean":"2023-09-23T20:35:26.636485+0000","last_became_active":"2023-09-23T20:32:33.224699+0000","last_became_peered":"2023-09-23T20:32:33.224699+0000","last_unstale":"2023-09-23T20:35:26.636485+0000","last_undegraded":"2023-09-23T20:35:26.636485+0000","last_fullsized":"2023-09-23T20:35:26.636485+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:10:06.267878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"364'11926","reported_seq":11995,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T20:35:48.964374+0000","last_change":"2023-09-23T20:31:49.154418+0000","last_active":"2023-09-23T20:35:48.964374+0000","last_peered":"2023-09-23T20:35:48.964374+0000","last_clean":"2023-09-23T20:35:48.964374+0000","last_became_active":"2023-09-23T20:31:49.154182+0000","last_became_peered":"2023-09-23T20:31:49.154182+0000","last_unstale":"2023-09-23T20:35:48.964374+0000","last_undegraded":"2023-09-23T20:35:48.964374+0000","last_fullsized":"2023-09-23T20:35:48.964374+0000","mapping_epoch":303,"log_start":"329'5718","ondisk_log_start":"329'5718","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":6208,"log_dups_size":0,"ondisk_log_size":6208,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:44:10.087486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11926,"num_write_kb":381632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":657,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T20:35:26.620315+0000","last_change":"2023-09-23T20:35:02.238632+0000","last_active":"2023-09-23T20:35:26.620315+0000","last_peered":"2023-09-23T20:35:26.620315+0000","last_clean":"2023-09-23T20:35:26.620315+0000","last_became_active":"2023-09-23T20:35:02.237812+0000","last_became_peered":"2023-09-23T20:35:02.237812+0000","last_unstale":"2023-09-23T20:35:26.620315+0000","last_undegraded":"2023-09-23T20:35:26.620315+0000","last_fullsized":"2023-09-23T20:35:26.620315+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:43.582146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:43.582146+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:25:04.582696+0000","scrub_duration":0.0016219730000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"86'80","reported_seq":627,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T20:35:26.635424+0000","last_change":"2023-09-23T20:29:45.536796+0000","last_active":"2023-09-23T20:35:26.635424+0000","last_peered":"2023-09-23T20:35:26.635424+0000","last_clean":"2023-09-23T20:35:26.635424+0000","last_became_active":"2023-09-23T20:29:45.536439+0000","last_became_peered":"2023-09-23T20:29:45.536439+0000","last_unstale":"2023-09-23T20:35:26.635424+0000","last_undegraded":"2023-09-23T20:35:26.635424+0000","last_fullsized":"2023-09-23T20:35:26.635424+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":80,"log_dups_size":0,"ondisk_log_size":80,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:30:57.793132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035115,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":395,"num_read_kb":1370,"num_write":66,"num_write_kb":1061,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":366,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T20:35:26.637778+0000","last_change":"2023-09-23T20:34:59.984286+0000","last_active":"2023-09-23T20:35:26.637778+0000","last_peered":"2023-09-23T20:35:26.637778+0000","last_clean":"2023-09-23T20:35:26.637778+0000","last_became_active":"2023-09-23T20:34:59.983950+0000","last_became_peered":"2023-09-23T20:34:59.983950+0000","last_unstale":"2023-09-23T20:35:26.637778+0000","last_undegraded":"2023-09-23T20:35:26.637778+0000","last_fullsized":"2023-09-23T20:35:26.637778+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:31:53.557035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":361,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T20:35:26.636691+0000","last_change":"2023-09-23T20:29:45.690815+0000","last_active":"2023-09-23T20:35:26.636691+0000","last_peered":"2023-09-23T20:35:26.636691+0000","last_clean":"2023-09-23T20:35:26.636691+0000","last_became_active":"2023-09-23T20:29:45.690478+0000","last_became_peered":"2023-09-23T20:29:45.690478+0000","last_unstale":"2023-09-23T20:35:26.636691+0000","last_undegraded":"2023-09-23T20:35:26.636691+0000","last_fullsized":"2023-09-23T20:35:26.636691+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:38.288193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.0","version":"364'11898","reported_seq":11967,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T20:35:48.908009+0000","last_change":"2023-09-23T20:31:49.157205+0000","last_active":"2023-09-23T20:35:48.908009+0000","last_peered":"2023-09-23T20:35:48.908009+0000","last_clean":"2023-09-23T20:35:48.908009+0000","last_became_active":"2023-09-23T20:31:49.156922+0000","last_became_peered":"2023-09-23T20:31:49.156922+0000","last_unstale":"2023-09-23T20:35:48.908009+0000","last_undegraded":"2023-09-23T20:35:48.908009+0000","last_fullsized":"2023-09-23T20:35:48.908009+0000","mapping_epoch":303,"log_start":"318'4187","ondisk_log_start":"318'4187","created":303,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":7711,"log_dups_size":0,"ondisk_log_size":7711,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:51:36.120059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11898,"num_write_kb":380736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":606,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T20:35:26.636450+0000","last_change":"2023-09-23T20:35:00.151823+0000","last_active":"2023-09-23T20:35:26.636450+0000","last_peered":"2023-09-23T20:35:26.636450+0000","last_clean":"2023-09-23T20:35:26.636450+0000","last_became_active":"2023-09-23T20:33:38.991030+0000","last_became_peered":"2023-09-23T20:33:38.991030+0000","last_unstale":"2023-09-23T20:35:26.636450+0000","last_undegraded":"2023-09-23T20:35:26.636450+0000","last_fullsized":"2023-09-23T20:35:26.636450+0000","mapping_epoch":253,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:35:00.151695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:35:00.151695+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:09:55.183927+0000","scrub_duration":0.0016030090000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"86'102","reported_seq":958,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T20:35:26.636281+0000","last_change":"2023-09-23T20:32:36.314672+0000","last_active":"2023-09-23T20:35:26.636281+0000","last_peered":"2023-09-23T20:35:26.636281+0000","last_clean":"2023-09-23T20:35:26.636281+0000","last_became_active":"2023-09-23T20:32:36.314231+0000","last_became_peered":"2023-09-23T20:32:36.314231+0000","last_unstale":"2023-09-23T20:35:26.636281+0000","last_undegraded":"2023-09-23T20:35:26.636281+0000","last_fullsized":"2023-09-23T20:35:26.636281+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":102,"log_dups_size":0,"ondisk_log_size":102,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:11:40.514709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030656500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":963,"num_read_kb":1234,"num_write":60,"num_write_kb":225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":288,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T20:35:26.621102+0000","last_change":"2023-09-23T20:29:45.538283+0000","last_active":"2023-09-23T20:35:26.621102+0000","last_peered":"2023-09-23T20:35:26.621102+0000","last_clean":"2023-09-23T20:35:26.621102+0000","last_became_active":"2023-09-23T20:29:45.537922+0000","last_became_peered":"2023-09-23T20:29:45.537922+0000","last_unstale":"2023-09-23T20:35:26.621102+0000","last_undegraded":"2023-09-23T20:35:26.621102+0000","last_fullsized":"2023-09-23T20:35:26.621102+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:30:52.397836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":298,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T20:35:26.637187+0000","last_change":"2023-09-23T20:20:14.888874+0000","last_active":"2023-09-23T20:35:26.637187+0000","last_peered":"2023-09-23T20:35:26.637187+0000","last_clean":"2023-09-23T20:35:26.637187+0000","last_became_active":"2023-09-23T20:20:14.887724+0000","last_became_peered":"2023-09-23T20:20:14.887724+0000","last_unstale":"2023-09-23T20:35:26.637187+0000","last_undegraded":"2023-09-23T20:35:26.637187+0000","last_fullsized":"2023-09-23T20:35:26.637187+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:09:24.510326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"364'11494","reported_seq":18672,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T20:35:48.889877+0000","last_change":"2023-09-23T20:34:50.587971+0000","last_active":"2023-09-23T20:35:48.889877+0000","last_peered":"2023-09-23T20:35:48.889877+0000","last_clean":"2023-09-23T20:35:48.889877+0000","last_became_active":"2023-09-23T20:34:50.555328+0000","last_became_peered":"2023-09-23T20:34:50.555328+0000","last_unstale":"2023-09-23T20:35:48.889877+0000","last_undegraded":"2023-09-23T20:35:48.889877+0000","last_fullsized":"2023-09-23T20:35:48.889877+0000","mapping_epoch":349,"log_start":"310'2400","ondisk_log_start":"310'2400","created":303,"last_epoch_clean":350,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":9094,"log_dups_size":0,"ondisk_log_size":9094,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:20:26.250987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11494,"num_write_kb":367808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3542,"num_bytes_recovered":170328064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":572,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T20:35:26.633852+0000","last_change":"2023-09-23T20:34:55.534723+0000","last_active":"2023-09-23T20:35:26.633852+0000","last_peered":"2023-09-23T20:35:26.633852+0000","last_clean":"2023-09-23T20:35:26.633852+0000","last_became_active":"2023-09-23T20:33:38.985300+0000","last_became_peered":"2023-09-23T20:33:38.985300+0000","last_unstale":"2023-09-23T20:35:26.633852+0000","last_undegraded":"2023-09-23T20:35:26.633852+0000","last_fullsized":"2023-09-23T20:35:26.633852+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:44.509012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:44.509012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0032186319999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":101,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T20:35:26.635699+0000","last_change":"2023-09-23T20:32:50.549695+0000","last_active":"2023-09-23T20:35:26.635699+0000","last_peered":"2023-09-23T20:35:26.635699+0000","last_clean":"2023-09-23T20:35:26.635699+0000","last_became_active":"2023-09-23T20:32:50.549431+0000","last_became_peered":"2023-09-23T20:32:50.549431+0000","last_unstale":"2023-09-23T20:35:26.635699+0000","last_undegraded":"2023-09-23T20:35:26.635699+0000","last_fullsized":"2023-09-23T20:35:26.635699+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:45:53.666569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"86'128","reported_seq":754,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T20:35:26.619549+0000","last_change":"2023-09-23T20:32:36.310656+0000","last_active":"2023-09-23T20:35:26.619549+0000","last_peered":"2023-09-23T20:35:26.619549+0000","last_clean":"2023-09-23T20:35:26.619549+0000","last_became_active":"2023-09-23T20:32:36.310155+0000","last_became_peered":"2023-09-23T20:32:36.310155+0000","last_unstale":"2023-09-23T20:35:26.619549+0000","last_undegraded":"2023-09-23T20:35:26.619549+0000","last_fullsized":"2023-09-23T20:35:26.619549+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:58:14.217466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000293202,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":579,"num_read_kb":1473,"num_write":96,"num_write_kb":1062,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":134,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T20:35:26.636026+0000","last_change":"2023-09-23T20:32:50.539051+0000","last_active":"2023-09-23T20:35:26.636026+0000","last_peered":"2023-09-23T20:35:26.636026+0000","last_clean":"2023-09-23T20:35:26.636026+0000","last_became_active":"2023-09-23T20:32:50.538755+0000","last_became_peered":"2023-09-23T20:32:50.538755+0000","last_unstale":"2023-09-23T20:35:26.636026+0000","last_undegraded":"2023-09-23T20:35:26.636026+0000","last_fullsized":"2023-09-23T20:35:26.636026+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:16.656613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.2","version":"364'11692","reported_seq":11770,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T20:35:48.968820+0000","last_change":"2023-09-23T20:32:50.557165+0000","last_active":"2023-09-23T20:35:48.968820+0000","last_peered":"2023-09-23T20:35:48.968820+0000","last_clean":"2023-09-23T20:35:48.968820+0000","last_became_active":"2023-09-23T20:32:50.553409+0000","last_became_peered":"2023-09-23T20:32:50.553409+0000","last_unstale":"2023-09-23T20:35:48.968820+0000","last_undegraded":"2023-09-23T20:35:48.968820+0000","last_fullsized":"2023-09-23T20:35:48.968820+0000","mapping_epoch":319,"log_start":"329'5493","ondisk_log_start":"329'5493","created":303,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:31:48.135169+0000","last_clean_scrub_stamp":"2023-09-23T20:31:48.135169+0000","objects_scrubbed":0,"log_size":6199,"log_dups_size":0,"ondisk_log_size":6199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:55:13.578195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11692,"num_write_kb":374144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"90'161","reported_seq":513,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T20:35:26.637361+0000","last_change":"2023-09-23T20:34:54.504652+0000","last_active":"2023-09-23T20:35:26.637361+0000","last_peered":"2023-09-23T20:35:26.637361+0000","last_clean":"2023-09-23T20:35:26.637361+0000","last_became_active":"2023-09-23T20:32:50.533470+0000","last_became_peered":"2023-09-23T20:32:50.533470+0000","last_unstale":"2023-09-23T20:35:26.637361+0000","last_undegraded":"2023-09-23T20:35:26.637361+0000","last_fullsized":"2023-09-23T20:35:26.637361+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:34:54.504554+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:34:54.504554+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:17:03.108888+0000","scrub_duration":0.013031021,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.7","version":"86'117","reported_seq":633,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T20:35:26.636589+0000","last_change":"2023-09-23T20:31:32.814392+0000","last_active":"2023-09-23T20:35:26.636589+0000","last_peered":"2023-09-23T20:35:26.636589+0000","last_clean":"2023-09-23T20:35:26.636589+0000","last_became_active":"2023-09-23T20:31:32.813857+0000","last_became_peered":"2023-09-23T20:31:32.813857+0000","last_unstale":"2023-09-23T20:35:26.636589+0000","last_undegraded":"2023-09-23T20:35:26.636589+0000","last_fullsized":"2023-09-23T20:35:26.636589+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:56:57.863005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046891400000000002,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":893,"num_read_kb":922,"num_write":82,"num_write_kb":4193,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":361,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T20:35:26.636723+0000","last_change":"2023-09-23T20:32:33.277827+0000","last_active":"2023-09-23T20:35:26.636723+0000","last_peered":"2023-09-23T20:35:26.636723+0000","last_clean":"2023-09-23T20:35:26.636723+0000","last_became_active":"2023-09-23T20:32:33.236087+0000","last_became_peered":"2023-09-23T20:32:33.236087+0000","last_unstale":"2023-09-23T20:35:26.636723+0000","last_undegraded":"2023-09-23T20:35:26.636723+0000","last_fullsized":"2023-09-23T20:35:26.636723+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:20:58.176854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":61,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T20:35:26.632576+0000","last_change":"2023-09-23T20:32:33.238290+0000","last_active":"2023-09-23T20:35:26.632576+0000","last_peered":"2023-09-23T20:35:26.632576+0000","last_clean":"2023-09-23T20:35:26.632576+0000","last_became_active":"2023-09-23T20:32:33.237984+0000","last_became_peered":"2023-09-23T20:32:33.237984+0000","last_unstale":"2023-09-23T20:35:26.632576+0000","last_undegraded":"2023-09-23T20:35:26.632576+0000","last_fullsized":"2023-09-23T20:35:26.632576+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:49:02.089307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"90'161","reported_seq":585,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T20:35:30.625326+0000","last_change":"2023-09-23T20:35:26.651952+0000","last_active":"2023-09-23T20:35:30.625326+0000","last_peered":"2023-09-23T20:35:30.625326+0000","last_clean":"2023-09-23T20:35:30.625326+0000","last_became_active":"2023-09-23T20:35:26.651601+0000","last_became_peered":"2023-09-23T20:35:26.651601+0000","last_unstale":"2023-09-23T20:35:30.625326+0000","last_undegraded":"2023-09-23T20:35:30.625326+0000","last_fullsized":"2023-09-23T20:35:30.625326+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:28:05.872074+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:28:05.872074+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":329,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T20:35:26.637120+0000","last_change":"2023-09-23T20:32:33.237234+0000","last_active":"2023-09-23T20:35:26.637120+0000","last_peered":"2023-09-23T20:35:26.637120+0000","last_clean":"2023-09-23T20:35:26.637120+0000","last_became_active":"2023-09-23T20:32:33.236881+0000","last_became_peered":"2023-09-23T20:32:33.236881+0000","last_unstale":"2023-09-23T20:35:26.637120+0000","last_undegraded":"2023-09-23T20:35:26.637120+0000","last_fullsized":"2023-09-23T20:35:26.637120+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:56:17.384831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":217,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T20:35:41.650188+0000","last_change":"2023-09-23T20:29:45.538314+0000","last_active":"2023-09-23T20:35:41.650188+0000","last_peered":"2023-09-23T20:35:41.650188+0000","last_clean":"2023-09-23T20:35:41.650188+0000","last_became_active":"2023-09-23T20:29:45.536847+0000","last_became_peered":"2023-09-23T20:29:45.536847+0000","last_unstale":"2023-09-23T20:35:41.650188+0000","last_undegraded":"2023-09-23T20:35:41.650188+0000","last_fullsized":"2023-09-23T20:35:41.650188+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:51.948721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"90'161","reported_seq":448,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T20:35:42.864501+0000","last_change":"2023-09-23T20:35:42.864462+0000","last_active":"2023-09-23T20:35:42.864501+0000","last_peered":"2023-09-23T20:35:42.864501+0000","last_clean":"2023-09-23T20:35:42.864501+0000","last_became_active":"2023-09-23T20:35:26.665630+0000","last_became_peered":"2023-09-23T20:35:26.665630+0000","last_unstale":"2023-09-23T20:35:42.864501+0000","last_undegraded":"2023-09-23T20:35:42.864501+0000","last_fullsized":"2023-09-23T20:35:42.864501+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:35:42.864363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:35:42.864363+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:14:34.984081+0000","scrub_duration":0.0070144939999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":373,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T20:35:26.637237+0000","last_change":"2023-09-23T20:32:33.235987+0000","last_active":"2023-09-23T20:35:26.637237+0000","last_peered":"2023-09-23T20:35:26.637237+0000","last_clean":"2023-09-23T20:35:26.637237+0000","last_became_active":"2023-09-23T20:32:33.226544+0000","last_became_peered":"2023-09-23T20:32:33.226544+0000","last_unstale":"2023-09-23T20:35:26.637237+0000","last_undegraded":"2023-09-23T20:35:26.637237+0000","last_fullsized":"2023-09-23T20:35:26.637237+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:53:22.794856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":20,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T20:35:26.634024+0000","last_change":"2023-09-23T20:34:59.987575+0000","last_active":"2023-09-23T20:35:26.634024+0000","last_peered":"2023-09-23T20:35:26.634024+0000","last_clean":"2023-09-23T20:35:26.634024+0000","last_became_active":"2023-09-23T20:34:59.987229+0000","last_became_peered":"2023-09-23T20:34:59.987229+0000","last_unstale":"2023-09-23T20:35:26.634024+0000","last_undegraded":"2023-09-23T20:35:26.634024+0000","last_fullsized":"2023-09-23T20:35:26.634024+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:34:01.820159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":381,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T20:35:26.637200+0000","last_change":"2023-09-23T20:32:33.268260+0000","last_active":"2023-09-23T20:35:26.637200+0000","last_peered":"2023-09-23T20:35:26.637200+0000","last_clean":"2023-09-23T20:35:26.637200+0000","last_became_active":"2023-09-23T20:32:33.233695+0000","last_became_peered":"2023-09-23T20:32:33.233695+0000","last_unstale":"2023-09-23T20:35:26.637200+0000","last_undegraded":"2023-09-23T20:35:26.637200+0000","last_fullsized":"2023-09-23T20:35:26.637200+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:19:29.299623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":20,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T20:35:26.632919+0000","last_change":"2023-09-23T20:34:59.987416+0000","last_active":"2023-09-23T20:35:26.632919+0000","last_peered":"2023-09-23T20:35:26.632919+0000","last_clean":"2023-09-23T20:35:26.632919+0000","last_became_active":"2023-09-23T20:34:59.987117+0000","last_became_peered":"2023-09-23T20:34:59.987117+0000","last_unstale":"2023-09-23T20:35:26.632919+0000","last_undegraded":"2023-09-23T20:35:26.632919+0000","last_fullsized":"2023-09-23T20:35:26.632919+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:36:16.318623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"90'161","reported_seq":549,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T20:35:54.459682+0000","last_change":"2023-09-23T20:35:54.459682+0000","last_active":"2023-09-23T20:35:54.459682+0000","last_peered":"2023-09-23T20:35:54.459682+0000","last_clean":"2023-09-23T20:35:54.459682+0000","last_became_active":"2023-09-23T20:35:26.643160+0000","last_became_peered":"2023-09-23T20:35:26.643160+0000","last_unstale":"2023-09-23T20:35:54.459682+0000","last_undegraded":"2023-09-23T20:35:54.459682+0000","last_fullsized":"2023-09-23T20:35:54.459682+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:41.701171+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:41.701171+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0019955509999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":54614,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":179,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T20:35:26.636369+0000","last_change":"2023-09-23T20:32:50.537447+0000","last_active":"2023-09-23T20:35:26.636369+0000","last_peered":"2023-09-23T20:35:26.636369+0000","last_clean":"2023-09-23T20:35:26.636369+0000","last_became_active":"2023-09-23T20:32:50.537136+0000","last_became_peered":"2023-09-23T20:32:50.537136+0000","last_unstale":"2023-09-23T20:35:26.636369+0000","last_undegraded":"2023-09-23T20:35:26.636369+0000","last_fullsized":"2023-09-23T20:35:26.636369+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:46:51.599492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":79,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T20:35:26.635224+0000","last_change":"2023-09-23T20:32:50.537450+0000","last_active":"2023-09-23T20:35:26.635224+0000","last_peered":"2023-09-23T20:35:26.635224+0000","last_clean":"2023-09-23T20:35:26.635224+0000","last_became_active":"2023-09-23T20:32:50.537128+0000","last_became_peered":"2023-09-23T20:32:50.537128+0000","last_unstale":"2023-09-23T20:35:26.635224+0000","last_undegraded":"2023-09-23T20:35:26.635224+0000","last_fullsized":"2023-09-23T20:35:26.635224+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:37:15.889029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1d","version":"90'161","reported_seq":442,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T20:35:26.649058+0000","last_change":"2023-09-23T20:35:26.645398+0000","last_active":"2023-09-23T20:35:26.649058+0000","last_peered":"2023-09-23T20:35:26.649058+0000","last_clean":"2023-09-23T20:35:26.649058+0000","last_became_active":"2023-09-23T20:35:26.644707+0000","last_became_peered":"2023-09-23T20:35:26.644707+0000","last_unstale":"2023-09-23T20:35:26.649058+0000","last_undegraded":"2023-09-23T20:35:26.649058+0000","last_fullsized":"2023-09-23T20:35:26.649058+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:46.388545+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:46.388545+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":40253,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":346,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T20:35:26.642319+0000","last_change":"2023-09-23T20:25:39.668882+0000","last_active":"2023-09-23T20:35:26.642319+0000","last_peered":"2023-09-23T20:35:26.642319+0000","last_clean":"2023-09-23T20:35:26.642319+0000","last_became_active":"2023-09-23T20:25:39.637128+0000","last_became_peered":"2023-09-23T20:25:39.637128+0000","last_unstale":"2023-09-23T20:35:26.642319+0000","last_undegraded":"2023-09-23T20:35:26.642319+0000","last_fullsized":"2023-09-23T20:35:26.642319+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:56.406298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":61,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T20:35:26.633014+0000","last_change":"2023-09-23T20:32:33.232281+0000","last_active":"2023-09-23T20:35:26.633014+0000","last_peered":"2023-09-23T20:35:26.633014+0000","last_clean":"2023-09-23T20:35:26.633014+0000","last_became_active":"2023-09-23T20:32:33.232003+0000","last_became_peered":"2023-09-23T20:32:33.232003+0000","last_unstale":"2023-09-23T20:35:26.633014+0000","last_undegraded":"2023-09-23T20:35:26.633014+0000","last_fullsized":"2023-09-23T20:35:26.633014+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:58:27.314355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":189100,"num_write_kb":6051137,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":13549,"num_bytes_recovered":652214272,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":349437952,"data_stored":349437952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":114543,"ondisk_log_size":114543,"up":48,"acting":47,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":49,"num_bytes_recovered":6522,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26691,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":479256,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":89,"num_read_kb":168,"num_write":45,"num_write_kb":440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":584707,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":58,"num_bytes_recovered":4194342,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6082,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201577,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":316708,"kb_used_data":8156,"kb_used_omap":0,"kb_used_meta":308544,"kb_avail":93428444,"statfs":{"total":95995035648,"available":95670726656,"internally_reserved":0,"allocated":8351744,"data_stored":7190630,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":315949056},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:35:44 2023","interfaces":[{"interface":"back","average":{"1min":3.5600000000000001,"5min":2.5299999999999998,"15min":1.8160000000000001},"min":{"1min":0.28100000000000003,"5min":0.23599999999999999,"15min":0.20799999999999999},"max":{"1min":17.399999999999999,"5min":17.399999999999999,"15min":48.798999999999999},"last":0.34300000000000003},{"interface":"front","average":{"1min":3.1890000000000001,"5min":2.21,"15min":1.877},"min":{"1min":0.249,"5min":0.249,"15min":0.24399999999999999},"max":{"1min":19.355,"5min":19.355,"15min":48.247},"last":0.27300000000000002}]},{"osd":1,"last update":"Sat Sep 23 20:34:54 2023","interfaces":[{"interface":"back","average":{"1min":2.3319999999999999,"5min":1.984,"15min":1.8360000000000001},"min":{"1min":0.29999999999999999,"5min":0.254,"15min":0.254},"max":{"1min":5.5030000000000001,"5min":14.026,"15min":14.026},"last":0.39500000000000002},{"interface":"front","average":{"1min":2.5139999999999998,"5min":1.8340000000000001,"15min":1.9319999999999999},"min":{"1min":0.32000000000000001,"5min":0.248,"15min":0.248},"max":{"1min":6.7220000000000004,"5min":6.7220000000000004,"15min":6.7220000000000004},"last":0.35999999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:35:44 2023","interfaces":[{"interface":"back","average":{"1min":3.3079999999999998,"5min":2.492,"15min":1.8620000000000001},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27600000000000002},"max":{"1min":9.8629999999999995,"5min":13.173999999999999,"15min":13.173999999999999},"last":0.45000000000000001},{"interface":"front","average":{"1min":3.7130000000000001,"5min":2.8799999999999999,"15min":2.113},"min":{"1min":0.36099999999999999,"5min":0.27400000000000002,"15min":0.26600000000000001},"max":{"1min":12.551,"5min":23.494,"15min":23.494},"last":0.33100000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:35:11 2023","interfaces":[{"interface":"back","average":{"1min":4.8280000000000003,"5min":2.8500000000000001,"15min":1.9850000000000001},"min":{"1min":0.375,"5min":0.33700000000000002,"15min":0.26200000000000001},"max":{"1min":14.789,"5min":17.155999999999999,"15min":17.155999999999999},"last":0.42799999999999999},{"interface":"front","average":{"1min":4.819,"5min":2.952,"15min":1.9179999999999999},"min":{"1min":0.46200000000000002,"5min":0.34799999999999998,"15min":0.27600000000000002},"max":{"1min":14.805,"5min":16.960000000000001,"15min":16.960000000000001},"last":0.47799999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:35:11 2023","interfaces":[{"interface":"back","average":{"1min":1.248,"5min":0.81699999999999995,"15min":1.101},"min":{"1min":0.254,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":5.0460000000000003,"5min":5.8819999999999997,"15min":15.177},"last":0.313},{"interface":"front","average":{"1min":1.389,"5min":0.91700000000000004,"15min":1.2350000000000001},"min":{"1min":0.32400000000000001,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":6.1779999999999999,"5min":6.1779999999999999,"15min":11.122},"last":0.41099999999999998}]},{"osd":5,"last update":"Sat Sep 23 20:35:11 2023","interfaces":[{"interface":"back","average":{"1min":0.71299999999999997,"5min":0.70299999999999996,"15min":1.145},"min":{"1min":0.27600000000000002,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":4.133,"5min":4.133,"15min":13.241},"last":0.377},{"interface":"front","average":{"1min":0.90200000000000002,"5min":0.66100000000000003,"15min":1.2170000000000001},"min":{"1min":0.32400000000000001,"5min":0.32200000000000001,"15min":0.315},"max":{"1min":4.7590000000000003,"5min":5.7869999999999999,"15min":18.831},"last":0.46800000000000003}]},{"osd":6,"last update":"Sat Sep 23 20:35:11 2023","interfaces":[{"interface":"back","average":{"1min":0.85299999999999998,"5min":0.67800000000000005,"15min":1.1160000000000001},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":4.819,"5min":5.7629999999999999,"15min":13.185},"last":0.439},{"interface":"front","average":{"1min":1.2649999999999999,"5min":0.94299999999999995,"15min":1.137},"min":{"1min":0.35299999999999998,"5min":0.318,"15min":0.29899999999999999},"max":{"1min":6.2770000000000001,"5min":6.2770000000000001,"15min":9.9120000000000008},"last":0.50600000000000001}]}]},{"osd":6,"up_from":47,"seq":201863463213,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":502616,"kb_used_data":225040,"kb_used_omap":0,"kb_used_meta":277568,"kb_avail":93242536,"statfs":{"total":95995035648,"available":95480356864,"internally_reserved":0,"allocated":230440960,"data_stored":229291463,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":284229632},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:35:41 2023","interfaces":[{"interface":"back","average":{"1min":1.754,"5min":2.589,"15min":1.667},"min":{"1min":0.26200000000000001,"5min":0.22800000000000001,"15min":0.221},"max":{"1min":4.8959999999999999,"5min":53.701999999999998,"15min":53.701999999999998},"last":0.29399999999999998},{"interface":"front","average":{"1min":1.7290000000000001,"5min":2.9180000000000001,"15min":2.016},"min":{"1min":0.252,"5min":0.252,"15min":0.25},"max":{"1min":6.093,"5min":53.875999999999998,"15min":53.875999999999998},"last":0.56899999999999995}]},{"osd":1,"last update":"Sat Sep 23 20:34:57 2023","interfaces":[{"interface":"back","average":{"1min":3.4510000000000001,"5min":2.2719999999999998,"15min":2.5910000000000002},"min":{"1min":0.25600000000000001,"5min":0.249,"15min":0.249},"max":{"1min":34.173999999999999,"5min":34.173999999999999,"15min":34.173999999999999},"last":0.40999999999999998},{"interface":"front","average":{"1min":4.29,"5min":2.569,"15min":3.181},"min":{"1min":0.32900000000000001,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":34.490000000000002,"5min":34.490000000000002,"15min":34.490000000000002},"last":0.35299999999999998}]},{"osd":2,"last update":"Sat Sep 23 20:35:41 2023","interfaces":[{"interface":"back","average":{"1min":2.6259999999999999,"5min":2.512,"15min":1.9430000000000001},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":13.429,"5min":34.317999999999998,"15min":34.317999999999998},"last":0.46600000000000003},{"interface":"front","average":{"1min":2.4649999999999999,"5min":2.5289999999999999,"15min":1.859},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":13.457000000000001,"5min":34.292999999999999,"15min":34.292999999999999},"last":0.5}]},{"osd":3,"last update":"Sat Sep 23 20:35:25 2023","interfaces":[{"interface":"back","average":{"1min":2.0099999999999998,"5min":2.0369999999999999,"15min":1.7050000000000001},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29599999999999999},"max":{"1min":6.46,"5min":7.5620000000000003,"15min":9.4239999999999995},"last":0.45200000000000001},{"interface":"front","average":{"1min":2.8290000000000002,"5min":2.4140000000000001,"15min":1.851},"min":{"1min":0.39400000000000002,"5min":0.317,"15min":0.317},"max":{"1min":8.5600000000000005,"5min":12.875999999999999,"15min":12.875999999999999},"last":0.53500000000000003}]},{"osd":4,"last update":"Sat Sep 23 20:35:02 2023","interfaces":[{"interface":"back","average":{"1min":0.89200000000000002,"5min":0.58599999999999997,"15min":0.89500000000000002},"min":{"1min":0.27400000000000002,"5min":0.251,"15min":0.251},"max":{"1min":4.9279999999999999,"5min":4.9279999999999999,"15min":10.173},"last":0.441},{"interface":"front","average":{"1min":0.77200000000000002,"5min":0.84699999999999998,"15min":1.0620000000000001},"min":{"1min":0.29099999999999998,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":3.1139999999999999,"5min":7.7629999999999999,"15min":8.3249999999999993},"last":0.48499999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:35:02 2023","interfaces":[{"interface":"back","average":{"1min":0.70599999999999996,"5min":0.66400000000000003,"15min":0.95199999999999996},"min":{"1min":0.31,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":4.8259999999999996,"5min":6.8310000000000004,"15min":14.085000000000001},"last":0.47499999999999998},{"interface":"front","average":{"1min":0.69199999999999995,"5min":0.84199999999999997,"15min":1.3100000000000001},"min":{"1min":0.32000000000000001,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":3.0920000000000001,"5min":7.7300000000000004,"15min":18.192},"last":0.51600000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:35:25 2023","interfaces":[{"interface":"back","average":{"1min":0.82599999999999996,"5min":0.69799999999999995,"15min":1.085},"min":{"1min":0.36599999999999999,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":4.851,"5min":5.0350000000000001,"15min":10.217000000000001},"last":0.39100000000000001},{"interface":"front","average":{"1min":0.93500000000000005,"5min":0.93999999999999995,"15min":1.2509999999999999},"min":{"1min":0.35999999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":5.6820000000000004,"5min":7.601,"15min":9.7360000000000007},"last":0.55200000000000005}]}]},{"osd":1,"up_from":266,"seq":1142461300811,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":278180,"kb_used_data":4700,"kb_used_omap":8,"kb_used_meta":273463,"kb_avail":93466972,"statfs":{"total":95995035648,"available":95710179328,"internally_reserved":0,"allocated":4812800,"data_stored":2837872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8206,"internal_metadata":280027122},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:34:54 2023","interfaces":[{"interface":"back","average":{"1min":1.9470000000000001,"5min":2.9750000000000001,"15min":2.5449999999999999},"min":{"1min":0.221,"5min":0.218,"15min":0.218},"max":{"1min":8.6080000000000005,"5min":52.689999999999998,"15min":52.689999999999998},"last":0.33900000000000002},{"interface":"front","average":{"1min":5.3540000000000001,"5min":3.4809999999999999,"15min":3.2909999999999999},"min":{"1min":0.20599999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":26.588000000000001,"5min":52.662999999999997,"15min":52.662999999999997},"last":0.25}]},{"osd":2,"last update":"Sat Sep 23 20:34:54 2023","interfaces":[{"interface":"back","average":{"1min":7.2110000000000003,"5min":4.069,"15min":4.7729999999999997},"min":{"1min":0.312,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":28.359000000000002,"5min":28.359000000000002,"15min":28.359000000000002},"last":0.41099999999999998},{"interface":"front","average":{"1min":3.4550000000000001,"5min":2.6179999999999999,"15min":2.2679999999999998},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":12.801,"5min":12.801,"15min":12.801},"last":0.26700000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:34:54 2023","interfaces":[{"interface":"back","average":{"1min":4.1239999999999997,"5min":4.3799999999999999,"15min":4.0380000000000003},"min":{"1min":0.31,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":15.247999999999999,"5min":80.745999999999995,"15min":80.745999999999995},"last":0.308},{"interface":"front","average":{"1min":5.3380000000000001,"5min":4.6689999999999996,"15min":4.7869999999999999},"min":{"1min":0.35799999999999998,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":17.893999999999998,"5min":82.988,"15min":82.988},"last":0.372}]},{"osd":4,"last update":"Sat Sep 23 20:34:54 2023","interfaces":[{"interface":"back","average":{"1min":4.1449999999999996,"5min":2.2989999999999999,"15min":2.1520000000000001},"min":{"1min":0.34200000000000003,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":24.327999999999999,"5min":24.327999999999999,"15min":24.327999999999999},"last":0.35599999999999998},{"interface":"front","average":{"1min":1.2889999999999999,"5min":1.6759999999999999,"15min":1.9079999999999999},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":4.7889999999999997,"5min":10.760999999999999,"15min":10.760999999999999},"last":0.38800000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:34:54 2023","interfaces":[{"interface":"back","average":{"1min":3.3170000000000002,"5min":2.1760000000000002,"15min":2.593},"min":{"1min":0.45000000000000001,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":7.6909999999999998,"5min":9.6479999999999997,"15min":9.6479999999999997},"last":0.40300000000000002},{"interface":"front","average":{"1min":4.1059999999999999,"5min":2.4809999999999999,"15min":2.2200000000000002},"min":{"1min":0.38400000000000001,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":24.404,"5min":24.404,"15min":24.404},"last":0.44800000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:34:54 2023","interfaces":[{"interface":"back","average":{"1min":3.3250000000000002,"5min":3.1059999999999999,"15min":6.0010000000000003},"min":{"1min":0.51200000000000001,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":7.7199999999999998,"5min":72.682000000000002,"15min":72.682000000000002},"last":0.505},{"interface":"front","average":{"1min":4.2489999999999997,"5min":3.4180000000000001,"15min":5.4320000000000004},"min":{"1min":0.41099999999999998,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":24.376000000000001,"5min":72.563999999999993,"15min":72.563999999999993},"last":0.46899999999999997}]},{"osd":7,"last update":"Sat Sep 23 20:34:54 2023","interfaces":[{"interface":"back","average":{"1min":3.4489999999999998,"5min":2.3759999999999999,"15min":2.98},"min":{"1min":0.54400000000000004,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":9.0830000000000002,"5min":9.6850000000000005,"15min":9.6850000000000005},"last":0.52700000000000002},{"interface":"front","average":{"1min":4.2519999999999998,"5min":2.6230000000000002,"15min":2.4470000000000001},"min":{"1min":0.441,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":24.524000000000001,"5min":24.524000000000001,"15min":24.524000000000001},"last":0.51500000000000001}]}]},{"osd":0,"up_from":94,"seq":403726926014,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":485188,"kb_used_data":4860,"kb_used_omap":7,"kb_used_meta":480312,"kb_avail":93259964,"statfs":{"total":95995035648,"available":95498203136,"internally_reserved":0,"allocated":4976640,"data_stored":3148821,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":491839505},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:35:02 2023","interfaces":[{"interface":"back","average":{"1min":3.8130000000000002,"5min":3.5190000000000001,"15min":3.7639999999999998},"min":{"1min":0.183,"5min":0.17999999999999999,"15min":0.17999999999999999},"max":{"1min":12.301,"5min":42.000999999999998,"15min":42.000999999999998},"last":0.23000000000000001},{"interface":"front","average":{"1min":4.5780000000000003,"5min":3.585,"15min":4.1660000000000004},"min":{"1min":0.247,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":18.355,"5min":23.036000000000001,"15min":23.036000000000001},"last":0.29499999999999998}]},{"osd":2,"last update":"Sat Sep 23 20:35:37 2023","interfaces":[{"interface":"back","average":{"1min":3.5840000000000001,"5min":2.9820000000000002,"15min":2.4569999999999999},"min":{"1min":0.24099999999999999,"5min":0.193,"15min":0.185},"max":{"1min":11.064,"5min":22.460999999999999,"15min":77.448999999999998},"last":0.25700000000000001},{"interface":"front","average":{"1min":3.4489999999999998,"5min":2.8570000000000002,"15min":2.294},"min":{"1min":0.25700000000000001,"5min":0.188,"15min":0.188},"max":{"1min":12.071,"5min":22.260000000000002,"15min":81.724999999999994},"last":0.33000000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:35:07 2023","interfaces":[{"interface":"back","average":{"1min":5.1449999999999996,"5min":4.0899999999999999,"15min":2.5219999999999998},"min":{"1min":0.27500000000000002,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":17.692,"5min":81.400999999999996,"15min":81.400999999999996},"last":0.318},{"interface":"front","average":{"1min":5.085,"5min":4.7759999999999998,"15min":2.6200000000000001},"min":{"1min":0.30299999999999999,"5min":0.25700000000000001,"15min":0.23699999999999999},"max":{"1min":12.262,"5min":81.137,"15min":81.137},"last":0.34399999999999997}]},{"osd":4,"last update":"Sat Sep 23 20:35:37 2023","interfaces":[{"interface":"back","average":{"1min":3.331,"5min":2.1749999999999998,"15min":1.9350000000000001},"min":{"1min":0.32000000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":11.068,"5min":12.587999999999999,"15min":23.876999999999999},"last":0.374},{"interface":"front","average":{"1min":2.7650000000000001,"5min":2.347,"15min":1.8160000000000001},"min":{"1min":0.316,"5min":0.308,"15min":0.29299999999999998},"max":{"1min":11.458,"5min":23.001000000000001,"15min":23.681999999999999},"last":0.53500000000000003}]},{"osd":5,"last update":"Sat Sep 23 20:35:37 2023","interfaces":[{"interface":"back","average":{"1min":2.5960000000000001,"5min":2.2360000000000002,"15min":2.129},"min":{"1min":0.35099999999999998,"5min":0.34399999999999997,"15min":0.34200000000000003},"max":{"1min":11.497999999999999,"5min":22.969999999999999,"15min":24.198},"last":0.57799999999999996},{"interface":"front","average":{"1min":2.8170000000000002,"5min":2.3199999999999998,"15min":1.9970000000000001},"min":{"1min":0.38300000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":11.974,"5min":22.236000000000001,"15min":22.236000000000001},"last":0.52600000000000002}]},{"osd":6,"last update":"Sat Sep 23 20:35:37 2023","interfaces":[{"interface":"back","average":{"1min":3.1640000000000001,"5min":2.4590000000000001,"15min":1.923},"min":{"1min":0.38,"5min":0.32300000000000001,"15min":0.318},"max":{"1min":11.526999999999999,"5min":22.934999999999999,"15min":24.408000000000001},"last":0.60799999999999998},{"interface":"front","average":{"1min":2.8010000000000002,"5min":2.399,"15min":1.841},"min":{"1min":0.35799999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":12.015000000000001,"5min":22.164999999999999,"15min":25.271000000000001},"last":0.36599999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:35:37 2023","interfaces":[{"interface":"back","average":{"1min":3.1899999999999999,"5min":2.5630000000000002,"15min":2.0459999999999998},"min":{"1min":0.40200000000000002,"5min":0.32200000000000001,"15min":0.30599999999999999},"max":{"1min":12.042,"5min":22.206,"15min":25.359000000000002},"last":0.504},{"interface":"front","average":{"1min":3.9740000000000002,"5min":2.6230000000000002,"15min":2.3319999999999999},"min":{"1min":0.36199999999999999,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":11.112,"5min":22.422999999999998,"15min":24.669},"last":0.51200000000000001}]}]},{"osd":2,"up_from":104,"seq":446676598961,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":493884,"kb_used_data":8884,"kb_used_omap":11,"kb_used_meta":484980,"kb_avail":93251268,"statfs":{"total":95995035648,"available":95489298432,"internally_reserved":0,"allocated":9097216,"data_stored":7274245,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11878,"internal_metadata":496619930},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,0,0,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:35:47 2023","interfaces":[{"interface":"back","average":{"1min":3.944,"5min":4.274,"15min":2.5819999999999999},"min":{"1min":0.25800000000000001,"5min":0.185,"15min":0.17999999999999999},"max":{"1min":14.124000000000001,"5min":82.424999999999997,"15min":82.424999999999997},"last":2.2690000000000001},{"interface":"front","average":{"1min":4.4859999999999998,"5min":3.7890000000000001,"15min":2.3460000000000001},"min":{"1min":0.24199999999999999,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":14.055,"5min":82.524000000000001,"15min":82.524000000000001},"last":6.327}]},{"osd":1,"last update":"Sat Sep 23 20:34:52 2023","interfaces":[{"interface":"back","average":{"1min":2.7080000000000002,"5min":3.0699999999999998,"15min":4.2569999999999997},"min":{"1min":0.27100000000000002,"5min":0.221,"15min":0.221},"max":{"1min":10.205,"5min":16.280000000000001,"15min":16.280000000000001},"last":7.335},{"interface":"front","average":{"1min":4.1100000000000003,"5min":2.8029999999999999,"15min":2.9260000000000002},"min":{"1min":0.36699999999999999,"5min":0.248,"15min":0.248},"max":{"1min":13.130000000000001,"5min":30.565000000000001,"15min":30.565000000000001},"last":3.0019999999999998}]},{"osd":3,"last update":"Sat Sep 23 20:35:13 2023","interfaces":[{"interface":"back","average":{"1min":2.6520000000000001,"5min":2.9529999999999998,"15min":1.905},"min":{"1min":0.32700000000000001,"5min":0.27800000000000002,"15min":0.19800000000000001},"max":{"1min":12.664,"5min":15.804,"15min":16.331},"last":7.3739999999999997},{"interface":"front","average":{"1min":4.577,"5min":3.5139999999999998,"15min":2.242},"min":{"1min":0.33400000000000002,"5min":0.27300000000000002,"15min":0.23899999999999999},"max":{"1min":13.157,"5min":30.704999999999998,"15min":30.704999999999998},"last":13.148999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:35:47 2023","interfaces":[{"interface":"back","average":{"1min":3.3730000000000002,"5min":2.1440000000000001,"15min":1.754},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":9.2430000000000003,"5min":9.2430000000000003,"15min":16.797999999999998},"last":6.4320000000000004},{"interface":"front","average":{"1min":3.1819999999999999,"5min":2.5110000000000001,"15min":1.903},"min":{"1min":0.307,"5min":0.30499999999999999,"15min":0.27600000000000002},"max":{"1min":11.044,"5min":15.853999999999999,"15min":15.853999999999999},"last":7.4009999999999998}]},{"osd":5,"last update":"Sat Sep 23 20:35:47 2023","interfaces":[{"interface":"back","average":{"1min":3.3690000000000002,"5min":2.2069999999999999,"15min":2.2869999999999999},"min":{"1min":0.36599999999999999,"5min":0.314,"15min":0.309},"max":{"1min":9.1799999999999997,"5min":9.1799999999999997,"15min":95.393000000000001},"last":6.46},{"interface":"front","average":{"1min":3.1920000000000002,"5min":2.77,"15min":2.5049999999999999},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.32300000000000001},"max":{"1min":10.856999999999999,"5min":15.885,"15min":95.414000000000001},"last":7.4359999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:35:47 2023","interfaces":[{"interface":"back","average":{"1min":3.0680000000000001,"5min":2.6560000000000001,"15min":1.9890000000000001},"min":{"1min":0.37,"5min":0.35799999999999998,"15min":0.32100000000000001},"max":{"1min":11.253,"5min":15.914,"15min":15.914},"last":7.4649999999999999},{"interface":"front","average":{"1min":2.968,"5min":2.3570000000000002,"15min":1.9670000000000001},"min":{"1min":0.49099999999999999,"5min":0.318,"15min":0.316},"max":{"1min":10.811,"5min":11.523,"15min":30.670999999999999},"last":2.1960000000000002}]},{"osd":7,"last update":"Sat Sep 23 20:35:47 2023","interfaces":[{"interface":"back","average":{"1min":2.7730000000000001,"5min":2.306,"15min":2.0259999999999998},"min":{"1min":0.439,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":10.231,"5min":12.734999999999999,"15min":30.741},"last":2.2400000000000002},{"interface":"front","average":{"1min":3.2250000000000001,"5min":2.3170000000000002,"15min":2.0089999999999999},"min":{"1min":0.42999999999999999,"5min":0.38,"15min":0.38},"max":{"1min":9.2720000000000002,"5min":10.861000000000001,"15min":16.719000000000001},"last":6.5449999999999999}]}]},{"osd":3,"up_from":109,"seq":468151435435,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":582520,"kb_used_data":30384,"kb_used_omap":4,"kb_used_meta":552123,"kb_avail":93162632,"statfs":{"total":95995035648,"available":95398535168,"internally_reserved":0,"allocated":31113216,"data_stored":29292352,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4514,"internal_metadata":565374558},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:35:11 2023","interfaces":[{"interface":"back","average":{"1min":3.742,"5min":3.242,"15min":2.2770000000000001},"min":{"1min":0.215,"5min":0.191,"15min":0.186},"max":{"1min":17.007000000000001,"5min":17.129000000000001,"15min":67.305999999999997},"last":0.26500000000000001},{"interface":"front","average":{"1min":3.3140000000000001,"5min":3.0550000000000002,"15min":2.1190000000000002},"min":{"1min":0.23699999999999999,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":9.5380000000000003,"5min":12.781000000000001,"15min":67.403999999999996},"last":0.28399999999999997}]},{"osd":1,"last update":"Sat Sep 23 20:34:55 2023","interfaces":[{"interface":"back","average":{"1min":2.7589999999999999,"5min":2.7189999999999999,"15min":2.9649999999999999},"min":{"1min":0.28799999999999998,"5min":0.222,"15min":0.222},"max":{"1min":14.641,"5min":16.189,"15min":16.189},"last":0.30499999999999999},{"interface":"front","average":{"1min":2.1139999999999999,"5min":2.6299999999999999,"15min":3.0449999999999999},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":10.811,"5min":32.603999999999999,"15min":32.603999999999999},"last":0.34599999999999997}]},{"osd":2,"last update":"Sat Sep 23 20:35:11 2023","interfaces":[{"interface":"back","average":{"1min":2.883,"5min":3.4980000000000002,"15min":2.2589999999999999},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.22800000000000001},"max":{"1min":9.5879999999999992,"5min":32.857999999999997,"15min":32.857999999999997},"last":0.32000000000000001},{"interface":"front","average":{"1min":3.4369999999999998,"5min":3.1699999999999999,"15min":2.165},"min":{"1min":0.251,"5min":0.246,"15min":0.20899999999999999},"max":{"1min":14.492000000000001,"5min":24.146999999999998,"15min":24.146999999999998},"last":0.40999999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:35:11 2023","interfaces":[{"interface":"back","average":{"1min":1.5669999999999999,"5min":1.9750000000000001,"15min":1.698},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.29799999999999999},"max":{"1min":5.3929999999999998,"5min":12.151,"15min":46.042999999999999},"last":0.51600000000000001},{"interface":"front","average":{"1min":2.6949999999999998,"5min":2.6339999999999999,"15min":2.0579999999999998},"min":{"1min":0.308,"5min":0.29399999999999998,"15min":0.28100000000000003},"max":{"1min":14.422000000000001,"5min":19.635000000000002,"15min":46.579999999999998},"last":0.64000000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:35:11 2023","interfaces":[{"interface":"back","average":{"1min":2.056,"5min":2.2120000000000002,"15min":1.9399999999999999},"min":{"1min":0.36699999999999999,"5min":0.32200000000000001,"15min":0.311},"max":{"1min":13.228999999999999,"5min":13.228999999999999,"15min":45.292000000000002},"last":0.55500000000000005},{"interface":"front","average":{"1min":1.647,"5min":2.1190000000000002,"15min":1.764},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.313},"max":{"1min":8.782,"5min":11.028,"15min":44.679000000000002},"last":0.73399999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:35:11 2023","interfaces":[{"interface":"back","average":{"1min":1.4590000000000001,"5min":2.266,"15min":1.79},"min":{"1min":0.40500000000000003,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":5.2640000000000002,"5min":17.524999999999999,"15min":17.524999999999999},"last":0.76000000000000001},{"interface":"front","average":{"1min":3.1150000000000002,"5min":2.4180000000000001,"15min":1.8100000000000001},"min":{"1min":0.41299999999999998,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":14.464,"5min":19.524000000000001,"15min":19.524000000000001},"last":0.61799999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:35:11 2023","interfaces":[{"interface":"back","average":{"1min":2.9740000000000002,"5min":2.6230000000000002,"15min":1.964},"min":{"1min":0.44700000000000001,"5min":0.36899999999999999,"15min":0.34999999999999998},"max":{"1min":14.521000000000001,"5min":19.571999999999999,"15min":19.571999999999999},"last":0.66300000000000003},{"interface":"front","average":{"1min":2.016,"5min":2.149,"15min":1.998},"min":{"1min":0.44600000000000001,"5min":0.42599999999999999,"15min":0.371},"max":{"1min":8.3360000000000003,"5min":12.186,"15min":45.387999999999998},"last":0.58699999999999997}]}]},{"osd":4,"up_from":30,"seq":128849019189,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":369240,"kb_used_data":102672,"kb_used_omap":0,"kb_used_meta":266560,"kb_avail":93375912,"statfs":{"total":95995035648,"available":95616933888,"internally_reserved":0,"allocated":105136128,"data_stored":103986365,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":272957440},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:35:40 2023","interfaces":[{"interface":"back","average":{"1min":2.2799999999999998,"5min":1.6579999999999999,"15min":1.5229999999999999},"min":{"1min":0.27300000000000002,"5min":0.255,"15min":0.222},"max":{"1min":5.2359999999999998,"5min":8.0800000000000001,"15min":20.411999999999999},"last":0.52000000000000002},{"interface":"front","average":{"1min":2.3849999999999998,"5min":1.835,"15min":1.6739999999999999},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.253},"max":{"1min":5.2229999999999999,"5min":10.675000000000001,"15min":20.454999999999998},"last":0.54200000000000004}]},{"osd":1,"last update":"Sat Sep 23 20:34:56 2023","interfaces":[{"interface":"back","average":{"1min":1.4350000000000001,"5min":1.885,"15min":1.9670000000000001},"min":{"1min":0.27600000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":6.0869999999999997,"5min":11.303000000000001,"15min":11.303000000000001},"last":0.47399999999999998},{"interface":"front","average":{"1min":1.2110000000000001,"5min":1.6299999999999999,"15min":1.8859999999999999},"min":{"1min":0.30499999999999999,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":3.9169999999999998,"5min":11.287000000000001,"15min":11.287000000000001},"last":0.48699999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:35:53 2023","interfaces":[{"interface":"back","average":{"1min":2.702,"5min":1.9570000000000001,"15min":1.798},"min":{"1min":0.32300000000000001,"5min":0.311,"15min":0.26600000000000001},"max":{"1min":10.416,"5min":10.416,"15min":32.363},"last":0.42999999999999999},{"interface":"front","average":{"1min":2.9470000000000001,"5min":2.21,"15min":2.0950000000000002},"min":{"1min":0.41999999999999998,"5min":0.28999999999999998,"15min":0.26800000000000002},"max":{"1min":8.4939999999999998,"5min":14.614000000000001,"15min":34.042999999999999},"last":0.504}]},{"osd":3,"last update":"Sat Sep 23 20:35:09 2023","interfaces":[{"interface":"back","average":{"1min":2.085,"5min":2.2839999999999998,"15min":1.849},"min":{"1min":0.34899999999999998,"5min":0.34799999999999998,"15min":0.29099999999999998},"max":{"1min":7.2060000000000004,"5min":14.515000000000001,"15min":32.332999999999998},"last":0.57099999999999995},{"interface":"front","average":{"1min":2.5150000000000001,"5min":2.2599999999999998,"15min":1.9379999999999999},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":7.9210000000000003,"5min":15.991,"15min":32.399999999999999},"last":0.45400000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:35:32 2023","interfaces":[{"interface":"back","average":{"1min":0.67800000000000005,"5min":0.72199999999999998,"15min":1.5409999999999999},"min":{"1min":0.26800000000000002,"5min":0.25800000000000001,"15min":0.247},"max":{"1min":2.9369999999999998,"5min":5.4340000000000002,"15min":82.866},"last":0.34599999999999997},{"interface":"front","average":{"1min":0.61799999999999999,"5min":0.79400000000000004,"15min":1.4370000000000001},"min":{"1min":0.33700000000000002,"5min":0.30299999999999999,"15min":0.27800000000000002},"max":{"1min":2.7509999999999999,"5min":6.8520000000000003,"15min":82.546000000000006},"last":0.32400000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:35:53 2023","interfaces":[{"interface":"back","average":{"1min":1.018,"5min":1.028,"15min":1.389},"min":{"1min":0.312,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":3.008,"5min":10.178000000000001,"15min":41.371000000000002},"last":0.439},{"interface":"front","average":{"1min":1.097,"5min":0.85099999999999998,"15min":1.27},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.27200000000000002},"max":{"1min":4.2949999999999999,"5min":5.4619999999999997,"15min":41.140999999999998},"last":0.42099999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:35:09 2023","interfaces":[{"interface":"back","average":{"1min":0.86499999999999999,"5min":1.0880000000000001,"15min":1.425},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.33600000000000002},"max":{"1min":3.2970000000000002,"5min":6.8849999999999998,"15min":16.427},"last":0.377},{"interface":"front","average":{"1min":0.65200000000000002,"5min":0.81599999999999995,"15min":1.2989999999999999},"min":{"1min":0.36399999999999999,"5min":0.33000000000000002,"15min":0.307},"max":{"1min":1.4490000000000001,"5min":6.3970000000000002,"15min":19.018999999999998},"last":0.38700000000000001}]}]},{"osd":5,"up_from":39,"seq":167503724850,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":262036,"kb_used_data":4044,"kb_used_omap":0,"kb_used_meta":257984,"kb_avail":93483116,"statfs":{"total":95995035648,"available":95726710784,"internally_reserved":0,"allocated":4141056,"data_stored":2992019,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":264175616},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:35:42 2023","interfaces":[{"interface":"back","average":{"1min":4.7649999999999997,"5min":2.125,"15min":1.6919999999999999},"min":{"1min":0.29599999999999999,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":18.16,"5min":18.16,"15min":32.475000000000001},"last":0.32600000000000001},{"interface":"front","average":{"1min":3.3809999999999998,"5min":1.837,"15min":1.615},"min":{"1min":0.33900000000000002,"5min":0.26300000000000001,"15min":0.20599999999999999},"max":{"1min":9.0570000000000004,"5min":9.0570000000000004,"15min":32.457000000000001},"last":0.50600000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:34:59 2023","interfaces":[{"interface":"back","average":{"1min":2.1640000000000001,"5min":1.694,"15min":1.9339999999999999},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":9.3539999999999992,"5min":9.3539999999999992,"15min":9.3539999999999992},"last":0.48999999999999999},{"interface":"front","average":{"1min":2.1070000000000002,"5min":1.516,"15min":1.546},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":9.3900000000000006,"5min":9.3900000000000006,"15min":9.3900000000000006},"last":0.56200000000000006}]},{"osd":2,"last update":"Sat Sep 23 20:35:42 2023","interfaces":[{"interface":"back","average":{"1min":2.4020000000000001,"5min":2.1070000000000002,"15min":1.8440000000000001},"min":{"1min":0.32300000000000001,"5min":0.27400000000000002,"15min":0.26700000000000002},"max":{"1min":11.347,"5min":27.603999999999999,"15min":34.514000000000003},"last":0.66900000000000004},{"interface":"front","average":{"1min":2.4159999999999999,"5min":2.4060000000000001,"15min":1.8720000000000001},"min":{"1min":0.34100000000000003,"5min":0.255,"15min":0.255},"max":{"1min":13.545,"5min":27.626999999999999,"15min":32.515000000000001},"last":0.52400000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:35:09 2023","interfaces":[{"interface":"back","average":{"1min":2.2130000000000001,"5min":2.5499999999999998,"15min":1.8149999999999999},"min":{"1min":0.314,"5min":0.29599999999999999,"15min":0.25700000000000001},"max":{"1min":9.4770000000000003,"5min":16.189,"15min":16.189},"last":0.51700000000000002},{"interface":"front","average":{"1min":3.7400000000000002,"5min":2.2749999999999999,"15min":1.669},"min":{"1min":0.36099999999999999,"5min":0.32300000000000001,"15min":0.28799999999999998},"max":{"1min":10.191000000000001,"5min":10.577999999999999,"15min":12.005000000000001},"last":0.63800000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:35:33 2023","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.67800000000000005,"15min":1.2989999999999999},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.25800000000000001},"max":{"1min":2.2280000000000002,"5min":6.282,"15min":28.015999999999998},"last":0.34300000000000003},{"interface":"front","average":{"1min":0.84899999999999998,"5min":0.69899999999999995,"15min":1.087},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":3.0659999999999998,"5min":3.0659999999999998,"15min":12.300000000000001},"last":0.46100000000000002}]},{"osd":6,"last update":"Sat Sep 23 20:35:48 2023","interfaces":[{"interface":"back","average":{"1min":0.77500000000000002,"5min":0.70699999999999996,"15min":1.2190000000000001},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":4.3300000000000001,"5min":4.3330000000000002,"15min":12.890000000000001},"last":0.377},{"interface":"front","average":{"1min":0.83699999999999997,"5min":0.73799999999999999,"15min":1.1879999999999999},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.34499999999999997},"max":{"1min":4.2400000000000002,"5min":5.1680000000000001,"15min":16.265999999999998},"last":0.38800000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:35:09 2023","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.68999999999999995,"15min":1.226},"min":{"1min":0.36399999999999999,"5min":0.32100000000000001,"15min":0.29699999999999999},"max":{"1min":1.946,"5min":2.77,"15min":36.621000000000002},"last":0.35999999999999999},{"interface":"front","average":{"1min":0.76100000000000001,"5min":0.72799999999999998,"15min":1.1359999999999999},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":5.093,"5min":5.093,"15min":37.331000000000003},"last":0.47599999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4722,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1360,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8206,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7156,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3154,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":26148864,"data_stored":26148864,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":101253120,"data_stored":101253120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":222035968,"data_stored":222035968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:36:02.201 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-23T20:36:02.261 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:36:02.306 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:02.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:02 smithi018 ceph-mon[184144]: failed to encode map e366 with expected crc 2023-09-23T20:36:02.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:02 smithi018 ceph-mon[184144]: osdmap e366: 8 total, 8 up, 8 in 2023-09-23T20:36:02.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:02 smithi018 ceph-mon[184144]: failed to encode map e366 with expected crc 2023-09-23T20:36:02.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:02 smithi018 ceph-mon[184144]: 1.1a scrub starts 2023-09-23T20:36:02.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:02 smithi018 ceph-mon[184144]: 1.1a scrub ok 2023-09-23T20:36:02.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:02 smithi018 ceph-mon[184144]: failed to encode map e366 with expected crc 2023-09-23T20:36:02.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:02 smithi018 ceph-mon[184144]: 1.1d scrub starts 2023-09-23T20:36:02.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:02 smithi018 ceph-mon[184144]: 1.1d scrub ok 2023-09-23T20:36:02.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:02 smithi018 ceph-mon[179647]: failed to encode map e366 with expected crc 2023-09-23T20:36:02.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:02 smithi018 ceph-mon[179647]: osdmap e366: 8 total, 8 up, 8 in 2023-09-23T20:36:02.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:02 smithi018 ceph-mon[179647]: failed to encode map e366 with expected crc 2023-09-23T20:36:02.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:02 smithi018 ceph-mon[179647]: 1.1a scrub starts 2023-09-23T20:36:02.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:02 smithi018 ceph-mon[179647]: 1.1a scrub ok 2023-09-23T20:36:02.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:02 smithi018 ceph-mon[179647]: failed to encode map e366 with expected crc 2023-09-23T20:36:02.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:02 smithi018 ceph-mon[179647]: 1.1d scrub starts 2023-09-23T20:36:02.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:02 smithi018 ceph-mon[179647]: 1.1d scrub ok 2023-09-23T20:36:02.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:02 smithi149 ceph-mon[170394]: failed to encode map e366 with expected crc 2023-09-23T20:36:02.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:02 smithi149 ceph-mon[170394]: osdmap e366: 8 total, 8 up, 8 in 2023-09-23T20:36:02.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:02 smithi149 ceph-mon[170394]: failed to encode map e366 with expected crc 2023-09-23T20:36:02.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:02 smithi149 ceph-mon[170394]: 1.1a scrub starts 2023-09-23T20:36:02.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:02 smithi149 ceph-mon[170394]: 1.1a scrub ok 2023-09-23T20:36:02.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:02 smithi149 ceph-mon[170394]: failed to encode map e366 with expected crc 2023-09-23T20:36:02.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:02 smithi149 ceph-mon[170394]: 1.1d scrub starts 2023-09-23T20:36:02.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:02 smithi149 ceph-mon[170394]: 1.1d scrub ok 2023-09-23T20:36:03.402 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:36:03.405 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:03.405 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:36:03.506 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:36:03.518 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:03 smithi018 ceph-mon[184144]: pgmap v880: 119 pgs: 3 peering, 116 active+clean; 4.4 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s, 307 objects/s recovering 2023-09-23T20:36:03.518 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:03 smithi018 ceph-mon[184144]: 1.1c scrub starts 2023-09-23T20:36:03.518 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:03 smithi018 ceph-mon[184144]: 1.1c scrub ok 2023-09-23T20:36:03.518 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:03 smithi018 ceph-mon[184144]: 1.16 scrub starts 2023-09-23T20:36:03.518 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:03 smithi018 ceph-mon[184144]: 1.16 scrub ok 2023-09-23T20:36:03.518 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:03 smithi018 ceph-mon[179647]: pgmap v880: 119 pgs: 3 peering, 116 active+clean; 4.4 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s, 307 objects/s recovering 2023-09-23T20:36:03.518 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:03 smithi018 ceph-mon[179647]: 1.1c scrub starts 2023-09-23T20:36:03.518 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:03 smithi018 ceph-mon[179647]: 1.1c scrub ok 2023-09-23T20:36:03.519 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:03 smithi018 ceph-mon[179647]: 1.16 scrub starts 2023-09-23T20:36:03.519 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:03 smithi018 ceph-mon[179647]: 1.16 scrub ok 2023-09-23T20:36:03.545 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:03.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:03 smithi149 ceph-mon[170394]: pgmap v880: 119 pgs: 3 peering, 116 active+clean; 4.4 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s, 307 objects/s recovering 2023-09-23T20:36:03.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:03 smithi149 ceph-mon[170394]: 1.1c scrub starts 2023-09-23T20:36:03.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:03 smithi149 ceph-mon[170394]: 1.1c scrub ok 2023-09-23T20:36:03.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:03 smithi149 ceph-mon[170394]: 1.16 scrub starts 2023-09-23T20:36:03.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:03 smithi149 ceph-mon[170394]: 1.16 scrub ok 2023-09-23T20:36:03.778 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:36:03.783 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:03.783 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:36:03.885 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:36:03.917 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:03.930 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:36:03.953 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:36:03.954 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:36:03.956 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:03.957 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:36:03.975 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:36:03.995 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:36:04.014 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:36:04.032 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:36:04.052 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:36:04.057 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:36:04.071 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:36:04.073 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:36:04.073 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:36:04.073 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:36:04.073 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:36:04.073 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:36:04.073 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:36:04.073 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:36:04.073 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:36:04.085 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:04.112 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:36:04.120 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:36:04.123 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:04.124 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:36:04.224 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:36:04.252 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:04.263 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:36:04.279 INFO:teuthology.orchestra.run.smithi018.stderr:nodeep-scrub is unset 2023-09-23T20:36:04.287 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:36:04.290 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:04.290 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:36:04.357 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:04 smithi018 ceph-mon[184144]: 1.5 scrub starts 2023-09-23T20:36:04.358 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:04 smithi018 ceph-mon[184144]: 1.5 scrub ok 2023-09-23T20:36:04.358 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:04 smithi018 ceph-mon[184144]: 1.15 scrub starts 2023-09-23T20:36:04.358 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:04 smithi018 ceph-mon[184144]: 1.15 scrub ok 2023-09-23T20:36:04.358 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:04 smithi018 ceph-mon[184144]: 1.17 scrub starts 2023-09-23T20:36:04.358 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:04 smithi018 ceph-mon[184144]: 1.17 scrub ok 2023-09-23T20:36:04.358 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:04 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/662533850' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:36:04.358 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:04 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/4267845994' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:36:04.358 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:04 smithi018 ceph-mon[179647]: 1.5 scrub starts 2023-09-23T20:36:04.359 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:04 smithi018 ceph-mon[179647]: 1.5 scrub ok 2023-09-23T20:36:04.359 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:04 smithi018 ceph-mon[179647]: 1.15 scrub starts 2023-09-23T20:36:04.359 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:04 smithi018 ceph-mon[179647]: 1.15 scrub ok 2023-09-23T20:36:04.359 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:04 smithi018 ceph-mon[179647]: 1.17 scrub starts 2023-09-23T20:36:04.359 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:04 smithi018 ceph-mon[179647]: 1.17 scrub ok 2023-09-23T20:36:04.359 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:04 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/662533850' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:36:04.359 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:04 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/4267845994' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:36:04.391 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:36:04.418 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:04.449 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:36:04.452 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:04.454 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:36:04.554 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:36:04.578 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:04 smithi149 ceph-mon[170394]: 1.5 scrub starts 2023-09-23T20:36:04.578 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:04 smithi149 ceph-mon[170394]: 1.5 scrub ok 2023-09-23T20:36:04.579 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:04 smithi149 ceph-mon[170394]: 1.15 scrub starts 2023-09-23T20:36:04.579 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:04 smithi149 ceph-mon[170394]: 1.15 scrub ok 2023-09-23T20:36:04.579 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:04 smithi149 ceph-mon[170394]: 1.17 scrub starts 2023-09-23T20:36:04.579 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:04 smithi149 ceph-mon[170394]: 1.17 scrub ok 2023-09-23T20:36:04.579 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:04 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/662533850' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:36:04.579 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:04 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/4267845994' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:36:04.582 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:04.613 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:36:04.615 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:04.616 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:36:04.716 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:36:04.742 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:04.774 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:36:04.777 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:04.777 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:36:04.878 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:36:04.914 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:05.141 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:36:04 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:36:04] "GET /metrics HTTP/1.1" 200 36515 "" "Prometheus/2.43.0" 2023-09-23T20:36:05.160 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:36:05.163 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:05.164 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:36:05.264 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:36:05.282 INFO:teuthology.orchestra.run.smithi018.stderr:pool 'unique_pool_2' removed 2023-09-23T20:36:05.297 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:05.379 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:36:05.382 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:05.383 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:36:05.483 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:36:05.500 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:05 smithi018 ceph-mon[184144]: pgmap v881: 119 pgs: 3 peering, 116 active+clean; 4.4 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s, 155 objects/s recovering 2023-09-23T20:36:05.500 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:05 smithi018 ceph-mon[184144]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:36:05.500 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:05 smithi018 ceph-mon[184144]: Cluster is now healthy 2023-09-23T20:36:05.500 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:05 smithi018 ceph-mon[184144]: failed to encode map e367 with expected crc 2023-09-23T20:36:05.500 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:05 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/4267845994' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:36:05.500 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:05 smithi018 ceph-mon[184144]: osdmap e367: 8 total, 8 up, 8 in 2023-09-23T20:36:05.500 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:05 smithi018 ceph-mon[184144]: failed to encode map e367 with expected crc 2023-09-23T20:36:05.500 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:05 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/772734982' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T20:36:05.501 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:05 smithi018 ceph-mon[179647]: pgmap v881: 119 pgs: 3 peering, 116 active+clean; 4.4 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s, 155 objects/s recovering 2023-09-23T20:36:05.501 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:05 smithi018 ceph-mon[179647]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:36:05.501 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:05 smithi018 ceph-mon[179647]: Cluster is now healthy 2023-09-23T20:36:05.501 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:05 smithi018 ceph-mon[179647]: failed to encode map e367 with expected crc 2023-09-23T20:36:05.501 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:05 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/4267845994' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:36:05.501 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:05 smithi018 ceph-mon[179647]: osdmap e367: 8 total, 8 up, 8 in 2023-09-23T20:36:05.501 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:05 smithi018 ceph-mon[179647]: failed to encode map e367 with expected crc 2023-09-23T20:36:05.501 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:05 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/772734982' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T20:36:05.509 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:05.541 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:36:05.543 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:05 smithi149 ceph-mon[170394]: pgmap v881: 119 pgs: 3 peering, 116 active+clean; 4.4 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 0 B/s, 155 objects/s recovering 2023-09-23T20:36:05.543 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:05 smithi149 ceph-mon[170394]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:36:05.544 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:05 smithi149 ceph-mon[170394]: Cluster is now healthy 2023-09-23T20:36:05.544 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:05 smithi149 ceph-mon[170394]: failed to encode map e367 with expected crc 2023-09-23T20:36:05.544 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:05 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/4267845994' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:36:05.544 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:05 smithi149 ceph-mon[170394]: osdmap e367: 8 total, 8 up, 8 in 2023-09-23T20:36:05.544 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:05 smithi149 ceph-mon[170394]: failed to encode map e367 with expected crc 2023-09-23T20:36:05.544 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:05 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/772734982' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T20:36:05.544 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:05.545 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:36:05.646 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:36:05.674 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:05.706 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:36:05.708 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:05.709 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:36:05.810 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:36:05.841 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:06.102 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:36:06.102 INFO:teuthology.orchestra.run.smithi018.stdout:{"fsid":"a3b3b17e-5a4c-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_OK","checks":{},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1042,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":368,"num_osds":8,"num_up_osds":8,"osd_up_since":1695500984,"num_in_osds":8,"osd_in_since":1695501152,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":116},{"state_name":"peering","count":3}],"num_pgs":119,"num_pools":5,"num_objects":28,"data_bytes":4665666,"bytes_used":3236364288,"bytes_avail":764723920896,"bytes_total":767960285184,"inactive_pgs_ratio":0.025210084393620491,"recovering_objects_per_sec":155,"recovering_bytes_per_sec":0,"recovering_keys_per_sec":0,"num_objects_recovered":1242,"num_bytes_recovered":0,"num_keys_recovered":0},"fsmap":{"epoch":9,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi018.hduiol","status":"up:active","gid":24566}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.18:8443/","prometheus":"http://172.21.15.18:9283/"}},"servicemap":{"epoch":23,"modified":"2023-09-23T20:34:43.862209+0000","services":{}},"progress_events":{"09b28c4a-46d3-4634-bc08-b7c059c5a8c4":{"message":"Global Recovery Event (3m)\n [===========================.] (remaining: 4s)","progress":0.97478991746902466,"add_to_ceph_s":true},"8e7eff18-0e45-4130-a6e1-662b3902f0d2":{"message":"Upgrade to 18.2.0-460-gbb0daeff (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2023-09-23T20:36:06.180 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:36:06.187 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:06.187 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:36:06.189 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-09-23T20:36:06.189 INFO:tasks.radosbench:Beginning radosbench... 2023-09-23T20:36:06.190 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_3 2023-09-23T20:36:06.190 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd pool create unique_pool_3 16 2023-09-23T20:36:06.288 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:36:06.324 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:06.362 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:36:06.364 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:06.365 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:36:06.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:06 smithi018 ceph-mon[184144]: failed to encode map e368 with expected crc 2023-09-23T20:36:06.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:06 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/772734982' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T20:36:06.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:06 smithi018 ceph-mon[184144]: osdmap e368: 8 total, 8 up, 8 in 2023-09-23T20:36:06.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:06 smithi018 ceph-mon[184144]: 1.1b deep-scrub starts 2023-09-23T20:36:06.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:06 smithi018 ceph-mon[184144]: 1.1b deep-scrub ok 2023-09-23T20:36:06.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:06 smithi018 ceph-mon[184144]: 1.c scrub starts 2023-09-23T20:36:06.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:06 smithi018 ceph-mon[184144]: 1.c scrub ok 2023-09-23T20:36:06.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:06 smithi018 ceph-mon[184144]: 1.b scrub starts 2023-09-23T20:36:06.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:06 smithi018 ceph-mon[184144]: 1.b scrub ok 2023-09-23T20:36:06.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:06 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/559842031' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T20:36:06.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:06 smithi018 ceph-mon[179647]: failed to encode map e368 with expected crc 2023-09-23T20:36:06.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:06 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/772734982' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T20:36:06.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:06 smithi018 ceph-mon[179647]: osdmap e368: 8 total, 8 up, 8 in 2023-09-23T20:36:06.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:06 smithi018 ceph-mon[179647]: 1.1b deep-scrub starts 2023-09-23T20:36:06.394 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:06 smithi018 ceph-mon[179647]: 1.1b deep-scrub ok 2023-09-23T20:36:06.394 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:06 smithi018 ceph-mon[179647]: 1.c scrub starts 2023-09-23T20:36:06.394 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:06 smithi018 ceph-mon[179647]: 1.c scrub ok 2023-09-23T20:36:06.394 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:06 smithi018 ceph-mon[179647]: 1.b scrub starts 2023-09-23T20:36:06.394 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:06 smithi018 ceph-mon[179647]: 1.b scrub ok 2023-09-23T20:36:06.394 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:06 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/559842031' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T20:36:06.466 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:36:06.495 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-09-23T20:36:06.495 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:36:06.496 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:06.536 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:36:06.539 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:06.540 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:36:06.640 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:36:06.667 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:06 smithi149 ceph-mon[170394]: failed to encode map e368 with expected crc 2023-09-23T20:36:06.667 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:06 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/772734982' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T20:36:06.667 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:06 smithi149 ceph-mon[170394]: osdmap e368: 8 total, 8 up, 8 in 2023-09-23T20:36:06.668 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:06 smithi149 ceph-mon[170394]: 1.1b deep-scrub starts 2023-09-23T20:36:06.668 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:06 smithi149 ceph-mon[170394]: 1.1b deep-scrub ok 2023-09-23T20:36:06.668 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:06 smithi149 ceph-mon[170394]: 1.c scrub starts 2023-09-23T20:36:06.668 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:06 smithi149 ceph-mon[170394]: 1.c scrub ok 2023-09-23T20:36:06.668 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:06 smithi149 ceph-mon[170394]: 1.b scrub starts 2023-09-23T20:36:06.668 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:06 smithi149 ceph-mon[170394]: 1.b scrub ok 2023-09-23T20:36:06.668 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:06 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/559842031' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T20:36:06.669 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:06.703 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:36:06.706 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:06.706 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:36:06.807 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:36:06.834 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:06.869 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:36:06.872 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:06.873 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:36:06.902 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T20:36:06.928 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:36:06.953 INFO:teuthology.orchestra.run.smithi018.stdout:ERROR: (22) Invalid argument 2023-09-23T20:36:06.954 INFO:teuthology.orchestra.run.smithi018.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:36:06.974 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:36:07.007 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:07.064 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:36:07.330 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:36:07.334 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:07.334 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:36:07.420 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:36:07.435 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:36:07.456 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T20:36:07.462 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:07.495 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:36:07.498 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:07.498 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:36:07.544 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:07 smithi018 ceph-mon[184144]: pgmap v884: 103 pgs: 103 active+clean; 4.6 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-09-23T20:36:07.544 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:07 smithi018 ceph-mon[184144]: 1.19 scrub starts 2023-09-23T20:36:07.544 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:07 smithi018 ceph-mon[184144]: 1.19 scrub ok 2023-09-23T20:36:07.544 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:07 smithi018 ceph-mon[179647]: pgmap v884: 103 pgs: 103 active+clean; 4.6 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-09-23T20:36:07.544 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:07 smithi018 ceph-mon[179647]: 1.19 scrub starts 2023-09-23T20:36:07.545 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:07 smithi018 ceph-mon[179647]: 1.19 scrub ok 2023-09-23T20:36:07.599 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:36:07.624 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:07 smithi149 ceph-mon[170394]: pgmap v884: 103 pgs: 103 active+clean; 4.6 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-09-23T20:36:07.624 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:07 smithi149 ceph-mon[170394]: 1.19 scrub starts 2023-09-23T20:36:07.624 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:07 smithi149 ceph-mon[170394]: 1.19 scrub ok 2023-09-23T20:36:07.626 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:07.659 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:36:07.662 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:07.662 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:36:07.763 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:36:07.790 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:07.822 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:36:07.825 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:07.826 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:36:07.926 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:36:07.953 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:07.988 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:36:07.991 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:07.992 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:36:08.092 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:36:08.118 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:08.152 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:36:08.155 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:08.155 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:36:08.165 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:36:08.256 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:36:08.287 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:08.559 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:36:09.178 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:36:09.182 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:09.182 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:36:09.283 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:36:09.312 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:09.348 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:36:09.350 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:09.351 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:36:09.429 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:09 smithi018 ceph-mon[184144]: pgmap v885: 103 pgs: 103 active+clean; 4.6 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-09-23T20:36:09.429 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:09 smithi018 ceph-mon[179647]: pgmap v885: 103 pgs: 103 active+clean; 4.6 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-09-23T20:36:09.452 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:36:09.479 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:09.514 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:36:09.516 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:09.517 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:36:09.618 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:36:09.652 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:09.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:09 smithi149 ceph-mon[170394]: pgmap v885: 103 pgs: 103 active+clean; 4.6 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-09-23T20:36:09.932 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:36:09.935 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:09.935 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:36:09.965 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:36:10.036 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:36:10.064 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:10.098 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:36:10.101 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:10.102 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:36:10.202 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:36:10.236 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:10.282 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:36:10.286 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:10.287 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:36:10.304 INFO:teuthology.orchestra.run.smithi018.stderr:pool 'unique_pool_3' created 2023-09-23T20:36:10.388 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:36:10.424 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:10.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:10 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2984397524' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-09-23T20:36:10.502 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:36:10.504 INFO:teuthology.orchestra.run.smithi018.stderr:dumped all 2023-09-23T20:36:10.511 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:36:10.525 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:10 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2984397524' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-09-23T20:36:10.526 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:10 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2984397524' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-09-23T20:36:10.531 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:36:10.550 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:36:10.570 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:36:10.589 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:36:10.609 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:36:10.633 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:36:10.652 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:36:10.654 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:36:10.654 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:36:10.654 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:36:10.654 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:36:10.654 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:36:10.654 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:36:10.654 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:36:10.654 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:36:10.741 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:36:10.745 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:10.745 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:36:10.846 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:36:10.874 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:10.908 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:36:10.911 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:10.911 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:36:11.012 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:36:11.044 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:11.081 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:36:11.084 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:11.085 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:36:11.185 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:36:11.209 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd pool application enable unique_pool_3 rados --yes-i-really-mean-it || true 2023-09-23T20:36:11.213 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:11.313 INFO:teuthology.orchestra.run.smithi018.stderr:noscrub is set 2023-09-23T20:36:11.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:11 smithi018 ceph-mon[179647]: pgmap v886: 103 pgs: 103 active+clean; 4.6 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-09-23T20:36:11.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:11 smithi018 ceph-mon[179647]: failed to encode map e369 with expected crc 2023-09-23T20:36:11.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:11 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2984397524' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2023-09-23T20:36:11.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:11 smithi018 ceph-mon[179647]: osdmap e369: 8 total, 8 up, 8 in 2023-09-23T20:36:11.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:11 smithi018 ceph-mon[179647]: from='client.35967 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:36:11.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:11 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3389547375' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:36:11.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:11 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2937804545' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:36:11.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:11 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:36:11.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:11 smithi018 ceph-mon[184144]: pgmap v886: 103 pgs: 103 active+clean; 4.6 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-09-23T20:36:11.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:11 smithi018 ceph-mon[184144]: failed to encode map e369 with expected crc 2023-09-23T20:36:11.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:11 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2984397524' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2023-09-23T20:36:11.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:11 smithi018 ceph-mon[184144]: osdmap e369: 8 total, 8 up, 8 in 2023-09-23T20:36:11.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:11 smithi018 ceph-mon[184144]: from='client.35967 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:36:11.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:11 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3389547375' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:36:11.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:11 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2937804545' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:36:11.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:11 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:36:11.602 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:36:11.605 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:11.605 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:36:11.706 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:36:11.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:11 smithi149 ceph-mon[170394]: pgmap v886: 103 pgs: 103 active+clean; 4.6 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-09-23T20:36:11.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:11 smithi149 ceph-mon[170394]: failed to encode map e369 with expected crc 2023-09-23T20:36:11.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:11 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2984397524' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2023-09-23T20:36:11.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:11 smithi149 ceph-mon[170394]: osdmap e369: 8 total, 8 up, 8 in 2023-09-23T20:36:11.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:11 smithi149 ceph-mon[170394]: from='client.35967 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:36:11.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:11 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3389547375' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:36:11.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:11 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2937804545' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:36:11.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:11 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:36:11.734 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:11.857 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:36:11.860 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:11.861 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:36:11.962 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:36:11.994 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:12.032 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:36:12.035 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:12.036 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:36:12.136 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:36:12.168 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:12.246 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2023-09-23T20:36:12.247 INFO:teuthology.orchestra.run.smithi018.stdout: "ops": [], 2023-09-23T20:36:12.247 INFO:teuthology.orchestra.run.smithi018.stdout: "complaint_time": 30, 2023-09-23T20:36:12.247 INFO:teuthology.orchestra.run.smithi018.stdout: "num_blocked_ops": 0 2023-09-23T20:36:12.247 INFO:teuthology.orchestra.run.smithi018.stdout:} 2023-09-23T20:36:12.253 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:36:12.256 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:12.256 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:36:12.350 INFO:teuthology.orchestra.run.smithi018.stdout:{"pg_ready":true,"pg_map":{"version":887,"stamp":"2023-09-23T20:36:10.288847+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4817030,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6018,"num_read_kb":13254,"num_write":918,"num_write_kb":12835,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":108,"num_bytes_recovered":4544057,"num_keys_recovered":40,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6022,"ondisk_log_size":6022,"up":309,"acting":309,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":345,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":3138572,"kb_used_data":237772,"kb_used_omap":32,"kb_used_meta":2900703,"kb_avail":746822644,"statfs":{"total":767960285184,"available":764746387456,"internally_reserved":0,"allocated":243478528,"data_stored":231375995,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32773,"internal_metadata":2970320891},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":66,"apply_latency_ms":66,"commit_latency_ns":66000000,"apply_latency_ns":66000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":151364,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":-56,"num_read_kb":-106,"num_write":-189129,"num_write_kb":-6051412,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-14792,"num_bytes_recovered":-652415534,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.362987"},"pg_stats":[{"pgid":"1.18","version":"90'161","reported_seq":641,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-09-23T20:36:01.455206+0000","last_change":"2023-09-23T20:35:26.644897+0000","last_active":"2023-09-23T20:36:01.455206+0000","last_peered":"2023-09-23T20:36:01.455206+0000","last_clean":"2023-09-23T20:36:01.455206+0000","last_became_active":"2023-09-23T20:35:26.644564+0000","last_became_peered":"2023-09-23T20:35:26.644564+0000","last_unstale":"2023-09-23T20:36:01.455206+0000","last_undegraded":"2023-09-23T20:36:01.455206+0000","last_fullsized":"2023-09-23T20:36:01.455206+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:35:03.551143+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:35:03.551143+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:55:55.872961+0000","scrub_duration":0.033122453000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"69'1","reported_seq":64,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-09-23T20:36:01.261864+0000","last_change":"2023-09-23T20:32:33.233853+0000","last_active":"2023-09-23T20:36:01.261864+0000","last_peered":"2023-09-23T20:36:01.261864+0000","last_clean":"2023-09-23T20:36:01.261864+0000","last_became_active":"2023-09-23T20:32:33.233546+0000","last_became_peered":"2023-09-23T20:32:33.233546+0000","last_unstale":"2023-09-23T20:36:01.261864+0000","last_undegraded":"2023-09-23T20:36:01.261864+0000","last_fullsized":"2023-09-23T20:36:01.261864+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:50:27.130808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":247,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-09-23T20:36:04.583153+0000","last_change":"2023-09-23T20:32:50.536495+0000","last_active":"2023-09-23T20:36:04.583153+0000","last_peered":"2023-09-23T20:36:04.583153+0000","last_clean":"2023-09-23T20:36:04.583153+0000","last_became_active":"2023-09-23T20:32:50.536180+0000","last_became_peered":"2023-09-23T20:32:50.536180+0000","last_unstale":"2023-09-23T20:36:04.583153+0000","last_undegraded":"2023-09-23T20:36:04.583153+0000","last_fullsized":"2023-09-23T20:36:04.583153+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:01:05.524308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1b","version":"90'161","reported_seq":507,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-09-23T20:36:01.455412+0000","last_change":"2023-09-23T20:35:56.480143+0000","last_active":"2023-09-23T20:36:01.455412+0000","last_peered":"2023-09-23T20:36:01.455412+0000","last_clean":"2023-09-23T20:36:01.455412+0000","last_became_active":"2023-09-23T20:35:26.647261+0000","last_became_peered":"2023-09-23T20:35:26.647261+0000","last_unstale":"2023-09-23T20:36:01.455412+0000","last_undegraded":"2023-09-23T20:36:01.455412+0000","last_fullsized":"2023-09-23T20:36:01.455412+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:46.558292+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:46.558292+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0019500940000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":334,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-09-23T20:36:04.582549+0000","last_change":"2023-09-23T20:34:59.983035+0000","last_active":"2023-09-23T20:36:04.582549+0000","last_peered":"2023-09-23T20:36:04.582549+0000","last_clean":"2023-09-23T20:36:04.582549+0000","last_became_active":"2023-09-23T20:34:59.982459+0000","last_became_peered":"2023-09-23T20:34:59.982459+0000","last_unstale":"2023-09-23T20:36:04.582549+0000","last_undegraded":"2023-09-23T20:36:04.582549+0000","last_fullsized":"2023-09-23T20:36:04.582549+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:16:48.510783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":335,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-09-23T20:36:04.585875+0000","last_change":"2023-09-23T20:25:02.957185+0000","last_active":"2023-09-23T20:36:04.585875+0000","last_peered":"2023-09-23T20:36:04.585875+0000","last_clean":"2023-09-23T20:36:04.585875+0000","last_became_active":"2023-09-23T20:25:02.956689+0000","last_became_peered":"2023-09-23T20:25:02.956689+0000","last_unstale":"2023-09-23T20:36:04.585875+0000","last_undegraded":"2023-09-23T20:36:04.585875+0000","last_fullsized":"2023-09-23T20:36:04.585875+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:04:28.524015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"90'161","reported_seq":446,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-09-23T20:36:01.292151+0000","last_change":"2023-09-23T20:36:01.292103+0000","last_active":"2023-09-23T20:36:01.292151+0000","last_peered":"2023-09-23T20:36:01.292151+0000","last_clean":"2023-09-23T20:36:01.292151+0000","last_became_active":"2023-09-23T20:35:26.639912+0000","last_became_peered":"2023-09-23T20:35:26.639912+0000","last_unstale":"2023-09-23T20:36:01.292151+0000","last_undegraded":"2023-09-23T20:36:01.292151+0000","last_fullsized":"2023-09-23T20:36:01.292151+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:01.292032+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:01.292032+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:04:53.926640+0000","scrub_duration":0.0032983159999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":180,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-09-23T20:36:04.584633+0000","last_change":"2023-09-23T20:32:50.546428+0000","last_active":"2023-09-23T20:36:04.584633+0000","last_peered":"2023-09-23T20:36:04.584633+0000","last_clean":"2023-09-23T20:36:04.584633+0000","last_became_active":"2023-09-23T20:32:50.546120+0000","last_became_peered":"2023-09-23T20:32:50.546120+0000","last_unstale":"2023-09-23T20:36:04.584633+0000","last_undegraded":"2023-09-23T20:36:04.584633+0000","last_fullsized":"2023-09-23T20:36:04.584633+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:00:10.792410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":141,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-09-23T20:36:04.584602+0000","last_change":"2023-09-23T20:29:46.112977+0000","last_active":"2023-09-23T20:36:04.584602+0000","last_peered":"2023-09-23T20:36:04.584602+0000","last_clean":"2023-09-23T20:36:04.584602+0000","last_became_active":"2023-09-23T20:29:46.112600+0000","last_became_peered":"2023-09-23T20:29:46.112600+0000","last_unstale":"2023-09-23T20:36:04.584602+0000","last_undegraded":"2023-09-23T20:36:04.584602+0000","last_fullsized":"2023-09-23T20:36:04.584602+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:06:34.132247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"90'161","reported_seq":540,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-09-23T20:36:04.374913+0000","last_change":"2023-09-23T20:36:03.411105+0000","last_active":"2023-09-23T20:36:04.374913+0000","last_peered":"2023-09-23T20:36:04.374913+0000","last_clean":"2023-09-23T20:36:04.374913+0000","last_became_active":"2023-09-23T20:35:26.642130+0000","last_became_peered":"2023-09-23T20:35:26.642130+0000","last_unstale":"2023-09-23T20:36:04.374913+0000","last_undegraded":"2023-09-23T20:36:04.374913+0000","last_fullsized":"2023-09-23T20:36:04.374913+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:03.411025+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:03.411025+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:31:28.138183+0000","scrub_duration":0.001802922,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":333,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-09-23T20:36:04.585373+0000","last_change":"2023-09-23T20:25:02.969432+0000","last_active":"2023-09-23T20:36:04.585373+0000","last_peered":"2023-09-23T20:36:04.585373+0000","last_clean":"2023-09-23T20:36:04.585373+0000","last_became_active":"2023-09-23T20:25:02.969118+0000","last_became_peered":"2023-09-23T20:25:02.969118+0000","last_unstale":"2023-09-23T20:36:04.585373+0000","last_undegraded":"2023-09-23T20:36:04.585373+0000","last_fullsized":"2023-09-23T20:36:04.585373+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:02:22.453831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":71,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-09-23T20:36:04.584555+0000","last_change":"2023-09-23T20:32:01.360359+0000","last_active":"2023-09-23T20:36:04.584555+0000","last_peered":"2023-09-23T20:36:04.584555+0000","last_clean":"2023-09-23T20:36:04.584555+0000","last_became_active":"2023-09-23T20:32:01.360008+0000","last_became_peered":"2023-09-23T20:32:01.360008+0000","last_unstale":"2023-09-23T20:36:04.584555+0000","last_undegraded":"2023-09-23T20:36:04.584555+0000","last_fullsized":"2023-09-23T20:36:04.584555+0000","mapping_epoch":308,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:33:43.135001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.14","version":"90'161","reported_seq":695,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-09-23T20:36:04.585400+0000","last_change":"2023-09-23T20:35:02.236398+0000","last_active":"2023-09-23T20:36:04.585400+0000","last_peered":"2023-09-23T20:36:04.585400+0000","last_clean":"2023-09-23T20:36:04.585400+0000","last_became_active":"2023-09-23T20:35:02.235988+0000","last_became_peered":"2023-09-23T20:35:02.235988+0000","last_unstale":"2023-09-23T20:36:04.585400+0000","last_undegraded":"2023-09-23T20:36:04.585400+0000","last_fullsized":"2023-09-23T20:36:04.585400+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:47.479012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:47.479012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:13.519648+0000","scrub_duration":0.0020136590000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":343,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-09-23T20:36:01.455689+0000","last_change":"2023-09-23T20:29:45.523808+0000","last_active":"2023-09-23T20:36:01.455689+0000","last_peered":"2023-09-23T20:36:01.455689+0000","last_clean":"2023-09-23T20:36:01.455689+0000","last_became_active":"2023-09-23T20:29:45.523508+0000","last_became_peered":"2023-09-23T20:29:45.523508+0000","last_unstale":"2023-09-23T20:36:01.455689+0000","last_undegraded":"2023-09-23T20:36:01.455689+0000","last_fullsized":"2023-09-23T20:36:01.455689+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:50:55.565352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":167,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-09-23T20:36:04.374838+0000","last_change":"2023-09-23T20:32:50.539003+0000","last_active":"2023-09-23T20:36:04.374838+0000","last_peered":"2023-09-23T20:36:04.374838+0000","last_clean":"2023-09-23T20:36:04.374838+0000","last_became_active":"2023-09-23T20:32:50.538596+0000","last_became_peered":"2023-09-23T20:32:50.538596+0000","last_unstale":"2023-09-23T20:36:04.374838+0000","last_undegraded":"2023-09-23T20:36:04.374838+0000","last_fullsized":"2023-09-23T20:36:04.374838+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:56:03.314375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.17","version":"90'161","reported_seq":508,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-09-23T20:36:03.435617+0000","last_change":"2023-09-23T20:36:03.435557+0000","last_active":"2023-09-23T20:36:03.435617+0000","last_peered":"2023-09-23T20:36:03.435617+0000","last_clean":"2023-09-23T20:36:03.435617+0000","last_became_active":"2023-09-23T20:35:26.652669+0000","last_became_peered":"2023-09-23T20:35:26.652669+0000","last_unstale":"2023-09-23T20:36:03.435617+0000","last_undegraded":"2023-09-23T20:36:03.435617+0000","last_fullsized":"2023-09-23T20:36:03.435617+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:03.435480+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:03.435480+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:27:05.066931+0000","scrub_duration":0.001636689,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":395,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-09-23T20:36:04.375134+0000","last_change":"2023-09-23T20:32:33.270617+0000","last_active":"2023-09-23T20:36:04.375134+0000","last_peered":"2023-09-23T20:36:04.375134+0000","last_clean":"2023-09-23T20:36:04.375134+0000","last_became_active":"2023-09-23T20:32:33.227902+0000","last_became_peered":"2023-09-23T20:32:33.227902+0000","last_unstale":"2023-09-23T20:36:04.375134+0000","last_undegraded":"2023-09-23T20:36:04.375134+0000","last_fullsized":"2023-09-23T20:36:04.375134+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:30:26.847831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":360,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":317,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-09-23T20:36:04.582936+0000","last_change":"2023-09-23T20:29:46.080766+0000","last_active":"2023-09-23T20:36:04.582936+0000","last_peered":"2023-09-23T20:36:04.582936+0000","last_clean":"2023-09-23T20:36:04.582936+0000","last_became_active":"2023-09-23T20:29:46.080356+0000","last_became_peered":"2023-09-23T20:29:46.080356+0000","last_unstale":"2023-09-23T20:36:04.582936+0000","last_undegraded":"2023-09-23T20:36:04.582936+0000","last_fullsized":"2023-09-23T20:36:04.582936+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:08:01.862270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"90'161","reported_seq":490,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-09-23T20:36:04.583327+0000","last_change":"2023-09-23T20:36:02.548387+0000","last_active":"2023-09-23T20:36:04.583327+0000","last_peered":"2023-09-23T20:36:04.583327+0000","last_clean":"2023-09-23T20:36:04.583327+0000","last_became_active":"2023-09-23T20:35:26.644787+0000","last_became_peered":"2023-09-23T20:35:26.644787+0000","last_unstale":"2023-09-23T20:36:04.583327+0000","last_undegraded":"2023-09-23T20:36:04.583327+0000","last_fullsized":"2023-09-23T20:36:04.583327+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:02.548285+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:02.548285+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:37:15.216553+0000","scrub_duration":0.0015514459999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":74,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-09-23T20:36:01.261770+0000","last_change":"2023-09-23T20:32:33.266787+0000","last_active":"2023-09-23T20:36:01.261770+0000","last_peered":"2023-09-23T20:36:01.261770+0000","last_clean":"2023-09-23T20:36:01.261770+0000","last_became_active":"2023-09-23T20:32:33.228482+0000","last_became_peered":"2023-09-23T20:32:33.228482+0000","last_unstale":"2023-09-23T20:36:01.261770+0000","last_undegraded":"2023-09-23T20:36:01.261770+0000","last_fullsized":"2023-09-23T20:36:01.261770+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:13:01.812073+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":64,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-09-23T20:36:01.262243+0000","last_change":"2023-09-23T20:32:33.228929+0000","last_active":"2023-09-23T20:36:01.262243+0000","last_peered":"2023-09-23T20:36:01.262243+0000","last_clean":"2023-09-23T20:36:01.262243+0000","last_became_active":"2023-09-23T20:32:33.228595+0000","last_became_peered":"2023-09-23T20:32:33.228595+0000","last_unstale":"2023-09-23T20:36:01.262243+0000","last_undegraded":"2023-09-23T20:36:01.262243+0000","last_fullsized":"2023-09-23T20:36:01.262243+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:17:53.053808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"90'161","reported_seq":598,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-09-23T20:36:04.584282+0000","last_change":"2023-09-23T20:31:43.578810+0000","last_active":"2023-09-23T20:36:04.584282+0000","last_peered":"2023-09-23T20:36:04.584282+0000","last_clean":"2023-09-23T20:36:04.584282+0000","last_became_active":"2023-09-23T20:28:05.991304+0000","last_became_peered":"2023-09-23T20:28:05.991304+0000","last_unstale":"2023-09-23T20:36:04.584282+0000","last_undegraded":"2023-09-23T20:36:04.584282+0000","last_fullsized":"2023-09-23T20:36:04.584282+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:43.578666+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:43.578666+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:46:18.999991+0000","scrub_duration":0.0021890379999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":394,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-09-23T20:36:04.374912+0000","last_change":"2023-09-23T20:32:33.242943+0000","last_active":"2023-09-23T20:36:04.374912+0000","last_peered":"2023-09-23T20:36:04.374912+0000","last_clean":"2023-09-23T20:36:04.374912+0000","last_became_active":"2023-09-23T20:32:33.225468+0000","last_became_peered":"2023-09-23T20:32:33.225468+0000","last_unstale":"2023-09-23T20:36:04.374912+0000","last_undegraded":"2023-09-23T20:36:04.374912+0000","last_fullsized":"2023-09-23T20:36:04.374912+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:40:47.867573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":68,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-09-23T20:36:01.455171+0000","last_change":"2023-09-23T20:32:01.358590+0000","last_active":"2023-09-23T20:36:01.455171+0000","last_peered":"2023-09-23T20:36:01.455171+0000","last_clean":"2023-09-23T20:36:01.455171+0000","last_became_active":"2023-09-23T20:32:01.358345+0000","last_became_peered":"2023-09-23T20:32:01.358345+0000","last_unstale":"2023-09-23T20:36:01.455171+0000","last_undegraded":"2023-09-23T20:36:01.455171+0000","last_fullsized":"2023-09-23T20:36:01.455171+0000","mapping_epoch":308,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:22:21.159708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.10","version":"90'161","reported_seq":597,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-09-23T20:36:01.455309+0000","last_change":"2023-09-23T20:31:49.726214+0000","last_active":"2023-09-23T20:36:01.455309+0000","last_peered":"2023-09-23T20:36:01.455309+0000","last_clean":"2023-09-23T20:36:01.455309+0000","last_became_active":"2023-09-23T20:29:07.044892+0000","last_became_peered":"2023-09-23T20:29:07.044892+0000","last_unstale":"2023-09-23T20:36:01.455309+0000","last_undegraded":"2023-09-23T20:36:01.455309+0000","last_fullsized":"2023-09-23T20:36:01.455309+0000","mapping_epoch":253,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":254,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:49.726139+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:49.726139+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:59:11.810516+0000","scrub_duration":0.0015441470000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":333,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-09-23T20:36:04.584671+0000","last_change":"2023-09-23T20:25:02.953993+0000","last_active":"2023-09-23T20:36:04.584671+0000","last_peered":"2023-09-23T20:36:04.584671+0000","last_clean":"2023-09-23T20:36:04.584671+0000","last_became_active":"2023-09-23T20:25:02.953672+0000","last_became_peered":"2023-09-23T20:25:02.953672+0000","last_unstale":"2023-09-23T20:36:04.584671+0000","last_undegraded":"2023-09-23T20:36:04.584671+0000","last_fullsized":"2023-09-23T20:36:04.584671+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:26:08.603577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":345,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-09-23T20:36:01.455757+0000","last_change":"2023-09-23T20:25:02.949584+0000","last_active":"2023-09-23T20:36:01.455757+0000","last_peered":"2023-09-23T20:36:01.455757+0000","last_clean":"2023-09-23T20:36:01.455757+0000","last_became_active":"2023-09-23T20:25:02.949132+0000","last_became_peered":"2023-09-23T20:25:02.949132+0000","last_unstale":"2023-09-23T20:36:01.455757+0000","last_undegraded":"2023-09-23T20:36:01.455757+0000","last_fullsized":"2023-09-23T20:36:01.455757+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:34:20.509823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"90'161","reported_seq":651,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-09-23T20:36:04.582875+0000","last_change":"2023-09-23T20:35:02.238923+0000","last_active":"2023-09-23T20:36:04.582875+0000","last_peered":"2023-09-23T20:36:04.582875+0000","last_clean":"2023-09-23T20:36:04.582875+0000","last_became_active":"2023-09-23T20:35:02.238293+0000","last_became_peered":"2023-09-23T20:35:02.238293+0000","last_unstale":"2023-09-23T20:36:04.582875+0000","last_undegraded":"2023-09-23T20:36:04.582875+0000","last_fullsized":"2023-09-23T20:36:04.582875+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:42.544308+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:42.544308+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:51:22.281932+0000","scrub_duration":0.0029958179999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":351,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-09-23T20:36:04.374771+0000","last_change":"2023-09-23T20:32:33.225148+0000","last_active":"2023-09-23T20:36:04.374771+0000","last_peered":"2023-09-23T20:36:04.374771+0000","last_clean":"2023-09-23T20:36:04.374771+0000","last_became_active":"2023-09-23T20:32:33.224805+0000","last_became_peered":"2023-09-23T20:32:33.224805+0000","last_unstale":"2023-09-23T20:36:04.374771+0000","last_undegraded":"2023-09-23T20:36:04.374771+0000","last_fullsized":"2023-09-23T20:36:04.374771+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:55:30.891398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":343,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-09-23T20:36:04.585951+0000","last_change":"2023-09-23T20:25:02.957010+0000","last_active":"2023-09-23T20:36:04.585951+0000","last_peered":"2023-09-23T20:36:04.585951+0000","last_clean":"2023-09-23T20:36:04.585951+0000","last_became_active":"2023-09-23T20:25:02.956267+0000","last_became_peered":"2023-09-23T20:25:02.956267+0000","last_unstale":"2023-09-23T20:36:04.585951+0000","last_undegraded":"2023-09-23T20:36:04.585951+0000","last_fullsized":"2023-09-23T20:36:04.585951+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:10:55.299422+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"90'161","reported_seq":616,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-09-23T20:36:01.455665+0000","last_change":"2023-09-23T20:31:53.707033+0000","last_active":"2023-09-23T20:36:01.455665+0000","last_peered":"2023-09-23T20:36:01.455665+0000","last_clean":"2023-09-23T20:36:01.455665+0000","last_became_active":"2023-09-23T20:29:46.078873+0000","last_became_peered":"2023-09-23T20:29:46.078873+0000","last_unstale":"2023-09-23T20:36:01.455665+0000","last_undegraded":"2023-09-23T20:36:01.455665+0000","last_fullsized":"2023-09-23T20:36:01.455665+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:53.706934+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:53.706934+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:51:58.039657+0000","scrub_duration":0.0016840900000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":137,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-09-23T20:36:04.583811+0000","last_change":"2023-09-23T20:29:04.179171+0000","last_active":"2023-09-23T20:36:04.583811+0000","last_peered":"2023-09-23T20:36:04.583811+0000","last_clean":"2023-09-23T20:36:04.583811+0000","last_became_active":"2023-09-23T20:29:04.128672+0000","last_became_peered":"2023-09-23T20:29:04.128672+0000","last_unstale":"2023-09-23T20:36:04.583811+0000","last_undegraded":"2023-09-23T20:36:04.583811+0000","last_fullsized":"2023-09-23T20:36:04.583811+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:49:15.506106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":180,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-09-23T20:36:04.375287+0000","last_change":"2023-09-23T20:32:50.539157+0000","last_active":"2023-09-23T20:36:04.375287+0000","last_peered":"2023-09-23T20:36:04.375287+0000","last_clean":"2023-09-23T20:36:04.375287+0000","last_became_active":"2023-09-23T20:32:50.538889+0000","last_became_peered":"2023-09-23T20:32:50.538889+0000","last_unstale":"2023-09-23T20:36:04.375287+0000","last_undegraded":"2023-09-23T20:36:04.375287+0000","last_fullsized":"2023-09-23T20:36:04.375287+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:28:28.857256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.4","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T20:36:10.283416+0000","last_change":"2023-09-23T20:36:10.283416+0000","last_active":"2023-09-23T20:36:10.283416+0000","last_peered":"2023-09-23T20:36:10.283416+0000","last_clean":"2023-09-23T20:36:10.283416+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T20:36:10.283416+0000","last_undegraded":"2023-09-23T20:36:10.283416+0000","last_fullsized":"2023-09-23T20:36:10.283416+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.d","version":"90'161","reported_seq":660,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-09-23T20:36:04.375032+0000","last_change":"2023-09-23T20:34:59.496438+0000","last_active":"2023-09-23T20:36:04.375032+0000","last_peered":"2023-09-23T20:36:04.375032+0000","last_clean":"2023-09-23T20:36:04.375032+0000","last_became_active":"2023-09-23T20:33:38.970527+0000","last_became_peered":"2023-09-23T20:33:38.970527+0000","last_unstale":"2023-09-23T20:36:04.375032+0000","last_undegraded":"2023-09-23T20:36:04.375032+0000","last_fullsized":"2023-09-23T20:36:04.375032+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:34:59.496375+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:34:59.496375+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:46:18.244734+0000","scrub_duration":0.0017843189999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":371,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-09-23T20:36:04.375230+0000","last_change":"2023-09-23T20:32:33.283752+0000","last_active":"2023-09-23T20:36:04.375230+0000","last_peered":"2023-09-23T20:36:04.375230+0000","last_clean":"2023-09-23T20:36:04.375230+0000","last_became_active":"2023-09-23T20:32:33.230503+0000","last_became_peered":"2023-09-23T20:32:33.230503+0000","last_unstale":"2023-09-23T20:36:04.375230+0000","last_undegraded":"2023-09-23T20:36:04.375230+0000","last_fullsized":"2023-09-23T20:36:04.375230+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:29.084410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":328,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-09-23T20:36:01.455510+0000","last_change":"2023-09-23T20:29:04.128604+0000","last_active":"2023-09-23T20:36:01.455510+0000","last_peered":"2023-09-23T20:36:01.455510+0000","last_clean":"2023-09-23T20:36:01.455510+0000","last_became_active":"2023-09-23T20:29:04.128003+0000","last_became_peered":"2023-09-23T20:29:04.128003+0000","last_unstale":"2023-09-23T20:36:01.455510+0000","last_undegraded":"2023-09-23T20:36:01.455510+0000","last_fullsized":"2023-09-23T20:36:01.455510+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:50:49.385599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T20:36:10.283416+0000","last_change":"2023-09-23T20:36:10.283416+0000","last_active":"2023-09-23T20:36:10.283416+0000","last_peered":"2023-09-23T20:36:10.283416+0000","last_clean":"2023-09-23T20:36:10.283416+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T20:36:10.283416+0000","last_undegraded":"2023-09-23T20:36:10.283416+0000","last_fullsized":"2023-09-23T20:36:10.283416+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.c","version":"90'161","reported_seq":551,"reported_epoch":368,"state":"active+clean","last_fresh":"2023-09-23T20:36:05.485796+0000","last_change":"2023-09-23T20:36:05.485974+0000","last_active":"2023-09-23T20:36:05.485796+0000","last_peered":"2023-09-23T20:36:05.485796+0000","last_clean":"2023-09-23T20:36:05.485796+0000","last_became_active":"2023-09-23T20:33:38.989255+0000","last_became_peered":"2023-09-23T20:33:38.989255+0000","last_unstale":"2023-09-23T20:36:05.485796+0000","last_undegraded":"2023-09-23T20:36:05.485796+0000","last_fullsized":"2023-09-23T20:36:05.485796+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:05.485751+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:05.485751+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:48:05.480750+0000","scrub_duration":0.0057838120000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":406,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-09-23T20:36:04.375181+0000","last_change":"2023-09-23T20:34:59.987981+0000","last_active":"2023-09-23T20:36:04.375181+0000","last_peered":"2023-09-23T20:36:04.375181+0000","last_clean":"2023-09-23T20:36:04.375181+0000","last_became_active":"2023-09-23T20:34:59.985368+0000","last_became_peered":"2023-09-23T20:34:59.985368+0000","last_unstale":"2023-09-23T20:36:04.375181+0000","last_undegraded":"2023-09-23T20:36:04.375181+0000","last_fullsized":"2023-09-23T20:36:04.375181+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:12:43.082493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":64,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-09-23T20:36:01.262628+0000","last_change":"2023-09-23T20:32:33.232142+0000","last_active":"2023-09-23T20:36:01.262628+0000","last_peered":"2023-09-23T20:36:01.262628+0000","last_clean":"2023-09-23T20:36:01.262628+0000","last_became_active":"2023-09-23T20:32:33.231683+0000","last_became_peered":"2023-09-23T20:32:33.231683+0000","last_unstale":"2023-09-23T20:36:01.262628+0000","last_undegraded":"2023-09-23T20:36:01.262628+0000","last_fullsized":"2023-09-23T20:36:01.262628+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:02:58.336348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T20:36:10.283416+0000","last_change":"2023-09-23T20:36:10.283416+0000","last_active":"2023-09-23T20:36:10.283416+0000","last_peered":"2023-09-23T20:36:10.283416+0000","last_clean":"2023-09-23T20:36:10.283416+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T20:36:10.283416+0000","last_undegraded":"2023-09-23T20:36:10.283416+0000","last_fullsized":"2023-09-23T20:36:10.283416+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.f","version":"90'161","reported_seq":590,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-09-23T20:36:04.583609+0000","last_change":"2023-09-23T20:32:36.302809+0000","last_active":"2023-09-23T20:36:04.583609+0000","last_peered":"2023-09-23T20:36:04.583609+0000","last_clean":"2023-09-23T20:36:04.583609+0000","last_became_active":"2023-09-23T20:32:36.302465+0000","last_became_peered":"2023-09-23T20:32:36.302465+0000","last_unstale":"2023-09-23T20:36:04.583609+0000","last_undegraded":"2023-09-23T20:36:04.583609+0000","last_fullsized":"2023-09-23T20:36:04.583609+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:46.217489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:46.217489+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:35:40.488427+0000","scrub_duration":0.001456429,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":340,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-09-23T20:36:04.375082+0000","last_change":"2023-09-23T20:29:04.184893+0000","last_active":"2023-09-23T20:36:04.375082+0000","last_peered":"2023-09-23T20:36:04.375082+0000","last_clean":"2023-09-23T20:36:04.375082+0000","last_became_active":"2023-09-23T20:29:04.152723+0000","last_became_peered":"2023-09-23T20:29:04.152723+0000","last_unstale":"2023-09-23T20:36:04.375082+0000","last_undegraded":"2023-09-23T20:36:04.375082+0000","last_fullsized":"2023-09-23T20:36:04.375082+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:51:27.017151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":141,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-09-23T20:36:04.583909+0000","last_change":"2023-09-23T20:29:46.086912+0000","last_active":"2023-09-23T20:36:04.583909+0000","last_peered":"2023-09-23T20:36:04.583909+0000","last_clean":"2023-09-23T20:36:04.583909+0000","last_became_active":"2023-09-23T20:29:46.085471+0000","last_became_peered":"2023-09-23T20:29:46.085471+0000","last_unstale":"2023-09-23T20:36:04.583909+0000","last_undegraded":"2023-09-23T20:36:04.583909+0000","last_fullsized":"2023-09-23T20:36:04.583909+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:04:55.751660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T20:36:10.283416+0000","last_change":"2023-09-23T20:36:10.283416+0000","last_active":"2023-09-23T20:36:10.283416+0000","last_peered":"2023-09-23T20:36:10.283416+0000","last_clean":"2023-09-23T20:36:10.283416+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T20:36:10.283416+0000","last_undegraded":"2023-09-23T20:36:10.283416+0000","last_fullsized":"2023-09-23T20:36:10.283416+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.e","version":"90'161","reported_seq":467,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-09-23T20:36:01.262313+0000","last_change":"2023-09-23T20:36:00.322453+0000","last_active":"2023-09-23T20:36:01.262313+0000","last_peered":"2023-09-23T20:36:01.262313+0000","last_clean":"2023-09-23T20:36:01.262313+0000","last_became_active":"2023-09-23T20:35:02.233536+0000","last_became_peered":"2023-09-23T20:35:02.233536+0000","last_unstale":"2023-09-23T20:36:01.262313+0000","last_undegraded":"2023-09-23T20:36:01.262313+0000","last_fullsized":"2023-09-23T20:36:01.262313+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:00.322381+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:00.322381+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:02:15.489069+0000","scrub_duration":0.0017321859999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":373,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-09-23T20:36:01.455739+0000","last_change":"2023-09-23T20:29:04.138284+0000","last_active":"2023-09-23T20:36:01.455739+0000","last_peered":"2023-09-23T20:36:01.455739+0000","last_clean":"2023-09-23T20:36:01.455739+0000","last_became_active":"2023-09-23T20:29:04.137960+0000","last_became_peered":"2023-09-23T20:29:04.137960+0000","last_unstale":"2023-09-23T20:36:01.455739+0000","last_undegraded":"2023-09-23T20:36:01.455739+0000","last_fullsized":"2023-09-23T20:36:01.455739+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:18:59.548311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":352,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-09-23T20:36:04.585210+0000","last_change":"2023-09-23T20:32:33.236122+0000","last_active":"2023-09-23T20:36:04.585210+0000","last_peered":"2023-09-23T20:36:04.585210+0000","last_clean":"2023-09-23T20:36:04.585210+0000","last_became_active":"2023-09-23T20:32:33.235384+0000","last_became_peered":"2023-09-23T20:32:33.235384+0000","last_unstale":"2023-09-23T20:36:04.585210+0000","last_undegraded":"2023-09-23T20:36:04.585210+0000","last_fullsized":"2023-09-23T20:36:04.585210+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:56.843733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T20:36:10.283416+0000","last_change":"2023-09-23T20:36:10.283416+0000","last_active":"2023-09-23T20:36:10.283416+0000","last_peered":"2023-09-23T20:36:10.283416+0000","last_clean":"2023-09-23T20:36:10.283416+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T20:36:10.283416+0000","last_undegraded":"2023-09-23T20:36:10.283416+0000","last_fullsized":"2023-09-23T20:36:10.283416+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":773,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-09-23T20:36:04.374671+0000","last_change":"2023-09-23T20:35:02.523308+0000","last_active":"2023-09-23T20:36:04.374671+0000","last_peered":"2023-09-23T20:36:04.374671+0000","last_clean":"2023-09-23T20:36:04.374671+0000","last_became_active":"2023-09-23T20:33:38.979989+0000","last_became_peered":"2023-09-23T20:33:38.979989+0000","last_unstale":"2023-09-23T20:36:04.374671+0000","last_undegraded":"2023-09-23T20:36:04.374671+0000","last_fullsized":"2023-09-23T20:36:04.374671+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:35:02.523203+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:35:02.523203+0000","last_clean_scrub_stamp":"2023-09-23T20:35:02.523203+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:02:41.472958+0000","scrub_duration":0.0085927989999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":103,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-09-23T20:36:04.585300+0000","last_change":"2023-09-23T20:32:50.538500+0000","last_active":"2023-09-23T20:36:04.585300+0000","last_peered":"2023-09-23T20:36:04.585300+0000","last_clean":"2023-09-23T20:36:04.585300+0000","last_became_active":"2023-09-23T20:32:50.538152+0000","last_became_peered":"2023-09-23T20:32:50.538152+0000","last_unstale":"2023-09-23T20:36:04.585300+0000","last_undegraded":"2023-09-23T20:36:04.585300+0000","last_fullsized":"2023-09-23T20:36:04.585300+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:00:41.103508+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":64,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-09-23T20:36:01.261936+0000","last_change":"2023-09-23T20:32:33.231614+0000","last_active":"2023-09-23T20:36:01.261936+0000","last_peered":"2023-09-23T20:36:01.261936+0000","last_clean":"2023-09-23T20:36:01.261936+0000","last_became_active":"2023-09-23T20:32:33.231207+0000","last_became_peered":"2023-09-23T20:32:33.231207+0000","last_unstale":"2023-09-23T20:36:01.261936+0000","last_undegraded":"2023-09-23T20:36:01.261936+0000","last_fullsized":"2023-09-23T20:36:01.261936+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:46:09.780829+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T20:36:10.283416+0000","last_change":"2023-09-23T20:36:10.283416+0000","last_active":"2023-09-23T20:36:10.283416+0000","last_peered":"2023-09-23T20:36:10.283416+0000","last_clean":"2023-09-23T20:36:10.283416+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T20:36:10.283416+0000","last_undegraded":"2023-09-23T20:36:10.283416+0000","last_fullsized":"2023-09-23T20:36:10.283416+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":617,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-09-23T20:36:01.455403+0000","last_change":"2023-09-23T20:35:00.496271+0000","last_active":"2023-09-23T20:36:01.455403+0000","last_peered":"2023-09-23T20:36:01.455403+0000","last_clean":"2023-09-23T20:36:01.455403+0000","last_became_active":"2023-09-23T20:33:38.996273+0000","last_became_peered":"2023-09-23T20:33:38.996273+0000","last_unstale":"2023-09-23T20:36:01.455403+0000","last_undegraded":"2023-09-23T20:36:01.455403+0000","last_fullsized":"2023-09-23T20:36:01.455403+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:35:00.496197+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:35:00.496197+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:30:15.813493+0000","scrub_duration":0.0016339779999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":163,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-09-23T20:36:04.583954+0000","last_change":"2023-09-23T20:32:33.229700+0000","last_active":"2023-09-23T20:36:04.583954+0000","last_peered":"2023-09-23T20:36:04.583954+0000","last_clean":"2023-09-23T20:36:04.583954+0000","last_became_active":"2023-09-23T20:32:33.229372+0000","last_became_peered":"2023-09-23T20:32:33.229372+0000","last_unstale":"2023-09-23T20:36:04.583954+0000","last_undegraded":"2023-09-23T20:36:04.583954+0000","last_fullsized":"2023-09-23T20:36:04.583954+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:18:10.958227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":23,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-09-23T20:36:01.261476+0000","last_change":"2023-09-23T20:34:59.985545+0000","last_active":"2023-09-23T20:36:01.261476+0000","last_peered":"2023-09-23T20:36:01.261476+0000","last_clean":"2023-09-23T20:36:01.261476+0000","last_became_active":"2023-09-23T20:34:59.985090+0000","last_became_peered":"2023-09-23T20:34:59.985090+0000","last_unstale":"2023-09-23T20:36:01.261476+0000","last_undegraded":"2023-09-23T20:36:01.261476+0000","last_fullsized":"2023-09-23T20:36:01.261476+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:21:40.825065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T20:36:10.283416+0000","last_change":"2023-09-23T20:36:10.283416+0000","last_active":"2023-09-23T20:36:10.283416+0000","last_peered":"2023-09-23T20:36:10.283416+0000","last_clean":"2023-09-23T20:36:10.283416+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T20:36:10.283416+0000","last_undegraded":"2023-09-23T20:36:10.283416+0000","last_fullsized":"2023-09-23T20:36:10.283416+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.b","version":"90'161","reported_seq":633,"reported_epoch":368,"state":"active+clean","last_fresh":"2023-09-23T20:36:05.595064+0000","last_change":"2023-09-23T20:36:05.595168+0000","last_active":"2023-09-23T20:36:05.595064+0000","last_peered":"2023-09-23T20:36:05.595064+0000","last_clean":"2023-09-23T20:36:05.595064+0000","last_became_active":"2023-09-23T20:33:38.991910+0000","last_became_peered":"2023-09-23T20:33:38.991910+0000","last_unstale":"2023-09-23T20:36:05.595064+0000","last_undegraded":"2023-09-23T20:36:05.595064+0000","last_fullsized":"2023-09-23T20:36:05.595064+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:05.595042+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:05.595042+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:06:55.394493+0000","scrub_duration":0.007271675,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":23,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-09-23T20:36:01.262136+0000","last_change":"2023-09-23T20:34:59.983122+0000","last_active":"2023-09-23T20:36:01.262136+0000","last_peered":"2023-09-23T20:36:01.262136+0000","last_clean":"2023-09-23T20:36:01.262136+0000","last_became_active":"2023-09-23T20:34:59.982809+0000","last_became_peered":"2023-09-23T20:34:59.982809+0000","last_unstale":"2023-09-23T20:36:01.262136+0000","last_undegraded":"2023-09-23T20:36:01.262136+0000","last_fullsized":"2023-09-23T20:36:01.262136+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:49:40.839723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":64,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-09-23T20:36:01.262524+0000","last_change":"2023-09-23T20:32:33.224963+0000","last_active":"2023-09-23T20:36:01.262524+0000","last_peered":"2023-09-23T20:36:01.262524+0000","last_clean":"2023-09-23T20:36:01.262524+0000","last_became_active":"2023-09-23T20:32:33.224634+0000","last_became_peered":"2023-09-23T20:32:33.224634+0000","last_unstale":"2023-09-23T20:36:01.262524+0000","last_undegraded":"2023-09-23T20:36:01.262524+0000","last_fullsized":"2023-09-23T20:36:01.262524+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:41:40.314140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T20:36:10.283416+0000","last_change":"2023-09-23T20:36:10.283416+0000","last_active":"2023-09-23T20:36:10.283416+0000","last_peered":"2023-09-23T20:36:10.283416+0000","last_clean":"2023-09-23T20:36:10.283416+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T20:36:10.283416+0000","last_undegraded":"2023-09-23T20:36:10.283416+0000","last_fullsized":"2023-09-23T20:36:10.283416+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":660,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-09-23T20:36:01.455522+0000","last_change":"2023-09-23T20:34:59.481511+0000","last_active":"2023-09-23T20:36:01.455522+0000","last_peered":"2023-09-23T20:36:01.455522+0000","last_clean":"2023-09-23T20:36:01.455522+0000","last_became_active":"2023-09-23T20:33:38.988954+0000","last_became_peered":"2023-09-23T20:33:38.988954+0000","last_unstale":"2023-09-23T20:36:01.455522+0000","last_undegraded":"2023-09-23T20:36:01.455522+0000","last_fullsized":"2023-09-23T20:36:01.455522+0000","mapping_epoch":253,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:34:59.481446+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:34:59.481446+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:53:42.768648+0000","scrub_duration":0.001554773,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":181,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-09-23T20:36:04.582689+0000","last_change":"2023-09-23T20:32:50.530690+0000","last_active":"2023-09-23T20:36:04.582689+0000","last_peered":"2023-09-23T20:36:04.582689+0000","last_clean":"2023-09-23T20:36:04.582689+0000","last_became_active":"2023-09-23T20:32:50.530376+0000","last_became_peered":"2023-09-23T20:32:50.530376+0000","last_unstale":"2023-09-23T20:36:04.582689+0000","last_undegraded":"2023-09-23T20:36:04.582689+0000","last_fullsized":"2023-09-23T20:36:04.582689+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:24:53.184959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":189,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-09-23T20:36:04.583220+0000","last_change":"2023-09-23T20:32:33.230582+0000","last_active":"2023-09-23T20:36:04.583220+0000","last_peered":"2023-09-23T20:36:04.583220+0000","last_clean":"2023-09-23T20:36:04.583220+0000","last_became_active":"2023-09-23T20:32:33.230257+0000","last_became_peered":"2023-09-23T20:32:33.230257+0000","last_unstale":"2023-09-23T20:36:04.583220+0000","last_undegraded":"2023-09-23T20:36:04.583220+0000","last_fullsized":"2023-09-23T20:36:04.583220+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:11:45.333096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T20:36:10.283416+0000","last_change":"2023-09-23T20:36:10.283416+0000","last_active":"2023-09-23T20:36:10.283416+0000","last_peered":"2023-09-23T20:36:10.283416+0000","last_clean":"2023-09-23T20:36:10.283416+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T20:36:10.283416+0000","last_undegraded":"2023-09-23T20:36:10.283416+0000","last_fullsized":"2023-09-23T20:36:10.283416+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":617,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-09-23T20:36:04.584560+0000","last_change":"2023-09-23T20:35:04.102089+0000","last_active":"2023-09-23T20:36:04.584560+0000","last_peered":"2023-09-23T20:36:04.584560+0000","last_clean":"2023-09-23T20:36:04.584560+0000","last_became_active":"2023-09-23T20:33:38.978580+0000","last_became_peered":"2023-09-23T20:33:38.978580+0000","last_unstale":"2023-09-23T20:36:04.584560+0000","last_undegraded":"2023-09-23T20:36:04.584560+0000","last_fullsized":"2023-09-23T20:36:04.584560+0000","mapping_epoch":253,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:35:04.101972+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:35:04.101972+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:44:38.790944+0000","scrub_duration":0.0063459650000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"86'119","reported_seq":674,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-09-23T20:36:04.583101+0000","last_change":"2023-09-23T20:29:45.521257+0000","last_active":"2023-09-23T20:36:04.583101+0000","last_peered":"2023-09-23T20:36:04.583101+0000","last_clean":"2023-09-23T20:36:04.583101+0000","last_became_active":"2023-09-23T20:29:45.520546+0000","last_became_peered":"2023-09-23T20:29:45.520546+0000","last_unstale":"2023-09-23T20:36:04.583101+0000","last_undegraded":"2023-09-23T20:36:04.583101+0000","last_fullsized":"2023-09-23T20:36:04.583101+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":119,"log_dups_size":0,"ondisk_log_size":119,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:58:56.085777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032705499999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":523,"num_read_kb":3503,"num_write":136,"num_write_kb":3128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":383,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-09-23T20:36:01.455674+0000","last_change":"2023-09-23T20:25:39.637160+0000","last_active":"2023-09-23T20:36:01.455674+0000","last_peered":"2023-09-23T20:36:01.455674+0000","last_clean":"2023-09-23T20:36:01.455674+0000","last_became_active":"2023-09-23T20:25:39.636860+0000","last_became_peered":"2023-09-23T20:25:39.636860+0000","last_unstale":"2023-09-23T20:36:01.455674+0000","last_undegraded":"2023-09-23T20:36:01.455674+0000","last_fullsized":"2023-09-23T20:36:01.455674+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:21:47.860063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":303,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-09-23T20:36:04.582598+0000","last_change":"2023-09-23T20:34:59.985572+0000","last_active":"2023-09-23T20:36:04.582598+0000","last_peered":"2023-09-23T20:36:04.582598+0000","last_clean":"2023-09-23T20:36:04.582598+0000","last_became_active":"2023-09-23T20:34:59.985241+0000","last_became_peered":"2023-09-23T20:34:59.985241+0000","last_unstale":"2023-09-23T20:36:04.582598+0000","last_undegraded":"2023-09-23T20:36:04.582598+0000","last_fullsized":"2023-09-23T20:36:04.582598+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:54:57.081334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.8","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T20:36:10.283416+0000","last_change":"2023-09-23T20:36:10.283416+0000","last_active":"2023-09-23T20:36:10.283416+0000","last_peered":"2023-09-23T20:36:10.283416+0000","last_clean":"2023-09-23T20:36:10.283416+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T20:36:10.283416+0000","last_undegraded":"2023-09-23T20:36:10.283416+0000","last_fullsized":"2023-09-23T20:36:10.283416+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":611,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-09-23T20:36:01.455316+0000","last_change":"2023-09-23T20:31:45.785521+0000","last_active":"2023-09-23T20:36:01.455316+0000","last_peered":"2023-09-23T20:36:01.455316+0000","last_clean":"2023-09-23T20:36:01.455316+0000","last_became_active":"2023-09-23T20:27:55.213394+0000","last_became_peered":"2023-09-23T20:27:55.213394+0000","last_unstale":"2023-09-23T20:36:01.455316+0000","last_undegraded":"2023-09-23T20:36:01.455316+0000","last_fullsized":"2023-09-23T20:36:01.455316+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:45.785450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:45.785450+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:21:26.092578+0000","scrub_duration":0.001558306,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"86'106","reported_seq":1085,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-09-23T20:36:01.455353+0000","last_change":"2023-09-23T20:31:31.531882+0000","last_active":"2023-09-23T20:36:01.455353+0000","last_peered":"2023-09-23T20:36:01.455353+0000","last_clean":"2023-09-23T20:36:01.455353+0000","last_became_active":"2023-09-23T20:29:46.078501+0000","last_became_peered":"2023-09-23T20:29:46.078501+0000","last_unstale":"2023-09-23T20:36:01.455353+0000","last_undegraded":"2023-09-23T20:36:01.455353+0000","last_fullsized":"2023-09-23T20:36:01.455353+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":106,"log_dups_size":0,"ondisk_log_size":106,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:37.015974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00099302599999999993,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1105,"num_read_kb":1689,"num_write":100,"num_write_kb":404,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":64,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-09-23T20:36:01.262078+0000","last_change":"2023-09-23T20:32:33.230242+0000","last_active":"2023-09-23T20:36:01.262078+0000","last_peered":"2023-09-23T20:36:01.262078+0000","last_clean":"2023-09-23T20:36:01.262078+0000","last_became_active":"2023-09-23T20:32:33.229732+0000","last_became_peered":"2023-09-23T20:32:33.229732+0000","last_unstale":"2023-09-23T20:36:01.262078+0000","last_undegraded":"2023-09-23T20:36:01.262078+0000","last_fullsized":"2023-09-23T20:36:01.262078+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:27:55.538429+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":327,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-09-23T20:36:04.583035+0000","last_change":"2023-09-23T20:29:45.533406+0000","last_active":"2023-09-23T20:36:04.583035+0000","last_peered":"2023-09-23T20:36:04.583035+0000","last_clean":"2023-09-23T20:36:04.583035+0000","last_became_active":"2023-09-23T20:29:45.526001+0000","last_became_peered":"2023-09-23T20:29:45.526001+0000","last_unstale":"2023-09-23T20:36:04.583035+0000","last_undegraded":"2023-09-23T20:36:04.583035+0000","last_fullsized":"2023-09-23T20:36:04.583035+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:43.787251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.b","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T20:36:10.283416+0000","last_change":"2023-09-23T20:36:10.283416+0000","last_active":"2023-09-23T20:36:10.283416+0000","last_peered":"2023-09-23T20:36:10.283416+0000","last_clean":"2023-09-23T20:36:10.283416+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T20:36:10.283416+0000","last_undegraded":"2023-09-23T20:36:10.283416+0000","last_fullsized":"2023-09-23T20:36:10.283416+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":840,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-09-23T20:36:04.374798+0000","last_change":"2023-09-23T20:35:02.260708+0000","last_active":"2023-09-23T20:36:04.374798+0000","last_peered":"2023-09-23T20:36:04.374798+0000","last_clean":"2023-09-23T20:36:04.374798+0000","last_became_active":"2023-09-23T20:35:02.260348+0000","last_became_peered":"2023-09-23T20:35:02.260348+0000","last_unstale":"2023-09-23T20:36:04.374798+0000","last_undegraded":"2023-09-23T20:36:04.374798+0000","last_fullsized":"2023-09-23T20:36:04.374798+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:43.390908+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:43.390908+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:09:14.144070+0000","scrub_duration":0.0032721,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"86'130","reported_seq":682,"reported_epoch":368,"state":"active+clean","last_fresh":"2023-09-23T20:36:05.628877+0000","last_change":"2023-09-23T20:25:39.637458+0000","last_active":"2023-09-23T20:36:05.628877+0000","last_peered":"2023-09-23T20:36:05.628877+0000","last_clean":"2023-09-23T20:36:05.628877+0000","last_became_active":"2023-09-23T20:25:39.636773+0000","last_became_peered":"2023-09-23T20:25:39.636773+0000","last_unstale":"2023-09-23T20:36:05.628877+0000","last_undegraded":"2023-09-23T20:36:05.628877+0000","last_fullsized":"2023-09-23T20:36:05.628877+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:00:32.810564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011300049999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":512,"num_read_kb":1820,"num_write":133,"num_write_kb":1347,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":317,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-09-23T20:36:04.582557+0000","last_change":"2023-09-23T20:29:46.073394+0000","last_active":"2023-09-23T20:36:04.582557+0000","last_peered":"2023-09-23T20:36:04.582557+0000","last_clean":"2023-09-23T20:36:04.582557+0000","last_became_active":"2023-09-23T20:29:46.072067+0000","last_became_peered":"2023-09-23T20:29:46.072067+0000","last_unstale":"2023-09-23T20:36:04.582557+0000","last_undegraded":"2023-09-23T20:36:04.582557+0000","last_fullsized":"2023-09-23T20:36:04.582557+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:01:56.944614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":308,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-09-23T20:36:04.583173+0000","last_change":"2023-09-23T20:29:45.533185+0000","last_active":"2023-09-23T20:36:04.583173+0000","last_peered":"2023-09-23T20:36:04.583173+0000","last_clean":"2023-09-23T20:36:04.583173+0000","last_became_active":"2023-09-23T20:29:45.526765+0000","last_became_peered":"2023-09-23T20:29:45.526765+0000","last_unstale":"2023-09-23T20:36:04.583173+0000","last_undegraded":"2023-09-23T20:36:04.583173+0000","last_fullsized":"2023-09-23T20:36:04.583173+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:18:33.063215+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T20:36:10.283416+0000","last_change":"2023-09-23T20:36:10.283416+0000","last_active":"2023-09-23T20:36:10.283416+0000","last_peered":"2023-09-23T20:36:10.283416+0000","last_clean":"2023-09-23T20:36:10.283416+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T20:36:10.283416+0000","last_undegraded":"2023-09-23T20:36:10.283416+0000","last_fullsized":"2023-09-23T20:36:10.283416+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.3","version":"86'194","reported_seq":1015,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-09-23T20:36:01.455555+0000","last_change":"2023-09-23T20:25:40.618665+0000","last_active":"2023-09-23T20:36:01.455555+0000","last_peered":"2023-09-23T20:36:01.455555+0000","last_clean":"2023-09-23T20:36:01.455555+0000","last_became_active":"2023-09-23T20:25:40.618087+0000","last_became_peered":"2023-09-23T20:25:40.618087+0000","last_unstale":"2023-09-23T20:36:01.455555+0000","last_undegraded":"2023-09-23T20:36:01.455555+0000","last_fullsized":"2023-09-23T20:36:01.455555+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":194,"log_dups_size":0,"ondisk_log_size":194,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:07:16.114968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00037995999999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1025,"num_read_kb":1201,"num_write":173,"num_write_kb":1276,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":801,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-09-23T20:36:04.374673+0000","last_change":"2023-09-23T20:31:45.343835+0000","last_active":"2023-09-23T20:36:04.374673+0000","last_peered":"2023-09-23T20:36:04.374673+0000","last_clean":"2023-09-23T20:36:04.374673+0000","last_became_active":"2023-09-23T20:29:07.042407+0000","last_became_peered":"2023-09-23T20:29:07.042407+0000","last_unstale":"2023-09-23T20:36:04.374673+0000","last_undegraded":"2023-09-23T20:36:04.374673+0000","last_fullsized":"2023-09-23T20:36:04.374673+0000","mapping_epoch":253,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":254,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:45.343742+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:45.343742+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:33:12.745366+0000","scrub_duration":0.001475802,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":166,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-09-23T20:36:01.455445+0000","last_change":"2023-09-23T20:32:50.534315+0000","last_active":"2023-09-23T20:36:01.455445+0000","last_peered":"2023-09-23T20:36:01.455445+0000","last_clean":"2023-09-23T20:36:01.455445+0000","last_became_active":"2023-09-23T20:32:50.533896+0000","last_became_peered":"2023-09-23T20:32:50.533896+0000","last_unstale":"2023-09-23T20:36:01.455445+0000","last_undegraded":"2023-09-23T20:36:01.455445+0000","last_fullsized":"2023-09-23T20:36:01.455445+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:06:53.464923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":147,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-09-23T20:36:04.584336+0000","last_change":"2023-09-23T20:32:33.225012+0000","last_active":"2023-09-23T20:36:04.584336+0000","last_peered":"2023-09-23T20:36:04.584336+0000","last_clean":"2023-09-23T20:36:04.584336+0000","last_became_active":"2023-09-23T20:32:33.224699+0000","last_became_peered":"2023-09-23T20:32:33.224699+0000","last_unstale":"2023-09-23T20:36:04.584336+0000","last_undegraded":"2023-09-23T20:36:04.584336+0000","last_fullsized":"2023-09-23T20:36:04.584336+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:10:06.267878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.a","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T20:36:10.283416+0000","last_change":"2023-09-23T20:36:10.283416+0000","last_active":"2023-09-23T20:36:10.283416+0000","last_peered":"2023-09-23T20:36:10.283416+0000","last_clean":"2023-09-23T20:36:10.283416+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T20:36:10.283416+0000","last_undegraded":"2023-09-23T20:36:10.283416+0000","last_fullsized":"2023-09-23T20:36:10.283416+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":662,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-09-23T20:36:04.582836+0000","last_change":"2023-09-23T20:35:02.238632+0000","last_active":"2023-09-23T20:36:04.582836+0000","last_peered":"2023-09-23T20:36:04.582836+0000","last_clean":"2023-09-23T20:36:04.582836+0000","last_became_active":"2023-09-23T20:35:02.237812+0000","last_became_peered":"2023-09-23T20:35:02.237812+0000","last_unstale":"2023-09-23T20:36:04.582836+0000","last_undegraded":"2023-09-23T20:36:04.582836+0000","last_fullsized":"2023-09-23T20:36:04.582836+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:43.582146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:43.582146+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:25:04.582696+0000","scrub_duration":0.0016219730000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"86'80","reported_seq":631,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-09-23T20:36:04.374637+0000","last_change":"2023-09-23T20:29:45.536796+0000","last_active":"2023-09-23T20:36:04.374637+0000","last_peered":"2023-09-23T20:36:04.374637+0000","last_clean":"2023-09-23T20:36:04.374637+0000","last_became_active":"2023-09-23T20:29:45.536439+0000","last_became_peered":"2023-09-23T20:29:45.536439+0000","last_unstale":"2023-09-23T20:36:04.374637+0000","last_undegraded":"2023-09-23T20:36:04.374637+0000","last_fullsized":"2023-09-23T20:36:04.374637+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":80,"log_dups_size":0,"ondisk_log_size":80,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:30:57.793132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035115,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":395,"num_read_kb":1370,"num_write":66,"num_write_kb":1061,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":370,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-09-23T20:36:04.584483+0000","last_change":"2023-09-23T20:34:59.984286+0000","last_active":"2023-09-23T20:36:04.584483+0000","last_peered":"2023-09-23T20:36:04.584483+0000","last_clean":"2023-09-23T20:36:04.584483+0000","last_became_active":"2023-09-23T20:34:59.983950+0000","last_became_peered":"2023-09-23T20:34:59.983950+0000","last_unstale":"2023-09-23T20:36:04.584483+0000","last_undegraded":"2023-09-23T20:36:04.584483+0000","last_fullsized":"2023-09-23T20:36:04.584483+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:31:53.557035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":365,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-09-23T20:36:04.585692+0000","last_change":"2023-09-23T20:29:45.690815+0000","last_active":"2023-09-23T20:36:04.585692+0000","last_peered":"2023-09-23T20:36:04.585692+0000","last_clean":"2023-09-23T20:36:04.585692+0000","last_became_active":"2023-09-23T20:29:45.690478+0000","last_became_peered":"2023-09-23T20:29:45.690478+0000","last_unstale":"2023-09-23T20:36:04.585692+0000","last_undegraded":"2023-09-23T20:36:04.585692+0000","last_fullsized":"2023-09-23T20:36:04.585692+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:38.288193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.f","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T20:36:10.283416+0000","last_change":"2023-09-23T20:36:10.283416+0000","last_active":"2023-09-23T20:36:10.283416+0000","last_peered":"2023-09-23T20:36:10.283416+0000","last_clean":"2023-09-23T20:36:10.283416+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T20:36:10.283416+0000","last_undegraded":"2023-09-23T20:36:10.283416+0000","last_fullsized":"2023-09-23T20:36:10.283416+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":610,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-09-23T20:36:04.584439+0000","last_change":"2023-09-23T20:35:00.151823+0000","last_active":"2023-09-23T20:36:04.584439+0000","last_peered":"2023-09-23T20:36:04.584439+0000","last_clean":"2023-09-23T20:36:04.584439+0000","last_became_active":"2023-09-23T20:33:38.991030+0000","last_became_peered":"2023-09-23T20:33:38.991030+0000","last_unstale":"2023-09-23T20:36:04.584439+0000","last_undegraded":"2023-09-23T20:36:04.584439+0000","last_fullsized":"2023-09-23T20:36:04.584439+0000","mapping_epoch":253,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:35:00.151695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:35:00.151695+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:09:55.183927+0000","scrub_duration":0.0016030090000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"86'102","reported_seq":962,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-09-23T20:36:04.374533+0000","last_change":"2023-09-23T20:32:36.314672+0000","last_active":"2023-09-23T20:36:04.374533+0000","last_peered":"2023-09-23T20:36:04.374533+0000","last_clean":"2023-09-23T20:36:04.374533+0000","last_became_active":"2023-09-23T20:32:36.314231+0000","last_became_peered":"2023-09-23T20:32:36.314231+0000","last_unstale":"2023-09-23T20:36:04.374533+0000","last_undegraded":"2023-09-23T20:36:04.374533+0000","last_fullsized":"2023-09-23T20:36:04.374533+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":102,"log_dups_size":0,"ondisk_log_size":102,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:11:40.514709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030656500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":963,"num_read_kb":1234,"num_write":60,"num_write_kb":225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":293,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-09-23T20:36:04.583129+0000","last_change":"2023-09-23T20:29:45.538283+0000","last_active":"2023-09-23T20:36:04.583129+0000","last_peered":"2023-09-23T20:36:04.583129+0000","last_clean":"2023-09-23T20:36:04.583129+0000","last_became_active":"2023-09-23T20:29:45.537922+0000","last_became_peered":"2023-09-23T20:29:45.537922+0000","last_unstale":"2023-09-23T20:36:04.583129+0000","last_undegraded":"2023-09-23T20:36:04.583129+0000","last_fullsized":"2023-09-23T20:36:04.583129+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:30:52.397836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":302,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-09-23T20:36:04.585478+0000","last_change":"2023-09-23T20:20:14.888874+0000","last_active":"2023-09-23T20:36:04.585478+0000","last_peered":"2023-09-23T20:36:04.585478+0000","last_clean":"2023-09-23T20:36:04.585478+0000","last_became_active":"2023-09-23T20:20:14.887724+0000","last_became_peered":"2023-09-23T20:20:14.887724+0000","last_unstale":"2023-09-23T20:36:04.585478+0000","last_undegraded":"2023-09-23T20:36:04.585478+0000","last_fullsized":"2023-09-23T20:36:04.585478+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:09:24.510326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T20:36:10.283416+0000","last_change":"2023-09-23T20:36:10.283416+0000","last_active":"2023-09-23T20:36:10.283416+0000","last_peered":"2023-09-23T20:36:10.283416+0000","last_clean":"2023-09-23T20:36:10.283416+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T20:36:10.283416+0000","last_undegraded":"2023-09-23T20:36:10.283416+0000","last_fullsized":"2023-09-23T20:36:10.283416+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":585,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-09-23T20:36:03.246427+0000","last_change":"2023-09-23T20:36:03.246372+0000","last_active":"2023-09-23T20:36:03.246427+0000","last_peered":"2023-09-23T20:36:03.246427+0000","last_clean":"2023-09-23T20:36:03.246427+0000","last_became_active":"2023-09-23T20:33:38.985300+0000","last_became_peered":"2023-09-23T20:33:38.985300+0000","last_unstale":"2023-09-23T20:36:03.246427+0000","last_undegraded":"2023-09-23T20:36:03.246427+0000","last_fullsized":"2023-09-23T20:36:03.246427+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:03.246271+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:03.246271+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:12:15.732632+0000","scrub_duration":0.0019016700000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":105,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-09-23T20:36:04.375297+0000","last_change":"2023-09-23T20:32:50.549695+0000","last_active":"2023-09-23T20:36:04.375297+0000","last_peered":"2023-09-23T20:36:04.375297+0000","last_clean":"2023-09-23T20:36:04.375297+0000","last_became_active":"2023-09-23T20:32:50.549431+0000","last_became_peered":"2023-09-23T20:32:50.549431+0000","last_unstale":"2023-09-23T20:36:04.375297+0000","last_undegraded":"2023-09-23T20:36:04.375297+0000","last_fullsized":"2023-09-23T20:36:04.375297+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:45:53.666569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"86'128","reported_seq":758,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-09-23T20:36:04.582440+0000","last_change":"2023-09-23T20:32:36.310656+0000","last_active":"2023-09-23T20:36:04.582440+0000","last_peered":"2023-09-23T20:36:04.582440+0000","last_clean":"2023-09-23T20:36:04.582440+0000","last_became_active":"2023-09-23T20:32:36.310155+0000","last_became_peered":"2023-09-23T20:32:36.310155+0000","last_unstale":"2023-09-23T20:36:04.582440+0000","last_undegraded":"2023-09-23T20:36:04.582440+0000","last_fullsized":"2023-09-23T20:36:04.582440+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:58:14.217466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000293202,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":579,"num_read_kb":1473,"num_write":96,"num_write_kb":1062,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":138,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-09-23T20:36:04.374891+0000","last_change":"2023-09-23T20:32:50.539051+0000","last_active":"2023-09-23T20:36:04.374891+0000","last_peered":"2023-09-23T20:36:04.374891+0000","last_clean":"2023-09-23T20:36:04.374891+0000","last_became_active":"2023-09-23T20:32:50.538755+0000","last_became_peered":"2023-09-23T20:32:50.538755+0000","last_unstale":"2023-09-23T20:36:04.374891+0000","last_undegraded":"2023-09-23T20:36:04.374891+0000","last_fullsized":"2023-09-23T20:36:04.374891+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:16.656613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.d","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T20:36:10.283416+0000","last_change":"2023-09-23T20:36:10.283416+0000","last_active":"2023-09-23T20:36:10.283416+0000","last_peered":"2023-09-23T20:36:10.283416+0000","last_clean":"2023-09-23T20:36:10.283416+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T20:36:10.283416+0000","last_undegraded":"2023-09-23T20:36:10.283416+0000","last_fullsized":"2023-09-23T20:36:10.283416+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.4","version":"90'161","reported_seq":516,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-09-23T20:36:01.455798+0000","last_change":"2023-09-23T20:34:54.504652+0000","last_active":"2023-09-23T20:36:01.455798+0000","last_peered":"2023-09-23T20:36:01.455798+0000","last_clean":"2023-09-23T20:36:01.455798+0000","last_became_active":"2023-09-23T20:32:50.533470+0000","last_became_peered":"2023-09-23T20:32:50.533470+0000","last_unstale":"2023-09-23T20:36:01.455798+0000","last_undegraded":"2023-09-23T20:36:01.455798+0000","last_fullsized":"2023-09-23T20:36:01.455798+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:34:54.504554+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:34:54.504554+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:17:03.108888+0000","scrub_duration":0.013031021,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.7","version":"86'117","reported_seq":637,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-09-23T20:36:04.584645+0000","last_change":"2023-09-23T20:31:32.814392+0000","last_active":"2023-09-23T20:36:04.584645+0000","last_peered":"2023-09-23T20:36:04.584645+0000","last_clean":"2023-09-23T20:36:04.584645+0000","last_became_active":"2023-09-23T20:31:32.813857+0000","last_became_peered":"2023-09-23T20:31:32.813857+0000","last_unstale":"2023-09-23T20:36:04.584645+0000","last_undegraded":"2023-09-23T20:36:04.584645+0000","last_fullsized":"2023-09-23T20:36:04.584645+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:56:57.863005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046891400000000002,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":893,"num_read_kb":922,"num_write":82,"num_write_kb":4193,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":365,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-09-23T20:36:04.585784+0000","last_change":"2023-09-23T20:32:33.277827+0000","last_active":"2023-09-23T20:36:04.585784+0000","last_peered":"2023-09-23T20:36:04.585784+0000","last_clean":"2023-09-23T20:36:04.585784+0000","last_became_active":"2023-09-23T20:32:33.236087+0000","last_became_peered":"2023-09-23T20:32:33.236087+0000","last_unstale":"2023-09-23T20:36:04.585784+0000","last_undegraded":"2023-09-23T20:36:04.585784+0000","last_fullsized":"2023-09-23T20:36:04.585784+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:20:58.176854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":64,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-09-23T20:36:01.262533+0000","last_change":"2023-09-23T20:32:33.238290+0000","last_active":"2023-09-23T20:36:01.262533+0000","last_peered":"2023-09-23T20:36:01.262533+0000","last_clean":"2023-09-23T20:36:01.262533+0000","last_became_active":"2023-09-23T20:32:33.237984+0000","last_became_peered":"2023-09-23T20:32:33.237984+0000","last_unstale":"2023-09-23T20:36:01.262533+0000","last_undegraded":"2023-09-23T20:36:01.262533+0000","last_fullsized":"2023-09-23T20:36:01.262533+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:49:02.089307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"90'161","reported_seq":589,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-09-23T20:36:04.584090+0000","last_change":"2023-09-23T20:35:26.651952+0000","last_active":"2023-09-23T20:36:04.584090+0000","last_peered":"2023-09-23T20:36:04.584090+0000","last_clean":"2023-09-23T20:36:04.584090+0000","last_became_active":"2023-09-23T20:35:26.651601+0000","last_became_peered":"2023-09-23T20:35:26.651601+0000","last_unstale":"2023-09-23T20:36:04.584090+0000","last_undegraded":"2023-09-23T20:36:04.584090+0000","last_fullsized":"2023-09-23T20:36:04.584090+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:28:05.872074+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:28:05.872074+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":333,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-09-23T20:36:04.585397+0000","last_change":"2023-09-23T20:32:33.237234+0000","last_active":"2023-09-23T20:36:04.585397+0000","last_peered":"2023-09-23T20:36:04.585397+0000","last_clean":"2023-09-23T20:36:04.585397+0000","last_became_active":"2023-09-23T20:32:33.236881+0000","last_became_peered":"2023-09-23T20:32:33.236881+0000","last_unstale":"2023-09-23T20:36:04.585397+0000","last_undegraded":"2023-09-23T20:36:04.585397+0000","last_fullsized":"2023-09-23T20:36:04.585397+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:56:17.384831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":221,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-09-23T20:36:04.585594+0000","last_change":"2023-09-23T20:29:45.538314+0000","last_active":"2023-09-23T20:36:04.585594+0000","last_peered":"2023-09-23T20:36:04.585594+0000","last_clean":"2023-09-23T20:36:04.585594+0000","last_became_active":"2023-09-23T20:29:45.536847+0000","last_became_peered":"2023-09-23T20:29:45.536847+0000","last_unstale":"2023-09-23T20:36:04.585594+0000","last_undegraded":"2023-09-23T20:36:04.585594+0000","last_fullsized":"2023-09-23T20:36:04.585594+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:51.948721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"90'161","reported_seq":452,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-09-23T20:36:04.584261+0000","last_change":"2023-09-23T20:35:42.864462+0000","last_active":"2023-09-23T20:36:04.584261+0000","last_peered":"2023-09-23T20:36:04.584261+0000","last_clean":"2023-09-23T20:36:04.584261+0000","last_became_active":"2023-09-23T20:35:26.665630+0000","last_became_peered":"2023-09-23T20:35:26.665630+0000","last_unstale":"2023-09-23T20:36:04.584261+0000","last_undegraded":"2023-09-23T20:36:04.584261+0000","last_fullsized":"2023-09-23T20:36:04.584261+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:35:42.864363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:35:42.864363+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:14:34.984081+0000","scrub_duration":0.0070144939999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":376,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-09-23T20:36:01.455595+0000","last_change":"2023-09-23T20:32:33.235987+0000","last_active":"2023-09-23T20:36:01.455595+0000","last_peered":"2023-09-23T20:36:01.455595+0000","last_clean":"2023-09-23T20:36:01.455595+0000","last_became_active":"2023-09-23T20:32:33.226544+0000","last_became_peered":"2023-09-23T20:32:33.226544+0000","last_unstale":"2023-09-23T20:36:01.455595+0000","last_undegraded":"2023-09-23T20:36:01.455595+0000","last_fullsized":"2023-09-23T20:36:01.455595+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:53:22.794856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":23,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-09-23T20:36:01.261865+0000","last_change":"2023-09-23T20:34:59.987575+0000","last_active":"2023-09-23T20:36:01.261865+0000","last_peered":"2023-09-23T20:36:01.261865+0000","last_clean":"2023-09-23T20:36:01.261865+0000","last_became_active":"2023-09-23T20:34:59.987229+0000","last_became_peered":"2023-09-23T20:34:59.987229+0000","last_unstale":"2023-09-23T20:36:01.261865+0000","last_undegraded":"2023-09-23T20:36:01.261865+0000","last_fullsized":"2023-09-23T20:36:01.261865+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:34:01.820159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":384,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-09-23T20:36:01.455553+0000","last_change":"2023-09-23T20:32:33.268260+0000","last_active":"2023-09-23T20:36:01.455553+0000","last_peered":"2023-09-23T20:36:01.455553+0000","last_clean":"2023-09-23T20:36:01.455553+0000","last_became_active":"2023-09-23T20:32:33.233695+0000","last_became_peered":"2023-09-23T20:32:33.233695+0000","last_unstale":"2023-09-23T20:36:01.455553+0000","last_undegraded":"2023-09-23T20:36:01.455553+0000","last_fullsized":"2023-09-23T20:36:01.455553+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:19:29.299623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":23,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-09-23T20:36:01.261735+0000","last_change":"2023-09-23T20:34:59.987416+0000","last_active":"2023-09-23T20:36:01.261735+0000","last_peered":"2023-09-23T20:36:01.261735+0000","last_clean":"2023-09-23T20:36:01.261735+0000","last_became_active":"2023-09-23T20:34:59.987117+0000","last_became_peered":"2023-09-23T20:34:59.987117+0000","last_unstale":"2023-09-23T20:36:01.261735+0000","last_undegraded":"2023-09-23T20:36:01.261735+0000","last_fullsized":"2023-09-23T20:36:01.261735+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:36:16.318623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"90'161","reported_seq":563,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-09-23T20:36:04.374903+0000","last_change":"2023-09-23T20:36:02.384213+0000","last_active":"2023-09-23T20:36:04.374903+0000","last_peered":"2023-09-23T20:36:04.374903+0000","last_clean":"2023-09-23T20:36:04.374903+0000","last_became_active":"2023-09-23T20:35:26.643160+0000","last_became_peered":"2023-09-23T20:35:26.643160+0000","last_unstale":"2023-09-23T20:36:04.374903+0000","last_undegraded":"2023-09-23T20:36:04.374903+0000","last_fullsized":"2023-09-23T20:36:04.374903+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:02.384133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:02.384133+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:34:12.654342+0000","scrub_duration":0.001769098,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":183,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-09-23T20:36:04.584369+0000","last_change":"2023-09-23T20:32:50.537447+0000","last_active":"2023-09-23T20:36:04.584369+0000","last_peered":"2023-09-23T20:36:04.584369+0000","last_clean":"2023-09-23T20:36:04.584369+0000","last_became_active":"2023-09-23T20:32:50.537136+0000","last_became_peered":"2023-09-23T20:32:50.537136+0000","last_unstale":"2023-09-23T20:36:04.584369+0000","last_undegraded":"2023-09-23T20:36:04.584369+0000","last_fullsized":"2023-09-23T20:36:04.584369+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:46:51.599492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":83,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-09-23T20:36:04.584778+0000","last_change":"2023-09-23T20:32:50.537450+0000","last_active":"2023-09-23T20:36:04.584778+0000","last_peered":"2023-09-23T20:36:04.584778+0000","last_clean":"2023-09-23T20:36:04.584778+0000","last_became_active":"2023-09-23T20:32:50.537128+0000","last_became_peered":"2023-09-23T20:32:50.537128+0000","last_unstale":"2023-09-23T20:36:04.584778+0000","last_undegraded":"2023-09-23T20:36:04.584778+0000","last_fullsized":"2023-09-23T20:36:04.584778+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:37:15.889029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1d","version":"90'161","reported_seq":458,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-09-23T20:36:04.583052+0000","last_change":"2023-09-23T20:36:01.505098+0000","last_active":"2023-09-23T20:36:04.583052+0000","last_peered":"2023-09-23T20:36:04.583052+0000","last_clean":"2023-09-23T20:36:04.583052+0000","last_became_active":"2023-09-23T20:35:26.644707+0000","last_became_peered":"2023-09-23T20:35:26.644707+0000","last_unstale":"2023-09-23T20:36:04.583052+0000","last_undegraded":"2023-09-23T20:36:04.583052+0000","last_fullsized":"2023-09-23T20:36:04.583052+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:01.504973+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:01.504973+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:02:10.880809+0000","scrub_duration":0.00202344,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":350,"reported_epoch":367,"state":"active+clean","last_fresh":"2023-09-23T20:36:04.585466+0000","last_change":"2023-09-23T20:25:39.668882+0000","last_active":"2023-09-23T20:36:04.585466+0000","last_peered":"2023-09-23T20:36:04.585466+0000","last_clean":"2023-09-23T20:36:04.585466+0000","last_became_active":"2023-09-23T20:25:39.637128+0000","last_became_peered":"2023-09-23T20:25:39.637128+0000","last_unstale":"2023-09-23T20:36:04.585466+0000","last_undegraded":"2023-09-23T20:36:04.585466+0000","last_fullsized":"2023-09-23T20:36:04.585466+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:56.406298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":64,"reported_epoch":366,"state":"active+clean","last_fresh":"2023-09-23T20:36:01.261362+0000","last_change":"2023-09-23T20:32:33.232281+0000","last_active":"2023-09-23T20:36:01.261362+0000","last_peered":"2023-09-23T20:36:01.261362+0000","last_clean":"2023-09-23T20:36:01.261362+0000","last_became_active":"2023-09-23T20:32:33.232003+0000","last_became_peered":"2023-09-23T20:32:33.232003+0000","last_unstale":"2023-09-23T20:36:01.261362+0000","last_undegraded":"2023-09-23T20:36:01.261362+0000","last_fullsized":"2023-09-23T20:36:01.261362+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:58:27.314355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":49,"num_bytes_recovered":6522,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26691,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":617203,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":343193,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":58,"num_bytes_recovered":4194342,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6082,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201579,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":318024,"kb_used_data":8192,"kb_used_omap":0,"kb_used_meta":309824,"kb_avail":93427128,"statfs":{"total":95995035648,"available":95669379072,"internally_reserved":0,"allocated":8388608,"data_stored":7223270,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":317259776},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:35:44 2023","interfaces":[{"interface":"back","average":{"1min":3.5600000000000001,"5min":2.5299999999999998,"15min":1.8160000000000001},"min":{"1min":0.28100000000000003,"5min":0.23599999999999999,"15min":0.20799999999999999},"max":{"1min":17.399999999999999,"5min":17.399999999999999,"15min":48.798999999999999},"last":0.55100000000000005},{"interface":"front","average":{"1min":3.1890000000000001,"5min":2.21,"15min":1.877},"min":{"1min":0.249,"5min":0.249,"15min":0.24399999999999999},"max":{"1min":19.355,"5min":19.355,"15min":48.247},"last":0.71099999999999997}]},{"osd":1,"last update":"Sat Sep 23 20:35:59 2023","interfaces":[{"interface":"back","average":{"1min":3.0880000000000001,"5min":2.25,"15min":1.9239999999999999},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.254},"max":{"1min":16.103000000000002,"5min":16.103000000000002,"15min":16.103000000000002},"last":0.60699999999999998},{"interface":"front","average":{"1min":3.0249999999999999,"5min":2.0430000000000001,"15min":2.0009999999999999},"min":{"1min":0.32700000000000001,"5min":0.248,"15min":0.248},"max":{"1min":12.02,"5min":12.02,"15min":12.02},"last":0.57799999999999996}]},{"osd":2,"last update":"Sat Sep 23 20:35:44 2023","interfaces":[{"interface":"back","average":{"1min":3.3079999999999998,"5min":2.492,"15min":1.8620000000000001},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27600000000000002},"max":{"1min":9.8629999999999995,"5min":13.173999999999999,"15min":13.173999999999999},"last":0.52000000000000002},{"interface":"front","average":{"1min":3.7130000000000001,"5min":2.8799999999999999,"15min":2.113},"min":{"1min":0.36099999999999999,"5min":0.27400000000000002,"15min":0.26600000000000001},"max":{"1min":12.551,"5min":23.494,"15min":23.494},"last":0.63800000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:35:11 2023","interfaces":[{"interface":"back","average":{"1min":4.8280000000000003,"5min":2.8500000000000001,"15min":1.9850000000000001},"min":{"1min":0.375,"5min":0.33700000000000002,"15min":0.26200000000000001},"max":{"1min":14.789,"5min":17.155999999999999,"15min":17.155999999999999},"last":0.69299999999999995},{"interface":"front","average":{"1min":4.819,"5min":2.952,"15min":1.9179999999999999},"min":{"1min":0.46200000000000002,"5min":0.34799999999999998,"15min":0.27600000000000002},"max":{"1min":14.805,"5min":16.960000000000001,"15min":16.960000000000001},"last":0.66200000000000003}]},{"osd":4,"last update":"Sat Sep 23 20:35:11 2023","interfaces":[{"interface":"back","average":{"1min":1.248,"5min":0.81699999999999995,"15min":1.101},"min":{"1min":0.254,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":5.0460000000000003,"5min":5.8819999999999997,"15min":15.177},"last":0.68200000000000005},{"interface":"front","average":{"1min":1.389,"5min":0.91700000000000004,"15min":1.2350000000000001},"min":{"1min":0.32400000000000001,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":6.1779999999999999,"5min":6.1779999999999999,"15min":11.122},"last":0.80400000000000005}]},{"osd":5,"last update":"Sat Sep 23 20:35:11 2023","interfaces":[{"interface":"back","average":{"1min":0.71299999999999997,"5min":0.70299999999999996,"15min":1.145},"min":{"1min":0.27600000000000002,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":4.133,"5min":4.133,"15min":13.241},"last":0.74399999999999999},{"interface":"front","average":{"1min":0.90200000000000002,"5min":0.66100000000000003,"15min":1.2170000000000001},"min":{"1min":0.32400000000000001,"5min":0.32200000000000001,"15min":0.315},"max":{"1min":4.7590000000000003,"5min":5.7869999999999999,"15min":18.831},"last":0.72999999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:35:11 2023","interfaces":[{"interface":"back","average":{"1min":0.85299999999999998,"5min":0.67800000000000005,"15min":1.1160000000000001},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":4.819,"5min":5.7629999999999999,"15min":13.185},"last":0.77300000000000002},{"interface":"front","average":{"1min":1.2649999999999999,"5min":0.94299999999999995,"15min":1.137},"min":{"1min":0.35299999999999998,"5min":0.318,"15min":0.29899999999999999},"max":{"1min":6.2770000000000001,"5min":6.2770000000000001,"15min":9.9120000000000008},"last":0.76100000000000001}]}]},{"osd":6,"up_from":47,"seq":201863463215,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":475888,"kb_used_data":198248,"kb_used_omap":0,"kb_used_meta":277632,"kb_avail":93269264,"statfs":{"total":95995035648,"available":95507726336,"internally_reserved":0,"allocated":203005952,"data_stored":201856359,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":284295168},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:35:41 2023","interfaces":[{"interface":"back","average":{"1min":1.754,"5min":2.589,"15min":1.667},"min":{"1min":0.26200000000000001,"5min":0.22800000000000001,"15min":0.221},"max":{"1min":4.8959999999999999,"5min":53.701999999999998,"15min":53.701999999999998},"last":0.57099999999999995},{"interface":"front","average":{"1min":1.7290000000000001,"5min":2.9180000000000001,"15min":2.016},"min":{"1min":0.252,"5min":0.252,"15min":0.25},"max":{"1min":6.093,"5min":53.875999999999998,"15min":53.875999999999998},"last":0.44800000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:36:01 2023","interfaces":[{"interface":"back","average":{"1min":2.3340000000000001,"5min":2.1880000000000002,"15min":2.5630000000000002},"min":{"1min":0.309,"5min":0.249,"15min":0.249},"max":{"1min":10.331,"5min":34.173999999999999,"15min":34.173999999999999},"last":0.55700000000000005},{"interface":"front","average":{"1min":2.0539999999999998,"5min":2.282,"15min":3.085},"min":{"1min":0.35299999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":6.6020000000000003,"5min":34.490000000000002,"15min":34.490000000000002},"last":0.46999999999999997}]},{"osd":2,"last update":"Sat Sep 23 20:35:41 2023","interfaces":[{"interface":"back","average":{"1min":2.6259999999999999,"5min":2.512,"15min":1.9430000000000001},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":13.429,"5min":34.317999999999998,"15min":34.317999999999998},"last":0.497},{"interface":"front","average":{"1min":2.4649999999999999,"5min":2.5289999999999999,"15min":1.859},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":13.457000000000001,"5min":34.292999999999999,"15min":34.292999999999999},"last":0.63100000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:35:25 2023","interfaces":[{"interface":"back","average":{"1min":2.0099999999999998,"5min":2.0369999999999999,"15min":1.7050000000000001},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29599999999999999},"max":{"1min":6.46,"5min":7.5620000000000003,"15min":9.4239999999999995},"last":0.54000000000000004},{"interface":"front","average":{"1min":2.8290000000000002,"5min":2.4140000000000001,"15min":1.851},"min":{"1min":0.39400000000000002,"5min":0.317,"15min":0.317},"max":{"1min":8.5600000000000005,"5min":12.875999999999999,"15min":12.875999999999999},"last":0.64300000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:36:05 2023","interfaces":[{"interface":"back","average":{"1min":0.60499999999999998,"5min":0.61899999999999999,"15min":0.90400000000000003},"min":{"1min":0.26900000000000002,"5min":0.251,"15min":0.251},"max":{"1min":2.1600000000000001,"5min":4.9279999999999999,"15min":10.173},"last":0.59699999999999998},{"interface":"front","average":{"1min":1.236,"5min":0.88800000000000001,"15min":1.1100000000000001},"min":{"1min":0.35099999999999998,"5min":0.28199999999999997,"15min":0.27000000000000002},"max":{"1min":4.194,"5min":7.7629999999999999,"15min":8.3249999999999993},"last":0.58599999999999997}]},{"osd":5,"last update":"Sat Sep 23 20:36:05 2023","interfaces":[{"interface":"back","average":{"1min":0.879,"5min":0.747,"15min":0.97599999999999998},"min":{"1min":0.32800000000000001,"5min":0.30499999999999999,"15min":0.28699999999999998},"max":{"1min":2.8849999999999998,"5min":6.8310000000000004,"15min":14.085000000000001},"last":0.52400000000000002},{"interface":"front","average":{"1min":0.879,"5min":0.82199999999999995,"15min":1.329},"min":{"1min":0.35099999999999998,"5min":0.32000000000000001,"15min":0.30599999999999999},"max":{"1min":4.2489999999999997,"5min":7.7300000000000004,"15min":18.192},"last":0.61399999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:35:25 2023","interfaces":[{"interface":"back","average":{"1min":0.82599999999999996,"5min":0.69799999999999995,"15min":1.085},"min":{"1min":0.36599999999999999,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":4.851,"5min":5.0350000000000001,"15min":10.217000000000001},"last":0.65700000000000003},{"interface":"front","average":{"1min":0.93500000000000005,"5min":0.93999999999999995,"15min":1.2509999999999999},"min":{"1min":0.35999999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":5.6820000000000004,"5min":7.601,"15min":9.7360000000000007},"last":0.66700000000000004}]}]},{"osd":1,"up_from":266,"seq":1142461300814,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":278228,"kb_used_data":4748,"kb_used_omap":8,"kb_used_meta":273463,"kb_avail":93466924,"statfs":{"total":95995035648,"available":95710130176,"internally_reserved":0,"allocated":4861952,"data_stored":2871774,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8206,"internal_metadata":280027122},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":15,"apply_latency_ms":15,"commit_latency_ns":15000000,"apply_latency_ns":15000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:35:58 2023","interfaces":[{"interface":"back","average":{"1min":3.254,"5min":3.1600000000000001,"15min":2.6059999999999999},"min":{"1min":0.222,"5min":0.218,"15min":0.218},"max":{"1min":14.531000000000001,"5min":52.689999999999998,"15min":52.689999999999998},"last":0.379},{"interface":"front","average":{"1min":4.46,"5min":3.734,"15min":3.375},"min":{"1min":0.19900000000000001,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":19.41,"5min":52.662999999999997,"15min":52.662999999999997},"last":0.29799999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:35:58 2023","interfaces":[{"interface":"back","average":{"1min":4.4960000000000004,"5min":3.9430000000000001,"15min":4.7309999999999999},"min":{"1min":0.25900000000000001,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":14.474,"5min":28.359000000000002,"15min":28.359000000000002},"last":0.35699999999999998},{"interface":"front","average":{"1min":2.9969999999999999,"5min":2.7989999999999999,"15min":2.3290000000000002},"min":{"1min":0.26700000000000002,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":11.606999999999999,"5min":12.801,"15min":12.801},"last":0.44800000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:35:58 2023","interfaces":[{"interface":"back","average":{"1min":4.468,"5min":4.5,"15min":4.0780000000000003},"min":{"1min":0.308,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":14.359,"5min":80.745999999999995,"15min":80.745999999999995},"last":0.41399999999999998},{"interface":"front","average":{"1min":2.7360000000000002,"5min":4.2469999999999999,"15min":4.6459999999999999},"min":{"1min":0.35599999999999998,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":19.315000000000001,"5min":82.988,"15min":82.988},"last":0.34399999999999997}]},{"osd":4,"last update":"Sat Sep 23 20:35:58 2023","interfaces":[{"interface":"back","average":{"1min":3.4609999999999999,"5min":2.5750000000000002,"15min":2.2440000000000002},"min":{"1min":0.35599999999999998,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":9.4510000000000005,"5min":24.327999999999999,"15min":24.327999999999999},"last":0.49199999999999999},{"interface":"front","average":{"1min":2.4430000000000001,"5min":1.76,"15min":1.9359999999999999},"min":{"1min":0.38800000000000001,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":11.574999999999999,"5min":11.574999999999999,"15min":11.574999999999999},"last":0.48299999999999998}]},{"osd":5,"last update":"Sat Sep 23 20:35:58 2023","interfaces":[{"interface":"back","average":{"1min":2.0760000000000001,"5min":2.0310000000000001,"15min":2.5449999999999999},"min":{"1min":0.34100000000000003,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":6.4059999999999997,"5min":8.1950000000000003,"15min":9.6479999999999997},"last":0.42599999999999999},{"interface":"front","average":{"1min":3.5289999999999999,"5min":2.7690000000000001,"15min":2.3159999999999998},"min":{"1min":0.44800000000000001,"5min":0.36899999999999999,"15min":0.36799999999999999},"max":{"1min":9.5150000000000006,"5min":24.404,"15min":24.404},"last":0.55100000000000005}]},{"osd":6,"last update":"Sat Sep 23 20:35:58 2023","interfaces":[{"interface":"back","average":{"1min":1.79,"5min":1.974,"15min":5.6239999999999997},"min":{"1min":0.36899999999999999,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":5.6109999999999998,"5min":8.1620000000000008,"15min":72.682000000000002},"last":0.45700000000000002},{"interface":"front","average":{"1min":3.569,"5min":2.8439999999999999,"15min":5.2409999999999997},"min":{"1min":0.45800000000000002,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":9.4849999999999994,"5min":24.376000000000001,"15min":72.563999999999993},"last":0.59399999999999997}]},{"osd":7,"last update":"Sat Sep 23 20:35:58 2023","interfaces":[{"interface":"back","average":{"1min":2.6949999999999998,"5min":2.258,"15min":2.9409999999999998},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":18.437000000000001,"5min":18.437000000000001,"15min":18.437000000000001},"last":0.51300000000000001},{"interface":"front","average":{"1min":3.734,"5min":2.8980000000000001,"15min":2.5390000000000001},"min":{"1min":0.51500000000000001,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":12.021000000000001,"5min":24.524000000000001,"15min":24.524000000000001},"last":0.63400000000000001}]}]},{"osd":0,"up_from":94,"seq":403726926016,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":489316,"kb_used_data":4892,"kb_used_omap":7,"kb_used_meta":484408,"kb_avail":93255836,"statfs":{"total":95995035648,"available":95493976064,"internally_reserved":0,"allocated":5009408,"data_stored":3174174,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":496033809},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":11,"apply_latency_ms":11,"commit_latency_ns":11000000,"apply_latency_ns":11000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:36:03 2023","interfaces":[{"interface":"back","average":{"1min":2.6989999999999998,"5min":3.2810000000000001,"15min":3.6850000000000001},"min":{"1min":0.23000000000000001,"5min":0.183,"15min":0.17999999999999999},"max":{"1min":11.554,"5min":42.000999999999998,"15min":42.000999999999998},"last":0.254},{"interface":"front","average":{"1min":2.9199999999999999,"5min":3.278,"15min":4.0640000000000001},"min":{"1min":0.218,"5min":0.218,"15min":0.218},"max":{"1min":9.6059999999999999,"5min":23.036000000000001,"15min":23.036000000000001},"last":0.33500000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:35:37 2023","interfaces":[{"interface":"back","average":{"1min":3.5840000000000001,"5min":2.9820000000000002,"15min":2.4569999999999999},"min":{"1min":0.24099999999999999,"5min":0.193,"15min":0.185},"max":{"1min":11.064,"5min":22.460999999999999,"15min":77.448999999999998},"last":0.27300000000000002},{"interface":"front","average":{"1min":3.4489999999999998,"5min":2.8570000000000002,"15min":2.294},"min":{"1min":0.25700000000000001,"5min":0.188,"15min":0.188},"max":{"1min":12.071,"5min":22.260000000000002,"15min":81.724999999999994},"last":0.35199999999999998}]},{"osd":3,"last update":"Sat Sep 23 20:35:07 2023","interfaces":[{"interface":"back","average":{"1min":5.1449999999999996,"5min":4.0899999999999999,"15min":2.5219999999999998},"min":{"1min":0.27500000000000002,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":17.692,"5min":81.400999999999996,"15min":81.400999999999996},"last":0.36899999999999999},{"interface":"front","average":{"1min":5.085,"5min":4.7759999999999998,"15min":2.6200000000000001},"min":{"1min":0.30299999999999999,"5min":0.25700000000000001,"15min":0.23699999999999999},"max":{"1min":12.262,"5min":81.137,"15min":81.137},"last":0.29499999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:35:37 2023","interfaces":[{"interface":"back","average":{"1min":3.331,"5min":2.1749999999999998,"15min":1.9350000000000001},"min":{"1min":0.32000000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":11.068,"5min":12.587999999999999,"15min":23.876999999999999},"last":0.58799999999999997},{"interface":"front","average":{"1min":2.7650000000000001,"5min":2.347,"15min":1.8160000000000001},"min":{"1min":0.316,"5min":0.308,"15min":0.29299999999999998},"max":{"1min":11.458,"5min":23.001000000000001,"15min":23.681999999999999},"last":0.63400000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:35:37 2023","interfaces":[{"interface":"back","average":{"1min":2.5960000000000001,"5min":2.2360000000000002,"15min":2.129},"min":{"1min":0.35099999999999998,"5min":0.34399999999999997,"15min":0.34200000000000003},"max":{"1min":11.497999999999999,"5min":22.969999999999999,"15min":24.198},"last":0.65200000000000002},{"interface":"front","average":{"1min":2.8170000000000002,"5min":2.3199999999999998,"15min":1.9970000000000001},"min":{"1min":0.38300000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":11.974,"5min":22.236000000000001,"15min":22.236000000000001},"last":0.74299999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:35:37 2023","interfaces":[{"interface":"back","average":{"1min":3.1640000000000001,"5min":2.4590000000000001,"15min":1.923},"min":{"1min":0.38,"5min":0.32300000000000001,"15min":0.318},"max":{"1min":11.526999999999999,"5min":22.934999999999999,"15min":24.408000000000001},"last":0.60999999999999999},{"interface":"front","average":{"1min":2.8010000000000002,"5min":2.399,"15min":1.841},"min":{"1min":0.35799999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":12.015000000000001,"5min":22.164999999999999,"15min":25.271000000000001},"last":0.68400000000000005}]},{"osd":7,"last update":"Sat Sep 23 20:35:37 2023","interfaces":[{"interface":"back","average":{"1min":3.1899999999999999,"5min":2.5630000000000002,"15min":2.0459999999999998},"min":{"1min":0.40200000000000002,"5min":0.32200000000000001,"15min":0.30599999999999999},"max":{"1min":12.042,"5min":22.206,"15min":25.359000000000002},"last":0.71499999999999997},{"interface":"front","average":{"1min":3.9740000000000002,"5min":2.6230000000000002,"15min":2.3319999999999999},"min":{"1min":0.36199999999999999,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":11.112,"5min":22.422999999999998,"15min":24.669},"last":0.62}]}]},{"osd":2,"up_from":104,"seq":446676598964,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":493928,"kb_used_data":8928,"kb_used_omap":11,"kb_used_meta":484980,"kb_avail":93251224,"statfs":{"total":95995035648,"available":95489253376,"internally_reserved":0,"allocated":9142272,"data_stored":7307758,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11878,"internal_metadata":496619930},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":10,"apply_latency_ms":10,"commit_latency_ns":10000000,"apply_latency_ns":10000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:35:47 2023","interfaces":[{"interface":"back","average":{"1min":3.944,"5min":4.274,"15min":2.5819999999999999},"min":{"1min":0.25800000000000001,"5min":0.185,"15min":0.17999999999999999},"max":{"1min":14.124000000000001,"5min":82.424999999999997,"15min":82.424999999999997},"last":0.59899999999999998},{"interface":"front","average":{"1min":4.4859999999999998,"5min":3.7890000000000001,"15min":2.3460000000000001},"min":{"1min":0.24199999999999999,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":14.055,"5min":82.524000000000001,"15min":82.524000000000001},"last":0.53100000000000003}]},{"osd":1,"last update":"Sat Sep 23 20:35:53 2023","interfaces":[{"interface":"back","average":{"1min":3.5960000000000001,"5min":2.819,"15min":4.173},"min":{"1min":0.29299999999999998,"5min":0.22900000000000001,"15min":0.221},"max":{"1min":11.227,"5min":15.956,"15min":16.280000000000001},"last":0.54900000000000004},{"interface":"front","average":{"1min":5.9560000000000004,"5min":3.3959999999999999,"15min":3.1240000000000001},"min":{"1min":0.46400000000000002,"5min":0.248,"15min":0.248},"max":{"1min":21.585999999999999,"5min":21.585999999999999,"15min":30.565000000000001},"last":0.50600000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:35:13 2023","interfaces":[{"interface":"back","average":{"1min":2.6520000000000001,"5min":2.9529999999999998,"15min":1.905},"min":{"1min":0.32700000000000001,"5min":0.27800000000000002,"15min":0.19800000000000001},"max":{"1min":12.664,"5min":15.804,"15min":16.331},"last":0.49199999999999999},{"interface":"front","average":{"1min":4.577,"5min":3.5139999999999998,"15min":2.242},"min":{"1min":0.33400000000000002,"5min":0.27300000000000002,"15min":0.23899999999999999},"max":{"1min":13.157,"5min":30.704999999999998,"15min":30.704999999999998},"last":0.56899999999999995}]},{"osd":4,"last update":"Sat Sep 23 20:35:47 2023","interfaces":[{"interface":"back","average":{"1min":3.3730000000000002,"5min":2.1440000000000001,"15min":1.754},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":9.2430000000000003,"5min":9.2430000000000003,"15min":16.797999999999998},"last":0.91700000000000004},{"interface":"front","average":{"1min":3.1819999999999999,"5min":2.5110000000000001,"15min":1.903},"min":{"1min":0.307,"5min":0.30499999999999999,"15min":0.27600000000000002},"max":{"1min":11.044,"5min":15.853999999999999,"15min":15.853999999999999},"last":0.747}]},{"osd":5,"last update":"Sat Sep 23 20:35:47 2023","interfaces":[{"interface":"back","average":{"1min":3.3690000000000002,"5min":2.2069999999999999,"15min":2.2869999999999999},"min":{"1min":0.36599999999999999,"5min":0.314,"15min":0.309},"max":{"1min":9.1799999999999997,"5min":9.1799999999999997,"15min":95.393000000000001},"last":0.98399999999999999},{"interface":"front","average":{"1min":3.1920000000000002,"5min":2.77,"15min":2.5049999999999999},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.32300000000000001},"max":{"1min":10.856999999999999,"5min":15.885,"15min":95.414000000000001},"last":0.999}]},{"osd":6,"last update":"Sat Sep 23 20:35:47 2023","interfaces":[{"interface":"back","average":{"1min":3.0680000000000001,"5min":2.6560000000000001,"15min":1.9890000000000001},"min":{"1min":0.37,"5min":0.35799999999999998,"15min":0.32100000000000001},"max":{"1min":11.253,"5min":15.914,"15min":15.914},"last":0.71099999999999997},{"interface":"front","average":{"1min":2.968,"5min":2.3570000000000002,"15min":1.9670000000000001},"min":{"1min":0.49099999999999999,"5min":0.318,"15min":0.316},"max":{"1min":10.811,"5min":11.523,"15min":30.670999999999999},"last":0.90400000000000003}]},{"osd":7,"last update":"Sat Sep 23 20:35:47 2023","interfaces":[{"interface":"back","average":{"1min":2.7730000000000001,"5min":2.306,"15min":2.0259999999999998},"min":{"1min":0.439,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":10.231,"5min":12.734999999999999,"15min":30.741},"last":0.69399999999999995},{"interface":"front","average":{"1min":3.2250000000000001,"5min":2.3170000000000002,"15min":2.0089999999999999},"min":{"1min":0.42999999999999999,"5min":0.38,"15min":0.38},"max":{"1min":9.2720000000000002,"5min":10.861000000000001,"15min":16.719000000000001},"last":0.95199999999999996}]}]},{"osd":3,"up_from":109,"seq":468151435438,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":557024,"kb_used_data":4888,"kb_used_omap":4,"kb_used_meta":552123,"kb_avail":93188128,"statfs":{"total":95995035648,"available":95424643072,"internally_reserved":0,"allocated":5005312,"data_stored":3176596,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4514,"internal_metadata":565374558},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":21,"apply_latency_ms":21,"commit_latency_ns":21000000,"apply_latency_ns":21000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:35:11 2023","interfaces":[{"interface":"back","average":{"1min":3.742,"5min":3.242,"15min":2.2770000000000001},"min":{"1min":0.215,"5min":0.191,"15min":0.186},"max":{"1min":17.007000000000001,"5min":17.129000000000001,"15min":67.305999999999997},"last":0.40899999999999997},{"interface":"front","average":{"1min":3.3140000000000001,"5min":3.0550000000000002,"15min":2.1190000000000002},"min":{"1min":0.23699999999999999,"5min":0.19500000000000001,"15min":0.19500000000000001},"max":{"1min":9.5380000000000003,"5min":12.781000000000001,"15min":67.403999999999996},"last":0.34999999999999998}]},{"osd":1,"last update":"Sat Sep 23 20:35:56 2023","interfaces":[{"interface":"back","average":{"1min":1.7689999999999999,"5min":2.456,"15min":2.8769999999999998},"min":{"1min":0.251,"5min":0.222,"15min":0.222},"max":{"1min":9.8940000000000001,"5min":16.189,"15min":16.189},"last":0.374},{"interface":"front","average":{"1min":2.9140000000000001,"5min":2.5619999999999998,"15min":3.0219999999999998},"min":{"1min":0.34599999999999997,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":13.727,"5min":32.603999999999999,"15min":32.603999999999999},"last":0.39100000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:35:11 2023","interfaces":[{"interface":"back","average":{"1min":2.883,"5min":3.4980000000000002,"15min":2.2589999999999999},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.22800000000000001},"max":{"1min":9.5879999999999992,"5min":32.857999999999997,"15min":32.857999999999997},"last":0.432},{"interface":"front","average":{"1min":3.4369999999999998,"5min":3.1699999999999999,"15min":2.165},"min":{"1min":0.251,"5min":0.246,"15min":0.20899999999999999},"max":{"1min":14.492000000000001,"5min":24.146999999999998,"15min":24.146999999999998},"last":0.32900000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:35:11 2023","interfaces":[{"interface":"back","average":{"1min":1.5669999999999999,"5min":1.9750000000000001,"15min":1.698},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.29799999999999999},"max":{"1min":5.3929999999999998,"5min":12.151,"15min":46.042999999999999},"last":0.52800000000000002},{"interface":"front","average":{"1min":2.6949999999999998,"5min":2.6339999999999999,"15min":2.0579999999999998},"min":{"1min":0.308,"5min":0.29399999999999998,"15min":0.28100000000000003},"max":{"1min":14.422000000000001,"5min":19.635000000000002,"15min":46.579999999999998},"last":0.44400000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:35:11 2023","interfaces":[{"interface":"back","average":{"1min":2.056,"5min":2.2120000000000002,"15min":1.9399999999999999},"min":{"1min":0.36699999999999999,"5min":0.32200000000000001,"15min":0.311},"max":{"1min":13.228999999999999,"5min":13.228999999999999,"15min":45.292000000000002},"last":0.46700000000000003},{"interface":"front","average":{"1min":1.647,"5min":2.1190000000000002,"15min":1.764},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.313},"max":{"1min":8.782,"5min":11.028,"15min":44.679000000000002},"last":0.55800000000000005}]},{"osd":6,"last update":"Sat Sep 23 20:35:11 2023","interfaces":[{"interface":"back","average":{"1min":1.4590000000000001,"5min":2.266,"15min":1.79},"min":{"1min":0.40500000000000003,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":5.2640000000000002,"5min":17.524999999999999,"15min":17.524999999999999},"last":0.50700000000000001},{"interface":"front","average":{"1min":3.1150000000000002,"5min":2.4180000000000001,"15min":1.8100000000000001},"min":{"1min":0.41299999999999998,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":14.464,"5min":19.524000000000001,"15min":19.524000000000001},"last":0.626}]},{"osd":7,"last update":"Sat Sep 23 20:35:11 2023","interfaces":[{"interface":"back","average":{"1min":2.9740000000000002,"5min":2.6230000000000002,"15min":1.964},"min":{"1min":0.44700000000000001,"5min":0.36899999999999999,"15min":0.34999999999999998},"max":{"1min":14.521000000000001,"5min":19.571999999999999,"15min":19.571999999999999},"last":0.49399999999999999},{"interface":"front","average":{"1min":2.016,"5min":2.149,"15min":1.998},"min":{"1min":0.44600000000000001,"5min":0.42599999999999999,"15min":0.371},"max":{"1min":8.3360000000000003,"5min":12.186,"15min":45.387999999999998},"last":0.60499999999999998}]}]},{"osd":4,"up_from":30,"seq":128849019191,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":264048,"kb_used_data":3816,"kb_used_omap":0,"kb_used_meta":260224,"kb_avail":93481104,"statfs":{"total":95995035648,"available":95724650496,"internally_reserved":0,"allocated":3907584,"data_stored":2757725,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":266469376},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:35:40 2023","interfaces":[{"interface":"back","average":{"1min":2.2799999999999998,"5min":1.6579999999999999,"15min":1.5229999999999999},"min":{"1min":0.27300000000000002,"5min":0.255,"15min":0.222},"max":{"1min":5.2359999999999998,"5min":8.0800000000000001,"15min":20.411999999999999},"last":0.27000000000000002},{"interface":"front","average":{"1min":2.3849999999999998,"5min":1.835,"15min":1.6739999999999999},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.253},"max":{"1min":5.2229999999999999,"5min":10.675000000000001,"15min":20.454999999999998},"last":0.34899999999999998}]},{"osd":1,"last update":"Sat Sep 23 20:36:01 2023","interfaces":[{"interface":"back","average":{"1min":1.5649999999999999,"5min":1.7969999999999999,"15min":1.9370000000000001},"min":{"1min":0.32000000000000001,"5min":0.27600000000000002,"15min":0.27300000000000002},"max":{"1min":7.8600000000000003,"5min":11.303000000000001,"15min":11.303000000000001},"last":0.503},{"interface":"front","average":{"1min":1.7809999999999999,"5min":1.583,"15min":1.871},"min":{"1min":0.30099999999999999,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":10.454000000000001,"5min":11.287000000000001,"15min":11.287000000000001},"last":0.53800000000000003}]},{"osd":2,"last update":"Sat Sep 23 20:35:53 2023","interfaces":[{"interface":"back","average":{"1min":2.702,"5min":1.9570000000000001,"15min":1.798},"min":{"1min":0.32300000000000001,"5min":0.311,"15min":0.26600000000000001},"max":{"1min":10.416,"5min":10.416,"15min":32.363},"last":0.41399999999999998},{"interface":"front","average":{"1min":2.9470000000000001,"5min":2.21,"15min":2.0950000000000002},"min":{"1min":0.41999999999999998,"5min":0.28999999999999998,"15min":0.26800000000000002},"max":{"1min":8.4939999999999998,"5min":14.614000000000001,"15min":34.042999999999999},"last":0.52300000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:35:09 2023","interfaces":[{"interface":"back","average":{"1min":2.085,"5min":2.2839999999999998,"15min":1.849},"min":{"1min":0.34899999999999998,"5min":0.34799999999999998,"15min":0.29099999999999998},"max":{"1min":7.2060000000000004,"5min":14.515000000000001,"15min":32.332999999999998},"last":0.372},{"interface":"front","average":{"1min":2.5150000000000001,"5min":2.2599999999999998,"15min":1.9379999999999999},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":7.9210000000000003,"5min":15.991,"15min":32.399999999999999},"last":0.48999999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:35:32 2023","interfaces":[{"interface":"back","average":{"1min":0.67800000000000005,"5min":0.72199999999999998,"15min":1.5409999999999999},"min":{"1min":0.26800000000000002,"5min":0.25800000000000001,"15min":0.247},"max":{"1min":2.9369999999999998,"5min":5.4340000000000002,"15min":82.866},"last":0.65000000000000002},{"interface":"front","average":{"1min":0.61799999999999999,"5min":0.79400000000000004,"15min":1.4370000000000001},"min":{"1min":0.33700000000000002,"5min":0.30299999999999999,"15min":0.27800000000000002},"max":{"1min":2.7509999999999999,"5min":6.8520000000000003,"15min":82.546000000000006},"last":0.39400000000000002}]},{"osd":6,"last update":"Sat Sep 23 20:35:53 2023","interfaces":[{"interface":"back","average":{"1min":1.018,"5min":1.028,"15min":1.389},"min":{"1min":0.312,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":3.008,"5min":10.178000000000001,"15min":41.371000000000002},"last":0.44700000000000001},{"interface":"front","average":{"1min":1.097,"5min":0.85099999999999998,"15min":1.27},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.27200000000000002},"max":{"1min":4.2949999999999999,"5min":5.4619999999999997,"15min":41.140999999999998},"last":0.55000000000000004}]},{"osd":7,"last update":"Sat Sep 23 20:35:09 2023","interfaces":[{"interface":"back","average":{"1min":0.86499999999999999,"5min":1.0880000000000001,"15min":1.425},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.33600000000000002},"max":{"1min":3.2970000000000002,"5min":6.8849999999999998,"15min":16.427},"last":0.61299999999999999},{"interface":"front","average":{"1min":0.65200000000000002,"5min":0.81599999999999995,"15min":1.2989999999999999},"min":{"1min":0.36399999999999999,"5min":0.33000000000000002,"15min":0.307},"max":{"1min":1.4490000000000001,"5min":6.3970000000000002,"15min":19.018999999999998},"last":0.59799999999999998}]}]},{"osd":5,"up_from":39,"seq":167503724852,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":262116,"kb_used_data":4060,"kb_used_omap":0,"kb_used_meta":258048,"kb_avail":93483036,"statfs":{"total":95995035648,"available":95726628864,"internally_reserved":0,"allocated":4157440,"data_stored":3008339,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":264241152},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:35:42 2023","interfaces":[{"interface":"back","average":{"1min":4.7649999999999997,"5min":2.125,"15min":1.6919999999999999},"min":{"1min":0.29599999999999999,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":18.16,"5min":18.16,"15min":32.475000000000001},"last":0.38700000000000001},{"interface":"front","average":{"1min":3.3809999999999998,"5min":1.837,"15min":1.615},"min":{"1min":0.33900000000000002,"5min":0.26300000000000001,"15min":0.20599999999999999},"max":{"1min":9.0570000000000004,"5min":9.0570000000000004,"15min":32.457000000000001},"last":0.46800000000000003}]},{"osd":1,"last update":"Sat Sep 23 20:35:59 2023","interfaces":[{"interface":"back","average":{"1min":1.675,"5min":1.6180000000000001,"15min":1.909},"min":{"1min":0.26300000000000001,"5min":0.253,"15min":0.253},"max":{"1min":4.5910000000000002,"5min":9.3539999999999992,"15min":9.3539999999999992},"last":0.60299999999999998},{"interface":"front","average":{"1min":3.375,"5min":1.879,"15min":1.667},"min":{"1min":0.41899999999999998,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":8.4220000000000006,"5min":9.3900000000000006,"15min":9.3900000000000006},"last":0.42699999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:35:42 2023","interfaces":[{"interface":"back","average":{"1min":2.4020000000000001,"5min":2.1070000000000002,"15min":1.8440000000000001},"min":{"1min":0.32300000000000001,"5min":0.27400000000000002,"15min":0.26700000000000002},"max":{"1min":11.347,"5min":27.603999999999999,"15min":34.514000000000003},"last":0.45800000000000002},{"interface":"front","average":{"1min":2.4159999999999999,"5min":2.4060000000000001,"15min":1.8720000000000001},"min":{"1min":0.34100000000000003,"5min":0.255,"15min":0.255},"max":{"1min":13.545,"5min":27.626999999999999,"15min":32.515000000000001},"last":0.502}]},{"osd":3,"last update":"Sat Sep 23 20:35:09 2023","interfaces":[{"interface":"back","average":{"1min":2.2130000000000001,"5min":2.5499999999999998,"15min":1.8149999999999999},"min":{"1min":0.314,"5min":0.29599999999999999,"15min":0.25700000000000001},"max":{"1min":9.4770000000000003,"5min":16.189,"15min":16.189},"last":0.67800000000000005},{"interface":"front","average":{"1min":3.7400000000000002,"5min":2.2749999999999999,"15min":1.669},"min":{"1min":0.36099999999999999,"5min":0.32300000000000001,"15min":0.28799999999999998},"max":{"1min":10.191000000000001,"5min":10.577999999999999,"15min":12.005000000000001},"last":0.48299999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:35:33 2023","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.67800000000000005,"15min":1.2989999999999999},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.25800000000000001},"max":{"1min":2.2280000000000002,"5min":6.282,"15min":28.015999999999998},"last":0.73699999999999999},{"interface":"front","average":{"1min":0.84899999999999998,"5min":0.69899999999999995,"15min":1.087},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":3.0659999999999998,"5min":3.0659999999999998,"15min":12.300000000000001},"last":0.623}]},{"osd":6,"last update":"Sat Sep 23 20:35:48 2023","interfaces":[{"interface":"back","average":{"1min":0.77500000000000002,"5min":0.70699999999999996,"15min":1.2190000000000001},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":4.3300000000000001,"5min":4.3330000000000002,"15min":12.890000000000001},"last":0.64600000000000002},{"interface":"front","average":{"1min":0.83699999999999997,"5min":0.73799999999999999,"15min":1.1879999999999999},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.34499999999999997},"max":{"1min":4.2400000000000002,"5min":5.1680000000000001,"15min":16.265999999999998},"last":0.57499999999999996}]},{"osd":7,"last update":"Sat Sep 23 20:35:09 2023","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.68999999999999995,"15min":1.226},"min":{"1min":0.36399999999999999,"5min":0.32100000000000001,"15min":0.29699999999999999},"max":{"1min":1.946,"5min":2.77,"15min":36.621000000000002},"last":0.65900000000000003},{"interface":"front","average":{"1min":0.76100000000000001,"5min":0.72799999999999998,"15min":1.1359999999999999},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":5.093,"5min":5.093,"15min":37.331000000000003},"last":0.56000000000000005}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4722,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1360,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8206,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7156,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3154,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:36:12.358 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:36:12.388 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:12.421 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:36:12.423 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:12 smithi018 ceph-mon[184144]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:36:12.423 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:12 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:36:12.423 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:12 smithi018 ceph-mon[184144]: osdmap e370: 8 total, 8 up, 8 in 2023-09-23T20:36:12.423 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:12 smithi018 ceph-mon[184144]: failed to encode map e370 with expected crc 2023-09-23T20:36:12.423 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:12 smithi018 ceph-mon[184144]: failed to encode map e370 with expected crc 2023-09-23T20:36:12.424 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:12 smithi018 ceph-mon[184144]: failed to encode map e370 with expected crc 2023-09-23T20:36:12.424 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:12 smithi018 ceph-mon[184144]: failed to encode map e370 with expected crc 2023-09-23T20:36:12.424 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:12 smithi018 ceph-mon[179647]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:36:12.424 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:12 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:36:12.424 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:12 smithi018 ceph-mon[179647]: osdmap e370: 8 total, 8 up, 8 in 2023-09-23T20:36:12.424 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:12 smithi018 ceph-mon[179647]: failed to encode map e370 with expected crc 2023-09-23T20:36:12.424 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:12 smithi018 ceph-mon[179647]: failed to encode map e370 with expected crc 2023-09-23T20:36:12.424 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:12 smithi018 ceph-mon[179647]: failed to encode map e370 with expected crc 2023-09-23T20:36:12.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:12 smithi018 ceph-mon[179647]: failed to encode map e370 with expected crc 2023-09-23T20:36:12.425 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:12.425 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:36:12.526 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:36:12.555 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:12 smithi149 ceph-mon[170394]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:36:12.555 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:12 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:36:12.555 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:12 smithi149 ceph-mon[170394]: osdmap e370: 8 total, 8 up, 8 in 2023-09-23T20:36:12.555 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:12 smithi149 ceph-mon[170394]: failed to encode map e370 with expected crc 2023-09-23T20:36:12.555 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:12 smithi149 ceph-mon[170394]: failed to encode map e370 with expected crc 2023-09-23T20:36:12.555 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:12 smithi149 ceph-mon[170394]: failed to encode map e370 with expected crc 2023-09-23T20:36:12.556 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:12 smithi149 ceph-mon[170394]: failed to encode map e370 with expected crc 2023-09-23T20:36:12.558 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:12.595 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:36:12.598 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:12.598 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:36:12.699 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:36:12.728 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:12.747 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:36:12.762 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:36:12.764 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:12.765 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:36:12.786 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T20:36:12.865 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:36:12.893 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:12.927 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:36:12.929 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:12.930 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:36:13.031 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:36:13.058 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:13.092 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:36:13.094 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:13.095 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:36:13.196 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:36:13.222 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:13.254 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:36:13.257 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:13.258 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:36:13.358 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:36:13.388 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:13.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:13 smithi149 ceph-mon[170394]: pgmap v889: 119 pgs: 11 creating+peering, 5 unknown, 103 active+clean; 4.6 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-09-23T20:36:13.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:13 smithi149 ceph-mon[170394]: osdmap e371: 8 total, 8 up, 8 in 2023-09-23T20:36:13.474 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:36:13.476 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:13 smithi018 ceph-mon[184144]: pgmap v889: 119 pgs: 11 creating+peering, 5 unknown, 103 active+clean; 4.6 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-09-23T20:36:13.476 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:13 smithi018 ceph-mon[184144]: osdmap e371: 8 total, 8 up, 8 in 2023-09-23T20:36:13.477 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:13 smithi018 ceph-mon[179647]: pgmap v889: 119 pgs: 11 creating+peering, 5 unknown, 103 active+clean; 4.6 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-09-23T20:36:13.477 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:13 smithi018 ceph-mon[179647]: osdmap e371: 8 total, 8 up, 8 in 2023-09-23T20:36:13.477 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:13.478 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:36:13.578 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:36:13.609 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:13.666 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:36:13.668 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:36:13.668 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:13.669 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:36:13.770 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:36:13.784 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:36:13.801 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:13.893 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:36:13.896 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:13.897 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:36:13.906 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T20:36:13.998 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:36:14.028 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:14.063 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:36:14.066 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:14.067 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:36:14.168 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:36:14.198 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:14.306 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:14 smithi018 ceph-mon[184144]: failed to encode map e371 with expected crc 2023-09-23T20:36:14.307 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:14 smithi018 ceph-mon[179647]: failed to encode map e371 with expected crc 2023-09-23T20:36:14.307 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:14 smithi018 ceph-mon[179647]: failed to encode map e371 with expected crc 2023-09-23T20:36:14.307 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:14 smithi018 ceph-mon[179647]: failed to encode map e371 with expected crc 2023-09-23T20:36:14.308 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:14 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/4190821611' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T20:36:14.313 INFO:teuthology.orchestra.run.smithi018.stderr:enabled application 'rados' on pool 'unique_pool_3' 2023-09-23T20:36:14.490 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:36:14.494 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:14.494 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:36:14.598 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:36:14.624 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:14 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T20:36:14.624 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:14 smithi018 ceph-mon[184144]: failed to encode map e371 with expected crc 2023-09-23T20:36:14.624 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:14 smithi018 ceph-mon[184144]: failed to encode map e371 with expected crc 2023-09-23T20:36:14.625 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:14 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/4190821611' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T20:36:14.625 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:14 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T20:36:14.627 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:14.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:14 smithi149 ceph-mon[170394]: failed to encode map e371 with expected crc 2023-09-23T20:36:14.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:14 smithi149 ceph-mon[170394]: failed to encode map e371 with expected crc 2023-09-23T20:36:14.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:14 smithi149 ceph-mon[170394]: failed to encode map e371 with expected crc 2023-09-23T20:36:14.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:14 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/4190821611' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T20:36:14.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:14 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T20:36:14.907 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:36:14.910 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:14.910 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:36:15.012 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:36:15.019 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:36:15.043 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:15.095 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:36:15.141 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:36:14 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:36:14] "GET /metrics HTTP/1.1" 200 36435 "" "Prometheus/2.43.0" 2023-09-23T20:36:15.212 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:36:15.215 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:15.216 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:36:15.316 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:36:15.344 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2023-09-23T20:36:15.344 INFO:teuthology.orchestra.run.smithi018.stdout: "size": 20, 2023-09-23T20:36:15.344 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 600, 2023-09-23T20:36:15.344 INFO:teuthology.orchestra.run.smithi018.stdout: "ops": [ 2023-09-23T20:36:15.344 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.344 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:57603 7.4 7:280a08d4:::benchmark_data_smithi018_308601_object57602:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e311)", 2023-09-23T20:36:15.344 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:32:32.217458+0000", 2023-09-23T20:36:15.345 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 223.123106245, 2023-09-23T20:36:15.345 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.4379806410000002, 2023-09-23T20:36:15.345 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:36:15.345 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:36:15.345 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:36:15.345 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:36:15.345 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:36:15.345 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 57603 2023-09-23T20:36:15.345 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.345 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:36:15.345 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.346 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:36:15.346 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.217458+0000", 2023-09-23T20:36:15.346 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:36:15.346 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.346 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.346 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:36:15.346 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.217458+0000", 2023-09-23T20:36:15.346 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:36:15.346 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.346 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.346 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:36:15.347 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.217457+0000", 2023-09-23T20:36:15.347 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:36:15.347 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.347 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.347 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:36:15.347 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.217483+0000", 2023-09-23T20:36:15.347 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.5765000000000001e-05 2023-09-23T20:36:15.347 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.347 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.347 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:36:15.347 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.217492+0000", 2023-09-23T20:36:15.348 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.4179999999999997e-06 2023-09-23T20:36:15.348 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.348 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.348 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:36:15.348 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.217499+0000", 2023-09-23T20:36:15.348 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.9210000000000003e-06 2023-09-23T20:36:15.348 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.348 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.348 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:15.348 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.221000+0000", 2023-09-23T20:36:15.348 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0035019550000000002 2023-09-23T20:36:15.348 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.349 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.349 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:36:15.349 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.221153+0000", 2023-09-23T20:36:15.349 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00015237900000000001 2023-09-23T20:36:15.349 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.349 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.349 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,6", 2023-09-23T20:36:15.349 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.221299+0000", 2023-09-23T20:36:15.349 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00014614399999999999 2023-09-23T20:36:15.349 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.349 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.350 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:36:15.350 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.223762+0000", 2023-09-23T20:36:15.350 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0024632769999999998 2023-09-23T20:36:15.350 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.350 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.350 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:15.350 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.234020+0000", 2023-09-23T20:36:15.350 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.010257943 2023-09-23T20:36:15.350 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.350 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.350 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:36:15.351 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.234037+0000", 2023-09-23T20:36:15.351 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.6432000000000001e-05 2023-09-23T20:36:15.351 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.351 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.351 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:15.351 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.213974+0000", 2023-09-23T20:36:15.351 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.97993717700000005 2023-09-23T20:36:15.351 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.351 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.351 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:36:15.351 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.213988+0000", 2023-09-23T20:36:15.352 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.4001e-05 2023-09-23T20:36:15.352 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.352 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.352 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:15.352 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.634540+0000", 2023-09-23T20:36:15.352 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.4205522720000001 2023-09-23T20:36:15.352 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.352 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.352 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for degraded object", 2023-09-23T20:36:15.352 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.635049+0000", 2023-09-23T20:36:15.352 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00050854999999999997 2023-09-23T20:36:15.352 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.353 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.353 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:15.353 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.655344+0000", 2023-09-23T20:36:15.353 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.020295167999999999 2023-09-23T20:36:15.353 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.353 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.353 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:36:15.353 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.655439+0000", 2023-09-23T20:36:15.353 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.4845000000000004e-05 2023-09-23T20:36:15.353 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:15.353 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:36:15.353 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:15.354 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.354 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.354 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:57616 7.4 7:26fa0786:::benchmark_data_smithi018_308601_object57615:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e312)", 2023-09-23T20:36:15.354 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:32:32.242374+0000", 2023-09-23T20:36:15.354 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 223.09819063899999, 2023-09-23T20:36:15.354 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.4160429900000002, 2023-09-23T20:36:15.354 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:36:15.354 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:36:15.354 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:36:15.354 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:36:15.354 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:36:15.355 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 57616 2023-09-23T20:36:15.355 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.355 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:36:15.355 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.355 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:36:15.355 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.242374+0000", 2023-09-23T20:36:15.355 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:36:15.355 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.355 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.355 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:36:15.355 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.242374+0000", 2023-09-23T20:36:15.355 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:36:15.356 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.356 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.356 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:36:15.356 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.242372+0000", 2023-09-23T20:36:15.356 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:36:15.356 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.356 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.356 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:36:15.356 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.242398+0000", 2023-09-23T20:36:15.357 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.6188000000000001e-05 2023-09-23T20:36:15.357 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.357 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.357 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:36:15.357 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.242753+0000", 2023-09-23T20:36:15.357 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000354607 2023-09-23T20:36:15.357 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.357 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.357 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:36:15.357 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.242764+0000", 2023-09-23T20:36:15.357 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1443999999999999e-05 2023-09-23T20:36:15.358 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.358 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.358 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:15.358 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.242827+0000", 2023-09-23T20:36:15.358 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.3237000000000005e-05 2023-09-23T20:36:15.358 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.358 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.358 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:36:15.358 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.242840+0000", 2023-09-23T20:36:15.358 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3046e-05 2023-09-23T20:36:15.358 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.359 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.359 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:15.359 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.214015+0000", 2023-09-23T20:36:15.359 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.97117513 2023-09-23T20:36:15.359 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.359 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.359 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:36:15.359 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.214024+0000", 2023-09-23T20:36:15.359 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.7690000000000002e-06 2023-09-23T20:36:15.359 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.359 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.359 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:15.360 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.635115+0000", 2023-09-23T20:36:15.360 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.4210909899999999 2023-09-23T20:36:15.360 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.360 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.360 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:36:15.360 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.635297+0000", 2023-09-23T20:36:15.360 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00018154400000000001 2023-09-23T20:36:15.360 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.360 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.360 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,4", 2023-09-23T20:36:15.360 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.635493+0000", 2023-09-23T20:36:15.361 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00019634200000000001 2023-09-23T20:36:15.361 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.361 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.361 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:36:15.361 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.640427+0000", 2023-09-23T20:36:15.361 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0049342559999999997 2023-09-23T20:36:15.361 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.362 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.362 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:36:15.362 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.655477+0000", 2023-09-23T20:36:15.362 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.015049794 2023-09-23T20:36:15.362 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.362 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.363 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:36:15.363 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.655547+0000", 2023-09-23T20:36:15.363 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.9598999999999996e-05 2023-09-23T20:36:15.363 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.363 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.363 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:36:15.363 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.655597+0000", 2023-09-23T20:36:15.363 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.0489999999999999e-05 2023-09-23T20:36:15.363 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.363 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.363 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:36:15.364 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.658417+0000", 2023-09-23T20:36:15.364 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0028193939999999998 2023-09-23T20:36:15.364 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:15.364 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:36:15.364 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:15.364 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.364 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.364 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:57660 7.9 7:90d29d10:::benchmark_data_smithi018_308601_object57659:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e312)", 2023-09-23T20:36:15.364 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:32:32.288586+0000", 2023-09-23T20:36:15.364 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 223.05197858099999, 2023-09-23T20:36:15.365 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.3662949609999999, 2023-09-23T20:36:15.365 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:36:15.365 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:36:15.365 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:36:15.365 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:36:15.365 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:36:15.365 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 57660 2023-09-23T20:36:15.365 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.365 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:36:15.365 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.365 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:36:15.366 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.288586+0000", 2023-09-23T20:36:15.366 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:36:15.366 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.366 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.366 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:36:15.366 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.288586+0000", 2023-09-23T20:36:15.366 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:36:15.366 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.366 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.366 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:36:15.366 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.288584+0000", 2023-09-23T20:36:15.367 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:36:15.367 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.367 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.367 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:36:15.367 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.288608+0000", 2023-09-23T20:36:15.367 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.3784000000000001e-05 2023-09-23T20:36:15.367 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.367 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.367 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:36:15.367 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.288644+0000", 2023-09-23T20:36:15.367 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.5930000000000003e-05 2023-09-23T20:36:15.367 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.368 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.368 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:36:15.368 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.288656+0000", 2023-09-23T20:36:15.368 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1977999999999999e-05 2023-09-23T20:36:15.368 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.368 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.368 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:15.368 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.290685+0000", 2023-09-23T20:36:15.368 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.002029552 2023-09-23T20:36:15.368 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.368 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.368 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:36:15.369 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.290697+0000", 2023-09-23T20:36:15.369 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2155e-05 2023-09-23T20:36:15.369 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.369 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.369 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:15.369 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.219112+0000", 2023-09-23T20:36:15.369 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.92841424500000003 2023-09-23T20:36:15.369 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.369 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.369 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:36:15.369 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.219129+0000", 2023-09-23T20:36:15.370 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.7359000000000001e-05 2023-09-23T20:36:15.370 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.370 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.370 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:15.370 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.630777+0000", 2023-09-23T20:36:15.370 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.4116482610000001 2023-09-23T20:36:15.370 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.370 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.370 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:36:15.370 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.630922+0000", 2023-09-23T20:36:15.371 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000144912 2023-09-23T20:36:15.371 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.371 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.371 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,4", 2023-09-23T20:36:15.371 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.631075+0000", 2023-09-23T20:36:15.371 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00015256699999999999 2023-09-23T20:36:15.371 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.371 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.371 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:36:15.371 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.639027+0000", 2023-09-23T20:36:15.372 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0079517019999999997 2023-09-23T20:36:15.372 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.372 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.372 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:36:15.372 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.646083+0000", 2023-09-23T20:36:15.372 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0070564820000000002 2023-09-23T20:36:15.372 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.372 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.372 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:36:15.372 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.651941+0000", 2023-09-23T20:36:15.372 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0058583680000000001 2023-09-23T20:36:15.372 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.373 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.373 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:36:15.373 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.652009+0000", 2023-09-23T20:36:15.373 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.7535999999999998e-05 2023-09-23T20:36:15.373 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.373 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.373 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:36:15.373 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.654881+0000", 2023-09-23T20:36:15.373 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0028717500000000002 2023-09-23T20:36:15.373 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:15.373 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:36:15.374 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:15.374 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.374 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.374 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:57665 7.4 7:22060733:::benchmark_data_smithi018_308601_object57664:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e312)", 2023-09-23T20:36:15.374 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:32:32.297484+0000", 2023-09-23T20:36:15.374 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 223.043079944, 2023-09-23T20:36:15.374 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.3639945290000002, 2023-09-23T20:36:15.374 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:36:15.374 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:36:15.374 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:36:15.374 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:36:15.375 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:36:15.375 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 57665 2023-09-23T20:36:15.375 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.375 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:36:15.375 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.375 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:36:15.375 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.297484+0000", 2023-09-23T20:36:15.375 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:36:15.375 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.375 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.375 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:36:15.375 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.297484+0000", 2023-09-23T20:36:15.376 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:36:15.376 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.376 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.376 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:36:15.376 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.297484+0000", 2023-09-23T20:36:15.376 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.999999 2023-09-23T20:36:15.376 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.376 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.376 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:36:15.377 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.297505+0000", 2023-09-23T20:36:15.377 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.1407e-05 2023-09-23T20:36:15.377 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.377 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.377 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:36:15.377 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.297514+0000", 2023-09-23T20:36:15.377 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.8419999999999994e-06 2023-09-23T20:36:15.377 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.377 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.377 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:36:15.377 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.297520+0000", 2023-09-23T20:36:15.378 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.8200000000000002e-06 2023-09-23T20:36:15.378 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.378 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.378 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:15.378 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.297573+0000", 2023-09-23T20:36:15.378 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.3565e-05 2023-09-23T20:36:15.378 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.378 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.378 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:36:15.378 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.297583+0000", 2023-09-23T20:36:15.378 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.5130000000000002e-06 2023-09-23T20:36:15.379 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.379 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.379 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:15.379 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.214047+0000", 2023-09-23T20:36:15.379 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.916464218 2023-09-23T20:36:15.379 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.379 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.379 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:36:15.379 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.214055+0000", 2023-09-23T20:36:15.379 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.9710000000000004e-06 2023-09-23T20:36:15.379 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.380 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.380 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:15.380 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.637794+0000", 2023-09-23T20:36:15.380 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.4237388100000001 2023-09-23T20:36:15.380 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.380 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.380 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:36:15.380 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.637941+0000", 2023-09-23T20:36:15.380 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00014709200000000001 2023-09-23T20:36:15.380 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.380 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.381 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,4", 2023-09-23T20:36:15.381 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.638110+0000", 2023-09-23T20:36:15.381 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00016960900000000001 2023-09-23T20:36:15.381 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.381 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.381 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:36:15.381 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.640435+0000", 2023-09-23T20:36:15.381 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.002324137 2023-09-23T20:36:15.381 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.381 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.381 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:36:15.381 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.655512+0000", 2023-09-23T20:36:15.382 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.015077337999999999 2023-09-23T20:36:15.382 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.382 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.382 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:36:15.382 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.658496+0000", 2023-09-23T20:36:15.382 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00298446 2023-09-23T20:36:15.382 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.382 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.382 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:36:15.382 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.658545+0000", 2023-09-23T20:36:15.382 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.8789999999999999e-05 2023-09-23T20:36:15.383 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.383 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.383 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:36:15.383 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.661479+0000", 2023-09-23T20:36:15.383 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.002933747 2023-09-23T20:36:15.383 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:15.383 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:36:15.383 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:15.383 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.383 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.383 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:135798 7.9 7:9c5d0a75:::benchmark_data_smithi018_308601_object41246:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e345)", 2023-09-23T20:36:15.384 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:34:28.925910+0000", 2023-09-23T20:36:15.384 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 106.41465421700001, 2023-09-23T20:36:15.384 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.21189239000000001, 2023-09-23T20:36:15.384 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:36:15.384 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:36:15.384 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:36:15.384 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:36:15.384 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:36:15.384 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 135798 2023-09-23T20:36:15.384 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.384 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:36:15.384 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.385 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:36:15.385 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:28.925910+0000", 2023-09-23T20:36:15.385 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:36:15.385 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.385 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.385 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:36:15.385 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:28.925910+0000", 2023-09-23T20:36:15.385 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:36:15.385 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.385 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.385 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:36:15.386 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:28.925909+0000", 2023-09-23T20:36:15.386 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:36:15.386 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.386 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.386 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:36:15.386 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:28.925913+0000", 2023-09-23T20:36:15.386 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.6830000000000001e-06 2023-09-23T20:36:15.386 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.386 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.386 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:36:15.386 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:28.925921+0000", 2023-09-23T20:36:15.386 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.3590000000000008e-06 2023-09-23T20:36:15.387 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.387 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.387 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:36:15.387 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:28.925927+0000", 2023-09-23T20:36:15.387 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.6799999999999998e-06 2023-09-23T20:36:15.387 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.387 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.387 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:15.387 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:28.925982+0000", 2023-09-23T20:36:15.387 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.5687999999999999e-05 2023-09-23T20:36:15.387 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.387 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.388 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:36:15.388 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:28.926097+0000", 2023-09-23T20:36:15.388 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000115173 2023-09-23T20:36:15.388 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.388 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.388 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,7", 2023-09-23T20:36:15.388 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:28.926209+0000", 2023-09-23T20:36:15.388 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00011126800000000001 2023-09-23T20:36:15.388 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.388 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.388 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:36:15.388 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:28.930138+0000", 2023-09-23T20:36:15.389 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0039295650000000003 2023-09-23T20:36:15.389 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.389 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.389 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:36:15.389 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:29.120703+0000", 2023-09-23T20:36:15.389 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.19056478600000001 2023-09-23T20:36:15.389 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.389 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.389 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:36:15.389 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:29.137696+0000", 2023-09-23T20:36:15.390 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.016993438 2023-09-23T20:36:15.390 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.390 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.390 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:36:15.390 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:29.137747+0000", 2023-09-23T20:36:15.390 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.0720999999999998e-05 2023-09-23T20:36:15.390 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.390 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.390 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:36:15.390 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:29.137802+0000", 2023-09-23T20:36:15.390 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.524e-05 2023-09-23T20:36:15.391 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:15.391 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:36:15.391 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:15.391 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.391 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.391 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:135807 7.9 7:936f4f71:::benchmark_data_smithi018_308601_object41255:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e345)", 2023-09-23T20:36:15.391 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:34:28.931691+0000", 2023-09-23T20:36:15.391 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 106.408873689, 2023-09-23T20:36:15.391 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.20622343900000001, 2023-09-23T20:36:15.391 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:36:15.391 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:36:15.392 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:36:15.392 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:36:15.392 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:36:15.392 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 135807 2023-09-23T20:36:15.392 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.392 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:36:15.392 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.392 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:36:15.392 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:28.931691+0000", 2023-09-23T20:36:15.392 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:36:15.392 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.392 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.393 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:36:15.393 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:28.931691+0000", 2023-09-23T20:36:15.393 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:36:15.393 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.393 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.393 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:36:15.393 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:28.931690+0000", 2023-09-23T20:36:15.393 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.999999 2023-09-23T20:36:15.393 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.393 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.393 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:36:15.394 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:28.931695+0000", 2023-09-23T20:36:15.394 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.6180000000000003e-06 2023-09-23T20:36:15.394 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.394 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.394 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:36:15.394 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:28.931704+0000", 2023-09-23T20:36:15.394 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.0100000000000001e-06 2023-09-23T20:36:15.394 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.394 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.394 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:36:15.395 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:28.931710+0000", 2023-09-23T20:36:15.395 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.5289999999999999e-06 2023-09-23T20:36:15.395 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.395 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.395 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:15.395 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:28.931766+0000", 2023-09-23T20:36:15.395 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.6552999999999999e-05 2023-09-23T20:36:15.395 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.395 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.395 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:36:15.395 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:28.931866+0000", 2023-09-23T20:36:15.396 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.9159000000000004e-05 2023-09-23T20:36:15.396 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.396 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.396 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,7", 2023-09-23T20:36:15.396 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:29.112327+0000", 2023-09-23T20:36:15.396 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.18046110900000001 2023-09-23T20:36:15.396 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.396 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.396 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:36:15.396 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:29.121128+0000", 2023-09-23T20:36:15.396 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0088009980000000008 2023-09-23T20:36:15.397 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.397 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.397 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:36:15.397 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:29.121324+0000", 2023-09-23T20:36:15.397 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00019586000000000001 2023-09-23T20:36:15.397 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.397 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.397 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:36:15.397 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:29.137842+0000", 2023-09-23T20:36:15.397 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.016518617999999999 2023-09-23T20:36:15.397 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.397 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.398 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:36:15.398 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:29.137879+0000", 2023-09-23T20:36:15.398 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.6402999999999997e-05 2023-09-23T20:36:15.398 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.398 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.398 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:36:15.398 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:29.137914+0000", 2023-09-23T20:36:15.398 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.5503000000000002e-05 2023-09-23T20:36:15.398 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:15.398 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:36:15.398 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:15.399 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.399 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.399 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:156873 7.4 7:2413b99c:::benchmark_data_smithi018_308601_object62321:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e352)", 2023-09-23T20:36:15.399 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:34:58.961681+0000", 2023-09-23T20:36:15.399 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 76.378883643999998, 2023-09-23T20:36:15.399 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2142744990000001, 2023-09-23T20:36:15.399 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:36:15.399 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:36:15.399 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:36:15.399 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:36:15.399 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:36:15.400 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 156873 2023-09-23T20:36:15.400 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.400 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:36:15.400 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.400 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:36:15.400 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:58.961681+0000", 2023-09-23T20:36:15.400 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:36:15.400 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.400 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.400 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:36:15.400 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:58.961681+0000", 2023-09-23T20:36:15.401 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:36:15.401 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.401 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.401 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:36:15.401 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:58.961679+0000", 2023-09-23T20:36:15.401 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:36:15.401 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.401 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.401 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:36:15.401 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:58.961685+0000", 2023-09-23T20:36:15.401 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.7289999999999997e-06 2023-09-23T20:36:15.402 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.402 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.402 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:36:15.402 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:58.961698+0000", 2023-09-23T20:36:15.402 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3456e-05 2023-09-23T20:36:15.402 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.402 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.402 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:36:15.402 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:58.961707+0000", 2023-09-23T20:36:15.402 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.371e-06 2023-09-23T20:36:15.403 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.403 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.403 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:15.403 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:58.961766+0000", 2023-09-23T20:36:15.403 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.8736000000000001e-05 2023-09-23T20:36:15.403 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.403 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.403 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:36:15.403 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:58.961919+0000", 2023-09-23T20:36:15.403 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000153473 2023-09-23T20:36:15.403 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.404 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.404 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,6", 2023-09-23T20:36:15.404 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:58.962066+0000", 2023-09-23T20:36:15.404 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00014647799999999999 2023-09-23T20:36:15.404 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.404 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.404 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:15.404 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:58.982292+0000", 2023-09-23T20:36:15.404 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.020226839 2023-09-23T20:36:15.404 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.404 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.405 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:36:15.405 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:58.982307+0000", 2023-09-23T20:36:15.405 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.4323e-05 2023-09-23T20:36:15.405 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.405 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.405 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:15.405 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.150194+0000", 2023-09-23T20:36:15.405 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.167887355 2023-09-23T20:36:15.405 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.405 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.405 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for missing object", 2023-09-23T20:36:15.406 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.150339+0000", 2023-09-23T20:36:15.406 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00014458400000000001 2023-09-23T20:36:15.406 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.406 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.406 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:15.406 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.175879+0000", 2023-09-23T20:36:15.406 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.025540245999999999 2023-09-23T20:36:15.406 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.406 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.406 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:36:15.406 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.175955+0000", 2023-09-23T20:36:15.407 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.6230999999999996e-05 2023-09-23T20:36:15.407 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:15.407 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:36:15.407 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:15.407 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.407 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.407 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:156874 7.4 7:2210d725:::benchmark_data_smithi018_308601_object62322:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e352)", 2023-09-23T20:36:15.407 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:34:58.965887+0000", 2023-09-23T20:36:15.407 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 76.374677677999998, 2023-09-23T20:36:15.407 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.22316603, 2023-09-23T20:36:15.407 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:36:15.408 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:36:15.408 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:36:15.408 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:36:15.408 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:36:15.408 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 156874 2023-09-23T20:36:15.408 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.408 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:36:15.408 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.408 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:36:15.408 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:58.965887+0000", 2023-09-23T20:36:15.408 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:36:15.408 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.409 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.409 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:36:15.409 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:58.965887+0000", 2023-09-23T20:36:15.409 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:36:15.409 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.409 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.409 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:36:15.409 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:58.965885+0000", 2023-09-23T20:36:15.409 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:36:15.409 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.409 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.410 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:36:15.410 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:58.965895+0000", 2023-09-23T20:36:15.410 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.9150000000000008e-06 2023-09-23T20:36:15.410 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.410 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.410 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:36:15.410 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:58.965908+0000", 2023-09-23T20:36:15.410 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3349e-05 2023-09-23T20:36:15.410 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.410 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.411 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:36:15.411 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:58.965918+0000", 2023-09-23T20:36:15.411 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.6139999999999998e-06 2023-09-23T20:36:15.411 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.411 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.411 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:15.411 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:58.966970+0000", 2023-09-23T20:36:15.411 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.001052545 2023-09-23T20:36:15.411 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.411 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.411 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:36:15.412 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:58.967159+0000", 2023-09-23T20:36:15.412 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000189087 2023-09-23T20:36:15.412 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.412 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.412 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,6", 2023-09-23T20:36:15.412 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:58.967370+0000", 2023-09-23T20:36:15.412 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000210795 2023-09-23T20:36:15.412 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.412 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.412 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:15.412 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:58.982341+0000", 2023-09-23T20:36:15.412 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.014970255 2023-09-23T20:36:15.413 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.413 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.413 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:36:15.413 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:58.982350+0000", 2023-09-23T20:36:15.413 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.6029999999999994e-06 2023-09-23T20:36:15.413 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.413 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.413 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:15.413 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.151266+0000", 2023-09-23T20:36:15.413 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1689161379999999 2023-09-23T20:36:15.413 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.414 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.414 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for missing object", 2023-09-23T20:36:15.414 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.151368+0000", 2023-09-23T20:36:15.414 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00010191699999999999 2023-09-23T20:36:15.414 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.414 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.414 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:15.414 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.188980+0000", 2023-09-23T20:36:15.414 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.037611376000000002 2023-09-23T20:36:15.414 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.414 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.415 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:36:15.415 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.189053+0000", 2023-09-23T20:36:15.415 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.3058000000000004e-05 2023-09-23T20:36:15.415 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:15.415 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:36:15.415 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:15.415 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.415 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.415 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:156880 7.9 7:97f9ea4b:::benchmark_data_smithi018_308601_object62328:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e353)", 2023-09-23T20:36:15.415 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:34:58.989864+0000", 2023-09-23T20:36:15.417 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 76.350700273000001, 2023-09-23T20:36:15.417 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.1443719099999998, 2023-09-23T20:36:15.417 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:36:15.417 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:36:15.417 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:36:15.417 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:36:15.417 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:36:15.417 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 156880 2023-09-23T20:36:15.417 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.418 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:36:15.418 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.418 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:36:15.418 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:58.989864+0000", 2023-09-23T20:36:15.418 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:36:15.418 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.418 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.418 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:36:15.418 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:58.989864+0000", 2023-09-23T20:36:15.418 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:36:15.418 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.419 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.419 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:36:15.419 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:58.989862+0000", 2023-09-23T20:36:15.419 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:36:15.419 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.419 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.419 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:36:15.419 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:58.989870+0000", 2023-09-23T20:36:15.419 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.7959999999999996e-06 2023-09-23T20:36:15.419 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.419 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.419 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:36:15.420 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:58.989888+0000", 2023-09-23T20:36:15.420 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.8008000000000001e-05 2023-09-23T20:36:15.420 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.420 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.420 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:36:15.420 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:58.989898+0000", 2023-09-23T20:36:15.420 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.4930000000000004e-06 2023-09-23T20:36:15.420 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.420 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.420 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:15.420 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:58.994377+0000", 2023-09-23T20:36:15.420 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0044797750000000001 2023-09-23T20:36:15.421 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.421 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.421 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:36:15.421 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:58.994389+0000", 2023-09-23T20:36:15.421 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1224e-05 2023-09-23T20:36:15.421 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.421 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.421 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:15.421 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.969523+0000", 2023-09-23T20:36:15.421 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.97513417700000005 2023-09-23T20:36:15.421 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.421 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.422 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:36:15.422 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.969537+0000", 2023-09-23T20:36:15.422 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.4653e-05 2023-09-23T20:36:15.422 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.422 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.422 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:15.422 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.107362+0000", 2023-09-23T20:36:15.422 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.13782403 2023-09-23T20:36:15.422 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.422 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.422 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:36:15.423 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.107518+0000", 2023-09-23T20:36:15.423 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00015667200000000001 2023-09-23T20:36:15.423 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.423 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.423 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,4,7", 2023-09-23T20:36:15.423 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.107667+0000", 2023-09-23T20:36:15.423 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00014836800000000001 2023-09-23T20:36:15.423 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.423 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.423 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:36:15.423 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.111586+0000", 2023-09-23T20:36:15.423 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0039193520000000001 2023-09-23T20:36:15.424 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.424 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.424 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:36:15.424 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.111613+0000", 2023-09-23T20:36:15.424 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.6597999999999998e-05 2023-09-23T20:36:15.424 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.424 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.424 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:36:15.424 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.115895+0000", 2023-09-23T20:36:15.424 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.004282526 2023-09-23T20:36:15.424 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.424 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.425 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:36:15.425 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.134027+0000", 2023-09-23T20:36:15.425 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.018131807 2023-09-23T20:36:15.425 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.425 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.425 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:36:15.425 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.134094+0000", 2023-09-23T20:36:15.425 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.7551000000000004e-05 2023-09-23T20:36:15.425 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.425 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.425 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:36:15.425 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.134236+0000", 2023-09-23T20:36:15.426 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00014152500000000001 2023-09-23T20:36:15.426 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:15.426 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:36:15.426 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:15.426 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.426 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.426 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:156884 7.9 7:90c28959:::benchmark_data_smithi018_308601_object62332:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e353)", 2023-09-23T20:36:15.426 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:34:58.995119+0000", 2023-09-23T20:36:15.426 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 76.345445181000002, 2023-09-23T20:36:15.426 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.146166928, 2023-09-23T20:36:15.426 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:36:15.426 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:36:15.427 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:36:15.427 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:36:15.427 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:36:15.427 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 156884 2023-09-23T20:36:15.427 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.427 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:36:15.427 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.427 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:36:15.427 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:58.995119+0000", 2023-09-23T20:36:15.427 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:36:15.427 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.427 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.428 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:36:15.428 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:58.995119+0000", 2023-09-23T20:36:15.428 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:36:15.428 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.428 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.428 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:36:15.428 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:58.995118+0000", 2023-09-23T20:36:15.428 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.999999 2023-09-23T20:36:15.428 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.428 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.428 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:36:15.428 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:58.995124+0000", 2023-09-23T20:36:15.429 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.4960000000000004e-06 2023-09-23T20:36:15.429 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.429 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.429 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:36:15.429 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:58.995138+0000", 2023-09-23T20:36:15.429 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.4413000000000001e-05 2023-09-23T20:36:15.429 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.429 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.429 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:36:15.429 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:58.995146+0000", 2023-09-23T20:36:15.429 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.0150000000000003e-06 2023-09-23T20:36:15.430 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.430 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.430 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:15.430 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:58.995225+0000", 2023-09-23T20:36:15.430 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.9101e-05 2023-09-23T20:36:15.430 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.430 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.430 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:36:15.430 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:58.995235+0000", 2023-09-23T20:36:15.430 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.9699999999999994e-06 2023-09-23T20:36:15.430 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.430 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.431 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:15.431 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.969600+0000", 2023-09-23T20:36:15.431 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.97436495300000003 2023-09-23T20:36:15.431 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.431 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.431 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:36:15.431 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.969609+0000", 2023-09-23T20:36:15.431 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.5699999999999993e-06 2023-09-23T20:36:15.431 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.431 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.431 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:15.432 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.111637+0000", 2023-09-23T20:36:15.432 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.142028147 2023-09-23T20:36:15.432 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.432 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.432 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:36:15.432 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.111784+0000", 2023-09-23T20:36:15.432 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000146883 2023-09-23T20:36:15.432 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.432 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.432 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,4,7", 2023-09-23T20:36:15.432 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.111947+0000", 2023-09-23T20:36:15.432 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00016295400000000001 2023-09-23T20:36:15.433 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.433 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.433 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:36:15.433 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.119811+0000", 2023-09-23T20:36:15.433 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0078639850000000004 2023-09-23T20:36:15.433 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.433 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.433 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:36:15.433 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.119872+0000", 2023-09-23T20:36:15.433 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.1822000000000005e-05 2023-09-23T20:36:15.433 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.433 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.434 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:36:15.434 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.119937+0000", 2023-09-23T20:36:15.434 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.5036000000000005e-05 2023-09-23T20:36:15.434 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.434 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.434 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:36:15.434 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.141091+0000", 2023-09-23T20:36:15.434 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.021153885000000001 2023-09-23T20:36:15.434 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.434 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.434 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:36:15.435 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.141158+0000", 2023-09-23T20:36:15.435 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.6433000000000006e-05 2023-09-23T20:36:15.435 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.435 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.435 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:36:15.435 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.141286+0000", 2023-09-23T20:36:15.435 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00012822300000000001 2023-09-23T20:36:15.435 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:15.435 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:36:15.435 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:15.435 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.436 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.436 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:156891 7.9 7:900a49cc:::benchmark_data_smithi018_308601_object62339:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e353)", 2023-09-23T20:36:15.436 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:34:59.007075+0000", 2023-09-23T20:36:15.436 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 76.333489266000001, 2023-09-23T20:36:15.436 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.1390987259999998, 2023-09-23T20:36:15.436 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:36:15.436 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:36:15.436 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:36:15.436 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:36:15.436 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:36:15.436 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 156891 2023-09-23T20:36:15.436 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.437 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:36:15.437 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.437 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:36:15.437 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.007075+0000", 2023-09-23T20:36:15.437 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:36:15.437 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.437 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.437 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:36:15.437 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.007075+0000", 2023-09-23T20:36:15.437 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:36:15.437 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.437 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.438 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:36:15.438 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.007073+0000", 2023-09-23T20:36:15.438 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:36:15.438 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.438 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.438 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:36:15.438 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.007079+0000", 2023-09-23T20:36:15.438 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.9619999999999998e-06 2023-09-23T20:36:15.438 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.438 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.438 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:36:15.438 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.007090+0000", 2023-09-23T20:36:15.439 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0694e-05 2023-09-23T20:36:15.439 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.439 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.439 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:36:15.439 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.007097+0000", 2023-09-23T20:36:15.439 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.6729999999999992e-06 2023-09-23T20:36:15.439 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.439 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.439 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:15.439 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.012873+0000", 2023-09-23T20:36:15.439 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0057759999999999999 2023-09-23T20:36:15.440 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.440 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.440 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:36:15.440 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.012890+0000", 2023-09-23T20:36:15.440 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.6566999999999999e-05 2023-09-23T20:36:15.440 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.440 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.440 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:15.440 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.969636+0000", 2023-09-23T20:36:15.440 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.95674641400000004 2023-09-23T20:36:15.440 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.440 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.441 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:36:15.441 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.969644+0000", 2023-09-23T20:36:15.441 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.2989999999999999e-06 2023-09-23T20:36:15.441 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.441 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.441 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:15.441 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.115914+0000", 2023-09-23T20:36:15.441 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1462701470000001 2023-09-23T20:36:15.441 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.441 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.441 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:36:15.441 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.116062+0000", 2023-09-23T20:36:15.442 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00014790800000000001 2023-09-23T20:36:15.442 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.442 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.442 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,4,7", 2023-09-23T20:36:15.442 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.116218+0000", 2023-09-23T20:36:15.442 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000156725 2023-09-23T20:36:15.442 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.442 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.442 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:36:15.442 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.119974+0000", 2023-09-23T20:36:15.442 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0037559519999999999 2023-09-23T20:36:15.443 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.443 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.443 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:36:15.443 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.120032+0000", 2023-09-23T20:36:15.443 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.7485999999999998e-05 2023-09-23T20:36:15.443 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.443 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.443 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:36:15.443 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.126273+0000", 2023-09-23T20:36:15.443 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0062413069999999998 2023-09-23T20:36:15.443 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.443 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.444 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:36:15.444 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.146012+0000", 2023-09-23T20:36:15.444 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.019739014999999999 2023-09-23T20:36:15.444 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.444 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.444 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:36:15.444 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.146065+0000", 2023-09-23T20:36:15.444 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.2939000000000003e-05 2023-09-23T20:36:15.444 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.444 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.444 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:36:15.445 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.146174+0000", 2023-09-23T20:36:15.445 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000108713 2023-09-23T20:36:15.445 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:15.445 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:36:15.445 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:15.445 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.445 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.445 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:156904 7.4 7:256a86e5:::benchmark_data_smithi018_308601_object62352:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e353)", 2023-09-23T20:36:15.445 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:34:59.030138+0000", 2023-09-23T20:36:15.445 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 76.310425812000005, 2023-09-23T20:36:15.445 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1549561939999999, 2023-09-23T20:36:15.446 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:36:15.446 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:36:15.446 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:36:15.446 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:36:15.446 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:36:15.446 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 156904 2023-09-23T20:36:15.446 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.446 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:36:15.446 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.446 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:36:15.446 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.030138+0000", 2023-09-23T20:36:15.447 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:36:15.447 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.447 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.447 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:36:15.447 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.030138+0000", 2023-09-23T20:36:15.447 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:36:15.447 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.447 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.447 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:36:15.447 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.030138+0000", 2023-09-23T20:36:15.447 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:36:15.448 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.448 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.448 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:36:15.448 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.030141+0000", 2023-09-23T20:36:15.448 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.8260000000000001e-06 2023-09-23T20:36:15.448 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.448 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.448 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:36:15.448 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.030147+0000", 2023-09-23T20:36:15.448 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.3749999999999999e-06 2023-09-23T20:36:15.448 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.449 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.449 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:36:15.449 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.030152+0000", 2023-09-23T20:36:15.449 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.0340000000000004e-06 2023-09-23T20:36:15.449 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.449 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.449 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:15.449 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.078211+0000", 2023-09-23T20:36:15.449 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.048058233999999998 2023-09-23T20:36:15.449 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.449 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.449 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:36:15.450 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.078229+0000", 2023-09-23T20:36:15.450 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.8342000000000002e-05 2023-09-23T20:36:15.450 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.450 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.450 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:15.450 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.151421+0000", 2023-09-23T20:36:15.450 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0731922810000001 2023-09-23T20:36:15.450 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.450 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.450 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:36:15.451 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.151535+0000", 2023-09-23T20:36:15.451 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000113695 2023-09-23T20:36:15.451 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.451 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.451 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,4", 2023-09-23T20:36:15.451 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.151626+0000", 2023-09-23T20:36:15.451 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.0983999999999995e-05 2023-09-23T20:36:15.451 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.451 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.451 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:36:15.451 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.158726+0000", 2023-09-23T20:36:15.451 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0070999100000000001 2023-09-23T20:36:15.452 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.452 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.452 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:36:15.452 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.159871+0000", 2023-09-23T20:36:15.452 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.001145177 2023-09-23T20:36:15.452 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.452 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.452 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:36:15.452 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.183778+0000", 2023-09-23T20:36:15.452 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.023906849000000001 2023-09-23T20:36:15.452 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.453 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.453 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:36:15.453 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.183826+0000", 2023-09-23T20:36:15.453 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.8149999999999998e-05 2023-09-23T20:36:15.453 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.453 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.453 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:36:15.453 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.185095+0000", 2023-09-23T20:36:15.453 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0012687359999999999 2023-09-23T20:36:15.453 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:15.453 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:36:15.454 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:15.454 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.454 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.454 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:156909 7.9 7:9eb097ac:::benchmark_data_smithi018_308601_object62357:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e353)", 2023-09-23T20:36:15.454 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:34:59.038642+0000", 2023-09-23T20:36:15.454 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 76.301921832999994, 2023-09-23T20:36:15.454 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.1077584620000001, 2023-09-23T20:36:15.454 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:36:15.454 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:36:15.454 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:36:15.454 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:36:15.455 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:36:15.455 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 156909 2023-09-23T20:36:15.455 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.455 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:36:15.455 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.455 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:36:15.455 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.038642+0000", 2023-09-23T20:36:15.455 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:36:15.455 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.455 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.455 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:36:15.456 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.038642+0000", 2023-09-23T20:36:15.456 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:36:15.456 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.456 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.456 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:36:15.456 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.038641+0000", 2023-09-23T20:36:15.456 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:36:15.456 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.456 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.456 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:36:15.456 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.038649+0000", 2023-09-23T20:36:15.457 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.7000000000000008e-06 2023-09-23T20:36:15.457 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.457 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.457 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:36:15.457 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.038665+0000", 2023-09-23T20:36:15.457 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.6545000000000002e-05 2023-09-23T20:36:15.457 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.457 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.457 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:36:15.457 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.038674+0000", 2023-09-23T20:36:15.457 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.1440000000000003e-06 2023-09-23T20:36:15.458 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.458 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.458 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:15.458 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.038726+0000", 2023-09-23T20:36:15.458 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.2135000000000001e-05 2023-09-23T20:36:15.458 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.458 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.458 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:36:15.458 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.038737+0000", 2023-09-23T20:36:15.458 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1823e-05 2023-09-23T20:36:15.459 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.459 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.459 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:15.459 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.969669+0000", 2023-09-23T20:36:15.459 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.93093119400000002 2023-09-23T20:36:15.459 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.459 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.459 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:36:15.459 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.969676+0000", 2023-09-23T20:36:15.459 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.9480000000000002e-06 2023-09-23T20:36:15.459 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.459 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.460 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:15.460 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.120069+0000", 2023-09-23T20:36:15.460 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1503932180000001 2023-09-23T20:36:15.460 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.460 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.460 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:36:15.460 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.120165+0000", 2023-09-23T20:36:15.460 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.5729000000000005e-05 2023-09-23T20:36:15.460 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.460 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.460 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,4,7", 2023-09-23T20:36:15.461 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.120247+0000", 2023-09-23T20:36:15.461 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.2736000000000002e-05 2023-09-23T20:36:15.461 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.461 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.461 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:36:15.461 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.126258+0000", 2023-09-23T20:36:15.461 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0060109810000000003 2023-09-23T20:36:15.461 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.461 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.461 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:36:15.461 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.126315+0000", 2023-09-23T20:36:15.462 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.6663999999999998e-05 2023-09-23T20:36:15.462 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.462 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.462 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:36:15.462 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.126326+0000", 2023-09-23T20:36:15.462 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0812000000000001e-05 2023-09-23T20:36:15.462 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.462 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.462 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:36:15.462 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.146229+0000", 2023-09-23T20:36:15.462 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.019902835000000001 2023-09-23T20:36:15.463 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.463 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.463 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:36:15.463 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.146283+0000", 2023-09-23T20:36:15.463 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.3922000000000003e-05 2023-09-23T20:36:15.463 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.463 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.463 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:36:15.463 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.146401+0000", 2023-09-23T20:36:15.463 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000118396 2023-09-23T20:36:15.463 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:15.464 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:36:15.464 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:15.464 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.464 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.464 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:156910 7.4 7:20d2ed44:::benchmark_data_smithi018_308601_object62358:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e353)", 2023-09-23T20:36:15.464 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:34:59.038685+0000", 2023-09-23T20:36:15.464 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 76.301879038999999, 2023-09-23T20:36:15.464 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1477232370000001, 2023-09-23T20:36:15.464 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:36:15.464 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:36:15.464 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:36:15.465 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:36:15.465 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:36:15.465 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 156910 2023-09-23T20:36:15.465 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.465 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:36:15.465 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.465 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:36:15.465 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.038685+0000", 2023-09-23T20:36:15.465 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:36:15.465 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.465 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.466 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:36:15.466 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.038685+0000", 2023-09-23T20:36:15.466 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:36:15.466 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.466 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.466 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:36:15.466 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.038685+0000", 2023-09-23T20:36:15.466 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:36:15.466 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.466 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.466 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:36:15.467 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.038687+0000", 2023-09-23T20:36:15.467 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.5620000000000002e-06 2023-09-23T20:36:15.467 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.467 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.467 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:36:15.467 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.038692+0000", 2023-09-23T20:36:15.467 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.5560000000000001e-06 2023-09-23T20:36:15.467 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.467 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.467 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:36:15.467 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.038696+0000", 2023-09-23T20:36:15.468 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.6040000000000001e-06 2023-09-23T20:36:15.468 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.468 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.468 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:15.468 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.078252+0000", 2023-09-23T20:36:15.468 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.039556674999999999 2023-09-23T20:36:15.468 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.468 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.468 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:36:15.468 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.078261+0000", 2023-09-23T20:36:15.468 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.3999999999999992e-06 2023-09-23T20:36:15.469 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.469 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.469 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:15.469 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.154318+0000", 2023-09-23T20:36:15.469 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.076057134 2023-09-23T20:36:15.469 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.469 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.469 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:36:15.469 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.154417+0000", 2023-09-23T20:36:15.469 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.9007999999999998e-05 2023-09-23T20:36:15.469 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.470 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.470 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,4", 2023-09-23T20:36:15.470 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.154500+0000", 2023-09-23T20:36:15.470 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.3410000000000003e-05 2023-09-23T20:36:15.470 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.470 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.470 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:36:15.470 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.158731+0000", 2023-09-23T20:36:15.470 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0042308340000000002 2023-09-23T20:36:15.470 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.470 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.471 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:36:15.471 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.159915+0000", 2023-09-23T20:36:15.471 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0011838739999999999 2023-09-23T20:36:15.471 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.471 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.471 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:36:15.471 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.185179+0000", 2023-09-23T20:36:15.471 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.025264134000000001 2023-09-23T20:36:15.471 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.471 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.471 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:36:15.471 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.185248+0000", 2023-09-23T20:36:15.472 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.8959000000000002e-05 2023-09-23T20:36:15.472 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.472 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.472 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:36:15.472 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.186408+0000", 2023-09-23T20:36:15.472 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.001160424 2023-09-23T20:36:15.472 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:15.472 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:36:15.472 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:15.472 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.472 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.473 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:156918 7.9 7:969f6c77:::benchmark_data_smithi018_308601_object62366:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e353)", 2023-09-23T20:36:15.473 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:34:59.046909+0000", 2023-09-23T20:36:15.473 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 76.293655616999999, 2023-09-23T20:36:15.473 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.0997079030000001, 2023-09-23T20:36:15.473 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:36:15.473 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:36:15.473 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:36:15.473 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:36:15.473 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:36:15.473 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 156918 2023-09-23T20:36:15.473 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.474 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:36:15.474 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.474 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:36:15.474 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.046909+0000", 2023-09-23T20:36:15.474 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:36:15.474 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.474 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.474 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:36:15.474 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.046909+0000", 2023-09-23T20:36:15.474 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:36:15.474 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.475 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.475 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:36:15.475 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.046907+0000", 2023-09-23T20:36:15.475 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:36:15.475 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.475 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.475 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:36:15.475 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.046914+0000", 2023-09-23T20:36:15.475 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.657e-06 2023-09-23T20:36:15.475 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.475 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.476 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:36:15.476 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.046929+0000", 2023-09-23T20:36:15.476 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.4713e-05 2023-09-23T20:36:15.476 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.476 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.476 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:36:15.476 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.046936+0000", 2023-09-23T20:36:15.476 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.5900000000000002e-06 2023-09-23T20:36:15.476 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.476 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.476 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:15.476 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.046998+0000", 2023-09-23T20:36:15.477 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.1655000000000005e-05 2023-09-23T20:36:15.477 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.477 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.477 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:36:15.477 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.047008+0000", 2023-09-23T20:36:15.477 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0223e-05 2023-09-23T20:36:15.477 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.477 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.477 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:15.477 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.969700+0000", 2023-09-23T20:36:15.477 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.92269210300000004 2023-09-23T20:36:15.478 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.478 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.478 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:36:15.478 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.969707+0000", 2023-09-23T20:36:15.478 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.861e-06 2023-09-23T20:36:15.478 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.478 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.478 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:15.478 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.123042+0000", 2023-09-23T20:36:15.478 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.153334992 2023-09-23T20:36:15.478 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.479 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.479 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:36:15.479 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.123185+0000", 2023-09-23T20:36:15.479 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00014291400000000001 2023-09-23T20:36:15.479 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.479 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.479 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,4,7", 2023-09-23T20:36:15.479 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.123329+0000", 2023-09-23T20:36:15.479 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000144344 2023-09-23T20:36:15.479 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.479 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.480 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:36:15.480 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.126389+0000", 2023-09-23T20:36:15.480 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0030597329999999998 2023-09-23T20:36:15.480 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.480 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.480 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:36:15.480 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.130226+0000", 2023-09-23T20:36:15.480 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0038367789999999998 2023-09-23T20:36:15.480 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.480 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.480 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:36:15.481 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.133972+0000", 2023-09-23T20:36:15.481 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0037460340000000001 2023-09-23T20:36:15.481 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.481 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.481 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:36:15.481 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.146449+0000", 2023-09-23T20:36:15.481 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.012477626 2023-09-23T20:36:15.481 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.481 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.481 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:36:15.481 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.146505+0000", 2023-09-23T20:36:15.482 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.5926e-05 2023-09-23T20:36:15.482 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.482 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.482 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:36:15.482 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.146617+0000", 2023-09-23T20:36:15.482 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00011121899999999999 2023-09-23T20:36:15.482 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:15.482 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:36:15.482 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:15.482 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.482 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.483 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:156927 7.4 7:2c3ea5a8:::benchmark_data_smithi018_308601_object62375:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e353)", 2023-09-23T20:36:15.483 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:34:59.064897+0000", 2023-09-23T20:36:15.483 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 76.275667630000001, 2023-09-23T20:36:15.483 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.12259446, 2023-09-23T20:36:15.483 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:36:15.483 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:36:15.483 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:36:15.483 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:36:15.483 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:36:15.483 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 156927 2023-09-23T20:36:15.483 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.484 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:36:15.484 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.484 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:36:15.484 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.064897+0000", 2023-09-23T20:36:15.484 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:36:15.484 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.484 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.484 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:36:15.484 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.064897+0000", 2023-09-23T20:36:15.484 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:36:15.484 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.485 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.485 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:36:15.485 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.064895+0000", 2023-09-23T20:36:15.485 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:36:15.485 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.485 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.485 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:36:15.485 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.064903+0000", 2023-09-23T20:36:15.485 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.2300000000000008e-06 2023-09-23T20:36:15.485 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.485 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.486 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:36:15.486 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.064916+0000", 2023-09-23T20:36:15.486 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3355e-05 2023-09-23T20:36:15.486 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.486 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.486 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:36:15.486 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.064925+0000", 2023-09-23T20:36:15.486 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.8799999999999997e-06 2023-09-23T20:36:15.486 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.486 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.486 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:15.487 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.078285+0000", 2023-09-23T20:36:15.487 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.013359861000000001 2023-09-23T20:36:15.487 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.487 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.487 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:36:15.487 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.078293+0000", 2023-09-23T20:36:15.487 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.1489999999999994e-06 2023-09-23T20:36:15.487 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.487 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.487 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:15.487 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.156712+0000", 2023-09-23T20:36:15.488 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0784186039999999 2023-09-23T20:36:15.488 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.488 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.488 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:36:15.488 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.156823+0000", 2023-09-23T20:36:15.488 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000111264 2023-09-23T20:36:15.488 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.488 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.488 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,4", 2023-09-23T20:36:15.488 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.156902+0000", 2023-09-23T20:36:15.488 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.9296999999999997e-05 2023-09-23T20:36:15.488 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.489 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.489 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:36:15.489 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.159936+0000", 2023-09-23T20:36:15.489 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.003034003 2023-09-23T20:36:15.489 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.489 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.489 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:36:15.489 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.160017+0000", 2023-09-23T20:36:15.489 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.0453999999999999e-05 2023-09-23T20:36:15.489 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.489 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.490 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:36:15.490 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.186490+0000", 2023-09-23T20:36:15.490 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.026473625000000001 2023-09-23T20:36:15.490 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.490 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.490 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:36:15.490 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.186551+0000", 2023-09-23T20:36:15.490 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.0473000000000002e-05 2023-09-23T20:36:15.490 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.490 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.490 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:36:15.491 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.187491+0000", 2023-09-23T20:36:15.491 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000940229 2023-09-23T20:36:15.491 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:15.491 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:36:15.491 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:15.491 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.491 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.491 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:156946 7.9 7:9c9b215c:::benchmark_data_smithi018_308601_object62394:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e353)", 2023-09-23T20:36:15.491 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:34:59.100567+0000", 2023-09-23T20:36:15.491 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 76.239997360999993, 2023-09-23T20:36:15.491 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.0462718, 2023-09-23T20:36:15.492 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:36:15.492 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:36:15.492 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:36:15.492 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:36:15.492 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:36:15.492 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 156946 2023-09-23T20:36:15.492 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.492 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:36:15.492 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.492 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:36:15.492 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.100567+0000", 2023-09-23T20:36:15.492 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:36:15.493 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.493 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.493 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:36:15.493 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.100567+0000", 2023-09-23T20:36:15.493 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:36:15.493 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.493 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.493 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:36:15.493 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.100566+0000", 2023-09-23T20:36:15.493 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:36:15.493 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.494 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.494 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:36:15.494 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.100571+0000", 2023-09-23T20:36:15.494 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.3759999999999999e-06 2023-09-23T20:36:15.494 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.494 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.494 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:36:15.494 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.100584+0000", 2023-09-23T20:36:15.494 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2679e-05 2023-09-23T20:36:15.494 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.494 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.495 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:36:15.495 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.100590+0000", 2023-09-23T20:36:15.495 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.2929999999999997e-06 2023-09-23T20:36:15.495 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.495 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.495 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:15.495 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.100652+0000", 2023-09-23T20:36:15.495 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.2476000000000003e-05 2023-09-23T20:36:15.495 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.495 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.495 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:36:15.495 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.100667+0000", 2023-09-23T20:36:15.496 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.4270999999999999e-05 2023-09-23T20:36:15.496 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.496 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.496 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:15.496 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.971028+0000", 2023-09-23T20:36:15.496 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.87036148599999996 2023-09-23T20:36:15.496 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.496 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.496 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:36:15.496 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.971042+0000", 2023-09-23T20:36:15.496 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3366999999999999e-05 2023-09-23T20:36:15.497 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.497 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.497 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:15.497 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.126433+0000", 2023-09-23T20:36:15.497 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1553918599999999 2023-09-23T20:36:15.497 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.497 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.497 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:36:15.497 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.126572+0000", 2023-09-23T20:36:15.497 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00013825299999999999 2023-09-23T20:36:15.497 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.497 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.498 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,4,7", 2023-09-23T20:36:15.498 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.126718+0000", 2023-09-23T20:36:15.498 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00014583900000000001 2023-09-23T20:36:15.498 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.498 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.498 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:36:15.498 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.134301+0000", 2023-09-23T20:36:15.498 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0075839439999999996 2023-09-23T20:36:15.498 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.498 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.499 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:36:15.499 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.134357+0000", 2023-09-23T20:36:15.499 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.5522000000000001e-05 2023-09-23T20:36:15.499 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.499 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.499 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:36:15.499 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.134367+0000", 2023-09-23T20:36:15.499 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0118e-05 2023-09-23T20:36:15.499 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.499 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.499 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:36:15.500 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.146664+0000", 2023-09-23T20:36:15.500 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.012296848000000001 2023-09-23T20:36:15.500 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.500 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.500 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:36:15.500 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.146717+0000", 2023-09-23T20:36:15.500 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.2800999999999998e-05 2023-09-23T20:36:15.500 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.500 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.501 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:36:15.501 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.146839+0000", 2023-09-23T20:36:15.501 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000121931 2023-09-23T20:36:15.501 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:15.501 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:36:15.501 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:15.501 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.501 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.501 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:156966 7.9 7:9ee3fd12:::benchmark_data_smithi018_308601_object62414:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e354)", 2023-09-23T20:36:15.501 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:35:00.195629+0000", 2023-09-23T20:36:15.501 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 75.144935669000006, 2023-09-23T20:36:15.502 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.95484696700000005, 2023-09-23T20:36:15.502 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:36:15.502 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:36:15.502 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:36:15.502 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:36:15.502 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:36:15.502 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 156966 2023-09-23T20:36:15.502 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.502 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:36:15.502 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.502 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:36:15.503 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.195629+0000", 2023-09-23T20:36:15.503 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:36:15.503 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.503 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.503 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:36:15.503 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.195629+0000", 2023-09-23T20:36:15.503 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:36:15.503 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.503 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.503 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:36:15.503 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.195627+0000", 2023-09-23T20:36:15.503 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:36:15.504 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.504 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.504 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:36:15.504 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.195635+0000", 2023-09-23T20:36:15.504 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.8909999999999995e-06 2023-09-23T20:36:15.504 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.504 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.504 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:36:15.504 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.195652+0000", 2023-09-23T20:36:15.504 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.6968000000000001e-05 2023-09-23T20:36:15.504 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.505 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.505 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:36:15.505 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.195659+0000", 2023-09-23T20:36:15.505 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.8019999999999992e-06 2023-09-23T20:36:15.505 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.505 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.505 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:15.505 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.197096+0000", 2023-09-23T20:36:15.505 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.001436751 2023-09-23T20:36:15.505 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.505 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.506 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:36:15.506 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.197108+0000", 2023-09-23T20:36:15.506 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1445e-05 2023-09-23T20:36:15.506 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.506 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.506 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:15.506 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.130266+0000", 2023-09-23T20:36:15.506 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.933158288 2023-09-23T20:36:15.506 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.506 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.506 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:36:15.506 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.130363+0000", 2023-09-23T20:36:15.507 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.7326999999999998e-05 2023-09-23T20:36:15.507 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.507 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.507 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,4,7", 2023-09-23T20:36:15.507 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.130454+0000", 2023-09-23T20:36:15.507 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.0653000000000001e-05 2023-09-23T20:36:15.507 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.507 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.507 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:36:15.507 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.134376+0000", 2023-09-23T20:36:15.507 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0039225140000000002 2023-09-23T20:36:15.508 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.508 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.508 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:36:15.508 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.134421+0000", 2023-09-23T20:36:15.508 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.4378000000000001e-05 2023-09-23T20:36:15.508 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.508 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.508 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:36:15.508 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.134474+0000", 2023-09-23T20:36:15.508 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.3263999999999997e-05 2023-09-23T20:36:15.508 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.509 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.509 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:36:15.509 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.150283+0000", 2023-09-23T20:36:15.509 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.015809519000000001 2023-09-23T20:36:15.509 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.509 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.509 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:36:15.509 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.150341+0000", 2023-09-23T20:36:15.509 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.7182999999999998e-05 2023-09-23T20:36:15.509 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.509 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.510 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:36:15.510 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.150476+0000", 2023-09-23T20:36:15.510 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000134893 2023-09-23T20:36:15.510 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:15.510 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:36:15.510 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:15.510 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.510 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.510 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:156978 7.9 7:90ac27fe:::benchmark_data_smithi018_308601_object62426:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e354)", 2023-09-23T20:36:15.510 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:35:00.215473+0000", 2023-09-23T20:36:15.510 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 75.125091561999994, 2023-09-23T20:36:15.511 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.93521590300000002, 2023-09-23T20:36:15.511 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:36:15.511 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:36:15.511 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:36:15.511 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:36:15.511 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:36:15.511 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 156978 2023-09-23T20:36:15.511 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.511 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:36:15.511 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.511 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:36:15.511 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.215473+0000", 2023-09-23T20:36:15.512 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:36:15.512 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.512 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.512 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:36:15.512 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.215473+0000", 2023-09-23T20:36:15.512 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:36:15.512 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.512 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.512 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:36:15.512 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.215471+0000", 2023-09-23T20:36:15.512 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:36:15.513 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.513 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.513 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:36:15.513 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.215478+0000", 2023-09-23T20:36:15.513 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.6710000000000002e-06 2023-09-23T20:36:15.513 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.513 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.513 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:36:15.513 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.215494+0000", 2023-09-23T20:36:15.513 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.6480000000000001e-05 2023-09-23T20:36:15.513 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.514 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.514 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:36:15.514 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.215503+0000", 2023-09-23T20:36:15.514 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.3030000000000005e-06 2023-09-23T20:36:15.514 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.514 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.514 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:15.514 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.217858+0000", 2023-09-23T20:36:15.514 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0023550530000000002 2023-09-23T20:36:15.514 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.514 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.515 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:36:15.515 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.217873+0000", 2023-09-23T20:36:15.515 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.428e-05 2023-09-23T20:36:15.515 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.515 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.515 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:15.515 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.134499+0000", 2023-09-23T20:36:15.515 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.91662635100000001 2023-09-23T20:36:15.515 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.515 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.515 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:36:15.516 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.134590+0000", 2023-09-23T20:36:15.516 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.0661999999999994e-05 2023-09-23T20:36:15.516 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.516 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.516 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,4,7", 2023-09-23T20:36:15.516 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.134659+0000", 2023-09-23T20:36:15.516 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.9627000000000003e-05 2023-09-23T20:36:15.516 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.516 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.516 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:36:15.516 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.140958+0000", 2023-09-23T20:36:15.517 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0062990199999999998 2023-09-23T20:36:15.517 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.517 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.517 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:36:15.517 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.140971+0000", 2023-09-23T20:36:15.517 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2649e-05 2023-09-23T20:36:15.517 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.517 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.517 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:36:15.517 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.141033+0000", 2023-09-23T20:36:15.517 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.1580000000000003e-05 2023-09-23T20:36:15.517 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.518 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.518 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:36:15.518 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.150526+0000", 2023-09-23T20:36:15.518 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0094936429999999995 2023-09-23T20:36:15.518 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.518 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.518 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:36:15.518 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.150571+0000", 2023-09-23T20:36:15.518 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.4948999999999999e-05 2023-09-23T20:36:15.518 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.518 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.519 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:36:15.519 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.150689+0000", 2023-09-23T20:36:15.519 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00011736800000000001 2023-09-23T20:36:15.519 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:15.519 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:36:15.519 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:15.519 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.519 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.519 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:156997 7.9 7:926c23dd:::benchmark_data_smithi018_308601_object62445:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e354)", 2023-09-23T20:36:15.519 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:35:00.331744+0000", 2023-09-23T20:36:15.519 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 75.008820100999998, 2023-09-23T20:36:15.520 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.83941280200000001, 2023-09-23T20:36:15.520 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:36:15.520 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:36:15.520 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:36:15.520 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:36:15.520 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:36:15.520 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 156997 2023-09-23T20:36:15.520 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.520 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:36:15.520 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.520 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:36:15.520 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.331744+0000", 2023-09-23T20:36:15.521 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:36:15.521 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.521 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.521 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:36:15.521 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.331744+0000", 2023-09-23T20:36:15.521 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:36:15.521 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.521 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.521 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:36:15.521 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.331743+0000", 2023-09-23T20:36:15.521 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:36:15.522 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.522 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.522 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:36:15.522 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.331749+0000", 2023-09-23T20:36:15.522 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.1140000000000002e-06 2023-09-23T20:36:15.522 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.522 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.522 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:36:15.522 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.331762+0000", 2023-09-23T20:36:15.522 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3275000000000001e-05 2023-09-23T20:36:15.522 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.523 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.523 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:36:15.523 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.331768+0000", 2023-09-23T20:36:15.523 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.7930000000000003e-06 2023-09-23T20:36:15.523 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.523 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.523 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:15.523 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.331839+0000", 2023-09-23T20:36:15.523 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.0685999999999994e-05 2023-09-23T20:36:15.523 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.523 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.524 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:36:15.524 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.331853+0000", 2023-09-23T20:36:15.524 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.4698999999999999e-05 2023-09-23T20:36:15.524 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.524 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.524 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:15.524 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.137742+0000", 2023-09-23T20:36:15.524 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.80588849600000001 2023-09-23T20:36:15.524 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.524 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.524 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:36:15.524 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.137885+0000", 2023-09-23T20:36:15.525 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00014333 2023-09-23T20:36:15.525 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.525 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.525 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,4,7", 2023-09-23T20:36:15.525 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.138005+0000", 2023-09-23T20:36:15.525 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000120022 2023-09-23T20:36:15.525 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.525 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.525 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:36:15.525 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.141352+0000", 2023-09-23T20:36:15.525 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.003346768 2023-09-23T20:36:15.526 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.526 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.526 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:36:15.526 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.141417+0000", 2023-09-23T20:36:15.526 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.4555999999999997e-05 2023-09-23T20:36:15.526 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.526 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.526 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:36:15.526 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.141425+0000", 2023-09-23T20:36:15.526 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.3869999999999995e-06 2023-09-23T20:36:15.527 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.527 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.527 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:36:15.527 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.170973+0000", 2023-09-23T20:36:15.527 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.02954824 2023-09-23T20:36:15.527 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.527 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.527 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:36:15.527 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.171028+0000", 2023-09-23T20:36:15.527 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.5040999999999997e-05 2023-09-23T20:36:15.527 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:15.527 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:15.528 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:36:15.528 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.171157+0000", 2023-09-23T20:36:15.528 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00012873999999999999 2023-09-23T20:36:15.528 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:15.528 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:36:15.528 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:15.528 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:15.528 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:36:15.528 INFO:teuthology.orchestra.run.smithi018.stdout:} 2023-09-23T20:36:15.529 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:36:15.530 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:15.530 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:15.530 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:36:15.531 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:36:15.552 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:15 smithi018 ceph-mon[179647]: pgmap v891: 119 pgs: 11 creating+peering, 4 unknown, 104 active+clean; 4.6 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-09-23T20:36:15.552 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:15 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T20:36:15.552 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:15 smithi018 ceph-mon[179647]: osdmap e372: 8 total, 8 up, 8 in 2023-09-23T20:36:15.552 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:15 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:36:15.552 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:15 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:36:15.552 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:15 smithi018 ceph-mon[184144]: pgmap v891: 119 pgs: 11 creating+peering, 4 unknown, 104 active+clean; 4.6 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-09-23T20:36:15.552 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:15 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T20:36:15.553 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:15 smithi018 ceph-mon[184144]: osdmap e372: 8 total, 8 up, 8 in 2023-09-23T20:36:15.553 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:15 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:36:15.553 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:15 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:36:15.631 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:36:15.657 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:15 smithi149 ceph-mon[170394]: pgmap v891: 119 pgs: 11 creating+peering, 4 unknown, 104 active+clean; 4.6 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-09-23T20:36:15.657 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:15 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T20:36:15.657 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:15 smithi149 ceph-mon[170394]: osdmap e372: 8 total, 8 up, 8 in 2023-09-23T20:36:15.658 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:15 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:36:15.658 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:15 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:36:15.659 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:15.694 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:36:15.697 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:15.697 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:36:15.798 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:36:15.825 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:15.859 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:36:15.862 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:15.863 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:36:15.893 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T20:36:15.916 DEBUG:teuthology.orchestra.run.smithi018:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_3 bench 90 write' 2023-09-23T20:36:15.936 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-09-23T20:36:15.937 INFO:teuthology.orchestra.run:waiting for 3000 2023-09-23T20:36:15.963 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:36:16.008 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:hints = 1 2023-09-23T20:36:16.008 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-09-23T20:36:16.008 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Object prefix: benchmark_data_smithi018_341490 2023-09-23T20:36:16.009 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:16.009 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T20:36:16.009 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 0 0 0 0 0 0 - 0 2023-09-23T20:36:16.375 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:36:16.381 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:16.382 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:36:16.483 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:36:16.527 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:16.585 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:36:16.590 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:16.591 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:16 smithi149 ceph-mon[170394]: failed to encode map e372 with expected crc 2023-09-23T20:36:16.591 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:16 smithi149 ceph-mon[170394]: failed to encode map e372 with expected crc 2023-09-23T20:36:16.591 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:16 smithi149 ceph-mon[170394]: failed to encode map e372 with expected crc 2023-09-23T20:36:16.591 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:16 smithi149 ceph-mon[170394]: failed to encode map e372 with expected crc 2023-09-23T20:36:16.591 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:16 smithi149 ceph-mon[170394]: failed to encode map e372 with expected crc 2023-09-23T20:36:16.592 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:36:16.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:16 smithi018 ceph-mon[179647]: failed to encode map e372 with expected crc 2023-09-23T20:36:16.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:16 smithi018 ceph-mon[179647]: failed to encode map e372 with expected crc 2023-09-23T20:36:16.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:16 smithi018 ceph-mon[179647]: failed to encode map e372 with expected crc 2023-09-23T20:36:16.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:16 smithi018 ceph-mon[179647]: failed to encode map e372 with expected crc 2023-09-23T20:36:16.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:16 smithi018 ceph-mon[179647]: failed to encode map e372 with expected crc 2023-09-23T20:36:16.643 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:16 smithi018 ceph-mon[184144]: failed to encode map e372 with expected crc 2023-09-23T20:36:16.643 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:16 smithi018 ceph-mon[184144]: failed to encode map e372 with expected crc 2023-09-23T20:36:16.643 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:16 smithi018 ceph-mon[184144]: failed to encode map e372 with expected crc 2023-09-23T20:36:16.643 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:16 smithi018 ceph-mon[184144]: failed to encode map e372 with expected crc 2023-09-23T20:36:16.643 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:16 smithi018 ceph-mon[184144]: failed to encode map e372 with expected crc 2023-09-23T20:36:16.694 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:36:16.758 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:16.797 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:36:16.829 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:36:16.834 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:16.834 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:36:16.936 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:36:16.971 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:17.011 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 1 16 3077 3061 190.884 191.312 0.00477502 0.00522019 2023-09-23T20:36:17.025 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:36:17.031 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:17.032 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:36:17.133 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:36:17.204 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:17.216 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:36:17.323 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:36:17.347 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:17.347 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:36:17.448 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:36:17.481 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:17.532 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:36:17.536 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:17.537 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:36:17.576 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:17 smithi018 ceph-mon[184144]: pgmap v893: 119 pgs: 119 active+clean; 4.6 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-09-23T20:36:17.577 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:17 smithi018 ceph-mon[179647]: pgmap v893: 119 pgs: 119 active+clean; 4.6 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-09-23T20:36:17.638 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:36:17.665 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:36:17.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:17 smithi149 ceph-mon[170394]: pgmap v893: 119 pgs: 119 active+clean; 4.6 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail 2023-09-23T20:36:17.721 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:36:17.739 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:17.770 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:36:17.805 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:36:17.813 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:17.813 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:36:17.814 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:36:17.858 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:36:17.899 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:36:17.915 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:36:17.939 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:36:17.986 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:36:17.991 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:36:17.991 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:36:17.991 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:36:17.991 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:36:17.991 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:36:17.992 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:36:17.992 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:36:17.992 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:36:17.992 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:18.011 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 2 16 5654 5638 175.982 161.062 0.00598331 0.00566161 2023-09-23T20:36:18.357 INFO:teuthology.orchestra.run.smithi018.stderr:nodeep-scrub is set 2023-09-23T20:36:18.586 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:36:18.587 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:18 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3243325557' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:36:18.587 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:18 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1687877899' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:36:18.587 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:18 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3243325557' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:36:18.588 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:18 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1687877899' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:36:18.598 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:18.599 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:36:18.700 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:36:18.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:18 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3243325557' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:36:18.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:18 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1687877899' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:36:18.745 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:18.807 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:36:18.811 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:18.812 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:36:18.913 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:36:18.978 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:19.012 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 3 16 8699 8683 180.724 190.312 0.00919463 0.005524 2023-09-23T20:36:19.256 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:36:19.263 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:19.264 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:36:19.365 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:36:19.412 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:19.460 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:36:19.466 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:19.467 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:36:19.481 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:19 smithi018 ceph-mon[184144]: pgmap v894: 119 pgs: 119 active+clean; 23 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 2.8 MiB/s wr, 45 op/s 2023-09-23T20:36:19.481 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:19 smithi018 ceph-mon[184144]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:36:19.481 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:19 smithi018 ceph-mon[184144]: failed to encode map e373 with expected crc 2023-09-23T20:36:19.481 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:19 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1687877899' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:36:19.481 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:19 smithi018 ceph-mon[184144]: osdmap e373: 8 total, 8 up, 8 in 2023-09-23T20:36:19.483 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:19 smithi018 ceph-mon[179647]: pgmap v894: 119 pgs: 119 active+clean; 23 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 2.8 MiB/s wr, 45 op/s 2023-09-23T20:36:19.484 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:19 smithi018 ceph-mon[179647]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:36:19.484 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:19 smithi018 ceph-mon[179647]: failed to encode map e373 with expected crc 2023-09-23T20:36:19.484 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:19 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1687877899' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:36:19.484 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:19 smithi018 ceph-mon[179647]: osdmap e373: 8 total, 8 up, 8 in 2023-09-23T20:36:19.550 INFO:teuthology.orchestra.run.smithi018.stdout:ERROR: (22) Invalid argument 2023-09-23T20:36:19.550 INFO:teuthology.orchestra.run.smithi018.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:36:19.568 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:36:19.613 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:19 smithi149 ceph-mon[170394]: pgmap v894: 119 pgs: 119 active+clean; 23 MiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 2.8 MiB/s wr, 45 op/s 2023-09-23T20:36:19.613 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:19 smithi149 ceph-mon[170394]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:36:19.613 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:19 smithi149 ceph-mon[170394]: failed to encode map e373 with expected crc 2023-09-23T20:36:19.614 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:19 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1687877899' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:36:19.614 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:19 smithi149 ceph-mon[170394]: osdmap e373: 8 total, 8 up, 8 in 2023-09-23T20:36:19.615 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:19.668 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:36:19.672 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:19.672 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:36:19.773 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:36:19.819 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:19.874 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:36:19.881 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:19.882 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:36:19.973 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T20:36:19.982 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:36:20.012 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 4 16 11363 11347 177.144 166.5 0.00517453 0.00563569 2023-09-23T20:36:20.119 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:20.189 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:36:20.197 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:36:20.203 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:20.203 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:36:20.205 INFO:teuthology.orchestra.run.smithi018.stderr:dumped all 2023-09-23T20:36:20.304 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:36:20.343 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:20.401 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:36:20.406 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:20.406 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:36:20.507 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:36:20.552 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:20.593 INFO:teuthology.orchestra.run.smithi018.stdout:{"pg_ready":true,"pg_map":{"version":896,"stamp":"2023-09-23T20:36:19.930849+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":179574707,"num_objects":2695,"num_object_clones":0,"num_object_copies":8085,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2695,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":3574,"num_write_kb":183413,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":108,"num_bytes_recovered":4463552,"num_keys_recovered":40,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":8689,"ondisk_log_size":8689,"up":357,"acting":357,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":359,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":3544364,"kb_used_data":646636,"kb_used_omap":32,"kb_used_meta":2897631,"kb_avail":746416852,"statfs":{"total":767960285184,"available":764330856448,"internally_reserved":0,"allocated":662155264,"data_stored":649993419,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32773,"internal_metadata":2967175163},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,1,1,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":174784512,"num_objects":2667,"num_object_clones":0,"num_object_copies":8001,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2667,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2667,"num_write_kb":170688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"7.631009"},"pg_stats":[{"pgid":"1.18","version":"90'161","reported_seq":646,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T20:36:13.191232+0000","last_change":"2023-09-23T20:35:26.644897+0000","last_active":"2023-09-23T20:36:13.191232+0000","last_peered":"2023-09-23T20:36:13.191232+0000","last_clean":"2023-09-23T20:36:13.191232+0000","last_became_active":"2023-09-23T20:35:26.644564+0000","last_became_peered":"2023-09-23T20:35:26.644564+0000","last_unstale":"2023-09-23T20:36:13.191232+0000","last_undegraded":"2023-09-23T20:36:13.191232+0000","last_fullsized":"2023-09-23T20:36:13.191232+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:35:03.551143+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:35:03.551143+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:55:55.872961+0000","scrub_duration":0.033122453000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"69'1","reported_seq":69,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T20:36:13.154634+0000","last_change":"2023-09-23T20:32:33.233853+0000","last_active":"2023-09-23T20:36:13.154634+0000","last_peered":"2023-09-23T20:36:13.154634+0000","last_clean":"2023-09-23T20:36:13.154634+0000","last_became_active":"2023-09-23T20:32:33.233546+0000","last_became_peered":"2023-09-23T20:32:33.233546+0000","last_unstale":"2023-09-23T20:36:13.154634+0000","last_undegraded":"2023-09-23T20:36:13.154634+0000","last_fullsized":"2023-09-23T20:36:13.154634+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:50:27.130808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":252,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T20:36:15.193656+0000","last_change":"2023-09-23T20:32:50.536495+0000","last_active":"2023-09-23T20:36:15.193656+0000","last_peered":"2023-09-23T20:36:15.193656+0000","last_clean":"2023-09-23T20:36:15.193656+0000","last_became_active":"2023-09-23T20:32:50.536180+0000","last_became_peered":"2023-09-23T20:32:50.536180+0000","last_unstale":"2023-09-23T20:36:15.193656+0000","last_undegraded":"2023-09-23T20:36:15.193656+0000","last_fullsized":"2023-09-23T20:36:15.193656+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:01:05.524308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1b","version":"90'161","reported_seq":522,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T20:36:13.191228+0000","last_change":"2023-09-23T20:36:05.442367+0000","last_active":"2023-09-23T20:36:13.191228+0000","last_peered":"2023-09-23T20:36:13.191228+0000","last_clean":"2023-09-23T20:36:13.191228+0000","last_became_active":"2023-09-23T20:35:26.647261+0000","last_became_peered":"2023-09-23T20:35:26.647261+0000","last_unstale":"2023-09-23T20:36:13.191228+0000","last_undegraded":"2023-09-23T20:36:13.191228+0000","last_fullsized":"2023-09-23T20:36:13.191228+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:05.442288+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:36:05.442288+0000","last_clean_scrub_stamp":"2023-09-23T20:36:05.442288+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:08:03.875152+0000","scrub_duration":0.00146853,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":339,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T20:36:15.183457+0000","last_change":"2023-09-23T20:34:59.983035+0000","last_active":"2023-09-23T20:36:15.183457+0000","last_peered":"2023-09-23T20:36:15.183457+0000","last_clean":"2023-09-23T20:36:15.183457+0000","last_became_active":"2023-09-23T20:34:59.982459+0000","last_became_peered":"2023-09-23T20:34:59.982459+0000","last_unstale":"2023-09-23T20:36:15.183457+0000","last_undegraded":"2023-09-23T20:36:15.183457+0000","last_fullsized":"2023-09-23T20:36:15.183457+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:16:48.510783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":340,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T20:36:14.684111+0000","last_change":"2023-09-23T20:25:02.957185+0000","last_active":"2023-09-23T20:36:14.684111+0000","last_peered":"2023-09-23T20:36:14.684111+0000","last_clean":"2023-09-23T20:36:14.684111+0000","last_became_active":"2023-09-23T20:25:02.956689+0000","last_became_peered":"2023-09-23T20:25:02.956689+0000","last_unstale":"2023-09-23T20:36:14.684111+0000","last_undegraded":"2023-09-23T20:36:14.684111+0000","last_fullsized":"2023-09-23T20:36:14.684111+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:04:28.524015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"90'161","reported_seq":451,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T20:36:13.154387+0000","last_change":"2023-09-23T20:36:01.292103+0000","last_active":"2023-09-23T20:36:13.154387+0000","last_peered":"2023-09-23T20:36:13.154387+0000","last_clean":"2023-09-23T20:36:13.154387+0000","last_became_active":"2023-09-23T20:35:26.639912+0000","last_became_peered":"2023-09-23T20:35:26.639912+0000","last_unstale":"2023-09-23T20:36:13.154387+0000","last_undegraded":"2023-09-23T20:36:13.154387+0000","last_fullsized":"2023-09-23T20:36:13.154387+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:01.292032+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:01.292032+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:04:53.926640+0000","scrub_duration":0.0032983159999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":184,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T20:36:13.324103+0000","last_change":"2023-09-23T20:32:50.546428+0000","last_active":"2023-09-23T20:36:13.324103+0000","last_peered":"2023-09-23T20:36:13.324103+0000","last_clean":"2023-09-23T20:36:13.324103+0000","last_became_active":"2023-09-23T20:32:50.546120+0000","last_became_peered":"2023-09-23T20:32:50.546120+0000","last_unstale":"2023-09-23T20:36:13.324103+0000","last_undegraded":"2023-09-23T20:36:13.324103+0000","last_fullsized":"2023-09-23T20:36:13.324103+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:00:10.792410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":145,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T20:36:12.943116+0000","last_change":"2023-09-23T20:29:46.112977+0000","last_active":"2023-09-23T20:36:12.943116+0000","last_peered":"2023-09-23T20:36:12.943116+0000","last_clean":"2023-09-23T20:36:12.943116+0000","last_became_active":"2023-09-23T20:29:46.112600+0000","last_became_peered":"2023-09-23T20:29:46.112600+0000","last_unstale":"2023-09-23T20:36:12.943116+0000","last_undegraded":"2023-09-23T20:36:12.943116+0000","last_fullsized":"2023-09-23T20:36:12.943116+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:06:34.132247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"90'161","reported_seq":544,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T20:36:13.326714+0000","last_change":"2023-09-23T20:36:03.411105+0000","last_active":"2023-09-23T20:36:13.326714+0000","last_peered":"2023-09-23T20:36:13.326714+0000","last_clean":"2023-09-23T20:36:13.326714+0000","last_became_active":"2023-09-23T20:35:26.642130+0000","last_became_peered":"2023-09-23T20:35:26.642130+0000","last_unstale":"2023-09-23T20:36:13.326714+0000","last_undegraded":"2023-09-23T20:36:13.326714+0000","last_fullsized":"2023-09-23T20:36:13.326714+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:03.411025+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:03.411025+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:31:28.138183+0000","scrub_duration":0.001802922,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":337,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T20:36:13.324497+0000","last_change":"2023-09-23T20:25:02.969432+0000","last_active":"2023-09-23T20:36:13.324497+0000","last_peered":"2023-09-23T20:36:13.324497+0000","last_clean":"2023-09-23T20:36:13.324497+0000","last_became_active":"2023-09-23T20:25:02.969118+0000","last_became_peered":"2023-09-23T20:25:02.969118+0000","last_unstale":"2023-09-23T20:36:13.324497+0000","last_undegraded":"2023-09-23T20:36:13.324497+0000","last_fullsized":"2023-09-23T20:36:13.324497+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:02:22.453831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":75,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T20:36:13.324241+0000","last_change":"2023-09-23T20:32:01.360359+0000","last_active":"2023-09-23T20:36:13.324241+0000","last_peered":"2023-09-23T20:36:13.324241+0000","last_clean":"2023-09-23T20:36:13.324241+0000","last_became_active":"2023-09-23T20:32:01.360008+0000","last_became_peered":"2023-09-23T20:32:01.360008+0000","last_unstale":"2023-09-23T20:36:13.324241+0000","last_undegraded":"2023-09-23T20:36:13.324241+0000","last_fullsized":"2023-09-23T20:36:13.324241+0000","mapping_epoch":308,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:33:43.135001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.14","version":"90'161","reported_seq":699,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T20:36:13.324509+0000","last_change":"2023-09-23T20:35:02.236398+0000","last_active":"2023-09-23T20:36:13.324509+0000","last_peered":"2023-09-23T20:36:13.324509+0000","last_clean":"2023-09-23T20:36:13.324509+0000","last_became_active":"2023-09-23T20:35:02.235988+0000","last_became_peered":"2023-09-23T20:35:02.235988+0000","last_unstale":"2023-09-23T20:36:13.324509+0000","last_undegraded":"2023-09-23T20:36:13.324509+0000","last_fullsized":"2023-09-23T20:36:13.324509+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:47.479012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:47.479012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:13.519648+0000","scrub_duration":0.0020136590000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":348,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T20:36:13.191477+0000","last_change":"2023-09-23T20:29:45.523808+0000","last_active":"2023-09-23T20:36:13.191477+0000","last_peered":"2023-09-23T20:36:13.191477+0000","last_clean":"2023-09-23T20:36:13.191477+0000","last_became_active":"2023-09-23T20:29:45.523508+0000","last_became_peered":"2023-09-23T20:29:45.523508+0000","last_unstale":"2023-09-23T20:36:13.191477+0000","last_undegraded":"2023-09-23T20:36:13.191477+0000","last_fullsized":"2023-09-23T20:36:13.191477+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:50:55.565352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":171,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T20:36:13.326768+0000","last_change":"2023-09-23T20:32:50.539003+0000","last_active":"2023-09-23T20:36:13.326768+0000","last_peered":"2023-09-23T20:36:13.326768+0000","last_clean":"2023-09-23T20:36:13.326768+0000","last_became_active":"2023-09-23T20:32:50.538596+0000","last_became_peered":"2023-09-23T20:32:50.538596+0000","last_unstale":"2023-09-23T20:36:13.326768+0000","last_undegraded":"2023-09-23T20:36:13.326768+0000","last_fullsized":"2023-09-23T20:36:13.326768+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:56:03.314375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.17","version":"90'161","reported_seq":513,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T20:36:13.191341+0000","last_change":"2023-09-23T20:36:03.435557+0000","last_active":"2023-09-23T20:36:13.191341+0000","last_peered":"2023-09-23T20:36:13.191341+0000","last_clean":"2023-09-23T20:36:13.191341+0000","last_became_active":"2023-09-23T20:35:26.652669+0000","last_became_peered":"2023-09-23T20:35:26.652669+0000","last_unstale":"2023-09-23T20:36:13.191341+0000","last_undegraded":"2023-09-23T20:36:13.191341+0000","last_fullsized":"2023-09-23T20:36:13.191341+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:03.435480+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:03.435480+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:27:05.066931+0000","scrub_duration":0.001636689,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":399,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T20:36:13.327062+0000","last_change":"2023-09-23T20:32:33.270617+0000","last_active":"2023-09-23T20:36:13.327062+0000","last_peered":"2023-09-23T20:36:13.327062+0000","last_clean":"2023-09-23T20:36:13.327062+0000","last_became_active":"2023-09-23T20:32:33.227902+0000","last_became_peered":"2023-09-23T20:32:33.227902+0000","last_unstale":"2023-09-23T20:36:13.327062+0000","last_undegraded":"2023-09-23T20:36:13.327062+0000","last_fullsized":"2023-09-23T20:36:13.327062+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:30:26.847831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":360,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":322,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T20:36:15.183812+0000","last_change":"2023-09-23T20:29:46.080766+0000","last_active":"2023-09-23T20:36:15.183812+0000","last_peered":"2023-09-23T20:36:15.183812+0000","last_clean":"2023-09-23T20:36:15.183812+0000","last_became_active":"2023-09-23T20:29:46.080356+0000","last_became_peered":"2023-09-23T20:29:46.080356+0000","last_unstale":"2023-09-23T20:36:15.183812+0000","last_undegraded":"2023-09-23T20:36:15.183812+0000","last_fullsized":"2023-09-23T20:36:15.183812+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:08:01.862270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"90'161","reported_seq":495,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T20:36:15.183308+0000","last_change":"2023-09-23T20:36:02.548387+0000","last_active":"2023-09-23T20:36:15.183308+0000","last_peered":"2023-09-23T20:36:15.183308+0000","last_clean":"2023-09-23T20:36:15.183308+0000","last_became_active":"2023-09-23T20:35:26.644787+0000","last_became_peered":"2023-09-23T20:35:26.644787+0000","last_unstale":"2023-09-23T20:36:15.183308+0000","last_undegraded":"2023-09-23T20:36:15.183308+0000","last_fullsized":"2023-09-23T20:36:15.183308+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:02.548285+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:02.548285+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:37:15.216553+0000","scrub_duration":0.0015514459999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":79,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T20:36:13.154662+0000","last_change":"2023-09-23T20:32:33.266787+0000","last_active":"2023-09-23T20:36:13.154662+0000","last_peered":"2023-09-23T20:36:13.154662+0000","last_clean":"2023-09-23T20:36:13.154662+0000","last_became_active":"2023-09-23T20:32:33.228482+0000","last_became_peered":"2023-09-23T20:32:33.228482+0000","last_unstale":"2023-09-23T20:36:13.154662+0000","last_undegraded":"2023-09-23T20:36:13.154662+0000","last_fullsized":"2023-09-23T20:36:13.154662+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:13:01.812073+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":69,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T20:36:13.154972+0000","last_change":"2023-09-23T20:32:33.228929+0000","last_active":"2023-09-23T20:36:13.154972+0000","last_peered":"2023-09-23T20:36:13.154972+0000","last_clean":"2023-09-23T20:36:13.154972+0000","last_became_active":"2023-09-23T20:32:33.228595+0000","last_became_peered":"2023-09-23T20:32:33.228595+0000","last_unstale":"2023-09-23T20:36:13.154972+0000","last_undegraded":"2023-09-23T20:36:13.154972+0000","last_fullsized":"2023-09-23T20:36:13.154972+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:17:53.053808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"90'161","reported_seq":602,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T20:36:13.324829+0000","last_change":"2023-09-23T20:31:43.578810+0000","last_active":"2023-09-23T20:36:13.324829+0000","last_peered":"2023-09-23T20:36:13.324829+0000","last_clean":"2023-09-23T20:36:13.324829+0000","last_became_active":"2023-09-23T20:28:05.991304+0000","last_became_peered":"2023-09-23T20:28:05.991304+0000","last_unstale":"2023-09-23T20:36:13.324829+0000","last_undegraded":"2023-09-23T20:36:13.324829+0000","last_fullsized":"2023-09-23T20:36:13.324829+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:43.578666+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:43.578666+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:46:18.999991+0000","scrub_duration":0.0021890379999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":398,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T20:36:13.326608+0000","last_change":"2023-09-23T20:32:33.242943+0000","last_active":"2023-09-23T20:36:13.326608+0000","last_peered":"2023-09-23T20:36:13.326608+0000","last_clean":"2023-09-23T20:36:13.326608+0000","last_became_active":"2023-09-23T20:32:33.225468+0000","last_became_peered":"2023-09-23T20:32:33.225468+0000","last_unstale":"2023-09-23T20:36:13.326608+0000","last_undegraded":"2023-09-23T20:36:13.326608+0000","last_fullsized":"2023-09-23T20:36:13.326608+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:40:47.867573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":73,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T20:36:13.191312+0000","last_change":"2023-09-23T20:32:01.358590+0000","last_active":"2023-09-23T20:36:13.191312+0000","last_peered":"2023-09-23T20:36:13.191312+0000","last_clean":"2023-09-23T20:36:13.191312+0000","last_became_active":"2023-09-23T20:32:01.358345+0000","last_became_peered":"2023-09-23T20:32:01.358345+0000","last_unstale":"2023-09-23T20:36:13.191312+0000","last_undegraded":"2023-09-23T20:36:13.191312+0000","last_fullsized":"2023-09-23T20:36:13.191312+0000","mapping_epoch":308,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:22:21.159708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.10","version":"90'161","reported_seq":602,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T20:36:13.191135+0000","last_change":"2023-09-23T20:31:49.726214+0000","last_active":"2023-09-23T20:36:13.191135+0000","last_peered":"2023-09-23T20:36:13.191135+0000","last_clean":"2023-09-23T20:36:13.191135+0000","last_became_active":"2023-09-23T20:29:07.044892+0000","last_became_peered":"2023-09-23T20:29:07.044892+0000","last_unstale":"2023-09-23T20:36:13.191135+0000","last_undegraded":"2023-09-23T20:36:13.191135+0000","last_fullsized":"2023-09-23T20:36:13.191135+0000","mapping_epoch":253,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":254,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:49.726139+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:49.726139+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:59:11.810516+0000","scrub_duration":0.0015441470000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":337,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T20:36:13.324089+0000","last_change":"2023-09-23T20:25:02.953993+0000","last_active":"2023-09-23T20:36:13.324089+0000","last_peered":"2023-09-23T20:36:13.324089+0000","last_clean":"2023-09-23T20:36:13.324089+0000","last_became_active":"2023-09-23T20:25:02.953672+0000","last_became_peered":"2023-09-23T20:25:02.953672+0000","last_unstale":"2023-09-23T20:36:13.324089+0000","last_undegraded":"2023-09-23T20:36:13.324089+0000","last_fullsized":"2023-09-23T20:36:13.324089+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:26:08.603577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":350,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T20:36:13.191626+0000","last_change":"2023-09-23T20:25:02.949584+0000","last_active":"2023-09-23T20:36:13.191626+0000","last_peered":"2023-09-23T20:36:13.191626+0000","last_clean":"2023-09-23T20:36:13.191626+0000","last_became_active":"2023-09-23T20:25:02.949132+0000","last_became_peered":"2023-09-23T20:25:02.949132+0000","last_unstale":"2023-09-23T20:36:13.191626+0000","last_undegraded":"2023-09-23T20:36:13.191626+0000","last_fullsized":"2023-09-23T20:36:13.191626+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:34:20.509823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"90'161","reported_seq":656,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T20:36:15.183816+0000","last_change":"2023-09-23T20:35:02.238923+0000","last_active":"2023-09-23T20:36:15.183816+0000","last_peered":"2023-09-23T20:36:15.183816+0000","last_clean":"2023-09-23T20:36:15.183816+0000","last_became_active":"2023-09-23T20:35:02.238293+0000","last_became_peered":"2023-09-23T20:35:02.238293+0000","last_unstale":"2023-09-23T20:36:15.183816+0000","last_undegraded":"2023-09-23T20:36:15.183816+0000","last_fullsized":"2023-09-23T20:36:15.183816+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:42.544308+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:42.544308+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:51:22.281932+0000","scrub_duration":0.0029958179999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":355,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T20:36:13.326557+0000","last_change":"2023-09-23T20:32:33.225148+0000","last_active":"2023-09-23T20:36:13.326557+0000","last_peered":"2023-09-23T20:36:13.326557+0000","last_clean":"2023-09-23T20:36:13.326557+0000","last_became_active":"2023-09-23T20:32:33.224805+0000","last_became_peered":"2023-09-23T20:32:33.224805+0000","last_unstale":"2023-09-23T20:36:13.326557+0000","last_undegraded":"2023-09-23T20:36:13.326557+0000","last_fullsized":"2023-09-23T20:36:13.326557+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:55:30.891398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":347,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T20:36:13.325125+0000","last_change":"2023-09-23T20:25:02.957010+0000","last_active":"2023-09-23T20:36:13.325125+0000","last_peered":"2023-09-23T20:36:13.325125+0000","last_clean":"2023-09-23T20:36:13.325125+0000","last_became_active":"2023-09-23T20:25:02.956267+0000","last_became_peered":"2023-09-23T20:25:02.956267+0000","last_unstale":"2023-09-23T20:36:13.325125+0000","last_undegraded":"2023-09-23T20:36:13.325125+0000","last_fullsized":"2023-09-23T20:36:13.325125+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:10:55.299422+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"90'161","reported_seq":621,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T20:36:13.191395+0000","last_change":"2023-09-23T20:31:53.707033+0000","last_active":"2023-09-23T20:36:13.191395+0000","last_peered":"2023-09-23T20:36:13.191395+0000","last_clean":"2023-09-23T20:36:13.191395+0000","last_became_active":"2023-09-23T20:29:46.078873+0000","last_became_peered":"2023-09-23T20:29:46.078873+0000","last_unstale":"2023-09-23T20:36:13.191395+0000","last_undegraded":"2023-09-23T20:36:13.191395+0000","last_fullsized":"2023-09-23T20:36:13.191395+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:53.706934+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:53.706934+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:51:58.039657+0000","scrub_duration":0.0016840900000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":141,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T20:36:12.942659+0000","last_change":"2023-09-23T20:29:04.179171+0000","last_active":"2023-09-23T20:36:12.942659+0000","last_peered":"2023-09-23T20:36:12.942659+0000","last_clean":"2023-09-23T20:36:12.942659+0000","last_became_active":"2023-09-23T20:29:04.128672+0000","last_became_peered":"2023-09-23T20:29:04.128672+0000","last_unstale":"2023-09-23T20:36:12.942659+0000","last_undegraded":"2023-09-23T20:36:12.942659+0000","last_fullsized":"2023-09-23T20:36:12.942659+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:49:15.506106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":184,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T20:36:13.326997+0000","last_change":"2023-09-23T20:32:50.539157+0000","last_active":"2023-09-23T20:36:13.326997+0000","last_peered":"2023-09-23T20:36:13.326997+0000","last_clean":"2023-09-23T20:36:13.326997+0000","last_became_active":"2023-09-23T20:32:50.538889+0000","last_became_peered":"2023-09-23T20:32:50.538889+0000","last_unstale":"2023-09-23T20:36:13.326997+0000","last_undegraded":"2023-09-23T20:36:13.326997+0000","last_fullsized":"2023-09-23T20:36:13.326997+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:28:28.857256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.4","version":"0'0","reported_seq":11,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T20:36:13.191224+0000","last_change":"2023-09-23T20:36:11.310529+0000","last_active":"2023-09-23T20:36:13.191224+0000","last_peered":"2023-09-23T20:36:13.191224+0000","last_clean":"2023-09-23T20:36:13.191224+0000","last_became_active":"2023-09-23T20:36:11.310209+0000","last_became_peered":"2023-09-23T20:36:11.310209+0000","last_unstale":"2023-09-23T20:36:13.191224+0000","last_undegraded":"2023-09-23T20:36:13.191224+0000","last_fullsized":"2023-09-23T20:36:13.191224+0000","mapping_epoch":369,"log_start":"0'0","ondisk_log_start":"0'0","created":369,"last_epoch_clean":370,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:44:59.621229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"90'161","reported_seq":664,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T20:36:13.326772+0000","last_change":"2023-09-23T20:34:59.496438+0000","last_active":"2023-09-23T20:36:13.326772+0000","last_peered":"2023-09-23T20:36:13.326772+0000","last_clean":"2023-09-23T20:36:13.326772+0000","last_became_active":"2023-09-23T20:33:38.970527+0000","last_became_peered":"2023-09-23T20:33:38.970527+0000","last_unstale":"2023-09-23T20:36:13.326772+0000","last_undegraded":"2023-09-23T20:36:13.326772+0000","last_fullsized":"2023-09-23T20:36:13.326772+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:34:59.496375+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:34:59.496375+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:46:18.244734+0000","scrub_duration":0.0017843189999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":375,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T20:36:13.326938+0000","last_change":"2023-09-23T20:32:33.283752+0000","last_active":"2023-09-23T20:36:13.326938+0000","last_peered":"2023-09-23T20:36:13.326938+0000","last_clean":"2023-09-23T20:36:13.326938+0000","last_became_active":"2023-09-23T20:32:33.230503+0000","last_became_peered":"2023-09-23T20:32:33.230503+0000","last_unstale":"2023-09-23T20:36:13.326938+0000","last_undegraded":"2023-09-23T20:36:13.326938+0000","last_fullsized":"2023-09-23T20:36:13.326938+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:29.084410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":333,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T20:36:13.190953+0000","last_change":"2023-09-23T20:29:04.128604+0000","last_active":"2023-09-23T20:36:13.190953+0000","last_peered":"2023-09-23T20:36:13.190953+0000","last_clean":"2023-09-23T20:36:13.190953+0000","last_became_active":"2023-09-23T20:29:04.128003+0000","last_became_peered":"2023-09-23T20:29:04.128003+0000","last_unstale":"2023-09-23T20:36:13.190953+0000","last_undegraded":"2023-09-23T20:36:13.190953+0000","last_fullsized":"2023-09-23T20:36:13.190953+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:50:49.385599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"0'0","reported_seq":11,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T20:36:12.943004+0000","last_change":"2023-09-23T20:36:11.483045+0000","last_active":"2023-09-23T20:36:12.943004+0000","last_peered":"2023-09-23T20:36:12.943004+0000","last_clean":"2023-09-23T20:36:12.943004+0000","last_became_active":"2023-09-23T20:36:11.482781+0000","last_became_peered":"2023-09-23T20:36:11.482781+0000","last_unstale":"2023-09-23T20:36:12.943004+0000","last_undegraded":"2023-09-23T20:36:12.943004+0000","last_fullsized":"2023-09-23T20:36:12.943004+0000","mapping_epoch":369,"log_start":"0'0","ondisk_log_start":"0'0","created":369,"last_epoch_clean":370,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:02:26.550668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.c","version":"90'161","reported_seq":555,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T20:36:13.324454+0000","last_change":"2023-09-23T20:36:05.485974+0000","last_active":"2023-09-23T20:36:13.324454+0000","last_peered":"2023-09-23T20:36:13.324454+0000","last_clean":"2023-09-23T20:36:13.324454+0000","last_became_active":"2023-09-23T20:33:38.989255+0000","last_became_peered":"2023-09-23T20:33:38.989255+0000","last_unstale":"2023-09-23T20:36:13.324454+0000","last_undegraded":"2023-09-23T20:36:13.324454+0000","last_fullsized":"2023-09-23T20:36:13.324454+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:05.485751+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:05.485751+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:48:05.480750+0000","scrub_duration":0.0057838120000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":410,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T20:36:13.326939+0000","last_change":"2023-09-23T20:34:59.987981+0000","last_active":"2023-09-23T20:36:13.326939+0000","last_peered":"2023-09-23T20:36:13.326939+0000","last_clean":"2023-09-23T20:36:13.326939+0000","last_became_active":"2023-09-23T20:34:59.985368+0000","last_became_peered":"2023-09-23T20:34:59.985368+0000","last_unstale":"2023-09-23T20:36:13.326939+0000","last_undegraded":"2023-09-23T20:36:13.326939+0000","last_fullsized":"2023-09-23T20:36:13.326939+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:12:43.082493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":69,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T20:36:13.154472+0000","last_change":"2023-09-23T20:32:33.232142+0000","last_active":"2023-09-23T20:36:13.154472+0000","last_peered":"2023-09-23T20:36:13.154472+0000","last_clean":"2023-09-23T20:36:13.154472+0000","last_became_active":"2023-09-23T20:32:33.231683+0000","last_became_peered":"2023-09-23T20:32:33.231683+0000","last_unstale":"2023-09-23T20:36:13.154472+0000","last_undegraded":"2023-09-23T20:36:13.154472+0000","last_fullsized":"2023-09-23T20:36:13.154472+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:02:58.336348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"0'0","reported_seq":12,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T20:36:13.324811+0000","last_change":"2023-09-23T20:36:11.482244+0000","last_active":"2023-09-23T20:36:13.324811+0000","last_peered":"2023-09-23T20:36:13.324811+0000","last_clean":"2023-09-23T20:36:13.324811+0000","last_became_active":"2023-09-23T20:36:11.481929+0000","last_became_peered":"2023-09-23T20:36:11.481929+0000","last_unstale":"2023-09-23T20:36:13.324811+0000","last_undegraded":"2023-09-23T20:36:13.324811+0000","last_fullsized":"2023-09-23T20:36:13.324811+0000","mapping_epoch":369,"log_start":"0'0","ondisk_log_start":"0'0","created":369,"last_epoch_clean":370,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:29:51.534913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"90'161","reported_seq":594,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T20:36:12.943051+0000","last_change":"2023-09-23T20:32:36.302809+0000","last_active":"2023-09-23T20:36:12.943051+0000","last_peered":"2023-09-23T20:36:12.943051+0000","last_clean":"2023-09-23T20:36:12.943051+0000","last_became_active":"2023-09-23T20:32:36.302465+0000","last_became_peered":"2023-09-23T20:32:36.302465+0000","last_unstale":"2023-09-23T20:36:12.943051+0000","last_undegraded":"2023-09-23T20:36:12.943051+0000","last_fullsized":"2023-09-23T20:36:12.943051+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:46.217489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:46.217489+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:35:40.488427+0000","scrub_duration":0.001456429,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":344,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T20:36:13.326817+0000","last_change":"2023-09-23T20:29:04.184893+0000","last_active":"2023-09-23T20:36:13.326817+0000","last_peered":"2023-09-23T20:36:13.326817+0000","last_clean":"2023-09-23T20:36:13.326817+0000","last_became_active":"2023-09-23T20:29:04.152723+0000","last_became_peered":"2023-09-23T20:29:04.152723+0000","last_unstale":"2023-09-23T20:36:13.326817+0000","last_undegraded":"2023-09-23T20:36:13.326817+0000","last_fullsized":"2023-09-23T20:36:13.326817+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:51:27.017151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":145,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T20:36:12.942751+0000","last_change":"2023-09-23T20:29:46.086912+0000","last_active":"2023-09-23T20:36:12.942751+0000","last_peered":"2023-09-23T20:36:12.942751+0000","last_clean":"2023-09-23T20:36:12.942751+0000","last_became_active":"2023-09-23T20:29:46.085471+0000","last_became_peered":"2023-09-23T20:29:46.085471+0000","last_unstale":"2023-09-23T20:36:12.942751+0000","last_undegraded":"2023-09-23T20:36:12.942751+0000","last_fullsized":"2023-09-23T20:36:12.942751+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:04:55.751660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"373'573","reported_seq":586,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T20:36:19.195310+0000","last_change":"2023-09-23T20:36:11.307165+0000","last_active":"2023-09-23T20:36:19.195310+0000","last_peered":"2023-09-23T20:36:19.195310+0000","last_clean":"2023-09-23T20:36:19.195310+0000","last_became_active":"2023-09-23T20:36:11.306749+0000","last_became_peered":"2023-09-23T20:36:11.306749+0000","last_unstale":"2023-09-23T20:36:19.195310+0000","last_undegraded":"2023-09-23T20:36:19.195310+0000","last_fullsized":"2023-09-23T20:36:19.195310+0000","mapping_epoch":369,"log_start":"0'0","ondisk_log_start":"0'0","created":369,"last_epoch_clean":370,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":573,"log_dups_size":0,"ondisk_log_size":573,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:14:02.892561+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":37552128,"num_objects":573,"num_object_clones":0,"num_object_copies":1719,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":573,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":573,"num_write_kb":36672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.e","version":"90'161","reported_seq":472,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T20:36:13.154864+0000","last_change":"2023-09-23T20:36:00.322453+0000","last_active":"2023-09-23T20:36:13.154864+0000","last_peered":"2023-09-23T20:36:13.154864+0000","last_clean":"2023-09-23T20:36:13.154864+0000","last_became_active":"2023-09-23T20:35:02.233536+0000","last_became_peered":"2023-09-23T20:35:02.233536+0000","last_unstale":"2023-09-23T20:36:13.154864+0000","last_undegraded":"2023-09-23T20:36:13.154864+0000","last_fullsized":"2023-09-23T20:36:13.154864+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:00.322381+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:00.322381+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:02:15.489069+0000","scrub_duration":0.0017321859999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":378,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T20:36:13.191414+0000","last_change":"2023-09-23T20:29:04.138284+0000","last_active":"2023-09-23T20:36:13.191414+0000","last_peered":"2023-09-23T20:36:13.191414+0000","last_clean":"2023-09-23T20:36:13.191414+0000","last_became_active":"2023-09-23T20:29:04.137960+0000","last_became_peered":"2023-09-23T20:29:04.137960+0000","last_unstale":"2023-09-23T20:36:13.191414+0000","last_undegraded":"2023-09-23T20:36:13.191414+0000","last_fullsized":"2023-09-23T20:36:13.191414+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:18:59.548311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":356,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T20:36:13.324277+0000","last_change":"2023-09-23T20:32:33.236122+0000","last_active":"2023-09-23T20:36:13.324277+0000","last_peered":"2023-09-23T20:36:13.324277+0000","last_clean":"2023-09-23T20:36:13.324277+0000","last_became_active":"2023-09-23T20:32:33.235384+0000","last_became_peered":"2023-09-23T20:32:33.235384+0000","last_unstale":"2023-09-23T20:36:13.324277+0000","last_undegraded":"2023-09-23T20:36:13.324277+0000","last_fullsized":"2023-09-23T20:36:13.324277+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:56.843733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"0'0","reported_seq":11,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T20:36:13.191136+0000","last_change":"2023-09-23T20:36:11.311613+0000","last_active":"2023-09-23T20:36:13.191136+0000","last_peered":"2023-09-23T20:36:13.191136+0000","last_clean":"2023-09-23T20:36:13.191136+0000","last_became_active":"2023-09-23T20:36:11.311359+0000","last_became_peered":"2023-09-23T20:36:11.311359+0000","last_unstale":"2023-09-23T20:36:13.191136+0000","last_undegraded":"2023-09-23T20:36:13.191136+0000","last_fullsized":"2023-09-23T20:36:13.191136+0000","mapping_epoch":369,"log_start":"0'0","ondisk_log_start":"0'0","created":369,"last_epoch_clean":370,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:32:06.448223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":777,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T20:36:13.326394+0000","last_change":"2023-09-23T20:35:02.523308+0000","last_active":"2023-09-23T20:36:13.326394+0000","last_peered":"2023-09-23T20:36:13.326394+0000","last_clean":"2023-09-23T20:36:13.326394+0000","last_became_active":"2023-09-23T20:33:38.979989+0000","last_became_peered":"2023-09-23T20:33:38.979989+0000","last_unstale":"2023-09-23T20:36:13.326394+0000","last_undegraded":"2023-09-23T20:36:13.326394+0000","last_fullsized":"2023-09-23T20:36:13.326394+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:35:02.523203+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:35:02.523203+0000","last_clean_scrub_stamp":"2023-09-23T20:35:02.523203+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:02:41.472958+0000","scrub_duration":0.0085927989999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":107,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T20:36:13.324370+0000","last_change":"2023-09-23T20:32:50.538500+0000","last_active":"2023-09-23T20:36:13.324370+0000","last_peered":"2023-09-23T20:36:13.324370+0000","last_clean":"2023-09-23T20:36:13.324370+0000","last_became_active":"2023-09-23T20:32:50.538152+0000","last_became_peered":"2023-09-23T20:32:50.538152+0000","last_unstale":"2023-09-23T20:36:13.324370+0000","last_undegraded":"2023-09-23T20:36:13.324370+0000","last_fullsized":"2023-09-23T20:36:13.324370+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:00:41.103508+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":69,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T20:36:13.154659+0000","last_change":"2023-09-23T20:32:33.231614+0000","last_active":"2023-09-23T20:36:13.154659+0000","last_peered":"2023-09-23T20:36:13.154659+0000","last_clean":"2023-09-23T20:36:13.154659+0000","last_became_active":"2023-09-23T20:32:33.231207+0000","last_became_peered":"2023-09-23T20:32:33.231207+0000","last_unstale":"2023-09-23T20:36:13.154659+0000","last_undegraded":"2023-09-23T20:36:13.154659+0000","last_fullsized":"2023-09-23T20:36:13.154659+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:46:09.780829+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"373'592","reported_seq":604,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T20:36:19.213296+0000","last_change":"2023-09-23T20:36:11.307387+0000","last_active":"2023-09-23T20:36:19.213296+0000","last_peered":"2023-09-23T20:36:19.213296+0000","last_clean":"2023-09-23T20:36:19.213296+0000","last_became_active":"2023-09-23T20:36:11.307082+0000","last_became_peered":"2023-09-23T20:36:11.307082+0000","last_unstale":"2023-09-23T20:36:19.213296+0000","last_undegraded":"2023-09-23T20:36:19.213296+0000","last_fullsized":"2023-09-23T20:36:19.213296+0000","mapping_epoch":369,"log_start":"0'0","ondisk_log_start":"0'0","created":369,"last_epoch_clean":370,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":592,"log_dups_size":0,"ondisk_log_size":592,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:06:28.610503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":38797312,"num_objects":592,"num_object_clones":0,"num_object_copies":1776,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":592,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":592,"num_write_kb":37888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":622,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T20:36:13.191034+0000","last_change":"2023-09-23T20:35:00.496271+0000","last_active":"2023-09-23T20:36:13.191034+0000","last_peered":"2023-09-23T20:36:13.191034+0000","last_clean":"2023-09-23T20:36:13.191034+0000","last_became_active":"2023-09-23T20:33:38.996273+0000","last_became_peered":"2023-09-23T20:33:38.996273+0000","last_unstale":"2023-09-23T20:36:13.191034+0000","last_undegraded":"2023-09-23T20:36:13.191034+0000","last_fullsized":"2023-09-23T20:36:13.191034+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:35:00.496197+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:35:00.496197+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:30:15.813493+0000","scrub_duration":0.0016339779999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":167,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T20:36:12.942795+0000","last_change":"2023-09-23T20:32:33.229700+0000","last_active":"2023-09-23T20:36:12.942795+0000","last_peered":"2023-09-23T20:36:12.942795+0000","last_clean":"2023-09-23T20:36:12.942795+0000","last_became_active":"2023-09-23T20:32:33.229372+0000","last_became_peered":"2023-09-23T20:32:33.229372+0000","last_unstale":"2023-09-23T20:36:12.942795+0000","last_undegraded":"2023-09-23T20:36:12.942795+0000","last_fullsized":"2023-09-23T20:36:12.942795+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:18:10.958227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":28,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T20:36:13.154554+0000","last_change":"2023-09-23T20:34:59.985545+0000","last_active":"2023-09-23T20:36:13.154554+0000","last_peered":"2023-09-23T20:36:13.154554+0000","last_clean":"2023-09-23T20:36:13.154554+0000","last_became_active":"2023-09-23T20:34:59.985090+0000","last_became_peered":"2023-09-23T20:34:59.985090+0000","last_unstale":"2023-09-23T20:36:13.154554+0000","last_undegraded":"2023-09-23T20:36:13.154554+0000","last_fullsized":"2023-09-23T20:36:13.154554+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:21:40.825065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"0'0","reported_seq":11,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T20:36:13.154350+0000","last_change":"2023-09-23T20:36:11.309600+0000","last_active":"2023-09-23T20:36:13.154350+0000","last_peered":"2023-09-23T20:36:13.154350+0000","last_clean":"2023-09-23T20:36:13.154350+0000","last_became_active":"2023-09-23T20:36:11.309343+0000","last_became_peered":"2023-09-23T20:36:11.309343+0000","last_unstale":"2023-09-23T20:36:13.154350+0000","last_undegraded":"2023-09-23T20:36:13.154350+0000","last_fullsized":"2023-09-23T20:36:13.154350+0000","mapping_epoch":369,"log_start":"0'0","ondisk_log_start":"0'0","created":369,"last_epoch_clean":370,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:20:32.774345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"90'161","reported_seq":638,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T20:36:15.193573+0000","last_change":"2023-09-23T20:36:05.595168+0000","last_active":"2023-09-23T20:36:15.193573+0000","last_peered":"2023-09-23T20:36:15.193573+0000","last_clean":"2023-09-23T20:36:15.193573+0000","last_became_active":"2023-09-23T20:33:38.991910+0000","last_became_peered":"2023-09-23T20:33:38.991910+0000","last_unstale":"2023-09-23T20:36:15.193573+0000","last_undegraded":"2023-09-23T20:36:15.193573+0000","last_fullsized":"2023-09-23T20:36:15.193573+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:05.595042+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:05.595042+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:06:55.394493+0000","scrub_duration":0.007271675,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":28,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T20:36:13.154311+0000","last_change":"2023-09-23T20:34:59.983122+0000","last_active":"2023-09-23T20:36:13.154311+0000","last_peered":"2023-09-23T20:36:13.154311+0000","last_clean":"2023-09-23T20:36:13.154311+0000","last_became_active":"2023-09-23T20:34:59.982809+0000","last_became_peered":"2023-09-23T20:34:59.982809+0000","last_unstale":"2023-09-23T20:36:13.154311+0000","last_undegraded":"2023-09-23T20:36:13.154311+0000","last_fullsized":"2023-09-23T20:36:13.154311+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:49:40.839723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":69,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T20:36:13.154926+0000","last_change":"2023-09-23T20:32:33.224963+0000","last_active":"2023-09-23T20:36:13.154926+0000","last_peered":"2023-09-23T20:36:13.154926+0000","last_clean":"2023-09-23T20:36:13.154926+0000","last_became_active":"2023-09-23T20:32:33.224634+0000","last_became_peered":"2023-09-23T20:32:33.224634+0000","last_unstale":"2023-09-23T20:36:13.154926+0000","last_undegraded":"2023-09-23T20:36:13.154926+0000","last_fullsized":"2023-09-23T20:36:13.154926+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:41:40.314140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"373'583","reported_seq":596,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T20:36:19.195169+0000","last_change":"2023-09-23T20:36:11.307835+0000","last_active":"2023-09-23T20:36:19.195169+0000","last_peered":"2023-09-23T20:36:19.195169+0000","last_clean":"2023-09-23T20:36:19.195169+0000","last_became_active":"2023-09-23T20:36:11.307592+0000","last_became_peered":"2023-09-23T20:36:11.307592+0000","last_unstale":"2023-09-23T20:36:19.195169+0000","last_undegraded":"2023-09-23T20:36:19.195169+0000","last_fullsized":"2023-09-23T20:36:19.195169+0000","mapping_epoch":369,"log_start":"0'0","ondisk_log_start":"0'0","created":369,"last_epoch_clean":370,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":583,"log_dups_size":0,"ondisk_log_size":583,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:27:55.106087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":38207488,"num_objects":583,"num_object_clones":0,"num_object_copies":1749,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":583,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":583,"num_write_kb":37312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":665,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T20:36:13.191480+0000","last_change":"2023-09-23T20:34:59.481511+0000","last_active":"2023-09-23T20:36:13.191480+0000","last_peered":"2023-09-23T20:36:13.191480+0000","last_clean":"2023-09-23T20:36:13.191480+0000","last_became_active":"2023-09-23T20:33:38.988954+0000","last_became_peered":"2023-09-23T20:33:38.988954+0000","last_unstale":"2023-09-23T20:36:13.191480+0000","last_undegraded":"2023-09-23T20:36:13.191480+0000","last_fullsized":"2023-09-23T20:36:13.191480+0000","mapping_epoch":253,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:34:59.481446+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:34:59.481446+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:53:42.768648+0000","scrub_duration":0.001554773,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":186,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T20:36:15.183529+0000","last_change":"2023-09-23T20:32:50.530690+0000","last_active":"2023-09-23T20:36:15.183529+0000","last_peered":"2023-09-23T20:36:15.183529+0000","last_clean":"2023-09-23T20:36:15.183529+0000","last_became_active":"2023-09-23T20:32:50.530376+0000","last_became_peered":"2023-09-23T20:32:50.530376+0000","last_unstale":"2023-09-23T20:36:15.183529+0000","last_undegraded":"2023-09-23T20:36:15.183529+0000","last_fullsized":"2023-09-23T20:36:15.183529+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:24:53.184959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":194,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T20:36:15.193988+0000","last_change":"2023-09-23T20:32:33.230582+0000","last_active":"2023-09-23T20:36:15.193988+0000","last_peered":"2023-09-23T20:36:15.193988+0000","last_clean":"2023-09-23T20:36:15.193988+0000","last_became_active":"2023-09-23T20:32:33.230257+0000","last_became_peered":"2023-09-23T20:32:33.230257+0000","last_unstale":"2023-09-23T20:36:15.193988+0000","last_undegraded":"2023-09-23T20:36:15.193988+0000","last_fullsized":"2023-09-23T20:36:15.193988+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:11:45.333096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"0'0","reported_seq":12,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T20:36:13.324898+0000","last_change":"2023-09-23T20:36:11.310149+0000","last_active":"2023-09-23T20:36:13.324898+0000","last_peered":"2023-09-23T20:36:13.324898+0000","last_clean":"2023-09-23T20:36:13.324898+0000","last_became_active":"2023-09-23T20:36:11.309844+0000","last_became_peered":"2023-09-23T20:36:11.309844+0000","last_unstale":"2023-09-23T20:36:13.324898+0000","last_undegraded":"2023-09-23T20:36:13.324898+0000","last_fullsized":"2023-09-23T20:36:13.324898+0000","mapping_epoch":369,"log_start":"0'0","ondisk_log_start":"0'0","created":369,"last_epoch_clean":370,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:14:57.442108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":621,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T20:36:12.943115+0000","last_change":"2023-09-23T20:35:04.102089+0000","last_active":"2023-09-23T20:36:12.943115+0000","last_peered":"2023-09-23T20:36:12.943115+0000","last_clean":"2023-09-23T20:36:12.943115+0000","last_became_active":"2023-09-23T20:33:38.978580+0000","last_became_peered":"2023-09-23T20:33:38.978580+0000","last_unstale":"2023-09-23T20:36:12.943115+0000","last_undegraded":"2023-09-23T20:36:12.943115+0000","last_fullsized":"2023-09-23T20:36:12.943115+0000","mapping_epoch":253,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:35:04.101972+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:35:04.101972+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:44:38.790944+0000","scrub_duration":0.0063459650000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"86'119","reported_seq":679,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T20:36:15.183818+0000","last_change":"2023-09-23T20:29:45.521257+0000","last_active":"2023-09-23T20:36:15.183818+0000","last_peered":"2023-09-23T20:36:15.183818+0000","last_clean":"2023-09-23T20:36:15.183818+0000","last_became_active":"2023-09-23T20:29:45.520546+0000","last_became_peered":"2023-09-23T20:29:45.520546+0000","last_unstale":"2023-09-23T20:36:15.183818+0000","last_undegraded":"2023-09-23T20:36:15.183818+0000","last_fullsized":"2023-09-23T20:36:15.183818+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":119,"log_dups_size":0,"ondisk_log_size":119,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:58:56.085777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032705499999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":523,"num_read_kb":3503,"num_write":136,"num_write_kb":3128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":388,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T20:36:13.191588+0000","last_change":"2023-09-23T20:25:39.637160+0000","last_active":"2023-09-23T20:36:13.191588+0000","last_peered":"2023-09-23T20:36:13.191588+0000","last_clean":"2023-09-23T20:36:13.191588+0000","last_became_active":"2023-09-23T20:25:39.636860+0000","last_became_peered":"2023-09-23T20:25:39.636860+0000","last_unstale":"2023-09-23T20:36:13.191588+0000","last_undegraded":"2023-09-23T20:36:13.191588+0000","last_fullsized":"2023-09-23T20:36:13.191588+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:21:47.860063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":308,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T20:36:15.183466+0000","last_change":"2023-09-23T20:34:59.985572+0000","last_active":"2023-09-23T20:36:15.183466+0000","last_peered":"2023-09-23T20:36:15.183466+0000","last_clean":"2023-09-23T20:36:15.183466+0000","last_became_active":"2023-09-23T20:34:59.985241+0000","last_became_peered":"2023-09-23T20:34:59.985241+0000","last_unstale":"2023-09-23T20:36:15.183466+0000","last_undegraded":"2023-09-23T20:36:15.183466+0000","last_fullsized":"2023-09-23T20:36:15.183466+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:54:57.081334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.8","version":"0'0","reported_seq":11,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T20:36:13.191033+0000","last_change":"2023-09-23T20:36:11.482049+0000","last_active":"2023-09-23T20:36:13.191033+0000","last_peered":"2023-09-23T20:36:13.191033+0000","last_clean":"2023-09-23T20:36:13.191033+0000","last_became_active":"2023-09-23T20:36:11.481701+0000","last_became_peered":"2023-09-23T20:36:11.481701+0000","last_unstale":"2023-09-23T20:36:13.191033+0000","last_undegraded":"2023-09-23T20:36:13.191033+0000","last_fullsized":"2023-09-23T20:36:13.191033+0000","mapping_epoch":369,"log_start":"0'0","ondisk_log_start":"0'0","created":369,"last_epoch_clean":370,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:27:21.469954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":616,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T20:36:13.191107+0000","last_change":"2023-09-23T20:31:45.785521+0000","last_active":"2023-09-23T20:36:13.191107+0000","last_peered":"2023-09-23T20:36:13.191107+0000","last_clean":"2023-09-23T20:36:13.191107+0000","last_became_active":"2023-09-23T20:27:55.213394+0000","last_became_peered":"2023-09-23T20:27:55.213394+0000","last_unstale":"2023-09-23T20:36:13.191107+0000","last_undegraded":"2023-09-23T20:36:13.191107+0000","last_fullsized":"2023-09-23T20:36:13.191107+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:45.785450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:45.785450+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:21:26.092578+0000","scrub_duration":0.001558306,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"86'106","reported_seq":1090,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T20:36:13.191136+0000","last_change":"2023-09-23T20:31:31.531882+0000","last_active":"2023-09-23T20:36:13.191136+0000","last_peered":"2023-09-23T20:36:13.191136+0000","last_clean":"2023-09-23T20:36:13.191136+0000","last_became_active":"2023-09-23T20:29:46.078501+0000","last_became_peered":"2023-09-23T20:29:46.078501+0000","last_unstale":"2023-09-23T20:36:13.191136+0000","last_undegraded":"2023-09-23T20:36:13.191136+0000","last_fullsized":"2023-09-23T20:36:13.191136+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":106,"log_dups_size":0,"ondisk_log_size":106,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:37.015974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00099302599999999993,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1105,"num_read_kb":1689,"num_write":100,"num_write_kb":404,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":69,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T20:36:13.154909+0000","last_change":"2023-09-23T20:32:33.230242+0000","last_active":"2023-09-23T20:36:13.154909+0000","last_peered":"2023-09-23T20:36:13.154909+0000","last_clean":"2023-09-23T20:36:13.154909+0000","last_became_active":"2023-09-23T20:32:33.229732+0000","last_became_peered":"2023-09-23T20:32:33.229732+0000","last_unstale":"2023-09-23T20:36:13.154909+0000","last_undegraded":"2023-09-23T20:36:13.154909+0000","last_fullsized":"2023-09-23T20:36:13.154909+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:27:55.538429+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":332,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T20:36:15.183767+0000","last_change":"2023-09-23T20:29:45.533406+0000","last_active":"2023-09-23T20:36:15.183767+0000","last_peered":"2023-09-23T20:36:15.183767+0000","last_clean":"2023-09-23T20:36:15.183767+0000","last_became_active":"2023-09-23T20:29:45.526001+0000","last_became_peered":"2023-09-23T20:29:45.526001+0000","last_unstale":"2023-09-23T20:36:15.183767+0000","last_undegraded":"2023-09-23T20:36:15.183767+0000","last_fullsized":"2023-09-23T20:36:15.183767+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:43.787251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.b","version":"0'0","reported_seq":11,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T20:36:13.154446+0000","last_change":"2023-09-23T20:36:11.310405+0000","last_active":"2023-09-23T20:36:13.154446+0000","last_peered":"2023-09-23T20:36:13.154446+0000","last_clean":"2023-09-23T20:36:13.154446+0000","last_became_active":"2023-09-23T20:36:11.310095+0000","last_became_peered":"2023-09-23T20:36:11.310095+0000","last_unstale":"2023-09-23T20:36:13.154446+0000","last_undegraded":"2023-09-23T20:36:13.154446+0000","last_fullsized":"2023-09-23T20:36:13.154446+0000","mapping_epoch":369,"log_start":"0'0","ondisk_log_start":"0'0","created":369,"last_epoch_clean":370,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:28:16.000557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":844,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T20:36:13.326521+0000","last_change":"2023-09-23T20:35:02.260708+0000","last_active":"2023-09-23T20:36:13.326521+0000","last_peered":"2023-09-23T20:36:13.326521+0000","last_clean":"2023-09-23T20:36:13.326521+0000","last_became_active":"2023-09-23T20:35:02.260348+0000","last_became_peered":"2023-09-23T20:35:02.260348+0000","last_unstale":"2023-09-23T20:36:13.326521+0000","last_undegraded":"2023-09-23T20:36:13.326521+0000","last_fullsized":"2023-09-23T20:36:13.326521+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:43.390908+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:43.390908+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:09:14.144070+0000","scrub_duration":0.0032721,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"86'130","reported_seq":686,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T20:36:13.324343+0000","last_change":"2023-09-23T20:25:39.637458+0000","last_active":"2023-09-23T20:36:13.324343+0000","last_peered":"2023-09-23T20:36:13.324343+0000","last_clean":"2023-09-23T20:36:13.324343+0000","last_became_active":"2023-09-23T20:25:39.636773+0000","last_became_peered":"2023-09-23T20:25:39.636773+0000","last_unstale":"2023-09-23T20:36:13.324343+0000","last_undegraded":"2023-09-23T20:36:13.324343+0000","last_fullsized":"2023-09-23T20:36:13.324343+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:00:32.810564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011300049999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":512,"num_read_kb":1820,"num_write":133,"num_write_kb":1347,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":322,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T20:36:15.183438+0000","last_change":"2023-09-23T20:29:46.073394+0000","last_active":"2023-09-23T20:36:15.183438+0000","last_peered":"2023-09-23T20:36:15.183438+0000","last_clean":"2023-09-23T20:36:15.183438+0000","last_became_active":"2023-09-23T20:29:46.072067+0000","last_became_peered":"2023-09-23T20:29:46.072067+0000","last_unstale":"2023-09-23T20:36:15.183438+0000","last_undegraded":"2023-09-23T20:36:15.183438+0000","last_fullsized":"2023-09-23T20:36:15.183438+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:01:56.944614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":314,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T20:36:18.350112+0000","last_change":"2023-09-23T20:29:45.533185+0000","last_active":"2023-09-23T20:36:18.350112+0000","last_peered":"2023-09-23T20:36:18.350112+0000","last_clean":"2023-09-23T20:36:18.350112+0000","last_became_active":"2023-09-23T20:29:45.526765+0000","last_became_peered":"2023-09-23T20:29:45.526765+0000","last_unstale":"2023-09-23T20:36:18.350112+0000","last_undegraded":"2023-09-23T20:36:18.350112+0000","last_fullsized":"2023-09-23T20:36:18.350112+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:18:33.063215+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"0'0","reported_seq":11,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T20:36:12.942561+0000","last_change":"2023-09-23T20:36:11.311818+0000","last_active":"2023-09-23T20:36:12.942561+0000","last_peered":"2023-09-23T20:36:12.942561+0000","last_clean":"2023-09-23T20:36:12.942561+0000","last_became_active":"2023-09-23T20:36:11.311525+0000","last_became_peered":"2023-09-23T20:36:11.311525+0000","last_unstale":"2023-09-23T20:36:12.942561+0000","last_undegraded":"2023-09-23T20:36:12.942561+0000","last_fullsized":"2023-09-23T20:36:12.942561+0000","mapping_epoch":369,"log_start":"0'0","ondisk_log_start":"0'0","created":369,"last_epoch_clean":370,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:16:49.945652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"86'194","reported_seq":1020,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T20:36:13.191315+0000","last_change":"2023-09-23T20:25:40.618665+0000","last_active":"2023-09-23T20:36:13.191315+0000","last_peered":"2023-09-23T20:36:13.191315+0000","last_clean":"2023-09-23T20:36:13.191315+0000","last_became_active":"2023-09-23T20:25:40.618087+0000","last_became_peered":"2023-09-23T20:25:40.618087+0000","last_unstale":"2023-09-23T20:36:13.191315+0000","last_undegraded":"2023-09-23T20:36:13.191315+0000","last_fullsized":"2023-09-23T20:36:13.191315+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":194,"log_dups_size":0,"ondisk_log_size":194,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:07:16.114968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00037995999999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1025,"num_read_kb":1201,"num_write":173,"num_write_kb":1276,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":805,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T20:36:13.326425+0000","last_change":"2023-09-23T20:31:45.343835+0000","last_active":"2023-09-23T20:36:13.326425+0000","last_peered":"2023-09-23T20:36:13.326425+0000","last_clean":"2023-09-23T20:36:13.326425+0000","last_became_active":"2023-09-23T20:29:07.042407+0000","last_became_peered":"2023-09-23T20:29:07.042407+0000","last_unstale":"2023-09-23T20:36:13.326425+0000","last_undegraded":"2023-09-23T20:36:13.326425+0000","last_fullsized":"2023-09-23T20:36:13.326425+0000","mapping_epoch":253,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":254,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:45.343742+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:45.343742+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:33:12.745366+0000","scrub_duration":0.001475802,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":171,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T20:36:13.191206+0000","last_change":"2023-09-23T20:32:50.534315+0000","last_active":"2023-09-23T20:36:13.191206+0000","last_peered":"2023-09-23T20:36:13.191206+0000","last_clean":"2023-09-23T20:36:13.191206+0000","last_became_active":"2023-09-23T20:32:50.533896+0000","last_became_peered":"2023-09-23T20:32:50.533896+0000","last_unstale":"2023-09-23T20:36:13.191206+0000","last_undegraded":"2023-09-23T20:36:13.191206+0000","last_fullsized":"2023-09-23T20:36:13.191206+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:06:53.464923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":151,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T20:36:12.943079+0000","last_change":"2023-09-23T20:32:33.225012+0000","last_active":"2023-09-23T20:36:12.943079+0000","last_peered":"2023-09-23T20:36:12.943079+0000","last_clean":"2023-09-23T20:36:12.943079+0000","last_became_active":"2023-09-23T20:32:33.224699+0000","last_became_peered":"2023-09-23T20:32:33.224699+0000","last_unstale":"2023-09-23T20:36:12.943079+0000","last_undegraded":"2023-09-23T20:36:12.943079+0000","last_fullsized":"2023-09-23T20:36:12.943079+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:10:06.267878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.a","version":"373'618","reported_seq":631,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T20:36:19.213212+0000","last_change":"2023-09-23T20:36:11.311940+0000","last_active":"2023-09-23T20:36:19.213212+0000","last_peered":"2023-09-23T20:36:19.213212+0000","last_clean":"2023-09-23T20:36:19.213212+0000","last_became_active":"2023-09-23T20:36:11.311558+0000","last_became_peered":"2023-09-23T20:36:11.311558+0000","last_unstale":"2023-09-23T20:36:19.213212+0000","last_undegraded":"2023-09-23T20:36:19.213212+0000","last_fullsized":"2023-09-23T20:36:19.213212+0000","mapping_epoch":369,"log_start":"0'0","ondisk_log_start":"0'0","created":369,"last_epoch_clean":370,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":618,"log_dups_size":0,"ondisk_log_size":618,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:58:12.645832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":40501248,"num_objects":618,"num_object_clones":0,"num_object_copies":1854,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":618,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":618,"num_write_kb":39552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":667,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T20:36:15.183737+0000","last_change":"2023-09-23T20:35:02.238632+0000","last_active":"2023-09-23T20:36:15.183737+0000","last_peered":"2023-09-23T20:36:15.183737+0000","last_clean":"2023-09-23T20:36:15.183737+0000","last_became_active":"2023-09-23T20:35:02.237812+0000","last_became_peered":"2023-09-23T20:35:02.237812+0000","last_unstale":"2023-09-23T20:36:15.183737+0000","last_undegraded":"2023-09-23T20:36:15.183737+0000","last_fullsized":"2023-09-23T20:36:15.183737+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:43.582146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:43.582146+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:25:04.582696+0000","scrub_duration":0.0016219730000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"86'80","reported_seq":635,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T20:36:13.326381+0000","last_change":"2023-09-23T20:29:45.536796+0000","last_active":"2023-09-23T20:36:13.326381+0000","last_peered":"2023-09-23T20:36:13.326381+0000","last_clean":"2023-09-23T20:36:13.326381+0000","last_became_active":"2023-09-23T20:29:45.536439+0000","last_became_peered":"2023-09-23T20:29:45.536439+0000","last_unstale":"2023-09-23T20:36:13.326381+0000","last_undegraded":"2023-09-23T20:36:13.326381+0000","last_fullsized":"2023-09-23T20:36:13.326381+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":80,"log_dups_size":0,"ondisk_log_size":80,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:30:57.793132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035115,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":395,"num_read_kb":1370,"num_write":66,"num_write_kb":1061,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":374,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T20:36:13.324943+0000","last_change":"2023-09-23T20:34:59.984286+0000","last_active":"2023-09-23T20:36:13.324943+0000","last_peered":"2023-09-23T20:36:13.324943+0000","last_clean":"2023-09-23T20:36:13.324943+0000","last_became_active":"2023-09-23T20:34:59.983950+0000","last_became_peered":"2023-09-23T20:34:59.983950+0000","last_unstale":"2023-09-23T20:36:13.324943+0000","last_undegraded":"2023-09-23T20:36:13.324943+0000","last_fullsized":"2023-09-23T20:36:13.324943+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:31:53.557035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":369,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T20:36:13.325043+0000","last_change":"2023-09-23T20:29:45.690815+0000","last_active":"2023-09-23T20:36:13.325043+0000","last_peered":"2023-09-23T20:36:13.325043+0000","last_clean":"2023-09-23T20:36:13.325043+0000","last_became_active":"2023-09-23T20:29:45.690478+0000","last_became_peered":"2023-09-23T20:29:45.690478+0000","last_unstale":"2023-09-23T20:36:13.325043+0000","last_undegraded":"2023-09-23T20:36:13.325043+0000","last_fullsized":"2023-09-23T20:36:13.325043+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:38.288193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.f","version":"372'301","reported_seq":313,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T20:36:17.752430+0000","last_change":"2023-09-23T20:36:11.309536+0000","last_active":"2023-09-23T20:36:17.752430+0000","last_peered":"2023-09-23T20:36:17.752430+0000","last_clean":"2023-09-23T20:36:17.752430+0000","last_became_active":"2023-09-23T20:36:11.309208+0000","last_became_peered":"2023-09-23T20:36:11.309208+0000","last_unstale":"2023-09-23T20:36:17.752430+0000","last_undegraded":"2023-09-23T20:36:17.752430+0000","last_fullsized":"2023-09-23T20:36:17.752430+0000","mapping_epoch":369,"log_start":"0'0","ondisk_log_start":"0'0","created":369,"last_epoch_clean":370,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":301,"log_dups_size":0,"ondisk_log_size":301,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:47:51.785158+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19726336,"num_objects":301,"num_object_clones":0,"num_object_copies":903,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":301,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":301,"num_write_kb":19264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":614,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T20:36:12.943156+0000","last_change":"2023-09-23T20:35:00.151823+0000","last_active":"2023-09-23T20:36:12.943156+0000","last_peered":"2023-09-23T20:36:12.943156+0000","last_clean":"2023-09-23T20:36:12.943156+0000","last_became_active":"2023-09-23T20:33:38.991030+0000","last_became_peered":"2023-09-23T20:33:38.991030+0000","last_unstale":"2023-09-23T20:36:12.943156+0000","last_undegraded":"2023-09-23T20:36:12.943156+0000","last_fullsized":"2023-09-23T20:36:12.943156+0000","mapping_epoch":253,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:35:00.151695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:35:00.151695+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:09:55.183927+0000","scrub_duration":0.0016030090000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"86'102","reported_seq":966,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T20:36:13.326909+0000","last_change":"2023-09-23T20:32:36.314672+0000","last_active":"2023-09-23T20:36:13.326909+0000","last_peered":"2023-09-23T20:36:13.326909+0000","last_clean":"2023-09-23T20:36:13.326909+0000","last_became_active":"2023-09-23T20:32:36.314231+0000","last_became_peered":"2023-09-23T20:32:36.314231+0000","last_unstale":"2023-09-23T20:36:13.326909+0000","last_undegraded":"2023-09-23T20:36:13.326909+0000","last_fullsized":"2023-09-23T20:36:13.326909+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":102,"log_dups_size":0,"ondisk_log_size":102,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:11:40.514709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030656500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":963,"num_read_kb":1234,"num_write":60,"num_write_kb":225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":298,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T20:36:15.193681+0000","last_change":"2023-09-23T20:29:45.538283+0000","last_active":"2023-09-23T20:36:15.193681+0000","last_peered":"2023-09-23T20:36:15.193681+0000","last_clean":"2023-09-23T20:36:15.193681+0000","last_became_active":"2023-09-23T20:29:45.537922+0000","last_became_peered":"2023-09-23T20:29:45.537922+0000","last_unstale":"2023-09-23T20:36:15.193681+0000","last_undegraded":"2023-09-23T20:36:15.193681+0000","last_fullsized":"2023-09-23T20:36:15.193681+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:30:52.397836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":306,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T20:36:13.324548+0000","last_change":"2023-09-23T20:20:14.888874+0000","last_active":"2023-09-23T20:36:13.324548+0000","last_peered":"2023-09-23T20:36:13.324548+0000","last_clean":"2023-09-23T20:36:13.324548+0000","last_became_active":"2023-09-23T20:20:14.887724+0000","last_became_peered":"2023-09-23T20:20:14.887724+0000","last_unstale":"2023-09-23T20:36:13.324548+0000","last_undegraded":"2023-09-23T20:36:13.324548+0000","last_fullsized":"2023-09-23T20:36:13.324548+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:09:24.510326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"0'0","reported_seq":12,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T20:36:13.324369+0000","last_change":"2023-09-23T20:36:11.482126+0000","last_active":"2023-09-23T20:36:13.324369+0000","last_peered":"2023-09-23T20:36:13.324369+0000","last_clean":"2023-09-23T20:36:13.324369+0000","last_became_active":"2023-09-23T20:36:11.481802+0000","last_became_peered":"2023-09-23T20:36:11.481802+0000","last_unstale":"2023-09-23T20:36:13.324369+0000","last_undegraded":"2023-09-23T20:36:13.324369+0000","last_fullsized":"2023-09-23T20:36:13.324369+0000","mapping_epoch":369,"log_start":"0'0","ondisk_log_start":"0'0","created":369,"last_epoch_clean":370,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:13:12.263455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":590,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T20:36:13.154974+0000","last_change":"2023-09-23T20:36:03.246372+0000","last_active":"2023-09-23T20:36:13.154974+0000","last_peered":"2023-09-23T20:36:13.154974+0000","last_clean":"2023-09-23T20:36:13.154974+0000","last_became_active":"2023-09-23T20:33:38.985300+0000","last_became_peered":"2023-09-23T20:33:38.985300+0000","last_unstale":"2023-09-23T20:36:13.154974+0000","last_undegraded":"2023-09-23T20:36:13.154974+0000","last_fullsized":"2023-09-23T20:36:13.154974+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:03.246271+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:03.246271+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:12:15.732632+0000","scrub_duration":0.0019016700000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":109,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T20:36:13.327034+0000","last_change":"2023-09-23T20:32:50.549695+0000","last_active":"2023-09-23T20:36:13.327034+0000","last_peered":"2023-09-23T20:36:13.327034+0000","last_clean":"2023-09-23T20:36:13.327034+0000","last_became_active":"2023-09-23T20:32:50.549431+0000","last_became_peered":"2023-09-23T20:32:50.549431+0000","last_unstale":"2023-09-23T20:36:13.327034+0000","last_undegraded":"2023-09-23T20:36:13.327034+0000","last_fullsized":"2023-09-23T20:36:13.327034+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:45:53.666569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"86'128","reported_seq":764,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T20:36:18.349435+0000","last_change":"2023-09-23T20:32:36.310656+0000","last_active":"2023-09-23T20:36:18.349435+0000","last_peered":"2023-09-23T20:36:18.349435+0000","last_clean":"2023-09-23T20:36:18.349435+0000","last_became_active":"2023-09-23T20:32:36.310155+0000","last_became_peered":"2023-09-23T20:32:36.310155+0000","last_unstale":"2023-09-23T20:36:18.349435+0000","last_undegraded":"2023-09-23T20:36:18.349435+0000","last_fullsized":"2023-09-23T20:36:18.349435+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:58:14.217466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000293202,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":579,"num_read_kb":1473,"num_write":96,"num_write_kb":1062,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":142,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T20:36:13.326560+0000","last_change":"2023-09-23T20:32:50.539051+0000","last_active":"2023-09-23T20:36:13.326560+0000","last_peered":"2023-09-23T20:36:13.326560+0000","last_clean":"2023-09-23T20:36:13.326560+0000","last_became_active":"2023-09-23T20:32:50.538755+0000","last_became_peered":"2023-09-23T20:32:50.538755+0000","last_unstale":"2023-09-23T20:36:13.326560+0000","last_undegraded":"2023-09-23T20:36:13.326560+0000","last_fullsized":"2023-09-23T20:36:13.326560+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:16.656613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.d","version":"0'0","reported_seq":11,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T20:36:13.191478+0000","last_change":"2023-09-23T20:36:11.482064+0000","last_active":"2023-09-23T20:36:13.191478+0000","last_peered":"2023-09-23T20:36:13.191478+0000","last_clean":"2023-09-23T20:36:13.191478+0000","last_became_active":"2023-09-23T20:36:11.481816+0000","last_became_peered":"2023-09-23T20:36:11.481816+0000","last_unstale":"2023-09-23T20:36:13.191478+0000","last_undegraded":"2023-09-23T20:36:13.191478+0000","last_fullsized":"2023-09-23T20:36:13.191478+0000","mapping_epoch":369,"log_start":"0'0","ondisk_log_start":"0'0","created":369,"last_epoch_clean":370,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:03:56.861034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.4","version":"90'161","reported_seq":521,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T20:36:13.191575+0000","last_change":"2023-09-23T20:34:54.504652+0000","last_active":"2023-09-23T20:36:13.191575+0000","last_peered":"2023-09-23T20:36:13.191575+0000","last_clean":"2023-09-23T20:36:13.191575+0000","last_became_active":"2023-09-23T20:32:50.533470+0000","last_became_peered":"2023-09-23T20:32:50.533470+0000","last_unstale":"2023-09-23T20:36:13.191575+0000","last_undegraded":"2023-09-23T20:36:13.191575+0000","last_fullsized":"2023-09-23T20:36:13.191575+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:34:54.504554+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:34:54.504554+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:17:03.108888+0000","scrub_duration":0.013031021,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.7","version":"86'117","reported_seq":641,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T20:36:12.943171+0000","last_change":"2023-09-23T20:31:32.814392+0000","last_active":"2023-09-23T20:36:12.943171+0000","last_peered":"2023-09-23T20:36:12.943171+0000","last_clean":"2023-09-23T20:36:12.943171+0000","last_became_active":"2023-09-23T20:31:32.813857+0000","last_became_peered":"2023-09-23T20:31:32.813857+0000","last_unstale":"2023-09-23T20:36:12.943171+0000","last_undegraded":"2023-09-23T20:36:12.943171+0000","last_fullsized":"2023-09-23T20:36:12.943171+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:56:57.863005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046891400000000002,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":893,"num_read_kb":922,"num_write":82,"num_write_kb":4193,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":369,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T20:36:13.325056+0000","last_change":"2023-09-23T20:32:33.277827+0000","last_active":"2023-09-23T20:36:13.325056+0000","last_peered":"2023-09-23T20:36:13.325056+0000","last_clean":"2023-09-23T20:36:13.325056+0000","last_became_active":"2023-09-23T20:32:33.236087+0000","last_became_peered":"2023-09-23T20:32:33.236087+0000","last_unstale":"2023-09-23T20:36:13.325056+0000","last_undegraded":"2023-09-23T20:36:13.325056+0000","last_fullsized":"2023-09-23T20:36:13.325056+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:20:58.176854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":69,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T20:36:13.154433+0000","last_change":"2023-09-23T20:32:33.238290+0000","last_active":"2023-09-23T20:36:13.154433+0000","last_peered":"2023-09-23T20:36:13.154433+0000","last_clean":"2023-09-23T20:36:13.154433+0000","last_became_active":"2023-09-23T20:32:33.237984+0000","last_became_peered":"2023-09-23T20:32:33.237984+0000","last_unstale":"2023-09-23T20:36:13.154433+0000","last_undegraded":"2023-09-23T20:36:13.154433+0000","last_fullsized":"2023-09-23T20:36:13.154433+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:49:02.089307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"90'161","reported_seq":605,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T20:36:13.324760+0000","last_change":"2023-09-23T20:36:06.487465+0000","last_active":"2023-09-23T20:36:13.324760+0000","last_peered":"2023-09-23T20:36:13.324760+0000","last_clean":"2023-09-23T20:36:13.324760+0000","last_became_active":"2023-09-23T20:35:26.651601+0000","last_became_peered":"2023-09-23T20:35:26.651601+0000","last_unstale":"2023-09-23T20:36:13.324760+0000","last_undegraded":"2023-09-23T20:36:13.324760+0000","last_fullsized":"2023-09-23T20:36:13.324760+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:06.487351+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:06.487351+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:19:22.102078+0000","scrub_duration":0.001964379,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":337,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T20:36:13.324466+0000","last_change":"2023-09-23T20:32:33.237234+0000","last_active":"2023-09-23T20:36:13.324466+0000","last_peered":"2023-09-23T20:36:13.324466+0000","last_clean":"2023-09-23T20:36:13.324466+0000","last_became_active":"2023-09-23T20:32:33.236881+0000","last_became_peered":"2023-09-23T20:32:33.236881+0000","last_unstale":"2023-09-23T20:36:13.324466+0000","last_undegraded":"2023-09-23T20:36:13.324466+0000","last_fullsized":"2023-09-23T20:36:13.324466+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:56:17.384831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":225,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T20:36:13.324688+0000","last_change":"2023-09-23T20:29:45.538314+0000","last_active":"2023-09-23T20:36:13.324688+0000","last_peered":"2023-09-23T20:36:13.324688+0000","last_clean":"2023-09-23T20:36:13.324688+0000","last_became_active":"2023-09-23T20:29:45.536847+0000","last_became_peered":"2023-09-23T20:29:45.536847+0000","last_unstale":"2023-09-23T20:36:13.324688+0000","last_undegraded":"2023-09-23T20:36:13.324688+0000","last_fullsized":"2023-09-23T20:36:13.324688+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:51.948721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"90'161","reported_seq":456,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T20:36:12.942971+0000","last_change":"2023-09-23T20:35:42.864462+0000","last_active":"2023-09-23T20:36:12.942971+0000","last_peered":"2023-09-23T20:36:12.942971+0000","last_clean":"2023-09-23T20:36:12.942971+0000","last_became_active":"2023-09-23T20:35:26.665630+0000","last_became_peered":"2023-09-23T20:35:26.665630+0000","last_unstale":"2023-09-23T20:36:12.942971+0000","last_undegraded":"2023-09-23T20:36:12.942971+0000","last_fullsized":"2023-09-23T20:36:12.942971+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:35:42.864363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:35:42.864363+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:14:34.984081+0000","scrub_duration":0.0070144939999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":381,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T20:36:13.191368+0000","last_change":"2023-09-23T20:32:33.235987+0000","last_active":"2023-09-23T20:36:13.191368+0000","last_peered":"2023-09-23T20:36:13.191368+0000","last_clean":"2023-09-23T20:36:13.191368+0000","last_became_active":"2023-09-23T20:32:33.226544+0000","last_became_peered":"2023-09-23T20:32:33.226544+0000","last_unstale":"2023-09-23T20:36:13.191368+0000","last_undegraded":"2023-09-23T20:36:13.191368+0000","last_fullsized":"2023-09-23T20:36:13.191368+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:53:22.794856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":28,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T20:36:13.154617+0000","last_change":"2023-09-23T20:34:59.987575+0000","last_active":"2023-09-23T20:36:13.154617+0000","last_peered":"2023-09-23T20:36:13.154617+0000","last_clean":"2023-09-23T20:36:13.154617+0000","last_became_active":"2023-09-23T20:34:59.987229+0000","last_became_peered":"2023-09-23T20:34:59.987229+0000","last_unstale":"2023-09-23T20:36:13.154617+0000","last_undegraded":"2023-09-23T20:36:13.154617+0000","last_fullsized":"2023-09-23T20:36:13.154617+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:34:01.820159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":389,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T20:36:13.191483+0000","last_change":"2023-09-23T20:32:33.268260+0000","last_active":"2023-09-23T20:36:13.191483+0000","last_peered":"2023-09-23T20:36:13.191483+0000","last_clean":"2023-09-23T20:36:13.191483+0000","last_became_active":"2023-09-23T20:32:33.233695+0000","last_became_peered":"2023-09-23T20:32:33.233695+0000","last_unstale":"2023-09-23T20:36:13.191483+0000","last_undegraded":"2023-09-23T20:36:13.191483+0000","last_fullsized":"2023-09-23T20:36:13.191483+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:19:29.299623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":28,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T20:36:13.154473+0000","last_change":"2023-09-23T20:34:59.987416+0000","last_active":"2023-09-23T20:36:13.154473+0000","last_peered":"2023-09-23T20:36:13.154473+0000","last_clean":"2023-09-23T20:36:13.154473+0000","last_became_active":"2023-09-23T20:34:59.987117+0000","last_became_peered":"2023-09-23T20:34:59.987117+0000","last_unstale":"2023-09-23T20:36:13.154473+0000","last_undegraded":"2023-09-23T20:36:13.154473+0000","last_fullsized":"2023-09-23T20:36:13.154473+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:36:16.318623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"90'161","reported_seq":567,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T20:36:13.326592+0000","last_change":"2023-09-23T20:36:02.384213+0000","last_active":"2023-09-23T20:36:13.326592+0000","last_peered":"2023-09-23T20:36:13.326592+0000","last_clean":"2023-09-23T20:36:13.326592+0000","last_became_active":"2023-09-23T20:35:26.643160+0000","last_became_peered":"2023-09-23T20:35:26.643160+0000","last_unstale":"2023-09-23T20:36:13.326592+0000","last_undegraded":"2023-09-23T20:36:13.326592+0000","last_fullsized":"2023-09-23T20:36:13.326592+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:02.384133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:02.384133+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:34:12.654342+0000","scrub_duration":0.001769098,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":187,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T20:36:12.943084+0000","last_change":"2023-09-23T20:32:50.537447+0000","last_active":"2023-09-23T20:36:12.943084+0000","last_peered":"2023-09-23T20:36:12.943084+0000","last_clean":"2023-09-23T20:36:12.943084+0000","last_became_active":"2023-09-23T20:32:50.537136+0000","last_became_peered":"2023-09-23T20:32:50.537136+0000","last_unstale":"2023-09-23T20:36:12.943084+0000","last_undegraded":"2023-09-23T20:36:12.943084+0000","last_fullsized":"2023-09-23T20:36:12.943084+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:46:51.599492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":87,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T20:36:13.323990+0000","last_change":"2023-09-23T20:32:50.537450+0000","last_active":"2023-09-23T20:36:13.323990+0000","last_peered":"2023-09-23T20:36:13.323990+0000","last_clean":"2023-09-23T20:36:13.323990+0000","last_became_active":"2023-09-23T20:32:50.537128+0000","last_became_peered":"2023-09-23T20:32:50.537128+0000","last_unstale":"2023-09-23T20:36:13.323990+0000","last_undegraded":"2023-09-23T20:36:13.323990+0000","last_fullsized":"2023-09-23T20:36:13.323990+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:37:15.889029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1d","version":"90'161","reported_seq":463,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T20:36:15.183289+0000","last_change":"2023-09-23T20:36:01.505098+0000","last_active":"2023-09-23T20:36:15.183289+0000","last_peered":"2023-09-23T20:36:15.183289+0000","last_clean":"2023-09-23T20:36:15.183289+0000","last_became_active":"2023-09-23T20:35:26.644707+0000","last_became_peered":"2023-09-23T20:35:26.644707+0000","last_unstale":"2023-09-23T20:36:15.183289+0000","last_undegraded":"2023-09-23T20:36:15.183289+0000","last_fullsized":"2023-09-23T20:36:15.183289+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:01.504973+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:01.504973+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:02:10.880809+0000","scrub_duration":0.00202344,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":354,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T20:36:13.324626+0000","last_change":"2023-09-23T20:25:39.668882+0000","last_active":"2023-09-23T20:36:13.324626+0000","last_peered":"2023-09-23T20:36:13.324626+0000","last_clean":"2023-09-23T20:36:13.324626+0000","last_became_active":"2023-09-23T20:25:39.637128+0000","last_became_peered":"2023-09-23T20:25:39.637128+0000","last_unstale":"2023-09-23T20:36:13.324626+0000","last_undegraded":"2023-09-23T20:36:13.324626+0000","last_fullsized":"2023-09-23T20:36:13.324626+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:56.406298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":69,"reported_epoch":371,"state":"active+clean","last_fresh":"2023-09-23T20:36:13.154273+0000","last_change":"2023-09-23T20:32:33.232281+0000","last_active":"2023-09-23T20:36:13.154273+0000","last_peered":"2023-09-23T20:36:13.154273+0000","last_clean":"2023-09-23T20:36:13.154273+0000","last_became_active":"2023-09-23T20:32:33.232003+0000","last_became_peered":"2023-09-23T20:32:33.232003+0000","last_unstale":"2023-09-23T20:36:13.154273+0000","last_undegraded":"2023-09-23T20:36:13.154273+0000","last_fullsized":"2023-09-23T20:36:13.154273+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:58:27.314355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":49,"num_bytes_recovered":6522,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26691,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":174784512,"num_objects":2667,"num_object_clones":0,"num_object_copies":8001,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2667,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2667,"num_write_kb":170688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":573243392,"data_stored":573243392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":2667,"ondisk_log_size":2667,"up":48,"acting":48,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":58,"num_bytes_recovered":4194342,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6082,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201581,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":318696,"kb_used_data":8224,"kb_used_omap":0,"kb_used_meta":310464,"kb_avail":93426456,"statfs":{"total":95995035648,"available":95668690944,"internally_reserved":0,"allocated":8421376,"data_stored":7255891,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":317915136},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:35:44 2023","interfaces":[{"interface":"back","average":{"1min":3.5600000000000001,"5min":2.5299999999999998,"15min":1.8160000000000001},"min":{"1min":0.28100000000000003,"5min":0.23599999999999999,"15min":0.20799999999999999},"max":{"1min":17.399999999999999,"5min":17.399999999999999,"15min":48.798999999999999},"last":0.39900000000000002},{"interface":"front","average":{"1min":3.1890000000000001,"5min":2.21,"15min":1.877},"min":{"1min":0.249,"5min":0.249,"15min":0.24399999999999999},"max":{"1min":19.355,"5min":19.355,"15min":48.247},"last":0.33900000000000002}]},{"osd":1,"last update":"Sat Sep 23 20:35:59 2023","interfaces":[{"interface":"back","average":{"1min":3.0880000000000001,"5min":2.25,"15min":1.9239999999999999},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.254},"max":{"1min":16.103000000000002,"5min":16.103000000000002,"15min":16.103000000000002},"last":0.46100000000000002},{"interface":"front","average":{"1min":3.0249999999999999,"5min":2.0430000000000001,"15min":2.0009999999999999},"min":{"1min":0.32700000000000001,"5min":0.248,"15min":0.248},"max":{"1min":12.02,"5min":12.02,"15min":12.02},"last":0.61799999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:35:44 2023","interfaces":[{"interface":"back","average":{"1min":3.3079999999999998,"5min":2.492,"15min":1.8620000000000001},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27600000000000002},"max":{"1min":9.8629999999999995,"5min":13.173999999999999,"15min":13.173999999999999},"last":0.48999999999999999},{"interface":"front","average":{"1min":3.7130000000000001,"5min":2.8799999999999999,"15min":2.113},"min":{"1min":0.36099999999999999,"5min":0.27400000000000002,"15min":0.26600000000000001},"max":{"1min":12.551,"5min":23.494,"15min":23.494},"last":0.50600000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:36:15 2023","interfaces":[{"interface":"back","average":{"1min":2.1960000000000002,"5min":2.7879999999999998,"15min":2.0880000000000001},"min":{"1min":0.39100000000000001,"5min":0.33700000000000002,"15min":0.26200000000000001},"max":{"1min":7.3630000000000004,"5min":17.155999999999999,"15min":17.155999999999999},"last":0.442},{"interface":"front","average":{"1min":2.5609999999999999,"5min":2.9049999999999998,"15min":2.044},"min":{"1min":0.40899999999999997,"5min":0.34799999999999998,"15min":0.27600000000000002},"max":{"1min":7.3719999999999999,"5min":16.960000000000001,"15min":16.960000000000001},"last":0.64900000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:36:15 2023","interfaces":[{"interface":"back","average":{"1min":0.746,"5min":0.81899999999999995,"15min":1.1140000000000001},"min":{"1min":0.26900000000000002,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":4.4189999999999996,"5min":5.8819999999999997,"15min":15.177},"last":0.53100000000000003},{"interface":"front","average":{"1min":1.0169999999999999,"5min":0.96099999999999997,"15min":1.266},"min":{"1min":0.28399999999999997,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":3.1099999999999999,"5min":6.1779999999999999,"15min":11.122},"last":0.67800000000000005}]},{"osd":5,"last update":"Sat Sep 23 20:36:15 2023","interfaces":[{"interface":"back","average":{"1min":0.996,"5min":0.75,"15min":1.1739999999999999},"min":{"1min":0.30599999999999999,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":3.0939999999999999,"5min":4.133,"15min":13.241},"last":0.47699999999999998},{"interface":"front","average":{"1min":0.88600000000000001,"5min":0.68000000000000005,"15min":1.2350000000000001},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.315},"max":{"1min":4.4610000000000003,"5min":4.7590000000000003,"15min":18.831},"last":0.55700000000000005}]},{"osd":6,"last update":"Sat Sep 23 20:36:15 2023","interfaces":[{"interface":"back","average":{"1min":0.97899999999999998,"5min":0.70899999999999996,"15min":1.139},"min":{"1min":0.34799999999999998,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":5.0339999999999998,"5min":5.0339999999999998,"15min":13.185},"last":0.57599999999999996},{"interface":"front","average":{"1min":1.0860000000000001,"5min":0.97299999999999998,"15min":1.167},"min":{"1min":0.378,"5min":0.318,"15min":0.29899999999999999},"max":{"1min":3.7890000000000001,"5min":6.2770000000000001,"15min":9.9120000000000008},"last":0.54600000000000004}]}]},{"osd":6,"up_from":47,"seq":201863463217,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":441044,"kb_used_data":161932,"kb_used_omap":0,"kb_used_meta":279104,"kb_avail":93304108,"statfs":{"total":95995035648,"available":95543406592,"internally_reserved":0,"allocated":165818368,"data_stored":164664532,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":285802496},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:35:41 2023","interfaces":[{"interface":"back","average":{"1min":1.754,"5min":2.589,"15min":1.667},"min":{"1min":0.26200000000000001,"5min":0.22800000000000001,"15min":0.221},"max":{"1min":4.8959999999999999,"5min":53.701999999999998,"15min":53.701999999999998},"last":0.63800000000000001},{"interface":"front","average":{"1min":1.7290000000000001,"5min":2.9180000000000001,"15min":2.016},"min":{"1min":0.252,"5min":0.252,"15min":0.25},"max":{"1min":6.093,"5min":53.875999999999998,"15min":53.875999999999998},"last":0.56100000000000005}]},{"osd":1,"last update":"Sat Sep 23 20:36:01 2023","interfaces":[{"interface":"back","average":{"1min":2.3340000000000001,"5min":2.1880000000000002,"15min":2.5630000000000002},"min":{"1min":0.309,"5min":0.249,"15min":0.249},"max":{"1min":10.331,"5min":34.173999999999999,"15min":34.173999999999999},"last":0.54900000000000004},{"interface":"front","average":{"1min":2.0539999999999998,"5min":2.282,"15min":3.085},"min":{"1min":0.35299999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":6.6020000000000003,"5min":34.490000000000002,"15min":34.490000000000002},"last":0.58599999999999997}]},{"osd":2,"last update":"Sat Sep 23 20:35:41 2023","interfaces":[{"interface":"back","average":{"1min":2.6259999999999999,"5min":2.512,"15min":1.9430000000000001},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":13.429,"5min":34.317999999999998,"15min":34.317999999999998},"last":0.48999999999999999},{"interface":"front","average":{"1min":2.4649999999999999,"5min":2.5289999999999999,"15min":1.859},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":13.457000000000001,"5min":34.292999999999999,"15min":34.292999999999999},"last":0.66300000000000003}]},{"osd":3,"last update":"Sat Sep 23 20:35:25 2023","interfaces":[{"interface":"back","average":{"1min":2.0099999999999998,"5min":2.0369999999999999,"15min":1.7050000000000001},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29599999999999999},"max":{"1min":6.46,"5min":7.5620000000000003,"15min":9.4239999999999995},"last":0.67800000000000005},{"interface":"front","average":{"1min":2.8290000000000002,"5min":2.4140000000000001,"15min":1.851},"min":{"1min":0.39400000000000002,"5min":0.317,"15min":0.317},"max":{"1min":8.5600000000000005,"5min":12.875999999999999,"15min":12.875999999999999},"last":0.76400000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:36:05 2023","interfaces":[{"interface":"back","average":{"1min":0.60499999999999998,"5min":0.61899999999999999,"15min":0.90400000000000003},"min":{"1min":0.26900000000000002,"5min":0.251,"15min":0.251},"max":{"1min":2.1600000000000001,"5min":4.9279999999999999,"15min":10.173},"last":0.69699999999999995},{"interface":"front","average":{"1min":1.236,"5min":0.88800000000000001,"15min":1.1100000000000001},"min":{"1min":0.35099999999999998,"5min":0.28199999999999997,"15min":0.27000000000000002},"max":{"1min":4.194,"5min":7.7629999999999999,"15min":8.3249999999999993},"last":0.72199999999999998}]},{"osd":5,"last update":"Sat Sep 23 20:36:05 2023","interfaces":[{"interface":"back","average":{"1min":0.879,"5min":0.747,"15min":0.97599999999999998},"min":{"1min":0.32800000000000001,"5min":0.30499999999999999,"15min":0.28699999999999998},"max":{"1min":2.8849999999999998,"5min":6.8310000000000004,"15min":14.085000000000001},"last":0.74299999999999999},{"interface":"front","average":{"1min":0.879,"5min":0.82199999999999995,"15min":1.329},"min":{"1min":0.35099999999999998,"5min":0.32000000000000001,"15min":0.30599999999999999},"max":{"1min":4.2489999999999997,"5min":7.7300000000000004,"15min":18.192},"last":0.80400000000000005}]},{"osd":7,"last update":"Sat Sep 23 20:35:25 2023","interfaces":[{"interface":"back","average":{"1min":0.82599999999999996,"5min":0.69799999999999995,"15min":1.085},"min":{"1min":0.36599999999999999,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":4.851,"5min":5.0350000000000001,"15min":10.217000000000001},"last":0.77800000000000002},{"interface":"front","average":{"1min":0.93500000000000005,"5min":0.93999999999999995,"15min":1.2509999999999999},"min":{"1min":0.35999999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":5.6820000000000004,"5min":7.601,"15min":9.7360000000000007},"last":0.82999999999999996}]}]},{"osd":1,"up_from":266,"seq":1142461300816,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":482440,"kb_used_data":213056,"kb_used_omap":8,"kb_used_meta":269367,"kb_avail":93262712,"statfs":{"total":95995035648,"available":95501017088,"internally_reserved":0,"allocated":218169344,"data_stored":216168581,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8206,"internal_metadata":275832818},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,1,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:35:58 2023","interfaces":[{"interface":"back","average":{"1min":3.254,"5min":3.1600000000000001,"15min":2.6059999999999999},"min":{"1min":0.222,"5min":0.218,"15min":0.218},"max":{"1min":14.531000000000001,"5min":52.689999999999998,"15min":52.689999999999998},"last":0.42999999999999999},{"interface":"front","average":{"1min":4.46,"5min":3.734,"15min":3.375},"min":{"1min":0.19900000000000001,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":19.41,"5min":52.662999999999997,"15min":52.662999999999997},"last":0.308}]},{"osd":2,"last update":"Sat Sep 23 20:35:58 2023","interfaces":[{"interface":"back","average":{"1min":4.4960000000000004,"5min":3.9430000000000001,"15min":4.7309999999999999},"min":{"1min":0.25900000000000001,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":14.474,"5min":28.359000000000002,"15min":28.359000000000002},"last":1.3200000000000001},{"interface":"front","average":{"1min":2.9969999999999999,"5min":2.7989999999999999,"15min":2.3290000000000002},"min":{"1min":0.26700000000000002,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":11.606999999999999,"5min":12.801,"15min":12.801},"last":0.34699999999999998}]},{"osd":3,"last update":"Sat Sep 23 20:35:58 2023","interfaces":[{"interface":"back","average":{"1min":4.468,"5min":4.5,"15min":4.0780000000000003},"min":{"1min":0.308,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":14.359,"5min":80.745999999999995,"15min":80.745999999999995},"last":0.68300000000000005},{"interface":"front","average":{"1min":2.7360000000000002,"5min":4.2469999999999999,"15min":4.6459999999999999},"min":{"1min":0.35599999999999998,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":19.315000000000001,"5min":82.988,"15min":82.988},"last":0.75600000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:35:58 2023","interfaces":[{"interface":"back","average":{"1min":3.4609999999999999,"5min":2.5750000000000002,"15min":2.2440000000000002},"min":{"1min":0.35599999999999998,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":9.4510000000000005,"5min":24.327999999999999,"15min":24.327999999999999},"last":1.4550000000000001},{"interface":"front","average":{"1min":2.4430000000000001,"5min":1.76,"15min":1.9359999999999999},"min":{"1min":0.38800000000000001,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":11.574999999999999,"5min":11.574999999999999,"15min":11.574999999999999},"last":0.63100000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:35:58 2023","interfaces":[{"interface":"back","average":{"1min":2.0760000000000001,"5min":2.0310000000000001,"15min":2.5449999999999999},"min":{"1min":0.34100000000000003,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":6.4059999999999997,"5min":8.1950000000000003,"15min":9.6479999999999997},"last":0.97099999999999997},{"interface":"front","average":{"1min":3.5289999999999999,"5min":2.7690000000000001,"15min":2.3159999999999998},"min":{"1min":0.44800000000000001,"5min":0.36899999999999999,"15min":0.36799999999999999},"max":{"1min":9.5150000000000006,"5min":24.404,"15min":24.404},"last":1.4850000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:35:58 2023","interfaces":[{"interface":"back","average":{"1min":1.79,"5min":1.974,"15min":5.6239999999999997},"min":{"1min":0.36899999999999999,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":5.6109999999999998,"5min":8.1620000000000008,"15min":72.682000000000002},"last":0.72299999999999998},{"interface":"front","average":{"1min":3.569,"5min":2.8439999999999999,"15min":5.2409999999999997},"min":{"1min":0.45800000000000002,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":9.4849999999999994,"5min":24.376000000000001,"15min":72.563999999999993},"last":1.3939999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:35:58 2023","interfaces":[{"interface":"back","average":{"1min":2.6949999999999998,"5min":2.258,"15min":2.9409999999999998},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":18.437000000000001,"5min":18.437000000000001,"15min":18.437000000000001},"last":0.83699999999999997},{"interface":"front","average":{"1min":3.734,"5min":2.8980000000000001,"15min":2.5390000000000001},"min":{"1min":0.51500000000000001,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":12.021000000000001,"5min":24.524000000000001,"15min":24.524000000000001},"last":1.429}]}]},{"osd":0,"up_from":94,"seq":403726926018,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":489372,"kb_used_data":4948,"kb_used_omap":7,"kb_used_meta":484408,"kb_avail":93255780,"statfs":{"total":95995035648,"available":95493918720,"internally_reserved":0,"allocated":5066752,"data_stored":3216837,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":496033809},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:36:03 2023","interfaces":[{"interface":"back","average":{"1min":2.6989999999999998,"5min":3.2810000000000001,"15min":3.6850000000000001},"min":{"1min":0.23000000000000001,"5min":0.183,"15min":0.17999999999999999},"max":{"1min":11.554,"5min":42.000999999999998,"15min":42.000999999999998},"last":0.27800000000000002},{"interface":"front","average":{"1min":2.9199999999999999,"5min":3.278,"15min":4.0640000000000001},"min":{"1min":0.218,"5min":0.218,"15min":0.218},"max":{"1min":9.6059999999999999,"5min":23.036000000000001,"15min":23.036000000000001},"last":0.36399999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:35:37 2023","interfaces":[{"interface":"back","average":{"1min":3.5840000000000001,"5min":2.9820000000000002,"15min":2.4569999999999999},"min":{"1min":0.24099999999999999,"5min":0.193,"15min":0.185},"max":{"1min":11.064,"5min":22.460999999999999,"15min":77.448999999999998},"last":0.218},{"interface":"front","average":{"1min":3.4489999999999998,"5min":2.8570000000000002,"15min":2.294},"min":{"1min":0.25700000000000001,"5min":0.188,"15min":0.188},"max":{"1min":12.071,"5min":22.260000000000002,"15min":81.724999999999994},"last":0.254}]},{"osd":3,"last update":"Sat Sep 23 20:36:10 2023","interfaces":[{"interface":"back","average":{"1min":2.5390000000000001,"5min":3.1829999999999998,"15min":2.6520000000000001},"min":{"1min":0.30099999999999999,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":9.5730000000000004,"5min":22.486000000000001,"15min":81.400999999999996},"last":0.26800000000000002},{"interface":"front","average":{"1min":2.915,"5min":3.4849999999999999,"15min":2.7869999999999999},"min":{"1min":0.29499999999999998,"5min":0.25700000000000001,"15min":0.23699999999999999},"max":{"1min":11.58,"5min":22.867999999999999,"15min":81.137},"last":0.33500000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:35:37 2023","interfaces":[{"interface":"back","average":{"1min":3.331,"5min":2.1749999999999998,"15min":1.9350000000000001},"min":{"1min":0.32000000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":11.068,"5min":12.587999999999999,"15min":23.876999999999999},"last":0.442},{"interface":"front","average":{"1min":2.7650000000000001,"5min":2.347,"15min":1.8160000000000001},"min":{"1min":0.316,"5min":0.308,"15min":0.29299999999999998},"max":{"1min":11.458,"5min":23.001000000000001,"15min":23.681999999999999},"last":0.38500000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:35:37 2023","interfaces":[{"interface":"back","average":{"1min":2.5960000000000001,"5min":2.2360000000000002,"15min":2.129},"min":{"1min":0.35099999999999998,"5min":0.34399999999999997,"15min":0.34200000000000003},"max":{"1min":11.497999999999999,"5min":22.969999999999999,"15min":24.198},"last":0.46899999999999997},{"interface":"front","average":{"1min":2.8170000000000002,"5min":2.3199999999999998,"15min":1.9970000000000001},"min":{"1min":0.38300000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":11.974,"5min":22.236000000000001,"15min":22.236000000000001},"last":0.54300000000000004}]},{"osd":6,"last update":"Sat Sep 23 20:35:37 2023","interfaces":[{"interface":"back","average":{"1min":3.1640000000000001,"5min":2.4590000000000001,"15min":1.923},"min":{"1min":0.38,"5min":0.32300000000000001,"15min":0.318},"max":{"1min":11.526999999999999,"5min":22.934999999999999,"15min":24.408000000000001},"last":0.52600000000000002},{"interface":"front","average":{"1min":2.8010000000000002,"5min":2.399,"15min":1.841},"min":{"1min":0.35799999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":12.015000000000001,"5min":22.164999999999999,"15min":25.271000000000001},"last":0.45400000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:35:37 2023","interfaces":[{"interface":"back","average":{"1min":3.1899999999999999,"5min":2.5630000000000002,"15min":2.0459999999999998},"min":{"1min":0.40200000000000002,"5min":0.32200000000000001,"15min":0.30599999999999999},"max":{"1min":12.042,"5min":22.206,"15min":25.359000000000002},"last":0.48599999999999999},{"interface":"front","average":{"1min":3.9740000000000002,"5min":2.6230000000000002,"15min":2.3319999999999999},"min":{"1min":0.36199999999999999,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":11.112,"5min":22.422999999999998,"15min":24.669},"last":0.51000000000000001}]}]},{"osd":2,"up_from":104,"seq":446676598966,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":567124,"kb_used_data":78028,"kb_used_omap":11,"kb_used_meta":489076,"kb_avail":93178028,"statfs":{"total":95995035648,"available":95414300672,"internally_reserved":0,"allocated":79900672,"data_stored":78055605,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11878,"internal_metadata":500814234},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:35:47 2023","interfaces":[{"interface":"back","average":{"1min":3.944,"5min":4.274,"15min":2.5819999999999999},"min":{"1min":0.25800000000000001,"5min":0.185,"15min":0.17999999999999999},"max":{"1min":14.124000000000001,"5min":82.424999999999997,"15min":82.424999999999997},"last":1.522},{"interface":"front","average":{"1min":4.4859999999999998,"5min":3.7890000000000001,"15min":2.3460000000000001},"min":{"1min":0.24199999999999999,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":14.055,"5min":82.524000000000001,"15min":82.524000000000001},"last":1.5429999999999999}]},{"osd":1,"last update":"Sat Sep 23 20:35:53 2023","interfaces":[{"interface":"back","average":{"1min":3.5960000000000001,"5min":2.819,"15min":4.173},"min":{"1min":0.29299999999999998,"5min":0.22900000000000001,"15min":0.221},"max":{"1min":11.227,"5min":15.956,"15min":16.280000000000001},"last":0.50600000000000001},{"interface":"front","average":{"1min":5.9560000000000004,"5min":3.3959999999999999,"15min":3.1240000000000001},"min":{"1min":0.46400000000000002,"5min":0.248,"15min":0.248},"max":{"1min":21.585999999999999,"5min":21.585999999999999,"15min":30.565000000000001},"last":0.56399999999999995}]},{"osd":3,"last update":"Sat Sep 23 20:36:15 2023","interfaces":[{"interface":"back","average":{"1min":2.52,"5min":2.8799999999999999,"15min":2.0489999999999999},"min":{"1min":0.31900000000000001,"5min":0.30199999999999999,"15min":0.19800000000000001},"max":{"1min":11.129,"5min":15.804,"15min":16.331},"last":0.57799999999999996},{"interface":"front","average":{"1min":2.8100000000000001,"5min":3.2250000000000001,"15min":2.403},"min":{"1min":0.33200000000000002,"5min":0.28999999999999998,"15min":0.23899999999999999},"max":{"1min":13.148999999999999,"5min":15.266,"15min":30.704999999999998},"last":0.53100000000000003}]},{"osd":4,"last update":"Sat Sep 23 20:35:47 2023","interfaces":[{"interface":"back","average":{"1min":3.3730000000000002,"5min":2.1440000000000001,"15min":1.754},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":9.2430000000000003,"5min":9.2430000000000003,"15min":16.797999999999998},"last":0.64700000000000002},{"interface":"front","average":{"1min":3.1819999999999999,"5min":2.5110000000000001,"15min":1.903},"min":{"1min":0.307,"5min":0.30499999999999999,"15min":0.27600000000000002},"max":{"1min":11.044,"5min":15.853999999999999,"15min":15.853999999999999},"last":0.61899999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:35:47 2023","interfaces":[{"interface":"back","average":{"1min":3.3690000000000002,"5min":2.2069999999999999,"15min":2.2869999999999999},"min":{"1min":0.36599999999999999,"5min":0.314,"15min":0.309},"max":{"1min":9.1799999999999997,"5min":9.1799999999999997,"15min":95.393000000000001},"last":0.71199999999999997},{"interface":"front","average":{"1min":3.1920000000000002,"5min":2.77,"15min":2.5049999999999999},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.32300000000000001},"max":{"1min":10.856999999999999,"5min":15.885,"15min":95.414000000000001},"last":0.69999999999999996}]},{"osd":6,"last update":"Sat Sep 23 20:35:47 2023","interfaces":[{"interface":"back","average":{"1min":3.0680000000000001,"5min":2.6560000000000001,"15min":1.9890000000000001},"min":{"1min":0.37,"5min":0.35799999999999998,"15min":0.32100000000000001},"max":{"1min":11.253,"5min":15.914,"15min":15.914},"last":0.73599999999999999},{"interface":"front","average":{"1min":2.968,"5min":2.3570000000000002,"15min":1.9670000000000001},"min":{"1min":0.49099999999999999,"5min":0.318,"15min":0.316},"max":{"1min":10.811,"5min":11.523,"15min":30.670999999999999},"last":0.65900000000000003}]},{"osd":7,"last update":"Sat Sep 23 20:35:47 2023","interfaces":[{"interface":"back","average":{"1min":2.7730000000000001,"5min":2.306,"15min":2.0259999999999998},"min":{"1min":0.439,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":10.231,"5min":12.734999999999999,"15min":30.741},"last":0.68999999999999995},{"interface":"front","average":{"1min":3.2250000000000001,"5min":2.3170000000000002,"15min":2.0089999999999999},"min":{"1min":0.42999999999999999,"5min":0.38,"15min":0.38},"max":{"1min":9.2720000000000002,"5min":10.861000000000001,"15min":16.719000000000001},"last":0.75}]}]},{"osd":3,"up_from":109,"seq":468151435440,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":716436,"kb_used_data":172492,"kb_used_omap":4,"kb_used_meta":543931,"kb_avail":93028716,"statfs":{"total":95995035648,"available":95261405184,"internally_reserved":0,"allocated":176631808,"data_stored":174792507,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4514,"internal_metadata":556985950},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:36:12 2023","interfaces":[{"interface":"back","average":{"1min":1.3129999999999999,"5min":2.992,"15min":2.3410000000000002},"min":{"1min":0.22500000000000001,"5min":0.191,"15min":0.186},"max":{"1min":7.0309999999999997,"5min":17.129000000000001,"15min":67.305999999999997},"last":0.373},{"interface":"front","average":{"1min":1.734,"5min":2.8340000000000001,"15min":2.2090000000000001},"min":{"1min":0.28399999999999997,"5min":0.22600000000000001,"15min":0.19500000000000001},"max":{"1min":9.8520000000000003,"5min":12.502000000000001,"15min":67.403999999999996},"last":0.41699999999999998}]},{"osd":1,"last update":"Sat Sep 23 20:35:56 2023","interfaces":[{"interface":"back","average":{"1min":1.7689999999999999,"5min":2.456,"15min":2.8769999999999998},"min":{"1min":0.251,"5min":0.222,"15min":0.222},"max":{"1min":9.8940000000000001,"5min":16.189,"15min":16.189},"last":0.34100000000000003},{"interface":"front","average":{"1min":2.9140000000000001,"5min":2.5619999999999998,"15min":3.0219999999999998},"min":{"1min":0.34599999999999997,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":13.727,"5min":32.603999999999999,"15min":32.603999999999999},"last":0.35499999999999998}]},{"osd":2,"last update":"Sat Sep 23 20:36:12 2023","interfaces":[{"interface":"back","average":{"1min":1.1859999999999999,"5min":3.1070000000000002,"15min":2.3140000000000001},"min":{"1min":0.30399999999999999,"5min":0.23799999999999999,"15min":0.22800000000000001},"max":{"1min":9.6820000000000004,"5min":32.857999999999997,"15min":32.857999999999997},"last":0.39100000000000001},{"interface":"front","average":{"1min":1.0660000000000001,"5min":2.7490000000000001,"15min":2.2050000000000001},"min":{"1min":0.311,"5min":0.246,"15min":0.20899999999999999},"max":{"1min":3.4630000000000001,"5min":24.146999999999998,"15min":24.146999999999998},"last":0.32200000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:36:12 2023","interfaces":[{"interface":"back","average":{"1min":0.76400000000000001,"5min":1.661,"15min":1.7090000000000001},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.29799999999999999},"max":{"1min":2.3109999999999999,"5min":12.151,"15min":46.042999999999999},"last":0.625},{"interface":"front","average":{"1min":1.1659999999999999,"5min":2.125,"15min":2.0939999999999999},"min":{"1min":0.376,"5min":0.29399999999999998,"15min":0.28100000000000003},"max":{"1min":3.5110000000000001,"5min":14.422000000000001,"15min":46.579999999999998},"last":0.51700000000000002}]},{"osd":5,"last update":"Sat Sep 23 20:36:12 2023","interfaces":[{"interface":"back","average":{"1min":0.77800000000000002,"5min":1.9039999999999999,"15min":1.9510000000000001},"min":{"1min":0.32300000000000001,"5min":0.32200000000000001,"15min":0.311},"max":{"1min":2.383,"5min":13.228999999999999,"15min":45.292000000000002},"last":0.66300000000000003},{"interface":"front","average":{"1min":1.272,"5min":1.8500000000000001,"15min":1.806},"min":{"1min":0.41299999999999998,"5min":0.32000000000000001,"15min":0.313},"max":{"1min":9.0839999999999996,"5min":10.300000000000001,"15min":44.679000000000002},"last":0.53300000000000003}]},{"osd":6,"last update":"Sat Sep 23 20:36:12 2023","interfaces":[{"interface":"back","average":{"1min":1.28,"5min":1.851,"15min":1.833},"min":{"1min":0.437,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":9.0470000000000006,"5min":12.464,"15min":17.524999999999999},"last":0.73299999999999998},{"interface":"front","average":{"1min":1.2330000000000001,"5min":2.0510000000000002,"15min":1.847},"min":{"1min":0.39800000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":4.2450000000000001,"5min":14.464,"15min":19.524000000000001},"last":0.58599999999999997}]},{"osd":7,"last update":"Sat Sep 23 20:36:12 2023","interfaces":[{"interface":"back","average":{"1min":1.556,"5min":2.0990000000000002,"15min":2.0209999999999999},"min":{"1min":0.41499999999999998,"5min":0.36899999999999999,"15min":0.34999999999999998},"max":{"1min":5.8860000000000001,"5min":14.521000000000001,"15min":19.571999999999999},"last":0.747},{"interface":"front","average":{"1min":0.94799999999999995,"5min":1.877,"15min":2.0179999999999998},"min":{"1min":0.45200000000000001,"5min":0.42599999999999999,"15min":0.371},"max":{"1min":2.3580000000000001,"5min":12.186,"15min":45.387999999999998},"last":0.57199999999999995}]}]},{"osd":4,"up_from":30,"seq":128849019193,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":265684,"kb_used_data":3852,"kb_used_omap":0,"kb_used_meta":261824,"kb_avail":93479468,"statfs":{"total":95995035648,"available":95722975232,"internally_reserved":0,"allocated":3944448,"data_stored":2790346,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":268107776},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:35:40 2023","interfaces":[{"interface":"back","average":{"1min":2.2799999999999998,"5min":1.6579999999999999,"15min":1.5229999999999999},"min":{"1min":0.27300000000000002,"5min":0.255,"15min":0.222},"max":{"1min":5.2359999999999998,"5min":8.0800000000000001,"15min":20.411999999999999},"last":0.32900000000000001},{"interface":"front","average":{"1min":2.3849999999999998,"5min":1.835,"15min":1.6739999999999999},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.253},"max":{"1min":5.2229999999999999,"5min":10.675000000000001,"15min":20.454999999999998},"last":0.35199999999999998}]},{"osd":1,"last update":"Sat Sep 23 20:36:01 2023","interfaces":[{"interface":"back","average":{"1min":1.5649999999999999,"5min":1.7969999999999999,"15min":1.9370000000000001},"min":{"1min":0.32000000000000001,"5min":0.27600000000000002,"15min":0.27300000000000002},"max":{"1min":7.8600000000000003,"5min":11.303000000000001,"15min":11.303000000000001},"last":0.64900000000000002},{"interface":"front","average":{"1min":1.7809999999999999,"5min":1.583,"15min":1.871},"min":{"1min":0.30099999999999999,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":10.454000000000001,"5min":11.287000000000001,"15min":11.287000000000001},"last":0.38600000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:35:53 2023","interfaces":[{"interface":"back","average":{"1min":2.702,"5min":1.9570000000000001,"15min":1.798},"min":{"1min":0.32300000000000001,"5min":0.311,"15min":0.26600000000000001},"max":{"1min":10.416,"5min":10.416,"15min":32.363},"last":0.46600000000000003},{"interface":"front","average":{"1min":2.9470000000000001,"5min":2.21,"15min":2.0950000000000002},"min":{"1min":0.41999999999999998,"5min":0.28999999999999998,"15min":0.26800000000000002},"max":{"1min":8.4939999999999998,"5min":14.614000000000001,"15min":34.042999999999999},"last":0.505}]},{"osd":3,"last update":"Sat Sep 23 20:36:10 2023","interfaces":[{"interface":"back","average":{"1min":2.4630000000000001,"5min":1.9359999999999999,"15min":1.9710000000000001},"min":{"1min":0.372,"5min":0.34799999999999998,"15min":0.29099999999999998},"max":{"1min":9.5239999999999991,"5min":9.5239999999999991,"15min":32.332999999999998},"last":0.42699999999999999},{"interface":"front","average":{"1min":2.262,"5min":2.1509999999999998,"15min":2.044},"min":{"1min":0.38300000000000001,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":8.2240000000000002,"5min":15.991,"15min":32.399999999999999},"last":0.56399999999999995}]},{"osd":5,"last update":"Sat Sep 23 20:35:32 2023","interfaces":[{"interface":"back","average":{"1min":0.67800000000000005,"5min":0.72199999999999998,"15min":1.5409999999999999},"min":{"1min":0.26800000000000002,"5min":0.25800000000000001,"15min":0.247},"max":{"1min":2.9369999999999998,"5min":5.4340000000000002,"15min":82.866},"last":0.79000000000000004},{"interface":"front","average":{"1min":0.61799999999999999,"5min":0.79400000000000004,"15min":1.4370000000000001},"min":{"1min":0.33700000000000002,"5min":0.30299999999999999,"15min":0.27800000000000002},"max":{"1min":2.7509999999999999,"5min":6.8520000000000003,"15min":82.546000000000006},"last":0.58899999999999997}]},{"osd":6,"last update":"Sat Sep 23 20:35:53 2023","interfaces":[{"interface":"back","average":{"1min":1.018,"5min":1.028,"15min":1.389},"min":{"1min":0.312,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":3.008,"5min":10.178000000000001,"15min":41.371000000000002},"last":0.67200000000000004},{"interface":"front","average":{"1min":1.097,"5min":0.85099999999999998,"15min":1.27},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.27200000000000002},"max":{"1min":4.2949999999999999,"5min":5.4619999999999997,"15min":41.140999999999998},"last":0.84099999999999997}]},{"osd":7,"last update":"Sat Sep 23 20:36:10 2023","interfaces":[{"interface":"back","average":{"1min":0.81499999999999995,"5min":1.028,"15min":1.4419999999999999},"min":{"1min":0.377,"5min":0.34000000000000002,"15min":0.33600000000000002},"max":{"1min":2.4369999999999998,"5min":6.8849999999999998,"15min":16.427},"last":0.72099999999999997},{"interface":"front","average":{"1min":0.68300000000000005,"5min":0.79100000000000004,"15min":1.298},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.307},"max":{"1min":2.0550000000000002,"5min":6.3970000000000002,"15min":19.018999999999998},"last":0.86399999999999999}]}]},{"osd":5,"up_from":39,"seq":167503724854,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":263568,"kb_used_data":4104,"kb_used_omap":0,"kb_used_meta":259456,"kb_avail":93481584,"statfs":{"total":95995035648,"available":95725142016,"internally_reserved":0,"allocated":4202496,"data_stored":3049120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":265682944},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:35:42 2023","interfaces":[{"interface":"back","average":{"1min":4.7649999999999997,"5min":2.125,"15min":1.6919999999999999},"min":{"1min":0.29599999999999999,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":18.16,"5min":18.16,"15min":32.475000000000001},"last":0.52500000000000002},{"interface":"front","average":{"1min":3.3809999999999998,"5min":1.837,"15min":1.615},"min":{"1min":0.33900000000000002,"5min":0.26300000000000001,"15min":0.20599999999999999},"max":{"1min":9.0570000000000004,"5min":9.0570000000000004,"15min":32.457000000000001},"last":0.58799999999999997}]},{"osd":1,"last update":"Sat Sep 23 20:35:59 2023","interfaces":[{"interface":"back","average":{"1min":1.675,"5min":1.6180000000000001,"15min":1.909},"min":{"1min":0.26300000000000001,"5min":0.253,"15min":0.253},"max":{"1min":4.5910000000000002,"5min":9.3539999999999992,"15min":9.3539999999999992},"last":0.61599999999999999},{"interface":"front","average":{"1min":3.375,"5min":1.879,"15min":1.667},"min":{"1min":0.41899999999999998,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":8.4220000000000006,"5min":9.3900000000000006,"15min":9.3900000000000006},"last":0.441}]},{"osd":2,"last update":"Sat Sep 23 20:35:42 2023","interfaces":[{"interface":"back","average":{"1min":2.4020000000000001,"5min":2.1070000000000002,"15min":1.8440000000000001},"min":{"1min":0.32300000000000001,"5min":0.27400000000000002,"15min":0.26700000000000002},"max":{"1min":11.347,"5min":27.603999999999999,"15min":34.514000000000003},"last":0.36199999999999999},{"interface":"front","average":{"1min":2.4159999999999999,"5min":2.4060000000000001,"15min":1.8720000000000001},"min":{"1min":0.34100000000000003,"5min":0.255,"15min":0.255},"max":{"1min":13.545,"5min":27.626999999999999,"15min":32.515000000000001},"last":0.54100000000000004}]},{"osd":3,"last update":"Sat Sep 23 20:36:09 2023","interfaces":[{"interface":"back","average":{"1min":4.0209999999999999,"5min":2.7290000000000001,"15min":2.0459999999999998},"min":{"1min":0.35799999999999998,"5min":0.29599999999999999,"15min":0.25700000000000001},"max":{"1min":36.716000000000001,"5min":36.716000000000001,"15min":36.716000000000001},"last":0.55900000000000005},{"interface":"front","average":{"1min":2.0710000000000002,"5min":2.2200000000000002,"15min":1.766},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.28799999999999998},"max":{"1min":9.6959999999999997,"5min":10.577999999999999,"15min":12.005000000000001},"last":0.72899999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:35:33 2023","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.67800000000000005,"15min":1.2989999999999999},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.25800000000000001},"max":{"1min":2.2280000000000002,"5min":6.282,"15min":28.015999999999998},"last":0.42299999999999999},{"interface":"front","average":{"1min":0.84899999999999998,"5min":0.69899999999999995,"15min":1.087},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":3.0659999999999998,"5min":3.0659999999999998,"15min":12.300000000000001},"last":0.57399999999999995}]},{"osd":6,"last update":"Sat Sep 23 20:35:48 2023","interfaces":[{"interface":"back","average":{"1min":0.77500000000000002,"5min":0.70699999999999996,"15min":1.2190000000000001},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":4.3300000000000001,"5min":4.3330000000000002,"15min":12.890000000000001},"last":0.496},{"interface":"front","average":{"1min":0.83699999999999997,"5min":0.73799999999999999,"15min":1.1879999999999999},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.34499999999999997},"max":{"1min":4.2400000000000002,"5min":5.1680000000000001,"15min":16.265999999999998},"last":0.63200000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:36:09 2023","interfaces":[{"interface":"back","average":{"1min":0.70999999999999996,"5min":0.63,"15min":1.2310000000000001},"min":{"1min":0.33600000000000002,"5min":0.32100000000000001,"15min":0.29699999999999999},"max":{"1min":2.8180000000000001,"5min":2.8180000000000001,"15min":36.621000000000002},"last":0.66600000000000004},{"interface":"front","average":{"1min":0.62,"5min":0.66400000000000003,"15min":1.135},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":2.6360000000000001,"5min":5.093,"15min":37.331000000000003},"last":0.46300000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4722,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1360,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8206,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7156,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3154,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":265224192,"data_stored":265224192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":122421248,"data_stored":122421248,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":185597952,"data_stored":185597952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:36:20.602 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-09-23T20:36:20.602 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:36:20.602 INFO:tasks.thrashosds.thrasher:inject_pause on osd.6 2023-09-23T20:36:20.602 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T20:36:20.602 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T20:36:20.603 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 config set filestore_inject_stall 3 2023-09-23T20:36:20.609 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:36:20.614 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:20.614 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:36:20.715 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:36:20.774 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:20.843 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:36:20.847 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:20.848 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:36:20.949 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:36:20.962 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:36:20.964 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:36:21.012 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 5 16 13833 13817 172.579 154.375 0.00880835 0.00578429 2023-09-23T20:36:21.015 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:21.129 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:36:21.135 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:21.136 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:36:21.237 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:36:21.279 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:36:21.280 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:21.311 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T20:36:21.346 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:36:21.351 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:21.351 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:36:21.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:21 smithi018 ceph-mon[179647]: pgmap v896: 119 pgs: 119 active+clean; 171 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 22 MiB/s wr, 349 op/s 2023-09-23T20:36:21.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:21 smithi018 ceph-mon[179647]: from='client.55714 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:36:21.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:21 smithi018 ceph-mon[184144]: pgmap v896: 119 pgs: 119 active+clean; 171 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 22 MiB/s wr, 349 op/s 2023-09-23T20:36:21.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:21 smithi018 ceph-mon[184144]: from='client.55714 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:36:21.452 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:36:21.551 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:21 smithi149 ceph-mon[170394]: pgmap v896: 119 pgs: 119 active+clean; 171 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 22 MiB/s wr, 349 op/s 2023-09-23T20:36:21.551 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:21 smithi149 ceph-mon[170394]: from='client.55714 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:36:21.594 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:22.012 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 6 16 16158 16142 168.035 145.312 0.00776357 0.00594453 2023-09-23T20:36:23.013 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 7 16 18758 18742 167.242 162.5 0.00453456 0.00597276 2023-09-23T20:36:23.029 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:36:23.045 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:23.046 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:36:23.146 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:36:23.211 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:23.299 INFO:teuthology.orchestra.run.smithi149.stdout:{ 2023-09-23T20:36:23.300 INFO:teuthology.orchestra.run.smithi149.stdout: "success": "" 2023-09-23T20:36:23.300 INFO:teuthology.orchestra.run.smithi149.stdout:} 2023-09-23T20:36:23.641 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:23 smithi149 ceph-mon[170394]: pgmap v897: 119 pgs: 119 active+clean; 712 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s 2023-09-23T20:36:23.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:23 smithi018 ceph-mon[179647]: pgmap v897: 119 pgs: 119 active+clean; 712 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s 2023-09-23T20:36:23.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:23 smithi018 ceph-mon[184144]: pgmap v897: 119 pgs: 119 active+clean; 712 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s 2023-09-23T20:36:24.013 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 8 16 20931 20915 163.314 135.812 0.00187452 0.00605293 2023-09-23T20:36:24.077 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:36:24.126 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:36:24.129 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:24.129 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:36:24.230 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:36:24.265 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:36:24.288 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:24.328 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:36:24.330 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:24.331 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:36:24.432 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:36:24.498 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:25.013 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 9 16 22967 22951 159.307 127.25 0.00437216 0.00627099 2023-09-23T20:36:25.353 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:36:24 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:36:24] "GET /metrics HTTP/1.1" 200 36435 "" "Prometheus/2.43.0" 2023-09-23T20:36:25.608 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:36:25.614 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:25 smithi018 ceph-mon[179647]: pgmap v898: 119 pgs: 119 active+clean; 760 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2023-09-23T20:36:25.615 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:25 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:36:25.615 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:25 smithi018 ceph-mon[184144]: pgmap v898: 119 pgs: 119 active+clean; 760 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2023-09-23T20:36:25.616 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:25 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:36:25.616 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:25.617 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:36:25.717 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:36:25.832 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:25 smithi149 ceph-mon[170394]: pgmap v898: 119 pgs: 119 active+clean; 760 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2023-09-23T20:36:25.832 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:25 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:36:25.877 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:25.935 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:36:25.940 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:25.940 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:36:26.013 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 10 16 24806 24790 154.871 114.938 0.00931979 0.00644872 2023-09-23T20:36:26.019 INFO:teuthology.orchestra.run.smithi018.stdout:ERROR: (22) Invalid argument 2023-09-23T20:36:26.020 INFO:teuthology.orchestra.run.smithi018.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:36:26.041 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:36:26.098 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:27.013 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 11 16 26887 26871 152.614 130.062 0.00993988 0.00654737 2023-09-23T20:36:27.016 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:36:27.022 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:27.022 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:36:27.123 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:36:27.184 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:27.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:27 smithi018 ceph-mon[184144]: pgmap v899: 119 pgs: 119 active+clean; 1.4 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 143 MiB/s wr, 2.28k op/s 2023-09-23T20:36:27.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:27 smithi018 ceph-mon[179647]: pgmap v899: 119 pgs: 119 active+clean; 1.4 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 143 MiB/s wr, 2.28k op/s 2023-09-23T20:36:27.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:27 smithi149 ceph-mon[170394]: pgmap v899: 119 pgs: 119 active+clean; 1.4 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 143 MiB/s wr, 2.28k op/s 2023-09-23T20:36:28.013 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 12 16 29075 29059 151.289 136.75 0.00353196 0.00660399 2023-09-23T20:36:28.649 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 5, 7, 3, 2, 6, 4] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T20:36:28.649 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:36:28.649 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_3 2023-09-23T20:36:28.650 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:36:29.017 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 13 16 31183 31167 149.786 131.75 0.00447514 0.00667232 2023-09-23T20:36:29.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:29 smithi018 ceph-mon[184144]: pgmap v900: 119 pgs: 119 active+clean; 1.4 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 145 MiB/s wr, 2.32k op/s 2023-09-23T20:36:29.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:29 smithi018 ceph-mon[179647]: pgmap v900: 119 pgs: 119 active+clean; 1.4 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 145 MiB/s wr, 2.32k op/s 2023-09-23T20:36:29.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:29 smithi149 ceph-mon[170394]: pgmap v900: 119 pgs: 119 active+clean; 1.4 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 145 MiB/s wr, 2.32k op/s 2023-09-23T20:36:29.748 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T20:36:30.014 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 14 16 33081 33065 147.56 118.625 0.00659264 0.00677146 2023-09-23T20:36:30.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:30 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:36:30.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:30 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:36:30.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:30 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:36:31.013 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 15 16 34977 34961 145.622 118.5 0.00156285 0.00684631 2023-09-23T20:36:31.390 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:36:31.396 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:31.397 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:36:31.497 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:36:31.590 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:31.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:31 smithi149 ceph-mon[170394]: pgmap v901: 119 pgs: 119 active+clean; 1.6 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 139 MiB/s wr, 2.22k op/s 2023-09-23T20:36:31.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:31 smithi018 ceph-mon[179647]: pgmap v901: 119 pgs: 119 active+clean; 1.6 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 139 MiB/s wr, 2.22k op/s 2023-09-23T20:36:31.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:31 smithi018 ceph-mon[184144]: pgmap v901: 119 pgs: 119 active+clean; 1.6 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 139 MiB/s wr, 2.22k op/s 2023-09-23T20:36:32.015 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 16 16 36459 36443 142.296 92.625 0.00588046 0.00702285 2023-09-23T20:36:32.592 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:36:32.600 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:32.601 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:36:32.702 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:36:32.773 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:32.852 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:36:33.015 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 17 16 38010 37994 139.628 96.9375 0.00961553 0.00715617 2023-09-23T20:36:33.018 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:36:33.022 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:36:33.030 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:33.030 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:36:33.131 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:36:33.210 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:33.343 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:36:33.351 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:33.351 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:36:33.452 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:36:33.522 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:33.631 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:33 smithi018 ceph-mon[184144]: pgmap v902: 119 pgs: 119 active+clean; 2.0 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 159 MiB/s wr, 2.54k op/s 2023-09-23T20:36:33.631 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:33 smithi018 ceph-mon[179647]: pgmap v902: 119 pgs: 119 active+clean; 2.0 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 159 MiB/s wr, 2.54k op/s 2023-09-23T20:36:33.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:33 smithi149 ceph-mon[170394]: pgmap v902: 119 pgs: 119 active+clean; 2.0 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 159 MiB/s wr, 2.54k op/s 2023-09-23T20:36:34.015 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 18 16 39579 39563 137.319 98.0625 0.00533399 0.00727823 2023-09-23T20:36:34.657 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:36:34.663 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:34.663 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:36:34.764 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:36:34.813 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:34.868 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:36:34.874 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:34.875 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:36:34.975 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:36:35.016 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 19 16 41086 41070 135.043 94.1875 0.0111108 0.00739973 2023-09-23T20:36:35.022 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:35.075 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:36:35.079 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:35.079 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:36:35.176 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:36:34 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:36:34] "GET /metrics HTTP/1.1" 200 36500 "" "Prometheus/2.43.0" 2023-09-23T20:36:35.180 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:36:35.202 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:36:35.219 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:35.260 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:36:35.272 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:36:35.277 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:35.278 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:36:35.293 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:36:35.339 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:36:35.378 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:36:35.379 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:36:35.420 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:36:35.427 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:35 smithi018 ceph-mon[179647]: pgmap v903: 119 pgs: 119 active+clean; 2.1 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 117 MiB/s wr, 1.87k op/s 2023-09-23T20:36:35.428 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:35 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1571200272' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:36:35.428 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:35 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:36:35.428 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:35 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1661326808' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:36:35.444 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:35.462 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:36:35.470 INFO:teuthology.orchestra.run.smithi018.stderr:noscrub is unset 2023-09-23T20:36:35.510 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:36:35.521 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:36:35.521 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:36:35.521 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:36:35.521 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:36:35.521 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:36:35.522 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:36:35.522 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:36:35.522 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:36:35.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:35 smithi149 ceph-mon[170394]: pgmap v903: 119 pgs: 119 active+clean; 2.1 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 117 MiB/s wr, 1.87k op/s 2023-09-23T20:36:35.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:35 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1571200272' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:36:35.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:35 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:36:35.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:35 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1661326808' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:36:35.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:35 smithi018 ceph-mon[184144]: pgmap v903: 119 pgs: 119 active+clean; 2.1 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 117 MiB/s wr, 1.87k op/s 2023-09-23T20:36:35.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:35 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1571200272' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:36:35.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:35 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:36:35.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:35 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1661326808' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:36:36.016 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:2023-09-23T20:36:36.017020+0000 min lat: 0.00120245 max lat: 0.186851 avg lat: 0.00750647 2023-09-23T20:36:36.016 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T20:36:36.017 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 20 16 42593 42577 133 94.1875 0.00205336 0.00750647 2023-09-23T20:36:36.443 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:36 smithi018 ceph-mon[179647]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:36:36.443 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:36 smithi018 ceph-mon[179647]: failed to encode map e374 with expected crc 2023-09-23T20:36:36.443 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:36 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:36:36.443 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:36 smithi018 ceph-mon[179647]: osdmap e374: 8 total, 8 up, 8 in 2023-09-23T20:36:36.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:36 smithi149 ceph-mon[170394]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:36:36.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:36 smithi149 ceph-mon[170394]: failed to encode map e374 with expected crc 2023-09-23T20:36:36.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:36 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:36:36.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:36 smithi149 ceph-mon[170394]: osdmap e374: 8 total, 8 up, 8 in 2023-09-23T20:36:36.890 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:36 smithi018 ceph-mon[184144]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:36:36.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:36 smithi018 ceph-mon[184144]: failed to encode map e374 with expected crc 2023-09-23T20:36:36.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:36 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:36:36.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:36 smithi018 ceph-mon[184144]: osdmap e374: 8 total, 8 up, 8 in 2023-09-23T20:36:36.934 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:36:36.950 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:36.951 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:36:37.016 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 21 16 44110 44094 131.182 94.8125 0.00868995 0.00761845 2023-09-23T20:36:37.051 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:36:37.106 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:37.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:37 smithi018 ceph-mon[179647]: pgmap v905: 119 pgs: 119 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 114 MiB/s wr, 1.82k op/s 2023-09-23T20:36:37.643 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:37 smithi018 ceph-mon[184144]: pgmap v905: 119 pgs: 119 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 114 MiB/s wr, 1.82k op/s 2023-09-23T20:36:37.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:37 smithi149 ceph-mon[170394]: pgmap v905: 119 pgs: 119 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 114 MiB/s wr, 1.82k op/s 2023-09-23T20:36:38.017 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 22 16 45479 45463 129.108 85.5625 0.00461522 0.00773973 2023-09-23T20:36:38.166 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:36:38.183 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:38.184 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:36:38.285 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:36:38.323 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:38.369 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:36:38.379 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:38.380 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:36:38.481 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:36:38.500 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:36:38.511 INFO:teuthology.orchestra.run.smithi018.stderr:dumped all 2023-09-23T20:36:38.535 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:38.609 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:36:38.616 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:38.617 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:36:38.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:38 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:36:38.643 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:38 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:36:38.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:38 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:36:38.719 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:36:38.784 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:38.841 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:36:38.848 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:38.849 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:36:38.950 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:36:38.983 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2023-09-23T20:36:38.983 INFO:teuthology.orchestra.run.smithi018.stdout: "size": 20, 2023-09-23T20:36:38.984 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 600, 2023-09-23T20:36:38.984 INFO:teuthology.orchestra.run.smithi018.stdout: "ops": [ 2023-09-23T20:36:38.984 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:38.984 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.45161.0:136049 6.f 6:f1f0f499:::benchmark_data_smithi018_270558_object31310:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e258)", 2023-09-23T20:36:38.984 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:29:27.360919+0000", 2023-09-23T20:36:38.984 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 431.61448490599997, 2023-09-23T20:36:38.984 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.1770357090000001, 2023-09-23T20:36:38.984 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:36:38.985 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:36:38.985 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:36:38.985 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.45161", 2023-09-23T20:36:38.985 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3795614987", 2023-09-23T20:36:38.985 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 136049 2023-09-23T20:36:38.985 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:38.985 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:36:38.985 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:38.986 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:36:38.986 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.360919+0000", 2023-09-23T20:36:38.986 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:36:38.986 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:38.986 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:38.986 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:36:38.986 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.360919+0000", 2023-09-23T20:36:38.986 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:36:38.987 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:38.987 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:38.987 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:36:38.987 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.360917+0000", 2023-09-23T20:36:38.987 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:36:38.987 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:38.987 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:38.987 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:36:38.987 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.360925+0000", 2023-09-23T20:36:38.987 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.8480000000000001e-06 2023-09-23T20:36:38.987 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:38.987 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:38.988 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:36:38.988 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.360944+0000", 2023-09-23T20:36:38.988 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.8439000000000001e-05 2023-09-23T20:36:38.988 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:38.988 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:38.988 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:36:38.988 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.360954+0000", 2023-09-23T20:36:38.988 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0327e-05 2023-09-23T20:36:38.988 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:38.988 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:38.988 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:38.989 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.363434+0000", 2023-09-23T20:36:38.989 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0024801100000000002 2023-09-23T20:36:38.989 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:38.989 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:38.989 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:36:38.989 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.363450+0000", 2023-09-23T20:36:38.989 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.5359999999999999e-05 2023-09-23T20:36:38.989 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:38.989 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:38.989 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:38.990 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:28.342983+0000", 2023-09-23T20:36:38.990 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.97953384099999996 2023-09-23T20:36:38.990 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:38.990 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:38.990 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:36:38.990 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:28.342997+0000", 2023-09-23T20:36:38.991 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3641e-05 2023-09-23T20:36:38.991 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:38.991 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:38.991 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:38.991 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.518718+0000", 2023-09-23T20:36:38.991 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.175721432 2023-09-23T20:36:38.991 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:38.991 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:38.991 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:36:38.992 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.518815+0000", 2023-09-23T20:36:38.992 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.6764999999999999e-05 2023-09-23T20:36:38.992 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:38.992 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:38.992 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 3,4,6", 2023-09-23T20:36:38.992 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.519272+0000", 2023-09-23T20:36:38.992 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00045647899999999998 2023-09-23T20:36:38.992 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:38.992 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:38.992 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:36:38.993 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.537608+0000", 2023-09-23T20:36:38.993 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.018336363000000001 2023-09-23T20:36:38.993 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:38.993 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:38.993 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:36:38.993 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.537675+0000", 2023-09-23T20:36:38.993 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.7039999999999995e-05 2023-09-23T20:36:38.993 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:38.994 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:38.994 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:36:38.994 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.537777+0000", 2023-09-23T20:36:38.994 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000102147 2023-09-23T20:36:38.994 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:38.994 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:38.994 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:36:38.994 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.537821+0000", 2023-09-23T20:36:38.995 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.3547999999999997e-05 2023-09-23T20:36:38.995 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:38.995 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:38.995 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:36:38.995 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.537858+0000", 2023-09-23T20:36:38.995 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.7132999999999998e-05 2023-09-23T20:36:38.995 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:38.995 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:38.995 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:36:38.996 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.537955+0000", 2023-09-23T20:36:38.996 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.6804000000000004e-05 2023-09-23T20:36:38.996 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:38.996 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:36:38.996 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:38.996 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:38.996 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:38.996 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.45161.0:136051 6.f 6:fcaf49a8:::benchmark_data_smithi018_270558_object31312:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e258)", 2023-09-23T20:36:38.996 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:29:27.375493+0000", 2023-09-23T20:36:38.997 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 431.59991053300001, 2023-09-23T20:36:38.997 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.174356832, 2023-09-23T20:36:38.997 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:36:38.997 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:36:38.997 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:36:38.997 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.45161", 2023-09-23T20:36:38.997 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3795614987", 2023-09-23T20:36:38.997 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 136051 2023-09-23T20:36:38.997 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:38.997 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:36:38.997 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:38.997 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:36:38.998 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.375493+0000", 2023-09-23T20:36:38.998 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:36:38.998 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:38.998 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:38.998 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:36:38.998 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.375493+0000", 2023-09-23T20:36:38.998 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:36:38.998 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:38.998 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:38.998 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:36:38.998 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.375491+0000", 2023-09-23T20:36:38.999 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:36:38.999 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:38.999 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:38.999 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:36:38.999 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.375501+0000", 2023-09-23T20:36:38.999 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.149e-06 2023-09-23T20:36:38.999 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:38.999 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:38.999 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:36:38.999 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.375516+0000", 2023-09-23T20:36:38.999 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.5463000000000001e-05 2023-09-23T20:36:38.999 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.000 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.000 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:36:39.000 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.375528+0000", 2023-09-23T20:36:39.000 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1800999999999999e-05 2023-09-23T20:36:39.000 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.000 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.000 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:39.000 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.375579+0000", 2023-09-23T20:36:39.000 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.096e-05 2023-09-23T20:36:39.000 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.000 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.000 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:36:39.001 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.375592+0000", 2023-09-23T20:36:39.001 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.343e-05 2023-09-23T20:36:39.001 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.001 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.001 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:39.001 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:28.343022+0000", 2023-09-23T20:36:39.001 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.96742986500000006 2023-09-23T20:36:39.001 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.001 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.001 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:36:39.001 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:28.343030+0000", 2023-09-23T20:36:39.001 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.2400000000000007e-06 2023-09-23T20:36:39.002 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.002 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.002 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:39.002 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.530153+0000", 2023-09-23T20:36:39.002 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.187122427 2023-09-23T20:36:39.002 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.002 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.002 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:36:39.002 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.530266+0000", 2023-09-23T20:36:39.002 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00011287500000000001 2023-09-23T20:36:39.002 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.003 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.003 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 3,4,6", 2023-09-23T20:36:39.003 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.530349+0000", 2023-09-23T20:36:39.003 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.3065999999999994e-05 2023-09-23T20:36:39.003 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.003 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.003 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:36:39.003 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.537739+0000", 2023-09-23T20:36:39.003 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0073898840000000002 2023-09-23T20:36:39.003 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.003 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.004 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:36:39.004 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.537745+0000", 2023-09-23T20:36:39.004 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.5860000000000001e-06 2023-09-23T20:36:39.004 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.004 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.004 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:36:39.004 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.538012+0000", 2023-09-23T20:36:39.004 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000267018 2023-09-23T20:36:39.004 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.004 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.004 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:36:39.005 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.549701+0000", 2023-09-23T20:36:39.005 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.011688426 2023-09-23T20:36:39.005 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.005 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.005 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:36:39.005 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.549735+0000", 2023-09-23T20:36:39.005 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.4161e-05 2023-09-23T20:36:39.005 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.005 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.005 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:36:39.005 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.549850+0000", 2023-09-23T20:36:39.006 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00011539099999999999 2023-09-23T20:36:39.006 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:39.006 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:36:39.006 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:39.006 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.007 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.007 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.45161.0:136053 6.f 6:f2209241:::benchmark_data_smithi018_270558_object31314:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e258)", 2023-09-23T20:36:39.007 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:29:27.379666+0000", 2023-09-23T20:36:39.007 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 431.59573800499999, 2023-09-23T20:36:39.007 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.1768631190000001, 2023-09-23T20:36:39.007 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:36:39.007 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:36:39.007 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:36:39.007 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.45161", 2023-09-23T20:36:39.008 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3795614987", 2023-09-23T20:36:39.008 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 136053 2023-09-23T20:36:39.008 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.008 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:36:39.008 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.008 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:36:39.008 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.379666+0000", 2023-09-23T20:36:39.008 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:36:39.008 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.008 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.008 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:36:39.008 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.379666+0000", 2023-09-23T20:36:39.009 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:36:39.009 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.009 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.009 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:36:39.009 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.379664+0000", 2023-09-23T20:36:39.009 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999976 2023-09-23T20:36:39.009 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.009 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.009 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:36:39.009 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.379674+0000", 2023-09-23T20:36:39.009 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0271999999999999e-05 2023-09-23T20:36:39.009 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.010 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.010 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:36:39.010 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.379698+0000", 2023-09-23T20:36:39.010 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.3941999999999999e-05 2023-09-23T20:36:39.010 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.010 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.010 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:36:39.010 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.379711+0000", 2023-09-23T20:36:39.010 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2743e-05 2023-09-23T20:36:39.010 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.010 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.011 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:39.011 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.379865+0000", 2023-09-23T20:36:39.011 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00015445199999999999 2023-09-23T20:36:39.011 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.011 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.011 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:36:39.011 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.379881+0000", 2023-09-23T20:36:39.011 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.5423000000000001e-05 2023-09-23T20:36:39.011 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.011 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.011 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:39.011 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:28.343053+0000", 2023-09-23T20:36:39.012 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.963171993 2023-09-23T20:36:39.012 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.012 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.012 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:36:39.012 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:28.343061+0000", 2023-09-23T20:36:39.012 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.9480000000000008e-06 2023-09-23T20:36:39.012 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.012 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.012 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:39.012 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.534004+0000", 2023-09-23T20:36:39.012 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1909433570000001 2023-09-23T20:36:39.012 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.013 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.013 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:36:39.013 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.534089+0000", 2023-09-23T20:36:39.013 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.5394000000000003e-05 2023-09-23T20:36:39.013 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.013 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.013 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 3,4,6", 2023-09-23T20:36:39.013 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.534172+0000", 2023-09-23T20:36:39.013 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.2553999999999996e-05 2023-09-23T20:36:39.013 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.013 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.013 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:36:39.014 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.537964+0000", 2023-09-23T20:36:39.014 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0037922590000000001 2023-09-23T20:36:39.014 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.014 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.014 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:36:39.014 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.538056+0000", 2023-09-23T20:36:39.014 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.1833e-05 2023-09-23T20:36:39.014 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.014 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.014 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:36:39.014 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.538092+0000", 2023-09-23T20:36:39.015 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.6528e-05 2023-09-23T20:36:39.015 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.015 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.015 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:36:39.015 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.556365+0000", 2023-09-23T20:36:39.015 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.018272594 2023-09-23T20:36:39.015 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.015 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.015 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:36:39.015 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.556403+0000", 2023-09-23T20:36:39.015 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.8034999999999998e-05 2023-09-23T20:36:39.016 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.016 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.016 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:36:39.016 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.556529+0000", 2023-09-23T20:36:39.016 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00012598199999999999 2023-09-23T20:36:39.016 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:39.016 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:36:39.016 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:39.017 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.017 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.017 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.45161.0:136065 6.e 6:7df88343:::benchmark_data_smithi018_270558_object31326:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e258)", 2023-09-23T20:36:39.017 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:29:27.389482+0000", 2023-09-23T20:36:39.017 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 431.58592244099998, 2023-09-23T20:36:39.017 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.1285085910000001, 2023-09-23T20:36:39.017 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:36:39.017 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:36:39.017 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:36:39.017 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.45161", 2023-09-23T20:36:39.017 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3795614987", 2023-09-23T20:36:39.017 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 136065 2023-09-23T20:36:39.018 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.018 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:36:39.018 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.018 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:36:39.018 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.389482+0000", 2023-09-23T20:36:39.018 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:36:39.018 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.019 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.019 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:36:39.019 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.389482+0000", 2023-09-23T20:36:39.019 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:36:39.019 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.019 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.019 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:36:39.019 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.389480+0000", 2023-09-23T20:36:39.019 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:36:39.019 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.019 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.020 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:36:39.020 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.389488+0000", 2023-09-23T20:36:39.020 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.6120000000000001e-06 2023-09-23T20:36:39.020 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.020 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.020 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:36:39.020 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.389503+0000", 2023-09-23T20:36:39.020 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.5339999999999999e-05 2023-09-23T20:36:39.020 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.021 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.021 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:36:39.021 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.389516+0000", 2023-09-23T20:36:39.021 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3467e-05 2023-09-23T20:36:39.021 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.021 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.021 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:39.021 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.389586+0000", 2023-09-23T20:36:39.021 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.9797999999999998e-05 2023-09-23T20:36:39.022 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.022 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.022 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:36:39.022 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.389597+0000", 2023-09-23T20:36:39.022 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0961999999999999e-05 2023-09-23T20:36:39.022 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.022 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.022 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:39.022 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:28.343523+0000", 2023-09-23T20:36:39.022 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.95392566499999998 2023-09-23T20:36:39.023 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.023 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.023 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:36:39.023 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:28.343532+0000", 2023-09-23T20:36:39.023 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.8070000000000006e-06 2023-09-23T20:36:39.023 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.023 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.023 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:39.023 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.508816+0000", 2023-09-23T20:36:39.024 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1652842480000001 2023-09-23T20:36:39.024 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.024 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.024 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:36:39.024 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.511009+0000", 2023-09-23T20:36:39.024 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0021932660000000001 2023-09-23T20:36:39.024 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.024 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.024 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 4,5", 2023-09-23T20:36:39.025 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.511162+0000", 2023-09-23T20:36:39.025 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00015258099999999999 2023-09-23T20:36:39.025 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.025 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.025 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:36:39.025 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.516451+0000", 2023-09-23T20:36:39.025 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.005289261 2023-09-23T20:36:39.025 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.025 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.026 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:36:39.026 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.516483+0000", 2023-09-23T20:36:39.026 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.2236000000000001e-05 2023-09-23T20:36:39.026 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.026 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.026 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:36:39.026 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.517808+0000", 2023-09-23T20:36:39.026 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0013247910000000001 2023-09-23T20:36:39.027 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.027 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.027 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:36:39.027 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.517866+0000", 2023-09-23T20:36:39.027 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.7624000000000003e-05 2023-09-23T20:36:39.027 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.027 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.027 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:36:39.027 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.517990+0000", 2023-09-23T20:36:39.028 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000124411 2023-09-23T20:36:39.028 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:39.028 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:36:39.028 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:39.028 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.028 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.028 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.45161.0:136068 6.f 6:f6357533:::benchmark_data_smithi018_270558_object31329:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e258)", 2023-09-23T20:36:39.028 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:29:27.391642+0000", 2023-09-23T20:36:39.028 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 431.58376148100001, 2023-09-23T20:36:39.028 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.1651141979999999, 2023-09-23T20:36:39.029 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:36:39.029 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:36:39.029 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:36:39.029 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.45161", 2023-09-23T20:36:39.029 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3795614987", 2023-09-23T20:36:39.029 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 136068 2023-09-23T20:36:39.029 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.029 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:36:39.029 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.029 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:36:39.029 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.391642+0000", 2023-09-23T20:36:39.030 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:36:39.030 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.030 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.030 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:36:39.030 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.391642+0000", 2023-09-23T20:36:39.030 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:36:39.030 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.030 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.030 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:36:39.030 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.391641+0000", 2023-09-23T20:36:39.030 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.999999 2023-09-23T20:36:39.031 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.031 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.031 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:36:39.031 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.391648+0000", 2023-09-23T20:36:39.031 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.0350000000000004e-06 2023-09-23T20:36:39.031 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.031 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.031 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:36:39.031 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.391663+0000", 2023-09-23T20:36:39.031 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.4722000000000001e-05 2023-09-23T20:36:39.031 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.031 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.032 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:36:39.032 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.391672+0000", 2023-09-23T20:36:39.032 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.8359999999999998e-06 2023-09-23T20:36:39.032 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.032 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.032 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:39.032 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.391746+0000", 2023-09-23T20:36:39.032 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.3660999999999998e-05 2023-09-23T20:36:39.032 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.032 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.032 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:36:39.032 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.391758+0000", 2023-09-23T20:36:39.033 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.236e-05 2023-09-23T20:36:39.033 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.033 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.033 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:39.033 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:28.343081+0000", 2023-09-23T20:36:39.033 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.95132316299999997 2023-09-23T20:36:39.033 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.033 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.033 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:36:39.033 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:28.343089+0000", 2023-09-23T20:36:39.033 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.0809999999999994e-06 2023-09-23T20:36:39.033 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.034 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.034 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:39.034 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.538120+0000", 2023-09-23T20:36:39.034 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.195030437 2023-09-23T20:36:39.034 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.034 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.034 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:36:39.034 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.538205+0000", 2023-09-23T20:36:39.034 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.5142999999999994e-05 2023-09-23T20:36:39.034 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.034 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.035 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 3,4,6", 2023-09-23T20:36:39.035 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.538283+0000", 2023-09-23T20:36:39.035 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.8652999999999994e-05 2023-09-23T20:36:39.035 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.035 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.035 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:36:39.035 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.549604+0000", 2023-09-23T20:36:39.035 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.011320436 2023-09-23T20:36:39.035 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.035 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.035 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:36:39.035 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.549613+0000", 2023-09-23T20:36:39.036 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.0680000000000003e-06 2023-09-23T20:36:39.036 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.036 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.036 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:36:39.036 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.549680+0000", 2023-09-23T20:36:39.036 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.6829999999999995e-05 2023-09-23T20:36:39.036 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.036 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.036 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:36:39.036 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.556587+0000", 2023-09-23T20:36:39.036 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0069069190000000001 2023-09-23T20:36:39.036 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.037 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.037 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:36:39.037 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.556645+0000", 2023-09-23T20:36:39.037 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.8740999999999999e-05 2023-09-23T20:36:39.037 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.037 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.037 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:36:39.037 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.556757+0000", 2023-09-23T20:36:39.037 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000111208 2023-09-23T20:36:39.037 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:39.037 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:36:39.038 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:39.038 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.038 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.038 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.45161.0:136069 6.e 6:7343e225:::benchmark_data_smithi018_270558_object31330:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e258)", 2023-09-23T20:36:39.038 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:29:27.396169+0000", 2023-09-23T20:36:39.038 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 431.57923464499999, 2023-09-23T20:36:39.038 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.122030412, 2023-09-23T20:36:39.038 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:36:39.038 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:36:39.038 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:36:39.038 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.45161", 2023-09-23T20:36:39.038 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3795614987", 2023-09-23T20:36:39.039 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 136069 2023-09-23T20:36:39.039 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.039 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:36:39.039 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.039 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:36:39.039 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.396169+0000", 2023-09-23T20:36:39.039 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:36:39.039 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.039 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.039 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:36:39.039 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.396169+0000", 2023-09-23T20:36:39.039 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:36:39.040 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.040 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.040 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:36:39.040 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.396168+0000", 2023-09-23T20:36:39.040 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:36:39.040 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.040 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.040 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:36:39.040 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.396175+0000", 2023-09-23T20:36:39.040 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.8310000000000003e-06 2023-09-23T20:36:39.040 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.041 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.041 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:36:39.041 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.396191+0000", 2023-09-23T20:36:39.041 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.5736999999999999e-05 2023-09-23T20:36:39.041 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.041 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.041 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:36:39.041 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.396199+0000", 2023-09-23T20:36:39.041 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.1839999999999999e-06 2023-09-23T20:36:39.041 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.041 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.041 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:39.042 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.396257+0000", 2023-09-23T20:36:39.042 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.8400999999999999e-05 2023-09-23T20:36:39.042 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.042 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.042 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:36:39.042 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.396268+0000", 2023-09-23T20:36:39.042 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1099e-05 2023-09-23T20:36:39.042 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.042 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.042 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:39.042 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:28.345135+0000", 2023-09-23T20:36:39.043 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.948866459 2023-09-23T20:36:39.043 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.043 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.043 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:36:39.043 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:28.345145+0000", 2023-09-23T20:36:39.043 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.9210000000000004e-06 2023-09-23T20:36:39.043 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.043 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.043 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:39.043 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.514448+0000", 2023-09-23T20:36:39.044 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1693032249999999 2023-09-23T20:36:39.044 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.044 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.044 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:36:39.044 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.514537+0000", 2023-09-23T20:36:39.044 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.9529000000000003e-05 2023-09-23T20:36:39.044 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.044 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.044 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 4,5", 2023-09-23T20:36:39.044 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.514611+0000", 2023-09-23T20:36:39.044 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.3763999999999994e-05 2023-09-23T20:36:39.044 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.045 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.045 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:36:39.045 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.516564+0000", 2023-09-23T20:36:39.045 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.001952992 2023-09-23T20:36:39.045 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.045 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.045 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:36:39.045 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.516815+0000", 2023-09-23T20:36:39.045 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00025055699999999999 2023-09-23T20:36:39.045 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.045 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.046 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:36:39.046 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.518040+0000", 2023-09-23T20:36:39.046 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0012252529999999999 2023-09-23T20:36:39.046 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.046 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.046 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:36:39.046 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.518094+0000", 2023-09-23T20:36:39.046 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.4104000000000002e-05 2023-09-23T20:36:39.046 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.046 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.046 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:36:39.047 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.518200+0000", 2023-09-23T20:36:39.047 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000105715 2023-09-23T20:36:39.047 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:39.047 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:36:39.047 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:39.047 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.047 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.047 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:57620 7.8 7:1c6d8580:::benchmark_data_smithi018_308601_object57619:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e312)", 2023-09-23T20:36:39.047 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:32:32.247325+0000", 2023-09-23T20:36:39.048 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 246.72807936300001, 2023-09-23T20:36:39.048 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.3963792850000001, 2023-09-23T20:36:39.048 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:36:39.048 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:36:39.048 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:36:39.048 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:36:39.048 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:36:39.048 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 57620 2023-09-23T20:36:39.048 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.048 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:36:39.049 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.049 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:36:39.049 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.247325+0000", 2023-09-23T20:36:39.049 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:36:39.049 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.049 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.049 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:36:39.049 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.247325+0000", 2023-09-23T20:36:39.049 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:36:39.049 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.049 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.050 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:36:39.050 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.247323+0000", 2023-09-23T20:36:39.050 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:36:39.050 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.050 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.050 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:36:39.050 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.247348+0000", 2023-09-23T20:36:39.050 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.5418e-05 2023-09-23T20:36:39.051 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.051 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.051 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:36:39.051 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.247362+0000", 2023-09-23T20:36:39.051 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.4116e-05 2023-09-23T20:36:39.051 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.051 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.051 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:36:39.051 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.247370+0000", 2023-09-23T20:36:39.051 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.9459999999999998e-06 2023-09-23T20:36:39.051 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.052 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.052 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:39.052 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.247428+0000", 2023-09-23T20:36:39.052 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.7472999999999997e-05 2023-09-23T20:36:39.052 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.052 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.052 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:36:39.052 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.247439+0000", 2023-09-23T20:36:39.053 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0764000000000001e-05 2023-09-23T20:36:39.053 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.053 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.053 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:39.053 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.213131+0000", 2023-09-23T20:36:39.053 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.96569261699999998 2023-09-23T20:36:39.053 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.053 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.053 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:36:39.054 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.213144+0000", 2023-09-23T20:36:39.054 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3129000000000001e-05 2023-09-23T20:36:39.054 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.054 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.054 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:39.054 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.625268+0000", 2023-09-23T20:36:39.054 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.4121238840000001 2023-09-23T20:36:39.054 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.054 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.054 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:36:39.055 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.625437+0000", 2023-09-23T20:36:39.055 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000168968 2023-09-23T20:36:39.055 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.055 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.055 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 4,7", 2023-09-23T20:36:39.055 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.625602+0000", 2023-09-23T20:36:39.055 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000164635 2023-09-23T20:36:39.055 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.056 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.056 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:36:39.056 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.633312+0000", 2023-09-23T20:36:39.056 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.007709975 2023-09-23T20:36:39.056 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.056 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.056 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:36:39.056 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.635263+0000", 2023-09-23T20:36:39.056 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0019514930000000001 2023-09-23T20:36:39.056 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.056 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.057 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:36:39.057 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.639458+0000", 2023-09-23T20:36:39.057 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0041947290000000003 2023-09-23T20:36:39.057 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.057 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.057 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:36:39.057 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.639516+0000", 2023-09-23T20:36:39.057 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.8232000000000001e-05 2023-09-23T20:36:39.057 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.057 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.057 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:36:39.058 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.643704+0000", 2023-09-23T20:36:39.058 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0041875940000000002 2023-09-23T20:36:39.058 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:39.058 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:36:39.058 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:39.058 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.058 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.058 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:69152 7.3 7:c62f424f:::benchmark_data_smithi018_308601_object69151:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e320)", 2023-09-23T20:36:39.058 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:32:50.526348+0000", 2023-09-23T20:36:39.058 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 228.44905588099999, 2023-09-23T20:36:39.058 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0568542780000001, 2023-09-23T20:36:39.059 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:36:39.059 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:36:39.059 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:36:39.059 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:36:39.059 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:36:39.059 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 69152 2023-09-23T20:36:39.059 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.059 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:36:39.059 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.059 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:36:39.060 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:50.526348+0000", 2023-09-23T20:36:39.060 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:36:39.060 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.060 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.060 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:36:39.060 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:50.526348+0000", 2023-09-23T20:36:39.060 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:36:39.060 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.061 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.061 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:36:39.061 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:50.526348+0000", 2023-09-23T20:36:39.061 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:36:39.061 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.061 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.061 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:36:39.062 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:50.526389+0000", 2023-09-23T20:36:39.062 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.1758000000000002e-05 2023-09-23T20:36:39.062 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.062 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.062 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:36:39.062 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:50.526397+0000", 2023-09-23T20:36:39.062 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.0339999999999997e-06 2023-09-23T20:36:39.063 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.063 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.063 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:36:39.063 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:50.526404+0000", 2023-09-23T20:36:39.063 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.2319999999999998e-06 2023-09-23T20:36:39.063 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.063 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.063 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:39.063 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:50.526656+0000", 2023-09-23T20:36:39.063 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00025193600000000002 2023-09-23T20:36:39.063 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.064 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.064 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:36:39.064 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:50.526664+0000", 2023-09-23T20:36:39.064 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.1759999999999993e-06 2023-09-23T20:36:39.064 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.064 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.064 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:39.064 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:51.572432+0000", 2023-09-23T20:36:39.064 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.045768684 2023-09-23T20:36:39.064 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.064 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.064 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:36:39.065 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:51.572983+0000", 2023-09-23T20:36:39.065 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00055033100000000004 2023-09-23T20:36:39.065 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.065 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.065 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:36:39.065 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:51.573177+0000", 2023-09-23T20:36:39.065 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00019465899999999999 2023-09-23T20:36:39.065 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.065 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.065 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:36:39.065 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:51.581286+0000", 2023-09-23T20:36:39.065 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0081080490000000009 2023-09-23T20:36:39.066 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.066 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.066 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:36:39.066 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:51.581339+0000", 2023-09-23T20:36:39.066 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.3736000000000002e-05 2023-09-23T20:36:39.066 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.066 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.066 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:36:39.066 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:51.581415+0000", 2023-09-23T20:36:39.066 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.5952999999999996e-05 2023-09-23T20:36:39.066 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.067 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.067 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:36:39.067 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:51.581485+0000", 2023-09-23T20:36:39.067 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.991e-05 2023-09-23T20:36:39.067 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.067 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.067 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:36:39.067 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:51.583202+0000", 2023-09-23T20:36:39.067 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.001717239 2023-09-23T20:36:39.067 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:39.067 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:36:39.068 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:39.068 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.068 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.068 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:69159 7.3 7:c3cb5c7b:::benchmark_data_smithi018_308601_object69158:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e320)", 2023-09-23T20:36:39.068 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:32:50.526409+0000", 2023-09-23T20:36:39.068 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 228.44899540599999, 2023-09-23T20:36:39.068 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0737358429999999, 2023-09-23T20:36:39.068 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:36:39.068 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:36:39.068 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:36:39.068 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:36:39.069 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:36:39.069 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 69159 2023-09-23T20:36:39.069 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.069 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:36:39.069 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.069 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:36:39.069 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:50.526409+0000", 2023-09-23T20:36:39.069 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:36:39.069 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.069 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.069 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:36:39.069 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:50.526409+0000", 2023-09-23T20:36:39.070 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:36:39.070 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.070 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.070 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:36:39.070 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:50.526408+0000", 2023-09-23T20:36:39.070 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:36:39.070 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.070 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.070 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:36:39.070 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:50.526448+0000", 2023-09-23T20:36:39.070 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.9480999999999997e-05 2023-09-23T20:36:39.070 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.071 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.071 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:36:39.071 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:50.526455+0000", 2023-09-23T20:36:39.071 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.4170000000000003e-06 2023-09-23T20:36:39.071 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.071 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.071 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:36:39.071 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:50.526460+0000", 2023-09-23T20:36:39.071 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.4179999999999996e-06 2023-09-23T20:36:39.071 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.071 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.072 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:39.072 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:50.526681+0000", 2023-09-23T20:36:39.072 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00022085599999999999 2023-09-23T20:36:39.072 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.072 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.072 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:36:39.072 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:50.526686+0000", 2023-09-23T20:36:39.072 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.2529999999999998e-06 2023-09-23T20:36:39.072 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.072 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.072 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:39.073 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:51.575808+0000", 2023-09-23T20:36:39.073 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.04912205 2023-09-23T20:36:39.073 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.073 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.073 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:36:39.073 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:51.575976+0000", 2023-09-23T20:36:39.073 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00016865599999999999 2023-09-23T20:36:39.073 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.073 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.073 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:36:39.073 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:51.576200+0000", 2023-09-23T20:36:39.073 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00022404900000000001 2023-09-23T20:36:39.074 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.074 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.074 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:36:39.074 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:51.581402+0000", 2023-09-23T20:36:39.074 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0052012289999999999 2023-09-23T20:36:39.074 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.074 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.074 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:36:39.074 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:51.583225+0000", 2023-09-23T20:36:39.074 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.001823304 2023-09-23T20:36:39.074 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.074 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.075 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:36:39.075 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:51.598437+0000", 2023-09-23T20:36:39.075 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.015212652 2023-09-23T20:36:39.075 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.075 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.075 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:36:39.075 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:51.598500+0000", 2023-09-23T20:36:39.075 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.2810000000000003e-05 2023-09-23T20:36:39.075 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.075 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.075 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:36:39.076 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:51.600144+0000", 2023-09-23T20:36:39.076 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0016441050000000001 2023-09-23T20:36:39.076 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:39.076 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:36:39.076 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:39.076 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.076 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.076 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:69197 7.3 7:c10ecde9:::benchmark_data_smithi018_308601_object69196:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e320)", 2023-09-23T20:36:39.076 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:32:50.526466+0000", 2023-09-23T20:36:39.076 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 228.44893820499999, 2023-09-23T20:36:39.076 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0751176520000001, 2023-09-23T20:36:39.077 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:36:39.077 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:36:39.077 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:36:39.077 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:36:39.077 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:36:39.077 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 69197 2023-09-23T20:36:39.077 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.077 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:36:39.077 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.077 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:36:39.077 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:50.526466+0000", 2023-09-23T20:36:39.077 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:36:39.078 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.078 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.078 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:36:39.078 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:50.526466+0000", 2023-09-23T20:36:39.078 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:36:39.078 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.078 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.078 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:36:39.078 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:50.526465+0000", 2023-09-23T20:36:39.078 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:36:39.078 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.078 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.079 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:36:39.079 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:50.526504+0000", 2023-09-23T20:36:39.079 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.8685e-05 2023-09-23T20:36:39.079 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.079 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.079 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:36:39.079 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:50.526511+0000", 2023-09-23T20:36:39.079 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.1729999999999998e-06 2023-09-23T20:36:39.079 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.079 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.080 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:36:39.080 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:50.526516+0000", 2023-09-23T20:36:39.080 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.2599999999999996e-06 2023-09-23T20:36:39.080 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.080 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.080 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:39.080 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:50.526702+0000", 2023-09-23T20:36:39.080 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00018522600000000001 2023-09-23T20:36:39.080 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.080 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.080 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:36:39.081 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:50.526710+0000", 2023-09-23T20:36:39.081 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.2430000000000005e-06 2023-09-23T20:36:39.081 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.081 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.081 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:39.081 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:51.578646+0000", 2023-09-23T20:36:39.081 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0519360019999999 2023-09-23T20:36:39.081 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.081 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.081 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:36:39.081 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:51.578833+0000", 2023-09-23T20:36:39.081 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00018657499999999999 2023-09-23T20:36:39.082 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.082 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.082 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:36:39.082 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:51.579044+0000", 2023-09-23T20:36:39.082 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00021137099999999999 2023-09-23T20:36:39.082 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.082 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.082 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:36:39.082 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:51.583303+0000", 2023-09-23T20:36:39.083 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0042587650000000003 2023-09-23T20:36:39.083 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.083 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.083 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:36:39.083 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:51.588240+0000", 2023-09-23T20:36:39.083 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0049370669999999998 2023-09-23T20:36:39.083 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.083 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.083 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:36:39.084 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:51.600222+0000", 2023-09-23T20:36:39.084 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.011982418999999999 2023-09-23T20:36:39.084 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.084 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.084 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:36:39.084 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:51.600288+0000", 2023-09-23T20:36:39.084 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.6079000000000002e-05 2023-09-23T20:36:39.084 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.084 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.084 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:36:39.084 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:51.601583+0000", 2023-09-23T20:36:39.085 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.001295156 2023-09-23T20:36:39.085 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:39.085 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:36:39.085 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:39.085 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.085 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.085 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:69207 7.3 7:c71b5f56:::benchmark_data_smithi018_308601_object69206:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e320)", 2023-09-23T20:36:39.085 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:32:50.526521+0000", 2023-09-23T20:36:39.085 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 228.44888328799999, 2023-09-23T20:36:39.086 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0910981740000001, 2023-09-23T20:36:39.086 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:36:39.086 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:36:39.086 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:36:39.086 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:36:39.086 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:36:39.086 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 69207 2023-09-23T20:36:39.086 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.086 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:36:39.086 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.087 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:36:39.087 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:50.526521+0000", 2023-09-23T20:36:39.087 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:36:39.087 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.087 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.087 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:36:39.087 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:50.526521+0000", 2023-09-23T20:36:39.087 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:36:39.087 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.088 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.088 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:36:39.088 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:50.526520+0000", 2023-09-23T20:36:39.088 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:36:39.088 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.088 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.088 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:36:39.088 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:50.526561+0000", 2023-09-23T20:36:39.088 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.0983999999999999e-05 2023-09-23T20:36:39.088 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.089 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.089 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:36:39.089 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:50.526570+0000", 2023-09-23T20:36:39.089 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.8939999999999999e-06 2023-09-23T20:36:39.089 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.089 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.089 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:36:39.089 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:50.526576+0000", 2023-09-23T20:36:39.089 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.536e-06 2023-09-23T20:36:39.089 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.090 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.090 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:39.090 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:50.526731+0000", 2023-09-23T20:36:39.090 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00015522800000000001 2023-09-23T20:36:39.090 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.090 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.090 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:36:39.090 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:50.526739+0000", 2023-09-23T20:36:39.090 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.4169999999999992e-06 2023-09-23T20:36:39.091 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.091 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.091 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:39.091 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:51.583339+0000", 2023-09-23T20:36:39.091 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.056599356 2023-09-23T20:36:39.091 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.091 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.091 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:36:39.091 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:51.583498+0000", 2023-09-23T20:36:39.091 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00015957699999999999 2023-09-23T20:36:39.092 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.092 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.092 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:36:39.092 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:51.583675+0000", 2023-09-23T20:36:39.092 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000177157 2023-09-23T20:36:39.092 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.092 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.092 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:36:39.092 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:51.594254+0000", 2023-09-23T20:36:39.093 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.010578110999999999 2023-09-23T20:36:39.093 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.093 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.093 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:36:39.093 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:51.594314+0000", 2023-09-23T20:36:39.093 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.0528000000000001e-05 2023-09-23T20:36:39.093 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.093 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.093 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:36:39.094 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:51.616006+0000", 2023-09-23T20:36:39.094 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.021692211999999999 2023-09-23T20:36:39.094 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.094 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.094 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:36:39.094 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:51.616045+0000", 2023-09-23T20:36:39.094 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.8838999999999999e-05 2023-09-23T20:36:39.094 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.095 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.095 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:36:39.095 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:51.617619+0000", 2023-09-23T20:36:39.095 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.001573704 2023-09-23T20:36:39.095 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:39.095 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:36:39.095 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:39.095 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.095 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.095 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:120680 7.8 7:19893268:::benchmark_data_smithi018_308601_object26128:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e337)", 2023-09-23T20:36:39.095 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:34:04.594933+0000", 2023-09-23T20:36:39.096 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 154.380470524, 2023-09-23T20:36:39.096 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.98937692300000002, 2023-09-23T20:36:39.096 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:36:39.096 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:36:39.096 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:36:39.096 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:36:39.096 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:36:39.096 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 120680 2023-09-23T20:36:39.096 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.096 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:36:39.096 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.097 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:36:39.097 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.594933+0000", 2023-09-23T20:36:39.097 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:36:39.097 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.097 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.097 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:36:39.097 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.594933+0000", 2023-09-23T20:36:39.097 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:36:39.097 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.097 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.097 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:36:39.097 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.594933+0000", 2023-09-23T20:36:39.098 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:36:39.098 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.098 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.098 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:36:39.098 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.594936+0000", 2023-09-23T20:36:39.098 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.5360000000000001e-06 2023-09-23T20:36:39.098 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.098 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.098 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:36:39.098 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.594943+0000", 2023-09-23T20:36:39.098 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.4300000000000003e-06 2023-09-23T20:36:39.098 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.099 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.099 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:36:39.099 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.594949+0000", 2023-09-23T20:36:39.099 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.4099999999999996e-06 2023-09-23T20:36:39.099 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.099 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.099 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:39.099 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.598565+0000", 2023-09-23T20:36:39.099 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0036157450000000001 2023-09-23T20:36:39.099 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.099 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.100 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:36:39.100 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.598581+0000", 2023-09-23T20:36:39.100 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.5749000000000001e-05 2023-09-23T20:36:39.100 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.100 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.100 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:39.100 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.577402+0000", 2023-09-23T20:36:39.100 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.97882154499999996 2023-09-23T20:36:39.100 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.100 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.100 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:36:39.100 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.577521+0000", 2023-09-23T20:36:39.101 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000118606 2023-09-23T20:36:39.101 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.101 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.101 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 4,7", 2023-09-23T20:36:39.101 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.577601+0000", 2023-09-23T20:36:39.101 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.0010000000000001e-05 2023-09-23T20:36:39.101 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.101 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.101 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:36:39.101 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.581327+0000", 2023-09-23T20:36:39.101 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0037262950000000001 2023-09-23T20:36:39.102 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.102 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.102 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:36:39.102 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.581357+0000", 2023-09-23T20:36:39.102 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.9507999999999999e-05 2023-09-23T20:36:39.102 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.102 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.102 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:36:39.102 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.584253+0000", 2023-09-23T20:36:39.102 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0028957710000000001 2023-09-23T20:36:39.102 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.102 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.103 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:36:39.103 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.584284+0000", 2023-09-23T20:36:39.103 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.1109999999999999e-05 2023-09-23T20:36:39.103 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.103 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.103 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:36:39.103 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.584310+0000", 2023-09-23T20:36:39.103 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.6721e-05 2023-09-23T20:36:39.103 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:39.103 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:36:39.103 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:39.103 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.104 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.104 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:120746 7.8 7:13a4d181:::benchmark_data_smithi018_308601_object26194:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e338)", 2023-09-23T20:36:39.104 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:34:04.743456+0000", 2023-09-23T20:36:39.104 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 154.23194761400001, 2023-09-23T20:36:39.104 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.84858167699999998, 2023-09-23T20:36:39.104 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:36:39.104 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:36:39.104 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:36:39.104 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:36:39.104 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:36:39.104 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 120746 2023-09-23T20:36:39.104 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.105 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:36:39.105 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.105 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:36:39.105 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.743456+0000", 2023-09-23T20:36:39.105 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:36:39.105 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.105 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.105 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:36:39.105 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.743456+0000", 2023-09-23T20:36:39.105 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:36:39.105 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.106 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.106 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:36:39.106 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.743455+0000", 2023-09-23T20:36:39.106 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:36:39.106 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.106 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.106 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:36:39.106 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.743462+0000", 2023-09-23T20:36:39.106 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.3270000000000003e-06 2023-09-23T20:36:39.106 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.106 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.107 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:36:39.107 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.743480+0000", 2023-09-23T20:36:39.107 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.7609999999999999e-05 2023-09-23T20:36:39.107 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.107 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.107 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:36:39.107 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.743488+0000", 2023-09-23T20:36:39.107 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.7100000000000007e-06 2023-09-23T20:36:39.107 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.107 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.107 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:39.107 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.743543+0000", 2023-09-23T20:36:39.108 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.5819000000000002e-05 2023-09-23T20:36:39.108 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.108 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.108 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:36:39.108 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.743558+0000", 2023-09-23T20:36:39.108 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.4195e-05 2023-09-23T20:36:39.108 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.108 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.108 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:39.108 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.581377+0000", 2023-09-23T20:36:39.108 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.83781900600000003 2023-09-23T20:36:39.108 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.109 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.109 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:36:39.109 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.581461+0000", 2023-09-23T20:36:39.109 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.4777000000000005e-05 2023-09-23T20:36:39.109 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.109 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.109 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 4,7", 2023-09-23T20:36:39.109 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.581536+0000", 2023-09-23T20:36:39.109 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.4791999999999998e-05 2023-09-23T20:36:39.109 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.109 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.109 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:36:39.110 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.591780+0000", 2023-09-23T20:36:39.110 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.010243533000000001 2023-09-23T20:36:39.110 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.110 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.110 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:36:39.110 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.591833+0000", 2023-09-23T20:36:39.110 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.2916e-05 2023-09-23T20:36:39.110 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.110 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.110 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:36:39.110 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.591955+0000", 2023-09-23T20:36:39.111 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00012233 2023-09-23T20:36:39.111 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.111 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.111 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:36:39.111 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.592001+0000", 2023-09-23T20:36:39.111 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.6439e-05 2023-09-23T20:36:39.111 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.111 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.111 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:36:39.111 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.592038+0000", 2023-09-23T20:36:39.111 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.6650000000000003e-05 2023-09-23T20:36:39.111 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:39.112 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:36:39.112 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:39.112 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.112 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.112 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:120761 7.8 7:12380617:::benchmark_data_smithi018_308601_object26209:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e338)", 2023-09-23T20:36:39.112 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:34:04.772652+0000", 2023-09-23T20:36:39.112 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 154.202752336, 2023-09-23T20:36:39.112 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.81952633799999997, 2023-09-23T20:36:39.112 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:36:39.112 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:36:39.112 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:36:39.113 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:36:39.113 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:36:39.113 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 120761 2023-09-23T20:36:39.113 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.113 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:36:39.113 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.113 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:36:39.113 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.772652+0000", 2023-09-23T20:36:39.113 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:36:39.113 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.113 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.113 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:36:39.114 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.772652+0000", 2023-09-23T20:36:39.114 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:36:39.114 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.114 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.114 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:36:39.114 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.772650+0000", 2023-09-23T20:36:39.114 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:36:39.114 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.114 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.115 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:36:39.115 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.772660+0000", 2023-09-23T20:36:39.115 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0069e-05 2023-09-23T20:36:39.115 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.115 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.115 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:36:39.115 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.772684+0000", 2023-09-23T20:36:39.115 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.3852000000000001e-05 2023-09-23T20:36:39.115 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.116 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.116 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:36:39.116 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.772697+0000", 2023-09-23T20:36:39.116 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3505e-05 2023-09-23T20:36:39.116 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.116 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.116 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:39.116 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.772764+0000", 2023-09-23T20:36:39.116 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.7130000000000003e-05 2023-09-23T20:36:39.116 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.116 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.117 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:36:39.117 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.772778+0000", 2023-09-23T20:36:39.117 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.4035e-05 2023-09-23T20:36:39.117 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.117 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.117 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:39.117 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.584345+0000", 2023-09-23T20:36:39.117 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.81156667500000002 2023-09-23T20:36:39.117 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.118 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.118 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:36:39.118 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.584431+0000", 2023-09-23T20:36:39.118 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.6496999999999995e-05 2023-09-23T20:36:39.118 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.118 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.118 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 4,7", 2023-09-23T20:36:39.118 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.584502+0000", 2023-09-23T20:36:39.118 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.0326000000000004e-05 2023-09-23T20:36:39.118 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.119 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.119 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:36:39.119 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.591948+0000", 2023-09-23T20:36:39.119 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0074458609999999998 2023-09-23T20:36:39.119 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.119 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.119 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:36:39.119 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.592045+0000", 2023-09-23T20:36:39.119 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.7040000000000006e-05 2023-09-23T20:36:39.119 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.120 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.120 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:36:39.120 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.592097+0000", 2023-09-23T20:36:39.120 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.2416000000000001e-05 2023-09-23T20:36:39.120 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.120 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.120 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:36:39.120 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.592142+0000", 2023-09-23T20:36:39.120 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.4814999999999997e-05 2023-09-23T20:36:39.121 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.121 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.121 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:36:39.121 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.592178+0000", 2023-09-23T20:36:39.121 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.6016999999999998e-05 2023-09-23T20:36:39.121 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:39.121 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:36:39.121 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:39.121 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.121 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.121 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:120780 7.8 7:13c48ce1:::benchmark_data_smithi018_308601_object26228:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e338)", 2023-09-23T20:36:39.122 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:34:04.802248+0000", 2023-09-23T20:36:39.122 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 154.17315629300001, 2023-09-23T20:36:39.122 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.79010917300000005, 2023-09-23T20:36:39.122 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:36:39.122 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:36:39.122 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:36:39.122 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:36:39.122 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:36:39.122 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 120780 2023-09-23T20:36:39.123 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.123 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:36:39.123 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.123 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:36:39.123 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.802248+0000", 2023-09-23T20:36:39.123 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:36:39.123 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.123 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.123 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:36:39.124 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.802248+0000", 2023-09-23T20:36:39.124 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:36:39.124 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.124 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.124 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:36:39.124 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.802247+0000", 2023-09-23T20:36:39.124 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.999999 2023-09-23T20:36:39.124 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.124 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.124 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:36:39.124 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.802252+0000", 2023-09-23T20:36:39.125 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.541e-06 2023-09-23T20:36:39.125 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.125 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.125 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:36:39.125 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.802264+0000", 2023-09-23T20:36:39.125 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2283000000000001e-05 2023-09-23T20:36:39.125 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.125 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.125 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:36:39.125 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.802273+0000", 2023-09-23T20:36:39.125 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.49e-06 2023-09-23T20:36:39.126 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.126 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.126 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:39.126 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.802382+0000", 2023-09-23T20:36:39.126 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00010888 2023-09-23T20:36:39.126 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.126 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.126 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:36:39.126 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.802390+0000", 2023-09-23T20:36:39.126 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.9559999999999997e-06 2023-09-23T20:36:39.126 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.127 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.127 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:39.127 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.587735+0000", 2023-09-23T20:36:39.127 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.78534481 2023-09-23T20:36:39.127 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.127 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.127 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:36:39.127 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.587860+0000", 2023-09-23T20:36:39.127 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000125525 2023-09-23T20:36:39.128 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.128 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.128 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 4,7", 2023-09-23T20:36:39.128 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.587968+0000", 2023-09-23T20:36:39.128 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00010744899999999999 2023-09-23T20:36:39.128 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.128 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.129 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:36:39.129 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.592049+0000", 2023-09-23T20:36:39.129 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0040818549999999997 2023-09-23T20:36:39.129 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.129 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.129 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:36:39.129 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.592227+0000", 2023-09-23T20:36:39.129 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00017758699999999999 2023-09-23T20:36:39.129 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.130 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.130 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:36:39.130 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.592281+0000", 2023-09-23T20:36:39.130 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.4135999999999998e-05 2023-09-23T20:36:39.130 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.130 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.130 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:36:39.130 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.592323+0000", 2023-09-23T20:36:39.130 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.1607999999999999e-05 2023-09-23T20:36:39.130 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.130 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.131 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:36:39.131 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.592357+0000", 2023-09-23T20:36:39.131 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.4076999999999999e-05 2023-09-23T20:36:39.131 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:39.131 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:36:39.131 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:39.131 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.131 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.131 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:120781 7.8 7:1dfbb747:::benchmark_data_smithi018_308601_object26229:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e338)", 2023-09-23T20:36:39.131 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:34:04.802289+0000", 2023-09-23T20:36:39.131 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 154.17311497599999, 2023-09-23T20:36:39.132 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.80573468199999998, 2023-09-23T20:36:39.132 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:36:39.132 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:36:39.132 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:36:39.132 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:36:39.132 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:36:39.132 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 120781 2023-09-23T20:36:39.132 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.132 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:36:39.132 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.132 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:36:39.133 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.802289+0000", 2023-09-23T20:36:39.133 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:36:39.133 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.133 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.133 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:36:39.133 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.802289+0000", 2023-09-23T20:36:39.133 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:36:39.133 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.133 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.133 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:36:39.133 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.802288+0000", 2023-09-23T20:36:39.134 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:36:39.134 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.134 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.134 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:36:39.134 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.802292+0000", 2023-09-23T20:36:39.134 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.4769999999999999e-06 2023-09-23T20:36:39.134 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.134 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.134 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:36:39.134 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.802304+0000", 2023-09-23T20:36:39.134 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1613e-05 2023-09-23T20:36:39.135 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.135 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.135 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:36:39.135 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.802310+0000", 2023-09-23T20:36:39.135 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.3820000000000001e-06 2023-09-23T20:36:39.135 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.135 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.135 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:39.135 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.802404+0000", 2023-09-23T20:36:39.135 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.3603000000000005e-05 2023-09-23T20:36:39.135 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.136 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.136 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:36:39.136 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.802408+0000", 2023-09-23T20:36:39.136 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.5240000000000002e-06 2023-09-23T20:36:39.136 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.136 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.136 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:39.136 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.592389+0000", 2023-09-23T20:36:39.137 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.78998126300000004 2023-09-23T20:36:39.137 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.137 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.137 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:36:39.137 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.592492+0000", 2023-09-23T20:36:39.137 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000102662 2023-09-23T20:36:39.137 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.137 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.137 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 4,7", 2023-09-23T20:36:39.137 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.592606+0000", 2023-09-23T20:36:39.137 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00011369 2023-09-23T20:36:39.138 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.138 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.138 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:36:39.138 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.600025+0000", 2023-09-23T20:36:39.138 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0074188470000000001 2023-09-23T20:36:39.138 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.138 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.138 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:36:39.138 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.600036+0000", 2023-09-23T20:36:39.138 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0987e-05 2023-09-23T20:36:39.138 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.139 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.139 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:36:39.139 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.607931+0000", 2023-09-23T20:36:39.139 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0078951549999999992 2023-09-23T20:36:39.139 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.139 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.139 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:36:39.139 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.607977+0000", 2023-09-23T20:36:39.139 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.6479e-05 2023-09-23T20:36:39.139 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.139 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.140 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:36:39.140 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.608024+0000", 2023-09-23T20:36:39.140 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.6487999999999999e-05 2023-09-23T20:36:39.140 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:39.140 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:36:39.140 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:39.140 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.140 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.140 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:120828 7.8 7:1842f669:::benchmark_data_smithi018_308601_object26276:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e338)", 2023-09-23T20:36:39.140 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:34:04.911638+0000", 2023-09-23T20:36:39.140 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 154.063766042, 2023-09-23T20:36:39.140 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.69669138900000005, 2023-09-23T20:36:39.141 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:36:39.141 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:36:39.141 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:36:39.141 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:36:39.141 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:36:39.141 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 120828 2023-09-23T20:36:39.141 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.141 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:36:39.141 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.141 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:36:39.141 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.911638+0000", 2023-09-23T20:36:39.141 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:36:39.142 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.142 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.142 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:36:39.142 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.911638+0000", 2023-09-23T20:36:39.142 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:36:39.142 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.142 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.142 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:36:39.142 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.911636+0000", 2023-09-23T20:36:39.142 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:36:39.142 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.143 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.143 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:36:39.143 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.911651+0000", 2023-09-23T20:36:39.143 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.5067e-05 2023-09-23T20:36:39.143 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.143 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.143 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:36:39.143 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.911668+0000", 2023-09-23T20:36:39.143 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.6957999999999999e-05 2023-09-23T20:36:39.143 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.143 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.144 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:36:39.144 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.911677+0000", 2023-09-23T20:36:39.144 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.0909999999999999e-06 2023-09-23T20:36:39.144 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.144 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.144 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:39.144 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.911744+0000", 2023-09-23T20:36:39.144 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.6674000000000006e-05 2023-09-23T20:36:39.144 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.144 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.144 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:36:39.145 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.911757+0000", 2023-09-23T20:36:39.145 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2653999999999999e-05 2023-09-23T20:36:39.145 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.145 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.145 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:39.145 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.596774+0000", 2023-09-23T20:36:39.145 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.68501761699999997 2023-09-23T20:36:39.145 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.145 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.145 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:36:39.145 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.596892+0000", 2023-09-23T20:36:39.145 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000117624 2023-09-23T20:36:39.146 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.146 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.146 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 4,7", 2023-09-23T20:36:39.146 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.597011+0000", 2023-09-23T20:36:39.146 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000118956 2023-09-23T20:36:39.146 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.146 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.146 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:36:39.146 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.608085+0000", 2023-09-23T20:36:39.146 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.011074392000000001 2023-09-23T20:36:39.146 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.146 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.147 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:36:39.147 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.608101+0000", 2023-09-23T20:36:39.147 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.5710999999999999e-05 2023-09-23T20:36:39.147 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.147 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.147 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:36:39.147 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.608242+0000", 2023-09-23T20:36:39.147 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000141266 2023-09-23T20:36:39.147 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.147 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.147 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:36:39.147 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.608285+0000", 2023-09-23T20:36:39.148 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.2870000000000001e-05 2023-09-23T20:36:39.148 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.148 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.148 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:36:39.148 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.608329+0000", 2023-09-23T20:36:39.148 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.4153000000000002e-05 2023-09-23T20:36:39.148 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:39.148 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:36:39.148 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:39.148 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.148 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.149 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:184296 7.e 7:78fe8854:::benchmark_data_smithi018_308601_object89744:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e364)", 2023-09-23T20:36:39.149 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:35:42.303829+0000", 2023-09-23T20:36:39.149 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 56.671574858, 2023-09-23T20:36:39.149 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.087049301999999995, 2023-09-23T20:36:39.149 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:36:39.149 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:36:39.149 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:36:39.149 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:36:39.149 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:36:39.149 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 184296 2023-09-23T20:36:39.150 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.150 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:36:39.150 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.150 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:36:39.150 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:42.303829+0000", 2023-09-23T20:36:39.150 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:36:39.150 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.150 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.150 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:36:39.151 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:42.303829+0000", 2023-09-23T20:36:39.151 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:36:39.151 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.151 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.151 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:36:39.151 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:42.303828+0000", 2023-09-23T20:36:39.151 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:36:39.151 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.151 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.151 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:36:39.152 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:42.303834+0000", 2023-09-23T20:36:39.152 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.4819999999999999e-06 2023-09-23T20:36:39.152 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.152 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.152 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:36:39.152 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:42.303851+0000", 2023-09-23T20:36:39.152 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.6449000000000001e-05 2023-09-23T20:36:39.152 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.152 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.153 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:36:39.153 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:42.303860+0000", 2023-09-23T20:36:39.153 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.1099999999999992e-06 2023-09-23T20:36:39.153 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.153 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.153 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:39.153 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:42.303962+0000", 2023-09-23T20:36:39.153 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000102042 2023-09-23T20:36:39.153 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.153 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.154 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:36:39.154 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:42.304076+0000", 2023-09-23T20:36:39.154 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000113737 2023-09-23T20:36:39.154 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.154 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.154 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:36:39.154 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:42.304174+0000", 2023-09-23T20:36:39.154 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.8439999999999999e-05 2023-09-23T20:36:39.155 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.155 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.155 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:36:39.155 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:42.309650+0000", 2023-09-23T20:36:39.155 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0054761280000000002 2023-09-23T20:36:39.155 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.155 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.155 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:36:39.155 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:42.309805+0000", 2023-09-23T20:36:39.155 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00015525599999999999 2023-09-23T20:36:39.156 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.156 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.156 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:36:39.156 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:42.390764+0000", 2023-09-23T20:36:39.156 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.080958764000000003 2023-09-23T20:36:39.156 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.156 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.156 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:36:39.156 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:42.390836+0000", 2023-09-23T20:36:39.156 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.2026999999999993e-05 2023-09-23T20:36:39.156 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.157 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.157 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:36:39.157 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:42.390878+0000", 2023-09-23T20:36:39.157 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.2211e-05 2023-09-23T20:36:39.157 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:39.157 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:36:39.157 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:39.157 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.158 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.158 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35994.0:43476 8.f 8:fcb5d1df:::benchmark_data_smithi018_341490_object43475:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e374)", 2023-09-23T20:36:39.158 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:36:36.556754+0000", 2023-09-23T20:36:39.158 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 2.4186500240000002, 2023-09-23T20:36:39.158 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.088543183999999997, 2023-09-23T20:36:39.158 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:36:39.158 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:36:39.158 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:36:39.158 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35994", 2023-09-23T20:36:39.159 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:36:39.159 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 43476 2023-09-23T20:36:39.159 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.159 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:36:39.159 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.159 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:36:39.159 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:36:36.556754+0000", 2023-09-23T20:36:39.159 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:36:39.160 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.160 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.160 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:36:39.160 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:36:36.556754+0000", 2023-09-23T20:36:39.160 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:36:39.160 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.160 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.160 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:36:39.161 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:36:36.556752+0000", 2023-09-23T20:36:39.161 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:36:39.161 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.161 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.161 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:36:39.161 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:36:36.556774+0000", 2023-09-23T20:36:39.161 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.1746000000000002e-05 2023-09-23T20:36:39.161 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.162 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.162 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:36:39.162 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:36:36.556792+0000", 2023-09-23T20:36:39.162 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.7929000000000001e-05 2023-09-23T20:36:39.162 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.162 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.162 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:36:39.162 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:36:36.556801+0000", 2023-09-23T20:36:39.162 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.3340000000000008e-06 2023-09-23T20:36:39.162 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.162 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.162 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:39.163 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:36:36.556857+0000", 2023-09-23T20:36:39.163 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.5889e-05 2023-09-23T20:36:39.163 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.163 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.163 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:36:39.163 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:36:36.557025+0000", 2023-09-23T20:36:39.163 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00016774499999999999 2023-09-23T20:36:39.163 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.163 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.163 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:36:39.163 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:36:36.557172+0000", 2023-09-23T20:36:39.163 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00014666700000000001 2023-09-23T20:36:39.164 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.164 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.164 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:36:39.164 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:36:36.558958+0000", 2023-09-23T20:36:39.164 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.001785774 2023-09-23T20:36:39.164 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.164 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.164 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:36:39.164 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:36:36.561988+0000", 2023-09-23T20:36:39.164 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0030299760000000002 2023-09-23T20:36:39.164 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.165 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.165 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:36:39.165 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:36:36.645209+0000", 2023-09-23T20:36:39.165 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.083221672999999996 2023-09-23T20:36:39.165 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.165 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.165 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:36:39.165 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:36:36.645257+0000", 2023-09-23T20:36:39.165 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.8235999999999997e-05 2023-09-23T20:36:39.165 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.165 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.165 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:36:39.166 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:36:36.645297+0000", 2023-09-23T20:36:39.166 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.9674e-05 2023-09-23T20:36:39.166 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:39.166 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:36:39.166 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:39.166 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.166 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.166 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35994.0:45797 8.f 8:f4a4f4e0:::benchmark_data_smithi018_341490_object45796:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e374)", 2023-09-23T20:36:39.166 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:36:38.286046+0000", 2023-09-23T20:36:39.166 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.68935793000000001, 2023-09-23T20:36:39.166 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.076539772000000006, 2023-09-23T20:36:39.166 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:36:39.167 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:36:39.167 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:36:39.167 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35994", 2023-09-23T20:36:39.167 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:36:39.167 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 45797 2023-09-23T20:36:39.167 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.167 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:36:39.167 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.167 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:36:39.167 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:36:38.286046+0000", 2023-09-23T20:36:39.167 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:36:39.167 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.168 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.168 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:36:39.168 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:36:38.286046+0000", 2023-09-23T20:36:39.168 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:36:39.168 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.168 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.168 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:36:39.168 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:36:38.286044+0000", 2023-09-23T20:36:39.168 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:36:39.168 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.168 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.169 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:36:39.169 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:36:38.286070+0000", 2023-09-23T20:36:39.169 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.5789e-05 2023-09-23T20:36:39.169 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.169 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.169 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:36:39.169 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:36:38.286083+0000", 2023-09-23T20:36:39.169 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.313e-05 2023-09-23T20:36:39.169 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.169 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.169 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:36:39.169 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:36:38.286092+0000", 2023-09-23T20:36:39.170 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.1030000000000008e-06 2023-09-23T20:36:39.170 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.170 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.170 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:39.170 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:36:38.286147+0000", 2023-09-23T20:36:39.170 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.4716999999999998e-05 2023-09-23T20:36:39.170 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.170 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.170 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:36:39.170 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:36:38.286280+0000", 2023-09-23T20:36:39.170 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00013324300000000001 2023-09-23T20:36:39.171 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.171 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.171 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:36:39.171 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:36:38.286396+0000", 2023-09-23T20:36:39.171 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000115708 2023-09-23T20:36:39.171 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.171 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.171 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:36:39.171 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:36:38.289035+0000", 2023-09-23T20:36:39.171 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.002638952 2023-09-23T20:36:39.171 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.171 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.172 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:36:39.172 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:36:38.291538+0000", 2023-09-23T20:36:39.172 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0025031760000000002 2023-09-23T20:36:39.172 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.172 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.172 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:36:39.172 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:36:38.362492+0000", 2023-09-23T20:36:39.172 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.070953287000000004 2023-09-23T20:36:39.172 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.172 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.172 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:36:39.172 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:36:38.362547+0000", 2023-09-23T20:36:39.173 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.5757000000000002e-05 2023-09-23T20:36:39.173 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:39.173 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:39.173 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:36:39.173 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:36:38.362586+0000", 2023-09-23T20:36:39.173 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.8513000000000002e-05 2023-09-23T20:36:39.173 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:39.173 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:36:39.173 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:39.173 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:39.173 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:36:39.173 INFO:teuthology.orchestra.run.smithi018.stdout:} 2023-09-23T20:36:39.174 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 23 16 46711 46695 126.835 77 0.00952972 0.00787891 2023-09-23T20:36:39.175 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:39.176 INFO:teuthology.orchestra.run.smithi018.stdout:{"pg_ready":true,"pg_map":{"version":906,"stamp":"2023-09-23T20:36:37.936967+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2729908147,"num_objects":41610,"num_object_clones":0,"num_object_copies":124830,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":41610,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":42489,"num_write_kb":2673973,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":108,"num_bytes_recovered":4463552,"num_keys_recovered":40,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":47604,"ondisk_log_size":47604,"up":357,"acting":357,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":359,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":11080140,"kb_used_data":7979852,"kb_used_omap":32,"kb_used_meta":3100191,"kb_avail":738881076,"statfs":{"total":767960285184,"available":756614221824,"internally_reserved":0,"allocated":8171368448,"data_stored":8159206219,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32773,"internal_metadata":3174596603},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":1186791424,"num_objects":18109,"num_object_clones":0,"num_object_copies":54327,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":18109,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":18109,"num_write_kb":1158976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003129"},"pg_stats":[{"pgid":"1.18","version":"90'161","reported_seq":647,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T20:36:15.186177+0000","last_change":"2023-09-23T20:35:26.644897+0000","last_active":"2023-09-23T20:36:15.186177+0000","last_peered":"2023-09-23T20:36:15.186177+0000","last_clean":"2023-09-23T20:36:15.186177+0000","last_became_active":"2023-09-23T20:35:26.644564+0000","last_became_peered":"2023-09-23T20:35:26.644564+0000","last_unstale":"2023-09-23T20:36:15.186177+0000","last_undegraded":"2023-09-23T20:36:15.186177+0000","last_fullsized":"2023-09-23T20:36:15.186177+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:35:03.551143+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:35:03.551143+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:55:55.872961+0000","scrub_duration":0.033122453000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"69'1","reported_seq":70,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T20:36:15.187018+0000","last_change":"2023-09-23T20:32:33.233853+0000","last_active":"2023-09-23T20:36:15.187018+0000","last_peered":"2023-09-23T20:36:15.187018+0000","last_clean":"2023-09-23T20:36:15.187018+0000","last_became_active":"2023-09-23T20:32:33.233546+0000","last_became_peered":"2023-09-23T20:32:33.233546+0000","last_unstale":"2023-09-23T20:36:15.187018+0000","last_undegraded":"2023-09-23T20:36:15.187018+0000","last_fullsized":"2023-09-23T20:36:15.187018+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:50:27.130808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":252,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T20:36:15.193656+0000","last_change":"2023-09-23T20:32:50.536495+0000","last_active":"2023-09-23T20:36:15.193656+0000","last_peered":"2023-09-23T20:36:15.193656+0000","last_clean":"2023-09-23T20:36:15.193656+0000","last_became_active":"2023-09-23T20:32:50.536180+0000","last_became_peered":"2023-09-23T20:32:50.536180+0000","last_unstale":"2023-09-23T20:36:15.193656+0000","last_undegraded":"2023-09-23T20:36:15.193656+0000","last_fullsized":"2023-09-23T20:36:15.193656+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:01:05.524308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1b","version":"90'161","reported_seq":523,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T20:36:15.187038+0000","last_change":"2023-09-23T20:36:05.442367+0000","last_active":"2023-09-23T20:36:15.187038+0000","last_peered":"2023-09-23T20:36:15.187038+0000","last_clean":"2023-09-23T20:36:15.187038+0000","last_became_active":"2023-09-23T20:35:26.647261+0000","last_became_peered":"2023-09-23T20:35:26.647261+0000","last_unstale":"2023-09-23T20:36:15.187038+0000","last_undegraded":"2023-09-23T20:36:15.187038+0000","last_fullsized":"2023-09-23T20:36:15.187038+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:05.442288+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:36:05.442288+0000","last_clean_scrub_stamp":"2023-09-23T20:36:05.442288+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:08:03.875152+0000","scrub_duration":0.00146853,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":339,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T20:36:15.183457+0000","last_change":"2023-09-23T20:34:59.983035+0000","last_active":"2023-09-23T20:36:15.183457+0000","last_peered":"2023-09-23T20:36:15.183457+0000","last_clean":"2023-09-23T20:36:15.183457+0000","last_became_active":"2023-09-23T20:34:59.982459+0000","last_became_peered":"2023-09-23T20:34:59.982459+0000","last_unstale":"2023-09-23T20:36:15.183457+0000","last_undegraded":"2023-09-23T20:36:15.183457+0000","last_fullsized":"2023-09-23T20:36:15.183457+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:16:48.510783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":342,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T20:36:18.350852+0000","last_change":"2023-09-23T20:25:02.957185+0000","last_active":"2023-09-23T20:36:18.350852+0000","last_peered":"2023-09-23T20:36:18.350852+0000","last_clean":"2023-09-23T20:36:18.350852+0000","last_became_active":"2023-09-23T20:25:02.956689+0000","last_became_peered":"2023-09-23T20:25:02.956689+0000","last_unstale":"2023-09-23T20:36:18.350852+0000","last_undegraded":"2023-09-23T20:36:18.350852+0000","last_fullsized":"2023-09-23T20:36:18.350852+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:04:28.524015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"90'161","reported_seq":452,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T20:36:15.186635+0000","last_change":"2023-09-23T20:36:01.292103+0000","last_active":"2023-09-23T20:36:15.186635+0000","last_peered":"2023-09-23T20:36:15.186635+0000","last_clean":"2023-09-23T20:36:15.186635+0000","last_became_active":"2023-09-23T20:35:26.639912+0000","last_became_peered":"2023-09-23T20:35:26.639912+0000","last_unstale":"2023-09-23T20:36:15.186635+0000","last_undegraded":"2023-09-23T20:36:15.186635+0000","last_fullsized":"2023-09-23T20:36:15.186635+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:01.292032+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:01.292032+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:04:53.926640+0000","scrub_duration":0.0032983159999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":186,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T20:36:18.349536+0000","last_change":"2023-09-23T20:32:50.546428+0000","last_active":"2023-09-23T20:36:18.349536+0000","last_peered":"2023-09-23T20:36:18.349536+0000","last_clean":"2023-09-23T20:36:18.349536+0000","last_became_active":"2023-09-23T20:32:50.546120+0000","last_became_peered":"2023-09-23T20:32:50.546120+0000","last_unstale":"2023-09-23T20:36:18.349536+0000","last_undegraded":"2023-09-23T20:36:18.349536+0000","last_fullsized":"2023-09-23T20:36:18.349536+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:00:10.792410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":147,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T20:36:18.347372+0000","last_change":"2023-09-23T20:29:46.112977+0000","last_active":"2023-09-23T20:36:18.347372+0000","last_peered":"2023-09-23T20:36:18.347372+0000","last_clean":"2023-09-23T20:36:18.347372+0000","last_became_active":"2023-09-23T20:29:46.112600+0000","last_became_peered":"2023-09-23T20:29:46.112600+0000","last_unstale":"2023-09-23T20:36:18.347372+0000","last_undegraded":"2023-09-23T20:36:18.347372+0000","last_fullsized":"2023-09-23T20:36:18.347372+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:06:34.132247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"90'161","reported_seq":546,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T20:36:18.346350+0000","last_change":"2023-09-23T20:36:03.411105+0000","last_active":"2023-09-23T20:36:18.346350+0000","last_peered":"2023-09-23T20:36:18.346350+0000","last_clean":"2023-09-23T20:36:18.346350+0000","last_became_active":"2023-09-23T20:35:26.642130+0000","last_became_peered":"2023-09-23T20:35:26.642130+0000","last_unstale":"2023-09-23T20:36:18.346350+0000","last_undegraded":"2023-09-23T20:36:18.346350+0000","last_fullsized":"2023-09-23T20:36:18.346350+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:03.411025+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:03.411025+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:31:28.138183+0000","scrub_duration":0.001802922,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":339,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T20:36:18.351711+0000","last_change":"2023-09-23T20:25:02.969432+0000","last_active":"2023-09-23T20:36:18.351711+0000","last_peered":"2023-09-23T20:36:18.351711+0000","last_clean":"2023-09-23T20:36:18.351711+0000","last_became_active":"2023-09-23T20:25:02.969118+0000","last_became_peered":"2023-09-23T20:25:02.969118+0000","last_unstale":"2023-09-23T20:36:18.351711+0000","last_undegraded":"2023-09-23T20:36:18.351711+0000","last_fullsized":"2023-09-23T20:36:18.351711+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:02:22.453831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":77,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T20:36:18.346734+0000","last_change":"2023-09-23T20:32:01.360359+0000","last_active":"2023-09-23T20:36:18.346734+0000","last_peered":"2023-09-23T20:36:18.346734+0000","last_clean":"2023-09-23T20:36:18.346734+0000","last_became_active":"2023-09-23T20:32:01.360008+0000","last_became_peered":"2023-09-23T20:32:01.360008+0000","last_unstale":"2023-09-23T20:36:18.346734+0000","last_undegraded":"2023-09-23T20:36:18.346734+0000","last_fullsized":"2023-09-23T20:36:18.346734+0000","mapping_epoch":308,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:33:43.135001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.14","version":"90'161","reported_seq":701,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T20:36:18.347578+0000","last_change":"2023-09-23T20:35:02.236398+0000","last_active":"2023-09-23T20:36:18.347578+0000","last_peered":"2023-09-23T20:36:18.347578+0000","last_clean":"2023-09-23T20:36:18.347578+0000","last_became_active":"2023-09-23T20:35:02.235988+0000","last_became_peered":"2023-09-23T20:35:02.235988+0000","last_unstale":"2023-09-23T20:36:18.347578+0000","last_undegraded":"2023-09-23T20:36:18.347578+0000","last_fullsized":"2023-09-23T20:36:18.347578+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:47.479012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:47.479012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:13.519648+0000","scrub_duration":0.0020136590000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":349,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T20:36:15.187302+0000","last_change":"2023-09-23T20:29:45.523808+0000","last_active":"2023-09-23T20:36:15.187302+0000","last_peered":"2023-09-23T20:36:15.187302+0000","last_clean":"2023-09-23T20:36:15.187302+0000","last_became_active":"2023-09-23T20:29:45.523508+0000","last_became_peered":"2023-09-23T20:29:45.523508+0000","last_unstale":"2023-09-23T20:36:15.187302+0000","last_undegraded":"2023-09-23T20:36:15.187302+0000","last_fullsized":"2023-09-23T20:36:15.187302+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:50:55.565352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":173,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T20:36:18.346510+0000","last_change":"2023-09-23T20:32:50.539003+0000","last_active":"2023-09-23T20:36:18.346510+0000","last_peered":"2023-09-23T20:36:18.346510+0000","last_clean":"2023-09-23T20:36:18.346510+0000","last_became_active":"2023-09-23T20:32:50.538596+0000","last_became_peered":"2023-09-23T20:32:50.538596+0000","last_unstale":"2023-09-23T20:36:18.346510+0000","last_undegraded":"2023-09-23T20:36:18.346510+0000","last_fullsized":"2023-09-23T20:36:18.346510+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:56:03.314375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.17","version":"90'161","reported_seq":514,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T20:36:15.187711+0000","last_change":"2023-09-23T20:36:03.435557+0000","last_active":"2023-09-23T20:36:15.187711+0000","last_peered":"2023-09-23T20:36:15.187711+0000","last_clean":"2023-09-23T20:36:15.187711+0000","last_became_active":"2023-09-23T20:35:26.652669+0000","last_became_peered":"2023-09-23T20:35:26.652669+0000","last_unstale":"2023-09-23T20:36:15.187711+0000","last_undegraded":"2023-09-23T20:36:15.187711+0000","last_fullsized":"2023-09-23T20:36:15.187711+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:03.435480+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:03.435480+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:27:05.066931+0000","scrub_duration":0.001636689,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":401,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T20:36:18.346603+0000","last_change":"2023-09-23T20:32:33.270617+0000","last_active":"2023-09-23T20:36:18.346603+0000","last_peered":"2023-09-23T20:36:18.346603+0000","last_clean":"2023-09-23T20:36:18.346603+0000","last_became_active":"2023-09-23T20:32:33.227902+0000","last_became_peered":"2023-09-23T20:32:33.227902+0000","last_unstale":"2023-09-23T20:36:18.346603+0000","last_undegraded":"2023-09-23T20:36:18.346603+0000","last_fullsized":"2023-09-23T20:36:18.346603+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:30:26.847831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":360,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":322,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T20:36:15.183812+0000","last_change":"2023-09-23T20:29:46.080766+0000","last_active":"2023-09-23T20:36:15.183812+0000","last_peered":"2023-09-23T20:36:15.183812+0000","last_clean":"2023-09-23T20:36:15.183812+0000","last_became_active":"2023-09-23T20:29:46.080356+0000","last_became_peered":"2023-09-23T20:29:46.080356+0000","last_unstale":"2023-09-23T20:36:15.183812+0000","last_undegraded":"2023-09-23T20:36:15.183812+0000","last_fullsized":"2023-09-23T20:36:15.183812+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:08:01.862270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"90'161","reported_seq":495,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T20:36:15.183308+0000","last_change":"2023-09-23T20:36:02.548387+0000","last_active":"2023-09-23T20:36:15.183308+0000","last_peered":"2023-09-23T20:36:15.183308+0000","last_clean":"2023-09-23T20:36:15.183308+0000","last_became_active":"2023-09-23T20:35:26.644787+0000","last_became_peered":"2023-09-23T20:35:26.644787+0000","last_unstale":"2023-09-23T20:36:15.183308+0000","last_undegraded":"2023-09-23T20:36:15.183308+0000","last_fullsized":"2023-09-23T20:36:15.183308+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:02.548285+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:02.548285+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:37:15.216553+0000","scrub_duration":0.0015514459999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":80,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T20:36:15.188897+0000","last_change":"2023-09-23T20:32:33.266787+0000","last_active":"2023-09-23T20:36:15.188897+0000","last_peered":"2023-09-23T20:36:15.188897+0000","last_clean":"2023-09-23T20:36:15.188897+0000","last_became_active":"2023-09-23T20:32:33.228482+0000","last_became_peered":"2023-09-23T20:32:33.228482+0000","last_unstale":"2023-09-23T20:36:15.188897+0000","last_undegraded":"2023-09-23T20:36:15.188897+0000","last_fullsized":"2023-09-23T20:36:15.188897+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:13:01.812073+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":70,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T20:36:15.188758+0000","last_change":"2023-09-23T20:32:33.228929+0000","last_active":"2023-09-23T20:36:15.188758+0000","last_peered":"2023-09-23T20:36:15.188758+0000","last_clean":"2023-09-23T20:36:15.188758+0000","last_became_active":"2023-09-23T20:32:33.228595+0000","last_became_peered":"2023-09-23T20:32:33.228595+0000","last_unstale":"2023-09-23T20:36:15.188758+0000","last_undegraded":"2023-09-23T20:36:15.188758+0000","last_fullsized":"2023-09-23T20:36:15.188758+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:17:53.053808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"90'161","reported_seq":604,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T20:36:18.350097+0000","last_change":"2023-09-23T20:31:43.578810+0000","last_active":"2023-09-23T20:36:18.350097+0000","last_peered":"2023-09-23T20:36:18.350097+0000","last_clean":"2023-09-23T20:36:18.350097+0000","last_became_active":"2023-09-23T20:28:05.991304+0000","last_became_peered":"2023-09-23T20:28:05.991304+0000","last_unstale":"2023-09-23T20:36:18.350097+0000","last_undegraded":"2023-09-23T20:36:18.350097+0000","last_fullsized":"2023-09-23T20:36:18.350097+0000","mapping_epoch":231,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":232,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:43.578666+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:43.578666+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:46:18.999991+0000","scrub_duration":0.0021890379999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":400,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T20:36:18.346309+0000","last_change":"2023-09-23T20:32:33.242943+0000","last_active":"2023-09-23T20:36:18.346309+0000","last_peered":"2023-09-23T20:36:18.346309+0000","last_clean":"2023-09-23T20:36:18.346309+0000","last_became_active":"2023-09-23T20:32:33.225468+0000","last_became_peered":"2023-09-23T20:32:33.225468+0000","last_unstale":"2023-09-23T20:36:18.346309+0000","last_undegraded":"2023-09-23T20:36:18.346309+0000","last_fullsized":"2023-09-23T20:36:18.346309+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:40:47.867573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":74,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T20:36:15.186731+0000","last_change":"2023-09-23T20:32:01.358590+0000","last_active":"2023-09-23T20:36:15.186731+0000","last_peered":"2023-09-23T20:36:15.186731+0000","last_clean":"2023-09-23T20:36:15.186731+0000","last_became_active":"2023-09-23T20:32:01.358345+0000","last_became_peered":"2023-09-23T20:32:01.358345+0000","last_unstale":"2023-09-23T20:36:15.186731+0000","last_undegraded":"2023-09-23T20:36:15.186731+0000","last_fullsized":"2023-09-23T20:36:15.186731+0000","mapping_epoch":308,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":309,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:22:21.159708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.10","version":"90'161","reported_seq":603,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T20:36:15.186657+0000","last_change":"2023-09-23T20:31:49.726214+0000","last_active":"2023-09-23T20:36:15.186657+0000","last_peered":"2023-09-23T20:36:15.186657+0000","last_clean":"2023-09-23T20:36:15.186657+0000","last_became_active":"2023-09-23T20:29:07.044892+0000","last_became_peered":"2023-09-23T20:29:07.044892+0000","last_unstale":"2023-09-23T20:36:15.186657+0000","last_undegraded":"2023-09-23T20:36:15.186657+0000","last_fullsized":"2023-09-23T20:36:15.186657+0000","mapping_epoch":253,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":254,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:49.726139+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:49.726139+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:59:11.810516+0000","scrub_duration":0.0015441470000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":340,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T20:36:35.455719+0000","last_change":"2023-09-23T20:25:02.953993+0000","last_active":"2023-09-23T20:36:35.455719+0000","last_peered":"2023-09-23T20:36:35.455719+0000","last_clean":"2023-09-23T20:36:35.455719+0000","last_became_active":"2023-09-23T20:25:02.953672+0000","last_became_peered":"2023-09-23T20:25:02.953672+0000","last_unstale":"2023-09-23T20:36:35.455719+0000","last_undegraded":"2023-09-23T20:36:35.455719+0000","last_fullsized":"2023-09-23T20:36:35.455719+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:26:08.603577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":351,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T20:36:15.187174+0000","last_change":"2023-09-23T20:25:02.949584+0000","last_active":"2023-09-23T20:36:15.187174+0000","last_peered":"2023-09-23T20:36:15.187174+0000","last_clean":"2023-09-23T20:36:15.187174+0000","last_became_active":"2023-09-23T20:25:02.949132+0000","last_became_peered":"2023-09-23T20:25:02.949132+0000","last_unstale":"2023-09-23T20:36:15.187174+0000","last_undegraded":"2023-09-23T20:36:15.187174+0000","last_fullsized":"2023-09-23T20:36:15.187174+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:34:20.509823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"90'161","reported_seq":656,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T20:36:15.183816+0000","last_change":"2023-09-23T20:35:02.238923+0000","last_active":"2023-09-23T20:36:15.183816+0000","last_peered":"2023-09-23T20:36:15.183816+0000","last_clean":"2023-09-23T20:36:15.183816+0000","last_became_active":"2023-09-23T20:35:02.238293+0000","last_became_peered":"2023-09-23T20:35:02.238293+0000","last_unstale":"2023-09-23T20:36:15.183816+0000","last_undegraded":"2023-09-23T20:36:15.183816+0000","last_fullsized":"2023-09-23T20:36:15.183816+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:42.544308+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:42.544308+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:51:22.281932+0000","scrub_duration":0.0029958179999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":357,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T20:36:18.345619+0000","last_change":"2023-09-23T20:32:33.225148+0000","last_active":"2023-09-23T20:36:18.345619+0000","last_peered":"2023-09-23T20:36:18.345619+0000","last_clean":"2023-09-23T20:36:18.345619+0000","last_became_active":"2023-09-23T20:32:33.224805+0000","last_became_peered":"2023-09-23T20:32:33.224805+0000","last_unstale":"2023-09-23T20:36:18.345619+0000","last_undegraded":"2023-09-23T20:36:18.345619+0000","last_fullsized":"2023-09-23T20:36:18.345619+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:55:30.891398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":349,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T20:36:18.350924+0000","last_change":"2023-09-23T20:25:02.957010+0000","last_active":"2023-09-23T20:36:18.350924+0000","last_peered":"2023-09-23T20:36:18.350924+0000","last_clean":"2023-09-23T20:36:18.350924+0000","last_became_active":"2023-09-23T20:25:02.956267+0000","last_became_peered":"2023-09-23T20:25:02.956267+0000","last_unstale":"2023-09-23T20:36:18.350924+0000","last_undegraded":"2023-09-23T20:36:18.350924+0000","last_fullsized":"2023-09-23T20:36:18.350924+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:10:55.299422+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"90'161","reported_seq":622,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T20:36:15.187681+0000","last_change":"2023-09-23T20:31:53.707033+0000","last_active":"2023-09-23T20:36:15.187681+0000","last_peered":"2023-09-23T20:36:15.187681+0000","last_clean":"2023-09-23T20:36:15.187681+0000","last_became_active":"2023-09-23T20:29:46.078873+0000","last_became_peered":"2023-09-23T20:29:46.078873+0000","last_unstale":"2023-09-23T20:36:15.187681+0000","last_undegraded":"2023-09-23T20:36:15.187681+0000","last_fullsized":"2023-09-23T20:36:15.187681+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:53.706934+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:53.706934+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:51:58.039657+0000","scrub_duration":0.0016840900000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":143,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T20:36:18.346937+0000","last_change":"2023-09-23T20:29:04.179171+0000","last_active":"2023-09-23T20:36:18.346937+0000","last_peered":"2023-09-23T20:36:18.346937+0000","last_clean":"2023-09-23T20:36:18.346937+0000","last_became_active":"2023-09-23T20:29:04.128672+0000","last_became_peered":"2023-09-23T20:29:04.128672+0000","last_unstale":"2023-09-23T20:36:18.346937+0000","last_undegraded":"2023-09-23T20:36:18.346937+0000","last_fullsized":"2023-09-23T20:36:18.346937+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:49:15.506106+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":186,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T20:36:18.347765+0000","last_change":"2023-09-23T20:32:50.539157+0000","last_active":"2023-09-23T20:36:18.347765+0000","last_peered":"2023-09-23T20:36:18.347765+0000","last_clean":"2023-09-23T20:36:18.347765+0000","last_became_active":"2023-09-23T20:32:50.538889+0000","last_became_peered":"2023-09-23T20:32:50.538889+0000","last_unstale":"2023-09-23T20:36:18.347765+0000","last_undegraded":"2023-09-23T20:36:18.347765+0000","last_fullsized":"2023-09-23T20:36:18.347765+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:28:28.857256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.4","version":"373'2701","reported_seq":2713,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T20:36:35.400706+0000","last_change":"2023-09-23T20:36:11.310529+0000","last_active":"2023-09-23T20:36:35.400706+0000","last_peered":"2023-09-23T20:36:35.400706+0000","last_clean":"2023-09-23T20:36:35.400706+0000","last_became_active":"2023-09-23T20:36:11.310209+0000","last_became_peered":"2023-09-23T20:36:11.310209+0000","last_unstale":"2023-09-23T20:36:35.400706+0000","last_undegraded":"2023-09-23T20:36:35.400706+0000","last_fullsized":"2023-09-23T20:36:35.400706+0000","mapping_epoch":369,"log_start":"0'0","ondisk_log_start":"0'0","created":369,"last_epoch_clean":370,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":2701,"log_dups_size":0,"ondisk_log_size":2701,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:44:59.621229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":177012736,"num_objects":2701,"num_object_clones":0,"num_object_copies":8103,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2701,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2701,"num_write_kb":172864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"90'161","reported_seq":666,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T20:36:18.346411+0000","last_change":"2023-09-23T20:34:59.496438+0000","last_active":"2023-09-23T20:36:18.346411+0000","last_peered":"2023-09-23T20:36:18.346411+0000","last_clean":"2023-09-23T20:36:18.346411+0000","last_became_active":"2023-09-23T20:33:38.970527+0000","last_became_peered":"2023-09-23T20:33:38.970527+0000","last_unstale":"2023-09-23T20:36:18.346411+0000","last_undegraded":"2023-09-23T20:36:18.346411+0000","last_fullsized":"2023-09-23T20:36:18.346411+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:34:59.496375+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:34:59.496375+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:46:18.244734+0000","scrub_duration":0.0017843189999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":377,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T20:36:18.347633+0000","last_change":"2023-09-23T20:32:33.283752+0000","last_active":"2023-09-23T20:36:18.347633+0000","last_peered":"2023-09-23T20:36:18.347633+0000","last_clean":"2023-09-23T20:36:18.347633+0000","last_became_active":"2023-09-23T20:32:33.230503+0000","last_became_peered":"2023-09-23T20:32:33.230503+0000","last_unstale":"2023-09-23T20:36:18.347633+0000","last_undegraded":"2023-09-23T20:36:18.347633+0000","last_fullsized":"2023-09-23T20:36:18.347633+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:29.084410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":334,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T20:36:15.186306+0000","last_change":"2023-09-23T20:29:04.128604+0000","last_active":"2023-09-23T20:36:15.186306+0000","last_peered":"2023-09-23T20:36:15.186306+0000","last_clean":"2023-09-23T20:36:15.186306+0000","last_became_active":"2023-09-23T20:29:04.128003+0000","last_became_peered":"2023-09-23T20:29:04.128003+0000","last_unstale":"2023-09-23T20:36:15.186306+0000","last_undegraded":"2023-09-23T20:36:15.186306+0000","last_fullsized":"2023-09-23T20:36:15.186306+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:50:49.385599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"374'2621","reported_seq":2634,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T20:36:35.675168+0000","last_change":"2023-09-23T20:36:11.483045+0000","last_active":"2023-09-23T20:36:35.675168+0000","last_peered":"2023-09-23T20:36:35.675168+0000","last_clean":"2023-09-23T20:36:35.675168+0000","last_became_active":"2023-09-23T20:36:11.482781+0000","last_became_peered":"2023-09-23T20:36:11.482781+0000","last_unstale":"2023-09-23T20:36:35.675168+0000","last_undegraded":"2023-09-23T20:36:35.675168+0000","last_fullsized":"2023-09-23T20:36:35.675168+0000","mapping_epoch":369,"log_start":"0'0","ondisk_log_start":"0'0","created":369,"last_epoch_clean":370,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":2621,"log_dups_size":0,"ondisk_log_size":2621,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:02:26.550668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":171769856,"num_objects":2621,"num_object_clones":0,"num_object_copies":7863,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2621,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2621,"num_write_kb":167744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.c","version":"90'161","reported_seq":557,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T20:36:18.349953+0000","last_change":"2023-09-23T20:36:05.485974+0000","last_active":"2023-09-23T20:36:18.349953+0000","last_peered":"2023-09-23T20:36:18.349953+0000","last_clean":"2023-09-23T20:36:18.349953+0000","last_became_active":"2023-09-23T20:33:38.989255+0000","last_became_peered":"2023-09-23T20:33:38.989255+0000","last_unstale":"2023-09-23T20:36:18.349953+0000","last_undegraded":"2023-09-23T20:36:18.349953+0000","last_fullsized":"2023-09-23T20:36:18.349953+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:05.485751+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:05.485751+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:48:05.480750+0000","scrub_duration":0.0057838120000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":412,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T20:36:18.345721+0000","last_change":"2023-09-23T20:34:59.987981+0000","last_active":"2023-09-23T20:36:18.345721+0000","last_peered":"2023-09-23T20:36:18.345721+0000","last_clean":"2023-09-23T20:36:18.345721+0000","last_became_active":"2023-09-23T20:34:59.985368+0000","last_became_peered":"2023-09-23T20:34:59.985368+0000","last_unstale":"2023-09-23T20:36:18.345721+0000","last_undegraded":"2023-09-23T20:36:18.345721+0000","last_fullsized":"2023-09-23T20:36:18.345721+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:12:43.082493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":70,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T20:36:15.188649+0000","last_change":"2023-09-23T20:32:33.232142+0000","last_active":"2023-09-23T20:36:15.188649+0000","last_peered":"2023-09-23T20:36:15.188649+0000","last_clean":"2023-09-23T20:36:15.188649+0000","last_became_active":"2023-09-23T20:32:33.231683+0000","last_became_peered":"2023-09-23T20:32:33.231683+0000","last_unstale":"2023-09-23T20:36:15.188649+0000","last_undegraded":"2023-09-23T20:36:15.188649+0000","last_fullsized":"2023-09-23T20:36:15.188649+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:02:58.336348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"374'2612","reported_seq":2626,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T20:36:35.677358+0000","last_change":"2023-09-23T20:36:11.482244+0000","last_active":"2023-09-23T20:36:35.677358+0000","last_peered":"2023-09-23T20:36:35.677358+0000","last_clean":"2023-09-23T20:36:35.677358+0000","last_became_active":"2023-09-23T20:36:11.481929+0000","last_became_peered":"2023-09-23T20:36:11.481929+0000","last_unstale":"2023-09-23T20:36:35.677358+0000","last_undegraded":"2023-09-23T20:36:35.677358+0000","last_fullsized":"2023-09-23T20:36:35.677358+0000","mapping_epoch":369,"log_start":"0'0","ondisk_log_start":"0'0","created":369,"last_epoch_clean":370,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":2612,"log_dups_size":0,"ondisk_log_size":2612,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:29:51.534913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":171180032,"num_objects":2612,"num_object_clones":0,"num_object_copies":7836,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2612,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2612,"num_write_kb":167168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"90'161","reported_seq":596,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T20:36:18.347283+0000","last_change":"2023-09-23T20:32:36.302809+0000","last_active":"2023-09-23T20:36:18.347283+0000","last_peered":"2023-09-23T20:36:18.347283+0000","last_clean":"2023-09-23T20:36:18.347283+0000","last_became_active":"2023-09-23T20:32:36.302465+0000","last_became_peered":"2023-09-23T20:32:36.302465+0000","last_unstale":"2023-09-23T20:36:18.347283+0000","last_undegraded":"2023-09-23T20:36:18.347283+0000","last_fullsized":"2023-09-23T20:36:18.347283+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:46.217489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:46.217489+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:35:40.488427+0000","scrub_duration":0.001456429,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":346,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T20:36:18.346448+0000","last_change":"2023-09-23T20:29:04.184893+0000","last_active":"2023-09-23T20:36:18.346448+0000","last_peered":"2023-09-23T20:36:18.346448+0000","last_clean":"2023-09-23T20:36:18.346448+0000","last_became_active":"2023-09-23T20:29:04.152723+0000","last_became_peered":"2023-09-23T20:29:04.152723+0000","last_unstale":"2023-09-23T20:36:18.346448+0000","last_undegraded":"2023-09-23T20:36:18.346448+0000","last_fullsized":"2023-09-23T20:36:18.346448+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:51:27.017151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":147,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T20:36:18.347060+0000","last_change":"2023-09-23T20:29:46.086912+0000","last_active":"2023-09-23T20:36:18.347060+0000","last_peered":"2023-09-23T20:36:18.347060+0000","last_clean":"2023-09-23T20:36:18.347060+0000","last_became_active":"2023-09-23T20:29:46.085471+0000","last_became_peered":"2023-09-23T20:29:46.085471+0000","last_unstale":"2023-09-23T20:36:18.347060+0000","last_undegraded":"2023-09-23T20:36:18.347060+0000","last_fullsized":"2023-09-23T20:36:18.347060+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:04:55.751660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"373'2468","reported_seq":2481,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T20:36:34.203843+0000","last_change":"2023-09-23T20:36:11.307165+0000","last_active":"2023-09-23T20:36:34.203843+0000","last_peered":"2023-09-23T20:36:34.203843+0000","last_clean":"2023-09-23T20:36:34.203843+0000","last_became_active":"2023-09-23T20:36:11.306749+0000","last_became_peered":"2023-09-23T20:36:11.306749+0000","last_unstale":"2023-09-23T20:36:34.203843+0000","last_undegraded":"2023-09-23T20:36:34.203843+0000","last_fullsized":"2023-09-23T20:36:34.203843+0000","mapping_epoch":369,"log_start":"0'0","ondisk_log_start":"0'0","created":369,"last_epoch_clean":370,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":2468,"log_dups_size":0,"ondisk_log_size":2468,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:14:02.892561+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":161742848,"num_objects":2468,"num_object_clones":0,"num_object_copies":7404,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2468,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2468,"num_write_kb":157952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.e","version":"90'161","reported_seq":473,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T20:36:15.187930+0000","last_change":"2023-09-23T20:36:00.322453+0000","last_active":"2023-09-23T20:36:15.187930+0000","last_peered":"2023-09-23T20:36:15.187930+0000","last_clean":"2023-09-23T20:36:15.187930+0000","last_became_active":"2023-09-23T20:35:02.233536+0000","last_became_peered":"2023-09-23T20:35:02.233536+0000","last_unstale":"2023-09-23T20:36:15.187930+0000","last_undegraded":"2023-09-23T20:36:15.187930+0000","last_fullsized":"2023-09-23T20:36:15.187930+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:00.322381+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:00.322381+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:02:15.489069+0000","scrub_duration":0.0017321859999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":379,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T20:36:15.187588+0000","last_change":"2023-09-23T20:29:04.138284+0000","last_active":"2023-09-23T20:36:15.187588+0000","last_peered":"2023-09-23T20:36:15.187588+0000","last_clean":"2023-09-23T20:36:15.187588+0000","last_became_active":"2023-09-23T20:29:04.137960+0000","last_became_peered":"2023-09-23T20:29:04.137960+0000","last_unstale":"2023-09-23T20:36:15.187588+0000","last_undegraded":"2023-09-23T20:36:15.187588+0000","last_fullsized":"2023-09-23T20:36:15.187588+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":251,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:18:59.548311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":358,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T20:36:18.350041+0000","last_change":"2023-09-23T20:32:33.236122+0000","last_active":"2023-09-23T20:36:18.350041+0000","last_peered":"2023-09-23T20:36:18.350041+0000","last_clean":"2023-09-23T20:36:18.350041+0000","last_became_active":"2023-09-23T20:32:33.235384+0000","last_became_peered":"2023-09-23T20:32:33.235384+0000","last_unstale":"2023-09-23T20:36:18.350041+0000","last_undegraded":"2023-09-23T20:36:18.350041+0000","last_fullsized":"2023-09-23T20:36:18.350041+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:56.843733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"373'2547","reported_seq":2559,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T20:36:35.396004+0000","last_change":"2023-09-23T20:36:11.311613+0000","last_active":"2023-09-23T20:36:35.396004+0000","last_peered":"2023-09-23T20:36:35.396004+0000","last_clean":"2023-09-23T20:36:35.396004+0000","last_became_active":"2023-09-23T20:36:11.311359+0000","last_became_peered":"2023-09-23T20:36:11.311359+0000","last_unstale":"2023-09-23T20:36:35.396004+0000","last_undegraded":"2023-09-23T20:36:35.396004+0000","last_fullsized":"2023-09-23T20:36:35.396004+0000","mapping_epoch":369,"log_start":"0'0","ondisk_log_start":"0'0","created":369,"last_epoch_clean":370,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":2547,"log_dups_size":0,"ondisk_log_size":2547,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:32:06.448223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":166920192,"num_objects":2547,"num_object_clones":0,"num_object_copies":7641,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2547,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2547,"num_write_kb":163008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":779,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T20:36:18.345473+0000","last_change":"2023-09-23T20:35:02.523308+0000","last_active":"2023-09-23T20:36:18.345473+0000","last_peered":"2023-09-23T20:36:18.345473+0000","last_clean":"2023-09-23T20:36:18.345473+0000","last_became_active":"2023-09-23T20:33:38.979989+0000","last_became_peered":"2023-09-23T20:33:38.979989+0000","last_unstale":"2023-09-23T20:36:18.345473+0000","last_undegraded":"2023-09-23T20:36:18.345473+0000","last_fullsized":"2023-09-23T20:36:18.345473+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:35:02.523203+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:35:02.523203+0000","last_clean_scrub_stamp":"2023-09-23T20:35:02.523203+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:02:41.472958+0000","scrub_duration":0.0085927989999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":109,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T20:36:18.350140+0000","last_change":"2023-09-23T20:32:50.538500+0000","last_active":"2023-09-23T20:36:18.350140+0000","last_peered":"2023-09-23T20:36:18.350140+0000","last_clean":"2023-09-23T20:36:18.350140+0000","last_became_active":"2023-09-23T20:32:50.538152+0000","last_became_peered":"2023-09-23T20:32:50.538152+0000","last_unstale":"2023-09-23T20:36:18.350140+0000","last_undegraded":"2023-09-23T20:36:18.350140+0000","last_fullsized":"2023-09-23T20:36:18.350140+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:00:41.103508+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":70,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T20:36:15.188922+0000","last_change":"2023-09-23T20:32:33.231614+0000","last_active":"2023-09-23T20:36:15.188922+0000","last_peered":"2023-09-23T20:36:15.188922+0000","last_clean":"2023-09-23T20:36:15.188922+0000","last_became_active":"2023-09-23T20:32:33.231207+0000","last_became_peered":"2023-09-23T20:32:33.231207+0000","last_unstale":"2023-09-23T20:36:15.188922+0000","last_undegraded":"2023-09-23T20:36:15.188922+0000","last_fullsized":"2023-09-23T20:36:15.188922+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:46:09.780829+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"373'2497","reported_seq":2510,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T20:36:34.195215+0000","last_change":"2023-09-23T20:36:11.307387+0000","last_active":"2023-09-23T20:36:34.195215+0000","last_peered":"2023-09-23T20:36:34.195215+0000","last_clean":"2023-09-23T20:36:34.195215+0000","last_became_active":"2023-09-23T20:36:11.307082+0000","last_became_peered":"2023-09-23T20:36:11.307082+0000","last_unstale":"2023-09-23T20:36:34.195215+0000","last_undegraded":"2023-09-23T20:36:34.195215+0000","last_fullsized":"2023-09-23T20:36:34.195215+0000","mapping_epoch":369,"log_start":"0'0","ondisk_log_start":"0'0","created":369,"last_epoch_clean":370,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":2497,"log_dups_size":0,"ondisk_log_size":2497,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:06:28.610503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163643392,"num_objects":2497,"num_object_clones":0,"num_object_copies":7491,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2497,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2497,"num_write_kb":159808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":623,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T20:36:15.186470+0000","last_change":"2023-09-23T20:35:00.496271+0000","last_active":"2023-09-23T20:36:15.186470+0000","last_peered":"2023-09-23T20:36:15.186470+0000","last_clean":"2023-09-23T20:36:15.186470+0000","last_became_active":"2023-09-23T20:33:38.996273+0000","last_became_peered":"2023-09-23T20:33:38.996273+0000","last_unstale":"2023-09-23T20:36:15.186470+0000","last_undegraded":"2023-09-23T20:36:15.186470+0000","last_fullsized":"2023-09-23T20:36:15.186470+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:35:00.496197+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:35:00.496197+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:30:15.813493+0000","scrub_duration":0.0016339779999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":169,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T20:36:18.347139+0000","last_change":"2023-09-23T20:32:33.229700+0000","last_active":"2023-09-23T20:36:18.347139+0000","last_peered":"2023-09-23T20:36:18.347139+0000","last_clean":"2023-09-23T20:36:18.347139+0000","last_became_active":"2023-09-23T20:32:33.229372+0000","last_became_peered":"2023-09-23T20:32:33.229372+0000","last_unstale":"2023-09-23T20:36:18.347139+0000","last_undegraded":"2023-09-23T20:36:18.347139+0000","last_fullsized":"2023-09-23T20:36:18.347139+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:18:10.958227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":29,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T20:36:15.186418+0000","last_change":"2023-09-23T20:34:59.985545+0000","last_active":"2023-09-23T20:36:15.186418+0000","last_peered":"2023-09-23T20:36:15.186418+0000","last_clean":"2023-09-23T20:36:15.186418+0000","last_became_active":"2023-09-23T20:34:59.985090+0000","last_became_peered":"2023-09-23T20:34:59.985090+0000","last_unstale":"2023-09-23T20:36:15.186418+0000","last_undegraded":"2023-09-23T20:36:15.186418+0000","last_fullsized":"2023-09-23T20:36:15.186418+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:21:40.825065+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"373'2632","reported_seq":2644,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T20:36:35.386679+0000","last_change":"2023-09-23T20:36:11.309600+0000","last_active":"2023-09-23T20:36:35.386679+0000","last_peered":"2023-09-23T20:36:35.386679+0000","last_clean":"2023-09-23T20:36:35.386679+0000","last_became_active":"2023-09-23T20:36:11.309343+0000","last_became_peered":"2023-09-23T20:36:11.309343+0000","last_unstale":"2023-09-23T20:36:35.386679+0000","last_undegraded":"2023-09-23T20:36:35.386679+0000","last_fullsized":"2023-09-23T20:36:35.386679+0000","mapping_epoch":369,"log_start":"0'0","ondisk_log_start":"0'0","created":369,"last_epoch_clean":370,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":2632,"log_dups_size":0,"ondisk_log_size":2632,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:20:32.774345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":172490752,"num_objects":2632,"num_object_clones":0,"num_object_copies":7896,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2632,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2632,"num_write_kb":168448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"90'161","reported_seq":639,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T20:36:18.350282+0000","last_change":"2023-09-23T20:36:05.595168+0000","last_active":"2023-09-23T20:36:18.350282+0000","last_peered":"2023-09-23T20:36:18.350282+0000","last_clean":"2023-09-23T20:36:18.350282+0000","last_became_active":"2023-09-23T20:33:38.991910+0000","last_became_peered":"2023-09-23T20:33:38.991910+0000","last_unstale":"2023-09-23T20:36:18.350282+0000","last_undegraded":"2023-09-23T20:36:18.350282+0000","last_fullsized":"2023-09-23T20:36:18.350282+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:05.595042+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:05.595042+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:06:55.394493+0000","scrub_duration":0.007271675,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":29,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T20:36:15.188346+0000","last_change":"2023-09-23T20:34:59.983122+0000","last_active":"2023-09-23T20:36:15.188346+0000","last_peered":"2023-09-23T20:36:15.188346+0000","last_clean":"2023-09-23T20:36:15.188346+0000","last_became_active":"2023-09-23T20:34:59.982809+0000","last_became_peered":"2023-09-23T20:34:59.982809+0000","last_unstale":"2023-09-23T20:36:15.188346+0000","last_undegraded":"2023-09-23T20:36:15.188346+0000","last_fullsized":"2023-09-23T20:36:15.188346+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:49:40.839723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":70,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T20:36:15.188041+0000","last_change":"2023-09-23T20:32:33.224963+0000","last_active":"2023-09-23T20:36:15.188041+0000","last_peered":"2023-09-23T20:36:15.188041+0000","last_clean":"2023-09-23T20:36:15.188041+0000","last_became_active":"2023-09-23T20:32:33.224634+0000","last_became_peered":"2023-09-23T20:32:33.224634+0000","last_unstale":"2023-09-23T20:36:15.188041+0000","last_undegraded":"2023-09-23T20:36:15.188041+0000","last_fullsized":"2023-09-23T20:36:15.188041+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:41:40.314140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"373'2480","reported_seq":2493,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T20:36:34.226974+0000","last_change":"2023-09-23T20:36:11.307835+0000","last_active":"2023-09-23T20:36:34.226974+0000","last_peered":"2023-09-23T20:36:34.226974+0000","last_clean":"2023-09-23T20:36:34.226974+0000","last_became_active":"2023-09-23T20:36:11.307592+0000","last_became_peered":"2023-09-23T20:36:11.307592+0000","last_unstale":"2023-09-23T20:36:34.226974+0000","last_undegraded":"2023-09-23T20:36:34.226974+0000","last_fullsized":"2023-09-23T20:36:34.226974+0000","mapping_epoch":369,"log_start":"0'0","ondisk_log_start":"0'0","created":369,"last_epoch_clean":370,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":2480,"log_dups_size":0,"ondisk_log_size":2480,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:27:55.106087+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":162529280,"num_objects":2480,"num_object_clones":0,"num_object_copies":7440,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2480,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2480,"num_write_kb":158720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":666,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T20:36:15.187554+0000","last_change":"2023-09-23T20:34:59.481511+0000","last_active":"2023-09-23T20:36:15.187554+0000","last_peered":"2023-09-23T20:36:15.187554+0000","last_clean":"2023-09-23T20:36:15.187554+0000","last_became_active":"2023-09-23T20:33:38.988954+0000","last_became_peered":"2023-09-23T20:33:38.988954+0000","last_unstale":"2023-09-23T20:36:15.187554+0000","last_undegraded":"2023-09-23T20:36:15.187554+0000","last_fullsized":"2023-09-23T20:36:15.187554+0000","mapping_epoch":253,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:34:59.481446+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:34:59.481446+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:53:42.768648+0000","scrub_duration":0.001554773,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":186,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T20:36:15.183529+0000","last_change":"2023-09-23T20:32:50.530690+0000","last_active":"2023-09-23T20:36:15.183529+0000","last_peered":"2023-09-23T20:36:15.183529+0000","last_clean":"2023-09-23T20:36:15.183529+0000","last_became_active":"2023-09-23T20:32:50.530376+0000","last_became_peered":"2023-09-23T20:32:50.530376+0000","last_unstale":"2023-09-23T20:36:15.183529+0000","last_undegraded":"2023-09-23T20:36:15.183529+0000","last_fullsized":"2023-09-23T20:36:15.183529+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:24:53.184959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":195,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T20:36:18.350189+0000","last_change":"2023-09-23T20:32:33.230582+0000","last_active":"2023-09-23T20:36:18.350189+0000","last_peered":"2023-09-23T20:36:18.350189+0000","last_clean":"2023-09-23T20:36:18.350189+0000","last_became_active":"2023-09-23T20:32:33.230257+0000","last_became_peered":"2023-09-23T20:32:33.230257+0000","last_unstale":"2023-09-23T20:36:18.350189+0000","last_undegraded":"2023-09-23T20:36:18.350189+0000","last_fullsized":"2023-09-23T20:36:18.350189+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:11:45.333096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"374'2589","reported_seq":2603,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T20:36:35.670937+0000","last_change":"2023-09-23T20:36:11.310149+0000","last_active":"2023-09-23T20:36:35.670937+0000","last_peered":"2023-09-23T20:36:35.670937+0000","last_clean":"2023-09-23T20:36:35.670937+0000","last_became_active":"2023-09-23T20:36:11.309844+0000","last_became_peered":"2023-09-23T20:36:11.309844+0000","last_unstale":"2023-09-23T20:36:35.670937+0000","last_undegraded":"2023-09-23T20:36:35.670937+0000","last_fullsized":"2023-09-23T20:36:35.670937+0000","mapping_epoch":369,"log_start":"0'0","ondisk_log_start":"0'0","created":369,"last_epoch_clean":370,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":2589,"log_dups_size":0,"ondisk_log_size":2589,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:14:57.442108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":169672704,"num_objects":2589,"num_object_clones":0,"num_object_copies":7767,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2589,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2589,"num_write_kb":165696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":623,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T20:36:18.347537+0000","last_change":"2023-09-23T20:35:04.102089+0000","last_active":"2023-09-23T20:36:18.347537+0000","last_peered":"2023-09-23T20:36:18.347537+0000","last_clean":"2023-09-23T20:36:18.347537+0000","last_became_active":"2023-09-23T20:33:38.978580+0000","last_became_peered":"2023-09-23T20:33:38.978580+0000","last_unstale":"2023-09-23T20:36:18.347537+0000","last_undegraded":"2023-09-23T20:36:18.347537+0000","last_fullsized":"2023-09-23T20:36:18.347537+0000","mapping_epoch":253,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:35:04.101972+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:35:04.101972+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:44:38.790944+0000","scrub_duration":0.0063459650000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"86'119","reported_seq":679,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T20:36:15.183818+0000","last_change":"2023-09-23T20:29:45.521257+0000","last_active":"2023-09-23T20:36:15.183818+0000","last_peered":"2023-09-23T20:36:15.183818+0000","last_clean":"2023-09-23T20:36:15.183818+0000","last_became_active":"2023-09-23T20:29:45.520546+0000","last_became_peered":"2023-09-23T20:29:45.520546+0000","last_unstale":"2023-09-23T20:36:15.183818+0000","last_undegraded":"2023-09-23T20:36:15.183818+0000","last_fullsized":"2023-09-23T20:36:15.183818+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":119,"log_dups_size":0,"ondisk_log_size":119,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:58:56.085777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032705499999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":523,"num_read_kb":3503,"num_write":136,"num_write_kb":3128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":389,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T20:36:15.187577+0000","last_change":"2023-09-23T20:25:39.637160+0000","last_active":"2023-09-23T20:36:15.187577+0000","last_peered":"2023-09-23T20:36:15.187577+0000","last_clean":"2023-09-23T20:36:15.187577+0000","last_became_active":"2023-09-23T20:25:39.636860+0000","last_became_peered":"2023-09-23T20:25:39.636860+0000","last_unstale":"2023-09-23T20:36:15.187577+0000","last_undegraded":"2023-09-23T20:36:15.187577+0000","last_fullsized":"2023-09-23T20:36:15.187577+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:21:47.860063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":308,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T20:36:15.183466+0000","last_change":"2023-09-23T20:34:59.985572+0000","last_active":"2023-09-23T20:36:15.183466+0000","last_peered":"2023-09-23T20:36:15.183466+0000","last_clean":"2023-09-23T20:36:15.183466+0000","last_became_active":"2023-09-23T20:34:59.985241+0000","last_became_peered":"2023-09-23T20:34:59.985241+0000","last_unstale":"2023-09-23T20:36:15.183466+0000","last_undegraded":"2023-09-23T20:36:15.183466+0000","last_fullsized":"2023-09-23T20:36:15.183466+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:54:57.081334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.8","version":"373'2617","reported_seq":2629,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T20:36:35.396071+0000","last_change":"2023-09-23T20:36:11.482049+0000","last_active":"2023-09-23T20:36:35.396071+0000","last_peered":"2023-09-23T20:36:35.396071+0000","last_clean":"2023-09-23T20:36:35.396071+0000","last_became_active":"2023-09-23T20:36:11.481701+0000","last_became_peered":"2023-09-23T20:36:11.481701+0000","last_unstale":"2023-09-23T20:36:35.396071+0000","last_undegraded":"2023-09-23T20:36:35.396071+0000","last_fullsized":"2023-09-23T20:36:35.396071+0000","mapping_epoch":369,"log_start":"0'0","ondisk_log_start":"0'0","created":369,"last_epoch_clean":370,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":2617,"log_dups_size":0,"ondisk_log_size":2617,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:27:21.469954+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":171507712,"num_objects":2617,"num_object_clones":0,"num_object_copies":7851,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2617,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2617,"num_write_kb":167488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":617,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T20:36:15.186527+0000","last_change":"2023-09-23T20:31:45.785521+0000","last_active":"2023-09-23T20:36:15.186527+0000","last_peered":"2023-09-23T20:36:15.186527+0000","last_clean":"2023-09-23T20:36:15.186527+0000","last_became_active":"2023-09-23T20:27:55.213394+0000","last_became_peered":"2023-09-23T20:27:55.213394+0000","last_unstale":"2023-09-23T20:36:15.186527+0000","last_undegraded":"2023-09-23T20:36:15.186527+0000","last_fullsized":"2023-09-23T20:36:15.186527+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:45.785450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:45.785450+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:21:26.092578+0000","scrub_duration":0.001558306,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"86'106","reported_seq":1091,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T20:36:15.187281+0000","last_change":"2023-09-23T20:31:31.531882+0000","last_active":"2023-09-23T20:36:15.187281+0000","last_peered":"2023-09-23T20:36:15.187281+0000","last_clean":"2023-09-23T20:36:15.187281+0000","last_became_active":"2023-09-23T20:29:46.078501+0000","last_became_peered":"2023-09-23T20:29:46.078501+0000","last_unstale":"2023-09-23T20:36:15.187281+0000","last_undegraded":"2023-09-23T20:36:15.187281+0000","last_fullsized":"2023-09-23T20:36:15.187281+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":106,"log_dups_size":0,"ondisk_log_size":106,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:37.015974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00099302599999999993,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1105,"num_read_kb":1689,"num_write":100,"num_write_kb":404,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":70,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T20:36:15.188606+0000","last_change":"2023-09-23T20:32:33.230242+0000","last_active":"2023-09-23T20:36:15.188606+0000","last_peered":"2023-09-23T20:36:15.188606+0000","last_clean":"2023-09-23T20:36:15.188606+0000","last_became_active":"2023-09-23T20:32:33.229732+0000","last_became_peered":"2023-09-23T20:32:33.229732+0000","last_unstale":"2023-09-23T20:36:15.188606+0000","last_undegraded":"2023-09-23T20:36:15.188606+0000","last_fullsized":"2023-09-23T20:36:15.188606+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:27:55.538429+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":332,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T20:36:15.183767+0000","last_change":"2023-09-23T20:29:45.533406+0000","last_active":"2023-09-23T20:36:15.183767+0000","last_peered":"2023-09-23T20:36:15.183767+0000","last_clean":"2023-09-23T20:36:15.183767+0000","last_became_active":"2023-09-23T20:29:45.526001+0000","last_became_peered":"2023-09-23T20:29:45.526001+0000","last_unstale":"2023-09-23T20:36:15.183767+0000","last_undegraded":"2023-09-23T20:36:15.183767+0000","last_fullsized":"2023-09-23T20:36:15.183767+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:43.787251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.b","version":"373'2571","reported_seq":2583,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T20:36:35.386814+0000","last_change":"2023-09-23T20:36:11.310405+0000","last_active":"2023-09-23T20:36:35.386814+0000","last_peered":"2023-09-23T20:36:35.386814+0000","last_clean":"2023-09-23T20:36:35.386814+0000","last_became_active":"2023-09-23T20:36:11.310095+0000","last_became_peered":"2023-09-23T20:36:11.310095+0000","last_unstale":"2023-09-23T20:36:35.386814+0000","last_undegraded":"2023-09-23T20:36:35.386814+0000","last_fullsized":"2023-09-23T20:36:35.386814+0000","mapping_epoch":369,"log_start":"0'0","ondisk_log_start":"0'0","created":369,"last_epoch_clean":370,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":2571,"log_dups_size":0,"ondisk_log_size":2571,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:28:16.000557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":168493056,"num_objects":2571,"num_object_clones":0,"num_object_copies":7713,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2571,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2571,"num_write_kb":164544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":846,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T20:36:18.346279+0000","last_change":"2023-09-23T20:35:02.260708+0000","last_active":"2023-09-23T20:36:18.346279+0000","last_peered":"2023-09-23T20:36:18.346279+0000","last_clean":"2023-09-23T20:36:18.346279+0000","last_became_active":"2023-09-23T20:35:02.260348+0000","last_became_peered":"2023-09-23T20:35:02.260348+0000","last_unstale":"2023-09-23T20:36:18.346279+0000","last_undegraded":"2023-09-23T20:36:18.346279+0000","last_fullsized":"2023-09-23T20:36:18.346279+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:43.390908+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:43.390908+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:09:14.144070+0000","scrub_duration":0.0032721,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"86'130","reported_seq":688,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T20:36:18.347131+0000","last_change":"2023-09-23T20:25:39.637458+0000","last_active":"2023-09-23T20:36:18.347131+0000","last_peered":"2023-09-23T20:36:18.347131+0000","last_clean":"2023-09-23T20:36:18.347131+0000","last_became_active":"2023-09-23T20:25:39.636773+0000","last_became_peered":"2023-09-23T20:25:39.636773+0000","last_unstale":"2023-09-23T20:36:18.347131+0000","last_undegraded":"2023-09-23T20:36:18.347131+0000","last_fullsized":"2023-09-23T20:36:18.347131+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:00:32.810564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011300049999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":512,"num_read_kb":1820,"num_write":133,"num_write_kb":1347,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":322,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T20:36:15.183438+0000","last_change":"2023-09-23T20:29:46.073394+0000","last_active":"2023-09-23T20:36:15.183438+0000","last_peered":"2023-09-23T20:36:15.183438+0000","last_clean":"2023-09-23T20:36:15.183438+0000","last_became_active":"2023-09-23T20:29:46.072067+0000","last_became_peered":"2023-09-23T20:29:46.072067+0000","last_unstale":"2023-09-23T20:36:15.183438+0000","last_undegraded":"2023-09-23T20:36:15.183438+0000","last_fullsized":"2023-09-23T20:36:15.183438+0000","mapping_epoch":267,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:01:56.944614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":314,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T20:36:18.350112+0000","last_change":"2023-09-23T20:29:45.533185+0000","last_active":"2023-09-23T20:36:18.350112+0000","last_peered":"2023-09-23T20:36:18.350112+0000","last_clean":"2023-09-23T20:36:18.350112+0000","last_became_active":"2023-09-23T20:29:45.526765+0000","last_became_peered":"2023-09-23T20:29:45.526765+0000","last_unstale":"2023-09-23T20:36:18.350112+0000","last_undegraded":"2023-09-23T20:36:18.350112+0000","last_fullsized":"2023-09-23T20:36:18.350112+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:18:33.063215+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"374'2599","reported_seq":2612,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T20:36:35.671637+0000","last_change":"2023-09-23T20:36:11.311818+0000","last_active":"2023-09-23T20:36:35.671637+0000","last_peered":"2023-09-23T20:36:35.671637+0000","last_clean":"2023-09-23T20:36:35.671637+0000","last_became_active":"2023-09-23T20:36:11.311525+0000","last_became_peered":"2023-09-23T20:36:11.311525+0000","last_unstale":"2023-09-23T20:36:35.671637+0000","last_undegraded":"2023-09-23T20:36:35.671637+0000","last_fullsized":"2023-09-23T20:36:35.671637+0000","mapping_epoch":369,"log_start":"0'0","ondisk_log_start":"0'0","created":369,"last_epoch_clean":370,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":2599,"log_dups_size":0,"ondisk_log_size":2599,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:16:49.945652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":170328064,"num_objects":2599,"num_object_clones":0,"num_object_copies":7797,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2599,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2599,"num_write_kb":166336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"86'194","reported_seq":1021,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T20:36:15.187064+0000","last_change":"2023-09-23T20:25:40.618665+0000","last_active":"2023-09-23T20:36:15.187064+0000","last_peered":"2023-09-23T20:36:15.187064+0000","last_clean":"2023-09-23T20:36:15.187064+0000","last_became_active":"2023-09-23T20:25:40.618087+0000","last_became_peered":"2023-09-23T20:25:40.618087+0000","last_unstale":"2023-09-23T20:36:15.187064+0000","last_undegraded":"2023-09-23T20:36:15.187064+0000","last_fullsized":"2023-09-23T20:36:15.187064+0000","mapping_epoch":177,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":178,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":194,"log_dups_size":0,"ondisk_log_size":194,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:07:16.114968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00037995999999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1025,"num_read_kb":1201,"num_write":173,"num_write_kb":1276,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":807,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T20:36:18.345768+0000","last_change":"2023-09-23T20:31:45.343835+0000","last_active":"2023-09-23T20:36:18.345768+0000","last_peered":"2023-09-23T20:36:18.345768+0000","last_clean":"2023-09-23T20:36:18.345768+0000","last_became_active":"2023-09-23T20:29:07.042407+0000","last_became_peered":"2023-09-23T20:29:07.042407+0000","last_unstale":"2023-09-23T20:36:18.345768+0000","last_undegraded":"2023-09-23T20:36:18.345768+0000","last_fullsized":"2023-09-23T20:36:18.345768+0000","mapping_epoch":253,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":254,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:45.343742+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:45.343742+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:33:12.745366+0000","scrub_duration":0.001475802,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":172,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T20:36:15.187389+0000","last_change":"2023-09-23T20:32:50.534315+0000","last_active":"2023-09-23T20:36:15.187389+0000","last_peered":"2023-09-23T20:36:15.187389+0000","last_clean":"2023-09-23T20:36:15.187389+0000","last_became_active":"2023-09-23T20:32:50.533896+0000","last_became_peered":"2023-09-23T20:32:50.533896+0000","last_unstale":"2023-09-23T20:36:15.187389+0000","last_undegraded":"2023-09-23T20:36:15.187389+0000","last_fullsized":"2023-09-23T20:36:15.187389+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:06:53.464923+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":153,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T20:36:18.347250+0000","last_change":"2023-09-23T20:32:33.225012+0000","last_active":"2023-09-23T20:36:18.347250+0000","last_peered":"2023-09-23T20:36:18.347250+0000","last_clean":"2023-09-23T20:36:18.347250+0000","last_became_active":"2023-09-23T20:32:33.224699+0000","last_became_peered":"2023-09-23T20:32:33.224699+0000","last_unstale":"2023-09-23T20:36:18.347250+0000","last_undegraded":"2023-09-23T20:36:18.347250+0000","last_fullsized":"2023-09-23T20:36:18.347250+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:10:06.267878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.a","version":"373'2576","reported_seq":2587,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T20:36:34.233844+0000","last_change":"2023-09-23T20:36:11.311940+0000","last_active":"2023-09-23T20:36:34.233844+0000","last_peered":"2023-09-23T20:36:34.233844+0000","last_clean":"2023-09-23T20:36:34.233844+0000","last_became_active":"2023-09-23T20:36:11.311558+0000","last_became_peered":"2023-09-23T20:36:11.311558+0000","last_unstale":"2023-09-23T20:36:34.233844+0000","last_undegraded":"2023-09-23T20:36:34.233844+0000","last_fullsized":"2023-09-23T20:36:34.233844+0000","mapping_epoch":369,"log_start":"0'0","ondisk_log_start":"0'0","created":369,"last_epoch_clean":370,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":2576,"log_dups_size":0,"ondisk_log_size":2576,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:58:12.645832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":168820736,"num_objects":2576,"num_object_clones":0,"num_object_copies":7728,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2576,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2576,"num_write_kb":164864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":667,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T20:36:15.183737+0000","last_change":"2023-09-23T20:35:02.238632+0000","last_active":"2023-09-23T20:36:15.183737+0000","last_peered":"2023-09-23T20:36:15.183737+0000","last_clean":"2023-09-23T20:36:15.183737+0000","last_became_active":"2023-09-23T20:35:02.237812+0000","last_became_peered":"2023-09-23T20:35:02.237812+0000","last_unstale":"2023-09-23T20:36:15.183737+0000","last_undegraded":"2023-09-23T20:36:15.183737+0000","last_fullsized":"2023-09-23T20:36:15.183737+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:43.582146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:43.582146+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:25:04.582696+0000","scrub_duration":0.0016219730000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"86'80","reported_seq":637,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T20:36:18.345693+0000","last_change":"2023-09-23T20:29:45.536796+0000","last_active":"2023-09-23T20:36:18.345693+0000","last_peered":"2023-09-23T20:36:18.345693+0000","last_clean":"2023-09-23T20:36:18.345693+0000","last_became_active":"2023-09-23T20:29:45.536439+0000","last_became_peered":"2023-09-23T20:29:45.536439+0000","last_unstale":"2023-09-23T20:36:18.345693+0000","last_undegraded":"2023-09-23T20:36:18.345693+0000","last_fullsized":"2023-09-23T20:36:18.345693+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":80,"log_dups_size":0,"ondisk_log_size":80,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:30:57.793132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035115,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":395,"num_read_kb":1370,"num_write":66,"num_write_kb":1061,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":376,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T20:36:18.350311+0000","last_change":"2023-09-23T20:34:59.984286+0000","last_active":"2023-09-23T20:36:18.350311+0000","last_peered":"2023-09-23T20:36:18.350311+0000","last_clean":"2023-09-23T20:36:18.350311+0000","last_became_active":"2023-09-23T20:34:59.983950+0000","last_became_peered":"2023-09-23T20:34:59.983950+0000","last_unstale":"2023-09-23T20:36:18.350311+0000","last_undegraded":"2023-09-23T20:36:18.350311+0000","last_fullsized":"2023-09-23T20:36:18.350311+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:31:53.557035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":371,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T20:36:18.350680+0000","last_change":"2023-09-23T20:29:45.690815+0000","last_active":"2023-09-23T20:36:18.350680+0000","last_peered":"2023-09-23T20:36:18.350680+0000","last_clean":"2023-09-23T20:36:18.350680+0000","last_became_active":"2023-09-23T20:29:45.690478+0000","last_became_peered":"2023-09-23T20:29:45.690478+0000","last_unstale":"2023-09-23T20:36:18.350680+0000","last_undegraded":"2023-09-23T20:36:18.350680+0000","last_fullsized":"2023-09-23T20:36:18.350680+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:38.288193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.f","version":"374'2780","reported_seq":2794,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T20:36:37.787985+0000","last_change":"2023-09-23T20:36:11.309536+0000","last_active":"2023-09-23T20:36:37.787985+0000","last_peered":"2023-09-23T20:36:37.787985+0000","last_clean":"2023-09-23T20:36:37.787985+0000","last_became_active":"2023-09-23T20:36:11.309208+0000","last_became_peered":"2023-09-23T20:36:11.309208+0000","last_unstale":"2023-09-23T20:36:37.787985+0000","last_undegraded":"2023-09-23T20:36:37.787985+0000","last_fullsized":"2023-09-23T20:36:37.787985+0000","mapping_epoch":369,"log_start":"0'0","ondisk_log_start":"0'0","created":369,"last_epoch_clean":370,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":2780,"log_dups_size":0,"ondisk_log_size":2780,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:47:51.785158+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":182190080,"num_objects":2780,"num_object_clones":0,"num_object_copies":8340,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2780,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2780,"num_write_kb":177920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":616,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T20:36:18.348709+0000","last_change":"2023-09-23T20:35:00.151823+0000","last_active":"2023-09-23T20:36:18.348709+0000","last_peered":"2023-09-23T20:36:18.348709+0000","last_clean":"2023-09-23T20:36:18.348709+0000","last_became_active":"2023-09-23T20:33:38.991030+0000","last_became_peered":"2023-09-23T20:33:38.991030+0000","last_unstale":"2023-09-23T20:36:18.348709+0000","last_undegraded":"2023-09-23T20:36:18.348709+0000","last_fullsized":"2023-09-23T20:36:18.348709+0000","mapping_epoch":253,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:35:00.151695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:35:00.151695+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:09:55.183927+0000","scrub_duration":0.0016030090000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"86'102","reported_seq":968,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T20:36:18.346499+0000","last_change":"2023-09-23T20:32:36.314672+0000","last_active":"2023-09-23T20:36:18.346499+0000","last_peered":"2023-09-23T20:36:18.346499+0000","last_clean":"2023-09-23T20:36:18.346499+0000","last_became_active":"2023-09-23T20:32:36.314231+0000","last_became_peered":"2023-09-23T20:32:36.314231+0000","last_unstale":"2023-09-23T20:36:18.346499+0000","last_undegraded":"2023-09-23T20:36:18.346499+0000","last_fullsized":"2023-09-23T20:36:18.346499+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":102,"log_dups_size":0,"ondisk_log_size":102,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:11:40.514709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030656500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":963,"num_read_kb":1234,"num_write":60,"num_write_kb":225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":298,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T20:36:15.193681+0000","last_change":"2023-09-23T20:29:45.538283+0000","last_active":"2023-09-23T20:36:15.193681+0000","last_peered":"2023-09-23T20:36:15.193681+0000","last_clean":"2023-09-23T20:36:15.193681+0000","last_became_active":"2023-09-23T20:29:45.537922+0000","last_became_peered":"2023-09-23T20:29:45.537922+0000","last_unstale":"2023-09-23T20:36:15.193681+0000","last_undegraded":"2023-09-23T20:36:15.193681+0000","last_fullsized":"2023-09-23T20:36:15.193681+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:30:52.397836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":308,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T20:36:18.350326+0000","last_change":"2023-09-23T20:20:14.888874+0000","last_active":"2023-09-23T20:36:18.350326+0000","last_peered":"2023-09-23T20:36:18.350326+0000","last_clean":"2023-09-23T20:36:18.350326+0000","last_became_active":"2023-09-23T20:20:14.887724+0000","last_became_peered":"2023-09-23T20:20:14.887724+0000","last_unstale":"2023-09-23T20:36:18.350326+0000","last_undegraded":"2023-09-23T20:36:18.350326+0000","last_fullsized":"2023-09-23T20:36:18.350326+0000","mapping_epoch":94,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":95,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:09:24.510326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"374'2650","reported_seq":2664,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T20:36:35.671949+0000","last_change":"2023-09-23T20:36:11.482126+0000","last_active":"2023-09-23T20:36:35.671949+0000","last_peered":"2023-09-23T20:36:35.671949+0000","last_clean":"2023-09-23T20:36:35.671949+0000","last_became_active":"2023-09-23T20:36:11.481802+0000","last_became_peered":"2023-09-23T20:36:11.481802+0000","last_unstale":"2023-09-23T20:36:35.671949+0000","last_undegraded":"2023-09-23T20:36:35.671949+0000","last_fullsized":"2023-09-23T20:36:35.671949+0000","mapping_epoch":369,"log_start":"0'0","ondisk_log_start":"0'0","created":369,"last_epoch_clean":370,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":2650,"log_dups_size":0,"ondisk_log_size":2650,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:13:12.263455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":173670400,"num_objects":2650,"num_object_clones":0,"num_object_copies":7950,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2650,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2650,"num_write_kb":169600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":591,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T20:36:15.187715+0000","last_change":"2023-09-23T20:36:03.246372+0000","last_active":"2023-09-23T20:36:15.187715+0000","last_peered":"2023-09-23T20:36:15.187715+0000","last_clean":"2023-09-23T20:36:15.187715+0000","last_became_active":"2023-09-23T20:33:38.985300+0000","last_became_peered":"2023-09-23T20:33:38.985300+0000","last_unstale":"2023-09-23T20:36:15.187715+0000","last_undegraded":"2023-09-23T20:36:15.187715+0000","last_fullsized":"2023-09-23T20:36:15.187715+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:03.246271+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:03.246271+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:12:15.732632+0000","scrub_duration":0.0019016700000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":111,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T20:36:18.347854+0000","last_change":"2023-09-23T20:32:50.549695+0000","last_active":"2023-09-23T20:36:18.347854+0000","last_peered":"2023-09-23T20:36:18.347854+0000","last_clean":"2023-09-23T20:36:18.347854+0000","last_became_active":"2023-09-23T20:32:50.549431+0000","last_became_peered":"2023-09-23T20:32:50.549431+0000","last_unstale":"2023-09-23T20:36:18.347854+0000","last_undegraded":"2023-09-23T20:36:18.347854+0000","last_fullsized":"2023-09-23T20:36:18.347854+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:45:53.666569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"86'128","reported_seq":764,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T20:36:18.349435+0000","last_change":"2023-09-23T20:32:36.310656+0000","last_active":"2023-09-23T20:36:18.349435+0000","last_peered":"2023-09-23T20:36:18.349435+0000","last_clean":"2023-09-23T20:36:18.349435+0000","last_became_active":"2023-09-23T20:32:36.310155+0000","last_became_peered":"2023-09-23T20:32:36.310155+0000","last_unstale":"2023-09-23T20:36:18.349435+0000","last_undegraded":"2023-09-23T20:36:18.349435+0000","last_fullsized":"2023-09-23T20:36:18.349435+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:58:14.217466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000293202,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":579,"num_read_kb":1473,"num_write":96,"num_write_kb":1062,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":144,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T20:36:18.345937+0000","last_change":"2023-09-23T20:32:50.539051+0000","last_active":"2023-09-23T20:36:18.345937+0000","last_peered":"2023-09-23T20:36:18.345937+0000","last_clean":"2023-09-23T20:36:18.345937+0000","last_became_active":"2023-09-23T20:32:50.538755+0000","last_became_peered":"2023-09-23T20:32:50.538755+0000","last_unstale":"2023-09-23T20:36:18.345937+0000","last_undegraded":"2023-09-23T20:36:18.345937+0000","last_fullsized":"2023-09-23T20:36:18.345937+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:16.656613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.d","version":"373'2642","reported_seq":2654,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T20:36:35.397261+0000","last_change":"2023-09-23T20:36:11.482064+0000","last_active":"2023-09-23T20:36:35.397261+0000","last_peered":"2023-09-23T20:36:35.397261+0000","last_clean":"2023-09-23T20:36:35.397261+0000","last_became_active":"2023-09-23T20:36:11.481816+0000","last_became_peered":"2023-09-23T20:36:11.481816+0000","last_unstale":"2023-09-23T20:36:35.397261+0000","last_undegraded":"2023-09-23T20:36:35.397261+0000","last_fullsized":"2023-09-23T20:36:35.397261+0000","mapping_epoch":369,"log_start":"0'0","ondisk_log_start":"0'0","created":369,"last_epoch_clean":370,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":2642,"log_dups_size":0,"ondisk_log_size":2642,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:03:56.861034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":173146112,"num_objects":2642,"num_object_clones":0,"num_object_copies":7926,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2642,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2642,"num_write_kb":169088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.4","version":"90'161","reported_seq":522,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T20:36:15.187556+0000","last_change":"2023-09-23T20:34:54.504652+0000","last_active":"2023-09-23T20:36:15.187556+0000","last_peered":"2023-09-23T20:36:15.187556+0000","last_clean":"2023-09-23T20:36:15.187556+0000","last_became_active":"2023-09-23T20:32:50.533470+0000","last_became_peered":"2023-09-23T20:32:50.533470+0000","last_unstale":"2023-09-23T20:36:15.187556+0000","last_undegraded":"2023-09-23T20:36:15.187556+0000","last_fullsized":"2023-09-23T20:36:15.187556+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:34:54.504554+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:34:54.504554+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:17:03.108888+0000","scrub_duration":0.013031021,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.7","version":"86'117","reported_seq":643,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T20:36:18.347478+0000","last_change":"2023-09-23T20:31:32.814392+0000","last_active":"2023-09-23T20:36:18.347478+0000","last_peered":"2023-09-23T20:36:18.347478+0000","last_clean":"2023-09-23T20:36:18.347478+0000","last_became_active":"2023-09-23T20:31:32.813857+0000","last_became_peered":"2023-09-23T20:31:32.813857+0000","last_unstale":"2023-09-23T20:36:18.347478+0000","last_undegraded":"2023-09-23T20:36:18.347478+0000","last_fullsized":"2023-09-23T20:36:18.347478+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:56:57.863005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046891400000000002,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":893,"num_read_kb":922,"num_write":82,"num_write_kb":4193,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":371,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T20:36:18.350768+0000","last_change":"2023-09-23T20:32:33.277827+0000","last_active":"2023-09-23T20:36:18.350768+0000","last_peered":"2023-09-23T20:36:18.350768+0000","last_clean":"2023-09-23T20:36:18.350768+0000","last_became_active":"2023-09-23T20:32:33.236087+0000","last_became_peered":"2023-09-23T20:32:33.236087+0000","last_unstale":"2023-09-23T20:36:18.350768+0000","last_undegraded":"2023-09-23T20:36:18.350768+0000","last_fullsized":"2023-09-23T20:36:18.350768+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:20:58.176854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":70,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T20:36:15.188566+0000","last_change":"2023-09-23T20:32:33.238290+0000","last_active":"2023-09-23T20:36:15.188566+0000","last_peered":"2023-09-23T20:36:15.188566+0000","last_clean":"2023-09-23T20:36:15.188566+0000","last_became_active":"2023-09-23T20:32:33.237984+0000","last_became_peered":"2023-09-23T20:32:33.237984+0000","last_unstale":"2023-09-23T20:36:15.188566+0000","last_undegraded":"2023-09-23T20:36:15.188566+0000","last_fullsized":"2023-09-23T20:36:15.188566+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:49:02.089307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"90'161","reported_seq":607,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T20:36:18.349786+0000","last_change":"2023-09-23T20:36:06.487465+0000","last_active":"2023-09-23T20:36:18.349786+0000","last_peered":"2023-09-23T20:36:18.349786+0000","last_clean":"2023-09-23T20:36:18.349786+0000","last_became_active":"2023-09-23T20:35:26.651601+0000","last_became_peered":"2023-09-23T20:35:26.651601+0000","last_unstale":"2023-09-23T20:36:18.349786+0000","last_undegraded":"2023-09-23T20:36:18.349786+0000","last_fullsized":"2023-09-23T20:36:18.349786+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:06.487351+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:06.487351+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:19:22.102078+0000","scrub_duration":0.001964379,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":339,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T20:36:18.350235+0000","last_change":"2023-09-23T20:32:33.237234+0000","last_active":"2023-09-23T20:36:18.350235+0000","last_peered":"2023-09-23T20:36:18.350235+0000","last_clean":"2023-09-23T20:36:18.350235+0000","last_became_active":"2023-09-23T20:32:33.236881+0000","last_became_peered":"2023-09-23T20:32:33.236881+0000","last_unstale":"2023-09-23T20:36:18.350235+0000","last_undegraded":"2023-09-23T20:36:18.350235+0000","last_fullsized":"2023-09-23T20:36:18.350235+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:56:17.384831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":227,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T20:36:18.347771+0000","last_change":"2023-09-23T20:29:45.538314+0000","last_active":"2023-09-23T20:36:18.347771+0000","last_peered":"2023-09-23T20:36:18.347771+0000","last_clean":"2023-09-23T20:36:18.347771+0000","last_became_active":"2023-09-23T20:29:45.536847+0000","last_became_peered":"2023-09-23T20:29:45.536847+0000","last_unstale":"2023-09-23T20:36:18.347771+0000","last_undegraded":"2023-09-23T20:36:18.347771+0000","last_fullsized":"2023-09-23T20:36:18.347771+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:51.948721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"90'161","reported_seq":458,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T20:36:18.348416+0000","last_change":"2023-09-23T20:35:42.864462+0000","last_active":"2023-09-23T20:36:18.348416+0000","last_peered":"2023-09-23T20:36:18.348416+0000","last_clean":"2023-09-23T20:36:18.348416+0000","last_became_active":"2023-09-23T20:35:26.665630+0000","last_became_peered":"2023-09-23T20:35:26.665630+0000","last_unstale":"2023-09-23T20:36:18.348416+0000","last_undegraded":"2023-09-23T20:36:18.348416+0000","last_fullsized":"2023-09-23T20:36:18.348416+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:35:42.864363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:35:42.864363+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:14:34.984081+0000","scrub_duration":0.0070144939999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":382,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T20:36:15.187025+0000","last_change":"2023-09-23T20:32:33.235987+0000","last_active":"2023-09-23T20:36:15.187025+0000","last_peered":"2023-09-23T20:36:15.187025+0000","last_clean":"2023-09-23T20:36:15.187025+0000","last_became_active":"2023-09-23T20:32:33.226544+0000","last_became_peered":"2023-09-23T20:32:33.226544+0000","last_unstale":"2023-09-23T20:36:15.187025+0000","last_undegraded":"2023-09-23T20:36:15.187025+0000","last_fullsized":"2023-09-23T20:36:15.187025+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:53:22.794856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":29,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T20:36:15.188262+0000","last_change":"2023-09-23T20:34:59.987575+0000","last_active":"2023-09-23T20:36:15.188262+0000","last_peered":"2023-09-23T20:36:15.188262+0000","last_clean":"2023-09-23T20:36:15.188262+0000","last_became_active":"2023-09-23T20:34:59.987229+0000","last_became_peered":"2023-09-23T20:34:59.987229+0000","last_unstale":"2023-09-23T20:36:15.188262+0000","last_undegraded":"2023-09-23T20:36:15.188262+0000","last_fullsized":"2023-09-23T20:36:15.188262+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:34:01.820159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":390,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T20:36:15.187368+0000","last_change":"2023-09-23T20:32:33.268260+0000","last_active":"2023-09-23T20:36:15.187368+0000","last_peered":"2023-09-23T20:36:15.187368+0000","last_clean":"2023-09-23T20:36:15.187368+0000","last_became_active":"2023-09-23T20:32:33.233695+0000","last_became_peered":"2023-09-23T20:32:33.233695+0000","last_unstale":"2023-09-23T20:36:15.187368+0000","last_undegraded":"2023-09-23T20:36:15.187368+0000","last_fullsized":"2023-09-23T20:36:15.187368+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:19:29.299623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":29,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T20:36:15.186809+0000","last_change":"2023-09-23T20:34:59.987416+0000","last_active":"2023-09-23T20:36:15.186809+0000","last_peered":"2023-09-23T20:36:15.186809+0000","last_clean":"2023-09-23T20:36:15.186809+0000","last_became_active":"2023-09-23T20:34:59.987117+0000","last_became_peered":"2023-09-23T20:34:59.987117+0000","last_unstale":"2023-09-23T20:36:15.186809+0000","last_undegraded":"2023-09-23T20:36:15.186809+0000","last_fullsized":"2023-09-23T20:36:15.186809+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:36:16.318623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"90'161","reported_seq":569,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T20:36:18.345538+0000","last_change":"2023-09-23T20:36:02.384213+0000","last_active":"2023-09-23T20:36:18.345538+0000","last_peered":"2023-09-23T20:36:18.345538+0000","last_clean":"2023-09-23T20:36:18.345538+0000","last_became_active":"2023-09-23T20:35:26.643160+0000","last_became_peered":"2023-09-23T20:35:26.643160+0000","last_unstale":"2023-09-23T20:36:18.345538+0000","last_undegraded":"2023-09-23T20:36:18.345538+0000","last_fullsized":"2023-09-23T20:36:18.345538+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:02.384133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:02.384133+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:34:12.654342+0000","scrub_duration":0.001769098,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":189,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T20:36:18.348562+0000","last_change":"2023-09-23T20:32:50.537447+0000","last_active":"2023-09-23T20:36:18.348562+0000","last_peered":"2023-09-23T20:36:18.348562+0000","last_clean":"2023-09-23T20:36:18.348562+0000","last_became_active":"2023-09-23T20:32:50.537136+0000","last_became_peered":"2023-09-23T20:32:50.537136+0000","last_unstale":"2023-09-23T20:36:18.348562+0000","last_undegraded":"2023-09-23T20:36:18.348562+0000","last_fullsized":"2023-09-23T20:36:18.348562+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:46:51.599492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":89,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T20:36:18.347231+0000","last_change":"2023-09-23T20:32:50.537450+0000","last_active":"2023-09-23T20:36:18.347231+0000","last_peered":"2023-09-23T20:36:18.347231+0000","last_clean":"2023-09-23T20:36:18.347231+0000","last_became_active":"2023-09-23T20:32:50.537128+0000","last_became_peered":"2023-09-23T20:32:50.537128+0000","last_unstale":"2023-09-23T20:36:18.347231+0000","last_undegraded":"2023-09-23T20:36:18.347231+0000","last_fullsized":"2023-09-23T20:36:18.347231+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:37:15.889029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1d","version":"90'161","reported_seq":463,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T20:36:15.183289+0000","last_change":"2023-09-23T20:36:01.505098+0000","last_active":"2023-09-23T20:36:15.183289+0000","last_peered":"2023-09-23T20:36:15.183289+0000","last_clean":"2023-09-23T20:36:15.183289+0000","last_became_active":"2023-09-23T20:35:26.644707+0000","last_became_peered":"2023-09-23T20:35:26.644707+0000","last_unstale":"2023-09-23T20:36:15.183289+0000","last_undegraded":"2023-09-23T20:36:15.183289+0000","last_fullsized":"2023-09-23T20:36:15.183289+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:01.504973+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:01.504973+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:02:10.880809+0000","scrub_duration":0.00202344,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":356,"reported_epoch":373,"state":"active+clean","last_fresh":"2023-09-23T20:36:18.351865+0000","last_change":"2023-09-23T20:25:39.668882+0000","last_active":"2023-09-23T20:36:18.351865+0000","last_peered":"2023-09-23T20:36:18.351865+0000","last_clean":"2023-09-23T20:36:18.351865+0000","last_became_active":"2023-09-23T20:25:39.637128+0000","last_became_peered":"2023-09-23T20:25:39.637128+0000","last_unstale":"2023-09-23T20:36:18.351865+0000","last_undegraded":"2023-09-23T20:36:18.351865+0000","last_fullsized":"2023-09-23T20:36:18.351865+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:56.406298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":70,"reported_epoch":372,"state":"active+clean","last_fresh":"2023-09-23T20:36:15.186411+0000","last_change":"2023-09-23T20:32:33.232281+0000","last_active":"2023-09-23T20:36:15.186411+0000","last_peered":"2023-09-23T20:36:15.186411+0000","last_clean":"2023-09-23T20:36:15.186411+0000","last_became_active":"2023-09-23T20:32:33.232003+0000","last_became_peered":"2023-09-23T20:32:33.232003+0000","last_unstale":"2023-09-23T20:36:15.186411+0000","last_undegraded":"2023-09-23T20:36:15.186411+0000","last_fullsized":"2023-09-23T20:36:15.186411+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:58:27.314355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":49,"num_bytes_recovered":6522,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26691,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":2725117952,"num_objects":41582,"num_object_clones":0,"num_object_copies":124746,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":41582,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":41582,"num_write_kb":2661248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8196325376,"data_stored":8196325376,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":41582,"ondisk_log_size":41582,"up":48,"acting":48,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":58,"num_bytes_recovered":4194342,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6082,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201585,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1166008,"kb_used_data":846768,"kb_used_omap":0,"kb_used_meta":319232,"kb_avail":92579144,"statfs":{"total":95995035648,"available":94801043456,"internally_reserved":0,"allocated":867090432,"data_stored":865924883,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":326893568},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:35:44 2023","interfaces":[{"interface":"back","average":{"1min":3.5600000000000001,"5min":2.5299999999999998,"15min":1.8160000000000001},"min":{"1min":0.28100000000000003,"5min":0.23599999999999999,"15min":0.20799999999999999},"max":{"1min":17.399999999999999,"5min":17.399999999999999,"15min":48.798999999999999},"last":3.024},{"interface":"front","average":{"1min":3.1890000000000001,"5min":2.21,"15min":1.877},"min":{"1min":0.249,"5min":0.249,"15min":0.24399999999999999},"max":{"1min":19.355,"5min":19.355,"15min":48.247},"last":2.4550000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:35:59 2023","interfaces":[{"interface":"back","average":{"1min":3.0880000000000001,"5min":2.25,"15min":1.9239999999999999},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.254},"max":{"1min":16.103000000000002,"5min":16.103000000000002,"15min":16.103000000000002},"last":5.1360000000000001},{"interface":"front","average":{"1min":3.0249999999999999,"5min":2.0430000000000001,"15min":2.0009999999999999},"min":{"1min":0.32700000000000001,"5min":0.248,"15min":0.248},"max":{"1min":12.02,"5min":12.02,"15min":12.02},"last":2.4769999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:35:44 2023","interfaces":[{"interface":"back","average":{"1min":3.3079999999999998,"5min":2.492,"15min":1.8620000000000001},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27600000000000002},"max":{"1min":9.8629999999999995,"5min":13.173999999999999,"15min":13.173999999999999},"last":2.3820000000000001},{"interface":"front","average":{"1min":3.7130000000000001,"5min":2.8799999999999999,"15min":2.113},"min":{"1min":0.36099999999999999,"5min":0.27400000000000002,"15min":0.26600000000000001},"max":{"1min":12.551,"5min":23.494,"15min":23.494},"last":5.468}]},{"osd":3,"last update":"Sat Sep 23 20:36:15 2023","interfaces":[{"interface":"back","average":{"1min":2.1960000000000002,"5min":2.7879999999999998,"15min":2.0880000000000001},"min":{"1min":0.39100000000000001,"5min":0.33700000000000002,"15min":0.26200000000000001},"max":{"1min":7.3630000000000004,"5min":17.155999999999999,"15min":17.155999999999999},"last":1.5580000000000001},{"interface":"front","average":{"1min":2.5609999999999999,"5min":2.9049999999999998,"15min":2.044},"min":{"1min":0.40899999999999997,"5min":0.34799999999999998,"15min":0.27600000000000002},"max":{"1min":7.3719999999999999,"5min":16.960000000000001,"15min":16.960000000000001},"last":2.419}]},{"osd":4,"last update":"Sat Sep 23 20:36:15 2023","interfaces":[{"interface":"back","average":{"1min":0.746,"5min":0.81899999999999995,"15min":1.1140000000000001},"min":{"1min":0.26900000000000002,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":4.4189999999999996,"5min":5.8819999999999997,"15min":15.177},"last":2.9329999999999998},{"interface":"front","average":{"1min":1.0169999999999999,"5min":0.96099999999999997,"15min":1.266},"min":{"1min":0.28399999999999997,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":3.1099999999999999,"5min":6.1779999999999999,"15min":11.122},"last":0.47699999999999998}]},{"osd":5,"last update":"Sat Sep 23 20:36:15 2023","interfaces":[{"interface":"back","average":{"1min":0.996,"5min":0.75,"15min":1.1739999999999999},"min":{"1min":0.30599999999999999,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":3.0939999999999999,"5min":4.133,"15min":13.241},"last":3.403},{"interface":"front","average":{"1min":0.88600000000000001,"5min":0.68000000000000005,"15min":1.2350000000000001},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.315},"max":{"1min":4.4610000000000003,"5min":4.7590000000000003,"15min":18.831},"last":3.391}]},{"osd":6,"last update":"Sat Sep 23 20:36:15 2023","interfaces":[{"interface":"back","average":{"1min":0.97899999999999998,"5min":0.70899999999999996,"15min":1.139},"min":{"1min":0.34799999999999998,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":5.0339999999999998,"5min":5.0339999999999998,"15min":13.185},"last":3.0619999999999998},{"interface":"front","average":{"1min":1.0860000000000001,"5min":0.97299999999999998,"15min":1.167},"min":{"1min":0.378,"5min":0.318,"15min":0.29899999999999999},"max":{"1min":3.7890000000000001,"5min":6.2770000000000001,"15min":9.9120000000000008},"last":0.50600000000000001}]}]},{"osd":6,"up_from":47,"seq":201863463221,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1189604,"kb_used_data":901916,"kb_used_omap":0,"kb_used_meta":287680,"kb_avail":92555548,"statfs":{"total":95995035648,"available":94776881152,"internally_reserved":0,"allocated":923561984,"data_stored":922408084,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":294584320},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:35:41 2023","interfaces":[{"interface":"back","average":{"1min":1.754,"5min":2.589,"15min":1.667},"min":{"1min":0.26200000000000001,"5min":0.22800000000000001,"15min":0.221},"max":{"1min":4.8959999999999999,"5min":53.701999999999998,"15min":53.701999999999998},"last":0.26300000000000001},{"interface":"front","average":{"1min":1.7290000000000001,"5min":2.9180000000000001,"15min":2.016},"min":{"1min":0.252,"5min":0.252,"15min":0.25},"max":{"1min":6.093,"5min":53.875999999999998,"15min":53.875999999999998},"last":0.64100000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:36:01 2023","interfaces":[{"interface":"back","average":{"1min":2.3340000000000001,"5min":2.1880000000000002,"15min":2.5630000000000002},"min":{"1min":0.309,"5min":0.249,"15min":0.249},"max":{"1min":10.331,"5min":34.173999999999999,"15min":34.173999999999999},"last":0.56200000000000006},{"interface":"front","average":{"1min":2.0539999999999998,"5min":2.282,"15min":3.085},"min":{"1min":0.35299999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":6.6020000000000003,"5min":34.490000000000002,"15min":34.490000000000002},"last":0.307}]},{"osd":2,"last update":"Sat Sep 23 20:35:41 2023","interfaces":[{"interface":"back","average":{"1min":2.6259999999999999,"5min":2.512,"15min":1.9430000000000001},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":13.429,"5min":34.317999999999998,"15min":34.317999999999998},"last":0.65500000000000003},{"interface":"front","average":{"1min":2.4649999999999999,"5min":2.5289999999999999,"15min":1.859},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":13.457000000000001,"5min":34.292999999999999,"15min":34.292999999999999},"last":0.48399999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:36:27 2023","interfaces":[{"interface":"back","average":{"1min":1.2110000000000001,"5min":1.6910000000000001,"15min":1.7450000000000001},"min":{"1min":0.39800000000000002,"5min":0.29999999999999999,"15min":0.29599999999999999},"max":{"1min":4.3520000000000003,"5min":7.5620000000000003,"15min":9.4239999999999995},"last":0.59399999999999997},{"interface":"front","average":{"1min":1.0329999999999999,"5min":2.0019999999999998,"15min":1.875},"min":{"1min":0.34899999999999998,"5min":0.317,"15min":0.317},"max":{"1min":4.383,"5min":12.875999999999999,"15min":12.875999999999999},"last":1.5409999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:36:05 2023","interfaces":[{"interface":"back","average":{"1min":0.60499999999999998,"5min":0.61899999999999999,"15min":0.90400000000000003},"min":{"1min":0.26900000000000002,"5min":0.251,"15min":0.251},"max":{"1min":2.1600000000000001,"5min":4.9279999999999999,"15min":10.173},"last":0.33700000000000002},{"interface":"front","average":{"1min":1.236,"5min":0.88800000000000001,"15min":1.1100000000000001},"min":{"1min":0.35099999999999998,"5min":0.28199999999999997,"15min":0.27000000000000002},"max":{"1min":4.194,"5min":7.7629999999999999,"15min":8.3249999999999993},"last":0.53500000000000003}]},{"osd":5,"last update":"Sat Sep 23 20:36:05 2023","interfaces":[{"interface":"back","average":{"1min":0.879,"5min":0.747,"15min":0.97599999999999998},"min":{"1min":0.32800000000000001,"5min":0.30499999999999999,"15min":0.28699999999999998},"max":{"1min":2.8849999999999998,"5min":6.8310000000000004,"15min":14.085000000000001},"last":0.36699999999999999},{"interface":"front","average":{"1min":0.879,"5min":0.82199999999999995,"15min":1.329},"min":{"1min":0.35099999999999998,"5min":0.32000000000000001,"15min":0.30599999999999999},"max":{"1min":4.2489999999999997,"5min":7.7300000000000004,"15min":18.192},"last":0.82099999999999995}]},{"osd":7,"last update":"Sat Sep 23 20:36:27 2023","interfaces":[{"interface":"back","average":{"1min":0.83899999999999997,"5min":0.77500000000000002,"15min":1.1020000000000001},"min":{"1min":0.379,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":2.3479999999999999,"5min":5.0350000000000001,"15min":10.217000000000001},"last":1.048},{"interface":"front","average":{"1min":0.90600000000000003,"5min":0.90800000000000003,"15min":1.27},"min":{"1min":0.38900000000000001,"5min":0.34499999999999997,"15min":0.30299999999999999},"max":{"1min":4.3150000000000004,"5min":7.601,"15min":9.7360000000000007},"last":0.67600000000000005}]}]},{"osd":1,"up_from":266,"seq":1142461300819,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1408648,"kb_used_data":1094976,"kb_used_omap":8,"kb_used_meta":313655,"kb_avail":92336504,"statfs":{"total":95995035648,"available":94552580096,"internally_reserved":0,"allocated":1121255424,"data_stored":1119254661,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8206,"internal_metadata":321183730},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:35:58 2023","interfaces":[{"interface":"back","average":{"1min":3.254,"5min":3.1600000000000001,"15min":2.6059999999999999},"min":{"1min":0.222,"5min":0.218,"15min":0.218},"max":{"1min":14.531000000000001,"5min":52.689999999999998,"15min":52.689999999999998},"last":0.25900000000000001},{"interface":"front","average":{"1min":4.46,"5min":3.734,"15min":3.375},"min":{"1min":0.19900000000000001,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":19.41,"5min":52.662999999999997,"15min":52.662999999999997},"last":0.23100000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:35:58 2023","interfaces":[{"interface":"back","average":{"1min":4.4960000000000004,"5min":3.9430000000000001,"15min":4.7309999999999999},"min":{"1min":0.25900000000000001,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":14.474,"5min":28.359000000000002,"15min":28.359000000000002},"last":0.245},{"interface":"front","average":{"1min":2.9969999999999999,"5min":2.7989999999999999,"15min":2.3290000000000002},"min":{"1min":0.26700000000000002,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":11.606999999999999,"5min":12.801,"15min":12.801},"last":0.35599999999999998}]},{"osd":3,"last update":"Sat Sep 23 20:35:58 2023","interfaces":[{"interface":"back","average":{"1min":4.468,"5min":4.5,"15min":4.0780000000000003},"min":{"1min":0.308,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":14.359,"5min":80.745999999999995,"15min":80.745999999999995},"last":0.32300000000000001},{"interface":"front","average":{"1min":2.7360000000000002,"5min":4.2469999999999999,"15min":4.6459999999999999},"min":{"1min":0.35599999999999998,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":19.315000000000001,"5min":82.988,"15min":82.988},"last":0.31}]},{"osd":4,"last update":"Sat Sep 23 20:35:58 2023","interfaces":[{"interface":"back","average":{"1min":3.4609999999999999,"5min":2.5750000000000002,"15min":2.2440000000000002},"min":{"1min":0.35599999999999998,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":9.4510000000000005,"5min":24.327999999999999,"15min":24.327999999999999},"last":0.61299999999999999},{"interface":"front","average":{"1min":2.4430000000000001,"5min":1.76,"15min":1.9359999999999999},"min":{"1min":0.38800000000000001,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":11.574999999999999,"5min":11.574999999999999,"15min":11.574999999999999},"last":0.57099999999999995}]},{"osd":5,"last update":"Sat Sep 23 20:35:58 2023","interfaces":[{"interface":"back","average":{"1min":2.0760000000000001,"5min":2.0310000000000001,"15min":2.5449999999999999},"min":{"1min":0.34100000000000003,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":6.4059999999999997,"5min":8.1950000000000003,"15min":9.6479999999999997},"last":33.401000000000003},{"interface":"front","average":{"1min":3.5289999999999999,"5min":2.7690000000000001,"15min":2.3159999999999998},"min":{"1min":0.44800000000000001,"5min":0.36899999999999999,"15min":0.36799999999999999},"max":{"1min":9.5150000000000006,"5min":24.404,"15min":24.404},"last":33.417000000000002}]},{"osd":6,"last update":"Sat Sep 23 20:35:58 2023","interfaces":[{"interface":"back","average":{"1min":1.79,"5min":1.974,"15min":5.6239999999999997},"min":{"1min":0.36899999999999999,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":5.6109999999999998,"5min":8.1620000000000008,"15min":72.682000000000002},"last":0.58399999999999996},{"interface":"front","average":{"1min":3.569,"5min":2.8439999999999999,"15min":5.2409999999999997},"min":{"1min":0.45800000000000002,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":9.4849999999999994,"5min":24.376000000000001,"15min":72.563999999999993},"last":0.65200000000000002}]},{"osd":7,"last update":"Sat Sep 23 20:35:58 2023","interfaces":[{"interface":"back","average":{"1min":2.6949999999999998,"5min":2.258,"15min":2.9409999999999998},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":18.437000000000001,"5min":18.437000000000001,"15min":18.437000000000001},"last":0.67300000000000004},{"interface":"front","average":{"1min":3.734,"5min":2.8980000000000001,"15min":2.5390000000000001},"min":{"1min":0.51500000000000001,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":12.021000000000001,"5min":24.524000000000001,"15min":24.524000000000001},"last":0.68700000000000006}]}]},{"osd":0,"up_from":94,"seq":403726926022,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1507820,"kb_used_data":1005348,"kb_used_omap":7,"kb_used_meta":502456,"kb_avail":92237332,"statfs":{"total":95995035648,"available":94451027968,"internally_reserved":0,"allocated":1029476352,"data_stored":1027626373,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":514514961},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:36:03 2023","interfaces":[{"interface":"back","average":{"1min":2.6989999999999998,"5min":3.2810000000000001,"15min":3.6850000000000001},"min":{"1min":0.23000000000000001,"5min":0.183,"15min":0.17999999999999999},"max":{"1min":11.554,"5min":42.000999999999998,"15min":42.000999999999998},"last":1.853},{"interface":"front","average":{"1min":2.9199999999999999,"5min":3.278,"15min":4.0640000000000001},"min":{"1min":0.218,"5min":0.218,"15min":0.218},"max":{"1min":9.6059999999999999,"5min":23.036000000000001,"15min":23.036000000000001},"last":1.998}]},{"osd":2,"last update":"Sat Sep 23 20:35:37 2023","interfaces":[{"interface":"back","average":{"1min":3.5840000000000001,"5min":2.9820000000000002,"15min":2.4569999999999999},"min":{"1min":0.24099999999999999,"5min":0.193,"15min":0.185},"max":{"1min":11.064,"5min":22.460999999999999,"15min":77.448999999999998},"last":1.0289999999999999},{"interface":"front","average":{"1min":3.4489999999999998,"5min":2.8570000000000002,"15min":2.294},"min":{"1min":0.25700000000000001,"5min":0.188,"15min":0.188},"max":{"1min":12.071,"5min":22.260000000000002,"15min":81.724999999999994},"last":4.5259999999999998}]},{"osd":3,"last update":"Sat Sep 23 20:36:10 2023","interfaces":[{"interface":"back","average":{"1min":2.5390000000000001,"5min":3.1829999999999998,"15min":2.6520000000000001},"min":{"1min":0.30099999999999999,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":9.5730000000000004,"5min":22.486000000000001,"15min":81.400999999999996},"last":0.54500000000000004},{"interface":"front","average":{"1min":2.915,"5min":3.4849999999999999,"15min":2.7869999999999999},"min":{"1min":0.29499999999999998,"5min":0.25700000000000001,"15min":0.23699999999999999},"max":{"1min":11.58,"5min":22.867999999999999,"15min":81.137},"last":1.8799999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:35:37 2023","interfaces":[{"interface":"back","average":{"1min":3.331,"5min":2.1749999999999998,"15min":1.9350000000000001},"min":{"1min":0.32000000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":11.068,"5min":12.587999999999999,"15min":23.876999999999999},"last":0.59099999999999997},{"interface":"front","average":{"1min":2.7650000000000001,"5min":2.347,"15min":1.8160000000000001},"min":{"1min":0.316,"5min":0.308,"15min":0.29299999999999998},"max":{"1min":11.458,"5min":23.001000000000001,"15min":23.681999999999999},"last":1.8220000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:35:37 2023","interfaces":[{"interface":"back","average":{"1min":2.5960000000000001,"5min":2.2360000000000002,"15min":2.129},"min":{"1min":0.35099999999999998,"5min":0.34399999999999997,"15min":0.34200000000000003},"max":{"1min":11.497999999999999,"5min":22.969999999999999,"15min":24.198},"last":1.7609999999999999},{"interface":"front","average":{"1min":2.8170000000000002,"5min":2.3199999999999998,"15min":1.9970000000000001},"min":{"1min":0.38300000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":11.974,"5min":22.236000000000001,"15min":22.236000000000001},"last":2.452}]},{"osd":6,"last update":"Sat Sep 23 20:35:37 2023","interfaces":[{"interface":"back","average":{"1min":3.1640000000000001,"5min":2.4590000000000001,"15min":1.923},"min":{"1min":0.38,"5min":0.32300000000000001,"15min":0.318},"max":{"1min":11.526999999999999,"5min":22.934999999999999,"15min":24.408000000000001},"last":1.798},{"interface":"front","average":{"1min":2.8010000000000002,"5min":2.399,"15min":1.841},"min":{"1min":0.35799999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":12.015000000000001,"5min":22.164999999999999,"15min":25.271000000000001},"last":2.508}]},{"osd":7,"last update":"Sat Sep 23 20:35:37 2023","interfaces":[{"interface":"back","average":{"1min":3.1899999999999999,"5min":2.5630000000000002,"15min":2.0459999999999998},"min":{"1min":0.40200000000000002,"5min":0.32200000000000001,"15min":0.30599999999999999},"max":{"1min":12.042,"5min":22.206,"15min":25.359000000000002},"last":2.484},{"interface":"front","average":{"1min":3.9740000000000002,"5min":2.6230000000000002,"15min":2.3319999999999999},"min":{"1min":0.36199999999999999,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":11.112,"5min":22.422999999999998,"15min":24.669},"last":0.70299999999999996}]}]},{"osd":2,"up_from":104,"seq":446676598970,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1576164,"kb_used_data":1069020,"kb_used_omap":11,"kb_used_meta":507124,"kb_avail":92168988,"statfs":{"total":95995035648,"available":94381043712,"internally_reserved":0,"allocated":1094676480,"data_stored":1092831349,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11878,"internal_metadata":519295386},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:35:47 2023","interfaces":[{"interface":"back","average":{"1min":3.944,"5min":4.274,"15min":2.5819999999999999},"min":{"1min":0.25800000000000001,"5min":0.185,"15min":0.17999999999999999},"max":{"1min":14.124000000000001,"5min":82.424999999999997,"15min":82.424999999999997},"last":0.27000000000000002},{"interface":"front","average":{"1min":4.4859999999999998,"5min":3.7890000000000001,"15min":2.3460000000000001},"min":{"1min":0.24199999999999999,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":14.055,"5min":82.524000000000001,"15min":82.524000000000001},"last":0.621}]},{"osd":1,"last update":"Sat Sep 23 20:35:53 2023","interfaces":[{"interface":"back","average":{"1min":3.5960000000000001,"5min":2.819,"15min":4.173},"min":{"1min":0.29299999999999998,"5min":0.22900000000000001,"15min":0.221},"max":{"1min":11.227,"5min":15.956,"15min":16.280000000000001},"last":1.448},{"interface":"front","average":{"1min":5.9560000000000004,"5min":3.3959999999999999,"15min":3.1240000000000001},"min":{"1min":0.46400000000000002,"5min":0.248,"15min":0.248},"max":{"1min":21.585999999999999,"5min":21.585999999999999,"15min":30.565000000000001},"last":1.5680000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:36:15 2023","interfaces":[{"interface":"back","average":{"1min":2.52,"5min":2.8799999999999999,"15min":2.0489999999999999},"min":{"1min":0.31900000000000001,"5min":0.30199999999999999,"15min":0.19800000000000001},"max":{"1min":11.129,"5min":15.804,"15min":16.331},"last":0.66600000000000004},{"interface":"front","average":{"1min":2.8100000000000001,"5min":3.2250000000000001,"15min":2.403},"min":{"1min":0.33200000000000002,"5min":0.28999999999999998,"15min":0.23899999999999999},"max":{"1min":13.148999999999999,"5min":15.266,"15min":30.704999999999998},"last":4.7009999999999996}]},{"osd":4,"last update":"Sat Sep 23 20:35:47 2023","interfaces":[{"interface":"back","average":{"1min":3.3730000000000002,"5min":2.1440000000000001,"15min":1.754},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":9.2430000000000003,"5min":9.2430000000000003,"15min":16.797999999999998},"last":0.71699999999999997},{"interface":"front","average":{"1min":3.1819999999999999,"5min":2.5110000000000001,"15min":1.903},"min":{"1min":0.307,"5min":0.30499999999999999,"15min":0.27600000000000002},"max":{"1min":11.044,"5min":15.853999999999999,"15min":15.853999999999999},"last":2.5139999999999998}]},{"osd":5,"last update":"Sat Sep 23 20:35:47 2023","interfaces":[{"interface":"back","average":{"1min":3.3690000000000002,"5min":2.2069999999999999,"15min":2.2869999999999999},"min":{"1min":0.36599999999999999,"5min":0.314,"15min":0.309},"max":{"1min":9.1799999999999997,"5min":9.1799999999999997,"15min":95.393000000000001},"last":0.65300000000000002},{"interface":"front","average":{"1min":3.1920000000000002,"5min":2.77,"15min":2.5049999999999999},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.32300000000000001},"max":{"1min":10.856999999999999,"5min":15.885,"15min":95.414000000000001},"last":0.69899999999999995}]},{"osd":6,"last update":"Sat Sep 23 20:35:47 2023","interfaces":[{"interface":"back","average":{"1min":3.0680000000000001,"5min":2.6560000000000001,"15min":1.9890000000000001},"min":{"1min":0.37,"5min":0.35799999999999998,"15min":0.32100000000000001},"max":{"1min":11.253,"5min":15.914,"15min":15.914},"last":1.4139999999999999},{"interface":"front","average":{"1min":2.968,"5min":2.3570000000000002,"15min":1.9670000000000001},"min":{"1min":0.49099999999999999,"5min":0.318,"15min":0.316},"max":{"1min":10.811,"5min":11.523,"15min":30.670999999999999},"last":1.373}]},{"osd":7,"last update":"Sat Sep 23 20:35:47 2023","interfaces":[{"interface":"back","average":{"1min":2.7730000000000001,"5min":2.306,"15min":2.0259999999999998},"min":{"1min":0.439,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":10.231,"5min":12.734999999999999,"15min":30.741},"last":0.53400000000000003},{"interface":"front","average":{"1min":3.2250000000000001,"5min":2.3170000000000002,"15min":2.0089999999999999},"min":{"1min":0.42999999999999999,"5min":0.38,"15min":0.38},"max":{"1min":9.2720000000000002,"5min":10.861000000000001,"15min":16.719000000000001},"last":0.68200000000000005}]}]},{"osd":3,"up_from":109,"seq":468151435443,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1550868,"kb_used_data":930636,"kb_used_omap":4,"kb_used_meta":620219,"kb_avail":92194284,"statfs":{"total":95995035648,"available":94406946816,"internally_reserved":0,"allocated":952971264,"data_stored":951131963,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4514,"internal_metadata":635104862},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:36:12 2023","interfaces":[{"interface":"back","average":{"1min":1.3129999999999999,"5min":2.992,"15min":2.3410000000000002},"min":{"1min":0.22500000000000001,"5min":0.191,"15min":0.186},"max":{"1min":7.0309999999999997,"5min":17.129000000000001,"15min":67.305999999999997},"last":0.434},{"interface":"front","average":{"1min":1.734,"5min":2.8340000000000001,"15min":2.2090000000000001},"min":{"1min":0.28399999999999997,"5min":0.22600000000000001,"15min":0.19500000000000001},"max":{"1min":9.8520000000000003,"5min":12.502000000000001,"15min":67.403999999999996},"last":0.504}]},{"osd":1,"last update":"Sat Sep 23 20:35:56 2023","interfaces":[{"interface":"back","average":{"1min":1.7689999999999999,"5min":2.456,"15min":2.8769999999999998},"min":{"1min":0.251,"5min":0.222,"15min":0.222},"max":{"1min":9.8940000000000001,"5min":16.189,"15min":16.189},"last":0.46999999999999997},{"interface":"front","average":{"1min":2.9140000000000001,"5min":2.5619999999999998,"15min":3.0219999999999998},"min":{"1min":0.34599999999999997,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":13.727,"5min":32.603999999999999,"15min":32.603999999999999},"last":0.42099999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:36:12 2023","interfaces":[{"interface":"back","average":{"1min":1.1859999999999999,"5min":3.1070000000000002,"15min":2.3140000000000001},"min":{"1min":0.30399999999999999,"5min":0.23799999999999999,"15min":0.22800000000000001},"max":{"1min":9.6820000000000004,"5min":32.857999999999997,"15min":32.857999999999997},"last":0.47799999999999998},{"interface":"front","average":{"1min":1.0660000000000001,"5min":2.7490000000000001,"15min":2.2050000000000001},"min":{"1min":0.311,"5min":0.246,"15min":0.20899999999999999},"max":{"1min":3.4630000000000001,"5min":24.146999999999998,"15min":24.146999999999998},"last":0.51800000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:36:12 2023","interfaces":[{"interface":"back","average":{"1min":0.76400000000000001,"5min":1.661,"15min":1.7090000000000001},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.29799999999999999},"max":{"1min":2.3109999999999999,"5min":12.151,"15min":46.042999999999999},"last":0.83599999999999997},{"interface":"front","average":{"1min":1.1659999999999999,"5min":2.125,"15min":2.0939999999999999},"min":{"1min":0.376,"5min":0.29399999999999998,"15min":0.28100000000000003},"max":{"1min":3.5110000000000001,"5min":14.422000000000001,"15min":46.579999999999998},"last":0.65300000000000002}]},{"osd":5,"last update":"Sat Sep 23 20:36:12 2023","interfaces":[{"interface":"back","average":{"1min":0.77800000000000002,"5min":1.9039999999999999,"15min":1.9510000000000001},"min":{"1min":0.32300000000000001,"5min":0.32200000000000001,"15min":0.311},"max":{"1min":2.383,"5min":13.228999999999999,"15min":45.292000000000002},"last":3.6779999999999999},{"interface":"front","average":{"1min":1.272,"5min":1.8500000000000001,"15min":1.806},"min":{"1min":0.41299999999999998,"5min":0.32000000000000001,"15min":0.313},"max":{"1min":9.0839999999999996,"5min":10.300000000000001,"15min":44.679000000000002},"last":3.3410000000000002}]},{"osd":6,"last update":"Sat Sep 23 20:36:12 2023","interfaces":[{"interface":"back","average":{"1min":1.28,"5min":1.851,"15min":1.833},"min":{"1min":0.437,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":9.0470000000000006,"5min":12.464,"15min":17.524999999999999},"last":0.79100000000000004},{"interface":"front","average":{"1min":1.2330000000000001,"5min":2.0510000000000002,"15min":1.847},"min":{"1min":0.39800000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":4.2450000000000001,"5min":14.464,"15min":19.524000000000001},"last":0.74299999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:36:12 2023","interfaces":[{"interface":"back","average":{"1min":1.556,"5min":2.0990000000000002,"15min":2.0209999999999999},"min":{"1min":0.41499999999999998,"5min":0.36899999999999999,"15min":0.34999999999999998},"max":{"1min":5.8860000000000001,"5min":14.521000000000001,"15min":19.571999999999999},"last":0.72099999999999997},{"interface":"front","average":{"1min":0.94799999999999995,"5min":1.877,"15min":2.0179999999999998},"min":{"1min":0.45200000000000001,"5min":0.42599999999999999,"15min":0.371},"max":{"1min":2.3580000000000001,"5min":12.186,"15min":45.387999999999998},"last":0.81100000000000005}]}]},{"osd":4,"up_from":30,"seq":128849019197,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1100708,"kb_used_data":830236,"kb_used_omap":0,"kb_used_meta":270464,"kb_avail":92644444,"statfs":{"total":95995035648,"available":94867910656,"internally_reserved":0,"allocated":850161664,"data_stored":849007498,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":276955136},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:35:40 2023","interfaces":[{"interface":"back","average":{"1min":2.2799999999999998,"5min":1.6579999999999999,"15min":1.5229999999999999},"min":{"1min":0.27300000000000002,"5min":0.255,"15min":0.222},"max":{"1min":5.2359999999999998,"5min":8.0800000000000001,"15min":20.411999999999999},"last":0.254},{"interface":"front","average":{"1min":2.3849999999999998,"5min":1.835,"15min":1.6739999999999999},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.253},"max":{"1min":5.2229999999999999,"5min":10.675000000000001,"15min":20.454999999999998},"last":0.35799999999999998}]},{"osd":1,"last update":"Sat Sep 23 20:36:01 2023","interfaces":[{"interface":"back","average":{"1min":1.5649999999999999,"5min":1.7969999999999999,"15min":1.9370000000000001},"min":{"1min":0.32000000000000001,"5min":0.27600000000000002,"15min":0.27300000000000002},"max":{"1min":7.8600000000000003,"5min":11.303000000000001,"15min":11.303000000000001},"last":0.308},{"interface":"front","average":{"1min":1.7809999999999999,"5min":1.583,"15min":1.871},"min":{"1min":0.30099999999999999,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":10.454000000000001,"5min":11.287000000000001,"15min":11.287000000000001},"last":0.29499999999999998}]},{"osd":2,"last update":"Sat Sep 23 20:35:53 2023","interfaces":[{"interface":"back","average":{"1min":2.702,"5min":1.9570000000000001,"15min":1.798},"min":{"1min":0.32300000000000001,"5min":0.311,"15min":0.26600000000000001},"max":{"1min":10.416,"5min":10.416,"15min":32.363},"last":0.83799999999999997},{"interface":"front","average":{"1min":2.9470000000000001,"5min":2.21,"15min":2.0950000000000002},"min":{"1min":0.41999999999999998,"5min":0.28999999999999998,"15min":0.26800000000000002},"max":{"1min":8.4939999999999998,"5min":14.614000000000001,"15min":34.042999999999999},"last":2.1970000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:36:10 2023","interfaces":[{"interface":"back","average":{"1min":2.4630000000000001,"5min":1.9359999999999999,"15min":1.9710000000000001},"min":{"1min":0.372,"5min":0.34799999999999998,"15min":0.29099999999999998},"max":{"1min":9.5239999999999991,"5min":9.5239999999999991,"15min":32.332999999999998},"last":2.0019999999999998},{"interface":"front","average":{"1min":2.262,"5min":2.1509999999999998,"15min":2.044},"min":{"1min":0.38300000000000001,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":8.2240000000000002,"5min":15.991,"15min":32.399999999999999},"last":1.7689999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:36:33 2023","interfaces":[{"interface":"back","average":{"1min":0.64500000000000002,"5min":0.68500000000000005,"15min":1.542},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.247},"max":{"1min":2.0049999999999999,"5min":5.4340000000000002,"15min":82.866},"last":0.55500000000000005},{"interface":"front","average":{"1min":0.73199999999999998,"5min":0.754,"15min":1.4490000000000001},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.27800000000000002},"max":{"1min":2.1230000000000002,"5min":6.8520000000000003,"15min":82.546000000000006},"last":0.33200000000000002}]},{"osd":6,"last update":"Sat Sep 23 20:35:53 2023","interfaces":[{"interface":"back","average":{"1min":1.018,"5min":1.028,"15min":1.389},"min":{"1min":0.312,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":3.008,"5min":10.178000000000001,"15min":41.371000000000002},"last":0.39100000000000001},{"interface":"front","average":{"1min":1.097,"5min":0.85099999999999998,"15min":1.27},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.27200000000000002},"max":{"1min":4.2949999999999999,"5min":5.4619999999999997,"15min":41.140999999999998},"last":0.45000000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:36:10 2023","interfaces":[{"interface":"back","average":{"1min":0.81499999999999995,"5min":1.028,"15min":1.4419999999999999},"min":{"1min":0.377,"5min":0.34000000000000002,"15min":0.33600000000000002},"max":{"1min":2.4369999999999998,"5min":6.8849999999999998,"15min":16.427},"last":0.59899999999999998},{"interface":"front","average":{"1min":0.68300000000000005,"5min":0.79100000000000004,"15min":1.298},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.307},"max":{"1min":2.0550000000000002,"5min":6.3970000000000002,"15min":19.018999999999998},"last":0.48599999999999999}]}]},{"osd":5,"up_from":39,"seq":167503724858,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1580320,"kb_used_data":1300952,"kb_used_omap":0,"kb_used_meta":279360,"kb_avail":92164832,"statfs":{"total":95995035648,"available":94376787968,"internally_reserved":0,"allocated":1332174848,"data_stored":1331021408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":286064640},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:35:42 2023","interfaces":[{"interface":"back","average":{"1min":4.7649999999999997,"5min":2.125,"15min":1.6919999999999999},"min":{"1min":0.29599999999999999,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":18.16,"5min":18.16,"15min":32.475000000000001},"last":7.5839999999999996},{"interface":"front","average":{"1min":3.3809999999999998,"5min":1.837,"15min":1.615},"min":{"1min":0.33900000000000002,"5min":0.26300000000000001,"15min":0.20599999999999999},"max":{"1min":9.0570000000000004,"5min":9.0570000000000004,"15min":32.457000000000001},"last":7.5709999999999997}]},{"osd":1,"last update":"Sat Sep 23 20:35:59 2023","interfaces":[{"interface":"back","average":{"1min":1.675,"5min":1.6180000000000001,"15min":1.909},"min":{"1min":0.26300000000000001,"5min":0.253,"15min":0.253},"max":{"1min":4.5910000000000002,"5min":9.3539999999999992,"15min":9.3539999999999992},"last":0.30599999999999999},{"interface":"front","average":{"1min":3.375,"5min":1.879,"15min":1.667},"min":{"1min":0.41899999999999998,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":8.4220000000000006,"5min":9.3900000000000006,"15min":9.3900000000000006},"last":0.35999999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:35:42 2023","interfaces":[{"interface":"back","average":{"1min":2.4020000000000001,"5min":2.1070000000000002,"15min":1.8440000000000001},"min":{"1min":0.32300000000000001,"5min":0.27400000000000002,"15min":0.26700000000000002},"max":{"1min":11.347,"5min":27.603999999999999,"15min":34.514000000000003},"last":0.39100000000000001},{"interface":"front","average":{"1min":2.4159999999999999,"5min":2.4060000000000001,"15min":1.8720000000000001},"min":{"1min":0.34100000000000003,"5min":0.255,"15min":0.255},"max":{"1min":13.545,"5min":27.626999999999999,"15min":32.515000000000001},"last":0.379}]},{"osd":3,"last update":"Sat Sep 23 20:36:09 2023","interfaces":[{"interface":"back","average":{"1min":4.0209999999999999,"5min":2.7290000000000001,"15min":2.0459999999999998},"min":{"1min":0.35799999999999998,"5min":0.29599999999999999,"15min":0.25700000000000001},"max":{"1min":36.716000000000001,"5min":36.716000000000001,"15min":36.716000000000001},"last":0.35299999999999998},{"interface":"front","average":{"1min":2.0710000000000002,"5min":2.2200000000000002,"15min":1.766},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.28799999999999998},"max":{"1min":9.6959999999999997,"5min":10.577999999999999,"15min":12.005000000000001},"last":0.46300000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:35:33 2023","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.67800000000000005,"15min":1.2989999999999999},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.25800000000000001},"max":{"1min":2.2280000000000002,"5min":6.282,"15min":28.015999999999998},"last":0.33500000000000002},{"interface":"front","average":{"1min":0.84899999999999998,"5min":0.69899999999999995,"15min":1.087},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":3.0659999999999998,"5min":3.0659999999999998,"15min":12.300000000000001},"last":0.42099999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:35:48 2023","interfaces":[{"interface":"back","average":{"1min":0.77500000000000002,"5min":0.70699999999999996,"15min":1.2190000000000001},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":4.3300000000000001,"5min":4.3330000000000002,"15min":12.890000000000001},"last":0.372},{"interface":"front","average":{"1min":0.83699999999999997,"5min":0.73799999999999999,"15min":1.1879999999999999},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.34499999999999997},"max":{"1min":4.2400000000000002,"5min":5.1680000000000001,"15min":16.265999999999998},"last":0.44500000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:36:09 2023","interfaces":[{"interface":"back","average":{"1min":0.70999999999999996,"5min":0.63,"15min":1.2310000000000001},"min":{"1min":0.33600000000000002,"5min":0.32100000000000001,"15min":0.29699999999999999},"max":{"1min":2.8180000000000001,"5min":2.8180000000000001,"15min":36.621000000000002},"last":0.316},{"interface":"front","average":{"1min":0.62,"5min":0.66400000000000003,"15min":1.135},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":2.6360000000000001,"5min":5.093,"15min":37.331000000000003},"last":0.40999999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4722,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1360,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8206,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7156,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3154,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1031208960,"data_stored":1031208960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1138163712,"data_stored":1138163712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1120600064,"data_stored":1120600064,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":951320576,"data_stored":951320576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":856227840,"data_stored":856227840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1376387072,"data_stored":1376387072,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":855375872,"data_stored":855375872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":867041280,"data_stored":867041280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:36:39.178 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:39.179 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:36:39.184 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd pool set unique_pool_3 pgp_num 16 2023-09-23T20:36:39.185 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:36:39.286 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:36:39.345 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:39.415 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:36:39.420 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:39.421 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:36:39.470 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T20:36:39.521 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:36:39.554 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:39 smithi018 ceph-mon[179647]: pgmap v906: 119 pgs: 119 active+clean; 2.5 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 113 MiB/s wr, 1.81k op/s 2023-09-23T20:36:39.555 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:39 smithi018 ceph-mon[179647]: from='client.55729 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:36:39.557 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:39 smithi018 ceph-mon[184144]: pgmap v906: 119 pgs: 119 active+clean; 2.5 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 113 MiB/s wr, 1.81k op/s 2023-09-23T20:36:39.557 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:39 smithi018 ceph-mon[184144]: from='client.55729 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:36:39.563 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:39.621 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:36:39.626 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:39 smithi149 ceph-mon[170394]: pgmap v906: 119 pgs: 119 active+clean; 2.5 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 113 MiB/s wr, 1.81k op/s 2023-09-23T20:36:39.626 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:39 smithi149 ceph-mon[170394]: from='client.55729 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:36:39.627 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:39.627 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:36:39.728 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:36:39.766 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:39.817 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:36:39.820 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:39.821 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:36:39.922 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:36:39.967 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:40.019 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 24 16 47985 47969 124.862 79.625 0.00942379 0.008003 2023-09-23T20:36:40.028 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:36:40.033 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:40.033 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:36:40.050 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T20:36:40.134 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:36:40.190 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:40.254 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:36:40.264 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:40.265 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:36:40.333 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:36:40.365 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:36:40.427 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:40.432 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:36:40.433 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:36:40.526 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:36:40.535 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:40.535 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:36:40.636 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:36:40.674 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:40.733 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:36:40.738 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:40.738 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:36:40.839 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:36:40.883 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:40.941 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:36:40.947 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:40.948 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:36:41.020 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 25 16 49161 49145 122.808 73.5 0.0130344 0.00813804 2023-09-23T20:36:41.049 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:36:41.114 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:41.832 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:41 smithi018 ceph-mon[179647]: pgmap v907: 119 pgs: 119 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 108 MiB/s wr, 1.73k op/s 2023-09-23T20:36:41.836 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:41 smithi018 ceph-mon[184144]: pgmap v907: 119 pgs: 119 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 108 MiB/s wr, 1.73k op/s 2023-09-23T20:36:41.843 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:36:41.851 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:41.852 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:36:41.952 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:36:41.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:41 smithi149 ceph-mon[170394]: pgmap v907: 119 pgs: 119 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 108 MiB/s wr, 1.73k op/s 2023-09-23T20:36:42.000 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:42.020 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 26 16 50422 50406 121.116 78.8125 0.0554858 0.00824465 2023-09-23T20:36:42.051 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:36:42.058 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:42.058 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:36:42.159 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:36:42.231 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:43.022 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 27 16 51471 51455 119.048 65.5625 0.0120541 0.00839489 2023-09-23T20:36:43.326 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:36:43.335 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:43.335 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:36:43.435 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:36:43.436 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:36:43.442 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:36:43.474 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:43.540 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:36:43.555 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:43.556 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:36:43.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:43 smithi018 ceph-mon[179647]: pgmap v908: 119 pgs: 119 active+clean; 2.9 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 93 MiB/s wr, 1.48k op/s 2023-09-23T20:36:43.644 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:43 smithi018 ceph-mon[184144]: pgmap v908: 119 pgs: 119 active+clean; 2.9 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 93 MiB/s wr, 1.48k op/s 2023-09-23T20:36:43.657 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:36:43.721 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:43 smithi149 ceph-mon[170394]: pgmap v908: 119 pgs: 119 active+clean; 2.9 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 93 MiB/s wr, 1.48k op/s 2023-09-23T20:36:43.729 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:43.789 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:36:43.796 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:43.796 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:36:43.897 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:36:43.985 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:44.023 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 28 16 52672 52656 117.473 75.0625 0.0104604 0.00850686 2023-09-23T20:36:44.745 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:36:44.757 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:44.757 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:36:44.768 INFO:teuthology.orchestra.run.smithi018.stderr:set pool 8 pgp_num to 16 2023-09-23T20:36:44.803 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:44 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:36:44.804 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:44 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:36:44.804 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:44 smithi018 ceph-mon[179647]: pgmap v909: 119 pgs: 119 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 91 MiB/s wr, 1.46k op/s 2023-09-23T20:36:44.804 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:44 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:36:44.804 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:44 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2269006226' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T20:36:44.804 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:44 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:36:44.805 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:44 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:36:44.805 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:44 smithi018 ceph-mon[184144]: pgmap v909: 119 pgs: 119 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 91 MiB/s wr, 1.46k op/s 2023-09-23T20:36:44.805 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:44 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:36:44.805 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:44 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2269006226' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T20:36:44.859 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:36:44.926 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:44.969 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:36:44.975 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:44.975 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:36:45.023 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 29 16 53880 53864 116.026 75.5 0.00750734 0.00861309 2023-09-23T20:36:45.076 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:36:45.141 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:36:44 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:36:44] "GET /metrics HTTP/1.1" 200 36499 "" "Prometheus/2.43.0" 2023-09-23T20:36:45.156 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:45.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:44 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:36:45.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:44 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:36:45.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:44 smithi149 ceph-mon[170394]: pgmap v909: 119 pgs: 119 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 91 MiB/s wr, 1.46k op/s 2023-09-23T20:36:45.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:44 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:36:45.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:44 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2269006226' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T20:36:45.285 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:36:45.293 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:45.294 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:36:45.395 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:36:45.496 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:45.534 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2023-09-23T20:36:45.534 INFO:teuthology.orchestra.run.smithi018.stdout: "ops": [ 2023-09-23T20:36:45.535 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:45.535 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_repop(client.35994.0:54478 8.7 e375/369 8:e5a8f7b1:::benchmark_data_smithi018_341490_object54477:head v 375'3384, mlcod=375'3381)", 2023-09-23T20:36:45.535 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:36:45.525861+0000", 2023-09-23T20:36:45.535 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.003571131, 2023-09-23T20:36:45.535 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0036023130000000002, 2023-09-23T20:36:45.536 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:36:45.536 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "started", 2023-09-23T20:36:45.536 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:36:45.536 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:45.536 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:36:45.536 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:36:45.525861+0000", 2023-09-23T20:36:45.536 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:36:45.537 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:45.537 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:45.537 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:36:45.537 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:36:45.525861+0000", 2023-09-23T20:36:45.537 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:36:45.538 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:45.538 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:45.538 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:36:45.538 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:36:45.525860+0000", 2023-09-23T20:36:45.538 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.999999 2023-09-23T20:36:45.538 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:45.538 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:45.539 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:36:45.539 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:36:45.525886+0000", 2023-09-23T20:36:45.539 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.5301999999999999e-05 2023-09-23T20:36:45.539 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:45.539 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:45.539 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:36:45.539 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:36:45.525902+0000", 2023-09-23T20:36:45.540 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.6013999999999999e-05 2023-09-23T20:36:45.540 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:45.540 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:45.540 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:36:45.540 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:36:45.525910+0000", 2023-09-23T20:36:45.541 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.5890000000000003e-06 2023-09-23T20:36:45.541 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:45.541 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:45.541 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:45.541 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:36:45.525979+0000", 2023-09-23T20:36:45.542 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.8440000000000002e-05 2023-09-23T20:36:45.542 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:45.542 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:45.542 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:36:45.542 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:36:45.526005+0000", 2023-09-23T20:36:45.542 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.6084e-05 2023-09-23T20:36:45.542 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:45.542 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:36:45.542 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:45.543 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:45.543 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:45.543 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_repop(client.35994.0:54485 8.c e375/369 8:3b2fca08:::benchmark_data_smithi018_341490_object54484:head v 375'3468, mlcod=375'3466)", 2023-09-23T20:36:45.543 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:36:45.528315+0000", 2023-09-23T20:36:45.543 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.001117205, 2023-09-23T20:36:45.543 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.001221604, 2023-09-23T20:36:45.543 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:36:45.543 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "started", 2023-09-23T20:36:45.543 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:36:45.543 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:45.543 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:36:45.543 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:36:45.528315+0000", 2023-09-23T20:36:45.544 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:36:45.544 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:45.544 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:45.544 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:36:45.544 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:36:45.528315+0000", 2023-09-23T20:36:45.544 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:36:45.544 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:45.544 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:45.544 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:36:45.545 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:36:45.528314+0000", 2023-09-23T20:36:45.545 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.999999 2023-09-23T20:36:45.545 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:45.545 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:45.545 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:36:45.545 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:36:45.528340+0000", 2023-09-23T20:36:45.546 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.6217e-05 2023-09-23T20:36:45.546 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:45.546 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:45.546 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:36:45.546 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:36:45.528352+0000", 2023-09-23T20:36:45.547 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2053e-05 2023-09-23T20:36:45.547 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:45.547 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:45.547 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:36:45.547 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:36:45.528361+0000", 2023-09-23T20:36:45.548 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.106e-06 2023-09-23T20:36:45.548 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:45.548 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:45.548 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:45.548 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:36:45.528413+0000", 2023-09-23T20:36:45.548 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.2556000000000003e-05 2023-09-23T20:36:45.549 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:45.549 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:45.549 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:36:45.549 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:36:45.528437+0000", 2023-09-23T20:36:45.549 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.4292999999999999e-05 2023-09-23T20:36:45.550 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:45.550 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:36:45.550 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:45.550 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:45.550 INFO:teuthology.orchestra.run.smithi018.stdout: ], 2023-09-23T20:36:45.550 INFO:teuthology.orchestra.run.smithi018.stdout: "num_ops": 2 2023-09-23T20:36:45.550 INFO:teuthology.orchestra.run.smithi018.stdout:} 2023-09-23T20:36:45.760 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:36:45.768 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:45.769 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:36:45.771 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:45 smithi018 ceph-mon[179647]: failed to encode map e375 with expected crc 2023-09-23T20:36:45.772 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:45 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2269006226' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num", "val": "16"}]': finished 2023-09-23T20:36:45.772 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:45 smithi018 ceph-mon[179647]: osdmap e375: 8 total, 8 up, 8 in 2023-09-23T20:36:45.772 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:45 smithi018 ceph-mon[184144]: failed to encode map e375 with expected crc 2023-09-23T20:36:45.772 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:45 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2269006226' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num", "val": "16"}]': finished 2023-09-23T20:36:45.772 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:45 smithi018 ceph-mon[184144]: osdmap e375: 8 total, 8 up, 8 in 2023-09-23T20:36:45.870 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:36:45.947 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:46.023 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 30 16 55122 55106 114.747 77.625 0.0119605 0.0087111 2023-09-23T20:36:46.029 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:36:46.034 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:46.035 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:36:46.136 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:36:46.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:45 smithi149 ceph-mon[170394]: failed to encode map e375 with expected crc 2023-09-23T20:36:46.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:45 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2269006226' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num", "val": "16"}]': finished 2023-09-23T20:36:46.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:45 smithi149 ceph-mon[170394]: osdmap e375: 8 total, 8 up, 8 in 2023-09-23T20:36:46.223 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:46.705 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:36:46.710 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:46.710 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:36:46.809 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T20:36:46.811 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:36:46.909 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:46.997 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:36:47.007 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:47.008 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:36:47.023 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 31 16 56330 56314 113.481 75.5 0.0121392 0.00880552 2023-09-23T20:36:47.109 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:36:47.153 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:47.218 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:36:47.221 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:47.222 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:36:47.323 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:36:47.420 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:47.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:47 smithi149 ceph-mon[170394]: pgmap v911: 119 pgs: 119 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s 2023-09-23T20:36:47.497 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:36:47.503 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:47.504 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:36:47.537 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:36:47.575 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:36:47.605 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:36:47.614 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:36:47.643 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:47 smithi018 ceph-mon[184144]: pgmap v911: 119 pgs: 119 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s 2023-09-23T20:36:47.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:47 smithi018 ceph-mon[179647]: pgmap v911: 119 pgs: 119 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s 2023-09-23T20:36:47.650 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:36:47.665 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:47.687 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:36:47.724 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:36:47.731 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:47.731 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:36:47.735 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:36:47.782 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:36:47.826 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:36:47.832 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:36:47.835 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:36:47.835 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:36:47.835 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:36:47.835 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:36:47.836 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:36:47.836 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:36:47.836 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:36:47.836 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:36:47.869 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:36:47.922 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:48.011 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:36:48.016 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:48.017 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:36:48.026 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 32 16 57407 57391 112.03 67.3125 0.0183869 0.00892121 2023-09-23T20:36:48.118 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:36:48.184 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:48.242 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:36:48.247 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:48.247 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:36:48.295 INFO:teuthology.orchestra.run.smithi018.stderr:nodeep-scrub is unset 2023-09-23T20:36:48.348 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:36:48.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:48 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1106438751' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:36:48.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:48 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1660389879' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:36:48.502 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:48.645 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:48 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1106438751' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:36:48.645 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:48 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1660389879' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:36:48.645 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:48 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1106438751' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:36:48.646 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:48 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1660389879' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:36:49.026 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 33 16 58671 58655 111.029 79 0.0142327 0.00900019 2023-09-23T20:36:49.027 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:36:49.031 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:49.031 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:36:49.132 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:36:49.176 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:49.227 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:36:49.234 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:49.234 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:36:49.322 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:49 smithi018 ceph-mon[179647]: pgmap v912: 119 pgs: 119 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s 2023-09-23T20:36:49.322 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:49 smithi018 ceph-mon[179647]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:36:49.322 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:49 smithi018 ceph-mon[179647]: Cluster is now healthy 2023-09-23T20:36:49.322 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:49 smithi018 ceph-mon[179647]: failed to encode map e376 with expected crc 2023-09-23T20:36:49.322 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:49 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1106438751' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:36:49.323 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:49 smithi018 ceph-mon[179647]: osdmap e376: 8 total, 8 up, 8 in 2023-09-23T20:36:49.323 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:49 smithi018 ceph-mon[179647]: failed to encode map e376 with expected crc 2023-09-23T20:36:49.323 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:49 smithi018 ceph-mon[184144]: pgmap v912: 119 pgs: 119 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s 2023-09-23T20:36:49.323 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:49 smithi018 ceph-mon[184144]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:36:49.323 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:49 smithi018 ceph-mon[184144]: Cluster is now healthy 2023-09-23T20:36:49.323 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:49 smithi018 ceph-mon[184144]: failed to encode map e376 with expected crc 2023-09-23T20:36:49.323 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:49 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1106438751' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:36:49.323 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:49 smithi018 ceph-mon[184144]: osdmap e376: 8 total, 8 up, 8 in 2023-09-23T20:36:49.323 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:49 smithi018 ceph-mon[184144]: failed to encode map e376 with expected crc 2023-09-23T20:36:49.335 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:36:49.384 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:49.446 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:36:49.451 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:49.452 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:36:49.452 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:49 smithi149 ceph-mon[170394]: pgmap v912: 119 pgs: 119 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s 2023-09-23T20:36:49.452 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:49 smithi149 ceph-mon[170394]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:36:49.452 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:49 smithi149 ceph-mon[170394]: Cluster is now healthy 2023-09-23T20:36:49.452 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:49 smithi149 ceph-mon[170394]: failed to encode map e376 with expected crc 2023-09-23T20:36:49.452 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:49 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1106438751' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:36:49.452 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:49 smithi149 ceph-mon[170394]: osdmap e376: 8 total, 8 up, 8 in 2023-09-23T20:36:49.453 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:49 smithi149 ceph-mon[170394]: failed to encode map e376 with expected crc 2023-09-23T20:36:49.552 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:36:49.612 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:49.792 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:36:50.147 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:50.148 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:36:50.148 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 34 16 59881 59865 109.988 75.625 0.00575953 0.00908674 2023-09-23T20:36:50.241 INFO:teuthology.orchestra.run.smithi018.stdout:ERROR: (22) Invalid argument 2023-09-23T20:36:50.242 INFO:teuthology.orchestra.run.smithi018.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:36:50.248 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:36:50.293 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:50.342 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:36:50.356 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:36:50.360 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:50.361 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:36:50.461 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:36:50.569 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:50 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:36:50.569 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:50 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:36:50.569 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:50 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:36:50.569 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:50 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:36:50.569 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:50 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:36:50.570 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:50 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:36:50.570 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:50 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:36:50.570 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:50 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:36:50.570 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:50 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:36:50.570 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:50 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:36:50.578 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:50.660 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:36:50.666 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:50.667 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:36:50.718 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:50 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:36:50.718 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:50 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:36:50.718 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:50 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:36:50.718 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:50 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:36:50.718 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:50 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:36:50.722 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T20:36:50.767 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:36:50.832 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:50.903 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T20:36:50.935 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:36:50.943 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:50.944 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:36:51.032 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 35 16 60959 60943 108.769 67.375 0.016137 0.00918712 2023-09-23T20:36:51.045 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:36:51.184 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:51.200 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 5, 7, 3, 2, 6, 4] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T20:36:51.201 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:36:51.201 INFO:tasks.thrashosds.thrasher:inject_pause on osd.6 2023-09-23T20:36:51.201 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T20:36:51.201 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T20:36:51.201 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 config set filestore_inject_stall 3 2023-09-23T20:36:51.239 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:36:51.266 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:36:51.277 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:51.277 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:36:51.378 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:36:51.472 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:51.538 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:51 smithi018 ceph-mon[184144]: pgmap v914: 119 pgs: 119 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 61 MiB/s wr, 975 op/s 2023-09-23T20:36:51.539 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:51 smithi018 ceph-mon[179647]: pgmap v914: 119 pgs: 119 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 61 MiB/s wr, 975 op/s 2023-09-23T20:36:51.540 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:51 smithi149 ceph-mon[170394]: pgmap v914: 119 pgs: 119 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 61 MiB/s wr, 975 op/s 2023-09-23T20:36:51.545 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:36:51.549 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:51.550 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:36:51.650 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:36:51.730 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:51.800 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:36:51.808 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:51.808 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:36:51.909 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:36:51.956 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:51.988 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:36:52.027 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 36 16 62078 62062 107.69 69.9375 0.0245672 0.00928004 2023-09-23T20:36:52.033 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:36:52.045 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:52.046 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:36:52.146 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:36:52.222 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:52.309 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:36:52.319 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:52.319 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:36:52.420 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:36:52.472 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:52.494 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:36:52.519 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:36:52.549 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:36:52.555 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:52.556 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:36:52.657 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:36:52.730 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:53.028 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 37 16 63216 63200 106.701 71.125 0.004882 0.00935469 2023-09-23T20:36:53.486 INFO:teuthology.orchestra.run.smithi149.stdout:{ 2023-09-23T20:36:53.486 INFO:teuthology.orchestra.run.smithi149.stdout: "success": "" 2023-09-23T20:36:53.486 INFO:teuthology.orchestra.run.smithi149.stdout:} 2023-09-23T20:36:53.526 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:36:53.533 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:53.534 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:36:53.634 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:36:53.695 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:53.841 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:53 smithi018 ceph-mon[184144]: pgmap v915: 119 pgs: 119 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s 2023-09-23T20:36:53.841 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:53 smithi018 ceph-mon[179647]: pgmap v915: 119 pgs: 119 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s 2023-09-23T20:36:53.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:53 smithi149 ceph-mon[170394]: pgmap v915: 119 pgs: 119 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s 2023-09-23T20:36:54.028 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 38 16 64419 64403 105.872 75.1875 0.00830946 0.00944044 2023-09-23T20:36:54.361 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:36:54.370 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:54.371 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:36:54.471 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:36:54.577 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:55.028 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 39 16 65693 65677 105.199 79.625 0.0104287 0.0095008 2023-09-23T20:36:55.144 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:36:54 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:36:54] "GET /metrics HTTP/1.1" 200 36499 "" "Prometheus/2.43.0" 2023-09-23T20:36:55.667 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:36:55.674 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:55.675 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:36:55.776 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:36:55.786 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:55 smithi018 ceph-mon[179647]: pgmap v916: 119 pgs: 119 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-09-23T20:36:55.787 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:55 smithi018 ceph-mon[184144]: pgmap v916: 119 pgs: 119 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-09-23T20:36:55.813 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:55.875 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:36:55.880 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:55 smithi149 ceph-mon[170394]: pgmap v916: 119 pgs: 119 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-09-23T20:36:55.881 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:55.881 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:36:55.982 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:36:56.026 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:56.028 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:2023-09-23T20:36:56.029051+0000 min lat: 0.00120245 max lat: 0.186851 avg lat: 0.00957304 2023-09-23T20:36:56.028 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T20:36:56.028 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 40 16 66861 66845 104.393 73 0.0188478 0.00957304 2023-09-23T20:36:56.064 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:36:56.094 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:36:56.100 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:36:56.101 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:56.101 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:36:56.134 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:36:56.168 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:36:56.202 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:36:56.233 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:36:56.246 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:56.271 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:36:56.296 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:36:56.302 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:56.303 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:36:56.315 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:36:56.355 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:36:56.363 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:36:56.363 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:36:56.363 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:36:56.363 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:36:56.363 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:36:56.363 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:36:56.363 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:36:56.363 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:36:56.404 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:36:56.475 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:56.572 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:56 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1976501263' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:36:56.572 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:56 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1976501263' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:36:56.717 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:36:56.724 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:56.725 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:36:56.750 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2023-09-23T20:36:56.750 INFO:teuthology.orchestra.run.smithi018.stdout: "size": 20, 2023-09-23T20:36:56.750 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 600, 2023-09-23T20:36:56.750 INFO:teuthology.orchestra.run.smithi018.stdout: "ops": [ 2023-09-23T20:36:56.750 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.750 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.45161.0:68466 6.c 6:320f8bfb:::benchmark_data_smithi018_270558_object68465:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e230)", 2023-09-23T20:36:56.750 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:28:04.042968+0000", 2023-09-23T20:36:56.750 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 532.70334069499995, 2023-09-23T20:36:56.750 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.1849001010000002, 2023-09-23T20:36:56.750 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:36:56.751 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:36:56.751 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:36:56.751 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.45161", 2023-09-23T20:36:56.751 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3795614987", 2023-09-23T20:36:56.751 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 68466 2023-09-23T20:36:56.751 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.751 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:36:56.751 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.751 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:36:56.751 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:04.042968+0000", 2023-09-23T20:36:56.751 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:36:56.752 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.752 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.752 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:36:56.752 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:04.042968+0000", 2023-09-23T20:36:56.752 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:36:56.752 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.752 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.752 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:36:56.752 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:04.042967+0000", 2023-09-23T20:36:56.752 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:36:56.752 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.753 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.753 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:36:56.753 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:04.043005+0000", 2023-09-23T20:36:56.753 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.7880999999999999e-05 2023-09-23T20:36:56.753 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.753 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.753 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:36:56.753 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:04.043021+0000", 2023-09-23T20:36:56.753 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.5979999999999999e-05 2023-09-23T20:36:56.753 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.753 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.754 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:36:56.754 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:04.043030+0000", 2023-09-23T20:36:56.754 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.5910000000000002e-06 2023-09-23T20:36:56.754 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.754 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.754 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:56.754 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:04.043170+0000", 2023-09-23T20:36:56.754 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00014017599999999999 2023-09-23T20:36:56.754 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.754 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.754 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:36:56.755 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:04.043185+0000", 2023-09-23T20:36:56.755 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.4789e-05 2023-09-23T20:36:56.755 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.755 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.755 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:56.755 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:05.022058+0000", 2023-09-23T20:36:56.755 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.97887254400000001 2023-09-23T20:36:56.755 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.755 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.755 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:36:56.755 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:05.022071+0000", 2023-09-23T20:36:56.756 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3016999999999999e-05 2023-09-23T20:36:56.756 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.756 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.756 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:56.756 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:06.207745+0000", 2023-09-23T20:36:56.756 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1856746280000001 2023-09-23T20:36:56.756 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.756 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.756 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:36:56.756 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:06.207958+0000", 2023-09-23T20:36:56.756 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000212408 2023-09-23T20:36:56.756 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.757 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.757 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 5,7", 2023-09-23T20:36:56.757 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:06.208139+0000", 2023-09-23T20:36:56.757 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00018073899999999999 2023-09-23T20:36:56.757 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.757 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.757 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:36:56.757 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:06.215444+0000", 2023-09-23T20:36:56.757 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0073050779999999996 2023-09-23T20:36:56.757 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.757 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.757 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:36:56.758 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:06.215482+0000", 2023-09-23T20:36:56.758 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.7985999999999998e-05 2023-09-23T20:36:56.758 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.758 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.758 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:36:56.758 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:06.225399+0000", 2023-09-23T20:36:56.758 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0099177599999999994 2023-09-23T20:36:56.758 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.758 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.758 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:36:56.758 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:06.225470+0000", 2023-09-23T20:36:56.759 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.0081999999999997e-05 2023-09-23T20:36:56.759 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.759 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.759 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:36:56.759 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:06.227869+0000", 2023-09-23T20:36:56.759 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0023990259999999998 2023-09-23T20:36:56.759 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:56.759 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:36:56.759 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:56.759 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.759 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.759 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.45161.0:68467 6.c 6:33c0ab26:::benchmark_data_smithi018_270558_object68466:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e230)", 2023-09-23T20:36:56.760 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:28:04.043047+0000", 2023-09-23T20:36:56.760 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 532.70326186499994, 2023-09-23T20:36:56.760 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.1959208280000002, 2023-09-23T20:36:56.760 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:36:56.760 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:36:56.760 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:36:56.760 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.45161", 2023-09-23T20:36:56.760 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3795614987", 2023-09-23T20:36:56.760 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 68467 2023-09-23T20:36:56.760 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.760 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:36:56.761 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.761 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:36:56.761 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:04.043047+0000", 2023-09-23T20:36:56.761 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:36:56.761 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.761 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.761 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:36:56.761 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:04.043047+0000", 2023-09-23T20:36:56.761 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:36:56.761 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.761 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.761 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:36:56.762 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:04.043046+0000", 2023-09-23T20:36:56.762 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.999999 2023-09-23T20:36:56.762 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.762 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.762 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:36:56.762 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:04.043101+0000", 2023-09-23T20:36:56.762 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.5018000000000001e-05 2023-09-23T20:36:56.762 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.762 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.762 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:36:56.762 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:04.043111+0000", 2023-09-23T20:36:56.763 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.4590000000000004e-06 2023-09-23T20:36:56.763 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.763 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.763 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:36:56.763 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:04.043118+0000", 2023-09-23T20:36:56.763 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.6340000000000003e-06 2023-09-23T20:36:56.763 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.763 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.763 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:56.763 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:04.043212+0000", 2023-09-23T20:36:56.763 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.399e-05 2023-09-23T20:36:56.764 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.764 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.764 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:36:56.764 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:04.043221+0000", 2023-09-23T20:36:56.764 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.6579999999999997e-06 2023-09-23T20:36:56.764 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.764 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.764 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:56.764 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:05.022095+0000", 2023-09-23T20:36:56.764 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.97887364200000004 2023-09-23T20:36:56.764 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.764 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.765 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:36:56.765 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:05.022103+0000", 2023-09-23T20:36:56.765 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.0090000000000007e-06 2023-09-23T20:36:56.765 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.765 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.765 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:56.765 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:06.213032+0000", 2023-09-23T20:36:56.765 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1909290669999999 2023-09-23T20:36:56.765 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.765 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.765 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:36:56.765 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:06.213177+0000", 2023-09-23T20:36:56.766 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00014537999999999999 2023-09-23T20:36:56.766 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.766 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.766 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 5,7", 2023-09-23T20:36:56.766 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:06.213344+0000", 2023-09-23T20:36:56.766 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00016685100000000001 2023-09-23T20:36:56.766 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.766 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.766 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:36:56.766 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:06.225333+0000", 2023-09-23T20:36:56.767 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.011988403999999999 2023-09-23T20:36:56.767 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.767 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.767 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:36:56.767 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:06.227963+0000", 2023-09-23T20:36:56.767 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0026302629999999999 2023-09-23T20:36:56.767 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.767 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.767 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:36:56.767 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:06.235942+0000", 2023-09-23T20:36:56.767 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0079790829999999997 2023-09-23T20:36:56.768 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.768 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.768 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:36:56.768 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:06.236003+0000", 2023-09-23T20:36:56.768 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.0649000000000002e-05 2023-09-23T20:36:56.768 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.768 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.768 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:36:56.768 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:06.238968+0000", 2023-09-23T20:36:56.768 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0029655139999999998 2023-09-23T20:36:56.768 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:56.768 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:36:56.769 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:56.769 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.769 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.769 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.45161.0:68472 6.c 6:3a94d71c:::benchmark_data_smithi018_270558_object68471:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e230)", 2023-09-23T20:36:56.769 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:28:04.058107+0000", 2023-09-23T20:36:56.769 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 532.68820245400002, 2023-09-23T20:36:56.769 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.1836921600000001, 2023-09-23T20:36:56.769 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:36:56.769 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:36:56.769 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:36:56.769 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.45161", 2023-09-23T20:36:56.769 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3795614987", 2023-09-23T20:36:56.770 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 68472 2023-09-23T20:36:56.770 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.770 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:36:56.770 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.770 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:36:56.770 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:04.058107+0000", 2023-09-23T20:36:56.770 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:36:56.770 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.770 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.770 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:36:56.770 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:04.058107+0000", 2023-09-23T20:36:56.771 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:36:56.771 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.771 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.771 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:36:56.771 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:04.058106+0000", 2023-09-23T20:36:56.771 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.999999 2023-09-23T20:36:56.771 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.771 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.771 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:36:56.771 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:04.058127+0000", 2023-09-23T20:36:56.771 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.143e-05 2023-09-23T20:36:56.771 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.772 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.772 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:36:56.772 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:04.058147+0000", 2023-09-23T20:36:56.772 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.0217999999999999e-05 2023-09-23T20:36:56.772 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.772 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.772 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:36:56.772 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:04.058154+0000", 2023-09-23T20:36:56.772 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.8229999999999997e-06 2023-09-23T20:36:56.772 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.772 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.772 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:56.772 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:04.058206+0000", 2023-09-23T20:36:56.773 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.2494000000000003e-05 2023-09-23T20:36:56.773 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.773 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.773 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:36:56.773 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:04.058216+0000", 2023-09-23T20:36:56.773 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.5189999999999998e-06 2023-09-23T20:36:56.773 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.773 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.773 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:56.773 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:05.022124+0000", 2023-09-23T20:36:56.773 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.96390836099999999 2023-09-23T20:36:56.773 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.773 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.774 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:36:56.774 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:05.022132+0000", 2023-09-23T20:36:56.774 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.0099999999999995e-06 2023-09-23T20:36:56.774 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.774 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.774 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:56.774 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:06.215585+0000", 2023-09-23T20:36:56.774 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1934526110000001 2023-09-23T20:36:56.774 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.774 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.774 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:36:56.774 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:06.215718+0000", 2023-09-23T20:36:56.775 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000132941 2023-09-23T20:36:56.775 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.775 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.775 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 5,7", 2023-09-23T20:36:56.775 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:06.215888+0000", 2023-09-23T20:36:56.775 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000169908 2023-09-23T20:36:56.775 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.775 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.775 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:36:56.775 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:06.225349+0000", 2023-09-23T20:36:56.775 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0094613979999999993 2023-09-23T20:36:56.775 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.776 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.776 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:36:56.776 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:06.228020+0000", 2023-09-23T20:36:56.776 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0026708610000000001 2023-09-23T20:36:56.776 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.776 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.776 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:36:56.776 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:06.239066+0000", 2023-09-23T20:36:56.776 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.011046344 2023-09-23T20:36:56.776 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.776 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.777 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:36:56.777 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:06.239126+0000", 2023-09-23T20:36:56.777 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.9778999999999998e-05 2023-09-23T20:36:56.777 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.777 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.777 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:36:56.777 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:06.241799+0000", 2023-09-23T20:36:56.777 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0026726219999999999 2023-09-23T20:36:56.777 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:56.777 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:36:56.777 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:56.778 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.778 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.778 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.45161.0:68480 6.c 6:3eaf7db7:::benchmark_data_smithi018_270558_object68479:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e230)", 2023-09-23T20:36:56.778 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:28:04.068993+0000", 2023-09-23T20:36:56.778 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 532.67731572599996, 2023-09-23T20:36:56.778 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.1783830000000002, 2023-09-23T20:36:56.778 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:36:56.778 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:36:56.778 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:36:56.778 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.45161", 2023-09-23T20:36:56.778 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3795614987", 2023-09-23T20:36:56.779 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 68480 2023-09-23T20:36:56.779 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.779 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:36:56.779 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.779 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:36:56.779 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:04.068993+0000", 2023-09-23T20:36:56.779 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:36:56.779 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.779 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.779 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:36:56.779 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:04.068993+0000", 2023-09-23T20:36:56.779 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:36:56.780 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.780 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.780 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:36:56.780 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:04.068986+0000", 2023-09-23T20:36:56.780 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999924 2023-09-23T20:36:56.780 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.780 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.780 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:36:56.780 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:04.069020+0000", 2023-09-23T20:36:56.780 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.4041000000000002e-05 2023-09-23T20:36:56.780 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.781 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.781 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:36:56.781 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:04.069034+0000", 2023-09-23T20:36:56.781 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3719e-05 2023-09-23T20:36:56.781 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.781 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.781 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:36:56.781 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:04.069043+0000", 2023-09-23T20:36:56.781 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.8429999999999999e-06 2023-09-23T20:36:56.781 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.781 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.781 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:56.782 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:04.069107+0000", 2023-09-23T20:36:56.782 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.3947000000000003e-05 2023-09-23T20:36:56.782 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.782 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.782 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:36:56.782 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:04.069119+0000", 2023-09-23T20:36:56.782 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2758999999999999e-05 2023-09-23T20:36:56.782 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.782 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.782 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:56.782 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:05.022156+0000", 2023-09-23T20:36:56.783 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.95303622600000004 2023-09-23T20:36:56.783 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.783 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.783 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:36:56.783 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:05.022164+0000", 2023-09-23T20:36:56.783 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.977e-06 2023-09-23T20:36:56.783 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.783 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.783 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:56.783 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:06.217835+0000", 2023-09-23T20:36:56.783 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.195671851 2023-09-23T20:36:56.783 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.784 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.784 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:36:56.784 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:06.217933+0000", 2023-09-23T20:36:56.784 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.7876999999999998e-05 2023-09-23T20:36:56.784 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.784 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.784 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 5,7", 2023-09-23T20:36:56.784 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:06.218034+0000", 2023-09-23T20:36:56.784 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00010027900000000001 2023-09-23T20:36:56.784 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.784 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.784 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:36:56.785 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:06.227890+0000", 2023-09-23T20:36:56.785 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0098562379999999998 2023-09-23T20:36:56.785 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.785 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.785 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:36:56.785 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:06.228062+0000", 2023-09-23T20:36:56.785 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00017186299999999999 2023-09-23T20:36:56.785 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.785 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.785 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:36:56.785 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:06.245432+0000", 2023-09-23T20:36:56.785 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.017370072 2023-09-23T20:36:56.785 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.786 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.786 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:36:56.786 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:06.245492+0000", 2023-09-23T20:36:56.786 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.0316999999999999e-05 2023-09-23T20:36:56.786 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.786 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.786 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:36:56.786 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:06.247376+0000", 2023-09-23T20:36:56.786 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0018844000000000001 2023-09-23T20:36:56.786 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:56.786 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:36:56.786 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:56.787 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.787 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.787 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.45161.0:68499 6.c 6:3ab32f48:::benchmark_data_smithi018_270558_object68498:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e230)", 2023-09-23T20:36:56.787 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:28:04.092063+0000", 2023-09-23T20:36:56.787 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 532.65424564800003, 2023-09-23T20:36:56.787 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.1571343239999998, 2023-09-23T20:36:56.787 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:36:56.787 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:36:56.787 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:36:56.787 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.45161", 2023-09-23T20:36:56.787 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3795614987", 2023-09-23T20:36:56.787 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 68499 2023-09-23T20:36:56.788 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.788 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:36:56.788 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.788 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:36:56.788 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:04.092063+0000", 2023-09-23T20:36:56.788 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:36:56.788 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.788 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.788 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:36:56.788 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:04.092063+0000", 2023-09-23T20:36:56.788 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:36:56.788 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.788 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.789 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:36:56.789 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:04.092062+0000", 2023-09-23T20:36:56.789 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:36:56.789 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.789 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.789 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:36:56.789 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:04.092088+0000", 2023-09-23T20:36:56.789 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.5961000000000001e-05 2023-09-23T20:36:56.789 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.789 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.789 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:36:56.790 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:04.092105+0000", 2023-09-23T20:36:56.790 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.6957999999999999e-05 2023-09-23T20:36:56.790 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.790 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.790 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:36:56.790 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:04.092113+0000", 2023-09-23T20:36:56.790 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.8110000000000003e-06 2023-09-23T20:36:56.790 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.790 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.790 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:56.790 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:04.092184+0000", 2023-09-23T20:36:56.791 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.1218999999999997e-05 2023-09-23T20:36:56.791 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.791 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.791 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:36:56.791 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:04.092199+0000", 2023-09-23T20:36:56.791 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.5506999999999999e-05 2023-09-23T20:36:56.791 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.791 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.791 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:56.791 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:05.022185+0000", 2023-09-23T20:36:56.791 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.92998589899999995 2023-09-23T20:36:56.791 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.792 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.792 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:36:56.792 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:05.022193+0000", 2023-09-23T20:36:56.792 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.8650000000000001e-06 2023-09-23T20:36:56.792 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.792 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.792 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:56.792 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:06.220364+0000", 2023-09-23T20:36:56.792 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1981710510000001 2023-09-23T20:36:56.792 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.792 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.793 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:36:56.793 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:06.220506+0000", 2023-09-23T20:36:56.793 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000142046 2023-09-23T20:36:56.793 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.793 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.793 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 5,7", 2023-09-23T20:36:56.793 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:06.220664+0000", 2023-09-23T20:36:56.793 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00015750000000000001 2023-09-23T20:36:56.793 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.793 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.793 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:36:56.794 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:06.227899+0000", 2023-09-23T20:36:56.794 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0072355859999999996 2023-09-23T20:36:56.794 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.794 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.794 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:36:56.794 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:06.228196+0000", 2023-09-23T20:36:56.794 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00029671400000000001 2023-09-23T20:36:56.794 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.794 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.794 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:36:56.794 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:06.247435+0000", 2023-09-23T20:36:56.795 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.019239136 2023-09-23T20:36:56.795 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.795 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.795 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:36:56.795 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:06.247483+0000", 2023-09-23T20:36:56.795 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.8130000000000002e-05 2023-09-23T20:36:56.795 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.795 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.795 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:36:56.795 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:28:06.249198+0000", 2023-09-23T20:36:56.795 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0017144510000000001 2023-09-23T20:36:56.795 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:56.796 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:36:56.796 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:56.796 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.796 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.796 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.45161.0:136070 6.16 6:6af4a005:::benchmark_data_smithi018_270558_object31331:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e258)", 2023-09-23T20:36:56.796 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:29:27.396634+0000", 2023-09-23T20:36:56.796 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 449.34967483600002, 2023-09-23T20:36:56.796 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.1461166540000001, 2023-09-23T20:36:56.796 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:36:56.796 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:36:56.796 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:36:56.797 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.45161", 2023-09-23T20:36:56.797 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3795614987", 2023-09-23T20:36:56.797 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 136070 2023-09-23T20:36:56.797 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.797 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:36:56.797 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.797 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:36:56.797 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.396634+0000", 2023-09-23T20:36:56.797 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:36:56.797 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.797 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.797 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:36:56.797 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.396634+0000", 2023-09-23T20:36:56.798 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:36:56.798 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.798 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.798 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:36:56.798 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.396633+0000", 2023-09-23T20:36:56.798 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.999999 2023-09-23T20:36:56.798 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.798 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.798 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:36:56.798 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.396639+0000", 2023-09-23T20:36:56.798 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.6099999999999997e-06 2023-09-23T20:36:56.799 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.799 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.799 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:36:56.799 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.396654+0000", 2023-09-23T20:36:56.799 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.5432999999999999e-05 2023-09-23T20:36:56.799 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.799 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.799 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:36:56.799 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.396661+0000", 2023-09-23T20:36:56.799 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.4799999999999998e-06 2023-09-23T20:36:56.799 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.799 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.800 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:56.800 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.396728+0000", 2023-09-23T20:36:56.800 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.6884999999999994e-05 2023-09-23T20:36:56.800 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.800 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.800 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:36:56.800 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.396741+0000", 2023-09-23T20:36:56.800 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3933e-05 2023-09-23T20:36:56.800 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.800 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.800 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:56.800 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:28.349683+0000", 2023-09-23T20:36:56.801 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.95294156299999999 2023-09-23T20:36:56.801 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.801 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.801 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:36:56.801 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:28.349696+0000", 2023-09-23T20:36:56.801 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3191e-05 2023-09-23T20:36:56.801 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.801 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.801 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:56.801 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.535166+0000", 2023-09-23T20:36:56.801 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1854697679999999 2023-09-23T20:36:56.802 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.802 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.802 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:36:56.802 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.535254+0000", 2023-09-23T20:36:56.802 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.7965000000000002e-05 2023-09-23T20:36:56.802 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.802 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.802 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 4,6,7", 2023-09-23T20:36:56.802 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.535550+0000", 2023-09-23T20:36:56.802 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00029616100000000003 2023-09-23T20:36:56.802 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.803 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.803 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:36:56.803 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.538940+0000", 2023-09-23T20:36:56.803 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.003389651 2023-09-23T20:36:56.803 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.803 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.803 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:36:56.803 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.538996+0000", 2023-09-23T20:36:56.803 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.6674e-05 2023-09-23T20:36:56.803 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.803 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.803 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:36:56.804 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.540647+0000", 2023-09-23T20:36:56.804 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0016507220000000001 2023-09-23T20:36:56.804 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.804 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.804 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:36:56.804 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.542633+0000", 2023-09-23T20:36:56.804 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0019862809999999999 2023-09-23T20:36:56.804 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.804 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.804 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:36:56.804 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.542666+0000", 2023-09-23T20:36:56.804 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.2740999999999997e-05 2023-09-23T20:36:56.805 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.805 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.805 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:36:56.805 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.542751+0000", 2023-09-23T20:36:56.805 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.4754999999999997e-05 2023-09-23T20:36:56.805 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:56.805 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:36:56.805 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:56.805 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.805 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.805 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.45161.0:136077 6.16 6:6b87af5f:::benchmark_data_smithi018_270558_object31338:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e258)", 2023-09-23T20:36:56.805 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:29:27.409208+0000", 2023-09-23T20:36:56.806 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 449.33710064000002, 2023-09-23T20:36:56.806 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.1337409109999999, 2023-09-23T20:36:56.806 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:36:56.806 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:36:56.806 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:36:56.806 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.45161", 2023-09-23T20:36:56.806 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3795614987", 2023-09-23T20:36:56.806 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 136077 2023-09-23T20:36:56.806 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.806 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:36:56.806 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.806 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:36:56.807 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.409208+0000", 2023-09-23T20:36:56.807 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:36:56.807 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.807 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.807 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:36:56.807 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.409208+0000", 2023-09-23T20:36:56.807 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:36:56.807 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.807 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.807 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:36:56.807 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.409207+0000", 2023-09-23T20:36:56.807 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:36:56.808 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.808 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.808 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:36:56.808 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.409214+0000", 2023-09-23T20:36:56.808 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.3930000000000002e-06 2023-09-23T20:36:56.808 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.808 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.808 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:36:56.808 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.409229+0000", 2023-09-23T20:36:56.808 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.4535000000000001e-05 2023-09-23T20:36:56.808 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.808 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.809 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:36:56.809 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.409238+0000", 2023-09-23T20:36:56.809 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.5899999999999997e-06 2023-09-23T20:36:56.809 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.809 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.809 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:56.809 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.409307+0000", 2023-09-23T20:36:56.809 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.9023000000000006e-05 2023-09-23T20:36:56.809 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.809 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.809 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:36:56.809 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.409318+0000", 2023-09-23T20:36:56.809 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0387999999999999e-05 2023-09-23T20:36:56.810 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.810 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.810 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:56.810 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:28.349722+0000", 2023-09-23T20:36:56.810 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.94040487299999997 2023-09-23T20:36:56.810 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.810 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.810 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:36:56.810 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:28.349733+0000", 2023-09-23T20:36:56.810 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0152e-05 2023-09-23T20:36:56.810 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.811 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.811 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:56.811 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.536807+0000", 2023-09-23T20:36:56.811 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.187074006 2023-09-23T20:36:56.811 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.811 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.811 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:36:56.811 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.536968+0000", 2023-09-23T20:36:56.811 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00016101399999999999 2023-09-23T20:36:56.811 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.811 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.812 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 4,6,7", 2023-09-23T20:36:56.812 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.537271+0000", 2023-09-23T20:36:56.812 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000302887 2023-09-23T20:36:56.812 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.812 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.812 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:36:56.812 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.539134+0000", 2023-09-23T20:36:56.812 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.001863611 2023-09-23T20:36:56.812 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.812 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.812 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:36:56.812 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.542232+0000", 2023-09-23T20:36:56.813 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.003097533 2023-09-23T20:36:56.813 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.813 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.813 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:36:56.813 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.542819+0000", 2023-09-23T20:36:56.813 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00058755999999999999 2023-09-23T20:36:56.813 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.813 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.813 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:36:56.813 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.542846+0000", 2023-09-23T20:36:56.813 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.6855e-05 2023-09-23T20:36:56.813 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.814 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.814 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:36:56.814 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.542874+0000", 2023-09-23T20:36:56.814 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.7798e-05 2023-09-23T20:36:56.814 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.814 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.814 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:36:56.814 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.542949+0000", 2023-09-23T20:36:56.814 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.5531999999999994e-05 2023-09-23T20:36:56.814 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:56.814 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:36:56.815 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:56.815 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.815 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.815 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:57605 7.3 7:c29712e0:::benchmark_data_smithi018_308601_object57604:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e311)", 2023-09-23T20:36:56.815 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:32:32.219481+0000", 2023-09-23T20:36:56.815 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 264.52682805799998, 2023-09-23T20:36:56.815 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.3860234459999998, 2023-09-23T20:36:56.815 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:36:56.815 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:36:56.815 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:36:56.815 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:36:56.815 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:36:56.816 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 57605 2023-09-23T20:36:56.816 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.816 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:36:56.816 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.816 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:36:56.816 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.219481+0000", 2023-09-23T20:36:56.816 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:36:56.816 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.816 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.816 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:36:56.816 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.219481+0000", 2023-09-23T20:36:56.816 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:36:56.817 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.817 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.817 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:36:56.817 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.219480+0000", 2023-09-23T20:36:56.817 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:36:56.817 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.817 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.817 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:36:56.817 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.219513+0000", 2023-09-23T20:36:56.817 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.3324999999999997e-05 2023-09-23T20:36:56.817 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.817 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.818 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:36:56.818 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.219526+0000", 2023-09-23T20:36:56.818 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3145999999999999e-05 2023-09-23T20:36:56.818 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.818 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.818 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:36:56.818 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.219534+0000", 2023-09-23T20:36:56.818 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.729e-06 2023-09-23T20:36:56.818 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.818 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.818 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:56.819 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.231964+0000", 2023-09-23T20:36:56.819 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.012430274999999999 2023-09-23T20:36:56.819 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.819 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.819 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:36:56.819 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.231974+0000", 2023-09-23T20:36:56.819 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.4939999999999992e-06 2023-09-23T20:36:56.819 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.819 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.819 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:56.819 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.215331+0000", 2023-09-23T20:36:56.819 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.983357276 2023-09-23T20:36:56.820 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.820 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.820 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:36:56.820 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.215344+0000", 2023-09-23T20:36:56.820 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3061000000000001e-05 2023-09-23T20:36:56.820 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.820 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.820 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:56.820 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.573779+0000", 2023-09-23T20:36:56.820 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.358434581 2023-09-23T20:36:56.820 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.821 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.821 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:36:56.821 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.573953+0000", 2023-09-23T20:36:56.821 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00017463999999999999 2023-09-23T20:36:56.821 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.821 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.821 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 2,4", 2023-09-23T20:36:56.821 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.574063+0000", 2023-09-23T20:36:56.821 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000110059 2023-09-23T20:36:56.821 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.821 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.821 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:36:56.822 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.578839+0000", 2023-09-23T20:36:56.822 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0047753709999999996 2023-09-23T20:36:56.822 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.822 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.822 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:36:56.822 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.578852+0000", 2023-09-23T20:36:56.822 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3360000000000001e-05 2023-09-23T20:36:56.822 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.822 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.822 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:36:56.822 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.602935+0000", 2023-09-23T20:36:56.823 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.024082883999999999 2023-09-23T20:36:56.823 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.823 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.823 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:36:56.823 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.602997+0000", 2023-09-23T20:36:56.823 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.2113000000000007e-05 2023-09-23T20:36:56.823 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.823 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.823 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:36:56.823 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.605505+0000", 2023-09-23T20:36:56.823 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.002507493 2023-09-23T20:36:56.824 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:56.824 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:36:56.824 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:56.824 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.824 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.824 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:57610 7.3 7:cbaa52d6:::benchmark_data_smithi018_308601_object57609:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e311)", 2023-09-23T20:36:56.824 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:32:32.219584+0000", 2023-09-23T20:36:56.824 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 264.52672542300002, 2023-09-23T20:36:56.824 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.38833787, 2023-09-23T20:36:56.824 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:36:56.824 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:36:56.824 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:36:56.825 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:36:56.825 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:36:56.825 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 57610 2023-09-23T20:36:56.825 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.825 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:36:56.825 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.825 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:36:56.825 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.219584+0000", 2023-09-23T20:36:56.825 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:36:56.825 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.825 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.825 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:36:56.826 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.219584+0000", 2023-09-23T20:36:56.826 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:36:56.826 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.826 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.826 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:36:56.826 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.219583+0000", 2023-09-23T20:36:56.826 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:36:56.826 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.826 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.826 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:36:56.826 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.219607+0000", 2023-09-23T20:36:56.827 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.4170000000000001e-05 2023-09-23T20:36:56.827 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.827 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.827 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:36:56.827 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.219614+0000", 2023-09-23T20:36:56.827 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.4790000000000001e-06 2023-09-23T20:36:56.827 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.827 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.827 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:36:56.827 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.219619+0000", 2023-09-23T20:36:56.827 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.079e-06 2023-09-23T20:36:56.827 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.828 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.828 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:56.828 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.231994+0000", 2023-09-23T20:36:56.828 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.012374484999999999 2023-09-23T20:36:56.828 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.828 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.828 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:36:56.828 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.232000+0000", 2023-09-23T20:36:56.828 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.4790000000000001e-06 2023-09-23T20:36:56.828 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.828 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.828 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:56.829 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.215369+0000", 2023-09-23T20:36:56.829 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.98336916699999999 2023-09-23T20:36:56.829 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.829 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.829 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:36:56.829 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.215378+0000", 2023-09-23T20:36:56.829 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.6479999999999992e-06 2023-09-23T20:36:56.829 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.829 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.829 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:56.829 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.577198+0000", 2023-09-23T20:36:56.829 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.36182066 2023-09-23T20:36:56.830 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.830 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.830 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:36:56.830 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.577312+0000", 2023-09-23T20:36:56.830 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000113257 2023-09-23T20:36:56.830 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.830 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.830 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 2,4", 2023-09-23T20:36:56.830 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.577424+0000", 2023-09-23T20:36:56.830 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00011234999999999999 2023-09-23T20:36:56.830 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.831 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.831 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:36:56.831 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.580718+0000", 2023-09-23T20:36:56.831 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0032940090000000001 2023-09-23T20:36:56.831 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.831 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.831 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:36:56.831 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.582068+0000", 2023-09-23T20:36:56.831 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0013497990000000001 2023-09-23T20:36:56.831 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.831 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.832 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:36:56.832 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.605587+0000", 2023-09-23T20:36:56.832 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.023518779 2023-09-23T20:36:56.832 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.832 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.832 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:36:56.832 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.605666+0000", 2023-09-23T20:36:56.832 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.9777999999999994e-05 2023-09-23T20:36:56.832 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.832 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.832 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:36:56.832 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.607922+0000", 2023-09-23T20:36:56.833 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00225515 2023-09-23T20:36:56.833 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:56.833 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:36:56.833 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:56.833 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.833 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.833 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:57645 7.3 7:cb6be1ad:::benchmark_data_smithi018_308601_object57644:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e312)", 2023-09-23T20:36:56.833 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:32:32.275903+0000", 2023-09-23T20:36:56.833 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 264.47040606899998, 2023-09-23T20:36:56.833 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.3348626370000001, 2023-09-23T20:36:56.833 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:36:56.833 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:36:56.834 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:36:56.834 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:36:56.834 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:36:56.834 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 57645 2023-09-23T20:36:56.834 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.834 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:36:56.834 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.834 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:36:56.834 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.275903+0000", 2023-09-23T20:36:56.834 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:36:56.834 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.835 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.835 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:36:56.835 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.275903+0000", 2023-09-23T20:36:56.835 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:36:56.835 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.835 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.835 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:36:56.835 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.275902+0000", 2023-09-23T20:36:56.835 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:36:56.835 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.835 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.835 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:36:56.836 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.275926+0000", 2023-09-23T20:36:56.836 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.4209000000000001e-05 2023-09-23T20:36:56.836 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.836 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.836 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:36:56.836 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.275938+0000", 2023-09-23T20:36:56.836 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1992e-05 2023-09-23T20:36:56.836 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.836 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.836 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:36:56.836 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.275946+0000", 2023-09-23T20:36:56.836 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.8839999999999994e-06 2023-09-23T20:36:56.836 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.837 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.837 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:56.837 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.276851+0000", 2023-09-23T20:36:56.837 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00090492800000000005 2023-09-23T20:36:56.837 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.837 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.837 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:36:56.837 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.276861+0000", 2023-09-23T20:36:56.837 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0208e-05 2023-09-23T20:36:56.837 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.837 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.837 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:56.838 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.215401+0000", 2023-09-23T20:36:56.838 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.93854029800000005 2023-09-23T20:36:56.838 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.838 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.838 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:36:56.838 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.215410+0000", 2023-09-23T20:36:56.838 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.3510000000000002e-06 2023-09-23T20:36:56.838 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.838 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.838 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:56.838 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.578900+0000", 2023-09-23T20:36:56.839 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.363490173 2023-09-23T20:36:56.839 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.839 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.839 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:36:56.839 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.579070+0000", 2023-09-23T20:36:56.839 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00017048899999999999 2023-09-23T20:36:56.839 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.839 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.839 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 2,4", 2023-09-23T20:36:56.839 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.579260+0000", 2023-09-23T20:36:56.839 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000189667 2023-09-23T20:36:56.839 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.839 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.840 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:36:56.840 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.583356+0000", 2023-09-23T20:36:56.840 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0040960520000000002 2023-09-23T20:36:56.840 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.840 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.840 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:36:56.840 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.583406+0000", 2023-09-23T20:36:56.840 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.9449e-05 2023-09-23T20:36:56.840 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.840 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.840 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:36:56.840 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.607983+0000", 2023-09-23T20:36:56.841 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.024577576 2023-09-23T20:36:56.841 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.841 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.841 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:36:56.841 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.608038+0000", 2023-09-23T20:36:56.841 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.4787000000000002e-05 2023-09-23T20:36:56.841 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.841 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.841 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:36:56.841 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.610766+0000", 2023-09-23T20:36:56.841 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0027278319999999999 2023-09-23T20:36:56.841 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:56.842 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:36:56.842 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:56.842 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.842 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.842 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:57662 7.3 7:ce017a4a:::benchmark_data_smithi018_308601_object57661:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e312)", 2023-09-23T20:36:56.842 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:32:32.295077+0000", 2023-09-23T20:36:56.842 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 264.45123199199998, 2023-09-23T20:36:56.842 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.3221079059999998, 2023-09-23T20:36:56.842 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:36:56.842 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:36:56.842 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:36:56.842 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:36:56.843 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:36:56.843 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 57662 2023-09-23T20:36:56.843 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.843 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:36:56.843 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.843 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:36:56.843 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.295077+0000", 2023-09-23T20:36:56.843 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:36:56.843 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.843 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.843 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:36:56.843 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.295077+0000", 2023-09-23T20:36:56.844 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:36:56.844 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.844 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.844 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:36:56.844 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.295075+0000", 2023-09-23T20:36:56.844 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:36:56.844 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.844 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.844 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:36:56.844 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.295104+0000", 2023-09-23T20:36:56.844 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.8374999999999999e-05 2023-09-23T20:36:56.844 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.845 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.845 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:36:56.845 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.295117+0000", 2023-09-23T20:36:56.845 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2943e-05 2023-09-23T20:36:56.845 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.845 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.845 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:36:56.845 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.295123+0000", 2023-09-23T20:36:56.845 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.5989999999999998e-06 2023-09-23T20:36:56.845 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.845 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.845 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:56.846 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.295189+0000", 2023-09-23T20:36:56.846 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.5988000000000006e-05 2023-09-23T20:36:56.846 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.846 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.846 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:36:56.846 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.295205+0000", 2023-09-23T20:36:56.846 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.5434000000000002e-05 2023-09-23T20:36:56.846 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.846 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.846 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:56.846 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.215432+0000", 2023-09-23T20:36:56.846 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.92022715499999996 2023-09-23T20:36:56.847 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.847 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.847 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:36:56.847 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.215440+0000", 2023-09-23T20:36:56.847 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.2519999999999999e-06 2023-09-23T20:36:56.847 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.847 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.847 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:56.847 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.580753+0000", 2023-09-23T20:36:56.847 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.365312673 2023-09-23T20:36:56.847 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.847 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.848 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:36:56.848 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.580896+0000", 2023-09-23T20:36:56.848 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00014352000000000001 2023-09-23T20:36:56.848 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.848 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.848 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 2,4", 2023-09-23T20:36:56.848 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.581023+0000", 2023-09-23T20:36:56.848 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000127112 2023-09-23T20:36:56.848 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.848 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.848 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:36:56.848 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.583459+0000", 2023-09-23T20:36:56.849 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0024356450000000002 2023-09-23T20:36:56.849 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.849 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.849 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:36:56.849 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.583468+0000", 2023-09-23T20:36:56.849 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.4840000000000004e-06 2023-09-23T20:36:56.849 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.849 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.849 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:36:56.849 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.613992+0000", 2023-09-23T20:36:56.849 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.030524193000000002 2023-09-23T20:36:56.849 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.850 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.850 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:36:56.850 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.614070+0000", 2023-09-23T20:36:56.850 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.8054999999999997e-05 2023-09-23T20:36:56.850 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.850 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.850 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:36:56.850 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.617185+0000", 2023-09-23T20:36:56.850 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0031151690000000001 2023-09-23T20:36:56.850 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:56.850 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:36:56.851 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:56.851 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.851 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.851 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:57667 7.3 7:cbc699af:::benchmark_data_smithi018_308601_object57666:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e312)", 2023-09-23T20:36:56.851 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:32:32.300970+0000", 2023-09-23T20:36:56.851 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 264.44533903899998, 2023-09-23T20:36:56.851 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.3193818780000002, 2023-09-23T20:36:56.851 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:36:56.851 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:36:56.851 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:36:56.851 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:36:56.851 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:36:56.852 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 57667 2023-09-23T20:36:56.852 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.852 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:36:56.852 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.852 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:36:56.852 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.300970+0000", 2023-09-23T20:36:56.852 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:36:56.852 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.852 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.852 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:36:56.852 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.300970+0000", 2023-09-23T20:36:56.852 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:36:56.853 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.853 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.853 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:36:56.853 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.300969+0000", 2023-09-23T20:36:56.853 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.999999 2023-09-23T20:36:56.853 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.853 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.853 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:36:56.853 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.300992+0000", 2023-09-23T20:36:56.853 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.2997e-05 2023-09-23T20:36:56.853 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.853 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.853 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:36:56.854 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.301006+0000", 2023-09-23T20:36:56.854 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3509000000000001e-05 2023-09-23T20:36:56.854 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.854 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.854 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:36:56.854 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.301014+0000", 2023-09-23T20:36:56.854 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.9189999999999999e-06 2023-09-23T20:36:56.854 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.854 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.854 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:56.854 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.301060+0000", 2023-09-23T20:36:56.855 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.6597999999999997e-05 2023-09-23T20:36:56.855 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.855 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.855 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:36:56.855 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.301067+0000", 2023-09-23T20:36:56.855 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.6039999999999998e-06 2023-09-23T20:36:56.855 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.855 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.855 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:56.855 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.215463+0000", 2023-09-23T20:36:56.855 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.914396019 2023-09-23T20:36:56.856 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.856 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.856 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:36:56.856 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.215471+0000", 2023-09-23T20:36:56.856 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.1880000000000002e-06 2023-09-23T20:36:56.856 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.856 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.856 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:56.856 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.582113+0000", 2023-09-23T20:36:56.856 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.366641642 2023-09-23T20:36:56.856 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.856 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.856 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:36:56.857 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.582227+0000", 2023-09-23T20:36:56.857 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000114545 2023-09-23T20:36:56.857 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.857 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.857 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 2,4", 2023-09-23T20:36:56.857 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.582342+0000", 2023-09-23T20:36:56.857 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000114294 2023-09-23T20:36:56.857 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.857 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.857 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:36:56.857 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.584646+0000", 2023-09-23T20:36:56.857 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.002304038 2023-09-23T20:36:56.858 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.858 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.858 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:36:56.858 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.584666+0000", 2023-09-23T20:36:56.858 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.0596000000000001e-05 2023-09-23T20:36:56.858 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.858 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.858 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:36:56.858 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.617277+0000", 2023-09-23T20:36:56.858 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.032610293999999998 2023-09-23T20:36:56.858 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.859 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.859 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:36:56.859 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.617329+0000", 2023-09-23T20:36:56.859 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.2527e-05 2023-09-23T20:36:56.859 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.859 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.859 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:36:56.859 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.620352+0000", 2023-09-23T20:36:56.859 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0030229179999999999 2023-09-23T20:36:56.859 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:56.859 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:36:56.859 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:56.860 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.860 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.860 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:57675 7.3 7:c7bba963:::benchmark_data_smithi018_308601_object57674:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e312)", 2023-09-23T20:36:56.860 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:32:32.327987+0000", 2023-09-23T20:36:56.860 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 264.418322515, 2023-09-23T20:36:56.860 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.2956588660000001, 2023-09-23T20:36:56.860 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:36:56.860 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:36:56.860 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:36:56.860 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:36:56.860 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:36:56.860 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 57675 2023-09-23T20:36:56.861 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.861 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:36:56.861 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.861 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:36:56.861 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.327987+0000", 2023-09-23T20:36:56.861 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:36:56.861 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.861 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.861 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:36:56.861 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.327987+0000", 2023-09-23T20:36:56.861 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:36:56.861 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.862 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.862 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:36:56.862 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.327985+0000", 2023-09-23T20:36:56.862 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:36:56.862 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.862 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.862 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:36:56.862 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.328012+0000", 2023-09-23T20:36:56.862 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.6673e-05 2023-09-23T20:36:56.862 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.863 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.863 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:36:56.863 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.328033+0000", 2023-09-23T20:36:56.863 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.1865000000000002e-05 2023-09-23T20:36:56.863 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.863 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.863 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:36:56.863 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.328046+0000", 2023-09-23T20:36:56.863 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2138e-05 2023-09-23T20:36:56.863 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.863 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.863 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:56.864 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.328112+0000", 2023-09-23T20:36:56.864 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.6155999999999995e-05 2023-09-23T20:36:56.864 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.864 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.864 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:36:56.864 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.328123+0000", 2023-09-23T20:36:56.864 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1323e-05 2023-09-23T20:36:56.864 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.864 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.864 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:56.864 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.215494+0000", 2023-09-23T20:36:56.864 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.88737101299999999 2023-09-23T20:36:56.865 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.865 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.865 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:36:56.865 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.215502+0000", 2023-09-23T20:36:56.865 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.3140000000000004e-06 2023-09-23T20:36:56.865 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.865 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.865 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:56.865 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.583495+0000", 2023-09-23T20:36:56.865 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.367992519 2023-09-23T20:36:56.865 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.865 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.866 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:36:56.866 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.583598+0000", 2023-09-23T20:36:56.866 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00010268399999999999 2023-09-23T20:36:56.866 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.866 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.866 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 2,4", 2023-09-23T20:36:56.866 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.583704+0000", 2023-09-23T20:36:56.866 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000106096 2023-09-23T20:36:56.866 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.866 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.866 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:36:56.866 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.584776+0000", 2023-09-23T20:36:56.867 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00107215 2023-09-23T20:36:56.867 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.867 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.867 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:36:56.867 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.585126+0000", 2023-09-23T20:36:56.867 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00035052599999999998 2023-09-23T20:36:56.867 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.867 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.867 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:36:56.867 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.620444+0000", 2023-09-23T20:36:56.867 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.035318081000000001 2023-09-23T20:36:56.867 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.868 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.868 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:36:56.868 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.620519+0000", 2023-09-23T20:36:56.868 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.4976999999999997e-05 2023-09-23T20:36:56.868 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.868 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.868 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:36:56.868 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.623645+0000", 2023-09-23T20:36:56.868 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.003126017 2023-09-23T20:36:56.868 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:56.868 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:36:56.868 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:56.868 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.869 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.869 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:69129 7.3 7:cf914298:::benchmark_data_smithi018_308601_object69128:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e318)", 2023-09-23T20:36:56.869 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:32:49.475881+0000", 2023-09-23T20:36:56.869 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 247.270428127, 2023-09-23T20:36:56.869 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.048825159, 2023-09-23T20:36:56.869 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:36:56.869 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:36:56.869 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:36:56.869 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:36:56.869 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:36:56.869 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 69129 2023-09-23T20:36:56.869 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.870 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:36:56.870 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.870 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:36:56.870 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.475881+0000", 2023-09-23T20:36:56.870 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:36:56.870 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.870 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.870 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:36:56.870 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.475881+0000", 2023-09-23T20:36:56.870 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:36:56.870 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.870 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.871 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:36:56.871 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.475881+0000", 2023-09-23T20:36:56.871 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:36:56.871 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.871 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.871 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:36:56.871 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.475925+0000", 2023-09-23T20:36:56.871 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.4274000000000003e-05 2023-09-23T20:36:56.871 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.871 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.871 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:36:56.871 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.475932+0000", 2023-09-23T20:36:56.872 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.2300000000000002e-06 2023-09-23T20:36:56.872 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.872 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.872 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:36:56.872 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.475937+0000", 2023-09-23T20:36:56.872 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.4600000000000002e-06 2023-09-23T20:36:56.872 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.872 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.872 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:56.872 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.480114+0000", 2023-09-23T20:36:56.872 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0041766099999999999 2023-09-23T20:36:56.872 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.873 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.873 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:36:56.873 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.480293+0000", 2023-09-23T20:36:56.873 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00017894300000000001 2023-09-23T20:36:56.873 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.873 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.873 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 2,4", 2023-09-23T20:36:56.873 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.480478+0000", 2023-09-23T20:36:56.873 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00018538100000000001 2023-09-23T20:36:56.873 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.873 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.873 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:36:56.874 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.520416+0000", 2023-09-23T20:36:56.874 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.039937451999999998 2023-09-23T20:36:56.874 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.874 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.874 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:36:56.874 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.523198+0000", 2023-09-23T20:36:56.874 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0027818999999999999 2023-09-23T20:36:56.874 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.874 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.874 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:56.874 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.528277+0000", 2023-09-23T20:36:56.875 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0050791120000000002 2023-09-23T20:36:56.875 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.875 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.875 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:36:56.875 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.528303+0000", 2023-09-23T20:36:56.875 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.5732e-05 2023-09-23T20:36:56.875 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.875 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.875 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:56.875 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:50.524690+0000", 2023-09-23T20:36:56.875 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.99638690299999999 2023-09-23T20:36:56.875 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.876 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.876 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:36:56.876 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:50.524706+0000", 2023-09-23T20:36:56.876 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.6634e-05 2023-09-23T20:36:56.876 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:56.876 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:36:56.876 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:56.876 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.876 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.876 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:69152 7.3 7.f242f463 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e318)", 2023-09-23T20:36:56.876 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:32:49.520371+0000", 2023-09-23T20:36:56.876 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 247.22593843999999, 2023-09-23T20:36:56.876 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0043691610000001, 2023-09-23T20:36:56.877 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:36:56.877 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:36:56.877 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:36:56.877 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:36:56.877 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:36:56.877 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 69152 2023-09-23T20:36:56.877 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.877 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:36:56.877 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.877 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:36:56.877 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.520371+0000", 2023-09-23T20:36:56.877 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:36:56.877 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.878 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.878 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:36:56.878 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.520371+0000", 2023-09-23T20:36:56.878 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:36:56.878 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.878 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.878 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:36:56.879 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:36:56.879 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.520369+0000", 2023-09-23T20:36:56.879 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:36:56.879 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.880 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.880 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:36:56.880 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.520395+0000", 2023-09-23T20:36:56.880 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.5843999999999999e-05 2023-09-23T20:36:56.880 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.880 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.880 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:36:56.880 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.520408+0000", 2023-09-23T20:36:56.880 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3189e-05 2023-09-23T20:36:56.880 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.880 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.880 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:36:56.881 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.520417+0000", 2023-09-23T20:36:56.881 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.6899999999999998e-06 2023-09-23T20:36:56.881 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.881 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.881 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:56.881 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.529310+0000", 2023-09-23T20:36:56.881 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0088929609999999996 2023-09-23T20:36:56.881 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.881 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.881 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:36:56.881 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.529330+0000", 2023-09-23T20:36:56.882 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.9673e-05 2023-09-23T20:36:56.882 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.882 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.882 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:56.882 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:50.524730+0000", 2023-09-23T20:36:56.882 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.99540061999999996 2023-09-23T20:36:56.882 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.882 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.882 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:36:56.882 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:50.524740+0000", 2023-09-23T20:36:56.882 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.6139999999999998e-06 2023-09-23T20:36:56.883 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:56.883 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:36:56.883 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:56.883 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.883 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.883 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:69159 7.3 7.de3ad3c3 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e318)", 2023-09-23T20:36:56.883 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:32:49.534755+0000", 2023-09-23T20:36:56.883 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 247.21155370899999, 2023-09-23T20:36:56.883 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.99001591200000005, 2023-09-23T20:36:56.883 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:36:56.883 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:36:56.884 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:36:56.884 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:36:56.884 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:36:56.884 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 69159 2023-09-23T20:36:56.884 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.884 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:36:56.884 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.884 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:36:56.884 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.534755+0000", 2023-09-23T20:36:56.884 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:36:56.884 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.884 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.885 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:36:56.885 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.534755+0000", 2023-09-23T20:36:56.885 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:36:56.885 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.885 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.885 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:36:56.885 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.534754+0000", 2023-09-23T20:36:56.885 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:36:56.885 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.885 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.885 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:36:56.885 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.534782+0000", 2023-09-23T20:36:56.885 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.8348999999999999e-05 2023-09-23T20:36:56.886 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.886 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.886 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:36:56.886 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.534808+0000", 2023-09-23T20:36:56.886 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.5941000000000001e-05 2023-09-23T20:36:56.886 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.886 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.886 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:36:56.886 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.534818+0000", 2023-09-23T20:36:56.886 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0132999999999999e-05 2023-09-23T20:36:56.886 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.887 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.887 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:56.887 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.535076+0000", 2023-09-23T20:36:56.887 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00025796400000000001 2023-09-23T20:36:56.887 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.887 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.887 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:36:56.887 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.535090+0000", 2023-09-23T20:36:56.887 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3505e-05 2023-09-23T20:36:56.887 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.887 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.887 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:56.887 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:50.524762+0000", 2023-09-23T20:36:56.888 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.98967230399999995 2023-09-23T20:36:56.888 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.888 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.888 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:36:56.888 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:50.524771+0000", 2023-09-23T20:36:56.888 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.5070000000000006e-06 2023-09-23T20:36:56.888 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:56.888 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:36:56.888 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:56.888 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.888 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.889 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_repop(client.35700.0:110175 7.3 e334/320 7:ce0ca0fc:::benchmark_data_smithi018_308601_object15623:head v 334'6694, mlcod=308'587)", 2023-09-23T20:36:56.889 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:33:49.477493+0000", 2023-09-23T20:36:56.889 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 187.26881654799999, 2023-09-23T20:36:56.889 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.052026827999999997, 2023-09-23T20:36:56.889 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:36:56.889 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:36:56.889 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:36:56.889 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.889 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:36:56.889 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:33:49.477493+0000", 2023-09-23T20:36:56.889 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:36:56.889 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.890 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.890 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:36:56.890 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:33:49.477493+0000", 2023-09-23T20:36:56.890 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:36:56.890 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.890 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.890 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:36:56.890 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:33:49.477492+0000", 2023-09-23T20:36:56.890 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.999999 2023-09-23T20:36:56.890 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.890 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.891 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:36:56.891 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:33:49.477504+0000", 2023-09-23T20:36:56.891 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2262000000000001e-05 2023-09-23T20:36:56.891 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.891 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.891 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:36:56.891 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:33:49.477524+0000", 2023-09-23T20:36:56.891 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.0196000000000001e-05 2023-09-23T20:36:56.891 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.891 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.891 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:36:56.891 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:33:49.477533+0000", 2023-09-23T20:36:56.892 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.8310000000000007e-06 2023-09-23T20:36:56.892 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.892 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.892 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:56.892 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:33:49.477615+0000", 2023-09-23T20:36:56.892 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.2155000000000002e-05 2023-09-23T20:36:56.892 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.892 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.892 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:36:56.892 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:33:49.477644+0000", 2023-09-23T20:36:56.892 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.8843000000000002e-05 2023-09-23T20:36:56.892 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.893 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.893 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:36:56.893 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:33:49.529474+0000", 2023-09-23T20:36:56.893 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.051830214999999999 2023-09-23T20:36:56.893 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.893 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.893 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:36:56.893 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:33:49.529519+0000", 2023-09-23T20:36:56.893 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.5237000000000001e-05 2023-09-23T20:36:56.893 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:56.893 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:36:56.893 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:56.894 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.894 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.894 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_repop(client.35700.0:166342 7.a e358/319 7:506dc27b:::benchmark_data_smithi018_308601_object71790:head v 358'10397, mlcod=358'10395)", 2023-09-23T20:36:56.894 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:35:15.202112+0000", 2023-09-23T20:36:56.894 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 101.54419668200001, 2023-09-23T20:36:56.894 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.051485393999999997, 2023-09-23T20:36:56.894 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:36:56.894 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:36:56.894 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:36:56.894 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.894 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:36:56.895 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:15.202112+0000", 2023-09-23T20:36:56.895 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:36:56.895 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.895 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.895 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:36:56.895 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:15.202112+0000", 2023-09-23T20:36:56.895 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:36:56.895 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.895 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.895 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:36:56.895 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:15.202112+0000", 2023-09-23T20:36:56.895 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.999999 2023-09-23T20:36:56.896 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.896 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.896 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:36:56.896 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:15.202120+0000", 2023-09-23T20:36:56.896 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.0709999999999995e-06 2023-09-23T20:36:56.896 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.896 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.896 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:36:56.896 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:15.202129+0000", 2023-09-23T20:36:56.896 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.6260000000000007e-06 2023-09-23T20:36:56.896 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.896 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.897 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:36:56.897 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:15.202135+0000", 2023-09-23T20:36:56.897 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.84e-06 2023-09-23T20:36:56.897 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.897 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.897 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:56.897 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:15.202208+0000", 2023-09-23T20:36:56.897 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.2428999999999994e-05 2023-09-23T20:36:56.897 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.897 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.897 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:36:56.897 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:15.202230+0000", 2023-09-23T20:36:56.898 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.2433999999999999e-05 2023-09-23T20:36:56.898 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.898 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.898 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:36:56.898 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:15.253548+0000", 2023-09-23T20:36:56.898 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.051317507999999998 2023-09-23T20:36:56.898 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.898 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.898 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:36:56.898 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:15.253598+0000", 2023-09-23T20:36:56.898 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.0303999999999998e-05 2023-09-23T20:36:56.898 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:56.899 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:36:56.899 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:56.899 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.899 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.899 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_repop(client.35700.0:172339 7.6 e360/303 7:66f07c66:::benchmark_data_smithi018_308601_object77787:head v 360'10819, mlcod=360'10817)", 2023-09-23T20:36:56.899 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:35:24.116755+0000", 2023-09-23T20:36:56.899 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 92.629554592999995, 2023-09-23T20:36:56.899 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.051403868999999998, 2023-09-23T20:36:56.899 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:36:56.899 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:36:56.899 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:36:56.899 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.900 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:36:56.900 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:24.116755+0000", 2023-09-23T20:36:56.900 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:36:56.900 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.900 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.900 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:36:56.900 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:24.116755+0000", 2023-09-23T20:36:56.900 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:36:56.900 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.900 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.900 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:36:56.900 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:24.116754+0000", 2023-09-23T20:36:56.901 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.999999 2023-09-23T20:36:56.901 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.901 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.901 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:36:56.901 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:24.116762+0000", 2023-09-23T20:36:56.901 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.3529999999999995e-06 2023-09-23T20:36:56.901 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.901 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.901 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:36:56.901 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:24.116773+0000", 2023-09-23T20:36:56.901 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0992000000000001e-05 2023-09-23T20:36:56.901 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.902 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.902 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:36:56.902 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:24.116780+0000", 2023-09-23T20:36:56.902 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.0940000000000002e-06 2023-09-23T20:36:56.902 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.902 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.902 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:56.902 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:24.146276+0000", 2023-09-23T20:36:56.902 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.02949622 2023-09-23T20:36:56.902 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.902 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.903 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:36:56.903 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:24.146297+0000", 2023-09-23T20:36:56.903 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.0607000000000001e-05 2023-09-23T20:36:56.903 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.903 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.903 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:36:56.903 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:24.168117+0000", 2023-09-23T20:36:56.903 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.021819761999999999 2023-09-23T20:36:56.903 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.903 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.903 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:36:56.903 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:24.168158+0000", 2023-09-23T20:36:56.904 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.1797e-05 2023-09-23T20:36:56.904 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:56.904 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:36:56.904 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:56.904 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.904 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.904 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_repop(client.35700.0:186911 7.3 e364/349 7:c0a1e58d:::benchmark_data_smithi018_308601_object92359:head v 364'11338, mlcod=364'11335)", 2023-09-23T20:36:56.904 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:35:45.875072+0000", 2023-09-23T20:36:56.904 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 70.871237531000006, 2023-09-23T20:36:56.904 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.051084923999999997, 2023-09-23T20:36:56.904 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:36:56.904 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:36:56.905 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:36:56.905 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.905 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:36:56.905 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:45.875072+0000", 2023-09-23T20:36:56.905 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:36:56.905 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.905 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.905 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:36:56.905 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:45.875072+0000", 2023-09-23T20:36:56.905 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:36:56.905 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.905 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.906 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:36:56.906 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:45.875071+0000", 2023-09-23T20:36:56.906 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:36:56.906 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.906 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.906 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:36:56.906 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:45.875075+0000", 2023-09-23T20:36:56.906 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.7670000000000001e-06 2023-09-23T20:36:56.906 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.906 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.906 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:36:56.907 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:45.875081+0000", 2023-09-23T20:36:56.907 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.5389999999999998e-06 2023-09-23T20:36:56.907 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.907 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.907 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:36:56.907 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:45.875085+0000", 2023-09-23T20:36:56.907 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.1659999999999996e-06 2023-09-23T20:36:56.907 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.907 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.907 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:36:56.907 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:45.880268+0000", 2023-09-23T20:36:56.907 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0051831940000000003 2023-09-23T20:36:56.908 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.908 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.908 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:36:56.908 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:45.880290+0000", 2023-09-23T20:36:56.908 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.1880999999999999e-05 2023-09-23T20:36:56.908 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.908 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.908 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:36:56.908 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:45.926112+0000", 2023-09-23T20:36:56.908 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.045822073999999997 2023-09-23T20:36:56.908 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:36:56.908 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:36:56.909 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:36:56.909 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:45.926157+0000", 2023-09-23T20:36:56.909 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.4554000000000001e-05 2023-09-23T20:36:56.909 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:56.909 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:36:56.909 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:56.909 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:36:56.909 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:36:56.909 INFO:teuthology.orchestra.run.smithi018.stdout:} 2023-09-23T20:36:56.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:56 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1976501263' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:36:56.973 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:57.028 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 41 16 67903 67887 103.436 65.125 0.0281916 0.00966242 2023-09-23T20:36:57.213 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:36:57.223 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:57.224 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:36:57.260 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T20:36:57.324 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:36:57.389 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:57.475 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:36:57.482 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:57.482 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:36:57.577 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:57 smithi149 ceph-mon[170394]: pgmap v917: 119 pgs: 119 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2023-09-23T20:36:57.577 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:57 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2589121321' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:36:57.577 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:57 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:36:57.583 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:36:57.599 INFO:teuthology.orchestra.run.smithi018.stderr:noscrub is set 2023-09-23T20:36:57.655 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:57.717 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:36:57.724 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:57.725 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:36:57.825 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:36:57.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:57 smithi018 ceph-mon[179647]: pgmap v917: 119 pgs: 119 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2023-09-23T20:36:57.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:57 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2589121321' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:36:57.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:57 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:36:57.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:57 smithi018 ceph-mon[184144]: pgmap v917: 119 pgs: 119 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2023-09-23T20:36:57.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:57 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2589121321' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:36:57.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:57 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:36:57.900 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:58.014 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:36:58.023 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:58.024 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:36:58.031 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 42 16 69011 68995 102.617 69.25 0.0117969 0.00973955 2023-09-23T20:36:58.085 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:36:58.125 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:36:58.195 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:58.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:58 smithi018 ceph-mon[184144]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:36:58.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:58 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:36:58.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:58 smithi018 ceph-mon[184144]: osdmap e377: 8 total, 8 up, 8 in 2023-09-23T20:36:58.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:58 smithi018 ceph-mon[184144]: failed to encode map e377 with expected crc 2023-09-23T20:36:58.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:58 smithi018 ceph-mon[179647]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:36:58.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:58 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:36:58.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:58 smithi018 ceph-mon[179647]: osdmap e377: 8 total, 8 up, 8 in 2023-09-23T20:36:58.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:58 smithi018 ceph-mon[179647]: failed to encode map e377 with expected crc 2023-09-23T20:36:58.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:58 smithi149 ceph-mon[170394]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:36:58.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:58 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:36:58.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:58 smithi149 ceph-mon[170394]: osdmap e377: 8 total, 8 up, 8 in 2023-09-23T20:36:58.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:58 smithi149 ceph-mon[170394]: failed to encode map e377 with expected crc 2023-09-23T20:36:59.015 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:36:59.018 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:59.019 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:36:59.033 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 43 16 70179 70163 101.925 73 0.00785399 0.00980626 2023-09-23T20:36:59.119 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:36:59.178 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:59.247 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:36:59.252 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:59.253 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:36:59.353 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:36:59.423 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:59.493 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:36:59.500 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:59.500 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:36:59.539 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:36:59.601 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:36:59.601 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:59 smithi018 ceph-mon[184144]: pgmap v919: 119 pgs: 119 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 76 MiB/s wr, 1.21k op/s 2023-09-23T20:36:59.601 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:36:59 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:36:59.602 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:59 smithi018 ceph-mon[179647]: pgmap v919: 119 pgs: 119 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 76 MiB/s wr, 1.21k op/s 2023-09-23T20:36:59.602 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:36:59 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:36:59.644 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:59 smithi149 ceph-mon[170394]: pgmap v919: 119 pgs: 119 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 76 MiB/s wr, 1.21k op/s 2023-09-23T20:36:59.645 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:36:59 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:36:59.648 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:36:59.762 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:36:59.771 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:36:59.772 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:36:59.873 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:36:59.947 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:00.024 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:37:00.030 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:00.030 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:37:00.032 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 44 16 71297 71281 101.197 69.875 0.0070685 0.00987678 2023-09-23T20:37:00.082 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T20:37:00.131 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:37:00.256 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:00.317 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:37:00.326 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:00.326 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:37:00.356 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:37:00.400 INFO:teuthology.orchestra.run.smithi149.stdout:{ 2023-09-23T20:37:00.400 INFO:teuthology.orchestra.run.smithi149.stdout: "ops": [ 2023-09-23T20:37:00.400 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:00.400 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:71642 8.b 8:d7554365:::benchmark_data_smithi018_341490_object71641:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e377)", 2023-09-23T20:37:00.400 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:37:00.396046+0000", 2023-09-23T20:37:00.400 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 0.002092095, 2023-09-23T20:37:00.400 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.002128853, 2023-09-23T20:37:00.401 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:37:00.401 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:37:00.401 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:37:00.401 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:37:00.401 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:37:00.401 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 71642 2023-09-23T20:37:00.401 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:00.401 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:37:00.401 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:00.401 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:37:00.401 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:00.396046+0000", 2023-09-23T20:37:00.401 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:37:00.402 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:00.402 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:00.402 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:37:00.402 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:00.396046+0000", 2023-09-23T20:37:00.402 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:37:00.402 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:00.402 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:00.402 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:37:00.402 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:00.396045+0000", 2023-09-23T20:37:00.402 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:37:00.402 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:00.402 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:00.403 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:37:00.403 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:00.396170+0000", 2023-09-23T20:37:00.403 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00012467399999999999 2023-09-23T20:37:00.403 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:00.403 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:00.403 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:37:00.403 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:00.396191+0000", 2023-09-23T20:37:00.403 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.1421e-05 2023-09-23T20:37:00.403 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:00.403 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:00.403 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:37:00.403 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:00.396211+0000", 2023-09-23T20:37:00.404 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.0256999999999999e-05 2023-09-23T20:37:00.404 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:00.404 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:00.404 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:37:00.404 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:00.396279+0000", 2023-09-23T20:37:00.404 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.7582000000000004e-05 2023-09-23T20:37:00.404 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:00.404 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:00.404 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:37:00.404 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:00.396445+0000", 2023-09-23T20:37:00.404 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00016596300000000001 2023-09-23T20:37:00.404 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:00.404 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:00.405 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:37:00.405 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:00.396585+0000", 2023-09-23T20:37:00.405 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00013969300000000001 2023-09-23T20:37:00.405 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:37:00.405 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:37:00.405 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:37:00.405 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:00.405 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:00.405 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:71644 8.b 8.a0963f9b (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e377)", 2023-09-23T20:37:00.405 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:37:00.396241+0000", 2023-09-23T20:37:00.405 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 0.0018973799999999999, 2023-09-23T20:37:00.405 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0020136429999999999, 2023-09-23T20:37:00.406 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:37:00.406 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "queued for pg", 2023-09-23T20:37:00.406 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:37:00.406 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:37:00.406 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:37:00.406 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 71644 2023-09-23T20:37:00.406 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:00.406 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:37:00.406 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:00.406 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:37:00.406 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:00.396241+0000", 2023-09-23T20:37:00.407 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:37:00.407 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:00.407 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:00.407 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:37:00.407 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:00.396241+0000", 2023-09-23T20:37:00.407 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:37:00.407 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:00.407 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:00.407 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:37:00.407 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:00.396240+0000", 2023-09-23T20:37:00.407 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:37:00.407 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:00.407 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:00.408 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:37:00.408 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:00.396287+0000", 2023-09-23T20:37:00.408 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.7215000000000001e-05 2023-09-23T20:37:00.408 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:00.408 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:00.408 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:37:00.408 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:00.396299+0000", 2023-09-23T20:37:00.408 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1464999999999999e-05 2023-09-23T20:37:00.408 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:00.408 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:00.408 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:37:00.408 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:00.396305+0000", 2023-09-23T20:37:00.409 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.6559999999999997e-06 2023-09-23T20:37:00.409 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:37:00.409 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:37:00.409 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:37:00.409 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:00.409 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:00.409 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:71645 8.b 8.b42b0c0b (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e377)", 2023-09-23T20:37:00.409 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:37:00.396354+0000", 2023-09-23T20:37:00.409 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 0.0017842789999999999, 2023-09-23T20:37:00.409 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.001953785, 2023-09-23T20:37:00.409 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:37:00.409 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "queued for pg", 2023-09-23T20:37:00.410 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:37:00.410 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:37:00.410 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:37:00.410 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 71645 2023-09-23T20:37:00.410 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:00.410 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:37:00.410 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:00.410 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:37:00.410 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:00.396354+0000", 2023-09-23T20:37:00.410 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:37:00.410 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:00.410 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:00.411 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:37:00.411 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:00.396354+0000", 2023-09-23T20:37:00.411 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:37:00.411 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:00.411 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:00.411 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:37:00.411 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:00.396353+0000", 2023-09-23T20:37:00.411 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:37:00.411 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:00.411 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:00.411 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:37:00.411 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:00.396445+0000", 2023-09-23T20:37:00.411 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.1677999999999999e-05 2023-09-23T20:37:00.412 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:00.412 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:00.412 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:37:00.412 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:00.396457+0000", 2023-09-23T20:37:00.412 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1501999999999999e-05 2023-09-23T20:37:00.412 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:00.412 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:00.412 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:37:00.412 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:00.396462+0000", 2023-09-23T20:37:00.412 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.784e-06 2023-09-23T20:37:00.412 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:37:00.412 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:37:00.413 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:37:00.413 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:37:00.413 INFO:teuthology.orchestra.run.smithi149.stdout: ], 2023-09-23T20:37:00.413 INFO:teuthology.orchestra.run.smithi149.stdout: "num_ops": 3 2023-09-23T20:37:00.413 INFO:teuthology.orchestra.run.smithi149.stdout:} 2023-09-23T20:37:00.427 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:37:00.497 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:00.659 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:37:00.672 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:00.673 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:37:00.741 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 5, 7, 3, 2, 6, 4] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T20:37:00.741 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:37:00.741 INFO:tasks.thrashosds.thrasher:inject_pause on osd.6 2023-09-23T20:37:00.741 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T20:37:00.741 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T20:37:00.741 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 config set filestore_inject_stall 3 2023-09-23T20:37:00.773 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:37:00.845 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T20:37:00.872 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:00.945 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:37:00.958 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:00.959 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:37:01.032 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 45 16 72275 72259 100.307 61.125 0.0106572 0.0099634 2023-09-23T20:37:01.061 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:37:01.201 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:01.263 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:37:01.272 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:01.273 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:37:01.290 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:37:01.374 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:37:01.416 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:01.500 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:37:01.503 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:37:01.513 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:01.513 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:37:01.614 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:37:01.692 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:01.726 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:01 smithi018 ceph-mon[184144]: pgmap v920: 119 pgs: 119 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2023-09-23T20:37:01.727 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:01 smithi018 ceph-mon[179647]: pgmap v920: 119 pgs: 119 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2023-09-23T20:37:01.759 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:37:01.769 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:01.770 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:37:01.788 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:37:01.871 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:37:01.940 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:01.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:01 smithi149 ceph-mon[170394]: pgmap v920: 119 pgs: 119 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2023-09-23T20:37:02.032 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 46 16 73250 73234 99.451 60.9375 0.00901492 0.0100501 2023-09-23T20:37:02.572 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:37:02.577 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:02.578 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:37:02.679 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:37:02.736 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:03.032 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 47 16 74291 74275 98.7195 65.0625 0.015856 0.0101256 2023-09-23T20:37:03.102 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:37:03.109 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:03.110 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:37:03.210 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:37:03.271 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:03.402 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:37:03.406 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:03.407 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:37:03.436 INFO:teuthology.orchestra.run.smithi149.stdout:{ 2023-09-23T20:37:03.436 INFO:teuthology.orchestra.run.smithi149.stdout: "ops": [], 2023-09-23T20:37:03.436 INFO:teuthology.orchestra.run.smithi149.stdout: "complaint_time": 30, 2023-09-23T20:37:03.436 INFO:teuthology.orchestra.run.smithi149.stdout: "num_blocked_ops": 0 2023-09-23T20:37:03.437 INFO:teuthology.orchestra.run.smithi149.stdout:} 2023-09-23T20:37:03.508 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:37:03.610 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:03.611 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:37:03.648 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:03 smithi149 ceph-mon[170394]: pgmap v921: 119 pgs: 119 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-09-23T20:37:03.648 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:03 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2026376441' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:37:03.651 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:37:03.690 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:37:03.709 INFO:teuthology.orchestra.run.smithi149.stdout:{ 2023-09-23T20:37:03.710 INFO:teuthology.orchestra.run.smithi149.stdout: "success": "" 2023-09-23T20:37:03.710 INFO:teuthology.orchestra.run.smithi149.stdout:} 2023-09-23T20:37:03.774 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:37:03.827 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:37:03.870 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:37:03.904 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:37:03.949 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:37:03.958 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:37:03.958 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:37:03.958 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:37:03.958 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:37:03.958 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:37:03.959 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:37:03.959 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:37:03.959 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:37:03.987 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:03 smithi018 ceph-mon[179647]: pgmap v921: 119 pgs: 119 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-09-23T20:37:03.987 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:03 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2026376441' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:37:03.987 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:03 smithi018 ceph-mon[184144]: pgmap v921: 119 pgs: 119 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-09-23T20:37:03.987 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:03 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2026376441' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:37:04.009 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T20:37:04.032 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 48 16 75288 75272 97.9613 62.3125 0.012203 0.0102035 2023-09-23T20:37:04.095 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:37:04.107 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:04.108 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:37:04.208 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:37:04.281 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:04.811 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:37:04.958 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:37:04.963 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:04.964 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:37:05.033 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 49 16 76206 76190 97.1332 57.375 0.0178637 0.0102893 2023-09-23T20:37:05.065 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:37:05.143 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:05.320 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:37:04 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:37:04] "GET /metrics HTTP/1.1" 200 36501 "" "Prometheus/2.43.0" 2023-09-23T20:37:05.663 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:05 smithi149 ceph-mon[170394]: pgmap v922: 119 pgs: 119 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-09-23T20:37:06.033 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 50 16 77281 77265 96.5344 67.1875 0.0191829 0.0103546 2023-09-23T20:37:06.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:05 smithi018 ceph-mon[184144]: pgmap v922: 119 pgs: 119 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-09-23T20:37:06.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:05 smithi018 ceph-mon[179647]: pgmap v922: 119 pgs: 119 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-09-23T20:37:06.214 INFO:teuthology.orchestra.run.smithi149.stdout:ERROR: (22) Invalid argument 2023-09-23T20:37:06.214 INFO:teuthology.orchestra.run.smithi149.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:37:06.315 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:37:06.319 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:06.320 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:37:06.421 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:37:06.478 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:06.585 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:37:06.590 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:06.590 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:37:06.667 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T20:37:06.691 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:37:06.769 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:06.827 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:37:06.840 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:06.840 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:37:06.941 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:37:07.019 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:07.033 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 51 16 78281 78265 95.8672 62.5 0.0264761 0.0104261 2023-09-23T20:37:07.061 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:37:07.064 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:07.065 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:37:07.166 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:37:07.242 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:07.292 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:37:07.301 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:07.301 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:37:07.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:07 smithi018 ceph-mon[179647]: pgmap v923: 119 pgs: 119 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-09-23T20:37:07.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:07 smithi018 ceph-mon[184144]: pgmap v923: 119 pgs: 119 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-09-23T20:37:07.401 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:37:07.446 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:37:07.478 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:07.566 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:37:07.571 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:07.572 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:37:07.673 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:37:07.675 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:07 smithi149 ceph-mon[170394]: pgmap v923: 119 pgs: 119 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-09-23T20:37:07.720 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:07.787 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:37:07.793 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:07.794 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:37:07.895 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:37:07.943 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:08.033 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 52 16 79196 79180 95.1235 57.1875 0.0171353 0.0105078 2023-09-23T20:37:08.281 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:37:08.292 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:08.292 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:37:08.329 INFO:teuthology.orchestra.run.smithi018.stderr:nodeep-scrub is set 2023-09-23T20:37:08.393 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:37:08.451 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:37:08.465 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:08.626 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:08 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2295601091' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:37:08.630 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:08 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2295601091' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:37:08.638 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:08 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2295601091' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:37:08.729 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:37:08.739 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:08.740 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:37:08.840 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:37:08.918 INFO:teuthology.orchestra.run.smithi149.stdout:ERROR: (22) Invalid argument 2023-09-23T20:37:08.918 INFO:teuthology.orchestra.run.smithi149.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:37:08.919 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:09.033 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 53 16 80127 80111 94.4268 58.1875 0.0166359 0.0105843 2023-09-23T20:37:09.269 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:37:09.274 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:09.274 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:37:09.283 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T20:37:09.291 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 5, 7, 3, 2, 6, 4] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T20:37:09.292 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:37:09.292 INFO:tasks.thrashosds.thrasher:Reweighting osd 5 to 0.2886195609608352 2023-09-23T20:37:09.292 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd reweight 5 0.2886195609608352 2023-09-23T20:37:09.375 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:37:09.376 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:37:09.420 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:09.495 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:37:09.502 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:09.503 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:37:09.604 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:37:09.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:09 smithi018 ceph-mon[184144]: pgmap v924: 119 pgs: 119 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-09-23T20:37:09.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:09 smithi018 ceph-mon[184144]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:37:09.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:09 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2295601091' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:37:09.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:09 smithi018 ceph-mon[184144]: osdmap e378: 8 total, 8 up, 8 in 2023-09-23T20:37:09.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:09 smithi018 ceph-mon[184144]: failed to encode map e378 with expected crc 2023-09-23T20:37:09.647 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:09 smithi018 ceph-mon[179647]: pgmap v924: 119 pgs: 119 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-09-23T20:37:09.648 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:09 smithi018 ceph-mon[179647]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:37:09.648 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:09 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2295601091' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:37:09.648 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:09 smithi018 ceph-mon[179647]: osdmap e378: 8 total, 8 up, 8 in 2023-09-23T20:37:09.648 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:09 smithi018 ceph-mon[179647]: failed to encode map e378 with expected crc 2023-09-23T20:37:09.697 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:09.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:09 smithi149 ceph-mon[170394]: pgmap v924: 119 pgs: 119 active+clean; 4.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2023-09-23T20:37:09.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:09 smithi149 ceph-mon[170394]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:37:09.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:09 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2295601091' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:37:09.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:09 smithi149 ceph-mon[170394]: osdmap e378: 8 total, 8 up, 8 in 2023-09-23T20:37:09.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:09 smithi149 ceph-mon[170394]: failed to encode map e378 with expected crc 2023-09-23T20:37:09.781 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:37:09.793 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:09.794 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:37:09.894 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:37:09.976 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:10.033 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 54 16 80985 80969 93.6714 53.625 0.0112886 0.0106693 2023-09-23T20:37:10.063 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:37:10.418 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:37:10.422 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:10.422 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:37:10.422 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:37:10.523 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:37:10.587 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:10.765 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T20:37:10.794 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:37:10.799 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:10.800 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:37:10.901 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:37:10.986 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:11.033 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 55 16 81923 81907 93.0343 58.625 0.0344616 0.0107425 2023-09-23T20:37:11.069 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:37:11.076 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:11.077 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:37:11.177 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:37:11.268 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:11.328 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:37:11.381 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:37:11.414 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:37:11.424 INFO:teuthology.orchestra.run.smithi149.stdout:ERROR: (22) Invalid argument 2023-09-23T20:37:11.424 INFO:teuthology.orchestra.run.smithi149.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:37:11.444 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:11 smithi149 ceph-mon[170394]: pgmap v926: 119 pgs: 119 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-09-23T20:37:11.444 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:11 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1408850602' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:37:11.466 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:37:11.504 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:37:11.509 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:11.510 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:37:11.514 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:11 smithi018 ceph-mon[184144]: pgmap v926: 119 pgs: 119 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-09-23T20:37:11.514 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:11 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1408850602' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:37:11.515 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:37:11.518 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:11 smithi018 ceph-mon[179647]: pgmap v926: 119 pgs: 119 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-09-23T20:37:11.519 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:11 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1408850602' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:37:11.572 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:37:11.611 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:37:11.613 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:37:11.665 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:11.669 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:37:11.675 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:37:11.675 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:37:11.675 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:37:11.675 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:37:11.675 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:37:11.675 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:37:11.676 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:37:11.676 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:37:11.794 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:37:11.817 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:37:11.821 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:11.822 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:37:11.871 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T20:37:11.923 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:37:11.969 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:12.033 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 56 16 82809 82793 92.362 55.375 0.0132926 0.0108211 2023-09-23T20:37:12.062 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:37:12.072 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:12.073 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:37:12.173 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:37:12.270 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:12.333 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:37:12.338 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:12.339 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:37:12.439 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:37:12.520 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:12.575 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:37:12.579 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:12.579 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:37:12.606 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:37:12.680 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:37:12.734 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:12.839 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:37:12.844 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:12.845 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:37:12.946 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:37:13.014 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:13.033 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 57 16 83782 83766 91.8086 60.8125 0.0110123 0.0108871 2023-09-23T20:37:13.093 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:37:13.102 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:13.103 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:37:13.204 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:37:13.256 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:13.338 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:13 smithi018 ceph-mon[179647]: pgmap v927: 119 pgs: 119 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 61 MiB/s wr, 982 op/s 2023-09-23T20:37:13.339 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:13 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2008845680' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 0.2886195609608352}]: dispatch 2023-09-23T20:37:13.365 INFO:teuthology.orchestra.run.smithi018.stderr:reweighted osd.5 to 0.28862 (49e2) 2023-09-23T20:37:13.438 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:37:13.442 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:13.442 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:37:13.543 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:37:13.574 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:13.624 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:13 smithi018 ceph-mon[184144]: pgmap v927: 119 pgs: 119 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 61 MiB/s wr, 982 op/s 2023-09-23T20:37:13.625 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:13 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2008845680' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 0.2886195609608352}]: dispatch 2023-09-23T20:37:13.715 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:13 smithi149 ceph-mon[170394]: pgmap v927: 119 pgs: 119 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 61 MiB/s wr, 982 op/s 2023-09-23T20:37:13.715 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:13 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2008845680' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 0.2886195609608352}]: dispatch 2023-09-23T20:37:13.796 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:37:13.799 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:13.802 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:37:13.903 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:37:13.914 INFO:teuthology.orchestra.run.smithi149.stdout:{ 2023-09-23T20:37:13.915 INFO:teuthology.orchestra.run.smithi149.stdout: "size": 20, 2023-09-23T20:37:13.915 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 600, 2023-09-23T20:37:13.915 INFO:teuthology.orchestra.run.smithi149.stdout: "ops": [ 2023-09-23T20:37:13.915 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.915 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.45161.0:68483 6.a 6:5eb5ac9f:::benchmark_data_smithi018_270558_object68482:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e230)", 2023-09-23T20:37:13.915 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:28:04.072353+0000", 2023-09-23T20:37:13.916 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 549.83922937800003, 2023-09-23T20:37:13.916 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.1210222989999998, 2023-09-23T20:37:13.916 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:37:13.916 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:37:13.916 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:37:13.916 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.45161", 2023-09-23T20:37:13.916 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3795614987", 2023-09-23T20:37:13.917 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 68483 2023-09-23T20:37:13.917 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.917 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:37:13.917 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.917 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:37:13.917 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.072353+0000", 2023-09-23T20:37:13.917 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:37:13.917 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.917 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.917 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:37:13.917 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.072353+0000", 2023-09-23T20:37:13.918 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:37:13.918 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.918 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.918 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:37:13.918 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.072352+0000", 2023-09-23T20:37:13.918 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:37:13.918 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.918 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.918 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:37:13.918 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.072388+0000", 2023-09-23T20:37:13.918 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.6284e-05 2023-09-23T20:37:13.919 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.919 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.919 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:37:13.919 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.072401+0000", 2023-09-23T20:37:13.919 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.2706e-05 2023-09-23T20:37:13.919 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.919 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.919 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:37:13.919 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.072407+0000", 2023-09-23T20:37:13.919 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.7679999999999997e-06 2023-09-23T20:37:13.919 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.919 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.920 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:37:13.920 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.072536+0000", 2023-09-23T20:37:13.920 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000129694 2023-09-23T20:37:13.920 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.920 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.920 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:37:13.920 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.072547+0000", 2023-09-23T20:37:13.920 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0692e-05 2023-09-23T20:37:13.920 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.920 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.920 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:37:13.920 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:05.030894+0000", 2023-09-23T20:37:13.921 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.95834700699999997 2023-09-23T20:37:13.921 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.921 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.921 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:37:13.921 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:05.030907+0000", 2023-09-23T20:37:13.921 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.3186e-05 2023-09-23T20:37:13.921 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.921 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.921 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:37:13.921 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.177110+0000", 2023-09-23T20:37:13.921 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1462023859999999 2023-09-23T20:37:13.921 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.922 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.922 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:37:13.922 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.177301+0000", 2023-09-23T20:37:13.922 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00019089000000000001 2023-09-23T20:37:13.922 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.922 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.922 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,2", 2023-09-23T20:37:13.922 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.177511+0000", 2023-09-23T20:37:13.922 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00021060299999999999 2023-09-23T20:37:13.922 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.922 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.922 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:37:13.923 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.187865+0000", 2023-09-23T20:37:13.923 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.010353431 2023-09-23T20:37:13.923 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.923 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.923 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:37:13.923 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.188200+0000", 2023-09-23T20:37:13.923 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00033555299999999999 2023-09-23T20:37:13.923 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.923 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.923 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:37:13.923 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.189703+0000", 2023-09-23T20:37:13.923 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.001502499 2023-09-23T20:37:13.924 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.924 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.924 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:37:13.924 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.189754+0000", 2023-09-23T20:37:13.924 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.1598999999999998e-05 2023-09-23T20:37:13.924 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.924 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.924 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:37:13.924 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.193376+0000", 2023-09-23T20:37:13.924 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0036214429999999998 2023-09-23T20:37:13.924 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:37:13.924 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:37:13.925 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:37:13.925 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.925 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.925 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.45161.0:68495 6.a 6:5fb49a37:::benchmark_data_smithi018_270558_object68494:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e230)", 2023-09-23T20:37:13.925 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:28:04.084817+0000", 2023-09-23T20:37:13.925 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 549.826765368, 2023-09-23T20:37:13.925 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.110988222, 2023-09-23T20:37:13.925 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:37:13.925 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:37:13.926 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:37:13.926 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.45161", 2023-09-23T20:37:13.926 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3795614987", 2023-09-23T20:37:13.926 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 68495 2023-09-23T20:37:13.926 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.926 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:37:13.926 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.926 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:37:13.926 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.084817+0000", 2023-09-23T20:37:13.926 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:37:13.926 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.927 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.927 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:37:13.927 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.084817+0000", 2023-09-23T20:37:13.927 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:37:13.927 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.927 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.927 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:37:13.927 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.084815+0000", 2023-09-23T20:37:13.927 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999981 2023-09-23T20:37:13.927 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.927 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.927 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:37:13.928 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.084859+0000", 2023-09-23T20:37:13.928 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.3095e-05 2023-09-23T20:37:13.928 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.928 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.928 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:37:13.928 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.084875+0000", 2023-09-23T20:37:13.928 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.6739e-05 2023-09-23T20:37:13.928 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.928 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.928 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:37:13.928 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.084883+0000", 2023-09-23T20:37:13.928 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.0010000000000001e-06 2023-09-23T20:37:13.929 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.929 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.929 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:37:13.929 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.084956+0000", 2023-09-23T20:37:13.929 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.2482000000000002e-05 2023-09-23T20:37:13.929 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.929 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.929 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:37:13.929 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.084970+0000", 2023-09-23T20:37:13.929 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.4154999999999999e-05 2023-09-23T20:37:13.929 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.929 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.930 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:37:13.930 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:05.030931+0000", 2023-09-23T20:37:13.930 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.94596105600000002 2023-09-23T20:37:13.930 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.930 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.930 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:37:13.930 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:05.030940+0000", 2023-09-23T20:37:13.930 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.6419999999999996e-06 2023-09-23T20:37:13.930 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.930 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.930 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:37:13.930 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.180341+0000", 2023-09-23T20:37:13.931 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1494014969999999 2023-09-23T20:37:13.931 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.931 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.931 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:37:13.931 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.180523+0000", 2023-09-23T20:37:13.931 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00018138299999999999 2023-09-23T20:37:13.931 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.931 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.931 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,2", 2023-09-23T20:37:13.931 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.180726+0000", 2023-09-23T20:37:13.931 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00020395100000000001 2023-09-23T20:37:13.932 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.932 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.932 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:37:13.932 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.188188+0000", 2023-09-23T20:37:13.932 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0074612180000000004 2023-09-23T20:37:13.932 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.932 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.932 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:37:13.932 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.188341+0000", 2023-09-23T20:37:13.932 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000153776 2023-09-23T20:37:13.932 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.933 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.933 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:37:13.933 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.193438+0000", 2023-09-23T20:37:13.933 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0050964640000000002 2023-09-23T20:37:13.933 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.933 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.933 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:37:13.933 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.193481+0000", 2023-09-23T20:37:13.933 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.2864999999999997e-05 2023-09-23T20:37:13.933 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.933 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.934 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:37:13.934 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.195806+0000", 2023-09-23T20:37:13.934 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0023247419999999999 2023-09-23T20:37:13.934 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:37:13.934 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:37:13.934 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:37:13.934 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.934 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.934 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.45161.0:136033 6.7 6:e12c7d04:::benchmark_data_smithi018_270558_object31294:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e257)", 2023-09-23T20:37:13.934 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:29:27.327366+0000", 2023-09-23T20:37:13.934 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 466.58421706399997, 2023-09-23T20:37:13.935 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.0930456, 2023-09-23T20:37:13.935 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:37:13.935 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:37:13.935 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:37:13.935 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.45161", 2023-09-23T20:37:13.935 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3795614987", 2023-09-23T20:37:13.935 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 136033 2023-09-23T20:37:13.935 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.935 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:37:13.935 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.935 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:37:13.936 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:27.327366+0000", 2023-09-23T20:37:13.936 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:37:13.936 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.936 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.936 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:37:13.936 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:27.327366+0000", 2023-09-23T20:37:13.936 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:37:13.936 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.936 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.936 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:37:13.936 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:27.327363+0000", 2023-09-23T20:37:13.937 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999976 2023-09-23T20:37:13.937 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.937 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.937 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:37:13.937 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:27.327372+0000", 2023-09-23T20:37:13.937 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.6750000000000008e-06 2023-09-23T20:37:13.937 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.938 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.938 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:37:13.938 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:27.327386+0000", 2023-09-23T20:37:13.938 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.4151999999999999e-05 2023-09-23T20:37:13.938 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.938 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.938 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:37:13.938 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:27.327393+0000", 2023-09-23T20:37:13.938 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.9210000000000003e-06 2023-09-23T20:37:13.938 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.938 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.938 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:37:13.939 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:27.327440+0000", 2023-09-23T20:37:13.939 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.7227e-05 2023-09-23T20:37:13.939 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.939 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.939 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:37:13.939 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:27.327599+0000", 2023-09-23T20:37:13.939 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00015890000000000001 2023-09-23T20:37:13.939 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.939 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.940 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:37:13.940 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:27.328068+0000", 2023-09-23T20:37:13.940 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00046883200000000003 2023-09-23T20:37:13.940 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.940 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.940 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:37:13.940 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:27.330709+0000", 2023-09-23T20:37:13.940 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0026409200000000002 2023-09-23T20:37:13.940 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.940 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.941 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:37:13.941 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:27.344609+0000", 2023-09-23T20:37:13.941 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.013900341 2023-09-23T20:37:13.941 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.941 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.941 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:37:13.941 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:27.369483+0000", 2023-09-23T20:37:13.941 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.024873704 2023-09-23T20:37:13.941 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.942 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.942 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:37:13.942 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:27.369498+0000", 2023-09-23T20:37:13.942 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.4545999999999999e-05 2023-09-23T20:37:13.942 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.942 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.942 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:37:13.942 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:28.355543+0000", 2023-09-23T20:37:13.942 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.98604489900000003 2023-09-23T20:37:13.942 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.943 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.943 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:37:13.943 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:28.355555+0000", 2023-09-23T20:37:13.943 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.2612000000000001e-05 2023-09-23T20:37:13.943 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.943 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.943 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:37:13.943 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:29.391776+0000", 2023-09-23T20:37:13.944 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0362208989999999 2023-09-23T20:37:13.944 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.944 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.944 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for missing object", 2023-09-23T20:37:13.944 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:29.391962+0000", 2023-09-23T20:37:13.944 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00018626499999999999 2023-09-23T20:37:13.944 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.944 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.944 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:37:13.944 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:29.420342+0000", 2023-09-23T20:37:13.944 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.028379945 2023-09-23T20:37:13.945 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.945 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.945 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:37:13.945 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:29.420411+0000", 2023-09-23T20:37:13.945 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.8942000000000005e-05 2023-09-23T20:37:13.945 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:37:13.945 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:37:13.945 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:37:13.946 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.946 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.946 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.45161.0:136047 6.7 6:e530c251:::benchmark_data_smithi018_270558_object31308:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e257)", 2023-09-23T20:37:13.946 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:29:27.363510+0000", 2023-09-23T20:37:13.946 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 466.54807260199999, 2023-09-23T20:37:13.946 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.0564319210000002, 2023-09-23T20:37:13.946 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:37:13.946 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:37:13.946 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:37:13.947 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.45161", 2023-09-23T20:37:13.947 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3795614987", 2023-09-23T20:37:13.947 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 136047 2023-09-23T20:37:13.947 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.947 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:37:13.947 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.947 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:37:13.947 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:27.363510+0000", 2023-09-23T20:37:13.947 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:37:13.948 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.948 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.948 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:37:13.948 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:27.363510+0000", 2023-09-23T20:37:13.948 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:37:13.948 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.948 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.948 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:37:13.949 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:27.363508+0000", 2023-09-23T20:37:13.949 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999981 2023-09-23T20:37:13.949 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.949 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.949 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:37:13.949 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:27.363514+0000", 2023-09-23T20:37:13.949 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.3580000000000002e-06 2023-09-23T20:37:13.949 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.950 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.950 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:37:13.950 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:27.363524+0000", 2023-09-23T20:37:13.950 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0543000000000001e-05 2023-09-23T20:37:13.950 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.950 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.950 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:37:13.950 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:27.363535+0000", 2023-09-23T20:37:13.950 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.044e-05 2023-09-23T20:37:13.950 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.950 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.951 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:37:13.951 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:27.369525+0000", 2023-09-23T20:37:13.951 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0059905039999999998 2023-09-23T20:37:13.951 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.951 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.951 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:37:13.951 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:27.369532+0000", 2023-09-23T20:37:13.951 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.6619999999999999e-06 2023-09-23T20:37:13.951 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.951 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.951 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:37:13.952 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:28.355582+0000", 2023-09-23T20:37:13.952 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.98605059900000003 2023-09-23T20:37:13.952 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.952 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.952 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:37:13.952 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:28.355589+0000", 2023-09-23T20:37:13.952 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.8929999999999999e-06 2023-09-23T20:37:13.952 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.952 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.952 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:37:13.952 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:29.392324+0000", 2023-09-23T20:37:13.953 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0367343600000001 2023-09-23T20:37:13.953 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.953 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.953 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:37:13.953 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:29.392475+0000", 2023-09-23T20:37:13.953 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00015086799999999999 2023-09-23T20:37:13.953 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.953 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.953 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:37:13.953 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:29.393040+0000", 2023-09-23T20:37:13.953 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00056566799999999997 2023-09-23T20:37:13.954 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.954 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.954 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:37:13.954 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:29.399625+0000", 2023-09-23T20:37:13.954 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0065850880000000002 2023-09-23T20:37:13.954 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.954 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.954 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:37:13.954 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:29.399866+0000", 2023-09-23T20:37:13.954 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00024026000000000001 2023-09-23T20:37:13.954 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.954 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.955 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:37:13.955 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:29.419769+0000", 2023-09-23T20:37:13.955 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.019903745 2023-09-23T20:37:13.955 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.955 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.955 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:37:13.955 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:29.419830+0000", 2023-09-23T20:37:13.955 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.0566000000000003e-05 2023-09-23T20:37:13.955 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.955 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.955 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:37:13.956 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:29.419942+0000", 2023-09-23T20:37:13.956 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000112093 2023-09-23T20:37:13.956 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:37:13.956 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:37:13.956 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:37:13.956 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.956 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.956 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.45161.0:136059 6.17 6:ed3cc3ed:::benchmark_data_smithi018_270558_object31320:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e258)", 2023-09-23T20:37:13.956 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:29:27.380270+0000", 2023-09-23T20:37:13.956 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 466.53131291199998, 2023-09-23T20:37:13.956 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.019361118, 2023-09-23T20:37:13.957 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:37:13.957 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:37:13.957 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:37:13.957 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.45161", 2023-09-23T20:37:13.957 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3795614987", 2023-09-23T20:37:13.957 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 136059 2023-09-23T20:37:13.957 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.957 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:37:13.957 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.957 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:37:13.957 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:27.380270+0000", 2023-09-23T20:37:13.958 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:37:13.958 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.958 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.958 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:37:13.958 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:27.380270+0000", 2023-09-23T20:37:13.958 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:37:13.958 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.958 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.958 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:37:13.958 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:27.380269+0000", 2023-09-23T20:37:13.958 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:37:13.959 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.959 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.959 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:37:13.959 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:27.380274+0000", 2023-09-23T20:37:13.959 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.169e-06 2023-09-23T20:37:13.959 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.959 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.959 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:37:13.959 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:27.380284+0000", 2023-09-23T20:37:13.959 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.7580000000000004e-06 2023-09-23T20:37:13.959 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.960 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.960 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:37:13.960 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:27.380290+0000", 2023-09-23T20:37:13.960 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.8370000000000002e-06 2023-09-23T20:37:13.960 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.960 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.960 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:37:13.960 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:27.380359+0000", 2023-09-23T20:37:13.960 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.9441000000000001e-05 2023-09-23T20:37:13.960 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.960 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.961 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:37:13.961 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:27.380368+0000", 2023-09-23T20:37:13.961 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.5879999999999998e-06 2023-09-23T20:37:13.961 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.961 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.961 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:37:13.961 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:28.355613+0000", 2023-09-23T20:37:13.961 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.97524513400000001 2023-09-23T20:37:13.961 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.961 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.961 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:37:13.962 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:28.355621+0000", 2023-09-23T20:37:13.962 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.0209999999999999e-06 2023-09-23T20:37:13.962 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.962 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.962 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:37:13.962 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:29.376357+0000", 2023-09-23T20:37:13.962 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.020736445 2023-09-23T20:37:13.962 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.962 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.962 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:37:13.962 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:29.376497+0000", 2023-09-23T20:37:13.962 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00014008400000000001 2023-09-23T20:37:13.963 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.963 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.963 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:37:13.963 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:29.376961+0000", 2023-09-23T20:37:13.963 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00046314499999999998 2023-09-23T20:37:13.963 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.963 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.963 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:37:13.963 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:29.392111+0000", 2023-09-23T20:37:13.963 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.015150788 2023-09-23T20:37:13.963 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.964 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.964 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:37:13.964 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:29.392146+0000", 2023-09-23T20:37:13.964 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.4604999999999998e-05 2023-09-23T20:37:13.964 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.964 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.964 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:37:13.964 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:29.399462+0000", 2023-09-23T20:37:13.964 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0073164119999999996 2023-09-23T20:37:13.964 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.964 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.964 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:37:13.965 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:29.399517+0000", 2023-09-23T20:37:13.965 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.4987000000000001e-05 2023-09-23T20:37:13.965 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.965 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.965 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:37:13.965 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:29.399631+0000", 2023-09-23T20:37:13.965 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00011351 2023-09-23T20:37:13.965 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:37:13.965 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:37:13.965 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:37:13.965 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.965 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.966 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.45161.0:136060 6.7 6:e35290e5:::benchmark_data_smithi018_270558_object31321:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e258)", 2023-09-23T20:37:13.966 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:29:27.380303+0000", 2023-09-23T20:37:13.966 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 466.531279729, 2023-09-23T20:37:13.966 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.0397914629999998, 2023-09-23T20:37:13.966 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:37:13.966 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:37:13.966 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:37:13.966 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.45161", 2023-09-23T20:37:13.966 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3795614987", 2023-09-23T20:37:13.966 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 136060 2023-09-23T20:37:13.966 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.967 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:37:13.967 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.967 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:37:13.967 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:27.380303+0000", 2023-09-23T20:37:13.967 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:37:13.967 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.967 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.967 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:37:13.967 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:27.380303+0000", 2023-09-23T20:37:13.967 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:37:13.967 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.968 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.968 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:37:13.968 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:27.380303+0000", 2023-09-23T20:37:13.968 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:37:13.968 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.968 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.968 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:37:13.968 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:27.380306+0000", 2023-09-23T20:37:13.968 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.0070000000000002e-06 2023-09-23T20:37:13.968 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.969 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.969 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:37:13.969 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:27.380312+0000", 2023-09-23T20:37:13.969 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.6200000000000001e-06 2023-09-23T20:37:13.969 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.969 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.969 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:37:13.969 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:27.380321+0000", 2023-09-23T20:37:13.969 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.3200000000000006e-06 2023-09-23T20:37:13.970 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.970 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.970 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:37:13.970 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:27.380389+0000", 2023-09-23T20:37:13.970 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.7699000000000004e-05 2023-09-23T20:37:13.970 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.970 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.970 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:37:13.971 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:27.380395+0000", 2023-09-23T20:37:13.971 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.139e-06 2023-09-23T20:37:13.971 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.971 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.971 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:37:13.971 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:28.355749+0000", 2023-09-23T20:37:13.971 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.97535333199999996 2023-09-23T20:37:13.971 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.971 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.971 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:37:13.971 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:28.355757+0000", 2023-09-23T20:37:13.972 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.0430000000000008e-06 2023-09-23T20:37:13.972 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.972 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.972 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:37:13.972 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:29.396046+0000", 2023-09-23T20:37:13.972 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0402891299999999 2023-09-23T20:37:13.972 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.972 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.973 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:37:13.973 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:29.396225+0000", 2023-09-23T20:37:13.973 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00017960099999999999 2023-09-23T20:37:13.973 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.973 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.973 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:37:13.973 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:29.396790+0000", 2023-09-23T20:37:13.973 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00056465699999999998 2023-09-23T20:37:13.973 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.973 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.973 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:37:13.974 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:29.399660+0000", 2023-09-23T20:37:13.974 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.002869969 2023-09-23T20:37:13.974 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.974 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.974 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:37:13.974 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:29.400003+0000", 2023-09-23T20:37:13.974 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00034285899999999999 2023-09-23T20:37:13.974 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.974 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.975 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:37:13.975 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:29.419979+0000", 2023-09-23T20:37:13.975 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.019976224000000001 2023-09-23T20:37:13.975 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.975 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.975 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:37:13.975 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:29.420015+0000", 2023-09-23T20:37:13.975 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.6282999999999998e-05 2023-09-23T20:37:13.975 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.975 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.976 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:37:13.976 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:29:29.420094+0000", 2023-09-23T20:37:13.976 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.9040000000000002e-05 2023-09-23T20:37:13.976 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:37:13.976 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:37:13.976 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:37:13.976 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.977 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.977 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:43488 8.0 8:0364fc62:::benchmark_data_smithi018_341490_object43487:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e374)", 2023-09-23T20:37:13.977 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:36:36.564188+0000", 2023-09-23T20:37:13.977 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 37.347394393999998, 2023-09-23T20:37:13.977 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.078957933999999994, 2023-09-23T20:37:13.977 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:37:13.977 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:37:13.977 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:37:13.977 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:37:13.977 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:37:13.978 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 43488 2023-09-23T20:37:13.978 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.978 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:37:13.978 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.978 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:37:13.978 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:36.564188+0000", 2023-09-23T20:37:13.978 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:37:13.978 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.978 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.979 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:37:13.979 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:36.564188+0000", 2023-09-23T20:37:13.979 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:37:13.979 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.979 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.979 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:37:13.979 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:36.564188+0000", 2023-09-23T20:37:13.979 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:37:13.980 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.980 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.980 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:37:13.980 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:36.564244+0000", 2023-09-23T20:37:13.980 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.6781999999999999e-05 2023-09-23T20:37:13.980 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.980 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.980 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:37:13.980 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:36.564254+0000", 2023-09-23T20:37:13.980 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.2010000000000005e-06 2023-09-23T20:37:13.980 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.981 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.981 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:37:13.981 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:36.564258+0000", 2023-09-23T20:37:13.981 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.9809999999999997e-06 2023-09-23T20:37:13.981 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.981 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.981 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:37:13.981 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:36.564301+0000", 2023-09-23T20:37:13.981 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.3541000000000002e-05 2023-09-23T20:37:13.981 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.981 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.981 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:37:13.982 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:36.564389+0000", 2023-09-23T20:37:13.982 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.7837999999999995e-05 2023-09-23T20:37:13.982 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.982 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.982 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:37:13.982 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:36.564509+0000", 2023-09-23T20:37:13.982 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000119882 2023-09-23T20:37:13.982 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.982 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.982 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:37:13.983 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:36.566248+0000", 2023-09-23T20:37:13.983 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00173883 2023-09-23T20:37:13.983 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.983 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.983 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:37:13.983 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:36.568338+0000", 2023-09-23T20:37:13.983 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.002090668 2023-09-23T20:37:13.984 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.984 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.984 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:37:13.984 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:36.643080+0000", 2023-09-23T20:37:13.984 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.074741780999999993 2023-09-23T20:37:13.984 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.984 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.984 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:37:13.985 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:36.643117+0000", 2023-09-23T20:37:13.985 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.7178999999999998e-05 2023-09-23T20:37:13.985 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.985 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.985 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:37:13.985 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:36.643146+0000", 2023-09-23T20:37:13.985 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.8867e-05 2023-09-23T20:37:13.985 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:37:13.986 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:37:13.986 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:37:13.986 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.986 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.986 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:45464 8.d 8:ba447c44:::benchmark_data_smithi018_341490_object45463:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e374)", 2023-09-23T20:37:13.986 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:36:38.007549+0000", 2023-09-23T20:37:13.986 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 35.904033769999998, 2023-09-23T20:37:13.986 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.078667916000000004, 2023-09-23T20:37:13.986 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:37:13.986 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:37:13.986 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:37:13.986 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:37:13.987 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:37:13.987 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 45464 2023-09-23T20:37:13.987 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.987 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:37:13.987 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.987 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:37:13.987 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:38.007549+0000", 2023-09-23T20:37:13.987 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:37:13.987 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.987 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.987 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:37:13.987 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:38.007549+0000", 2023-09-23T20:37:13.988 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:37:13.988 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.988 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.988 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:37:13.988 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:38.007548+0000", 2023-09-23T20:37:13.988 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:37:13.988 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.988 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.988 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:37:13.988 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:38.007589+0000", 2023-09-23T20:37:13.988 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.0552000000000001e-05 2023-09-23T20:37:13.989 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.989 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.989 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:37:13.989 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:38.007596+0000", 2023-09-23T20:37:13.989 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.7449999999999995e-06 2023-09-23T20:37:13.989 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.989 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.989 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:37:13.989 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:38.007602+0000", 2023-09-23T20:37:13.989 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.4589999999999997e-06 2023-09-23T20:37:13.989 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.989 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.990 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:37:13.990 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:38.007759+0000", 2023-09-23T20:37:13.990 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00015674299999999999 2023-09-23T20:37:13.990 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.990 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.990 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:37:13.990 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:38.007873+0000", 2023-09-23T20:37:13.990 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000114814 2023-09-23T20:37:13.990 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.990 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.990 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 2,7", 2023-09-23T20:37:13.991 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:38.008037+0000", 2023-09-23T20:37:13.991 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00016373099999999999 2023-09-23T20:37:13.991 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.991 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.991 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:37:13.991 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:38.012622+0000", 2023-09-23T20:37:13.991 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0045852760000000001 2023-09-23T20:37:13.991 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.991 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.991 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:37:13.991 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:38.014508+0000", 2023-09-23T20:37:13.992 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.001885287 2023-09-23T20:37:13.992 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.992 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.992 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:37:13.992 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:38.086115+0000", 2023-09-23T20:37:13.992 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.071607335999999994 2023-09-23T20:37:13.992 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.992 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.992 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:37:13.992 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:38.086171+0000", 2023-09-23T20:37:13.992 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.6150000000000003e-05 2023-09-23T20:37:13.993 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.993 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.993 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:37:13.993 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:38.086217+0000", 2023-09-23T20:37:13.993 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.5636999999999997e-05 2023-09-23T20:37:13.993 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:37:13.993 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:37:13.993 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:37:13.993 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.993 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.993 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:50918 8.d 8:bd511cac:::benchmark_data_smithi018_341490_object50917:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e374)", 2023-09-23T20:37:13.993 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:36:42.401925+0000", 2023-09-23T20:37:13.994 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 31.50965763, 2023-09-23T20:37:13.994 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.097822600999999995, 2023-09-23T20:37:13.994 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:37:13.994 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:37:13.994 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:37:13.994 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:37:13.994 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:37:13.994 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 50918 2023-09-23T20:37:13.994 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.994 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:37:13.995 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.995 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:37:13.995 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.401925+0000", 2023-09-23T20:37:13.995 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:37:13.995 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.995 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.995 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:37:13.995 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.401925+0000", 2023-09-23T20:37:13.995 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:37:13.995 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.995 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.996 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:37:13.996 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.401925+0000", 2023-09-23T20:37:13.996 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:37:13.996 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.996 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.996 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:37:13.996 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.401966+0000", 2023-09-23T20:37:13.996 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.1180999999999998e-05 2023-09-23T20:37:13.996 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.996 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.996 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:37:13.997 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.401973+0000", 2023-09-23T20:37:13.997 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.2880000000000004e-06 2023-09-23T20:37:13.997 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.997 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.997 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:37:13.997 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.401978+0000", 2023-09-23T20:37:13.997 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.4900000000000002e-06 2023-09-23T20:37:13.997 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.997 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.997 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:37:13.997 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.402248+0000", 2023-09-23T20:37:13.998 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00027098600000000002 2023-09-23T20:37:13.998 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.998 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.998 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:37:13.998 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.402364+0000", 2023-09-23T20:37:13.998 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000115437 2023-09-23T20:37:13.998 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.998 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.998 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 2,7", 2023-09-23T20:37:13.998 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.402489+0000", 2023-09-23T20:37:13.998 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00012556500000000001 2023-09-23T20:37:13.999 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.999 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.999 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:37:13.999 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.410785+0000", 2023-09-23T20:37:13.999 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0082954139999999992 2023-09-23T20:37:13.999 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.999 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:13.999 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:37:13.999 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.411956+0000", 2023-09-23T20:37:13.999 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.001171102 2023-09-23T20:37:13.999 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:13.999 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.000 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:37:14.000 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.499657+0000", 2023-09-23T20:37:14.000 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.087701055 2023-09-23T20:37:14.000 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.000 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.000 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:37:14.000 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.499699+0000", 2023-09-23T20:37:14.000 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.2315999999999999e-05 2023-09-23T20:37:14.000 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.000 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.000 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:37:14.001 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.499748+0000", 2023-09-23T20:37:14.001 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.8297000000000002e-05 2023-09-23T20:37:14.001 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:37:14.001 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:37:14.001 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:37:14.001 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.001 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.001 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:50920 8.4 8:26c76582:::benchmark_data_smithi018_341490_object50919:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e374)", 2023-09-23T20:37:14.001 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:36:42.401992+0000", 2023-09-23T20:37:14.001 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 31.509590331999998, 2023-09-23T20:37:14.001 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.099793881000000001, 2023-09-23T20:37:14.002 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:37:14.002 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:37:14.002 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:37:14.002 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:37:14.002 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:37:14.002 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 50920 2023-09-23T20:37:14.002 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.002 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:37:14.002 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.002 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:37:14.002 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.401992+0000", 2023-09-23T20:37:14.002 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:37:14.003 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.003 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.003 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:37:14.003 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.401992+0000", 2023-09-23T20:37:14.003 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:37:14.003 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.003 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.003 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:37:14.003 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.401992+0000", 2023-09-23T20:37:14.003 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:37:14.003 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.003 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.005 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:37:14.005 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.402039+0000", 2023-09-23T20:37:14.005 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.7437e-05 2023-09-23T20:37:14.005 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.005 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.005 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:37:14.005 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.402046+0000", 2023-09-23T20:37:14.006 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.6039999999999998e-06 2023-09-23T20:37:14.006 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.006 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.006 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:37:14.006 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.402050+0000", 2023-09-23T20:37:14.006 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.1910000000000002e-06 2023-09-23T20:37:14.006 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.006 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.006 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:37:14.006 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.402085+0000", 2023-09-23T20:37:14.006 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.4932999999999999e-05 2023-09-23T20:37:14.007 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.007 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.007 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:37:14.007 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.402196+0000", 2023-09-23T20:37:14.007 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00011066200000000001 2023-09-23T20:37:14.007 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.007 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.007 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 2,4", 2023-09-23T20:37:14.007 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.402335+0000", 2023-09-23T20:37:14.007 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00013887099999999999 2023-09-23T20:37:14.007 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.008 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.008 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:37:14.008 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.411209+0000", 2023-09-23T20:37:14.008 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0088743020000000006 2023-09-23T20:37:14.008 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.008 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.008 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:37:14.009 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.411925+0000", 2023-09-23T20:37:14.009 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00071639299999999996 2023-09-23T20:37:14.009 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.009 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.009 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:37:14.009 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.501572+0000", 2023-09-23T20:37:14.009 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.089646669999999998 2023-09-23T20:37:14.009 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.009 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.009 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:37:14.009 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.501696+0000", 2023-09-23T20:37:14.010 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000124401 2023-09-23T20:37:14.010 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.010 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.010 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:37:14.010 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.501786+0000", 2023-09-23T20:37:14.010 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.9876999999999993e-05 2023-09-23T20:37:14.010 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:37:14.010 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:37:14.011 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:37:14.011 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.011 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.011 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:50921 8.d 8:b5eb3a99:::benchmark_data_smithi018_341490_object50920:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e374)", 2023-09-23T20:37:14.011 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:36:42.402064+0000", 2023-09-23T20:37:14.011 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 31.509518570000001, 2023-09-23T20:37:14.011 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.101541197, 2023-09-23T20:37:14.011 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:37:14.011 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:37:14.011 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:37:14.011 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:37:14.012 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:37:14.012 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 50921 2023-09-23T20:37:14.012 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.012 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:37:14.012 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.012 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:37:14.012 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.402064+0000", 2023-09-23T20:37:14.012 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:37:14.012 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.013 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.013 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:37:14.013 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.402064+0000", 2023-09-23T20:37:14.013 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:37:14.013 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.013 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.013 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:37:14.013 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.402064+0000", 2023-09-23T20:37:14.013 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:37:14.013 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.013 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.014 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:37:14.014 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.402100+0000", 2023-09-23T20:37:14.014 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.6505999999999999e-05 2023-09-23T20:37:14.014 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.014 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.014 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:37:14.014 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.402108+0000", 2023-09-23T20:37:14.015 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.909e-06 2023-09-23T20:37:14.015 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.015 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.015 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:37:14.015 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.402113+0000", 2023-09-23T20:37:14.015 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.8389999999999997e-06 2023-09-23T20:37:14.015 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.015 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.015 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:37:14.015 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.404426+0000", 2023-09-23T20:37:14.015 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.002312634 2023-09-23T20:37:14.016 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.016 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.016 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:37:14.016 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.404560+0000", 2023-09-23T20:37:14.016 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00013467500000000001 2023-09-23T20:37:14.016 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.016 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.016 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 2,7", 2023-09-23T20:37:14.017 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.404700+0000", 2023-09-23T20:37:14.017 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000140221 2023-09-23T20:37:14.017 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.017 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.017 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:37:14.017 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.410792+0000", 2023-09-23T20:37:14.017 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0060912309999999999 2023-09-23T20:37:14.017 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.018 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.018 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:37:14.018 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.411972+0000", 2023-09-23T20:37:14.018 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0011804809999999999 2023-09-23T20:37:14.018 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.018 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.018 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:37:14.018 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.503540+0000", 2023-09-23T20:37:14.018 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.091568288999999997 2023-09-23T20:37:14.019 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.019 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.019 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:37:14.019 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.503573+0000", 2023-09-23T20:37:14.019 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.2759999999999998e-05 2023-09-23T20:37:14.019 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.019 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.019 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:37:14.020 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.503605+0000", 2023-09-23T20:37:14.020 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.2152e-05 2023-09-23T20:37:14.020 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:37:14.020 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:37:14.020 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:37:14.020 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.020 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.020 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:50927 8.4 8:290beff9:::benchmark_data_smithi018_341490_object50926:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e374)", 2023-09-23T20:37:14.020 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:36:42.406872+0000", 2023-09-23T20:37:14.021 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 31.504710327000002, 2023-09-23T20:37:14.021 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.096241193000000003, 2023-09-23T20:37:14.021 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:37:14.021 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:37:14.021 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:37:14.021 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:37:14.021 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:37:14.021 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 50927 2023-09-23T20:37:14.021 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.021 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:37:14.021 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.022 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:37:14.022 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.406872+0000", 2023-09-23T20:37:14.022 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:37:14.022 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.022 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.022 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:37:14.022 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.406872+0000", 2023-09-23T20:37:14.022 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:37:14.022 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.022 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.022 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:37:14.023 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.406872+0000", 2023-09-23T20:37:14.023 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:37:14.023 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.023 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.023 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:37:14.023 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.406927+0000", 2023-09-23T20:37:14.023 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.5055000000000001e-05 2023-09-23T20:37:14.023 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.023 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.023 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:37:14.023 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.406933+0000", 2023-09-23T20:37:14.023 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.6060000000000002e-06 2023-09-23T20:37:14.024 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.024 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.024 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:37:14.024 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.406936+0000", 2023-09-23T20:37:14.024 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.3720000000000001e-06 2023-09-23T20:37:14.024 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.024 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.024 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:37:14.024 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.406962+0000", 2023-09-23T20:37:14.024 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.5809e-05 2023-09-23T20:37:14.024 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.025 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.025 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:37:14.025 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.407067+0000", 2023-09-23T20:37:14.025 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000105581 2023-09-23T20:37:14.025 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.025 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.025 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 2,4", 2023-09-23T20:37:14.025 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.407212+0000", 2023-09-23T20:37:14.025 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00014454100000000001 2023-09-23T20:37:14.025 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.025 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.026 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:37:14.026 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.411215+0000", 2023-09-23T20:37:14.026 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.004003581 2023-09-23T20:37:14.026 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.026 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.026 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:37:14.026 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.411965+0000", 2023-09-23T20:37:14.026 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00074908199999999998 2023-09-23T20:37:14.026 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.026 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.026 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:37:14.026 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.503041+0000", 2023-09-23T20:37:14.027 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.091075955 2023-09-23T20:37:14.027 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.027 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.027 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:37:14.027 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.503077+0000", 2023-09-23T20:37:14.027 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.6531e-05 2023-09-23T20:37:14.027 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.027 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.027 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:37:14.027 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.503114+0000", 2023-09-23T20:37:14.028 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.6514000000000003e-05 2023-09-23T20:37:14.028 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:37:14.028 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:37:14.028 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:37:14.028 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.028 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.028 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:50929 8.4 8:28988421:::benchmark_data_smithi018_341490_object50928:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e374)", 2023-09-23T20:37:14.028 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:36:42.409489+0000", 2023-09-23T20:37:14.028 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 31.502093701, 2023-09-23T20:37:14.028 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.095383402000000006, 2023-09-23T20:37:14.028 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:37:14.029 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:37:14.029 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:37:14.029 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:37:14.029 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:37:14.029 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 50929 2023-09-23T20:37:14.029 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.029 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:37:14.029 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.029 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:37:14.029 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.409489+0000", 2023-09-23T20:37:14.029 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:37:14.029 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.030 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.030 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:37:14.030 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.409489+0000", 2023-09-23T20:37:14.030 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:37:14.030 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.030 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.030 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:37:14.030 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.409488+0000", 2023-09-23T20:37:14.030 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:37:14.030 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.031 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.031 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:37:14.031 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.409528+0000", 2023-09-23T20:37:14.031 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.9983e-05 2023-09-23T20:37:14.031 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.031 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.031 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:37:14.031 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.409539+0000", 2023-09-23T20:37:14.031 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.148e-05 2023-09-23T20:37:14.031 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.031 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.031 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:37:14.031 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.409546+0000", 2023-09-23T20:37:14.032 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.4679999999999997e-06 2023-09-23T20:37:14.032 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.032 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.032 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:37:14.032 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.409591+0000", 2023-09-23T20:37:14.032 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.4879000000000002e-05 2023-09-23T20:37:14.032 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.032 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.032 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:37:14.032 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.409679+0000", 2023-09-23T20:37:14.033 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.8792e-05 2023-09-23T20:37:14.033 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.033 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.033 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 2,4", 2023-09-23T20:37:14.033 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.409777+0000", 2023-09-23T20:37:14.033 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.7485000000000006e-05 2023-09-23T20:37:14.033 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.033 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.033 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:37:14.033 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.414076+0000", 2023-09-23T20:37:14.033 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0042990090000000003 2023-09-23T20:37:14.034 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.034 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.034 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:37:14.034 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.418948+0000", 2023-09-23T20:37:14.034 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.004871673 2023-09-23T20:37:14.034 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.034 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.034 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:37:14.034 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.504816+0000", 2023-09-23T20:37:14.034 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.085868710000000001 2023-09-23T20:37:14.034 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.035 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.035 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:37:14.035 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.504851+0000", 2023-09-23T20:37:14.035 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.4762999999999999e-05 2023-09-23T20:37:14.035 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.035 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.035 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:37:14.035 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.504872+0000", 2023-09-23T20:37:14.035 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.1365999999999998e-05 2023-09-23T20:37:14.035 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:37:14.035 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:37:14.035 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:37:14.036 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.036 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.036 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:50932 8.4 8:263dc8f1:::benchmark_data_smithi018_341490_object50931:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e374)", 2023-09-23T20:37:14.036 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:36:42.411965+0000", 2023-09-23T20:37:14.036 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 31.499618045999998, 2023-09-23T20:37:14.036 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.093201201999999997, 2023-09-23T20:37:14.036 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:37:14.036 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:37:14.036 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:37:14.036 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:37:14.036 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:37:14.037 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 50932 2023-09-23T20:37:14.037 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.037 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:37:14.037 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.037 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:37:14.037 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.411965+0000", 2023-09-23T20:37:14.037 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:37:14.037 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.037 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.037 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:37:14.037 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.411965+0000", 2023-09-23T20:37:14.037 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:37:14.037 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.038 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.038 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:37:14.038 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.411964+0000", 2023-09-23T20:37:14.038 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:37:14.038 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.038 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.038 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:37:14.038 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.412019+0000", 2023-09-23T20:37:14.038 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.5523999999999999e-05 2023-09-23T20:37:14.038 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.038 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.039 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:37:14.039 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.412029+0000", 2023-09-23T20:37:14.039 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.9690000000000006e-06 2023-09-23T20:37:14.039 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.039 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.039 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:37:14.039 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.412035+0000", 2023-09-23T20:37:14.039 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.5470000000000004e-06 2023-09-23T20:37:14.039 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.039 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.039 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:37:14.040 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.412079+0000", 2023-09-23T20:37:14.040 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.4742e-05 2023-09-23T20:37:14.040 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.040 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.040 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:37:14.040 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.412211+0000", 2023-09-23T20:37:14.040 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00013197799999999999 2023-09-23T20:37:14.040 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.040 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.040 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 2,4", 2023-09-23T20:37:14.040 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.412364+0000", 2023-09-23T20:37:14.040 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000152862 2023-09-23T20:37:14.041 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.041 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.041 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:37:14.041 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.414234+0000", 2023-09-23T20:37:14.041 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0018700170000000001 2023-09-23T20:37:14.041 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.042 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.042 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:37:14.042 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.419011+0000", 2023-09-23T20:37:14.042 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0047765309999999997 2023-09-23T20:37:14.042 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.042 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.042 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:37:14.042 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.505083+0000", 2023-09-23T20:37:14.042 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.086072420999999996 2023-09-23T20:37:14.042 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.043 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.043 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:37:14.043 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.505133+0000", 2023-09-23T20:37:14.043 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.0142000000000002e-05 2023-09-23T20:37:14.043 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.043 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.043 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:37:14.043 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.505166+0000", 2023-09-23T20:37:14.043 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.2428000000000002e-05 2023-09-23T20:37:14.044 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:37:14.044 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:37:14.044 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:37:14.044 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.044 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.044 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:50939 8.d 8:b0da9d67:::benchmark_data_smithi018_341490_object50938:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e374)", 2023-09-23T20:37:14.044 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:36:42.414794+0000", 2023-09-23T20:37:14.044 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 31.496788845000001, 2023-09-23T20:37:14.044 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.089907521000000004, 2023-09-23T20:37:14.044 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:37:14.045 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:37:14.045 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:37:14.045 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:37:14.045 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:37:14.045 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 50939 2023-09-23T20:37:14.045 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.045 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:37:14.045 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.045 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:37:14.045 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.414794+0000", 2023-09-23T20:37:14.046 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:37:14.046 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.046 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.046 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:37:14.046 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.414794+0000", 2023-09-23T20:37:14.046 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:37:14.046 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.046 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.046 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:37:14.047 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.414793+0000", 2023-09-23T20:37:14.047 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:37:14.047 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.047 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.047 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:37:14.047 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.414874+0000", 2023-09-23T20:37:14.047 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.1625999999999994e-05 2023-09-23T20:37:14.047 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.048 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.048 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:37:14.048 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.414889+0000", 2023-09-23T20:37:14.048 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.4651000000000001e-05 2023-09-23T20:37:14.048 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.048 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.048 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:37:14.048 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.414895+0000", 2023-09-23T20:37:14.048 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.0109999999999997e-06 2023-09-23T20:37:14.048 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.048 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.049 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:37:14.049 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.414967+0000", 2023-09-23T20:37:14.049 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.2558000000000006e-05 2023-09-23T20:37:14.049 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.049 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.049 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:37:14.049 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.415105+0000", 2023-09-23T20:37:14.049 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00013778999999999999 2023-09-23T20:37:14.050 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.050 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.050 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 2,7", 2023-09-23T20:37:14.050 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.415282+0000", 2023-09-23T20:37:14.050 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000176949 2023-09-23T20:37:14.050 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.050 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.050 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:37:14.051 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.417706+0000", 2023-09-23T20:37:14.051 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0024242449999999998 2023-09-23T20:37:14.051 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.051 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.051 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:37:14.051 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.418833+0000", 2023-09-23T20:37:14.051 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.001127081 2023-09-23T20:37:14.051 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.052 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.052 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:37:14.052 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.504617+0000", 2023-09-23T20:37:14.052 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.085783662999999996 2023-09-23T20:37:14.052 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.052 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.052 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:37:14.052 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.504665+0000", 2023-09-23T20:37:14.053 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.7744000000000002e-05 2023-09-23T20:37:14.053 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.053 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.053 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:37:14.053 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.504701+0000", 2023-09-23T20:37:14.053 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.6535999999999997e-05 2023-09-23T20:37:14.053 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:37:14.053 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:37:14.053 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:37:14.053 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.053 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.054 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:50941 8.d 8:b90c3afe:::benchmark_data_smithi018_341490_object50940:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e374)", 2023-09-23T20:37:14.054 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:36:42.418950+0000", 2023-09-23T20:37:14.054 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 31.492632263000001, 2023-09-23T20:37:14.054 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.085864218000000006, 2023-09-23T20:37:14.054 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:37:14.054 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:37:14.054 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:37:14.054 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:37:14.054 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:37:14.054 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 50941 2023-09-23T20:37:14.054 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.054 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:37:14.055 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.055 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:37:14.055 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.418950+0000", 2023-09-23T20:37:14.055 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:37:14.055 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.055 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.055 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:37:14.055 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.418950+0000", 2023-09-23T20:37:14.055 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:37:14.055 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.055 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.056 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:37:14.056 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.418950+0000", 2023-09-23T20:37:14.056 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:37:14.056 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.056 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.056 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:37:14.056 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.419106+0000", 2023-09-23T20:37:14.056 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00015659699999999999 2023-09-23T20:37:14.057 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.057 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.057 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:37:14.057 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.419119+0000", 2023-09-23T20:37:14.057 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.2643000000000001e-05 2023-09-23T20:37:14.057 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.057 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.057 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:37:14.057 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.419124+0000", 2023-09-23T20:37:14.057 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.6149999999999996e-06 2023-09-23T20:37:14.057 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.057 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.058 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:37:14.058 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.419167+0000", 2023-09-23T20:37:14.058 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.2536000000000001e-05 2023-09-23T20:37:14.058 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.058 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.058 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:37:14.058 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.419284+0000", 2023-09-23T20:37:14.058 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000117138 2023-09-23T20:37:14.058 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.058 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.058 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 2,7", 2023-09-23T20:37:14.058 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.419439+0000", 2023-09-23T20:37:14.059 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000155317 2023-09-23T20:37:14.059 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.059 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.059 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:37:14.059 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.421564+0000", 2023-09-23T20:37:14.059 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0021250050000000001 2023-09-23T20:37:14.059 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.059 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.059 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:37:14.059 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.421897+0000", 2023-09-23T20:37:14.059 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00033218100000000002 2023-09-23T20:37:14.060 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.060 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.060 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:37:14.060 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.504738+0000", 2023-09-23T20:37:14.060 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.082841434000000005 2023-09-23T20:37:14.060 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.060 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.060 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:37:14.060 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.504769+0000", 2023-09-23T20:37:14.060 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.1479000000000001e-05 2023-09-23T20:37:14.061 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.061 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.061 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:37:14.061 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.504815+0000", 2023-09-23T20:37:14.061 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.5172000000000001e-05 2023-09-23T20:37:14.061 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:37:14.061 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:37:14.061 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:37:14.061 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.061 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.061 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:51043 8.0 8:0f697c15:::benchmark_data_smithi018_341490_object51042:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e374)", 2023-09-23T20:37:14.061 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:36:42.561436+0000", 2023-09-23T20:37:14.062 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 31.350146829, 2023-09-23T20:37:14.062 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.14033216600000001, 2023-09-23T20:37:14.062 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:37:14.062 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:37:14.062 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:37:14.062 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:37:14.062 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:37:14.062 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 51043 2023-09-23T20:37:14.062 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.062 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:37:14.063 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.063 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:37:14.063 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.561436+0000", 2023-09-23T20:37:14.063 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:37:14.063 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.063 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.063 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:37:14.063 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.561436+0000", 2023-09-23T20:37:14.063 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:37:14.063 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.063 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.064 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:37:14.064 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.561435+0000", 2023-09-23T20:37:14.064 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:37:14.064 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.064 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.064 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:37:14.064 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.561495+0000", 2023-09-23T20:37:14.064 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.0056999999999998e-05 2023-09-23T20:37:14.064 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.064 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.064 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:37:14.065 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.561507+0000", 2023-09-23T20:37:14.065 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1627999999999999e-05 2023-09-23T20:37:14.065 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.065 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.065 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:37:14.065 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.561513+0000", 2023-09-23T20:37:14.065 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.5969999999999997e-06 2023-09-23T20:37:14.065 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.065 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.065 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:37:14.065 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.561542+0000", 2023-09-23T20:37:14.066 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.8665000000000001e-05 2023-09-23T20:37:14.066 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.066 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.066 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:37:14.066 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.561627+0000", 2023-09-23T20:37:14.066 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.4596000000000001e-05 2023-09-23T20:37:14.066 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.066 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.066 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:37:14.066 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.561720+0000", 2023-09-23T20:37:14.066 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.3153999999999996e-05 2023-09-23T20:37:14.066 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.067 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.067 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:37:14.067 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.563303+0000", 2023-09-23T20:37:14.067 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0015831689999999999 2023-09-23T20:37:14.067 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.067 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.067 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:37:14.067 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.563545+0000", 2023-09-23T20:37:14.067 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00024228800000000001 2023-09-23T20:37:14.067 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.067 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.068 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:37:14.068 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.701697+0000", 2023-09-23T20:37:14.068 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.138151417 2023-09-23T20:37:14.068 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.068 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.068 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:37:14.068 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.701739+0000", 2023-09-23T20:37:14.068 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.2009999999999999e-05 2023-09-23T20:37:14.068 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.068 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.068 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:37:14.069 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.701768+0000", 2023-09-23T20:37:14.069 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.9332e-05 2023-09-23T20:37:14.069 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:37:14.069 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:37:14.069 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:37:14.069 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.069 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.069 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:70064 8.0 8:0bf37180:::benchmark_data_smithi018_341490_object70063:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e377)", 2023-09-23T20:37:14.069 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:36:58.876024+0000", 2023-09-23T20:37:14.069 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 15.035558432, 2023-09-23T20:37:14.069 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.077114127000000005, 2023-09-23T20:37:14.070 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:37:14.070 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:37:14.070 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:37:14.070 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:37:14.070 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:37:14.070 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 70064 2023-09-23T20:37:14.070 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.070 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:37:14.070 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.070 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:37:14.070 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:58.876024+0000", 2023-09-23T20:37:14.071 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:37:14.071 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.071 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.071 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:37:14.071 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:58.876024+0000", 2023-09-23T20:37:14.071 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:37:14.071 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.071 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.071 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:37:14.071 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:58.876023+0000", 2023-09-23T20:37:14.071 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:37:14.072 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.072 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.072 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:37:14.072 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:58.876082+0000", 2023-09-23T20:37:14.072 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.8848000000000003e-05 2023-09-23T20:37:14.072 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.072 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.072 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:37:14.072 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:58.876099+0000", 2023-09-23T20:37:14.072 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.7478000000000001e-05 2023-09-23T20:37:14.072 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.073 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.073 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:37:14.073 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:58.876104+0000", 2023-09-23T20:37:14.073 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.9169999999999997e-06 2023-09-23T20:37:14.073 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.073 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.073 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:37:14.073 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:58.876192+0000", 2023-09-23T20:37:14.073 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.7218000000000004e-05 2023-09-23T20:37:14.073 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.073 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.074 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:37:14.074 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:58.876268+0000", 2023-09-23T20:37:14.074 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.6905999999999999e-05 2023-09-23T20:37:14.074 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.074 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.074 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:37:14.074 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:58.876356+0000", 2023-09-23T20:37:14.074 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.8021000000000003e-05 2023-09-23T20:37:14.074 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.074 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.074 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:37:14.074 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:58.880361+0000", 2023-09-23T20:37:14.075 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0040048899999999997 2023-09-23T20:37:14.075 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.075 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.075 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:37:14.075 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:58.952873+0000", 2023-09-23T20:37:14.075 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.072511663000000004 2023-09-23T20:37:14.075 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.075 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.075 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:37:14.075 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:58.953056+0000", 2023-09-23T20:37:14.075 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00018334699999999999 2023-09-23T20:37:14.076 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.076 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.076 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:37:14.076 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:58.953101+0000", 2023-09-23T20:37:14.076 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.4669999999999998e-05 2023-09-23T20:37:14.076 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.076 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.076 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:37:14.076 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:58.953138+0000", 2023-09-23T20:37:14.076 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.7339000000000003e-05 2023-09-23T20:37:14.077 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:37:14.077 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:37:14.077 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:37:14.077 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.077 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.077 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:70065 8.0 8:0c404290:::benchmark_data_smithi018_341490_object70064:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e377)", 2023-09-23T20:37:14.077 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:36:58.876120+0000", 2023-09-23T20:37:14.077 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 15.035462709000001, 2023-09-23T20:37:14.077 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.077118790000000007, 2023-09-23T20:37:14.078 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:37:14.078 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:37:14.078 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:37:14.078 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:37:14.078 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:37:14.078 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 70065 2023-09-23T20:37:14.078 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.078 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:37:14.078 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.078 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:37:14.079 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:58.876120+0000", 2023-09-23T20:37:14.079 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:37:14.079 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.079 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.079 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:37:14.079 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:58.876120+0000", 2023-09-23T20:37:14.079 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:37:14.080 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.080 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.080 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:37:14.080 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:58.876120+0000", 2023-09-23T20:37:14.080 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:37:14.080 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.080 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.080 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:37:14.080 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:58.876153+0000", 2023-09-23T20:37:14.080 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.3509000000000001e-05 2023-09-23T20:37:14.080 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.080 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.081 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:37:14.081 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:58.876166+0000", 2023-09-23T20:37:14.081 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.2628e-05 2023-09-23T20:37:14.081 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.081 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.081 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:37:14.081 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:58.876169+0000", 2023-09-23T20:37:14.082 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.3759999999999999e-06 2023-09-23T20:37:14.082 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.082 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.082 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:37:14.082 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:58.878268+0000", 2023-09-23T20:37:14.082 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00209881 2023-09-23T20:37:14.082 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.082 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.082 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:37:14.082 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:58.878396+0000", 2023-09-23T20:37:14.082 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00012822699999999999 2023-09-23T20:37:14.082 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.083 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.083 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:37:14.083 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:58.878478+0000", 2023-09-23T20:37:14.083 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.1601999999999996e-05 2023-09-23T20:37:14.083 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.083 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.083 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:37:14.083 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:58.882441+0000", 2023-09-23T20:37:14.084 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0039637040000000002 2023-09-23T20:37:14.084 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.084 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.084 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:37:14.084 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:58.952923+0000", 2023-09-23T20:37:14.084 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.070481849999999999 2023-09-23T20:37:14.084 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.084 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.084 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:37:14.084 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:58.953171+0000", 2023-09-23T20:37:14.085 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000247545 2023-09-23T20:37:14.085 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.085 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.085 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:37:14.085 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:58.953206+0000", 2023-09-23T20:37:14.085 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.5212000000000001e-05 2023-09-23T20:37:14.086 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.086 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.086 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:37:14.086 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:58.953239+0000", 2023-09-23T20:37:14.086 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.2694000000000002e-05 2023-09-23T20:37:14.086 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:37:14.086 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:37:14.087 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:37:14.087 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.087 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.087 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:70070 8.8 8:1c559d00:::benchmark_data_smithi018_341490_object70069:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e377)", 2023-09-23T20:37:14.087 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:36:58.876180+0000", 2023-09-23T20:37:14.087 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 15.035402287, 2023-09-23T20:37:14.087 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.076670715, 2023-09-23T20:37:14.087 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:37:14.087 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:37:14.087 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:37:14.088 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:37:14.088 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:37:14.088 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 70070 2023-09-23T20:37:14.088 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.088 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:37:14.088 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.088 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:37:14.088 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:58.876180+0000", 2023-09-23T20:37:14.089 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:37:14.089 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.089 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.089 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:37:14.089 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:58.876180+0000", 2023-09-23T20:37:14.089 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:37:14.089 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.089 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.090 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:37:14.090 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:58.876180+0000", 2023-09-23T20:37:14.090 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:37:14.090 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.090 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.090 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:37:14.090 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:58.876214+0000", 2023-09-23T20:37:14.090 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.3793e-05 2023-09-23T20:37:14.090 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.090 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.090 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:37:14.090 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:58.876229+0000", 2023-09-23T20:37:14.091 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.4929999999999999e-05 2023-09-23T20:37:14.091 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.091 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.091 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:37:14.091 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:58.876232+0000", 2023-09-23T20:37:14.091 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.2059999999999999e-06 2023-09-23T20:37:14.091 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.091 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.091 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:37:14.091 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:58.878259+0000", 2023-09-23T20:37:14.091 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0020270240000000001 2023-09-23T20:37:14.091 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.092 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.092 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:37:14.092 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:58.878368+0000", 2023-09-23T20:37:14.092 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000109116 2023-09-23T20:37:14.092 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.092 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.092 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 3,7", 2023-09-23T20:37:14.092 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:58.878481+0000", 2023-09-23T20:37:14.092 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00011245 2023-09-23T20:37:14.092 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.092 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.093 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:37:14.093 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:58.882452+0000", 2023-09-23T20:37:14.093 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0039712289999999997 2023-09-23T20:37:14.093 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.093 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.093 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:37:14.093 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:58.882460+0000", 2023-09-23T20:37:14.093 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.3860000000000007e-06 2023-09-23T20:37:14.093 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.093 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.093 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:37:14.093 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:58.952751+0000", 2023-09-23T20:37:14.094 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.070290602999999993 2023-09-23T20:37:14.094 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.094 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.094 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:37:14.094 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:58.952806+0000", 2023-09-23T20:37:14.094 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.4895000000000002e-05 2023-09-23T20:37:14.094 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:14.094 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:14.094 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:37:14.094 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:58.952851+0000", 2023-09-23T20:37:14.094 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.5454000000000002e-05 2023-09-23T20:37:14.094 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:37:14.095 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:37:14.095 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:37:14.095 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:37:14.095 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:37:14.095 INFO:teuthology.orchestra.run.smithi149.stdout:} 2023-09-23T20:37:14.096 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:14.097 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:14.097 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:37:14.097 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:37:14.097 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 58 16 84113 84097 90.5827 20.6875 0.00399029 0.0109086 2023-09-23T20:37:14.198 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:37:14.229 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:14.267 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:37:14.269 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T20:37:14.270 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:14.271 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:37:14.362 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:37:14.371 INFO:teuthology.orchestra.run.smithi018.stderr:noscrub is unset 2023-09-23T20:37:14.372 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:37:14.408 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:14.451 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:37:14.454 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:14.454 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:37:14.555 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:37:14.584 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:14.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:14 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2008845680' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 0.2886195609608352}]': finished 2023-09-23T20:37:14.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:14 smithi018 ceph-mon[184144]: osdmap e379: 8 total, 8 up, 8 in 2023-09-23T20:37:14.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:14 smithi018 ceph-mon[184144]: failed to encode map e379 with expected crc 2023-09-23T20:37:14.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:14 smithi018 ceph-mon[184144]: failed to encode map e379 with expected crc 2023-09-23T20:37:14.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:14 smithi018 ceph-mon[184144]: failed to encode map e379 with expected crc 2023-09-23T20:37:14.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:14 smithi018 ceph-mon[184144]: failed to encode map e379 with expected crc 2023-09-23T20:37:14.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:14 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2698730305' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:37:14.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:14 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2008845680' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 0.2886195609608352}]': finished 2023-09-23T20:37:14.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:14 smithi018 ceph-mon[179647]: osdmap e379: 8 total, 8 up, 8 in 2023-09-23T20:37:14.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:14 smithi018 ceph-mon[179647]: failed to encode map e379 with expected crc 2023-09-23T20:37:14.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:14 smithi018 ceph-mon[179647]: failed to encode map e379 with expected crc 2023-09-23T20:37:14.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:14 smithi018 ceph-mon[179647]: failed to encode map e379 with expected crc 2023-09-23T20:37:14.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:14 smithi018 ceph-mon[179647]: failed to encode map e379 with expected crc 2023-09-23T20:37:14.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:14 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2698730305' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:37:14.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:14 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2008845680' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 0.2886195609608352}]': finished 2023-09-23T20:37:14.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:14 smithi149 ceph-mon[170394]: osdmap e379: 8 total, 8 up, 8 in 2023-09-23T20:37:14.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:14 smithi149 ceph-mon[170394]: failed to encode map e379 with expected crc 2023-09-23T20:37:14.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:14 smithi149 ceph-mon[170394]: failed to encode map e379 with expected crc 2023-09-23T20:37:14.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:14 smithi149 ceph-mon[170394]: failed to encode map e379 with expected crc 2023-09-23T20:37:14.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:14 smithi149 ceph-mon[170394]: failed to encode map e379 with expected crc 2023-09-23T20:37:14.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:14 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2698730305' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:37:14.730 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:37:14.733 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:14.734 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:37:14.835 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:37:14.862 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:14.895 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:37:14.897 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:14.898 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:37:14.943 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:37:14.998 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:37:15.026 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:15.033 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 59 16 84113 84097 89.048 0 - 0.0109086 2023-09-23T20:37:15.060 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:37:15.060 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:37:15.062 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:15.063 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:37:15.164 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:37:15.191 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:15.268 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:37:14 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:37:14] "GET /metrics HTTP/1.1" 200 36536 "" "Prometheus/2.43.0" 2023-09-23T20:37:15.276 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:37:15.279 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:15.280 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:37:15.380 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:37:15.464 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:15.470 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:15 smithi149 ceph-mon[170394]: pgmap v929: 119 pgs: 119 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 40 MiB/s wr, 633 op/s 2023-09-23T20:37:15.470 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:15 smithi149 ceph-mon[170394]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:37:15.470 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:15 smithi149 ceph-mon[170394]: failed to encode map e380 with expected crc 2023-09-23T20:37:15.470 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:15 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2698730305' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:37:15.470 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:15 smithi149 ceph-mon[170394]: failed to encode map e380 with expected crc 2023-09-23T20:37:15.470 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:15 smithi149 ceph-mon[170394]: osdmap e380: 8 total, 8 up, 8 in 2023-09-23T20:37:15.470 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:15 smithi149 ceph-mon[170394]: failed to encode map e380 with expected crc 2023-09-23T20:37:15.471 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:15 smithi149 ceph-mon[170394]: failed to encode map e380 with expected crc 2023-09-23T20:37:15.471 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:15 smithi149 ceph-mon[170394]: failed to encode map e380 with expected crc 2023-09-23T20:37:15.471 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:15 smithi149 ceph-mon[170394]: failed to encode map e380 with expected crc 2023-09-23T20:37:15.471 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:15 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:37:15.526 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:15 smithi018 ceph-mon[184144]: pgmap v929: 119 pgs: 119 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 40 MiB/s wr, 633 op/s 2023-09-23T20:37:15.526 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:15 smithi018 ceph-mon[184144]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:37:15.526 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:15 smithi018 ceph-mon[184144]: failed to encode map e380 with expected crc 2023-09-23T20:37:15.526 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:15 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2698730305' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:37:15.526 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:15 smithi018 ceph-mon[184144]: failed to encode map e380 with expected crc 2023-09-23T20:37:15.526 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:15 smithi018 ceph-mon[184144]: osdmap e380: 8 total, 8 up, 8 in 2023-09-23T20:37:15.527 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:15 smithi018 ceph-mon[184144]: failed to encode map e380 with expected crc 2023-09-23T20:37:15.527 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:15 smithi018 ceph-mon[184144]: failed to encode map e380 with expected crc 2023-09-23T20:37:15.527 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:15 smithi018 ceph-mon[184144]: failed to encode map e380 with expected crc 2023-09-23T20:37:15.527 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:15 smithi018 ceph-mon[184144]: failed to encode map e380 with expected crc 2023-09-23T20:37:15.527 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:15 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:37:15.527 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:15 smithi018 ceph-mon[179647]: pgmap v929: 119 pgs: 119 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 40 MiB/s wr, 633 op/s 2023-09-23T20:37:15.527 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:15 smithi018 ceph-mon[179647]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:37:15.527 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:15 smithi018 ceph-mon[179647]: failed to encode map e380 with expected crc 2023-09-23T20:37:15.527 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:15 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2698730305' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:37:15.527 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:15 smithi018 ceph-mon[179647]: failed to encode map e380 with expected crc 2023-09-23T20:37:15.528 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:15 smithi018 ceph-mon[179647]: osdmap e380: 8 total, 8 up, 8 in 2023-09-23T20:37:15.528 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:15 smithi018 ceph-mon[179647]: failed to encode map e380 with expected crc 2023-09-23T20:37:15.528 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:15 smithi018 ceph-mon[179647]: failed to encode map e380 with expected crc 2023-09-23T20:37:15.528 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:15 smithi018 ceph-mon[179647]: failed to encode map e380 with expected crc 2023-09-23T20:37:15.528 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:15 smithi018 ceph-mon[179647]: failed to encode map e380 with expected crc 2023-09-23T20:37:15.528 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:15 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:37:15.583 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:37:15.602 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:15.603 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:37:15.704 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:37:15.741 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:15.795 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:37:15.801 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:15.802 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:37:15.902 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:37:15.952 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:16.007 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:37:16.011 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:16.011 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:37:16.034 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:2023-09-23T20:37:16.034243+0000 min lat: 0.00120245 max lat: 2.3822 avg lat: 0.0112387 2023-09-23T20:37:16.034 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T20:37:16.034 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 60 16 84245 84229 87.7018 4.125 0.00536488 0.0112387 2023-09-23T20:37:16.112 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:37:16.155 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:16.202 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:37:16.205 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:16.205 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:37:16.306 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:37:16.340 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:16.387 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:37:16.391 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:16.391 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:37:16.403 INFO:teuthology.orchestra.run.smithi149.stdout:{ 2023-09-23T20:37:16.403 INFO:teuthology.orchestra.run.smithi149.stdout: "ops": [], 2023-09-23T20:37:16.403 INFO:teuthology.orchestra.run.smithi149.stdout: "num_ops": 0 2023-09-23T20:37:16.403 INFO:teuthology.orchestra.run.smithi149.stdout:} 2023-09-23T20:37:16.432 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:16 smithi149 ceph-mon[170394]: failed to encode map e381 with expected crc 2023-09-23T20:37:16.432 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:16 smithi149 ceph-mon[170394]: osdmap e381: 8 total, 8 up, 8 in 2023-09-23T20:37:16.432 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:16 smithi149 ceph-mon[170394]: failed to encode map e381 with expected crc 2023-09-23T20:37:16.432 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:16 smithi149 ceph-mon[170394]: failed to encode map e381 with expected crc 2023-09-23T20:37:16.432 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:16 smithi149 ceph-mon[170394]: failed to encode map e381 with expected crc 2023-09-23T20:37:16.432 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:16 smithi149 ceph-mon[170394]: failed to encode map e381 with expected crc 2023-09-23T20:37:16.432 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:16 smithi149 ceph-mon[170394]: failed to encode map e381 with expected crc 2023-09-23T20:37:16.433 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:16 smithi149 ceph-mon[170394]: failed to encode map e381 with expected crc 2023-09-23T20:37:16.433 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:16 smithi149 ceph-mon[170394]: failed to encode map e381 with expected crc 2023-09-23T20:37:16.433 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:16 smithi149 ceph-mon[170394]: osdmap e382: 8 total, 8 up, 8 in 2023-09-23T20:37:16.492 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:37:16.522 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:16.584 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:37:16.605 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:37:16.626 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:37:16.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:16 smithi018 ceph-mon[184144]: failed to encode map e381 with expected crc 2023-09-23T20:37:16.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:16 smithi018 ceph-mon[184144]: osdmap e381: 8 total, 8 up, 8 in 2023-09-23T20:37:16.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:16 smithi018 ceph-mon[184144]: failed to encode map e381 with expected crc 2023-09-23T20:37:16.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:16 smithi018 ceph-mon[184144]: failed to encode map e381 with expected crc 2023-09-23T20:37:16.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:16 smithi018 ceph-mon[184144]: failed to encode map e381 with expected crc 2023-09-23T20:37:16.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:16 smithi018 ceph-mon[184144]: failed to encode map e381 with expected crc 2023-09-23T20:37:16.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:16 smithi018 ceph-mon[184144]: failed to encode map e381 with expected crc 2023-09-23T20:37:16.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:16 smithi018 ceph-mon[184144]: failed to encode map e381 with expected crc 2023-09-23T20:37:16.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:16 smithi018 ceph-mon[184144]: failed to encode map e381 with expected crc 2023-09-23T20:37:16.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:16 smithi018 ceph-mon[184144]: osdmap e382: 8 total, 8 up, 8 in 2023-09-23T20:37:16.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:16 smithi018 ceph-mon[179647]: failed to encode map e381 with expected crc 2023-09-23T20:37:16.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:16 smithi018 ceph-mon[179647]: osdmap e381: 8 total, 8 up, 8 in 2023-09-23T20:37:16.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:16 smithi018 ceph-mon[179647]: failed to encode map e381 with expected crc 2023-09-23T20:37:16.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:16 smithi018 ceph-mon[179647]: failed to encode map e381 with expected crc 2023-09-23T20:37:16.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:16 smithi018 ceph-mon[179647]: failed to encode map e381 with expected crc 2023-09-23T20:37:16.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:16 smithi018 ceph-mon[179647]: failed to encode map e381 with expected crc 2023-09-23T20:37:16.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:16 smithi018 ceph-mon[179647]: failed to encode map e381 with expected crc 2023-09-23T20:37:16.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:16 smithi018 ceph-mon[179647]: failed to encode map e381 with expected crc 2023-09-23T20:37:16.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:16 smithi018 ceph-mon[179647]: failed to encode map e381 with expected crc 2023-09-23T20:37:16.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:16 smithi018 ceph-mon[179647]: osdmap e382: 8 total, 8 up, 8 in 2023-09-23T20:37:16.652 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:37:16.674 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:37:16.696 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:37:16.716 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:37:16.760 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:37:16.764 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:16.764 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:37:16.790 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T20:37:16.844 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T20:37:16.865 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:37:16.894 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:16.941 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:37:16.944 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:16.945 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:37:17.034 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 61 16 84270 84254 86.2901 1.5625 0.00403114 0.011237 2023-09-23T20:37:17.045 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:37:17.077 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:17.116 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:37:17.119 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:17.120 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:37:17.220 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:37:17.252 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:17.308 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:37:17.311 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:17.312 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:37:17.413 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:37:17.426 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:37:17.444 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:17.520 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:37:17.537 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:37:17.540 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:17.541 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:37:17.619 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:17 smithi018 ceph-mon[179647]: pgmap v932: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+remapped, 2 unknown, 6 peering, 106 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 17 KiB/s rd, 62 MiB/s wr, 1.01k op/s; 5302/251352 objects degraded (2.109%); 33 KiB/s, 1 objects/s recovering 2023-09-23T20:37:17.620 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:17 smithi018 ceph-mon[179647]: failed to encode map e382 with expected crc 2023-09-23T20:37:17.620 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:17 smithi018 ceph-mon[179647]: failed to encode map e382 with expected crc 2023-09-23T20:37:17.620 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:17 smithi018 ceph-mon[179647]: failed to encode map e382 with expected crc 2023-09-23T20:37:17.620 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:17 smithi018 ceph-mon[179647]: failed to encode map e382 with expected crc 2023-09-23T20:37:17.620 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:17 smithi018 ceph-mon[179647]: Health check failed: Reduced data availability: 1 pg inactive (PG_AVAILABILITY) 2023-09-23T20:37:17.620 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:17 smithi018 ceph-mon[179647]: Health check failed: Degraded data redundancy: 5302/251352 objects degraded (2.109%), 1 pg degraded (PG_DEGRADED) 2023-09-23T20:37:17.620 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:17 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2107485736' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:37:17.620 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:17 smithi018 ceph-mon[179647]: osdmap e383: 8 total, 8 up, 8 in 2023-09-23T20:37:17.620 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:17 smithi018 ceph-mon[184144]: pgmap v932: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+remapped, 2 unknown, 6 peering, 106 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 17 KiB/s rd, 62 MiB/s wr, 1.01k op/s; 5302/251352 objects degraded (2.109%); 33 KiB/s, 1 objects/s recovering 2023-09-23T20:37:17.621 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:17 smithi018 ceph-mon[184144]: failed to encode map e382 with expected crc 2023-09-23T20:37:17.621 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:17 smithi018 ceph-mon[184144]: failed to encode map e382 with expected crc 2023-09-23T20:37:17.621 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:17 smithi018 ceph-mon[184144]: failed to encode map e382 with expected crc 2023-09-23T20:37:17.621 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:17 smithi018 ceph-mon[184144]: failed to encode map e382 with expected crc 2023-09-23T20:37:17.621 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:17 smithi018 ceph-mon[184144]: Health check failed: Reduced data availability: 1 pg inactive (PG_AVAILABILITY) 2023-09-23T20:37:17.621 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:17 smithi018 ceph-mon[184144]: Health check failed: Degraded data redundancy: 5302/251352 objects degraded (2.109%), 1 pg degraded (PG_DEGRADED) 2023-09-23T20:37:17.621 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:17 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2107485736' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:37:17.621 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:17 smithi018 ceph-mon[184144]: osdmap e383: 8 total, 8 up, 8 in 2023-09-23T20:37:17.642 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:37:17.669 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:17 smithi149 ceph-mon[170394]: pgmap v932: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+remapped, 2 unknown, 6 peering, 106 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 17 KiB/s rd, 62 MiB/s wr, 1.01k op/s; 5302/251352 objects degraded (2.109%); 33 KiB/s, 1 objects/s recovering 2023-09-23T20:37:17.669 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:17 smithi149 ceph-mon[170394]: failed to encode map e382 with expected crc 2023-09-23T20:37:17.669 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:17 smithi149 ceph-mon[170394]: failed to encode map e382 with expected crc 2023-09-23T20:37:17.669 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:17 smithi149 ceph-mon[170394]: failed to encode map e382 with expected crc 2023-09-23T20:37:17.669 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:17 smithi149 ceph-mon[170394]: failed to encode map e382 with expected crc 2023-09-23T20:37:17.669 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:17 smithi149 ceph-mon[170394]: Health check failed: Reduced data availability: 1 pg inactive (PG_AVAILABILITY) 2023-09-23T20:37:17.669 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:17 smithi149 ceph-mon[170394]: Health check failed: Degraded data redundancy: 5302/251352 objects degraded (2.109%), 1 pg degraded (PG_DEGRADED) 2023-09-23T20:37:17.669 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:17 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2107485736' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:37:17.669 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:17 smithi149 ceph-mon[170394]: osdmap e383: 8 total, 8 up, 8 in 2023-09-23T20:37:17.674 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:17.771 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:37:17.775 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:17.775 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:37:17.876 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:37:17.910 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:18.000 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:37:18.004 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:18.004 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:37:18.034 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 62 16 84270 84254 84.8988 0 - 0.011237 2023-09-23T20:37:18.105 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:37:18.136 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:18.227 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:37:18.230 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:18.231 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:37:18.331 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:37:18.363 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:18.452 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:37:18.455 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:18.456 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:37:18.482 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:18 smithi018 ceph-mon[184144]: failed to encode map e383 with expected crc 2023-09-23T20:37:18.482 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:18 smithi018 ceph-mon[184144]: failed to encode map e383 with expected crc 2023-09-23T20:37:18.482 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:18 smithi018 ceph-mon[184144]: failed to encode map e383 with expected crc 2023-09-23T20:37:18.482 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:18 smithi018 ceph-mon[184144]: failed to encode map e383 with expected crc 2023-09-23T20:37:18.482 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:18 smithi018 ceph-mon[184144]: failed to encode map e383 with expected crc 2023-09-23T20:37:18.482 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:18 smithi018 ceph-mon[184144]: failed to encode map e383 with expected crc 2023-09-23T20:37:18.482 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:18 smithi018 ceph-mon[179647]: failed to encode map e383 with expected crc 2023-09-23T20:37:18.483 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:18 smithi018 ceph-mon[179647]: failed to encode map e383 with expected crc 2023-09-23T20:37:18.483 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:18 smithi018 ceph-mon[179647]: failed to encode map e383 with expected crc 2023-09-23T20:37:18.483 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:18 smithi018 ceph-mon[179647]: failed to encode map e383 with expected crc 2023-09-23T20:37:18.483 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:18 smithi018 ceph-mon[179647]: failed to encode map e383 with expected crc 2023-09-23T20:37:18.483 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:18 smithi018 ceph-mon[179647]: failed to encode map e383 with expected crc 2023-09-23T20:37:18.556 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:37:18.557 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:18 smithi149 ceph-mon[170394]: failed to encode map e383 with expected crc 2023-09-23T20:37:18.557 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:18 smithi149 ceph-mon[170394]: failed to encode map e383 with expected crc 2023-09-23T20:37:18.557 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:18 smithi149 ceph-mon[170394]: failed to encode map e383 with expected crc 2023-09-23T20:37:18.558 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:18 smithi149 ceph-mon[170394]: failed to encode map e383 with expected crc 2023-09-23T20:37:18.558 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:18 smithi149 ceph-mon[170394]: failed to encode map e383 with expected crc 2023-09-23T20:37:18.558 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:18 smithi149 ceph-mon[170394]: failed to encode map e383 with expected crc 2023-09-23T20:37:18.588 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:18.626 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:37:18.629 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:18.630 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:37:18.730 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:37:18.763 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:18.805 INFO:teuthology.orchestra.run.smithi149.stdout:ERROR: (22) Invalid argument 2023-09-23T20:37:18.805 INFO:teuthology.orchestra.run.smithi149.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:37:18.807 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:37:18.812 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:18.812 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:37:18.913 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:37:18.946 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:19.034 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 63 16 84270 84254 83.5516 0 - 0.011237 2023-09-23T20:37:19.096 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 5, 7, 3, 2, 6, 4] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T20:37:19.097 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:37:19.097 INFO:tasks.thrashosds.thrasher:Removing osd 6, in_osds are: [0, 1, 5, 7, 3, 2, 6, 4] 2023-09-23T20:37:19.097 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd out 6 2023-09-23T20:37:19.199 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T20:37:19.203 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:37:19.206 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:19.206 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:37:19.307 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:37:19.339 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:19.371 INFO:teuthology.orchestra.run.smithi018.stderr:nodeep-scrub is unset 2023-09-23T20:37:19.378 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:37:19.380 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:19 smithi018 ceph-mon[179647]: pgmap v935: 119 pgs: 1 activating+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+remapped, 2 unknown, 6 peering, 105 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 46 KiB/s rd, 39 MiB/s wr, 652 op/s; 5303/251475 objects degraded (2.109%); 24 KiB/s, 4 objects/s recovering 2023-09-23T20:37:19.381 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:19 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/677757736' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:37:19.381 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:19 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:37:19.381 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:19 smithi018 ceph-mon[184144]: pgmap v935: 119 pgs: 1 activating+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+remapped, 2 unknown, 6 peering, 105 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 46 KiB/s rd, 39 MiB/s wr, 652 op/s; 5303/251475 objects degraded (2.109%); 24 KiB/s, 4 objects/s recovering 2023-09-23T20:37:19.381 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:19 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/677757736' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:37:19.381 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:19 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:37:19.382 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:19.382 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:37:19.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:19 smithi149 ceph-mon[170394]: pgmap v935: 119 pgs: 1 activating+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+remapped, 2 unknown, 6 peering, 105 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 46 KiB/s rd, 39 MiB/s wr, 652 op/s; 5303/251475 objects degraded (2.109%); 24 KiB/s, 4 objects/s recovering 2023-09-23T20:37:19.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:19 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/677757736' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:37:19.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:19 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:37:19.483 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:37:19.517 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:19.728 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:37:19.731 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:19.731 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:37:19.833 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:37:19.848 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:37:19.868 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:19.942 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:37:20.034 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 64 16 84270 84254 82.2465 0 - 0.011237 2023-09-23T20:37:20.138 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:37:20.141 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:20.141 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:37:20.242 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:37:20.273 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:20.313 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:37:20.317 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:20.317 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:37:20.418 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:37:20.428 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:20 smithi149 ceph-mon[170394]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:37:20.428 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:20 smithi149 ceph-mon[170394]: failed to encode map e384 with expected crc 2023-09-23T20:37:20.428 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:20 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:37:20.429 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:20 smithi149 ceph-mon[170394]: osdmap e384: 8 total, 8 up, 8 in 2023-09-23T20:37:20.450 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:20.459 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:20 smithi018 ceph-mon[184144]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:37:20.459 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:20 smithi018 ceph-mon[184144]: failed to encode map e384 with expected crc 2023-09-23T20:37:20.460 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:20 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:37:20.460 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:20 smithi018 ceph-mon[184144]: osdmap e384: 8 total, 8 up, 8 in 2023-09-23T20:37:20.460 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:20 smithi018 ceph-mon[179647]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:37:20.460 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:20 smithi018 ceph-mon[179647]: failed to encode map e384 with expected crc 2023-09-23T20:37:20.460 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:20 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:37:20.460 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:20 smithi018 ceph-mon[179647]: osdmap e384: 8 total, 8 up, 8 in 2023-09-23T20:37:20.542 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:37:20.545 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:20.545 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:37:20.646 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:37:20.674 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:20.765 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:37:20.768 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:20.769 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:37:20.869 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:37:20.901 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:20.991 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:37:20.993 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:20.994 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:37:21.034 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 65 16 84270 84254 80.9815 0 - 0.011237 2023-09-23T20:37:21.095 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:37:21.129 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:21.227 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:37:21.232 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:21.232 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:37:21.307 INFO:teuthology.orchestra.run.smithi149.stdout:ERROR: (22) Invalid argument 2023-09-23T20:37:21.307 INFO:teuthology.orchestra.run.smithi149.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:37:21.332 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:37:21.364 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:21.404 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:37:21.407 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:21.407 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:37:21.508 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:37:21.542 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:21.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:21 smithi018 ceph-mon[184144]: pgmap v937: 119 pgs: 1 active+recovering+undersized+remapped, 1 activating+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+remapped, 110 active+clean; 5.1 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 31 KiB/s rd, 19 MiB/s wr, 333 op/s; 15902/251634 objects degraded (6.319%); 4851/251634 objects misplaced (1.928%); 3.4 MiB/s, 58 objects/s recovering 2023-09-23T20:37:21.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:21 smithi018 ceph-mon[179647]: pgmap v937: 119 pgs: 1 active+recovering+undersized+remapped, 1 activating+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+remapped, 110 active+clean; 5.1 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 31 KiB/s rd, 19 MiB/s wr, 333 op/s; 15902/251634 objects degraded (6.319%); 4851/251634 objects misplaced (1.928%); 3.4 MiB/s, 58 objects/s recovering 2023-09-23T20:37:21.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:21 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mon-a[179623]: 2023-09-23T20:37:21.491+0000 7f160fef9700 -1 mon.a@0(leader).osd e384 definitely_dead 0 2023-09-23T20:37:21.653 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:21 smithi149 ceph-mon[170394]: pgmap v937: 119 pgs: 1 active+recovering+undersized+remapped, 1 activating+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+remapped, 110 active+clean; 5.1 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 31 KiB/s rd, 19 MiB/s wr, 333 op/s; 15902/251634 objects degraded (6.319%); 4851/251634 objects misplaced (1.928%); 3.4 MiB/s, 58 objects/s recovering 2023-09-23T20:37:21.696 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:37:21.701 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:21.701 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:37:21.725 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T20:37:21.790 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T20:37:21.802 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:37:21.839 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:21.880 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:37:21.883 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:21.884 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:37:21.985 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:37:22.016 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:22.034 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 66 16 84270 84254 79.7549 0 - 0.011237 2023-09-23T20:37:22.057 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:37:22.063 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:22.063 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:37:22.164 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:37:22.193 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:22.230 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:37:22.233 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:22.233 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:37:22.333 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:37:22.334 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:37:22.362 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:22.390 INFO:teuthology.orchestra.run.smithi018.stderr:marked out osd.6. 2023-09-23T20:37:22.415 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:37:22.419 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:37:22.423 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:22.423 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:37:22.512 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:22 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1257991535' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-09-23T20:37:22.512 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:22 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-09-23T20:37:22.512 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:22 smithi018 ceph-mon[184144]: Client client.admin marked osd.6 out, while it was still marked up 2023-09-23T20:37:22.512 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:22 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1257991535' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-09-23T20:37:22.512 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:22 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-09-23T20:37:22.512 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:22 smithi018 ceph-mon[179647]: Client client.admin marked osd.6 out, while it was still marked up 2023-09-23T20:37:22.524 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:37:22.553 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:22.624 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:22 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1257991535' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-09-23T20:37:22.624 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:22 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-09-23T20:37:22.624 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:22 smithi149 ceph-mon[170394]: Client client.admin marked osd.6 out, while it was still marked up 2023-09-23T20:37:22.695 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:37:22.698 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:22.698 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:37:22.799 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:37:22.827 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:22.915 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:37:22.918 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:22.919 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:37:23.020 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:37:23.034 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 67 16 84270 84254 78.5649 0 - 0.011237 2023-09-23T20:37:23.049 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:23.095 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:37:23.099 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:23.100 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:37:23.200 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:37:23.231 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:23.395 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:23 smithi018 ceph-mon[179647]: pgmap v938: 119 pgs: 1 active+recovering+undersized+remapped, 1 activating+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 5.1 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 14 KiB/s rd, 1.7 MiB/s wr, 37 op/s; 15902/251826 objects degraded (6.315%); 4851/251826 objects misplaced (1.926%); 2.6 MiB/s, 43 objects/s recovering 2023-09-23T20:37:23.395 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:23 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 15902/251826 objects degraded (6.315%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T20:37:23.395 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:23 smithi018 ceph-mon[179647]: failed to encode map e385 with expected crc 2023-09-23T20:37:23.396 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:23 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2023-09-23T20:37:23.396 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:23 smithi018 ceph-mon[179647]: osdmap e385: 8 total, 8 up, 7 in 2023-09-23T20:37:23.396 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:23 smithi018 ceph-mon[184144]: pgmap v938: 119 pgs: 1 active+recovering+undersized+remapped, 1 activating+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 5.1 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 14 KiB/s rd, 1.7 MiB/s wr, 37 op/s; 15902/251826 objects degraded (6.315%); 4851/251826 objects misplaced (1.926%); 2.6 MiB/s, 43 objects/s recovering 2023-09-23T20:37:23.396 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:23 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 15902/251826 objects degraded (6.315%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T20:37:23.396 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:23 smithi018 ceph-mon[184144]: failed to encode map e385 with expected crc 2023-09-23T20:37:23.396 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:23 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2023-09-23T20:37:23.396 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:23 smithi018 ceph-mon[184144]: osdmap e385: 8 total, 8 up, 7 in 2023-09-23T20:37:23.449 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:37:23.458 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:23.458 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:37:23.559 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:37:23.595 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:23.647 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:37:23.651 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:23 smithi149 ceph-mon[170394]: pgmap v938: 119 pgs: 1 active+recovering+undersized+remapped, 1 activating+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 5.1 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 14 KiB/s rd, 1.7 MiB/s wr, 37 op/s; 15902/251826 objects degraded (6.315%); 4851/251826 objects misplaced (1.926%); 2.6 MiB/s, 43 objects/s recovering 2023-09-23T20:37:23.651 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:23 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 15902/251826 objects degraded (6.315%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T20:37:23.651 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:23 smithi149 ceph-mon[170394]: failed to encode map e385 with expected crc 2023-09-23T20:37:23.651 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:23 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2023-09-23T20:37:23.651 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:23 smithi149 ceph-mon[170394]: osdmap e385: 8 total, 8 up, 7 in 2023-09-23T20:37:23.651 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:23.652 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:37:23.667 INFO:teuthology.orchestra.run.smithi149.stdout:{ 2023-09-23T20:37:23.668 INFO:teuthology.orchestra.run.smithi149.stdout: "ops": [ 2023-09-23T20:37:23.668 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.668 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:84270 8.8 8.25532748 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e381)", 2023-09-23T20:37:23.668 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:37:16.141290+0000", 2023-09-23T20:37:23.668 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 7.5247988479999997, 2023-09-23T20:37:23.668 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.5248308589999997, 2023-09-23T20:37:23.668 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:37:23.668 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "queued for pg", 2023-09-23T20:37:23.668 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:37:23.669 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:37:23.669 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:37:23.669 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 84270 2023-09-23T20:37:23.669 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.669 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:37:23.669 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.669 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:37:23.669 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:16.141290+0000", 2023-09-23T20:37:23.669 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:37:23.669 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.669 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.669 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:37:23.670 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:16.141290+0000", 2023-09-23T20:37:23.670 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:37:23.670 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.670 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.670 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:37:23.670 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:16.141284+0000", 2023-09-23T20:37:23.670 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999948 2023-09-23T20:37:23.670 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.670 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.670 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:37:23.670 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:16.141322+0000", 2023-09-23T20:37:23.671 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.8040000000000002e-05 2023-09-23T20:37:23.671 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.671 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.671 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:37:23.671 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:16.141340+0000", 2023-09-23T20:37:23.671 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.7844000000000001e-05 2023-09-23T20:37:23.671 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.671 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.671 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:37:23.671 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:16.141348+0000", 2023-09-23T20:37:23.671 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.4869999999999996e-06 2023-09-23T20:37:23.672 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:37:23.672 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:37:23.672 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:37:23.672 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.672 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.672 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:84173 8.8 8.db844068 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e381)", 2023-09-23T20:37:23.672 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:37:15.773994+0000", 2023-09-23T20:37:23.672 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 7.8920944669999997, 2023-09-23T20:37:23.672 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.89219195, 2023-09-23T20:37:23.672 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:37:23.672 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "queued for pg", 2023-09-23T20:37:23.673 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:37:23.673 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:37:23.673 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:37:23.673 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 84173 2023-09-23T20:37:23.673 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.673 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:37:23.673 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.673 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:37:23.673 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:15.773994+0000", 2023-09-23T20:37:23.673 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:37:23.673 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.673 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.674 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:37:23.674 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:15.773994+0000", 2023-09-23T20:37:23.674 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:37:23.674 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.674 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.674 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:37:23.674 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:15.773993+0000", 2023-09-23T20:37:23.674 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:37:23.674 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.674 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.674 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:37:23.674 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:15.774038+0000", 2023-09-23T20:37:23.675 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.5278000000000003e-05 2023-09-23T20:37:23.675 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.675 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.675 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:37:23.675 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:15.774055+0000", 2023-09-23T20:37:23.675 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.7181999999999999e-05 2023-09-23T20:37:23.675 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.675 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.675 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:37:23.675 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:15.774065+0000", 2023-09-23T20:37:23.675 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.9229999999999997e-06 2023-09-23T20:37:23.676 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:37:23.676 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:37:23.676 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:37:23.676 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.676 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.676 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "MOSDPGPushReply(8.d 386/385 [PushReplyOp(8:bd34abb8:::benchmark_data_smithi018_341490_object88:head)])", 2023-09-23T20:37:23.676 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:37:23.665306+0000", 2023-09-23T20:37:23.676 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 0.00078264700000000003, 2023-09-23T20:37:23.676 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00091774499999999998, 2023-09-23T20:37:23.676 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:37:23.676 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:37:23.677 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:37:23.677 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.677 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:37:23.677 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:23.665306+0000", 2023-09-23T20:37:23.677 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:37:23.677 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.677 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.677 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:37:23.677 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:23.665306+0000", 2023-09-23T20:37:23.677 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:37:23.677 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.677 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.678 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:37:23.678 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:23.665305+0000", 2023-09-23T20:37:23.678 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:37:23.678 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.678 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.678 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:37:23.678 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:23.665313+0000", 2023-09-23T20:37:23.678 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.6340000000000007e-06 2023-09-23T20:37:23.678 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.678 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.678 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:37:23.679 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:23.665332+0000", 2023-09-23T20:37:23.679 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.8519999999999999e-05 2023-09-23T20:37:23.679 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.679 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.679 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:37:23.679 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:23.665339+0000", 2023-09-23T20:37:23.679 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.9809999999999997e-06 2023-09-23T20:37:23.679 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.679 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.679 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:37:23.679 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:23.665397+0000", 2023-09-23T20:37:23.680 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.7923000000000001e-05 2023-09-23T20:37:23.680 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:37:23.680 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:37:23.680 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:37:23.680 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.680 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.680 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:84133 8.8 8.9de74298 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e381)", 2023-09-23T20:37:23.680 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:37:15.686039+0000", 2023-09-23T20:37:23.680 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 7.9800493939999999, 2023-09-23T20:37:23.680 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.9802206529999999, 2023-09-23T20:37:23.680 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:37:23.680 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "queued for pg", 2023-09-23T20:37:23.681 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:37:23.681 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:37:23.681 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:37:23.681 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 84133 2023-09-23T20:37:23.681 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.681 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:37:23.681 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.681 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:37:23.681 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:15.686039+0000", 2023-09-23T20:37:23.681 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:37:23.681 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.682 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.682 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:37:23.682 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:15.686039+0000", 2023-09-23T20:37:23.682 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:37:23.682 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.682 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.682 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:37:23.682 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:15.686038+0000", 2023-09-23T20:37:23.682 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999981 2023-09-23T20:37:23.682 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.682 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.682 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:37:23.683 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:15.686083+0000", 2023-09-23T20:37:23.683 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.5479000000000003e-05 2023-09-23T20:37:23.683 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.683 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.683 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:37:23.683 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:15.686101+0000", 2023-09-23T20:37:23.683 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.7918999999999999e-05 2023-09-23T20:37:23.683 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.683 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.683 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:37:23.683 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:15.686109+0000", 2023-09-23T20:37:23.683 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.0539999999999995e-06 2023-09-23T20:37:23.684 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:37:23.684 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:37:23.684 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:37:23.684 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.684 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.684 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:84067 8.8 8.41697458 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e379)", 2023-09-23T20:37:23.684 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:37:13.364912+0000", 2023-09-23T20:37:23.684 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 10.301177026, 2023-09-23T20:37:23.684 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 10.301380186999999, 2023-09-23T20:37:23.684 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:37:23.684 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "delayed", 2023-09-23T20:37:23.684 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:37:23.685 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:37:23.685 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:37:23.685 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 84067 2023-09-23T20:37:23.685 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.685 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:37:23.685 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.685 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:37:23.685 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.364912+0000", 2023-09-23T20:37:23.685 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:37:23.685 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.685 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.686 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:37:23.686 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.364912+0000", 2023-09-23T20:37:23.686 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:37:23.686 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.686 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.686 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:37:23.686 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.364910+0000", 2023-09-23T20:37:23.686 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:37:23.686 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.686 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.686 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:37:23.687 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.365067+0000", 2023-09-23T20:37:23.687 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00015717199999999999 2023-09-23T20:37:23.687 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.687 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.687 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:37:23.687 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.365079+0000", 2023-09-23T20:37:23.687 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.2218999999999999e-05 2023-09-23T20:37:23.687 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.687 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.687 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:37:23.687 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.365087+0000", 2023-09-23T20:37:23.687 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.2760000000000003e-06 2023-09-23T20:37:23.688 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.688 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.688 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:37:23.688 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.365156+0000", 2023-09-23T20:37:23.688 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.9232000000000004e-05 2023-09-23T20:37:23.688 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.688 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.688 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:37:23.688 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.365169+0000", 2023-09-23T20:37:23.688 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.3376e-05 2023-09-23T20:37:23.688 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.689 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.689 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:37:23.689 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:14.350109+0000", 2023-09-23T20:37:23.689 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.98493943900000003 2023-09-23T20:37:23.689 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.689 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.689 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:37:23.689 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:14.350124+0000", 2023-09-23T20:37:23.689 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.4892000000000001e-05 2023-09-23T20:37:23.689 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:37:23.689 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:37:23.689 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:37:23.690 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.690 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.690 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:84078 8.8 8.8c4a4288 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e379)", 2023-09-23T20:37:23.690 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:37:13.365108+0000", 2023-09-23T20:37:23.690 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 10.30098108, 2023-09-23T20:37:23.690 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 10.301249725, 2023-09-23T20:37:23.690 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:37:23.690 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "delayed", 2023-09-23T20:37:23.690 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:37:23.690 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:37:23.690 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:37:23.691 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 84078 2023-09-23T20:37:23.691 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.691 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:37:23.691 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.691 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:37:23.691 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.365108+0000", 2023-09-23T20:37:23.691 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:37:23.691 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.691 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.691 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:37:23.691 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.365108+0000", 2023-09-23T20:37:23.691 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:37:23.692 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.692 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.692 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:37:23.692 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.365107+0000", 2023-09-23T20:37:23.692 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:37:23.692 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.692 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.692 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:37:23.692 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.365134+0000", 2023-09-23T20:37:23.692 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.6896000000000001e-05 2023-09-23T20:37:23.692 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.693 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.693 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:37:23.693 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.365142+0000", 2023-09-23T20:37:23.693 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.8259999999999993e-06 2023-09-23T20:37:23.693 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.693 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.693 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:37:23.693 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.365147+0000", 2023-09-23T20:37:23.693 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.2100000000000001e-06 2023-09-23T20:37:23.693 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.693 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.693 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:37:23.694 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.365234+0000", 2023-09-23T20:37:23.694 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.7615999999999996e-05 2023-09-23T20:37:23.694 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.694 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.694 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:37:23.694 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.365246+0000", 2023-09-23T20:37:23.694 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1131000000000001e-05 2023-09-23T20:37:23.694 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.694 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.694 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:37:23.694 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:14.350158+0000", 2023-09-23T20:37:23.695 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.98491213899999996 2023-09-23T20:37:23.695 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.695 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.695 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:37:23.695 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:14.350175+0000", 2023-09-23T20:37:23.695 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.7547999999999999e-05 2023-09-23T20:37:23.695 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:37:23.695 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:37:23.695 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:37:23.695 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.695 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.695 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:84191 8.8 8.7ab23d78 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e381)", 2023-09-23T20:37:23.696 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:37:15.830787+0000", 2023-09-23T20:37:23.696 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 7.8353017410000003, 2023-09-23T20:37:23.696 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.8356141849999998, 2023-09-23T20:37:23.696 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:37:23.696 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "queued for pg", 2023-09-23T20:37:23.696 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:37:23.696 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:37:23.696 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:37:23.696 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 84191 2023-09-23T20:37:23.696 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.696 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:37:23.696 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.697 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:37:23.697 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:15.830787+0000", 2023-09-23T20:37:23.697 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:37:23.697 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.697 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.697 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:37:23.697 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:15.830787+0000", 2023-09-23T20:37:23.697 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:37:23.697 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.697 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.697 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:37:23.698 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:15.830785+0000", 2023-09-23T20:37:23.698 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999981 2023-09-23T20:37:23.698 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.698 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.698 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:37:23.698 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:15.830827+0000", 2023-09-23T20:37:23.698 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.2339000000000002e-05 2023-09-23T20:37:23.698 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.698 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.698 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:37:23.698 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:15.830840+0000", 2023-09-23T20:37:23.698 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.3118999999999999e-05 2023-09-23T20:37:23.699 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.699 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.699 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:37:23.699 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:15.830846+0000", 2023-09-23T20:37:23.699 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.8749999999999997e-06 2023-09-23T20:37:23.699 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:37:23.699 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:37:23.699 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:37:23.699 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.699 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.699 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:84137 8.8 8.5dc6b238 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e381)", 2023-09-23T20:37:23.700 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:37:15.692928+0000", 2023-09-23T20:37:23.700 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 7.9731605339999998, 2023-09-23T20:37:23.700 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.9735029520000005, 2023-09-23T20:37:23.700 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:37:23.700 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "queued for pg", 2023-09-23T20:37:23.700 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:37:23.700 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:37:23.700 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:37:23.700 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 84137 2023-09-23T20:37:23.700 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.700 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:37:23.700 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.701 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:37:23.701 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:15.692928+0000", 2023-09-23T20:37:23.701 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:37:23.701 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.701 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.701 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:37:23.701 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:15.692928+0000", 2023-09-23T20:37:23.701 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:37:23.701 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.701 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.701 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:37:23.701 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:15.692927+0000", 2023-09-23T20:37:23.702 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:37:23.702 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.702 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.702 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:37:23.702 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:15.692982+0000", 2023-09-23T20:37:23.702 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.5213999999999997e-05 2023-09-23T20:37:23.702 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.702 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.702 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:37:23.702 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:15.692994+0000", 2023-09-23T20:37:23.702 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1107000000000001e-05 2023-09-23T20:37:23.703 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.703 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.703 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:37:23.703 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:15.693000+0000", 2023-09-23T20:37:23.703 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.5350000000000001e-06 2023-09-23T20:37:23.703 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:37:23.703 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:37:23.703 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:37:23.703 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.703 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.703 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:84101 8.8 8.33c4d18 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e379)", 2023-09-23T20:37:23.704 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:37:13.380076+0000", 2023-09-23T20:37:23.704 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 10.286012704000001, 2023-09-23T20:37:23.704 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 10.286388403, 2023-09-23T20:37:23.704 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:37:23.704 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "delayed", 2023-09-23T20:37:23.704 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:37:23.704 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:37:23.704 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:37:23.704 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 84101 2023-09-23T20:37:23.704 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.704 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:37:23.704 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.705 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:37:23.705 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.380076+0000", 2023-09-23T20:37:23.705 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:37:23.705 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.705 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.705 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:37:23.705 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.380076+0000", 2023-09-23T20:37:23.705 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:37:23.705 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.705 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.705 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:37:23.706 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.380074+0000", 2023-09-23T20:37:23.706 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:37:23.706 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.706 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.706 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:37:23.706 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.380115+0000", 2023-09-23T20:37:23.706 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.0809000000000002e-05 2023-09-23T20:37:23.706 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.706 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.706 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:37:23.706 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.380135+0000", 2023-09-23T20:37:23.706 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.0129e-05 2023-09-23T20:37:23.707 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.707 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.707 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:37:23.707 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.380144+0000", 2023-09-23T20:37:23.707 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.4400000000000005e-06 2023-09-23T20:37:23.707 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.707 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.707 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:37:23.707 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.380211+0000", 2023-09-23T20:37:23.707 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.7817000000000005e-05 2023-09-23T20:37:23.707 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.707 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.708 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:37:23.708 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.380221+0000", 2023-09-23T20:37:23.708 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.5440000000000004e-06 2023-09-23T20:37:23.708 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.708 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.708 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:37:23.708 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:14.350207+0000", 2023-09-23T20:37:23.708 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.96998578000000002 2023-09-23T20:37:23.708 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.708 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.708 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:37:23.709 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:14.350216+0000", 2023-09-23T20:37:23.709 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.9299999999999992e-06 2023-09-23T20:37:23.709 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:37:23.709 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:37:23.709 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:37:23.709 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.709 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.709 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:84107 8.8 8.d6b1a648 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e379)", 2023-09-23T20:37:23.709 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:37:13.385027+0000", 2023-09-23T20:37:23.709 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 10.281061553000001, 2023-09-23T20:37:23.709 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 10.281480921, 2023-09-23T20:37:23.710 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:37:23.710 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "delayed", 2023-09-23T20:37:23.710 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:37:23.710 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:37:23.710 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:37:23.710 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 84107 2023-09-23T20:37:23.710 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.710 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:37:23.710 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.710 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:37:23.710 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.385027+0000", 2023-09-23T20:37:23.710 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:37:23.711 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.711 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.711 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:37:23.711 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.385027+0000", 2023-09-23T20:37:23.711 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:37:23.711 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.711 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.711 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:37:23.711 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.385027+0000", 2023-09-23T20:37:23.711 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:37:23.711 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.712 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.712 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:37:23.712 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.385083+0000", 2023-09-23T20:37:23.712 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.6094999999999997e-05 2023-09-23T20:37:23.712 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.712 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.712 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:37:23.712 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.385092+0000", 2023-09-23T20:37:23.712 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.3670000000000003e-06 2023-09-23T20:37:23.712 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.712 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.712 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:37:23.713 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.385096+0000", 2023-09-23T20:37:23.713 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.6210000000000001e-06 2023-09-23T20:37:23.713 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.713 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.713 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:37:23.713 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.385156+0000", 2023-09-23T20:37:23.713 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.9840999999999998e-05 2023-09-23T20:37:23.713 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.713 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.713 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:37:23.713 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.385168+0000", 2023-09-23T20:37:23.713 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.2523e-05 2023-09-23T20:37:23.714 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.714 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.714 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:37:23.714 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:14.350342+0000", 2023-09-23T20:37:23.714 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.96517391100000005 2023-09-23T20:37:23.714 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.714 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.714 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:37:23.714 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:14.350354+0000", 2023-09-23T20:37:23.714 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.183e-05 2023-09-23T20:37:23.714 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:37:23.715 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:37:23.715 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:37:23.715 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.715 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.715 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:84146 8.8 8.be5258d8 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e381)", 2023-09-23T20:37:23.715 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:37:15.714158+0000", 2023-09-23T20:37:23.715 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 7.9519310389999998, 2023-09-23T20:37:23.715 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.9523934050000005, 2023-09-23T20:37:23.715 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:37:23.715 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "queued for pg", 2023-09-23T20:37:23.715 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:37:23.715 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:37:23.716 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:37:23.716 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 84146 2023-09-23T20:37:23.716 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.716 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:37:23.716 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.716 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:37:23.716 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:15.714158+0000", 2023-09-23T20:37:23.716 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:37:23.716 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.716 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.716 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:37:23.716 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:15.714158+0000", 2023-09-23T20:37:23.717 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:37:23.717 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.717 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.717 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:37:23.717 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:15.714156+0000", 2023-09-23T20:37:23.717 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999981 2023-09-23T20:37:23.717 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.717 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.717 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:37:23.717 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:15.714217+0000", 2023-09-23T20:37:23.717 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.1391999999999998e-05 2023-09-23T20:37:23.717 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.717 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.718 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:37:23.718 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:15.714236+0000", 2023-09-23T20:37:23.718 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.9188e-05 2023-09-23T20:37:23.718 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.718 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.718 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:37:23.718 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:15.714247+0000", 2023-09-23T20:37:23.718 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0304e-05 2023-09-23T20:37:23.718 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:37:23.718 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:37:23.718 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:37:23.719 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.719 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.719 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:84148 8.8 8.b4963108 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e381)", 2023-09-23T20:37:23.719 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:37:15.721454+0000", 2023-09-23T20:37:23.719 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 7.9446350969999999, 2023-09-23T20:37:23.719 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.9451334679999999, 2023-09-23T20:37:23.719 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:37:23.719 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "queued for pg", 2023-09-23T20:37:23.719 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:37:23.719 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:37:23.719 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:37:23.720 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 84148 2023-09-23T20:37:23.720 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.720 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:37:23.720 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.720 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:37:23.720 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:15.721454+0000", 2023-09-23T20:37:23.720 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:37:23.720 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.720 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.720 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:37:23.720 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:15.721454+0000", 2023-09-23T20:37:23.720 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:37:23.721 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.721 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.721 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:37:23.721 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:15.721452+0000", 2023-09-23T20:37:23.721 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999981 2023-09-23T20:37:23.721 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.721 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.721 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:37:23.721 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:15.721513+0000", 2023-09-23T20:37:23.721 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.0766000000000001e-05 2023-09-23T20:37:23.721 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.722 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.722 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:37:23.722 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:15.721526+0000", 2023-09-23T20:37:23.722 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.3271000000000001e-05 2023-09-23T20:37:23.722 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.722 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.722 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:37:23.722 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:15.721532+0000", 2023-09-23T20:37:23.722 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.1109999999999996e-06 2023-09-23T20:37:23.722 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:37:23.722 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:37:23.723 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:37:23.723 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.723 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.723 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:84213 8.8 8.3ba16b98 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e381)", 2023-09-23T20:37:23.723 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:37:15.897214+0000", 2023-09-23T20:37:23.723 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 7.7688741480000001, 2023-09-23T20:37:23.723 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.7694029919999998, 2023-09-23T20:37:23.723 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:37:23.723 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "queued for pg", 2023-09-23T20:37:23.723 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:37:23.723 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:37:23.724 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:37:23.724 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 84213 2023-09-23T20:37:23.724 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.724 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:37:23.724 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.724 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:37:23.724 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:15.897214+0000", 2023-09-23T20:37:23.724 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:37:23.724 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.724 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.724 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:37:23.724 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:15.897214+0000", 2023-09-23T20:37:23.725 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:37:23.725 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.725 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.725 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:37:23.725 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:15.897213+0000", 2023-09-23T20:37:23.725 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:37:23.725 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.725 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.725 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:37:23.725 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:15.897272+0000", 2023-09-23T20:37:23.725 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.8961e-05 2023-09-23T20:37:23.726 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.726 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.726 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:37:23.726 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:15.897287+0000", 2023-09-23T20:37:23.726 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.4656e-05 2023-09-23T20:37:23.726 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.726 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.726 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:37:23.726 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:15.897295+0000", 2023-09-23T20:37:23.726 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.123e-06 2023-09-23T20:37:23.726 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:37:23.726 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:37:23.727 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:37:23.727 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.727 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.727 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:84255 8.8 8.b16fe688 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e381)", 2023-09-23T20:37:23.727 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:37:16.064440+0000", 2023-09-23T20:37:23.727 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 7.601648247, 2023-09-23T20:37:23.727 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.6022073629999998, 2023-09-23T20:37:23.727 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:37:23.727 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "queued for pg", 2023-09-23T20:37:23.727 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:37:23.727 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:37:23.727 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:37:23.728 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 84255 2023-09-23T20:37:23.728 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.728 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:37:23.728 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.728 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:37:23.728 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:16.064440+0000", 2023-09-23T20:37:23.728 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:37:23.728 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.728 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.728 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:37:23.728 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:16.064440+0000", 2023-09-23T20:37:23.729 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:37:23.729 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.729 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.729 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:37:23.729 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:16.064438+0000", 2023-09-23T20:37:23.729 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999981 2023-09-23T20:37:23.729 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.729 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.729 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:37:23.729 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:16.064485+0000", 2023-09-23T20:37:23.729 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.7073999999999997e-05 2023-09-23T20:37:23.729 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.730 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.730 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:37:23.730 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:16.064565+0000", 2023-09-23T20:37:23.730 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.9803999999999997e-05 2023-09-23T20:37:23.730 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.730 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.730 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:37:23.730 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:16.064573+0000", 2023-09-23T20:37:23.730 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.0450000000000001e-06 2023-09-23T20:37:23.730 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:37:23.730 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:37:23.731 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:37:23.731 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.731 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.731 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:84156 8.8 8.d91e0758 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e381)", 2023-09-23T20:37:23.731 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:37:15.745716+0000", 2023-09-23T20:37:23.731 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 7.9203729950000001, 2023-09-23T20:37:23.731 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.9209636080000001, 2023-09-23T20:37:23.731 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:37:23.731 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "queued for pg", 2023-09-23T20:37:23.731 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:37:23.731 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:37:23.731 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:37:23.732 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 84156 2023-09-23T20:37:23.732 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.732 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:37:23.732 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.732 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:37:23.732 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:15.745716+0000", 2023-09-23T20:37:23.732 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:37:23.732 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.732 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.732 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:37:23.732 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:15.745716+0000", 2023-09-23T20:37:23.732 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:37:23.733 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.733 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.733 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:37:23.733 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:15.745713+0000", 2023-09-23T20:37:23.733 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999971 2023-09-23T20:37:23.733 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.733 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.733 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:37:23.733 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:15.745800+0000", 2023-09-23T20:37:23.733 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.7334000000000001e-05 2023-09-23T20:37:23.733 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.734 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.734 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:37:23.734 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:15.745825+0000", 2023-09-23T20:37:23.734 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.4630999999999998e-05 2023-09-23T20:37:23.734 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.734 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.734 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:37:23.734 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:15.745838+0000", 2023-09-23T20:37:23.734 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.293e-05 2023-09-23T20:37:23.734 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:37:23.734 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:37:23.735 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:37:23.735 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.735 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.735 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:84158 8.8 8.6faa2e28 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e381)", 2023-09-23T20:37:23.735 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:37:15.745852+0000", 2023-09-23T20:37:23.735 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 7.9202368090000004, 2023-09-23T20:37:23.735 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.9208573070000003, 2023-09-23T20:37:23.735 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:37:23.735 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "queued for pg", 2023-09-23T20:37:23.735 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:37:23.735 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:37:23.736 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:37:23.736 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 84158 2023-09-23T20:37:23.736 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.736 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:37:23.736 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.736 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:37:23.736 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:15.745852+0000", 2023-09-23T20:37:23.736 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:37:23.736 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.736 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.736 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:37:23.736 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:15.745852+0000", 2023-09-23T20:37:23.737 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:37:23.737 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.737 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.737 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:37:23.737 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:15.745851+0000", 2023-09-23T20:37:23.737 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:37:23.737 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.737 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.737 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:37:23.737 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:15.745920+0000", 2023-09-23T20:37:23.737 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.8277999999999999e-05 2023-09-23T20:37:23.738 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.738 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.738 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:37:23.738 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:15.745927+0000", 2023-09-23T20:37:23.738 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.7540000000000006e-06 2023-09-23T20:37:23.738 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.738 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.738 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:37:23.738 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:15.745932+0000", 2023-09-23T20:37:23.738 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.0710000000000002e-06 2023-09-23T20:37:23.738 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:37:23.739 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:37:23.739 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:37:23.739 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.739 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.739 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:84160 8.8 8.6eac6168 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e381)", 2023-09-23T20:37:23.739 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:37:15.747064+0000", 2023-09-23T20:37:23.739 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 7.9190245859999999, 2023-09-23T20:37:23.739 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.9196884880000002, 2023-09-23T20:37:23.739 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:37:23.739 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "queued for pg", 2023-09-23T20:37:23.739 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:37:23.739 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:37:23.740 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:37:23.740 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 84160 2023-09-23T20:37:23.740 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.740 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:37:23.740 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.740 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:37:23.740 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:15.747064+0000", 2023-09-23T20:37:23.740 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:37:23.740 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.740 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.740 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:37:23.740 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:15.747064+0000", 2023-09-23T20:37:23.741 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:37:23.741 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.741 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.741 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:37:23.741 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:15.747062+0000", 2023-09-23T20:37:23.741 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999981 2023-09-23T20:37:23.741 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.741 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.741 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:37:23.741 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:15.747140+0000", 2023-09-23T20:37:23.741 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.7566999999999998e-05 2023-09-23T20:37:23.741 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.742 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.742 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:37:23.742 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:15.747157+0000", 2023-09-23T20:37:23.742 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.7223000000000001e-05 2023-09-23T20:37:23.742 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:23.742 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:23.742 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:37:23.742 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:15.747166+0000", 2023-09-23T20:37:23.742 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.5779999999999999e-06 2023-09-23T20:37:23.742 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:37:23.742 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:37:23.742 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:37:23.743 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:37:23.743 INFO:teuthology.orchestra.run.smithi149.stdout: ], 2023-09-23T20:37:23.743 INFO:teuthology.orchestra.run.smithi149.stdout: "num_ops": 17 2023-09-23T20:37:23.743 INFO:teuthology.orchestra.run.smithi149.stdout:} 2023-09-23T20:37:23.753 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:37:23.785 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:23.977 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:37:23.980 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:23.981 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:37:24.017 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T20:37:24.034 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 68 16 84270 84254 77.4099 0 - 0.011237 2023-09-23T20:37:24.081 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:37:24.132 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:24.172 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:37:24.175 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:24.176 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:37:24.277 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:37:24.311 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:24.353 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:37:24.356 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:24.356 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:37:24.393 INFO:teuthology.orchestra.run.smithi018.stderr:noscrub is set 2023-09-23T20:37:24.457 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:37:24.520 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:24.561 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:37:24.563 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:24 smithi149 ceph-mon[170394]: osdmap e386: 8 total, 8 up, 7 in 2023-09-23T20:37:24.563 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:24 smithi149 ceph-mon[170394]: failed to encode map e386 with expected crc 2023-09-23T20:37:24.563 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:24 smithi149 ceph-mon[170394]: failed to encode map e386 with expected crc 2023-09-23T20:37:24.563 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:24 smithi149 ceph-mon[170394]: failed to encode map e386 with expected crc 2023-09-23T20:37:24.563 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:24 smithi149 ceph-mon[170394]: failed to encode map e386 with expected crc 2023-09-23T20:37:24.564 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:24 smithi149 ceph-mon[170394]: failed to encode map e386 with expected crc 2023-09-23T20:37:24.564 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:24 smithi149 ceph-mon[170394]: failed to encode map e386 with expected crc 2023-09-23T20:37:24.564 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:24 smithi149 ceph-mon[170394]: failed to encode map e386 with expected crc 2023-09-23T20:37:24.564 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:24 smithi149 ceph-mon[170394]: failed to encode map e386 with expected crc 2023-09-23T20:37:24.564 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:24 smithi149 ceph-mon[170394]: failed to encode map e386 with expected crc 2023-09-23T20:37:24.564 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:24 smithi149 ceph-mon[170394]: failed to encode map e386 with expected crc 2023-09-23T20:37:24.564 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:24 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2766999381' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:37:24.564 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:24 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:37:24.565 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:24.565 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:37:24.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:24 smithi018 ceph-mon[179647]: osdmap e386: 8 total, 8 up, 7 in 2023-09-23T20:37:24.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:24 smithi018 ceph-mon[179647]: failed to encode map e386 with expected crc 2023-09-23T20:37:24.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:24 smithi018 ceph-mon[179647]: failed to encode map e386 with expected crc 2023-09-23T20:37:24.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:24 smithi018 ceph-mon[179647]: failed to encode map e386 with expected crc 2023-09-23T20:37:24.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:24 smithi018 ceph-mon[179647]: failed to encode map e386 with expected crc 2023-09-23T20:37:24.644 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:24 smithi018 ceph-mon[179647]: failed to encode map e386 with expected crc 2023-09-23T20:37:24.644 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:24 smithi018 ceph-mon[179647]: failed to encode map e386 with expected crc 2023-09-23T20:37:24.644 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:24 smithi018 ceph-mon[179647]: failed to encode map e386 with expected crc 2023-09-23T20:37:24.644 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:24 smithi018 ceph-mon[179647]: failed to encode map e386 with expected crc 2023-09-23T20:37:24.644 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:24 smithi018 ceph-mon[179647]: failed to encode map e386 with expected crc 2023-09-23T20:37:24.644 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:24 smithi018 ceph-mon[179647]: failed to encode map e386 with expected crc 2023-09-23T20:37:24.645 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:24 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2766999381' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:37:24.645 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:24 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:37:24.645 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:24 smithi018 ceph-mon[184144]: osdmap e386: 8 total, 8 up, 7 in 2023-09-23T20:37:24.645 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:24 smithi018 ceph-mon[184144]: failed to encode map e386 with expected crc 2023-09-23T20:37:24.645 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:24 smithi018 ceph-mon[184144]: failed to encode map e386 with expected crc 2023-09-23T20:37:24.646 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:24 smithi018 ceph-mon[184144]: failed to encode map e386 with expected crc 2023-09-23T20:37:24.646 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:24 smithi018 ceph-mon[184144]: failed to encode map e386 with expected crc 2023-09-23T20:37:24.646 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:24 smithi018 ceph-mon[184144]: failed to encode map e386 with expected crc 2023-09-23T20:37:24.646 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:24 smithi018 ceph-mon[184144]: failed to encode map e386 with expected crc 2023-09-23T20:37:24.646 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:24 smithi018 ceph-mon[184144]: failed to encode map e386 with expected crc 2023-09-23T20:37:24.647 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:24 smithi018 ceph-mon[184144]: failed to encode map e386 with expected crc 2023-09-23T20:37:24.647 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:24 smithi018 ceph-mon[184144]: failed to encode map e386 with expected crc 2023-09-23T20:37:24.647 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:24 smithi018 ceph-mon[184144]: failed to encode map e386 with expected crc 2023-09-23T20:37:24.647 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:24 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2766999381' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:37:24.647 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:24 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:37:24.666 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:37:24.695 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:24.740 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:37:24.743 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:37:24.745 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:24.745 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:37:24.846 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:37:24.876 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:25.035 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 69 16 84270 84254 76.2879 0 - 0.011237 2023-09-23T20:37:25.141 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:37:24 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:37:24] "GET /metrics HTTP/1.1" 200 36536 "" "Prometheus/2.43.0" 2023-09-23T20:37:25.617 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:37:25.620 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:25.620 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:37:25.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:25 smithi018 ceph-mon[179647]: pgmap v941: 119 pgs: 1 activating+undersized+degraded+remapped, 4 peering, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 5.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 1.2 MiB/s wr, 19 op/s; 15902/251826 objects degraded (6.315%); 4851/251826 objects misplaced (1.926%); 2.6 MiB/s, 42 objects/s recovering 2023-09-23T20:37:25.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:25 smithi018 ceph-mon[179647]: Health check update: Reduced data availability: 2 pgs inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T20:37:25.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:25 smithi018 ceph-mon[179647]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:37:25.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:25 smithi018 ceph-mon[179647]: failed to encode map e387 with expected crc 2023-09-23T20:37:25.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:25 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:37:25.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:25 smithi018 ceph-mon[179647]: osdmap e387: 8 total, 8 up, 7 in 2023-09-23T20:37:25.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:25 smithi018 ceph-mon[179647]: failed to encode map e387 with expected crc 2023-09-23T20:37:25.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:25 smithi018 ceph-mon[179647]: failed to encode map e387 with expected crc 2023-09-23T20:37:25.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:25 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:37:25.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:25 smithi018 ceph-mon[184144]: pgmap v941: 119 pgs: 1 activating+undersized+degraded+remapped, 4 peering, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 5.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 1.2 MiB/s wr, 19 op/s; 15902/251826 objects degraded (6.315%); 4851/251826 objects misplaced (1.926%); 2.6 MiB/s, 42 objects/s recovering 2023-09-23T20:37:25.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:25 smithi018 ceph-mon[184144]: Health check update: Reduced data availability: 2 pgs inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T20:37:25.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:25 smithi018 ceph-mon[184144]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:37:25.643 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:25 smithi018 ceph-mon[184144]: failed to encode map e387 with expected crc 2023-09-23T20:37:25.643 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:25 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:37:25.643 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:25 smithi018 ceph-mon[184144]: osdmap e387: 8 total, 8 up, 7 in 2023-09-23T20:37:25.643 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:25 smithi018 ceph-mon[184144]: failed to encode map e387 with expected crc 2023-09-23T20:37:25.643 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:25 smithi018 ceph-mon[184144]: failed to encode map e387 with expected crc 2023-09-23T20:37:25.643 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:25 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:37:25.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:25 smithi149 ceph-mon[170394]: pgmap v941: 119 pgs: 1 activating+undersized+degraded+remapped, 4 peering, 1 active+recovering+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 5.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 1.2 MiB/s wr, 19 op/s; 15902/251826 objects degraded (6.315%); 4851/251826 objects misplaced (1.926%); 2.6 MiB/s, 42 objects/s recovering 2023-09-23T20:37:25.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:25 smithi149 ceph-mon[170394]: Health check update: Reduced data availability: 2 pgs inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T20:37:25.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:25 smithi149 ceph-mon[170394]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:37:25.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:25 smithi149 ceph-mon[170394]: failed to encode map e387 with expected crc 2023-09-23T20:37:25.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:25 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:37:25.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:25 smithi149 ceph-mon[170394]: osdmap e387: 8 total, 8 up, 7 in 2023-09-23T20:37:25.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:25 smithi149 ceph-mon[170394]: failed to encode map e387 with expected crc 2023-09-23T20:37:25.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:25 smithi149 ceph-mon[170394]: failed to encode map e387 with expected crc 2023-09-23T20:37:25.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:25 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:37:25.722 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:37:25.750 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:25.789 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:37:25.792 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:25.793 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:37:25.894 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:37:25.927 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:25.966 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:37:25.968 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:25.969 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:37:26.035 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 70 16 84270 84254 75.1984 0 - 0.011237 2023-09-23T20:37:26.070 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:37:26.099 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:26.133 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:37:26.136 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:26.136 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:37:26.237 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:37:26.266 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:26.456 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:37:26.462 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:26.462 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:37:26.519 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:26 smithi149 ceph-mon[170394]: osdmap e388: 8 total, 8 up, 7 in 2023-09-23T20:37:26.519 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:26 smithi149 ceph-mon[170394]: failed to encode map e388 with expected crc 2023-09-23T20:37:26.519 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:26 smithi149 ceph-mon[170394]: failed to encode map e388 with expected crc 2023-09-23T20:37:26.520 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:26 smithi149 ceph-mon[170394]: failed to encode map e388 with expected crc 2023-09-23T20:37:26.520 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:26 smithi149 ceph-mon[170394]: failed to encode map e388 with expected crc 2023-09-23T20:37:26.520 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:26 smithi149 ceph-mon[170394]: failed to encode map e388 with expected crc 2023-09-23T20:37:26.520 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:26 smithi149 ceph-mon[170394]: failed to encode map e388 with expected crc 2023-09-23T20:37:26.520 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:26 smithi149 ceph-mon[170394]: failed to encode map e388 with expected crc 2023-09-23T20:37:26.563 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:37:26.593 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:26.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:26 smithi018 ceph-mon[179647]: osdmap e388: 8 total, 8 up, 7 in 2023-09-23T20:37:26.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:26 smithi018 ceph-mon[179647]: failed to encode map e388 with expected crc 2023-09-23T20:37:26.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:26 smithi018 ceph-mon[179647]: failed to encode map e388 with expected crc 2023-09-23T20:37:26.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:26 smithi018 ceph-mon[179647]: failed to encode map e388 with expected crc 2023-09-23T20:37:26.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:26 smithi018 ceph-mon[179647]: failed to encode map e388 with expected crc 2023-09-23T20:37:26.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:26 smithi018 ceph-mon[179647]: failed to encode map e388 with expected crc 2023-09-23T20:37:26.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:26 smithi018 ceph-mon[179647]: failed to encode map e388 with expected crc 2023-09-23T20:37:26.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:26 smithi018 ceph-mon[179647]: failed to encode map e388 with expected crc 2023-09-23T20:37:26.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:26 smithi018 ceph-mon[184144]: osdmap e388: 8 total, 8 up, 7 in 2023-09-23T20:37:26.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:26 smithi018 ceph-mon[184144]: failed to encode map e388 with expected crc 2023-09-23T20:37:26.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:26 smithi018 ceph-mon[184144]: failed to encode map e388 with expected crc 2023-09-23T20:37:26.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:26 smithi018 ceph-mon[184144]: failed to encode map e388 with expected crc 2023-09-23T20:37:26.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:26 smithi018 ceph-mon[184144]: failed to encode map e388 with expected crc 2023-09-23T20:37:26.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:26 smithi018 ceph-mon[184144]: failed to encode map e388 with expected crc 2023-09-23T20:37:26.643 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:26 smithi018 ceph-mon[184144]: failed to encode map e388 with expected crc 2023-09-23T20:37:26.643 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:26 smithi018 ceph-mon[184144]: failed to encode map e388 with expected crc 2023-09-23T20:37:26.793 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:37:26.797 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:26.797 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:37:26.898 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:37:26.914 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T20:37:26.929 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:26.969 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:37:26.971 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:26.972 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:37:27.035 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 71 16 84270 84254 74.1395 0 - 0.011237 2023-09-23T20:37:27.072 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:37:27.104 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:27.149 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:37:27.152 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:27.152 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:37:27.253 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:37:27.281 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:27.321 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:37:27.326 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:27.327 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:37:27.427 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:37:27.461 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:27.560 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:37:27.563 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:27.564 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:37:27.575 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:37:27.637 INFO:teuthology.orchestra.run.smithi149.stdout:{ 2023-09-23T20:37:27.637 INFO:teuthology.orchestra.run.smithi149.stdout: "ops": [], 2023-09-23T20:37:27.637 INFO:teuthology.orchestra.run.smithi149.stdout: "complaint_time": 30, 2023-09-23T20:37:27.637 INFO:teuthology.orchestra.run.smithi149.stdout: "num_blocked_ops": 0 2023-09-23T20:37:27.637 INFO:teuthology.orchestra.run.smithi149.stdout:} 2023-09-23T20:37:27.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:27 smithi018 ceph-mon[184144]: pgmap v944: 119 pgs: 1 active+recovering+undersized+remapped, 5 remapped+peering, 1 activating+undersized+degraded+remapped, 10 peering, 6 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 15 KiB/s rd, 41 KiB/s wr, 12 op/s; 26432/251826 objects degraded (10.496%); 4483/251826 objects misplaced (1.780%); 5.8 MiB/s, 92 objects/s recovering 2023-09-23T20:37:27.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:27 smithi018 ceph-mon[184144]: failed to encode map e389 with expected crc 2023-09-23T20:37:27.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:27 smithi018 ceph-mon[184144]: osdmap e389: 8 total, 8 up, 7 in 2023-09-23T20:37:27.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:27 smithi018 ceph-mon[184144]: failed to encode map e389 with expected crc 2023-09-23T20:37:27.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:27 smithi018 ceph-mon[184144]: failed to encode map e389 with expected crc 2023-09-23T20:37:27.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:27 smithi018 ceph-mon[179647]: pgmap v944: 119 pgs: 1 active+recovering+undersized+remapped, 5 remapped+peering, 1 activating+undersized+degraded+remapped, 10 peering, 6 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 15 KiB/s rd, 41 KiB/s wr, 12 op/s; 26432/251826 objects degraded (10.496%); 4483/251826 objects misplaced (1.780%); 5.8 MiB/s, 92 objects/s recovering 2023-09-23T20:37:27.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:27 smithi018 ceph-mon[179647]: failed to encode map e389 with expected crc 2023-09-23T20:37:27.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:27 smithi018 ceph-mon[179647]: osdmap e389: 8 total, 8 up, 7 in 2023-09-23T20:37:27.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:27 smithi018 ceph-mon[179647]: failed to encode map e389 with expected crc 2023-09-23T20:37:27.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:27 smithi018 ceph-mon[179647]: failed to encode map e389 with expected crc 2023-09-23T20:37:27.664 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:37:27.665 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:27 smithi149 ceph-mon[170394]: pgmap v944: 119 pgs: 1 active+recovering+undersized+remapped, 5 remapped+peering, 1 activating+undersized+degraded+remapped, 10 peering, 6 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 15 KiB/s rd, 41 KiB/s wr, 12 op/s; 26432/251826 objects degraded (10.496%); 4483/251826 objects misplaced (1.780%); 5.8 MiB/s, 92 objects/s recovering 2023-09-23T20:37:27.665 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:27 smithi149 ceph-mon[170394]: failed to encode map e389 with expected crc 2023-09-23T20:37:27.665 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:27 smithi149 ceph-mon[170394]: osdmap e389: 8 total, 8 up, 7 in 2023-09-23T20:37:27.665 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:27 smithi149 ceph-mon[170394]: failed to encode map e389 with expected crc 2023-09-23T20:37:27.666 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:27 smithi149 ceph-mon[170394]: failed to encode map e389 with expected crc 2023-09-23T20:37:27.694 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:27.705 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 5, 7, 3, 2, 4] out_osds: [6] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T20:37:27.705 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:37:27.705 INFO:tasks.thrashosds.thrasher:Removing osd 5, in_osds are: [0, 1, 5, 7, 3, 2, 4] 2023-09-23T20:37:27.705 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd out 5 2023-09-23T20:37:27.785 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:37:27.790 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:27.790 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:37:27.891 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:37:27.922 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:27.968 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:37:27.971 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:27.972 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:37:28.035 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 72 16 84270 84254 73.1101 0 - 0.011237 2023-09-23T20:37:28.042 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T20:37:28.072 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:37:28.111 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:28.341 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:37:28.345 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:28.345 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:37:28.446 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:37:28.475 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:28.518 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:37:28.520 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:28.521 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:37:28.594 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:28 smithi018 ceph-mon[179647]: failed to encode map e389 with expected crc 2023-09-23T20:37:28.594 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:28 smithi018 ceph-mon[179647]: failed to encode map e389 with expected crc 2023-09-23T20:37:28.595 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:28 smithi018 ceph-mon[184144]: failed to encode map e389 with expected crc 2023-09-23T20:37:28.595 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:28 smithi018 ceph-mon[184144]: failed to encode map e389 with expected crc 2023-09-23T20:37:28.606 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:37:28.622 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:37:28.661 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:28.689 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:37:28.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:28 smithi149 ceph-mon[170394]: failed to encode map e389 with expected crc 2023-09-23T20:37:28.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:28 smithi149 ceph-mon[170394]: failed to encode map e389 with expected crc 2023-09-23T20:37:28.786 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:37:28.789 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:28.790 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:37:28.890 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:37:28.934 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:28.977 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:37:28.981 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:28.982 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:37:29.035 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 73 16 84270 84254 72.1089 0 - 0.011237 2023-09-23T20:37:29.082 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:37:29.113 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:29.402 INFO:teuthology.orchestra.run.smithi018.stderr:nodeep-scrub is set 2023-09-23T20:37:29.519 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:29 smithi018 ceph-mon[184144]: pgmap v946: 119 pgs: 1 activating+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 5 remapped+peering, 6 peering, 6 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 5.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 13 KiB/s rd, 36 KiB/s wr, 10 op/s; 26432/251826 objects degraded (10.496%); 4483/251826 objects misplaced (1.780%); 5.0 MiB/s, 80 objects/s recovering 2023-09-23T20:37:29.519 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:29 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/595015129' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:37:29.519 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:29 smithi018 ceph-mon[179647]: pgmap v946: 119 pgs: 1 activating+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 5 remapped+peering, 6 peering, 6 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 5.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 13 KiB/s rd, 36 KiB/s wr, 10 op/s; 26432/251826 objects degraded (10.496%); 4483/251826 objects misplaced (1.780%); 5.0 MiB/s, 80 objects/s recovering 2023-09-23T20:37:29.519 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:29 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/595015129' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:37:29.523 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:37:29.526 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:29.526 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:37:29.627 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:37:29.659 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:29.698 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:37:29.703 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:29.703 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:37:29.704 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:29 smithi149 ceph-mon[170394]: pgmap v946: 119 pgs: 1 activating+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 5 remapped+peering, 6 peering, 6 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 5.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 13 KiB/s rd, 36 KiB/s wr, 10 op/s; 26432/251826 objects degraded (10.496%); 4483/251826 objects misplaced (1.780%); 5.0 MiB/s, 80 objects/s recovering 2023-09-23T20:37:29.705 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:29 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/595015129' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:37:29.804 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:37:29.836 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:29.919 INFO:teuthology.orchestra.run.smithi149.stdout:{ 2023-09-23T20:37:29.919 INFO:teuthology.orchestra.run.smithi149.stdout: "size": 20, 2023-09-23T20:37:29.920 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 600, 2023-09-23T20:37:29.920 INFO:teuthology.orchestra.run.smithi149.stdout: "ops": [ 2023-09-23T20:37:29.920 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.920 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.45161.0:68446 6.5 6:aa6b0392:::benchmark_data_smithi018_270558_object68445:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e229)", 2023-09-23T20:37:29.920 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:28:04.006452+0000", 2023-09-23T20:37:29.920 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 565.90997753099998, 2023-09-23T20:37:29.920 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.1775093220000001, 2023-09-23T20:37:29.920 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:37:29.920 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:37:29.920 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:37:29.920 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.45161", 2023-09-23T20:37:29.921 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3795614987", 2023-09-23T20:37:29.921 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 68446 2023-09-23T20:37:29.921 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.921 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:37:29.921 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.921 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:37:29.921 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.006452+0000", 2023-09-23T20:37:29.921 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:37:29.921 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.921 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.921 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:37:29.921 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.006452+0000", 2023-09-23T20:37:29.922 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:37:29.922 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.922 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.922 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:37:29.922 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.006451+0000", 2023-09-23T20:37:29.922 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:37:29.922 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.922 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.922 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:37:29.922 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.006483+0000", 2023-09-23T20:37:29.923 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.1628999999999998e-05 2023-09-23T20:37:29.923 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.923 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.923 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:37:29.923 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.006492+0000", 2023-09-23T20:37:29.923 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.85e-06 2023-09-23T20:37:29.923 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.923 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.923 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:37:29.923 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.006496+0000", 2023-09-23T20:37:29.923 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.6800000000000001e-06 2023-09-23T20:37:29.923 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.924 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.924 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:37:29.924 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.009246+0000", 2023-09-23T20:37:29.924 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.002750106 2023-09-23T20:37:29.924 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.924 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.924 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:37:29.924 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.009403+0000", 2023-09-23T20:37:29.924 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00015714200000000001 2023-09-23T20:37:29.924 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.924 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.925 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 3,6", 2023-09-23T20:37:29.925 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.009573+0000", 2023-09-23T20:37:29.925 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00016996700000000001 2023-09-23T20:37:29.925 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.925 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.925 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:37:29.925 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.012816+0000", 2023-09-23T20:37:29.925 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0032425520000000001 2023-09-23T20:37:29.925 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.925 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.925 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:37:29.925 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.039131+0000", 2023-09-23T20:37:29.926 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.026315412 2023-09-23T20:37:29.926 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.926 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.926 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:37:29.926 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.051095+0000", 2023-09-23T20:37:29.926 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.011963381 2023-09-23T20:37:29.926 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.926 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.926 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:37:29.926 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.051104+0000", 2023-09-23T20:37:29.926 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.3260000000000002e-06 2023-09-23T20:37:29.927 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.927 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.927 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:37:29.927 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:05.014602+0000", 2023-09-23T20:37:29.927 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.96349793900000003 2023-09-23T20:37:29.927 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.927 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.927 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:37:29.927 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:05.014610+0000", 2023-09-23T20:37:29.927 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.1079999999999993e-06 2023-09-23T20:37:29.927 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.927 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.928 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:37:29.928 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.183859+0000", 2023-09-23T20:37:29.928 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.169248861 2023-09-23T20:37:29.928 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.928 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.928 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:37:29.928 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.183961+0000", 2023-09-23T20:37:29.928 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00010183099999999999 2023-09-23T20:37:29.928 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:37:29.928 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:37:29.928 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:37:29.928 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.929 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.929 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.45161.0:68457 6.3 6:c96b4b0b:::benchmark_data_smithi018_270558_object68456:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e229)", 2023-09-23T20:37:29.929 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:28:04.017421+0000", 2023-09-23T20:37:29.929 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 565.89900798999997, 2023-09-23T20:37:29.929 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.1984804609999999, 2023-09-23T20:37:29.929 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:37:29.929 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:37:29.929 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:37:29.929 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.45161", 2023-09-23T20:37:29.929 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3795614987", 2023-09-23T20:37:29.929 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 68457 2023-09-23T20:37:29.930 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.930 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:37:29.930 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.930 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:37:29.930 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.017421+0000", 2023-09-23T20:37:29.930 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:37:29.930 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.930 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.930 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:37:29.930 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.017421+0000", 2023-09-23T20:37:29.930 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:37:29.930 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.931 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.931 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:37:29.931 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.017419+0000", 2023-09-23T20:37:29.931 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999976 2023-09-23T20:37:29.931 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.931 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.931 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:37:29.931 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.017470+0000", 2023-09-23T20:37:29.931 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.1082999999999999e-05 2023-09-23T20:37:29.931 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.931 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.932 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:37:29.932 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.017490+0000", 2023-09-23T20:37:29.932 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.9848999999999999e-05 2023-09-23T20:37:29.932 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.932 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.932 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:37:29.932 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.017500+0000", 2023-09-23T20:37:29.932 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0094e-05 2023-09-23T20:37:29.932 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.932 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.932 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:37:29.932 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.021025+0000", 2023-09-23T20:37:29.933 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0035254679999999999 2023-09-23T20:37:29.933 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.933 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.933 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:37:29.933 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.021234+0000", 2023-09-23T20:37:29.933 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000208805 2023-09-23T20:37:29.933 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.933 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.933 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 2,6", 2023-09-23T20:37:29.933 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.021415+0000", 2023-09-23T20:37:29.933 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00018162199999999999 2023-09-23T20:37:29.933 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.934 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.934 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:37:29.934 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.039193+0000", 2023-09-23T20:37:29.934 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.017777167999999999 2023-09-23T20:37:29.934 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.934 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.937 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:37:29.937 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.058121+0000", 2023-09-23T20:37:29.937 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.018928479000000002 2023-09-23T20:37:29.937 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.937 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.937 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:37:29.937 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.058131+0000", 2023-09-23T20:37:29.937 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0193999999999999e-05 2023-09-23T20:37:29.937 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.937 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.938 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:37:29.938 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:05.013114+0000", 2023-09-23T20:37:29.938 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.95498276500000001 2023-09-23T20:37:29.938 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.938 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.938 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:37:29.938 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:05.013126+0000", 2023-09-23T20:37:29.938 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.2140999999999999e-05 2023-09-23T20:37:29.938 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.938 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.938 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:37:29.939 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.163597+0000", 2023-09-23T20:37:29.939 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1504710300000001 2023-09-23T20:37:29.939 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.939 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.939 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for degraded object", 2023-09-23T20:37:29.939 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.164537+0000", 2023-09-23T20:37:29.939 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00094005900000000003 2023-09-23T20:37:29.939 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.939 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.939 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:37:29.939 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.215805+0000", 2023-09-23T20:37:29.939 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.05126737 2023-09-23T20:37:29.939 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.940 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.940 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:37:29.940 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.215902+0000", 2023-09-23T20:37:29.940 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.6806999999999997e-05 2023-09-23T20:37:29.940 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:37:29.940 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:37:29.940 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:37:29.940 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.940 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.940 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.45161.0:68471 6.5 6:a85ec5ad:::benchmark_data_smithi018_270558_object68470:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e230)", 2023-09-23T20:37:29.940 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:28:04.058014+0000", 2023-09-23T20:37:29.941 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 565.85841479999999, 2023-09-23T20:37:29.941 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.1399790890000001, 2023-09-23T20:37:29.941 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:37:29.941 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:37:29.941 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:37:29.941 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.45161", 2023-09-23T20:37:29.941 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3795614987", 2023-09-23T20:37:29.941 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 68471 2023-09-23T20:37:29.941 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.941 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:37:29.941 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.942 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:37:29.942 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.058014+0000", 2023-09-23T20:37:29.942 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:37:29.942 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.942 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.942 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:37:29.942 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.058014+0000", 2023-09-23T20:37:29.942 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:37:29.942 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.942 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.942 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:37:29.942 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.058013+0000", 2023-09-23T20:37:29.943 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:37:29.943 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.943 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.943 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:37:29.943 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.058072+0000", 2023-09-23T20:37:29.943 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.8891999999999998e-05 2023-09-23T20:37:29.943 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.943 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.943 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:37:29.943 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.058083+0000", 2023-09-23T20:37:29.943 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1775e-05 2023-09-23T20:37:29.943 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.944 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.944 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:37:29.944 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.058091+0000", 2023-09-23T20:37:29.944 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.6680000000000001e-06 2023-09-23T20:37:29.944 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.944 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.944 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:37:29.944 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.058348+0000", 2023-09-23T20:37:29.944 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00025689700000000002 2023-09-23T20:37:29.944 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.944 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.944 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:37:29.945 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.058358+0000", 2023-09-23T20:37:29.945 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.023e-05 2023-09-23T20:37:29.945 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.945 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.945 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:37:29.945 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:05.014631+0000", 2023-09-23T20:37:29.945 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.95627271700000005 2023-09-23T20:37:29.945 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.945 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.945 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:37:29.945 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:05.014638+0000", 2023-09-23T20:37:29.945 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.5449999999999998e-06 2023-09-23T20:37:29.946 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.946 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.946 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:37:29.946 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.183989+0000", 2023-09-23T20:37:29.946 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1693501770000001 2023-09-23T20:37:29.946 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.946 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.946 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:37:29.946 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.184136+0000", 2023-09-23T20:37:29.946 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00014772 2023-09-23T20:37:29.946 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.947 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.947 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 2,3", 2023-09-23T20:37:29.947 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.184314+0000", 2023-09-23T20:37:29.947 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00017729399999999999 2023-09-23T20:37:29.947 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.947 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.947 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:37:29.947 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.193577+0000", 2023-09-23T20:37:29.947 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0092632889999999992 2023-09-23T20:37:29.947 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.947 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.947 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:37:29.948 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.193688+0000", 2023-09-23T20:37:29.948 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00011158900000000001 2023-09-23T20:37:29.948 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.948 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.948 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:37:29.948 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.194841+0000", 2023-09-23T20:37:29.948 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0011521319999999999 2023-09-23T20:37:29.948 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.948 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.948 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:37:29.948 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.194879+0000", 2023-09-23T20:37:29.949 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.8649999999999998e-05 2023-09-23T20:37:29.949 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.949 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.949 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:37:29.949 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.197993+0000", 2023-09-23T20:37:29.949 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0031141210000000001 2023-09-23T20:37:29.949 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:37:29.949 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:37:29.949 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:37:29.949 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.949 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.949 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.45161.0:68482 6.5 6:a8dc35b6:::benchmark_data_smithi018_270558_object68481:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e230)", 2023-09-23T20:37:29.950 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:28:04.068966+0000", 2023-09-23T20:37:29.950 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 565.84746295399998, 2023-09-23T20:37:29.950 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.1433483820000001, 2023-09-23T20:37:29.950 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:37:29.950 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:37:29.950 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:37:29.950 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.45161", 2023-09-23T20:37:29.950 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3795614987", 2023-09-23T20:37:29.950 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 68482 2023-09-23T20:37:29.950 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.950 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:37:29.951 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.951 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:37:29.951 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.068966+0000", 2023-09-23T20:37:29.951 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:37:29.951 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.951 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.951 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:37:29.951 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.068966+0000", 2023-09-23T20:37:29.951 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:37:29.951 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.951 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.952 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:37:29.952 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.068965+0000", 2023-09-23T20:37:29.952 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:37:29.952 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.952 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.952 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:37:29.952 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.069008+0000", 2023-09-23T20:37:29.952 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.3155000000000003e-05 2023-09-23T20:37:29.952 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.952 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.952 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:37:29.952 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.069020+0000", 2023-09-23T20:37:29.952 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.2804e-05 2023-09-23T20:37:29.953 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.953 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.953 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:37:29.953 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.069027+0000", 2023-09-23T20:37:29.953 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.5080000000000002e-06 2023-09-23T20:37:29.953 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.953 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.953 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:37:29.953 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.069101+0000", 2023-09-23T20:37:29.953 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.4017999999999995e-05 2023-09-23T20:37:29.953 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.954 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.954 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:37:29.954 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.069111+0000", 2023-09-23T20:37:29.954 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0426e-05 2023-09-23T20:37:29.954 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.954 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.954 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:37:29.954 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:05.014659+0000", 2023-09-23T20:37:29.954 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.94554741200000003 2023-09-23T20:37:29.954 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.954 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.954 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:37:29.955 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:05.014666+0000", 2023-09-23T20:37:29.955 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.3170000000000004e-06 2023-09-23T20:37:29.955 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.955 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.955 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:37:29.955 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.187136+0000", 2023-09-23T20:37:29.955 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.172469896 2023-09-23T20:37:29.955 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.955 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.955 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:37:29.955 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.187291+0000", 2023-09-23T20:37:29.956 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000155437 2023-09-23T20:37:29.956 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.956 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.956 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 2,3", 2023-09-23T20:37:29.956 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.187476+0000", 2023-09-23T20:37:29.956 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00018494599999999999 2023-09-23T20:37:29.956 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.956 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.956 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:37:29.956 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.193695+0000", 2023-09-23T20:37:29.956 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0062182280000000001 2023-09-23T20:37:29.956 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.957 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.957 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:37:29.957 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.198072+0000", 2023-09-23T20:37:29.957 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0043777499999999997 2023-09-23T20:37:29.957 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.957 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.957 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:37:29.957 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.208878+0000", 2023-09-23T20:37:29.957 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.010805189999999999 2023-09-23T20:37:29.957 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.957 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.957 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:37:29.958 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.208936+0000", 2023-09-23T20:37:29.958 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.7957000000000001e-05 2023-09-23T20:37:29.958 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.958 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.958 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:37:29.958 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.212314+0000", 2023-09-23T20:37:29.958 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.003378902 2023-09-23T20:37:29.958 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:37:29.958 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:37:29.958 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:37:29.958 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.958 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.959 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.45161.0:68484 6.3 6:c8f6e5db:::benchmark_data_smithi018_270558_object68483:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e230)", 2023-09-23T20:37:29.959 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:28:04.071814+0000", 2023-09-23T20:37:29.959 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 565.84461548599995, 2023-09-23T20:37:29.959 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.1657179320000002, 2023-09-23T20:37:29.959 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:37:29.959 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:37:29.959 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:37:29.959 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.45161", 2023-09-23T20:37:29.959 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3795614987", 2023-09-23T20:37:29.959 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 68484 2023-09-23T20:37:29.959 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.960 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:37:29.960 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.960 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:37:29.960 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.071814+0000", 2023-09-23T20:37:29.960 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:37:29.960 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.960 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.960 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:37:29.960 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.071814+0000", 2023-09-23T20:37:29.960 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:37:29.960 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.960 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.961 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:37:29.961 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.071811+0000", 2023-09-23T20:37:29.961 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999976 2023-09-23T20:37:29.961 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.961 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.961 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:37:29.961 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.071871+0000", 2023-09-23T20:37:29.961 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.9790000000000002e-05 2023-09-23T20:37:29.961 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.961 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.961 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:37:29.962 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.071901+0000", 2023-09-23T20:37:29.962 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.9560000000000002e-05 2023-09-23T20:37:29.962 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.962 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.962 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:37:29.962 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.071909+0000", 2023-09-23T20:37:29.962 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.3890000000000005e-06 2023-09-23T20:37:29.962 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.962 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.962 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:37:29.962 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.072396+0000", 2023-09-23T20:37:29.962 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00048719099999999999 2023-09-23T20:37:29.963 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.963 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.963 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:37:29.963 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.072410+0000", 2023-09-23T20:37:29.963 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.3373000000000001e-05 2023-09-23T20:37:29.963 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.963 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.963 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:37:29.963 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:05.013152+0000", 2023-09-23T20:37:29.963 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.94074252199999997 2023-09-23T20:37:29.963 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.963 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.964 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:37:29.964 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:05.013160+0000", 2023-09-23T20:37:29.964 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.1990000000000006e-06 2023-09-23T20:37:29.964 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.964 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.964 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:37:29.964 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.215926+0000", 2023-09-23T20:37:29.964 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.2027656010000001 2023-09-23T20:37:29.964 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.964 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.964 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:37:29.965 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.216074+0000", 2023-09-23T20:37:29.965 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000148264 2023-09-23T20:37:29.965 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.965 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.965 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 1,2", 2023-09-23T20:37:29.965 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.216322+0000", 2023-09-23T20:37:29.965 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00024790499999999998 2023-09-23T20:37:29.965 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.965 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.965 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:37:29.965 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.228067+0000", 2023-09-23T20:37:29.965 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.011744747 2023-09-23T20:37:29.965 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.966 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.966 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:37:29.966 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.228080+0000", 2023-09-23T20:37:29.966 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.3380000000000001e-05 2023-09-23T20:37:29.966 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.966 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.966 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:37:29.966 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.234399+0000", 2023-09-23T20:37:29.966 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0063193010000000003 2023-09-23T20:37:29.966 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.966 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.967 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:37:29.967 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.234457+0000", 2023-09-23T20:37:29.967 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.7456e-05 2023-09-23T20:37:29.967 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.967 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.967 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:37:29.967 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.237531+0000", 2023-09-23T20:37:29.967 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0030745450000000001 2023-09-23T20:37:29.967 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:37:29.967 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:37:29.967 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:37:29.967 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.968 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.968 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.45161.0:68485 6.3 6:ceb1ac5f:::benchmark_data_smithi018_270558_object68484:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e230)", 2023-09-23T20:37:29.968 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:28:04.071924+0000", 2023-09-23T20:37:29.968 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 565.84450508700002, 2023-09-23T20:37:29.968 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.1713503310000002, 2023-09-23T20:37:29.968 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:37:29.968 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:37:29.968 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:37:29.968 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.45161", 2023-09-23T20:37:29.968 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3795614987", 2023-09-23T20:37:29.968 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 68485 2023-09-23T20:37:29.968 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.969 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:37:29.969 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.969 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:37:29.969 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.071924+0000", 2023-09-23T20:37:29.969 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:37:29.969 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.969 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.969 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:37:29.969 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.071924+0000", 2023-09-23T20:37:29.969 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:37:29.969 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.969 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.969 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:37:29.970 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.071923+0000", 2023-09-23T20:37:29.970 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:37:29.970 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.970 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.970 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:37:29.970 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.071978+0000", 2023-09-23T20:37:29.970 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.4419999999999997e-05 2023-09-23T20:37:29.970 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.970 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.970 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:37:29.970 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.071985+0000", 2023-09-23T20:37:29.971 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.0520000000000004e-06 2023-09-23T20:37:29.971 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.971 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.971 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:37:29.971 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.071989+0000", 2023-09-23T20:37:29.971 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.4850000000000003e-06 2023-09-23T20:37:29.971 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.971 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.971 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:37:29.971 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.072433+0000", 2023-09-23T20:37:29.971 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00044325099999999997 2023-09-23T20:37:29.971 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.972 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.972 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:37:29.972 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.072440+0000", 2023-09-23T20:37:29.972 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.3989999999999998e-06 2023-09-23T20:37:29.972 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.972 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.972 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:37:29.972 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:05.013184+0000", 2023-09-23T20:37:29.972 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.940744213 2023-09-23T20:37:29.972 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.972 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.972 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:37:29.972 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:05.013193+0000", 2023-09-23T20:37:29.973 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.6340000000000007e-06 2023-09-23T20:37:29.973 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.973 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.973 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:37:29.973 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.218351+0000", 2023-09-23T20:37:29.973 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.205158497 2023-09-23T20:37:29.973 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.973 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.973 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:37:29.973 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.218571+0000", 2023-09-23T20:37:29.973 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00022005200000000001 2023-09-23T20:37:29.974 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.974 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.974 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 1,2", 2023-09-23T20:37:29.974 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.218740+0000", 2023-09-23T20:37:29.974 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00016879699999999999 2023-09-23T20:37:29.974 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.974 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.974 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:37:29.974 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.228087+0000", 2023-09-23T20:37:29.974 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0093472799999999995 2023-09-23T20:37:29.974 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.974 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.975 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:37:29.975 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.230676+0000", 2023-09-23T20:37:29.975 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0025880819999999998 2023-09-23T20:37:29.975 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.975 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.975 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:37:29.975 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.240032+0000", 2023-09-23T20:37:29.975 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0093564019999999998 2023-09-23T20:37:29.975 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.975 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.975 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:37:29.975 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.240107+0000", 2023-09-23T20:37:29.976 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.5167999999999996e-05 2023-09-23T20:37:29.976 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.976 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.976 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:37:29.976 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.243274+0000", 2023-09-23T20:37:29.976 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.003167128 2023-09-23T20:37:29.976 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:37:29.976 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:37:29.976 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:37:29.976 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.976 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.976 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.45161.0:68486 6.3 6:c4625da8:::benchmark_data_smithi018_270558_object68485:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e230)", 2023-09-23T20:37:29.977 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:28:04.072143+0000", 2023-09-23T20:37:29.977 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 565.84428641299996, 2023-09-23T20:37:29.977 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.173711715, 2023-09-23T20:37:29.977 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:37:29.977 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:37:29.977 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:37:29.977 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.45161", 2023-09-23T20:37:29.977 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3795614987", 2023-09-23T20:37:29.977 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 68486 2023-09-23T20:37:29.977 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.977 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:37:29.978 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.978 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:37:29.978 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.072143+0000", 2023-09-23T20:37:29.978 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:37:29.978 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.978 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.978 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:37:29.978 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.072143+0000", 2023-09-23T20:37:29.978 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:37:29.978 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.978 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.978 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:37:29.979 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.072142+0000", 2023-09-23T20:37:29.979 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:37:29.979 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.979 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.979 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:37:29.979 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.072222+0000", 2023-09-23T20:37:29.979 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.0106999999999997e-05 2023-09-23T20:37:29.979 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.979 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.979 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:37:29.979 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.072236+0000", 2023-09-23T20:37:29.979 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.3760999999999999e-05 2023-09-23T20:37:29.980 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.980 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.980 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:37:29.980 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.072242+0000", 2023-09-23T20:37:29.980 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.7749999999999998e-06 2023-09-23T20:37:29.980 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.980 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.980 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:37:29.980 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.072471+0000", 2023-09-23T20:37:29.980 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000229779 2023-09-23T20:37:29.980 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.980 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.981 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:37:29.981 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.072479+0000", 2023-09-23T20:37:29.981 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.452e-06 2023-09-23T20:37:29.981 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.981 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.981 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:37:29.981 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:05.013217+0000", 2023-09-23T20:37:29.981 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.94073840099999995 2023-09-23T20:37:29.981 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.981 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.981 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:37:29.981 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:05.013224+0000", 2023-09-23T20:37:29.982 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.2919999999999998e-06 2023-09-23T20:37:29.982 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.982 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.982 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:37:29.982 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.221302+0000", 2023-09-23T20:37:29.982 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.208077751 2023-09-23T20:37:29.982 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.982 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.982 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:37:29.982 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.221476+0000", 2023-09-23T20:37:29.982 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00017371800000000001 2023-09-23T20:37:29.983 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.983 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.983 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 1,2", 2023-09-23T20:37:29.983 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.221657+0000", 2023-09-23T20:37:29.983 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00018101599999999999 2023-09-23T20:37:29.983 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.983 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.983 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:37:29.983 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.228092+0000", 2023-09-23T20:37:29.983 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0064346450000000001 2023-09-23T20:37:29.983 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.983 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.984 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:37:29.984 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.230712+0000", 2023-09-23T20:37:29.984 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0026200059999999998 2023-09-23T20:37:29.984 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.984 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.984 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:37:29.984 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.243356+0000", 2023-09-23T20:37:29.984 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.012643972 2023-09-23T20:37:29.984 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.984 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.984 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:37:29.984 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.243422+0000", 2023-09-23T20:37:29.985 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.6008999999999998e-05 2023-09-23T20:37:29.985 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.985 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.985 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:37:29.985 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.245854+0000", 2023-09-23T20:37:29.985 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0024327659999999998 2023-09-23T20:37:29.985 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:37:29.985 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:37:29.985 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:37:29.985 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.985 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.986 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.45161.0:68497 6.3 6:c484dd05:::benchmark_data_smithi018_270558_object68496:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e230)", 2023-09-23T20:37:29.986 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:28:04.088615+0000", 2023-09-23T20:37:29.986 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 565.82781370700002, 2023-09-23T20:37:29.986 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.1605494919999999, 2023-09-23T20:37:29.986 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:37:29.986 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:37:29.986 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:37:29.986 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.45161", 2023-09-23T20:37:29.986 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3795614987", 2023-09-23T20:37:29.986 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 68497 2023-09-23T20:37:29.986 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.987 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:37:29.987 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.987 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:37:29.987 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.088615+0000", 2023-09-23T20:37:29.987 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:37:29.987 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.987 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.987 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:37:29.987 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.088615+0000", 2023-09-23T20:37:29.987 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:37:29.987 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.987 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.988 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:37:29.988 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.088613+0000", 2023-09-23T20:37:29.988 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999981 2023-09-23T20:37:29.988 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.988 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.988 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:37:29.988 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.088682+0000", 2023-09-23T20:37:29.988 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.8730000000000001e-05 2023-09-23T20:37:29.988 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.988 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.988 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:37:29.989 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.088704+0000", 2023-09-23T20:37:29.989 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.2246999999999999e-05 2023-09-23T20:37:29.989 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.989 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.989 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:37:29.989 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.088714+0000", 2023-09-23T20:37:29.989 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.7280000000000007e-06 2023-09-23T20:37:29.989 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.989 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.989 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:37:29.989 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.088771+0000", 2023-09-23T20:37:29.989 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.6453000000000003e-05 2023-09-23T20:37:29.990 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.990 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.990 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:37:29.990 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:04.088779+0000", 2023-09-23T20:37:29.990 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.7050000000000005e-06 2023-09-23T20:37:29.990 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.990 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.990 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:37:29.990 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:05.013542+0000", 2023-09-23T20:37:29.990 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.92476320499999998 2023-09-23T20:37:29.990 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.990 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.991 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:37:29.991 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:05.013553+0000", 2023-09-23T20:37:29.991 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0118e-05 2023-09-23T20:37:29.991 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.991 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.991 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:37:29.991 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.223980+0000", 2023-09-23T20:37:29.991 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.2104271390000001 2023-09-23T20:37:29.991 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.991 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.991 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:37:29.991 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.224129+0000", 2023-09-23T20:37:29.992 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000149438 2023-09-23T20:37:29.993 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.993 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.993 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 1,2", 2023-09-23T20:37:29.993 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.224312+0000", 2023-09-23T20:37:29.993 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00018292499999999999 2023-09-23T20:37:29.993 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.994 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.994 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:37:29.994 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.228096+0000", 2023-09-23T20:37:29.994 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0037838569999999998 2023-09-23T20:37:29.994 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.994 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.994 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:37:29.994 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.230803+0000", 2023-09-23T20:37:29.994 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0027065489999999999 2023-09-23T20:37:29.994 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.994 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.994 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:37:29.995 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.245942+0000", 2023-09-23T20:37:29.995 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.015139832000000001 2023-09-23T20:37:29.995 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.995 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.995 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:37:29.995 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.245993+0000", 2023-09-23T20:37:29.995 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.0849000000000001e-05 2023-09-23T20:37:29.995 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.995 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.995 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:37:29.995 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:28:06.249165+0000", 2023-09-23T20:37:29.996 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0031716169999999998 2023-09-23T20:37:29.996 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:37:29.996 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:37:29.996 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:37:29.996 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.996 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.996 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.45161.0:193673 6.5 6:a4b932b5:::benchmark_data_smithi018_270558_object88934:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e288)", 2023-09-23T20:37:29.996 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:30:56.146817+0000", 2023-09-23T20:37:29.996 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 393.76961176600003, 2023-09-23T20:37:29.996 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0049539489999999, 2023-09-23T20:37:29.996 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:37:29.996 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:37:29.997 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:37:29.997 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.45161", 2023-09-23T20:37:29.997 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3795614987", 2023-09-23T20:37:29.997 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 193673 2023-09-23T20:37:29.997 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.997 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:37:29.997 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.997 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:37:29.997 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:56.146817+0000", 2023-09-23T20:37:29.997 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:37:29.997 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.997 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.997 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:37:29.998 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:56.146817+0000", 2023-09-23T20:37:29.998 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:37:29.998 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.998 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.998 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:37:29.998 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:56.146817+0000", 2023-09-23T20:37:29.998 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:37:29.998 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.998 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.998 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:37:29.998 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:56.146820+0000", 2023-09-23T20:37:29.998 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.5710000000000002e-06 2023-09-23T20:37:29.999 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.999 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.999 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:37:29.999 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:56.146828+0000", 2023-09-23T20:37:29.999 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.2960000000000001e-06 2023-09-23T20:37:29.999 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.999 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:29.999 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:37:29.999 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:56.146832+0000", 2023-09-23T20:37:29.999 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.8350000000000003e-06 2023-09-23T20:37:29.999 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:29.999 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.000 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:37:30.000 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:56.146935+0000", 2023-09-23T20:37:30.000 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000102823 2023-09-23T20:37:30.000 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.000 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.000 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:37:30.000 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:56.146944+0000", 2023-09-23T20:37:30.000 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.1330000000000005e-06 2023-09-23T20:37:30.000 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.000 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.000 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:37:30.001 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:57.114740+0000", 2023-09-23T20:37:30.001 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.96779585899999998 2023-09-23T20:37:30.001 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.001 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.001 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:37:30.001 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:57.114847+0000", 2023-09-23T20:37:30.001 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00010665799999999999 2023-09-23T20:37:30.001 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.001 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.001 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 3,6", 2023-09-23T20:37:30.001 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:57.115266+0000", 2023-09-23T20:37:30.001 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00041946699999999997 2023-09-23T20:37:30.002 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.002 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.002 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:37:30.002 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:57.126835+0000", 2023-09-23T20:37:30.002 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.011568983 2023-09-23T20:37:30.002 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.002 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.002 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:37:30.002 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:57.151518+0000", 2023-09-23T20:37:30.002 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.024682470000000001 2023-09-23T20:37:30.002 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.002 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.003 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:37:30.003 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:57.151681+0000", 2023-09-23T20:37:30.003 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00016317600000000001 2023-09-23T20:37:30.003 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.003 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.003 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:37:30.003 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:57.151739+0000", 2023-09-23T20:37:30.003 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.8257999999999997e-05 2023-09-23T20:37:30.003 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.003 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.003 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:37:30.004 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:57.151771+0000", 2023-09-23T20:37:30.004 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.1924000000000001e-05 2023-09-23T20:37:30.004 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:37:30.004 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:37:30.004 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:37:30.004 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.004 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.004 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.45161.0:193696 6.5 6:a5a4014d:::benchmark_data_smithi018_270558_object88957:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e288)", 2023-09-23T20:37:30.004 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:30:56.192672+0000", 2023-09-23T20:37:30.004 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 393.72375654000001, 2023-09-23T20:37:30.004 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.95918431800000004, 2023-09-23T20:37:30.004 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:37:30.005 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:37:30.005 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:37:30.005 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.45161", 2023-09-23T20:37:30.005 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3795614987", 2023-09-23T20:37:30.005 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 193696 2023-09-23T20:37:30.005 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.005 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:37:30.005 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.005 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:37:30.005 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:56.192672+0000", 2023-09-23T20:37:30.005 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:37:30.005 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.006 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.006 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:37:30.006 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:56.192672+0000", 2023-09-23T20:37:30.006 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:37:30.006 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.006 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.006 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:37:30.006 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:56.192672+0000", 2023-09-23T20:37:30.006 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:37:30.006 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.006 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.007 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:37:30.007 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:56.192676+0000", 2023-09-23T20:37:30.007 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.5480000000000002e-06 2023-09-23T20:37:30.007 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.007 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.007 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:37:30.007 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:56.192686+0000", 2023-09-23T20:37:30.007 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0149999999999999e-05 2023-09-23T20:37:30.007 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.007 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.007 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:37:30.008 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:56.192689+0000", 2023-09-23T20:37:30.008 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.0910000000000001e-06 2023-09-23T20:37:30.008 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.008 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.008 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:37:30.008 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:56.192739+0000", 2023-09-23T20:37:30.008 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.0253000000000001e-05 2023-09-23T20:37:30.008 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.008 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.008 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:37:30.008 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:56.192746+0000", 2023-09-23T20:37:30.008 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.0010000000000004e-06 2023-09-23T20:37:30.009 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.009 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.009 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:37:30.009 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:57.117501+0000", 2023-09-23T20:37:30.009 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.92475526299999999 2023-09-23T20:37:30.009 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.009 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.009 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:37:30.009 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:57.117635+0000", 2023-09-23T20:37:30.009 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00013371699999999999 2023-09-23T20:37:30.009 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.009 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.010 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 3,6", 2023-09-23T20:37:30.010 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:57.117987+0000", 2023-09-23T20:37:30.010 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00035175700000000001 2023-09-23T20:37:30.010 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.010 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.010 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:37:30.010 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:57.126842+0000", 2023-09-23T20:37:30.010 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0088546649999999994 2023-09-23T20:37:30.010 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.010 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.010 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:37:30.010 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:57.151620+0000", 2023-09-23T20:37:30.011 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.024778901999999998 2023-09-23T20:37:30.011 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.011 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.011 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:37:30.011 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:57.151810+0000", 2023-09-23T20:37:30.011 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00018969399999999999 2023-09-23T20:37:30.011 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.011 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.011 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:37:30.011 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:57.151837+0000", 2023-09-23T20:37:30.011 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.6838999999999998e-05 2023-09-23T20:37:30.011 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.012 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.012 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:37:30.012 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:57.151857+0000", 2023-09-23T20:37:30.012 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.9848e-05 2023-09-23T20:37:30.012 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:37:30.012 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:37:30.012 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:37:30.012 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.012 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.012 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.45161.0:193704 6.5 6:a27b446f:::benchmark_data_smithi018_270558_object88965:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e288)", 2023-09-23T20:37:30.012 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:30:56.201495+0000", 2023-09-23T20:37:30.012 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 393.71493421000002, 2023-09-23T20:37:30.013 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.95056226799999999, 2023-09-23T20:37:30.013 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:37:30.013 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:37:30.013 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:37:30.013 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.45161", 2023-09-23T20:37:30.013 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3795614987", 2023-09-23T20:37:30.013 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 193704 2023-09-23T20:37:30.013 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.013 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:37:30.013 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.013 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:37:30.013 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:56.201495+0000", 2023-09-23T20:37:30.014 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:37:30.014 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.014 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.014 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:37:30.014 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:56.201495+0000", 2023-09-23T20:37:30.014 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:37:30.014 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.014 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.014 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:37:30.014 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:56.201494+0000", 2023-09-23T20:37:30.014 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:37:30.015 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.015 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.015 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:37:30.015 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:56.201499+0000", 2023-09-23T20:37:30.015 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.1819999999999997e-06 2023-09-23T20:37:30.015 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.015 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.015 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:37:30.015 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:56.201508+0000", 2023-09-23T20:37:30.015 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.7440000000000002e-06 2023-09-23T20:37:30.015 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.015 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.016 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:37:30.016 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:56.201514+0000", 2023-09-23T20:37:30.016 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.8019999999999997e-06 2023-09-23T20:37:30.016 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.016 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.016 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:37:30.016 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:56.201581+0000", 2023-09-23T20:37:30.016 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.7176999999999997e-05 2023-09-23T20:37:30.016 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.016 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.016 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:37:30.016 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:56.201591+0000", 2023-09-23T20:37:30.017 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.2050000000000008e-06 2023-09-23T20:37:30.017 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.017 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.017 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:37:30.017 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:57.120171+0000", 2023-09-23T20:37:30.017 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.91858044599999999 2023-09-23T20:37:30.017 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.017 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.017 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:37:30.017 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:57.120254+0000", 2023-09-23T20:37:30.017 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.2799999999999993e-05 2023-09-23T20:37:30.018 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.018 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.018 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 3,6", 2023-09-23T20:37:30.018 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:57.120540+0000", 2023-09-23T20:37:30.018 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00028637999999999998 2023-09-23T20:37:30.018 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.018 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.018 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:37:30.018 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:57.129589+0000", 2023-09-23T20:37:30.018 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0090484699999999994 2023-09-23T20:37:30.018 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.019 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.019 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:37:30.019 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:57.151887+0000", 2023-09-23T20:37:30.019 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.022298370000000001 2023-09-23T20:37:30.019 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.019 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.019 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:37:30.019 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:57.151987+0000", 2023-09-23T20:37:30.019 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.9631999999999997e-05 2023-09-23T20:37:30.019 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.019 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.019 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:37:30.020 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:57.152031+0000", 2023-09-23T20:37:30.020 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.4570999999999997e-05 2023-09-23T20:37:30.020 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.020 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.020 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:37:30.020 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:57.152057+0000", 2023-09-23T20:37:30.020 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.5738999999999999e-05 2023-09-23T20:37:30.020 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:37:30.020 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:37:30.020 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:37:30.020 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.020 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.021 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.45161.0:193713 6.5 6:a53a13e1:::benchmark_data_smithi018_270558_object88974:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e288)", 2023-09-23T20:37:30.021 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:30:56.213097+0000", 2023-09-23T20:37:30.021 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 393.703332031, 2023-09-23T20:37:30.021 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.93902963399999995, 2023-09-23T20:37:30.021 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:37:30.021 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:37:30.021 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:37:30.021 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.45161", 2023-09-23T20:37:30.021 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3795614987", 2023-09-23T20:37:30.021 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 193713 2023-09-23T20:37:30.022 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.022 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:37:30.022 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.022 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:37:30.022 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:56.213097+0000", 2023-09-23T20:37:30.022 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:37:30.022 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.022 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.022 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:37:30.022 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:56.213097+0000", 2023-09-23T20:37:30.022 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:37:30.022 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.023 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.023 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:37:30.023 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:56.213096+0000", 2023-09-23T20:37:30.023 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:37:30.023 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.023 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.023 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:37:30.023 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:56.213102+0000", 2023-09-23T20:37:30.023 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.6579999999999999e-06 2023-09-23T20:37:30.023 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.023 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.024 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:37:30.024 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:56.213114+0000", 2023-09-23T20:37:30.024 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.2734e-05 2023-09-23T20:37:30.024 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.024 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.024 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:37:30.024 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:56.213120+0000", 2023-09-23T20:37:30.024 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.5899999999999998e-06 2023-09-23T20:37:30.024 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.024 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.024 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:37:30.024 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:56.213171+0000", 2023-09-23T20:37:30.025 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.1e-05 2023-09-23T20:37:30.025 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.025 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.025 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:37:30.025 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:56.213180+0000", 2023-09-23T20:37:30.025 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.8089999999999999e-06 2023-09-23T20:37:30.025 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.025 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.025 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:37:30.025 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:57.122356+0000", 2023-09-23T20:37:30.025 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.90917587 2023-09-23T20:37:30.025 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.026 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.026 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:37:30.026 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:57.122485+0000", 2023-09-23T20:37:30.026 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000129178 2023-09-23T20:37:30.026 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.026 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.026 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 3,6", 2023-09-23T20:37:30.026 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:57.122803+0000", 2023-09-23T20:37:30.026 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00031803799999999998 2023-09-23T20:37:30.026 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.026 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.027 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:37:30.027 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:57.129595+0000", 2023-09-23T20:37:30.027 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0067919410000000001 2023-09-23T20:37:30.027 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.027 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.027 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:37:30.027 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:57.151961+0000", 2023-09-23T20:37:30.027 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.022366456 2023-09-23T20:37:30.027 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.027 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.027 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:37:30.027 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:57.152083+0000", 2023-09-23T20:37:30.027 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000121366 2023-09-23T20:37:30.028 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.028 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.028 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:37:30.028 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:57.152108+0000", 2023-09-23T20:37:30.028 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.5058999999999998e-05 2023-09-23T20:37:30.028 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.028 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.028 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:37:30.028 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:57.152127+0000", 2023-09-23T20:37:30.028 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.9037000000000001e-05 2023-09-23T20:37:30.028 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:37:30.029 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:37:30.029 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:37:30.029 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.029 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.029 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.45161.0:193722 6.5 6:a673ba13:::benchmark_data_smithi018_270558_object88983:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e288)", 2023-09-23T20:37:30.029 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:30:56.236163+0000", 2023-09-23T20:37:30.029 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 393.68026589800002, 2023-09-23T20:37:30.029 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.91627206999999999, 2023-09-23T20:37:30.029 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:37:30.029 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:37:30.029 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:37:30.029 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.45161", 2023-09-23T20:37:30.030 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3795614987", 2023-09-23T20:37:30.030 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 193722 2023-09-23T20:37:30.030 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.030 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:37:30.030 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.030 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:37:30.030 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:56.236163+0000", 2023-09-23T20:37:30.030 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:37:30.030 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.030 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.030 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:37:30.030 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:56.236163+0000", 2023-09-23T20:37:30.031 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:37:30.031 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.031 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.031 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:37:30.031 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:56.236161+0000", 2023-09-23T20:37:30.031 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:37:30.031 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.031 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.031 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:37:30.031 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:56.236170+0000", 2023-09-23T20:37:30.031 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.0819999999999999e-06 2023-09-23T20:37:30.031 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.032 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.032 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:37:30.032 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:56.236188+0000", 2023-09-23T20:37:30.032 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.8587e-05 2023-09-23T20:37:30.032 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.032 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.032 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:37:30.032 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:56.236196+0000", 2023-09-23T20:37:30.032 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.9929999999999995e-06 2023-09-23T20:37:30.032 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.032 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.032 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:37:30.033 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:56.236267+0000", 2023-09-23T20:37:30.033 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.0878999999999997e-05 2023-09-23T20:37:30.033 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.033 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.033 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:37:30.033 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:56.236279+0000", 2023-09-23T20:37:30.033 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.2215000000000001e-05 2023-09-23T20:37:30.033 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.033 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.033 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:37:30.033 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:57.124823+0000", 2023-09-23T20:37:30.033 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.88854340200000004 2023-09-23T20:37:30.033 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.034 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.034 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:37:30.034 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:57.124900+0000", 2023-09-23T20:37:30.034 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.7144000000000005e-05 2023-09-23T20:37:30.034 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.034 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.034 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 3,6", 2023-09-23T20:37:30.034 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:57.125132+0000", 2023-09-23T20:37:30.034 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000232232 2023-09-23T20:37:30.034 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.034 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.034 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:37:30.035 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:57.131669+0000", 2023-09-23T20:37:30.035 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0065368040000000002 2023-09-23T20:37:30.035 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.035 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.035 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:37:30.035 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:57.152154+0000", 2023-09-23T20:37:30.035 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.020485493 2023-09-23T20:37:30.035 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.035 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.035 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:37:30.035 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:57.152360+0000", 2023-09-23T20:37:30.035 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00020602399999999999 2023-09-23T20:37:30.036 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.036 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.036 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:37:30.036 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:57.152403+0000", 2023-09-23T20:37:30.036 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.2268000000000003e-05 2023-09-23T20:37:30.036 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.036 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.036 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:37:30.036 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:30:57.152435+0000", 2023-09-23T20:37:30.036 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.2604999999999997e-05 2023-09-23T20:37:30.036 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:37:30.036 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:37:30.037 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:37:30.037 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.037 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.037 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_repop(client.35700.0:125805 7.9 e340/322 7:97d108be:::benchmark_data_smithi018_308601_object31253:head v 340'7797, mlcod=318'4185)", 2023-09-23T20:37:30.037 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:34:12.963312+0000", 2023-09-23T20:37:30.037 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 196.95311716399999, 2023-09-23T20:37:30.037 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.37645642600000001, 2023-09-23T20:37:30.037 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:37:30.037 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:37:30.037 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:37:30.037 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.037 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:37:30.038 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:12.963312+0000", 2023-09-23T20:37:30.038 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:37:30.038 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.038 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.038 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:37:30.038 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:12.963312+0000", 2023-09-23T20:37:30.038 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:37:30.038 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.038 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.038 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:37:30.038 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:12.963310+0000", 2023-09-23T20:37:30.039 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:37:30.039 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.039 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.039 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:37:30.039 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:12.963341+0000", 2023-09-23T20:37:30.039 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.0323000000000001e-05 2023-09-23T20:37:30.039 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.039 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.039 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:37:30.039 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:12.963363+0000", 2023-09-23T20:37:30.039 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.2563000000000001e-05 2023-09-23T20:37:30.039 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.040 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.040 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:37:30.040 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:12.963374+0000", 2023-09-23T20:37:30.040 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.111e-05 2023-09-23T20:37:30.040 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.040 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.040 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:37:30.040 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:12.963470+0000", 2023-09-23T20:37:30.040 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.5302000000000003e-05 2023-09-23T20:37:30.040 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.040 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.040 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:37:30.041 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:12.963511+0000", 2023-09-23T20:37:30.041 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.1306000000000001e-05 2023-09-23T20:37:30.041 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.041 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.041 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:37:30.041 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:13.339739+0000", 2023-09-23T20:37:30.041 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.37622825100000001 2023-09-23T20:37:30.041 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.041 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.041 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:37:30.041 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:13.339768+0000", 2023-09-23T20:37:30.041 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.9235e-05 2023-09-23T20:37:30.042 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:37:30.042 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:37:30.042 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:37:30.042 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.042 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.042 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "MOSDPGPush(7.9 340/322 [PushOp(7:975f1708:::benchmark_data_smithi018_308601_object68059:head, version: 318'4196, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(7:975f1708:::benchmark_data_smithi018_308601_object68059:head@318'4196, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-09-23T20:37:30.042 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:34:12.966007+0000", 2023-09-23T20:37:30.042 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 196.95042218399999, 2023-09-23T20:37:30.042 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.36607237799999998, 2023-09-23T20:37:30.042 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:37:30.042 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "started", 2023-09-23T20:37:30.042 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:37:30.043 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.043 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:37:30.043 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:12.966007+0000", 2023-09-23T20:37:30.043 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:37:30.043 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.043 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.043 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:37:30.043 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:12.966007+0000", 2023-09-23T20:37:30.043 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:37:30.043 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.043 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.043 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:37:30.044 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:12.966006+0000", 2023-09-23T20:37:30.044 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:37:30.044 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.044 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.044 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:37:30.044 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:12.966038+0000", 2023-09-23T20:37:30.044 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.1473000000000002e-05 2023-09-23T20:37:30.044 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.044 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.044 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:37:30.044 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:13.325876+0000", 2023-09-23T20:37:30.044 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.35983845800000003 2023-09-23T20:37:30.045 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.045 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.045 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:37:30.045 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:13.325895+0000", 2023-09-23T20:37:30.045 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.9037000000000001e-05 2023-09-23T20:37:30.045 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.045 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.045 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:37:30.045 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:13.329519+0000", 2023-09-23T20:37:30.045 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0036242380000000001 2023-09-23T20:37:30.045 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.045 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.046 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:37:30.046 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:13.329528+0000", 2023-09-23T20:37:30.046 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.4859999999999997e-06 2023-09-23T20:37:30.046 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.046 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.046 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:37:30.046 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:13.332079+0000", 2023-09-23T20:37:30.046 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0025512830000000001 2023-09-23T20:37:30.046 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:37:30.046 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:37:30.046 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:37:30.046 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.047 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.047 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35700.0:184301 7.5 7:ad344ba2:::benchmark_data_smithi018_308601_object89749:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e364)", 2023-09-23T20:37:30.047 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:35:42.313972+0000", 2023-09-23T20:37:30.047 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 107.60245745, 2023-09-23T20:37:30.047 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.081416009999999997, 2023-09-23T20:37:30.047 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:37:30.047 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:37:30.047 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:37:30.047 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35700", 2023-09-23T20:37:30.047 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:37:30.047 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 184301 2023-09-23T20:37:30.047 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.048 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:37:30.048 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.048 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:37:30.048 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:35:42.313972+0000", 2023-09-23T20:37:30.048 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:37:30.048 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.048 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.048 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:37:30.048 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:35:42.313972+0000", 2023-09-23T20:37:30.048 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:37:30.048 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.048 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.049 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:37:30.049 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:35:42.313970+0000", 2023-09-23T20:37:30.049 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:37:30.049 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.049 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.049 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:37:30.049 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:35:42.313978+0000", 2023-09-23T20:37:30.049 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.6830000000000008e-06 2023-09-23T20:37:30.049 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.049 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.049 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:37:30.049 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:35:42.313995+0000", 2023-09-23T20:37:30.049 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.7365000000000001e-05 2023-09-23T20:37:30.050 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.050 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.050 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:37:30.050 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:35:42.314003+0000", 2023-09-23T20:37:30.050 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.1349999999999992e-06 2023-09-23T20:37:30.050 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.050 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.050 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:37:30.050 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:35:42.314076+0000", 2023-09-23T20:37:30.050 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.2515000000000006e-05 2023-09-23T20:37:30.050 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.050 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.051 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:37:30.051 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:35:42.314214+0000", 2023-09-23T20:37:30.051 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000137892 2023-09-23T20:37:30.051 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.051 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.051 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,2", 2023-09-23T20:37:30.051 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:35:42.314348+0000", 2023-09-23T20:37:30.051 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000134468 2023-09-23T20:37:30.051 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.051 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.051 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:37:30.052 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:35:42.317206+0000", 2023-09-23T20:37:30.052 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0028581489999999999 2023-09-23T20:37:30.052 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.052 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.052 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:37:30.052 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:35:42.318962+0000", 2023-09-23T20:37:30.052 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.001755316 2023-09-23T20:37:30.052 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.052 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.052 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:37:30.052 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:35:42.395289+0000", 2023-09-23T20:37:30.052 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.076326918999999993 2023-09-23T20:37:30.053 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.053 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.053 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:37:30.053 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:35:42.395342+0000", 2023-09-23T20:37:30.053 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.3315e-05 2023-09-23T20:37:30.053 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.053 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.053 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:37:30.053 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:35:42.395388+0000", 2023-09-23T20:37:30.053 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.5708999999999999e-05 2023-09-23T20:37:30.053 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:37:30.053 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:37:30.054 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:37:30.054 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.054 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.054 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35700.0:184310 7.5 7:afbdcf8d:::benchmark_data_smithi018_308601_object89758:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e364)", 2023-09-23T20:37:30.054 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:35:42.326361+0000", 2023-09-23T20:37:30.054 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 107.59006833799999, 2023-09-23T20:37:30.054 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.069168101999999995, 2023-09-23T20:37:30.054 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:37:30.054 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:37:30.054 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:37:30.054 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35700", 2023-09-23T20:37:30.055 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:37:30.055 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 184310 2023-09-23T20:37:30.055 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.055 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:37:30.055 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.055 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:37:30.055 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:35:42.326361+0000", 2023-09-23T20:37:30.055 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:37:30.055 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.055 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.055 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:37:30.055 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:35:42.326361+0000", 2023-09-23T20:37:30.056 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:37:30.056 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.056 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.056 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:37:30.056 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:35:42.326359+0000", 2023-09-23T20:37:30.056 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:37:30.056 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.056 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.056 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:37:30.056 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:35:42.326367+0000", 2023-09-23T20:37:30.056 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.0509999999999997e-06 2023-09-23T20:37:30.056 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.057 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.057 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:37:30.057 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:35:42.326384+0000", 2023-09-23T20:37:30.057 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.6548000000000001e-05 2023-09-23T20:37:30.057 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.057 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.057 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:37:30.057 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:35:42.326390+0000", 2023-09-23T20:37:30.057 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.6679999999999998e-06 2023-09-23T20:37:30.057 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.057 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.057 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:37:30.058 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:35:42.326445+0000", 2023-09-23T20:37:30.058 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.5312999999999997e-05 2023-09-23T20:37:30.058 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.058 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.058 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:37:30.058 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:35:42.326581+0000", 2023-09-23T20:37:30.058 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000135921 2023-09-23T20:37:30.058 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.058 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.058 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,2", 2023-09-23T20:37:30.058 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:35:42.326732+0000", 2023-09-23T20:37:30.058 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00015113600000000001 2023-09-23T20:37:30.059 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.059 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.059 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:37:30.059 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:35:42.329407+0000", 2023-09-23T20:37:30.059 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0026753440000000001 2023-09-23T20:37:30.059 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.059 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.059 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:37:30.059 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:35:42.329985+0000", 2023-09-23T20:37:30.059 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00057795000000000003 2023-09-23T20:37:30.059 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.059 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.060 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:37:30.060 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:35:42.395441+0000", 2023-09-23T20:37:30.060 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.065455673000000006 2023-09-23T20:37:30.060 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.060 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.060 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:37:30.060 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:35:42.395491+0000", 2023-09-23T20:37:30.060 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.9954999999999999e-05 2023-09-23T20:37:30.060 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.060 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.060 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:37:30.060 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:35:42.395529+0000", 2023-09-23T20:37:30.061 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.7796000000000002e-05 2023-09-23T20:37:30.061 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:37:30.061 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:37:30.061 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:37:30.061 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.061 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.061 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35700.0:184314 7.5 7:ad9d377a:::benchmark_data_smithi018_308601_object89762:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e364)", 2023-09-23T20:37:30.061 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:35:42.331696+0000", 2023-09-23T20:37:30.061 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 107.58473281099999, 2023-09-23T20:37:30.061 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.063921882999999999, 2023-09-23T20:37:30.061 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:37:30.061 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:37:30.062 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:37:30.062 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35700", 2023-09-23T20:37:30.062 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:37:30.062 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 184314 2023-09-23T20:37:30.062 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.062 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:37:30.062 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.062 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:37:30.062 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:35:42.331696+0000", 2023-09-23T20:37:30.062 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:37:30.062 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.062 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.063 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:37:30.063 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:35:42.331696+0000", 2023-09-23T20:37:30.063 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:37:30.063 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.063 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.063 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:37:30.063 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:35:42.331695+0000", 2023-09-23T20:37:30.063 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:37:30.063 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.063 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.063 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:37:30.063 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:35:42.331702+0000", 2023-09-23T20:37:30.063 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.3270000000000003e-06 2023-09-23T20:37:30.064 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.064 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.064 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:37:30.064 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:35:42.331717+0000", 2023-09-23T20:37:30.064 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.4795e-05 2023-09-23T20:37:30.064 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.064 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.064 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:37:30.064 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:35:42.331723+0000", 2023-09-23T20:37:30.064 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.8810000000000001e-06 2023-09-23T20:37:30.064 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.064 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.065 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:37:30.065 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:35:42.331754+0000", 2023-09-23T20:37:30.065 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.1133000000000002e-05 2023-09-23T20:37:30.065 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.065 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.065 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:37:30.065 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:35:42.331835+0000", 2023-09-23T20:37:30.065 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.1201999999999999e-05 2023-09-23T20:37:30.065 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.065 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.065 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,2", 2023-09-23T20:37:30.065 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:35:42.331909+0000", 2023-09-23T20:37:30.066 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.4214000000000005e-05 2023-09-23T20:37:30.066 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.066 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.066 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:37:30.066 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:35:42.334523+0000", 2023-09-23T20:37:30.066 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0026135640000000001 2023-09-23T20:37:30.066 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.066 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.066 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:37:30.066 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:35:42.336092+0000", 2023-09-23T20:37:30.066 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0015691419999999999 2023-09-23T20:37:30.066 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.067 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.067 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:37:30.067 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:35:42.395562+0000", 2023-09-23T20:37:30.067 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.059469635999999999 2023-09-23T20:37:30.067 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.067 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.067 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:37:30.067 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:35:42.395588+0000", 2023-09-23T20:37:30.067 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.6363000000000002e-05 2023-09-23T20:37:30.067 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.067 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.067 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:37:30.068 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:35:42.395618+0000", 2023-09-23T20:37:30.068 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.9921e-05 2023-09-23T20:37:30.068 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:37:30.068 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:37:30.068 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:37:30.068 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.068 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.068 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:84102 8.d 8:b20f6310:::benchmark_data_smithi018_341490_object84101:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e379)", 2023-09-23T20:37:30.068 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:37:13.381094+0000", 2023-09-23T20:37:30.068 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 16.535335116999999, 2023-09-23T20:37:30.068 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.221042502, 2023-09-23T20:37:30.068 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:37:30.069 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:37:30.069 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:37:30.069 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:37:30.069 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:37:30.069 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 84102 2023-09-23T20:37:30.069 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.069 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:37:30.069 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.069 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:37:30.069 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.381094+0000", 2023-09-23T20:37:30.069 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:37:30.069 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.070 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.070 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:37:30.070 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.381094+0000", 2023-09-23T20:37:30.070 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:37:30.070 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.070 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.070 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:37:30.070 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.381093+0000", 2023-09-23T20:37:30.070 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:37:30.070 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.070 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.070 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:37:30.071 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.381160+0000", 2023-09-23T20:37:30.071 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.7119999999999994e-05 2023-09-23T20:37:30.071 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.071 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.071 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:37:30.071 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.381175+0000", 2023-09-23T20:37:30.071 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.5091e-05 2023-09-23T20:37:30.071 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.071 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.071 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:37:30.071 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.381181+0000", 2023-09-23T20:37:30.072 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.1160000000000004e-06 2023-09-23T20:37:30.072 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.072 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.072 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:37:30.072 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.381234+0000", 2023-09-23T20:37:30.072 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.2218e-05 2023-09-23T20:37:30.072 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.072 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.072 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:37:30.072 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.381242+0000", 2023-09-23T20:37:30.072 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.3149999999999992e-06 2023-09-23T20:37:30.072 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.072 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.073 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:37:30.073 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:14.352689+0000", 2023-09-23T20:37:30.073 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.97144663399999998 2023-09-23T20:37:30.073 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.073 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.073 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:37:30.073 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:14.352699+0000", 2023-09-23T20:37:30.073 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0249e-05 2023-09-23T20:37:30.073 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.073 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.073 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:37:30.073 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:15.563152+0000", 2023-09-23T20:37:30.074 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.210452927 2023-09-23T20:37:30.074 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.074 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.074 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:37:30.074 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:15.564607+0000", 2023-09-23T20:37:30.074 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00145528 2023-09-23T20:37:30.074 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.074 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.074 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 2,6", 2023-09-23T20:37:30.074 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:15.564732+0000", 2023-09-23T20:37:30.074 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000125208 2023-09-23T20:37:30.074 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.075 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.075 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:37:30.075 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:15.572216+0000", 2023-09-23T20:37:30.075 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.007483712 2023-09-23T20:37:30.075 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.075 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.075 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:37:30.075 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:15.572291+0000", 2023-09-23T20:37:30.075 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.4611999999999996e-05 2023-09-23T20:37:30.075 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.075 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.075 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:37:30.076 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:15.600899+0000", 2023-09-23T20:37:30.076 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.028608602 2023-09-23T20:37:30.076 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.076 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.076 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:37:30.076 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:15.600945+0000", 2023-09-23T20:37:30.076 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.6074e-05 2023-09-23T20:37:30.076 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.076 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.076 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:37:30.076 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:15.602136+0000", 2023-09-23T20:37:30.076 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.001191147 2023-09-23T20:37:30.076 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:37:30.077 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:37:30.077 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:37:30.077 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.077 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.077 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:84105 8.d 8:b309f62d:::benchmark_data_smithi018_341490_object84104:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e379)", 2023-09-23T20:37:30.077 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:37:13.384749+0000", 2023-09-23T20:37:30.077 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 16.531680497, 2023-09-23T20:37:30.077 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.2195194549999999, 2023-09-23T20:37:30.077 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:37:30.077 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:37:30.077 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:37:30.078 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:37:30.078 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:37:30.078 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 84105 2023-09-23T20:37:30.078 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.078 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:37:30.078 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.078 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:37:30.078 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.384749+0000", 2023-09-23T20:37:30.078 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:37:30.078 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.078 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.079 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:37:30.079 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.384749+0000", 2023-09-23T20:37:30.079 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:37:30.079 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.079 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.079 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:37:30.079 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.384748+0000", 2023-09-23T20:37:30.079 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:37:30.079 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.079 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.079 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:37:30.080 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.384787+0000", 2023-09-23T20:37:30.080 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.9078000000000001e-05 2023-09-23T20:37:30.080 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.080 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.080 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:37:30.080 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.384803+0000", 2023-09-23T20:37:30.080 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.6277e-05 2023-09-23T20:37:30.080 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.080 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.080 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:37:30.080 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.384810+0000", 2023-09-23T20:37:30.080 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.9009999999999997e-06 2023-09-23T20:37:30.081 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.081 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.081 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:37:30.081 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.384862+0000", 2023-09-23T20:37:30.081 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.1962000000000002e-05 2023-09-23T20:37:30.081 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.081 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.081 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:37:30.081 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.384872+0000", 2023-09-23T20:37:30.081 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.781e-06 2023-09-23T20:37:30.081 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.081 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.082 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:37:30.082 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:14.352738+0000", 2023-09-23T20:37:30.082 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.96786639799999996 2023-09-23T20:37:30.082 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.082 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.082 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:37:30.082 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:14.352747+0000", 2023-09-23T20:37:30.082 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.9649999999999997e-06 2023-09-23T20:37:30.082 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.082 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.082 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:37:30.083 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:15.566789+0000", 2023-09-23T20:37:30.083 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.2140422710000001 2023-09-23T20:37:30.083 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.083 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.083 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:37:30.083 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:15.566946+0000", 2023-09-23T20:37:30.083 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00015720799999999999 2023-09-23T20:37:30.083 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.083 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.083 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 2,6", 2023-09-23T20:37:30.084 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:15.567141+0000", 2023-09-23T20:37:30.084 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000194215 2023-09-23T20:37:30.084 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.084 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.084 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:37:30.084 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:15.572233+0000", 2023-09-23T20:37:30.084 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.005092343 2023-09-23T20:37:30.084 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.084 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.084 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:37:30.084 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:15.577236+0000", 2023-09-23T20:37:30.084 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0050031850000000003 2023-09-23T20:37:30.085 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.085 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.085 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:37:30.085 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:15.602198+0000", 2023-09-23T20:37:30.085 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.024961555 2023-09-23T20:37:30.085 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.085 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.085 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:37:30.085 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:15.602245+0000", 2023-09-23T20:37:30.085 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.7126999999999998e-05 2023-09-23T20:37:30.085 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:37:30.086 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:37:30.086 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:37:30.086 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:15.604268+0000", 2023-09-23T20:37:30.086 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0020230959999999998 2023-09-23T20:37:30.086 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:37:30.086 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:37:30.086 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:37:30.086 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:37:30.086 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:37:30.086 INFO:teuthology.orchestra.run.smithi149.stdout:} 2023-09-23T20:37:30.087 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:30.088 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:37:30.088 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:37:30.088 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 74 16 84270 84254 71.1347 0 - 0.011237 2023-09-23T20:37:30.189 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:37:30.225 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:30.319 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:37:30.322 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:30.323 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:37:30.366 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T20:37:30.423 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:37:30.455 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:30.486 INFO:teuthology.orchestra.run.smithi018.stderr:marked out osd.5. 2023-09-23T20:37:30.512 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:37:30.515 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:30.515 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:37:30.616 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:37:30.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:30 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mon-a[179623]: 2023-09-23T20:37:30.330+0000 7f160fef9700 -1 mon.a@0(leader).osd e390 definitely_dead 0 2023-09-23T20:37:30.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:30 smithi018 ceph-mon[179647]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:37:30.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:30 smithi018 ceph-mon[179647]: failed to encode map e390 with expected crc 2023-09-23T20:37:30.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:30 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/595015129' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:37:30.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:30 smithi018 ceph-mon[179647]: osdmap e390: 8 total, 8 up, 7 in 2023-09-23T20:37:30.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:30 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:37:30.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:30 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:37:30.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:30 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3024493556' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2023-09-23T20:37:30.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:30 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2023-09-23T20:37:30.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:30 smithi018 ceph-mon[179647]: Client client.admin marked osd.5 out, while it was still marked up 2023-09-23T20:37:30.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:30 smithi018 ceph-mon[184144]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:37:30.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:30 smithi018 ceph-mon[184144]: failed to encode map e390 with expected crc 2023-09-23T20:37:30.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:30 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/595015129' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:37:30.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:30 smithi018 ceph-mon[184144]: osdmap e390: 8 total, 8 up, 7 in 2023-09-23T20:37:30.643 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:30 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:37:30.643 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:30 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:37:30.643 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:30 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3024493556' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2023-09-23T20:37:30.643 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:30 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2023-09-23T20:37:30.643 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:30 smithi018 ceph-mon[184144]: Client client.admin marked osd.5 out, while it was still marked up 2023-09-23T20:37:30.643 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:30 smithi149 ceph-mon[170394]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:37:30.643 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:30 smithi149 ceph-mon[170394]: failed to encode map e390 with expected crc 2023-09-23T20:37:30.643 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:30 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/595015129' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:37:30.644 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:30 smithi149 ceph-mon[170394]: osdmap e390: 8 total, 8 up, 7 in 2023-09-23T20:37:30.644 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:30 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:37:30.644 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:30 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:37:30.644 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:30 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3024493556' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2023-09-23T20:37:30.644 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:30 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2023-09-23T20:37:30.644 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:30 smithi149 ceph-mon[170394]: Client client.admin marked osd.5 out, while it was still marked up 2023-09-23T20:37:30.646 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:30.692 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:37:30.697 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:30.697 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:37:30.798 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:37:30.828 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:30.865 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:37:30.868 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:30.868 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:37:30.968 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:37:30.999 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:31.035 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 75 16 84270 84254 70.1865 0 - 0.011237 2023-09-23T20:37:31.039 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:37:31.042 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:31.043 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:37:31.144 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:37:31.173 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:31.207 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:37:31.210 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:37:31.218 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:31.219 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:37:31.319 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:37:31.374 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:31.419 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:37:31.420 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:31.420 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:37:31.479 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:31 smithi018 ceph-mon[184144]: pgmap v948: 119 pgs: 1 active+recovering+undersized+remapped, 1 activating+undersized+degraded+remapped, 6 peering, 9 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 5.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 11 KiB/s rd, 29 KiB/s wr, 8 op/s; 42226/251826 objects degraded (16.768%); 4098/251826 objects misplaced (1.627%); 4.3 MiB/s, 69 objects/s recovering 2023-09-23T20:37:31.479 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:31 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 42226/251826 objects degraded (16.768%), 10 pgs degraded (PG_DEGRADED) 2023-09-23T20:37:31.479 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:31 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["5"]}]': finished 2023-09-23T20:37:31.479 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:31 smithi018 ceph-mon[184144]: osdmap e391: 8 total, 8 up, 6 in 2023-09-23T20:37:31.479 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:31 smithi018 ceph-mon[184144]: failed to encode map e391 with expected crc 2023-09-23T20:37:31.480 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:31 smithi018 ceph-mon[184144]: failed to encode map e391 with expected crc 2023-09-23T20:37:31.480 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:31 smithi018 ceph-mon[184144]: failed to encode map e391 with expected crc 2023-09-23T20:37:31.480 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:31 smithi018 ceph-mon[184144]: failed to encode map e391 with expected crc 2023-09-23T20:37:31.480 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:31 smithi018 ceph-mon[184144]: Health check update: Reduced data availability: 1 pg inactive (PG_AVAILABILITY) 2023-09-23T20:37:31.480 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:31 smithi018 ceph-mon[184144]: osdmap e392: 8 total, 8 up, 6 in 2023-09-23T20:37:31.480 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:31 smithi018 ceph-mon[179647]: pgmap v948: 119 pgs: 1 active+recovering+undersized+remapped, 1 activating+undersized+degraded+remapped, 6 peering, 9 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 5.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 11 KiB/s rd, 29 KiB/s wr, 8 op/s; 42226/251826 objects degraded (16.768%); 4098/251826 objects misplaced (1.627%); 4.3 MiB/s, 69 objects/s recovering 2023-09-23T20:37:31.480 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:31 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 42226/251826 objects degraded (16.768%), 10 pgs degraded (PG_DEGRADED) 2023-09-23T20:37:31.480 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:31 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["5"]}]': finished 2023-09-23T20:37:31.480 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:31 smithi018 ceph-mon[179647]: osdmap e391: 8 total, 8 up, 6 in 2023-09-23T20:37:31.480 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:31 smithi018 ceph-mon[179647]: failed to encode map e391 with expected crc 2023-09-23T20:37:31.481 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:31 smithi018 ceph-mon[179647]: failed to encode map e391 with expected crc 2023-09-23T20:37:31.481 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:31 smithi018 ceph-mon[179647]: failed to encode map e391 with expected crc 2023-09-23T20:37:31.481 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:31 smithi018 ceph-mon[179647]: failed to encode map e391 with expected crc 2023-09-23T20:37:31.481 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:31 smithi018 ceph-mon[179647]: Health check update: Reduced data availability: 1 pg inactive (PG_AVAILABILITY) 2023-09-23T20:37:31.481 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:31 smithi018 ceph-mon[179647]: osdmap e392: 8 total, 8 up, 6 in 2023-09-23T20:37:31.520 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:37:31.552 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:31.592 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:37:31.595 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:31.595 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:37:31.696 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:37:31.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:31 smithi149 ceph-mon[170394]: pgmap v948: 119 pgs: 1 active+recovering+undersized+remapped, 1 activating+undersized+degraded+remapped, 6 peering, 9 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 5.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 11 KiB/s rd, 29 KiB/s wr, 8 op/s; 42226/251826 objects degraded (16.768%); 4098/251826 objects misplaced (1.627%); 4.3 MiB/s, 69 objects/s recovering 2023-09-23T20:37:31.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:31 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 42226/251826 objects degraded (16.768%), 10 pgs degraded (PG_DEGRADED) 2023-09-23T20:37:31.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:31 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["5"]}]': finished 2023-09-23T20:37:31.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:31 smithi149 ceph-mon[170394]: osdmap e391: 8 total, 8 up, 6 in 2023-09-23T20:37:31.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:31 smithi149 ceph-mon[170394]: failed to encode map e391 with expected crc 2023-09-23T20:37:31.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:31 smithi149 ceph-mon[170394]: failed to encode map e391 with expected crc 2023-09-23T20:37:31.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:31 smithi149 ceph-mon[170394]: failed to encode map e391 with expected crc 2023-09-23T20:37:31.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:31 smithi149 ceph-mon[170394]: failed to encode map e391 with expected crc 2023-09-23T20:37:31.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:31 smithi149 ceph-mon[170394]: Health check update: Reduced data availability: 1 pg inactive (PG_AVAILABILITY) 2023-09-23T20:37:31.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:31 smithi149 ceph-mon[170394]: osdmap e392: 8 total, 8 up, 6 in 2023-09-23T20:37:31.730 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:31.822 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T20:37:32.035 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 76 16 84270 84254 69.2632 0 - 0.011237 2023-09-23T20:37:32.305 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:37:32.306 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:37:32.306 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:37:32.306 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:37:32.306 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:37:32.306 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:37:32.307 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:37:32.307 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:37:32.307 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:37:32.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:32 smithi149 ceph-mon[170394]: failed to encode map e392 with expected crc 2023-09-23T20:37:32.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:32 smithi149 ceph-mon[170394]: failed to encode map e392 with expected crc 2023-09-23T20:37:32.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:32 smithi149 ceph-mon[170394]: failed to encode map e392 with expected crc 2023-09-23T20:37:32.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:32 smithi149 ceph-mon[170394]: failed to encode map e392 with expected crc 2023-09-23T20:37:32.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:32 smithi149 ceph-mon[170394]: failed to encode map e392 with expected crc 2023-09-23T20:37:32.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:32 smithi149 ceph-mon[170394]: failed to encode map e392 with expected crc 2023-09-23T20:37:32.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:32 smithi149 ceph-mon[170394]: osdmap e393: 8 total, 8 up, 6 in 2023-09-23T20:37:33.010 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:32 smithi018 ceph-mon[184144]: failed to encode map e392 with expected crc 2023-09-23T20:37:33.011 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:32 smithi018 ceph-mon[184144]: failed to encode map e392 with expected crc 2023-09-23T20:37:33.011 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:32 smithi018 ceph-mon[184144]: failed to encode map e392 with expected crc 2023-09-23T20:37:33.011 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:32 smithi018 ceph-mon[184144]: failed to encode map e392 with expected crc 2023-09-23T20:37:33.011 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:32 smithi018 ceph-mon[184144]: failed to encode map e392 with expected crc 2023-09-23T20:37:33.011 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:32 smithi018 ceph-mon[184144]: failed to encode map e392 with expected crc 2023-09-23T20:37:33.011 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:32 smithi018 ceph-mon[184144]: osdmap e393: 8 total, 8 up, 6 in 2023-09-23T20:37:33.011 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:32 smithi018 ceph-mon[179647]: failed to encode map e392 with expected crc 2023-09-23T20:37:33.011 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:32 smithi018 ceph-mon[179647]: failed to encode map e392 with expected crc 2023-09-23T20:37:33.011 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:32 smithi018 ceph-mon[179647]: failed to encode map e392 with expected crc 2023-09-23T20:37:33.011 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:32 smithi018 ceph-mon[179647]: failed to encode map e392 with expected crc 2023-09-23T20:37:33.012 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:32 smithi018 ceph-mon[179647]: failed to encode map e392 with expected crc 2023-09-23T20:37:33.012 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:32 smithi018 ceph-mon[179647]: failed to encode map e392 with expected crc 2023-09-23T20:37:33.012 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:32 smithi018 ceph-mon[179647]: osdmap e393: 8 total, 8 up, 6 in 2023-09-23T20:37:33.036 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 77 16 84463 84447 68.52 0.753906 0.0068155 0.0145875 2023-09-23T20:37:33.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:33 smithi149 ceph-mon[170394]: pgmap v951: 119 pgs: 2 active+recovering+undersized+remapped, 1 activating+undersized+degraded+remapped, 8 peering, 8 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 5.1 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 37000/251826 objects degraded (14.693%); 9320/251826 objects misplaced (3.701%); 4.3 MiB/s, 69 objects/s recovering 2023-09-23T20:37:33.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:33 smithi149 ceph-mon[170394]: failed to encode map e393 with expected crc 2023-09-23T20:37:33.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:33 smithi149 ceph-mon[170394]: failed to encode map e393 with expected crc 2023-09-23T20:37:33.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:33 smithi149 ceph-mon[170394]: failed to encode map e393 with expected crc 2023-09-23T20:37:33.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:33 smithi149 ceph-mon[170394]: failed to encode map e393 with expected crc 2023-09-23T20:37:33.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:33 smithi149 ceph-mon[170394]: failed to encode map e393 with expected crc 2023-09-23T20:37:33.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:33 smithi149 ceph-mon[170394]: failed to encode map e393 with expected crc 2023-09-23T20:37:33.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:33 smithi149 ceph-mon[170394]: failed to encode map e393 with expected crc 2023-09-23T20:37:33.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:33 smithi149 ceph-mon[170394]: osdmap e394: 8 total, 8 up, 6 in 2023-09-23T20:37:33.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:33 smithi018 ceph-mon[184144]: pgmap v951: 119 pgs: 2 active+recovering+undersized+remapped, 1 activating+undersized+degraded+remapped, 8 peering, 8 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 5.1 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 37000/251826 objects degraded (14.693%); 9320/251826 objects misplaced (3.701%); 4.3 MiB/s, 69 objects/s recovering 2023-09-23T20:37:33.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:33 smithi018 ceph-mon[184144]: failed to encode map e393 with expected crc 2023-09-23T20:37:33.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:33 smithi018 ceph-mon[184144]: failed to encode map e393 with expected crc 2023-09-23T20:37:33.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:33 smithi018 ceph-mon[184144]: failed to encode map e393 with expected crc 2023-09-23T20:37:33.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:33 smithi018 ceph-mon[184144]: failed to encode map e393 with expected crc 2023-09-23T20:37:33.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:33 smithi018 ceph-mon[184144]: failed to encode map e393 with expected crc 2023-09-23T20:37:33.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:33 smithi018 ceph-mon[184144]: failed to encode map e393 with expected crc 2023-09-23T20:37:33.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:33 smithi018 ceph-mon[184144]: failed to encode map e393 with expected crc 2023-09-23T20:37:33.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:33 smithi018 ceph-mon[184144]: osdmap e394: 8 total, 8 up, 6 in 2023-09-23T20:37:33.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:33 smithi018 ceph-mon[179647]: pgmap v951: 119 pgs: 2 active+recovering+undersized+remapped, 1 activating+undersized+degraded+remapped, 8 peering, 8 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 5.1 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 37000/251826 objects degraded (14.693%); 9320/251826 objects misplaced (3.701%); 4.3 MiB/s, 69 objects/s recovering 2023-09-23T20:37:33.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:33 smithi018 ceph-mon[179647]: failed to encode map e393 with expected crc 2023-09-23T20:37:33.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:33 smithi018 ceph-mon[179647]: failed to encode map e393 with expected crc 2023-09-23T20:37:33.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:33 smithi018 ceph-mon[179647]: failed to encode map e393 with expected crc 2023-09-23T20:37:33.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:33 smithi018 ceph-mon[179647]: failed to encode map e393 with expected crc 2023-09-23T20:37:33.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:33 smithi018 ceph-mon[179647]: failed to encode map e393 with expected crc 2023-09-23T20:37:33.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:33 smithi018 ceph-mon[179647]: failed to encode map e393 with expected crc 2023-09-23T20:37:33.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:33 smithi018 ceph-mon[179647]: failed to encode map e393 with expected crc 2023-09-23T20:37:33.895 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:33 smithi018 ceph-mon[179647]: osdmap e394: 8 total, 8 up, 6 in 2023-09-23T20:37:33.910 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:37:33.918 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:33.918 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:37:34.019 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:37:34.039 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 78 16 84958 84942 68.036 30.9375 0.0129358 0.0146902 2023-09-23T20:37:34.094 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:34.172 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:37:34.183 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:34.184 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:37:34.285 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:37:34.327 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:34.364 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:37:34.369 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:34.369 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:37:34.447 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:34 smithi018 ceph-mon[184144]: failed to encode map e394 with expected crc 2023-09-23T20:37:34.447 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:34 smithi018 ceph-mon[184144]: failed to encode map e394 with expected crc 2023-09-23T20:37:34.470 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:37:34.504 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:34.547 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:37:34.553 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:34.553 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:37:34.654 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:37:34.680 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:34 smithi149 ceph-mon[170394]: failed to encode map e394 with expected crc 2023-09-23T20:37:34.680 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:34 smithi149 ceph-mon[170394]: failed to encode map e394 with expected crc 2023-09-23T20:37:34.681 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:34 smithi149 ceph-mon[170394]: failed to encode map e394 with expected crc 2023-09-23T20:37:34.681 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:34 smithi149 ceph-mon[170394]: osdmap e395: 8 total, 8 up, 6 in 2023-09-23T20:37:34.683 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:34.729 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:37:34.732 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:34.732 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:37:34.833 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:37:34.865 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:34.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:34 smithi018 ceph-mon[184144]: failed to encode map e394 with expected crc 2023-09-23T20:37:34.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:34 smithi018 ceph-mon[184144]: osdmap e395: 8 total, 8 up, 6 in 2023-09-23T20:37:34.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:34 smithi018 ceph-mon[179647]: failed to encode map e394 with expected crc 2023-09-23T20:37:34.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:34 smithi018 ceph-mon[179647]: failed to encode map e394 with expected crc 2023-09-23T20:37:34.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:34 smithi018 ceph-mon[179647]: failed to encode map e394 with expected crc 2023-09-23T20:37:34.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:34 smithi018 ceph-mon[179647]: osdmap e395: 8 total, 8 up, 6 in 2023-09-23T20:37:34.909 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:37:34.914 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:34.914 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:37:35.015 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:37:35.039 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 79 16 85446 85430 67.561 30.5 0.0307428 0.0147941 2023-09-23T20:37:35.133 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:35.393 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:37:35 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:37:34] "GET /metrics HTTP/1.1" 200 36589 "" "Prometheus/2.43.0" 2023-09-23T20:37:35.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:35 smithi149 ceph-mon[170394]: pgmap v954: 119 pgs: 2 remapped+peering, 2 active+recovering+undersized+remapped, 1 activating+undersized+degraded+remapped, 8 peering, 2 active+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 5.1 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 12 KiB/s rd, 5.4 MiB/s wr, 95 op/s; 47063/252864 objects degraded (18.612%); 9320/252864 objects misplaced (3.686%); 15 MiB/s, 230 objects/s recovering 2023-09-23T20:37:35.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:35 smithi149 ceph-mon[170394]: failed to encode map e395 with expected crc 2023-09-23T20:37:35.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:35 smithi149 ceph-mon[170394]: failed to encode map e395 with expected crc 2023-09-23T20:37:35.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:35 smithi149 ceph-mon[170394]: failed to encode map e395 with expected crc 2023-09-23T20:37:35.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:35 smithi149 ceph-mon[170394]: failed to encode map e395 with expected crc 2023-09-23T20:37:35.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:35 smithi149 ceph-mon[170394]: failed to encode map e395 with expected crc 2023-09-23T20:37:35.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:35 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:37:35.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:35 smithi149 ceph-mon[170394]: osdmap e396: 8 total, 8 up, 6 in 2023-09-23T20:37:35.884 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:37:35.893 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:35.894 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:37:35.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:35 smithi018 ceph-mon[179647]: pgmap v954: 119 pgs: 2 remapped+peering, 2 active+recovering+undersized+remapped, 1 activating+undersized+degraded+remapped, 8 peering, 2 active+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 5.1 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 12 KiB/s rd, 5.4 MiB/s wr, 95 op/s; 47063/252864 objects degraded (18.612%); 9320/252864 objects misplaced (3.686%); 15 MiB/s, 230 objects/s recovering 2023-09-23T20:37:35.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:35 smithi018 ceph-mon[179647]: failed to encode map e395 with expected crc 2023-09-23T20:37:35.895 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:35 smithi018 ceph-mon[179647]: failed to encode map e395 with expected crc 2023-09-23T20:37:35.895 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:35 smithi018 ceph-mon[179647]: failed to encode map e395 with expected crc 2023-09-23T20:37:35.895 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:35 smithi018 ceph-mon[179647]: failed to encode map e395 with expected crc 2023-09-23T20:37:35.895 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:35 smithi018 ceph-mon[179647]: failed to encode map e395 with expected crc 2023-09-23T20:37:35.895 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:35 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:37:35.895 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:35 smithi018 ceph-mon[179647]: osdmap e396: 8 total, 8 up, 6 in 2023-09-23T20:37:35.895 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:35 smithi018 ceph-mon[184144]: pgmap v954: 119 pgs: 2 remapped+peering, 2 active+recovering+undersized+remapped, 1 activating+undersized+degraded+remapped, 8 peering, 2 active+undersized+remapped, 10 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 5.1 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 12 KiB/s rd, 5.4 MiB/s wr, 95 op/s; 47063/252864 objects degraded (18.612%); 9320/252864 objects misplaced (3.686%); 15 MiB/s, 230 objects/s recovering 2023-09-23T20:37:35.895 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:35 smithi018 ceph-mon[184144]: failed to encode map e395 with expected crc 2023-09-23T20:37:35.895 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:35 smithi018 ceph-mon[184144]: failed to encode map e395 with expected crc 2023-09-23T20:37:35.896 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:35 smithi018 ceph-mon[184144]: failed to encode map e395 with expected crc 2023-09-23T20:37:35.896 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:35 smithi018 ceph-mon[184144]: failed to encode map e395 with expected crc 2023-09-23T20:37:35.896 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:35 smithi018 ceph-mon[184144]: failed to encode map e395 with expected crc 2023-09-23T20:37:35.896 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:35 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:37:35.896 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:35 smithi018 ceph-mon[184144]: osdmap e396: 8 total, 8 up, 6 in 2023-09-23T20:37:35.950 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 7, 3, 2, 4] out_osds: [6, 5] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T20:37:35.951 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:37:35.951 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-09-23T20:37:35.951 INFO:tasks.thrashosds.thrasher:Setting osd 3 primary_affinity to 0.905976 2023-09-23T20:37:35.951 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 3 0.9059760403326326 2023-09-23T20:37:35.995 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:37:36.026 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:36.039 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:2023-09-23T20:37:36.040156+0000 min lat: 0.00120245 max lat: 19.3764 avg lat: 0.0148991 2023-09-23T20:37:36.040 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T20:37:36.040 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 80 16 85919 85903 67.0857 29.5625 0.0498068 0.0148991 2023-09-23T20:37:36.073 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:37:36.077 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:36.077 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:37:36.178 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:37:36.210 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:36.249 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:37:36.253 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:36.253 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:37:36.354 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:37:36.383 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:36.424 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:37:36.429 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:36.429 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:37:36.530 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:37:36.561 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:36.600 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:37:36.606 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:36.607 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:37:36.707 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:37:36.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:36 smithi149 ceph-mon[170394]: failed to encode map e396 with expected crc 2023-09-23T20:37:36.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:36 smithi149 ceph-mon[170394]: Health check update: Reduced data availability: 3 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2023-09-23T20:37:36.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:36 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 47063/252864 objects degraded (18.612%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T20:37:36.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:36 smithi149 ceph-mon[170394]: osdmap e397: 8 total, 8 up, 6 in 2023-09-23T20:37:36.738 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:36.784 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:37:36.787 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:36.787 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:37:36.888 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:37:36.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:36 smithi018 ceph-mon[179647]: failed to encode map e396 with expected crc 2023-09-23T20:37:36.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:36 smithi018 ceph-mon[179647]: Health check update: Reduced data availability: 3 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2023-09-23T20:37:36.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:36 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 47063/252864 objects degraded (18.612%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T20:37:36.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:36 smithi018 ceph-mon[179647]: osdmap e397: 8 total, 8 up, 6 in 2023-09-23T20:37:36.895 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:36 smithi018 ceph-mon[184144]: failed to encode map e396 with expected crc 2023-09-23T20:37:36.895 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:36 smithi018 ceph-mon[184144]: Health check update: Reduced data availability: 3 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2023-09-23T20:37:36.895 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:36 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 47063/252864 objects degraded (18.612%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T20:37:36.895 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:36 smithi018 ceph-mon[184144]: osdmap e397: 8 total, 8 up, 6 in 2023-09-23T20:37:37.028 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:37:37.046 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 81 16 86352 86336 66.5897 27.0625 0.0246103 0.0150096 2023-09-23T20:37:37.058 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:37.672 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:37:37.679 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:37.679 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:37:37.744 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:37 smithi018 ceph-mon[179647]: pgmap v957: 119 pgs: 2 active+recovering+undersized+remapped, 2 remapped+peering, 1 activating+undersized+degraded+remapped, 2 active+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 5.2 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 20 KiB/s rd, 21 MiB/s wr, 352 op/s; 51044/255882 objects degraded (19.948%); 15387/255882 objects misplaced (6.013%); 23 MiB/s, 353 objects/s recovering 2023-09-23T20:37:37.745 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:37 smithi018 ceph-mon[179647]: failed to encode map e397 with expected crc 2023-09-23T20:37:37.745 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:37 smithi018 ceph-mon[179647]: failed to encode map e397 with expected crc 2023-09-23T20:37:37.745 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:37 smithi018 ceph-mon[184144]: pgmap v957: 119 pgs: 2 active+recovering+undersized+remapped, 2 remapped+peering, 1 activating+undersized+degraded+remapped, 2 active+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 5.2 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 20 KiB/s rd, 21 MiB/s wr, 352 op/s; 51044/255882 objects degraded (19.948%); 15387/255882 objects misplaced (6.013%); 23 MiB/s, 353 objects/s recovering 2023-09-23T20:37:37.745 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:37 smithi018 ceph-mon[184144]: failed to encode map e397 with expected crc 2023-09-23T20:37:37.745 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:37 smithi018 ceph-mon[184144]: failed to encode map e397 with expected crc 2023-09-23T20:37:37.780 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:37:37.807 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:37 smithi149 ceph-mon[170394]: pgmap v957: 119 pgs: 2 active+recovering+undersized+remapped, 2 remapped+peering, 1 activating+undersized+degraded+remapped, 2 active+undersized+remapped, 11 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 5.2 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 20 KiB/s rd, 21 MiB/s wr, 352 op/s; 51044/255882 objects degraded (19.948%); 15387/255882 objects misplaced (6.013%); 23 MiB/s, 353 objects/s recovering 2023-09-23T20:37:37.807 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:37 smithi149 ceph-mon[170394]: failed to encode map e397 with expected crc 2023-09-23T20:37:37.807 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:37 smithi149 ceph-mon[170394]: failed to encode map e397 with expected crc 2023-09-23T20:37:37.810 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:37.862 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:37:37.868 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:37.868 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:37:37.969 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:37:38.046 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 82 16 86838 86822 66.1454 30.375 0.0252268 0.0151094 2023-09-23T20:37:38.064 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:38.351 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:37:38.373 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:38.374 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:37:38.475 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:37:38.506 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:38.546 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:37:38.550 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:38.550 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:37:38.604 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:38 smithi018 ceph-mon[179647]: failed to encode map e397 with expected crc 2023-09-23T20:37:38.605 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:38 smithi018 ceph-mon[184144]: failed to encode map e397 with expected crc 2023-09-23T20:37:38.609 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:37:38.651 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:37:38.670 INFO:teuthology.orchestra.run.smithi018.stdout:ERROR: (22) Invalid argument 2023-09-23T20:37:38.670 INFO:teuthology.orchestra.run.smithi018.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:37:38.786 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:38.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:38 smithi149 ceph-mon[170394]: failed to encode map e397 with expected crc 2023-09-23T20:37:38.978 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:37:39.046 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 83 15 87261 87246 65.6677 26.5 0.0246161 0.015222 2023-09-23T20:37:39.307 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:37:39.312 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:39.313 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:37:39.413 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:37:39.421 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T20:37:39.563 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:39 smithi018 ceph-mon[179647]: pgmap v959: 119 pgs: 2 active+recovering+undersized+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 5.2 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 19 MiB/s wr, 299 op/s; 51041/257082 objects degraded (19.854%); 15387/257082 objects misplaced (5.985%); 6.9 MiB/s, 105 objects/s recovering 2023-09-23T20:37:39.563 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:39 smithi018 ceph-mon[179647]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-09-23T20:37:39.563 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:39 smithi018 ceph-mon[184144]: pgmap v959: 119 pgs: 2 active+recovering+undersized+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 5.2 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 19 MiB/s wr, 299 op/s; 51041/257082 objects degraded (19.854%); 15387/257082 objects misplaced (5.985%); 6.9 MiB/s, 105 objects/s recovering 2023-09-23T20:37:39.563 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:39 smithi018 ceph-mon[184144]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-09-23T20:37:39.582 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:39.711 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:37:39.718 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:39.718 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:37:39.819 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:37:39.885 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:39.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:39 smithi149 ceph-mon[170394]: pgmap v959: 119 pgs: 2 active+recovering+undersized+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 5.2 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 19 MiB/s wr, 299 op/s; 51041/257082 objects degraded (19.854%); 15387/257082 objects misplaced (5.985%); 6.9 MiB/s, 105 objects/s recovering 2023-09-23T20:37:39.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:39 smithi149 ceph-mon[170394]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-09-23T20:37:40.016 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:37:40.022 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:40.022 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:37:40.046 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 84 16 87629 87613 65.1591 22.9375 0.0517913 0.0153369 2023-09-23T20:37:40.124 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:37:40.153 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:40.364 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:37:40.368 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:40.368 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:37:40.469 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:37:40.595 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:40.738 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:37:40.949 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:37:40.959 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:40.959 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:37:41.048 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 85 16 88047 88031 64.6988 26.125 0.0409513 0.0154488 2023-09-23T20:37:41.060 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:37:41.192 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:41.567 INFO:teuthology.orchestra.run.smithi018.stderr:noscrub is unset 2023-09-23T20:37:41.667 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:41 smithi018 ceph-mon[179647]: pgmap v960: 119 pgs: 2 active+recovering+undersized+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 5.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 22 MiB/s wr, 352 op/s; 51041/259212 objects degraded (19.691%); 15387/259212 objects misplaced (5.936%); 5.4 MiB/s, 82 objects/s recovering 2023-09-23T20:37:41.667 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:41 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/68989640' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:37:41.667 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:41 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:37:41.667 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:41 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 51041/259212 objects degraded (19.691%), 12 pgs degraded (PG_DEGRADED) 2023-09-23T20:37:41.668 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:41 smithi018 ceph-mon[184144]: pgmap v960: 119 pgs: 2 active+recovering+undersized+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 5.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 22 MiB/s wr, 352 op/s; 51041/259212 objects degraded (19.691%); 15387/259212 objects misplaced (5.936%); 5.4 MiB/s, 82 objects/s recovering 2023-09-23T20:37:41.668 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:41 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/68989640' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:37:41.668 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:41 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:37:41.668 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:41 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 51041/259212 objects degraded (19.691%), 12 pgs degraded (PG_DEGRADED) 2023-09-23T20:37:41.689 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:37:41.695 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:41.696 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:37:41.797 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:37:41.849 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:37:41.932 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:41.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:41 smithi149 ceph-mon[170394]: pgmap v960: 119 pgs: 2 active+recovering+undersized+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 5.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 22 MiB/s wr, 352 op/s; 51041/259212 objects degraded (19.691%); 15387/259212 objects misplaced (5.936%); 5.4 MiB/s, 82 objects/s recovering 2023-09-23T20:37:41.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:41 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/68989640' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:37:41.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:41 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:37:41.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:41 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 51041/259212 objects degraded (19.691%), 12 pgs degraded (PG_DEGRADED) 2023-09-23T20:37:42.048 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 86 16 88476 88460 64.2584 26.8125 0.0592863 0.0155553 2023-09-23T20:37:42.260 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:37:42.270 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:42.271 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:37:42.372 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:37:42.498 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:42.555 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:42 smithi018 ceph-mon[179647]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:37:42.556 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:42 smithi018 ceph-mon[179647]: failed to encode map e398 with expected crc 2023-09-23T20:37:42.556 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:42 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:37:42.556 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:42 smithi018 ceph-mon[179647]: osdmap e398: 8 total, 8 up, 6 in 2023-09-23T20:37:42.556 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:42 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1542122015' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.9059760403326326}]: dispatch 2023-09-23T20:37:42.562 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:42 smithi018 ceph-mon[184144]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:37:42.562 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:42 smithi018 ceph-mon[184144]: failed to encode map e398 with expected crc 2023-09-23T20:37:42.562 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:42 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:37:42.562 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:42 smithi018 ceph-mon[184144]: osdmap e398: 8 total, 8 up, 6 in 2023-09-23T20:37:42.562 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:42 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1542122015' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.9059760403326326}]: dispatch 2023-09-23T20:37:42.606 INFO:teuthology.orchestra.run.smithi018.stderr:set osd.3 primary-affinity to 0.905976 (8593742) 2023-09-23T20:37:42.912 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:37:42.915 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:42.915 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:37:42.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:42 smithi149 ceph-mon[170394]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:37:42.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:42 smithi149 ceph-mon[170394]: failed to encode map e398 with expected crc 2023-09-23T20:37:42.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:42 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:37:42.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:42 smithi149 ceph-mon[170394]: osdmap e398: 8 total, 8 up, 6 in 2023-09-23T20:37:42.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:42 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1542122015' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.9059760403326326}]: dispatch 2023-09-23T20:37:43.016 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:37:43.048 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 87 16 88763 88747 63.7261 17.9375 0.00660782 0.0156173 2023-09-23T20:37:43.168 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:43.601 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:37:43.606 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:43.606 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:37:43.707 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:37:43.745 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:43.838 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:43 smithi018 ceph-mon[179647]: pgmap v962: 119 pgs: 2 active+recovering+undersized+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 5.3 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 28 MiB/s wr, 443 op/s; 51041/262539 objects degraded (19.441%); 15066/262539 objects misplaced (5.739%); 5.6 MiB/s, 89 objects/s recovering 2023-09-23T20:37:43.838 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:43 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1542122015' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.9059760403326326}]': finished 2023-09-23T20:37:43.838 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:43 smithi018 ceph-mon[179647]: osdmap e399: 8 total, 8 up, 6 in 2023-09-23T20:37:43.838 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:43 smithi018 ceph-mon[179647]: failed to encode map e399 with expected crc 2023-09-23T20:37:43.838 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:43 smithi018 ceph-mon[179647]: failed to encode map e399 with expected crc 2023-09-23T20:37:43.839 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:43 smithi018 ceph-mon[184144]: pgmap v962: 119 pgs: 2 active+recovering+undersized+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 5.3 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 28 MiB/s wr, 443 op/s; 51041/262539 objects degraded (19.441%); 15066/262539 objects misplaced (5.739%); 5.6 MiB/s, 89 objects/s recovering 2023-09-23T20:37:43.839 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:43 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1542122015' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.9059760403326326}]': finished 2023-09-23T20:37:43.839 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:43 smithi018 ceph-mon[184144]: osdmap e399: 8 total, 8 up, 6 in 2023-09-23T20:37:43.839 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:43 smithi018 ceph-mon[184144]: failed to encode map e399 with expected crc 2023-09-23T20:37:43.839 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:43 smithi018 ceph-mon[184144]: failed to encode map e399 with expected crc 2023-09-23T20:37:43.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:43 smithi149 ceph-mon[170394]: pgmap v962: 119 pgs: 2 active+recovering+undersized+remapped, 12 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 5.3 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 28 MiB/s wr, 443 op/s; 51041/262539 objects degraded (19.441%); 15066/262539 objects misplaced (5.739%); 5.6 MiB/s, 89 objects/s recovering 2023-09-23T20:37:43.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:43 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1542122015' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.9059760403326326}]': finished 2023-09-23T20:37:43.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:43 smithi149 ceph-mon[170394]: osdmap e399: 8 total, 8 up, 6 in 2023-09-23T20:37:43.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:43 smithi149 ceph-mon[170394]: failed to encode map e399 with expected crc 2023-09-23T20:37:43.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:43 smithi149 ceph-mon[170394]: failed to encode map e399 with expected crc 2023-09-23T20:37:44.048 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 88 16 88832 88816 63.0512 4.3125 0.00328147 0.0157111 2023-09-23T20:37:44.287 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:37:44.291 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:44.291 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:37:44.391 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:37:44.420 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:44.710 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:37:44.715 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:44.715 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:37:44.716 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:44 smithi018 ceph-mon[184144]: failed to encode map e399 with expected crc 2023-09-23T20:37:44.716 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:44 smithi018 ceph-mon[184144]: osdmap e400: 8 total, 8 up, 6 in 2023-09-23T20:37:44.716 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:44 smithi018 ceph-mon[184144]: failed to encode map e400 with expected crc 2023-09-23T20:37:44.716 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:44 smithi018 ceph-mon[184144]: failed to encode map e400 with expected crc 2023-09-23T20:37:44.716 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:44 smithi018 ceph-mon[184144]: failed to encode map e400 with expected crc 2023-09-23T20:37:44.716 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:44 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:37:44.716 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:44 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:37:44.717 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:44 smithi018 ceph-mon[179647]: failed to encode map e399 with expected crc 2023-09-23T20:37:44.717 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:44 smithi018 ceph-mon[179647]: osdmap e400: 8 total, 8 up, 6 in 2023-09-23T20:37:44.717 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:44 smithi018 ceph-mon[179647]: failed to encode map e400 with expected crc 2023-09-23T20:37:44.717 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:44 smithi018 ceph-mon[179647]: failed to encode map e400 with expected crc 2023-09-23T20:37:44.717 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:44 smithi018 ceph-mon[179647]: failed to encode map e400 with expected crc 2023-09-23T20:37:44.717 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:44 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:37:44.717 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:44 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:37:44.817 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:37:44.842 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:44 smithi149 ceph-mon[170394]: failed to encode map e399 with expected crc 2023-09-23T20:37:44.842 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:44 smithi149 ceph-mon[170394]: osdmap e400: 8 total, 8 up, 6 in 2023-09-23T20:37:44.842 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:44 smithi149 ceph-mon[170394]: failed to encode map e400 with expected crc 2023-09-23T20:37:44.842 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:44 smithi149 ceph-mon[170394]: failed to encode map e400 with expected crc 2023-09-23T20:37:44.842 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:44 smithi149 ceph-mon[170394]: failed to encode map e400 with expected crc 2023-09-23T20:37:44.842 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:44 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:37:44.843 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:44 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:37:44.845 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:44.891 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:37:44.894 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:44.894 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:37:44.993 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T20:37:44.995 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:37:45.010 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:37:44 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:37:44] "GET /metrics HTTP/1.1" 200 36559 "" "Prometheus/2.43.0" 2023-09-23T20:37:45.048 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 89 16 88952 88936 62.4272 7.5 0.0266472 0.0160116 2023-09-23T20:37:45.143 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:45.253 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:37:45.257 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:45.257 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:37:45.358 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:37:45.385 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:45.436 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:37:45.439 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:45.439 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:37:45.540 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:37:45.561 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:45 smithi018 ceph-mon[179647]: failed to encode map e400 with expected crc 2023-09-23T20:37:45.562 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:45 smithi018 ceph-mon[179647]: pgmap v965: 119 pgs: 2 peering, 2 active+recovering+undersized+remapped, 1 active+recovery_wait, 10 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 5.4 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 25 MiB/s wr, 392 op/s; 41859/264162 objects degraded (15.846%); 15066/264162 objects misplaced (5.703%); 2.5 MiB/s, 41 objects/s recovering 2023-09-23T20:37:45.562 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:45 smithi018 ceph-mon[179647]: failed to encode map e401 with expected crc 2023-09-23T20:37:45.562 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:45 smithi018 ceph-mon[179647]: osdmap e401: 8 total, 8 up, 6 in 2023-09-23T20:37:45.562 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:45 smithi018 ceph-mon[179647]: failed to encode map e401 with expected crc 2023-09-23T20:37:45.562 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:45 smithi018 ceph-mon[179647]: failed to encode map e401 with expected crc 2023-09-23T20:37:45.566 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:45 smithi018 ceph-mon[184144]: failed to encode map e400 with expected crc 2023-09-23T20:37:45.566 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:45 smithi018 ceph-mon[184144]: pgmap v965: 119 pgs: 2 peering, 2 active+recovering+undersized+remapped, 1 active+recovery_wait, 10 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 5.4 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 25 MiB/s wr, 392 op/s; 41859/264162 objects degraded (15.846%); 15066/264162 objects misplaced (5.703%); 2.5 MiB/s, 41 objects/s recovering 2023-09-23T20:37:45.566 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:45 smithi018 ceph-mon[184144]: failed to encode map e401 with expected crc 2023-09-23T20:37:45.566 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:45 smithi018 ceph-mon[184144]: osdmap e401: 8 total, 8 up, 6 in 2023-09-23T20:37:45.567 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:45 smithi018 ceph-mon[184144]: failed to encode map e401 with expected crc 2023-09-23T20:37:45.567 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:45 smithi018 ceph-mon[184144]: failed to encode map e401 with expected crc 2023-09-23T20:37:45.569 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:45.610 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:37:45.615 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:45.615 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:37:45.707 INFO:teuthology.orchestra.run.smithi018.stdout:ERROR: (22) Invalid argument 2023-09-23T20:37:45.707 INFO:teuthology.orchestra.run.smithi018.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:37:45.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:45 smithi149 ceph-mon[170394]: failed to encode map e400 with expected crc 2023-09-23T20:37:45.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:45 smithi149 ceph-mon[170394]: pgmap v965: 119 pgs: 2 peering, 2 active+recovering+undersized+remapped, 1 active+recovery_wait, 10 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 5.4 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 25 MiB/s wr, 392 op/s; 41859/264162 objects degraded (15.846%); 15066/264162 objects misplaced (5.703%); 2.5 MiB/s, 41 objects/s recovering 2023-09-23T20:37:45.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:45 smithi149 ceph-mon[170394]: failed to encode map e401 with expected crc 2023-09-23T20:37:45.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:45 smithi149 ceph-mon[170394]: osdmap e401: 8 total, 8 up, 6 in 2023-09-23T20:37:45.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:45 smithi149 ceph-mon[170394]: failed to encode map e401 with expected crc 2023-09-23T20:37:45.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:45 smithi149 ceph-mon[170394]: failed to encode map e401 with expected crc 2023-09-23T20:37:45.717 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:37:45.747 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:45.784 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:37:45.784 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:37:45.787 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:45.787 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:37:45.831 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:37:45.885 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:37:45.888 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:37:45.951 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:37:45.987 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:46.015 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:37:46.048 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 90 2 89320 89318 61.999 23.875 0.0625055 0.0161219 2023-09-23T20:37:46.055 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:37:46.109 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:37:46.109 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:37:46.109 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:37:46.110 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:37:46.110 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:37:46.110 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:37:46.110 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:37:46.110 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:37:46.110 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:37:46.110 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:37:46.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:46 smithi018 ceph-mon[179647]: failed to encode map e401 with expected crc 2023-09-23T20:37:46.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:46 smithi018 ceph-mon[179647]: failed to encode map e401 with expected crc 2023-09-23T20:37:46.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:46 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2888801238' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:37:46.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:46 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 41859/264162 objects degraded (15.846%), 10 pgs degraded (PG_DEGRADED) 2023-09-23T20:37:46.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:46 smithi018 ceph-mon[184144]: failed to encode map e401 with expected crc 2023-09-23T20:37:46.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:46 smithi018 ceph-mon[184144]: failed to encode map e401 with expected crc 2023-09-23T20:37:46.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:46 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2888801238' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:37:46.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:46 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 41859/264162 objects degraded (15.846%), 10 pgs degraded (PG_DEGRADED) 2023-09-23T20:37:46.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:46 smithi149 ceph-mon[170394]: failed to encode map e401 with expected crc 2023-09-23T20:37:46.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:46 smithi149 ceph-mon[170394]: failed to encode map e401 with expected crc 2023-09-23T20:37:46.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:46 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2888801238' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:37:46.968 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:46 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 41859/264162 objects degraded (15.846%), 10 pgs degraded (PG_DEGRADED) 2023-09-23T20:37:47.048 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Total time run: 90.0414 2023-09-23T20:37:47.048 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Total writes made: 89320 2023-09-23T20:37:47.049 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Write size: 65536 2023-09-23T20:37:47.049 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Object size: 65536 2023-09-23T20:37:47.049 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Bandwidth (MB/sec): 61.9993 2023-09-23T20:37:47.049 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Stddev Bandwidth: 49.5305 2023-09-23T20:37:47.049 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Max bandwidth (MB/sec): 191.312 2023-09-23T20:37:47.049 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Min bandwidth (MB/sec): 0 2023-09-23T20:37:47.049 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Average IOPS: 991 2023-09-23T20:37:47.049 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Stddev IOPS: 792.489 2023-09-23T20:37:47.049 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Max IOPS: 3061 2023-09-23T20:37:47.049 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Min IOPS: 0 2023-09-23T20:37:47.049 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Average Latency(s): 0.0161229 2023-09-23T20:37:47.050 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Stddev Latency(s): 0.23715 2023-09-23T20:37:47.050 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Max latency(s): 19.3764 2023-09-23T20:37:47.050 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Min latency(s): 0.00120245 2023-09-23T20:37:47.060 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Cleaning up (deleting benchmark objects) 2023-09-23T20:37:47.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:47 smithi018 ceph-mon[179647]: pgmap v967: 119 pgs: 2 peering, 2 active+recovering+undersized+remapped, 1 active+recovery_wait, 10 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 5.4 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 21 MiB/s wr, 331 op/s; 41859/266976 objects degraded (15.679%); 14511/266976 objects misplaced (5.435%); 5.8 MiB/s, 94 objects/s recovering 2023-09-23T20:37:47.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:47 smithi018 ceph-mon[184144]: pgmap v967: 119 pgs: 2 peering, 2 active+recovering+undersized+remapped, 1 active+recovery_wait, 10 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 5.4 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 21 MiB/s wr, 331 op/s; 41859/266976 objects degraded (15.679%); 14511/266976 objects misplaced (5.435%); 5.8 MiB/s, 94 objects/s recovering 2023-09-23T20:37:47.932 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:37:47.937 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:47.938 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:37:47.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:47 smithi149 ceph-mon[170394]: pgmap v967: 119 pgs: 2 peering, 2 active+recovering+undersized+remapped, 1 active+recovery_wait, 10 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 5.4 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 21 MiB/s wr, 331 op/s; 41859/266976 objects degraded (15.679%); 14511/266976 objects misplaced (5.435%); 5.8 MiB/s, 94 objects/s recovering 2023-09-23T20:37:48.039 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:37:48.140 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:48.602 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T20:37:48.811 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:37:48.821 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:48.822 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:37:48.908 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:37:48.923 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:37:49.039 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:49.306 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 7, 3, 2, 4] out_osds: [6, 5] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T20:37:49.307 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:37:49.307 INFO:tasks.thrashosds.thrasher:Adding osd 6 2023-09-23T20:37:49.307 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd in 6 2023-09-23T20:37:49.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:49 smithi018 ceph-mon[184144]: pgmap v968: 119 pgs: 2 peering, 2 active+recovering+undersized+remapped, 1 active+recovery_wait, 10 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 5.4 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 16 MiB/s wr, 262 op/s; 41859/267141 objects degraded (15.669%); 14511/267141 objects misplaced (5.432%); 4.3 MiB/s, 69 objects/s recovering 2023-09-23T20:37:49.896 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:49 smithi018 ceph-mon[179647]: pgmap v968: 119 pgs: 2 peering, 2 active+recovering+undersized+remapped, 1 active+recovery_wait, 10 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 5.4 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 16 MiB/s wr, 262 op/s; 41859/267141 objects degraded (15.669%); 14511/267141 objects misplaced (5.432%); 4.3 MiB/s, 69 objects/s recovering 2023-09-23T20:37:49.904 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:37:49.912 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:49.913 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:37:49.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:49 smithi149 ceph-mon[170394]: pgmap v968: 119 pgs: 2 peering, 2 active+recovering+undersized+remapped, 1 active+recovery_wait, 10 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 5.4 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 16 MiB/s wr, 262 op/s; 41859/267141 objects degraded (15.669%); 14511/267141 objects misplaced (5.432%); 4.3 MiB/s, 69 objects/s recovering 2023-09-23T20:37:50.013 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:37:50.107 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:50.246 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:37:50.255 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:50.256 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:37:50.357 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:37:50.466 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:50.797 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:37:50.813 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:50.814 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:37:50.886 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:37:50.915 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:37:50.972 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:51.012 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:37:51.015 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:51.015 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:37:51.116 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:37:51.145 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:51.186 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:37:51.194 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:51.194 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:37:51.295 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:37:51.324 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:51.359 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:37:51.364 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:51.365 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:37:51.466 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:37:51.494 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:51.537 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:37:51.540 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:51.541 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:37:51.642 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:37:51.672 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:51.766 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:37:51.768 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:51.769 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:37:51.870 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:37:51.898 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:51 smithi149 ceph-mon[170394]: pgmap v969: 119 pgs: 2 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 1 active+recovery_wait, 10 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 5.4 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 13 MiB/s wr, 254 op/s; 50878/266748 objects degraded (19.073%); 14511/266748 objects misplaced (5.440%); 4.3 MiB/s, 69 objects/s recovering 2023-09-23T20:37:51.898 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:51 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:37:51.899 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:51 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 50878/266748 objects degraded (19.073%), 12 pgs degraded (PG_DEGRADED) 2023-09-23T20:37:51.902 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:51.941 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:37:51.945 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:51.946 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:37:52.046 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:37:52.124 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:51 smithi018 ceph-mon[179647]: pgmap v969: 119 pgs: 2 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 1 active+recovery_wait, 10 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 5.4 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 13 MiB/s wr, 254 op/s; 50878/266748 objects degraded (19.073%); 14511/266748 objects misplaced (5.440%); 4.3 MiB/s, 69 objects/s recovering 2023-09-23T20:37:52.125 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:51 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:37:52.125 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:51 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 50878/266748 objects degraded (19.073%), 12 pgs degraded (PG_DEGRADED) 2023-09-23T20:37:52.125 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:51 smithi018 ceph-mon[184144]: pgmap v969: 119 pgs: 2 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 1 active+recovery_wait, 10 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 5.4 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 13 MiB/s wr, 254 op/s; 50878/266748 objects degraded (19.073%); 14511/266748 objects misplaced (5.440%); 4.3 MiB/s, 69 objects/s recovering 2023-09-23T20:37:52.125 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:51 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:37:52.125 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:51 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 50878/266748 objects degraded (19.073%), 12 pgs degraded (PG_DEGRADED) 2023-09-23T20:37:52.134 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:53.921 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:37:53.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:53 smithi149 ceph-mon[170394]: pgmap v970: 119 pgs: 2 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 1 active+recovery_wait, 10 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 5.4 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 11 MiB/s wr, 298 op/s; 50878/264825 objects degraded (19.212%); 14018/264825 objects misplaced (5.293%); 6.5 MiB/s, 105 objects/s recovering 2023-09-23T20:37:54.012 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:53 smithi018 ceph-mon[179647]: pgmap v970: 119 pgs: 2 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 1 active+recovery_wait, 10 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 5.4 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 11 MiB/s wr, 298 op/s; 50878/264825 objects degraded (19.212%); 14018/264825 objects misplaced (5.293%); 6.5 MiB/s, 105 objects/s recovering 2023-09-23T20:37:54.013 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:53 smithi018 ceph-mon[184144]: pgmap v970: 119 pgs: 2 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 1 active+recovery_wait, 10 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 5.4 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 11 MiB/s wr, 298 op/s; 50878/264825 objects degraded (19.212%); 14018/264825 objects misplaced (5.293%); 6.5 MiB/s, 105 objects/s recovering 2023-09-23T20:37:54.017 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:37:54.032 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:54.032 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:37:54.075 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:37:54.134 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:37:54.243 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:55.302 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:37:55.315 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:55.316 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:37:55.319 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:37:54 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:37:54] "GET /metrics HTTP/1.1" 200 36559 "" "Prometheus/2.43.0" 2023-09-23T20:37:55.417 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:37:55.454 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:55.498 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:37:55.501 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:55.502 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:37:55.502 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:37:55.603 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:37:55.631 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:55.668 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:37:55.672 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:55.673 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:37:55.773 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:37:55.802 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:55.846 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:37:55.850 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:55.851 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:37:55.857 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:55 smithi018 ceph-mon[179647]: pgmap v971: 119 pgs: 2 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 1 active+recovery_wait, 10 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 5.4 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 102 B/s rd, 8.1 MiB/s wr, 291 op/s; 50761/263169 objects degraded (19.288%); 14018/263169 objects misplaced (5.327%); 6.3 MiB/s, 100 objects/s recovering 2023-09-23T20:37:55.857 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:55 smithi018 ceph-mon[184144]: pgmap v971: 119 pgs: 2 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 1 active+recovery_wait, 10 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 5.4 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 102 B/s rd, 8.1 MiB/s wr, 291 op/s; 50761/263169 objects degraded (19.288%); 14018/263169 objects misplaced (5.327%); 6.3 MiB/s, 100 objects/s recovering 2023-09-23T20:37:55.952 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:37:55.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:55 smithi149 ceph-mon[170394]: pgmap v971: 119 pgs: 2 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 1 active+recovery_wait, 10 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 5.4 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 102 B/s rd, 8.1 MiB/s wr, 291 op/s; 50761/263169 objects degraded (19.288%); 14018/263169 objects misplaced (5.327%); 6.3 MiB/s, 100 objects/s recovering 2023-09-23T20:37:55.984 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:56.023 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:37:56.027 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:56.027 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:37:56.128 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:37:56.162 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:56.199 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:37:56.202 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:56.202 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:37:56.303 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:37:56.334 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:56.382 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:37:56.388 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:56.388 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:37:56.489 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:37:56.637 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:56.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:56 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:37:56.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:56 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:37:56.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:56 smithi018 ceph-mon[184144]: pgmap v972: 119 pgs: 2 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 1 active+recovery_wait, 10 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 5.3 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 89 B/s rd, 5.0 MiB/s wr, 342 op/s; 50471/259065 objects degraded (19.482%); 13643/259065 objects misplaced (5.266%); 7.8 MiB/s, 125 objects/s recovering 2023-09-23T20:37:56.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:56 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 50761/263169 objects degraded (19.288%), 12 pgs degraded (PG_DEGRADED) 2023-09-23T20:37:56.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:56 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi149", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:37:56.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:56 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:37:56.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:56 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:37:56.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:56 smithi018 ceph-mon[179647]: pgmap v972: 119 pgs: 2 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 1 active+recovery_wait, 10 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 5.3 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 89 B/s rd, 5.0 MiB/s wr, 342 op/s; 50471/259065 objects degraded (19.482%); 13643/259065 objects misplaced (5.266%); 7.8 MiB/s, 125 objects/s recovering 2023-09-23T20:37:56.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:56 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 50761/263169 objects degraded (19.288%), 12 pgs degraded (PG_DEGRADED) 2023-09-23T20:37:56.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:56 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi149", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:37:57.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:56 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:37:57.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:56 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:37:57.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:56 smithi149 ceph-mon[170394]: pgmap v972: 119 pgs: 2 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 1 active+recovery_wait, 10 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 5.3 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 89 B/s rd, 5.0 MiB/s wr, 342 op/s; 50471/259065 objects degraded (19.482%); 13643/259065 objects misplaced (5.266%); 7.8 MiB/s, 125 objects/s recovering 2023-09-23T20:37:57.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:56 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 50761/263169 objects degraded (19.288%), 12 pgs degraded (PG_DEGRADED) 2023-09-23T20:37:57.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:56 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi149", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:37:57.968 INFO:teuthology.orchestra.run.smithi018.stderr:nodeep-scrub is unset 2023-09-23T20:37:58.049 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:37:58.057 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:58.057 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:37:58.061 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:57 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2159653337' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:37:58.061 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:57 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2159653337' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:37:58.158 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:37:58.184 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:57 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2159653337' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:37:58.186 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:58.227 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:37:58.230 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:58.231 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:37:58.332 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:37:58.360 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:58.398 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:37:58.402 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:58.402 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:37:58.503 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:37:58.533 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:58.570 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:37:58.573 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:58.574 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:37:58.674 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:37:58.780 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:59.009 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:58 smithi018 ceph-mon[184144]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:37:59.010 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:58 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2159653337' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:37:59.010 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:58 smithi018 ceph-mon[184144]: osdmap e402: 8 total, 8 up, 6 in 2023-09-23T20:37:59.010 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:58 smithi018 ceph-mon[184144]: failed to encode map e402 with expected crc 2023-09-23T20:37:59.010 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:58 smithi018 ceph-mon[184144]: failed to encode map e402 with expected crc 2023-09-23T20:37:59.010 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:37:58 smithi018 ceph-mon[184144]: pgmap v974: 119 pgs: 2 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 1 active+recovery_wait, 10 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 5.3 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 1.7 MiB/s wr, 348 op/s; 50471/258354 objects degraded (19.536%); 13643/258354 objects misplaced (5.281%); 6.4 MiB/s, 101 objects/s recovering 2023-09-23T20:37:59.013 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:58 smithi018 ceph-mon[179647]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:37:59.013 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:58 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2159653337' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:37:59.013 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:58 smithi018 ceph-mon[179647]: osdmap e402: 8 total, 8 up, 6 in 2023-09-23T20:37:59.013 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:58 smithi018 ceph-mon[179647]: failed to encode map e402 with expected crc 2023-09-23T20:37:59.013 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:58 smithi018 ceph-mon[179647]: failed to encode map e402 with expected crc 2023-09-23T20:37:59.013 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:37:58 smithi018 ceph-mon[179647]: pgmap v974: 119 pgs: 2 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 1 active+recovery_wait, 10 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 5.3 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 1.7 MiB/s wr, 348 op/s; 50471/258354 objects degraded (19.536%); 13643/258354 objects misplaced (5.281%); 6.4 MiB/s, 101 objects/s recovering 2023-09-23T20:37:59.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:58 smithi149 ceph-mon[170394]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:37:59.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:58 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2159653337' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:37:59.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:58 smithi149 ceph-mon[170394]: osdmap e402: 8 total, 8 up, 6 in 2023-09-23T20:37:59.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:58 smithi149 ceph-mon[170394]: failed to encode map e402 with expected crc 2023-09-23T20:37:59.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:58 smithi149 ceph-mon[170394]: failed to encode map e402 with expected crc 2023-09-23T20:37:59.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:37:58 smithi149 ceph-mon[170394]: pgmap v974: 119 pgs: 2 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 1 active+recovery_wait, 10 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 5.3 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 1.7 MiB/s wr, 348 op/s; 50471/258354 objects degraded (19.536%); 13643/258354 objects misplaced (5.281%); 6.4 MiB/s, 101 objects/s recovering 2023-09-23T20:37:59.281 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:37:59.293 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:59.293 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:37:59.394 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:37:59.422 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:59.464 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:37:59.467 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:59.467 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:37:59.520 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2023-09-23T20:37:59.521 INFO:teuthology.orchestra.run.smithi018.stdout: "size": 20, 2023-09-23T20:37:59.521 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 600, 2023-09-23T20:37:59.521 INFO:teuthology.orchestra.run.smithi018.stdout: "ops": [ 2023-09-23T20:37:59.521 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.521 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.45161.0:205351 6.11 6:8940b4e2:::benchmark_data_smithi018_270558_object100612:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e292)", 2023-09-23T20:37:59.521 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:31:14.403632+0000", 2023-09-23T20:37:59.522 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 405.11255485599997, 2023-09-23T20:37:59.522 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.030579779, 2023-09-23T20:37:59.522 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:37:59.522 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:37:59.522 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:37:59.522 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.45161", 2023-09-23T20:37:59.523 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3795614987", 2023-09-23T20:37:59.523 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 205351 2023-09-23T20:37:59.523 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.523 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:37:59.523 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.523 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:37:59.523 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:14.403632+0000", 2023-09-23T20:37:59.524 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:37:59.524 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.524 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.524 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:37:59.524 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:14.403632+0000", 2023-09-23T20:37:59.524 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:37:59.524 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.525 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.525 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:37:59.525 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:14.403630+0000", 2023-09-23T20:37:59.525 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:37:59.525 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.525 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.525 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:37:59.525 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:14.403641+0000", 2023-09-23T20:37:59.525 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1354e-05 2023-09-23T20:37:59.525 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.525 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.525 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:37:59.525 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:14.403656+0000", 2023-09-23T20:37:59.526 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.435e-05 2023-09-23T20:37:59.526 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.526 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.526 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:37:59.526 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:14.403665+0000", 2023-09-23T20:37:59.526 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.5519999999999993e-06 2023-09-23T20:37:59.526 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.526 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.526 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:37:59.526 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:14.403729+0000", 2023-09-23T20:37:59.526 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.3886999999999993e-05 2023-09-23T20:37:59.527 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.527 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.527 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:37:59.527 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:14.403741+0000", 2023-09-23T20:37:59.527 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1620999999999999e-05 2023-09-23T20:37:59.527 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.527 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.527 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:37:59.527 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:15.410699+0000", 2023-09-23T20:37:59.527 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0069581620000001 2023-09-23T20:37:59.527 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.527 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.528 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:37:59.528 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:15.410843+0000", 2023-09-23T20:37:59.528 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00014438 2023-09-23T20:37:59.528 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.528 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.528 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 2,6", 2023-09-23T20:37:59.528 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:15.411260+0000", 2023-09-23T20:37:59.528 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00041685600000000001 2023-09-23T20:37:59.528 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.528 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.528 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:37:59.528 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:15.416401+0000", 2023-09-23T20:37:59.528 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0051409209999999997 2023-09-23T20:37:59.529 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.529 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.529 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:37:59.529 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:15.422105+0000", 2023-09-23T20:37:59.529 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0057035250000000001 2023-09-23T20:37:59.529 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.529 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.529 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:37:59.530 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:15.434115+0000", 2023-09-23T20:37:59.530 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.012010758 2023-09-23T20:37:59.530 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.530 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.530 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:37:59.530 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:15.434165+0000", 2023-09-23T20:37:59.530 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.9317999999999998e-05 2023-09-23T20:37:59.530 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.531 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.531 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:37:59.531 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:15.434212+0000", 2023-09-23T20:37:59.531 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.6968000000000002e-05 2023-09-23T20:37:59.531 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:37:59.531 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:37:59.531 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:37:59.531 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.531 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.531 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.45161.0:205365 6.11 6:8afc9a6e:::benchmark_data_smithi018_270558_object100626:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e292)", 2023-09-23T20:37:59.531 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:31:14.428052+0000", 2023-09-23T20:37:59.531 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 405.08813433500001, 2023-09-23T20:37:59.532 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0435979980000001, 2023-09-23T20:37:59.532 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:37:59.532 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:37:59.532 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:37:59.532 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.45161", 2023-09-23T20:37:59.532 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3795614987", 2023-09-23T20:37:59.532 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 205365 2023-09-23T20:37:59.532 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.532 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:37:59.532 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.532 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:37:59.532 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:14.428052+0000", 2023-09-23T20:37:59.532 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:37:59.533 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.533 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.533 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:37:59.533 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:14.428052+0000", 2023-09-23T20:37:59.533 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:37:59.533 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.533 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.533 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:37:59.533 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:14.428050+0000", 2023-09-23T20:37:59.533 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999971 2023-09-23T20:37:59.533 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.534 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.534 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:37:59.534 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:14.428065+0000", 2023-09-23T20:37:59.534 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.5152e-05 2023-09-23T20:37:59.534 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.534 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.534 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:37:59.534 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:14.428084+0000", 2023-09-23T20:37:59.534 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.9346000000000001e-05 2023-09-23T20:37:59.534 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.534 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.535 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:37:59.535 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:14.428097+0000", 2023-09-23T20:37:59.535 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2559e-05 2023-09-23T20:37:59.535 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.535 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.535 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:37:59.535 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:14.435340+0000", 2023-09-23T20:37:59.535 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0072437339999999999 2023-09-23T20:37:59.535 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.535 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.535 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:37:59.535 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:14.435360+0000", 2023-09-23T20:37:59.535 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.9942e-05 2023-09-23T20:37:59.536 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.536 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.536 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:37:59.536 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:15.416434+0000", 2023-09-23T20:37:59.536 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.98107409499999998 2023-09-23T20:37:59.536 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.536 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.536 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:37:59.536 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:15.416578+0000", 2023-09-23T20:37:59.536 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00014369799999999999 2023-09-23T20:37:59.536 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.536 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.536 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 2,6", 2023-09-23T20:37:59.537 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:15.417093+0000", 2023-09-23T20:37:59.537 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00051506499999999999 2023-09-23T20:37:59.537 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.537 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.537 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:37:59.537 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:15.424390+0000", 2023-09-23T20:37:59.537 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0072970149999999996 2023-09-23T20:37:59.537 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.537 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.538 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:37:59.538 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:15.424416+0000", 2023-09-23T20:37:59.538 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.6003000000000002e-05 2023-09-23T20:37:59.538 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.538 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.538 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:37:59.538 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:15.471556+0000", 2023-09-23T20:37:59.538 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.047139595999999999 2023-09-23T20:37:59.538 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.538 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.538 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:37:59.538 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:15.471611+0000", 2023-09-23T20:37:59.538 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.5606999999999998e-05 2023-09-23T20:37:59.539 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.539 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.539 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:37:59.539 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:31:15.471650+0000", 2023-09-23T20:37:59.539 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.8862999999999997e-05 2023-09-23T20:37:59.539 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:37:59.539 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:37:59.539 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:37:59.539 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.539 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.539 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:57591 7.1 7:8554f45e:::benchmark_data_smithi018_308601_object57590:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e313)", 2023-09-23T20:37:59.540 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:32:33.220481+0000", 2023-09-23T20:37:59.540 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 326.29570583999998, 2023-09-23T20:37:59.540 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3899836859999999, 2023-09-23T20:37:59.540 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:37:59.540 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:37:59.540 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:37:59.540 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:37:59.540 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:37:59.540 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 57591 2023-09-23T20:37:59.540 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.540 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:37:59.540 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.541 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:37:59.541 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.220481+0000", 2023-09-23T20:37:59.541 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:37:59.541 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.541 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.541 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:37:59.541 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.220481+0000", 2023-09-23T20:37:59.541 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:37:59.541 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.541 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.541 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:37:59.541 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.220479+0000", 2023-09-23T20:37:59.541 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:37:59.542 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.542 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.542 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:37:59.542 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.220511+0000", 2023-09-23T20:37:59.542 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.1970000000000001e-05 2023-09-23T20:37:59.542 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.542 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.542 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:37:59.542 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.220523+0000", 2023-09-23T20:37:59.542 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2105e-05 2023-09-23T20:37:59.542 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.542 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.543 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:37:59.543 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.220535+0000", 2023-09-23T20:37:59.543 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1235e-05 2023-09-23T20:37:59.543 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.543 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.543 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:37:59.543 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.221636+0000", 2023-09-23T20:37:59.543 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0011011759999999999 2023-09-23T20:37:59.543 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.543 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.543 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:37:59.543 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.221643+0000", 2023-09-23T20:37:59.544 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.5610000000000001e-06 2023-09-23T20:37:59.544 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.544 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.544 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:37:59.544 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.609130+0000", 2023-09-23T20:37:59.544 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.387486223 2023-09-23T20:37:59.544 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.544 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.544 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:37:59.544 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.610465+0000", 2023-09-23T20:37:59.544 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0013348940000000001 2023-09-23T20:37:59.544 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:37:59.544 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:37:59.545 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:37:59.545 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.545 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.545 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:57595 7.1 7:8b518598:::benchmark_data_smithi018_308601_object57594:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e313)", 2023-09-23T20:37:59.545 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:32:33.220549+0000", 2023-09-23T20:37:59.545 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 326.29563762800001, 2023-09-23T20:37:59.545 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.4131732800000001, 2023-09-23T20:37:59.545 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:37:59.545 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:37:59.545 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:37:59.545 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:37:59.545 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:37:59.545 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 57595 2023-09-23T20:37:59.546 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.546 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:37:59.546 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.546 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:37:59.546 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.220549+0000", 2023-09-23T20:37:59.546 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:37:59.546 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.546 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.546 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:37:59.546 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.220549+0000", 2023-09-23T20:37:59.546 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:37:59.546 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.547 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.547 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:37:59.547 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.220549+0000", 2023-09-23T20:37:59.547 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:37:59.547 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.547 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.547 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:37:59.547 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.220574+0000", 2023-09-23T20:37:59.547 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.5612000000000001e-05 2023-09-23T20:37:59.547 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.547 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.547 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:37:59.547 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.220582+0000", 2023-09-23T20:37:59.548 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.8269999999999998e-06 2023-09-23T20:37:59.548 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.548 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.548 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:37:59.548 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.220588+0000", 2023-09-23T20:37:59.548 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.1349999999999997e-06 2023-09-23T20:37:59.548 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.548 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.548 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:37:59.548 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.221656+0000", 2023-09-23T20:37:59.548 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.001068098 2023-09-23T20:37:59.548 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.548 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.549 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:37:59.549 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.221660+0000", 2023-09-23T20:37:59.549 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.58e-06 2023-09-23T20:37:59.549 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.549 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.549 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:37:59.549 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.613460+0000", 2023-09-23T20:37:59.549 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3918001280000001 2023-09-23T20:37:59.549 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.549 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.549 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for degraded object", 2023-09-23T20:37:59.549 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.614021+0000", 2023-09-23T20:37:59.550 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00056140200000000002 2023-09-23T20:37:59.550 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.550 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.550 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:37:59.550 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.633574+0000", 2023-09-23T20:37:59.550 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.019552403999999999 2023-09-23T20:37:59.550 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.550 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.550 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:37:59.551 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.633722+0000", 2023-09-23T20:37:59.551 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00014861 2023-09-23T20:37:59.551 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:37:59.551 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:37:59.551 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:37:59.551 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.551 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.551 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:57641 7.1 7:82e04a7d:::benchmark_data_smithi018_308601_object57640:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e313)", 2023-09-23T20:37:59.551 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:32:33.220594+0000", 2023-09-23T20:37:59.551 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 326.29559316500001, 2023-09-23T20:37:59.551 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.41538217, 2023-09-23T20:37:59.551 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:37:59.551 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:37:59.552 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:37:59.552 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:37:59.552 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:37:59.552 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 57641 2023-09-23T20:37:59.552 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.552 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:37:59.552 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.552 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:37:59.552 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.220594+0000", 2023-09-23T20:37:59.552 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:37:59.552 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.552 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.553 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:37:59.553 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.220594+0000", 2023-09-23T20:37:59.553 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:37:59.553 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.553 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.553 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:37:59.553 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.220593+0000", 2023-09-23T20:37:59.553 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:37:59.553 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.553 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.553 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:37:59.553 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.220619+0000", 2023-09-23T20:37:59.553 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.6431000000000002e-05 2023-09-23T20:37:59.554 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.554 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.554 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:37:59.554 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.220626+0000", 2023-09-23T20:37:59.554 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.6259999999999997e-06 2023-09-23T20:37:59.554 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.554 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.554 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:37:59.554 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.220631+0000", 2023-09-23T20:37:59.554 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.215e-06 2023-09-23T20:37:59.554 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.555 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.555 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:37:59.555 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.221678+0000", 2023-09-23T20:37:59.555 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.001046892 2023-09-23T20:37:59.555 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.555 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.555 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:37:59.555 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.221682+0000", 2023-09-23T20:37:59.555 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.0300000000000004e-06 2023-09-23T20:37:59.555 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.555 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.555 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:37:59.555 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.614054+0000", 2023-09-23T20:37:59.556 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3923719809999999 2023-09-23T20:37:59.556 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.556 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.556 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:37:59.556 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.614542+0000", 2023-09-23T20:37:59.556 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00048730699999999997 2023-09-23T20:37:59.556 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.556 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.556 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 3,4,6", 2023-09-23T20:37:59.556 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.614721+0000", 2023-09-23T20:37:59.556 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00017952699999999999 2023-09-23T20:37:59.556 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.556 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.557 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:37:59.557 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.625753+0000", 2023-09-23T20:37:59.557 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.011032036 2023-09-23T20:37:59.557 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.557 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.557 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:37:59.557 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.625939+0000", 2023-09-23T20:37:59.557 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00018612199999999999 2023-09-23T20:37:59.557 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.557 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.557 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:37:59.557 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.628311+0000", 2023-09-23T20:37:59.557 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0023717389999999999 2023-09-23T20:37:59.558 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.558 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.558 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:37:59.558 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.633817+0000", 2023-09-23T20:37:59.558 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0055056690000000004 2023-09-23T20:37:59.558 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.558 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.558 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:37:59.558 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.633889+0000", 2023-09-23T20:37:59.558 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.2037000000000002e-05 2023-09-23T20:37:59.558 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.558 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.559 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:37:59.559 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.635976+0000", 2023-09-23T20:37:59.559 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0020870390000000002 2023-09-23T20:37:59.559 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:37:59.559 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:37:59.559 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:37:59.559 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.559 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.559 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:57647 7.1 7:8e361ecd:::benchmark_data_smithi018_308601_object57646:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e313)", 2023-09-23T20:37:59.559 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:32:33.220636+0000", 2023-09-23T20:37:59.559 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 326.29555081299998, 2023-09-23T20:37:59.559 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.417278721, 2023-09-23T20:37:59.559 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:37:59.560 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:37:59.560 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:37:59.560 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:37:59.560 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:37:59.560 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 57647 2023-09-23T20:37:59.560 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.560 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:37:59.560 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.560 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:37:59.560 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.220636+0000", 2023-09-23T20:37:59.561 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:37:59.561 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.561 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.561 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:37:59.561 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.220636+0000", 2023-09-23T20:37:59.561 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:37:59.561 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.561 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.561 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:37:59.561 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.220635+0000", 2023-09-23T20:37:59.561 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:37:59.561 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.561 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.562 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:37:59.562 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.220659+0000", 2023-09-23T20:37:59.562 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.3969999999999999e-05 2023-09-23T20:37:59.562 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.562 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.562 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:37:59.562 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.220666+0000", 2023-09-23T20:37:59.562 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.2299999999999996e-06 2023-09-23T20:37:59.562 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.562 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.562 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:37:59.562 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.220671+0000", 2023-09-23T20:37:59.562 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.9540000000000003e-06 2023-09-23T20:37:59.563 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.563 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.563 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:37:59.563 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.221699+0000", 2023-09-23T20:37:59.563 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0010287359999999999 2023-09-23T20:37:59.563 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.563 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.563 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:37:59.563 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.221703+0000", 2023-09-23T20:37:59.563 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.765e-06 2023-09-23T20:37:59.563 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.563 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.564 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:37:59.564 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.616911+0000", 2023-09-23T20:37:59.564 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3952080200000001 2023-09-23T20:37:59.564 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.564 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.564 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:37:59.564 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.617068+0000", 2023-09-23T20:37:59.564 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000156484 2023-09-23T20:37:59.564 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.564 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.564 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 3,4,6", 2023-09-23T20:37:59.564 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.617232+0000", 2023-09-23T20:37:59.565 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000163974 2023-09-23T20:37:59.565 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.565 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.565 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:37:59.565 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.625797+0000", 2023-09-23T20:37:59.565 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0085653410000000006 2023-09-23T20:37:59.565 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.565 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.565 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:37:59.565 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.625946+0000", 2023-09-23T20:37:59.565 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000148947 2023-09-23T20:37:59.565 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.565 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.566 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:37:59.566 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.629107+0000", 2023-09-23T20:37:59.566 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0031612279999999999 2023-09-23T20:37:59.566 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.566 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.566 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:37:59.566 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.636021+0000", 2023-09-23T20:37:59.566 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0069139689999999998 2023-09-23T20:37:59.566 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.566 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.566 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:37:59.566 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.636062+0000", 2023-09-23T20:37:59.566 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.0567e-05 2023-09-23T20:37:59.567 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.567 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.567 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:37:59.567 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.637915+0000", 2023-09-23T20:37:59.567 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.001852958 2023-09-23T20:37:59.567 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:37:59.567 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:37:59.567 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:37:59.567 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.567 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.567 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:57652 7.1 7:803e82a9:::benchmark_data_smithi018_308601_object57651:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e313)", 2023-09-23T20:37:59.567 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:32:33.220675+0000", 2023-09-23T20:37:59.567 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 326.29551136399999, 2023-09-23T20:37:59.568 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.4243310039999999, 2023-09-23T20:37:59.568 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:37:59.568 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:37:59.568 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:37:59.568 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:37:59.568 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:37:59.568 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 57652 2023-09-23T20:37:59.568 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.568 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:37:59.568 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.568 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:37:59.568 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.220675+0000", 2023-09-23T20:37:59.568 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:37:59.569 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.569 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.569 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:37:59.569 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.220675+0000", 2023-09-23T20:37:59.569 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:37:59.569 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.569 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.569 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:37:59.569 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.220675+0000", 2023-09-23T20:37:59.569 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:37:59.569 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.569 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.569 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:37:59.570 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.220708+0000", 2023-09-23T20:37:59.570 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.3080000000000002e-05 2023-09-23T20:37:59.570 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.570 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.570 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:37:59.570 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.220716+0000", 2023-09-23T20:37:59.570 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.7600000000000002e-06 2023-09-23T20:37:59.570 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.570 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.570 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:37:59.570 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.220721+0000", 2023-09-23T20:37:59.571 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.2510000000000002e-06 2023-09-23T20:37:59.571 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.571 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.571 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:37:59.571 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.221715+0000", 2023-09-23T20:37:59.571 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000993774 2023-09-23T20:37:59.571 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.571 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.571 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:37:59.572 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.221718+0000", 2023-09-23T20:37:59.572 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.5090000000000002e-06 2023-09-23T20:37:59.572 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.572 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.572 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:37:59.572 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.620216+0000", 2023-09-23T20:37:59.572 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.398498002 2023-09-23T20:37:59.572 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.572 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.572 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:37:59.572 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.620417+0000", 2023-09-23T20:37:59.572 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00020063700000000001 2023-09-23T20:37:59.572 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.573 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.573 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 3,4,6", 2023-09-23T20:37:59.573 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.620597+0000", 2023-09-23T20:37:59.573 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000180306 2023-09-23T20:37:59.573 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.573 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.573 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:37:59.573 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.625927+0000", 2023-09-23T20:37:59.573 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.005329828 2023-09-23T20:37:59.573 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.573 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.573 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:37:59.574 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.628080+0000", 2023-09-23T20:37:59.574 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0021524550000000002 2023-09-23T20:37:59.574 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.574 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.574 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:37:59.574 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.629309+0000", 2023-09-23T20:37:59.574 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.001229162 2023-09-23T20:37:59.574 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.574 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.574 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:37:59.574 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.641922+0000", 2023-09-23T20:37:59.575 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.012613177999999999 2023-09-23T20:37:59.575 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.575 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.575 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:37:59.575 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.641958+0000", 2023-09-23T20:37:59.575 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.5973000000000003e-05 2023-09-23T20:37:59.575 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.575 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.575 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:37:59.575 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.645006+0000", 2023-09-23T20:37:59.575 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0030484900000000001 2023-09-23T20:37:59.575 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:37:59.575 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:37:59.576 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:37:59.576 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.576 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.576 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:156899 7.1 7:894832cf:::benchmark_data_smithi018_308601_object62347:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e354)", 2023-09-23T20:37:59.576 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:34:59.972856+0000", 2023-09-23T20:37:59.576 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 179.54333092300001, 2023-09-23T20:37:59.576 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1521833800000001, 2023-09-23T20:37:59.576 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:37:59.576 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:37:59.576 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:37:59.576 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:37:59.576 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:37:59.576 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 156899 2023-09-23T20:37:59.577 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.577 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:37:59.577 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.577 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:37:59.577 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.972856+0000", 2023-09-23T20:37:59.577 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:37:59.577 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.577 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.577 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:37:59.577 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.972856+0000", 2023-09-23T20:37:59.577 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:37:59.577 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.577 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.578 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:37:59.578 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.972854+0000", 2023-09-23T20:37:59.578 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:37:59.578 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.578 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.578 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:37:59.578 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.972863+0000", 2023-09-23T20:37:59.578 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.738e-06 2023-09-23T20:37:59.578 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.578 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.578 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:37:59.579 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.972876+0000", 2023-09-23T20:37:59.579 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3362e-05 2023-09-23T20:37:59.579 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.579 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.579 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:37:59.579 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.972888+0000", 2023-09-23T20:37:59.579 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1856e-05 2023-09-23T20:37:59.579 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.579 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.579 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:37:59.579 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.975112+0000", 2023-09-23T20:37:59.579 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0022238980000000002 2023-09-23T20:37:59.579 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.580 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.580 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:37:59.580 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.975128+0000", 2023-09-23T20:37:59.580 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.6325999999999999e-05 2023-09-23T20:37:59.580 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.580 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.580 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:37:59.580 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.115938+0000", 2023-09-23T20:37:59.580 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1408093079999999 2023-09-23T20:37:59.580 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.580 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.580 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:37:59.581 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.116085+0000", 2023-09-23T20:37:59.581 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00014754199999999999 2023-09-23T20:37:59.581 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.581 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.581 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 3,4,6", 2023-09-23T20:37:59.581 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.116220+0000", 2023-09-23T20:37:59.581 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000135308 2023-09-23T20:37:59.581 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.581 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.581 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:37:59.582 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.122330+0000", 2023-09-23T20:37:59.582 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.006109377 2023-09-23T20:37:59.582 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.582 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.582 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:37:59.582 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.124884+0000", 2023-09-23T20:37:59.582 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0025545559999999999 2023-09-23T20:37:59.582 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.582 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.582 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:37:59.582 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.124904+0000", 2023-09-23T20:37:59.583 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.9967999999999999e-05 2023-09-23T20:37:59.583 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.583 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.583 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:37:59.583 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.124921+0000", 2023-09-23T20:37:59.583 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.7166000000000001e-05 2023-09-23T20:37:59.583 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.583 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.583 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:37:59.583 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.124957+0000", 2023-09-23T20:37:59.583 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.5417000000000003e-05 2023-09-23T20:37:59.583 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.584 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.584 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:37:59.584 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.125039+0000", 2023-09-23T20:37:59.584 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.2295999999999999e-05 2023-09-23T20:37:59.584 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:37:59.584 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:37:59.584 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:37:59.584 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.584 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.584 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:156906 7.1 7:8761197e:::benchmark_data_smithi018_308601_object62354:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e354)", 2023-09-23T20:37:59.584 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:34:59.972906+0000", 2023-09-23T20:37:59.584 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 179.54328041599999, 2023-09-23T20:37:59.585 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.204579289, 2023-09-23T20:37:59.585 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:37:59.585 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:37:59.585 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:37:59.585 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:37:59.585 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:37:59.585 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 156906 2023-09-23T20:37:59.585 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.585 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:37:59.585 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.585 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:37:59.585 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.972906+0000", 2023-09-23T20:37:59.585 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:37:59.586 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.586 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.586 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:37:59.586 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.972906+0000", 2023-09-23T20:37:59.586 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:37:59.586 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.586 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.586 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:37:59.586 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.972906+0000", 2023-09-23T20:37:59.586 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:37:59.586 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.586 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.587 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:37:59.587 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.972909+0000", 2023-09-23T20:37:59.587 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.6569999999999999e-06 2023-09-23T20:37:59.587 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.587 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.587 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:37:59.587 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.972917+0000", 2023-09-23T20:37:59.587 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.6389999999999992e-06 2023-09-23T20:37:59.587 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.587 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.587 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:37:59.587 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.972923+0000", 2023-09-23T20:37:59.587 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.6810000000000003e-06 2023-09-23T20:37:59.588 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.588 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.588 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:37:59.588 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.975155+0000", 2023-09-23T20:37:59.588 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.002232445 2023-09-23T20:37:59.588 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.588 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.588 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:37:59.588 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.975164+0000", 2023-09-23T20:37:59.588 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.6309999999999992e-06 2023-09-23T20:37:59.588 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.588 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.588 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:37:59.589 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.146302+0000", 2023-09-23T20:37:59.589 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.171138427 2023-09-23T20:37:59.589 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.589 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.589 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:37:59.589 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.146426+0000", 2023-09-23T20:37:59.589 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00012366800000000001 2023-09-23T20:37:59.589 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.589 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.589 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 3,4,6", 2023-09-23T20:37:59.589 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.146558+0000", 2023-09-23T20:37:59.589 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00013182499999999999 2023-09-23T20:37:59.589 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.590 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.590 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:37:59.590 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.150022+0000", 2023-09-23T20:37:59.590 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0034646070000000002 2023-09-23T20:37:59.590 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.590 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.590 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:37:59.590 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.150058+0000", 2023-09-23T20:37:59.590 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.5277000000000001e-05 2023-09-23T20:37:59.590 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.590 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.590 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:37:59.591 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.166990+0000", 2023-09-23T20:37:59.591 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.016932365000000001 2023-09-23T20:37:59.591 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.591 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.591 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:37:59.591 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.177360+0000", 2023-09-23T20:37:59.591 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.010369682 2023-09-23T20:37:59.591 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.591 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.591 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:37:59.591 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.177412+0000", 2023-09-23T20:37:59.592 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.2039000000000001e-05 2023-09-23T20:37:59.592 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.592 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.592 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:37:59.592 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.177486+0000", 2023-09-23T20:37:59.592 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.3830000000000003e-05 2023-09-23T20:37:59.592 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:37:59.592 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:37:59.592 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:37:59.592 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.592 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.592 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:156911 7.1 7:8bb3fc92:::benchmark_data_smithi018_308601_object62359:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e354)", 2023-09-23T20:37:59.593 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:34:59.972928+0000", 2023-09-23T20:37:59.593 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 179.543258597, 2023-09-23T20:37:59.593 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.204718468, 2023-09-23T20:37:59.593 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:37:59.593 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:37:59.593 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:37:59.593 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:37:59.593 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:37:59.593 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 156911 2023-09-23T20:37:59.593 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.593 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:37:59.593 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.594 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:37:59.594 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.972928+0000", 2023-09-23T20:37:59.594 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:37:59.594 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.594 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.594 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:37:59.594 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.972928+0000", 2023-09-23T20:37:59.594 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:37:59.594 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.594 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.594 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:37:59.594 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.972928+0000", 2023-09-23T20:37:59.595 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:37:59.595 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.595 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.595 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:37:59.595 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.972930+0000", 2023-09-23T20:37:59.595 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.5359999999999999e-06 2023-09-23T20:37:59.595 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.595 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.595 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:37:59.595 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.972936+0000", 2023-09-23T20:37:59.595 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.1190000000000002e-06 2023-09-23T20:37:59.595 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.595 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.596 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:37:59.596 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.972942+0000", 2023-09-23T20:37:59.596 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.5369999999999997e-06 2023-09-23T20:37:59.596 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.596 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.596 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:37:59.596 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.975188+0000", 2023-09-23T20:37:59.596 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.002246521 2023-09-23T20:37:59.596 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.596 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.596 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:37:59.596 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.975196+0000", 2023-09-23T20:37:59.596 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.0900000000000005e-06 2023-09-23T20:37:59.597 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.597 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.597 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:37:59.597 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.150087+0000", 2023-09-23T20:37:59.597 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.174890802 2023-09-23T20:37:59.597 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.597 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.597 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:37:59.597 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.150236+0000", 2023-09-23T20:37:59.597 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00014853699999999999 2023-09-23T20:37:59.597 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.597 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.597 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 3,4,6", 2023-09-23T20:37:59.598 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.150378+0000", 2023-09-23T20:37:59.598 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00014263000000000001 2023-09-23T20:37:59.598 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.598 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.598 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:37:59.598 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.157143+0000", 2023-09-23T20:37:59.598 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0067646909999999998 2023-09-23T20:37:59.598 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.598 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.598 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:37:59.598 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.157179+0000", 2023-09-23T20:37:59.599 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.6099000000000001e-05 2023-09-23T20:37:59.599 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.599 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.599 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:37:59.599 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.167022+0000", 2023-09-23T20:37:59.599 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0098431739999999997 2023-09-23T20:37:59.599 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.599 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.599 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:37:59.599 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.177564+0000", 2023-09-23T20:37:59.599 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.010541867999999999 2023-09-23T20:37:59.599 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.599 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.599 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:37:59.600 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.177590+0000", 2023-09-23T20:37:59.600 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.62e-05 2023-09-23T20:37:59.600 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.600 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.600 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:37:59.600 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.177647+0000", 2023-09-23T20:37:59.600 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.6063999999999997e-05 2023-09-23T20:37:59.600 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:37:59.600 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:37:59.600 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:37:59.600 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.600 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.601 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:156944 7.1 7:848d83ce:::benchmark_data_smithi018_308601_object62392:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e354)", 2023-09-23T20:37:59.601 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:34:59.972947+0000", 2023-09-23T20:37:59.601 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 179.54323990699999, 2023-09-23T20:37:59.601 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.207883992, 2023-09-23T20:37:59.601 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:37:59.601 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:37:59.601 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:37:59.601 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:37:59.601 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:37:59.601 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 156944 2023-09-23T20:37:59.601 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.601 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:37:59.601 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.602 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:37:59.602 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.972947+0000", 2023-09-23T20:37:59.602 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:37:59.602 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.602 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.602 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:37:59.602 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.972947+0000", 2023-09-23T20:37:59.602 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:37:59.602 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.602 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.603 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:37:59.603 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.972946+0000", 2023-09-23T20:37:59.603 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:37:59.603 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.603 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.603 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:37:59.603 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.972949+0000", 2023-09-23T20:37:59.603 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.503e-06 2023-09-23T20:37:59.603 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.603 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.603 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:37:59.603 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.972956+0000", 2023-09-23T20:37:59.603 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.9870000000000002e-06 2023-09-23T20:37:59.604 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.604 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.604 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:37:59.604 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.972962+0000", 2023-09-23T20:37:59.604 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.7339999999999996e-06 2023-09-23T20:37:59.604 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.604 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.604 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:37:59.604 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.975220+0000", 2023-09-23T20:37:59.604 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0022582489999999999 2023-09-23T20:37:59.604 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.604 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.605 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:37:59.605 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.975229+0000", 2023-09-23T20:37:59.605 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.3540000000000006e-06 2023-09-23T20:37:59.605 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.605 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.605 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:37:59.605 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.153614+0000", 2023-09-23T20:37:59.605 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1783843549999999 2023-09-23T20:37:59.605 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.605 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.605 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:37:59.605 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.153771+0000", 2023-09-23T20:37:59.606 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000156935 2023-09-23T20:37:59.606 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.606 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.606 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 3,4,6", 2023-09-23T20:37:59.606 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.153909+0000", 2023-09-23T20:37:59.606 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00013803099999999999 2023-09-23T20:37:59.606 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.606 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.606 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:37:59.606 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.166810+0000", 2023-09-23T20:37:59.606 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.012901347000000001 2023-09-23T20:37:59.606 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.606 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.607 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:37:59.607 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.166863+0000", 2023-09-23T20:37:59.607 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.3046e-05 2023-09-23T20:37:59.607 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.607 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.607 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:37:59.607 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.177657+0000", 2023-09-23T20:37:59.607 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.010794419 2023-09-23T20:37:59.607 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.607 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.607 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:37:59.607 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.180631+0000", 2023-09-23T20:37:59.607 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0029738070000000002 2023-09-23T20:37:59.608 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.608 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.608 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:37:59.608 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.180693+0000", 2023-09-23T20:37:59.608 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.1752000000000001e-05 2023-09-23T20:37:59.608 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.608 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.608 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:37:59.608 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.180831+0000", 2023-09-23T20:37:59.608 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00013783600000000001 2023-09-23T20:37:59.608 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:37:59.608 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:37:59.608 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:37:59.609 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.609 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.609 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:156954 7.1 7:8fbb61c7:::benchmark_data_smithi018_308601_object62402:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e354)", 2023-09-23T20:37:59.609 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:34:59.972967+0000", 2023-09-23T20:37:59.609 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 179.54322006999999, 2023-09-23T20:37:59.609 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.208111494, 2023-09-23T20:37:59.609 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:37:59.609 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:37:59.609 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:37:59.609 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:37:59.609 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:37:59.609 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 156954 2023-09-23T20:37:59.609 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.610 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:37:59.610 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.610 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:37:59.610 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.972967+0000", 2023-09-23T20:37:59.610 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:37:59.610 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.610 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.610 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:37:59.610 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.972967+0000", 2023-09-23T20:37:59.610 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:37:59.610 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.610 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.611 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:37:59.611 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.972966+0000", 2023-09-23T20:37:59.611 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:37:59.611 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.611 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.611 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:37:59.611 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.972969+0000", 2023-09-23T20:37:59.611 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.548e-06 2023-09-23T20:37:59.611 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.611 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.611 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:37:59.611 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.972974+0000", 2023-09-23T20:37:59.611 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.6389999999999997e-06 2023-09-23T20:37:59.611 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.612 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.612 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:37:59.612 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.972979+0000", 2023-09-23T20:37:59.612 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.2639999999999999e-06 2023-09-23T20:37:59.612 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.612 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.612 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:37:59.612 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.975256+0000", 2023-09-23T20:37:59.612 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0022768879999999999 2023-09-23T20:37:59.612 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.613 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.613 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:37:59.613 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.975264+0000", 2023-09-23T20:37:59.613 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.0260000000000007e-06 2023-09-23T20:37:59.613 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.613 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.613 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:37:59.613 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.157204+0000", 2023-09-23T20:37:59.613 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.181939297 2023-09-23T20:37:59.613 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.613 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.613 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:37:59.613 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.157336+0000", 2023-09-23T20:37:59.614 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00013253599999999999 2023-09-23T20:37:59.614 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.614 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.614 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 3,4,6", 2023-09-23T20:37:59.614 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.157473+0000", 2023-09-23T20:37:59.614 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000137219 2023-09-23T20:37:59.614 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.614 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.614 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:37:59.614 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.166957+0000", 2023-09-23T20:37:59.614 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0094833369999999997 2023-09-23T20:37:59.614 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.615 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.615 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:37:59.615 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.174176+0000", 2023-09-23T20:37:59.615 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.007219306 2023-09-23T20:37:59.615 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.615 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.615 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:37:59.615 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.177664+0000", 2023-09-23T20:37:59.615 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0034882519999999998 2023-09-23T20:37:59.615 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.615 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.615 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:37:59.615 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.180887+0000", 2023-09-23T20:37:59.615 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0032224430000000002 2023-09-23T20:37:59.616 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.616 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.616 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:37:59.616 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.180953+0000", 2023-09-23T20:37:59.616 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.5775e-05 2023-09-23T20:37:59.616 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.616 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.616 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:37:59.616 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.181078+0000", 2023-09-23T20:37:59.616 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00012554600000000001 2023-09-23T20:37:59.616 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:37:59.616 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:37:59.617 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:37:59.617 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.617 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.617 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35994.0:84076 8.a 8:51aef8e7:::benchmark_data_smithi018_341490_object84075:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e378)", 2023-09-23T20:37:59.617 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:37:13.334851+0000", 2023-09-23T20:37:59.617 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 46.181335265999998, 2023-09-23T20:37:59.617 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.367426676, 2023-09-23T20:37:59.617 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:37:59.617 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:37:59.617 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:37:59.617 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35994", 2023-09-23T20:37:59.617 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:37:59.617 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 84076 2023-09-23T20:37:59.618 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.618 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:37:59.618 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.618 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:37:59.618 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.334851+0000", 2023-09-23T20:37:59.618 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:37:59.618 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.618 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.618 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:37:59.618 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.334851+0000", 2023-09-23T20:37:59.618 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:37:59.618 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.618 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.619 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:37:59.619 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.334851+0000", 2023-09-23T20:37:59.619 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:37:59.619 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.619 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.619 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:37:59.619 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.334872+0000", 2023-09-23T20:37:59.619 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.1464999999999999e-05 2023-09-23T20:37:59.619 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.619 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.619 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:37:59.619 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.334880+0000", 2023-09-23T20:37:59.619 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.7489999999999998e-06 2023-09-23T20:37:59.619 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.620 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.620 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:37:59.620 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.334886+0000", 2023-09-23T20:37:59.620 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.626e-06 2023-09-23T20:37:59.620 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.620 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.620 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:37:59.620 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.338890+0000", 2023-09-23T20:37:59.620 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0040048039999999998 2023-09-23T20:37:59.620 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.620 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.620 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:37:59.621 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.339048+0000", 2023-09-23T20:37:59.621 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000157548 2023-09-23T20:37:59.621 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.621 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.621 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 2,5", 2023-09-23T20:37:59.621 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.339203+0000", 2023-09-23T20:37:59.621 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00015535000000000001 2023-09-23T20:37:59.621 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.621 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.621 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:37:59.621 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.350871+0000", 2023-09-23T20:37:59.621 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0116682 2023-09-23T20:37:59.621 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.622 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.622 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:37:59.622 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.354938+0000", 2023-09-23T20:37:59.622 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0040660749999999997 2023-09-23T20:37:59.622 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.622 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.622 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:37:59.622 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.361832+0000", 2023-09-23T20:37:59.622 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0068944369999999998 2023-09-23T20:37:59.623 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.623 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.623 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:37:59.623 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.361845+0000", 2023-09-23T20:37:59.623 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2889e-05 2023-09-23T20:37:59.623 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.623 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.623 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:37:59.623 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:14.353806+0000", 2023-09-23T20:37:59.623 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.99196156400000002 2023-09-23T20:37:59.623 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.624 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.624 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:37:59.624 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:14.353825+0000", 2023-09-23T20:37:59.624 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.8437999999999999e-05 2023-09-23T20:37:59.624 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.624 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.624 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:37:59.624 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:15.688941+0000", 2023-09-23T20:37:59.624 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.335116411 2023-09-23T20:37:59.624 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.624 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.624 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for degraded object", 2023-09-23T20:37:59.625 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:15.689523+0000", 2023-09-23T20:37:59.625 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00058162999999999997 2023-09-23T20:37:59.625 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.625 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.625 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:37:59.625 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:15.702170+0000", 2023-09-23T20:37:59.625 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.012646734999999999 2023-09-23T20:37:59.625 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.625 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.625 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:37:59.625 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:15.702278+0000", 2023-09-23T20:37:59.625 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00010842 2023-09-23T20:37:59.625 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:37:59.626 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:37:59.626 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:37:59.626 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.626 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.626 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35994.0:84079 8.a 8:53d5899b:::benchmark_data_smithi018_341490_object84078:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e378)", 2023-09-23T20:37:59.626 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:37:13.344056+0000", 2023-09-23T20:37:59.626 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 46.172130940999999, 2023-09-23T20:37:59.626 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.3675740109999999, 2023-09-23T20:37:59.626 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:37:59.626 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:37:59.626 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:37:59.626 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35994", 2023-09-23T20:37:59.626 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:37:59.627 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 84079 2023-09-23T20:37:59.627 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.627 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:37:59.627 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.627 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:37:59.627 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.344056+0000", 2023-09-23T20:37:59.627 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:37:59.627 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.627 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.627 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:37:59.627 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.344056+0000", 2023-09-23T20:37:59.627 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:37:59.628 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.628 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.628 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:37:59.628 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.344053+0000", 2023-09-23T20:37:59.628 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999976 2023-09-23T20:37:59.628 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.628 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.628 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:37:59.628 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.344081+0000", 2023-09-23T20:37:59.628 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.7042999999999998e-05 2023-09-23T20:37:59.628 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.628 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.628 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:37:59.629 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.344099+0000", 2023-09-23T20:37:59.629 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.8505e-05 2023-09-23T20:37:59.629 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.629 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.629 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:37:59.629 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.344109+0000", 2023-09-23T20:37:59.629 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.6350000000000001e-06 2023-09-23T20:37:59.629 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.629 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.629 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:37:59.629 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.350901+0000", 2023-09-23T20:37:59.629 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0067922520000000004 2023-09-23T20:37:59.629 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.630 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.630 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:37:59.630 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.351025+0000", 2023-09-23T20:37:59.630 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00012372899999999999 2023-09-23T20:37:59.630 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.630 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.630 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 2,5", 2023-09-23T20:37:59.630 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.351144+0000", 2023-09-23T20:37:59.630 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00011948 2023-09-23T20:37:59.630 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.630 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.630 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:37:59.631 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.354963+0000", 2023-09-23T20:37:59.631 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0038190049999999999 2023-09-23T20:37:59.631 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.631 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.631 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:37:59.631 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.361878+0000", 2023-09-23T20:37:59.631 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0069147660000000001 2023-09-23T20:37:59.631 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.631 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.631 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:37:59.631 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.361889+0000", 2023-09-23T20:37:59.631 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0721e-05 2023-09-23T20:37:59.631 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.632 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.632 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:37:59.632 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:14.353869+0000", 2023-09-23T20:37:59.632 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.991980046 2023-09-23T20:37:59.632 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.632 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.632 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:37:59.632 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:14.353879+0000", 2023-09-23T20:37:59.632 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0455000000000001e-05 2023-09-23T20:37:59.632 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.632 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.632 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:37:59.633 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:15.692953+0000", 2023-09-23T20:37:59.633 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3390743249999999 2023-09-23T20:37:59.633 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.633 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.633 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for degraded object", 2023-09-23T20:37:59.633 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:15.693411+0000", 2023-09-23T20:37:59.633 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00045711999999999999 2023-09-23T20:37:59.633 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.633 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.633 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:37:59.634 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:15.711539+0000", 2023-09-23T20:37:59.634 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.018128634000000001 2023-09-23T20:37:59.634 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.634 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.634 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:37:59.634 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:15.711630+0000", 2023-09-23T20:37:59.634 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.0568000000000004e-05 2023-09-23T20:37:59.634 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:37:59.634 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:37:59.634 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:37:59.634 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.634 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.635 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35994.0:84089 8.a 8:5f99afa4:::benchmark_data_smithi018_341490_object84088:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e379)", 2023-09-23T20:37:59.635 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:37:13.363463+0000", 2023-09-23T20:37:59.635 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 46.152723217999998, 2023-09-23T20:37:59.635 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.3832335360000001, 2023-09-23T20:37:59.635 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:37:59.635 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:37:59.635 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:37:59.635 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35994", 2023-09-23T20:37:59.635 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:37:59.635 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 84089 2023-09-23T20:37:59.635 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.635 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:37:59.635 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.636 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:37:59.636 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.363463+0000", 2023-09-23T20:37:59.636 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:37:59.636 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.636 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.636 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:37:59.636 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.363463+0000", 2023-09-23T20:37:59.636 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:37:59.636 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.636 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.636 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:37:59.636 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.363462+0000", 2023-09-23T20:37:59.637 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:37:59.637 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.637 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.637 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:37:59.637 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.363487+0000", 2023-09-23T20:37:59.637 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.5588999999999998e-05 2023-09-23T20:37:59.637 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.637 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.637 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:37:59.637 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.363504+0000", 2023-09-23T20:37:59.637 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.7118000000000001e-05 2023-09-23T20:37:59.637 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.637 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.637 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:37:59.638 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.363513+0000", 2023-09-23T20:37:59.638 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.337e-06 2023-09-23T20:37:59.638 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.638 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.638 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:37:59.638 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.363563+0000", 2023-09-23T20:37:59.638 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.0002e-05 2023-09-23T20:37:59.638 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.638 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.638 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:37:59.638 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.363572+0000", 2023-09-23T20:37:59.638 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.9269999999999994e-06 2023-09-23T20:37:59.639 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.639 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.639 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:37:59.639 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:14.353905+0000", 2023-09-23T20:37:59.639 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.99033294699999996 2023-09-23T20:37:59.639 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.639 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.639 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:37:59.639 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:14.353913+0000", 2023-09-23T20:37:59.639 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.4440000000000008e-06 2023-09-23T20:37:59.639 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.639 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.639 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:37:59.639 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:15.711657+0000", 2023-09-23T20:37:59.640 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3577443149999999 2023-09-23T20:37:59.640 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.640 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.640 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:37:59.640 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:15.711807+0000", 2023-09-23T20:37:59.640 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00014948199999999999 2023-09-23T20:37:59.640 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.640 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.640 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 2,3", 2023-09-23T20:37:59.640 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:15.711983+0000", 2023-09-23T20:37:59.640 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00017621999999999999 2023-09-23T20:37:59.640 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.641 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.641 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:37:59.641 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:15.727869+0000", 2023-09-23T20:37:59.641 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.015886246 2023-09-23T20:37:59.641 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.641 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.641 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:37:59.641 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:15.727928+0000", 2023-09-23T20:37:59.641 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.8464000000000001e-05 2023-09-23T20:37:59.641 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.641 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.641 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:37:59.641 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:15.743251+0000", 2023-09-23T20:37:59.642 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.015322794000000001 2023-09-23T20:37:59.642 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.642 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.642 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:37:59.642 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:15.743305+0000", 2023-09-23T20:37:59.642 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.4292000000000001e-05 2023-09-23T20:37:59.642 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.642 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.642 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:37:59.642 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:15.746697+0000", 2023-09-23T20:37:59.642 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0033921620000000002 2023-09-23T20:37:59.642 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:37:59.642 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:37:59.643 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:37:59.643 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.643 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.643 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35994.0:84091 8.3 8:c9851101:::benchmark_data_smithi018_341490_object84090:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e379)", 2023-09-23T20:37:59.643 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:37:13.366065+0000", 2023-09-23T20:37:59.643 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 46.150121222000003, 2023-09-23T20:37:59.643 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.2838538150000001, 2023-09-23T20:37:59.643 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:37:59.643 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:37:59.643 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:37:59.644 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35994", 2023-09-23T20:37:59.644 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:37:59.644 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 84091 2023-09-23T20:37:59.644 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.644 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:37:59.644 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.644 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:37:59.644 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.366065+0000", 2023-09-23T20:37:59.644 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:37:59.644 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.644 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.644 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:37:59.645 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.366065+0000", 2023-09-23T20:37:59.645 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:37:59.645 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.645 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.645 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:37:59.645 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.366065+0000", 2023-09-23T20:37:59.645 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:37:59.645 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.645 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.645 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:37:59.645 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.366089+0000", 2023-09-23T20:37:59.645 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.3904e-05 2023-09-23T20:37:59.645 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.646 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.646 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:37:59.646 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.366098+0000", 2023-09-23T20:37:59.646 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.9949999999999994e-06 2023-09-23T20:37:59.646 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.646 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.646 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:37:59.646 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.366117+0000", 2023-09-23T20:37:59.646 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.8924e-05 2023-09-23T20:37:59.646 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.646 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.646 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:37:59.647 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.366220+0000", 2023-09-23T20:37:59.647 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000103059 2023-09-23T20:37:59.647 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.647 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.647 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:37:59.647 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.366232+0000", 2023-09-23T20:37:59.647 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.206e-05 2023-09-23T20:37:59.647 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.647 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.647 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:37:59.647 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:14.349352+0000", 2023-09-23T20:37:59.647 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.98312001199999999 2023-09-23T20:37:59.647 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.648 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.648 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:37:59.648 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:14.349365+0000", 2023-09-23T20:37:59.648 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2999e-05 2023-09-23T20:37:59.648 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.648 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.648 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:37:59.648 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:15.629266+0000", 2023-09-23T20:37:59.648 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2799014390000001 2023-09-23T20:37:59.648 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.648 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.648 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:37:59.648 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:15.629429+0000", 2023-09-23T20:37:59.648 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00016256800000000001 2023-09-23T20:37:59.649 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.649 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.649 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 2,6", 2023-09-23T20:37:59.649 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:15.629608+0000", 2023-09-23T20:37:59.649 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00017943599999999999 2023-09-23T20:37:59.649 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.649 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.649 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:37:59.649 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:15.646076+0000", 2023-09-23T20:37:59.649 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.016468046 2023-09-23T20:37:59.649 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.649 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.649 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:37:59.650 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:15.646127+0000", 2023-09-23T20:37:59.650 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.0583e-05 2023-09-23T20:37:59.650 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.650 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.650 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:37:59.650 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:15.646140+0000", 2023-09-23T20:37:59.650 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3519e-05 2023-09-23T20:37:59.650 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.650 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.650 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:37:59.650 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:15.646219+0000", 2023-09-23T20:37:59.650 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.8881000000000006e-05 2023-09-23T20:37:59.650 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.651 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.651 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:37:59.651 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:15.649919+0000", 2023-09-23T20:37:59.651 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0036999530000000002 2023-09-23T20:37:59.651 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:37:59.651 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:37:59.651 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:37:59.651 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.651 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.651 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35994.0:84092 8.a 8:5d9db4b0:::benchmark_data_smithi018_341490_object84091:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e379)", 2023-09-23T20:37:59.651 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:37:13.366132+0000", 2023-09-23T20:37:59.651 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 46.150054930000003, 2023-09-23T20:37:59.651 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.3842849990000001, 2023-09-23T20:37:59.652 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:37:59.652 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:37:59.652 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:37:59.652 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35994", 2023-09-23T20:37:59.652 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:37:59.652 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 84092 2023-09-23T20:37:59.652 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.652 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:37:59.652 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.652 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:37:59.652 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.366132+0000", 2023-09-23T20:37:59.652 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:37:59.652 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.652 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.653 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:37:59.653 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.366132+0000", 2023-09-23T20:37:59.653 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:37:59.653 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.653 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.653 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:37:59.653 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.366131+0000", 2023-09-23T20:37:59.653 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:37:59.653 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.653 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.653 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:37:59.653 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.366154+0000", 2023-09-23T20:37:59.654 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.3272e-05 2023-09-23T20:37:59.654 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.654 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.654 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:37:59.654 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.366161+0000", 2023-09-23T20:37:59.654 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.046e-06 2023-09-23T20:37:59.654 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.654 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.654 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:37:59.654 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.366168+0000", 2023-09-23T20:37:59.655 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.3620000000000002e-06 2023-09-23T20:37:59.655 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.655 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.655 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:37:59.655 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.366275+0000", 2023-09-23T20:37:59.655 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000107376 2023-09-23T20:37:59.655 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.655 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.655 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:37:59.655 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.366286+0000", 2023-09-23T20:37:59.655 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1239e-05 2023-09-23T20:37:59.655 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.655 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.655 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:37:59.656 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:14.353936+0000", 2023-09-23T20:37:59.656 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.98764978299999995 2023-09-23T20:37:59.656 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.656 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.656 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:37:59.656 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:14.353944+0000", 2023-09-23T20:37:59.656 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.1340000000000004e-06 2023-09-23T20:37:59.656 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.656 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.656 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:37:59.656 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:15.714442+0000", 2023-09-23T20:37:59.656 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3604973929999999 2023-09-23T20:37:59.656 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.657 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.657 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:37:59.657 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:15.714618+0000", 2023-09-23T20:37:59.657 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00017675500000000001 2023-09-23T20:37:59.657 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.657 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.657 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 2,3", 2023-09-23T20:37:59.657 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:15.714894+0000", 2023-09-23T20:37:59.657 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00027531799999999998 2023-09-23T20:37:59.657 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.657 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.657 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:37:59.657 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:15.727877+0000", 2023-09-23T20:37:59.658 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.012983112999999999 2023-09-23T20:37:59.658 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.658 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.658 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:37:59.658 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:15.727983+0000", 2023-09-23T20:37:59.658 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000106573 2023-09-23T20:37:59.658 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.658 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.658 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:37:59.658 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:15.746758+0000", 2023-09-23T20:37:59.658 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.018774036000000001 2023-09-23T20:37:59.658 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.658 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.659 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:37:59.659 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:15.746834+0000", 2023-09-23T20:37:59.659 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.6039999999999997e-05 2023-09-23T20:37:59.659 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.659 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.659 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:37:59.659 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:15.750417+0000", 2023-09-23T20:37:59.659 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0035831880000000002 2023-09-23T20:37:59.659 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:37:59.659 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:37:59.659 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:37:59.659 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.659 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.660 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35994.0:84104 8.a 8:510db2a2:::benchmark_data_smithi018_341490_object84103:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e379)", 2023-09-23T20:37:59.660 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:37:13.384561+0000", 2023-09-23T20:37:59.660 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 46.131625192000001, 2023-09-23T20:37:59.660 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.368910107, 2023-09-23T20:37:59.660 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:37:59.660 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:37:59.660 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:37:59.660 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35994", 2023-09-23T20:37:59.660 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:37:59.660 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 84104 2023-09-23T20:37:59.660 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.660 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:37:59.660 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.661 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:37:59.661 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.384561+0000", 2023-09-23T20:37:59.661 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:37:59.662 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:37:59.662 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.662 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.662 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:37:59.662 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.384561+0000", 2023-09-23T20:37:59.662 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:37:59.662 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.662 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.663 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:37:59.663 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.384559+0000", 2023-09-23T20:37:59.663 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999976 2023-09-23T20:37:59.663 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.663 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.663 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:37:59.663 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.384588+0000", 2023-09-23T20:37:59.663 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.9560000000000002e-05 2023-09-23T20:37:59.663 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.663 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.663 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:37:59.663 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.384600+0000", 2023-09-23T20:37:59.664 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1868999999999999e-05 2023-09-23T20:37:59.664 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.664 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.664 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:37:59.664 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.384608+0000", 2023-09-23T20:37:59.664 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.0290000000000005e-06 2023-09-23T20:37:59.664 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.664 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.664 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:37:59.664 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.384667+0000", 2023-09-23T20:37:59.665 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.8554999999999998e-05 2023-09-23T20:37:59.665 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.665 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.665 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:37:59.665 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.384679+0000", 2023-09-23T20:37:59.665 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2193e-05 2023-09-23T20:37:59.665 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.665 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.665 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:37:59.665 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:14.353967+0000", 2023-09-23T20:37:59.665 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.96928828600000005 2023-09-23T20:37:59.665 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.666 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.666 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:37:59.666 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:14.353976+0000", 2023-09-23T20:37:59.666 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.4509999999999992e-06 2023-09-23T20:37:59.666 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.666 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.666 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:37:59.666 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:15.718028+0000", 2023-09-23T20:37:59.666 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3640520330000001 2023-09-23T20:37:59.666 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.666 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.666 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:37:59.666 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:15.718486+0000", 2023-09-23T20:37:59.667 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00045779600000000001 2023-09-23T20:37:59.667 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.667 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.667 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 2,3", 2023-09-23T20:37:59.667 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:15.718646+0000", 2023-09-23T20:37:59.667 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000160545 2023-09-23T20:37:59.667 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.667 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.667 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:37:59.667 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:15.727882+0000", 2023-09-23T20:37:59.667 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0092361939999999997 2023-09-23T20:37:59.667 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.667 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.667 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:37:59.668 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:15.730716+0000", 2023-09-23T20:37:59.668 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0028335600000000002 2023-09-23T20:37:59.668 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.668 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.668 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:37:59.668 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:15.750486+0000", 2023-09-23T20:37:59.668 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.019770167000000002 2023-09-23T20:37:59.668 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.668 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.668 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:37:59.668 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:15.750549+0000", 2023-09-23T20:37:59.668 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.2545999999999994e-05 2023-09-23T20:37:59.668 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.669 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.669 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:37:59.669 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:15.753472+0000", 2023-09-23T20:37:59.669 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0029228790000000002 2023-09-23T20:37:59.669 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:37:59.669 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:37:59.669 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:37:59.669 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.669 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.669 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35994.0:84111 8.0 8:0d10ef15:::benchmark_data_smithi018_341490_object84110:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e379)", 2023-09-23T20:37:59.669 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:37:13.393609+0000", 2023-09-23T20:37:59.669 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 46.122578083999997, 2023-09-23T20:37:59.669 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.2585651910000002, 2023-09-23T20:37:59.670 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:37:59.670 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:37:59.670 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:37:59.670 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35994", 2023-09-23T20:37:59.670 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:37:59.670 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 84111 2023-09-23T20:37:59.670 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.670 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:37:59.670 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.670 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:37:59.670 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.393609+0000", 2023-09-23T20:37:59.670 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:37:59.670 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.671 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.671 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:37:59.671 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.393609+0000", 2023-09-23T20:37:59.671 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:37:59.671 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.671 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.671 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:37:59.671 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.393608+0000", 2023-09-23T20:37:59.671 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:37:59.671 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.671 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.671 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:37:59.671 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.393626+0000", 2023-09-23T20:37:59.672 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.7727000000000002e-05 2023-09-23T20:37:59.672 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.672 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.672 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:37:59.672 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.393631+0000", 2023-09-23T20:37:59.672 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.9339999999999997e-06 2023-09-23T20:37:59.672 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.672 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.672 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:37:59.672 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.393635+0000", 2023-09-23T20:37:59.672 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.924e-06 2023-09-23T20:37:59.672 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.672 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.673 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:37:59.673 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.393697+0000", 2023-09-23T20:37:59.673 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.1972999999999998e-05 2023-09-23T20:37:59.673 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.673 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.673 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:37:59.673 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.393711+0000", 2023-09-23T20:37:59.673 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.4250000000000001e-05 2023-09-23T20:37:59.673 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.673 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.673 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:37:59.673 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:14.347541+0000", 2023-09-23T20:37:59.673 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.95382987600000002 2023-09-23T20:37:59.673 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.674 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.674 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:37:59.674 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:14.347552+0000", 2023-09-23T20:37:59.674 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1e-05 2023-09-23T20:37:59.674 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.674 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.674 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:37:59.674 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:15.603057+0000", 2023-09-23T20:37:59.674 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2555049060000001 2023-09-23T20:37:59.675 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.675 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.675 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:37:59.675 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:15.605409+0000", 2023-09-23T20:37:59.675 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0023523680000000001 2023-09-23T20:37:59.675 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.675 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.675 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:37:59.675 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:15.605629+0000", 2023-09-23T20:37:59.675 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00021984899999999999 2023-09-23T20:37:59.675 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.675 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.676 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:37:59.676 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:15.615897+0000", 2023-09-23T20:37:59.676 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.010267716 2023-09-23T20:37:59.676 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.676 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.676 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:37:59.676 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:15.631851+0000", 2023-09-23T20:37:59.676 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.015954349 2023-09-23T20:37:59.676 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.676 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.676 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:37:59.676 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:15.648581+0000", 2023-09-23T20:37:59.676 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.016730098999999998 2023-09-23T20:37:59.677 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.677 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.677 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:37:59.677 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:15.648651+0000", 2023-09-23T20:37:59.677 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.0188e-05 2023-09-23T20:37:59.677 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.677 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.677 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:37:59.677 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:15.652174+0000", 2023-09-23T20:37:59.677 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0035223810000000002 2023-09-23T20:37:59.677 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:37:59.677 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:37:59.677 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:37:59.678 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.678 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.678 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35994.0:84113 8.3 8:ca914450:::benchmark_data_smithi018_341490_object84112:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e379)", 2023-09-23T20:37:59.678 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:37:13.401619+0000", 2023-09-23T20:37:59.678 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 46.114567373, 2023-09-23T20:37:59.678 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.2600310069999998, 2023-09-23T20:37:59.678 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:37:59.678 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:37:59.678 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:37:59.678 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35994", 2023-09-23T20:37:59.678 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:37:59.678 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 84113 2023-09-23T20:37:59.679 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.679 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:37:59.679 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.679 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:37:59.679 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.401619+0000", 2023-09-23T20:37:59.679 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:37:59.679 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.679 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.679 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:37:59.679 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.401619+0000", 2023-09-23T20:37:59.679 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:37:59.679 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.679 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.680 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:37:59.680 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.401618+0000", 2023-09-23T20:37:59.680 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:37:59.680 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.680 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.680 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:37:59.680 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.401642+0000", 2023-09-23T20:37:59.680 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.3949000000000001e-05 2023-09-23T20:37:59.680 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.680 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.680 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:37:59.680 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.401654+0000", 2023-09-23T20:37:59.680 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2651e-05 2023-09-23T20:37:59.681 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.681 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.681 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:37:59.681 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.401661+0000", 2023-09-23T20:37:59.681 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.2389999999999999e-06 2023-09-23T20:37:59.681 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.681 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.681 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:37:59.681 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.401714+0000", 2023-09-23T20:37:59.681 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.3087000000000002e-05 2023-09-23T20:37:59.681 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.681 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.682 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:37:59.682 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.401721+0000", 2023-09-23T20:37:59.682 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.8339999999999999e-06 2023-09-23T20:37:59.682 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.682 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.682 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:37:59.682 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:14.349394+0000", 2023-09-23T20:37:59.682 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.94767251600000002 2023-09-23T20:37:59.682 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.682 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.682 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:37:59.682 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:14.349402+0000", 2023-09-23T20:37:59.683 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.3110000000000006e-06 2023-09-23T20:37:59.683 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.683 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.683 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:37:59.683 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:15.649982+0000", 2023-09-23T20:37:59.683 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3005794319999999 2023-09-23T20:37:59.683 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.683 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.683 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:37:59.683 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:15.650187+0000", 2023-09-23T20:37:59.683 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000205181 2023-09-23T20:37:59.683 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.684 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.684 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 2,6", 2023-09-23T20:37:59.684 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:15.650404+0000", 2023-09-23T20:37:59.684 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000217035 2023-09-23T20:37:59.684 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.684 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.684 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:37:59.684 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:15.653830+0000", 2023-09-23T20:37:59.684 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0034262009999999998 2023-09-23T20:37:59.684 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.684 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.684 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:37:59.684 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:15.658265+0000", 2023-09-23T20:37:59.685 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0044345979999999997 2023-09-23T20:37:59.685 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.685 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.685 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:37:59.685 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:15.658288+0000", 2023-09-23T20:37:59.685 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.3549000000000001e-05 2023-09-23T20:37:59.685 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.685 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.685 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:37:59.685 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:15.658336+0000", 2023-09-23T20:37:59.686 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.7372e-05 2023-09-23T20:37:59.686 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:37:59.686 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:37:59.686 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:37:59.686 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:15.661650+0000", 2023-09-23T20:37:59.686 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0033146180000000001 2023-09-23T20:37:59.686 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:37:59.686 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:37:59.686 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:37:59.686 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:37:59.686 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:37:59.686 INFO:teuthology.orchestra.run.smithi018.stdout:} 2023-09-23T20:37:59.687 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:59.723 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:37:59.726 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:59.727 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:37:59.827 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:37:59.855 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:37:59.897 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:37:59.909 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:37:59.910 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:38:00.010 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:38:00.057 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T20:38:00.129 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:00.371 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:38:00.399 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:00.400 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:38:00.501 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:38:00.531 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:00.568 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:38:00.571 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:00.572 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:38:00.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:00 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:38:00.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:00 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:38:00.645 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:00 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:38:00.645 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:00 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:38:00.673 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:38:00.700 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:00.736 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:38:00.739 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:00.740 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:38:00.759 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:38:00.813 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:38:00.817 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T20:38:00.840 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:38:00.883 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:38:00.943 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:38:00.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:00 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:38:00.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:00 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:38:00.984 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:38:00.993 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:01.059 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:38:01.082 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:38:01.086 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:01.087 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:38:01.119 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:38:01.163 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:38:01.180 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:38:01.188 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:38:01.215 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:01.224 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:38:01.224 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:38:01.224 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:38:01.224 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:38:01.224 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:38:01.224 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:38:01.225 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:38:01.225 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:38:01.252 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:38:01.255 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:01.256 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:38:01.256 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:00 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mon-a[179623]: 2023-09-23T20:38:00.965+0000 7f160fef9700 -1 mon.a@0(leader).osd e402 definitely_dead 0 2023-09-23T20:38:01.357 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:38:01.385 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:01.423 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:38:01.428 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:01.429 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:38:01.507 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:01 smithi018 ceph-mon[184144]: pgmap v975: 119 pgs: 2 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 1 active+recovery_wait, 10 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 5.2 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 320 KiB/s wr, 372 op/s; 50138/255852 objects degraded (19.596%); 13643/255852 objects misplaced (5.332%); 7.1 MiB/s, 114 objects/s recovering 2023-09-23T20:38:01.507 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:01 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/243092763' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:38:01.507 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:01 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1886871503' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-09-23T20:38:01.507 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:01 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-09-23T20:38:01.507 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:01 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 50138/255852 objects degraded (19.596%), 12 pgs degraded (PG_DEGRADED) 2023-09-23T20:38:01.508 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:01 smithi018 ceph-mon[179647]: pgmap v975: 119 pgs: 2 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 1 active+recovery_wait, 10 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 5.2 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 320 KiB/s wr, 372 op/s; 50138/255852 objects degraded (19.596%); 13643/255852 objects misplaced (5.332%); 7.1 MiB/s, 114 objects/s recovering 2023-09-23T20:38:01.508 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:01 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/243092763' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:38:01.508 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:01 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1886871503' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-09-23T20:38:01.508 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:01 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-09-23T20:38:01.508 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:01 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 50138/255852 objects degraded (19.596%), 12 pgs degraded (PG_DEGRADED) 2023-09-23T20:38:01.530 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:38:01.560 INFO:teuthology.orchestra.run.smithi018.stderr:marked in osd.6. 2023-09-23T20:38:01.644 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:01.859 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:38:01.863 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:01.863 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:38:01.964 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:38:01.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:01 smithi149 ceph-mon[170394]: pgmap v975: 119 pgs: 2 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 1 active+recovery_wait, 10 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 5.2 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 320 KiB/s wr, 372 op/s; 50138/255852 objects degraded (19.596%); 13643/255852 objects misplaced (5.332%); 7.1 MiB/s, 114 objects/s recovering 2023-09-23T20:38:01.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:01 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/243092763' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:38:01.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:01 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1886871503' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-09-23T20:38:01.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:01 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-09-23T20:38:01.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:01 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 50138/255852 objects degraded (19.596%), 12 pgs degraded (PG_DEGRADED) 2023-09-23T20:38:01.992 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:02.029 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:38:02.030 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:02.031 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:38:02.131 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:38:02.164 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:02.203 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:38:02.208 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:02.208 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:38:02.309 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:38:02.335 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:02.368 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:38:02.370 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:02.371 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:38:02.472 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:38:02.504 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:02.596 INFO:tasks.thrashosds.thrasher:Added osd 6 2023-09-23T20:38:02.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:02 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2023-09-23T20:38:02.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:02 smithi149 ceph-mon[170394]: osdmap e403: 8 total, 8 up, 7 in 2023-09-23T20:38:02.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:02 smithi149 ceph-mon[170394]: failed to encode map e403 with expected crc 2023-09-23T20:38:02.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:02 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2023-09-23T20:38:02.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:02 smithi018 ceph-mon[179647]: osdmap e403: 8 total, 8 up, 7 in 2023-09-23T20:38:02.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:02 smithi018 ceph-mon[179647]: failed to encode map e403 with expected crc 2023-09-23T20:38:02.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:02 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2023-09-23T20:38:02.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:02 smithi018 ceph-mon[184144]: osdmap e403: 8 total, 8 up, 7 in 2023-09-23T20:38:02.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:02 smithi018 ceph-mon[184144]: failed to encode map e403 with expected crc 2023-09-23T20:38:03.268 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:38:03.270 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:03.271 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:38:03.371 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:38:03.397 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:03.420 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:38:03.430 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:38:03.433 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:03.434 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:38:03.534 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:38:03.584 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:03.628 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:38:03.632 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:03.632 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:38:03.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:03 smithi018 ceph-mon[179647]: pgmap v977: 119 pgs: 2 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 1 active+recovery_wait, 10 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 5.2 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 404 op/s; 50138/253443 objects degraded (19.783%); 13266/253443 objects misplaced (5.234%); 7.4 MiB/s, 118 objects/s recovering 2023-09-23T20:38:03.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:03 smithi018 ceph-mon[179647]: failed to encode map e404 with expected crc 2023-09-23T20:38:03.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:03 smithi018 ceph-mon[179647]: failed to encode map e404 with expected crc 2023-09-23T20:38:03.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:03 smithi018 ceph-mon[179647]: osdmap e404: 8 total, 8 up, 7 in 2023-09-23T20:38:03.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:03 smithi018 ceph-mon[179647]: failed to encode map e404 with expected crc 2023-09-23T20:38:03.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:03 smithi018 ceph-mon[179647]: failed to encode map e404 with expected crc 2023-09-23T20:38:03.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:03 smithi018 ceph-mon[179647]: failed to encode map e404 with expected crc 2023-09-23T20:38:03.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:03 smithi018 ceph-mon[184144]: pgmap v977: 119 pgs: 2 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 1 active+recovery_wait, 10 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 5.2 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 404 op/s; 50138/253443 objects degraded (19.783%); 13266/253443 objects misplaced (5.234%); 7.4 MiB/s, 118 objects/s recovering 2023-09-23T20:38:03.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:03 smithi018 ceph-mon[184144]: failed to encode map e404 with expected crc 2023-09-23T20:38:03.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:03 smithi018 ceph-mon[184144]: failed to encode map e404 with expected crc 2023-09-23T20:38:03.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:03 smithi018 ceph-mon[184144]: osdmap e404: 8 total, 8 up, 7 in 2023-09-23T20:38:03.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:03 smithi018 ceph-mon[184144]: failed to encode map e404 with expected crc 2023-09-23T20:38:03.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:03 smithi018 ceph-mon[184144]: failed to encode map e404 with expected crc 2023-09-23T20:38:03.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:03 smithi018 ceph-mon[184144]: failed to encode map e404 with expected crc 2023-09-23T20:38:03.698 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:38:03.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:03 smithi149 ceph-mon[170394]: pgmap v977: 119 pgs: 2 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 1 active+recovery_wait, 10 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 5.2 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 404 op/s; 50138/253443 objects degraded (19.783%); 13266/253443 objects misplaced (5.234%); 7.4 MiB/s, 118 objects/s recovering 2023-09-23T20:38:03.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:03 smithi149 ceph-mon[170394]: failed to encode map e404 with expected crc 2023-09-23T20:38:03.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:03 smithi149 ceph-mon[170394]: failed to encode map e404 with expected crc 2023-09-23T20:38:03.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:03 smithi149 ceph-mon[170394]: osdmap e404: 8 total, 8 up, 7 in 2023-09-23T20:38:03.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:03 smithi149 ceph-mon[170394]: failed to encode map e404 with expected crc 2023-09-23T20:38:03.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:03 smithi149 ceph-mon[170394]: failed to encode map e404 with expected crc 2023-09-23T20:38:03.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:03 smithi149 ceph-mon[170394]: failed to encode map e404 with expected crc 2023-09-23T20:38:03.733 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:38:03.866 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:04.032 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:38:04.053 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:04.053 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:38:04.154 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:38:04.246 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:04.399 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:38:04.412 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:04.412 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:38:04.513 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:38:04.546 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:04.601 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:38:04.603 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:04.604 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:38:04.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:04 smithi018 ceph-mon[184144]: osdmap e405: 8 total, 8 up, 7 in 2023-09-23T20:38:04.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:04 smithi018 ceph-mon[184144]: failed to encode map e405 with expected crc 2023-09-23T20:38:04.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:04 smithi018 ceph-mon[184144]: failed to encode map e405 with expected crc 2023-09-23T20:38:04.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:04 smithi018 ceph-mon[184144]: failed to encode map e405 with expected crc 2023-09-23T20:38:04.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:04 smithi018 ceph-mon[184144]: failed to encode map e405 with expected crc 2023-09-23T20:38:04.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:04 smithi018 ceph-mon[184144]: failed to encode map e405 with expected crc 2023-09-23T20:38:04.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:04 smithi018 ceph-mon[184144]: failed to encode map e405 with expected crc 2023-09-23T20:38:04.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:04 smithi018 ceph-mon[184144]: failed to encode map e405 with expected crc 2023-09-23T20:38:04.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:04 smithi018 ceph-mon[184144]: failed to encode map e405 with expected crc 2023-09-23T20:38:04.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:04 smithi018 ceph-mon[184144]: failed to encode map e405 with expected crc 2023-09-23T20:38:04.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:04 smithi018 ceph-mon[184144]: failed to encode map e405 with expected crc 2023-09-23T20:38:04.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:04 smithi018 ceph-mon[184144]: failed to encode map e405 with expected crc 2023-09-23T20:38:04.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:04 smithi018 ceph-mon[184144]: 8.4 starting backfill to osd.6 from (0'0,0'0] MAX to 402'6078 2023-09-23T20:38:04.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:04 smithi018 ceph-mon[179647]: osdmap e405: 8 total, 8 up, 7 in 2023-09-23T20:38:04.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:04 smithi018 ceph-mon[179647]: failed to encode map e405 with expected crc 2023-09-23T20:38:04.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:04 smithi018 ceph-mon[179647]: failed to encode map e405 with expected crc 2023-09-23T20:38:04.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:04 smithi018 ceph-mon[179647]: failed to encode map e405 with expected crc 2023-09-23T20:38:04.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:04 smithi018 ceph-mon[179647]: failed to encode map e405 with expected crc 2023-09-23T20:38:04.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:04 smithi018 ceph-mon[179647]: failed to encode map e405 with expected crc 2023-09-23T20:38:04.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:04 smithi018 ceph-mon[179647]: failed to encode map e405 with expected crc 2023-09-23T20:38:04.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:04 smithi018 ceph-mon[179647]: failed to encode map e405 with expected crc 2023-09-23T20:38:04.644 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:04 smithi018 ceph-mon[179647]: failed to encode map e405 with expected crc 2023-09-23T20:38:04.644 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:04 smithi018 ceph-mon[179647]: failed to encode map e405 with expected crc 2023-09-23T20:38:04.644 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:04 smithi018 ceph-mon[179647]: failed to encode map e405 with expected crc 2023-09-23T20:38:04.644 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:04 smithi018 ceph-mon[179647]: failed to encode map e405 with expected crc 2023-09-23T20:38:04.644 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:04 smithi018 ceph-mon[179647]: 8.4 starting backfill to osd.6 from (0'0,0'0] MAX to 402'6078 2023-09-23T20:38:04.704 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:38:04.811 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:04.924 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:38:04.930 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:04.930 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:38:04.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:04 smithi149 ceph-mon[170394]: osdmap e405: 8 total, 8 up, 7 in 2023-09-23T20:38:04.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:04 smithi149 ceph-mon[170394]: failed to encode map e405 with expected crc 2023-09-23T20:38:04.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:04 smithi149 ceph-mon[170394]: failed to encode map e405 with expected crc 2023-09-23T20:38:04.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:04 smithi149 ceph-mon[170394]: failed to encode map e405 with expected crc 2023-09-23T20:38:04.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:04 smithi149 ceph-mon[170394]: failed to encode map e405 with expected crc 2023-09-23T20:38:04.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:04 smithi149 ceph-mon[170394]: failed to encode map e405 with expected crc 2023-09-23T20:38:04.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:04 smithi149 ceph-mon[170394]: failed to encode map e405 with expected crc 2023-09-23T20:38:04.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:04 smithi149 ceph-mon[170394]: failed to encode map e405 with expected crc 2023-09-23T20:38:04.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:04 smithi149 ceph-mon[170394]: failed to encode map e405 with expected crc 2023-09-23T20:38:04.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:04 smithi149 ceph-mon[170394]: failed to encode map e405 with expected crc 2023-09-23T20:38:04.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:04 smithi149 ceph-mon[170394]: failed to encode map e405 with expected crc 2023-09-23T20:38:04.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:04 smithi149 ceph-mon[170394]: failed to encode map e405 with expected crc 2023-09-23T20:38:04.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:04 smithi149 ceph-mon[170394]: 8.4 starting backfill to osd.6 from (0'0,0'0] MAX to 402'6078 2023-09-23T20:38:05.031 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:38:05.066 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:05.117 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:38:05.126 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:05.127 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:38:05.141 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:38:04 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:38:04] "GET /metrics HTTP/1.1" 200 36564 "" "Prometheus/2.43.0" 2023-09-23T20:38:05.228 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:38:05.262 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:05.303 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:38:05.307 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:05.308 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:38:05.408 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:38:05.490 INFO:teuthology.orchestra.run.smithi018.stdout:ERROR: (22) Invalid argument 2023-09-23T20:38:05.490 INFO:teuthology.orchestra.run.smithi018.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:38:05.514 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:05.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:05 smithi018 ceph-mon[184144]: pgmap v980: 119 pgs: 6 remapped+peering, 8 peering, 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 1 active+recovering, 2 active+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 5.1 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 332 op/s; 40266/252354 objects degraded (15.956%); 18101/252354 objects misplaced (7.173%); 7.1 MiB/s, 122 objects/s recovering 2023-09-23T20:38:05.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:05 smithi018 ceph-mon[184144]: Health check failed: Reduced data availability: 1 pg inactive, 3 pgs peering (PG_AVAILABILITY) 2023-09-23T20:38:05.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:05 smithi018 ceph-mon[184144]: osdmap e406: 8 total, 8 up, 7 in 2023-09-23T20:38:05.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:05 smithi018 ceph-mon[184144]: failed to encode map e406 with expected crc 2023-09-23T20:38:05.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:05 smithi018 ceph-mon[184144]: failed to encode map e406 with expected crc 2023-09-23T20:38:05.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:05 smithi018 ceph-mon[184144]: failed to encode map e406 with expected crc 2023-09-23T20:38:05.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:05 smithi018 ceph-mon[184144]: failed to encode map e406 with expected crc 2023-09-23T20:38:05.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:05 smithi018 ceph-mon[179647]: pgmap v980: 119 pgs: 6 remapped+peering, 8 peering, 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 1 active+recovering, 2 active+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 5.1 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 332 op/s; 40266/252354 objects degraded (15.956%); 18101/252354 objects misplaced (7.173%); 7.1 MiB/s, 122 objects/s recovering 2023-09-23T20:38:05.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:05 smithi018 ceph-mon[179647]: Health check failed: Reduced data availability: 1 pg inactive, 3 pgs peering (PG_AVAILABILITY) 2023-09-23T20:38:05.895 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:05 smithi018 ceph-mon[179647]: osdmap e406: 8 total, 8 up, 7 in 2023-09-23T20:38:05.895 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:05 smithi018 ceph-mon[179647]: failed to encode map e406 with expected crc 2023-09-23T20:38:05.895 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:05 smithi018 ceph-mon[179647]: failed to encode map e406 with expected crc 2023-09-23T20:38:05.895 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:05 smithi018 ceph-mon[179647]: failed to encode map e406 with expected crc 2023-09-23T20:38:05.895 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:05 smithi018 ceph-mon[179647]: failed to encode map e406 with expected crc 2023-09-23T20:38:05.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:05 smithi149 ceph-mon[170394]: pgmap v980: 119 pgs: 6 remapped+peering, 8 peering, 1 active+recovery_wait+degraded, 2 active+recovering+undersized+remapped, 1 active+recovering, 2 active+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 90 active+clean; 5.1 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 332 op/s; 40266/252354 objects degraded (15.956%); 18101/252354 objects misplaced (7.173%); 7.1 MiB/s, 122 objects/s recovering 2023-09-23T20:38:05.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:05 smithi149 ceph-mon[170394]: Health check failed: Reduced data availability: 1 pg inactive, 3 pgs peering (PG_AVAILABILITY) 2023-09-23T20:38:05.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:05 smithi149 ceph-mon[170394]: osdmap e406: 8 total, 8 up, 7 in 2023-09-23T20:38:05.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:05 smithi149 ceph-mon[170394]: failed to encode map e406 with expected crc 2023-09-23T20:38:05.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:05 smithi149 ceph-mon[170394]: failed to encode map e406 with expected crc 2023-09-23T20:38:05.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:05 smithi149 ceph-mon[170394]: failed to encode map e406 with expected crc 2023-09-23T20:38:05.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:05 smithi149 ceph-mon[170394]: failed to encode map e406 with expected crc 2023-09-23T20:38:06.387 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:38:06.394 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:06.394 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:38:06.495 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:38:06.606 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:06.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:06 smithi018 ceph-mon[179647]: osdmap e407: 8 total, 8 up, 7 in 2023-09-23T20:38:06.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:06 smithi018 ceph-mon[179647]: failed to encode map e407 with expected crc 2023-09-23T20:38:06.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:06 smithi018 ceph-mon[179647]: failed to encode map e407 with expected crc 2023-09-23T20:38:06.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:06 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 40266/252354 objects degraded (15.956%), 10 pgs degraded (PG_DEGRADED) 2023-09-23T20:38:06.643 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:06 smithi018 ceph-mon[184144]: osdmap e407: 8 total, 8 up, 7 in 2023-09-23T20:38:06.643 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:06 smithi018 ceph-mon[184144]: failed to encode map e407 with expected crc 2023-09-23T20:38:06.643 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:06 smithi018 ceph-mon[184144]: failed to encode map e407 with expected crc 2023-09-23T20:38:06.643 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:06 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 40266/252354 objects degraded (15.956%), 10 pgs degraded (PG_DEGRADED) 2023-09-23T20:38:06.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:06 smithi149 ceph-mon[170394]: osdmap e407: 8 total, 8 up, 7 in 2023-09-23T20:38:06.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:06 smithi149 ceph-mon[170394]: failed to encode map e407 with expected crc 2023-09-23T20:38:06.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:06 smithi149 ceph-mon[170394]: failed to encode map e407 with expected crc 2023-09-23T20:38:06.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:06 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 40266/252354 objects degraded (15.956%), 10 pgs degraded (PG_DEGRADED) 2023-09-23T20:38:07.597 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 7, 3, 2, 4, 6] out_osds: [5] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T20:38:07.597 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:38:07.597 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_3 2023-09-23T20:38:07.597 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:38:07.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:07 smithi018 ceph-mon[179647]: failed to encode map e407 with expected crc 2023-09-23T20:38:07.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:07 smithi018 ceph-mon[179647]: failed to encode map e407 with expected crc 2023-09-23T20:38:07.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:07 smithi018 ceph-mon[179647]: pgmap v983: 119 pgs: 4 remapped+peering, 8 peering, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 4 active+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 91 active+clean; 5.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 220 op/s; 26889/250800 objects degraded (10.721%); 19075/250800 objects misplaced (7.606%); 6.9 MiB/s, 126 objects/s recovering 2023-09-23T20:38:07.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:07 smithi018 ceph-mon[184144]: failed to encode map e407 with expected crc 2023-09-23T20:38:07.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:07 smithi018 ceph-mon[184144]: failed to encode map e407 with expected crc 2023-09-23T20:38:07.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:07 smithi018 ceph-mon[184144]: pgmap v983: 119 pgs: 4 remapped+peering, 8 peering, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 4 active+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 91 active+clean; 5.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 220 op/s; 26889/250800 objects degraded (10.721%); 19075/250800 objects misplaced (7.606%); 6.9 MiB/s, 126 objects/s recovering 2023-09-23T20:38:07.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:07 smithi149 ceph-mon[170394]: failed to encode map e407 with expected crc 2023-09-23T20:38:07.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:07 smithi149 ceph-mon[170394]: failed to encode map e407 with expected crc 2023-09-23T20:38:07.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:07 smithi149 ceph-mon[170394]: pgmap v983: 119 pgs: 4 remapped+peering, 8 peering, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 4 active+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 91 active+clean; 5.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 220 op/s; 26889/250800 objects degraded (10.721%); 19075/250800 objects misplaced (7.606%); 6.9 MiB/s, 126 objects/s recovering 2023-09-23T20:38:08.922 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:38:08.930 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:08.931 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:38:08.931 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:08 smithi018 ceph-mon[184144]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-09-23T20:38:08.932 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:08 smithi018 ceph-mon[179647]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-09-23T20:38:08.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:08 smithi149 ceph-mon[170394]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-09-23T20:38:09.032 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:38:09.118 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:09.866 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:38:09.873 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:09.873 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:38:09.961 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T20:38:09.961 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:09 smithi018 ceph-mon[179647]: pgmap v984: 119 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 4 active+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 101 active+clean; 5.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 250 op/s; 33048/249063 objects degraded (13.269%); 9690/249063 objects misplaced (3.891%); 5.1 MiB/s, 94 objects/s recovering 2023-09-23T20:38:09.962 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:09 smithi018 ceph-mon[184144]: pgmap v984: 119 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 4 active+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 101 active+clean; 5.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 250 op/s; 33048/249063 objects degraded (13.269%); 9690/249063 objects misplaced (3.891%); 5.1 MiB/s, 94 objects/s recovering 2023-09-23T20:38:09.962 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:38:09.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:09 smithi149 ceph-mon[170394]: pgmap v984: 119 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 4 active+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 101 active+clean; 5.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 250 op/s; 33048/249063 objects degraded (13.269%); 9690/249063 objects misplaced (3.891%); 5.1 MiB/s, 94 objects/s recovering 2023-09-23T20:38:09.974 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:38:10.021 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:10.063 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:38:10.066 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:10.066 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:38:10.167 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:38:10.207 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:10.244 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:38:10.247 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:10.247 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:38:10.348 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:38:10.445 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:10.708 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:38:10.715 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:10.715 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:38:10.816 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:38:10.961 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:11.216 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:38:11.223 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:11.223 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:38:11.324 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:38:11.342 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:38:11.464 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:11.657 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:38:11.670 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:11.670 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:38:11.710 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:38:11.771 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:38:11.798 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:11.840 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:38:11.843 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:11.844 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:38:11.933 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:11 smithi018 ceph-mon[179647]: pgmap v985: 119 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 105 active+clean; 5.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 393 op/s; 33048/245310 objects degraded (13.472%); 9465/245310 objects misplaced (3.858%); 7.1 MiB/s, 115 objects/s recovering 2023-09-23T20:38:11.933 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:11 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:38:11.933 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:11 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:38:11.933 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:11 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 33048/245310 objects degraded (13.472%), 13 pgs degraded (PG_DEGRADED) 2023-09-23T20:38:11.933 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:11 smithi018 ceph-mon[184144]: pgmap v985: 119 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 105 active+clean; 5.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 393 op/s; 33048/245310 objects degraded (13.472%); 9465/245310 objects misplaced (3.858%); 7.1 MiB/s, 115 objects/s recovering 2023-09-23T20:38:11.933 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:11 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:38:11.934 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:11 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:38:11.934 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:11 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 33048/245310 objects degraded (13.472%), 13 pgs degraded (PG_DEGRADED) 2023-09-23T20:38:11.944 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:38:11.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:11 smithi149 ceph-mon[170394]: pgmap v985: 119 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 105 active+clean; 5.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 393 op/s; 33048/245310 objects degraded (13.472%); 9465/245310 objects misplaced (3.858%); 7.1 MiB/s, 115 objects/s recovering 2023-09-23T20:38:11.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:11 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:38:11.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:11 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:38:11.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:11 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 33048/245310 objects degraded (13.472%), 13 pgs degraded (PG_DEGRADED) 2023-09-23T20:38:12.060 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:12.511 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:38:12.525 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:12.526 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:38:12.626 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:38:12.728 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:12.736 INFO:teuthology.orchestra.run.smithi018.stderr:noscrub is set 2023-09-23T20:38:12.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:12 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1197303284' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:38:12.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:12 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:38:12.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:12 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:38:12.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:12 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:38:12.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:12 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:38:12.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:12 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1197303284' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:38:12.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:12 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:38:12.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:12 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:38:12.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:12 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:38:12.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:12 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:38:12.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:12 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1197303284' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:38:12.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:12 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:38:12.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:12 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:38:12.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:12 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:38:12.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:12 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:38:13.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:13 smithi149 ceph-mon[170394]: pgmap v986: 119 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 105 active+clean; 4.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 438 op/s; 32876/241821 objects degraded (13.595%); 9465/241821 objects misplaced (3.914%); 5.3 MiB/s, 87 objects/s recovering 2023-09-23T20:38:13.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:13 smithi149 ceph-mon[170394]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:38:13.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:13 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:38:13.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:13 smithi149 ceph-mon[170394]: osdmap e408: 8 total, 8 up, 7 in 2023-09-23T20:38:13.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:13 smithi149 ceph-mon[170394]: failed to encode map e408 with expected crc 2023-09-23T20:38:13.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:13 smithi149 ceph-mon[170394]: failed to encode map e408 with expected crc 2023-09-23T20:38:13.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:13 smithi149 ceph-mon[170394]: failed to encode map e408 with expected crc 2023-09-23T20:38:13.974 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:13 smithi018 ceph-mon[184144]: pgmap v986: 119 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 105 active+clean; 4.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 438 op/s; 32876/241821 objects degraded (13.595%); 9465/241821 objects misplaced (3.914%); 5.3 MiB/s, 87 objects/s recovering 2023-09-23T20:38:13.974 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:13 smithi018 ceph-mon[184144]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:38:13.975 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:13 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:38:13.975 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:13 smithi018 ceph-mon[184144]: osdmap e408: 8 total, 8 up, 7 in 2023-09-23T20:38:13.975 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:13 smithi018 ceph-mon[184144]: failed to encode map e408 with expected crc 2023-09-23T20:38:13.975 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:13 smithi018 ceph-mon[184144]: failed to encode map e408 with expected crc 2023-09-23T20:38:13.975 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:13 smithi018 ceph-mon[184144]: failed to encode map e408 with expected crc 2023-09-23T20:38:13.975 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:13 smithi018 ceph-mon[179647]: pgmap v986: 119 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 105 active+clean; 4.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 438 op/s; 32876/241821 objects degraded (13.595%); 9465/241821 objects misplaced (3.914%); 5.3 MiB/s, 87 objects/s recovering 2023-09-23T20:38:13.975 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:13 smithi018 ceph-mon[179647]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:38:13.975 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:13 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:38:13.975 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:13 smithi018 ceph-mon[179647]: osdmap e408: 8 total, 8 up, 7 in 2023-09-23T20:38:13.976 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:13 smithi018 ceph-mon[179647]: failed to encode map e408 with expected crc 2023-09-23T20:38:13.976 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:13 smithi018 ceph-mon[179647]: failed to encode map e408 with expected crc 2023-09-23T20:38:13.976 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:13 smithi018 ceph-mon[179647]: failed to encode map e408 with expected crc 2023-09-23T20:38:14.159 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:38:14.168 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:14.168 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:38:14.269 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:38:14.388 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:14.499 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:38:14.549 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:38:14.602 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:38:14.654 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:38:14.710 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:38:14.752 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:38:14.785 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:38:14.827 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:38:14.859 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:38:14.859 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:38:14.860 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:38:14.860 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:38:14.860 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:38:14.860 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:38:14.860 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:38:14.860 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:38:14.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:14 smithi018 ceph-mon[179647]: pgmap v988: 119 pgs: 2 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 105 active+clean; 4.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 477 op/s; 32876/239526 objects degraded (13.725%); 9348/239526 objects misplaced (3.903%); 4.6 MiB/s, 76 objects/s recovering 2023-09-23T20:38:14.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:14 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1427336057' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:38:14.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:14 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:38:14.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:14 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:38:14.897 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:14 smithi018 ceph-mon[184144]: pgmap v988: 119 pgs: 2 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 105 active+clean; 4.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 477 op/s; 32876/239526 objects degraded (13.725%); 9348/239526 objects misplaced (3.903%); 4.6 MiB/s, 76 objects/s recovering 2023-09-23T20:38:14.897 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:14 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1427336057' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:38:14.897 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:14 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:38:14.897 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:14 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:38:15.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:14 smithi149 ceph-mon[170394]: pgmap v988: 119 pgs: 2 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 105 active+clean; 4.9 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 477 op/s; 32876/239526 objects degraded (13.725%); 9348/239526 objects misplaced (3.903%); 4.6 MiB/s, 76 objects/s recovering 2023-09-23T20:38:15.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:14 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1427336057' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:38:15.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:14 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:38:15.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:14 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:38:15.393 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:38:14 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:38:14] "GET /metrics HTTP/1.1" 200 36563 "" "Prometheus/2.43.0" 2023-09-23T20:38:15.750 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:38:15.756 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:15.757 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:38:15.857 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:38:15.885 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:15.927 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:38:15.934 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:15.934 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:38:16.035 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:38:16.119 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:16.321 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:38:16.336 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:16.337 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:38:16.437 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:38:16.472 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:16.512 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:38:16.517 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:16.517 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:38:16.618 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:38:16.646 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:16.685 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:38:16.689 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:16.690 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:38:16.749 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2023-09-23T20:38:16.749 INFO:teuthology.orchestra.run.smithi018.stdout: "ops": [], 2023-09-23T20:38:16.749 INFO:teuthology.orchestra.run.smithi018.stdout: "complaint_time": 30, 2023-09-23T20:38:16.749 INFO:teuthology.orchestra.run.smithi018.stdout: "num_blocked_ops": 0 2023-09-23T20:38:16.750 INFO:teuthology.orchestra.run.smithi018.stdout:} 2023-09-23T20:38:16.790 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:38:16.826 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:16.867 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:38:16.874 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:16.874 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:38:16.975 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:38:17.050 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:17.127 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:38:17.146 INFO:teuthology.orchestra.run.smithi018.stderr:dumped all 2023-09-23T20:38:17.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:17 smithi149 ceph-mon[170394]: pgmap v989: 119 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 105 active+clean; 4.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 479 op/s; 32788/236418 objects degraded (13.869%); 9181/236418 objects misplaced (3.883%); 4.9 MiB/s, 78 objects/s recovering 2023-09-23T20:38:17.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:17 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 32788/236418 objects degraded (13.869%), 13 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T20:38:17.258 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:38:17.264 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:17.265 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:38:17.331 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:17 smithi018 ceph-mon[179647]: pgmap v989: 119 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 105 active+clean; 4.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 479 op/s; 32788/236418 objects degraded (13.869%); 9181/236418 objects misplaced (3.883%); 4.9 MiB/s, 78 objects/s recovering 2023-09-23T20:38:17.331 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:17 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 32788/236418 objects degraded (13.869%), 13 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T20:38:17.333 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T20:38:17.333 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:17 smithi018 ceph-mon[184144]: pgmap v989: 119 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 105 active+clean; 4.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 479 op/s; 32788/236418 objects degraded (13.869%); 9181/236418 objects misplaced (3.883%); 4.9 MiB/s, 78 objects/s recovering 2023-09-23T20:38:17.333 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:17 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 32788/236418 objects degraded (13.869%), 13 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T20:38:17.366 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:38:17.537 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:17.589 INFO:teuthology.orchestra.run.smithi018.stdout:{"pg_ready":true,"pg_map":{"version":989,"stamp":"2023-09-23T20:38:16.010982+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5167346543,"num_objects":78806,"num_object_clones":0,"num_object_copies":236418,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":32788,"num_objects_misplaced":9181,"num_objects_unfound":0,"num_objects_dirty":78806,"num_whiteouts":0,"num_read":6139,"num_read_kb":13491,"num_write":100846,"num_write_kb":5729974,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6044,"num_bytes_recovered":392552754,"num_keys_recovered":40,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":85396,"ondisk_log_size":85396,"up":357,"acting":347,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":398,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":16317000,"kb_used_data":13106832,"kb_used_omap":32,"kb_used_meta":3210015,"kb_avail":639899064,"statfs":{"total":671965249536,"available":655256641536,"internally_reserved":0,"allocated":13421395968,"data_stored":13409297357,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32773,"internal_metadata":3287056379},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,1,1,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":11,"apply_latency_ms":11,"commit_latency_ns":11000000,"apply_latency_ns":11000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-314179584,"num_objects":-4794,"num_object_clones":0,"num_object_copies":-14382,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5899,"num_objects_misplaced":-9894,"num_objects_unfound":0,"num_objects_dirty":-4794,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4794,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":782,"num_bytes_recovered":50922096,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.007256"},"pg_stats":[{"pgid":"1.18","version":"90'161","reported_seq":468,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T20:38:05.583768+0000","last_change":"2023-09-23T20:38:03.548189+0000","last_active":"2023-09-23T20:38:05.583768+0000","last_peered":"2023-09-23T20:38:05.583768+0000","last_clean":"2023-09-23T20:38:05.583768+0000","last_became_active":"2023-09-23T20:38:03.547630+0000","last_became_peered":"2023-09-23T20:38:03.547630+0000","last_unstale":"2023-09-23T20:38:05.583768+0000","last_undegraded":"2023-09-23T20:38:05.583768+0000","last_fullsized":"2023-09-23T20:38:05.583768+0000","mapping_epoch":404,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":405,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:35:03.551143+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:35:03.551143+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":343193,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"69'1","reported_seq":105,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T20:38:05.584987+0000","last_change":"2023-09-23T20:32:33.233853+0000","last_active":"2023-09-23T20:38:05.584987+0000","last_peered":"2023-09-23T20:38:05.584987+0000","last_clean":"2023-09-23T20:38:05.584987+0000","last_became_active":"2023-09-23T20:32:33.233546+0000","last_became_peered":"2023-09-23T20:32:33.233546+0000","last_unstale":"2023-09-23T20:38:05.584987+0000","last_undegraded":"2023-09-23T20:38:05.584987+0000","last_fullsized":"2023-09-23T20:38:05.584987+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:50:27.130808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":220,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T20:38:05.597117+0000","last_change":"2023-09-23T20:37:43.579611+0000","last_active":"2023-09-23T20:38:05.597117+0000","last_peered":"2023-09-23T20:38:05.597117+0000","last_clean":"2023-09-23T20:38:05.597117+0000","last_became_active":"2023-09-23T20:37:43.579323+0000","last_became_peered":"2023-09-23T20:37:43.579323+0000","last_unstale":"2023-09-23T20:38:05.597117+0000","last_undegraded":"2023-09-23T20:38:05.597117+0000","last_fullsized":"2023-09-23T20:38:05.597117+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:17:32.311427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"90'161","reported_seq":523,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T20:38:05.578461+0000","last_change":"2023-09-23T20:37:17.292313+0000","last_active":"2023-09-23T20:38:05.578461+0000","last_peered":"2023-09-23T20:38:05.578461+0000","last_clean":"2023-09-23T20:38:05.578461+0000","last_became_active":"2023-09-23T20:37:17.291961+0000","last_became_peered":"2023-09-23T20:37:17.291961+0000","last_unstale":"2023-09-23T20:38:05.578461+0000","last_undegraded":"2023-09-23T20:38:05.578461+0000","last_fullsized":"2023-09-23T20:38:05.578461+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:05.442288+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:36:05.442288+0000","last_clean_scrub_stamp":"2023-09-23T20:36:05.442288+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0019500940000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,2],"acting":[4,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":375,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T20:38:05.584364+0000","last_change":"2023-09-23T20:34:59.983035+0000","last_active":"2023-09-23T20:38:05.584364+0000","last_peered":"2023-09-23T20:38:05.584364+0000","last_clean":"2023-09-23T20:38:05.584364+0000","last_became_active":"2023-09-23T20:34:59.982459+0000","last_became_peered":"2023-09-23T20:34:59.982459+0000","last_unstale":"2023-09-23T20:38:05.584364+0000","last_undegraded":"2023-09-23T20:38:05.584364+0000","last_fullsized":"2023-09-23T20:38:05.584364+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:16:48.510783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":375,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T20:38:04.565079+0000","last_change":"2023-09-23T20:25:02.957185+0000","last_active":"2023-09-23T20:38:04.565079+0000","last_peered":"2023-09-23T20:38:04.565079+0000","last_clean":"2023-09-23T20:38:04.565079+0000","last_became_active":"2023-09-23T20:25:02.956689+0000","last_became_peered":"2023-09-23T20:25:02.956689+0000","last_unstale":"2023-09-23T20:38:04.565079+0000","last_undegraded":"2023-09-23T20:38:04.565079+0000","last_fullsized":"2023-09-23T20:38:04.565079+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:04:28.524015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"90'161","reported_seq":487,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T20:38:05.584612+0000","last_change":"2023-09-23T20:36:01.292103+0000","last_active":"2023-09-23T20:38:05.584612+0000","last_peered":"2023-09-23T20:38:05.584612+0000","last_clean":"2023-09-23T20:38:05.584612+0000","last_became_active":"2023-09-23T20:35:26.639912+0000","last_became_peered":"2023-09-23T20:35:26.639912+0000","last_unstale":"2023-09-23T20:38:05.584612+0000","last_undegraded":"2023-09-23T20:38:05.584612+0000","last_fullsized":"2023-09-23T20:38:05.584612+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:01.292032+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:01.292032+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:04:53.926640+0000","scrub_duration":0.0032983159999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":200,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T20:38:05.612820+0000","last_change":"2023-09-23T20:37:43.584414+0000","last_active":"2023-09-23T20:38:05.612820+0000","last_peered":"2023-09-23T20:38:05.612820+0000","last_clean":"2023-09-23T20:38:05.612820+0000","last_became_active":"2023-09-23T20:37:43.584116+0000","last_became_peered":"2023-09-23T20:37:43.584116+0000","last_unstale":"2023-09-23T20:38:05.612820+0000","last_undegraded":"2023-09-23T20:38:05.612820+0000","last_fullsized":"2023-09-23T20:38:05.612820+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:17:54.258172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":14,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T20:38:05.582470+0000","last_change":"2023-09-23T20:38:02.590667+0000","last_active":"2023-09-23T20:38:05.582470+0000","last_peered":"2023-09-23T20:38:05.582470+0000","last_clean":"2023-09-23T20:38:05.582470+0000","last_became_active":"2023-09-23T20:38:02.590287+0000","last_became_peered":"2023-09-23T20:38:02.590287+0000","last_unstale":"2023-09-23T20:38:05.582470+0000","last_undegraded":"2023-09-23T20:38:05.582470+0000","last_fullsized":"2023-09-23T20:38:05.582470+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:29:29.989747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"90'161","reported_seq":599,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T20:38:04.559229+0000","last_change":"2023-09-23T20:37:17.304763+0000","last_active":"2023-09-23T20:38:04.559229+0000","last_peered":"2023-09-23T20:38:04.559229+0000","last_clean":"2023-09-23T20:38:04.559229+0000","last_became_active":"2023-09-23T20:37:17.304362+0000","last_became_peered":"2023-09-23T20:37:17.304362+0000","last_unstale":"2023-09-23T20:38:04.559229+0000","last_undegraded":"2023-09-23T20:38:04.559229+0000","last_fullsized":"2023-09-23T20:38:04.559229+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:03.411025+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:03.411025+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:11:42.762720+0000","scrub_duration":0.001802922,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":400,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T20:38:04.563640+0000","last_change":"2023-09-23T20:38:02.606871+0000","last_active":"2023-09-23T20:38:04.563640+0000","last_peered":"2023-09-23T20:38:04.563640+0000","last_clean":"2023-09-23T20:38:04.563640+0000","last_became_active":"2023-09-23T20:38:02.606596+0000","last_became_peered":"2023-09-23T20:38:02.606596+0000","last_unstale":"2023-09-23T20:38:04.563640+0000","last_undegraded":"2023-09-23T20:38:04.563640+0000","last_fullsized":"2023-09-23T20:38:04.563640+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:27:33.341667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":130,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T20:38:05.583916+0000","last_change":"2023-09-23T20:38:02.579343+0000","last_active":"2023-09-23T20:38:05.583916+0000","last_peered":"2023-09-23T20:38:05.583916+0000","last_clean":"2023-09-23T20:38:05.583916+0000","last_became_active":"2023-09-23T20:38:02.579047+0000","last_became_peered":"2023-09-23T20:38:02.579047+0000","last_unstale":"2023-09-23T20:38:05.583916+0000","last_undegraded":"2023-09-23T20:38:05.583916+0000","last_fullsized":"2023-09-23T20:38:05.583916+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:44:03.166556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.14","version":"90'161","reported_seq":736,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T20:38:05.583300+0000","last_change":"2023-09-23T20:35:02.236398+0000","last_active":"2023-09-23T20:38:05.583300+0000","last_peered":"2023-09-23T20:38:05.583300+0000","last_clean":"2023-09-23T20:38:05.583300+0000","last_became_active":"2023-09-23T20:35:02.235988+0000","last_became_peered":"2023-09-23T20:35:02.235988+0000","last_unstale":"2023-09-23T20:38:05.583300+0000","last_undegraded":"2023-09-23T20:38:05.583300+0000","last_fullsized":"2023-09-23T20:38:05.583300+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:47.479012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:47.479012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:13.519648+0000","scrub_duration":0.0020136590000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":274,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T20:38:04.560624+0000","last_change":"2023-09-23T20:37:14.362069+0000","last_active":"2023-09-23T20:38:04.560624+0000","last_peered":"2023-09-23T20:38:04.560624+0000","last_clean":"2023-09-23T20:38:04.560624+0000","last_became_active":"2023-09-23T20:37:14.361772+0000","last_became_peered":"2023-09-23T20:37:14.361772+0000","last_unstale":"2023-09-23T20:38:04.560624+0000","last_undegraded":"2023-09-23T20:38:04.560624+0000","last_fullsized":"2023-09-23T20:38:04.560624+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:20:31.626104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":231,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T20:38:05.595481+0000","last_change":"2023-09-23T20:38:02.542034+0000","last_active":"2023-09-23T20:38:05.595481+0000","last_peered":"2023-09-23T20:38:05.595481+0000","last_clean":"2023-09-23T20:38:05.595481+0000","last_became_active":"2023-09-23T20:38:02.541575+0000","last_became_peered":"2023-09-23T20:38:02.541575+0000","last_unstale":"2023-09-23T20:38:05.595481+0000","last_undegraded":"2023-09-23T20:38:05.595481+0000","last_fullsized":"2023-09-23T20:38:05.595481+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:50:54.591529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"90'161","reported_seq":549,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T20:38:05.611362+0000","last_change":"2023-09-23T20:38:05.611362+0000","last_active":"2023-09-23T20:38:05.611362+0000","last_peered":"2023-09-23T20:38:05.611362+0000","last_clean":"2023-09-23T20:38:05.611362+0000","last_became_active":"2023-09-23T20:38:05.611031+0000","last_became_peered":"2023-09-23T20:38:05.611031+0000","last_unstale":"2023-09-23T20:38:05.611362+0000","last_undegraded":"2023-09-23T20:38:05.611362+0000","last_fullsized":"2023-09-23T20:38:05.611362+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:03.435480+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:03.435480+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":434,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T20:38:04.560609+0000","last_change":"2023-09-23T20:32:33.270617+0000","last_active":"2023-09-23T20:38:04.560609+0000","last_peered":"2023-09-23T20:38:04.560609+0000","last_clean":"2023-09-23T20:38:04.560609+0000","last_became_active":"2023-09-23T20:32:33.227902+0000","last_became_peered":"2023-09-23T20:32:33.227902+0000","last_unstale":"2023-09-23T20:38:04.560609+0000","last_undegraded":"2023-09-23T20:38:04.560609+0000","last_fullsized":"2023-09-23T20:38:04.560609+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:30:26.847831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":360,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":376,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T20:38:05.584328+0000","last_change":"2023-09-23T20:38:02.599068+0000","last_active":"2023-09-23T20:38:05.584328+0000","last_peered":"2023-09-23T20:38:05.584328+0000","last_clean":"2023-09-23T20:38:05.584328+0000","last_became_active":"2023-09-23T20:38:02.597602+0000","last_became_peered":"2023-09-23T20:38:02.597602+0000","last_unstale":"2023-09-23T20:38:05.584328+0000","last_undegraded":"2023-09-23T20:38:05.584328+0000","last_fullsized":"2023-09-23T20:38:05.584328+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:34:06.337029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"90'161","reported_seq":531,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T20:38:05.593994+0000","last_change":"2023-09-23T20:36:02.548387+0000","last_active":"2023-09-23T20:38:05.593994+0000","last_peered":"2023-09-23T20:38:05.593994+0000","last_clean":"2023-09-23T20:38:05.593994+0000","last_became_active":"2023-09-23T20:35:26.644787+0000","last_became_peered":"2023-09-23T20:35:26.644787+0000","last_unstale":"2023-09-23T20:38:05.593994+0000","last_undegraded":"2023-09-23T20:38:05.593994+0000","last_fullsized":"2023-09-23T20:38:05.593994+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:02.548285+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:02.548285+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:37:15.216553+0000","scrub_duration":0.0015514459999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":155,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T20:38:05.578582+0000","last_change":"2023-09-23T20:38:02.560646+0000","last_active":"2023-09-23T20:38:05.578582+0000","last_peered":"2023-09-23T20:38:05.578582+0000","last_clean":"2023-09-23T20:38:05.578582+0000","last_became_active":"2023-09-23T20:38:02.539814+0000","last_became_peered":"2023-09-23T20:38:02.539814+0000","last_unstale":"2023-09-23T20:38:05.578582+0000","last_undegraded":"2023-09-23T20:38:05.578582+0000","last_fullsized":"2023-09-23T20:38:05.578582+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:00:09.435023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":135,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T20:38:05.579009+0000","last_change":"2023-09-23T20:38:02.579222+0000","last_active":"2023-09-23T20:38:05.579009+0000","last_peered":"2023-09-23T20:38:05.579009+0000","last_clean":"2023-09-23T20:38:05.579009+0000","last_became_active":"2023-09-23T20:38:02.578813+0000","last_became_peered":"2023-09-23T20:38:02.578813+0000","last_unstale":"2023-09-23T20:38:05.579009+0000","last_undegraded":"2023-09-23T20:38:05.579009+0000","last_fullsized":"2023-09-23T20:38:05.579009+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:57:31.080027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"90'161","reported_seq":655,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T20:38:04.578909+0000","last_change":"2023-09-23T20:37:17.282654+0000","last_active":"2023-09-23T20:38:04.578909+0000","last_peered":"2023-09-23T20:38:04.578909+0000","last_clean":"2023-09-23T20:38:04.578909+0000","last_became_active":"2023-09-23T20:37:17.282316+0000","last_became_peered":"2023-09-23T20:37:17.282316+0000","last_unstale":"2023-09-23T20:38:04.578909+0000","last_undegraded":"2023-09-23T20:38:04.578909+0000","last_fullsized":"2023-09-23T20:38:04.578909+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:43.578666+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:43.578666+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:53:37.101450+0000","scrub_duration":0.0021890379999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":433,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T20:38:04.560411+0000","last_change":"2023-09-23T20:32:33.242943+0000","last_active":"2023-09-23T20:38:04.560411+0000","last_peered":"2023-09-23T20:38:04.560411+0000","last_clean":"2023-09-23T20:38:04.560411+0000","last_became_active":"2023-09-23T20:32:33.225468+0000","last_became_peered":"2023-09-23T20:32:33.225468+0000","last_unstale":"2023-09-23T20:38:04.560411+0000","last_undegraded":"2023-09-23T20:38:04.560411+0000","last_fullsized":"2023-09-23T20:38:04.560411+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:40:47.867573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":142,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T20:38:05.603078+0000","last_change":"2023-09-23T20:37:43.571836+0000","last_active":"2023-09-23T20:38:05.603078+0000","last_peered":"2023-09-23T20:38:05.603078+0000","last_clean":"2023-09-23T20:38:05.603078+0000","last_became_active":"2023-09-23T20:37:43.571511+0000","last_became_peered":"2023-09-23T20:37:43.571511+0000","last_unstale":"2023-09-23T20:38:05.603078+0000","last_undegraded":"2023-09-23T20:38:05.603078+0000","last_fullsized":"2023-09-23T20:38:05.603078+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:59:25.507917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"90'161","reported_seq":666,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T20:38:05.590032+0000","last_change":"2023-09-23T20:38:05.590032+0000","last_active":"2023-09-23T20:38:05.590032+0000","last_peered":"2023-09-23T20:38:05.590032+0000","last_clean":"2023-09-23T20:38:05.590032+0000","last_became_active":"2023-09-23T20:38:05.589703+0000","last_became_peered":"2023-09-23T20:38:05.589703+0000","last_unstale":"2023-09-23T20:38:05.590032+0000","last_undegraded":"2023-09-23T20:38:05.590032+0000","last_fullsized":"2023-09-23T20:38:05.590032+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:49.726139+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:49.726139+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0096333110000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":373,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T20:38:04.562785+0000","last_change":"2023-09-23T20:25:02.953993+0000","last_active":"2023-09-23T20:38:04.562785+0000","last_peered":"2023-09-23T20:38:04.562785+0000","last_clean":"2023-09-23T20:38:04.562785+0000","last_became_active":"2023-09-23T20:25:02.953672+0000","last_became_peered":"2023-09-23T20:25:02.953672+0000","last_unstale":"2023-09-23T20:38:04.562785+0000","last_undegraded":"2023-09-23T20:38:04.562785+0000","last_fullsized":"2023-09-23T20:38:04.562785+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:26:08.603577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":195,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T20:38:04.560744+0000","last_change":"2023-09-23T20:38:02.712917+0000","last_active":"2023-09-23T20:38:04.560744+0000","last_peered":"2023-09-23T20:38:04.560744+0000","last_clean":"2023-09-23T20:38:04.560744+0000","last_became_active":"2023-09-23T20:38:02.658528+0000","last_became_peered":"2023-09-23T20:38:02.658528+0000","last_unstale":"2023-09-23T20:38:04.560744+0000","last_undegraded":"2023-09-23T20:38:04.560744+0000","last_fullsized":"2023-09-23T20:38:04.560744+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:42:32.680506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.13","version":"90'161","reported_seq":692,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T20:38:05.595612+0000","last_change":"2023-09-23T20:35:02.238923+0000","last_active":"2023-09-23T20:38:05.595612+0000","last_peered":"2023-09-23T20:38:05.595612+0000","last_clean":"2023-09-23T20:38:05.595612+0000","last_became_active":"2023-09-23T20:35:02.238293+0000","last_became_peered":"2023-09-23T20:35:02.238293+0000","last_unstale":"2023-09-23T20:38:05.595612+0000","last_undegraded":"2023-09-23T20:38:05.595612+0000","last_fullsized":"2023-09-23T20:38:05.595612+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:42.544308+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:42.544308+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:51:22.281932+0000","scrub_duration":0.0029958179999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":410,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T20:38:04.558671+0000","last_change":"2023-09-23T20:38:02.539774+0000","last_active":"2023-09-23T20:38:04.558671+0000","last_peered":"2023-09-23T20:38:04.558671+0000","last_clean":"2023-09-23T20:38:04.558671+0000","last_became_active":"2023-09-23T20:38:02.539436+0000","last_became_peered":"2023-09-23T20:38:02.539436+0000","last_unstale":"2023-09-23T20:38:04.558671+0000","last_undegraded":"2023-09-23T20:38:04.558671+0000","last_fullsized":"2023-09-23T20:38:04.558671+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:50:57.375803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":383,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T20:38:04.565146+0000","last_change":"2023-09-23T20:25:02.957010+0000","last_active":"2023-09-23T20:38:04.565146+0000","last_peered":"2023-09-23T20:38:04.565146+0000","last_clean":"2023-09-23T20:38:04.565146+0000","last_became_active":"2023-09-23T20:25:02.956267+0000","last_became_peered":"2023-09-23T20:25:02.956267+0000","last_unstale":"2023-09-23T20:38:04.565146+0000","last_undegraded":"2023-09-23T20:38:04.565146+0000","last_fullsized":"2023-09-23T20:38:04.565146+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:10:55.299422+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"90'161","reported_seq":502,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T20:38:05.621518+0000","last_change":"2023-09-23T20:38:05.606521+0000","last_active":"2023-09-23T20:38:05.621518+0000","last_peered":"2023-09-23T20:38:05.621518+0000","last_clean":"2023-09-23T20:38:05.621518+0000","last_became_active":"2023-09-23T20:38:05.606164+0000","last_became_peered":"2023-09-23T20:38:05.606164+0000","last_unstale":"2023-09-23T20:38:05.621518+0000","last_undegraded":"2023-09-23T20:38:05.621518+0000","last_fullsized":"2023-09-23T20:38:05.621518+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:53.706934+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:53.706934+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":24,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T20:38:05.581786+0000","last_change":"2023-09-23T20:38:02.626973+0000","last_active":"2023-09-23T20:38:05.581786+0000","last_peered":"2023-09-23T20:38:05.581786+0000","last_clean":"2023-09-23T20:38:05.581786+0000","last_became_active":"2023-09-23T20:38:02.545190+0000","last_became_peered":"2023-09-23T20:38:02.545190+0000","last_unstale":"2023-09-23T20:38:05.581786+0000","last_undegraded":"2023-09-23T20:38:05.581786+0000","last_fullsized":"2023-09-23T20:38:05.581786+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:38:12.102685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":200,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T20:38:05.614344+0000","last_change":"2023-09-23T20:37:43.571083+0000","last_active":"2023-09-23T20:38:05.614344+0000","last_peered":"2023-09-23T20:38:05.614344+0000","last_clean":"2023-09-23T20:38:05.614344+0000","last_became_active":"2023-09-23T20:37:43.570306+0000","last_became_peered":"2023-09-23T20:37:43.570306+0000","last_unstale":"2023-09-23T20:38:05.614344+0000","last_undegraded":"2023-09-23T20:38:05.614344+0000","last_fullsized":"2023-09-23T20:38:05.614344+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:42:02.396718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.4","version":"408'6481","reported_seq":7345,"reported_epoch":408,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T20:38:15.598650+0000","last_change":"2023-09-23T20:38:03.731185+0000","last_active":"2023-09-23T20:38:15.598650+0000","last_peered":"2023-09-23T20:38:15.598650+0000","last_clean":"2023-09-23T20:37:15.925522+0000","last_became_active":"2023-09-23T20:38:03.683171+0000","last_became_peered":"2023-09-23T20:38:03.683171+0000","last_unstale":"2023-09-23T20:38:15.598650+0000","last_undegraded":"2023-09-23T20:38:03.670916+0000","last_fullsized":"2023-09-23T20:38:15.598650+0000","mapping_epoch":404,"log_start":"373'1780","ondisk_log_start":"373'1780","created":369,"last_epoch_clean":370,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":4701,"log_dups_size":0,"ondisk_log_size":4701,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:45:50.794312+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":321454080,"num_objects":4905,"num_object_clones":0,"num_object_copies":14715,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4905,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4905,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6481,"num_write_kb":364352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":404,"num_bytes_recovered":26476544,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,3],"avail_no_missing":["4","2","5"],"object_location_counts":[{"shards":"2,3,4","objects":327},{"shards":"2,4,5","objects":4578}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.d","version":"90'161","reported_seq":699,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T20:38:04.559324+0000","last_change":"2023-09-23T20:34:59.496438+0000","last_active":"2023-09-23T20:38:04.559324+0000","last_peered":"2023-09-23T20:38:04.559324+0000","last_clean":"2023-09-23T20:38:04.559324+0000","last_became_active":"2023-09-23T20:33:38.970527+0000","last_became_peered":"2023-09-23T20:33:38.970527+0000","last_unstale":"2023-09-23T20:38:04.559324+0000","last_undegraded":"2023-09-23T20:38:04.559324+0000","last_fullsized":"2023-09-23T20:38:04.559324+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:34:59.496375+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:34:59.496375+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:46:18.244734+0000","scrub_duration":0.0017843189999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":410,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T20:38:04.559699+0000","last_change":"2023-09-23T20:32:33.283752+0000","last_active":"2023-09-23T20:38:04.559699+0000","last_peered":"2023-09-23T20:38:04.559699+0000","last_clean":"2023-09-23T20:38:04.559699+0000","last_became_active":"2023-09-23T20:32:33.230503+0000","last_became_peered":"2023-09-23T20:32:33.230503+0000","last_unstale":"2023-09-23T20:38:04.559699+0000","last_undegraded":"2023-09-23T20:38:04.559699+0000","last_fullsized":"2023-09-23T20:38:04.559699+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:29.084410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":252,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T20:38:04.558970+0000","last_change":"2023-09-23T20:38:02.539067+0000","last_active":"2023-09-23T20:38:04.558970+0000","last_peered":"2023-09-23T20:38:04.558970+0000","last_clean":"2023-09-23T20:38:04.558970+0000","last_became_active":"2023-09-23T20:38:02.538745+0000","last_became_peered":"2023-09-23T20:38:02.538745+0000","last_unstale":"2023-09-23T20:38:04.558970+0000","last_undegraded":"2023-09-23T20:38:04.558970+0000","last_fullsized":"2023-09-23T20:38:04.558970+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:36:31.615062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.5","version":"408'6305","reported_seq":9125,"reported_epoch":408,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:38:15.483171+0000","last_change":"2023-09-23T20:38:02.583026+0000","last_active":"2023-09-23T20:38:15.483171+0000","last_peered":"2023-09-23T20:38:15.483171+0000","last_clean":"2023-09-23T20:37:15.983935+0000","last_became_active":"2023-09-23T20:38:02.575092+0000","last_became_peered":"2023-09-23T20:38:02.575092+0000","last_unstale":"2023-09-23T20:38:15.483171+0000","last_undegraded":"2023-09-23T20:38:02.534834+0000","last_fullsized":"2023-09-23T20:38:02.524629+0000","mapping_epoch":403,"log_start":"373'1539","ondisk_log_start":"373'1539","created":369,"last_epoch_clean":370,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":4766,"log_dups_size":0,"ondisk_log_size":4766,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:45:50.447569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":314638336,"num_objects":4801,"num_object_clones":0,"num_object_copies":14403,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":702,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4801,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6305,"num_write_kb":355392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1377,"num_bytes_recovered":90243072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[4,7],"avail_no_missing":["4","7"],"object_location_counts":[{"shards":"0,4,7","objects":327},{"shards":"4,7","objects":4474}],"blocked_by":[],"up_primary":6,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.c","version":"90'161","reported_seq":641,"reported_epoch":407,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:38:05.603390+0000","last_change":"2023-09-23T20:38:03.545142+0000","last_active":"2023-09-23T20:38:05.603390+0000","last_peered":"2023-09-23T20:38:05.603390+0000","last_clean":"2023-09-23T20:37:57.920598+0000","last_became_active":"2023-09-23T20:38:03.544662+0000","last_became_peered":"2023-09-23T20:38:03.544662+0000","last_unstale":"2023-09-23T20:38:05.603390+0000","last_undegraded":"2023-09-23T20:38:03.528737+0000","last_fullsized":"2023-09-23T20:38:03.528331+0000","mapping_epoch":404,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:05.485751+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:05.485751+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.002180043,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163840,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0],"avail_no_missing":["3","0","4"],"object_location_counts":[{"shards":"0,3","objects":-1},{"shards":"0,3,4","objects":1}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":445,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T20:38:04.560705+0000","last_change":"2023-09-23T20:34:59.987981+0000","last_active":"2023-09-23T20:38:04.560705+0000","last_peered":"2023-09-23T20:38:04.560705+0000","last_clean":"2023-09-23T20:38:04.560705+0000","last_became_active":"2023-09-23T20:34:59.985368+0000","last_became_peered":"2023-09-23T20:34:59.985368+0000","last_unstale":"2023-09-23T20:38:04.560705+0000","last_undegraded":"2023-09-23T20:38:04.560705+0000","last_fullsized":"2023-09-23T20:38:04.560705+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:12:43.082493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":105,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T20:38:05.577054+0000","last_change":"2023-09-23T20:32:33.232142+0000","last_active":"2023-09-23T20:38:05.577054+0000","last_peered":"2023-09-23T20:38:05.577054+0000","last_clean":"2023-09-23T20:38:05.577054+0000","last_became_active":"2023-09-23T20:32:33.231683+0000","last_became_peered":"2023-09-23T20:32:33.231683+0000","last_unstale":"2023-09-23T20:38:05.577054+0000","last_undegraded":"2023-09-23T20:38:05.577054+0000","last_fullsized":"2023-09-23T20:38:05.577054+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:02:58.336348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"407'6135","reported_seq":6198,"reported_epoch":407,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:38:11.009705+0000","last_change":"2023-09-23T20:38:02.694017+0000","last_active":"2023-09-23T20:38:11.009705+0000","last_peered":"2023-09-23T20:38:11.009705+0000","last_clean":"2023-09-23T20:37:19.797265+0000","last_became_active":"2023-09-23T20:38:02.605275+0000","last_became_peered":"2023-09-23T20:38:02.605275+0000","last_unstale":"2023-09-23T20:38:11.009705+0000","last_undegraded":"2023-09-23T20:38:02.553891+0000","last_fullsized":"2023-09-23T20:38:02.536494+0000","mapping_epoch":403,"log_start":"373'961","ondisk_log_start":"373'961","created":369,"last_epoch_clean":370,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":5174,"log_dups_size":0,"ondisk_log_size":5174,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:32:35.154280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":325910528,"num_objects":4973,"num_object_clones":0,"num_object_copies":14919,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":640,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4973,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6135,"num_write_kb":355456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[{"shards":"0,2,7","objects":307},{"shards":"0,7","objects":4666}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"90'161","reported_seq":544,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T20:38:05.623056+0000","last_change":"2023-09-23T20:38:05.622695+0000","last_active":"2023-09-23T20:38:05.623056+0000","last_peered":"2023-09-23T20:38:05.623056+0000","last_clean":"2023-09-23T20:38:05.623056+0000","last_became_active":"2023-09-23T20:38:05.622356+0000","last_became_peered":"2023-09-23T20:38:05.622356+0000","last_unstale":"2023-09-23T20:38:05.623056+0000","last_undegraded":"2023-09-23T20:38:05.623056+0000","last_fullsized":"2023-09-23T20:38:05.623056+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:46.217489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:46.217489+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:17:38.331779+0000","scrub_duration":0.001456429,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":421,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T20:38:04.559616+0000","last_change":"2023-09-23T20:38:02.715585+0000","last_active":"2023-09-23T20:38:04.559616+0000","last_peered":"2023-09-23T20:38:04.559616+0000","last_clean":"2023-09-23T20:38:04.559616+0000","last_became_active":"2023-09-23T20:38:02.658973+0000","last_became_peered":"2023-09-23T20:38:02.658973+0000","last_unstale":"2023-09-23T20:38:04.559616+0000","last_undegraded":"2023-09-23T20:38:04.559616+0000","last_fullsized":"2023-09-23T20:38:04.559616+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:32:03.044825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":350,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":14,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T20:38:05.581313+0000","last_change":"2023-09-23T20:38:02.590613+0000","last_active":"2023-09-23T20:38:05.581313+0000","last_peered":"2023-09-23T20:38:05.581313+0000","last_clean":"2023-09-23T20:38:05.581313+0000","last_became_active":"2023-09-23T20:38:02.590284+0000","last_became_peered":"2023-09-23T20:38:02.590284+0000","last_unstale":"2023-09-23T20:38:05.581313+0000","last_undegraded":"2023-09-23T20:38:05.581313+0000","last_fullsized":"2023-09-23T20:38:05.581313+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:40:07.825054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"408'6226","reported_seq":6291,"reported_epoch":408,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:38:14.317206+0000","last_change":"2023-09-23T20:38:02.636016+0000","last_active":"2023-09-23T20:38:14.317206+0000","last_peered":"2023-09-23T20:38:14.317206+0000","last_clean":"2023-09-23T20:37:19.384160+0000","last_became_active":"2023-09-23T20:38:02.626676+0000","last_became_peered":"2023-09-23T20:38:02.626676+0000","last_unstale":"2023-09-23T20:38:14.317206+0000","last_undegraded":"2023-09-23T20:38:02.574062+0000","last_fullsized":"2023-09-23T20:38:02.533567+0000","mapping_epoch":403,"log_start":"373'1258","ondisk_log_start":"373'1258","created":369,"last_epoch_clean":370,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":4968,"log_dups_size":0,"ondisk_log_size":4968,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:04:06.427567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":320339968,"num_objects":4888,"num_object_clones":0,"num_object_copies":14664,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":635,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4888,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6226,"num_write_kb":355648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3],"avail_no_missing":["1","3"],"object_location_counts":[{"shards":"1,2,3","objects":306},{"shards":"1,3","objects":4582}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.e","version":"90'161","reported_seq":508,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T20:38:05.577447+0000","last_change":"2023-09-23T20:36:00.322453+0000","last_active":"2023-09-23T20:38:05.577447+0000","last_peered":"2023-09-23T20:38:05.577447+0000","last_clean":"2023-09-23T20:38:05.577447+0000","last_became_active":"2023-09-23T20:35:02.233536+0000","last_became_peered":"2023-09-23T20:35:02.233536+0000","last_unstale":"2023-09-23T20:38:05.577447+0000","last_undegraded":"2023-09-23T20:38:05.577447+0000","last_fullsized":"2023-09-23T20:38:05.577447+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:00.322381+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:00.322381+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:02:15.489069+0000","scrub_duration":0.0017321859999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":424,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T20:38:05.603749+0000","last_change":"2023-09-23T20:38:02.590403+0000","last_active":"2023-09-23T20:38:05.603749+0000","last_peered":"2023-09-23T20:38:05.603749+0000","last_clean":"2023-09-23T20:38:05.603749+0000","last_became_active":"2023-09-23T20:38:02.590122+0000","last_became_peered":"2023-09-23T20:38:02.590122+0000","last_unstale":"2023-09-23T20:38:05.603749+0000","last_undegraded":"2023-09-23T20:38:05.603749+0000","last_fullsized":"2023-09-23T20:38:05.603749+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:54:19.913605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":391,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T20:38:04.562187+0000","last_change":"2023-09-23T20:32:33.236122+0000","last_active":"2023-09-23T20:38:04.562187+0000","last_peered":"2023-09-23T20:38:04.562187+0000","last_clean":"2023-09-23T20:38:04.562187+0000","last_became_active":"2023-09-23T20:32:33.235384+0000","last_became_peered":"2023-09-23T20:32:33.235384+0000","last_unstale":"2023-09-23T20:38:04.562187+0000","last_undegraded":"2023-09-23T20:38:04.562187+0000","last_fullsized":"2023-09-23T20:38:04.562187+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:56.843733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"408'6129","reported_seq":8705,"reported_epoch":408,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T20:38:13.844684+0000","last_change":"2023-09-23T20:38:02.606728+0000","last_active":"2023-09-23T20:38:13.844684+0000","last_peered":"2023-09-23T20:38:13.844684+0000","last_clean":"2023-09-23T20:37:13.293179+0000","last_became_active":"2023-09-23T20:37:44.588978+0000","last_became_peered":"2023-09-23T20:37:44.588978+0000","last_unstale":"2023-09-23T20:38:13.844684+0000","last_undegraded":"2023-09-23T20:38:02.600937+0000","last_fullsized":"2023-09-23T20:38:13.844684+0000","mapping_epoch":400,"log_start":"373'1887","ondisk_log_start":"373'1887","created":369,"last_epoch_clean":370,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":4242,"log_dups_size":0,"ondisk_log_size":4242,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:30:02.912864+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":313327616,"num_objects":4781,"num_object_clones":0,"num_object_copies":14343,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3848,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4781,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6129,"num_write_kb":349120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1254,"num_bytes_recovered":82182144,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":["3","1","5"],"object_location_counts":[{"shards":"1,3,4","objects":933},{"shards":"1,3,5","objects":3848}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":812,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T20:38:04.558531+0000","last_change":"2023-09-23T20:35:02.523308+0000","last_active":"2023-09-23T20:38:04.558531+0000","last_peered":"2023-09-23T20:38:04.558531+0000","last_clean":"2023-09-23T20:38:04.558531+0000","last_became_active":"2023-09-23T20:33:38.979989+0000","last_became_peered":"2023-09-23T20:33:38.979989+0000","last_unstale":"2023-09-23T20:38:04.558531+0000","last_undegraded":"2023-09-23T20:38:04.558531+0000","last_fullsized":"2023-09-23T20:38:04.558531+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:35:02.523203+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:35:02.523203+0000","last_clean_scrub_stamp":"2023-09-23T20:35:02.523203+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:02:41.472958+0000","scrub_duration":0.0085927989999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":226,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T20:38:05.600940+0000","last_change":"2023-09-23T20:37:43.583731+0000","last_active":"2023-09-23T20:38:05.600940+0000","last_peered":"2023-09-23T20:38:05.600940+0000","last_clean":"2023-09-23T20:38:05.600940+0000","last_became_active":"2023-09-23T20:37:43.583066+0000","last_became_peered":"2023-09-23T20:37:43.583066+0000","last_unstale":"2023-09-23T20:38:05.600940+0000","last_undegraded":"2023-09-23T20:38:05.600940+0000","last_fullsized":"2023-09-23T20:38:05.600940+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:03:45.211900+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":125,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T20:38:05.578504+0000","last_change":"2023-09-23T20:38:02.579814+0000","last_active":"2023-09-23T20:38:05.578504+0000","last_peered":"2023-09-23T20:38:05.578504+0000","last_clean":"2023-09-23T20:38:05.578504+0000","last_became_active":"2023-09-23T20:38:02.579253+0000","last_became_peered":"2023-09-23T20:38:02.579253+0000","last_unstale":"2023-09-23T20:38:05.578504+0000","last_undegraded":"2023-09-23T20:38:05.578504+0000","last_fullsized":"2023-09-23T20:38:05.578504+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:05:07.298198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"408'6292","reported_seq":6340,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T20:38:14.393194+0000","last_change":"2023-09-23T20:36:11.307387+0000","last_active":"2023-09-23T20:38:14.393194+0000","last_peered":"2023-09-23T20:38:14.393194+0000","last_clean":"2023-09-23T20:38:14.393194+0000","last_became_active":"2023-09-23T20:36:11.307082+0000","last_became_peered":"2023-09-23T20:36:11.307082+0000","last_unstale":"2023-09-23T20:38:14.393194+0000","last_undegraded":"2023-09-23T20:38:14.393194+0000","last_fullsized":"2023-09-23T20:38:14.393194+0000","mapping_epoch":369,"log_start":"373'1294","ondisk_log_start":"373'1294","created":369,"last_epoch_clean":370,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":4998,"log_dups_size":0,"ondisk_log_size":4998,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:06:28.610503+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":319422464,"num_objects":4874,"num_object_clones":0,"num_object_copies":14622,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4874,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6292,"num_write_kb":357312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":498,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T20:38:05.633044+0000","last_change":"2023-09-23T20:38:05.632243+0000","last_active":"2023-09-23T20:38:05.633044+0000","last_peered":"2023-09-23T20:38:05.633044+0000","last_clean":"2023-09-23T20:38:05.633044+0000","last_became_active":"2023-09-23T20:38:05.631907+0000","last_became_peered":"2023-09-23T20:38:05.631907+0000","last_unstale":"2023-09-23T20:38:05.633044+0000","last_undegraded":"2023-09-23T20:38:05.633044+0000","last_fullsized":"2023-09-23T20:38:05.633044+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:35:00.496197+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:35:00.496197+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":14,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T20:38:05.581415+0000","last_change":"2023-09-23T20:38:02.578703+0000","last_active":"2023-09-23T20:38:05.581415+0000","last_peered":"2023-09-23T20:38:05.581415+0000","last_clean":"2023-09-23T20:38:05.581415+0000","last_became_active":"2023-09-23T20:38:02.578414+0000","last_became_peered":"2023-09-23T20:38:02.578414+0000","last_unstale":"2023-09-23T20:38:05.581415+0000","last_undegraded":"2023-09-23T20:38:05.581415+0000","last_fullsized":"2023-09-23T20:38:05.581415+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:43:59.779295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":74,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T20:38:05.577346+0000","last_change":"2023-09-23T20:37:31.296606+0000","last_active":"2023-09-23T20:38:05.577346+0000","last_peered":"2023-09-23T20:38:05.577346+0000","last_clean":"2023-09-23T20:38:05.577346+0000","last_became_active":"2023-09-23T20:37:31.295924+0000","last_became_peered":"2023-09-23T20:37:31.295924+0000","last_unstale":"2023-09-23T20:38:05.577346+0000","last_undegraded":"2023-09-23T20:38:05.577346+0000","last_fullsized":"2023-09-23T20:38:05.577346+0000","mapping_epoch":391,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":392,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:27:50.948510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"408'6361","reported_seq":6431,"reported_epoch":408,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:38:15.567684+0000","last_change":"2023-09-23T20:38:02.728507+0000","last_active":"2023-09-23T20:38:15.567684+0000","last_peered":"2023-09-23T20:38:15.567684+0000","last_clean":"2023-09-23T20:37:29.399979+0000","last_became_active":"2023-09-23T20:37:32.488640+0000","last_became_peered":"2023-09-23T20:37:32.488640+0000","last_unstale":"2023-09-23T20:38:15.567684+0000","last_undegraded":"2023-09-23T20:37:32.323451+0000","last_fullsized":"2023-09-23T20:37:32.285677+0000","mapping_epoch":392,"log_start":"373'1596","ondisk_log_start":"373'1596","created":369,"last_epoch_clean":370,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":4765,"log_dups_size":0,"ondisk_log_size":4765,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:20:50.993764+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":318963712,"num_objects":4867,"num_object_clones":0,"num_object_copies":14601,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5278,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4867,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6361,"num_write_kb":359296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0],"avail_no_missing":["4","0","5"],"object_location_counts":[{"shards":"0,4","objects":-411},{"shards":"0,4,5","objects":4536}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"90'161","reported_seq":691,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T20:38:05.595696+0000","last_change":"2023-09-23T20:37:17.282621+0000","last_active":"2023-09-23T20:38:05.595696+0000","last_peered":"2023-09-23T20:38:05.595696+0000","last_clean":"2023-09-23T20:38:05.595696+0000","last_became_active":"2023-09-23T20:37:17.282243+0000","last_became_peered":"2023-09-23T20:37:17.282243+0000","last_unstale":"2023-09-23T20:38:05.595696+0000","last_undegraded":"2023-09-23T20:38:05.595696+0000","last_fullsized":"2023-09-23T20:38:05.595696+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:05.595042+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:05.595042+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:42:06.244026+0000","scrub_duration":0.007271675,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":64,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T20:38:05.576801+0000","last_change":"2023-09-23T20:34:59.983122+0000","last_active":"2023-09-23T20:38:05.576801+0000","last_peered":"2023-09-23T20:38:05.576801+0000","last_clean":"2023-09-23T20:38:05.576801+0000","last_became_active":"2023-09-23T20:34:59.982809+0000","last_became_peered":"2023-09-23T20:34:59.982809+0000","last_unstale":"2023-09-23T20:38:05.576801+0000","last_undegraded":"2023-09-23T20:38:05.576801+0000","last_fullsized":"2023-09-23T20:38:05.576801+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:49:40.839723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":125,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T20:38:05.577653+0000","last_change":"2023-09-23T20:38:02.543866+0000","last_active":"2023-09-23T20:38:05.577653+0000","last_peered":"2023-09-23T20:38:05.577653+0000","last_clean":"2023-09-23T20:38:05.577653+0000","last_became_active":"2023-09-23T20:38:02.543563+0000","last_became_peered":"2023-09-23T20:38:02.543563+0000","last_unstale":"2023-09-23T20:38:05.577653+0000","last_undegraded":"2023-09-23T20:38:05.577653+0000","last_fullsized":"2023-09-23T20:38:05.577653+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:33.050536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"408'6274","reported_seq":7409,"reported_epoch":408,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:38:14.341956+0000","last_change":"2023-09-23T20:38:03.738377+0000","last_active":"2023-09-23T20:38:14.341956+0000","last_peered":"2023-09-23T20:38:14.341956+0000","last_clean":"2023-09-23T20:37:13.297905+0000","last_became_active":"2023-09-23T20:38:03.673416+0000","last_became_peered":"2023-09-23T20:38:03.673416+0000","last_unstale":"2023-09-23T20:38:14.341956+0000","last_undegraded":"2023-09-23T20:38:03.658992+0000","last_fullsized":"2023-09-23T20:38:03.525897+0000","mapping_epoch":404,"log_start":"373'1280","ondisk_log_start":"373'1280","created":369,"last_epoch_clean":370,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":4994,"log_dups_size":0,"ondisk_log_size":4994,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:45:56.016050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":321912832,"num_objects":4912,"num_object_clones":0,"num_object_copies":14736,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5580,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4912,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6274,"num_write_kb":357952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":354,"num_bytes_recovered":23134208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6],"avail_no_missing":["1","5"],"object_location_counts":[{"shards":"1,2,3","objects":319},{"shards":"1,5,6","objects":4581},{"shards":"1,6","objects":-316}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":469,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T20:38:05.616545+0000","last_change":"2023-09-23T20:38:05.604552+0000","last_active":"2023-09-23T20:38:05.616545+0000","last_peered":"2023-09-23T20:38:05.616545+0000","last_clean":"2023-09-23T20:38:05.616545+0000","last_became_active":"2023-09-23T20:38:05.604235+0000","last_became_peered":"2023-09-23T20:38:05.604235+0000","last_unstale":"2023-09-23T20:38:05.616545+0000","last_undegraded":"2023-09-23T20:38:05.616545+0000","last_fullsized":"2023-09-23T20:38:05.616545+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:34:59.481446+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:34:59.481446+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26834,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,4],"acting":[2,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":200,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T20:38:05.612738+0000","last_change":"2023-09-23T20:37:43.570610+0000","last_active":"2023-09-23T20:38:05.612738+0000","last_peered":"2023-09-23T20:38:05.612738+0000","last_clean":"2023-09-23T20:38:05.612738+0000","last_became_active":"2023-09-23T20:37:43.570216+0000","last_became_peered":"2023-09-23T20:37:43.570216+0000","last_unstale":"2023-09-23T20:38:05.612738+0000","last_undegraded":"2023-09-23T20:38:05.612738+0000","last_fullsized":"2023-09-23T20:38:05.612738+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:39:26.555329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":230,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T20:38:05.597271+0000","last_change":"2023-09-23T20:32:33.230582+0000","last_active":"2023-09-23T20:38:05.597271+0000","last_peered":"2023-09-23T20:38:05.597271+0000","last_clean":"2023-09-23T20:38:05.597271+0000","last_became_active":"2023-09-23T20:32:33.230257+0000","last_became_peered":"2023-09-23T20:32:33.230257+0000","last_unstale":"2023-09-23T20:38:05.597271+0000","last_undegraded":"2023-09-23T20:38:05.597271+0000","last_fullsized":"2023-09-23T20:38:05.597271+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:11:45.333096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"407'6084","reported_seq":6131,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T20:38:10.920083+0000","last_change":"2023-09-23T20:36:11.310149+0000","last_active":"2023-09-23T20:38:10.920083+0000","last_peered":"2023-09-23T20:38:10.920083+0000","last_clean":"2023-09-23T20:38:10.920083+0000","last_became_active":"2023-09-23T20:36:11.309844+0000","last_became_peered":"2023-09-23T20:36:11.309844+0000","last_unstale":"2023-09-23T20:38:10.920083+0000","last_undegraded":"2023-09-23T20:38:10.920083+0000","last_fullsized":"2023-09-23T20:38:10.920083+0000","mapping_epoch":369,"log_start":"373'1019","ondisk_log_start":"373'1019","created":369,"last_epoch_clean":370,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":5065,"log_dups_size":0,"ondisk_log_size":5065,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:14:57.442108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":325844992,"num_objects":4972,"num_object_clones":0,"num_object_copies":14916,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4972,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6084,"num_write_kb":353792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":587,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T20:38:05.622963+0000","last_change":"2023-09-23T20:38:05.622597+0000","last_active":"2023-09-23T20:38:05.622963+0000","last_peered":"2023-09-23T20:38:05.622963+0000","last_clean":"2023-09-23T20:38:05.622963+0000","last_became_active":"2023-09-23T20:38:05.622261+0000","last_became_peered":"2023-09-23T20:38:05.622261+0000","last_unstale":"2023-09-23T20:38:05.622963+0000","last_undegraded":"2023-09-23T20:38:05.622963+0000","last_fullsized":"2023-09-23T20:38:05.622963+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:35:04.101972+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:35:04.101972+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"86'119","reported_seq":715,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T20:38:09.382463+0000","last_change":"2023-09-23T20:29:45.521257+0000","last_active":"2023-09-23T20:38:09.382463+0000","last_peered":"2023-09-23T20:38:09.382463+0000","last_clean":"2023-09-23T20:38:09.382463+0000","last_became_active":"2023-09-23T20:29:45.520546+0000","last_became_peered":"2023-09-23T20:29:45.520546+0000","last_unstale":"2023-09-23T20:38:09.382463+0000","last_undegraded":"2023-09-23T20:38:09.382463+0000","last_fullsized":"2023-09-23T20:38:09.382463+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":119,"log_dups_size":0,"ondisk_log_size":119,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:58:56.085777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032705499999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":523,"num_read_kb":3503,"num_write":136,"num_write_kb":3128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":231,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T20:38:05.597259+0000","last_change":"2023-09-23T20:38:02.597306+0000","last_active":"2023-09-23T20:38:05.597259+0000","last_peered":"2023-09-23T20:38:05.597259+0000","last_clean":"2023-09-23T20:38:05.597259+0000","last_became_active":"2023-09-23T20:38:02.596966+0000","last_became_peered":"2023-09-23T20:38:02.596966+0000","last_unstale":"2023-09-23T20:38:05.597259+0000","last_undegraded":"2023-09-23T20:38:05.597259+0000","last_fullsized":"2023-09-23T20:38:05.597259+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:30:41.962907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":353,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T20:38:09.382324+0000","last_change":"2023-09-23T20:37:14.359424+0000","last_active":"2023-09-23T20:38:09.382324+0000","last_peered":"2023-09-23T20:38:09.382324+0000","last_clean":"2023-09-23T20:38:09.382324+0000","last_became_active":"2023-09-23T20:37:14.359059+0000","last_became_peered":"2023-09-23T20:37:14.359059+0000","last_unstale":"2023-09-23T20:38:09.382324+0000","last_undegraded":"2023-09-23T20:38:09.382324+0000","last_fullsized":"2023-09-23T20:38:09.382324+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:07:10.779153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.8","version":"408'6246","reported_seq":7622,"reported_epoch":408,"state":"active+recovering","last_fresh":"2023-09-23T20:38:13.829676+0000","last_change":"2023-09-23T20:38:02.644256+0000","last_active":"2023-09-23T20:38:13.829676+0000","last_peered":"2023-09-23T20:38:13.829676+0000","last_clean":"2023-09-23T20:37:13.320723+0000","last_became_active":"2023-09-23T20:37:44.600883+0000","last_became_peered":"2023-09-23T20:37:44.600883+0000","last_unstale":"2023-09-23T20:38:13.829676+0000","last_undegraded":"2023-09-23T20:38:13.829676+0000","last_fullsized":"2023-09-23T20:38:13.829676+0000","mapping_epoch":400,"log_start":"373'1934","ondisk_log_start":"373'1934","created":369,"last_epoch_clean":370,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":4312,"log_dups_size":0,"ondisk_log_size":4312,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:01:54.455070+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":323878912,"num_objects":4942,"num_object_clones":0,"num_object_copies":14826,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4591,"num_objects_unfound":0,"num_objects_dirty":4942,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6246,"num_write_kb":358016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":659,"num_bytes_recovered":43188224,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":["3","5","7"],"object_location_counts":[{"shards":"0,3,7","objects":351},{"shards":"3,5,7","objects":4591}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":469,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T20:38:04.579183+0000","last_change":"2023-09-23T20:37:17.290846+0000","last_active":"2023-09-23T20:38:04.579183+0000","last_peered":"2023-09-23T20:38:04.579183+0000","last_clean":"2023-09-23T20:38:04.579183+0000","last_became_active":"2023-09-23T20:37:17.290470+0000","last_became_peered":"2023-09-23T20:37:17.290470+0000","last_unstale":"2023-09-23T20:38:04.579183+0000","last_undegraded":"2023-09-23T20:38:04.579183+0000","last_fullsized":"2023-09-23T20:38:04.579183+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:45.785450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:45.785450+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"86'106","reported_seq":715,"reported_epoch":407,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T20:38:05.584163+0000","last_change":"2023-09-23T20:38:03.543047+0000","last_active":"2023-09-23T20:38:05.584163+0000","last_peered":"2023-09-23T20:38:05.584163+0000","last_clean":"2023-09-23T20:13:52.922055+0000","last_became_active":"2023-09-23T20:38:03.542561+0000","last_became_peered":"2023-09-23T20:38:03.542561+0000","last_unstale":"2023-09-23T20:38:05.584163+0000","last_undegraded":"2023-09-23T20:38:03.531295+0000","last_fullsized":"2023-09-23T20:38:05.584163+0000","mapping_epoch":404,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":106,"log_dups_size":0,"ondisk_log_size":106,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:59:55.031916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00099302599999999993,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1094,"num_read_kb":1678,"num_write":100,"num_write_kb":404,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":["1","5","6"],"object_location_counts":[{"shards":"1,5,6","objects":1}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":125,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T20:38:05.578893+0000","last_change":"2023-09-23T20:38:02.552008+0000","last_active":"2023-09-23T20:38:05.578893+0000","last_peered":"2023-09-23T20:38:05.578893+0000","last_clean":"2023-09-23T20:38:05.578893+0000","last_became_active":"2023-09-23T20:38:02.551684+0000","last_became_peered":"2023-09-23T20:38:02.551684+0000","last_unstale":"2023-09-23T20:38:05.578893+0000","last_undegraded":"2023-09-23T20:38:05.578893+0000","last_fullsized":"2023-09-23T20:38:05.578893+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:40:37.147538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":368,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T20:38:09.382448+0000","last_change":"2023-09-23T20:29:45.533406+0000","last_active":"2023-09-23T20:38:09.382448+0000","last_peered":"2023-09-23T20:38:09.382448+0000","last_clean":"2023-09-23T20:38:09.382448+0000","last_became_active":"2023-09-23T20:29:45.526001+0000","last_became_peered":"2023-09-23T20:29:45.526001+0000","last_unstale":"2023-09-23T20:38:09.382448+0000","last_undegraded":"2023-09-23T20:38:09.382448+0000","last_fullsized":"2023-09-23T20:38:09.382448+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:43.787251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.b","version":"408'6212","reported_seq":6274,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T20:38:13.804393+0000","last_change":"2023-09-23T20:38:01.517845+0000","last_active":"2023-09-23T20:38:13.804393+0000","last_peered":"2023-09-23T20:38:13.804393+0000","last_clean":"2023-09-23T20:38:13.804393+0000","last_became_active":"2023-09-23T20:37:43.579403+0000","last_became_peered":"2023-09-23T20:37:43.579403+0000","last_unstale":"2023-09-23T20:38:13.804393+0000","last_undegraded":"2023-09-23T20:38:13.804393+0000","last_fullsized":"2023-09-23T20:38:13.804393+0000","mapping_epoch":399,"log_start":"373'1900","ondisk_log_start":"373'1900","created":369,"last_epoch_clean":403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":4312,"log_dups_size":0,"ondisk_log_size":4312,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:33:13.920936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":322699264,"num_objects":4924,"num_object_clones":0,"num_object_copies":14772,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4924,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6212,"num_write_kb":356352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":879,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T20:38:04.558853+0000","last_change":"2023-09-23T20:35:02.260708+0000","last_active":"2023-09-23T20:38:04.558853+0000","last_peered":"2023-09-23T20:38:04.558853+0000","last_clean":"2023-09-23T20:38:04.558853+0000","last_became_active":"2023-09-23T20:35:02.260348+0000","last_became_peered":"2023-09-23T20:35:02.260348+0000","last_unstale":"2023-09-23T20:38:04.558853+0000","last_undegraded":"2023-09-23T20:38:04.558853+0000","last_fullsized":"2023-09-23T20:38:04.558853+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:43.390908+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:43.390908+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:09:14.144070+0000","scrub_duration":0.0032721,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"86'130","reported_seq":723,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T20:38:12.697643+0000","last_change":"2023-09-23T20:25:39.637458+0000","last_active":"2023-09-23T20:38:12.697643+0000","last_peered":"2023-09-23T20:38:12.697643+0000","last_clean":"2023-09-23T20:38:12.697643+0000","last_became_active":"2023-09-23T20:25:39.636773+0000","last_became_peered":"2023-09-23T20:25:39.636773+0000","last_unstale":"2023-09-23T20:38:12.697643+0000","last_undegraded":"2023-09-23T20:38:12.697643+0000","last_fullsized":"2023-09-23T20:38:12.697643+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:00:32.810564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011300049999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":512,"num_read_kb":1820,"num_write":133,"num_write_kb":1347,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":376,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T20:38:05.583905+0000","last_change":"2023-09-23T20:38:02.591335+0000","last_active":"2023-09-23T20:38:05.583905+0000","last_peered":"2023-09-23T20:38:05.583905+0000","last_clean":"2023-09-23T20:38:05.583905+0000","last_became_active":"2023-09-23T20:38:02.591015+0000","last_became_peered":"2023-09-23T20:38:02.591015+0000","last_unstale":"2023-09-23T20:38:05.583905+0000","last_undegraded":"2023-09-23T20:38:05.583905+0000","last_fullsized":"2023-09-23T20:38:05.583905+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:55:32.272195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":358,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T20:38:05.597206+0000","last_change":"2023-09-23T20:37:14.360153+0000","last_active":"2023-09-23T20:38:05.597206+0000","last_peered":"2023-09-23T20:38:05.597206+0000","last_clean":"2023-09-23T20:38:05.597206+0000","last_became_active":"2023-09-23T20:37:14.359892+0000","last_became_peered":"2023-09-23T20:37:14.359892+0000","last_unstale":"2023-09-23T20:38:05.597206+0000","last_undegraded":"2023-09-23T20:38:05.597206+0000","last_fullsized":"2023-09-23T20:38:05.597206+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:59:37.031659+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"407'6213","reported_seq":8141,"reported_epoch":407,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:38:10.888630+0000","last_change":"2023-09-23T20:38:02.790622+0000","last_active":"2023-09-23T20:38:10.888630+0000","last_peered":"2023-09-23T20:38:10.888630+0000","last_clean":"2023-09-23T20:37:13.347998+0000","last_became_active":"2023-09-23T20:38:02.705735+0000","last_became_peered":"2023-09-23T20:38:02.705735+0000","last_unstale":"2023-09-23T20:38:10.888630+0000","last_undegraded":"2023-09-23T20:38:02.691707+0000","last_fullsized":"2023-09-23T20:38:02.528692+0000","mapping_epoch":403,"log_start":"373'1116","ondisk_log_start":"373'1116","created":369,"last_epoch_clean":370,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":5097,"log_dups_size":0,"ondisk_log_size":5097,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:40:10.104223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":330891264,"num_objects":5049,"num_object_clones":0,"num_object_copies":15147,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5338,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5049,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6213,"num_write_kb":360384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":622,"num_bytes_recovered":40632320,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[0,3],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,3","objects":46},{"shards":"0,3,4","objects":322},{"shards":"0,5,6","objects":4681}],"blocked_by":[],"up_primary":6,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"86'194","reported_seq":756,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T20:38:05.586889+0000","last_change":"2023-09-23T20:37:17.289352+0000","last_active":"2023-09-23T20:38:05.586889+0000","last_peered":"2023-09-23T20:38:05.586889+0000","last_clean":"2023-09-23T20:38:05.586889+0000","last_became_active":"2023-09-23T20:37:17.288695+0000","last_became_peered":"2023-09-23T20:37:17.288695+0000","last_unstale":"2023-09-23T20:38:05.586889+0000","last_undegraded":"2023-09-23T20:38:05.586889+0000","last_fullsized":"2023-09-23T20:38:05.586889+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":194,"log_dups_size":0,"ondisk_log_size":194,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:43:22.047407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00037995999999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1024,"num_read_kb":1200,"num_write":173,"num_write_kb":1276,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":869,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T20:38:05.602508+0000","last_change":"2023-09-23T20:38:05.602508+0000","last_active":"2023-09-23T20:38:05.602508+0000","last_peered":"2023-09-23T20:38:05.602508+0000","last_clean":"2023-09-23T20:38:05.602508+0000","last_became_active":"2023-09-23T20:38:05.602129+0000","last_became_peered":"2023-09-23T20:38:05.602129+0000","last_unstale":"2023-09-23T20:38:05.602508+0000","last_undegraded":"2023-09-23T20:38:05.602508+0000","last_fullsized":"2023-09-23T20:38:05.602508+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:45.343742+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:45.343742+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:48.910343+0000","scrub_duration":0.001475802,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":14,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T20:38:05.581682+0000","last_change":"2023-09-23T20:38:02.541452+0000","last_active":"2023-09-23T20:38:05.581682+0000","last_peered":"2023-09-23T20:38:05.581682+0000","last_clean":"2023-09-23T20:38:05.581682+0000","last_became_active":"2023-09-23T20:38:02.541112+0000","last_became_peered":"2023-09-23T20:38:02.541112+0000","last_unstale":"2023-09-23T20:38:05.581682+0000","last_undegraded":"2023-09-23T20:38:05.581682+0000","last_fullsized":"2023-09-23T20:38:05.581682+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:03:51.385905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,4],"acting":[6,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":14,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T20:38:05.582125+0000","last_change":"2023-09-23T20:38:02.573200+0000","last_active":"2023-09-23T20:38:05.582125+0000","last_peered":"2023-09-23T20:38:05.582125+0000","last_clean":"2023-09-23T20:38:05.582125+0000","last_became_active":"2023-09-23T20:38:02.572945+0000","last_became_peered":"2023-09-23T20:38:02.572945+0000","last_unstale":"2023-09-23T20:38:05.582125+0000","last_undegraded":"2023-09-23T20:38:05.582125+0000","last_fullsized":"2023-09-23T20:38:05.582125+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:57.246533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.a","version":"408'6410","reported_seq":7126,"reported_epoch":408,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T20:38:14.376522+0000","last_change":"2023-09-23T20:38:02.722059+0000","last_active":"2023-09-23T20:38:14.376522+0000","last_peered":"2023-09-23T20:38:14.376522+0000","last_clean":"2023-09-23T20:37:13.317597+0000","last_became_active":"2023-09-23T20:37:15.678492+0000","last_became_peered":"2023-09-23T20:37:15.678492+0000","last_unstale":"2023-09-23T20:38:14.376522+0000","last_undegraded":"2023-09-23T20:37:23.465606+0000","last_fullsized":"2023-09-23T20:37:15.349179+0000","mapping_epoch":380,"log_start":"373'1466","ondisk_log_start":"373'1466","created":369,"last_epoch_clean":370,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":4944,"log_dups_size":0,"ondisk_log_size":4944,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:52:53.085450+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":322437120,"num_objects":4920,"num_object_clones":0,"num_object_copies":14760,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":423,"num_objects_misplaced":4590,"num_objects_unfound":0,"num_objects_dirty":4920,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6410,"num_write_kb":362560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":326,"num_bytes_recovered":21364736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2],"avail_no_missing":["1","2","5"],"object_location_counts":[{"shards":"1,2","objects":-93},{"shards":"1,2,5","objects":4590}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":737,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T20:38:09.382410+0000","last_change":"2023-09-23T20:38:05.615711+0000","last_active":"2023-09-23T20:38:09.382410+0000","last_peered":"2023-09-23T20:38:09.382410+0000","last_clean":"2023-09-23T20:38:09.382410+0000","last_became_active":"2023-09-23T20:38:05.615390+0000","last_became_peered":"2023-09-23T20:38:05.615390+0000","last_unstale":"2023-09-23T20:38:09.382410+0000","last_undegraded":"2023-09-23T20:38:09.382410+0000","last_fullsized":"2023-09-23T20:38:09.382410+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:43.582146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:43.582146+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:45:11.917871+0000","scrub_duration":0.0016219730000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"86'80","reported_seq":659,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T20:38:04.557881+0000","last_change":"2023-09-23T20:29:45.536796+0000","last_active":"2023-09-23T20:38:04.557881+0000","last_peered":"2023-09-23T20:38:04.557881+0000","last_clean":"2023-09-23T20:38:04.557881+0000","last_became_active":"2023-09-23T20:29:45.536439+0000","last_became_peered":"2023-09-23T20:29:45.536439+0000","last_unstale":"2023-09-23T20:38:04.557881+0000","last_undegraded":"2023-09-23T20:38:04.557881+0000","last_fullsized":"2023-09-23T20:38:04.557881+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":80,"log_dups_size":0,"ondisk_log_size":80,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:30:57.793132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035115,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":395,"num_read_kb":1370,"num_write":66,"num_write_kb":1061,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":410,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T20:38:04.579103+0000","last_change":"2023-09-23T20:34:59.984286+0000","last_active":"2023-09-23T20:38:04.579103+0000","last_peered":"2023-09-23T20:38:04.579103+0000","last_clean":"2023-09-23T20:38:04.579103+0000","last_became_active":"2023-09-23T20:34:59.983950+0000","last_became_peered":"2023-09-23T20:34:59.983950+0000","last_unstale":"2023-09-23T20:38:04.579103+0000","last_undegraded":"2023-09-23T20:38:04.579103+0000","last_fullsized":"2023-09-23T20:38:04.579103+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:31:53.557035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":405,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T20:38:04.564937+0000","last_change":"2023-09-23T20:29:45.690815+0000","last_active":"2023-09-23T20:38:04.564937+0000","last_peered":"2023-09-23T20:38:04.564937+0000","last_clean":"2023-09-23T20:38:04.564937+0000","last_became_active":"2023-09-23T20:29:45.690478+0000","last_became_peered":"2023-09-23T20:29:45.690478+0000","last_unstale":"2023-09-23T20:38:04.564937+0000","last_undegraded":"2023-09-23T20:38:04.564937+0000","last_fullsized":"2023-09-23T20:38:04.564937+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:38.288193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.f","version":"408'6118","reported_seq":6166,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T20:38:12.837964+0000","last_change":"2023-09-23T20:36:11.309536+0000","last_active":"2023-09-23T20:38:12.837964+0000","last_peered":"2023-09-23T20:38:12.837964+0000","last_clean":"2023-09-23T20:38:12.837964+0000","last_became_active":"2023-09-23T20:36:11.309208+0000","last_became_peered":"2023-09-23T20:36:11.309208+0000","last_unstale":"2023-09-23T20:38:12.837964+0000","last_undegraded":"2023-09-23T20:38:12.837964+0000","last_fullsized":"2023-09-23T20:38:12.837964+0000","mapping_epoch":369,"log_start":"0'0","ondisk_log_start":"0'0","created":369,"last_epoch_clean":370,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":6118,"log_dups_size":0,"ondisk_log_size":6118,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:47:51.785158+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":322174976,"num_objects":4916,"num_object_clones":0,"num_object_copies":14748,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4916,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6118,"num_write_kb":353088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":510,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T20:38:05.611413+0000","last_change":"2023-09-23T20:38:05.611181+0000","last_active":"2023-09-23T20:38:05.611413+0000","last_peered":"2023-09-23T20:38:05.611413+0000","last_clean":"2023-09-23T20:38:05.611413+0000","last_became_active":"2023-09-23T20:38:05.610881+0000","last_became_peered":"2023-09-23T20:38:05.610881+0000","last_unstale":"2023-09-23T20:38:05.611413+0000","last_undegraded":"2023-09-23T20:38:05.611413+0000","last_fullsized":"2023-09-23T20:38:05.611413+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:35:00.151695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:35:00.151695+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":20,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"86'102","reported_seq":1001,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T20:38:04.559709+0000","last_change":"2023-09-23T20:32:36.314672+0000","last_active":"2023-09-23T20:38:04.559709+0000","last_peered":"2023-09-23T20:38:04.559709+0000","last_clean":"2023-09-23T20:38:04.559709+0000","last_became_active":"2023-09-23T20:32:36.314231+0000","last_became_peered":"2023-09-23T20:32:36.314231+0000","last_unstale":"2023-09-23T20:38:04.559709+0000","last_undegraded":"2023-09-23T20:38:04.559709+0000","last_fullsized":"2023-09-23T20:38:04.559709+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":102,"log_dups_size":0,"ondisk_log_size":102,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:11:40.514709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030656500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":963,"num_read_kb":1234,"num_write":60,"num_write_kb":225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":343,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T20:38:09.382481+0000","last_change":"2023-09-23T20:37:14.359334+0000","last_active":"2023-09-23T20:38:09.382481+0000","last_peered":"2023-09-23T20:38:09.382481+0000","last_clean":"2023-09-23T20:38:09.382481+0000","last_became_active":"2023-09-23T20:37:14.358982+0000","last_became_peered":"2023-09-23T20:37:14.358982+0000","last_unstale":"2023-09-23T20:38:09.382481+0000","last_undegraded":"2023-09-23T20:38:09.382481+0000","last_fullsized":"2023-09-23T20:38:09.382481+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:50:00.008145+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":351,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T20:38:04.562438+0000","last_change":"2023-09-23T20:37:14.359068+0000","last_active":"2023-09-23T20:38:04.562438+0000","last_peered":"2023-09-23T20:38:04.562438+0000","last_clean":"2023-09-23T20:38:04.562438+0000","last_became_active":"2023-09-23T20:37:14.358729+0000","last_became_peered":"2023-09-23T20:37:14.358729+0000","last_unstale":"2023-09-23T20:38:04.562438+0000","last_undegraded":"2023-09-23T20:38:04.562438+0000","last_fullsized":"2023-09-23T20:38:04.562438+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:19:17.084920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"407'6211","reported_seq":6258,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T20:38:10.758110+0000","last_change":"2023-09-23T20:36:11.482126+0000","last_active":"2023-09-23T20:38:10.758110+0000","last_peered":"2023-09-23T20:38:10.758110+0000","last_clean":"2023-09-23T20:38:10.758110+0000","last_became_active":"2023-09-23T20:36:11.481802+0000","last_became_peered":"2023-09-23T20:36:11.481802+0000","last_unstale":"2023-09-23T20:38:10.758110+0000","last_undegraded":"2023-09-23T20:38:10.758110+0000","last_fullsized":"2023-09-23T20:38:10.758110+0000","mapping_epoch":369,"log_start":"373'1059","ondisk_log_start":"373'1059","created":369,"last_epoch_clean":370,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":5152,"log_dups_size":0,"ondisk_log_size":5152,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:13:12.263455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":333774848,"num_objects":5093,"num_object_clones":0,"num_object_copies":15279,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5093,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6211,"num_write_kb":361728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":686,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T20:38:05.590976+0000","last_change":"2023-09-23T20:38:05.590361+0000","last_active":"2023-09-23T20:38:05.590976+0000","last_peered":"2023-09-23T20:38:05.590976+0000","last_clean":"2023-09-23T20:38:05.590976+0000","last_became_active":"2023-09-23T20:38:05.589985+0000","last_became_peered":"2023-09-23T20:38:05.589985+0000","last_unstale":"2023-09-23T20:38:05.590976+0000","last_undegraded":"2023-09-23T20:38:05.590976+0000","last_fullsized":"2023-09-23T20:38:05.590976+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:03.246271+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:03.246271+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:38:54.351576+0000","scrub_duration":0.0019016700000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":144,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T20:38:04.560270+0000","last_change":"2023-09-23T20:32:50.549695+0000","last_active":"2023-09-23T20:38:04.560270+0000","last_peered":"2023-09-23T20:38:04.560270+0000","last_clean":"2023-09-23T20:38:04.560270+0000","last_became_active":"2023-09-23T20:32:50.549431+0000","last_became_peered":"2023-09-23T20:32:50.549431+0000","last_unstale":"2023-09-23T20:38:04.560270+0000","last_undegraded":"2023-09-23T20:38:04.560270+0000","last_fullsized":"2023-09-23T20:38:04.560270+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:45:53.666569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"86'128","reported_seq":817,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T20:38:05.594218+0000","last_change":"2023-09-23T20:38:03.541765+0000","last_active":"2023-09-23T20:38:05.594218+0000","last_peered":"2023-09-23T20:38:05.594218+0000","last_clean":"2023-09-23T20:38:05.594218+0000","last_became_active":"2023-09-23T20:38:03.541277+0000","last_became_peered":"2023-09-23T20:38:03.541277+0000","last_unstale":"2023-09-23T20:38:05.594218+0000","last_undegraded":"2023-09-23T20:38:05.594218+0000","last_fullsized":"2023-09-23T20:38:05.594218+0000","mapping_epoch":404,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":405,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:20:53.834554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000293202,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":579,"num_read_kb":1473,"num_write":96,"num_write_kb":1062,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":185,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T20:38:05.612562+0000","last_change":"2023-09-23T20:37:43.582105+0000","last_active":"2023-09-23T20:38:05.612562+0000","last_peered":"2023-09-23T20:38:05.612562+0000","last_clean":"2023-09-23T20:38:05.612562+0000","last_became_active":"2023-09-23T20:37:43.581860+0000","last_became_peered":"2023-09-23T20:37:43.581860+0000","last_unstale":"2023-09-23T20:38:05.612562+0000","last_undegraded":"2023-09-23T20:38:05.612562+0000","last_fullsized":"2023-09-23T20:38:05.612562+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:18:38.765486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"407'6166","reported_seq":8067,"reported_epoch":407,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:38:12.169259+0000","last_change":"2023-09-23T20:38:02.710271+0000","last_active":"2023-09-23T20:38:12.169259+0000","last_peered":"2023-09-23T20:38:12.169259+0000","last_clean":"2023-09-23T20:37:13.317209+0000","last_became_active":"2023-09-23T20:38:02.636987+0000","last_became_peered":"2023-09-23T20:38:02.636987+0000","last_unstale":"2023-09-23T20:38:12.169259+0000","last_undegraded":"2023-09-23T20:38:02.547074+0000","last_fullsized":"2023-09-23T20:38:02.526059+0000","mapping_epoch":403,"log_start":"373'400","ondisk_log_start":"373'400","created":369,"last_epoch_clean":370,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":5766,"log_dups_size":0,"ondisk_log_size":5766,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:15:55.380701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":325124120,"num_objects":4962,"num_object_clones":0,"num_object_copies":14886,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5420,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4962,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":6166,"num_write_kb":356033,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":913,"num_bytes_recovered":59834368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2],"avail_no_missing":["7","2","5"],"object_location_counts":[{"shards":"1,2,7","objects":280},{"shards":"2,5,7","objects":4667},{"shards":"2,7","objects":-147}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.4","version":"90'161","reported_seq":602,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T20:38:05.630046+0000","last_change":"2023-09-23T20:38:05.607009+0000","last_active":"2023-09-23T20:38:05.630046+0000","last_peered":"2023-09-23T20:38:05.630046+0000","last_clean":"2023-09-23T20:38:05.630046+0000","last_became_active":"2023-09-23T20:38:05.606621+0000","last_became_peered":"2023-09-23T20:38:05.606621+0000","last_unstale":"2023-09-23T20:38:05.630046+0000","last_undegraded":"2023-09-23T20:38:05.630046+0000","last_fullsized":"2023-09-23T20:38:05.630046+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:34:54.504554+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:34:54.504554+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:47:23.494026+0000","scrub_duration":0.011176864,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"86'117","reported_seq":641,"reported_epoch":407,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:38:05.584510+0000","last_change":"2023-09-23T20:38:03.549038+0000","last_active":"2023-09-23T20:38:05.584510+0000","last_peered":"2023-09-23T20:38:05.584510+0000","last_clean":"2023-09-23T20:13:53.392235+0000","last_became_active":"2023-09-23T20:38:03.548537+0000","last_became_peered":"2023-09-23T20:38:03.548537+0000","last_unstale":"2023-09-23T20:38:05.584510+0000","last_undegraded":"2023-09-23T20:38:03.531263+0000","last_fullsized":"2023-09-23T20:38:03.528724+0000","mapping_epoch":404,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":394,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:51:10.106959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046891400000000002,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":893,"num_read_kb":922,"num_write":82,"num_write_kb":4193,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[2,7],"avail_no_missing":["2","3","7"],"object_location_counts":[{"shards":"2,3,7","objects":1}],"blocked_by":[],"up_primary":6,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":405,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T20:38:04.565013+0000","last_change":"2023-09-23T20:32:33.277827+0000","last_active":"2023-09-23T20:38:04.565013+0000","last_peered":"2023-09-23T20:38:04.565013+0000","last_clean":"2023-09-23T20:38:04.565013+0000","last_became_active":"2023-09-23T20:32:33.236087+0000","last_became_peered":"2023-09-23T20:32:33.236087+0000","last_unstale":"2023-09-23T20:38:04.565013+0000","last_undegraded":"2023-09-23T20:38:04.565013+0000","last_fullsized":"2023-09-23T20:38:04.565013+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:20:58.176854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":135,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T20:38:05.576998+0000","last_change":"2023-09-23T20:38:02.700897+0000","last_active":"2023-09-23T20:38:05.576998+0000","last_peered":"2023-09-23T20:38:05.576998+0000","last_clean":"2023-09-23T20:38:05.576998+0000","last_became_active":"2023-09-23T20:38:02.700561+0000","last_became_peered":"2023-09-23T20:38:02.700561+0000","last_unstale":"2023-09-23T20:38:05.576998+0000","last_undegraded":"2023-09-23T20:38:05.576998+0000","last_fullsized":"2023-09-23T20:38:05.576998+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:31:11.316211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"90'161","reported_seq":675,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T20:38:05.630221+0000","last_change":"2023-09-23T20:38:05.630221+0000","last_active":"2023-09-23T20:38:05.630221+0000","last_peered":"2023-09-23T20:38:05.630221+0000","last_clean":"2023-09-23T20:38:05.630221+0000","last_became_active":"2023-09-23T20:38:05.629839+0000","last_became_peered":"2023-09-23T20:38:05.629839+0000","last_unstale":"2023-09-23T20:38:05.630221+0000","last_undegraded":"2023-09-23T20:38:05.630221+0000","last_fullsized":"2023-09-23T20:38:05.630221+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:06.487351+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:06.487351+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:25:51.325529+0000","scrub_duration":0.001964379,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":373,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T20:38:04.562354+0000","last_change":"2023-09-23T20:32:33.237234+0000","last_active":"2023-09-23T20:38:04.562354+0000","last_peered":"2023-09-23T20:38:04.562354+0000","last_clean":"2023-09-23T20:38:04.562354+0000","last_became_active":"2023-09-23T20:32:33.236881+0000","last_became_peered":"2023-09-23T20:32:33.236881+0000","last_unstale":"2023-09-23T20:38:04.562354+0000","last_undegraded":"2023-09-23T20:38:04.562354+0000","last_fullsized":"2023-09-23T20:38:04.562354+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:56:17.384831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":262,"reported_epoch":408,"state":"active+clean","last_fresh":"2023-09-23T20:38:12.698059+0000","last_change":"2023-09-23T20:29:45.538314+0000","last_active":"2023-09-23T20:38:12.698059+0000","last_peered":"2023-09-23T20:38:12.698059+0000","last_clean":"2023-09-23T20:38:12.698059+0000","last_became_active":"2023-09-23T20:29:45.536847+0000","last_became_peered":"2023-09-23T20:29:45.536847+0000","last_unstale":"2023-09-23T20:38:12.698059+0000","last_undegraded":"2023-09-23T20:38:12.698059+0000","last_fullsized":"2023-09-23T20:38:12.698059+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:51.948721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"90'161","reported_seq":466,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T20:38:05.602477+0000","last_change":"2023-09-23T20:38:05.602250+0000","last_active":"2023-09-23T20:38:05.602477+0000","last_peered":"2023-09-23T20:38:05.602477+0000","last_clean":"2023-09-23T20:38:05.602477+0000","last_became_active":"2023-09-23T20:38:05.601874+0000","last_became_peered":"2023-09-23T20:38:05.601874+0000","last_unstale":"2023-09-23T20:38:05.602477+0000","last_undegraded":"2023-09-23T20:38:05.602477+0000","last_fullsized":"2023-09-23T20:38:05.602477+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:35:42.864363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:35:42.864363+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":40253,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":356,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T20:38:05.578657+0000","last_change":"2023-09-23T20:38:02.657296+0000","last_active":"2023-09-23T20:38:05.578657+0000","last_peered":"2023-09-23T20:38:05.578657+0000","last_clean":"2023-09-23T20:38:05.578657+0000","last_became_active":"2023-09-23T20:38:02.579619+0000","last_became_peered":"2023-09-23T20:38:02.579619+0000","last_unstale":"2023-09-23T20:38:05.578657+0000","last_undegraded":"2023-09-23T20:38:05.578657+0000","last_fullsized":"2023-09-23T20:38:05.578657+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:32:51.000990+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":64,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T20:38:05.578555+0000","last_change":"2023-09-23T20:34:59.987575+0000","last_active":"2023-09-23T20:38:05.578555+0000","last_peered":"2023-09-23T20:38:05.578555+0000","last_clean":"2023-09-23T20:38:05.578555+0000","last_became_active":"2023-09-23T20:34:59.987229+0000","last_became_peered":"2023-09-23T20:34:59.987229+0000","last_unstale":"2023-09-23T20:38:05.578555+0000","last_undegraded":"2023-09-23T20:38:05.578555+0000","last_fullsized":"2023-09-23T20:38:05.578555+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:34:01.820159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":364,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T20:38:05.577628+0000","last_change":"2023-09-23T20:38:02.641841+0000","last_active":"2023-09-23T20:38:05.577628+0000","last_peered":"2023-09-23T20:38:05.577628+0000","last_clean":"2023-09-23T20:38:05.577628+0000","last_became_active":"2023-09-23T20:38:02.547519+0000","last_became_peered":"2023-09-23T20:38:02.547519+0000","last_unstale":"2023-09-23T20:38:05.577628+0000","last_undegraded":"2023-09-23T20:38:05.577628+0000","last_fullsized":"2023-09-23T20:38:05.577628+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:44:56.981910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":64,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T20:38:05.584802+0000","last_change":"2023-09-23T20:34:59.987416+0000","last_active":"2023-09-23T20:38:05.584802+0000","last_peered":"2023-09-23T20:38:05.584802+0000","last_clean":"2023-09-23T20:38:05.584802+0000","last_became_active":"2023-09-23T20:34:59.987117+0000","last_became_peered":"2023-09-23T20:34:59.987117+0000","last_unstale":"2023-09-23T20:38:05.584802+0000","last_undegraded":"2023-09-23T20:38:05.584802+0000","last_fullsized":"2023-09-23T20:38:05.584802+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:36:16.318623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"90'161","reported_seq":662,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T20:38:05.591992+0000","last_change":"2023-09-23T20:38:05.591992+0000","last_active":"2023-09-23T20:38:05.591992+0000","last_peered":"2023-09-23T20:38:05.591992+0000","last_clean":"2023-09-23T20:38:05.591992+0000","last_became_active":"2023-09-23T20:38:05.591701+0000","last_became_peered":"2023-09-23T20:38:05.591701+0000","last_unstale":"2023-09-23T20:38:05.591992+0000","last_undegraded":"2023-09-23T20:38:05.591992+0000","last_fullsized":"2023-09-23T20:38:05.591992+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:02.384133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:02.384133+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:41.722542+0000","scrub_duration":0.001769098,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":227,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T20:38:05.597807+0000","last_change":"2023-09-23T20:38:02.542719+0000","last_active":"2023-09-23T20:38:05.597807+0000","last_peered":"2023-09-23T20:38:05.597807+0000","last_clean":"2023-09-23T20:38:05.597807+0000","last_became_active":"2023-09-23T20:38:02.542385+0000","last_became_peered":"2023-09-23T20:38:02.542385+0000","last_unstale":"2023-09-23T20:38:05.597807+0000","last_undegraded":"2023-09-23T20:38:05.597807+0000","last_fullsized":"2023-09-23T20:38:05.597807+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:36:23.673757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":194,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T20:38:05.595406+0000","last_change":"2023-09-23T20:37:43.584253+0000","last_active":"2023-09-23T20:38:05.595406+0000","last_peered":"2023-09-23T20:38:05.595406+0000","last_clean":"2023-09-23T20:38:05.595406+0000","last_became_active":"2023-09-23T20:37:43.580069+0000","last_became_peered":"2023-09-23T20:37:43.580069+0000","last_unstale":"2023-09-23T20:38:05.595406+0000","last_undegraded":"2023-09-23T20:38:05.595406+0000","last_fullsized":"2023-09-23T20:38:05.595406+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:39:11.482581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"90'161","reported_seq":499,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T20:38:05.593647+0000","last_change":"2023-09-23T20:36:01.505098+0000","last_active":"2023-09-23T20:38:05.593647+0000","last_peered":"2023-09-23T20:38:05.593647+0000","last_clean":"2023-09-23T20:38:05.593647+0000","last_became_active":"2023-09-23T20:35:26.644707+0000","last_became_peered":"2023-09-23T20:35:26.644707+0000","last_unstale":"2023-09-23T20:38:05.593647+0000","last_undegraded":"2023-09-23T20:38:05.593647+0000","last_fullsized":"2023-09-23T20:38:05.593647+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:01.504973+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:01.504973+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:02:10.880809+0000","scrub_duration":0.00202344,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":435,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T20:38:04.563714+0000","last_change":"2023-09-23T20:38:02.655107+0000","last_active":"2023-09-23T20:38:04.563714+0000","last_peered":"2023-09-23T20:38:04.563714+0000","last_clean":"2023-09-23T20:38:04.563714+0000","last_became_active":"2023-09-23T20:38:02.574184+0000","last_became_peered":"2023-09-23T20:38:02.574184+0000","last_unstale":"2023-09-23T20:38:04.563714+0000","last_undegraded":"2023-09-23T20:38:04.563714+0000","last_fullsized":"2023-09-23T20:38:04.563714+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:59:37.398085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2770,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":125,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T20:38:05.578654+0000","last_change":"2023-09-23T20:38:02.538333+0000","last_active":"2023-09-23T20:38:05.578654+0000","last_peered":"2023-09-23T20:38:05.578654+0000","last_clean":"2023-09-23T20:38:05.578654+0000","last_became_active":"2023-09-23T20:38:02.538008+0000","last_became_peered":"2023-09-23T20:38:02.538008+0000","last_unstale":"2023-09-23T20:38:05.578654+0000","last_undegraded":"2023-09-23T20:38:05.578654+0000","last_fullsized":"2023-09-23T20:38:05.578654+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:02:38.789530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":75,"num_bytes_recovered":17862,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26691,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":5162795032,"num_objects":78779,"num_object_clones":0,"num_object_copies":236337,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":32769,"num_objects_misplaced":9181,"num_objects_unfound":0,"num_objects_dirty":78779,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":99863,"num_write_kb":5716481,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5917,"num_bytes_recovered":387514368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16098668544,"data_stored":16098656328,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":79374,"ondisk_log_size":79374,"up":48,"acting":40,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":351684,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":155,"num_read_kb":290,"num_write":76,"num_write_kb":768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":826220,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2048000,"data_stored":2033792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":92,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":18,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5983,"num_read_kb":13200,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":49,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6082,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":23,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201601,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1944436,"kb_used_data":1605420,"kb_used_omap":0,"kb_used_meta":339008,"kb_avail":91800716,"statfs":{"total":95995035648,"available":94003933184,"internally_reserved":0,"allocated":1643950080,"data_stored":1642676347,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":347144192},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:37:48 2023","interfaces":[{"interface":"back","average":{"1min":2.8460000000000001,"5min":2.9020000000000001,"15min":2.0609999999999999},"min":{"1min":0.26500000000000001,"5min":0.23599999999999999,"15min":0.20799999999999999},"max":{"1min":12.640000000000001,"5min":17.399999999999999,"15min":48.798999999999999},"last":8.8930000000000007},{"interface":"front","average":{"1min":1.0049999999999999,"5min":2.0579999999999998,"15min":1.952},"min":{"1min":0.247,"5min":0.247,"15min":0.24399999999999999},"max":{"1min":4.1959999999999997,"5min":19.355,"15min":48.247},"last":8.8439999999999994}]},{"osd":1,"last update":"Sat Sep 23 20:38:03 2023","interfaces":[{"interface":"back","average":{"1min":1.4530000000000001,"5min":2.5419999999999998,"15min":2.0630000000000002},"min":{"1min":0.32000000000000001,"5min":0.25900000000000001,"15min":0.254},"max":{"1min":5.8399999999999999,"5min":49.965000000000003,"15min":49.965000000000003},"last":4.6029999999999998},{"interface":"front","average":{"1min":1.9570000000000001,"5min":2.859,"15min":2.1699999999999999},"min":{"1min":0.315,"5min":0.29999999999999999,"15min":0.248},"max":{"1min":6.2960000000000003,"5min":53.451999999999998,"15min":53.451999999999998},"last":2.9649999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:37:48 2023","interfaces":[{"interface":"back","average":{"1min":1.46,"5min":2.3879999999999999,"15min":1.992},"min":{"1min":0.30399999999999999,"5min":0.27800000000000002,"15min":0.27600000000000002},"max":{"1min":5.3879999999999999,"5min":12.196999999999999,"15min":13.173999999999999},"last":0.28599999999999998},{"interface":"front","average":{"1min":1.8839999999999999,"5min":2.6960000000000002,"15min":2.2530000000000001},"min":{"1min":0.32500000000000001,"5min":0.27400000000000002,"15min":0.26600000000000001},"max":{"1min":9.1159999999999997,"5min":23.494,"15min":23.494},"last":14.984999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:37:16 2023","interfaces":[{"interface":"back","average":{"1min":2.145,"5min":2.9020000000000001,"15min":2.1880000000000002},"min":{"1min":0.39700000000000002,"5min":0.35199999999999998,"15min":0.26200000000000001},"max":{"1min":7.4370000000000003,"5min":14.789,"15min":17.155999999999999},"last":4.8799999999999999},{"interface":"front","average":{"1min":1.851,"5min":2.948,"15min":2.1230000000000002},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.27600000000000002},"max":{"1min":5.1210000000000004,"5min":14.805,"15min":16.960000000000001},"last":4.9020000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:37:16 2023","interfaces":[{"interface":"back","average":{"1min":1.1839999999999999,"5min":0.90100000000000002,"15min":1.157},"min":{"1min":0.25600000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":6.3449999999999998,"5min":6.3449999999999998,"15min":15.177},"last":0.33600000000000002},{"interface":"front","average":{"1min":1.5640000000000001,"5min":1.145,"15min":1.3320000000000001},"min":{"1min":0.32800000000000001,"5min":0.28399999999999997,"15min":0.27600000000000002},"max":{"1min":6.5339999999999998,"5min":6.5339999999999998,"15min":11.122},"last":0.34599999999999997}]},{"osd":5,"last update":"Sat Sep 23 20:37:16 2023","interfaces":[{"interface":"back","average":{"1min":1.641,"5min":0.98099999999999998,"15min":1.246},"min":{"1min":0.29199999999999998,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":7.1159999999999997,"5min":7.1159999999999997,"15min":13.241},"last":0.378},{"interface":"front","average":{"1min":1.9910000000000001,"5min":0.96099999999999997,"15min":1.3260000000000001},"min":{"1min":0.35199999999999998,"5min":0.31900000000000001,"15min":0.315},"max":{"1min":8.4809999999999999,"5min":8.4809999999999999,"15min":18.831},"last":0.39600000000000002}]},{"osd":6,"last update":"Sat Sep 23 20:37:16 2023","interfaces":[{"interface":"back","average":{"1min":1.6799999999999999,"5min":0.93500000000000005,"15min":1.208},"min":{"1min":0.312,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":6.5540000000000003,"5min":6.5540000000000003,"15min":13.185},"last":0.36699999999999999},{"interface":"front","average":{"1min":1.087,"5min":1.069,"15min":1.1990000000000001},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.29899999999999999},"max":{"1min":4.9610000000000003,"5min":6.2770000000000001,"15min":9.9120000000000008},"last":0.45100000000000001}]}]},{"osd":6,"up_from":47,"seq":201863463241,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1918388,"kb_used_data":1597612,"kb_used_omap":0,"kb_used_meta":320768,"kb_avail":91826764,"statfs":{"total":95995035648,"available":94030606336,"internally_reserved":0,"allocated":1635954688,"data_stored":1634689239,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":328466432},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:37:45 2023","interfaces":[{"interface":"back","average":{"1min":2.6739999999999999,"5min":2.3719999999999999,"15min":1.865},"min":{"1min":0.23300000000000001,"5min":0.22800000000000001,"15min":0.221},"max":{"1min":17.492999999999999,"5min":34.143999999999998,"15min":53.701999999999998},"last":0.51200000000000001},{"interface":"front","average":{"1min":2.4700000000000002,"5min":2.4449999999999998,"15min":2.2000000000000002},"min":{"1min":0.23999999999999999,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":11.058999999999999,"5min":34.197000000000003,"15min":53.875999999999998},"last":2.4950000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:38:04 2023","interfaces":[{"interface":"back","average":{"1min":1.633,"5min":2.165,"15min":2.3929999999999998},"min":{"1min":0.28599999999999998,"5min":0.25600000000000001,"15min":0.249},"max":{"1min":6.0170000000000003,"5min":34.173999999999999,"15min":34.173999999999999},"last":0.27600000000000002},{"interface":"front","average":{"1min":1.379,"5min":2.137,"15min":2.7759999999999998},"min":{"1min":0.30399999999999999,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":5.569,"5min":34.490000000000002,"15min":34.490000000000002},"last":2.3849999999999998}]},{"osd":2,"last update":"Sat Sep 23 20:37:45 2023","interfaces":[{"interface":"back","average":{"1min":1.3560000000000001,"5min":2.4180000000000001,"15min":2.0529999999999999},"min":{"1min":0.40100000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":5.5140000000000002,"5min":34.317999999999998,"15min":34.317999999999998},"last":2.8010000000000002},{"interface":"front","average":{"1min":1.149,"5min":2.3180000000000001,"15min":1.946},"min":{"1min":0.41399999999999998,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":4.5730000000000004,"5min":34.292999999999999,"15min":34.292999999999999},"last":2.8149999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:37:32 2023","interfaces":[{"interface":"back","average":{"1min":2.3239999999999998,"5min":1.9790000000000001,"15min":1.859},"min":{"1min":0.375,"5min":0.29999999999999999,"15min":0.29599999999999999},"max":{"1min":8.9670000000000005,"5min":8.9670000000000005,"15min":9.4239999999999995},"last":2.609},{"interface":"front","average":{"1min":2.1099999999999999,"5min":2.2389999999999999,"15min":1.9710000000000001},"min":{"1min":0.42299999999999999,"5min":0.317,"15min":0.317},"max":{"1min":9.1850000000000005,"5min":12.875999999999999,"15min":12.875999999999999},"last":6.5430000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:38:09 2023","interfaces":[{"interface":"back","average":{"1min":0.73699999999999999,"5min":0.67300000000000004,"15min":0.92000000000000004},"min":{"1min":0.28000000000000003,"5min":0.26900000000000002,"15min":0.251},"max":{"1min":3.1749999999999998,"5min":4.9279999999999999,"15min":10.173},"last":0.30099999999999999},{"interface":"front","average":{"1min":0.93100000000000005,"5min":1.0349999999999999,"15min":1.177},"min":{"1min":0.33000000000000002,"5min":0.29099999999999998,"15min":0.27000000000000002},"max":{"1min":6.335,"5min":8.2720000000000002,"15min":8.3249999999999993},"last":1.5629999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:38:09 2023","interfaces":[{"interface":"back","average":{"1min":0.77200000000000002,"5min":0.79700000000000004,"15min":1.012},"min":{"1min":0.38,"5min":0.31,"15min":0.28699999999999998},"max":{"1min":3.141,"5min":4.8259999999999996,"15min":14.085000000000001},"last":0.48699999999999999},{"interface":"front","average":{"1min":0.91400000000000003,"5min":0.93999999999999995,"15min":1.3839999999999999},"min":{"1min":0.38700000000000001,"5min":0.32000000000000001,"15min":0.30599999999999999},"max":{"1min":4.2969999999999997,"5min":6.7320000000000002,"15min":18.192},"last":0.55500000000000005}]},{"osd":7,"last update":"Sat Sep 23 20:37:32 2023","interfaces":[{"interface":"back","average":{"1min":0.995,"5min":0.87,"15min":1.131},"min":{"1min":0.376,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":4.8209999999999997,"5min":5.0350000000000001,"15min":10.217000000000001},"last":0.44900000000000001},{"interface":"front","average":{"1min":1.554,"5min":1.1080000000000001,"15min":1.333},"min":{"1min":0.39400000000000002,"5min":0.34499999999999997,"15min":0.30299999999999999},"max":{"1min":8.8119999999999994,"5min":8.8119999999999994,"15min":9.7360000000000007},"last":0.46999999999999997}]}]},{"osd":1,"up_from":266,"seq":1142461300839,"num_pgs":61,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2613940,"kb_used_data":2222252,"kb_used_omap":8,"kb_used_meta":391607,"kb_avail":91131212,"statfs":{"total":95995035648,"available":93318361088,"internally_reserved":0,"allocated":2275586048,"data_stored":2273392124,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8206,"internal_metadata":401006578},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:38:01 2023","interfaces":[{"interface":"back","average":{"1min":3.6200000000000001,"5min":2.9550000000000001,"15min":2.6850000000000001},"min":{"1min":0.221,"5min":0.221,"15min":0.218},"max":{"1min":10.991,"5min":14.531000000000001,"15min":52.689999999999998},"last":2.5659999999999998},{"interface":"front","average":{"1min":4.6130000000000004,"5min":3.863,"15min":3.395},"min":{"1min":0.253,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":17.169,"5min":26.588000000000001,"15min":52.662999999999997},"last":14.097}]},{"osd":2,"last update":"Sat Sep 23 20:38:01 2023","interfaces":[{"interface":"back","average":{"1min":4.75,"5min":4.4950000000000001,"15min":4.484},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.20899999999999999},"max":{"1min":32.255000000000003,"5min":32.255000000000003,"15min":32.255000000000003},"last":0.309},{"interface":"front","average":{"1min":4.0910000000000002,"5min":3.052,"15min":2.383},"min":{"1min":0.27700000000000002,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":13.404999999999999,"5min":13.404999999999999,"15min":13.404999999999999},"last":5.8209999999999997}]},{"osd":3,"last update":"Sat Sep 23 20:38:01 2023","interfaces":[{"interface":"back","average":{"1min":3.1059999999999999,"5min":3.3460000000000001,"15min":3.8660000000000001},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":10.935,"5min":15.247999999999999,"15min":80.745999999999995},"last":2.6459999999999999},{"interface":"front","average":{"1min":3.008,"5min":3.2869999999999999,"15min":4.2949999999999999},"min":{"1min":0.36299999999999999,"5min":0.30399999999999999,"15min":0.22700000000000001},"max":{"1min":10.083,"5min":19.315000000000001,"15min":82.988},"last":14.048999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:38:01 2023","interfaces":[{"interface":"back","average":{"1min":2.2879999999999998,"5min":2.7930000000000001,"15min":2.242},"min":{"1min":0.315,"5min":0.315,"15min":0.30299999999999999},"max":{"1min":7.6120000000000001,"5min":24.327999999999999,"15min":24.327999999999999},"last":0.372},{"interface":"front","average":{"1min":1.8680000000000001,"5min":1.8009999999999999,"15min":1.847},"min":{"1min":0.36699999999999999,"5min":0.311,"15min":0.311},"max":{"1min":9.3339999999999996,"5min":11.574999999999999,"15min":11.574999999999999},"last":2.6070000000000002}]},{"osd":5,"last update":"Sat Sep 23 20:38:01 2023","interfaces":[{"interface":"back","average":{"1min":2.27,"5min":2.6869999999999998,"15min":2.536},"min":{"1min":0.41999999999999998,"5min":0.34100000000000003,"15min":0.33200000000000002},"max":{"1min":9.6419999999999995,"5min":33.401000000000003,"15min":33.401000000000003},"last":6.0750000000000002},{"interface":"front","average":{"1min":2.4020000000000001,"5min":3.206,"15min":2.4199999999999999},"min":{"1min":0.46700000000000003,"5min":0.376,"15min":0.36799999999999999},"max":{"1min":7.7199999999999998,"5min":33.417000000000002,"15min":33.417000000000002},"last":0.40000000000000002}]},{"osd":6,"last update":"Sat Sep 23 20:38:01 2023","interfaces":[{"interface":"back","average":{"1min":2.9670000000000001,"5min":2.3700000000000001,"15min":4.9390000000000001},"min":{"1min":0.432,"5min":0.35899999999999999,"15min":0.35699999999999998},"max":{"1min":9.6999999999999993,"5min":9.6999999999999993,"15min":72.682000000000002},"last":6.1459999999999999},{"interface":"front","average":{"1min":2.343,"5min":2.976,"15min":4.6619999999999999},"min":{"1min":0.46200000000000002,"5min":0.41099999999999998,"15min":0.38800000000000001},"max":{"1min":7.6900000000000004,"5min":24.376000000000001,"15min":72.563999999999993},"last":0.503}]},{"osd":7,"last update":"Sat Sep 23 20:38:01 2023","interfaces":[{"interface":"back","average":{"1min":2.4969999999999999,"5min":2.6320000000000001,"15min":2.7770000000000001},"min":{"1min":0.39500000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":9.6720000000000006,"5min":18.437000000000001,"15min":18.437000000000001},"last":6.1189999999999998},{"interface":"front","average":{"1min":2.5470000000000002,"5min":3.0169999999999999,"15min":2.508},"min":{"1min":0.46800000000000003,"5min":0.441,"15min":0.42499999999999999},"max":{"1min":7.6580000000000004,"5min":24.524000000000001,"15min":24.524000000000001},"last":0.53300000000000003}]}]},{"osd":0,"up_from":94,"seq":403726926041,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2611260,"kb_used_data":2040692,"kb_used_omap":7,"kb_used_meta":570552,"kb_avail":91133892,"statfs":{"total":95995035648,"available":93321105408,"internally_reserved":0,"allocated":2089668608,"data_stored":2087636503,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":584245265},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:38:05 2023","interfaces":[{"interface":"back","average":{"1min":1.667,"5min":3.3479999999999999,"15min":3.403},"min":{"1min":0.193,"5min":0.17199999999999999,"15min":0.17199999999999999},"max":{"1min":8.8659999999999997,"5min":42.000999999999998,"15min":42.000999999999998},"last":0.39500000000000002},{"interface":"front","average":{"1min":1.952,"5min":3.5329999999999999,"15min":3.7650000000000001},"min":{"1min":0.20399999999999999,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":5.7830000000000004,"5min":23.036000000000001,"15min":23.036000000000001},"last":0.496}]},{"osd":2,"last update":"Sat Sep 23 20:37:41 2023","interfaces":[{"interface":"back","average":{"1min":5.3470000000000004,"5min":3.3809999999999998,"15min":2.8029999999999999},"min":{"1min":0.22600000000000001,"5min":0.193,"15min":0.185},"max":{"1min":62.726999999999997,"5min":62.726999999999997,"15min":77.448999999999998},"last":2.3199999999999998},{"interface":"front","average":{"1min":5.2919999999999998,"5min":3.5819999999999999,"15min":2.6499999999999999},"min":{"1min":0.247,"5min":0.247,"15min":0.188},"max":{"1min":62.689,"5min":62.689,"15min":81.724999999999994},"last":2.1019999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:37:12 2023","interfaces":[{"interface":"back","average":{"1min":1.4299999999999999,"5min":3.137,"15min":2.7069999999999999},"min":{"1min":0.24399999999999999,"5min":0.24099999999999999,"15min":0.23400000000000001},"max":{"1min":6.875,"5min":22.486000000000001,"15min":81.400999999999996},"last":1.7849999999999999},{"interface":"front","average":{"1min":1.6220000000000001,"5min":3.6230000000000002,"15min":2.867},"min":{"1min":0.21199999999999999,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":8.2899999999999991,"5min":22.867999999999999,"15min":81.137},"last":0.435}]},{"osd":4,"last update":"Sat Sep 23 20:37:41 2023","interfaces":[{"interface":"back","average":{"1min":1.5069999999999999,"5min":2.1640000000000001,"15min":2.0129999999999999},"min":{"1min":0.28899999999999998,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":6.9340000000000002,"5min":12.587999999999999,"15min":23.876999999999999},"last":1.8129999999999999},{"interface":"front","average":{"1min":1.5,"5min":2.387,"15min":1.9039999999999999},"min":{"1min":0.34100000000000003,"5min":0.308,"15min":0.29299999999999998},"max":{"1min":4.3949999999999996,"5min":23.001000000000001,"15min":23.681999999999999},"last":0.46999999999999997}]},{"osd":5,"last update":"Sat Sep 23 20:37:41 2023","interfaces":[{"interface":"back","average":{"1min":1.526,"5min":2.2959999999999998,"15min":2.2269999999999999},"min":{"1min":0.44700000000000001,"5min":0.34399999999999997,"15min":0.34200000000000003},"max":{"1min":4.101,"5min":22.969999999999999,"15min":24.198},"last":0.52800000000000002},{"interface":"front","average":{"1min":2.2639999999999998,"5min":2.5790000000000002,"15min":2.1440000000000001},"min":{"1min":0.44700000000000001,"5min":0.307,"15min":0.25600000000000001},"max":{"1min":11.545999999999999,"5min":22.236000000000001,"15min":22.236000000000001},"last":2.1299999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:37:41 2023","interfaces":[{"interface":"back","average":{"1min":1.6779999999999999,"5min":2.5880000000000001,"15min":2.04},"min":{"1min":0.43099999999999999,"5min":0.32300000000000001,"15min":0.318},"max":{"1min":4.2149999999999999,"5min":22.934999999999999,"15min":24.408000000000001},"last":0.55200000000000005},{"interface":"front","average":{"1min":2.129,"5min":2.5289999999999999,"15min":1.966},"min":{"1min":0.41999999999999998,"5min":0.34100000000000003,"15min":0.29399999999999998},"max":{"1min":11.590999999999999,"5min":22.164999999999999,"15min":25.271000000000001},"last":2.1589999999999998}]},{"osd":7,"last update":"Sat Sep 23 20:37:41 2023","interfaces":[{"interface":"back","average":{"1min":2.0579999999999998,"5min":2.6099999999999999,"15min":2.1659999999999999},"min":{"1min":0.378,"5min":0.36099999999999999,"15min":0.30599999999999999},"max":{"1min":11.617000000000001,"5min":22.206,"15min":25.359000000000002},"last":2.1869999999999998},{"interface":"front","average":{"1min":1.6819999999999999,"5min":2.4900000000000002,"15min":2.4079999999999999},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34499999999999997},"max":{"1min":6.9699999999999998,"5min":22.422999999999998,"15min":24.669},"last":1.75}]}]},{"osd":2,"up_from":104,"seq":446676598989,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2588224,"kb_used_data":1976888,"kb_used_omap":11,"kb_used_meta":611316,"kb_avail":91156928,"statfs":{"total":95995035648,"available":93344694272,"internally_reserved":0,"allocated":2024333312,"data_stored":2022295143,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11878,"internal_metadata":625987994},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:37:54 2023","interfaces":[{"interface":"back","average":{"1min":1.2649999999999999,"5min":3.1869999999999998,"15min":2.673},"min":{"1min":0.26600000000000001,"5min":0.20300000000000001,"15min":0.17999999999999999},"max":{"1min":4.6479999999999997,"5min":28.234999999999999,"15min":82.424999999999997},"last":12.007},{"interface":"front","average":{"1min":2.0219999999999998,"5min":2.8759999999999999,"15min":2.5190000000000001},"min":{"1min":0.251,"5min":0.19,"15min":0.19},"max":{"1min":6.3940000000000001,"5min":14.055,"15min":82.524000000000001},"last":5.8220000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:37:57 2023","interfaces":[{"interface":"back","average":{"1min":3.161,"5min":3.0859999999999999,"15min":3.8380000000000001},"min":{"1min":0.252,"5min":0.246,"15min":0.221},"max":{"1min":19.393999999999998,"5min":19.393999999999998,"15min":19.393999999999998},"last":2.8599999999999999},{"interface":"front","average":{"1min":2.4540000000000002,"5min":3.5049999999999999,"15min":2.964},"min":{"1min":0.28000000000000003,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":11.253,"5min":21.585999999999999,"15min":30.565000000000001},"last":0.46200000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:37:18 2023","interfaces":[{"interface":"back","average":{"1min":1.3009999999999999,"5min":2.79,"15min":2.1120000000000001},"min":{"1min":0.35399999999999998,"5min":0.317,"15min":0.19800000000000001},"max":{"1min":4.0700000000000003,"5min":15.804,"15min":16.331},"last":2.899},{"interface":"front","average":{"1min":1.089,"5min":3.1709999999999998,"15min":2.4489999999999998},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.23899999999999999},"max":{"1min":4.7009999999999996,"5min":15.266,"15min":30.704999999999998},"last":0.42799999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:37:54 2023","interfaces":[{"interface":"back","average":{"1min":1.8120000000000001,"5min":2.028,"15min":1.8660000000000001},"min":{"1min":0.34399999999999997,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":6.2869999999999999,"5min":9.2430000000000003,"15min":16.797999999999998},"last":4.5220000000000002},{"interface":"front","average":{"1min":2.6989999999999998,"5min":2.6920000000000002,"15min":2.0720000000000001},"min":{"1min":0.38600000000000001,"5min":0.30499999999999999,"15min":0.27600000000000002},"max":{"1min":19.440999999999999,"5min":19.440999999999999,"15min":19.440999999999999},"last":2.3290000000000002}]},{"osd":5,"last update":"Sat Sep 23 20:37:54 2023","interfaces":[{"interface":"back","average":{"1min":1.7190000000000001,"5min":2.1120000000000001,"15min":2.3849999999999998},"min":{"1min":0.38600000000000001,"5min":0.34799999999999998,"15min":0.309},"max":{"1min":6.3520000000000003,"5min":9.1799999999999997,"15min":95.393000000000001},"last":4.5970000000000004},{"interface":"front","average":{"1min":2.4159999999999999,"5min":2.7959999999999998,"15min":2.6499999999999999},"min":{"1min":0.41399999999999998,"5min":0.35299999999999998,"15min":0.32300000000000001},"max":{"1min":19.513000000000002,"5min":19.513000000000002,"15min":95.414000000000001},"last":2.3580000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:37:54 2023","interfaces":[{"interface":"back","average":{"1min":2.4500000000000002,"5min":2.6520000000000001,"15min":2.1299999999999999},"min":{"1min":0.434,"5min":0.36599999999999999,"15min":0.32100000000000001},"max":{"1min":19.486000000000001,"5min":19.486000000000001,"15min":19.486000000000001},"last":2.2789999999999999},{"interface":"front","average":{"1min":1.276,"5min":2.23,"15min":2.0169999999999999},"min":{"1min":0.374,"5min":0.318,"15min":0.316},"max":{"1min":4.4029999999999996,"5min":11.523,"15min":30.670999999999999},"last":0.55200000000000005}]},{"osd":7,"last update":"Sat Sep 23 20:37:54 2023","interfaces":[{"interface":"back","average":{"1min":1.3879999999999999,"5min":2.2410000000000001,"15min":2.085},"min":{"1min":0.5,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":4.4500000000000002,"5min":12.734999999999999,"15min":30.741},"last":0.59399999999999997},{"interface":"front","average":{"1min":2.1989999999999998,"5min":2.2450000000000001,"15min":2.145},"min":{"1min":0.443,"5min":0.39800000000000002,"15min":0.38},"max":{"1min":6.3220000000000001,"5min":9.2720000000000002,"15min":16.719000000000001},"last":4.569}]}]},{"osd":3,"up_from":109,"seq":468151435463,"num_pgs":71,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2676216,"kb_used_data":2005424,"kb_used_omap":4,"kb_used_meta":670779,"kb_avail":91068936,"statfs":{"total":95995035648,"available":93254590464,"internally_reserved":0,"allocated":2053554176,"data_stored":2051528686,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4514,"internal_metadata":686878302},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:37:16 2023","interfaces":[{"interface":"back","average":{"1min":2.8650000000000002,"5min":3.2269999999999999,"15min":2.508},"min":{"1min":0.23000000000000001,"5min":0.191,"15min":0.186},"max":{"1min":14.638,"5min":17.129000000000001,"15min":67.305999999999997},"last":7.6479999999999997},{"interface":"front","average":{"1min":2.3690000000000002,"5min":3.048,"15min":2.3410000000000002},"min":{"1min":0.26600000000000001,"5min":0.22600000000000001,"15min":0.19500000000000001},"max":{"1min":10.140000000000001,"5min":12.502000000000001,"15min":67.403999999999996},"last":3.2570000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:37:58 2023","interfaces":[{"interface":"back","average":{"1min":1.3660000000000001,"5min":2.1259999999999999,"15min":2.6240000000000001},"min":{"1min":0.23999999999999999,"5min":0.23799999999999999,"15min":0.222},"max":{"1min":6.4489999999999998,"5min":16.189,"15min":16.189},"last":3.6469999999999998},{"interface":"front","average":{"1min":2.6339999999999999,"5min":2.3759999999999999,"15min":2.8719999999999999},"min":{"1min":0.27300000000000002,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":12.702999999999999,"5min":13.727,"15min":32.603999999999999},"last":0.44600000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:37:16 2023","interfaces":[{"interface":"back","average":{"1min":1.6180000000000001,"5min":3.1779999999999999,"15min":2.399},"min":{"1min":0.251,"5min":0.23799999999999999,"15min":0.22800000000000001},"max":{"1min":10.108000000000001,"5min":32.857999999999997,"15min":32.857999999999997},"last":7.7649999999999997},{"interface":"front","average":{"1min":1.444,"5min":2.6600000000000001,"15min":2.27},"min":{"1min":0.317,"5min":0.251,"15min":0.20899999999999999},"max":{"1min":4.7880000000000003,"5min":24.146999999999998,"15min":24.146999999999998},"last":10.470000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:37:16 2023","interfaces":[{"interface":"back","average":{"1min":1.3759999999999999,"5min":1.6419999999999999,"15min":1.76},"min":{"1min":0.39100000000000001,"5min":0.32200000000000001,"15min":0.29799999999999999},"max":{"1min":9.3160000000000007,"5min":12.151,"15min":46.042999999999999},"last":3.7250000000000001},{"interface":"front","average":{"1min":1.3500000000000001,"5min":2.0499999999999998,"15min":2.1429999999999998},"min":{"1min":0.38,"5min":0.29399999999999998,"15min":0.28100000000000003},"max":{"1min":4.0650000000000004,"5min":14.422000000000001,"15min":46.579999999999998},"last":0.55900000000000005}]},{"osd":5,"last update":"Sat Sep 23 20:37:16 2023","interfaces":[{"interface":"back","average":{"1min":1.629,"5min":1.849,"15min":2.02},"min":{"1min":0.41599999999999998,"5min":0.32200000000000001,"15min":0.311},"max":{"1min":9.4130000000000003,"5min":13.228999999999999,"15min":45.292000000000002},"last":3.7519999999999998},{"interface":"front","average":{"1min":1.6240000000000001,"5min":1.841,"15min":1.873},"min":{"1min":0.34399999999999997,"5min":0.32000000000000001,"15min":0.313},"max":{"1min":6.8630000000000004,"5min":10.300000000000001,"15min":44.679000000000002},"last":0.502}]},{"osd":6,"last update":"Sat Sep 23 20:37:16 2023","interfaces":[{"interface":"back","average":{"1min":1.2490000000000001,"5min":1.7949999999999999,"15min":1.873},"min":{"1min":0.46600000000000003,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":6.6459999999999999,"5min":12.464,"15min":17.524999999999999},"last":0.47599999999999998},{"interface":"front","average":{"1min":1.835,"5min":1.9970000000000001,"15min":1.925},"min":{"1min":0.41699999999999998,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":6.5970000000000004,"5min":14.464,"15min":19.524000000000001},"last":0.58399999999999996}]},{"osd":7,"last update":"Sat Sep 23 20:37:16 2023","interfaces":[{"interface":"back","average":{"1min":1.794,"5min":2.012,"15min":2.0939999999999999},"min":{"1min":0.45700000000000002,"5min":0.36899999999999999,"15min":0.34999999999999998},"max":{"1min":5.9240000000000004,"5min":14.521000000000001,"15min":19.571999999999999},"last":10.417},{"interface":"front","average":{"1min":1.6519999999999999,"5min":1.9099999999999999,"15min":2.0859999999999999},"min":{"1min":0.50600000000000001,"5min":0.42599999999999999,"15min":0.371},"max":{"1min":9.3719999999999999,"5min":12.186,"15min":45.387999999999998},"last":3.694}]}]},{"osd":4,"up_from":30,"seq":128849019217,"num_pgs":64,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1964536,"kb_used_data":1658544,"kb_used_omap":0,"kb_used_meta":305984,"kb_avail":91780616,"statfs":{"total":95995035648,"available":93983350784,"internally_reserved":0,"allocated":1698349056,"data_stored":1697079315,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":313327616},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,1,1,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:37:46 2023","interfaces":[{"interface":"back","average":{"1min":0.85599999999999998,"5min":1.917,"15min":1.667},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.222},"max":{"1min":3.3759999999999999,"5min":24.114999999999998,"15min":24.114999999999998},"last":2.1379999999999999},{"interface":"front","average":{"1min":0.96399999999999997,"5min":2.0779999999999998,"15min":1.8100000000000001},"min":{"1min":0.25,"5min":0.246,"15min":0.246},"max":{"1min":3.6709999999999998,"5min":24.135999999999999,"15min":24.135999999999999},"last":2.1469999999999998}]},{"osd":1,"last update":"Sat Sep 23 20:38:02 2023","interfaces":[{"interface":"back","average":{"1min":1.7929999999999999,"5min":1.9059999999999999,"15min":1.877},"min":{"1min":0.28599999999999998,"5min":0.27600000000000002,"15min":0.27300000000000002},"max":{"1min":18.190999999999999,"5min":18.190999999999999,"15min":18.190999999999999},"last":0.42499999999999999},{"interface":"front","average":{"1min":2.0800000000000001,"5min":1.79,"15min":1.8320000000000001},"min":{"1min":0.29299999999999998,"5min":0.28599999999999998,"15min":0.28199999999999997},"max":{"1min":18.173999999999999,"5min":18.173999999999999,"15min":18.173999999999999},"last":0.40300000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:37:57 2023","interfaces":[{"interface":"back","average":{"1min":2.1110000000000002,"5min":2.0070000000000001,"15min":1.9379999999999999},"min":{"1min":0.374,"5min":0.311,"15min":0.26600000000000001},"max":{"1min":7.75,"5min":10.416,"15min":32.363},"last":0.36299999999999999},{"interface":"front","average":{"1min":2.399,"5min":2.4500000000000002,"15min":2.2549999999999999},"min":{"1min":0.372,"5min":0.30599999999999999,"15min":0.26800000000000002},"max":{"1min":8.2680000000000007,"5min":14.614000000000001,"15min":34.042999999999999},"last":0.28899999999999998}]},{"osd":3,"last update":"Sat Sep 23 20:38:14 2023","interfaces":[{"interface":"back","average":{"1min":1.607,"5min":1.9370000000000001,"15min":2.0950000000000002},"min":{"1min":0.42899999999999999,"5min":0.34899999999999998,"15min":0.29099999999999998},"max":{"1min":7.3170000000000002,"5min":9.5239999999999991,"15min":32.332999999999998},"last":0.73399999999999999},{"interface":"front","average":{"1min":1.9099999999999999,"5min":2.3039999999999998,"15min":2.2189999999999999},"min":{"1min":0.41399999999999998,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":7.2350000000000003,"5min":14.384,"15min":32.399999999999999},"last":1.9870000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:37:38 2023","interfaces":[{"interface":"back","average":{"1min":0.84599999999999997,"5min":0.75,"15min":1.5660000000000001},"min":{"1min":0.308,"5min":0.26400000000000001,"15min":0.247},"max":{"1min":3.5960000000000001,"5min":5.4340000000000002,"15min":82.866},"last":0.32000000000000001},{"interface":"front","average":{"1min":1.157,"5min":0.88900000000000001,"15min":1.49},"min":{"1min":0.36699999999999999,"5min":0.28999999999999998,"15min":0.27800000000000002},"max":{"1min":4.306,"5min":6.8520000000000003,"15min":82.546000000000006},"last":0.33000000000000002}]},{"osd":6,"last update":"Sat Sep 23 20:37:57 2023","interfaces":[{"interface":"back","average":{"1min":0.96899999999999997,"5min":1.1359999999999999,"15min":1.4390000000000001},"min":{"1min":0.33200000000000002,"5min":0.312,"15min":0.27600000000000002},"max":{"1min":4.8730000000000002,"5min":10.178000000000001,"15min":41.371000000000002},"last":0.38100000000000001},{"interface":"front","average":{"1min":1.113,"5min":0.995,"15min":1.327},"min":{"1min":0.374,"5min":0.29899999999999999,"15min":0.27200000000000002},"max":{"1min":7.2850000000000001,"5min":7.2850000000000001,"15min":41.140999999999998},"last":0.35099999999999998}]},{"osd":7,"last update":"Sat Sep 23 20:38:14 2023","interfaces":[{"interface":"back","average":{"1min":0.64800000000000002,"5min":1.204,"15min":1.5309999999999999},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":3.036,"5min":10.548999999999999,"15min":16.427},"last":0.41499999999999998},{"interface":"front","average":{"1min":0.56599999999999995,"5min":0.81599999999999995,"15min":1.321},"min":{"1min":0.33200000000000002,"5min":0.32700000000000001,"15min":0.307},"max":{"1min":1.9350000000000001,"5min":7.2400000000000002,"15min":19.018999999999998},"last":0.39300000000000002}]}]},{"osd":5,"up_from":0,"seq":167503724878,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4722,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1360,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8206,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7156,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3154,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2081030144,"data_stored":2081030144,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2265976832,"data_stored":2265972760,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2006716416,"data_stored":2006712344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2037121024,"data_stored":2037121024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1689780224,"data_stored":1689780224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2763915264,"data_stored":2763915264,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1623982080,"data_stored":1623982080,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1630146560,"data_stored":1630142488,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:38:17.598 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-09-23T20:38:17.598 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd pool set unique_pool_3 pg_num 26 2023-09-23T20:38:17.630 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:38:17.637 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:17.637 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:38:17.738 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:38:17.765 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:17.801 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:38:17.803 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:17.804 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:38:17.872 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T20:38:17.904 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:38:17.932 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:17.970 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:38:17.973 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:17.974 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:38:18.075 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:38:18.105 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:18.129 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:38:18.141 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:38:18.144 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:18.144 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:38:18.245 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:38:18.256 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:18 smithi018 ceph-mon[179647]: from='client.45980 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:38:18.256 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:18 smithi018 ceph-mon[184144]: from='client.45980 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:38:18.341 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:38:18.390 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:18.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:18 smithi149 ceph-mon[170394]: from='client.45980 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:38:18.474 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:38:18.489 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:18.489 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:38:18.590 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:38:18.689 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:38:18.690 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:18.770 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:38:18.785 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:18.786 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:38:18.886 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:38:18.917 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:18.958 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:38:18.964 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:18.964 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:38:19.065 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:38:19.092 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:19.130 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:38:19.134 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:19.134 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:38:19.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:19 smithi149 ceph-mon[170394]: pgmap v990: 119 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 105 active+clean; 4.8 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 500 op/s; 32688/234042 objects degraded (13.967%); 9181/234042 objects misplaced (3.923%); 5.4 MiB/s, 86 objects/s recovering 2023-09-23T20:38:19.236 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:38:19.270 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:19.311 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:38:19.314 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:19.314 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:38:19.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:19 smithi018 ceph-mon[179647]: pgmap v990: 119 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 105 active+clean; 4.8 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 500 op/s; 32688/234042 objects degraded (13.967%); 9181/234042 objects misplaced (3.923%); 5.4 MiB/s, 86 objects/s recovering 2023-09-23T20:38:19.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:19 smithi018 ceph-mon[184144]: pgmap v990: 119 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 105 active+clean; 4.8 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 500 op/s; 32688/234042 objects degraded (13.967%); 9181/234042 objects misplaced (3.923%); 5.4 MiB/s, 86 objects/s recovering 2023-09-23T20:38:19.414 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:38:19.450 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:19.494 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:38:19.497 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:19.497 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:38:19.598 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:38:19.639 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:19.684 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:38:19.689 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:19.690 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:38:19.790 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:38:19.886 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:21.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:21 smithi018 ceph-mon[184144]: pgmap v991: 119 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 105 active+clean; 4.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 474 op/s; 32746/231066 objects degraded (14.172%); 8878/231066 objects misplaced (3.842%); 5.6 MiB/s, 88 objects/s recovering 2023-09-23T20:38:21.394 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:21 smithi018 ceph-mon[179647]: pgmap v991: 119 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 105 active+clean; 4.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 474 op/s; 32746/231066 objects degraded (14.172%); 8878/231066 objects misplaced (3.842%); 5.6 MiB/s, 88 objects/s recovering 2023-09-23T20:38:21.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:21 smithi149 ceph-mon[170394]: pgmap v991: 119 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 105 active+clean; 4.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 474 op/s; 32746/231066 objects degraded (14.172%); 8878/231066 objects misplaced (3.842%); 5.6 MiB/s, 88 objects/s recovering 2023-09-23T20:38:22.117 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:22 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 32746/231066 objects degraded (14.172%), 13 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T20:38:22.185 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:38:22.197 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:22.197 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:38:22.245 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:38:22.249 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:38:22.298 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:38:22.397 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:22 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 32746/231066 objects degraded (14.172%), 13 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T20:38:22.445 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:22.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:22 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 32746/231066 objects degraded (14.172%), 13 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T20:38:22.620 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:38:22.628 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:22.629 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:38:22.730 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:38:22.870 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:23.415 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:38:23.426 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:23.426 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:38:23.527 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:38:23.565 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:23.606 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:38:23.610 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:23 smithi149 ceph-mon[170394]: pgmap v992: 119 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 105 active+clean; 4.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 458 op/s; 32456/228057 objects degraded (14.232%); 8878/228057 objects misplaced (3.893%); 6.3 MiB/s, 100 objects/s recovering 2023-09-23T20:38:23.610 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:23.611 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:38:23.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:23 smithi018 ceph-mon[179647]: pgmap v992: 119 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 105 active+clean; 4.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 458 op/s; 32456/228057 objects degraded (14.232%); 8878/228057 objects misplaced (3.893%); 6.3 MiB/s, 100 objects/s recovering 2023-09-23T20:38:23.643 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:23 smithi018 ceph-mon[184144]: pgmap v992: 119 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 105 active+clean; 4.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 458 op/s; 32456/228057 objects degraded (14.232%); 8878/228057 objects misplaced (3.893%); 6.3 MiB/s, 100 objects/s recovering 2023-09-23T20:38:23.712 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:38:23.820 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2023-09-23T20:38:23.820 INFO:teuthology.orchestra.run.smithi018.stdout: "size": 20, 2023-09-23T20:38:23.820 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 600, 2023-09-23T20:38:23.820 INFO:teuthology.orchestra.run.smithi018.stdout: "ops": [ 2023-09-23T20:38:23.821 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.821 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:57603 7.4 7:280a08d4:::benchmark_data_smithi018_308601_object57602:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e311)", 2023-09-23T20:38:23.821 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:32:32.217458+0000", 2023-09-23T20:38:23.821 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 351.591865547, 2023-09-23T20:38:23.821 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.4379806410000002, 2023-09-23T20:38:23.822 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:38:23.822 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:38:23.822 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:38:23.822 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:38:23.822 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:38:23.823 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 57603 2023-09-23T20:38:23.823 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.823 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:38:23.823 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.823 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:38:23.823 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.217458+0000", 2023-09-23T20:38:23.824 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:38:23.824 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.824 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.824 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:38:23.824 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.217458+0000", 2023-09-23T20:38:23.824 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:38:23.825 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.825 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.825 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:38:23.825 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.217457+0000", 2023-09-23T20:38:23.825 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:38:23.825 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.826 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.826 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:38:23.826 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.217483+0000", 2023-09-23T20:38:23.826 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.5765000000000001e-05 2023-09-23T20:38:23.826 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.827 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.827 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:38:23.827 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.217492+0000", 2023-09-23T20:38:23.827 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.4179999999999997e-06 2023-09-23T20:38:23.827 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.827 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.828 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:38:23.828 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.217499+0000", 2023-09-23T20:38:23.828 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.9210000000000003e-06 2023-09-23T20:38:23.828 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.828 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.828 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:38:23.829 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.221000+0000", 2023-09-23T20:38:23.829 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0035019550000000002 2023-09-23T20:38:23.829 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.829 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.829 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:38:23.830 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.221153+0000", 2023-09-23T20:38:23.830 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00015237900000000001 2023-09-23T20:38:23.830 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.830 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.830 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,6", 2023-09-23T20:38:23.830 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.221299+0000", 2023-09-23T20:38:23.830 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00014614399999999999 2023-09-23T20:38:23.831 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.831 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.831 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:38:23.831 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.223762+0000", 2023-09-23T20:38:23.831 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0024632769999999998 2023-09-23T20:38:23.831 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.832 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.832 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:38:23.832 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.234020+0000", 2023-09-23T20:38:23.832 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.010257943 2023-09-23T20:38:23.832 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.832 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.833 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:38:23.833 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.234037+0000", 2023-09-23T20:38:23.833 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.6432000000000001e-05 2023-09-23T20:38:23.833 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.833 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.833 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:38:23.834 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.213974+0000", 2023-09-23T20:38:23.834 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.97993717700000005 2023-09-23T20:38:23.834 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.834 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.834 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:38:23.835 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.213988+0000", 2023-09-23T20:38:23.835 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.4001e-05 2023-09-23T20:38:23.835 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.835 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.835 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:38:23.835 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.634540+0000", 2023-09-23T20:38:23.835 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.4205522720000001 2023-09-23T20:38:23.836 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.836 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.836 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for degraded object", 2023-09-23T20:38:23.836 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.635049+0000", 2023-09-23T20:38:23.836 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00050854999999999997 2023-09-23T20:38:23.836 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.836 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.837 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:38:23.837 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.655344+0000", 2023-09-23T20:38:23.837 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.020295167999999999 2023-09-23T20:38:23.837 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.837 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.837 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:38:23.837 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.655439+0000", 2023-09-23T20:38:23.837 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.4845000000000004e-05 2023-09-23T20:38:23.838 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:38:23.838 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:38:23.838 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:38:23.838 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.838 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.838 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:57616 7.4 7:26fa0786:::benchmark_data_smithi018_308601_object57615:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e312)", 2023-09-23T20:38:23.838 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:32:32.242374+0000", 2023-09-23T20:38:23.839 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 351.56694994100002, 2023-09-23T20:38:23.839 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.4160429900000002, 2023-09-23T20:38:23.839 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:38:23.839 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:38:23.839 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:38:23.839 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:38:23.839 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:38:23.840 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 57616 2023-09-23T20:38:23.840 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.840 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:38:23.840 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.840 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:38:23.840 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.242374+0000", 2023-09-23T20:38:23.840 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:38:23.841 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.841 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.841 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:38:23.841 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.242374+0000", 2023-09-23T20:38:23.841 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:38:23.841 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.841 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.842 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:38:23.842 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.242372+0000", 2023-09-23T20:38:23.842 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:38:23.842 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.842 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.842 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:38:23.843 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.242398+0000", 2023-09-23T20:38:23.843 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.6188000000000001e-05 2023-09-23T20:38:23.843 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.843 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.843 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:38:23.843 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.242753+0000", 2023-09-23T20:38:23.843 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000354607 2023-09-23T20:38:23.844 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.844 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.844 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:38:23.844 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.242764+0000", 2023-09-23T20:38:23.844 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1443999999999999e-05 2023-09-23T20:38:23.844 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.845 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.845 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:38:23.845 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.242827+0000", 2023-09-23T20:38:23.845 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.3237000000000005e-05 2023-09-23T20:38:23.845 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.845 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.846 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:38:23.846 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.242840+0000", 2023-09-23T20:38:23.846 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3046e-05 2023-09-23T20:38:23.846 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.846 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.846 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:38:23.847 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.214015+0000", 2023-09-23T20:38:23.847 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.97117513 2023-09-23T20:38:23.847 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.847 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.847 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:38:23.847 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.214024+0000", 2023-09-23T20:38:23.848 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.7690000000000002e-06 2023-09-23T20:38:23.848 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.848 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.848 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:38:23.848 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.635115+0000", 2023-09-23T20:38:23.848 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.4210909899999999 2023-09-23T20:38:23.849 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.849 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.849 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:38:23.849 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.635297+0000", 2023-09-23T20:38:23.849 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00018154400000000001 2023-09-23T20:38:23.849 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.850 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.850 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,4", 2023-09-23T20:38:23.850 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.635493+0000", 2023-09-23T20:38:23.850 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00019634200000000001 2023-09-23T20:38:23.850 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.851 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.851 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:38:23.851 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.640427+0000", 2023-09-23T20:38:23.851 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0049342559999999997 2023-09-23T20:38:23.851 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.851 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.851 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:38:23.852 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.655477+0000", 2023-09-23T20:38:23.852 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.015049794 2023-09-23T20:38:23.852 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.852 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.852 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:38:23.852 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.655547+0000", 2023-09-23T20:38:23.853 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.9598999999999996e-05 2023-09-23T20:38:23.853 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.853 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.853 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:38:23.853 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.655597+0000", 2023-09-23T20:38:23.853 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.0489999999999999e-05 2023-09-23T20:38:23.854 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.854 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.854 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:38:23.854 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.658417+0000", 2023-09-23T20:38:23.854 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0028193939999999998 2023-09-23T20:38:23.854 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:38:23.855 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:38:23.855 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:38:23.855 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.855 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.855 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:57660 7.9 7:90d29d10:::benchmark_data_smithi018_308601_object57659:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e312)", 2023-09-23T20:38:23.855 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:32:32.288586+0000", 2023-09-23T20:38:23.855 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 351.52073788299998, 2023-09-23T20:38:23.856 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.3662949609999999, 2023-09-23T20:38:23.856 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:38:23.856 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:38:23.856 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:38:23.856 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:38:23.856 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:38:23.857 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 57660 2023-09-23T20:38:23.857 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.857 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:38:23.857 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.857 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:38:23.857 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.288586+0000", 2023-09-23T20:38:23.858 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:38:23.858 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.858 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.858 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:38:23.858 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.288586+0000", 2023-09-23T20:38:23.858 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:38:23.859 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.859 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.859 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:38:23.859 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.288584+0000", 2023-09-23T20:38:23.859 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:38:23.859 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.859 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.860 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:38:23.860 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.288608+0000", 2023-09-23T20:38:23.860 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.3784000000000001e-05 2023-09-23T20:38:23.860 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.860 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.860 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:38:23.861 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.288644+0000", 2023-09-23T20:38:23.861 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.5930000000000003e-05 2023-09-23T20:38:23.861 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.861 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.861 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:38:23.861 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.288656+0000", 2023-09-23T20:38:23.862 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1977999999999999e-05 2023-09-23T20:38:23.862 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.862 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.862 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:38:23.862 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.290685+0000", 2023-09-23T20:38:23.863 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.002029552 2023-09-23T20:38:23.863 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.863 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.863 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:38:23.863 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.290697+0000", 2023-09-23T20:38:23.863 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2155e-05 2023-09-23T20:38:23.864 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.864 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.864 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:38:23.864 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.219112+0000", 2023-09-23T20:38:23.864 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.92841424500000003 2023-09-23T20:38:23.865 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.865 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.865 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:38:23.865 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.219129+0000", 2023-09-23T20:38:23.865 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.7359000000000001e-05 2023-09-23T20:38:23.865 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.866 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.866 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:38:23.866 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.630777+0000", 2023-09-23T20:38:23.866 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.4116482610000001 2023-09-23T20:38:23.866 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.867 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.867 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:38:23.867 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.630922+0000", 2023-09-23T20:38:23.867 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000144912 2023-09-23T20:38:23.867 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.867 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.868 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,4", 2023-09-23T20:38:23.868 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.631075+0000", 2023-09-23T20:38:23.868 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00015256699999999999 2023-09-23T20:38:23.868 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.868 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.869 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:38:23.869 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.639027+0000", 2023-09-23T20:38:23.869 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0079517019999999997 2023-09-23T20:38:23.869 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.869 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.869 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:38:23.869 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.646083+0000", 2023-09-23T20:38:23.870 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0070564820000000002 2023-09-23T20:38:23.870 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.870 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.870 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:38:23.870 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.651941+0000", 2023-09-23T20:38:23.870 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0058583680000000001 2023-09-23T20:38:23.871 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.871 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.871 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:38:23.871 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.652009+0000", 2023-09-23T20:38:23.871 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.7535999999999998e-05 2023-09-23T20:38:23.871 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.872 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.872 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:38:23.872 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.654881+0000", 2023-09-23T20:38:23.872 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0028717500000000002 2023-09-23T20:38:23.872 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:38:23.873 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:38:23.873 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:38:23.873 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.873 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.873 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:57665 7.4 7:22060733:::benchmark_data_smithi018_308601_object57664:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e312)", 2023-09-23T20:38:23.873 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:32:32.297484+0000", 2023-09-23T20:38:23.874 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 351.51183924600002, 2023-09-23T20:38:23.874 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.3639945290000002, 2023-09-23T20:38:23.874 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:38:23.874 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:38:23.874 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:38:23.874 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:38:23.875 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:38:23.875 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 57665 2023-09-23T20:38:23.875 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.875 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:38:23.875 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.875 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:38:23.876 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.297484+0000", 2023-09-23T20:38:23.876 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:38:23.876 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.876 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.876 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:38:23.877 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.297484+0000", 2023-09-23T20:38:23.877 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:38:23.877 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.877 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.877 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:38:23.877 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.297484+0000", 2023-09-23T20:38:23.878 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.999999 2023-09-23T20:38:23.878 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.878 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.878 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:38:23.878 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.297505+0000", 2023-09-23T20:38:23.878 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.1407e-05 2023-09-23T20:38:23.879 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.879 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.879 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:38:23.879 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.297514+0000", 2023-09-23T20:38:23.879 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.8419999999999994e-06 2023-09-23T20:38:23.879 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.880 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.880 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:38:23.880 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.297520+0000", 2023-09-23T20:38:23.880 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.8200000000000002e-06 2023-09-23T20:38:23.880 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.880 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.881 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:38:23.881 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.297573+0000", 2023-09-23T20:38:23.881 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.3565e-05 2023-09-23T20:38:23.881 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.881 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.881 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:38:23.881 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.297583+0000", 2023-09-23T20:38:23.881 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.5130000000000002e-06 2023-09-23T20:38:23.881 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.881 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.882 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:23.883 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:38:23.883 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.214047+0000", 2023-09-23T20:38:23.883 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.916464218 2023-09-23T20:38:23.883 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.883 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.883 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:38:23.884 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.214055+0000", 2023-09-23T20:38:23.884 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.9710000000000004e-06 2023-09-23T20:38:23.884 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.884 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.884 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:38:23.884 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.637794+0000", 2023-09-23T20:38:23.884 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.4237388100000001 2023-09-23T20:38:23.884 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.884 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.884 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:38:23.884 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.637941+0000", 2023-09-23T20:38:23.885 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00014709200000000001 2023-09-23T20:38:23.885 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.885 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.885 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,4", 2023-09-23T20:38:23.885 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.638110+0000", 2023-09-23T20:38:23.885 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00016960900000000001 2023-09-23T20:38:23.885 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.885 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.885 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:38:23.885 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.640435+0000", 2023-09-23T20:38:23.885 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.002324137 2023-09-23T20:38:23.885 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.886 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.886 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:38:23.886 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.655512+0000", 2023-09-23T20:38:23.886 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.015077337999999999 2023-09-23T20:38:23.886 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.886 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.886 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:38:23.886 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.658496+0000", 2023-09-23T20:38:23.886 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00298446 2023-09-23T20:38:23.886 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.886 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.887 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:38:23.887 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.658545+0000", 2023-09-23T20:38:23.887 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.8789999999999999e-05 2023-09-23T20:38:23.887 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.887 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.887 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:38:23.887 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.661479+0000", 2023-09-23T20:38:23.887 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.002933747 2023-09-23T20:38:23.887 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:38:23.887 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:38:23.887 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:38:23.887 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.888 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.888 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:135798 7.9 7:9c5d0a75:::benchmark_data_smithi018_308601_object41246:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e345)", 2023-09-23T20:38:23.888 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:34:28.925910+0000", 2023-09-23T20:38:23.888 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 234.88341351899999, 2023-09-23T20:38:23.888 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.21189239000000001, 2023-09-23T20:38:23.888 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:38:23.888 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:38:23.888 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:38:23.888 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:38:23.888 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:38:23.888 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 135798 2023-09-23T20:38:23.889 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.889 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:38:23.889 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.889 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:38:23.889 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:28.925910+0000", 2023-09-23T20:38:23.889 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:38:23.889 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.889 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.889 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:38:23.889 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:28.925910+0000", 2023-09-23T20:38:23.889 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:38:23.889 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.890 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.890 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:38:23.890 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:28.925909+0000", 2023-09-23T20:38:23.890 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:38:23.890 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.890 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.890 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:38:23.890 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:28.925913+0000", 2023-09-23T20:38:23.890 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.6830000000000001e-06 2023-09-23T20:38:23.890 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.890 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.891 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:38:23.891 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:28.925921+0000", 2023-09-23T20:38:23.891 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.3590000000000008e-06 2023-09-23T20:38:23.891 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.891 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.891 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:38:23.891 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:28.925927+0000", 2023-09-23T20:38:23.891 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.6799999999999998e-06 2023-09-23T20:38:23.891 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.891 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.891 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:38:23.891 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:28.925982+0000", 2023-09-23T20:38:23.892 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.5687999999999999e-05 2023-09-23T20:38:23.892 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.892 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.892 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:38:23.892 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:28.926097+0000", 2023-09-23T20:38:23.892 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000115173 2023-09-23T20:38:23.892 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.892 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.892 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,7", 2023-09-23T20:38:23.892 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:28.926209+0000", 2023-09-23T20:38:23.892 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00011126800000000001 2023-09-23T20:38:23.893 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.893 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.893 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:38:23.893 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:28.930138+0000", 2023-09-23T20:38:23.893 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0039295650000000003 2023-09-23T20:38:23.893 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.893 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.893 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:38:23.893 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:29.120703+0000", 2023-09-23T20:38:23.893 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.19056478600000001 2023-09-23T20:38:23.893 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.893 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.894 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:38:23.894 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:29.137696+0000", 2023-09-23T20:38:23.894 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.016993438 2023-09-23T20:38:23.894 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.894 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.894 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:38:23.894 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:29.137747+0000", 2023-09-23T20:38:23.894 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.0720999999999998e-05 2023-09-23T20:38:23.894 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.894 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.894 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:38:23.895 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:29.137802+0000", 2023-09-23T20:38:23.895 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.524e-05 2023-09-23T20:38:23.895 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:38:23.895 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:38:23.895 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:38:23.895 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.895 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.895 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:135807 7.9 7:936f4f71:::benchmark_data_smithi018_308601_object41255:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e345)", 2023-09-23T20:38:23.895 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:34:28.931691+0000", 2023-09-23T20:38:23.895 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 234.87763299100001, 2023-09-23T20:38:23.895 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.20622343900000001, 2023-09-23T20:38:23.895 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:38:23.896 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:38:23.896 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:38:23.896 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:38:23.896 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:38:23.896 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 135807 2023-09-23T20:38:23.896 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.896 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:38:23.896 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.896 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:38:23.896 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:28.931691+0000", 2023-09-23T20:38:23.896 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:38:23.896 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.897 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.897 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:38:23.897 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:28.931691+0000", 2023-09-23T20:38:23.897 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:38:23.897 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.897 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.897 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:38:23.897 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:28.931690+0000", 2023-09-23T20:38:23.897 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.999999 2023-09-23T20:38:23.897 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.897 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.897 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:38:23.898 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:28.931695+0000", 2023-09-23T20:38:23.898 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.6180000000000003e-06 2023-09-23T20:38:23.898 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.898 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.898 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:38:23.898 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:28.931704+0000", 2023-09-23T20:38:23.898 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.0100000000000001e-06 2023-09-23T20:38:23.898 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.898 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.898 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:38:23.898 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:28.931710+0000", 2023-09-23T20:38:23.898 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.5289999999999999e-06 2023-09-23T20:38:23.899 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.899 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.899 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:38:23.899 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:28.931766+0000", 2023-09-23T20:38:23.899 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.6552999999999999e-05 2023-09-23T20:38:23.899 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.899 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.899 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:38:23.899 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:28.931866+0000", 2023-09-23T20:38:23.899 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.9159000000000004e-05 2023-09-23T20:38:23.899 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.899 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.900 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,7", 2023-09-23T20:38:23.900 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:29.112327+0000", 2023-09-23T20:38:23.900 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.18046110900000001 2023-09-23T20:38:23.900 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.900 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.900 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:38:23.900 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:29.121128+0000", 2023-09-23T20:38:23.900 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0088009980000000008 2023-09-23T20:38:23.900 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.900 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.900 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:38:23.900 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:29.121324+0000", 2023-09-23T20:38:23.901 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00019586000000000001 2023-09-23T20:38:23.901 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.901 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.901 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:38:23.901 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:29.137842+0000", 2023-09-23T20:38:23.901 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.016518617999999999 2023-09-23T20:38:23.901 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.901 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.901 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:38:23.901 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:29.137879+0000", 2023-09-23T20:38:23.901 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.6402999999999997e-05 2023-09-23T20:38:23.901 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.902 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.902 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:38:23.902 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:29.137914+0000", 2023-09-23T20:38:23.902 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.5503000000000002e-05 2023-09-23T20:38:23.902 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:38:23.902 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:38:23.902 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:38:23.902 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.902 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.902 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:156873 7.4 7:2413b99c:::benchmark_data_smithi018_308601_object62321:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e352)", 2023-09-23T20:38:23.902 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:34:58.961681+0000", 2023-09-23T20:38:23.903 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 204.84764294600001, 2023-09-23T20:38:23.903 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2142744990000001, 2023-09-23T20:38:23.903 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:38:23.903 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:38:23.903 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:38:23.903 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:38:23.903 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:38:23.903 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 156873 2023-09-23T20:38:23.903 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.903 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:38:23.903 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.903 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:38:23.904 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:58.961681+0000", 2023-09-23T20:38:23.904 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:38:23.904 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.904 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.904 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:38:23.904 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:58.961681+0000", 2023-09-23T20:38:23.904 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:38:23.904 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.904 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.904 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:38:23.904 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:58.961679+0000", 2023-09-23T20:38:23.904 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:38:23.905 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.905 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.905 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:38:23.905 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:58.961685+0000", 2023-09-23T20:38:23.905 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.7289999999999997e-06 2023-09-23T20:38:23.905 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.905 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.905 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:38:23.905 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:58.961698+0000", 2023-09-23T20:38:23.905 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3456e-05 2023-09-23T20:38:23.905 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.905 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.906 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:38:23.906 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:58.961707+0000", 2023-09-23T20:38:23.906 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.371e-06 2023-09-23T20:38:23.906 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.906 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.906 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:38:23.906 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:58.961766+0000", 2023-09-23T20:38:23.906 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.8736000000000001e-05 2023-09-23T20:38:23.906 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.906 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.906 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:38:23.906 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:58.961919+0000", 2023-09-23T20:38:23.907 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000153473 2023-09-23T20:38:23.907 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.907 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.907 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,6", 2023-09-23T20:38:23.907 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:58.962066+0000", 2023-09-23T20:38:23.907 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00014647799999999999 2023-09-23T20:38:23.907 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.907 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.907 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:38:23.907 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:58.982292+0000", 2023-09-23T20:38:23.907 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.020226839 2023-09-23T20:38:23.907 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.908 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.908 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:38:23.908 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:58.982307+0000", 2023-09-23T20:38:23.908 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.4323e-05 2023-09-23T20:38:23.908 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.908 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.908 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:38:23.908 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.150194+0000", 2023-09-23T20:38:23.908 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.167887355 2023-09-23T20:38:23.908 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.908 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.909 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for missing object", 2023-09-23T20:38:23.909 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.150339+0000", 2023-09-23T20:38:23.909 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00014458400000000001 2023-09-23T20:38:23.909 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.909 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.909 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:38:23.909 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.175879+0000", 2023-09-23T20:38:23.909 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.025540245999999999 2023-09-23T20:38:23.909 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.909 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.909 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:38:23.909 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.175955+0000", 2023-09-23T20:38:23.910 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.6230999999999996e-05 2023-09-23T20:38:23.910 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:38:23.910 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:38:23.910 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:38:23.910 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.910 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.910 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:156874 7.4 7:2210d725:::benchmark_data_smithi018_308601_object62322:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e352)", 2023-09-23T20:38:23.910 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:34:58.965887+0000", 2023-09-23T20:38:23.910 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 204.84343698000001, 2023-09-23T20:38:23.910 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.22316603, 2023-09-23T20:38:23.910 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:38:23.910 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:38:23.911 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:38:23.911 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:38:23.911 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:38:23.911 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 156874 2023-09-23T20:38:23.911 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.911 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:38:23.911 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.911 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:38:23.911 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:58.965887+0000", 2023-09-23T20:38:23.911 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:38:23.911 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.911 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.912 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:38:23.912 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:58.965887+0000", 2023-09-23T20:38:23.912 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:38:23.912 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.912 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.912 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:38:23.912 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:58.965885+0000", 2023-09-23T20:38:23.912 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:38:23.912 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.912 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.912 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:38:23.912 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:58.965895+0000", 2023-09-23T20:38:23.913 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.9150000000000008e-06 2023-09-23T20:38:23.913 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.913 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.913 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:38:23.913 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:58.965908+0000", 2023-09-23T20:38:23.913 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3349e-05 2023-09-23T20:38:23.913 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.913 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.913 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:38:23.913 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:58.965918+0000", 2023-09-23T20:38:23.913 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.6139999999999998e-06 2023-09-23T20:38:23.913 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.914 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.914 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:38:23.914 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:58.966970+0000", 2023-09-23T20:38:23.914 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.001052545 2023-09-23T20:38:23.914 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.914 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.914 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:38:23.914 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:58.967159+0000", 2023-09-23T20:38:23.914 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000189087 2023-09-23T20:38:23.914 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.914 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.914 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,6", 2023-09-23T20:38:23.915 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:58.967370+0000", 2023-09-23T20:38:23.915 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000210795 2023-09-23T20:38:23.915 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.915 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.915 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:38:23.915 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:58.982341+0000", 2023-09-23T20:38:23.915 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.014970255 2023-09-23T20:38:23.915 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.915 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.915 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:38:23.915 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:58.982350+0000", 2023-09-23T20:38:23.916 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.6029999999999994e-06 2023-09-23T20:38:23.916 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.916 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.916 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:38:23.916 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.151266+0000", 2023-09-23T20:38:23.916 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1689161379999999 2023-09-23T20:38:23.916 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.916 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.916 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for missing object", 2023-09-23T20:38:23.916 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.151368+0000", 2023-09-23T20:38:23.916 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00010191699999999999 2023-09-23T20:38:23.916 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.917 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.917 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:38:23.917 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.188980+0000", 2023-09-23T20:38:23.917 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.037611376000000002 2023-09-23T20:38:23.917 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.917 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.917 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:38:23.917 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.189053+0000", 2023-09-23T20:38:23.917 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.3058000000000004e-05 2023-09-23T20:38:23.917 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:38:23.917 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:38:23.917 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:38:23.918 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.918 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.918 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:156880 7.9 7:97f9ea4b:::benchmark_data_smithi018_308601_object62328:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e353)", 2023-09-23T20:38:23.918 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:34:58.989864+0000", 2023-09-23T20:38:23.918 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 204.819459575, 2023-09-23T20:38:23.918 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.1443719099999998, 2023-09-23T20:38:23.918 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:38:23.918 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:38:23.918 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:38:23.918 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:38:23.918 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:38:23.918 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 156880 2023-09-23T20:38:23.919 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.919 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:38:23.919 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.919 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:38:23.919 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:58.989864+0000", 2023-09-23T20:38:23.919 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:38:23.919 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.919 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.919 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:38:23.919 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:58.989864+0000", 2023-09-23T20:38:23.919 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:38:23.920 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.920 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.920 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:38:23.920 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:58.989862+0000", 2023-09-23T20:38:23.920 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:38:23.920 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.920 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.920 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:38:23.920 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:58.989870+0000", 2023-09-23T20:38:23.920 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.7959999999999996e-06 2023-09-23T20:38:23.920 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.920 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.921 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:38:23.921 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:58.989888+0000", 2023-09-23T20:38:23.921 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.8008000000000001e-05 2023-09-23T20:38:23.921 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.921 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.921 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:38:23.921 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:58.989898+0000", 2023-09-23T20:38:23.921 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.4930000000000004e-06 2023-09-23T20:38:23.921 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.921 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.921 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:38:23.921 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:58.994377+0000", 2023-09-23T20:38:23.922 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0044797750000000001 2023-09-23T20:38:23.922 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.922 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.922 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:38:23.922 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:58.994389+0000", 2023-09-23T20:38:23.922 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1224e-05 2023-09-23T20:38:23.922 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.922 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.922 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:38:23.922 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.969523+0000", 2023-09-23T20:38:23.922 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.97513417700000005 2023-09-23T20:38:23.923 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.923 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.923 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:38:23.923 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.969537+0000", 2023-09-23T20:38:23.923 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.4653e-05 2023-09-23T20:38:23.923 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.923 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.923 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:38:23.923 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.107362+0000", 2023-09-23T20:38:23.923 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.13782403 2023-09-23T20:38:23.923 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.923 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.924 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:38:23.924 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.107518+0000", 2023-09-23T20:38:23.924 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00015667200000000001 2023-09-23T20:38:23.924 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.924 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.924 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,4,7", 2023-09-23T20:38:23.924 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.107667+0000", 2023-09-23T20:38:23.924 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00014836800000000001 2023-09-23T20:38:23.924 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.924 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.924 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:38:23.924 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.111586+0000", 2023-09-23T20:38:23.925 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0039193520000000001 2023-09-23T20:38:23.925 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.925 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.925 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:38:23.925 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.111613+0000", 2023-09-23T20:38:23.925 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.6597999999999998e-05 2023-09-23T20:38:23.925 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.925 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.925 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:38:23.925 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.115895+0000", 2023-09-23T20:38:23.925 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.004282526 2023-09-23T20:38:23.925 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.926 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.926 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:38:23.926 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.134027+0000", 2023-09-23T20:38:23.926 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.018131807 2023-09-23T20:38:23.926 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.926 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.926 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:38:23.926 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.134094+0000", 2023-09-23T20:38:23.926 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.7551000000000004e-05 2023-09-23T20:38:23.926 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.926 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.927 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:38:23.927 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.134236+0000", 2023-09-23T20:38:23.927 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00014152500000000001 2023-09-23T20:38:23.927 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:38:23.927 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:38:23.927 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:38:23.927 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.927 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.927 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:156884 7.9 7:90c28959:::benchmark_data_smithi018_308601_object62332:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e353)", 2023-09-23T20:38:23.927 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:34:58.995119+0000", 2023-09-23T20:38:23.927 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 204.814204483, 2023-09-23T20:38:23.927 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.146166928, 2023-09-23T20:38:23.928 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:38:23.928 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:38:23.928 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:38:23.928 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:38:23.928 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:38:23.928 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 156884 2023-09-23T20:38:23.928 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.928 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:38:23.928 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.928 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:38:23.928 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:58.995119+0000", 2023-09-23T20:38:23.928 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:38:23.929 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.929 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.929 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:38:23.929 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:58.995119+0000", 2023-09-23T20:38:23.929 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:38:23.929 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.929 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.929 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:38:23.929 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:58.995118+0000", 2023-09-23T20:38:23.929 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.999999 2023-09-23T20:38:23.929 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.929 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.930 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:38:23.930 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:58.995124+0000", 2023-09-23T20:38:23.930 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.4960000000000004e-06 2023-09-23T20:38:23.930 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.930 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.930 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:38:23.930 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:58.995138+0000", 2023-09-23T20:38:23.930 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.4413000000000001e-05 2023-09-23T20:38:23.930 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.930 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.930 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:38:23.931 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:58.995146+0000", 2023-09-23T20:38:23.931 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.0150000000000003e-06 2023-09-23T20:38:23.931 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.931 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.931 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:38:23.931 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:58.995225+0000", 2023-09-23T20:38:23.931 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.9101e-05 2023-09-23T20:38:23.931 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.931 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.931 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:38:23.931 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:58.995235+0000", 2023-09-23T20:38:23.931 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.9699999999999994e-06 2023-09-23T20:38:23.932 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.932 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.932 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:38:23.932 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.969600+0000", 2023-09-23T20:38:23.932 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.97436495300000003 2023-09-23T20:38:23.932 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.932 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.932 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:38:23.932 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.969609+0000", 2023-09-23T20:38:23.932 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.5699999999999993e-06 2023-09-23T20:38:23.932 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.932 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.933 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:38:23.933 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.111637+0000", 2023-09-23T20:38:23.933 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.142028147 2023-09-23T20:38:23.933 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.933 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.933 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:38:23.933 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.111784+0000", 2023-09-23T20:38:23.933 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000146883 2023-09-23T20:38:23.933 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.933 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.933 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,4,7", 2023-09-23T20:38:23.934 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.111947+0000", 2023-09-23T20:38:23.934 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00016295400000000001 2023-09-23T20:38:23.934 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.934 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.934 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:38:23.934 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.119811+0000", 2023-09-23T20:38:23.934 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0078639850000000004 2023-09-23T20:38:23.934 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.934 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.934 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:38:23.934 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.119872+0000", 2023-09-23T20:38:23.934 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.1822000000000005e-05 2023-09-23T20:38:23.935 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.935 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.935 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:38:23.935 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.119937+0000", 2023-09-23T20:38:23.935 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.5036000000000005e-05 2023-09-23T20:38:23.935 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.935 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.935 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:38:23.935 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.141091+0000", 2023-09-23T20:38:23.935 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.021153885000000001 2023-09-23T20:38:23.935 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.935 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.936 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:38:23.936 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.141158+0000", 2023-09-23T20:38:23.936 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.6433000000000006e-05 2023-09-23T20:38:23.936 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.936 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.936 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:38:23.936 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.141286+0000", 2023-09-23T20:38:23.936 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00012822300000000001 2023-09-23T20:38:23.936 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:38:23.936 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:38:23.936 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:38:23.936 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.937 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.937 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:156891 7.9 7:900a49cc:::benchmark_data_smithi018_308601_object62339:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e353)", 2023-09-23T20:38:23.937 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:34:59.007075+0000", 2023-09-23T20:38:23.937 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 204.80224856800001, 2023-09-23T20:38:23.937 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.1390987259999998, 2023-09-23T20:38:23.937 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:38:23.937 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:38:23.937 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:38:23.937 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:38:23.937 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:38:23.937 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 156891 2023-09-23T20:38:23.937 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.938 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:38:23.938 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.938 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:38:23.938 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.007075+0000", 2023-09-23T20:38:23.938 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:38:23.938 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.938 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.938 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:38:23.938 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.007075+0000", 2023-09-23T20:38:23.938 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:38:23.938 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.939 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.939 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:38:23.939 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.007073+0000", 2023-09-23T20:38:23.939 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:38:23.939 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.939 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.939 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:38:23.939 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.007079+0000", 2023-09-23T20:38:23.939 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.9619999999999998e-06 2023-09-23T20:38:23.939 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.939 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.939 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:38:23.940 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.007090+0000", 2023-09-23T20:38:23.940 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0694e-05 2023-09-23T20:38:23.940 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.940 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.940 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:38:23.940 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.007097+0000", 2023-09-23T20:38:23.940 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.6729999999999992e-06 2023-09-23T20:38:23.940 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.940 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.940 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:38:23.940 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.012873+0000", 2023-09-23T20:38:23.940 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0057759999999999999 2023-09-23T20:38:23.941 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.941 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.941 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:38:23.941 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.012890+0000", 2023-09-23T20:38:23.941 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.6566999999999999e-05 2023-09-23T20:38:23.941 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.941 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.941 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:38:23.941 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.969636+0000", 2023-09-23T20:38:23.941 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.95674641400000004 2023-09-23T20:38:23.941 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.941 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.942 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:38:23.942 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.969644+0000", 2023-09-23T20:38:23.942 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.2989999999999999e-06 2023-09-23T20:38:23.942 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.942 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.942 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:38:23.942 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.115914+0000", 2023-09-23T20:38:23.942 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1462701470000001 2023-09-23T20:38:23.942 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.942 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.942 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:38:23.942 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.116062+0000", 2023-09-23T20:38:23.943 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00014790800000000001 2023-09-23T20:38:23.943 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.943 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.943 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,4,7", 2023-09-23T20:38:23.943 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.116218+0000", 2023-09-23T20:38:23.943 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000156725 2023-09-23T20:38:23.943 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.943 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.943 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:38:23.943 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.119974+0000", 2023-09-23T20:38:23.943 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0037559519999999999 2023-09-23T20:38:23.944 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.944 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.944 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:38:23.944 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.120032+0000", 2023-09-23T20:38:23.944 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.7485999999999998e-05 2023-09-23T20:38:23.944 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.944 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.944 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:38:23.944 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.126273+0000", 2023-09-23T20:38:23.944 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0062413069999999998 2023-09-23T20:38:23.944 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.944 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.945 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:38:23.945 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.146012+0000", 2023-09-23T20:38:23.945 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.019739014999999999 2023-09-23T20:38:23.945 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.945 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.945 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:38:23.945 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.146065+0000", 2023-09-23T20:38:23.945 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.2939000000000003e-05 2023-09-23T20:38:23.945 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.945 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.945 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:38:23.945 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.146174+0000", 2023-09-23T20:38:23.946 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000108713 2023-09-23T20:38:23.946 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:38:23.946 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:38:23.946 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:38:23.946 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.946 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.946 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:156904 7.4 7:256a86e5:::benchmark_data_smithi018_308601_object62352:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e353)", 2023-09-23T20:38:23.946 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:34:59.030138+0000", 2023-09-23T20:38:23.946 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 204.779185114, 2023-09-23T20:38:23.946 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1549561939999999, 2023-09-23T20:38:23.946 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:38:23.947 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:38:23.947 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:38:23.947 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:38:23.947 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:38:23.947 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 156904 2023-09-23T20:38:23.947 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.947 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:38:23.947 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.947 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:38:23.947 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.030138+0000", 2023-09-23T20:38:23.947 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:38:23.947 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.948 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.948 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:38:23.948 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.030138+0000", 2023-09-23T20:38:23.948 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:38:23.948 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.948 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.948 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:38:23.948 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.030138+0000", 2023-09-23T20:38:23.948 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:38:23.948 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.948 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.948 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:38:23.949 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.030141+0000", 2023-09-23T20:38:23.949 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.8260000000000001e-06 2023-09-23T20:38:23.949 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.949 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.949 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:38:23.949 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.030147+0000", 2023-09-23T20:38:23.949 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.3749999999999999e-06 2023-09-23T20:38:23.949 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.949 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.949 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:38:23.949 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.030152+0000", 2023-09-23T20:38:23.949 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.0340000000000004e-06 2023-09-23T20:38:23.950 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.950 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.950 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:38:23.950 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.078211+0000", 2023-09-23T20:38:23.950 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.048058233999999998 2023-09-23T20:38:23.950 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.950 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.950 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:38:23.950 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.078229+0000", 2023-09-23T20:38:23.950 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.8342000000000002e-05 2023-09-23T20:38:23.950 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.950 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.951 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:38:23.951 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.151421+0000", 2023-09-23T20:38:23.951 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0731922810000001 2023-09-23T20:38:23.951 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.951 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.951 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:38:23.951 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.151535+0000", 2023-09-23T20:38:23.951 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000113695 2023-09-23T20:38:23.951 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.951 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.951 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,4", 2023-09-23T20:38:23.952 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.151626+0000", 2023-09-23T20:38:23.952 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.0983999999999995e-05 2023-09-23T20:38:23.952 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.952 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.952 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:38:23.952 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.158726+0000", 2023-09-23T20:38:23.952 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0070999100000000001 2023-09-23T20:38:23.952 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.952 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.952 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:38:23.952 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.159871+0000", 2023-09-23T20:38:23.952 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.001145177 2023-09-23T20:38:23.953 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.953 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.953 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:38:23.953 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.183778+0000", 2023-09-23T20:38:23.953 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.023906849000000001 2023-09-23T20:38:23.953 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.953 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.953 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:38:23.953 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.183826+0000", 2023-09-23T20:38:23.953 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.8149999999999998e-05 2023-09-23T20:38:23.953 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.954 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.954 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:38:23.954 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.185095+0000", 2023-09-23T20:38:23.954 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0012687359999999999 2023-09-23T20:38:23.954 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:38:23.954 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:38:23.954 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:38:23.954 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.954 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.954 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:156909 7.9 7:9eb097ac:::benchmark_data_smithi018_308601_object62357:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e353)", 2023-09-23T20:38:23.954 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:34:59.038642+0000", 2023-09-23T20:38:23.954 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 204.77068113499999, 2023-09-23T20:38:23.955 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.1077584620000001, 2023-09-23T20:38:23.955 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:38:23.955 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:38:23.955 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:38:23.955 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:38:23.955 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:38:23.955 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 156909 2023-09-23T20:38:23.955 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.955 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:38:23.955 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.955 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:38:23.955 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.038642+0000", 2023-09-23T20:38:23.956 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:38:23.956 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.956 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.956 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:38:23.956 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.038642+0000", 2023-09-23T20:38:23.956 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:38:23.956 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.956 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.956 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:38:23.956 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.038641+0000", 2023-09-23T20:38:23.956 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:38:23.956 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.957 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.957 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:38:23.957 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.038649+0000", 2023-09-23T20:38:23.957 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.7000000000000008e-06 2023-09-23T20:38:23.957 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.957 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.957 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:38:23.957 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.038665+0000", 2023-09-23T20:38:23.957 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.6545000000000002e-05 2023-09-23T20:38:23.957 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.957 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.958 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:38:23.958 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.038674+0000", 2023-09-23T20:38:23.958 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.1440000000000003e-06 2023-09-23T20:38:23.958 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.958 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.958 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:38:23.958 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.038726+0000", 2023-09-23T20:38:23.958 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.2135000000000001e-05 2023-09-23T20:38:23.958 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.958 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.958 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:38:23.958 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.038737+0000", 2023-09-23T20:38:23.959 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1823e-05 2023-09-23T20:38:23.959 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.959 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.959 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:38:23.959 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.969669+0000", 2023-09-23T20:38:23.959 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.93093119400000002 2023-09-23T20:38:23.959 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.959 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.959 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:38:23.959 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.969676+0000", 2023-09-23T20:38:23.959 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.9480000000000002e-06 2023-09-23T20:38:23.959 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.960 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.960 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:38:23.960 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.120069+0000", 2023-09-23T20:38:23.960 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1503932180000001 2023-09-23T20:38:23.960 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.960 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.960 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:38:23.960 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.120165+0000", 2023-09-23T20:38:23.960 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.5729000000000005e-05 2023-09-23T20:38:23.960 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.960 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.961 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,4,7", 2023-09-23T20:38:23.961 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.120247+0000", 2023-09-23T20:38:23.961 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.2736000000000002e-05 2023-09-23T20:38:23.961 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.961 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.961 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:38:23.961 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.126258+0000", 2023-09-23T20:38:23.961 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0060109810000000003 2023-09-23T20:38:23.961 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.961 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.961 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:38:23.961 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.126315+0000", 2023-09-23T20:38:23.962 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.6663999999999998e-05 2023-09-23T20:38:23.962 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.962 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.962 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:38:23.962 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.126326+0000", 2023-09-23T20:38:23.962 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0812000000000001e-05 2023-09-23T20:38:23.962 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.962 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.962 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:38:23.962 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.146229+0000", 2023-09-23T20:38:23.962 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.019902835000000001 2023-09-23T20:38:23.963 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.963 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.963 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:38:23.963 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.146283+0000", 2023-09-23T20:38:23.963 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.3922000000000003e-05 2023-09-23T20:38:23.963 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.963 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.963 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:38:23.963 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.146401+0000", 2023-09-23T20:38:23.963 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000118396 2023-09-23T20:38:23.963 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:38:23.963 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:38:23.964 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:38:23.964 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.964 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.964 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:156910 7.4 7:20d2ed44:::benchmark_data_smithi018_308601_object62358:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e353)", 2023-09-23T20:38:23.964 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:34:59.038685+0000", 2023-09-23T20:38:23.964 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 204.77063834099999, 2023-09-23T20:38:23.964 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1477232370000001, 2023-09-23T20:38:23.964 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:38:23.964 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:38:23.964 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:38:23.964 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:38:23.965 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:38:23.965 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 156910 2023-09-23T20:38:23.965 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.965 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:38:23.965 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.965 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:38:23.965 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.038685+0000", 2023-09-23T20:38:23.965 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:38:23.965 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.965 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.965 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:38:23.965 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.038685+0000", 2023-09-23T20:38:23.966 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:38:23.966 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.966 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.966 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:38:23.966 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.038685+0000", 2023-09-23T20:38:23.966 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:38:23.966 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.966 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.966 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:38:23.966 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.038687+0000", 2023-09-23T20:38:23.966 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.5620000000000002e-06 2023-09-23T20:38:23.967 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.967 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.967 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:38:23.967 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.038692+0000", 2023-09-23T20:38:23.967 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.5560000000000001e-06 2023-09-23T20:38:23.967 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.967 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.967 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:38:23.967 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.038696+0000", 2023-09-23T20:38:23.967 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.6040000000000001e-06 2023-09-23T20:38:23.967 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.967 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.968 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:38:23.968 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.078252+0000", 2023-09-23T20:38:23.968 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.039556674999999999 2023-09-23T20:38:23.968 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.968 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.968 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:38:23.968 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.078261+0000", 2023-09-23T20:38:23.968 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.3999999999999992e-06 2023-09-23T20:38:23.968 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.968 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.968 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:38:23.968 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.154318+0000", 2023-09-23T20:38:23.969 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.076057134 2023-09-23T20:38:23.969 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.969 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.969 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:38:23.969 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.154417+0000", 2023-09-23T20:38:23.969 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.9007999999999998e-05 2023-09-23T20:38:23.969 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.969 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.969 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,4", 2023-09-23T20:38:23.969 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.154500+0000", 2023-09-23T20:38:23.969 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.3410000000000003e-05 2023-09-23T20:38:23.969 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.970 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.970 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:38:23.970 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.158731+0000", 2023-09-23T20:38:23.970 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0042308340000000002 2023-09-23T20:38:23.970 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.970 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.970 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:38:23.970 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.159915+0000", 2023-09-23T20:38:23.970 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0011838739999999999 2023-09-23T20:38:23.970 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.970 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.970 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:38:23.971 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.185179+0000", 2023-09-23T20:38:23.971 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.025264134000000001 2023-09-23T20:38:23.971 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.971 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.971 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:38:23.971 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.185248+0000", 2023-09-23T20:38:23.971 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.8959000000000002e-05 2023-09-23T20:38:23.971 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.971 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.971 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:38:23.971 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.186408+0000", 2023-09-23T20:38:23.971 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.001160424 2023-09-23T20:38:23.972 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:38:23.972 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:38:23.972 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:38:23.972 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.972 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.972 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:156918 7.9 7:969f6c77:::benchmark_data_smithi018_308601_object62366:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e353)", 2023-09-23T20:38:23.972 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:34:59.046909+0000", 2023-09-23T20:38:23.972 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 204.76241491900001, 2023-09-23T20:38:23.972 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.0997079030000001, 2023-09-23T20:38:23.972 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:38:23.972 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:38:23.972 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:38:23.973 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:38:23.973 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:38:23.973 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 156918 2023-09-23T20:38:23.973 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.973 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:38:23.973 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.973 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:38:23.973 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.046909+0000", 2023-09-23T20:38:23.973 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:38:23.973 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.973 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.973 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:38:23.974 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.046909+0000", 2023-09-23T20:38:23.974 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:38:23.974 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.974 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.974 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:38:23.974 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.046907+0000", 2023-09-23T20:38:23.974 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:38:23.974 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.974 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.974 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:38:23.974 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.046914+0000", 2023-09-23T20:38:23.975 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.657e-06 2023-09-23T20:38:23.975 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.975 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.975 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:38:23.975 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.046929+0000", 2023-09-23T20:38:23.975 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.4713e-05 2023-09-23T20:38:23.975 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.975 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.975 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:38:23.975 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.046936+0000", 2023-09-23T20:38:23.975 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.5900000000000002e-06 2023-09-23T20:38:23.975 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.976 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.976 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:38:23.976 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.046998+0000", 2023-09-23T20:38:23.976 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.1655000000000005e-05 2023-09-23T20:38:23.976 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.976 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.976 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:38:23.976 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.047008+0000", 2023-09-23T20:38:23.976 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0223e-05 2023-09-23T20:38:23.976 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.976 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.976 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:38:23.977 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.969700+0000", 2023-09-23T20:38:23.977 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.92269210300000004 2023-09-23T20:38:23.977 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.977 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.977 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:38:23.977 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.969707+0000", 2023-09-23T20:38:23.977 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.861e-06 2023-09-23T20:38:23.977 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.977 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.977 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:38:23.977 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.123042+0000", 2023-09-23T20:38:23.977 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.153334992 2023-09-23T20:38:23.978 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.978 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.978 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:38:23.978 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.123185+0000", 2023-09-23T20:38:23.978 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00014291400000000001 2023-09-23T20:38:23.978 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.978 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.978 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,4,7", 2023-09-23T20:38:23.978 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.123329+0000", 2023-09-23T20:38:23.978 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000144344 2023-09-23T20:38:23.978 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.979 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.979 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:38:23.979 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.126389+0000", 2023-09-23T20:38:23.979 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0030597329999999998 2023-09-23T20:38:23.979 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.979 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.979 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:38:23.979 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.130226+0000", 2023-09-23T20:38:23.979 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0038367789999999998 2023-09-23T20:38:23.979 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.979 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.979 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:38:23.980 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.133972+0000", 2023-09-23T20:38:23.980 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0037460340000000001 2023-09-23T20:38:23.980 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.980 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.980 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:38:23.980 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.146449+0000", 2023-09-23T20:38:23.980 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.012477626 2023-09-23T20:38:23.980 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.980 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.980 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:38:23.980 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.146505+0000", 2023-09-23T20:38:23.981 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.5926e-05 2023-09-23T20:38:23.981 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.981 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.981 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:38:23.981 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.146617+0000", 2023-09-23T20:38:23.981 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00011121899999999999 2023-09-23T20:38:23.981 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:38:23.981 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:38:23.981 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:38:23.981 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.981 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.981 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:156927 7.4 7:2c3ea5a8:::benchmark_data_smithi018_308601_object62375:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e353)", 2023-09-23T20:38:23.982 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:34:59.064897+0000", 2023-09-23T20:38:23.982 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 204.74442693200001, 2023-09-23T20:38:23.982 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.12259446, 2023-09-23T20:38:23.982 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:38:23.982 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:38:23.982 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:38:23.982 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:38:23.982 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:38:23.982 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 156927 2023-09-23T20:38:23.982 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.982 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:38:23.982 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.983 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:38:23.983 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.064897+0000", 2023-09-23T20:38:23.983 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:38:23.983 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.983 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.983 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:38:23.983 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.064897+0000", 2023-09-23T20:38:23.983 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:38:23.983 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.983 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.983 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:38:23.984 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.064895+0000", 2023-09-23T20:38:23.984 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:38:23.984 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.984 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.984 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:38:23.984 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.064903+0000", 2023-09-23T20:38:23.984 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.2300000000000008e-06 2023-09-23T20:38:23.984 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.984 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.984 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:38:23.984 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.064916+0000", 2023-09-23T20:38:23.984 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3355e-05 2023-09-23T20:38:23.985 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.985 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.985 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:38:23.985 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.064925+0000", 2023-09-23T20:38:23.985 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.8799999999999997e-06 2023-09-23T20:38:23.985 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.985 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.985 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:38:23.985 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.078285+0000", 2023-09-23T20:38:23.985 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.013359861000000001 2023-09-23T20:38:23.985 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.986 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.986 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:38:23.986 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.078293+0000", 2023-09-23T20:38:23.986 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.1489999999999994e-06 2023-09-23T20:38:23.986 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.986 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.986 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:38:23.986 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.156712+0000", 2023-09-23T20:38:23.986 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0784186039999999 2023-09-23T20:38:23.986 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.986 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.987 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:38:23.987 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.156823+0000", 2023-09-23T20:38:23.987 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000111264 2023-09-23T20:38:23.987 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.987 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.987 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,4", 2023-09-23T20:38:23.987 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.156902+0000", 2023-09-23T20:38:23.987 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.9296999999999997e-05 2023-09-23T20:38:23.987 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.987 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.987 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:38:23.987 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.159936+0000", 2023-09-23T20:38:23.988 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.003034003 2023-09-23T20:38:23.988 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.988 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.988 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:38:23.988 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.160017+0000", 2023-09-23T20:38:23.988 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.0453999999999999e-05 2023-09-23T20:38:23.988 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.988 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.988 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:38:23.988 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.186490+0000", 2023-09-23T20:38:23.988 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.026473625000000001 2023-09-23T20:38:23.988 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.989 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.989 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:38:23.989 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.186551+0000", 2023-09-23T20:38:23.989 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.0473000000000002e-05 2023-09-23T20:38:23.989 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.989 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.989 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:38:23.989 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.187491+0000", 2023-09-23T20:38:23.989 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000940229 2023-09-23T20:38:23.989 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:38:23.989 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:38:23.990 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:38:23.990 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.990 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.990 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:156946 7.9 7:9c9b215c:::benchmark_data_smithi018_308601_object62394:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e353)", 2023-09-23T20:38:23.990 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:34:59.100567+0000", 2023-09-23T20:38:23.990 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 204.708756663, 2023-09-23T20:38:23.990 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.0462718, 2023-09-23T20:38:23.990 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:38:23.990 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:38:23.990 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:38:23.990 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:38:23.991 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:38:23.991 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 156946 2023-09-23T20:38:23.991 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.991 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:38:23.991 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.991 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:38:23.991 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.100567+0000", 2023-09-23T20:38:23.991 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:38:23.991 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.991 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.991 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:38:23.991 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.100567+0000", 2023-09-23T20:38:23.992 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:38:23.992 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.992 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.992 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:38:23.992 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.100566+0000", 2023-09-23T20:38:23.992 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:38:23.992 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.992 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.992 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:38:23.992 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.100571+0000", 2023-09-23T20:38:23.992 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.3759999999999999e-06 2023-09-23T20:38:23.993 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.993 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.993 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:38:23.993 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.100584+0000", 2023-09-23T20:38:23.993 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2679e-05 2023-09-23T20:38:23.993 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.993 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.993 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:38:23.993 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.100590+0000", 2023-09-23T20:38:23.993 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.2929999999999997e-06 2023-09-23T20:38:23.993 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.993 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.994 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:38:23.994 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.100652+0000", 2023-09-23T20:38:23.994 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.2476000000000003e-05 2023-09-23T20:38:23.994 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.994 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.994 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:38:23.994 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.100667+0000", 2023-09-23T20:38:23.994 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.4270999999999999e-05 2023-09-23T20:38:23.994 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.994 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.994 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:38:23.994 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.971028+0000", 2023-09-23T20:38:23.995 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.87036148599999996 2023-09-23T20:38:23.995 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.995 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.995 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:38:23.995 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.971042+0000", 2023-09-23T20:38:23.995 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3366999999999999e-05 2023-09-23T20:38:23.995 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.995 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.995 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:38:23.995 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.126433+0000", 2023-09-23T20:38:23.995 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1553918599999999 2023-09-23T20:38:23.996 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.996 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.996 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:38:23.996 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.126572+0000", 2023-09-23T20:38:23.996 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00013825299999999999 2023-09-23T20:38:23.996 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.996 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.996 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,4,7", 2023-09-23T20:38:23.996 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.126718+0000", 2023-09-23T20:38:23.996 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00014583900000000001 2023-09-23T20:38:23.996 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.996 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.997 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:38:23.997 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.134301+0000", 2023-09-23T20:38:23.997 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0075839439999999996 2023-09-23T20:38:23.997 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.997 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.997 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:38:23.997 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.134357+0000", 2023-09-23T20:38:23.997 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.5522000000000001e-05 2023-09-23T20:38:23.997 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.997 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.997 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:38:23.997 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.134367+0000", 2023-09-23T20:38:23.998 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0118e-05 2023-09-23T20:38:23.998 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.998 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.998 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:38:23.998 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.146664+0000", 2023-09-23T20:38:23.998 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.012296848000000001 2023-09-23T20:38:23.998 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.998 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.998 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:38:23.998 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.146717+0000", 2023-09-23T20:38:23.998 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.2800999999999998e-05 2023-09-23T20:38:23.999 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.999 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.999 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:38:23.999 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.146839+0000", 2023-09-23T20:38:23.999 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000121931 2023-09-23T20:38:23.999 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:38:23.999 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:38:23.999 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:38:23.999 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:23.999 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:23.999 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:156966 7.9 7:9ee3fd12:::benchmark_data_smithi018_308601_object62414:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e354)", 2023-09-23T20:38:23.999 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:35:00.195629+0000", 2023-09-23T20:38:24.000 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 203.613694971, 2023-09-23T20:38:24.000 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.95484696700000005, 2023-09-23T20:38:24.000 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:38:24.000 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:38:24.000 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:38:24.000 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:38:24.000 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:38:24.000 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 156966 2023-09-23T20:38:24.000 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:24.000 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:38:24.000 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:24.000 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:38:24.001 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.195629+0000", 2023-09-23T20:38:24.001 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:38:24.001 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:24.001 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:24.001 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:38:24.001 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.195629+0000", 2023-09-23T20:38:24.001 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:38:24.001 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:24.001 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:24.001 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:38:24.001 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.195627+0000", 2023-09-23T20:38:24.001 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:38:24.002 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:24.002 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:24.002 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:38:24.002 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.195635+0000", 2023-09-23T20:38:24.002 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.8909999999999995e-06 2023-09-23T20:38:24.002 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:24.002 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:24.002 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:38:24.002 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.195652+0000", 2023-09-23T20:38:24.002 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.6968000000000001e-05 2023-09-23T20:38:24.002 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:24.002 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:24.003 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:38:24.003 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.195659+0000", 2023-09-23T20:38:24.003 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.8019999999999992e-06 2023-09-23T20:38:24.003 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:24.003 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:24.003 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:38:24.003 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.197096+0000", 2023-09-23T20:38:24.003 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.001436751 2023-09-23T20:38:24.003 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:24.003 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:24.003 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:38:24.003 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.197108+0000", 2023-09-23T20:38:24.004 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1445e-05 2023-09-23T20:38:24.004 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:24.004 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:24.004 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:38:24.004 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.130266+0000", 2023-09-23T20:38:24.004 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.933158288 2023-09-23T20:38:24.004 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:24.004 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:24.004 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:38:24.004 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.130363+0000", 2023-09-23T20:38:24.004 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.7326999999999998e-05 2023-09-23T20:38:24.004 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:24.005 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:24.005 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,4,7", 2023-09-23T20:38:24.005 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.130454+0000", 2023-09-23T20:38:24.005 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.0653000000000001e-05 2023-09-23T20:38:24.005 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:24.005 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:24.005 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:38:24.005 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.134376+0000", 2023-09-23T20:38:24.005 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0039225140000000002 2023-09-23T20:38:24.005 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:24.005 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:24.006 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:38:24.006 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.134421+0000", 2023-09-23T20:38:24.006 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.4378000000000001e-05 2023-09-23T20:38:24.006 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:24.006 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:24.006 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:38:24.006 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.134474+0000", 2023-09-23T20:38:24.006 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.3263999999999997e-05 2023-09-23T20:38:24.006 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:24.006 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:24.006 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:38:24.006 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.150283+0000", 2023-09-23T20:38:24.007 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.015809519000000001 2023-09-23T20:38:24.007 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:24.007 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:24.007 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:38:24.007 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.150341+0000", 2023-09-23T20:38:24.007 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.7182999999999998e-05 2023-09-23T20:38:24.007 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:24.007 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:24.007 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:38:24.007 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.150476+0000", 2023-09-23T20:38:24.007 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000134893 2023-09-23T20:38:24.007 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:38:24.008 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:38:24.008 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:38:24.008 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:24.008 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:24.008 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:156978 7.9 7:90ac27fe:::benchmark_data_smithi018_308601_object62426:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e354)", 2023-09-23T20:38:24.008 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:35:00.215473+0000", 2023-09-23T20:38:24.008 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 203.59385086399999, 2023-09-23T20:38:24.008 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.93521590300000002, 2023-09-23T20:38:24.008 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:38:24.008 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:38:24.008 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:38:24.008 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:38:24.009 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:38:24.009 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 156978 2023-09-23T20:38:24.009 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:24.009 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:38:24.009 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:24.009 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:38:24.009 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.215473+0000", 2023-09-23T20:38:24.009 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:38:24.009 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:24.009 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:24.009 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:38:24.009 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.215473+0000", 2023-09-23T20:38:24.010 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:38:24.010 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:24.010 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:24.010 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:38:24.010 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.215471+0000", 2023-09-23T20:38:24.010 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:38:24.010 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:24.010 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:24.010 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:38:24.010 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.215478+0000", 2023-09-23T20:38:24.010 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.6710000000000002e-06 2023-09-23T20:38:24.011 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:24.011 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:24.011 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:38:24.011 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.215494+0000", 2023-09-23T20:38:24.011 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.6480000000000001e-05 2023-09-23T20:38:24.011 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:24.011 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:24.011 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:38:24.011 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.215503+0000", 2023-09-23T20:38:24.011 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.3030000000000005e-06 2023-09-23T20:38:24.011 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:24.011 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:24.012 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:38:24.012 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.217858+0000", 2023-09-23T20:38:24.012 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0023550530000000002 2023-09-23T20:38:24.012 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:24.012 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:24.012 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:38:24.012 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.217873+0000", 2023-09-23T20:38:24.012 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.428e-05 2023-09-23T20:38:24.012 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:24.012 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:24.012 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:38:24.012 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.134499+0000", 2023-09-23T20:38:24.013 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.91662635100000001 2023-09-23T20:38:24.013 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:24.013 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:24.013 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:38:24.013 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.134590+0000", 2023-09-23T20:38:24.013 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.0661999999999994e-05 2023-09-23T20:38:24.013 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:24.013 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:24.013 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,4,7", 2023-09-23T20:38:24.013 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.134659+0000", 2023-09-23T20:38:24.013 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.9627000000000003e-05 2023-09-23T20:38:24.013 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:24.014 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:24.014 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:38:24.014 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.140958+0000", 2023-09-23T20:38:24.014 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0062990199999999998 2023-09-23T20:38:24.014 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:24.014 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:24.014 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:38:24.014 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.140971+0000", 2023-09-23T20:38:24.014 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2649e-05 2023-09-23T20:38:24.014 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:24.014 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:24.015 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:38:24.015 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.141033+0000", 2023-09-23T20:38:24.015 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.1580000000000003e-05 2023-09-23T20:38:24.015 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:24.015 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:24.015 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:38:24.015 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.150526+0000", 2023-09-23T20:38:24.015 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0094936429999999995 2023-09-23T20:38:24.015 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:24.015 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:24.015 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:38:24.015 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.150571+0000", 2023-09-23T20:38:24.016 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.4948999999999999e-05 2023-09-23T20:38:24.016 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:24.016 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:24.016 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:38:24.016 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.150689+0000", 2023-09-23T20:38:24.016 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00011736800000000001 2023-09-23T20:38:24.016 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:38:24.016 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:38:24.016 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:38:24.016 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:24.016 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:24.016 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:156997 7.9 7:926c23dd:::benchmark_data_smithi018_308601_object62445:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e354)", 2023-09-23T20:38:24.017 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:35:00.331744+0000", 2023-09-23T20:38:24.017 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 203.47757940299999, 2023-09-23T20:38:24.017 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.83941280200000001, 2023-09-23T20:38:24.017 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:38:24.017 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:38:24.017 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:38:24.017 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:38:24.017 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:38:24.017 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 156997 2023-09-23T20:38:24.017 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:24.017 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:38:24.017 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:24.018 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:38:24.018 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.331744+0000", 2023-09-23T20:38:24.018 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:38:24.018 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:24.018 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:24.018 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:38:24.018 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.331744+0000", 2023-09-23T20:38:24.018 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:38:24.018 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:24.018 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:24.018 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:38:24.018 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.331743+0000", 2023-09-23T20:38:24.019 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:38:24.019 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:24.019 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:24.019 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:38:24.019 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.331749+0000", 2023-09-23T20:38:24.019 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.1140000000000002e-06 2023-09-23T20:38:24.019 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:24.019 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:24.019 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:38:24.019 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.331762+0000", 2023-09-23T20:38:24.019 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3275000000000001e-05 2023-09-23T20:38:24.020 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:24.020 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:24.020 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:38:24.020 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.331768+0000", 2023-09-23T20:38:24.020 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.7930000000000003e-06 2023-09-23T20:38:24.020 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:24.020 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:24.020 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:38:24.020 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.331839+0000", 2023-09-23T20:38:24.020 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.0685999999999994e-05 2023-09-23T20:38:24.020 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:24.020 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:24.021 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:38:24.021 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.331853+0000", 2023-09-23T20:38:24.021 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.4698999999999999e-05 2023-09-23T20:38:24.021 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:24.021 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:24.021 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:38:24.021 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.137742+0000", 2023-09-23T20:38:24.021 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.80588849600000001 2023-09-23T20:38:24.021 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:24.021 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:24.021 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:38:24.021 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.137885+0000", 2023-09-23T20:38:24.022 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00014333 2023-09-23T20:38:24.022 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:24.022 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:24.022 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,4,7", 2023-09-23T20:38:24.022 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.138005+0000", 2023-09-23T20:38:24.022 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000120022 2023-09-23T20:38:24.022 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:24.022 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:24.022 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:38:24.022 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.141352+0000", 2023-09-23T20:38:24.022 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.003346768 2023-09-23T20:38:24.023 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:24.023 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:24.023 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:38:24.023 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.141417+0000", 2023-09-23T20:38:24.023 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.4555999999999997e-05 2023-09-23T20:38:24.023 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:24.023 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:24.023 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:38:24.023 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.141425+0000", 2023-09-23T20:38:24.023 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.3869999999999995e-06 2023-09-23T20:38:24.023 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:24.024 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:24.024 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:38:24.024 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.170973+0000", 2023-09-23T20:38:24.024 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.02954824 2023-09-23T20:38:24.024 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:24.024 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:24.024 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:38:24.024 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.171028+0000", 2023-09-23T20:38:24.024 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.5040999999999997e-05 2023-09-23T20:38:24.024 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:24.024 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:24.024 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:38:24.025 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.171157+0000", 2023-09-23T20:38:24.025 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00012873999999999999 2023-09-23T20:38:24.025 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:38:24.025 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:38:24.025 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:38:24.025 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:38:24.025 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:38:24.025 INFO:teuthology.orchestra.run.smithi018.stdout:} 2023-09-23T20:38:24.026 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:24.026 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:38:24.027 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:38:24.127 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:38:24.157 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:24.194 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:38:24.199 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:24.199 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:38:24.300 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:38:24.346 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:24 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3531594605' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T20:38:24.346 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:24 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T20:38:24.410 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:24.415 INFO:teuthology.orchestra.run.smithi018.stderr:set pool 8 pg_num to 26 2023-09-23T20:38:24.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:24 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3531594605' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T20:38:24.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:24 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T20:38:24.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:24 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3531594605' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T20:38:24.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:24 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T20:38:25.373 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:38:24 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:38:24] "GET /metrics HTTP/1.1" 200 36563 "" "Prometheus/2.43.0" 2023-09-23T20:38:25.376 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:25 smithi018 ceph-mon[179647]: pgmap v993: 119 pgs: 2 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 105 active+clean; 4.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 457 op/s; 32456/225825 objects degraded (14.372%); 8723/225825 objects misplaced (3.863%); 6.4 MiB/s, 102 objects/s recovering 2023-09-23T20:38:25.376 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:25 smithi018 ceph-mon[179647]: failed to encode map e409 with expected crc 2023-09-23T20:38:25.377 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:25 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "26"}]': finished 2023-09-23T20:38:25.377 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:25 smithi018 ceph-mon[179647]: osdmap e409: 8 total, 8 up, 7 in 2023-09-23T20:38:25.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:25 smithi018 ceph-mon[184144]: pgmap v993: 119 pgs: 2 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 105 active+clean; 4.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 457 op/s; 32456/225825 objects degraded (14.372%); 8723/225825 objects misplaced (3.863%); 6.4 MiB/s, 102 objects/s recovering 2023-09-23T20:38:25.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:25 smithi018 ceph-mon[184144]: failed to encode map e409 with expected crc 2023-09-23T20:38:25.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:25 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "26"}]': finished 2023-09-23T20:38:25.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:25 smithi018 ceph-mon[184144]: osdmap e409: 8 total, 8 up, 7 in 2023-09-23T20:38:25.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:25 smithi149 ceph-mon[170394]: pgmap v993: 119 pgs: 2 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 105 active+clean; 4.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 457 op/s; 32456/225825 objects degraded (14.372%); 8723/225825 objects misplaced (3.863%); 6.4 MiB/s, 102 objects/s recovering 2023-09-23T20:38:25.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:25 smithi149 ceph-mon[170394]: failed to encode map e409 with expected crc 2023-09-23T20:38:25.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:25 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "26"}]': finished 2023-09-23T20:38:25.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:25 smithi149 ceph-mon[170394]: osdmap e409: 8 total, 8 up, 7 in 2023-09-23T20:38:25.839 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T20:38:26.407 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:38:26.412 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:26.412 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:38:26.482 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:26 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-09-23T20:38:26.483 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:26 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 32456/225825 objects degraded (14.372%), 13 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T20:38:26.490 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:26 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-09-23T20:38:26.490 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:26 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 32456/225825 objects degraded (14.372%), 13 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T20:38:26.513 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:38:26.728 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:26 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-09-23T20:38:26.728 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:26 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 32456/225825 objects degraded (14.372%), 13 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T20:38:26.796 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:26.915 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:38:26.918 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:26.919 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:38:27.019 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:38:27.078 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:27.494 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:27.495 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:38:27.495 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:38:27.596 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:38:27.624 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:27.661 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:38:27.665 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:27 smithi149 ceph-mon[170394]: pgmap v995: 119 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 105 active+clean; 4.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 459 op/s; 32350/222615 objects degraded (14.532%); 8560/222615 objects misplaced (3.845%); 6.6 MiB/s, 105 objects/s recovering 2023-09-23T20:38:27.666 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:27 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num_actual", "val": "26"}]': finished 2023-09-23T20:38:27.666 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:27 smithi149 ceph-mon[170394]: osdmap e410: 8 total, 8 up, 7 in 2023-09-23T20:38:27.666 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:27 smithi149 ceph-mon[170394]: failed to encode map e410 with expected crc 2023-09-23T20:38:27.666 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:27 smithi149 ceph-mon[170394]: failed to encode map e410 with expected crc 2023-09-23T20:38:27.666 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:27 smithi149 ceph-mon[170394]: failed to encode map e410 with expected crc 2023-09-23T20:38:27.667 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:27.667 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:38:27.768 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:38:27.795 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:38:27.813 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:27 smithi018 ceph-mon[184144]: pgmap v995: 119 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 105 active+clean; 4.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 459 op/s; 32350/222615 objects degraded (14.532%); 8560/222615 objects misplaced (3.845%); 6.6 MiB/s, 105 objects/s recovering 2023-09-23T20:38:27.813 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:27 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num_actual", "val": "26"}]': finished 2023-09-23T20:38:27.813 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:27 smithi018 ceph-mon[184144]: osdmap e410: 8 total, 8 up, 7 in 2023-09-23T20:38:27.814 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:27 smithi018 ceph-mon[184144]: failed to encode map e410 with expected crc 2023-09-23T20:38:27.814 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:27 smithi018 ceph-mon[184144]: failed to encode map e410 with expected crc 2023-09-23T20:38:27.814 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:27 smithi018 ceph-mon[184144]: failed to encode map e410 with expected crc 2023-09-23T20:38:27.814 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:27 smithi018 ceph-mon[179647]: pgmap v995: 119 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 105 active+clean; 4.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 459 op/s; 32350/222615 objects degraded (14.532%); 8560/222615 objects misplaced (3.845%); 6.6 MiB/s, 105 objects/s recovering 2023-09-23T20:38:27.814 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:27 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num_actual", "val": "26"}]': finished 2023-09-23T20:38:27.814 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:27 smithi018 ceph-mon[179647]: osdmap e410: 8 total, 8 up, 7 in 2023-09-23T20:38:27.814 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:27 smithi018 ceph-mon[179647]: failed to encode map e410 with expected crc 2023-09-23T20:38:27.815 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:27 smithi018 ceph-mon[179647]: failed to encode map e410 with expected crc 2023-09-23T20:38:27.815 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:27 smithi018 ceph-mon[179647]: failed to encode map e410 with expected crc 2023-09-23T20:38:27.819 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:27.898 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:38:27.922 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:38:27.945 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:38:27.948 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:38:27.950 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:27.950 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:38:27.979 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:38:28.011 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:38:28.035 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:38:28.051 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:38:28.061 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:38:28.079 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:28.087 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:38:28.091 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:38:28.091 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:38:28.091 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:38:28.091 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:38:28.091 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:38:28.091 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:38:28.091 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:38:28.091 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:38:28.114 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:38:28.117 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:28.117 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:38:28.218 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:38:28.244 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:28.280 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:38:28.283 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:28.283 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:38:28.384 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:38:28.419 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:28.451 INFO:teuthology.orchestra.run.smithi018.stderr:nodeep-scrub is set 2023-09-23T20:38:28.488 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:38:28.492 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:28.493 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:38:28.593 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:38:28.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:28 smithi018 ceph-mon[184144]: failed to encode map e411 with expected crc 2023-09-23T20:38:28.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:28 smithi018 ceph-mon[184144]: osdmap e411: 8 total, 8 up, 7 in 2023-09-23T20:38:28.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:28 smithi018 ceph-mon[184144]: failed to encode map e411 with expected crc 2023-09-23T20:38:28.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:28 smithi018 ceph-mon[184144]: failed to encode map e411 with expected crc 2023-09-23T20:38:28.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:28 smithi018 ceph-mon[184144]: failed to encode map e411 with expected crc 2023-09-23T20:38:28.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:28 smithi018 ceph-mon[184144]: 8.4 continuing backfill to osd.6 from (373'2080,409'6814] MIN to 409'6814 2023-09-23T20:38:28.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:28 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/4139664560' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:38:28.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:28 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2205173780' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:38:28.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:28 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:38:28.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:28 smithi018 ceph-mon[179647]: failed to encode map e411 with expected crc 2023-09-23T20:38:28.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:28 smithi018 ceph-mon[179647]: osdmap e411: 8 total, 8 up, 7 in 2023-09-23T20:38:28.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:28 smithi018 ceph-mon[179647]: failed to encode map e411 with expected crc 2023-09-23T20:38:28.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:28 smithi018 ceph-mon[179647]: failed to encode map e411 with expected crc 2023-09-23T20:38:28.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:28 smithi018 ceph-mon[179647]: failed to encode map e411 with expected crc 2023-09-23T20:38:28.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:28 smithi018 ceph-mon[179647]: 8.4 continuing backfill to osd.6 from (373'2080,409'6814] MIN to 409'6814 2023-09-23T20:38:28.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:28 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/4139664560' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:38:28.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:28 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2205173780' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:38:28.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:28 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:38:28.676 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:28.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:28 smithi149 ceph-mon[170394]: failed to encode map e411 with expected crc 2023-09-23T20:38:28.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:28 smithi149 ceph-mon[170394]: osdmap e411: 8 total, 8 up, 7 in 2023-09-23T20:38:28.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:28 smithi149 ceph-mon[170394]: failed to encode map e411 with expected crc 2023-09-23T20:38:28.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:28 smithi149 ceph-mon[170394]: failed to encode map e411 with expected crc 2023-09-23T20:38:28.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:28 smithi149 ceph-mon[170394]: failed to encode map e411 with expected crc 2023-09-23T20:38:28.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:28 smithi149 ceph-mon[170394]: 8.4 continuing backfill to osd.6 from (373'2080,409'6814] MIN to 409'6814 2023-09-23T20:38:28.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:28 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/4139664560' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:38:28.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:28 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2205173780' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:38:28.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:28 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:38:28.902 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:38:28.910 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:28.911 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:38:29.012 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:38:29.045 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:29.145 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:38:29.149 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:29.150 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:38:29.251 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:38:29.279 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:29.314 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:38:29.317 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:29.318 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:38:29.418 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:38:29.445 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:29.488 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:38:29.491 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:29.492 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:38:29.492 INFO:teuthology.orchestra.run.smithi018.stdout:ERROR: (22) Invalid argument 2023-09-23T20:38:29.492 INFO:teuthology.orchestra.run.smithi018.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:38:29.517 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:29 smithi018 ceph-mon[184144]: pgmap v998: 129 pgs: 10 unknown, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 105 active+clean; 4.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 435 op/s; 32205/220200 objects degraded (14.625%); 8560/220200 objects misplaced (3.887%); 5.9 MiB/s, 94 objects/s recovering 2023-09-23T20:38:29.517 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:29 smithi018 ceph-mon[184144]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:38:29.517 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:29 smithi018 ceph-mon[184144]: failed to encode map e412 with expected crc 2023-09-23T20:38:29.518 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:29 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:38:29.518 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:29 smithi018 ceph-mon[184144]: osdmap e412: 8 total, 8 up, 7 in 2023-09-23T20:38:29.518 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:29 smithi018 ceph-mon[184144]: failed to encode map e412 with expected crc 2023-09-23T20:38:29.518 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:29 smithi018 ceph-mon[184144]: failed to encode map e412 with expected crc 2023-09-23T20:38:29.518 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:29 smithi018 ceph-mon[184144]: failed to encode map e412 with expected crc 2023-09-23T20:38:29.518 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:29 smithi018 ceph-mon[184144]: failed to encode map e412 with expected crc 2023-09-23T20:38:29.518 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:29 smithi018 ceph-mon[179647]: pgmap v998: 129 pgs: 10 unknown, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 105 active+clean; 4.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 435 op/s; 32205/220200 objects degraded (14.625%); 8560/220200 objects misplaced (3.887%); 5.9 MiB/s, 94 objects/s recovering 2023-09-23T20:38:29.518 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:29 smithi018 ceph-mon[179647]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:38:29.519 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:29 smithi018 ceph-mon[179647]: failed to encode map e412 with expected crc 2023-09-23T20:38:29.519 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:29 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:38:29.519 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:29 smithi018 ceph-mon[179647]: osdmap e412: 8 total, 8 up, 7 in 2023-09-23T20:38:29.519 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:29 smithi018 ceph-mon[179647]: failed to encode map e412 with expected crc 2023-09-23T20:38:29.519 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:29 smithi018 ceph-mon[179647]: failed to encode map e412 with expected crc 2023-09-23T20:38:29.519 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:29 smithi018 ceph-mon[179647]: failed to encode map e412 with expected crc 2023-09-23T20:38:29.519 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:29 smithi018 ceph-mon[179647]: failed to encode map e412 with expected crc 2023-09-23T20:38:29.593 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:38:29.644 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:29.692 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:38:29.697 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:29.698 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:38:29.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:29 smithi149 ceph-mon[170394]: pgmap v998: 129 pgs: 10 unknown, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 9 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 105 active+clean; 4.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 435 op/s; 32205/220200 objects degraded (14.625%); 8560/220200 objects misplaced (3.887%); 5.9 MiB/s, 94 objects/s recovering 2023-09-23T20:38:29.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:29 smithi149 ceph-mon[170394]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:38:29.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:29 smithi149 ceph-mon[170394]: failed to encode map e412 with expected crc 2023-09-23T20:38:29.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:29 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:38:29.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:29 smithi149 ceph-mon[170394]: osdmap e412: 8 total, 8 up, 7 in 2023-09-23T20:38:29.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:29 smithi149 ceph-mon[170394]: failed to encode map e412 with expected crc 2023-09-23T20:38:29.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:29 smithi149 ceph-mon[170394]: failed to encode map e412 with expected crc 2023-09-23T20:38:29.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:29 smithi149 ceph-mon[170394]: failed to encode map e412 with expected crc 2023-09-23T20:38:29.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:29 smithi149 ceph-mon[170394]: failed to encode map e412 with expected crc 2023-09-23T20:38:29.798 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:38:29.829 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:29.865 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:38:29.868 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:29.869 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:38:29.901 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T20:38:29.970 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:38:30.018 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:30.057 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:38:30.060 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:30.061 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:38:30.162 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:38:30.191 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:30.235 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:38:30.239 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:30.239 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:38:30.340 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:38:30.366 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:30.401 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:38:30.404 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:30.404 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:38:30.505 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:38:30.531 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:30.573 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:38:30.577 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:30.577 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:38:30.623 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:38:30.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:30 smithi018 ceph-mon[184144]: osdmap e413: 8 total, 8 up, 7 in 2023-09-23T20:38:30.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:30 smithi018 ceph-mon[184144]: failed to encode map e413 with expected crc 2023-09-23T20:38:30.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:30 smithi018 ceph-mon[184144]: 8.14 continuing backfill to osd.6 from (373'2080,409'6814] MIN to 409'6814 2023-09-23T20:38:30.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:30 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:38:30.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:30 smithi018 ceph-mon[179647]: osdmap e413: 8 total, 8 up, 7 in 2023-09-23T20:38:30.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:30 smithi018 ceph-mon[179647]: failed to encode map e413 with expected crc 2023-09-23T20:38:30.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:30 smithi018 ceph-mon[179647]: 8.14 continuing backfill to osd.6 from (373'2080,409'6814] MIN to 409'6814 2023-09-23T20:38:30.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:30 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:38:30.677 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:38:30.702 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:30 smithi149 ceph-mon[170394]: osdmap e413: 8 total, 8 up, 7 in 2023-09-23T20:38:30.702 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:30 smithi149 ceph-mon[170394]: failed to encode map e413 with expected crc 2023-09-23T20:38:30.702 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:30 smithi149 ceph-mon[170394]: 8.14 continuing backfill to osd.6 from (373'2080,409'6814] MIN to 409'6814 2023-09-23T20:38:30.702 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:30 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:38:30.705 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:30.740 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:38:30.743 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:30.744 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:38:30.844 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:38:30.880 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:30.914 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:38:30.918 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:38:30.921 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:30.922 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:38:31.023 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:38:31.048 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:31.082 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:38:31.085 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:31.085 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:38:31.153 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T20:38:31.186 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:38:31.229 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:31.399 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:38:31.402 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:31.403 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:38:31.424 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:31 smithi018 ceph-mon[179647]: pgmap v1001: 129 pgs: 5 unknown, 8 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 107 active+clean; 4.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 333 op/s; 34484/218613 objects degraded (15.774%); 5834/218613 objects misplaced (2.669%); 9.3 MiB/s, 147 objects/s recovering 2023-09-23T20:38:31.425 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:31 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 34484/218613 objects degraded (15.774%), 16 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T20:38:31.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:31 smithi018 ceph-mon[184144]: pgmap v1001: 129 pgs: 5 unknown, 8 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 107 active+clean; 4.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 333 op/s; 34484/218613 objects degraded (15.774%); 5834/218613 objects misplaced (2.669%); 9.3 MiB/s, 147 objects/s recovering 2023-09-23T20:38:31.425 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:31 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 34484/218613 objects degraded (15.774%), 16 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T20:38:31.504 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:38:31.542 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:31.670 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:38:31.673 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:31.674 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:38:31.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:31 smithi149 ceph-mon[170394]: pgmap v1001: 129 pgs: 5 unknown, 8 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 107 active+clean; 4.5 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 333 op/s; 34484/218613 objects degraded (15.774%); 5834/218613 objects misplaced (2.669%); 9.3 MiB/s, 147 objects/s recovering 2023-09-23T20:38:31.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:31 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 34484/218613 objects degraded (15.774%), 16 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T20:38:31.774 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:38:31.817 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:31.907 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:38:31.910 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:31.910 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:38:31.974 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:38:32.000 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:38:32.007 INFO:teuthology.orchestra.run.smithi018.stdout:ERROR: (22) Invalid argument 2023-09-23T20:38:32.007 INFO:teuthology.orchestra.run.smithi018.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:38:32.011 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:38:32.056 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:32.156 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 7, 3, 2, 4, 6] out_osds: [5] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T20:38:32.156 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:38:32.157 INFO:tasks.thrashosds.thrasher:Adding osd 5 2023-09-23T20:38:32.157 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd in 5 2023-09-23T20:38:32.282 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:38:32.286 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:32.287 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:38:32.387 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:38:32.422 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:32.718 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:38:32.721 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:32.722 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:38:32.823 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:38:32.858 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:33.163 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:38:33.166 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:33.167 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:38:33.225 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T20:38:33.268 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:38:33.294 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:33.331 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:38:33.334 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:33.335 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:38:33.435 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:38:33.468 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:33 smithi018 ceph-mon[184144]: pgmap v1002: 129 pgs: 1 peering, 1 activating+degraded, 12 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 110 active+clean; 4.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 164 op/s; 31288/218085 objects degraded (14.347%); 5834/218085 objects misplaced (2.675%); 9.7 MiB/s, 204 objects/s recovering 2023-09-23T20:38:33.468 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:33 smithi018 ceph-mon[179647]: pgmap v1002: 129 pgs: 1 peering, 1 activating+degraded, 12 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 110 active+clean; 4.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 164 op/s; 31288/218085 objects degraded (14.347%); 5834/218085 objects misplaced (2.675%); 9.7 MiB/s, 204 objects/s recovering 2023-09-23T20:38:33.472 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:33.585 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:38:33.588 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:33.589 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:38:33.690 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:38:33.690 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:38:33.714 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:33 smithi149 ceph-mon[170394]: pgmap v1002: 129 pgs: 1 peering, 1 activating+degraded, 12 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 110 active+clean; 4.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 164 op/s; 31288/218085 objects degraded (14.347%); 5834/218085 objects misplaced (2.675%); 9.7 MiB/s, 204 objects/s recovering 2023-09-23T20:38:33.716 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:33.752 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:38:33.756 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:33.756 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:38:33.857 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:38:33.883 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:33.918 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:38:33.921 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:33.921 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:38:33.953 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:38:33.976 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:38:34.003 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:38:34.022 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:38:34.029 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:38:34.056 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:38:34.068 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:34.086 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:38:34.116 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:38:34.120 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:34.120 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:38:34.122 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:38:34.149 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:38:34.154 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:38:34.154 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:38:34.154 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:38:34.154 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:38:34.154 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:38:34.154 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:38:34.155 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:38:34.155 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:38:34.221 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:38:34.267 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:34.433 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:34 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1459744365' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:38:34.434 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:34 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1459744365' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:38:34.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:34 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1459744365' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:38:34.833 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:38:34.836 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:34.836 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:38:34.838 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:38:34.937 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:38:34.968 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:35.052 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:38:34 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:38:34] "GET /metrics HTTP/1.1" 200 36575 "" "Prometheus/2.43.0" 2023-09-23T20:38:35.129 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:38:35.132 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:35.133 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:38:35.233 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:38:35.273 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:35.317 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:38:35.320 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:35.320 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:38:35.421 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:38:35.436 INFO:teuthology.orchestra.run.smithi018.stderr:noscrub is unset 2023-09-23T20:38:35.473 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:35.571 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:38:35.574 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:35 smithi018 ceph-mon[179647]: pgmap v1003: 129 pgs: 1 peering, 1 activating+degraded, 12 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 110 active+clean; 4.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 125 op/s; 31283/217971 objects degraded (14.352%); 4861/217971 objects misplaced (2.230%); 17 MiB/s, 320 objects/s recovering 2023-09-23T20:38:35.574 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:35 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2184574201' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:38:35.574 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:35 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:38:35.574 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:35 smithi018 ceph-mon[184144]: pgmap v1003: 129 pgs: 1 peering, 1 activating+degraded, 12 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 110 active+clean; 4.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 125 op/s; 31283/217971 objects degraded (14.352%); 4861/217971 objects misplaced (2.230%); 17 MiB/s, 320 objects/s recovering 2023-09-23T20:38:35.575 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:35 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2184574201' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:38:35.575 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:35 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:38:35.575 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:35.576 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:38:35.676 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:38:35.709 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:35.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:35 smithi149 ceph-mon[170394]: pgmap v1003: 129 pgs: 1 peering, 1 activating+degraded, 12 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 110 active+clean; 4.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 125 op/s; 31283/217971 objects degraded (14.352%); 4861/217971 objects misplaced (2.230%); 17 MiB/s, 320 objects/s recovering 2023-09-23T20:38:35.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:35 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2184574201' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:38:35.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:35 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:38:36.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:35 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mon-a[179623]: 2023-09-23T20:38:35.826+0000 7f160fef9700 -1 mon.a@0(leader).osd e414 definitely_dead 0 2023-09-23T20:38:36.445 INFO:teuthology.orchestra.run.smithi018.stderr:marked in osd.5. 2023-09-23T20:38:36.489 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:38:36.493 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:36 smithi018 ceph-mon[184144]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:38:36.493 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:36 smithi018 ceph-mon[184144]: failed to encode map e414 with expected crc 2023-09-23T20:38:36.493 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:36 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:38:36.493 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:36 smithi018 ceph-mon[184144]: osdmap e414: 8 total, 8 up, 7 in 2023-09-23T20:38:36.493 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:36 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2177316976' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["5"]}]: dispatch 2023-09-23T20:38:36.493 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:36 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["5"]}]: dispatch 2023-09-23T20:38:36.493 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:36 smithi018 ceph-mon[184144]: failed to encode map e414 with expected crc 2023-09-23T20:38:36.493 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:36 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 31283/217971 objects degraded (14.352%), 17 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T20:38:36.494 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:36 smithi018 ceph-mon[179647]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:38:36.494 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:36 smithi018 ceph-mon[179647]: failed to encode map e414 with expected crc 2023-09-23T20:38:36.494 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:36 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:38:36.494 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:36 smithi018 ceph-mon[179647]: osdmap e414: 8 total, 8 up, 7 in 2023-09-23T20:38:36.495 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:36 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2177316976' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["5"]}]: dispatch 2023-09-23T20:38:36.495 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:36 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["5"]}]: dispatch 2023-09-23T20:38:36.495 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:36 smithi018 ceph-mon[179647]: failed to encode map e414 with expected crc 2023-09-23T20:38:36.495 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:36 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 31283/217971 objects degraded (14.352%), 17 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T20:38:36.495 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:36.496 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:38:36.596 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:38:36.624 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:36.659 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:38:36.662 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:36.662 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:38:36.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:36 smithi149 ceph-mon[170394]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:38:36.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:36 smithi149 ceph-mon[170394]: failed to encode map e414 with expected crc 2023-09-23T20:38:36.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:36 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:38:36.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:36 smithi149 ceph-mon[170394]: osdmap e414: 8 total, 8 up, 7 in 2023-09-23T20:38:36.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:36 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2177316976' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["5"]}]: dispatch 2023-09-23T20:38:36.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:36 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["5"]}]: dispatch 2023-09-23T20:38:36.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:36 smithi149 ceph-mon[170394]: failed to encode map e414 with expected crc 2023-09-23T20:38:36.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:36 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 31283/217971 objects degraded (14.352%), 17 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T20:38:36.763 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:38:36.794 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:36.800 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:38:37.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:37 smithi149 ceph-mon[170394]: failed to encode map e414 with expected crc 2023-09-23T20:38:37.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:37 smithi149 ceph-mon[170394]: pgmap v1005: 129 pgs: 1 peering, 1 activating+degraded, 12 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 2 active+recovery_wait+degraded+remapped, 110 active+clean; 4.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 98 op/s; 31277/217938 objects degraded (14.351%); 4503/217938 objects misplaced (2.066%); 18 MiB/s, 326 objects/s recovering 2023-09-23T20:38:37.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:37 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["5"]}]': finished 2023-09-23T20:38:37.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:37 smithi149 ceph-mon[170394]: osdmap e415: 8 total, 8 up, 8 in 2023-09-23T20:38:37.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:37 smithi149 ceph-mon[170394]: failed to encode map e415 with expected crc 2023-09-23T20:38:37.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:37 smithi149 ceph-mon[170394]: failed to encode map e415 with expected crc 2023-09-23T20:38:37.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:37 smithi018 ceph-mon[184144]: failed to encode map e414 with expected crc 2023-09-23T20:38:37.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:37 smithi018 ceph-mon[184144]: pgmap v1005: 129 pgs: 1 peering, 1 activating+degraded, 12 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 2 active+recovery_wait+degraded+remapped, 110 active+clean; 4.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 98 op/s; 31277/217938 objects degraded (14.351%); 4503/217938 objects misplaced (2.066%); 18 MiB/s, 326 objects/s recovering 2023-09-23T20:38:37.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:37 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["5"]}]': finished 2023-09-23T20:38:37.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:37 smithi018 ceph-mon[184144]: osdmap e415: 8 total, 8 up, 8 in 2023-09-23T20:38:37.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:37 smithi018 ceph-mon[184144]: failed to encode map e415 with expected crc 2023-09-23T20:38:37.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:37 smithi018 ceph-mon[184144]: failed to encode map e415 with expected crc 2023-09-23T20:38:37.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:37 smithi018 ceph-mon[179647]: failed to encode map e414 with expected crc 2023-09-23T20:38:37.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:37 smithi018 ceph-mon[179647]: pgmap v1005: 129 pgs: 1 peering, 1 activating+degraded, 12 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering, 2 active+recovery_wait+degraded+remapped, 110 active+clean; 4.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 98 op/s; 31277/217938 objects degraded (14.351%); 4503/217938 objects misplaced (2.066%); 18 MiB/s, 326 objects/s recovering 2023-09-23T20:38:37.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:37 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["5"]}]': finished 2023-09-23T20:38:37.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:37 smithi018 ceph-mon[179647]: osdmap e415: 8 total, 8 up, 8 in 2023-09-23T20:38:37.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:37 smithi018 ceph-mon[179647]: failed to encode map e415 with expected crc 2023-09-23T20:38:37.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:37 smithi018 ceph-mon[179647]: failed to encode map e415 with expected crc 2023-09-23T20:38:38.229 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:38:38.234 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:38.235 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:38:38.335 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:38:38.385 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:38.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:38 smithi018 ceph-mon[184144]: osdmap e416: 8 total, 8 up, 8 in 2023-09-23T20:38:38.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:38 smithi018 ceph-mon[184144]: failed to encode map e416 with expected crc 2023-09-23T20:38:38.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:38 smithi018 ceph-mon[184144]: failed to encode map e416 with expected crc 2023-09-23T20:38:38.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:38 smithi018 ceph-mon[184144]: failed to encode map e416 with expected crc 2023-09-23T20:38:38.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:38 smithi018 ceph-mon[179647]: osdmap e416: 8 total, 8 up, 8 in 2023-09-23T20:38:38.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:38 smithi018 ceph-mon[179647]: failed to encode map e416 with expected crc 2023-09-23T20:38:38.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:38 smithi018 ceph-mon[179647]: failed to encode map e416 with expected crc 2023-09-23T20:38:38.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:38 smithi018 ceph-mon[179647]: failed to encode map e416 with expected crc 2023-09-23T20:38:38.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:38 smithi149 ceph-mon[170394]: osdmap e416: 8 total, 8 up, 8 in 2023-09-23T20:38:38.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:38 smithi149 ceph-mon[170394]: failed to encode map e416 with expected crc 2023-09-23T20:38:38.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:38 smithi149 ceph-mon[170394]: failed to encode map e416 with expected crc 2023-09-23T20:38:38.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:38 smithi149 ceph-mon[170394]: failed to encode map e416 with expected crc 2023-09-23T20:38:39.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:39 smithi149 ceph-mon[170394]: pgmap v1008: 129 pgs: 2 remapped+peering, 12 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 1 active+degraded, 2 active+recovery_wait+degraded+remapped, 109 active+clean; 4.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 8 op/s; 33420/217935 objects degraded (15.335%); 4503/217935 objects misplaced (2.066%); 16 MiB/s, 281 objects/s recovering 2023-09-23T20:38:39.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:39 smithi149 ceph-mon[170394]: osdmap e417: 8 total, 8 up, 8 in 2023-09-23T20:38:39.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:39 smithi149 ceph-mon[170394]: failed to encode map e417 with expected crc 2023-09-23T20:38:39.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:39 smithi149 ceph-mon[170394]: failed to encode map e417 with expected crc 2023-09-23T20:38:39.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:39 smithi018 ceph-mon[179647]: pgmap v1008: 129 pgs: 2 remapped+peering, 12 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 1 active+degraded, 2 active+recovery_wait+degraded+remapped, 109 active+clean; 4.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 8 op/s; 33420/217935 objects degraded (15.335%); 4503/217935 objects misplaced (2.066%); 16 MiB/s, 281 objects/s recovering 2023-09-23T20:38:39.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:39 smithi018 ceph-mon[179647]: osdmap e417: 8 total, 8 up, 8 in 2023-09-23T20:38:39.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:39 smithi018 ceph-mon[179647]: failed to encode map e417 with expected crc 2023-09-23T20:38:39.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:39 smithi018 ceph-mon[179647]: failed to encode map e417 with expected crc 2023-09-23T20:38:39.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:39 smithi018 ceph-mon[184144]: pgmap v1008: 129 pgs: 2 remapped+peering, 12 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 1 active+degraded, 2 active+recovery_wait+degraded+remapped, 109 active+clean; 4.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 8 op/s; 33420/217935 objects degraded (15.335%); 4503/217935 objects misplaced (2.066%); 16 MiB/s, 281 objects/s recovering 2023-09-23T20:38:39.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:39 smithi018 ceph-mon[184144]: osdmap e417: 8 total, 8 up, 8 in 2023-09-23T20:38:39.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:39 smithi018 ceph-mon[184144]: failed to encode map e417 with expected crc 2023-09-23T20:38:39.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:39 smithi018 ceph-mon[184144]: failed to encode map e417 with expected crc 2023-09-23T20:38:40.328 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:38:40.333 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:40.333 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:38:40.421 INFO:tasks.thrashosds.thrasher:Added osd 5 2023-09-23T20:38:40.434 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:38:40.463 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:40.513 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:38:40.518 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:40.519 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:38:40.609 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:40 smithi018 ceph-mon[184144]: 1.1b scrub starts 2023-09-23T20:38:40.609 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:40 smithi018 ceph-mon[184144]: failed to encode map e418 with expected crc 2023-09-23T20:38:40.609 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:40 smithi018 ceph-mon[184144]: osdmap e418: 8 total, 8 up, 8 in 2023-09-23T20:38:40.609 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:40 smithi018 ceph-mon[184144]: failed to encode map e418 with expected crc 2023-09-23T20:38:40.609 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:40 smithi018 ceph-mon[184144]: failed to encode map e418 with expected crc 2023-09-23T20:38:40.609 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:40 smithi018 ceph-mon[184144]: failed to encode map e418 with expected crc 2023-09-23T20:38:40.610 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:40 smithi018 ceph-mon[179647]: 1.1b scrub starts 2023-09-23T20:38:40.610 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:40 smithi018 ceph-mon[179647]: failed to encode map e418 with expected crc 2023-09-23T20:38:40.610 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:40 smithi018 ceph-mon[179647]: osdmap e418: 8 total, 8 up, 8 in 2023-09-23T20:38:40.610 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:40 smithi018 ceph-mon[179647]: failed to encode map e418 with expected crc 2023-09-23T20:38:40.610 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:40 smithi018 ceph-mon[179647]: failed to encode map e418 with expected crc 2023-09-23T20:38:40.610 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:40 smithi018 ceph-mon[179647]: failed to encode map e418 with expected crc 2023-09-23T20:38:40.620 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:38:40.650 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:40.695 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:38:40.698 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:40.698 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:38:40.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:40 smithi149 ceph-mon[170394]: 1.1b scrub starts 2023-09-23T20:38:40.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:40 smithi149 ceph-mon[170394]: failed to encode map e418 with expected crc 2023-09-23T20:38:40.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:40 smithi149 ceph-mon[170394]: osdmap e418: 8 total, 8 up, 8 in 2023-09-23T20:38:40.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:40 smithi149 ceph-mon[170394]: failed to encode map e418 with expected crc 2023-09-23T20:38:40.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:40 smithi149 ceph-mon[170394]: failed to encode map e418 with expected crc 2023-09-23T20:38:40.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:40 smithi149 ceph-mon[170394]: failed to encode map e418 with expected crc 2023-09-23T20:38:40.800 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:38:40.864 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T20:38:40.865 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:40.979 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:38:40.983 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:38:41.003 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:41.004 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:38:41.105 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:38:41.136 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:41.181 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:38:41.185 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:41.185 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:38:41.272 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2023-09-23T20:38:41.273 INFO:teuthology.orchestra.run.smithi018.stdout: "size": 20, 2023-09-23T20:38:41.273 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 600, 2023-09-23T20:38:41.273 INFO:teuthology.orchestra.run.smithi018.stdout: "ops": [ 2023-09-23T20:38:41.273 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.273 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.45161.0:136049 6.f 6:f1f0f499:::benchmark_data_smithi018_270558_object31310:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e258)", 2023-09-23T20:38:41.273 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:29:27.360919+0000", 2023-09-23T20:38:41.273 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 553.906275876, 2023-09-23T20:38:41.273 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.1770357090000001, 2023-09-23T20:38:41.273 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:38:41.273 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:38:41.274 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:38:41.274 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.45161", 2023-09-23T20:38:41.274 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3795614987", 2023-09-23T20:38:41.274 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 136049 2023-09-23T20:38:41.274 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.274 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:38:41.274 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.274 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:38:41.274 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.360919+0000", 2023-09-23T20:38:41.274 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:38:41.274 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.275 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.275 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:38:41.275 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.360919+0000", 2023-09-23T20:38:41.275 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:38:41.275 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.275 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.275 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:38:41.275 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.360917+0000", 2023-09-23T20:38:41.275 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:38:41.275 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.275 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.275 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:38:41.276 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.360925+0000", 2023-09-23T20:38:41.276 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.8480000000000001e-06 2023-09-23T20:38:41.276 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.276 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.276 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:38:41.276 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.360944+0000", 2023-09-23T20:38:41.276 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.8439000000000001e-05 2023-09-23T20:38:41.276 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.276 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.276 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:38:41.276 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.360954+0000", 2023-09-23T20:38:41.277 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0327e-05 2023-09-23T20:38:41.277 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.277 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.277 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:38:41.277 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.363434+0000", 2023-09-23T20:38:41.277 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0024801100000000002 2023-09-23T20:38:41.277 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.277 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.277 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:38:41.277 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.363450+0000", 2023-09-23T20:38:41.277 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.5359999999999999e-05 2023-09-23T20:38:41.278 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.278 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.278 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:38:41.278 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:28.342983+0000", 2023-09-23T20:38:41.278 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.97953384099999996 2023-09-23T20:38:41.278 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.278 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.278 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:38:41.278 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:28.342997+0000", 2023-09-23T20:38:41.278 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3641e-05 2023-09-23T20:38:41.278 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.279 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.279 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:38:41.279 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.518718+0000", 2023-09-23T20:38:41.279 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.175721432 2023-09-23T20:38:41.279 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.279 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.279 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:38:41.279 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.518815+0000", 2023-09-23T20:38:41.279 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.6764999999999999e-05 2023-09-23T20:38:41.279 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.279 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.280 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 3,4,6", 2023-09-23T20:38:41.280 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.519272+0000", 2023-09-23T20:38:41.280 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00045647899999999998 2023-09-23T20:38:41.280 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.280 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.280 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:38:41.280 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.537608+0000", 2023-09-23T20:38:41.280 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.018336363000000001 2023-09-23T20:38:41.280 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.280 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.280 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:38:41.280 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.537675+0000", 2023-09-23T20:38:41.281 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.7039999999999995e-05 2023-09-23T20:38:41.281 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.281 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.281 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:38:41.281 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.537777+0000", 2023-09-23T20:38:41.281 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000102147 2023-09-23T20:38:41.281 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.281 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.281 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:38:41.281 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.537821+0000", 2023-09-23T20:38:41.281 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.3547999999999997e-05 2023-09-23T20:38:41.282 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.282 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.282 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:38:41.282 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.537858+0000", 2023-09-23T20:38:41.282 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.7132999999999998e-05 2023-09-23T20:38:41.282 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.282 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.282 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:38:41.282 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.537955+0000", 2023-09-23T20:38:41.282 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.6804000000000004e-05 2023-09-23T20:38:41.282 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:38:41.283 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:38:41.283 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:38:41.283 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.283 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.283 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.45161.0:136051 6.f 6:fcaf49a8:::benchmark_data_smithi018_270558_object31312:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e258)", 2023-09-23T20:38:41.283 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:29:27.375493+0000", 2023-09-23T20:38:41.283 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 553.89170150300004, 2023-09-23T20:38:41.283 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.174356832, 2023-09-23T20:38:41.283 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:38:41.283 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:38:41.283 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:38:41.284 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.45161", 2023-09-23T20:38:41.284 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3795614987", 2023-09-23T20:38:41.284 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 136051 2023-09-23T20:38:41.284 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.284 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:38:41.284 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.284 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:38:41.284 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.375493+0000", 2023-09-23T20:38:41.284 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:38:41.284 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.284 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.284 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:38:41.285 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.375493+0000", 2023-09-23T20:38:41.285 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:38:41.285 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.285 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.285 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:38:41.285 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.375491+0000", 2023-09-23T20:38:41.285 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:38:41.285 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.285 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.285 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:38:41.285 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.375501+0000", 2023-09-23T20:38:41.286 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.149e-06 2023-09-23T20:38:41.286 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.286 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.286 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:38:41.286 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.375516+0000", 2023-09-23T20:38:41.286 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.5463000000000001e-05 2023-09-23T20:38:41.286 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.286 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.286 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:38:41.286 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.375528+0000", 2023-09-23T20:38:41.286 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1800999999999999e-05 2023-09-23T20:38:41.286 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.287 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.287 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:38:41.287 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.375579+0000", 2023-09-23T20:38:41.287 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.096e-05 2023-09-23T20:38:41.287 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.287 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.287 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:38:41.287 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.375592+0000", 2023-09-23T20:38:41.287 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.343e-05 2023-09-23T20:38:41.287 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.287 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.288 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:38:41.288 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:28.343022+0000", 2023-09-23T20:38:41.288 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.96742986500000006 2023-09-23T20:38:41.288 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.288 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.288 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:38:41.288 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:28.343030+0000", 2023-09-23T20:38:41.288 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.2400000000000007e-06 2023-09-23T20:38:41.288 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.288 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.288 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:38:41.288 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.530153+0000", 2023-09-23T20:38:41.289 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.187122427 2023-09-23T20:38:41.289 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.289 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.289 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:38:41.289 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.530266+0000", 2023-09-23T20:38:41.289 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00011287500000000001 2023-09-23T20:38:41.289 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.289 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.289 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 3,4,6", 2023-09-23T20:38:41.289 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.530349+0000", 2023-09-23T20:38:41.289 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.3065999999999994e-05 2023-09-23T20:38:41.289 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.290 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.290 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:38:41.290 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.537739+0000", 2023-09-23T20:38:41.290 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0073898840000000002 2023-09-23T20:38:41.290 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.290 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.290 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:38:41.290 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.537745+0000", 2023-09-23T20:38:41.290 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.5860000000000001e-06 2023-09-23T20:38:41.290 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.290 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.291 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:38:41.291 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.538012+0000", 2023-09-23T20:38:41.291 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000267018 2023-09-23T20:38:41.291 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.291 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.291 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:38:41.291 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.549701+0000", 2023-09-23T20:38:41.291 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.011688426 2023-09-23T20:38:41.291 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.291 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.291 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:38:41.291 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.549735+0000", 2023-09-23T20:38:41.292 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.4161e-05 2023-09-23T20:38:41.292 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.292 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.292 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:38:41.292 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.549850+0000", 2023-09-23T20:38:41.292 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00011539099999999999 2023-09-23T20:38:41.292 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:38:41.292 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:38:41.292 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:38:41.292 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.292 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.292 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.45161.0:136053 6.f 6:f2209241:::benchmark_data_smithi018_270558_object31314:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e258)", 2023-09-23T20:38:41.293 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:29:27.379666+0000", 2023-09-23T20:38:41.293 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 553.88752897500001, 2023-09-23T20:38:41.293 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.1768631190000001, 2023-09-23T20:38:41.293 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:38:41.293 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:38:41.293 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:38:41.293 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.45161", 2023-09-23T20:38:41.293 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3795614987", 2023-09-23T20:38:41.293 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 136053 2023-09-23T20:38:41.293 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.293 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:38:41.293 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.294 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:38:41.294 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.379666+0000", 2023-09-23T20:38:41.294 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:38:41.294 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.294 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.294 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:38:41.294 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.379666+0000", 2023-09-23T20:38:41.294 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:38:41.294 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.294 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.294 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:38:41.295 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.379664+0000", 2023-09-23T20:38:41.295 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999976 2023-09-23T20:38:41.295 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.295 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.295 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:38:41.295 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.379674+0000", 2023-09-23T20:38:41.295 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0271999999999999e-05 2023-09-23T20:38:41.295 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.295 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.295 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:38:41.295 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.379698+0000", 2023-09-23T20:38:41.296 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.3941999999999999e-05 2023-09-23T20:38:41.296 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.296 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.296 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:38:41.296 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.379711+0000", 2023-09-23T20:38:41.296 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2743e-05 2023-09-23T20:38:41.296 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.296 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.296 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:38:41.296 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.379865+0000", 2023-09-23T20:38:41.296 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00015445199999999999 2023-09-23T20:38:41.296 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.297 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.297 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:38:41.297 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.379881+0000", 2023-09-23T20:38:41.297 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.5423000000000001e-05 2023-09-23T20:38:41.297 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.297 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.297 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:38:41.297 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:28.343053+0000", 2023-09-23T20:38:41.297 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.963171993 2023-09-23T20:38:41.297 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.297 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.297 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:38:41.301 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:28.343061+0000", 2023-09-23T20:38:41.301 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.9480000000000008e-06 2023-09-23T20:38:41.301 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.301 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.301 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:38:41.301 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.534004+0000", 2023-09-23T20:38:41.301 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1909433570000001 2023-09-23T20:38:41.301 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.301 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.301 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:38:41.302 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.534089+0000", 2023-09-23T20:38:41.302 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.5394000000000003e-05 2023-09-23T20:38:41.302 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.302 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.302 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 3,4,6", 2023-09-23T20:38:41.302 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.534172+0000", 2023-09-23T20:38:41.302 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.2553999999999996e-05 2023-09-23T20:38:41.302 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.302 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.302 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:38:41.302 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.537964+0000", 2023-09-23T20:38:41.303 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0037922590000000001 2023-09-23T20:38:41.303 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.303 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.303 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:38:41.303 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.538056+0000", 2023-09-23T20:38:41.303 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.1833e-05 2023-09-23T20:38:41.303 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.303 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.303 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:38:41.303 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.538092+0000", 2023-09-23T20:38:41.303 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.6528e-05 2023-09-23T20:38:41.304 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.304 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.304 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:38:41.304 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.556365+0000", 2023-09-23T20:38:41.304 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.018272594 2023-09-23T20:38:41.304 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.304 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.304 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:38:41.304 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.556403+0000", 2023-09-23T20:38:41.304 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.8034999999999998e-05 2023-09-23T20:38:41.304 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.304 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.305 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:38:41.305 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.556529+0000", 2023-09-23T20:38:41.305 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00012598199999999999 2023-09-23T20:38:41.305 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:38:41.305 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:38:41.305 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:38:41.305 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.305 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.305 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.45161.0:136065 6.e 6:7df88343:::benchmark_data_smithi018_270558_object31326:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e258)", 2023-09-23T20:38:41.305 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:29:27.389482+0000", 2023-09-23T20:38:41.305 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 553.87771341099995, 2023-09-23T20:38:41.306 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.1285085910000001, 2023-09-23T20:38:41.306 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:38:41.306 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:38:41.306 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:38:41.306 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.45161", 2023-09-23T20:38:41.306 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3795614987", 2023-09-23T20:38:41.306 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 136065 2023-09-23T20:38:41.306 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.306 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:38:41.306 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.306 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:38:41.306 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.389482+0000", 2023-09-23T20:38:41.307 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:38:41.307 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.307 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.307 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:38:41.307 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.389482+0000", 2023-09-23T20:38:41.307 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:38:41.307 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.307 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.307 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:38:41.307 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.389480+0000", 2023-09-23T20:38:41.307 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:38:41.307 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.308 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.308 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:38:41.308 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.389488+0000", 2023-09-23T20:38:41.308 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.6120000000000001e-06 2023-09-23T20:38:41.308 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.308 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.308 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:38:41.308 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.389503+0000", 2023-09-23T20:38:41.308 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.5339999999999999e-05 2023-09-23T20:38:41.308 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.308 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.308 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:38:41.309 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.389516+0000", 2023-09-23T20:38:41.309 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3467e-05 2023-09-23T20:38:41.309 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.309 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.309 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:38:41.309 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.389586+0000", 2023-09-23T20:38:41.309 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.9797999999999998e-05 2023-09-23T20:38:41.309 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.309 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.309 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:38:41.309 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.389597+0000", 2023-09-23T20:38:41.309 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0961999999999999e-05 2023-09-23T20:38:41.310 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.310 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.310 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:38:41.310 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:28.343523+0000", 2023-09-23T20:38:41.310 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.95392566499999998 2023-09-23T20:38:41.310 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.310 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.310 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:38:41.310 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:28.343532+0000", 2023-09-23T20:38:41.310 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.8070000000000006e-06 2023-09-23T20:38:41.310 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.311 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.311 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:38:41.311 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.508816+0000", 2023-09-23T20:38:41.311 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1652842480000001 2023-09-23T20:38:41.311 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.311 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.311 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:38:41.311 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.511009+0000", 2023-09-23T20:38:41.311 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0021932660000000001 2023-09-23T20:38:41.311 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.311 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.311 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 4,5", 2023-09-23T20:38:41.312 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.511162+0000", 2023-09-23T20:38:41.312 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00015258099999999999 2023-09-23T20:38:41.312 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.312 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.312 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:38:41.312 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.516451+0000", 2023-09-23T20:38:41.312 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.005289261 2023-09-23T20:38:41.312 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.312 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.312 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:38:41.312 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.516483+0000", 2023-09-23T20:38:41.312 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.2236000000000001e-05 2023-09-23T20:38:41.313 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.313 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.313 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:38:41.313 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.517808+0000", 2023-09-23T20:38:41.313 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0013247910000000001 2023-09-23T20:38:41.313 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.313 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.313 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:38:41.313 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.517866+0000", 2023-09-23T20:38:41.313 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.7624000000000003e-05 2023-09-23T20:38:41.313 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.313 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.313 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:38:41.314 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.517990+0000", 2023-09-23T20:38:41.314 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000124411 2023-09-23T20:38:41.314 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:38:41.314 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:38:41.314 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:38:41.314 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.314 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.314 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.45161.0:136068 6.f 6:f6357533:::benchmark_data_smithi018_270558_object31329:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e258)", 2023-09-23T20:38:41.314 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:29:27.391642+0000", 2023-09-23T20:38:41.314 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 553.87555245099998, 2023-09-23T20:38:41.314 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.1651141979999999, 2023-09-23T20:38:41.315 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:38:41.315 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:38:41.315 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:38:41.315 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.45161", 2023-09-23T20:38:41.315 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3795614987", 2023-09-23T20:38:41.315 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 136068 2023-09-23T20:38:41.315 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.315 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:38:41.315 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.315 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:38:41.315 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.391642+0000", 2023-09-23T20:38:41.315 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:38:41.316 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.316 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.316 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:38:41.316 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.391642+0000", 2023-09-23T20:38:41.316 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:38:41.316 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.316 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.316 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:38:41.316 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.391641+0000", 2023-09-23T20:38:41.316 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.999999 2023-09-23T20:38:41.316 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.317 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.317 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:38:41.317 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.391648+0000", 2023-09-23T20:38:41.317 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.0350000000000004e-06 2023-09-23T20:38:41.317 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.317 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.317 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:38:41.317 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.391663+0000", 2023-09-23T20:38:41.317 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.4722000000000001e-05 2023-09-23T20:38:41.317 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.317 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.317 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:38:41.318 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.391672+0000", 2023-09-23T20:38:41.318 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.8359999999999998e-06 2023-09-23T20:38:41.318 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.318 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.318 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:38:41.318 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.391746+0000", 2023-09-23T20:38:41.318 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.3660999999999998e-05 2023-09-23T20:38:41.318 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.318 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.318 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:38:41.318 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.391758+0000", 2023-09-23T20:38:41.319 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.236e-05 2023-09-23T20:38:41.319 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.319 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.319 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:38:41.319 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:28.343081+0000", 2023-09-23T20:38:41.319 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.95132316299999997 2023-09-23T20:38:41.319 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.319 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.319 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:38:41.319 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:28.343089+0000", 2023-09-23T20:38:41.319 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.0809999999999994e-06 2023-09-23T20:38:41.319 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.319 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.320 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:38:41.320 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.538120+0000", 2023-09-23T20:38:41.320 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.195030437 2023-09-23T20:38:41.320 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.320 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.320 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:38:41.320 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.538205+0000", 2023-09-23T20:38:41.320 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.5142999999999994e-05 2023-09-23T20:38:41.320 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.320 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.320 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 3,4,6", 2023-09-23T20:38:41.321 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.538283+0000", 2023-09-23T20:38:41.321 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.8652999999999994e-05 2023-09-23T20:38:41.321 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.321 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.321 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:38:41.321 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.549604+0000", 2023-09-23T20:38:41.321 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.011320436 2023-09-23T20:38:41.321 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.321 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.321 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:38:41.321 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.549613+0000", 2023-09-23T20:38:41.321 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.0680000000000003e-06 2023-09-23T20:38:41.322 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.322 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.322 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:38:41.322 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.549680+0000", 2023-09-23T20:38:41.322 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.6829999999999995e-05 2023-09-23T20:38:41.322 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.322 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.322 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:38:41.322 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.556587+0000", 2023-09-23T20:38:41.322 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0069069190000000001 2023-09-23T20:38:41.322 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.322 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.323 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:38:41.323 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.556645+0000", 2023-09-23T20:38:41.323 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.8740999999999999e-05 2023-09-23T20:38:41.323 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.323 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.323 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:38:41.323 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.556757+0000", 2023-09-23T20:38:41.323 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000111208 2023-09-23T20:38:41.323 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:38:41.323 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:38:41.323 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:38:41.323 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.324 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.324 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.45161.0:136069 6.e 6:7343e225:::benchmark_data_smithi018_270558_object31330:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e258)", 2023-09-23T20:38:41.324 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:29:27.396169+0000", 2023-09-23T20:38:41.324 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 553.87102561500001, 2023-09-23T20:38:41.324 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.122030412, 2023-09-23T20:38:41.324 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:38:41.324 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:38:41.324 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:38:41.324 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.45161", 2023-09-23T20:38:41.324 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3795614987", 2023-09-23T20:38:41.324 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 136069 2023-09-23T20:38:41.325 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.325 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:38:41.325 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.325 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:38:41.325 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.396169+0000", 2023-09-23T20:38:41.325 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:38:41.325 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.325 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.325 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:38:41.325 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.396169+0000", 2023-09-23T20:38:41.325 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:38:41.326 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.326 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.326 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:38:41.326 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.396168+0000", 2023-09-23T20:38:41.326 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:38:41.326 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.326 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.326 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:38:41.326 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.396175+0000", 2023-09-23T20:38:41.326 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.8310000000000003e-06 2023-09-23T20:38:41.326 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.327 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.327 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:38:41.327 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.396191+0000", 2023-09-23T20:38:41.327 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.5736999999999999e-05 2023-09-23T20:38:41.327 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.327 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.327 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:38:41.327 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.396199+0000", 2023-09-23T20:38:41.327 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.1839999999999999e-06 2023-09-23T20:38:41.327 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.327 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.327 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:38:41.328 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.396257+0000", 2023-09-23T20:38:41.328 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.8400999999999999e-05 2023-09-23T20:38:41.328 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.328 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.328 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:38:41.328 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.396268+0000", 2023-09-23T20:38:41.328 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1099e-05 2023-09-23T20:38:41.328 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.328 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.328 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:38:41.328 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:28.345135+0000", 2023-09-23T20:38:41.328 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.948866459 2023-09-23T20:38:41.328 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.329 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.329 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:38:41.329 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:28.345145+0000", 2023-09-23T20:38:41.329 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.9210000000000004e-06 2023-09-23T20:38:41.329 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.329 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.329 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:38:41.329 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.514448+0000", 2023-09-23T20:38:41.329 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1693032249999999 2023-09-23T20:38:41.329 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.329 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.330 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:38:41.330 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.514537+0000", 2023-09-23T20:38:41.330 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.9529000000000003e-05 2023-09-23T20:38:41.330 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.330 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.330 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 4,5", 2023-09-23T20:38:41.330 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.514611+0000", 2023-09-23T20:38:41.330 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.3763999999999994e-05 2023-09-23T20:38:41.330 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.330 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.330 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:38:41.330 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.516564+0000", 2023-09-23T20:38:41.331 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.001952992 2023-09-23T20:38:41.331 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.331 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.331 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:38:41.331 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.516815+0000", 2023-09-23T20:38:41.331 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00025055699999999999 2023-09-23T20:38:41.331 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.331 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.331 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:38:41.331 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.518040+0000", 2023-09-23T20:38:41.331 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0012252529999999999 2023-09-23T20:38:41.332 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.332 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.332 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:38:41.332 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.518094+0000", 2023-09-23T20:38:41.332 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.4104000000000002e-05 2023-09-23T20:38:41.332 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.332 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.332 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:38:41.332 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.518200+0000", 2023-09-23T20:38:41.332 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000105715 2023-09-23T20:38:41.332 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:38:41.332 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:38:41.333 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:38:41.333 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.333 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.333 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:57620 7.8 7:1c6d8580:::benchmark_data_smithi018_308601_object57619:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e312)", 2023-09-23T20:38:41.333 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:32:32.247325+0000", 2023-09-23T20:38:41.333 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 369.01987033299997, 2023-09-23T20:38:41.333 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.3963792850000001, 2023-09-23T20:38:41.333 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:38:41.333 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:38:41.333 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:38:41.333 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:38:41.334 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:38:41.334 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 57620 2023-09-23T20:38:41.334 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.334 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:38:41.334 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.334 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:38:41.334 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.247325+0000", 2023-09-23T20:38:41.334 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:38:41.334 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.334 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.334 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:38:41.335 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.247325+0000", 2023-09-23T20:38:41.335 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:38:41.335 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.335 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.335 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:38:41.335 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.247323+0000", 2023-09-23T20:38:41.335 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:38:41.335 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.335 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.335 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:38:41.335 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.247348+0000", 2023-09-23T20:38:41.335 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.5418e-05 2023-09-23T20:38:41.336 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.336 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.336 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:38:41.336 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.247362+0000", 2023-09-23T20:38:41.336 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.4116e-05 2023-09-23T20:38:41.336 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.336 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.336 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:38:41.336 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.247370+0000", 2023-09-23T20:38:41.336 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.9459999999999998e-06 2023-09-23T20:38:41.336 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.336 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.337 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:38:41.337 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.247428+0000", 2023-09-23T20:38:41.337 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.7472999999999997e-05 2023-09-23T20:38:41.337 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.337 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.337 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:38:41.337 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.247439+0000", 2023-09-23T20:38:41.337 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0764000000000001e-05 2023-09-23T20:38:41.337 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.337 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.337 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:38:41.337 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.213131+0000", 2023-09-23T20:38:41.338 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.96569261699999998 2023-09-23T20:38:41.338 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.338 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.338 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:38:41.338 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.213144+0000", 2023-09-23T20:38:41.338 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3129000000000001e-05 2023-09-23T20:38:41.338 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.338 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.338 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:38:41.338 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.625268+0000", 2023-09-23T20:38:41.338 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.4121238840000001 2023-09-23T20:38:41.338 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.339 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.339 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:38:41.339 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.625437+0000", 2023-09-23T20:38:41.339 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000168968 2023-09-23T20:38:41.339 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.339 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.339 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 4,7", 2023-09-23T20:38:41.339 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.625602+0000", 2023-09-23T20:38:41.339 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000164635 2023-09-23T20:38:41.339 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.339 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.340 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:38:41.340 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.633312+0000", 2023-09-23T20:38:41.340 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.007709975 2023-09-23T20:38:41.340 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.340 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.340 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:38:41.340 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.635263+0000", 2023-09-23T20:38:41.340 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0019514930000000001 2023-09-23T20:38:41.340 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.340 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.340 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:38:41.340 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.639458+0000", 2023-09-23T20:38:41.340 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0041947290000000003 2023-09-23T20:38:41.341 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.341 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.341 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:38:41.341 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.639516+0000", 2023-09-23T20:38:41.341 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.8232000000000001e-05 2023-09-23T20:38:41.341 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.341 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.341 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:38:41.341 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.643704+0000", 2023-09-23T20:38:41.341 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0041875940000000002 2023-09-23T20:38:41.341 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:38:41.342 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:38:41.342 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:38:41.342 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.342 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.342 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:69152 7.3 7:c62f424f:::benchmark_data_smithi018_308601_object69151:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e320)", 2023-09-23T20:38:41.342 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:32:50.526348+0000", 2023-09-23T20:38:41.342 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 350.74084685100001, 2023-09-23T20:38:41.342 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0568542780000001, 2023-09-23T20:38:41.343 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:38:41.343 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:38:41.343 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:38:41.343 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:38:41.343 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:38:41.343 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 69152 2023-09-23T20:38:41.343 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.343 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:38:41.343 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.343 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:38:41.343 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:50.526348+0000", 2023-09-23T20:38:41.344 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:38:41.344 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.344 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.344 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:38:41.344 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:50.526348+0000", 2023-09-23T20:38:41.344 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:38:41.344 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.344 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.344 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:38:41.344 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:50.526348+0000", 2023-09-23T20:38:41.344 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:38:41.344 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.345 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.345 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:38:41.345 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:50.526389+0000", 2023-09-23T20:38:41.345 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.1758000000000002e-05 2023-09-23T20:38:41.345 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.345 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.345 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:38:41.345 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:50.526397+0000", 2023-09-23T20:38:41.345 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.0339999999999997e-06 2023-09-23T20:38:41.345 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.345 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.346 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:38:41.346 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:50.526404+0000", 2023-09-23T20:38:41.346 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.2319999999999998e-06 2023-09-23T20:38:41.346 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.346 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.346 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:38:41.346 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:50.526656+0000", 2023-09-23T20:38:41.346 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00025193600000000002 2023-09-23T20:38:41.346 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.346 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.346 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:38:41.347 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:50.526664+0000", 2023-09-23T20:38:41.347 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.1759999999999993e-06 2023-09-23T20:38:41.347 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.347 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.347 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:38:41.347 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:51.572432+0000", 2023-09-23T20:38:41.347 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.045768684 2023-09-23T20:38:41.347 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.347 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.347 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:38:41.347 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:51.572983+0000", 2023-09-23T20:38:41.347 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00055033100000000004 2023-09-23T20:38:41.348 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.348 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.348 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:38:41.348 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:51.573177+0000", 2023-09-23T20:38:41.348 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00019465899999999999 2023-09-23T20:38:41.348 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.348 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.348 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:38:41.348 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:51.581286+0000", 2023-09-23T20:38:41.348 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0081080490000000009 2023-09-23T20:38:41.348 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.349 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.349 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:38:41.349 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:51.581339+0000", 2023-09-23T20:38:41.349 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.3736000000000002e-05 2023-09-23T20:38:41.349 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.349 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.349 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:38:41.349 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:51.581415+0000", 2023-09-23T20:38:41.349 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.5952999999999996e-05 2023-09-23T20:38:41.349 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.349 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.350 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:38:41.350 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:51.581485+0000", 2023-09-23T20:38:41.350 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.991e-05 2023-09-23T20:38:41.350 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.350 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.350 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:38:41.350 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:51.583202+0000", 2023-09-23T20:38:41.350 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.001717239 2023-09-23T20:38:41.350 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:38:41.350 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:38:41.350 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:38:41.351 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.351 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.351 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:69159 7.3 7:c3cb5c7b:::benchmark_data_smithi018_308601_object69158:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e320)", 2023-09-23T20:38:41.351 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:32:50.526409+0000", 2023-09-23T20:38:41.351 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 350.74078637600002, 2023-09-23T20:38:41.351 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0737358429999999, 2023-09-23T20:38:41.351 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:38:41.351 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:38:41.351 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:38:41.351 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:38:41.351 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:38:41.351 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 69159 2023-09-23T20:38:41.351 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.352 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:38:41.352 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.352 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:38:41.352 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:50.526409+0000", 2023-09-23T20:38:41.352 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:38:41.352 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.352 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.352 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:38:41.352 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:50.526409+0000", 2023-09-23T20:38:41.352 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:38:41.352 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.352 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.353 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:38:41.353 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:50.526408+0000", 2023-09-23T20:38:41.353 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:38:41.353 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.353 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.353 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:38:41.353 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:50.526448+0000", 2023-09-23T20:38:41.353 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.9480999999999997e-05 2023-09-23T20:38:41.353 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.353 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.353 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:38:41.353 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:50.526455+0000", 2023-09-23T20:38:41.354 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.4170000000000003e-06 2023-09-23T20:38:41.354 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.354 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.354 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:38:41.354 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:50.526460+0000", 2023-09-23T20:38:41.354 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.4179999999999996e-06 2023-09-23T20:38:41.354 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.354 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.354 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:38:41.354 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:50.526681+0000", 2023-09-23T20:38:41.354 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00022085599999999999 2023-09-23T20:38:41.355 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.355 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.355 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:38:41.355 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:50.526686+0000", 2023-09-23T20:38:41.355 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.2529999999999998e-06 2023-09-23T20:38:41.355 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.355 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.355 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:38:41.355 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:51.575808+0000", 2023-09-23T20:38:41.355 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.04912205 2023-09-23T20:38:41.355 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.356 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.356 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:38:41.356 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:51.575976+0000", 2023-09-23T20:38:41.356 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00016865599999999999 2023-09-23T20:38:41.356 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.356 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.356 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:38:41.356 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:51.576200+0000", 2023-09-23T20:38:41.356 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00022404900000000001 2023-09-23T20:38:41.356 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.356 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.356 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:38:41.356 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:51.581402+0000", 2023-09-23T20:38:41.357 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0052012289999999999 2023-09-23T20:38:41.357 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.357 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.357 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:38:41.357 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:51.583225+0000", 2023-09-23T20:38:41.357 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.001823304 2023-09-23T20:38:41.357 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.357 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.357 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:38:41.357 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:51.598437+0000", 2023-09-23T20:38:41.357 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.015212652 2023-09-23T20:38:41.357 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.357 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.357 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:38:41.358 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:51.598500+0000", 2023-09-23T20:38:41.358 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.2810000000000003e-05 2023-09-23T20:38:41.358 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.358 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.358 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:38:41.358 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:51.600144+0000", 2023-09-23T20:38:41.358 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0016441050000000001 2023-09-23T20:38:41.358 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:38:41.358 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:38:41.358 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:38:41.358 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.359 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.359 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:69197 7.3 7:c10ecde9:::benchmark_data_smithi018_308601_object69196:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e320)", 2023-09-23T20:38:41.359 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:32:50.526466+0000", 2023-09-23T20:38:41.359 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 350.74072917500001, 2023-09-23T20:38:41.359 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0751176520000001, 2023-09-23T20:38:41.359 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:38:41.359 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:38:41.359 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:38:41.359 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:38:41.359 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:38:41.359 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 69197 2023-09-23T20:38:41.360 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.360 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:38:41.360 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.360 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:38:41.360 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:50.526466+0000", 2023-09-23T20:38:41.360 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:38:41.360 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.360 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.360 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:38:41.360 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:50.526466+0000", 2023-09-23T20:38:41.360 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:38:41.360 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.361 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.361 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:38:41.361 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:50.526465+0000", 2023-09-23T20:38:41.361 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:38:41.361 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.361 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.361 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:38:41.361 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:50.526504+0000", 2023-09-23T20:38:41.361 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.8685e-05 2023-09-23T20:38:41.361 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.361 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.362 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:38:41.362 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:50.526511+0000", 2023-09-23T20:38:41.362 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.1729999999999998e-06 2023-09-23T20:38:41.362 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.362 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.362 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:38:41.362 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:50.526516+0000", 2023-09-23T20:38:41.362 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.2599999999999996e-06 2023-09-23T20:38:41.362 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.362 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.362 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:38:41.363 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:50.526702+0000", 2023-09-23T20:38:41.363 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00018522600000000001 2023-09-23T20:38:41.363 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.363 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.363 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:38:41.363 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:50.526710+0000", 2023-09-23T20:38:41.363 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.2430000000000005e-06 2023-09-23T20:38:41.363 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.363 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.363 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:38:41.364 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:51.578646+0000", 2023-09-23T20:38:41.364 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0519360019999999 2023-09-23T20:38:41.364 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.364 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.364 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:38:41.364 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:51.578833+0000", 2023-09-23T20:38:41.364 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00018657499999999999 2023-09-23T20:38:41.364 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.364 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.364 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:38:41.364 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:51.579044+0000", 2023-09-23T20:38:41.365 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00021137099999999999 2023-09-23T20:38:41.365 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.365 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.365 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:38:41.365 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:51.583303+0000", 2023-09-23T20:38:41.365 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0042587650000000003 2023-09-23T20:38:41.365 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.365 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.365 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:38:41.365 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:51.588240+0000", 2023-09-23T20:38:41.366 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0049370669999999998 2023-09-23T20:38:41.366 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.366 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.366 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:38:41.366 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:51.600222+0000", 2023-09-23T20:38:41.366 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.011982418999999999 2023-09-23T20:38:41.366 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.366 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.366 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:38:41.366 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:51.600288+0000", 2023-09-23T20:38:41.367 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.6079000000000002e-05 2023-09-23T20:38:41.367 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.367 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.367 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:38:41.367 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:51.601583+0000", 2023-09-23T20:38:41.367 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.001295156 2023-09-23T20:38:41.367 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:38:41.367 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:38:41.367 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:38:41.367 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.367 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.368 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:69207 7.3 7:c71b5f56:::benchmark_data_smithi018_308601_object69206:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e320)", 2023-09-23T20:38:41.368 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:32:50.526521+0000", 2023-09-23T20:38:41.368 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 350.74067425800001, 2023-09-23T20:38:41.368 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0910981740000001, 2023-09-23T20:38:41.368 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:38:41.368 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:38:41.368 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:38:41.368 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:38:41.368 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:38:41.368 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 69207 2023-09-23T20:38:41.369 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.369 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:38:41.369 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.369 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:38:41.369 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:50.526521+0000", 2023-09-23T20:38:41.369 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:38:41.369 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.369 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.369 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:38:41.369 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:50.526521+0000", 2023-09-23T20:38:41.369 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:38:41.369 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.370 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.370 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:38:41.370 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:50.526520+0000", 2023-09-23T20:38:41.370 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:38:41.370 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.370 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.370 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:38:41.370 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:50.526561+0000", 2023-09-23T20:38:41.370 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.0983999999999999e-05 2023-09-23T20:38:41.370 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.370 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.371 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:38:41.371 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:50.526570+0000", 2023-09-23T20:38:41.371 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.8939999999999999e-06 2023-09-23T20:38:41.371 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.371 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.371 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:38:41.371 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:50.526576+0000", 2023-09-23T20:38:41.371 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.536e-06 2023-09-23T20:38:41.371 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.371 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.371 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:38:41.371 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:50.526731+0000", 2023-09-23T20:38:41.372 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00015522800000000001 2023-09-23T20:38:41.372 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.372 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.372 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:38:41.372 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:50.526739+0000", 2023-09-23T20:38:41.372 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.4169999999999992e-06 2023-09-23T20:38:41.372 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.372 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.372 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:38:41.372 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:51.583339+0000", 2023-09-23T20:38:41.372 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.056599356 2023-09-23T20:38:41.372 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.373 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.373 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:38:41.373 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:51.583498+0000", 2023-09-23T20:38:41.373 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00015957699999999999 2023-09-23T20:38:41.373 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.373 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.373 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:38:41.373 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:51.583675+0000", 2023-09-23T20:38:41.373 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000177157 2023-09-23T20:38:41.373 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.373 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.373 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:38:41.374 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:51.594254+0000", 2023-09-23T20:38:41.374 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.010578110999999999 2023-09-23T20:38:41.374 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.374 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.374 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:38:41.374 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:51.594314+0000", 2023-09-23T20:38:41.374 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.0528000000000001e-05 2023-09-23T20:38:41.374 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.374 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.374 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:38:41.374 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:51.616006+0000", 2023-09-23T20:38:41.375 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.021692211999999999 2023-09-23T20:38:41.375 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.375 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.375 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:38:41.375 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:51.616045+0000", 2023-09-23T20:38:41.375 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.8838999999999999e-05 2023-09-23T20:38:41.375 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.375 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.375 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:38:41.375 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:51.617619+0000", 2023-09-23T20:38:41.376 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.001573704 2023-09-23T20:38:41.376 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:38:41.376 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:38:41.376 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:38:41.376 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.376 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.376 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:120680 7.8 7:19893268:::benchmark_data_smithi018_308601_object26128:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e337)", 2023-09-23T20:38:41.376 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:34:04.594933+0000", 2023-09-23T20:38:41.376 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 276.672261494, 2023-09-23T20:38:41.376 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.98937692300000002, 2023-09-23T20:38:41.377 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:38:41.377 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:38:41.377 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:38:41.377 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:38:41.377 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:38:41.377 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 120680 2023-09-23T20:38:41.377 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.377 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:38:41.377 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.377 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:38:41.377 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.594933+0000", 2023-09-23T20:38:41.377 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:38:41.378 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.378 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.378 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:38:41.378 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.594933+0000", 2023-09-23T20:38:41.378 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:38:41.378 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.378 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.378 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:38:41.378 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.594933+0000", 2023-09-23T20:38:41.378 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:38:41.378 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.378 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.379 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:38:41.379 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.594936+0000", 2023-09-23T20:38:41.379 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.5360000000000001e-06 2023-09-23T20:38:41.379 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.379 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.379 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:38:41.379 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.594943+0000", 2023-09-23T20:38:41.379 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.4300000000000003e-06 2023-09-23T20:38:41.379 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.379 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.379 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:38:41.379 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.594949+0000", 2023-09-23T20:38:41.380 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.4099999999999996e-06 2023-09-23T20:38:41.380 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.380 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.380 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:38:41.380 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.598565+0000", 2023-09-23T20:38:41.380 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0036157450000000001 2023-09-23T20:38:41.380 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.380 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.380 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:38:41.380 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.598581+0000", 2023-09-23T20:38:41.380 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.5749000000000001e-05 2023-09-23T20:38:41.381 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.381 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.381 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:38:41.381 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.577402+0000", 2023-09-23T20:38:41.381 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.97882154499999996 2023-09-23T20:38:41.381 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.381 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.381 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:38:41.381 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.577521+0000", 2023-09-23T20:38:41.381 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000118606 2023-09-23T20:38:41.381 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.381 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.382 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 4,7", 2023-09-23T20:38:41.382 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.577601+0000", 2023-09-23T20:38:41.382 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.0010000000000001e-05 2023-09-23T20:38:41.382 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.382 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.382 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:38:41.382 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.581327+0000", 2023-09-23T20:38:41.382 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0037262950000000001 2023-09-23T20:38:41.382 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.382 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.382 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:38:41.383 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.581357+0000", 2023-09-23T20:38:41.383 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.9507999999999999e-05 2023-09-23T20:38:41.383 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.383 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.383 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:38:41.383 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.584253+0000", 2023-09-23T20:38:41.383 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0028957710000000001 2023-09-23T20:38:41.383 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.383 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.383 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:38:41.383 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.584284+0000", 2023-09-23T20:38:41.383 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.1109999999999999e-05 2023-09-23T20:38:41.384 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.384 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.384 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:38:41.384 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.584310+0000", 2023-09-23T20:38:41.384 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.6721e-05 2023-09-23T20:38:41.384 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:38:41.384 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:38:41.384 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:38:41.384 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.384 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.384 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:120746 7.8 7:13a4d181:::benchmark_data_smithi018_308601_object26194:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e338)", 2023-09-23T20:38:41.384 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:34:04.743456+0000", 2023-09-23T20:38:41.385 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 276.523738584, 2023-09-23T20:38:41.385 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.84858167699999998, 2023-09-23T20:38:41.385 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:38:41.385 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:38:41.385 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:38:41.385 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:38:41.385 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:38:41.385 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 120746 2023-09-23T20:38:41.385 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.385 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:38:41.385 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.385 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:38:41.386 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.743456+0000", 2023-09-23T20:38:41.386 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:38:41.386 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.386 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.386 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:38:41.386 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.743456+0000", 2023-09-23T20:38:41.386 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:38:41.386 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.386 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.386 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:38:41.386 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.743455+0000", 2023-09-23T20:38:41.387 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:38:41.387 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.387 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.387 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:38:41.387 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.743462+0000", 2023-09-23T20:38:41.387 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.3270000000000003e-06 2023-09-23T20:38:41.387 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.387 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.387 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:38:41.387 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.743480+0000", 2023-09-23T20:38:41.387 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.7609999999999999e-05 2023-09-23T20:38:41.387 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.387 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.388 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:38:41.388 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.743488+0000", 2023-09-23T20:38:41.388 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.7100000000000007e-06 2023-09-23T20:38:41.388 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.388 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.388 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:38:41.388 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.743543+0000", 2023-09-23T20:38:41.388 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.5819000000000002e-05 2023-09-23T20:38:41.388 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.388 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.388 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:38:41.388 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.743558+0000", 2023-09-23T20:38:41.389 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.4195e-05 2023-09-23T20:38:41.389 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.389 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.389 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:38:41.389 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.581377+0000", 2023-09-23T20:38:41.389 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.83781900600000003 2023-09-23T20:38:41.389 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.389 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.389 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:38:41.389 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.581461+0000", 2023-09-23T20:38:41.389 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.4777000000000005e-05 2023-09-23T20:38:41.389 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.390 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.390 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 4,7", 2023-09-23T20:38:41.390 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.581536+0000", 2023-09-23T20:38:41.390 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.4791999999999998e-05 2023-09-23T20:38:41.390 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.390 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.390 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:38:41.390 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.591780+0000", 2023-09-23T20:38:41.390 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.010243533000000001 2023-09-23T20:38:41.390 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.390 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.390 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:38:41.391 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.591833+0000", 2023-09-23T20:38:41.391 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.2916e-05 2023-09-23T20:38:41.391 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.391 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.391 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:38:41.391 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.591955+0000", 2023-09-23T20:38:41.391 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00012233 2023-09-23T20:38:41.391 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.391 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.391 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:38:41.391 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.592001+0000", 2023-09-23T20:38:41.392 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.6439e-05 2023-09-23T20:38:41.392 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.392 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.392 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:38:41.392 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.592038+0000", 2023-09-23T20:38:41.392 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.6650000000000003e-05 2023-09-23T20:38:41.392 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:38:41.392 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:38:41.392 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:38:41.392 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.392 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.393 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:120761 7.8 7:12380617:::benchmark_data_smithi018_308601_object26209:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e338)", 2023-09-23T20:38:41.393 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:34:04.772652+0000", 2023-09-23T20:38:41.393 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 276.49454330600003, 2023-09-23T20:38:41.393 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.81952633799999997, 2023-09-23T20:38:41.393 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:38:41.393 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:38:41.393 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:38:41.393 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:38:41.393 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:38:41.393 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 120761 2023-09-23T20:38:41.393 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.393 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:38:41.394 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.394 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:38:41.394 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.772652+0000", 2023-09-23T20:38:41.394 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:38:41.394 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.394 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.394 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:38:41.394 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.772652+0000", 2023-09-23T20:38:41.394 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:38:41.394 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.394 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.395 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:38:41.395 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.772650+0000", 2023-09-23T20:38:41.395 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:38:41.395 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.395 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.395 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:38:41.395 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.772660+0000", 2023-09-23T20:38:41.395 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0069e-05 2023-09-23T20:38:41.395 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.395 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.395 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:38:41.396 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.772684+0000", 2023-09-23T20:38:41.396 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.3852000000000001e-05 2023-09-23T20:38:41.396 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.396 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.396 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:38:41.396 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.772697+0000", 2023-09-23T20:38:41.396 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3505e-05 2023-09-23T20:38:41.396 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.396 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.396 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:38:41.396 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.772764+0000", 2023-09-23T20:38:41.396 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.7130000000000003e-05 2023-09-23T20:38:41.397 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.397 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.397 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:38:41.397 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.772778+0000", 2023-09-23T20:38:41.397 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.4035e-05 2023-09-23T20:38:41.397 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.397 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.397 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:38:41.397 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.584345+0000", 2023-09-23T20:38:41.397 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.81156667500000002 2023-09-23T20:38:41.398 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.398 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.398 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:38:41.398 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.584431+0000", 2023-09-23T20:38:41.398 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.6496999999999995e-05 2023-09-23T20:38:41.398 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.398 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.398 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 4,7", 2023-09-23T20:38:41.398 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.584502+0000", 2023-09-23T20:38:41.398 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.0326000000000004e-05 2023-09-23T20:38:41.398 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.399 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.399 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:38:41.399 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.591948+0000", 2023-09-23T20:38:41.399 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0074458609999999998 2023-09-23T20:38:41.399 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.399 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.399 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:38:41.399 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.592045+0000", 2023-09-23T20:38:41.399 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.7040000000000006e-05 2023-09-23T20:38:41.399 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.399 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.400 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:38:41.400 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.592097+0000", 2023-09-23T20:38:41.400 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.2416000000000001e-05 2023-09-23T20:38:41.400 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.400 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.400 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:38:41.400 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.592142+0000", 2023-09-23T20:38:41.400 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.4814999999999997e-05 2023-09-23T20:38:41.400 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.400 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.400 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:38:41.400 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.592178+0000", 2023-09-23T20:38:41.400 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.6016999999999998e-05 2023-09-23T20:38:41.401 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:38:41.401 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:38:41.401 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:38:41.401 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.401 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.401 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:120780 7.8 7:13c48ce1:::benchmark_data_smithi018_308601_object26228:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e338)", 2023-09-23T20:38:41.401 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:34:04.802248+0000", 2023-09-23T20:38:41.401 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 276.464947263, 2023-09-23T20:38:41.401 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.79010917300000005, 2023-09-23T20:38:41.401 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:38:41.401 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:38:41.401 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:38:41.402 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:38:41.402 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:38:41.402 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 120780 2023-09-23T20:38:41.402 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.402 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:38:41.402 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.402 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:38:41.402 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.802248+0000", 2023-09-23T20:38:41.402 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:38:41.402 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.402 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.402 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:38:41.403 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.802248+0000", 2023-09-23T20:38:41.403 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:38:41.403 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.403 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.403 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:38:41.403 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.802247+0000", 2023-09-23T20:38:41.403 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.999999 2023-09-23T20:38:41.403 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.403 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.403 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:38:41.403 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.802252+0000", 2023-09-23T20:38:41.404 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.541e-06 2023-09-23T20:38:41.404 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.404 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.404 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:38:41.404 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.802264+0000", 2023-09-23T20:38:41.404 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2283000000000001e-05 2023-09-23T20:38:41.404 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.404 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.404 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:38:41.404 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.802273+0000", 2023-09-23T20:38:41.404 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.49e-06 2023-09-23T20:38:41.404 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.404 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.405 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:38:41.405 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.802382+0000", 2023-09-23T20:38:41.405 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00010888 2023-09-23T20:38:41.405 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.405 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.405 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:38:41.405 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.802390+0000", 2023-09-23T20:38:41.405 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.9559999999999997e-06 2023-09-23T20:38:41.405 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.405 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.405 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:38:41.405 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.587735+0000", 2023-09-23T20:38:41.406 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.78534481 2023-09-23T20:38:41.406 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.406 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.406 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:38:41.406 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.587860+0000", 2023-09-23T20:38:41.406 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000125525 2023-09-23T20:38:41.406 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.406 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.406 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 4,7", 2023-09-23T20:38:41.406 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.587968+0000", 2023-09-23T20:38:41.406 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00010744899999999999 2023-09-23T20:38:41.407 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.407 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.407 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:38:41.407 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.592049+0000", 2023-09-23T20:38:41.407 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0040818549999999997 2023-09-23T20:38:41.407 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.407 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.407 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:38:41.407 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.592227+0000", 2023-09-23T20:38:41.407 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00017758699999999999 2023-09-23T20:38:41.407 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.408 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.408 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:38:41.408 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.592281+0000", 2023-09-23T20:38:41.408 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.4135999999999998e-05 2023-09-23T20:38:41.408 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.408 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.408 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:38:41.408 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.592323+0000", 2023-09-23T20:38:41.408 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.1607999999999999e-05 2023-09-23T20:38:41.409 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.409 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.409 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:38:41.409 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.592357+0000", 2023-09-23T20:38:41.409 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.4076999999999999e-05 2023-09-23T20:38:41.409 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:38:41.409 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:38:41.409 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:38:41.409 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.409 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.409 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:120781 7.8 7:1dfbb747:::benchmark_data_smithi018_308601_object26229:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e338)", 2023-09-23T20:38:41.410 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:34:04.802289+0000", 2023-09-23T20:38:41.410 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 276.46490594599999, 2023-09-23T20:38:41.410 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.80573468199999998, 2023-09-23T20:38:41.410 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:38:41.410 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:38:41.410 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:38:41.410 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:38:41.410 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:38:41.410 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 120781 2023-09-23T20:38:41.410 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.410 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:38:41.410 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.411 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:38:41.411 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.802289+0000", 2023-09-23T20:38:41.411 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:38:41.411 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.411 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.411 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:38:41.411 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.802289+0000", 2023-09-23T20:38:41.411 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:38:41.411 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.411 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.411 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:38:41.411 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.802288+0000", 2023-09-23T20:38:41.411 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:38:41.412 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.412 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.412 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:38:41.412 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.802292+0000", 2023-09-23T20:38:41.412 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.4769999999999999e-06 2023-09-23T20:38:41.412 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.412 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.412 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:38:41.412 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.802304+0000", 2023-09-23T20:38:41.412 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1613e-05 2023-09-23T20:38:41.412 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.412 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.413 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:38:41.413 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.802310+0000", 2023-09-23T20:38:41.413 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.3820000000000001e-06 2023-09-23T20:38:41.413 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.413 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.413 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:38:41.413 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.802404+0000", 2023-09-23T20:38:41.413 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.3603000000000005e-05 2023-09-23T20:38:41.413 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.413 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.413 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:38:41.413 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.802408+0000", 2023-09-23T20:38:41.413 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.5240000000000002e-06 2023-09-23T20:38:41.414 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.414 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.414 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:38:41.414 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.592389+0000", 2023-09-23T20:38:41.414 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.78998126300000004 2023-09-23T20:38:41.414 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.414 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.414 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:38:41.414 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.592492+0000", 2023-09-23T20:38:41.414 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000102662 2023-09-23T20:38:41.414 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.414 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.415 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 4,7", 2023-09-23T20:38:41.415 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.592606+0000", 2023-09-23T20:38:41.415 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00011369 2023-09-23T20:38:41.415 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.415 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.415 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:38:41.415 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.600025+0000", 2023-09-23T20:38:41.415 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0074188470000000001 2023-09-23T20:38:41.415 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.415 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.415 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:38:41.415 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.600036+0000", 2023-09-23T20:38:41.416 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0987e-05 2023-09-23T20:38:41.416 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.416 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.416 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:38:41.416 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.607931+0000", 2023-09-23T20:38:41.416 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0078951549999999992 2023-09-23T20:38:41.416 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.416 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.416 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:38:41.416 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.607977+0000", 2023-09-23T20:38:41.416 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.6479e-05 2023-09-23T20:38:41.416 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.417 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.417 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:38:41.417 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.608024+0000", 2023-09-23T20:38:41.417 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.6487999999999999e-05 2023-09-23T20:38:41.417 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:38:41.417 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:38:41.417 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:38:41.417 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.417 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.417 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:120828 7.8 7:1842f669:::benchmark_data_smithi018_308601_object26276:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e338)", 2023-09-23T20:38:41.417 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:34:04.911638+0000", 2023-09-23T20:38:41.417 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 276.35555701200002, 2023-09-23T20:38:41.418 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.69669138900000005, 2023-09-23T20:38:41.418 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:38:41.418 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:38:41.418 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:38:41.418 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:38:41.418 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:38:41.418 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 120828 2023-09-23T20:38:41.418 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.418 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:38:41.418 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.418 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:38:41.419 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.911638+0000", 2023-09-23T20:38:41.419 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:38:41.419 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.419 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.419 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:38:41.419 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.911638+0000", 2023-09-23T20:38:41.419 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:38:41.419 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.419 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.419 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:38:41.419 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.911636+0000", 2023-09-23T20:38:41.419 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:38:41.420 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.420 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.420 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:38:41.420 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.911651+0000", 2023-09-23T20:38:41.420 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.5067e-05 2023-09-23T20:38:41.420 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.420 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.420 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:38:41.420 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.911668+0000", 2023-09-23T20:38:41.420 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.6957999999999999e-05 2023-09-23T20:38:41.420 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.420 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.421 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:38:41.421 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.911677+0000", 2023-09-23T20:38:41.421 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.0909999999999999e-06 2023-09-23T20:38:41.421 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.421 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.421 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:38:41.421 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.911744+0000", 2023-09-23T20:38:41.421 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.6674000000000006e-05 2023-09-23T20:38:41.421 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.421 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.421 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:38:41.421 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:04.911757+0000", 2023-09-23T20:38:41.422 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2653999999999999e-05 2023-09-23T20:38:41.422 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.422 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.422 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:38:41.422 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.596774+0000", 2023-09-23T20:38:41.422 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.68501761699999997 2023-09-23T20:38:41.422 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.422 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.422 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:38:41.422 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.596892+0000", 2023-09-23T20:38:41.422 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000117624 2023-09-23T20:38:41.423 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.423 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.423 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 4,7", 2023-09-23T20:38:41.423 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.597011+0000", 2023-09-23T20:38:41.423 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000118956 2023-09-23T20:38:41.423 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.423 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.423 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:38:41.423 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.608085+0000", 2023-09-23T20:38:41.423 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.011074392000000001 2023-09-23T20:38:41.423 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.423 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.424 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:38:41.424 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.608101+0000", 2023-09-23T20:38:41.424 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.5710999999999999e-05 2023-09-23T20:38:41.424 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.424 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.424 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:38:41.424 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.608242+0000", 2023-09-23T20:38:41.424 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000141266 2023-09-23T20:38:41.424 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.424 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.424 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:38:41.424 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.608285+0000", 2023-09-23T20:38:41.425 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.2870000000000001e-05 2023-09-23T20:38:41.425 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.425 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.425 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:38:41.425 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:05.608329+0000", 2023-09-23T20:38:41.425 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.4153000000000002e-05 2023-09-23T20:38:41.425 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:38:41.425 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:38:41.425 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:38:41.425 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.425 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.425 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35994.0:51024 8.f 8:f455c68e:::benchmark_data_smithi018_341490_object51023:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e374)", 2023-09-23T20:38:41.426 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:36:42.547256+0000", 2023-09-23T20:38:41.426 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 118.71993852599999, 2023-09-23T20:38:41.426 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.15709083700000001, 2023-09-23T20:38:41.426 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:38:41.426 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:38:41.426 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:38:41.426 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35994", 2023-09-23T20:38:41.426 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:38:41.426 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 51024 2023-09-23T20:38:41.426 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.426 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:38:41.427 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.427 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:38:41.427 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:36:42.547256+0000", 2023-09-23T20:38:41.427 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:38:41.427 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.427 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.427 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:38:41.427 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:36:42.547256+0000", 2023-09-23T20:38:41.427 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:38:41.427 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.427 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.427 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:38:41.427 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:36:42.547255+0000", 2023-09-23T20:38:41.428 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:38:41.428 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.428 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.428 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:38:41.428 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:36:42.547278+0000", 2023-09-23T20:38:41.428 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.3464000000000001e-05 2023-09-23T20:38:41.428 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.428 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.428 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:38:41.428 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:36:42.547298+0000", 2023-09-23T20:38:41.428 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.9731000000000001e-05 2023-09-23T20:38:41.428 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.428 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.429 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:38:41.429 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:36:42.547308+0000", 2023-09-23T20:38:41.429 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.5759999999999995e-06 2023-09-23T20:38:41.429 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.429 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.429 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:38:41.429 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:36:42.547369+0000", 2023-09-23T20:38:41.429 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.1072000000000001e-05 2023-09-23T20:38:41.429 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.429 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.429 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:38:41.429 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:36:42.547539+0000", 2023-09-23T20:38:41.429 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00017053399999999999 2023-09-23T20:38:41.430 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.430 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.430 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:38:41.430 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:36:42.547677+0000", 2023-09-23T20:38:41.430 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00013801500000000001 2023-09-23T20:38:41.430 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.430 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.430 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:38:41.430 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:36:42.549518+0000", 2023-09-23T20:38:41.430 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.001841028 2023-09-23T20:38:41.430 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.430 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.431 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:38:41.431 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:36:42.550140+0000", 2023-09-23T20:38:41.431 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00062157399999999995 2023-09-23T20:38:41.431 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.431 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.431 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:38:41.431 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:36:42.704243+0000", 2023-09-23T20:38:41.431 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.154103238 2023-09-23T20:38:41.431 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.431 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.431 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:38:41.431 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:36:42.704307+0000", 2023-09-23T20:38:41.432 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.4120000000000003e-05 2023-09-23T20:38:41.432 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.432 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.432 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:38:41.432 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:36:42.704347+0000", 2023-09-23T20:38:41.432 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.0099999999999999e-05 2023-09-23T20:38:41.432 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:38:41.432 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:38:41.432 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:38:41.432 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.432 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.432 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35994.0:51038 8.f 8:f070768c:::benchmark_data_smithi018_341490_object51037:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e374)", 2023-09-23T20:38:41.433 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:36:42.557492+0000", 2023-09-23T20:38:41.433 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 118.709702978, 2023-09-23T20:38:41.433 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.14697069099999999, 2023-09-23T20:38:41.433 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:38:41.433 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:38:41.433 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:38:41.433 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35994", 2023-09-23T20:38:41.433 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:38:41.433 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 51038 2023-09-23T20:38:41.433 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.433 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:38:41.433 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.434 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:38:41.434 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:36:42.557492+0000", 2023-09-23T20:38:41.434 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:38:41.434 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.434 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.434 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:38:41.434 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:36:42.557492+0000", 2023-09-23T20:38:41.434 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:38:41.434 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.434 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.435 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:38:41.435 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:36:42.557491+0000", 2023-09-23T20:38:41.435 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.999999 2023-09-23T20:38:41.435 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.435 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.435 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:38:41.435 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:36:42.557508+0000", 2023-09-23T20:38:41.435 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.751e-05 2023-09-23T20:38:41.435 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.435 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.435 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:38:41.436 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:36:42.557523+0000", 2023-09-23T20:38:41.436 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.4435e-05 2023-09-23T20:38:41.436 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.436 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.436 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:38:41.436 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:36:42.557531+0000", 2023-09-23T20:38:41.436 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.4849999999999992e-06 2023-09-23T20:38:41.436 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.436 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.436 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:38:41.436 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:36:42.557591+0000", 2023-09-23T20:38:41.437 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.9945999999999998e-05 2023-09-23T20:38:41.437 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.437 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.437 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:38:41.437 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:36:42.557779+0000", 2023-09-23T20:38:41.437 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00018756799999999999 2023-09-23T20:38:41.437 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.437 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.437 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:38:41.437 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:36:42.557983+0000", 2023-09-23T20:38:41.437 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00020390100000000001 2023-09-23T20:38:41.437 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.438 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.438 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:38:41.438 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:36:42.559708+0000", 2023-09-23T20:38:41.438 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0017257520000000001 2023-09-23T20:38:41.438 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.438 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.438 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:38:41.438 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:36:42.559949+0000", 2023-09-23T20:38:41.438 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000240809 2023-09-23T20:38:41.438 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.438 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.439 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:38:41.439 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:36:42.704388+0000", 2023-09-23T20:38:41.439 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.14443837900000001 2023-09-23T20:38:41.439 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.439 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.439 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:38:41.439 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:36:42.704427+0000", 2023-09-23T20:38:41.439 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.9910000000000002e-05 2023-09-23T20:38:41.439 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.439 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.440 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:38:41.440 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:36:42.704463+0000", 2023-09-23T20:38:41.440 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.5175000000000001e-05 2023-09-23T20:38:41.440 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:38:41.440 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:38:41.440 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:38:41.440 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.440 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.440 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35994.0:105795 8.f 8:fbd22ea9:::benchmark_data_smithi018_341490_object16472:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e409)", 2023-09-23T20:38:41.440 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:38:26.417976+0000", 2023-09-23T20:38:41.440 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 14.84921859, 2023-09-23T20:38:41.441 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.35500995099999999, 2023-09-23T20:38:41.441 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:38:41.441 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:38:41.441 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:38:41.441 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35994", 2023-09-23T20:38:41.441 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:38:41.441 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 105795 2023-09-23T20:38:41.441 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.441 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:38:41.441 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.441 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:38:41.441 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.417976+0000", 2023-09-23T20:38:41.441 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:38:41.442 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.442 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.442 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:38:41.442 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.417976+0000", 2023-09-23T20:38:41.442 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:38:41.442 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.442 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.442 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:38:41.442 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.417974+0000", 2023-09-23T20:38:41.442 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:38:41.442 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.443 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.443 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:38:41.443 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.417985+0000", 2023-09-23T20:38:41.443 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0900000000000001e-05 2023-09-23T20:38:41.443 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.443 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.443 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:38:41.443 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.418013+0000", 2023-09-23T20:38:41.443 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.7627000000000002e-05 2023-09-23T20:38:41.443 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.443 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.443 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:38:41.444 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.418025+0000", 2023-09-23T20:38:41.444 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2130000000000001e-05 2023-09-23T20:38:41.444 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.444 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.444 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:38:41.444 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.418124+0000", 2023-09-23T20:38:41.444 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.8815999999999997e-05 2023-09-23T20:38:41.444 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.444 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.444 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:38:41.444 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.418247+0000", 2023-09-23T20:38:41.444 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000123249 2023-09-23T20:38:41.445 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.445 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.445 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:38:41.445 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.418326+0000", 2023-09-23T20:38:41.445 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.9097000000000005e-05 2023-09-23T20:38:41.445 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.445 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.445 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:38:41.445 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.423030+0000", 2023-09-23T20:38:41.445 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.004704387 2023-09-23T20:38:41.445 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.446 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.446 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:38:41.446 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.517001+0000", 2023-09-23T20:38:41.446 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.093970674000000004 2023-09-23T20:38:41.446 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.446 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.446 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:38:41.446 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.772873+0000", 2023-09-23T20:38:41.446 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.25587215499999999 2023-09-23T20:38:41.446 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.446 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.446 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:38:41.447 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.772933+0000", 2023-09-23T20:38:41.447 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.0099999999999997e-05 2023-09-23T20:38:41.447 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:38:41.447 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:38:41.447 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:38:41.447 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.772986+0000", 2023-09-23T20:38:41.447 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.2918999999999999e-05 2023-09-23T20:38:41.447 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:38:41.447 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:38:41.447 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:38:41.447 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:38:41.447 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:38:41.448 INFO:teuthology.orchestra.run.smithi018.stdout:} 2023-09-23T20:38:41.448 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:38:41.502 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:41.566 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:41 smithi018 ceph-mon[184144]: pgmap v1011: 129 pgs: 2 remapped+peering, 11 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+degraded, 2 active+recovery_wait+degraded+remapped, 110 active+clean; 4.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 26 op/s; 31343/217617 objects degraded (14.403%); 5036/217617 objects misplaced (2.314%); 27 MiB/s, 472 objects/s recovering 2023-09-23T20:38:41.566 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:41 smithi018 ceph-mon[184144]: osdmap e419: 8 total, 8 up, 8 in 2023-09-23T20:38:41.566 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:41 smithi018 ceph-mon[184144]: failed to encode map e419 with expected crc 2023-09-23T20:38:41.566 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:41 smithi018 ceph-mon[184144]: failed to encode map e419 with expected crc 2023-09-23T20:38:41.566 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:41 smithi018 ceph-mon[184144]: failed to encode map e419 with expected crc 2023-09-23T20:38:41.566 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:41 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 31343/217617 objects degraded (14.403%), 16 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T20:38:41.567 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:41 smithi018 ceph-mon[179647]: pgmap v1011: 129 pgs: 2 remapped+peering, 11 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+degraded, 2 active+recovery_wait+degraded+remapped, 110 active+clean; 4.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 26 op/s; 31343/217617 objects degraded (14.403%); 5036/217617 objects misplaced (2.314%); 27 MiB/s, 472 objects/s recovering 2023-09-23T20:38:41.567 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:41 smithi018 ceph-mon[179647]: osdmap e419: 8 total, 8 up, 8 in 2023-09-23T20:38:41.567 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:41 smithi018 ceph-mon[179647]: failed to encode map e419 with expected crc 2023-09-23T20:38:41.567 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:41 smithi018 ceph-mon[179647]: failed to encode map e419 with expected crc 2023-09-23T20:38:41.567 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:41 smithi018 ceph-mon[179647]: failed to encode map e419 with expected crc 2023-09-23T20:38:41.567 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:41 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 31343/217617 objects degraded (14.403%), 16 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T20:38:41.698 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:38:41.703 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:41.704 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:38:41.804 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:38:41.830 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:41 smithi149 ceph-mon[170394]: pgmap v1011: 129 pgs: 2 remapped+peering, 11 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+degraded, 2 active+recovery_wait+degraded+remapped, 110 active+clean; 4.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 26 op/s; 31343/217617 objects degraded (14.403%); 5036/217617 objects misplaced (2.314%); 27 MiB/s, 472 objects/s recovering 2023-09-23T20:38:41.830 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:41 smithi149 ceph-mon[170394]: osdmap e419: 8 total, 8 up, 8 in 2023-09-23T20:38:41.830 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:41 smithi149 ceph-mon[170394]: failed to encode map e419 with expected crc 2023-09-23T20:38:41.830 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:41 smithi149 ceph-mon[170394]: failed to encode map e419 with expected crc 2023-09-23T20:38:41.830 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:41 smithi149 ceph-mon[170394]: failed to encode map e419 with expected crc 2023-09-23T20:38:41.830 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:41 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 31343/217617 objects degraded (14.403%), 16 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T20:38:41.833 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:41.874 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:38:41.878 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:41.878 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:38:41.889 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T20:38:41.979 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:38:42.040 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:42.192 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:38:42.197 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:42.198 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:38:42.298 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:38:42.328 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:42.373 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:38:42.375 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:42.376 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:38:42.477 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:38:42.488 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:38:42.540 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:42.588 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:38:42.593 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:42.593 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:38:42.694 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:38:42.738 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:42.906 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:38:42.911 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:42.912 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:38:42.921 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:38:42.947 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:38:42.982 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:38:43.013 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:38:43.018 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:38:43.054 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:38:43.083 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:43.088 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:38:43.133 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:38:43.167 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:38:43.204 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:38:43.210 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:38:43.210 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:38:43.210 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:38:43.211 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:38:43.211 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:38:43.211 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:38:43.211 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:38:43.211 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:38:43.745 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:43 smithi018 ceph-mon[179647]: pgmap v1013: 129 pgs: 1 peering, 1 remapped+peering, 8 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+degraded, 111 active+clean; 4.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 95 op/s; 24423/216624 objects degraded (11.274%); 5036/216624 objects misplaced (2.325%); 23 MiB/s, 411 objects/s recovering 2023-09-23T20:38:43.746 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:43 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2781674509' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:38:43.746 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:43 smithi018 ceph-mon[184144]: pgmap v1013: 129 pgs: 1 peering, 1 remapped+peering, 8 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+degraded, 111 active+clean; 4.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 95 op/s; 24423/216624 objects degraded (11.274%); 5036/216624 objects misplaced (2.325%); 23 MiB/s, 411 objects/s recovering 2023-09-23T20:38:43.746 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:43 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2781674509' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:38:43.757 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:38:43.764 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:43.764 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:38:43.865 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:38:43.892 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:43 smithi149 ceph-mon[170394]: pgmap v1013: 129 pgs: 1 peering, 1 remapped+peering, 8 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 1 active+degraded, 111 active+clean; 4.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 95 op/s; 24423/216624 objects degraded (11.274%); 5036/216624 objects misplaced (2.325%); 23 MiB/s, 411 objects/s recovering 2023-09-23T20:38:43.892 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:43 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2781674509' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:38:43.895 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:43.935 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:38:43.938 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:43.938 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:38:44.039 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:38:44.089 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:44.513 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:38:44.517 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:44.518 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:38:44.618 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:38:44.675 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:44.796 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:38:44.800 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:44 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:38:44.800 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:44 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:38:44.801 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:44.801 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:38:44.902 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:38:44.961 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:44.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:44 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:38:45.069 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:38:45.074 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:45.074 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:38:44 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:38:44] "GET /metrics HTTP/1.1" 200 36566 "" "Prometheus/2.43.0" 2023-09-23T20:38:45.075 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:38:45.176 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:38:45.213 INFO:teuthology.orchestra.run.smithi018.stdout:ERROR: (22) Invalid argument 2023-09-23T20:38:45.214 INFO:teuthology.orchestra.run.smithi018.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:38:45.235 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:45.351 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:38:45.356 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:45.357 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:38:45.421 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 7, 3, 2, 4, 6, 5] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T20:38:45.422 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:38:45.422 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_3 2023-09-23T20:38:45.423 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:38:45.458 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:38:45.520 INFO:teuthology.orchestra.run.smithi018.stderr:nodeep-scrub is unset 2023-09-23T20:38:45.545 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:45.644 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:45 smithi018 ceph-mon[179647]: pgmap v1014: 129 pgs: 1 peering, 9 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 4.4 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 219 op/s; 24219/213984 objects degraded (11.318%); 4521/213984 objects misplaced (2.113%); 24 MiB/s, 376 objects/s recovering 2023-09-23T20:38:45.644 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:45 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1179359836' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:38:45.644 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:45 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:38:45.645 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:45 smithi018 ceph-mon[184144]: pgmap v1014: 129 pgs: 1 peering, 9 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 4.4 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 219 op/s; 24219/213984 objects degraded (11.318%); 4521/213984 objects misplaced (2.113%); 24 MiB/s, 376 objects/s recovering 2023-09-23T20:38:45.645 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:45 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1179359836' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:38:45.645 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:45 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:38:45.672 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T20:38:45.692 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:38:45.697 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:45.698 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:38:45.748 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:38:45.799 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:38:45.825 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:45 smithi149 ceph-mon[170394]: pgmap v1014: 129 pgs: 1 peering, 9 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 4.4 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 219 op/s; 24219/213984 objects degraded (11.318%); 4521/213984 objects misplaced (2.113%); 24 MiB/s, 376 objects/s recovering 2023-09-23T20:38:45.825 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:45 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1179359836' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:38:45.825 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:45 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:38:45.828 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:45.869 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:38:45.873 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:45.874 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:38:45.974 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:38:46.004 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:46.049 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:38:46.056 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:46.057 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:38:46.158 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:38:46.186 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:46.233 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:38:46.236 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:46.236 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:38:46.292 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:38:46.337 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:38:46.404 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:46.584 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:38:46.589 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:46.589 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:38:46.589 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:38:46.690 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:38:46.718 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:46.760 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:38:46.763 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:46.763 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:38:46.801 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:46 smithi018 ceph-mon[184144]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:38:46.801 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:46 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:38:46.801 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:46 smithi018 ceph-mon[184144]: osdmap e420: 8 total, 8 up, 8 in 2023-09-23T20:38:46.801 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:46 smithi018 ceph-mon[184144]: failed to encode map e420 with expected crc 2023-09-23T20:38:46.801 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:46 smithi018 ceph-mon[184144]: failed to encode map e420 with expected crc 2023-09-23T20:38:46.801 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:46 smithi018 ceph-mon[184144]: failed to encode map e420 with expected crc 2023-09-23T20:38:46.801 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:46 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 24219/213984 objects degraded (11.318%), 15 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T20:38:46.805 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:46 smithi018 ceph-mon[179647]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:38:46.805 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:46 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:38:46.806 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:46 smithi018 ceph-mon[179647]: osdmap e420: 8 total, 8 up, 8 in 2023-09-23T20:38:46.806 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:46 smithi018 ceph-mon[179647]: failed to encode map e420 with expected crc 2023-09-23T20:38:46.806 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:46 smithi018 ceph-mon[179647]: failed to encode map e420 with expected crc 2023-09-23T20:38:46.806 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:46 smithi018 ceph-mon[179647]: failed to encode map e420 with expected crc 2023-09-23T20:38:46.806 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:46 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 24219/213984 objects degraded (11.318%), 15 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T20:38:46.863 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:38:46.930 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:46.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:46 smithi149 ceph-mon[170394]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:38:46.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:46 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:38:46.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:46 smithi149 ceph-mon[170394]: osdmap e420: 8 total, 8 up, 8 in 2023-09-23T20:38:46.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:46 smithi149 ceph-mon[170394]: failed to encode map e420 with expected crc 2023-09-23T20:38:46.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:46 smithi149 ceph-mon[170394]: failed to encode map e420 with expected crc 2023-09-23T20:38:46.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:46 smithi149 ceph-mon[170394]: failed to encode map e420 with expected crc 2023-09-23T20:38:46.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:46 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 24219/213984 objects degraded (11.318%), 15 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T20:38:47.089 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:38:47.094 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:47.094 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:38:47.193 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:38:47.195 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:38:47.258 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:47.332 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:38:47.339 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:47.340 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:38:47.441 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:38:47.498 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:47.813 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:47 smithi018 ceph-mon[179647]: pgmap v1016: 129 pgs: 9 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 4.3 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 331 op/s; 24079/211344 objects degraded (11.393%); 4279/211344 objects misplaced (2.025%); 14 MiB/s, 230 objects/s recovering 2023-09-23T20:38:47.814 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:47 smithi018 ceph-mon[184144]: pgmap v1016: 129 pgs: 9 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 4.3 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 331 op/s; 24079/211344 objects degraded (11.393%); 4279/211344 objects misplaced (2.025%); 14 MiB/s, 230 objects/s recovering 2023-09-23T20:38:47.937 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T20:38:47.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:47 smithi149 ceph-mon[170394]: pgmap v1016: 129 pgs: 9 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 4.3 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 331 op/s; 24079/211344 objects degraded (11.393%); 4279/211344 objects misplaced (2.025%); 14 MiB/s, 230 objects/s recovering 2023-09-23T20:38:49.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:49 smithi018 ceph-mon[184144]: pgmap v1017: 129 pgs: 9 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 4.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 383 op/s; 23941/208398 objects degraded (11.488%); 4279/208398 objects misplaced (2.053%); 9.7 MiB/s, 154 objects/s recovering 2023-09-23T20:38:49.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:49 smithi018 ceph-mon[179647]: pgmap v1017: 129 pgs: 9 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 4.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 383 op/s; 23941/208398 objects degraded (11.488%); 4279/208398 objects misplaced (2.053%); 9.7 MiB/s, 154 objects/s recovering 2023-09-23T20:38:49.924 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:38:49.929 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:49.930 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:38:49.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:49 smithi149 ceph-mon[170394]: pgmap v1017: 129 pgs: 9 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 4.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 383 op/s; 23941/208398 objects degraded (11.488%); 4279/208398 objects misplaced (2.053%); 9.7 MiB/s, 154 objects/s recovering 2023-09-23T20:38:50.031 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:38:50.062 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:50.104 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:38:50.107 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:50.107 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:38:50.208 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:38:50.269 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:50.530 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:50 smithi018 ceph-mon[179647]: 1.a scrub starts 2023-09-23T20:38:50.691 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:38:50.698 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:50.698 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:38:50.791 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:50 smithi018 ceph-mon[184144]: 1.a scrub starts 2023-09-23T20:38:50.799 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:38:50.858 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:50.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:50 smithi149 ceph-mon[170394]: 1.a scrub starts 2023-09-23T20:38:51.097 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:38:51.101 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:51.102 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:38:51.202 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:38:51.250 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:51.315 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:38:51.320 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:51.320 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:38:51.421 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:38:51.470 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:38:51.491 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:51.628 INFO:teuthology.orchestra.run.smithi018.stdout:ERROR: (22) Invalid argument 2023-09-23T20:38:51.628 INFO:teuthology.orchestra.run.smithi018.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:38:51.799 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:38:51.807 INFO:teuthology.orchestra.run.smithi018.stderr:dumped all 2023-09-23T20:38:51.842 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:51 smithi018 ceph-mon[179647]: pgmap v1018: 129 pgs: 9 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 4.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 418 op/s; 23715/205614 objects degraded (11.534%); 3348/205614 objects misplaced (1.628%); 16 MiB/s, 250 objects/s recovering 2023-09-23T20:38:51.842 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:51 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 23715/205614 objects degraded (11.534%), 15 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T20:38:51.843 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:51 smithi018 ceph-mon[184144]: pgmap v1018: 129 pgs: 9 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 4.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 418 op/s; 23715/205614 objects degraded (11.534%); 3348/205614 objects misplaced (1.628%); 16 MiB/s, 250 objects/s recovering 2023-09-23T20:38:51.843 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:51 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 23715/205614 objects degraded (11.534%), 15 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T20:38:51.863 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:38:51.868 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:51.868 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:38:51.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:51 smithi149 ceph-mon[170394]: pgmap v1018: 129 pgs: 9 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 4.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 418 op/s; 23715/205614 objects degraded (11.534%); 3348/205614 objects misplaced (1.628%); 16 MiB/s, 250 objects/s recovering 2023-09-23T20:38:51.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:51 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 23715/205614 objects degraded (11.534%), 15 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T20:38:51.970 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:38:52.002 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:52.038 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:38:52.042 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:52.043 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:38:52.143 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:38:52.171 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:52.214 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:38:52.218 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:52.219 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:38:52.319 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:38:52.354 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:52.403 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:38:52.408 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:52.408 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:38:52.480 INFO:teuthology.orchestra.run.smithi018.stdout:{"pg_ready":true,"pg_map":{"version":1018,"stamp":"2023-09-23T20:38:50.036507+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4494422895,"num_objects":68538,"num_object_clones":0,"num_object_copies":205614,"num_objects_missing_on_primary":1011,"num_objects_missing":1011,"num_objects_degraded":23715,"num_objects_misplaced":3348,"num_objects_unfound":0,"num_objects_dirty":68538,"num_whiteouts":0,"num_read":6139,"num_read_kb":13491,"num_write":111114,"num_write_kb":5729974,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14027,"num_bytes_recovered":885383474,"num_keys_recovered":40,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":137501,"ondisk_log_size":137501,"up":387,"acting":381,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":424,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":18033376,"kb_used_data":14320224,"kb_used_omap":32,"kb_used_meta":3712927,"kb_avail":731927840,"statfs":{"total":767960285184,"available":749494108160,"internally_reserved":0,"allocated":14663909376,"data_stored":14650074068,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32773,"internal_metadata":3802038267},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-262209536,"num_objects":-4001,"num_object_clones":0,"num_object_copies":-12003,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-7618,"num_objects_misplaced":-1688,"num_objects_unfound":0,"num_objects_dirty":-4001,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4001,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2396,"num_bytes_recovered":157024256,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.564494"},"pg_stats":[{"pgid":"8.11","version":"420'6782","reported_seq":6853,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-23T20:38:49.451270+0000","last_change":"2023-09-23T20:38:27.478425+0000","last_active":"2023-09-23T20:38:49.451270+0000","last_peered":"2023-09-23T20:38:49.451270+0000","last_clean":"2023-09-23T20:38:49.451270+0000","last_became_active":"2023-09-23T20:38:27.472168+0000","last_became_peered":"2023-09-23T20:38:27.472168+0000","last_unstale":"2023-09-23T20:38:49.451270+0000","last_undegraded":"2023-09-23T20:38:49.451270+0000","last_fullsized":"2023-09-23T20:38:49.451270+0000","mapping_epoch":410,"log_start":"373'1694","ondisk_log_start":"373'1694","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":5088,"log_dups_size":0,"ondisk_log_size":5088,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":138313728,"num_objects":2111,"num_object_clones":0,"num_object_copies":6333,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2111,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3473,"num_write_kb":178656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.18","version":"90'161","reported_seq":480,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T20:38:40.478513+0000","last_change":"2023-09-23T20:38:03.548189+0000","last_active":"2023-09-23T20:38:40.478513+0000","last_peered":"2023-09-23T20:38:40.478513+0000","last_clean":"2023-09-23T20:38:40.478513+0000","last_became_active":"2023-09-23T20:38:03.547630+0000","last_became_peered":"2023-09-23T20:38:03.547630+0000","last_unstale":"2023-09-23T20:38:40.478513+0000","last_undegraded":"2023-09-23T20:38:40.478513+0000","last_fullsized":"2023-09-23T20:38:40.478513+0000","mapping_epoch":404,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":405,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:35:03.551143+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:35:03.551143+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":343193,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"69'1","reported_seq":117,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T20:38:40.486955+0000","last_change":"2023-09-23T20:32:33.233853+0000","last_active":"2023-09-23T20:38:40.486955+0000","last_peered":"2023-09-23T20:38:40.486955+0000","last_clean":"2023-09-23T20:38:40.486955+0000","last_became_active":"2023-09-23T20:32:33.233546+0000","last_became_peered":"2023-09-23T20:32:33.233546+0000","last_unstale":"2023-09-23T20:38:40.486955+0000","last_undegraded":"2023-09-23T20:38:40.486955+0000","last_fullsized":"2023-09-23T20:38:40.486955+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:50:27.130808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":232,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T20:38:40.493919+0000","last_change":"2023-09-23T20:37:43.579611+0000","last_active":"2023-09-23T20:38:40.493919+0000","last_peered":"2023-09-23T20:38:40.493919+0000","last_clean":"2023-09-23T20:38:40.493919+0000","last_became_active":"2023-09-23T20:37:43.579323+0000","last_became_peered":"2023-09-23T20:37:43.579323+0000","last_unstale":"2023-09-23T20:38:40.493919+0000","last_undegraded":"2023-09-23T20:38:40.493919+0000","last_fullsized":"2023-09-23T20:38:40.493919+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:17:32.311427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.12","version":"420'6803","reported_seq":6922,"reported_epoch":420,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:38:45.511880+0000","last_change":"2023-09-23T20:38:39.273131+0000","last_active":"2023-09-23T20:38:45.511880+0000","last_peered":"2023-09-23T20:38:45.511880+0000","last_clean":"2023-09-23T20:37:29.399979+0000","last_became_active":"2023-09-23T20:38:39.252214+0000","last_became_peered":"2023-09-23T20:38:39.252214+0000","last_unstale":"2023-09-23T20:38:45.511880+0000","last_undegraded":"2023-09-23T20:38:38.456024+0000","last_fullsized":"2023-09-23T20:38:38.449466+0000","mapping_epoch":416,"log_start":"373'1996","ondisk_log_start":"373'1996","created":410,"last_epoch_clean":370,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":4807,"log_dups_size":0,"ondisk_log_size":4807,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":141623296,"num_objects":2161,"num_object_clones":0,"num_object_copies":6483,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":715,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2161,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3453,"num_write_kb":179648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":753664,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0],"avail_no_missing":["4","0"],"object_location_counts":[{"shards":"0,3,4","objects":141},{"shards":"0,4","objects":2020}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1b","version":"90'161","reported_seq":538,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T20:38:40.485561+0000","last_change":"2023-09-23T20:38:39.106622+0000","last_active":"2023-09-23T20:38:40.485561+0000","last_peered":"2023-09-23T20:38:40.485561+0000","last_clean":"2023-09-23T20:38:40.485561+0000","last_became_active":"2023-09-23T20:37:17.291961+0000","last_became_peered":"2023-09-23T20:37:17.291961+0000","last_unstale":"2023-09-23T20:38:40.485561+0000","last_undegraded":"2023-09-23T20:38:40.485561+0000","last_fullsized":"2023-09-23T20:38:40.485561+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:05.442288+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:36:05.442288+0000","last_clean_scrub_stamp":"2023-09-23T20:36:05.442288+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0019500940000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,2],"acting":[4,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":387,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T20:38:40.474218+0000","last_change":"2023-09-23T20:34:59.983035+0000","last_active":"2023-09-23T20:38:40.474218+0000","last_peered":"2023-09-23T20:38:40.474218+0000","last_clean":"2023-09-23T20:38:40.474218+0000","last_became_active":"2023-09-23T20:34:59.982459+0000","last_became_peered":"2023-09-23T20:34:59.982459+0000","last_unstale":"2023-09-23T20:38:40.474218+0000","last_undegraded":"2023-09-23T20:38:40.474218+0000","last_fullsized":"2023-09-23T20:38:40.474218+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:16:48.510783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":389,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T20:38:40.477768+0000","last_change":"2023-09-23T20:25:02.957185+0000","last_active":"2023-09-23T20:38:40.477768+0000","last_peered":"2023-09-23T20:38:40.477768+0000","last_clean":"2023-09-23T20:38:40.477768+0000","last_became_active":"2023-09-23T20:25:02.956689+0000","last_became_peered":"2023-09-23T20:25:02.956689+0000","last_unstale":"2023-09-23T20:38:40.477768+0000","last_undegraded":"2023-09-23T20:38:40.477768+0000","last_fullsized":"2023-09-23T20:38:40.477768+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:04:28.524015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.13","version":"420'6744","reported_seq":8162,"reported_epoch":420,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T20:38:49.451034+0000","last_change":"2023-09-23T20:38:27.598846+0000","last_active":"2023-09-23T20:38:49.451034+0000","last_peered":"2023-09-23T20:38:49.451034+0000","last_clean":"2023-09-23T20:37:13.297905+0000","last_became_active":"2023-09-23T20:38:27.564876+0000","last_became_peered":"2023-09-23T20:38:27.564876+0000","last_unstale":"2023-09-23T20:38:49.451034+0000","last_undegraded":"2023-09-23T20:38:27.556546+0000","last_fullsized":"2023-09-23T20:38:49.451034+0000","mapping_epoch":410,"log_start":"373'1680","ondisk_log_start":"373'1680","created":410,"last_epoch_clean":370,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":5064,"log_dups_size":0,"ondisk_log_size":5064,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":141262848,"num_objects":2156,"num_object_clones":0,"num_object_copies":6468,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2660,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2156,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3438,"num_write_kb":178976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":308,"num_bytes_recovered":20152320,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":["1","5"],"object_location_counts":[{"shards":"1,2,3","objects":163},{"shards":"1,2,6","objects":-6},{"shards":"1,5,6","objects":1999}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1a","version":"90'161","reported_seq":499,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T20:38:40.486149+0000","last_change":"2023-09-23T20:36:01.292103+0000","last_active":"2023-09-23T20:38:40.486149+0000","last_peered":"2023-09-23T20:38:40.486149+0000","last_clean":"2023-09-23T20:38:40.486149+0000","last_became_active":"2023-09-23T20:35:26.639912+0000","last_became_peered":"2023-09-23T20:35:26.639912+0000","last_unstale":"2023-09-23T20:38:40.486149+0000","last_undegraded":"2023-09-23T20:38:40.486149+0000","last_fullsized":"2023-09-23T20:38:40.486149+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:01.292032+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:01.292032+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:04:53.926640+0000","scrub_duration":0.0032983159999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":212,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T20:38:40.495207+0000","last_change":"2023-09-23T20:37:43.584414+0000","last_active":"2023-09-23T20:38:40.495207+0000","last_peered":"2023-09-23T20:38:40.495207+0000","last_clean":"2023-09-23T20:38:40.495207+0000","last_became_active":"2023-09-23T20:37:43.584116+0000","last_became_peered":"2023-09-23T20:37:43.584116+0000","last_unstale":"2023-09-23T20:38:40.495207+0000","last_undegraded":"2023-09-23T20:38:40.495207+0000","last_fullsized":"2023-09-23T20:38:40.495207+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:17:54.258172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":36,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T20:38:40.482637+0000","last_change":"2023-09-23T20:38:37.457427+0000","last_active":"2023-09-23T20:38:40.482637+0000","last_peered":"2023-09-23T20:38:40.482637+0000","last_clean":"2023-09-23T20:38:40.482637+0000","last_became_active":"2023-09-23T20:38:37.457099+0000","last_became_peered":"2023-09-23T20:38:37.457099+0000","last_unstale":"2023-09-23T20:38:40.482637+0000","last_undegraded":"2023-09-23T20:38:40.482637+0000","last_fullsized":"2023-09-23T20:38:40.482637+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:31:27.084994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"90'161","reported_seq":612,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T20:38:40.500486+0000","last_change":"2023-09-23T20:37:17.304763+0000","last_active":"2023-09-23T20:38:40.500486+0000","last_peered":"2023-09-23T20:38:40.500486+0000","last_clean":"2023-09-23T20:38:40.500486+0000","last_became_active":"2023-09-23T20:37:17.304362+0000","last_became_peered":"2023-09-23T20:37:17.304362+0000","last_unstale":"2023-09-23T20:38:40.500486+0000","last_undegraded":"2023-09-23T20:38:40.500486+0000","last_fullsized":"2023-09-23T20:38:40.500486+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:03.411025+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:03.411025+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:11:42.762720+0000","scrub_duration":0.001802922,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":413,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T20:38:40.477061+0000","last_change":"2023-09-23T20:38:02.606871+0000","last_active":"2023-09-23T20:38:40.477061+0000","last_peered":"2023-09-23T20:38:40.477061+0000","last_clean":"2023-09-23T20:38:40.477061+0000","last_became_active":"2023-09-23T20:38:02.606596+0000","last_became_peered":"2023-09-23T20:38:02.606596+0000","last_unstale":"2023-09-23T20:38:40.477061+0000","last_undegraded":"2023-09-23T20:38:40.477061+0000","last_fullsized":"2023-09-23T20:38:40.477061+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:27:33.341667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":142,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T20:38:40.478633+0000","last_change":"2023-09-23T20:38:02.579343+0000","last_active":"2023-09-23T20:38:40.478633+0000","last_peered":"2023-09-23T20:38:40.478633+0000","last_clean":"2023-09-23T20:38:40.478633+0000","last_became_active":"2023-09-23T20:38:02.579047+0000","last_became_peered":"2023-09-23T20:38:02.579047+0000","last_unstale":"2023-09-23T20:38:40.478633+0000","last_undegraded":"2023-09-23T20:38:40.478633+0000","last_fullsized":"2023-09-23T20:38:40.478633+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:44:03.166556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.14","version":"90'161","reported_seq":748,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T20:38:40.479228+0000","last_change":"2023-09-23T20:35:02.236398+0000","last_active":"2023-09-23T20:38:40.479228+0000","last_peered":"2023-09-23T20:38:40.479228+0000","last_clean":"2023-09-23T20:38:40.479228+0000","last_became_active":"2023-09-23T20:35:02.235988+0000","last_became_peered":"2023-09-23T20:35:02.235988+0000","last_unstale":"2023-09-23T20:38:40.479228+0000","last_undegraded":"2023-09-23T20:38:40.479228+0000","last_fullsized":"2023-09-23T20:38:40.479228+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:47.479012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:47.479012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:13.519648+0000","scrub_duration":0.0020136590000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":287,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T20:38:40.500734+0000","last_change":"2023-09-23T20:37:14.362069+0000","last_active":"2023-09-23T20:38:40.500734+0000","last_peered":"2023-09-23T20:38:40.500734+0000","last_clean":"2023-09-23T20:38:40.500734+0000","last_became_active":"2023-09-23T20:37:14.361772+0000","last_became_peered":"2023-09-23T20:37:14.361772+0000","last_unstale":"2023-09-23T20:38:40.500734+0000","last_undegraded":"2023-09-23T20:38:40.500734+0000","last_fullsized":"2023-09-23T20:38:40.500734+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:20:31.626104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":243,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T20:38:40.494108+0000","last_change":"2023-09-23T20:38:02.542034+0000","last_active":"2023-09-23T20:38:40.494108+0000","last_peered":"2023-09-23T20:38:40.494108+0000","last_clean":"2023-09-23T20:38:40.494108+0000","last_became_active":"2023-09-23T20:38:02.541575+0000","last_became_peered":"2023-09-23T20:38:02.541575+0000","last_unstale":"2023-09-23T20:38:40.494108+0000","last_undegraded":"2023-09-23T20:38:40.494108+0000","last_fullsized":"2023-09-23T20:38:40.494108+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:50:54.591529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"90'161","reported_seq":562,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T20:38:40.476848+0000","last_change":"2023-09-23T20:38:05.611362+0000","last_active":"2023-09-23T20:38:40.476848+0000","last_peered":"2023-09-23T20:38:40.476848+0000","last_clean":"2023-09-23T20:38:40.476848+0000","last_became_active":"2023-09-23T20:38:05.611031+0000","last_became_peered":"2023-09-23T20:38:05.611031+0000","last_unstale":"2023-09-23T20:38:40.476848+0000","last_undegraded":"2023-09-23T20:38:40.476848+0000","last_fullsized":"2023-09-23T20:38:40.476848+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:03.435480+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:03.435480+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":447,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T20:38:40.500796+0000","last_change":"2023-09-23T20:32:33.270617+0000","last_active":"2023-09-23T20:38:40.500796+0000","last_peered":"2023-09-23T20:38:40.500796+0000","last_clean":"2023-09-23T20:38:40.500796+0000","last_became_active":"2023-09-23T20:32:33.227902+0000","last_became_peered":"2023-09-23T20:32:33.227902+0000","last_unstale":"2023-09-23T20:38:40.500796+0000","last_undegraded":"2023-09-23T20:38:40.500796+0000","last_fullsized":"2023-09-23T20:38:40.500796+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:30:26.847831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":360,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":388,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T20:38:40.474956+0000","last_change":"2023-09-23T20:38:02.599068+0000","last_active":"2023-09-23T20:38:40.474956+0000","last_peered":"2023-09-23T20:38:40.474956+0000","last_clean":"2023-09-23T20:38:40.474956+0000","last_became_active":"2023-09-23T20:38:02.597602+0000","last_became_peered":"2023-09-23T20:38:02.597602+0000","last_unstale":"2023-09-23T20:38:40.474956+0000","last_undegraded":"2023-09-23T20:38:40.474956+0000","last_fullsized":"2023-09-23T20:38:40.474956+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:34:06.337029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"90'161","reported_seq":543,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T20:38:40.475022+0000","last_change":"2023-09-23T20:36:02.548387+0000","last_active":"2023-09-23T20:38:40.475022+0000","last_peered":"2023-09-23T20:38:40.475022+0000","last_clean":"2023-09-23T20:38:40.475022+0000","last_became_active":"2023-09-23T20:35:26.644787+0000","last_became_peered":"2023-09-23T20:35:26.644787+0000","last_unstale":"2023-09-23T20:38:40.475022+0000","last_undegraded":"2023-09-23T20:38:40.475022+0000","last_fullsized":"2023-09-23T20:38:40.475022+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:02.548285+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:02.548285+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:37:15.216553+0000","scrub_duration":0.0015514459999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":167,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T20:38:40.486171+0000","last_change":"2023-09-23T20:38:02.560646+0000","last_active":"2023-09-23T20:38:40.486171+0000","last_peered":"2023-09-23T20:38:40.486171+0000","last_clean":"2023-09-23T20:38:40.486171+0000","last_became_active":"2023-09-23T20:38:02.539814+0000","last_became_peered":"2023-09-23T20:38:02.539814+0000","last_unstale":"2023-09-23T20:38:40.486171+0000","last_undegraded":"2023-09-23T20:38:40.486171+0000","last_fullsized":"2023-09-23T20:38:40.486171+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:00:09.435023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":147,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T20:38:40.486183+0000","last_change":"2023-09-23T20:38:02.579222+0000","last_active":"2023-09-23T20:38:40.486183+0000","last_peered":"2023-09-23T20:38:40.486183+0000","last_clean":"2023-09-23T20:38:40.486183+0000","last_became_active":"2023-09-23T20:38:02.578813+0000","last_became_peered":"2023-09-23T20:38:02.578813+0000","last_unstale":"2023-09-23T20:38:40.486183+0000","last_undegraded":"2023-09-23T20:38:40.486183+0000","last_fullsized":"2023-09-23T20:38:40.486183+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:57:31.080027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.18","version":"420'6773","reported_seq":9211,"reported_epoch":420,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T20:38:48.851139+0000","last_change":"2023-09-23T20:38:27.606849+0000","last_active":"2023-09-23T20:38:48.851139+0000","last_peered":"2023-09-23T20:38:48.851139+0000","last_clean":"2023-09-23T20:37:13.320723+0000","last_became_active":"2023-09-23T20:38:27.595968+0000","last_became_peered":"2023-09-23T20:38:27.595968+0000","last_unstale":"2023-09-23T20:38:48.851139+0000","last_undegraded":"2023-09-23T20:38:27.539992+0000","last_fullsized":"2023-09-23T20:38:48.851139+0000","mapping_epoch":410,"log_start":"373'2334","ondisk_log_start":"373'2334","created":410,"last_epoch_clean":370,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":4439,"log_dups_size":0,"ondisk_log_size":4439,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":139558912,"num_objects":2130,"num_object_clones":0,"num_object_copies":6390,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1995,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2130,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3465,"num_write_kb":179008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":667,"num_bytes_recovered":43712512,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":["3","5","7"],"object_location_counts":[{"shards":"0,3,7","objects":135},{"shards":"3,5,7","objects":1995}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.11","version":"90'161","reported_seq":668,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T20:38:40.477580+0000","last_change":"2023-09-23T20:37:17.282654+0000","last_active":"2023-09-23T20:38:40.477580+0000","last_peered":"2023-09-23T20:38:40.477580+0000","last_clean":"2023-09-23T20:38:40.477580+0000","last_became_active":"2023-09-23T20:37:17.282316+0000","last_became_peered":"2023-09-23T20:37:17.282316+0000","last_unstale":"2023-09-23T20:38:40.477580+0000","last_undegraded":"2023-09-23T20:38:40.477580+0000","last_fullsized":"2023-09-23T20:38:40.477580+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:43.578666+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:43.578666+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:53:37.101450+0000","scrub_duration":0.0021890379999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":446,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T20:38:40.497638+0000","last_change":"2023-09-23T20:32:33.242943+0000","last_active":"2023-09-23T20:38:40.497638+0000","last_peered":"2023-09-23T20:38:40.497638+0000","last_clean":"2023-09-23T20:38:40.497638+0000","last_became_active":"2023-09-23T20:32:33.225468+0000","last_became_peered":"2023-09-23T20:32:33.225468+0000","last_unstale":"2023-09-23T20:38:40.497638+0000","last_undegraded":"2023-09-23T20:38:40.497638+0000","last_fullsized":"2023-09-23T20:38:40.497638+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:40:47.867573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":154,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T20:38:40.500225+0000","last_change":"2023-09-23T20:37:43.571836+0000","last_active":"2023-09-23T20:38:40.500225+0000","last_peered":"2023-09-23T20:38:40.500225+0000","last_clean":"2023-09-23T20:38:40.500225+0000","last_became_active":"2023-09-23T20:37:43.571511+0000","last_became_peered":"2023-09-23T20:37:43.571511+0000","last_unstale":"2023-09-23T20:38:40.500225+0000","last_undegraded":"2023-09-23T20:38:40.500225+0000","last_fullsized":"2023-09-23T20:38:40.500225+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:59:25.507917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.19","version":"420'6764","reported_seq":9735,"reported_epoch":420,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T20:38:47.863062+0000","last_change":"2023-09-23T20:38:36.375475+0000","last_active":"2023-09-23T20:38:47.863062+0000","last_peered":"2023-09-23T20:38:47.863062+0000","last_clean":"2023-09-23T20:37:13.347998+0000","last_became_active":"2023-09-23T20:38:35.428186+0000","last_became_peered":"2023-09-23T20:38:35.428186+0000","last_unstale":"2023-09-23T20:38:47.863062+0000","last_undegraded":"2023-09-23T20:38:32.857390+0000","last_fullsized":"2023-09-23T20:38:47.863062+0000","mapping_epoch":410,"log_start":"373'1316","ondisk_log_start":"373'1316","created":410,"last_epoch_clean":370,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":5448,"log_dups_size":0,"ondisk_log_size":5448,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":142934016,"num_objects":2181,"num_object_clones":0,"num_object_copies":6543,"num_objects_missing_on_primary":323,"num_objects_missing":323,"num_objects_degraded":2354,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2181,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3450,"num_write_kb":180192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":636,"num_bytes_recovered":41615360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,3,4","objects":156},{"shards":"0,3,6","objects":-6},{"shards":"0,5,6","objects":2031}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.10","version":"90'161","reported_seq":679,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T20:38:40.497431+0000","last_change":"2023-09-23T20:38:05.590032+0000","last_active":"2023-09-23T20:38:40.497431+0000","last_peered":"2023-09-23T20:38:40.497431+0000","last_clean":"2023-09-23T20:38:40.497431+0000","last_became_active":"2023-09-23T20:38:05.589703+0000","last_became_peered":"2023-09-23T20:38:05.589703+0000","last_unstale":"2023-09-23T20:38:40.497431+0000","last_undegraded":"2023-09-23T20:38:40.497431+0000","last_fullsized":"2023-09-23T20:38:40.497431+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:49.726139+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:49.726139+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0096333110000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":386,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T20:38:40.477287+0000","last_change":"2023-09-23T20:25:02.953993+0000","last_active":"2023-09-23T20:38:40.477287+0000","last_peered":"2023-09-23T20:38:40.477287+0000","last_clean":"2023-09-23T20:38:40.477287+0000","last_became_active":"2023-09-23T20:25:02.953672+0000","last_became_peered":"2023-09-23T20:25:02.953672+0000","last_unstale":"2023-09-23T20:38:40.477287+0000","last_undegraded":"2023-09-23T20:38:40.477287+0000","last_fullsized":"2023-09-23T20:38:40.477287+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:26:08.603577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":208,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T20:38:40.500936+0000","last_change":"2023-09-23T20:38:02.712917+0000","last_active":"2023-09-23T20:38:40.500936+0000","last_peered":"2023-09-23T20:38:40.500936+0000","last_clean":"2023-09-23T20:38:40.500936+0000","last_became_active":"2023-09-23T20:38:02.658528+0000","last_became_peered":"2023-09-23T20:38:02.658528+0000","last_unstale":"2023-09-23T20:38:40.500936+0000","last_undegraded":"2023-09-23T20:38:40.500936+0000","last_fullsized":"2023-09-23T20:38:40.500936+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:42:32.680506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.13","version":"90'161","reported_seq":704,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T20:38:40.479889+0000","last_change":"2023-09-23T20:35:02.238923+0000","last_active":"2023-09-23T20:38:40.479889+0000","last_peered":"2023-09-23T20:38:40.479889+0000","last_clean":"2023-09-23T20:38:40.479889+0000","last_became_active":"2023-09-23T20:35:02.238293+0000","last_became_peered":"2023-09-23T20:35:02.238293+0000","last_unstale":"2023-09-23T20:38:40.479889+0000","last_undegraded":"2023-09-23T20:38:40.479889+0000","last_fullsized":"2023-09-23T20:38:40.479889+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:42.544308+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:42.544308+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:51:22.281932+0000","scrub_duration":0.0029958179999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":423,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T20:38:40.497532+0000","last_change":"2023-09-23T20:38:02.539774+0000","last_active":"2023-09-23T20:38:40.497532+0000","last_peered":"2023-09-23T20:38:40.497532+0000","last_clean":"2023-09-23T20:38:40.497532+0000","last_became_active":"2023-09-23T20:38:02.539436+0000","last_became_peered":"2023-09-23T20:38:02.539436+0000","last_unstale":"2023-09-23T20:38:40.497532+0000","last_undegraded":"2023-09-23T20:38:40.497532+0000","last_fullsized":"2023-09-23T20:38:40.497532+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:50:57.375803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":396,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T20:38:40.477853+0000","last_change":"2023-09-23T20:25:02.957010+0000","last_active":"2023-09-23T20:38:40.477853+0000","last_peered":"2023-09-23T20:38:40.477853+0000","last_clean":"2023-09-23T20:38:40.477853+0000","last_became_active":"2023-09-23T20:25:02.956267+0000","last_became_peered":"2023-09-23T20:25:02.956267+0000","last_unstale":"2023-09-23T20:38:40.477853+0000","last_undegraded":"2023-09-23T20:38:40.477853+0000","last_fullsized":"2023-09-23T20:38:40.477853+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:10:55.299422+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"90'161","reported_seq":514,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T20:38:40.474885+0000","last_change":"2023-09-23T20:38:05.606521+0000","last_active":"2023-09-23T20:38:40.474885+0000","last_peered":"2023-09-23T20:38:40.474885+0000","last_clean":"2023-09-23T20:38:40.474885+0000","last_became_active":"2023-09-23T20:38:05.606164+0000","last_became_peered":"2023-09-23T20:38:05.606164+0000","last_unstale":"2023-09-23T20:38:40.474885+0000","last_undegraded":"2023-09-23T20:38:40.474885+0000","last_fullsized":"2023-09-23T20:38:40.474885+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:53.706934+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:53.706934+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":36,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T20:38:40.481851+0000","last_change":"2023-09-23T20:38:02.626973+0000","last_active":"2023-09-23T20:38:40.481851+0000","last_peered":"2023-09-23T20:38:40.481851+0000","last_clean":"2023-09-23T20:38:40.481851+0000","last_became_active":"2023-09-23T20:38:02.545190+0000","last_became_peered":"2023-09-23T20:38:02.545190+0000","last_unstale":"2023-09-23T20:38:40.481851+0000","last_undegraded":"2023-09-23T20:38:40.481851+0000","last_fullsized":"2023-09-23T20:38:40.481851+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:38:12.102685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":212,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T20:38:40.493908+0000","last_change":"2023-09-23T20:37:43.571083+0000","last_active":"2023-09-23T20:38:40.493908+0000","last_peered":"2023-09-23T20:38:40.493908+0000","last_clean":"2023-09-23T20:38:40.493908+0000","last_became_active":"2023-09-23T20:37:43.570306+0000","last_became_peered":"2023-09-23T20:37:43.570306+0000","last_unstale":"2023-09-23T20:38:40.493908+0000","last_undegraded":"2023-09-23T20:38:40.493908+0000","last_fullsized":"2023-09-23T20:38:40.493908+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:42:02.396718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.4","version":"420'6902","reported_seq":8483,"reported_epoch":420,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:38:45.509088+0000","last_change":"2023-09-23T20:38:39.260380+0000","last_active":"2023-09-23T20:38:45.509088+0000","last_peered":"2023-09-23T20:38:45.509088+0000","last_clean":"2023-09-23T20:37:15.925522+0000","last_became_active":"2023-09-23T20:38:39.251413+0000","last_became_peered":"2023-09-23T20:38:39.251413+0000","last_unstale":"2023-09-23T20:38:45.509088+0000","last_undegraded":"2023-09-23T20:38:38.457471+0000","last_fullsized":"2023-09-23T20:38:38.449461+0000","mapping_epoch":416,"log_start":"373'2080","ondisk_log_start":"373'2080","created":369,"last_epoch_clean":370,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":4822,"log_dups_size":0,"ondisk_log_size":4822,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":144048128,"num_objects":2198,"num_object_clones":0,"num_object_copies":6594,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":771,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2198,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3495,"num_write_kb":182176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":377,"num_bytes_recovered":24707072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[4,2],"avail_no_missing":["4","2","6"],"object_location_counts":[{"shards":"2,3,4","objects":177},{"shards":"2,4","objects":2021}],"blocked_by":[],"up_primary":5,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.d","version":"90'161","reported_seq":712,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T20:38:40.500598+0000","last_change":"2023-09-23T20:34:59.496438+0000","last_active":"2023-09-23T20:38:40.500598+0000","last_peered":"2023-09-23T20:38:40.500598+0000","last_clean":"2023-09-23T20:38:40.500598+0000","last_became_active":"2023-09-23T20:33:38.970527+0000","last_became_peered":"2023-09-23T20:33:38.970527+0000","last_unstale":"2023-09-23T20:38:40.500598+0000","last_undegraded":"2023-09-23T20:38:40.500598+0000","last_fullsized":"2023-09-23T20:38:40.500598+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:34:59.496375+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:34:59.496375+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:46:18.244734+0000","scrub_duration":0.0017843189999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":423,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T20:38:40.501097+0000","last_change":"2023-09-23T20:32:33.283752+0000","last_active":"2023-09-23T20:38:40.501097+0000","last_peered":"2023-09-23T20:38:40.501097+0000","last_clean":"2023-09-23T20:38:40.501097+0000","last_became_active":"2023-09-23T20:32:33.230503+0000","last_became_peered":"2023-09-23T20:32:33.230503+0000","last_unstale":"2023-09-23T20:38:40.501097+0000","last_undegraded":"2023-09-23T20:38:40.501097+0000","last_fullsized":"2023-09-23T20:38:40.501097+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:29.084410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":265,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T20:38:40.497617+0000","last_change":"2023-09-23T20:38:02.539067+0000","last_active":"2023-09-23T20:38:40.497617+0000","last_peered":"2023-09-23T20:38:40.497617+0000","last_clean":"2023-09-23T20:38:40.497617+0000","last_became_active":"2023-09-23T20:38:02.538745+0000","last_became_peered":"2023-09-23T20:38:02.538745+0000","last_unstale":"2023-09-23T20:38:40.497617+0000","last_undegraded":"2023-09-23T20:38:40.497617+0000","last_fullsized":"2023-09-23T20:38:40.497617+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:36:31.615062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.5","version":"420'6747","reported_seq":10286,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-23T20:38:47.801159+0000","last_change":"2023-09-23T20:38:36.632834+0000","last_active":"2023-09-23T20:38:47.801159+0000","last_peered":"2023-09-23T20:38:47.801159+0000","last_clean":"2023-09-23T20:38:47.801159+0000","last_became_active":"2023-09-23T20:38:28.423549+0000","last_became_peered":"2023-09-23T20:38:28.423549+0000","last_unstale":"2023-09-23T20:38:47.801159+0000","last_undegraded":"2023-09-23T20:38:47.801159+0000","last_fullsized":"2023-09-23T20:38:47.801159+0000","mapping_epoch":411,"log_start":"373'1839","ondisk_log_start":"373'1839","created":369,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":4908,"log_dups_size":0,"ondisk_log_size":4908,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":137723904,"num_objects":2101,"num_object_clones":0,"num_object_copies":6303,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2101,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3451,"num_write_kb":177696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1036,"num_bytes_recovered":55345152,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.c","version":"90'161","reported_seq":666,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T20:38:40.500474+0000","last_change":"2023-09-23T20:38:28.442536+0000","last_active":"2023-09-23T20:38:40.500474+0000","last_peered":"2023-09-23T20:38:40.500474+0000","last_clean":"2023-09-23T20:38:40.500474+0000","last_became_active":"2023-09-23T20:38:28.442229+0000","last_became_peered":"2023-09-23T20:38:28.442229+0000","last_unstale":"2023-09-23T20:38:40.500474+0000","last_undegraded":"2023-09-23T20:38:40.500474+0000","last_fullsized":"2023-09-23T20:38:40.500474+0000","mapping_epoch":411,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":412,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:05.485751+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:05.485751+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.002180043,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163840,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":458,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T20:38:40.500823+0000","last_change":"2023-09-23T20:34:59.987981+0000","last_active":"2023-09-23T20:38:40.500823+0000","last_peered":"2023-09-23T20:38:40.500823+0000","last_clean":"2023-09-23T20:38:40.500823+0000","last_became_active":"2023-09-23T20:34:59.985368+0000","last_became_peered":"2023-09-23T20:34:59.985368+0000","last_unstale":"2023-09-23T20:38:40.500823+0000","last_undegraded":"2023-09-23T20:38:40.500823+0000","last_fullsized":"2023-09-23T20:38:40.500823+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:12:43.082493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":117,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T20:38:40.486675+0000","last_change":"2023-09-23T20:32:33.232142+0000","last_active":"2023-09-23T20:38:40.486675+0000","last_peered":"2023-09-23T20:38:40.486675+0000","last_clean":"2023-09-23T20:38:40.486675+0000","last_became_active":"2023-09-23T20:32:33.231683+0000","last_became_peered":"2023-09-23T20:32:33.231683+0000","last_unstale":"2023-09-23T20:38:40.486675+0000","last_undegraded":"2023-09-23T20:38:40.486675+0000","last_fullsized":"2023-09-23T20:38:40.486675+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:02:58.336348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"420'6658","reported_seq":7358,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-23T20:38:45.761022+0000","last_change":"2023-09-23T20:38:29.214109+0000","last_active":"2023-09-23T20:38:45.761022+0000","last_peered":"2023-09-23T20:38:45.761022+0000","last_clean":"2023-09-23T20:38:45.761022+0000","last_became_active":"2023-09-23T20:38:28.425504+0000","last_became_peered":"2023-09-23T20:38:28.425504+0000","last_unstale":"2023-09-23T20:38:45.761022+0000","last_undegraded":"2023-09-23T20:38:45.761022+0000","last_fullsized":"2023-09-23T20:38:45.761022+0000","mapping_epoch":411,"log_start":"373'1261","ondisk_log_start":"373'1261","created":369,"last_epoch_clean":412,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":5397,"log_dups_size":0,"ondisk_log_size":5397,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":142606336,"num_objects":2176,"num_object_clones":0,"num_object_copies":6528,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2176,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3378,"num_write_kb":177728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":305,"num_bytes_recovered":10420224,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"90'161","reported_seq":576,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T20:38:40.492937+0000","last_change":"2023-09-23T20:38:40.491902+0000","last_active":"2023-09-23T20:38:40.492937+0000","last_peered":"2023-09-23T20:38:40.492937+0000","last_clean":"2023-09-23T20:38:40.492937+0000","last_became_active":"2023-09-23T20:38:40.491563+0000","last_became_peered":"2023-09-23T20:38:40.491563+0000","last_unstale":"2023-09-23T20:38:40.492937+0000","last_undegraded":"2023-09-23T20:38:40.492937+0000","last_fullsized":"2023-09-23T20:38:40.492937+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:46.217489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:46.217489+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:39:25.008796+0000","scrub_duration":0.001456429,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":434,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T20:38:40.500663+0000","last_change":"2023-09-23T20:38:02.715585+0000","last_active":"2023-09-23T20:38:40.500663+0000","last_peered":"2023-09-23T20:38:40.500663+0000","last_clean":"2023-09-23T20:38:40.500663+0000","last_became_active":"2023-09-23T20:38:02.658973+0000","last_became_peered":"2023-09-23T20:38:02.658973+0000","last_unstale":"2023-09-23T20:38:40.500663+0000","last_undegraded":"2023-09-23T20:38:40.500663+0000","last_fullsized":"2023-09-23T20:38:40.500663+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:32:03.044825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":350,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":26,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T20:38:40.481782+0000","last_change":"2023-09-23T20:38:02.590613+0000","last_active":"2023-09-23T20:38:40.481782+0000","last_peered":"2023-09-23T20:38:40.481782+0000","last_clean":"2023-09-23T20:38:40.481782+0000","last_became_active":"2023-09-23T20:38:02.590284+0000","last_became_peered":"2023-09-23T20:38:02.590284+0000","last_unstale":"2023-09-23T20:38:40.481782+0000","last_undegraded":"2023-09-23T20:38:40.481782+0000","last_fullsized":"2023-09-23T20:38:40.481782+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:40:07.825054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"420'6760","reported_seq":6847,"reported_epoch":420,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T20:38:49.364435+0000","last_change":"2023-09-23T20:38:28.482576+0000","last_active":"2023-09-23T20:38:49.364435+0000","last_peered":"2023-09-23T20:38:49.364435+0000","last_clean":"2023-09-23T20:37:19.384160+0000","last_became_active":"2023-09-23T20:38:28.436345+0000","last_became_peered":"2023-09-23T20:38:28.436345+0000","last_unstale":"2023-09-23T20:38:49.364435+0000","last_undegraded":"2023-09-23T20:38:28.418749+0000","last_fullsized":"2023-09-23T20:38:49.364435+0000","mapping_epoch":411,"log_start":"373'1658","ondisk_log_start":"373'1658","created":369,"last_epoch_clean":370,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":5102,"log_dups_size":0,"ondisk_log_size":5102,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":136773632,"num_objects":2087,"num_object_clones":0,"num_object_copies":6261,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":309,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2087,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3470,"num_write_kb":177824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":["1","3"],"object_location_counts":[{"shards":"1,2,3","objects":148},{"shards":"1,3,6","objects":1939}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.e","version":"90'161","reported_seq":520,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T20:38:40.487723+0000","last_change":"2023-09-23T20:36:00.322453+0000","last_active":"2023-09-23T20:38:40.487723+0000","last_peered":"2023-09-23T20:38:40.487723+0000","last_clean":"2023-09-23T20:38:40.487723+0000","last_became_active":"2023-09-23T20:35:02.233536+0000","last_became_peered":"2023-09-23T20:35:02.233536+0000","last_unstale":"2023-09-23T20:38:40.487723+0000","last_undegraded":"2023-09-23T20:38:40.487723+0000","last_fullsized":"2023-09-23T20:38:40.487723+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:00.322381+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:00.322381+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:02:15.489069+0000","scrub_duration":0.0017321859999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":13,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T20:38:39.480014+0000","last_change":"2023-09-23T20:38:37.456683+0000","last_active":"2023-09-23T20:38:39.480014+0000","last_peered":"2023-09-23T20:38:39.480014+0000","last_clean":"2023-09-23T20:38:39.480014+0000","last_became_active":"2023-09-23T20:38:37.456289+0000","last_became_peered":"2023-09-23T20:38:37.456289+0000","last_unstale":"2023-09-23T20:38:39.480014+0000","last_undegraded":"2023-09-23T20:38:39.480014+0000","last_fullsized":"2023-09-23T20:38:39.480014+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:03:10.507333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":405,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T20:38:40.476632+0000","last_change":"2023-09-23T20:32:33.236122+0000","last_active":"2023-09-23T20:38:40.476632+0000","last_peered":"2023-09-23T20:38:40.476632+0000","last_clean":"2023-09-23T20:38:40.476632+0000","last_became_active":"2023-09-23T20:32:33.235384+0000","last_became_peered":"2023-09-23T20:32:33.235384+0000","last_unstale":"2023-09-23T20:38:40.476632+0000","last_undegraded":"2023-09-23T20:38:40.476632+0000","last_fullsized":"2023-09-23T20:38:40.476632+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:56.843733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"420'6636","reported_seq":13112,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-23T20:38:48.471051+0000","last_change":"2023-09-23T20:38:36.017904+0000","last_active":"2023-09-23T20:38:48.471051+0000","last_peered":"2023-09-23T20:38:48.471051+0000","last_clean":"2023-09-23T20:38:48.471051+0000","last_became_active":"2023-09-23T20:38:27.481141+0000","last_became_peered":"2023-09-23T20:38:27.481141+0000","last_unstale":"2023-09-23T20:38:48.471051+0000","last_undegraded":"2023-09-23T20:38:48.471051+0000","last_fullsized":"2023-09-23T20:38:48.471051+0000","mapping_epoch":400,"log_start":"373'2187","ondisk_log_start":"373'2187","created":369,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":4449,"log_dups_size":0,"ondisk_log_size":4449,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":135397376,"num_objects":2066,"num_object_clones":0,"num_object_copies":6198,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2066,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3389,"num_write_kb":174560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2565,"num_bytes_recovered":168099840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":825,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T20:38:40.497435+0000","last_change":"2023-09-23T20:35:02.523308+0000","last_active":"2023-09-23T20:38:40.497435+0000","last_peered":"2023-09-23T20:38:40.497435+0000","last_clean":"2023-09-23T20:38:40.497435+0000","last_became_active":"2023-09-23T20:33:38.979989+0000","last_became_peered":"2023-09-23T20:33:38.979989+0000","last_unstale":"2023-09-23T20:38:40.497435+0000","last_undegraded":"2023-09-23T20:38:40.497435+0000","last_fullsized":"2023-09-23T20:38:40.497435+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:35:02.523203+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:35:02.523203+0000","last_clean_scrub_stamp":"2023-09-23T20:35:02.523203+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:02:41.472958+0000","scrub_duration":0.0085927989999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":238,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T20:38:40.496367+0000","last_change":"2023-09-23T20:37:43.583731+0000","last_active":"2023-09-23T20:38:40.496367+0000","last_peered":"2023-09-23T20:38:40.496367+0000","last_clean":"2023-09-23T20:38:40.496367+0000","last_became_active":"2023-09-23T20:37:43.583066+0000","last_became_peered":"2023-09-23T20:37:43.583066+0000","last_unstale":"2023-09-23T20:38:40.496367+0000","last_undegraded":"2023-09-23T20:38:40.496367+0000","last_fullsized":"2023-09-23T20:38:40.496367+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:03:45.211900+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":137,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T20:38:40.486079+0000","last_change":"2023-09-23T20:38:02.579814+0000","last_active":"2023-09-23T20:38:40.486079+0000","last_peered":"2023-09-23T20:38:40.486079+0000","last_clean":"2023-09-23T20:38:40.486079+0000","last_became_active":"2023-09-23T20:38:02.579253+0000","last_became_peered":"2023-09-23T20:38:02.579253+0000","last_unstale":"2023-09-23T20:38:40.486079+0000","last_undegraded":"2023-09-23T20:38:40.486079+0000","last_fullsized":"2023-09-23T20:38:40.486079+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:05:07.298198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"420'6784","reported_seq":6855,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-23T20:38:49.268981+0000","last_change":"2023-09-23T20:38:27.447178+0000","last_active":"2023-09-23T20:38:49.268981+0000","last_peered":"2023-09-23T20:38:49.268981+0000","last_clean":"2023-09-23T20:38:49.268981+0000","last_became_active":"2023-09-23T20:38:27.437204+0000","last_became_peered":"2023-09-23T20:38:27.437204+0000","last_unstale":"2023-09-23T20:38:49.268981+0000","last_undegraded":"2023-09-23T20:38:49.268981+0000","last_fullsized":"2023-09-23T20:38:49.268981+0000","mapping_epoch":369,"log_start":"373'1694","ondisk_log_start":"373'1694","created":369,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":5090,"log_dups_size":0,"ondisk_log_size":5090,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":138182656,"num_objects":2108,"num_object_clones":0,"num_object_copies":6324,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2108,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3474,"num_write_kb":178656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":510,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T20:38:40.478755+0000","last_change":"2023-09-23T20:38:05.632243+0000","last_active":"2023-09-23T20:38:40.478755+0000","last_peered":"2023-09-23T20:38:40.478755+0000","last_clean":"2023-09-23T20:38:40.478755+0000","last_became_active":"2023-09-23T20:38:05.631907+0000","last_became_peered":"2023-09-23T20:38:05.631907+0000","last_unstale":"2023-09-23T20:38:40.478755+0000","last_undegraded":"2023-09-23T20:38:40.478755+0000","last_fullsized":"2023-09-23T20:38:40.478755+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:35:00.496197+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:35:00.496197+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":26,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T20:38:40.481839+0000","last_change":"2023-09-23T20:38:02.578703+0000","last_active":"2023-09-23T20:38:40.481839+0000","last_peered":"2023-09-23T20:38:40.481839+0000","last_clean":"2023-09-23T20:38:40.481839+0000","last_became_active":"2023-09-23T20:38:02.578414+0000","last_became_peered":"2023-09-23T20:38:02.578414+0000","last_unstale":"2023-09-23T20:38:40.481839+0000","last_undegraded":"2023-09-23T20:38:40.481839+0000","last_fullsized":"2023-09-23T20:38:40.481839+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:43:59.779295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":86,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T20:38:40.488065+0000","last_change":"2023-09-23T20:37:31.296606+0000","last_active":"2023-09-23T20:38:40.488065+0000","last_peered":"2023-09-23T20:38:40.488065+0000","last_clean":"2023-09-23T20:38:40.488065+0000","last_became_active":"2023-09-23T20:37:31.295924+0000","last_became_peered":"2023-09-23T20:37:31.295924+0000","last_unstale":"2023-09-23T20:38:40.488065+0000","last_undegraded":"2023-09-23T20:38:40.488065+0000","last_fullsized":"2023-09-23T20:38:40.488065+0000","mapping_epoch":391,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":392,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:27:50.948510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"420'6802","reported_seq":6930,"reported_epoch":420,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:38:45.523952+0000","last_change":"2023-09-23T20:38:39.273152+0000","last_active":"2023-09-23T20:38:45.523952+0000","last_peered":"2023-09-23T20:38:45.523952+0000","last_clean":"2023-09-23T20:37:29.399979+0000","last_became_active":"2023-09-23T20:38:39.253967+0000","last_became_peered":"2023-09-23T20:38:39.253967+0000","last_unstale":"2023-09-23T20:38:45.523952+0000","last_undegraded":"2023-09-23T20:38:38.450129+0000","last_fullsized":"2023-09-23T20:38:38.446971+0000","mapping_epoch":416,"log_start":"373'1996","ondisk_log_start":"373'1996","created":369,"last_epoch_clean":370,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":4806,"log_dups_size":0,"ondisk_log_size":4806,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":141688832,"num_objects":2162,"num_object_clones":0,"num_object_copies":6486,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":723,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2162,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3452,"num_write_kb":179648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":950272,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0],"avail_no_missing":["4","0"],"object_location_counts":[{"shards":"0,3,4","objects":190},{"shards":"0,4","objects":1972}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"90'161","reported_seq":704,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-23T20:38:45.520535+0000","last_change":"2023-09-23T20:37:17.282621+0000","last_active":"2023-09-23T20:38:45.520535+0000","last_peered":"2023-09-23T20:38:45.520535+0000","last_clean":"2023-09-23T20:38:45.520535+0000","last_became_active":"2023-09-23T20:37:17.282243+0000","last_became_peered":"2023-09-23T20:37:17.282243+0000","last_unstale":"2023-09-23T20:38:45.520535+0000","last_undegraded":"2023-09-23T20:38:45.520535+0000","last_fullsized":"2023-09-23T20:38:45.520535+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:05.595042+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:05.595042+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:42:06.244026+0000","scrub_duration":0.007271675,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":76,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T20:38:40.486339+0000","last_change":"2023-09-23T20:34:59.983122+0000","last_active":"2023-09-23T20:38:40.486339+0000","last_peered":"2023-09-23T20:38:40.486339+0000","last_clean":"2023-09-23T20:38:40.486339+0000","last_became_active":"2023-09-23T20:34:59.982809+0000","last_became_peered":"2023-09-23T20:34:59.982809+0000","last_unstale":"2023-09-23T20:38:40.486339+0000","last_undegraded":"2023-09-23T20:38:40.486339+0000","last_fullsized":"2023-09-23T20:38:40.486339+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:49:40.839723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":137,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T20:38:40.487943+0000","last_change":"2023-09-23T20:38:02.543866+0000","last_active":"2023-09-23T20:38:40.487943+0000","last_peered":"2023-09-23T20:38:40.487943+0000","last_clean":"2023-09-23T20:38:40.487943+0000","last_became_active":"2023-09-23T20:38:02.543563+0000","last_became_peered":"2023-09-23T20:38:02.543563+0000","last_unstale":"2023-09-23T20:38:40.487943+0000","last_undegraded":"2023-09-23T20:38:40.487943+0000","last_fullsized":"2023-09-23T20:38:40.487943+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:33.050536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"420'6768","reported_seq":8235,"reported_epoch":420,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T20:38:49.399534+0000","last_change":"2023-09-23T20:38:28.540352+0000","last_active":"2023-09-23T20:38:49.399534+0000","last_peered":"2023-09-23T20:38:49.399534+0000","last_clean":"2023-09-23T20:37:13.297905+0000","last_became_active":"2023-09-23T20:38:28.499228+0000","last_became_peered":"2023-09-23T20:38:28.499228+0000","last_unstale":"2023-09-23T20:38:49.399534+0000","last_undegraded":"2023-09-23T20:38:28.490806+0000","last_fullsized":"2023-09-23T20:38:49.399534+0000","mapping_epoch":411,"log_start":"373'1680","ondisk_log_start":"373'1680","created":369,"last_epoch_clean":370,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":5088,"log_dups_size":0,"ondisk_log_size":5088,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":139689984,"num_objects":2131,"num_object_clones":0,"num_object_copies":6393,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2634,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2131,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3461,"num_write_kb":178976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":332,"num_bytes_recovered":21725184,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":["1","5"],"object_location_counts":[{"shards":"1,2,3","objects":156},{"shards":"1,2,6","objects":18},{"shards":"1,5,6","objects":1957}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":490,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T20:38:40.494403+0000","last_change":"2023-09-23T20:38:40.494403+0000","last_active":"2023-09-23T20:38:40.494403+0000","last_peered":"2023-09-23T20:38:40.494403+0000","last_clean":"2023-09-23T20:38:40.494403+0000","last_became_active":"2023-09-23T20:38:40.494049+0000","last_became_peered":"2023-09-23T20:38:40.494049+0000","last_unstale":"2023-09-23T20:38:40.494403+0000","last_undegraded":"2023-09-23T20:38:40.494403+0000","last_fullsized":"2023-09-23T20:38:40.494403+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:34:59.481446+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:34:59.481446+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26834,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":212,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T20:38:40.495137+0000","last_change":"2023-09-23T20:37:43.570610+0000","last_active":"2023-09-23T20:38:40.495137+0000","last_peered":"2023-09-23T20:38:40.495137+0000","last_clean":"2023-09-23T20:38:40.495137+0000","last_became_active":"2023-09-23T20:37:43.570216+0000","last_became_peered":"2023-09-23T20:37:43.570216+0000","last_unstale":"2023-09-23T20:38:40.495137+0000","last_undegraded":"2023-09-23T20:38:40.495137+0000","last_fullsized":"2023-09-23T20:38:40.495137+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:39:26.555329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":243,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-23T20:38:45.515729+0000","last_change":"2023-09-23T20:32:33.230582+0000","last_active":"2023-09-23T20:38:45.515729+0000","last_peered":"2023-09-23T20:38:45.515729+0000","last_clean":"2023-09-23T20:38:45.515729+0000","last_became_active":"2023-09-23T20:32:33.230257+0000","last_became_peered":"2023-09-23T20:32:33.230257+0000","last_unstale":"2023-09-23T20:38:45.515729+0000","last_undegraded":"2023-09-23T20:38:45.515729+0000","last_fullsized":"2023-09-23T20:38:45.515729+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:11:45.333096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"420'6717","reported_seq":6778,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-23T20:38:46.111767+0000","last_change":"2023-09-23T20:36:11.310149+0000","last_active":"2023-09-23T20:38:46.111767+0000","last_peered":"2023-09-23T20:38:46.111767+0000","last_clean":"2023-09-23T20:38:46.111767+0000","last_became_active":"2023-09-23T20:36:11.309844+0000","last_became_peered":"2023-09-23T20:36:11.309844+0000","last_unstale":"2023-09-23T20:38:46.111767+0000","last_undegraded":"2023-09-23T20:38:46.111767+0000","last_fullsized":"2023-09-23T20:38:46.111767+0000","mapping_epoch":369,"log_start":"373'1219","ondisk_log_start":"373'1219","created":369,"last_epoch_clean":370,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":5498,"log_dups_size":0,"ondisk_log_size":5498,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:14:57.442108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":284360704,"num_objects":4339,"num_object_clones":0,"num_object_copies":13017,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4339,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6717,"num_write_kb":353792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":599,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T20:38:40.482861+0000","last_change":"2023-09-23T20:38:05.622597+0000","last_active":"2023-09-23T20:38:40.482861+0000","last_peered":"2023-09-23T20:38:40.482861+0000","last_clean":"2023-09-23T20:38:40.482861+0000","last_became_active":"2023-09-23T20:38:05.622261+0000","last_became_peered":"2023-09-23T20:38:05.622261+0000","last_unstale":"2023-09-23T20:38:40.482861+0000","last_undegraded":"2023-09-23T20:38:40.482861+0000","last_fullsized":"2023-09-23T20:38:40.482861+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:35:04.101972+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:35:04.101972+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"86'119","reported_seq":727,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T20:38:40.474491+0000","last_change":"2023-09-23T20:29:45.521257+0000","last_active":"2023-09-23T20:38:40.474491+0000","last_peered":"2023-09-23T20:38:40.474491+0000","last_clean":"2023-09-23T20:38:40.474491+0000","last_became_active":"2023-09-23T20:29:45.520546+0000","last_became_peered":"2023-09-23T20:29:45.520546+0000","last_unstale":"2023-09-23T20:38:40.474491+0000","last_undegraded":"2023-09-23T20:38:40.474491+0000","last_fullsized":"2023-09-23T20:38:40.474491+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":119,"log_dups_size":0,"ondisk_log_size":119,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:58:56.085777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032705499999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":523,"num_read_kb":3503,"num_write":136,"num_write_kb":3128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":243,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T20:38:40.494074+0000","last_change":"2023-09-23T20:38:02.597306+0000","last_active":"2023-09-23T20:38:40.494074+0000","last_peered":"2023-09-23T20:38:40.494074+0000","last_clean":"2023-09-23T20:38:40.494074+0000","last_became_active":"2023-09-23T20:38:02.596966+0000","last_became_peered":"2023-09-23T20:38:02.596966+0000","last_unstale":"2023-09-23T20:38:40.494074+0000","last_undegraded":"2023-09-23T20:38:40.494074+0000","last_fullsized":"2023-09-23T20:38:40.494074+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:30:41.962907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":365,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T20:38:40.474528+0000","last_change":"2023-09-23T20:37:14.359424+0000","last_active":"2023-09-23T20:38:40.474528+0000","last_peered":"2023-09-23T20:38:40.474528+0000","last_clean":"2023-09-23T20:38:40.474528+0000","last_became_active":"2023-09-23T20:37:14.359059+0000","last_became_peered":"2023-09-23T20:37:14.359059+0000","last_unstale":"2023-09-23T20:38:40.474528+0000","last_undegraded":"2023-09-23T20:38:40.474528+0000","last_fullsized":"2023-09-23T20:38:40.474528+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:07:10.779153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.8","version":"420'6785","reported_seq":11924,"reported_epoch":420,"state":"active+recovering","last_fresh":"2023-09-23T20:38:48.915798+0000","last_change":"2023-09-23T20:38:36.641167+0000","last_active":"2023-09-23T20:38:48.915798+0000","last_peered":"2023-09-23T20:38:48.915798+0000","last_clean":"2023-09-23T20:37:13.320723+0000","last_became_active":"2023-09-23T20:38:27.525964+0000","last_became_peered":"2023-09-23T20:38:27.525964+0000","last_unstale":"2023-09-23T20:38:48.915798+0000","last_undegraded":"2023-09-23T20:38:48.915798+0000","last_fullsized":"2023-09-23T20:38:48.915798+0000","mapping_epoch":400,"log_start":"373'2334","ondisk_log_start":"373'2334","created":369,"last_epoch_clean":370,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":4451,"log_dups_size":0,"ondisk_log_size":4451,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":138772480,"num_objects":2117,"num_object_clones":0,"num_object_copies":6351,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":571,"num_objects_unfound":0,"num_objects_dirty":2117,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3476,"num_write_kb":179008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2017,"num_bytes_recovered":132186112,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":["3","5","7"],"object_location_counts":[{"shards":"0,3,7","objects":1546},{"shards":"3,5,7","objects":571}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":482,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T20:38:40.477994+0000","last_change":"2023-09-23T20:37:17.290846+0000","last_active":"2023-09-23T20:38:40.477994+0000","last_peered":"2023-09-23T20:38:40.477994+0000","last_clean":"2023-09-23T20:38:40.477994+0000","last_became_active":"2023-09-23T20:37:17.290470+0000","last_became_peered":"2023-09-23T20:37:17.290470+0000","last_unstale":"2023-09-23T20:38:40.477994+0000","last_undegraded":"2023-09-23T20:38:40.477994+0000","last_fullsized":"2023-09-23T20:38:40.477994+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:45.785450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:45.785450+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"86'106","reported_seq":1138,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T20:38:39.480879+0000","last_change":"2023-09-23T20:38:37.476512+0000","last_active":"2023-09-23T20:38:39.480879+0000","last_peered":"2023-09-23T20:38:39.480879+0000","last_clean":"2023-09-23T20:38:39.480879+0000","last_became_active":"2023-09-23T20:38:37.475980+0000","last_became_peered":"2023-09-23T20:38:37.475980+0000","last_unstale":"2023-09-23T20:38:39.480879+0000","last_undegraded":"2023-09-23T20:38:39.480879+0000","last_fullsized":"2023-09-23T20:38:39.480879+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":106,"log_dups_size":0,"ondisk_log_size":106,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:58:45.508831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00099302599999999993,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1094,"num_read_kb":1678,"num_write":100,"num_write_kb":404,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":137,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T20:38:40.486042+0000","last_change":"2023-09-23T20:38:02.552008+0000","last_active":"2023-09-23T20:38:40.486042+0000","last_peered":"2023-09-23T20:38:40.486042+0000","last_clean":"2023-09-23T20:38:40.486042+0000","last_became_active":"2023-09-23T20:38:02.551684+0000","last_became_peered":"2023-09-23T20:38:02.551684+0000","last_unstale":"2023-09-23T20:38:40.486042+0000","last_undegraded":"2023-09-23T20:38:40.486042+0000","last_fullsized":"2023-09-23T20:38:40.486042+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:40:37.147538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":380,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T20:38:40.474440+0000","last_change":"2023-09-23T20:29:45.533406+0000","last_active":"2023-09-23T20:38:40.474440+0000","last_peered":"2023-09-23T20:38:40.474440+0000","last_clean":"2023-09-23T20:38:40.474440+0000","last_became_active":"2023-09-23T20:29:45.526001+0000","last_became_peered":"2023-09-23T20:29:45.526001+0000","last_unstale":"2023-09-23T20:38:40.474440+0000","last_undegraded":"2023-09-23T20:38:40.474440+0000","last_fullsized":"2023-09-23T20:38:40.474440+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:43.787251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.b","version":"420'6856","reported_seq":6930,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-23T20:38:48.789900+0000","last_change":"2023-09-23T20:38:01.517845+0000","last_active":"2023-09-23T20:38:48.789900+0000","last_peered":"2023-09-23T20:38:48.789900+0000","last_clean":"2023-09-23T20:38:48.789900+0000","last_became_active":"2023-09-23T20:37:43.579403+0000","last_became_peered":"2023-09-23T20:37:43.579403+0000","last_unstale":"2023-09-23T20:38:48.789900+0000","last_undegraded":"2023-09-23T20:38:48.789900+0000","last_fullsized":"2023-09-23T20:38:48.789900+0000","mapping_epoch":399,"log_start":"373'2500","ondisk_log_start":"373'2500","created":369,"last_epoch_clean":403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":4356,"log_dups_size":0,"ondisk_log_size":4356,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:33:13.920936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":280494080,"num_objects":4280,"num_object_clones":0,"num_object_copies":12840,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4280,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6856,"num_write_kb":356352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":892,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T20:38:40.497839+0000","last_change":"2023-09-23T20:35:02.260708+0000","last_active":"2023-09-23T20:38:40.497839+0000","last_peered":"2023-09-23T20:38:40.497839+0000","last_clean":"2023-09-23T20:38:40.497839+0000","last_became_active":"2023-09-23T20:35:02.260348+0000","last_became_peered":"2023-09-23T20:35:02.260348+0000","last_unstale":"2023-09-23T20:38:40.497839+0000","last_undegraded":"2023-09-23T20:38:40.497839+0000","last_fullsized":"2023-09-23T20:38:40.497839+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:43.390908+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:43.390908+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:09:14.144070+0000","scrub_duration":0.0032721,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"86'130","reported_seq":735,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T20:38:40.478860+0000","last_change":"2023-09-23T20:25:39.637458+0000","last_active":"2023-09-23T20:38:40.478860+0000","last_peered":"2023-09-23T20:38:40.478860+0000","last_clean":"2023-09-23T20:38:40.478860+0000","last_became_active":"2023-09-23T20:25:39.636773+0000","last_became_peered":"2023-09-23T20:25:39.636773+0000","last_unstale":"2023-09-23T20:38:40.478860+0000","last_undegraded":"2023-09-23T20:38:40.478860+0000","last_fullsized":"2023-09-23T20:38:40.478860+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:00:32.810564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011300049999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":512,"num_read_kb":1820,"num_write":133,"num_write_kb":1347,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":388,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T20:38:40.473960+0000","last_change":"2023-09-23T20:38:02.591335+0000","last_active":"2023-09-23T20:38:40.473960+0000","last_peered":"2023-09-23T20:38:40.473960+0000","last_clean":"2023-09-23T20:38:40.473960+0000","last_became_active":"2023-09-23T20:38:02.591015+0000","last_became_peered":"2023-09-23T20:38:02.591015+0000","last_unstale":"2023-09-23T20:38:40.473960+0000","last_undegraded":"2023-09-23T20:38:40.473960+0000","last_fullsized":"2023-09-23T20:38:40.473960+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:55:32.272195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":371,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T20:38:40.477782+0000","last_change":"2023-09-23T20:37:14.360153+0000","last_active":"2023-09-23T20:38:40.477782+0000","last_peered":"2023-09-23T20:38:40.477782+0000","last_clean":"2023-09-23T20:38:40.477782+0000","last_became_active":"2023-09-23T20:37:14.359892+0000","last_became_peered":"2023-09-23T20:37:14.359892+0000","last_unstale":"2023-09-23T20:38:40.477782+0000","last_undegraded":"2023-09-23T20:38:40.477782+0000","last_fullsized":"2023-09-23T20:38:40.477782+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:59:37.031659+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"420'6759","reported_seq":9721,"reported_epoch":420,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T20:38:47.784171+0000","last_change":"2023-09-23T20:38:37.846664+0000","last_active":"2023-09-23T20:38:47.784171+0000","last_peered":"2023-09-23T20:38:47.784171+0000","last_clean":"2023-09-23T20:37:13.347998+0000","last_became_active":"2023-09-23T20:38:35.428647+0000","last_became_peered":"2023-09-23T20:38:35.428647+0000","last_unstale":"2023-09-23T20:38:47.784171+0000","last_undegraded":"2023-09-23T20:38:30.822487+0000","last_fullsized":"2023-09-23T20:38:47.784171+0000","mapping_epoch":411,"log_start":"373'1316","ondisk_log_start":"373'1316","created":369,"last_epoch_clean":370,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":5443,"log_dups_size":0,"ondisk_log_size":5443,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":143261696,"num_objects":2186,"num_object_clones":0,"num_object_copies":6558,"num_objects_missing_on_primary":334,"num_objects_missing":334,"num_objects_degraded":2340,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2186,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3445,"num_write_kb":180192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":630,"num_bytes_recovered":41222144,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,3,4","objects":166},{"shards":"0,3,6","objects":14},{"shards":"0,5,6","objects":2006}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"86'194","reported_seq":768,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T20:38:40.479403+0000","last_change":"2023-09-23T20:37:17.289352+0000","last_active":"2023-09-23T20:38:40.479403+0000","last_peered":"2023-09-23T20:38:40.479403+0000","last_clean":"2023-09-23T20:38:40.479403+0000","last_became_active":"2023-09-23T20:37:17.288695+0000","last_became_peered":"2023-09-23T20:37:17.288695+0000","last_unstale":"2023-09-23T20:38:40.479403+0000","last_undegraded":"2023-09-23T20:38:40.479403+0000","last_fullsized":"2023-09-23T20:38:40.479403+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":194,"log_dups_size":0,"ondisk_log_size":194,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:43:22.047407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00037995999999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1024,"num_read_kb":1200,"num_write":173,"num_write_kb":1276,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":882,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T20:38:40.497526+0000","last_change":"2023-09-23T20:38:05.602508+0000","last_active":"2023-09-23T20:38:40.497526+0000","last_peered":"2023-09-23T20:38:40.497526+0000","last_clean":"2023-09-23T20:38:40.497526+0000","last_became_active":"2023-09-23T20:38:05.602129+0000","last_became_peered":"2023-09-23T20:38:05.602129+0000","last_unstale":"2023-09-23T20:38:40.497526+0000","last_undegraded":"2023-09-23T20:38:40.497526+0000","last_fullsized":"2023-09-23T20:38:40.497526+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:45.343742+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:45.343742+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:48.910343+0000","scrub_duration":0.001475802,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":26,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T20:38:40.481865+0000","last_change":"2023-09-23T20:38:02.541452+0000","last_active":"2023-09-23T20:38:40.481865+0000","last_peered":"2023-09-23T20:38:40.481865+0000","last_clean":"2023-09-23T20:38:40.481865+0000","last_became_active":"2023-09-23T20:38:02.541112+0000","last_became_peered":"2023-09-23T20:38:02.541112+0000","last_unstale":"2023-09-23T20:38:40.481865+0000","last_undegraded":"2023-09-23T20:38:40.481865+0000","last_fullsized":"2023-09-23T20:38:40.481865+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:03:51.385905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,4],"acting":[6,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":26,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T20:38:40.482953+0000","last_change":"2023-09-23T20:38:02.573200+0000","last_active":"2023-09-23T20:38:40.482953+0000","last_peered":"2023-09-23T20:38:40.482953+0000","last_clean":"2023-09-23T20:38:40.482953+0000","last_became_active":"2023-09-23T20:38:02.572945+0000","last_became_peered":"2023-09-23T20:38:02.572945+0000","last_unstale":"2023-09-23T20:38:40.482953+0000","last_undegraded":"2023-09-23T20:38:40.482953+0000","last_fullsized":"2023-09-23T20:38:40.482953+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:57.246533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.a","version":"420'7125","reported_seq":11226,"reported_epoch":420,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T20:38:49.452869+0000","last_change":"2023-09-23T20:38:02.722059+0000","last_active":"2023-09-23T20:38:49.452869+0000","last_peered":"2023-09-23T20:38:49.452869+0000","last_clean":"2023-09-23T20:37:13.317597+0000","last_became_active":"2023-09-23T20:37:15.678492+0000","last_became_peered":"2023-09-23T20:37:15.678492+0000","last_unstale":"2023-09-23T20:38:49.452869+0000","last_undegraded":"2023-09-23T20:37:23.465606+0000","last_fullsized":"2023-09-23T20:37:15.349179+0000","mapping_epoch":380,"log_start":"373'2479","ondisk_log_start":"373'2479","created":369,"last_epoch_clean":370,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":4646,"log_dups_size":0,"ondisk_log_size":4646,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:52:53.085450+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":275578880,"num_objects":4205,"num_object_clones":0,"num_object_copies":12615,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":550,"num_objects_misplaced":2777,"num_objects_unfound":0,"num_objects_dirty":4205,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7125,"num_write_kb":362560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2012,"num_bytes_recovered":131858432,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2],"avail_no_missing":["1","2","5"],"object_location_counts":[{"shards":"1,2","objects":878},{"shards":"1,2,5","objects":2777}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":749,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T20:38:40.479794+0000","last_change":"2023-09-23T20:38:05.615711+0000","last_active":"2023-09-23T20:38:40.479794+0000","last_peered":"2023-09-23T20:38:40.479794+0000","last_clean":"2023-09-23T20:38:40.479794+0000","last_became_active":"2023-09-23T20:38:05.615390+0000","last_became_peered":"2023-09-23T20:38:05.615390+0000","last_unstale":"2023-09-23T20:38:40.479794+0000","last_undegraded":"2023-09-23T20:38:40.479794+0000","last_fullsized":"2023-09-23T20:38:40.479794+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:43.582146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:43.582146+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:45:11.917871+0000","scrub_duration":0.0016219730000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"86'80","reported_seq":672,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T20:38:40.497487+0000","last_change":"2023-09-23T20:29:45.536796+0000","last_active":"2023-09-23T20:38:40.497487+0000","last_peered":"2023-09-23T20:38:40.497487+0000","last_clean":"2023-09-23T20:38:40.497487+0000","last_became_active":"2023-09-23T20:29:45.536439+0000","last_became_peered":"2023-09-23T20:29:45.536439+0000","last_unstale":"2023-09-23T20:38:40.497487+0000","last_undegraded":"2023-09-23T20:38:40.497487+0000","last_fullsized":"2023-09-23T20:38:40.497487+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":80,"log_dups_size":0,"ondisk_log_size":80,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:30:57.793132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035115,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":395,"num_read_kb":1370,"num_write":66,"num_write_kb":1061,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":423,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T20:38:40.477831+0000","last_change":"2023-09-23T20:34:59.984286+0000","last_active":"2023-09-23T20:38:40.477831+0000","last_peered":"2023-09-23T20:38:40.477831+0000","last_clean":"2023-09-23T20:38:40.477831+0000","last_became_active":"2023-09-23T20:34:59.983950+0000","last_became_peered":"2023-09-23T20:34:59.983950+0000","last_unstale":"2023-09-23T20:38:40.477831+0000","last_undegraded":"2023-09-23T20:38:40.477831+0000","last_fullsized":"2023-09-23T20:38:40.477831+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:31:53.557035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":418,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T20:38:40.477616+0000","last_change":"2023-09-23T20:29:45.690815+0000","last_active":"2023-09-23T20:38:40.477616+0000","last_peered":"2023-09-23T20:38:40.477616+0000","last_clean":"2023-09-23T20:38:40.477616+0000","last_became_active":"2023-09-23T20:29:45.690478+0000","last_became_peered":"2023-09-23T20:29:45.690478+0000","last_unstale":"2023-09-23T20:38:40.477616+0000","last_undegraded":"2023-09-23T20:38:40.477616+0000","last_fullsized":"2023-09-23T20:38:40.477616+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:38.288193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.f","version":"420'6764","reported_seq":6825,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-23T20:38:47.649594+0000","last_change":"2023-09-23T20:36:11.309536+0000","last_active":"2023-09-23T20:38:47.649594+0000","last_peered":"2023-09-23T20:38:47.649594+0000","last_clean":"2023-09-23T20:38:47.649594+0000","last_became_active":"2023-09-23T20:36:11.309208+0000","last_became_peered":"2023-09-23T20:36:11.309208+0000","last_unstale":"2023-09-23T20:38:47.649594+0000","last_undegraded":"2023-09-23T20:38:47.649594+0000","last_fullsized":"2023-09-23T20:38:47.649594+0000","mapping_epoch":369,"log_start":"0'0","ondisk_log_start":"0'0","created":369,"last_epoch_clean":370,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":6764,"log_dups_size":0,"ondisk_log_size":6764,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:47:51.785158+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":279838720,"num_objects":4270,"num_object_clones":0,"num_object_copies":12810,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4270,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6764,"num_write_kb":353088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":522,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T20:38:40.482438+0000","last_change":"2023-09-23T20:38:05.611181+0000","last_active":"2023-09-23T20:38:40.482438+0000","last_peered":"2023-09-23T20:38:40.482438+0000","last_clean":"2023-09-23T20:38:40.482438+0000","last_became_active":"2023-09-23T20:38:05.610881+0000","last_became_peered":"2023-09-23T20:38:05.610881+0000","last_unstale":"2023-09-23T20:38:40.482438+0000","last_undegraded":"2023-09-23T20:38:40.482438+0000","last_fullsized":"2023-09-23T20:38:40.482438+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:35:00.151695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:35:00.151695+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":20,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"86'102","reported_seq":1014,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T20:38:40.500692+0000","last_change":"2023-09-23T20:32:36.314672+0000","last_active":"2023-09-23T20:38:40.500692+0000","last_peered":"2023-09-23T20:38:40.500692+0000","last_clean":"2023-09-23T20:38:40.500692+0000","last_became_active":"2023-09-23T20:32:36.314231+0000","last_became_peered":"2023-09-23T20:32:36.314231+0000","last_unstale":"2023-09-23T20:38:40.500692+0000","last_undegraded":"2023-09-23T20:38:40.500692+0000","last_fullsized":"2023-09-23T20:38:40.500692+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":102,"log_dups_size":0,"ondisk_log_size":102,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:11:40.514709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030656500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":963,"num_read_kb":1234,"num_write":60,"num_write_kb":225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":355,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T20:38:40.474541+0000","last_change":"2023-09-23T20:37:14.359334+0000","last_active":"2023-09-23T20:38:40.474541+0000","last_peered":"2023-09-23T20:38:40.474541+0000","last_clean":"2023-09-23T20:38:40.474541+0000","last_became_active":"2023-09-23T20:37:14.358982+0000","last_became_peered":"2023-09-23T20:37:14.358982+0000","last_unstale":"2023-09-23T20:38:40.474541+0000","last_undegraded":"2023-09-23T20:38:40.474541+0000","last_fullsized":"2023-09-23T20:38:40.474541+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:50:00.008145+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":364,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T20:38:40.476874+0000","last_change":"2023-09-23T20:37:14.359068+0000","last_active":"2023-09-23T20:38:40.476874+0000","last_peered":"2023-09-23T20:38:40.476874+0000","last_clean":"2023-09-23T20:38:40.476874+0000","last_became_active":"2023-09-23T20:37:14.358729+0000","last_became_peered":"2023-09-23T20:37:14.358729+0000","last_unstale":"2023-09-23T20:38:40.476874+0000","last_undegraded":"2023-09-23T20:38:40.476874+0000","last_fullsized":"2023-09-23T20:38:40.476874+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:19:17.084920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"420'6878","reported_seq":6939,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-23T20:38:46.174924+0000","last_change":"2023-09-23T20:36:11.482126+0000","last_active":"2023-09-23T20:38:46.174924+0000","last_peered":"2023-09-23T20:38:46.174924+0000","last_clean":"2023-09-23T20:38:46.174924+0000","last_became_active":"2023-09-23T20:36:11.481802+0000","last_became_peered":"2023-09-23T20:36:11.481802+0000","last_unstale":"2023-09-23T20:38:46.174924+0000","last_undegraded":"2023-09-23T20:38:46.174924+0000","last_fullsized":"2023-09-23T20:38:46.174924+0000","mapping_epoch":369,"log_start":"373'1359","ondisk_log_start":"373'1359","created":369,"last_epoch_clean":370,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":5519,"log_dups_size":0,"ondisk_log_size":5519,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:13:12.263455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":290062336,"num_objects":4426,"num_object_clones":0,"num_object_copies":13278,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4426,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6878,"num_write_kb":361728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":698,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T20:38:40.488623+0000","last_change":"2023-09-23T20:38:05.590361+0000","last_active":"2023-09-23T20:38:40.488623+0000","last_peered":"2023-09-23T20:38:40.488623+0000","last_clean":"2023-09-23T20:38:40.488623+0000","last_became_active":"2023-09-23T20:38:05.589985+0000","last_became_peered":"2023-09-23T20:38:05.589985+0000","last_unstale":"2023-09-23T20:38:40.488623+0000","last_undegraded":"2023-09-23T20:38:40.488623+0000","last_fullsized":"2023-09-23T20:38:40.488623+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:03.246271+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:03.246271+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:38:54.351576+0000","scrub_duration":0.0019016700000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":157,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T20:38:40.501359+0000","last_change":"2023-09-23T20:32:50.549695+0000","last_active":"2023-09-23T20:38:40.501359+0000","last_peered":"2023-09-23T20:38:40.501359+0000","last_clean":"2023-09-23T20:38:40.501359+0000","last_became_active":"2023-09-23T20:32:50.549431+0000","last_became_peered":"2023-09-23T20:32:50.549431+0000","last_unstale":"2023-09-23T20:38:40.501359+0000","last_undegraded":"2023-09-23T20:38:40.501359+0000","last_fullsized":"2023-09-23T20:38:40.501359+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:45:53.666569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"86'128","reported_seq":830,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T20:38:40.475197+0000","last_change":"2023-09-23T20:38:03.541765+0000","last_active":"2023-09-23T20:38:40.475197+0000","last_peered":"2023-09-23T20:38:40.475197+0000","last_clean":"2023-09-23T20:38:40.475197+0000","last_became_active":"2023-09-23T20:38:03.541277+0000","last_became_peered":"2023-09-23T20:38:03.541277+0000","last_unstale":"2023-09-23T20:38:40.475197+0000","last_undegraded":"2023-09-23T20:38:40.475197+0000","last_fullsized":"2023-09-23T20:38:40.475197+0000","mapping_epoch":404,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":405,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:20:53.834554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000293202,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":579,"num_read_kb":1473,"num_write":96,"num_write_kb":1062,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":197,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T20:38:40.495062+0000","last_change":"2023-09-23T20:37:43.582105+0000","last_active":"2023-09-23T20:38:40.495062+0000","last_peered":"2023-09-23T20:38:40.495062+0000","last_clean":"2023-09-23T20:38:40.495062+0000","last_became_active":"2023-09-23T20:37:43.581860+0000","last_became_peered":"2023-09-23T20:37:43.581860+0000","last_unstale":"2023-09-23T20:38:40.495062+0000","last_undegraded":"2023-09-23T20:38:40.495062+0000","last_fullsized":"2023-09-23T20:38:40.495062+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:18:38.765486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"420'6852","reported_seq":8766,"reported_epoch":420,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:38:47.179242+0000","last_change":"2023-09-23T20:38:02.710271+0000","last_active":"2023-09-23T20:38:47.179242+0000","last_peered":"2023-09-23T20:38:47.179242+0000","last_clean":"2023-09-23T20:37:13.317209+0000","last_became_active":"2023-09-23T20:38:02.636987+0000","last_became_peered":"2023-09-23T20:38:02.636987+0000","last_unstale":"2023-09-23T20:38:47.179242+0000","last_undegraded":"2023-09-23T20:38:02.547074+0000","last_fullsized":"2023-09-23T20:38:02.526059+0000","mapping_epoch":403,"log_start":"373'1221","ondisk_log_start":"373'1221","created":369,"last_epoch_clean":370,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":5631,"log_dups_size":0,"ondisk_log_size":5631,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:15:55.380701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":280166424,"num_objects":4276,"num_object_clones":0,"num_object_copies":12828,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5420,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4276,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":6852,"num_write_kb":356033,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":913,"num_bytes_recovered":59834368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2],"avail_no_missing":["7","2","5"],"object_location_counts":[{"shards":"1,2,7","objects":280},{"shards":"2,5,7","objects":3981},{"shards":"2,7","objects":-833}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.4","version":"90'161","reported_seq":614,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T20:38:40.500559+0000","last_change":"2023-09-23T20:38:05.607009+0000","last_active":"2023-09-23T20:38:40.500559+0000","last_peered":"2023-09-23T20:38:40.500559+0000","last_clean":"2023-09-23T20:38:40.500559+0000","last_became_active":"2023-09-23T20:38:05.606621+0000","last_became_peered":"2023-09-23T20:38:05.606621+0000","last_unstale":"2023-09-23T20:38:40.500559+0000","last_undegraded":"2023-09-23T20:38:40.500559+0000","last_fullsized":"2023-09-23T20:38:40.500559+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:34:54.504554+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:34:54.504554+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:47:23.494026+0000","scrub_duration":0.011176864,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"86'117","reported_seq":653,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T20:38:40.482779+0000","last_change":"2023-09-23T20:38:28.446107+0000","last_active":"2023-09-23T20:38:40.482779+0000","last_peered":"2023-09-23T20:38:40.482779+0000","last_clean":"2023-09-23T20:38:40.482779+0000","last_became_active":"2023-09-23T20:38:28.445667+0000","last_became_peered":"2023-09-23T20:38:28.445667+0000","last_unstale":"2023-09-23T20:38:40.482779+0000","last_undegraded":"2023-09-23T20:38:40.482779+0000","last_fullsized":"2023-09-23T20:38:40.482779+0000","mapping_epoch":411,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":412,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:58:47.430360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046891400000000002,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":893,"num_read_kb":922,"num_write":82,"num_write_kb":4193,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":418,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T20:38:40.477698+0000","last_change":"2023-09-23T20:32:33.277827+0000","last_active":"2023-09-23T20:38:40.477698+0000","last_peered":"2023-09-23T20:38:40.477698+0000","last_clean":"2023-09-23T20:38:40.477698+0000","last_became_active":"2023-09-23T20:32:33.236087+0000","last_became_peered":"2023-09-23T20:32:33.236087+0000","last_unstale":"2023-09-23T20:38:40.477698+0000","last_undegraded":"2023-09-23T20:38:40.477698+0000","last_fullsized":"2023-09-23T20:38:40.477698+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:20:58.176854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":147,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T20:38:40.486600+0000","last_change":"2023-09-23T20:38:02.700897+0000","last_active":"2023-09-23T20:38:40.486600+0000","last_peered":"2023-09-23T20:38:40.486600+0000","last_clean":"2023-09-23T20:38:40.486600+0000","last_became_active":"2023-09-23T20:38:02.700561+0000","last_became_peered":"2023-09-23T20:38:02.700561+0000","last_unstale":"2023-09-23T20:38:40.486600+0000","last_undegraded":"2023-09-23T20:38:40.486600+0000","last_fullsized":"2023-09-23T20:38:40.486600+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:31:11.316211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.10","version":"420'6637","reported_seq":9311,"reported_epoch":420,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T20:38:48.783457+0000","last_change":"2023-09-23T20:38:27.597257+0000","last_active":"2023-09-23T20:38:48.783457+0000","last_peered":"2023-09-23T20:38:48.783457+0000","last_clean":"2023-09-23T20:37:13.293179+0000","last_became_active":"2023-09-23T20:38:27.574914+0000","last_became_peered":"2023-09-23T20:38:27.574914+0000","last_unstale":"2023-09-23T20:38:48.783457+0000","last_undegraded":"2023-09-23T20:38:27.519334+0000","last_fullsized":"2023-09-23T20:38:48.783457+0000","mapping_epoch":410,"log_start":"373'2187","ondisk_log_start":"373'2187","created":410,"last_epoch_clean":370,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":4450,"log_dups_size":0,"ondisk_log_size":4450,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":135331840,"num_objects":2065,"num_object_clones":0,"num_object_copies":6195,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1871,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2065,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3390,"num_write_kb":174560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":666,"num_bytes_recovered":43646976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":["3","1","5"],"object_location_counts":[{"shards":"1,3,4","objects":194},{"shards":"1,3,5","objects":1871}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.19","version":"90'161","reported_seq":688,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T20:38:40.477496+0000","last_change":"2023-09-23T20:38:05.630221+0000","last_active":"2023-09-23T20:38:40.477496+0000","last_peered":"2023-09-23T20:38:40.477496+0000","last_clean":"2023-09-23T20:38:40.477496+0000","last_became_active":"2023-09-23T20:38:05.629839+0000","last_became_peered":"2023-09-23T20:38:05.629839+0000","last_unstale":"2023-09-23T20:38:40.477496+0000","last_undegraded":"2023-09-23T20:38:40.477496+0000","last_fullsized":"2023-09-23T20:38:40.477496+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:06.487351+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:06.487351+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:25:51.325529+0000","scrub_duration":0.001964379,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":386,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T20:38:40.476781+0000","last_change":"2023-09-23T20:32:33.237234+0000","last_active":"2023-09-23T20:38:40.476781+0000","last_peered":"2023-09-23T20:38:40.476781+0000","last_clean":"2023-09-23T20:38:40.476781+0000","last_became_active":"2023-09-23T20:32:33.236881+0000","last_became_peered":"2023-09-23T20:32:33.236881+0000","last_unstale":"2023-09-23T20:38:40.476781+0000","last_undegraded":"2023-09-23T20:38:40.476781+0000","last_fullsized":"2023-09-23T20:38:40.476781+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:56:17.384831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":274,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T20:38:40.479434+0000","last_change":"2023-09-23T20:29:45.538314+0000","last_active":"2023-09-23T20:38:40.479434+0000","last_peered":"2023-09-23T20:38:40.479434+0000","last_clean":"2023-09-23T20:38:40.479434+0000","last_became_active":"2023-09-23T20:29:45.536847+0000","last_became_peered":"2023-09-23T20:29:45.536847+0000","last_unstale":"2023-09-23T20:38:40.479434+0000","last_undegraded":"2023-09-23T20:38:40.479434+0000","last_fullsized":"2023-09-23T20:38:40.479434+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:51.948721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"8.17","version":"420'6738","reported_seq":6824,"reported_epoch":420,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T20:38:49.349538+0000","last_change":"2023-09-23T20:38:27.525765+0000","last_active":"2023-09-23T20:38:49.349538+0000","last_peered":"2023-09-23T20:38:49.349538+0000","last_clean":"2023-09-23T20:37:19.384160+0000","last_became_active":"2023-09-23T20:38:27.474930+0000","last_became_peered":"2023-09-23T20:38:27.474930+0000","last_unstale":"2023-09-23T20:38:49.349538+0000","last_undegraded":"2023-09-23T20:38:27.459263+0000","last_fullsized":"2023-09-23T20:38:49.349538+0000","mapping_epoch":410,"log_start":"373'1658","ondisk_log_start":"373'1658","created":410,"last_epoch_clean":370,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":5080,"log_dups_size":0,"ondisk_log_size":5080,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":138215424,"num_objects":2109,"num_object_clones":0,"num_object_copies":6327,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":326,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2109,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3448,"num_write_kb":177824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":["1","3"],"object_location_counts":[{"shards":"1,2,3","objects":158},{"shards":"1,3,6","objects":1951}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1e","version":"90'161","reported_seq":478,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T20:38:40.494305+0000","last_change":"2023-09-23T20:38:05.602250+0000","last_active":"2023-09-23T20:38:40.494305+0000","last_peered":"2023-09-23T20:38:40.494305+0000","last_clean":"2023-09-23T20:38:40.494305+0000","last_became_active":"2023-09-23T20:38:05.601874+0000","last_became_peered":"2023-09-23T20:38:05.601874+0000","last_unstale":"2023-09-23T20:38:40.494305+0000","last_undegraded":"2023-09-23T20:38:40.494305+0000","last_fullsized":"2023-09-23T20:38:40.494305+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:35:42.864363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:35:42.864363+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":40253,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":17,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T20:38:39.479970+0000","last_change":"2023-09-23T20:38:37.481551+0000","last_active":"2023-09-23T20:38:39.479970+0000","last_peered":"2023-09-23T20:38:39.479970+0000","last_clean":"2023-09-23T20:38:39.479970+0000","last_became_active":"2023-09-23T20:38:37.458687+0000","last_became_peered":"2023-09-23T20:38:37.458687+0000","last_unstale":"2023-09-23T20:38:39.479970+0000","last_undegraded":"2023-09-23T20:38:39.479970+0000","last_fullsized":"2023-09-23T20:38:39.479970+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:22:15.033243+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":76,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T20:38:40.485713+0000","last_change":"2023-09-23T20:34:59.987575+0000","last_active":"2023-09-23T20:38:40.485713+0000","last_peered":"2023-09-23T20:38:40.485713+0000","last_clean":"2023-09-23T20:38:40.485713+0000","last_became_active":"2023-09-23T20:34:59.987229+0000","last_became_peered":"2023-09-23T20:34:59.987229+0000","last_unstale":"2023-09-23T20:38:40.485713+0000","last_undegraded":"2023-09-23T20:38:40.485713+0000","last_fullsized":"2023-09-23T20:38:40.485713+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:34:01.820159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.16","version":"420'6661","reported_seq":7420,"reported_epoch":420,"state":"active+clean","last_fresh":"2023-09-23T20:38:46.166217+0000","last_change":"2023-09-23T20:38:28.264940+0000","last_active":"2023-09-23T20:38:46.166217+0000","last_peered":"2023-09-23T20:38:46.166217+0000","last_clean":"2023-09-23T20:38:46.166217+0000","last_became_active":"2023-09-23T20:38:27.484333+0000","last_became_peered":"2023-09-23T20:38:27.484333+0000","last_unstale":"2023-09-23T20:38:46.166217+0000","last_undegraded":"2023-09-23T20:38:46.166217+0000","last_fullsized":"2023-09-23T20:38:46.166217+0000","mapping_epoch":410,"log_start":"373'1261","ondisk_log_start":"373'1261","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":5400,"log_dups_size":0,"ondisk_log_size":5400,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":142409728,"num_objects":2173,"num_object_clones":0,"num_object_copies":6519,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2173,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3381,"num_write_kb":177728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":335,"num_bytes_recovered":9699328,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":17,"reported_epoch":418,"state":"active+clean","last_fresh":"2023-09-23T20:38:39.480828+0000","last_change":"2023-09-23T20:38:37.457634+0000","last_active":"2023-09-23T20:38:39.480828+0000","last_peered":"2023-09-23T20:38:39.480828+0000","last_clean":"2023-09-23T20:38:39.480828+0000","last_became_active":"2023-09-23T20:38:37.452208+0000","last_became_peered":"2023-09-23T20:38:37.452208+0000","last_unstale":"2023-09-23T20:38:39.480828+0000","last_undegraded":"2023-09-23T20:38:39.480828+0000","last_fullsized":"2023-09-23T20:38:39.480828+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:09:41.754569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":76,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T20:38:40.486295+0000","last_change":"2023-09-23T20:34:59.987416+0000","last_active":"2023-09-23T20:38:40.486295+0000","last_peered":"2023-09-23T20:38:40.486295+0000","last_clean":"2023-09-23T20:38:40.486295+0000","last_became_active":"2023-09-23T20:34:59.987117+0000","last_became_peered":"2023-09-23T20:34:59.987117+0000","last_unstale":"2023-09-23T20:38:40.486295+0000","last_undegraded":"2023-09-23T20:38:40.486295+0000","last_fullsized":"2023-09-23T20:38:40.486295+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:36:16.318623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.15","version":"420'6730","reported_seq":9568,"reported_epoch":420,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T20:38:47.475060+0000","last_change":"2023-09-23T20:38:28.599996+0000","last_active":"2023-09-23T20:38:47.475060+0000","last_peered":"2023-09-23T20:38:47.475060+0000","last_clean":"2023-09-23T20:37:15.983935+0000","last_became_active":"2023-09-23T20:38:28.579407+0000","last_became_peered":"2023-09-23T20:38:28.579407+0000","last_unstale":"2023-09-23T20:38:47.475060+0000","last_undegraded":"2023-09-23T20:38:28.414306+0000","last_fullsized":"2023-09-23T20:38:47.475060+0000","mapping_epoch":410,"log_start":"373'1839","ondisk_log_start":"373'1839","created":410,"last_epoch_clean":370,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":4891,"log_dups_size":0,"ondisk_log_size":4891,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":138838016,"num_objects":2119,"num_object_clones":0,"num_object_copies":6357,"num_objects_missing_on_primary":354,"num_objects_missing":354,"num_objects_degraded":354,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2119,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3435,"num_write_kb":177696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":689,"num_bytes_recovered":45121536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":["4","7"],"object_location_counts":[{"shards":"0,4,7","objects":171},{"shards":"4,6,7","objects":1948}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1c","version":"90'161","reported_seq":675,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T20:38:40.497642+0000","last_change":"2023-09-23T20:38:05.591992+0000","last_active":"2023-09-23T20:38:40.497642+0000","last_peered":"2023-09-23T20:38:40.497642+0000","last_clean":"2023-09-23T20:38:40.497642+0000","last_became_active":"2023-09-23T20:38:05.591701+0000","last_became_peered":"2023-09-23T20:38:05.591701+0000","last_unstale":"2023-09-23T20:38:40.497642+0000","last_undegraded":"2023-09-23T20:38:40.497642+0000","last_fullsized":"2023-09-23T20:38:40.497642+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:02.384133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:02.384133+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:41.722542+0000","scrub_duration":0.001769098,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":239,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T20:38:40.494510+0000","last_change":"2023-09-23T20:38:02.542719+0000","last_active":"2023-09-23T20:38:40.494510+0000","last_peered":"2023-09-23T20:38:40.494510+0000","last_clean":"2023-09-23T20:38:40.494510+0000","last_became_active":"2023-09-23T20:38:02.542385+0000","last_became_peered":"2023-09-23T20:38:02.542385+0000","last_unstale":"2023-09-23T20:38:40.494510+0000","last_undegraded":"2023-09-23T20:38:40.494510+0000","last_fullsized":"2023-09-23T20:38:40.494510+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:36:23.673757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":206,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T20:38:40.494033+0000","last_change":"2023-09-23T20:37:43.584253+0000","last_active":"2023-09-23T20:38:40.494033+0000","last_peered":"2023-09-23T20:38:40.494033+0000","last_clean":"2023-09-23T20:38:40.494033+0000","last_became_active":"2023-09-23T20:37:43.580069+0000","last_became_peered":"2023-09-23T20:37:43.580069+0000","last_unstale":"2023-09-23T20:38:40.494033+0000","last_undegraded":"2023-09-23T20:38:40.494033+0000","last_fullsized":"2023-09-23T20:38:40.494033+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:39:11.482581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.14","version":"419'6922","reported_seq":8502,"reported_epoch":419,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:38:45.497265+0000","last_change":"2023-09-23T20:38:39.260381+0000","last_active":"2023-09-23T20:38:45.497265+0000","last_peered":"2023-09-23T20:38:45.497265+0000","last_clean":"2023-09-23T20:37:15.925522+0000","last_became_active":"2023-09-23T20:38:39.251212+0000","last_became_peered":"2023-09-23T20:38:39.251212+0000","last_unstale":"2023-09-23T20:38:45.497265+0000","last_undegraded":"2023-09-23T20:38:38.457918+0000","last_fullsized":"2023-09-23T20:38:38.450156+0000","mapping_epoch":416,"log_start":"373'2080","ondisk_log_start":"373'2080","created":410,"last_epoch_clean":370,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":4842,"log_dups_size":0,"ondisk_log_size":4842,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":142737408,"num_objects":2178,"num_object_clones":0,"num_object_copies":6534,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":693,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2178,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3515,"num_write_kb":182176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":376,"num_bytes_recovered":24641536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[4,2],"avail_no_missing":["4","2","6"],"object_location_counts":[{"shards":"2,3,4","objects":149},{"shards":"2,4","objects":2029}],"blocked_by":[],"up_primary":5,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1d","version":"90'161","reported_seq":511,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T20:38:40.474767+0000","last_change":"2023-09-23T20:36:01.505098+0000","last_active":"2023-09-23T20:38:40.474767+0000","last_peered":"2023-09-23T20:38:40.474767+0000","last_clean":"2023-09-23T20:38:40.474767+0000","last_became_active":"2023-09-23T20:35:26.644707+0000","last_became_peered":"2023-09-23T20:35:26.644707+0000","last_unstale":"2023-09-23T20:38:40.474767+0000","last_undegraded":"2023-09-23T20:38:40.474767+0000","last_fullsized":"2023-09-23T20:38:40.474767+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:01.504973+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:01.504973+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:02:10.880809+0000","scrub_duration":0.00202344,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":448,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T20:38:40.477155+0000","last_change":"2023-09-23T20:38:02.655107+0000","last_active":"2023-09-23T20:38:40.477155+0000","last_peered":"2023-09-23T20:38:40.477155+0000","last_clean":"2023-09-23T20:38:40.477155+0000","last_became_active":"2023-09-23T20:38:02.574184+0000","last_became_peered":"2023-09-23T20:38:02.574184+0000","last_unstale":"2023-09-23T20:38:40.477155+0000","last_undegraded":"2023-09-23T20:38:40.477155+0000","last_fullsized":"2023-09-23T20:38:40.477155+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:59:37.398085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2770,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":137,"reported_epoch":419,"state":"active+clean","last_fresh":"2023-09-23T20:38:40.486605+0000","last_change":"2023-09-23T20:38:02.538333+0000","last_active":"2023-09-23T20:38:40.486605+0000","last_peered":"2023-09-23T20:38:40.486605+0000","last_clean":"2023-09-23T20:38:40.486605+0000","last_became_active":"2023-09-23T20:38:02.538008+0000","last_became_peered":"2023-09-23T20:38:02.538008+0000","last_unstale":"2023-09-23T20:38:40.486605+0000","last_undegraded":"2023-09-23T20:38:40.486605+0000","last_fullsized":"2023-09-23T20:38:40.486605+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:02:38.789530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":73,"num_bytes_recovered":17862,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26691,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":8,"num_pg":26,"stat_sum":{"num_bytes":4489871384,"num_objects":68511,"num_object_clones":0,"num_object_copies":205533,"num_objects_missing_on_primary":1011,"num_objects_missing":1011,"num_objects_degraded":23715,"num_objects_misplaced":3348,"num_objects_unfound":0,"num_objects_dirty":68511,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":110131,"num_write_kb":5716481,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":13893,"num_bytes_recovered":875823104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":14573973504,"data_stored":14573961288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":131479,"ondisk_log_size":131479,"up":78,"acting":72,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":351684,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":155,"num_read_kb":290,"num_write":76,"num_write_kb":768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1153900,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5983,"num_read_kb":13200,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":57,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16789504,"data_stored":16777273,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6082,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201608,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1747600,"kb_used_data":1391048,"kb_used_omap":0,"kb_used_meta":356544,"kb_avail":91997552,"statfs":{"total":95995035648,"available":94205493248,"internally_reserved":0,"allocated":1424433152,"data_stored":1423107551,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":365101056},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:37:48 2023","interfaces":[{"interface":"back","average":{"1min":2.8460000000000001,"5min":2.9020000000000001,"15min":2.0609999999999999},"min":{"1min":0.26500000000000001,"5min":0.23599999999999999,"15min":0.20799999999999999},"max":{"1min":12.640000000000001,"5min":17.399999999999999,"15min":48.798999999999999},"last":0.34699999999999998},{"interface":"front","average":{"1min":1.0049999999999999,"5min":2.0579999999999998,"15min":1.952},"min":{"1min":0.247,"5min":0.247,"15min":0.24399999999999999},"max":{"1min":4.1959999999999997,"5min":19.355,"15min":48.247},"last":0.503}]},{"osd":1,"last update":"Sat Sep 23 20:38:03 2023","interfaces":[{"interface":"back","average":{"1min":1.4530000000000001,"5min":2.5419999999999998,"15min":2.0630000000000002},"min":{"1min":0.32000000000000001,"5min":0.25900000000000001,"15min":0.254},"max":{"1min":5.8399999999999999,"5min":49.965000000000003,"15min":49.965000000000003},"last":0.307},{"interface":"front","average":{"1min":1.9570000000000001,"5min":2.859,"15min":2.1699999999999999},"min":{"1min":0.315,"5min":0.29999999999999999,"15min":0.248},"max":{"1min":6.2960000000000003,"5min":53.451999999999998,"15min":53.451999999999998},"last":2.3620000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:37:48 2023","interfaces":[{"interface":"back","average":{"1min":1.46,"5min":2.3879999999999999,"15min":1.992},"min":{"1min":0.30399999999999999,"5min":0.27800000000000002,"15min":0.27600000000000002},"max":{"1min":5.3879999999999999,"5min":12.196999999999999,"15min":13.173999999999999},"last":0.439},{"interface":"front","average":{"1min":1.8839999999999999,"5min":2.6960000000000002,"15min":2.2530000000000001},"min":{"1min":0.32500000000000001,"5min":0.27400000000000002,"15min":0.26600000000000001},"max":{"1min":9.1159999999999997,"5min":23.494,"15min":23.494},"last":0.85399999999999998}]},{"osd":3,"last update":"Sat Sep 23 20:38:18 2023","interfaces":[{"interface":"back","average":{"1min":3.173,"5min":2.956,"15min":2.355},"min":{"1min":0.376,"5min":0.35199999999999998,"15min":0.26200000000000001},"max":{"1min":21.018000000000001,"5min":21.018000000000001,"15min":21.018000000000001},"last":0.55400000000000005},{"interface":"front","average":{"1min":3.1760000000000002,"5min":3.0259999999999998,"15min":2.2890000000000001},"min":{"1min":0.45000000000000001,"5min":0.34399999999999997,"15min":0.27600000000000002},"max":{"1min":20.969999999999999,"5min":20.969999999999999,"15min":20.969999999999999},"last":0.56499999999999995}]},{"osd":4,"last update":"Sat Sep 23 20:38:18 2023","interfaces":[{"interface":"back","average":{"1min":0.44600000000000001,"5min":0.85899999999999999,"15min":1.1519999999999999},"min":{"1min":0.27200000000000002,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":1.5429999999999999,"5min":6.3449999999999998,"15min":15.177},"last":0.39000000000000001},{"interface":"front","average":{"1min":0.46500000000000002,"5min":1.0529999999999999,"15min":1.3220000000000001},"min":{"1min":0.30399999999999999,"5min":0.28399999999999997,"15min":0.27600000000000002},"max":{"1min":0.73199999999999998,"5min":6.5339999999999998,"15min":11.122},"last":0.47299999999999998}]},{"osd":5,"last update":"Sat Sep 23 20:38:18 2023","interfaces":[{"interface":"back","average":{"1min":0.47399999999999998,"5min":0.92100000000000004,"15min":1.24},"min":{"1min":0.29699999999999999,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":1.1850000000000001,"5min":7.1159999999999997,"15min":13.241},"last":0.40500000000000003},{"interface":"front","average":{"1min":0.495,"5min":0.94999999999999996,"15min":1.3200000000000001},"min":{"1min":0.378,"5min":0.31900000000000001,"15min":0.315},"max":{"1min":0.69399999999999995,"5min":8.4809999999999999,"15min":18.831},"last":0.42199999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:38:18 2023","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":0.92300000000000004,"15min":1.1990000000000001},"min":{"1min":0.36699999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.69199999999999995,"5min":6.5540000000000003,"15min":13.185},"last":0.45600000000000002},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.98899999999999999,"15min":1.1910000000000001},"min":{"1min":0.35099999999999998,"5min":0.34699999999999998,"15min":0.318},"max":{"1min":0.77400000000000002,"5min":6.2770000000000001,"15min":9.9120000000000008},"last":0.53400000000000003}]}]},{"osd":6,"up_from":47,"seq":201863463247,"num_pgs":60,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1755716,"kb_used_data":1408700,"kb_used_omap":0,"kb_used_meta":346880,"kb_avail":91989436,"statfs":{"total":95995035648,"available":94197182464,"internally_reserved":0,"allocated":1442508800,"data_stored":1441195346,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":355205120},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:38:45 2023","interfaces":[{"interface":"back","average":{"1min":2.532,"5min":2.476,"15min":1.9890000000000001},"min":{"1min":0.26400000000000001,"5min":0.23300000000000001,"15min":0.221},"max":{"1min":14.891,"5min":34.143999999999998,"15min":53.701999999999998},"last":0.27100000000000002},{"interface":"front","average":{"1min":4.8259999999999996,"5min":2.9289999999999998,"15min":2.4710000000000001},"min":{"1min":0.26900000000000002,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":60.328000000000003,"5min":60.328000000000003,"15min":60.328000000000003},"last":0.84099999999999997}]},{"osd":1,"last update":"Sat Sep 23 20:38:04 2023","interfaces":[{"interface":"back","average":{"1min":1.633,"5min":2.165,"15min":2.3929999999999998},"min":{"1min":0.28599999999999998,"5min":0.25600000000000001,"15min":0.249},"max":{"1min":6.0170000000000003,"5min":34.173999999999999,"15min":34.173999999999999},"last":0.52300000000000002},{"interface":"front","average":{"1min":1.379,"5min":2.137,"15min":2.7759999999999998},"min":{"1min":0.30399999999999999,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":5.569,"5min":34.490000000000002,"15min":34.490000000000002},"last":0.91800000000000004}]},{"osd":2,"last update":"Sat Sep 23 20:38:45 2023","interfaces":[{"interface":"back","average":{"1min":2.2109999999999999,"5min":2.3100000000000001,"15min":2.1579999999999999},"min":{"1min":0.32100000000000001,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":13.010999999999999,"5min":34.317999999999998,"15min":34.317999999999998},"last":0.68200000000000005},{"interface":"front","average":{"1min":2.218,"5min":2.27,"15min":2.0449999999999999},"min":{"1min":0.41899999999999998,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":12.917999999999999,"5min":34.292999999999999,"15min":34.292999999999999},"last":0.65900000000000003}]},{"osd":3,"last update":"Sat Sep 23 20:38:33 2023","interfaces":[{"interface":"back","average":{"1min":2.0089999999999999,"5min":1.9590000000000001,"15min":1.952},"min":{"1min":0.308,"5min":0.29999999999999999,"15min":0.29599999999999999},"max":{"1min":8.7799999999999994,"5min":8.9670000000000005,"15min":9.4239999999999995},"last":4.5570000000000004},{"interface":"front","average":{"1min":3.0339999999999998,"5min":2.3399999999999999,"15min":2.1280000000000001},"min":{"1min":0.39100000000000001,"5min":0.34899999999999998,"15min":0.317},"max":{"1min":11.913,"5min":12.875999999999999,"15min":12.875999999999999},"last":1.536}]},{"osd":4,"last update":"Sat Sep 23 20:38:09 2023","interfaces":[{"interface":"back","average":{"1min":0.73699999999999999,"5min":0.67300000000000004,"15min":0.92000000000000004},"min":{"1min":0.28000000000000003,"5min":0.26900000000000002,"15min":0.251},"max":{"1min":3.1749999999999998,"5min":4.9279999999999999,"15min":10.173},"last":0.47699999999999998},{"interface":"front","average":{"1min":0.93100000000000005,"5min":1.0349999999999999,"15min":1.177},"min":{"1min":0.33000000000000002,"5min":0.29099999999999998,"15min":0.27000000000000002},"max":{"1min":6.335,"5min":8.2720000000000002,"15min":8.3249999999999993},"last":0.34399999999999997}]},{"osd":5,"last update":"Sat Sep 23 20:38:09 2023","interfaces":[{"interface":"back","average":{"1min":0.77200000000000002,"5min":0.79700000000000004,"15min":1.012},"min":{"1min":0.38,"5min":0.31,"15min":0.28699999999999998},"max":{"1min":3.141,"5min":4.8259999999999996,"15min":14.085000000000001},"last":0.443},{"interface":"front","average":{"1min":0.91400000000000003,"5min":0.93999999999999995,"15min":1.3839999999999999},"min":{"1min":0.38700000000000001,"5min":0.32000000000000001,"15min":0.30599999999999999},"max":{"1min":4.2969999999999997,"5min":6.7320000000000002,"15min":18.192},"last":0.496}]},{"osd":7,"last update":"Sat Sep 23 20:38:33 2023","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":0.76700000000000002,"15min":1.127},"min":{"1min":0.377,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":2.04,"5min":4.851,"15min":10.217000000000001},"last":0.56100000000000005},{"interface":"front","average":{"1min":0.60999999999999999,"5min":0.94499999999999995,"15min":1.329},"min":{"1min":0.36799999999999999,"5min":0.34499999999999997,"15min":0.30299999999999999},"max":{"1min":2.1389999999999998,"5min":8.8119999999999994,"15min":9.7360000000000007},"last":0.46000000000000002}]}]},{"osd":1,"up_from":266,"seq":1142461300846,"num_pgs":65,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2333540,"kb_used_data":1923868,"kb_used_omap":8,"kb_used_meta":409655,"kb_avail":91411612,"statfs":{"total":95995035648,"available":93605490688,"internally_reserved":0,"allocated":1970040832,"data_stored":1967769699,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8206,"internal_metadata":419487730},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:38:01 2023","interfaces":[{"interface":"back","average":{"1min":3.6200000000000001,"5min":2.9550000000000001,"15min":2.6850000000000001},"min":{"1min":0.221,"5min":0.221,"15min":0.218},"max":{"1min":10.991,"5min":14.531000000000001,"15min":52.689999999999998},"last":0.28499999999999998},{"interface":"front","average":{"1min":4.6130000000000004,"5min":3.863,"15min":3.395},"min":{"1min":0.253,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":17.169,"5min":26.588000000000001,"15min":52.662999999999997},"last":0.67500000000000004}]},{"osd":2,"last update":"Sat Sep 23 20:38:01 2023","interfaces":[{"interface":"back","average":{"1min":4.75,"5min":4.4950000000000001,"15min":4.484},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.20899999999999999},"max":{"1min":32.255000000000003,"5min":32.255000000000003,"15min":32.255000000000003},"last":3.613},{"interface":"front","average":{"1min":4.0910000000000002,"5min":3.052,"15min":2.383},"min":{"1min":0.27700000000000002,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":13.404999999999999,"5min":13.404999999999999,"15min":13.404999999999999},"last":0.254}]},{"osd":3,"last update":"Sat Sep 23 20:38:01 2023","interfaces":[{"interface":"back","average":{"1min":3.1059999999999999,"5min":3.3460000000000001,"15min":3.8660000000000001},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":10.935,"5min":15.247999999999999,"15min":80.745999999999995},"last":0.439},{"interface":"front","average":{"1min":3.008,"5min":3.2869999999999999,"15min":4.2949999999999999},"min":{"1min":0.36299999999999999,"5min":0.30399999999999999,"15min":0.22700000000000001},"max":{"1min":10.083,"5min":19.315000000000001,"15min":82.988},"last":0.73599999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:38:01 2023","interfaces":[{"interface":"back","average":{"1min":2.2879999999999998,"5min":2.7930000000000001,"15min":2.242},"min":{"1min":0.315,"5min":0.315,"15min":0.30299999999999999},"max":{"1min":7.6120000000000001,"5min":24.327999999999999,"15min":24.327999999999999},"last":3.6600000000000001},{"interface":"front","average":{"1min":1.8680000000000001,"5min":1.8009999999999999,"15min":1.847},"min":{"1min":0.36699999999999999,"5min":0.311,"15min":0.311},"max":{"1min":9.3339999999999996,"5min":11.574999999999999,"15min":11.574999999999999},"last":0.52500000000000002}]},{"osd":5,"last update":"Sat Sep 23 20:38:01 2023","interfaces":[{"interface":"back","average":{"1min":2.27,"5min":2.6869999999999998,"15min":2.536},"min":{"1min":0.41999999999999998,"5min":0.34100000000000003,"15min":0.33200000000000002},"max":{"1min":9.6419999999999995,"5min":33.401000000000003,"15min":33.401000000000003},"last":0.82999999999999996},{"interface":"front","average":{"1min":2.4020000000000001,"5min":3.206,"15min":2.4199999999999999},"min":{"1min":0.46700000000000003,"5min":0.376,"15min":0.36799999999999999},"max":{"1min":7.7199999999999998,"5min":33.417000000000002,"15min":33.417000000000002},"last":3.698}]},{"osd":6,"last update":"Sat Sep 23 20:38:01 2023","interfaces":[{"interface":"back","average":{"1min":2.9670000000000001,"5min":2.3700000000000001,"15min":4.9390000000000001},"min":{"1min":0.432,"5min":0.35899999999999999,"15min":0.35699999999999998},"max":{"1min":9.6999999999999993,"5min":9.6999999999999993,"15min":72.682000000000002},"last":0.77200000000000002},{"interface":"front","average":{"1min":2.343,"5min":2.976,"15min":4.6619999999999999},"min":{"1min":0.46200000000000002,"5min":0.41099999999999998,"15min":0.38800000000000001},"max":{"1min":7.6900000000000004,"5min":24.376000000000001,"15min":72.563999999999993},"last":3.766}]},{"osd":7,"last update":"Sat Sep 23 20:38:01 2023","interfaces":[{"interface":"back","average":{"1min":2.4969999999999999,"5min":2.6320000000000001,"15min":2.7770000000000001},"min":{"1min":0.39500000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":9.6720000000000006,"5min":18.437000000000001,"15min":18.437000000000001},"last":0.80300000000000005},{"interface":"front","average":{"1min":2.5470000000000002,"5min":3.0169999999999999,"15min":2.508},"min":{"1min":0.46800000000000003,"5min":0.441,"15min":0.42499999999999999},"max":{"1min":7.6580000000000004,"5min":24.524000000000001,"15min":24.524000000000001},"last":3.7320000000000002}]}]},{"osd":0,"up_from":94,"seq":403726926048,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2451960,"kb_used_data":1835440,"kb_used_omap":7,"kb_used_meta":616504,"kb_avail":91293192,"statfs":{"total":95995035648,"available":93484228608,"internally_reserved":0,"allocated":1879490560,"data_stored":1877377112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":631300113},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:38:05 2023","interfaces":[{"interface":"back","average":{"1min":1.667,"5min":3.3479999999999999,"15min":3.403},"min":{"1min":0.193,"5min":0.17199999999999999,"15min":0.17199999999999999},"max":{"1min":8.8659999999999997,"5min":42.000999999999998,"15min":42.000999999999998},"last":0.40699999999999997},{"interface":"front","average":{"1min":1.952,"5min":3.5329999999999999,"15min":3.7650000000000001},"min":{"1min":0.20399999999999999,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":5.7830000000000004,"5min":23.036000000000001,"15min":23.036000000000001},"last":0.89400000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:38:41 2023","interfaces":[{"interface":"back","average":{"1min":3.0099999999999998,"5min":3.2429999999999999,"15min":2.9609999999999999},"min":{"1min":0.29299999999999998,"5min":0.218,"15min":0.185},"max":{"1min":10.17,"5min":62.726999999999997,"15min":77.448999999999998},"last":0.29299999999999998},{"interface":"front","average":{"1min":3.8359999999999999,"5min":3.3490000000000002,"15min":2.8559999999999999},"min":{"1min":0.27300000000000002,"5min":0.247,"15min":0.188},"max":{"1min":26.445,"5min":62.689,"15min":81.724999999999994},"last":0.70199999999999996}]},{"osd":3,"last update":"Sat Sep 23 20:38:15 2023","interfaces":[{"interface":"back","average":{"1min":3.4670000000000001,"5min":3.23,"15min":2.899},"min":{"1min":0.32600000000000001,"5min":0.24099999999999999,"15min":0.23400000000000001},"max":{"1min":9.9079999999999995,"5min":22.486000000000001,"15min":81.400999999999996},"last":1.8600000000000001},{"interface":"front","average":{"1min":4.6989999999999998,"5min":3.9399999999999999,"15min":3.1520000000000001},"min":{"1min":0.33600000000000002,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":43.229999999999997,"5min":43.229999999999997,"15min":81.137},"last":0.33100000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:38:41 2023","interfaces":[{"interface":"back","average":{"1min":2.8170000000000002,"5min":2.1749999999999998,"15min":2.1589999999999998},"min":{"1min":0.311,"5min":0.28899999999999998,"15min":0.25900000000000001},"max":{"1min":9.6110000000000007,"5min":11.068,"15min":23.876999999999999},"last":0.47399999999999998},{"interface":"front","average":{"1min":1.4510000000000001,"5min":1.9570000000000001,"15min":1.958},"min":{"1min":0.46400000000000002,"5min":0.308,"15min":0.29299999999999998},"max":{"1min":5.5990000000000002,"5min":11.458,"15min":23.681999999999999},"last":0.46400000000000002}]},{"osd":5,"last update":"Sat Sep 23 20:38:41 2023","interfaces":[{"interface":"back","average":{"1min":1.4550000000000001,"5min":1.976,"15min":2.2789999999999999},"min":{"1min":0.52700000000000002,"5min":0.34399999999999997,"15min":0.34200000000000003},"max":{"1min":5.5410000000000004,"5min":11.497999999999999,"15min":24.198},"last":0.52700000000000002},{"interface":"front","average":{"1min":4.3460000000000001,"5min":2.5499999999999998,"15min":2.3900000000000001},"min":{"1min":0.41399999999999998,"5min":0.307,"15min":0.25600000000000001},"max":{"1min":26.228000000000002,"5min":26.228000000000002,"15min":26.228000000000002},"last":0.745}]},{"osd":6,"last update":"Sat Sep 23 20:38:41 2023","interfaces":[{"interface":"back","average":{"1min":1.911,"5min":2.2949999999999999,"15min":2.1190000000000002},"min":{"1min":0.496,"5min":0.372,"15min":0.318},"max":{"1min":7.742,"5min":11.526999999999999,"15min":24.408000000000001},"last":0.496},{"interface":"front","average":{"1min":4.3810000000000002,"5min":2.488,"15min":2.2130000000000001},"min":{"1min":0.39600000000000002,"5min":0.35499999999999998,"15min":0.29399999999999998},"max":{"1min":26.268000000000001,"5min":26.268000000000001,"15min":26.268000000000001},"last":0.81799999999999995}]},{"osd":7,"last update":"Sat Sep 23 20:38:41 2023","interfaces":[{"interface":"back","average":{"1min":4.3609999999999998,"5min":2.5510000000000002,"15min":2.411},"min":{"1min":0.46899999999999997,"5min":0.36099999999999999,"15min":0.30599999999999999},"max":{"1min":26.177,"5min":26.177,"15min":26.177},"last":0.79300000000000004},{"interface":"front","average":{"1min":2.8519999999999999,"5min":2.3410000000000002,"15min":2.5539999999999998},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":9.7200000000000006,"5min":11.112,"15min":24.669},"last":0.54000000000000004}]}]},{"osd":2,"up_from":104,"seq":446676598996,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2328252,"kb_used_data":1698868,"kb_used_omap":11,"kb_used_meta":629364,"kb_avail":91416900,"statfs":{"total":95995035648,"available":93610905600,"internally_reserved":0,"allocated":1739640832,"data_stored":1737521321,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11878,"internal_metadata":644469146},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:37:54 2023","interfaces":[{"interface":"back","average":{"1min":1.2649999999999999,"5min":3.1869999999999998,"15min":2.673},"min":{"1min":0.26600000000000001,"5min":0.20300000000000001,"15min":0.17999999999999999},"max":{"1min":4.6479999999999997,"5min":28.234999999999999,"15min":82.424999999999997},"last":0.68700000000000006},{"interface":"front","average":{"1min":2.0219999999999998,"5min":2.8759999999999999,"15min":2.5190000000000001},"min":{"1min":0.251,"5min":0.19,"15min":0.19},"max":{"1min":6.3940000000000001,"5min":14.055,"15min":82.524000000000001},"last":1.28}]},{"osd":1,"last update":"Sat Sep 23 20:37:57 2023","interfaces":[{"interface":"back","average":{"1min":3.161,"5min":3.0859999999999999,"15min":3.8380000000000001},"min":{"1min":0.252,"5min":0.246,"15min":0.221},"max":{"1min":19.393999999999998,"5min":19.393999999999998,"15min":19.393999999999998},"last":4.4669999999999996},{"interface":"front","average":{"1min":2.4540000000000002,"5min":3.5049999999999999,"15min":2.964},"min":{"1min":0.28000000000000003,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":11.253,"5min":21.585999999999999,"15min":30.565000000000001},"last":1.9370000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:38:19 2023","interfaces":[{"interface":"back","average":{"1min":3.9900000000000002,"5min":3.0539999999999998,"15min":2.3540000000000001},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.19800000000000001},"max":{"1min":19.562999999999999,"5min":19.562999999999999,"15min":19.562999999999999},"last":5.7389999999999999},{"interface":"front","average":{"1min":1.228,"5min":2.8439999999999999,"15min":2.504},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.23899999999999999},"max":{"1min":4.3520000000000003,"5min":15.173,"15min":30.704999999999998},"last":1.974}]},{"osd":4,"last update":"Sat Sep 23 20:37:54 2023","interfaces":[{"interface":"back","average":{"1min":1.8120000000000001,"5min":2.028,"15min":1.8660000000000001},"min":{"1min":0.34399999999999997,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":6.2869999999999999,"5min":9.2430000000000003,"15min":16.797999999999998},"last":1.3759999999999999},{"interface":"front","average":{"1min":2.6989999999999998,"5min":2.6920000000000002,"15min":2.0720000000000001},"min":{"1min":0.38600000000000001,"5min":0.30499999999999999,"15min":0.27600000000000002},"max":{"1min":19.440999999999999,"5min":19.440999999999999,"15min":19.440999999999999},"last":4.5220000000000002}]},{"osd":5,"last update":"Sat Sep 23 20:37:54 2023","interfaces":[{"interface":"back","average":{"1min":1.7190000000000001,"5min":2.1120000000000001,"15min":2.3849999999999998},"min":{"1min":0.38600000000000001,"5min":0.34799999999999998,"15min":0.309},"max":{"1min":6.3520000000000003,"5min":9.1799999999999997,"15min":95.393000000000001},"last":1.4239999999999999},{"interface":"front","average":{"1min":2.4159999999999999,"5min":2.7959999999999998,"15min":2.6499999999999999},"min":{"1min":0.41399999999999998,"5min":0.35299999999999998,"15min":0.32300000000000001},"max":{"1min":19.513000000000002,"5min":19.513000000000002,"15min":95.414000000000001},"last":4.5570000000000004}]},{"osd":6,"last update":"Sat Sep 23 20:37:54 2023","interfaces":[{"interface":"back","average":{"1min":2.4500000000000002,"5min":2.6520000000000001,"15min":2.1299999999999999},"min":{"1min":0.434,"5min":0.36599999999999999,"15min":0.32100000000000001},"max":{"1min":19.486000000000001,"5min":19.486000000000001,"15min":19.486000000000001},"last":4.5869999999999997},{"interface":"front","average":{"1min":1.276,"5min":2.23,"15min":2.0169999999999999},"min":{"1min":0.374,"5min":0.318,"15min":0.316},"max":{"1min":4.4029999999999996,"5min":11.523,"15min":30.670999999999999},"last":0.76400000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:37:54 2023","interfaces":[{"interface":"back","average":{"1min":1.3879999999999999,"5min":2.2410000000000001,"15min":2.085},"min":{"1min":0.5,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":4.4500000000000002,"5min":12.734999999999999,"15min":30.741},"last":0.72999999999999998},{"interface":"front","average":{"1min":2.1989999999999998,"5min":2.2450000000000001,"15min":2.145},"min":{"1min":0.443,"5min":0.39800000000000002,"15min":0.38},"max":{"1min":6.3220000000000001,"5min":9.2720000000000002,"15min":16.719000000000001},"last":1.341}]}]},{"osd":3,"up_from":109,"seq":468151435470,"num_pgs":69,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2520924,"kb_used_data":1814036,"kb_used_omap":4,"kb_used_meta":706875,"kb_avail":91224228,"statfs":{"total":95995035648,"available":93413609472,"internally_reserved":0,"allocated":1857572864,"data_stored":1855481189,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4514,"internal_metadata":723840606},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:38:20 2023","interfaces":[{"interface":"back","average":{"1min":2.306,"5min":3.0249999999999999,"15min":2.6389999999999998},"min":{"1min":0.29099999999999998,"5min":0.215,"15min":0.186},"max":{"1min":9.0850000000000009,"5min":17.129000000000001,"15min":67.305999999999997},"last":0.39700000000000002},{"interface":"front","average":{"1min":2.0529999999999999,"5min":2.8559999999999999,"15min":2.452},"min":{"1min":0.28599999999999998,"5min":0.23699999999999999,"15min":0.19500000000000001},"max":{"1min":8.8200000000000003,"5min":12.502000000000001,"15min":67.403999999999996},"last":0.26800000000000002}]},{"osd":1,"last update":"Sat Sep 23 20:37:58 2023","interfaces":[{"interface":"back","average":{"1min":1.3660000000000001,"5min":2.1259999999999999,"15min":2.6240000000000001},"min":{"1min":0.23999999999999999,"5min":0.23799999999999999,"15min":0.222},"max":{"1min":6.4489999999999998,"5min":16.189,"15min":16.189},"last":0.56299999999999994},{"interface":"front","average":{"1min":2.6339999999999999,"5min":2.3759999999999999,"15min":2.8719999999999999},"min":{"1min":0.27300000000000002,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":12.702999999999999,"5min":13.727,"15min":32.603999999999999},"last":0.47299999999999998}]},{"osd":2,"last update":"Sat Sep 23 20:38:20 2023","interfaces":[{"interface":"back","average":{"1min":3.3620000000000001,"5min":2.927,"15min":2.5990000000000002},"min":{"1min":0.25800000000000001,"5min":0.23799999999999999,"15min":0.22800000000000001},"max":{"1min":10.747999999999999,"5min":20.405000000000001,"15min":32.857999999999997},"last":0.313},{"interface":"front","average":{"1min":4.9720000000000004,"5min":3.129,"15min":2.5710000000000002},"min":{"1min":0.28100000000000003,"5min":0.251,"15min":0.20899999999999999},"max":{"1min":22.32,"5min":24.146999999999998,"15min":24.146999999999998},"last":1.8129999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:38:20 2023","interfaces":[{"interface":"back","average":{"1min":1.7110000000000001,"5min":1.679,"15min":1.8340000000000001},"min":{"1min":0.377,"5min":0.32200000000000001,"15min":0.29799999999999999},"max":{"1min":5.9930000000000003,"5min":12.151,"15min":46.042999999999999},"last":0.60799999999999998},{"interface":"front","average":{"1min":3.7320000000000002,"5min":2.4750000000000001,"15min":2.351},"min":{"1min":0.373,"5min":0.29399999999999998,"15min":0.28100000000000003},"max":{"1min":13.571999999999999,"5min":14.422000000000001,"15min":46.579999999999998},"last":0.374}]},{"osd":5,"last update":"Sat Sep 23 20:38:20 2023","interfaces":[{"interface":"back","average":{"1min":1.75,"5min":1.883,"15min":2.0960000000000001},"min":{"1min":0.40000000000000002,"5min":0.32300000000000001,"15min":0.311},"max":{"1min":6.024,"5min":13.228999999999999,"15min":45.292000000000002},"last":0.64900000000000002},{"interface":"front","average":{"1min":1.5589999999999999,"5min":1.7110000000000001,"15min":1.9339999999999999},"min":{"1min":0.442,"5min":0.32000000000000001,"15min":0.313},"max":{"1min":4.9770000000000003,"5min":10.300000000000001,"15min":44.679000000000002},"last":0.42599999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:38:20 2023","interfaces":[{"interface":"back","average":{"1min":1.571,"5min":1.671,"15min":1.9359999999999999},"min":{"1min":0.41599999999999998,"5min":0.40500000000000003,"15min":0.34899999999999998},"max":{"1min":5.0620000000000003,"5min":12.464,"15min":17.524999999999999},"last":0.36199999999999999},{"interface":"front","average":{"1min":3.774,"5min":2.5070000000000001,"15min":2.1309999999999998},"min":{"1min":0.47799999999999998,"5min":0.371,"15min":0.33700000000000002},"max":{"1min":13.500999999999999,"5min":14.464,"15min":19.524000000000001},"last":0.58499999999999996}]},{"osd":7,"last update":"Sat Sep 23 20:38:20 2023","interfaces":[{"interface":"back","average":{"1min":4.3659999999999997,"5min":2.637,"15min":2.3380000000000001},"min":{"1min":0.44700000000000001,"5min":0.41499999999999998,"15min":0.34999999999999998},"max":{"1min":13.544,"5min":14.521000000000001,"15min":19.571999999999999},"last":0.63},{"interface":"front","average":{"1min":1.762,"5min":1.9610000000000001,"15min":2.1600000000000001},"min":{"1min":0.50700000000000001,"5min":0.44600000000000001,"15min":0.371},"max":{"1min":6.0579999999999998,"5min":12.186,"15min":45.387999999999998},"last":0.68100000000000005}]}]},{"osd":4,"up_from":30,"seq":128849019223,"num_pgs":61,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1911924,"kb_used_data":1586412,"kb_used_omap":0,"kb_used_meta":325504,"kb_avail":91833228,"statfs":{"total":95995035648,"available":94037225472,"internally_reserved":0,"allocated":1624485888,"data_stored":1623179722,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":333316096},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:37:46 2023","interfaces":[{"interface":"back","average":{"1min":0.85599999999999998,"5min":1.917,"15min":1.667},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.222},"max":{"1min":3.3759999999999999,"5min":24.114999999999998,"15min":24.114999999999998},"last":1.383},{"interface":"front","average":{"1min":0.96399999999999997,"5min":2.0779999999999998,"15min":1.8100000000000001},"min":{"1min":0.25,"5min":0.246,"15min":0.246},"max":{"1min":3.6709999999999998,"5min":24.135999999999999,"15min":24.135999999999999},"last":1.397}]},{"osd":1,"last update":"Sat Sep 23 20:38:02 2023","interfaces":[{"interface":"back","average":{"1min":1.7929999999999999,"5min":1.9059999999999999,"15min":1.877},"min":{"1min":0.28599999999999998,"5min":0.27600000000000002,"15min":0.27300000000000002},"max":{"1min":18.190999999999999,"5min":18.190999999999999,"15min":18.190999999999999},"last":0.372},{"interface":"front","average":{"1min":2.0800000000000001,"5min":1.79,"15min":1.8320000000000001},"min":{"1min":0.29299999999999998,"5min":0.28599999999999998,"15min":0.28199999999999997},"max":{"1min":18.173999999999999,"5min":18.173999999999999,"15min":18.173999999999999},"last":0.80700000000000005}]},{"osd":2,"last update":"Sat Sep 23 20:37:57 2023","interfaces":[{"interface":"back","average":{"1min":2.1110000000000002,"5min":2.0070000000000001,"15min":1.9379999999999999},"min":{"1min":0.374,"5min":0.311,"15min":0.26600000000000001},"max":{"1min":7.75,"5min":10.416,"15min":32.363},"last":0.77800000000000002},{"interface":"front","average":{"1min":2.399,"5min":2.4500000000000002,"15min":2.2549999999999999},"min":{"1min":0.372,"5min":0.30599999999999999,"15min":0.26800000000000002},"max":{"1min":8.2680000000000007,"5min":14.614000000000001,"15min":34.042999999999999},"last":0.875}]},{"osd":3,"last update":"Sat Sep 23 20:38:14 2023","interfaces":[{"interface":"back","average":{"1min":1.607,"5min":1.9370000000000001,"15min":2.0950000000000002},"min":{"1min":0.42899999999999999,"5min":0.34899999999999998,"15min":0.29099999999999998},"max":{"1min":7.3170000000000002,"5min":9.5239999999999991,"15min":32.332999999999998},"last":0.92900000000000005},{"interface":"front","average":{"1min":1.9099999999999999,"5min":2.3039999999999998,"15min":2.2189999999999999},"min":{"1min":0.41399999999999998,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":7.2350000000000003,"5min":14.384,"15min":32.399999999999999},"last":0.84999999999999998}]},{"osd":5,"last update":"Sat Sep 23 20:38:42 2023","interfaces":[{"interface":"back","average":{"1min":0.47599999999999998,"5min":0.65200000000000002,"15min":1.5620000000000001},"min":{"1min":0.29999999999999999,"5min":0.26400000000000001,"15min":0.247},"max":{"1min":1.968,"5min":3.5960000000000001,"15min":82.866},"last":0.58099999999999996},{"interface":"front","average":{"1min":0.64500000000000002,"5min":0.77100000000000002,"15min":1.4970000000000001},"min":{"1min":0.33000000000000002,"5min":0.28999999999999998,"15min":0.27800000000000002},"max":{"1min":4.5590000000000002,"5min":4.5590000000000002,"15min":82.546000000000006},"last":1.0269999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:37:57 2023","interfaces":[{"interface":"back","average":{"1min":0.96899999999999997,"5min":1.1359999999999999,"15min":1.4390000000000001},"min":{"1min":0.33200000000000002,"5min":0.312,"15min":0.27600000000000002},"max":{"1min":4.8730000000000002,"5min":10.178000000000001,"15min":41.371000000000002},"last":0.83599999999999997},{"interface":"front","average":{"1min":1.113,"5min":0.995,"15min":1.327},"min":{"1min":0.374,"5min":0.29899999999999999,"15min":0.27200000000000002},"max":{"1min":7.2850000000000001,"5min":7.2850000000000001,"15min":41.140999999999998},"last":0.51000000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:38:14 2023","interfaces":[{"interface":"back","average":{"1min":0.64800000000000002,"5min":1.204,"15min":1.5309999999999999},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":3.036,"5min":10.548999999999999,"15min":16.427},"last":0.90100000000000002},{"interface":"front","average":{"1min":0.56599999999999995,"5min":0.81599999999999995,"15min":1.321},"min":{"1min":0.33200000000000002,"5min":0.32700000000000001,"15min":0.307},"max":{"1min":1.9350000000000001,"5min":7.2400000000000002,"15min":19.018999999999998},"last":0.54600000000000004}]}]},{"osd":5,"up_from":39,"seq":167503724884,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2983460,"kb_used_data":2661852,"kb_used_omap":0,"kb_used_meta":321600,"kb_avail":90761692,"statfs":{"total":95995035648,"available":92939972608,"internally_reserved":0,"allocated":2725736448,"data_stored":2724442128,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":329318400},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:37:47 2023","interfaces":[{"interface":"back","average":{"1min":1.2130000000000001,"5min":2.2200000000000002,"15min":1.784},"min":{"1min":0.26900000000000002,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":6.7229999999999999,"5min":18.16,"15min":32.475000000000001},"last":0.26600000000000001},{"interface":"front","average":{"1min":1.5069999999999999,"5min":2.1120000000000001,"15min":1.744},"min":{"1min":0.317,"5min":0.26300000000000001,"15min":0.20599999999999999},"max":{"1min":6.0620000000000003,"5min":9.0570000000000004,"15min":32.457000000000001},"last":0.63}]},{"osd":1,"last update":"Sat Sep 23 20:38:05 2023","interfaces":[{"interface":"back","average":{"1min":1.1779999999999999,"5min":2.101,"15min":1.952},"min":{"1min":0.29599999999999999,"5min":0.253,"15min":0.253},"max":{"1min":6.9450000000000003,"5min":36.588999999999999,"15min":36.588999999999999},"last":0.75},{"interface":"front","average":{"1min":0.90700000000000003,"5min":2.2749999999999999,"15min":1.714},"min":{"1min":0.27700000000000002,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":6.6859999999999999,"5min":36.920999999999999,"15min":36.920999999999999},"last":2.2709999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:37:47 2023","interfaces":[{"interface":"back","average":{"1min":1.2549999999999999,"5min":2.3679999999999999,"15min":2.0710000000000002},"min":{"1min":0.32700000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":6.7599999999999998,"5min":37.011000000000003,"15min":37.011000000000003},"last":2.2330000000000001},{"interface":"front","average":{"1min":1.7330000000000001,"5min":2.657,"15min":2.117},"min":{"1min":0.33100000000000002,"5min":0.255,"15min":0.255},"max":{"1min":8.7560000000000002,"5min":36.835000000000001,"15min":36.835000000000001},"last":0.55900000000000005}]},{"osd":3,"last update":"Sat Sep 23 20:38:11 2023","interfaces":[{"interface":"back","average":{"1min":1.357,"5min":2.6829999999999998,"15min":2.1779999999999999},"min":{"1min":0.34699999999999998,"5min":0.314,"15min":0.25700000000000001},"max":{"1min":6.9669999999999996,"5min":36.716000000000001,"15min":36.716000000000001},"last":0.71399999999999997},{"interface":"front","average":{"1min":2.0139999999999998,"5min":2.4940000000000002,"15min":1.9510000000000001},"min":{"1min":0.372,"5min":0.34399999999999997,"15min":0.28799999999999998},"max":{"1min":12.977,"5min":12.977,"15min":12.977},"last":2.319}]},{"osd":4,"last update":"Sat Sep 23 20:38:43 2023","interfaces":[{"interface":"back","average":{"1min":0.61699999999999999,"5min":0.70599999999999996,"15min":1.339},"min":{"1min":0.28699999999999998,"5min":0.255,"15min":0.255},"max":{"1min":3.331,"5min":6.9180000000000001,"15min":28.015999999999998},"last":0.68799999999999994},{"interface":"front","average":{"1min":0.78000000000000003,"5min":0.76300000000000001,"15min":1.1120000000000001},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":3.2999999999999998,"5min":6.8380000000000001,"15min":12.300000000000001},"last":2.343}]},{"osd":6,"last update":"Sat Sep 23 20:37:49 2023","interfaces":[{"interface":"back","average":{"1min":0.91800000000000004,"5min":0.83199999999999996,"15min":1.2829999999999999},"min":{"1min":0.34899999999999998,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":6.9960000000000004,"5min":6.9960000000000004,"15min":12.890000000000001},"last":2.798},{"interface":"front","average":{"1min":0.97299999999999998,"5min":0.84199999999999997,"15min":1.232},"min":{"1min":0.35999999999999999,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":6.7930000000000001,"5min":6.7930000000000001,"15min":16.265999999999998},"last":2.7749999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:38:11 2023","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.66200000000000003,"15min":1.244},"min":{"1min":0.33400000000000002,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.63400000000000001,"5min":4.0380000000000003,"15min":36.621000000000002},"last":2.2959999999999998},{"interface":"front","average":{"1min":0.52800000000000002,"5min":0.89600000000000002,"15min":1.212},"min":{"1min":0.35699999999999998,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":1.2889999999999999,"5min":6.5970000000000004,"15min":37.331000000000003},"last":0.59499999999999997}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4722,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1360,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8206,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7156,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3154,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1867841536,"data_stored":1867841536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1961496576,"data_stored":1961492504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1722814464,"data_stored":1722810392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1845362688,"data_stored":1845362688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1613561856,"data_stored":1613561856,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2720989184,"data_stored":2720989184,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1432813568,"data_stored":1432813568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1409093632,"data_stored":1409089560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:38:52.489 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd pool set unique_pool_3 pgp_num 26 2023-09-23T20:38:52.509 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:38:52.568 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T20:38:52.616 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:52.712 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:38:52.719 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:52.720 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:38:52.820 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:38:52.851 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:52.894 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:38:52.899 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:52.900 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:38:53.000 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:38:53.059 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:53.125 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:38:53.153 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:38:53.185 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:38:53.220 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:38:53.252 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:38:53.283 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:38:53.284 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:38:53.288 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:53.289 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:38:53.326 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:38:53.364 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:38:53.371 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:38:53.371 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:38:53.371 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:38:53.371 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:38:53.371 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:38:53.371 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:38:53.371 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:38:53.372 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:38:53.390 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:38:53.425 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:53.468 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:38:53.472 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:53.473 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:38:53.509 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:38:53.561 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:53 smithi018 ceph-mon[184144]: from='client.46019 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:38:53.561 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:53 smithi018 ceph-mon[184144]: pgmap v1019: 129 pgs: 9 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 4.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 438 op/s; 23715/203457 objects degraded (11.656%); 3348/203457 objects misplaced (1.646%); 15 MiB/s, 239 objects/s recovering 2023-09-23T20:38:53.561 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:53 smithi018 ceph-mon[184144]: 1.a scrub starts 2023-09-23T20:38:53.561 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:53 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1759982733' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:38:53.574 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:38:53.608 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:53.650 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:38:53.653 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:53.653 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:38:53.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:53 smithi149 ceph-mon[170394]: from='client.46019 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:38:53.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:53 smithi149 ceph-mon[170394]: pgmap v1019: 129 pgs: 9 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 4.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 438 op/s; 23715/203457 objects degraded (11.656%); 3348/203457 objects misplaced (1.646%); 15 MiB/s, 239 objects/s recovering 2023-09-23T20:38:53.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:53 smithi149 ceph-mon[170394]: 1.a scrub starts 2023-09-23T20:38:53.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:53 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1759982733' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:38:53.755 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:38:53.788 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:53.836 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:38:53.840 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:53.840 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:38:53.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:53 smithi018 ceph-mon[179647]: from='client.46019 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:38:53.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:53 smithi018 ceph-mon[179647]: pgmap v1019: 129 pgs: 9 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 4.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 438 op/s; 23715/203457 objects degraded (11.656%); 3348/203457 objects misplaced (1.646%); 15 MiB/s, 239 objects/s recovering 2023-09-23T20:38:53.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:53 smithi018 ceph-mon[179647]: 1.a scrub starts 2023-09-23T20:38:53.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:53 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1759982733' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:38:53.941 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:38:53.983 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:54.519 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:38:54.614 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:38:54.626 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:54.627 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:38:54.727 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:38:54.790 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:54.859 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:38:54.866 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:54.866 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:38:54.967 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:38:55.001 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:55.046 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:38:55.051 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:55.051 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:38:55.152 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:38:55.202 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:38:54 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:38:54] "GET /metrics HTTP/1.1" 200 36566 "" "Prometheus/2.43.0" 2023-09-23T20:38:55.217 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:55.564 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:55 smithi018 ceph-mon[184144]: 1.10 scrub starts 2023-09-23T20:38:55.564 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:55 smithi018 ceph-mon[184144]: pgmap v1020: 129 pgs: 7 active+recovery_wait+degraded, 2 active+recovering, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 4.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 446 op/s; 21259/200577 objects degraded (10.599%); 4438/200577 objects misplaced (2.213%); 18 MiB/s, 285 objects/s recovering 2023-09-23T20:38:55.564 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:55 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/926810151' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:38:55.564 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:55 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:38:55.564 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:55 smithi018 ceph-mon[179647]: 1.10 scrub starts 2023-09-23T20:38:55.564 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:55 smithi018 ceph-mon[179647]: pgmap v1020: 129 pgs: 7 active+recovery_wait+degraded, 2 active+recovering, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 4.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 446 op/s; 21259/200577 objects degraded (10.599%); 4438/200577 objects misplaced (2.213%); 18 MiB/s, 285 objects/s recovering 2023-09-23T20:38:55.564 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:55 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/926810151' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:38:55.565 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:55 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:38:55.587 INFO:teuthology.orchestra.run.smithi018.stderr:noscrub is set 2023-09-23T20:38:55.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:55 smithi149 ceph-mon[170394]: 1.10 scrub starts 2023-09-23T20:38:55.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:55 smithi149 ceph-mon[170394]: pgmap v1020: 129 pgs: 7 active+recovery_wait+degraded, 2 active+recovering, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 4.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 446 op/s; 21259/200577 objects degraded (10.599%); 4438/200577 objects misplaced (2.213%); 18 MiB/s, 285 objects/s recovering 2023-09-23T20:38:55.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:55 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/926810151' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:38:55.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:55 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:38:56.562 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:38:56.817 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:56 smithi018 ceph-mon[184144]: 1.a deep-scrub starts 2023-09-23T20:38:56.817 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:56 smithi018 ceph-mon[184144]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:38:56.817 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:56 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:38:56.817 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:56 smithi018 ceph-mon[184144]: osdmap e421: 8 total, 8 up, 8 in 2023-09-23T20:38:56.817 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:56 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 21259/200577 objects degraded (10.599%), 13 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T20:38:56.817 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:56 smithi018 ceph-mon[179647]: 1.a deep-scrub starts 2023-09-23T20:38:56.817 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:56 smithi018 ceph-mon[179647]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:38:56.817 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:56 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:38:56.818 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:56 smithi018 ceph-mon[179647]: osdmap e421: 8 total, 8 up, 8 in 2023-09-23T20:38:56.818 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:56 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 21259/200577 objects degraded (10.599%), 13 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T20:38:56.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:56 smithi149 ceph-mon[170394]: 1.a deep-scrub starts 2023-09-23T20:38:56.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:56 smithi149 ceph-mon[170394]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:38:56.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:56 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:38:56.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:56 smithi149 ceph-mon[170394]: osdmap e421: 8 total, 8 up, 8 in 2023-09-23T20:38:56.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:56 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 21259/200577 objects degraded (10.599%), 13 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T20:38:57.164 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:38:57.173 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:57.174 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:38:57.274 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:38:57.341 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:57.573 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:57 smithi018 ceph-mon[179647]: failed to encode map e421 with expected crc 2023-09-23T20:38:57.573 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:57 smithi018 ceph-mon[179647]: failed to encode map e421 with expected crc 2023-09-23T20:38:57.573 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:57 smithi018 ceph-mon[179647]: pgmap v1022: 129 pgs: 7 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+clean+scrubbing+deep, 2 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 4.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 442 op/s; 21116/198063 objects degraded (10.661%); 4159/198063 objects misplaced (2.100%); 18 MiB/s, 289 objects/s recovering 2023-09-23T20:38:57.573 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:57 smithi018 ceph-mon[184144]: failed to encode map e421 with expected crc 2023-09-23T20:38:57.573 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:57 smithi018 ceph-mon[184144]: failed to encode map e421 with expected crc 2023-09-23T20:38:57.573 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:57 smithi018 ceph-mon[184144]: pgmap v1022: 129 pgs: 7 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+clean+scrubbing+deep, 2 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 4.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 442 op/s; 21116/198063 objects degraded (10.661%); 4159/198063 objects misplaced (2.100%); 18 MiB/s, 289 objects/s recovering 2023-09-23T20:38:57.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:57 smithi149 ceph-mon[170394]: failed to encode map e421 with expected crc 2023-09-23T20:38:57.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:57 smithi149 ceph-mon[170394]: failed to encode map e421 with expected crc 2023-09-23T20:38:57.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:57 smithi149 ceph-mon[170394]: pgmap v1022: 129 pgs: 7 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+clean+scrubbing+deep, 2 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 4.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 442 op/s; 21116/198063 objects degraded (10.661%); 4159/198063 objects misplaced (2.100%); 18 MiB/s, 289 objects/s recovering 2023-09-23T20:38:58.716 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:38:58.720 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:58.721 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:38:58.821 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:38:58.853 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:58.901 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:38:58.905 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:58.905 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:38:59.006 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:38:59.055 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:59.207 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:38:59.212 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:59.212 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:38:59.313 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:38:59.358 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:59.390 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:38:59.427 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:38:59.433 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:59.433 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:38:59.534 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:38:59.577 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:59 smithi018 ceph-mon[184144]: pgmap v1023: 129 pgs: 7 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+clean+scrubbing+deep, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 4.0 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 447 op/s; 20966/194970 objects degraded (10.753%); 4154/194970 objects misplaced (2.131%); 18 MiB/s, 309 objects/s recovering 2023-09-23T20:38:59.578 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:38:59 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:38:59.581 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:59 smithi018 ceph-mon[179647]: pgmap v1023: 129 pgs: 7 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+clean+scrubbing+deep, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 4.0 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 447 op/s; 20966/194970 objects degraded (10.753%); 4154/194970 objects misplaced (2.131%); 18 MiB/s, 309 objects/s recovering 2023-09-23T20:38:59.581 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:38:59 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:38:59.597 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:59.714 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:38:59.720 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:59.721 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:38:59.822 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:38:59.901 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:38:59.955 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:38:59.960 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:38:59.961 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:38:59.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:59 smithi149 ceph-mon[170394]: pgmap v1023: 129 pgs: 7 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+clean+scrubbing+deep, 1 active+recovering, 5 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 4.0 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 447 op/s; 20966/194970 objects degraded (10.753%); 4154/194970 objects misplaced (2.131%); 18 MiB/s, 309 objects/s recovering 2023-09-23T20:38:59.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:38:59 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:39:00.061 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:39:00.088 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:00.130 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:39:00.136 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:00.136 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:39:00.237 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:39:00.247 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2023-09-23T20:39:00.247 INFO:teuthology.orchestra.run.smithi018.stdout: "size": 20, 2023-09-23T20:39:00.247 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 600, 2023-09-23T20:39:00.248 INFO:teuthology.orchestra.run.smithi018.stdout: "ops": [ 2023-09-23T20:39:00.248 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.248 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.45161.0:136070 6.16 6:6af4a005:::benchmark_data_smithi018_270558_object31331:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e258)", 2023-09-23T20:39:00.248 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:29:27.396634+0000", 2023-09-23T20:39:00.248 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 572.764433844, 2023-09-23T20:39:00.248 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.1461166540000001, 2023-09-23T20:39:00.248 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:39:00.248 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:39:00.248 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:39:00.248 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.45161", 2023-09-23T20:39:00.248 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3795614987", 2023-09-23T20:39:00.249 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 136070 2023-09-23T20:39:00.249 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.249 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:39:00.249 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.249 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:39:00.249 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.396634+0000", 2023-09-23T20:39:00.249 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:39:00.249 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.249 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.249 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:39:00.249 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.396634+0000", 2023-09-23T20:39:00.250 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:39:00.250 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.250 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.250 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:39:00.250 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.396633+0000", 2023-09-23T20:39:00.250 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.999999 2023-09-23T20:39:00.250 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.250 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.250 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:39:00.250 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.396639+0000", 2023-09-23T20:39:00.250 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.6099999999999997e-06 2023-09-23T20:39:00.251 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.251 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.251 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:39:00.251 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.396654+0000", 2023-09-23T20:39:00.251 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.5432999999999999e-05 2023-09-23T20:39:00.251 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.251 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.251 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:39:00.251 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.396661+0000", 2023-09-23T20:39:00.251 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.4799999999999998e-06 2023-09-23T20:39:00.251 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.251 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.252 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:39:00.252 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.396728+0000", 2023-09-23T20:39:00.252 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.6884999999999994e-05 2023-09-23T20:39:00.252 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.252 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.252 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:39:00.252 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.396741+0000", 2023-09-23T20:39:00.252 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3933e-05 2023-09-23T20:39:00.252 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.252 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.252 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:39:00.253 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:28.349683+0000", 2023-09-23T20:39:00.253 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.95294156299999999 2023-09-23T20:39:00.253 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.253 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.253 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:39:00.253 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:28.349696+0000", 2023-09-23T20:39:00.253 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3191e-05 2023-09-23T20:39:00.253 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.253 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.253 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:39:00.253 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.535166+0000", 2023-09-23T20:39:00.253 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1854697679999999 2023-09-23T20:39:00.254 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.254 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.254 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:39:00.254 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.535254+0000", 2023-09-23T20:39:00.254 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.7965000000000002e-05 2023-09-23T20:39:00.254 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.254 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.254 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 4,6,7", 2023-09-23T20:39:00.254 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.535550+0000", 2023-09-23T20:39:00.254 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00029616100000000003 2023-09-23T20:39:00.254 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.255 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.255 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:39:00.255 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.538940+0000", 2023-09-23T20:39:00.255 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.003389651 2023-09-23T20:39:00.255 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.255 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.255 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:39:00.255 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.538996+0000", 2023-09-23T20:39:00.255 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.6674e-05 2023-09-23T20:39:00.255 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.255 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.255 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:39:00.256 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.540647+0000", 2023-09-23T20:39:00.256 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0016507220000000001 2023-09-23T20:39:00.256 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.256 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.256 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:39:00.256 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.542633+0000", 2023-09-23T20:39:00.256 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0019862809999999999 2023-09-23T20:39:00.256 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.256 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.256 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:39:00.256 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.542666+0000", 2023-09-23T20:39:00.256 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.2740999999999997e-05 2023-09-23T20:39:00.257 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.257 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.257 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:39:00.257 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.542751+0000", 2023-09-23T20:39:00.257 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.4754999999999997e-05 2023-09-23T20:39:00.257 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:39:00.257 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:39:00.257 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:39:00.257 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.257 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.257 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.45161.0:136077 6.16 6:6b87af5f:::benchmark_data_smithi018_270558_object31338:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e258)", 2023-09-23T20:39:00.257 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:29:27.409208+0000", 2023-09-23T20:39:00.258 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 572.75185964800005, 2023-09-23T20:39:00.258 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.1337409109999999, 2023-09-23T20:39:00.258 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:39:00.258 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:39:00.258 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:39:00.258 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.45161", 2023-09-23T20:39:00.258 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3795614987", 2023-09-23T20:39:00.258 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 136077 2023-09-23T20:39:00.258 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.258 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:39:00.258 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.258 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:39:00.259 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.409208+0000", 2023-09-23T20:39:00.259 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:39:00.259 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.259 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.259 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:39:00.259 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.409208+0000", 2023-09-23T20:39:00.259 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:39:00.259 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.259 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.259 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:39:00.259 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.409207+0000", 2023-09-23T20:39:00.259 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:39:00.260 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.260 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.260 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:39:00.260 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.409214+0000", 2023-09-23T20:39:00.260 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.3930000000000002e-06 2023-09-23T20:39:00.260 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.260 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.260 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:39:00.260 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.409229+0000", 2023-09-23T20:39:00.260 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.4535000000000001e-05 2023-09-23T20:39:00.260 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.261 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.261 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:39:00.261 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.409238+0000", 2023-09-23T20:39:00.261 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.5899999999999997e-06 2023-09-23T20:39:00.261 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.261 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.261 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:39:00.261 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.409307+0000", 2023-09-23T20:39:00.261 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.9023000000000006e-05 2023-09-23T20:39:00.261 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.261 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.261 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:39:00.262 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:27.409318+0000", 2023-09-23T20:39:00.262 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0387999999999999e-05 2023-09-23T20:39:00.262 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.262 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.262 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:39:00.262 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:28.349722+0000", 2023-09-23T20:39:00.262 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.94040487299999997 2023-09-23T20:39:00.262 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.262 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.262 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:39:00.262 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:28.349733+0000", 2023-09-23T20:39:00.262 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0152e-05 2023-09-23T20:39:00.263 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.263 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.263 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:39:00.263 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.536807+0000", 2023-09-23T20:39:00.263 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.187074006 2023-09-23T20:39:00.263 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.263 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.263 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:39:00.263 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.536968+0000", 2023-09-23T20:39:00.263 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00016101399999999999 2023-09-23T20:39:00.263 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.263 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.264 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 4,6,7", 2023-09-23T20:39:00.264 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.537271+0000", 2023-09-23T20:39:00.264 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000302887 2023-09-23T20:39:00.264 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.264 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.264 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:39:00.264 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.539134+0000", 2023-09-23T20:39:00.264 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.001863611 2023-09-23T20:39:00.264 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.264 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.264 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:39:00.265 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.542232+0000", 2023-09-23T20:39:00.265 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.003097533 2023-09-23T20:39:00.265 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.265 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.265 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:39:00.265 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.542819+0000", 2023-09-23T20:39:00.265 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00058755999999999999 2023-09-23T20:39:00.265 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.265 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.265 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:39:00.265 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.542846+0000", 2023-09-23T20:39:00.265 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.6855e-05 2023-09-23T20:39:00.266 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.266 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.266 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:39:00.266 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.542874+0000", 2023-09-23T20:39:00.266 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.7798e-05 2023-09-23T20:39:00.266 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.266 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.266 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:39:00.266 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:29:29.542949+0000", 2023-09-23T20:39:00.266 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.5531999999999994e-05 2023-09-23T20:39:00.266 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:39:00.267 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:39:00.267 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:39:00.267 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.267 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.267 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:57605 7.3 7:c29712e0:::benchmark_data_smithi018_308601_object57604:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e311)", 2023-09-23T20:39:00.267 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:32:32.219481+0000", 2023-09-23T20:39:00.267 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 387.94158706600001, 2023-09-23T20:39:00.267 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.3860234459999998, 2023-09-23T20:39:00.267 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:39:00.267 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:39:00.267 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:39:00.267 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:39:00.268 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:39:00.268 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 57605 2023-09-23T20:39:00.268 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.268 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:39:00.268 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.268 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:39:00.268 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.219481+0000", 2023-09-23T20:39:00.268 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:39:00.268 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.268 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.268 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:39:00.268 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.219481+0000", 2023-09-23T20:39:00.269 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:39:00.269 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.269 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.269 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:39:00.269 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.219480+0000", 2023-09-23T20:39:00.269 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:39:00.269 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.269 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.269 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:39:00.269 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.219513+0000", 2023-09-23T20:39:00.269 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.3324999999999997e-05 2023-09-23T20:39:00.269 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.269 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.270 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:39:00.270 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.219526+0000", 2023-09-23T20:39:00.270 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3145999999999999e-05 2023-09-23T20:39:00.270 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.270 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.270 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:39:00.270 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.219534+0000", 2023-09-23T20:39:00.270 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.729e-06 2023-09-23T20:39:00.270 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.270 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.270 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:39:00.271 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.231964+0000", 2023-09-23T20:39:00.271 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.012430274999999999 2023-09-23T20:39:00.271 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.271 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.271 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:39:00.271 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.231974+0000", 2023-09-23T20:39:00.271 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.4939999999999992e-06 2023-09-23T20:39:00.271 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.271 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.271 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:39:00.271 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.215331+0000", 2023-09-23T20:39:00.271 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.983357276 2023-09-23T20:39:00.272 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.272 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.272 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:39:00.272 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.215344+0000", 2023-09-23T20:39:00.272 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3061000000000001e-05 2023-09-23T20:39:00.272 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.272 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.272 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:39:00.272 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.573779+0000", 2023-09-23T20:39:00.272 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.358434581 2023-09-23T20:39:00.272 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.272 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.273 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:39:00.273 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.573953+0000", 2023-09-23T20:39:00.273 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00017463999999999999 2023-09-23T20:39:00.273 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.273 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.273 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 2,4", 2023-09-23T20:39:00.273 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.574063+0000", 2023-09-23T20:39:00.273 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000110059 2023-09-23T20:39:00.273 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.273 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.273 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:39:00.274 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.578839+0000", 2023-09-23T20:39:00.274 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0047753709999999996 2023-09-23T20:39:00.274 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.274 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.274 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:39:00.274 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.578852+0000", 2023-09-23T20:39:00.274 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3360000000000001e-05 2023-09-23T20:39:00.274 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.274 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.274 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:39:00.274 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.602935+0000", 2023-09-23T20:39:00.275 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.024082883999999999 2023-09-23T20:39:00.275 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.275 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.275 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:39:00.275 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.602997+0000", 2023-09-23T20:39:00.275 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.2113000000000007e-05 2023-09-23T20:39:00.275 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.275 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.275 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:39:00.275 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.605505+0000", 2023-09-23T20:39:00.275 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.002507493 2023-09-23T20:39:00.275 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:39:00.276 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:39:00.276 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:39:00.276 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.276 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.276 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:57610 7.3 7:cbaa52d6:::benchmark_data_smithi018_308601_object57609:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e311)", 2023-09-23T20:39:00.276 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:32:32.219584+0000", 2023-09-23T20:39:00.276 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 387.94148443099999, 2023-09-23T20:39:00.276 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.38833787, 2023-09-23T20:39:00.276 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:39:00.276 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:39:00.276 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:39:00.276 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:39:00.277 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:39:00.277 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 57610 2023-09-23T20:39:00.277 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.277 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:39:00.277 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.277 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:39:00.277 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.219584+0000", 2023-09-23T20:39:00.277 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:39:00.277 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.277 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.277 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:39:00.278 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.219584+0000", 2023-09-23T20:39:00.278 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:39:00.278 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.278 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.278 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:39:00.278 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.219583+0000", 2023-09-23T20:39:00.278 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:39:00.278 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.278 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.278 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:39:00.278 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.219607+0000", 2023-09-23T20:39:00.278 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.4170000000000001e-05 2023-09-23T20:39:00.279 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.279 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.279 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:39:00.279 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.219614+0000", 2023-09-23T20:39:00.279 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.4790000000000001e-06 2023-09-23T20:39:00.279 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.279 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.279 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:39:00.279 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.219619+0000", 2023-09-23T20:39:00.279 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.079e-06 2023-09-23T20:39:00.279 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.279 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.280 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:39:00.280 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.231994+0000", 2023-09-23T20:39:00.280 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.012374484999999999 2023-09-23T20:39:00.280 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.280 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.280 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:39:00.280 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.232000+0000", 2023-09-23T20:39:00.280 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.4790000000000001e-06 2023-09-23T20:39:00.280 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.280 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.280 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:39:00.281 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.215369+0000", 2023-09-23T20:39:00.281 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.98336916699999999 2023-09-23T20:39:00.281 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.281 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.281 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:39:00.281 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.215378+0000", 2023-09-23T20:39:00.281 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.6479999999999992e-06 2023-09-23T20:39:00.281 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.281 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.281 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:39:00.281 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.577198+0000", 2023-09-23T20:39:00.281 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.36182066 2023-09-23T20:39:00.282 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.282 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.282 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:39:00.282 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.577312+0000", 2023-09-23T20:39:00.282 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000113257 2023-09-23T20:39:00.282 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.282 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.282 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 2,4", 2023-09-23T20:39:00.282 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.577424+0000", 2023-09-23T20:39:00.282 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00011234999999999999 2023-09-23T20:39:00.282 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.282 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.283 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:39:00.283 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.580718+0000", 2023-09-23T20:39:00.283 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0032940090000000001 2023-09-23T20:39:00.283 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.283 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.283 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:39:00.283 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.582068+0000", 2023-09-23T20:39:00.283 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0013497990000000001 2023-09-23T20:39:00.283 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.283 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.283 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:39:00.284 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.605587+0000", 2023-09-23T20:39:00.284 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.023518779 2023-09-23T20:39:00.284 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.284 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.284 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:39:00.284 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.605666+0000", 2023-09-23T20:39:00.284 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.9777999999999994e-05 2023-09-23T20:39:00.284 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.284 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.284 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:39:00.284 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.607922+0000", 2023-09-23T20:39:00.284 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00225515 2023-09-23T20:39:00.285 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:39:00.285 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:39:00.285 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:39:00.285 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.285 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.285 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:57645 7.3 7:cb6be1ad:::benchmark_data_smithi018_308601_object57644:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e312)", 2023-09-23T20:39:00.285 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:32:32.275903+0000", 2023-09-23T20:39:00.285 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 387.88516507700001, 2023-09-23T20:39:00.285 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.3348626370000001, 2023-09-23T20:39:00.285 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:39:00.285 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:39:00.285 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:39:00.286 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:39:00.286 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:39:00.286 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 57645 2023-09-23T20:39:00.286 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.286 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:39:00.286 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.286 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:39:00.286 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.275903+0000", 2023-09-23T20:39:00.286 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:39:00.286 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.286 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.287 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:39:00.287 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.275903+0000", 2023-09-23T20:39:00.287 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:39:00.287 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.287 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.287 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:39:00.287 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.275902+0000", 2023-09-23T20:39:00.287 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:39:00.287 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.287 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.287 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:39:00.288 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.275926+0000", 2023-09-23T20:39:00.288 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.4209000000000001e-05 2023-09-23T20:39:00.288 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.288 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.288 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:39:00.288 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.275938+0000", 2023-09-23T20:39:00.288 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1992e-05 2023-09-23T20:39:00.288 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.288 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.288 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:39:00.288 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.275946+0000", 2023-09-23T20:39:00.288 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.8839999999999994e-06 2023-09-23T20:39:00.289 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.289 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.289 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:39:00.289 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.276851+0000", 2023-09-23T20:39:00.289 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00090492800000000005 2023-09-23T20:39:00.289 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.289 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.289 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:39:00.289 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.276861+0000", 2023-09-23T20:39:00.289 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0208e-05 2023-09-23T20:39:00.289 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.289 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.290 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:39:00.290 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.215401+0000", 2023-09-23T20:39:00.290 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.93854029800000005 2023-09-23T20:39:00.290 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.290 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.290 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:39:00.290 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.215410+0000", 2023-09-23T20:39:00.290 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.3510000000000002e-06 2023-09-23T20:39:00.290 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.290 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.290 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:39:00.290 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.578900+0000", 2023-09-23T20:39:00.291 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.363490173 2023-09-23T20:39:00.291 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.291 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.291 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:39:00.291 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.579070+0000", 2023-09-23T20:39:00.291 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00017048899999999999 2023-09-23T20:39:00.291 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.291 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.291 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 2,4", 2023-09-23T20:39:00.291 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.579260+0000", 2023-09-23T20:39:00.291 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000189667 2023-09-23T20:39:00.293 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.293 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.293 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:39:00.293 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.583356+0000", 2023-09-23T20:39:00.293 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0040960520000000002 2023-09-23T20:39:00.293 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.293 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.293 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:39:00.293 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.583406+0000", 2023-09-23T20:39:00.293 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.9449e-05 2023-09-23T20:39:00.293 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.294 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.294 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:39:00.294 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.607983+0000", 2023-09-23T20:39:00.294 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.024577576 2023-09-23T20:39:00.294 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.294 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.294 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:39:00.294 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.608038+0000", 2023-09-23T20:39:00.294 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.4787000000000002e-05 2023-09-23T20:39:00.294 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.294 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.294 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:39:00.295 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.610766+0000", 2023-09-23T20:39:00.295 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0027278319999999999 2023-09-23T20:39:00.295 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:39:00.295 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:39:00.295 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:39:00.295 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.295 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.295 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:57662 7.3 7:ce017a4a:::benchmark_data_smithi018_308601_object57661:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e312)", 2023-09-23T20:39:00.295 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:32:32.295077+0000", 2023-09-23T20:39:00.295 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 387.86599100000001, 2023-09-23T20:39:00.295 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.3221079059999998, 2023-09-23T20:39:00.296 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:39:00.296 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:39:00.296 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:39:00.296 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:39:00.296 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:39:00.296 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 57662 2023-09-23T20:39:00.296 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.296 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:39:00.296 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.296 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:39:00.296 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.295077+0000", 2023-09-23T20:39:00.296 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:39:00.296 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.297 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.297 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:39:00.297 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.295077+0000", 2023-09-23T20:39:00.297 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:39:00.297 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.297 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.297 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:39:00.297 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.295075+0000", 2023-09-23T20:39:00.297 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:39:00.297 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.297 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.297 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:39:00.298 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.295104+0000", 2023-09-23T20:39:00.298 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.8374999999999999e-05 2023-09-23T20:39:00.298 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.298 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.298 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:39:00.298 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.295117+0000", 2023-09-23T20:39:00.298 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2943e-05 2023-09-23T20:39:00.298 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.298 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.298 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:39:00.298 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.295123+0000", 2023-09-23T20:39:00.298 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.5989999999999998e-06 2023-09-23T20:39:00.299 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.299 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.299 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:39:00.299 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.295189+0000", 2023-09-23T20:39:00.299 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.5988000000000006e-05 2023-09-23T20:39:00.299 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.299 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.299 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:39:00.299 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.295205+0000", 2023-09-23T20:39:00.299 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.5434000000000002e-05 2023-09-23T20:39:00.299 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.300 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.300 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:39:00.300 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.215432+0000", 2023-09-23T20:39:00.300 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.92022715499999996 2023-09-23T20:39:00.300 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.300 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.300 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:39:00.300 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.215440+0000", 2023-09-23T20:39:00.300 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.2519999999999999e-06 2023-09-23T20:39:00.300 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.300 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.301 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:39:00.301 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.580753+0000", 2023-09-23T20:39:00.301 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.365312673 2023-09-23T20:39:00.301 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.301 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.301 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:39:00.301 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.580896+0000", 2023-09-23T20:39:00.301 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00014352000000000001 2023-09-23T20:39:00.301 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.301 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.302 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 2,4", 2023-09-23T20:39:00.302 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.581023+0000", 2023-09-23T20:39:00.302 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000127112 2023-09-23T20:39:00.302 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.302 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.302 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:39:00.302 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.583459+0000", 2023-09-23T20:39:00.302 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0024356450000000002 2023-09-23T20:39:00.302 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.302 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.303 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:39:00.303 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.583468+0000", 2023-09-23T20:39:00.303 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.4840000000000004e-06 2023-09-23T20:39:00.303 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.303 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.303 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:39:00.303 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.613992+0000", 2023-09-23T20:39:00.303 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.030524193000000002 2023-09-23T20:39:00.303 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.303 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.303 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:39:00.303 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.614070+0000", 2023-09-23T20:39:00.304 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.8054999999999997e-05 2023-09-23T20:39:00.304 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.304 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.304 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:39:00.304 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.617185+0000", 2023-09-23T20:39:00.304 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0031151690000000001 2023-09-23T20:39:00.304 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:39:00.304 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:39:00.304 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:39:00.304 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.305 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.305 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:57667 7.3 7:cbc699af:::benchmark_data_smithi018_308601_object57666:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e312)", 2023-09-23T20:39:00.305 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:32:32.300970+0000", 2023-09-23T20:39:00.305 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 387.86009804700001, 2023-09-23T20:39:00.305 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.3193818780000002, 2023-09-23T20:39:00.305 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:39:00.305 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:39:00.305 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:39:00.305 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:39:00.305 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:39:00.305 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 57667 2023-09-23T20:39:00.306 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.306 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:39:00.306 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.306 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:39:00.306 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.300970+0000", 2023-09-23T20:39:00.306 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:39:00.306 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.306 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.306 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:39:00.306 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.300970+0000", 2023-09-23T20:39:00.306 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:39:00.307 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.307 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.307 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:39:00.307 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.300969+0000", 2023-09-23T20:39:00.307 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.999999 2023-09-23T20:39:00.307 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.307 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.307 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:39:00.307 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.300992+0000", 2023-09-23T20:39:00.307 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.2997e-05 2023-09-23T20:39:00.307 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.307 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.308 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:39:00.308 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.301006+0000", 2023-09-23T20:39:00.308 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3509000000000001e-05 2023-09-23T20:39:00.308 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.308 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.308 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:39:00.308 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.301014+0000", 2023-09-23T20:39:00.308 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.9189999999999999e-06 2023-09-23T20:39:00.308 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.308 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.308 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:39:00.308 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.301060+0000", 2023-09-23T20:39:00.309 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.6597999999999997e-05 2023-09-23T20:39:00.309 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.309 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.309 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:39:00.309 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.301067+0000", 2023-09-23T20:39:00.309 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.6039999999999998e-06 2023-09-23T20:39:00.309 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.309 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.309 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:39:00.309 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.215463+0000", 2023-09-23T20:39:00.309 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.914396019 2023-09-23T20:39:00.310 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.310 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.310 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:39:00.310 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.215471+0000", 2023-09-23T20:39:00.310 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.1880000000000002e-06 2023-09-23T20:39:00.310 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.310 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.310 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:39:00.310 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.582113+0000", 2023-09-23T20:39:00.310 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.366641642 2023-09-23T20:39:00.310 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.310 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.311 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:39:00.311 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.582227+0000", 2023-09-23T20:39:00.311 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000114545 2023-09-23T20:39:00.311 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.311 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.311 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 2,4", 2023-09-23T20:39:00.311 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.582342+0000", 2023-09-23T20:39:00.311 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000114294 2023-09-23T20:39:00.311 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.311 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.311 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:39:00.311 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.584646+0000", 2023-09-23T20:39:00.311 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.002304038 2023-09-23T20:39:00.312 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.312 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.312 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:39:00.312 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.584666+0000", 2023-09-23T20:39:00.312 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.0596000000000001e-05 2023-09-23T20:39:00.312 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.312 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.312 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:39:00.312 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.617277+0000", 2023-09-23T20:39:00.312 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.032610293999999998 2023-09-23T20:39:00.312 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.312 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.312 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:39:00.313 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.617329+0000", 2023-09-23T20:39:00.313 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.2527e-05 2023-09-23T20:39:00.313 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.313 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.313 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:39:00.313 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.620352+0000", 2023-09-23T20:39:00.313 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0030229179999999999 2023-09-23T20:39:00.313 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:39:00.313 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:39:00.313 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:39:00.313 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.313 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.314 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:57675 7.3 7:c7bba963:::benchmark_data_smithi018_308601_object57674:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e312)", 2023-09-23T20:39:00.314 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:32:32.327987+0000", 2023-09-23T20:39:00.314 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 387.83308152299998, 2023-09-23T20:39:00.314 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.2956588660000001, 2023-09-23T20:39:00.314 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:39:00.314 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:39:00.314 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:39:00.314 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:39:00.314 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:39:00.314 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 57675 2023-09-23T20:39:00.314 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.315 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:39:00.315 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.315 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:39:00.315 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.327987+0000", 2023-09-23T20:39:00.315 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:39:00.315 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.315 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.315 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:39:00.315 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.327987+0000", 2023-09-23T20:39:00.315 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:39:00.315 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.315 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.315 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:39:00.316 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.327985+0000", 2023-09-23T20:39:00.316 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:39:00.316 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.316 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.316 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:39:00.316 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.328012+0000", 2023-09-23T20:39:00.316 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.6673e-05 2023-09-23T20:39:00.316 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.316 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.316 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:39:00.316 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.328033+0000", 2023-09-23T20:39:00.316 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.1865000000000002e-05 2023-09-23T20:39:00.317 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.317 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.317 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:39:00.317 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.328046+0000", 2023-09-23T20:39:00.317 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2138e-05 2023-09-23T20:39:00.317 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.317 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.317 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:39:00.317 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.328112+0000", 2023-09-23T20:39:00.317 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.6155999999999995e-05 2023-09-23T20:39:00.317 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.317 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.318 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:39:00.318 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.328123+0000", 2023-09-23T20:39:00.318 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1323e-05 2023-09-23T20:39:00.318 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.318 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.318 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:39:00.318 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.215494+0000", 2023-09-23T20:39:00.318 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.88737101299999999 2023-09-23T20:39:00.318 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.318 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.318 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:39:00.318 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.215502+0000", 2023-09-23T20:39:00.319 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.3140000000000004e-06 2023-09-23T20:39:00.319 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.319 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.319 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:39:00.319 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.583495+0000", 2023-09-23T20:39:00.319 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.367992519 2023-09-23T20:39:00.319 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.319 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.319 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:39:00.319 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.583598+0000", 2023-09-23T20:39:00.319 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00010268399999999999 2023-09-23T20:39:00.319 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.320 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.320 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 2,4", 2023-09-23T20:39:00.320 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.583704+0000", 2023-09-23T20:39:00.320 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000106096 2023-09-23T20:39:00.320 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.320 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.320 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:39:00.320 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.584776+0000", 2023-09-23T20:39:00.320 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00107215 2023-09-23T20:39:00.320 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.320 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.320 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:39:00.320 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.585126+0000", 2023-09-23T20:39:00.321 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00035052599999999998 2023-09-23T20:39:00.321 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.321 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.321 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:39:00.321 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.620444+0000", 2023-09-23T20:39:00.321 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.035318081000000001 2023-09-23T20:39:00.321 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.321 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.321 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:39:00.321 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.620519+0000", 2023-09-23T20:39:00.321 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.4976999999999997e-05 2023-09-23T20:39:00.322 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.322 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.322 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:39:00.322 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.623645+0000", 2023-09-23T20:39:00.322 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.003126017 2023-09-23T20:39:00.322 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:39:00.322 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:39:00.322 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:39:00.322 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.322 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.322 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:69129 7.3 7:cf914298:::benchmark_data_smithi018_308601_object69128:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e318)", 2023-09-23T20:39:00.323 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:32:49.475881+0000", 2023-09-23T20:39:00.323 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 370.68518713499998, 2023-09-23T20:39:00.323 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.048825159, 2023-09-23T20:39:00.323 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:39:00.323 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:39:00.323 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:39:00.323 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:39:00.323 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:39:00.323 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 69129 2023-09-23T20:39:00.323 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.323 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:39:00.323 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.324 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:39:00.324 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.475881+0000", 2023-09-23T20:39:00.324 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:39:00.324 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.324 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.324 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:39:00.324 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.475881+0000", 2023-09-23T20:39:00.324 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:39:00.324 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.324 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.324 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:39:00.325 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.475881+0000", 2023-09-23T20:39:00.325 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:39:00.325 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.325 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.325 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:39:00.325 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.475925+0000", 2023-09-23T20:39:00.325 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.4274000000000003e-05 2023-09-23T20:39:00.325 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.325 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.325 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:39:00.325 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.475932+0000", 2023-09-23T20:39:00.325 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.2300000000000002e-06 2023-09-23T20:39:00.326 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.326 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.326 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:39:00.326 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.475937+0000", 2023-09-23T20:39:00.326 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.4600000000000002e-06 2023-09-23T20:39:00.326 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.326 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.326 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:39:00.326 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.480114+0000", 2023-09-23T20:39:00.326 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0041766099999999999 2023-09-23T20:39:00.326 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.327 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.327 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:39:00.327 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.480293+0000", 2023-09-23T20:39:00.327 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00017894300000000001 2023-09-23T20:39:00.327 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.327 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.327 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 2,4", 2023-09-23T20:39:00.327 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.480478+0000", 2023-09-23T20:39:00.327 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00018538100000000001 2023-09-23T20:39:00.327 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.327 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.327 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:39:00.327 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.520416+0000", 2023-09-23T20:39:00.327 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.039937451999999998 2023-09-23T20:39:00.328 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.328 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.328 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:39:00.328 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.523198+0000", 2023-09-23T20:39:00.328 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0027818999999999999 2023-09-23T20:39:00.328 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.328 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.328 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:39:00.328 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.528277+0000", 2023-09-23T20:39:00.328 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0050791120000000002 2023-09-23T20:39:00.328 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.329 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.329 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:39:00.329 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.528303+0000", 2023-09-23T20:39:00.329 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.5732e-05 2023-09-23T20:39:00.329 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.329 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.329 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:39:00.329 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:50.524690+0000", 2023-09-23T20:39:00.329 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.99638690299999999 2023-09-23T20:39:00.329 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.329 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.330 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:39:00.330 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:50.524706+0000", 2023-09-23T20:39:00.330 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.6634e-05 2023-09-23T20:39:00.330 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:39:00.330 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:39:00.330 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:39:00.330 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.330 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.330 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:69152 7.3 7.f242f463 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e318)", 2023-09-23T20:39:00.330 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:32:49.520371+0000", 2023-09-23T20:39:00.330 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 370.64069744800003, 2023-09-23T20:39:00.330 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0043691610000001, 2023-09-23T20:39:00.331 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:39:00.331 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:39:00.331 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:39:00.331 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:39:00.331 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:39:00.331 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 69152 2023-09-23T20:39:00.331 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.331 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:39:00.331 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.331 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:39:00.331 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.520371+0000", 2023-09-23T20:39:00.331 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:39:00.331 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.332 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.332 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:39:00.332 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.520371+0000", 2023-09-23T20:39:00.332 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:39:00.332 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.332 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.332 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:39:00.332 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.520369+0000", 2023-09-23T20:39:00.332 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:39:00.332 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.332 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.333 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:39:00.333 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.520395+0000", 2023-09-23T20:39:00.333 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.5843999999999999e-05 2023-09-23T20:39:00.333 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.333 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.333 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:39:00.333 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.520408+0000", 2023-09-23T20:39:00.333 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3189e-05 2023-09-23T20:39:00.333 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.333 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.333 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:39:00.333 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.520417+0000", 2023-09-23T20:39:00.334 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.6899999999999998e-06 2023-09-23T20:39:00.334 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.334 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.334 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:39:00.334 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.529310+0000", 2023-09-23T20:39:00.334 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0088929609999999996 2023-09-23T20:39:00.334 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.334 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.334 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:39:00.334 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.529330+0000", 2023-09-23T20:39:00.334 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.9673e-05 2023-09-23T20:39:00.334 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.335 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.335 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:39:00.335 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:50.524730+0000", 2023-09-23T20:39:00.335 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.99540061999999996 2023-09-23T20:39:00.335 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.335 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.335 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:39:00.335 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:50.524740+0000", 2023-09-23T20:39:00.335 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.6139999999999998e-06 2023-09-23T20:39:00.335 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:39:00.335 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:39:00.336 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:39:00.336 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.336 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.336 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:69159 7.3 7.de3ad3c3 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e318)", 2023-09-23T20:39:00.336 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:32:49.534755+0000", 2023-09-23T20:39:00.336 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 370.62631271700002, 2023-09-23T20:39:00.336 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.99001591200000005, 2023-09-23T20:39:00.336 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:39:00.336 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:39:00.336 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:39:00.336 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:39:00.336 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:39:00.337 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 69159 2023-09-23T20:39:00.337 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.337 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:39:00.337 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.337 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:39:00.337 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.534755+0000", 2023-09-23T20:39:00.337 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:39:00.337 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.337 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.337 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:39:00.337 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.534755+0000", 2023-09-23T20:39:00.337 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:39:00.338 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.338 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.338 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:39:00.338 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.534754+0000", 2023-09-23T20:39:00.338 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:39:00.338 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.338 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.338 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:39:00.338 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.534782+0000", 2023-09-23T20:39:00.338 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.8348999999999999e-05 2023-09-23T20:39:00.338 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.338 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.339 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:39:00.339 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.534808+0000", 2023-09-23T20:39:00.339 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.5941000000000001e-05 2023-09-23T20:39:00.339 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.339 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.339 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:39:00.339 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.534818+0000", 2023-09-23T20:39:00.339 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0132999999999999e-05 2023-09-23T20:39:00.339 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.339 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.339 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:39:00.340 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.535076+0000", 2023-09-23T20:39:00.340 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00025796400000000001 2023-09-23T20:39:00.340 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.340 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.340 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:39:00.340 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.535090+0000", 2023-09-23T20:39:00.340 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3505e-05 2023-09-23T20:39:00.340 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.340 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.340 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:39:00.340 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:50.524762+0000", 2023-09-23T20:39:00.340 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.98967230399999995 2023-09-23T20:39:00.341 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.341 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.341 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:39:00.341 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:50.524771+0000", 2023-09-23T20:39:00.341 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.5070000000000006e-06 2023-09-23T20:39:00.341 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:39:00.341 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:39:00.341 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:39:00.341 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.341 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.341 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35994.0:105785 8.0 8.a76e08d0 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e409)", 2023-09-23T20:39:00.341 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:38:26.401578+0000", 2023-09-23T20:39:00.342 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 33.759490602, 2023-09-23T20:39:00.342 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.31127717599999999, 2023-09-23T20:39:00.342 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:39:00.342 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:39:00.342 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:39:00.342 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35994", 2023-09-23T20:39:00.342 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:39:00.342 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 105785 2023-09-23T20:39:00.342 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.342 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:39:00.342 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.343 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:39:00.343 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.401578+0000", 2023-09-23T20:39:00.343 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:39:00.343 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.343 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.343 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:39:00.343 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.401578+0000", 2023-09-23T20:39:00.343 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:39:00.343 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.343 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.343 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:39:00.344 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.401576+0000", 2023-09-23T20:39:00.344 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.999999 2023-09-23T20:39:00.344 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.344 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.344 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:39:00.344 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.401582+0000", 2023-09-23T20:39:00.344 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.7899999999999996e-06 2023-09-23T20:39:00.344 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.344 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.344 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:39:00.344 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.401595+0000", 2023-09-23T20:39:00.344 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2649e-05 2023-09-23T20:39:00.345 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.345 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.345 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:39:00.345 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.401602+0000", 2023-09-23T20:39:00.345 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.81e-06 2023-09-23T20:39:00.345 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.345 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.345 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:39:00.345 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.712843+0000", 2023-09-23T20:39:00.345 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.31124165199999998 2023-09-23T20:39:00.345 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.346 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.346 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:39:00.346 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.712855+0000", 2023-09-23T20:39:00.346 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1389000000000001e-05 2023-09-23T20:39:00.346 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:39:00.346 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:39:00.346 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:39:00.346 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.346 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.346 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35994.0:105785 8.10 8:0b1076e5:::benchmark_data_smithi018_341490_object16462:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e410)", 2023-09-23T20:39:00.346 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:38:26.443871+0000", 2023-09-23T20:39:00.347 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 33.717196852999997, 2023-09-23T20:39:00.347 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1767037899999999, 2023-09-23T20:39:00.347 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:39:00.347 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:39:00.347 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:39:00.347 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35994", 2023-09-23T20:39:00.347 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:39:00.347 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 105785 2023-09-23T20:39:00.347 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.347 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:39:00.347 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.347 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:39:00.348 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.443871+0000", 2023-09-23T20:39:00.348 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:39:00.348 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.348 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.348 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:39:00.348 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.443871+0000", 2023-09-23T20:39:00.348 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:39:00.348 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.348 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.348 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:39:00.348 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.443871+0000", 2023-09-23T20:39:00.349 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:39:00.349 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.349 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.349 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:39:00.349 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.443875+0000", 2023-09-23T20:39:00.349 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.9509999999999999e-06 2023-09-23T20:39:00.349 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.349 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.349 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:39:00.349 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.443882+0000", 2023-09-23T20:39:00.349 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.8390000000000007e-06 2023-09-23T20:39:00.349 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.350 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.350 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:39:00.350 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.443889+0000", 2023-09-23T20:39:00.350 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.5230000000000001e-06 2023-09-23T20:39:00.350 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.350 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.350 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:39:00.350 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.713126+0000", 2023-09-23T20:39:00.350 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.269236743 2023-09-23T20:39:00.350 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.350 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.351 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:39:00.351 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.713135+0000", 2023-09-23T20:39:00.351 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.0790000000000007e-06 2023-09-23T20:39:00.351 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.351 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.351 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:39:00.351 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.610121+0000", 2023-09-23T20:39:00.351 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.89698586199999997 2023-09-23T20:39:00.351 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.351 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.351 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:39:00.351 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.610236+0000", 2023-09-23T20:39:00.352 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000114833 2023-09-23T20:39:00.352 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.352 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.352 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,4", 2023-09-23T20:39:00.352 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.610355+0000", 2023-09-23T20:39:00.352 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00011921099999999999 2023-09-23T20:39:00.352 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.352 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.352 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:39:00.352 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.615886+0000", 2023-09-23T20:39:00.352 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0055312039999999996 2023-09-23T20:39:00.353 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.353 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.353 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:39:00.353 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.618509+0000", 2023-09-23T20:39:00.353 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.002623434 2023-09-23T20:39:00.353 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.353 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.353 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:39:00.353 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.619136+0000", 2023-09-23T20:39:00.353 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00062698799999999998 2023-09-23T20:39:00.353 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.353 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.354 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:39:00.354 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.619190+0000", 2023-09-23T20:39:00.354 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.3492000000000002e-05 2023-09-23T20:39:00.354 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.354 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.354 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:39:00.354 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.620575+0000", 2023-09-23T20:39:00.354 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.001385231 2023-09-23T20:39:00.354 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:39:00.354 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:39:00.354 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:39:00.355 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.355 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.355 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35994.0:105786 8.10 8:0f8943c0:::benchmark_data_smithi018_341490_object16463:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e410)", 2023-09-23T20:39:00.355 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:38:26.443896+0000", 2023-09-23T20:39:00.355 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 33.717171931000003, 2023-09-23T20:39:00.355 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1781387480000001, 2023-09-23T20:39:00.355 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:39:00.355 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:39:00.355 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:39:00.355 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35994", 2023-09-23T20:39:00.355 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:39:00.356 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 105786 2023-09-23T20:39:00.356 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.356 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:39:00.356 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.356 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:39:00.356 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.443896+0000", 2023-09-23T20:39:00.356 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:39:00.356 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.356 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.356 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:39:00.356 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.443896+0000", 2023-09-23T20:39:00.357 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:39:00.357 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.357 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.357 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:39:00.357 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.443896+0000", 2023-09-23T20:39:00.357 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:39:00.357 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.357 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.357 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:39:00.357 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.443899+0000", 2023-09-23T20:39:00.357 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.337e-06 2023-09-23T20:39:00.357 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.358 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.358 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:39:00.358 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.443905+0000", 2023-09-23T20:39:00.358 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.2260000000000002e-06 2023-09-23T20:39:00.358 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.358 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.358 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:39:00.358 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.443911+0000", 2023-09-23T20:39:00.358 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.135e-06 2023-09-23T20:39:00.358 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.358 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.359 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:39:00.359 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.713155+0000", 2023-09-23T20:39:00.359 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.26924421900000001 2023-09-23T20:39:00.359 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.359 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.359 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:39:00.359 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.713161+0000", 2023-09-23T20:39:00.359 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.7799999999999997e-06 2023-09-23T20:39:00.359 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.359 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.359 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:39:00.359 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.611522+0000", 2023-09-23T20:39:00.359 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.89836117699999996 2023-09-23T20:39:00.360 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.360 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.360 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:39:00.360 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.611650+0000", 2023-09-23T20:39:00.360 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000128677 2023-09-23T20:39:00.360 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.360 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.360 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,4", 2023-09-23T20:39:00.360 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.611781+0000", 2023-09-23T20:39:00.360 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00013079300000000001 2023-09-23T20:39:00.360 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.360 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.360 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:39:00.361 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.618515+0000", 2023-09-23T20:39:00.361 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0067336820000000004 2023-09-23T20:39:00.361 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.361 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.361 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:39:00.361 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.619100+0000", 2023-09-23T20:39:00.361 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00058515599999999998 2023-09-23T20:39:00.361 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.361 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.361 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:39:00.361 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.620654+0000", 2023-09-23T20:39:00.361 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.001553476 2023-09-23T20:39:00.361 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.361 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.362 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:39:00.362 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.620709+0000", 2023-09-23T20:39:00.362 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.5781e-05 2023-09-23T20:39:00.362 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.362 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.362 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:39:00.362 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.622035+0000", 2023-09-23T20:39:00.362 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0013256679999999999 2023-09-23T20:39:00.362 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:39:00.362 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:39:00.362 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:39:00.362 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.363 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.363 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35994.0:105787 8.0 8:015f4eb0:::benchmark_data_smithi018_341490_object16464:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e410)", 2023-09-23T20:39:00.363 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:38:26.443916+0000", 2023-09-23T20:39:00.363 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 33.717152405, 2023-09-23T20:39:00.363 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1854093080000001, 2023-09-23T20:39:00.363 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:39:00.363 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:39:00.363 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:39:00.363 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35994", 2023-09-23T20:39:00.363 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:39:00.363 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 105787 2023-09-23T20:39:00.363 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.363 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:39:00.364 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.364 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:39:00.364 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.443916+0000", 2023-09-23T20:39:00.364 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:39:00.364 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.364 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.364 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:39:00.364 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.443916+0000", 2023-09-23T20:39:00.364 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:39:00.364 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.364 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.364 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:39:00.365 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.443915+0000", 2023-09-23T20:39:00.365 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:39:00.365 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.365 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.365 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:39:00.365 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.443918+0000", 2023-09-23T20:39:00.365 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.509e-06 2023-09-23T20:39:00.365 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.365 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.365 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:39:00.365 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.443924+0000", 2023-09-23T20:39:00.365 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.6740000000000002e-06 2023-09-23T20:39:00.366 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.366 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.366 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:39:00.366 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.443929+0000", 2023-09-23T20:39:00.366 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.5199999999999997e-06 2023-09-23T20:39:00.366 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.366 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.366 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:39:00.366 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.713182+0000", 2023-09-23T20:39:00.366 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.26925305700000002 2023-09-23T20:39:00.366 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.366 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.367 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:39:00.367 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.713190+0000", 2023-09-23T20:39:00.367 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.7980000000000006e-06 2023-09-23T20:39:00.367 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.367 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.367 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:39:00.367 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.613650+0000", 2023-09-23T20:39:00.367 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.90046006099999998 2023-09-23T20:39:00.367 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.367 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.367 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:39:00.367 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.613744+0000", 2023-09-23T20:39:00.367 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.4349000000000001e-05 2023-09-23T20:39:00.368 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.368 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.368 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,4", 2023-09-23T20:39:00.368 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.613821+0000", 2023-09-23T20:39:00.368 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.6915000000000005e-05 2023-09-23T20:39:00.368 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.368 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.368 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:39:00.368 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.619293+0000", 2023-09-23T20:39:00.368 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0054714159999999998 2023-09-23T20:39:00.368 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.368 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.368 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:39:00.368 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.627285+0000", 2023-09-23T20:39:00.369 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0079923000000000008 2023-09-23T20:39:00.369 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.369 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.369 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:39:00.369 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.627385+0000", 2023-09-23T20:39:00.369 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.9766000000000007e-05 2023-09-23T20:39:00.369 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.369 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.369 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:39:00.369 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.627433+0000", 2023-09-23T20:39:00.369 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.7963999999999997e-05 2023-09-23T20:39:00.369 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.369 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.370 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:39:00.370 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.629325+0000", 2023-09-23T20:39:00.370 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0018923880000000001 2023-09-23T20:39:00.370 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:39:00.370 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:39:00.370 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:39:00.370 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.370 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.370 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35994.0:105791 8.10 8:0d4c47c5:::benchmark_data_smithi018_341490_object16468:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e410)", 2023-09-23T20:39:00.370 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:38:26.443934+0000", 2023-09-23T20:39:00.370 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 33.717134223000002, 2023-09-23T20:39:00.371 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1850523630000001, 2023-09-23T20:39:00.371 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:39:00.371 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:39:00.371 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:39:00.371 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35994", 2023-09-23T20:39:00.371 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:39:00.371 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 105791 2023-09-23T20:39:00.371 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.371 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:39:00.371 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.371 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:39:00.371 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.443934+0000", 2023-09-23T20:39:00.372 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:39:00.372 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.372 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.372 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:39:00.372 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.443934+0000", 2023-09-23T20:39:00.372 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:39:00.372 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.372 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.372 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:39:00.372 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.443934+0000", 2023-09-23T20:39:00.372 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:39:00.372 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.372 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.373 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:39:00.373 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.443936+0000", 2023-09-23T20:39:00.373 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.1469999999999999e-06 2023-09-23T20:39:00.373 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.373 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.373 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:39:00.373 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.443942+0000", 2023-09-23T20:39:00.373 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.9050000000000002e-06 2023-09-23T20:39:00.373 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.373 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.373 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:39:00.373 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.443947+0000", 2023-09-23T20:39:00.374 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.8699999999999997e-06 2023-09-23T20:39:00.374 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.374 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.374 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:39:00.374 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.713210+0000", 2023-09-23T20:39:00.374 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.269262891 2023-09-23T20:39:00.374 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.374 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.374 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:39:00.374 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.713216+0000", 2023-09-23T20:39:00.375 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.0769999999999996e-06 2023-09-23T20:39:00.375 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.375 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.375 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:39:00.375 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.612639+0000", 2023-09-23T20:39:00.375 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.89942242299999997 2023-09-23T20:39:00.375 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.375 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.375 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:39:00.375 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.612710+0000", 2023-09-23T20:39:00.375 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.0743999999999999e-05 2023-09-23T20:39:00.375 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.376 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.376 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,4", 2023-09-23T20:39:00.376 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.612781+0000", 2023-09-23T20:39:00.376 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.0965999999999998e-05 2023-09-23T20:39:00.376 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.376 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.376 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:39:00.376 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.618520+0000", 2023-09-23T20:39:00.376 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0057398170000000004 2023-09-23T20:39:00.376 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.376 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.376 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:39:00.377 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.627222+0000", 2023-09-23T20:39:00.377 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0087017199999999996 2023-09-23T20:39:00.377 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.377 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.377 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:39:00.377 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.627275+0000", 2023-09-23T20:39:00.377 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.3081000000000002e-05 2023-09-23T20:39:00.377 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.377 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.377 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:39:00.377 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.627336+0000", 2023-09-23T20:39:00.378 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.0630000000000001e-05 2023-09-23T20:39:00.378 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.378 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.378 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:39:00.378 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.628986+0000", 2023-09-23T20:39:00.378 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0016504309999999999 2023-09-23T20:39:00.378 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:39:00.378 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:39:00.378 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:39:00.378 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.378 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.379 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35994.0:105792 8.0 8:02af189b:::benchmark_data_smithi018_341490_object16469:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e410)", 2023-09-23T20:39:00.379 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:38:26.443952+0000", 2023-09-23T20:39:00.379 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 33.717115950999997, 2023-09-23T20:39:00.379 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1871479650000001, 2023-09-23T20:39:00.379 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:39:00.379 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:39:00.379 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:39:00.379 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35994", 2023-09-23T20:39:00.379 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:39:00.379 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 105792 2023-09-23T20:39:00.379 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.380 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:39:00.380 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.380 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:39:00.380 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.443952+0000", 2023-09-23T20:39:00.380 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:39:00.380 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.380 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.380 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:39:00.380 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.443952+0000", 2023-09-23T20:39:00.380 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:39:00.380 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.380 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.381 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:39:00.381 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.443952+0000", 2023-09-23T20:39:00.381 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:39:00.381 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.381 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.381 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:39:00.381 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.443954+0000", 2023-09-23T20:39:00.381 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.6089999999999999e-06 2023-09-23T20:39:00.381 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.381 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.381 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:39:00.382 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.443961+0000", 2023-09-23T20:39:00.382 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.3029999999999996e-06 2023-09-23T20:39:00.382 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.382 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.382 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:39:00.382 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.443966+0000", 2023-09-23T20:39:00.382 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.9729999999999997e-06 2023-09-23T20:39:00.382 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.382 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.382 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:39:00.382 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.713239+0000", 2023-09-23T20:39:00.383 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.26927288300000002 2023-09-23T20:39:00.383 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.383 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.383 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:39:00.383 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.713246+0000", 2023-09-23T20:39:00.383 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.1690000000000003e-06 2023-09-23T20:39:00.383 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.383 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.383 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:39:00.383 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.614811+0000", 2023-09-23T20:39:00.383 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.90156508599999996 2023-09-23T20:39:00.384 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.384 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.384 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:39:00.384 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.614929+0000", 2023-09-23T20:39:00.384 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000117846 2023-09-23T20:39:00.384 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.384 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.384 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,4", 2023-09-23T20:39:00.384 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.615050+0000", 2023-09-23T20:39:00.384 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00012100900000000001 2023-09-23T20:39:00.384 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.384 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.385 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:39:00.385 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.619299+0000", 2023-09-23T20:39:00.385 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0042493160000000004 2023-09-23T20:39:00.385 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.385 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.385 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:39:00.385 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.627319+0000", 2023-09-23T20:39:00.385 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0080197559999999994 2023-09-23T20:39:00.385 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.385 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.385 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:39:00.385 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.629386+0000", 2023-09-23T20:39:00.386 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0020676959999999999 2023-09-23T20:39:00.386 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.386 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.386 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:39:00.386 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.629448+0000", 2023-09-23T20:39:00.386 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.1155999999999995e-05 2023-09-23T20:39:00.386 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.386 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.386 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:39:00.386 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.631100+0000", 2023-09-23T20:39:00.386 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.001652518 2023-09-23T20:39:00.387 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:39:00.387 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:39:00.387 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:39:00.387 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.387 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.387 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35994.0:105793 8.18 8:193032ff:::benchmark_data_smithi018_341490_object16470:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e410)", 2023-09-23T20:39:00.387 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:38:26.443971+0000", 2023-09-23T20:39:00.387 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 33.71709671, 2023-09-23T20:39:00.387 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.173527269, 2023-09-23T20:39:00.387 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:39:00.387 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:39:00.388 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:39:00.388 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35994", 2023-09-23T20:39:00.388 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:39:00.388 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 105793 2023-09-23T20:39:00.388 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.388 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:39:00.388 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.388 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:39:00.388 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.443971+0000", 2023-09-23T20:39:00.388 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:39:00.388 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.389 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.389 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:39:00.389 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.443971+0000", 2023-09-23T20:39:00.389 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:39:00.389 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.389 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.389 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:39:00.389 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.443971+0000", 2023-09-23T20:39:00.389 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:39:00.389 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.389 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.390 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:39:00.390 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.443979+0000", 2023-09-23T20:39:00.390 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.4270000000000008e-06 2023-09-23T20:39:00.390 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.390 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.390 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:39:00.390 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.443986+0000", 2023-09-23T20:39:00.390 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.5520000000000001e-06 2023-09-23T20:39:00.390 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.390 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.390 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:39:00.390 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.443991+0000", 2023-09-23T20:39:00.391 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.2429999999999996e-06 2023-09-23T20:39:00.391 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.391 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.391 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:39:00.391 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.713266+0000", 2023-09-23T20:39:00.391 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.26927488199999999 2023-09-23T20:39:00.391 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.391 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.391 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:39:00.391 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.713273+0000", 2023-09-23T20:39:00.391 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.4309999999999999e-06 2023-09-23T20:39:00.391 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.391 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.392 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:39:00.392 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.605188+0000", 2023-09-23T20:39:00.392 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.89191558699999995 2023-09-23T20:39:00.392 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.392 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.392 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:39:00.392 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.605348+0000", 2023-09-23T20:39:00.392 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00016024400000000001 2023-09-23T20:39:00.392 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.392 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.392 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 0,7", 2023-09-23T20:39:00.392 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.605486+0000", 2023-09-23T20:39:00.392 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00013745700000000001 2023-09-23T20:39:00.393 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.393 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.393 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:39:00.393 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.610094+0000", 2023-09-23T20:39:00.393 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0046085149999999997 2023-09-23T20:39:00.393 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.393 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.393 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:39:00.393 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.612624+0000", 2023-09-23T20:39:00.393 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0025298859999999999 2023-09-23T20:39:00.394 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.394 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.394 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:39:00.394 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.615895+0000", 2023-09-23T20:39:00.394 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0032704919999999998 2023-09-23T20:39:00.394 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.394 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.394 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:39:00.394 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.615938+0000", 2023-09-23T20:39:00.394 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.3510999999999997e-05 2023-09-23T20:39:00.394 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.395 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.395 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:39:00.395 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.617499+0000", 2023-09-23T20:39:00.395 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.001560475 2023-09-23T20:39:00.395 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:39:00.395 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:39:00.395 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:39:00.395 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.395 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.395 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35994.0:105794 8.0 8:06d1e15f:::benchmark_data_smithi018_341490_object16471:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e410)", 2023-09-23T20:39:00.395 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:38:26.443996+0000", 2023-09-23T20:39:00.395 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 33.717072420999997, 2023-09-23T20:39:00.396 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1894021189999999, 2023-09-23T20:39:00.396 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:39:00.396 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:39:00.396 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:39:00.396 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35994", 2023-09-23T20:39:00.396 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:39:00.396 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 105794 2023-09-23T20:39:00.396 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.396 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:39:00.396 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.396 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:39:00.397 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.443996+0000", 2023-09-23T20:39:00.397 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:39:00.397 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.397 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.397 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:39:00.397 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.443996+0000", 2023-09-23T20:39:00.397 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:39:00.397 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.397 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.397 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:39:00.398 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.443995+0000", 2023-09-23T20:39:00.398 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:39:00.398 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.398 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.398 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:39:00.398 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.443998+0000", 2023-09-23T20:39:00.398 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.7020000000000001e-06 2023-09-23T20:39:00.398 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.398 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.398 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:39:00.398 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.444004+0000", 2023-09-23T20:39:00.398 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.9479999999999996e-06 2023-09-23T20:39:00.399 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.399 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.399 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:39:00.399 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.444009+0000", 2023-09-23T20:39:00.399 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.502e-06 2023-09-23T20:39:00.399 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.399 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.399 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:39:00.399 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.713294+0000", 2023-09-23T20:39:00.399 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.26928485600000002 2023-09-23T20:39:00.399 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.399 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.399 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:39:00.400 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.713302+0000", 2023-09-23T20:39:00.400 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.4440000000000002e-06 2023-09-23T20:39:00.400 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.400 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.400 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:39:00.400 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.615633+0000", 2023-09-23T20:39:00.400 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.90233140000000001 2023-09-23T20:39:00.400 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.400 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.400 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:39:00.400 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.615759+0000", 2023-09-23T20:39:00.400 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00012542599999999999 2023-09-23T20:39:00.401 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.401 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.401 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,4", 2023-09-23T20:39:00.401 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.617918+0000", 2023-09-23T20:39:00.401 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0021595719999999998 2023-09-23T20:39:00.401 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.401 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.401 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:39:00.401 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.627197+0000", 2023-09-23T20:39:00.401 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0092789759999999995 2023-09-23T20:39:00.401 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.401 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.402 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:39:00.402 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.627353+0000", 2023-09-23T20:39:00.402 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00015556599999999999 2023-09-23T20:39:00.402 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.402 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.402 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:39:00.402 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.631674+0000", 2023-09-23T20:39:00.402 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0043214890000000004 2023-09-23T20:39:00.402 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.402 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.403 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:39:00.403 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.631719+0000", 2023-09-23T20:39:00.403 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.5314e-05 2023-09-23T20:39:00.403 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.403 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.403 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:39:00.403 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.633398+0000", 2023-09-23T20:39:00.403 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.001678368 2023-09-23T20:39:00.403 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:39:00.403 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:39:00.403 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:39:00.403 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.404 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.404 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35994.0:105797 8.18 8:1843db78:::benchmark_data_smithi018_341490_object16474:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e410)", 2023-09-23T20:39:00.404 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:38:26.444014+0000", 2023-09-23T20:39:00.404 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 33.717053851999999, 2023-09-23T20:39:00.404 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.17448456, 2023-09-23T20:39:00.404 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:39:00.404 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:39:00.404 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:39:00.404 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35994", 2023-09-23T20:39:00.404 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:39:00.405 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 105797 2023-09-23T20:39:00.405 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.405 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:39:00.405 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.405 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:39:00.405 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.444014+0000", 2023-09-23T20:39:00.405 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:39:00.405 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.405 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.405 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:39:00.405 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.444014+0000", 2023-09-23T20:39:00.406 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:39:00.406 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.406 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.406 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:39:00.406 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.444014+0000", 2023-09-23T20:39:00.406 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:39:00.406 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.406 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.406 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:39:00.406 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.444016+0000", 2023-09-23T20:39:00.406 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.097e-06 2023-09-23T20:39:00.407 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.407 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.407 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:39:00.407 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.444022+0000", 2023-09-23T20:39:00.407 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.8200000000000002e-06 2023-09-23T20:39:00.407 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.407 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.407 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:39:00.407 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.444027+0000", 2023-09-23T20:39:00.407 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.4360000000000001e-06 2023-09-23T20:39:00.407 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.408 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.408 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:39:00.408 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.713323+0000", 2023-09-23T20:39:00.408 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.26929555599999999 2023-09-23T20:39:00.408 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.408 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.408 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:39:00.408 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.713329+0000", 2023-09-23T20:39:00.408 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.9819999999999996e-06 2023-09-23T20:39:00.408 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.408 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.408 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:39:00.409 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.608546+0000", 2023-09-23T20:39:00.409 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.895217021 2023-09-23T20:39:00.409 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.409 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.409 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:39:00.409 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.608698+0000", 2023-09-23T20:39:00.409 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00015241000000000001 2023-09-23T20:39:00.409 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.409 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.409 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 0,7", 2023-09-23T20:39:00.409 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.608843+0000", 2023-09-23T20:39:00.410 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00014468799999999999 2023-09-23T20:39:00.410 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.410 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.410 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:39:00.410 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.613634+0000", 2023-09-23T20:39:00.410 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0047907979999999998 2023-09-23T20:39:00.410 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.410 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.410 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:39:00.410 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.615841+0000", 2023-09-23T20:39:00.410 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0022071690000000001 2023-09-23T20:39:00.410 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.411 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.411 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:39:00.411 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.617508+0000", 2023-09-23T20:39:00.411 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0016669390000000001 2023-09-23T20:39:00.411 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.411 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.411 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:39:00.411 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.617562+0000", 2023-09-23T20:39:00.411 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.3891999999999998e-05 2023-09-23T20:39:00.411 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:00.411 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:00.412 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:39:00.412 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.618499+0000", 2023-09-23T20:39:00.412 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00093712400000000001 2023-09-23T20:39:00.412 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:39:00.412 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:39:00.412 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:39:00.412 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:39:00.412 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:39:00.412 INFO:teuthology.orchestra.run.smithi018.stdout:} 2023-09-23T20:39:00.413 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:00.414 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:00.414 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:39:00.414 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:39:00.515 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:39:00.586 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:00.629 INFO:teuthology.orchestra.run.smithi018.stderr:set pool 8 pgp_num to 26 2023-09-23T20:39:00.883 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T20:39:00.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:00 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/976753666' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num", "val": "26"}]: dispatch 2023-09-23T20:39:00.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:00 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num", "val": "26"}]: dispatch 2023-09-23T20:39:00.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:00 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/976753666' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num", "val": "26"}]: dispatch 2023-09-23T20:39:00.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:00 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num", "val": "26"}]: dispatch 2023-09-23T20:39:00.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:00 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/976753666' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num", "val": "26"}]: dispatch 2023-09-23T20:39:00.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:00 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num", "val": "26"}]: dispatch 2023-09-23T20:39:01.265 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:39:01.271 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:01.271 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:39:01.372 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:39:01.414 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:01.457 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:39:01.463 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:01.463 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:39:01.564 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:39:01.610 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:01.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:01 smithi149 ceph-mon[170394]: pgmap v1024: 129 pgs: 7 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 1 active+clean+scrubbing+deep, 5 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 3.9 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 463 op/s; 20709/191715 objects degraded (10.802%); 3181/191715 objects misplaced (1.659%); 19 MiB/s, 317 objects/s recovering 2023-09-23T20:39:01.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:01 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num", "val": "26"}]': finished 2023-09-23T20:39:01.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:01 smithi149 ceph-mon[170394]: osdmap e422: 8 total, 8 up, 8 in 2023-09-23T20:39:01.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:01 smithi149 ceph-mon[170394]: failed to encode map e422 with expected crc 2023-09-23T20:39:01.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:01 smithi149 ceph-mon[170394]: failed to encode map e422 with expected crc 2023-09-23T20:39:01.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:01 smithi149 ceph-mon[170394]: failed to encode map e422 with expected crc 2023-09-23T20:39:01.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:01 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 20709/191715 objects degraded (10.802%), 13 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T20:39:01.830 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:01 smithi018 ceph-mon[184144]: pgmap v1024: 129 pgs: 7 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 1 active+clean+scrubbing+deep, 5 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 3.9 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 463 op/s; 20709/191715 objects degraded (10.802%); 3181/191715 objects misplaced (1.659%); 19 MiB/s, 317 objects/s recovering 2023-09-23T20:39:01.830 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:01 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num", "val": "26"}]': finished 2023-09-23T20:39:01.830 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:01 smithi018 ceph-mon[184144]: osdmap e422: 8 total, 8 up, 8 in 2023-09-23T20:39:01.830 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:01 smithi018 ceph-mon[184144]: failed to encode map e422 with expected crc 2023-09-23T20:39:01.830 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:01 smithi018 ceph-mon[184144]: failed to encode map e422 with expected crc 2023-09-23T20:39:01.830 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:01 smithi018 ceph-mon[184144]: failed to encode map e422 with expected crc 2023-09-23T20:39:01.830 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:01 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 20709/191715 objects degraded (10.802%), 13 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T20:39:01.831 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:01 smithi018 ceph-mon[179647]: pgmap v1024: 129 pgs: 7 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 1 active+clean+scrubbing+deep, 5 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 3.9 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 463 op/s; 20709/191715 objects degraded (10.802%); 3181/191715 objects misplaced (1.659%); 19 MiB/s, 317 objects/s recovering 2023-09-23T20:39:01.831 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:01 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num", "val": "26"}]': finished 2023-09-23T20:39:01.831 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:01 smithi018 ceph-mon[179647]: osdmap e422: 8 total, 8 up, 8 in 2023-09-23T20:39:01.831 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:01 smithi018 ceph-mon[179647]: failed to encode map e422 with expected crc 2023-09-23T20:39:01.831 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:01 smithi018 ceph-mon[179647]: failed to encode map e422 with expected crc 2023-09-23T20:39:01.831 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:01 smithi018 ceph-mon[179647]: failed to encode map e422 with expected crc 2023-09-23T20:39:01.831 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:01 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 20709/191715 objects degraded (10.802%), 13 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T20:39:03.844 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:39:03.847 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:03 smithi018 ceph-mon[179647]: pgmap v1026: 129 pgs: 7 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 3.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 462 op/s; 20709/189468 objects degraded (10.930%); 3181/189468 objects misplaced (1.679%); 14 MiB/s, 248 objects/s recovering 2023-09-23T20:39:03.847 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:03.848 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:39:03.848 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:03 smithi018 ceph-mon[184144]: pgmap v1026: 129 pgs: 7 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 3.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 462 op/s; 20709/189468 objects degraded (10.930%); 3181/189468 objects misplaced (1.679%); 14 MiB/s, 248 objects/s recovering 2023-09-23T20:39:03.949 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:39:03.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:03 smithi149 ceph-mon[170394]: pgmap v1026: 129 pgs: 7 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 3.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 462 op/s; 20709/189468 objects degraded (10.930%); 3181/189468 objects misplaced (1.679%); 14 MiB/s, 248 objects/s recovering 2023-09-23T20:39:03.989 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:04.324 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T20:39:05.210 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:39:05.391 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:39:04 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:39:04] "GET /metrics HTTP/1.1" 200 36564 "" "Prometheus/2.43.0" 2023-09-23T20:39:05.545 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:39:05.548 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:05.549 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:39:05.649 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:39:05.681 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:05.783 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:05 smithi149 ceph-mon[170394]: pgmap v1027: 129 pgs: 7 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 3.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 492 op/s; 20481/186402 objects degraded (10.988%); 2192/186402 objects misplaced (1.176%); 19 MiB/s, 328 objects/s recovering 2023-09-23T20:39:05.785 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:39:05.793 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:05.794 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:39:05.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:05 smithi018 ceph-mon[184144]: pgmap v1027: 129 pgs: 7 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 3.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 492 op/s; 20481/186402 objects degraded (10.988%); 2192/186402 objects misplaced (1.176%); 19 MiB/s, 328 objects/s recovering 2023-09-23T20:39:05.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:05 smithi018 ceph-mon[179647]: pgmap v1027: 129 pgs: 7 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 3.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 492 op/s; 20481/186402 objects degraded (10.988%); 2192/186402 objects misplaced (1.176%); 19 MiB/s, 328 objects/s recovering 2023-09-23T20:39:05.895 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:39:05.929 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:06.081 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:39:06.089 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:06.089 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:39:06.190 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:39:06.228 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:06.373 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:39:06.380 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:06.381 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:39:06.481 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:39:06.517 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:06.567 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:39:06.571 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:06.572 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:39:06.596 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:06 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 20481/186402 objects degraded (10.988%), 13 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T20:39:06.596 INFO:teuthology.orchestra.run.smithi149.stdout:{ 2023-09-23T20:39:06.597 INFO:teuthology.orchestra.run.smithi149.stdout: "ops": [ 2023-09-23T20:39:06.597 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:06.597 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_repop(client.35994.0:118937 8.5 e422/411 8:a0c2b618:::benchmark_data_smithi018_341490_object29614:head v 422'6995, mlcod=422'6993)", 2023-09-23T20:39:06.597 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:39:06.595061+0000", 2023-09-23T20:39:06.597 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 0.001511082, 2023-09-23T20:39:06.597 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00154104, 2023-09-23T20:39:06.597 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:39:06.597 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "started", 2023-09-23T20:39:06.597 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:39:06.597 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:06.597 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:39:06.598 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:06.595061+0000", 2023-09-23T20:39:06.598 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:39:06.598 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:06.598 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:06.598 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:39:06.598 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:06.595061+0000", 2023-09-23T20:39:06.598 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:39:06.598 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:06.598 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:06.598 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:39:06.599 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:06.595060+0000", 2023-09-23T20:39:06.599 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:39:06.599 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:06.599 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:06.599 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:39:06.599 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:06.595069+0000", 2023-09-23T20:39:06.599 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.2530000000000004e-06 2023-09-23T20:39:06.599 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:06.599 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:06.599 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:39:06.599 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:06.595083+0000", 2023-09-23T20:39:06.599 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.4224e-05 2023-09-23T20:39:06.600 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:06.600 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:06.600 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:39:06.600 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:06.595089+0000", 2023-09-23T20:39:06.600 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.1210000000000003e-06 2023-09-23T20:39:06.600 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:06.600 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:06.600 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:39:06.600 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:06.595132+0000", 2023-09-23T20:39:06.600 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.2861999999999997e-05 2023-09-23T20:39:06.600 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:06.600 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:06.601 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:39:06.601 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:06.595154+0000", 2023-09-23T20:39:06.601 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.1959000000000001e-05 2023-09-23T20:39:06.601 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:39:06.601 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:39:06.601 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:39:06.601 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:39:06.601 INFO:teuthology.orchestra.run.smithi149.stdout: ], 2023-09-23T20:39:06.601 INFO:teuthology.orchestra.run.smithi149.stdout: "num_ops": 1 2023-09-23T20:39:06.601 INFO:teuthology.orchestra.run.smithi149.stdout:} 2023-09-23T20:39:06.673 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:39:06.710 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:06.899 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:06 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 20481/186402 objects degraded (10.988%), 13 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T20:39:06.936 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:39:06.938 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:06 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 20481/186402 objects degraded (10.988%), 13 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T20:39:06.939 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:06.939 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:39:07.021 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T20:39:07.041 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:39:07.088 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:07.277 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:39:07.279 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:39:07.282 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:07.282 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:39:07.383 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:39:07.422 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:07.703 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:39:07.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:07 smithi018 ceph-mon[184144]: pgmap v1028: 129 pgs: 7 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 3.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 477 op/s; 20335/183723 objects degraded (11.068%); 1832/183723 objects misplaced (0.997%); 19 MiB/s, 328 objects/s recovering 2023-09-23T20:39:07.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:07 smithi018 ceph-mon[179647]: pgmap v1028: 129 pgs: 7 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 3.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 477 op/s; 20335/183723 objects degraded (11.068%); 1832/183723 objects misplaced (0.997%); 19 MiB/s, 328 objects/s recovering 2023-09-23T20:39:07.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:07 smithi149 ceph-mon[170394]: pgmap v1028: 129 pgs: 7 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 3.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 477 op/s; 20335/183723 objects degraded (11.068%); 1832/183723 objects misplaced (0.997%); 19 MiB/s, 328 objects/s recovering 2023-09-23T20:39:08.060 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:39:08.067 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:08.067 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:39:08.168 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:39:08.246 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:08.474 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:39:08.478 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:08.479 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:39:08.574 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:39:08.580 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:39:08.603 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:39:08.611 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:08.637 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:39:08.670 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:39:08.675 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:39:08.678 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:08.679 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:39:08.699 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:39:08.726 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:39:08.758 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:39:08.780 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:39:08.792 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:39:08.798 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:08 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/175397541' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:39:08.799 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:08 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/175397541' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:39:08.804 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:08 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/175397541' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:39:08.804 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:39:08.804 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:39:08.805 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:39:08.805 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:39:08.805 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:39:08.805 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:39:08.805 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:39:08.805 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:39:08.834 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:09.017 INFO:teuthology.orchestra.run.smithi149.stdout:ERROR: (22) Invalid argument 2023-09-23T20:39:09.017 INFO:teuthology.orchestra.run.smithi149.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:39:09.329 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:39:09.332 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:09.333 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:39:09.403 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T20:39:09.434 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:39:09.488 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:09.551 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:39:09.562 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:09.563 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:39:09.582 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 7, 3, 2, 4, 6, 5] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T20:39:09.582 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:39:09.582 INFO:tasks.thrashosds.thrasher:inject_pause on osd.4 2023-09-23T20:39:09.582 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T20:39:09.582 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T20:39:09.583 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 config set filestore_inject_stall 3 2023-09-23T20:39:09.619 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:09 smithi018 ceph-mon[179647]: pgmap v1029: 129 pgs: 7 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 3.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 477 op/s; 20179/180633 objects degraded (11.171%); 1832/180633 objects misplaced (1.014%); 19 MiB/s, 310 objects/s recovering 2023-09-23T20:39:09.623 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:09 smithi149 ceph-mon[170394]: pgmap v1029: 129 pgs: 7 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 3.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 477 op/s; 20179/180633 objects degraded (11.171%); 1832/180633 objects misplaced (1.014%); 19 MiB/s, 310 objects/s recovering 2023-09-23T20:39:09.664 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:39:09.717 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:09.827 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:39:09.833 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:09.834 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:39:09.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:09 smithi018 ceph-mon[184144]: pgmap v1029: 129 pgs: 7 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 3.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 477 op/s; 20179/180633 objects degraded (11.171%); 1832/180633 objects misplaced (1.014%); 19 MiB/s, 310 objects/s recovering 2023-09-23T20:39:09.934 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:39:09.975 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:10.039 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:39:10.045 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:10.045 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:39:10.146 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:39:10.149 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:39:10.185 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:10.239 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:39:10.247 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:10.248 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:39:10.348 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:39:10.389 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:10.410 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:39:10.440 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:39:10.446 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:10.447 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:39:10.547 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:39:10.587 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:10.648 INFO:teuthology.orchestra.run.smithi018.stderr:nodeep-scrub is set 2023-09-23T20:39:10.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:10 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1039385984' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:39:10.793 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:39:10.801 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:10.802 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:39:10.816 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:10 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1039385984' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:39:10.816 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:10 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1039385984' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:39:10.902 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:39:10.943 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:11.002 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:39:11.012 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:11.013 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:39:11.114 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:39:11.150 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:11.210 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:39:11.214 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:11.214 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:39:11.315 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:39:11.353 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:39:11.353 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:11.408 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:39:11.414 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:11.414 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:39:11.515 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:39:11.552 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:11.636 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:11 smithi149 ceph-mon[170394]: pgmap v1030: 129 pgs: 7 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 473 op/s; 19949/177510 objects degraded (11.238%); 1361/177510 objects misplaced (0.767%); 16 MiB/s, 257 objects/s recovering 2023-09-23T20:39:11.636 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:11 smithi149 ceph-mon[170394]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:39:11.636 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:11 smithi149 ceph-mon[170394]: failed to encode map e423 with expected crc 2023-09-23T20:39:11.636 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:11 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1039385984' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:39:11.636 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:11 smithi149 ceph-mon[170394]: osdmap e423: 8 total, 8 up, 8 in 2023-09-23T20:39:11.636 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:11 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 19949/177510 objects degraded (11.238%), 13 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T20:39:11.637 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:11 smithi018 ceph-mon[179647]: pgmap v1030: 129 pgs: 7 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 473 op/s; 19949/177510 objects degraded (11.238%); 1361/177510 objects misplaced (0.767%); 16 MiB/s, 257 objects/s recovering 2023-09-23T20:39:11.637 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:11 smithi018 ceph-mon[179647]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:39:11.637 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:11 smithi018 ceph-mon[179647]: failed to encode map e423 with expected crc 2023-09-23T20:39:11.637 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:11 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1039385984' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:39:11.637 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:11 smithi018 ceph-mon[179647]: osdmap e423: 8 total, 8 up, 8 in 2023-09-23T20:39:11.637 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:11 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 19949/177510 objects degraded (11.238%), 13 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T20:39:11.648 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:39:11.654 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:11.654 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:39:11.755 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:39:11.776 INFO:teuthology.orchestra.run.smithi149.stdout:ERROR: (22) Invalid argument 2023-09-23T20:39:11.777 INFO:teuthology.orchestra.run.smithi149.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:39:11.818 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:11.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:11 smithi018 ceph-mon[184144]: pgmap v1030: 129 pgs: 7 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 473 op/s; 19949/177510 objects degraded (11.238%); 1361/177510 objects misplaced (0.767%); 16 MiB/s, 257 objects/s recovering 2023-09-23T20:39:11.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:11 smithi018 ceph-mon[184144]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:39:11.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:11 smithi018 ceph-mon[184144]: failed to encode map e423 with expected crc 2023-09-23T20:39:11.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:11 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1039385984' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:39:11.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:11 smithi018 ceph-mon[184144]: osdmap e423: 8 total, 8 up, 8 in 2023-09-23T20:39:11.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:11 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 19949/177510 objects degraded (11.238%), 13 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T20:39:12.058 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:39:12.113 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:39:12.117 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:12.117 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:39:12.218 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:39:12.266 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:12.319 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:39:12.324 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:12.325 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:39:12.329 INFO:teuthology.orchestra.run.smithi149.stdout:{ 2023-09-23T20:39:12.329 INFO:teuthology.orchestra.run.smithi149.stdout: "success": "" 2023-09-23T20:39:12.329 INFO:teuthology.orchestra.run.smithi149.stdout:} 2023-09-23T20:39:12.360 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T20:39:12.426 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:39:12.459 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:12.680 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:39:12.688 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:12.689 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:39:12.790 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:39:12.864 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:12.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:12 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:39:12.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:12 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:39:12.915 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:39:12.920 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:12.921 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:12 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:39:12.921 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:39:13.014 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T20:39:13.022 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:39:13.103 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:39:13.161 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:13.278 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:39:13.282 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:13.282 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:39:13.383 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:39:13.415 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:13.475 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:39:13.481 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:13.481 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:39:13.532 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:39:13.567 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:39:13.582 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:39:13.597 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:39:13.627 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:39:13.631 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:13.666 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:39:13.679 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:39:13.684 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:13.684 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:39:13.700 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:39:13.735 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:39:13.765 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:39:13.768 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:39:13.769 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:39:13.769 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:39:13.769 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:39:13.769 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:39:13.769 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:39:13.769 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:39:13.769 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:39:13.785 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:39:13.822 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:39:13.848 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:13.904 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:13 smithi018 ceph-mon[179647]: pgmap v1032: 129 pgs: 7 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 3.6 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 481 op/s; 19949/175020 objects degraded (11.398%); 1361/175020 objects misplaced (0.778%); 16 MiB/s, 257 objects/s recovering 2023-09-23T20:39:13.904 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:13 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/81280382' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:39:13.906 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:13 smithi018 ceph-mon[184144]: pgmap v1032: 129 pgs: 7 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 3.6 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 481 op/s; 19949/175020 objects degraded (11.398%); 1361/175020 objects misplaced (0.778%); 16 MiB/s, 257 objects/s recovering 2023-09-23T20:39:13.906 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:13 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/81280382' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:39:13.943 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:13 smithi149 ceph-mon[170394]: pgmap v1032: 129 pgs: 7 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 3.6 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 481 op/s; 19949/175020 objects degraded (11.398%); 1361/175020 objects misplaced (0.778%); 16 MiB/s, 257 objects/s recovering 2023-09-23T20:39:13.944 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:13 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/81280382' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:39:14.125 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:39:14.129 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:14.130 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:39:14.231 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:39:14.268 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:14.309 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:39:14.314 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:14.315 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:39:14.416 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:39:14.465 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:14.517 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:39:14.523 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:14.524 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:39:14.624 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:39:14.667 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:14.714 INFO:teuthology.orchestra.run.smithi149.stdout:{ 2023-09-23T20:39:14.714 INFO:teuthology.orchestra.run.smithi149.stdout: "ops": [], 2023-09-23T20:39:14.714 INFO:teuthology.orchestra.run.smithi149.stdout: "num_ops": 0 2023-09-23T20:39:14.714 INFO:teuthology.orchestra.run.smithi149.stdout:} 2023-09-23T20:39:14.721 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:39:14.726 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:14.726 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:39:14.827 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:39:14.834 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:14 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:39:14.835 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:14 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:39:14.869 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:14.902 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:14 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:39:15.073 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:39:15.077 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:15.077 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:39:15.128 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T20:39:15.144 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:39:14 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:39:14] "GET /metrics HTTP/1.1" 200 36563 "" "Prometheus/2.43.0" 2023-09-23T20:39:15.178 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:39:15.224 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:15.306 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:39:15.310 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:15.311 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:39:15.411 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:39:15.453 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:15.508 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:39:15.514 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:15.515 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:39:15.615 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:39:15.689 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:15.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:15 smithi149 ceph-mon[170394]: pgmap v1033: 129 pgs: 7 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 3.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 490 op/s; 19726/171681 objects degraded (11.490%); 1361/171681 objects misplaced (0.793%); 9.9 MiB/s, 158 objects/s recovering 2023-09-23T20:39:15.750 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:39:15.757 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:15.758 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:39:15.842 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:39:15.859 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:39:15.919 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:15 smithi018 ceph-mon[179647]: pgmap v1033: 129 pgs: 7 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 3.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 490 op/s; 19726/171681 objects degraded (11.490%); 1361/171681 objects misplaced (0.793%); 9.9 MiB/s, 158 objects/s recovering 2023-09-23T20:39:15.919 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:15 smithi018 ceph-mon[184144]: pgmap v1033: 129 pgs: 7 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 3.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 490 op/s; 19726/171681 objects degraded (11.490%); 1361/171681 objects misplaced (0.793%); 9.9 MiB/s, 158 objects/s recovering 2023-09-23T20:39:15.922 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:15.989 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:39:15.996 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:15.997 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:39:16.097 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:39:16.152 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:16.228 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:39:16.233 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:16.234 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:39:16.272 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:39:16.334 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:39:16.377 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:16.637 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:39:16.641 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:16.641 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:39:16.689 INFO:teuthology.orchestra.run.smithi018.stderr:noscrub is unset 2023-09-23T20:39:16.742 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:39:16.804 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:16.854 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:39:16.858 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:16.859 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:39:16.959 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:39:16.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:16 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2420147872' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:39:16.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:16 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 19726/171681 objects degraded (11.490%), 13 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T20:39:16.994 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:17.045 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:39:17.055 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:17.055 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:39:17.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:16 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2420147872' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:39:17.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:16 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 19726/171681 objects degraded (11.490%), 13 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T20:39:17.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:16 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2420147872' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:39:17.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:16 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 19726/171681 objects degraded (11.490%), 13 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T20:39:17.156 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:39:17.210 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:17.274 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:39:17.278 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:17.279 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:39:17.380 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:39:17.388 INFO:teuthology.orchestra.run.smithi149.stdout:{ 2023-09-23T20:39:17.388 INFO:teuthology.orchestra.run.smithi149.stdout: "ops": [], 2023-09-23T20:39:17.388 INFO:teuthology.orchestra.run.smithi149.stdout: "complaint_time": 30, 2023-09-23T20:39:17.388 INFO:teuthology.orchestra.run.smithi149.stdout: "num_blocked_ops": 0 2023-09-23T20:39:17.388 INFO:teuthology.orchestra.run.smithi149.stdout:} 2023-09-23T20:39:17.397 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:39:17.423 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:17.539 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:39:17.544 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:17.544 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:39:17.645 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:39:17.678 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:17 smithi149 ceph-mon[170394]: pgmap v1034: 129 pgs: 7 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 3.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 491 op/s; 19573/168981 objects degraded (11.583%); 870/168981 objects misplaced (0.515%); 11 MiB/s, 172 objects/s recovering 2023-09-23T20:39:17.678 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:17 smithi149 ceph-mon[170394]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:39:17.678 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:17 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2420147872' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:39:17.678 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:17 smithi149 ceph-mon[170394]: osdmap e424: 8 total, 8 up, 8 in 2023-09-23T20:39:17.678 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:17 smithi149 ceph-mon[170394]: failed to encode map e424 with expected crc 2023-09-23T20:39:17.679 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:17 smithi149 ceph-mon[170394]: failed to encode map e424 with expected crc 2023-09-23T20:39:17.706 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:17.749 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 7, 3, 2, 4, 6, 5] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T20:39:17.749 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:39:17.749 INFO:tasks.thrashosds.thrasher:fixing pg num pool .mgr 2023-09-23T20:39:17.750 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:39:17.762 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:39:17.770 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:17.770 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:39:17.779 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T20:39:17.871 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:39:17.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:17 smithi018 ceph-mon[184144]: pgmap v1034: 129 pgs: 7 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 3.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 491 op/s; 19573/168981 objects degraded (11.583%); 870/168981 objects misplaced (0.515%); 11 MiB/s, 172 objects/s recovering 2023-09-23T20:39:17.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:17 smithi018 ceph-mon[184144]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:39:17.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:17 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2420147872' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:39:17.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:17 smithi018 ceph-mon[184144]: osdmap e424: 8 total, 8 up, 8 in 2023-09-23T20:39:17.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:17 smithi018 ceph-mon[184144]: failed to encode map e424 with expected crc 2023-09-23T20:39:17.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:17 smithi018 ceph-mon[184144]: failed to encode map e424 with expected crc 2023-09-23T20:39:17.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:17 smithi018 ceph-mon[179647]: pgmap v1034: 129 pgs: 7 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 3.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 491 op/s; 19573/168981 objects degraded (11.583%); 870/168981 objects misplaced (0.515%); 11 MiB/s, 172 objects/s recovering 2023-09-23T20:39:17.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:17 smithi018 ceph-mon[179647]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:39:17.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:17 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2420147872' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:39:17.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:17 smithi018 ceph-mon[179647]: osdmap e424: 8 total, 8 up, 8 in 2023-09-23T20:39:17.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:17 smithi018 ceph-mon[179647]: failed to encode map e424 with expected crc 2023-09-23T20:39:17.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:17 smithi018 ceph-mon[179647]: failed to encode map e424 with expected crc 2023-09-23T20:39:17.984 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:18.080 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:39:18.087 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:18.088 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:39:18.189 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:39:18.237 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:18.405 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:39:18.412 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:18.412 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:39:18.513 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:39:18.548 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:18.580 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:39:18.931 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:39:18.946 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:18.947 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:39:19.048 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:39:19.054 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:39:19.084 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:19.182 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:39:19.186 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:19.186 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:39:19.226 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:39:19.265 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:39:19.267 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T20:39:19.287 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:39:19.304 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:39:19.316 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:19.344 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:39:19.359 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:39:19.363 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:19.363 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:39:19.381 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:39:19.424 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:39:19.464 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:39:19.468 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:39:19.506 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:39:19.523 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:39:19.523 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:39:19.523 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:39:19.524 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:39:19.524 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:39:19.524 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:39:19.524 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:39:19.524 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:39:19.599 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:19.720 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:39:19.727 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:19.727 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:39:19.828 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:39:19.851 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:19 smithi018 ceph-mon[179647]: pgmap v1036: 129 pgs: 7 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 3.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 496 op/s; 19418/165597 objects degraded (11.726%); 870/165597 objects misplaced (0.525%); 8.0 MiB/s, 127 objects/s recovering 2023-09-23T20:39:19.851 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:19 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1399854981' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:39:19.852 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:19 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:39:19.852 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:19 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:39:19.852 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:19 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:39:19.852 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:19 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:39:19.852 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:19 smithi018 ceph-mon[184144]: pgmap v1036: 129 pgs: 7 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 3.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 496 op/s; 19418/165597 objects degraded (11.726%); 870/165597 objects misplaced (0.525%); 8.0 MiB/s, 127 objects/s recovering 2023-09-23T20:39:19.852 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:19 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1399854981' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:39:19.852 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:19 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:39:19.852 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:19 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:39:19.853 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:19 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:39:19.853 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:19 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:39:19.882 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:19.975 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:19 smithi149 ceph-mon[170394]: pgmap v1036: 129 pgs: 7 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 3.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 496 op/s; 19418/165597 objects degraded (11.726%); 870/165597 objects misplaced (0.525%); 8.0 MiB/s, 127 objects/s recovering 2023-09-23T20:39:19.975 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:19 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1399854981' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:39:19.975 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:19 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:39:19.975 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:19 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:39:19.975 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:19 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:39:19.975 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:19 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:39:20.148 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:39:20.151 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:20.152 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:39:20.243 INFO:teuthology.orchestra.run.smithi149.stdout:ERROR: (22) Invalid argument 2023-09-23T20:39:20.243 INFO:teuthology.orchestra.run.smithi149.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:39:20.252 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:39:20.310 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:20.372 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:39:20.378 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:20.379 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:39:20.480 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:39:20.512 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:20.671 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:39:20.674 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:20.675 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:39:20.737 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T20:39:20.775 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:39:20.786 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:39:20.842 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:20 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:39:20.842 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:20 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:39:20.843 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:20 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:39:20.843 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:20 smithi018 ceph-mon[184144]: 1.10 scrub starts 2023-09-23T20:39:20.843 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:20 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:39:20.843 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:20 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:39:20.843 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:20 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:39:20.843 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:20 smithi018 ceph-mon[179647]: 1.10 scrub starts 2023-09-23T20:39:20.846 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:20.926 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:39:20.929 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:20.930 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:39:20.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:20 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:39:20.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:20 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:39:20.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:20 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:39:20.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:20 smithi149 ceph-mon[170394]: 1.10 scrub starts 2023-09-23T20:39:21.031 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:39:21.058 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:21.101 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:39:21.105 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:21.106 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:39:21.206 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:39:21.209 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:39:21.226 INFO:teuthology.orchestra.run.smithi018.stderr:dumped all 2023-09-23T20:39:21.241 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:21.290 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:39:21.293 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:21.294 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:39:21.394 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:39:21.425 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:39:21.430 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:21.490 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:39:21.497 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:21.498 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:39:21.599 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:39:21.634 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:21.678 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:39:21.687 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:21.687 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:39:21.788 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:39:21.838 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:21.874 INFO:teuthology.orchestra.run.smithi018.stdout:{"pg_ready":true,"pg_map":{"version":1037,"stamp":"2023-09-23T20:39:20.048011+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3547820911,"num_objects":54094,"num_object_clones":0,"num_object_copies":162282,"num_objects_missing_on_primary":657,"num_objects_missing":657,"num_objects_degraded":19177,"num_objects_misplaced":343,"num_objects_unfound":0,"num_objects_dirty":54094,"num_whiteouts":0,"num_read":6139,"num_read_kb":13491,"num_write":125558,"num_write_kb":5729974,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21570,"num_bytes_recovered":1367728434,"num_keys_recovered":40,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":147845,"ondisk_log_size":147845,"up":387,"acting":381,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":423,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":15750836,"kb_used_data":11908596,"kb_used_omap":32,"kb_used_meta":3842079,"kb_avail":734210380,"statfs":{"total":767960285184,"available":751831429120,"internally_reserved":0,"allocated":12194402304,"data_stored":12180406241,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32773,"internal_metadata":3934289915},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,3,3,4,2,1,0,1,5],"upper_bound":512},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-311754752,"num_objects":-4757,"num_object_clones":0,"num_object_copies":-14271,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-772,"num_objects_misplaced":-1018,"num_objects_unfound":0,"num_objects_dirty":-4757,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4757,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1790,"num_bytes_recovered":117309440,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.413097"},"pg_stats":[{"pgid":"8.11","version":"424'7233","reported_seq":7308,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-09-23T20:39:19.406111+0000","last_change":"2023-09-23T20:38:27.478425+0000","last_active":"2023-09-23T20:39:19.406111+0000","last_peered":"2023-09-23T20:39:19.406111+0000","last_clean":"2023-09-23T20:39:19.406111+0000","last_became_active":"2023-09-23T20:38:27.472168+0000","last_became_peered":"2023-09-23T20:38:27.472168+0000","last_unstale":"2023-09-23T20:39:19.406111+0000","last_undegraded":"2023-09-23T20:39:19.406111+0000","last_fullsized":"2023-09-23T20:39:19.406111+0000","mapping_epoch":410,"log_start":"373'1694","ondisk_log_start":"373'1694","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":5539,"log_dups_size":0,"ondisk_log_size":5539,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":108756992,"num_objects":1660,"num_object_clones":0,"num_object_copies":4980,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1660,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3924,"num_write_kb":178656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.18","version":"90'161","reported_seq":484,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-23T20:39:10.646917+0000","last_change":"2023-09-23T20:38:03.548189+0000","last_active":"2023-09-23T20:39:10.646917+0000","last_peered":"2023-09-23T20:39:10.646917+0000","last_clean":"2023-09-23T20:39:10.646917+0000","last_became_active":"2023-09-23T20:38:03.547630+0000","last_became_peered":"2023-09-23T20:38:03.547630+0000","last_unstale":"2023-09-23T20:39:10.646917+0000","last_undegraded":"2023-09-23T20:39:10.646917+0000","last_fullsized":"2023-09-23T20:39:10.646917+0000","mapping_epoch":404,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":405,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:35:03.551143+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:35:03.551143+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":343193,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"69'1","reported_seq":120,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-09-23T20:39:00.602734+0000","last_change":"2023-09-23T20:32:33.233853+0000","last_active":"2023-09-23T20:39:00.602734+0000","last_peered":"2023-09-23T20:39:00.602734+0000","last_clean":"2023-09-23T20:39:00.602734+0000","last_became_active":"2023-09-23T20:32:33.233546+0000","last_became_peered":"2023-09-23T20:32:33.233546+0000","last_unstale":"2023-09-23T20:39:00.602734+0000","last_undegraded":"2023-09-23T20:39:00.602734+0000","last_fullsized":"2023-09-23T20:39:00.602734+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:50:27.130808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":236,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-23T20:39:10.663218+0000","last_change":"2023-09-23T20:37:43.579611+0000","last_active":"2023-09-23T20:39:10.663218+0000","last_peered":"2023-09-23T20:39:10.663218+0000","last_clean":"2023-09-23T20:39:10.663218+0000","last_became_active":"2023-09-23T20:37:43.579323+0000","last_became_peered":"2023-09-23T20:37:43.579323+0000","last_unstale":"2023-09-23T20:39:10.663218+0000","last_undegraded":"2023-09-23T20:39:10.663218+0000","last_fullsized":"2023-09-23T20:39:10.663218+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:17:32.311427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.12","version":"423'7243","reported_seq":7365,"reported_epoch":423,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:39:15.472480+0000","last_change":"2023-09-23T20:38:39.273131+0000","last_active":"2023-09-23T20:39:15.472480+0000","last_peered":"2023-09-23T20:39:15.472480+0000","last_clean":"2023-09-23T20:37:29.399979+0000","last_became_active":"2023-09-23T20:38:39.252214+0000","last_became_peered":"2023-09-23T20:38:39.252214+0000","last_unstale":"2023-09-23T20:39:15.472480+0000","last_undegraded":"2023-09-23T20:38:38.456024+0000","last_fullsized":"2023-09-23T20:38:38.449466+0000","mapping_epoch":416,"log_start":"373'1996","ondisk_log_start":"373'1996","created":410,"last_epoch_clean":370,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":5247,"log_dups_size":0,"ondisk_log_size":5247,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112787456,"num_objects":1721,"num_object_clones":0,"num_object_copies":5163,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":715,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1721,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3893,"num_write_kb":179648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":753664,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0],"avail_no_missing":["4","0"],"object_location_counts":[{"shards":"0,3,4","objects":141},{"shards":"0,4","objects":1580}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1b","version":"90'161","reported_seq":541,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-09-23T20:39:00.602287+0000","last_change":"2023-09-23T20:38:39.106622+0000","last_active":"2023-09-23T20:39:00.602287+0000","last_peered":"2023-09-23T20:39:00.602287+0000","last_clean":"2023-09-23T20:39:00.602287+0000","last_became_active":"2023-09-23T20:37:17.291961+0000","last_became_peered":"2023-09-23T20:37:17.291961+0000","last_unstale":"2023-09-23T20:39:00.602287+0000","last_undegraded":"2023-09-23T20:39:00.602287+0000","last_fullsized":"2023-09-23T20:39:00.602287+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:05.442288+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:36:05.442288+0000","last_clean_scrub_stamp":"2023-09-23T20:36:05.442288+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0019500940000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,2],"acting":[4,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":391,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-23T20:39:10.651260+0000","last_change":"2023-09-23T20:34:59.983035+0000","last_active":"2023-09-23T20:39:10.651260+0000","last_peered":"2023-09-23T20:39:10.651260+0000","last_clean":"2023-09-23T20:39:10.651260+0000","last_became_active":"2023-09-23T20:34:59.982459+0000","last_became_peered":"2023-09-23T20:34:59.982459+0000","last_unstale":"2023-09-23T20:39:10.651260+0000","last_undegraded":"2023-09-23T20:39:10.651260+0000","last_fullsized":"2023-09-23T20:39:10.651260+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:16:48.510783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":392,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-09-23T20:39:00.603093+0000","last_change":"2023-09-23T20:25:02.957185+0000","last_active":"2023-09-23T20:39:00.603093+0000","last_peered":"2023-09-23T20:39:00.603093+0000","last_clean":"2023-09-23T20:39:00.603093+0000","last_became_active":"2023-09-23T20:25:02.956689+0000","last_became_peered":"2023-09-23T20:25:02.956689+0000","last_unstale":"2023-09-23T20:39:00.603093+0000","last_undegraded":"2023-09-23T20:39:00.603093+0000","last_fullsized":"2023-09-23T20:39:00.603093+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:04:28.524015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.13","version":"424'7212","reported_seq":9572,"reported_epoch":424,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T20:39:19.510085+0000","last_change":"2023-09-23T20:38:27.598846+0000","last_active":"2023-09-23T20:39:19.510085+0000","last_peered":"2023-09-23T20:39:19.510085+0000","last_clean":"2023-09-23T20:37:13.297905+0000","last_became_active":"2023-09-23T20:38:27.564876+0000","last_became_peered":"2023-09-23T20:38:27.564876+0000","last_unstale":"2023-09-23T20:39:19.510085+0000","last_undegraded":"2023-09-23T20:38:27.556546+0000","last_fullsized":"2023-09-23T20:39:19.510085+0000","mapping_epoch":410,"log_start":"373'1680","ondisk_log_start":"373'1680","created":410,"last_epoch_clean":370,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":5532,"log_dups_size":0,"ondisk_log_size":5532,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":110592000,"num_objects":1688,"num_object_clones":0,"num_object_copies":5064,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2191,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1688,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3906,"num_write_kb":178976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":777,"num_bytes_recovered":50888704,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":["1","5"],"object_location_counts":[{"shards":"1,2,3","objects":163},{"shards":"1,2,6","objects":-5},{"shards":"1,5,6","objects":1530}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1a","version":"90'161","reported_seq":502,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-09-23T20:39:00.602291+0000","last_change":"2023-09-23T20:36:01.292103+0000","last_active":"2023-09-23T20:39:00.602291+0000","last_peered":"2023-09-23T20:39:00.602291+0000","last_clean":"2023-09-23T20:39:00.602291+0000","last_became_active":"2023-09-23T20:35:26.639912+0000","last_became_peered":"2023-09-23T20:35:26.639912+0000","last_unstale":"2023-09-23T20:39:00.602291+0000","last_undegraded":"2023-09-23T20:39:00.602291+0000","last_fullsized":"2023-09-23T20:39:00.602291+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:01.292032+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:01.292032+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:04:53.926640+0000","scrub_duration":0.0032983159999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":216,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-23T20:39:10.661397+0000","last_change":"2023-09-23T20:37:43.584414+0000","last_active":"2023-09-23T20:39:10.661397+0000","last_peered":"2023-09-23T20:39:10.661397+0000","last_clean":"2023-09-23T20:39:10.661397+0000","last_became_active":"2023-09-23T20:37:43.584116+0000","last_became_peered":"2023-09-23T20:37:43.584116+0000","last_unstale":"2023-09-23T20:39:10.661397+0000","last_undegraded":"2023-09-23T20:39:10.661397+0000","last_fullsized":"2023-09-23T20:39:10.661397+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:17:54.258172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":40,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-23T20:39:10.649207+0000","last_change":"2023-09-23T20:38:37.457427+0000","last_active":"2023-09-23T20:39:10.649207+0000","last_peered":"2023-09-23T20:39:10.649207+0000","last_clean":"2023-09-23T20:39:10.649207+0000","last_became_active":"2023-09-23T20:38:37.457099+0000","last_became_peered":"2023-09-23T20:38:37.457099+0000","last_unstale":"2023-09-23T20:39:10.649207+0000","last_undegraded":"2023-09-23T20:39:10.649207+0000","last_fullsized":"2023-09-23T20:39:10.649207+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:31:27.084994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"90'161","reported_seq":616,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-23T20:39:10.651509+0000","last_change":"2023-09-23T20:37:17.304763+0000","last_active":"2023-09-23T20:39:10.651509+0000","last_peered":"2023-09-23T20:39:10.651509+0000","last_clean":"2023-09-23T20:39:10.651509+0000","last_became_active":"2023-09-23T20:37:17.304362+0000","last_became_peered":"2023-09-23T20:37:17.304362+0000","last_unstale":"2023-09-23T20:39:10.651509+0000","last_undegraded":"2023-09-23T20:39:10.651509+0000","last_fullsized":"2023-09-23T20:39:10.651509+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:03.411025+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:03.411025+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:11:42.762720+0000","scrub_duration":0.001802922,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":416,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-09-23T20:39:00.602654+0000","last_change":"2023-09-23T20:38:02.606871+0000","last_active":"2023-09-23T20:39:00.602654+0000","last_peered":"2023-09-23T20:39:00.602654+0000","last_clean":"2023-09-23T20:39:00.602654+0000","last_became_active":"2023-09-23T20:38:02.606596+0000","last_became_peered":"2023-09-23T20:38:02.606596+0000","last_unstale":"2023-09-23T20:39:00.602654+0000","last_undegraded":"2023-09-23T20:39:00.602654+0000","last_fullsized":"2023-09-23T20:39:00.602654+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:27:33.341667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":147,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-23T20:39:10.647089+0000","last_change":"2023-09-23T20:38:02.579343+0000","last_active":"2023-09-23T20:39:10.647089+0000","last_peered":"2023-09-23T20:39:10.647089+0000","last_clean":"2023-09-23T20:39:10.647089+0000","last_became_active":"2023-09-23T20:38:02.579047+0000","last_became_peered":"2023-09-23T20:38:02.579047+0000","last_unstale":"2023-09-23T20:39:10.647089+0000","last_undegraded":"2023-09-23T20:39:10.647089+0000","last_fullsized":"2023-09-23T20:39:10.647089+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:44:03.166556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.14","version":"90'161","reported_seq":752,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-23T20:39:10.647906+0000","last_change":"2023-09-23T20:35:02.236398+0000","last_active":"2023-09-23T20:39:10.647906+0000","last_peered":"2023-09-23T20:39:10.647906+0000","last_clean":"2023-09-23T20:39:10.647906+0000","last_became_active":"2023-09-23T20:35:02.235988+0000","last_became_peered":"2023-09-23T20:35:02.235988+0000","last_unstale":"2023-09-23T20:39:10.647906+0000","last_undegraded":"2023-09-23T20:39:10.647906+0000","last_fullsized":"2023-09-23T20:39:10.647906+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:47.479012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:47.479012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:13.519648+0000","scrub_duration":0.0020136590000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":291,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-23T20:39:10.651592+0000","last_change":"2023-09-23T20:37:14.362069+0000","last_active":"2023-09-23T20:39:10.651592+0000","last_peered":"2023-09-23T20:39:10.651592+0000","last_clean":"2023-09-23T20:39:10.651592+0000","last_became_active":"2023-09-23T20:37:14.361772+0000","last_became_peered":"2023-09-23T20:37:14.361772+0000","last_unstale":"2023-09-23T20:39:10.651592+0000","last_undegraded":"2023-09-23T20:39:10.651592+0000","last_fullsized":"2023-09-23T20:39:10.651592+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:20:31.626104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":247,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-23T20:39:10.661981+0000","last_change":"2023-09-23T20:38:02.542034+0000","last_active":"2023-09-23T20:39:10.661981+0000","last_peered":"2023-09-23T20:39:10.661981+0000","last_clean":"2023-09-23T20:39:10.661981+0000","last_became_active":"2023-09-23T20:38:02.541575+0000","last_became_peered":"2023-09-23T20:38:02.541575+0000","last_unstale":"2023-09-23T20:39:10.661981+0000","last_undegraded":"2023-09-23T20:39:10.661981+0000","last_fullsized":"2023-09-23T20:39:10.661981+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:50:54.591529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"90'161","reported_seq":565,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-09-23T20:39:00.602448+0000","last_change":"2023-09-23T20:38:05.611362+0000","last_active":"2023-09-23T20:39:00.602448+0000","last_peered":"2023-09-23T20:39:00.602448+0000","last_clean":"2023-09-23T20:39:00.602448+0000","last_became_active":"2023-09-23T20:38:05.611031+0000","last_became_peered":"2023-09-23T20:38:05.611031+0000","last_unstale":"2023-09-23T20:39:00.602448+0000","last_undegraded":"2023-09-23T20:39:00.602448+0000","last_fullsized":"2023-09-23T20:39:00.602448+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:03.435480+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:03.435480+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":451,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-23T20:39:10.652495+0000","last_change":"2023-09-23T20:32:33.270617+0000","last_active":"2023-09-23T20:39:10.652495+0000","last_peered":"2023-09-23T20:39:10.652495+0000","last_clean":"2023-09-23T20:39:10.652495+0000","last_became_active":"2023-09-23T20:32:33.227902+0000","last_became_peered":"2023-09-23T20:32:33.227902+0000","last_unstale":"2023-09-23T20:39:10.652495+0000","last_undegraded":"2023-09-23T20:39:10.652495+0000","last_fullsized":"2023-09-23T20:39:10.652495+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:30:26.847831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":360,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":393,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-23T20:39:10.652312+0000","last_change":"2023-09-23T20:38:02.599068+0000","last_active":"2023-09-23T20:39:10.652312+0000","last_peered":"2023-09-23T20:39:10.652312+0000","last_clean":"2023-09-23T20:39:10.652312+0000","last_became_active":"2023-09-23T20:38:02.597602+0000","last_became_peered":"2023-09-23T20:38:02.597602+0000","last_unstale":"2023-09-23T20:39:10.652312+0000","last_undegraded":"2023-09-23T20:39:10.652312+0000","last_fullsized":"2023-09-23T20:39:10.652312+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:34:06.337029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"90'161","reported_seq":547,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-23T20:39:10.651839+0000","last_change":"2023-09-23T20:36:02.548387+0000","last_active":"2023-09-23T20:39:10.651839+0000","last_peered":"2023-09-23T20:39:10.651839+0000","last_clean":"2023-09-23T20:39:10.651839+0000","last_became_active":"2023-09-23T20:35:26.644787+0000","last_became_peered":"2023-09-23T20:35:26.644787+0000","last_unstale":"2023-09-23T20:39:10.651839+0000","last_undegraded":"2023-09-23T20:39:10.651839+0000","last_fullsized":"2023-09-23T20:39:10.651839+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:02.548285+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:02.548285+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:37:15.216553+0000","scrub_duration":0.0015514459999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":170,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-09-23T20:39:00.603139+0000","last_change":"2023-09-23T20:38:02.560646+0000","last_active":"2023-09-23T20:39:00.603139+0000","last_peered":"2023-09-23T20:39:00.603139+0000","last_clean":"2023-09-23T20:39:00.603139+0000","last_became_active":"2023-09-23T20:38:02.539814+0000","last_became_peered":"2023-09-23T20:38:02.539814+0000","last_unstale":"2023-09-23T20:39:00.603139+0000","last_undegraded":"2023-09-23T20:39:00.603139+0000","last_fullsized":"2023-09-23T20:39:00.603139+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:00:09.435023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":150,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-09-23T20:39:00.602613+0000","last_change":"2023-09-23T20:38:02.579222+0000","last_active":"2023-09-23T20:39:00.602613+0000","last_peered":"2023-09-23T20:39:00.602613+0000","last_clean":"2023-09-23T20:39:00.602613+0000","last_became_active":"2023-09-23T20:38:02.578813+0000","last_became_peered":"2023-09-23T20:38:02.578813+0000","last_unstale":"2023-09-23T20:39:00.602613+0000","last_undegraded":"2023-09-23T20:39:00.602613+0000","last_fullsized":"2023-09-23T20:39:00.602613+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:57:31.080027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.18","version":"424'7256","reported_seq":10664,"reported_epoch":424,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T20:39:18.748132+0000","last_change":"2023-09-23T20:38:27.606849+0000","last_active":"2023-09-23T20:39:18.748132+0000","last_peered":"2023-09-23T20:39:18.748132+0000","last_clean":"2023-09-23T20:37:13.320723+0000","last_became_active":"2023-09-23T20:38:27.595968+0000","last_became_peered":"2023-09-23T20:38:27.595968+0000","last_unstale":"2023-09-23T20:39:18.748132+0000","last_undegraded":"2023-09-23T20:38:27.539992+0000","last_fullsized":"2023-09-23T20:39:18.748132+0000","mapping_epoch":410,"log_start":"373'2334","ondisk_log_start":"373'2334","created":410,"last_epoch_clean":370,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":4922,"log_dups_size":0,"ondisk_log_size":4922,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":107905024,"num_objects":1647,"num_object_clones":0,"num_object_copies":4941,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1512,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1647,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3948,"num_write_kb":179008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1150,"num_bytes_recovered":75366400,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":["3","5","7"],"object_location_counts":[{"shards":"0,3,7","objects":135},{"shards":"3,5,7","objects":1512}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.11","version":"90'161","reported_seq":671,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-09-23T20:39:00.601509+0000","last_change":"2023-09-23T20:37:17.282654+0000","last_active":"2023-09-23T20:39:00.601509+0000","last_peered":"2023-09-23T20:39:00.601509+0000","last_clean":"2023-09-23T20:39:00.601509+0000","last_became_active":"2023-09-23T20:37:17.282316+0000","last_became_peered":"2023-09-23T20:37:17.282316+0000","last_unstale":"2023-09-23T20:39:00.601509+0000","last_undegraded":"2023-09-23T20:39:00.601509+0000","last_fullsized":"2023-09-23T20:39:00.601509+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:43.578666+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:43.578666+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:53:37.101450+0000","scrub_duration":0.0021890379999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":450,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-23T20:39:10.650849+0000","last_change":"2023-09-23T20:32:33.242943+0000","last_active":"2023-09-23T20:39:10.650849+0000","last_peered":"2023-09-23T20:39:10.650849+0000","last_clean":"2023-09-23T20:39:10.650849+0000","last_became_active":"2023-09-23T20:32:33.225468+0000","last_became_peered":"2023-09-23T20:32:33.225468+0000","last_unstale":"2023-09-23T20:39:10.650849+0000","last_undegraded":"2023-09-23T20:39:10.650849+0000","last_fullsized":"2023-09-23T20:39:10.650849+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:40:47.867573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":158,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-23T20:39:10.662321+0000","last_change":"2023-09-23T20:37:43.571836+0000","last_active":"2023-09-23T20:39:10.662321+0000","last_peered":"2023-09-23T20:39:10.662321+0000","last_clean":"2023-09-23T20:39:10.662321+0000","last_became_active":"2023-09-23T20:37:43.571511+0000","last_became_peered":"2023-09-23T20:37:43.571511+0000","last_unstale":"2023-09-23T20:39:10.662321+0000","last_undegraded":"2023-09-23T20:39:10.662321+0000","last_fullsized":"2023-09-23T20:39:10.662321+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:59:25.507917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.19","version":"424'7224","reported_seq":11120,"reported_epoch":424,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T20:39:17.944024+0000","last_change":"2023-09-23T20:38:36.375475+0000","last_active":"2023-09-23T20:39:17.944024+0000","last_peered":"2023-09-23T20:39:17.944024+0000","last_clean":"2023-09-23T20:37:13.347998+0000","last_became_active":"2023-09-23T20:38:35.428186+0000","last_became_peered":"2023-09-23T20:38:35.428186+0000","last_unstale":"2023-09-23T20:39:17.944024+0000","last_undegraded":"2023-09-23T20:38:32.857390+0000","last_fullsized":"2023-09-23T20:39:17.944024+0000","mapping_epoch":410,"log_start":"373'1316","ondisk_log_start":"373'1316","created":410,"last_epoch_clean":370,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":5908,"log_dups_size":0,"ondisk_log_size":5908,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112787456,"num_objects":1721,"num_object_clones":0,"num_object_copies":5163,"num_objects_missing_on_primary":323,"num_objects_missing":323,"num_objects_degraded":1894,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1721,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3910,"num_write_kb":180192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1096,"num_bytes_recovered":71761920,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,3,4","objects":156},{"shards":"0,3,6","objects":-6},{"shards":"0,5,6","objects":1571}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.10","version":"90'161","reported_seq":686,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-23T20:39:10.650482+0000","last_change":"2023-09-23T20:38:53.672030+0000","last_active":"2023-09-23T20:39:10.650482+0000","last_peered":"2023-09-23T20:39:10.650482+0000","last_clean":"2023-09-23T20:39:10.650482+0000","last_became_active":"2023-09-23T20:38:05.589703+0000","last_became_peered":"2023-09-23T20:38:05.589703+0000","last_unstale":"2023-09-23T20:39:10.650482+0000","last_undegraded":"2023-09-23T20:39:10.650482+0000","last_fullsized":"2023-09-23T20:39:10.650482+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:49.726139+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:49.726139+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0096333110000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":390,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-09-23T20:39:00.601585+0000","last_change":"2023-09-23T20:25:02.953993+0000","last_active":"2023-09-23T20:39:00.601585+0000","last_peered":"2023-09-23T20:39:00.601585+0000","last_clean":"2023-09-23T20:39:00.601585+0000","last_became_active":"2023-09-23T20:25:02.953672+0000","last_became_peered":"2023-09-23T20:25:02.953672+0000","last_unstale":"2023-09-23T20:39:00.601585+0000","last_undegraded":"2023-09-23T20:39:00.601585+0000","last_fullsized":"2023-09-23T20:39:00.601585+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:26:08.603577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":212,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-23T20:39:10.652626+0000","last_change":"2023-09-23T20:38:02.712917+0000","last_active":"2023-09-23T20:39:10.652626+0000","last_peered":"2023-09-23T20:39:10.652626+0000","last_clean":"2023-09-23T20:39:10.652626+0000","last_became_active":"2023-09-23T20:38:02.658528+0000","last_became_peered":"2023-09-23T20:38:02.658528+0000","last_unstale":"2023-09-23T20:39:10.652626+0000","last_undegraded":"2023-09-23T20:39:10.652626+0000","last_fullsized":"2023-09-23T20:39:10.652626+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:42:32.680506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.13","version":"90'161","reported_seq":709,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-23T20:39:10.654030+0000","last_change":"2023-09-23T20:35:02.238923+0000","last_active":"2023-09-23T20:39:10.654030+0000","last_peered":"2023-09-23T20:39:10.654030+0000","last_clean":"2023-09-23T20:39:10.654030+0000","last_became_active":"2023-09-23T20:35:02.238293+0000","last_became_peered":"2023-09-23T20:35:02.238293+0000","last_unstale":"2023-09-23T20:39:10.654030+0000","last_undegraded":"2023-09-23T20:39:10.654030+0000","last_fullsized":"2023-09-23T20:39:10.654030+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:42.544308+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:42.544308+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:51:22.281932+0000","scrub_duration":0.0029958179999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":427,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-23T20:39:10.650620+0000","last_change":"2023-09-23T20:38:02.539774+0000","last_active":"2023-09-23T20:39:10.650620+0000","last_peered":"2023-09-23T20:39:10.650620+0000","last_clean":"2023-09-23T20:39:10.650620+0000","last_became_active":"2023-09-23T20:38:02.539436+0000","last_became_peered":"2023-09-23T20:38:02.539436+0000","last_unstale":"2023-09-23T20:39:10.650620+0000","last_undegraded":"2023-09-23T20:39:10.650620+0000","last_fullsized":"2023-09-23T20:39:10.650620+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:50:57.375803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":399,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-09-23T20:39:00.603175+0000","last_change":"2023-09-23T20:25:02.957010+0000","last_active":"2023-09-23T20:39:00.603175+0000","last_peered":"2023-09-23T20:39:00.603175+0000","last_clean":"2023-09-23T20:39:00.603175+0000","last_became_active":"2023-09-23T20:25:02.956267+0000","last_became_peered":"2023-09-23T20:25:02.956267+0000","last_unstale":"2023-09-23T20:39:00.603175+0000","last_undegraded":"2023-09-23T20:39:00.603175+0000","last_fullsized":"2023-09-23T20:39:00.603175+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:10:55.299422+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"90'161","reported_seq":518,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-23T20:39:10.652210+0000","last_change":"2023-09-23T20:38:05.606521+0000","last_active":"2023-09-23T20:39:10.652210+0000","last_peered":"2023-09-23T20:39:10.652210+0000","last_clean":"2023-09-23T20:39:10.652210+0000","last_became_active":"2023-09-23T20:38:05.606164+0000","last_became_peered":"2023-09-23T20:38:05.606164+0000","last_unstale":"2023-09-23T20:39:10.652210+0000","last_undegraded":"2023-09-23T20:39:10.652210+0000","last_fullsized":"2023-09-23T20:39:10.652210+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:53.706934+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:53.706934+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":40,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-23T20:39:10.647548+0000","last_change":"2023-09-23T20:38:02.626973+0000","last_active":"2023-09-23T20:39:10.647548+0000","last_peered":"2023-09-23T20:39:10.647548+0000","last_clean":"2023-09-23T20:39:10.647548+0000","last_became_active":"2023-09-23T20:38:02.545190+0000","last_became_peered":"2023-09-23T20:38:02.545190+0000","last_unstale":"2023-09-23T20:39:10.647548+0000","last_undegraded":"2023-09-23T20:39:10.647548+0000","last_fullsized":"2023-09-23T20:39:10.647548+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:38:12.102685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":216,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-23T20:39:10.663425+0000","last_change":"2023-09-23T20:37:43.571083+0000","last_active":"2023-09-23T20:39:10.663425+0000","last_peered":"2023-09-23T20:39:10.663425+0000","last_clean":"2023-09-23T20:39:10.663425+0000","last_became_active":"2023-09-23T20:37:43.570306+0000","last_became_peered":"2023-09-23T20:37:43.570306+0000","last_unstale":"2023-09-23T20:39:10.663425+0000","last_undegraded":"2023-09-23T20:39:10.663425+0000","last_fullsized":"2023-09-23T20:39:10.663425+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:42:02.396718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.4","version":"423'7314","reported_seq":8898,"reported_epoch":423,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:39:15.283796+0000","last_change":"2023-09-23T20:38:39.260380+0000","last_active":"2023-09-23T20:39:15.283796+0000","last_peered":"2023-09-23T20:39:15.283796+0000","last_clean":"2023-09-23T20:37:15.925522+0000","last_became_active":"2023-09-23T20:38:39.251413+0000","last_became_peered":"2023-09-23T20:38:39.251413+0000","last_unstale":"2023-09-23T20:39:15.283796+0000","last_undegraded":"2023-09-23T20:38:38.457471+0000","last_fullsized":"2023-09-23T20:38:38.449461+0000","mapping_epoch":416,"log_start":"373'2080","ondisk_log_start":"373'2080","created":369,"last_epoch_clean":370,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":5234,"log_dups_size":0,"ondisk_log_size":5234,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":117047296,"num_objects":1786,"num_object_clones":0,"num_object_copies":5358,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":771,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1786,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3907,"num_write_kb":182176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":377,"num_bytes_recovered":24707072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[4,2],"avail_no_missing":["4","2","6"],"object_location_counts":[{"shards":"2,3,4","objects":177},{"shards":"2,4","objects":1609}],"blocked_by":[],"up_primary":5,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.d","version":"90'161","reported_seq":716,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-23T20:39:10.651650+0000","last_change":"2023-09-23T20:34:59.496438+0000","last_active":"2023-09-23T20:39:10.651650+0000","last_peered":"2023-09-23T20:39:10.651650+0000","last_clean":"2023-09-23T20:39:10.651650+0000","last_became_active":"2023-09-23T20:33:38.970527+0000","last_became_peered":"2023-09-23T20:33:38.970527+0000","last_unstale":"2023-09-23T20:39:10.651650+0000","last_undegraded":"2023-09-23T20:39:10.651650+0000","last_fullsized":"2023-09-23T20:39:10.651650+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:34:59.496375+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:34:59.496375+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:46:18.244734+0000","scrub_duration":0.0017843189999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":427,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-23T20:39:10.651567+0000","last_change":"2023-09-23T20:32:33.283752+0000","last_active":"2023-09-23T20:39:10.651567+0000","last_peered":"2023-09-23T20:39:10.651567+0000","last_clean":"2023-09-23T20:39:10.651567+0000","last_became_active":"2023-09-23T20:32:33.230503+0000","last_became_peered":"2023-09-23T20:32:33.230503+0000","last_unstale":"2023-09-23T20:39:10.651567+0000","last_undegraded":"2023-09-23T20:39:10.651567+0000","last_fullsized":"2023-09-23T20:39:10.651567+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:29.084410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":269,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-23T20:39:10.650651+0000","last_change":"2023-09-23T20:38:02.539067+0000","last_active":"2023-09-23T20:39:10.650651+0000","last_peered":"2023-09-23T20:39:10.650651+0000","last_clean":"2023-09-23T20:39:10.650651+0000","last_became_active":"2023-09-23T20:38:02.538745+0000","last_became_peered":"2023-09-23T20:38:02.538745+0000","last_unstale":"2023-09-23T20:39:10.650651+0000","last_undegraded":"2023-09-23T20:39:10.650651+0000","last_fullsized":"2023-09-23T20:39:10.650651+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:36:31.615062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.5","version":"424'7179","reported_seq":10722,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-09-23T20:39:17.809066+0000","last_change":"2023-09-23T20:38:36.632834+0000","last_active":"2023-09-23T20:39:17.809066+0000","last_peered":"2023-09-23T20:39:17.809066+0000","last_clean":"2023-09-23T20:39:17.809066+0000","last_became_active":"2023-09-23T20:38:28.423549+0000","last_became_peered":"2023-09-23T20:38:28.423549+0000","last_unstale":"2023-09-23T20:39:17.809066+0000","last_undegraded":"2023-09-23T20:39:17.809066+0000","last_fullsized":"2023-09-23T20:39:17.809066+0000","mapping_epoch":411,"log_start":"373'1839","ondisk_log_start":"373'1839","created":369,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":5340,"log_dups_size":0,"ondisk_log_size":5340,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109412352,"num_objects":1669,"num_object_clones":0,"num_object_copies":5007,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1669,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3883,"num_write_kb":177696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1036,"num_bytes_recovered":55345152,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.c","version":"90'161","reported_seq":670,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-23T20:39:10.662551+0000","last_change":"2023-09-23T20:38:28.442536+0000","last_active":"2023-09-23T20:39:10.662551+0000","last_peered":"2023-09-23T20:39:10.662551+0000","last_clean":"2023-09-23T20:39:10.662551+0000","last_became_active":"2023-09-23T20:38:28.442229+0000","last_became_peered":"2023-09-23T20:38:28.442229+0000","last_unstale":"2023-09-23T20:39:10.662551+0000","last_undegraded":"2023-09-23T20:39:10.662551+0000","last_fullsized":"2023-09-23T20:39:10.662551+0000","mapping_epoch":411,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":412,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:05.485751+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:05.485751+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.002180043,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163840,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":462,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-23T20:39:10.651674+0000","last_change":"2023-09-23T20:34:59.987981+0000","last_active":"2023-09-23T20:39:10.651674+0000","last_peered":"2023-09-23T20:39:10.651674+0000","last_clean":"2023-09-23T20:39:10.651674+0000","last_became_active":"2023-09-23T20:34:59.985368+0000","last_became_peered":"2023-09-23T20:34:59.985368+0000","last_unstale":"2023-09-23T20:39:10.651674+0000","last_undegraded":"2023-09-23T20:39:10.651674+0000","last_fullsized":"2023-09-23T20:39:10.651674+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:12:43.082493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":120,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-09-23T20:39:00.601937+0000","last_change":"2023-09-23T20:32:33.232142+0000","last_active":"2023-09-23T20:39:00.601937+0000","last_peered":"2023-09-23T20:39:00.601937+0000","last_clean":"2023-09-23T20:39:00.601937+0000","last_became_active":"2023-09-23T20:32:33.231683+0000","last_became_peered":"2023-09-23T20:32:33.231683+0000","last_unstale":"2023-09-23T20:39:00.601937+0000","last_undegraded":"2023-09-23T20:39:00.601937+0000","last_fullsized":"2023-09-23T20:39:00.601937+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:02:58.336348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"423'7145","reported_seq":7848,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-23T20:39:16.093543+0000","last_change":"2023-09-23T20:38:29.214109+0000","last_active":"2023-09-23T20:39:16.093543+0000","last_peered":"2023-09-23T20:39:16.093543+0000","last_clean":"2023-09-23T20:39:16.093543+0000","last_became_active":"2023-09-23T20:38:28.425504+0000","last_became_peered":"2023-09-23T20:38:28.425504+0000","last_unstale":"2023-09-23T20:39:16.093543+0000","last_undegraded":"2023-09-23T20:39:16.093543+0000","last_fullsized":"2023-09-23T20:39:16.093543+0000","mapping_epoch":411,"log_start":"373'1261","ondisk_log_start":"373'1261","created":369,"last_epoch_clean":412,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":5884,"log_dups_size":0,"ondisk_log_size":5884,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":110690304,"num_objects":1689,"num_object_clones":0,"num_object_copies":5067,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1689,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3865,"num_write_kb":177728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":305,"num_bytes_recovered":10420224,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"90'161","reported_seq":580,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-23T20:39:10.649517+0000","last_change":"2023-09-23T20:38:40.491902+0000","last_active":"2023-09-23T20:39:10.649517+0000","last_peered":"2023-09-23T20:39:10.649517+0000","last_clean":"2023-09-23T20:39:10.649517+0000","last_became_active":"2023-09-23T20:38:40.491563+0000","last_became_peered":"2023-09-23T20:38:40.491563+0000","last_unstale":"2023-09-23T20:39:10.649517+0000","last_undegraded":"2023-09-23T20:39:10.649517+0000","last_fullsized":"2023-09-23T20:39:10.649517+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:46.217489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:46.217489+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:39:25.008796+0000","scrub_duration":0.001456429,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":438,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-23T20:39:10.651739+0000","last_change":"2023-09-23T20:38:02.715585+0000","last_active":"2023-09-23T20:39:10.651739+0000","last_peered":"2023-09-23T20:39:10.651739+0000","last_clean":"2023-09-23T20:39:10.651739+0000","last_became_active":"2023-09-23T20:38:02.658973+0000","last_became_peered":"2023-09-23T20:38:02.658973+0000","last_unstale":"2023-09-23T20:39:10.651739+0000","last_undegraded":"2023-09-23T20:39:10.651739+0000","last_fullsized":"2023-09-23T20:39:10.651739+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:32:03.044825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":350,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":30,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-23T20:39:10.647661+0000","last_change":"2023-09-23T20:38:02.590613+0000","last_active":"2023-09-23T20:39:10.647661+0000","last_peered":"2023-09-23T20:39:10.647661+0000","last_clean":"2023-09-23T20:39:10.647661+0000","last_became_active":"2023-09-23T20:38:02.590284+0000","last_became_peered":"2023-09-23T20:38:02.590284+0000","last_unstale":"2023-09-23T20:39:10.647661+0000","last_undegraded":"2023-09-23T20:39:10.647661+0000","last_fullsized":"2023-09-23T20:39:10.647661+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:40:07.825054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"424'7210","reported_seq":7301,"reported_epoch":424,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T20:39:19.308594+0000","last_change":"2023-09-23T20:38:28.482576+0000","last_active":"2023-09-23T20:39:19.308594+0000","last_peered":"2023-09-23T20:39:19.308594+0000","last_clean":"2023-09-23T20:37:19.384160+0000","last_became_active":"2023-09-23T20:38:28.436345+0000","last_became_peered":"2023-09-23T20:38:28.436345+0000","last_unstale":"2023-09-23T20:39:19.308594+0000","last_undegraded":"2023-09-23T20:38:28.418749+0000","last_fullsized":"2023-09-23T20:39:19.308594+0000","mapping_epoch":411,"log_start":"373'1658","ondisk_log_start":"373'1658","created":369,"last_epoch_clean":370,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":5552,"log_dups_size":0,"ondisk_log_size":5552,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":107282432,"num_objects":1637,"num_object_clones":0,"num_object_copies":4911,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":309,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1637,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3920,"num_write_kb":177824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":["1","3"],"object_location_counts":[{"shards":"1,2,3","objects":148},{"shards":"1,3,6","objects":1489}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.e","version":"90'161","reported_seq":523,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-09-23T20:39:00.603227+0000","last_change":"2023-09-23T20:36:00.322453+0000","last_active":"2023-09-23T20:39:00.603227+0000","last_peered":"2023-09-23T20:39:00.603227+0000","last_clean":"2023-09-23T20:39:00.603227+0000","last_became_active":"2023-09-23T20:35:02.233536+0000","last_became_peered":"2023-09-23T20:35:02.233536+0000","last_unstale":"2023-09-23T20:39:00.603227+0000","last_undegraded":"2023-09-23T20:39:00.603227+0000","last_fullsized":"2023-09-23T20:39:00.603227+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:00.322381+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:00.322381+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:02:15.489069+0000","scrub_duration":0.0017321859999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":17,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-09-23T20:39:00.624996+0000","last_change":"2023-09-23T20:38:37.456683+0000","last_active":"2023-09-23T20:39:00.624996+0000","last_peered":"2023-09-23T20:39:00.624996+0000","last_clean":"2023-09-23T20:39:00.624996+0000","last_became_active":"2023-09-23T20:38:37.456289+0000","last_became_peered":"2023-09-23T20:38:37.456289+0000","last_unstale":"2023-09-23T20:39:00.624996+0000","last_undegraded":"2023-09-23T20:39:00.624996+0000","last_fullsized":"2023-09-23T20:39:00.624996+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:03:10.507333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":408,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-09-23T20:39:00.602737+0000","last_change":"2023-09-23T20:32:33.236122+0000","last_active":"2023-09-23T20:39:00.602737+0000","last_peered":"2023-09-23T20:39:00.602737+0000","last_clean":"2023-09-23T20:39:00.602737+0000","last_became_active":"2023-09-23T20:32:33.235384+0000","last_became_peered":"2023-09-23T20:32:33.235384+0000","last_unstale":"2023-09-23T20:39:00.602737+0000","last_undegraded":"2023-09-23T20:39:00.602737+0000","last_fullsized":"2023-09-23T20:39:00.602737+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:56.843733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"424'7092","reported_seq":13572,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-09-23T20:39:18.915970+0000","last_change":"2023-09-23T20:38:36.017904+0000","last_active":"2023-09-23T20:39:18.915970+0000","last_peered":"2023-09-23T20:39:18.915970+0000","last_clean":"2023-09-23T20:39:18.915970+0000","last_became_active":"2023-09-23T20:38:27.481141+0000","last_became_peered":"2023-09-23T20:38:27.481141+0000","last_unstale":"2023-09-23T20:39:18.915970+0000","last_undegraded":"2023-09-23T20:39:18.915970+0000","last_fullsized":"2023-09-23T20:39:18.915970+0000","mapping_epoch":400,"log_start":"373'2187","ondisk_log_start":"373'2187","created":369,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":4905,"log_dups_size":0,"ondisk_log_size":4905,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":105512960,"num_objects":1610,"num_object_clones":0,"num_object_copies":4830,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1610,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3845,"num_write_kb":174560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2565,"num_bytes_recovered":168099840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":829,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-23T20:39:10.650465+0000","last_change":"2023-09-23T20:35:02.523308+0000","last_active":"2023-09-23T20:39:10.650465+0000","last_peered":"2023-09-23T20:39:10.650465+0000","last_clean":"2023-09-23T20:39:10.650465+0000","last_became_active":"2023-09-23T20:33:38.979989+0000","last_became_peered":"2023-09-23T20:33:38.979989+0000","last_unstale":"2023-09-23T20:39:10.650465+0000","last_undegraded":"2023-09-23T20:39:10.650465+0000","last_fullsized":"2023-09-23T20:39:10.650465+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:35:02.523203+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:35:02.523203+0000","last_clean_scrub_stamp":"2023-09-23T20:35:02.523203+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:02:41.472958+0000","scrub_duration":0.0085927989999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":242,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-23T20:39:10.662535+0000","last_change":"2023-09-23T20:37:43.583731+0000","last_active":"2023-09-23T20:39:10.662535+0000","last_peered":"2023-09-23T20:39:10.662535+0000","last_clean":"2023-09-23T20:39:10.662535+0000","last_became_active":"2023-09-23T20:37:43.583066+0000","last_became_peered":"2023-09-23T20:37:43.583066+0000","last_unstale":"2023-09-23T20:39:10.662535+0000","last_undegraded":"2023-09-23T20:39:10.662535+0000","last_fullsized":"2023-09-23T20:39:10.662535+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:03:45.211900+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":140,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-09-23T20:39:00.603048+0000","last_change":"2023-09-23T20:38:02.579814+0000","last_active":"2023-09-23T20:39:00.603048+0000","last_peered":"2023-09-23T20:39:00.603048+0000","last_clean":"2023-09-23T20:39:00.603048+0000","last_became_active":"2023-09-23T20:38:02.579253+0000","last_became_peered":"2023-09-23T20:38:02.579253+0000","last_unstale":"2023-09-23T20:39:00.603048+0000","last_undegraded":"2023-09-23T20:39:00.603048+0000","last_fullsized":"2023-09-23T20:39:00.603048+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:05:07.298198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"424'7230","reported_seq":7305,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-09-23T20:39:19.272870+0000","last_change":"2023-09-23T20:38:27.447178+0000","last_active":"2023-09-23T20:39:19.272870+0000","last_peered":"2023-09-23T20:39:19.272870+0000","last_clean":"2023-09-23T20:39:19.272870+0000","last_became_active":"2023-09-23T20:38:27.437204+0000","last_became_peered":"2023-09-23T20:38:27.437204+0000","last_unstale":"2023-09-23T20:39:19.272870+0000","last_undegraded":"2023-09-23T20:39:19.272870+0000","last_fullsized":"2023-09-23T20:39:19.272870+0000","mapping_epoch":369,"log_start":"373'1694","ondisk_log_start":"373'1694","created":369,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":5536,"log_dups_size":0,"ondisk_log_size":5536,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":108953600,"num_objects":1662,"num_object_clones":0,"num_object_copies":4986,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1662,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3920,"num_write_kb":178656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":514,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-23T20:39:10.647230+0000","last_change":"2023-09-23T20:38:05.632243+0000","last_active":"2023-09-23T20:39:10.647230+0000","last_peered":"2023-09-23T20:39:10.647230+0000","last_clean":"2023-09-23T20:39:10.647230+0000","last_became_active":"2023-09-23T20:38:05.631907+0000","last_became_peered":"2023-09-23T20:38:05.631907+0000","last_unstale":"2023-09-23T20:39:10.647230+0000","last_undegraded":"2023-09-23T20:39:10.647230+0000","last_fullsized":"2023-09-23T20:39:10.647230+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:35:00.496197+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:35:00.496197+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":30,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-23T20:39:10.647651+0000","last_change":"2023-09-23T20:38:02.578703+0000","last_active":"2023-09-23T20:39:10.647651+0000","last_peered":"2023-09-23T20:39:10.647651+0000","last_clean":"2023-09-23T20:39:10.647651+0000","last_became_active":"2023-09-23T20:38:02.578414+0000","last_became_peered":"2023-09-23T20:38:02.578414+0000","last_unstale":"2023-09-23T20:39:10.647651+0000","last_undegraded":"2023-09-23T20:39:10.647651+0000","last_fullsized":"2023-09-23T20:39:10.647651+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:43:59.779295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":89,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-09-23T20:39:00.602489+0000","last_change":"2023-09-23T20:37:31.296606+0000","last_active":"2023-09-23T20:39:00.602489+0000","last_peered":"2023-09-23T20:39:00.602489+0000","last_clean":"2023-09-23T20:39:00.602489+0000","last_became_active":"2023-09-23T20:37:31.295924+0000","last_became_peered":"2023-09-23T20:37:31.295924+0000","last_unstale":"2023-09-23T20:39:00.602489+0000","last_undegraded":"2023-09-23T20:39:00.602489+0000","last_fullsized":"2023-09-23T20:39:00.602489+0000","mapping_epoch":391,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":392,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:27:50.948510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"423'7243","reported_seq":7374,"reported_epoch":423,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:39:15.491622+0000","last_change":"2023-09-23T20:38:39.273152+0000","last_active":"2023-09-23T20:39:15.491622+0000","last_peered":"2023-09-23T20:39:15.491622+0000","last_clean":"2023-09-23T20:37:29.399979+0000","last_became_active":"2023-09-23T20:38:39.253967+0000","last_became_peered":"2023-09-23T20:38:39.253967+0000","last_unstale":"2023-09-23T20:39:15.491622+0000","last_undegraded":"2023-09-23T20:38:38.450129+0000","last_fullsized":"2023-09-23T20:38:38.446971+0000","mapping_epoch":416,"log_start":"373'1996","ondisk_log_start":"373'1996","created":369,"last_epoch_clean":370,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":5247,"log_dups_size":0,"ondisk_log_size":5247,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112787456,"num_objects":1721,"num_object_clones":0,"num_object_copies":5163,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":723,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1721,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3893,"num_write_kb":179648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":950272,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0],"avail_no_missing":["4","0"],"object_location_counts":[{"shards":"0,3,4","objects":190},{"shards":"0,4","objects":1531}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"90'161","reported_seq":708,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-23T20:39:10.654868+0000","last_change":"2023-09-23T20:37:17.282621+0000","last_active":"2023-09-23T20:39:10.654868+0000","last_peered":"2023-09-23T20:39:10.654868+0000","last_clean":"2023-09-23T20:39:10.654868+0000","last_became_active":"2023-09-23T20:37:17.282243+0000","last_became_peered":"2023-09-23T20:37:17.282243+0000","last_unstale":"2023-09-23T20:39:10.654868+0000","last_undegraded":"2023-09-23T20:39:10.654868+0000","last_fullsized":"2023-09-23T20:39:10.654868+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:05.595042+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:05.595042+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:42:06.244026+0000","scrub_duration":0.007271675,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":79,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-09-23T20:39:00.601760+0000","last_change":"2023-09-23T20:34:59.983122+0000","last_active":"2023-09-23T20:39:00.601760+0000","last_peered":"2023-09-23T20:39:00.601760+0000","last_clean":"2023-09-23T20:39:00.601760+0000","last_became_active":"2023-09-23T20:34:59.982809+0000","last_became_peered":"2023-09-23T20:34:59.982809+0000","last_unstale":"2023-09-23T20:39:00.601760+0000","last_undegraded":"2023-09-23T20:39:00.601760+0000","last_fullsized":"2023-09-23T20:39:00.601760+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:49:40.839723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":140,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-09-23T20:39:00.603367+0000","last_change":"2023-09-23T20:38:02.543866+0000","last_active":"2023-09-23T20:39:00.603367+0000","last_peered":"2023-09-23T20:39:00.603367+0000","last_clean":"2023-09-23T20:39:00.603367+0000","last_became_active":"2023-09-23T20:38:02.543563+0000","last_became_peered":"2023-09-23T20:38:02.543563+0000","last_unstale":"2023-09-23T20:39:00.603367+0000","last_undegraded":"2023-09-23T20:39:00.603367+0000","last_fullsized":"2023-09-23T20:39:00.603367+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:33.050536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"424'7219","reported_seq":9592,"reported_epoch":424,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T20:39:19.462153+0000","last_change":"2023-09-23T20:38:28.540352+0000","last_active":"2023-09-23T20:39:19.462153+0000","last_peered":"2023-09-23T20:39:19.462153+0000","last_clean":"2023-09-23T20:37:13.297905+0000","last_became_active":"2023-09-23T20:38:28.499228+0000","last_became_peered":"2023-09-23T20:38:28.499228+0000","last_unstale":"2023-09-23T20:39:19.462153+0000","last_undegraded":"2023-09-23T20:38:28.490806+0000","last_fullsized":"2023-09-23T20:39:19.462153+0000","mapping_epoch":411,"log_start":"373'1680","ondisk_log_start":"373'1680","created":369,"last_epoch_clean":370,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":5539,"log_dups_size":0,"ondisk_log_size":5539,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":110133248,"num_objects":1680,"num_object_clones":0,"num_object_copies":5040,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2183,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1680,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3912,"num_write_kb":178976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":783,"num_bytes_recovered":51281920,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":["1","5"],"object_location_counts":[{"shards":"1,2,3","objects":156},{"shards":"1,2,6","objects":18},{"shards":"1,5,6","objects":1506}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":503,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-09-23T20:39:00.625112+0000","last_change":"2023-09-23T20:38:55.432995+0000","last_active":"2023-09-23T20:39:00.625112+0000","last_peered":"2023-09-23T20:39:00.625112+0000","last_clean":"2023-09-23T20:39:00.625112+0000","last_became_active":"2023-09-23T20:38:40.494049+0000","last_became_peered":"2023-09-23T20:38:40.494049+0000","last_unstale":"2023-09-23T20:39:00.625112+0000","last_undegraded":"2023-09-23T20:39:00.625112+0000","last_fullsized":"2023-09-23T20:39:00.625112+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:34:59.481446+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:34:59.481446+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26834,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":216,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-23T20:39:10.661333+0000","last_change":"2023-09-23T20:37:43.570610+0000","last_active":"2023-09-23T20:39:10.661333+0000","last_peered":"2023-09-23T20:39:10.661333+0000","last_clean":"2023-09-23T20:39:10.661333+0000","last_became_active":"2023-09-23T20:37:43.570216+0000","last_became_peered":"2023-09-23T20:37:43.570216+0000","last_unstale":"2023-09-23T20:39:10.661333+0000","last_undegraded":"2023-09-23T20:39:10.661333+0000","last_fullsized":"2023-09-23T20:39:10.661333+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:39:26.555329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":247,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-23T20:39:10.652603+0000","last_change":"2023-09-23T20:32:33.230582+0000","last_active":"2023-09-23T20:39:10.652603+0000","last_peered":"2023-09-23T20:39:10.652603+0000","last_clean":"2023-09-23T20:39:10.652603+0000","last_became_active":"2023-09-23T20:32:33.230257+0000","last_became_peered":"2023-09-23T20:32:33.230257+0000","last_unstale":"2023-09-23T20:39:10.652603+0000","last_undegraded":"2023-09-23T20:39:10.652603+0000","last_fullsized":"2023-09-23T20:39:10.652603+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:11:45.333096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"423'7664","reported_seq":7728,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-23T20:39:16.104660+0000","last_change":"2023-09-23T20:36:11.310149+0000","last_active":"2023-09-23T20:39:16.104660+0000","last_peered":"2023-09-23T20:39:16.104660+0000","last_clean":"2023-09-23T20:39:16.104660+0000","last_became_active":"2023-09-23T20:36:11.309844+0000","last_became_peered":"2023-09-23T20:36:11.309844+0000","last_unstale":"2023-09-23T20:39:16.104660+0000","last_undegraded":"2023-09-23T20:39:16.104660+0000","last_fullsized":"2023-09-23T20:39:16.104660+0000","mapping_epoch":369,"log_start":"373'1819","ondisk_log_start":"373'1819","created":369,"last_epoch_clean":370,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":5845,"log_dups_size":0,"ondisk_log_size":5845,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:14:57.442108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":222298112,"num_objects":3392,"num_object_clones":0,"num_object_copies":10176,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3392,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7664,"num_write_kb":353792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":603,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-23T20:39:10.649384+0000","last_change":"2023-09-23T20:38:05.622597+0000","last_active":"2023-09-23T20:39:10.649384+0000","last_peered":"2023-09-23T20:39:10.649384+0000","last_clean":"2023-09-23T20:39:10.649384+0000","last_became_active":"2023-09-23T20:38:05.622261+0000","last_became_peered":"2023-09-23T20:38:05.622261+0000","last_unstale":"2023-09-23T20:39:10.649384+0000","last_undegraded":"2023-09-23T20:39:10.649384+0000","last_fullsized":"2023-09-23T20:39:10.649384+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:35:04.101972+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:35:04.101972+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"86'119","reported_seq":731,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-23T20:39:10.651710+0000","last_change":"2023-09-23T20:29:45.521257+0000","last_active":"2023-09-23T20:39:10.651710+0000","last_peered":"2023-09-23T20:39:10.651710+0000","last_clean":"2023-09-23T20:39:10.651710+0000","last_became_active":"2023-09-23T20:29:45.520546+0000","last_became_peered":"2023-09-23T20:29:45.520546+0000","last_unstale":"2023-09-23T20:39:10.651710+0000","last_undegraded":"2023-09-23T20:39:10.651710+0000","last_fullsized":"2023-09-23T20:39:10.651710+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":119,"log_dups_size":0,"ondisk_log_size":119,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:58:56.085777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032705499999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":523,"num_read_kb":3503,"num_write":136,"num_write_kb":3128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":247,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-23T20:39:10.663375+0000","last_change":"2023-09-23T20:38:02.597306+0000","last_active":"2023-09-23T20:39:10.663375+0000","last_peered":"2023-09-23T20:39:10.663375+0000","last_clean":"2023-09-23T20:39:10.663375+0000","last_became_active":"2023-09-23T20:38:02.596966+0000","last_became_peered":"2023-09-23T20:38:02.596966+0000","last_unstale":"2023-09-23T20:39:10.663375+0000","last_undegraded":"2023-09-23T20:39:10.663375+0000","last_fullsized":"2023-09-23T20:39:10.663375+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:30:41.962907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":369,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-23T20:39:10.651376+0000","last_change":"2023-09-23T20:37:14.359424+0000","last_active":"2023-09-23T20:39:10.651376+0000","last_peered":"2023-09-23T20:39:10.651376+0000","last_clean":"2023-09-23T20:39:10.651376+0000","last_became_active":"2023-09-23T20:37:14.359059+0000","last_became_peered":"2023-09-23T20:37:14.359059+0000","last_unstale":"2023-09-23T20:39:10.651376+0000","last_undegraded":"2023-09-23T20:39:10.651376+0000","last_fullsized":"2023-09-23T20:39:10.651376+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:07:10.779153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.8","version":"424'7219","reported_seq":13507,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-09-23T20:39:18.742059+0000","last_change":"2023-09-23T20:38:52.185962+0000","last_active":"2023-09-23T20:39:18.742059+0000","last_peered":"2023-09-23T20:39:18.742059+0000","last_clean":"2023-09-23T20:39:18.742059+0000","last_became_active":"2023-09-23T20:38:27.525964+0000","last_became_peered":"2023-09-23T20:38:27.525964+0000","last_unstale":"2023-09-23T20:39:18.742059+0000","last_undegraded":"2023-09-23T20:39:18.742059+0000","last_fullsized":"2023-09-23T20:39:18.742059+0000","mapping_epoch":400,"log_start":"373'2334","ondisk_log_start":"373'2334","created":369,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":4885,"log_dups_size":0,"ondisk_log_size":4885,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":110329856,"num_objects":1683,"num_object_clones":0,"num_object_copies":5049,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1683,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3910,"num_write_kb":179008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2588,"num_bytes_recovered":169607168,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":485,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-09-23T20:39:00.601818+0000","last_change":"2023-09-23T20:37:17.290846+0000","last_active":"2023-09-23T20:39:00.601818+0000","last_peered":"2023-09-23T20:39:00.601818+0000","last_clean":"2023-09-23T20:39:00.601818+0000","last_became_active":"2023-09-23T20:37:17.290470+0000","last_became_peered":"2023-09-23T20:37:17.290470+0000","last_unstale":"2023-09-23T20:39:00.601818+0000","last_undegraded":"2023-09-23T20:39:00.601818+0000","last_fullsized":"2023-09-23T20:39:00.601818+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:45.785450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:45.785450+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"86'106","reported_seq":1142,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-09-23T20:39:00.625242+0000","last_change":"2023-09-23T20:38:37.476512+0000","last_active":"2023-09-23T20:39:00.625242+0000","last_peered":"2023-09-23T20:39:00.625242+0000","last_clean":"2023-09-23T20:39:00.625242+0000","last_became_active":"2023-09-23T20:38:37.475980+0000","last_became_peered":"2023-09-23T20:38:37.475980+0000","last_unstale":"2023-09-23T20:39:00.625242+0000","last_undegraded":"2023-09-23T20:39:00.625242+0000","last_fullsized":"2023-09-23T20:39:00.625242+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":106,"log_dups_size":0,"ondisk_log_size":106,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:58:45.508831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00099302599999999993,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1094,"num_read_kb":1678,"num_write":100,"num_write_kb":404,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":140,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-09-23T20:39:00.602533+0000","last_change":"2023-09-23T20:38:02.552008+0000","last_active":"2023-09-23T20:39:00.602533+0000","last_peered":"2023-09-23T20:39:00.602533+0000","last_clean":"2023-09-23T20:39:00.602533+0000","last_became_active":"2023-09-23T20:38:02.551684+0000","last_became_peered":"2023-09-23T20:38:02.551684+0000","last_unstale":"2023-09-23T20:39:00.602533+0000","last_undegraded":"2023-09-23T20:39:00.602533+0000","last_fullsized":"2023-09-23T20:39:00.602533+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:40:37.147538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":384,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-23T20:39:10.651613+0000","last_change":"2023-09-23T20:29:45.533406+0000","last_active":"2023-09-23T20:39:10.651613+0000","last_peered":"2023-09-23T20:39:10.651613+0000","last_clean":"2023-09-23T20:39:10.651613+0000","last_became_active":"2023-09-23T20:29:45.526001+0000","last_became_peered":"2023-09-23T20:29:45.526001+0000","last_unstale":"2023-09-23T20:39:10.651613+0000","last_undegraded":"2023-09-23T20:39:10.651613+0000","last_fullsized":"2023-09-23T20:39:10.651613+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:43.787251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.b","version":"424'7789","reported_seq":7867,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-09-23T20:39:18.919633+0000","last_change":"2023-09-23T20:38:01.517845+0000","last_active":"2023-09-23T20:39:18.919633+0000","last_peered":"2023-09-23T20:39:18.919633+0000","last_clean":"2023-09-23T20:39:18.919633+0000","last_became_active":"2023-09-23T20:37:43.579403+0000","last_became_peered":"2023-09-23T20:37:43.579403+0000","last_unstale":"2023-09-23T20:39:18.919633+0000","last_undegraded":"2023-09-23T20:39:18.919633+0000","last_fullsized":"2023-09-23T20:39:18.919633+0000","mapping_epoch":399,"log_start":"375'3400","ondisk_log_start":"375'3400","created":369,"last_epoch_clean":403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":4389,"log_dups_size":0,"ondisk_log_size":4389,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:33:13.920936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":219348992,"num_objects":3347,"num_object_clones":0,"num_object_copies":10041,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3347,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7789,"num_write_kb":356352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":896,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-23T20:39:10.650652+0000","last_change":"2023-09-23T20:35:02.260708+0000","last_active":"2023-09-23T20:39:10.650652+0000","last_peered":"2023-09-23T20:39:10.650652+0000","last_clean":"2023-09-23T20:39:10.650652+0000","last_became_active":"2023-09-23T20:35:02.260348+0000","last_became_peered":"2023-09-23T20:35:02.260348+0000","last_unstale":"2023-09-23T20:39:10.650652+0000","last_undegraded":"2023-09-23T20:39:10.650652+0000","last_fullsized":"2023-09-23T20:39:10.650652+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:43.390908+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:43.390908+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:09:14.144070+0000","scrub_duration":0.0032721,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"86'130","reported_seq":739,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-23T20:39:10.647210+0000","last_change":"2023-09-23T20:25:39.637458+0000","last_active":"2023-09-23T20:39:10.647210+0000","last_peered":"2023-09-23T20:39:10.647210+0000","last_clean":"2023-09-23T20:39:10.647210+0000","last_became_active":"2023-09-23T20:25:39.636773+0000","last_became_peered":"2023-09-23T20:25:39.636773+0000","last_unstale":"2023-09-23T20:39:10.647210+0000","last_undegraded":"2023-09-23T20:39:10.647210+0000","last_fullsized":"2023-09-23T20:39:10.647210+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:00:32.810564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011300049999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":512,"num_read_kb":1820,"num_write":133,"num_write_kb":1347,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":392,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-23T20:39:10.651179+0000","last_change":"2023-09-23T20:38:02.591335+0000","last_active":"2023-09-23T20:39:10.651179+0000","last_peered":"2023-09-23T20:39:10.651179+0000","last_clean":"2023-09-23T20:39:10.651179+0000","last_became_active":"2023-09-23T20:38:02.591015+0000","last_became_peered":"2023-09-23T20:38:02.591015+0000","last_unstale":"2023-09-23T20:39:10.651179+0000","last_undegraded":"2023-09-23T20:39:10.651179+0000","last_fullsized":"2023-09-23T20:39:10.651179+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:55:32.272195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":375,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-23T20:39:10.652507+0000","last_change":"2023-09-23T20:37:14.360153+0000","last_active":"2023-09-23T20:39:10.652507+0000","last_peered":"2023-09-23T20:39:10.652507+0000","last_clean":"2023-09-23T20:39:10.652507+0000","last_became_active":"2023-09-23T20:37:14.359892+0000","last_became_peered":"2023-09-23T20:37:14.359892+0000","last_unstale":"2023-09-23T20:39:10.652507+0000","last_undegraded":"2023-09-23T20:39:10.652507+0000","last_fullsized":"2023-09-23T20:39:10.652507+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:59:37.031659+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"424'7209","reported_seq":11075,"reported_epoch":424,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T20:39:17.909050+0000","last_change":"2023-09-23T20:38:37.846664+0000","last_active":"2023-09-23T20:39:17.909050+0000","last_peered":"2023-09-23T20:39:17.909050+0000","last_clean":"2023-09-23T20:37:13.347998+0000","last_became_active":"2023-09-23T20:38:35.428647+0000","last_became_peered":"2023-09-23T20:38:35.428647+0000","last_unstale":"2023-09-23T20:39:17.909050+0000","last_undegraded":"2023-09-23T20:38:30.822487+0000","last_fullsized":"2023-09-23T20:39:17.909050+0000","mapping_epoch":411,"log_start":"373'1316","ondisk_log_start":"373'1316","created":369,"last_epoch_clean":370,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":5893,"log_dups_size":0,"ondisk_log_size":5893,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":113770496,"num_objects":1736,"num_object_clones":0,"num_object_copies":5208,"num_objects_missing_on_primary":334,"num_objects_missing":334,"num_objects_degraded":1890,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1736,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3895,"num_write_kb":180192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1080,"num_bytes_recovered":70713344,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,3,4","objects":166},{"shards":"0,3,6","objects":14},{"shards":"0,5,6","objects":1556}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"86'194","reported_seq":772,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-23T20:39:10.647977+0000","last_change":"2023-09-23T20:37:17.289352+0000","last_active":"2023-09-23T20:39:10.647977+0000","last_peered":"2023-09-23T20:39:10.647977+0000","last_clean":"2023-09-23T20:39:10.647977+0000","last_became_active":"2023-09-23T20:37:17.288695+0000","last_became_peered":"2023-09-23T20:37:17.288695+0000","last_unstale":"2023-09-23T20:39:10.647977+0000","last_undegraded":"2023-09-23T20:39:10.647977+0000","last_fullsized":"2023-09-23T20:39:10.647977+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":194,"log_dups_size":0,"ondisk_log_size":194,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:43:22.047407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00037995999999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1024,"num_read_kb":1200,"num_write":173,"num_write_kb":1276,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":886,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-23T20:39:10.650573+0000","last_change":"2023-09-23T20:38:05.602508+0000","last_active":"2023-09-23T20:39:10.650573+0000","last_peered":"2023-09-23T20:39:10.650573+0000","last_clean":"2023-09-23T20:39:10.650573+0000","last_became_active":"2023-09-23T20:38:05.602129+0000","last_became_peered":"2023-09-23T20:38:05.602129+0000","last_unstale":"2023-09-23T20:39:10.650573+0000","last_undegraded":"2023-09-23T20:39:10.650573+0000","last_fullsized":"2023-09-23T20:39:10.650573+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:45.343742+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:45.343742+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:48.910343+0000","scrub_duration":0.001475802,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":30,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-23T20:39:10.647697+0000","last_change":"2023-09-23T20:38:02.541452+0000","last_active":"2023-09-23T20:39:10.647697+0000","last_peered":"2023-09-23T20:39:10.647697+0000","last_clean":"2023-09-23T20:39:10.647697+0000","last_became_active":"2023-09-23T20:38:02.541112+0000","last_became_peered":"2023-09-23T20:38:02.541112+0000","last_unstale":"2023-09-23T20:39:10.647697+0000","last_undegraded":"2023-09-23T20:39:10.647697+0000","last_fullsized":"2023-09-23T20:39:10.647697+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:03:51.385905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,4],"acting":[6,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":30,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-23T20:39:10.649032+0000","last_change":"2023-09-23T20:38:02.573200+0000","last_active":"2023-09-23T20:39:10.649032+0000","last_peered":"2023-09-23T20:39:10.649032+0000","last_clean":"2023-09-23T20:39:10.649032+0000","last_became_active":"2023-09-23T20:38:02.572945+0000","last_became_peered":"2023-09-23T20:38:02.572945+0000","last_unstale":"2023-09-23T20:39:10.649032+0000","last_undegraded":"2023-09-23T20:39:10.649032+0000","last_fullsized":"2023-09-23T20:39:10.649032+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:57.246533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.a","version":"424'7993","reported_seq":16966,"reported_epoch":424,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T20:39:19.504988+0000","last_change":"2023-09-23T20:38:02.722059+0000","last_active":"2023-09-23T20:39:19.504988+0000","last_peered":"2023-09-23T20:39:19.504988+0000","last_clean":"2023-09-23T20:37:13.317597+0000","last_became_active":"2023-09-23T20:37:15.678492+0000","last_became_peered":"2023-09-23T20:37:15.678492+0000","last_unstale":"2023-09-23T20:39:19.504988+0000","last_undegraded":"2023-09-23T20:37:23.465606+0000","last_fullsized":"2023-09-23T20:37:15.349179+0000","mapping_epoch":380,"log_start":"374'3279","ondisk_log_start":"374'3279","created":369,"last_epoch_clean":370,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":4714,"log_dups_size":0,"ondisk_log_size":4714,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:52:53.085450+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":218693632,"num_objects":3337,"num_object_clones":0,"num_object_copies":10011,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":550,"num_objects_misplaced":343,"num_objects_unfound":0,"num_objects_dirty":3337,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7993,"num_write_kb":362560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4446,"num_bytes_recovered":291373056,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2],"avail_no_missing":["1","2","5"],"object_location_counts":[{"shards":"1,2","objects":2444},{"shards":"1,2,5","objects":343}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":753,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-23T20:39:10.653962+0000","last_change":"2023-09-23T20:38:05.615711+0000","last_active":"2023-09-23T20:39:10.653962+0000","last_peered":"2023-09-23T20:39:10.653962+0000","last_clean":"2023-09-23T20:39:10.653962+0000","last_became_active":"2023-09-23T20:38:05.615390+0000","last_became_peered":"2023-09-23T20:38:05.615390+0000","last_unstale":"2023-09-23T20:39:10.653962+0000","last_undegraded":"2023-09-23T20:39:10.653962+0000","last_fullsized":"2023-09-23T20:39:10.653962+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:43.582146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:43.582146+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:45:11.917871+0000","scrub_duration":0.0016219730000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"86'80","reported_seq":676,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-23T20:39:10.650539+0000","last_change":"2023-09-23T20:29:45.536796+0000","last_active":"2023-09-23T20:39:10.650539+0000","last_peered":"2023-09-23T20:39:10.650539+0000","last_clean":"2023-09-23T20:39:10.650539+0000","last_became_active":"2023-09-23T20:29:45.536439+0000","last_became_peered":"2023-09-23T20:29:45.536439+0000","last_unstale":"2023-09-23T20:39:10.650539+0000","last_undegraded":"2023-09-23T20:39:10.650539+0000","last_fullsized":"2023-09-23T20:39:10.650539+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":80,"log_dups_size":0,"ondisk_log_size":80,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:30:57.793132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035115,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":395,"num_read_kb":1370,"num_write":66,"num_write_kb":1061,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":427,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-09-23T20:39:00.601688+0000","last_change":"2023-09-23T20:34:59.984286+0000","last_active":"2023-09-23T20:39:00.601688+0000","last_peered":"2023-09-23T20:39:00.601688+0000","last_clean":"2023-09-23T20:39:00.601688+0000","last_became_active":"2023-09-23T20:34:59.983950+0000","last_became_peered":"2023-09-23T20:34:59.983950+0000","last_unstale":"2023-09-23T20:39:00.601688+0000","last_undegraded":"2023-09-23T20:39:00.601688+0000","last_fullsized":"2023-09-23T20:39:00.601688+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:31:53.557035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":421,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-09-23T20:39:00.602909+0000","last_change":"2023-09-23T20:29:45.690815+0000","last_active":"2023-09-23T20:39:00.602909+0000","last_peered":"2023-09-23T20:39:00.602909+0000","last_clean":"2023-09-23T20:39:00.602909+0000","last_became_active":"2023-09-23T20:29:45.690478+0000","last_became_peered":"2023-09-23T20:29:45.690478+0000","last_unstale":"2023-09-23T20:39:00.602909+0000","last_undegraded":"2023-09-23T20:39:00.602909+0000","last_fullsized":"2023-09-23T20:39:00.602909+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:38.288193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.f","version":"424'7648","reported_seq":7713,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-09-23T20:39:17.853579+0000","last_change":"2023-09-23T20:36:11.309536+0000","last_active":"2023-09-23T20:39:17.853579+0000","last_peered":"2023-09-23T20:39:17.853579+0000","last_clean":"2023-09-23T20:39:17.853579+0000","last_became_active":"2023-09-23T20:36:11.309208+0000","last_became_peered":"2023-09-23T20:36:11.309208+0000","last_unstale":"2023-09-23T20:39:17.853579+0000","last_undegraded":"2023-09-23T20:39:17.853579+0000","last_fullsized":"2023-09-23T20:39:17.853579+0000","mapping_epoch":369,"log_start":"372'300","ondisk_log_start":"372'300","created":369,"last_epoch_clean":370,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":7348,"log_dups_size":0,"ondisk_log_size":7348,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:47:51.785158+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":221904896,"num_objects":3386,"num_object_clones":0,"num_object_copies":10158,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3386,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7648,"num_write_kb":353088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":526,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-23T20:39:10.649749+0000","last_change":"2023-09-23T20:38:05.611181+0000","last_active":"2023-09-23T20:39:10.649749+0000","last_peered":"2023-09-23T20:39:10.649749+0000","last_clean":"2023-09-23T20:39:10.649749+0000","last_became_active":"2023-09-23T20:38:05.610881+0000","last_became_peered":"2023-09-23T20:38:05.610881+0000","last_unstale":"2023-09-23T20:39:10.649749+0000","last_undegraded":"2023-09-23T20:39:10.649749+0000","last_fullsized":"2023-09-23T20:39:10.649749+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:35:00.151695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:35:00.151695+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":20,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"86'102","reported_seq":1018,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-23T20:39:10.651818+0000","last_change":"2023-09-23T20:32:36.314672+0000","last_active":"2023-09-23T20:39:10.651818+0000","last_peered":"2023-09-23T20:39:10.651818+0000","last_clean":"2023-09-23T20:39:10.651818+0000","last_became_active":"2023-09-23T20:32:36.314231+0000","last_became_peered":"2023-09-23T20:32:36.314231+0000","last_unstale":"2023-09-23T20:39:10.651818+0000","last_undegraded":"2023-09-23T20:39:10.651818+0000","last_fullsized":"2023-09-23T20:39:10.651818+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":102,"log_dups_size":0,"ondisk_log_size":102,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:11:40.514709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030656500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":963,"num_read_kb":1234,"num_write":60,"num_write_kb":225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":359,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-23T20:39:10.651838+0000","last_change":"2023-09-23T20:37:14.359334+0000","last_active":"2023-09-23T20:39:10.651838+0000","last_peered":"2023-09-23T20:39:10.651838+0000","last_clean":"2023-09-23T20:39:10.651838+0000","last_became_active":"2023-09-23T20:37:14.358982+0000","last_became_peered":"2023-09-23T20:37:14.358982+0000","last_unstale":"2023-09-23T20:39:10.651838+0000","last_undegraded":"2023-09-23T20:39:10.651838+0000","last_fullsized":"2023-09-23T20:39:10.651838+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:50:00.008145+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":368,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-09-23T20:39:01.201150+0000","last_change":"2023-09-23T20:37:14.359068+0000","last_active":"2023-09-23T20:39:01.201150+0000","last_peered":"2023-09-23T20:39:01.201150+0000","last_clean":"2023-09-23T20:39:01.201150+0000","last_became_active":"2023-09-23T20:37:14.358729+0000","last_became_peered":"2023-09-23T20:37:14.358729+0000","last_unstale":"2023-09-23T20:39:01.201150+0000","last_undegraded":"2023-09-23T20:39:01.201150+0000","last_fullsized":"2023-09-23T20:39:01.201150+0000","mapping_epoch":379,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:19:17.084920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"423'7795","reported_seq":7859,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-23T20:39:16.183164+0000","last_change":"2023-09-23T20:36:11.482126+0000","last_active":"2023-09-23T20:39:16.183164+0000","last_peered":"2023-09-23T20:39:16.183164+0000","last_clean":"2023-09-23T20:39:16.183164+0000","last_became_active":"2023-09-23T20:36:11.481802+0000","last_became_peered":"2023-09-23T20:36:11.481802+0000","last_unstale":"2023-09-23T20:39:16.183164+0000","last_undegraded":"2023-09-23T20:39:16.183164+0000","last_fullsized":"2023-09-23T20:39:16.183164+0000","mapping_epoch":369,"log_start":"373'1959","ondisk_log_start":"373'1959","created":369,"last_epoch_clean":370,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":5836,"log_dups_size":0,"ondisk_log_size":5836,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:13:12.263455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":229965824,"num_objects":3509,"num_object_clones":0,"num_object_copies":10527,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3509,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7795,"num_write_kb":361728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":701,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-09-23T20:39:00.602853+0000","last_change":"2023-09-23T20:38:05.590361+0000","last_active":"2023-09-23T20:39:00.602853+0000","last_peered":"2023-09-23T20:39:00.602853+0000","last_clean":"2023-09-23T20:39:00.602853+0000","last_became_active":"2023-09-23T20:38:05.589985+0000","last_became_peered":"2023-09-23T20:38:05.589985+0000","last_unstale":"2023-09-23T20:39:00.602853+0000","last_undegraded":"2023-09-23T20:39:00.602853+0000","last_fullsized":"2023-09-23T20:39:00.602853+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:03.246271+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:03.246271+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:38:54.351576+0000","scrub_duration":0.0019016700000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":161,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-23T20:39:10.651749+0000","last_change":"2023-09-23T20:32:50.549695+0000","last_active":"2023-09-23T20:39:10.651749+0000","last_peered":"2023-09-23T20:39:10.651749+0000","last_clean":"2023-09-23T20:39:10.651749+0000","last_became_active":"2023-09-23T20:32:50.549431+0000","last_became_peered":"2023-09-23T20:32:50.549431+0000","last_unstale":"2023-09-23T20:39:10.651749+0000","last_undegraded":"2023-09-23T20:39:10.651749+0000","last_fullsized":"2023-09-23T20:39:10.651749+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:45:53.666569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"86'128","reported_seq":834,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-23T20:39:10.652026+0000","last_change":"2023-09-23T20:38:03.541765+0000","last_active":"2023-09-23T20:39:10.652026+0000","last_peered":"2023-09-23T20:39:10.652026+0000","last_clean":"2023-09-23T20:39:10.652026+0000","last_became_active":"2023-09-23T20:38:03.541277+0000","last_became_peered":"2023-09-23T20:38:03.541277+0000","last_unstale":"2023-09-23T20:39:10.652026+0000","last_undegraded":"2023-09-23T20:39:10.652026+0000","last_fullsized":"2023-09-23T20:39:10.652026+0000","mapping_epoch":404,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":405,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:20:53.834554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000293202,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":579,"num_read_kb":1473,"num_write":96,"num_write_kb":1062,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":201,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-23T20:39:10.661267+0000","last_change":"2023-09-23T20:37:43.582105+0000","last_active":"2023-09-23T20:39:10.661267+0000","last_peered":"2023-09-23T20:39:10.661267+0000","last_clean":"2023-09-23T20:39:10.661267+0000","last_became_active":"2023-09-23T20:37:43.581860+0000","last_became_peered":"2023-09-23T20:37:43.581860+0000","last_unstale":"2023-09-23T20:39:10.661267+0000","last_undegraded":"2023-09-23T20:39:10.661267+0000","last_fullsized":"2023-09-23T20:39:10.661267+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:18:38.765486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"424'7774","reported_seq":9692,"reported_epoch":424,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:39:17.145558+0000","last_change":"2023-09-23T20:38:02.710271+0000","last_active":"2023-09-23T20:39:17.145558+0000","last_peered":"2023-09-23T20:39:17.145558+0000","last_clean":"2023-09-23T20:37:13.317209+0000","last_became_active":"2023-09-23T20:38:02.636987+0000","last_became_peered":"2023-09-23T20:38:02.636987+0000","last_unstale":"2023-09-23T20:39:17.145558+0000","last_undegraded":"2023-09-23T20:38:02.547074+0000","last_fullsized":"2023-09-23T20:38:02.526059+0000","mapping_epoch":403,"log_start":"373'2121","ondisk_log_start":"373'2121","created":369,"last_epoch_clean":370,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":5653,"log_dups_size":0,"ondisk_log_size":5653,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:15:55.380701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":219742232,"num_objects":3354,"num_object_clones":0,"num_object_copies":10062,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5420,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3354,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7774,"num_write_kb":356033,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":913,"num_bytes_recovered":59834368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2],"avail_no_missing":["7","2","5"],"object_location_counts":[{"shards":"1,2,7","objects":280},{"shards":"2,5,7","objects":3059},{"shards":"2,7","objects":-1755}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.4","version":"90'161","reported_seq":618,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-23T20:39:10.662653+0000","last_change":"2023-09-23T20:38:05.607009+0000","last_active":"2023-09-23T20:39:10.662653+0000","last_peered":"2023-09-23T20:39:10.662653+0000","last_clean":"2023-09-23T20:39:10.662653+0000","last_became_active":"2023-09-23T20:38:05.606621+0000","last_became_peered":"2023-09-23T20:38:05.606621+0000","last_unstale":"2023-09-23T20:39:10.662653+0000","last_undegraded":"2023-09-23T20:39:10.662653+0000","last_fullsized":"2023-09-23T20:39:10.662653+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:34:54.504554+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:34:54.504554+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:47:23.494026+0000","scrub_duration":0.011176864,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"86'117","reported_seq":657,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-23T20:39:10.649310+0000","last_change":"2023-09-23T20:38:28.446107+0000","last_active":"2023-09-23T20:39:10.649310+0000","last_peered":"2023-09-23T20:39:10.649310+0000","last_clean":"2023-09-23T20:39:10.649310+0000","last_became_active":"2023-09-23T20:38:28.445667+0000","last_became_peered":"2023-09-23T20:38:28.445667+0000","last_unstale":"2023-09-23T20:39:10.649310+0000","last_undegraded":"2023-09-23T20:39:10.649310+0000","last_fullsized":"2023-09-23T20:39:10.649310+0000","mapping_epoch":411,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":412,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:58:47.430360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046891400000000002,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":893,"num_read_kb":922,"num_write":82,"num_write_kb":4193,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":421,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-09-23T20:39:00.602995+0000","last_change":"2023-09-23T20:32:33.277827+0000","last_active":"2023-09-23T20:39:00.602995+0000","last_peered":"2023-09-23T20:39:00.602995+0000","last_clean":"2023-09-23T20:39:00.602995+0000","last_became_active":"2023-09-23T20:32:33.236087+0000","last_became_peered":"2023-09-23T20:32:33.236087+0000","last_unstale":"2023-09-23T20:39:00.602995+0000","last_undegraded":"2023-09-23T20:39:00.602995+0000","last_fullsized":"2023-09-23T20:39:00.602995+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:20:58.176854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":150,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-09-23T20:39:00.601899+0000","last_change":"2023-09-23T20:38:02.700897+0000","last_active":"2023-09-23T20:39:00.601899+0000","last_peered":"2023-09-23T20:39:00.601899+0000","last_clean":"2023-09-23T20:39:00.601899+0000","last_became_active":"2023-09-23T20:38:02.700561+0000","last_became_peered":"2023-09-23T20:38:02.700561+0000","last_unstale":"2023-09-23T20:39:00.601899+0000","last_undegraded":"2023-09-23T20:39:00.601899+0000","last_fullsized":"2023-09-23T20:39:00.601899+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:31:11.316211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.10","version":"424'7072","reported_seq":13496,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-09-23T20:39:18.525519+0000","last_change":"2023-09-23T20:39:03.521418+0000","last_active":"2023-09-23T20:39:18.525519+0000","last_peered":"2023-09-23T20:39:18.525519+0000","last_clean":"2023-09-23T20:39:18.525519+0000","last_became_active":"2023-09-23T20:38:27.574914+0000","last_became_peered":"2023-09-23T20:38:27.574914+0000","last_unstale":"2023-09-23T20:39:18.525519+0000","last_undegraded":"2023-09-23T20:39:18.525519+0000","last_fullsized":"2023-09-23T20:39:18.525519+0000","mapping_epoch":410,"log_start":"373'2187","ondisk_log_start":"373'2187","created":410,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":4885,"log_dups_size":0,"ondisk_log_size":4885,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":106823680,"num_objects":1630,"num_object_clones":0,"num_object_copies":4890,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1630,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3825,"num_write_kb":174560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2537,"num_bytes_recovered":166264832,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.19","version":"90'161","reported_seq":691,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-09-23T20:39:00.601424+0000","last_change":"2023-09-23T20:38:05.630221+0000","last_active":"2023-09-23T20:39:00.601424+0000","last_peered":"2023-09-23T20:39:00.601424+0000","last_clean":"2023-09-23T20:39:00.601424+0000","last_became_active":"2023-09-23T20:38:05.629839+0000","last_became_peered":"2023-09-23T20:38:05.629839+0000","last_unstale":"2023-09-23T20:39:00.601424+0000","last_undegraded":"2023-09-23T20:39:00.601424+0000","last_fullsized":"2023-09-23T20:39:00.601424+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:06.487351+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:06.487351+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:25:51.325529+0000","scrub_duration":0.001964379,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":389,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-09-23T20:39:00.602901+0000","last_change":"2023-09-23T20:32:33.237234+0000","last_active":"2023-09-23T20:39:00.602901+0000","last_peered":"2023-09-23T20:39:00.602901+0000","last_clean":"2023-09-23T20:39:00.602901+0000","last_became_active":"2023-09-23T20:32:33.236881+0000","last_became_peered":"2023-09-23T20:32:33.236881+0000","last_unstale":"2023-09-23T20:39:00.602901+0000","last_undegraded":"2023-09-23T20:39:00.602901+0000","last_fullsized":"2023-09-23T20:39:00.602901+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:56:17.384831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":278,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-23T20:39:10.648093+0000","last_change":"2023-09-23T20:29:45.538314+0000","last_active":"2023-09-23T20:39:10.648093+0000","last_peered":"2023-09-23T20:39:10.648093+0000","last_clean":"2023-09-23T20:39:10.648093+0000","last_became_active":"2023-09-23T20:29:45.536847+0000","last_became_peered":"2023-09-23T20:29:45.536847+0000","last_unstale":"2023-09-23T20:39:10.648093+0000","last_undegraded":"2023-09-23T20:39:10.648093+0000","last_fullsized":"2023-09-23T20:39:10.648093+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:51.948721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"8.17","version":"424'7164","reported_seq":7254,"reported_epoch":424,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T20:39:19.391980+0000","last_change":"2023-09-23T20:38:27.525765+0000","last_active":"2023-09-23T20:39:19.391980+0000","last_peered":"2023-09-23T20:39:19.391980+0000","last_clean":"2023-09-23T20:37:19.384160+0000","last_became_active":"2023-09-23T20:38:27.474930+0000","last_became_peered":"2023-09-23T20:38:27.474930+0000","last_unstale":"2023-09-23T20:39:19.391980+0000","last_undegraded":"2023-09-23T20:38:27.459263+0000","last_fullsized":"2023-09-23T20:39:19.391980+0000","mapping_epoch":410,"log_start":"373'1658","ondisk_log_start":"373'1658","created":410,"last_epoch_clean":370,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":5506,"log_dups_size":0,"ondisk_log_size":5506,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":110297088,"num_objects":1683,"num_object_clones":0,"num_object_copies":5049,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":326,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1683,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3874,"num_write_kb":177824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":["1","3"],"object_location_counts":[{"shards":"1,2,3","objects":158},{"shards":"1,3,6","objects":1525}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1e","version":"90'161","reported_seq":482,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-23T20:39:10.662638+0000","last_change":"2023-09-23T20:38:05.602250+0000","last_active":"2023-09-23T20:39:10.662638+0000","last_peered":"2023-09-23T20:39:10.662638+0000","last_clean":"2023-09-23T20:39:10.662638+0000","last_became_active":"2023-09-23T20:38:05.601874+0000","last_became_peered":"2023-09-23T20:38:05.601874+0000","last_unstale":"2023-09-23T20:39:10.662638+0000","last_undegraded":"2023-09-23T20:39:10.662638+0000","last_fullsized":"2023-09-23T20:39:10.662638+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:35:42.864363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:35:42.864363+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":40253,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":21,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-09-23T20:39:00.624918+0000","last_change":"2023-09-23T20:38:37.481551+0000","last_active":"2023-09-23T20:39:00.624918+0000","last_peered":"2023-09-23T20:39:00.624918+0000","last_clean":"2023-09-23T20:39:00.624918+0000","last_became_active":"2023-09-23T20:38:37.458687+0000","last_became_peered":"2023-09-23T20:38:37.458687+0000","last_unstale":"2023-09-23T20:39:00.624918+0000","last_undegraded":"2023-09-23T20:39:00.624918+0000","last_fullsized":"2023-09-23T20:39:00.624918+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:22:15.033243+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":79,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-09-23T20:39:00.602379+0000","last_change":"2023-09-23T20:34:59.987575+0000","last_active":"2023-09-23T20:39:00.602379+0000","last_peered":"2023-09-23T20:39:00.602379+0000","last_clean":"2023-09-23T20:39:00.602379+0000","last_became_active":"2023-09-23T20:34:59.987229+0000","last_became_peered":"2023-09-23T20:34:59.987229+0000","last_unstale":"2023-09-23T20:39:00.602379+0000","last_undegraded":"2023-09-23T20:39:00.602379+0000","last_fullsized":"2023-09-23T20:39:00.602379+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:34:01.820159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.16","version":"423'7103","reported_seq":7865,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-23T20:39:16.111809+0000","last_change":"2023-09-23T20:38:28.264940+0000","last_active":"2023-09-23T20:39:16.111809+0000","last_peered":"2023-09-23T20:39:16.111809+0000","last_clean":"2023-09-23T20:39:16.111809+0000","last_became_active":"2023-09-23T20:38:27.484333+0000","last_became_peered":"2023-09-23T20:38:27.484333+0000","last_unstale":"2023-09-23T20:39:16.111809+0000","last_undegraded":"2023-09-23T20:39:16.111809+0000","last_fullsized":"2023-09-23T20:39:16.111809+0000","mapping_epoch":410,"log_start":"373'1261","ondisk_log_start":"373'1261","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":5842,"log_dups_size":0,"ondisk_log_size":5842,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":113442816,"num_objects":1731,"num_object_clones":0,"num_object_copies":5193,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1731,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3823,"num_write_kb":177728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":335,"num_bytes_recovered":9699328,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":21,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-09-23T20:39:00.626435+0000","last_change":"2023-09-23T20:38:37.457634+0000","last_active":"2023-09-23T20:39:00.626435+0000","last_peered":"2023-09-23T20:39:00.626435+0000","last_clean":"2023-09-23T20:39:00.626435+0000","last_became_active":"2023-09-23T20:38:37.452208+0000","last_became_peered":"2023-09-23T20:38:37.452208+0000","last_unstale":"2023-09-23T20:39:00.626435+0000","last_undegraded":"2023-09-23T20:39:00.626435+0000","last_fullsized":"2023-09-23T20:39:00.626435+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:09:41.754569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":79,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-09-23T20:39:00.602451+0000","last_change":"2023-09-23T20:34:59.987416+0000","last_active":"2023-09-23T20:39:00.602451+0000","last_peered":"2023-09-23T20:39:00.602451+0000","last_clean":"2023-09-23T20:39:00.602451+0000","last_became_active":"2023-09-23T20:34:59.987117+0000","last_became_peered":"2023-09-23T20:34:59.987117+0000","last_unstale":"2023-09-23T20:39:00.602451+0000","last_undegraded":"2023-09-23T20:39:00.602451+0000","last_fullsized":"2023-09-23T20:39:00.602451+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:36:16.318623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.15","version":"424'7182","reported_seq":10736,"reported_epoch":424,"state":"active+clean","last_fresh":"2023-09-23T20:39:17.799201+0000","last_change":"2023-09-23T20:38:53.167534+0000","last_active":"2023-09-23T20:39:17.799201+0000","last_peered":"2023-09-23T20:39:17.799201+0000","last_clean":"2023-09-23T20:39:17.799201+0000","last_became_active":"2023-09-23T20:38:28.579407+0000","last_became_peered":"2023-09-23T20:38:28.579407+0000","last_unstale":"2023-09-23T20:39:17.799201+0000","last_undegraded":"2023-09-23T20:39:17.799201+0000","last_fullsized":"2023-09-23T20:39:17.799201+0000","mapping_epoch":410,"log_start":"373'1839","ondisk_log_start":"373'1839","created":410,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":5343,"log_dups_size":0,"ondisk_log_size":5343,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109215744,"num_objects":1667,"num_object_clones":0,"num_object_copies":5001,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1667,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3887,"num_write_kb":177696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1043,"num_bytes_recovered":56328192,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1c","version":"90'161","reported_seq":679,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-23T20:39:10.651107+0000","last_change":"2023-09-23T20:38:05.591992+0000","last_active":"2023-09-23T20:39:10.651107+0000","last_peered":"2023-09-23T20:39:10.651107+0000","last_clean":"2023-09-23T20:39:10.651107+0000","last_became_active":"2023-09-23T20:38:05.591701+0000","last_became_peered":"2023-09-23T20:38:05.591701+0000","last_unstale":"2023-09-23T20:39:10.651107+0000","last_undegraded":"2023-09-23T20:39:10.651107+0000","last_fullsized":"2023-09-23T20:39:10.651107+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:02.384133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:02.384133+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:41.722542+0000","scrub_duration":0.001769098,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":243,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-23T20:39:10.662878+0000","last_change":"2023-09-23T20:38:02.542719+0000","last_active":"2023-09-23T20:39:10.662878+0000","last_peered":"2023-09-23T20:39:10.662878+0000","last_clean":"2023-09-23T20:39:10.662878+0000","last_became_active":"2023-09-23T20:38:02.542385+0000","last_became_peered":"2023-09-23T20:38:02.542385+0000","last_unstale":"2023-09-23T20:39:10.662878+0000","last_undegraded":"2023-09-23T20:39:10.662878+0000","last_fullsized":"2023-09-23T20:39:10.662878+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:36:23.673757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":210,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-23T20:39:10.661991+0000","last_change":"2023-09-23T20:37:43.584253+0000","last_active":"2023-09-23T20:39:10.661991+0000","last_peered":"2023-09-23T20:39:10.661991+0000","last_clean":"2023-09-23T20:39:10.661991+0000","last_became_active":"2023-09-23T20:37:43.580069+0000","last_became_peered":"2023-09-23T20:37:43.580069+0000","last_unstale":"2023-09-23T20:39:10.661991+0000","last_undegraded":"2023-09-23T20:39:10.661991+0000","last_fullsized":"2023-09-23T20:39:10.661991+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:39:11.482581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.14","version":"423'7379","reported_seq":8962,"reported_epoch":423,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:39:15.416636+0000","last_change":"2023-09-23T20:38:39.260381+0000","last_active":"2023-09-23T20:39:15.416636+0000","last_peered":"2023-09-23T20:39:15.416636+0000","last_clean":"2023-09-23T20:37:15.925522+0000","last_became_active":"2023-09-23T20:38:39.251212+0000","last_became_peered":"2023-09-23T20:38:39.251212+0000","last_unstale":"2023-09-23T20:39:15.416636+0000","last_undegraded":"2023-09-23T20:38:38.457918+0000","last_fullsized":"2023-09-23T20:38:38.450156+0000","mapping_epoch":416,"log_start":"373'2080","ondisk_log_start":"373'2080","created":410,"last_epoch_clean":370,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":5299,"log_dups_size":0,"ondisk_log_size":5299,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112787456,"num_objects":1721,"num_object_clones":0,"num_object_copies":5163,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":693,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1721,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3972,"num_write_kb":182176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":376,"num_bytes_recovered":24641536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[4,2],"avail_no_missing":["4","2","6"],"object_location_counts":[{"shards":"2,3,4","objects":149},{"shards":"2,4","objects":1572}],"blocked_by":[],"up_primary":5,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1d","version":"90'161","reported_seq":515,"reported_epoch":423,"state":"active+clean","last_fresh":"2023-09-23T20:39:10.652394+0000","last_change":"2023-09-23T20:36:01.505098+0000","last_active":"2023-09-23T20:39:10.652394+0000","last_peered":"2023-09-23T20:39:10.652394+0000","last_clean":"2023-09-23T20:39:10.652394+0000","last_became_active":"2023-09-23T20:35:26.644707+0000","last_became_peered":"2023-09-23T20:35:26.644707+0000","last_unstale":"2023-09-23T20:39:10.652394+0000","last_undegraded":"2023-09-23T20:39:10.652394+0000","last_fullsized":"2023-09-23T20:39:10.652394+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:01.504973+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:01.504973+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:02:10.880809+0000","scrub_duration":0.00202344,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":452,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-09-23T20:39:00.602747+0000","last_change":"2023-09-23T20:38:02.655107+0000","last_active":"2023-09-23T20:39:00.602747+0000","last_peered":"2023-09-23T20:39:00.602747+0000","last_clean":"2023-09-23T20:39:00.602747+0000","last_became_active":"2023-09-23T20:38:02.574184+0000","last_became_peered":"2023-09-23T20:38:02.574184+0000","last_unstale":"2023-09-23T20:39:00.602747+0000","last_undegraded":"2023-09-23T20:39:00.602747+0000","last_fullsized":"2023-09-23T20:39:00.602747+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:59:37.398085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2770,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":140,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-09-23T20:39:00.601554+0000","last_change":"2023-09-23T20:38:02.538333+0000","last_active":"2023-09-23T20:39:00.601554+0000","last_peered":"2023-09-23T20:39:00.601554+0000","last_clean":"2023-09-23T20:39:00.601554+0000","last_became_active":"2023-09-23T20:38:02.538008+0000","last_became_peered":"2023-09-23T20:38:02.538008+0000","last_unstale":"2023-09-23T20:39:00.601554+0000","last_undegraded":"2023-09-23T20:39:00.601554+0000","last_fullsized":"2023-09-23T20:39:00.601554+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:02:38.789530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":73,"num_bytes_recovered":17862,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26691,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":8,"num_pg":26,"stat_sum":{"num_bytes":3543269400,"num_objects":54067,"num_object_clones":0,"num_object_copies":162201,"num_objects_missing_on_primary":657,"num_objects_missing":657,"num_objects_degraded":19177,"num_objects_misplaced":343,"num_objects_unfound":0,"num_objects_dirty":54067,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":124575,"num_write_kb":5716481,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21436,"num_bytes_recovered":1358168064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12099399680,"data_stored":12099387464,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":141823,"ondisk_log_size":141823,"up":78,"acting":72,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":351684,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":155,"num_read_kb":290,"num_write":76,"num_write_kb":768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1153900,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5983,"num_read_kb":13200,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":57,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16789504,"data_stored":16777273,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6082,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201614,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1464884,"kb_used_data":1100268,"kb_used_omap":0,"kb_used_meta":364608,"kb_avail":92280268,"statfs":{"total":95995035648,"available":94494994432,"internally_reserved":0,"allocated":1126674432,"data_stored":1125336795,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":373358592},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:38:50 2023","interfaces":[{"interface":"back","average":{"1min":2.8180000000000001,"5min":2.6960000000000002,"15min":2.21},"min":{"1min":0.255,"5min":0.23599999999999999,"15min":0.20799999999999999},"max":{"1min":18.315000000000001,"5min":18.315000000000001,"15min":48.798999999999999},"last":0.27900000000000003},{"interface":"front","average":{"1min":2.3330000000000002,"5min":2.1259999999999999,"15min":2.0680000000000001},"min":{"1min":0.29399999999999998,"5min":0.247,"15min":0.24399999999999999},"max":{"1min":9.4429999999999996,"5min":19.355,"15min":48.247},"last":0.30399999999999999}]},{"osd":1,"last update":"Sat Sep 23 20:39:03 2023","interfaces":[{"interface":"back","average":{"1min":3.9340000000000002,"5min":2.9929999999999999,"15min":2.2080000000000002},"min":{"1min":0.28100000000000003,"5min":0.25900000000000001,"15min":0.254},"max":{"1min":29.254999999999999,"5min":49.965000000000003,"15min":49.965000000000003},"last":1.2130000000000001},{"interface":"front","average":{"1min":3.0899999999999999,"5min":3.0230000000000001,"15min":2.2440000000000002},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.248},"max":{"1min":16.882000000000001,"5min":53.451999999999998,"15min":53.451999999999998},"last":0.5}]},{"osd":2,"last update":"Sat Sep 23 20:38:50 2023","interfaces":[{"interface":"back","average":{"1min":3.0310000000000001,"5min":2.4180000000000001,"15min":2.149},"min":{"1min":0.28599999999999998,"5min":0.27800000000000002,"15min":0.27600000000000002},"max":{"1min":16.965,"5min":16.965,"15min":16.965},"last":2.9849999999999999},{"interface":"front","average":{"1min":5.1319999999999997,"5min":2.9489999999999998,"15min":2.5409999999999999},"min":{"1min":0.33500000000000002,"5min":0.308,"15min":0.26600000000000001},"max":{"1min":23.414000000000001,"5min":23.414000000000001,"15min":23.494},"last":0.33700000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:38:18 2023","interfaces":[{"interface":"back","average":{"1min":3.173,"5min":2.956,"15min":2.355},"min":{"1min":0.376,"5min":0.35199999999999998,"15min":0.26200000000000001},"max":{"1min":21.018000000000001,"5min":21.018000000000001,"15min":21.018000000000001},"last":1.333},{"interface":"front","average":{"1min":3.1760000000000002,"5min":3.0259999999999998,"15min":2.2890000000000001},"min":{"1min":0.45000000000000001,"5min":0.34399999999999997,"15min":0.27600000000000002},"max":{"1min":20.969999999999999,"5min":20.969999999999999,"15min":20.969999999999999},"last":1.276}]},{"osd":4,"last update":"Sat Sep 23 20:38:18 2023","interfaces":[{"interface":"back","average":{"1min":0.44600000000000001,"5min":0.85899999999999999,"15min":1.1519999999999999},"min":{"1min":0.27200000000000002,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":1.5429999999999999,"5min":6.3449999999999998,"15min":15.177},"last":0.35399999999999998},{"interface":"front","average":{"1min":0.46500000000000002,"5min":1.0529999999999999,"15min":1.3220000000000001},"min":{"1min":0.30399999999999999,"5min":0.28399999999999997,"15min":0.27600000000000002},"max":{"1min":0.73199999999999998,"5min":6.5339999999999998,"15min":11.122},"last":0.52800000000000002}]},{"osd":5,"last update":"Sat Sep 23 20:38:18 2023","interfaces":[{"interface":"back","average":{"1min":0.47399999999999998,"5min":0.92100000000000004,"15min":1.24},"min":{"1min":0.29699999999999999,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":1.1850000000000001,"5min":7.1159999999999997,"15min":13.241},"last":0.44800000000000001},{"interface":"front","average":{"1min":0.495,"5min":0.94999999999999996,"15min":1.3200000000000001},"min":{"1min":0.378,"5min":0.31900000000000001,"15min":0.315},"max":{"1min":0.69399999999999995,"5min":8.4809999999999999,"15min":18.831},"last":0.48699999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:38:18 2023","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":0.92300000000000004,"15min":1.1990000000000001},"min":{"1min":0.36699999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.69199999999999995,"5min":6.5540000000000003,"15min":13.185},"last":0.437},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.98899999999999999,"15min":1.1910000000000001},"min":{"1min":0.35099999999999998,"5min":0.34699999999999998,"15min":0.318},"max":{"1min":0.77400000000000002,"5min":6.2770000000000001,"15min":9.9120000000000008},"last":0.46200000000000002}]}]},{"osd":6,"up_from":47,"seq":201863463253,"num_pgs":60,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1480564,"kb_used_data":1118316,"kb_used_omap":0,"kb_used_meta":362240,"kb_avail":92264588,"statfs":{"total":95995035648,"available":94478938112,"internally_reserved":0,"allocated":1145155584,"data_stored":1143826082,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":370933760},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,1,1,0,1,5],"upper_bound":512},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:38:45 2023","interfaces":[{"interface":"back","average":{"1min":2.532,"5min":2.476,"15min":1.9890000000000001},"min":{"1min":0.26400000000000001,"5min":0.23300000000000001,"15min":0.221},"max":{"1min":14.891,"5min":34.143999999999998,"15min":53.701999999999998},"last":1.4079999999999999},{"interface":"front","average":{"1min":4.8259999999999996,"5min":2.9289999999999998,"15min":2.4710000000000001},"min":{"1min":0.26900000000000002,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":60.328000000000003,"5min":60.328000000000003,"15min":60.328000000000003},"last":1.514}]},{"osd":1,"last update":"Sat Sep 23 20:39:06 2023","interfaces":[{"interface":"back","average":{"1min":3.8690000000000002,"5min":2.5190000000000001,"15min":2.4670000000000001},"min":{"1min":0.26000000000000001,"5min":0.25600000000000001,"15min":0.249},"max":{"1min":64.274000000000001,"5min":64.274000000000001,"15min":64.274000000000001},"last":2.0880000000000001},{"interface":"front","average":{"1min":1.3340000000000001,"5min":2.004,"15min":2.633},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":5.9569999999999999,"5min":34.490000000000002,"15min":34.490000000000002},"last":1.897}]},{"osd":2,"last update":"Sat Sep 23 20:38:45 2023","interfaces":[{"interface":"back","average":{"1min":2.2109999999999999,"5min":2.3100000000000001,"15min":2.1579999999999999},"min":{"1min":0.32100000000000001,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":13.010999999999999,"5min":34.317999999999998,"15min":34.317999999999998},"last":1.5409999999999999},{"interface":"front","average":{"1min":2.218,"5min":2.27,"15min":2.0449999999999999},"min":{"1min":0.41899999999999998,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":12.917999999999999,"5min":34.292999999999999,"15min":34.292999999999999},"last":1.5309999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:38:33 2023","interfaces":[{"interface":"back","average":{"1min":2.0089999999999999,"5min":1.9590000000000001,"15min":1.952},"min":{"1min":0.308,"5min":0.29999999999999999,"15min":0.29599999999999999},"max":{"1min":8.7799999999999994,"5min":8.9670000000000005,"15min":9.4239999999999995},"last":0.65200000000000002},{"interface":"front","average":{"1min":3.0339999999999998,"5min":2.3399999999999999,"15min":2.1280000000000001},"min":{"1min":0.39100000000000001,"5min":0.34899999999999998,"15min":0.317},"max":{"1min":11.913,"5min":12.875999999999999,"15min":12.875999999999999},"last":1.214}]},{"osd":4,"last update":"Sat Sep 23 20:39:11 2023","interfaces":[{"interface":"back","average":{"1min":1.472,"5min":0.84599999999999997,"15min":0.97999999999999998},"min":{"1min":0.28299999999999997,"5min":0.26900000000000002,"15min":0.251},"max":{"1min":23.904,"5min":23.904,"15min":23.904},"last":0.30099999999999999},{"interface":"front","average":{"1min":1.135,"5min":1.0429999999999999,"15min":1.21},"min":{"1min":0.29199999999999998,"5min":0.29099999999999998,"15min":0.27000000000000002},"max":{"1min":11.515000000000001,"5min":11.515000000000001,"15min":11.515000000000001},"last":0.52000000000000002}]},{"osd":5,"last update":"Sat Sep 23 20:39:11 2023","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":0.75,"15min":1.012},"min":{"1min":0.32200000000000001,"5min":0.31,"15min":0.28699999999999998},"max":{"1min":1.952,"5min":4.8259999999999996,"15min":14.085000000000001},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.77200000000000002,"5min":0.86699999999999999,"15min":1.385},"min":{"1min":0.32500000000000001,"5min":0.32000000000000001,"15min":0.30599999999999999},"max":{"1min":2.3879999999999999,"5min":6.7320000000000002,"15min":18.192},"last":0.56499999999999995}]},{"osd":7,"last update":"Sat Sep 23 20:38:33 2023","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":0.76700000000000002,"15min":1.127},"min":{"1min":0.377,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":2.04,"5min":4.851,"15min":10.217000000000001},"last":0.39400000000000002},{"interface":"front","average":{"1min":0.60999999999999999,"5min":0.94499999999999995,"15min":1.329},"min":{"1min":0.36799999999999999,"5min":0.34499999999999997,"15min":0.30299999999999999},"max":{"1min":2.1389999999999998,"5min":8.8119999999999994,"15min":9.7360000000000007},"last":0.61299999999999999}]}]},{"osd":1,"up_from":266,"seq":1142461300852,"num_pgs":65,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1941348,"kb_used_data":1523420,"kb_used_omap":8,"kb_used_meta":417847,"kb_avail":91803804,"statfs":{"total":95995035648,"available":94007095296,"internally_reserved":0,"allocated":1559982080,"data_stored":1557680179,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8206,"internal_metadata":427876338},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:39:04 2023","interfaces":[{"interface":"back","average":{"1min":0.66200000000000003,"5min":2.3399999999999999,"15min":2.5739999999999998},"min":{"1min":0.20599999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":2.5659999999999998,"5min":14.531000000000001,"15min":52.689999999999998},"last":0.22600000000000001},{"interface":"front","average":{"1min":2.036,"5min":3.7080000000000002,"15min":3.3170000000000002},"min":{"1min":0.254,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":14.097,"5min":26.588000000000001,"15min":52.662999999999997},"last":0.36499999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:39:04 2023","interfaces":[{"interface":"back","average":{"1min":1.4790000000000001,"5min":3.9460000000000002,"15min":4.2409999999999997},"min":{"1min":0.23999999999999999,"5min":0.23300000000000001,"15min":0.20899999999999999},"max":{"1min":6.8259999999999996,"5min":32.255000000000003,"15min":32.255000000000003},"last":0.438},{"interface":"front","average":{"1min":1.284,"5min":2.5470000000000002,"15min":2.3290000000000002},"min":{"1min":0.254,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":5.8209999999999997,"5min":13.404999999999999,"15min":13.404999999999999},"last":0.59799999999999998}]},{"osd":3,"last update":"Sat Sep 23 20:39:04 2023","interfaces":[{"interface":"back","average":{"1min":1.216,"5min":2.8730000000000002,"15min":3.6890000000000001},"min":{"1min":0.35899999999999999,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":6.4669999999999996,"5min":15.247999999999999,"15min":80.745999999999995},"last":0.47399999999999998},{"interface":"front","average":{"1min":1.5900000000000001,"5min":2.8180000000000001,"15min":4.0780000000000003},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.22700000000000001},"max":{"1min":14.048999999999999,"5min":19.315000000000001,"15min":82.988},"last":0.316}]},{"osd":4,"last update":"Sat Sep 23 20:39:04 2023","interfaces":[{"interface":"back","average":{"1min":1.581,"5min":2.6640000000000001,"15min":2.2090000000000001},"min":{"1min":0.35299999999999998,"5min":0.315,"15min":0.30299999999999999},"max":{"1min":6.8799999999999999,"5min":24.327999999999999,"15min":24.327999999999999},"last":0.55800000000000005},{"interface":"front","average":{"1min":0.75700000000000001,"5min":1.4399999999999999,"15min":1.762},"min":{"1min":0.33500000000000002,"5min":0.311,"15min":0.311},"max":{"1min":2.6070000000000002,"5min":11.574999999999999,"15min":11.574999999999999},"last":0.42199999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:39:04 2023","interfaces":[{"interface":"back","average":{"1min":1.1539999999999999,"5min":2.4039999999999999,"15min":2.4260000000000002},"min":{"1min":0.439,"5min":0.34100000000000003,"15min":0.33200000000000002},"max":{"1min":6.0750000000000002,"5min":33.401000000000003,"15min":33.401000000000003},"last":1.3220000000000001},{"interface":"front","average":{"1min":1.708,"5min":3.0169999999999999,"15min":2.395},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":6.9370000000000003,"5min":33.417000000000002,"15min":33.417000000000002},"last":1.3029999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:39:04 2023","interfaces":[{"interface":"back","average":{"1min":1.3680000000000001,"5min":2.2210000000000001,"15min":4.5330000000000004},"min":{"1min":0.42599999999999999,"5min":0.36899999999999999,"15min":0.35699999999999998},"max":{"1min":6.1459999999999999,"5min":9.6999999999999993,"15min":72.682000000000002},"last":0.52900000000000003},{"interface":"front","average":{"1min":1.6879999999999999,"5min":2.7410000000000001,"15min":4.3460000000000001},"min":{"1min":0.41799999999999998,"5min":0.41099999999999998,"15min":0.38800000000000001},"max":{"1min":6.9189999999999996,"5min":24.376000000000001,"15min":72.563999999999993},"last":0.498}]},{"osd":7,"last update":"Sat Sep 23 20:39:04 2023","interfaces":[{"interface":"back","average":{"1min":1.349,"5min":2.3199999999999998,"15min":2.649},"min":{"1min":0.40999999999999998,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":6.1189999999999998,"5min":18.437000000000001,"15min":18.437000000000001},"last":0.625},{"interface":"front","average":{"1min":1.679,"5min":2.7839999999999998,"15min":2.4630000000000001},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":6.9029999999999996,"5min":24.524000000000001,"15min":24.524000000000001},"last":0.60899999999999999}]}]},{"osd":0,"up_from":94,"seq":403726926054,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2082912,"kb_used_data":1454104,"kb_used_omap":7,"kb_used_meta":628792,"kb_avail":91662240,"statfs":{"total":95995035648,"available":93862133760,"internally_reserved":0,"allocated":1489002496,"data_stored":1486873587,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":643883025},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:39:07 2023","interfaces":[{"interface":"back","average":{"1min":1.036,"5min":2.2200000000000002,"15min":3.2130000000000001},"min":{"1min":0.23499999999999999,"5min":0.17199999999999999,"15min":0.17199999999999999},"max":{"1min":5.1150000000000002,"5min":12.301,"15min":42.000999999999998},"last":0.246},{"interface":"front","average":{"1min":0.93400000000000005,"5min":2.573,"15min":3.5299999999999998},"min":{"1min":0.32400000000000001,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":2.8959999999999999,"5min":18.355,"15min":23.036000000000001},"last":0.85799999999999998}]},{"osd":2,"last update":"Sat Sep 23 20:38:41 2023","interfaces":[{"interface":"back","average":{"1min":3.0099999999999998,"5min":3.2429999999999999,"15min":2.9609999999999999},"min":{"1min":0.29299999999999998,"5min":0.218,"15min":0.185},"max":{"1min":10.17,"5min":62.726999999999997,"15min":77.448999999999998},"last":1.895},{"interface":"front","average":{"1min":3.8359999999999999,"5min":3.3490000000000002,"15min":2.8559999999999999},"min":{"1min":0.27300000000000002,"5min":0.247,"15min":0.188},"max":{"1min":26.445,"5min":62.689,"15min":81.724999999999994},"last":6.0430000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:38:15 2023","interfaces":[{"interface":"back","average":{"1min":3.4670000000000001,"5min":3.23,"15min":2.899},"min":{"1min":0.32600000000000001,"5min":0.24099999999999999,"15min":0.23400000000000001},"max":{"1min":9.9079999999999995,"5min":22.486000000000001,"15min":81.400999999999996},"last":2.0939999999999999},{"interface":"front","average":{"1min":4.6989999999999998,"5min":3.9399999999999999,"15min":3.1520000000000001},"min":{"1min":0.33600000000000002,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":43.229999999999997,"5min":43.229999999999997,"15min":81.137},"last":0.45200000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:38:41 2023","interfaces":[{"interface":"back","average":{"1min":2.8170000000000002,"5min":2.1749999999999998,"15min":2.1589999999999998},"min":{"1min":0.311,"5min":0.28899999999999998,"15min":0.25900000000000001},"max":{"1min":9.6110000000000007,"5min":11.068,"15min":23.876999999999999},"last":1.929},{"interface":"front","average":{"1min":1.4510000000000001,"5min":1.9570000000000001,"15min":1.958},"min":{"1min":0.46400000000000002,"5min":0.308,"15min":0.29299999999999998},"max":{"1min":5.5990000000000002,"5min":11.458,"15min":23.681999999999999},"last":0.48999999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:38:41 2023","interfaces":[{"interface":"back","average":{"1min":1.4550000000000001,"5min":1.976,"15min":2.2789999999999999},"min":{"1min":0.52700000000000002,"5min":0.34399999999999997,"15min":0.34200000000000003},"max":{"1min":5.5410000000000004,"5min":11.497999999999999,"15min":24.198},"last":0.54400000000000004},{"interface":"front","average":{"1min":4.3460000000000001,"5min":2.5499999999999998,"15min":2.3900000000000001},"min":{"1min":0.41399999999999998,"5min":0.307,"15min":0.25600000000000001},"max":{"1min":26.228000000000002,"5min":26.228000000000002,"15min":26.228000000000002},"last":1.677}]},{"osd":6,"last update":"Sat Sep 23 20:38:41 2023","interfaces":[{"interface":"back","average":{"1min":1.911,"5min":2.2949999999999999,"15min":2.1190000000000002},"min":{"1min":0.496,"5min":0.372,"15min":0.318},"max":{"1min":7.742,"5min":11.526999999999999,"15min":24.408000000000001},"last":0.51800000000000002},{"interface":"front","average":{"1min":4.3810000000000002,"5min":2.488,"15min":2.2130000000000001},"min":{"1min":0.39600000000000002,"5min":0.35499999999999998,"15min":0.29399999999999998},"max":{"1min":26.268000000000001,"5min":26.268000000000001,"15min":26.268000000000001},"last":1.615}]},{"osd":7,"last update":"Sat Sep 23 20:38:41 2023","interfaces":[{"interface":"back","average":{"1min":4.3609999999999998,"5min":2.5510000000000002,"15min":2.411},"min":{"1min":0.46899999999999997,"5min":0.36099999999999999,"15min":0.30599999999999999},"max":{"1min":26.177,"5min":26.177,"15min":26.177},"last":1.7210000000000001},{"interface":"front","average":{"1min":2.8519999999999999,"5min":2.3410000000000002,"15min":2.5539999999999998},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":9.7200000000000006,"5min":11.112,"15min":24.669},"last":1.9610000000000001}]}]},{"osd":2,"up_from":104,"seq":446676599002,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2015924,"kb_used_data":1356204,"kb_used_omap":11,"kb_used_meta":659700,"kb_avail":91729228,"statfs":{"total":95995035648,"available":93930729472,"internally_reserved":0,"allocated":1388752896,"data_stored":1386609807,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11878,"internal_metadata":675533210},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:38:55 2023","interfaces":[{"interface":"back","average":{"1min":1.403,"5min":2.907,"15min":2.7010000000000001},"min":{"1min":0.20399999999999999,"5min":0.20300000000000001,"15min":0.17999999999999999},"max":{"1min":12.007,"5min":28.234999999999999,"15min":82.424999999999997},"last":0.33000000000000002},{"interface":"front","average":{"1min":1.7170000000000001,"5min":2.7949999999999999,"15min":2.5670000000000002},"min":{"1min":0.28899999999999998,"5min":0.19,"15min":0.19},"max":{"1min":7.2510000000000003,"5min":14.055,"15min":82.524000000000001},"last":0.504}]},{"osd":1,"last update":"Sat Sep 23 20:38:58 2023","interfaces":[{"interface":"back","average":{"1min":4.0090000000000003,"5min":2.9980000000000002,"15min":3.782},"min":{"1min":0.245,"5min":0.245,"15min":0.221},"max":{"1min":46.405000000000001,"5min":46.405000000000001,"15min":46.405000000000001},"last":0.44400000000000001},{"interface":"front","average":{"1min":3.5249999999999999,"5min":3.4340000000000002,"15min":3},"min":{"1min":0.26800000000000002,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":46.459000000000003,"5min":46.459000000000003,"15min":46.459000000000003},"last":0.70599999999999996}]},{"osd":3,"last update":"Sat Sep 23 20:38:19 2023","interfaces":[{"interface":"back","average":{"1min":3.9900000000000002,"5min":3.0539999999999998,"15min":2.3540000000000001},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.19800000000000001},"max":{"1min":19.562999999999999,"5min":19.562999999999999,"15min":19.562999999999999},"last":0.308},{"interface":"front","average":{"1min":1.228,"5min":2.8439999999999999,"15min":2.504},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.23899999999999999},"max":{"1min":4.3520000000000003,"5min":15.173,"15min":30.704999999999998},"last":0.52400000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:38:55 2023","interfaces":[{"interface":"back","average":{"1min":1.4279999999999999,"5min":1.9590000000000001,"15min":1.9099999999999999},"min":{"1min":0.34000000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":4.5220000000000002,"5min":9.2430000000000003,"15min":16.797999999999998},"last":0.54700000000000004},{"interface":"front","average":{"1min":1.6779999999999999,"5min":2.3330000000000002,"15min":2.1320000000000001},"min":{"1min":0.40899999999999997,"5min":0.30499999999999999,"15min":0.27600000000000002},"max":{"1min":7.0800000000000001,"5min":19.440999999999999,"15min":19.440999999999999},"last":0.40600000000000003}]},{"osd":5,"last update":"Sat Sep 23 20:38:55 2023","interfaces":[{"interface":"back","average":{"1min":1.4670000000000001,"5min":2.0270000000000001,"15min":2.4260000000000002},"min":{"1min":0.377,"5min":0.36599999999999999,"15min":0.309},"max":{"1min":4.5970000000000004,"5min":9.1799999999999997,"15min":95.393000000000001},"last":0.91000000000000003},{"interface":"front","average":{"1min":1.73,"5min":2.4399999999999999,"15min":2.7000000000000002},"min":{"1min":0.502,"5min":0.35299999999999998,"15min":0.32300000000000001},"max":{"1min":7.0350000000000001,"5min":19.513000000000002,"15min":95.414000000000001},"last":0.93400000000000005}]},{"osd":6,"last update":"Sat Sep 23 20:38:55 2023","interfaces":[{"interface":"back","average":{"1min":1.7390000000000001,"5min":2.2679999999999998,"15min":2.1869999999999998},"min":{"1min":0.42399999999999999,"5min":0.36599999999999999,"15min":0.32100000000000001},"max":{"1min":7.1079999999999997,"5min":19.486000000000001,"15min":19.486000000000001},"last":1.119},{"interface":"front","average":{"1min":0.70599999999999996,"5min":1.8759999999999999,"15min":2.0139999999999998},"min":{"1min":0.38300000000000001,"5min":0.374,"15min":0.316},"max":{"1min":1.97,"5min":10.811,"15min":30.670999999999999},"last":1.1359999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:38:55 2023","interfaces":[{"interface":"back","average":{"1min":0.72199999999999998,"5min":1.8149999999999999,"15min":2.069},"min":{"1min":0.39300000000000002,"5min":0.372,"15min":0.35199999999999998},"max":{"1min":2.0179999999999998,"5min":10.231,"15min":30.741},"last":0.64900000000000002},{"interface":"front","average":{"1min":1.4950000000000001,"5min":2.1669999999999998,"15min":2.177},"min":{"1min":0.42799999999999999,"5min":0.40999999999999998,"15min":0.38},"max":{"1min":4.569,"5min":9.2720000000000002,"15min":16.719000000000001},"last":0.57899999999999996}]}]},{"osd":3,"up_from":109,"seq":468151435476,"num_pgs":69,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2294364,"kb_used_data":1565332,"kb_used_omap":4,"kb_used_meta":729019,"kb_avail":91450788,"statfs":{"total":95995035648,"available":93645606912,"internally_reserved":0,"allocated":1602899968,"data_stored":1600777525,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4514,"internal_metadata":746516062},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,2],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:38:20 2023","interfaces":[{"interface":"back","average":{"1min":2.306,"5min":3.0249999999999999,"15min":2.6389999999999998},"min":{"1min":0.29099999999999998,"5min":0.215,"15min":0.186},"max":{"1min":9.0850000000000009,"5min":17.129000000000001,"15min":67.305999999999997},"last":0.41599999999999998},{"interface":"front","average":{"1min":2.0529999999999999,"5min":2.8559999999999999,"15min":2.452},"min":{"1min":0.28599999999999998,"5min":0.23699999999999999,"15min":0.19500000000000001},"max":{"1min":8.8200000000000003,"5min":12.502000000000001,"15min":67.403999999999996},"last":0.45000000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:38:58 2023","interfaces":[{"interface":"back","average":{"1min":1.6200000000000001,"5min":1.7050000000000001,"15min":2.5259999999999998},"min":{"1min":0.27500000000000002,"5min":0.23999999999999999,"15min":0.222},"max":{"1min":8.1159999999999997,"5min":14.641,"15min":16.189},"last":0.27400000000000002},{"interface":"front","average":{"1min":1.3280000000000001,"5min":2.121,"15min":2.7440000000000002},"min":{"1min":0.26200000000000001,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":6.1820000000000004,"5min":13.727,"15min":32.603999999999999},"last":0.48699999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:38:20 2023","interfaces":[{"interface":"back","average":{"1min":3.3620000000000001,"5min":2.927,"15min":2.5990000000000002},"min":{"1min":0.25800000000000001,"5min":0.23799999999999999,"15min":0.22800000000000001},"max":{"1min":10.747999999999999,"5min":20.405000000000001,"15min":32.857999999999997},"last":0.25700000000000001},{"interface":"front","average":{"1min":4.9720000000000004,"5min":3.129,"15min":2.5710000000000002},"min":{"1min":0.28100000000000003,"5min":0.251,"15min":0.20899999999999999},"max":{"1min":22.32,"5min":24.146999999999998,"15min":24.146999999999998},"last":1.1519999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:38:20 2023","interfaces":[{"interface":"back","average":{"1min":1.7110000000000001,"5min":1.679,"15min":1.8340000000000001},"min":{"1min":0.377,"5min":0.32200000000000001,"15min":0.29799999999999999},"max":{"1min":5.9930000000000003,"5min":12.151,"15min":46.042999999999999},"last":0.46800000000000003},{"interface":"front","average":{"1min":3.7320000000000002,"5min":2.4750000000000001,"15min":2.351},"min":{"1min":0.373,"5min":0.29399999999999998,"15min":0.28100000000000003},"max":{"1min":13.571999999999999,"5min":14.422000000000001,"15min":46.579999999999998},"last":1.1839999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:38:20 2023","interfaces":[{"interface":"back","average":{"1min":1.75,"5min":1.883,"15min":2.0960000000000001},"min":{"1min":0.40000000000000002,"5min":0.32300000000000001,"15min":0.311},"max":{"1min":6.024,"5min":13.228999999999999,"15min":45.292000000000002},"last":0.501},{"interface":"front","average":{"1min":1.5589999999999999,"5min":1.7110000000000001,"15min":1.9339999999999999},"min":{"1min":0.442,"5min":0.32000000000000001,"15min":0.313},"max":{"1min":4.9770000000000003,"5min":10.300000000000001,"15min":44.679000000000002},"last":0.51200000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:38:20 2023","interfaces":[{"interface":"back","average":{"1min":1.571,"5min":1.671,"15min":1.9359999999999999},"min":{"1min":0.41599999999999998,"5min":0.40500000000000003,"15min":0.34899999999999998},"max":{"1min":5.0620000000000003,"5min":12.464,"15min":17.524999999999999},"last":0.54000000000000004},{"interface":"front","average":{"1min":3.774,"5min":2.5070000000000001,"15min":2.1309999999999998},"min":{"1min":0.47799999999999998,"5min":0.371,"15min":0.33700000000000002},"max":{"1min":13.500999999999999,"5min":14.464,"15min":19.524000000000001},"last":1.2210000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:38:20 2023","interfaces":[{"interface":"back","average":{"1min":4.3659999999999997,"5min":2.637,"15min":2.3380000000000001},"min":{"1min":0.44700000000000001,"5min":0.41499999999999998,"15min":0.34999999999999998},"max":{"1min":13.544,"5min":14.521000000000001,"15min":19.571999999999999},"last":3.4329999999999998},{"interface":"front","average":{"1min":1.762,"5min":1.9610000000000001,"15min":2.1600000000000001},"min":{"1min":0.50700000000000001,"5min":0.44600000000000001,"15min":0.371},"max":{"1min":6.0579999999999998,"5min":12.186,"15min":45.387999999999998},"last":0.53100000000000003}]}]},{"osd":4,"up_from":30,"seq":128849019229,"num_pgs":61,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1715300,"kb_used_data":1367900,"kb_used_omap":0,"kb_used_meta":347392,"kb_avail":92029852,"statfs":{"total":95995035648,"available":94238568448,"internally_reserved":0,"allocated":1400729600,"data_stored":1399407386,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":355729408},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:38:47 2023","interfaces":[{"interface":"back","average":{"1min":2.2959999999999998,"5min":2.0190000000000001,"15min":1.7829999999999999},"min":{"1min":0.28699999999999998,"5min":0.24099999999999999,"15min":0.222},"max":{"1min":15.553000000000001,"5min":24.114999999999998,"15min":24.114999999999998},"last":0.36799999999999999},{"interface":"front","average":{"1min":2.2930000000000001,"5min":2.0449999999999999,"15min":1.9239999999999999},"min":{"1min":0.27700000000000002,"5min":0.246,"15min":0.246},"max":{"1min":15.686,"5min":24.135999999999999,"15min":24.135999999999999},"last":0.38400000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:39:03 2023","interfaces":[{"interface":"back","average":{"1min":1.024,"5min":1.4259999999999999,"15min":1.8109999999999999},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":6.4880000000000004,"5min":18.190999999999999,"15min":18.190999999999999},"last":0.32900000000000001},{"interface":"front","average":{"1min":1.0189999999999999,"5min":1.4910000000000001,"15min":1.7649999999999999},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28199999999999997},"max":{"1min":6.508,"5min":18.173999999999999,"15min":18.173999999999999},"last":0.40899999999999997}]},{"osd":2,"last update":"Sat Sep 23 20:38:58 2023","interfaces":[{"interface":"back","average":{"1min":0.67800000000000005,"5min":1.645,"15min":1.9370000000000001},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.26600000000000001},"max":{"1min":3.9860000000000002,"5min":10.416,"15min":32.363},"last":0.97699999999999998},{"interface":"front","average":{"1min":1.7669999999999999,"5min":2.1800000000000002,"15min":2.3069999999999999},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.27600000000000002},"max":{"1min":6.6710000000000003,"5min":8.4939999999999998,"15min":34.042999999999999},"last":0.48499999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:38:14 2023","interfaces":[{"interface":"back","average":{"1min":1.607,"5min":1.9370000000000001,"15min":2.0950000000000002},"min":{"1min":0.42899999999999999,"5min":0.34899999999999998,"15min":0.29099999999999998},"max":{"1min":7.3170000000000002,"5min":9.5239999999999991,"15min":32.332999999999998},"last":0.65300000000000002},{"interface":"front","average":{"1min":1.9099999999999999,"5min":2.3039999999999998,"15min":2.2189999999999999},"min":{"1min":0.41399999999999998,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":7.2350000000000003,"5min":14.384,"15min":32.399999999999999},"last":0.57499999999999996}]},{"osd":5,"last update":"Sat Sep 23 20:38:42 2023","interfaces":[{"interface":"back","average":{"1min":0.47599999999999998,"5min":0.65200000000000002,"15min":1.5620000000000001},"min":{"1min":0.29999999999999999,"5min":0.26400000000000001,"15min":0.247},"max":{"1min":1.968,"5min":3.5960000000000001,"15min":82.866},"last":0.29399999999999998},{"interface":"front","average":{"1min":0.64500000000000002,"5min":0.77100000000000002,"15min":1.4970000000000001},"min":{"1min":0.33000000000000002,"5min":0.28999999999999998,"15min":0.27800000000000002},"max":{"1min":4.5590000000000002,"5min":4.5590000000000002,"15min":82.546000000000006},"last":0.42999999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:38:58 2023","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.95699999999999996,"15min":1.4390000000000001},"min":{"1min":0.33100000000000002,"5min":0.312,"15min":0.27600000000000002},"max":{"1min":1.2410000000000001,"5min":10.178000000000001,"15min":41.371000000000002},"last":0.34399999999999997},{"interface":"front","average":{"1min":0.75900000000000001,"5min":0.89500000000000002,"15min":1.337},"min":{"1min":0.34499999999999997,"5min":0.29899999999999999,"15min":0.27200000000000002},"max":{"1min":4.7160000000000002,"5min":7.2850000000000001,"15min":41.140999999999998},"last":0.56200000000000006}]},{"osd":7,"last update":"Sat Sep 23 20:38:14 2023","interfaces":[{"interface":"back","average":{"1min":0.64800000000000002,"5min":1.204,"15min":1.5309999999999999},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":3.036,"5min":10.548999999999999,"15min":16.427},"last":0.46100000000000002},{"interface":"front","average":{"1min":0.56599999999999995,"5min":0.81599999999999995,"15min":1.321},"min":{"1min":0.33200000000000002,"5min":0.32700000000000001,"15min":0.307},"max":{"1min":1.9350000000000001,"5min":7.2400000000000002,"15min":19.018999999999998},"last":0.5}]}]},{"osd":5,"up_from":39,"seq":167503724890,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2755540,"kb_used_data":2423052,"kb_used_omap":0,"kb_used_meta":332480,"kb_avail":90989612,"statfs":{"total":95995035648,"available":93173362688,"internally_reserved":0,"allocated":2481205248,"data_stored":2479894880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":340459520},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:38:48 2023","interfaces":[{"interface":"back","average":{"1min":1.5449999999999999,"5min":2.1760000000000002,"15min":1.845},"min":{"1min":0.23899999999999999,"5min":0.23400000000000001,"15min":0.20699999999999999},"max":{"1min":7.819,"5min":18.16,"15min":32.475000000000001},"last":1.0760000000000001},{"interface":"front","average":{"1min":1.6000000000000001,"5min":2.1000000000000001,"15min":1.798},"min":{"1min":0.29999999999999999,"5min":0.29499999999999998,"15min":0.20599999999999999},"max":{"1min":7.9859999999999998,"5min":9.0570000000000004,"15min":32.457000000000001},"last":1.099}]},{"osd":1,"last update":"Sat Sep 23 20:39:07 2023","interfaces":[{"interface":"back","average":{"1min":1.921,"5min":2.1040000000000001,"15min":1.9430000000000001},"min":{"1min":0.28299999999999997,"5min":0.253,"15min":0.253},"max":{"1min":8.6820000000000004,"5min":36.588999999999999,"15min":36.588999999999999},"last":1.2390000000000001},{"interface":"front","average":{"1min":1.613,"5min":2.1869999999999998,"15min":1.718},"min":{"1min":0.313,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":6.4080000000000004,"5min":36.920999999999999,"15min":36.920999999999999},"last":0.91400000000000003}]},{"osd":2,"last update":"Sat Sep 23 20:38:48 2023","interfaces":[{"interface":"back","average":{"1min":2.008,"5min":2.536,"15min":2.1459999999999999},"min":{"1min":0.32200000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":10.743,"5min":37.011000000000003,"15min":37.011000000000003},"last":0.96499999999999997},{"interface":"front","average":{"1min":0.94699999999999995,"5min":2.3769999999999998,"15min":2.1259999999999999},"min":{"1min":0.28999999999999998,"5min":0.255,"15min":0.255},"max":{"1min":3.5510000000000002,"5min":36.835000000000001,"15min":36.835000000000001},"last":1.0529999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:39:14 2023","interfaces":[{"interface":"back","average":{"1min":1.4199999999999999,"5min":2.1499999999999999,"15min":2.2029999999999998},"min":{"1min":0.38500000000000001,"5min":0.314,"15min":0.25700000000000001},"max":{"1min":5.3799999999999999,"5min":36.716000000000001,"15min":36.716000000000001},"last":1.141},{"interface":"front","average":{"1min":3.5190000000000001,"5min":2.6669999999999998,"15min":2.133},"min":{"1min":0.34499999999999997,"5min":0.34399999999999997,"15min":0.28799999999999998},"max":{"1min":18.504000000000001,"5min":18.504000000000001,"15min":18.504000000000001},"last":0.94099999999999995}]},{"osd":4,"last update":"Sat Sep 23 20:38:43 2023","interfaces":[{"interface":"back","average":{"1min":0.61699999999999999,"5min":0.70599999999999996,"15min":1.339},"min":{"1min":0.28699999999999998,"5min":0.255,"15min":0.255},"max":{"1min":3.331,"5min":6.9180000000000001,"15min":28.015999999999998},"last":0.72999999999999998},{"interface":"front","average":{"1min":0.78000000000000003,"5min":0.76300000000000001,"15min":1.1120000000000001},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":3.2999999999999998,"5min":6.8380000000000001,"15min":12.300000000000001},"last":1.6519999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:38:53 2023","interfaces":[{"interface":"back","average":{"1min":0.94299999999999995,"5min":0.88300000000000001,"15min":1.294},"min":{"1min":0.35999999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":5.3559999999999999,"5min":6.9960000000000004,"15min":12.890000000000001},"last":1.7130000000000001},{"interface":"front","average":{"1min":0.91600000000000004,"5min":0.89300000000000002,"15min":1.238},"min":{"1min":0.38700000000000001,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":5.3019999999999996,"5min":6.7930000000000001,"15min":16.265999999999998},"last":1.6970000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:39:14 2023","interfaces":[{"interface":"back","average":{"1min":0.69499999999999995,"5min":0.68000000000000005,"15min":1.2310000000000001},"min":{"1min":0.34100000000000003,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":2.2959999999999998,"5min":4.0380000000000003,"15min":36.621000000000002},"last":0.56399999999999995},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.85599999999999998,"15min":1.1919999999999999},"min":{"1min":0.33200000000000002,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":1.2290000000000001,"5min":6.5970000000000004,"15min":37.331000000000003},"last":0.47999999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4722,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1360,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8206,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7156,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3154,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1477050368,"data_stored":1477050368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1552224256,"data_stored":1552220184,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1368068096,"data_stored":1368064024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1591672832,"data_stored":1591672832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1389101056,"data_stored":1389101056,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2476670976,"data_stored":2476670976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1135017984,"data_stored":1135017984,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1109594112,"data_stored":1109590040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:39:21.882 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd pool set .mgr pgp_num 31 2023-09-23T20:39:21.937 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:39:21.941 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:21 smithi149 ceph-mon[170394]: pgmap v1037: 129 pgs: 7 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 3.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 505 op/s; 19177/162282 objects degraded (11.817%); 343/162282 objects misplaced (0.211%); 12 MiB/s, 190 objects/s recovering 2023-09-23T20:39:21.941 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:21 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 19177/162282 objects degraded (11.817%), 13 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T20:39:21.942 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:21.943 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:39:21.974 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:21 smithi018 ceph-mon[184144]: pgmap v1037: 129 pgs: 7 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 3.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 505 op/s; 19177/162282 objects degraded (11.817%); 343/162282 objects misplaced (0.211%); 12 MiB/s, 190 objects/s recovering 2023-09-23T20:39:21.974 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:21 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 19177/162282 objects degraded (11.817%), 13 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T20:39:21.974 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:21 smithi018 ceph-mon[179647]: pgmap v1037: 129 pgs: 7 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 3.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 505 op/s; 19177/162282 objects degraded (11.817%); 343/162282 objects misplaced (0.211%); 12 MiB/s, 190 objects/s recovering 2023-09-23T20:39:21.974 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:21 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 19177/162282 objects degraded (11.817%), 13 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T20:39:22.044 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:39:22.126 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:22.204 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:39:22.213 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:22.214 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:39:22.315 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:39:22.347 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:22.404 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:39:22.408 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:22.408 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:39:22.509 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:39:22.512 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:39:22.609 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:22.673 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:39:22.685 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:39:22.700 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:22.700 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:39:22.733 INFO:teuthology.orchestra.run.smithi149.stdout:ERROR: (22) Invalid argument 2023-09-23T20:39:22.733 INFO:teuthology.orchestra.run.smithi149.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:39:22.768 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:22 smithi149 ceph-mon[170394]: from='client.36225 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:39:22.801 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:39:22.852 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:22.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:22 smithi018 ceph-mon[184144]: from='client.36225 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:39:22.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:22 smithi018 ceph-mon[179647]: from='client.36225 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:39:23.097 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:39:23.102 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:23.103 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:39:23.149 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T20:39:23.204 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:39:23.237 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:23.289 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:39:23.297 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:23.298 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:39:23.399 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:39:23.450 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:23.499 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:39:23.502 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:23.503 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:39:23.585 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:39:23.604 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:39:23.666 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:23.738 INFO:teuthology.orchestra.run.smithi018.stderr:nodeep-scrub is unset 2023-09-23T20:39:23.755 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:39:23.760 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:23.760 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:39:23.861 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:39:23.870 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:39:23.872 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:23 smithi018 ceph-mon[184144]: pgmap v1038: 129 pgs: 7 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 3.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 501 op/s; 19177/159972 objects degraded (11.988%); 343/159972 objects misplaced (0.214%); 11 MiB/s, 178 objects/s recovering 2023-09-23T20:39:23.872 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:23 smithi018 ceph-mon[184144]: 1.a scrub starts 2023-09-23T20:39:23.872 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:23 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2660594210' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:39:23.872 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:23 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:39:23.872 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:23 smithi018 ceph-mon[179647]: pgmap v1038: 129 pgs: 7 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 3.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 501 op/s; 19177/159972 objects degraded (11.988%); 343/159972 objects misplaced (0.214%); 11 MiB/s, 178 objects/s recovering 2023-09-23T20:39:23.872 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:23 smithi018 ceph-mon[179647]: 1.a scrub starts 2023-09-23T20:39:23.873 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:23 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2660594210' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:39:23.873 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:23 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:39:23.892 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:24.007 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:39:24.009 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:23 smithi149 ceph-mon[170394]: pgmap v1038: 129 pgs: 7 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 3.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 501 op/s; 19177/159972 objects degraded (11.988%); 343/159972 objects misplaced (0.214%); 11 MiB/s, 178 objects/s recovering 2023-09-23T20:39:24.009 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:23 smithi149 ceph-mon[170394]: 1.a scrub starts 2023-09-23T20:39:24.009 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:23 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2660594210' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:39:24.010 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:23 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:39:24.010 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:24.011 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:39:24.111 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:39:24.146 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:24.196 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:39:24.200 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:24.200 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:39:24.301 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:39:24.338 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:24.618 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:39:24.627 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:24.627 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:39:24.728 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:39:24.742 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:24 smithi018 ceph-mon[179647]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:39:24.742 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:24 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:39:24.742 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:24 smithi018 ceph-mon[179647]: osdmap e425: 8 total, 8 up, 8 in 2023-09-23T20:39:24.742 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:24 smithi018 ceph-mon[179647]: failed to encode map e425 with expected crc 2023-09-23T20:39:24.772 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:24.828 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:39:24.834 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:24.834 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:39:24.932 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:24 smithi149 ceph-mon[170394]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:39:24.933 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:24 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:39:24.933 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:24 smithi149 ceph-mon[170394]: osdmap e425: 8 total, 8 up, 8 in 2023-09-23T20:39:24.933 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:24 smithi149 ceph-mon[170394]: failed to encode map e425 with expected crc 2023-09-23T20:39:24.935 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:39:24.992 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:24 smithi018 ceph-mon[184144]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:39:24.993 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:24 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:39:24.993 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:24 smithi018 ceph-mon[184144]: osdmap e425: 8 total, 8 up, 8 in 2023-09-23T20:39:24.993 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:24 smithi018 ceph-mon[184144]: failed to encode map e425 with expected crc 2023-09-23T20:39:25.001 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:25.070 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:39:25.073 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:25.073 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:39:25.175 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:39:25.186 INFO:teuthology.orchestra.run.smithi149.stdout:ERROR: (22) Invalid argument 2023-09-23T20:39:25.186 INFO:teuthology.orchestra.run.smithi149.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:39:25.209 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:25.258 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:39:25.268 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:25.268 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:39:25.296 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:39:24 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:39:24] "GET /metrics HTTP/1.1" 200 36563 "" "Prometheus/2.43.0" 2023-09-23T20:39:25.369 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:39:25.401 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:25.446 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:39:25.451 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:25.452 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:39:25.539 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:39:25.553 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:39:25.561 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:39:25.586 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:39:25.589 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:25.611 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:39:25.637 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:39:25.664 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:39:25.695 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:39:25.720 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:39:25.727 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:39:25.727 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:39:25.727 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:39:25.727 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:39:25.727 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:39:25.727 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:39:25.727 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:39:25.727 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:39:25.767 INFO:teuthology.orchestra.run.smithi018.stderr:set pool 1 pgp_num to 31 2023-09-23T20:39:25.782 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:39:25.786 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:25.786 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:39:25.871 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:25 smithi149 ceph-mon[170394]: pgmap v1040: 129 pgs: 7 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 3.2 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 510 op/s; 18947/156714 objects degraded (12.090%); 343/156714 objects misplaced (0.219%); 9.0 MiB/s, 144 objects/s recovering 2023-09-23T20:39:25.871 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:25 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1306930334' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "31"}]: dispatch 2023-09-23T20:39:25.871 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:25 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3023454616' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:39:25.872 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T20:39:25.887 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:39:25.923 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:26.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:25 smithi018 ceph-mon[179647]: pgmap v1040: 129 pgs: 7 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 3.2 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 510 op/s; 18947/156714 objects degraded (12.090%); 343/156714 objects misplaced (0.219%); 9.0 MiB/s, 144 objects/s recovering 2023-09-23T20:39:26.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:25 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1306930334' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "31"}]: dispatch 2023-09-23T20:39:26.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:25 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3023454616' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:39:26.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:25 smithi018 ceph-mon[184144]: pgmap v1040: 129 pgs: 7 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 3.2 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 510 op/s; 18947/156714 objects degraded (12.090%); 343/156714 objects misplaced (0.219%); 9.0 MiB/s, 144 objects/s recovering 2023-09-23T20:39:26.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:25 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1306930334' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "31"}]: dispatch 2023-09-23T20:39:26.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:25 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3023454616' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:39:26.217 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:39:26.224 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:26.225 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:39:26.326 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:39:26.356 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:26.402 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:39:26.406 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:26.406 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:39:26.507 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:39:26.541 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:26.576 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:39:26.590 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:39:26.594 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:26.594 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:39:26.695 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:39:26.723 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:26.770 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:39:26.774 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:26.774 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:39:26.807 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:26 smithi149 ceph-mon[170394]: failed to encode map e426 with expected crc 2023-09-23T20:39:26.807 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:26 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1306930334' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "31"}]': finished 2023-09-23T20:39:26.807 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:26 smithi149 ceph-mon[170394]: osdmap e426: 8 total, 8 up, 8 in 2023-09-23T20:39:26.808 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:26 smithi149 ceph-mon[170394]: failed to encode map e426 with expected crc 2023-09-23T20:39:26.808 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:26 smithi149 ceph-mon[170394]: failed to encode map e426 with expected crc 2023-09-23T20:39:26.808 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:26 smithi149 ceph-mon[170394]: failed to encode map e426 with expected crc 2023-09-23T20:39:26.808 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:26 smithi149 ceph-mon[170394]: pgmap v1042: 129 pgs: 7 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 3.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 499 op/s; 18335/153603 objects degraded (11.937%); 12 MiB/s, 244 objects/s recovering 2023-09-23T20:39:26.808 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:26 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 18947/156714 objects degraded (12.090%), 13 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T20:39:26.808 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:26 smithi149 ceph-mon[170394]: 1.10 scrub starts 2023-09-23T20:39:26.875 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:39:26.934 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:26.989 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:39:26.994 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:26.994 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:39:27.076 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T20:39:27.095 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:39:27.127 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:26 smithi018 ceph-mon[179647]: failed to encode map e426 with expected crc 2023-09-23T20:39:27.127 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:26 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1306930334' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "31"}]': finished 2023-09-23T20:39:27.127 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:26 smithi018 ceph-mon[179647]: osdmap e426: 8 total, 8 up, 8 in 2023-09-23T20:39:27.127 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:26 smithi018 ceph-mon[179647]: failed to encode map e426 with expected crc 2023-09-23T20:39:27.127 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:26 smithi018 ceph-mon[179647]: failed to encode map e426 with expected crc 2023-09-23T20:39:27.127 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:26 smithi018 ceph-mon[179647]: failed to encode map e426 with expected crc 2023-09-23T20:39:27.127 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:26 smithi018 ceph-mon[179647]: pgmap v1042: 129 pgs: 7 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 3.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 499 op/s; 18335/153603 objects degraded (11.937%); 12 MiB/s, 244 objects/s recovering 2023-09-23T20:39:27.128 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:26 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 18947/156714 objects degraded (12.090%), 13 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T20:39:27.128 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:26 smithi018 ceph-mon[179647]: 1.10 scrub starts 2023-09-23T20:39:27.128 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:26 smithi018 ceph-mon[184144]: failed to encode map e426 with expected crc 2023-09-23T20:39:27.128 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:26 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1306930334' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "31"}]': finished 2023-09-23T20:39:27.128 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:26 smithi018 ceph-mon[184144]: osdmap e426: 8 total, 8 up, 8 in 2023-09-23T20:39:27.128 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:26 smithi018 ceph-mon[184144]: failed to encode map e426 with expected crc 2023-09-23T20:39:27.128 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:26 smithi018 ceph-mon[184144]: failed to encode map e426 with expected crc 2023-09-23T20:39:27.128 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:26 smithi018 ceph-mon[184144]: failed to encode map e426 with expected crc 2023-09-23T20:39:27.129 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:26 smithi018 ceph-mon[184144]: pgmap v1042: 129 pgs: 7 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 3.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 499 op/s; 18335/153603 objects degraded (11.937%); 12 MiB/s, 244 objects/s recovering 2023-09-23T20:39:27.129 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:26 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 18947/156714 objects degraded (12.090%), 13 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T20:39:27.129 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:26 smithi018 ceph-mon[184144]: 1.10 scrub starts 2023-09-23T20:39:27.178 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:27.230 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:39:27.235 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:27.236 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:39:27.336 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:39:27.392 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:27.449 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:39:27.456 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:27.457 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:39:27.558 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:39:27.602 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:27.651 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:39:27.654 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:27.654 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:39:27.755 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:39:27.788 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:27.829 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:39:27.833 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:27.834 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:39:27.848 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:39:27.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:27 smithi018 ceph-mon[179647]: 1.a scrub starts 2023-09-23T20:39:27.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:27 smithi018 ceph-mon[179647]: failed to encode map e427 with expected crc 2023-09-23T20:39:27.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:27 smithi018 ceph-mon[179647]: osdmap e427: 8 total, 8 up, 8 in 2023-09-23T20:39:27.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:27 smithi018 ceph-mon[179647]: failed to encode map e427 with expected crc 2023-09-23T20:39:27.894 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:27 smithi018 ceph-mon[184144]: 1.a scrub starts 2023-09-23T20:39:27.894 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:27 smithi018 ceph-mon[184144]: failed to encode map e427 with expected crc 2023-09-23T20:39:27.894 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:27 smithi018 ceph-mon[184144]: osdmap e427: 8 total, 8 up, 8 in 2023-09-23T20:39:27.894 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:27 smithi018 ceph-mon[184144]: failed to encode map e427 with expected crc 2023-09-23T20:39:27.934 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:39:27.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:27 smithi149 ceph-mon[170394]: 1.a scrub starts 2023-09-23T20:39:27.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:27 smithi149 ceph-mon[170394]: failed to encode map e427 with expected crc 2023-09-23T20:39:27.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:27 smithi149 ceph-mon[170394]: osdmap e427: 8 total, 8 up, 8 in 2023-09-23T20:39:27.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:27 smithi149 ceph-mon[170394]: failed to encode map e427 with expected crc 2023-09-23T20:39:27.976 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:27.993 INFO:teuthology.orchestra.run.smithi149.stdout:{ 2023-09-23T20:39:27.994 INFO:teuthology.orchestra.run.smithi149.stdout: "size": 20, 2023-09-23T20:39:27.994 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 600, 2023-09-23T20:39:27.994 INFO:teuthology.orchestra.run.smithi149.stdout: "ops": [ 2023-09-23T20:39:27.994 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:27.994 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:84069 8.9 8:9824f39b:::benchmark_data_smithi018_341490_object84068:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e378)", 2023-09-23T20:39:27.994 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:37:13.327117+0000", 2023-09-23T20:39:27.994 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 134.655483895, 2023-09-23T20:39:27.994 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.3519245670000002, 2023-09-23T20:39:27.994 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:39:27.994 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:39:27.995 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:39:27.995 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:39:27.995 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:39:27.995 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 84069 2023-09-23T20:39:27.995 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:27.995 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:39:27.995 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:27.995 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:39:27.995 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.327117+0000", 2023-09-23T20:39:27.995 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:39:27.995 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:27.996 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:27.996 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:39:27.996 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.327117+0000", 2023-09-23T20:39:27.996 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:39:27.996 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:27.996 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:27.996 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:39:27.996 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.327116+0000", 2023-09-23T20:39:27.996 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:39:27.996 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:27.996 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:27.996 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:39:27.997 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.327148+0000", 2023-09-23T20:39:27.997 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.1741e-05 2023-09-23T20:39:27.997 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:27.997 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:27.997 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:39:27.997 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.327160+0000", 2023-09-23T20:39:27.997 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.185e-05 2023-09-23T20:39:27.997 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:27.997 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:27.997 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:39:27.997 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.327165+0000", 2023-09-23T20:39:27.997 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.1749999999999996e-06 2023-09-23T20:39:27.998 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:27.998 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:27.998 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:39:27.998 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.330726+0000", 2023-09-23T20:39:27.998 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00356121 2023-09-23T20:39:27.998 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:27.998 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:27.998 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:39:27.998 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.330874+0000", 2023-09-23T20:39:27.998 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000148097 2023-09-23T20:39:27.998 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:27.998 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:27.999 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:39:27.999 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.331021+0000", 2023-09-23T20:39:27.999 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00014648499999999999 2023-09-23T20:39:27.999 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:27.999 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:27.999 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:39:27.999 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.334676+0000", 2023-09-23T20:39:27.999 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0036551869999999998 2023-09-23T20:39:27.999 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:27.999 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:27.999 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:39:27.999 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.335154+0000", 2023-09-23T20:39:28.000 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00047863799999999998 2023-09-23T20:39:28.000 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.000 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.000 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:39:28.000 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.368539+0000", 2023-09-23T20:39:28.000 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.033384692000000001 2023-09-23T20:39:28.000 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.000 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.000 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:39:28.000 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.368550+0000", 2023-09-23T20:39:28.000 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1178e-05 2023-09-23T20:39:28.000 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.001 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.001 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:39:28.001 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:14.362533+0000", 2023-09-23T20:39:28.001 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.993982649 2023-09-23T20:39:28.001 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.001 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.001 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:39:28.001 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:14.362546+0000", 2023-09-23T20:39:28.001 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.2660000000000001e-05 2023-09-23T20:39:28.001 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.001 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.001 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:39:28.002 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:15.678962+0000", 2023-09-23T20:39:28.002 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.316415911 2023-09-23T20:39:28.002 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.002 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.002 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:39:28.002 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:15.679041+0000", 2023-09-23T20:39:28.002 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.9655999999999998e-05 2023-09-23T20:39:28.002 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:39:28.002 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:39:28.002 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:39:28.002 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.003 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.003 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:84081 8.9 8:97fd4d67:::benchmark_data_smithi018_341490_object84080:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e378)", 2023-09-23T20:39:28.003 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:37:13.341077+0000", 2023-09-23T20:39:28.003 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 134.64152382399999, 2023-09-23T20:39:28.003 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.3380652359999998, 2023-09-23T20:39:28.003 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:39:28.003 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:39:28.003 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:39:28.003 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:39:28.003 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:39:28.003 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 84081 2023-09-23T20:39:28.004 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.004 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:39:28.004 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.004 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:39:28.004 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.341077+0000", 2023-09-23T20:39:28.004 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:39:28.004 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.004 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.004 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:39:28.004 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.341077+0000", 2023-09-23T20:39:28.004 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:39:28.004 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.005 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.005 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:39:28.005 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.341075+0000", 2023-09-23T20:39:28.005 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999981 2023-09-23T20:39:28.005 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.005 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.005 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:39:28.005 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.341116+0000", 2023-09-23T20:39:28.005 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.1675000000000003e-05 2023-09-23T20:39:28.005 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.005 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.005 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:39:28.006 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.341135+0000", 2023-09-23T20:39:28.006 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.8110000000000001e-05 2023-09-23T20:39:28.006 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.006 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.006 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:39:28.006 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.341141+0000", 2023-09-23T20:39:28.006 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.849e-06 2023-09-23T20:39:28.006 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.006 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.006 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:39:28.006 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.341194+0000", 2023-09-23T20:39:28.006 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.2553000000000003e-05 2023-09-23T20:39:28.007 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.007 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.007 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:39:28.007 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.341313+0000", 2023-09-23T20:39:28.007 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00011935899999999999 2023-09-23T20:39:28.007 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.007 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.007 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:39:28.007 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.341466+0000", 2023-09-23T20:39:28.007 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00015230300000000001 2023-09-23T20:39:28.007 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.007 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.008 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:39:28.008 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.344704+0000", 2023-09-23T20:39:28.008 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0032382510000000002 2023-09-23T20:39:28.008 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.008 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.008 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:39:28.008 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.348055+0000", 2023-09-23T20:39:28.008 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.003350872 2023-09-23T20:39:28.008 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.008 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.008 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:39:28.008 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.368594+0000", 2023-09-23T20:39:28.009 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.020538948000000001 2023-09-23T20:39:28.009 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.009 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.009 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:39:28.009 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.368601+0000", 2023-09-23T20:39:28.009 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.6529999999999994e-06 2023-09-23T20:39:28.009 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.009 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.009 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:39:28.009 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:14.362580+0000", 2023-09-23T20:39:28.009 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.99397867299999998 2023-09-23T20:39:28.009 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.010 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.010 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:39:28.010 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:14.362589+0000", 2023-09-23T20:39:28.010 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.9509999999999995e-06 2023-09-23T20:39:28.010 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.010 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.010 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:39:28.010 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:15.679073+0000", 2023-09-23T20:39:28.010 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.316483598 2023-09-23T20:39:28.010 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.010 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.011 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:39:28.011 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:15.679142+0000", 2023-09-23T20:39:28.011 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.9389999999999998e-05 2023-09-23T20:39:28.011 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:39:28.011 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:39:28.011 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:39:28.011 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.011 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.011 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:105775 8.9 8:97ec68b7:::benchmark_data_smithi018_341490_object16452:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e411)", 2023-09-23T20:39:28.011 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:38:27.408390+0000", 2023-09-23T20:39:28.011 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 60.574210032000003, 2023-09-23T20:39:28.011 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.1501579759999991, 2023-09-23T20:39:28.012 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:39:28.012 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:39:28.012 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:39:28.012 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:39:28.012 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:39:28.012 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 105775 2023-09-23T20:39:28.012 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.012 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:39:28.012 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.012 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:39:28.012 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.408390+0000", 2023-09-23T20:39:28.012 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:39:28.013 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.013 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.013 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:39:28.013 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.408390+0000", 2023-09-23T20:39:28.013 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:39:28.013 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.013 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.013 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:39:28.013 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.408389+0000", 2023-09-23T20:39:28.013 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:39:28.013 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.013 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.014 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:39:28.014 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.408397+0000", 2023-09-23T20:39:28.014 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.7009999999999996e-06 2023-09-23T20:39:28.014 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.014 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.014 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:39:28.014 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.408411+0000", 2023-09-23T20:39:28.014 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.4228e-05 2023-09-23T20:39:28.014 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.014 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.014 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:39:28.015 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.408419+0000", 2023-09-23T20:39:28.015 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.5599999999999994e-06 2023-09-23T20:39:28.015 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.015 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.015 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:39:28.015 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.414829+0000", 2023-09-23T20:39:28.015 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0064094740000000001 2023-09-23T20:39:28.015 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.015 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.015 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:39:28.015 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.414843+0000", 2023-09-23T20:39:28.016 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.4561e-05 2023-09-23T20:39:28.016 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.016 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.016 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:39:28.016 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.556066+0000", 2023-09-23T20:39:28.016 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.1412221710000008 2023-09-23T20:39:28.016 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.016 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.016 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:39:28.016 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.558548+0000", 2023-09-23T20:39:28.016 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0024828070000000001 2023-09-23T20:39:28.016 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:39:28.017 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:39:28.017 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:39:28.017 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.017 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.017 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:105788 8.9 8:94f5ae14:::benchmark_data_smithi018_341490_object16465:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e411)", 2023-09-23T20:39:28.017 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:38:27.415518+0000", 2023-09-23T20:39:28.017 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 60.567082116000002, 2023-09-23T20:39:28.017 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 10.492815438999999, 2023-09-23T20:39:28.017 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:39:28.017 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:39:28.017 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:39:28.017 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:39:28.018 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:39:28.018 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 105788 2023-09-23T20:39:28.018 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.018 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:39:28.018 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.018 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:39:28.018 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.415518+0000", 2023-09-23T20:39:28.018 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:39:28.018 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.018 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.018 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:39:28.019 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.415518+0000", 2023-09-23T20:39:28.019 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:39:28.019 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.019 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.019 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:39:28.019 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.415517+0000", 2023-09-23T20:39:28.019 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:39:28.019 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.019 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.019 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:39:28.019 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.415524+0000", 2023-09-23T20:39:28.019 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.0530000000000001e-06 2023-09-23T20:39:28.020 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.020 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.020 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:39:28.020 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.415539+0000", 2023-09-23T20:39:28.020 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.4847e-05 2023-09-23T20:39:28.020 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.020 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.020 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:39:28.020 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.415548+0000", 2023-09-23T20:39:28.020 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.1600000000000004e-06 2023-09-23T20:39:28.020 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.020 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.021 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:39:28.021 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.423501+0000", 2023-09-23T20:39:28.021 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0079528459999999995 2023-09-23T20:39:28.021 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.021 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.021 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:39:28.021 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.423513+0000", 2023-09-23T20:39:28.021 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.2523e-05 2023-09-23T20:39:28.021 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.021 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.021 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:39:28.021 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.559074+0000", 2023-09-23T20:39:28.022 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.1355603290000005 2023-09-23T20:39:28.022 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.022 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.022 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for degraded object", 2023-09-23T20:39:28.022 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.567834+0000", 2023-09-23T20:39:28.022 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0087603060000000007 2023-09-23T20:39:28.022 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.022 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.022 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:39:28.022 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.866509+0000", 2023-09-23T20:39:28.022 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.2986752469999998 2023-09-23T20:39:28.022 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.023 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.023 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:39:28.023 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.869596+0000", 2023-09-23T20:39:28.023 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0030866829999999998 2023-09-23T20:39:28.023 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.023 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.023 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:39:28.023 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.875831+0000", 2023-09-23T20:39:28.023 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0062352570000000001 2023-09-23T20:39:28.023 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.023 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.023 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:39:28.024 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.893196+0000", 2023-09-23T20:39:28.024 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.017365463000000001 2023-09-23T20:39:28.024 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.024 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.024 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:39:28.024 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.898820+0000", 2023-09-23T20:39:28.024 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0056230400000000002 2023-09-23T20:39:28.024 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.024 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.024 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:39:28.024 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.899854+0000", 2023-09-23T20:39:28.024 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0010348670000000001 2023-09-23T20:39:28.025 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.025 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.025 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:39:28.025 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.901497+0000", 2023-09-23T20:39:28.025 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0016428160000000001 2023-09-23T20:39:28.025 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.025 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.025 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:39:28.025 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.908334+0000", 2023-09-23T20:39:28.025 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0068366219999999997 2023-09-23T20:39:28.025 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:39:28.026 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:39:28.026 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:39:28.026 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.026 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.026 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:105815 8.19 8:9f25a704:::benchmark_data_smithi018_341490_object16492:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e411)", 2023-09-23T20:39:28.026 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:38:27.621206+0000", 2023-09-23T20:39:28.026 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 60.361394896, 2023-09-23T20:39:28.026 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 10.285912701000001, 2023-09-23T20:39:28.026 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:39:28.026 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:39:28.027 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:39:28.027 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:39:28.027 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:39:28.027 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 105815 2023-09-23T20:39:28.027 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.027 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:39:28.027 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.027 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:39:28.027 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.621206+0000", 2023-09-23T20:39:28.027 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:39:28.027 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.027 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.027 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:39:28.028 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.621206+0000", 2023-09-23T20:39:28.028 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:39:28.028 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.028 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.028 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:39:28.028 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.621204+0000", 2023-09-23T20:39:28.028 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999981 2023-09-23T20:39:28.028 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.028 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.028 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:39:28.028 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.621213+0000", 2023-09-23T20:39:28.028 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.6399999999999992e-06 2023-09-23T20:39:28.029 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.029 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.029 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:39:28.029 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.621236+0000", 2023-09-23T20:39:28.029 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.2016000000000001e-05 2023-09-23T20:39:28.029 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.029 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.029 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:39:28.029 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.621244+0000", 2023-09-23T20:39:28.029 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.7339999999999997e-06 2023-09-23T20:39:28.030 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.030 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.030 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:39:28.030 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.621311+0000", 2023-09-23T20:39:28.030 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.7037000000000002e-05 2023-09-23T20:39:28.030 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.030 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.030 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:39:28.030 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.621338+0000", 2023-09-23T20:39:28.030 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.6506999999999999e-05 2023-09-23T20:39:28.030 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.030 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.031 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:39:28.031 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.559059+0000", 2023-09-23T20:39:28.031 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.9377208670000003 2023-09-23T20:39:28.031 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.031 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.031 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for degraded object", 2023-09-23T20:39:28.031 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.566949+0000", 2023-09-23T20:39:28.031 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0078899190000000004 2023-09-23T20:39:28.031 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.031 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.031 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:39:28.031 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.857769+0000", 2023-09-23T20:39:28.032 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.290820509 2023-09-23T20:39:28.032 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.032 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.032 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:39:28.032 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.860457+0000", 2023-09-23T20:39:28.032 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0026880620000000002 2023-09-23T20:39:28.032 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.032 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.032 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:39:28.032 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.865930+0000", 2023-09-23T20:39:28.032 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0054732979999999997 2023-09-23T20:39:28.033 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.033 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.033 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:39:28.033 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.881728+0000", 2023-09-23T20:39:28.033 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.015797813000000001 2023-09-23T20:39:28.033 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.033 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.033 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:39:28.033 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.897756+0000", 2023-09-23T20:39:28.033 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.016027501 2023-09-23T20:39:28.033 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.033 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.034 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:39:28.034 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.898850+0000", 2023-09-23T20:39:28.034 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.001094292 2023-09-23T20:39:28.034 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.034 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.034 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:39:28.034 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.900364+0000", 2023-09-23T20:39:28.034 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0015141250000000001 2023-09-23T20:39:28.034 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.034 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.034 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:39:28.035 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.907118+0000", 2023-09-23T20:39:28.035 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0067541490000000001 2023-09-23T20:39:28.035 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:39:28.035 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:39:28.035 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:39:28.035 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.035 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.035 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:105818 8.19 8:9e8d5bb0:::benchmark_data_smithi018_341490_object16495:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e411)", 2023-09-23T20:39:28.035 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:38:27.629200+0000", 2023-09-23T20:39:28.035 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 60.353400815000001, 2023-09-23T20:39:28.035 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 10.286819241, 2023-09-23T20:39:28.035 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:39:28.036 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:39:28.036 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:39:28.036 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:39:28.036 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:39:28.036 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 105818 2023-09-23T20:39:28.036 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.036 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:39:28.036 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.036 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:39:28.036 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.629200+0000", 2023-09-23T20:39:28.036 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:39:28.036 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.037 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.037 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:39:28.037 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.629200+0000", 2023-09-23T20:39:28.037 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:39:28.037 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.037 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.037 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:39:28.037 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.629198+0000", 2023-09-23T20:39:28.037 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999981 2023-09-23T20:39:28.037 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.037 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.037 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:39:28.038 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.629208+0000", 2023-09-23T20:39:28.038 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.9119999999999993e-06 2023-09-23T20:39:28.038 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.038 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.038 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:39:28.038 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.629231+0000", 2023-09-23T20:39:28.038 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.3258999999999999e-05 2023-09-23T20:39:28.038 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.038 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.038 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:39:28.038 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.629240+0000", 2023-09-23T20:39:28.039 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.5719999999999992e-06 2023-09-23T20:39:28.039 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.039 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.039 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:39:28.039 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.629296+0000", 2023-09-23T20:39:28.039 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.5918999999999998e-05 2023-09-23T20:39:28.039 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.039 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.039 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:39:28.039 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.629305+0000", 2023-09-23T20:39:28.039 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.0909999999999999e-06 2023-09-23T20:39:28.040 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.040 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.040 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:39:28.040 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.567292+0000", 2023-09-23T20:39:28.040 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.937986414 2023-09-23T20:39:28.040 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.040 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.040 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for degraded object", 2023-09-23T20:39:28.040 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.576261+0000", 2023-09-23T20:39:28.040 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0089692720000000004 2023-09-23T20:39:28.040 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.040 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.041 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:39:28.041 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.882072+0000", 2023-09-23T20:39:28.041 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.3058112469999998 2023-09-23T20:39:28.041 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.041 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.041 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:39:28.041 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.884583+0000", 2023-09-23T20:39:28.041 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0025109939999999999 2023-09-23T20:39:28.041 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.041 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.041 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:39:28.042 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.889724+0000", 2023-09-23T20:39:28.042 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0051411909999999998 2023-09-23T20:39:28.042 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.042 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.042 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:39:28.042 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.894676+0000", 2023-09-23T20:39:28.042 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0049515119999999999 2023-09-23T20:39:28.042 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.042 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.042 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:39:28.042 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.908009+0000", 2023-09-23T20:39:28.043 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.013332899 2023-09-23T20:39:28.043 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.043 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.043 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:39:28.043 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.908929+0000", 2023-09-23T20:39:28.043 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00092006300000000004 2023-09-23T20:39:28.043 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.043 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.043 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:39:28.043 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.910490+0000", 2023-09-23T20:39:28.043 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.001560931 2023-09-23T20:39:28.044 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.044 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.044 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:39:28.044 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.916019+0000", 2023-09-23T20:39:28.044 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0055290570000000004 2023-09-23T20:39:28.044 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:39:28.044 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:39:28.044 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:39:28.044 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.044 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.044 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "MOSDPGPushReply(8.19 413/410 [PushReplyOp(8:9f25a704:::benchmark_data_smithi018_341490_object16492:head)])", 2023-09-23T20:39:28.044 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:38:35.569213+0000", 2023-09-23T20:39:28.045 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 52.413387528000001, 2023-09-23T20:39:28.045 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.2881636309999998, 2023-09-23T20:39:28.045 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:39:28.045 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:39:28.045 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:39:28.045 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.045 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:39:28.045 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.569213+0000", 2023-09-23T20:39:28.045 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:39:28.045 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.045 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.045 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:39:28.046 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.569213+0000", 2023-09-23T20:39:28.046 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:39:28.046 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.046 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.046 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:39:28.046 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.569213+0000", 2023-09-23T20:39:28.046 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:39:28.046 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.046 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.046 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:39:28.046 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.569217+0000", 2023-09-23T20:39:28.046 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.5480000000000003e-06 2023-09-23T20:39:28.047 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.047 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.047 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:39:28.047 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.569231+0000", 2023-09-23T20:39:28.047 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.3505e-05 2023-09-23T20:39:28.047 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.047 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.047 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:39:28.047 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.569241+0000", 2023-09-23T20:39:28.047 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0695e-05 2023-09-23T20:39:28.047 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.048 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.048 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:39:28.048 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.845386+0000", 2023-09-23T20:39:28.048 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.2761449049999998 2023-09-23T20:39:28.048 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.048 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.048 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:39:28.048 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.857377+0000", 2023-09-23T20:39:28.048 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.011990341999999999 2023-09-23T20:39:28.048 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:39:28.048 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:39:28.048 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:39:28.049 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.049 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.049 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "MOSDPGPushReply(8.9 413/411 [PushReplyOp(8:94f5ae14:::benchmark_data_smithi018_341490_object16465:head)])", 2023-09-23T20:39:28.049 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:38:35.570981+0000", 2023-09-23T20:39:28.049 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 52.411619344000002, 2023-09-23T20:39:28.049 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.295177131, 2023-09-23T20:39:28.049 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:39:28.049 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:39:28.049 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:39:28.049 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.049 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:39:28.049 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.570981+0000", 2023-09-23T20:39:28.050 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:39:28.050 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.050 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.050 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:39:28.050 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.570981+0000", 2023-09-23T20:39:28.050 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:39:28.050 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.050 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.050 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:39:28.050 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.570981+0000", 2023-09-23T20:39:28.050 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:39:28.050 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.051 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.051 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:39:28.051 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.570985+0000", 2023-09-23T20:39:28.051 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.2690000000000001e-06 2023-09-23T20:39:28.051 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.051 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.051 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:39:28.051 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.570995+0000", 2023-09-23T20:39:28.051 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.9799999999999993e-06 2023-09-23T20:39:28.051 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.051 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.051 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:39:28.052 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.570999+0000", 2023-09-23T20:39:28.052 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.5739999999999997e-06 2023-09-23T20:39:28.052 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.052 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.052 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:39:28.052 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.852442+0000", 2023-09-23T20:39:28.052 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.2814426320000001 2023-09-23T20:39:28.052 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.052 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.052 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:39:28.052 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.866158+0000", 2023-09-23T20:39:28.052 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.013716178000000001 2023-09-23T20:39:28.053 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:39:28.053 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:39:28.053 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:39:28.053 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.053 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.053 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "MOSDPGPushReply(8.19 413/410 [PushReplyOp(8:9e8d5bb0:::benchmark_data_smithi018_341490_object16495:head)])", 2023-09-23T20:39:28.053 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:38:35.577963+0000", 2023-09-23T20:39:28.053 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 52.404637186999999, 2023-09-23T20:39:28.053 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.303767224, 2023-09-23T20:39:28.053 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:39:28.053 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:39:28.053 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:39:28.054 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.054 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:39:28.054 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.577963+0000", 2023-09-23T20:39:28.054 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:39:28.054 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.054 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.054 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:39:28.054 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.577963+0000", 2023-09-23T20:39:28.054 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:39:28.054 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.054 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.055 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:39:28.055 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.577963+0000", 2023-09-23T20:39:28.055 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:39:28.055 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.055 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.055 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:39:28.055 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.577968+0000", 2023-09-23T20:39:28.055 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.13e-06 2023-09-23T20:39:28.055 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.055 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.055 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:39:28.055 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.577980+0000", 2023-09-23T20:39:28.056 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1749000000000001e-05 2023-09-23T20:39:28.056 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.056 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.056 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:39:28.056 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.577986+0000", 2023-09-23T20:39:28.056 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.8210000000000004e-06 2023-09-23T20:39:28.056 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.056 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.056 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:39:28.056 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.869571+0000", 2023-09-23T20:39:28.056 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.2915846719999999 2023-09-23T20:39:28.056 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.057 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.057 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:39:28.057 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.881731+0000", 2023-09-23T20:39:28.057 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.012159467 2023-09-23T20:39:28.057 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:39:28.057 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:39:28.057 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:39:28.057 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.057 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.057 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:106043 8.9 8:949649ba:::benchmark_data_smithi018_341490_object16720:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e414)", 2023-09-23T20:39:28.057 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:38:35.582779+0000", 2023-09-23T20:39:28.057 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 52.399821492999997, 2023-09-23T20:39:28.058 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.7561004659999999, 2023-09-23T20:39:28.058 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:39:28.058 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:39:28.058 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:39:28.058 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:39:28.058 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:39:28.058 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 106043 2023-09-23T20:39:28.058 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.058 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:39:28.058 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.058 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:39:28.058 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.582779+0000", 2023-09-23T20:39:28.059 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:39:28.059 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.059 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.059 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:39:28.059 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.582779+0000", 2023-09-23T20:39:28.059 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:39:28.059 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.059 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.059 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:39:28.059 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.582778+0000", 2023-09-23T20:39:28.059 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:39:28.059 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.060 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.060 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:39:28.060 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.582784+0000", 2023-09-23T20:39:28.060 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.0050000000000001e-06 2023-09-23T20:39:28.060 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.060 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.060 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:39:28.060 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.582798+0000", 2023-09-23T20:39:28.060 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.3788000000000001e-05 2023-09-23T20:39:28.060 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.060 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.061 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:39:28.061 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.582804+0000", 2023-09-23T20:39:28.061 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.6760000000000001e-06 2023-09-23T20:39:28.061 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.061 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.061 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:39:28.061 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.238120+0000", 2023-09-23T20:39:28.061 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.655315308 2023-09-23T20:39:28.061 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.061 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.061 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for degraded object", 2023-09-23T20:39:28.061 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.246651+0000", 2023-09-23T20:39:28.062 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0085312070000000007 2023-09-23T20:39:28.062 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.062 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.062 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:39:28.062 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.290876+0000", 2023-09-23T20:39:28.062 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.044225546999999997 2023-09-23T20:39:28.062 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.062 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.062 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:39:28.062 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.294002+0000", 2023-09-23T20:39:28.062 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0031256779999999998 2023-09-23T20:39:28.062 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.063 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.063 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:39:28.063 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.300293+0000", 2023-09-23T20:39:28.063 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0062908959999999998 2023-09-23T20:39:28.063 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.063 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.063 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:39:28.063 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.328008+0000", 2023-09-23T20:39:28.063 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.027715281000000001 2023-09-23T20:39:28.063 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.063 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.064 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:39:28.064 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.329088+0000", 2023-09-23T20:39:28.064 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.001079775 2023-09-23T20:39:28.064 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.064 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.064 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:39:28.064 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.330303+0000", 2023-09-23T20:39:28.064 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0012148199999999999 2023-09-23T20:39:28.064 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.064 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.064 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:39:28.064 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.332003+0000", 2023-09-23T20:39:28.065 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0016998580000000001 2023-09-23T20:39:28.065 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.065 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.065 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:39:28.065 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.338879+0000", 2023-09-23T20:39:28.065 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0068769119999999998 2023-09-23T20:39:28.065 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:39:28.065 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:39:28.065 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:39:28.065 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.065 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.065 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:106806 8.19 8:9c697886:::benchmark_data_smithi018_341490_object17483:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e418)", 2023-09-23T20:39:28.066 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:38:40.203477+0000", 2023-09-23T20:39:28.066 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 47.779123783000003, 2023-09-23T20:39:28.066 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.58349355700000005, 2023-09-23T20:39:28.066 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:39:28.066 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:39:28.066 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:39:28.066 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:39:28.066 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:39:28.066 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 106806 2023-09-23T20:39:28.066 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.066 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:39:28.067 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.067 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:39:28.067 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:40.203477+0000", 2023-09-23T20:39:28.067 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:39:28.067 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.067 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.067 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:39:28.067 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:40.203477+0000", 2023-09-23T20:39:28.067 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:39:28.067 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.067 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.067 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:39:28.068 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:40.203475+0000", 2023-09-23T20:39:28.068 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:39:28.068 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.068 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.068 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:39:28.068 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:40.203482+0000", 2023-09-23T20:39:28.068 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.2190000000000001e-06 2023-09-23T20:39:28.068 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.068 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.069 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:39:28.069 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:40.203496+0000", 2023-09-23T20:39:28.069 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.3974e-05 2023-09-23T20:39:28.070 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.070 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.070 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:39:28.070 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:40.203504+0000", 2023-09-23T20:39:28.070 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.32e-06 2023-09-23T20:39:28.070 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.070 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.070 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:39:28.070 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:40.554479+0000", 2023-09-23T20:39:28.070 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.35097457500000001 2023-09-23T20:39:28.070 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.071 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.071 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for degraded object", 2023-09-23T20:39:28.071 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:40.561879+0000", 2023-09-23T20:39:28.071 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0074007860000000003 2023-09-23T20:39:28.071 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.071 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.071 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:39:28.071 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:40.619042+0000", 2023-09-23T20:39:28.071 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.057163024999999999 2023-09-23T20:39:28.071 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.071 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.072 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:39:28.072 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:40.621716+0000", 2023-09-23T20:39:28.072 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0026739810000000002 2023-09-23T20:39:28.072 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.072 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.072 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:39:28.072 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:40.627239+0000", 2023-09-23T20:39:28.072 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0055227169999999999 2023-09-23T20:39:28.072 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.072 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.072 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:39:28.072 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:40.646592+0000", 2023-09-23T20:39:28.073 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.019353386 2023-09-23T20:39:28.073 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.073 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.073 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:39:28.073 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:40.778031+0000", 2023-09-23T20:39:28.073 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.131438103 2023-09-23T20:39:28.073 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.073 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.073 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:39:28.073 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:40.779250+0000", 2023-09-23T20:39:28.073 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0012194829999999999 2023-09-23T20:39:28.074 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.074 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.074 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:39:28.074 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:40.781214+0000", 2023-09-23T20:39:28.074 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.001964389 2023-09-23T20:39:28.074 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.074 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.074 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:39:28.074 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:40.786970+0000", 2023-09-23T20:39:28.074 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0057558599999999998 2023-09-23T20:39:28.074 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:39:28.074 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:39:28.075 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:39:28.075 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.075 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.075 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:106827 8.9 8:9397eda9:::benchmark_data_smithi018_341490_object17504:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e418)", 2023-09-23T20:39:28.075 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:38:40.241109+0000", 2023-09-23T20:39:28.075 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 47.741491947, 2023-09-23T20:39:28.075 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.55994143200000002, 2023-09-23T20:39:28.075 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:39:28.075 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:39:28.075 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:39:28.075 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:39:28.075 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:39:28.076 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 106827 2023-09-23T20:39:28.076 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.076 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:39:28.076 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.076 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:39:28.076 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:40.241109+0000", 2023-09-23T20:39:28.076 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:39:28.076 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.076 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.076 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:39:28.076 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:40.241109+0000", 2023-09-23T20:39:28.076 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:39:28.077 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.077 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.077 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:39:28.077 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:40.241108+0000", 2023-09-23T20:39:28.077 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:39:28.077 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.077 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.077 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:39:28.077 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:40.241112+0000", 2023-09-23T20:39:28.077 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.6720000000000003e-06 2023-09-23T20:39:28.077 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.078 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.078 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:39:28.078 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:40.241121+0000", 2023-09-23T20:39:28.078 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.9160000000000007e-06 2023-09-23T20:39:28.078 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.078 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.078 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:39:28.078 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:40.241126+0000", 2023-09-23T20:39:28.078 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.7709999999999997e-06 2023-09-23T20:39:28.078 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.078 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.078 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:39:28.079 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:40.554772+0000", 2023-09-23T20:39:28.079 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.31364581800000002 2023-09-23T20:39:28.079 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.079 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.079 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for degraded object", 2023-09-23T20:39:28.079 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:40.563264+0000", 2023-09-23T20:39:28.079 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0084926870000000005 2023-09-23T20:39:28.079 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.079 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.079 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:39:28.079 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:40.622402+0000", 2023-09-23T20:39:28.080 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.059138010999999997 2023-09-23T20:39:28.080 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.080 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.080 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:39:28.080 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:40.625707+0000", 2023-09-23T20:39:28.080 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0033045940000000001 2023-09-23T20:39:28.080 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.080 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.080 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:39:28.080 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:40.632910+0000", 2023-09-23T20:39:28.080 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0072032349999999997 2023-09-23T20:39:28.080 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.081 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.081 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:39:28.081 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:40.675242+0000", 2023-09-23T20:39:28.081 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.042331972000000002 2023-09-23T20:39:28.081 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.081 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.081 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:39:28.081 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:40.780889+0000", 2023-09-23T20:39:28.081 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.10564657199999999 2023-09-23T20:39:28.081 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.081 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.081 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:39:28.082 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:40.788709+0000", 2023-09-23T20:39:28.082 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0078198160000000003 2023-09-23T20:39:28.082 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.082 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.082 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:39:28.082 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:40.791167+0000", 2023-09-23T20:39:28.082 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.002458661 2023-09-23T20:39:28.082 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.082 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.082 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:39:28.082 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:40.801050+0000", 2023-09-23T20:39:28.082 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0098826720000000003 2023-09-23T20:39:28.083 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:39:28.083 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:39:28.083 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:39:28.083 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.083 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.083 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:107411 8.9 8:93727162:::benchmark_data_smithi018_341490_object18088:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e419)", 2023-09-23T20:39:28.083 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:38:41.536053+0000", 2023-09-23T20:39:28.083 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 46.446547832999997, 2023-09-23T20:39:28.083 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.545918711, 2023-09-23T20:39:28.083 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:39:28.083 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:39:28.083 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:39:28.084 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:39:28.084 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:39:28.084 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 107411 2023-09-23T20:39:28.084 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.084 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:39:28.084 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.084 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:39:28.084 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:41.536053+0000", 2023-09-23T20:39:28.084 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:39:28.084 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.084 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.085 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:39:28.085 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:41.536053+0000", 2023-09-23T20:39:28.085 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:39:28.085 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.085 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.085 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:39:28.085 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:41.536052+0000", 2023-09-23T20:39:28.085 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:39:28.085 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.085 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.085 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:39:28.086 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:41.536057+0000", 2023-09-23T20:39:28.086 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.31e-06 2023-09-23T20:39:28.086 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.086 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.086 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:39:28.086 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:41.536070+0000", 2023-09-23T20:39:28.086 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.3277e-05 2023-09-23T20:39:28.086 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.086 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.086 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:39:28.086 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:41.536075+0000", 2023-09-23T20:39:28.087 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.1390000000000003e-06 2023-09-23T20:39:28.087 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.087 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.087 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:39:28.087 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:41.981654+0000", 2023-09-23T20:39:28.087 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.445578633 2023-09-23T20:39:28.087 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.087 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.087 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for degraded object", 2023-09-23T20:39:28.087 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:41.990299+0000", 2023-09-23T20:39:28.087 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0086450859999999997 2023-09-23T20:39:28.087 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.088 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.088 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:39:28.088 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:42.014870+0000", 2023-09-23T20:39:28.088 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.024571401 2023-09-23T20:39:28.088 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.088 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.088 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:39:28.088 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:42.018005+0000", 2023-09-23T20:39:28.088 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0031345050000000001 2023-09-23T20:39:28.088 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.088 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.088 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:39:28.089 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:42.024399+0000", 2023-09-23T20:39:28.089 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0063945089999999996 2023-09-23T20:39:28.089 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.089 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.089 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:39:28.089 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:42.058397+0000", 2023-09-23T20:39:28.089 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.033997918000000002 2023-09-23T20:39:28.089 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.089 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.089 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:39:28.089 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:42.072100+0000", 2023-09-23T20:39:28.090 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.01370296 2023-09-23T20:39:28.090 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.090 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.090 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:39:28.090 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:42.073284+0000", 2023-09-23T20:39:28.090 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.001183447 2023-09-23T20:39:28.090 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.090 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.090 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:39:28.090 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:42.075038+0000", 2023-09-23T20:39:28.090 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0017545799999999999 2023-09-23T20:39:28.090 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.091 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.091 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:39:28.091 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:42.081971+0000", 2023-09-23T20:39:28.091 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0069330750000000003 2023-09-23T20:39:28.091 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:39:28.091 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:39:28.091 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:39:28.091 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.091 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.091 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:114780 8.19 8:99d20faf:::benchmark_data_smithi018_341490_object25457:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e421)", 2023-09-23T20:39:28.091 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:38:57.795331+0000", 2023-09-23T20:39:28.092 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 30.187269961999998, 2023-09-23T20:39:28.092 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.57046175899999996, 2023-09-23T20:39:28.092 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:39:28.092 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:39:28.092 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:39:28.092 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:39:28.092 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:39:28.092 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 114780 2023-09-23T20:39:28.092 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.092 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:39:28.092 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.092 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:39:28.092 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:57.795331+0000", 2023-09-23T20:39:28.093 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:39:28.093 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.093 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.093 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:39:28.093 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:57.795331+0000", 2023-09-23T20:39:28.093 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:39:28.093 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.093 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.093 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:39:28.093 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:57.795330+0000", 2023-09-23T20:39:28.093 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:39:28.094 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.094 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.094 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:39:28.094 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:57.795334+0000", 2023-09-23T20:39:28.094 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.8470000000000003e-06 2023-09-23T20:39:28.094 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.094 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.094 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:39:28.094 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:57.795346+0000", 2023-09-23T20:39:28.094 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1746000000000001e-05 2023-09-23T20:39:28.094 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.095 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.095 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:39:28.095 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:57.795351+0000", 2023-09-23T20:39:28.095 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.0629999999999996e-06 2023-09-23T20:39:28.095 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.095 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.095 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:39:28.095 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:58.258901+0000", 2023-09-23T20:39:28.095 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.463549977 2023-09-23T20:39:28.095 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.095 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.095 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for degraded object", 2023-09-23T20:39:28.096 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:58.267411+0000", 2023-09-23T20:39:28.096 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0085102670000000002 2023-09-23T20:39:28.096 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.096 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.096 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:39:28.096 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:58.297813+0000", 2023-09-23T20:39:28.096 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.030401557999999999 2023-09-23T20:39:28.096 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.096 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.096 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:39:28.096 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:58.300388+0000", 2023-09-23T20:39:28.097 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0025745809999999998 2023-09-23T20:39:28.097 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.097 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.097 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:39:28.097 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:58.305183+0000", 2023-09-23T20:39:28.097 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0047954850000000004 2023-09-23T20:39:28.097 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.097 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.097 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:39:28.097 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:58.331209+0000", 2023-09-23T20:39:28.097 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.026026198 2023-09-23T20:39:28.097 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.098 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.098 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:39:28.098 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:58.357178+0000", 2023-09-23T20:39:28.098 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.025968600000000001 2023-09-23T20:39:28.098 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.098 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.098 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:39:28.098 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:58.358388+0000", 2023-09-23T20:39:28.098 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.001210051 2023-09-23T20:39:28.098 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.098 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.098 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:39:28.099 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:58.359794+0000", 2023-09-23T20:39:28.099 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0014056769999999999 2023-09-23T20:39:28.099 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.099 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.099 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:39:28.099 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:58.365792+0000", 2023-09-23T20:39:28.099 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0059987219999999997 2023-09-23T20:39:28.099 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:39:28.099 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:39:28.099 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:39:28.099 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.099 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.100 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:116668 8.9 8:90978f56:::benchmark_data_smithi018_341490_object27345:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e422)", 2023-09-23T20:39:28.100 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:39:01.643813+0000", 2023-09-23T20:39:28.100 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 26.338787095000001, 2023-09-23T20:39:28.100 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.54435829499999999, 2023-09-23T20:39:28.100 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:39:28.100 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:39:28.100 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:39:28.100 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:39:28.100 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:39:28.100 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 116668 2023-09-23T20:39:28.100 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.101 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:39:28.101 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.101 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:39:28.101 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:01.643813+0000", 2023-09-23T20:39:28.101 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:39:28.101 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.101 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.101 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:39:28.101 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:01.643813+0000", 2023-09-23T20:39:28.101 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:39:28.101 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.101 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.102 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:39:28.102 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:01.643812+0000", 2023-09-23T20:39:28.102 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:39:28.102 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.102 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.102 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:39:28.102 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:01.643822+0000", 2023-09-23T20:39:28.102 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.7329999999999998e-06 2023-09-23T20:39:28.102 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.102 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.102 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:39:28.102 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:01.643835+0000", 2023-09-23T20:39:28.103 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.3393999999999999e-05 2023-09-23T20:39:28.103 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.103 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.103 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:39:28.103 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:01.643844+0000", 2023-09-23T20:39:28.103 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.0459999999999994e-06 2023-09-23T20:39:28.103 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.103 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.103 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:39:28.103 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:02.073276+0000", 2023-09-23T20:39:28.103 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.42943203400000002 2023-09-23T20:39:28.103 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.104 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.104 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for degraded object", 2023-09-23T20:39:28.104 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:02.081614+0000", 2023-09-23T20:39:28.104 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0083381830000000007 2023-09-23T20:39:28.104 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.104 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.104 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:39:28.104 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:02.112203+0000", 2023-09-23T20:39:28.104 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.030588539000000001 2023-09-23T20:39:28.104 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.104 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.104 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:39:28.105 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:02.115619+0000", 2023-09-23T20:39:28.105 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0034162989999999998 2023-09-23T20:39:28.105 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.105 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.105 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:39:28.105 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:02.122425+0000", 2023-09-23T20:39:28.105 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0068054669999999999 2023-09-23T20:39:28.105 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.105 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.105 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:39:28.105 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:02.145892+0000", 2023-09-23T20:39:28.106 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.023467054000000001 2023-09-23T20:39:28.106 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.106 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.106 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:39:28.106 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:02.170932+0000", 2023-09-23T20:39:28.106 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.025040007999999999 2023-09-23T20:39:28.106 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.106 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.106 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:39:28.106 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:02.178930+0000", 2023-09-23T20:39:28.106 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0079978259999999995 2023-09-23T20:39:28.106 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.107 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.107 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:39:28.107 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:02.180842+0000", 2023-09-23T20:39:28.107 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0019124770000000001 2023-09-23T20:39:28.107 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.107 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.107 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:39:28.107 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:02.188172+0000", 2023-09-23T20:39:28.107 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0073296020000000002 2023-09-23T20:39:28.107 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:39:28.107 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:39:28.107 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:39:28.108 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.108 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.108 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:117102 8.9 8:94476c01:::benchmark_data_smithi018_341490_object27779:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e422)", 2023-09-23T20:39:28.108 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:39:02.534326+0000", 2023-09-23T20:39:28.108 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 25.448274524999999, 2023-09-23T20:39:28.108 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.54634636000000003, 2023-09-23T20:39:28.108 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:39:28.108 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:39:28.108 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:39:28.108 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:39:28.108 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:39:28.109 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 117102 2023-09-23T20:39:28.109 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.109 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:39:28.109 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.109 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:39:28.109 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:02.534326+0000", 2023-09-23T20:39:28.109 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:39:28.109 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.109 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.109 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:39:28.109 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:02.534326+0000", 2023-09-23T20:39:28.109 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:39:28.110 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.110 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.110 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:39:28.110 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:02.534325+0000", 2023-09-23T20:39:28.110 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:39:28.110 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.110 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.110 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:39:28.110 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:02.534330+0000", 2023-09-23T20:39:28.110 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.5400000000000003e-06 2023-09-23T20:39:28.110 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.111 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.111 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:39:28.111 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:02.534342+0000", 2023-09-23T20:39:28.111 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.2149e-05 2023-09-23T20:39:28.111 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.111 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.111 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:39:28.111 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:02.534349+0000", 2023-09-23T20:39:28.111 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.7449999999999998e-06 2023-09-23T20:39:28.111 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.111 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.111 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:39:28.112 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:02.986050+0000", 2023-09-23T20:39:28.112 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.45170089200000002 2023-09-23T20:39:28.112 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.112 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.112 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for degraded object", 2023-09-23T20:39:28.112 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:02.994585+0000", 2023-09-23T20:39:28.112 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0085348030000000005 2023-09-23T20:39:28.112 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.112 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.112 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:39:28.112 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:03.017680+0000", 2023-09-23T20:39:28.113 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.023095015999999999 2023-09-23T20:39:28.113 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.113 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.113 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:39:28.113 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:03.020526+0000", 2023-09-23T20:39:28.113 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.002845686 2023-09-23T20:39:28.113 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.113 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.113 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:39:28.113 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:03.026613+0000", 2023-09-23T20:39:28.113 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0060874500000000003 2023-09-23T20:39:28.113 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.114 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.114 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:39:28.114 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:03.043901+0000", 2023-09-23T20:39:28.114 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.017287535 2023-09-23T20:39:28.114 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.114 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.114 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:39:28.114 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:03.057977+0000", 2023-09-23T20:39:28.114 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.014076422 2023-09-23T20:39:28.114 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.114 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.115 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:39:28.115 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:03.069852+0000", 2023-09-23T20:39:28.115 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0118748 2023-09-23T20:39:28.115 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.115 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.115 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:39:28.115 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:03.072063+0000", 2023-09-23T20:39:28.115 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0022112500000000001 2023-09-23T20:39:28.115 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.115 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.115 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:39:28.115 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:03.080672+0000", 2023-09-23T20:39:28.116 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0086092779999999997 2023-09-23T20:39:28.116 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:39:28.116 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:39:28.116 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:39:28.116 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.116 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.116 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:117920 8.9 8:937fdfb1:::benchmark_data_smithi018_341490_object28597:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e422)", 2023-09-23T20:39:28.116 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:39:04.522027+0000", 2023-09-23T20:39:28.116 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 23.460573707000002, 2023-09-23T20:39:28.116 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.54337784300000003, 2023-09-23T20:39:28.116 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:39:28.117 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:39:28.117 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:39:28.117 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:39:28.117 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:39:28.117 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 117920 2023-09-23T20:39:28.117 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.117 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:39:28.117 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.117 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:39:28.117 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:04.522027+0000", 2023-09-23T20:39:28.117 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:39:28.117 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.118 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.118 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:39:28.118 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:04.522027+0000", 2023-09-23T20:39:28.118 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:39:28.118 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.118 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.118 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:39:28.118 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:04.522025+0000", 2023-09-23T20:39:28.118 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999981 2023-09-23T20:39:28.118 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.118 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.118 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:39:28.119 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:04.522031+0000", 2023-09-23T20:39:28.119 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.2670000000000003e-06 2023-09-23T20:39:28.119 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.119 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.119 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:39:28.119 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:04.522043+0000", 2023-09-23T20:39:28.119 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.2116999999999999e-05 2023-09-23T20:39:28.119 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.119 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.119 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:39:28.119 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:04.522048+0000", 2023-09-23T20:39:28.120 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.0499999999999999e-06 2023-09-23T20:39:28.120 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.120 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.120 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:39:28.120 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:04.977677+0000", 2023-09-23T20:39:28.120 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.455628335 2023-09-23T20:39:28.120 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.120 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.120 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for degraded object", 2023-09-23T20:39:28.120 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:04.985879+0000", 2023-09-23T20:39:28.120 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0082020240000000005 2023-09-23T20:39:28.120 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.121 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.121 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:39:28.121 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:05.013327+0000", 2023-09-23T20:39:28.121 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.027448713 2023-09-23T20:39:28.121 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.121 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.121 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:39:28.121 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:05.016787+0000", 2023-09-23T20:39:28.121 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0034594320000000001 2023-09-23T20:39:28.121 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.121 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.122 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:39:28.122 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:05.022748+0000", 2023-09-23T20:39:28.122 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0059614949999999998 2023-09-23T20:39:28.122 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.122 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.122 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:39:28.122 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:05.053421+0000", 2023-09-23T20:39:28.122 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.030672725000000001 2023-09-23T20:39:28.122 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.122 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.122 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:39:28.122 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:05.054266+0000", 2023-09-23T20:39:28.123 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00084444800000000003 2023-09-23T20:39:28.123 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.123 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.123 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:39:28.123 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:05.055763+0000", 2023-09-23T20:39:28.123 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.001497113 2023-09-23T20:39:28.123 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.123 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.123 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:39:28.123 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:05.057839+0000", 2023-09-23T20:39:28.123 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.002076423 2023-09-23T20:39:28.123 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.124 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.124 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:39:28.124 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:05.065405+0000", 2023-09-23T20:39:28.124 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.007565581 2023-09-23T20:39:28.124 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:39:28.124 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:39:28.124 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:39:28.124 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.124 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.124 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:117951 8.9 8:9109a280:::benchmark_data_smithi018_341490_object28628:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e422)", 2023-09-23T20:39:28.124 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:39:04.627035+0000", 2023-09-23T20:39:28.125 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 23.355565447, 2023-09-23T20:39:28.125 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.56794783400000004, 2023-09-23T20:39:28.125 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:39:28.125 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:39:28.125 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:39:28.125 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:39:28.125 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:39:28.125 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 117951 2023-09-23T20:39:28.125 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.125 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:39:28.125 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.125 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:39:28.126 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:04.627035+0000", 2023-09-23T20:39:28.126 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:39:28.126 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.126 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.126 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:39:28.126 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:04.627035+0000", 2023-09-23T20:39:28.126 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:39:28.126 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.126 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.126 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:39:28.126 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:04.627034+0000", 2023-09-23T20:39:28.126 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:39:28.127 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.127 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.127 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:39:28.127 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:04.627041+0000", 2023-09-23T20:39:28.127 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.6450000000000005e-06 2023-09-23T20:39:28.127 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.127 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.127 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:39:28.127 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:04.627055+0000", 2023-09-23T20:39:28.127 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.3978999999999999e-05 2023-09-23T20:39:28.127 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.127 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.128 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:39:28.128 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:04.627064+0000", 2023-09-23T20:39:28.128 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.3590000000000008e-06 2023-09-23T20:39:28.128 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.128 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.128 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:39:28.128 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:05.078303+0000", 2023-09-23T20:39:28.128 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.45123901199999999 2023-09-23T20:39:28.128 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.128 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.128 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for degraded object", 2023-09-23T20:39:28.129 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:05.087537+0000", 2023-09-23T20:39:28.129 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0092337489999999994 2023-09-23T20:39:28.129 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.129 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.129 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:39:28.129 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:05.118409+0000", 2023-09-23T20:39:28.129 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.030872439000000002 2023-09-23T20:39:28.129 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.129 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.129 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:39:28.129 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:05.121485+0000", 2023-09-23T20:39:28.129 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0030759479999999998 2023-09-23T20:39:28.130 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.130 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.130 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:39:28.130 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:05.126808+0000", 2023-09-23T20:39:28.130 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0053229540000000004 2023-09-23T20:39:28.130 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.130 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.130 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:39:28.130 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:05.153916+0000", 2023-09-23T20:39:28.130 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.027107971000000002 2023-09-23T20:39:28.130 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.130 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.131 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:39:28.131 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:05.179878+0000", 2023-09-23T20:39:28.131 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.025962030000000001 2023-09-23T20:39:28.131 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.131 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.131 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:39:28.131 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:05.187231+0000", 2023-09-23T20:39:28.131 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.007353488 2023-09-23T20:39:28.131 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.131 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.131 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:39:28.132 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:05.188772+0000", 2023-09-23T20:39:28.132 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0015410350000000001 2023-09-23T20:39:28.132 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.132 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.132 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:39:28.132 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:05.194983+0000", 2023-09-23T20:39:28.132 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0062104589999999998 2023-09-23T20:39:28.132 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:39:28.132 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:39:28.132 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:39:28.132 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.132 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.133 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:118954 8.9 8:94282a85:::benchmark_data_smithi018_341490_object29631:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e422)", 2023-09-23T20:39:28.133 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:39:06.633258+0000", 2023-09-23T20:39:28.133 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 21.349342227000001, 2023-09-23T20:39:28.133 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.54584307600000004, 2023-09-23T20:39:28.133 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:39:28.133 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:39:28.133 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:39:28.133 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:39:28.133 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:39:28.133 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 118954 2023-09-23T20:39:28.133 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.134 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:39:28.134 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.134 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:39:28.134 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:06.633258+0000", 2023-09-23T20:39:28.134 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:39:28.134 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.134 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.134 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:39:28.134 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:06.633258+0000", 2023-09-23T20:39:28.134 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:39:28.134 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.135 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.135 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:39:28.135 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:06.633257+0000", 2023-09-23T20:39:28.135 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:39:28.135 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.135 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.135 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:39:28.135 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:06.633262+0000", 2023-09-23T20:39:28.135 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.0860000000000004e-06 2023-09-23T20:39:28.135 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.135 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.135 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:39:28.136 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:06.633270+0000", 2023-09-23T20:39:28.136 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.8619999999999992e-06 2023-09-23T20:39:28.136 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.136 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.136 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:39:28.136 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:06.633274+0000", 2023-09-23T20:39:28.136 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.0280000000000002e-06 2023-09-23T20:39:28.136 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.136 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.136 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:39:28.136 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:07.056940+0000", 2023-09-23T20:39:28.136 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.42366564000000001 2023-09-23T20:39:28.137 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.137 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.137 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for degraded object", 2023-09-23T20:39:28.137 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:07.070034+0000", 2023-09-23T20:39:28.137 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.013094006 2023-09-23T20:39:28.137 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.137 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.137 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:39:28.137 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:07.103251+0000", 2023-09-23T20:39:28.137 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.033216491000000001 2023-09-23T20:39:28.137 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.137 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.138 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:39:28.138 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:07.105891+0000", 2023-09-23T20:39:28.138 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0026404670000000001 2023-09-23T20:39:28.138 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.138 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.138 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:39:28.138 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:07.111462+0000", 2023-09-23T20:39:28.138 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0055709499999999999 2023-09-23T20:39:28.138 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.138 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.138 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:39:28.139 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:07.157460+0000", 2023-09-23T20:39:28.139 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.045998209999999998 2023-09-23T20:39:28.139 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.139 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.139 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:39:28.139 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:07.170250+0000", 2023-09-23T20:39:28.139 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.012790073000000001 2023-09-23T20:39:28.139 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.139 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.139 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:39:28.139 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:07.171476+0000", 2023-09-23T20:39:28.139 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.001225514 2023-09-23T20:39:28.140 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.140 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.140 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:39:28.140 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:07.172978+0000", 2023-09-23T20:39:28.140 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.001502275 2023-09-23T20:39:28.140 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.140 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.140 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:39:28.140 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:07.179101+0000", 2023-09-23T20:39:28.140 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0061233290000000003 2023-09-23T20:39:28.140 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:39:28.141 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:39:28.141 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:39:28.141 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.141 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.141 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:119363 8.19 8:9b94513b:::benchmark_data_smithi018_341490_object30040:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e422)", 2023-09-23T20:39:28.141 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:39:07.530370+0000", 2023-09-23T20:39:28.141 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 20.452230958000001, 2023-09-23T20:39:28.141 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.56285548100000005, 2023-09-23T20:39:28.141 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:39:28.141 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:39:28.141 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:39:28.141 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:39:28.142 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:39:28.142 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 119363 2023-09-23T20:39:28.142 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.142 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:39:28.142 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.142 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:39:28.142 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:07.530370+0000", 2023-09-23T20:39:28.142 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:39:28.142 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.142 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.142 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:39:28.143 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:07.530370+0000", 2023-09-23T20:39:28.143 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:39:28.143 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.143 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.143 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:39:28.143 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:07.530368+0000", 2023-09-23T20:39:28.143 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:39:28.143 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.143 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.143 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:39:28.143 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:07.530374+0000", 2023-09-23T20:39:28.143 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.7250000000000002e-06 2023-09-23T20:39:28.144 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.144 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.144 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:39:28.144 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:07.530387+0000", 2023-09-23T20:39:28.144 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.3284e-05 2023-09-23T20:39:28.144 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.144 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.144 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:39:28.144 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:07.530393+0000", 2023-09-23T20:39:28.144 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.3789999999999997e-06 2023-09-23T20:39:28.144 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.144 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.145 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:39:28.145 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:07.948569+0000", 2023-09-23T20:39:28.145 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.41817622599999998 2023-09-23T20:39:28.145 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.145 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.145 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for degraded object", 2023-09-23T20:39:28.145 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:07.959914+0000", 2023-09-23T20:39:28.145 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.011345434999999999 2023-09-23T20:39:28.145 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.145 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.145 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:39:28.145 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:07.999724+0000", 2023-09-23T20:39:28.146 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.039809125000000001 2023-09-23T20:39:28.146 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.146 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.146 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:39:28.146 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:08.002702+0000", 2023-09-23T20:39:28.146 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0029784859999999998 2023-09-23T20:39:28.146 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.146 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.146 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:39:28.146 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:08.008078+0000", 2023-09-23T20:39:28.146 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0053759610000000003 2023-09-23T20:39:28.147 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.147 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.147 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:39:28.147 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:08.036970+0000", 2023-09-23T20:39:28.147 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.028891567999999999 2023-09-23T20:39:28.147 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.147 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.147 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:39:28.147 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:08.084972+0000", 2023-09-23T20:39:28.147 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.048002351999999998 2023-09-23T20:39:28.147 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.147 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.148 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:39:28.148 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:08.086297+0000", 2023-09-23T20:39:28.148 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.001325562 2023-09-23T20:39:28.148 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.148 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.148 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:39:28.148 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:08.087523+0000", 2023-09-23T20:39:28.148 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.001225659 2023-09-23T20:39:28.148 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:28.148 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:28.148 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:39:28.148 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:08.093225+0000", 2023-09-23T20:39:28.148 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0057018900000000003 2023-09-23T20:39:28.149 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:39:28.149 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:39:28.149 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:39:28.149 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:39:28.149 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:39:28.149 INFO:teuthology.orchestra.run.smithi149.stdout:} 2023-09-23T20:39:28.150 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:28.150 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:39:28.150 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:39:28.240 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:39:28.251 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:39:28.327 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T20:39:28.336 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:28.573 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:39:28.576 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:28.577 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:39:28.678 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:39:28.710 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:28.768 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:39:28.773 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:28.773 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:39:28.874 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:39:28.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:28 smithi018 ceph-mon[179647]: 1.a scrub starts 2023-09-23T20:39:28.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:28 smithi018 ceph-mon[179647]: pgmap v1044: 129 pgs: 6 active+recovery_wait+degraded, 1 active+recovering, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 3.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 541 op/s; 13069/150225 objects degraded (8.700%); 4524/150225 objects misplaced (3.011%); 11 MiB/s, 241 objects/s recovering 2023-09-23T20:39:28.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:28 smithi018 ceph-mon[184144]: 1.a scrub starts 2023-09-23T20:39:28.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:28 smithi018 ceph-mon[184144]: pgmap v1044: 129 pgs: 6 active+recovery_wait+degraded, 1 active+recovering, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 3.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 541 op/s; 13069/150225 objects degraded (8.700%); 4524/150225 objects misplaced (3.011%); 11 MiB/s, 241 objects/s recovering 2023-09-23T20:39:28.908 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:28.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:28 smithi149 ceph-mon[170394]: 1.a scrub starts 2023-09-23T20:39:28.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:28 smithi149 ceph-mon[170394]: pgmap v1044: 129 pgs: 6 active+recovery_wait+degraded, 1 active+recovering, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 3.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 541 op/s; 13069/150225 objects degraded (8.700%); 4524/150225 objects misplaced (3.011%); 11 MiB/s, 241 objects/s recovering 2023-09-23T20:39:29.002 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:39:29.008 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:29.008 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:39:29.037 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:39:29.059 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:39:29.109 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:39:29.151 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:29.243 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:39:29.253 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:29.253 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:39:29.354 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:39:29.387 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:29.433 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:39:29.438 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:29.439 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:39:29.539 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:39:29.583 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:29.773 INFO:teuthology.orchestra.run.smithi018.stderr:noscrub is set 2023-09-23T20:39:29.805 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:39:29.811 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:29.811 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:39:29.912 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:39:29.960 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:29 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1935273794' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:39:29.960 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:29 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:39:29.960 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:29 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:39:29.965 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:30.051 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:39:30.058 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:30.058 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:39:30.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:29 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1935273794' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:39:30.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:29 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:39:30.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:29 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:39:30.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:29 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1935273794' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:39:30.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:29 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:39:30.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:29 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:39:30.159 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:39:30.205 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:30.508 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:39:30.514 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:30.514 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:39:30.525 INFO:teuthology.orchestra.run.smithi149.stdout:{ 2023-09-23T20:39:30.526 INFO:teuthology.orchestra.run.smithi149.stdout: "ops": [ 2023-09-23T20:39:30.526 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:30.526 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:131090 8.d 8:baeede6b:::benchmark_data_smithi018_341490_object41767:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e428)", 2023-09-23T20:39:30.526 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:39:30.488664+0000", 2023-09-23T20:39:30.526 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 0.035979627, 2023-09-23T20:39:30.526 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.036003805999999999, 2023-09-23T20:39:30.526 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:39:30.526 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:39:30.526 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:39:30.526 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:39:30.526 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:39:30.527 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 131090 2023-09-23T20:39:30.527 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:30.527 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:39:30.527 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:30.527 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:39:30.527 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:30.488664+0000", 2023-09-23T20:39:30.527 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:39:30.527 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:30.527 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:30.527 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:39:30.527 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:30.488664+0000", 2023-09-23T20:39:30.528 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:39:30.528 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:30.528 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:30.528 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:39:30.528 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:30.488663+0000", 2023-09-23T20:39:30.528 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:39:30.528 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:30.528 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:30.528 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:39:30.528 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:30.488669+0000", 2023-09-23T20:39:30.528 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.3829999999999997e-06 2023-09-23T20:39:30.529 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:30.529 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:30.529 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:39:30.529 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:30.488683+0000", 2023-09-23T20:39:30.529 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.3577000000000001e-05 2023-09-23T20:39:30.529 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:30.529 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:30.529 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:39:30.529 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:30.488689+0000", 2023-09-23T20:39:30.529 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.4570000000000002e-06 2023-09-23T20:39:30.529 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:30.529 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:30.530 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:39:30.530 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:30.495663+0000", 2023-09-23T20:39:30.530 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0069738780000000002 2023-09-23T20:39:30.530 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:30.530 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:30.530 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:39:30.530 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:30.495836+0000", 2023-09-23T20:39:30.530 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00017234699999999999 2023-09-23T20:39:30.530 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:30.530 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:30.530 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 2,6", 2023-09-23T20:39:30.531 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:30.496004+0000", 2023-09-23T20:39:30.531 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00016843699999999999 2023-09-23T20:39:30.531 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:30.531 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:30.531 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:39:30.531 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:30.504545+0000", 2023-09-23T20:39:30.531 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0085406490000000009 2023-09-23T20:39:30.531 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:30.531 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:30.531 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:39:30.531 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:30.504599+0000", 2023-09-23T20:39:30.531 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.4747000000000003e-05 2023-09-23T20:39:30.532 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:30.532 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:30.532 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:39:30.532 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:30.523077+0000", 2023-09-23T20:39:30.532 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.018477463 2023-09-23T20:39:30.532 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:30.532 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:30.532 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:39:30.532 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:30.523116+0000", 2023-09-23T20:39:30.532 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.9168000000000002e-05 2023-09-23T20:39:30.532 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:39:30.533 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:39:30.533 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:39:30.533 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:30.533 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:30.533 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:131095 8.d 8:b4829aa0:::benchmark_data_smithi018_341490_object41772:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e428)", 2023-09-23T20:39:30.533 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:39:30.502270+0000", 2023-09-23T20:39:30.533 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 0.022373885, 2023-09-23T20:39:30.533 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.022464397000000001, 2023-09-23T20:39:30.533 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:39:30.533 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:39:30.533 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:39:30.533 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:39:30.534 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:39:30.534 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 131095 2023-09-23T20:39:30.534 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:30.534 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:39:30.534 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:30.534 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:39:30.534 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:30.502270+0000", 2023-09-23T20:39:30.534 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:39:30.535 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:30.535 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:30.535 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:39:30.535 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:30.502270+0000", 2023-09-23T20:39:30.535 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:39:30.535 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:30.535 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:30.535 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:39:30.535 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:30.502269+0000", 2023-09-23T20:39:30.536 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:39:30.536 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:30.536 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:30.536 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:39:30.536 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:30.502276+0000", 2023-09-23T20:39:30.536 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.6039999999999998e-06 2023-09-23T20:39:30.536 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:30.536 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:30.536 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:39:30.536 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:30.502287+0000", 2023-09-23T20:39:30.536 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1029000000000001e-05 2023-09-23T20:39:30.536 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:30.537 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:30.537 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:39:30.537 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:30.502292+0000", 2023-09-23T20:39:30.537 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.3109999999999996e-06 2023-09-23T20:39:30.537 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:30.537 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:30.537 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:39:30.537 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:30.507011+0000", 2023-09-23T20:39:30.537 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0047196369999999996 2023-09-23T20:39:30.537 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:30.537 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:30.537 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:39:30.538 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:30.507181+0000", 2023-09-23T20:39:30.538 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00016913200000000001 2023-09-23T20:39:30.538 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:30.538 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:30.538 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 2,6", 2023-09-23T20:39:30.538 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:30.507366+0000", 2023-09-23T20:39:30.538 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000185813 2023-09-23T20:39:30.538 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:39:30.538 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:39:30.538 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:39:30.538 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:30.538 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:30.539 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_repop_reply(client.35994.0:131090 8.d e428/403 ondisk, result = 0)", 2023-09-23T20:39:30.539 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:39:30.507372+0000", 2023-09-23T20:39:30.539 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 0.017272035000000002, 2023-09-23T20:39:30.539 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.017411919000000001, 2023-09-23T20:39:30.539 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:39:30.539 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "started", 2023-09-23T20:39:30.539 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:39:30.539 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:30.539 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:39:30.539 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:30.507372+0000", 2023-09-23T20:39:30.539 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:39:30.539 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:30.540 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:30.540 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:39:30.540 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:30.507372+0000", 2023-09-23T20:39:30.540 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:39:30.540 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:30.540 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:30.540 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:39:30.540 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:30.507372+0000", 2023-09-23T20:39:30.540 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:39:30.540 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:30.540 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:30.540 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:39:30.541 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:30.507375+0000", 2023-09-23T20:39:30.541 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.8739999999999999e-06 2023-09-23T20:39:30.541 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:30.541 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:30.541 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:39:30.541 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:30.507380+0000", 2023-09-23T20:39:30.541 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.2159999999999997e-06 2023-09-23T20:39:30.541 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:30.541 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:30.541 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:39:30.541 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:30.507383+0000", 2023-09-23T20:39:30.541 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.4699999999999998e-06 2023-09-23T20:39:30.542 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:30.542 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:30.542 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:39:30.542 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:30.523066+0000", 2023-09-23T20:39:30.542 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.015682506999999998 2023-09-23T20:39:30.542 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:30.542 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:30.542 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:39:30.542 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:30.523072+0000", 2023-09-23T20:39:30.542 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.472e-06 2023-09-23T20:39:30.542 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:39:30.543 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:39:30.543 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:39:30.543 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:30.543 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:30.543 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_repop(client.35994.0:131093 8.15 e428/410 8:af96e80e:::benchmark_data_smithi018_341490_object41770:head v 428'7366, mlcod=428'7364)", 2023-09-23T20:39:30.543 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:39:30.508736+0000", 2023-09-23T20:39:30.543 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 0.015908048000000001, 2023-09-23T20:39:30.543 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.016107092999999999, 2023-09-23T20:39:30.543 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:39:30.543 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "started", 2023-09-23T20:39:30.543 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:39:30.544 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:30.544 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:39:30.544 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:30.508736+0000", 2023-09-23T20:39:30.544 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:39:30.544 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:30.544 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:30.544 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:39:30.544 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:30.508736+0000", 2023-09-23T20:39:30.544 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:39:30.544 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:30.544 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:30.544 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:39:30.545 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:30.508735+0000", 2023-09-23T20:39:30.545 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:39:30.545 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:30.545 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:30.545 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:39:30.545 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:30.508743+0000", 2023-09-23T20:39:30.545 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.3089999999999998e-06 2023-09-23T20:39:30.545 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:30.545 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:30.545 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:39:30.545 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:30.508759+0000", 2023-09-23T20:39:30.546 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.6350000000000001e-05 2023-09-23T20:39:30.546 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:30.546 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:30.546 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:39:30.546 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:30.508766+0000", 2023-09-23T20:39:30.546 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.6429999999999998e-06 2023-09-23T20:39:30.546 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:30.546 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:30.546 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:39:30.546 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:30.523057+0000", 2023-09-23T20:39:30.546 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.014291557 2023-09-23T20:39:30.546 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:30.547 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:30.547 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:39:30.547 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:30.523077+0000", 2023-09-23T20:39:30.547 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.9514e-05 2023-09-23T20:39:30.547 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:39:30.547 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:39:30.547 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:39:30.547 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:30.547 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:30.547 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_repop_reply(client.35994.0:131095 8.d e428/403)", 2023-09-23T20:39:30.547 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:39:30.510957+0000", 2023-09-23T20:39:30.547 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 0.013686903, 2023-09-23T20:39:30.548 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.013920833000000001, 2023-09-23T20:39:30.548 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:39:30.548 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "queued for pg", 2023-09-23T20:39:30.548 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:39:30.548 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:30.548 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:39:30.548 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:30.510957+0000", 2023-09-23T20:39:30.548 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:39:30.548 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:30.548 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:30.548 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:39:30.548 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:30.510957+0000", 2023-09-23T20:39:30.549 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:39:30.549 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:30.549 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:30.549 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:39:30.549 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:30.510956+0000", 2023-09-23T20:39:30.549 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:39:30.549 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:30.549 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:30.549 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:39:30.549 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:30.510962+0000", 2023-09-23T20:39:30.549 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.0310000000000004e-06 2023-09-23T20:39:30.550 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:30.550 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:30.550 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:39:30.550 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:30.510977+0000", 2023-09-23T20:39:30.550 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.4608e-05 2023-09-23T20:39:30.550 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:30.550 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:30.550 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:39:30.550 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:30.510984+0000", 2023-09-23T20:39:30.550 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.1960000000000002e-06 2023-09-23T20:39:30.550 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:39:30.550 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:39:30.551 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:39:30.551 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:30.551 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:30.551 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_repop_reply(client.35994.0:131095 8.d e428/403)", 2023-09-23T20:39:30.551 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:39:30.515235+0000", 2023-09-23T20:39:30.551 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 0.0094085249999999992, 2023-09-23T20:39:30.551 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0096791289999999999, 2023-09-23T20:39:30.551 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:39:30.551 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "queued for pg", 2023-09-23T20:39:30.551 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:39:30.551 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:30.551 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:39:30.552 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:30.515235+0000", 2023-09-23T20:39:30.552 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:39:30.552 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:30.552 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:30.552 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:39:30.552 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:30.515235+0000", 2023-09-23T20:39:30.552 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:39:30.552 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:30.552 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:30.552 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:39:30.552 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:30.515235+0000", 2023-09-23T20:39:30.552 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:39:30.553 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:30.553 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:30.553 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:39:30.553 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:30.515242+0000", 2023-09-23T20:39:30.553 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.2019999999999998e-06 2023-09-23T20:39:30.553 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:30.553 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:30.553 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:39:30.553 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:30.515260+0000", 2023-09-23T20:39:30.553 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.8479e-05 2023-09-23T20:39:30.553 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:30.554 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:30.554 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:39:30.554 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:30.515266+0000", 2023-09-23T20:39:30.554 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.1469999999999998e-06 2023-09-23T20:39:30.554 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:39:30.554 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:39:30.554 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:39:30.554 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:30.554 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:30.554 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:131102 8.d 8.4c864b7d (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e428)", 2023-09-23T20:39:30.554 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:39:30.517540+0000", 2023-09-23T20:39:30.555 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 0.0071041910000000002, 2023-09-23T20:39:30.555 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0074075429999999999, 2023-09-23T20:39:30.555 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:39:30.555 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "queued for pg", 2023-09-23T20:39:30.555 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:39:30.555 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:39:30.555 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:39:30.555 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 131102 2023-09-23T20:39:30.555 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:30.555 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:39:30.555 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:30.555 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:39:30.556 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:30.517540+0000", 2023-09-23T20:39:30.556 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:39:30.556 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:30.556 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:30.556 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:39:30.556 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:30.517540+0000", 2023-09-23T20:39:30.556 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:39:30.556 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:30.556 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:30.556 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:39:30.556 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:30.517538+0000", 2023-09-23T20:39:30.556 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:39:30.557 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:30.557 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:30.557 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:39:30.557 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:30.517547+0000", 2023-09-23T20:39:30.557 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.7190000000000007e-06 2023-09-23T20:39:30.557 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:30.557 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:30.557 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:39:30.557 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:30.517564+0000", 2023-09-23T20:39:30.557 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.7034999999999999e-05 2023-09-23T20:39:30.557 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:30.558 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:30.558 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:39:30.558 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:30.517571+0000", 2023-09-23T20:39:30.558 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.9800000000000001e-06 2023-09-23T20:39:30.558 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:39:30.558 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:39:30.558 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:39:30.558 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:30.558 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:30.558 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "MOSDPGPushReply(8.d 428/403 [PushReplyOp(8:b27ed576:::benchmark_data_smithi018_341490_object42846:head)])", 2023-09-23T20:39:30.558 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:39:30.522104+0000", 2023-09-23T20:39:30.559 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 0.0025400290000000001, 2023-09-23T20:39:30.559 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0028799720000000002, 2023-09-23T20:39:30.559 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:39:30.559 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "queued for pg", 2023-09-23T20:39:30.559 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:39:30.559 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:30.559 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:39:30.559 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:30.522104+0000", 2023-09-23T20:39:30.559 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:39:30.559 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:30.559 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:30.560 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:39:30.560 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:30.522104+0000", 2023-09-23T20:39:30.560 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:39:30.560 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:30.560 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:30.560 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:39:30.560 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:30.522103+0000", 2023-09-23T20:39:30.560 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:39:30.560 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:30.560 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:30.560 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:39:30.560 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:30.522110+0000", 2023-09-23T20:39:30.561 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.7700000000000004e-06 2023-09-23T20:39:30.561 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:30.561 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:30.561 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:39:30.561 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:30.522122+0000", 2023-09-23T20:39:30.561 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1885999999999999e-05 2023-09-23T20:39:30.561 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:30.561 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:30.561 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:39:30.561 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:30.522129+0000", 2023-09-23T20:39:30.562 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.6660000000000002e-06 2023-09-23T20:39:30.562 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:39:30.562 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:39:30.562 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:39:30.562 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:30.562 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:30.562 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:131106 8.d 8.b4e793fd (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e428)", 2023-09-23T20:39:30.562 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:39:30.523167+0000", 2023-09-23T20:39:30.562 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 0.0014769270000000001, 2023-09-23T20:39:30.562 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0018455609999999999, 2023-09-23T20:39:30.562 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:39:30.563 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "queued for pg", 2023-09-23T20:39:30.563 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:39:30.563 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:39:30.563 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:39:30.563 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 131106 2023-09-23T20:39:30.563 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:30.563 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:39:30.563 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:30.563 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:39:30.563 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:30.523167+0000", 2023-09-23T20:39:30.563 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:39:30.563 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:30.564 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:30.564 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:39:30.564 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:30.523167+0000", 2023-09-23T20:39:30.564 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:39:30.564 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:30.564 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:30.564 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:39:30.564 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:30.523166+0000", 2023-09-23T20:39:30.564 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:39:30.564 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:30.565 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:30.565 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:39:30.565 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:30.523173+0000", 2023-09-23T20:39:30.565 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.7789999999999998e-06 2023-09-23T20:39:30.565 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:30.565 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:30.565 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:39:30.565 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:30.523185+0000", 2023-09-23T20:39:30.565 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.2262000000000001e-05 2023-09-23T20:39:30.565 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:39:30.565 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:39:30.566 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:39:30.566 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:30.523192+0000", 2023-09-23T20:39:30.566 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.1969999999999999e-06 2023-09-23T20:39:30.566 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:39:30.566 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:39:30.566 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:39:30.566 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:39:30.566 INFO:teuthology.orchestra.run.smithi149.stdout: ], 2023-09-23T20:39:30.566 INFO:teuthology.orchestra.run.smithi149.stdout: "num_ops": 9 2023-09-23T20:39:30.566 INFO:teuthology.orchestra.run.smithi149.stdout:} 2023-09-23T20:39:30.615 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:39:30.666 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:30.845 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:39:30.853 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:30.854 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:39:30.855 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:30 smithi149 ceph-mon[170394]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:39:30.855 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:30 smithi149 ceph-mon[170394]: failed to encode map e428 with expected crc 2023-09-23T20:39:30.855 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:30 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1935273794' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:39:30.855 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:30 smithi149 ceph-mon[170394]: osdmap e428: 8 total, 8 up, 8 in 2023-09-23T20:39:30.855 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:30 smithi149 ceph-mon[170394]: pgmap v1046: 129 pgs: 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 3.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 542 op/s; 12716/146955 objects degraded (8.653%); 4524/146955 objects misplaced (3.078%); 11 MiB/s, 261 objects/s recovering 2023-09-23T20:39:30.896 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T20:39:30.955 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:39:31.014 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:31.083 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:39:31.088 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:39:31.093 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:31.094 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:39:31.116 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:39:31.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:30 smithi018 ceph-mon[179647]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:39:31.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:30 smithi018 ceph-mon[179647]: failed to encode map e428 with expected crc 2023-09-23T20:39:31.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:30 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1935273794' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:39:31.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:30 smithi018 ceph-mon[179647]: osdmap e428: 8 total, 8 up, 8 in 2023-09-23T20:39:31.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:30 smithi018 ceph-mon[179647]: pgmap v1046: 129 pgs: 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 3.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 542 op/s; 12716/146955 objects degraded (8.653%); 4524/146955 objects misplaced (3.078%); 11 MiB/s, 261 objects/s recovering 2023-09-23T20:39:31.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:30 smithi018 ceph-mon[184144]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:39:31.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:30 smithi018 ceph-mon[184144]: failed to encode map e428 with expected crc 2023-09-23T20:39:31.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:30 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1935273794' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:39:31.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:30 smithi018 ceph-mon[184144]: osdmap e428: 8 total, 8 up, 8 in 2023-09-23T20:39:31.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:30 smithi018 ceph-mon[184144]: pgmap v1046: 129 pgs: 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 3.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 542 op/s; 12716/146955 objects degraded (8.653%); 4524/146955 objects misplaced (3.078%); 11 MiB/s, 261 objects/s recovering 2023-09-23T20:39:31.146 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:39:31.170 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:39:31.194 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:39:31.194 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:39:31.216 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:39:31.233 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:31.244 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:39:31.277 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:39:31.282 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:39:31.282 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:39:31.282 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:39:31.282 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:39:31.282 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:39:31.282 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:39:31.282 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:39:31.283 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:39:31.298 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:39:31.299 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 7, 3, 2, 4, 6, 5] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T20:39:31.300 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd reweight 0 1 2023-09-23T20:39:31.304 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:31.305 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:39:31.406 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:39:31.452 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:31.679 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:39:31.683 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:31.684 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:39:31.691 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:39:31.785 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:39:31.831 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:31.882 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:39:31.890 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:31.890 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:39:31.991 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:39:32.028 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:31 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3458018892' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:39:32.028 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:31 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 12716/146955 objects degraded (8.653%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T20:39:32.029 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:31 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3458018892' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:39:32.029 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:31 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 12716/146955 objects degraded (8.653%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T20:39:32.034 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:32.071 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:39:32.089 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:39:32.095 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:32.095 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:39:32.148 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:31 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3458018892' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:39:32.148 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:31 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 12716/146955 objects degraded (8.653%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T20:39:32.196 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:39:32.226 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:32.265 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:39:32.270 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:32.271 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:39:32.371 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:39:32.408 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:32.503 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:39:32.507 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:32.508 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:39:32.575 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T20:39:32.608 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:39:32.658 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:32.705 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:39:32.712 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:32.712 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:39:32.813 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:39:32.854 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:32.894 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:32 smithi018 ceph-mon[184144]: pgmap v1047: 129 pgs: 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 2.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 485 op/s; 12716/144429 objects degraded (8.804%); 4524/144429 objects misplaced (3.132%); 5.0 MiB/s, 97 objects/s recovering 2023-09-23T20:39:32.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:32 smithi018 ceph-mon[179647]: pgmap v1047: 129 pgs: 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 2.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 485 op/s; 12716/144429 objects degraded (8.804%); 4524/144429 objects misplaced (3.132%); 5.0 MiB/s, 97 objects/s recovering 2023-09-23T20:39:32.917 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:39:32.923 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:32.923 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:39:33.024 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:39:33.063 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:32 smithi149 ceph-mon[170394]: pgmap v1047: 129 pgs: 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 2.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 485 op/s; 12716/144429 objects degraded (8.804%); 4524/144429 objects misplaced (3.132%); 5.0 MiB/s, 97 objects/s recovering 2023-09-23T20:39:33.069 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:33.128 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:39:33.133 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:33.134 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:39:33.148 INFO:teuthology.orchestra.run.smithi149.stdout:ERROR: (22) Invalid argument 2023-09-23T20:39:33.148 INFO:teuthology.orchestra.run.smithi149.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:39:33.235 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:39:33.272 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:33.365 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:39:33.369 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:33.370 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:39:33.450 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:39:33.470 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:39:33.505 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:33.564 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:39:33.570 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:33.570 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:39:33.607 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T20:39:33.671 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:39:33.735 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:33.781 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:39:33.782 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:39:33.792 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:33.792 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:39:33.812 INFO:teuthology.orchestra.run.smithi018.stderr:reweighted osd.0 to 1 (10000) 2023-09-23T20:39:33.893 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:39:33.921 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:33 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3011853640' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T20:39:33.929 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:33.983 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:39:33.986 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:33.987 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:39:34.088 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:39:34.134 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:34.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:33 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3011853640' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T20:39:34.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:33 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3011853640' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T20:39:34.180 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:39:34.184 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:34.184 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:39:34.188 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd reweight 1 1 2023-09-23T20:39:34.285 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:39:34.341 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:34.392 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:39:34.397 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:34.398 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:39:34.413 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:39:34.499 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:39:34.546 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:34.691 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:39:34.702 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:34.702 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:39:34.803 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:39:34.855 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:34.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:34 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3011853640' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T20:39:34.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:34 smithi149 ceph-mon[170394]: osdmap e429: 8 total, 8 up, 8 in 2023-09-23T20:39:34.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:34 smithi149 ceph-mon[170394]: failed to encode map e429 with expected crc 2023-09-23T20:39:34.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:34 smithi149 ceph-mon[170394]: failed to encode map e429 with expected crc 2023-09-23T20:39:34.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:34 smithi149 ceph-mon[170394]: pgmap v1049: 129 pgs: 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 2.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 509 op/s; 12556/140988 objects degraded (8.906%); 4233/140988 objects misplaced (3.002%); 8.1 MiB/s, 146 objects/s recovering 2023-09-23T20:39:34.995 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:34 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3011853640' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T20:39:34.995 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:34 smithi018 ceph-mon[179647]: osdmap e429: 8 total, 8 up, 8 in 2023-09-23T20:39:34.995 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:34 smithi018 ceph-mon[179647]: failed to encode map e429 with expected crc 2023-09-23T20:39:34.995 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:34 smithi018 ceph-mon[179647]: failed to encode map e429 with expected crc 2023-09-23T20:39:34.996 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:34 smithi018 ceph-mon[179647]: pgmap v1049: 129 pgs: 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 2.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 509 op/s; 12556/140988 objects degraded (8.906%); 4233/140988 objects misplaced (3.002%); 8.1 MiB/s, 146 objects/s recovering 2023-09-23T20:39:34.996 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:39:34 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:39:34] "GET /metrics HTTP/1.1" 200 36562 "" "Prometheus/2.43.0" 2023-09-23T20:39:34.996 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:34 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3011853640' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T20:39:34.996 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:34 smithi018 ceph-mon[184144]: osdmap e429: 8 total, 8 up, 8 in 2023-09-23T20:39:34.996 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:34 smithi018 ceph-mon[184144]: failed to encode map e429 with expected crc 2023-09-23T20:39:34.997 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:34 smithi018 ceph-mon[184144]: failed to encode map e429 with expected crc 2023-09-23T20:39:34.997 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:34 smithi018 ceph-mon[184144]: pgmap v1049: 129 pgs: 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 2.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 509 op/s; 12556/140988 objects degraded (8.906%); 4233/140988 objects misplaced (3.002%); 8.1 MiB/s, 146 objects/s recovering 2023-09-23T20:39:35.016 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:39:35.177 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:39:35.183 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:35.184 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:39:35.284 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:39:35.332 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:35.436 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:39:35.438 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:39:35.438 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:35.439 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:39:35.540 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:39:35.583 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:35.822 INFO:teuthology.orchestra.run.smithi018.stderr:nodeep-scrub is set 2023-09-23T20:39:35.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:35 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/117818156' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:39:35.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:35 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:39:36.074 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:39:36.078 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:36.078 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:39:36.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:35 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/117818156' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:39:36.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:35 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:39:36.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:35 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/117818156' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:39:36.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:35 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:39:36.179 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:39:36.234 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:36.331 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:39:36.335 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:36.335 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:39:36.436 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:39:36.468 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:36.497 INFO:teuthology.orchestra.run.smithi149.stdout:ERROR: (22) Invalid argument 2023-09-23T20:39:36.497 INFO:teuthology.orchestra.run.smithi149.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:39:36.816 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:36 smithi018 ceph-mon[179647]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:39:36.816 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:36 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:39:36.817 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:36 smithi018 ceph-mon[179647]: osdmap e430: 8 total, 8 up, 8 in 2023-09-23T20:39:36.817 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:36 smithi018 ceph-mon[179647]: failed to encode map e430 with expected crc 2023-09-23T20:39:36.817 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:36 smithi018 ceph-mon[179647]: failed to encode map e430 with expected crc 2023-09-23T20:39:36.817 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:36 smithi018 ceph-mon[179647]: pgmap v1051: 129 pgs: 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 2.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 474 op/s; 12361/138009 objects degraded (8.957%); 4233/138009 objects misplaced (3.067%); 6.4 MiB/s, 102 objects/s recovering 2023-09-23T20:39:36.817 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:36 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 12556/140988 objects degraded (8.906%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T20:39:36.818 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:36 smithi018 ceph-mon[184144]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:39:36.818 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:36 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:39:36.818 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:36 smithi018 ceph-mon[184144]: osdmap e430: 8 total, 8 up, 8 in 2023-09-23T20:39:36.818 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:36 smithi018 ceph-mon[184144]: failed to encode map e430 with expected crc 2023-09-23T20:39:36.818 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:36 smithi018 ceph-mon[184144]: failed to encode map e430 with expected crc 2023-09-23T20:39:36.818 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:36 smithi018 ceph-mon[184144]: pgmap v1051: 129 pgs: 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 2.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 474 op/s; 12361/138009 objects degraded (8.957%); 4233/138009 objects misplaced (3.067%); 6.4 MiB/s, 102 objects/s recovering 2023-09-23T20:39:36.818 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:36 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 12556/140988 objects degraded (8.906%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T20:39:36.939 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T20:39:36.965 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:39:36.970 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:36.970 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:39:37.071 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:39:37.117 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:37.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:36 smithi149 ceph-mon[170394]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:39:37.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:36 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:39:37.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:36 smithi149 ceph-mon[170394]: osdmap e430: 8 total, 8 up, 8 in 2023-09-23T20:39:37.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:36 smithi149 ceph-mon[170394]: failed to encode map e430 with expected crc 2023-09-23T20:39:37.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:36 smithi149 ceph-mon[170394]: failed to encode map e430 with expected crc 2023-09-23T20:39:37.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:36 smithi149 ceph-mon[170394]: pgmap v1051: 129 pgs: 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 2.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 474 op/s; 12361/138009 objects degraded (8.957%); 4233/138009 objects misplaced (3.067%); 6.4 MiB/s, 102 objects/s recovering 2023-09-23T20:39:37.218 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:36 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 12556/140988 objects degraded (8.906%), 11 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T20:39:37.368 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:39:37.371 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:37.372 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:39:37.472 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:39:37.520 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:37.572 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:39:37.575 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:37.576 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:39:37.676 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:39:37.745 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:38.076 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:39:38.083 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:38.083 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:39:38.184 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:39:38.221 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:38.268 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:39:38.276 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:38.276 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:39:38.328 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:39:38.360 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:39:38.377 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:39:38.400 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:39:38.437 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:38.438 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:39:38.484 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:39:38.524 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:39:38.556 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:39:38.589 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:39:38.620 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:39:38.624 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:39:38.624 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:39:38.624 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:39:38.625 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:39:38.625 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:39:38.625 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:39:38.625 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:39:38.625 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:39:38.644 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:39:38.648 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:38.648 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:39:38.749 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:39:38.785 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:38.847 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:39:38.851 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:38.851 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:39:38.952 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:39:39.023 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:39.102 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:39:39.108 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:39.108 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:39:39.142 INFO:teuthology.orchestra.run.smithi018.stderr:reweighted osd.1 to 1 (10000) 2023-09-23T20:39:39.209 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:39:39.263 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:39.314 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:39:39.318 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:39.318 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:39:39.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:39 smithi018 ceph-mon[184144]: pgmap v1052: 129 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 4 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 2.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 478 op/s; 12361/135459 objects degraded (9.125%); 3995/135459 objects misplaced (2.949%); 6.9 MiB/s, 110 objects/s recovering 2023-09-23T20:39:39.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:39 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1018651196' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:39:39.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:39 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/458131250' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T20:39:39.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:39 smithi018 ceph-mon[179647]: pgmap v1052: 129 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 4 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 2.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 478 op/s; 12361/135459 objects degraded (9.125%); 3995/135459 objects misplaced (2.949%); 6.9 MiB/s, 110 objects/s recovering 2023-09-23T20:39:39.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:39 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1018651196' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:39:39.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:39 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/458131250' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T20:39:39.419 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:39:39.428 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T20:39:39.455 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:39 smithi149 ceph-mon[170394]: pgmap v1052: 129 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 4 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 2.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 478 op/s; 12361/135459 objects degraded (9.125%); 3995/135459 objects misplaced (2.949%); 6.9 MiB/s, 110 objects/s recovering 2023-09-23T20:39:39.455 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:39 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1018651196' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:39:39.455 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:39 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/458131250' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T20:39:39.459 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:39.515 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:39:39.525 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:39.525 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:39:39.626 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:39:39.664 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:39.711 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:39:39.718 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:39.719 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:39:39.820 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:39:39.869 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd reweight 7 1 2023-09-23T20:39:39.872 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:39.922 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:39:39.926 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:39.926 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:39:40.027 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:39:40.065 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:40.116 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:39:40.122 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:40.123 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:39:40.223 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:39:40.281 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:40.339 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:39:40.343 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:40.344 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:39:40.371 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2023-09-23T20:39:40.371 INFO:teuthology.orchestra.run.smithi018.stdout: "ops": [], 2023-09-23T20:39:40.371 INFO:teuthology.orchestra.run.smithi018.stdout: "num_ops": 0 2023-09-23T20:39:40.372 INFO:teuthology.orchestra.run.smithi018.stdout:} 2023-09-23T20:39:40.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:40 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/458131250' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T20:39:40.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:40 smithi018 ceph-mon[184144]: osdmap e431: 8 total, 8 up, 8 in 2023-09-23T20:39:40.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:40 smithi018 ceph-mon[184144]: failed to encode map e431 with expected crc 2023-09-23T20:39:40.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:40 smithi018 ceph-mon[184144]: failed to encode map e431 with expected crc 2023-09-23T20:39:40.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:40 smithi018 ceph-mon[184144]: failed to encode map e431 with expected crc 2023-09-23T20:39:40.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:40 smithi018 ceph-mon[184144]: failed to encode map e431 with expected crc 2023-09-23T20:39:40.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:40 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/458131250' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T20:39:40.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:40 smithi018 ceph-mon[179647]: osdmap e431: 8 total, 8 up, 8 in 2023-09-23T20:39:40.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:40 smithi018 ceph-mon[179647]: failed to encode map e431 with expected crc 2023-09-23T20:39:40.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:40 smithi018 ceph-mon[179647]: failed to encode map e431 with expected crc 2023-09-23T20:39:40.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:40 smithi018 ceph-mon[179647]: failed to encode map e431 with expected crc 2023-09-23T20:39:40.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:40 smithi018 ceph-mon[179647]: failed to encode map e431 with expected crc 2023-09-23T20:39:40.445 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:39:40.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:40 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/458131250' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T20:39:40.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:40 smithi149 ceph-mon[170394]: osdmap e431: 8 total, 8 up, 8 in 2023-09-23T20:39:40.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:40 smithi149 ceph-mon[170394]: failed to encode map e431 with expected crc 2023-09-23T20:39:40.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:40 smithi149 ceph-mon[170394]: failed to encode map e431 with expected crc 2023-09-23T20:39:40.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:40 smithi149 ceph-mon[170394]: failed to encode map e431 with expected crc 2023-09-23T20:39:40.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:40 smithi149 ceph-mon[170394]: failed to encode map e431 with expected crc 2023-09-23T20:39:40.473 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:39:40.486 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:40.591 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:39:40.594 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:40.595 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:39:40.696 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:39:40.733 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:40.777 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:39:40.781 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:40.781 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:39:40.882 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:39:40.919 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:40.964 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:39:40.973 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:40.974 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:39:41.074 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:39:41.109 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:39:41.118 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:41.174 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:39:41.178 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:41.178 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:39:41.196 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:39:41.255 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T20:39:41.279 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:39:41.311 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:41 smithi018 ceph-mon[179647]: pgmap v1054: 129 pgs: 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 2.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 622 op/s; 11993/130698 objects degraded (9.176%); 3910/130698 objects misplaced (2.992%); 9.6 MiB/s, 153 objects/s recovering 2023-09-23T20:39:41.311 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:41 smithi018 ceph-mon[184144]: pgmap v1054: 129 pgs: 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 2.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 622 op/s; 11993/130698 objects degraded (9.176%); 3910/130698 objects misplaced (2.992%); 9.6 MiB/s, 153 objects/s recovering 2023-09-23T20:39:41.360 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:41.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:41 smithi149 ceph-mon[170394]: pgmap v1054: 129 pgs: 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 2.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 622 op/s; 11993/130698 objects degraded (9.176%); 3910/130698 objects misplaced (2.992%); 9.6 MiB/s, 153 objects/s recovering 2023-09-23T20:39:41.550 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:39:41.557 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:41.557 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:39:41.659 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:39:41.714 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:41.820 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:39:41.824 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:41.824 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:39:41.925 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:39:41.966 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:42.024 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:39:42.030 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:42.031 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:39:42.131 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:39:42.168 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:42.214 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:39:42.221 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:42.221 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:39:42.322 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:39:42.384 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:42.437 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:39:42.451 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:42.452 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:39:42.538 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:39:42.552 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:39:42.552 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:39:42.587 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:42 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 11993/130698 objects degraded (9.176%), 11 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T20:39:42.590 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:42.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:42 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 11993/130698 objects degraded (9.176%), 11 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T20:39:42.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:42 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 11993/130698 objects degraded (9.176%), 11 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T20:39:42.649 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:39:42.658 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:42.659 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:39:42.760 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:39:42.812 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:42.859 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:39:42.865 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:42.866 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:39:42.967 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:39:43.014 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:43.063 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:39:43.070 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:43.070 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:39:43.171 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:39:43.207 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:43.274 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:39:43.279 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:43.279 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:39:43.380 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:39:43.416 INFO:teuthology.orchestra.run.smithi018.stderr:noscrub is unset 2023-09-23T20:39:43.420 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:43.473 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:43 smithi149 ceph-mon[170394]: pgmap v1055: 129 pgs: 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 2.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 543 op/s; 11993/127926 objects degraded (9.375%); 3910/127926 objects misplaced (3.056%); 6.9 MiB/s, 110 objects/s recovering 2023-09-23T20:39:43.473 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:43 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2958712112' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:39:43.490 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:39:43.495 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:43.495 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:39:43.596 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:39:43.632 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:43 smithi018 ceph-mon[184144]: pgmap v1055: 129 pgs: 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 2.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 543 op/s; 11993/127926 objects degraded (9.375%); 3910/127926 objects misplaced (3.056%); 6.9 MiB/s, 110 objects/s recovering 2023-09-23T20:39:43.633 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:43 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2958712112' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:39:43.633 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:43 smithi018 ceph-mon[179647]: pgmap v1055: 129 pgs: 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 2.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 543 op/s; 11993/127926 objects degraded (9.375%); 3910/127926 objects misplaced (3.056%); 6.9 MiB/s, 110 objects/s recovering 2023-09-23T20:39:43.633 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:43 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2958712112' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:39:43.634 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:44.192 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:39:44.196 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:44.196 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:39:44.298 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:39:44.341 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:44.391 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:39:44.395 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:44.395 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:39:44.436 INFO:teuthology.orchestra.run.smithi018.stderr:reweighted osd.7 to 1 (10000) 2023-09-23T20:39:44.496 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:39:44.537 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:44 smithi018 ceph-mon[179647]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:39:44.537 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:44 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2958712112' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:39:44.538 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:44 smithi018 ceph-mon[179647]: osdmap e432: 8 total, 8 up, 8 in 2023-09-23T20:39:44.538 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:44 smithi018 ceph-mon[179647]: failed to encode map e432 with expected crc 2023-09-23T20:39:44.538 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:44 smithi018 ceph-mon[179647]: failed to encode map e432 with expected crc 2023-09-23T20:39:44.538 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:44 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/796400041' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T20:39:44.538 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:44 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T20:39:44.538 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:44 smithi018 ceph-mon[179647]: 1.a scrub starts 2023-09-23T20:39:44.539 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:44 smithi018 ceph-mon[184144]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:39:44.539 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:44 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2958712112' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:39:44.539 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:44 smithi018 ceph-mon[184144]: osdmap e432: 8 total, 8 up, 8 in 2023-09-23T20:39:44.539 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:44 smithi018 ceph-mon[184144]: failed to encode map e432 with expected crc 2023-09-23T20:39:44.539 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:44 smithi018 ceph-mon[184144]: failed to encode map e432 with expected crc 2023-09-23T20:39:44.540 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:44 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/796400041' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T20:39:44.540 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:44 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T20:39:44.540 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:44 smithi018 ceph-mon[184144]: 1.a scrub starts 2023-09-23T20:39:44.565 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:44.617 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:39:44.620 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:44.620 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:39:44.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:44 smithi149 ceph-mon[170394]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:39:44.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:44 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2958712112' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:39:44.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:44 smithi149 ceph-mon[170394]: osdmap e432: 8 total, 8 up, 8 in 2023-09-23T20:39:44.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:44 smithi149 ceph-mon[170394]: failed to encode map e432 with expected crc 2023-09-23T20:39:44.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:44 smithi149 ceph-mon[170394]: failed to encode map e432 with expected crc 2023-09-23T20:39:44.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:44 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/796400041' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T20:39:44.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:44 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T20:39:44.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:44 smithi149 ceph-mon[170394]: 1.a scrub starts 2023-09-23T20:39:44.721 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:39:44.772 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:44.875 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:39:44.882 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:44.883 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:39:44.935 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd reweight 3 1 2023-09-23T20:39:44.984 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:39:45.038 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:45.130 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:39:45.142 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:39:44 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:39:44] "GET /metrics HTTP/1.1" 200 36559 "" "Prometheus/2.43.0" 2023-09-23T20:39:45.143 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:45.143 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:39:45.209 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2023-09-23T20:39:45.210 INFO:teuthology.orchestra.run.smithi018.stdout: "ops": [], 2023-09-23T20:39:45.210 INFO:teuthology.orchestra.run.smithi018.stdout: "complaint_time": 30, 2023-09-23T20:39:45.210 INFO:teuthology.orchestra.run.smithi018.stdout: "num_blocked_ops": 0 2023-09-23T20:39:45.210 INFO:teuthology.orchestra.run.smithi018.stdout:} 2023-09-23T20:39:45.244 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:39:45.286 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:45.342 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:39:45.347 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:45.348 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:39:45.366 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:39:45.402 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:39:45.405 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:45 smithi018 ceph-mon[179647]: pgmap v1057: 129 pgs: 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 2.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 584 op/s; 11787/123969 objects degraded (9.508%); 3577/123969 objects misplaced (2.885%); 9.6 MiB/s, 153 objects/s recovering 2023-09-23T20:39:45.405 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:45 smithi018 ceph-mon[179647]: failed to encode map e433 with expected crc 2023-09-23T20:39:45.405 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:45 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T20:39:45.405 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:45 smithi018 ceph-mon[179647]: osdmap e433: 8 total, 8 up, 8 in 2023-09-23T20:39:45.405 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:45 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:39:45.405 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:45 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:39:45.405 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:45 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1313298720' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:39:45.443 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:39:45.448 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:39:45.483 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:39:45.490 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:45.521 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:39:45.551 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:39:45.553 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:39:45.556 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:45.557 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:39:45.557 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:45 smithi149 ceph-mon[170394]: pgmap v1057: 129 pgs: 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 2.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 584 op/s; 11787/123969 objects degraded (9.508%); 3577/123969 objects misplaced (2.885%); 9.6 MiB/s, 153 objects/s recovering 2023-09-23T20:39:45.558 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:45 smithi149 ceph-mon[170394]: failed to encode map e433 with expected crc 2023-09-23T20:39:45.558 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:45 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T20:39:45.558 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:45 smithi149 ceph-mon[170394]: osdmap e433: 8 total, 8 up, 8 in 2023-09-23T20:39:45.558 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:45 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:39:45.558 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:45 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:39:45.558 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:45 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1313298720' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:39:45.589 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:39:45.624 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:39:45.634 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:39:45.634 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:39:45.635 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:39:45.635 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:39:45.635 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:39:45.635 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:39:45.635 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:39:45.635 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:39:45.656 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T20:39:45.657 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:45 smithi018 ceph-mon[184144]: pgmap v1057: 129 pgs: 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 2.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 584 op/s; 11787/123969 objects degraded (9.508%); 3577/123969 objects misplaced (2.885%); 9.6 MiB/s, 153 objects/s recovering 2023-09-23T20:39:45.657 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:45 smithi018 ceph-mon[184144]: failed to encode map e433 with expected crc 2023-09-23T20:39:45.658 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:45 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T20:39:45.658 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:45 smithi018 ceph-mon[184144]: osdmap e433: 8 total, 8 up, 8 in 2023-09-23T20:39:45.658 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:45 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:39:45.658 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:45 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:39:45.658 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:45 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1313298720' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:39:45.658 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:39:45.740 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:45.846 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:39:45.855 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:45.856 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:39:45.957 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:39:46.014 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:46.072 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:39:46.075 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:46.075 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:39:46.177 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:39:46.217 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:46.248 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T20:39:46.282 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:39:46.286 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:46.286 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:39:46.289 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:39:46.387 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:39:46.430 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:46.481 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:39:46.485 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:46.485 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:39:46.555 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:39:46.586 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:39:46.644 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:46 smithi149 ceph-mon[170394]: 1.a scrub starts 2023-09-23T20:39:46.644 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:46 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 11787/123969 objects degraded (9.508%), 11 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T20:39:46.644 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:46.712 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:39:46.716 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:46.716 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:39:46.802 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:46 smithi018 ceph-mon[184144]: 1.a scrub starts 2023-09-23T20:39:46.802 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:46 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 11787/123969 objects degraded (9.508%), 11 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T20:39:46.804 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:46 smithi018 ceph-mon[179647]: 1.a scrub starts 2023-09-23T20:39:46.804 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:46 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 11787/123969 objects degraded (9.508%), 11 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T20:39:46.817 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:39:46.854 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:46.913 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:39:46.921 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:46.921 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:39:47.022 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:39:47.094 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:47.157 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:39:47.165 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:47.166 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:39:47.266 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:39:47.313 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:47.388 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:39:47.397 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:47.398 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:39:47.498 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:39:47.559 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:47.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:47 smithi149 ceph-mon[170394]: pgmap v1059: 129 pgs: 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 2.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 596 op/s; 11594/120444 objects degraded (9.626%); 3577/120444 objects misplaced (2.970%); 8.2 MiB/s, 131 objects/s recovering 2023-09-23T20:39:47.733 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:47 smithi018 ceph-mon[184144]: pgmap v1059: 129 pgs: 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 2.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 596 op/s; 11594/120444 objects degraded (9.626%); 3577/120444 objects misplaced (2.970%); 8.2 MiB/s, 131 objects/s recovering 2023-09-23T20:39:47.735 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:47 smithi018 ceph-mon[179647]: pgmap v1059: 129 pgs: 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 2.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 596 op/s; 11594/120444 objects degraded (9.626%); 3577/120444 objects misplaced (2.970%); 8.2 MiB/s, 131 objects/s recovering 2023-09-23T20:39:48.141 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:39:48.144 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:39:48.145 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:48.146 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:39:48.192 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:39:48.247 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:39:48.307 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:48.451 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:39:48.458 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:48.458 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:39:48.559 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:39:48.637 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:48.765 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:39:48.774 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:48.774 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:39:48.875 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:39:48.910 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:48.963 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:39:48.967 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:48.967 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:39:49.068 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:39:49.110 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:49.130 INFO:teuthology.orchestra.run.smithi018.stdout:ERROR: (22) Invalid argument 2023-09-23T20:39:49.130 INFO:teuthology.orchestra.run.smithi018.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:39:49.159 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:39:49.164 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:49.165 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:39:49.165 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:39:49.266 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:39:49.318 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:49.423 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:39:49.427 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:49.427 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:39:49.448 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:49 smithi149 ceph-mon[170394]: pgmap v1060: 129 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 4 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 2.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 557 op/s; 11594/117306 objects degraded (9.884%); 3354/117306 objects misplaced (2.859%); 7.5 MiB/s, 119 objects/s recovering 2023-09-23T20:39:49.448 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:49 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1017670352' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T20:39:49.448 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:49 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T20:39:49.467 INFO:teuthology.orchestra.run.smithi018.stderr:reweighted osd.3 to 1 (10000) 2023-09-23T20:39:49.528 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:39:49.570 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:49.806 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:49 smithi018 ceph-mon[184144]: pgmap v1060: 129 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 4 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 2.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 557 op/s; 11594/117306 objects degraded (9.884%); 3354/117306 objects misplaced (2.859%); 7.5 MiB/s, 119 objects/s recovering 2023-09-23T20:39:49.806 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:49 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1017670352' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T20:39:49.806 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:49 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T20:39:49.808 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:49 smithi018 ceph-mon[179647]: pgmap v1060: 129 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 4 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 2.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 557 op/s; 11594/117306 objects degraded (9.884%); 3354/117306 objects misplaced (2.859%); 7.5 MiB/s, 119 objects/s recovering 2023-09-23T20:39:49.808 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:49 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1017670352' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T20:39:49.809 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:49 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T20:39:49.862 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:39:49.868 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:49.869 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:39:49.914 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T20:39:49.970 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:39:50.013 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:50.074 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:39:50.078 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:50.078 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:39:50.159 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd reweight 2 1 2023-09-23T20:39:50.179 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:39:50.215 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:50.262 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:39:50.267 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:50.268 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:39:50.368 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:39:50.447 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:50 smithi018 ceph-mon[179647]: failed to encode map e434 with expected crc 2023-09-23T20:39:50.448 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:50 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T20:39:50.448 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:50 smithi018 ceph-mon[179647]: osdmap e434: 8 total, 8 up, 8 in 2023-09-23T20:39:50.448 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:50 smithi018 ceph-mon[179647]: failed to encode map e434 with expected crc 2023-09-23T20:39:50.455 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:50.593 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:39:50.600 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:50.601 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:39:50.702 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:39:50.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:50 smithi149 ceph-mon[170394]: failed to encode map e434 with expected crc 2023-09-23T20:39:50.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:50 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T20:39:50.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:50 smithi149 ceph-mon[170394]: osdmap e434: 8 total, 8 up, 8 in 2023-09-23T20:39:50.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:50 smithi149 ceph-mon[170394]: failed to encode map e434 with expected crc 2023-09-23T20:39:50.734 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:50.795 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:39:50.799 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:50.799 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:39:50.886 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:50 smithi018 ceph-mon[184144]: failed to encode map e434 with expected crc 2023-09-23T20:39:50.887 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:50 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T20:39:50.887 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:50 smithi018 ceph-mon[184144]: osdmap e434: 8 total, 8 up, 8 in 2023-09-23T20:39:50.887 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:50 smithi018 ceph-mon[184144]: failed to encode map e434 with expected crc 2023-09-23T20:39:50.887 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:39:50.900 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:39:50.997 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:51.160 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:39:51.176 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:51.176 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:39:51.277 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:39:51.337 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:39:51.345 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:51.390 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:39:51.418 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:39:51.424 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:51.424 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:39:51.428 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:39:51.454 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:51 smithi018 ceph-mon[179647]: pgmap v1062: 129 pgs: 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 2.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 709 op/s; 11143/111525 objects degraded (9.991%); 3244/111525 objects misplaced (2.909%); 10 MiB/s, 161 objects/s recovering 2023-09-23T20:39:51.463 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:39:51.491 INFO:teuthology.orchestra.run.smithi018.stderr:nodeep-scrub is unset 2023-09-23T20:39:51.503 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:39:51.520 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:39:51.525 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:39:51.553 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:39:51.575 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:51.599 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:39:51.621 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:39:51.628 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:51.628 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:39:51.637 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:39:51.648 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:39:51.648 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:39:51.648 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:39:51.648 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:39:51.648 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:39:51.648 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:39:51.648 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:39:51.648 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:39:51.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:51 smithi149 ceph-mon[170394]: pgmap v1062: 129 pgs: 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 2.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 709 op/s; 11143/111525 objects degraded (9.991%); 3244/111525 objects misplaced (2.909%); 10 MiB/s, 161 objects/s recovering 2023-09-23T20:39:51.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:51 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3690463152' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:39:51.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:51 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3063437189' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:39:51.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:51 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 11143/111525 objects degraded (9.991%), 11 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T20:39:51.730 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:39:51.808 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:51 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3690463152' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:39:51.809 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:51 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3063437189' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:39:51.809 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:51 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 11143/111525 objects degraded (9.991%), 11 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T20:39:51.813 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:51.814 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:51 smithi018 ceph-mon[184144]: pgmap v1062: 129 pgs: 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 2.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 709 op/s; 11143/111525 objects degraded (9.991%); 3244/111525 objects misplaced (2.909%); 10 MiB/s, 161 objects/s recovering 2023-09-23T20:39:51.814 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:51 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3690463152' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:39:51.814 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:51 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3063437189' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:39:51.814 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:51 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 11143/111525 objects degraded (9.991%), 11 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T20:39:52.254 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:39:52.268 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:52.268 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:39:52.369 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:39:52.407 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:52.454 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:39:52.465 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:52.466 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:39:52.565 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:52 smithi018 ceph-mon[179647]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:39:52.565 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:52 smithi018 ceph-mon[179647]: failed to encode map e435 with expected crc 2023-09-23T20:39:52.566 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:52 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3063437189' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:39:52.566 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:52 smithi018 ceph-mon[179647]: osdmap e435: 8 total, 8 up, 8 in 2023-09-23T20:39:52.566 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:52 smithi018 ceph-mon[184144]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:39:52.566 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:52 smithi018 ceph-mon[184144]: failed to encode map e435 with expected crc 2023-09-23T20:39:52.566 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:52 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3063437189' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:39:52.566 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:52 smithi018 ceph-mon[184144]: osdmap e435: 8 total, 8 up, 8 in 2023-09-23T20:39:52.566 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:39:52.612 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:52.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:52 smithi149 ceph-mon[170394]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:39:52.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:52 smithi149 ceph-mon[170394]: failed to encode map e435 with expected crc 2023-09-23T20:39:52.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:52 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3063437189' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:39:52.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:52 smithi149 ceph-mon[170394]: osdmap e435: 8 total, 8 up, 8 in 2023-09-23T20:39:53.060 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:39:53.068 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:53.068 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:39:53.169 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:39:53.200 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:53.252 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:39:53.261 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:53.261 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:39:53.270 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2023-09-23T20:39:53.270 INFO:teuthology.orchestra.run.smithi018.stdout: "ops": [ 2023-09-23T20:39:53.270 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:53.271 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35994.0:145114 8.e 8:7961a3a1:::benchmark_data_smithi018_341490_object55791:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e435)", 2023-09-23T20:39:53.271 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:39:53.266704+0000", 2023-09-23T20:39:53.271 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.0034541390000000002, 2023-09-23T20:39:53.271 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0034833619999999998, 2023-09-23T20:39:53.271 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:39:53.271 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:39:53.271 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:39:53.271 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35994", 2023-09-23T20:39:53.271 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:39:53.271 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 145114 2023-09-23T20:39:53.271 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:53.272 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:39:53.272 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:53.272 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:39:53.272 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:39:53.266704+0000", 2023-09-23T20:39:53.272 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:39:53.272 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:53.272 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:53.272 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:39:53.272 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:39:53.266704+0000", 2023-09-23T20:39:53.272 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:39:53.272 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:53.273 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:53.273 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:39:53.273 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:39:53.266703+0000", 2023-09-23T20:39:53.273 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:39:53.273 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:53.273 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:53.273 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:39:53.273 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:39:53.266711+0000", 2023-09-23T20:39:53.273 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.9829999999999996e-06 2023-09-23T20:39:53.273 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:53.273 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:53.273 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:39:53.274 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:39:53.266729+0000", 2023-09-23T20:39:53.274 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.7716000000000001e-05 2023-09-23T20:39:53.274 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:53.274 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:53.274 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:39:53.274 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:39:53.266738+0000", 2023-09-23T20:39:53.274 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.0720000000000006e-06 2023-09-23T20:39:53.274 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:53.274 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:53.274 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:39:53.274 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:39:53.266811+0000", 2023-09-23T20:39:53.274 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.3615000000000006e-05 2023-09-23T20:39:53.275 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:53.275 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:53.275 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:39:53.275 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:39:53.266955+0000", 2023-09-23T20:39:53.275 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000144015 2023-09-23T20:39:53.275 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:39:53.275 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:39:53.275 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 2,4", 2023-09-23T20:39:53.275 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:39:53.267085+0000", 2023-09-23T20:39:53.275 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000129367 2023-09-23T20:39:53.275 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:39:53.275 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:39:53.275 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:39:53.276 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:39:53.276 INFO:teuthology.orchestra.run.smithi018.stdout: ], 2023-09-23T20:39:53.276 INFO:teuthology.orchestra.run.smithi018.stdout: "num_ops": 1 2023-09-23T20:39:53.276 INFO:teuthology.orchestra.run.smithi018.stdout:} 2023-09-23T20:39:53.361 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:39:53.442 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:53.583 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:53 smithi018 ceph-mon[184144]: pgmap v1064: 129 pgs: 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 2.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 673 op/s; 11143/108486 objects degraded (10.271%); 3244/108486 objects misplaced (2.990%); 8.0 MiB/s, 127 objects/s recovering 2023-09-23T20:39:53.583 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:53 smithi018 ceph-mon[184144]: 1.10 scrub starts 2023-09-23T20:39:53.583 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:53 smithi018 ceph-mon[179647]: pgmap v1064: 129 pgs: 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 2.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 673 op/s; 11143/108486 objects degraded (10.271%); 3244/108486 objects misplaced (2.990%); 8.0 MiB/s, 127 objects/s recovering 2023-09-23T20:39:53.584 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:53 smithi018 ceph-mon[179647]: 1.10 scrub starts 2023-09-23T20:39:53.715 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:39:53.722 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:53.723 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:39:53.824 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:39:53.874 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:53 smithi149 ceph-mon[170394]: pgmap v1064: 129 pgs: 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 2.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 673 op/s; 11143/108486 objects degraded (10.271%); 3244/108486 objects misplaced (2.990%); 8.0 MiB/s, 127 objects/s recovering 2023-09-23T20:39:53.874 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:53 smithi149 ceph-mon[170394]: 1.10 scrub starts 2023-09-23T20:39:53.881 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:53.941 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:39:53.945 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:53.946 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:39:54.046 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:39:54.102 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:54.143 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T20:39:54.154 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:39:54.159 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:54.159 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:39:54.260 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:39:54.282 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:39:54.378 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:54.468 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:39:54.476 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:54.476 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:39:54.577 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:39:54.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:54 smithi018 ceph-mon[179647]: 1.a scrub starts 2023-09-23T20:39:54.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:54 smithi018 ceph-mon[179647]: 1.10 scrub starts 2023-09-23T20:39:54.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:54 smithi018 ceph-mon[184144]: 1.a scrub starts 2023-09-23T20:39:54.643 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:54 smithi018 ceph-mon[184144]: 1.10 scrub starts 2023-09-23T20:39:54.673 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:54.769 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:39:54.776 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:54.776 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:39:54.877 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:39:54.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:54 smithi149 ceph-mon[170394]: 1.a scrub starts 2023-09-23T20:39:54.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:54 smithi149 ceph-mon[170394]: 1.10 scrub starts 2023-09-23T20:39:54.967 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:55.043 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:39:55.047 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:55.048 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:39:55.062 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:39:55.148 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:39:55.178 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T20:39:55.204 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:39:54 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:39:54] "GET /metrics HTTP/1.1" 200 36559 "" "Prometheus/2.43.0" 2023-09-23T20:39:55.208 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:55.226 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:39:55.343 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:39:55.353 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:55.353 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:39:55.454 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:39:55.497 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:55.501 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:55 smithi018 ceph-mon[179647]: pgmap v1065: 129 pgs: 1 active+clean+scrubbing, 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 2.1 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 682 op/s; 10913/104070 objects degraded (10.486%); 2849/104070 objects misplaced (2.738%); 11 MiB/s, 175 objects/s recovering 2023-09-23T20:39:55.502 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:55 smithi018 ceph-mon[179647]: 1.10 deep-scrub starts 2023-09-23T20:39:55.506 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:55 smithi018 ceph-mon[184144]: pgmap v1065: 129 pgs: 1 active+clean+scrubbing, 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 2.1 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 682 op/s; 10913/104070 objects degraded (10.486%); 2849/104070 objects misplaced (2.738%); 11 MiB/s, 175 objects/s recovering 2023-09-23T20:39:55.506 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:55 smithi018 ceph-mon[184144]: 1.10 deep-scrub starts 2023-09-23T20:39:55.506 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:55 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/304834443' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T20:39:55.506 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:55 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T20:39:55.542 INFO:teuthology.orchestra.run.smithi018.stderr:reweighted osd.2 to 1 (10000) 2023-09-23T20:39:55.555 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:39:55.559 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:55.559 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:39:55.660 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:39:55.724 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:55.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:55 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/304834443' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T20:39:55.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:55 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T20:39:55.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:55 smithi149 ceph-mon[170394]: pgmap v1065: 129 pgs: 1 active+clean+scrubbing, 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 2.1 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 682 op/s; 10913/104070 objects degraded (10.486%); 2849/104070 objects misplaced (2.738%); 11 MiB/s, 175 objects/s recovering 2023-09-23T20:39:55.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:55 smithi149 ceph-mon[170394]: 1.10 deep-scrub starts 2023-09-23T20:39:55.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:55 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/304834443' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T20:39:55.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:55 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T20:39:56.393 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:39:56.399 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:56.399 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:39:56.429 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd reweight 4 1 2023-09-23T20:39:56.500 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:39:56.524 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:56 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T20:39:56.524 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:56 smithi018 ceph-mon[179647]: osdmap e436: 8 total, 8 up, 8 in 2023-09-23T20:39:56.524 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:56 smithi018 ceph-mon[179647]: failed to encode map e436 with expected crc 2023-09-23T20:39:56.524 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:56 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 10913/104070 objects degraded (10.486%), 11 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T20:39:56.524 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:56 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T20:39:56.524 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:56 smithi018 ceph-mon[184144]: osdmap e436: 8 total, 8 up, 8 in 2023-09-23T20:39:56.524 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:56 smithi018 ceph-mon[184144]: failed to encode map e436 with expected crc 2023-09-23T20:39:56.524 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:56 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 10913/104070 objects degraded (10.486%), 11 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T20:39:56.550 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:56.610 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:39:56.619 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:56.619 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:39:56.720 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:39:56.829 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:56.945 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:39:56.951 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:56.951 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:39:56.955 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:39:56.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:56 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T20:39:56.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:56 smithi149 ceph-mon[170394]: osdmap e436: 8 total, 8 up, 8 in 2023-09-23T20:39:56.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:56 smithi149 ceph-mon[170394]: failed to encode map e436 with expected crc 2023-09-23T20:39:56.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:56 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 10913/104070 objects degraded (10.486%), 11 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T20:39:57.052 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:39:57.084 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:57.151 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:39:57.157 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:57.157 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:39:57.258 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:39:57.307 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:57.359 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:39:57.364 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:57.365 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:39:57.466 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:39:57.479 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:39:57.504 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:57.530 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:39:57.548 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:39:57.555 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:57.555 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:39:57.574 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:39:57.623 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:39:57.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:57 smithi018 ceph-mon[179647]: pgmap v1067: 129 pgs: 1 active+clean+scrubbing, 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 703 op/s; 10680/100140 objects degraded (10.665%); 2849/100140 objects misplaced (2.845%); 10 MiB/s, 163 objects/s recovering 2023-09-23T20:39:57.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:57 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2890795853' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:39:57.644 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:57 smithi018 ceph-mon[184144]: pgmap v1067: 129 pgs: 1 active+clean+scrubbing, 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 703 op/s; 10680/100140 objects degraded (10.665%); 2849/100140 objects misplaced (2.845%); 10 MiB/s, 163 objects/s recovering 2023-09-23T20:39:57.644 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:57 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2890795853' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:39:57.656 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:39:57.659 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:39:57.694 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:57 smithi149 ceph-mon[170394]: pgmap v1067: 129 pgs: 1 active+clean+scrubbing, 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 703 op/s; 10680/100140 objects degraded (10.665%); 2849/100140 objects misplaced (2.845%); 10 MiB/s, 163 objects/s recovering 2023-09-23T20:39:57.694 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:57 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2890795853' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:39:57.699 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:57.707 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:39:57.734 INFO:teuthology.orchestra.run.smithi018.stdout:ERROR: (22) Invalid argument 2023-09-23T20:39:57.734 INFO:teuthology.orchestra.run.smithi018.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:39:57.753 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:39:57.756 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:39:57.760 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:57.761 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:39:57.806 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:39:57.815 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:39:57.816 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:39:57.816 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:39:57.816 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:39:57.816 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:39:57.816 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:39:57.816 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:39:57.816 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:39:57.862 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:39:57.911 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:57.962 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:39:57.967 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:57.968 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:39:58.069 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:39:58.115 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:58.174 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:39:58.183 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:58.183 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:39:58.284 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:39:58.350 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:58.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:58 smithi018 ceph-mon[179647]: 1.10 scrub starts 2023-09-23T20:39:58.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:58 smithi018 ceph-mon[179647]: 1.a scrub starts 2023-09-23T20:39:58.643 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:58 smithi018 ceph-mon[184144]: 1.10 scrub starts 2023-09-23T20:39:58.643 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:58 smithi018 ceph-mon[184144]: 1.a scrub starts 2023-09-23T20:39:58.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:58 smithi149 ceph-mon[170394]: 1.10 scrub starts 2023-09-23T20:39:58.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:58 smithi149 ceph-mon[170394]: 1.a scrub starts 2023-09-23T20:39:59.205 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T20:39:59.222 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:39:59.226 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:59.227 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:39:59.328 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:39:59.360 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:59.410 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:39:59.411 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:39:59.419 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:59.419 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:39:59.520 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:39:59.592 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:39:59.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:59 smithi149 ceph-mon[170394]: pgmap v1068: 129 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 4 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 616 op/s; 10680/96714 objects degraded (11.043%); 2590/96714 objects misplaced (2.678%); 8.7 MiB/s, 139 objects/s recovering 2023-09-23T20:39:59.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:59 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:39:59.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:39:59 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:39:59.733 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:59 smithi018 ceph-mon[184144]: pgmap v1068: 129 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 4 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 616 op/s; 10680/96714 objects degraded (11.043%); 2590/96714 objects misplaced (2.678%); 8.7 MiB/s, 139 objects/s recovering 2023-09-23T20:39:59.733 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:59 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:39:59.733 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:39:59 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:39:59.734 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:59 smithi018 ceph-mon[179647]: pgmap v1068: 129 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 4 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 616 op/s; 10680/96714 objects degraded (11.043%); 2590/96714 objects misplaced (2.678%); 8.7 MiB/s, 139 objects/s recovering 2023-09-23T20:39:59.734 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:59 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:39:59.734 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:39:59 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:39:59.770 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:39:59.774 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:39:59.775 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:39:59.876 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:39:59.942 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:00.015 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:40:00.021 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:00.021 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:40:00.122 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:40:00.200 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:00.265 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:40:00.276 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:00.276 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:40:00.377 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:40:00.457 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:00.742 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:40:00.745 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:00.745 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:40:00.796 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:40:00.846 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:40:00.890 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:00 smithi149 ceph-mon[170394]: Health detail: HEALTH_WARN Degraded data redundancy: 10680/96714 objects degraded (11.043%), 11 pgs degraded, 5 pgs undersized 2023-09-23T20:40:00.890 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:00 smithi149 ceph-mon[170394]: [WRN] PG_DEGRADED: Degraded data redundancy: 10680/96714 objects degraded (11.043%), 11 pgs degraded, 5 pgs undersized 2023-09-23T20:40:00.890 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:00 smithi149 ceph-mon[170394]: pg 8.2 is stuck undersized for 79s, current state active+recovery_wait+undersized+degraded+remapped, last acting [4,0] 2023-09-23T20:40:00.891 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:00 smithi149 ceph-mon[170394]: pg 8.3 is active+recovery_wait+degraded, acting [1,6,2] 2023-09-23T20:40:00.891 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:00 smithi149 ceph-mon[170394]: pg 8.4 is stuck undersized for 79s, current state active+recovery_wait+undersized+degraded+remapped, last acting [4,2] 2023-09-23T20:40:00.891 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:00 smithi149 ceph-mon[170394]: pg 8.7 is active+recovery_wait+degraded, acting [1,3,6] 2023-09-23T20:40:00.891 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:00 smithi149 ceph-mon[170394]: pg 8.9 is active+recovery_wait+degraded, acting [6,0,3] 2023-09-23T20:40:00.891 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:00 smithi149 ceph-mon[170394]: pg 8.d is stuck undersized for 115s, current state active+recovering+undersized+degraded+remapped, last acting [7,2] 2023-09-23T20:40:00.891 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:00 smithi149 ceph-mon[170394]: pg 8.12 is stuck undersized for 79s, current state active+recovery_wait+undersized+degraded+remapped, last acting [4,0] 2023-09-23T20:40:00.891 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:00 smithi149 ceph-mon[170394]: pg 8.13 is active+recovery_wait+degraded, acting [1,6,2] 2023-09-23T20:40:00.891 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:00 smithi149 ceph-mon[170394]: pg 8.14 is stuck undersized for 79s, current state active+recovery_wait+undersized+degraded+remapped, last acting [4,2] 2023-09-23T20:40:00.891 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:00 smithi149 ceph-mon[170394]: pg 8.17 is active+recovery_wait+degraded, acting [1,3,6] 2023-09-23T20:40:00.891 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:00 smithi149 ceph-mon[170394]: pg 8.18 is active+recovery_wait+degraded, acting [3,7,0] 2023-09-23T20:40:00.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:00 smithi018 ceph-mon[184144]: Health detail: HEALTH_WARN Degraded data redundancy: 10680/96714 objects degraded (11.043%), 11 pgs degraded, 5 pgs undersized 2023-09-23T20:40:00.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:00 smithi018 ceph-mon[184144]: [WRN] PG_DEGRADED: Degraded data redundancy: 10680/96714 objects degraded (11.043%), 11 pgs degraded, 5 pgs undersized 2023-09-23T20:40:00.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:00 smithi018 ceph-mon[184144]: pg 8.2 is stuck undersized for 79s, current state active+recovery_wait+undersized+degraded+remapped, last acting [4,0] 2023-09-23T20:40:00.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:00 smithi018 ceph-mon[184144]: pg 8.3 is active+recovery_wait+degraded, acting [1,6,2] 2023-09-23T20:40:00.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:00 smithi018 ceph-mon[184144]: pg 8.4 is stuck undersized for 79s, current state active+recovery_wait+undersized+degraded+remapped, last acting [4,2] 2023-09-23T20:40:00.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:00 smithi018 ceph-mon[184144]: pg 8.7 is active+recovery_wait+degraded, acting [1,3,6] 2023-09-23T20:40:00.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:00 smithi018 ceph-mon[184144]: pg 8.9 is active+recovery_wait+degraded, acting [6,0,3] 2023-09-23T20:40:00.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:00 smithi018 ceph-mon[184144]: pg 8.d is stuck undersized for 115s, current state active+recovering+undersized+degraded+remapped, last acting [7,2] 2023-09-23T20:40:00.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:00 smithi018 ceph-mon[184144]: pg 8.12 is stuck undersized for 79s, current state active+recovery_wait+undersized+degraded+remapped, last acting [4,0] 2023-09-23T20:40:00.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:00 smithi018 ceph-mon[184144]: pg 8.13 is active+recovery_wait+degraded, acting [1,6,2] 2023-09-23T20:40:00.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:00 smithi018 ceph-mon[184144]: pg 8.14 is stuck undersized for 79s, current state active+recovery_wait+undersized+degraded+remapped, last acting [4,2] 2023-09-23T20:40:00.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:00 smithi018 ceph-mon[184144]: pg 8.17 is active+recovery_wait+degraded, acting [1,3,6] 2023-09-23T20:40:00.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:00 smithi018 ceph-mon[184144]: pg 8.18 is active+recovery_wait+degraded, acting [3,7,0] 2023-09-23T20:40:00.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:00 smithi018 ceph-mon[179647]: Health detail: HEALTH_WARN Degraded data redundancy: 10680/96714 objects degraded (11.043%), 11 pgs degraded, 5 pgs undersized 2023-09-23T20:40:00.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:00 smithi018 ceph-mon[179647]: [WRN] PG_DEGRADED: Degraded data redundancy: 10680/96714 objects degraded (11.043%), 11 pgs degraded, 5 pgs undersized 2023-09-23T20:40:00.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:00 smithi018 ceph-mon[179647]: pg 8.2 is stuck undersized for 79s, current state active+recovery_wait+undersized+degraded+remapped, last acting [4,0] 2023-09-23T20:40:00.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:00 smithi018 ceph-mon[179647]: pg 8.3 is active+recovery_wait+degraded, acting [1,6,2] 2023-09-23T20:40:00.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:00 smithi018 ceph-mon[179647]: pg 8.4 is stuck undersized for 79s, current state active+recovery_wait+undersized+degraded+remapped, last acting [4,2] 2023-09-23T20:40:00.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:00 smithi018 ceph-mon[179647]: pg 8.7 is active+recovery_wait+degraded, acting [1,3,6] 2023-09-23T20:40:00.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:00 smithi018 ceph-mon[179647]: pg 8.9 is active+recovery_wait+degraded, acting [6,0,3] 2023-09-23T20:40:00.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:00 smithi018 ceph-mon[179647]: pg 8.d is stuck undersized for 115s, current state active+recovering+undersized+degraded+remapped, last acting [7,2] 2023-09-23T20:40:00.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:00 smithi018 ceph-mon[179647]: pg 8.12 is stuck undersized for 79s, current state active+recovery_wait+undersized+degraded+remapped, last acting [4,0] 2023-09-23T20:40:00.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:00 smithi018 ceph-mon[179647]: pg 8.13 is active+recovery_wait+degraded, acting [1,6,2] 2023-09-23T20:40:00.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:00 smithi018 ceph-mon[179647]: pg 8.14 is stuck undersized for 79s, current state active+recovery_wait+undersized+degraded+remapped, last acting [4,2] 2023-09-23T20:40:00.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:00 smithi018 ceph-mon[179647]: pg 8.17 is active+recovery_wait+degraded, acting [1,3,6] 2023-09-23T20:40:00.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:00 smithi018 ceph-mon[179647]: pg 8.18 is active+recovery_wait+degraded, acting [3,7,0] 2023-09-23T20:40:00.895 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:00.972 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:40:00.978 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:00.979 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:40:01.079 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:40:01.123 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:01.177 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:40:01.185 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:01.185 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:40:01.286 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:40:01.302 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:40:01.341 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:01.527 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:40:01.531 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:01.531 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:40:01.628 INFO:teuthology.orchestra.run.smithi018.stderr:noscrub is set 2023-09-23T20:40:01.633 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:40:01.677 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:01.729 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:40:01.736 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:01.736 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:40:01.837 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:40:01.878 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:01 smithi149 ceph-mon[170394]: pgmap v1069: 129 pgs: 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 1.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 695 op/s; 10212/90558 objects degraded (11.277%); 2433/90558 objects misplaced (2.687%); 13 MiB/s, 202 objects/s recovering 2023-09-23T20:40:01.878 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:01 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/413803169' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:40:01.879 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:01 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 10212/90558 objects degraded (11.277%), 11 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T20:40:01.881 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:01.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:01 smithi018 ceph-mon[184144]: pgmap v1069: 129 pgs: 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 1.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 695 op/s; 10212/90558 objects degraded (11.277%); 2433/90558 objects misplaced (2.687%); 13 MiB/s, 202 objects/s recovering 2023-09-23T20:40:01.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:01 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/413803169' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:40:01.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:01 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 10212/90558 objects degraded (11.277%), 11 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T20:40:01.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:01 smithi018 ceph-mon[179647]: pgmap v1069: 129 pgs: 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 1.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 695 op/s; 10212/90558 objects degraded (11.277%); 2433/90558 objects misplaced (2.687%); 13 MiB/s, 202 objects/s recovering 2023-09-23T20:40:01.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:01 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/413803169' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:40:01.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:01 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 10212/90558 objects degraded (11.277%), 11 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T20:40:01.930 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:40:01.937 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:01.937 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:40:02.038 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:40:02.079 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:02.152 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:40:02.161 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:02.161 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:40:02.262 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:40:02.351 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:02.422 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:40:02.429 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:02.430 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:40:02.530 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:40:02.546 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:40:02.553 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:02 smithi018 ceph-mon[179647]: 1.a scrub starts 2023-09-23T20:40:02.554 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:02 smithi018 ceph-mon[179647]: 1.10 deep-scrub starts 2023-09-23T20:40:02.554 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:02 smithi018 ceph-mon[179647]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:40:02.554 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:02 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/413803169' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:40:02.554 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:02 smithi018 ceph-mon[179647]: osdmap e437: 8 total, 8 up, 8 in 2023-09-23T20:40:02.554 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:02 smithi018 ceph-mon[179647]: failed to encode map e437 with expected crc 2023-09-23T20:40:02.554 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:02 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/4144176268' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T20:40:02.568 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:02.589 INFO:teuthology.orchestra.run.smithi018.stderr:reweighted osd.4 to 1 (10000) 2023-09-23T20:40:02.615 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:40:02.621 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:02.621 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:40:02.662 INFO:teuthology.orchestra.run.smithi018.stdout:ERROR: (22) Invalid argument 2023-09-23T20:40:02.662 INFO:teuthology.orchestra.run.smithi018.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:40:02.722 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:40:02.764 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:02.809 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:40:02.812 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:02 smithi149 ceph-mon[170394]: 1.a scrub starts 2023-09-23T20:40:02.812 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:02 smithi149 ceph-mon[170394]: 1.10 deep-scrub starts 2023-09-23T20:40:02.812 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:02 smithi149 ceph-mon[170394]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:40:02.812 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:02 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/413803169' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:40:02.812 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:02 smithi149 ceph-mon[170394]: osdmap e437: 8 total, 8 up, 8 in 2023-09-23T20:40:02.812 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:02 smithi149 ceph-mon[170394]: failed to encode map e437 with expected crc 2023-09-23T20:40:02.813 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:02 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/4144176268' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T20:40:02.813 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:02.813 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:40:02.877 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:02 smithi018 ceph-mon[184144]: 1.a scrub starts 2023-09-23T20:40:02.878 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:02 smithi018 ceph-mon[184144]: 1.10 deep-scrub starts 2023-09-23T20:40:02.878 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:02 smithi018 ceph-mon[184144]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:40:02.878 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:02 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/413803169' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:40:02.878 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:02 smithi018 ceph-mon[184144]: osdmap e437: 8 total, 8 up, 8 in 2023-09-23T20:40:02.878 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:02 smithi018 ceph-mon[184144]: failed to encode map e437 with expected crc 2023-09-23T20:40:02.878 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:02 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/4144176268' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T20:40:02.914 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:40:02.995 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:03.246 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd reweight 6 1 2023-09-23T20:40:03.368 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:40:03.374 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:03.374 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:40:03.475 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:40:03.545 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:03.770 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:40:03.781 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:03.781 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:40:03.856 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T20:40:03.861 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:03 smithi018 ceph-mon[179647]: pgmap v1071: 129 pgs: 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 1.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 708 op/s; 10212/87048 objects degraded (11.731%); 2433/87048 objects misplaced (2.795%); 8.7 MiB/s, 139 objects/s recovering 2023-09-23T20:40:03.862 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:03 smithi018 ceph-mon[179647]: failed to encode map e438 with expected crc 2023-09-23T20:40:03.862 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:03 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/4144176268' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T20:40:03.862 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:03 smithi018 ceph-mon[179647]: osdmap e438: 8 total, 8 up, 8 in 2023-09-23T20:40:03.862 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:03 smithi018 ceph-mon[184144]: pgmap v1071: 129 pgs: 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 1.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 708 op/s; 10212/87048 objects degraded (11.731%); 2433/87048 objects misplaced (2.795%); 8.7 MiB/s, 139 objects/s recovering 2023-09-23T20:40:03.862 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:03 smithi018 ceph-mon[184144]: failed to encode map e438 with expected crc 2023-09-23T20:40:03.862 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:03 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/4144176268' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T20:40:03.862 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:03 smithi018 ceph-mon[184144]: osdmap e438: 8 total, 8 up, 8 in 2023-09-23T20:40:03.882 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:40:03.926 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:03 smithi149 ceph-mon[170394]: pgmap v1071: 129 pgs: 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 1.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 708 op/s; 10212/87048 objects degraded (11.731%); 2433/87048 objects misplaced (2.795%); 8.7 MiB/s, 139 objects/s recovering 2023-09-23T20:40:03.926 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:03 smithi149 ceph-mon[170394]: failed to encode map e438 with expected crc 2023-09-23T20:40:03.926 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:03 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/4144176268' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T20:40:03.926 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:03 smithi149 ceph-mon[170394]: osdmap e438: 8 total, 8 up, 8 in 2023-09-23T20:40:03.928 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:03.981 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:40:03.988 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:03.989 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:40:04.090 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:40:04.204 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:04.275 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T20:40:04.296 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:40:04.305 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:04.305 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:40:04.406 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:40:04.443 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:04.495 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:40:04.501 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:04.502 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:40:04.603 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:40:04.638 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:04.707 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:40:04.714 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:04.715 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:40:04.816 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:40:04.842 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:40:04.843 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:40:04.843 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:40:04.866 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:04.879 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:40:04.918 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:40:04.919 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:40:04.922 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:04.923 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:40:04.942 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:40:04.958 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:40:05.010 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:40:05.024 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:40:05.059 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:40:05.099 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:40:05.100 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:40:05.100 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:40:05.100 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:40:05.100 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:40:05.100 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:40:05.100 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:40:05.100 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:40:05.127 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:05.392 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:40:04 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:40:04] "GET /metrics HTTP/1.1" 200 36558 "" "Prometheus/2.43.0" 2023-09-23T20:40:05.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:05 smithi018 ceph-mon[179647]: pgmap v1073: 129 pgs: 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 1.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 751 op/s; 9994/82092 objects degraded (12.174%); 1996/82092 objects misplaced (2.431%); 12 MiB/s, 192 objects/s recovering 2023-09-23T20:40:05.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:05 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3253132103' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:40:05.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:05 smithi018 ceph-mon[184144]: pgmap v1073: 129 pgs: 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 1.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 751 op/s; 9994/82092 objects degraded (12.174%); 1996/82092 objects misplaced (2.431%); 12 MiB/s, 192 objects/s recovering 2023-09-23T20:40:05.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:05 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3253132103' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:40:05.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:05 smithi149 ceph-mon[170394]: pgmap v1073: 129 pgs: 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 1.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 751 op/s; 9994/82092 objects degraded (12.174%); 1996/82092 objects misplaced (2.431%); 12 MiB/s, 192 objects/s recovering 2023-09-23T20:40:05.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:05 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3253132103' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:40:06.143 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:40:06.189 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:40:06.198 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:06.198 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:40:06.299 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:40:06.345 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:06.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:06 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 9994/82092 objects degraded (12.174%), 11 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T20:40:06.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:06 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 9994/82092 objects degraded (12.174%), 11 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T20:40:06.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:06 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 9994/82092 objects degraded (12.174%), 11 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T20:40:07.830 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:40:07.834 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:07.835 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:40:07.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:07 smithi018 ceph-mon[184144]: pgmap v1074: 129 pgs: 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 1.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 788 op/s; 9778/77790 objects degraded (12.570%); 1996/77790 objects misplaced (2.566%); 12 MiB/s, 186 objects/s recovering 2023-09-23T20:40:07.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:07 smithi018 ceph-mon[179647]: pgmap v1074: 129 pgs: 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 1.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 788 op/s; 9778/77790 objects degraded (12.570%); 1996/77790 objects misplaced (2.566%); 12 MiB/s, 186 objects/s recovering 2023-09-23T20:40:07.936 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:40:07.969 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:07 smithi149 ceph-mon[170394]: pgmap v1074: 129 pgs: 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 1.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 788 op/s; 9778/77790 objects degraded (12.570%); 1996/77790 objects misplaced (2.566%); 12 MiB/s, 186 objects/s recovering 2023-09-23T20:40:07.986 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:08.450 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:40:08.457 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:08.458 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:40:08.559 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:40:08.599 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:08.637 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:40:08.640 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:08.641 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:40:08.742 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:40:08.790 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:08.811 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:40:08.847 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:40:08.851 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:08.851 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:40:08.953 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:40:09.006 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:09.065 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:40:09.075 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:09.075 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:40:09.176 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:40:09.232 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:09.346 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:40:09.355 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:09.355 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:40:09.456 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:40:09.511 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:09.617 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:09 smithi018 ceph-mon[179647]: pgmap v1075: 129 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 4 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 1.5 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 683 op/s; 9778/74148 objects degraded (13.187%); 1718/74148 objects misplaced (2.317%); 9.0 MiB/s, 143 objects/s recovering 2023-09-23T20:40:09.617 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:09 smithi018 ceph-mon[184144]: pgmap v1075: 129 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 4 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 1.5 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 683 op/s; 9778/74148 objects degraded (13.187%); 1718/74148 objects misplaced (2.317%); 9.0 MiB/s, 143 objects/s recovering 2023-09-23T20:40:09.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:09 smithi149 ceph-mon[170394]: pgmap v1075: 129 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 4 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 1.5 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 683 op/s; 9778/74148 objects degraded (13.187%); 1718/74148 objects misplaced (2.317%); 9.0 MiB/s, 143 objects/s recovering 2023-09-23T20:40:10.173 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:40:10.182 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:10.182 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:40:10.283 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:40:10.337 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:10.457 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:40:10.461 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:10.461 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:40:10.562 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:40:10.651 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:10.772 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:40:10.778 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:10.779 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:40:10.853 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:40:10.880 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:40:10.916 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:10.961 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:40:10.969 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:10.969 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:40:11.070 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:40:11.188 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:11.490 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:40:11.519 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:11.520 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:40:11.621 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:40:11.770 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:11 smithi018 ceph-mon[179647]: pgmap v1076: 129 pgs: 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 1.4 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 785 op/s; 9271/67002 objects degraded (13.837%); 1560/67002 objects misplaced (2.328%); 13 MiB/s, 224 objects/s recovering 2023-09-23T20:40:11.770 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:11 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 9271/67002 objects degraded (13.837%), 11 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T20:40:11.771 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:11.771 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:11 smithi018 ceph-mon[184144]: pgmap v1076: 129 pgs: 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 1.4 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 785 op/s; 9271/67002 objects degraded (13.837%); 1560/67002 objects misplaced (2.328%); 13 MiB/s, 224 objects/s recovering 2023-09-23T20:40:11.771 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:11 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 9271/67002 objects degraded (13.837%), 11 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T20:40:11.850 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:40:11.860 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:11.860 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:40:11.908 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2023-09-23T20:40:11.908 INFO:teuthology.orchestra.run.smithi018.stdout: "ops": [ 2023-09-23T20:40:11.908 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:11.908 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_repop(client.35994.0:159394 8.a e438/426 8:57c5a6ef:::benchmark_data_smithi018_341490_object70071:head v 438'10101, mlcod=438'10098)", 2023-09-23T20:40:11.908 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:40:11.881278+0000", 2023-09-23T20:40:11.908 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.018690559999999998, 2023-09-23T20:40:11.908 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.018722592, 2023-09-23T20:40:11.909 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:40:11.909 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "started", 2023-09-23T20:40:11.909 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:40:11.909 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:11.909 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:40:11.909 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:40:11.881278+0000", 2023-09-23T20:40:11.909 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:40:11.909 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:11.909 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:11.910 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:40:11.910 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:40:11.881278+0000", 2023-09-23T20:40:11.910 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:40:11.910 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:11.910 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:11.910 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:40:11.910 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:40:11.881277+0000", 2023-09-23T20:40:11.910 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:40:11.910 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:11.911 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:11.911 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:40:11.911 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:40:11.881282+0000", 2023-09-23T20:40:11.911 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.994e-06 2023-09-23T20:40:11.911 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:11.911 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:11.911 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:40:11.911 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:40:11.881289+0000", 2023-09-23T20:40:11.911 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.9890000000000003e-06 2023-09-23T20:40:11.912 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:11.912 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:11.912 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:40:11.912 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:40:11.881294+0000", 2023-09-23T20:40:11.912 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.9699999999999998e-06 2023-09-23T20:40:11.912 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:11.912 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:11.912 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:40:11.912 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:40:11.884296+0000", 2023-09-23T20:40:11.912 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0030018969999999999 2023-09-23T20:40:11.913 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:11.913 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:11.913 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:40:11.913 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:40:11.884319+0000", 2023-09-23T20:40:11.913 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.2640999999999999e-05 2023-09-23T20:40:11.913 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:40:11.913 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:40:11.913 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:40:11.913 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:11.913 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:11.914 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_repop(client.35994.0:159395 8.a e438/426 8:54cef455:::benchmark_data_smithi018_341490_object70072:head v 438'10102, mlcod=438'10098)", 2023-09-23T20:40:11.914 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:40:11.884161+0000", 2023-09-23T20:40:11.914 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.015807378, 2023-09-23T20:40:11.914 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.015917130000000002, 2023-09-23T20:40:11.914 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:40:11.914 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "started", 2023-09-23T20:40:11.914 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:40:11.914 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:11.914 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:40:11.914 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:40:11.884161+0000", 2023-09-23T20:40:11.915 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:40:11.915 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:11.915 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:11.915 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:40:11.915 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:40:11.884161+0000", 2023-09-23T20:40:11.915 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:40:11.915 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:11.915 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:11.915 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:40:11.916 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:40:11.884160+0000", 2023-09-23T20:40:11.916 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.999999 2023-09-23T20:40:11.916 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:11.916 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:11.916 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:40:11.916 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:40:11.884167+0000", 2023-09-23T20:40:11.916 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.401e-06 2023-09-23T20:40:11.916 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:11.916 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:11.916 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:40:11.917 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:40:11.884179+0000", 2023-09-23T20:40:11.917 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2159e-05 2023-09-23T20:40:11.917 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:11.917 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:11.917 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:40:11.917 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:40:11.884188+0000", 2023-09-23T20:40:11.917 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.1470000000000001e-06 2023-09-23T20:40:11.917 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:11.917 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:11.918 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:40:11.918 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:40:11.887029+0000", 2023-09-23T20:40:11.918 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0028412960000000001 2023-09-23T20:40:11.918 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:11.918 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:11.918 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:40:11.918 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:40:11.887053+0000", 2023-09-23T20:40:11.918 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.4077999999999999e-05 2023-09-23T20:40:11.918 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:40:11.919 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:40:11.919 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:40:11.919 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:11.919 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:11.919 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_repop(client.35994.0:159398 8.a e438/426)", 2023-09-23T20:40:11.919 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:40:11.899176+0000", 2023-09-23T20:40:11.919 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.000792623, 2023-09-23T20:40:11.919 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00096743100000000004, 2023-09-23T20:40:11.919 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:40:11.920 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "queued for pg", 2023-09-23T20:40:11.920 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:40:11.920 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:11.920 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:40:11.920 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:40:11.899176+0000", 2023-09-23T20:40:11.920 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:40:11.920 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:11.920 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:11.920 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:40:11.921 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:40:11.899176+0000", 2023-09-23T20:40:11.921 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:40:11.921 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:11.921 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:11.921 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:40:11.921 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:40:11.899175+0000", 2023-09-23T20:40:11.921 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.999999 2023-09-23T20:40:11.921 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:11.921 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:11.921 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:40:11.921 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:40:11.899184+0000", 2023-09-23T20:40:11.922 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.9260000000000006e-06 2023-09-23T20:40:11.922 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:11.922 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:11.922 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:40:11.922 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:40:11.899203+0000", 2023-09-23T20:40:11.922 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.9108999999999999e-05 2023-09-23T20:40:11.922 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:11.922 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:11.922 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:40:11.923 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:40:11.899212+0000", 2023-09-23T20:40:11.923 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.4010000000000003e-06 2023-09-23T20:40:11.923 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:40:11.923 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:40:11.923 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:40:11.923 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:11.923 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:11.923 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_repop(client.35994.0:159403 8.11 e438/410 8:8d45ce5b:::benchmark_data_smithi018_341490_object70080:head v 438'8315, mlcod=438'8313)", 2023-09-23T20:40:11.923 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:40:11.899228+0000", 2023-09-23T20:40:11.923 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.00073970000000000004, 2023-09-23T20:40:11.924 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00096991999999999998, 2023-09-23T20:40:11.924 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:40:11.924 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "started", 2023-09-23T20:40:11.924 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:40:11.924 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:11.924 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:40:11.924 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:40:11.899228+0000", 2023-09-23T20:40:11.924 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:40:11.924 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:11.924 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:11.925 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:40:11.925 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:40:11.899228+0000", 2023-09-23T20:40:11.925 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:40:11.925 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:11.925 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:11.925 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:40:11.925 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:40:11.899228+0000", 2023-09-23T20:40:11.925 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:40:11.925 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:11.926 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:11.926 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:40:11.926 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:40:11.899233+0000", 2023-09-23T20:40:11.926 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.8049999999999997e-06 2023-09-23T20:40:11.926 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:11.926 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:11.926 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:40:11.926 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:40:11.899239+0000", 2023-09-23T20:40:11.926 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.6980000000000001e-06 2023-09-23T20:40:11.926 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:11.927 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:11.927 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:40:11.927 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:40:11.899246+0000", 2023-09-23T20:40:11.927 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.0499999999999997e-06 2023-09-23T20:40:11.927 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:11.927 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:11.927 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:40:11.927 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:40:11.899340+0000", 2023-09-23T20:40:11.927 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.4921999999999997e-05 2023-09-23T20:40:11.927 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:11.928 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:11.928 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:40:11.928 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:40:11.899364+0000", 2023-09-23T20:40:11.928 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.3187e-05 2023-09-23T20:40:11.928 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:40:11.928 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:40:11.928 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:40:11.928 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:40:11.928 INFO:teuthology.orchestra.run.smithi018.stdout: ], 2023-09-23T20:40:11.928 INFO:teuthology.orchestra.run.smithi018.stdout: "num_ops": 4 2023-09-23T20:40:11.928 INFO:teuthology.orchestra.run.smithi018.stdout:} 2023-09-23T20:40:11.951 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:40:11.961 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:40:11.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:11 smithi149 ceph-mon[170394]: pgmap v1076: 129 pgs: 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 1.4 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 785 op/s; 9271/67002 objects degraded (13.837%); 1560/67002 objects misplaced (2.328%); 13 MiB/s, 224 objects/s recovering 2023-09-23T20:40:11.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:11 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 9271/67002 objects degraded (13.837%), 11 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T20:40:12.068 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:12.262 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:40:12.270 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:12.271 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:40:12.372 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:40:12.402 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:12.437 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T20:40:12.450 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:40:12.454 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:12.455 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:40:12.555 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:40:12.677 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:12.689 INFO:teuthology.orchestra.run.smithi018.stderr:reweighted osd.6 to 1 (10000) 2023-09-23T20:40:12.700 INFO:teuthology.orchestra.run.smithi018.stderr:nodeep-scrub is set 2023-09-23T20:40:12.722 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:12 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2307851128' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:40:12.722 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:12 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:40:12.722 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:12 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1058741772' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T20:40:12.723 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:12 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2307851128' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:40:12.723 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:12 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:40:12.723 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:12 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1058741772' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T20:40:12.794 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:40:12.798 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:12.799 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:40:12.899 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:40:12.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:12 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2307851128' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:40:12.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:12 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:40:12.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:12 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1058741772' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T20:40:13.008 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:13.583 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:40:13.589 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:13.589 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:40:13.622 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd reweight 5 1 2023-09-23T20:40:13.691 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:40:13.736 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:13.788 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:40:13.799 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:13.800 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:40:13.834 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:13 smithi018 ceph-mon[184144]: pgmap v1077: 129 pgs: 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 1.3 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 794 op/s; 9271/63213 objects degraded (14.666%); 1560/63213 objects misplaced (2.468%); 11 MiB/s, 191 objects/s recovering 2023-09-23T20:40:13.834 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:13 smithi018 ceph-mon[184144]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:40:13.835 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:13 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:40:13.835 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:13 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1058741772' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T20:40:13.835 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:13 smithi018 ceph-mon[184144]: osdmap e439: 8 total, 8 up, 8 in 2023-09-23T20:40:13.835 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:13 smithi018 ceph-mon[184144]: failed to encode map e439 with expected crc 2023-09-23T20:40:13.835 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:13 smithi018 ceph-mon[184144]: failed to encode map e439 with expected crc 2023-09-23T20:40:13.835 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:13 smithi018 ceph-mon[179647]: pgmap v1077: 129 pgs: 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 1.3 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 794 op/s; 9271/63213 objects degraded (14.666%); 1560/63213 objects misplaced (2.468%); 11 MiB/s, 191 objects/s recovering 2023-09-23T20:40:13.835 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:13 smithi018 ceph-mon[179647]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:40:13.836 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:13 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:40:13.836 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:13 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1058741772' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T20:40:13.836 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:13 smithi018 ceph-mon[179647]: osdmap e439: 8 total, 8 up, 8 in 2023-09-23T20:40:13.836 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:13 smithi018 ceph-mon[179647]: failed to encode map e439 with expected crc 2023-09-23T20:40:13.836 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:13 smithi018 ceph-mon[179647]: failed to encode map e439 with expected crc 2023-09-23T20:40:13.900 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:40:13.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:13 smithi149 ceph-mon[170394]: pgmap v1077: 129 pgs: 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 1.3 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 794 op/s; 9271/63213 objects degraded (14.666%); 1560/63213 objects misplaced (2.468%); 11 MiB/s, 191 objects/s recovering 2023-09-23T20:40:13.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:13 smithi149 ceph-mon[170394]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:40:13.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:13 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:40:13.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:13 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1058741772' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T20:40:13.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:13 smithi149 ceph-mon[170394]: osdmap e439: 8 total, 8 up, 8 in 2023-09-23T20:40:13.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:13 smithi149 ceph-mon[170394]: failed to encode map e439 with expected crc 2023-09-23T20:40:13.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:13 smithi149 ceph-mon[170394]: failed to encode map e439 with expected crc 2023-09-23T20:40:14.019 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:14.143 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:40:14.151 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:14.151 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:40:14.154 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:40:14.252 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:40:14.296 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:14.345 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:40:14.352 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:14.353 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:40:14.454 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:40:14.523 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:14.615 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:40:14.630 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:14.630 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:40:14.720 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:40:14.731 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:40:14.767 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:40:14.777 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:14.805 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:40:14.850 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:40:14.851 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:40:14.858 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:40:14.862 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:14.863 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:40:14.895 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:40:14.952 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:40:14.963 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:40:14.991 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:40:15.029 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:40:15.037 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:40:15.037 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:40:15.037 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:40:15.037 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:40:15.037 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:40:15.037 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:40:15.037 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:40:15.037 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:40:15.052 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:15.141 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:40:14 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:40:14] "GET /metrics HTTP/1.1" 200 36557 "" "Prometheus/2.43.0" 2023-09-23T20:40:15.318 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:40:15.327 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:15.327 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:40:15.428 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:40:15.466 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:15.474 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:15 smithi018 ceph-mon[179647]: pgmap v1079: 129 pgs: 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 1.2 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 794 op/s; 9035/58257 objects degraded (15.509%); 1019/58257 objects misplaced (1.749%); 12 MiB/s, 210 objects/s recovering 2023-09-23T20:40:15.474 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:15 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:40:15.474 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:15 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:40:15.474 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:15 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2847462029' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:40:15.521 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:40:15.525 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:15.526 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:40:15.566 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T20:40:15.627 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:40:15.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:15 smithi149 ceph-mon[170394]: pgmap v1079: 129 pgs: 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 1.2 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 794 op/s; 9035/58257 objects degraded (15.509%); 1019/58257 objects misplaced (1.749%); 12 MiB/s, 210 objects/s recovering 2023-09-23T20:40:15.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:15 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:40:15.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:15 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:40:15.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:15 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2847462029' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:40:15.719 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:15.818 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:40:15.830 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:15.830 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:15 smithi018 ceph-mon[184144]: pgmap v1079: 129 pgs: 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 1.2 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 794 op/s; 9035/58257 objects degraded (15.509%); 1019/58257 objects misplaced (1.749%); 12 MiB/s, 210 objects/s recovering 2023-09-23T20:40:15.830 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:15 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:40:15.830 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:15 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:40:15.830 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:15 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2847462029' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:40:15.831 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:40:15.932 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:40:15.974 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:16.030 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:40:16.034 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:16.035 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:40:16.136 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:40:16.171 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:16.220 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:40:16.224 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:16.224 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:40:16.236 INFO:teuthology.orchestra.run.smithi018.stdout:ERROR: (22) Invalid argument 2023-09-23T20:40:16.236 INFO:teuthology.orchestra.run.smithi018.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:40:16.325 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:40:16.417 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:16.491 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:16 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 9035/58257 objects degraded (15.509%), 11 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T20:40:16.563 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:40:16.570 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:16.570 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:40:16.671 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:40:16.747 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:16 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 9035/58257 objects degraded (15.509%), 11 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T20:40:16.751 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:16.893 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:40:16.897 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:16.897 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:40:16.937 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T20:40:16.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:16 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 9035/58257 objects degraded (15.509%), 11 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T20:40:16.997 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:40:17.033 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:17.079 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:40:17.085 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:17.085 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:40:17.186 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:40:17.253 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:40:17.265 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:17.462 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:40:17.471 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:17.471 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:40:17.572 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:40:17.647 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:17.709 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:40:17.714 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:17.715 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:40:17.816 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:40:17.849 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:17 smithi018 ceph-mon[179647]: pgmap v1080: 129 pgs: 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 1.1 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 795 op/s; 8791/53913 objects degraded (16.306%); 1019/53913 objects misplaced (1.890%); 12 MiB/s, 213 objects/s recovering 2023-09-23T20:40:17.850 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:17 smithi018 ceph-mon[184144]: pgmap v1080: 129 pgs: 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 1.1 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 795 op/s; 8791/53913 objects degraded (16.306%); 1019/53913 objects misplaced (1.890%); 12 MiB/s, 213 objects/s recovering 2023-09-23T20:40:17.903 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:17.938 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:40:17.968 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:17 smithi149 ceph-mon[170394]: pgmap v1080: 129 pgs: 6 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 1.1 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 795 op/s; 8791/53913 objects degraded (16.306%); 1019/53913 objects misplaced (1.890%); 12 MiB/s, 213 objects/s recovering 2023-09-23T20:40:18.147 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:40:18.152 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:18.152 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:40:18.215 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:40:18.253 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:40:18.288 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:18.334 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:40:18.337 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:18.337 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:40:18.438 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:40:18.480 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:18.522 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:18 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3242340542' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T20:40:18.522 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:18 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T20:40:18.545 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:40:18.551 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:18 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3242340542' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T20:40:18.551 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:18 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T20:40:18.552 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:18.552 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:40:18.556 INFO:teuthology.orchestra.run.smithi018.stderr:reweighted osd.5 to 1 (10000) 2023-09-23T20:40:18.653 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:40:18.686 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:18.781 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:40:18.786 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:18 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3242340542' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T20:40:18.786 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:18 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T20:40:18.787 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:18.787 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:40:18.888 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:40:18.920 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:19.165 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:40:19.168 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:19.168 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:40:19.196 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-09-23T20:40:19.196 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:40:19.269 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:40:19.318 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:19.413 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:40:19.416 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:19.416 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:40:19.450 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:40:19.518 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:40:19.526 INFO:teuthology.orchestra.run.smithi018.stderr:noscrub is unset 2023-09-23T20:40:19.579 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:19.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:19 smithi018 ceph-mon[184144]: pgmap v1081: 129 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 4 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 1.0 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 793 op/s; 8791/50337 objects degraded (17.464%); 646/50337 objects misplaced (1.283%); 13 MiB/s, 222 objects/s recovering 2023-09-23T20:40:19.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:19 smithi018 ceph-mon[184144]: failed to encode map e440 with expected crc 2023-09-23T20:40:19.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:19 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T20:40:19.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:19 smithi018 ceph-mon[184144]: osdmap e440: 8 total, 8 up, 8 in 2023-09-23T20:40:19.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:19 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/87750483' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:40:19.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:19 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:40:19.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:19 smithi018 ceph-mon[184144]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:40:19.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:19 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:40:19.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:19 smithi018 ceph-mon[184144]: osdmap e441: 8 total, 8 up, 8 in 2023-09-23T20:40:19.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:19 smithi018 ceph-mon[179647]: pgmap v1081: 129 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 4 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 1.0 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 793 op/s; 8791/50337 objects degraded (17.464%); 646/50337 objects misplaced (1.283%); 13 MiB/s, 222 objects/s recovering 2023-09-23T20:40:19.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:19 smithi018 ceph-mon[179647]: failed to encode map e440 with expected crc 2023-09-23T20:40:19.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:19 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T20:40:19.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:19 smithi018 ceph-mon[179647]: osdmap e440: 8 total, 8 up, 8 in 2023-09-23T20:40:19.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:19 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/87750483' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:40:19.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:19 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:40:19.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:19 smithi018 ceph-mon[179647]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:40:19.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:19 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:40:19.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:19 smithi018 ceph-mon[179647]: osdmap e441: 8 total, 8 up, 8 in 2023-09-23T20:40:19.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:19 smithi149 ceph-mon[170394]: pgmap v1081: 129 pgs: 6 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 4 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 1.0 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 793 op/s; 8791/50337 objects degraded (17.464%); 646/50337 objects misplaced (1.283%); 13 MiB/s, 222 objects/s recovering 2023-09-23T20:40:19.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:19 smithi149 ceph-mon[170394]: failed to encode map e440 with expected crc 2023-09-23T20:40:19.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:19 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T20:40:19.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:19 smithi149 ceph-mon[170394]: osdmap e440: 8 total, 8 up, 8 in 2023-09-23T20:40:19.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:19 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/87750483' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:40:19.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:19 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:40:19.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:19 smithi149 ceph-mon[170394]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:40:19.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:19 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:40:19.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:19 smithi149 ceph-mon[170394]: osdmap e441: 8 total, 8 up, 8 in 2023-09-23T20:40:20.668 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:40:20.677 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:20.678 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:40:20.717 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:20 smithi018 ceph-mon[184144]: failed to encode map e441 with expected crc 2023-09-23T20:40:20.717 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:20 smithi018 ceph-mon[184144]: failed to encode map e441 with expected crc 2023-09-23T20:40:20.717 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:20 smithi018 ceph-mon[184144]: failed to encode map e441 with expected crc 2023-09-23T20:40:20.717 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:20 smithi018 ceph-mon[184144]: failed to encode map e441 with expected crc 2023-09-23T20:40:20.717 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:20 smithi018 ceph-mon[184144]: failed to encode map e441 with expected crc 2023-09-23T20:40:20.717 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:20 smithi018 ceph-mon[184144]: failed to encode map e441 with expected crc 2023-09-23T20:40:20.717 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:20 smithi018 ceph-mon[184144]: failed to encode map e441 with expected crc 2023-09-23T20:40:20.717 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:20 smithi018 ceph-mon[184144]: failed to encode map e441 with expected crc 2023-09-23T20:40:20.717 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:20 smithi018 ceph-mon[184144]: failed to encode map e441 with expected crc 2023-09-23T20:40:20.718 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:20 smithi018 ceph-mon[184144]: 8.1 scrub starts 2023-09-23T20:40:20.718 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:20 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:40:20.718 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:20 smithi018 ceph-mon[184144]: osdmap e442: 8 total, 8 up, 8 in 2023-09-23T20:40:20.718 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:20 smithi018 ceph-mon[179647]: failed to encode map e441 with expected crc 2023-09-23T20:40:20.718 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:20 smithi018 ceph-mon[179647]: failed to encode map e441 with expected crc 2023-09-23T20:40:20.718 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:20 smithi018 ceph-mon[179647]: failed to encode map e441 with expected crc 2023-09-23T20:40:20.718 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:20 smithi018 ceph-mon[179647]: failed to encode map e441 with expected crc 2023-09-23T20:40:20.718 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:20 smithi018 ceph-mon[179647]: failed to encode map e441 with expected crc 2023-09-23T20:40:20.719 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:20 smithi018 ceph-mon[179647]: failed to encode map e441 with expected crc 2023-09-23T20:40:20.719 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:20 smithi018 ceph-mon[179647]: failed to encode map e441 with expected crc 2023-09-23T20:40:20.719 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:20 smithi018 ceph-mon[179647]: failed to encode map e441 with expected crc 2023-09-23T20:40:20.719 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:20 smithi018 ceph-mon[179647]: failed to encode map e441 with expected crc 2023-09-23T20:40:20.719 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:20 smithi018 ceph-mon[179647]: 8.1 scrub starts 2023-09-23T20:40:20.719 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:20 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:40:20.719 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:20 smithi018 ceph-mon[179647]: osdmap e442: 8 total, 8 up, 8 in 2023-09-23T20:40:20.779 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:40:20.850 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:20 smithi149 ceph-mon[170394]: failed to encode map e441 with expected crc 2023-09-23T20:40:20.850 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:20 smithi149 ceph-mon[170394]: failed to encode map e441 with expected crc 2023-09-23T20:40:20.850 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:20 smithi149 ceph-mon[170394]: failed to encode map e441 with expected crc 2023-09-23T20:40:20.850 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:20 smithi149 ceph-mon[170394]: failed to encode map e441 with expected crc 2023-09-23T20:40:20.850 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:20 smithi149 ceph-mon[170394]: failed to encode map e441 with expected crc 2023-09-23T20:40:20.850 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:20 smithi149 ceph-mon[170394]: failed to encode map e441 with expected crc 2023-09-23T20:40:20.850 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:20 smithi149 ceph-mon[170394]: failed to encode map e441 with expected crc 2023-09-23T20:40:20.851 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:20 smithi149 ceph-mon[170394]: failed to encode map e441 with expected crc 2023-09-23T20:40:20.851 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:20 smithi149 ceph-mon[170394]: failed to encode map e441 with expected crc 2023-09-23T20:40:20.851 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:20 smithi149 ceph-mon[170394]: 8.1 scrub starts 2023-09-23T20:40:20.851 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:20 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:40:20.851 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:20 smithi149 ceph-mon[170394]: osdmap e442: 8 total, 8 up, 8 in 2023-09-23T20:40:20.863 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:20.928 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:40:20.941 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:20.942 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:40:21.043 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:40:21.143 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:21.151 INFO:teuthology.orchestra.run.smithi018.stdout:ERROR: (22) Invalid argument 2023-09-23T20:40:21.151 INFO:teuthology.orchestra.run.smithi018.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:40:21.212 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:40:21.230 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:21.231 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:40:21.332 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:40:21.380 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:21.608 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:40:21.611 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:21.612 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:40:21.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:21 smithi018 ceph-mon[179647]: pgmap v1084: 129 pgs: 2 activating, 6 remapped+peering, 3 peering, 2 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 948 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 768 op/s; 5988/45381 objects degraded (13.195%); 356/45381 objects misplaced (0.784%); 16 MiB/s, 294 objects/s recovering 2023-09-23T20:40:21.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:21 smithi018 ceph-mon[179647]: failed to encode map e442 with expected crc 2023-09-23T20:40:21.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:21 smithi018 ceph-mon[179647]: failed to encode map e442 with expected crc 2023-09-23T20:40:21.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:21 smithi018 ceph-mon[179647]: failed to encode map e442 with expected crc 2023-09-23T20:40:21.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:21 smithi018 ceph-mon[179647]: 8.8 starting backfill to osd.5 from (0'0,378'5250] MIN to 439'8414 2023-09-23T20:40:21.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:21 smithi018 ceph-mon[179647]: failed to encode map e442 with expected crc 2023-09-23T20:40:21.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:21 smithi018 ceph-mon[179647]: failed to encode map e442 with expected crc 2023-09-23T20:40:21.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:21 smithi018 ceph-mon[179647]: 8.19 starting backfill to osd.5 from (0'0,378'5301] MIN to 439'8507 2023-09-23T20:40:21.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:21 smithi018 ceph-mon[179647]: 8.a starting backfill to osd.5 from (0'0,378'5339] MIN to 439'10433 2023-09-23T20:40:21.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:21 smithi018 ceph-mon[179647]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T20:40:21.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:21 smithi018 ceph-mon[179647]: 8.10 starting backfill to osd.5 from (0'0,378'5100] MIN to 439'8255 2023-09-23T20:40:21.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:21 smithi018 ceph-mon[179647]: 8.0 starting backfill to osd.5 from (0'0,0'0] MAX to 439'8277 2023-09-23T20:40:21.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:21 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:40:21.644 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:21 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:40:21.644 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:21 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:40:21.644 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:21 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:40:21.644 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:21 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 5988/45381 objects degraded (13.195%), 7 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T20:40:21.644 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:21 smithi018 ceph-mon[179647]: osdmap e443: 8 total, 8 up, 8 in 2023-09-23T20:40:21.644 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:21 smithi018 ceph-mon[184144]: pgmap v1084: 129 pgs: 2 activating, 6 remapped+peering, 3 peering, 2 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 948 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 768 op/s; 5988/45381 objects degraded (13.195%); 356/45381 objects misplaced (0.784%); 16 MiB/s, 294 objects/s recovering 2023-09-23T20:40:21.644 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:21 smithi018 ceph-mon[184144]: failed to encode map e442 with expected crc 2023-09-23T20:40:21.644 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:21 smithi018 ceph-mon[184144]: failed to encode map e442 with expected crc 2023-09-23T20:40:21.644 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:21 smithi018 ceph-mon[184144]: failed to encode map e442 with expected crc 2023-09-23T20:40:21.644 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:21 smithi018 ceph-mon[184144]: 8.8 starting backfill to osd.5 from (0'0,378'5250] MIN to 439'8414 2023-09-23T20:40:21.645 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:21 smithi018 ceph-mon[184144]: failed to encode map e442 with expected crc 2023-09-23T20:40:21.645 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:21 smithi018 ceph-mon[184144]: failed to encode map e442 with expected crc 2023-09-23T20:40:21.645 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:21 smithi018 ceph-mon[184144]: 8.19 starting backfill to osd.5 from (0'0,378'5301] MIN to 439'8507 2023-09-23T20:40:21.645 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:21 smithi018 ceph-mon[184144]: 8.a starting backfill to osd.5 from (0'0,378'5339] MIN to 439'10433 2023-09-23T20:40:21.645 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:21 smithi018 ceph-mon[184144]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T20:40:21.645 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:21 smithi018 ceph-mon[184144]: 8.10 starting backfill to osd.5 from (0'0,378'5100] MIN to 439'8255 2023-09-23T20:40:21.645 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:21 smithi018 ceph-mon[184144]: 8.0 starting backfill to osd.5 from (0'0,0'0] MAX to 439'8277 2023-09-23T20:40:21.645 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:21 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:40:21.645 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:21 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:40:21.645 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:21 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:40:21.645 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:21 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:40:21.646 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:21 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 5988/45381 objects degraded (13.195%), 7 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T20:40:21.646 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:21 smithi018 ceph-mon[184144]: osdmap e443: 8 total, 8 up, 8 in 2023-09-23T20:40:21.713 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:40:21.723 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:40:21.755 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T20:40:21.760 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:21.808 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:40:21.818 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:21.818 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:40:21.818 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:21 smithi149 ceph-mon[170394]: pgmap v1084: 129 pgs: 2 activating, 6 remapped+peering, 3 peering, 2 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 948 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 768 op/s; 5988/45381 objects degraded (13.195%); 356/45381 objects misplaced (0.784%); 16 MiB/s, 294 objects/s recovering 2023-09-23T20:40:21.818 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:21 smithi149 ceph-mon[170394]: failed to encode map e442 with expected crc 2023-09-23T20:40:21.819 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:21 smithi149 ceph-mon[170394]: failed to encode map e442 with expected crc 2023-09-23T20:40:21.819 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:21 smithi149 ceph-mon[170394]: failed to encode map e442 with expected crc 2023-09-23T20:40:21.819 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:21 smithi149 ceph-mon[170394]: 8.8 starting backfill to osd.5 from (0'0,378'5250] MIN to 439'8414 2023-09-23T20:40:21.819 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:21 smithi149 ceph-mon[170394]: failed to encode map e442 with expected crc 2023-09-23T20:40:21.819 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:21 smithi149 ceph-mon[170394]: failed to encode map e442 with expected crc 2023-09-23T20:40:21.819 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:21 smithi149 ceph-mon[170394]: 8.19 starting backfill to osd.5 from (0'0,378'5301] MIN to 439'8507 2023-09-23T20:40:21.819 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:21 smithi149 ceph-mon[170394]: 8.a starting backfill to osd.5 from (0'0,378'5339] MIN to 439'10433 2023-09-23T20:40:21.819 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:21 smithi149 ceph-mon[170394]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T20:40:21.819 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:21 smithi149 ceph-mon[170394]: 8.10 starting backfill to osd.5 from (0'0,378'5100] MIN to 439'8255 2023-09-23T20:40:21.819 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:21 smithi149 ceph-mon[170394]: 8.0 starting backfill to osd.5 from (0'0,0'0] MAX to 439'8277 2023-09-23T20:40:21.819 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:21 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:40:21.820 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:21 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:40:21.820 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:21 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:40:21.820 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:21 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:40:21.820 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:21 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 5988/45381 objects degraded (13.195%), 7 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T20:40:21.820 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:21 smithi149 ceph-mon[170394]: osdmap e443: 8 total, 8 up, 8 in 2023-09-23T20:40:21.919 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:40:21.956 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:22.008 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:40:22.013 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:22.014 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:40:22.114 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:40:22.243 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:22.279 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:40:22.354 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:40:22.394 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:40:22.449 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:40:22.479 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:40:22.494 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:22.495 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:40:22.499 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:40:22.538 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:40:22.570 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:40:22.596 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:40:22.612 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:40:22.636 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:40:22.636 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:40:22.636 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:40:22.636 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:40:22.636 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:40:22.637 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:40:22.637 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:40:22.637 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:40:22.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:22 smithi018 ceph-mon[179647]: failed to encode map e443 with expected crc 2023-09-23T20:40:22.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:22 smithi018 ceph-mon[179647]: failed to encode map e443 with expected crc 2023-09-23T20:40:22.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:22 smithi018 ceph-mon[179647]: failed to encode map e443 with expected crc 2023-09-23T20:40:22.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:22 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1578309873' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:40:22.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:22 smithi018 ceph-mon[179647]: osdmap e444: 8 total, 8 up, 8 in 2023-09-23T20:40:22.645 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:22 smithi018 ceph-mon[184144]: failed to encode map e443 with expected crc 2023-09-23T20:40:22.645 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:22 smithi018 ceph-mon[184144]: failed to encode map e443 with expected crc 2023-09-23T20:40:22.645 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:22 smithi018 ceph-mon[184144]: failed to encode map e443 with expected crc 2023-09-23T20:40:22.645 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:22 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1578309873' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:40:22.645 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:22 smithi018 ceph-mon[184144]: osdmap e444: 8 total, 8 up, 8 in 2023-09-23T20:40:22.686 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:22.774 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T20:40:22.973 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:22 smithi149 ceph-mon[170394]: failed to encode map e443 with expected crc 2023-09-23T20:40:22.973 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:22 smithi149 ceph-mon[170394]: failed to encode map e443 with expected crc 2023-09-23T20:40:22.973 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:22 smithi149 ceph-mon[170394]: failed to encode map e443 with expected crc 2023-09-23T20:40:22.973 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:22 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1578309873' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:40:22.973 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:22 smithi149 ceph-mon[170394]: osdmap e444: 8 total, 8 up, 8 in 2023-09-23T20:40:23.135 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:40:23.149 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:23.149 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:40:23.250 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:40:23.297 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:23.374 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:40:23.425 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:40:23.435 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:23.436 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:40:23.536 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:40:23.638 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:23.768 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:40:23.777 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:23.778 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:40:23.840 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:23 smithi018 ceph-mon[179647]: pgmap v1087: 129 pgs: 2 active+recovering+undersized+remapped, 2 activating, 6 remapped+peering, 3 peering, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 4 active+undersized+remapped, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped, 104 active+clean; 896 MiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 622 op/s; 4380/42864 objects degraded (10.218%); 1130/42864 objects misplaced (2.636%); 19 MiB/s, 407 objects/s recovering 2023-09-23T20:40:23.840 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:23 smithi018 ceph-mon[179647]: failed to encode map e444 with expected crc 2023-09-23T20:40:23.841 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:23 smithi018 ceph-mon[179647]: failed to encode map e444 with expected crc 2023-09-23T20:40:23.841 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:23 smithi018 ceph-mon[179647]: failed to encode map e444 with expected crc 2023-09-23T20:40:23.841 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:23 smithi018 ceph-mon[179647]: failed to encode map e444 with expected crc 2023-09-23T20:40:23.841 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:23 smithi018 ceph-mon[179647]: failed to encode map e444 with expected crc 2023-09-23T20:40:23.841 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:23 smithi018 ceph-mon[179647]: failed to encode map e444 with expected crc 2023-09-23T20:40:23.841 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:23 smithi018 ceph-mon[184144]: pgmap v1087: 129 pgs: 2 active+recovering+undersized+remapped, 2 activating, 6 remapped+peering, 3 peering, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 4 active+undersized+remapped, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped, 104 active+clean; 896 MiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 622 op/s; 4380/42864 objects degraded (10.218%); 1130/42864 objects misplaced (2.636%); 19 MiB/s, 407 objects/s recovering 2023-09-23T20:40:23.841 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:23 smithi018 ceph-mon[184144]: failed to encode map e444 with expected crc 2023-09-23T20:40:23.842 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:23 smithi018 ceph-mon[184144]: failed to encode map e444 with expected crc 2023-09-23T20:40:23.842 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:23 smithi018 ceph-mon[184144]: failed to encode map e444 with expected crc 2023-09-23T20:40:23.842 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:23 smithi018 ceph-mon[184144]: failed to encode map e444 with expected crc 2023-09-23T20:40:23.842 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:23 smithi018 ceph-mon[184144]: failed to encode map e444 with expected crc 2023-09-23T20:40:23.842 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:23 smithi018 ceph-mon[184144]: failed to encode map e444 with expected crc 2023-09-23T20:40:23.878 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:40:23.966 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:23.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:23 smithi149 ceph-mon[170394]: pgmap v1087: 129 pgs: 2 active+recovering+undersized+remapped, 2 activating, 6 remapped+peering, 3 peering, 1 active+recovery_wait+degraded, 1 active+remapped+backfill_wait, 4 active+undersized+remapped, 1 active+recovering, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped, 104 active+clean; 896 MiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 622 op/s; 4380/42864 objects degraded (10.218%); 1130/42864 objects misplaced (2.636%); 19 MiB/s, 407 objects/s recovering 2023-09-23T20:40:23.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:23 smithi149 ceph-mon[170394]: failed to encode map e444 with expected crc 2023-09-23T20:40:23.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:23 smithi149 ceph-mon[170394]: failed to encode map e444 with expected crc 2023-09-23T20:40:23.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:23 smithi149 ceph-mon[170394]: failed to encode map e444 with expected crc 2023-09-23T20:40:23.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:23 smithi149 ceph-mon[170394]: failed to encode map e444 with expected crc 2023-09-23T20:40:23.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:23 smithi149 ceph-mon[170394]: failed to encode map e444 with expected crc 2023-09-23T20:40:23.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:23 smithi149 ceph-mon[170394]: failed to encode map e444 with expected crc 2023-09-23T20:40:24.042 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:40:24.054 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:24.054 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:40:24.155 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:40:24.172 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:40:24.259 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:24.478 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:40:24.492 INFO:teuthology.orchestra.run.smithi018.stderr:dumped all 2023-09-23T20:40:24.854 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:40:24.863 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:24.864 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:40:24.965 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:40:25.031 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:25.082 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:40:24 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:40:24] "GET /metrics HTTP/1.1" 200 36557 "" "Prometheus/2.43.0" 2023-09-23T20:40:25.108 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:40:25.114 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:25.115 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:40:25.178 INFO:teuthology.orchestra.run.smithi018.stdout:{"pg_ready":true,"pg_map":{"version":1089,"stamp":"2023-09-23T20:40:24.080592+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":880964463,"num_objects":13401,"num_object_clones":0,"num_object_copies":40203,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":9512,"num_objects_misplaced":1976,"num_objects_unfound":0,"num_objects_dirty":13401,"num_whiteouts":0,"num_read":6139,"num_read_kb":13491,"num_write":166251,"num_write_kb":5729974,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33206,"num_bytes_recovered":2056446338,"num_keys_recovered":40,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":173438,"ondisk_log_size":173438,"up":387,"acting":371,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":420,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":8258148,"kb_used_data":4261540,"kb_used_omap":32,"kb_used_meta":3996383,"kb_avail":741703068,"statfs":{"total":767960285184,"available":759503941632,"internally_reserved":0,"allocated":4363816960,"data_stored":4349070851,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32773,"internal_metadata":4092297211},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,5,5,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":9,"apply_latency_ms":9,"commit_latency_ns":9000000,"apply_latency_ns":9000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-161939456,"num_objects":-2471,"num_object_clones":0,"num_object_copies":-7413,"num_objects_missing_on_primary":-334,"num_objects_missing":-334,"num_objects_degraded":832,"num_objects_misplaced":1685,"num_objects_unfound":0,"num_objects_dirty":-2471,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2471,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1398,"num_bytes_recovered":64749648,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"4.574015"},"pg_stats":[{"pgid":"8.11","version":"440'8496","reported_seq":8588,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-09-23T20:40:18.601989+0000","last_change":"2023-09-23T20:38:27.478425+0000","last_active":"2023-09-23T20:40:18.601989+0000","last_peered":"2023-09-23T20:40:18.601989+0000","last_clean":"2023-09-23T20:40:18.601989+0000","last_became_active":"2023-09-23T20:38:27.472168+0000","last_became_peered":"2023-09-23T20:38:27.472168+0000","last_unstale":"2023-09-23T20:40:18.601989+0000","last_undegraded":"2023-09-23T20:40:18.601989+0000","last_fullsized":"2023-09-23T20:40:18.601989+0000","mapping_epoch":410,"log_start":"373'1694","ondisk_log_start":"373'1694","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":6802,"log_dups_size":0,"ondisk_log_size":6802,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":25985024,"num_objects":397,"num_object_clones":0,"num_object_copies":1191,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":397,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5187,"num_write_kb":178656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.18","version":"90'161","reported_seq":423,"reported_epoch":441,"state":"remapped+peering","last_fresh":"2023-09-23T20:40:19.512410+0000","last_change":"2023-09-23T20:37:14.346873+0000","last_active":"2023-09-23T20:27:09.188873+0000","last_peered":"2023-09-23T20:15:59.452160+0000","last_clean":"2023-09-23T20:15:59.452160+0000","last_became_active":"2023-09-23T20:11:27.848678+0000","last_became_peered":"2023-09-23T20:11:27.848678+0000","last_unstale":"2023-09-23T20:40:19.512410+0000","last_undegraded":"2023-09-23T20:40:19.512410+0000","last_fullsized":"2023-09-23T20:40:19.512410+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":405,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:35:03.551143+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:35:03.551143+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1a","version":"69'1","reported_seq":139,"reported_epoch":441,"state":"active+clean","last_fresh":"2023-09-23T20:40:19.518577+0000","last_change":"2023-09-23T20:32:33.233853+0000","last_active":"2023-09-23T20:40:19.518577+0000","last_peered":"2023-09-23T20:40:19.518577+0000","last_clean":"2023-09-23T20:40:19.518577+0000","last_became_active":"2023-09-23T20:32:33.233546+0000","last_became_peered":"2023-09-23T20:32:33.233546+0000","last_unstale":"2023-09-23T20:40:19.518577+0000","last_undegraded":"2023-09-23T20:40:19.518577+0000","last_fullsized":"2023-09-23T20:40:19.518577+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:50:27.130808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":257,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T20:40:21.403559+0000","last_change":"2023-09-23T20:37:43.579611+0000","last_active":"2023-09-23T20:40:21.403559+0000","last_peered":"2023-09-23T20:40:21.403559+0000","last_clean":"2023-09-23T20:40:21.403559+0000","last_became_active":"2023-09-23T20:37:43.579323+0000","last_became_peered":"2023-09-23T20:37:43.579323+0000","last_unstale":"2023-09-23T20:40:21.403559+0000","last_undegraded":"2023-09-23T20:40:21.403559+0000","last_fullsized":"2023-09-23T20:40:21.403559+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:17:32.311427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.12","version":"440'8545","reported_seq":8686,"reported_epoch":441,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:40:19.518233+0000","last_change":"2023-09-23T20:38:39.273131+0000","last_active":"2023-09-23T20:40:19.518233+0000","last_peered":"2023-09-23T20:40:19.518233+0000","last_clean":"2023-09-23T20:37:29.399979+0000","last_became_active":"2023-09-23T20:38:39.252214+0000","last_became_peered":"2023-09-23T20:38:39.252214+0000","last_unstale":"2023-09-23T20:40:19.518233+0000","last_undegraded":"2023-09-23T20:38:38.456024+0000","last_fullsized":"2023-09-23T20:38:38.449466+0000","mapping_epoch":416,"log_start":"373'1996","ondisk_log_start":"373'1996","created":410,"last_epoch_clean":370,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":6549,"log_dups_size":0,"ondisk_log_size":6549,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":27459584,"num_objects":419,"num_object_clones":0,"num_object_copies":1257,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":715,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":419,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5195,"num_write_kb":179648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":753664,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0],"avail_no_missing":["4","0"],"object_location_counts":[{"shards":"0,3,4","objects":141},{"shards":"0,4","objects":278}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1b","version":"90'161","reported_seq":511,"reported_epoch":442,"state":"active+undersized+remapped","last_fresh":"2023-09-23T20:40:20.531192+0000","last_change":"2023-09-23T20:40:20.530948+0000","last_active":"2023-09-23T20:40:20.531192+0000","last_peered":"2023-09-23T20:40:20.531192+0000","last_clean":"2023-09-23T20:27:55.206527+0000","last_became_active":"2023-09-23T20:40:20.530948+0000","last_became_peered":"2023-09-23T20:40:20.530948+0000","last_unstale":"2023-09-23T20:40:20.531192+0000","last_undegraded":"2023-09-23T20:40:20.531192+0000","last_fullsized":"2023-09-23T20:40:20.510572+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:05.442288+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:36:05.442288+0000","last_clean_scrub_stamp":"2023-09-23T20:36:05.442288+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0019500940000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[0,4],"avail_no_missing":["0","2","4","5"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":409,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-09-23T20:40:18.538383+0000","last_change":"2023-09-23T20:34:59.983035+0000","last_active":"2023-09-23T20:40:18.538383+0000","last_peered":"2023-09-23T20:40:18.538383+0000","last_clean":"2023-09-23T20:40:18.538383+0000","last_became_active":"2023-09-23T20:34:59.982459+0000","last_became_peered":"2023-09-23T20:34:59.982459+0000","last_unstale":"2023-09-23T20:40:18.538383+0000","last_undegraded":"2023-09-23T20:40:18.538383+0000","last_fullsized":"2023-09-23T20:40:18.538383+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:16:48.510783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":412,"reported_epoch":441,"state":"active+clean","last_fresh":"2023-09-23T20:40:19.511488+0000","last_change":"2023-09-23T20:25:02.957185+0000","last_active":"2023-09-23T20:40:19.511488+0000","last_peered":"2023-09-23T20:40:19.511488+0000","last_clean":"2023-09-23T20:40:19.511488+0000","last_became_active":"2023-09-23T20:25:02.956689+0000","last_became_peered":"2023-09-23T20:25:02.956689+0000","last_unstale":"2023-09-23T20:40:19.511488+0000","last_undegraded":"2023-09-23T20:40:19.511488+0000","last_fullsized":"2023-09-23T20:40:19.511488+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:04:28.524015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.13","version":"439'8465","reported_seq":13349,"reported_epoch":441,"state":"remapped+peering","last_fresh":"2023-09-23T20:40:19.522377+0000","last_change":"2023-09-23T20:40:19.522377+0000","last_active":"2023-09-23T20:40:18.535408+0000","last_peered":"2023-09-23T20:40:18.508952+0000","last_clean":"2023-09-23T20:37:13.297905+0000","last_became_active":"2023-09-23T20:38:27.564876+0000","last_became_peered":"2023-09-23T20:38:27.564876+0000","last_unstale":"2023-09-23T20:40:19.522377+0000","last_undegraded":"2023-09-23T20:40:19.522377+0000","last_fullsized":"2023-09-23T20:40:19.522377+0000","mapping_epoch":441,"log_start":"373'1680","ondisk_log_start":"373'1680","created":410,"last_epoch_clean":370,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":6785,"log_dups_size":0,"ondisk_log_size":6785,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":28475392,"num_objects":435,"num_object_clones":0,"num_object_copies":1305,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":435,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5159,"num_write_kb":178976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2029,"num_bytes_recovered":132939776,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[5,6],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1a","version":"90'161","reported_seq":521,"reported_epoch":441,"state":"active+clean","last_fresh":"2023-09-23T20:40:19.517686+0000","last_change":"2023-09-23T20:36:01.292103+0000","last_active":"2023-09-23T20:40:19.517686+0000","last_peered":"2023-09-23T20:40:19.517686+0000","last_clean":"2023-09-23T20:40:19.517686+0000","last_became_active":"2023-09-23T20:35:26.639912+0000","last_became_peered":"2023-09-23T20:35:26.639912+0000","last_unstale":"2023-09-23T20:40:19.517686+0000","last_undegraded":"2023-09-23T20:40:19.517686+0000","last_fullsized":"2023-09-23T20:40:19.517686+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:01.292032+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:01.292032+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:04:53.926640+0000","scrub_duration":0.0032983159999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":237,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T20:40:21.405091+0000","last_change":"2023-09-23T20:37:43.584414+0000","last_active":"2023-09-23T20:40:21.405091+0000","last_peered":"2023-09-23T20:40:21.405091+0000","last_clean":"2023-09-23T20:40:21.405091+0000","last_became_active":"2023-09-23T20:37:43.584116+0000","last_became_peered":"2023-09-23T20:37:43.584116+0000","last_unstale":"2023-09-23T20:40:21.405091+0000","last_undegraded":"2023-09-23T20:40:21.405091+0000","last_fullsized":"2023-09-23T20:40:21.405091+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:17:54.258172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":60,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T20:40:21.407590+0000","last_change":"2023-09-23T20:38:37.457427+0000","last_active":"2023-09-23T20:40:21.407590+0000","last_peered":"2023-09-23T20:40:21.407590+0000","last_clean":"2023-09-23T20:40:21.407590+0000","last_became_active":"2023-09-23T20:38:37.457099+0000","last_became_peered":"2023-09-23T20:38:37.457099+0000","last_unstale":"2023-09-23T20:40:21.407590+0000","last_undegraded":"2023-09-23T20:40:21.407590+0000","last_fullsized":"2023-09-23T20:40:21.407590+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:31:27.084994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"90'161","reported_seq":647,"reported_epoch":443,"state":"peering","last_fresh":"2023-09-23T20:40:21.405426+0000","last_change":"2023-09-23T20:40:21.405638+0000","last_active":"2023-09-23T20:40:21.405397+0000","last_peered":"2023-09-23T20:40:20.541444+0000","last_clean":"2023-09-23T20:40:12.677529+0000","last_became_active":"2023-09-23T20:40:20.541126+0000","last_became_peered":"2023-09-23T20:40:20.541126+0000","last_unstale":"2023-09-23T20:40:21.405426+0000","last_undegraded":"2023-09-23T20:40:21.405426+0000","last_fullsized":"2023-09-23T20:40:21.405426+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:03.411025+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:03.411025+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:02:54.842285+0000","scrub_duration":0.001802922,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[1,3,5],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":445,"reported_epoch":441,"state":"active+clean","last_fresh":"2023-09-23T20:40:19.539535+0000","last_change":"2023-09-23T20:40:19.538844+0000","last_active":"2023-09-23T20:40:19.539535+0000","last_peered":"2023-09-23T20:40:19.539535+0000","last_clean":"2023-09-23T20:40:19.539535+0000","last_became_active":"2023-09-23T20:40:19.538434+0000","last_became_peered":"2023-09-23T20:40:19.538434+0000","last_unstale":"2023-09-23T20:40:19.539535+0000","last_undegraded":"2023-09-23T20:40:19.539535+0000","last_fullsized":"2023-09-23T20:40:19.539535+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:53:59.418427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":167,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T20:40:21.390764+0000","last_change":"2023-09-23T20:38:02.579343+0000","last_active":"2023-09-23T20:40:21.390764+0000","last_peered":"2023-09-23T20:40:21.390764+0000","last_clean":"2023-09-23T20:40:21.390764+0000","last_became_active":"2023-09-23T20:38:02.579047+0000","last_became_peered":"2023-09-23T20:38:02.579047+0000","last_unstale":"2023-09-23T20:40:21.390764+0000","last_undegraded":"2023-09-23T20:40:21.390764+0000","last_fullsized":"2023-09-23T20:40:21.390764+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:44:03.166556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.14","version":"90'161","reported_seq":773,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T20:40:21.394614+0000","last_change":"2023-09-23T20:35:02.236398+0000","last_active":"2023-09-23T20:40:21.394614+0000","last_peered":"2023-09-23T20:40:21.394614+0000","last_clean":"2023-09-23T20:40:21.394614+0000","last_became_active":"2023-09-23T20:35:02.235988+0000","last_became_peered":"2023-09-23T20:35:02.235988+0000","last_unstale":"2023-09-23T20:40:21.394614+0000","last_undegraded":"2023-09-23T20:40:21.394614+0000","last_fullsized":"2023-09-23T20:40:21.394614+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:47.479012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:47.479012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:13.519648+0000","scrub_duration":0.0020136590000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":10,"reported_epoch":441,"state":"active+clean","last_fresh":"2023-09-23T20:40:19.545472+0000","last_change":"2023-09-23T20:40:19.545472+0000","last_active":"2023-09-23T20:40:19.545472+0000","last_peered":"2023-09-23T20:40:19.545472+0000","last_clean":"2023-09-23T20:40:19.545472+0000","last_became_active":"2023-09-23T20:40:19.543853+0000","last_became_peered":"2023-09-23T20:40:19.543853+0000","last_unstale":"2023-09-23T20:40:19.545472+0000","last_undegraded":"2023-09-23T20:40:19.545472+0000","last_fullsized":"2023-09-23T20:40:19.545472+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:08.014744+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":268,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T20:40:21.403947+0000","last_change":"2023-09-23T20:38:02.542034+0000","last_active":"2023-09-23T20:40:21.403947+0000","last_peered":"2023-09-23T20:40:21.403947+0000","last_clean":"2023-09-23T20:40:21.403947+0000","last_became_active":"2023-09-23T20:38:02.541575+0000","last_became_peered":"2023-09-23T20:38:02.541575+0000","last_unstale":"2023-09-23T20:40:21.403947+0000","last_undegraded":"2023-09-23T20:40:21.403947+0000","last_fullsized":"2023-09-23T20:40:21.403947+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:50:54.591529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"90'161","reported_seq":590,"reported_epoch":442,"state":"active+undersized+remapped","last_fresh":"2023-09-23T20:40:20.541008+0000","last_change":"2023-09-23T20:40:20.540908+0000","last_active":"2023-09-23T20:40:20.541008+0000","last_peered":"2023-09-23T20:40:20.541008+0000","last_clean":"2023-09-23T20:40:12.675053+0000","last_became_active":"2023-09-23T20:40:20.540908+0000","last_became_peered":"2023-09-23T20:40:20.540908+0000","last_unstale":"2023-09-23T20:40:20.541008+0000","last_undegraded":"2023-09-23T20:40:20.541008+0000","last_fullsized":"2023-09-23T20:40:20.515520+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:03.435480+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:03.435480+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[0,6],"avail_no_missing":["0","5","6","7"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":470,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T20:40:20.518089+0000","last_change":"2023-09-23T20:32:33.270617+0000","last_active":"2023-09-23T20:40:20.518089+0000","last_peered":"2023-09-23T20:40:20.518089+0000","last_clean":"2023-09-23T20:40:20.518089+0000","last_became_active":"2023-09-23T20:32:33.227902+0000","last_became_peered":"2023-09-23T20:32:33.227902+0000","last_unstale":"2023-09-23T20:40:20.518089+0000","last_undegraded":"2023-09-23T20:40:20.518089+0000","last_fullsized":"2023-09-23T20:40:20.518089+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:30:26.847831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":360,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":410,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-09-23T20:40:18.539212+0000","last_change":"2023-09-23T20:38:02.599068+0000","last_active":"2023-09-23T20:40:18.539212+0000","last_peered":"2023-09-23T20:40:18.539212+0000","last_clean":"2023-09-23T20:40:18.539212+0000","last_became_active":"2023-09-23T20:38:02.597602+0000","last_became_peered":"2023-09-23T20:38:02.597602+0000","last_unstale":"2023-09-23T20:40:18.539212+0000","last_undegraded":"2023-09-23T20:40:18.539212+0000","last_fullsized":"2023-09-23T20:40:18.539212+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:34:06.337029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"90'161","reported_seq":565,"reported_epoch":441,"state":"active+clean","last_fresh":"2023-09-23T20:40:19.518371+0000","last_change":"2023-09-23T20:36:02.548387+0000","last_active":"2023-09-23T20:40:19.518371+0000","last_peered":"2023-09-23T20:40:19.518371+0000","last_clean":"2023-09-23T20:40:19.518371+0000","last_became_active":"2023-09-23T20:35:26.644787+0000","last_became_peered":"2023-09-23T20:35:26.644787+0000","last_unstale":"2023-09-23T20:40:19.518371+0000","last_undegraded":"2023-09-23T20:40:19.518371+0000","last_fullsized":"2023-09-23T20:40:19.518371+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:02.548285+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:02.548285+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:37:15.216553+0000","scrub_duration":0.0015514459999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":189,"reported_epoch":441,"state":"active+clean","last_fresh":"2023-09-23T20:40:19.518173+0000","last_change":"2023-09-23T20:38:02.560646+0000","last_active":"2023-09-23T20:40:19.518173+0000","last_peered":"2023-09-23T20:40:19.518173+0000","last_clean":"2023-09-23T20:40:19.518173+0000","last_became_active":"2023-09-23T20:38:02.539814+0000","last_became_peered":"2023-09-23T20:38:02.539814+0000","last_unstale":"2023-09-23T20:40:19.518173+0000","last_undegraded":"2023-09-23T20:40:19.518173+0000","last_fullsized":"2023-09-23T20:40:19.518173+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:00:09.435023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":169,"reported_epoch":441,"state":"active+clean","last_fresh":"2023-09-23T20:40:19.519507+0000","last_change":"2023-09-23T20:38:02.579222+0000","last_active":"2023-09-23T20:40:19.519507+0000","last_peered":"2023-09-23T20:40:19.519507+0000","last_clean":"2023-09-23T20:40:19.519507+0000","last_became_active":"2023-09-23T20:38:02.578813+0000","last_became_peered":"2023-09-23T20:38:02.578813+0000","last_unstale":"2023-09-23T20:40:19.519507+0000","last_undegraded":"2023-09-23T20:40:19.519507+0000","last_fullsized":"2023-09-23T20:40:19.519507+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:57:31.080027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.18","version":"444'8564","reported_seq":14504,"reported_epoch":444,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:40:23.910225+0000","last_change":"2023-09-23T20:40:20.742770+0000","last_active":"2023-09-23T20:40:23.910225+0000","last_peered":"2023-09-23T20:40:23.910225+0000","last_clean":"2023-09-23T20:37:13.320723+0000","last_became_active":"2023-09-23T20:40:20.713497+0000","last_became_peered":"2023-09-23T20:40:20.713497+0000","last_unstale":"2023-09-23T20:40:23.910225+0000","last_undegraded":"2023-09-23T20:40:20.533808+0000","last_fullsized":"2023-09-23T20:40:20.514147+0000","mapping_epoch":441,"log_start":"373'2334","ondisk_log_start":"373'2334","created":410,"last_epoch_clean":370,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":6230,"log_dups_size":0,"ondisk_log_size":6230,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22183936,"num_objects":339,"num_object_clones":0,"num_object_copies":1017,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2531,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":339,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5256,"num_write_kb":179008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2403,"num_bytes_recovered":157417472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"0,3,7","objects":179},{"shards":"3,7","objects":160}],"blocked_by":[],"up_primary":5,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.11","version":"90'161","reported_seq":699,"reported_epoch":442,"state":"active+undersized+remapped","last_fresh":"2023-09-23T20:40:20.597465+0000","last_change":"2023-09-23T20:40:20.597412+0000","last_active":"2023-09-23T20:40:20.597465+0000","last_peered":"2023-09-23T20:40:20.597465+0000","last_clean":"2023-09-23T20:40:12.672702+0000","last_became_active":"2023-09-23T20:40:20.597412+0000","last_became_peered":"2023-09-23T20:40:20.597412+0000","last_unstale":"2023-09-23T20:40:20.597465+0000","last_undegraded":"2023-09-23T20:40:20.597465+0000","last_fullsized":"2023-09-23T20:40:20.512147+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:43.578666+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:43.578666+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:19:15.267147+0000","scrub_duration":0.0021890379999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,3],"avail_no_missing":["0","1","3","5"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":469,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T20:40:20.516131+0000","last_change":"2023-09-23T20:32:33.242943+0000","last_active":"2023-09-23T20:40:20.516131+0000","last_peered":"2023-09-23T20:40:20.516131+0000","last_clean":"2023-09-23T20:40:20.516131+0000","last_became_active":"2023-09-23T20:32:33.225468+0000","last_became_peered":"2023-09-23T20:32:33.225468+0000","last_unstale":"2023-09-23T20:40:20.516131+0000","last_undegraded":"2023-09-23T20:40:20.516131+0000","last_fullsized":"2023-09-23T20:40:20.516131+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:40:47.867573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":179,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T20:40:21.403546+0000","last_change":"2023-09-23T20:37:43.571836+0000","last_active":"2023-09-23T20:40:21.403546+0000","last_peered":"2023-09-23T20:40:21.403546+0000","last_clean":"2023-09-23T20:40:21.403546+0000","last_became_active":"2023-09-23T20:37:43.571511+0000","last_became_peered":"2023-09-23T20:37:43.571511+0000","last_unstale":"2023-09-23T20:40:21.403546+0000","last_undegraded":"2023-09-23T20:40:21.403546+0000","last_fullsized":"2023-09-23T20:40:21.403546+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:59:25.507917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.19","version":"444'8554","reported_seq":16275,"reported_epoch":444,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T20:40:23.054889+0000","last_change":"2023-09-23T20:40:20.669720+0000","last_active":"2023-09-23T20:40:23.054889+0000","last_peered":"2023-09-23T20:40:23.054889+0000","last_clean":"2023-09-23T20:40:18.450551+0000","last_became_active":"2023-09-23T20:40:20.635841+0000","last_became_peered":"2023-09-23T20:40:20.635841+0000","last_unstale":"2023-09-23T20:40:23.054889+0000","last_undegraded":"2023-09-23T20:40:23.054889+0000","last_fullsized":"2023-09-23T20:40:23.054889+0000","mapping_epoch":441,"log_start":"373'1316","ondisk_log_start":"373'1316","created":410,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":7238,"log_dups_size":0,"ondisk_log_size":7238,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":25624576,"num_objects":391,"num_object_clones":0,"num_object_copies":1173,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":391,"num_objects_unfound":0,"num_objects_dirty":391,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5240,"num_write_kb":180192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2990,"num_bytes_recovered":184942592,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,0,3],"avail_no_missing":["6","0","3","4"],"object_location_counts":[{"shards":"0,3,6","objects":391}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.10","version":"90'161","reported_seq":620,"reported_epoch":442,"state":"active+remapped","last_fresh":"2023-09-23T20:40:20.533718+0000","last_change":"2023-09-23T20:40:20.533638+0000","last_active":"2023-09-23T20:40:20.533718+0000","last_peered":"2023-09-23T20:40:20.533718+0000","last_clean":"2023-09-23T20:27:55.215933+0000","last_became_active":"2023-09-23T20:40:20.533638+0000","last_became_peered":"2023-09-23T20:40:20.533638+0000","last_unstale":"2023-09-23T20:40:20.533718+0000","last_undegraded":"2023-09-23T20:40:20.533718+0000","last_fullsized":"2023-09-23T20:40:20.533718+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:49.726139+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:49.726139+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0096333110000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[4,7,6],"avail_no_missing":["4","5","6","7"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":409,"reported_epoch":441,"state":"active+clean","last_fresh":"2023-09-23T20:40:19.509836+0000","last_change":"2023-09-23T20:25:02.953993+0000","last_active":"2023-09-23T20:40:19.509836+0000","last_peered":"2023-09-23T20:40:19.509836+0000","last_clean":"2023-09-23T20:40:19.509836+0000","last_became_active":"2023-09-23T20:25:02.953672+0000","last_became_peered":"2023-09-23T20:25:02.953672+0000","last_unstale":"2023-09-23T20:40:19.509836+0000","last_undegraded":"2023-09-23T20:40:19.509836+0000","last_fullsized":"2023-09-23T20:40:19.509836+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:26:08.603577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":10,"reported_epoch":441,"state":"active+clean","last_fresh":"2023-09-23T20:40:19.561512+0000","last_change":"2023-09-23T20:40:19.561512+0000","last_active":"2023-09-23T20:40:19.561512+0000","last_peered":"2023-09-23T20:40:19.561512+0000","last_clean":"2023-09-23T20:40:19.561512+0000","last_became_active":"2023-09-23T20:40:19.561123+0000","last_became_peered":"2023-09-23T20:40:19.561123+0000","last_unstale":"2023-09-23T20:40:19.561512+0000","last_undegraded":"2023-09-23T20:40:19.561512+0000","last_fullsized":"2023-09-23T20:40:19.561512+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:50:02.093978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"90'161","reported_seq":726,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-09-23T20:40:18.542246+0000","last_change":"2023-09-23T20:35:02.238923+0000","last_active":"2023-09-23T20:40:18.542246+0000","last_peered":"2023-09-23T20:40:18.542246+0000","last_clean":"2023-09-23T20:40:18.542246+0000","last_became_active":"2023-09-23T20:35:02.238293+0000","last_became_peered":"2023-09-23T20:35:02.238293+0000","last_unstale":"2023-09-23T20:40:18.542246+0000","last_undegraded":"2023-09-23T20:40:18.542246+0000","last_fullsized":"2023-09-23T20:40:18.542246+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:42.544308+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:42.544308+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:51:22.281932+0000","scrub_duration":0.0029958179999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":446,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T20:40:20.515300+0000","last_change":"2023-09-23T20:38:02.539774+0000","last_active":"2023-09-23T20:40:20.515300+0000","last_peered":"2023-09-23T20:40:20.515300+0000","last_clean":"2023-09-23T20:40:20.515300+0000","last_became_active":"2023-09-23T20:38:02.539436+0000","last_became_peered":"2023-09-23T20:38:02.539436+0000","last_unstale":"2023-09-23T20:40:20.515300+0000","last_undegraded":"2023-09-23T20:40:20.515300+0000","last_fullsized":"2023-09-23T20:40:20.515300+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:50:57.375803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":419,"reported_epoch":441,"state":"active+clean","last_fresh":"2023-09-23T20:40:19.511567+0000","last_change":"2023-09-23T20:25:02.957010+0000","last_active":"2023-09-23T20:40:19.511567+0000","last_peered":"2023-09-23T20:40:19.511567+0000","last_clean":"2023-09-23T20:40:19.511567+0000","last_became_active":"2023-09-23T20:25:02.956267+0000","last_became_peered":"2023-09-23T20:25:02.956267+0000","last_unstale":"2023-09-23T20:40:19.511567+0000","last_undegraded":"2023-09-23T20:40:19.511567+0000","last_fullsized":"2023-09-23T20:40:19.511567+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:10:55.299422+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"90'161","reported_seq":535,"reported_epoch":439,"state":"remapped+peering","last_fresh":"2023-09-23T20:40:12.692011+0000","last_change":"2023-09-23T20:40:19.521960+0000","last_active":"2023-09-23T20:40:19.521960+0000","last_peered":"2023-09-23T20:40:12.692011+0000","last_clean":"2023-09-23T20:40:12.692011+0000","last_became_active":"2023-09-23T20:38:05.606164+0000","last_became_peered":"2023-09-23T20:38:05.606164+0000","last_unstale":"2023-09-23T20:40:12.692011+0000","last_undegraded":"2023-09-23T20:40:12.692011+0000","last_fullsized":"2023-09-23T20:40:12.692011+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:53.706934+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:53.706934+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[5,6,7],"up_primary":5,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":80,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T20:40:21.405935+0000","last_change":"2023-09-23T20:40:19.564378+0000","last_active":"2023-09-23T20:40:21.405935+0000","last_peered":"2023-09-23T20:40:21.405935+0000","last_clean":"2023-09-23T20:40:21.405935+0000","last_became_active":"2023-09-23T20:40:19.532991+0000","last_became_peered":"2023-09-23T20:40:19.532991+0000","last_unstale":"2023-09-23T20:40:21.405935+0000","last_undegraded":"2023-09-23T20:40:21.405935+0000","last_fullsized":"2023-09-23T20:40:21.405935+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:22:42.832115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":237,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T20:40:21.403510+0000","last_change":"2023-09-23T20:37:43.571083+0000","last_active":"2023-09-23T20:40:21.403510+0000","last_peered":"2023-09-23T20:40:21.403510+0000","last_clean":"2023-09-23T20:40:21.403510+0000","last_became_active":"2023-09-23T20:37:43.570306+0000","last_became_peered":"2023-09-23T20:37:43.570306+0000","last_unstale":"2023-09-23T20:40:21.403510+0000","last_undegraded":"2023-09-23T20:40:21.403510+0000","last_fullsized":"2023-09-23T20:40:21.403510+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:42:02.396718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.4","version":"440'8663","reported_seq":10805,"reported_epoch":441,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:40:19.553186+0000","last_change":"2023-09-23T20:40:19.540505+0000","last_active":"2023-09-23T20:40:19.553186+0000","last_peered":"2023-09-23T20:40:19.553186+0000","last_clean":"2023-09-23T20:37:15.925522+0000","last_became_active":"2023-09-23T20:38:39.251413+0000","last_became_peered":"2023-09-23T20:38:39.251413+0000","last_unstale":"2023-09-23T20:40:19.553186+0000","last_undegraded":"2023-09-23T20:40:19.539602+0000","last_fullsized":"2023-09-23T20:38:38.449461+0000","mapping_epoch":416,"log_start":"373'2080","ondisk_log_start":"373'2080","created":369,"last_epoch_clean":370,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":6583,"log_dups_size":0,"ondisk_log_size":6583,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":28639232,"num_objects":437,"num_object_clones":0,"num_object_copies":1311,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":503,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":437,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5256,"num_write_kb":182176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":645,"num_bytes_recovered":36306944,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[4,2],"avail_no_missing":["4","2","6"],"object_location_counts":[{"shards":"2,4","objects":437}],"blocked_by":[],"up_primary":5,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.d","version":"90'161","reported_seq":735,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T20:40:20.516214+0000","last_change":"2023-09-23T20:34:59.496438+0000","last_active":"2023-09-23T20:40:20.516214+0000","last_peered":"2023-09-23T20:40:20.516214+0000","last_clean":"2023-09-23T20:40:20.516214+0000","last_became_active":"2023-09-23T20:33:38.970527+0000","last_became_peered":"2023-09-23T20:33:38.970527+0000","last_unstale":"2023-09-23T20:40:20.516214+0000","last_undegraded":"2023-09-23T20:40:20.516214+0000","last_fullsized":"2023-09-23T20:40:20.516214+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:34:59.496375+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:34:59.496375+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:46:18.244734+0000","scrub_duration":0.0017843189999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":446,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T20:40:20.515746+0000","last_change":"2023-09-23T20:32:33.283752+0000","last_active":"2023-09-23T20:40:20.515746+0000","last_peered":"2023-09-23T20:40:20.515746+0000","last_clean":"2023-09-23T20:40:20.515746+0000","last_became_active":"2023-09-23T20:32:33.230503+0000","last_became_peered":"2023-09-23T20:32:33.230503+0000","last_unstale":"2023-09-23T20:40:20.515746+0000","last_undegraded":"2023-09-23T20:40:20.515746+0000","last_fullsized":"2023-09-23T20:40:20.515746+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:29.084410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":10,"reported_epoch":441,"state":"active+clean","last_fresh":"2023-09-23T20:40:19.545668+0000","last_change":"2023-09-23T20:40:19.545668+0000","last_active":"2023-09-23T20:40:19.545668+0000","last_peered":"2023-09-23T20:40:19.545668+0000","last_clean":"2023-09-23T20:40:19.545668+0000","last_became_active":"2023-09-23T20:40:19.543857+0000","last_became_peered":"2023-09-23T20:40:19.543857+0000","last_unstale":"2023-09-23T20:40:19.545668+0000","last_undegraded":"2023-09-23T20:40:19.545668+0000","last_fullsized":"2023-09-23T20:40:19.545668+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:27:09.470605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"444'8473","reported_seq":12036,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-09-23T20:40:23.061559+0000","last_change":"2023-09-23T20:38:36.632834+0000","last_active":"2023-09-23T20:40:23.061559+0000","last_peered":"2023-09-23T20:40:23.061559+0000","last_clean":"2023-09-23T20:40:23.061559+0000","last_became_active":"2023-09-23T20:38:28.423549+0000","last_became_peered":"2023-09-23T20:38:28.423549+0000","last_unstale":"2023-09-23T20:40:23.061559+0000","last_undegraded":"2023-09-23T20:40:23.061559+0000","last_fullsized":"2023-09-23T20:40:23.061559+0000","mapping_epoch":411,"log_start":"373'1839","ondisk_log_start":"373'1839","created":369,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":6634,"log_dups_size":0,"ondisk_log_size":6634,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":24608768,"num_objects":375,"num_object_clones":0,"num_object_copies":1125,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":375,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5177,"num_write_kb":177696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1036,"num_bytes_recovered":55345152,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.c","version":"90'161","reported_seq":691,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T20:40:21.403950+0000","last_change":"2023-09-23T20:38:28.442536+0000","last_active":"2023-09-23T20:40:21.403950+0000","last_peered":"2023-09-23T20:40:21.403950+0000","last_clean":"2023-09-23T20:40:21.403950+0000","last_became_active":"2023-09-23T20:38:28.442229+0000","last_became_peered":"2023-09-23T20:38:28.442229+0000","last_unstale":"2023-09-23T20:40:21.403950+0000","last_undegraded":"2023-09-23T20:40:21.403950+0000","last_fullsized":"2023-09-23T20:40:21.403950+0000","mapping_epoch":411,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":412,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:05.485751+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:05.485751+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.002180043,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163840,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":481,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T20:40:20.516200+0000","last_change":"2023-09-23T20:34:59.987981+0000","last_active":"2023-09-23T20:40:20.516200+0000","last_peered":"2023-09-23T20:40:20.516200+0000","last_clean":"2023-09-23T20:40:20.516200+0000","last_became_active":"2023-09-23T20:34:59.985368+0000","last_became_peered":"2023-09-23T20:34:59.985368+0000","last_unstale":"2023-09-23T20:40:20.516200+0000","last_undegraded":"2023-09-23T20:40:20.516200+0000","last_fullsized":"2023-09-23T20:40:20.516200+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:12:43.082493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":139,"reported_epoch":441,"state":"active+clean","last_fresh":"2023-09-23T20:40:19.518222+0000","last_change":"2023-09-23T20:32:33.232142+0000","last_active":"2023-09-23T20:40:19.518222+0000","last_peered":"2023-09-23T20:40:19.518222+0000","last_clean":"2023-09-23T20:40:19.518222+0000","last_became_active":"2023-09-23T20:32:33.231683+0000","last_became_peered":"2023-09-23T20:32:33.231683+0000","last_unstale":"2023-09-23T20:40:19.518222+0000","last_undegraded":"2023-09-23T20:40:19.518222+0000","last_fullsized":"2023-09-23T20:40:19.518222+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:02:58.336348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"442'8458","reported_seq":9180,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T20:40:21.117431+0000","last_change":"2023-09-23T20:38:29.214109+0000","last_active":"2023-09-23T20:40:21.117431+0000","last_peered":"2023-09-23T20:40:21.117431+0000","last_clean":"2023-09-23T20:40:21.117431+0000","last_became_active":"2023-09-23T20:38:28.425504+0000","last_became_peered":"2023-09-23T20:38:28.425504+0000","last_unstale":"2023-09-23T20:40:21.117431+0000","last_undegraded":"2023-09-23T20:40:21.117431+0000","last_fullsized":"2023-09-23T20:40:21.117431+0000","mapping_epoch":411,"log_start":"373'1261","ondisk_log_start":"373'1261","created":369,"last_epoch_clean":412,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":7197,"log_dups_size":0,"ondisk_log_size":7197,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":24641536,"num_objects":376,"num_object_clones":0,"num_object_copies":1128,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":376,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5178,"num_write_kb":177728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":305,"num_bytes_recovered":10420224,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"90'161","reported_seq":600,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T20:40:21.407411+0000","last_change":"2023-09-23T20:38:40.491902+0000","last_active":"2023-09-23T20:40:21.407411+0000","last_peered":"2023-09-23T20:40:21.407411+0000","last_clean":"2023-09-23T20:40:21.407411+0000","last_became_active":"2023-09-23T20:38:40.491563+0000","last_became_peered":"2023-09-23T20:38:40.491563+0000","last_unstale":"2023-09-23T20:40:21.407411+0000","last_undegraded":"2023-09-23T20:40:21.407411+0000","last_fullsized":"2023-09-23T20:40:21.407411+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:46.217489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:46.217489+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:39:25.008796+0000","scrub_duration":0.001456429,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":471,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T20:40:20.516415+0000","last_change":"2023-09-23T20:40:19.677655+0000","last_active":"2023-09-23T20:40:20.516415+0000","last_peered":"2023-09-23T20:40:20.516415+0000","last_clean":"2023-09-23T20:40:20.516415+0000","last_became_active":"2023-09-23T20:40:19.551189+0000","last_became_peered":"2023-09-23T20:40:19.551189+0000","last_unstale":"2023-09-23T20:40:20.516415+0000","last_undegraded":"2023-09-23T20:40:20.516415+0000","last_fullsized":"2023-09-23T20:40:20.516415+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:11:01.559576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":50,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T20:40:21.404701+0000","last_change":"2023-09-23T20:38:02.590613+0000","last_active":"2023-09-23T20:40:21.404701+0000","last_peered":"2023-09-23T20:40:21.404701+0000","last_clean":"2023-09-23T20:40:21.404701+0000","last_became_active":"2023-09-23T20:38:02.590284+0000","last_became_peered":"2023-09-23T20:38:02.590284+0000","last_unstale":"2023-09-23T20:40:21.404701+0000","last_undegraded":"2023-09-23T20:40:21.404701+0000","last_fullsized":"2023-09-23T20:40:21.404701+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:40:07.825054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"439'8398","reported_seq":8673,"reported_epoch":441,"state":"active+recovering","last_fresh":"2023-09-23T20:40:19.536767+0000","last_change":"2023-09-23T20:40:19.285856+0000","last_active":"2023-09-23T20:40:19.536767+0000","last_peered":"2023-09-23T20:40:19.536767+0000","last_clean":"2023-09-23T20:37:19.384160+0000","last_became_active":"2023-09-23T20:38:28.436345+0000","last_became_peered":"2023-09-23T20:38:28.436345+0000","last_unstale":"2023-09-23T20:40:19.536767+0000","last_undegraded":"2023-09-23T20:40:19.536767+0000","last_fullsized":"2023-09-23T20:40:19.536767+0000","mapping_epoch":411,"log_start":"373'1658","ondisk_log_start":"373'1658","created":369,"last_epoch_clean":370,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":6740,"log_dups_size":0,"ondisk_log_size":6740,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":29425664,"num_objects":449,"num_object_clones":0,"num_object_copies":1347,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":65,"num_objects_unfound":0,"num_objects_dirty":449,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5108,"num_write_kb":177824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":83,"num_bytes_recovered":5439488,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":["1","3"],"object_location_counts":[{"shards":"1,2,3","objects":65},{"shards":"1,3,6","objects":384}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.e","version":"90'161","reported_seq":542,"reported_epoch":441,"state":"active+clean","last_fresh":"2023-09-23T20:40:19.516915+0000","last_change":"2023-09-23T20:36:00.322453+0000","last_active":"2023-09-23T20:40:19.516915+0000","last_peered":"2023-09-23T20:40:19.516915+0000","last_clean":"2023-09-23T20:40:19.516915+0000","last_became_active":"2023-09-23T20:35:02.233536+0000","last_became_peered":"2023-09-23T20:35:02.233536+0000","last_unstale":"2023-09-23T20:40:19.516915+0000","last_undegraded":"2023-09-23T20:40:19.516915+0000","last_fullsized":"2023-09-23T20:40:19.516915+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:00.322381+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:00.322381+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:02:15.489069+0000","scrub_duration":0.0017321859999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":35,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-09-23T20:40:18.527951+0000","last_change":"2023-09-23T20:38:37.456683+0000","last_active":"2023-09-23T20:40:18.527951+0000","last_peered":"2023-09-23T20:40:18.527951+0000","last_clean":"2023-09-23T20:40:18.527951+0000","last_became_active":"2023-09-23T20:38:37.456289+0000","last_became_peered":"2023-09-23T20:38:37.456289+0000","last_unstale":"2023-09-23T20:40:18.527951+0000","last_undegraded":"2023-09-23T20:40:18.527951+0000","last_fullsized":"2023-09-23T20:40:18.527951+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:03:10.507333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":427,"reported_epoch":441,"state":"active+clean","last_fresh":"2023-09-23T20:40:19.508704+0000","last_change":"2023-09-23T20:32:33.236122+0000","last_active":"2023-09-23T20:40:19.508704+0000","last_peered":"2023-09-23T20:40:19.508704+0000","last_clean":"2023-09-23T20:40:19.508704+0000","last_became_active":"2023-09-23T20:32:33.235384+0000","last_became_peered":"2023-09-23T20:32:33.235384+0000","last_unstale":"2023-09-23T20:40:19.508704+0000","last_undegraded":"2023-09-23T20:40:19.508704+0000","last_fullsized":"2023-09-23T20:40:19.508704+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:56.843733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"444'8338","reported_seq":14848,"reported_epoch":444,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T20:40:23.882877+0000","last_change":"2023-09-23T20:40:20.768066+0000","last_active":"2023-09-23T20:40:23.882877+0000","last_peered":"2023-09-23T20:40:23.882877+0000","last_clean":"2023-09-23T20:40:18.483118+0000","last_became_active":"2023-09-23T20:40:20.740650+0000","last_became_peered":"2023-09-23T20:40:20.740650+0000","last_unstale":"2023-09-23T20:40:23.882877+0000","last_undegraded":"2023-09-23T20:40:23.882877+0000","last_fullsized":"2023-09-23T20:40:23.882877+0000","mapping_epoch":441,"log_start":"373'2187","ondisk_log_start":"373'2187","created":369,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":6151,"log_dups_size":0,"ondisk_log_size":6151,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":23855104,"num_objects":364,"num_object_clones":0,"num_object_copies":1092,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":364,"num_objects_unfound":0,"num_objects_dirty":364,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5091,"num_write_kb":174560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2565,"num_bytes_recovered":168099840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,1,4],"avail_no_missing":["3","1","4"],"object_location_counts":[{"shards":"1,3,4","objects":364}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":848,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T20:40:20.515214+0000","last_change":"2023-09-23T20:35:02.523308+0000","last_active":"2023-09-23T20:40:20.515214+0000","last_peered":"2023-09-23T20:40:20.515214+0000","last_clean":"2023-09-23T20:40:20.515214+0000","last_became_active":"2023-09-23T20:33:38.979989+0000","last_became_peered":"2023-09-23T20:33:38.979989+0000","last_unstale":"2023-09-23T20:40:20.515214+0000","last_undegraded":"2023-09-23T20:40:20.515214+0000","last_fullsized":"2023-09-23T20:40:20.515214+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:35:02.523203+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:35:02.523203+0000","last_clean_scrub_stamp":"2023-09-23T20:35:02.523203+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:02:41.472958+0000","scrub_duration":0.0085927989999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":263,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T20:40:21.404273+0000","last_change":"2023-09-23T20:37:43.583731+0000","last_active":"2023-09-23T20:40:21.404273+0000","last_peered":"2023-09-23T20:40:21.404273+0000","last_clean":"2023-09-23T20:40:21.404273+0000","last_became_active":"2023-09-23T20:37:43.583066+0000","last_became_peered":"2023-09-23T20:37:43.583066+0000","last_unstale":"2023-09-23T20:40:21.404273+0000","last_undegraded":"2023-09-23T20:40:21.404273+0000","last_fullsized":"2023-09-23T20:40:21.404273+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:03:45.211900+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":159,"reported_epoch":441,"state":"active+clean","last_fresh":"2023-09-23T20:40:19.520117+0000","last_change":"2023-09-23T20:38:02.579814+0000","last_active":"2023-09-23T20:40:19.520117+0000","last_peered":"2023-09-23T20:40:19.520117+0000","last_clean":"2023-09-23T20:40:19.520117+0000","last_became_active":"2023-09-23T20:38:02.579253+0000","last_became_peered":"2023-09-23T20:38:02.579253+0000","last_unstale":"2023-09-23T20:40:19.520117+0000","last_undegraded":"2023-09-23T20:40:19.520117+0000","last_fullsized":"2023-09-23T20:40:19.520117+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:05:07.298198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"440'8447","reported_seq":8539,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-09-23T20:40:18.578991+0000","last_change":"2023-09-23T20:38:27.447178+0000","last_active":"2023-09-23T20:40:18.578991+0000","last_peered":"2023-09-23T20:40:18.578991+0000","last_clean":"2023-09-23T20:40:18.578991+0000","last_became_active":"2023-09-23T20:38:27.437204+0000","last_became_peered":"2023-09-23T20:38:27.437204+0000","last_unstale":"2023-09-23T20:40:18.578991+0000","last_undegraded":"2023-09-23T20:40:18.578991+0000","last_fullsized":"2023-09-23T20:40:18.578991+0000","mapping_epoch":369,"log_start":"373'1694","ondisk_log_start":"373'1694","created":369,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":6753,"log_dups_size":0,"ondisk_log_size":6753,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":29196288,"num_objects":445,"num_object_clones":0,"num_object_copies":1335,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":445,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5137,"num_write_kb":178656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":530,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-09-23T20:40:02.609194+0000","last_change":"2023-09-23T20:38:05.632243+0000","last_active":"2023-09-23T20:40:02.609194+0000","last_peered":"2023-09-23T20:40:02.609194+0000","last_clean":"2023-09-23T20:40:02.609194+0000","last_became_active":"2023-09-23T20:38:05.631907+0000","last_became_peered":"2023-09-23T20:38:05.631907+0000","last_unstale":"2023-09-23T20:40:02.609194+0000","last_undegraded":"2023-09-23T20:40:02.609194+0000","last_fullsized":"2023-09-23T20:40:02.609194+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:35:00.496197+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:35:00.496197+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":50,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T20:40:21.404780+0000","last_change":"2023-09-23T20:38:02.578703+0000","last_active":"2023-09-23T20:40:21.404780+0000","last_peered":"2023-09-23T20:40:21.404780+0000","last_clean":"2023-09-23T20:40:21.404780+0000","last_became_active":"2023-09-23T20:38:02.578414+0000","last_became_peered":"2023-09-23T20:38:02.578414+0000","last_unstale":"2023-09-23T20:40:21.404780+0000","last_undegraded":"2023-09-23T20:40:21.404780+0000","last_fullsized":"2023-09-23T20:40:21.404780+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:43:59.779295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":108,"reported_epoch":441,"state":"active+clean","last_fresh":"2023-09-23T20:40:19.516749+0000","last_change":"2023-09-23T20:37:31.296606+0000","last_active":"2023-09-23T20:40:19.516749+0000","last_peered":"2023-09-23T20:40:19.516749+0000","last_clean":"2023-09-23T20:40:19.516749+0000","last_became_active":"2023-09-23T20:37:31.295924+0000","last_became_peered":"2023-09-23T20:37:31.295924+0000","last_unstale":"2023-09-23T20:40:19.516749+0000","last_undegraded":"2023-09-23T20:40:19.516749+0000","last_fullsized":"2023-09-23T20:40:19.516749+0000","mapping_epoch":391,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":392,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:27:50.948510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"439'8466","reported_seq":8616,"reported_epoch":441,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:40:19.517386+0000","last_change":"2023-09-23T20:38:39.273152+0000","last_active":"2023-09-23T20:40:19.517386+0000","last_peered":"2023-09-23T20:40:19.517386+0000","last_clean":"2023-09-23T20:37:29.399979+0000","last_became_active":"2023-09-23T20:38:39.253967+0000","last_became_peered":"2023-09-23T20:38:39.253967+0000","last_unstale":"2023-09-23T20:40:19.517386+0000","last_undegraded":"2023-09-23T20:38:38.450129+0000","last_fullsized":"2023-09-23T20:38:38.446971+0000","mapping_epoch":416,"log_start":"373'1996","ondisk_log_start":"373'1996","created":369,"last_epoch_clean":370,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":6470,"log_dups_size":0,"ondisk_log_size":6470,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":32636928,"num_objects":498,"num_object_clones":0,"num_object_copies":1494,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":723,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":498,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5116,"num_write_kb":179648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":950272,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0],"avail_no_missing":["4","0"],"object_location_counts":[{"shards":"0,3,4","objects":190},{"shards":"0,4","objects":308}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"90'161","reported_seq":727,"reported_epoch":441,"state":"remapped+peering","last_fresh":"2023-09-23T20:40:19.519297+0000","last_change":"2023-09-23T20:40:19.519432+0000","last_active":"2023-09-23T20:40:18.529291+0000","last_peered":"2023-09-23T20:40:12.689441+0000","last_clean":"2023-09-23T20:40:12.689441+0000","last_became_active":"2023-09-23T20:37:17.282243+0000","last_became_peered":"2023-09-23T20:37:17.282243+0000","last_unstale":"2023-09-23T20:40:19.519297+0000","last_undegraded":"2023-09-23T20:40:19.519297+0000","last_fullsized":"2023-09-23T20:40:19.519297+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:05.595042+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:05.595042+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:18:18.402248+0000","scrub_duration":0.007271675,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[0,2,5],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":98,"reported_epoch":441,"state":"active+clean","last_fresh":"2023-09-23T20:40:19.517546+0000","last_change":"2023-09-23T20:34:59.983122+0000","last_active":"2023-09-23T20:40:19.517546+0000","last_peered":"2023-09-23T20:40:19.517546+0000","last_clean":"2023-09-23T20:40:19.517546+0000","last_became_active":"2023-09-23T20:34:59.982809+0000","last_became_peered":"2023-09-23T20:34:59.982809+0000","last_unstale":"2023-09-23T20:40:19.517546+0000","last_undegraded":"2023-09-23T20:40:19.517546+0000","last_fullsized":"2023-09-23T20:40:19.517546+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:49:40.839723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":159,"reported_epoch":441,"state":"active+clean","last_fresh":"2023-09-23T20:40:19.517113+0000","last_change":"2023-09-23T20:38:02.543866+0000","last_active":"2023-09-23T20:40:19.517113+0000","last_peered":"2023-09-23T20:40:19.517113+0000","last_clean":"2023-09-23T20:40:19.517113+0000","last_became_active":"2023-09-23T20:38:02.543563+0000","last_became_peered":"2023-09-23T20:38:02.543563+0000","last_unstale":"2023-09-23T20:40:19.517113+0000","last_undegraded":"2023-09-23T20:40:19.517113+0000","last_fullsized":"2023-09-23T20:40:19.517113+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:33.050536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"439'8459","reported_seq":13331,"reported_epoch":441,"state":"remapped+peering","last_fresh":"2023-09-23T20:40:19.523015+0000","last_change":"2023-09-23T20:40:19.523015+0000","last_active":"2023-09-23T20:40:18.540631+0000","last_peered":"2023-09-23T20:40:18.507527+0000","last_clean":"2023-09-23T20:37:13.297905+0000","last_became_active":"2023-09-23T20:38:28.499228+0000","last_became_peered":"2023-09-23T20:38:28.499228+0000","last_unstale":"2023-09-23T20:40:19.523015+0000","last_undegraded":"2023-09-23T20:40:19.523015+0000","last_fullsized":"2023-09-23T20:40:19.523015+0000","mapping_epoch":441,"log_start":"373'1680","ondisk_log_start":"373'1680","created":369,"last_epoch_clean":370,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":6779,"log_dups_size":0,"ondisk_log_size":6779,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":28868608,"num_objects":440,"num_object_clones":0,"num_object_copies":1320,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":440,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5152,"num_write_kb":178976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2023,"num_bytes_recovered":132546560,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[5,6],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":545,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-09-23T20:40:18.525410+0000","last_change":"2023-09-23T20:40:01.179672+0000","last_active":"2023-09-23T20:40:18.525410+0000","last_peered":"2023-09-23T20:40:18.525410+0000","last_clean":"2023-09-23T20:40:18.525410+0000","last_became_active":"2023-09-23T20:38:40.494049+0000","last_became_peered":"2023-09-23T20:38:40.494049+0000","last_unstale":"2023-09-23T20:40:18.525410+0000","last_undegraded":"2023-09-23T20:40:18.525410+0000","last_fullsized":"2023-09-23T20:40:18.525410+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:34:59.481446+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:34:59.481446+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26834,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":237,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T20:40:21.404960+0000","last_change":"2023-09-23T20:37:43.570610+0000","last_active":"2023-09-23T20:40:21.404960+0000","last_peered":"2023-09-23T20:40:21.404960+0000","last_clean":"2023-09-23T20:40:21.404960+0000","last_became_active":"2023-09-23T20:37:43.570216+0000","last_became_peered":"2023-09-23T20:37:43.570216+0000","last_unstale":"2023-09-23T20:40:21.404960+0000","last_undegraded":"2023-09-23T20:40:21.404960+0000","last_fullsized":"2023-09-23T20:40:21.404960+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:39:26.555329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":264,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-09-23T20:40:18.545469+0000","last_change":"2023-09-23T20:32:33.230582+0000","last_active":"2023-09-23T20:40:18.545469+0000","last_peered":"2023-09-23T20:40:18.545469+0000","last_clean":"2023-09-23T20:40:18.545469+0000","last_became_active":"2023-09-23T20:32:33.230257+0000","last_became_peered":"2023-09-23T20:32:33.230257+0000","last_unstale":"2023-09-23T20:40:18.545469+0000","last_undegraded":"2023-09-23T20:40:18.545469+0000","last_fullsized":"2023-09-23T20:40:18.545469+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:11:45.333096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"442'10193","reported_seq":10276,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T20:40:21.201097+0000","last_change":"2023-09-23T20:36:11.310149+0000","last_active":"2023-09-23T20:40:21.201097+0000","last_peered":"2023-09-23T20:40:21.201097+0000","last_clean":"2023-09-23T20:40:21.201097+0000","last_became_active":"2023-09-23T20:36:11.309844+0000","last_became_peered":"2023-09-23T20:36:11.309844+0000","last_unstale":"2023-09-23T20:40:21.201097+0000","last_undegraded":"2023-09-23T20:40:21.201097+0000","last_fullsized":"2023-09-23T20:40:21.201097+0000","mapping_epoch":369,"log_start":"376'4219","ondisk_log_start":"376'4219","created":369,"last_epoch_clean":370,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":5974,"log_dups_size":0,"ondisk_log_size":5974,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:14:57.442108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":56557568,"num_objects":863,"num_object_clones":0,"num_object_copies":2589,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":863,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10193,"num_write_kb":353792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":623,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T20:40:21.407260+0000","last_change":"2023-09-23T20:38:05.622597+0000","last_active":"2023-09-23T20:40:21.407260+0000","last_peered":"2023-09-23T20:40:21.407260+0000","last_clean":"2023-09-23T20:40:21.407260+0000","last_became_active":"2023-09-23T20:38:05.622261+0000","last_became_peered":"2023-09-23T20:38:05.622261+0000","last_unstale":"2023-09-23T20:40:21.407260+0000","last_undegraded":"2023-09-23T20:40:21.407260+0000","last_fullsized":"2023-09-23T20:40:21.407260+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:35:04.101972+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:35:04.101972+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"86'119","reported_seq":749,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-09-23T20:40:18.537732+0000","last_change":"2023-09-23T20:29:45.521257+0000","last_active":"2023-09-23T20:40:18.537732+0000","last_peered":"2023-09-23T20:40:18.537732+0000","last_clean":"2023-09-23T20:40:18.537732+0000","last_became_active":"2023-09-23T20:29:45.520546+0000","last_became_peered":"2023-09-23T20:29:45.520546+0000","last_unstale":"2023-09-23T20:40:18.537732+0000","last_undegraded":"2023-09-23T20:40:18.537732+0000","last_fullsized":"2023-09-23T20:40:18.537732+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":119,"log_dups_size":0,"ondisk_log_size":119,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:58:56.085777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032705499999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":523,"num_read_kb":3503,"num_write":136,"num_write_kb":3128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":10,"reported_epoch":441,"state":"active+clean","last_fresh":"2023-09-23T20:40:19.557009+0000","last_change":"2023-09-23T20:40:19.557009+0000","last_active":"2023-09-23T20:40:19.557009+0000","last_peered":"2023-09-23T20:40:19.557009+0000","last_clean":"2023-09-23T20:40:19.557009+0000","last_became_active":"2023-09-23T20:40:19.550473+0000","last_became_peered":"2023-09-23T20:40:19.550473+0000","last_unstale":"2023-09-23T20:40:19.557009+0000","last_undegraded":"2023-09-23T20:40:19.557009+0000","last_fullsized":"2023-09-23T20:40:19.557009+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:35:01.140258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":396,"reported_epoch":441,"state":"active+clean","last_fresh":"2023-09-23T20:40:19.530942+0000","last_change":"2023-09-23T20:40:19.530942+0000","last_active":"2023-09-23T20:40:19.530942+0000","last_peered":"2023-09-23T20:40:19.530942+0000","last_clean":"2023-09-23T20:40:19.530942+0000","last_became_active":"2023-09-23T20:40:19.530617+0000","last_became_peered":"2023-09-23T20:40:19.530617+0000","last_unstale":"2023-09-23T20:40:19.530942+0000","last_undegraded":"2023-09-23T20:40:19.530942+0000","last_fullsized":"2023-09-23T20:40:19.530942+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:24:09.231934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.8","version":"442'8427","reported_seq":14741,"reported_epoch":442,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T20:40:21.163340+0000","last_change":"2023-09-23T20:40:20.738122+0000","last_active":"2023-09-23T20:40:21.163340+0000","last_peered":"2023-09-23T20:40:21.163340+0000","last_clean":"2023-09-23T20:40:18.482315+0000","last_became_active":"2023-09-23T20:40:20.708683+0000","last_became_peered":"2023-09-23T20:40:20.708683+0000","last_unstale":"2023-09-23T20:40:21.163340+0000","last_undegraded":"2023-09-23T20:40:21.163340+0000","last_fullsized":"2023-09-23T20:40:21.163340+0000","mapping_epoch":441,"log_start":"373'2334","ondisk_log_start":"373'2334","created":369,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":6093,"log_dups_size":0,"ondisk_log_size":6093,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":31162368,"num_objects":475,"num_object_clones":0,"num_object_copies":1425,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":475,"num_objects_unfound":0,"num_objects_dirty":475,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5118,"num_write_kb":179008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2588,"num_bytes_recovered":169607168,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[0,3,7],"avail_no_missing":["0","3","7"],"object_location_counts":[{"shards":"0,3,7","objects":475}],"blocked_by":[],"up_primary":5,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":510,"reported_epoch":442,"state":"active+undersized+remapped","last_fresh":"2023-09-23T20:40:20.597162+0000","last_change":"2023-09-23T20:40:20.597089+0000","last_active":"2023-09-23T20:40:20.597162+0000","last_peered":"2023-09-23T20:40:20.597162+0000","last_clean":"2023-09-23T20:40:12.673032+0000","last_became_active":"2023-09-23T20:40:20.597089+0000","last_became_peered":"2023-09-23T20:40:20.597089+0000","last_unstale":"2023-09-23T20:40:20.597162+0000","last_undegraded":"2023-09-23T20:40:20.597162+0000","last_fullsized":"2023-09-23T20:40:20.510572+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:45.785450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:45.785450+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[0,2],"avail_no_missing":["0","2","4","5"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"86'106","reported_seq":1160,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-09-23T20:40:18.525585+0000","last_change":"2023-09-23T20:38:37.476512+0000","last_active":"2023-09-23T20:40:18.525585+0000","last_peered":"2023-09-23T20:40:18.525585+0000","last_clean":"2023-09-23T20:40:18.525585+0000","last_became_active":"2023-09-23T20:38:37.475980+0000","last_became_peered":"2023-09-23T20:38:37.475980+0000","last_unstale":"2023-09-23T20:40:18.525585+0000","last_undegraded":"2023-09-23T20:40:18.525585+0000","last_fullsized":"2023-09-23T20:40:18.525585+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":106,"log_dups_size":0,"ondisk_log_size":106,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:58:45.508831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00099302599999999993,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1094,"num_read_kb":1678,"num_write":100,"num_write_kb":404,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":159,"reported_epoch":441,"state":"active+clean","last_fresh":"2023-09-23T20:40:19.519430+0000","last_change":"2023-09-23T20:38:02.552008+0000","last_active":"2023-09-23T20:40:19.519430+0000","last_peered":"2023-09-23T20:40:19.519430+0000","last_clean":"2023-09-23T20:40:19.519430+0000","last_became_active":"2023-09-23T20:38:02.551684+0000","last_became_peered":"2023-09-23T20:38:02.551684+0000","last_unstale":"2023-09-23T20:40:19.519430+0000","last_undegraded":"2023-09-23T20:40:19.519430+0000","last_fullsized":"2023-09-23T20:40:19.519430+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:40:37.147538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":402,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-09-23T20:40:18.537681+0000","last_change":"2023-09-23T20:29:45.533406+0000","last_active":"2023-09-23T20:40:18.537681+0000","last_peered":"2023-09-23T20:40:18.537681+0000","last_clean":"2023-09-23T20:40:18.537681+0000","last_became_active":"2023-09-23T20:29:45.526001+0000","last_became_peered":"2023-09-23T20:29:45.526001+0000","last_unstale":"2023-09-23T20:40:18.537681+0000","last_undegraded":"2023-09-23T20:40:18.537681+0000","last_fullsized":"2023-09-23T20:40:18.537681+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:43.787251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.b","version":"444'10407","reported_seq":10506,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-09-23T20:40:23.933930+0000","last_change":"2023-09-23T20:38:01.517845+0000","last_active":"2023-09-23T20:40:23.933930+0000","last_peered":"2023-09-23T20:40:23.933930+0000","last_clean":"2023-09-23T20:40:23.933930+0000","last_became_active":"2023-09-23T20:37:43.579403+0000","last_became_peered":"2023-09-23T20:37:43.579403+0000","last_unstale":"2023-09-23T20:40:23.933930+0000","last_undegraded":"2023-09-23T20:40:23.933930+0000","last_fullsized":"2023-09-23T20:40:23.933930+0000","mapping_epoch":399,"log_start":"405'5900","ondisk_log_start":"405'5900","created":369,"last_epoch_clean":403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":4507,"log_dups_size":0,"ondisk_log_size":4507,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:33:13.920936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47775744,"num_objects":729,"num_object_clones":0,"num_object_copies":2187,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":729,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10407,"num_write_kb":356352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":915,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T20:40:20.516296+0000","last_change":"2023-09-23T20:35:02.260708+0000","last_active":"2023-09-23T20:40:20.516296+0000","last_peered":"2023-09-23T20:40:20.516296+0000","last_clean":"2023-09-23T20:40:20.516296+0000","last_became_active":"2023-09-23T20:35:02.260348+0000","last_became_peered":"2023-09-23T20:35:02.260348+0000","last_unstale":"2023-09-23T20:40:20.516296+0000","last_undegraded":"2023-09-23T20:40:20.516296+0000","last_fullsized":"2023-09-23T20:40:20.516296+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:43.390908+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:43.390908+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:09:14.144070+0000","scrub_duration":0.0032721,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"86'130","reported_seq":760,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T20:40:21.393669+0000","last_change":"2023-09-23T20:25:39.637458+0000","last_active":"2023-09-23T20:40:21.393669+0000","last_peered":"2023-09-23T20:40:21.393669+0000","last_clean":"2023-09-23T20:40:21.393669+0000","last_became_active":"2023-09-23T20:25:39.636773+0000","last_became_peered":"2023-09-23T20:25:39.636773+0000","last_unstale":"2023-09-23T20:40:21.393669+0000","last_undegraded":"2023-09-23T20:40:21.393669+0000","last_fullsized":"2023-09-23T20:40:21.393669+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:00:32.810564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011300049999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":512,"num_read_kb":1820,"num_write":133,"num_write_kb":1347,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":410,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-09-23T20:40:18.542657+0000","last_change":"2023-09-23T20:38:02.591335+0000","last_active":"2023-09-23T20:40:18.542657+0000","last_peered":"2023-09-23T20:40:18.542657+0000","last_clean":"2023-09-23T20:40:18.542657+0000","last_became_active":"2023-09-23T20:38:02.591015+0000","last_became_peered":"2023-09-23T20:38:02.591015+0000","last_unstale":"2023-09-23T20:40:18.542657+0000","last_undegraded":"2023-09-23T20:40:18.542657+0000","last_fullsized":"2023-09-23T20:40:18.542657+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:55:32.272195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":395,"reported_epoch":441,"state":"activating","last_fresh":"2023-09-23T20:40:19.511500+0000","last_change":"2023-09-23T20:40:19.511500+0000","last_active":"2023-09-23T20:40:18.529322+0000","last_peered":"2023-09-23T20:40:12.686842+0000","last_clean":"2023-09-23T20:40:12.686842+0000","last_became_active":"2023-09-23T20:37:14.359892+0000","last_became_peered":"2023-09-23T20:37:14.359892+0000","last_unstale":"2023-09-23T20:40:19.511500+0000","last_undegraded":"2023-09-23T20:40:19.511500+0000","last_fullsized":"2023-09-23T20:40:19.511500+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:12:59.203296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":["1","0","5","7"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"442'8472","reported_seq":14911,"reported_epoch":442,"state":"active+recovering+undersized+remapped","last_fresh":"2023-09-23T20:40:21.232506+0000","last_change":"2023-09-23T20:40:20.787351+0000","last_active":"2023-09-23T20:40:21.232506+0000","last_peered":"2023-09-23T20:40:21.232506+0000","last_clean":"2023-09-23T20:37:13.347998+0000","last_became_active":"2023-09-23T20:40:20.702197+0000","last_became_peered":"2023-09-23T20:40:20.702197+0000","last_unstale":"2023-09-23T20:40:21.232506+0000","last_undegraded":"2023-09-23T20:40:21.232506+0000","last_fullsized":"2023-09-23T20:40:20.511184+0000","mapping_epoch":441,"log_start":"373'1316","ondisk_log_start":"373'1316","created":369,"last_epoch_clean":370,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":7156,"log_dups_size":0,"ondisk_log_size":7156,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":30998528,"num_objects":473,"num_object_clones":0,"num_object_copies":1419,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":299,"num_objects_unfound":0,"num_objects_dirty":473,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5158,"num_write_kb":180192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2345,"num_bytes_recovered":153616384,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[0,6],"avail_no_missing":["0"],"object_location_counts":[{"shards":"0,3,4","objects":148},{"shards":"0,3,6","objects":3},{"shards":"0,6","objects":322}],"blocked_by":[],"up_primary":6,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"86'194","reported_seq":798,"reported_epoch":443,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:40:21.393071+0000","last_change":"2023-09-23T20:40:20.532327+0000","last_active":"2023-09-23T20:40:21.393071+0000","last_peered":"2023-09-23T20:40:21.393071+0000","last_clean":"2023-09-23T20:40:12.685357+0000","last_became_active":"2023-09-23T20:40:20.531854+0000","last_became_peered":"2023-09-23T20:40:20.531854+0000","last_unstale":"2023-09-23T20:40:21.393071+0000","last_undegraded":"2023-09-23T20:40:20.510463+0000","last_fullsized":"2023-09-23T20:40:20.509782+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":194,"log_dups_size":0,"ondisk_log_size":194,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:35:05.823248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00037995999999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1024,"num_read_kb":1200,"num_write":173,"num_write_kb":1276,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[2,7],"avail_no_missing":["2","3","7"],"object_location_counts":[{"shards":"2,3,7","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":905,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T20:40:20.515008+0000","last_change":"2023-09-23T20:38:05.602508+0000","last_active":"2023-09-23T20:40:20.515008+0000","last_peered":"2023-09-23T20:40:20.515008+0000","last_clean":"2023-09-23T20:40:20.515008+0000","last_became_active":"2023-09-23T20:38:05.602129+0000","last_became_peered":"2023-09-23T20:38:05.602129+0000","last_unstale":"2023-09-23T20:40:20.515008+0000","last_undegraded":"2023-09-23T20:40:20.515008+0000","last_fullsized":"2023-09-23T20:40:20.515008+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:45.343742+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:45.343742+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:48.910343+0000","scrub_duration":0.001475802,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":10,"reported_epoch":441,"state":"active+clean","last_fresh":"2023-09-23T20:40:19.542142+0000","last_change":"2023-09-23T20:40:19.542142+0000","last_active":"2023-09-23T20:40:19.542142+0000","last_peered":"2023-09-23T20:40:19.542142+0000","last_clean":"2023-09-23T20:40:19.542142+0000","last_became_active":"2023-09-23T20:40:19.541870+0000","last_became_peered":"2023-09-23T20:40:19.541870+0000","last_unstale":"2023-09-23T20:40:19.542142+0000","last_undegraded":"2023-09-23T20:40:19.542142+0000","last_fullsized":"2023-09-23T20:40:19.542142+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:01.518149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":50,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T20:40:21.410991+0000","last_change":"2023-09-23T20:38:02.573200+0000","last_active":"2023-09-23T20:40:21.410991+0000","last_peered":"2023-09-23T20:40:21.410991+0000","last_clean":"2023-09-23T20:40:21.410991+0000","last_became_active":"2023-09-23T20:38:02.572945+0000","last_became_peered":"2023-09-23T20:38:02.572945+0000","last_unstale":"2023-09-23T20:40:21.410991+0000","last_undegraded":"2023-09-23T20:40:21.410991+0000","last_fullsized":"2023-09-23T20:40:21.410991+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:57.246533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.a","version":"439'10433","reported_seq":21220,"reported_epoch":441,"state":"remapped+peering","last_fresh":"2023-09-23T20:40:19.521912+0000","last_change":"2023-09-23T20:40:19.522074+0000","last_active":"2023-09-23T20:40:18.529992+0000","last_peered":"2023-09-23T20:40:18.471731+0000","last_clean":"2023-09-23T20:40:18.471731+0000","last_became_active":"2023-09-23T20:39:26.762740+0000","last_became_peered":"2023-09-23T20:39:26.762740+0000","last_unstale":"2023-09-23T20:40:19.521912+0000","last_undegraded":"2023-09-23T20:40:19.521912+0000","last_fullsized":"2023-09-23T20:40:19.521912+0000","mapping_epoch":441,"log_start":"401'5779","ondisk_log_start":"401'5779","created":369,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":4654,"log_dups_size":0,"ondisk_log_size":4654,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:41:49.972051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":58785792,"num_objects":897,"num_object_clones":0,"num_object_copies":2691,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":897,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10433,"num_write_kb":362560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5339,"num_bytes_recovered":313851904,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[2,3,5],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":771,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-09-23T20:40:18.542193+0000","last_change":"2023-09-23T20:38:05.615711+0000","last_active":"2023-09-23T20:40:18.542193+0000","last_peered":"2023-09-23T20:40:18.542193+0000","last_clean":"2023-09-23T20:40:18.542193+0000","last_became_active":"2023-09-23T20:38:05.615390+0000","last_became_peered":"2023-09-23T20:38:05.615390+0000","last_unstale":"2023-09-23T20:40:18.542193+0000","last_undegraded":"2023-09-23T20:40:18.542193+0000","last_fullsized":"2023-09-23T20:40:18.542193+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:43.582146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:43.582146+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:45:11.917871+0000","scrub_duration":0.0016219730000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"86'80","reported_seq":695,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T20:40:20.514928+0000","last_change":"2023-09-23T20:29:45.536796+0000","last_active":"2023-09-23T20:40:20.514928+0000","last_peered":"2023-09-23T20:40:20.514928+0000","last_clean":"2023-09-23T20:40:20.514928+0000","last_became_active":"2023-09-23T20:29:45.536439+0000","last_became_peered":"2023-09-23T20:29:45.536439+0000","last_unstale":"2023-09-23T20:40:20.514928+0000","last_undegraded":"2023-09-23T20:40:20.514928+0000","last_fullsized":"2023-09-23T20:40:20.514928+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":80,"log_dups_size":0,"ondisk_log_size":80,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:30:57.793132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035115,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":395,"num_read_kb":1370,"num_write":66,"num_write_kb":1061,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":446,"reported_epoch":441,"state":"active+clean","last_fresh":"2023-09-23T20:40:19.513755+0000","last_change":"2023-09-23T20:34:59.984286+0000","last_active":"2023-09-23T20:40:19.513755+0000","last_peered":"2023-09-23T20:40:19.513755+0000","last_clean":"2023-09-23T20:40:19.513755+0000","last_became_active":"2023-09-23T20:34:59.983950+0000","last_became_peered":"2023-09-23T20:34:59.983950+0000","last_unstale":"2023-09-23T20:40:19.513755+0000","last_undegraded":"2023-09-23T20:40:19.513755+0000","last_fullsized":"2023-09-23T20:40:19.513755+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:31:53.557035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":441,"reported_epoch":441,"state":"active+clean","last_fresh":"2023-09-23T20:40:19.511312+0000","last_change":"2023-09-23T20:29:45.690815+0000","last_active":"2023-09-23T20:40:19.511312+0000","last_peered":"2023-09-23T20:40:19.511312+0000","last_clean":"2023-09-23T20:40:19.511312+0000","last_became_active":"2023-09-23T20:29:45.690478+0000","last_became_peered":"2023-09-23T20:29:45.690478+0000","last_unstale":"2023-09-23T20:40:19.511312+0000","last_undegraded":"2023-09-23T20:40:19.511312+0000","last_fullsized":"2023-09-23T20:40:19.511312+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:38.288193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.f","version":"444'10274","reported_seq":10358,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-09-23T20:40:22.916574+0000","last_change":"2023-09-23T20:36:11.309536+0000","last_active":"2023-09-23T20:40:22.916574+0000","last_peered":"2023-09-23T20:40:22.916574+0000","last_clean":"2023-09-23T20:40:22.916574+0000","last_became_active":"2023-09-23T20:36:11.309208+0000","last_became_peered":"2023-09-23T20:36:11.309208+0000","last_unstale":"2023-09-23T20:40:22.916574+0000","last_undegraded":"2023-09-23T20:40:22.916574+0000","last_fullsized":"2023-09-23T20:40:22.916574+0000","mapping_epoch":369,"log_start":"374'2900","ondisk_log_start":"374'2900","created":369,"last_epoch_clean":370,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":7374,"log_dups_size":0,"ondisk_log_size":7374,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:47:51.785158+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":49807360,"num_objects":760,"num_object_clones":0,"num_object_copies":2280,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":760,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10274,"num_write_kb":353088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":546,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T20:40:21.405933+0000","last_change":"2023-09-23T20:38:05.611181+0000","last_active":"2023-09-23T20:40:21.405933+0000","last_peered":"2023-09-23T20:40:21.405933+0000","last_clean":"2023-09-23T20:40:21.405933+0000","last_became_active":"2023-09-23T20:38:05.610881+0000","last_became_peered":"2023-09-23T20:38:05.610881+0000","last_unstale":"2023-09-23T20:40:21.405933+0000","last_undegraded":"2023-09-23T20:40:21.405933+0000","last_fullsized":"2023-09-23T20:40:21.405933+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:35:00.151695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:35:00.151695+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":20,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"86'102","reported_seq":1037,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T20:40:20.516491+0000","last_change":"2023-09-23T20:32:36.314672+0000","last_active":"2023-09-23T20:40:20.516491+0000","last_peered":"2023-09-23T20:40:20.516491+0000","last_clean":"2023-09-23T20:40:20.516491+0000","last_became_active":"2023-09-23T20:32:36.314231+0000","last_became_peered":"2023-09-23T20:32:36.314231+0000","last_unstale":"2023-09-23T20:40:20.516491+0000","last_undegraded":"2023-09-23T20:40:20.516491+0000","last_fullsized":"2023-09-23T20:40:20.516491+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":102,"log_dups_size":0,"ondisk_log_size":102,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:11:40.514709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030656500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":963,"num_read_kb":1234,"num_write":60,"num_write_kb":225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":380,"reported_epoch":441,"state":"activating","last_fresh":"2023-09-23T20:40:19.523327+0000","last_change":"2023-09-23T20:40:19.523327+0000","last_active":"2023-09-23T20:40:18.528160+0000","last_peered":"2023-09-23T20:40:14.542472+0000","last_clean":"2023-09-23T20:40:14.542472+0000","last_became_active":"2023-09-23T20:37:14.358982+0000","last_became_peered":"2023-09-23T20:37:14.358982+0000","last_unstale":"2023-09-23T20:40:19.523327+0000","last_undegraded":"2023-09-23T20:40:19.523327+0000","last_fullsized":"2023-09-23T20:40:19.523327+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":380,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:00:19.498631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":["1","0","2","5"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":396,"reported_epoch":441,"state":"active+clean","last_fresh":"2023-09-23T20:40:19.533949+0000","last_change":"2023-09-23T20:40:19.532976+0000","last_active":"2023-09-23T20:40:19.533949+0000","last_peered":"2023-09-23T20:40:19.533949+0000","last_clean":"2023-09-23T20:40:19.533949+0000","last_became_active":"2023-09-23T20:40:19.532639+0000","last_became_peered":"2023-09-23T20:40:19.532639+0000","last_unstale":"2023-09-23T20:40:19.533949+0000","last_undegraded":"2023-09-23T20:40:19.533949+0000","last_fullsized":"2023-09-23T20:40:19.533949+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:02:35.301884+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"442'10453","reported_seq":10536,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T20:40:21.198902+0000","last_change":"2023-09-23T20:36:11.482126+0000","last_active":"2023-09-23T20:40:21.198902+0000","last_peered":"2023-09-23T20:40:21.198902+0000","last_clean":"2023-09-23T20:40:21.198902+0000","last_became_active":"2023-09-23T20:36:11.481802+0000","last_became_peered":"2023-09-23T20:36:11.481802+0000","last_unstale":"2023-09-23T20:40:21.198902+0000","last_undegraded":"2023-09-23T20:40:21.198902+0000","last_fullsized":"2023-09-23T20:40:21.198902+0000","mapping_epoch":369,"log_start":"377'4459","ondisk_log_start":"377'4459","created":369,"last_epoch_clean":370,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":5994,"log_dups_size":0,"ondisk_log_size":5994,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:13:12.263455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":55771136,"num_objects":851,"num_object_clones":0,"num_object_copies":2553,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":851,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10453,"num_write_kb":361728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":720,"reported_epoch":441,"state":"active+clean","last_fresh":"2023-09-23T20:40:19.517308+0000","last_change":"2023-09-23T20:38:05.590361+0000","last_active":"2023-09-23T20:40:19.517308+0000","last_peered":"2023-09-23T20:40:19.517308+0000","last_clean":"2023-09-23T20:40:19.517308+0000","last_became_active":"2023-09-23T20:38:05.589985+0000","last_became_peered":"2023-09-23T20:38:05.589985+0000","last_unstale":"2023-09-23T20:40:19.517308+0000","last_undegraded":"2023-09-23T20:40:19.517308+0000","last_fullsized":"2023-09-23T20:40:19.517308+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:03.246271+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:03.246271+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:38:54.351576+0000","scrub_duration":0.0019016700000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":180,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T20:40:20.516036+0000","last_change":"2023-09-23T20:32:50.549695+0000","last_active":"2023-09-23T20:40:20.516036+0000","last_peered":"2023-09-23T20:40:20.516036+0000","last_clean":"2023-09-23T20:40:20.516036+0000","last_became_active":"2023-09-23T20:32:50.549431+0000","last_became_peered":"2023-09-23T20:32:50.549431+0000","last_unstale":"2023-09-23T20:40:20.516036+0000","last_undegraded":"2023-09-23T20:40:20.516036+0000","last_fullsized":"2023-09-23T20:40:20.516036+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:45:53.666569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"86'128","reported_seq":851,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-09-23T20:40:18.541897+0000","last_change":"2023-09-23T20:38:03.541765+0000","last_active":"2023-09-23T20:40:18.541897+0000","last_peered":"2023-09-23T20:40:18.541897+0000","last_clean":"2023-09-23T20:40:18.541897+0000","last_became_active":"2023-09-23T20:38:03.541277+0000","last_became_peered":"2023-09-23T20:38:03.541277+0000","last_unstale":"2023-09-23T20:40:18.541897+0000","last_undegraded":"2023-09-23T20:40:18.541897+0000","last_fullsized":"2023-09-23T20:40:18.541897+0000","mapping_epoch":404,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":405,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:20:53.834554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000293202,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":579,"num_read_kb":1473,"num_write":96,"num_write_kb":1062,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":222,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T20:40:21.404563+0000","last_change":"2023-09-23T20:37:43.582105+0000","last_active":"2023-09-23T20:40:21.404563+0000","last_peered":"2023-09-23T20:40:21.404563+0000","last_clean":"2023-09-23T20:40:21.404563+0000","last_became_active":"2023-09-23T20:37:43.581860+0000","last_became_peered":"2023-09-23T20:37:43.581860+0000","last_unstale":"2023-09-23T20:40:21.404563+0000","last_undegraded":"2023-09-23T20:40:21.404563+0000","last_fullsized":"2023-09-23T20:40:21.404563+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:18:38.765486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"443'10355","reported_seq":18036,"reported_epoch":443,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:40:22.268498+0000","last_change":"2023-09-23T20:40:19.729086+0000","last_active":"2023-09-23T20:40:22.268498+0000","last_peered":"2023-09-23T20:40:22.268498+0000","last_clean":"2023-09-23T20:37:13.317209+0000","last_became_active":"2023-09-23T20:40:19.694465+0000","last_became_peered":"2023-09-23T20:40:19.694465+0000","last_unstale":"2023-09-23T20:40:22.268498+0000","last_undegraded":"2023-09-23T20:40:19.549561+0000","last_fullsized":"2023-09-23T20:40:19.517168+0000","mapping_epoch":440,"log_start":"377'4721","ondisk_log_start":"377'4721","created":369,"last_epoch_clean":370,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":5634,"log_dups_size":0,"ondisk_log_size":5634,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:13:44.257914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":50593816,"num_objects":773,"num_object_clones":0,"num_object_copies":2319,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5029,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":773,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10355,"num_write_kb":356033,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3780,"num_bytes_recovered":247726080,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[7,2],"avail_no_missing":["7","2"],"object_location_counts":[{"shards":"1,2,6,7","objects":87},{"shards":"1,2,7","objects":193},{"shards":"2,6,7","objects":15},{"shards":"2,7","objects":478}],"blocked_by":[],"up_primary":5,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.4","version":"90'161","reported_seq":656,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-09-23T20:40:22.427435+0000","last_change":"2023-09-23T20:40:22.427435+0000","last_active":"2023-09-23T20:40:22.427435+0000","last_peered":"2023-09-23T20:40:22.427435+0000","last_clean":"2023-09-23T20:40:22.427435+0000","last_became_active":"2023-09-23T20:40:22.427051+0000","last_became_peered":"2023-09-23T20:40:22.427051+0000","last_unstale":"2023-09-23T20:40:22.427435+0000","last_undegraded":"2023-09-23T20:40:22.427435+0000","last_fullsized":"2023-09-23T20:40:22.427435+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:34:54.504554+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:34:54.504554+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:42.001652+0000","scrub_duration":0.011176864,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"86'117","reported_seq":677,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T20:40:21.407176+0000","last_change":"2023-09-23T20:38:28.446107+0000","last_active":"2023-09-23T20:40:21.407176+0000","last_peered":"2023-09-23T20:40:21.407176+0000","last_clean":"2023-09-23T20:40:21.407176+0000","last_became_active":"2023-09-23T20:38:28.445667+0000","last_became_peered":"2023-09-23T20:38:28.445667+0000","last_unstale":"2023-09-23T20:40:21.407176+0000","last_undegraded":"2023-09-23T20:40:21.407176+0000","last_fullsized":"2023-09-23T20:40:21.407176+0000","mapping_epoch":411,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":412,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:58:47.430360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046891400000000002,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":893,"num_read_kb":922,"num_write":82,"num_write_kb":4193,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":441,"reported_epoch":441,"state":"active+clean","last_fresh":"2023-09-23T20:40:19.511402+0000","last_change":"2023-09-23T20:32:33.277827+0000","last_active":"2023-09-23T20:40:19.511402+0000","last_peered":"2023-09-23T20:40:19.511402+0000","last_clean":"2023-09-23T20:40:19.511402+0000","last_became_active":"2023-09-23T20:32:33.236087+0000","last_became_peered":"2023-09-23T20:32:33.236087+0000","last_unstale":"2023-09-23T20:40:19.511402+0000","last_undegraded":"2023-09-23T20:40:19.511402+0000","last_fullsized":"2023-09-23T20:40:19.511402+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:20:58.176854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":179,"reported_epoch":441,"state":"active+clean","last_fresh":"2023-09-23T20:40:19.531488+0000","last_change":"2023-09-23T20:40:19.530602+0000","last_active":"2023-09-23T20:40:19.531488+0000","last_peered":"2023-09-23T20:40:19.531488+0000","last_clean":"2023-09-23T20:40:19.531488+0000","last_became_active":"2023-09-23T20:40:19.530256+0000","last_became_peered":"2023-09-23T20:40:19.530256+0000","last_unstale":"2023-09-23T20:40:19.531488+0000","last_undegraded":"2023-09-23T20:40:19.531488+0000","last_fullsized":"2023-09-23T20:40:19.531488+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:18:45.658867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.10","version":"444'8320","reported_seq":14773,"reported_epoch":444,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T20:40:23.751625+0000","last_change":"2023-09-23T20:40:20.743359+0000","last_active":"2023-09-23T20:40:23.751625+0000","last_peered":"2023-09-23T20:40:23.751625+0000","last_clean":"2023-09-23T20:40:18.470747+0000","last_became_active":"2023-09-23T20:40:20.710426+0000","last_became_peered":"2023-09-23T20:40:20.710426+0000","last_unstale":"2023-09-23T20:40:23.751625+0000","last_undegraded":"2023-09-23T20:40:23.751625+0000","last_fullsized":"2023-09-23T20:40:23.751625+0000","mapping_epoch":441,"log_start":"373'2187","ondisk_log_start":"373'2187","created":410,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":6133,"log_dups_size":0,"ondisk_log_size":6133,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":25034752,"num_objects":382,"num_object_clones":0,"num_object_copies":1146,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":382,"num_objects_unfound":0,"num_objects_dirty":382,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5073,"num_write_kb":174560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2537,"num_bytes_recovered":166264832,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,1,4],"avail_no_missing":["3","1","4"],"object_location_counts":[{"shards":"1,3,4","objects":382}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.19","version":"90'161","reported_seq":711,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T20:40:20.512032+0000","last_change":"2023-09-23T20:38:05.630221+0000","last_active":"2023-09-23T20:40:20.512032+0000","last_peered":"2023-09-23T20:40:20.512032+0000","last_clean":"2023-09-23T20:40:20.512032+0000","last_became_active":"2023-09-23T20:38:05.629839+0000","last_became_peered":"2023-09-23T20:38:05.629839+0000","last_unstale":"2023-09-23T20:40:20.512032+0000","last_undegraded":"2023-09-23T20:40:20.512032+0000","last_fullsized":"2023-09-23T20:40:20.512032+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:06.487351+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:06.487351+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:25:51.325529+0000","scrub_duration":0.001964379,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":409,"reported_epoch":441,"state":"active+clean","last_fresh":"2023-09-23T20:40:19.508817+0000","last_change":"2023-09-23T20:32:33.237234+0000","last_active":"2023-09-23T20:40:19.508817+0000","last_peered":"2023-09-23T20:40:19.508817+0000","last_clean":"2023-09-23T20:40:19.508817+0000","last_became_active":"2023-09-23T20:32:33.236881+0000","last_became_peered":"2023-09-23T20:32:33.236881+0000","last_unstale":"2023-09-23T20:40:19.508817+0000","last_undegraded":"2023-09-23T20:40:19.508817+0000","last_fullsized":"2023-09-23T20:40:19.508817+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:56:17.384831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":298,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T20:40:21.394803+0000","last_change":"2023-09-23T20:29:45.538314+0000","last_active":"2023-09-23T20:40:21.394803+0000","last_peered":"2023-09-23T20:40:21.394803+0000","last_clean":"2023-09-23T20:40:21.394803+0000","last_became_active":"2023-09-23T20:29:45.536847+0000","last_became_peered":"2023-09-23T20:29:45.536847+0000","last_unstale":"2023-09-23T20:40:21.394803+0000","last_undegraded":"2023-09-23T20:40:21.394803+0000","last_fullsized":"2023-09-23T20:40:21.394803+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:51.948721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"8.17","version":"440'8417","reported_seq":9180,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-09-23T20:40:19.283970+0000","last_change":"2023-09-23T20:40:19.281439+0000","last_active":"2023-09-23T20:40:19.283970+0000","last_peered":"2023-09-23T20:40:19.283970+0000","last_clean":"2023-09-23T20:40:19.283970+0000","last_became_active":"2023-09-23T20:38:27.474930+0000","last_became_peered":"2023-09-23T20:38:27.474930+0000","last_unstale":"2023-09-23T20:40:19.283970+0000","last_undegraded":"2023-09-23T20:40:19.283970+0000","last_fullsized":"2023-09-23T20:40:19.283970+0000","mapping_epoch":410,"log_start":"373'1658","ondisk_log_start":"373'1658","created":410,"last_epoch_clean":440,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":6759,"log_dups_size":0,"ondisk_log_size":6759,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":28180480,"num_objects":430,"num_object_clones":0,"num_object_copies":1290,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":430,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5127,"num_write_kb":177824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":327,"num_bytes_recovered":10354688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1e","version":"90'161","reported_seq":503,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T20:40:21.403403+0000","last_change":"2023-09-23T20:38:05.602250+0000","last_active":"2023-09-23T20:40:21.403403+0000","last_peered":"2023-09-23T20:40:21.403403+0000","last_clean":"2023-09-23T20:40:21.403403+0000","last_became_active":"2023-09-23T20:38:05.601874+0000","last_became_peered":"2023-09-23T20:38:05.601874+0000","last_unstale":"2023-09-23T20:40:21.403403+0000","last_undegraded":"2023-09-23T20:40:21.403403+0000","last_fullsized":"2023-09-23T20:40:21.403403+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:35:42.864363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:35:42.864363+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":40253,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":39,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-09-23T20:40:18.527873+0000","last_change":"2023-09-23T20:38:37.481551+0000","last_active":"2023-09-23T20:40:18.527873+0000","last_peered":"2023-09-23T20:40:18.527873+0000","last_clean":"2023-09-23T20:40:18.527873+0000","last_became_active":"2023-09-23T20:38:37.458687+0000","last_became_peered":"2023-09-23T20:38:37.458687+0000","last_unstale":"2023-09-23T20:40:18.527873+0000","last_undegraded":"2023-09-23T20:40:18.527873+0000","last_fullsized":"2023-09-23T20:40:18.527873+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:22:15.033243+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":98,"reported_epoch":441,"state":"active+clean","last_fresh":"2023-09-23T20:40:19.519178+0000","last_change":"2023-09-23T20:34:59.987575+0000","last_active":"2023-09-23T20:40:19.519178+0000","last_peered":"2023-09-23T20:40:19.519178+0000","last_clean":"2023-09-23T20:40:19.519178+0000","last_became_active":"2023-09-23T20:34:59.987229+0000","last_became_peered":"2023-09-23T20:34:59.987229+0000","last_unstale":"2023-09-23T20:40:19.519178+0000","last_undegraded":"2023-09-23T20:40:19.519178+0000","last_fullsized":"2023-09-23T20:40:19.519178+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:34:01.820159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.16","version":"442'8328","reported_seq":9109,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T20:40:21.047734+0000","last_change":"2023-09-23T20:38:28.264940+0000","last_active":"2023-09-23T20:40:21.047734+0000","last_peered":"2023-09-23T20:40:21.047734+0000","last_clean":"2023-09-23T20:40:21.047734+0000","last_became_active":"2023-09-23T20:38:27.484333+0000","last_became_peered":"2023-09-23T20:38:27.484333+0000","last_unstale":"2023-09-23T20:40:21.047734+0000","last_undegraded":"2023-09-23T20:40:21.047734+0000","last_fullsized":"2023-09-23T20:40:21.047734+0000","mapping_epoch":410,"log_start":"373'1261","ondisk_log_start":"373'1261","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":7067,"log_dups_size":0,"ondisk_log_size":7067,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":33161216,"num_objects":506,"num_object_clones":0,"num_object_copies":1518,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":506,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5048,"num_write_kb":177728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":335,"num_bytes_recovered":9699328,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":39,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-09-23T20:40:18.527020+0000","last_change":"2023-09-23T20:38:37.457634+0000","last_active":"2023-09-23T20:40:18.527020+0000","last_peered":"2023-09-23T20:40:18.527020+0000","last_clean":"2023-09-23T20:40:18.527020+0000","last_became_active":"2023-09-23T20:38:37.452208+0000","last_became_peered":"2023-09-23T20:38:37.452208+0000","last_unstale":"2023-09-23T20:40:18.527020+0000","last_undegraded":"2023-09-23T20:40:18.527020+0000","last_fullsized":"2023-09-23T20:40:18.527020+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:09:41.754569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":98,"reported_epoch":441,"state":"active+clean","last_fresh":"2023-09-23T20:40:19.517829+0000","last_change":"2023-09-23T20:34:59.987416+0000","last_active":"2023-09-23T20:40:19.517829+0000","last_peered":"2023-09-23T20:40:19.517829+0000","last_clean":"2023-09-23T20:40:19.517829+0000","last_became_active":"2023-09-23T20:34:59.987117+0000","last_became_peered":"2023-09-23T20:34:59.987117+0000","last_unstale":"2023-09-23T20:40:19.517829+0000","last_undegraded":"2023-09-23T20:40:19.517829+0000","last_fullsized":"2023-09-23T20:40:19.517829+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:36:16.318623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.15","version":"444'8426","reported_seq":12000,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-09-23T20:40:23.034844+0000","last_change":"2023-09-23T20:38:53.167534+0000","last_active":"2023-09-23T20:40:23.034844+0000","last_peered":"2023-09-23T20:40:23.034844+0000","last_clean":"2023-09-23T20:40:23.034844+0000","last_became_active":"2023-09-23T20:38:28.579407+0000","last_became_peered":"2023-09-23T20:38:28.579407+0000","last_unstale":"2023-09-23T20:40:23.034844+0000","last_undegraded":"2023-09-23T20:40:23.034844+0000","last_fullsized":"2023-09-23T20:40:23.034844+0000","mapping_epoch":410,"log_start":"373'1839","ondisk_log_start":"373'1839","created":410,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":6587,"log_dups_size":0,"ondisk_log_size":6587,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":27688960,"num_objects":423,"num_object_clones":0,"num_object_copies":1269,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":423,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5131,"num_write_kb":177696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1043,"num_bytes_recovered":56328192,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1c","version":"90'161","reported_seq":698,"reported_epoch":442,"state":"active+clean","last_fresh":"2023-09-23T20:40:20.515431+0000","last_change":"2023-09-23T20:38:05.591992+0000","last_active":"2023-09-23T20:40:20.515431+0000","last_peered":"2023-09-23T20:40:20.515431+0000","last_clean":"2023-09-23T20:40:20.515431+0000","last_became_active":"2023-09-23T20:38:05.591701+0000","last_became_peered":"2023-09-23T20:38:05.591701+0000","last_unstale":"2023-09-23T20:40:20.515431+0000","last_undegraded":"2023-09-23T20:40:20.515431+0000","last_fullsized":"2023-09-23T20:40:20.515431+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:02.384133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:02.384133+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:41.722542+0000","scrub_duration":0.001769098,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":264,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T20:40:21.403845+0000","last_change":"2023-09-23T20:38:02.542719+0000","last_active":"2023-09-23T20:40:21.403845+0000","last_peered":"2023-09-23T20:40:21.403845+0000","last_clean":"2023-09-23T20:40:21.403845+0000","last_became_active":"2023-09-23T20:38:02.542385+0000","last_became_peered":"2023-09-23T20:38:02.542385+0000","last_unstale":"2023-09-23T20:40:21.403845+0000","last_undegraded":"2023-09-23T20:40:21.403845+0000","last_fullsized":"2023-09-23T20:40:21.403845+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:36:23.673757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":231,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T20:40:21.403371+0000","last_change":"2023-09-23T20:37:43.584253+0000","last_active":"2023-09-23T20:40:21.403371+0000","last_peered":"2023-09-23T20:40:21.403371+0000","last_clean":"2023-09-23T20:40:21.403371+0000","last_became_active":"2023-09-23T20:37:43.580069+0000","last_became_peered":"2023-09-23T20:37:43.580069+0000","last_unstale":"2023-09-23T20:40:21.403371+0000","last_undegraded":"2023-09-23T20:40:21.403371+0000","last_fullsized":"2023-09-23T20:40:21.403371+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:39:11.482581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.14","version":"440'8653","reported_seq":10834,"reported_epoch":442,"state":"active+recovering+undersized+remapped","last_fresh":"2023-09-23T20:40:20.627523+0000","last_change":"2023-09-23T20:40:19.683603+0000","last_active":"2023-09-23T20:40:20.627523+0000","last_peered":"2023-09-23T20:40:20.627523+0000","last_clean":"2023-09-23T20:37:15.925522+0000","last_became_active":"2023-09-23T20:38:39.251212+0000","last_became_peered":"2023-09-23T20:38:39.251212+0000","last_unstale":"2023-09-23T20:40:20.627523+0000","last_undegraded":"2023-09-23T20:40:20.627523+0000","last_fullsized":"2023-09-23T20:38:38.450156+0000","mapping_epoch":416,"log_start":"373'2080","ondisk_log_start":"373'2080","created":410,"last_epoch_clean":370,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":6573,"log_dups_size":0,"ondisk_log_size":6573,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":29294592,"num_objects":447,"num_object_clones":0,"num_object_copies":1341,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":447,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5246,"num_write_kb":182176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":665,"num_bytes_recovered":34406400,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[4,2],"avail_no_missing":["4","2","6"],"object_location_counts":[{"shards":"2,4","objects":447}],"blocked_by":[],"up_primary":5,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1d","version":"90'161","reported_seq":532,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-09-23T20:40:18.541165+0000","last_change":"2023-09-23T20:36:01.505098+0000","last_active":"2023-09-23T20:40:18.541165+0000","last_peered":"2023-09-23T20:40:18.541165+0000","last_clean":"2023-09-23T20:40:18.541165+0000","last_became_active":"2023-09-23T20:35:26.644707+0000","last_became_peered":"2023-09-23T20:35:26.644707+0000","last_unstale":"2023-09-23T20:40:18.541165+0000","last_undegraded":"2023-09-23T20:40:18.541165+0000","last_fullsized":"2023-09-23T20:40:18.541165+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:01.504973+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:01.504973+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:02:10.880809+0000","scrub_duration":0.00202344,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":486,"reported_epoch":441,"state":"active+clean","last_fresh":"2023-09-23T20:40:19.569435+0000","last_change":"2023-09-23T20:40:19.569032+0000","last_active":"2023-09-23T20:40:19.569435+0000","last_peered":"2023-09-23T20:40:19.569435+0000","last_clean":"2023-09-23T20:40:19.569435+0000","last_became_active":"2023-09-23T20:40:19.538476+0000","last_became_peered":"2023-09-23T20:40:19.538476+0000","last_unstale":"2023-09-23T20:40:19.569435+0000","last_undegraded":"2023-09-23T20:40:19.569435+0000","last_fullsized":"2023-09-23T20:40:19.569435+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:31:58.998168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3324,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":159,"reported_epoch":441,"state":"active+clean","last_fresh":"2023-09-23T20:40:19.517928+0000","last_change":"2023-09-23T20:38:02.538333+0000","last_active":"2023-09-23T20:40:19.517928+0000","last_peered":"2023-09-23T20:40:19.517928+0000","last_clean":"2023-09-23T20:40:19.517928+0000","last_became_active":"2023-09-23T20:38:02.538008+0000","last_became_peered":"2023-09-23T20:38:02.538008+0000","last_unstale":"2023-09-23T20:40:19.517928+0000","last_undegraded":"2023-09-23T20:40:19.517928+0000","last_fullsized":"2023-09-23T20:40:19.517928+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:02:38.789530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":80,"num_bytes_recovered":18486,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26691,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":8,"num_pg":26,"stat_sum":{"num_bytes":876412952,"num_objects":13374,"num_object_clones":0,"num_object_copies":40122,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":9501,"num_objects_misplaced":1976,"num_objects_unfound":0,"num_objects_dirty":13374,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":165268,"num_write_kb":5716481,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33066,"num_bytes_recovered":2047148032,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":4272566272,"data_stored":4272554056,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":167416,"ondisk_log_size":167416,"up":78,"acting":69,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":351684,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":155,"num_read_kb":290,"num_write":76,"num_write_kb":768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":891212,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":87,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5983,"num_read_kb":13200,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":57,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6082,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":23,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201627,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":647908,"kb_used_data":272220,"kb_used_omap":0,"kb_used_meta":375680,"kb_avail":93097244,"statfs":{"total":95995035648,"available":95331577856,"internally_reserved":0,"allocated":278753280,"data_stored":277336498,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":384696320},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:39:51 2023","interfaces":[{"interface":"back","average":{"1min":0.81899999999999995,"5min":2.395,"15min":2.177},"min":{"1min":0.25800000000000001,"5min":0.24399999999999999,"15min":0.20799999999999999},"max":{"1min":3.7639999999999998,"5min":18.315000000000001,"15min":48.798999999999999},"last":5.6310000000000002},{"interface":"front","average":{"1min":0.81599999999999995,"5min":1.7150000000000001,"15min":2.008},"min":{"1min":0.29199999999999998,"5min":0.247,"15min":0.24399999999999999},"max":{"1min":7.109,"5min":19.355,"15min":48.247},"last":1.1899999999999999}]},{"osd":1,"last update":"Sat Sep 23 20:40:03 2023","interfaces":[{"interface":"back","average":{"1min":0.92600000000000005,"5min":2.7120000000000002,"15min":2.1520000000000001},"min":{"1min":0.28799999999999998,"5min":0.25900000000000001,"15min":0.254},"max":{"1min":2.9020000000000001,"5min":49.965000000000003,"15min":49.965000000000003},"last":1.04},{"interface":"front","average":{"1min":0.89300000000000002,"5min":2.6989999999999998,"15min":2.1709999999999998},"min":{"1min":0.30299999999999999,"5min":0.26900000000000002,"15min":0.248},"max":{"1min":5.508,"5min":53.451999999999998,"15min":53.451999999999998},"last":0.42899999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:39:51 2023","interfaces":[{"interface":"back","average":{"1min":1.23,"5min":2.129,"15min":2.1469999999999998},"min":{"1min":0.32800000000000001,"5min":0.27800000000000002,"15min":0.27600000000000002},"max":{"1min":7.2910000000000004,"5min":16.965,"15min":16.965},"last":4.5039999999999996},{"interface":"front","average":{"1min":1.1140000000000001,"5min":2.6459999999999999,"15min":2.48},"min":{"1min":0.32500000000000001,"5min":0.308,"15min":0.26600000000000001},"max":{"1min":3.6880000000000002,"5min":23.414000000000001,"15min":23.494},"last":0.31900000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:40:21 2023","interfaces":[{"interface":"back","average":{"1min":1.105,"5min":2.2480000000000002,"15min":2.4470000000000001},"min":{"1min":0.35999999999999999,"5min":0.311,"15min":0.26200000000000001},"max":{"1min":4.2149999999999999,"5min":23.352,"15min":23.352},"last":0.35999999999999999},{"interface":"front","average":{"1min":1.139,"5min":2.2709999999999999,"15min":2.423},"min":{"1min":0.46500000000000002,"5min":0.34399999999999997,"15min":0.32500000000000001},"max":{"1min":4.2359999999999998,"5min":20.969999999999999,"15min":20.969999999999999},"last":1.843}]},{"osd":4,"last update":"Sat Sep 23 20:40:21 2023","interfaces":[{"interface":"back","average":{"1min":1.046,"5min":1.085,"15min":1.2170000000000001},"min":{"1min":0.35799999999999998,"5min":0.25600000000000001,"15min":0.23200000000000001},"max":{"1min":4.1639999999999997,"5min":18.364999999999998,"15min":18.364999999999998},"last":0.57599999999999996},{"interface":"front","average":{"1min":0.94999999999999996,"5min":1.296,"15min":1.417},"min":{"1min":0.39100000000000001,"5min":0.28399999999999997,"15min":0.27600000000000002},"max":{"1min":2.605,"5min":23.617000000000001,"15min":23.617000000000001},"last":1.5840000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:40:21 2023","interfaces":[{"interface":"back","average":{"1min":1.0780000000000001,"5min":1.1499999999999999,"15min":1.2509999999999999},"min":{"1min":0.30099999999999999,"5min":0.29199999999999998,"15min":0.26600000000000001},"max":{"1min":4.3780000000000001,"5min":12.516,"15min":13.241},"last":0.38600000000000001},{"interface":"front","average":{"1min":0.76400000000000001,"5min":1.286,"15min":1.3540000000000001},"min":{"1min":0.38400000000000001,"5min":0.31900000000000001,"15min":0.315},"max":{"1min":3.1030000000000002,"5min":18.390000000000001,"15min":18.831},"last":0.48799999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:40:21 2023","interfaces":[{"interface":"back","average":{"1min":1.2629999999999999,"5min":1.4350000000000001,"15min":1.329},"min":{"1min":0.44,"5min":0.312,"15min":0.30399999999999999},"max":{"1min":4.0540000000000003,"5min":18.417000000000002,"15min":18.417000000000002},"last":0.68400000000000005},{"interface":"front","average":{"1min":1.1659999999999999,"5min":1.113,"15min":1.252},"min":{"1min":0.38200000000000001,"5min":0.32700000000000001,"15min":0.318},"max":{"1min":6.4340000000000002,"5min":7.266,"15min":9.9120000000000008},"last":2.306}]}]},{"osd":6,"up_from":47,"seq":201863463266,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":722964,"kb_used_data":344332,"kb_used_omap":0,"kb_used_meta":378624,"kb_avail":93022188,"statfs":{"total":95995035648,"available":95254720512,"internally_reserved":0,"allocated":352595968,"data_stored":351194875,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":387710976},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,5,2,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:39:49 2023","interfaces":[{"interface":"back","average":{"1min":0.89500000000000002,"5min":1.859,"15min":1.9339999999999999},"min":{"1min":0.23699999999999999,"5min":0.23300000000000001,"15min":0.221},"max":{"1min":2.887,"5min":17.492999999999999,"15min":53.701999999999998},"last":0.28699999999999998},{"interface":"front","average":{"1min":1.4219999999999999,"5min":2.3730000000000002,"15min":2.4409999999999998},"min":{"1min":0.24099999999999999,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":6.625,"5min":60.328000000000003,"15min":60.328000000000003},"last":3.3780000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:40:06 2023","interfaces":[{"interface":"back","average":{"1min":1.157,"5min":2.0600000000000001,"15min":2.3610000000000002},"min":{"1min":0.26600000000000001,"5min":0.26000000000000001,"15min":0.249},"max":{"1min":5.8630000000000004,"5min":64.274000000000001,"15min":64.274000000000001},"last":0.90100000000000002},{"interface":"front","average":{"1min":1.2390000000000001,"5min":1.393,"15min":2.4830000000000001},"min":{"1min":0.34499999999999997,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":6.8460000000000001,"5min":6.8460000000000001,"15min":34.490000000000002},"last":0.32900000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:39:49 2023","interfaces":[{"interface":"back","average":{"1min":1.069,"5min":1.748,"15min":2.1499999999999999},"min":{"1min":0.30299999999999999,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":4.3399999999999999,"5min":13.429,"15min":34.317999999999998},"last":3.226},{"interface":"front","average":{"1min":1.0740000000000001,"5min":1.657,"15min":2.024},"min":{"1min":0.40000000000000002,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":4.3630000000000004,"5min":13.457000000000001,"15min":34.292999999999999},"last":3.3170000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:39:35 2023","interfaces":[{"interface":"back","average":{"1min":1.829,"5min":1.8759999999999999,"15min":1.9970000000000001},"min":{"1min":0.36699999999999999,"5min":0.29999999999999999,"15min":0.29599999999999999},"max":{"1min":6.7690000000000001,"5min":8.9670000000000005,"15min":9.4239999999999995},"last":0.38700000000000001},{"interface":"front","average":{"1min":1.742,"5min":2.149,"15min":2.1459999999999999},"min":{"1min":0.36899999999999999,"5min":0.34899999999999998,"15min":0.317},"max":{"1min":7.6230000000000002,"5min":11.913,"15min":12.875999999999999},"last":3.3460000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:40:11 2023","interfaces":[{"interface":"back","average":{"1min":0.90200000000000002,"5min":0.84799999999999998,"15min":0.95399999999999996},"min":{"1min":0.30099999999999999,"5min":0.26900000000000002,"15min":0.251},"max":{"1min":2.859,"5min":23.904,"15min":23.904},"last":4.0830000000000002},{"interface":"front","average":{"1min":1.498,"5min":1.1879999999999999,"15min":1.204},"min":{"1min":0.36799999999999999,"5min":0.29199999999999998,"15min":0.27000000000000002},"max":{"1min":6.7359999999999998,"5min":11.515000000000001,"15min":11.515000000000001},"last":0.41999999999999998}]},{"osd":5,"last update":"Sat Sep 23 20:40:11 2023","interfaces":[{"interface":"back","average":{"1min":1.2210000000000001,"5min":0.85299999999999998,"15min":0.96199999999999997},"min":{"1min":0.375,"5min":0.318,"15min":0.28699999999999998},"max":{"1min":6.6340000000000003,"5min":6.6340000000000003,"15min":14.085000000000001},"last":0.46700000000000003},{"interface":"front","average":{"1min":1.2529999999999999,"5min":0.97899999999999998,"15min":1.355},"min":{"1min":0.35399999999999998,"5min":0.32500000000000001,"15min":0.30599999999999999},"max":{"1min":6.6909999999999998,"5min":6.7320000000000002,"15min":18.192},"last":0.629}]},{"osd":7,"last update":"Sat Sep 23 20:39:35 2023","interfaces":[{"interface":"back","average":{"1min":0.877,"5min":0.82299999999999995,"15min":1.137},"min":{"1min":0.39400000000000002,"5min":0.36599999999999999,"15min":0.28599999999999998},"max":{"1min":4.1520000000000001,"5min":4.851,"15min":10.217000000000001},"last":0.64100000000000001},{"interface":"front","average":{"1min":1.1910000000000001,"5min":1.0389999999999999,"15min":1.3460000000000001},"min":{"1min":0.379,"5min":0.35999999999999999,"15min":0.30299999999999999},"max":{"1min":6.7119999999999997,"5min":8.8119999999999994,"15min":9.7360000000000007},"last":0.59799999999999998}]}]},{"osd":1,"up_from":266,"seq":1142461300864,"num_pgs":65,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":860292,"kb_used_data":424060,"kb_used_omap":8,"kb_used_meta":436215,"kb_avail":92884860,"statfs":{"total":95995035648,"available":95114096640,"internally_reserved":0,"allocated":434237440,"data_stored":431841223,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8206,"internal_metadata":446685170},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:40:05 2023","interfaces":[{"interface":"back","average":{"1min":1.3680000000000001,"5min":2.2250000000000001,"15min":2.5099999999999998},"min":{"1min":0.218,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":8.1150000000000002,"5min":14.531000000000001,"15min":52.689999999999998},"last":8.2799999999999994},{"interface":"front","average":{"1min":2.4140000000000001,"5min":3.1200000000000001,"15min":3.2650000000000001},"min":{"1min":0.316,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":11.994,"5min":19.41,"15min":52.662999999999997},"last":2.8730000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:40:05 2023","interfaces":[{"interface":"back","average":{"1min":2.7280000000000002,"5min":3.0489999999999999,"15min":4.0810000000000004},"min":{"1min":0.23400000000000001,"5min":0.23300000000000001,"15min":0.20899999999999999},"max":{"1min":14.148,"5min":32.255000000000003,"15min":32.255000000000003},"last":0.28799999999999998},{"interface":"front","average":{"1min":1.911,"5min":2.238,"15min":2.3170000000000002},"min":{"1min":0.28599999999999998,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":8.8079999999999998,"5min":13.404999999999999,"15min":13.404999999999999},"last":2.7149999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:40:05 2023","interfaces":[{"interface":"back","average":{"1min":1.6659999999999999,"5min":2.3809999999999998,"15min":3.5430000000000001},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":8.0079999999999991,"5min":14.359,"15min":80.745999999999995},"last":2.6179999999999999},{"interface":"front","average":{"1min":2.6509999999999998,"5min":2.2810000000000001,"15min":3.9319999999999999},"min":{"1min":0.316,"5min":0.27500000000000002,"15min":0.22700000000000001},"max":{"1min":12.045999999999999,"5min":19.315000000000001,"15min":82.988},"last":6.5279999999999996}]},{"osd":4,"last update":"Sat Sep 23 20:40:05 2023","interfaces":[{"interface":"back","average":{"1min":2.0710000000000002,"5min":2.2490000000000001,"15min":2.2090000000000001},"min":{"1min":0.33400000000000002,"5min":0.315,"15min":0.30299999999999999},"max":{"1min":7.8399999999999999,"5min":9.4510000000000005,"15min":24.327999999999999},"last":3.149},{"interface":"front","average":{"1min":2.056,"5min":1.593,"15min":1.764},"min":{"1min":0.33400000000000002,"5min":0.311,"15min":0.311},"max":{"1min":8.1509999999999998,"5min":11.574999999999999,"15min":11.574999999999999},"last":2.677}]},{"osd":5,"last update":"Sat Sep 23 20:40:05 2023","interfaces":[{"interface":"back","average":{"1min":1.996,"5min":2.1400000000000001,"15min":2.3730000000000002},"min":{"1min":0.45800000000000002,"5min":0.34100000000000003,"15min":0.33200000000000002},"max":{"1min":5.5529999999999999,"5min":33.401000000000003,"15min":33.401000000000003},"last":2.7669999999999999},{"interface":"front","average":{"1min":2.5790000000000002,"5min":2.7120000000000002,"15min":2.427},"min":{"1min":0.36199999999999999,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":7.9340000000000002,"5min":33.417000000000002,"15min":33.417000000000002},"last":3.238}]},{"osd":6,"last update":"Sat Sep 23 20:40:05 2023","interfaces":[{"interface":"back","average":{"1min":2.0019999999999998,"5min":1.9570000000000001,"15min":4.1699999999999999},"min":{"1min":0.42899999999999999,"5min":0.36899999999999999,"15min":0.35699999999999998},"max":{"1min":6.8099999999999996,"5min":9.6999999999999993,"15min":72.682000000000002},"last":2.839},{"interface":"front","average":{"1min":1.9419999999999999,"5min":2.2799999999999998,"15min":4.0460000000000003},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":7.883,"5min":9.4849999999999994,"15min":72.563999999999993},"last":3.2050000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:40:05 2023","interfaces":[{"interface":"back","average":{"1min":2.1930000000000001,"5min":2.069,"15min":2.5760000000000001},"min":{"1min":0.48399999999999999,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":5.0819999999999999,"5min":18.437000000000001,"15min":18.437000000000001},"last":2.6429999999999998},{"interface":"front","average":{"1min":2.3149999999999999,"5min":2.3959999999999999,"15min":2.46},"min":{"1min":0.45100000000000001,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":7.9089999999999998,"5min":12.021000000000001,"15min":24.524000000000001},"last":3.27}]}]},{"osd":0,"up_from":94,"seq":403726926067,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1024048,"kb_used_data":385256,"kb_used_omap":7,"kb_used_meta":638648,"kb_avail":92721104,"statfs":{"total":95995035648,"available":94946410496,"internally_reserved":0,"allocated":394502144,"data_stored":392257170,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":653975569},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:40:11 2023","interfaces":[{"interface":"back","average":{"1min":1.712,"5min":1.8,"15min":3.0680000000000001},"min":{"1min":0.20699999999999999,"5min":0.17199999999999999,"15min":0.17199999999999999},"max":{"1min":9.8879999999999999,"5min":11.554,"15min":42.000999999999998},"last":6.2089999999999996},{"interface":"front","average":{"1min":1.5660000000000001,"5min":1.9710000000000001,"15min":3.3370000000000002},"min":{"1min":0.218,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":10.097,"5min":12.35,"15min":23.036000000000001},"last":6.1289999999999996}]},{"osd":2,"last update":"Sat Sep 23 20:39:47 2023","interfaces":[{"interface":"back","average":{"1min":0.83099999999999996,"5min":2.7610000000000001,"15min":2.907},"min":{"1min":0.23400000000000001,"5min":0.218,"15min":0.185},"max":{"1min":2.9670000000000001,"5min":62.726999999999997,"15min":77.448999999999998},"last":0.66200000000000003},{"interface":"front","average":{"1min":1.5489999999999999,"5min":3.0329999999999999,"15min":2.8010000000000002},"min":{"1min":0.27300000000000002,"5min":0.247,"15min":0.188},"max":{"1min":6.4800000000000004,"5min":62.689,"15min":81.724999999999994},"last":0.54900000000000004}]},{"osd":3,"last update":"Sat Sep 23 20:39:17 2023","interfaces":[{"interface":"back","average":{"1min":3.8900000000000001,"5min":3.294,"15min":3.1059999999999999},"min":{"1min":0.28899999999999998,"5min":0.24399999999999999,"15min":0.23400000000000001},"max":{"1min":51.536000000000001,"5min":51.536000000000001,"15min":81.400999999999996},"last":0.95999999999999996},{"interface":"front","average":{"1min":3.5779999999999998,"5min":3.5790000000000002,"15min":3.347},"min":{"1min":0.316,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":44.811999999999998,"5min":44.811999999999998,"15min":81.137},"last":6.1689999999999996}]},{"osd":4,"last update":"Sat Sep 23 20:39:47 2023","interfaces":[{"interface":"back","average":{"1min":1.333,"5min":2.008,"15min":2.1539999999999999},"min":{"1min":0.31900000000000001,"5min":0.28899999999999998,"15min":0.25900000000000001},"max":{"1min":6.2450000000000001,"5min":11.068,"15min":23.876999999999999},"last":0.627},{"interface":"front","average":{"1min":1.0940000000000001,"5min":1.5700000000000001,"15min":1.954},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":6.3310000000000004,"5min":11.458,"15min":23.681999999999999},"last":0.36299999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:39:47 2023","interfaces":[{"interface":"back","average":{"1min":0.97299999999999998,"5min":1.5580000000000001,"15min":2.1600000000000001},"min":{"1min":0.371,"5min":0.35099999999999998,"15min":0.34200000000000003},"max":{"1min":3.29,"5min":11.497999999999999,"15min":24.198},"last":0.44},{"interface":"front","average":{"1min":1.4910000000000001,"5min":2.4399999999999999,"15min":2.3010000000000002},"min":{"1min":0.318,"5min":0.318,"15min":0.25600000000000001},"max":{"1min":6.593,"5min":26.228000000000002,"15min":26.228000000000002},"last":0.45800000000000002}]},{"osd":6,"last update":"Sat Sep 23 20:39:47 2023","interfaces":[{"interface":"back","average":{"1min":1.1599999999999999,"5min":1.867,"15min":2.1309999999999998},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.318},"max":{"1min":6.3680000000000003,"5min":11.526999999999999,"15min":24.408000000000001},"last":0.39800000000000002},{"interface":"front","average":{"1min":1.6599999999999999,"5min":2.4129999999999998,"15min":2.2480000000000002},"min":{"1min":0.36799999999999999,"5min":0.35499999999999998,"15min":0.29399999999999998},"max":{"1min":6.5469999999999997,"5min":26.268000000000001,"15min":26.268000000000001},"last":0.49199999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:39:47 2023","interfaces":[{"interface":"back","average":{"1min":1.6539999999999999,"5min":2.4710000000000001,"15min":2.4249999999999998},"min":{"1min":0.39800000000000002,"5min":0.378,"15min":0.30599999999999999},"max":{"1min":6.9160000000000004,"5min":26.177,"15min":26.177},"last":0.52100000000000002},{"interface":"front","average":{"1min":1.327,"5min":2.1629999999999998,"15min":2.5409999999999999},"min":{"1min":0.39200000000000002,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":4.4210000000000003,"5min":11.112,"15min":24.669},"last":0.59099999999999997}]}]},{"osd":2,"up_from":104,"seq":446676599015,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1011088,"kb_used_data":337416,"kb_used_omap":11,"kb_used_meta":673652,"kb_avail":92734064,"statfs":{"total":95995035648,"available":94959681536,"internally_reserved":0,"allocated":345513984,"data_stored":343252119,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11878,"internal_metadata":689820058},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:40:00 2023","interfaces":[{"interface":"back","average":{"1min":2.1339999999999999,"5min":1.921,"15min":2.7160000000000002},"min":{"1min":0.26400000000000001,"5min":0.20300000000000001,"15min":0.17999999999999999},"max":{"1min":8.9710000000000001,"5min":14.124000000000001,"15min":82.424999999999997},"last":1.6799999999999999},{"interface":"front","average":{"1min":2.2919999999999998,"5min":2.375,"15min":2.5920000000000001},"min":{"1min":0.23000000000000001,"5min":0.19,"15min":0.19},"max":{"1min":9.5600000000000005,"5min":14.055,"15min":82.524000000000001},"last":0.40799999999999997}]},{"osd":1,"last update":"Sat Sep 23 20:40:00 2023","interfaces":[{"interface":"back","average":{"1min":2.472,"5min":2.9500000000000002,"15min":3.6240000000000001},"min":{"1min":0.433,"5min":0.245,"15min":0.221},"max":{"1min":11.279999999999999,"5min":46.405000000000001,"15min":46.405000000000001},"last":1.899},{"interface":"front","average":{"1min":1.506,"5min":2.9129999999999998,"15min":2.9009999999999998},"min":{"1min":0.24399999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":8.8699999999999992,"5min":46.459000000000003,"15min":46.459000000000003},"last":1.9750000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:40:22 2023","interfaces":[{"interface":"back","average":{"1min":2.851,"5min":2.613,"15min":2.4769999999999999},"min":{"1min":0.28499999999999998,"5min":0.27100000000000002,"15min":0.19800000000000001},"max":{"1min":11.304,"5min":19.562999999999999,"15min":19.562999999999999},"last":1.3109999999999999},{"interface":"front","average":{"1min":1.7909999999999999,"5min":1.6160000000000001,"15min":2.5089999999999999},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.23899999999999999},"max":{"1min":8.9369999999999994,"5min":13.148999999999999,"15min":30.704999999999998},"last":1.7769999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:40:00 2023","interfaces":[{"interface":"back","average":{"1min":1.587,"5min":1.8460000000000001,"15min":1.931},"min":{"1min":0.33300000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":6.6849999999999996,"5min":9.2430000000000003,"15min":16.797999999999998},"last":0.33300000000000002},{"interface":"front","average":{"1min":1.4530000000000001,"5min":2.0009999999999999,"15min":2.117},"min":{"1min":0.38,"5min":0.307,"15min":0.27600000000000002},"max":{"1min":7.9400000000000004,"5min":19.440999999999999,"15min":19.440999999999999},"last":1.5840000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:40:00 2023","interfaces":[{"interface":"back","average":{"1min":1.7290000000000001,"5min":1.8680000000000001,"15min":2.4329999999999998},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.309},"max":{"1min":6.6319999999999997,"5min":9.1799999999999997,"15min":95.393000000000001},"last":0.43099999999999999},{"interface":"front","average":{"1min":1.7110000000000001,"5min":2.0230000000000001,"15min":2.694},"min":{"1min":0.42399999999999999,"5min":0.35299999999999998,"15min":0.32300000000000001},"max":{"1min":7.8179999999999996,"5min":19.513000000000002,"15min":95.414000000000001},"last":1.625}]},{"osd":6,"last update":"Sat Sep 23 20:40:00 2023","interfaces":[{"interface":"back","average":{"1min":1.6339999999999999,"5min":1.9730000000000001,"15min":2.1909999999999998},"min":{"1min":0.47099999999999997,"5min":0.37,"15min":0.32100000000000001},"max":{"1min":7.8579999999999997,"5min":19.486000000000001,"15min":19.486000000000001},"last":1.6559999999999999},{"interface":"front","average":{"1min":1.512,"5min":1.4550000000000001,"15min":2.0329999999999999},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.316},"max":{"1min":9.3010000000000002,"5min":10.811,"15min":30.670999999999999},"last":1.5600000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:40:00 2023","interfaces":[{"interface":"back","average":{"1min":1.073,"5min":1.3520000000000001,"15min":2.028},"min":{"1min":0.42499999999999999,"5min":0.39300000000000002,"15min":0.35199999999999998},"max":{"1min":4.4260000000000002,"5min":10.231,"15min":30.741},"last":1.6020000000000001},{"interface":"front","average":{"1min":1.752,"5min":1.9650000000000001,"15min":2.1760000000000002},"min":{"1min":0.44,"5min":0.40999999999999998,"15min":0.38},"max":{"1min":6.7220000000000004,"5min":9.2720000000000002,"15min":16.719000000000001},"last":0.38}]}]},{"osd":3,"up_from":109,"seq":468151435489,"num_pgs":63,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1232312,"kb_used_data":439280,"kb_used_omap":4,"kb_used_meta":793019,"kb_avail":92512840,"statfs":{"total":95995035648,"available":94733148160,"internally_reserved":0,"allocated":449822720,"data_stored":447581778,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4514,"internal_metadata":812052062},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:39:23 2023","interfaces":[{"interface":"back","average":{"1min":0.78500000000000003,"5min":2.202,"15min":2.5649999999999999},"min":{"1min":0.224,"5min":0.215,"15min":0.186},"max":{"1min":5.2480000000000002,"5min":17.007000000000001,"15min":67.305999999999997},"last":7.6879999999999997},{"interface":"front","average":{"1min":0.71499999999999997,"5min":2.0369999999999999,"15min":2.3679999999999999},"min":{"1min":0.26800000000000002,"5min":0.23699999999999999,"15min":0.19500000000000001},"max":{"1min":2.3980000000000001,"5min":10.140000000000001,"15min":67.403999999999996},"last":2.5920000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:39:59 2023","interfaces":[{"interface":"back","average":{"1min":0.81699999999999995,"5min":1.3169999999999999,"15min":2.375},"min":{"1min":0.246,"5min":0.23999999999999999,"15min":0.222},"max":{"1min":7.6219999999999999,"5min":9.8940000000000001,"15min":16.189},"last":1.6659999999999999},{"interface":"front","average":{"1min":2.343,"5min":2.1669999999999998,"15min":2.6829999999999998},"min":{"1min":0.26500000000000001,"5min":0.26200000000000001,"15min":0.24199999999999999},"max":{"1min":9.1590000000000007,"5min":13.727,"15min":32.603999999999999},"last":8.7449999999999992}]},{"osd":2,"last update":"Sat Sep 23 20:39:23 2023","interfaces":[{"interface":"back","average":{"1min":0.70799999999999996,"5min":1.9510000000000001,"15min":2.468},"min":{"1min":0.246,"5min":0.23799999999999999,"15min":0.22800000000000001},"max":{"1min":2.4849999999999999,"5min":10.747999999999999,"15min":32.857999999999997},"last":2.6230000000000002},{"interface":"front","average":{"1min":1.1359999999999999,"5min":2.411,"15min":2.5150000000000001},"min":{"1min":0.31,"5min":0.251,"15min":0.20899999999999999},"max":{"1min":4.0389999999999997,"5min":22.32,"15min":24.146999999999998},"last":0.63400000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:39:23 2023","interfaces":[{"interface":"back","average":{"1min":0.75700000000000001,"5min":1.2350000000000001,"15min":1.8240000000000001},"min":{"1min":0.34399999999999997,"5min":0.32200000000000001,"15min":0.29799999999999999},"max":{"1min":2.8599999999999999,"5min":9.3160000000000007,"15min":46.042999999999999},"last":1.7030000000000001},{"interface":"front","average":{"1min":1.5149999999999999,"5min":2.0910000000000002,"15min":2.3929999999999998},"min":{"1min":0.33000000000000002,"5min":0.308,"15min":0.28100000000000003},"max":{"1min":6.7519999999999998,"5min":14.422000000000001,"15min":46.579999999999998},"last":0.70699999999999996}]},{"osd":5,"last update":"Sat Sep 23 20:39:23 2023","interfaces":[{"interface":"back","average":{"1min":0.85799999999999998,"5min":1.4139999999999999,"15min":2.0539999999999998},"min":{"1min":0.379,"5min":0.32300000000000001,"15min":0.311},"max":{"1min":3.827,"5min":13.228999999999999,"15min":45.292000000000002},"last":1.768},{"interface":"front","average":{"1min":1.2709999999999999,"5min":1.474,"15min":1.919},"min":{"1min":0.42599999999999999,"5min":0.32000000000000001,"15min":0.313},"max":{"1min":7.3200000000000003,"5min":9.0839999999999996,"15min":44.679000000000002},"last":2.496}]},{"osd":6,"last update":"Sat Sep 23 20:39:23 2023","interfaces":[{"interface":"back","average":{"1min":1.016,"5min":1.3149999999999999,"15min":1.9059999999999999},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.34899999999999998},"max":{"1min":5.4409999999999998,"5min":9.0470000000000006,"15min":17.524999999999999},"last":2.5510000000000002},{"interface":"front","average":{"1min":1.1060000000000001,"5min":2.2120000000000002,"15min":2.1400000000000001},"min":{"1min":0.41899999999999998,"5min":0.39800000000000002,"15min":0.33700000000000002},"max":{"1min":3.9660000000000002,"5min":14.464,"15min":19.524000000000001},"last":0.67200000000000004}]},{"osd":7,"last update":"Sat Sep 23 20:39:23 2023","interfaces":[{"interface":"back","average":{"1min":1.6910000000000001,"5min":2.476,"15min":2.3849999999999998},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.34999999999999998},"max":{"1min":7.6399999999999997,"5min":14.521000000000001,"15min":19.571999999999999},"last":0.74099999999999999},{"interface":"front","average":{"1min":1.0129999999999999,"5min":1.478,"15min":2.1320000000000001},"min":{"1min":0.45000000000000001,"5min":0.44600000000000001,"15min":0.371},"max":{"1min":5.3259999999999996,"5min":9.3719999999999999,"15min":45.387999999999998},"last":1.738}]}]},{"osd":4,"up_from":30,"seq":128849019242,"num_pgs":61,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":753724,"kb_used_data":384564,"kb_used_omap":0,"kb_used_meta":369152,"kb_avail":92991428,"statfs":{"total":95995035648,"available":95223222272,"internally_reserved":0,"allocated":393793536,"data_stored":392399279,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":378011648},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:39:48 2023","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":1.6910000000000001,"15min":1.6819999999999999},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.222},"max":{"1min":2.4279999999999999,"5min":24.114999999999998,"15min":24.114999999999998},"last":0.45800000000000002},{"interface":"front","average":{"1min":1.145,"5min":1.829,"15min":1.8580000000000001},"min":{"1min":0.29899999999999999,"5min":0.246,"15min":0.246},"max":{"1min":6.9119999999999999,"5min":24.135999999999999,"15min":24.135999999999999},"last":0.36099999999999999}]},{"osd":1,"last update":"Sat Sep 23 20:40:06 2023","interfaces":[{"interface":"back","average":{"1min":0.95899999999999996,"5min":1.331,"15min":1.7410000000000001},"min":{"1min":0.309,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":4.1669999999999998,"5min":18.190999999999999,"15min":18.190999999999999},"last":0.41499999999999998},{"interface":"front","average":{"1min":0.88300000000000001,"5min":1.4259999999999999,"15min":1.6899999999999999},"min":{"1min":0.307,"5min":0.28499999999999998,"15min":0.28199999999999997},"max":{"1min":3.7109999999999999,"5min":18.173999999999999,"15min":18.173999999999999},"last":0.52000000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:40:03 2023","interfaces":[{"interface":"back","average":{"1min":1.3400000000000001,"5min":1.6060000000000001,"15min":1.8740000000000001},"min":{"1min":0.34200000000000003,"5min":0.29999999999999999,"15min":0.26600000000000001},"max":{"1min":3.8010000000000002,"5min":10.416,"15min":32.363},"last":0.42699999999999999},{"interface":"front","average":{"1min":2.2069999999999999,"5min":2.1019999999999999,"15min":2.3109999999999999},"min":{"1min":0.34300000000000003,"5min":0.28899999999999998,"15min":0.27600000000000002},"max":{"1min":11.917999999999999,"5min":11.917999999999999,"15min":34.042999999999999},"last":0.39900000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:40:17 2023","interfaces":[{"interface":"back","average":{"1min":2.4209999999999998,"5min":1.8480000000000001,"15min":2.113},"min":{"1min":0.40500000000000003,"5min":0.372,"15min":0.29099999999999998},"max":{"1min":12.147,"5min":12.147,"15min":32.332999999999998},"last":0.48199999999999998},{"interface":"front","average":{"1min":1.8280000000000001,"5min":1.8360000000000001,"15min":2.206},"min":{"1min":0.371,"5min":0.371,"15min":0.30299999999999999},"max":{"1min":11.914,"5min":11.914,"15min":32.399999999999999},"last":0.54500000000000004}]},{"osd":5,"last update":"Sat Sep 23 20:39:43 2023","interfaces":[{"interface":"back","average":{"1min":0.84499999999999997,"5min":0.69799999999999995,"15min":1.54},"min":{"1min":0.29399999999999998,"5min":0.26400000000000001,"15min":0.247},"max":{"1min":4.1260000000000003,"5min":4.1260000000000003,"15min":82.866},"last":0.34100000000000003},{"interface":"front","average":{"1min":1.3129999999999999,"5min":0.89300000000000002,"15min":1.528},"min":{"1min":0.35699999999999998,"5min":0.28999999999999998,"15min":0.27800000000000002},"max":{"1min":7.0129999999999999,"5min":7.0129999999999999,"15min":82.546000000000006},"last":0.44400000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:40:03 2023","interfaces":[{"interface":"back","average":{"1min":1.722,"5min":1.032,"15min":1.381},"min":{"1min":0.32200000000000001,"5min":0.312,"15min":0.27600000000000002},"max":{"1min":14.731999999999999,"5min":14.731999999999999,"15min":15.252000000000001},"last":0.47399999999999998},{"interface":"front","average":{"1min":1.024,"5min":0.97199999999999998,"15min":1.2450000000000001},"min":{"1min":0.40500000000000003,"5min":0.29899999999999999,"15min":0.27200000000000002},"max":{"1min":4.976,"5min":7.2850000000000001,"15min":11.199},"last":0.377}]},{"osd":7,"last update":"Sat Sep 23 20:40:17 2023","interfaces":[{"interface":"back","average":{"1min":1.0389999999999999,"5min":1.143,"15min":1.5369999999999999},"min":{"1min":0.39100000000000001,"5min":0.318,"15min":0.318},"max":{"1min":4.21,"5min":14.534000000000001,"15min":16.427},"last":0.50600000000000001},{"interface":"front","average":{"1min":1.472,"5min":0.84599999999999997,"15min":1.3180000000000001},"min":{"1min":0.36399999999999999,"5min":0.32700000000000001,"15min":0.307},"max":{"1min":7.9390000000000001,"5min":7.9390000000000001,"15min":19.018999999999998},"last":0.49299999999999999}]}]},{"osd":5,"up_from":39,"seq":167503724903,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2005812,"kb_used_data":1674412,"kb_used_omap":0,"kb_used_meta":331392,"kb_avail":91739340,"statfs":{"total":95995035648,"available":93941084160,"internally_reserved":0,"allocated":1714597888,"data_stored":1713207909,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":339345408},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:39:48 2023","interfaces":[{"interface":"back","average":{"1min":1.4970000000000001,"5min":2.077,"15min":1.8620000000000001},"min":{"1min":0.253,"5min":0.23899999999999999,"15min":0.20699999999999999},"max":{"1min":5.3079999999999998,"5min":18.16,"15min":32.475000000000001},"last":0.27500000000000002},{"interface":"front","average":{"1min":1.3600000000000001,"5min":1.8899999999999999,"15min":1.7589999999999999},"min":{"1min":0.32100000000000001,"5min":0.29499999999999998,"15min":0.245},"max":{"1min":5.2969999999999997,"5min":9.0570000000000004,"15min":32.457000000000001},"last":0.34999999999999998}]},{"osd":1,"last update":"Sat Sep 23 20:40:07 2023","interfaces":[{"interface":"back","average":{"1min":1.6799999999999999,"5min":2.0070000000000001,"15min":1.9179999999999999},"min":{"1min":0.26800000000000002,"5min":0.26300000000000001,"15min":0.253},"max":{"1min":5.71,"5min":36.588999999999999,"15min":36.588999999999999},"last":0.52400000000000002},{"interface":"front","average":{"1min":1.2470000000000001,"5min":2.0150000000000001,"15min":1.6970000000000001},"min":{"1min":0.29799999999999999,"5min":0.27700000000000002,"15min":0.23499999999999999},"max":{"1min":5.298,"5min":36.920999999999999,"15min":36.920999999999999},"last":0.32600000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:39:48 2023","interfaces":[{"interface":"back","average":{"1min":1.145,"5min":2.0099999999999998,"15min":2.073},"min":{"1min":0.33200000000000002,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":3.4670000000000001,"5min":37.011000000000003,"15min":37.011000000000003},"last":0.37},{"interface":"front","average":{"1min":1.4570000000000001,"5min":1.9239999999999999,"15min":2.0649999999999999},"min":{"1min":0.308,"5min":0.28999999999999998,"15min":0.255},"max":{"1min":4.399,"5min":36.835000000000001,"15min":36.835000000000001},"last":0.433}]},{"osd":3,"last update":"Sat Sep 23 20:40:14 2023","interfaces":[{"interface":"back","average":{"1min":2.2320000000000002,"5min":2.153,"15min":2.2360000000000002},"min":{"1min":0.435,"5min":0.34699999999999998,"15min":0.25700000000000001},"max":{"1min":6.0309999999999997,"5min":36.716000000000001,"15min":36.716000000000001},"last":0.44800000000000001},{"interface":"front","average":{"1min":1.5289999999999999,"5min":2.2240000000000002,"15min":2.117},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.28799999999999998},"max":{"1min":8.3599999999999994,"5min":18.504000000000001,"15min":18.504000000000001},"last":0.40400000000000003}]},{"osd":4,"last update":"Sat Sep 23 20:39:43 2023","interfaces":[{"interface":"back","average":{"1min":0.82899999999999996,"5min":0.76800000000000002,"15min":1.329},"min":{"1min":0.28000000000000003,"5min":0.255,"15min":0.255},"max":{"1min":4.4939999999999998,"5min":6.9180000000000001,"15min":28.015999999999998},"last":0.49399999999999999},{"interface":"front","average":{"1min":0.82699999999999996,"5min":0.79200000000000004,"15min":1.1020000000000001},"min":{"1min":0.311,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":1.9059999999999999,"5min":6.8380000000000001,"15min":12.300000000000001},"last":0.42399999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:39:54 2023","interfaces":[{"interface":"back","average":{"1min":1.329,"5min":1.02,"15min":1.2589999999999999},"min":{"1min":0.32900000000000001,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":4.7679999999999998,"5min":6.9960000000000004,"15min":12.890000000000001},"last":0.41299999999999998},{"interface":"front","average":{"1min":1.1799999999999999,"5min":0.97499999999999998,"15min":1.2030000000000001},"min":{"1min":0.34000000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":4.7199999999999998,"5min":6.7930000000000001,"15min":16.265999999999998},"last":0.46500000000000002}]},{"osd":7,"last update":"Sat Sep 23 20:40:14 2023","interfaces":[{"interface":"back","average":{"1min":1.3939999999999999,"5min":0.84299999999999997,"15min":1.244},"min":{"1min":0.38400000000000001,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":5.2430000000000003,"5min":5.2430000000000003,"15min":36.621000000000002},"last":0.48599999999999999},{"interface":"front","average":{"1min":1.0189999999999999,"5min":0.90700000000000003,"15min":1.151},"min":{"1min":0.35699999999999998,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":4.3250000000000002,"5min":6.5970000000000004,"15min":37.331000000000003},"last":0.38700000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4722,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1360,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8206,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7156,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3154,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":383909888,"data_stored":383909888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":428085248,"data_stored":428081176,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":330108928,"data_stored":330104856,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":435879936,"data_stored":435879936,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":389021696,"data_stored":389021696,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1705508864,"data_stored":1705508864,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":332464128,"data_stored":332464128,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":267587584,"data_stored":267583512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:40:25.188 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:40:25.189 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:40:25.215 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:40:25.329 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:25.859 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:25 smithi018 ceph-mon[179647]: pgmap v1089: 129 pgs: 2 active+recovering+undersized+remapped, 2 activating, 6 remapped+peering, 1 peering, 4 active+remapped+backfill_wait, 4 active+undersized+remapped, 1 active+recovering, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped, 102 active+clean; 840 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 540 op/s; 9512/40203 objects degraded (23.660%); 1976/40203 objects misplaced (4.915%); 14 MiB/s, 305 objects/s recovering 2023-09-23T20:40:25.860 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:25 smithi018 ceph-mon[179647]: from='client.46154 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:40:25.862 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:25 smithi018 ceph-mon[184144]: pgmap v1089: 129 pgs: 2 active+recovering+undersized+remapped, 2 activating, 6 remapped+peering, 1 peering, 4 active+remapped+backfill_wait, 4 active+undersized+remapped, 1 active+recovering, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped, 102 active+clean; 840 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 540 op/s; 9512/40203 objects degraded (23.660%); 1976/40203 objects misplaced (4.915%); 14 MiB/s, 305 objects/s recovering 2023-09-23T20:40:25.862 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:25 smithi018 ceph-mon[184144]: from='client.46154 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:40:25.970 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:25 smithi149 ceph-mon[170394]: pgmap v1089: 129 pgs: 2 active+recovering+undersized+remapped, 2 activating, 6 remapped+peering, 1 peering, 4 active+remapped+backfill_wait, 4 active+undersized+remapped, 1 active+recovering, 6 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped, 102 active+clean; 840 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 540 op/s; 9512/40203 objects degraded (23.660%); 1976/40203 objects misplaced (4.915%); 14 MiB/s, 305 objects/s recovering 2023-09-23T20:40:25.970 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:25 smithi149 ceph-mon[170394]: from='client.46154 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:40:26.123 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:40:26.135 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:26.135 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:40:26.236 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:40:26.330 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:26.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:26 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 9512/40203 objects degraded (23.660%), 6 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T20:40:26.895 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:26 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 9512/40203 objects degraded (23.660%), 6 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T20:40:26.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:26 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 9512/40203 objects degraded (23.660%), 6 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T20:40:27.178 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2023-09-23T20:40:27.178 INFO:teuthology.orchestra.run.smithi018.stdout: "ops": [ 2023-09-23T20:40:27.178 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:27.178 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35994.0:168712 8.b 8:d8b29d00:::benchmark_data_smithi018_341490_object79389:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e444)", 2023-09-23T20:40:27.178 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:40:26.329256+0000", 2023-09-23T20:40:27.178 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.0016149059999999999, 2023-09-23T20:40:27.179 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0016480240000000001, 2023-09-23T20:40:27.179 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:40:27.179 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:40:27.179 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:40:27.179 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35994", 2023-09-23T20:40:27.179 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:40:27.179 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 168712 2023-09-23T20:40:27.179 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:27.179 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:40:27.179 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:27.179 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:40:27.180 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:40:26.329256+0000", 2023-09-23T20:40:27.180 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:40:27.180 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:27.180 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:27.180 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:40:27.180 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:40:26.329256+0000", 2023-09-23T20:40:27.180 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:40:27.180 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:27.180 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:27.180 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:40:27.180 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:40:26.329254+0000", 2023-09-23T20:40:27.180 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:40:27.181 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:27.181 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:27.181 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:40:27.181 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:40:26.329261+0000", 2023-09-23T20:40:27.181 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.0879999999999997e-06 2023-09-23T20:40:27.181 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:27.181 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:27.181 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:40:27.181 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:40:26.329274+0000", 2023-09-23T20:40:27.181 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3174e-05 2023-09-23T20:40:27.181 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:27.181 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:27.182 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:40:27.182 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:40:26.329282+0000", 2023-09-23T20:40:27.182 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.8099999999999998e-06 2023-09-23T20:40:27.182 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:27.182 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:27.182 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:40:27.182 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:40:26.329648+0000", 2023-09-23T20:40:27.182 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000366564 2023-09-23T20:40:27.182 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:27.182 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:27.183 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:40:27.183 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:40:26.329775+0000", 2023-09-23T20:40:27.183 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00012645000000000001 2023-09-23T20:40:27.183 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:27.183 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:27.183 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,4", 2023-09-23T20:40:27.183 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:40:26.329875+0000", 2023-09-23T20:40:27.183 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00010045800000000001 2023-09-23T20:40:27.183 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:40:27.183 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:40:27.183 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:40:27.183 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:27.184 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:27.184 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35994.0:168703 8.b 8:da88d32a:::benchmark_data_smithi018_341490_object79380:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e444)", 2023-09-23T20:40:27.184 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:40:26.318371+0000", 2023-09-23T20:40:27.184 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.012499227999999999, 2023-09-23T20:40:27.184 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.012623265999999999, 2023-09-23T20:40:27.184 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:40:27.184 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:40:27.184 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:40:27.184 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35994", 2023-09-23T20:40:27.184 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:40:27.184 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 168703 2023-09-23T20:40:27.184 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:27.185 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:40:27.185 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:27.185 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:40:27.185 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:40:26.318371+0000", 2023-09-23T20:40:27.185 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:40:27.185 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:27.185 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:27.185 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:40:27.185 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:40:26.318371+0000", 2023-09-23T20:40:27.185 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:40:27.185 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:27.185 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:27.186 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:40:27.186 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:40:26.318370+0000", 2023-09-23T20:40:27.186 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:40:27.186 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:27.186 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:27.186 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:40:27.186 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:40:26.318376+0000", 2023-09-23T20:40:27.186 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.8279999999999999e-06 2023-09-23T20:40:27.186 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:27.186 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:27.186 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:40:27.187 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:40:26.318392+0000", 2023-09-23T20:40:27.187 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.5998000000000002e-05 2023-09-23T20:40:27.187 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:27.187 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:27.187 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:40:27.187 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:40:26.318400+0000", 2023-09-23T20:40:27.187 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.7209999999999994e-06 2023-09-23T20:40:27.187 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:27.187 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:27.187 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:40:27.187 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:40:26.319232+0000", 2023-09-23T20:40:27.187 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00083252499999999998 2023-09-23T20:40:27.188 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:27.188 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:27.188 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:40:27.188 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:40:26.319405+0000", 2023-09-23T20:40:27.188 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00017264 2023-09-23T20:40:27.188 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:27.188 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:27.188 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,4", 2023-09-23T20:40:27.188 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:40:26.319544+0000", 2023-09-23T20:40:27.188 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000139209 2023-09-23T20:40:27.188 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:27.188 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:27.189 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:40:27.189 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:40:26.329591+0000", 2023-09-23T20:40:27.189 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.010046738 2023-09-23T20:40:27.189 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:27.189 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:27.189 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:40:27.189 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:40:26.329622+0000", 2023-09-23T20:40:27.189 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.1538999999999997e-05 2023-09-23T20:40:27.189 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:40:27.189 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:40:27.189 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:40:27.189 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:40:27.190 INFO:teuthology.orchestra.run.smithi018.stdout: ], 2023-09-23T20:40:27.190 INFO:teuthology.orchestra.run.smithi018.stdout: "num_ops": 2 2023-09-23T20:40:27.190 INFO:teuthology.orchestra.run.smithi018.stdout:} 2023-09-23T20:40:27.268 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:40:27.277 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:27.278 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:40:27.378 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:40:27.467 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:27.636 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:40:27.642 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:27.643 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:40:27.744 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:40:27.808 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:27 smithi018 ceph-mon[184144]: pgmap v1090: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 peering, 5 active+remapped+backfill_wait, 1 active+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 767 MiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 481 op/s; 15618/36702 objects degraded (42.554%); 2619/36702 objects misplaced (7.136%); 6.7 MiB/s, 176 objects/s recovering 2023-09-23T20:40:27.808 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:27 smithi018 ceph-mon[184144]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T20:40:27.809 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:27 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:40:27.809 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:27 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:40:27.810 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:27 smithi018 ceph-mon[179647]: pgmap v1090: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 peering, 5 active+remapped+backfill_wait, 1 active+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 767 MiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 481 op/s; 15618/36702 objects degraded (42.554%); 2619/36702 objects misplaced (7.136%); 6.7 MiB/s, 176 objects/s recovering 2023-09-23T20:40:27.811 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:27 smithi018 ceph-mon[179647]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T20:40:27.811 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:27 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:40:27.811 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:27 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:40:27.815 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:27.918 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T20:40:27.960 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:40:27.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:27 smithi149 ceph-mon[170394]: pgmap v1090: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 peering, 5 active+remapped+backfill_wait, 1 active+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 767 MiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 481 op/s; 15618/36702 objects degraded (42.554%); 2619/36702 objects misplaced (7.136%); 6.7 MiB/s, 176 objects/s recovering 2023-09-23T20:40:27.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:27 smithi149 ceph-mon[170394]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T20:40:27.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:27 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:40:27.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:27 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:40:27.969 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:40:27.969 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:27.970 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:40:28.070 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:40:28.155 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:28.258 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:40:28.269 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:28.269 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:40:28.370 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:40:28.415 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:28.483 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:40:28.488 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:28.488 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:40:28.489 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:40:28.590 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:40:28.674 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:29.156 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:40:29.164 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:29.165 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:40:29.265 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:40:29.343 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:29.534 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:40:29.544 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:29.544 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:40:29.645 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:40:29.705 INFO:teuthology.orchestra.run.smithi018.stderr:nodeep-scrub is unset 2023-09-23T20:40:29.730 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:29 smithi018 ceph-mon[184144]: pgmap v1091: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 676 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 574 op/s; 15618/32334 objects degraded (48.302%); 2207/32334 objects misplaced (6.826%); 6.5 MiB/s, 177 objects/s recovering 2023-09-23T20:40:29.730 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:29 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2826799700' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:40:29.730 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:29 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:40:29.730 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:29 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:40:29.731 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:29 smithi018 ceph-mon[179647]: pgmap v1091: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 676 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 574 op/s; 15618/32334 objects degraded (48.302%); 2207/32334 objects misplaced (6.826%); 6.5 MiB/s, 177 objects/s recovering 2023-09-23T20:40:29.731 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:29 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2826799700' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:40:29.731 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:29 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:40:29.731 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:29 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:40:29.739 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:29.750 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:40:29.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:29 smithi149 ceph-mon[170394]: pgmap v1091: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 676 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 574 op/s; 15618/32334 objects degraded (48.302%); 2207/32334 objects misplaced (6.826%); 6.5 MiB/s, 177 objects/s recovering 2023-09-23T20:40:29.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:29 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2826799700' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:40:29.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:29 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:40:29.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:29 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:40:30.262 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:40:30.273 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:30.274 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:40:30.375 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:40:30.470 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:30.761 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:40:30.770 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:30.771 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:40:30.835 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:30 smithi018 ceph-mon[184144]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:40:30.835 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:30 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:40:30.836 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:30 smithi018 ceph-mon[184144]: osdmap e445: 8 total, 8 up, 8 in 2023-09-23T20:40:30.836 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:30 smithi018 ceph-mon[184144]: failed to encode map e445 with expected crc 2023-09-23T20:40:30.836 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:30 smithi018 ceph-mon[179647]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:40:30.836 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:30 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:40:30.836 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:30 smithi018 ceph-mon[179647]: osdmap e445: 8 total, 8 up, 8 in 2023-09-23T20:40:30.836 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:30 smithi018 ceph-mon[179647]: failed to encode map e445 with expected crc 2023-09-23T20:40:30.872 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:40:30.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:30 smithi149 ceph-mon[170394]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:40:30.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:30 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:40:30.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:30 smithi149 ceph-mon[170394]: osdmap e445: 8 total, 8 up, 8 in 2023-09-23T20:40:30.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:30 smithi149 ceph-mon[170394]: failed to encode map e445 with expected crc 2023-09-23T20:40:30.993 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:31.056 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:40:31.134 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:40:31.138 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:40:31.149 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:31.150 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:40:31.184 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:40:31.195 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:40:31.206 INFO:teuthology.orchestra.run.smithi018.stderr:dumped all 2023-09-23T20:40:31.233 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:40:31.251 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:40:31.283 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:40:31.289 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:31.322 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:40:31.326 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:31.326 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:40:31.328 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:40:31.368 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:40:31.405 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:40:31.427 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:40:31.427 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:40:31.428 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:40:31.428 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:40:31.428 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:40:31.428 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:40:31.428 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:40:31.428 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:40:31.428 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:40:31.506 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:31.599 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:40:31.605 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:31.605 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:40:31.681 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:31 smithi149 ceph-mon[170394]: pgmap v1093: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 557 MiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 677 op/s; 15618/26604 objects degraded (58.705%); 1614/26604 objects misplaced (6.067%); 3.4 MiB/s, 95 objects/s recovering 2023-09-23T20:40:31.681 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:31 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/788450659' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:40:31.681 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:31 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 15618/26604 objects degraded (58.705%), 10 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T20:40:31.706 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:40:31.779 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:31 smithi018 ceph-mon[184144]: pgmap v1093: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 557 MiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 677 op/s; 15618/26604 objects degraded (58.705%); 1614/26604 objects misplaced (6.067%); 3.4 MiB/s, 95 objects/s recovering 2023-09-23T20:40:31.779 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:31 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/788450659' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:40:31.779 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:31 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 15618/26604 objects degraded (58.705%), 10 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T20:40:31.782 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:31 smithi018 ceph-mon[179647]: pgmap v1093: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 557 MiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 677 op/s; 15618/26604 objects degraded (58.705%); 1614/26604 objects misplaced (6.067%); 3.4 MiB/s, 95 objects/s recovering 2023-09-23T20:40:31.782 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:31 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/788450659' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:40:31.782 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:31 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 15618/26604 objects degraded (58.705%), 10 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T20:40:31.801 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:32.228 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:40:32.231 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:32.232 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:40:32.333 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:40:32.393 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:32.420 INFO:teuthology.orchestra.run.smithi018.stdout:{"pg_ready":true,"pg_map":{"version":1093,"stamp":"2023-09-23T20:40:30.082855+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":583889775,"num_objects":8868,"num_object_clones":0,"num_object_copies":26604,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":15618,"num_objects_misplaced":1614,"num_objects_unfound":0,"num_objects_dirty":8868,"num_whiteouts":0,"num_read":6139,"num_read_kb":13491,"num_write":170784,"num_write_kb":5729974,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33735,"num_bytes_recovered":2070405506,"num_keys_recovered":40,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":177671,"ondisk_log_size":177671,"up":387,"acting":377,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":423,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":7532068,"kb_used_data":3469476,"kb_used_omap":32,"kb_used_meta":4062175,"kb_avail":742429148,"statfs":{"total":767960285184,"available":760247447552,"internally_reserved":0,"allocated":3552743424,"data_stored":3537925474,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32773,"internal_metadata":4159668219},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,6,5,4,6,3,3],"upper_bound":128},"perf_stat":{"commit_latency_ms":11,"apply_latency_ms":11,"commit_latency_ns":11000000,"apply_latency_ns":11000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-355205120,"num_objects":-5420,"num_object_clones":0,"num_object_copies":-16260,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":11238,"num_objects_misplaced":484,"num_objects_unfound":0,"num_objects_dirty":-5420,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5420,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":763,"num_bytes_recovered":28901446,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.003134"},"pg_stats":[{"pgid":"8.11","version":"444'8671","reported_seq":8766,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-09-23T20:40:29.383102+0000","last_change":"2023-09-23T20:38:27.478425+0000","last_active":"2023-09-23T20:40:29.383102+0000","last_peered":"2023-09-23T20:40:29.383102+0000","last_clean":"2023-09-23T20:40:29.383102+0000","last_became_active":"2023-09-23T20:38:27.472168+0000","last_became_peered":"2023-09-23T20:38:27.472168+0000","last_unstale":"2023-09-23T20:40:29.383102+0000","last_undegraded":"2023-09-23T20:40:29.383102+0000","last_fullsized":"2023-09-23T20:40:29.383102+0000","mapping_epoch":410,"log_start":"373'1694","ondisk_log_start":"373'1694","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":6977,"log_dups_size":0,"ondisk_log_size":6977,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":14516224,"num_objects":222,"num_object_clones":0,"num_object_copies":666,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":222,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5362,"num_write_kb":178656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.18","version":"90'161","reported_seq":433,"reported_epoch":443,"state":"active+recovery_wait+remapped","last_fresh":"2023-09-23T20:40:21.393481+0000","last_change":"2023-09-23T20:40:20.543976+0000","last_active":"2023-09-23T20:40:21.393481+0000","last_peered":"2023-09-23T20:40:21.393481+0000","last_clean":"2023-09-23T20:15:59.452160+0000","last_became_active":"2023-09-23T20:40:20.539716+0000","last_became_peered":"2023-09-23T20:40:20.539716+0000","last_unstale":"2023-09-23T20:40:21.393481+0000","last_undegraded":"2023-09-23T20:40:21.393481+0000","last_fullsized":"2023-09-23T20:40:21.393481+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":405,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:35:03.551143+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:35:03.551143+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[1,2,7],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2,7","objects":-1}],"blocked_by":[],"up_primary":5,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1a","version":"69'1","reported_seq":141,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T20:40:21.415509+0000","last_change":"2023-09-23T20:32:33.233853+0000","last_active":"2023-09-23T20:40:21.415509+0000","last_peered":"2023-09-23T20:40:21.415509+0000","last_clean":"2023-09-23T20:40:21.415509+0000","last_became_active":"2023-09-23T20:32:33.233546+0000","last_became_peered":"2023-09-23T20:32:33.233546+0000","last_unstale":"2023-09-23T20:40:21.415509+0000","last_undegraded":"2023-09-23T20:40:21.415509+0000","last_fullsized":"2023-09-23T20:40:21.415509+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:50:27.130808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":257,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T20:40:21.403559+0000","last_change":"2023-09-23T20:37:43.579611+0000","last_active":"2023-09-23T20:40:21.403559+0000","last_peered":"2023-09-23T20:40:21.403559+0000","last_clean":"2023-09-23T20:40:21.403559+0000","last_became_active":"2023-09-23T20:37:43.579323+0000","last_became_peered":"2023-09-23T20:37:43.579323+0000","last_unstale":"2023-09-23T20:40:21.403559+0000","last_undegraded":"2023-09-23T20:40:21.403559+0000","last_fullsized":"2023-09-23T20:40:21.403559+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:17:32.311427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.12","version":"444'8671","reported_seq":8814,"reported_epoch":444,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:40:25.537301+0000","last_change":"2023-09-23T20:38:39.273131+0000","last_active":"2023-09-23T20:40:25.537301+0000","last_peered":"2023-09-23T20:40:25.537301+0000","last_clean":"2023-09-23T20:37:29.399979+0000","last_became_active":"2023-09-23T20:38:39.252214+0000","last_became_peered":"2023-09-23T20:38:39.252214+0000","last_unstale":"2023-09-23T20:40:25.537301+0000","last_undegraded":"2023-09-23T20:38:38.456024+0000","last_fullsized":"2023-09-23T20:38:38.449466+0000","mapping_epoch":416,"log_start":"373'1996","ondisk_log_start":"373'1996","created":410,"last_epoch_clean":370,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":6675,"log_dups_size":0,"ondisk_log_size":6675,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19202048,"num_objects":293,"num_object_clones":0,"num_object_copies":879,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":715,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":293,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5321,"num_write_kb":179648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":753664,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0],"avail_no_missing":["4","0"],"object_location_counts":[{"shards":"0,3,4","objects":141},{"shards":"0,4","objects":152}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1b","version":"90'161","reported_seq":517,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-09-23T20:40:22.445059+0000","last_change":"2023-09-23T20:40:22.445059+0000","last_active":"2023-09-23T20:40:22.445059+0000","last_peered":"2023-09-23T20:40:22.445059+0000","last_clean":"2023-09-23T20:40:22.445059+0000","last_became_active":"2023-09-23T20:40:22.444637+0000","last_became_peered":"2023-09-23T20:40:22.444637+0000","last_unstale":"2023-09-23T20:40:22.445059+0000","last_undegraded":"2023-09-23T20:40:22.445059+0000","last_fullsized":"2023-09-23T20:40:22.445059+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:05.442288+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:36:05.442288+0000","last_clean_scrub_stamp":"2023-09-23T20:36:05.442288+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0019500940000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":412,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T20:40:21.402763+0000","last_change":"2023-09-23T20:34:59.983035+0000","last_active":"2023-09-23T20:40:21.402763+0000","last_peered":"2023-09-23T20:40:21.402763+0000","last_clean":"2023-09-23T20:40:21.402763+0000","last_became_active":"2023-09-23T20:34:59.982459+0000","last_became_peered":"2023-09-23T20:34:59.982459+0000","last_unstale":"2023-09-23T20:40:21.402763+0000","last_undegraded":"2023-09-23T20:40:21.402763+0000","last_fullsized":"2023-09-23T20:40:21.402763+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:16:48.510783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":414,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T20:40:21.407524+0000","last_change":"2023-09-23T20:25:02.957185+0000","last_active":"2023-09-23T20:40:21.407524+0000","last_peered":"2023-09-23T20:40:21.407524+0000","last_clean":"2023-09-23T20:40:21.407524+0000","last_became_active":"2023-09-23T20:25:02.956689+0000","last_became_peered":"2023-09-23T20:25:02.956689+0000","last_unstale":"2023-09-23T20:40:21.407524+0000","last_undegraded":"2023-09-23T20:40:21.407524+0000","last_fullsized":"2023-09-23T20:40:21.407524+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:04:28.524015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.13","version":"444'8676","reported_seq":13569,"reported_epoch":444,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:40:29.479912+0000","last_change":"2023-09-23T20:40:20.783376+0000","last_active":"2023-09-23T20:40:29.479912+0000","last_peered":"2023-09-23T20:40:29.479912+0000","last_clean":"2023-09-23T20:37:13.297905+0000","last_became_active":"2023-09-23T20:40:20.744428+0000","last_became_peered":"2023-09-23T20:40:20.744428+0000","last_unstale":"2023-09-23T20:40:29.479912+0000","last_undegraded":"2023-09-23T20:40:20.622760+0000","last_fullsized":"2023-09-23T20:40:20.514696+0000","mapping_epoch":441,"log_start":"373'1680","ondisk_log_start":"373'1680","created":410,"last_epoch_clean":370,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":6996,"log_dups_size":0,"ondisk_log_size":6996,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":14647296,"num_objects":224,"num_object_clones":0,"num_object_copies":672,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2851,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":224,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5370,"num_write_kb":178976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2029,"num_bytes_recovered":132939776,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,6],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1,2,3","objects":163},{"shards":"1,2,6","objects":7},{"shards":"1,6","objects":54}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1a","version":"90'161","reported_seq":523,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T20:40:21.414969+0000","last_change":"2023-09-23T20:36:01.292103+0000","last_active":"2023-09-23T20:40:21.414969+0000","last_peered":"2023-09-23T20:40:21.414969+0000","last_clean":"2023-09-23T20:40:21.414969+0000","last_became_active":"2023-09-23T20:35:26.639912+0000","last_became_peered":"2023-09-23T20:35:26.639912+0000","last_unstale":"2023-09-23T20:40:21.414969+0000","last_undegraded":"2023-09-23T20:40:21.414969+0000","last_fullsized":"2023-09-23T20:40:21.414969+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:01.292032+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:01.292032+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:04:53.926640+0000","scrub_duration":0.0032983159999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":237,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T20:40:21.405091+0000","last_change":"2023-09-23T20:37:43.584414+0000","last_active":"2023-09-23T20:40:21.405091+0000","last_peered":"2023-09-23T20:40:21.405091+0000","last_clean":"2023-09-23T20:40:21.405091+0000","last_became_active":"2023-09-23T20:37:43.584116+0000","last_became_peered":"2023-09-23T20:37:43.584116+0000","last_unstale":"2023-09-23T20:40:21.405091+0000","last_undegraded":"2023-09-23T20:40:21.405091+0000","last_fullsized":"2023-09-23T20:40:21.405091+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:17:54.258172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":60,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T20:40:21.407590+0000","last_change":"2023-09-23T20:38:37.457427+0000","last_active":"2023-09-23T20:40:21.407590+0000","last_peered":"2023-09-23T20:40:21.407590+0000","last_clean":"2023-09-23T20:40:21.407590+0000","last_became_active":"2023-09-23T20:38:37.457099+0000","last_became_peered":"2023-09-23T20:38:37.457099+0000","last_unstale":"2023-09-23T20:40:21.407590+0000","last_undegraded":"2023-09-23T20:40:21.407590+0000","last_fullsized":"2023-09-23T20:40:21.407590+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:31:27.084994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"90'161","reported_seq":656,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-09-23T20:40:22.449754+0000","last_change":"2023-09-23T20:40:22.449754+0000","last_active":"2023-09-23T20:40:22.449754+0000","last_peered":"2023-09-23T20:40:22.449754+0000","last_clean":"2023-09-23T20:40:22.449754+0000","last_became_active":"2023-09-23T20:40:22.448838+0000","last_became_peered":"2023-09-23T20:40:22.448838+0000","last_unstale":"2023-09-23T20:40:22.449754+0000","last_undegraded":"2023-09-23T20:40:22.449754+0000","last_fullsized":"2023-09-23T20:40:22.449754+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:03.411025+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:03.411025+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:22:07.948856+0000","scrub_duration":0.001802922,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":447,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T20:40:21.406470+0000","last_change":"2023-09-23T20:40:19.538844+0000","last_active":"2023-09-23T20:40:21.406470+0000","last_peered":"2023-09-23T20:40:21.406470+0000","last_clean":"2023-09-23T20:40:21.406470+0000","last_became_active":"2023-09-23T20:40:19.538434+0000","last_became_peered":"2023-09-23T20:40:19.538434+0000","last_unstale":"2023-09-23T20:40:21.406470+0000","last_undegraded":"2023-09-23T20:40:21.406470+0000","last_fullsized":"2023-09-23T20:40:21.406470+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:53:59.418427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":167,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T20:40:21.390764+0000","last_change":"2023-09-23T20:38:02.579343+0000","last_active":"2023-09-23T20:40:21.390764+0000","last_peered":"2023-09-23T20:40:21.390764+0000","last_clean":"2023-09-23T20:40:21.390764+0000","last_became_active":"2023-09-23T20:38:02.579047+0000","last_became_peered":"2023-09-23T20:38:02.579047+0000","last_unstale":"2023-09-23T20:40:21.390764+0000","last_undegraded":"2023-09-23T20:40:21.390764+0000","last_fullsized":"2023-09-23T20:40:21.390764+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:44:03.166556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.14","version":"90'161","reported_seq":773,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T20:40:21.394614+0000","last_change":"2023-09-23T20:35:02.236398+0000","last_active":"2023-09-23T20:40:21.394614+0000","last_peered":"2023-09-23T20:40:21.394614+0000","last_clean":"2023-09-23T20:40:21.394614+0000","last_became_active":"2023-09-23T20:35:02.235988+0000","last_became_peered":"2023-09-23T20:35:02.235988+0000","last_unstale":"2023-09-23T20:40:21.394614+0000","last_undegraded":"2023-09-23T20:40:21.394614+0000","last_fullsized":"2023-09-23T20:40:21.394614+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:47.479012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:47.479012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:13.519648+0000","scrub_duration":0.0020136590000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":13,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T20:40:21.395277+0000","last_change":"2023-09-23T20:40:19.545472+0000","last_active":"2023-09-23T20:40:21.395277+0000","last_peered":"2023-09-23T20:40:21.395277+0000","last_clean":"2023-09-23T20:40:21.395277+0000","last_became_active":"2023-09-23T20:40:19.543853+0000","last_became_peered":"2023-09-23T20:40:19.543853+0000","last_unstale":"2023-09-23T20:40:21.395277+0000","last_undegraded":"2023-09-23T20:40:21.395277+0000","last_fullsized":"2023-09-23T20:40:21.395277+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:08.014744+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":268,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T20:40:21.403947+0000","last_change":"2023-09-23T20:38:02.542034+0000","last_active":"2023-09-23T20:40:21.403947+0000","last_peered":"2023-09-23T20:40:21.403947+0000","last_clean":"2023-09-23T20:40:21.403947+0000","last_became_active":"2023-09-23T20:38:02.541575+0000","last_became_peered":"2023-09-23T20:38:02.541575+0000","last_unstale":"2023-09-23T20:40:21.403947+0000","last_undegraded":"2023-09-23T20:40:21.403947+0000","last_fullsized":"2023-09-23T20:40:21.403947+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:50:54.591529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"90'161","reported_seq":597,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-09-23T20:40:22.444898+0000","last_change":"2023-09-23T20:40:22.444898+0000","last_active":"2023-09-23T20:40:22.444898+0000","last_peered":"2023-09-23T20:40:22.444898+0000","last_clean":"2023-09-23T20:40:22.444898+0000","last_became_active":"2023-09-23T20:40:22.444543+0000","last_became_peered":"2023-09-23T20:40:22.444543+0000","last_unstale":"2023-09-23T20:40:22.444898+0000","last_undegraded":"2023-09-23T20:40:22.444898+0000","last_fullsized":"2023-09-23T20:40:22.444898+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:03.435480+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:03.435480+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":471,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T20:40:21.410175+0000","last_change":"2023-09-23T20:32:33.270617+0000","last_active":"2023-09-23T20:40:21.410175+0000","last_peered":"2023-09-23T20:40:21.410175+0000","last_clean":"2023-09-23T20:40:21.410175+0000","last_became_active":"2023-09-23T20:32:33.227902+0000","last_became_peered":"2023-09-23T20:32:33.227902+0000","last_unstale":"2023-09-23T20:40:21.410175+0000","last_undegraded":"2023-09-23T20:40:21.410175+0000","last_fullsized":"2023-09-23T20:40:21.410175+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:30:26.847831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":360,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":413,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T20:40:21.399314+0000","last_change":"2023-09-23T20:38:02.599068+0000","last_active":"2023-09-23T20:40:21.399314+0000","last_peered":"2023-09-23T20:40:21.399314+0000","last_clean":"2023-09-23T20:40:21.399314+0000","last_became_active":"2023-09-23T20:38:02.597602+0000","last_became_peered":"2023-09-23T20:38:02.597602+0000","last_unstale":"2023-09-23T20:40:21.399314+0000","last_undegraded":"2023-09-23T20:40:21.399314+0000","last_fullsized":"2023-09-23T20:40:21.399314+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:34:06.337029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"90'161","reported_seq":568,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T20:40:21.401391+0000","last_change":"2023-09-23T20:36:02.548387+0000","last_active":"2023-09-23T20:40:21.401391+0000","last_peered":"2023-09-23T20:40:21.401391+0000","last_clean":"2023-09-23T20:40:21.401391+0000","last_became_active":"2023-09-23T20:35:26.644787+0000","last_became_peered":"2023-09-23T20:35:26.644787+0000","last_unstale":"2023-09-23T20:40:21.401391+0000","last_undegraded":"2023-09-23T20:40:21.401391+0000","last_fullsized":"2023-09-23T20:40:21.401391+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:02.548285+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:02.548285+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:37:15.216553+0000","scrub_duration":0.0015514459999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":191,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T20:40:21.413855+0000","last_change":"2023-09-23T20:38:02.560646+0000","last_active":"2023-09-23T20:40:21.413855+0000","last_peered":"2023-09-23T20:40:21.413855+0000","last_clean":"2023-09-23T20:40:21.413855+0000","last_became_active":"2023-09-23T20:38:02.539814+0000","last_became_peered":"2023-09-23T20:38:02.539814+0000","last_unstale":"2023-09-23T20:40:21.413855+0000","last_undegraded":"2023-09-23T20:40:21.413855+0000","last_fullsized":"2023-09-23T20:40:21.413855+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:00:09.435023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":171,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T20:40:21.413044+0000","last_change":"2023-09-23T20:38:02.579222+0000","last_active":"2023-09-23T20:40:21.413044+0000","last_peered":"2023-09-23T20:40:21.413044+0000","last_clean":"2023-09-23T20:40:21.413044+0000","last_became_active":"2023-09-23T20:38:02.578813+0000","last_became_peered":"2023-09-23T20:38:02.578813+0000","last_unstale":"2023-09-23T20:40:21.413044+0000","last_undegraded":"2023-09-23T20:40:21.413044+0000","last_fullsized":"2023-09-23T20:40:21.413044+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:57:31.080027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.18","version":"444'8685","reported_seq":14625,"reported_epoch":444,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:40:28.983175+0000","last_change":"2023-09-23T20:40:20.742770+0000","last_active":"2023-09-23T20:40:28.983175+0000","last_peered":"2023-09-23T20:40:28.983175+0000","last_clean":"2023-09-23T20:37:13.320723+0000","last_became_active":"2023-09-23T20:40:20.713497+0000","last_became_peered":"2023-09-23T20:40:20.713497+0000","last_unstale":"2023-09-23T20:40:28.983175+0000","last_undegraded":"2023-09-23T20:40:20.533808+0000","last_fullsized":"2023-09-23T20:40:20.514147+0000","mapping_epoch":441,"log_start":"373'2334","ondisk_log_start":"373'2334","created":410,"last_epoch_clean":370,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":6351,"log_dups_size":0,"ondisk_log_size":6351,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":14254080,"num_objects":218,"num_object_clones":0,"num_object_copies":654,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2531,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":218,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5377,"num_write_kb":179008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2403,"num_bytes_recovered":157417472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"0,3,7","objects":179},{"shards":"3,7","objects":39}],"blocked_by":[],"up_primary":5,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.11","version":"90'161","reported_seq":710,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-09-23T20:40:22.427437+0000","last_change":"2023-09-23T20:40:22.427437+0000","last_active":"2023-09-23T20:40:22.427437+0000","last_peered":"2023-09-23T20:40:22.427437+0000","last_clean":"2023-09-23T20:40:22.427437+0000","last_became_active":"2023-09-23T20:40:22.427090+0000","last_became_peered":"2023-09-23T20:40:22.427090+0000","last_unstale":"2023-09-23T20:40:22.427437+0000","last_undegraded":"2023-09-23T20:40:22.427437+0000","last_fullsized":"2023-09-23T20:40:22.427437+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:43.578666+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:43.578666+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:08:43.995672+0000","scrub_duration":0.0021890379999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":470,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T20:40:21.404800+0000","last_change":"2023-09-23T20:32:33.242943+0000","last_active":"2023-09-23T20:40:21.404800+0000","last_peered":"2023-09-23T20:40:21.404800+0000","last_clean":"2023-09-23T20:40:21.404800+0000","last_became_active":"2023-09-23T20:32:33.225468+0000","last_became_peered":"2023-09-23T20:32:33.225468+0000","last_unstale":"2023-09-23T20:40:21.404800+0000","last_undegraded":"2023-09-23T20:40:21.404800+0000","last_fullsized":"2023-09-23T20:40:21.404800+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:40:47.867573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":179,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T20:40:21.403546+0000","last_change":"2023-09-23T20:37:43.571836+0000","last_active":"2023-09-23T20:40:21.403546+0000","last_peered":"2023-09-23T20:40:21.403546+0000","last_clean":"2023-09-23T20:40:21.403546+0000","last_became_active":"2023-09-23T20:37:43.571511+0000","last_became_peered":"2023-09-23T20:37:43.571511+0000","last_unstale":"2023-09-23T20:40:21.403546+0000","last_undegraded":"2023-09-23T20:40:21.403546+0000","last_fullsized":"2023-09-23T20:40:21.403546+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:59:25.507917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.19","version":"444'8676","reported_seq":16397,"reported_epoch":444,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T20:40:28.076030+0000","last_change":"2023-09-23T20:40:20.669720+0000","last_active":"2023-09-23T20:40:28.076030+0000","last_peered":"2023-09-23T20:40:28.076030+0000","last_clean":"2023-09-23T20:40:18.450551+0000","last_became_active":"2023-09-23T20:40:20.635841+0000","last_became_peered":"2023-09-23T20:40:20.635841+0000","last_unstale":"2023-09-23T20:40:28.076030+0000","last_undegraded":"2023-09-23T20:40:28.076030+0000","last_fullsized":"2023-09-23T20:40:28.076030+0000","mapping_epoch":441,"log_start":"373'1316","ondisk_log_start":"373'1316","created":410,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":7360,"log_dups_size":0,"ondisk_log_size":7360,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17629184,"num_objects":269,"num_object_clones":0,"num_object_copies":807,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":269,"num_objects_unfound":0,"num_objects_dirty":269,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5362,"num_write_kb":180192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2990,"num_bytes_recovered":184942592,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,0,3],"avail_no_missing":["6","0","3","4"],"object_location_counts":[{"shards":"0,3,6","objects":269}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.10","version":"90'161","reported_seq":626,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-09-23T20:40:22.427706+0000","last_change":"2023-09-23T20:40:22.427706+0000","last_active":"2023-09-23T20:40:22.427706+0000","last_peered":"2023-09-23T20:40:22.427706+0000","last_clean":"2023-09-23T20:40:22.427706+0000","last_became_active":"2023-09-23T20:40:22.427114+0000","last_became_peered":"2023-09-23T20:40:22.427114+0000","last_unstale":"2023-09-23T20:40:22.427706+0000","last_undegraded":"2023-09-23T20:40:22.427706+0000","last_fullsized":"2023-09-23T20:40:22.427706+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:49.726139+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:49.726139+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0096333110000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":411,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T20:40:21.406449+0000","last_change":"2023-09-23T20:25:02.953993+0000","last_active":"2023-09-23T20:40:21.406449+0000","last_peered":"2023-09-23T20:40:21.406449+0000","last_clean":"2023-09-23T20:40:21.406449+0000","last_became_active":"2023-09-23T20:25:02.953672+0000","last_became_peered":"2023-09-23T20:25:02.953672+0000","last_unstale":"2023-09-23T20:40:21.406449+0000","last_undegraded":"2023-09-23T20:40:21.406449+0000","last_fullsized":"2023-09-23T20:40:21.406449+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:26:08.603577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":13,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T20:40:21.396325+0000","last_change":"2023-09-23T20:40:19.561512+0000","last_active":"2023-09-23T20:40:21.396325+0000","last_peered":"2023-09-23T20:40:21.396325+0000","last_clean":"2023-09-23T20:40:21.396325+0000","last_became_active":"2023-09-23T20:40:19.561123+0000","last_became_peered":"2023-09-23T20:40:19.561123+0000","last_unstale":"2023-09-23T20:40:21.396325+0000","last_undegraded":"2023-09-23T20:40:21.396325+0000","last_fullsized":"2023-09-23T20:40:21.396325+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:50:02.093978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"90'161","reported_seq":729,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T20:40:21.399148+0000","last_change":"2023-09-23T20:35:02.238923+0000","last_active":"2023-09-23T20:40:21.399148+0000","last_peered":"2023-09-23T20:40:21.399148+0000","last_clean":"2023-09-23T20:40:21.399148+0000","last_became_active":"2023-09-23T20:35:02.238293+0000","last_became_peered":"2023-09-23T20:35:02.238293+0000","last_unstale":"2023-09-23T20:40:21.399148+0000","last_undegraded":"2023-09-23T20:40:21.399148+0000","last_fullsized":"2023-09-23T20:40:21.399148+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:42.544308+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:42.544308+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:51:22.281932+0000","scrub_duration":0.0029958179999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":447,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T20:40:21.404629+0000","last_change":"2023-09-23T20:38:02.539774+0000","last_active":"2023-09-23T20:40:21.404629+0000","last_peered":"2023-09-23T20:40:21.404629+0000","last_clean":"2023-09-23T20:40:21.404629+0000","last_became_active":"2023-09-23T20:38:02.539436+0000","last_became_peered":"2023-09-23T20:38:02.539436+0000","last_unstale":"2023-09-23T20:40:21.404629+0000","last_undegraded":"2023-09-23T20:40:21.404629+0000","last_fullsized":"2023-09-23T20:40:21.404629+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:50:57.375803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":421,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T20:40:21.407569+0000","last_change":"2023-09-23T20:25:02.957010+0000","last_active":"2023-09-23T20:40:21.407569+0000","last_peered":"2023-09-23T20:40:21.407569+0000","last_clean":"2023-09-23T20:40:21.407569+0000","last_became_active":"2023-09-23T20:25:02.956267+0000","last_became_peered":"2023-09-23T20:25:02.956267+0000","last_unstale":"2023-09-23T20:40:21.407569+0000","last_undegraded":"2023-09-23T20:40:21.407569+0000","last_fullsized":"2023-09-23T20:40:21.407569+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:10:55.299422+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"90'161","reported_seq":549,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-09-23T20:40:22.431074+0000","last_change":"2023-09-23T20:40:22.431074+0000","last_active":"2023-09-23T20:40:22.431074+0000","last_peered":"2023-09-23T20:40:22.431074+0000","last_clean":"2023-09-23T20:40:22.431074+0000","last_became_active":"2023-09-23T20:40:22.430717+0000","last_became_peered":"2023-09-23T20:40:22.430717+0000","last_unstale":"2023-09-23T20:40:22.431074+0000","last_undegraded":"2023-09-23T20:40:22.431074+0000","last_fullsized":"2023-09-23T20:40:22.431074+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:53.706934+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:53.706934+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":80,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T20:40:21.405935+0000","last_change":"2023-09-23T20:40:19.564378+0000","last_active":"2023-09-23T20:40:21.405935+0000","last_peered":"2023-09-23T20:40:21.405935+0000","last_clean":"2023-09-23T20:40:21.405935+0000","last_became_active":"2023-09-23T20:40:19.532991+0000","last_became_peered":"2023-09-23T20:40:19.532991+0000","last_unstale":"2023-09-23T20:40:21.405935+0000","last_undegraded":"2023-09-23T20:40:21.405935+0000","last_fullsized":"2023-09-23T20:40:21.405935+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:22:42.832115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":237,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T20:40:21.403510+0000","last_change":"2023-09-23T20:37:43.571083+0000","last_active":"2023-09-23T20:40:21.403510+0000","last_peered":"2023-09-23T20:40:21.403510+0000","last_clean":"2023-09-23T20:40:21.403510+0000","last_became_active":"2023-09-23T20:37:43.570306+0000","last_became_peered":"2023-09-23T20:37:43.570306+0000","last_unstale":"2023-09-23T20:40:21.403510+0000","last_undegraded":"2023-09-23T20:40:21.403510+0000","last_fullsized":"2023-09-23T20:40:21.403510+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:42:02.396718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.4","version":"444'8781","reported_seq":10925,"reported_epoch":444,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:40:25.517046+0000","last_change":"2023-09-23T20:40:19.540505+0000","last_active":"2023-09-23T20:40:25.517046+0000","last_peered":"2023-09-23T20:40:25.517046+0000","last_clean":"2023-09-23T20:37:15.925522+0000","last_became_active":"2023-09-23T20:38:39.251413+0000","last_became_peered":"2023-09-23T20:38:39.251413+0000","last_unstale":"2023-09-23T20:40:25.517046+0000","last_undegraded":"2023-09-23T20:40:19.539602+0000","last_fullsized":"2023-09-23T20:38:38.449461+0000","mapping_epoch":416,"log_start":"373'2080","ondisk_log_start":"373'2080","created":369,"last_epoch_clean":370,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":6701,"log_dups_size":0,"ondisk_log_size":6701,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":20905984,"num_objects":319,"num_object_clones":0,"num_object_copies":957,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":503,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":319,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5374,"num_write_kb":182176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":645,"num_bytes_recovered":36306944,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[4,2],"avail_no_missing":["4","2","6"],"object_location_counts":[{"shards":"2,4","objects":319}],"blocked_by":[],"up_primary":5,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.d","version":"90'161","reported_seq":736,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T20:40:21.408250+0000","last_change":"2023-09-23T20:34:59.496438+0000","last_active":"2023-09-23T20:40:21.408250+0000","last_peered":"2023-09-23T20:40:21.408250+0000","last_clean":"2023-09-23T20:40:21.408250+0000","last_became_active":"2023-09-23T20:33:38.970527+0000","last_became_peered":"2023-09-23T20:33:38.970527+0000","last_unstale":"2023-09-23T20:40:21.408250+0000","last_undegraded":"2023-09-23T20:40:21.408250+0000","last_fullsized":"2023-09-23T20:40:21.408250+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:34:59.496375+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:34:59.496375+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:46:18.244734+0000","scrub_duration":0.0017843189999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":447,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T20:40:21.406809+0000","last_change":"2023-09-23T20:32:33.283752+0000","last_active":"2023-09-23T20:40:21.406809+0000","last_peered":"2023-09-23T20:40:21.406809+0000","last_clean":"2023-09-23T20:40:21.406809+0000","last_became_active":"2023-09-23T20:32:33.230503+0000","last_became_peered":"2023-09-23T20:32:33.230503+0000","last_unstale":"2023-09-23T20:40:21.406809+0000","last_undegraded":"2023-09-23T20:40:21.406809+0000","last_fullsized":"2023-09-23T20:40:21.406809+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:29.084410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":13,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T20:40:21.395899+0000","last_change":"2023-09-23T20:40:19.545668+0000","last_active":"2023-09-23T20:40:21.395899+0000","last_peered":"2023-09-23T20:40:21.395899+0000","last_clean":"2023-09-23T20:40:21.395899+0000","last_became_active":"2023-09-23T20:40:19.543857+0000","last_became_peered":"2023-09-23T20:40:19.543857+0000","last_unstale":"2023-09-23T20:40:21.395899+0000","last_undegraded":"2023-09-23T20:40:21.395899+0000","last_fullsized":"2023-09-23T20:40:21.395899+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:27:09.470605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"444'8602","reported_seq":12165,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-09-23T20:40:28.074572+0000","last_change":"2023-09-23T20:38:36.632834+0000","last_active":"2023-09-23T20:40:28.074572+0000","last_peered":"2023-09-23T20:40:28.074572+0000","last_clean":"2023-09-23T20:40:28.074572+0000","last_became_active":"2023-09-23T20:38:28.423549+0000","last_became_peered":"2023-09-23T20:38:28.423549+0000","last_unstale":"2023-09-23T20:40:28.074572+0000","last_undegraded":"2023-09-23T20:40:28.074572+0000","last_fullsized":"2023-09-23T20:40:28.074572+0000","mapping_epoch":411,"log_start":"373'1839","ondisk_log_start":"373'1839","created":369,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":6763,"log_dups_size":0,"ondisk_log_size":6763,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":16154624,"num_objects":246,"num_object_clones":0,"num_object_copies":738,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":246,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5306,"num_write_kb":177696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1036,"num_bytes_recovered":55345152,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.c","version":"90'161","reported_seq":691,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T20:40:21.403950+0000","last_change":"2023-09-23T20:38:28.442536+0000","last_active":"2023-09-23T20:40:21.403950+0000","last_peered":"2023-09-23T20:40:21.403950+0000","last_clean":"2023-09-23T20:40:21.403950+0000","last_became_active":"2023-09-23T20:38:28.442229+0000","last_became_peered":"2023-09-23T20:38:28.442229+0000","last_unstale":"2023-09-23T20:40:21.403950+0000","last_undegraded":"2023-09-23T20:40:21.403950+0000","last_fullsized":"2023-09-23T20:40:21.403950+0000","mapping_epoch":411,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":412,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:05.485751+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:05.485751+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.002180043,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163840,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":482,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T20:40:21.406219+0000","last_change":"2023-09-23T20:34:59.987981+0000","last_active":"2023-09-23T20:40:21.406219+0000","last_peered":"2023-09-23T20:40:21.406219+0000","last_clean":"2023-09-23T20:40:21.406219+0000","last_became_active":"2023-09-23T20:34:59.985368+0000","last_became_peered":"2023-09-23T20:34:59.985368+0000","last_unstale":"2023-09-23T20:40:21.406219+0000","last_undegraded":"2023-09-23T20:40:21.406219+0000","last_fullsized":"2023-09-23T20:40:21.406219+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:12:43.082493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":141,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T20:40:21.413525+0000","last_change":"2023-09-23T20:32:33.232142+0000","last_active":"2023-09-23T20:40:21.413525+0000","last_peered":"2023-09-23T20:40:21.413525+0000","last_clean":"2023-09-23T20:40:21.413525+0000","last_became_active":"2023-09-23T20:32:33.231683+0000","last_became_peered":"2023-09-23T20:32:33.231683+0000","last_unstale":"2023-09-23T20:40:21.413525+0000","last_undegraded":"2023-09-23T20:40:21.413525+0000","last_fullsized":"2023-09-23T20:40:21.413525+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:02:58.336348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"444'8587","reported_seq":9311,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-09-23T20:40:26.168809+0000","last_change":"2023-09-23T20:38:29.214109+0000","last_active":"2023-09-23T20:40:26.168809+0000","last_peered":"2023-09-23T20:40:26.168809+0000","last_clean":"2023-09-23T20:40:26.168809+0000","last_became_active":"2023-09-23T20:38:28.425504+0000","last_became_peered":"2023-09-23T20:38:28.425504+0000","last_unstale":"2023-09-23T20:40:26.168809+0000","last_undegraded":"2023-09-23T20:40:26.168809+0000","last_fullsized":"2023-09-23T20:40:26.168809+0000","mapping_epoch":411,"log_start":"373'1261","ondisk_log_start":"373'1261","created":369,"last_epoch_clean":412,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":7326,"log_dups_size":0,"ondisk_log_size":7326,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":16187392,"num_objects":247,"num_object_clones":0,"num_object_copies":741,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":247,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5307,"num_write_kb":177728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":305,"num_bytes_recovered":10420224,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"90'161","reported_seq":600,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T20:40:21.407411+0000","last_change":"2023-09-23T20:38:40.491902+0000","last_active":"2023-09-23T20:40:21.407411+0000","last_peered":"2023-09-23T20:40:21.407411+0000","last_clean":"2023-09-23T20:40:21.407411+0000","last_became_active":"2023-09-23T20:38:40.491563+0000","last_became_peered":"2023-09-23T20:38:40.491563+0000","last_unstale":"2023-09-23T20:40:21.407411+0000","last_undegraded":"2023-09-23T20:40:21.407411+0000","last_fullsized":"2023-09-23T20:40:21.407411+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:46.217489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:46.217489+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:39:25.008796+0000","scrub_duration":0.001456429,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":472,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T20:40:21.408333+0000","last_change":"2023-09-23T20:40:19.677655+0000","last_active":"2023-09-23T20:40:21.408333+0000","last_peered":"2023-09-23T20:40:21.408333+0000","last_clean":"2023-09-23T20:40:21.408333+0000","last_became_active":"2023-09-23T20:40:19.551189+0000","last_became_peered":"2023-09-23T20:40:19.551189+0000","last_unstale":"2023-09-23T20:40:21.408333+0000","last_undegraded":"2023-09-23T20:40:21.408333+0000","last_fullsized":"2023-09-23T20:40:21.408333+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:11:01.559576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":50,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T20:40:21.404701+0000","last_change":"2023-09-23T20:38:02.590613+0000","last_active":"2023-09-23T20:40:21.404701+0000","last_peered":"2023-09-23T20:40:21.404701+0000","last_clean":"2023-09-23T20:40:21.404701+0000","last_became_active":"2023-09-23T20:38:02.590284+0000","last_became_peered":"2023-09-23T20:38:02.590284+0000","last_unstale":"2023-09-23T20:40:21.404701+0000","last_undegraded":"2023-09-23T20:40:21.404701+0000","last_fullsized":"2023-09-23T20:40:21.404701+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:40:07.825054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"444'8611","reported_seq":9347,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-09-23T20:40:29.534441+0000","last_change":"2023-09-23T20:40:20.140860+0000","last_active":"2023-09-23T20:40:29.534441+0000","last_peered":"2023-09-23T20:40:29.534441+0000","last_clean":"2023-09-23T20:40:29.534441+0000","last_became_active":"2023-09-23T20:38:28.436345+0000","last_became_peered":"2023-09-23T20:38:28.436345+0000","last_unstale":"2023-09-23T20:40:29.534441+0000","last_undegraded":"2023-09-23T20:40:29.534441+0000","last_fullsized":"2023-09-23T20:40:29.534441+0000","mapping_epoch":411,"log_start":"373'1658","ondisk_log_start":"373'1658","created":369,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":6953,"log_dups_size":0,"ondisk_log_size":6953,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":15466496,"num_objects":236,"num_object_clones":0,"num_object_copies":708,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":236,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5321,"num_write_kb":177824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":309,"num_bytes_recovered":9699328,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.e","version":"90'161","reported_seq":544,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T20:40:21.412884+0000","last_change":"2023-09-23T20:36:00.322453+0000","last_active":"2023-09-23T20:40:21.412884+0000","last_peered":"2023-09-23T20:40:21.412884+0000","last_clean":"2023-09-23T20:40:21.412884+0000","last_became_active":"2023-09-23T20:35:02.233536+0000","last_became_peered":"2023-09-23T20:35:02.233536+0000","last_unstale":"2023-09-23T20:40:21.412884+0000","last_undegraded":"2023-09-23T20:40:21.412884+0000","last_fullsized":"2023-09-23T20:40:21.412884+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:00.322381+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:00.322381+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:02:15.489069+0000","scrub_duration":0.0017321859999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":38,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T20:40:21.397362+0000","last_change":"2023-09-23T20:38:37.456683+0000","last_active":"2023-09-23T20:40:21.397362+0000","last_peered":"2023-09-23T20:40:21.397362+0000","last_clean":"2023-09-23T20:40:21.397362+0000","last_became_active":"2023-09-23T20:38:37.456289+0000","last_became_peered":"2023-09-23T20:38:37.456289+0000","last_unstale":"2023-09-23T20:40:21.397362+0000","last_undegraded":"2023-09-23T20:40:21.397362+0000","last_fullsized":"2023-09-23T20:40:21.397362+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:03:10.507333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":429,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T20:40:21.405603+0000","last_change":"2023-09-23T20:32:33.236122+0000","last_active":"2023-09-23T20:40:21.405603+0000","last_peered":"2023-09-23T20:40:21.405603+0000","last_clean":"2023-09-23T20:40:21.405603+0000","last_became_active":"2023-09-23T20:32:33.235384+0000","last_became_peered":"2023-09-23T20:32:33.235384+0000","last_unstale":"2023-09-23T20:40:21.405603+0000","last_undegraded":"2023-09-23T20:40:21.405603+0000","last_fullsized":"2023-09-23T20:40:21.405603+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:56.843733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"444'8451","reported_seq":14961,"reported_epoch":444,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T20:40:28.982747+0000","last_change":"2023-09-23T20:40:20.768066+0000","last_active":"2023-09-23T20:40:28.982747+0000","last_peered":"2023-09-23T20:40:28.982747+0000","last_clean":"2023-09-23T20:40:18.483118+0000","last_became_active":"2023-09-23T20:40:20.740650+0000","last_became_peered":"2023-09-23T20:40:20.740650+0000","last_unstale":"2023-09-23T20:40:28.982747+0000","last_undegraded":"2023-09-23T20:40:28.982747+0000","last_fullsized":"2023-09-23T20:40:28.982747+0000","mapping_epoch":441,"log_start":"373'2187","ondisk_log_start":"373'2187","created":369,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":6264,"log_dups_size":0,"ondisk_log_size":6264,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":16449536,"num_objects":251,"num_object_clones":0,"num_object_copies":753,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":251,"num_objects_unfound":0,"num_objects_dirty":251,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5204,"num_write_kb":174560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2565,"num_bytes_recovered":168099840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,1,4],"avail_no_missing":["3","1","4"],"object_location_counts":[{"shards":"1,3,4","objects":251}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":849,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T20:40:21.404480+0000","last_change":"2023-09-23T20:35:02.523308+0000","last_active":"2023-09-23T20:40:21.404480+0000","last_peered":"2023-09-23T20:40:21.404480+0000","last_clean":"2023-09-23T20:40:21.404480+0000","last_became_active":"2023-09-23T20:33:38.979989+0000","last_became_peered":"2023-09-23T20:33:38.979989+0000","last_unstale":"2023-09-23T20:40:21.404480+0000","last_undegraded":"2023-09-23T20:40:21.404480+0000","last_fullsized":"2023-09-23T20:40:21.404480+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:35:02.523203+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:35:02.523203+0000","last_clean_scrub_stamp":"2023-09-23T20:35:02.523203+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:02:41.472958+0000","scrub_duration":0.0085927989999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":263,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T20:40:21.404273+0000","last_change":"2023-09-23T20:37:43.583731+0000","last_active":"2023-09-23T20:40:21.404273+0000","last_peered":"2023-09-23T20:40:21.404273+0000","last_clean":"2023-09-23T20:40:21.404273+0000","last_became_active":"2023-09-23T20:37:43.583066+0000","last_became_peered":"2023-09-23T20:37:43.583066+0000","last_unstale":"2023-09-23T20:40:21.404273+0000","last_undegraded":"2023-09-23T20:40:21.404273+0000","last_fullsized":"2023-09-23T20:40:21.404273+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:03:45.211900+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":161,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T20:40:21.413776+0000","last_change":"2023-09-23T20:38:02.579814+0000","last_active":"2023-09-23T20:40:21.413776+0000","last_peered":"2023-09-23T20:40:21.413776+0000","last_clean":"2023-09-23T20:40:21.413776+0000","last_became_active":"2023-09-23T20:38:02.579253+0000","last_became_peered":"2023-09-23T20:38:02.579253+0000","last_unstale":"2023-09-23T20:40:21.413776+0000","last_undegraded":"2023-09-23T20:40:21.413776+0000","last_fullsized":"2023-09-23T20:40:21.413776+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:05:07.298198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"444'8663","reported_seq":8760,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-09-23T20:40:29.377429+0000","last_change":"2023-09-23T20:40:20.514587+0000","last_active":"2023-09-23T20:40:29.377429+0000","last_peered":"2023-09-23T20:40:29.377429+0000","last_clean":"2023-09-23T20:40:29.377429+0000","last_became_active":"2023-09-23T20:38:27.437204+0000","last_became_peered":"2023-09-23T20:38:27.437204+0000","last_unstale":"2023-09-23T20:40:29.377429+0000","last_undegraded":"2023-09-23T20:40:29.377429+0000","last_fullsized":"2023-09-23T20:40:29.377429+0000","mapping_epoch":369,"log_start":"373'1694","ondisk_log_start":"373'1694","created":369,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":6969,"log_dups_size":0,"ondisk_log_size":6969,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":15040512,"num_objects":229,"num_object_clones":0,"num_object_copies":687,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":229,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5353,"num_write_kb":178656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":545,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-09-23T20:40:22.433408+0000","last_change":"2023-09-23T20:40:22.433408+0000","last_active":"2023-09-23T20:40:22.433408+0000","last_peered":"2023-09-23T20:40:22.433408+0000","last_clean":"2023-09-23T20:40:22.433408+0000","last_became_active":"2023-09-23T20:40:22.433023+0000","last_became_peered":"2023-09-23T20:40:22.433023+0000","last_unstale":"2023-09-23T20:40:22.433408+0000","last_undegraded":"2023-09-23T20:40:22.433408+0000","last_fullsized":"2023-09-23T20:40:22.433408+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:35:00.496197+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:35:00.496197+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":50,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T20:40:21.404780+0000","last_change":"2023-09-23T20:38:02.578703+0000","last_active":"2023-09-23T20:40:21.404780+0000","last_peered":"2023-09-23T20:40:21.404780+0000","last_clean":"2023-09-23T20:40:21.404780+0000","last_became_active":"2023-09-23T20:38:02.578414+0000","last_became_peered":"2023-09-23T20:38:02.578414+0000","last_unstale":"2023-09-23T20:40:21.404780+0000","last_undegraded":"2023-09-23T20:40:21.404780+0000","last_fullsized":"2023-09-23T20:40:21.404780+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:43:59.779295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":110,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T20:40:21.412074+0000","last_change":"2023-09-23T20:37:31.296606+0000","last_active":"2023-09-23T20:40:21.412074+0000","last_peered":"2023-09-23T20:40:21.412074+0000","last_clean":"2023-09-23T20:40:21.412074+0000","last_became_active":"2023-09-23T20:37:31.295924+0000","last_became_peered":"2023-09-23T20:37:31.295924+0000","last_unstale":"2023-09-23T20:40:21.412074+0000","last_undegraded":"2023-09-23T20:40:21.412074+0000","last_fullsized":"2023-09-23T20:40:21.412074+0000","mapping_epoch":391,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":392,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:27:50.948510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"444'8584","reported_seq":8736,"reported_epoch":444,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:40:25.602554+0000","last_change":"2023-09-23T20:38:39.273152+0000","last_active":"2023-09-23T20:40:25.602554+0000","last_peered":"2023-09-23T20:40:25.602554+0000","last_clean":"2023-09-23T20:37:29.399979+0000","last_became_active":"2023-09-23T20:38:39.253967+0000","last_became_peered":"2023-09-23T20:38:39.253967+0000","last_unstale":"2023-09-23T20:40:25.602554+0000","last_undegraded":"2023-09-23T20:38:38.450129+0000","last_fullsized":"2023-09-23T20:38:38.446971+0000","mapping_epoch":416,"log_start":"373'1996","ondisk_log_start":"373'1996","created":369,"last_epoch_clean":370,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":6588,"log_dups_size":0,"ondisk_log_size":6588,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":24903680,"num_objects":380,"num_object_clones":0,"num_object_copies":1140,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":723,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":380,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5234,"num_write_kb":179648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":950272,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0],"avail_no_missing":["4","0"],"object_location_counts":[{"shards":"0,3,4","objects":190},{"shards":"0,4","objects":190}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"90'161","reported_seq":745,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-09-23T20:40:22.445808+0000","last_change":"2023-09-23T20:40:22.445808+0000","last_active":"2023-09-23T20:40:22.445808+0000","last_peered":"2023-09-23T20:40:22.445808+0000","last_clean":"2023-09-23T20:40:22.445808+0000","last_became_active":"2023-09-23T20:40:22.445456+0000","last_became_peered":"2023-09-23T20:40:22.445456+0000","last_unstale":"2023-09-23T20:40:22.445808+0000","last_undegraded":"2023-09-23T20:40:22.445808+0000","last_fullsized":"2023-09-23T20:40:22.445808+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:05.595042+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:05.595042+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:39:23.678154+0000","scrub_duration":0.007271675,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":100,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T20:40:21.413217+0000","last_change":"2023-09-23T20:34:59.983122+0000","last_active":"2023-09-23T20:40:21.413217+0000","last_peered":"2023-09-23T20:40:21.413217+0000","last_clean":"2023-09-23T20:40:21.413217+0000","last_became_active":"2023-09-23T20:34:59.982809+0000","last_became_peered":"2023-09-23T20:34:59.982809+0000","last_unstale":"2023-09-23T20:40:21.413217+0000","last_undegraded":"2023-09-23T20:40:21.413217+0000","last_fullsized":"2023-09-23T20:40:21.413217+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:49:40.839723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":161,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T20:40:21.412983+0000","last_change":"2023-09-23T20:38:02.543866+0000","last_active":"2023-09-23T20:40:21.412983+0000","last_peered":"2023-09-23T20:40:21.412983+0000","last_clean":"2023-09-23T20:40:21.412983+0000","last_became_active":"2023-09-23T20:38:02.543563+0000","last_became_peered":"2023-09-23T20:38:02.543563+0000","last_unstale":"2023-09-23T20:40:21.412983+0000","last_undegraded":"2023-09-23T20:40:21.412983+0000","last_fullsized":"2023-09-23T20:40:21.412983+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:33.050536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"444'8657","reported_seq":13542,"reported_epoch":444,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:40:29.496064+0000","last_change":"2023-09-23T20:40:20.781471+0000","last_active":"2023-09-23T20:40:29.496064+0000","last_peered":"2023-09-23T20:40:29.496064+0000","last_clean":"2023-09-23T20:37:13.297905+0000","last_became_active":"2023-09-23T20:40:20.744231+0000","last_became_peered":"2023-09-23T20:40:20.744231+0000","last_unstale":"2023-09-23T20:40:29.496064+0000","last_undegraded":"2023-09-23T20:40:20.609553+0000","last_fullsized":"2023-09-23T20:40:20.515058+0000","mapping_epoch":441,"log_start":"373'1680","ondisk_log_start":"373'1680","created":369,"last_epoch_clean":370,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":6977,"log_dups_size":0,"ondisk_log_size":6977,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":15892480,"num_objects":242,"num_object_clones":0,"num_object_copies":726,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2868,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":242,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5350,"num_write_kb":178976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2024,"num_bytes_recovered":132546560,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,6],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1,2,3","objects":156},{"shards":"1,2,6","objects":5},{"shards":"1,6","objects":81}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":548,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T20:40:21.399840+0000","last_change":"2023-09-23T20:40:01.179672+0000","last_active":"2023-09-23T20:40:21.399840+0000","last_peered":"2023-09-23T20:40:21.399840+0000","last_clean":"2023-09-23T20:40:21.399840+0000","last_became_active":"2023-09-23T20:38:40.494049+0000","last_became_peered":"2023-09-23T20:38:40.494049+0000","last_unstale":"2023-09-23T20:40:21.399840+0000","last_undegraded":"2023-09-23T20:40:21.399840+0000","last_fullsized":"2023-09-23T20:40:21.399840+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:34:59.481446+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:34:59.481446+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26834,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":237,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T20:40:21.404960+0000","last_change":"2023-09-23T20:37:43.570610+0000","last_active":"2023-09-23T20:40:21.404960+0000","last_peered":"2023-09-23T20:40:21.404960+0000","last_clean":"2023-09-23T20:40:21.404960+0000","last_became_active":"2023-09-23T20:37:43.570216+0000","last_became_peered":"2023-09-23T20:37:43.570216+0000","last_unstale":"2023-09-23T20:40:21.404960+0000","last_undegraded":"2023-09-23T20:40:21.404960+0000","last_fullsized":"2023-09-23T20:40:21.404960+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:39:26.555329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":267,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T20:40:21.400293+0000","last_change":"2023-09-23T20:32:33.230582+0000","last_active":"2023-09-23T20:40:21.400293+0000","last_peered":"2023-09-23T20:40:21.400293+0000","last_clean":"2023-09-23T20:40:21.400293+0000","last_became_active":"2023-09-23T20:32:33.230257+0000","last_became_peered":"2023-09-23T20:32:33.230257+0000","last_unstale":"2023-09-23T20:40:21.400293+0000","last_undegraded":"2023-09-23T20:40:21.400293+0000","last_fullsized":"2023-09-23T20:40:21.400293+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:11:45.333096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"444'10426","reported_seq":10511,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-09-23T20:40:26.201283+0000","last_change":"2023-09-23T20:36:11.310149+0000","last_active":"2023-09-23T20:40:26.201283+0000","last_peered":"2023-09-23T20:40:26.201283+0000","last_clean":"2023-09-23T20:40:26.201283+0000","last_became_active":"2023-09-23T20:36:11.309844+0000","last_became_peered":"2023-09-23T20:36:11.309844+0000","last_unstale":"2023-09-23T20:40:26.201283+0000","last_undegraded":"2023-09-23T20:40:26.201283+0000","last_fullsized":"2023-09-23T20:40:26.201283+0000","mapping_epoch":369,"log_start":"376'4219","ondisk_log_start":"376'4219","created":369,"last_epoch_clean":370,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":6207,"log_dups_size":0,"ondisk_log_size":6207,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:14:57.442108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":41287680,"num_objects":630,"num_object_clones":0,"num_object_copies":1890,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":630,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10426,"num_write_kb":353792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":623,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T20:40:21.407260+0000","last_change":"2023-09-23T20:38:05.622597+0000","last_active":"2023-09-23T20:40:21.407260+0000","last_peered":"2023-09-23T20:40:21.407260+0000","last_clean":"2023-09-23T20:40:21.407260+0000","last_became_active":"2023-09-23T20:38:05.622261+0000","last_became_peered":"2023-09-23T20:38:05.622261+0000","last_unstale":"2023-09-23T20:40:21.407260+0000","last_undegraded":"2023-09-23T20:40:21.407260+0000","last_fullsized":"2023-09-23T20:40:21.407260+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:35:04.101972+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:35:04.101972+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"86'119","reported_seq":752,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T20:40:21.401936+0000","last_change":"2023-09-23T20:29:45.521257+0000","last_active":"2023-09-23T20:40:21.401936+0000","last_peered":"2023-09-23T20:40:21.401936+0000","last_clean":"2023-09-23T20:40:21.401936+0000","last_became_active":"2023-09-23T20:29:45.520546+0000","last_became_peered":"2023-09-23T20:29:45.520546+0000","last_unstale":"2023-09-23T20:40:21.401936+0000","last_undegraded":"2023-09-23T20:40:21.401936+0000","last_fullsized":"2023-09-23T20:40:21.401936+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":119,"log_dups_size":0,"ondisk_log_size":119,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:58:56.085777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032705499999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":523,"num_read_kb":3503,"num_write":136,"num_write_kb":3128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":13,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T20:40:21.396242+0000","last_change":"2023-09-23T20:40:19.557009+0000","last_active":"2023-09-23T20:40:21.396242+0000","last_peered":"2023-09-23T20:40:21.396242+0000","last_clean":"2023-09-23T20:40:21.396242+0000","last_became_active":"2023-09-23T20:40:19.550473+0000","last_became_peered":"2023-09-23T20:40:19.550473+0000","last_unstale":"2023-09-23T20:40:21.396242+0000","last_undegraded":"2023-09-23T20:40:21.396242+0000","last_fullsized":"2023-09-23T20:40:21.396242+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:35:01.140258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":399,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T20:40:21.398124+0000","last_change":"2023-09-23T20:40:19.530942+0000","last_active":"2023-09-23T20:40:21.398124+0000","last_peered":"2023-09-23T20:40:21.398124+0000","last_clean":"2023-09-23T20:40:21.398124+0000","last_became_active":"2023-09-23T20:40:19.530617+0000","last_became_peered":"2023-09-23T20:40:19.530617+0000","last_unstale":"2023-09-23T20:40:21.398124+0000","last_undegraded":"2023-09-23T20:40:21.398124+0000","last_fullsized":"2023-09-23T20:40:21.398124+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:24:09.231934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.8","version":"444'8543","reported_seq":14859,"reported_epoch":444,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T20:40:26.204436+0000","last_change":"2023-09-23T20:40:20.738122+0000","last_active":"2023-09-23T20:40:26.204436+0000","last_peered":"2023-09-23T20:40:26.204436+0000","last_clean":"2023-09-23T20:40:18.482315+0000","last_became_active":"2023-09-23T20:40:20.708683+0000","last_became_peered":"2023-09-23T20:40:20.708683+0000","last_unstale":"2023-09-23T20:40:26.204436+0000","last_undegraded":"2023-09-23T20:40:26.204436+0000","last_fullsized":"2023-09-23T20:40:26.204436+0000","mapping_epoch":441,"log_start":"373'2334","ondisk_log_start":"373'2334","created":369,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":6209,"log_dups_size":0,"ondisk_log_size":6209,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":23560192,"num_objects":359,"num_object_clones":0,"num_object_copies":1077,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":359,"num_objects_unfound":0,"num_objects_dirty":359,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5234,"num_write_kb":179008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2588,"num_bytes_recovered":169607168,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[0,3,7],"avail_no_missing":["0","3","7"],"object_location_counts":[{"shards":"0,3,7","objects":359}],"blocked_by":[],"up_primary":5,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":517,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-09-23T20:40:22.445201+0000","last_change":"2023-09-23T20:40:22.445201+0000","last_active":"2023-09-23T20:40:22.445201+0000","last_peered":"2023-09-23T20:40:22.445201+0000","last_clean":"2023-09-23T20:40:22.445201+0000","last_became_active":"2023-09-23T20:40:22.444703+0000","last_became_peered":"2023-09-23T20:40:22.444703+0000","last_unstale":"2023-09-23T20:40:22.445201+0000","last_undegraded":"2023-09-23T20:40:22.445201+0000","last_fullsized":"2023-09-23T20:40:22.445201+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:45.785450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:45.785450+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"86'106","reported_seq":1163,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T20:40:21.400047+0000","last_change":"2023-09-23T20:38:37.476512+0000","last_active":"2023-09-23T20:40:21.400047+0000","last_peered":"2023-09-23T20:40:21.400047+0000","last_clean":"2023-09-23T20:40:21.400047+0000","last_became_active":"2023-09-23T20:38:37.475980+0000","last_became_peered":"2023-09-23T20:38:37.475980+0000","last_unstale":"2023-09-23T20:40:21.400047+0000","last_undegraded":"2023-09-23T20:40:21.400047+0000","last_fullsized":"2023-09-23T20:40:21.400047+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":106,"log_dups_size":0,"ondisk_log_size":106,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:58:45.508831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00099302599999999993,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1094,"num_read_kb":1678,"num_write":100,"num_write_kb":404,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":161,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T20:40:21.413206+0000","last_change":"2023-09-23T20:38:02.552008+0000","last_active":"2023-09-23T20:40:21.413206+0000","last_peered":"2023-09-23T20:40:21.413206+0000","last_clean":"2023-09-23T20:40:21.413206+0000","last_became_active":"2023-09-23T20:38:02.551684+0000","last_became_peered":"2023-09-23T20:38:02.551684+0000","last_unstale":"2023-09-23T20:40:21.413206+0000","last_undegraded":"2023-09-23T20:40:21.413206+0000","last_fullsized":"2023-09-23T20:40:21.413206+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:40:37.147538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":405,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T20:40:21.398568+0000","last_change":"2023-09-23T20:29:45.533406+0000","last_active":"2023-09-23T20:40:21.398568+0000","last_peered":"2023-09-23T20:40:21.398568+0000","last_clean":"2023-09-23T20:40:21.398568+0000","last_became_active":"2023-09-23T20:29:45.526001+0000","last_became_peered":"2023-09-23T20:29:45.526001+0000","last_unstale":"2023-09-23T20:40:21.398568+0000","last_undegraded":"2023-09-23T20:40:21.398568+0000","last_fullsized":"2023-09-23T20:40:21.398568+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:43.787251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.b","version":"444'10653","reported_seq":10752,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-09-23T20:40:28.959403+0000","last_change":"2023-09-23T20:38:01.517845+0000","last_active":"2023-09-23T20:40:28.959403+0000","last_peered":"2023-09-23T20:40:28.959403+0000","last_clean":"2023-09-23T20:40:28.959403+0000","last_became_active":"2023-09-23T20:37:43.579403+0000","last_became_peered":"2023-09-23T20:37:43.579403+0000","last_unstale":"2023-09-23T20:40:28.959403+0000","last_undegraded":"2023-09-23T20:40:28.959403+0000","last_fullsized":"2023-09-23T20:40:28.959403+0000","mapping_epoch":399,"log_start":"405'5900","ondisk_log_start":"405'5900","created":369,"last_epoch_clean":403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":4753,"log_dups_size":0,"ondisk_log_size":4753,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:33:13.920936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":31653888,"num_objects":483,"num_object_clones":0,"num_object_copies":1449,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":483,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10653,"num_write_kb":356352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":916,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T20:40:21.405912+0000","last_change":"2023-09-23T20:35:02.260708+0000","last_active":"2023-09-23T20:40:21.405912+0000","last_peered":"2023-09-23T20:40:21.405912+0000","last_clean":"2023-09-23T20:40:21.405912+0000","last_became_active":"2023-09-23T20:35:02.260348+0000","last_became_peered":"2023-09-23T20:35:02.260348+0000","last_unstale":"2023-09-23T20:40:21.405912+0000","last_undegraded":"2023-09-23T20:40:21.405912+0000","last_fullsized":"2023-09-23T20:40:21.405912+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:43.390908+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:43.390908+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:09:14.144070+0000","scrub_duration":0.0032721,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"86'130","reported_seq":760,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T20:40:21.393669+0000","last_change":"2023-09-23T20:25:39.637458+0000","last_active":"2023-09-23T20:40:21.393669+0000","last_peered":"2023-09-23T20:40:21.393669+0000","last_clean":"2023-09-23T20:40:21.393669+0000","last_became_active":"2023-09-23T20:25:39.636773+0000","last_became_peered":"2023-09-23T20:25:39.636773+0000","last_unstale":"2023-09-23T20:40:21.393669+0000","last_undegraded":"2023-09-23T20:40:21.393669+0000","last_fullsized":"2023-09-23T20:40:21.393669+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:00:32.810564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011300049999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":512,"num_read_kb":1820,"num_write":133,"num_write_kb":1347,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":413,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T20:40:21.393769+0000","last_change":"2023-09-23T20:38:02.591335+0000","last_active":"2023-09-23T20:40:21.393769+0000","last_peered":"2023-09-23T20:40:21.393769+0000","last_clean":"2023-09-23T20:40:21.393769+0000","last_became_active":"2023-09-23T20:38:02.591015+0000","last_became_peered":"2023-09-23T20:38:02.591015+0000","last_unstale":"2023-09-23T20:40:21.393769+0000","last_undegraded":"2023-09-23T20:40:21.393769+0000","last_fullsized":"2023-09-23T20:40:21.393769+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:55:32.272195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":404,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T20:40:21.400232+0000","last_change":"2023-09-23T20:40:19.544333+0000","last_active":"2023-09-23T20:40:21.400232+0000","last_peered":"2023-09-23T20:40:21.400232+0000","last_clean":"2023-09-23T20:40:21.400232+0000","last_became_active":"2023-09-23T20:40:19.543987+0000","last_became_peered":"2023-09-23T20:40:19.543987+0000","last_unstale":"2023-09-23T20:40:21.400232+0000","last_undegraded":"2023-09-23T20:40:21.400232+0000","last_fullsized":"2023-09-23T20:40:21.400232+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:12:59.203296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"444'8602","reported_seq":15898,"reported_epoch":444,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T20:40:26.235443+0000","last_change":"2023-09-23T20:40:26.235752+0000","last_active":"2023-09-23T20:40:26.235443+0000","last_peered":"2023-09-23T20:40:26.235443+0000","last_clean":"2023-09-23T20:37:13.347998+0000","last_became_active":"2023-09-23T20:40:20.702197+0000","last_became_peered":"2023-09-23T20:40:20.702197+0000","last_unstale":"2023-09-23T20:40:26.235443+0000","last_undegraded":"2023-09-23T20:40:26.235443+0000","last_fullsized":"2023-09-23T20:40:20.511184+0000","mapping_epoch":441,"log_start":"373'1316","ondisk_log_start":"373'1316","created":369,"last_epoch_clean":370,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":7286,"log_dups_size":0,"ondisk_log_size":7286,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22478848,"num_objects":343,"num_object_clones":0,"num_object_copies":1029,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3,"num_objects_unfound":0,"num_objects_dirty":343,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5288,"num_write_kb":180192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2630,"num_bytes_recovered":163315712,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[0,6],"avail_no_missing":["0"],"object_location_counts":[{"shards":"0,3,6","objects":3},{"shards":"0,6","objects":340}],"blocked_by":[],"up_primary":6,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"86'194","reported_seq":798,"reported_epoch":443,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:40:21.393071+0000","last_change":"2023-09-23T20:40:20.532327+0000","last_active":"2023-09-23T20:40:21.393071+0000","last_peered":"2023-09-23T20:40:21.393071+0000","last_clean":"2023-09-23T20:40:12.685357+0000","last_became_active":"2023-09-23T20:40:20.531854+0000","last_became_peered":"2023-09-23T20:40:20.531854+0000","last_unstale":"2023-09-23T20:40:21.393071+0000","last_undegraded":"2023-09-23T20:40:20.510463+0000","last_fullsized":"2023-09-23T20:40:20.509782+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":194,"log_dups_size":0,"ondisk_log_size":194,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:35:05.823248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00037995999999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1024,"num_read_kb":1200,"num_write":173,"num_write_kb":1276,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[2,7],"avail_no_missing":["2","3","7"],"object_location_counts":[{"shards":"2,3,7","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":906,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T20:40:21.407614+0000","last_change":"2023-09-23T20:38:05.602508+0000","last_active":"2023-09-23T20:40:21.407614+0000","last_peered":"2023-09-23T20:40:21.407614+0000","last_clean":"2023-09-23T20:40:21.407614+0000","last_became_active":"2023-09-23T20:38:05.602129+0000","last_became_peered":"2023-09-23T20:38:05.602129+0000","last_unstale":"2023-09-23T20:40:21.407614+0000","last_undegraded":"2023-09-23T20:40:21.407614+0000","last_fullsized":"2023-09-23T20:40:21.407614+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:45.343742+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:45.343742+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:48.910343+0000","scrub_duration":0.001475802,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":13,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T20:40:21.399751+0000","last_change":"2023-09-23T20:40:19.542142+0000","last_active":"2023-09-23T20:40:21.399751+0000","last_peered":"2023-09-23T20:40:21.399751+0000","last_clean":"2023-09-23T20:40:21.399751+0000","last_became_active":"2023-09-23T20:40:19.541870+0000","last_became_peered":"2023-09-23T20:40:19.541870+0000","last_unstale":"2023-09-23T20:40:21.399751+0000","last_undegraded":"2023-09-23T20:40:21.399751+0000","last_fullsized":"2023-09-23T20:40:21.399751+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:01.518149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":50,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T20:40:21.410991+0000","last_change":"2023-09-23T20:38:02.573200+0000","last_active":"2023-09-23T20:40:21.410991+0000","last_peered":"2023-09-23T20:40:21.410991+0000","last_clean":"2023-09-23T20:40:21.410991+0000","last_became_active":"2023-09-23T20:38:02.572945+0000","last_became_peered":"2023-09-23T20:38:02.572945+0000","last_unstale":"2023-09-23T20:40:21.410991+0000","last_undegraded":"2023-09-23T20:40:21.410991+0000","last_fullsized":"2023-09-23T20:40:21.410991+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:57.246533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.a","version":"444'10866","reported_seq":21663,"reported_epoch":444,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T20:40:29.517745+0000","last_change":"2023-09-23T20:40:20.700955+0000","last_active":"2023-09-23T20:40:29.517745+0000","last_peered":"2023-09-23T20:40:29.517745+0000","last_clean":"2023-09-23T20:40:18.471731+0000","last_became_active":"2023-09-23T20:40:20.661915+0000","last_became_peered":"2023-09-23T20:40:20.661915+0000","last_unstale":"2023-09-23T20:40:29.517745+0000","last_undegraded":"2023-09-23T20:40:29.517745+0000","last_fullsized":"2023-09-23T20:40:29.517745+0000","mapping_epoch":441,"log_start":"401'5779","ondisk_log_start":"401'5779","created":369,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":5087,"log_dups_size":0,"ondisk_log_size":5087,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:31:16.196574+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":30408704,"num_objects":464,"num_object_clones":0,"num_object_copies":1392,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":464,"num_objects_unfound":0,"num_objects_dirty":464,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10866,"num_write_kb":362560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5339,"num_bytes_recovered":313851904,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,3],"avail_no_missing":["1","2","3"],"object_location_counts":[{"shards":"1,2,3","objects":464}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":774,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T20:40:21.399058+0000","last_change":"2023-09-23T20:38:05.615711+0000","last_active":"2023-09-23T20:40:21.399058+0000","last_peered":"2023-09-23T20:40:21.399058+0000","last_clean":"2023-09-23T20:40:21.399058+0000","last_became_active":"2023-09-23T20:38:05.615390+0000","last_became_peered":"2023-09-23T20:38:05.615390+0000","last_unstale":"2023-09-23T20:40:21.399058+0000","last_undegraded":"2023-09-23T20:40:21.399058+0000","last_fullsized":"2023-09-23T20:40:21.399058+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:43.582146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:43.582146+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:45:11.917871+0000","scrub_duration":0.0016219730000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"86'80","reported_seq":696,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T20:40:21.407527+0000","last_change":"2023-09-23T20:29:45.536796+0000","last_active":"2023-09-23T20:40:21.407527+0000","last_peered":"2023-09-23T20:40:21.407527+0000","last_clean":"2023-09-23T20:40:21.407527+0000","last_became_active":"2023-09-23T20:29:45.536439+0000","last_became_peered":"2023-09-23T20:29:45.536439+0000","last_unstale":"2023-09-23T20:40:21.407527+0000","last_undegraded":"2023-09-23T20:40:21.407527+0000","last_fullsized":"2023-09-23T20:40:21.407527+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":80,"log_dups_size":0,"ondisk_log_size":80,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:30:57.793132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035115,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":395,"num_read_kb":1370,"num_write":66,"num_write_kb":1061,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":448,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T20:40:21.413543+0000","last_change":"2023-09-23T20:34:59.984286+0000","last_active":"2023-09-23T20:40:21.413543+0000","last_peered":"2023-09-23T20:40:21.413543+0000","last_clean":"2023-09-23T20:40:21.413543+0000","last_became_active":"2023-09-23T20:34:59.983950+0000","last_became_peered":"2023-09-23T20:34:59.983950+0000","last_unstale":"2023-09-23T20:40:21.413543+0000","last_undegraded":"2023-09-23T20:40:21.413543+0000","last_fullsized":"2023-09-23T20:40:21.413543+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:31:53.557035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":443,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T20:40:21.407395+0000","last_change":"2023-09-23T20:29:45.690815+0000","last_active":"2023-09-23T20:40:21.407395+0000","last_peered":"2023-09-23T20:40:21.407395+0000","last_clean":"2023-09-23T20:40:21.407395+0000","last_became_active":"2023-09-23T20:29:45.690478+0000","last_became_peered":"2023-09-23T20:29:45.690478+0000","last_unstale":"2023-09-23T20:40:21.407395+0000","last_undegraded":"2023-09-23T20:40:21.407395+0000","last_fullsized":"2023-09-23T20:40:21.407395+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:38.288193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.f","version":"444'10527","reported_seq":10612,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-09-23T20:40:27.889231+0000","last_change":"2023-09-23T20:36:11.309536+0000","last_active":"2023-09-23T20:40:27.889231+0000","last_peered":"2023-09-23T20:40:27.889231+0000","last_clean":"2023-09-23T20:40:27.889231+0000","last_became_active":"2023-09-23T20:36:11.309208+0000","last_became_peered":"2023-09-23T20:36:11.309208+0000","last_unstale":"2023-09-23T20:40:27.889231+0000","last_undegraded":"2023-09-23T20:40:27.889231+0000","last_fullsized":"2023-09-23T20:40:27.889231+0000","mapping_epoch":369,"log_start":"374'3000","ondisk_log_start":"374'3000","created":369,"last_epoch_clean":370,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":7527,"log_dups_size":0,"ondisk_log_size":7527,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:47:51.785158+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":33226752,"num_objects":507,"num_object_clones":0,"num_object_copies":1521,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":507,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10527,"num_write_kb":353088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":546,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T20:40:21.405933+0000","last_change":"2023-09-23T20:38:05.611181+0000","last_active":"2023-09-23T20:40:21.405933+0000","last_peered":"2023-09-23T20:40:21.405933+0000","last_clean":"2023-09-23T20:40:21.405933+0000","last_became_active":"2023-09-23T20:38:05.610881+0000","last_became_peered":"2023-09-23T20:38:05.610881+0000","last_unstale":"2023-09-23T20:40:21.405933+0000","last_undegraded":"2023-09-23T20:40:21.405933+0000","last_fullsized":"2023-09-23T20:40:21.405933+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:35:00.151695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:35:00.151695+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":20,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"86'102","reported_seq":1038,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T20:40:21.408404+0000","last_change":"2023-09-23T20:32:36.314672+0000","last_active":"2023-09-23T20:40:21.408404+0000","last_peered":"2023-09-23T20:40:21.408404+0000","last_clean":"2023-09-23T20:40:21.408404+0000","last_became_active":"2023-09-23T20:32:36.314231+0000","last_became_peered":"2023-09-23T20:32:36.314231+0000","last_unstale":"2023-09-23T20:40:21.408404+0000","last_undegraded":"2023-09-23T20:40:21.408404+0000","last_fullsized":"2023-09-23T20:40:21.408404+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":102,"log_dups_size":0,"ondisk_log_size":102,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:11:40.514709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030656500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":963,"num_read_kb":1234,"num_write":60,"num_write_kb":225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":389,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T20:40:21.402024+0000","last_change":"2023-09-23T20:40:19.544384+0000","last_active":"2023-09-23T20:40:21.402024+0000","last_peered":"2023-09-23T20:40:21.402024+0000","last_clean":"2023-09-23T20:40:21.402024+0000","last_became_active":"2023-09-23T20:40:19.544079+0000","last_became_peered":"2023-09-23T20:40:19.544079+0000","last_unstale":"2023-09-23T20:40:21.402024+0000","last_undegraded":"2023-09-23T20:40:21.402024+0000","last_fullsized":"2023-09-23T20:40:21.402024+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:00:19.498631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":398,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T20:40:21.405771+0000","last_change":"2023-09-23T20:40:19.532976+0000","last_active":"2023-09-23T20:40:21.405771+0000","last_peered":"2023-09-23T20:40:21.405771+0000","last_clean":"2023-09-23T20:40:21.405771+0000","last_became_active":"2023-09-23T20:40:19.532639+0000","last_became_peered":"2023-09-23T20:40:19.532639+0000","last_unstale":"2023-09-23T20:40:21.405771+0000","last_undegraded":"2023-09-23T20:40:21.405771+0000","last_fullsized":"2023-09-23T20:40:21.405771+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:02:35.301884+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"444'10685","reported_seq":10770,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-09-23T20:40:26.240021+0000","last_change":"2023-09-23T20:36:11.482126+0000","last_active":"2023-09-23T20:40:26.240021+0000","last_peered":"2023-09-23T20:40:26.240021+0000","last_clean":"2023-09-23T20:40:26.240021+0000","last_became_active":"2023-09-23T20:36:11.481802+0000","last_became_peered":"2023-09-23T20:36:11.481802+0000","last_unstale":"2023-09-23T20:40:26.240021+0000","last_undegraded":"2023-09-23T20:40:26.240021+0000","last_fullsized":"2023-09-23T20:40:26.240021+0000","mapping_epoch":369,"log_start":"377'4459","ondisk_log_start":"377'4459","created":369,"last_epoch_clean":370,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":6226,"log_dups_size":0,"ondisk_log_size":6226,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:13:12.263455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":40566784,"num_objects":619,"num_object_clones":0,"num_object_copies":1857,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":619,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10685,"num_write_kb":361728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":722,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T20:40:21.412656+0000","last_change":"2023-09-23T20:38:05.590361+0000","last_active":"2023-09-23T20:40:21.412656+0000","last_peered":"2023-09-23T20:40:21.412656+0000","last_clean":"2023-09-23T20:40:21.412656+0000","last_became_active":"2023-09-23T20:38:05.589985+0000","last_became_peered":"2023-09-23T20:38:05.589985+0000","last_unstale":"2023-09-23T20:40:21.412656+0000","last_undegraded":"2023-09-23T20:40:21.412656+0000","last_fullsized":"2023-09-23T20:40:21.412656+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:03.246271+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:03.246271+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:38:54.351576+0000","scrub_duration":0.0019016700000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":181,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T20:40:21.407043+0000","last_change":"2023-09-23T20:32:50.549695+0000","last_active":"2023-09-23T20:40:21.407043+0000","last_peered":"2023-09-23T20:40:21.407043+0000","last_clean":"2023-09-23T20:40:21.407043+0000","last_became_active":"2023-09-23T20:32:50.549431+0000","last_became_peered":"2023-09-23T20:32:50.549431+0000","last_unstale":"2023-09-23T20:40:21.407043+0000","last_undegraded":"2023-09-23T20:40:21.407043+0000","last_fullsized":"2023-09-23T20:40:21.407043+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:45:53.666569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"86'128","reported_seq":854,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T20:40:21.401612+0000","last_change":"2023-09-23T20:38:03.541765+0000","last_active":"2023-09-23T20:40:21.401612+0000","last_peered":"2023-09-23T20:40:21.401612+0000","last_clean":"2023-09-23T20:40:21.401612+0000","last_became_active":"2023-09-23T20:38:03.541277+0000","last_became_peered":"2023-09-23T20:38:03.541277+0000","last_unstale":"2023-09-23T20:40:21.401612+0000","last_undegraded":"2023-09-23T20:40:21.401612+0000","last_fullsized":"2023-09-23T20:40:21.401612+0000","mapping_epoch":404,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":405,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:20:53.834554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000293202,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":579,"num_read_kb":1473,"num_write":96,"num_write_kb":1062,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":222,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T20:40:21.404563+0000","last_change":"2023-09-23T20:37:43.582105+0000","last_active":"2023-09-23T20:40:21.404563+0000","last_peered":"2023-09-23T20:40:21.404563+0000","last_clean":"2023-09-23T20:40:21.404563+0000","last_became_active":"2023-09-23T20:37:43.581860+0000","last_became_peered":"2023-09-23T20:37:43.581860+0000","last_unstale":"2023-09-23T20:40:21.404563+0000","last_undegraded":"2023-09-23T20:40:21.404563+0000","last_fullsized":"2023-09-23T20:40:21.404563+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:18:38.765486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"444'10606","reported_seq":18288,"reported_epoch":444,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:40:27.312997+0000","last_change":"2023-09-23T20:40:19.729086+0000","last_active":"2023-09-23T20:40:27.312997+0000","last_peered":"2023-09-23T20:40:27.312997+0000","last_clean":"2023-09-23T20:37:13.317209+0000","last_became_active":"2023-09-23T20:40:19.694465+0000","last_became_peered":"2023-09-23T20:40:19.694465+0000","last_unstale":"2023-09-23T20:40:27.312997+0000","last_undegraded":"2023-09-23T20:40:19.549561+0000","last_fullsized":"2023-09-23T20:40:19.517168+0000","mapping_epoch":440,"log_start":"377'4921","ondisk_log_start":"377'4921","created":369,"last_epoch_clean":370,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":5685,"log_dups_size":0,"ondisk_log_size":5685,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:13:44.257914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34144280,"num_objects":522,"num_object_clones":0,"num_object_copies":1566,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5029,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":522,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10606,"num_write_kb":356033,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3780,"num_bytes_recovered":247726080,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[7,2],"avail_no_missing":["7","2"],"object_location_counts":[{"shards":"1,2,6,7","objects":87},{"shards":"1,2,7","objects":193},{"shards":"2,6,7","objects":15},{"shards":"2,7","objects":227}],"blocked_by":[],"up_primary":5,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.4","version":"90'161","reported_seq":656,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-09-23T20:40:22.427435+0000","last_change":"2023-09-23T20:40:22.427435+0000","last_active":"2023-09-23T20:40:22.427435+0000","last_peered":"2023-09-23T20:40:22.427435+0000","last_clean":"2023-09-23T20:40:22.427435+0000","last_became_active":"2023-09-23T20:40:22.427051+0000","last_became_peered":"2023-09-23T20:40:22.427051+0000","last_unstale":"2023-09-23T20:40:22.427435+0000","last_undegraded":"2023-09-23T20:40:22.427435+0000","last_fullsized":"2023-09-23T20:40:22.427435+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:34:54.504554+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:34:54.504554+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:42.001652+0000","scrub_duration":0.011176864,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"86'117","reported_seq":677,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T20:40:21.407176+0000","last_change":"2023-09-23T20:38:28.446107+0000","last_active":"2023-09-23T20:40:21.407176+0000","last_peered":"2023-09-23T20:40:21.407176+0000","last_clean":"2023-09-23T20:40:21.407176+0000","last_became_active":"2023-09-23T20:38:28.445667+0000","last_became_peered":"2023-09-23T20:38:28.445667+0000","last_unstale":"2023-09-23T20:40:21.407176+0000","last_undegraded":"2023-09-23T20:40:21.407176+0000","last_fullsized":"2023-09-23T20:40:21.407176+0000","mapping_epoch":411,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":412,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:58:47.430360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046891400000000002,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":893,"num_read_kb":922,"num_write":82,"num_write_kb":4193,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":443,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T20:40:21.407478+0000","last_change":"2023-09-23T20:32:33.277827+0000","last_active":"2023-09-23T20:40:21.407478+0000","last_peered":"2023-09-23T20:40:21.407478+0000","last_clean":"2023-09-23T20:40:21.407478+0000","last_became_active":"2023-09-23T20:32:33.236087+0000","last_became_peered":"2023-09-23T20:32:33.236087+0000","last_unstale":"2023-09-23T20:40:21.407478+0000","last_undegraded":"2023-09-23T20:40:21.407478+0000","last_fullsized":"2023-09-23T20:40:21.407478+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:20:58.176854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":181,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T20:40:21.413460+0000","last_change":"2023-09-23T20:40:19.530602+0000","last_active":"2023-09-23T20:40:21.413460+0000","last_peered":"2023-09-23T20:40:21.413460+0000","last_clean":"2023-09-23T20:40:21.413460+0000","last_became_active":"2023-09-23T20:40:19.530256+0000","last_became_peered":"2023-09-23T20:40:19.530256+0000","last_unstale":"2023-09-23T20:40:21.413460+0000","last_undegraded":"2023-09-23T20:40:21.413460+0000","last_fullsized":"2023-09-23T20:40:21.413460+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:18:45.658867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.10","version":"444'8435","reported_seq":14888,"reported_epoch":444,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T20:40:28.888943+0000","last_change":"2023-09-23T20:40:20.743359+0000","last_active":"2023-09-23T20:40:28.888943+0000","last_peered":"2023-09-23T20:40:28.888943+0000","last_clean":"2023-09-23T20:40:18.470747+0000","last_became_active":"2023-09-23T20:40:20.710426+0000","last_became_peered":"2023-09-23T20:40:20.710426+0000","last_unstale":"2023-09-23T20:40:28.888943+0000","last_undegraded":"2023-09-23T20:40:28.888943+0000","last_fullsized":"2023-09-23T20:40:28.888943+0000","mapping_epoch":441,"log_start":"373'2187","ondisk_log_start":"373'2187","created":410,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":6248,"log_dups_size":0,"ondisk_log_size":6248,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17498112,"num_objects":267,"num_object_clones":0,"num_object_copies":801,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":267,"num_objects_unfound":0,"num_objects_dirty":267,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5188,"num_write_kb":174560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2537,"num_bytes_recovered":166264832,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,1,4],"avail_no_missing":["3","1","4"],"object_location_counts":[{"shards":"1,3,4","objects":267}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.19","version":"90'161","reported_seq":713,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T20:40:21.413274+0000","last_change":"2023-09-23T20:38:05.630221+0000","last_active":"2023-09-23T20:40:21.413274+0000","last_peered":"2023-09-23T20:40:21.413274+0000","last_clean":"2023-09-23T20:40:21.413274+0000","last_became_active":"2023-09-23T20:38:05.629839+0000","last_became_peered":"2023-09-23T20:38:05.629839+0000","last_unstale":"2023-09-23T20:40:21.413274+0000","last_undegraded":"2023-09-23T20:40:21.413274+0000","last_fullsized":"2023-09-23T20:40:21.413274+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:06.487351+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:06.487351+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:25:51.325529+0000","scrub_duration":0.001964379,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":411,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T20:40:21.405722+0000","last_change":"2023-09-23T20:32:33.237234+0000","last_active":"2023-09-23T20:40:21.405722+0000","last_peered":"2023-09-23T20:40:21.405722+0000","last_clean":"2023-09-23T20:40:21.405722+0000","last_became_active":"2023-09-23T20:32:33.236881+0000","last_became_peered":"2023-09-23T20:32:33.236881+0000","last_unstale":"2023-09-23T20:40:21.405722+0000","last_undegraded":"2023-09-23T20:40:21.405722+0000","last_fullsized":"2023-09-23T20:40:21.405722+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:56:17.384831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":298,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T20:40:21.394803+0000","last_change":"2023-09-23T20:29:45.538314+0000","last_active":"2023-09-23T20:40:21.394803+0000","last_peered":"2023-09-23T20:40:21.394803+0000","last_clean":"2023-09-23T20:40:21.394803+0000","last_became_active":"2023-09-23T20:29:45.536847+0000","last_became_peered":"2023-09-23T20:29:45.536847+0000","last_unstale":"2023-09-23T20:40:21.394803+0000","last_undegraded":"2023-09-23T20:40:21.394803+0000","last_fullsized":"2023-09-23T20:40:21.394803+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:51.948721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"8.17","version":"444'8619","reported_seq":9385,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-09-23T20:40:29.501019+0000","last_change":"2023-09-23T20:40:19.281439+0000","last_active":"2023-09-23T20:40:29.501019+0000","last_peered":"2023-09-23T20:40:29.501019+0000","last_clean":"2023-09-23T20:40:29.501019+0000","last_became_active":"2023-09-23T20:38:27.474930+0000","last_became_peered":"2023-09-23T20:38:27.474930+0000","last_unstale":"2023-09-23T20:40:29.501019+0000","last_undegraded":"2023-09-23T20:40:29.501019+0000","last_fullsized":"2023-09-23T20:40:29.501019+0000","mapping_epoch":410,"log_start":"373'1658","ondisk_log_start":"373'1658","created":410,"last_epoch_clean":440,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":6961,"log_dups_size":0,"ondisk_log_size":6961,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":14942208,"num_objects":228,"num_object_clones":0,"num_object_copies":684,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":228,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5329,"num_write_kb":177824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":327,"num_bytes_recovered":10354688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1e","version":"90'161","reported_seq":503,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T20:40:21.403403+0000","last_change":"2023-09-23T20:38:05.602250+0000","last_active":"2023-09-23T20:40:21.403403+0000","last_peered":"2023-09-23T20:40:21.403403+0000","last_clean":"2023-09-23T20:40:21.403403+0000","last_became_active":"2023-09-23T20:38:05.601874+0000","last_became_peered":"2023-09-23T20:38:05.601874+0000","last_unstale":"2023-09-23T20:40:21.403403+0000","last_undegraded":"2023-09-23T20:40:21.403403+0000","last_fullsized":"2023-09-23T20:40:21.403403+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:35:42.864363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:35:42.864363+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":40253,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":42,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T20:40:21.397271+0000","last_change":"2023-09-23T20:38:37.481551+0000","last_active":"2023-09-23T20:40:21.397271+0000","last_peered":"2023-09-23T20:40:21.397271+0000","last_clean":"2023-09-23T20:40:21.397271+0000","last_became_active":"2023-09-23T20:38:37.458687+0000","last_became_peered":"2023-09-23T20:38:37.458687+0000","last_unstale":"2023-09-23T20:40:21.397271+0000","last_undegraded":"2023-09-23T20:40:21.397271+0000","last_fullsized":"2023-09-23T20:40:21.397271+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:22:15.033243+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":100,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T20:40:21.412571+0000","last_change":"2023-09-23T20:34:59.987575+0000","last_active":"2023-09-23T20:40:21.412571+0000","last_peered":"2023-09-23T20:40:21.412571+0000","last_clean":"2023-09-23T20:40:21.412571+0000","last_became_active":"2023-09-23T20:34:59.987229+0000","last_became_peered":"2023-09-23T20:34:59.987229+0000","last_unstale":"2023-09-23T20:40:21.412571+0000","last_undegraded":"2023-09-23T20:40:21.412571+0000","last_fullsized":"2023-09-23T20:40:21.412571+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:34:01.820159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.16","version":"444'8440","reported_seq":9223,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-09-23T20:40:26.217476+0000","last_change":"2023-09-23T20:38:28.264940+0000","last_active":"2023-09-23T20:40:26.217476+0000","last_peered":"2023-09-23T20:40:26.217476+0000","last_clean":"2023-09-23T20:40:26.217476+0000","last_became_active":"2023-09-23T20:38:27.484333+0000","last_became_peered":"2023-09-23T20:38:27.484333+0000","last_unstale":"2023-09-23T20:40:26.217476+0000","last_undegraded":"2023-09-23T20:40:26.217476+0000","last_fullsized":"2023-09-23T20:40:26.217476+0000","mapping_epoch":410,"log_start":"373'1261","ondisk_log_start":"373'1261","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":7179,"log_dups_size":0,"ondisk_log_size":7179,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":25821184,"num_objects":394,"num_object_clones":0,"num_object_copies":1182,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":394,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5160,"num_write_kb":177728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":335,"num_bytes_recovered":9699328,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":42,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T20:40:21.396412+0000","last_change":"2023-09-23T20:38:37.457634+0000","last_active":"2023-09-23T20:40:21.396412+0000","last_peered":"2023-09-23T20:40:21.396412+0000","last_clean":"2023-09-23T20:40:21.396412+0000","last_became_active":"2023-09-23T20:38:37.452208+0000","last_became_peered":"2023-09-23T20:38:37.452208+0000","last_unstale":"2023-09-23T20:40:21.396412+0000","last_undegraded":"2023-09-23T20:40:21.396412+0000","last_fullsized":"2023-09-23T20:40:21.396412+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:09:41.754569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":100,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T20:40:21.415102+0000","last_change":"2023-09-23T20:34:59.987416+0000","last_active":"2023-09-23T20:40:21.415102+0000","last_peered":"2023-09-23T20:40:21.415102+0000","last_clean":"2023-09-23T20:40:21.415102+0000","last_became_active":"2023-09-23T20:34:59.987117+0000","last_became_peered":"2023-09-23T20:34:59.987117+0000","last_unstale":"2023-09-23T20:40:21.415102+0000","last_undegraded":"2023-09-23T20:40:21.415102+0000","last_fullsized":"2023-09-23T20:40:21.415102+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:36:16.318623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.15","version":"444'8544","reported_seq":12118,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-09-23T20:40:28.032595+0000","last_change":"2023-09-23T20:38:53.167534+0000","last_active":"2023-09-23T20:40:28.032595+0000","last_peered":"2023-09-23T20:40:28.032595+0000","last_clean":"2023-09-23T20:40:28.032595+0000","last_became_active":"2023-09-23T20:38:28.579407+0000","last_became_peered":"2023-09-23T20:38:28.579407+0000","last_unstale":"2023-09-23T20:40:28.032595+0000","last_undegraded":"2023-09-23T20:40:28.032595+0000","last_fullsized":"2023-09-23T20:40:28.032595+0000","mapping_epoch":410,"log_start":"373'1839","ondisk_log_start":"373'1839","created":410,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":6705,"log_dups_size":0,"ondisk_log_size":6705,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19955712,"num_objects":305,"num_object_clones":0,"num_object_copies":915,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":305,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5249,"num_write_kb":177696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1043,"num_bytes_recovered":56328192,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1c","version":"90'161","reported_seq":699,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T20:40:21.410277+0000","last_change":"2023-09-23T20:38:05.591992+0000","last_active":"2023-09-23T20:40:21.410277+0000","last_peered":"2023-09-23T20:40:21.410277+0000","last_clean":"2023-09-23T20:40:21.410277+0000","last_became_active":"2023-09-23T20:38:05.591701+0000","last_became_peered":"2023-09-23T20:38:05.591701+0000","last_unstale":"2023-09-23T20:40:21.410277+0000","last_undegraded":"2023-09-23T20:40:21.410277+0000","last_fullsized":"2023-09-23T20:40:21.410277+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:02.384133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:02.384133+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:41.722542+0000","scrub_duration":0.001769098,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":264,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T20:40:21.403845+0000","last_change":"2023-09-23T20:38:02.542719+0000","last_active":"2023-09-23T20:40:21.403845+0000","last_peered":"2023-09-23T20:40:21.403845+0000","last_clean":"2023-09-23T20:40:21.403845+0000","last_became_active":"2023-09-23T20:38:02.542385+0000","last_became_peered":"2023-09-23T20:38:02.542385+0000","last_unstale":"2023-09-23T20:40:21.403845+0000","last_undegraded":"2023-09-23T20:40:21.403845+0000","last_fullsized":"2023-09-23T20:40:21.403845+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:36:23.673757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":231,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T20:40:21.403371+0000","last_change":"2023-09-23T20:37:43.584253+0000","last_active":"2023-09-23T20:40:21.403371+0000","last_peered":"2023-09-23T20:40:21.403371+0000","last_clean":"2023-09-23T20:40:21.403371+0000","last_became_active":"2023-09-23T20:37:43.580069+0000","last_became_peered":"2023-09-23T20:37:43.580069+0000","last_unstale":"2023-09-23T20:40:21.403371+0000","last_undegraded":"2023-09-23T20:40:21.403371+0000","last_fullsized":"2023-09-23T20:40:21.403371+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:39:11.482581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.14","version":"444'8756","reported_seq":10975,"reported_epoch":444,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:40:25.518296+0000","last_change":"2023-09-23T20:40:20.758265+0000","last_active":"2023-09-23T20:40:25.518296+0000","last_peered":"2023-09-23T20:40:25.518296+0000","last_clean":"2023-09-23T20:37:15.925522+0000","last_became_active":"2023-09-23T20:38:39.251212+0000","last_became_peered":"2023-09-23T20:38:39.251212+0000","last_unstale":"2023-09-23T20:40:25.518296+0000","last_undegraded":"2023-09-23T20:40:20.756509+0000","last_fullsized":"2023-09-23T20:38:38.450156+0000","mapping_epoch":416,"log_start":"373'2080","ondisk_log_start":"373'2080","created":410,"last_epoch_clean":370,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":6676,"log_dups_size":0,"ondisk_log_size":6676,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22544384,"num_objects":344,"num_object_clones":0,"num_object_copies":1032,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":387,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":344,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5349,"num_write_kb":182176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":682,"num_bytes_recovered":34406400,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[4,2],"avail_no_missing":["4","2","6"],"object_location_counts":[{"shards":"2,4","objects":344}],"blocked_by":[],"up_primary":5,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1d","version":"90'161","reported_seq":536,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-09-23T20:40:22.403981+0000","last_change":"2023-09-23T20:36:01.505098+0000","last_active":"2023-09-23T20:40:22.403981+0000","last_peered":"2023-09-23T20:40:22.403981+0000","last_clean":"2023-09-23T20:40:22.403981+0000","last_became_active":"2023-09-23T20:35:26.644707+0000","last_became_peered":"2023-09-23T20:35:26.644707+0000","last_unstale":"2023-09-23T20:40:22.403981+0000","last_undegraded":"2023-09-23T20:40:22.403981+0000","last_fullsized":"2023-09-23T20:40:22.403981+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:01.504973+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:01.504973+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:02:10.880809+0000","scrub_duration":0.00202344,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":488,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T20:40:21.406501+0000","last_change":"2023-09-23T20:40:19.569032+0000","last_active":"2023-09-23T20:40:21.406501+0000","last_peered":"2023-09-23T20:40:21.406501+0000","last_clean":"2023-09-23T20:40:21.406501+0000","last_became_active":"2023-09-23T20:40:19.538476+0000","last_became_peered":"2023-09-23T20:40:19.538476+0000","last_unstale":"2023-09-23T20:40:21.406501+0000","last_undegraded":"2023-09-23T20:40:21.406501+0000","last_fullsized":"2023-09-23T20:40:21.406501+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:31:58.998168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3324,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":161,"reported_epoch":443,"state":"active+clean","last_fresh":"2023-09-23T20:40:21.412083+0000","last_change":"2023-09-23T20:38:02.538333+0000","last_active":"2023-09-23T20:40:21.412083+0000","last_peered":"2023-09-23T20:40:21.412083+0000","last_clean":"2023-09-23T20:40:21.412083+0000","last_became_active":"2023-09-23T20:38:02.538008+0000","last_became_peered":"2023-09-23T20:38:02.538008+0000","last_unstale":"2023-09-23T20:40:21.412083+0000","last_undegraded":"2023-09-23T20:40:21.412083+0000","last_fullsized":"2023-09-23T20:40:21.412083+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:02:38.789530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":80,"num_bytes_recovered":18486,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26691,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":8,"num_pg":26,"stat_sum":{"num_bytes":579338264,"num_objects":8841,"num_object_clones":0,"num_object_copies":26523,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":15607,"num_objects_misplaced":1613,"num_objects_unfound":0,"num_objects_dirty":8841,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":169801,"num_write_kb":5716481,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33595,"num_bytes_recovered":2061107200,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3433246720,"data_stored":3433234504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":171649,"ondisk_log_size":171649,"up":78,"acting":69,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":351684,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":155,"num_read_kb":290,"num_write":76,"num_write_kb":768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":891212,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5983,"num_read_kb":13200,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":57,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6082,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":23,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201628,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":560432,"kb_used_data":196008,"kb_used_omap":0,"kb_used_meta":364416,"kb_avail":93184720,"statfs":{"total":95995035648,"available":95421153280,"internally_reserved":0,"allocated":200712192,"data_stored":199291712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":373161984},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,1,0,1,1,2],"upper_bound":128},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:39:51 2023","interfaces":[{"interface":"back","average":{"1min":0.81899999999999995,"5min":2.395,"15min":2.177},"min":{"1min":0.25800000000000001,"5min":0.24399999999999999,"15min":0.20799999999999999},"max":{"1min":3.7639999999999998,"5min":18.315000000000001,"15min":48.798999999999999},"last":0.27200000000000002},{"interface":"front","average":{"1min":0.81599999999999995,"5min":1.7150000000000001,"15min":2.008},"min":{"1min":0.29199999999999998,"5min":0.247,"15min":0.24399999999999999},"max":{"1min":7.109,"5min":19.355,"15min":48.247},"last":3.1549999999999998}]},{"osd":1,"last update":"Sat Sep 23 20:40:03 2023","interfaces":[{"interface":"back","average":{"1min":0.92600000000000005,"5min":2.7120000000000002,"15min":2.1520000000000001},"min":{"1min":0.28799999999999998,"5min":0.25900000000000001,"15min":0.254},"max":{"1min":2.9020000000000001,"5min":49.965000000000003,"15min":49.965000000000003},"last":0.30599999999999999},{"interface":"front","average":{"1min":0.89300000000000002,"5min":2.6989999999999998,"15min":2.1709999999999998},"min":{"1min":0.30299999999999999,"5min":0.26900000000000002,"15min":0.248},"max":{"1min":5.508,"5min":53.451999999999998,"15min":53.451999999999998},"last":3.105}]},{"osd":2,"last update":"Sat Sep 23 20:39:51 2023","interfaces":[{"interface":"back","average":{"1min":1.23,"5min":2.129,"15min":2.1469999999999998},"min":{"1min":0.32800000000000001,"5min":0.27800000000000002,"15min":0.27600000000000002},"max":{"1min":7.2910000000000004,"5min":16.965,"15min":16.965},"last":3.2360000000000002},{"interface":"front","average":{"1min":1.1140000000000001,"5min":2.6459999999999999,"15min":2.48},"min":{"1min":0.32500000000000001,"5min":0.308,"15min":0.26600000000000001},"max":{"1min":3.6880000000000002,"5min":23.414000000000001,"15min":23.494},"last":1.571}]},{"osd":3,"last update":"Sat Sep 23 20:40:21 2023","interfaces":[{"interface":"back","average":{"1min":1.105,"5min":2.2480000000000002,"15min":2.4470000000000001},"min":{"1min":0.35999999999999999,"5min":0.311,"15min":0.26200000000000001},"max":{"1min":4.2149999999999999,"5min":23.352,"15min":23.352},"last":1.9570000000000001},{"interface":"front","average":{"1min":1.139,"5min":2.2709999999999999,"15min":2.423},"min":{"1min":0.46500000000000002,"5min":0.34399999999999997,"15min":0.32500000000000001},"max":{"1min":4.2359999999999998,"5min":20.969999999999999,"15min":20.969999999999999},"last":3.331}]},{"osd":4,"last update":"Sat Sep 23 20:40:21 2023","interfaces":[{"interface":"back","average":{"1min":1.046,"5min":1.085,"15min":1.2170000000000001},"min":{"1min":0.35799999999999998,"5min":0.25600000000000001,"15min":0.23200000000000001},"max":{"1min":4.1639999999999997,"5min":18.364999999999998,"15min":18.364999999999998},"last":5.673},{"interface":"front","average":{"1min":0.94999999999999996,"5min":1.296,"15min":1.417},"min":{"1min":0.39100000000000001,"5min":0.28399999999999997,"15min":0.27600000000000002},"max":{"1min":2.605,"5min":23.617000000000001,"15min":23.617000000000001},"last":3.298}]},{"osd":5,"last update":"Sat Sep 23 20:40:21 2023","interfaces":[{"interface":"back","average":{"1min":1.0780000000000001,"5min":1.1499999999999999,"15min":1.2509999999999999},"min":{"1min":0.30099999999999999,"5min":0.29199999999999998,"15min":0.26600000000000001},"max":{"1min":4.3780000000000001,"5min":12.516,"15min":13.241},"last":1.6240000000000001},{"interface":"front","average":{"1min":0.76400000000000001,"5min":1.286,"15min":1.3540000000000001},"min":{"1min":0.38400000000000001,"5min":0.31900000000000001,"15min":0.315},"max":{"1min":3.1030000000000002,"5min":18.390000000000001,"15min":18.831},"last":5.9160000000000004}]},{"osd":6,"last update":"Sat Sep 23 20:40:21 2023","interfaces":[{"interface":"back","average":{"1min":1.2629999999999999,"5min":1.4350000000000001,"15min":1.329},"min":{"1min":0.44,"5min":0.312,"15min":0.30399999999999999},"max":{"1min":4.0540000000000003,"5min":18.417000000000002,"15min":18.417000000000002},"last":2.883},{"interface":"front","average":{"1min":1.1659999999999999,"5min":1.113,"15min":1.252},"min":{"1min":0.38200000000000001,"5min":0.32700000000000001,"15min":0.318},"max":{"1min":6.4340000000000002,"5min":7.266,"15min":9.9120000000000008},"last":3.2679999999999998}]}]},{"osd":6,"up_from":47,"seq":201863463267,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":646496,"kb_used_data":258200,"kb_used_omap":0,"kb_used_meta":388288,"kb_avail":93098656,"statfs":{"total":95995035648,"available":95333023744,"internally_reserved":0,"allocated":264396800,"data_stored":262992009,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":397606912},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2],"upper_bound":4},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:39:49 2023","interfaces":[{"interface":"back","average":{"1min":0.89500000000000002,"5min":1.859,"15min":1.9339999999999999},"min":{"1min":0.23699999999999999,"5min":0.23300000000000001,"15min":0.221},"max":{"1min":2.887,"5min":17.492999999999999,"15min":53.701999999999998},"last":0.315},{"interface":"front","average":{"1min":1.4219999999999999,"5min":2.3730000000000002,"15min":2.4409999999999998},"min":{"1min":0.24099999999999999,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":6.625,"5min":60.328000000000003,"15min":60.328000000000003},"last":6.0570000000000004}]},{"osd":1,"last update":"Sat Sep 23 20:40:06 2023","interfaces":[{"interface":"back","average":{"1min":1.157,"5min":2.0600000000000001,"15min":2.3610000000000002},"min":{"1min":0.26600000000000001,"5min":0.26000000000000001,"15min":0.249},"max":{"1min":5.8630000000000004,"5min":64.274000000000001,"15min":64.274000000000001},"last":0.254},{"interface":"front","average":{"1min":1.2390000000000001,"5min":1.393,"15min":2.4830000000000001},"min":{"1min":0.34499999999999997,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":6.8460000000000001,"5min":6.8460000000000001,"15min":34.490000000000002},"last":0.40500000000000003}]},{"osd":2,"last update":"Sat Sep 23 20:39:49 2023","interfaces":[{"interface":"back","average":{"1min":1.069,"5min":1.748,"15min":2.1499999999999999},"min":{"1min":0.30299999999999999,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":4.3399999999999999,"5min":13.429,"15min":34.317999999999998},"last":0.74399999999999999},{"interface":"front","average":{"1min":1.0740000000000001,"5min":1.657,"15min":2.024},"min":{"1min":0.40000000000000002,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":4.3630000000000004,"5min":13.457000000000001,"15min":34.292999999999999},"last":0.71499999999999997}]},{"osd":3,"last update":"Sat Sep 23 20:39:35 2023","interfaces":[{"interface":"back","average":{"1min":1.829,"5min":1.8759999999999999,"15min":1.9970000000000001},"min":{"1min":0.36699999999999999,"5min":0.29999999999999999,"15min":0.29599999999999999},"max":{"1min":6.7690000000000001,"5min":8.9670000000000005,"15min":9.4239999999999995},"last":5.9130000000000003},{"interface":"front","average":{"1min":1.742,"5min":2.149,"15min":2.1459999999999999},"min":{"1min":0.36899999999999999,"5min":0.34899999999999998,"15min":0.317},"max":{"1min":7.6230000000000002,"5min":11.913,"15min":12.875999999999999},"last":4.8789999999999996}]},{"osd":4,"last update":"Sat Sep 23 20:40:11 2023","interfaces":[{"interface":"back","average":{"1min":0.90200000000000002,"5min":0.84799999999999998,"15min":0.95399999999999996},"min":{"1min":0.30099999999999999,"5min":0.26900000000000002,"15min":0.251},"max":{"1min":2.859,"5min":23.904,"15min":23.904},"last":0.68700000000000006},{"interface":"front","average":{"1min":1.498,"5min":1.1879999999999999,"15min":1.204},"min":{"1min":0.36799999999999999,"5min":0.29199999999999998,"15min":0.27000000000000002},"max":{"1min":6.7359999999999998,"5min":11.515000000000001,"15min":11.515000000000001},"last":2.181}]},{"osd":5,"last update":"Sat Sep 23 20:40:11 2023","interfaces":[{"interface":"back","average":{"1min":1.2210000000000001,"5min":0.85299999999999998,"15min":0.96199999999999997},"min":{"1min":0.375,"5min":0.318,"15min":0.28699999999999998},"max":{"1min":6.6340000000000003,"5min":6.6340000000000003,"15min":14.085000000000001},"last":0.78500000000000003},{"interface":"front","average":{"1min":1.2529999999999999,"5min":0.97899999999999998,"15min":1.355},"min":{"1min":0.35399999999999998,"5min":0.32500000000000001,"15min":0.30599999999999999},"max":{"1min":6.6909999999999998,"5min":6.7320000000000002,"15min":18.192},"last":4.7910000000000004}]},{"osd":7,"last update":"Sat Sep 23 20:39:35 2023","interfaces":[{"interface":"back","average":{"1min":0.877,"5min":0.82299999999999995,"15min":1.137},"min":{"1min":0.39400000000000002,"5min":0.36599999999999999,"15min":0.28599999999999998},"max":{"1min":4.1520000000000001,"5min":4.851,"15min":10.217000000000001},"last":0.54200000000000004},{"interface":"front","average":{"1min":1.1910000000000001,"5min":1.0389999999999999,"15min":1.3460000000000001},"min":{"1min":0.379,"5min":0.35999999999999999,"15min":0.30299999999999999},"max":{"1min":6.7119999999999997,"5min":8.8119999999999994,"15min":9.7360000000000007},"last":2.1480000000000001}]}]},{"osd":1,"up_from":266,"seq":1142461300866,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":694532,"kb_used_data":245948,"kb_used_omap":8,"kb_used_meta":448503,"kb_avail":93050620,"statfs":{"total":95995035648,"available":95283834880,"internally_reserved":0,"allocated":251850752,"data_stored":249426318,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8206,"internal_metadata":459268082},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:40:05 2023","interfaces":[{"interface":"back","average":{"1min":1.3680000000000001,"5min":2.2250000000000001,"15min":2.5099999999999998},"min":{"1min":0.218,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":8.1150000000000002,"5min":14.531000000000001,"15min":52.689999999999998},"last":0.84299999999999997},{"interface":"front","average":{"1min":2.4140000000000001,"5min":3.1200000000000001,"15min":3.2650000000000001},"min":{"1min":0.316,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":11.994,"5min":19.41,"15min":52.662999999999997},"last":0.38700000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:40:05 2023","interfaces":[{"interface":"back","average":{"1min":2.7280000000000002,"5min":3.0489999999999999,"15min":4.0810000000000004},"min":{"1min":0.23400000000000001,"5min":0.23300000000000001,"15min":0.20899999999999999},"max":{"1min":14.148,"5min":32.255000000000003,"15min":32.255000000000003},"last":9.5359999999999996},{"interface":"front","average":{"1min":1.911,"5min":2.238,"15min":2.3170000000000002},"min":{"1min":0.28599999999999998,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":8.8079999999999998,"5min":13.404999999999999,"15min":13.404999999999999},"last":0.81599999999999995}]},{"osd":3,"last update":"Sat Sep 23 20:40:05 2023","interfaces":[{"interface":"back","average":{"1min":1.6659999999999999,"5min":2.3809999999999998,"15min":3.5430000000000001},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":8.0079999999999991,"5min":14.359,"15min":80.745999999999995},"last":0.87},{"interface":"front","average":{"1min":2.6509999999999998,"5min":2.2810000000000001,"15min":3.9319999999999999},"min":{"1min":0.316,"5min":0.27500000000000002,"15min":0.22700000000000001},"max":{"1min":12.045999999999999,"5min":19.315000000000001,"15min":82.988},"last":0.41899999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:40:05 2023","interfaces":[{"interface":"back","average":{"1min":2.0710000000000002,"5min":2.2490000000000001,"15min":2.2090000000000001},"min":{"1min":0.33400000000000002,"5min":0.315,"15min":0.30299999999999999},"max":{"1min":7.8399999999999999,"5min":9.4510000000000005,"15min":24.327999999999999},"last":0.53300000000000003},{"interface":"front","average":{"1min":2.056,"5min":1.593,"15min":1.764},"min":{"1min":0.33400000000000002,"5min":0.311,"15min":0.311},"max":{"1min":8.1509999999999998,"5min":11.574999999999999,"15min":11.574999999999999},"last":0.77500000000000002}]},{"osd":5,"last update":"Sat Sep 23 20:40:05 2023","interfaces":[{"interface":"back","average":{"1min":1.996,"5min":2.1400000000000001,"15min":2.3730000000000002},"min":{"1min":0.45800000000000002,"5min":0.34100000000000003,"15min":0.33200000000000002},"max":{"1min":5.5529999999999999,"5min":33.401000000000003,"15min":33.401000000000003},"last":0.45800000000000002},{"interface":"front","average":{"1min":2.5790000000000002,"5min":2.7120000000000002,"15min":2.427},"min":{"1min":0.36199999999999999,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":7.9340000000000002,"5min":33.417000000000002,"15min":33.417000000000002},"last":0.43099999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:40:05 2023","interfaces":[{"interface":"back","average":{"1min":2.0019999999999998,"5min":1.9570000000000001,"15min":4.1699999999999999},"min":{"1min":0.42899999999999999,"5min":0.36899999999999999,"15min":0.35699999999999998},"max":{"1min":6.8099999999999996,"5min":9.6999999999999993,"15min":72.682000000000002},"last":0.52200000000000002},{"interface":"front","average":{"1min":1.9419999999999999,"5min":2.2799999999999998,"15min":4.0460000000000003},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":7.883,"5min":9.4849999999999994,"15min":72.563999999999993},"last":0.47899999999999998}]},{"osd":7,"last update":"Sat Sep 23 20:40:05 2023","interfaces":[{"interface":"back","average":{"1min":2.1930000000000001,"5min":2.069,"15min":2.5760000000000001},"min":{"1min":0.48399999999999999,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":5.0819999999999999,"5min":18.437000000000001,"15min":18.437000000000001},"last":0.496},{"interface":"front","average":{"1min":2.3149999999999999,"5min":2.3959999999999999,"15min":2.46},"min":{"1min":0.45100000000000001,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":7.9089999999999998,"5min":12.021000000000001,"15min":24.524000000000001},"last":0.505}]}]},{"osd":0,"up_from":94,"seq":403726926068,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":933712,"kb_used_data":288648,"kb_used_omap":7,"kb_used_meta":644984,"kb_avail":92811440,"statfs":{"total":95995035648,"available":95038914560,"internally_reserved":0,"allocated":295575552,"data_stored":293315950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":660463633},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,2,0,1,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:40:11 2023","interfaces":[{"interface":"back","average":{"1min":1.712,"5min":1.8,"15min":3.0680000000000001},"min":{"1min":0.20699999999999999,"5min":0.17199999999999999,"15min":0.17199999999999999},"max":{"1min":9.8879999999999999,"5min":11.554,"15min":42.000999999999998},"last":0.78100000000000003},{"interface":"front","average":{"1min":1.5660000000000001,"5min":1.9710000000000001,"15min":3.3370000000000002},"min":{"1min":0.218,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":10.097,"5min":12.35,"15min":23.036000000000001},"last":0.75}]},{"osd":2,"last update":"Sat Sep 23 20:39:47 2023","interfaces":[{"interface":"back","average":{"1min":0.83099999999999996,"5min":2.7610000000000001,"15min":2.907},"min":{"1min":0.23400000000000001,"5min":0.218,"15min":0.185},"max":{"1min":2.9670000000000001,"5min":62.726999999999997,"15min":77.448999999999998},"last":0.80500000000000005},{"interface":"front","average":{"1min":1.5489999999999999,"5min":3.0329999999999999,"15min":2.8010000000000002},"min":{"1min":0.27300000000000002,"5min":0.247,"15min":0.188},"max":{"1min":6.4800000000000004,"5min":62.689,"15min":81.724999999999994},"last":0.311}]},{"osd":3,"last update":"Sat Sep 23 20:40:23 2023","interfaces":[{"interface":"back","average":{"1min":2.399,"5min":2.7450000000000001,"15min":3.133},"min":{"1min":0.34699999999999998,"5min":0.24399999999999999,"15min":0.23400000000000001},"max":{"1min":9.0039999999999996,"5min":51.536000000000001,"15min":81.400999999999996},"last":0.92600000000000005},{"interface":"front","average":{"1min":2.8839999999999999,"5min":3.1389999999999998,"15min":3.4590000000000001},"min":{"1min":0.245,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":14.237,"5min":44.811999999999998,"15min":81.137},"last":0.89100000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:39:47 2023","interfaces":[{"interface":"back","average":{"1min":1.333,"5min":2.008,"15min":2.1539999999999999},"min":{"1min":0.31900000000000001,"5min":0.28899999999999998,"15min":0.25900000000000001},"max":{"1min":6.2450000000000001,"5min":11.068,"15min":23.876999999999999},"last":2.6059999999999999},{"interface":"front","average":{"1min":1.0940000000000001,"5min":1.5700000000000001,"15min":1.954},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":6.3310000000000004,"5min":11.458,"15min":23.681999999999999},"last":9.1099999999999994}]},{"osd":5,"last update":"Sat Sep 23 20:39:47 2023","interfaces":[{"interface":"back","average":{"1min":0.97299999999999998,"5min":1.5580000000000001,"15min":2.1600000000000001},"min":{"1min":0.371,"5min":0.35099999999999998,"15min":0.34200000000000003},"max":{"1min":3.29,"5min":11.497999999999999,"15min":24.198},"last":9.5920000000000005},{"interface":"front","average":{"1min":1.4910000000000001,"5min":2.4399999999999999,"15min":2.3010000000000002},"min":{"1min":0.318,"5min":0.318,"15min":0.25600000000000001},"max":{"1min":6.593,"5min":26.228000000000002,"15min":26.228000000000002},"last":12.929}]},{"osd":6,"last update":"Sat Sep 23 20:39:47 2023","interfaces":[{"interface":"back","average":{"1min":1.1599999999999999,"5min":1.867,"15min":2.1309999999999998},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.318},"max":{"1min":6.3680000000000003,"5min":11.526999999999999,"15min":24.408000000000001},"last":1.238},{"interface":"front","average":{"1min":1.6599999999999999,"5min":2.4129999999999998,"15min":2.2480000000000002},"min":{"1min":0.36799999999999999,"5min":0.35499999999999998,"15min":0.29399999999999998},"max":{"1min":6.5469999999999997,"5min":26.268000000000001,"15min":26.268000000000001},"last":1.2549999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:39:47 2023","interfaces":[{"interface":"back","average":{"1min":1.6539999999999999,"5min":2.4710000000000001,"15min":2.4249999999999998},"min":{"1min":0.39800000000000002,"5min":0.378,"15min":0.30599999999999999},"max":{"1min":6.9160000000000004,"5min":26.177,"15min":26.177},"last":4.4509999999999996},{"interface":"front","average":{"1min":1.327,"5min":2.1629999999999998,"15min":2.5409999999999999},"min":{"1min":0.39200000000000002,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":4.4210000000000003,"5min":11.112,"15min":24.669},"last":4.4900000000000002}]}]},{"osd":2,"up_from":104,"seq":446676599016,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":917072,"kb_used_data":239304,"kb_used_omap":11,"kb_used_meta":677748,"kb_avail":92828080,"statfs":{"total":95995035648,"available":95055953920,"internally_reserved":0,"allocated":245047296,"data_stored":242785431,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11878,"internal_metadata":694014362},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:40:00 2023","interfaces":[{"interface":"back","average":{"1min":2.1339999999999999,"5min":1.921,"15min":2.7160000000000002},"min":{"1min":0.26400000000000001,"5min":0.20300000000000001,"15min":0.17999999999999999},"max":{"1min":8.9710000000000001,"5min":14.124000000000001,"15min":82.424999999999997},"last":1.9530000000000001},{"interface":"front","average":{"1min":2.2919999999999998,"5min":2.375,"15min":2.5920000000000001},"min":{"1min":0.23000000000000001,"5min":0.19,"15min":0.19},"max":{"1min":9.5600000000000005,"5min":14.055,"15min":82.524000000000001},"last":3.2919999999999998}]},{"osd":1,"last update":"Sat Sep 23 20:40:00 2023","interfaces":[{"interface":"back","average":{"1min":2.472,"5min":2.9500000000000002,"15min":3.6240000000000001},"min":{"1min":0.433,"5min":0.245,"15min":0.221},"max":{"1min":11.279999999999999,"5min":46.405000000000001,"15min":46.405000000000001},"last":1.7330000000000001},{"interface":"front","average":{"1min":1.506,"5min":2.9129999999999998,"15min":2.9009999999999998},"min":{"1min":0.24399999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":8.8699999999999992,"5min":46.459000000000003,"15min":46.459000000000003},"last":3.2290000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:40:22 2023","interfaces":[{"interface":"back","average":{"1min":2.851,"5min":2.613,"15min":2.4769999999999999},"min":{"1min":0.28499999999999998,"5min":0.27100000000000002,"15min":0.19800000000000001},"max":{"1min":11.304,"5min":19.562999999999999,"15min":19.562999999999999},"last":3.556},{"interface":"front","average":{"1min":1.7909999999999999,"5min":1.6160000000000001,"15min":2.5089999999999999},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.23899999999999999},"max":{"1min":8.9369999999999994,"5min":13.148999999999999,"15min":30.704999999999998},"last":1.891}]},{"osd":4,"last update":"Sat Sep 23 20:40:00 2023","interfaces":[{"interface":"back","average":{"1min":1.587,"5min":1.8460000000000001,"15min":1.931},"min":{"1min":0.33300000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":6.6849999999999996,"5min":9.2430000000000003,"15min":16.797999999999998},"last":4.5019999999999998},{"interface":"front","average":{"1min":1.4530000000000001,"5min":2.0009999999999999,"15min":2.117},"min":{"1min":0.38,"5min":0.307,"15min":0.27600000000000002},"max":{"1min":7.9400000000000004,"5min":19.440999999999999,"15min":19.440999999999999},"last":3.5299999999999998}]},{"osd":5,"last update":"Sat Sep 23 20:40:00 2023","interfaces":[{"interface":"back","average":{"1min":1.7290000000000001,"5min":1.8680000000000001,"15min":2.4329999999999998},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.309},"max":{"1min":6.6319999999999997,"5min":9.1799999999999997,"15min":95.393000000000001},"last":3.3439999999999999},{"interface":"front","average":{"1min":1.7110000000000001,"5min":2.0230000000000001,"15min":2.694},"min":{"1min":0.42399999999999999,"5min":0.35299999999999998,"15min":0.32300000000000001},"max":{"1min":7.8179999999999996,"5min":19.513000000000002,"15min":95.414000000000001},"last":3.5819999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:40:00 2023","interfaces":[{"interface":"back","average":{"1min":1.6339999999999999,"5min":1.9730000000000001,"15min":2.1909999999999998},"min":{"1min":0.47099999999999997,"5min":0.37,"15min":0.32100000000000001},"max":{"1min":7.8579999999999997,"5min":19.486000000000001,"15min":19.486000000000001},"last":3.492},{"interface":"front","average":{"1min":1.512,"5min":1.4550000000000001,"15min":2.0329999999999999},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.316},"max":{"1min":9.3010000000000002,"5min":10.811,"15min":30.670999999999999},"last":1.927}]},{"osd":7,"last update":"Sat Sep 23 20:40:00 2023","interfaces":[{"interface":"back","average":{"1min":1.073,"5min":1.3520000000000001,"15min":2.028},"min":{"1min":0.42499999999999999,"5min":0.39300000000000002,"15min":0.35199999999999998},"max":{"1min":4.4260000000000002,"5min":10.231,"15min":30.741},"last":3.7309999999999999},{"interface":"front","average":{"1min":1.752,"5min":1.9650000000000001,"15min":2.1760000000000002},"min":{"1min":0.44,"5min":0.40999999999999998,"15min":0.38},"max":{"1min":6.7220000000000004,"5min":9.2720000000000002,"15min":16.719000000000001},"last":5.5380000000000003}]}]},{"osd":3,"up_from":109,"seq":468151435490,"num_pgs":63,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1124856,"kb_used_data":323504,"kb_used_omap":4,"kb_used_meta":801211,"kb_avail":92620296,"statfs":{"total":95995035648,"available":94843183104,"internally_reserved":0,"allocated":331268096,"data_stored":329027154,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4514,"internal_metadata":820440670},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,2,1,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:40:26 2023","interfaces":[{"interface":"back","average":{"1min":1.409,"5min":1.7350000000000001,"15min":2.544},"min":{"1min":0.21199999999999999,"5min":0.21199999999999999,"15min":0.186},"max":{"1min":7.6879999999999997,"5min":14.638,"15min":67.305999999999997},"last":22.32},{"interface":"front","average":{"1min":2.4540000000000002,"5min":1.865,"15min":2.415},"min":{"1min":0.29499999999999998,"5min":0.26600000000000001,"15min":0.19500000000000001},"max":{"1min":8.8439999999999994,"5min":10.140000000000001,"15min":67.403999999999996},"last":22.073}]},{"osd":1,"last update":"Sat Sep 23 20:39:59 2023","interfaces":[{"interface":"back","average":{"1min":0.81699999999999995,"5min":1.3169999999999999,"15min":2.375},"min":{"1min":0.246,"5min":0.23999999999999999,"15min":0.222},"max":{"1min":7.6219999999999999,"5min":9.8940000000000001,"15min":16.189},"last":2.7210000000000001},{"interface":"front","average":{"1min":2.343,"5min":2.1669999999999998,"15min":2.6829999999999998},"min":{"1min":0.26500000000000001,"5min":0.26200000000000001,"15min":0.24199999999999999},"max":{"1min":9.1590000000000007,"5min":13.727,"15min":32.603999999999999},"last":0.97299999999999998}]},{"osd":2,"last update":"Sat Sep 23 20:40:26 2023","interfaces":[{"interface":"back","average":{"1min":2.3140000000000001,"5min":1.837,"15min":2.5030000000000001},"min":{"1min":0.26600000000000001,"5min":0.246,"15min":0.22800000000000001},"max":{"1min":6.3200000000000003,"5min":10.747999999999999,"15min":32.857999999999997},"last":21.992999999999999},{"interface":"front","average":{"1min":1.516,"5min":2.0259999999999998,"15min":2.4980000000000002},"min":{"1min":0.31,"5min":0.28100000000000003,"15min":0.20899999999999999},"max":{"1min":8.5530000000000008,"5min":22.32,"15min":24.146999999999998},"last":0.39000000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:40:26 2023","interfaces":[{"interface":"back","average":{"1min":0.82199999999999995,"5min":1.0860000000000001,"15min":1.786},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.29799999999999999},"max":{"1min":2.5899999999999999,"5min":9.3160000000000007,"15min":46.042999999999999},"last":22.157},{"interface":"front","average":{"1min":1.6359999999999999,"5min":1.879,"15min":2.4340000000000002},"min":{"1min":0.33200000000000002,"5min":0.33000000000000002,"15min":0.28100000000000003},"max":{"1min":5.141,"5min":13.571999999999999,"15min":46.579999999999998},"last":0.42599999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:40:26 2023","interfaces":[{"interface":"back","average":{"1min":0.82099999999999995,"5min":1.167,"15min":1.998},"min":{"1min":0.32700000000000001,"5min":0.32300000000000001,"15min":0.31900000000000001},"max":{"1min":2.9889999999999999,"5min":9.4130000000000003,"15min":45.292000000000002},"last":22.289999999999999},{"interface":"front","average":{"1min":1.6859999999999999,"5min":1.482,"15min":1.968},"min":{"1min":0.38500000000000001,"5min":0.34399999999999997,"15min":0.313},"max":{"1min":5.0060000000000002,"5min":9.0839999999999996,"15min":44.679000000000002},"last":1.046}]},{"osd":6,"last update":"Sat Sep 23 20:40:26 2023","interfaces":[{"interface":"back","average":{"1min":1.8640000000000001,"5min":1.3959999999999999,"15min":1.964},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":7.3630000000000004,"5min":9.0470000000000006,"15min":17.524999999999999},"last":1.0069999999999999},{"interface":"front","average":{"1min":1.4750000000000001,"5min":1.8839999999999999,"15min":2.165},"min":{"1min":0.434,"5min":0.39800000000000002,"15min":0.33700000000000002},"max":{"1min":4.8819999999999997,"5min":13.500999999999999,"15min":19.524000000000001},"last":0.45500000000000002}]},{"osd":7,"last update":"Sat Sep 23 20:40:26 2023","interfaces":[{"interface":"back","average":{"1min":1.5109999999999999,"5min":2.1829999999999998,"15min":2.3959999999999999},"min":{"1min":0.433,"5min":0.39900000000000002,"15min":0.34999999999999998},"max":{"1min":4.319,"5min":13.544,"15min":19.571999999999999},"last":0.48499999999999999},{"interface":"front","average":{"1min":0.996,"5min":1.274,"15min":2.0720000000000001},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.371},"max":{"1min":4.157,"5min":9.3719999999999999,"15min":45.387999999999998},"last":22.202999999999999}]}]},{"osd":4,"up_from":30,"seq":128849019243,"num_pgs":61,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":670496,"kb_used_data":312472,"kb_used_omap":0,"kb_used_meta":358016,"kb_avail":93074656,"statfs":{"total":95995035648,"available":95308447744,"internally_reserved":0,"allocated":319971328,"data_stored":318566271,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":366608384},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,1,1,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:39:48 2023","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":1.6910000000000001,"15min":1.6819999999999999},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.222},"max":{"1min":2.4279999999999999,"5min":24.114999999999998,"15min":24.114999999999998},"last":0.33200000000000002},{"interface":"front","average":{"1min":1.145,"5min":1.829,"15min":1.8580000000000001},"min":{"1min":0.29899999999999999,"5min":0.246,"15min":0.246},"max":{"1min":6.9119999999999999,"5min":24.135999999999999,"15min":24.135999999999999},"last":0.74099999999999999}]},{"osd":1,"last update":"Sat Sep 23 20:40:06 2023","interfaces":[{"interface":"back","average":{"1min":0.95899999999999996,"5min":1.331,"15min":1.7410000000000001},"min":{"1min":0.309,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":4.1669999999999998,"5min":18.190999999999999,"15min":18.190999999999999},"last":0.29499999999999998},{"interface":"front","average":{"1min":0.88300000000000001,"5min":1.4259999999999999,"15min":1.6899999999999999},"min":{"1min":0.307,"5min":0.28499999999999998,"15min":0.28199999999999997},"max":{"1min":3.7109999999999999,"5min":18.173999999999999,"15min":18.173999999999999},"last":0.378}]},{"osd":2,"last update":"Sat Sep 23 20:40:03 2023","interfaces":[{"interface":"back","average":{"1min":1.3400000000000001,"5min":1.6060000000000001,"15min":1.8740000000000001},"min":{"1min":0.34200000000000003,"5min":0.29999999999999999,"15min":0.26600000000000001},"max":{"1min":3.8010000000000002,"5min":10.416,"15min":32.363},"last":0.83199999999999996},{"interface":"front","average":{"1min":2.2069999999999999,"5min":2.1019999999999999,"15min":2.3109999999999999},"min":{"1min":0.34300000000000003,"5min":0.28899999999999998,"15min":0.27600000000000002},"max":{"1min":11.917999999999999,"5min":11.917999999999999,"15min":34.042999999999999},"last":0.92700000000000005}]},{"osd":3,"last update":"Sat Sep 23 20:40:17 2023","interfaces":[{"interface":"back","average":{"1min":2.4209999999999998,"5min":1.8480000000000001,"15min":2.113},"min":{"1min":0.40500000000000003,"5min":0.372,"15min":0.29099999999999998},"max":{"1min":12.147,"5min":12.147,"15min":32.332999999999998},"last":0.72899999999999998},{"interface":"front","average":{"1min":1.8280000000000001,"5min":1.8360000000000001,"15min":2.206},"min":{"1min":0.371,"5min":0.371,"15min":0.30299999999999999},"max":{"1min":11.914,"5min":11.914,"15min":32.399999999999999},"last":3.125}]},{"osd":5,"last update":"Sat Sep 23 20:39:43 2023","interfaces":[{"interface":"back","average":{"1min":0.84499999999999997,"5min":0.69799999999999995,"15min":1.54},"min":{"1min":0.29399999999999998,"5min":0.26400000000000001,"15min":0.247},"max":{"1min":4.1260000000000003,"5min":4.1260000000000003,"15min":82.866},"last":0.34999999999999998},{"interface":"front","average":{"1min":1.3129999999999999,"5min":0.89300000000000002,"15min":1.528},"min":{"1min":0.35699999999999998,"5min":0.28999999999999998,"15min":0.27800000000000002},"max":{"1min":7.0129999999999999,"5min":7.0129999999999999,"15min":82.546000000000006},"last":0.64700000000000002}]},{"osd":6,"last update":"Sat Sep 23 20:40:03 2023","interfaces":[{"interface":"back","average":{"1min":1.722,"5min":1.032,"15min":1.381},"min":{"1min":0.32200000000000001,"5min":0.312,"15min":0.27600000000000002},"max":{"1min":14.731999999999999,"5min":14.731999999999999,"15min":15.252000000000001},"last":0.68100000000000005},{"interface":"front","average":{"1min":1.024,"5min":0.97199999999999998,"15min":1.2450000000000001},"min":{"1min":0.40500000000000003,"5min":0.29899999999999999,"15min":0.27200000000000002},"max":{"1min":4.976,"5min":7.2850000000000001,"15min":11.199},"last":0.42599999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:40:17 2023","interfaces":[{"interface":"back","average":{"1min":1.0389999999999999,"5min":1.143,"15min":1.5369999999999999},"min":{"1min":0.39100000000000001,"5min":0.318,"15min":0.318},"max":{"1min":4.21,"5min":14.534000000000001,"15min":16.427},"last":0.76900000000000002},{"interface":"front","average":{"1min":1.472,"5min":0.84599999999999997,"15min":1.3180000000000001},"min":{"1min":0.36399999999999999,"5min":0.32700000000000001,"15min":0.307},"max":{"1min":7.9390000000000001,"5min":7.9390000000000001,"15min":19.018999999999998},"last":0.39700000000000002}]}]},{"osd":5,"up_from":39,"seq":167503724904,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1984472,"kb_used_data":1605392,"kb_used_omap":0,"kb_used_meta":379008,"kb_avail":91760680,"statfs":{"total":95995035648,"available":93962936320,"internally_reserved":0,"allocated":1643921408,"data_stored":1642520629,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":388104192},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:39:48 2023","interfaces":[{"interface":"back","average":{"1min":1.4970000000000001,"5min":2.077,"15min":1.8620000000000001},"min":{"1min":0.253,"5min":0.23899999999999999,"15min":0.20699999999999999},"max":{"1min":5.3079999999999998,"5min":18.16,"15min":32.475000000000001},"last":5.5720000000000001},{"interface":"front","average":{"1min":1.3600000000000001,"5min":1.8899999999999999,"15min":1.7589999999999999},"min":{"1min":0.32100000000000001,"5min":0.29499999999999998,"15min":0.245},"max":{"1min":5.2969999999999997,"5min":9.0570000000000004,"15min":32.457000000000001},"last":0.29999999999999999}]},{"osd":1,"last update":"Sat Sep 23 20:40:07 2023","interfaces":[{"interface":"back","average":{"1min":1.6799999999999999,"5min":2.0070000000000001,"15min":1.9179999999999999},"min":{"1min":0.26800000000000002,"5min":0.26300000000000001,"15min":0.253},"max":{"1min":5.71,"5min":36.588999999999999,"15min":36.588999999999999},"last":6.0839999999999996},{"interface":"front","average":{"1min":1.2470000000000001,"5min":2.0150000000000001,"15min":1.6970000000000001},"min":{"1min":0.29799999999999999,"5min":0.27700000000000002,"15min":0.23499999999999999},"max":{"1min":5.298,"5min":36.920999999999999,"15min":36.920999999999999},"last":5.6710000000000003}]},{"osd":2,"last update":"Sat Sep 23 20:39:48 2023","interfaces":[{"interface":"back","average":{"1min":1.145,"5min":2.0099999999999998,"15min":2.073},"min":{"1min":0.33200000000000002,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":3.4670000000000001,"5min":37.011000000000003,"15min":37.011000000000003},"last":5.6230000000000002},{"interface":"front","average":{"1min":1.4570000000000001,"5min":1.9239999999999999,"15min":2.0649999999999999},"min":{"1min":0.308,"5min":0.28999999999999998,"15min":0.255},"max":{"1min":4.399,"5min":36.835000000000001,"15min":36.835000000000001},"last":5.4889999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:40:14 2023","interfaces":[{"interface":"back","average":{"1min":2.2320000000000002,"5min":2.153,"15min":2.2360000000000002},"min":{"1min":0.435,"5min":0.34699999999999998,"15min":0.25700000000000001},"max":{"1min":6.0309999999999997,"5min":36.716000000000001,"15min":36.716000000000001},"last":3.512},{"interface":"front","average":{"1min":1.5289999999999999,"5min":2.2240000000000002,"15min":2.117},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.28799999999999998},"max":{"1min":8.3599999999999994,"5min":18.504000000000001,"15min":18.504000000000001},"last":5.6470000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:39:43 2023","interfaces":[{"interface":"back","average":{"1min":0.82899999999999996,"5min":0.76800000000000002,"15min":1.329},"min":{"1min":0.28000000000000003,"5min":0.255,"15min":0.255},"max":{"1min":4.4939999999999998,"5min":6.9180000000000001,"15min":28.015999999999998},"last":0.32100000000000001},{"interface":"front","average":{"1min":0.82699999999999996,"5min":0.79200000000000004,"15min":1.1020000000000001},"min":{"1min":0.311,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":1.9059999999999999,"5min":6.8380000000000001,"15min":12.300000000000001},"last":5.5449999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:39:54 2023","interfaces":[{"interface":"back","average":{"1min":1.329,"5min":1.02,"15min":1.2589999999999999},"min":{"1min":0.32900000000000001,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":4.7679999999999998,"5min":6.9960000000000004,"15min":12.890000000000001},"last":0.499},{"interface":"front","average":{"1min":1.1799999999999999,"5min":0.97499999999999998,"15min":1.2030000000000001},"min":{"1min":0.34000000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":4.7199999999999998,"5min":6.7930000000000001,"15min":16.265999999999998},"last":5.5010000000000003}]},{"osd":7,"last update":"Sat Sep 23 20:40:14 2023","interfaces":[{"interface":"back","average":{"1min":1.3939999999999999,"5min":0.84299999999999997,"15min":1.244},"min":{"1min":0.38400000000000001,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":5.2430000000000003,"5min":5.2430000000000003,"15min":36.621000000000002},"last":5.5979999999999999},{"interface":"front","average":{"1min":1.0189999999999999,"5min":0.90700000000000003,"15min":1.151},"min":{"1min":0.35699999999999998,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":4.3250000000000002,"5min":6.5970000000000004,"15min":37.331000000000003},"last":1.887}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4722,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1360,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8206,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7156,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3154,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":281214976,"data_stored":281214976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":241045504,"data_stored":241041432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":229380096,"data_stored":229376024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":317456384,"data_stored":317456384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":302448640,"data_stored":302448640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1633288192,"data_stored":1633288192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":243073024,"data_stored":243073024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":185339904,"data_stored":185335832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:40:32.429 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-23T20:40:32.460 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:40:32.468 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:32.469 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:40:32.470 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T20:40:32.570 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:40:32.711 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:32.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:32 smithi018 ceph-mon[179647]: from='client.46166 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:40:32.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:32 smithi018 ceph-mon[184144]: from='client.46166 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:40:32.968 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:32 smithi149 ceph-mon[170394]: from='client.46166 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:40:33.690 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:33 smithi018 ceph-mon[179647]: pgmap v1094: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 5 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 465 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 699 op/s; 15618/22185 objects degraded (70.399%); 1492/22185 objects misplaced (6.725%); 2.2 MiB/s, 100 objects/s recovering 2023-09-23T20:40:33.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:33 smithi149 ceph-mon[170394]: pgmap v1094: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 5 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 465 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 699 op/s; 15618/22185 objects degraded (70.399%); 1492/22185 objects misplaced (6.725%); 2.2 MiB/s, 100 objects/s recovering 2023-09-23T20:40:34.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:33 smithi018 ceph-mon[184144]: pgmap v1094: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 5 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 465 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 699 op/s; 15618/22185 objects degraded (70.399%); 1492/22185 objects misplaced (6.725%); 2.2 MiB/s, 100 objects/s recovering 2023-09-23T20:40:34.591 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:40:34.600 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:34.601 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:40:34.702 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:40:34.808 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:34.989 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:40:35.060 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:35.060 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:40:35.140 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:40:35.154 INFO:teuthology.orchestra.run.smithi018.stdout:ERROR: (22) Invalid argument 2023-09-23T20:40:35.154 INFO:teuthology.orchestra.run.smithi018.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:40:35.161 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:40:35.170 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:40:35.248 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:40:34 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:40:34] "GET /metrics HTTP/1.1" 200 36541 "" "Prometheus/2.43.0" 2023-09-23T20:40:35.249 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:35 smithi018 ceph-mon[184144]: pgmap v1095: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 5 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 374 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 744 op/s; 15618/17847 objects degraded (87.511%); 1107/17847 objects misplaced (6.203%); 1.3 MiB/s, 84 objects/s recovering 2023-09-23T20:40:35.249 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:35 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:40:35.249 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:35 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:40:35.249 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:35 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:40:35.249 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:35 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:40:35.249 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:35 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:40:35.250 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:35 smithi018 ceph-mon[179647]: pgmap v1095: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 5 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 374 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 744 op/s; 15618/17847 objects degraded (87.511%); 1107/17847 objects misplaced (6.203%); 1.3 MiB/s, 84 objects/s recovering 2023-09-23T20:40:35.251 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:35 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:40:35.251 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:35 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:40:35.251 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:35 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:40:35.251 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:35 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:40:35.251 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:35 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:40:35.253 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:35.466 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:40:35.469 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:35 smithi149 ceph-mon[170394]: pgmap v1095: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 5 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 374 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 744 op/s; 15618/17847 objects degraded (87.511%); 1107/17847 objects misplaced (6.203%); 1.3 MiB/s, 84 objects/s recovering 2023-09-23T20:40:35.469 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:35 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:40:35.470 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:35 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:40:35.470 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:35 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:40:35.470 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:35 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:40:35.470 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:35 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:40:35.478 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:40:35.482 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:35.482 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:40:35.583 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:40:35.684 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:35.840 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T20:40:36.016 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:40:36.035 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:36.036 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:40:36.137 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:40:36.274 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:36.358 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:40:36.363 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:36.364 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:40:36.465 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:40:36.505 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:36.560 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:40:36.570 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:36.571 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:40:36.671 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:40:36.751 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:36.901 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:40:36.909 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:36.909 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:40:37.010 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:40:37.100 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:37.175 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:40:37.181 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:37.181 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:40:37.220 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:40:37.281 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:40:37.326 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:37.389 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:40:37.392 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:37.393 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:40:37.455 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:37 smithi018 ceph-mon[179647]: pgmap v1096: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 283 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 774 op/s; 15424/13461 objects degraded (114.583%); 862/13461 objects misplaced (6.404%); 1.5 MiB/s, 69 objects/s recovering 2023-09-23T20:40:37.455 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:37 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 15424/13461 objects degraded (114.583%), 9 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T20:40:37.455 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:37 smithi018 ceph-mon[184144]: pgmap v1096: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 283 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 774 op/s; 15424/13461 objects degraded (114.583%); 862/13461 objects misplaced (6.404%); 1.5 MiB/s, 69 objects/s recovering 2023-09-23T20:40:37.455 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:37 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 15424/13461 objects degraded (114.583%), 9 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T20:40:37.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:37 smithi149 ceph-mon[170394]: pgmap v1096: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 283 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 774 op/s; 15424/13461 objects degraded (114.583%); 862/13461 objects misplaced (6.404%); 1.5 MiB/s, 69 objects/s recovering 2023-09-23T20:40:37.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:37 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 15424/13461 objects degraded (114.583%), 9 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T20:40:37.493 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:40:37.552 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:37.612 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:40:37.623 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:37.623 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:40:37.724 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:40:37.756 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:40:37.871 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:40:37.871 INFO:teuthology.orchestra.run.smithi018.stdout:{"fsid":"a3b3b17e-5a4c-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 15424/13461 objects degraded (114.583%), 9 pgs degraded, 4 pgs undersized","count":13},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1314,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":445,"num_osds":8,"num_up_osds":8,"osd_up_since":1695500984,"num_in_osds":8,"osd_in_since":1695501516,"num_remapped_pgs":16},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":113},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":9},{"state_name":"active+remapped+backfill_wait","count":5},{"state_name":"active+recovering+undersized+remapped","count":1},{"state_name":"active+recovery_wait+remapped","count":1}],"num_pgs":129,"num_pools":5,"num_objects":4487,"data_bytes":296776559,"bytes_used":6764433408,"bytes_avail":761195851776,"bytes_total":767960285184,"degraded_objects":15424,"degraded_total":13461,"degraded_ratio":1.1458286902904689,"misplaced_objects":862,"misplaced_total":13461,"misplaced_ratio":0.064036847188173235,"recovering_objects_per_sec":69,"recovering_bytes_per_sec":1624610,"recovering_keys_per_sec":0,"num_objects_recovered":695,"num_bytes_recovered":16252928,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":774},"fsmap":{"epoch":9,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi018.hduiol","status":"up:active","gid":24566}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.18:8443/","prometheus":"http://172.21.15.18:9283/"}},"servicemap":{"epoch":23,"modified":"2023-09-23T20:34:43.862209+0000","services":{}},"progress_events":{"8e7eff18-0e45-4130-a6e1-662b3902f0d2":{"message":"Upgrade to 18.2.0-460-gbb0daeff (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true},"aefdb80c-9ae7-446c-b728-33511e08db43":{"message":"Global Recovery Event (3m)\n [========================....] (remaining: 25s)","progress":0.87596899271011353,"add_to_ceph_s":true}}} 2023-09-23T20:40:37.881 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:37.998 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:40:38.004 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:38.004 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:40:38.105 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:40:38.152 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:38.217 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:40:38.223 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:38.223 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:40:38.258 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:38 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/514979320' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T20:40:38.258 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:38 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/514979320' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T20:40:38.324 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:40:38.363 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:38.427 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:40:38.436 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:38 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/514979320' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T20:40:38.436 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:38.437 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:40:38.537 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:40:38.584 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:38.639 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:40:38.648 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:38.649 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:40:38.750 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:40:38.833 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:38.913 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:40:38.921 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:38.921 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:40:39.020 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-09-23T20:40:39.020 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:40:39.022 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:40:39.099 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:39.177 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:40:39.181 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:39.181 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:40:39.198 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Removed 89320 objects 2023-09-23T20:40:39.198 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Clean up completed and total clean up time :172.137 2023-09-23T20:40:39.236 INFO:teuthology.orchestra.run.smithi018.stderr:noscrub is set 2023-09-23T20:40:39.282 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:40:39.324 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:39.370 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:39 smithi018 ceph-mon[184144]: pgmap v1097: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 5 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 195 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 771 op/s; 15424/9219 objects degraded (167.307%); 736/9219 objects misplaced (7.984%); 640 KiB/s, 76 objects/s recovering 2023-09-23T20:40:39.370 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:39 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3066636435' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:40:39.371 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:39 smithi018 ceph-mon[179647]: pgmap v1097: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 5 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 195 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 771 op/s; 15424/9219 objects degraded (167.307%); 736/9219 objects misplaced (7.984%); 640 KiB/s, 76 objects/s recovering 2023-09-23T20:40:39.371 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:39 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3066636435' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:40:39.649 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:40:39.652 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:39.653 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:40:39.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:39 smithi149 ceph-mon[170394]: pgmap v1097: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 5 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 195 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 771 op/s; 15424/9219 objects degraded (167.307%); 736/9219 objects misplaced (7.984%); 640 KiB/s, 76 objects/s recovering 2023-09-23T20:40:39.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:39 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3066636435' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:40:39.754 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:40:39.785 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:39.925 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:40:39.930 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:39.931 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:40:39.932 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:40:39.955 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:40:39.977 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:40:40.003 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:40:40.026 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:40:40.031 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:40:40.049 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:40:40.066 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:40.074 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:40:40.104 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:40:40.106 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:40:40.106 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:40:40.106 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:40:40.106 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:40:40.106 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:40:40.106 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:40:40.106 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:40:40.106 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:40:40.111 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:40:40.115 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:40.116 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:40:40.149 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:40:40.217 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:40:40.248 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:40.297 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2023-09-23T20:40:40.297 INFO:teuthology.orchestra.run.smithi018.stdout: "size": 20, 2023-09-23T20:40:40.297 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 600, 2023-09-23T20:40:40.297 INFO:teuthology.orchestra.run.smithi018.stdout: "ops": [ 2023-09-23T20:40:40.297 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.297 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:57605 7.3 7:c29712e0:::benchmark_data_smithi018_308601_object57604:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e311)", 2023-09-23T20:40:40.298 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:32:32.219481+0000", 2023-09-23T20:40:40.298 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 487.93896170800002, 2023-09-23T20:40:40.298 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.3860234459999998, 2023-09-23T20:40:40.298 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:40:40.298 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:40:40.298 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:40:40.298 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:40:40.298 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:40:40.298 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 57605 2023-09-23T20:40:40.298 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.299 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:40:40.299 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.299 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:40:40.300 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.219481+0000", 2023-09-23T20:40:40.300 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:40:40.300 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.300 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.300 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:40:40.300 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.219481+0000", 2023-09-23T20:40:40.300 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:40:40.300 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.300 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.300 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:40:40.301 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.219480+0000", 2023-09-23T20:40:40.301 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:40:40.301 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.301 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.301 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:40:40.301 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.219513+0000", 2023-09-23T20:40:40.301 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.3324999999999997e-05 2023-09-23T20:40:40.301 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.301 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.301 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:40:40.301 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.219526+0000", 2023-09-23T20:40:40.301 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3145999999999999e-05 2023-09-23T20:40:40.302 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.302 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.302 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:40:40.302 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.219534+0000", 2023-09-23T20:40:40.302 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.729e-06 2023-09-23T20:40:40.302 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.302 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.303 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:40:40.303 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.231964+0000", 2023-09-23T20:40:40.303 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.012430274999999999 2023-09-23T20:40:40.303 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.303 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.303 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:40:40.304 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.231974+0000", 2023-09-23T20:40:40.304 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.4939999999999992e-06 2023-09-23T20:40:40.304 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.304 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.304 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:40:40.304 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.215331+0000", 2023-09-23T20:40:40.304 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.983357276 2023-09-23T20:40:40.305 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.305 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.305 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:40:40.305 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.215344+0000", 2023-09-23T20:40:40.305 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3061000000000001e-05 2023-09-23T20:40:40.305 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.306 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.306 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:40:40.306 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.573779+0000", 2023-09-23T20:40:40.306 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.358434581 2023-09-23T20:40:40.306 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.306 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.307 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:40:40.307 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.573953+0000", 2023-09-23T20:40:40.307 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00017463999999999999 2023-09-23T20:40:40.307 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.307 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.307 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 2,4", 2023-09-23T20:40:40.307 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.574063+0000", 2023-09-23T20:40:40.308 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000110059 2023-09-23T20:40:40.308 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.308 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.308 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:40:40.308 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.578839+0000", 2023-09-23T20:40:40.308 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0047753709999999996 2023-09-23T20:40:40.308 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.309 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.309 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:40:40.309 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.578852+0000", 2023-09-23T20:40:40.309 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3360000000000001e-05 2023-09-23T20:40:40.309 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.309 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.310 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:40:40.310 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.602935+0000", 2023-09-23T20:40:40.310 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.024082883999999999 2023-09-23T20:40:40.310 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.310 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.310 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:40:40.311 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.602997+0000", 2023-09-23T20:40:40.311 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.2113000000000007e-05 2023-09-23T20:40:40.311 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.311 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.311 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:40:40.311 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.605505+0000", 2023-09-23T20:40:40.311 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.002507493 2023-09-23T20:40:40.311 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:40:40.312 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:40:40.312 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:40:40.312 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.312 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.312 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:57610 7.3 7:cbaa52d6:::benchmark_data_smithi018_308601_object57609:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e311)", 2023-09-23T20:40:40.312 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:32:32.219584+0000", 2023-09-23T20:40:40.312 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 487.938859073, 2023-09-23T20:40:40.312 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.38833787, 2023-09-23T20:40:40.312 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:40:40.312 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:40:40.312 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:40:40.313 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:40:40.313 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:40:40.313 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 57610 2023-09-23T20:40:40.313 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.313 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:40:40.313 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.313 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:40:40.313 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.219584+0000", 2023-09-23T20:40:40.313 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:40:40.313 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.313 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.314 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:40:40.314 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.219584+0000", 2023-09-23T20:40:40.314 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:40:40.314 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.314 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.314 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:40:40.314 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.219583+0000", 2023-09-23T20:40:40.314 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:40:40.314 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.314 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.314 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:40:40.315 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.219607+0000", 2023-09-23T20:40:40.315 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.4170000000000001e-05 2023-09-23T20:40:40.315 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.315 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.315 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:40:40.315 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.219614+0000", 2023-09-23T20:40:40.315 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.4790000000000001e-06 2023-09-23T20:40:40.315 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.315 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.315 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:40:40.315 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.219619+0000", 2023-09-23T20:40:40.315 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.079e-06 2023-09-23T20:40:40.316 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.316 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.316 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:40:40.316 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.231994+0000", 2023-09-23T20:40:40.316 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.012374484999999999 2023-09-23T20:40:40.316 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.316 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.316 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:40:40.316 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.232000+0000", 2023-09-23T20:40:40.316 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.4790000000000001e-06 2023-09-23T20:40:40.316 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.316 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.316 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:40:40.317 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.215369+0000", 2023-09-23T20:40:40.317 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.98336916699999999 2023-09-23T20:40:40.317 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.317 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.317 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:40:40.317 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.215378+0000", 2023-09-23T20:40:40.317 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.6479999999999992e-06 2023-09-23T20:40:40.317 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.317 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.317 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:40:40.317 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.577198+0000", 2023-09-23T20:40:40.317 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.36182066 2023-09-23T20:40:40.318 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.318 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.318 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:40:40.318 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.577312+0000", 2023-09-23T20:40:40.318 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000113257 2023-09-23T20:40:40.318 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.318 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.318 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 2,4", 2023-09-23T20:40:40.318 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.577424+0000", 2023-09-23T20:40:40.318 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00011234999999999999 2023-09-23T20:40:40.318 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.318 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.319 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:40:40.320 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.580718+0000", 2023-09-23T20:40:40.320 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0032940090000000001 2023-09-23T20:40:40.320 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.320 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.320 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:40:40.320 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.582068+0000", 2023-09-23T20:40:40.320 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0013497990000000001 2023-09-23T20:40:40.320 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.320 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.321 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:40:40.321 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.605587+0000", 2023-09-23T20:40:40.321 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.023518779 2023-09-23T20:40:40.321 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.321 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.321 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:40:40.321 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.605666+0000", 2023-09-23T20:40:40.321 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.9777999999999994e-05 2023-09-23T20:40:40.321 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.321 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.321 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:40:40.322 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.607922+0000", 2023-09-23T20:40:40.322 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00225515 2023-09-23T20:40:40.322 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:40:40.322 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:40:40.322 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:40:40.322 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.322 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.322 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:57645 7.3 7:cb6be1ad:::benchmark_data_smithi018_308601_object57644:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e312)", 2023-09-23T20:40:40.322 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:32:32.275903+0000", 2023-09-23T20:40:40.322 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 487.88253971900002, 2023-09-23T20:40:40.323 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.3348626370000001, 2023-09-23T20:40:40.323 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:40:40.323 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:40:40.323 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:40:40.323 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:40:40.323 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:40:40.323 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 57645 2023-09-23T20:40:40.323 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.323 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:40:40.323 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.323 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:40:40.324 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.275903+0000", 2023-09-23T20:40:40.324 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:40:40.324 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.324 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.324 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:40:40.324 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.275903+0000", 2023-09-23T20:40:40.324 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:40:40.324 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.324 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.324 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:40:40.324 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.275902+0000", 2023-09-23T20:40:40.324 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:40:40.325 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.325 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.325 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:40:40.325 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.275926+0000", 2023-09-23T20:40:40.325 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.4209000000000001e-05 2023-09-23T20:40:40.325 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.325 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.325 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:40:40.325 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.275938+0000", 2023-09-23T20:40:40.325 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1992e-05 2023-09-23T20:40:40.325 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.326 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.326 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:40:40.326 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.275946+0000", 2023-09-23T20:40:40.326 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.8839999999999994e-06 2023-09-23T20:40:40.326 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.326 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.326 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:40:40.326 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.276851+0000", 2023-09-23T20:40:40.326 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00090492800000000005 2023-09-23T20:40:40.326 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.326 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.327 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:40:40.327 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.276861+0000", 2023-09-23T20:40:40.327 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0208e-05 2023-09-23T20:40:40.327 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.327 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.327 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:40:40.327 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.215401+0000", 2023-09-23T20:40:40.327 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.93854029800000005 2023-09-23T20:40:40.327 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.328 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.328 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:40:40.328 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.215410+0000", 2023-09-23T20:40:40.328 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.3510000000000002e-06 2023-09-23T20:40:40.328 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.330 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.330 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:40:40.330 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.578900+0000", 2023-09-23T20:40:40.330 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.363490173 2023-09-23T20:40:40.330 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.330 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.330 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:40:40.331 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.579070+0000", 2023-09-23T20:40:40.331 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00017048899999999999 2023-09-23T20:40:40.331 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.331 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.331 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 2,4", 2023-09-23T20:40:40.331 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.579260+0000", 2023-09-23T20:40:40.331 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000189667 2023-09-23T20:40:40.331 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.331 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.331 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:40:40.331 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.583356+0000", 2023-09-23T20:40:40.331 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0040960520000000002 2023-09-23T20:40:40.332 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.332 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.332 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:40:40.332 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.583406+0000", 2023-09-23T20:40:40.332 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.9449e-05 2023-09-23T20:40:40.332 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.332 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.332 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:40:40.333 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.607983+0000", 2023-09-23T20:40:40.333 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.024577576 2023-09-23T20:40:40.333 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.333 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.333 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:40:40.333 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.608038+0000", 2023-09-23T20:40:40.333 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.4787000000000002e-05 2023-09-23T20:40:40.333 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.333 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.333 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:40:40.333 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.610766+0000", 2023-09-23T20:40:40.334 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0027278319999999999 2023-09-23T20:40:40.334 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:40:40.334 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:40:40.334 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:40:40.334 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.334 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.334 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:57662 7.3 7:ce017a4a:::benchmark_data_smithi018_308601_object57661:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e312)", 2023-09-23T20:40:40.334 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:32:32.295077+0000", 2023-09-23T20:40:40.334 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 487.86336564200002, 2023-09-23T20:40:40.334 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.3221079059999998, 2023-09-23T20:40:40.334 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:40:40.334 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:40:40.335 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:40:40.335 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:40:40.335 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:40:40.335 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 57662 2023-09-23T20:40:40.335 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.336 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:40:40.336 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.336 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:40:40.336 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.295077+0000", 2023-09-23T20:40:40.336 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:40:40.336 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.336 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.336 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:40:40.336 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.295077+0000", 2023-09-23T20:40:40.336 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:40:40.336 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.337 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.337 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:40:40.337 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.295075+0000", 2023-09-23T20:40:40.337 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:40:40.337 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.337 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.337 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:40:40.337 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.295104+0000", 2023-09-23T20:40:40.337 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.8374999999999999e-05 2023-09-23T20:40:40.337 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.337 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.337 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:40:40.338 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.295117+0000", 2023-09-23T20:40:40.338 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2943e-05 2023-09-23T20:40:40.338 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.338 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.338 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:40:40.338 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.295123+0000", 2023-09-23T20:40:40.338 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.5989999999999998e-06 2023-09-23T20:40:40.338 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.338 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.338 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:40:40.338 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.295189+0000", 2023-09-23T20:40:40.339 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.5988000000000006e-05 2023-09-23T20:40:40.339 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.339 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.339 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:40:40.339 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.295205+0000", 2023-09-23T20:40:40.339 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.5434000000000002e-05 2023-09-23T20:40:40.339 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.339 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.339 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:40:40.339 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.215432+0000", 2023-09-23T20:40:40.340 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.92022715499999996 2023-09-23T20:40:40.340 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.340 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.340 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:40:40.340 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.215440+0000", 2023-09-23T20:40:40.340 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.2519999999999999e-06 2023-09-23T20:40:40.340 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.340 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.340 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:40:40.340 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.580753+0000", 2023-09-23T20:40:40.340 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.365312673 2023-09-23T20:40:40.340 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.341 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.341 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:40:40.341 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.580896+0000", 2023-09-23T20:40:40.341 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00014352000000000001 2023-09-23T20:40:40.341 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.341 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.341 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 2,4", 2023-09-23T20:40:40.341 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.581023+0000", 2023-09-23T20:40:40.341 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000127112 2023-09-23T20:40:40.341 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.341 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.341 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:40:40.342 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.583459+0000", 2023-09-23T20:40:40.342 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0024356450000000002 2023-09-23T20:40:40.342 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.342 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.342 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:40:40.342 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.583468+0000", 2023-09-23T20:40:40.342 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.4840000000000004e-06 2023-09-23T20:40:40.342 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.342 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.342 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:40:40.343 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.613992+0000", 2023-09-23T20:40:40.343 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.030524193000000002 2023-09-23T20:40:40.343 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.343 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.343 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:40:40.343 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.614070+0000", 2023-09-23T20:40:40.343 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.8054999999999997e-05 2023-09-23T20:40:40.343 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.343 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.343 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:40:40.343 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.617185+0000", 2023-09-23T20:40:40.343 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0031151690000000001 2023-09-23T20:40:40.344 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:40:40.344 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:40:40.344 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:40:40.344 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.344 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.344 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:57667 7.3 7:cbc699af:::benchmark_data_smithi018_308601_object57666:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e312)", 2023-09-23T20:40:40.344 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:32:32.300970+0000", 2023-09-23T20:40:40.344 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 487.85747268900002, 2023-09-23T20:40:40.344 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.3193818780000002, 2023-09-23T20:40:40.344 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:40:40.344 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:40:40.344 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:40:40.345 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:40:40.345 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:40:40.345 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 57667 2023-09-23T20:40:40.345 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.345 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:40:40.345 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.345 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:40:40.345 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.300970+0000", 2023-09-23T20:40:40.345 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:40:40.345 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.345 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.345 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:40:40.346 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.300970+0000", 2023-09-23T20:40:40.346 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:40:40.346 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.346 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.346 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:40:40.346 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.300969+0000", 2023-09-23T20:40:40.346 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.999999 2023-09-23T20:40:40.346 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.346 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.346 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:40:40.347 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.300992+0000", 2023-09-23T20:40:40.347 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.2997e-05 2023-09-23T20:40:40.347 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.347 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.347 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:40:40.347 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.301006+0000", 2023-09-23T20:40:40.347 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3509000000000001e-05 2023-09-23T20:40:40.347 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.347 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.347 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:40:40.347 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.301014+0000", 2023-09-23T20:40:40.348 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.9189999999999999e-06 2023-09-23T20:40:40.348 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.348 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.348 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:40:40.348 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.301060+0000", 2023-09-23T20:40:40.348 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.6597999999999997e-05 2023-09-23T20:40:40.348 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.348 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.348 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:40:40.348 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.301067+0000", 2023-09-23T20:40:40.348 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.6039999999999998e-06 2023-09-23T20:40:40.348 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.349 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.349 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:40:40.349 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.215463+0000", 2023-09-23T20:40:40.349 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.914396019 2023-09-23T20:40:40.349 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.349 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.349 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:40:40.349 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.215471+0000", 2023-09-23T20:40:40.349 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.1880000000000002e-06 2023-09-23T20:40:40.349 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.349 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.349 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:40:40.350 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.582113+0000", 2023-09-23T20:40:40.350 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.366641642 2023-09-23T20:40:40.350 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.350 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.350 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:40:40.350 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.582227+0000", 2023-09-23T20:40:40.350 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000114545 2023-09-23T20:40:40.350 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.350 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.350 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 2,4", 2023-09-23T20:40:40.350 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.582342+0000", 2023-09-23T20:40:40.350 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000114294 2023-09-23T20:40:40.351 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.351 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.351 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:40:40.351 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.584646+0000", 2023-09-23T20:40:40.351 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.002304038 2023-09-23T20:40:40.351 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.351 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.351 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:40:40.351 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.584666+0000", 2023-09-23T20:40:40.351 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.0596000000000001e-05 2023-09-23T20:40:40.351 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.351 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.351 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:40:40.352 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.617277+0000", 2023-09-23T20:40:40.352 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.032610293999999998 2023-09-23T20:40:40.352 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.352 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.352 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:40:40.352 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.617329+0000", 2023-09-23T20:40:40.352 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.2527e-05 2023-09-23T20:40:40.352 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.352 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.352 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:40:40.352 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.620352+0000", 2023-09-23T20:40:40.352 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0030229179999999999 2023-09-23T20:40:40.353 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:40:40.353 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:40:40.353 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:40:40.353 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.353 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.353 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:57675 7.3 7:c7bba963:::benchmark_data_smithi018_308601_object57674:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e312)", 2023-09-23T20:40:40.353 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:32:32.327987+0000", 2023-09-23T20:40:40.353 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 487.83045616499999, 2023-09-23T20:40:40.353 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.2956588660000001, 2023-09-23T20:40:40.353 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:40:40.353 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:40:40.353 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:40:40.353 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:40:40.354 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:40:40.354 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 57675 2023-09-23T20:40:40.354 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.354 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:40:40.354 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.354 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:40:40.354 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.327987+0000", 2023-09-23T20:40:40.354 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:40:40.354 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.354 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.355 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:40:40.355 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.327987+0000", 2023-09-23T20:40:40.355 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:40:40.355 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.355 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.355 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:40:40.355 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.327985+0000", 2023-09-23T20:40:40.355 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:40:40.355 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.355 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.355 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:40:40.355 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.328012+0000", 2023-09-23T20:40:40.356 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.6673e-05 2023-09-23T20:40:40.356 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.356 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.356 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:40:40.356 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.328033+0000", 2023-09-23T20:40:40.356 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.1865000000000002e-05 2023-09-23T20:40:40.356 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.356 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.356 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:40:40.356 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.328046+0000", 2023-09-23T20:40:40.356 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2138e-05 2023-09-23T20:40:40.357 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.357 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.357 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:40:40.357 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.328112+0000", 2023-09-23T20:40:40.357 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.6155999999999995e-05 2023-09-23T20:40:40.357 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.357 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.357 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:40:40.357 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:32.328123+0000", 2023-09-23T20:40:40.357 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1323e-05 2023-09-23T20:40:40.357 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.357 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.358 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:40:40.358 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.215494+0000", 2023-09-23T20:40:40.358 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.88737101299999999 2023-09-23T20:40:40.358 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.358 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.358 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:40:40.358 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:33.215502+0000", 2023-09-23T20:40:40.358 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.3140000000000004e-06 2023-09-23T20:40:40.358 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.358 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.358 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:40:40.359 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.583495+0000", 2023-09-23T20:40:40.359 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.367992519 2023-09-23T20:40:40.359 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.359 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.359 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:40:40.359 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.583598+0000", 2023-09-23T20:40:40.359 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00010268399999999999 2023-09-23T20:40:40.359 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.359 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.359 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 2,4", 2023-09-23T20:40:40.359 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.583704+0000", 2023-09-23T20:40:40.360 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000106096 2023-09-23T20:40:40.360 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.360 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.360 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:40:40.360 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.584776+0000", 2023-09-23T20:40:40.360 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00107215 2023-09-23T20:40:40.360 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.360 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.360 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:40:40.360 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.585126+0000", 2023-09-23T20:40:40.360 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00035052599999999998 2023-09-23T20:40:40.361 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.361 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.361 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:40:40.361 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.620444+0000", 2023-09-23T20:40:40.361 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.035318081000000001 2023-09-23T20:40:40.361 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.361 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.361 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:40:40.361 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.620519+0000", 2023-09-23T20:40:40.361 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.4976999999999997e-05 2023-09-23T20:40:40.361 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.361 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.362 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:40:40.362 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:34.623645+0000", 2023-09-23T20:40:40.362 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.003126017 2023-09-23T20:40:40.362 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:40:40.362 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:40:40.362 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:40:40.362 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.362 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.362 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:69129 7.3 7:cf914298:::benchmark_data_smithi018_308601_object69128:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e318)", 2023-09-23T20:40:40.362 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:32:49.475881+0000", 2023-09-23T20:40:40.362 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 470.68256177699999, 2023-09-23T20:40:40.363 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.048825159, 2023-09-23T20:40:40.363 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:40:40.363 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:40:40.363 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:40:40.363 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:40:40.363 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:40:40.363 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 69129 2023-09-23T20:40:40.363 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.363 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:40:40.363 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.363 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:40:40.363 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.475881+0000", 2023-09-23T20:40:40.363 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:40:40.364 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.364 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.364 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:40:40.364 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.475881+0000", 2023-09-23T20:40:40.364 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:40:40.364 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.364 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.364 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:40:40.364 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.475881+0000", 2023-09-23T20:40:40.364 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:40:40.365 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.365 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.365 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:40:40.365 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.475925+0000", 2023-09-23T20:40:40.365 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.4274000000000003e-05 2023-09-23T20:40:40.365 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.365 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.365 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:40:40.365 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.475932+0000", 2023-09-23T20:40:40.365 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.2300000000000002e-06 2023-09-23T20:40:40.365 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.365 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.366 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:40:40.366 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.475937+0000", 2023-09-23T20:40:40.366 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.4600000000000002e-06 2023-09-23T20:40:40.366 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.366 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.366 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:40:40.366 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.480114+0000", 2023-09-23T20:40:40.366 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0041766099999999999 2023-09-23T20:40:40.366 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.366 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.366 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:40:40.366 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.480293+0000", 2023-09-23T20:40:40.367 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00017894300000000001 2023-09-23T20:40:40.367 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.367 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.367 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 2,4", 2023-09-23T20:40:40.367 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.480478+0000", 2023-09-23T20:40:40.367 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00018538100000000001 2023-09-23T20:40:40.367 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.367 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.367 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:40:40.367 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.520416+0000", 2023-09-23T20:40:40.367 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.039937451999999998 2023-09-23T20:40:40.367 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.368 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.368 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:40:40.368 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.523198+0000", 2023-09-23T20:40:40.368 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0027818999999999999 2023-09-23T20:40:40.368 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.368 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.369 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:40:40.369 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.528277+0000", 2023-09-23T20:40:40.369 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0050791120000000002 2023-09-23T20:40:40.369 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.369 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.369 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:40:40.370 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.528303+0000", 2023-09-23T20:40:40.370 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.5732e-05 2023-09-23T20:40:40.370 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.370 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.370 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:40:40.370 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:50.524690+0000", 2023-09-23T20:40:40.370 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.99638690299999999 2023-09-23T20:40:40.370 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.370 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.370 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:40:40.370 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:50.524706+0000", 2023-09-23T20:40:40.371 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.6634e-05 2023-09-23T20:40:40.371 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:40:40.371 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:40:40.371 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:40:40.371 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.371 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.371 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:69152 7.3 7.f242f463 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e318)", 2023-09-23T20:40:40.371 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:32:49.520371+0000", 2023-09-23T20:40:40.371 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 470.63807208999998, 2023-09-23T20:40:40.371 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0043691610000001, 2023-09-23T20:40:40.372 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:40:40.372 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:40:40.372 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:40:40.372 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:40:40.372 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:40:40.372 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 69152 2023-09-23T20:40:40.372 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.372 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:40:40.373 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.373 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:40:40.373 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.520371+0000", 2023-09-23T20:40:40.373 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:40:40.373 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.373 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.373 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:40:40.373 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.520371+0000", 2023-09-23T20:40:40.373 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:40:40.373 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.373 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.374 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:40:40.374 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.520369+0000", 2023-09-23T20:40:40.374 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:40:40.374 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.374 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.374 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:40:40.374 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.520395+0000", 2023-09-23T20:40:40.374 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.5843999999999999e-05 2023-09-23T20:40:40.374 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.374 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.374 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:40:40.375 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.520408+0000", 2023-09-23T20:40:40.375 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3189e-05 2023-09-23T20:40:40.375 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.375 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.375 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:40:40.375 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.520417+0000", 2023-09-23T20:40:40.375 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.6899999999999998e-06 2023-09-23T20:40:40.375 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.375 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.375 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:40:40.376 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.529310+0000", 2023-09-23T20:40:40.376 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0088929609999999996 2023-09-23T20:40:40.376 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.376 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.376 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:40:40.376 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.529330+0000", 2023-09-23T20:40:40.376 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.9673e-05 2023-09-23T20:40:40.377 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.377 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.377 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:40:40.377 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:50.524730+0000", 2023-09-23T20:40:40.377 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.99540061999999996 2023-09-23T20:40:40.377 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.377 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.377 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:40:40.377 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:50.524740+0000", 2023-09-23T20:40:40.377 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.6139999999999998e-06 2023-09-23T20:40:40.377 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:40:40.378 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:40:40.378 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:40:40.378 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.378 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.378 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:69159 7.3 7.de3ad3c3 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e318)", 2023-09-23T20:40:40.378 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:32:49.534755+0000", 2023-09-23T20:40:40.378 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 470.62368735899997, 2023-09-23T20:40:40.379 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.99001591200000005, 2023-09-23T20:40:40.379 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:40:40.379 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:40:40.379 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:40:40.379 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:40:40.379 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:40:40.379 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 69159 2023-09-23T20:40:40.379 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.379 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:40:40.379 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.380 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:40:40.380 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.534755+0000", 2023-09-23T20:40:40.380 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:40:40.380 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.380 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.380 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:40:40.380 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.534755+0000", 2023-09-23T20:40:40.380 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:40:40.380 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.380 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.380 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:40:40.380 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.534754+0000", 2023-09-23T20:40:40.381 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:40:40.381 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.381 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.381 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:40:40.381 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.534782+0000", 2023-09-23T20:40:40.381 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.8348999999999999e-05 2023-09-23T20:40:40.381 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.381 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.381 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:40:40.381 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.534808+0000", 2023-09-23T20:40:40.382 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.5941000000000001e-05 2023-09-23T20:40:40.382 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.382 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.382 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:40:40.382 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.534818+0000", 2023-09-23T20:40:40.382 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0132999999999999e-05 2023-09-23T20:40:40.382 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.382 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.382 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:40:40.382 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.535076+0000", 2023-09-23T20:40:40.383 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00025796400000000001 2023-09-23T20:40:40.383 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.383 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.383 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:40:40.383 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:49.535090+0000", 2023-09-23T20:40:40.383 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3505e-05 2023-09-23T20:40:40.383 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.383 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.383 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:40:40.383 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:50.524762+0000", 2023-09-23T20:40:40.383 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.98967230399999995 2023-09-23T20:40:40.384 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.384 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.384 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:40:40.384 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:32:50.524771+0000", 2023-09-23T20:40:40.384 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.5070000000000006e-06 2023-09-23T20:40:40.384 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:40:40.384 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:40:40.384 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:40:40.384 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.384 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.384 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35994.0:105785 8.0 8.a76e08d0 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e409)", 2023-09-23T20:40:40.385 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:38:26.401578+0000", 2023-09-23T20:40:40.385 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 133.75686524400001, 2023-09-23T20:40:40.385 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.31127717599999999, 2023-09-23T20:40:40.385 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:40:40.385 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:40:40.385 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:40:40.385 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35994", 2023-09-23T20:40:40.385 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:40:40.385 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 105785 2023-09-23T20:40:40.385 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.385 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:40:40.386 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.386 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:40:40.386 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.401578+0000", 2023-09-23T20:40:40.386 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:40:40.386 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.386 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.386 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:40:40.386 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.401578+0000", 2023-09-23T20:40:40.386 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:40:40.386 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.386 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.387 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:40:40.387 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.401576+0000", 2023-09-23T20:40:40.387 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.999999 2023-09-23T20:40:40.387 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.387 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.387 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:40:40.387 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.401582+0000", 2023-09-23T20:40:40.387 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.7899999999999996e-06 2023-09-23T20:40:40.387 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.387 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.387 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:40:40.387 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.401595+0000", 2023-09-23T20:40:40.387 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2649e-05 2023-09-23T20:40:40.388 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.388 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.388 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:40:40.388 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.401602+0000", 2023-09-23T20:40:40.388 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.81e-06 2023-09-23T20:40:40.388 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.388 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.388 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:40:40.388 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.712843+0000", 2023-09-23T20:40:40.388 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.31124165199999998 2023-09-23T20:40:40.388 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.388 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.389 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:40:40.389 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.712855+0000", 2023-09-23T20:40:40.389 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1389000000000001e-05 2023-09-23T20:40:40.389 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:40:40.389 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:40:40.389 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:40:40.389 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.389 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.389 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35994.0:105785 8.10 8:0b1076e5:::benchmark_data_smithi018_341490_object16462:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e410)", 2023-09-23T20:40:40.389 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:38:26.443871+0000", 2023-09-23T20:40:40.389 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 133.714571495, 2023-09-23T20:40:40.389 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1767037899999999, 2023-09-23T20:40:40.390 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:40:40.390 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:40:40.390 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:40:40.390 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35994", 2023-09-23T20:40:40.390 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:40:40.390 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 105785 2023-09-23T20:40:40.390 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.390 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:40:40.390 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.390 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:40:40.391 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.443871+0000", 2023-09-23T20:40:40.391 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:40:40.391 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.391 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.391 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:40:40.391 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.443871+0000", 2023-09-23T20:40:40.391 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:40:40.391 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.391 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.391 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:40:40.391 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.443871+0000", 2023-09-23T20:40:40.391 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:40:40.392 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.392 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.392 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:40:40.392 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.443875+0000", 2023-09-23T20:40:40.392 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.9509999999999999e-06 2023-09-23T20:40:40.392 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.392 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.392 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:40:40.392 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.443882+0000", 2023-09-23T20:40:40.392 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.8390000000000007e-06 2023-09-23T20:40:40.392 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.392 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.393 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:40:40.393 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.443889+0000", 2023-09-23T20:40:40.393 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.5230000000000001e-06 2023-09-23T20:40:40.393 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.393 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.393 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:40:40.393 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.713126+0000", 2023-09-23T20:40:40.393 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.269236743 2023-09-23T20:40:40.393 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.393 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.393 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:40:40.393 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.713135+0000", 2023-09-23T20:40:40.394 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.0790000000000007e-06 2023-09-23T20:40:40.394 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.394 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.394 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:40:40.394 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.610121+0000", 2023-09-23T20:40:40.394 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.89698586199999997 2023-09-23T20:40:40.394 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.394 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.394 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:40:40.394 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.610236+0000", 2023-09-23T20:40:40.394 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000114833 2023-09-23T20:40:40.395 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.395 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.395 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,4", 2023-09-23T20:40:40.395 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.610355+0000", 2023-09-23T20:40:40.395 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00011921099999999999 2023-09-23T20:40:40.395 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.395 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.395 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:40:40.395 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.615886+0000", 2023-09-23T20:40:40.395 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0055312039999999996 2023-09-23T20:40:40.395 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.395 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.396 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:40:40.396 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.618509+0000", 2023-09-23T20:40:40.396 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.002623434 2023-09-23T20:40:40.396 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.396 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.396 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:40:40.396 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.619136+0000", 2023-09-23T20:40:40.396 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00062698799999999998 2023-09-23T20:40:40.396 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.396 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.397 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:40:40.397 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.619190+0000", 2023-09-23T20:40:40.397 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.3492000000000002e-05 2023-09-23T20:40:40.397 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.397 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.397 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:40:40.397 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.620575+0000", 2023-09-23T20:40:40.397 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.001385231 2023-09-23T20:40:40.397 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:40:40.397 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:40:40.397 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:40:40.398 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.398 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.398 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35994.0:105786 8.10 8:0f8943c0:::benchmark_data_smithi018_341490_object16463:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e410)", 2023-09-23T20:40:40.398 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:38:26.443896+0000", 2023-09-23T20:40:40.398 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 133.71454657300001, 2023-09-23T20:40:40.398 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1781387480000001, 2023-09-23T20:40:40.398 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:40:40.398 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:40:40.398 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:40:40.398 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35994", 2023-09-23T20:40:40.399 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:40:40.399 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 105786 2023-09-23T20:40:40.399 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.399 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:40:40.399 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.399 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:40:40.399 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.443896+0000", 2023-09-23T20:40:40.399 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:40:40.399 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.399 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.399 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:40:40.400 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.443896+0000", 2023-09-23T20:40:40.400 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:40:40.400 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.400 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.400 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:40:40.400 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.443896+0000", 2023-09-23T20:40:40.400 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:40:40.400 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.400 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.400 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:40:40.400 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.443899+0000", 2023-09-23T20:40:40.401 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.337e-06 2023-09-23T20:40:40.401 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.401 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.401 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:40:40.401 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.443905+0000", 2023-09-23T20:40:40.401 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.2260000000000002e-06 2023-09-23T20:40:40.401 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.401 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.401 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:40:40.401 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.443911+0000", 2023-09-23T20:40:40.402 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.135e-06 2023-09-23T20:40:40.402 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.402 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.402 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:40:40.402 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.713155+0000", 2023-09-23T20:40:40.402 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.26924421900000001 2023-09-23T20:40:40.402 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.402 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.402 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:40:40.402 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.713161+0000", 2023-09-23T20:40:40.403 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.7799999999999997e-06 2023-09-23T20:40:40.403 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.403 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.403 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:40:40.403 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.611522+0000", 2023-09-23T20:40:40.403 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.89836117699999996 2023-09-23T20:40:40.403 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.403 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.403 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:40:40.403 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.611650+0000", 2023-09-23T20:40:40.403 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000128677 2023-09-23T20:40:40.404 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.404 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.404 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,4", 2023-09-23T20:40:40.404 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.611781+0000", 2023-09-23T20:40:40.404 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00013079300000000001 2023-09-23T20:40:40.404 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.404 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.404 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:40:40.404 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.618515+0000", 2023-09-23T20:40:40.404 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0067336820000000004 2023-09-23T20:40:40.404 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.404 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.405 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:40:40.405 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.619100+0000", 2023-09-23T20:40:40.405 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00058515599999999998 2023-09-23T20:40:40.405 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.405 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.405 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:40:40.405 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.620654+0000", 2023-09-23T20:40:40.405 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.001553476 2023-09-23T20:40:40.405 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.405 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.405 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:40:40.405 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.620709+0000", 2023-09-23T20:40:40.406 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.5781e-05 2023-09-23T20:40:40.406 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.406 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.406 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:40:40.406 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.622035+0000", 2023-09-23T20:40:40.406 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0013256679999999999 2023-09-23T20:40:40.406 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:40:40.406 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:40:40.406 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:40:40.406 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.406 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.407 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35994.0:105787 8.0 8:015f4eb0:::benchmark_data_smithi018_341490_object16464:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e410)", 2023-09-23T20:40:40.407 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:38:26.443916+0000", 2023-09-23T20:40:40.407 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 133.71452704699999, 2023-09-23T20:40:40.407 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1854093080000001, 2023-09-23T20:40:40.407 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:40:40.407 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:40:40.407 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:40:40.407 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35994", 2023-09-23T20:40:40.407 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:40:40.407 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 105787 2023-09-23T20:40:40.407 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.407 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:40:40.408 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.408 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:40:40.408 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.443916+0000", 2023-09-23T20:40:40.408 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:40:40.408 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.408 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.408 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:40:40.408 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.443916+0000", 2023-09-23T20:40:40.408 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:40:40.408 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.408 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.409 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:40:40.409 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.443915+0000", 2023-09-23T20:40:40.409 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:40:40.409 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.409 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.409 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:40:40.409 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.443918+0000", 2023-09-23T20:40:40.409 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.509e-06 2023-09-23T20:40:40.409 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.409 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.409 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:40:40.409 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.443924+0000", 2023-09-23T20:40:40.410 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.6740000000000002e-06 2023-09-23T20:40:40.410 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.410 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.410 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:40:40.410 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.443929+0000", 2023-09-23T20:40:40.410 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.5199999999999997e-06 2023-09-23T20:40:40.410 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.410 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.410 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:40:40.410 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.713182+0000", 2023-09-23T20:40:40.411 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.26925305700000002 2023-09-23T20:40:40.411 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.411 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.412 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:40:40.412 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.713190+0000", 2023-09-23T20:40:40.412 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.7980000000000006e-06 2023-09-23T20:40:40.412 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.412 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.412 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:40:40.412 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.613650+0000", 2023-09-23T20:40:40.412 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.90046006099999998 2023-09-23T20:40:40.412 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.412 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.413 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:40:40.413 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.613744+0000", 2023-09-23T20:40:40.413 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.4349000000000001e-05 2023-09-23T20:40:40.413 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.413 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.413 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,4", 2023-09-23T20:40:40.413 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.613821+0000", 2023-09-23T20:40:40.413 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.6915000000000005e-05 2023-09-23T20:40:40.413 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.413 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.413 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:40:40.414 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.619293+0000", 2023-09-23T20:40:40.414 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0054714159999999998 2023-09-23T20:40:40.414 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.414 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.414 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:40:40.414 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.627285+0000", 2023-09-23T20:40:40.414 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0079923000000000008 2023-09-23T20:40:40.414 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.415 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.415 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:40:40.415 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.627385+0000", 2023-09-23T20:40:40.415 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.9766000000000007e-05 2023-09-23T20:40:40.415 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.415 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.415 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:40:40.415 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.627433+0000", 2023-09-23T20:40:40.415 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.7963999999999997e-05 2023-09-23T20:40:40.415 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.415 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.415 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:40:40.416 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.629325+0000", 2023-09-23T20:40:40.416 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0018923880000000001 2023-09-23T20:40:40.416 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:40:40.416 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:40:40.416 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:40:40.416 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.416 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.416 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35994.0:105791 8.10 8:0d4c47c5:::benchmark_data_smithi018_341490_object16468:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e410)", 2023-09-23T20:40:40.416 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:38:26.443934+0000", 2023-09-23T20:40:40.416 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 133.714508865, 2023-09-23T20:40:40.416 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1850523630000001, 2023-09-23T20:40:40.416 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:40:40.417 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:40:40.417 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:40:40.417 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35994", 2023-09-23T20:40:40.417 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:40:40.417 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 105791 2023-09-23T20:40:40.417 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.417 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:40:40.417 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.417 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:40:40.417 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.443934+0000", 2023-09-23T20:40:40.417 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:40:40.417 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.418 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.418 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:40:40.418 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.443934+0000", 2023-09-23T20:40:40.418 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:40:40.418 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.418 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.418 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:40:40.418 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.443934+0000", 2023-09-23T20:40:40.418 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:40:40.418 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.419 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.419 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:40:40.419 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.443936+0000", 2023-09-23T20:40:40.419 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.1469999999999999e-06 2023-09-23T20:40:40.419 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.419 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.419 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:40:40.419 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.443942+0000", 2023-09-23T20:40:40.419 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.9050000000000002e-06 2023-09-23T20:40:40.419 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.419 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.420 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:40:40.420 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.443947+0000", 2023-09-23T20:40:40.420 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.8699999999999997e-06 2023-09-23T20:40:40.420 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.420 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.420 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:40:40.420 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.713210+0000", 2023-09-23T20:40:40.420 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.269262891 2023-09-23T20:40:40.420 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.420 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.420 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:40:40.420 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.713216+0000", 2023-09-23T20:40:40.421 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.0769999999999996e-06 2023-09-23T20:40:40.421 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.421 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.421 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:40:40.421 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.612639+0000", 2023-09-23T20:40:40.421 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.89942242299999997 2023-09-23T20:40:40.421 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.421 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.421 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:40:40.421 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.612710+0000", 2023-09-23T20:40:40.421 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.0743999999999999e-05 2023-09-23T20:40:40.422 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.422 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.422 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,4", 2023-09-23T20:40:40.422 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.612781+0000", 2023-09-23T20:40:40.422 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.0965999999999998e-05 2023-09-23T20:40:40.422 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.422 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.422 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:40:40.422 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.618520+0000", 2023-09-23T20:40:40.422 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0057398170000000004 2023-09-23T20:40:40.422 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.423 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.423 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:40:40.423 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.627222+0000", 2023-09-23T20:40:40.423 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0087017199999999996 2023-09-23T20:40:40.423 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.423 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.423 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:40:40.423 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.627275+0000", 2023-09-23T20:40:40.423 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.3081000000000002e-05 2023-09-23T20:40:40.423 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.423 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.423 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:40:40.424 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.627336+0000", 2023-09-23T20:40:40.424 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.0630000000000001e-05 2023-09-23T20:40:40.424 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.424 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.424 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:40:40.424 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.628986+0000", 2023-09-23T20:40:40.424 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0016504309999999999 2023-09-23T20:40:40.424 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:40:40.424 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:40:40.424 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:40:40.424 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.424 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.425 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35994.0:105792 8.0 8:02af189b:::benchmark_data_smithi018_341490_object16469:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e410)", 2023-09-23T20:40:40.425 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:38:26.443952+0000", 2023-09-23T20:40:40.425 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 133.71449059299999, 2023-09-23T20:40:40.425 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1871479650000001, 2023-09-23T20:40:40.425 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:40:40.425 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:40:40.425 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:40:40.425 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35994", 2023-09-23T20:40:40.425 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:40:40.425 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 105792 2023-09-23T20:40:40.425 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.425 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:40:40.426 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.426 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:40:40.426 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.443952+0000", 2023-09-23T20:40:40.426 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:40:40.426 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.426 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.426 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:40:40.426 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.443952+0000", 2023-09-23T20:40:40.426 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:40:40.426 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.426 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.427 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:40:40.427 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.443952+0000", 2023-09-23T20:40:40.427 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:40:40.427 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.427 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.427 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:40:40.427 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.443954+0000", 2023-09-23T20:40:40.427 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.6089999999999999e-06 2023-09-23T20:40:40.427 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.427 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.428 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:40:40.428 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.443961+0000", 2023-09-23T20:40:40.428 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.3029999999999996e-06 2023-09-23T20:40:40.428 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.428 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.428 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:40:40.428 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.443966+0000", 2023-09-23T20:40:40.428 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.9729999999999997e-06 2023-09-23T20:40:40.428 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.428 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.428 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:40:40.429 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.713239+0000", 2023-09-23T20:40:40.429 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.26927288300000002 2023-09-23T20:40:40.429 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.429 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.429 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:40:40.429 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.713246+0000", 2023-09-23T20:40:40.429 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.1690000000000003e-06 2023-09-23T20:40:40.429 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.429 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.429 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:40:40.429 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.614811+0000", 2023-09-23T20:40:40.430 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.90156508599999996 2023-09-23T20:40:40.430 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.430 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.430 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:40:40.430 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.614929+0000", 2023-09-23T20:40:40.430 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000117846 2023-09-23T20:40:40.430 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.430 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.430 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,4", 2023-09-23T20:40:40.430 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.615050+0000", 2023-09-23T20:40:40.430 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00012100900000000001 2023-09-23T20:40:40.431 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.431 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.431 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:40:40.431 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.619299+0000", 2023-09-23T20:40:40.431 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0042493160000000004 2023-09-23T20:40:40.431 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.431 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.431 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:40:40.431 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.627319+0000", 2023-09-23T20:40:40.431 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0080197559999999994 2023-09-23T20:40:40.431 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.431 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.432 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:40:40.432 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.629386+0000", 2023-09-23T20:40:40.432 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0020676959999999999 2023-09-23T20:40:40.432 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.432 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.432 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:40:40.432 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.629448+0000", 2023-09-23T20:40:40.432 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.1155999999999995e-05 2023-09-23T20:40:40.432 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.432 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.432 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:40:40.433 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.631100+0000", 2023-09-23T20:40:40.433 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.001652518 2023-09-23T20:40:40.433 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:40:40.433 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:40:40.433 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:40:40.434 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.434 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.434 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35994.0:105793 8.18 8:193032ff:::benchmark_data_smithi018_341490_object16470:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e410)", 2023-09-23T20:40:40.434 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:38:26.443971+0000", 2023-09-23T20:40:40.434 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 133.714471352, 2023-09-23T20:40:40.434 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.173527269, 2023-09-23T20:40:40.434 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:40:40.434 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:40:40.434 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:40:40.434 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35994", 2023-09-23T20:40:40.434 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:40:40.435 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 105793 2023-09-23T20:40:40.435 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.435 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:40:40.435 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.435 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:40:40.435 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.443971+0000", 2023-09-23T20:40:40.435 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:40:40.435 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.435 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.435 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:40:40.436 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.443971+0000", 2023-09-23T20:40:40.436 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:40:40.436 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.436 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.436 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:40:40.436 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.443971+0000", 2023-09-23T20:40:40.436 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:40:40.436 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.436 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.436 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:40:40.437 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.443979+0000", 2023-09-23T20:40:40.437 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.4270000000000008e-06 2023-09-23T20:40:40.437 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.437 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.437 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:40:40.437 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.443986+0000", 2023-09-23T20:40:40.437 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.5520000000000001e-06 2023-09-23T20:40:40.437 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.437 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.437 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:40:40.437 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.443991+0000", 2023-09-23T20:40:40.437 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.2429999999999996e-06 2023-09-23T20:40:40.438 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.438 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.438 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:40:40.438 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.713266+0000", 2023-09-23T20:40:40.438 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.26927488199999999 2023-09-23T20:40:40.438 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.438 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.438 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:40:40.438 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.713273+0000", 2023-09-23T20:40:40.438 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.4309999999999999e-06 2023-09-23T20:40:40.438 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.439 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.439 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:40:40.439 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.605188+0000", 2023-09-23T20:40:40.439 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.89191558699999995 2023-09-23T20:40:40.439 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.439 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.439 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:40:40.439 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.605348+0000", 2023-09-23T20:40:40.439 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00016024400000000001 2023-09-23T20:40:40.439 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.439 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.440 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 0,7", 2023-09-23T20:40:40.440 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.605486+0000", 2023-09-23T20:40:40.440 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00013745700000000001 2023-09-23T20:40:40.440 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.440 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.440 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:40:40.440 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.610094+0000", 2023-09-23T20:40:40.440 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0046085149999999997 2023-09-23T20:40:40.440 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.440 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.440 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:40:40.441 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.612624+0000", 2023-09-23T20:40:40.441 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0025298859999999999 2023-09-23T20:40:40.441 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.441 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.441 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:40:40.441 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.615895+0000", 2023-09-23T20:40:40.441 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0032704919999999998 2023-09-23T20:40:40.441 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.441 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.441 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:40:40.441 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.615938+0000", 2023-09-23T20:40:40.442 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.3510999999999997e-05 2023-09-23T20:40:40.442 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.442 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.442 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:40:40.442 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.617499+0000", 2023-09-23T20:40:40.442 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.001560475 2023-09-23T20:40:40.442 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:40:40.442 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:40:40.442 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:40:40.442 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.442 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.442 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35994.0:105794 8.0 8:06d1e15f:::benchmark_data_smithi018_341490_object16471:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e410)", 2023-09-23T20:40:40.443 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:38:26.443996+0000", 2023-09-23T20:40:40.443 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 133.71444706299999, 2023-09-23T20:40:40.443 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1894021189999999, 2023-09-23T20:40:40.443 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:40:40.443 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:40:40.443 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:40:40.443 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35994", 2023-09-23T20:40:40.443 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:40:40.443 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 105794 2023-09-23T20:40:40.443 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.443 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:40:40.444 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.444 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:40:40.444 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.443996+0000", 2023-09-23T20:40:40.444 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:40:40.444 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.444 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.444 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:40:40.444 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.443996+0000", 2023-09-23T20:40:40.444 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:40:40.444 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.444 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.445 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:40:40.445 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.443995+0000", 2023-09-23T20:40:40.445 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:40:40.445 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.445 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.445 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:40:40.445 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.443998+0000", 2023-09-23T20:40:40.445 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.7020000000000001e-06 2023-09-23T20:40:40.445 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.445 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.445 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:40:40.445 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.444004+0000", 2023-09-23T20:40:40.446 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.9479999999999996e-06 2023-09-23T20:40:40.446 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.446 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.446 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:40:40.446 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.444009+0000", 2023-09-23T20:40:40.446 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.502e-06 2023-09-23T20:40:40.446 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.446 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.446 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:40:40.446 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.713294+0000", 2023-09-23T20:40:40.446 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.26928485600000002 2023-09-23T20:40:40.447 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.447 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.447 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:40:40.447 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.713302+0000", 2023-09-23T20:40:40.447 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.4440000000000002e-06 2023-09-23T20:40:40.447 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.447 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.447 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:40:40.447 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.615633+0000", 2023-09-23T20:40:40.447 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.90233140000000001 2023-09-23T20:40:40.447 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.448 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.448 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:40:40.448 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.615759+0000", 2023-09-23T20:40:40.448 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00012542599999999999 2023-09-23T20:40:40.448 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.448 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.448 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,4", 2023-09-23T20:40:40.448 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.617918+0000", 2023-09-23T20:40:40.448 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0021595719999999998 2023-09-23T20:40:40.448 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.448 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.449 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:40:40.449 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.627197+0000", 2023-09-23T20:40:40.449 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0092789759999999995 2023-09-23T20:40:40.449 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.449 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.449 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:40:40.449 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.627353+0000", 2023-09-23T20:40:40.449 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00015556599999999999 2023-09-23T20:40:40.449 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.449 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.449 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:40:40.449 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.631674+0000", 2023-09-23T20:40:40.450 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0043214890000000004 2023-09-23T20:40:40.450 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.450 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.450 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:40:40.450 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.631719+0000", 2023-09-23T20:40:40.450 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.5314e-05 2023-09-23T20:40:40.450 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.450 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.450 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:40:40.450 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.633398+0000", 2023-09-23T20:40:40.450 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.001678368 2023-09-23T20:40:40.451 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:40:40.451 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:40:40.451 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:40:40.451 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.451 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.451 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35994.0:105797 8.18 8:1843db78:::benchmark_data_smithi018_341490_object16474:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e410)", 2023-09-23T20:40:40.451 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:38:26.444014+0000", 2023-09-23T20:40:40.451 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 133.714428494, 2023-09-23T20:40:40.451 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.17448456, 2023-09-23T20:40:40.451 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:40:40.451 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:40:40.452 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:40:40.452 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35994", 2023-09-23T20:40:40.452 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:40:40.452 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 105797 2023-09-23T20:40:40.452 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.452 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:40:40.452 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.452 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:40:40.452 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.444014+0000", 2023-09-23T20:40:40.452 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:40:40.452 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.453 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.453 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:40:40.453 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.444014+0000", 2023-09-23T20:40:40.453 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:40:40.453 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.453 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.453 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:40:40.453 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.444014+0000", 2023-09-23T20:40:40.453 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:40:40.453 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.453 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.453 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:40:40.454 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.444016+0000", 2023-09-23T20:40:40.454 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.097e-06 2023-09-23T20:40:40.454 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.454 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.454 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:40:40.454 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.444022+0000", 2023-09-23T20:40:40.454 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.8200000000000002e-06 2023-09-23T20:40:40.454 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.454 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.454 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:40:40.454 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.444027+0000", 2023-09-23T20:40:40.455 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.4360000000000001e-06 2023-09-23T20:40:40.455 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.455 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.455 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:40:40.455 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.713323+0000", 2023-09-23T20:40:40.455 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.26929555599999999 2023-09-23T20:40:40.455 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.455 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.455 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:40:40.455 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.713329+0000", 2023-09-23T20:40:40.455 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.9819999999999996e-06 2023-09-23T20:40:40.456 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.456 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.456 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:40:40.456 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.608546+0000", 2023-09-23T20:40:40.456 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.895217021 2023-09-23T20:40:40.456 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.456 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.456 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:40:40.456 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.608698+0000", 2023-09-23T20:40:40.456 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00015241000000000001 2023-09-23T20:40:40.456 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.457 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.457 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 0,7", 2023-09-23T20:40:40.457 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.608843+0000", 2023-09-23T20:40:40.457 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00014468799999999999 2023-09-23T20:40:40.457 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.457 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.457 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:40:40.457 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.613634+0000", 2023-09-23T20:40:40.457 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0047907979999999998 2023-09-23T20:40:40.457 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.457 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.458 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:40:40.458 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.615841+0000", 2023-09-23T20:40:40.458 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0022071690000000001 2023-09-23T20:40:40.458 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.458 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.458 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:40:40.458 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.617508+0000", 2023-09-23T20:40:40.458 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0016669390000000001 2023-09-23T20:40:40.458 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.458 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.458 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:40:40.459 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.617562+0000", 2023-09-23T20:40:40.459 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.3891999999999998e-05 2023-09-23T20:40:40.459 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.459 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.459 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:40:40.459 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.618499+0000", 2023-09-23T20:40:40.459 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00093712400000000001 2023-09-23T20:40:40.459 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:40:40.459 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:40:40.459 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:40:40.459 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.459 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.460 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35994.0:168760 8.b 8:da9c59b5:::benchmark_data_smithi018_341490_object79437:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e444)", 2023-09-23T20:40:40.460 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:40:26.391800+0000", 2023-09-23T20:40:40.460 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 13.766642710999999, 2023-09-23T20:40:40.460 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.127134791, 2023-09-23T20:40:40.460 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:40:40.460 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:40:40.460 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:40:40.460 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35994", 2023-09-23T20:40:40.460 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:40:40.460 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 168760 2023-09-23T20:40:40.460 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.461 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:40:40.461 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.461 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:40:40.461 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:40:26.391800+0000", 2023-09-23T20:40:40.461 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:40:40.461 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.461 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.461 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:40:40.461 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:40:26.391800+0000", 2023-09-23T20:40:40.461 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:40:40.461 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.462 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.462 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:40:40.462 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:40:26.391799+0000", 2023-09-23T20:40:40.462 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.999999 2023-09-23T20:40:40.462 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.462 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.462 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:40:40.462 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:40:26.391806+0000", 2023-09-23T20:40:40.462 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.9110000000000004e-06 2023-09-23T20:40:40.462 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.463 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.463 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:40:40.463 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:40:26.391817+0000", 2023-09-23T20:40:40.463 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1426e-05 2023-09-23T20:40:40.463 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.463 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.463 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:40:40.463 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:40:26.391825+0000", 2023-09-23T20:40:40.463 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.6869999999999994e-06 2023-09-23T20:40:40.463 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.463 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.463 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:40:40.464 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:40:26.391944+0000", 2023-09-23T20:40:40.464 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00011903799999999999 2023-09-23T20:40:40.464 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.464 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.464 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:40:40.464 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:40:26.392067+0000", 2023-09-23T20:40:40.464 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00012332599999999999 2023-09-23T20:40:40.464 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.464 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.464 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,4", 2023-09-23T20:40:40.464 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:40:26.392185+0000", 2023-09-23T20:40:40.465 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000118022 2023-09-23T20:40:40.465 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.465 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.465 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:40:40.465 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:40:26.397545+0000", 2023-09-23T20:40:40.465 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0053600540000000004 2023-09-23T20:40:40.465 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.465 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.465 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:40:40.465 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:40:26.406107+0000", 2023-09-23T20:40:40.465 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0085620439999999996 2023-09-23T20:40:40.465 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.466 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.466 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:40:40.466 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:40:26.518835+0000", 2023-09-23T20:40:40.466 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.112727956 2023-09-23T20:40:40.466 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.466 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.466 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:40:40.466 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:40:26.518889+0000", 2023-09-23T20:40:40.466 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.3835999999999998e-05 2023-09-23T20:40:40.466 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.466 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.467 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:40:40.467 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:40:26.518935+0000", 2023-09-23T20:40:40.467 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.5624000000000002e-05 2023-09-23T20:40:40.467 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:40:40.467 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:40:40.467 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:40:40.467 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.467 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.467 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35994.0:168765 8.b 8:d7f87d63:::benchmark_data_smithi018_341490_object79442:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e444)", 2023-09-23T20:40:40.467 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:40:26.391909+0000", 2023-09-23T20:40:40.468 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 13.766534012999999, 2023-09-23T20:40:40.468 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.12718057899999999, 2023-09-23T20:40:40.468 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:40:40.468 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:40:40.468 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:40:40.468 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35994", 2023-09-23T20:40:40.468 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:40:40.468 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 168765 2023-09-23T20:40:40.468 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.468 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:40:40.468 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.468 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:40:40.469 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:40:26.391909+0000", 2023-09-23T20:40:40.469 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:40:40.469 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.469 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.469 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:40:40.469 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:40:26.391909+0000", 2023-09-23T20:40:40.469 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:40:40.469 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.469 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.469 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:40:40.469 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:40:26.391908+0000", 2023-09-23T20:40:40.470 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:40:40.470 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.470 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.470 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:40:40.470 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:40:26.391912+0000", 2023-09-23T20:40:40.470 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.3919999999999999e-06 2023-09-23T20:40:40.470 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.470 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.470 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:40:40.470 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:40:26.391920+0000", 2023-09-23T20:40:40.470 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.9780000000000005e-06 2023-09-23T20:40:40.471 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.471 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.471 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:40:40.471 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:40:26.391925+0000", 2023-09-23T20:40:40.471 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.6509999999999998e-06 2023-09-23T20:40:40.471 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.471 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.471 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:40:40.471 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:40:26.394931+0000", 2023-09-23T20:40:40.471 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0030057170000000002 2023-09-23T20:40:40.471 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.471 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.472 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:40:40.472 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:40:26.395053+0000", 2023-09-23T20:40:40.472 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000121853 2023-09-23T20:40:40.472 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.472 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.472 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,4", 2023-09-23T20:40:40.472 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:40:26.395162+0000", 2023-09-23T20:40:40.472 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000109528 2023-09-23T20:40:40.472 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.472 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.472 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:40:40.472 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:40:26.397631+0000", 2023-09-23T20:40:40.473 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0024690810000000001 2023-09-23T20:40:40.473 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.473 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.473 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:40:40.473 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:40:26.406135+0000", 2023-09-23T20:40:40.473 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0085033060000000004 2023-09-23T20:40:40.473 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.473 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.473 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:40:40.473 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:40:26.519006+0000", 2023-09-23T20:40:40.473 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.11287143299999999 2023-09-23T20:40:40.473 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.474 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.474 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:40:40.474 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:40:26.519053+0000", 2023-09-23T20:40:40.474 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.7153000000000001e-05 2023-09-23T20:40:40.474 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:40:40.474 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:40:40.474 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:40:40.474 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:40:26.519089+0000", 2023-09-23T20:40:40.474 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.6047999999999998e-05 2023-09-23T20:40:40.474 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:40:40.474 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:40:40.475 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:40:40.475 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:40:40.475 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:40:40.475 INFO:teuthology.orchestra.run.smithi018.stdout:} 2023-09-23T20:40:40.629 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:40 smithi018 ceph-mon[184144]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:40:40.629 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:40 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3066636435' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:40:40.629 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:40 smithi018 ceph-mon[184144]: osdmap e446: 8 total, 8 up, 8 in 2023-09-23T20:40:40.630 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:40 smithi018 ceph-mon[184144]: failed to encode map e446 with expected crc 2023-09-23T20:40:40.630 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:40 smithi018 ceph-mon[184144]: failed to encode map e446 with expected crc 2023-09-23T20:40:40.630 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:40 smithi018 ceph-mon[184144]: failed to encode map e446 with expected crc 2023-09-23T20:40:40.630 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:40 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/533532987' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:40:40.630 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:40 smithi018 ceph-mon[179647]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:40:40.630 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:40 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3066636435' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:40:40.630 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:40 smithi018 ceph-mon[179647]: osdmap e446: 8 total, 8 up, 8 in 2023-09-23T20:40:40.630 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:40 smithi018 ceph-mon[179647]: failed to encode map e446 with expected crc 2023-09-23T20:40:40.630 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:40 smithi018 ceph-mon[179647]: failed to encode map e446 with expected crc 2023-09-23T20:40:40.630 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:40 smithi018 ceph-mon[179647]: failed to encode map e446 with expected crc 2023-09-23T20:40:40.631 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:40 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/533532987' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:40:40.634 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:40:40.638 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:40.638 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:40:40.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:40 smithi149 ceph-mon[170394]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:40:40.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:40 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3066636435' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:40:40.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:40 smithi149 ceph-mon[170394]: osdmap e446: 8 total, 8 up, 8 in 2023-09-23T20:40:40.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:40 smithi149 ceph-mon[170394]: failed to encode map e446 with expected crc 2023-09-23T20:40:40.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:40 smithi149 ceph-mon[170394]: failed to encode map e446 with expected crc 2023-09-23T20:40:40.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:40 smithi149 ceph-mon[170394]: failed to encode map e446 with expected crc 2023-09-23T20:40:40.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:40 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/533532987' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:40:40.739 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:40:40.766 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:40.800 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:40:40.802 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:40.803 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:40:40.904 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:40:40.930 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:40.964 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:40:40.967 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:40.967 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:40:41.068 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:40:41.096 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:41.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:41 smithi018 ceph-mon[184144]: failed to encode map e446 with expected crc 2023-09-23T20:40:41.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:41 smithi018 ceph-mon[184144]: failed to encode map e446 with expected crc 2023-09-23T20:40:41.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:41 smithi018 ceph-mon[184144]: pgmap v1099: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 82 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 768 op/s; 14980/3792 objects degraded (395.042%); 149/3792 objects misplaced (3.929%); 2.0 MiB/s, 98 objects/s recovering 2023-09-23T20:40:41.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:41 smithi018 ceph-mon[179647]: failed to encode map e446 with expected crc 2023-09-23T20:40:41.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:41 smithi018 ceph-mon[179647]: failed to encode map e446 with expected crc 2023-09-23T20:40:41.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:41 smithi018 ceph-mon[179647]: pgmap v1099: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 82 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 768 op/s; 14980/3792 objects degraded (395.042%); 149/3792 objects misplaced (3.929%); 2.0 MiB/s, 98 objects/s recovering 2023-09-23T20:40:41.669 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T20:40:41.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:41 smithi149 ceph-mon[170394]: failed to encode map e446 with expected crc 2023-09-23T20:40:41.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:41 smithi149 ceph-mon[170394]: failed to encode map e446 with expected crc 2023-09-23T20:40:41.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:41 smithi149 ceph-mon[170394]: pgmap v1099: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 5 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 82 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 768 op/s; 14980/3792 objects degraded (395.042%); 149/3792 objects misplaced (3.929%); 2.0 MiB/s, 98 objects/s recovering 2023-09-23T20:40:42.198 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:40:42.201 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:42.201 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:40:42.302 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:40:42.329 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:42.364 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:42 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 14980/3792 objects degraded (395.042%), 9 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T20:40:42.364 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:42 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 14980/3792 objects degraded (395.042%), 9 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T20:40:42.365 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T20:40:42.366 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:40:42.368 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:42.369 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:40:42.469 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:40:42.493 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:42 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 14980/3792 objects degraded (395.042%), 9 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T20:40:42.496 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:42.539 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:40:42.542 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:42.543 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:40:42.643 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:40:42.646 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:40:42.670 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:42.704 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:40:42.707 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:42.708 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:40:42.808 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:40:42.835 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:42.876 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:40:42.880 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:42.880 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:40:42.981 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:40:43.003 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:40:43.013 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:43.151 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:40:43.155 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:43.156 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:40:43.256 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:40:43.288 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:43.325 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:40:43.328 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:43.329 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:40:43.430 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:40:43.461 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:43.560 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:43 smithi149 ceph-mon[170394]: pgmap v1100: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 5 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 32 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 710 op/s; 14980/1434 objects degraded (1044.630%); 79/1434 objects misplaced (5.509%); 2.0 MiB/s, 146 objects/s recovering 2023-09-23T20:40:43.732 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:43 smithi018 ceph-mon[184144]: pgmap v1100: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 5 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 32 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 710 op/s; 14980/1434 objects degraded (1044.630%); 79/1434 objects misplaced (5.509%); 2.0 MiB/s, 146 objects/s recovering 2023-09-23T20:40:43.732 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:43 smithi018 ceph-mon[179647]: pgmap v1100: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 5 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 32 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 710 op/s; 14980/1434 objects degraded (1044.630%); 79/1434 objects misplaced (5.509%); 2.0 MiB/s, 146 objects/s recovering 2023-09-23T20:40:43.782 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:40:43.786 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:43.786 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:40:43.887 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:40:43.918 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:44.033 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:40:44.037 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:44.037 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:40:44.138 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:40:44.154 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:40:44.167 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:44.221 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:40:44.223 INFO:teuthology.orchestra.run.smithi018.stderr:dumped all 2023-09-23T20:40:44.254 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:40:44.258 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:44.258 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:40:44.339 INFO:teuthology.orchestra.run.smithi149.stdout:{ 2023-09-23T20:40:44.339 INFO:teuthology.orchestra.run.smithi149.stdout: "ops": [], 2023-09-23T20:40:44.339 INFO:teuthology.orchestra.run.smithi149.stdout: "num_ops": 0 2023-09-23T20:40:44.340 INFO:teuthology.orchestra.run.smithi149.stdout:} 2023-09-23T20:40:44.359 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:40:44.393 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:44.395 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_3 2023-09-23T20:40:44.395 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd pool rm unique_pool_3 unique_pool_3 --yes-i-really-really-mean-it 2023-09-23T20:40:44.564 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:40:44.568 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:44 smithi018 ceph-mon[184144]: osdmap e447: 8 total, 8 up, 8 in 2023-09-23T20:40:44.568 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:44 smithi018 ceph-mon[184144]: failed to encode map e447 with expected crc 2023-09-23T20:40:44.568 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:44 smithi018 ceph-mon[179647]: osdmap e447: 8 total, 8 up, 8 in 2023-09-23T20:40:44.568 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:44 smithi018 ceph-mon[179647]: failed to encode map e447 with expected crc 2023-09-23T20:40:44.569 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:44.569 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:40:44.648 INFO:teuthology.orchestra.run.smithi018.stdout:{"pg_ready":true,"pg_map":{"version":1102,"stamp":"2023-09-23T20:40:44.088463+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":18707287,"num_objects":243,"num_object_clones":0,"num_object_copies":729,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":14980,"num_objects_misplaced":46,"num_objects_unfound":0,"num_objects_dirty":243,"num_whiteouts":0,"num_read":6139,"num_read_kb":13491,"num_write":179625,"num_write_kb":5729974,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":35511,"num_bytes_recovered":2091508098,"num_keys_recovered":40,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":184112,"ondisk_log_size":184112,"up":387,"acting":377,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":421,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":5762856,"kb_used_data":1694376,"kb_used_omap":32,"kb_used_meta":4068383,"kb_avail":744198360,"statfs":{"total":767960285184,"available":762059120640,"internally_reserved":0,"allocated":1735041024,"data_stored":1720160726,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32773,"internal_metadata":4166025211},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":10,"apply_latency_ms":10,"commit_latency_ns":10000000,"apply_latency_ns":10000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-278069272,"num_objects":-4244,"num_object_clones":0,"num_object_copies":-12732,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-444,"num_objects_misplaced":-816,"num_objects_unfound":0,"num_objects_dirty":-4244,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4460,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1366,"num_bytes_recovered":14548992,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.002886"},"pg_stats":[{"pgid":"8.11","version":"445'8889","reported_seq":8985,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-09-23T20:40:39.101569+0000","last_change":"2023-09-23T20:38:27.478425+0000","last_active":"2023-09-23T20:40:39.101569+0000","last_peered":"2023-09-23T20:40:39.101569+0000","last_clean":"2023-09-23T20:40:39.101569+0000","last_became_active":"2023-09-23T20:38:27.472168+0000","last_became_peered":"2023-09-23T20:38:27.472168+0000","last_unstale":"2023-09-23T20:40:39.101569+0000","last_undegraded":"2023-09-23T20:40:39.101569+0000","last_fullsized":"2023-09-23T20:40:39.101569+0000","mapping_epoch":410,"log_start":"373'1694","ondisk_log_start":"373'1694","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":7195,"log_dups_size":0,"ondisk_log_size":7195,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":229376,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5580,"num_write_kb":178656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.18","version":"90'161","reported_seq":434,"reported_epoch":444,"state":"active+recovery_wait+remapped","last_fresh":"2023-09-23T20:40:22.405324+0000","last_change":"2023-09-23T20:40:20.543976+0000","last_active":"2023-09-23T20:40:22.405324+0000","last_peered":"2023-09-23T20:40:22.405324+0000","last_clean":"2023-09-23T20:15:59.452160+0000","last_became_active":"2023-09-23T20:40:20.539716+0000","last_became_peered":"2023-09-23T20:40:20.539716+0000","last_unstale":"2023-09-23T20:40:22.405324+0000","last_undegraded":"2023-09-23T20:40:22.405324+0000","last_fullsized":"2023-09-23T20:40:22.405324+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":405,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:35:03.551143+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:35:03.551143+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[1,2,7],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2,7","objects":-1}],"blocked_by":[],"up_primary":5,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1a","version":"69'1","reported_seq":143,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-09-23T20:40:29.682113+0000","last_change":"2023-09-23T20:32:33.233853+0000","last_active":"2023-09-23T20:40:29.682113+0000","last_peered":"2023-09-23T20:40:29.682113+0000","last_clean":"2023-09-23T20:40:29.682113+0000","last_became_active":"2023-09-23T20:32:33.233546+0000","last_became_peered":"2023-09-23T20:32:33.233546+0000","last_unstale":"2023-09-23T20:40:29.682113+0000","last_undegraded":"2023-09-23T20:40:29.682113+0000","last_fullsized":"2023-09-23T20:40:29.682113+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:50:27.130808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":260,"reported_epoch":446,"state":"active+clean","last_fresh":"2023-09-23T20:40:39.228975+0000","last_change":"2023-09-23T20:37:43.579611+0000","last_active":"2023-09-23T20:40:39.228975+0000","last_peered":"2023-09-23T20:40:39.228975+0000","last_clean":"2023-09-23T20:40:39.228975+0000","last_became_active":"2023-09-23T20:37:43.579323+0000","last_became_peered":"2023-09-23T20:37:43.579323+0000","last_unstale":"2023-09-23T20:40:39.228975+0000","last_undegraded":"2023-09-23T20:40:39.228975+0000","last_fullsized":"2023-09-23T20:40:39.228975+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:17:32.311427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.12","version":"445'8980","reported_seq":9125,"reported_epoch":445,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:40:39.102856+0000","last_change":"2023-09-23T20:38:39.273131+0000","last_active":"2023-09-23T20:40:39.102856+0000","last_peered":"2023-09-23T20:40:39.102856+0000","last_clean":"2023-09-23T20:37:29.399979+0000","last_became_active":"2023-09-23T20:38:39.252214+0000","last_became_peered":"2023-09-23T20:38:39.252214+0000","last_unstale":"2023-09-23T20:40:39.102856+0000","last_undegraded":"2023-09-23T20:38:38.456024+0000","last_fullsized":"2023-09-23T20:38:38.449466+0000","mapping_epoch":416,"log_start":"373'1996","ondisk_log_start":"373'1996","created":410,"last_epoch_clean":370,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":6984,"log_dups_size":0,"ondisk_log_size":6984,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":715,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5630,"num_write_kb":179648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":753664,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0],"avail_no_missing":["4","0"],"object_location_counts":[{"shards":"0,4","objects":-141}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1b","version":"90'161","reported_seq":519,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-09-23T20:40:29.692215+0000","last_change":"2023-09-23T20:40:22.445059+0000","last_active":"2023-09-23T20:40:29.692215+0000","last_peered":"2023-09-23T20:40:29.692215+0000","last_clean":"2023-09-23T20:40:29.692215+0000","last_became_active":"2023-09-23T20:40:22.444637+0000","last_became_peered":"2023-09-23T20:40:22.444637+0000","last_unstale":"2023-09-23T20:40:29.692215+0000","last_undegraded":"2023-09-23T20:40:29.692215+0000","last_fullsized":"2023-09-23T20:40:29.692215+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:05.442288+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:36:05.442288+0000","last_clean_scrub_stamp":"2023-09-23T20:36:05.442288+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0019500940000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":413,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-09-23T20:40:22.405025+0000","last_change":"2023-09-23T20:34:59.983035+0000","last_active":"2023-09-23T20:40:22.405025+0000","last_peered":"2023-09-23T20:40:22.405025+0000","last_clean":"2023-09-23T20:40:22.405025+0000","last_became_active":"2023-09-23T20:34:59.982459+0000","last_became_peered":"2023-09-23T20:34:59.982459+0000","last_unstale":"2023-09-23T20:40:22.405025+0000","last_undegraded":"2023-09-23T20:40:22.405025+0000","last_fullsized":"2023-09-23T20:40:22.405025+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:16:48.510783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":416,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-09-23T20:40:29.682807+0000","last_change":"2023-09-23T20:25:02.957185+0000","last_active":"2023-09-23T20:40:29.682807+0000","last_peered":"2023-09-23T20:40:29.682807+0000","last_clean":"2023-09-23T20:40:29.682807+0000","last_became_active":"2023-09-23T20:25:02.956689+0000","last_became_peered":"2023-09-23T20:25:02.956689+0000","last_unstale":"2023-09-23T20:40:29.682807+0000","last_undegraded":"2023-09-23T20:40:29.682807+0000","last_fullsized":"2023-09-23T20:40:29.682807+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:04:28.524015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.13","version":"445'8913","reported_seq":14296,"reported_epoch":445,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:40:39.183662+0000","last_change":"2023-09-23T20:40:20.783376+0000","last_active":"2023-09-23T20:40:39.183662+0000","last_peered":"2023-09-23T20:40:39.183662+0000","last_clean":"2023-09-23T20:37:13.297905+0000","last_became_active":"2023-09-23T20:40:20.744428+0000","last_became_peered":"2023-09-23T20:40:20.744428+0000","last_unstale":"2023-09-23T20:40:39.183662+0000","last_undegraded":"2023-09-23T20:40:20.622760+0000","last_fullsized":"2023-09-23T20:40:20.514696+0000","mapping_epoch":441,"log_start":"373'1680","ondisk_log_start":"373'1680","created":410,"last_epoch_clean":370,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":7233,"log_dups_size":0,"ondisk_log_size":7233,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2525,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5607,"num_write_kb":178976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2192,"num_bytes_recovered":143622144,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,6],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1,6","objects":-7}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1a","version":"90'161","reported_seq":525,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-09-23T20:40:29.678486+0000","last_change":"2023-09-23T20:36:01.292103+0000","last_active":"2023-09-23T20:40:29.678486+0000","last_peered":"2023-09-23T20:40:29.678486+0000","last_clean":"2023-09-23T20:40:29.678486+0000","last_became_active":"2023-09-23T20:35:26.639912+0000","last_became_peered":"2023-09-23T20:35:26.639912+0000","last_unstale":"2023-09-23T20:40:29.678486+0000","last_undegraded":"2023-09-23T20:40:29.678486+0000","last_fullsized":"2023-09-23T20:40:29.678486+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:01.292032+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:01.292032+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:04:53.926640+0000","scrub_duration":0.0032983159999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":240,"reported_epoch":446,"state":"active+clean","last_fresh":"2023-09-23T20:40:39.229710+0000","last_change":"2023-09-23T20:37:43.584414+0000","last_active":"2023-09-23T20:40:39.229710+0000","last_peered":"2023-09-23T20:40:39.229710+0000","last_clean":"2023-09-23T20:40:39.229710+0000","last_became_active":"2023-09-23T20:37:43.584116+0000","last_became_peered":"2023-09-23T20:37:43.584116+0000","last_unstale":"2023-09-23T20:40:39.229710+0000","last_undegraded":"2023-09-23T20:40:39.229710+0000","last_fullsized":"2023-09-23T20:40:39.229710+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:17:54.258172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":62,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-09-23T20:40:29.672025+0000","last_change":"2023-09-23T20:38:37.457427+0000","last_active":"2023-09-23T20:40:29.672025+0000","last_peered":"2023-09-23T20:40:29.672025+0000","last_clean":"2023-09-23T20:40:29.672025+0000","last_became_active":"2023-09-23T20:38:37.457099+0000","last_became_peered":"2023-09-23T20:38:37.457099+0000","last_unstale":"2023-09-23T20:40:29.672025+0000","last_undegraded":"2023-09-23T20:40:29.672025+0000","last_fullsized":"2023-09-23T20:40:29.672025+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:31:27.084994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"90'161","reported_seq":658,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-09-23T20:40:29.685336+0000","last_change":"2023-09-23T20:40:22.449754+0000","last_active":"2023-09-23T20:40:29.685336+0000","last_peered":"2023-09-23T20:40:29.685336+0000","last_clean":"2023-09-23T20:40:29.685336+0000","last_became_active":"2023-09-23T20:40:22.448838+0000","last_became_peered":"2023-09-23T20:40:22.448838+0000","last_unstale":"2023-09-23T20:40:29.685336+0000","last_undegraded":"2023-09-23T20:40:29.685336+0000","last_fullsized":"2023-09-23T20:40:29.685336+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:03.411025+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:03.411025+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:22:07.948856+0000","scrub_duration":0.001802922,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":449,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-09-23T20:40:29.681599+0000","last_change":"2023-09-23T20:40:19.538844+0000","last_active":"2023-09-23T20:40:29.681599+0000","last_peered":"2023-09-23T20:40:29.681599+0000","last_clean":"2023-09-23T20:40:29.681599+0000","last_became_active":"2023-09-23T20:40:19.538434+0000","last_became_peered":"2023-09-23T20:40:19.538434+0000","last_unstale":"2023-09-23T20:40:29.681599+0000","last_undegraded":"2023-09-23T20:40:29.681599+0000","last_fullsized":"2023-09-23T20:40:29.681599+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:53:59.418427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":169,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-09-23T20:40:29.685711+0000","last_change":"2023-09-23T20:38:02.579343+0000","last_active":"2023-09-23T20:40:29.685711+0000","last_peered":"2023-09-23T20:40:29.685711+0000","last_clean":"2023-09-23T20:40:29.685711+0000","last_became_active":"2023-09-23T20:38:02.579047+0000","last_became_peered":"2023-09-23T20:38:02.579047+0000","last_unstale":"2023-09-23T20:40:29.685711+0000","last_undegraded":"2023-09-23T20:40:29.685711+0000","last_fullsized":"2023-09-23T20:40:29.685711+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:44:03.166556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.14","version":"90'161","reported_seq":775,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-09-23T20:40:29.686912+0000","last_change":"2023-09-23T20:35:02.236398+0000","last_active":"2023-09-23T20:40:29.686912+0000","last_peered":"2023-09-23T20:40:29.686912+0000","last_clean":"2023-09-23T20:40:29.686912+0000","last_became_active":"2023-09-23T20:35:02.235988+0000","last_became_peered":"2023-09-23T20:35:02.235988+0000","last_unstale":"2023-09-23T20:40:29.686912+0000","last_undegraded":"2023-09-23T20:40:29.686912+0000","last_fullsized":"2023-09-23T20:40:29.686912+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:47.479012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:47.479012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:13.519648+0000","scrub_duration":0.0020136590000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":15,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-09-23T20:40:29.692946+0000","last_change":"2023-09-23T20:40:19.545472+0000","last_active":"2023-09-23T20:40:29.692946+0000","last_peered":"2023-09-23T20:40:29.692946+0000","last_clean":"2023-09-23T20:40:29.692946+0000","last_became_active":"2023-09-23T20:40:19.543853+0000","last_became_peered":"2023-09-23T20:40:19.543853+0000","last_unstale":"2023-09-23T20:40:29.692946+0000","last_undegraded":"2023-09-23T20:40:29.692946+0000","last_fullsized":"2023-09-23T20:40:29.692946+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:08.014744+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":271,"reported_epoch":446,"state":"active+clean","last_fresh":"2023-09-23T20:40:39.228885+0000","last_change":"2023-09-23T20:38:02.542034+0000","last_active":"2023-09-23T20:40:39.228885+0000","last_peered":"2023-09-23T20:40:39.228885+0000","last_clean":"2023-09-23T20:40:39.228885+0000","last_became_active":"2023-09-23T20:38:02.541575+0000","last_became_peered":"2023-09-23T20:38:02.541575+0000","last_unstale":"2023-09-23T20:40:39.228885+0000","last_undegraded":"2023-09-23T20:40:39.228885+0000","last_fullsized":"2023-09-23T20:40:39.228885+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:50:54.591529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"90'161","reported_seq":599,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-09-23T20:40:29.693220+0000","last_change":"2023-09-23T20:40:22.444898+0000","last_active":"2023-09-23T20:40:29.693220+0000","last_peered":"2023-09-23T20:40:29.693220+0000","last_clean":"2023-09-23T20:40:29.693220+0000","last_became_active":"2023-09-23T20:40:22.444543+0000","last_became_peered":"2023-09-23T20:40:22.444543+0000","last_unstale":"2023-09-23T20:40:29.693220+0000","last_undegraded":"2023-09-23T20:40:29.693220+0000","last_fullsized":"2023-09-23T20:40:29.693220+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:03.435480+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:03.435480+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":473,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-09-23T20:40:29.685475+0000","last_change":"2023-09-23T20:32:33.270617+0000","last_active":"2023-09-23T20:40:29.685475+0000","last_peered":"2023-09-23T20:40:29.685475+0000","last_clean":"2023-09-23T20:40:29.685475+0000","last_became_active":"2023-09-23T20:32:33.227902+0000","last_became_peered":"2023-09-23T20:32:33.227902+0000","last_unstale":"2023-09-23T20:40:29.685475+0000","last_undegraded":"2023-09-23T20:40:29.685475+0000","last_fullsized":"2023-09-23T20:40:29.685475+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:30:26.847831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":360,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":414,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-09-23T20:40:22.406684+0000","last_change":"2023-09-23T20:38:02.599068+0000","last_active":"2023-09-23T20:40:22.406684+0000","last_peered":"2023-09-23T20:40:22.406684+0000","last_clean":"2023-09-23T20:40:22.406684+0000","last_became_active":"2023-09-23T20:38:02.597602+0000","last_became_peered":"2023-09-23T20:38:02.597602+0000","last_unstale":"2023-09-23T20:40:22.406684+0000","last_undegraded":"2023-09-23T20:40:22.406684+0000","last_fullsized":"2023-09-23T20:40:22.406684+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:34:06.337029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"90'161","reported_seq":569,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-09-23T20:40:22.403190+0000","last_change":"2023-09-23T20:36:02.548387+0000","last_active":"2023-09-23T20:40:22.403190+0000","last_peered":"2023-09-23T20:40:22.403190+0000","last_clean":"2023-09-23T20:40:22.403190+0000","last_became_active":"2023-09-23T20:35:26.644787+0000","last_became_peered":"2023-09-23T20:35:26.644787+0000","last_unstale":"2023-09-23T20:40:22.403190+0000","last_undegraded":"2023-09-23T20:40:22.403190+0000","last_fullsized":"2023-09-23T20:40:22.403190+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:02.548285+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:02.548285+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:37:15.216553+0000","scrub_duration":0.0015514459999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":193,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-09-23T20:40:29.673745+0000","last_change":"2023-09-23T20:38:02.560646+0000","last_active":"2023-09-23T20:40:29.673745+0000","last_peered":"2023-09-23T20:40:29.673745+0000","last_clean":"2023-09-23T20:40:29.673745+0000","last_became_active":"2023-09-23T20:38:02.539814+0000","last_became_peered":"2023-09-23T20:38:02.539814+0000","last_unstale":"2023-09-23T20:40:29.673745+0000","last_undegraded":"2023-09-23T20:40:29.673745+0000","last_fullsized":"2023-09-23T20:40:29.673745+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:00:09.435023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":173,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-09-23T20:40:29.679894+0000","last_change":"2023-09-23T20:38:02.579222+0000","last_active":"2023-09-23T20:40:29.679894+0000","last_peered":"2023-09-23T20:40:29.679894+0000","last_clean":"2023-09-23T20:40:29.679894+0000","last_became_active":"2023-09-23T20:38:02.578813+0000","last_became_peered":"2023-09-23T20:38:02.578813+0000","last_unstale":"2023-09-23T20:40:29.679894+0000","last_undegraded":"2023-09-23T20:40:29.679894+0000","last_fullsized":"2023-09-23T20:40:29.679894+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:57:31.080027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.18","version":"445'8947","reported_seq":14890,"reported_epoch":446,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:40:39.228822+0000","last_change":"2023-09-23T20:40:20.742770+0000","last_active":"2023-09-23T20:40:39.228822+0000","last_peered":"2023-09-23T20:40:39.228822+0000","last_clean":"2023-09-23T20:37:13.320723+0000","last_became_active":"2023-09-23T20:40:20.713497+0000","last_became_peered":"2023-09-23T20:40:20.713497+0000","last_unstale":"2023-09-23T20:40:39.228822+0000","last_undegraded":"2023-09-23T20:40:20.533808+0000","last_fullsized":"2023-09-23T20:40:20.514147+0000","mapping_epoch":441,"log_start":"373'2334","ondisk_log_start":"373'2334","created":410,"last_epoch_clean":370,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":6613,"log_dups_size":0,"ondisk_log_size":6613,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2531,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5639,"num_write_kb":179008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2403,"num_bytes_recovered":157417472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":-179}],"blocked_by":[],"up_primary":5,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.11","version":"90'161","reported_seq":712,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-09-23T20:40:29.682079+0000","last_change":"2023-09-23T20:40:22.427437+0000","last_active":"2023-09-23T20:40:29.682079+0000","last_peered":"2023-09-23T20:40:29.682079+0000","last_clean":"2023-09-23T20:40:29.682079+0000","last_became_active":"2023-09-23T20:40:22.427090+0000","last_became_peered":"2023-09-23T20:40:22.427090+0000","last_unstale":"2023-09-23T20:40:29.682079+0000","last_undegraded":"2023-09-23T20:40:29.682079+0000","last_fullsized":"2023-09-23T20:40:29.682079+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:43.578666+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:43.578666+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:08:43.995672+0000","scrub_duration":0.0021890379999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":472,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-09-23T20:40:29.685802+0000","last_change":"2023-09-23T20:32:33.242943+0000","last_active":"2023-09-23T20:40:29.685802+0000","last_peered":"2023-09-23T20:40:29.685802+0000","last_clean":"2023-09-23T20:40:29.685802+0000","last_became_active":"2023-09-23T20:32:33.225468+0000","last_became_peered":"2023-09-23T20:32:33.225468+0000","last_unstale":"2023-09-23T20:40:29.685802+0000","last_undegraded":"2023-09-23T20:40:29.685802+0000","last_fullsized":"2023-09-23T20:40:29.685802+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:40:47.867573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":182,"reported_epoch":446,"state":"active+clean","last_fresh":"2023-09-23T20:40:39.229989+0000","last_change":"2023-09-23T20:37:43.571836+0000","last_active":"2023-09-23T20:40:39.229989+0000","last_peered":"2023-09-23T20:40:39.229989+0000","last_clean":"2023-09-23T20:40:39.229989+0000","last_became_active":"2023-09-23T20:37:43.571511+0000","last_became_peered":"2023-09-23T20:37:43.571511+0000","last_unstale":"2023-09-23T20:40:39.229989+0000","last_undegraded":"2023-09-23T20:40:39.229989+0000","last_fullsized":"2023-09-23T20:40:39.229989+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:59:25.507917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.19","version":"445'8956","reported_seq":16679,"reported_epoch":445,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T20:40:39.174806+0000","last_change":"2023-09-23T20:40:20.669720+0000","last_active":"2023-09-23T20:40:39.174806+0000","last_peered":"2023-09-23T20:40:39.174806+0000","last_clean":"2023-09-23T20:40:18.450551+0000","last_became_active":"2023-09-23T20:40:20.635841+0000","last_became_peered":"2023-09-23T20:40:20.635841+0000","last_unstale":"2023-09-23T20:40:39.174806+0000","last_undegraded":"2023-09-23T20:40:39.174806+0000","last_fullsized":"2023-09-23T20:40:39.174806+0000","mapping_epoch":441,"log_start":"373'1316","ondisk_log_start":"373'1316","created":410,"last_epoch_clean":439,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":7640,"log_dups_size":0,"ondisk_log_size":7640,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5642,"num_write_kb":180192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2990,"num_bytes_recovered":184942592,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,0,3],"avail_no_missing":["6","0","3","4","5"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.10","version":"90'161","reported_seq":628,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-09-23T20:40:29.692276+0000","last_change":"2023-09-23T20:40:22.427706+0000","last_active":"2023-09-23T20:40:29.692276+0000","last_peered":"2023-09-23T20:40:29.692276+0000","last_clean":"2023-09-23T20:40:29.692276+0000","last_became_active":"2023-09-23T20:40:22.427114+0000","last_became_peered":"2023-09-23T20:40:22.427114+0000","last_unstale":"2023-09-23T20:40:29.692276+0000","last_undegraded":"2023-09-23T20:40:29.692276+0000","last_fullsized":"2023-09-23T20:40:29.692276+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:49.726139+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:49.726139+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0096333110000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":413,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-09-23T20:40:29.682097+0000","last_change":"2023-09-23T20:25:02.953993+0000","last_active":"2023-09-23T20:40:29.682097+0000","last_peered":"2023-09-23T20:40:29.682097+0000","last_clean":"2023-09-23T20:40:29.682097+0000","last_became_active":"2023-09-23T20:25:02.953672+0000","last_became_peered":"2023-09-23T20:25:02.953672+0000","last_unstale":"2023-09-23T20:40:29.682097+0000","last_undegraded":"2023-09-23T20:40:29.682097+0000","last_fullsized":"2023-09-23T20:40:29.682097+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:26:08.603577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":15,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-09-23T20:40:29.693608+0000","last_change":"2023-09-23T20:40:19.561512+0000","last_active":"2023-09-23T20:40:29.693608+0000","last_peered":"2023-09-23T20:40:29.693608+0000","last_clean":"2023-09-23T20:40:29.693608+0000","last_became_active":"2023-09-23T20:40:19.561123+0000","last_became_peered":"2023-09-23T20:40:19.561123+0000","last_unstale":"2023-09-23T20:40:29.693608+0000","last_undegraded":"2023-09-23T20:40:29.693608+0000","last_fullsized":"2023-09-23T20:40:29.693608+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:50:02.093978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"90'161","reported_seq":730,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-09-23T20:40:22.404218+0000","last_change":"2023-09-23T20:35:02.238923+0000","last_active":"2023-09-23T20:40:22.404218+0000","last_peered":"2023-09-23T20:40:22.404218+0000","last_clean":"2023-09-23T20:40:22.404218+0000","last_became_active":"2023-09-23T20:35:02.238293+0000","last_became_peered":"2023-09-23T20:35:02.238293+0000","last_unstale":"2023-09-23T20:40:22.404218+0000","last_undegraded":"2023-09-23T20:40:22.404218+0000","last_fullsized":"2023-09-23T20:40:22.404218+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:42.544308+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:42.544308+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:51:22.281932+0000","scrub_duration":0.0029958179999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":449,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-09-23T20:40:29.684447+0000","last_change":"2023-09-23T20:38:02.539774+0000","last_active":"2023-09-23T20:40:29.684447+0000","last_peered":"2023-09-23T20:40:29.684447+0000","last_clean":"2023-09-23T20:40:29.684447+0000","last_became_active":"2023-09-23T20:38:02.539436+0000","last_became_peered":"2023-09-23T20:38:02.539436+0000","last_unstale":"2023-09-23T20:40:29.684447+0000","last_undegraded":"2023-09-23T20:40:29.684447+0000","last_fullsized":"2023-09-23T20:40:29.684447+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:50:57.375803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":423,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-09-23T20:40:29.682879+0000","last_change":"2023-09-23T20:25:02.957010+0000","last_active":"2023-09-23T20:40:29.682879+0000","last_peered":"2023-09-23T20:40:29.682879+0000","last_clean":"2023-09-23T20:40:29.682879+0000","last_became_active":"2023-09-23T20:25:02.956267+0000","last_became_peered":"2023-09-23T20:25:02.956267+0000","last_unstale":"2023-09-23T20:40:29.682879+0000","last_undegraded":"2023-09-23T20:40:29.682879+0000","last_fullsized":"2023-09-23T20:40:29.682879+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:10:55.299422+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"90'161","reported_seq":551,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-09-23T20:40:29.692604+0000","last_change":"2023-09-23T20:40:22.431074+0000","last_active":"2023-09-23T20:40:29.692604+0000","last_peered":"2023-09-23T20:40:29.692604+0000","last_clean":"2023-09-23T20:40:29.692604+0000","last_became_active":"2023-09-23T20:40:22.430717+0000","last_became_peered":"2023-09-23T20:40:22.430717+0000","last_unstale":"2023-09-23T20:40:29.692604+0000","last_undegraded":"2023-09-23T20:40:29.692604+0000","last_fullsized":"2023-09-23T20:40:29.692604+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:53.706934+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:53.706934+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":82,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-09-23T20:40:29.672069+0000","last_change":"2023-09-23T20:40:19.564378+0000","last_active":"2023-09-23T20:40:29.672069+0000","last_peered":"2023-09-23T20:40:29.672069+0000","last_clean":"2023-09-23T20:40:29.672069+0000","last_became_active":"2023-09-23T20:40:19.532991+0000","last_became_peered":"2023-09-23T20:40:19.532991+0000","last_unstale":"2023-09-23T20:40:29.672069+0000","last_undegraded":"2023-09-23T20:40:29.672069+0000","last_fullsized":"2023-09-23T20:40:29.672069+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:22:42.832115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":240,"reported_epoch":446,"state":"active+clean","last_fresh":"2023-09-23T20:40:39.230258+0000","last_change":"2023-09-23T20:37:43.571083+0000","last_active":"2023-09-23T20:40:39.230258+0000","last_peered":"2023-09-23T20:40:39.230258+0000","last_clean":"2023-09-23T20:40:39.230258+0000","last_became_active":"2023-09-23T20:37:43.570306+0000","last_became_peered":"2023-09-23T20:37:43.570306+0000","last_unstale":"2023-09-23T20:40:39.230258+0000","last_undegraded":"2023-09-23T20:40:39.230258+0000","last_fullsized":"2023-09-23T20:40:39.230258+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:42:02.396718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.4","version":"445'9120","reported_seq":11266,"reported_epoch":445,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:40:39.157679+0000","last_change":"2023-09-23T20:40:19.540505+0000","last_active":"2023-09-23T20:40:39.157679+0000","last_peered":"2023-09-23T20:40:39.157679+0000","last_clean":"2023-09-23T20:37:15.925522+0000","last_became_active":"2023-09-23T20:38:39.251413+0000","last_became_peered":"2023-09-23T20:38:39.251413+0000","last_unstale":"2023-09-23T20:40:39.157679+0000","last_undegraded":"2023-09-23T20:40:19.539602+0000","last_fullsized":"2023-09-23T20:38:38.449461+0000","mapping_epoch":416,"log_start":"373'2080","ondisk_log_start":"373'2080","created":369,"last_epoch_clean":370,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":7040,"log_dups_size":0,"ondisk_log_size":7040,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":503,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5713,"num_write_kb":182176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":645,"num_bytes_recovered":36306944,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[4,2],"avail_no_missing":["4","2","6"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.d","version":"90'161","reported_seq":738,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-09-23T20:40:29.685211+0000","last_change":"2023-09-23T20:34:59.496438+0000","last_active":"2023-09-23T20:40:29.685211+0000","last_peered":"2023-09-23T20:40:29.685211+0000","last_clean":"2023-09-23T20:40:29.685211+0000","last_became_active":"2023-09-23T20:33:38.970527+0000","last_became_peered":"2023-09-23T20:33:38.970527+0000","last_unstale":"2023-09-23T20:40:29.685211+0000","last_undegraded":"2023-09-23T20:40:29.685211+0000","last_fullsized":"2023-09-23T20:40:29.685211+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:34:59.496375+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:34:59.496375+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:46:18.244734+0000","scrub_duration":0.0017843189999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":449,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-09-23T20:40:29.684730+0000","last_change":"2023-09-23T20:32:33.283752+0000","last_active":"2023-09-23T20:40:29.684730+0000","last_peered":"2023-09-23T20:40:29.684730+0000","last_clean":"2023-09-23T20:40:29.684730+0000","last_became_active":"2023-09-23T20:32:33.230503+0000","last_became_peered":"2023-09-23T20:32:33.230503+0000","last_unstale":"2023-09-23T20:40:29.684730+0000","last_undegraded":"2023-09-23T20:40:29.684730+0000","last_fullsized":"2023-09-23T20:40:29.684730+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:29.084410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":15,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-09-23T20:40:29.691959+0000","last_change":"2023-09-23T20:40:19.545668+0000","last_active":"2023-09-23T20:40:29.691959+0000","last_peered":"2023-09-23T20:40:29.691959+0000","last_clean":"2023-09-23T20:40:29.691959+0000","last_became_active":"2023-09-23T20:40:19.543857+0000","last_became_peered":"2023-09-23T20:40:19.543857+0000","last_unstale":"2023-09-23T20:40:29.691959+0000","last_undegraded":"2023-09-23T20:40:29.691959+0000","last_fullsized":"2023-09-23T20:40:29.691959+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:27:09.470605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"445'8865","reported_seq":12430,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-09-23T20:40:39.182715+0000","last_change":"2023-09-23T20:38:36.632834+0000","last_active":"2023-09-23T20:40:39.182715+0000","last_peered":"2023-09-23T20:40:39.182715+0000","last_clean":"2023-09-23T20:40:39.182715+0000","last_became_active":"2023-09-23T20:38:28.423549+0000","last_became_peered":"2023-09-23T20:38:28.423549+0000","last_unstale":"2023-09-23T20:40:39.182715+0000","last_undegraded":"2023-09-23T20:40:39.182715+0000","last_fullsized":"2023-09-23T20:40:39.182715+0000","mapping_epoch":411,"log_start":"373'1839","ondisk_log_start":"373'1839","created":369,"last_epoch_clean":415,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":7026,"log_dups_size":0,"ondisk_log_size":7026,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5569,"num_write_kb":177696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1036,"num_bytes_recovered":55345152,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.c","version":"90'161","reported_seq":694,"reported_epoch":446,"state":"active+clean","last_fresh":"2023-09-23T20:40:39.230173+0000","last_change":"2023-09-23T20:38:28.442536+0000","last_active":"2023-09-23T20:40:39.230173+0000","last_peered":"2023-09-23T20:40:39.230173+0000","last_clean":"2023-09-23T20:40:39.230173+0000","last_became_active":"2023-09-23T20:38:28.442229+0000","last_became_peered":"2023-09-23T20:38:28.442229+0000","last_unstale":"2023-09-23T20:40:39.230173+0000","last_undegraded":"2023-09-23T20:40:39.230173+0000","last_fullsized":"2023-09-23T20:40:39.230173+0000","mapping_epoch":411,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":412,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:05.485751+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:05.485751+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.002180043,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163840,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":484,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-09-23T20:40:29.685167+0000","last_change":"2023-09-23T20:34:59.987981+0000","last_active":"2023-09-23T20:40:29.685167+0000","last_peered":"2023-09-23T20:40:29.685167+0000","last_clean":"2023-09-23T20:40:29.685167+0000","last_became_active":"2023-09-23T20:34:59.985368+0000","last_became_peered":"2023-09-23T20:34:59.985368+0000","last_unstale":"2023-09-23T20:40:29.685167+0000","last_undegraded":"2023-09-23T20:40:29.685167+0000","last_fullsized":"2023-09-23T20:40:29.685167+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:12:43.082493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":143,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-09-23T20:40:29.680510+0000","last_change":"2023-09-23T20:32:33.232142+0000","last_active":"2023-09-23T20:40:29.680510+0000","last_peered":"2023-09-23T20:40:29.680510+0000","last_clean":"2023-09-23T20:40:29.680510+0000","last_became_active":"2023-09-23T20:32:33.231683+0000","last_became_peered":"2023-09-23T20:32:33.231683+0000","last_unstale":"2023-09-23T20:40:29.680510+0000","last_undegraded":"2023-09-23T20:40:29.680510+0000","last_fullsized":"2023-09-23T20:40:29.680510+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:02:58.336348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"445'8913","reported_seq":9640,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-09-23T20:40:39.174929+0000","last_change":"2023-09-23T20:38:29.214109+0000","last_active":"2023-09-23T20:40:39.174929+0000","last_peered":"2023-09-23T20:40:39.174929+0000","last_clean":"2023-09-23T20:40:39.174929+0000","last_became_active":"2023-09-23T20:38:28.425504+0000","last_became_peered":"2023-09-23T20:38:28.425504+0000","last_unstale":"2023-09-23T20:40:39.174929+0000","last_undegraded":"2023-09-23T20:40:39.174929+0000","last_fullsized":"2023-09-23T20:40:39.174929+0000","mapping_epoch":411,"log_start":"373'1261","ondisk_log_start":"373'1261","created":369,"last_epoch_clean":412,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":7652,"log_dups_size":0,"ondisk_log_size":7652,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5633,"num_write_kb":177728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":305,"num_bytes_recovered":10420224,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"90'161","reported_seq":602,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-09-23T20:40:29.672316+0000","last_change":"2023-09-23T20:38:40.491902+0000","last_active":"2023-09-23T20:40:29.672316+0000","last_peered":"2023-09-23T20:40:29.672316+0000","last_clean":"2023-09-23T20:40:29.672316+0000","last_became_active":"2023-09-23T20:38:40.491563+0000","last_became_peered":"2023-09-23T20:38:40.491563+0000","last_unstale":"2023-09-23T20:40:29.672316+0000","last_undegraded":"2023-09-23T20:40:29.672316+0000","last_fullsized":"2023-09-23T20:40:29.672316+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:46.217489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:46.217489+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:39:25.008796+0000","scrub_duration":0.001456429,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":474,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-09-23T20:40:29.685298+0000","last_change":"2023-09-23T20:40:19.677655+0000","last_active":"2023-09-23T20:40:29.685298+0000","last_peered":"2023-09-23T20:40:29.685298+0000","last_clean":"2023-09-23T20:40:29.685298+0000","last_became_active":"2023-09-23T20:40:19.551189+0000","last_became_peered":"2023-09-23T20:40:19.551189+0000","last_unstale":"2023-09-23T20:40:29.685298+0000","last_undegraded":"2023-09-23T20:40:29.685298+0000","last_fullsized":"2023-09-23T20:40:29.685298+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:11:01.559576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":52,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-09-23T20:40:29.671448+0000","last_change":"2023-09-23T20:38:02.590613+0000","last_active":"2023-09-23T20:40:29.671448+0000","last_peered":"2023-09-23T20:40:29.671448+0000","last_clean":"2023-09-23T20:40:29.671448+0000","last_became_active":"2023-09-23T20:38:02.590284+0000","last_became_peered":"2023-09-23T20:38:02.590284+0000","last_unstale":"2023-09-23T20:40:29.671448+0000","last_undegraded":"2023-09-23T20:40:29.671448+0000","last_fullsized":"2023-09-23T20:40:29.671448+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:40:07.825054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"445'8836","reported_seq":9574,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-09-23T20:40:39.083906+0000","last_change":"2023-09-23T20:40:20.140860+0000","last_active":"2023-09-23T20:40:39.083906+0000","last_peered":"2023-09-23T20:40:39.083906+0000","last_clean":"2023-09-23T20:40:39.083906+0000","last_became_active":"2023-09-23T20:38:28.436345+0000","last_became_peered":"2023-09-23T20:38:28.436345+0000","last_unstale":"2023-09-23T20:40:39.083906+0000","last_undegraded":"2023-09-23T20:40:39.083906+0000","last_fullsized":"2023-09-23T20:40:39.083906+0000","mapping_epoch":411,"log_start":"373'1658","ondisk_log_start":"373'1658","created":369,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":7178,"log_dups_size":0,"ondisk_log_size":7178,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":720896,"num_objects":11,"num_object_clones":0,"num_object_copies":33,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":11,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5546,"num_write_kb":177824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":309,"num_bytes_recovered":9699328,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.e","version":"90'161","reported_seq":546,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-09-23T20:40:29.674073+0000","last_change":"2023-09-23T20:36:00.322453+0000","last_active":"2023-09-23T20:40:29.674073+0000","last_peered":"2023-09-23T20:40:29.674073+0000","last_clean":"2023-09-23T20:40:29.674073+0000","last_became_active":"2023-09-23T20:35:02.233536+0000","last_became_peered":"2023-09-23T20:35:02.233536+0000","last_unstale":"2023-09-23T20:40:29.674073+0000","last_undegraded":"2023-09-23T20:40:29.674073+0000","last_fullsized":"2023-09-23T20:40:29.674073+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:00.322381+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:00.322381+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:02:15.489069+0000","scrub_duration":0.0017321859999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":40,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-09-23T20:40:29.692985+0000","last_change":"2023-09-23T20:38:37.456683+0000","last_active":"2023-09-23T20:40:29.692985+0000","last_peered":"2023-09-23T20:40:29.692985+0000","last_clean":"2023-09-23T20:40:29.692985+0000","last_became_active":"2023-09-23T20:38:37.456289+0000","last_became_peered":"2023-09-23T20:38:37.456289+0000","last_unstale":"2023-09-23T20:40:29.692985+0000","last_undegraded":"2023-09-23T20:40:29.692985+0000","last_fullsized":"2023-09-23T20:40:29.692985+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:03:10.507333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":432,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-09-23T20:40:31.252462+0000","last_change":"2023-09-23T20:32:33.236122+0000","last_active":"2023-09-23T20:40:31.252462+0000","last_peered":"2023-09-23T20:40:31.252462+0000","last_clean":"2023-09-23T20:40:31.252462+0000","last_became_active":"2023-09-23T20:32:33.235384+0000","last_became_peered":"2023-09-23T20:32:33.235384+0000","last_unstale":"2023-09-23T20:40:31.252462+0000","last_undegraded":"2023-09-23T20:40:31.252462+0000","last_fullsized":"2023-09-23T20:40:31.252462+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:56.843733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"445'8703","reported_seq":15216,"reported_epoch":446,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T20:40:39.229306+0000","last_change":"2023-09-23T20:40:20.768066+0000","last_active":"2023-09-23T20:40:39.229306+0000","last_peered":"2023-09-23T20:40:39.229306+0000","last_clean":"2023-09-23T20:40:18.483118+0000","last_became_active":"2023-09-23T20:40:20.740650+0000","last_became_peered":"2023-09-23T20:40:20.740650+0000","last_unstale":"2023-09-23T20:40:39.229306+0000","last_undegraded":"2023-09-23T20:40:39.229306+0000","last_fullsized":"2023-09-23T20:40:39.229306+0000","mapping_epoch":441,"log_start":"373'2187","ondisk_log_start":"373'2187","created":369,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":6516,"log_dups_size":0,"ondisk_log_size":6516,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5456,"num_write_kb":174560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2565,"num_bytes_recovered":168099840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,1,4],"avail_no_missing":["3","1","4","5"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":851,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-09-23T20:40:29.680916+0000","last_change":"2023-09-23T20:35:02.523308+0000","last_active":"2023-09-23T20:40:29.680916+0000","last_peered":"2023-09-23T20:40:29.680916+0000","last_clean":"2023-09-23T20:40:29.680916+0000","last_became_active":"2023-09-23T20:33:38.979989+0000","last_became_peered":"2023-09-23T20:33:38.979989+0000","last_unstale":"2023-09-23T20:40:29.680916+0000","last_undegraded":"2023-09-23T20:40:29.680916+0000","last_fullsized":"2023-09-23T20:40:29.680916+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:35:02.523203+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:35:02.523203+0000","last_clean_scrub_stamp":"2023-09-23T20:35:02.523203+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:02:41.472958+0000","scrub_duration":0.0085927989999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":266,"reported_epoch":446,"state":"active+clean","last_fresh":"2023-09-23T20:40:39.229007+0000","last_change":"2023-09-23T20:37:43.583731+0000","last_active":"2023-09-23T20:40:39.229007+0000","last_peered":"2023-09-23T20:40:39.229007+0000","last_clean":"2023-09-23T20:40:39.229007+0000","last_became_active":"2023-09-23T20:37:43.583066+0000","last_became_peered":"2023-09-23T20:37:43.583066+0000","last_unstale":"2023-09-23T20:40:39.229007+0000","last_undegraded":"2023-09-23T20:40:39.229007+0000","last_fullsized":"2023-09-23T20:40:39.229007+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:03:45.211900+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":163,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-09-23T20:40:29.681076+0000","last_change":"2023-09-23T20:38:02.579814+0000","last_active":"2023-09-23T20:40:29.681076+0000","last_peered":"2023-09-23T20:40:29.681076+0000","last_clean":"2023-09-23T20:40:29.681076+0000","last_became_active":"2023-09-23T20:38:02.579253+0000","last_became_peered":"2023-09-23T20:38:02.579253+0000","last_unstale":"2023-09-23T20:40:29.681076+0000","last_undegraded":"2023-09-23T20:40:29.681076+0000","last_fullsized":"2023-09-23T20:40:29.681076+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:05:07.298198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"445'8896","reported_seq":8994,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-09-23T20:40:39.157479+0000","last_change":"2023-09-23T20:40:20.514587+0000","last_active":"2023-09-23T20:40:39.157479+0000","last_peered":"2023-09-23T20:40:39.157479+0000","last_clean":"2023-09-23T20:40:39.157479+0000","last_became_active":"2023-09-23T20:38:27.437204+0000","last_became_peered":"2023-09-23T20:38:27.437204+0000","last_unstale":"2023-09-23T20:40:39.157479+0000","last_undegraded":"2023-09-23T20:40:39.157479+0000","last_fullsized":"2023-09-23T20:40:39.157479+0000","mapping_epoch":369,"log_start":"373'1694","ondisk_log_start":"373'1694","created":369,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":7202,"log_dups_size":0,"ondisk_log_size":7202,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5586,"num_write_kb":178656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":547,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-09-23T20:40:29.691940+0000","last_change":"2023-09-23T20:40:22.433408+0000","last_active":"2023-09-23T20:40:29.691940+0000","last_peered":"2023-09-23T20:40:29.691940+0000","last_clean":"2023-09-23T20:40:29.691940+0000","last_became_active":"2023-09-23T20:40:22.433023+0000","last_became_peered":"2023-09-23T20:40:22.433023+0000","last_unstale":"2023-09-23T20:40:29.691940+0000","last_undegraded":"2023-09-23T20:40:29.691940+0000","last_fullsized":"2023-09-23T20:40:29.691940+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:35:00.496197+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:35:00.496197+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":52,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-09-23T20:40:29.671732+0000","last_change":"2023-09-23T20:38:02.578703+0000","last_active":"2023-09-23T20:40:29.671732+0000","last_peered":"2023-09-23T20:40:29.671732+0000","last_clean":"2023-09-23T20:40:29.671732+0000","last_became_active":"2023-09-23T20:38:02.578414+0000","last_became_peered":"2023-09-23T20:38:02.578414+0000","last_unstale":"2023-09-23T20:40:29.671732+0000","last_undegraded":"2023-09-23T20:40:29.671732+0000","last_fullsized":"2023-09-23T20:40:29.671732+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:43:59.779295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":112,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-09-23T20:40:29.679656+0000","last_change":"2023-09-23T20:37:31.296606+0000","last_active":"2023-09-23T20:40:29.679656+0000","last_peered":"2023-09-23T20:40:29.679656+0000","last_clean":"2023-09-23T20:40:29.679656+0000","last_became_active":"2023-09-23T20:37:31.295924+0000","last_became_peered":"2023-09-23T20:37:31.295924+0000","last_unstale":"2023-09-23T20:40:29.679656+0000","last_undegraded":"2023-09-23T20:40:29.679656+0000","last_fullsized":"2023-09-23T20:40:29.679656+0000","mapping_epoch":391,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":392,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:27:50.948510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"445'8948","reported_seq":9102,"reported_epoch":445,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:40:39.176892+0000","last_change":"2023-09-23T20:38:39.273152+0000","last_active":"2023-09-23T20:40:39.176892+0000","last_peered":"2023-09-23T20:40:39.176892+0000","last_clean":"2023-09-23T20:37:29.399979+0000","last_became_active":"2023-09-23T20:38:39.253967+0000","last_became_peered":"2023-09-23T20:38:39.253967+0000","last_unstale":"2023-09-23T20:40:39.176892+0000","last_undegraded":"2023-09-23T20:38:38.450129+0000","last_fullsized":"2023-09-23T20:38:38.446971+0000","mapping_epoch":416,"log_start":"373'1996","ondisk_log_start":"373'1996","created":369,"last_epoch_clean":370,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":6952,"log_dups_size":0,"ondisk_log_size":6952,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1048576,"num_objects":16,"num_object_clones":0,"num_object_copies":48,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":723,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":16,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5598,"num_write_kb":179648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":950272,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0],"avail_no_missing":["4","0"],"object_location_counts":[{"shards":"0,4","objects":-174}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"90'161","reported_seq":746,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-09-23T20:40:22.456699+0000","last_change":"2023-09-23T20:40:22.445808+0000","last_active":"2023-09-23T20:40:22.456699+0000","last_peered":"2023-09-23T20:40:22.456699+0000","last_clean":"2023-09-23T20:40:22.456699+0000","last_became_active":"2023-09-23T20:40:22.445456+0000","last_became_peered":"2023-09-23T20:40:22.445456+0000","last_unstale":"2023-09-23T20:40:22.456699+0000","last_undegraded":"2023-09-23T20:40:22.456699+0000","last_fullsized":"2023-09-23T20:40:22.456699+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:05.595042+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:05.595042+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:39:23.678154+0000","scrub_duration":0.007271675,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":102,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-09-23T20:40:29.674055+0000","last_change":"2023-09-23T20:34:59.983122+0000","last_active":"2023-09-23T20:40:29.674055+0000","last_peered":"2023-09-23T20:40:29.674055+0000","last_clean":"2023-09-23T20:40:29.674055+0000","last_became_active":"2023-09-23T20:34:59.982809+0000","last_became_peered":"2023-09-23T20:34:59.982809+0000","last_unstale":"2023-09-23T20:40:29.674055+0000","last_undegraded":"2023-09-23T20:40:29.674055+0000","last_fullsized":"2023-09-23T20:40:29.674055+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:49:40.839723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":163,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-09-23T20:40:29.681418+0000","last_change":"2023-09-23T20:38:02.543866+0000","last_active":"2023-09-23T20:40:29.681418+0000","last_peered":"2023-09-23T20:40:29.681418+0000","last_clean":"2023-09-23T20:40:29.681418+0000","last_became_active":"2023-09-23T20:38:02.543563+0000","last_became_peered":"2023-09-23T20:38:02.543563+0000","last_unstale":"2023-09-23T20:40:29.681418+0000","last_undegraded":"2023-09-23T20:40:29.681418+0000","last_fullsized":"2023-09-23T20:40:29.681418+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:33.050536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"445'8886","reported_seq":14240,"reported_epoch":445,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:40:39.195772+0000","last_change":"2023-09-23T20:40:20.781471+0000","last_active":"2023-09-23T20:40:39.195772+0000","last_peered":"2023-09-23T20:40:39.195772+0000","last_clean":"2023-09-23T20:37:13.297905+0000","last_became_active":"2023-09-23T20:40:20.744231+0000","last_became_peered":"2023-09-23T20:40:20.744231+0000","last_unstale":"2023-09-23T20:40:39.195772+0000","last_undegraded":"2023-09-23T20:40:20.609553+0000","last_fullsized":"2023-09-23T20:40:20.515058+0000","mapping_epoch":441,"log_start":"373'1680","ondisk_log_start":"373'1680","created":369,"last_epoch_clean":370,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":7206,"log_dups_size":0,"ondisk_log_size":7206,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":884736,"num_objects":13,"num_object_clones":0,"num_object_copies":39,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2556,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":13,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5579,"num_write_kb":178976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2180,"num_bytes_recovered":142770176,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,6],"avail_no_missing":["1"],"object_location_counts":[{"shards":"1,6","objects":8}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":550,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-09-23T20:40:29.692296+0000","last_change":"2023-09-23T20:40:01.179672+0000","last_active":"2023-09-23T20:40:29.692296+0000","last_peered":"2023-09-23T20:40:29.692296+0000","last_clean":"2023-09-23T20:40:29.692296+0000","last_became_active":"2023-09-23T20:38:40.494049+0000","last_became_peered":"2023-09-23T20:38:40.494049+0000","last_unstale":"2023-09-23T20:40:29.692296+0000","last_undegraded":"2023-09-23T20:40:29.692296+0000","last_fullsized":"2023-09-23T20:40:29.692296+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:34:59.481446+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:34:59.481446+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26834,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":240,"reported_epoch":446,"state":"active+clean","last_fresh":"2023-09-23T20:40:39.229623+0000","last_change":"2023-09-23T20:37:43.570610+0000","last_active":"2023-09-23T20:40:39.229623+0000","last_peered":"2023-09-23T20:40:39.229623+0000","last_clean":"2023-09-23T20:40:39.229623+0000","last_became_active":"2023-09-23T20:37:43.570216+0000","last_became_peered":"2023-09-23T20:37:43.570216+0000","last_unstale":"2023-09-23T20:40:39.229623+0000","last_undegraded":"2023-09-23T20:40:39.229623+0000","last_fullsized":"2023-09-23T20:40:39.229623+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:39:26.555329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":268,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-09-23T20:40:22.404934+0000","last_change":"2023-09-23T20:32:33.230582+0000","last_active":"2023-09-23T20:40:22.404934+0000","last_peered":"2023-09-23T20:40:22.404934+0000","last_clean":"2023-09-23T20:40:22.404934+0000","last_became_active":"2023-09-23T20:32:33.230257+0000","last_became_peered":"2023-09-23T20:32:33.230257+0000","last_unstale":"2023-09-23T20:40:22.404934+0000","last_undegraded":"2023-09-23T20:40:22.404934+0000","last_fullsized":"2023-09-23T20:40:22.404934+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:11:45.333096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"445'11056","reported_seq":11144,"reported_epoch":446,"state":"active+clean","last_fresh":"2023-09-23T20:40:39.697724+0000","last_change":"2023-09-23T20:36:11.310149+0000","last_active":"2023-09-23T20:40:39.697724+0000","last_peered":"2023-09-23T20:40:39.697724+0000","last_clean":"2023-09-23T20:40:39.697724+0000","last_became_active":"2023-09-23T20:36:11.309844+0000","last_became_peered":"2023-09-23T20:36:11.309844+0000","last_unstale":"2023-09-23T20:40:39.697724+0000","last_undegraded":"2023-09-23T20:40:39.697724+0000","last_fullsized":"2023-09-23T20:40:39.697724+0000","mapping_epoch":369,"log_start":"377'4719","ondisk_log_start":"377'4719","created":369,"last_epoch_clean":370,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":6337,"log_dups_size":0,"ondisk_log_size":6337,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:14:57.442108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11056,"num_write_kb":353792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":625,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-09-23T20:40:29.672188+0000","last_change":"2023-09-23T20:38:05.622597+0000","last_active":"2023-09-23T20:40:29.672188+0000","last_peered":"2023-09-23T20:40:29.672188+0000","last_clean":"2023-09-23T20:40:29.672188+0000","last_became_active":"2023-09-23T20:38:05.622261+0000","last_became_peered":"2023-09-23T20:38:05.622261+0000","last_unstale":"2023-09-23T20:40:29.672188+0000","last_undegraded":"2023-09-23T20:40:29.672188+0000","last_fullsized":"2023-09-23T20:40:29.672188+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:35:04.101972+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:35:04.101972+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"86'119","reported_seq":753,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-09-23T20:40:22.403454+0000","last_change":"2023-09-23T20:29:45.521257+0000","last_active":"2023-09-23T20:40:22.403454+0000","last_peered":"2023-09-23T20:40:22.403454+0000","last_clean":"2023-09-23T20:40:22.403454+0000","last_became_active":"2023-09-23T20:29:45.520546+0000","last_became_peered":"2023-09-23T20:29:45.520546+0000","last_unstale":"2023-09-23T20:40:22.403454+0000","last_undegraded":"2023-09-23T20:40:22.403454+0000","last_fullsized":"2023-09-23T20:40:22.403454+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":119,"log_dups_size":0,"ondisk_log_size":119,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:58:56.085777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032705499999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":523,"num_read_kb":3503,"num_write":136,"num_write_kb":3128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":15,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-09-23T20:40:29.693537+0000","last_change":"2023-09-23T20:40:19.557009+0000","last_active":"2023-09-23T20:40:29.693537+0000","last_peered":"2023-09-23T20:40:29.693537+0000","last_clean":"2023-09-23T20:40:29.693537+0000","last_became_active":"2023-09-23T20:40:19.550473+0000","last_became_peered":"2023-09-23T20:40:19.550473+0000","last_unstale":"2023-09-23T20:40:29.693537+0000","last_undegraded":"2023-09-23T20:40:29.693537+0000","last_fullsized":"2023-09-23T20:40:29.693537+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:35:01.140258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":400,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-09-23T20:40:22.406084+0000","last_change":"2023-09-23T20:40:19.530942+0000","last_active":"2023-09-23T20:40:22.406084+0000","last_peered":"2023-09-23T20:40:22.406084+0000","last_clean":"2023-09-23T20:40:22.406084+0000","last_became_active":"2023-09-23T20:40:19.530617+0000","last_became_peered":"2023-09-23T20:40:19.530617+0000","last_unstale":"2023-09-23T20:40:22.406084+0000","last_undegraded":"2023-09-23T20:40:22.406084+0000","last_fullsized":"2023-09-23T20:40:22.406084+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:24:09.231934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.8","version":"445'8858","reported_seq":15176,"reported_epoch":445,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T20:40:39.117301+0000","last_change":"2023-09-23T20:40:20.738122+0000","last_active":"2023-09-23T20:40:39.117301+0000","last_peered":"2023-09-23T20:40:39.117301+0000","last_clean":"2023-09-23T20:40:18.482315+0000","last_became_active":"2023-09-23T20:40:20.708683+0000","last_became_peered":"2023-09-23T20:40:20.708683+0000","last_unstale":"2023-09-23T20:40:39.117301+0000","last_undegraded":"2023-09-23T20:40:39.117301+0000","last_fullsized":"2023-09-23T20:40:39.117301+0000","mapping_epoch":441,"log_start":"373'2334","ondisk_log_start":"373'2334","created":369,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":6524,"log_dups_size":0,"ondisk_log_size":6524,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":2916352,"num_objects":44,"num_object_clones":0,"num_object_copies":132,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":44,"num_objects_unfound":0,"num_objects_dirty":44,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5549,"num_write_kb":179008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2588,"num_bytes_recovered":169607168,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[0,3,7],"avail_no_missing":["0","3","7"],"object_location_counts":[{"shards":"0,3,7","objects":44}],"blocked_by":[],"up_primary":5,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":519,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-09-23T20:40:29.692045+0000","last_change":"2023-09-23T20:40:22.445201+0000","last_active":"2023-09-23T20:40:29.692045+0000","last_peered":"2023-09-23T20:40:29.692045+0000","last_clean":"2023-09-23T20:40:29.692045+0000","last_became_active":"2023-09-23T20:40:22.444703+0000","last_became_peered":"2023-09-23T20:40:22.444703+0000","last_unstale":"2023-09-23T20:40:29.692045+0000","last_undegraded":"2023-09-23T20:40:29.692045+0000","last_fullsized":"2023-09-23T20:40:29.692045+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:45.785450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:45.785450+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"86'106","reported_seq":1165,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-09-23T20:40:29.692519+0000","last_change":"2023-09-23T20:38:37.476512+0000","last_active":"2023-09-23T20:40:29.692519+0000","last_peered":"2023-09-23T20:40:29.692519+0000","last_clean":"2023-09-23T20:40:29.692519+0000","last_became_active":"2023-09-23T20:38:37.475980+0000","last_became_peered":"2023-09-23T20:38:37.475980+0000","last_unstale":"2023-09-23T20:40:29.692519+0000","last_undegraded":"2023-09-23T20:40:29.692519+0000","last_fullsized":"2023-09-23T20:40:29.692519+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":106,"log_dups_size":0,"ondisk_log_size":106,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:58:45.508831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00099302599999999993,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1094,"num_read_kb":1678,"num_write":100,"num_write_kb":404,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":163,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-09-23T20:40:29.679733+0000","last_change":"2023-09-23T20:38:02.552008+0000","last_active":"2023-09-23T20:40:29.679733+0000","last_peered":"2023-09-23T20:40:29.679733+0000","last_clean":"2023-09-23T20:40:29.679733+0000","last_became_active":"2023-09-23T20:38:02.551684+0000","last_became_peered":"2023-09-23T20:38:02.551684+0000","last_unstale":"2023-09-23T20:40:29.679733+0000","last_undegraded":"2023-09-23T20:40:29.679733+0000","last_fullsized":"2023-09-23T20:40:29.679733+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:40:37.147538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":406,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-09-23T20:40:22.403372+0000","last_change":"2023-09-23T20:29:45.533406+0000","last_active":"2023-09-23T20:40:22.403372+0000","last_peered":"2023-09-23T20:40:22.403372+0000","last_clean":"2023-09-23T20:40:22.403372+0000","last_became_active":"2023-09-23T20:29:45.526001+0000","last_became_peered":"2023-09-23T20:29:45.526001+0000","last_unstale":"2023-09-23T20:40:22.403372+0000","last_undegraded":"2023-09-23T20:40:22.403372+0000","last_fullsized":"2023-09-23T20:40:22.403372+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:43.787251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.b","version":"445'11136","reported_seq":11238,"reported_epoch":446,"state":"active+clean","last_fresh":"2023-09-23T20:40:39.229097+0000","last_change":"2023-09-23T20:38:01.517845+0000","last_active":"2023-09-23T20:40:39.229097+0000","last_peered":"2023-09-23T20:40:39.229097+0000","last_clean":"2023-09-23T20:40:39.229097+0000","last_became_active":"2023-09-23T20:37:43.579403+0000","last_became_peered":"2023-09-23T20:37:43.579403+0000","last_unstale":"2023-09-23T20:40:39.229097+0000","last_undegraded":"2023-09-23T20:40:39.229097+0000","last_fullsized":"2023-09-23T20:40:39.229097+0000","mapping_epoch":399,"log_start":"408'6300","ondisk_log_start":"408'6300","created":369,"last_epoch_clean":403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":4836,"log_dups_size":0,"ondisk_log_size":4836,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:33:13.920936+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11136,"num_write_kb":356352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":918,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-09-23T20:40:29.686014+0000","last_change":"2023-09-23T20:35:02.260708+0000","last_active":"2023-09-23T20:40:29.686014+0000","last_peered":"2023-09-23T20:40:29.686014+0000","last_clean":"2023-09-23T20:40:29.686014+0000","last_became_active":"2023-09-23T20:35:02.260348+0000","last_became_peered":"2023-09-23T20:35:02.260348+0000","last_unstale":"2023-09-23T20:40:29.686014+0000","last_undegraded":"2023-09-23T20:40:29.686014+0000","last_fullsized":"2023-09-23T20:40:29.686014+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:43.390908+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:43.390908+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:09:14.144070+0000","scrub_duration":0.0032721,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"86'130","reported_seq":762,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-09-23T20:40:29.685727+0000","last_change":"2023-09-23T20:25:39.637458+0000","last_active":"2023-09-23T20:40:29.685727+0000","last_peered":"2023-09-23T20:40:29.685727+0000","last_clean":"2023-09-23T20:40:29.685727+0000","last_became_active":"2023-09-23T20:25:39.636773+0000","last_became_peered":"2023-09-23T20:25:39.636773+0000","last_unstale":"2023-09-23T20:40:29.685727+0000","last_undegraded":"2023-09-23T20:40:29.685727+0000","last_fullsized":"2023-09-23T20:40:29.685727+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:00:32.810564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011300049999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":512,"num_read_kb":1820,"num_write":133,"num_write_kb":1347,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":414,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-09-23T20:40:22.405621+0000","last_change":"2023-09-23T20:38:02.591335+0000","last_active":"2023-09-23T20:40:22.405621+0000","last_peered":"2023-09-23T20:40:22.405621+0000","last_clean":"2023-09-23T20:40:22.405621+0000","last_became_active":"2023-09-23T20:38:02.591015+0000","last_became_peered":"2023-09-23T20:38:02.591015+0000","last_unstale":"2023-09-23T20:40:22.405621+0000","last_undegraded":"2023-09-23T20:40:22.405621+0000","last_fullsized":"2023-09-23T20:40:22.405621+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:55:32.272195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":406,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-09-23T20:40:29.708142+0000","last_change":"2023-09-23T20:40:19.544333+0000","last_active":"2023-09-23T20:40:29.708142+0000","last_peered":"2023-09-23T20:40:29.708142+0000","last_clean":"2023-09-23T20:40:29.708142+0000","last_became_active":"2023-09-23T20:40:19.543987+0000","last_became_peered":"2023-09-23T20:40:19.543987+0000","last_unstale":"2023-09-23T20:40:29.708142+0000","last_undegraded":"2023-09-23T20:40:29.708142+0000","last_fullsized":"2023-09-23T20:40:29.708142+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:12:59.203296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"445'8934","reported_seq":19177,"reported_epoch":446,"state":"active+recovering+undersized+remapped","last_fresh":"2023-09-23T20:40:41.266990+0000","last_change":"2023-09-23T20:40:26.947190+0000","last_active":"2023-09-23T20:40:41.266990+0000","last_peered":"2023-09-23T20:40:41.266990+0000","last_clean":"2023-09-23T20:37:13.347998+0000","last_became_active":"2023-09-23T20:40:20.702197+0000","last_became_peered":"2023-09-23T20:40:20.702197+0000","last_unstale":"2023-09-23T20:40:41.266990+0000","last_undegraded":"2023-09-23T20:40:41.266990+0000","last_fullsized":"2023-09-23T20:40:20.511184+0000","mapping_epoch":441,"log_start":"373'1316","ondisk_log_start":"373'1316","created":369,"last_epoch_clean":370,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":7618,"log_dups_size":0,"ondisk_log_size":7618,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":720896,"num_objects":11,"num_object_clones":0,"num_object_copies":33,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":11,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5620,"num_write_kb":180192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4087,"num_bytes_recovered":163512320,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[0,6],"avail_no_missing":["0","6"],"object_location_counts":[{"shards":"0,6","objects":11}],"blocked_by":[],"up_primary":6,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"86'194","reported_seq":800,"reported_epoch":445,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:40:29.686196+0000","last_change":"2023-09-23T20:40:20.532327+0000","last_active":"2023-09-23T20:40:29.686196+0000","last_peered":"2023-09-23T20:40:29.686196+0000","last_clean":"2023-09-23T20:40:12.685357+0000","last_became_active":"2023-09-23T20:40:20.531854+0000","last_became_peered":"2023-09-23T20:40:20.531854+0000","last_unstale":"2023-09-23T20:40:29.686196+0000","last_undegraded":"2023-09-23T20:40:20.510463+0000","last_fullsized":"2023-09-23T20:40:20.509782+0000","mapping_epoch":441,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":194,"log_dups_size":0,"ondisk_log_size":194,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:35:05.823248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00037995999999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1024,"num_read_kb":1200,"num_write":173,"num_write_kb":1276,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[2,7],"avail_no_missing":["2","3","7"],"object_location_counts":[{"shards":"2,3,7","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":908,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-09-23T20:40:29.681412+0000","last_change":"2023-09-23T20:38:05.602508+0000","last_active":"2023-09-23T20:40:29.681412+0000","last_peered":"2023-09-23T20:40:29.681412+0000","last_clean":"2023-09-23T20:40:29.681412+0000","last_became_active":"2023-09-23T20:38:05.602129+0000","last_became_peered":"2023-09-23T20:38:05.602129+0000","last_unstale":"2023-09-23T20:40:29.681412+0000","last_undegraded":"2023-09-23T20:40:29.681412+0000","last_fullsized":"2023-09-23T20:40:29.681412+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:45.343742+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:45.343742+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:48.910343+0000","scrub_duration":0.001475802,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":15,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-09-23T20:40:29.692208+0000","last_change":"2023-09-23T20:40:19.542142+0000","last_active":"2023-09-23T20:40:29.692208+0000","last_peered":"2023-09-23T20:40:29.692208+0000","last_clean":"2023-09-23T20:40:29.692208+0000","last_became_active":"2023-09-23T20:40:19.541870+0000","last_became_peered":"2023-09-23T20:40:19.541870+0000","last_unstale":"2023-09-23T20:40:29.692208+0000","last_undegraded":"2023-09-23T20:40:29.692208+0000","last_fullsized":"2023-09-23T20:40:29.692208+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:01.518149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":52,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-09-23T20:40:29.677364+0000","last_change":"2023-09-23T20:38:02.573200+0000","last_active":"2023-09-23T20:40:29.677364+0000","last_peered":"2023-09-23T20:40:29.677364+0000","last_clean":"2023-09-23T20:40:29.677364+0000","last_became_active":"2023-09-23T20:38:02.572945+0000","last_became_peered":"2023-09-23T20:38:02.572945+0000","last_unstale":"2023-09-23T20:40:29.677364+0000","last_undegraded":"2023-09-23T20:40:29.677364+0000","last_fullsized":"2023-09-23T20:40:29.677364+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:57.246533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.a","version":"445'11330","reported_seq":22129,"reported_epoch":445,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T20:40:39.170837+0000","last_change":"2023-09-23T20:40:20.700955+0000","last_active":"2023-09-23T20:40:39.170837+0000","last_peered":"2023-09-23T20:40:39.170837+0000","last_clean":"2023-09-23T20:40:18.471731+0000","last_became_active":"2023-09-23T20:40:20.661915+0000","last_became_peered":"2023-09-23T20:40:20.661915+0000","last_unstale":"2023-09-23T20:40:39.170837+0000","last_undegraded":"2023-09-23T20:40:39.170837+0000","last_fullsized":"2023-09-23T20:40:39.170837+0000","mapping_epoch":441,"log_start":"402'5979","ondisk_log_start":"402'5979","created":369,"last_epoch_clean":427,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":5351,"log_dups_size":0,"ondisk_log_size":5351,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:31:16.196574+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11330,"num_write_kb":362560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5339,"num_bytes_recovered":313851904,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,3],"avail_no_missing":["1","2","3","5"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":775,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-09-23T20:40:22.404133+0000","last_change":"2023-09-23T20:38:05.615711+0000","last_active":"2023-09-23T20:40:22.404133+0000","last_peered":"2023-09-23T20:40:22.404133+0000","last_clean":"2023-09-23T20:40:22.404133+0000","last_became_active":"2023-09-23T20:38:05.615390+0000","last_became_peered":"2023-09-23T20:38:05.615390+0000","last_unstale":"2023-09-23T20:40:22.404133+0000","last_undegraded":"2023-09-23T20:40:22.404133+0000","last_fullsized":"2023-09-23T20:40:22.404133+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:43.582146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:43.582146+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:45:11.917871+0000","scrub_duration":0.0016219730000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"86'80","reported_seq":698,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-09-23T20:40:29.681315+0000","last_change":"2023-09-23T20:29:45.536796+0000","last_active":"2023-09-23T20:40:29.681315+0000","last_peered":"2023-09-23T20:40:29.681315+0000","last_clean":"2023-09-23T20:40:29.681315+0000","last_became_active":"2023-09-23T20:29:45.536439+0000","last_became_peered":"2023-09-23T20:29:45.536439+0000","last_unstale":"2023-09-23T20:40:29.681315+0000","last_undegraded":"2023-09-23T20:40:29.681315+0000","last_fullsized":"2023-09-23T20:40:29.681315+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":80,"log_dups_size":0,"ondisk_log_size":80,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:30:57.793132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035115,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":395,"num_read_kb":1370,"num_write":66,"num_write_kb":1061,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":450,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-09-23T20:40:29.682196+0000","last_change":"2023-09-23T20:34:59.984286+0000","last_active":"2023-09-23T20:40:29.682196+0000","last_peered":"2023-09-23T20:40:29.682196+0000","last_clean":"2023-09-23T20:40:29.682196+0000","last_became_active":"2023-09-23T20:34:59.983950+0000","last_became_peered":"2023-09-23T20:34:59.983950+0000","last_unstale":"2023-09-23T20:40:29.682196+0000","last_undegraded":"2023-09-23T20:40:29.682196+0000","last_fullsized":"2023-09-23T20:40:29.682196+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:31:53.557035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":445,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-09-23T20:40:29.682639+0000","last_change":"2023-09-23T20:29:45.690815+0000","last_active":"2023-09-23T20:40:29.682639+0000","last_peered":"2023-09-23T20:40:29.682639+0000","last_clean":"2023-09-23T20:40:29.682639+0000","last_became_active":"2023-09-23T20:29:45.690478+0000","last_became_peered":"2023-09-23T20:29:45.690478+0000","last_unstale":"2023-09-23T20:40:29.682639+0000","last_undegraded":"2023-09-23T20:40:29.682639+0000","last_fullsized":"2023-09-23T20:40:29.682639+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:38.288193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.f","version":"445'11034","reported_seq":11122,"reported_epoch":446,"state":"active+clean","last_fresh":"2023-09-23T20:40:39.657855+0000","last_change":"2023-09-23T20:36:11.309536+0000","last_active":"2023-09-23T20:40:39.657855+0000","last_peered":"2023-09-23T20:40:39.657855+0000","last_clean":"2023-09-23T20:40:39.657855+0000","last_became_active":"2023-09-23T20:36:11.309208+0000","last_became_peered":"2023-09-23T20:36:11.309208+0000","last_unstale":"2023-09-23T20:40:39.657855+0000","last_undegraded":"2023-09-23T20:40:39.657855+0000","last_fullsized":"2023-09-23T20:40:39.657855+0000","mapping_epoch":369,"log_start":"375'3300","ondisk_log_start":"375'3300","created":369,"last_epoch_clean":370,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":7734,"log_dups_size":0,"ondisk_log_size":7734,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:47:51.785158+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11034,"num_write_kb":353088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,1],"acting":[2,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":548,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-09-23T20:40:29.671956+0000","last_change":"2023-09-23T20:38:05.611181+0000","last_active":"2023-09-23T20:40:29.671956+0000","last_peered":"2023-09-23T20:40:29.671956+0000","last_clean":"2023-09-23T20:40:29.671956+0000","last_became_active":"2023-09-23T20:38:05.610881+0000","last_became_peered":"2023-09-23T20:38:05.610881+0000","last_unstale":"2023-09-23T20:40:29.671956+0000","last_undegraded":"2023-09-23T20:40:29.671956+0000","last_fullsized":"2023-09-23T20:40:29.671956+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:35:00.151695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:35:00.151695+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":20,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"86'102","reported_seq":1040,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-09-23T20:40:29.685381+0000","last_change":"2023-09-23T20:32:36.314672+0000","last_active":"2023-09-23T20:40:29.685381+0000","last_peered":"2023-09-23T20:40:29.685381+0000","last_clean":"2023-09-23T20:40:29.685381+0000","last_became_active":"2023-09-23T20:32:36.314231+0000","last_became_peered":"2023-09-23T20:32:36.314231+0000","last_unstale":"2023-09-23T20:40:29.685381+0000","last_undegraded":"2023-09-23T20:40:29.685381+0000","last_fullsized":"2023-09-23T20:40:29.685381+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":102,"log_dups_size":0,"ondisk_log_size":102,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:11:40.514709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030656500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":963,"num_read_kb":1234,"num_write":60,"num_write_kb":225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":390,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-09-23T20:40:22.403534+0000","last_change":"2023-09-23T20:40:19.544384+0000","last_active":"2023-09-23T20:40:22.403534+0000","last_peered":"2023-09-23T20:40:22.403534+0000","last_clean":"2023-09-23T20:40:22.403534+0000","last_became_active":"2023-09-23T20:40:19.544079+0000","last_became_peered":"2023-09-23T20:40:19.544079+0000","last_unstale":"2023-09-23T20:40:22.403534+0000","last_undegraded":"2023-09-23T20:40:22.403534+0000","last_fullsized":"2023-09-23T20:40:22.403534+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:00:19.498631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":400,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-09-23T20:40:29.681692+0000","last_change":"2023-09-23T20:40:19.532976+0000","last_active":"2023-09-23T20:40:29.681692+0000","last_peered":"2023-09-23T20:40:29.681692+0000","last_clean":"2023-09-23T20:40:29.681692+0000","last_became_active":"2023-09-23T20:40:19.532639+0000","last_became_peered":"2023-09-23T20:40:19.532639+0000","last_unstale":"2023-09-23T20:40:29.681692+0000","last_undegraded":"2023-09-23T20:40:29.681692+0000","last_fullsized":"2023-09-23T20:40:29.681692+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:02:35.301884+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"445'11304","reported_seq":11392,"reported_epoch":446,"state":"active+clean","last_fresh":"2023-09-23T20:40:39.696840+0000","last_change":"2023-09-23T20:36:11.482126+0000","last_active":"2023-09-23T20:40:39.696840+0000","last_peered":"2023-09-23T20:40:39.696840+0000","last_clean":"2023-09-23T20:40:39.696840+0000","last_became_active":"2023-09-23T20:36:11.481802+0000","last_became_peered":"2023-09-23T20:36:11.481802+0000","last_unstale":"2023-09-23T20:40:39.696840+0000","last_undegraded":"2023-09-23T20:40:39.696840+0000","last_fullsized":"2023-09-23T20:40:39.696840+0000","mapping_epoch":369,"log_start":"377'4959","ondisk_log_start":"377'4959","created":369,"last_epoch_clean":370,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":6345,"log_dups_size":0,"ondisk_log_size":6345,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:13:12.263455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11304,"num_write_kb":361728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":724,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-09-23T20:40:29.679138+0000","last_change":"2023-09-23T20:38:05.590361+0000","last_active":"2023-09-23T20:40:29.679138+0000","last_peered":"2023-09-23T20:40:29.679138+0000","last_clean":"2023-09-23T20:40:29.679138+0000","last_became_active":"2023-09-23T20:38:05.589985+0000","last_became_peered":"2023-09-23T20:38:05.589985+0000","last_unstale":"2023-09-23T20:40:29.679138+0000","last_undegraded":"2023-09-23T20:40:29.679138+0000","last_fullsized":"2023-09-23T20:40:29.679138+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:03.246271+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:03.246271+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:38:54.351576+0000","scrub_duration":0.0019016700000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":183,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-09-23T20:40:29.684988+0000","last_change":"2023-09-23T20:32:50.549695+0000","last_active":"2023-09-23T20:40:29.684988+0000","last_peered":"2023-09-23T20:40:29.684988+0000","last_clean":"2023-09-23T20:40:29.684988+0000","last_became_active":"2023-09-23T20:32:50.549431+0000","last_became_peered":"2023-09-23T20:32:50.549431+0000","last_unstale":"2023-09-23T20:40:29.684988+0000","last_undegraded":"2023-09-23T20:40:29.684988+0000","last_fullsized":"2023-09-23T20:40:29.684988+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:45:53.666569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"86'128","reported_seq":856,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-09-23T20:40:29.704666+0000","last_change":"2023-09-23T20:38:03.541765+0000","last_active":"2023-09-23T20:40:29.704666+0000","last_peered":"2023-09-23T20:40:29.704666+0000","last_clean":"2023-09-23T20:40:29.704666+0000","last_became_active":"2023-09-23T20:38:03.541277+0000","last_became_peered":"2023-09-23T20:38:03.541277+0000","last_unstale":"2023-09-23T20:40:29.704666+0000","last_undegraded":"2023-09-23T20:40:29.704666+0000","last_fullsized":"2023-09-23T20:40:29.704666+0000","mapping_epoch":404,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":405,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:20:53.834554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000293202,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":579,"num_read_kb":1473,"num_write":96,"num_write_kb":1062,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":225,"reported_epoch":446,"state":"active+clean","last_fresh":"2023-09-23T20:40:39.229537+0000","last_change":"2023-09-23T20:37:43.582105+0000","last_active":"2023-09-23T20:40:39.229537+0000","last_peered":"2023-09-23T20:40:39.229537+0000","last_clean":"2023-09-23T20:40:39.229537+0000","last_became_active":"2023-09-23T20:37:43.581860+0000","last_became_peered":"2023-09-23T20:37:43.581860+0000","last_unstale":"2023-09-23T20:40:39.229537+0000","last_undegraded":"2023-09-23T20:40:39.229537+0000","last_fullsized":"2023-09-23T20:40:39.229537+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:18:38.765486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"445'11128","reported_seq":18812,"reported_epoch":445,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:40:39.204995+0000","last_change":"2023-09-23T20:40:19.729086+0000","last_active":"2023-09-23T20:40:39.204995+0000","last_peered":"2023-09-23T20:40:39.204995+0000","last_clean":"2023-09-23T20:37:13.317209+0000","last_became_active":"2023-09-23T20:40:19.694465+0000","last_became_peered":"2023-09-23T20:40:19.694465+0000","last_unstale":"2023-09-23T20:40:39.204995+0000","last_undegraded":"2023-09-23T20:40:19.549561+0000","last_fullsized":"2023-09-23T20:40:19.517168+0000","mapping_epoch":440,"log_start":"397'5421","ondisk_log_start":"397'5421","created":369,"last_epoch_clean":370,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":5707,"log_dups_size":0,"ondisk_log_size":5707,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:13:44.257914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5029,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":11128,"num_write_kb":356033,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3780,"num_bytes_recovered":247726080,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[7,2],"avail_no_missing":["7","2"],"object_location_counts":[{"shards":"2,7","objects":-295}],"blocked_by":[],"up_primary":5,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.4","version":"90'161","reported_seq":659,"reported_epoch":446,"state":"active+clean","last_fresh":"2023-09-23T20:40:39.230206+0000","last_change":"2023-09-23T20:40:22.427435+0000","last_active":"2023-09-23T20:40:39.230206+0000","last_peered":"2023-09-23T20:40:39.230206+0000","last_clean":"2023-09-23T20:40:39.230206+0000","last_became_active":"2023-09-23T20:40:22.427051+0000","last_became_peered":"2023-09-23T20:40:22.427051+0000","last_unstale":"2023-09-23T20:40:39.230206+0000","last_undegraded":"2023-09-23T20:40:39.230206+0000","last_fullsized":"2023-09-23T20:40:39.230206+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:34:54.504554+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:34:54.504554+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:42.001652+0000","scrub_duration":0.011176864,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"86'117","reported_seq":679,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-09-23T20:40:29.672106+0000","last_change":"2023-09-23T20:38:28.446107+0000","last_active":"2023-09-23T20:40:29.672106+0000","last_peered":"2023-09-23T20:40:29.672106+0000","last_clean":"2023-09-23T20:40:29.672106+0000","last_became_active":"2023-09-23T20:38:28.445667+0000","last_became_peered":"2023-09-23T20:38:28.445667+0000","last_unstale":"2023-09-23T20:40:29.672106+0000","last_undegraded":"2023-09-23T20:40:29.672106+0000","last_fullsized":"2023-09-23T20:40:29.672106+0000","mapping_epoch":411,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":412,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:58:47.430360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046891400000000002,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":893,"num_read_kb":922,"num_write":82,"num_write_kb":4193,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":445,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-09-23T20:40:29.682725+0000","last_change":"2023-09-23T20:32:33.277827+0000","last_active":"2023-09-23T20:40:29.682725+0000","last_peered":"2023-09-23T20:40:29.682725+0000","last_clean":"2023-09-23T20:40:29.682725+0000","last_became_active":"2023-09-23T20:32:33.236087+0000","last_became_peered":"2023-09-23T20:32:33.236087+0000","last_unstale":"2023-09-23T20:40:29.682725+0000","last_undegraded":"2023-09-23T20:40:29.682725+0000","last_fullsized":"2023-09-23T20:40:29.682725+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:20:58.176854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":183,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-09-23T20:40:29.680428+0000","last_change":"2023-09-23T20:40:19.530602+0000","last_active":"2023-09-23T20:40:29.680428+0000","last_peered":"2023-09-23T20:40:29.680428+0000","last_clean":"2023-09-23T20:40:29.680428+0000","last_became_active":"2023-09-23T20:40:19.530256+0000","last_became_peered":"2023-09-23T20:40:19.530256+0000","last_unstale":"2023-09-23T20:40:29.680428+0000","last_undegraded":"2023-09-23T20:40:29.680428+0000","last_fullsized":"2023-09-23T20:40:29.680428+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:18:45.658867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.10","version":"445'8701","reported_seq":15157,"reported_epoch":446,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T20:40:39.229131+0000","last_change":"2023-09-23T20:40:20.743359+0000","last_active":"2023-09-23T20:40:39.229131+0000","last_peered":"2023-09-23T20:40:39.229131+0000","last_clean":"2023-09-23T20:40:18.470747+0000","last_became_active":"2023-09-23T20:40:20.710426+0000","last_became_peered":"2023-09-23T20:40:20.710426+0000","last_unstale":"2023-09-23T20:40:39.229131+0000","last_undegraded":"2023-09-23T20:40:39.229131+0000","last_fullsized":"2023-09-23T20:40:39.229131+0000","mapping_epoch":441,"log_start":"373'2187","ondisk_log_start":"373'2187","created":410,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":6514,"log_dups_size":0,"ondisk_log_size":6514,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":65536,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5454,"num_write_kb":174560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2537,"num_bytes_recovered":166264832,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,1,4],"avail_no_missing":["3","1","4"],"object_location_counts":[{"shards":"1,3,4","objects":1}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.19","version":"90'161","reported_seq":715,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-09-23T20:40:29.682029+0000","last_change":"2023-09-23T20:38:05.630221+0000","last_active":"2023-09-23T20:40:29.682029+0000","last_peered":"2023-09-23T20:40:29.682029+0000","last_clean":"2023-09-23T20:40:29.682029+0000","last_became_active":"2023-09-23T20:38:05.629839+0000","last_became_peered":"2023-09-23T20:38:05.629839+0000","last_unstale":"2023-09-23T20:40:29.682029+0000","last_undegraded":"2023-09-23T20:40:29.682029+0000","last_fullsized":"2023-09-23T20:40:29.682029+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:06.487351+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:06.487351+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:25:51.325529+0000","scrub_duration":0.001964379,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":413,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-09-23T20:40:29.681624+0000","last_change":"2023-09-23T20:32:33.237234+0000","last_active":"2023-09-23T20:40:29.681624+0000","last_peered":"2023-09-23T20:40:29.681624+0000","last_clean":"2023-09-23T20:40:29.681624+0000","last_became_active":"2023-09-23T20:32:33.236881+0000","last_became_peered":"2023-09-23T20:32:33.236881+0000","last_unstale":"2023-09-23T20:40:29.681624+0000","last_undegraded":"2023-09-23T20:40:29.681624+0000","last_fullsized":"2023-09-23T20:40:29.681624+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:56:17.384831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":301,"reported_epoch":446,"state":"active+clean","last_fresh":"2023-09-23T20:40:39.657516+0000","last_change":"2023-09-23T20:29:45.538314+0000","last_active":"2023-09-23T20:40:39.657516+0000","last_peered":"2023-09-23T20:40:39.657516+0000","last_clean":"2023-09-23T20:40:39.657516+0000","last_became_active":"2023-09-23T20:29:45.536847+0000","last_became_peered":"2023-09-23T20:29:45.536847+0000","last_unstale":"2023-09-23T20:40:39.657516+0000","last_undegraded":"2023-09-23T20:40:39.657516+0000","last_fullsized":"2023-09-23T20:40:39.657516+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:51.948721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"8.17","version":"445'8858","reported_seq":9625,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-09-23T20:40:39.130182+0000","last_change":"2023-09-23T20:40:19.281439+0000","last_active":"2023-09-23T20:40:39.130182+0000","last_peered":"2023-09-23T20:40:39.130182+0000","last_clean":"2023-09-23T20:40:39.130182+0000","last_became_active":"2023-09-23T20:38:27.474930+0000","last_became_peered":"2023-09-23T20:38:27.474930+0000","last_unstale":"2023-09-23T20:40:39.130182+0000","last_undegraded":"2023-09-23T20:40:39.130182+0000","last_fullsized":"2023-09-23T20:40:39.130182+0000","mapping_epoch":410,"log_start":"373'1658","ondisk_log_start":"373'1658","created":410,"last_epoch_clean":440,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":7200,"log_dups_size":0,"ondisk_log_size":7200,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5568,"num_write_kb":177824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":327,"num_bytes_recovered":10354688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1e","version":"90'161","reported_seq":506,"reported_epoch":446,"state":"active+clean","last_fresh":"2023-09-23T20:40:39.229256+0000","last_change":"2023-09-23T20:38:05.602250+0000","last_active":"2023-09-23T20:40:39.229256+0000","last_peered":"2023-09-23T20:40:39.229256+0000","last_clean":"2023-09-23T20:40:39.229256+0000","last_became_active":"2023-09-23T20:38:05.601874+0000","last_became_peered":"2023-09-23T20:38:05.601874+0000","last_unstale":"2023-09-23T20:40:39.229256+0000","last_undegraded":"2023-09-23T20:40:39.229256+0000","last_fullsized":"2023-09-23T20:40:39.229256+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:35:42.864363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:35:42.864363+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":40253,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":44,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-09-23T20:40:29.692935+0000","last_change":"2023-09-23T20:38:37.481551+0000","last_active":"2023-09-23T20:40:29.692935+0000","last_peered":"2023-09-23T20:40:29.692935+0000","last_clean":"2023-09-23T20:40:29.692935+0000","last_became_active":"2023-09-23T20:38:37.458687+0000","last_became_peered":"2023-09-23T20:38:37.458687+0000","last_unstale":"2023-09-23T20:40:29.692935+0000","last_undegraded":"2023-09-23T20:40:29.692935+0000","last_fullsized":"2023-09-23T20:40:29.692935+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:22:15.033243+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":102,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-09-23T20:40:29.679454+0000","last_change":"2023-09-23T20:34:59.987575+0000","last_active":"2023-09-23T20:40:29.679454+0000","last_peered":"2023-09-23T20:40:29.679454+0000","last_clean":"2023-09-23T20:40:29.679454+0000","last_became_active":"2023-09-23T20:34:59.987229+0000","last_became_peered":"2023-09-23T20:34:59.987229+0000","last_unstale":"2023-09-23T20:40:29.679454+0000","last_undegraded":"2023-09-23T20:40:29.679454+0000","last_fullsized":"2023-09-23T20:40:29.679454+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:34:01.820159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.16","version":"445'8755","reported_seq":9540,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-09-23T20:40:39.083496+0000","last_change":"2023-09-23T20:38:28.264940+0000","last_active":"2023-09-23T20:40:39.083496+0000","last_peered":"2023-09-23T20:40:39.083496+0000","last_clean":"2023-09-23T20:40:39.083496+0000","last_became_active":"2023-09-23T20:38:27.484333+0000","last_became_peered":"2023-09-23T20:38:27.484333+0000","last_unstale":"2023-09-23T20:40:39.083496+0000","last_undegraded":"2023-09-23T20:40:39.083496+0000","last_fullsized":"2023-09-23T20:40:39.083496+0000","mapping_epoch":410,"log_start":"373'1261","ondisk_log_start":"373'1261","created":410,"last_epoch_clean":411,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":7494,"log_dups_size":0,"ondisk_log_size":7494,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":5177344,"num_objects":79,"num_object_clones":0,"num_object_copies":237,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":79,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5475,"num_write_kb":177728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":335,"num_bytes_recovered":9699328,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":44,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-09-23T20:40:29.693689+0000","last_change":"2023-09-23T20:38:37.457634+0000","last_active":"2023-09-23T20:40:29.693689+0000","last_peered":"2023-09-23T20:40:29.693689+0000","last_clean":"2023-09-23T20:40:29.693689+0000","last_became_active":"2023-09-23T20:38:37.452208+0000","last_became_peered":"2023-09-23T20:38:37.452208+0000","last_unstale":"2023-09-23T20:40:29.693689+0000","last_undegraded":"2023-09-23T20:40:29.693689+0000","last_fullsized":"2023-09-23T20:40:29.693689+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:09:41.754569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":102,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-09-23T20:40:29.679458+0000","last_change":"2023-09-23T20:34:59.987416+0000","last_active":"2023-09-23T20:40:29.679458+0000","last_peered":"2023-09-23T20:40:29.679458+0000","last_clean":"2023-09-23T20:40:29.679458+0000","last_became_active":"2023-09-23T20:34:59.987117+0000","last_became_peered":"2023-09-23T20:34:59.987117+0000","last_unstale":"2023-09-23T20:40:29.679458+0000","last_undegraded":"2023-09-23T20:40:29.679458+0000","last_fullsized":"2023-09-23T20:40:29.679458+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:36:16.318623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.15","version":"445'8832","reported_seq":12408,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-09-23T20:40:39.114815+0000","last_change":"2023-09-23T20:38:53.167534+0000","last_active":"2023-09-23T20:40:39.114815+0000","last_peered":"2023-09-23T20:40:39.114815+0000","last_clean":"2023-09-23T20:40:39.114815+0000","last_became_active":"2023-09-23T20:38:28.579407+0000","last_became_peered":"2023-09-23T20:38:28.579407+0000","last_unstale":"2023-09-23T20:40:39.114815+0000","last_undegraded":"2023-09-23T20:40:39.114815+0000","last_fullsized":"2023-09-23T20:40:39.114815+0000","mapping_epoch":410,"log_start":"373'1839","ondisk_log_start":"373'1839","created":410,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":6993,"log_dups_size":0,"ondisk_log_size":6993,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1081344,"num_objects":17,"num_object_clones":0,"num_object_copies":51,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":17,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5537,"num_write_kb":177696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1043,"num_bytes_recovered":56328192,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1c","version":"90'161","reported_seq":701,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-09-23T20:40:29.684684+0000","last_change":"2023-09-23T20:38:05.591992+0000","last_active":"2023-09-23T20:40:29.684684+0000","last_peered":"2023-09-23T20:40:29.684684+0000","last_clean":"2023-09-23T20:40:29.684684+0000","last_became_active":"2023-09-23T20:38:05.591701+0000","last_became_peered":"2023-09-23T20:38:05.591701+0000","last_unstale":"2023-09-23T20:40:29.684684+0000","last_undegraded":"2023-09-23T20:40:29.684684+0000","last_fullsized":"2023-09-23T20:40:29.684684+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:02.384133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:02.384133+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:41.722542+0000","scrub_duration":0.001769098,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":267,"reported_epoch":446,"state":"active+clean","last_fresh":"2023-09-23T20:40:39.229392+0000","last_change":"2023-09-23T20:38:02.542719+0000","last_active":"2023-09-23T20:40:39.229392+0000","last_peered":"2023-09-23T20:40:39.229392+0000","last_clean":"2023-09-23T20:40:39.229392+0000","last_became_active":"2023-09-23T20:38:02.542385+0000","last_became_peered":"2023-09-23T20:38:02.542385+0000","last_unstale":"2023-09-23T20:40:39.229392+0000","last_undegraded":"2023-09-23T20:40:39.229392+0000","last_fullsized":"2023-09-23T20:40:39.229392+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:36:23.673757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":234,"reported_epoch":446,"state":"active+clean","last_fresh":"2023-09-23T20:40:39.228884+0000","last_change":"2023-09-23T20:37:43.584253+0000","last_active":"2023-09-23T20:40:39.228884+0000","last_peered":"2023-09-23T20:40:39.228884+0000","last_clean":"2023-09-23T20:40:39.228884+0000","last_became_active":"2023-09-23T20:37:43.580069+0000","last_became_peered":"2023-09-23T20:37:43.580069+0000","last_unstale":"2023-09-23T20:40:39.228884+0000","last_undegraded":"2023-09-23T20:40:39.228884+0000","last_fullsized":"2023-09-23T20:40:39.228884+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:39:11.482581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.14","version":"445'9080","reported_seq":11301,"reported_epoch":445,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T20:40:39.143145+0000","last_change":"2023-09-23T20:40:20.758265+0000","last_active":"2023-09-23T20:40:39.143145+0000","last_peered":"2023-09-23T20:40:39.143145+0000","last_clean":"2023-09-23T20:37:15.925522+0000","last_became_active":"2023-09-23T20:38:39.251212+0000","last_became_peered":"2023-09-23T20:38:39.251212+0000","last_unstale":"2023-09-23T20:40:39.143145+0000","last_undegraded":"2023-09-23T20:40:20.756509+0000","last_fullsized":"2023-09-23T20:38:38.450156+0000","mapping_epoch":416,"log_start":"373'2080","ondisk_log_start":"373'2080","created":410,"last_epoch_clean":370,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:36:10.283416+0000","last_clean_scrub_stamp":"2023-09-23T20:36:10.283416+0000","objects_scrubbed":0,"log_size":7000,"log_dups_size":0,"ondisk_log_size":7000,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1310720,"num_objects":20,"num_object_clones":0,"num_object_copies":60,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":387,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":20,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5673,"num_write_kb":182176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":682,"num_bytes_recovered":34406400,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[4,2],"avail_no_missing":["4","2","6"],"object_location_counts":[{"shards":"2,4","objects":20}],"blocked_by":[],"up_primary":5,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1d","version":"90'161","reported_seq":537,"reported_epoch":444,"state":"active+clean","last_fresh":"2023-09-23T20:40:24.544645+0000","last_change":"2023-09-23T20:36:01.505098+0000","last_active":"2023-09-23T20:40:24.544645+0000","last_peered":"2023-09-23T20:40:24.544645+0000","last_clean":"2023-09-23T20:40:24.544645+0000","last_became_active":"2023-09-23T20:35:26.644707+0000","last_became_peered":"2023-09-23T20:35:26.644707+0000","last_unstale":"2023-09-23T20:40:24.544645+0000","last_undegraded":"2023-09-23T20:40:24.544645+0000","last_fullsized":"2023-09-23T20:40:24.544645+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:01.504973+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:01.504973+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:02:10.880809+0000","scrub_duration":0.00202344,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":490,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-09-23T20:40:29.681696+0000","last_change":"2023-09-23T20:40:19.569032+0000","last_active":"2023-09-23T20:40:29.681696+0000","last_peered":"2023-09-23T20:40:29.681696+0000","last_clean":"2023-09-23T20:40:29.681696+0000","last_became_active":"2023-09-23T20:40:19.538476+0000","last_became_peered":"2023-09-23T20:40:19.538476+0000","last_unstale":"2023-09-23T20:40:29.681696+0000","last_undegraded":"2023-09-23T20:40:29.681696+0000","last_fullsized":"2023-09-23T20:40:29.681696+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:31:58.998168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3324,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":163,"reported_epoch":445,"state":"active+clean","last_fresh":"2023-09-23T20:40:29.673521+0000","last_change":"2023-09-23T20:38:02.538333+0000","last_active":"2023-09-23T20:40:29.673521+0000","last_peered":"2023-09-23T20:40:29.673521+0000","last_clean":"2023-09-23T20:40:29.673521+0000","last_became_active":"2023-09-23T20:38:02.538008+0000","last_became_peered":"2023-09-23T20:38:02.538008+0000","last_unstale":"2023-09-23T20:40:29.673521+0000","last_undegraded":"2023-09-23T20:40:29.673521+0000","last_fullsized":"2023-09-23T20:40:29.673521+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:02:38.789530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":80,"num_bytes_recovered":18486,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26691,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":8,"num_pg":26,"stat_sum":{"num_bytes":14155776,"num_objects":216,"num_object_clones":0,"num_object_copies":648,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":14969,"num_objects_misplaced":45,"num_objects_unfound":0,"num_objects_dirty":216,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":178642,"num_write_kb":5716481,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":35371,"num_bytes_recovered":2082209792,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1671892992,"data_stored":1671888920,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":178090,"ondisk_log_size":178090,"up":78,"acting":69,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":351684,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":155,"num_read_kb":290,"num_write":76,"num_write_kb":768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":891212,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5983,"num_read_kb":13200,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":57,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6082,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":23,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201631,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":388292,"kb_used_data":9084,"kb_used_omap":0,"kb_used_meta":379200,"kb_avail":93356860,"statfs":{"total":95995035648,"available":95597424640,"internally_reserved":0,"allocated":9302016,"data_stored":7878212,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":388300800},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:39:51 2023","interfaces":[{"interface":"back","average":{"1min":0.81899999999999995,"5min":2.395,"15min":2.177},"min":{"1min":0.25800000000000001,"5min":0.24399999999999999,"15min":0.20799999999999999},"max":{"1min":3.7639999999999998,"5min":18.315000000000001,"15min":48.798999999999999},"last":0.50600000000000001},{"interface":"front","average":{"1min":0.81599999999999995,"5min":1.7150000000000001,"15min":2.008},"min":{"1min":0.29199999999999998,"5min":0.247,"15min":0.24399999999999999},"max":{"1min":7.109,"5min":19.355,"15min":48.247},"last":0.25800000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:40:03 2023","interfaces":[{"interface":"back","average":{"1min":0.92600000000000005,"5min":2.7120000000000002,"15min":2.1520000000000001},"min":{"1min":0.28799999999999998,"5min":0.25900000000000001,"15min":0.254},"max":{"1min":2.9020000000000001,"5min":49.965000000000003,"15min":49.965000000000003},"last":0.40899999999999997},{"interface":"front","average":{"1min":0.89300000000000002,"5min":2.6989999999999998,"15min":2.1709999999999998},"min":{"1min":0.30299999999999999,"5min":0.26900000000000002,"15min":0.248},"max":{"1min":5.508,"5min":53.451999999999998,"15min":53.451999999999998},"last":0.40100000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:39:51 2023","interfaces":[{"interface":"back","average":{"1min":1.23,"5min":2.129,"15min":2.1469999999999998},"min":{"1min":0.32800000000000001,"5min":0.27800000000000002,"15min":0.27600000000000002},"max":{"1min":7.2910000000000004,"5min":16.965,"15min":16.965},"last":0.35799999999999998},{"interface":"front","average":{"1min":1.1140000000000001,"5min":2.6459999999999999,"15min":2.48},"min":{"1min":0.32500000000000001,"5min":0.308,"15min":0.26600000000000001},"max":{"1min":3.6880000000000002,"5min":23.414000000000001,"15min":23.494},"last":0.30299999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:40:21 2023","interfaces":[{"interface":"back","average":{"1min":1.105,"5min":2.2480000000000002,"15min":2.4470000000000001},"min":{"1min":0.35999999999999999,"5min":0.311,"15min":0.26200000000000001},"max":{"1min":4.2149999999999999,"5min":23.352,"15min":23.352},"last":0.42299999999999999},{"interface":"front","average":{"1min":1.139,"5min":2.2709999999999999,"15min":2.423},"min":{"1min":0.46500000000000002,"5min":0.34399999999999997,"15min":0.32500000000000001},"max":{"1min":4.2359999999999998,"5min":20.969999999999999,"15min":20.969999999999999},"last":0.46500000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:40:21 2023","interfaces":[{"interface":"back","average":{"1min":1.046,"5min":1.085,"15min":1.2170000000000001},"min":{"1min":0.35799999999999998,"5min":0.25600000000000001,"15min":0.23200000000000001},"max":{"1min":4.1639999999999997,"5min":18.364999999999998,"15min":18.364999999999998},"last":0.45200000000000001},{"interface":"front","average":{"1min":0.94999999999999996,"5min":1.296,"15min":1.417},"min":{"1min":0.39100000000000001,"5min":0.28399999999999997,"15min":0.27600000000000002},"max":{"1min":2.605,"5min":23.617000000000001,"15min":23.617000000000001},"last":0.495}]},{"osd":5,"last update":"Sat Sep 23 20:40:21 2023","interfaces":[{"interface":"back","average":{"1min":1.0780000000000001,"5min":1.1499999999999999,"15min":1.2509999999999999},"min":{"1min":0.30099999999999999,"5min":0.29199999999999998,"15min":0.26600000000000001},"max":{"1min":4.3780000000000001,"5min":12.516,"15min":13.241},"last":0.47499999999999998},{"interface":"front","average":{"1min":0.76400000000000001,"5min":1.286,"15min":1.3540000000000001},"min":{"1min":0.38400000000000001,"5min":0.31900000000000001,"15min":0.315},"max":{"1min":3.1030000000000002,"5min":18.390000000000001,"15min":18.831},"last":0.48499999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:40:21 2023","interfaces":[{"interface":"back","average":{"1min":1.2629999999999999,"5min":1.4350000000000001,"15min":1.329},"min":{"1min":0.44,"5min":0.312,"15min":0.30399999999999999},"max":{"1min":4.0540000000000003,"5min":18.417000000000002,"15min":18.417000000000002},"last":0.53000000000000003},{"interface":"front","average":{"1min":1.1659999999999999,"5min":1.113,"15min":1.252},"min":{"1min":0.38200000000000001,"5min":0.32700000000000001,"15min":0.318},"max":{"1min":6.4340000000000002,"5min":7.266,"15min":9.9120000000000008},"last":0.437}]}]},{"osd":6,"up_from":47,"seq":201863463270,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":479672,"kb_used_data":92336,"kb_used_omap":0,"kb_used_meta":387328,"kb_avail":93265480,"statfs":{"total":95995035648,"available":95503851520,"internally_reserved":0,"allocated":94552064,"data_stored":93139877,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":396623872},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:39:49 2023","interfaces":[{"interface":"back","average":{"1min":0.89500000000000002,"5min":1.859,"15min":1.9339999999999999},"min":{"1min":0.23699999999999999,"5min":0.23300000000000001,"15min":0.221},"max":{"1min":2.887,"5min":17.492999999999999,"15min":53.701999999999998},"last":0.39500000000000002},{"interface":"front","average":{"1min":1.4219999999999999,"5min":2.3730000000000002,"15min":2.4409999999999998},"min":{"1min":0.24099999999999999,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":6.625,"5min":60.328000000000003,"15min":60.328000000000003},"last":0.49199999999999999}]},{"osd":1,"last update":"Sat Sep 23 20:40:06 2023","interfaces":[{"interface":"back","average":{"1min":1.157,"5min":2.0600000000000001,"15min":2.3610000000000002},"min":{"1min":0.26600000000000001,"5min":0.26000000000000001,"15min":0.249},"max":{"1min":5.8630000000000004,"5min":64.274000000000001,"15min":64.274000000000001},"last":0.379},{"interface":"front","average":{"1min":1.2390000000000001,"5min":1.393,"15min":2.4830000000000001},"min":{"1min":0.34499999999999997,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":6.8460000000000001,"5min":6.8460000000000001,"15min":34.490000000000002},"last":0.48199999999999998}]},{"osd":2,"last update":"Sat Sep 23 20:39:49 2023","interfaces":[{"interface":"back","average":{"1min":1.069,"5min":1.748,"15min":2.1499999999999999},"min":{"1min":0.30299999999999999,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":4.3399999999999999,"5min":13.429,"15min":34.317999999999998},"last":0.42999999999999999},{"interface":"front","average":{"1min":1.0740000000000001,"5min":1.657,"15min":2.024},"min":{"1min":0.40000000000000002,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":4.3630000000000004,"5min":13.457000000000001,"15min":34.292999999999999},"last":0.503}]},{"osd":3,"last update":"Sat Sep 23 20:40:36 2023","interfaces":[{"interface":"back","average":{"1min":2.0099999999999998,"5min":1.8759999999999999,"15min":1.9910000000000001},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":5.9130000000000003,"5min":8.9670000000000005,"15min":9.4239999999999995},"last":0.41599999999999998},{"interface":"front","average":{"1min":1.921,"5min":1.968,"15min":2.1659999999999999},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.317},"max":{"1min":5.8380000000000001,"5min":11.913,"15min":12.875999999999999},"last":0.46800000000000003}]},{"osd":4,"last update":"Sat Sep 23 20:40:11 2023","interfaces":[{"interface":"back","average":{"1min":0.90200000000000002,"5min":0.84799999999999998,"15min":0.95399999999999996},"min":{"1min":0.30099999999999999,"5min":0.26900000000000002,"15min":0.251},"max":{"1min":2.859,"5min":23.904,"15min":23.904},"last":0.34399999999999997},{"interface":"front","average":{"1min":1.498,"5min":1.1879999999999999,"15min":1.204},"min":{"1min":0.36799999999999999,"5min":0.29199999999999998,"15min":0.27000000000000002},"max":{"1min":6.7359999999999998,"5min":11.515000000000001,"15min":11.515000000000001},"last":0.35999999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:40:11 2023","interfaces":[{"interface":"back","average":{"1min":1.2210000000000001,"5min":0.85299999999999998,"15min":0.96199999999999997},"min":{"1min":0.375,"5min":0.318,"15min":0.28699999999999998},"max":{"1min":6.6340000000000003,"5min":6.6340000000000003,"15min":14.085000000000001},"last":0.44},{"interface":"front","average":{"1min":1.2529999999999999,"5min":0.97899999999999998,"15min":1.355},"min":{"1min":0.35399999999999998,"5min":0.32500000000000001,"15min":0.30599999999999999},"max":{"1min":6.6909999999999998,"5min":6.7320000000000002,"15min":18.192},"last":0.308}]},{"osd":7,"last update":"Sat Sep 23 20:40:36 2023","interfaces":[{"interface":"back","average":{"1min":1.1240000000000001,"5min":0.88300000000000001,"15min":1.121},"min":{"1min":0.40400000000000003,"5min":0.376,"15min":0.28599999999999998},"max":{"1min":5.2729999999999997,"5min":5.2729999999999997,"15min":10.217000000000001},"last":0.46200000000000002},{"interface":"front","average":{"1min":0.97899999999999998,"5min":1.048,"15min":1.3120000000000001},"min":{"1min":0.45700000000000002,"5min":0.36799999999999999,"15min":0.30299999999999999},"max":{"1min":2.2909999999999999,"5min":8.8119999999999994,"15min":9.7360000000000007},"last":0.45000000000000001}]}]},{"osd":1,"up_from":266,"seq":1142461300868,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":474640,"kb_used_data":34312,"kb_used_omap":8,"kb_used_meta":440311,"kb_avail":93270512,"statfs":{"total":95995035648,"available":95509004288,"internally_reserved":0,"allocated":35135488,"data_stored":32707356,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8206,"internal_metadata":450879474},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:40:05 2023","interfaces":[{"interface":"back","average":{"1min":1.3680000000000001,"5min":2.2250000000000001,"15min":2.5099999999999998},"min":{"1min":0.218,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":8.1150000000000002,"5min":14.531000000000001,"15min":52.689999999999998},"last":0.52800000000000002},{"interface":"front","average":{"1min":2.4140000000000001,"5min":3.1200000000000001,"15min":3.2650000000000001},"min":{"1min":0.316,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":11.994,"5min":19.41,"15min":52.662999999999997},"last":2.7010000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:40:05 2023","interfaces":[{"interface":"back","average":{"1min":2.7280000000000002,"5min":3.0489999999999999,"15min":4.0810000000000004},"min":{"1min":0.23400000000000001,"5min":0.23300000000000001,"15min":0.20899999999999999},"max":{"1min":14.148,"5min":32.255000000000003,"15min":32.255000000000003},"last":1.754},{"interface":"front","average":{"1min":1.911,"5min":2.238,"15min":2.3170000000000002},"min":{"1min":0.28599999999999998,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":8.8079999999999998,"5min":13.404999999999999,"15min":13.404999999999999},"last":3.786}]},{"osd":3,"last update":"Sat Sep 23 20:40:05 2023","interfaces":[{"interface":"back","average":{"1min":1.6659999999999999,"5min":2.3809999999999998,"15min":3.5430000000000001},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":8.0079999999999991,"5min":14.359,"15min":80.745999999999995},"last":0.65100000000000002},{"interface":"front","average":{"1min":2.6509999999999998,"5min":2.2810000000000001,"15min":3.9319999999999999},"min":{"1min":0.316,"5min":0.27500000000000002,"15min":0.22700000000000001},"max":{"1min":12.045999999999999,"5min":19.315000000000001,"15min":82.988},"last":2.7400000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:40:05 2023","interfaces":[{"interface":"back","average":{"1min":2.0710000000000002,"5min":2.2490000000000001,"15min":2.2090000000000001},"min":{"1min":0.33400000000000002,"5min":0.315,"15min":0.30299999999999999},"max":{"1min":7.8399999999999999,"5min":9.4510000000000005,"15min":24.327999999999999},"last":0.44600000000000001},{"interface":"front","average":{"1min":2.056,"5min":1.593,"15min":1.764},"min":{"1min":0.33400000000000002,"5min":0.311,"15min":0.311},"max":{"1min":8.1509999999999998,"5min":11.574999999999999,"15min":11.574999999999999},"last":0.59499999999999997}]},{"osd":5,"last update":"Sat Sep 23 20:40:05 2023","interfaces":[{"interface":"back","average":{"1min":1.996,"5min":2.1400000000000001,"15min":2.3730000000000002},"min":{"1min":0.45800000000000002,"5min":0.34100000000000003,"15min":0.33200000000000002},"max":{"1min":5.5529999999999999,"5min":33.401000000000003,"15min":33.401000000000003},"last":6.4080000000000004},{"interface":"front","average":{"1min":2.5790000000000002,"5min":2.7120000000000002,"15min":2.427},"min":{"1min":0.36199999999999999,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":7.9340000000000002,"5min":33.417000000000002,"15min":33.417000000000002},"last":0.41099999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:40:05 2023","interfaces":[{"interface":"back","average":{"1min":2.0019999999999998,"5min":1.9570000000000001,"15min":4.1699999999999999},"min":{"1min":0.42899999999999999,"5min":0.36899999999999999,"15min":0.35699999999999998},"max":{"1min":6.8099999999999996,"5min":9.6999999999999993,"15min":72.682000000000002},"last":2.903},{"interface":"front","average":{"1min":1.9419999999999999,"5min":2.2799999999999998,"15min":4.0460000000000003},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":7.883,"5min":9.4849999999999994,"15min":72.563999999999993},"last":1.786}]},{"osd":7,"last update":"Sat Sep 23 20:40:05 2023","interfaces":[{"interface":"back","average":{"1min":2.1930000000000001,"5min":2.069,"15min":2.5760000000000001},"min":{"1min":0.48399999999999999,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":5.0819999999999999,"5min":18.437000000000001,"15min":18.437000000000001},"last":6.4859999999999998},{"interface":"front","average":{"1min":2.3149999999999999,"5min":2.3959999999999999,"15min":2.46},"min":{"1min":0.45100000000000001,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":7.9089999999999998,"5min":12.021000000000001,"15min":24.524000000000001},"last":0.47899999999999998}]}]},{"osd":0,"up_from":94,"seq":403726926071,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":664556,"kb_used_data":19556,"kb_used_omap":7,"kb_used_meta":644984,"kb_avail":93080596,"statfs":{"total":95995035648,"available":95314530304,"internally_reserved":0,"allocated":20025344,"data_stored":17754581,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":660463633},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:40:11 2023","interfaces":[{"interface":"back","average":{"1min":1.712,"5min":1.8,"15min":3.0680000000000001},"min":{"1min":0.20699999999999999,"5min":0.17199999999999999,"15min":0.17199999999999999},"max":{"1min":9.8879999999999999,"5min":11.554,"15min":42.000999999999998},"last":0.24299999999999999},{"interface":"front","average":{"1min":1.5660000000000001,"5min":1.9710000000000001,"15min":3.3370000000000002},"min":{"1min":0.218,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":10.097,"5min":12.35,"15min":23.036000000000001},"last":0.32600000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:39:47 2023","interfaces":[{"interface":"back","average":{"1min":0.83099999999999996,"5min":2.7610000000000001,"15min":2.907},"min":{"1min":0.23400000000000001,"5min":0.218,"15min":0.185},"max":{"1min":2.9670000000000001,"5min":62.726999999999997,"15min":77.448999999999998},"last":0.34599999999999997},{"interface":"front","average":{"1min":1.5489999999999999,"5min":3.0329999999999999,"15min":2.8010000000000002},"min":{"1min":0.27300000000000002,"5min":0.247,"15min":0.188},"max":{"1min":6.4800000000000004,"5min":62.689,"15min":81.724999999999994},"last":0.30099999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:40:23 2023","interfaces":[{"interface":"back","average":{"1min":2.399,"5min":2.7450000000000001,"15min":3.133},"min":{"1min":0.34699999999999998,"5min":0.24399999999999999,"15min":0.23400000000000001},"max":{"1min":9.0039999999999996,"5min":51.536000000000001,"15min":81.400999999999996},"last":0.27600000000000002},{"interface":"front","average":{"1min":2.8839999999999999,"5min":3.1389999999999998,"15min":3.4590000000000001},"min":{"1min":0.245,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":14.237,"5min":44.811999999999998,"15min":81.137},"last":0.42299999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:39:47 2023","interfaces":[{"interface":"back","average":{"1min":1.333,"5min":2.008,"15min":2.1539999999999999},"min":{"1min":0.31900000000000001,"5min":0.28899999999999998,"15min":0.25900000000000001},"max":{"1min":6.2450000000000001,"5min":11.068,"15min":23.876999999999999},"last":0.51600000000000001},{"interface":"front","average":{"1min":1.0940000000000001,"5min":1.5700000000000001,"15min":1.954},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":6.3310000000000004,"5min":11.458,"15min":23.681999999999999},"last":0.56699999999999995}]},{"osd":5,"last update":"Sat Sep 23 20:39:47 2023","interfaces":[{"interface":"back","average":{"1min":0.97299999999999998,"5min":1.5580000000000001,"15min":2.1600000000000001},"min":{"1min":0.371,"5min":0.35099999999999998,"15min":0.34200000000000003},"max":{"1min":3.29,"5min":11.497999999999999,"15min":24.198},"last":0.60899999999999999},{"interface":"front","average":{"1min":1.4910000000000001,"5min":2.4399999999999999,"15min":2.3010000000000002},"min":{"1min":0.318,"5min":0.318,"15min":0.25600000000000001},"max":{"1min":6.593,"5min":26.228000000000002,"15min":26.228000000000002},"last":0.60099999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:39:47 2023","interfaces":[{"interface":"back","average":{"1min":1.1599999999999999,"5min":1.867,"15min":2.1309999999999998},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.318},"max":{"1min":6.3680000000000003,"5min":11.526999999999999,"15min":24.408000000000001},"last":0.64200000000000002},{"interface":"front","average":{"1min":1.6599999999999999,"5min":2.4129999999999998,"15min":2.2480000000000002},"min":{"1min":0.36799999999999999,"5min":0.35499999999999998,"15min":0.29399999999999998},"max":{"1min":6.5469999999999997,"5min":26.268000000000001,"15min":26.268000000000001},"last":0.63100000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:39:47 2023","interfaces":[{"interface":"back","average":{"1min":1.6539999999999999,"5min":2.4710000000000001,"15min":2.4249999999999998},"min":{"1min":0.39800000000000002,"5min":0.378,"15min":0.30599999999999999},"max":{"1min":6.9160000000000004,"5min":26.177,"15min":26.177},"last":0.54500000000000004},{"interface":"front","average":{"1min":1.327,"5min":2.1629999999999998,"15min":2.5409999999999999},"min":{"1min":0.39200000000000002,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":4.4210000000000003,"5min":11.112,"15min":24.669},"last":0.62}]}]},{"osd":2,"up_from":104,"seq":446676599019,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":704936,"kb_used_data":31264,"kb_used_omap":11,"kb_used_meta":673652,"kb_avail":93040216,"statfs":{"total":95995035648,"available":95273181184,"internally_reserved":0,"allocated":32014336,"data_stored":29745382,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11878,"internal_metadata":689820058},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:40:00 2023","interfaces":[{"interface":"back","average":{"1min":2.1339999999999999,"5min":1.921,"15min":2.7160000000000002},"min":{"1min":0.26400000000000001,"5min":0.20300000000000001,"15min":0.17999999999999999},"max":{"1min":8.9710000000000001,"5min":14.124000000000001,"15min":82.424999999999997},"last":0.28399999999999997},{"interface":"front","average":{"1min":2.2919999999999998,"5min":2.375,"15min":2.5920000000000001},"min":{"1min":0.23000000000000001,"5min":0.19,"15min":0.19},"max":{"1min":9.5600000000000005,"5min":14.055,"15min":82.524000000000001},"last":0.41999999999999998}]},{"osd":1,"last update":"Sat Sep 23 20:40:00 2023","interfaces":[{"interface":"back","average":{"1min":2.472,"5min":2.9500000000000002,"15min":3.6240000000000001},"min":{"1min":0.433,"5min":0.245,"15min":0.221},"max":{"1min":11.279999999999999,"5min":46.405000000000001,"15min":46.405000000000001},"last":0.34799999999999998},{"interface":"front","average":{"1min":1.506,"5min":2.9129999999999998,"15min":2.9009999999999998},"min":{"1min":0.24399999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":8.8699999999999992,"5min":46.459000000000003,"15min":46.459000000000003},"last":0.40699999999999997}]},{"osd":3,"last update":"Sat Sep 23 20:40:22 2023","interfaces":[{"interface":"back","average":{"1min":2.851,"5min":2.613,"15min":2.4769999999999999},"min":{"1min":0.28499999999999998,"5min":0.27100000000000002,"15min":0.19800000000000001},"max":{"1min":11.304,"5min":19.562999999999999,"15min":19.562999999999999},"last":0.39400000000000002},{"interface":"front","average":{"1min":1.7909999999999999,"5min":1.6160000000000001,"15min":2.5089999999999999},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.23899999999999999},"max":{"1min":8.9369999999999994,"5min":13.148999999999999,"15min":30.704999999999998},"last":0.37}]},{"osd":4,"last update":"Sat Sep 23 20:40:00 2023","interfaces":[{"interface":"back","average":{"1min":1.587,"5min":1.8460000000000001,"15min":1.931},"min":{"1min":0.33300000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":6.6849999999999996,"5min":9.2430000000000003,"15min":16.797999999999998},"last":0.55600000000000005},{"interface":"front","average":{"1min":1.4530000000000001,"5min":2.0009999999999999,"15min":2.117},"min":{"1min":0.38,"5min":0.307,"15min":0.27600000000000002},"max":{"1min":7.9400000000000004,"5min":19.440999999999999,"15min":19.440999999999999},"last":0.44900000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:40:00 2023","interfaces":[{"interface":"back","average":{"1min":1.7290000000000001,"5min":1.8680000000000001,"15min":2.4329999999999998},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.309},"max":{"1min":6.6319999999999997,"5min":9.1799999999999997,"15min":95.393000000000001},"last":0.47599999999999998},{"interface":"front","average":{"1min":1.7110000000000001,"5min":2.0230000000000001,"15min":2.694},"min":{"1min":0.42399999999999999,"5min":0.35299999999999998,"15min":0.32300000000000001},"max":{"1min":7.8179999999999996,"5min":19.513000000000002,"15min":95.414000000000001},"last":0.54700000000000004}]},{"osd":6,"last update":"Sat Sep 23 20:40:00 2023","interfaces":[{"interface":"back","average":{"1min":1.6339999999999999,"5min":1.9730000000000001,"15min":2.1909999999999998},"min":{"1min":0.47099999999999997,"5min":0.37,"15min":0.32100000000000001},"max":{"1min":7.8579999999999997,"5min":19.486000000000001,"15min":19.486000000000001},"last":0.56799999999999995},{"interface":"front","average":{"1min":1.512,"5min":1.4550000000000001,"15min":2.0329999999999999},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.316},"max":{"1min":9.3010000000000002,"5min":10.811,"15min":30.670999999999999},"last":0.501}]},{"osd":7,"last update":"Sat Sep 23 20:40:00 2023","interfaces":[{"interface":"back","average":{"1min":1.073,"5min":1.3520000000000001,"15min":2.028},"min":{"1min":0.42499999999999999,"5min":0.39300000000000002,"15min":0.35199999999999998},"max":{"1min":4.4260000000000002,"5min":10.231,"15min":30.741},"last":0.52600000000000002},{"interface":"front","average":{"1min":1.752,"5min":1.9650000000000001,"15min":2.1760000000000002},"min":{"1min":0.44,"5min":0.40999999999999998,"15min":0.38},"max":{"1min":6.7220000000000004,"5min":9.2720000000000002,"15min":16.719000000000001},"last":0.64200000000000002}]}]},{"osd":3,"up_from":109,"seq":468151435493,"num_pgs":63,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":876384,"kb_used_data":79256,"kb_used_omap":4,"kb_used_meta":797115,"kb_avail":92868768,"statfs":{"total":95995035648,"available":95097618432,"internally_reserved":0,"allocated":81158144,"data_stored":78902314,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4514,"internal_metadata":816246366},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:40:26 2023","interfaces":[{"interface":"back","average":{"1min":1.409,"5min":1.7350000000000001,"15min":2.544},"min":{"1min":0.21199999999999999,"5min":0.21199999999999999,"15min":0.186},"max":{"1min":7.6879999999999997,"5min":14.638,"15min":67.305999999999997},"last":0.44500000000000001},{"interface":"front","average":{"1min":2.4540000000000002,"5min":1.865,"15min":2.415},"min":{"1min":0.29499999999999998,"5min":0.26600000000000001,"15min":0.19500000000000001},"max":{"1min":8.8439999999999994,"5min":10.140000000000001,"15min":67.403999999999996},"last":0.37}]},{"osd":1,"last update":"Sat Sep 23 20:39:59 2023","interfaces":[{"interface":"back","average":{"1min":0.81699999999999995,"5min":1.3169999999999999,"15min":2.375},"min":{"1min":0.246,"5min":0.23999999999999999,"15min":0.222},"max":{"1min":7.6219999999999999,"5min":9.8940000000000001,"15min":16.189},"last":0.40799999999999997},{"interface":"front","average":{"1min":2.343,"5min":2.1669999999999998,"15min":2.6829999999999998},"min":{"1min":0.26500000000000001,"5min":0.26200000000000001,"15min":0.24199999999999999},"max":{"1min":9.1590000000000007,"5min":13.727,"15min":32.603999999999999},"last":0.28999999999999998}]},{"osd":2,"last update":"Sat Sep 23 20:40:26 2023","interfaces":[{"interface":"back","average":{"1min":2.3140000000000001,"5min":1.837,"15min":2.5030000000000001},"min":{"1min":0.26600000000000001,"5min":0.246,"15min":0.22800000000000001},"max":{"1min":6.3200000000000003,"5min":10.747999999999999,"15min":32.857999999999997},"last":0.42799999999999999},{"interface":"front","average":{"1min":1.516,"5min":2.0259999999999998,"15min":2.4980000000000002},"min":{"1min":0.31,"5min":0.28100000000000003,"15min":0.20899999999999999},"max":{"1min":8.5530000000000008,"5min":22.32,"15min":24.146999999999998},"last":0.45600000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:40:26 2023","interfaces":[{"interface":"back","average":{"1min":0.82199999999999995,"5min":1.0860000000000001,"15min":1.786},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.29799999999999999},"max":{"1min":2.5899999999999999,"5min":9.3160000000000007,"15min":46.042999999999999},"last":0.65000000000000002},{"interface":"front","average":{"1min":1.6359999999999999,"5min":1.879,"15min":2.4340000000000002},"min":{"1min":0.33200000000000002,"5min":0.33000000000000002,"15min":0.28100000000000003},"max":{"1min":5.141,"5min":13.571999999999999,"15min":46.579999999999998},"last":0.501}]},{"osd":5,"last update":"Sat Sep 23 20:40:26 2023","interfaces":[{"interface":"back","average":{"1min":0.82099999999999995,"5min":1.167,"15min":1.998},"min":{"1min":0.32700000000000001,"5min":0.32300000000000001,"15min":0.31900000000000001},"max":{"1min":2.9889999999999999,"5min":9.4130000000000003,"15min":45.292000000000002},"last":0.68899999999999995},{"interface":"front","average":{"1min":1.6859999999999999,"5min":1.482,"15min":1.968},"min":{"1min":0.38500000000000001,"5min":0.34399999999999997,"15min":0.313},"max":{"1min":5.0060000000000002,"5min":9.0839999999999996,"15min":44.679000000000002},"last":4.0659999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:40:26 2023","interfaces":[{"interface":"back","average":{"1min":1.8640000000000001,"5min":1.3959999999999999,"15min":1.964},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":7.3630000000000004,"5min":9.0470000000000006,"15min":17.524999999999999},"last":0.51800000000000002},{"interface":"front","average":{"1min":1.4750000000000001,"5min":1.8839999999999999,"15min":2.165},"min":{"1min":0.434,"5min":0.39800000000000002,"15min":0.33700000000000002},"max":{"1min":4.8819999999999997,"5min":13.500999999999999,"15min":19.524000000000001},"last":0.81299999999999994}]},{"osd":7,"last update":"Sat Sep 23 20:40:26 2023","interfaces":[{"interface":"back","average":{"1min":1.5109999999999999,"5min":2.1829999999999998,"15min":2.3959999999999999},"min":{"1min":0.433,"5min":0.39900000000000002,"15min":0.34999999999999998},"max":{"1min":4.319,"5min":13.544,"15min":19.571999999999999},"last":0.84599999999999997},{"interface":"front","average":{"1min":0.996,"5min":1.274,"15min":2.0720000000000001},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.371},"max":{"1min":4.157,"5min":9.3719999999999999,"15min":45.387999999999998},"last":0.72699999999999998}]}]},{"osd":4,"up_from":30,"seq":128849019246,"num_pgs":61,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":405560,"kb_used_data":43568,"kb_used_omap":0,"kb_used_meta":361984,"kb_avail":93339592,"statfs":{"total":95995035648,"available":95579742208,"internally_reserved":0,"allocated":44613632,"data_stored":43201179,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":370671616},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:39:48 2023","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":1.6910000000000001,"15min":1.6819999999999999},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.222},"max":{"1min":2.4279999999999999,"5min":24.114999999999998,"15min":24.114999999999998},"last":4.2450000000000001},{"interface":"front","average":{"1min":1.145,"5min":1.829,"15min":1.8580000000000001},"min":{"1min":0.29899999999999999,"5min":0.246,"15min":0.246},"max":{"1min":6.9119999999999999,"5min":24.135999999999999,"15min":24.135999999999999},"last":4.266}]},{"osd":1,"last update":"Sat Sep 23 20:40:06 2023","interfaces":[{"interface":"back","average":{"1min":0.95899999999999996,"5min":1.331,"15min":1.7410000000000001},"min":{"1min":0.309,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":4.1669999999999998,"5min":18.190999999999999,"15min":18.190999999999999},"last":1.6459999999999999},{"interface":"front","average":{"1min":0.88300000000000001,"5min":1.4259999999999999,"15min":1.6899999999999999},"min":{"1min":0.307,"5min":0.28499999999999998,"15min":0.28199999999999997},"max":{"1min":3.7109999999999999,"5min":18.173999999999999,"15min":18.173999999999999},"last":1.659}]},{"osd":2,"last update":"Sat Sep 23 20:40:03 2023","interfaces":[{"interface":"back","average":{"1min":1.3400000000000001,"5min":1.6060000000000001,"15min":1.8740000000000001},"min":{"1min":0.34200000000000003,"5min":0.29999999999999999,"15min":0.26600000000000001},"max":{"1min":3.8010000000000002,"5min":10.416,"15min":32.363},"last":4.4640000000000004},{"interface":"front","average":{"1min":2.2069999999999999,"5min":2.1019999999999999,"15min":2.3109999999999999},"min":{"1min":0.34300000000000003,"5min":0.28899999999999998,"15min":0.27600000000000002},"max":{"1min":11.917999999999999,"5min":11.917999999999999,"15min":34.042999999999999},"last":0.40699999999999997}]},{"osd":3,"last update":"Sat Sep 23 20:40:17 2023","interfaces":[{"interface":"back","average":{"1min":2.4209999999999998,"5min":1.8480000000000001,"15min":2.113},"min":{"1min":0.40500000000000003,"5min":0.372,"15min":0.29099999999999998},"max":{"1min":12.147,"5min":12.147,"15min":32.332999999999998},"last":0.51100000000000001},{"interface":"front","average":{"1min":1.8280000000000001,"5min":1.8360000000000001,"15min":2.206},"min":{"1min":0.371,"5min":0.371,"15min":0.30299999999999999},"max":{"1min":11.914,"5min":11.914,"15min":32.399999999999999},"last":0.55400000000000005}]},{"osd":5,"last update":"Sat Sep 23 20:39:43 2023","interfaces":[{"interface":"back","average":{"1min":0.84499999999999997,"5min":0.69799999999999995,"15min":1.54},"min":{"1min":0.29399999999999998,"5min":0.26400000000000001,"15min":0.247},"max":{"1min":4.1260000000000003,"5min":4.1260000000000003,"15min":82.866},"last":0.34599999999999997},{"interface":"front","average":{"1min":1.3129999999999999,"5min":0.89300000000000002,"15min":1.528},"min":{"1min":0.35699999999999998,"5min":0.28999999999999998,"15min":0.27800000000000002},"max":{"1min":7.0129999999999999,"5min":7.0129999999999999,"15min":82.546000000000006},"last":0.48999999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:40:03 2023","interfaces":[{"interface":"back","average":{"1min":1.722,"5min":1.032,"15min":1.381},"min":{"1min":0.32200000000000001,"5min":0.312,"15min":0.27600000000000002},"max":{"1min":14.731999999999999,"5min":14.731999999999999,"15min":15.252000000000001},"last":1.1559999999999999},{"interface":"front","average":{"1min":1.024,"5min":0.97199999999999998,"15min":1.2450000000000001},"min":{"1min":0.40500000000000003,"5min":0.29899999999999999,"15min":0.27200000000000002},"max":{"1min":4.976,"5min":7.2850000000000001,"15min":11.199},"last":0.38200000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:40:17 2023","interfaces":[{"interface":"back","average":{"1min":1.0389999999999999,"5min":1.143,"15min":1.5369999999999999},"min":{"1min":0.39100000000000001,"5min":0.318,"15min":0.318},"max":{"1min":4.21,"5min":14.534000000000001,"15min":16.427},"last":0.442},{"interface":"front","average":{"1min":1.472,"5min":0.84599999999999997,"15min":1.3180000000000001},"min":{"1min":0.36399999999999999,"5min":0.32700000000000001,"15min":0.307},"max":{"1min":7.9390000000000001,"5min":7.9390000000000001,"15min":19.018999999999998},"last":0.42099999999999999}]}]},{"osd":5,"up_from":39,"seq":167503724907,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1768816,"kb_used_data":1385000,"kb_used_omap":0,"kb_used_meta":383808,"kb_avail":91976336,"statfs":{"total":95995035648,"available":94183768064,"internally_reserved":0,"allocated":1418240000,"data_stored":1416831825,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":393019392},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:39:48 2023","interfaces":[{"interface":"back","average":{"1min":1.4970000000000001,"5min":2.077,"15min":1.8620000000000001},"min":{"1min":0.253,"5min":0.23899999999999999,"15min":0.20699999999999999},"max":{"1min":5.3079999999999998,"5min":18.16,"15min":32.475000000000001},"last":0.46100000000000002},{"interface":"front","average":{"1min":1.3600000000000001,"5min":1.8899999999999999,"15min":1.7589999999999999},"min":{"1min":0.32100000000000001,"5min":0.29499999999999998,"15min":0.245},"max":{"1min":5.2969999999999997,"5min":9.0570000000000004,"15min":32.457000000000001},"last":0.435}]},{"osd":1,"last update":"Sat Sep 23 20:40:07 2023","interfaces":[{"interface":"back","average":{"1min":1.6799999999999999,"5min":2.0070000000000001,"15min":1.9179999999999999},"min":{"1min":0.26800000000000002,"5min":0.26300000000000001,"15min":0.253},"max":{"1min":5.71,"5min":36.588999999999999,"15min":36.588999999999999},"last":0.37},{"interface":"front","average":{"1min":1.2470000000000001,"5min":2.0150000000000001,"15min":1.6970000000000001},"min":{"1min":0.29799999999999999,"5min":0.27700000000000002,"15min":0.23499999999999999},"max":{"1min":5.298,"5min":36.920999999999999,"15min":36.920999999999999},"last":0.41599999999999998}]},{"osd":2,"last update":"Sat Sep 23 20:39:48 2023","interfaces":[{"interface":"back","average":{"1min":1.145,"5min":2.0099999999999998,"15min":2.073},"min":{"1min":0.33200000000000002,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":3.4670000000000001,"5min":37.011000000000003,"15min":37.011000000000003},"last":0.48099999999999998},{"interface":"front","average":{"1min":1.4570000000000001,"5min":1.9239999999999999,"15min":2.0649999999999999},"min":{"1min":0.308,"5min":0.28999999999999998,"15min":0.255},"max":{"1min":4.399,"5min":36.835000000000001,"15min":36.835000000000001},"last":0.34599999999999997}]},{"osd":3,"last update":"Sat Sep 23 20:40:14 2023","interfaces":[{"interface":"back","average":{"1min":2.2320000000000002,"5min":2.153,"15min":2.2360000000000002},"min":{"1min":0.435,"5min":0.34699999999999998,"15min":0.25700000000000001},"max":{"1min":6.0309999999999997,"5min":36.716000000000001,"15min":36.716000000000001},"last":0.40500000000000003},{"interface":"front","average":{"1min":1.5289999999999999,"5min":2.2240000000000002,"15min":2.117},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.28799999999999998},"max":{"1min":8.3599999999999994,"5min":18.504000000000001,"15min":18.504000000000001},"last":0.55500000000000005}]},{"osd":4,"last update":"Sat Sep 23 20:39:43 2023","interfaces":[{"interface":"back","average":{"1min":0.82899999999999996,"5min":0.76800000000000002,"15min":1.329},"min":{"1min":0.28000000000000003,"5min":0.255,"15min":0.255},"max":{"1min":4.4939999999999998,"5min":6.9180000000000001,"15min":28.015999999999998},"last":0.29999999999999999},{"interface":"front","average":{"1min":0.82699999999999996,"5min":0.79200000000000004,"15min":1.1020000000000001},"min":{"1min":0.311,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":1.9059999999999999,"5min":6.8380000000000001,"15min":12.300000000000001},"last":0.502}]},{"osd":6,"last update":"Sat Sep 23 20:39:54 2023","interfaces":[{"interface":"back","average":{"1min":1.329,"5min":1.02,"15min":1.2589999999999999},"min":{"1min":0.32900000000000001,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":4.7679999999999998,"5min":6.9960000000000004,"15min":12.890000000000001},"last":0.44600000000000001},{"interface":"front","average":{"1min":1.1799999999999999,"5min":0.97499999999999998,"15min":1.2030000000000001},"min":{"1min":0.34000000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":4.7199999999999998,"5min":6.7930000000000001,"15min":16.265999999999998},"last":0.52000000000000002}]},{"osd":7,"last update":"Sat Sep 23 20:40:14 2023","interfaces":[{"interface":"back","average":{"1min":1.3939999999999999,"5min":0.84299999999999997,"15min":1.244},"min":{"1min":0.38400000000000001,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":5.2430000000000003,"5min":5.2430000000000003,"15min":36.621000000000002},"last":0.53800000000000003},{"interface":"front","average":{"1min":1.0189999999999999,"5min":0.90700000000000003,"15min":1.151},"min":{"1min":0.35699999999999998,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":4.3250000000000002,"5min":6.5970000000000004,"15min":37.331000000000003},"last":0.39000000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4722,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1360,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8206,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7156,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3154,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":13893632,"data_stored":13893632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":28905472,"data_stored":28901400,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":21757952,"data_stored":21757952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":75038720,"data_stored":75038720,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":39780352,"data_stored":39780352,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1407385600,"data_stored":1407385600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":85131264,"data_stored":85131264,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:40:44.670 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:40:44.696 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:44 smithi149 ceph-mon[170394]: osdmap e447: 8 total, 8 up, 8 in 2023-09-23T20:40:44.696 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:44 smithi149 ceph-mon[170394]: failed to encode map e447 with expected crc 2023-09-23T20:40:44.699 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:44.739 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:40:44.742 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:44.743 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:40:44.762 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T20:40:44.844 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:40:44.876 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:44.911 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:40:44.916 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:40:44.920 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:44.921 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:40:45.021 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:40:45.049 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:45.083 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:40:45.086 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:45.087 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:40:45.141 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:40:44 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:40:44] "GET /metrics HTTP/1.1" 200 36565 "" "Prometheus/2.43.0" 2023-09-23T20:40:45.187 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:40:45.216 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:45.421 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:40:45.430 INFO:teuthology.orchestra.run.smithi018.stderr:nodeep-scrub is set 2023-09-23T20:40:45.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:45 smithi018 ceph-mon[179647]: pgmap v1102: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 5 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 18 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 557 op/s; 14980/729 objects degraded (2054.870%); 46/729 objects misplaced (6.310%); 1.7 MiB/s, 170 objects/s recovering 2023-09-23T20:40:45.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:45 smithi018 ceph-mon[179647]: from='client.36384 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:40:45.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:45 smithi018 ceph-mon[179647]: osdmap e448: 8 total, 8 up, 8 in 2023-09-23T20:40:45.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:45 smithi018 ceph-mon[179647]: failed to encode map e448 with expected crc 2023-09-23T20:40:45.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:45 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/502109288' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:40:45.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:45 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:40:45.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:45 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:40:45.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:45 smithi018 ceph-mon[184144]: pgmap v1102: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 5 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 18 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 557 op/s; 14980/729 objects degraded (2054.870%); 46/729 objects misplaced (6.310%); 1.7 MiB/s, 170 objects/s recovering 2023-09-23T20:40:45.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:45 smithi018 ceph-mon[184144]: from='client.36384 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:40:45.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:45 smithi018 ceph-mon[184144]: osdmap e448: 8 total, 8 up, 8 in 2023-09-23T20:40:45.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:45 smithi018 ceph-mon[184144]: failed to encode map e448 with expected crc 2023-09-23T20:40:45.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:45 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/502109288' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:40:45.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:45 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:40:45.643 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:45 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:40:45.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:45 smithi149 ceph-mon[170394]: pgmap v1102: 129 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+remapped, 5 active+remapped+backfill_wait, 9 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 18 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 557 op/s; 14980/729 objects degraded (2054.870%); 46/729 objects misplaced (6.310%); 1.7 MiB/s, 170 objects/s recovering 2023-09-23T20:40:45.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:45 smithi149 ceph-mon[170394]: from='client.36384 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:40:45.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:45 smithi149 ceph-mon[170394]: osdmap e448: 8 total, 8 up, 8 in 2023-09-23T20:40:45.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:45 smithi149 ceph-mon[170394]: failed to encode map e448 with expected crc 2023-09-23T20:40:45.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:45 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/502109288' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:40:45.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:45 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:40:45.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:45 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:40:45.845 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:40:45.848 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:45.849 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:40:45.950 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:40:45.978 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:46.150 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:40:46.667 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:40:46.670 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:46.671 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:40:46.679 INFO:teuthology.orchestra.run.smithi149.stdout:{ 2023-09-23T20:40:46.679 INFO:teuthology.orchestra.run.smithi149.stdout: "ops": [], 2023-09-23T20:40:46.679 INFO:teuthology.orchestra.run.smithi149.stdout: "complaint_time": 30, 2023-09-23T20:40:46.679 INFO:teuthology.orchestra.run.smithi149.stdout: "num_blocked_ops": 0 2023-09-23T20:40:46.679 INFO:teuthology.orchestra.run.smithi149.stdout:} 2023-09-23T20:40:46.701 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:46 smithi149 ceph-mon[170394]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:40:46.701 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:46 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:40:46.701 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:46 smithi149 ceph-mon[170394]: osdmap e449: 8 total, 8 up, 8 in 2023-09-23T20:40:46.701 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:46 smithi149 ceph-mon[170394]: failed to encode map e449 with expected crc 2023-09-23T20:40:46.701 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:46 smithi149 ceph-mon[170394]: failed to encode map e449 with expected crc 2023-09-23T20:40:46.701 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:46 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 14980/729 objects degraded (2054.870%), 9 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T20:40:46.771 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:40:46.798 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:46 smithi018 ceph-mon[184144]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:40:46.798 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:46 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:40:46.798 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:46 smithi018 ceph-mon[184144]: osdmap e449: 8 total, 8 up, 8 in 2023-09-23T20:40:46.798 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:46 smithi018 ceph-mon[184144]: failed to encode map e449 with expected crc 2023-09-23T20:40:46.798 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:46 smithi018 ceph-mon[184144]: failed to encode map e449 with expected crc 2023-09-23T20:40:46.799 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:46 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 14980/729 objects degraded (2054.870%), 9 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T20:40:46.799 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:46 smithi018 ceph-mon[179647]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:40:46.799 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:46 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:40:46.799 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:46 smithi018 ceph-mon[179647]: osdmap e449: 8 total, 8 up, 8 in 2023-09-23T20:40:46.799 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:46 smithi018 ceph-mon[179647]: failed to encode map e449 with expected crc 2023-09-23T20:40:46.799 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:46 smithi018 ceph-mon[179647]: failed to encode map e449 with expected crc 2023-09-23T20:40:46.799 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:46 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 14980/729 objects degraded (2054.870%), 9 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T20:40:46.801 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:47.092 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T20:40:47.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:47 smithi018 ceph-mon[184144]: failed to encode map e449 with expected crc 2023-09-23T20:40:47.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:47 smithi018 ceph-mon[184144]: failed to encode map e449 with expected crc 2023-09-23T20:40:47.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:47 smithi018 ceph-mon[184144]: failed to encode map e449 with expected crc 2023-09-23T20:40:47.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:47 smithi018 ceph-mon[184144]: pgmap v1105: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 18 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 192 op/s; 12429/729 objects degraded (1704.938%); 46/729 objects misplaced (6.310%); 0 B/s, 182 objects/s recovering 2023-09-23T20:40:47.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:47 smithi018 ceph-mon[179647]: failed to encode map e449 with expected crc 2023-09-23T20:40:47.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:47 smithi018 ceph-mon[179647]: failed to encode map e449 with expected crc 2023-09-23T20:40:47.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:47 smithi018 ceph-mon[179647]: failed to encode map e449 with expected crc 2023-09-23T20:40:47.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:47 smithi018 ceph-mon[179647]: pgmap v1105: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 18 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 192 op/s; 12429/729 objects degraded (1704.938%); 46/729 objects misplaced (6.310%); 0 B/s, 182 objects/s recovering 2023-09-23T20:40:47.658 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:40:47.693 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:40:47.696 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:47.697 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:40:47.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:47 smithi149 ceph-mon[170394]: failed to encode map e449 with expected crc 2023-09-23T20:40:47.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:47 smithi149 ceph-mon[170394]: failed to encode map e449 with expected crc 2023-09-23T20:40:47.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:47 smithi149 ceph-mon[170394]: failed to encode map e449 with expected crc 2023-09-23T20:40:47.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:47 smithi149 ceph-mon[170394]: pgmap v1105: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 18 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 192 op/s; 12429/729 objects degraded (1704.938%); 46/729 objects misplaced (6.310%); 0 B/s, 182 objects/s recovering 2023-09-23T20:40:47.724 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:40:47.797 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:40:47.830 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:48.962 INFO:teuthology.orchestra.run.smithi149.stdout:{ 2023-09-23T20:40:48.963 INFO:teuthology.orchestra.run.smithi149.stdout: "size": 20, 2023-09-23T20:40:48.963 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 600, 2023-09-23T20:40:48.963 INFO:teuthology.orchestra.run.smithi149.stdout: "ops": [ 2023-09-23T20:40:48.963 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:48.963 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35700.0:57591 7.1 7.7a2f2aa1 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e312)", 2023-09-23T20:40:48.963 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:32:32.227723+0000", 2023-09-23T20:40:48.963 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 496.73162041500001, 2023-09-23T20:40:48.963 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.99202805599999999, 2023-09-23T20:40:48.963 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:40:48.964 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:40:48.964 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:40:48.964 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35700", 2023-09-23T20:40:48.964 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:40:48.964 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 57591 2023-09-23T20:40:48.964 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:48.964 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:40:48.964 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:48.964 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:40:48.964 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:32:32.227723+0000", 2023-09-23T20:40:48.964 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:40:48.964 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:48.965 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:48.965 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:40:48.965 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:32:32.227723+0000", 2023-09-23T20:40:48.965 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:40:48.965 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:48.965 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:48.965 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:40:48.965 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:32:32.227722+0000", 2023-09-23T20:40:48.965 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:40:48.965 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:48.965 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:48.965 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:40:48.966 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:32:32.228185+0000", 2023-09-23T20:40:48.966 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00046322699999999997 2023-09-23T20:40:48.966 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:48.966 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:48.966 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:40:48.966 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:32:32.228205+0000", 2023-09-23T20:40:48.966 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.9927000000000001e-05 2023-09-23T20:40:48.966 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:48.966 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:48.966 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:40:48.966 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:32:32.228213+0000", 2023-09-23T20:40:48.966 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.6359999999999994e-06 2023-09-23T20:40:48.967 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:48.967 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:48.967 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:40:48.967 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:32:32.228446+0000", 2023-09-23T20:40:48.967 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00023344300000000001 2023-09-23T20:40:48.967 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:48.967 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:48.967 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:40:48.967 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:32:32.228454+0000", 2023-09-23T20:40:48.967 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.2029999999999992e-06 2023-09-23T20:40:48.967 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:48.967 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:48.968 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:40:48.968 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:32:33.219740+0000", 2023-09-23T20:40:48.968 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.99128563800000002 2023-09-23T20:40:48.968 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:48.968 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:48.968 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:40:48.968 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:32:33.219751+0000", 2023-09-23T20:40:48.968 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1409e-05 2023-09-23T20:40:48.968 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:40:48.968 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:40:48.968 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:40:48.968 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:48.969 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:48.969 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35700.0:57595 7.1 7.19a18ad1 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e312)", 2023-09-23T20:40:48.969 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:32:32.228233+0000", 2023-09-23T20:40:48.969 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 496.73111062700002, 2023-09-23T20:40:48.969 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.99154681200000006, 2023-09-23T20:40:48.969 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:40:48.969 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:40:48.969 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:40:48.969 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35700", 2023-09-23T20:40:48.969 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:40:48.969 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 57595 2023-09-23T20:40:48.969 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:48.970 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:40:48.970 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:48.970 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:40:48.970 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:32:32.228233+0000", 2023-09-23T20:40:48.970 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:40:48.970 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:48.970 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:48.970 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:40:48.970 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:32:32.228233+0000", 2023-09-23T20:40:48.970 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:40:48.970 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:48.971 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:48.971 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:40:48.971 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:32:32.228233+0000", 2023-09-23T20:40:48.971 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:40:48.971 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:48.971 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:48.971 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:40:48.971 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:32:32.228272+0000", 2023-09-23T20:40:48.971 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.9016e-05 2023-09-23T20:40:48.971 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:48.971 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:48.971 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:40:48.972 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:32:32.228283+0000", 2023-09-23T20:40:48.972 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1591e-05 2023-09-23T20:40:48.972 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:48.972 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:48.972 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:40:48.972 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:32:32.228289+0000", 2023-09-23T20:40:48.972 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.9989999999999997e-06 2023-09-23T20:40:48.972 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:48.972 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:48.972 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:40:48.972 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:32:32.228476+0000", 2023-09-23T20:40:48.972 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00018691899999999999 2023-09-23T20:40:48.972 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:48.973 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:48.973 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:40:48.973 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:32:32.228481+0000", 2023-09-23T20:40:48.973 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.6469999999999997e-06 2023-09-23T20:40:48.973 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:48.973 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:48.973 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:40:48.973 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:32:33.219773+0000", 2023-09-23T20:40:48.973 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.99129225700000001 2023-09-23T20:40:48.973 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:48.973 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:48.974 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:40:48.974 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:32:33.219780+0000", 2023-09-23T20:40:48.974 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.9800000000000001e-06 2023-09-23T20:40:48.974 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:40:48.974 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:40:48.974 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:40:48.974 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:48.974 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:48.974 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35700.0:57641 7.1 7.be520741 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e312)", 2023-09-23T20:40:48.974 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:32:32.269916+0000", 2023-09-23T20:40:48.974 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 496.68942801999998, 2023-09-23T20:40:48.974 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.94988938700000003, 2023-09-23T20:40:48.975 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:40:48.975 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:40:48.975 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:40:48.975 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35700", 2023-09-23T20:40:48.975 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:40:48.975 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 57641 2023-09-23T20:40:48.975 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:48.975 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:40:48.975 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:48.975 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:40:48.975 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:32:32.269916+0000", 2023-09-23T20:40:48.975 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:40:48.976 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:48.976 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:48.976 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:40:48.976 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:32:32.269916+0000", 2023-09-23T20:40:48.976 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:40:48.976 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:48.976 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:48.976 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:40:48.976 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:32:32.269913+0000", 2023-09-23T20:40:48.976 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999976 2023-09-23T20:40:48.976 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:48.976 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:48.977 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:40:48.977 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:32:32.269961+0000", 2023-09-23T20:40:48.977 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.7499e-05 2023-09-23T20:40:48.977 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:48.977 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:48.977 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:40:48.977 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:32:32.269987+0000", 2023-09-23T20:40:48.977 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.5522999999999999e-05 2023-09-23T20:40:48.977 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:48.977 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:48.977 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:40:48.977 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:32:32.269996+0000", 2023-09-23T20:40:48.978 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.4879999999999996e-06 2023-09-23T20:40:48.978 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:48.978 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:48.978 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:40:48.978 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:32:32.270066+0000", 2023-09-23T20:40:48.978 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.9770000000000005e-05 2023-09-23T20:40:48.978 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:48.978 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:48.978 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:40:48.978 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:32:32.270074+0000", 2023-09-23T20:40:48.978 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.8550000000000002e-06 2023-09-23T20:40:48.978 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:48.979 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:48.979 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:40:48.979 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:32:33.219798+0000", 2023-09-23T20:40:48.979 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.94972451999999996 2023-09-23T20:40:48.979 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:48.979 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:48.979 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:40:48.979 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:32:33.219805+0000", 2023-09-23T20:40:48.979 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.9630000000000001e-06 2023-09-23T20:40:48.979 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:40:48.980 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:40:48.980 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:40:48.980 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:48.980 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:48.980 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35700.0:57647 7.1 7.b3786c71 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e312)", 2023-09-23T20:40:48.980 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:32:32.275658+0000", 2023-09-23T20:40:48.980 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 496.68368588300001, 2023-09-23T20:40:48.980 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.94417131399999998, 2023-09-23T20:40:48.980 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:40:48.980 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:40:48.980 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:40:48.980 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35700", 2023-09-23T20:40:48.981 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:40:48.981 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 57647 2023-09-23T20:40:48.981 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:48.981 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:40:48.981 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:48.981 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:40:48.981 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:32:32.275658+0000", 2023-09-23T20:40:48.981 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:40:48.981 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:48.981 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:48.981 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:40:48.982 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:32:32.275658+0000", 2023-09-23T20:40:48.982 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:40:48.982 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:48.982 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:48.982 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:40:48.982 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:32:32.275655+0000", 2023-09-23T20:40:48.982 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999976 2023-09-23T20:40:48.982 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:48.982 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:48.982 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:40:48.982 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:32:32.275743+0000", 2023-09-23T20:40:48.983 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.7860000000000002e-05 2023-09-23T20:40:48.983 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:48.983 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:48.983 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:40:48.983 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:32:32.275769+0000", 2023-09-23T20:40:48.983 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.5711000000000002e-05 2023-09-23T20:40:48.983 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:48.983 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:48.983 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:40:48.983 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:32:32.275779+0000", 2023-09-23T20:40:48.983 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0327e-05 2023-09-23T20:40:48.983 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:48.984 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:48.984 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:40:48.984 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:32:32.275851+0000", 2023-09-23T20:40:48.984 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.1360999999999997e-05 2023-09-23T20:40:48.984 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:48.984 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:48.984 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:40:48.984 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:32:32.275862+0000", 2023-09-23T20:40:48.984 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1173e-05 2023-09-23T20:40:48.984 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:48.984 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:48.984 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:40:48.985 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:32:33.219822+0000", 2023-09-23T20:40:48.985 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.94396049000000004 2023-09-23T20:40:48.985 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:48.985 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:48.985 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:40:48.985 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:32:33.219829+0000", 2023-09-23T20:40:48.985 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.8399999999999997e-06 2023-09-23T20:40:48.985 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:40:48.985 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:40:48.985 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:40:48.985 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:48.985 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:48.986 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35700.0:57652 7.1 7.95417c01 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e312)", 2023-09-23T20:40:48.986 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:32:32.281114+0000", 2023-09-23T20:40:48.986 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 496.67823012899999, 2023-09-23T20:40:48.986 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.93874086199999995, 2023-09-23T20:40:48.986 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:40:48.986 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:40:48.986 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:40:48.986 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35700", 2023-09-23T20:40:48.986 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:40:48.986 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 57652 2023-09-23T20:40:48.986 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:48.986 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:40:48.987 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:48.987 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:40:48.987 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:32:32.281114+0000", 2023-09-23T20:40:48.987 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:40:48.987 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:48.987 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:48.987 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:40:48.987 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:32:32.281114+0000", 2023-09-23T20:40:48.987 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:40:48.987 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:48.987 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:48.987 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:40:48.988 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:32:32.281112+0000", 2023-09-23T20:40:48.988 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999981 2023-09-23T20:40:48.988 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:48.988 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:48.988 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:40:48.988 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:32:32.281163+0000", 2023-09-23T20:40:48.988 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.1251000000000001e-05 2023-09-23T20:40:48.988 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:48.988 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:48.988 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:40:48.988 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:32:32.281186+0000", 2023-09-23T20:40:48.988 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.2416999999999999e-05 2023-09-23T20:40:48.989 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:48.989 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:48.989 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:40:48.989 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:32:32.281195+0000", 2023-09-23T20:40:48.989 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.6180000000000001e-06 2023-09-23T20:40:48.989 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:48.989 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:48.989 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:40:48.989 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:32:32.281254+0000", 2023-09-23T20:40:48.989 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.8884000000000001e-05 2023-09-23T20:40:48.989 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:48.989 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:48.990 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:40:48.990 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:32:32.281265+0000", 2023-09-23T20:40:48.990 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.111e-05 2023-09-23T20:40:48.990 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:48.990 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:48.990 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:40:48.990 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:32:33.219848+0000", 2023-09-23T20:40:48.990 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.93858254200000002 2023-09-23T20:40:48.990 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:48.990 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:48.990 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:40:48.991 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:32:33.219854+0000", 2023-09-23T20:40:48.991 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.8190000000000003e-06 2023-09-23T20:40:48.991 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:40:48.991 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:40:48.991 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:40:48.991 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:48.991 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:48.991 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35700.0:156899 7.1 7.f34c1291 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e353)", 2023-09-23T20:40:48.991 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:34:59.025405+0000", 2023-09-23T20:40:48.991 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 349.933938797, 2023-09-23T20:40:48.991 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.95359936199999995, 2023-09-23T20:40:48.992 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:40:48.992 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:40:48.992 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:40:48.992 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35700", 2023-09-23T20:40:48.992 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:40:48.992 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 156899 2023-09-23T20:40:48.992 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:48.992 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:40:48.992 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:48.992 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:40:48.992 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:59.025405+0000", 2023-09-23T20:40:48.992 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:40:48.993 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:48.993 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:48.993 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:40:48.993 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:59.025405+0000", 2023-09-23T20:40:48.993 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:40:48.993 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:48.993 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:48.993 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:40:48.993 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:59.025404+0000", 2023-09-23T20:40:48.993 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:40:48.993 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:48.993 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:48.993 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:40:48.994 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:59.025410+0000", 2023-09-23T20:40:48.994 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.2659999999999998e-06 2023-09-23T20:40:48.994 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:48.994 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:48.994 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:40:48.994 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:59.025427+0000", 2023-09-23T20:40:48.994 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.6750000000000001e-05 2023-09-23T20:40:48.994 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:48.994 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:48.994 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:40:48.994 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:59.025434+0000", 2023-09-23T20:40:48.995 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.0430000000000002e-06 2023-09-23T20:40:48.995 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:48.995 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:48.995 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:40:48.995 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:59.025493+0000", 2023-09-23T20:40:48.995 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.9514e-05 2023-09-23T20:40:48.995 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:48.995 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:48.995 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:40:48.995 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:59.025503+0000", 2023-09-23T20:40:48.995 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0042e-05 2023-09-23T20:40:48.996 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:48.996 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:48.996 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:40:48.996 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:59.978991+0000", 2023-09-23T20:40:48.996 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.953488055 2023-09-23T20:40:48.996 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:48.996 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:48.996 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:40:48.996 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:59.979004+0000", 2023-09-23T20:40:48.996 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.307e-05 2023-09-23T20:40:48.996 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:40:48.997 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:40:48.997 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:40:48.997 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:48.997 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:48.997 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35700.0:156906 7.1 7.7e9886e1 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e353)", 2023-09-23T20:40:48.997 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:34:59.029901+0000", 2023-09-23T20:40:48.997 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 349.92944292300001, 2023-09-23T20:40:48.997 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.94915789500000003, 2023-09-23T20:40:48.997 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:40:48.997 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:40:48.997 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:40:48.998 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35700", 2023-09-23T20:40:48.998 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:40:48.998 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 156906 2023-09-23T20:40:48.998 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:48.998 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:40:48.998 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:48.998 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:40:48.998 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:59.029901+0000", 2023-09-23T20:40:48.998 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:40:48.998 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:48.998 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:48.998 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:40:48.999 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:59.029901+0000", 2023-09-23T20:40:48.999 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:40:48.999 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:48.999 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:48.999 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:40:48.999 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:59.029900+0000", 2023-09-23T20:40:48.999 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:40:48.999 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:48.999 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:48.999 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:40:48.999 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:59.029909+0000", 2023-09-23T20:40:49.000 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.7830000000000004e-06 2023-09-23T20:40:49.000 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.000 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.000 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:40:49.000 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:59.029919+0000", 2023-09-23T20:40:49.000 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0606e-05 2023-09-23T20:40:49.000 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.000 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.000 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:40:49.000 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:59.029923+0000", 2023-09-23T20:40:49.000 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.1899999999999997e-06 2023-09-23T20:40:49.000 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.001 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.001 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:40:49.001 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:59.029966+0000", 2023-09-23T20:40:49.001 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.3061e-05 2023-09-23T20:40:49.001 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.001 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.001 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:40:49.001 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:59.029979+0000", 2023-09-23T20:40:49.001 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.273e-05 2023-09-23T20:40:49.001 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.001 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.001 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:40:49.002 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:59.979028+0000", 2023-09-23T20:40:49.002 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.94904833600000005 2023-09-23T20:40:49.002 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.002 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.002 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:40:49.002 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:59.979059+0000", 2023-09-23T20:40:49.002 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.1163e-05 2023-09-23T20:40:49.002 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:40:49.002 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:40:49.002 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:40:49.002 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.003 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.003 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35700.0:156911 7.1 7.493fcdd1 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e353)", 2023-09-23T20:40:49.003 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:34:59.038682+0000", 2023-09-23T20:40:49.003 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 349.92066158400002, 2023-09-23T20:40:49.003 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.94040694800000002, 2023-09-23T20:40:49.003 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:40:49.003 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:40:49.003 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:40:49.003 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35700", 2023-09-23T20:40:49.003 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:40:49.004 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 156911 2023-09-23T20:40:49.004 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.004 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:40:49.004 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.004 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:40:49.004 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:59.038682+0000", 2023-09-23T20:40:49.004 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:40:49.004 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.004 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.004 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:40:49.004 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:59.038682+0000", 2023-09-23T20:40:49.004 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:40:49.005 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.005 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.005 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:40:49.005 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:59.038681+0000", 2023-09-23T20:40:49.005 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:40:49.005 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.005 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.005 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:40:49.005 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:59.038687+0000", 2023-09-23T20:40:49.005 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.3629999999999999e-06 2023-09-23T20:40:49.005 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.005 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.006 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:40:49.006 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:59.038702+0000", 2023-09-23T20:40:49.006 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.4983999999999999e-05 2023-09-23T20:40:49.006 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.006 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.006 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:40:49.006 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:59.038708+0000", 2023-09-23T20:40:49.006 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.9710000000000001e-06 2023-09-23T20:40:49.006 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.006 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.006 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:40:49.007 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:59.038768+0000", 2023-09-23T20:40:49.007 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.0174999999999999e-05 2023-09-23T20:40:49.007 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.007 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.007 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:40:49.007 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:59.038780+0000", 2023-09-23T20:40:49.007 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1742e-05 2023-09-23T20:40:49.007 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.007 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.007 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:40:49.007 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:59.979080+0000", 2023-09-23T20:40:49.007 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.94029980999999996 2023-09-23T20:40:49.008 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.008 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.008 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:40:49.008 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:59.979089+0000", 2023-09-23T20:40:49.008 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.0710000000000001e-06 2023-09-23T20:40:49.008 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:40:49.008 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:40:49.008 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:40:49.008 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.008 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.008 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35700.0:156944 7.1 7.73c1b121 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e353)", 2023-09-23T20:40:49.008 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:34:59.099576+0000", 2023-09-23T20:40:49.009 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 349.85976781800002, 2023-09-23T20:40:49.009 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.87954308699999995, 2023-09-23T20:40:49.009 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:40:49.009 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:40:49.009 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:40:49.009 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35700", 2023-09-23T20:40:49.009 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:40:49.009 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 156944 2023-09-23T20:40:49.009 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.009 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:40:49.009 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.010 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:40:49.010 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:59.099576+0000", 2023-09-23T20:40:49.010 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:40:49.010 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.010 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.010 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:40:49.010 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:59.099576+0000", 2023-09-23T20:40:49.010 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:40:49.010 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.010 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.010 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:40:49.011 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:59.099575+0000", 2023-09-23T20:40:49.011 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:40:49.011 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.011 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.011 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:40:49.011 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:59.099581+0000", 2023-09-23T20:40:49.011 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.7270000000000001e-06 2023-09-23T20:40:49.011 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.011 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.011 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:40:49.011 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:59.099594+0000", 2023-09-23T20:40:49.012 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.2323e-05 2023-09-23T20:40:49.012 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.012 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.012 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:40:49.012 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:59.099601+0000", 2023-09-23T20:40:49.012 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.1550000000000001e-06 2023-09-23T20:40:49.012 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.012 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.012 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:40:49.012 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:59.099672+0000", 2023-09-23T20:40:49.013 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.1493000000000001e-05 2023-09-23T20:40:49.013 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.013 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.013 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:40:49.013 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:59.099685+0000", 2023-09-23T20:40:49.013 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.2201999999999999e-05 2023-09-23T20:40:49.013 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.013 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.013 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:40:49.013 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:59.979110+0000", 2023-09-23T20:40:49.013 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.87942509400000002 2023-09-23T20:40:49.013 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.014 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.014 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:40:49.014 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:59.979119+0000", 2023-09-23T20:40:49.014 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.3719999999999994e-06 2023-09-23T20:40:49.014 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:40:49.014 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:40:49.014 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:40:49.014 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.014 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.014 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:105803 8.12 8:4d1c432a:::benchmark_data_smithi018_341490_object16480:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e410)", 2023-09-23T20:40:49.014 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:38:26.699836+0000", 2023-09-23T20:40:49.015 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 142.259508216, 2023-09-23T20:40:49.015 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.85041256099999996, 2023-09-23T20:40:49.015 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:40:49.015 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:40:49.015 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:40:49.015 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:40:49.015 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:40:49.015 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 105803 2023-09-23T20:40:49.015 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.015 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:40:49.015 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.016 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:40:49.016 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:26.699836+0000", 2023-09-23T20:40:49.016 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:40:49.016 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.016 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.016 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:40:49.016 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:26.699836+0000", 2023-09-23T20:40:49.016 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:40:49.016 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.016 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.016 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:40:49.017 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:26.699832+0000", 2023-09-23T20:40:49.017 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999967 2023-09-23T20:40:49.017 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.017 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.017 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:40:49.017 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:26.699849+0000", 2023-09-23T20:40:49.017 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.7223000000000001e-05 2023-09-23T20:40:49.017 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.017 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.017 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:40:49.017 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:26.699870+0000", 2023-09-23T20:40:49.018 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.1005e-05 2023-09-23T20:40:49.018 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.018 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.018 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:40:49.018 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:26.699884+0000", 2023-09-23T20:40:49.018 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.3349e-05 2023-09-23T20:40:49.018 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.018 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.018 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:40:49.018 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:26.703306+0000", 2023-09-23T20:40:49.018 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0034221799999999999 2023-09-23T20:40:49.019 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.019 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.019 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:40:49.019 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:26.703327+0000", 2023-09-23T20:40:49.019 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.0890000000000002e-05 2023-09-23T20:40:49.019 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.019 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.019 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:40:49.019 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.504227+0000", 2023-09-23T20:40:49.019 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.800900163 2023-09-23T20:40:49.019 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.019 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.020 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for degraded object", 2023-09-23T20:40:49.020 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.507106+0000", 2023-09-23T20:40:49.020 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0028788500000000001 2023-09-23T20:40:49.020 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.020 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.020 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:40:49.020 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.543770+0000", 2023-09-23T20:40:49.020 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.036664344000000001 2023-09-23T20:40:49.020 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.020 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.021 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:40:49.021 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.543848+0000", 2023-09-23T20:40:49.021 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.8217e-05 2023-09-23T20:40:49.021 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.021 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.021 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:40:49.021 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.543938+0000", 2023-09-23T20:40:49.021 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.9777000000000005e-05 2023-09-23T20:40:49.021 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.021 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.021 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:40:49.021 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.545834+0000", 2023-09-23T20:40:49.022 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0018956579999999999 2023-09-23T20:40:49.022 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.022 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.022 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:40:49.022 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.546152+0000", 2023-09-23T20:40:49.022 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000318311 2023-09-23T20:40:49.022 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.022 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.022 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:40:49.022 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.549292+0000", 2023-09-23T20:40:49.022 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0031395870000000001 2023-09-23T20:40:49.023 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.023 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.023 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:40:49.023 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.549336+0000", 2023-09-23T20:40:49.023 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.3982e-05 2023-09-23T20:40:49.023 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.023 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.023 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:40:49.023 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.550248+0000", 2023-09-23T20:40:49.023 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00091245400000000004 2023-09-23T20:40:49.023 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:40:49.023 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:40:49.024 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:40:49.024 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.024 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.024 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:106049 8.14 8:2acb269a:::benchmark_data_smithi018_341490_object16726:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e416)", 2023-09-23T20:40:49.024 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:38:37.916629+0000", 2023-09-23T20:40:49.024 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 131.042715001, 2023-09-23T20:40:49.024 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.3484781940000001, 2023-09-23T20:40:49.024 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:40:49.024 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:40:49.024 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:40:49.024 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:40:49.024 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:40:49.025 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 106049 2023-09-23T20:40:49.025 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.025 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:40:49.025 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.025 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:40:49.025 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.916629+0000", 2023-09-23T20:40:49.025 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:40:49.025 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.025 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.025 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:40:49.025 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.916629+0000", 2023-09-23T20:40:49.026 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:40:49.026 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.026 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.026 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:40:49.026 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.916627+0000", 2023-09-23T20:40:49.026 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999981 2023-09-23T20:40:49.026 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.026 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.026 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:40:49.026 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.916636+0000", 2023-09-23T20:40:49.026 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.0790000000000007e-06 2023-09-23T20:40:49.027 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.027 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.027 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:40:49.027 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.916657+0000", 2023-09-23T20:40:49.027 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.0982e-05 2023-09-23T20:40:49.027 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.027 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.027 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:40:49.027 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.916665+0000", 2023-09-23T20:40:49.027 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.3289999999999994e-06 2023-09-23T20:40:49.027 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.028 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.028 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:40:49.028 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.916720+0000", 2023-09-23T20:40:49.028 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.5334999999999998e-05 2023-09-23T20:40:49.028 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.028 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.028 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:40:49.028 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.916731+0000", 2023-09-23T20:40:49.028 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0329000000000001e-05 2023-09-23T20:40:49.028 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.028 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.028 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:40:49.029 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.258803+0000", 2023-09-23T20:40:49.029 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.342072444 2023-09-23T20:40:49.029 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.029 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.029 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:40:49.029 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.258947+0000", 2023-09-23T20:40:49.029 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00014386600000000001 2023-09-23T20:40:49.029 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.029 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.029 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 2,5", 2023-09-23T20:40:49.029 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.259044+0000", 2023-09-23T20:40:49.030 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.7132000000000004e-05 2023-09-23T20:40:49.030 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.030 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.030 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:40:49.030 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.261960+0000", 2023-09-23T20:40:49.030 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.002915866 2023-09-23T20:40:49.030 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.030 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.030 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:40:49.030 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.261969+0000", 2023-09-23T20:40:49.030 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.2299999999999997e-06 2023-09-23T20:40:49.030 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.031 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.031 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:40:49.031 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.264756+0000", 2023-09-23T20:40:49.031 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0027866610000000002 2023-09-23T20:40:49.031 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.031 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.031 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:40:49.031 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.264798+0000", 2023-09-23T20:40:49.031 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.2058999999999998e-05 2023-09-23T20:40:49.031 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.031 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.031 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:40:49.032 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.265107+0000", 2023-09-23T20:40:49.032 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000308943 2023-09-23T20:40:49.032 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:40:49.032 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:40:49.032 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:40:49.032 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.032 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.032 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:106054 8.12 8:4dc1e34a:::benchmark_data_smithi018_341490_object16731:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e416)", 2023-09-23T20:40:49.032 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:38:37.939136+0000", 2023-09-23T20:40:49.032 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 131.02020752300001, 2023-09-23T20:40:49.032 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.3347287539999999, 2023-09-23T20:40:49.033 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:40:49.033 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:40:49.033 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:40:49.033 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:40:49.033 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:40:49.033 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 106054 2023-09-23T20:40:49.033 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.033 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:40:49.033 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.033 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:40:49.033 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.939136+0000", 2023-09-23T20:40:49.033 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:40:49.034 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.034 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.034 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:40:49.034 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.939136+0000", 2023-09-23T20:40:49.034 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:40:49.034 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.034 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.034 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:40:49.034 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.939135+0000", 2023-09-23T20:40:49.034 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:40:49.034 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.034 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.035 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:40:49.035 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.939141+0000", 2023-09-23T20:40:49.035 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.0850000000000002e-06 2023-09-23T20:40:49.035 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.035 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.035 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:40:49.035 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.939155+0000", 2023-09-23T20:40:49.035 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.3912e-05 2023-09-23T20:40:49.035 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.035 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.036 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:40:49.036 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.939160+0000", 2023-09-23T20:40:49.036 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.6289999999999998e-06 2023-09-23T20:40:49.036 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.036 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.036 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:40:49.036 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.939210+0000", 2023-09-23T20:40:49.036 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.9122999999999997e-05 2023-09-23T20:40:49.036 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.036 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.036 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:40:49.037 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.939218+0000", 2023-09-23T20:40:49.037 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.2309999999999996e-06 2023-09-23T20:40:49.037 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.037 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.037 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:40:49.037 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.260728+0000", 2023-09-23T20:40:49.037 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.321510314 2023-09-23T20:40:49.037 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.037 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.037 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:40:49.037 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.260841+0000", 2023-09-23T20:40:49.038 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000112938 2023-09-23T20:40:49.038 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.038 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.038 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:40:49.038 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.260955+0000", 2023-09-23T20:40:49.038 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000113584 2023-09-23T20:40:49.038 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.038 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.038 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:40:49.038 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.266252+0000", 2023-09-23T20:40:49.038 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0052969810000000001 2023-09-23T20:40:49.039 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.039 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.039 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:40:49.039 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.273531+0000", 2023-09-23T20:40:49.039 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0072792580000000003 2023-09-23T20:40:49.039 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.039 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.039 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:40:49.039 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.273639+0000", 2023-09-23T20:40:49.039 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000107676 2023-09-23T20:40:49.039 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.039 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.039 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:40:49.040 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.273672+0000", 2023-09-23T20:40:49.040 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.3679000000000001e-05 2023-09-23T20:40:49.040 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.040 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.040 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:40:49.040 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.273865+0000", 2023-09-23T20:40:49.040 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00019271100000000001 2023-09-23T20:40:49.040 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:40:49.040 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:40:49.040 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:40:49.040 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.041 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.041 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:106060 8.4 8:25362c51:::benchmark_data_smithi018_341490_object16737:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e416)", 2023-09-23T20:40:49.041 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:38:38.009403+0000", 2023-09-23T20:40:49.041 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 130.949940392, 2023-09-23T20:40:49.041 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.262631214, 2023-09-23T20:40:49.041 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:40:49.041 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:40:49.041 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:40:49.041 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:40:49.041 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:40:49.041 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 106060 2023-09-23T20:40:49.042 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.042 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:40:49.042 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.042 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:40:49.042 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.009403+0000", 2023-09-23T20:40:49.042 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:40:49.042 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.042 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.042 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:40:49.042 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.009403+0000", 2023-09-23T20:40:49.043 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:40:49.043 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.043 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.043 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:40:49.043 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.009402+0000", 2023-09-23T20:40:49.043 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:40:49.043 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.043 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.043 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:40:49.043 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.009407+0000", 2023-09-23T20:40:49.043 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.3259999999999997e-06 2023-09-23T20:40:49.044 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.044 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.044 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:40:49.044 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.009419+0000", 2023-09-23T20:40:49.044 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.2813000000000001e-05 2023-09-23T20:40:49.044 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.044 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.044 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:40:49.044 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.009425+0000", 2023-09-23T20:40:49.044 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.2839999999999997e-06 2023-09-23T20:40:49.044 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.045 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.045 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:40:49.045 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.009487+0000", 2023-09-23T20:40:49.045 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.2145999999999998e-05 2023-09-23T20:40:49.045 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.045 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.045 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:40:49.045 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.009500+0000", 2023-09-23T20:40:49.045 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.3382e-05 2023-09-23T20:40:49.045 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.046 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.046 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:40:49.046 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.261986+0000", 2023-09-23T20:40:49.046 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.2524858210000001 2023-09-23T20:40:49.046 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.046 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.046 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:40:49.046 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.262103+0000", 2023-09-23T20:40:49.046 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000116661 2023-09-23T20:40:49.046 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.046 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.047 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 2,5", 2023-09-23T20:40:49.047 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.262187+0000", 2023-09-23T20:40:49.047 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.4349000000000002e-05 2023-09-23T20:40:49.047 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.047 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.047 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:40:49.047 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.266297+0000", 2023-09-23T20:40:49.047 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0041101369999999998 2023-09-23T20:40:49.047 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.047 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.047 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:40:49.048 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.266347+0000", 2023-09-23T20:40:49.048 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.9561999999999997e-05 2023-09-23T20:40:49.048 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.048 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.048 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:40:49.048 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.271596+0000", 2023-09-23T20:40:49.048 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0052490719999999996 2023-09-23T20:40:49.048 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.048 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.048 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:40:49.048 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.271648+0000", 2023-09-23T20:40:49.049 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.2011000000000001e-05 2023-09-23T20:40:49.049 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.049 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.049 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:40:49.049 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.272035+0000", 2023-09-23T20:40:49.049 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000386702 2023-09-23T20:40:49.049 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:40:49.049 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:40:49.049 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:40:49.049 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.050 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.050 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:106077 8.2 8:405ecd88:::benchmark_data_smithi018_341490_object16754:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e416)", 2023-09-23T20:40:49.050 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:38:38.055215+0000", 2023-09-23T20:40:49.050 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 130.90412827500001, 2023-09-23T20:40:49.050 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.218947317, 2023-09-23T20:40:49.050 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:40:49.050 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:40:49.050 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:40:49.050 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:40:49.050 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:40:49.050 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 106077 2023-09-23T20:40:49.051 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.051 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:40:49.051 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.051 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:40:49.051 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.055215+0000", 2023-09-23T20:40:49.051 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:40:49.051 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.051 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.051 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:40:49.051 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.055215+0000", 2023-09-23T20:40:49.051 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:40:49.051 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.051 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.052 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:40:49.052 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.055214+0000", 2023-09-23T20:40:49.052 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:40:49.052 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.052 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.052 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:40:49.052 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.055221+0000", 2023-09-23T20:40:49.052 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.3150000000000003e-06 2023-09-23T20:40:49.052 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.052 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.052 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:40:49.052 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.055238+0000", 2023-09-23T20:40:49.053 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.6225000000000001e-05 2023-09-23T20:40:49.053 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.053 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.053 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:40:49.053 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.055244+0000", 2023-09-23T20:40:49.053 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.6780000000000003e-06 2023-09-23T20:40:49.053 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.053 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.053 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:40:49.053 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.055311+0000", 2023-09-23T20:40:49.053 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.6406000000000001e-05 2023-09-23T20:40:49.054 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.054 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.054 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:40:49.054 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.055328+0000", 2023-09-23T20:40:49.054 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.7190999999999999e-05 2023-09-23T20:40:49.054 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.054 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.054 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:40:49.054 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.259906+0000", 2023-09-23T20:40:49.054 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.2045777580000001 2023-09-23T20:40:49.054 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.054 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.055 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:40:49.055 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.259995+0000", 2023-09-23T20:40:49.055 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.9866999999999999e-05 2023-09-23T20:40:49.055 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.055 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.055 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:40:49.055 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.260085+0000", 2023-09-23T20:40:49.055 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.9098999999999995e-05 2023-09-23T20:40:49.055 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.055 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.055 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:40:49.056 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.266243+0000", 2023-09-23T20:40:49.056 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0061579579999999998 2023-09-23T20:40:49.056 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.056 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.056 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:40:49.056 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.273553+0000", 2023-09-23T20:40:49.056 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0073100300000000003 2023-09-23T20:40:49.056 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.056 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.056 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:40:49.056 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.273929+0000", 2023-09-23T20:40:49.056 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00037609299999999998 2023-09-23T20:40:49.057 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.057 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.057 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:40:49.057 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.273960+0000", 2023-09-23T20:40:49.057 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.1851999999999999e-05 2023-09-23T20:40:49.057 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.057 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.057 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:40:49.057 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.274163+0000", 2023-09-23T20:40:49.057 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000202303 2023-09-23T20:40:49.057 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:40:49.057 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:40:49.058 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:40:49.058 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.058 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.058 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:106082 8.2 8:43f6991f:::benchmark_data_smithi018_341490_object16759:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e416)", 2023-09-23T20:40:49.058 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:38:38.078328+0000", 2023-09-23T20:40:49.058 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 130.881015351, 2023-09-23T20:40:49.058 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.200252715, 2023-09-23T20:40:49.058 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:40:49.058 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:40:49.058 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:40:49.058 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:40:49.059 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:40:49.059 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 106082 2023-09-23T20:40:49.059 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.059 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:40:49.059 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.059 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:40:49.059 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.078328+0000", 2023-09-23T20:40:49.059 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:40:49.059 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.059 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.059 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:40:49.059 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.078328+0000", 2023-09-23T20:40:49.060 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:40:49.060 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.060 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.060 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:40:49.060 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.078327+0000", 2023-09-23T20:40:49.060 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999981 2023-09-23T20:40:49.060 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.060 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.060 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:40:49.060 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.078334+0000", 2023-09-23T20:40:49.060 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.4769999999999997e-06 2023-09-23T20:40:49.060 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.061 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.061 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:40:49.061 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.078353+0000", 2023-09-23T20:40:49.061 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.8457e-05 2023-09-23T20:40:49.061 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.061 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.061 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:40:49.061 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.078359+0000", 2023-09-23T20:40:49.061 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.6359999999999997e-06 2023-09-23T20:40:49.061 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.061 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.061 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:40:49.062 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.078428+0000", 2023-09-23T20:40:49.062 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.8532e-05 2023-09-23T20:40:49.062 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.062 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.062 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:40:49.062 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.078441+0000", 2023-09-23T20:40:49.062 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.2943e-05 2023-09-23T20:40:49.062 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.062 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.062 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:40:49.062 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.268537+0000", 2023-09-23T20:40:49.063 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.190096729 2023-09-23T20:40:49.063 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.063 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.063 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:40:49.063 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.268680+0000", 2023-09-23T20:40:49.063 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00014273500000000001 2023-09-23T20:40:49.063 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.063 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.063 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:40:49.063 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.268813+0000", 2023-09-23T20:40:49.063 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00013306700000000001 2023-09-23T20:40:49.064 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.064 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.064 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:40:49.064 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.272998+0000", 2023-09-23T20:40:49.064 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0041846360000000003 2023-09-23T20:40:49.064 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.064 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.064 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:40:49.064 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.274587+0000", 2023-09-23T20:40:49.064 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0015889490000000001 2023-09-23T20:40:49.064 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.064 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.064 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:40:49.065 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.278215+0000", 2023-09-23T20:40:49.065 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0036286130000000002 2023-09-23T20:40:49.065 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.065 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.065 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:40:49.065 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.278259+0000", 2023-09-23T20:40:49.065 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.3995000000000001e-05 2023-09-23T20:40:49.065 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.065 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.065 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:40:49.065 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.278581+0000", 2023-09-23T20:40:49.065 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00032174500000000002 2023-09-23T20:40:49.066 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:40:49.066 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:40:49.066 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:40:49.066 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.066 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.066 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:106090 8.4 8:202238a2:::benchmark_data_smithi018_341490_object16767:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e416)", 2023-09-23T20:40:49.066 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:38:38.108972+0000", 2023-09-23T20:40:49.066 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 130.850371663, 2023-09-23T20:40:49.066 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1634425150000001, 2023-09-23T20:40:49.066 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:40:49.066 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:40:49.067 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:40:49.067 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:40:49.067 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:40:49.067 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 106090 2023-09-23T20:40:49.067 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.067 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:40:49.067 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.067 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:40:49.067 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.108972+0000", 2023-09-23T20:40:49.067 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:40:49.068 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.068 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.068 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:40:49.068 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.108972+0000", 2023-09-23T20:40:49.068 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:40:49.068 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.068 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.068 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:40:49.068 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.108971+0000", 2023-09-23T20:40:49.068 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:40:49.068 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.068 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.069 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:40:49.069 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.108976+0000", 2023-09-23T20:40:49.069 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.4110000000000004e-06 2023-09-23T20:40:49.069 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.069 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.069 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:40:49.069 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.108989+0000", 2023-09-23T20:40:49.069 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.3166000000000001e-05 2023-09-23T20:40:49.069 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.069 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.069 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:40:49.069 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.108995+0000", 2023-09-23T20:40:49.070 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.4480000000000002e-06 2023-09-23T20:40:49.070 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.070 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.070 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:40:49.070 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.109033+0000", 2023-09-23T20:40:49.070 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.8402000000000003e-05 2023-09-23T20:40:49.070 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.070 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.070 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:40:49.070 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.109042+0000", 2023-09-23T20:40:49.070 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.5790000000000004e-06 2023-09-23T20:40:49.070 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.071 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.071 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:40:49.071 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.263413+0000", 2023-09-23T20:40:49.071 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1543713289999999 2023-09-23T20:40:49.071 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.071 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.071 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:40:49.071 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.263504+0000", 2023-09-23T20:40:49.071 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.0820000000000001e-05 2023-09-23T20:40:49.071 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.071 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.071 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 2,5", 2023-09-23T20:40:49.072 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.263583+0000", 2023-09-23T20:40:49.072 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.9167999999999998e-05 2023-09-23T20:40:49.072 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.072 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.072 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:40:49.072 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.266401+0000", 2023-09-23T20:40:49.072 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0028183430000000001 2023-09-23T20:40:49.072 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.072 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.072 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:40:49.072 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.266409+0000", 2023-09-23T20:40:49.072 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.0190000000000006e-06 2023-09-23T20:40:49.073 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.073 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.073 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:40:49.073 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.272098+0000", 2023-09-23T20:40:49.073 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0056886230000000003 2023-09-23T20:40:49.073 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.073 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.073 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:40:49.073 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.272162+0000", 2023-09-23T20:40:49.073 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.3738999999999994e-05 2023-09-23T20:40:49.073 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.073 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.074 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:40:49.074 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.272415+0000", 2023-09-23T20:40:49.074 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000252768 2023-09-23T20:40:49.074 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:40:49.074 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:40:49.074 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:40:49.074 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.074 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.074 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:106105 8.12 8:4ed2b003:::benchmark_data_smithi018_341490_object16782:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e416)", 2023-09-23T20:40:49.074 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:38:38.247356+0000", 2023-09-23T20:40:49.074 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 130.71198778199999, 2023-09-23T20:40:49.075 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.027019473, 2023-09-23T20:40:49.075 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:40:49.075 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:40:49.075 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:40:49.075 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:40:49.075 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:40:49.075 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 106105 2023-09-23T20:40:49.075 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.075 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:40:49.075 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.075 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:40:49.076 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.247356+0000", 2023-09-23T20:40:49.076 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:40:49.076 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.076 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.076 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:40:49.076 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.247356+0000", 2023-09-23T20:40:49.076 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:40:49.076 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.076 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.076 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:40:49.076 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.247355+0000", 2023-09-23T20:40:49.076 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:40:49.077 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.077 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.077 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:40:49.077 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.247360+0000", 2023-09-23T20:40:49.077 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.5809999999999996e-06 2023-09-23T20:40:49.077 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.077 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.077 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:40:49.077 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.247374+0000", 2023-09-23T20:40:49.077 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.3869999999999999e-05 2023-09-23T20:40:49.077 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.077 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.078 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:40:49.078 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.247380+0000", 2023-09-23T20:40:49.078 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.5670000000000002e-06 2023-09-23T20:40:49.078 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.078 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.078 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:40:49.078 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.247458+0000", 2023-09-23T20:40:49.078 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.7976e-05 2023-09-23T20:40:49.078 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.078 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.078 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:40:49.079 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.247467+0000", 2023-09-23T20:40:49.079 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.8230000000000006e-06 2023-09-23T20:40:49.079 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.079 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.079 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:40:49.079 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.262332+0000", 2023-09-23T20:40:49.079 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.014864722 2023-09-23T20:40:49.079 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.079 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.079 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:40:49.079 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.262420+0000", 2023-09-23T20:40:49.080 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.7597999999999997e-05 2023-09-23T20:40:49.080 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.080 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.080 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:40:49.080 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.262520+0000", 2023-09-23T20:40:49.080 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.9769e-05 2023-09-23T20:40:49.080 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.080 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.080 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:40:49.080 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.266258+0000", 2023-09-23T20:40:49.080 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.003738285 2023-09-23T20:40:49.080 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.080 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.081 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:40:49.081 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.273571+0000", 2023-09-23T20:40:49.081 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0073127560000000001 2023-09-23T20:40:49.081 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.081 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.081 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:40:49.081 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.274186+0000", 2023-09-23T20:40:49.081 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00061492000000000003 2023-09-23T20:40:49.081 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.081 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.081 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:40:49.081 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.274218+0000", 2023-09-23T20:40:49.082 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.2221000000000002e-05 2023-09-23T20:40:49.082 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.082 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.082 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:40:49.082 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.274375+0000", 2023-09-23T20:40:49.082 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000157708 2023-09-23T20:40:49.082 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:40:49.082 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:40:49.082 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:40:49.082 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.082 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.082 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:106118 8.4 8:27850836:::benchmark_data_smithi018_341490_object16795:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e416)", 2023-09-23T20:40:49.083 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:38:38.309923+0000", 2023-09-23T20:40:49.083 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 130.649420553, 2023-09-23T20:40:49.083 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.96280872399999995, 2023-09-23T20:40:49.083 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:40:49.083 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:40:49.083 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:40:49.083 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:40:49.083 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:40:49.083 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 106118 2023-09-23T20:40:49.083 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.084 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:40:49.084 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.084 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:40:49.084 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.309923+0000", 2023-09-23T20:40:49.084 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:40:49.084 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.084 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.084 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:40:49.084 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.309923+0000", 2023-09-23T20:40:49.084 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:40:49.084 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.084 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.085 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:40:49.085 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.309922+0000", 2023-09-23T20:40:49.085 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:40:49.085 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.085 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.085 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:40:49.085 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.309927+0000", 2023-09-23T20:40:49.085 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.2959999999999998e-06 2023-09-23T20:40:49.085 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.085 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.085 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:40:49.086 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.309941+0000", 2023-09-23T20:40:49.086 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.3375e-05 2023-09-23T20:40:49.086 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.086 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.086 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:40:49.086 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.309947+0000", 2023-09-23T20:40:49.086 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.7440000000000001e-06 2023-09-23T20:40:49.086 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.086 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.086 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:40:49.086 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.309999+0000", 2023-09-23T20:40:49.087 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.1266e-05 2023-09-23T20:40:49.087 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.087 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.087 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:40:49.087 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.310006+0000", 2023-09-23T20:40:49.087 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.6499999999999996e-06 2023-09-23T20:40:49.087 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.087 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.087 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:40:49.087 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.264766+0000", 2023-09-23T20:40:49.087 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.95475926600000005 2023-09-23T20:40:49.087 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.087 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.088 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:40:49.088 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.264896+0000", 2023-09-23T20:40:49.088 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000130514 2023-09-23T20:40:49.088 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.088 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.088 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 2,5", 2023-09-23T20:40:49.088 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.265017+0000", 2023-09-23T20:40:49.088 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000120581 2023-09-23T20:40:49.088 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.088 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.088 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:40:49.088 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.266595+0000", 2023-09-23T20:40:49.088 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.001578217 2023-09-23T20:40:49.089 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.089 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.089 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:40:49.089 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.267280+0000", 2023-09-23T20:40:49.089 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00068537800000000005 2023-09-23T20:40:49.089 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.089 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.089 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:40:49.089 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.272466+0000", 2023-09-23T20:40:49.089 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0051855180000000001 2023-09-23T20:40:49.089 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.089 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.089 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:40:49.090 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.272522+0000", 2023-09-23T20:40:49.090 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.5914e-05 2023-09-23T20:40:49.090 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.090 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.090 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:40:49.090 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.272732+0000", 2023-09-23T20:40:49.090 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00021019499999999999 2023-09-23T20:40:49.090 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:40:49.090 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:40:49.090 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:40:49.090 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.090 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.091 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:106123 8.2 8:435cd3c6:::benchmark_data_smithi018_341490_object16800:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e416)", 2023-09-23T20:40:49.091 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:38:38.326800+0000", 2023-09-23T20:40:49.091 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 130.63254359000001, 2023-09-23T20:40:49.091 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.95465794400000004, 2023-09-23T20:40:49.091 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:40:49.091 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:40:49.091 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:40:49.091 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:40:49.091 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:40:49.091 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 106123 2023-09-23T20:40:49.091 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.092 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:40:49.092 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.092 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:40:49.092 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.326800+0000", 2023-09-23T20:40:49.092 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:40:49.092 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.092 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.092 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:40:49.092 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.326800+0000", 2023-09-23T20:40:49.092 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:40:49.092 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.092 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.093 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:40:49.093 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.326799+0000", 2023-09-23T20:40:49.093 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:40:49.093 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.093 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.093 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:40:49.093 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.326805+0000", 2023-09-23T20:40:49.093 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.6259999999999997e-06 2023-09-23T20:40:49.093 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.093 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.093 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:40:49.094 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.326820+0000", 2023-09-23T20:40:49.094 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.4508000000000001e-05 2023-09-23T20:40:49.094 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.094 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.094 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:40:49.094 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.326826+0000", 2023-09-23T20:40:49.094 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.1709999999999999e-06 2023-09-23T20:40:49.094 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.094 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.094 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:40:49.094 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.326899+0000", 2023-09-23T20:40:49.095 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.2838999999999998e-05 2023-09-23T20:40:49.095 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.095 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.095 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:40:49.095 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.326913+0000", 2023-09-23T20:40:49.095 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.4022999999999999e-05 2023-09-23T20:40:49.095 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.095 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.095 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:40:49.095 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.270145+0000", 2023-09-23T20:40:49.095 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.94323180900000003 2023-09-23T20:40:49.096 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.096 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.096 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:40:49.096 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.270297+0000", 2023-09-23T20:40:49.096 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00015283500000000001 2023-09-23T20:40:49.096 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.096 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.096 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:40:49.096 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.270461+0000", 2023-09-23T20:40:49.096 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00016353799999999999 2023-09-23T20:40:49.096 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.097 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.097 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:40:49.097 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.273062+0000", 2023-09-23T20:40:49.097 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0026012800000000001 2023-09-23T20:40:49.097 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.097 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.097 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:40:49.097 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.276398+0000", 2023-09-23T20:40:49.097 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.003335321 2023-09-23T20:40:49.097 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.097 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.098 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:40:49.098 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.281194+0000", 2023-09-23T20:40:49.098 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0047966939999999998 2023-09-23T20:40:49.098 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.098 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.098 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:40:49.098 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.281226+0000", 2023-09-23T20:40:49.098 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.1965000000000003e-05 2023-09-23T20:40:49.098 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.098 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.098 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:40:49.099 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.281458+0000", 2023-09-23T20:40:49.099 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000231876 2023-09-23T20:40:49.099 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:40:49.099 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:40:49.099 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:40:49.099 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.099 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.099 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:106135 8.14 8:2e4b5b89:::benchmark_data_smithi018_341490_object16812:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e416)", 2023-09-23T20:40:49.099 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:38:38.358297+0000", 2023-09-23T20:40:49.099 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 130.60104642300001, 2023-09-23T20:40:49.099 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.91161518500000005, 2023-09-23T20:40:49.099 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:40:49.100 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:40:49.100 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:40:49.100 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:40:49.100 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:40:49.100 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 106135 2023-09-23T20:40:49.100 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.100 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:40:49.100 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.100 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:40:49.100 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.358297+0000", 2023-09-23T20:40:49.100 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:40:49.100 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.101 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.101 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:40:49.101 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.358297+0000", 2023-09-23T20:40:49.101 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:40:49.101 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.101 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.101 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:40:49.101 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.358294+0000", 2023-09-23T20:40:49.101 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999971 2023-09-23T20:40:49.101 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.101 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.101 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:40:49.101 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.358302+0000", 2023-09-23T20:40:49.102 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.0039999999999999e-06 2023-09-23T20:40:49.102 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.102 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.102 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:40:49.102 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.358329+0000", 2023-09-23T20:40:49.102 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.6804999999999998e-05 2023-09-23T20:40:49.102 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.102 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.102 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:40:49.102 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.358337+0000", 2023-09-23T20:40:49.102 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.276e-06 2023-09-23T20:40:49.102 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.103 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.103 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:40:49.103 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.358408+0000", 2023-09-23T20:40:49.103 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.0525000000000007e-05 2023-09-23T20:40:49.103 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.103 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.103 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:40:49.103 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.358419+0000", 2023-09-23T20:40:49.103 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.083e-05 2023-09-23T20:40:49.103 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.103 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.103 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:40:49.103 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.260630+0000", 2023-09-23T20:40:49.104 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.90221140300000002 2023-09-23T20:40:49.104 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.104 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.104 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:40:49.104 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.260747+0000", 2023-09-23T20:40:49.104 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000116555 2023-09-23T20:40:49.104 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.104 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.104 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 2,5", 2023-09-23T20:40:49.104 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.260886+0000", 2023-09-23T20:40:49.104 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00013910600000000001 2023-09-23T20:40:49.104 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.105 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.105 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:40:49.105 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.263408+0000", 2023-09-23T20:40:49.105 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0025218670000000001 2023-09-23T20:40:49.105 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.105 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.105 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:40:49.105 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.265119+0000", 2023-09-23T20:40:49.105 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.001711661 2023-09-23T20:40:49.105 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.105 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.106 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:40:49.106 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.269557+0000", 2023-09-23T20:40:49.106 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0044376390000000002 2023-09-23T20:40:49.106 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.106 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.106 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:40:49.106 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.269618+0000", 2023-09-23T20:40:49.106 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.0798999999999998e-05 2023-09-23T20:40:49.106 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:40:49.106 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:40:49.106 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:40:49.107 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.269913+0000", 2023-09-23T20:40:49.107 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00029461999999999999 2023-09-23T20:40:49.107 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:40:49.107 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:40:49.107 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:40:49.107 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:40:49.107 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:40:49.107 INFO:teuthology.orchestra.run.smithi149.stdout:} 2023-09-23T20:40:49.378 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T20:40:49.546 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:49 smithi018 ceph-mon[184144]: pgmap v1106: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 18 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 43 op/s; 12429/729 objects degraded (1704.938%); 46/729 objects misplaced (6.310%); 0 B/s, 51 objects/s recovering 2023-09-23T20:40:49.547 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:49 smithi018 ceph-mon[179647]: pgmap v1106: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 18 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 43 op/s; 12429/729 objects degraded (1704.938%); 46/729 objects misplaced (6.310%); 0 B/s, 51 objects/s recovering 2023-09-23T20:40:49.666 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:40:49.669 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:49.669 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:40:49.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:49 smithi149 ceph-mon[170394]: pgmap v1106: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 8 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 18 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 43 op/s; 12429/729 objects degraded (1704.938%); 46/729 objects misplaced (6.310%); 0 B/s, 51 objects/s recovering 2023-09-23T20:40:49.770 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:40:49.798 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:49.832 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:40:49.836 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:49.836 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:40:49.937 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:40:49.964 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:50.000 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:40:50.005 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:50.006 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:40:50.006 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:40:50.106 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:40:50.134 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:50.221 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:40:50.224 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:50.224 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:40:50.247 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:40:50.325 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:40:50.386 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:51.328 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:40:51.330 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:51.331 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:40:51.339 INFO:teuthology.orchestra.run.smithi149.stdout:{ 2023-09-23T20:40:51.339 INFO:teuthology.orchestra.run.smithi149.stdout: "ops": [], 2023-09-23T20:40:51.339 INFO:teuthology.orchestra.run.smithi149.stdout: "num_ops": 0 2023-09-23T20:40:51.339 INFO:teuthology.orchestra.run.smithi149.stdout:} 2023-09-23T20:40:51.431 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:40:51.468 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:51.602 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T20:40:51.629 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:51 smithi018 ceph-mon[184144]: pgmap v1107: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 5 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 18 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 3 op/s; 9906/729 objects degraded (1358.848%); 46/729 objects misplaced (6.310%); 0 B/s, 272 objects/s recovering 2023-09-23T20:40:51.629 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:51 smithi018 ceph-mon[184144]: osdmap e450: 8 total, 8 up, 8 in 2023-09-23T20:40:51.629 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:51 smithi018 ceph-mon[184144]: failed to encode map e450 with expected crc 2023-09-23T20:40:51.629 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:51 smithi018 ceph-mon[184144]: failed to encode map e450 with expected crc 2023-09-23T20:40:51.629 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:51 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 9906/729 objects degraded (1358.848%), 8 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T20:40:51.629 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:51 smithi018 ceph-mon[179647]: pgmap v1107: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 5 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 18 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 3 op/s; 9906/729 objects degraded (1358.848%); 46/729 objects misplaced (6.310%); 0 B/s, 272 objects/s recovering 2023-09-23T20:40:51.630 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:51 smithi018 ceph-mon[179647]: osdmap e450: 8 total, 8 up, 8 in 2023-09-23T20:40:51.630 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:51 smithi018 ceph-mon[179647]: failed to encode map e450 with expected crc 2023-09-23T20:40:51.630 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:51 smithi018 ceph-mon[179647]: failed to encode map e450 with expected crc 2023-09-23T20:40:51.630 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:51 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 9906/729 objects degraded (1358.848%), 8 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T20:40:51.641 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:40:51.642 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:40:51.642 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:40:51.642 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:40:51.642 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:40:51.642 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:40:51.656 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:40:51.663 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:40:51.667 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:51.668 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:40:51.684 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:40:51.685 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:51 smithi149 ceph-mon[170394]: pgmap v1107: 129 pgs: 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 5 active+remapped+backfill_wait, 7 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 18 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 3 op/s; 9906/729 objects degraded (1358.848%); 46/729 objects misplaced (6.310%); 0 B/s, 272 objects/s recovering 2023-09-23T20:40:51.685 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:51 smithi149 ceph-mon[170394]: osdmap e450: 8 total, 8 up, 8 in 2023-09-23T20:40:51.685 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:51 smithi149 ceph-mon[170394]: failed to encode map e450 with expected crc 2023-09-23T20:40:51.685 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:51 smithi149 ceph-mon[170394]: failed to encode map e450 with expected crc 2023-09-23T20:40:51.685 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:51 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 9906/729 objects degraded (1358.848%), 8 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T20:40:51.685 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:40:51.686 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:40:51.686 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:40:51.686 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:40:51.686 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:40:51.686 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:40:51.686 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:40:51.686 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:40:51.767 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T20:40:51.768 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:40:51.797 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:51.834 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:40:51.837 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:51.838 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:40:51.939 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:40:51.966 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:52.004 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:40:52.008 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:52.009 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:40:52.110 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:40:52.137 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:52.175 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:40:52.178 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:52.179 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:40:52.279 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:40:52.307 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:52.341 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:40:52.344 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:52.345 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:40:52.398 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:40:52.445 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:40:52.473 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:52.522 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:40:52.524 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:52.525 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:40:52.607 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:52 smithi149 ceph-mon[170394]: osdmap e451: 8 total, 8 up, 8 in 2023-09-23T20:40:52.607 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:52 smithi149 ceph-mon[170394]: failed to encode map e451 with expected crc 2023-09-23T20:40:52.607 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:52 smithi149 ceph-mon[170394]: failed to encode map e451 with expected crc 2023-09-23T20:40:52.607 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:52 smithi149 ceph-mon[170394]: failed to encode map e451 with expected crc 2023-09-23T20:40:52.607 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:52 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2804645590' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:40:52.625 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:40:52.655 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:52.692 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:40:52.695 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:52.696 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:40:52.796 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:40:52.825 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:52 smithi018 ceph-mon[179647]: osdmap e451: 8 total, 8 up, 8 in 2023-09-23T20:40:52.825 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:52 smithi018 ceph-mon[179647]: failed to encode map e451 with expected crc 2023-09-23T20:40:52.825 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:52 smithi018 ceph-mon[179647]: failed to encode map e451 with expected crc 2023-09-23T20:40:52.825 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:52 smithi018 ceph-mon[179647]: failed to encode map e451 with expected crc 2023-09-23T20:40:52.825 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:52 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2804645590' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:40:52.826 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:52 smithi018 ceph-mon[184144]: osdmap e451: 8 total, 8 up, 8 in 2023-09-23T20:40:52.826 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:52 smithi018 ceph-mon[184144]: failed to encode map e451 with expected crc 2023-09-23T20:40:52.826 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:52 smithi018 ceph-mon[184144]: failed to encode map e451 with expected crc 2023-09-23T20:40:52.826 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:52 smithi018 ceph-mon[184144]: failed to encode map e451 with expected crc 2023-09-23T20:40:52.826 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:52 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2804645590' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:40:52.828 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:53.120 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:40:53.123 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:53.123 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:40:53.225 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:40:53.255 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:53.318 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:40:53.320 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:53.321 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:40:53.422 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:40:53.451 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:53.462 INFO:teuthology.orchestra.run.smithi018.stderr:pool 'unique_pool_3' removed 2023-09-23T20:40:53.539 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:40:53.542 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:53.543 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:40:53.642 INFO:teuthology.orchestra.run.smithi149.stdout:ERROR: (22) Invalid argument 2023-09-23T20:40:53.642 INFO:teuthology.orchestra.run.smithi149.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:40:53.644 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:40:53.663 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:53 smithi149 ceph-mon[170394]: pgmap v1110: 129 pgs: 1 peering, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 5 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 18 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 9895/729 objects degraded (1357.339%); 46/729 objects misplaced (6.310%); 0 B/s, 226 objects/s recovering 2023-09-23T20:40:53.663 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:53 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1136731097' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T20:40:53.663 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:53 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T20:40:53.672 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:53.707 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:40:53.709 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:53 smithi018 ceph-mon[184144]: pgmap v1110: 129 pgs: 1 peering, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 5 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 18 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 9895/729 objects degraded (1357.339%); 46/729 objects misplaced (6.310%); 0 B/s, 226 objects/s recovering 2023-09-23T20:40:53.709 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:53 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1136731097' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T20:40:53.709 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:53 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T20:40:53.710 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:53 smithi018 ceph-mon[179647]: pgmap v1110: 129 pgs: 1 peering, 1 active+recovery_wait+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 5 active+remapped+backfill_wait, 6 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 18 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 9895/729 objects degraded (1357.339%); 46/729 objects misplaced (6.310%); 0 B/s, 226 objects/s recovering 2023-09-23T20:40:53.710 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:53 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1136731097' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T20:40:53.710 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:53 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T20:40:53.710 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:53.711 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:40:53.811 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:40:53.841 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:54.051 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T20:40:54.580 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:40:54.583 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:54.584 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:40:54.648 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:54 smithi018 ceph-mon[179647]: failed to encode map e452 with expected crc 2023-09-23T20:40:54.648 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:54 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T20:40:54.649 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:54 smithi018 ceph-mon[179647]: osdmap e452: 8 total, 8 up, 8 in 2023-09-23T20:40:54.651 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:54 smithi018 ceph-mon[184144]: failed to encode map e452 with expected crc 2023-09-23T20:40:54.651 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:54 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T20:40:54.651 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:54 smithi018 ceph-mon[184144]: osdmap e452: 8 total, 8 up, 8 in 2023-09-23T20:40:54.685 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:40:54.711 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:40:54.712 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:54.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:54 smithi149 ceph-mon[170394]: failed to encode map e452 with expected crc 2023-09-23T20:40:54.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:54 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T20:40:54.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:54 smithi149 ceph-mon[170394]: osdmap e452: 8 total, 8 up, 8 in 2023-09-23T20:40:54.839 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:40:54.843 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:54.844 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:40:54.944 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:40:54.973 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:55.141 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:40:54 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:40:54] "GET /metrics HTTP/1.1" 200 36565 "" "Prometheus/2.43.0" 2023-09-23T20:40:55.194 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:40:55.359 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:40:55.361 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:55.362 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:40:55.463 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:40:55.494 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:55.540 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:40:55.544 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:55.545 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:40:55.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:55 smithi018 ceph-mon[184144]: pgmap v1112: 103 pgs: 1 peering, 1 active+recovery_wait+remapped, 101 active+clean; 4.3 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 1/81 objects misplaced (1.235%) 2023-09-23T20:40:55.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:55 smithi018 ceph-mon[184144]: failed to encode map e452 with expected crc 2023-09-23T20:40:55.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:55 smithi018 ceph-mon[184144]: failed to encode map e452 with expected crc 2023-09-23T20:40:55.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:55 smithi018 ceph-mon[184144]: failed to encode map e452 with expected crc 2023-09-23T20:40:55.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:55 smithi018 ceph-mon[184144]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 9895/729 objects degraded (1357.339%), 7 pgs degraded, 4 pgs undersized) 2023-09-23T20:40:55.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:55 smithi018 ceph-mon[179647]: pgmap v1112: 103 pgs: 1 peering, 1 active+recovery_wait+remapped, 101 active+clean; 4.3 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 1/81 objects misplaced (1.235%) 2023-09-23T20:40:55.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:55 smithi018 ceph-mon[179647]: failed to encode map e452 with expected crc 2023-09-23T20:40:55.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:55 smithi018 ceph-mon[179647]: failed to encode map e452 with expected crc 2023-09-23T20:40:55.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:55 smithi018 ceph-mon[179647]: failed to encode map e452 with expected crc 2023-09-23T20:40:55.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:55 smithi018 ceph-mon[179647]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 9895/729 objects degraded (1357.339%), 7 pgs degraded, 4 pgs undersized) 2023-09-23T20:40:55.645 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:40:55.672 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:55.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:55 smithi149 ceph-mon[170394]: pgmap v1112: 103 pgs: 1 peering, 1 active+recovery_wait+remapped, 101 active+clean; 4.3 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 1/81 objects misplaced (1.235%) 2023-09-23T20:40:55.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:55 smithi149 ceph-mon[170394]: failed to encode map e452 with expected crc 2023-09-23T20:40:55.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:55 smithi149 ceph-mon[170394]: failed to encode map e452 with expected crc 2023-09-23T20:40:55.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:55 smithi149 ceph-mon[170394]: failed to encode map e452 with expected crc 2023-09-23T20:40:55.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:55 smithi149 ceph-mon[170394]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 9895/729 objects degraded (1357.339%), 7 pgs degraded, 4 pgs undersized) 2023-09-23T20:40:55.759 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:40:55.762 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:55.763 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:40:55.864 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:40:55.893 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:55.946 INFO:teuthology.orchestra.run.smithi149.stdout:ERROR: (22) Invalid argument 2023-09-23T20:40:55.946 INFO:teuthology.orchestra.run.smithi149.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:40:56.279 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T20:40:56.281 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:40:56.285 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:56.285 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:40:56.385 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-09-23T20:40:56.386 INFO:tasks.radosbench:Beginning radosbench... 2023-09-23T20:40:56.386 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_4 2023-09-23T20:40:56.386 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd pool create unique_pool_4 16 2023-09-23T20:40:56.387 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:40:56.417 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:56.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:56 smithi018 ceph-mon[179647]: failed to encode map e453 with expected crc 2023-09-23T20:40:56.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:56 smithi018 ceph-mon[179647]: osdmap e453: 8 total, 8 up, 8 in 2023-09-23T20:40:56.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:56 smithi018 ceph-mon[179647]: failed to encode map e453 with expected crc 2023-09-23T20:40:56.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:56 smithi018 ceph-mon[184144]: failed to encode map e453 with expected crc 2023-09-23T20:40:56.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:56 smithi018 ceph-mon[184144]: osdmap e453: 8 total, 8 up, 8 in 2023-09-23T20:40:56.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:56 smithi018 ceph-mon[184144]: failed to encode map e453 with expected crc 2023-09-23T20:40:56.889 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:40:56.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:56 smithi149 ceph-mon[170394]: failed to encode map e453 with expected crc 2023-09-23T20:40:56.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:56 smithi149 ceph-mon[170394]: osdmap e453: 8 total, 8 up, 8 in 2023-09-23T20:40:56.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:56 smithi149 ceph-mon[170394]: failed to encode map e453 with expected crc 2023-09-23T20:40:57.643 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:57 smithi149 ceph-mon[170394]: pgmap v1114: 103 pgs: 1 peering, 102 active+clean; 4.3 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-09-23T20:40:57.643 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:57 smithi149 ceph-mon[170394]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T20:40:57.643 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:57 smithi149 ceph-mon[170394]: failed to encode map e453 with expected crc 2023-09-23T20:40:57.643 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:57 smithi149 ceph-mon[170394]: osdmap e454: 8 total, 8 up, 8 in 2023-09-23T20:40:57.643 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:57 smithi149 ceph-mon[170394]: failed to encode map e454 with expected crc 2023-09-23T20:40:57.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:57 smithi018 ceph-mon[179647]: pgmap v1114: 103 pgs: 1 peering, 102 active+clean; 4.3 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-09-23T20:40:57.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:57 smithi018 ceph-mon[179647]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T20:40:57.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:57 smithi018 ceph-mon[179647]: failed to encode map e453 with expected crc 2023-09-23T20:40:57.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:57 smithi018 ceph-mon[179647]: osdmap e454: 8 total, 8 up, 8 in 2023-09-23T20:40:57.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:57 smithi018 ceph-mon[179647]: failed to encode map e454 with expected crc 2023-09-23T20:40:57.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:57 smithi018 ceph-mon[184144]: pgmap v1114: 103 pgs: 1 peering, 102 active+clean; 4.3 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-09-23T20:40:57.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:57 smithi018 ceph-mon[184144]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T20:40:57.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:57 smithi018 ceph-mon[184144]: failed to encode map e453 with expected crc 2023-09-23T20:40:57.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:57 smithi018 ceph-mon[184144]: osdmap e454: 8 total, 8 up, 8 in 2023-09-23T20:40:57.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:57 smithi018 ceph-mon[184144]: failed to encode map e454 with expected crc 2023-09-23T20:40:58.069 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:40:58.073 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:58.074 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:40:58.100 INFO:teuthology.orchestra.run.smithi149.stdout:ERROR: (22) Invalid argument 2023-09-23T20:40:58.100 INFO:teuthology.orchestra.run.smithi149.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:40:58.107 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:40:58.175 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:40:58.204 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:58.436 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T20:40:58.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:58 smithi149 ceph-mon[170394]: failed to encode map e454 with expected crc 2023-09-23T20:40:58.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:58 smithi018 ceph-mon[179647]: failed to encode map e454 with expected crc 2023-09-23T20:40:58.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:58 smithi018 ceph-mon[184144]: failed to encode map e454 with expected crc 2023-09-23T20:40:58.933 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:40:58.936 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:58.937 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:40:59.038 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:40:59.040 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:40:59.069 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:59.170 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:40:59.220 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:40:59.222 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:59.223 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:40:59.324 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:40:59.352 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:59.388 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:40:59.391 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:59.392 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:40:59.480 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:59 smithi018 ceph-mon[179647]: pgmap v1116: 103 pgs: 1 peering, 102 active+clean; 4.3 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-09-23T20:40:59.480 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:59 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:40:59.480 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:40:59 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:40:59.493 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:40:59.518 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:59 smithi149 ceph-mon[170394]: pgmap v1116: 103 pgs: 1 peering, 102 active+clean; 4.3 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-09-23T20:40:59.518 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:59 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:40:59.518 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:40:59 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:40:59.521 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:59.708 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:40:59.711 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:59.711 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:40:59.812 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:40:59.839 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:40:59.874 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:40:59.878 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:40:59.878 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:40:59.885 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:40:59.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:59 smithi018 ceph-mon[184144]: pgmap v1116: 103 pgs: 1 peering, 102 active+clean; 4.3 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2023-09-23T20:40:59.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:59 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:40:59.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:40:59 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:40:59.979 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:41:00.010 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:00.216 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:41:00.222 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:00.222 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:41:00.323 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:41:00.353 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:00.363 INFO:teuthology.orchestra.run.smithi149.stdout:ERROR: (22) Invalid argument 2023-09-23T20:41:00.363 INFO:teuthology.orchestra.run.smithi149.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:41:00.392 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:41:00.397 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:00.397 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:41:00.498 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:41:00.529 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:00.663 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:41:00.666 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:00.667 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:41:00.695 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T20:41:00.768 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:41:00.806 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:00.841 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:41:00.845 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:00.846 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:41:00.947 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:41:00.977 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:01.201 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:41:01.203 INFO:teuthology.orchestra.run.smithi018.stderr:dumped all 2023-09-23T20:41:01.342 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:41:01.551 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:41:01.554 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:01.555 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:41:01.656 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:41:01.685 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:01 smithi018 ceph-mon[184144]: pgmap v1117: 103 pgs: 1 peering, 102 active+clean; 4.3 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s, 1 objects/s recovering 2023-09-23T20:41:01.686 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:01 smithi018 ceph-mon[179647]: pgmap v1117: 103 pgs: 1 peering, 102 active+clean; 4.3 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s, 1 objects/s recovering 2023-09-23T20:41:01.689 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:01.782 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:01 smithi149 ceph-mon[170394]: pgmap v1117: 103 pgs: 1 peering, 102 active+clean; 4.3 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s, 1 objects/s recovering 2023-09-23T20:41:02.161 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:41:02.165 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:02.166 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:41:02.192 INFO:teuthology.orchestra.run.smithi018.stdout:{"pg_ready":true,"pg_map":{"version":1117,"stamp":"2023-09-23T20:41:00.093149+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4551511,"num_objects":27,"num_object_clones":0,"num_object_copies":81,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":27,"num_whiteouts":0,"num_read":6138,"num_read_kb":13490,"num_write":983,"num_write_kb":13493,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":151,"num_bytes_recovered":9298306,"num_keys_recovered":40,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6022,"ondisk_log_size":6022,"up":309,"acting":309,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":364,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":5368144,"kb_used_data":1280464,"kb_used_omap":32,"kb_used_meta":4087583,"kb_avail":744593072,"statfs":{"total":767960285184,"available":762463305728,"internally_reserved":0,"allocated":1311195136,"data_stored":1296045363,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32773,"internal_metadata":4185686011},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":38,"apply_latency_ms":38,"commit_latency_ns":38000000,"apply_latency_ns":38000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-1,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.638452"},"pg_stats":[{"pgid":"1.18","version":"90'161","reported_seq":427,"reported_epoch":453,"state":"peering","last_fresh":"2023-09-23T20:40:55.476186+0000","last_change":"2023-09-23T20:40:55.476461+0000","last_active":"2023-09-23T20:27:09.188873+0000","last_peered":"2023-09-23T20:15:59.452160+0000","last_clean":"2023-09-23T20:15:59.452160+0000","last_became_active":"2023-09-23T20:11:27.848678+0000","last_became_peered":"2023-09-23T20:11:27.848678+0000","last_unstale":"2023-09-23T20:40:55.476186+0000","last_undegraded":"2023-09-23T20:40:55.476186+0000","last_fullsized":"2023-09-23T20:40:55.476186+0000","mapping_epoch":453,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":405,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:35:03.551143+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:35:03.551143+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[1,2,7],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"69'1","reported_seq":150,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-09-23T20:40:54.248475+0000","last_change":"2023-09-23T20:32:33.233853+0000","last_active":"2023-09-23T20:40:54.248475+0000","last_peered":"2023-09-23T20:40:54.248475+0000","last_clean":"2023-09-23T20:40:54.248475+0000","last_became_active":"2023-09-23T20:32:33.233546+0000","last_became_peered":"2023-09-23T20:32:33.233546+0000","last_unstale":"2023-09-23T20:40:54.248475+0000","last_undegraded":"2023-09-23T20:40:54.248475+0000","last_fullsized":"2023-09-23T20:40:54.248475+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:50:27.130808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":267,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T20:40:55.618134+0000","last_change":"2023-09-23T20:37:43.579611+0000","last_active":"2023-09-23T20:40:55.618134+0000","last_peered":"2023-09-23T20:40:55.618134+0000","last_clean":"2023-09-23T20:40:55.618134+0000","last_became_active":"2023-09-23T20:37:43.579323+0000","last_became_peered":"2023-09-23T20:37:43.579323+0000","last_unstale":"2023-09-23T20:40:55.618134+0000","last_undegraded":"2023-09-23T20:40:55.618134+0000","last_fullsized":"2023-09-23T20:40:55.618134+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:17:32.311427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"90'161","reported_seq":526,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-09-23T20:40:54.221925+0000","last_change":"2023-09-23T20:40:22.445059+0000","last_active":"2023-09-23T20:40:54.221925+0000","last_peered":"2023-09-23T20:40:54.221925+0000","last_clean":"2023-09-23T20:40:54.221925+0000","last_became_active":"2023-09-23T20:40:22.444637+0000","last_became_peered":"2023-09-23T20:40:22.444637+0000","last_unstale":"2023-09-23T20:40:54.221925+0000","last_undegraded":"2023-09-23T20:40:54.221925+0000","last_fullsized":"2023-09-23T20:40:54.221925+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:05.442288+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:36:05.442288+0000","last_clean_scrub_stamp":"2023-09-23T20:36:05.442288+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0019500940000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":422,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T20:40:55.472237+0000","last_change":"2023-09-23T20:34:59.983035+0000","last_active":"2023-09-23T20:40:55.472237+0000","last_peered":"2023-09-23T20:40:55.472237+0000","last_clean":"2023-09-23T20:40:55.472237+0000","last_became_active":"2023-09-23T20:34:59.982459+0000","last_became_peered":"2023-09-23T20:34:59.982459+0000","last_unstale":"2023-09-23T20:40:55.472237+0000","last_undegraded":"2023-09-23T20:40:55.472237+0000","last_fullsized":"2023-09-23T20:40:55.472237+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:16:48.510783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":423,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-09-23T20:40:53.908187+0000","last_change":"2023-09-23T20:25:02.957185+0000","last_active":"2023-09-23T20:40:53.908187+0000","last_peered":"2023-09-23T20:40:53.908187+0000","last_clean":"2023-09-23T20:40:53.908187+0000","last_became_active":"2023-09-23T20:25:02.956689+0000","last_became_peered":"2023-09-23T20:25:02.956689+0000","last_unstale":"2023-09-23T20:40:53.908187+0000","last_undegraded":"2023-09-23T20:40:53.908187+0000","last_fullsized":"2023-09-23T20:40:53.908187+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:04:28.524015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"90'161","reported_seq":532,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-09-23T20:40:54.243846+0000","last_change":"2023-09-23T20:36:01.292103+0000","last_active":"2023-09-23T20:40:54.243846+0000","last_peered":"2023-09-23T20:40:54.243846+0000","last_clean":"2023-09-23T20:40:54.243846+0000","last_became_active":"2023-09-23T20:35:26.639912+0000","last_became_peered":"2023-09-23T20:35:26.639912+0000","last_unstale":"2023-09-23T20:40:54.243846+0000","last_undegraded":"2023-09-23T20:40:54.243846+0000","last_fullsized":"2023-09-23T20:40:54.243846+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:01.292032+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:01.292032+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:04:53.926640+0000","scrub_duration":0.0032983159999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":247,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T20:40:55.618306+0000","last_change":"2023-09-23T20:37:43.584414+0000","last_active":"2023-09-23T20:40:55.618306+0000","last_peered":"2023-09-23T20:40:55.618306+0000","last_clean":"2023-09-23T20:40:55.618306+0000","last_became_active":"2023-09-23T20:37:43.584116+0000","last_became_peered":"2023-09-23T20:37:43.584116+0000","last_unstale":"2023-09-23T20:40:55.618306+0000","last_undegraded":"2023-09-23T20:40:55.618306+0000","last_fullsized":"2023-09-23T20:40:55.618306+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:17:54.258172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":69,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-09-23T20:40:53.461763+0000","last_change":"2023-09-23T20:38:37.457427+0000","last_active":"2023-09-23T20:40:53.461763+0000","last_peered":"2023-09-23T20:40:53.461763+0000","last_clean":"2023-09-23T20:40:53.461763+0000","last_became_active":"2023-09-23T20:38:37.457099+0000","last_became_peered":"2023-09-23T20:38:37.457099+0000","last_unstale":"2023-09-23T20:40:53.461763+0000","last_undegraded":"2023-09-23T20:40:53.461763+0000","last_fullsized":"2023-09-23T20:40:53.461763+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:31:27.084994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"90'161","reported_seq":666,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T20:40:55.478841+0000","last_change":"2023-09-23T20:40:22.449754+0000","last_active":"2023-09-23T20:40:55.478841+0000","last_peered":"2023-09-23T20:40:55.478841+0000","last_clean":"2023-09-23T20:40:55.478841+0000","last_became_active":"2023-09-23T20:40:22.448838+0000","last_became_peered":"2023-09-23T20:40:22.448838+0000","last_unstale":"2023-09-23T20:40:55.478841+0000","last_undegraded":"2023-09-23T20:40:55.478841+0000","last_fullsized":"2023-09-23T20:40:55.478841+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:03.411025+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:03.411025+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:22:07.948856+0000","scrub_duration":0.001802922,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":456,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-09-23T20:40:53.905574+0000","last_change":"2023-09-23T20:40:19.538844+0000","last_active":"2023-09-23T20:40:53.905574+0000","last_peered":"2023-09-23T20:40:53.905574+0000","last_clean":"2023-09-23T20:40:53.905574+0000","last_became_active":"2023-09-23T20:40:19.538434+0000","last_became_peered":"2023-09-23T20:40:19.538434+0000","last_unstale":"2023-09-23T20:40:53.905574+0000","last_undegraded":"2023-09-23T20:40:53.905574+0000","last_fullsized":"2023-09-23T20:40:53.905574+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:53:59.418427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":177,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T20:40:55.478461+0000","last_change":"2023-09-23T20:38:02.579343+0000","last_active":"2023-09-23T20:40:55.478461+0000","last_peered":"2023-09-23T20:40:55.478461+0000","last_clean":"2023-09-23T20:40:55.478461+0000","last_became_active":"2023-09-23T20:38:02.579047+0000","last_became_peered":"2023-09-23T20:38:02.579047+0000","last_unstale":"2023-09-23T20:40:55.478461+0000","last_undegraded":"2023-09-23T20:40:55.478461+0000","last_fullsized":"2023-09-23T20:40:55.478461+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:44:03.166556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.14","version":"90'161","reported_seq":783,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T20:40:55.478073+0000","last_change":"2023-09-23T20:35:02.236398+0000","last_active":"2023-09-23T20:40:55.478073+0000","last_peered":"2023-09-23T20:40:55.478073+0000","last_clean":"2023-09-23T20:40:55.478073+0000","last_became_active":"2023-09-23T20:35:02.235988+0000","last_became_peered":"2023-09-23T20:35:02.235988+0000","last_unstale":"2023-09-23T20:40:55.478073+0000","last_undegraded":"2023-09-23T20:40:55.478073+0000","last_fullsized":"2023-09-23T20:40:55.478073+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:47.479012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:47.479012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:13.519648+0000","scrub_duration":0.0020136590000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":22,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-09-23T20:40:54.221058+0000","last_change":"2023-09-23T20:40:19.545472+0000","last_active":"2023-09-23T20:40:54.221058+0000","last_peered":"2023-09-23T20:40:54.221058+0000","last_clean":"2023-09-23T20:40:54.221058+0000","last_became_active":"2023-09-23T20:40:19.543853+0000","last_became_peered":"2023-09-23T20:40:19.543853+0000","last_unstale":"2023-09-23T20:40:54.221058+0000","last_undegraded":"2023-09-23T20:40:54.221058+0000","last_fullsized":"2023-09-23T20:40:54.221058+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:08.014744+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":278,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T20:40:55.618551+0000","last_change":"2023-09-23T20:38:02.542034+0000","last_active":"2023-09-23T20:40:55.618551+0000","last_peered":"2023-09-23T20:40:55.618551+0000","last_clean":"2023-09-23T20:40:55.618551+0000","last_became_active":"2023-09-23T20:38:02.541575+0000","last_became_peered":"2023-09-23T20:38:02.541575+0000","last_unstale":"2023-09-23T20:40:55.618551+0000","last_undegraded":"2023-09-23T20:40:55.618551+0000","last_fullsized":"2023-09-23T20:40:55.618551+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:50:54.591529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"90'161","reported_seq":606,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-09-23T20:40:54.225423+0000","last_change":"2023-09-23T20:40:22.444898+0000","last_active":"2023-09-23T20:40:54.225423+0000","last_peered":"2023-09-23T20:40:54.225423+0000","last_clean":"2023-09-23T20:40:54.225423+0000","last_became_active":"2023-09-23T20:40:22.444543+0000","last_became_peered":"2023-09-23T20:40:22.444543+0000","last_unstale":"2023-09-23T20:40:54.225423+0000","last_undegraded":"2023-09-23T20:40:54.225423+0000","last_fullsized":"2023-09-23T20:40:54.225423+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:03.435480+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:03.435480+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":481,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T20:40:55.479165+0000","last_change":"2023-09-23T20:32:33.270617+0000","last_active":"2023-09-23T20:40:55.479165+0000","last_peered":"2023-09-23T20:40:55.479165+0000","last_clean":"2023-09-23T20:40:55.479165+0000","last_became_active":"2023-09-23T20:32:33.227902+0000","last_became_peered":"2023-09-23T20:32:33.227902+0000","last_unstale":"2023-09-23T20:40:55.479165+0000","last_undegraded":"2023-09-23T20:40:55.479165+0000","last_fullsized":"2023-09-23T20:40:55.479165+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:30:26.847831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":360,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":423,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T20:40:55.472520+0000","last_change":"2023-09-23T20:38:02.599068+0000","last_active":"2023-09-23T20:40:55.472520+0000","last_peered":"2023-09-23T20:40:55.472520+0000","last_clean":"2023-09-23T20:40:55.472520+0000","last_became_active":"2023-09-23T20:38:02.597602+0000","last_became_peered":"2023-09-23T20:38:02.597602+0000","last_unstale":"2023-09-23T20:40:55.472520+0000","last_undegraded":"2023-09-23T20:40:55.472520+0000","last_fullsized":"2023-09-23T20:40:55.472520+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:34:06.337029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"90'161","reported_seq":578,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T20:40:55.472737+0000","last_change":"2023-09-23T20:36:02.548387+0000","last_active":"2023-09-23T20:40:55.472737+0000","last_peered":"2023-09-23T20:40:55.472737+0000","last_clean":"2023-09-23T20:40:55.472737+0000","last_became_active":"2023-09-23T20:35:26.644787+0000","last_became_peered":"2023-09-23T20:35:26.644787+0000","last_unstale":"2023-09-23T20:40:55.472737+0000","last_undegraded":"2023-09-23T20:40:55.472737+0000","last_fullsized":"2023-09-23T20:40:55.472737+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:02.548285+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:02.548285+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:37:15.216553+0000","scrub_duration":0.0015514459999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":200,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-09-23T20:40:54.238497+0000","last_change":"2023-09-23T20:38:02.560646+0000","last_active":"2023-09-23T20:40:54.238497+0000","last_peered":"2023-09-23T20:40:54.238497+0000","last_clean":"2023-09-23T20:40:54.238497+0000","last_became_active":"2023-09-23T20:38:02.539814+0000","last_became_peered":"2023-09-23T20:38:02.539814+0000","last_unstale":"2023-09-23T20:40:54.238497+0000","last_undegraded":"2023-09-23T20:40:54.238497+0000","last_fullsized":"2023-09-23T20:40:54.238497+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:00:09.435023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":180,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-09-23T20:40:54.243879+0000","last_change":"2023-09-23T20:38:02.579222+0000","last_active":"2023-09-23T20:40:54.243879+0000","last_peered":"2023-09-23T20:40:54.243879+0000","last_clean":"2023-09-23T20:40:54.243879+0000","last_became_active":"2023-09-23T20:38:02.578813+0000","last_became_peered":"2023-09-23T20:38:02.578813+0000","last_unstale":"2023-09-23T20:40:54.243879+0000","last_undegraded":"2023-09-23T20:40:54.243879+0000","last_fullsized":"2023-09-23T20:40:54.243879+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:57:31.080027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"90'161","reported_seq":719,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-09-23T20:40:53.905914+0000","last_change":"2023-09-23T20:40:22.427437+0000","last_active":"2023-09-23T20:40:53.905914+0000","last_peered":"2023-09-23T20:40:53.905914+0000","last_clean":"2023-09-23T20:40:53.905914+0000","last_became_active":"2023-09-23T20:40:22.427090+0000","last_became_peered":"2023-09-23T20:40:22.427090+0000","last_unstale":"2023-09-23T20:40:53.905914+0000","last_undegraded":"2023-09-23T20:40:53.905914+0000","last_fullsized":"2023-09-23T20:40:53.905914+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:43.578666+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:43.578666+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:08:43.995672+0000","scrub_duration":0.0021890379999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":480,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T20:40:55.478063+0000","last_change":"2023-09-23T20:32:33.242943+0000","last_active":"2023-09-23T20:40:55.478063+0000","last_peered":"2023-09-23T20:40:55.478063+0000","last_clean":"2023-09-23T20:40:55.478063+0000","last_became_active":"2023-09-23T20:32:33.225468+0000","last_became_peered":"2023-09-23T20:32:33.225468+0000","last_unstale":"2023-09-23T20:40:55.478063+0000","last_undegraded":"2023-09-23T20:40:55.478063+0000","last_fullsized":"2023-09-23T20:40:55.478063+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:40:47.867573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":189,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T20:40:55.618088+0000","last_change":"2023-09-23T20:37:43.571836+0000","last_active":"2023-09-23T20:40:55.618088+0000","last_peered":"2023-09-23T20:40:55.618088+0000","last_clean":"2023-09-23T20:40:55.618088+0000","last_became_active":"2023-09-23T20:37:43.571511+0000","last_became_peered":"2023-09-23T20:37:43.571511+0000","last_unstale":"2023-09-23T20:40:55.618088+0000","last_undegraded":"2023-09-23T20:40:55.618088+0000","last_fullsized":"2023-09-23T20:40:55.618088+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:59:25.507917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"90'161","reported_seq":635,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-09-23T20:40:54.222029+0000","last_change":"2023-09-23T20:40:22.427706+0000","last_active":"2023-09-23T20:40:54.222029+0000","last_peered":"2023-09-23T20:40:54.222029+0000","last_clean":"2023-09-23T20:40:54.222029+0000","last_became_active":"2023-09-23T20:40:22.427114+0000","last_became_peered":"2023-09-23T20:40:22.427114+0000","last_unstale":"2023-09-23T20:40:54.222029+0000","last_undegraded":"2023-09-23T20:40:54.222029+0000","last_fullsized":"2023-09-23T20:40:54.222029+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:49.726139+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:49.726139+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0096333110000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":420,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-09-23T20:40:53.905639+0000","last_change":"2023-09-23T20:25:02.953993+0000","last_active":"2023-09-23T20:40:53.905639+0000","last_peered":"2023-09-23T20:40:53.905639+0000","last_clean":"2023-09-23T20:40:53.905639+0000","last_became_active":"2023-09-23T20:25:02.953672+0000","last_became_peered":"2023-09-23T20:25:02.953672+0000","last_unstale":"2023-09-23T20:40:53.905639+0000","last_undegraded":"2023-09-23T20:40:53.905639+0000","last_fullsized":"2023-09-23T20:40:53.905639+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:26:08.603577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":22,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-09-23T20:40:54.221078+0000","last_change":"2023-09-23T20:40:19.561512+0000","last_active":"2023-09-23T20:40:54.221078+0000","last_peered":"2023-09-23T20:40:54.221078+0000","last_clean":"2023-09-23T20:40:54.221078+0000","last_became_active":"2023-09-23T20:40:19.561123+0000","last_became_peered":"2023-09-23T20:40:19.561123+0000","last_unstale":"2023-09-23T20:40:54.221078+0000","last_undegraded":"2023-09-23T20:40:54.221078+0000","last_fullsized":"2023-09-23T20:40:54.221078+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:50:02.093978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"90'161","reported_seq":739,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T20:40:55.472467+0000","last_change":"2023-09-23T20:35:02.238923+0000","last_active":"2023-09-23T20:40:55.472467+0000","last_peered":"2023-09-23T20:40:55.472467+0000","last_clean":"2023-09-23T20:40:55.472467+0000","last_became_active":"2023-09-23T20:35:02.238293+0000","last_became_peered":"2023-09-23T20:35:02.238293+0000","last_unstale":"2023-09-23T20:40:55.472467+0000","last_undegraded":"2023-09-23T20:40:55.472467+0000","last_fullsized":"2023-09-23T20:40:55.472467+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:42.544308+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:42.544308+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:51:22.281932+0000","scrub_duration":0.0029958179999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":457,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T20:40:55.478107+0000","last_change":"2023-09-23T20:38:02.539774+0000","last_active":"2023-09-23T20:40:55.478107+0000","last_peered":"2023-09-23T20:40:55.478107+0000","last_clean":"2023-09-23T20:40:55.478107+0000","last_became_active":"2023-09-23T20:38:02.539436+0000","last_became_peered":"2023-09-23T20:38:02.539436+0000","last_unstale":"2023-09-23T20:40:55.478107+0000","last_undegraded":"2023-09-23T20:40:55.478107+0000","last_fullsized":"2023-09-23T20:40:55.478107+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:50:57.375803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":430,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-09-23T20:40:53.908263+0000","last_change":"2023-09-23T20:25:02.957010+0000","last_active":"2023-09-23T20:40:53.908263+0000","last_peered":"2023-09-23T20:40:53.908263+0000","last_clean":"2023-09-23T20:40:53.908263+0000","last_became_active":"2023-09-23T20:25:02.956267+0000","last_became_peered":"2023-09-23T20:25:02.956267+0000","last_unstale":"2023-09-23T20:40:53.908263+0000","last_undegraded":"2023-09-23T20:40:53.908263+0000","last_fullsized":"2023-09-23T20:40:53.908263+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:10:55.299422+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"90'161","reported_seq":558,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-09-23T20:40:54.220516+0000","last_change":"2023-09-23T20:40:22.431074+0000","last_active":"2023-09-23T20:40:54.220516+0000","last_peered":"2023-09-23T20:40:54.220516+0000","last_clean":"2023-09-23T20:40:54.220516+0000","last_became_active":"2023-09-23T20:40:22.430717+0000","last_became_peered":"2023-09-23T20:40:22.430717+0000","last_unstale":"2023-09-23T20:40:54.220516+0000","last_undegraded":"2023-09-23T20:40:54.220516+0000","last_fullsized":"2023-09-23T20:40:54.220516+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:53.706934+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:53.706934+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":89,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-09-23T20:40:53.460214+0000","last_change":"2023-09-23T20:40:19.564378+0000","last_active":"2023-09-23T20:40:53.460214+0000","last_peered":"2023-09-23T20:40:53.460214+0000","last_clean":"2023-09-23T20:40:53.460214+0000","last_became_active":"2023-09-23T20:40:19.532991+0000","last_became_peered":"2023-09-23T20:40:19.532991+0000","last_unstale":"2023-09-23T20:40:53.460214+0000","last_undegraded":"2023-09-23T20:40:53.460214+0000","last_fullsized":"2023-09-23T20:40:53.460214+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:22:42.832115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":247,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T20:40:55.618185+0000","last_change":"2023-09-23T20:37:43.571083+0000","last_active":"2023-09-23T20:40:55.618185+0000","last_peered":"2023-09-23T20:40:55.618185+0000","last_clean":"2023-09-23T20:40:55.618185+0000","last_became_active":"2023-09-23T20:37:43.570306+0000","last_became_peered":"2023-09-23T20:37:43.570306+0000","last_unstale":"2023-09-23T20:40:55.618185+0000","last_undegraded":"2023-09-23T20:40:55.618185+0000","last_fullsized":"2023-09-23T20:40:55.618185+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:42:02.396718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.d","version":"90'161","reported_seq":746,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T20:40:55.478917+0000","last_change":"2023-09-23T20:34:59.496438+0000","last_active":"2023-09-23T20:40:55.478917+0000","last_peered":"2023-09-23T20:40:55.478917+0000","last_clean":"2023-09-23T20:40:55.478917+0000","last_became_active":"2023-09-23T20:33:38.970527+0000","last_became_peered":"2023-09-23T20:33:38.970527+0000","last_unstale":"2023-09-23T20:40:55.478917+0000","last_undegraded":"2023-09-23T20:40:55.478917+0000","last_fullsized":"2023-09-23T20:40:55.478917+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:34:59.496375+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:34:59.496375+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:46:18.244734+0000","scrub_duration":0.0017843189999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":457,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T20:40:55.478415+0000","last_change":"2023-09-23T20:32:33.283752+0000","last_active":"2023-09-23T20:40:55.478415+0000","last_peered":"2023-09-23T20:40:55.478415+0000","last_clean":"2023-09-23T20:40:55.478415+0000","last_became_active":"2023-09-23T20:32:33.230503+0000","last_became_peered":"2023-09-23T20:32:33.230503+0000","last_unstale":"2023-09-23T20:40:55.478415+0000","last_undegraded":"2023-09-23T20:40:55.478415+0000","last_fullsized":"2023-09-23T20:40:55.478415+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:29.084410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":22,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-09-23T20:40:54.220896+0000","last_change":"2023-09-23T20:40:19.545668+0000","last_active":"2023-09-23T20:40:54.220896+0000","last_peered":"2023-09-23T20:40:54.220896+0000","last_clean":"2023-09-23T20:40:54.220896+0000","last_became_active":"2023-09-23T20:40:19.543857+0000","last_became_peered":"2023-09-23T20:40:19.543857+0000","last_unstale":"2023-09-23T20:40:54.220896+0000","last_undegraded":"2023-09-23T20:40:54.220896+0000","last_fullsized":"2023-09-23T20:40:54.220896+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:27:09.470605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"90'161","reported_seq":701,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T20:40:55.618678+0000","last_change":"2023-09-23T20:38:28.442536+0000","last_active":"2023-09-23T20:40:55.618678+0000","last_peered":"2023-09-23T20:40:55.618678+0000","last_clean":"2023-09-23T20:40:55.618678+0000","last_became_active":"2023-09-23T20:38:28.442229+0000","last_became_peered":"2023-09-23T20:38:28.442229+0000","last_unstale":"2023-09-23T20:40:55.618678+0000","last_undegraded":"2023-09-23T20:40:55.618678+0000","last_fullsized":"2023-09-23T20:40:55.618678+0000","mapping_epoch":411,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":412,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:05.485751+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:05.485751+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.002180043,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163840,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":492,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T20:40:55.478475+0000","last_change":"2023-09-23T20:34:59.987981+0000","last_active":"2023-09-23T20:40:55.478475+0000","last_peered":"2023-09-23T20:40:55.478475+0000","last_clean":"2023-09-23T20:40:55.478475+0000","last_became_active":"2023-09-23T20:34:59.985368+0000","last_became_peered":"2023-09-23T20:34:59.985368+0000","last_unstale":"2023-09-23T20:40:55.478475+0000","last_undegraded":"2023-09-23T20:40:55.478475+0000","last_fullsized":"2023-09-23T20:40:55.478475+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:12:43.082493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":150,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-09-23T20:40:54.238152+0000","last_change":"2023-09-23T20:32:33.232142+0000","last_active":"2023-09-23T20:40:54.238152+0000","last_peered":"2023-09-23T20:40:54.238152+0000","last_clean":"2023-09-23T20:40:54.238152+0000","last_became_active":"2023-09-23T20:32:33.231683+0000","last_became_peered":"2023-09-23T20:32:33.231683+0000","last_unstale":"2023-09-23T20:40:54.238152+0000","last_undegraded":"2023-09-23T20:40:54.238152+0000","last_fullsized":"2023-09-23T20:40:54.238152+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:02:58.336348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"90'161","reported_seq":609,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-09-23T20:40:53.462597+0000","last_change":"2023-09-23T20:38:40.491902+0000","last_active":"2023-09-23T20:40:53.462597+0000","last_peered":"2023-09-23T20:40:53.462597+0000","last_clean":"2023-09-23T20:40:53.462597+0000","last_became_active":"2023-09-23T20:38:40.491563+0000","last_became_peered":"2023-09-23T20:38:40.491563+0000","last_unstale":"2023-09-23T20:40:53.462597+0000","last_undegraded":"2023-09-23T20:40:53.462597+0000","last_fullsized":"2023-09-23T20:40:53.462597+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:46.217489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:46.217489+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:39:25.008796+0000","scrub_duration":0.001456429,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":482,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T20:40:55.479006+0000","last_change":"2023-09-23T20:40:19.677655+0000","last_active":"2023-09-23T20:40:55.479006+0000","last_peered":"2023-09-23T20:40:55.479006+0000","last_clean":"2023-09-23T20:40:55.479006+0000","last_became_active":"2023-09-23T20:40:19.551189+0000","last_became_peered":"2023-09-23T20:40:19.551189+0000","last_unstale":"2023-09-23T20:40:55.479006+0000","last_undegraded":"2023-09-23T20:40:55.479006+0000","last_fullsized":"2023-09-23T20:40:55.479006+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:11:01.559576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":59,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-09-23T20:40:53.460498+0000","last_change":"2023-09-23T20:38:02.590613+0000","last_active":"2023-09-23T20:40:53.460498+0000","last_peered":"2023-09-23T20:40:53.460498+0000","last_clean":"2023-09-23T20:40:53.460498+0000","last_became_active":"2023-09-23T20:38:02.590284+0000","last_became_peered":"2023-09-23T20:38:02.590284+0000","last_unstale":"2023-09-23T20:40:53.460498+0000","last_undegraded":"2023-09-23T20:40:53.460498+0000","last_fullsized":"2023-09-23T20:40:53.460498+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:40:07.825054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.e","version":"90'161","reported_seq":553,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-09-23T20:40:54.247023+0000","last_change":"2023-09-23T20:36:00.322453+0000","last_active":"2023-09-23T20:40:54.247023+0000","last_peered":"2023-09-23T20:40:54.247023+0000","last_clean":"2023-09-23T20:40:54.247023+0000","last_became_active":"2023-09-23T20:35:02.233536+0000","last_became_peered":"2023-09-23T20:35:02.233536+0000","last_unstale":"2023-09-23T20:40:54.247023+0000","last_undegraded":"2023-09-23T20:40:54.247023+0000","last_fullsized":"2023-09-23T20:40:54.247023+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:00.322381+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:00.322381+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:02:15.489069+0000","scrub_duration":0.0017321859999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":47,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-09-23T20:40:54.218345+0000","last_change":"2023-09-23T20:38:37.456683+0000","last_active":"2023-09-23T20:40:54.218345+0000","last_peered":"2023-09-23T20:40:54.218345+0000","last_clean":"2023-09-23T20:40:54.218345+0000","last_became_active":"2023-09-23T20:38:37.456289+0000","last_became_peered":"2023-09-23T20:38:37.456289+0000","last_unstale":"2023-09-23T20:40:54.218345+0000","last_undegraded":"2023-09-23T20:40:54.218345+0000","last_fullsized":"2023-09-23T20:40:54.218345+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:03:10.507333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":439,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-09-23T20:40:53.906148+0000","last_change":"2023-09-23T20:32:33.236122+0000","last_active":"2023-09-23T20:40:53.906148+0000","last_peered":"2023-09-23T20:40:53.906148+0000","last_clean":"2023-09-23T20:40:53.906148+0000","last_became_active":"2023-09-23T20:32:33.235384+0000","last_became_peered":"2023-09-23T20:32:33.235384+0000","last_unstale":"2023-09-23T20:40:53.906148+0000","last_undegraded":"2023-09-23T20:40:53.906148+0000","last_fullsized":"2023-09-23T20:40:53.906148+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:56.843733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":859,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T20:40:55.477937+0000","last_change":"2023-09-23T20:35:02.523308+0000","last_active":"2023-09-23T20:40:55.477937+0000","last_peered":"2023-09-23T20:40:55.477937+0000","last_clean":"2023-09-23T20:40:55.477937+0000","last_became_active":"2023-09-23T20:33:38.979989+0000","last_became_peered":"2023-09-23T20:33:38.979989+0000","last_unstale":"2023-09-23T20:40:55.477937+0000","last_undegraded":"2023-09-23T20:40:55.477937+0000","last_fullsized":"2023-09-23T20:40:55.477937+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:35:02.523203+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:35:02.523203+0000","last_clean_scrub_stamp":"2023-09-23T20:35:02.523203+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:02:41.472958+0000","scrub_duration":0.0085927989999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":273,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T20:40:55.618070+0000","last_change":"2023-09-23T20:37:43.583731+0000","last_active":"2023-09-23T20:40:55.618070+0000","last_peered":"2023-09-23T20:40:55.618070+0000","last_clean":"2023-09-23T20:40:55.618070+0000","last_became_active":"2023-09-23T20:37:43.583066+0000","last_became_peered":"2023-09-23T20:37:43.583066+0000","last_unstale":"2023-09-23T20:40:55.618070+0000","last_undegraded":"2023-09-23T20:40:55.618070+0000","last_fullsized":"2023-09-23T20:40:55.618070+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:03:45.211900+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":170,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-09-23T20:40:54.238417+0000","last_change":"2023-09-23T20:38:02.579814+0000","last_active":"2023-09-23T20:40:54.238417+0000","last_peered":"2023-09-23T20:40:54.238417+0000","last_clean":"2023-09-23T20:40:54.238417+0000","last_became_active":"2023-09-23T20:38:02.579253+0000","last_became_peered":"2023-09-23T20:38:02.579253+0000","last_unstale":"2023-09-23T20:40:54.238417+0000","last_undegraded":"2023-09-23T20:40:54.238417+0000","last_fullsized":"2023-09-23T20:40:54.238417+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:05:07.298198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":554,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-09-23T20:40:54.220809+0000","last_change":"2023-09-23T20:40:22.433408+0000","last_active":"2023-09-23T20:40:54.220809+0000","last_peered":"2023-09-23T20:40:54.220809+0000","last_clean":"2023-09-23T20:40:54.220809+0000","last_became_active":"2023-09-23T20:40:22.433023+0000","last_became_peered":"2023-09-23T20:40:22.433023+0000","last_unstale":"2023-09-23T20:40:54.220809+0000","last_undegraded":"2023-09-23T20:40:54.220809+0000","last_fullsized":"2023-09-23T20:40:54.220809+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:35:00.496197+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:35:00.496197+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":59,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-09-23T20:40:53.460580+0000","last_change":"2023-09-23T20:38:02.578703+0000","last_active":"2023-09-23T20:40:53.460580+0000","last_peered":"2023-09-23T20:40:53.460580+0000","last_clean":"2023-09-23T20:40:53.460580+0000","last_became_active":"2023-09-23T20:38:02.578414+0000","last_became_peered":"2023-09-23T20:38:02.578414+0000","last_unstale":"2023-09-23T20:40:53.460580+0000","last_undegraded":"2023-09-23T20:40:53.460580+0000","last_fullsized":"2023-09-23T20:40:53.460580+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:43:59.779295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":119,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-09-23T20:40:54.246110+0000","last_change":"2023-09-23T20:37:31.296606+0000","last_active":"2023-09-23T20:40:54.246110+0000","last_peered":"2023-09-23T20:40:54.246110+0000","last_clean":"2023-09-23T20:40:54.246110+0000","last_became_active":"2023-09-23T20:37:31.295924+0000","last_became_peered":"2023-09-23T20:37:31.295924+0000","last_unstale":"2023-09-23T20:40:54.246110+0000","last_undegraded":"2023-09-23T20:40:54.246110+0000","last_fullsized":"2023-09-23T20:40:54.246110+0000","mapping_epoch":391,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":392,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:27:50.948510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"90'161","reported_seq":755,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T20:40:55.472531+0000","last_change":"2023-09-23T20:40:22.445808+0000","last_active":"2023-09-23T20:40:55.472531+0000","last_peered":"2023-09-23T20:40:55.472531+0000","last_clean":"2023-09-23T20:40:55.472531+0000","last_became_active":"2023-09-23T20:40:22.445456+0000","last_became_peered":"2023-09-23T20:40:22.445456+0000","last_unstale":"2023-09-23T20:40:55.472531+0000","last_undegraded":"2023-09-23T20:40:55.472531+0000","last_fullsized":"2023-09-23T20:40:55.472531+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:05.595042+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:05.595042+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:39:23.678154+0000","scrub_duration":0.007271675,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":109,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-09-23T20:40:54.237434+0000","last_change":"2023-09-23T20:34:59.983122+0000","last_active":"2023-09-23T20:40:54.237434+0000","last_peered":"2023-09-23T20:40:54.237434+0000","last_clean":"2023-09-23T20:40:54.237434+0000","last_became_active":"2023-09-23T20:34:59.982809+0000","last_became_peered":"2023-09-23T20:34:59.982809+0000","last_unstale":"2023-09-23T20:40:54.237434+0000","last_undegraded":"2023-09-23T20:40:54.237434+0000","last_fullsized":"2023-09-23T20:40:54.237434+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:49:40.839723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":170,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-09-23T20:40:54.248986+0000","last_change":"2023-09-23T20:38:02.543866+0000","last_active":"2023-09-23T20:40:54.248986+0000","last_peered":"2023-09-23T20:40:54.248986+0000","last_clean":"2023-09-23T20:40:54.248986+0000","last_became_active":"2023-09-23T20:38:02.543563+0000","last_became_peered":"2023-09-23T20:38:02.543563+0000","last_unstale":"2023-09-23T20:40:54.248986+0000","last_undegraded":"2023-09-23T20:40:54.248986+0000","last_fullsized":"2023-09-23T20:40:54.248986+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:33.050536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":557,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-09-23T20:40:54.218748+0000","last_change":"2023-09-23T20:40:01.179672+0000","last_active":"2023-09-23T20:40:54.218748+0000","last_peered":"2023-09-23T20:40:54.218748+0000","last_clean":"2023-09-23T20:40:54.218748+0000","last_became_active":"2023-09-23T20:38:40.494049+0000","last_became_peered":"2023-09-23T20:38:40.494049+0000","last_unstale":"2023-09-23T20:40:54.218748+0000","last_undegraded":"2023-09-23T20:40:54.218748+0000","last_fullsized":"2023-09-23T20:40:54.218748+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:34:59.481446+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:34:59.481446+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26834,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":247,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T20:40:55.618058+0000","last_change":"2023-09-23T20:37:43.570610+0000","last_active":"2023-09-23T20:40:55.618058+0000","last_peered":"2023-09-23T20:40:55.618058+0000","last_clean":"2023-09-23T20:40:55.618058+0000","last_became_active":"2023-09-23T20:37:43.570216+0000","last_became_peered":"2023-09-23T20:37:43.570216+0000","last_unstale":"2023-09-23T20:40:55.618058+0000","last_undegraded":"2023-09-23T20:40:55.618058+0000","last_fullsized":"2023-09-23T20:40:55.618058+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:39:26.555329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":277,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T20:40:55.472870+0000","last_change":"2023-09-23T20:32:33.230582+0000","last_active":"2023-09-23T20:40:55.472870+0000","last_peered":"2023-09-23T20:40:55.472870+0000","last_clean":"2023-09-23T20:40:55.472870+0000","last_became_active":"2023-09-23T20:32:33.230257+0000","last_became_peered":"2023-09-23T20:32:33.230257+0000","last_unstale":"2023-09-23T20:40:55.472870+0000","last_undegraded":"2023-09-23T20:40:55.472870+0000","last_fullsized":"2023-09-23T20:40:55.472870+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:11:45.333096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":632,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-09-23T20:40:53.462527+0000","last_change":"2023-09-23T20:38:05.622597+0000","last_active":"2023-09-23T20:40:53.462527+0000","last_peered":"2023-09-23T20:40:53.462527+0000","last_clean":"2023-09-23T20:40:53.462527+0000","last_became_active":"2023-09-23T20:38:05.622261+0000","last_became_peered":"2023-09-23T20:38:05.622261+0000","last_unstale":"2023-09-23T20:40:53.462527+0000","last_undegraded":"2023-09-23T20:40:53.462527+0000","last_fullsized":"2023-09-23T20:40:53.462527+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:35:04.101972+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:35:04.101972+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"86'119","reported_seq":762,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T20:40:55.472657+0000","last_change":"2023-09-23T20:29:45.521257+0000","last_active":"2023-09-23T20:40:55.472657+0000","last_peered":"2023-09-23T20:40:55.472657+0000","last_clean":"2023-09-23T20:40:55.472657+0000","last_became_active":"2023-09-23T20:29:45.520546+0000","last_became_peered":"2023-09-23T20:29:45.520546+0000","last_unstale":"2023-09-23T20:40:55.472657+0000","last_undegraded":"2023-09-23T20:40:55.472657+0000","last_fullsized":"2023-09-23T20:40:55.472657+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":119,"log_dups_size":0,"ondisk_log_size":119,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:58:56.085777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032705499999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":523,"num_read_kb":3503,"num_write":136,"num_write_kb":3128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":22,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-09-23T20:40:54.221002+0000","last_change":"2023-09-23T20:40:19.557009+0000","last_active":"2023-09-23T20:40:54.221002+0000","last_peered":"2023-09-23T20:40:54.221002+0000","last_clean":"2023-09-23T20:40:54.221002+0000","last_became_active":"2023-09-23T20:40:19.550473+0000","last_became_peered":"2023-09-23T20:40:19.550473+0000","last_unstale":"2023-09-23T20:40:54.221002+0000","last_undegraded":"2023-09-23T20:40:54.221002+0000","last_fullsized":"2023-09-23T20:40:54.221002+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:35:01.140258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":409,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T20:40:55.472238+0000","last_change":"2023-09-23T20:40:19.530942+0000","last_active":"2023-09-23T20:40:55.472238+0000","last_peered":"2023-09-23T20:40:55.472238+0000","last_clean":"2023-09-23T20:40:55.472238+0000","last_became_active":"2023-09-23T20:40:19.530617+0000","last_became_peered":"2023-09-23T20:40:19.530617+0000","last_unstale":"2023-09-23T20:40:55.472238+0000","last_undegraded":"2023-09-23T20:40:55.472238+0000","last_fullsized":"2023-09-23T20:40:55.472238+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:24:09.231934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":526,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-09-23T20:40:54.218595+0000","last_change":"2023-09-23T20:40:22.445201+0000","last_active":"2023-09-23T20:40:54.218595+0000","last_peered":"2023-09-23T20:40:54.218595+0000","last_clean":"2023-09-23T20:40:54.218595+0000","last_became_active":"2023-09-23T20:40:22.444703+0000","last_became_peered":"2023-09-23T20:40:22.444703+0000","last_unstale":"2023-09-23T20:40:54.218595+0000","last_undegraded":"2023-09-23T20:40:54.218595+0000","last_fullsized":"2023-09-23T20:40:54.218595+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:45.785450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:45.785450+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"86'106","reported_seq":1172,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-09-23T20:40:54.218512+0000","last_change":"2023-09-23T20:38:37.476512+0000","last_active":"2023-09-23T20:40:54.218512+0000","last_peered":"2023-09-23T20:40:54.218512+0000","last_clean":"2023-09-23T20:40:54.218512+0000","last_became_active":"2023-09-23T20:38:37.475980+0000","last_became_peered":"2023-09-23T20:38:37.475980+0000","last_unstale":"2023-09-23T20:40:54.218512+0000","last_undegraded":"2023-09-23T20:40:54.218512+0000","last_fullsized":"2023-09-23T20:40:54.218512+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":106,"log_dups_size":0,"ondisk_log_size":106,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:58:45.508831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00099302599999999993,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1094,"num_read_kb":1678,"num_write":100,"num_write_kb":404,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":170,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-09-23T20:40:54.243802+0000","last_change":"2023-09-23T20:38:02.552008+0000","last_active":"2023-09-23T20:40:54.243802+0000","last_peered":"2023-09-23T20:40:54.243802+0000","last_clean":"2023-09-23T20:40:54.243802+0000","last_became_active":"2023-09-23T20:38:02.551684+0000","last_became_peered":"2023-09-23T20:38:02.551684+0000","last_unstale":"2023-09-23T20:40:54.243802+0000","last_undegraded":"2023-09-23T20:40:54.243802+0000","last_fullsized":"2023-09-23T20:40:54.243802+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:40:37.147538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":415,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T20:40:55.472587+0000","last_change":"2023-09-23T20:29:45.533406+0000","last_active":"2023-09-23T20:40:55.472587+0000","last_peered":"2023-09-23T20:40:55.472587+0000","last_clean":"2023-09-23T20:40:55.472587+0000","last_became_active":"2023-09-23T20:29:45.526001+0000","last_became_peered":"2023-09-23T20:29:45.526001+0000","last_unstale":"2023-09-23T20:40:55.472587+0000","last_undegraded":"2023-09-23T20:40:55.472587+0000","last_fullsized":"2023-09-23T20:40:55.472587+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:43.787251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":926,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T20:40:55.478081+0000","last_change":"2023-09-23T20:35:02.260708+0000","last_active":"2023-09-23T20:40:55.478081+0000","last_peered":"2023-09-23T20:40:55.478081+0000","last_clean":"2023-09-23T20:40:55.478081+0000","last_became_active":"2023-09-23T20:35:02.260348+0000","last_became_peered":"2023-09-23T20:35:02.260348+0000","last_unstale":"2023-09-23T20:40:55.478081+0000","last_undegraded":"2023-09-23T20:40:55.478081+0000","last_fullsized":"2023-09-23T20:40:55.478081+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:43.390908+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:43.390908+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:09:14.144070+0000","scrub_duration":0.0032721,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"86'130","reported_seq":770,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T20:40:55.477894+0000","last_change":"2023-09-23T20:25:39.637458+0000","last_active":"2023-09-23T20:40:55.477894+0000","last_peered":"2023-09-23T20:40:55.477894+0000","last_clean":"2023-09-23T20:40:55.477894+0000","last_became_active":"2023-09-23T20:25:39.636773+0000","last_became_peered":"2023-09-23T20:25:39.636773+0000","last_unstale":"2023-09-23T20:40:55.477894+0000","last_undegraded":"2023-09-23T20:40:55.477894+0000","last_fullsized":"2023-09-23T20:40:55.477894+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:00:32.810564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011300049999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":512,"num_read_kb":1820,"num_write":133,"num_write_kb":1347,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":423,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T20:40:55.472283+0000","last_change":"2023-09-23T20:38:02.591335+0000","last_active":"2023-09-23T20:40:55.472283+0000","last_peered":"2023-09-23T20:40:55.472283+0000","last_clean":"2023-09-23T20:40:55.472283+0000","last_became_active":"2023-09-23T20:38:02.591015+0000","last_became_peered":"2023-09-23T20:38:02.591015+0000","last_unstale":"2023-09-23T20:40:55.472283+0000","last_undegraded":"2023-09-23T20:40:55.472283+0000","last_fullsized":"2023-09-23T20:40:55.472283+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:55:32.272195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":415,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T20:40:55.472785+0000","last_change":"2023-09-23T20:40:19.544333+0000","last_active":"2023-09-23T20:40:55.472785+0000","last_peered":"2023-09-23T20:40:55.472785+0000","last_clean":"2023-09-23T20:40:55.472785+0000","last_became_active":"2023-09-23T20:40:19.543987+0000","last_became_peered":"2023-09-23T20:40:19.543987+0000","last_unstale":"2023-09-23T20:40:55.472785+0000","last_undegraded":"2023-09-23T20:40:55.472785+0000","last_fullsized":"2023-09-23T20:40:55.472785+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:12:59.203296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"86'194","reported_seq":805,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-09-23T20:40:54.221833+0000","last_change":"2023-09-23T20:40:51.458056+0000","last_active":"2023-09-23T20:40:54.221833+0000","last_peered":"2023-09-23T20:40:54.221833+0000","last_clean":"2023-09-23T20:40:54.221833+0000","last_became_active":"2023-09-23T20:40:51.457465+0000","last_became_peered":"2023-09-23T20:40:51.457465+0000","last_unstale":"2023-09-23T20:40:54.221833+0000","last_undegraded":"2023-09-23T20:40:54.221833+0000","last_fullsized":"2023-09-23T20:40:54.221833+0000","mapping_epoch":450,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":451,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":194,"log_dups_size":0,"ondisk_log_size":194,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:18:36.482553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00037995999999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1024,"num_read_kb":1200,"num_write":173,"num_write_kb":1276,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":916,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T20:40:55.478706+0000","last_change":"2023-09-23T20:38:05.602508+0000","last_active":"2023-09-23T20:40:55.478706+0000","last_peered":"2023-09-23T20:40:55.478706+0000","last_clean":"2023-09-23T20:40:55.478706+0000","last_became_active":"2023-09-23T20:38:05.602129+0000","last_became_peered":"2023-09-23T20:38:05.602129+0000","last_unstale":"2023-09-23T20:40:55.478706+0000","last_undegraded":"2023-09-23T20:40:55.478706+0000","last_fullsized":"2023-09-23T20:40:55.478706+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:45.343742+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:45.343742+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:48.910343+0000","scrub_duration":0.001475802,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":22,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-09-23T20:40:54.218670+0000","last_change":"2023-09-23T20:40:19.542142+0000","last_active":"2023-09-23T20:40:54.218670+0000","last_peered":"2023-09-23T20:40:54.218670+0000","last_clean":"2023-09-23T20:40:54.218670+0000","last_became_active":"2023-09-23T20:40:19.541870+0000","last_became_peered":"2023-09-23T20:40:19.541870+0000","last_unstale":"2023-09-23T20:40:54.218670+0000","last_undegraded":"2023-09-23T20:40:54.218670+0000","last_fullsized":"2023-09-23T20:40:54.218670+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:01.518149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":59,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-09-23T20:40:53.464861+0000","last_change":"2023-09-23T20:38:02.573200+0000","last_active":"2023-09-23T20:40:53.464861+0000","last_peered":"2023-09-23T20:40:53.464861+0000","last_clean":"2023-09-23T20:40:53.464861+0000","last_became_active":"2023-09-23T20:38:02.572945+0000","last_became_peered":"2023-09-23T20:38:02.572945+0000","last_unstale":"2023-09-23T20:40:53.464861+0000","last_undegraded":"2023-09-23T20:40:53.464861+0000","last_fullsized":"2023-09-23T20:40:53.464861+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:57.246533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":784,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T20:40:55.472423+0000","last_change":"2023-09-23T20:38:05.615711+0000","last_active":"2023-09-23T20:40:55.472423+0000","last_peered":"2023-09-23T20:40:55.472423+0000","last_clean":"2023-09-23T20:40:55.472423+0000","last_became_active":"2023-09-23T20:38:05.615390+0000","last_became_peered":"2023-09-23T20:38:05.615390+0000","last_unstale":"2023-09-23T20:40:55.472423+0000","last_undegraded":"2023-09-23T20:40:55.472423+0000","last_fullsized":"2023-09-23T20:40:55.472423+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:43.582146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:43.582146+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:45:11.917871+0000","scrub_duration":0.0016219730000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"86'80","reported_seq":706,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T20:40:55.478709+0000","last_change":"2023-09-23T20:29:45.536796+0000","last_active":"2023-09-23T20:40:55.478709+0000","last_peered":"2023-09-23T20:40:55.478709+0000","last_clean":"2023-09-23T20:40:55.478709+0000","last_became_active":"2023-09-23T20:29:45.536439+0000","last_became_peered":"2023-09-23T20:29:45.536439+0000","last_unstale":"2023-09-23T20:40:55.478709+0000","last_undegraded":"2023-09-23T20:40:55.478709+0000","last_fullsized":"2023-09-23T20:40:55.478709+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":80,"log_dups_size":0,"ondisk_log_size":80,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:30:57.793132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035115,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":395,"num_read_kb":1370,"num_write":66,"num_write_kb":1061,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":458,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T20:40:55.921182+0000","last_change":"2023-09-23T20:34:59.984286+0000","last_active":"2023-09-23T20:40:55.921182+0000","last_peered":"2023-09-23T20:40:55.921182+0000","last_clean":"2023-09-23T20:40:55.921182+0000","last_became_active":"2023-09-23T20:34:59.983950+0000","last_became_peered":"2023-09-23T20:34:59.983950+0000","last_unstale":"2023-09-23T20:40:55.921182+0000","last_undegraded":"2023-09-23T20:40:55.921182+0000","last_fullsized":"2023-09-23T20:40:55.921182+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:31:53.557035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":452,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-09-23T20:40:53.908001+0000","last_change":"2023-09-23T20:29:45.690815+0000","last_active":"2023-09-23T20:40:53.908001+0000","last_peered":"2023-09-23T20:40:53.908001+0000","last_clean":"2023-09-23T20:40:53.908001+0000","last_became_active":"2023-09-23T20:29:45.690478+0000","last_became_peered":"2023-09-23T20:29:45.690478+0000","last_unstale":"2023-09-23T20:40:53.908001+0000","last_undegraded":"2023-09-23T20:40:53.908001+0000","last_fullsized":"2023-09-23T20:40:53.908001+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:38.288193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":555,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-09-23T20:40:53.462434+0000","last_change":"2023-09-23T20:38:05.611181+0000","last_active":"2023-09-23T20:40:53.462434+0000","last_peered":"2023-09-23T20:40:53.462434+0000","last_clean":"2023-09-23T20:40:53.462434+0000","last_became_active":"2023-09-23T20:38:05.610881+0000","last_became_peered":"2023-09-23T20:38:05.610881+0000","last_unstale":"2023-09-23T20:40:53.462434+0000","last_undegraded":"2023-09-23T20:40:53.462434+0000","last_fullsized":"2023-09-23T20:40:53.462434+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:35:00.151695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:35:00.151695+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":20,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"86'102","reported_seq":1048,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T20:40:55.479090+0000","last_change":"2023-09-23T20:32:36.314672+0000","last_active":"2023-09-23T20:40:55.479090+0000","last_peered":"2023-09-23T20:40:55.479090+0000","last_clean":"2023-09-23T20:40:55.479090+0000","last_became_active":"2023-09-23T20:32:36.314231+0000","last_became_peered":"2023-09-23T20:32:36.314231+0000","last_unstale":"2023-09-23T20:40:55.479090+0000","last_undegraded":"2023-09-23T20:40:55.479090+0000","last_fullsized":"2023-09-23T20:40:55.479090+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":102,"log_dups_size":0,"ondisk_log_size":102,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:11:40.514709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030656500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":963,"num_read_kb":1234,"num_write":60,"num_write_kb":225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":399,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T20:40:55.472648+0000","last_change":"2023-09-23T20:40:19.544384+0000","last_active":"2023-09-23T20:40:55.472648+0000","last_peered":"2023-09-23T20:40:55.472648+0000","last_clean":"2023-09-23T20:40:55.472648+0000","last_became_active":"2023-09-23T20:40:19.544079+0000","last_became_peered":"2023-09-23T20:40:19.544079+0000","last_unstale":"2023-09-23T20:40:55.472648+0000","last_undegraded":"2023-09-23T20:40:55.472648+0000","last_fullsized":"2023-09-23T20:40:55.472648+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:00:19.498631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":407,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-09-23T20:40:53.906361+0000","last_change":"2023-09-23T20:40:19.532976+0000","last_active":"2023-09-23T20:40:53.906361+0000","last_peered":"2023-09-23T20:40:53.906361+0000","last_clean":"2023-09-23T20:40:53.906361+0000","last_became_active":"2023-09-23T20:40:19.532639+0000","last_became_peered":"2023-09-23T20:40:19.532639+0000","last_unstale":"2023-09-23T20:40:53.906361+0000","last_undegraded":"2023-09-23T20:40:53.906361+0000","last_fullsized":"2023-09-23T20:40:53.906361+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:02:35.301884+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":731,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-09-23T20:40:54.236604+0000","last_change":"2023-09-23T20:38:05.590361+0000","last_active":"2023-09-23T20:40:54.236604+0000","last_peered":"2023-09-23T20:40:54.236604+0000","last_clean":"2023-09-23T20:40:54.236604+0000","last_became_active":"2023-09-23T20:38:05.589985+0000","last_became_peered":"2023-09-23T20:38:05.589985+0000","last_unstale":"2023-09-23T20:40:54.236604+0000","last_undegraded":"2023-09-23T20:40:54.236604+0000","last_fullsized":"2023-09-23T20:40:54.236604+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:03.246271+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:03.246271+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:38:54.351576+0000","scrub_duration":0.0019016700000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":191,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T20:40:55.478713+0000","last_change":"2023-09-23T20:32:50.549695+0000","last_active":"2023-09-23T20:40:55.478713+0000","last_peered":"2023-09-23T20:40:55.478713+0000","last_clean":"2023-09-23T20:40:55.478713+0000","last_became_active":"2023-09-23T20:32:50.549431+0000","last_became_peered":"2023-09-23T20:32:50.549431+0000","last_unstale":"2023-09-23T20:40:55.478713+0000","last_undegraded":"2023-09-23T20:40:55.478713+0000","last_fullsized":"2023-09-23T20:40:55.478713+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:45:53.666569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"86'128","reported_seq":865,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T20:40:55.472921+0000","last_change":"2023-09-23T20:38:03.541765+0000","last_active":"2023-09-23T20:40:55.472921+0000","last_peered":"2023-09-23T20:40:55.472921+0000","last_clean":"2023-09-23T20:40:55.472921+0000","last_became_active":"2023-09-23T20:38:03.541277+0000","last_became_peered":"2023-09-23T20:38:03.541277+0000","last_unstale":"2023-09-23T20:40:55.472921+0000","last_undegraded":"2023-09-23T20:40:55.472921+0000","last_fullsized":"2023-09-23T20:40:55.472921+0000","mapping_epoch":404,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":405,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:20:53.834554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000293202,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":579,"num_read_kb":1473,"num_write":96,"num_write_kb":1062,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":232,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T20:40:55.618201+0000","last_change":"2023-09-23T20:37:43.582105+0000","last_active":"2023-09-23T20:40:55.618201+0000","last_peered":"2023-09-23T20:40:55.618201+0000","last_clean":"2023-09-23T20:40:55.618201+0000","last_became_active":"2023-09-23T20:37:43.581860+0000","last_became_peered":"2023-09-23T20:37:43.581860+0000","last_unstale":"2023-09-23T20:40:55.618201+0000","last_undegraded":"2023-09-23T20:40:55.618201+0000","last_fullsized":"2023-09-23T20:40:55.618201+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:18:38.765486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"90'161","reported_seq":666,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T20:40:55.618762+0000","last_change":"2023-09-23T20:40:22.427435+0000","last_active":"2023-09-23T20:40:55.618762+0000","last_peered":"2023-09-23T20:40:55.618762+0000","last_clean":"2023-09-23T20:40:55.618762+0000","last_became_active":"2023-09-23T20:40:22.427051+0000","last_became_peered":"2023-09-23T20:40:22.427051+0000","last_unstale":"2023-09-23T20:40:55.618762+0000","last_undegraded":"2023-09-23T20:40:55.618762+0000","last_fullsized":"2023-09-23T20:40:55.618762+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:34:54.504554+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:34:54.504554+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:42.001652+0000","scrub_duration":0.011176864,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"86'117","reported_seq":686,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-09-23T20:40:53.462506+0000","last_change":"2023-09-23T20:38:28.446107+0000","last_active":"2023-09-23T20:40:53.462506+0000","last_peered":"2023-09-23T20:40:53.462506+0000","last_clean":"2023-09-23T20:40:53.462506+0000","last_became_active":"2023-09-23T20:38:28.445667+0000","last_became_peered":"2023-09-23T20:38:28.445667+0000","last_unstale":"2023-09-23T20:40:53.462506+0000","last_undegraded":"2023-09-23T20:40:53.462506+0000","last_fullsized":"2023-09-23T20:40:53.462506+0000","mapping_epoch":411,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":412,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:58:47.430360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046891400000000002,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":893,"num_read_kb":922,"num_write":82,"num_write_kb":4193,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":452,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-09-23T20:40:53.908107+0000","last_change":"2023-09-23T20:32:33.277827+0000","last_active":"2023-09-23T20:40:53.908107+0000","last_peered":"2023-09-23T20:40:53.908107+0000","last_clean":"2023-09-23T20:40:53.908107+0000","last_became_active":"2023-09-23T20:32:33.236087+0000","last_became_peered":"2023-09-23T20:32:33.236087+0000","last_unstale":"2023-09-23T20:40:53.908107+0000","last_undegraded":"2023-09-23T20:40:53.908107+0000","last_fullsized":"2023-09-23T20:40:53.908107+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:20:58.176854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":190,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-09-23T20:40:54.238086+0000","last_change":"2023-09-23T20:40:19.530602+0000","last_active":"2023-09-23T20:40:54.238086+0000","last_peered":"2023-09-23T20:40:54.238086+0000","last_clean":"2023-09-23T20:40:54.238086+0000","last_became_active":"2023-09-23T20:40:19.530256+0000","last_became_peered":"2023-09-23T20:40:19.530256+0000","last_unstale":"2023-09-23T20:40:54.238086+0000","last_undegraded":"2023-09-23T20:40:54.238086+0000","last_fullsized":"2023-09-23T20:40:54.238086+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:18:45.658867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"90'161","reported_seq":722,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-09-23T20:40:53.904674+0000","last_change":"2023-09-23T20:38:05.630221+0000","last_active":"2023-09-23T20:40:53.904674+0000","last_peered":"2023-09-23T20:40:53.904674+0000","last_clean":"2023-09-23T20:40:53.904674+0000","last_became_active":"2023-09-23T20:38:05.629839+0000","last_became_peered":"2023-09-23T20:38:05.629839+0000","last_unstale":"2023-09-23T20:40:53.904674+0000","last_undegraded":"2023-09-23T20:40:53.904674+0000","last_fullsized":"2023-09-23T20:40:53.904674+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:06.487351+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:06.487351+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:25:51.325529+0000","scrub_duration":0.001964379,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":420,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-09-23T20:40:53.906383+0000","last_change":"2023-09-23T20:32:33.237234+0000","last_active":"2023-09-23T20:40:53.906383+0000","last_peered":"2023-09-23T20:40:53.906383+0000","last_clean":"2023-09-23T20:40:53.906383+0000","last_became_active":"2023-09-23T20:32:33.236881+0000","last_became_peered":"2023-09-23T20:32:33.236881+0000","last_unstale":"2023-09-23T20:40:53.906383+0000","last_undegraded":"2023-09-23T20:40:53.906383+0000","last_fullsized":"2023-09-23T20:40:53.906383+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:56:17.384831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":309,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T20:40:55.478292+0000","last_change":"2023-09-23T20:29:45.538314+0000","last_active":"2023-09-23T20:40:55.478292+0000","last_peered":"2023-09-23T20:40:55.478292+0000","last_clean":"2023-09-23T20:40:55.478292+0000","last_became_active":"2023-09-23T20:29:45.536847+0000","last_became_peered":"2023-09-23T20:29:45.536847+0000","last_unstale":"2023-09-23T20:40:55.478292+0000","last_undegraded":"2023-09-23T20:40:55.478292+0000","last_fullsized":"2023-09-23T20:40:55.478292+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:51.948721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"90'161","reported_seq":513,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T20:40:55.618743+0000","last_change":"2023-09-23T20:38:05.602250+0000","last_active":"2023-09-23T20:40:55.618743+0000","last_peered":"2023-09-23T20:40:55.618743+0000","last_clean":"2023-09-23T20:40:55.618743+0000","last_became_active":"2023-09-23T20:38:05.601874+0000","last_became_peered":"2023-09-23T20:38:05.601874+0000","last_unstale":"2023-09-23T20:40:55.618743+0000","last_undegraded":"2023-09-23T20:40:55.618743+0000","last_fullsized":"2023-09-23T20:40:55.618743+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:35:42.864363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:35:42.864363+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":40253,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":51,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-09-23T20:40:54.218719+0000","last_change":"2023-09-23T20:38:37.481551+0000","last_active":"2023-09-23T20:40:54.218719+0000","last_peered":"2023-09-23T20:40:54.218719+0000","last_clean":"2023-09-23T20:40:54.218719+0000","last_became_active":"2023-09-23T20:38:37.458687+0000","last_became_peered":"2023-09-23T20:38:37.458687+0000","last_unstale":"2023-09-23T20:40:54.218719+0000","last_undegraded":"2023-09-23T20:40:54.218719+0000","last_fullsized":"2023-09-23T20:40:54.218719+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:22:15.033243+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":109,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-09-23T20:40:54.243437+0000","last_change":"2023-09-23T20:34:59.987575+0000","last_active":"2023-09-23T20:40:54.243437+0000","last_peered":"2023-09-23T20:40:54.243437+0000","last_clean":"2023-09-23T20:40:54.243437+0000","last_became_active":"2023-09-23T20:34:59.987229+0000","last_became_peered":"2023-09-23T20:34:59.987229+0000","last_unstale":"2023-09-23T20:40:54.243437+0000","last_undegraded":"2023-09-23T20:40:54.243437+0000","last_fullsized":"2023-09-23T20:40:54.243437+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:34:01.820159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":51,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-09-23T20:40:54.218352+0000","last_change":"2023-09-23T20:38:37.457634+0000","last_active":"2023-09-23T20:40:54.218352+0000","last_peered":"2023-09-23T20:40:54.218352+0000","last_clean":"2023-09-23T20:40:54.218352+0000","last_became_active":"2023-09-23T20:38:37.452208+0000","last_became_peered":"2023-09-23T20:38:37.452208+0000","last_unstale":"2023-09-23T20:40:54.218352+0000","last_undegraded":"2023-09-23T20:40:54.218352+0000","last_fullsized":"2023-09-23T20:40:54.218352+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:09:41.754569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":109,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-09-23T20:40:54.243975+0000","last_change":"2023-09-23T20:34:59.987416+0000","last_active":"2023-09-23T20:40:54.243975+0000","last_peered":"2023-09-23T20:40:54.243975+0000","last_clean":"2023-09-23T20:40:54.243975+0000","last_became_active":"2023-09-23T20:34:59.987117+0000","last_became_peered":"2023-09-23T20:34:59.987117+0000","last_unstale":"2023-09-23T20:40:54.243975+0000","last_undegraded":"2023-09-23T20:40:54.243975+0000","last_fullsized":"2023-09-23T20:40:54.243975+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:36:16.318623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"90'161","reported_seq":709,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T20:40:55.478200+0000","last_change":"2023-09-23T20:38:05.591992+0000","last_active":"2023-09-23T20:40:55.478200+0000","last_peered":"2023-09-23T20:40:55.478200+0000","last_clean":"2023-09-23T20:40:55.478200+0000","last_became_active":"2023-09-23T20:38:05.591701+0000","last_became_peered":"2023-09-23T20:38:05.591701+0000","last_unstale":"2023-09-23T20:40:55.478200+0000","last_undegraded":"2023-09-23T20:40:55.478200+0000","last_fullsized":"2023-09-23T20:40:55.478200+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:02.384133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:02.384133+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:41.722542+0000","scrub_duration":0.001769098,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":274,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T20:40:55.618856+0000","last_change":"2023-09-23T20:38:02.542719+0000","last_active":"2023-09-23T20:40:55.618856+0000","last_peered":"2023-09-23T20:40:55.618856+0000","last_clean":"2023-09-23T20:40:55.618856+0000","last_became_active":"2023-09-23T20:38:02.542385+0000","last_became_peered":"2023-09-23T20:38:02.542385+0000","last_unstale":"2023-09-23T20:40:55.618856+0000","last_undegraded":"2023-09-23T20:40:55.618856+0000","last_fullsized":"2023-09-23T20:40:55.618856+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:36:23.673757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":241,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T20:40:55.618054+0000","last_change":"2023-09-23T20:37:43.584253+0000","last_active":"2023-09-23T20:40:55.618054+0000","last_peered":"2023-09-23T20:40:55.618054+0000","last_clean":"2023-09-23T20:40:55.618054+0000","last_became_active":"2023-09-23T20:37:43.580069+0000","last_became_peered":"2023-09-23T20:37:43.580069+0000","last_unstale":"2023-09-23T20:40:55.618054+0000","last_undegraded":"2023-09-23T20:40:55.618054+0000","last_fullsized":"2023-09-23T20:40:55.618054+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:39:11.482581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"90'161","reported_seq":546,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T20:40:55.472825+0000","last_change":"2023-09-23T20:36:01.505098+0000","last_active":"2023-09-23T20:40:55.472825+0000","last_peered":"2023-09-23T20:40:55.472825+0000","last_clean":"2023-09-23T20:40:55.472825+0000","last_became_active":"2023-09-23T20:35:26.644707+0000","last_became_peered":"2023-09-23T20:35:26.644707+0000","last_unstale":"2023-09-23T20:40:55.472825+0000","last_undegraded":"2023-09-23T20:40:55.472825+0000","last_fullsized":"2023-09-23T20:40:55.472825+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:01.504973+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:01.504973+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:02:10.880809+0000","scrub_duration":0.00202344,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":498,"reported_epoch":453,"state":"active+clean","last_fresh":"2023-09-23T20:40:55.922180+0000","last_change":"2023-09-23T20:40:19.569032+0000","last_active":"2023-09-23T20:40:55.922180+0000","last_peered":"2023-09-23T20:40:55.922180+0000","last_clean":"2023-09-23T20:40:55.922180+0000","last_became_active":"2023-09-23T20:40:19.538476+0000","last_became_peered":"2023-09-23T20:40:19.538476+0000","last_unstale":"2023-09-23T20:40:55.922180+0000","last_undegraded":"2023-09-23T20:40:55.922180+0000","last_fullsized":"2023-09-23T20:40:55.922180+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:31:58.998168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3324,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":170,"reported_epoch":452,"state":"active+clean","last_fresh":"2023-09-23T20:40:54.241982+0000","last_change":"2023-09-23T20:38:02.538333+0000","last_active":"2023-09-23T20:40:54.241982+0000","last_peered":"2023-09-23T20:40:54.241982+0000","last_clean":"2023-09-23T20:40:54.241982+0000","last_became_active":"2023-09-23T20:38:02.538008+0000","last_became_peered":"2023-09-23T20:38:02.538008+0000","last_unstale":"2023-09-23T20:40:54.241982+0000","last_undegraded":"2023-09-23T20:40:54.241982+0000","last_fullsized":"2023-09-23T20:40:54.241982+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:02:38.789530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":80,"num_bytes_recovered":18486,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26691,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":351684,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":155,"num_read_kb":290,"num_write":76,"num_write_kb":768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":891212,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5983,"num_read_kb":13200,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":68,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6082,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201634,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":390172,"kb_used_data":9172,"kb_used_omap":0,"kb_used_meta":380992,"kb_avail":93354980,"statfs":{"total":95995035648,"available":95595499520,"internally_reserved":0,"allocated":9392128,"data_stored":7944510,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":390135808},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:40:54 2023","interfaces":[{"interface":"back","average":{"1min":1.423,"5min":1.968,"15min":2.1520000000000001},"min":{"1min":0.27200000000000002,"5min":0.24399999999999999,"15min":0.20799999999999999},"max":{"1min":5.6310000000000002,"5min":18.315000000000001,"15min":48.798999999999999},"last":0.52700000000000002},{"interface":"front","average":{"1min":1.069,"5min":1.2909999999999999,"15min":1.974},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":5.3330000000000002,"5min":9.4429999999999996,"15min":48.247},"last":0.41099999999999998}]},{"osd":1,"last update":"Sat Sep 23 20:40:03 2023","interfaces":[{"interface":"back","average":{"1min":0.92600000000000005,"5min":2.7120000000000002,"15min":2.1520000000000001},"min":{"1min":0.28799999999999998,"5min":0.25900000000000001,"15min":0.254},"max":{"1min":2.9020000000000001,"5min":49.965000000000003,"15min":49.965000000000003},"last":0.48199999999999998},{"interface":"front","average":{"1min":0.89300000000000002,"5min":2.6989999999999998,"15min":2.1709999999999998},"min":{"1min":0.30299999999999999,"5min":0.26900000000000002,"15min":0.248},"max":{"1min":5.508,"5min":53.451999999999998,"15min":53.451999999999998},"last":0.434}]},{"osd":2,"last update":"Sat Sep 23 20:40:54 2023","interfaces":[{"interface":"back","average":{"1min":1.339,"5min":1.736,"15min":2.0840000000000001},"min":{"1min":0.32000000000000001,"5min":0.28599999999999998,"15min":0.27600000000000002},"max":{"1min":4.5039999999999996,"5min":16.965,"15min":16.965},"last":0.38500000000000001},{"interface":"front","average":{"1min":1.4630000000000001,"5min":2.1960000000000002,"15min":2.4489999999999998},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.26600000000000001},"max":{"1min":6.4329999999999998,"5min":23.414000000000001,"15min":23.494},"last":0.42099999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:40:21 2023","interfaces":[{"interface":"back","average":{"1min":1.105,"5min":2.2480000000000002,"15min":2.4470000000000001},"min":{"1min":0.35999999999999999,"5min":0.311,"15min":0.26200000000000001},"max":{"1min":4.2149999999999999,"5min":23.352,"15min":23.352},"last":0.45800000000000002},{"interface":"front","average":{"1min":1.139,"5min":2.2709999999999999,"15min":2.423},"min":{"1min":0.46500000000000002,"5min":0.34399999999999997,"15min":0.32500000000000001},"max":{"1min":4.2359999999999998,"5min":20.969999999999999,"15min":20.969999999999999},"last":0.47099999999999997}]},{"osd":4,"last update":"Sat Sep 23 20:40:21 2023","interfaces":[{"interface":"back","average":{"1min":1.046,"5min":1.085,"15min":1.2170000000000001},"min":{"1min":0.35799999999999998,"5min":0.25600000000000001,"15min":0.23200000000000001},"max":{"1min":4.1639999999999997,"5min":18.364999999999998,"15min":18.364999999999998},"last":0.39800000000000002},{"interface":"front","average":{"1min":0.94999999999999996,"5min":1.296,"15min":1.417},"min":{"1min":0.39100000000000001,"5min":0.28399999999999997,"15min":0.27600000000000002},"max":{"1min":2.605,"5min":23.617000000000001,"15min":23.617000000000001},"last":0.32100000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:40:21 2023","interfaces":[{"interface":"back","average":{"1min":1.0780000000000001,"5min":1.1499999999999999,"15min":1.2509999999999999},"min":{"1min":0.30099999999999999,"5min":0.29199999999999998,"15min":0.26600000000000001},"max":{"1min":4.3780000000000001,"5min":12.516,"15min":13.241},"last":0.34399999999999997},{"interface":"front","average":{"1min":0.76400000000000001,"5min":1.286,"15min":1.3540000000000001},"min":{"1min":0.38400000000000001,"5min":0.31900000000000001,"15min":0.315},"max":{"1min":3.1030000000000002,"5min":18.390000000000001,"15min":18.831},"last":0.44400000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:40:21 2023","interfaces":[{"interface":"back","average":{"1min":1.2629999999999999,"5min":1.4350000000000001,"15min":1.329},"min":{"1min":0.44,"5min":0.312,"15min":0.30399999999999999},"max":{"1min":4.0540000000000003,"5min":18.417000000000002,"15min":18.417000000000002},"last":0.50700000000000001},{"interface":"front","average":{"1min":1.1659999999999999,"5min":1.113,"15min":1.252},"min":{"1min":0.38200000000000001,"5min":0.32700000000000001,"15min":0.318},"max":{"1min":6.4340000000000002,"5min":7.266,"15min":9.9120000000000008},"last":0.36499999999999999}]}]},{"osd":6,"up_from":47,"seq":201863463273,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":452624,"kb_used_data":62280,"kb_used_omap":0,"kb_used_meta":390336,"kb_avail":93292528,"statfs":{"total":95995035648,"available":95531548672,"internally_reserved":0,"allocated":63774720,"data_stored":62338719,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":399704064},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:40:50 2023","interfaces":[{"interface":"back","average":{"1min":1.1910000000000001,"5min":1.7470000000000001,"15min":1.9159999999999999},"min":{"1min":0.28100000000000003,"5min":0.23300000000000001,"15min":0.221},"max":{"1min":4.1399999999999997,"5min":17.492999999999999,"15min":53.701999999999998},"last":0.56499999999999995},{"interface":"front","average":{"1min":1.395,"5min":2.306,"15min":2.3399999999999999},"min":{"1min":0.253,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":6.0570000000000004,"5min":60.328000000000003,"15min":60.328000000000003},"last":0.53600000000000003}]},{"osd":1,"last update":"Sat Sep 23 20:40:06 2023","interfaces":[{"interface":"back","average":{"1min":1.157,"5min":2.0600000000000001,"15min":2.3610000000000002},"min":{"1min":0.26600000000000001,"5min":0.26000000000000001,"15min":0.249},"max":{"1min":5.8630000000000004,"5min":64.274000000000001,"15min":64.274000000000001},"last":0.54900000000000004},{"interface":"front","average":{"1min":1.2390000000000001,"5min":1.393,"15min":2.4830000000000001},"min":{"1min":0.34499999999999997,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":6.8460000000000001,"5min":6.8460000000000001,"15min":34.490000000000002},"last":0.64300000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:40:50 2023","interfaces":[{"interface":"back","average":{"1min":2.097,"5min":1.6419999999999999,"15min":2.133},"min":{"1min":0.31,"5min":0.30299999999999999,"15min":0.27700000000000002},"max":{"1min":9.9049999999999994,"5min":13.010999999999999,"15min":34.317999999999998},"last":0.58999999999999997},{"interface":"front","average":{"1min":2.1080000000000001,"5min":1.585,"15min":2.0569999999999999},"min":{"1min":0.311,"5min":0.311,"15min":0.30399999999999999},"max":{"1min":9.9269999999999996,"5min":12.917999999999999,"15min":34.292999999999999},"last":0.61099999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:40:36 2023","interfaces":[{"interface":"back","average":{"1min":2.0099999999999998,"5min":1.8759999999999999,"15min":1.9910000000000001},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":5.9130000000000003,"5min":8.9670000000000005,"15min":9.4239999999999995},"last":0.59999999999999998},{"interface":"front","average":{"1min":1.921,"5min":1.968,"15min":2.1659999999999999},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.317},"max":{"1min":5.8380000000000001,"5min":11.913,"15min":12.875999999999999},"last":0.51400000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:40:11 2023","interfaces":[{"interface":"back","average":{"1min":0.90200000000000002,"5min":0.84799999999999998,"15min":0.95399999999999996},"min":{"1min":0.30099999999999999,"5min":0.26900000000000002,"15min":0.251},"max":{"1min":2.859,"5min":23.904,"15min":23.904},"last":0.46899999999999997},{"interface":"front","average":{"1min":1.498,"5min":1.1879999999999999,"15min":1.204},"min":{"1min":0.36799999999999999,"5min":0.29199999999999998,"15min":0.27000000000000002},"max":{"1min":6.7359999999999998,"5min":11.515000000000001,"15min":11.515000000000001},"last":0.35499999999999998}]},{"osd":5,"last update":"Sat Sep 23 20:40:11 2023","interfaces":[{"interface":"back","average":{"1min":1.2210000000000001,"5min":0.85299999999999998,"15min":0.96199999999999997},"min":{"1min":0.375,"5min":0.318,"15min":0.28699999999999998},"max":{"1min":6.6340000000000003,"5min":6.6340000000000003,"15min":14.085000000000001},"last":0.42799999999999999},{"interface":"front","average":{"1min":1.2529999999999999,"5min":0.97899999999999998,"15min":1.355},"min":{"1min":0.35399999999999998,"5min":0.32500000000000001,"15min":0.30599999999999999},"max":{"1min":6.6909999999999998,"5min":6.7320000000000002,"15min":18.192},"last":0.45500000000000002}]},{"osd":7,"last update":"Sat Sep 23 20:40:36 2023","interfaces":[{"interface":"back","average":{"1min":1.1240000000000001,"5min":0.88300000000000001,"15min":1.121},"min":{"1min":0.40400000000000003,"5min":0.376,"15min":0.28599999999999998},"max":{"1min":5.2729999999999997,"5min":5.2729999999999997,"15min":10.217000000000001},"last":0.52500000000000002},{"interface":"front","average":{"1min":0.97899999999999998,"5min":1.048,"15min":1.3120000000000001},"min":{"1min":0.45700000000000002,"5min":0.36799999999999999,"15min":0.30299999999999999},"max":{"1min":2.2909999999999999,"5min":8.8119999999999994,"15min":9.7360000000000007},"last":0.39300000000000002}]}]},{"osd":1,"up_from":266,"seq":1142461300872,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":464968,"kb_used_data":24640,"kb_used_omap":8,"kb_used_meta":440311,"kb_avail":93280184,"statfs":{"total":95995035648,"available":95518908416,"internally_reserved":0,"allocated":25231360,"data_stored":22756789,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8206,"internal_metadata":450879474},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:40:05 2023","interfaces":[{"interface":"back","average":{"1min":1.3680000000000001,"5min":2.2250000000000001,"15min":2.5099999999999998},"min":{"1min":0.218,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":8.1150000000000002,"5min":14.531000000000001,"15min":52.689999999999998},"last":0.51500000000000001},{"interface":"front","average":{"1min":2.4140000000000001,"5min":3.1200000000000001,"15min":3.2650000000000001},"min":{"1min":0.316,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":11.994,"5min":19.41,"15min":52.662999999999997},"last":0.38600000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:40:05 2023","interfaces":[{"interface":"back","average":{"1min":2.7280000000000002,"5min":3.0489999999999999,"15min":4.0810000000000004},"min":{"1min":0.23400000000000001,"5min":0.23300000000000001,"15min":0.20899999999999999},"max":{"1min":14.148,"5min":32.255000000000003,"15min":32.255000000000003},"last":0.46500000000000002},{"interface":"front","average":{"1min":1.911,"5min":2.238,"15min":2.3170000000000002},"min":{"1min":0.28599999999999998,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":8.8079999999999998,"5min":13.404999999999999,"15min":13.404999999999999},"last":0.437}]},{"osd":3,"last update":"Sat Sep 23 20:40:05 2023","interfaces":[{"interface":"back","average":{"1min":1.6659999999999999,"5min":2.3809999999999998,"15min":3.5430000000000001},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":8.0079999999999991,"5min":14.359,"15min":80.745999999999995},"last":0.57399999999999995},{"interface":"front","average":{"1min":2.6509999999999998,"5min":2.2810000000000001,"15min":3.9319999999999999},"min":{"1min":0.316,"5min":0.27500000000000002,"15min":0.22700000000000001},"max":{"1min":12.045999999999999,"5min":19.315000000000001,"15min":82.988},"last":0.495}]},{"osd":4,"last update":"Sat Sep 23 20:40:05 2023","interfaces":[{"interface":"back","average":{"1min":2.0710000000000002,"5min":2.2490000000000001,"15min":2.2090000000000001},"min":{"1min":0.33400000000000002,"5min":0.315,"15min":0.30299999999999999},"max":{"1min":7.8399999999999999,"5min":9.4510000000000005,"15min":24.327999999999999},"last":0.54900000000000004},{"interface":"front","average":{"1min":2.056,"5min":1.593,"15min":1.764},"min":{"1min":0.33400000000000002,"5min":0.311,"15min":0.311},"max":{"1min":8.1509999999999998,"5min":11.574999999999999,"15min":11.574999999999999},"last":0.63200000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:40:05 2023","interfaces":[{"interface":"back","average":{"1min":1.996,"5min":2.1400000000000001,"15min":2.3730000000000002},"min":{"1min":0.45800000000000002,"5min":0.34100000000000003,"15min":0.33200000000000002},"max":{"1min":5.5529999999999999,"5min":33.401000000000003,"15min":33.401000000000003},"last":0.62},{"interface":"front","average":{"1min":2.5790000000000002,"5min":2.7120000000000002,"15min":2.427},"min":{"1min":0.36199999999999999,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":7.9340000000000002,"5min":33.417000000000002,"15min":33.417000000000002},"last":0.69499999999999995}]},{"osd":6,"last update":"Sat Sep 23 20:40:05 2023","interfaces":[{"interface":"back","average":{"1min":2.0019999999999998,"5min":1.9570000000000001,"15min":4.1699999999999999},"min":{"1min":0.42899999999999999,"5min":0.36899999999999999,"15min":0.35699999999999998},"max":{"1min":6.8099999999999996,"5min":9.6999999999999993,"15min":72.682000000000002},"last":0.53400000000000003},{"interface":"front","average":{"1min":1.9419999999999999,"5min":2.2799999999999998,"15min":4.0460000000000003},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":7.883,"5min":9.4849999999999994,"15min":72.563999999999993},"last":0.60099999999999998}]},{"osd":7,"last update":"Sat Sep 23 20:40:05 2023","interfaces":[{"interface":"back","average":{"1min":2.1930000000000001,"5min":2.069,"15min":2.5760000000000001},"min":{"1min":0.48399999999999999,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":5.0819999999999999,"5min":18.437000000000001,"15min":18.437000000000001},"last":0.58799999999999997},{"interface":"front","average":{"1min":2.3149999999999999,"5min":2.3959999999999999,"15min":2.46},"min":{"1min":0.45100000000000001,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":7.9089999999999998,"5min":12.021000000000001,"15min":24.524000000000001},"last":0.65600000000000003}]}]},{"osd":0,"up_from":94,"seq":403726926074,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":660868,"kb_used_data":15868,"kb_used_omap":7,"kb_used_meta":644984,"kb_avail":93084284,"statfs":{"total":95995035648,"available":95318306816,"internally_reserved":0,"allocated":16248832,"data_stored":13940410,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":660463633},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:40:11 2023","interfaces":[{"interface":"back","average":{"1min":1.712,"5min":1.8,"15min":3.0680000000000001},"min":{"1min":0.20699999999999999,"5min":0.17199999999999999,"15min":0.17199999999999999},"max":{"1min":9.8879999999999999,"5min":11.554,"15min":42.000999999999998},"last":0.35399999999999998},{"interface":"front","average":{"1min":1.5660000000000001,"5min":1.9710000000000001,"15min":3.3370000000000002},"min":{"1min":0.218,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":10.097,"5min":12.35,"15min":23.036000000000001},"last":0.46600000000000003}]},{"osd":2,"last update":"Sat Sep 23 20:40:50 2023","interfaces":[{"interface":"back","average":{"1min":3.0609999999999999,"5min":2.657,"15min":2.9849999999999999},"min":{"1min":0.29199999999999998,"5min":0.218,"15min":0.185},"max":{"1min":13.871,"5min":62.726999999999997,"15min":77.448999999999998},"last":0.38700000000000001},{"interface":"front","average":{"1min":2.6600000000000001,"5min":2.875,"15min":2.8919999999999999},"min":{"1min":0.27800000000000002,"5min":0.247,"15min":0.188},"max":{"1min":13.356999999999999,"5min":62.689,"15min":81.724999999999994},"last":0.33800000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:40:23 2023","interfaces":[{"interface":"back","average":{"1min":2.399,"5min":2.7450000000000001,"15min":3.133},"min":{"1min":0.34699999999999998,"5min":0.24399999999999999,"15min":0.23400000000000001},"max":{"1min":9.0039999999999996,"5min":51.536000000000001,"15min":81.400999999999996},"last":0.318},{"interface":"front","average":{"1min":2.8839999999999999,"5min":3.1389999999999998,"15min":3.4590000000000001},"min":{"1min":0.245,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":14.237,"5min":44.811999999999998,"15min":81.137},"last":0.495}]},{"osd":4,"last update":"Sat Sep 23 20:40:50 2023","interfaces":[{"interface":"back","average":{"1min":2.7970000000000002,"5min":1.9019999999999999,"15min":2.1920000000000002},"min":{"1min":0.432,"5min":0.28899999999999998,"15min":0.25900000000000001},"max":{"1min":6.8970000000000002,"5min":9.6110000000000007,"15min":23.876999999999999},"last":0.441},{"interface":"front","average":{"1min":2.9929999999999999,"5min":1.615,"15min":1.9770000000000001},"min":{"1min":0.36299999999999999,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":9.7789999999999999,"5min":9.7789999999999999,"15min":23.681999999999999},"last":0.55800000000000005}]},{"osd":5,"last update":"Sat Sep 23 20:40:50 2023","interfaces":[{"interface":"back","average":{"1min":3.0649999999999999,"5min":1.6519999999999999,"15min":2.1819999999999999},"min":{"1min":0.41499999999999998,"5min":0.371,"15min":0.34200000000000003},"max":{"1min":9.8629999999999995,"5min":9.8629999999999995,"15min":24.198},"last":0.59999999999999998},{"interface":"front","average":{"1min":2.8439999999999999,"5min":2.4449999999999998,"15min":2.3170000000000002},"min":{"1min":0.38900000000000001,"5min":0.318,"15min":0.25600000000000001},"max":{"1min":12.929,"5min":26.228000000000002,"15min":26.228000000000002},"last":0.52800000000000002}]},{"osd":6,"last update":"Sat Sep 23 20:40:50 2023","interfaces":[{"interface":"back","average":{"1min":3.6520000000000001,"5min":1.9650000000000001,"15min":2.2549999999999999},"min":{"1min":0.39800000000000002,"5min":0.34499999999999997,"15min":0.318},"max":{"1min":24.056000000000001,"5min":24.056000000000001,"15min":24.408000000000001},"last":0.64600000000000002},{"interface":"front","average":{"1min":2.1219999999999999,"5min":2.2770000000000001,"15min":2.282},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.29399999999999998},"max":{"1min":8.0549999999999997,"5min":26.268000000000001,"15min":26.268000000000001},"last":0.58899999999999997}]},{"osd":7,"last update":"Sat Sep 23 20:40:50 2023","interfaces":[{"interface":"back","average":{"1min":2.4239999999999999,"5min":2.3180000000000001,"15min":2.4220000000000002},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.30599999999999999},"max":{"1min":8.1189999999999998,"5min":26.177,"15min":26.177},"last":0.61399999999999999},{"interface":"front","average":{"1min":2.9910000000000001,"5min":1.966,"15min":2.569},"min":{"1min":0.53700000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":6.8979999999999997,"5min":9.7200000000000006,"15min":24.669},"last":0.54300000000000004}]}]},{"osd":2,"up_from":104,"seq":446676599022,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":688984,"kb_used_data":15312,"kb_used_omap":11,"kb_used_meta":673652,"kb_avail":93056168,"statfs":{"total":95995035648,"available":95289516032,"internally_reserved":0,"allocated":15679488,"data_stored":13364501,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11878,"internal_metadata":689820058},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:40:00 2023","interfaces":[{"interface":"back","average":{"1min":2.1339999999999999,"5min":1.921,"15min":2.7160000000000002},"min":{"1min":0.26400000000000001,"5min":0.20300000000000001,"15min":0.17999999999999999},"max":{"1min":8.9710000000000001,"5min":14.124000000000001,"15min":82.424999999999997},"last":0.377},{"interface":"front","average":{"1min":2.2919999999999998,"5min":2.375,"15min":2.5920000000000001},"min":{"1min":0.23000000000000001,"5min":0.19,"15min":0.19},"max":{"1min":9.5600000000000005,"5min":14.055,"15min":82.524000000000001},"last":0.38600000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:40:00 2023","interfaces":[{"interface":"back","average":{"1min":2.472,"5min":2.9500000000000002,"15min":3.6240000000000001},"min":{"1min":0.433,"5min":0.245,"15min":0.221},"max":{"1min":11.279999999999999,"5min":46.405000000000001,"15min":46.405000000000001},"last":0.33200000000000002},{"interface":"front","average":{"1min":1.506,"5min":2.9129999999999998,"15min":2.9009999999999998},"min":{"1min":0.24399999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":8.8699999999999992,"5min":46.459000000000003,"15min":46.459000000000003},"last":0.40500000000000003}]},{"osd":3,"last update":"Sat Sep 23 20:40:22 2023","interfaces":[{"interface":"back","average":{"1min":2.851,"5min":2.613,"15min":2.4769999999999999},"min":{"1min":0.28499999999999998,"5min":0.27100000000000002,"15min":0.19800000000000001},"max":{"1min":11.304,"5min":19.562999999999999,"15min":19.562999999999999},"last":0.28799999999999998},{"interface":"front","average":{"1min":1.7909999999999999,"5min":1.6160000000000001,"15min":2.5089999999999999},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.23899999999999999},"max":{"1min":8.9369999999999994,"5min":13.148999999999999,"15min":30.704999999999998},"last":0.432}]},{"osd":4,"last update":"Sat Sep 23 20:40:00 2023","interfaces":[{"interface":"back","average":{"1min":1.587,"5min":1.8460000000000001,"15min":1.931},"min":{"1min":0.33300000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":6.6849999999999996,"5min":9.2430000000000003,"15min":16.797999999999998},"last":0.64700000000000002},{"interface":"front","average":{"1min":1.4530000000000001,"5min":2.0009999999999999,"15min":2.117},"min":{"1min":0.38,"5min":0.307,"15min":0.27600000000000002},"max":{"1min":7.9400000000000004,"5min":19.440999999999999,"15min":19.440999999999999},"last":0.71899999999999997}]},{"osd":5,"last update":"Sat Sep 23 20:40:00 2023","interfaces":[{"interface":"back","average":{"1min":1.7290000000000001,"5min":1.8680000000000001,"15min":2.4329999999999998},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.309},"max":{"1min":6.6319999999999997,"5min":9.1799999999999997,"15min":95.393000000000001},"last":0.73399999999999999},{"interface":"front","average":{"1min":1.7110000000000001,"5min":2.0230000000000001,"15min":2.694},"min":{"1min":0.42399999999999999,"5min":0.35299999999999998,"15min":0.32300000000000001},"max":{"1min":7.8179999999999996,"5min":19.513000000000002,"15min":95.414000000000001},"last":0.60799999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:40:00 2023","interfaces":[{"interface":"back","average":{"1min":1.6339999999999999,"5min":1.9730000000000001,"15min":2.1909999999999998},"min":{"1min":0.47099999999999997,"5min":0.37,"15min":0.32100000000000001},"max":{"1min":7.8579999999999997,"5min":19.486000000000001,"15min":19.486000000000001},"last":0.63200000000000001},{"interface":"front","average":{"1min":1.512,"5min":1.4550000000000001,"15min":2.0329999999999999},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.316},"max":{"1min":9.3010000000000002,"5min":10.811,"15min":30.670999999999999},"last":0.755}]},{"osd":7,"last update":"Sat Sep 23 20:40:00 2023","interfaces":[{"interface":"back","average":{"1min":1.073,"5min":1.3520000000000001,"15min":2.028},"min":{"1min":0.42499999999999999,"5min":0.39300000000000002,"15min":0.35199999999999998},"max":{"1min":4.4260000000000002,"5min":10.231,"15min":30.741},"last":0.67000000000000004},{"interface":"front","average":{"1min":1.752,"5min":1.9650000000000001,"15min":2.1760000000000002},"min":{"1min":0.44,"5min":0.40999999999999998,"15min":0.38},"max":{"1min":6.7220000000000004,"5min":9.2720000000000002,"15min":16.719000000000001},"last":0.78300000000000003}]}]},{"osd":3,"up_from":109,"seq":468151435496,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":823172,"kb_used_data":26044,"kb_used_omap":4,"kb_used_meta":797115,"kb_avail":92921980,"statfs":{"total":95995035648,"available":95152107520,"internally_reserved":0,"allocated":26669056,"data_stored":24370870,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4514,"internal_metadata":816246366},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:40:26 2023","interfaces":[{"interface":"back","average":{"1min":1.409,"5min":1.7350000000000001,"15min":2.544},"min":{"1min":0.21199999999999999,"5min":0.21199999999999999,"15min":0.186},"max":{"1min":7.6879999999999997,"5min":14.638,"15min":67.305999999999997},"last":0.35499999999999998},{"interface":"front","average":{"1min":2.4540000000000002,"5min":1.865,"15min":2.415},"min":{"1min":0.29499999999999998,"5min":0.26600000000000001,"15min":0.19500000000000001},"max":{"1min":8.8439999999999994,"5min":10.140000000000001,"15min":67.403999999999996},"last":0.39500000000000002}]},{"osd":1,"last update":"Sat Sep 23 20:39:59 2023","interfaces":[{"interface":"back","average":{"1min":0.81699999999999995,"5min":1.3169999999999999,"15min":2.375},"min":{"1min":0.246,"5min":0.23999999999999999,"15min":0.222},"max":{"1min":7.6219999999999999,"5min":9.8940000000000001,"15min":16.189},"last":0.38300000000000001},{"interface":"front","average":{"1min":2.343,"5min":2.1669999999999998,"15min":2.6829999999999998},"min":{"1min":0.26500000000000001,"5min":0.26200000000000001,"15min":0.24199999999999999},"max":{"1min":9.1590000000000007,"5min":13.727,"15min":32.603999999999999},"last":0.31900000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:40:26 2023","interfaces":[{"interface":"back","average":{"1min":2.3140000000000001,"5min":1.837,"15min":2.5030000000000001},"min":{"1min":0.26600000000000001,"5min":0.246,"15min":0.22800000000000001},"max":{"1min":6.3200000000000003,"5min":10.747999999999999,"15min":32.857999999999997},"last":0.36899999999999999},{"interface":"front","average":{"1min":1.516,"5min":2.0259999999999998,"15min":2.4980000000000002},"min":{"1min":0.31,"5min":0.28100000000000003,"15min":0.20899999999999999},"max":{"1min":8.5530000000000008,"5min":22.32,"15min":24.146999999999998},"last":0.41899999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:40:26 2023","interfaces":[{"interface":"back","average":{"1min":0.82199999999999995,"5min":1.0860000000000001,"15min":1.786},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.29799999999999999},"max":{"1min":2.5899999999999999,"5min":9.3160000000000007,"15min":46.042999999999999},"last":0.48999999999999999},{"interface":"front","average":{"1min":1.6359999999999999,"5min":1.879,"15min":2.4340000000000002},"min":{"1min":0.33200000000000002,"5min":0.33000000000000002,"15min":0.28100000000000003},"max":{"1min":5.141,"5min":13.571999999999999,"15min":46.579999999999998},"last":0.45300000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:40:26 2023","interfaces":[{"interface":"back","average":{"1min":0.82099999999999995,"5min":1.167,"15min":1.998},"min":{"1min":0.32700000000000001,"5min":0.32300000000000001,"15min":0.31900000000000001},"max":{"1min":2.9889999999999999,"5min":9.4130000000000003,"15min":45.292000000000002},"last":0.40799999999999997},{"interface":"front","average":{"1min":1.6859999999999999,"5min":1.482,"15min":1.968},"min":{"1min":0.38500000000000001,"5min":0.34399999999999997,"15min":0.313},"max":{"1min":5.0060000000000002,"5min":9.0839999999999996,"15min":44.679000000000002},"last":0.51000000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:40:26 2023","interfaces":[{"interface":"back","average":{"1min":1.8640000000000001,"5min":1.3959999999999999,"15min":1.964},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":7.3630000000000004,"5min":9.0470000000000006,"15min":17.524999999999999},"last":0.44400000000000001},{"interface":"front","average":{"1min":1.4750000000000001,"5min":1.8839999999999999,"15min":2.165},"min":{"1min":0.434,"5min":0.39800000000000002,"15min":0.33700000000000002},"max":{"1min":4.8819999999999997,"5min":13.500999999999999,"15min":19.524000000000001},"last":0.47399999999999998}]},{"osd":7,"last update":"Sat Sep 23 20:40:26 2023","interfaces":[{"interface":"back","average":{"1min":1.5109999999999999,"5min":2.1829999999999998,"15min":2.3959999999999999},"min":{"1min":0.433,"5min":0.39900000000000002,"15min":0.34999999999999998},"max":{"1min":4.319,"5min":13.544,"15min":19.571999999999999},"last":0.503},{"interface":"front","average":{"1min":0.996,"5min":1.274,"15min":2.0720000000000001},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.371},"max":{"1min":4.157,"5min":9.3719999999999999,"15min":45.387999999999998},"last":0.52000000000000002}]}]},{"osd":4,"up_from":30,"seq":128849019249,"num_pgs":61,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":404480,"kb_used_data":39928,"kb_used_omap":0,"kb_used_meta":364544,"kb_avail":93340672,"statfs":{"total":95995035648,"available":95580848128,"internally_reserved":0,"allocated":40886272,"data_stored":39450915,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":373293056},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:40:49 2023","interfaces":[{"interface":"back","average":{"1min":2.1949999999999998,"5min":1.6739999999999999,"15min":1.724},"min":{"1min":0.31,"5min":0.24099999999999999,"15min":0.23699999999999999},"max":{"1min":11.553000000000001,"5min":24.114999999999998,"15min":24.114999999999998},"last":0.26200000000000001},{"interface":"front","average":{"1min":2.2789999999999999,"5min":1.8080000000000001,"15min":1.895},"min":{"1min":0.28899999999999998,"5min":0.246,"15min":0.246},"max":{"1min":11.536,"5min":24.135999999999999,"15min":24.135999999999999},"last":0.39600000000000002}]},{"osd":1,"last update":"Sat Sep 23 20:40:06 2023","interfaces":[{"interface":"back","average":{"1min":0.95899999999999996,"5min":1.331,"15min":1.7410000000000001},"min":{"1min":0.309,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":4.1669999999999998,"5min":18.190999999999999,"15min":18.190999999999999},"last":0.34399999999999997},{"interface":"front","average":{"1min":0.88300000000000001,"5min":1.4259999999999999,"15min":1.6899999999999999},"min":{"1min":0.307,"5min":0.28499999999999998,"15min":0.28199999999999997},"max":{"1min":3.7109999999999999,"5min":18.173999999999999,"15min":18.173999999999999},"last":0.33400000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:40:03 2023","interfaces":[{"interface":"back","average":{"1min":1.3400000000000001,"5min":1.6060000000000001,"15min":1.8740000000000001},"min":{"1min":0.34200000000000003,"5min":0.29999999999999999,"15min":0.26600000000000001},"max":{"1min":3.8010000000000002,"5min":10.416,"15min":32.363},"last":0.30399999999999999},{"interface":"front","average":{"1min":2.2069999999999999,"5min":2.1019999999999999,"15min":2.3109999999999999},"min":{"1min":0.34300000000000003,"5min":0.28899999999999998,"15min":0.27600000000000002},"max":{"1min":11.917999999999999,"5min":11.917999999999999,"15min":34.042999999999999},"last":0.42299999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:40:17 2023","interfaces":[{"interface":"back","average":{"1min":2.4209999999999998,"5min":1.8480000000000001,"15min":2.113},"min":{"1min":0.40500000000000003,"5min":0.372,"15min":0.29099999999999998},"max":{"1min":12.147,"5min":12.147,"15min":32.332999999999998},"last":0.372},{"interface":"front","average":{"1min":1.8280000000000001,"5min":1.8360000000000001,"15min":2.206},"min":{"1min":0.371,"5min":0.371,"15min":0.30299999999999999},"max":{"1min":11.914,"5min":11.914,"15min":32.399999999999999},"last":0.35499999999999998}]},{"osd":5,"last update":"Sat Sep 23 20:40:47 2023","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.68200000000000005,"15min":1.51},"min":{"1min":0.28100000000000003,"5min":0.26400000000000001,"15min":0.247},"max":{"1min":2.202,"5min":4.1260000000000003,"15min":82.866},"last":0.28799999999999998},{"interface":"front","average":{"1min":1.006,"5min":0.96999999999999997,"15min":1.4810000000000001},"min":{"1min":0.32200000000000001,"5min":0.28999999999999998,"15min":0.27800000000000002},"max":{"1min":5.2670000000000003,"5min":7.0129999999999999,"15min":82.546000000000006},"last":0.314}]},{"osd":6,"last update":"Sat Sep 23 20:40:03 2023","interfaces":[{"interface":"back","average":{"1min":1.722,"5min":1.032,"15min":1.381},"min":{"1min":0.32200000000000001,"5min":0.312,"15min":0.27600000000000002},"max":{"1min":14.731999999999999,"5min":14.731999999999999,"15min":15.252000000000001},"last":0.36199999999999999},{"interface":"front","average":{"1min":1.024,"5min":0.97199999999999998,"15min":1.2450000000000001},"min":{"1min":0.40500000000000003,"5min":0.29899999999999999,"15min":0.27200000000000002},"max":{"1min":4.976,"5min":7.2850000000000001,"15min":11.199},"last":0.38500000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:40:17 2023","interfaces":[{"interface":"back","average":{"1min":1.0389999999999999,"5min":1.143,"15min":1.5369999999999999},"min":{"1min":0.39100000000000001,"5min":0.318,"15min":0.318},"max":{"1min":4.21,"5min":14.534000000000001,"15min":16.427},"last":0.442},{"interface":"front","average":{"1min":1.472,"5min":0.84599999999999997,"15min":1.3180000000000001},"min":{"1min":0.36399999999999999,"5min":0.32700000000000001,"15min":0.307},"max":{"1min":7.9390000000000001,"5min":7.9390000000000001,"15min":19.018999999999998},"last":0.41199999999999998}]}]},{"osd":5,"up_from":39,"seq":167503724910,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1482876,"kb_used_data":1087220,"kb_used_omap":0,"kb_used_meta":395648,"kb_avail":92262276,"statfs":{"total":95995035648,"available":94476570624,"internally_reserved":0,"allocated":1113313280,"data_stored":1111878649,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":405143552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:40:53 2023","interfaces":[{"interface":"back","average":{"1min":1.3640000000000001,"5min":1.397,"15min":1.758},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.20699999999999999},"max":{"1min":5.8330000000000002,"5min":7.819,"15min":18.16},"last":0.34599999999999997},{"interface":"front","average":{"1min":1.3149999999999999,"5min":1.4770000000000001,"15min":1.665},"min":{"1min":0.29899999999999999,"5min":0.29499999999999998,"15min":0.245},"max":{"1min":5.157,"5min":7.9859999999999998,"15min":9.0570000000000004},"last":0.41499999999999998}]},{"osd":1,"last update":"Sat Sep 23 20:40:07 2023","interfaces":[{"interface":"back","average":{"1min":1.6799999999999999,"5min":2.0070000000000001,"15min":1.9179999999999999},"min":{"1min":0.26800000000000002,"5min":0.26300000000000001,"15min":0.253},"max":{"1min":5.71,"5min":36.588999999999999,"15min":36.588999999999999},"last":0.439},{"interface":"front","average":{"1min":1.2470000000000001,"5min":2.0150000000000001,"15min":1.6970000000000001},"min":{"1min":0.29799999999999999,"5min":0.27700000000000002,"15min":0.23499999999999999},"max":{"1min":5.298,"5min":36.920999999999999,"15min":36.920999999999999},"last":0.30199999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:40:53 2023","interfaces":[{"interface":"back","average":{"1min":2.0840000000000001,"5min":1.9470000000000001,"15min":1.9590000000000001},"min":{"1min":0.28000000000000003,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":9.2119999999999997,"5min":37.011000000000003,"15min":37.011000000000003},"last":0.45400000000000001},{"interface":"front","average":{"1min":1.5,"5min":1.74,"15min":1.978},"min":{"1min":0.32800000000000001,"5min":0.28999999999999998,"15min":0.255},"max":{"1min":5.4889999999999999,"5min":36.835000000000001,"15min":36.835000000000001},"last":0.32800000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:40:14 2023","interfaces":[{"interface":"back","average":{"1min":2.2320000000000002,"5min":2.153,"15min":2.2360000000000002},"min":{"1min":0.435,"5min":0.34699999999999998,"15min":0.25700000000000001},"max":{"1min":6.0309999999999997,"5min":36.716000000000001,"15min":36.716000000000001},"last":0.36099999999999999},{"interface":"front","average":{"1min":1.5289999999999999,"5min":2.2240000000000002,"15min":2.117},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.28799999999999998},"max":{"1min":8.3599999999999994,"5min":18.504000000000001,"15min":18.504000000000001},"last":0.47899999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:40:44 2023","interfaces":[{"interface":"back","average":{"1min":1.2689999999999999,"5min":0.90600000000000003,"15min":1.3140000000000001},"min":{"1min":0.29999999999999999,"5min":0.255,"15min":0.255},"max":{"1min":6.3780000000000001,"5min":6.9180000000000001,"15min":28.015999999999998},"last":0.314},{"interface":"front","average":{"1min":1.5760000000000001,"5min":0.93700000000000006,"15min":1.0940000000000001},"min":{"1min":0.32500000000000001,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":5.7469999999999999,"5min":6.8380000000000001,"15min":12.300000000000001},"last":0.38700000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:39:54 2023","interfaces":[{"interface":"back","average":{"1min":1.329,"5min":1.02,"15min":1.2589999999999999},"min":{"1min":0.32900000000000001,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":4.7679999999999998,"5min":6.9960000000000004,"15min":12.890000000000001},"last":0.40100000000000002},{"interface":"front","average":{"1min":1.1799999999999999,"5min":0.97499999999999998,"15min":1.2030000000000001},"min":{"1min":0.34000000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":4.7199999999999998,"5min":6.7930000000000001,"15min":16.265999999999998},"last":0.503}]},{"osd":7,"last update":"Sat Sep 23 20:40:14 2023","interfaces":[{"interface":"back","average":{"1min":1.3939999999999999,"5min":0.84299999999999997,"15min":1.244},"min":{"1min":0.38400000000000001,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":5.2430000000000003,"5min":5.2430000000000003,"15min":36.621000000000002},"last":0.42699999999999999},{"interface":"front","average":{"1min":1.0189999999999999,"5min":0.90700000000000003,"15min":1.151},"min":{"1min":0.35699999999999998,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":4.3250000000000002,"5min":6.5970000000000004,"15min":37.331000000000003},"last":0.375}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4722,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1360,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8206,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7156,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3154,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:41:02.200 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-23T20:41:02.266 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:41:02.297 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:02.384 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:41:02.387 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:02.387 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:41:02.488 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:41:02.515 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:02.598 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:02 smithi018 ceph-mon[184144]: from='client.46211 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:41:02.598 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:02 smithi018 ceph-mon[179647]: from='client.46211 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:41:02.631 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:41:02.633 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:02 smithi149 ceph-mon[170394]: from='client.46211 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:41:02.634 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:02.634 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:41:02.738 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:41:02.803 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:41:02.807 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:02.823 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:41:02.842 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:41:02.866 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:41:02.887 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:41:02.894 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:41:02.897 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:02.897 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:41:02.909 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:41:02.928 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:41:02.946 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:41:02.948 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:41:02.948 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:41:02.948 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:41:02.948 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:41:02.948 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:41:02.948 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:41:02.949 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:41:02.949 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:41:02.990 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:41:02.998 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:41:03.027 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:03.066 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:41:03.069 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:03.069 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:41:03.130 INFO:teuthology.orchestra.run.smithi149.stdout:{ 2023-09-23T20:41:03.130 INFO:teuthology.orchestra.run.smithi149.stdout: "size": 20, 2023-09-23T20:41:03.130 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 600, 2023-09-23T20:41:03.130 INFO:teuthology.orchestra.run.smithi149.stdout: "ops": [ 2023-09-23T20:41:03.130 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.130 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:84069 8.9 8:9824f39b:::benchmark_data_smithi018_341490_object84068:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e378)", 2023-09-23T20:41:03.130 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:37:13.327117+0000", 2023-09-23T20:41:03.131 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 229.800041733, 2023-09-23T20:41:03.131 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.3519245670000002, 2023-09-23T20:41:03.131 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:41:03.131 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:41:03.131 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:41:03.131 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:41:03.131 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:41:03.131 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 84069 2023-09-23T20:41:03.131 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.131 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:41:03.131 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.131 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:41:03.132 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.327117+0000", 2023-09-23T20:41:03.132 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:41:03.132 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.132 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.132 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:41:03.132 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.327117+0000", 2023-09-23T20:41:03.132 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:41:03.132 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.132 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.132 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:41:03.132 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.327116+0000", 2023-09-23T20:41:03.133 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:41:03.133 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.133 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.133 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:41:03.133 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.327148+0000", 2023-09-23T20:41:03.133 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.1741e-05 2023-09-23T20:41:03.133 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.133 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.133 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:41:03.133 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.327160+0000", 2023-09-23T20:41:03.133 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.185e-05 2023-09-23T20:41:03.133 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.134 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.134 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:41:03.134 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.327165+0000", 2023-09-23T20:41:03.134 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.1749999999999996e-06 2023-09-23T20:41:03.134 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.134 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.134 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:41:03.134 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.330726+0000", 2023-09-23T20:41:03.134 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00356121 2023-09-23T20:41:03.134 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.134 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.135 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:41:03.135 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.330874+0000", 2023-09-23T20:41:03.135 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000148097 2023-09-23T20:41:03.135 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.135 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.135 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:41:03.135 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.331021+0000", 2023-09-23T20:41:03.135 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00014648499999999999 2023-09-23T20:41:03.135 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.135 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.135 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:41:03.135 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.334676+0000", 2023-09-23T20:41:03.136 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0036551869999999998 2023-09-23T20:41:03.136 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.136 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.136 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:41:03.136 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.335154+0000", 2023-09-23T20:41:03.136 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00047863799999999998 2023-09-23T20:41:03.136 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.136 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.136 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:41:03.136 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.368539+0000", 2023-09-23T20:41:03.136 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.033384692000000001 2023-09-23T20:41:03.137 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.137 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.137 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:41:03.137 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.368550+0000", 2023-09-23T20:41:03.137 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1178e-05 2023-09-23T20:41:03.137 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.137 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.137 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:41:03.137 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:14.362533+0000", 2023-09-23T20:41:03.137 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.993982649 2023-09-23T20:41:03.137 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.137 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.138 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:41:03.138 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:14.362546+0000", 2023-09-23T20:41:03.138 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.2660000000000001e-05 2023-09-23T20:41:03.138 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.138 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.138 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:41:03.138 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:15.678962+0000", 2023-09-23T20:41:03.138 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.316415911 2023-09-23T20:41:03.138 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.138 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.138 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:41:03.139 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:15.679041+0000", 2023-09-23T20:41:03.139 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.9655999999999998e-05 2023-09-23T20:41:03.139 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:41:03.139 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:41:03.139 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:41:03.139 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.139 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.139 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:84081 8.9 8:97fd4d67:::benchmark_data_smithi018_341490_object84080:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e378)", 2023-09-23T20:41:03.139 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:37:13.341077+0000", 2023-09-23T20:41:03.139 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 229.78608166199999, 2023-09-23T20:41:03.139 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.3380652359999998, 2023-09-23T20:41:03.140 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:41:03.140 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:41:03.140 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:41:03.140 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:41:03.140 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:41:03.140 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 84081 2023-09-23T20:41:03.140 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.140 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:41:03.140 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.140 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:41:03.140 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.341077+0000", 2023-09-23T20:41:03.141 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:41:03.141 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.141 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.141 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:41:03.141 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.341077+0000", 2023-09-23T20:41:03.141 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:41:03.141 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.141 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.141 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:41:03.141 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.341075+0000", 2023-09-23T20:41:03.141 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999981 2023-09-23T20:41:03.141 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.142 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.142 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:41:03.142 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.341116+0000", 2023-09-23T20:41:03.142 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.1675000000000003e-05 2023-09-23T20:41:03.142 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.142 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.142 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:41:03.142 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.341135+0000", 2023-09-23T20:41:03.142 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.8110000000000001e-05 2023-09-23T20:41:03.142 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.142 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.143 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:41:03.143 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.341141+0000", 2023-09-23T20:41:03.143 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.849e-06 2023-09-23T20:41:03.143 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.143 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.143 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:41:03.143 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.341194+0000", 2023-09-23T20:41:03.143 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.2553000000000003e-05 2023-09-23T20:41:03.143 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.143 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.143 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:41:03.143 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.341313+0000", 2023-09-23T20:41:03.144 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00011935899999999999 2023-09-23T20:41:03.144 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.144 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.144 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:41:03.144 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.341466+0000", 2023-09-23T20:41:03.144 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00015230300000000001 2023-09-23T20:41:03.144 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.144 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.144 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:41:03.144 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.344704+0000", 2023-09-23T20:41:03.144 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0032382510000000002 2023-09-23T20:41:03.145 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.145 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.145 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:41:03.145 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.348055+0000", 2023-09-23T20:41:03.145 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.003350872 2023-09-23T20:41:03.145 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.145 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.145 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:41:03.145 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.368594+0000", 2023-09-23T20:41:03.145 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.020538948000000001 2023-09-23T20:41:03.145 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.145 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.146 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:41:03.146 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.368601+0000", 2023-09-23T20:41:03.146 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.6529999999999994e-06 2023-09-23T20:41:03.146 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.146 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.146 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:41:03.146 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:14.362580+0000", 2023-09-23T20:41:03.146 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.99397867299999998 2023-09-23T20:41:03.146 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.146 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.146 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:41:03.147 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:14.362589+0000", 2023-09-23T20:41:03.147 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.9509999999999995e-06 2023-09-23T20:41:03.147 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.147 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.147 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:41:03.147 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:15.679073+0000", 2023-09-23T20:41:03.147 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.316483598 2023-09-23T20:41:03.147 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.147 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.147 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:41:03.147 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:15.679142+0000", 2023-09-23T20:41:03.148 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.9389999999999998e-05 2023-09-23T20:41:03.148 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:41:03.148 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:41:03.148 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:41:03.148 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.148 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.148 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:105775 8.9 8:97ec68b7:::benchmark_data_smithi018_341490_object16452:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e411)", 2023-09-23T20:41:03.148 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:38:27.408390+0000", 2023-09-23T20:41:03.148 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 155.71876786999999, 2023-09-23T20:41:03.148 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.1501579759999991, 2023-09-23T20:41:03.148 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:41:03.148 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:41:03.148 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:41:03.149 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:41:03.149 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:41:03.149 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 105775 2023-09-23T20:41:03.149 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.149 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:41:03.149 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.149 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:41:03.149 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.408390+0000", 2023-09-23T20:41:03.149 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:41:03.149 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.149 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.149 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:41:03.149 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.408390+0000", 2023-09-23T20:41:03.150 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:41:03.150 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.150 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.150 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:41:03.150 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.408389+0000", 2023-09-23T20:41:03.150 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:41:03.150 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.150 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.150 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:41:03.150 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.408397+0000", 2023-09-23T20:41:03.150 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.7009999999999996e-06 2023-09-23T20:41:03.151 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.151 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.151 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:41:03.151 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.408411+0000", 2023-09-23T20:41:03.151 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.4228e-05 2023-09-23T20:41:03.151 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.151 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.151 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:41:03.151 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.408419+0000", 2023-09-23T20:41:03.151 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.5599999999999994e-06 2023-09-23T20:41:03.151 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.151 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.152 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:41:03.152 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.414829+0000", 2023-09-23T20:41:03.152 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0064094740000000001 2023-09-23T20:41:03.152 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.152 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.152 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:41:03.152 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.414843+0000", 2023-09-23T20:41:03.152 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.4561e-05 2023-09-23T20:41:03.152 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.152 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.152 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:41:03.152 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.556066+0000", 2023-09-23T20:41:03.153 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.1412221710000008 2023-09-23T20:41:03.153 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.153 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.153 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:41:03.153 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.558548+0000", 2023-09-23T20:41:03.153 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0024828070000000001 2023-09-23T20:41:03.153 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:41:03.153 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:41:03.153 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:41:03.153 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.153 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.153 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:105788 8.9 8:94f5ae14:::benchmark_data_smithi018_341490_object16465:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e411)", 2023-09-23T20:41:03.154 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:38:27.415518+0000", 2023-09-23T20:41:03.154 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 155.71163995399999, 2023-09-23T20:41:03.154 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 10.492815438999999, 2023-09-23T20:41:03.154 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:41:03.154 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:41:03.154 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:41:03.154 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:41:03.154 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:41:03.154 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 105788 2023-09-23T20:41:03.154 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.154 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:41:03.155 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.155 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:41:03.155 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.415518+0000", 2023-09-23T20:41:03.155 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:41:03.155 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.155 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.155 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:41:03.155 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.415518+0000", 2023-09-23T20:41:03.155 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:41:03.155 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.155 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.155 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:41:03.156 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.415517+0000", 2023-09-23T20:41:03.156 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:41:03.156 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.156 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.156 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:41:03.156 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.415524+0000", 2023-09-23T20:41:03.156 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.0530000000000001e-06 2023-09-23T20:41:03.156 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.156 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.156 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:41:03.156 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.415539+0000", 2023-09-23T20:41:03.156 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.4847e-05 2023-09-23T20:41:03.157 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.157 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.157 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:41:03.157 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.415548+0000", 2023-09-23T20:41:03.157 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.1600000000000004e-06 2023-09-23T20:41:03.157 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.157 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.157 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:41:03.157 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.423501+0000", 2023-09-23T20:41:03.157 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0079528459999999995 2023-09-23T20:41:03.157 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.157 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.158 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:41:03.158 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.423513+0000", 2023-09-23T20:41:03.158 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.2523e-05 2023-09-23T20:41:03.158 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.158 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.158 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:41:03.158 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.559074+0000", 2023-09-23T20:41:03.158 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.1355603290000005 2023-09-23T20:41:03.158 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.158 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.158 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for degraded object", 2023-09-23T20:41:03.158 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.567834+0000", 2023-09-23T20:41:03.159 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0087603060000000007 2023-09-23T20:41:03.159 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.159 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.159 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:41:03.159 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.866509+0000", 2023-09-23T20:41:03.159 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.2986752469999998 2023-09-23T20:41:03.159 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.159 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.159 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:41:03.159 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.869596+0000", 2023-09-23T20:41:03.159 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0030866829999999998 2023-09-23T20:41:03.159 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.160 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.160 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:41:03.160 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.875831+0000", 2023-09-23T20:41:03.160 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0062352570000000001 2023-09-23T20:41:03.160 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.160 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.160 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:41:03.160 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.893196+0000", 2023-09-23T20:41:03.160 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.017365463000000001 2023-09-23T20:41:03.160 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.160 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.161 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:41:03.161 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.898820+0000", 2023-09-23T20:41:03.161 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0056230400000000002 2023-09-23T20:41:03.161 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.161 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.161 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:41:03.161 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.899854+0000", 2023-09-23T20:41:03.161 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0010348670000000001 2023-09-23T20:41:03.161 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.161 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.161 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:41:03.161 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.901497+0000", 2023-09-23T20:41:03.162 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0016428160000000001 2023-09-23T20:41:03.162 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.162 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.162 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:41:03.162 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.908334+0000", 2023-09-23T20:41:03.162 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0068366219999999997 2023-09-23T20:41:03.162 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:41:03.162 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:41:03.162 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:41:03.162 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.162 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.163 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:105815 8.19 8:9f25a704:::benchmark_data_smithi018_341490_object16492:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e411)", 2023-09-23T20:41:03.163 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:38:27.621206+0000", 2023-09-23T20:41:03.163 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 155.505952734, 2023-09-23T20:41:03.163 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 10.285912701000001, 2023-09-23T20:41:03.163 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:41:03.163 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:41:03.163 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:41:03.163 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:41:03.163 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:41:03.163 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 105815 2023-09-23T20:41:03.163 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.163 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:41:03.164 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.164 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:41:03.164 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.621206+0000", 2023-09-23T20:41:03.164 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:41:03.164 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.164 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.164 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:41:03.164 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.621206+0000", 2023-09-23T20:41:03.164 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:41:03.164 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.164 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.165 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:41:03.165 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.621204+0000", 2023-09-23T20:41:03.165 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999981 2023-09-23T20:41:03.165 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.165 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.165 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:41:03.165 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.621213+0000", 2023-09-23T20:41:03.165 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.6399999999999992e-06 2023-09-23T20:41:03.165 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.165 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.165 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:41:03.165 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.621236+0000", 2023-09-23T20:41:03.166 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.2016000000000001e-05 2023-09-23T20:41:03.166 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.166 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.166 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:41:03.166 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.621244+0000", 2023-09-23T20:41:03.166 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.7339999999999997e-06 2023-09-23T20:41:03.166 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.166 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.166 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:41:03.166 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.621311+0000", 2023-09-23T20:41:03.166 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.7037000000000002e-05 2023-09-23T20:41:03.166 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.167 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.167 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:41:03.167 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.621338+0000", 2023-09-23T20:41:03.167 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.6506999999999999e-05 2023-09-23T20:41:03.167 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.167 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.167 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:41:03.167 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.559059+0000", 2023-09-23T20:41:03.167 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.9377208670000003 2023-09-23T20:41:03.167 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.167 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.167 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for degraded object", 2023-09-23T20:41:03.168 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.566949+0000", 2023-09-23T20:41:03.168 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0078899190000000004 2023-09-23T20:41:03.168 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.168 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.168 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:41:03.168 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.857769+0000", 2023-09-23T20:41:03.168 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.290820509 2023-09-23T20:41:03.168 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.168 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.168 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:41:03.168 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.860457+0000", 2023-09-23T20:41:03.168 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0026880620000000002 2023-09-23T20:41:03.169 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.169 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.169 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:41:03.169 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.865930+0000", 2023-09-23T20:41:03.169 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0054732979999999997 2023-09-23T20:41:03.169 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.169 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.169 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:41:03.169 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.881728+0000", 2023-09-23T20:41:03.169 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.015797813000000001 2023-09-23T20:41:03.169 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.169 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.170 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:41:03.170 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.897756+0000", 2023-09-23T20:41:03.170 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.016027501 2023-09-23T20:41:03.170 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.170 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.170 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:41:03.170 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.898850+0000", 2023-09-23T20:41:03.170 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.001094292 2023-09-23T20:41:03.170 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.170 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.170 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:41:03.171 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.900364+0000", 2023-09-23T20:41:03.171 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0015141250000000001 2023-09-23T20:41:03.171 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.171 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.171 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:41:03.171 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.907118+0000", 2023-09-23T20:41:03.171 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0067541490000000001 2023-09-23T20:41:03.171 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:41:03.171 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:41:03.171 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:41:03.171 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.171 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.172 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:105818 8.19 8:9e8d5bb0:::benchmark_data_smithi018_341490_object16495:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e411)", 2023-09-23T20:41:03.172 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:38:27.629200+0000", 2023-09-23T20:41:03.172 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 155.49795865300001, 2023-09-23T20:41:03.172 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 10.286819241, 2023-09-23T20:41:03.172 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:41:03.172 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:41:03.172 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:41:03.172 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:41:03.172 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:41:03.172 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 105818 2023-09-23T20:41:03.172 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.172 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:41:03.173 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.173 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:41:03.173 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.629200+0000", 2023-09-23T20:41:03.173 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:41:03.173 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.173 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.173 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:41:03.173 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.629200+0000", 2023-09-23T20:41:03.173 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:41:03.173 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.173 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.174 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:41:03.174 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.629198+0000", 2023-09-23T20:41:03.174 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999981 2023-09-23T20:41:03.174 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.174 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.174 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:41:03.174 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.629208+0000", 2023-09-23T20:41:03.174 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.9119999999999993e-06 2023-09-23T20:41:03.174 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.174 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.174 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:41:03.174 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.629231+0000", 2023-09-23T20:41:03.175 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.3258999999999999e-05 2023-09-23T20:41:03.175 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.175 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.175 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:41:03.175 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.629240+0000", 2023-09-23T20:41:03.175 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.5719999999999992e-06 2023-09-23T20:41:03.175 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.175 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.175 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:41:03.175 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.629296+0000", 2023-09-23T20:41:03.175 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.5918999999999998e-05 2023-09-23T20:41:03.175 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.176 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.176 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:41:03.176 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.629305+0000", 2023-09-23T20:41:03.176 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.0909999999999999e-06 2023-09-23T20:41:03.176 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.176 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.176 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:41:03.176 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.567292+0000", 2023-09-23T20:41:03.176 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.937986414 2023-09-23T20:41:03.176 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.176 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.176 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for degraded object", 2023-09-23T20:41:03.177 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.576261+0000", 2023-09-23T20:41:03.177 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0089692720000000004 2023-09-23T20:41:03.177 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.177 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.177 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:41:03.177 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.882072+0000", 2023-09-23T20:41:03.177 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.3058112469999998 2023-09-23T20:41:03.177 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.177 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.177 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:41:03.177 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.884583+0000", 2023-09-23T20:41:03.177 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0025109939999999999 2023-09-23T20:41:03.178 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.178 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.178 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:41:03.178 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.889724+0000", 2023-09-23T20:41:03.178 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0051411909999999998 2023-09-23T20:41:03.178 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.178 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.178 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:41:03.178 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.894676+0000", 2023-09-23T20:41:03.178 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0049515119999999999 2023-09-23T20:41:03.178 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.179 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.179 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:41:03.179 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.908009+0000", 2023-09-23T20:41:03.179 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.013332899 2023-09-23T20:41:03.179 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.179 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.179 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:41:03.179 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.908929+0000", 2023-09-23T20:41:03.179 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00092006300000000004 2023-09-23T20:41:03.179 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.179 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.179 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:41:03.180 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.910490+0000", 2023-09-23T20:41:03.180 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.001560931 2023-09-23T20:41:03.180 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.180 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.180 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:41:03.180 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.916019+0000", 2023-09-23T20:41:03.180 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0055290570000000004 2023-09-23T20:41:03.180 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:41:03.180 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:41:03.180 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:41:03.180 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.181 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.181 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "MOSDPGPushReply(8.19 413/410 [PushReplyOp(8:9f25a704:::benchmark_data_smithi018_341490_object16492:head)])", 2023-09-23T20:41:03.181 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:38:35.569213+0000", 2023-09-23T20:41:03.181 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 147.55794536600001, 2023-09-23T20:41:03.181 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.2881636309999998, 2023-09-23T20:41:03.181 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:41:03.181 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:41:03.181 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:41:03.181 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.181 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:41:03.181 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.569213+0000", 2023-09-23T20:41:03.181 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:41:03.182 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.182 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.182 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:41:03.182 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.569213+0000", 2023-09-23T20:41:03.182 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:41:03.182 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.182 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.182 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:41:03.182 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.569213+0000", 2023-09-23T20:41:03.182 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:41:03.182 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.183 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.183 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:41:03.183 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.569217+0000", 2023-09-23T20:41:03.183 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.5480000000000003e-06 2023-09-23T20:41:03.183 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.183 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.183 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:41:03.183 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.569231+0000", 2023-09-23T20:41:03.183 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.3505e-05 2023-09-23T20:41:03.183 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.183 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.184 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:41:03.184 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.569241+0000", 2023-09-23T20:41:03.184 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0695e-05 2023-09-23T20:41:03.184 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.184 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.184 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:41:03.184 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.845386+0000", 2023-09-23T20:41:03.184 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.2761449049999998 2023-09-23T20:41:03.184 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.184 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.184 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:41:03.184 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.857377+0000", 2023-09-23T20:41:03.185 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.011990341999999999 2023-09-23T20:41:03.185 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:41:03.185 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:41:03.185 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:41:03.185 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.185 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.185 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "MOSDPGPushReply(8.9 413/411 [PushReplyOp(8:94f5ae14:::benchmark_data_smithi018_341490_object16465:head)])", 2023-09-23T20:41:03.185 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:38:35.570981+0000", 2023-09-23T20:41:03.185 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 147.556177182, 2023-09-23T20:41:03.185 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.295177131, 2023-09-23T20:41:03.185 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:41:03.185 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:41:03.186 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:41:03.186 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.186 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:41:03.186 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.570981+0000", 2023-09-23T20:41:03.186 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:41:03.186 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.186 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.186 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:41:03.186 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.570981+0000", 2023-09-23T20:41:03.186 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:41:03.186 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.187 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.187 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:41:03.187 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.570981+0000", 2023-09-23T20:41:03.187 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:41:03.187 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.187 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.187 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:41:03.187 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.570985+0000", 2023-09-23T20:41:03.187 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.2690000000000001e-06 2023-09-23T20:41:03.187 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.187 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.187 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:41:03.188 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.570995+0000", 2023-09-23T20:41:03.188 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.9799999999999993e-06 2023-09-23T20:41:03.188 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.188 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.188 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:41:03.188 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.570999+0000", 2023-09-23T20:41:03.188 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.5739999999999997e-06 2023-09-23T20:41:03.188 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.188 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.188 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:41:03.188 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.852442+0000", 2023-09-23T20:41:03.188 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.2814426320000001 2023-09-23T20:41:03.189 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.189 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.189 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:41:03.189 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.866158+0000", 2023-09-23T20:41:03.189 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.013716178000000001 2023-09-23T20:41:03.189 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:41:03.189 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:41:03.189 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:41:03.189 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.189 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.189 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "MOSDPGPushReply(8.19 413/410 [PushReplyOp(8:9e8d5bb0:::benchmark_data_smithi018_341490_object16495:head)])", 2023-09-23T20:41:03.189 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:38:35.577963+0000", 2023-09-23T20:41:03.190 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 147.54919502499999, 2023-09-23T20:41:03.190 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.303767224, 2023-09-23T20:41:03.190 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:41:03.190 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:41:03.190 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:41:03.190 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.190 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:41:03.190 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.577963+0000", 2023-09-23T20:41:03.190 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:41:03.190 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.190 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.190 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:41:03.191 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.577963+0000", 2023-09-23T20:41:03.191 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:41:03.191 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.191 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.191 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:41:03.191 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.577963+0000", 2023-09-23T20:41:03.191 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:41:03.191 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.191 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.191 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:41:03.191 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.577968+0000", 2023-09-23T20:41:03.191 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.13e-06 2023-09-23T20:41:03.192 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.192 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.192 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:41:03.192 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.577980+0000", 2023-09-23T20:41:03.192 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1749000000000001e-05 2023-09-23T20:41:03.192 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.192 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.192 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:41:03.192 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.577986+0000", 2023-09-23T20:41:03.192 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.8210000000000004e-06 2023-09-23T20:41:03.192 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.192 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.193 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:41:03.193 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.869571+0000", 2023-09-23T20:41:03.193 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.2915846719999999 2023-09-23T20:41:03.193 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.193 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.193 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:41:03.193 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.881731+0000", 2023-09-23T20:41:03.193 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.012159467 2023-09-23T20:41:03.193 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:41:03.193 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:41:03.193 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:41:03.193 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.193 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.194 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:106043 8.9 8:949649ba:::benchmark_data_smithi018_341490_object16720:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e414)", 2023-09-23T20:41:03.194 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:38:35.582779+0000", 2023-09-23T20:41:03.194 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 147.54437933099999, 2023-09-23T20:41:03.194 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.7561004659999999, 2023-09-23T20:41:03.194 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:41:03.194 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:41:03.194 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:41:03.194 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:41:03.194 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:41:03.194 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 106043 2023-09-23T20:41:03.194 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.195 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:41:03.195 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.195 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:41:03.195 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.582779+0000", 2023-09-23T20:41:03.195 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:41:03.195 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.195 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.195 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:41:03.195 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.582779+0000", 2023-09-23T20:41:03.195 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:41:03.195 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.195 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.196 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:41:03.196 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.582778+0000", 2023-09-23T20:41:03.196 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:41:03.196 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.196 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.196 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:41:03.196 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.582784+0000", 2023-09-23T20:41:03.196 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.0050000000000001e-06 2023-09-23T20:41:03.196 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.196 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.196 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:41:03.196 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.582798+0000", 2023-09-23T20:41:03.197 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.3788000000000001e-05 2023-09-23T20:41:03.197 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.197 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.197 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:41:03.197 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.582804+0000", 2023-09-23T20:41:03.197 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.6760000000000001e-06 2023-09-23T20:41:03.197 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.197 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.197 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:41:03.197 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.238120+0000", 2023-09-23T20:41:03.197 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.655315308 2023-09-23T20:41:03.198 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.198 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.198 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for degraded object", 2023-09-23T20:41:03.198 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.246651+0000", 2023-09-23T20:41:03.198 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0085312070000000007 2023-09-23T20:41:03.198 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.198 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.198 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:41:03.198 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.290876+0000", 2023-09-23T20:41:03.198 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.044225546999999997 2023-09-23T20:41:03.198 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.199 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.199 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:41:03.199 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.294002+0000", 2023-09-23T20:41:03.199 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0031256779999999998 2023-09-23T20:41:03.199 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.199 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.199 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:41:03.199 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.300293+0000", 2023-09-23T20:41:03.199 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0062908959999999998 2023-09-23T20:41:03.199 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.199 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.199 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:41:03.200 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.328008+0000", 2023-09-23T20:41:03.200 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.027715281000000001 2023-09-23T20:41:03.200 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.200 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.200 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:41:03.200 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.329088+0000", 2023-09-23T20:41:03.200 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.001079775 2023-09-23T20:41:03.200 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.200 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.200 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:41:03.200 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.330303+0000", 2023-09-23T20:41:03.200 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0012148199999999999 2023-09-23T20:41:03.200 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.201 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.201 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:41:03.201 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.332003+0000", 2023-09-23T20:41:03.201 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0016998580000000001 2023-09-23T20:41:03.201 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.201 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.201 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:41:03.201 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.338879+0000", 2023-09-23T20:41:03.201 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0068769119999999998 2023-09-23T20:41:03.201 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:41:03.201 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:41:03.201 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:41:03.202 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.202 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.202 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:106806 8.19 8:9c697886:::benchmark_data_smithi018_341490_object17483:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e418)", 2023-09-23T20:41:03.202 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:38:40.203477+0000", 2023-09-23T20:41:03.202 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 142.92368162099999, 2023-09-23T20:41:03.202 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.58349355700000005, 2023-09-23T20:41:03.202 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:41:03.202 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:41:03.202 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:41:03.202 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:41:03.202 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:41:03.203 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 106806 2023-09-23T20:41:03.203 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.203 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:41:03.203 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.203 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:41:03.203 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:40.203477+0000", 2023-09-23T20:41:03.203 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:41:03.203 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.203 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.203 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:41:03.203 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:40.203477+0000", 2023-09-23T20:41:03.203 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:41:03.204 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.204 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.204 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:41:03.204 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:40.203475+0000", 2023-09-23T20:41:03.204 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:41:03.204 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.204 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.204 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:41:03.204 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:40.203482+0000", 2023-09-23T20:41:03.204 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.2190000000000001e-06 2023-09-23T20:41:03.204 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.204 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.205 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:41:03.205 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:40.203496+0000", 2023-09-23T20:41:03.205 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.3974e-05 2023-09-23T20:41:03.205 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.205 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.205 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:41:03.205 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:40.203504+0000", 2023-09-23T20:41:03.205 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.32e-06 2023-09-23T20:41:03.205 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.205 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.205 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:41:03.206 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:40.554479+0000", 2023-09-23T20:41:03.206 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.35097457500000001 2023-09-23T20:41:03.206 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.206 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.206 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for degraded object", 2023-09-23T20:41:03.206 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:40.561879+0000", 2023-09-23T20:41:03.206 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0074007860000000003 2023-09-23T20:41:03.206 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.206 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.206 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:41:03.206 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:40.619042+0000", 2023-09-23T20:41:03.207 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.057163024999999999 2023-09-23T20:41:03.207 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.207 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.207 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:41:03.207 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:40.621716+0000", 2023-09-23T20:41:03.207 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0026739810000000002 2023-09-23T20:41:03.207 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.207 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.207 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:41:03.207 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:40.627239+0000", 2023-09-23T20:41:03.207 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0055227169999999999 2023-09-23T20:41:03.207 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.208 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.208 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:41:03.208 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:40.646592+0000", 2023-09-23T20:41:03.208 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.019353386 2023-09-23T20:41:03.208 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.208 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.208 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:41:03.208 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:40.778031+0000", 2023-09-23T20:41:03.208 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.131438103 2023-09-23T20:41:03.208 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.208 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.208 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:41:03.209 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:40.779250+0000", 2023-09-23T20:41:03.209 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0012194829999999999 2023-09-23T20:41:03.209 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.209 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.209 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:41:03.209 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:40.781214+0000", 2023-09-23T20:41:03.209 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.001964389 2023-09-23T20:41:03.209 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.209 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.209 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:41:03.209 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:40.786970+0000", 2023-09-23T20:41:03.210 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0057558599999999998 2023-09-23T20:41:03.210 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:41:03.210 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:41:03.210 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:41:03.210 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.210 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.210 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:106827 8.9 8:9397eda9:::benchmark_data_smithi018_341490_object17504:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e418)", 2023-09-23T20:41:03.210 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:38:40.241109+0000", 2023-09-23T20:41:03.210 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 142.88604978500001, 2023-09-23T20:41:03.210 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.55994143200000002, 2023-09-23T20:41:03.210 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:41:03.210 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:41:03.211 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:41:03.211 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:41:03.211 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:41:03.211 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 106827 2023-09-23T20:41:03.211 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.211 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:41:03.211 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.211 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:41:03.211 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:40.241109+0000", 2023-09-23T20:41:03.211 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:41:03.211 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.211 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.212 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:41:03.212 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:40.241109+0000", 2023-09-23T20:41:03.212 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:41:03.212 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.212 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.212 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:41:03.212 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:40.241108+0000", 2023-09-23T20:41:03.212 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:41:03.212 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.212 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.212 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:41:03.213 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:40.241112+0000", 2023-09-23T20:41:03.213 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.6720000000000003e-06 2023-09-23T20:41:03.213 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.213 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.213 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:41:03.213 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:40.241121+0000", 2023-09-23T20:41:03.213 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.9160000000000007e-06 2023-09-23T20:41:03.213 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.213 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.213 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:41:03.213 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:40.241126+0000", 2023-09-23T20:41:03.213 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.7709999999999997e-06 2023-09-23T20:41:03.214 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.214 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.214 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:41:03.214 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:40.554772+0000", 2023-09-23T20:41:03.214 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.31364581800000002 2023-09-23T20:41:03.214 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.214 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.214 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for degraded object", 2023-09-23T20:41:03.214 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:40.563264+0000", 2023-09-23T20:41:03.214 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0084926870000000005 2023-09-23T20:41:03.214 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.214 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.215 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:41:03.215 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:40.622402+0000", 2023-09-23T20:41:03.215 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.059138010999999997 2023-09-23T20:41:03.215 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.215 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.215 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:41:03.215 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:40.625707+0000", 2023-09-23T20:41:03.215 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0033045940000000001 2023-09-23T20:41:03.215 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.215 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.215 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:41:03.215 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:40.632910+0000", 2023-09-23T20:41:03.216 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0072032349999999997 2023-09-23T20:41:03.216 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.216 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.216 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:41:03.216 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:40.675242+0000", 2023-09-23T20:41:03.216 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.042331972000000002 2023-09-23T20:41:03.216 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.216 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.216 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:41:03.216 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:40.780889+0000", 2023-09-23T20:41:03.216 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.10564657199999999 2023-09-23T20:41:03.216 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.217 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.217 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:41:03.217 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:40.788709+0000", 2023-09-23T20:41:03.217 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0078198160000000003 2023-09-23T20:41:03.217 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.217 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.217 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:41:03.217 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:40.791167+0000", 2023-09-23T20:41:03.217 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.002458661 2023-09-23T20:41:03.217 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.217 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.218 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:41:03.218 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:40.801050+0000", 2023-09-23T20:41:03.218 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0098826720000000003 2023-09-23T20:41:03.218 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:41:03.218 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:41:03.218 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:41:03.218 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.218 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.218 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:107411 8.9 8:93727162:::benchmark_data_smithi018_341490_object18088:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e419)", 2023-09-23T20:41:03.218 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:38:41.536053+0000", 2023-09-23T20:41:03.218 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 141.59110567100001, 2023-09-23T20:41:03.218 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.545918711, 2023-09-23T20:41:03.219 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:41:03.219 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:41:03.219 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:41:03.219 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:41:03.219 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:41:03.219 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 107411 2023-09-23T20:41:03.219 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.219 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:41:03.219 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.219 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:41:03.219 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:41.536053+0000", 2023-09-23T20:41:03.219 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:41:03.220 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.220 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.220 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:41:03.220 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:41.536053+0000", 2023-09-23T20:41:03.220 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:41:03.220 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.220 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.220 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:41:03.220 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:41.536052+0000", 2023-09-23T20:41:03.220 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:41:03.220 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.220 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.221 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:41:03.221 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:41.536057+0000", 2023-09-23T20:41:03.221 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.31e-06 2023-09-23T20:41:03.221 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.221 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.221 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:41:03.221 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:41.536070+0000", 2023-09-23T20:41:03.221 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.3277e-05 2023-09-23T20:41:03.221 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.221 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.221 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:41:03.221 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:41.536075+0000", 2023-09-23T20:41:03.222 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.1390000000000003e-06 2023-09-23T20:41:03.222 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.222 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.222 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:41:03.222 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:41.981654+0000", 2023-09-23T20:41:03.222 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.445578633 2023-09-23T20:41:03.222 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.222 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.222 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for degraded object", 2023-09-23T20:41:03.222 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:41.990299+0000", 2023-09-23T20:41:03.222 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0086450859999999997 2023-09-23T20:41:03.223 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.223 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.223 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:41:03.223 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:42.014870+0000", 2023-09-23T20:41:03.223 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.024571401 2023-09-23T20:41:03.223 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.223 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.223 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:41:03.223 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:42.018005+0000", 2023-09-23T20:41:03.223 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0031345050000000001 2023-09-23T20:41:03.223 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.223 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.224 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:41:03.224 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:42.024399+0000", 2023-09-23T20:41:03.224 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0063945089999999996 2023-09-23T20:41:03.224 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.224 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.224 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:41:03.224 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:42.058397+0000", 2023-09-23T20:41:03.224 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.033997918000000002 2023-09-23T20:41:03.224 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.224 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.224 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:41:03.225 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:42.072100+0000", 2023-09-23T20:41:03.225 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.01370296 2023-09-23T20:41:03.225 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.225 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.225 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:41:03.225 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:42.073284+0000", 2023-09-23T20:41:03.225 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.001183447 2023-09-23T20:41:03.225 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.225 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.225 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:41:03.225 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:42.075038+0000", 2023-09-23T20:41:03.225 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0017545799999999999 2023-09-23T20:41:03.226 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.226 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.226 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:41:03.226 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:42.081971+0000", 2023-09-23T20:41:03.226 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0069330750000000003 2023-09-23T20:41:03.226 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:41:03.226 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:41:03.226 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:41:03.226 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.226 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.226 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:114780 8.19 8:99d20faf:::benchmark_data_smithi018_341490_object25457:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e421)", 2023-09-23T20:41:03.227 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:38:57.795331+0000", 2023-09-23T20:41:03.227 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 125.3318278, 2023-09-23T20:41:03.227 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.57046175899999996, 2023-09-23T20:41:03.227 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:41:03.227 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:41:03.227 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:41:03.227 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:41:03.227 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:41:03.227 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 114780 2023-09-23T20:41:03.227 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.227 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:41:03.227 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.228 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:41:03.228 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:57.795331+0000", 2023-09-23T20:41:03.228 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:41:03.228 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.228 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.228 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:41:03.228 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:57.795331+0000", 2023-09-23T20:41:03.228 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:41:03.228 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.228 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.228 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:41:03.229 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:57.795330+0000", 2023-09-23T20:41:03.229 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:41:03.229 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.229 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.229 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:41:03.229 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:57.795334+0000", 2023-09-23T20:41:03.229 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.8470000000000003e-06 2023-09-23T20:41:03.229 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.229 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.229 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:41:03.229 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:57.795346+0000", 2023-09-23T20:41:03.230 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1746000000000001e-05 2023-09-23T20:41:03.230 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.230 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.230 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:41:03.230 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:57.795351+0000", 2023-09-23T20:41:03.230 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.0629999999999996e-06 2023-09-23T20:41:03.230 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.230 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.230 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:41:03.230 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:58.258901+0000", 2023-09-23T20:41:03.230 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.463549977 2023-09-23T20:41:03.231 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.231 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.231 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for degraded object", 2023-09-23T20:41:03.231 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:58.267411+0000", 2023-09-23T20:41:03.231 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0085102670000000002 2023-09-23T20:41:03.231 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.231 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.231 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:41:03.231 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:58.297813+0000", 2023-09-23T20:41:03.231 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.030401557999999999 2023-09-23T20:41:03.231 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.231 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.231 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:41:03.232 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:58.300388+0000", 2023-09-23T20:41:03.232 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0025745809999999998 2023-09-23T20:41:03.232 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.232 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.232 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:41:03.232 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:58.305183+0000", 2023-09-23T20:41:03.232 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0047954850000000004 2023-09-23T20:41:03.232 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.232 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.232 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:41:03.232 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:58.331209+0000", 2023-09-23T20:41:03.232 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.026026198 2023-09-23T20:41:03.233 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.233 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.233 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:41:03.233 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:58.357178+0000", 2023-09-23T20:41:03.233 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.025968600000000001 2023-09-23T20:41:03.233 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.233 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.233 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:41:03.233 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:58.358388+0000", 2023-09-23T20:41:03.233 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.001210051 2023-09-23T20:41:03.233 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.233 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.234 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:41:03.234 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:58.359794+0000", 2023-09-23T20:41:03.234 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0014056769999999999 2023-09-23T20:41:03.234 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.234 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.234 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:41:03.234 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:58.365792+0000", 2023-09-23T20:41:03.234 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0059987219999999997 2023-09-23T20:41:03.234 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:41:03.234 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:41:03.234 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:41:03.235 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.235 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.235 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:117102 8.9 8:94476c01:::benchmark_data_smithi018_341490_object27779:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e422)", 2023-09-23T20:41:03.235 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:39:02.534326+0000", 2023-09-23T20:41:03.235 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 120.592832363, 2023-09-23T20:41:03.235 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.54634636000000003, 2023-09-23T20:41:03.235 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:41:03.235 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:41:03.235 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:41:03.235 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:41:03.235 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:41:03.235 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 117102 2023-09-23T20:41:03.235 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.236 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:41:03.236 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.236 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:41:03.236 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:02.534326+0000", 2023-09-23T20:41:03.236 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:41:03.236 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.236 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.236 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:41:03.236 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:02.534326+0000", 2023-09-23T20:41:03.236 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:41:03.236 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.236 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.236 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:41:03.237 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:02.534325+0000", 2023-09-23T20:41:03.237 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:41:03.237 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.237 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.237 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:41:03.237 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:02.534330+0000", 2023-09-23T20:41:03.237 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.5400000000000003e-06 2023-09-23T20:41:03.237 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.237 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.237 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:41:03.237 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:02.534342+0000", 2023-09-23T20:41:03.237 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.2149e-05 2023-09-23T20:41:03.238 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.238 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.238 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:41:03.238 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:02.534349+0000", 2023-09-23T20:41:03.238 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.7449999999999998e-06 2023-09-23T20:41:03.238 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.238 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.238 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:41:03.238 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:02.986050+0000", 2023-09-23T20:41:03.238 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.45170089200000002 2023-09-23T20:41:03.238 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.238 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.239 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for degraded object", 2023-09-23T20:41:03.239 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:02.994585+0000", 2023-09-23T20:41:03.239 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0085348030000000005 2023-09-23T20:41:03.239 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.239 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.239 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:41:03.239 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:03.017680+0000", 2023-09-23T20:41:03.239 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.023095015999999999 2023-09-23T20:41:03.239 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.239 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.239 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:41:03.239 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:03.020526+0000", 2023-09-23T20:41:03.240 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.002845686 2023-09-23T20:41:03.240 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.240 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.240 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:41:03.240 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:03.026613+0000", 2023-09-23T20:41:03.240 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0060874500000000003 2023-09-23T20:41:03.240 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.240 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.240 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:41:03.240 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:03.043901+0000", 2023-09-23T20:41:03.240 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.017287535 2023-09-23T20:41:03.240 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.241 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.241 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:41:03.241 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:03.057977+0000", 2023-09-23T20:41:03.241 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.014076422 2023-09-23T20:41:03.241 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.241 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.241 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:41:03.241 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:03.069852+0000", 2023-09-23T20:41:03.241 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0118748 2023-09-23T20:41:03.241 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.241 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.242 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:41:03.242 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:03.072063+0000", 2023-09-23T20:41:03.242 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0022112500000000001 2023-09-23T20:41:03.242 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.242 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.242 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:41:03.242 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:03.080672+0000", 2023-09-23T20:41:03.242 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0086092779999999997 2023-09-23T20:41:03.242 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:41:03.242 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:41:03.242 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:41:03.242 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.243 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.243 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:117951 8.9 8:9109a280:::benchmark_data_smithi018_341490_object28628:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e422)", 2023-09-23T20:41:03.243 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:39:04.627035+0000", 2023-09-23T20:41:03.243 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 118.500123285, 2023-09-23T20:41:03.243 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.56794783400000004, 2023-09-23T20:41:03.243 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:41:03.243 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:41:03.243 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:41:03.243 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:41:03.243 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:41:03.243 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 117951 2023-09-23T20:41:03.243 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.244 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:41:03.244 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.244 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:41:03.244 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:04.627035+0000", 2023-09-23T20:41:03.244 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:41:03.244 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.244 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.244 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:41:03.244 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:04.627035+0000", 2023-09-23T20:41:03.244 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:41:03.244 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.244 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.245 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:41:03.245 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:04.627034+0000", 2023-09-23T20:41:03.245 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:41:03.245 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.245 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.245 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:41:03.245 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:04.627041+0000", 2023-09-23T20:41:03.245 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.6450000000000005e-06 2023-09-23T20:41:03.245 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.245 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.245 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:41:03.245 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:04.627055+0000", 2023-09-23T20:41:03.246 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.3978999999999999e-05 2023-09-23T20:41:03.246 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.246 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.246 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:41:03.246 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:04.627064+0000", 2023-09-23T20:41:03.246 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.3590000000000008e-06 2023-09-23T20:41:03.246 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.246 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.246 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:41:03.246 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:05.078303+0000", 2023-09-23T20:41:03.246 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.45123901199999999 2023-09-23T20:41:03.246 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.247 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.247 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for degraded object", 2023-09-23T20:41:03.247 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:05.087537+0000", 2023-09-23T20:41:03.247 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0092337489999999994 2023-09-23T20:41:03.247 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.247 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.247 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:41:03.247 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:05.118409+0000", 2023-09-23T20:41:03.247 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.030872439000000002 2023-09-23T20:41:03.247 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.247 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.247 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:41:03.248 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:05.121485+0000", 2023-09-23T20:41:03.248 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0030759479999999998 2023-09-23T20:41:03.248 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.248 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.248 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:41:03.248 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:05.126808+0000", 2023-09-23T20:41:03.248 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0053229540000000004 2023-09-23T20:41:03.248 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.248 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.248 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:41:03.248 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:05.153916+0000", 2023-09-23T20:41:03.248 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.027107971000000002 2023-09-23T20:41:03.249 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.249 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.249 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:41:03.249 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:05.179878+0000", 2023-09-23T20:41:03.249 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.025962030000000001 2023-09-23T20:41:03.249 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.249 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.249 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:41:03.249 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:05.187231+0000", 2023-09-23T20:41:03.249 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.007353488 2023-09-23T20:41:03.249 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.250 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.250 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:41:03.250 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:05.188772+0000", 2023-09-23T20:41:03.250 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0015410350000000001 2023-09-23T20:41:03.250 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.250 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.250 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:41:03.250 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:05.194983+0000", 2023-09-23T20:41:03.250 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0062104589999999998 2023-09-23T20:41:03.250 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:41:03.250 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:41:03.250 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:41:03.251 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.251 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.251 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:118954 8.9 8:94282a85:::benchmark_data_smithi018_341490_object29631:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e422)", 2023-09-23T20:41:03.251 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:39:06.633258+0000", 2023-09-23T20:41:03.251 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 116.49390006500001, 2023-09-23T20:41:03.251 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.54584307600000004, 2023-09-23T20:41:03.251 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:41:03.251 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:41:03.251 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:41:03.251 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:41:03.251 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:41:03.251 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 118954 2023-09-23T20:41:03.252 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.252 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:41:03.252 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.252 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:41:03.252 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:06.633258+0000", 2023-09-23T20:41:03.252 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:41:03.252 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.252 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.252 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:41:03.252 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:06.633258+0000", 2023-09-23T20:41:03.252 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:41:03.252 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.252 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.253 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:41:03.253 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:06.633257+0000", 2023-09-23T20:41:03.253 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:41:03.253 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.253 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.253 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:41:03.253 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:06.633262+0000", 2023-09-23T20:41:03.253 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.0860000000000004e-06 2023-09-23T20:41:03.253 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.253 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.253 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:41:03.253 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:06.633270+0000", 2023-09-23T20:41:03.254 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.8619999999999992e-06 2023-09-23T20:41:03.254 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.254 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.254 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:41:03.254 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:06.633274+0000", 2023-09-23T20:41:03.254 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.0280000000000002e-06 2023-09-23T20:41:03.254 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.254 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.254 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:41:03.254 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:07.056940+0000", 2023-09-23T20:41:03.254 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.42366564000000001 2023-09-23T20:41:03.255 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.255 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.255 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for degraded object", 2023-09-23T20:41:03.255 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:07.070034+0000", 2023-09-23T20:41:03.255 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.013094006 2023-09-23T20:41:03.255 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.255 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.255 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:41:03.255 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:07.103251+0000", 2023-09-23T20:41:03.255 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.033216491000000001 2023-09-23T20:41:03.255 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.256 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.256 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:41:03.256 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:07.105891+0000", 2023-09-23T20:41:03.256 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0026404670000000001 2023-09-23T20:41:03.256 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.256 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.256 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:41:03.256 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:07.111462+0000", 2023-09-23T20:41:03.256 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0055709499999999999 2023-09-23T20:41:03.256 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.256 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.256 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:41:03.257 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:07.157460+0000", 2023-09-23T20:41:03.257 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.045998209999999998 2023-09-23T20:41:03.257 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.257 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.257 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:41:03.257 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:07.170250+0000", 2023-09-23T20:41:03.257 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.012790073000000001 2023-09-23T20:41:03.257 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.257 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.257 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:41:03.257 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:07.171476+0000", 2023-09-23T20:41:03.257 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.001225514 2023-09-23T20:41:03.258 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.258 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.258 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:41:03.258 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:07.172978+0000", 2023-09-23T20:41:03.258 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.001502275 2023-09-23T20:41:03.258 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.258 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.258 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:41:03.258 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:07.179101+0000", 2023-09-23T20:41:03.258 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0061233290000000003 2023-09-23T20:41:03.258 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:41:03.259 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:41:03.259 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:41:03.259 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.259 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.259 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:119363 8.19 8:9b94513b:::benchmark_data_smithi018_341490_object30040:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e422)", 2023-09-23T20:41:03.259 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:39:07.530370+0000", 2023-09-23T20:41:03.259 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 115.596788796, 2023-09-23T20:41:03.259 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.56285548100000005, 2023-09-23T20:41:03.259 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:41:03.259 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:41:03.259 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:41:03.259 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:41:03.259 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:41:03.260 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 119363 2023-09-23T20:41:03.260 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.260 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:41:03.260 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.260 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:41:03.260 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:07.530370+0000", 2023-09-23T20:41:03.260 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:41:03.260 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.260 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.260 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:41:03.260 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:07.530370+0000", 2023-09-23T20:41:03.260 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:41:03.261 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.261 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.261 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:41:03.261 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:07.530368+0000", 2023-09-23T20:41:03.261 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:41:03.261 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.261 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.261 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:41:03.261 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:07.530374+0000", 2023-09-23T20:41:03.261 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.7250000000000002e-06 2023-09-23T20:41:03.261 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.262 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.262 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:41:03.262 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:07.530387+0000", 2023-09-23T20:41:03.262 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.3284e-05 2023-09-23T20:41:03.262 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.262 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.262 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:41:03.262 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:07.530393+0000", 2023-09-23T20:41:03.262 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.3789999999999997e-06 2023-09-23T20:41:03.262 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.262 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.263 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:41:03.263 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:07.948569+0000", 2023-09-23T20:41:03.263 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.41817622599999998 2023-09-23T20:41:03.263 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.263 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.263 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for degraded object", 2023-09-23T20:41:03.263 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:07.959914+0000", 2023-09-23T20:41:03.263 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.011345434999999999 2023-09-23T20:41:03.263 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.263 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.263 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:41:03.263 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:07.999724+0000", 2023-09-23T20:41:03.264 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.039809125000000001 2023-09-23T20:41:03.264 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.264 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.264 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:41:03.264 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:08.002702+0000", 2023-09-23T20:41:03.264 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0029784859999999998 2023-09-23T20:41:03.264 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.264 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.264 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:41:03.264 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:08.008078+0000", 2023-09-23T20:41:03.264 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0053759610000000003 2023-09-23T20:41:03.264 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.265 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.265 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:41:03.265 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:08.036970+0000", 2023-09-23T20:41:03.265 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.028891567999999999 2023-09-23T20:41:03.265 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.265 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.265 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:41:03.265 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:08.084972+0000", 2023-09-23T20:41:03.265 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.048002351999999998 2023-09-23T20:41:03.265 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.266 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.266 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:41:03.266 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:08.086297+0000", 2023-09-23T20:41:03.266 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.001325562 2023-09-23T20:41:03.266 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.266 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.266 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:41:03.266 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:08.087523+0000", 2023-09-23T20:41:03.266 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.001225659 2023-09-23T20:41:03.266 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.266 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.266 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:41:03.267 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:08.093225+0000", 2023-09-23T20:41:03.267 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0057018900000000003 2023-09-23T20:41:03.267 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:41:03.267 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:41:03.267 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:41:03.267 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.267 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.267 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "MOSDPGPush(8.19 427/410 [PushOp(8:9f37acf0:::benchmark_data_smithi018_341490_object84819:head, version: 394'5338, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(8:9f37acf0:::benchmark_data_smithi018_341490_object84819:head@394'5338, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-09-23T20:41:03.267 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:39:26.795989+0000", 2023-09-23T20:41:03.267 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 96.331169165000006, 2023-09-23T20:41:03.267 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 18.381641347999999, 2023-09-23T20:41:03.268 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:41:03.268 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "started", 2023-09-23T20:41:03.268 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:41:03.268 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.268 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:41:03.268 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:26.795989+0000", 2023-09-23T20:41:03.268 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:41:03.268 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.268 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.268 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:41:03.268 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:26.795989+0000", 2023-09-23T20:41:03.268 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:41:03.269 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.269 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.269 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:41:03.269 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:26.795988+0000", 2023-09-23T20:41:03.269 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:41:03.269 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.269 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.269 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:41:03.269 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:26.796034+0000", 2023-09-23T20:41:03.269 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.5374999999999998e-05 2023-09-23T20:41:03.269 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.270 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.270 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:41:03.270 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:26.796055+0000", 2023-09-23T20:41:03.270 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.126e-05 2023-09-23T20:41:03.270 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.270 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.270 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:41:03.270 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:26.796067+0000", 2023-09-23T20:41:03.270 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.2043e-05 2023-09-23T20:41:03.270 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.270 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.270 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:41:03.271 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:45.167841+0000", 2023-09-23T20:41:03.271 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 18.371773717 2023-09-23T20:41:03.271 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.271 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.271 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:41:03.271 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:45.168079+0000", 2023-09-23T20:41:03.271 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000237839 2023-09-23T20:41:03.271 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.271 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.271 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:41:03.271 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:45.177631+0000", 2023-09-23T20:41:03.271 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0095519390000000006 2023-09-23T20:41:03.272 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:41:03.272 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:41:03.272 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:41:03.272 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.272 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.272 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "MOSDPGPush(8.19 435/410 [PushOp(8:9a33faf0:::benchmark_data_smithi018_341490_object85471:head, version: 395'5385, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(8:9a33faf0:::benchmark_data_smithi018_341490_object85471:head@395'5385, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-09-23T20:41:03.272 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:39:53.671016+0000", 2023-09-23T20:41:03.272 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 69.456142306000004, 2023-09-23T20:41:03.272 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.82055882099999999, 2023-09-23T20:41:03.272 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:41:03.272 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "started", 2023-09-23T20:41:03.272 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:41:03.273 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.273 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:41:03.273 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:53.671016+0000", 2023-09-23T20:41:03.273 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:41:03.273 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.273 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.273 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:41:03.273 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:53.671016+0000", 2023-09-23T20:41:03.273 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:41:03.273 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.273 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.273 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:41:03.274 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:53.671015+0000", 2023-09-23T20:41:03.274 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:41:03.274 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.274 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.274 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:41:03.274 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:53.671046+0000", 2023-09-23T20:41:03.274 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.0802000000000001e-05 2023-09-23T20:41:03.274 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.274 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.274 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:41:03.274 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:53.671061+0000", 2023-09-23T20:41:03.275 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.5446000000000001e-05 2023-09-23T20:41:03.275 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.275 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.275 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:41:03.275 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:53.671069+0000", 2023-09-23T20:41:03.275 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.9519999999999994e-06 2023-09-23T20:41:03.275 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.275 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.275 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:41:03.275 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:54.483764+0000", 2023-09-23T20:41:03.275 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.812694429 2023-09-23T20:41:03.276 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.276 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.276 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:41:03.276 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:54.483953+0000", 2023-09-23T20:41:03.276 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00018980099999999999 2023-09-23T20:41:03.276 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:41:03.276 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:41:03.276 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:41:03.276 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:54.491575+0000", 2023-09-23T20:41:03.276 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0076214999999999998 2023-09-23T20:41:03.276 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:41:03.276 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:41:03.277 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:41:03.277 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:41:03.277 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:41:03.277 INFO:teuthology.orchestra.run.smithi149.stdout:} 2023-09-23T20:41:03.277 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:41:03.308 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:03.372 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:41:03.378 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:03.378 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:41:03.480 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:41:03.508 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:03.547 INFO:teuthology.orchestra.run.smithi018.stderr:pool 'unique_pool_4' created 2023-09-23T20:41:03.552 INFO:teuthology.orchestra.run.smithi018.stderr:noscrub is unset 2023-09-23T20:41:03.552 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:41:03.555 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:03 smithi018 ceph-mon[179647]: pgmap v1118: 103 pgs: 103 active+clean; 4.3 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 32 KiB/s, 1 objects/s recovering 2023-09-23T20:41:03.555 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:03 smithi018 ceph-mon[179647]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T20:41:03.555 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:03 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3360825510' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:41:03.555 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:03 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/263149376' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:41:03.555 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:03 smithi018 ceph-mon[184144]: pgmap v1118: 103 pgs: 103 active+clean; 4.3 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 32 KiB/s, 1 objects/s recovering 2023-09-23T20:41:03.555 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:03 smithi018 ceph-mon[184144]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T20:41:03.556 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:03 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3360825510' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:41:03.556 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:03 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/263149376' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:41:03.556 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:03.556 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:41:03.595 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T20:41:03.657 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:41:03.689 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:03.727 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:41:03.731 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:03.732 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:41:03.833 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:41:03.860 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:03.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:03 smithi149 ceph-mon[170394]: pgmap v1118: 103 pgs: 103 active+clean; 4.3 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 32 KiB/s, 1 objects/s recovering 2023-09-23T20:41:03.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:03 smithi149 ceph-mon[170394]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T20:41:03.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:03 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3360825510' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:41:03.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:03 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/263149376' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:41:04.135 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd pool application enable unique_pool_4 rados --yes-i-really-mean-it || true 2023-09-23T20:41:04.147 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:41:04.149 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:04.150 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:41:04.199 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:41:04.251 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:41:04.282 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:04.402 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:41:04.405 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:04.405 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:41:04.506 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:41:04.538 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:04.586 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:41:04.589 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:04.589 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:41:04.690 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:41:04.694 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:04 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3253129604' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-09-23T20:41:04.695 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:04 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-09-23T20:41:04.695 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:04 smithi018 ceph-mon[179647]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:41:04.695 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:04 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3360825510' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:41:04.695 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:04 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2023-09-23T20:41:04.695 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:04 smithi018 ceph-mon[179647]: osdmap e455: 8 total, 8 up, 8 in 2023-09-23T20:41:04.695 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:04 smithi018 ceph-mon[179647]: failed to encode map e455 with expected crc 2023-09-23T20:41:04.695 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:04 smithi018 ceph-mon[179647]: failed to encode map e455 with expected crc 2023-09-23T20:41:04.695 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:04 smithi018 ceph-mon[179647]: failed to encode map e455 with expected crc 2023-09-23T20:41:04.695 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:04 smithi018 ceph-mon[179647]: failed to encode map e455 with expected crc 2023-09-23T20:41:04.695 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:04 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3253129604' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-09-23T20:41:04.696 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:04 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-09-23T20:41:04.696 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:04 smithi018 ceph-mon[184144]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:41:04.696 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:04 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3360825510' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:41:04.696 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:04 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2023-09-23T20:41:04.696 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:04 smithi018 ceph-mon[184144]: osdmap e455: 8 total, 8 up, 8 in 2023-09-23T20:41:04.696 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:04 smithi018 ceph-mon[184144]: failed to encode map e455 with expected crc 2023-09-23T20:41:04.696 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:04 smithi018 ceph-mon[184144]: failed to encode map e455 with expected crc 2023-09-23T20:41:04.696 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:04 smithi018 ceph-mon[184144]: failed to encode map e455 with expected crc 2023-09-23T20:41:04.696 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:04 smithi018 ceph-mon[184144]: failed to encode map e455 with expected crc 2023-09-23T20:41:04.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:04 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3253129604' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-09-23T20:41:04.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:04 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-09-23T20:41:04.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:04 smithi149 ceph-mon[170394]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:41:04.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:04 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3360825510' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:41:04.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:04 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2023-09-23T20:41:04.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:04 smithi149 ceph-mon[170394]: osdmap e455: 8 total, 8 up, 8 in 2023-09-23T20:41:04.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:04 smithi149 ceph-mon[170394]: failed to encode map e455 with expected crc 2023-09-23T20:41:04.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:04 smithi149 ceph-mon[170394]: failed to encode map e455 with expected crc 2023-09-23T20:41:04.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:04 smithi149 ceph-mon[170394]: failed to encode map e455 with expected crc 2023-09-23T20:41:04.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:04 smithi149 ceph-mon[170394]: failed to encode map e455 with expected crc 2023-09-23T20:41:04.719 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:04.755 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:41:04.758 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:04.759 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:41:04.859 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:41:04.889 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:05.014 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:41:05.082 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:41:05.082 INFO:teuthology.orchestra.run.smithi018.stdout:{"fsid":"a3b3b17e-5a4c-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodeep-scrub flag(s) set","count":12},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1341,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":456,"num_osds":8,"num_up_osds":8,"osd_up_since":1695500984,"num_in_osds":8,"osd_in_since":1695501516,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":103},{"state_name":"unknown","count":13},{"state_name":"creating+peering","count":3}],"num_pgs":119,"num_pools":5,"num_objects":27,"data_bytes":4551511,"bytes_used":5335621632,"bytes_avail":762624663552,"bytes_total":767960285184,"unknown_pgs_ratio":0.10924369841814041,"inactive_pgs_ratio":0.025210084393620491,"recovering_objects_per_sec":0,"recovering_bytes_per_sec":32825,"recovering_keys_per_sec":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0},"fsmap":{"epoch":9,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi018.hduiol","status":"up:active","gid":24566}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.18:8443/","prometheus":"http://172.21.15.18:9283/"}},"servicemap":{"epoch":23,"modified":"2023-09-23T20:34:43.862209+0000","services":{}},"progress_events":{"8e7eff18-0e45-4130-a6e1-662b3902f0d2":{"message":"Upgrade to 18.2.0-460-gbb0daeff (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true},"aefdb80c-9ae7-446c-b728-33511e08db43":{"message":"Global Recovery Event (3m)\n [===========================.] (remaining: 2s)","progress":0.99029123783111572,"add_to_ceph_s":true}}} 2023-09-23T20:41:05.115 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:41:05.118 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:05.118 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:41:05.219 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:41:05.249 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:05.299 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:41:05.303 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:05.303 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:41:05.391 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:41:04 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:41:04] "GET /metrics HTTP/1.1" 200 34170 "" "Prometheus/2.43.0" 2023-09-23T20:41:05.395 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:41:05.404 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:41:05.435 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:05.617 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-09-23T20:41:05.618 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:41:05.623 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:41:05.625 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:05.626 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:41:05.652 INFO:teuthology.orchestra.run.smithi149.stdout:{ 2023-09-23T20:41:05.652 INFO:teuthology.orchestra.run.smithi149.stdout: "ops": [], 2023-09-23T20:41:05.652 INFO:teuthology.orchestra.run.smithi149.stdout: "num_ops": 0 2023-09-23T20:41:05.652 INFO:teuthology.orchestra.run.smithi149.stdout:} 2023-09-23T20:41:05.687 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:05 smithi149 ceph-mon[170394]: pgmap v1120: 119 pgs: 3 creating+peering, 13 unknown, 103 active+clean; 4.3 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 32 KiB/s, 0 objects/s recovering 2023-09-23T20:41:05.715 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:05 smithi149 ceph-mon[170394]: failed to encode map e456 with expected crc 2023-09-23T20:41:05.715 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:05 smithi149 ceph-mon[170394]: osdmap e456: 8 total, 8 up, 8 in 2023-09-23T20:41:05.715 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:05 smithi149 ceph-mon[170394]: failed to encode map e456 with expected crc 2023-09-23T20:41:05.715 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:05 smithi149 ceph-mon[170394]: failed to encode map e456 with expected crc 2023-09-23T20:41:05.715 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:05 smithi149 ceph-mon[170394]: failed to encode map e456 with expected crc 2023-09-23T20:41:05.715 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:05 smithi149 ceph-mon[170394]: failed to encode map e456 with expected crc 2023-09-23T20:41:05.715 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:05 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1000096943' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T20:41:05.727 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:41:05.758 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:05 smithi018 ceph-mon[184144]: pgmap v1120: 119 pgs: 3 creating+peering, 13 unknown, 103 active+clean; 4.3 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 32 KiB/s, 0 objects/s recovering 2023-09-23T20:41:05.758 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:05 smithi018 ceph-mon[184144]: failed to encode map e456 with expected crc 2023-09-23T20:41:05.758 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:05 smithi018 ceph-mon[184144]: osdmap e456: 8 total, 8 up, 8 in 2023-09-23T20:41:05.758 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:05 smithi018 ceph-mon[184144]: failed to encode map e456 with expected crc 2023-09-23T20:41:05.758 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:05 smithi018 ceph-mon[184144]: failed to encode map e456 with expected crc 2023-09-23T20:41:05.758 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:05 smithi018 ceph-mon[184144]: failed to encode map e456 with expected crc 2023-09-23T20:41:05.758 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:05 smithi018 ceph-mon[184144]: failed to encode map e456 with expected crc 2023-09-23T20:41:05.759 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:05 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1000096943' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T20:41:05.759 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:05 smithi018 ceph-mon[179647]: pgmap v1120: 119 pgs: 3 creating+peering, 13 unknown, 103 active+clean; 4.3 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 32 KiB/s, 0 objects/s recovering 2023-09-23T20:41:05.759 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:05 smithi018 ceph-mon[179647]: failed to encode map e456 with expected crc 2023-09-23T20:41:05.759 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:05 smithi018 ceph-mon[179647]: osdmap e456: 8 total, 8 up, 8 in 2023-09-23T20:41:05.759 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:05 smithi018 ceph-mon[179647]: failed to encode map e456 with expected crc 2023-09-23T20:41:05.759 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:05 smithi018 ceph-mon[179647]: failed to encode map e456 with expected crc 2023-09-23T20:41:05.759 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:05 smithi018 ceph-mon[179647]: failed to encode map e456 with expected crc 2023-09-23T20:41:05.759 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:05 smithi018 ceph-mon[179647]: failed to encode map e456 with expected crc 2023-09-23T20:41:05.759 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:05 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1000096943' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T20:41:05.761 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:05.802 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:41:05.805 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:05.805 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:41:05.906 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:41:05.936 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:06.272 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:41:06.377 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:41:06.380 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:06.381 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:41:06.449 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T20:41:06.481 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:41:06.513 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:06.606 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:41:06.608 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:06.609 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:41:06.609 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:41:06.710 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:41:06.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:06 smithi149 ceph-mon[170394]: osdmap e457: 8 total, 8 up, 8 in 2023-09-23T20:41:06.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:06 smithi149 ceph-mon[170394]: failed to encode map e457 with expected crc 2023-09-23T20:41:06.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:06 smithi149 ceph-mon[170394]: failed to encode map e457 with expected crc 2023-09-23T20:41:06.741 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:06.795 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T20:41:06.799 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:06 smithi018 ceph-mon[179647]: osdmap e457: 8 total, 8 up, 8 in 2023-09-23T20:41:06.799 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:06 smithi018 ceph-mon[179647]: failed to encode map e457 with expected crc 2023-09-23T20:41:06.799 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:06 smithi018 ceph-mon[179647]: failed to encode map e457 with expected crc 2023-09-23T20:41:06.799 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:06 smithi018 ceph-mon[184144]: osdmap e457: 8 total, 8 up, 8 in 2023-09-23T20:41:06.799 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:06 smithi018 ceph-mon[184144]: failed to encode map e457 with expected crc 2023-09-23T20:41:06.799 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:06 smithi018 ceph-mon[184144]: failed to encode map e457 with expected crc 2023-09-23T20:41:06.879 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:41:06.882 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:06.883 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:41:06.984 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:41:07.014 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:07.067 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:41:07.235 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:41:07.239 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:07.240 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:41:07.340 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:41:07.372 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:07.559 INFO:teuthology.orchestra.run.smithi018.stderr:enabled application 'rados' on pool 'unique_pool_4' 2023-09-23T20:41:07.658 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:41:07.662 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:07.662 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:41:07.763 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:41:07.793 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:07.871 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:41:07.871 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:41:07.871 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:41:07.871 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:41:07.871 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:41:07.871 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:41:07.875 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:41:07.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:07 smithi018 ceph-mon[184144]: failed to encode map e457 with expected crc 2023-09-23T20:41:07.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:07 smithi018 ceph-mon[184144]: failed to encode map e457 with expected crc 2023-09-23T20:41:07.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:07 smithi018 ceph-mon[184144]: pgmap v1123: 119 pgs: 2 creating+activating, 3 creating+peering, 7 unknown, 107 active+clean; 4.3 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 43 KiB/s, 0 objects/s recovering 2023-09-23T20:41:07.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:07 smithi018 ceph-mon[184144]: 1.1e scrub starts 2023-09-23T20:41:07.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:07 smithi018 ceph-mon[184144]: 1.1e scrub ok 2023-09-23T20:41:07.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:07 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3867734778' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T20:41:07.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:07 smithi018 ceph-mon[179647]: failed to encode map e457 with expected crc 2023-09-23T20:41:07.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:07 smithi018 ceph-mon[179647]: failed to encode map e457 with expected crc 2023-09-23T20:41:07.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:07 smithi018 ceph-mon[179647]: pgmap v1123: 119 pgs: 2 creating+activating, 3 creating+peering, 7 unknown, 107 active+clean; 4.3 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 43 KiB/s, 0 objects/s recovering 2023-09-23T20:41:07.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:07 smithi018 ceph-mon[179647]: 1.1e scrub starts 2023-09-23T20:41:07.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:07 smithi018 ceph-mon[179647]: 1.1e scrub ok 2023-09-23T20:41:07.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:07 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3867734778' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T20:41:07.894 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:41:07.896 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:41:07.896 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:41:07.896 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:41:07.896 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:41:07.896 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:41:07.896 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:41:07.896 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:41:07.896 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:41:07.935 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:07 smithi149 ceph-mon[170394]: failed to encode map e457 with expected crc 2023-09-23T20:41:07.935 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:07 smithi149 ceph-mon[170394]: failed to encode map e457 with expected crc 2023-09-23T20:41:07.935 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:07 smithi149 ceph-mon[170394]: pgmap v1123: 119 pgs: 2 creating+activating, 3 creating+peering, 7 unknown, 107 active+clean; 4.3 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 43 KiB/s, 0 objects/s recovering 2023-09-23T20:41:07.935 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:07 smithi149 ceph-mon[170394]: 1.1e scrub starts 2023-09-23T20:41:07.935 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:07 smithi149 ceph-mon[170394]: 1.1e scrub ok 2023-09-23T20:41:07.935 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:07 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3867734778' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T20:41:07.967 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:41:07.970 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:07.970 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:41:08.071 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:41:08.099 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:08.309 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:41:08.312 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:08.313 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:41:08.384 INFO:teuthology.orchestra.run.smithi149.stdout:ERROR: (22) Invalid argument 2023-09-23T20:41:08.385 INFO:teuthology.orchestra.run.smithi149.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:41:08.413 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:41:08.440 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:08.569 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:41:08.571 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:08 smithi018 ceph-mon[184144]: 1.a scrub starts 2023-09-23T20:41:08.571 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:08 smithi018 ceph-mon[184144]: 1.a scrub ok 2023-09-23T20:41:08.571 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:08 smithi018 ceph-mon[184144]: 1.7 scrub starts 2023-09-23T20:41:08.571 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:08 smithi018 ceph-mon[184144]: 1.7 scrub ok 2023-09-23T20:41:08.571 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:08 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3867734778' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T20:41:08.571 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:08 smithi018 ceph-mon[184144]: osdmap e458: 8 total, 8 up, 8 in 2023-09-23T20:41:08.571 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:08 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2036749303' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:41:08.572 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:08 smithi018 ceph-mon[179647]: 1.a scrub starts 2023-09-23T20:41:08.572 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:08 smithi018 ceph-mon[179647]: 1.a scrub ok 2023-09-23T20:41:08.572 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:08 smithi018 ceph-mon[179647]: 1.7 scrub starts 2023-09-23T20:41:08.572 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:08 smithi018 ceph-mon[179647]: 1.7 scrub ok 2023-09-23T20:41:08.572 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:08 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3867734778' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T20:41:08.572 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:08 smithi018 ceph-mon[179647]: osdmap e458: 8 total, 8 up, 8 in 2023-09-23T20:41:08.572 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:08 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2036749303' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:41:08.573 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:08.573 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:41:08.573 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:41:08.674 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:41:08.705 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:08.727 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T20:41:08.782 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:08 smithi149 ceph-mon[170394]: 1.a scrub starts 2023-09-23T20:41:08.782 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:08 smithi149 ceph-mon[170394]: 1.a scrub ok 2023-09-23T20:41:08.782 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:08 smithi149 ceph-mon[170394]: 1.7 scrub starts 2023-09-23T20:41:08.782 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:08 smithi149 ceph-mon[170394]: 1.7 scrub ok 2023-09-23T20:41:08.783 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:08 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3867734778' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T20:41:08.783 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:08 smithi149 ceph-mon[170394]: osdmap e458: 8 total, 8 up, 8 in 2023-09-23T20:41:08.783 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:08 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2036749303' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:41:08.792 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:41:08.794 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:08.795 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:41:08.896 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:41:08.923 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:08.962 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:41:08.965 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:08.965 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:41:09.061 DEBUG:teuthology.orchestra.run.smithi018:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_4 bench 90 write' 2023-09-23T20:41:09.063 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-09-23T20:41:09.063 INFO:teuthology.orchestra.run:waiting for 3000 2023-09-23T20:41:09.066 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:41:09.095 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:09.127 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:hints = 1 2023-09-23T20:41:09.127 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-09-23T20:41:09.127 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Object prefix: benchmark_data_smithi018_381516 2023-09-23T20:41:09.128 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T20:41:09.128 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 0 0 0 0 0 0 - 0 2023-09-23T20:41:09.374 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:41:09.437 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:41:09.444 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:09.444 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:41:09.545 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:41:09.617 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:09.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:09 smithi018 ceph-mon[179647]: pgmap v1125: 119 pgs: 2 creating+activating, 3 creating+peering, 1 unknown, 113 active+clean; 4.3 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-23T20:41:09.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:09 smithi018 ceph-mon[179647]: 1.12 scrub starts 2023-09-23T20:41:09.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:09 smithi018 ceph-mon[179647]: 1.12 scrub ok 2023-09-23T20:41:09.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:09 smithi018 ceph-mon[179647]: failed to encode map e458 with expected crc 2023-09-23T20:41:09.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:09 smithi018 ceph-mon[179647]: 1.6 scrub starts 2023-09-23T20:41:09.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:09 smithi018 ceph-mon[179647]: failed to encode map e458 with expected crc 2023-09-23T20:41:09.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:09 smithi018 ceph-mon[179647]: 1.6 scrub ok 2023-09-23T20:41:09.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:09 smithi018 ceph-mon[184144]: pgmap v1125: 119 pgs: 2 creating+activating, 3 creating+peering, 1 unknown, 113 active+clean; 4.3 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-23T20:41:09.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:09 smithi018 ceph-mon[184144]: 1.12 scrub starts 2023-09-23T20:41:09.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:09 smithi018 ceph-mon[184144]: 1.12 scrub ok 2023-09-23T20:41:09.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:09 smithi018 ceph-mon[184144]: failed to encode map e458 with expected crc 2023-09-23T20:41:09.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:09 smithi018 ceph-mon[184144]: 1.6 scrub starts 2023-09-23T20:41:09.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:09 smithi018 ceph-mon[184144]: failed to encode map e458 with expected crc 2023-09-23T20:41:09.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:09 smithi018 ceph-mon[184144]: 1.6 scrub ok 2023-09-23T20:41:09.681 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:41:09.686 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:09.687 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:41:09.787 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:41:09.826 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:09 smithi149 ceph-mon[170394]: pgmap v1125: 119 pgs: 2 creating+activating, 3 creating+peering, 1 unknown, 113 active+clean; 4.3 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail 2023-09-23T20:41:09.827 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:09 smithi149 ceph-mon[170394]: 1.12 scrub starts 2023-09-23T20:41:09.827 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:09 smithi149 ceph-mon[170394]: 1.12 scrub ok 2023-09-23T20:41:09.827 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:09 smithi149 ceph-mon[170394]: failed to encode map e458 with expected crc 2023-09-23T20:41:09.827 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:09 smithi149 ceph-mon[170394]: 1.6 scrub starts 2023-09-23T20:41:09.827 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:09 smithi149 ceph-mon[170394]: failed to encode map e458 with expected crc 2023-09-23T20:41:09.827 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:09 smithi149 ceph-mon[170394]: 1.6 scrub ok 2023-09-23T20:41:09.830 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:10.128 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 1 16 3173 3157 197.244 197.312 0.00487896 0.0050487 2023-09-23T20:41:10.177 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:41:10.182 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:10.182 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:41:10.203 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:41:10.211 INFO:teuthology.orchestra.run.smithi018.stderr:dumped all 2023-09-23T20:41:10.283 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:41:10.328 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:10.336 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:41:10.384 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:41:10.390 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:10.390 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:41:10.491 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:41:10.547 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:10.577 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:10 smithi018 ceph-mon[179647]: failed to encode map e458 with expected crc 2023-09-23T20:41:10.577 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:10 smithi018 ceph-mon[179647]: failed to encode map e458 with expected crc 2023-09-23T20:41:10.578 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:10 smithi018 ceph-mon[179647]: failed to encode map e458 with expected crc 2023-09-23T20:41:10.578 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:10 smithi018 ceph-mon[179647]: failed to encode map e458 with expected crc 2023-09-23T20:41:10.578 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:10 smithi018 ceph-mon[179647]: failed to encode map e458 with expected crc 2023-09-23T20:41:10.578 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:10 smithi018 ceph-mon[179647]: 1.c scrub starts 2023-09-23T20:41:10.578 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:10 smithi018 ceph-mon[179647]: 1.c scrub ok 2023-09-23T20:41:10.580 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:10 smithi018 ceph-mon[184144]: failed to encode map e458 with expected crc 2023-09-23T20:41:10.580 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:10 smithi018 ceph-mon[184144]: failed to encode map e458 with expected crc 2023-09-23T20:41:10.580 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:10 smithi018 ceph-mon[184144]: failed to encode map e458 with expected crc 2023-09-23T20:41:10.580 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:10 smithi018 ceph-mon[184144]: failed to encode map e458 with expected crc 2023-09-23T20:41:10.580 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:10 smithi018 ceph-mon[184144]: failed to encode map e458 with expected crc 2023-09-23T20:41:10.580 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:10 smithi018 ceph-mon[184144]: 1.c scrub starts 2023-09-23T20:41:10.580 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:10 smithi018 ceph-mon[184144]: 1.c scrub ok 2023-09-23T20:41:10.603 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:41:10.610 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:10.610 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:41:10.711 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:41:10.762 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:10.834 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:41:10.840 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:10 smithi149 ceph-mon[170394]: failed to encode map e458 with expected crc 2023-09-23T20:41:10.840 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:10 smithi149 ceph-mon[170394]: failed to encode map e458 with expected crc 2023-09-23T20:41:10.840 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:10 smithi149 ceph-mon[170394]: failed to encode map e458 with expected crc 2023-09-23T20:41:10.840 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:10 smithi149 ceph-mon[170394]: failed to encode map e458 with expected crc 2023-09-23T20:41:10.840 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:10 smithi149 ceph-mon[170394]: failed to encode map e458 with expected crc 2023-09-23T20:41:10.840 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:10 smithi149 ceph-mon[170394]: 1.c scrub starts 2023-09-23T20:41:10.840 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:10 smithi149 ceph-mon[170394]: 1.c scrub ok 2023-09-23T20:41:10.841 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:10.841 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:41:10.941 INFO:teuthology.orchestra.run.smithi149.stdout:ERROR: (22) Invalid argument 2023-09-23T20:41:10.942 INFO:teuthology.orchestra.run.smithi149.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:41:10.942 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:41:10.997 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:11.129 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 2 16 6156 6140 191.739 186.438 0.00195843 0.0052062 2023-09-23T20:41:11.433 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T20:41:11.595 INFO:teuthology.orchestra.run.smithi018.stderr:nodeep-scrub is unset 2023-09-23T20:41:11.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:11 smithi149 ceph-mon[170394]: pgmap v1126: 119 pgs: 119 active+clean; 14 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 1.6 MiB/s wr, 23 op/s 2023-09-23T20:41:11.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:11 smithi149 ceph-mon[170394]: from='client.46262 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:41:11.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:11 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3077875587' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:41:11.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:11 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:41:11.855 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:11 smithi018 ceph-mon[184144]: pgmap v1126: 119 pgs: 119 active+clean; 14 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 1.6 MiB/s wr, 23 op/s 2023-09-23T20:41:11.855 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:11 smithi018 ceph-mon[184144]: from='client.46262 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:41:11.855 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:11 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3077875587' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:41:11.855 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:11 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:41:11.856 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:11 smithi018 ceph-mon[179647]: pgmap v1126: 119 pgs: 119 active+clean; 14 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 1.6 MiB/s wr, 23 op/s 2023-09-23T20:41:11.856 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:11 smithi018 ceph-mon[179647]: from='client.46262 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:41:11.856 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:11 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3077875587' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:41:11.856 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:11 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:41:11.969 INFO:teuthology.orchestra.run.smithi018.stdout:{"pg_ready":true,"pg_map":{"version":1126,"stamp":"2023-09-23T20:41:10.096061+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":14617220,"num_objects":181,"num_object_clones":0,"num_object_copies":543,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":181,"num_whiteouts":0,"num_read":6116,"num_read_kb":13448,"num_write":1126,"num_write_kb":23239,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":151,"num_bytes_recovered":9480489,"num_keys_recovered":40,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6176,"ondisk_log_size":6176,"up":357,"acting":357,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":364,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":5098160,"kb_used_data":1005680,"kb_used_omap":32,"kb_used_meta":4092383,"kb_avail":744863056,"statfs":{"total":767960285184,"available":762739769344,"internally_reserved":0,"allocated":1029816320,"data_stored":1014612503,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32773,"internal_metadata":4190601211},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":10065709,"num_objects":154,"num_object_clones":0,"num_object_copies":462,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":154,"num_whiteouts":0,"num_read":-22,"num_read_kb":-42,"num_write":143,"num_write_kb":9746,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-1,"num_bytes_recovered":-80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.001515"},"pg_stats":[{"pgid":"1.18","version":"90'161","reported_seq":439,"reported_epoch":455,"state":"active+clean","last_fresh":"2023-09-23T20:41:03.549075+0000","last_change":"2023-09-23T20:40:56.483418+0000","last_active":"2023-09-23T20:41:03.549075+0000","last_peered":"2023-09-23T20:41:03.549075+0000","last_clean":"2023-09-23T20:41:03.549075+0000","last_became_active":"2023-09-23T20:40:56.482886+0000","last_became_peered":"2023-09-23T20:40:56.482886+0000","last_unstale":"2023-09-23T20:41:03.549075+0000","last_undegraded":"2023-09-23T20:41:03.549075+0000","last_fullsized":"2023-09-23T20:41:03.549075+0000","mapping_epoch":453,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":454,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:35:03.551143+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:35:03.551143+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":343193,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"69'1","reported_seq":153,"reported_epoch":455,"state":"active+clean","last_fresh":"2023-09-23T20:41:03.544074+0000","last_change":"2023-09-23T20:32:33.233853+0000","last_active":"2023-09-23T20:41:03.544074+0000","last_peered":"2023-09-23T20:41:03.544074+0000","last_clean":"2023-09-23T20:41:03.544074+0000","last_became_active":"2023-09-23T20:32:33.233546+0000","last_became_peered":"2023-09-23T20:32:33.233546+0000","last_unstale":"2023-09-23T20:41:03.544074+0000","last_undegraded":"2023-09-23T20:41:03.544074+0000","last_fullsized":"2023-09-23T20:41:03.544074+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:50:27.130808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":271,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-09-23T20:41:05.548288+0000","last_change":"2023-09-23T20:37:43.579611+0000","last_active":"2023-09-23T20:41:05.548288+0000","last_peered":"2023-09-23T20:41:05.548288+0000","last_clean":"2023-09-23T20:41:05.548288+0000","last_became_active":"2023-09-23T20:37:43.579323+0000","last_became_peered":"2023-09-23T20:37:43.579323+0000","last_unstale":"2023-09-23T20:41:05.548288+0000","last_undegraded":"2023-09-23T20:41:05.548288+0000","last_fullsized":"2023-09-23T20:41:05.548288+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:17:32.311427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"90'161","reported_seq":529,"reported_epoch":455,"state":"active+clean","last_fresh":"2023-09-23T20:41:03.549558+0000","last_change":"2023-09-23T20:40:22.445059+0000","last_active":"2023-09-23T20:41:03.549558+0000","last_peered":"2023-09-23T20:41:03.549558+0000","last_clean":"2023-09-23T20:41:03.549558+0000","last_became_active":"2023-09-23T20:40:22.444637+0000","last_became_peered":"2023-09-23T20:40:22.444637+0000","last_unstale":"2023-09-23T20:41:03.549558+0000","last_undegraded":"2023-09-23T20:41:03.549558+0000","last_fullsized":"2023-09-23T20:41:03.549558+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:05.442288+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:36:05.442288+0000","last_clean_scrub_stamp":"2023-09-23T20:36:05.442288+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0019500940000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":426,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-09-23T20:41:05.639494+0000","last_change":"2023-09-23T20:34:59.983035+0000","last_active":"2023-09-23T20:41:05.639494+0000","last_peered":"2023-09-23T20:41:05.639494+0000","last_clean":"2023-09-23T20:41:05.639494+0000","last_became_active":"2023-09-23T20:34:59.982459+0000","last_became_peered":"2023-09-23T20:34:59.982459+0000","last_unstale":"2023-09-23T20:41:05.639494+0000","last_undegraded":"2023-09-23T20:41:05.639494+0000","last_fullsized":"2023-09-23T20:41:05.639494+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:16:48.510783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":427,"reported_epoch":456,"state":"active+clean","last_fresh":"2023-09-23T20:41:04.545179+0000","last_change":"2023-09-23T20:25:02.957185+0000","last_active":"2023-09-23T20:41:04.545179+0000","last_peered":"2023-09-23T20:41:04.545179+0000","last_clean":"2023-09-23T20:41:04.545179+0000","last_became_active":"2023-09-23T20:25:02.956689+0000","last_became_peered":"2023-09-23T20:25:02.956689+0000","last_unstale":"2023-09-23T20:41:04.545179+0000","last_undegraded":"2023-09-23T20:41:04.545179+0000","last_fullsized":"2023-09-23T20:41:04.545179+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:04:28.524015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"90'161","reported_seq":535,"reported_epoch":455,"state":"active+clean","last_fresh":"2023-09-23T20:41:03.543627+0000","last_change":"2023-09-23T20:36:01.292103+0000","last_active":"2023-09-23T20:41:03.543627+0000","last_peered":"2023-09-23T20:41:03.543627+0000","last_clean":"2023-09-23T20:41:03.543627+0000","last_became_active":"2023-09-23T20:35:26.639912+0000","last_became_peered":"2023-09-23T20:35:26.639912+0000","last_unstale":"2023-09-23T20:41:03.543627+0000","last_undegraded":"2023-09-23T20:41:03.543627+0000","last_fullsized":"2023-09-23T20:41:03.543627+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:01.292032+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:01.292032+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:04:53.926640+0000","scrub_duration":0.0032983159999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":251,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-09-23T20:41:05.547838+0000","last_change":"2023-09-23T20:37:43.584414+0000","last_active":"2023-09-23T20:41:05.547838+0000","last_peered":"2023-09-23T20:41:05.547838+0000","last_clean":"2023-09-23T20:41:05.547838+0000","last_became_active":"2023-09-23T20:37:43.584116+0000","last_became_peered":"2023-09-23T20:37:43.584116+0000","last_unstale":"2023-09-23T20:41:05.547838+0000","last_undegraded":"2023-09-23T20:41:05.547838+0000","last_fullsized":"2023-09-23T20:41:05.547838+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:17:54.258172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":72,"reported_epoch":456,"state":"active+clean","last_fresh":"2023-09-23T20:41:04.554267+0000","last_change":"2023-09-23T20:38:37.457427+0000","last_active":"2023-09-23T20:41:04.554267+0000","last_peered":"2023-09-23T20:41:04.554267+0000","last_clean":"2023-09-23T20:41:04.554267+0000","last_became_active":"2023-09-23T20:38:37.457099+0000","last_became_peered":"2023-09-23T20:38:37.457099+0000","last_unstale":"2023-09-23T20:41:04.554267+0000","last_undegraded":"2023-09-23T20:41:04.554267+0000","last_fullsized":"2023-09-23T20:41:04.554267+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:31:27.084994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"90'161","reported_seq":669,"reported_epoch":456,"state":"active+clean","last_fresh":"2023-09-23T20:41:04.553843+0000","last_change":"2023-09-23T20:40:22.449754+0000","last_active":"2023-09-23T20:41:04.553843+0000","last_peered":"2023-09-23T20:41:04.553843+0000","last_clean":"2023-09-23T20:41:04.553843+0000","last_became_active":"2023-09-23T20:40:22.448838+0000","last_became_peered":"2023-09-23T20:40:22.448838+0000","last_unstale":"2023-09-23T20:41:04.553843+0000","last_undegraded":"2023-09-23T20:41:04.553843+0000","last_fullsized":"2023-09-23T20:41:04.553843+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:03.411025+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:03.411025+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:22:07.948856+0000","scrub_duration":0.001802922,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":460,"reported_epoch":456,"state":"active+clean","last_fresh":"2023-09-23T20:41:04.544695+0000","last_change":"2023-09-23T20:40:19.538844+0000","last_active":"2023-09-23T20:41:04.544695+0000","last_peered":"2023-09-23T20:41:04.544695+0000","last_clean":"2023-09-23T20:41:04.544695+0000","last_became_active":"2023-09-23T20:40:19.538434+0000","last_became_peered":"2023-09-23T20:40:19.538434+0000","last_unstale":"2023-09-23T20:41:04.544695+0000","last_undegraded":"2023-09-23T20:41:04.544695+0000","last_fullsized":"2023-09-23T20:41:04.544695+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:53:59.418427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":180,"reported_epoch":456,"state":"active+clean","last_fresh":"2023-09-23T20:41:04.572059+0000","last_change":"2023-09-23T20:38:02.579343+0000","last_active":"2023-09-23T20:41:04.572059+0000","last_peered":"2023-09-23T20:41:04.572059+0000","last_clean":"2023-09-23T20:41:04.572059+0000","last_became_active":"2023-09-23T20:38:02.579047+0000","last_became_peered":"2023-09-23T20:38:02.579047+0000","last_unstale":"2023-09-23T20:41:04.572059+0000","last_undegraded":"2023-09-23T20:41:04.572059+0000","last_fullsized":"2023-09-23T20:41:04.572059+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:44:03.166556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.14","version":"90'161","reported_seq":786,"reported_epoch":456,"state":"active+clean","last_fresh":"2023-09-23T20:41:04.572277+0000","last_change":"2023-09-23T20:35:02.236398+0000","last_active":"2023-09-23T20:41:04.572277+0000","last_peered":"2023-09-23T20:41:04.572277+0000","last_clean":"2023-09-23T20:41:04.572277+0000","last_became_active":"2023-09-23T20:35:02.235988+0000","last_became_peered":"2023-09-23T20:35:02.235988+0000","last_unstale":"2023-09-23T20:41:04.572277+0000","last_undegraded":"2023-09-23T20:41:04.572277+0000","last_fullsized":"2023-09-23T20:41:04.572277+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:47.479012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:47.479012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:13.519648+0000","scrub_duration":0.0020136590000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":25,"reported_epoch":455,"state":"active+clean","last_fresh":"2023-09-23T20:41:03.549717+0000","last_change":"2023-09-23T20:40:19.545472+0000","last_active":"2023-09-23T20:41:03.549717+0000","last_peered":"2023-09-23T20:41:03.549717+0000","last_clean":"2023-09-23T20:41:03.549717+0000","last_became_active":"2023-09-23T20:40:19.543853+0000","last_became_peered":"2023-09-23T20:40:19.543853+0000","last_unstale":"2023-09-23T20:41:03.549717+0000","last_undegraded":"2023-09-23T20:41:03.549717+0000","last_fullsized":"2023-09-23T20:41:03.549717+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:08.014744+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":282,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-09-23T20:41:05.549083+0000","last_change":"2023-09-23T20:38:02.542034+0000","last_active":"2023-09-23T20:41:05.549083+0000","last_peered":"2023-09-23T20:41:05.549083+0000","last_clean":"2023-09-23T20:41:05.549083+0000","last_became_active":"2023-09-23T20:38:02.541575+0000","last_became_peered":"2023-09-23T20:38:02.541575+0000","last_unstale":"2023-09-23T20:41:05.549083+0000","last_undegraded":"2023-09-23T20:41:05.549083+0000","last_fullsized":"2023-09-23T20:41:05.549083+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:50:54.591529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"90'161","reported_seq":609,"reported_epoch":455,"state":"active+clean","last_fresh":"2023-09-23T20:41:03.549842+0000","last_change":"2023-09-23T20:40:22.444898+0000","last_active":"2023-09-23T20:41:03.549842+0000","last_peered":"2023-09-23T20:41:03.549842+0000","last_clean":"2023-09-23T20:41:03.549842+0000","last_became_active":"2023-09-23T20:40:22.444543+0000","last_became_peered":"2023-09-23T20:40:22.444543+0000","last_unstale":"2023-09-23T20:41:03.549842+0000","last_undegraded":"2023-09-23T20:41:03.549842+0000","last_fullsized":"2023-09-23T20:41:03.549842+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:03.435480+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:03.435480+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":484,"reported_epoch":456,"state":"active+clean","last_fresh":"2023-09-23T20:41:04.554194+0000","last_change":"2023-09-23T20:32:33.270617+0000","last_active":"2023-09-23T20:41:04.554194+0000","last_peered":"2023-09-23T20:41:04.554194+0000","last_clean":"2023-09-23T20:41:04.554194+0000","last_became_active":"2023-09-23T20:32:33.227902+0000","last_became_peered":"2023-09-23T20:32:33.227902+0000","last_unstale":"2023-09-23T20:41:04.554194+0000","last_undegraded":"2023-09-23T20:41:04.554194+0000","last_fullsized":"2023-09-23T20:41:04.554194+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:30:26.847831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":360,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":427,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-09-23T20:41:05.640256+0000","last_change":"2023-09-23T20:38:02.599068+0000","last_active":"2023-09-23T20:41:05.640256+0000","last_peered":"2023-09-23T20:41:05.640256+0000","last_clean":"2023-09-23T20:41:05.640256+0000","last_became_active":"2023-09-23T20:38:02.597602+0000","last_became_peered":"2023-09-23T20:38:02.597602+0000","last_unstale":"2023-09-23T20:41:05.640256+0000","last_undegraded":"2023-09-23T20:41:05.640256+0000","last_fullsized":"2023-09-23T20:41:05.640256+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:34:06.337029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"90'161","reported_seq":582,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-09-23T20:41:05.640382+0000","last_change":"2023-09-23T20:36:02.548387+0000","last_active":"2023-09-23T20:41:05.640382+0000","last_peered":"2023-09-23T20:41:05.640382+0000","last_clean":"2023-09-23T20:41:05.640382+0000","last_became_active":"2023-09-23T20:35:26.644787+0000","last_became_peered":"2023-09-23T20:35:26.644787+0000","last_unstale":"2023-09-23T20:41:05.640382+0000","last_undegraded":"2023-09-23T20:41:05.640382+0000","last_fullsized":"2023-09-23T20:41:05.640382+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:02.548285+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:02.548285+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:37:15.216553+0000","scrub_duration":0.0015514459999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":203,"reported_epoch":455,"state":"active+clean","last_fresh":"2023-09-23T20:41:03.543959+0000","last_change":"2023-09-23T20:38:02.560646+0000","last_active":"2023-09-23T20:41:03.543959+0000","last_peered":"2023-09-23T20:41:03.543959+0000","last_clean":"2023-09-23T20:41:03.543959+0000","last_became_active":"2023-09-23T20:38:02.539814+0000","last_became_peered":"2023-09-23T20:38:02.539814+0000","last_unstale":"2023-09-23T20:41:03.543959+0000","last_undegraded":"2023-09-23T20:41:03.543959+0000","last_fullsized":"2023-09-23T20:41:03.543959+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:00:09.435023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":183,"reported_epoch":455,"state":"active+clean","last_fresh":"2023-09-23T20:41:03.544241+0000","last_change":"2023-09-23T20:38:02.579222+0000","last_active":"2023-09-23T20:41:03.544241+0000","last_peered":"2023-09-23T20:41:03.544241+0000","last_clean":"2023-09-23T20:41:03.544241+0000","last_became_active":"2023-09-23T20:38:02.578813+0000","last_became_peered":"2023-09-23T20:38:02.578813+0000","last_unstale":"2023-09-23T20:41:03.544241+0000","last_undegraded":"2023-09-23T20:41:03.544241+0000","last_fullsized":"2023-09-23T20:41:03.544241+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:57:31.080027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"90'161","reported_seq":723,"reported_epoch":456,"state":"active+clean","last_fresh":"2023-09-23T20:41:04.547005+0000","last_change":"2023-09-23T20:40:22.427437+0000","last_active":"2023-09-23T20:41:04.547005+0000","last_peered":"2023-09-23T20:41:04.547005+0000","last_clean":"2023-09-23T20:41:04.547005+0000","last_became_active":"2023-09-23T20:40:22.427090+0000","last_became_peered":"2023-09-23T20:40:22.427090+0000","last_unstale":"2023-09-23T20:41:04.547005+0000","last_undegraded":"2023-09-23T20:41:04.547005+0000","last_fullsized":"2023-09-23T20:41:04.547005+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:43.578666+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:43.578666+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:08:43.995672+0000","scrub_duration":0.0021890379999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":483,"reported_epoch":456,"state":"active+clean","last_fresh":"2023-09-23T20:41:04.550962+0000","last_change":"2023-09-23T20:32:33.242943+0000","last_active":"2023-09-23T20:41:04.550962+0000","last_peered":"2023-09-23T20:41:04.550962+0000","last_clean":"2023-09-23T20:41:04.550962+0000","last_became_active":"2023-09-23T20:32:33.225468+0000","last_became_peered":"2023-09-23T20:32:33.225468+0000","last_unstale":"2023-09-23T20:41:04.550962+0000","last_undegraded":"2023-09-23T20:41:04.550962+0000","last_fullsized":"2023-09-23T20:41:04.550962+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:40:47.867573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":193,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-09-23T20:41:05.549097+0000","last_change":"2023-09-23T20:37:43.571836+0000","last_active":"2023-09-23T20:41:05.549097+0000","last_peered":"2023-09-23T20:41:05.549097+0000","last_clean":"2023-09-23T20:41:05.549097+0000","last_became_active":"2023-09-23T20:37:43.571511+0000","last_became_peered":"2023-09-23T20:37:43.571511+0000","last_unstale":"2023-09-23T20:41:05.549097+0000","last_undegraded":"2023-09-23T20:41:05.549097+0000","last_fullsized":"2023-09-23T20:41:05.549097+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:59:25.507917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"90'161","reported_seq":638,"reported_epoch":455,"state":"active+clean","last_fresh":"2023-09-23T20:41:03.549355+0000","last_change":"2023-09-23T20:40:22.427706+0000","last_active":"2023-09-23T20:41:03.549355+0000","last_peered":"2023-09-23T20:41:03.549355+0000","last_clean":"2023-09-23T20:41:03.549355+0000","last_became_active":"2023-09-23T20:40:22.427114+0000","last_became_peered":"2023-09-23T20:40:22.427114+0000","last_unstale":"2023-09-23T20:41:03.549355+0000","last_undegraded":"2023-09-23T20:41:03.549355+0000","last_fullsized":"2023-09-23T20:41:03.549355+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:49.726139+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:49.726139+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0096333110000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":424,"reported_epoch":456,"state":"active+clean","last_fresh":"2023-09-23T20:41:04.545717+0000","last_change":"2023-09-23T20:25:02.953993+0000","last_active":"2023-09-23T20:41:04.545717+0000","last_peered":"2023-09-23T20:41:04.545717+0000","last_clean":"2023-09-23T20:41:04.545717+0000","last_became_active":"2023-09-23T20:25:02.953672+0000","last_became_peered":"2023-09-23T20:25:02.953672+0000","last_unstale":"2023-09-23T20:41:04.545717+0000","last_undegraded":"2023-09-23T20:41:04.545717+0000","last_fullsized":"2023-09-23T20:41:04.545717+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:26:08.603577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":25,"reported_epoch":455,"state":"active+clean","last_fresh":"2023-09-23T20:41:03.549830+0000","last_change":"2023-09-23T20:40:19.561512+0000","last_active":"2023-09-23T20:41:03.549830+0000","last_peered":"2023-09-23T20:41:03.549830+0000","last_clean":"2023-09-23T20:41:03.549830+0000","last_became_active":"2023-09-23T20:40:19.561123+0000","last_became_peered":"2023-09-23T20:40:19.561123+0000","last_unstale":"2023-09-23T20:41:03.549830+0000","last_undegraded":"2023-09-23T20:41:03.549830+0000","last_fullsized":"2023-09-23T20:41:03.549830+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:50:02.093978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"90'161","reported_seq":743,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-09-23T20:41:05.639800+0000","last_change":"2023-09-23T20:35:02.238923+0000","last_active":"2023-09-23T20:41:05.639800+0000","last_peered":"2023-09-23T20:41:05.639800+0000","last_clean":"2023-09-23T20:41:05.639800+0000","last_became_active":"2023-09-23T20:35:02.238293+0000","last_became_peered":"2023-09-23T20:35:02.238293+0000","last_unstale":"2023-09-23T20:41:05.639800+0000","last_undegraded":"2023-09-23T20:41:05.639800+0000","last_fullsized":"2023-09-23T20:41:05.639800+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:42.544308+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:42.544308+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:51:22.281932+0000","scrub_duration":0.0029958179999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":460,"reported_epoch":456,"state":"active+clean","last_fresh":"2023-09-23T20:41:04.550831+0000","last_change":"2023-09-23T20:38:02.539774+0000","last_active":"2023-09-23T20:41:04.550831+0000","last_peered":"2023-09-23T20:41:04.550831+0000","last_clean":"2023-09-23T20:41:04.550831+0000","last_became_active":"2023-09-23T20:38:02.539436+0000","last_became_peered":"2023-09-23T20:38:02.539436+0000","last_unstale":"2023-09-23T20:41:04.550831+0000","last_undegraded":"2023-09-23T20:41:04.550831+0000","last_fullsized":"2023-09-23T20:41:04.550831+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:50:57.375803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":434,"reported_epoch":456,"state":"active+clean","last_fresh":"2023-09-23T20:41:04.545218+0000","last_change":"2023-09-23T20:25:02.957010+0000","last_active":"2023-09-23T20:41:04.545218+0000","last_peered":"2023-09-23T20:41:04.545218+0000","last_clean":"2023-09-23T20:41:04.545218+0000","last_became_active":"2023-09-23T20:25:02.956267+0000","last_became_peered":"2023-09-23T20:25:02.956267+0000","last_unstale":"2023-09-23T20:41:04.545218+0000","last_undegraded":"2023-09-23T20:41:04.545218+0000","last_fullsized":"2023-09-23T20:41:04.545218+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:10:55.299422+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"90'161","reported_seq":561,"reported_epoch":455,"state":"active+clean","last_fresh":"2023-09-23T20:41:03.549595+0000","last_change":"2023-09-23T20:40:22.431074+0000","last_active":"2023-09-23T20:41:03.549595+0000","last_peered":"2023-09-23T20:41:03.549595+0000","last_clean":"2023-09-23T20:41:03.549595+0000","last_became_active":"2023-09-23T20:40:22.430717+0000","last_became_peered":"2023-09-23T20:40:22.430717+0000","last_unstale":"2023-09-23T20:41:03.549595+0000","last_undegraded":"2023-09-23T20:41:03.549595+0000","last_fullsized":"2023-09-23T20:41:03.549595+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:53.706934+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:53.706934+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":92,"reported_epoch":456,"state":"active+clean","last_fresh":"2023-09-23T20:41:04.552665+0000","last_change":"2023-09-23T20:40:19.564378+0000","last_active":"2023-09-23T20:41:04.552665+0000","last_peered":"2023-09-23T20:41:04.552665+0000","last_clean":"2023-09-23T20:41:04.552665+0000","last_became_active":"2023-09-23T20:40:19.532991+0000","last_became_peered":"2023-09-23T20:40:19.532991+0000","last_unstale":"2023-09-23T20:41:04.552665+0000","last_undegraded":"2023-09-23T20:41:04.552665+0000","last_fullsized":"2023-09-23T20:41:04.552665+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:22:42.832115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":251,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-09-23T20:41:05.548464+0000","last_change":"2023-09-23T20:37:43.571083+0000","last_active":"2023-09-23T20:41:05.548464+0000","last_peered":"2023-09-23T20:41:05.548464+0000","last_clean":"2023-09-23T20:41:05.548464+0000","last_became_active":"2023-09-23T20:37:43.570306+0000","last_became_peered":"2023-09-23T20:37:43.570306+0000","last_unstale":"2023-09-23T20:41:05.548464+0000","last_undegraded":"2023-09-23T20:41:05.548464+0000","last_fullsized":"2023-09-23T20:41:05.548464+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:42:02.396718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.5","version":"0'0","reported_seq":11,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-09-23T20:41:05.641211+0000","last_change":"2023-09-23T20:41:04.556343+0000","last_active":"2023-09-23T20:41:05.641211+0000","last_peered":"2023-09-23T20:41:05.641211+0000","last_clean":"2023-09-23T20:41:05.641211+0000","last_became_active":"2023-09-23T20:41:04.556043+0000","last_became_peered":"2023-09-23T20:41:04.556043+0000","last_unstale":"2023-09-23T20:41:05.641211+0000","last_undegraded":"2023-09-23T20:41:05.641211+0000","last_fullsized":"2023-09-23T20:41:05.641211+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":455,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:33:41.345407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.d","version":"90'161","reported_seq":749,"reported_epoch":456,"state":"active+clean","last_fresh":"2023-09-23T20:41:04.553945+0000","last_change":"2023-09-23T20:34:59.496438+0000","last_active":"2023-09-23T20:41:04.553945+0000","last_peered":"2023-09-23T20:41:04.553945+0000","last_clean":"2023-09-23T20:41:04.553945+0000","last_became_active":"2023-09-23T20:33:38.970527+0000","last_became_peered":"2023-09-23T20:33:38.970527+0000","last_unstale":"2023-09-23T20:41:04.553945+0000","last_undegraded":"2023-09-23T20:41:04.553945+0000","last_fullsized":"2023-09-23T20:41:04.553945+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:34:59.496375+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:34:59.496375+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:46:18.244734+0000","scrub_duration":0.0017843189999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":460,"reported_epoch":456,"state":"active+clean","last_fresh":"2023-09-23T20:41:04.553483+0000","last_change":"2023-09-23T20:32:33.283752+0000","last_active":"2023-09-23T20:41:04.553483+0000","last_peered":"2023-09-23T20:41:04.553483+0000","last_clean":"2023-09-23T20:41:04.553483+0000","last_became_active":"2023-09-23T20:32:33.230503+0000","last_became_peered":"2023-09-23T20:32:33.230503+0000","last_unstale":"2023-09-23T20:41:04.553483+0000","last_undegraded":"2023-09-23T20:41:04.553483+0000","last_fullsized":"2023-09-23T20:41:04.553483+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:29.084410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":25,"reported_epoch":455,"state":"active+clean","last_fresh":"2023-09-23T20:41:03.549248+0000","last_change":"2023-09-23T20:40:19.545668+0000","last_active":"2023-09-23T20:41:03.549248+0000","last_peered":"2023-09-23T20:41:03.549248+0000","last_clean":"2023-09-23T20:41:03.549248+0000","last_became_active":"2023-09-23T20:40:19.543857+0000","last_became_peered":"2023-09-23T20:40:19.543857+0000","last_unstale":"2023-09-23T20:41:03.549248+0000","last_undegraded":"2023-09-23T20:41:03.549248+0000","last_fullsized":"2023-09-23T20:41:03.549248+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:27:09.470605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"458'81","reported_seq":93,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:09.549579+0000","last_change":"2023-09-23T20:41:04.575200+0000","last_active":"2023-09-23T20:41:09.549579+0000","last_peered":"2023-09-23T20:41:09.549579+0000","last_clean":"2023-09-23T20:41:09.549579+0000","last_became_active":"2023-09-23T20:41:04.574824+0000","last_became_peered":"2023-09-23T20:41:04.574824+0000","last_unstale":"2023-09-23T20:41:09.549579+0000","last_undegraded":"2023-09-23T20:41:09.549579+0000","last_fullsized":"2023-09-23T20:41:09.549579+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":455,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":81,"log_dups_size":0,"ondisk_log_size":81,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:50:46.714383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":5308416,"num_objects":81,"num_object_clones":0,"num_object_copies":243,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":81,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":81,"num_write_kb":5184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.c","version":"90'161","reported_seq":705,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-09-23T20:41:05.549258+0000","last_change":"2023-09-23T20:38:28.442536+0000","last_active":"2023-09-23T20:41:05.549258+0000","last_peered":"2023-09-23T20:41:05.549258+0000","last_clean":"2023-09-23T20:41:05.549258+0000","last_became_active":"2023-09-23T20:38:28.442229+0000","last_became_peered":"2023-09-23T20:38:28.442229+0000","last_unstale":"2023-09-23T20:41:05.549258+0000","last_undegraded":"2023-09-23T20:41:05.549258+0000","last_fullsized":"2023-09-23T20:41:05.549258+0000","mapping_epoch":411,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":412,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:05.485751+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:05.485751+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.002180043,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163840,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":495,"reported_epoch":456,"state":"active+clean","last_fresh":"2023-09-23T20:41:04.554251+0000","last_change":"2023-09-23T20:34:59.987981+0000","last_active":"2023-09-23T20:41:04.554251+0000","last_peered":"2023-09-23T20:41:04.554251+0000","last_clean":"2023-09-23T20:41:04.554251+0000","last_became_active":"2023-09-23T20:34:59.985368+0000","last_became_peered":"2023-09-23T20:34:59.985368+0000","last_unstale":"2023-09-23T20:41:04.554251+0000","last_undegraded":"2023-09-23T20:41:04.554251+0000","last_fullsized":"2023-09-23T20:41:04.554251+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:12:43.082493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":153,"reported_epoch":455,"state":"active+clean","last_fresh":"2023-09-23T20:41:03.543741+0000","last_change":"2023-09-23T20:32:33.232142+0000","last_active":"2023-09-23T20:41:03.543741+0000","last_peered":"2023-09-23T20:41:03.543741+0000","last_clean":"2023-09-23T20:41:03.543741+0000","last_became_active":"2023-09-23T20:32:33.231683+0000","last_became_peered":"2023-09-23T20:32:33.231683+0000","last_unstale":"2023-09-23T20:41:03.543741+0000","last_undegraded":"2023-09-23T20:41:03.543741+0000","last_fullsized":"2023-09-23T20:41:03.543741+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:02:58.336348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"0'0","reported_seq":9,"reported_epoch":456,"state":"active+clean","last_fresh":"2023-09-23T20:41:04.557945+0000","last_change":"2023-09-23T20:41:04.557945+0000","last_active":"2023-09-23T20:41:04.557945+0000","last_peered":"2023-09-23T20:41:04.557945+0000","last_clean":"2023-09-23T20:41:04.557945+0000","last_became_active":"2023-09-23T20:41:04.557435+0000","last_became_peered":"2023-09-23T20:41:04.557435+0000","last_unstale":"2023-09-23T20:41:04.557945+0000","last_undegraded":"2023-09-23T20:41:04.557945+0000","last_fullsized":"2023-09-23T20:41:04.557945+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":455,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:09:02.910592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.f","version":"90'161","reported_seq":612,"reported_epoch":456,"state":"active+clean","last_fresh":"2023-09-23T20:41:04.554466+0000","last_change":"2023-09-23T20:38:40.491902+0000","last_active":"2023-09-23T20:41:04.554466+0000","last_peered":"2023-09-23T20:41:04.554466+0000","last_clean":"2023-09-23T20:41:04.554466+0000","last_became_active":"2023-09-23T20:38:40.491563+0000","last_became_peered":"2023-09-23T20:38:40.491563+0000","last_unstale":"2023-09-23T20:41:04.554466+0000","last_undegraded":"2023-09-23T20:41:04.554466+0000","last_fullsized":"2023-09-23T20:41:04.554466+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:46.217489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:46.217489+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:39:25.008796+0000","scrub_duration":0.001456429,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":485,"reported_epoch":456,"state":"active+clean","last_fresh":"2023-09-23T20:41:04.554032+0000","last_change":"2023-09-23T20:40:19.677655+0000","last_active":"2023-09-23T20:41:04.554032+0000","last_peered":"2023-09-23T20:41:04.554032+0000","last_clean":"2023-09-23T20:41:04.554032+0000","last_became_active":"2023-09-23T20:40:19.551189+0000","last_became_peered":"2023-09-23T20:40:19.551189+0000","last_unstale":"2023-09-23T20:41:04.554032+0000","last_undegraded":"2023-09-23T20:41:04.554032+0000","last_fullsized":"2023-09-23T20:41:04.554032+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:11:01.559576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":62,"reported_epoch":456,"state":"active+clean","last_fresh":"2023-09-23T20:41:04.552965+0000","last_change":"2023-09-23T20:38:02.590613+0000","last_active":"2023-09-23T20:41:04.552965+0000","last_peered":"2023-09-23T20:41:04.552965+0000","last_clean":"2023-09-23T20:41:04.552965+0000","last_became_active":"2023-09-23T20:38:02.590284+0000","last_became_peered":"2023-09-23T20:38:02.590284+0000","last_unstale":"2023-09-23T20:41:04.552965+0000","last_undegraded":"2023-09-23T20:41:04.552965+0000","last_fullsized":"2023-09-23T20:41:04.552965+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:40:07.825054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"0'0","reported_seq":9,"reported_epoch":456,"state":"active+clean","last_fresh":"2023-09-23T20:41:04.573199+0000","last_change":"2023-09-23T20:41:04.573199+0000","last_active":"2023-09-23T20:41:04.573199+0000","last_peered":"2023-09-23T20:41:04.573199+0000","last_clean":"2023-09-23T20:41:04.573199+0000","last_became_active":"2023-09-23T20:41:04.572850+0000","last_became_peered":"2023-09-23T20:41:04.572850+0000","last_unstale":"2023-09-23T20:41:04.573199+0000","last_undegraded":"2023-09-23T20:41:04.573199+0000","last_fullsized":"2023-09-23T20:41:04.573199+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":455,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:53:49.894113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.e","version":"90'161","reported_seq":556,"reported_epoch":455,"state":"active+clean","last_fresh":"2023-09-23T20:41:03.544044+0000","last_change":"2023-09-23T20:36:00.322453+0000","last_active":"2023-09-23T20:41:03.544044+0000","last_peered":"2023-09-23T20:41:03.544044+0000","last_clean":"2023-09-23T20:41:03.544044+0000","last_became_active":"2023-09-23T20:35:02.233536+0000","last_became_peered":"2023-09-23T20:35:02.233536+0000","last_unstale":"2023-09-23T20:41:03.544044+0000","last_undegraded":"2023-09-23T20:41:03.544044+0000","last_fullsized":"2023-09-23T20:41:03.544044+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:00.322381+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:00.322381+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:02:15.489069+0000","scrub_duration":0.0017321859999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":50,"reported_epoch":455,"state":"active+clean","last_fresh":"2023-09-23T20:41:03.549637+0000","last_change":"2023-09-23T20:38:37.456683+0000","last_active":"2023-09-23T20:41:03.549637+0000","last_peered":"2023-09-23T20:41:03.549637+0000","last_clean":"2023-09-23T20:41:03.549637+0000","last_became_active":"2023-09-23T20:38:37.456289+0000","last_became_peered":"2023-09-23T20:38:37.456289+0000","last_unstale":"2023-09-23T20:41:03.549637+0000","last_undegraded":"2023-09-23T20:41:03.549637+0000","last_fullsized":"2023-09-23T20:41:03.549637+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:03:10.507333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":443,"reported_epoch":456,"state":"active+clean","last_fresh":"2023-09-23T20:41:04.546267+0000","last_change":"2023-09-23T20:32:33.236122+0000","last_active":"2023-09-23T20:41:04.546267+0000","last_peered":"2023-09-23T20:41:04.546267+0000","last_clean":"2023-09-23T20:41:04.546267+0000","last_became_active":"2023-09-23T20:32:33.235384+0000","last_became_peered":"2023-09-23T20:32:33.235384+0000","last_unstale":"2023-09-23T20:41:04.546267+0000","last_undegraded":"2023-09-23T20:41:04.546267+0000","last_fullsized":"2023-09-23T20:41:04.546267+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:56.843733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"0'0","reported_seq":10,"reported_epoch":456,"state":"active+clean","last_fresh":"2023-09-23T20:41:04.559102+0000","last_change":"2023-09-23T20:41:04.558473+0000","last_active":"2023-09-23T20:41:04.559102+0000","last_peered":"2023-09-23T20:41:04.559102+0000","last_clean":"2023-09-23T20:41:04.559102+0000","last_became_active":"2023-09-23T20:41:04.558173+0000","last_became_peered":"2023-09-23T20:41:04.558173+0000","last_unstale":"2023-09-23T20:41:04.559102+0000","last_undegraded":"2023-09-23T20:41:04.559102+0000","last_fullsized":"2023-09-23T20:41:04.559102+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":455,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:17:53.184877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":862,"reported_epoch":456,"state":"active+clean","last_fresh":"2023-09-23T20:41:04.550682+0000","last_change":"2023-09-23T20:35:02.523308+0000","last_active":"2023-09-23T20:41:04.550682+0000","last_peered":"2023-09-23T20:41:04.550682+0000","last_clean":"2023-09-23T20:41:04.550682+0000","last_became_active":"2023-09-23T20:33:38.979989+0000","last_became_peered":"2023-09-23T20:33:38.979989+0000","last_unstale":"2023-09-23T20:41:04.550682+0000","last_undegraded":"2023-09-23T20:41:04.550682+0000","last_fullsized":"2023-09-23T20:41:04.550682+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:35:02.523203+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:35:02.523203+0000","last_clean_scrub_stamp":"2023-09-23T20:35:02.523203+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:02:41.472958+0000","scrub_duration":0.0085927989999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":277,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-09-23T20:41:05.548091+0000","last_change":"2023-09-23T20:37:43.583731+0000","last_active":"2023-09-23T20:41:05.548091+0000","last_peered":"2023-09-23T20:41:05.548091+0000","last_clean":"2023-09-23T20:41:05.548091+0000","last_became_active":"2023-09-23T20:37:43.583066+0000","last_became_peered":"2023-09-23T20:37:43.583066+0000","last_unstale":"2023-09-23T20:41:05.548091+0000","last_undegraded":"2023-09-23T20:41:05.548091+0000","last_fullsized":"2023-09-23T20:41:05.548091+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:03:45.211900+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":173,"reported_epoch":455,"state":"active+clean","last_fresh":"2023-09-23T20:41:03.543799+0000","last_change":"2023-09-23T20:38:02.579814+0000","last_active":"2023-09-23T20:41:03.543799+0000","last_peered":"2023-09-23T20:41:03.543799+0000","last_clean":"2023-09-23T20:41:03.543799+0000","last_became_active":"2023-09-23T20:38:02.579253+0000","last_became_peered":"2023-09-23T20:38:02.579253+0000","last_unstale":"2023-09-23T20:41:03.543799+0000","last_undegraded":"2023-09-23T20:41:03.543799+0000","last_fullsized":"2023-09-23T20:41:03.543799+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:05:07.298198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"458'73","reported_seq":85,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:09.557556+0000","last_change":"2023-09-23T20:41:04.573101+0000","last_active":"2023-09-23T20:41:09.557556+0000","last_peered":"2023-09-23T20:41:09.557556+0000","last_clean":"2023-09-23T20:41:09.557556+0000","last_became_active":"2023-09-23T20:41:04.572728+0000","last_became_peered":"2023-09-23T20:41:04.572728+0000","last_unstale":"2023-09-23T20:41:09.557556+0000","last_undegraded":"2023-09-23T20:41:09.557556+0000","last_fullsized":"2023-09-23T20:41:09.557556+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":455,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":73,"log_dups_size":0,"ondisk_log_size":73,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:21:10.721593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4784128,"num_objects":73,"num_object_clones":0,"num_object_copies":219,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":73,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":73,"num_write_kb":4672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":557,"reported_epoch":455,"state":"active+clean","last_fresh":"2023-09-23T20:41:03.549177+0000","last_change":"2023-09-23T20:40:22.433408+0000","last_active":"2023-09-23T20:41:03.549177+0000","last_peered":"2023-09-23T20:41:03.549177+0000","last_clean":"2023-09-23T20:41:03.549177+0000","last_became_active":"2023-09-23T20:40:22.433023+0000","last_became_peered":"2023-09-23T20:40:22.433023+0000","last_unstale":"2023-09-23T20:41:03.549177+0000","last_undegraded":"2023-09-23T20:41:03.549177+0000","last_fullsized":"2023-09-23T20:41:03.549177+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:35:00.496197+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:35:00.496197+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":62,"reported_epoch":456,"state":"active+clean","last_fresh":"2023-09-23T20:41:04.553000+0000","last_change":"2023-09-23T20:38:02.578703+0000","last_active":"2023-09-23T20:41:04.553000+0000","last_peered":"2023-09-23T20:41:04.553000+0000","last_clean":"2023-09-23T20:41:04.553000+0000","last_became_active":"2023-09-23T20:38:02.578414+0000","last_became_peered":"2023-09-23T20:38:02.578414+0000","last_unstale":"2023-09-23T20:41:04.553000+0000","last_undegraded":"2023-09-23T20:41:04.553000+0000","last_fullsized":"2023-09-23T20:41:04.553000+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:43:59.779295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":122,"reported_epoch":455,"state":"active+clean","last_fresh":"2023-09-23T20:41:03.543891+0000","last_change":"2023-09-23T20:37:31.296606+0000","last_active":"2023-09-23T20:41:03.543891+0000","last_peered":"2023-09-23T20:41:03.543891+0000","last_clean":"2023-09-23T20:41:03.543891+0000","last_became_active":"2023-09-23T20:37:31.295924+0000","last_became_peered":"2023-09-23T20:37:31.295924+0000","last_unstale":"2023-09-23T20:41:03.543891+0000","last_undegraded":"2023-09-23T20:41:03.543891+0000","last_fullsized":"2023-09-23T20:41:03.543891+0000","mapping_epoch":391,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":392,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:27:50.948510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"0'0","reported_seq":10,"reported_epoch":456,"state":"active+clean","last_fresh":"2023-09-23T20:41:04.556805+0000","last_change":"2023-09-23T20:41:04.555980+0000","last_active":"2023-09-23T20:41:04.556805+0000","last_peered":"2023-09-23T20:41:04.556805+0000","last_clean":"2023-09-23T20:41:04.556805+0000","last_became_active":"2023-09-23T20:41:04.555688+0000","last_became_peered":"2023-09-23T20:41:04.555688+0000","last_unstale":"2023-09-23T20:41:04.556805+0000","last_undegraded":"2023-09-23T20:41:04.556805+0000","last_fullsized":"2023-09-23T20:41:04.556805+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":455,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:28:42.819404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.b","version":"90'161","reported_seq":760,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-09-23T20:41:05.639960+0000","last_change":"2023-09-23T20:40:22.445808+0000","last_active":"2023-09-23T20:41:05.639960+0000","last_peered":"2023-09-23T20:41:05.639960+0000","last_clean":"2023-09-23T20:41:05.639960+0000","last_became_active":"2023-09-23T20:40:22.445456+0000","last_became_peered":"2023-09-23T20:40:22.445456+0000","last_unstale":"2023-09-23T20:41:05.639960+0000","last_undegraded":"2023-09-23T20:41:05.639960+0000","last_fullsized":"2023-09-23T20:41:05.639960+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:05.595042+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:05.595042+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:39:23.678154+0000","scrub_duration":0.007271675,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":112,"reported_epoch":455,"state":"active+clean","last_fresh":"2023-09-23T20:41:03.543577+0000","last_change":"2023-09-23T20:34:59.983122+0000","last_active":"2023-09-23T20:41:03.543577+0000","last_peered":"2023-09-23T20:41:03.543577+0000","last_clean":"2023-09-23T20:41:03.543577+0000","last_became_active":"2023-09-23T20:34:59.982809+0000","last_became_peered":"2023-09-23T20:34:59.982809+0000","last_unstale":"2023-09-23T20:41:03.543577+0000","last_undegraded":"2023-09-23T20:41:03.543577+0000","last_fullsized":"2023-09-23T20:41:03.543577+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:49:40.839723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":173,"reported_epoch":455,"state":"active+clean","last_fresh":"2023-09-23T20:41:03.544452+0000","last_change":"2023-09-23T20:38:02.543866+0000","last_active":"2023-09-23T20:41:03.544452+0000","last_peered":"2023-09-23T20:41:03.544452+0000","last_clean":"2023-09-23T20:41:03.544452+0000","last_became_active":"2023-09-23T20:38:02.543563+0000","last_became_peered":"2023-09-23T20:38:02.543563+0000","last_unstale":"2023-09-23T20:41:03.544452+0000","last_undegraded":"2023-09-23T20:41:03.544452+0000","last_fullsized":"2023-09-23T20:41:03.544452+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:33.050536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"0'0","reported_seq":12,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-09-23T20:41:05.548007+0000","last_change":"2023-09-23T20:41:04.573300+0000","last_active":"2023-09-23T20:41:05.548007+0000","last_peered":"2023-09-23T20:41:05.548007+0000","last_clean":"2023-09-23T20:41:05.548007+0000","last_became_active":"2023-09-23T20:41:04.572919+0000","last_became_peered":"2023-09-23T20:41:04.572919+0000","last_unstale":"2023-09-23T20:41:05.548007+0000","last_undegraded":"2023-09-23T20:41:05.548007+0000","last_fullsized":"2023-09-23T20:41:05.548007+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":455,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:57:46.465243+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":560,"reported_epoch":455,"state":"active+clean","last_fresh":"2023-09-23T20:41:03.549430+0000","last_change":"2023-09-23T20:40:01.179672+0000","last_active":"2023-09-23T20:41:03.549430+0000","last_peered":"2023-09-23T20:41:03.549430+0000","last_clean":"2023-09-23T20:41:03.549430+0000","last_became_active":"2023-09-23T20:38:40.494049+0000","last_became_peered":"2023-09-23T20:38:40.494049+0000","last_unstale":"2023-09-23T20:41:03.549430+0000","last_undegraded":"2023-09-23T20:41:03.549430+0000","last_fullsized":"2023-09-23T20:41:03.549430+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:34:59.481446+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:34:59.481446+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26834,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":251,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-09-23T20:41:05.547838+0000","last_change":"2023-09-23T20:37:43.570610+0000","last_active":"2023-09-23T20:41:05.547838+0000","last_peered":"2023-09-23T20:41:05.547838+0000","last_clean":"2023-09-23T20:41:05.547838+0000","last_became_active":"2023-09-23T20:37:43.570216+0000","last_became_peered":"2023-09-23T20:37:43.570216+0000","last_unstale":"2023-09-23T20:41:05.547838+0000","last_undegraded":"2023-09-23T20:41:05.547838+0000","last_fullsized":"2023-09-23T20:41:05.547838+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:39:26.555329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":282,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-09-23T20:41:05.640762+0000","last_change":"2023-09-23T20:32:33.230582+0000","last_active":"2023-09-23T20:41:05.640762+0000","last_peered":"2023-09-23T20:41:05.640762+0000","last_clean":"2023-09-23T20:41:05.640762+0000","last_became_active":"2023-09-23T20:32:33.230257+0000","last_became_peered":"2023-09-23T20:32:33.230257+0000","last_unstale":"2023-09-23T20:41:05.640762+0000","last_undegraded":"2023-09-23T20:41:05.640762+0000","last_fullsized":"2023-09-23T20:41:05.640762+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:11:45.333096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.f","version":"0'0","reported_seq":9,"reported_epoch":456,"state":"active+clean","last_fresh":"2023-09-23T20:41:04.557353+0000","last_change":"2023-09-23T20:41:04.557353+0000","last_active":"2023-09-23T20:41:04.557353+0000","last_peered":"2023-09-23T20:41:04.557353+0000","last_clean":"2023-09-23T20:41:04.557353+0000","last_became_active":"2023-09-23T20:41:04.557037+0000","last_became_peered":"2023-09-23T20:41:04.557037+0000","last_unstale":"2023-09-23T20:41:04.557353+0000","last_undegraded":"2023-09-23T20:41:04.557353+0000","last_fullsized":"2023-09-23T20:41:04.557353+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":455,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:23:30.329193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":645,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-09-23T20:41:07.424662+0000","last_change":"2023-09-23T20:41:07.424609+0000","last_active":"2023-09-23T20:41:07.424662+0000","last_peered":"2023-09-23T20:41:07.424662+0000","last_clean":"2023-09-23T20:41:07.424662+0000","last_became_active":"2023-09-23T20:38:05.622261+0000","last_became_peered":"2023-09-23T20:38:05.622261+0000","last_unstale":"2023-09-23T20:41:07.424662+0000","last_undegraded":"2023-09-23T20:41:07.424662+0000","last_fullsized":"2023-09-23T20:41:07.424662+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:41:07.424492+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:41:07.424492+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:45:21.689123+0000","scrub_duration":0.0018960959999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"86'119","reported_seq":766,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-09-23T20:41:05.640075+0000","last_change":"2023-09-23T20:29:45.521257+0000","last_active":"2023-09-23T20:41:05.640075+0000","last_peered":"2023-09-23T20:41:05.640075+0000","last_clean":"2023-09-23T20:41:05.640075+0000","last_became_active":"2023-09-23T20:29:45.520546+0000","last_became_peered":"2023-09-23T20:29:45.520546+0000","last_unstale":"2023-09-23T20:41:05.640075+0000","last_undegraded":"2023-09-23T20:41:05.640075+0000","last_fullsized":"2023-09-23T20:41:05.640075+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":119,"log_dups_size":0,"ondisk_log_size":119,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:58:56.085777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032705499999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":523,"num_read_kb":3503,"num_write":136,"num_write_kb":3128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":25,"reported_epoch":455,"state":"active+clean","last_fresh":"2023-09-23T20:41:03.549831+0000","last_change":"2023-09-23T20:40:19.557009+0000","last_active":"2023-09-23T20:41:03.549831+0000","last_peered":"2023-09-23T20:41:03.549831+0000","last_clean":"2023-09-23T20:41:03.549831+0000","last_became_active":"2023-09-23T20:40:19.550473+0000","last_became_peered":"2023-09-23T20:40:19.550473+0000","last_unstale":"2023-09-23T20:41:03.549831+0000","last_undegraded":"2023-09-23T20:41:03.549831+0000","last_fullsized":"2023-09-23T20:41:03.549831+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:35:01.140258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":413,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-09-23T20:41:05.639811+0000","last_change":"2023-09-23T20:40:19.530942+0000","last_active":"2023-09-23T20:41:05.639811+0000","last_peered":"2023-09-23T20:41:05.639811+0000","last_clean":"2023-09-23T20:41:05.639811+0000","last_became_active":"2023-09-23T20:40:19.530617+0000","last_became_peered":"2023-09-23T20:40:19.530617+0000","last_unstale":"2023-09-23T20:41:05.639811+0000","last_undegraded":"2023-09-23T20:41:05.639811+0000","last_fullsized":"2023-09-23T20:41:05.639811+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:24:09.231934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.9","version":"0'0","reported_seq":11,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-09-23T20:41:05.640022+0000","last_change":"2023-09-23T20:41:04.556637+0000","last_active":"2023-09-23T20:41:05.640022+0000","last_peered":"2023-09-23T20:41:05.640022+0000","last_clean":"2023-09-23T20:41:05.640022+0000","last_became_active":"2023-09-23T20:41:04.555979+0000","last_became_peered":"2023-09-23T20:41:04.555979+0000","last_unstale":"2023-09-23T20:41:05.640022+0000","last_undegraded":"2023-09-23T20:41:05.640022+0000","last_fullsized":"2023-09-23T20:41:05.640022+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":455,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:45:38.600613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":529,"reported_epoch":455,"state":"active+clean","last_fresh":"2023-09-23T20:41:03.549194+0000","last_change":"2023-09-23T20:40:22.445201+0000","last_active":"2023-09-23T20:41:03.549194+0000","last_peered":"2023-09-23T20:41:03.549194+0000","last_clean":"2023-09-23T20:41:03.549194+0000","last_became_active":"2023-09-23T20:40:22.444703+0000","last_became_peered":"2023-09-23T20:40:22.444703+0000","last_unstale":"2023-09-23T20:41:03.549194+0000","last_undegraded":"2023-09-23T20:41:03.549194+0000","last_fullsized":"2023-09-23T20:41:03.549194+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:45.785450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:45.785450+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"86'106","reported_seq":1175,"reported_epoch":455,"state":"active+clean","last_fresh":"2023-09-23T20:41:03.549586+0000","last_change":"2023-09-23T20:38:37.476512+0000","last_active":"2023-09-23T20:41:03.549586+0000","last_peered":"2023-09-23T20:41:03.549586+0000","last_clean":"2023-09-23T20:41:03.549586+0000","last_became_active":"2023-09-23T20:38:37.475980+0000","last_became_peered":"2023-09-23T20:38:37.475980+0000","last_unstale":"2023-09-23T20:41:03.549586+0000","last_undegraded":"2023-09-23T20:41:03.549586+0000","last_fullsized":"2023-09-23T20:41:03.549586+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":106,"log_dups_size":0,"ondisk_log_size":106,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:58:45.508831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00099302599999999993,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1094,"num_read_kb":1678,"num_write":100,"num_write_kb":404,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":173,"reported_epoch":455,"state":"active+clean","last_fresh":"2023-09-23T20:41:03.544171+0000","last_change":"2023-09-23T20:38:02.552008+0000","last_active":"2023-09-23T20:41:03.544171+0000","last_peered":"2023-09-23T20:41:03.544171+0000","last_clean":"2023-09-23T20:41:03.544171+0000","last_became_active":"2023-09-23T20:38:02.551684+0000","last_became_peered":"2023-09-23T20:38:02.551684+0000","last_unstale":"2023-09-23T20:41:03.544171+0000","last_undegraded":"2023-09-23T20:41:03.544171+0000","last_fullsized":"2023-09-23T20:41:03.544171+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:40:37.147538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":419,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-09-23T20:41:05.639367+0000","last_change":"2023-09-23T20:29:45.533406+0000","last_active":"2023-09-23T20:41:05.639367+0000","last_peered":"2023-09-23T20:41:05.639367+0000","last_clean":"2023-09-23T20:41:05.639367+0000","last_became_active":"2023-09-23T20:29:45.526001+0000","last_became_peered":"2023-09-23T20:29:45.526001+0000","last_unstale":"2023-09-23T20:41:05.639367+0000","last_undegraded":"2023-09-23T20:41:05.639367+0000","last_fullsized":"2023-09-23T20:41:05.639367+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:43.787251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.a","version":"0'0","reported_seq":9,"reported_epoch":456,"state":"active+clean","last_fresh":"2023-09-23T20:41:04.556831+0000","last_change":"2023-09-23T20:41:04.556831+0000","last_active":"2023-09-23T20:41:04.556831+0000","last_peered":"2023-09-23T20:41:04.556831+0000","last_clean":"2023-09-23T20:41:04.556831+0000","last_became_active":"2023-09-23T20:41:04.556573+0000","last_became_peered":"2023-09-23T20:41:04.556573+0000","last_unstale":"2023-09-23T20:41:04.556831+0000","last_undegraded":"2023-09-23T20:41:04.556831+0000","last_fullsized":"2023-09-23T20:41:04.556831+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":455,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:55:53.953463+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":929,"reported_epoch":456,"state":"active+clean","last_fresh":"2023-09-23T20:41:04.552325+0000","last_change":"2023-09-23T20:35:02.260708+0000","last_active":"2023-09-23T20:41:04.552325+0000","last_peered":"2023-09-23T20:41:04.552325+0000","last_clean":"2023-09-23T20:41:04.552325+0000","last_became_active":"2023-09-23T20:35:02.260348+0000","last_became_peered":"2023-09-23T20:35:02.260348+0000","last_unstale":"2023-09-23T20:41:04.552325+0000","last_undegraded":"2023-09-23T20:41:04.552325+0000","last_fullsized":"2023-09-23T20:41:04.552325+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:43.390908+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:43.390908+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:09:14.144070+0000","scrub_duration":0.0032721,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"86'130","reported_seq":773,"reported_epoch":456,"state":"active+clean","last_fresh":"2023-09-23T20:41:04.572074+0000","last_change":"2023-09-23T20:25:39.637458+0000","last_active":"2023-09-23T20:41:04.572074+0000","last_peered":"2023-09-23T20:41:04.572074+0000","last_clean":"2023-09-23T20:41:04.572074+0000","last_became_active":"2023-09-23T20:25:39.636773+0000","last_became_peered":"2023-09-23T20:25:39.636773+0000","last_unstale":"2023-09-23T20:41:04.572074+0000","last_undegraded":"2023-09-23T20:41:04.572074+0000","last_fullsized":"2023-09-23T20:41:04.572074+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:00:32.810564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011300049999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":512,"num_read_kb":1820,"num_write":133,"num_write_kb":1347,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":427,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-09-23T20:41:05.640869+0000","last_change":"2023-09-23T20:38:02.591335+0000","last_active":"2023-09-23T20:41:05.640869+0000","last_peered":"2023-09-23T20:41:05.640869+0000","last_clean":"2023-09-23T20:41:05.640869+0000","last_became_active":"2023-09-23T20:38:02.591015+0000","last_became_peered":"2023-09-23T20:38:02.591015+0000","last_unstale":"2023-09-23T20:41:05.640869+0000","last_undegraded":"2023-09-23T20:41:05.640869+0000","last_fullsized":"2023-09-23T20:41:05.640869+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:55:32.272195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":419,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-09-23T20:41:05.640683+0000","last_change":"2023-09-23T20:40:19.544333+0000","last_active":"2023-09-23T20:41:05.640683+0000","last_peered":"2023-09-23T20:41:05.640683+0000","last_clean":"2023-09-23T20:41:05.640683+0000","last_became_active":"2023-09-23T20:40:19.543987+0000","last_became_peered":"2023-09-23T20:40:19.543987+0000","last_unstale":"2023-09-23T20:41:05.640683+0000","last_undegraded":"2023-09-23T20:41:05.640683+0000","last_fullsized":"2023-09-23T20:41:05.640683+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:12:59.203296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.8","version":"0'0","reported_seq":12,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-09-23T20:41:05.547716+0000","last_change":"2023-09-23T20:41:04.550745+0000","last_active":"2023-09-23T20:41:05.547716+0000","last_peered":"2023-09-23T20:41:05.547716+0000","last_clean":"2023-09-23T20:41:05.547716+0000","last_became_active":"2023-09-23T20:41:04.550493+0000","last_became_peered":"2023-09-23T20:41:04.550493+0000","last_unstale":"2023-09-23T20:41:05.547716+0000","last_undegraded":"2023-09-23T20:41:05.547716+0000","last_fullsized":"2023-09-23T20:41:05.547716+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":455,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:28:41.080518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"86'194","reported_seq":808,"reported_epoch":455,"state":"active+clean","last_fresh":"2023-09-23T20:41:03.549548+0000","last_change":"2023-09-23T20:40:51.458056+0000","last_active":"2023-09-23T20:41:03.549548+0000","last_peered":"2023-09-23T20:41:03.549548+0000","last_clean":"2023-09-23T20:41:03.549548+0000","last_became_active":"2023-09-23T20:40:51.457465+0000","last_became_peered":"2023-09-23T20:40:51.457465+0000","last_unstale":"2023-09-23T20:41:03.549548+0000","last_undegraded":"2023-09-23T20:41:03.549548+0000","last_fullsized":"2023-09-23T20:41:03.549548+0000","mapping_epoch":450,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":451,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":194,"log_dups_size":0,"ondisk_log_size":194,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:18:36.482553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00037995999999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1024,"num_read_kb":1200,"num_write":173,"num_write_kb":1276,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":919,"reported_epoch":456,"state":"active+clean","last_fresh":"2023-09-23T20:41:04.551182+0000","last_change":"2023-09-23T20:38:05.602508+0000","last_active":"2023-09-23T20:41:04.551182+0000","last_peered":"2023-09-23T20:41:04.551182+0000","last_clean":"2023-09-23T20:41:04.551182+0000","last_became_active":"2023-09-23T20:38:05.602129+0000","last_became_peered":"2023-09-23T20:38:05.602129+0000","last_unstale":"2023-09-23T20:41:04.551182+0000","last_undegraded":"2023-09-23T20:41:04.551182+0000","last_fullsized":"2023-09-23T20:41:04.551182+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:45.343742+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:45.343742+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:48.910343+0000","scrub_duration":0.001475802,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":25,"reported_epoch":455,"state":"active+clean","last_fresh":"2023-09-23T20:41:03.549422+0000","last_change":"2023-09-23T20:40:19.542142+0000","last_active":"2023-09-23T20:41:03.549422+0000","last_peered":"2023-09-23T20:41:03.549422+0000","last_clean":"2023-09-23T20:41:03.549422+0000","last_became_active":"2023-09-23T20:40:19.541870+0000","last_became_peered":"2023-09-23T20:40:19.541870+0000","last_unstale":"2023-09-23T20:41:03.549422+0000","last_undegraded":"2023-09-23T20:41:03.549422+0000","last_fullsized":"2023-09-23T20:41:03.549422+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:01.518149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":62,"reported_epoch":456,"state":"active+clean","last_fresh":"2023-09-23T20:41:04.554126+0000","last_change":"2023-09-23T20:38:02.573200+0000","last_active":"2023-09-23T20:41:04.554126+0000","last_peered":"2023-09-23T20:41:04.554126+0000","last_clean":"2023-09-23T20:41:04.554126+0000","last_became_active":"2023-09-23T20:38:02.572945+0000","last_became_peered":"2023-09-23T20:38:02.572945+0000","last_unstale":"2023-09-23T20:41:04.554126+0000","last_undegraded":"2023-09-23T20:41:04.554126+0000","last_fullsized":"2023-09-23T20:41:04.554126+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:57.246533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.b","version":"0'0","reported_seq":11,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-09-23T20:41:05.639518+0000","last_change":"2023-09-23T20:41:04.557718+0000","last_active":"2023-09-23T20:41:05.639518+0000","last_peered":"2023-09-23T20:41:05.639518+0000","last_clean":"2023-09-23T20:41:05.639518+0000","last_became_active":"2023-09-23T20:41:04.557420+0000","last_became_peered":"2023-09-23T20:41:04.557420+0000","last_unstale":"2023-09-23T20:41:05.639518+0000","last_undegraded":"2023-09-23T20:41:05.639518+0000","last_fullsized":"2023-09-23T20:41:05.639518+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":455,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:40:23.719326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":788,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-09-23T20:41:05.639693+0000","last_change":"2023-09-23T20:38:05.615711+0000","last_active":"2023-09-23T20:41:05.639693+0000","last_peered":"2023-09-23T20:41:05.639693+0000","last_clean":"2023-09-23T20:41:05.639693+0000","last_became_active":"2023-09-23T20:38:05.615390+0000","last_became_peered":"2023-09-23T20:38:05.615390+0000","last_unstale":"2023-09-23T20:41:05.639693+0000","last_undegraded":"2023-09-23T20:41:05.639693+0000","last_fullsized":"2023-09-23T20:41:05.639693+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:43.582146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:43.582146+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:45:11.917871+0000","scrub_duration":0.0016219730000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"86'80","reported_seq":709,"reported_epoch":456,"state":"active+clean","last_fresh":"2023-09-23T20:41:04.550552+0000","last_change":"2023-09-23T20:29:45.536796+0000","last_active":"2023-09-23T20:41:04.550552+0000","last_peered":"2023-09-23T20:41:04.550552+0000","last_clean":"2023-09-23T20:41:04.550552+0000","last_became_active":"2023-09-23T20:29:45.536439+0000","last_became_peered":"2023-09-23T20:29:45.536439+0000","last_unstale":"2023-09-23T20:41:04.550552+0000","last_undegraded":"2023-09-23T20:41:04.550552+0000","last_fullsized":"2023-09-23T20:41:04.550552+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":80,"log_dups_size":0,"ondisk_log_size":80,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:30:57.793132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035115,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":395,"num_read_kb":1370,"num_write":66,"num_write_kb":1061,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":462,"reported_epoch":456,"state":"active+clean","last_fresh":"2023-09-23T20:41:04.547184+0000","last_change":"2023-09-23T20:34:59.984286+0000","last_active":"2023-09-23T20:41:04.547184+0000","last_peered":"2023-09-23T20:41:04.547184+0000","last_clean":"2023-09-23T20:41:04.547184+0000","last_became_active":"2023-09-23T20:34:59.983950+0000","last_became_peered":"2023-09-23T20:34:59.983950+0000","last_unstale":"2023-09-23T20:41:04.547184+0000","last_undegraded":"2023-09-23T20:41:04.547184+0000","last_fullsized":"2023-09-23T20:41:04.547184+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:31:53.557035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":456,"reported_epoch":456,"state":"active+clean","last_fresh":"2023-09-23T20:41:04.545049+0000","last_change":"2023-09-23T20:29:45.690815+0000","last_active":"2023-09-23T20:41:04.545049+0000","last_peered":"2023-09-23T20:41:04.545049+0000","last_clean":"2023-09-23T20:41:04.545049+0000","last_became_active":"2023-09-23T20:29:45.690478+0000","last_became_peered":"2023-09-23T20:29:45.690478+0000","last_unstale":"2023-09-23T20:41:04.545049+0000","last_undegraded":"2023-09-23T20:41:04.545049+0000","last_fullsized":"2023-09-23T20:41:04.545049+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:38.288193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.e","version":"0'0","reported_seq":10,"reported_epoch":456,"state":"active+clean","last_fresh":"2023-09-23T20:41:04.577521+0000","last_change":"2023-09-23T20:41:04.574876+0000","last_active":"2023-09-23T20:41:04.577521+0000","last_peered":"2023-09-23T20:41:04.577521+0000","last_clean":"2023-09-23T20:41:04.577521+0000","last_became_active":"2023-09-23T20:41:04.574595+0000","last_became_peered":"2023-09-23T20:41:04.574595+0000","last_unstale":"2023-09-23T20:41:04.577521+0000","last_undegraded":"2023-09-23T20:41:04.577521+0000","last_fullsized":"2023-09-23T20:41:04.577521+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":455,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:31:00.147966+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":558,"reported_epoch":456,"state":"active+clean","last_fresh":"2023-09-23T20:41:04.553977+0000","last_change":"2023-09-23T20:38:05.611181+0000","last_active":"2023-09-23T20:41:04.553977+0000","last_peered":"2023-09-23T20:41:04.553977+0000","last_clean":"2023-09-23T20:41:04.553977+0000","last_became_active":"2023-09-23T20:38:05.610881+0000","last_became_peered":"2023-09-23T20:38:05.610881+0000","last_unstale":"2023-09-23T20:41:04.553977+0000","last_undegraded":"2023-09-23T20:41:04.553977+0000","last_fullsized":"2023-09-23T20:41:04.553977+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:35:00.151695+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:35:00.151695+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":20,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"86'102","reported_seq":1051,"reported_epoch":456,"state":"active+clean","last_fresh":"2023-09-23T20:41:04.554114+0000","last_change":"2023-09-23T20:32:36.314672+0000","last_active":"2023-09-23T20:41:04.554114+0000","last_peered":"2023-09-23T20:41:04.554114+0000","last_clean":"2023-09-23T20:41:04.554114+0000","last_became_active":"2023-09-23T20:32:36.314231+0000","last_became_peered":"2023-09-23T20:32:36.314231+0000","last_unstale":"2023-09-23T20:41:04.554114+0000","last_undegraded":"2023-09-23T20:41:04.554114+0000","last_fullsized":"2023-09-23T20:41:04.554114+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":102,"log_dups_size":0,"ondisk_log_size":102,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:11:40.514709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030656500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":963,"num_read_kb":1234,"num_write":60,"num_write_kb":225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":403,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-09-23T20:41:05.640127+0000","last_change":"2023-09-23T20:40:19.544384+0000","last_active":"2023-09-23T20:41:05.640127+0000","last_peered":"2023-09-23T20:41:05.640127+0000","last_clean":"2023-09-23T20:41:05.640127+0000","last_became_active":"2023-09-23T20:40:19.544079+0000","last_became_peered":"2023-09-23T20:40:19.544079+0000","last_unstale":"2023-09-23T20:41:05.640127+0000","last_undegraded":"2023-09-23T20:41:05.640127+0000","last_fullsized":"2023-09-23T20:41:05.640127+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:00:19.498631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":412,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-09-23T20:41:05.641994+0000","last_change":"2023-09-23T20:40:19.532976+0000","last_active":"2023-09-23T20:41:05.641994+0000","last_peered":"2023-09-23T20:41:05.641994+0000","last_clean":"2023-09-23T20:41:05.641994+0000","last_became_active":"2023-09-23T20:40:19.532639+0000","last_became_peered":"2023-09-23T20:40:19.532639+0000","last_unstale":"2023-09-23T20:41:05.641994+0000","last_undegraded":"2023-09-23T20:41:05.641994+0000","last_fullsized":"2023-09-23T20:41:05.641994+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:02:35.301884+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"0'0","reported_seq":10,"reported_epoch":456,"state":"active+clean","last_fresh":"2023-09-23T20:41:04.559617+0000","last_change":"2023-09-23T20:41:04.558941+0000","last_active":"2023-09-23T20:41:04.559617+0000","last_peered":"2023-09-23T20:41:04.559617+0000","last_clean":"2023-09-23T20:41:04.559617+0000","last_became_active":"2023-09-23T20:41:04.558482+0000","last_became_peered":"2023-09-23T20:41:04.558482+0000","last_unstale":"2023-09-23T20:41:04.559617+0000","last_undegraded":"2023-09-23T20:41:04.559617+0000","last_fullsized":"2023-09-23T20:41:04.559617+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":455,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:36:14.259949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":734,"reported_epoch":455,"state":"active+clean","last_fresh":"2023-09-23T20:41:03.544149+0000","last_change":"2023-09-23T20:38:05.590361+0000","last_active":"2023-09-23T20:41:03.544149+0000","last_peered":"2023-09-23T20:41:03.544149+0000","last_clean":"2023-09-23T20:41:03.544149+0000","last_became_active":"2023-09-23T20:38:05.589985+0000","last_became_peered":"2023-09-23T20:38:05.589985+0000","last_unstale":"2023-09-23T20:41:03.544149+0000","last_undegraded":"2023-09-23T20:41:03.544149+0000","last_fullsized":"2023-09-23T20:41:03.544149+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:03.246271+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:03.246271+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:38:54.351576+0000","scrub_duration":0.0019016700000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":194,"reported_epoch":456,"state":"active+clean","last_fresh":"2023-09-23T20:41:04.553682+0000","last_change":"2023-09-23T20:32:50.549695+0000","last_active":"2023-09-23T20:41:04.553682+0000","last_peered":"2023-09-23T20:41:04.553682+0000","last_clean":"2023-09-23T20:41:04.553682+0000","last_became_active":"2023-09-23T20:32:50.549431+0000","last_became_peered":"2023-09-23T20:32:50.549431+0000","last_unstale":"2023-09-23T20:41:04.553682+0000","last_undegraded":"2023-09-23T20:41:04.553682+0000","last_fullsized":"2023-09-23T20:41:04.553682+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:45:53.666569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"86'128","reported_seq":869,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-09-23T20:41:05.640566+0000","last_change":"2023-09-23T20:38:03.541765+0000","last_active":"2023-09-23T20:41:05.640566+0000","last_peered":"2023-09-23T20:41:05.640566+0000","last_clean":"2023-09-23T20:41:05.640566+0000","last_became_active":"2023-09-23T20:38:03.541277+0000","last_became_peered":"2023-09-23T20:38:03.541277+0000","last_unstale":"2023-09-23T20:41:05.640566+0000","last_undegraded":"2023-09-23T20:41:05.640566+0000","last_fullsized":"2023-09-23T20:41:05.640566+0000","mapping_epoch":404,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":405,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:20:53.834554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000293202,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":579,"num_read_kb":1473,"num_write":96,"num_write_kb":1062,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":236,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-09-23T20:41:05.547708+0000","last_change":"2023-09-23T20:37:43.582105+0000","last_active":"2023-09-23T20:41:05.547708+0000","last_peered":"2023-09-23T20:41:05.547708+0000","last_clean":"2023-09-23T20:41:05.547708+0000","last_became_active":"2023-09-23T20:37:43.581860+0000","last_became_peered":"2023-09-23T20:37:43.581860+0000","last_unstale":"2023-09-23T20:41:05.547708+0000","last_undegraded":"2023-09-23T20:41:05.547708+0000","last_fullsized":"2023-09-23T20:41:05.547708+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:18:38.765486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.c","version":"0'0","reported_seq":12,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-09-23T20:41:05.549350+0000","last_change":"2023-09-23T20:41:04.575496+0000","last_active":"2023-09-23T20:41:05.549350+0000","last_peered":"2023-09-23T20:41:05.549350+0000","last_clean":"2023-09-23T20:41:05.549350+0000","last_became_active":"2023-09-23T20:41:04.575242+0000","last_became_peered":"2023-09-23T20:41:04.575242+0000","last_unstale":"2023-09-23T20:41:05.549350+0000","last_undegraded":"2023-09-23T20:41:05.549350+0000","last_fullsized":"2023-09-23T20:41:05.549350+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":455,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:30:14.611529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"90'161","reported_seq":670,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-09-23T20:41:05.549437+0000","last_change":"2023-09-23T20:40:22.427435+0000","last_active":"2023-09-23T20:41:05.549437+0000","last_peered":"2023-09-23T20:41:05.549437+0000","last_clean":"2023-09-23T20:41:05.549437+0000","last_became_active":"2023-09-23T20:40:22.427051+0000","last_became_peered":"2023-09-23T20:40:22.427051+0000","last_unstale":"2023-09-23T20:41:05.549437+0000","last_undegraded":"2023-09-23T20:41:05.549437+0000","last_fullsized":"2023-09-23T20:41:05.549437+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:34:54.504554+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:34:54.504554+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:42.001652+0000","scrub_duration":0.011176864,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"86'117","reported_seq":689,"reported_epoch":456,"state":"active+clean","last_fresh":"2023-09-23T20:41:04.554361+0000","last_change":"2023-09-23T20:38:28.446107+0000","last_active":"2023-09-23T20:41:04.554361+0000","last_peered":"2023-09-23T20:41:04.554361+0000","last_clean":"2023-09-23T20:41:04.554361+0000","last_became_active":"2023-09-23T20:38:28.445667+0000","last_became_peered":"2023-09-23T20:38:28.445667+0000","last_unstale":"2023-09-23T20:41:04.554361+0000","last_undegraded":"2023-09-23T20:41:04.554361+0000","last_fullsized":"2023-09-23T20:41:04.554361+0000","mapping_epoch":411,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":412,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:58:47.430360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046891400000000002,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":893,"num_read_kb":922,"num_write":82,"num_write_kb":4193,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":456,"reported_epoch":456,"state":"active+clean","last_fresh":"2023-09-23T20:41:04.545134+0000","last_change":"2023-09-23T20:32:33.277827+0000","last_active":"2023-09-23T20:41:04.545134+0000","last_peered":"2023-09-23T20:41:04.545134+0000","last_clean":"2023-09-23T20:41:04.545134+0000","last_became_active":"2023-09-23T20:32:33.236087+0000","last_became_peered":"2023-09-23T20:32:33.236087+0000","last_unstale":"2023-09-23T20:41:04.545134+0000","last_undegraded":"2023-09-23T20:41:04.545134+0000","last_fullsized":"2023-09-23T20:41:04.545134+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:20:58.176854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":193,"reported_epoch":455,"state":"active+clean","last_fresh":"2023-09-23T20:41:03.543706+0000","last_change":"2023-09-23T20:40:19.530602+0000","last_active":"2023-09-23T20:41:03.543706+0000","last_peered":"2023-09-23T20:41:03.543706+0000","last_clean":"2023-09-23T20:41:03.543706+0000","last_became_active":"2023-09-23T20:40:19.530256+0000","last_became_peered":"2023-09-23T20:40:19.530256+0000","last_unstale":"2023-09-23T20:41:03.543706+0000","last_undegraded":"2023-09-23T20:41:03.543706+0000","last_fullsized":"2023-09-23T20:41:03.543706+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:18:45.658867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"90'161","reported_seq":726,"reported_epoch":456,"state":"active+clean","last_fresh":"2023-09-23T20:41:04.546993+0000","last_change":"2023-09-23T20:38:05.630221+0000","last_active":"2023-09-23T20:41:04.546993+0000","last_peered":"2023-09-23T20:41:04.546993+0000","last_clean":"2023-09-23T20:41:04.546993+0000","last_became_active":"2023-09-23T20:38:05.629839+0000","last_became_peered":"2023-09-23T20:38:05.629839+0000","last_unstale":"2023-09-23T20:41:04.546993+0000","last_undegraded":"2023-09-23T20:41:04.546993+0000","last_fullsized":"2023-09-23T20:41:04.546993+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:06.487351+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:06.487351+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:25:51.325529+0000","scrub_duration":0.001964379,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":424,"reported_epoch":456,"state":"active+clean","last_fresh":"2023-09-23T20:41:04.546421+0000","last_change":"2023-09-23T20:32:33.237234+0000","last_active":"2023-09-23T20:41:04.546421+0000","last_peered":"2023-09-23T20:41:04.546421+0000","last_clean":"2023-09-23T20:41:04.546421+0000","last_became_active":"2023-09-23T20:32:33.236881+0000","last_became_peered":"2023-09-23T20:32:33.236881+0000","last_unstale":"2023-09-23T20:41:04.546421+0000","last_undegraded":"2023-09-23T20:41:04.546421+0000","last_fullsized":"2023-09-23T20:41:04.546421+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:56:17.384831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":312,"reported_epoch":456,"state":"active+clean","last_fresh":"2023-09-23T20:41:04.572614+0000","last_change":"2023-09-23T20:29:45.538314+0000","last_active":"2023-09-23T20:41:04.572614+0000","last_peered":"2023-09-23T20:41:04.572614+0000","last_clean":"2023-09-23T20:41:04.572614+0000","last_became_active":"2023-09-23T20:29:45.536847+0000","last_became_peered":"2023-09-23T20:29:45.536847+0000","last_unstale":"2023-09-23T20:41:04.572614+0000","last_undegraded":"2023-09-23T20:41:04.572614+0000","last_fullsized":"2023-09-23T20:41:04.572614+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:51.948721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"90'161","reported_seq":529,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-09-23T20:41:06.499511+0000","last_change":"2023-09-23T20:41:06.499511+0000","last_active":"2023-09-23T20:41:06.499511+0000","last_peered":"2023-09-23T20:41:06.499511+0000","last_clean":"2023-09-23T20:41:06.499511+0000","last_became_active":"2023-09-23T20:38:05.601874+0000","last_became_peered":"2023-09-23T20:38:05.601874+0000","last_unstale":"2023-09-23T20:41:06.499511+0000","last_undegraded":"2023-09-23T20:41:06.499511+0000","last_fullsized":"2023-09-23T20:41:06.499511+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:41:06.499396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:41:06.499396+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:31:34.059077+0000","scrub_duration":0.0019670310000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":54,"reported_epoch":455,"state":"active+clean","last_fresh":"2023-09-23T20:41:03.549647+0000","last_change":"2023-09-23T20:38:37.481551+0000","last_active":"2023-09-23T20:41:03.549647+0000","last_peered":"2023-09-23T20:41:03.549647+0000","last_clean":"2023-09-23T20:41:03.549647+0000","last_became_active":"2023-09-23T20:38:37.458687+0000","last_became_peered":"2023-09-23T20:38:37.458687+0000","last_unstale":"2023-09-23T20:41:03.549647+0000","last_undegraded":"2023-09-23T20:41:03.549647+0000","last_fullsized":"2023-09-23T20:41:03.549647+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:22:15.033243+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":112,"reported_epoch":455,"state":"active+clean","last_fresh":"2023-09-23T20:41:03.543950+0000","last_change":"2023-09-23T20:34:59.987575+0000","last_active":"2023-09-23T20:41:03.543950+0000","last_peered":"2023-09-23T20:41:03.543950+0000","last_clean":"2023-09-23T20:41:03.543950+0000","last_became_active":"2023-09-23T20:34:59.987229+0000","last_became_peered":"2023-09-23T20:34:59.987229+0000","last_unstale":"2023-09-23T20:41:03.543950+0000","last_undegraded":"2023-09-23T20:41:03.543950+0000","last_fullsized":"2023-09-23T20:41:03.543950+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:34:01.820159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":54,"reported_epoch":455,"state":"active+clean","last_fresh":"2023-09-23T20:41:03.549917+0000","last_change":"2023-09-23T20:38:37.457634+0000","last_active":"2023-09-23T20:41:03.549917+0000","last_peered":"2023-09-23T20:41:03.549917+0000","last_clean":"2023-09-23T20:41:03.549917+0000","last_became_active":"2023-09-23T20:38:37.452208+0000","last_became_peered":"2023-09-23T20:38:37.452208+0000","last_unstale":"2023-09-23T20:41:03.549917+0000","last_undegraded":"2023-09-23T20:41:03.549917+0000","last_fullsized":"2023-09-23T20:41:03.549917+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:09:41.754569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":112,"reported_epoch":455,"state":"active+clean","last_fresh":"2023-09-23T20:41:03.543770+0000","last_change":"2023-09-23T20:34:59.987416+0000","last_active":"2023-09-23T20:41:03.543770+0000","last_peered":"2023-09-23T20:41:03.543770+0000","last_clean":"2023-09-23T20:41:03.543770+0000","last_became_active":"2023-09-23T20:34:59.987117+0000","last_became_peered":"2023-09-23T20:34:59.987117+0000","last_unstale":"2023-09-23T20:41:03.543770+0000","last_undegraded":"2023-09-23T20:41:03.543770+0000","last_fullsized":"2023-09-23T20:41:03.543770+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:36:16.318623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"90'161","reported_seq":712,"reported_epoch":456,"state":"active+clean","last_fresh":"2023-09-23T20:41:04.550724+0000","last_change":"2023-09-23T20:38:05.591992+0000","last_active":"2023-09-23T20:41:04.550724+0000","last_peered":"2023-09-23T20:41:04.550724+0000","last_clean":"2023-09-23T20:41:04.550724+0000","last_became_active":"2023-09-23T20:38:05.591701+0000","last_became_peered":"2023-09-23T20:38:05.591701+0000","last_unstale":"2023-09-23T20:41:04.550724+0000","last_undegraded":"2023-09-23T20:41:04.550724+0000","last_fullsized":"2023-09-23T20:41:04.550724+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:02.384133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:02.384133+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:41.722542+0000","scrub_duration":0.001769098,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":278,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-09-23T20:41:05.548110+0000","last_change":"2023-09-23T20:38:02.542719+0000","last_active":"2023-09-23T20:41:05.548110+0000","last_peered":"2023-09-23T20:41:05.548110+0000","last_clean":"2023-09-23T20:41:05.548110+0000","last_became_active":"2023-09-23T20:38:02.542385+0000","last_became_peered":"2023-09-23T20:38:02.542385+0000","last_unstale":"2023-09-23T20:41:05.548110+0000","last_undegraded":"2023-09-23T20:41:05.548110+0000","last_fullsized":"2023-09-23T20:41:05.548110+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:36:23.673757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":245,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-09-23T20:41:05.549083+0000","last_change":"2023-09-23T20:37:43.584253+0000","last_active":"2023-09-23T20:41:05.549083+0000","last_peered":"2023-09-23T20:41:05.549083+0000","last_clean":"2023-09-23T20:41:05.549083+0000","last_became_active":"2023-09-23T20:37:43.580069+0000","last_became_peered":"2023-09-23T20:37:43.580069+0000","last_unstale":"2023-09-23T20:41:05.549083+0000","last_undegraded":"2023-09-23T20:41:05.549083+0000","last_fullsized":"2023-09-23T20:41:05.549083+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:39:11.482581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"90'161","reported_seq":550,"reported_epoch":457,"state":"active+clean","last_fresh":"2023-09-23T20:41:05.640463+0000","last_change":"2023-09-23T20:36:01.505098+0000","last_active":"2023-09-23T20:41:05.640463+0000","last_peered":"2023-09-23T20:41:05.640463+0000","last_clean":"2023-09-23T20:41:05.640463+0000","last_became_active":"2023-09-23T20:35:26.644707+0000","last_became_peered":"2023-09-23T20:35:26.644707+0000","last_unstale":"2023-09-23T20:41:05.640463+0000","last_undegraded":"2023-09-23T20:41:05.640463+0000","last_fullsized":"2023-09-23T20:41:05.640463+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:01.504973+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:01.504973+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:02:10.880809+0000","scrub_duration":0.00202344,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":502,"reported_epoch":456,"state":"active+clean","last_fresh":"2023-09-23T20:41:04.544809+0000","last_change":"2023-09-23T20:40:19.569032+0000","last_active":"2023-09-23T20:41:04.544809+0000","last_peered":"2023-09-23T20:41:04.544809+0000","last_clean":"2023-09-23T20:41:04.544809+0000","last_became_active":"2023-09-23T20:40:19.538476+0000","last_became_peered":"2023-09-23T20:40:19.538476+0000","last_unstale":"2023-09-23T20:41:04.544809+0000","last_undegraded":"2023-09-23T20:41:04.544809+0000","last_fullsized":"2023-09-23T20:41:04.544809+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:31:58.998168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3324,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":173,"reported_epoch":455,"state":"active+clean","last_fresh":"2023-09-23T20:41:03.543478+0000","last_change":"2023-09-23T20:38:02.538333+0000","last_active":"2023-09-23T20:41:03.543478+0000","last_peered":"2023-09-23T20:41:03.543478+0000","last_clean":"2023-09-23T20:41:03.543478+0000","last_became_active":"2023-09-23T20:38:02.538008+0000","last_became_peered":"2023-09-23T20:38:02.538008+0000","last_unstale":"2023-09-23T20:41:03.543478+0000","last_undegraded":"2023-09-23T20:41:03.543478+0000","last_fullsized":"2023-09-23T20:41:03.543478+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:02:38.789530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":80,"num_bytes_recovered":18486,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26691,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":324849,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":133,"num_read_kb":248,"num_write":65,"num_write_kb":658,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":1073395,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":10092544,"num_objects":154,"num_object_clones":0,"num_object_copies":462,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":154,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":154,"num_write_kb":9856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":53018624,"data_stored":53018624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":154,"ondisk_log_size":154,"up":48,"acting":48,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5983,"num_read_kb":13200,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":68,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6082,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201636,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":390260,"kb_used_data":9196,"kb_used_omap":0,"kb_used_meta":381056,"kb_avail":93354892,"statfs":{"total":95995035648,"available":95595409408,"internally_reserved":0,"allocated":9416704,"data_stored":7968797,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":390201344},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:40:54 2023","interfaces":[{"interface":"back","average":{"1min":1.423,"5min":1.968,"15min":2.1520000000000001},"min":{"1min":0.27200000000000002,"5min":0.24399999999999999,"15min":0.20799999999999999},"max":{"1min":5.6310000000000002,"5min":18.315000000000001,"15min":48.798999999999999},"last":0.55600000000000005},{"interface":"front","average":{"1min":1.069,"5min":1.2909999999999999,"15min":1.974},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":5.3330000000000002,"5min":9.4429999999999996,"15min":48.247},"last":0.439}]},{"osd":1,"last update":"Sat Sep 23 20:41:06 2023","interfaces":[{"interface":"back","average":{"1min":0.91400000000000003,"5min":2.2770000000000001,"15min":2.0960000000000001},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.254},"max":{"1min":4.2460000000000004,"5min":49.965000000000003,"15min":49.965000000000003},"last":0.46100000000000002},{"interface":"front","average":{"1min":1.147,"5min":2.323,"15min":2.1160000000000001},"min":{"1min":0.29799999999999999,"5min":0.26900000000000002,"15min":0.248},"max":{"1min":4.2999999999999998,"5min":53.451999999999998,"15min":53.451999999999998},"last":0.63600000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:40:54 2023","interfaces":[{"interface":"back","average":{"1min":1.339,"5min":1.736,"15min":2.0840000000000001},"min":{"1min":0.32000000000000001,"5min":0.28599999999999998,"15min":0.27600000000000002},"max":{"1min":4.5039999999999996,"5min":16.965,"15min":16.965},"last":0.49199999999999999},{"interface":"front","average":{"1min":1.4630000000000001,"5min":2.1960000000000002,"15min":2.4489999999999998},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.26600000000000001},"max":{"1min":6.4329999999999998,"5min":23.414000000000001,"15min":23.494},"last":0.60299999999999998}]},{"osd":3,"last update":"Sat Sep 23 20:40:21 2023","interfaces":[{"interface":"back","average":{"1min":1.105,"5min":2.2480000000000002,"15min":2.4470000000000001},"min":{"1min":0.35999999999999999,"5min":0.311,"15min":0.26200000000000001},"max":{"1min":4.2149999999999999,"5min":23.352,"15min":23.352},"last":0.54500000000000004},{"interface":"front","average":{"1min":1.139,"5min":2.2709999999999999,"15min":2.423},"min":{"1min":0.46500000000000002,"5min":0.34399999999999997,"15min":0.32500000000000001},"max":{"1min":4.2359999999999998,"5min":20.969999999999999,"15min":20.969999999999999},"last":0.57299999999999995}]},{"osd":4,"last update":"Sat Sep 23 20:40:21 2023","interfaces":[{"interface":"back","average":{"1min":1.046,"5min":1.085,"15min":1.2170000000000001},"min":{"1min":0.35799999999999998,"5min":0.25600000000000001,"15min":0.23200000000000001},"max":{"1min":4.1639999999999997,"5min":18.364999999999998,"15min":18.364999999999998},"last":0.40000000000000002},{"interface":"front","average":{"1min":0.94999999999999996,"5min":1.296,"15min":1.417},"min":{"1min":0.39100000000000001,"5min":0.28399999999999997,"15min":0.27600000000000002},"max":{"1min":2.605,"5min":23.617000000000001,"15min":23.617000000000001},"last":0.53300000000000003}]},{"osd":5,"last update":"Sat Sep 23 20:40:21 2023","interfaces":[{"interface":"back","average":{"1min":1.0780000000000001,"5min":1.1499999999999999,"15min":1.2509999999999999},"min":{"1min":0.30099999999999999,"5min":0.29199999999999998,"15min":0.26600000000000001},"max":{"1min":4.3780000000000001,"5min":12.516,"15min":13.241},"last":0.50600000000000001},{"interface":"front","average":{"1min":0.76400000000000001,"5min":1.286,"15min":1.3540000000000001},"min":{"1min":0.38400000000000001,"5min":0.31900000000000001,"15min":0.315},"max":{"1min":3.1030000000000002,"5min":18.390000000000001,"15min":18.831},"last":0.51800000000000002}]},{"osd":6,"last update":"Sat Sep 23 20:40:21 2023","interfaces":[{"interface":"back","average":{"1min":1.2629999999999999,"5min":1.4350000000000001,"15min":1.329},"min":{"1min":0.44,"5min":0.312,"15min":0.30399999999999999},"max":{"1min":4.0540000000000003,"5min":18.417000000000002,"15min":18.417000000000002},"last":0.61799999999999999},{"interface":"front","average":{"1min":1.1659999999999999,"5min":1.113,"15min":1.252},"min":{"1min":0.38200000000000001,"5min":0.32700000000000001,"15min":0.318},"max":{"1min":6.4340000000000002,"5min":7.266,"15min":9.9120000000000008},"last":0.67800000000000005}]}]},{"osd":6,"up_from":47,"seq":201863463275,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":433512,"kb_used_data":43104,"kb_used_omap":0,"kb_used_meta":390400,"kb_avail":93311640,"statfs":{"total":95995035648,"available":95551119360,"internally_reserved":0,"allocated":44138496,"data_stored":42702206,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":399769600},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:40:50 2023","interfaces":[{"interface":"back","average":{"1min":1.1910000000000001,"5min":1.7470000000000001,"15min":1.9159999999999999},"min":{"1min":0.28100000000000003,"5min":0.23300000000000001,"15min":0.221},"max":{"1min":4.1399999999999997,"5min":17.492999999999999,"15min":53.701999999999998},"last":0.441},{"interface":"front","average":{"1min":1.395,"5min":2.306,"15min":2.3399999999999999},"min":{"1min":0.253,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":6.0570000000000004,"5min":60.328000000000003,"15min":60.328000000000003},"last":0.36399999999999999}]},{"osd":1,"last update":"Sat Sep 23 20:40:06 2023","interfaces":[{"interface":"back","average":{"1min":1.157,"5min":2.0600000000000001,"15min":2.3610000000000002},"min":{"1min":0.26600000000000001,"5min":0.26000000000000001,"15min":0.249},"max":{"1min":5.8630000000000004,"5min":64.274000000000001,"15min":64.274000000000001},"last":2.8250000000000002},{"interface":"front","average":{"1min":1.2390000000000001,"5min":1.393,"15min":2.4830000000000001},"min":{"1min":0.34499999999999997,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":6.8460000000000001,"5min":6.8460000000000001,"15min":34.490000000000002},"last":2.8879999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:40:50 2023","interfaces":[{"interface":"back","average":{"1min":2.097,"5min":1.6419999999999999,"15min":2.133},"min":{"1min":0.31,"5min":0.30299999999999999,"15min":0.27700000000000002},"max":{"1min":9.9049999999999994,"5min":13.010999999999999,"15min":34.317999999999998},"last":0.61399999999999999},{"interface":"front","average":{"1min":2.1080000000000001,"5min":1.585,"15min":2.0569999999999999},"min":{"1min":0.311,"5min":0.311,"15min":0.30399999999999999},"max":{"1min":9.9269999999999996,"5min":12.917999999999999,"15min":34.292999999999999},"last":0.626}]},{"osd":3,"last update":"Sat Sep 23 20:40:36 2023","interfaces":[{"interface":"back","average":{"1min":2.0099999999999998,"5min":1.8759999999999999,"15min":1.9910000000000001},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":5.9130000000000003,"5min":8.9670000000000005,"15min":9.4239999999999995},"last":0.46000000000000002},{"interface":"front","average":{"1min":1.921,"5min":1.968,"15min":2.1659999999999999},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.317},"max":{"1min":5.8380000000000001,"5min":11.913,"15min":12.875999999999999},"last":0.53400000000000003}]},{"osd":4,"last update":"Sat Sep 23 20:40:11 2023","interfaces":[{"interface":"back","average":{"1min":0.90200000000000002,"5min":0.84799999999999998,"15min":0.95399999999999996},"min":{"1min":0.30099999999999999,"5min":0.26900000000000002,"15min":0.251},"max":{"1min":2.859,"5min":23.904,"15min":23.904},"last":0.47399999999999998},{"interface":"front","average":{"1min":1.498,"5min":1.1879999999999999,"15min":1.204},"min":{"1min":0.36799999999999999,"5min":0.29199999999999998,"15min":0.27000000000000002},"max":{"1min":6.7359999999999998,"5min":11.515000000000001,"15min":11.515000000000001},"last":0.42799999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:40:11 2023","interfaces":[{"interface":"back","average":{"1min":1.2210000000000001,"5min":0.85299999999999998,"15min":0.96199999999999997},"min":{"1min":0.375,"5min":0.318,"15min":0.28699999999999998},"max":{"1min":6.6340000000000003,"5min":6.6340000000000003,"15min":14.085000000000001},"last":0.40400000000000003},{"interface":"front","average":{"1min":1.2529999999999999,"5min":0.97899999999999998,"15min":1.355},"min":{"1min":0.35399999999999998,"5min":0.32500000000000001,"15min":0.30599999999999999},"max":{"1min":6.6909999999999998,"5min":6.7320000000000002,"15min":18.192},"last":0.48799999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:40:36 2023","interfaces":[{"interface":"back","average":{"1min":1.1240000000000001,"5min":0.88300000000000001,"15min":1.121},"min":{"1min":0.40400000000000003,"5min":0.376,"15min":0.28599999999999998},"max":{"1min":5.2729999999999997,"5min":5.2729999999999997,"15min":10.217000000000001},"last":0.504},{"interface":"front","average":{"1min":0.97899999999999998,"5min":1.048,"15min":1.3120000000000001},"min":{"1min":0.45700000000000002,"5min":0.36799999999999999,"15min":0.30299999999999999},"max":{"1min":2.2909999999999999,"5min":8.8119999999999994,"15min":9.7360000000000007},"last":0.51300000000000001}]}]},{"osd":1,"up_from":266,"seq":1142461300874,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":470196,"kb_used_data":25772,"kb_used_omap":8,"kb_used_meta":444407,"kb_avail":93274956,"statfs":{"total":95995035648,"available":95513554944,"internally_reserved":0,"allocated":26390528,"data_stored":23905657,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8206,"internal_metadata":455073778},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:41:07 2023","interfaces":[{"interface":"back","average":{"1min":2.1469999999999998,"5min":2.0030000000000001,"15min":2.4980000000000002},"min":{"1min":0.215,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":16.292000000000002,"5min":16.292000000000002,"15min":52.689999999999998},"last":0.34599999999999997},{"interface":"front","average":{"1min":2.1419999999999999,"5min":2.6560000000000001,"15min":3.1949999999999998},"min":{"1min":0.20100000000000001,"5min":0.20100000000000001,"15min":0.19900000000000001},"max":{"1min":16.332999999999998,"5min":17.169,"15min":52.662999999999997},"last":0.44}]},{"osd":2,"last update":"Sat Sep 23 20:41:07 2023","interfaces":[{"interface":"back","average":{"1min":1.8220000000000001,"5min":2.5139999999999998,"15min":3.8610000000000002},"min":{"1min":0.216,"5min":0.216,"15min":0.20899999999999999},"max":{"1min":10.404,"5min":32.255000000000003,"15min":32.255000000000003},"last":0.47599999999999998},{"interface":"front","average":{"1min":1.611,"5min":1.9610000000000001,"15min":2.2839999999999998},"min":{"1min":0.24399999999999999,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":5.9699999999999998,"5min":13.404999999999999,"15min":13.404999999999999},"last":0.29999999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:41:07 2023","interfaces":[{"interface":"back","average":{"1min":1.7929999999999999,"5min":1.8460000000000001,"15min":3.4039999999999999},"min":{"1min":0.33800000000000002,"5min":0.247,"15min":0.247},"max":{"1min":16.245999999999999,"5min":16.245999999999999,"15min":80.745999999999995},"last":0.40300000000000002},{"interface":"front","average":{"1min":2.343,"5min":2.202,"15min":3.7650000000000001},"min":{"1min":0.34300000000000003,"5min":0.27500000000000002,"15min":0.22700000000000001},"max":{"1min":13.497999999999999,"5min":14.048999999999999,"15min":82.988},"last":0.624}]},{"osd":4,"last update":"Sat Sep 23 20:41:07 2023","interfaces":[{"interface":"back","average":{"1min":0.96899999999999997,"5min":1.75,"15min":2.1349999999999998},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30299999999999999},"max":{"1min":3.149,"5min":7.8399999999999999,"15min":24.327999999999999},"last":0.56599999999999995},{"interface":"front","average":{"1min":1.1240000000000001,"5min":1.3300000000000001,"15min":1.704},"min":{"1min":0.32600000000000001,"5min":0.311,"15min":0.311},"max":{"1min":4.3319999999999999,"5min":9.3339999999999996,"15min":11.574999999999999},"last":0.505}]},{"osd":5,"last update":"Sat Sep 23 20:41:07 2023","interfaces":[{"interface":"back","average":{"1min":2.02,"5min":2.129,"15min":2.3199999999999998},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.33200000000000002},"max":{"1min":13.550000000000001,"5min":33.401000000000003,"15min":33.401000000000003},"last":0.70999999999999996},{"interface":"front","average":{"1min":1.008,"5min":2.2069999999999999,"15min":2.355},"min":{"1min":0.377,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":3.238,"5min":33.417000000000002,"15min":33.417000000000002},"last":0.59799999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:41:07 2023","interfaces":[{"interface":"back","average":{"1min":2.0329999999999999,"5min":2.0049999999999999,"15min":3.8090000000000002},"min":{"1min":0.43099999999999999,"5min":0.42599999999999999,"15min":0.35699999999999998},"max":{"1min":13.593999999999999,"5min":13.593999999999999,"15min":72.682000000000002},"last":0.76900000000000002},{"interface":"front","average":{"1min":1.0669999999999999,"5min":1.7789999999999999,"15min":3.6869999999999998},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":3.2050000000000001,"5min":7.883,"15min":72.563999999999993},"last":0.52400000000000002}]},{"osd":7,"last update":"Sat Sep 23 20:41:07 2023","interfaces":[{"interface":"back","average":{"1min":2.1970000000000001,"5min":1.97,"15min":2.5030000000000001},"min":{"1min":0.40300000000000002,"5min":0.39500000000000002,"15min":0.39300000000000002},"max":{"1min":16.359000000000002,"5min":16.359000000000002,"15min":18.437000000000001},"last":0.81100000000000005},{"interface":"front","average":{"1min":1.1830000000000001,"5min":1.8859999999999999,"15min":2.3809999999999998},"min":{"1min":0.44800000000000001,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":3.27,"5min":7.9089999999999998,"15min":24.524000000000001},"last":0.75800000000000001}]}]},{"osd":0,"up_from":94,"seq":403726926076,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":651136,"kb_used_data":6136,"kb_used_omap":7,"kb_used_meta":644984,"kb_avail":93094016,"statfs":{"total":95995035648,"available":95328272384,"internally_reserved":0,"allocated":6283264,"data_stored":3955582,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":660463633},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:40:11 2023","interfaces":[{"interface":"back","average":{"1min":1.712,"5min":1.8,"15min":3.0680000000000001},"min":{"1min":0.20699999999999999,"5min":0.17199999999999999,"15min":0.17199999999999999},"max":{"1min":9.8879999999999999,"5min":11.554,"15min":42.000999999999998},"last":0.25600000000000001},{"interface":"front","average":{"1min":1.5660000000000001,"5min":1.9710000000000001,"15min":3.3370000000000002},"min":{"1min":0.218,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":10.097,"5min":12.35,"15min":23.036000000000001},"last":0.33600000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:40:50 2023","interfaces":[{"interface":"back","average":{"1min":3.0609999999999999,"5min":2.657,"15min":2.9849999999999999},"min":{"1min":0.29199999999999998,"5min":0.218,"15min":0.185},"max":{"1min":13.871,"5min":62.726999999999997,"15min":77.448999999999998},"last":0.34899999999999998},{"interface":"front","average":{"1min":2.6600000000000001,"5min":2.875,"15min":2.8919999999999999},"min":{"1min":0.27800000000000002,"5min":0.247,"15min":0.188},"max":{"1min":13.356999999999999,"5min":62.689,"15min":81.724999999999994},"last":0.307}]},{"osd":3,"last update":"Sat Sep 23 20:40:23 2023","interfaces":[{"interface":"back","average":{"1min":2.399,"5min":2.7450000000000001,"15min":3.133},"min":{"1min":0.34699999999999998,"5min":0.24399999999999999,"15min":0.23400000000000001},"max":{"1min":9.0039999999999996,"5min":51.536000000000001,"15min":81.400999999999996},"last":0.38700000000000001},{"interface":"front","average":{"1min":2.8839999999999999,"5min":3.1389999999999998,"15min":3.4590000000000001},"min":{"1min":0.245,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":14.237,"5min":44.811999999999998,"15min":81.137},"last":0.40899999999999997}]},{"osd":4,"last update":"Sat Sep 23 20:40:50 2023","interfaces":[{"interface":"back","average":{"1min":2.7970000000000002,"5min":1.9019999999999999,"15min":2.1920000000000002},"min":{"1min":0.432,"5min":0.28899999999999998,"15min":0.25900000000000001},"max":{"1min":6.8970000000000002,"5min":9.6110000000000007,"15min":23.876999999999999},"last":0.44500000000000001},{"interface":"front","average":{"1min":2.9929999999999999,"5min":1.615,"15min":1.9770000000000001},"min":{"1min":0.36299999999999999,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":9.7789999999999999,"5min":9.7789999999999999,"15min":23.681999999999999},"last":0.5}]},{"osd":5,"last update":"Sat Sep 23 20:40:50 2023","interfaces":[{"interface":"back","average":{"1min":3.0649999999999999,"5min":1.6519999999999999,"15min":2.1819999999999999},"min":{"1min":0.41499999999999998,"5min":0.371,"15min":0.34200000000000003},"max":{"1min":9.8629999999999995,"5min":9.8629999999999995,"15min":24.198},"last":0.54100000000000004},{"interface":"front","average":{"1min":2.8439999999999999,"5min":2.4449999999999998,"15min":2.3170000000000002},"min":{"1min":0.38900000000000001,"5min":0.318,"15min":0.25600000000000001},"max":{"1min":12.929,"5min":26.228000000000002,"15min":26.228000000000002},"last":0.52800000000000002}]},{"osd":6,"last update":"Sat Sep 23 20:40:50 2023","interfaces":[{"interface":"back","average":{"1min":3.6520000000000001,"5min":1.9650000000000001,"15min":2.2549999999999999},"min":{"1min":0.39800000000000002,"5min":0.34499999999999997,"15min":0.318},"max":{"1min":24.056000000000001,"5min":24.056000000000001,"15min":24.408000000000001},"last":0.56200000000000006},{"interface":"front","average":{"1min":2.1219999999999999,"5min":2.2770000000000001,"15min":2.282},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.29399999999999998},"max":{"1min":8.0549999999999997,"5min":26.268000000000001,"15min":26.268000000000001},"last":0.48999999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:40:50 2023","interfaces":[{"interface":"back","average":{"1min":2.4239999999999999,"5min":2.3180000000000001,"15min":2.4220000000000002},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.30599999999999999},"max":{"1min":8.1189999999999998,"5min":26.177,"15min":26.177},"last":0.46000000000000002},{"interface":"front","average":{"1min":2.9910000000000001,"5min":1.966,"15min":2.569},"min":{"1min":0.53700000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":6.8979999999999997,"5min":9.7200000000000006,"15min":24.669},"last":0.57199999999999995}]}]},{"osd":2,"up_from":104,"seq":446676599024,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":683836,"kb_used_data":10164,"kb_used_omap":11,"kb_used_meta":673652,"kb_avail":93061316,"statfs":{"total":95995035648,"available":95294787584,"internally_reserved":0,"allocated":10407936,"data_stored":8082092,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11878,"internal_metadata":689820058},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:41:03 2023","interfaces":[{"interface":"back","average":{"1min":0.97199999999999998,"5min":1.327,"15min":2.617},"min":{"1min":0.25,"5min":0.20300000000000001,"15min":0.17999999999999999},"max":{"1min":5.875,"5min":12.007,"15min":82.424999999999997},"last":0.37},{"interface":"front","average":{"1min":1.583,"5min":1.7949999999999999,"15min":2.6339999999999999},"min":{"1min":0.27000000000000002,"5min":0.19,"15min":0.19},"max":{"1min":9.2379999999999995,"5min":9.5600000000000005,"15min":82.524000000000001},"last":0.40600000000000003}]},{"osd":1,"last update":"Sat Sep 23 20:41:03 2023","interfaces":[{"interface":"back","average":{"1min":1.234,"5min":2.4780000000000002,"15min":3.3820000000000001},"min":{"1min":0.27100000000000002,"5min":0.245,"15min":0.221},"max":{"1min":7.8250000000000002,"5min":46.405000000000001,"15min":46.405000000000001},"last":0.35399999999999998},{"interface":"front","average":{"1min":1.105,"5min":1.9430000000000001,"15min":2.7749999999999999},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":5.5270000000000001,"5min":46.459000000000003,"15min":46.459000000000003},"last":0.29299999999999998}]},{"osd":3,"last update":"Sat Sep 23 20:40:22 2023","interfaces":[{"interface":"back","average":{"1min":2.851,"5min":2.613,"15min":2.4769999999999999},"min":{"1min":0.28499999999999998,"5min":0.27100000000000002,"15min":0.19800000000000001},"max":{"1min":11.304,"5min":19.562999999999999,"15min":19.562999999999999},"last":0.313},{"interface":"front","average":{"1min":1.7909999999999999,"5min":1.6160000000000001,"15min":2.5089999999999999},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.23899999999999999},"max":{"1min":8.9369999999999994,"5min":13.148999999999999,"15min":30.704999999999998},"last":0.42699999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:41:03 2023","interfaces":[{"interface":"back","average":{"1min":1.345,"5min":1.4399999999999999,"15min":1.917},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.27500000000000002},"max":{"1min":5.6150000000000002,"5min":6.6849999999999996,"15min":16.797999999999998},"last":0.56599999999999995},{"interface":"front","average":{"1min":1.1799999999999999,"5min":1.601,"15min":2.0899999999999999},"min":{"1min":0.379,"5min":0.379,"15min":0.27600000000000002},"max":{"1min":3.758,"5min":19.440999999999999,"15min":19.440999999999999},"last":0.38700000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:41:03 2023","interfaces":[{"interface":"back","average":{"1min":1.4730000000000001,"5min":1.488,"15min":2.403},"min":{"1min":0.43099999999999999,"5min":0.36599999999999999,"15min":0.309},"max":{"1min":5.6580000000000004,"5min":6.6319999999999997,"15min":95.393000000000001},"last":0.51700000000000002},{"interface":"front","average":{"1min":1.1419999999999999,"5min":1.613,"15min":2.6669999999999998},"min":{"1min":0.44800000000000001,"5min":0.41399999999999998,"15min":0.32300000000000001},"max":{"1min":3.8279999999999998,"5min":19.513000000000002,"15min":95.414000000000001},"last":0.44900000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:41:03 2023","interfaces":[{"interface":"back","average":{"1min":1.1279999999999999,"5min":1.585,"15min":2.177},"min":{"1min":0.432,"5min":0.42399999999999999,"15min":0.32100000000000001},"max":{"1min":3.7949999999999999,"5min":19.486000000000001,"15min":19.486000000000001},"last":0.503},{"interface":"front","average":{"1min":0.94199999999999995,"5min":1.05,"15min":1.9259999999999999},"min":{"1min":0.42099999999999999,"5min":0.36899999999999999,"15min":0.316},"max":{"1min":5.5789999999999997,"5min":9.3010000000000002,"15min":30.670999999999999},"last":0.48699999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:41:03 2023","interfaces":[{"interface":"back","average":{"1min":1.2030000000000001,"5min":1.038,"15min":1.9650000000000001},"min":{"1min":0.47399999999999998,"5min":0.39300000000000002,"15min":0.35199999999999998},"max":{"1min":7.5860000000000003,"5min":7.5860000000000003,"15min":30.741},"last":0.53800000000000003},{"interface":"front","average":{"1min":1.831,"5min":1.6870000000000001,"15min":2.1739999999999999},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":5.8529999999999998,"5min":6.7220000000000004,"15min":16.719000000000001},"last":0.60399999999999998}]}]},{"osd":3,"up_from":109,"seq":468151435498,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":803240,"kb_used_data":6112,"kb_used_omap":4,"kb_used_meta":797115,"kb_avail":92941912,"statfs":{"total":95995035648,"available":95172517888,"internally_reserved":0,"allocated":6258688,"data_stored":3949645,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4514,"internal_metadata":816246366},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:40:26 2023","interfaces":[{"interface":"back","average":{"1min":1.409,"5min":1.7350000000000001,"15min":2.544},"min":{"1min":0.21199999999999999,"5min":0.21199999999999999,"15min":0.186},"max":{"1min":7.6879999999999997,"5min":14.638,"15min":67.305999999999997},"last":0.32000000000000001},{"interface":"front","average":{"1min":2.4540000000000002,"5min":1.865,"15min":2.415},"min":{"1min":0.29499999999999998,"5min":0.26600000000000001,"15min":0.19500000000000001},"max":{"1min":8.8439999999999994,"5min":10.140000000000001,"15min":67.403999999999996},"last":0.436}]},{"osd":1,"last update":"Sat Sep 23 20:41:05 2023","interfaces":[{"interface":"back","average":{"1min":1.244,"5min":1.212,"15min":2.2519999999999998},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":4.8099999999999996,"5min":9.5299999999999994,"15min":16.189},"last":0.40400000000000003},{"interface":"front","average":{"1min":2.262,"5min":2.0369999999999999,"15min":2.617},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":8.8030000000000008,"5min":12.702999999999999,"15min":32.603999999999999},"last":2.0710000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:40:26 2023","interfaces":[{"interface":"back","average":{"1min":2.3140000000000001,"5min":1.837,"15min":2.5030000000000001},"min":{"1min":0.26600000000000001,"5min":0.246,"15min":0.22800000000000001},"max":{"1min":6.3200000000000003,"5min":10.747999999999999,"15min":32.857999999999997},"last":0.495},{"interface":"front","average":{"1min":1.516,"5min":2.0259999999999998,"15min":2.4980000000000002},"min":{"1min":0.31,"5min":0.28100000000000003,"15min":0.20899999999999999},"max":{"1min":8.5530000000000008,"5min":22.32,"15min":24.146999999999998},"last":0.38}]},{"osd":4,"last update":"Sat Sep 23 20:40:26 2023","interfaces":[{"interface":"back","average":{"1min":0.82199999999999995,"5min":1.0860000000000001,"15min":1.786},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.29799999999999999},"max":{"1min":2.5899999999999999,"5min":9.3160000000000007,"15min":46.042999999999999},"last":0.61899999999999999},{"interface":"front","average":{"1min":1.6359999999999999,"5min":1.879,"15min":2.4340000000000002},"min":{"1min":0.33200000000000002,"5min":0.33000000000000002,"15min":0.28100000000000003},"max":{"1min":5.141,"5min":13.571999999999999,"15min":46.579999999999998},"last":1.1419999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:40:26 2023","interfaces":[{"interface":"back","average":{"1min":0.82099999999999995,"5min":1.167,"15min":1.998},"min":{"1min":0.32700000000000001,"5min":0.32300000000000001,"15min":0.31900000000000001},"max":{"1min":2.9889999999999999,"5min":9.4130000000000003,"15min":45.292000000000002},"last":0.75900000000000001},{"interface":"front","average":{"1min":1.6859999999999999,"5min":1.482,"15min":1.968},"min":{"1min":0.38500000000000001,"5min":0.34399999999999997,"15min":0.313},"max":{"1min":5.0060000000000002,"5min":9.0839999999999996,"15min":44.679000000000002},"last":2.8420000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:40:26 2023","interfaces":[{"interface":"back","average":{"1min":1.8640000000000001,"5min":1.3959999999999999,"15min":1.964},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":7.3630000000000004,"5min":9.0470000000000006,"15min":17.524999999999999},"last":0.72499999999999998},{"interface":"front","average":{"1min":1.4750000000000001,"5min":1.8839999999999999,"15min":2.165},"min":{"1min":0.434,"5min":0.39800000000000002,"15min":0.33700000000000002},"max":{"1min":4.8819999999999997,"5min":13.500999999999999,"15min":19.524000000000001},"last":1.202}]},{"osd":7,"last update":"Sat Sep 23 20:40:26 2023","interfaces":[{"interface":"back","average":{"1min":1.5109999999999999,"5min":2.1829999999999998,"15min":2.3959999999999999},"min":{"1min":0.433,"5min":0.39900000000000002,"15min":0.34999999999999998},"max":{"1min":4.319,"5min":13.544,"15min":19.571999999999999},"last":1.069},{"interface":"front","average":{"1min":0.996,"5min":1.274,"15min":2.0720000000000001},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.371},"max":{"1min":4.157,"5min":9.3719999999999999,"15min":45.387999999999998},"last":0.67900000000000005}]}]},{"osd":4,"up_from":30,"seq":128849019251,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":369440,"kb_used_data":4824,"kb_used_omap":0,"kb_used_meta":364608,"kb_avail":93375712,"statfs":{"total":95995035648,"available":95616729088,"internally_reserved":0,"allocated":4939776,"data_stored":3503323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":373358592},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:40:49 2023","interfaces":[{"interface":"back","average":{"1min":2.1949999999999998,"5min":1.6739999999999999,"15min":1.724},"min":{"1min":0.31,"5min":0.24099999999999999,"15min":0.23699999999999999},"max":{"1min":11.553000000000001,"5min":24.114999999999998,"15min":24.114999999999998},"last":0.51700000000000002},{"interface":"front","average":{"1min":2.2789999999999999,"5min":1.8080000000000001,"15min":1.895},"min":{"1min":0.28899999999999998,"5min":0.246,"15min":0.246},"max":{"1min":11.536,"5min":24.135999999999999,"15min":24.135999999999999},"last":0.52800000000000002}]},{"osd":1,"last update":"Sat Sep 23 20:40:06 2023","interfaces":[{"interface":"back","average":{"1min":0.95899999999999996,"5min":1.331,"15min":1.7410000000000001},"min":{"1min":0.309,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":4.1669999999999998,"5min":18.190999999999999,"15min":18.190999999999999},"last":0.57199999999999995},{"interface":"front","average":{"1min":0.88300000000000001,"5min":1.4259999999999999,"15min":1.6899999999999999},"min":{"1min":0.307,"5min":0.28499999999999998,"15min":0.28199999999999997},"max":{"1min":3.7109999999999999,"5min":18.173999999999999,"15min":18.173999999999999},"last":0.58299999999999996}]},{"osd":2,"last update":"Sat Sep 23 20:41:05 2023","interfaces":[{"interface":"back","average":{"1min":0.99099999999999999,"5min":1.264,"15min":1.821},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.26600000000000001},"max":{"1min":4.4640000000000004,"5min":7.75,"15min":32.363},"last":0.60199999999999998},{"interface":"front","average":{"1min":0.82899999999999996,"5min":1.6779999999999999,"15min":2.2120000000000002},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.27600000000000002},"max":{"1min":2.4590000000000001,"5min":11.917999999999999,"15min":34.042999999999999},"last":0.48699999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:40:17 2023","interfaces":[{"interface":"back","average":{"1min":2.4209999999999998,"5min":1.8480000000000001,"15min":2.113},"min":{"1min":0.40500000000000003,"5min":0.372,"15min":0.29099999999999998},"max":{"1min":12.147,"5min":12.147,"15min":32.332999999999998},"last":0.47299999999999998},{"interface":"front","average":{"1min":1.8280000000000001,"5min":1.8360000000000001,"15min":2.206},"min":{"1min":0.371,"5min":0.371,"15min":0.30299999999999999},"max":{"1min":11.914,"5min":11.914,"15min":32.399999999999999},"last":0.55200000000000005}]},{"osd":5,"last update":"Sat Sep 23 20:40:47 2023","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.68200000000000005,"15min":1.51},"min":{"1min":0.28100000000000003,"5min":0.26400000000000001,"15min":0.247},"max":{"1min":2.202,"5min":4.1260000000000003,"15min":82.866},"last":0.497},{"interface":"front","average":{"1min":1.006,"5min":0.96999999999999997,"15min":1.4810000000000001},"min":{"1min":0.32200000000000001,"5min":0.28999999999999998,"15min":0.27800000000000002},"max":{"1min":5.2670000000000003,"5min":7.0129999999999999,"15min":82.546000000000006},"last":0.377}]},{"osd":6,"last update":"Sat Sep 23 20:41:05 2023","interfaces":[{"interface":"back","average":{"1min":0.67300000000000004,"5min":0.96299999999999997,"15min":1.258},"min":{"1min":0.35799999999999998,"5min":0.32200000000000001,"15min":0.27600000000000002},"max":{"1min":2.7450000000000001,"5min":14.731999999999999,"15min":15.252000000000001},"last":0.44900000000000001},{"interface":"front","average":{"1min":0.64500000000000002,"5min":0.88200000000000001,"15min":1.1240000000000001},"min":{"1min":0.314,"5min":0.309,"15min":0.27200000000000002},"max":{"1min":2.5539999999999998,"5min":7.2850000000000001,"15min":11.199},"last":0.40300000000000002}]},{"osd":7,"last update":"Sat Sep 23 20:40:17 2023","interfaces":[{"interface":"back","average":{"1min":1.0389999999999999,"5min":1.143,"15min":1.5369999999999999},"min":{"1min":0.39100000000000001,"5min":0.318,"15min":0.318},"max":{"1min":4.21,"5min":14.534000000000001,"15min":16.427},"last":0.43099999999999999},{"interface":"front","average":{"1min":1.472,"5min":0.84599999999999997,"15min":1.3180000000000001},"min":{"1min":0.36399999999999999,"5min":0.32700000000000001,"15min":0.307},"max":{"1min":7.9390000000000001,"5min":7.9390000000000001,"15min":19.018999999999998},"last":0.53900000000000003}]}]},{"osd":5,"up_from":39,"seq":167503724912,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1296540,"kb_used_data":900372,"kb_used_omap":0,"kb_used_meta":396160,"kb_avail":92448612,"statfs":{"total":95995035648,"available":94667378688,"internally_reserved":0,"allocated":921980928,"data_stored":920545201,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":405667840},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:40:53 2023","interfaces":[{"interface":"back","average":{"1min":1.3640000000000001,"5min":1.397,"15min":1.758},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.20699999999999999},"max":{"1min":5.8330000000000002,"5min":7.819,"15min":18.16},"last":0.47399999999999998},{"interface":"front","average":{"1min":1.3149999999999999,"5min":1.4770000000000001,"15min":1.665},"min":{"1min":0.29899999999999999,"5min":0.29499999999999998,"15min":0.245},"max":{"1min":5.157,"5min":7.9859999999999998,"15min":9.0570000000000004},"last":0.44500000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:40:07 2023","interfaces":[{"interface":"back","average":{"1min":1.6799999999999999,"5min":2.0070000000000001,"15min":1.9179999999999999},"min":{"1min":0.26800000000000002,"5min":0.26300000000000001,"15min":0.253},"max":{"1min":5.71,"5min":36.588999999999999,"15min":36.588999999999999},"last":0.32400000000000001},{"interface":"front","average":{"1min":1.2470000000000001,"5min":2.0150000000000001,"15min":1.6970000000000001},"min":{"1min":0.29799999999999999,"5min":0.27700000000000002,"15min":0.23499999999999999},"max":{"1min":5.298,"5min":36.920999999999999,"15min":36.920999999999999},"last":0.39400000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:40:53 2023","interfaces":[{"interface":"back","average":{"1min":2.0840000000000001,"5min":1.9470000000000001,"15min":1.9590000000000001},"min":{"1min":0.28000000000000003,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":9.2119999999999997,"5min":37.011000000000003,"15min":37.011000000000003},"last":0.45400000000000001},{"interface":"front","average":{"1min":1.5,"5min":1.74,"15min":1.978},"min":{"1min":0.32800000000000001,"5min":0.28999999999999998,"15min":0.255},"max":{"1min":5.4889999999999999,"5min":36.835000000000001,"15min":36.835000000000001},"last":0.38}]},{"osd":3,"last update":"Sat Sep 23 20:40:14 2023","interfaces":[{"interface":"back","average":{"1min":2.2320000000000002,"5min":2.153,"15min":2.2360000000000002},"min":{"1min":0.435,"5min":0.34699999999999998,"15min":0.25700000000000001},"max":{"1min":6.0309999999999997,"5min":36.716000000000001,"15min":36.716000000000001},"last":0.433},{"interface":"front","average":{"1min":1.5289999999999999,"5min":2.2240000000000002,"15min":2.117},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.28799999999999998},"max":{"1min":8.3599999999999994,"5min":18.504000000000001,"15min":18.504000000000001},"last":0.49099999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:40:44 2023","interfaces":[{"interface":"back","average":{"1min":1.2689999999999999,"5min":0.90600000000000003,"15min":1.3140000000000001},"min":{"1min":0.29999999999999999,"5min":0.255,"15min":0.255},"max":{"1min":6.3780000000000001,"5min":6.9180000000000001,"15min":28.015999999999998},"last":0.27300000000000002},{"interface":"front","average":{"1min":1.5760000000000001,"5min":0.93700000000000006,"15min":1.0940000000000001},"min":{"1min":0.32500000000000001,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":5.7469999999999999,"5min":6.8380000000000001,"15min":12.300000000000001},"last":0.30299999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:40:58 2023","interfaces":[{"interface":"back","average":{"1min":1.1160000000000001,"5min":1.089,"15min":1.1970000000000001},"min":{"1min":0.34599999999999997,"5min":0.318,"15min":0.30199999999999999},"max":{"1min":6.0739999999999998,"5min":6.9960000000000004,"15min":12.890000000000001},"last":0.36499999999999999},{"interface":"front","average":{"1min":1.339,"5min":1.075,"15min":1.117},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":5.8609999999999998,"5min":6.7930000000000001,"15min":16.265999999999998},"last":0.42399999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:40:14 2023","interfaces":[{"interface":"back","average":{"1min":1.3939999999999999,"5min":0.84299999999999997,"15min":1.244},"min":{"1min":0.38400000000000001,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":5.2430000000000003,"5min":5.2430000000000003,"15min":36.621000000000002},"last":0.34499999999999997},{"interface":"front","average":{"1min":1.0189999999999999,"5min":0.90700000000000003,"15min":1.151},"min":{"1min":0.35699999999999998,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":4.3250000000000002,"5min":6.5970000000000004,"15min":37.331000000000003},"last":0.41199999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4722,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1360,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8206,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7156,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3154,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":53018624,"data_stored":53018624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:41:12.009 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:41:12.016 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:12.017 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:41:12.118 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:41:12.130 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 3 16 9170 9154 190.577 188.375 0.00427254 0.00523772 2023-09-23T20:41:12.174 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:12.339 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:41:12.344 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:12.345 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:41:12.446 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:41:12.514 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:12.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:12 smithi018 ceph-mon[179647]: 1.17 scrub starts 2023-09-23T20:41:12.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:12 smithi018 ceph-mon[179647]: 1.17 scrub ok 2023-09-23T20:41:12.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:12 smithi018 ceph-mon[179647]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:41:12.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:12 smithi018 ceph-mon[179647]: Cluster is now healthy 2023-09-23T20:41:12.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:12 smithi018 ceph-mon[179647]: failed to encode map e459 with expected crc 2023-09-23T20:41:12.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:12 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:41:12.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:12 smithi018 ceph-mon[179647]: osdmap e459: 8 total, 8 up, 8 in 2023-09-23T20:41:12.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:12 smithi018 ceph-mon[184144]: 1.17 scrub starts 2023-09-23T20:41:12.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:12 smithi018 ceph-mon[184144]: 1.17 scrub ok 2023-09-23T20:41:12.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:12 smithi018 ceph-mon[184144]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:41:12.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:12 smithi018 ceph-mon[184144]: Cluster is now healthy 2023-09-23T20:41:12.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:12 smithi018 ceph-mon[184144]: failed to encode map e459 with expected crc 2023-09-23T20:41:12.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:12 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:41:12.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:12 smithi018 ceph-mon[184144]: osdmap e459: 8 total, 8 up, 8 in 2023-09-23T20:41:12.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:12 smithi149 ceph-mon[170394]: 1.17 scrub starts 2023-09-23T20:41:12.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:12 smithi149 ceph-mon[170394]: 1.17 scrub ok 2023-09-23T20:41:12.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:12 smithi149 ceph-mon[170394]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:41:12.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:12 smithi149 ceph-mon[170394]: Cluster is now healthy 2023-09-23T20:41:12.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:12 smithi149 ceph-mon[170394]: failed to encode map e459 with expected crc 2023-09-23T20:41:12.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:12 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:41:12.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:12 smithi149 ceph-mon[170394]: osdmap e459: 8 total, 8 up, 8 in 2023-09-23T20:41:13.130 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 4 16 12216 12200 190.521 190.375 0.00706784 0.00523967 2023-09-23T20:41:13.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:13 smithi018 ceph-mon[184144]: pgmap v1128: 119 pgs: 119 active+clean; 156 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 23 MiB/s wr, 366 op/s 2023-09-23T20:41:13.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:13 smithi018 ceph-mon[179647]: pgmap v1128: 119 pgs: 119 active+clean; 156 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 23 MiB/s wr, 366 op/s 2023-09-23T20:41:13.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:13 smithi149 ceph-mon[170394]: pgmap v1128: 119 pgs: 119 active+clean; 156 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 23 MiB/s wr, 366 op/s 2023-09-23T20:41:14.132 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 5 16 15066 15050 187.997 178.125 0.00618439 0.00530983 2023-09-23T20:41:14.191 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:41:14.192 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:41:14.204 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:14.204 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:41:14.305 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:41:14.357 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:14.429 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:41:14.432 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:14.433 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:41:14.533 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:41:14.588 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:14.977 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:41:14.996 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:41:15.000 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:15.001 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:41:15.001 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:41:15.101 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:41:15.131 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 6 16 17334 17318 180.291 141.75 0.0048527 0.00553943 2023-09-23T20:41:15.150 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:15.208 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:41:15.214 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:15.215 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:41:15.316 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:41:15.374 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:41:14 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:41:14] "GET /metrics HTTP/1.1" 200 36513 "" "Prometheus/2.43.0" 2023-09-23T20:41:15.381 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:15.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:15 smithi149 ceph-mon[170394]: pgmap v1129: 119 pgs: 119 active+clean; 496 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 61 MiB/s wr, 978 op/s 2023-09-23T20:41:15.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:15 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:41:15.468 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:15 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:41:15.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:15 smithi018 ceph-mon[179647]: pgmap v1129: 119 pgs: 119 active+clean; 496 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 61 MiB/s wr, 978 op/s 2023-09-23T20:41:15.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:15 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:41:15.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:15 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:41:15.643 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:15 smithi018 ceph-mon[184144]: pgmap v1129: 119 pgs: 119 active+clean; 496 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 61 MiB/s wr, 978 op/s 2023-09-23T20:41:15.643 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:15 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:41:15.643 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:15 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:41:16.132 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 7 16 19849 19833 176.99 157.188 0.00401934 0.00564337 2023-09-23T20:41:16.161 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T20:41:16.338 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:41:16.347 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:16.347 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:41:16.448 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:41:16.523 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:16.875 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:41:16.881 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:16.881 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:41:16.982 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:41:17.043 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:17.131 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 8 16 21981 21965 171.523 133.25 0.00433539 0.00582479 2023-09-23T20:41:17.759 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:17 smithi018 ceph-mon[179647]: pgmap v1130: 119 pgs: 119 active+clean; 824 MiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 96 MiB/s wr, 1.53k op/s 2023-09-23T20:41:17.759 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:17 smithi018 ceph-mon[179647]: 1.8 scrub starts 2023-09-23T20:41:17.760 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:17 smithi018 ceph-mon[179647]: 1.8 scrub ok 2023-09-23T20:41:17.761 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:17 smithi018 ceph-mon[184144]: pgmap v1130: 119 pgs: 119 active+clean; 824 MiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 96 MiB/s wr, 1.53k op/s 2023-09-23T20:41:17.761 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:17 smithi018 ceph-mon[184144]: 1.8 scrub starts 2023-09-23T20:41:17.761 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:17 smithi018 ceph-mon[184144]: 1.8 scrub ok 2023-09-23T20:41:17.775 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:41:17.787 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:17.788 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:41:17.889 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:41:17.951 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:41:17.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:17 smithi149 ceph-mon[170394]: pgmap v1130: 119 pgs: 119 active+clean; 824 MiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 96 MiB/s wr, 1.53k op/s 2023-09-23T20:41:17.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:17 smithi149 ceph-mon[170394]: 1.8 scrub starts 2023-09-23T20:41:17.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:17 smithi149 ceph-mon[170394]: 1.8 scrub ok 2023-09-23T20:41:17.967 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:18.132 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 9 16 23991 23975 166.423 125.625 0.0102708 0.00600118 2023-09-23T20:41:18.170 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:41:18.176 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:18.177 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:41:18.278 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:41:18.356 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:18.436 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:41:18.442 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:18.443 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:41:18.502 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:41:18.544 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:41:18.578 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:18.633 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:41:18.637 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:18.637 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:41:18.738 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:41:18.801 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:19.132 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 10 16 25753 25737 160.794 110.125 0.00461797 0.00621406 2023-09-23T20:41:19.481 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:41:19.489 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:19.489 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:41:19.540 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:41:19.590 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:41:19.591 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:41:19.627 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:41:19.656 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:19.671 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:41:19.708 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:41:19.747 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:41:19.787 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:41:19.823 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:41:19.831 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:41:19.832 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:41:19.832 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:41:19.832 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:41:19.832 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:41:19.832 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:41:19.832 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:41:19.832 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:41:19.869 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:19 smithi018 ceph-mon[179647]: pgmap v1131: 119 pgs: 119 active+clean; 1.1 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 112 MiB/s wr, 1.80k op/s 2023-09-23T20:41:19.869 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:19 smithi018 ceph-mon[184144]: pgmap v1131: 119 pgs: 119 active+clean; 1.1 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 112 MiB/s wr, 1.80k op/s 2023-09-23T20:41:19.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:19 smithi149 ceph-mon[170394]: pgmap v1131: 119 pgs: 119 active+clean; 1.1 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 112 MiB/s wr, 1.80k op/s 2023-09-23T20:41:20.132 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 11 16 27862 27846 158.159 131.812 0.00799243 0.00631705 2023-09-23T20:41:20.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:20 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2029867760' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:41:20.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:20 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:41:20.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:20 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2029867760' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:41:20.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:20 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:41:20.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:20 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2029867760' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:41:20.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:20 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:41:21.132 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 12 16 29573 29557 153.89 106.938 0.00169023 0.00645496 2023-09-23T20:41:21.202 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:41:21.206 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:21.206 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:41:21.307 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:41:21.357 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:21.428 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:41:21.436 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:21.436 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:41:21.491 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:21 smithi018 ceph-mon[179647]: pgmap v1132: 119 pgs: 119 active+clean; 1.4 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 137 MiB/s wr, 2.19k op/s 2023-09-23T20:41:21.492 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:21 smithi018 ceph-mon[184144]: pgmap v1132: 119 pgs: 119 active+clean; 1.4 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 137 MiB/s wr, 2.19k op/s 2023-09-23T20:41:21.536 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:41:21.583 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:21.637 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:41:21.640 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:21 smithi149 ceph-mon[170394]: pgmap v1132: 119 pgs: 119 active+clean; 1.4 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 137 MiB/s wr, 2.19k op/s 2023-09-23T20:41:21.641 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:21.642 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:41:21.743 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:41:21.814 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:22.134 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 13 16 31409 31393 150.874 114.75 0.0157423 0.00662299 2023-09-23T20:41:22.335 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2023-09-23T20:41:22.335 INFO:teuthology.orchestra.run.smithi018.stdout: "ops": [ 2023-09-23T20:41:22.336 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:41:22.336 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.36450.0:31779 9.4 9:23cfb9fc:::benchmark_data_smithi018_381516_object31778:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e459)", 2023-09-23T20:41:22.336 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:41:22.327485+0000", 2023-09-23T20:41:22.336 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.0070628410000000003, 2023-09-23T20:41:22.336 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0071000849999999999, 2023-09-23T20:41:22.336 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:41:22.336 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:41:22.336 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:41:22.337 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.36450", 2023-09-23T20:41:22.337 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:41:22.337 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 31779 2023-09-23T20:41:22.337 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:41:22.337 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:41:22.337 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:41:22.337 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:41:22.337 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:41:22.327485+0000", 2023-09-23T20:41:22.337 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:41:22.337 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:41:22.338 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:41:22.338 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:41:22.338 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:41:22.327485+0000", 2023-09-23T20:41:22.338 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:41:22.338 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:41:22.338 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:41:22.338 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:41:22.338 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:41:22.327484+0000", 2023-09-23T20:41:22.338 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:41:22.339 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:41:22.339 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:41:22.339 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:41:22.339 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:41:22.327508+0000", 2023-09-23T20:41:22.339 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.4239999999999998e-05 2023-09-23T20:41:22.339 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:41:22.339 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:41:22.339 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:41:22.339 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:41:22.327516+0000", 2023-09-23T20:41:22.339 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.9980000000000003e-06 2023-09-23T20:41:22.340 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:41:22.340 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:41:22.340 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:41:22.340 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:41:22.327523+0000", 2023-09-23T20:41:22.340 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.3589999999999996e-06 2023-09-23T20:41:22.340 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:41:22.340 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:41:22.340 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:41:22.340 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:41:22.329225+0000", 2023-09-23T20:41:22.340 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.001701934 2023-09-23T20:41:22.341 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:41:22.341 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:41:22.341 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:41:22.341 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:41:22.329361+0000", 2023-09-23T20:41:22.341 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00013679100000000001 2023-09-23T20:41:22.341 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:41:22.341 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:41:22.341 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:41:22.341 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:41:22.329506+0000", 2023-09-23T20:41:22.341 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000144828 2023-09-23T20:41:22.341 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:41:22.342 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:41:22.342 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:41:22.342 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:41:22.332133+0000", 2023-09-23T20:41:22.342 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.002626485 2023-09-23T20:41:22.342 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:41:22.342 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:41:22.342 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:41:22.342 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:41:22.332646+0000", 2023-09-23T20:41:22.342 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00051297300000000001 2023-09-23T20:41:22.343 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:41:22.343 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:41:22.343 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:41:22.343 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:41:22.343 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:41:22.343 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_repop(client.36450.0:31786 9.e e459/455 9:717aa9b3:::benchmark_data_smithi018_381516_object31785:head v 459'1978, mlcod=459'1976)", 2023-09-23T20:41:22.343 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:41:22.334261+0000", 2023-09-23T20:41:22.343 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.00028605699999999999, 2023-09-23T20:41:22.343 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00041149999999999997, 2023-09-23T20:41:22.343 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:41:22.344 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "started", 2023-09-23T20:41:22.344 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:41:22.344 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:41:22.344 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:41:22.344 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:41:22.334261+0000", 2023-09-23T20:41:22.344 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:41:22.344 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:41:22.344 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:41:22.344 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:41:22.344 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:41:22.334261+0000", 2023-09-23T20:41:22.345 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:41:22.345 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:41:22.345 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:41:22.345 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:41:22.345 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:41:22.334261+0000", 2023-09-23T20:41:22.345 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:41:22.345 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:41:22.345 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:41:22.345 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:41:22.346 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:41:22.334331+0000", 2023-09-23T20:41:22.346 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.9886000000000001e-05 2023-09-23T20:41:22.346 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:41:22.346 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:41:22.346 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:41:22.346 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:41:22.334346+0000", 2023-09-23T20:41:22.346 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.4904e-05 2023-09-23T20:41:22.346 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:41:22.346 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:41:22.346 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:41:22.347 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:41:22.334352+0000", 2023-09-23T20:41:22.347 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.8349999999999998e-06 2023-09-23T20:41:22.347 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:41:22.347 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:41:22.347 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:41:22.347 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:41:22.334387+0000", 2023-09-23T20:41:22.347 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.4254000000000001e-05 2023-09-23T20:41:22.347 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:41:22.347 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:41:22.347 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:41:22.348 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:41:22.334408+0000", 2023-09-23T20:41:22.348 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.1090999999999998e-05 2023-09-23T20:41:22.348 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:41:22.348 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:41:22.348 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:41:22.348 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:41:22.348 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:41:22.348 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_repop(client.36450.0:31788 9.3 e459/455 9:c4c0bd0a:::benchmark_data_smithi018_381516_object31787:head v 459'1977, mlcod=459'1975)", 2023-09-23T20:41:22.348 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:41:22.334365+0000", 2023-09-23T20:41:22.348 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.00018211, 2023-09-23T20:41:22.349 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000375767, 2023-09-23T20:41:22.349 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:41:22.349 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "started", 2023-09-23T20:41:22.349 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:41:22.349 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:41:22.349 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:41:22.349 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:41:22.334365+0000", 2023-09-23T20:41:22.349 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:41:22.349 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:41:22.349 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:41:22.349 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:41:22.350 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:41:22.334365+0000", 2023-09-23T20:41:22.350 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:41:22.350 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:41:22.350 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:41:22.350 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:41:22.350 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:41:22.334365+0000", 2023-09-23T20:41:22.350 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:41:22.350 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:41:22.350 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:41:22.351 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:41:22.351 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:41:22.334397+0000", 2023-09-23T20:41:22.351 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.2026000000000001e-05 2023-09-23T20:41:22.351 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:41:22.351 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:41:22.351 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:41:22.351 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:41:22.334404+0000", 2023-09-23T20:41:22.351 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.1189999999999999e-06 2023-09-23T20:41:22.351 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:41:22.351 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:41:22.351 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:41:22.352 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:41:22.334409+0000", 2023-09-23T20:41:22.352 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.9790000000000001e-06 2023-09-23T20:41:22.352 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:41:22.352 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:41:22.352 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:41:22.352 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:41:22.334518+0000", 2023-09-23T20:41:22.352 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000108554 2023-09-23T20:41:22.352 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:41:22.352 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:41:22.352 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:41:22.353 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:41:22.334538+0000", 2023-09-23T20:41:22.353 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.048e-05 2023-09-23T20:41:22.353 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:41:22.353 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:41:22.353 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:41:22.353 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:41:22.353 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:41:22.353 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_repop(client.36450.0:31790 9.d e459/455)", 2023-09-23T20:41:22.353 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:41:22.334423+0000", 2023-09-23T20:41:22.353 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.000124929, 2023-09-23T20:41:22.354 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00038585300000000002, 2023-09-23T20:41:22.354 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:41:22.354 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "queued for pg", 2023-09-23T20:41:22.354 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:41:22.354 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:41:22.354 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:41:22.354 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:41:22.334423+0000", 2023-09-23T20:41:22.354 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:41:22.354 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:41:22.354 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:41:22.355 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:41:22.355 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:41:22.334423+0000", 2023-09-23T20:41:22.355 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:41:22.355 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:41:22.355 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:41:22.355 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:41:22.355 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:41:22.334422+0000", 2023-09-23T20:41:22.355 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:41:22.355 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:41:22.356 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:41:22.356 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:41:22.356 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:41:22.334454+0000", 2023-09-23T20:41:22.356 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.1288000000000003e-05 2023-09-23T20:41:22.356 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:41:22.356 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:41:22.356 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:41:22.356 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:41:22.334460+0000", 2023-09-23T20:41:22.356 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.7070000000000003e-06 2023-09-23T20:41:22.357 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:41:22.357 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:41:22.357 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:41:22.357 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:41:22.334465+0000", 2023-09-23T20:41:22.357 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.4610000000000001e-06 2023-09-23T20:41:22.357 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:41:22.357 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:41:22.357 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:41:22.357 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:41:22.357 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:41:22.358 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.36450.0:31774 9.4 9:26b54179:::benchmark_data_smithi018_381516_object31773:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e459)", 2023-09-23T20:41:22.358 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:41:22.326547+0000", 2023-09-23T20:41:22.358 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.0080000169999999999, 2023-09-23T20:41:22.358 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0083117069999999998, 2023-09-23T20:41:22.358 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:41:22.358 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:41:22.358 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:41:22.358 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.36450", 2023-09-23T20:41:22.358 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:41:22.359 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 31774 2023-09-23T20:41:22.359 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:41:22.359 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:41:22.359 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:41:22.359 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:41:22.359 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:41:22.326547+0000", 2023-09-23T20:41:22.359 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:41:22.359 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:41:22.359 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:41:22.359 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:41:22.360 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:41:22.326547+0000", 2023-09-23T20:41:22.360 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:41:22.360 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:41:22.360 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:41:22.360 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:41:22.360 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:41:22.326546+0000", 2023-09-23T20:41:22.360 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:41:22.360 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:41:22.360 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:41:22.360 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:41:22.361 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:41:22.326571+0000", 2023-09-23T20:41:22.361 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.4797e-05 2023-09-23T20:41:22.361 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:41:22.361 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:41:22.361 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:41:22.361 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:41:22.326582+0000", 2023-09-23T20:41:22.361 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1253999999999999e-05 2023-09-23T20:41:22.361 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:41:22.361 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:41:22.362 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:41:22.362 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:41:22.326590+0000", 2023-09-23T20:41:22.362 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.0860000000000002e-06 2023-09-23T20:41:22.362 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:41:22.362 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:41:22.362 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:41:22.362 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:41:22.327637+0000", 2023-09-23T20:41:22.362 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0010468949999999999 2023-09-23T20:41:22.362 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:41:22.362 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:41:22.363 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:41:22.363 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:41:22.327780+0000", 2023-09-23T20:41:22.363 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00014339299999999999 2023-09-23T20:41:22.363 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:41:22.363 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:41:22.363 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:41:22.363 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:41:22.327942+0000", 2023-09-23T20:41:22.363 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00016207299999999999 2023-09-23T20:41:22.363 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:41:22.363 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:41:22.364 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:41:22.364 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:41:22.330836+0000", 2023-09-23T20:41:22.364 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0028938039999999998 2023-09-23T20:41:22.364 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:41:22.364 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:41:22.364 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:41:22.364 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:41:22.331468+0000", 2023-09-23T20:41:22.364 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00063206099999999995 2023-09-23T20:41:22.364 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:41:22.364 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:41:22.365 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:41:22.365 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:41:22.365 INFO:teuthology.orchestra.run.smithi018.stdout: ], 2023-09-23T20:41:22.365 INFO:teuthology.orchestra.run.smithi018.stdout: "num_ops": 5 2023-09-23T20:41:22.365 INFO:teuthology.orchestra.run.smithi018.stdout:} 2023-09-23T20:41:22.366 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:22.366 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:41:22.366 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:41:22.467 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:41:22.564 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:22.888 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:41:22.897 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:22.897 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:41:22.993 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T20:41:22.999 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:41:23.043 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:23.094 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:41:23.098 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:23.098 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:41:23.133 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 14 16 33181 33165 148.007 110.75 0.00758127 0.00675055 2023-09-23T20:41:23.199 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:41:23.286 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:23.457 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:41:23.466 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:23.466 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:41:23.505 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:23 smithi018 ceph-mon[179647]: pgmap v1133: 119 pgs: 119 active+clean; 1.6 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 143 MiB/s wr, 2.29k op/s 2023-09-23T20:41:23.506 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:23 smithi018 ceph-mon[179647]: 1.10 scrub starts 2023-09-23T20:41:23.506 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:23 smithi018 ceph-mon[179647]: 1.10 scrub ok 2023-09-23T20:41:23.567 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:41:23.572 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:41:23.597 INFO:teuthology.orchestra.run.smithi018.stderr:dumped all 2023-09-23T20:41:23.608 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:23.660 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:41:23.665 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:23.666 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:41:23.766 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:41:23.819 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:23 smithi018 ceph-mon[184144]: pgmap v1133: 119 pgs: 119 active+clean; 1.6 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 143 MiB/s wr, 2.29k op/s 2023-09-23T20:41:23.820 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:23 smithi018 ceph-mon[184144]: 1.10 scrub starts 2023-09-23T20:41:23.820 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:23 smithi018 ceph-mon[184144]: 1.10 scrub ok 2023-09-23T20:41:23.828 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:23.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:23 smithi149 ceph-mon[170394]: pgmap v1133: 119 pgs: 119 active+clean; 1.6 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 143 MiB/s wr, 2.29k op/s 2023-09-23T20:41:23.968 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:23 smithi149 ceph-mon[170394]: 1.10 scrub starts 2023-09-23T20:41:23.968 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:23 smithi149 ceph-mon[170394]: 1.10 scrub ok 2023-09-23T20:41:24.076 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:41:24.082 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:24.083 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:41:24.133 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 15 16 34806 34790 144.91 101.562 0.00832449 0.00689366 2023-09-23T20:41:24.154 INFO:teuthology.orchestra.run.smithi018.stdout:{"pg_ready":true,"pg_map":{"version":1133,"stamp":"2023-09-23T20:41:22.100913+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1746528057,"num_objects":26607,"num_object_clones":0,"num_object_copies":79821,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26607,"num_whiteouts":0,"num_read":6027,"num_read_kb":13283,"num_write":27509,"num_write_kb":1714065,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":149,"num_bytes_recovered":8830792,"num_keys_recovered":40,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":32602,"ondisk_log_size":32602,"up":357,"acting":357,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":363,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":9787780,"kb_used_data":5627844,"kb_used_omap":32,"kb_used_meta":4159775,"kb_avail":740173436,"statfs":{"total":767960285184,"available":757937598464,"internally_reserved":0,"allocated":5762912256,"data_stored":5747696608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32773,"internal_metadata":4259610619},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":1582739034,"num_objects":24150,"num_object_clones":0,"num_object_copies":72450,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24150,"num_whiteouts":0,"num_read":-44,"num_read_kb":-82,"num_write":24127,"num_write_kb":1545317,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-1,"num_bytes_recovered":-488688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.522538"},"pg_stats":[{"pgid":"1.18","version":"90'161","reported_seq":442,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:09.135550+0000","last_change":"2023-09-23T20:40:56.483418+0000","last_active":"2023-09-23T20:41:09.135550+0000","last_peered":"2023-09-23T20:41:09.135550+0000","last_clean":"2023-09-23T20:41:09.135550+0000","last_became_active":"2023-09-23T20:40:56.482886+0000","last_became_peered":"2023-09-23T20:40:56.482886+0000","last_unstale":"2023-09-23T20:41:09.135550+0000","last_undegraded":"2023-09-23T20:41:09.135550+0000","last_fullsized":"2023-09-23T20:41:09.135550+0000","mapping_epoch":453,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":454,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:35:03.551143+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:35:03.551143+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":343193,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"69'1","reported_seq":156,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:08.290015+0000","last_change":"2023-09-23T20:32:33.233853+0000","last_active":"2023-09-23T20:41:08.290015+0000","last_peered":"2023-09-23T20:41:08.290015+0000","last_clean":"2023-09-23T20:41:08.290015+0000","last_became_active":"2023-09-23T20:32:33.233546+0000","last_became_peered":"2023-09-23T20:32:33.233546+0000","last_unstale":"2023-09-23T20:41:08.290015+0000","last_undegraded":"2023-09-23T20:41:08.290015+0000","last_fullsized":"2023-09-23T20:41:08.290015+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:50:27.130808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":272,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:08.520708+0000","last_change":"2023-09-23T20:37:43.579611+0000","last_active":"2023-09-23T20:41:08.520708+0000","last_peered":"2023-09-23T20:41:08.520708+0000","last_clean":"2023-09-23T20:41:08.520708+0000","last_became_active":"2023-09-23T20:37:43.579323+0000","last_became_peered":"2023-09-23T20:37:43.579323+0000","last_unstale":"2023-09-23T20:41:08.520708+0000","last_undegraded":"2023-09-23T20:41:08.520708+0000","last_fullsized":"2023-09-23T20:41:08.520708+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:17:32.311427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"90'161","reported_seq":532,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:09.135862+0000","last_change":"2023-09-23T20:40:22.445059+0000","last_active":"2023-09-23T20:41:09.135862+0000","last_peered":"2023-09-23T20:41:09.135862+0000","last_clean":"2023-09-23T20:41:09.135862+0000","last_became_active":"2023-09-23T20:40:22.444637+0000","last_became_peered":"2023-09-23T20:40:22.444637+0000","last_unstale":"2023-09-23T20:41:09.135862+0000","last_undegraded":"2023-09-23T20:41:09.135862+0000","last_fullsized":"2023-09-23T20:41:09.135862+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:05.442288+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:36:05.442288+0000","last_clean_scrub_stamp":"2023-09-23T20:36:05.442288+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0019500940000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":427,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:09.132135+0000","last_change":"2023-09-23T20:34:59.983035+0000","last_active":"2023-09-23T20:41:09.132135+0000","last_peered":"2023-09-23T20:41:09.132135+0000","last_clean":"2023-09-23T20:41:09.132135+0000","last_became_active":"2023-09-23T20:34:59.982459+0000","last_became_peered":"2023-09-23T20:34:59.982459+0000","last_unstale":"2023-09-23T20:41:09.132135+0000","last_undegraded":"2023-09-23T20:41:09.132135+0000","last_fullsized":"2023-09-23T20:41:09.132135+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:16:48.510783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":429,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:07.553949+0000","last_change":"2023-09-23T20:25:02.957185+0000","last_active":"2023-09-23T20:41:07.553949+0000","last_peered":"2023-09-23T20:41:07.553949+0000","last_clean":"2023-09-23T20:41:07.553949+0000","last_became_active":"2023-09-23T20:25:02.956689+0000","last_became_peered":"2023-09-23T20:25:02.956689+0000","last_unstale":"2023-09-23T20:41:07.553949+0000","last_undegraded":"2023-09-23T20:41:07.553949+0000","last_fullsized":"2023-09-23T20:41:07.553949+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:04:28.524015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"90'161","reported_seq":538,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:08.289809+0000","last_change":"2023-09-23T20:36:01.292103+0000","last_active":"2023-09-23T20:41:08.289809+0000","last_peered":"2023-09-23T20:41:08.289809+0000","last_clean":"2023-09-23T20:41:08.289809+0000","last_became_active":"2023-09-23T20:35:26.639912+0000","last_became_peered":"2023-09-23T20:35:26.639912+0000","last_unstale":"2023-09-23T20:41:08.289809+0000","last_undegraded":"2023-09-23T20:41:08.289809+0000","last_fullsized":"2023-09-23T20:41:08.289809+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:01.292032+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:01.292032+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:04:53.926640+0000","scrub_duration":0.0032983159999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":252,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:08.520302+0000","last_change":"2023-09-23T20:37:43.584414+0000","last_active":"2023-09-23T20:41:08.520302+0000","last_peered":"2023-09-23T20:41:08.520302+0000","last_clean":"2023-09-23T20:41:08.520302+0000","last_became_active":"2023-09-23T20:37:43.584116+0000","last_became_peered":"2023-09-23T20:37:43.584116+0000","last_unstale":"2023-09-23T20:41:08.520302+0000","last_undegraded":"2023-09-23T20:41:08.520302+0000","last_fullsized":"2023-09-23T20:41:08.520302+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:17:54.258172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":74,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:08.381235+0000","last_change":"2023-09-23T20:38:37.457427+0000","last_active":"2023-09-23T20:41:08.381235+0000","last_peered":"2023-09-23T20:41:08.381235+0000","last_clean":"2023-09-23T20:41:08.381235+0000","last_became_active":"2023-09-23T20:38:37.457099+0000","last_became_peered":"2023-09-23T20:38:37.457099+0000","last_unstale":"2023-09-23T20:41:08.381235+0000","last_undegraded":"2023-09-23T20:41:08.381235+0000","last_fullsized":"2023-09-23T20:41:08.381235+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:31:27.084994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"90'161","reported_seq":671,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:08.769306+0000","last_change":"2023-09-23T20:40:22.449754+0000","last_active":"2023-09-23T20:41:08.769306+0000","last_peered":"2023-09-23T20:41:08.769306+0000","last_clean":"2023-09-23T20:41:08.769306+0000","last_became_active":"2023-09-23T20:40:22.448838+0000","last_became_peered":"2023-09-23T20:40:22.448838+0000","last_unstale":"2023-09-23T20:41:08.769306+0000","last_undegraded":"2023-09-23T20:41:08.769306+0000","last_fullsized":"2023-09-23T20:41:08.769306+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:03.411025+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:03.411025+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:22:07.948856+0000","scrub_duration":0.001802922,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":462,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:07.554042+0000","last_change":"2023-09-23T20:40:19.538844+0000","last_active":"2023-09-23T20:41:07.554042+0000","last_peered":"2023-09-23T20:41:07.554042+0000","last_clean":"2023-09-23T20:41:07.554042+0000","last_became_active":"2023-09-23T20:40:19.538434+0000","last_became_peered":"2023-09-23T20:40:19.538434+0000","last_unstale":"2023-09-23T20:41:07.554042+0000","last_undegraded":"2023-09-23T20:41:07.554042+0000","last_fullsized":"2023-09-23T20:41:07.554042+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:53:59.418427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":182,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:08.619423+0000","last_change":"2023-09-23T20:38:02.579343+0000","last_active":"2023-09-23T20:41:08.619423+0000","last_peered":"2023-09-23T20:41:08.619423+0000","last_clean":"2023-09-23T20:41:08.619423+0000","last_became_active":"2023-09-23T20:38:02.579047+0000","last_became_peered":"2023-09-23T20:38:02.579047+0000","last_unstale":"2023-09-23T20:41:08.619423+0000","last_undegraded":"2023-09-23T20:41:08.619423+0000","last_fullsized":"2023-09-23T20:41:08.619423+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:44:03.166556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.14","version":"90'161","reported_seq":788,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:08.619762+0000","last_change":"2023-09-23T20:35:02.236398+0000","last_active":"2023-09-23T20:41:08.619762+0000","last_peered":"2023-09-23T20:41:08.619762+0000","last_clean":"2023-09-23T20:41:08.619762+0000","last_became_active":"2023-09-23T20:35:02.235988+0000","last_became_peered":"2023-09-23T20:35:02.235988+0000","last_unstale":"2023-09-23T20:41:08.619762+0000","last_undegraded":"2023-09-23T20:41:08.619762+0000","last_fullsized":"2023-09-23T20:41:08.619762+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:47.479012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:47.479012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:13.519648+0000","scrub_duration":0.0020136590000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":28,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:09.136481+0000","last_change":"2023-09-23T20:40:19.545472+0000","last_active":"2023-09-23T20:41:09.136481+0000","last_peered":"2023-09-23T20:41:09.136481+0000","last_clean":"2023-09-23T20:41:09.136481+0000","last_became_active":"2023-09-23T20:40:19.543853+0000","last_became_peered":"2023-09-23T20:40:19.543853+0000","last_unstale":"2023-09-23T20:41:09.136481+0000","last_undegraded":"2023-09-23T20:41:09.136481+0000","last_fullsized":"2023-09-23T20:41:09.136481+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:08.014744+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":283,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:08.520523+0000","last_change":"2023-09-23T20:38:02.542034+0000","last_active":"2023-09-23T20:41:08.520523+0000","last_peered":"2023-09-23T20:41:08.520523+0000","last_clean":"2023-09-23T20:41:08.520523+0000","last_became_active":"2023-09-23T20:38:02.541575+0000","last_became_peered":"2023-09-23T20:38:02.541575+0000","last_unstale":"2023-09-23T20:41:08.520523+0000","last_undegraded":"2023-09-23T20:41:08.520523+0000","last_fullsized":"2023-09-23T20:41:08.520523+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:50:54.591529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"90'161","reported_seq":622,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:11.210079+0000","last_change":"2023-09-23T20:41:11.210009+0000","last_active":"2023-09-23T20:41:11.210079+0000","last_peered":"2023-09-23T20:41:11.210079+0000","last_clean":"2023-09-23T20:41:11.210079+0000","last_became_active":"2023-09-23T20:40:22.444543+0000","last_became_peered":"2023-09-23T20:40:22.444543+0000","last_unstale":"2023-09-23T20:41:11.210079+0000","last_undegraded":"2023-09-23T20:41:11.210079+0000","last_fullsized":"2023-09-23T20:41:11.210079+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:41:11.209916+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:41:11.209916+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:30:05.067685+0000","scrub_duration":0.0025146090000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":486,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:08.769513+0000","last_change":"2023-09-23T20:32:33.270617+0000","last_active":"2023-09-23T20:41:08.769513+0000","last_peered":"2023-09-23T20:41:08.769513+0000","last_clean":"2023-09-23T20:41:08.769513+0000","last_became_active":"2023-09-23T20:32:33.227902+0000","last_became_peered":"2023-09-23T20:32:33.227902+0000","last_unstale":"2023-09-23T20:41:08.769513+0000","last_undegraded":"2023-09-23T20:41:08.769513+0000","last_fullsized":"2023-09-23T20:41:08.769513+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:30:26.847831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":360,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":428,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:09.132505+0000","last_change":"2023-09-23T20:38:02.599068+0000","last_active":"2023-09-23T20:41:09.132505+0000","last_peered":"2023-09-23T20:41:09.132505+0000","last_clean":"2023-09-23T20:41:09.132505+0000","last_became_active":"2023-09-23T20:38:02.597602+0000","last_became_peered":"2023-09-23T20:38:02.597602+0000","last_unstale":"2023-09-23T20:41:09.132505+0000","last_undegraded":"2023-09-23T20:41:09.132505+0000","last_fullsized":"2023-09-23T20:41:09.132505+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:34:06.337029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"90'161","reported_seq":583,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:09.132772+0000","last_change":"2023-09-23T20:36:02.548387+0000","last_active":"2023-09-23T20:41:09.132772+0000","last_peered":"2023-09-23T20:41:09.132772+0000","last_clean":"2023-09-23T20:41:09.132772+0000","last_became_active":"2023-09-23T20:35:26.644787+0000","last_became_peered":"2023-09-23T20:35:26.644787+0000","last_unstale":"2023-09-23T20:41:09.132772+0000","last_undegraded":"2023-09-23T20:41:09.132772+0000","last_fullsized":"2023-09-23T20:41:09.132772+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:02.548285+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:02.548285+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:37:15.216553+0000","scrub_duration":0.0015514459999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":206,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:08.290232+0000","last_change":"2023-09-23T20:38:02.560646+0000","last_active":"2023-09-23T20:41:08.290232+0000","last_peered":"2023-09-23T20:41:08.290232+0000","last_clean":"2023-09-23T20:41:08.290232+0000","last_became_active":"2023-09-23T20:38:02.539814+0000","last_became_peered":"2023-09-23T20:38:02.539814+0000","last_unstale":"2023-09-23T20:41:08.290232+0000","last_undegraded":"2023-09-23T20:41:08.290232+0000","last_fullsized":"2023-09-23T20:41:08.290232+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:00:09.435023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":186,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:08.290455+0000","last_change":"2023-09-23T20:38:02.579222+0000","last_active":"2023-09-23T20:41:08.290455+0000","last_peered":"2023-09-23T20:41:08.290455+0000","last_clean":"2023-09-23T20:41:08.290455+0000","last_became_active":"2023-09-23T20:38:02.578813+0000","last_became_peered":"2023-09-23T20:38:02.578813+0000","last_unstale":"2023-09-23T20:41:08.290455+0000","last_undegraded":"2023-09-23T20:41:08.290455+0000","last_fullsized":"2023-09-23T20:41:08.290455+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:57:31.080027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"90'161","reported_seq":725,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:07.553128+0000","last_change":"2023-09-23T20:40:22.427437+0000","last_active":"2023-09-23T20:41:07.553128+0000","last_peered":"2023-09-23T20:41:07.553128+0000","last_clean":"2023-09-23T20:41:07.553128+0000","last_became_active":"2023-09-23T20:40:22.427090+0000","last_became_peered":"2023-09-23T20:40:22.427090+0000","last_unstale":"2023-09-23T20:41:07.553128+0000","last_undegraded":"2023-09-23T20:41:07.553128+0000","last_fullsized":"2023-09-23T20:41:07.553128+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:43.578666+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:43.578666+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:08:43.995672+0000","scrub_duration":0.0021890379999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":485,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:08.769039+0000","last_change":"2023-09-23T20:32:33.242943+0000","last_active":"2023-09-23T20:41:08.769039+0000","last_peered":"2023-09-23T20:41:08.769039+0000","last_clean":"2023-09-23T20:41:08.769039+0000","last_became_active":"2023-09-23T20:32:33.225468+0000","last_became_peered":"2023-09-23T20:32:33.225468+0000","last_unstale":"2023-09-23T20:41:08.769039+0000","last_undegraded":"2023-09-23T20:41:08.769039+0000","last_fullsized":"2023-09-23T20:41:08.769039+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:40:47.867573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":194,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:08.520407+0000","last_change":"2023-09-23T20:37:43.571836+0000","last_active":"2023-09-23T20:41:08.520407+0000","last_peered":"2023-09-23T20:41:08.520407+0000","last_clean":"2023-09-23T20:41:08.520407+0000","last_became_active":"2023-09-23T20:37:43.571511+0000","last_became_peered":"2023-09-23T20:37:43.571511+0000","last_unstale":"2023-09-23T20:41:08.520407+0000","last_undegraded":"2023-09-23T20:41:08.520407+0000","last_fullsized":"2023-09-23T20:41:08.520407+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:59:25.507917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"90'161","reported_seq":641,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:09.135788+0000","last_change":"2023-09-23T20:40:22.427706+0000","last_active":"2023-09-23T20:41:09.135788+0000","last_peered":"2023-09-23T20:41:09.135788+0000","last_clean":"2023-09-23T20:41:09.135788+0000","last_became_active":"2023-09-23T20:40:22.427114+0000","last_became_peered":"2023-09-23T20:40:22.427114+0000","last_unstale":"2023-09-23T20:41:09.135788+0000","last_undegraded":"2023-09-23T20:41:09.135788+0000","last_fullsized":"2023-09-23T20:41:09.135788+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:49.726139+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:49.726139+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0096333110000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":427,"reported_epoch":459,"state":"active+clean","last_fresh":"2023-09-23T20:41:11.599626+0000","last_change":"2023-09-23T20:25:02.953993+0000","last_active":"2023-09-23T20:41:11.599626+0000","last_peered":"2023-09-23T20:41:11.599626+0000","last_clean":"2023-09-23T20:41:11.599626+0000","last_became_active":"2023-09-23T20:25:02.953672+0000","last_became_peered":"2023-09-23T20:25:02.953672+0000","last_unstale":"2023-09-23T20:41:11.599626+0000","last_undegraded":"2023-09-23T20:41:11.599626+0000","last_fullsized":"2023-09-23T20:41:11.599626+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:26:08.603577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":28,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:09.136428+0000","last_change":"2023-09-23T20:40:19.561512+0000","last_active":"2023-09-23T20:41:09.136428+0000","last_peered":"2023-09-23T20:41:09.136428+0000","last_clean":"2023-09-23T20:41:09.136428+0000","last_became_active":"2023-09-23T20:40:19.561123+0000","last_became_peered":"2023-09-23T20:40:19.561123+0000","last_unstale":"2023-09-23T20:41:09.136428+0000","last_undegraded":"2023-09-23T20:41:09.136428+0000","last_fullsized":"2023-09-23T20:41:09.136428+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:50:02.093978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"90'161","reported_seq":744,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:09.132686+0000","last_change":"2023-09-23T20:35:02.238923+0000","last_active":"2023-09-23T20:41:09.132686+0000","last_peered":"2023-09-23T20:41:09.132686+0000","last_clean":"2023-09-23T20:41:09.132686+0000","last_became_active":"2023-09-23T20:35:02.238293+0000","last_became_peered":"2023-09-23T20:35:02.238293+0000","last_unstale":"2023-09-23T20:41:09.132686+0000","last_undegraded":"2023-09-23T20:41:09.132686+0000","last_fullsized":"2023-09-23T20:41:09.132686+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:42.544308+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:42.544308+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:51:22.281932+0000","scrub_duration":0.0029958179999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":462,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:08.768930+0000","last_change":"2023-09-23T20:38:02.539774+0000","last_active":"2023-09-23T20:41:08.768930+0000","last_peered":"2023-09-23T20:41:08.768930+0000","last_clean":"2023-09-23T20:41:08.768930+0000","last_became_active":"2023-09-23T20:38:02.539436+0000","last_became_peered":"2023-09-23T20:38:02.539436+0000","last_unstale":"2023-09-23T20:41:08.768930+0000","last_undegraded":"2023-09-23T20:41:08.768930+0000","last_fullsized":"2023-09-23T20:41:08.768930+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:50:57.375803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":436,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:07.554026+0000","last_change":"2023-09-23T20:25:02.957010+0000","last_active":"2023-09-23T20:41:07.554026+0000","last_peered":"2023-09-23T20:41:07.554026+0000","last_clean":"2023-09-23T20:41:07.554026+0000","last_became_active":"2023-09-23T20:25:02.956267+0000","last_became_peered":"2023-09-23T20:25:02.956267+0000","last_unstale":"2023-09-23T20:41:07.554026+0000","last_undegraded":"2023-09-23T20:41:07.554026+0000","last_fullsized":"2023-09-23T20:41:07.554026+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:10:55.299422+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"90'161","reported_seq":574,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:09.136215+0000","last_change":"2023-09-23T20:41:08.258019+0000","last_active":"2023-09-23T20:41:09.136215+0000","last_peered":"2023-09-23T20:41:09.136215+0000","last_clean":"2023-09-23T20:41:09.136215+0000","last_became_active":"2023-09-23T20:40:22.430717+0000","last_became_peered":"2023-09-23T20:40:22.430717+0000","last_unstale":"2023-09-23T20:41:09.136215+0000","last_undegraded":"2023-09-23T20:41:09.136215+0000","last_fullsized":"2023-09-23T20:41:09.136215+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:41:08.257952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:41:08.257952+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:47:51.982877+0000","scrub_duration":0.00158735,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":94,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:08.380892+0000","last_change":"2023-09-23T20:40:19.564378+0000","last_active":"2023-09-23T20:41:08.380892+0000","last_peered":"2023-09-23T20:41:08.380892+0000","last_clean":"2023-09-23T20:41:08.380892+0000","last_became_active":"2023-09-23T20:40:19.532991+0000","last_became_peered":"2023-09-23T20:40:19.532991+0000","last_unstale":"2023-09-23T20:41:08.380892+0000","last_undegraded":"2023-09-23T20:41:08.380892+0000","last_fullsized":"2023-09-23T20:41:08.380892+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:22:42.832115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":252,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:08.521100+0000","last_change":"2023-09-23T20:37:43.571083+0000","last_active":"2023-09-23T20:41:08.521100+0000","last_peered":"2023-09-23T20:41:08.521100+0000","last_clean":"2023-09-23T20:41:08.521100+0000","last_became_active":"2023-09-23T20:37:43.570306+0000","last_became_peered":"2023-09-23T20:37:43.570306+0000","last_unstale":"2023-09-23T20:41:08.521100+0000","last_undegraded":"2023-09-23T20:41:08.521100+0000","last_fullsized":"2023-09-23T20:41:08.521100+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:42:02.396718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.5","version":"459'1811","reported_seq":1824,"reported_epoch":459,"state":"active+clean","last_fresh":"2023-09-23T20:41:21.259952+0000","last_change":"2023-09-23T20:41:04.556343+0000","last_active":"2023-09-23T20:41:21.259952+0000","last_peered":"2023-09-23T20:41:21.259952+0000","last_clean":"2023-09-23T20:41:21.259952+0000","last_became_active":"2023-09-23T20:41:04.556043+0000","last_became_peered":"2023-09-23T20:41:04.556043+0000","last_unstale":"2023-09-23T20:41:21.259952+0000","last_undegraded":"2023-09-23T20:41:21.259952+0000","last_fullsized":"2023-09-23T20:41:21.259952+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":455,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":1811,"log_dups_size":0,"ondisk_log_size":1811,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:33:41.345407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":118685696,"num_objects":1811,"num_object_clones":0,"num_object_copies":5433,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1811,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1811,"num_write_kb":115904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.d","version":"90'161","reported_seq":751,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:08.769410+0000","last_change":"2023-09-23T20:34:59.496438+0000","last_active":"2023-09-23T20:41:08.769410+0000","last_peered":"2023-09-23T20:41:08.769410+0000","last_clean":"2023-09-23T20:41:08.769410+0000","last_became_active":"2023-09-23T20:33:38.970527+0000","last_became_peered":"2023-09-23T20:33:38.970527+0000","last_unstale":"2023-09-23T20:41:08.769410+0000","last_undegraded":"2023-09-23T20:41:08.769410+0000","last_fullsized":"2023-09-23T20:41:08.769410+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:34:59.496375+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:34:59.496375+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:46:18.244734+0000","scrub_duration":0.0017843189999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":462,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:08.769526+0000","last_change":"2023-09-23T20:32:33.283752+0000","last_active":"2023-09-23T20:41:08.769526+0000","last_peered":"2023-09-23T20:41:08.769526+0000","last_clean":"2023-09-23T20:41:08.769526+0000","last_became_active":"2023-09-23T20:32:33.230503+0000","last_became_peered":"2023-09-23T20:32:33.230503+0000","last_unstale":"2023-09-23T20:41:08.769526+0000","last_undegraded":"2023-09-23T20:41:08.769526+0000","last_fullsized":"2023-09-23T20:41:08.769526+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:29.084410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":28,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:09.135687+0000","last_change":"2023-09-23T20:40:19.545668+0000","last_active":"2023-09-23T20:41:09.135687+0000","last_peered":"2023-09-23T20:41:09.135687+0000","last_clean":"2023-09-23T20:41:09.135687+0000","last_became_active":"2023-09-23T20:40:19.543857+0000","last_became_peered":"2023-09-23T20:40:19.543857+0000","last_unstale":"2023-09-23T20:41:09.135687+0000","last_undegraded":"2023-09-23T20:41:09.135687+0000","last_fullsized":"2023-09-23T20:41:09.135687+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:27:09.470605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"459'1629","reported_seq":1641,"reported_epoch":459,"state":"active+clean","last_fresh":"2023-09-23T20:41:19.552843+0000","last_change":"2023-09-23T20:41:04.575200+0000","last_active":"2023-09-23T20:41:19.552843+0000","last_peered":"2023-09-23T20:41:19.552843+0000","last_clean":"2023-09-23T20:41:19.552843+0000","last_became_active":"2023-09-23T20:41:04.574824+0000","last_became_peered":"2023-09-23T20:41:04.574824+0000","last_unstale":"2023-09-23T20:41:19.552843+0000","last_undegraded":"2023-09-23T20:41:19.552843+0000","last_fullsized":"2023-09-23T20:41:19.552843+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":455,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":1629,"log_dups_size":0,"ondisk_log_size":1629,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:50:46.714383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":106758144,"num_objects":1629,"num_object_clones":0,"num_object_copies":4887,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1629,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1629,"num_write_kb":104256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.c","version":"90'161","reported_seq":718,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:09.506481+0000","last_change":"2023-09-23T20:41:09.506481+0000","last_active":"2023-09-23T20:41:09.506481+0000","last_peered":"2023-09-23T20:41:09.506481+0000","last_clean":"2023-09-23T20:41:09.506481+0000","last_became_active":"2023-09-23T20:38:28.442229+0000","last_became_peered":"2023-09-23T20:38:28.442229+0000","last_unstale":"2023-09-23T20:41:09.506481+0000","last_undegraded":"2023-09-23T20:41:09.506481+0000","last_fullsized":"2023-09-23T20:41:09.506481+0000","mapping_epoch":411,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":412,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:41:09.506338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:41:09.506338+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:19:46.265011+0000","scrub_duration":0.01112002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":497,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:08.769693+0000","last_change":"2023-09-23T20:34:59.987981+0000","last_active":"2023-09-23T20:41:08.769693+0000","last_peered":"2023-09-23T20:41:08.769693+0000","last_clean":"2023-09-23T20:41:08.769693+0000","last_became_active":"2023-09-23T20:34:59.985368+0000","last_became_peered":"2023-09-23T20:34:59.985368+0000","last_unstale":"2023-09-23T20:41:08.769693+0000","last_undegraded":"2023-09-23T20:41:08.769693+0000","last_fullsized":"2023-09-23T20:41:08.769693+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:12:43.082493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":156,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:08.289913+0000","last_change":"2023-09-23T20:32:33.232142+0000","last_active":"2023-09-23T20:41:08.289913+0000","last_peered":"2023-09-23T20:41:08.289913+0000","last_clean":"2023-09-23T20:41:08.289913+0000","last_became_active":"2023-09-23T20:32:33.231683+0000","last_became_peered":"2023-09-23T20:32:33.231683+0000","last_unstale":"2023-09-23T20:41:08.289913+0000","last_undegraded":"2023-09-23T20:41:08.289913+0000","last_fullsized":"2023-09-23T20:41:08.289913+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:02:58.336348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"459'1788","reported_seq":1800,"reported_epoch":459,"state":"active+clean","last_fresh":"2023-09-23T20:41:20.456551+0000","last_change":"2023-09-23T20:41:04.557945+0000","last_active":"2023-09-23T20:41:20.456551+0000","last_peered":"2023-09-23T20:41:20.456551+0000","last_clean":"2023-09-23T20:41:20.456551+0000","last_became_active":"2023-09-23T20:41:04.557435+0000","last_became_peered":"2023-09-23T20:41:04.557435+0000","last_unstale":"2023-09-23T20:41:20.456551+0000","last_undegraded":"2023-09-23T20:41:20.456551+0000","last_fullsized":"2023-09-23T20:41:20.456551+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":455,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":1788,"log_dups_size":0,"ondisk_log_size":1788,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:09:02.910592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":117178368,"num_objects":1788,"num_object_clones":0,"num_object_copies":5364,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1788,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1788,"num_write_kb":114432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.f","version":"90'161","reported_seq":614,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:08.381437+0000","last_change":"2023-09-23T20:38:40.491902+0000","last_active":"2023-09-23T20:41:08.381437+0000","last_peered":"2023-09-23T20:41:08.381437+0000","last_clean":"2023-09-23T20:41:08.381437+0000","last_became_active":"2023-09-23T20:38:40.491563+0000","last_became_peered":"2023-09-23T20:38:40.491563+0000","last_unstale":"2023-09-23T20:41:08.381437+0000","last_undegraded":"2023-09-23T20:41:08.381437+0000","last_fullsized":"2023-09-23T20:41:08.381437+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:46.217489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:46.217489+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:39:25.008796+0000","scrub_duration":0.001456429,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":487,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:08.769471+0000","last_change":"2023-09-23T20:40:19.677655+0000","last_active":"2023-09-23T20:41:08.769471+0000","last_peered":"2023-09-23T20:41:08.769471+0000","last_clean":"2023-09-23T20:41:08.769471+0000","last_became_active":"2023-09-23T20:40:19.551189+0000","last_became_peered":"2023-09-23T20:40:19.551189+0000","last_unstale":"2023-09-23T20:41:08.769471+0000","last_undegraded":"2023-09-23T20:41:08.769471+0000","last_fullsized":"2023-09-23T20:41:08.769471+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:11:01.559576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":64,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:08.380895+0000","last_change":"2023-09-23T20:38:02.590613+0000","last_active":"2023-09-23T20:41:08.380895+0000","last_peered":"2023-09-23T20:41:08.380895+0000","last_clean":"2023-09-23T20:41:08.380895+0000","last_became_active":"2023-09-23T20:38:02.590284+0000","last_became_peered":"2023-09-23T20:38:02.590284+0000","last_unstale":"2023-09-23T20:41:08.380895+0000","last_undegraded":"2023-09-23T20:41:08.380895+0000","last_fullsized":"2023-09-23T20:41:08.380895+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:40:07.825054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"459'1769","reported_seq":1781,"reported_epoch":459,"state":"active+clean","last_fresh":"2023-09-23T20:41:20.454959+0000","last_change":"2023-09-23T20:41:04.573199+0000","last_active":"2023-09-23T20:41:20.454959+0000","last_peered":"2023-09-23T20:41:20.454959+0000","last_clean":"2023-09-23T20:41:20.454959+0000","last_became_active":"2023-09-23T20:41:04.572850+0000","last_became_peered":"2023-09-23T20:41:04.572850+0000","last_unstale":"2023-09-23T20:41:20.454959+0000","last_undegraded":"2023-09-23T20:41:20.454959+0000","last_fullsized":"2023-09-23T20:41:20.454959+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":455,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":1769,"log_dups_size":0,"ondisk_log_size":1769,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:53:49.894113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":115933184,"num_objects":1769,"num_object_clones":0,"num_object_copies":5307,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1769,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1769,"num_write_kb":113216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.e","version":"90'161","reported_seq":559,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:08.290482+0000","last_change":"2023-09-23T20:36:00.322453+0000","last_active":"2023-09-23T20:41:08.290482+0000","last_peered":"2023-09-23T20:41:08.290482+0000","last_clean":"2023-09-23T20:41:08.290482+0000","last_became_active":"2023-09-23T20:35:02.233536+0000","last_became_peered":"2023-09-23T20:35:02.233536+0000","last_unstale":"2023-09-23T20:41:08.290482+0000","last_undegraded":"2023-09-23T20:41:08.290482+0000","last_fullsized":"2023-09-23T20:41:08.290482+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:00.322381+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:00.322381+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:02:15.489069+0000","scrub_duration":0.0017321859999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":53,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:09.136104+0000","last_change":"2023-09-23T20:38:37.456683+0000","last_active":"2023-09-23T20:41:09.136104+0000","last_peered":"2023-09-23T20:41:09.136104+0000","last_clean":"2023-09-23T20:41:09.136104+0000","last_became_active":"2023-09-23T20:38:37.456289+0000","last_became_peered":"2023-09-23T20:38:37.456289+0000","last_unstale":"2023-09-23T20:41:09.136104+0000","last_undegraded":"2023-09-23T20:41:09.136104+0000","last_fullsized":"2023-09-23T20:41:09.136104+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:03:10.507333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":445,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:07.553544+0000","last_change":"2023-09-23T20:32:33.236122+0000","last_active":"2023-09-23T20:41:07.553544+0000","last_peered":"2023-09-23T20:41:07.553544+0000","last_clean":"2023-09-23T20:41:07.553544+0000","last_became_active":"2023-09-23T20:32:33.235384+0000","last_became_peered":"2023-09-23T20:32:33.235384+0000","last_unstale":"2023-09-23T20:41:07.553544+0000","last_undegraded":"2023-09-23T20:41:07.553544+0000","last_fullsized":"2023-09-23T20:41:07.553544+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:56.843733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"459'1539","reported_seq":1551,"reported_epoch":459,"state":"active+clean","last_fresh":"2023-09-23T20:41:18.162903+0000","last_change":"2023-09-23T20:41:04.558473+0000","last_active":"2023-09-23T20:41:18.162903+0000","last_peered":"2023-09-23T20:41:18.162903+0000","last_clean":"2023-09-23T20:41:18.162903+0000","last_became_active":"2023-09-23T20:41:04.558173+0000","last_became_peered":"2023-09-23T20:41:04.558173+0000","last_unstale":"2023-09-23T20:41:18.162903+0000","last_undegraded":"2023-09-23T20:41:18.162903+0000","last_fullsized":"2023-09-23T20:41:18.162903+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":455,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":1539,"log_dups_size":0,"ondisk_log_size":1539,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:17:53.184877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":100859904,"num_objects":1539,"num_object_clones":0,"num_object_copies":4617,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1539,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1539,"num_write_kb":98496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":864,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:08.768866+0000","last_change":"2023-09-23T20:35:02.523308+0000","last_active":"2023-09-23T20:41:08.768866+0000","last_peered":"2023-09-23T20:41:08.768866+0000","last_clean":"2023-09-23T20:41:08.768866+0000","last_became_active":"2023-09-23T20:33:38.979989+0000","last_became_peered":"2023-09-23T20:33:38.979989+0000","last_unstale":"2023-09-23T20:41:08.768866+0000","last_undegraded":"2023-09-23T20:41:08.768866+0000","last_fullsized":"2023-09-23T20:41:08.768866+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:35:02.523203+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:35:02.523203+0000","last_clean_scrub_stamp":"2023-09-23T20:35:02.523203+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:02:41.472958+0000","scrub_duration":0.0085927989999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":278,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:08.520614+0000","last_change":"2023-09-23T20:37:43.583731+0000","last_active":"2023-09-23T20:41:08.520614+0000","last_peered":"2023-09-23T20:41:08.520614+0000","last_clean":"2023-09-23T20:41:08.520614+0000","last_became_active":"2023-09-23T20:37:43.583066+0000","last_became_peered":"2023-09-23T20:37:43.583066+0000","last_unstale":"2023-09-23T20:41:08.520614+0000","last_undegraded":"2023-09-23T20:41:08.520614+0000","last_fullsized":"2023-09-23T20:41:08.520614+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:03:45.211900+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":176,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:08.290216+0000","last_change":"2023-09-23T20:38:02.579814+0000","last_active":"2023-09-23T20:41:08.290216+0000","last_peered":"2023-09-23T20:41:08.290216+0000","last_clean":"2023-09-23T20:41:08.290216+0000","last_became_active":"2023-09-23T20:38:02.579253+0000","last_became_peered":"2023-09-23T20:38:02.579253+0000","last_unstale":"2023-09-23T20:41:08.290216+0000","last_undegraded":"2023-09-23T20:41:08.290216+0000","last_fullsized":"2023-09-23T20:41:08.290216+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:05:07.298198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"459'1665","reported_seq":1677,"reported_epoch":459,"state":"active+clean","last_fresh":"2023-09-23T20:41:19.562111+0000","last_change":"2023-09-23T20:41:04.573101+0000","last_active":"2023-09-23T20:41:19.562111+0000","last_peered":"2023-09-23T20:41:19.562111+0000","last_clean":"2023-09-23T20:41:19.562111+0000","last_became_active":"2023-09-23T20:41:04.572728+0000","last_became_peered":"2023-09-23T20:41:04.572728+0000","last_unstale":"2023-09-23T20:41:19.562111+0000","last_undegraded":"2023-09-23T20:41:19.562111+0000","last_fullsized":"2023-09-23T20:41:19.562111+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":455,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":1665,"log_dups_size":0,"ondisk_log_size":1665,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:21:10.721593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109117440,"num_objects":1665,"num_object_clones":0,"num_object_copies":4995,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1665,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1665,"num_write_kb":106560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":570,"reported_epoch":459,"state":"active+clean","last_fresh":"2023-09-23T20:41:16.281729+0000","last_change":"2023-09-23T20:41:16.281681+0000","last_active":"2023-09-23T20:41:16.281729+0000","last_peered":"2023-09-23T20:41:16.281729+0000","last_clean":"2023-09-23T20:41:16.281729+0000","last_became_active":"2023-09-23T20:40:22.433023+0000","last_became_peered":"2023-09-23T20:40:22.433023+0000","last_unstale":"2023-09-23T20:41:16.281729+0000","last_undegraded":"2023-09-23T20:41:16.281729+0000","last_fullsized":"2023-09-23T20:41:16.281729+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:41:16.281575+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:41:16.281575+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:51:51.938276+0000","scrub_duration":0.0055773209999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":64,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:08.380898+0000","last_change":"2023-09-23T20:38:02.578703+0000","last_active":"2023-09-23T20:41:08.380898+0000","last_peered":"2023-09-23T20:41:08.380898+0000","last_clean":"2023-09-23T20:41:08.380898+0000","last_became_active":"2023-09-23T20:38:02.578414+0000","last_became_peered":"2023-09-23T20:38:02.578414+0000","last_unstale":"2023-09-23T20:41:08.380898+0000","last_undegraded":"2023-09-23T20:41:08.380898+0000","last_fullsized":"2023-09-23T20:41:08.380898+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:43:59.779295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":125,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:08.290209+0000","last_change":"2023-09-23T20:37:31.296606+0000","last_active":"2023-09-23T20:41:08.290209+0000","last_peered":"2023-09-23T20:41:08.290209+0000","last_clean":"2023-09-23T20:41:08.290209+0000","last_became_active":"2023-09-23T20:37:31.295924+0000","last_became_peered":"2023-09-23T20:37:31.295924+0000","last_unstale":"2023-09-23T20:41:08.290209+0000","last_undegraded":"2023-09-23T20:41:08.290209+0000","last_fullsized":"2023-09-23T20:41:08.290209+0000","mapping_epoch":391,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":392,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:27:50.948510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"459'1412","reported_seq":1424,"reported_epoch":459,"state":"active+clean","last_fresh":"2023-09-23T20:41:17.364829+0000","last_change":"2023-09-23T20:41:04.555980+0000","last_active":"2023-09-23T20:41:17.364829+0000","last_peered":"2023-09-23T20:41:17.364829+0000","last_clean":"2023-09-23T20:41:17.364829+0000","last_became_active":"2023-09-23T20:41:04.555688+0000","last_became_peered":"2023-09-23T20:41:04.555688+0000","last_unstale":"2023-09-23T20:41:17.364829+0000","last_undegraded":"2023-09-23T20:41:17.364829+0000","last_fullsized":"2023-09-23T20:41:17.364829+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":455,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":1412,"log_dups_size":0,"ondisk_log_size":1412,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:28:42.819404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":92536832,"num_objects":1412,"num_object_clones":0,"num_object_copies":4236,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1412,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1412,"num_write_kb":90368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.b","version":"90'161","reported_seq":761,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:09.135450+0000","last_change":"2023-09-23T20:40:22.445808+0000","last_active":"2023-09-23T20:41:09.135450+0000","last_peered":"2023-09-23T20:41:09.135450+0000","last_clean":"2023-09-23T20:41:09.135450+0000","last_became_active":"2023-09-23T20:40:22.445456+0000","last_became_peered":"2023-09-23T20:40:22.445456+0000","last_unstale":"2023-09-23T20:41:09.135450+0000","last_undegraded":"2023-09-23T20:41:09.135450+0000","last_fullsized":"2023-09-23T20:41:09.135450+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:05.595042+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:05.595042+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:39:23.678154+0000","scrub_duration":0.007271675,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":115,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:08.289812+0000","last_change":"2023-09-23T20:34:59.983122+0000","last_active":"2023-09-23T20:41:08.289812+0000","last_peered":"2023-09-23T20:41:08.289812+0000","last_clean":"2023-09-23T20:41:08.289812+0000","last_became_active":"2023-09-23T20:34:59.982809+0000","last_became_peered":"2023-09-23T20:34:59.982809+0000","last_unstale":"2023-09-23T20:41:08.289812+0000","last_undegraded":"2023-09-23T20:41:08.289812+0000","last_fullsized":"2023-09-23T20:41:08.289812+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:49:40.839723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":176,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:08.290562+0000","last_change":"2023-09-23T20:38:02.543866+0000","last_active":"2023-09-23T20:41:08.290562+0000","last_peered":"2023-09-23T20:41:08.290562+0000","last_clean":"2023-09-23T20:41:08.290562+0000","last_became_active":"2023-09-23T20:38:02.543563+0000","last_became_peered":"2023-09-23T20:38:02.543563+0000","last_unstale":"2023-09-23T20:41:08.290562+0000","last_undegraded":"2023-09-23T20:41:08.290562+0000","last_fullsized":"2023-09-23T20:41:08.290562+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:33.050536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"459'1596","reported_seq":1609,"reported_epoch":459,"state":"active+clean","last_fresh":"2023-09-23T20:41:18.870755+0000","last_change":"2023-09-23T20:41:04.573300+0000","last_active":"2023-09-23T20:41:18.870755+0000","last_peered":"2023-09-23T20:41:18.870755+0000","last_clean":"2023-09-23T20:41:18.870755+0000","last_became_active":"2023-09-23T20:41:04.572919+0000","last_became_peered":"2023-09-23T20:41:04.572919+0000","last_unstale":"2023-09-23T20:41:18.870755+0000","last_undegraded":"2023-09-23T20:41:18.870755+0000","last_fullsized":"2023-09-23T20:41:18.870755+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":455,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":1596,"log_dups_size":0,"ondisk_log_size":1596,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:57:46.465243+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":104595456,"num_objects":1596,"num_object_clones":0,"num_object_copies":4788,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1596,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1596,"num_write_kb":102144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":573,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:09.135936+0000","last_change":"2023-09-23T20:41:07.289523+0000","last_active":"2023-09-23T20:41:09.135936+0000","last_peered":"2023-09-23T20:41:09.135936+0000","last_clean":"2023-09-23T20:41:09.135936+0000","last_became_active":"2023-09-23T20:38:40.494049+0000","last_became_peered":"2023-09-23T20:38:40.494049+0000","last_unstale":"2023-09-23T20:41:09.135936+0000","last_undegraded":"2023-09-23T20:41:09.135936+0000","last_fullsized":"2023-09-23T20:41:09.135936+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:41:07.289420+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:41:07.289420+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:12:25.551623+0000","scrub_duration":0.0015464249999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":252,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:08.520299+0000","last_change":"2023-09-23T20:37:43.570610+0000","last_active":"2023-09-23T20:41:08.520299+0000","last_peered":"2023-09-23T20:41:08.520299+0000","last_clean":"2023-09-23T20:41:08.520299+0000","last_became_active":"2023-09-23T20:37:43.570216+0000","last_became_peered":"2023-09-23T20:37:43.570216+0000","last_unstale":"2023-09-23T20:41:08.520299+0000","last_undegraded":"2023-09-23T20:41:08.520299+0000","last_fullsized":"2023-09-23T20:41:08.520299+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:39:26.555329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":283,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:09.133072+0000","last_change":"2023-09-23T20:32:33.230582+0000","last_active":"2023-09-23T20:41:09.133072+0000","last_peered":"2023-09-23T20:41:09.133072+0000","last_clean":"2023-09-23T20:41:09.133072+0000","last_became_active":"2023-09-23T20:32:33.230257+0000","last_became_peered":"2023-09-23T20:32:33.230257+0000","last_unstale":"2023-09-23T20:41:09.133072+0000","last_undegraded":"2023-09-23T20:41:09.133072+0000","last_fullsized":"2023-09-23T20:41:09.133072+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:11:45.333096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.f","version":"459'1799","reported_seq":1809,"reported_epoch":459,"state":"active+clean","last_fresh":"2023-09-23T20:41:20.478934+0000","last_change":"2023-09-23T20:41:04.557353+0000","last_active":"2023-09-23T20:41:20.478934+0000","last_peered":"2023-09-23T20:41:20.478934+0000","last_clean":"2023-09-23T20:41:20.478934+0000","last_became_active":"2023-09-23T20:41:04.557037+0000","last_became_peered":"2023-09-23T20:41:04.557037+0000","last_unstale":"2023-09-23T20:41:20.478934+0000","last_undegraded":"2023-09-23T20:41:20.478934+0000","last_fullsized":"2023-09-23T20:41:20.478934+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":455,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":1799,"log_dups_size":0,"ondisk_log_size":1799,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:23:30.329193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":117899264,"num_objects":1799,"num_object_clones":0,"num_object_copies":5397,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1799,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1799,"num_write_kb":115136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":647,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:08.381395+0000","last_change":"2023-09-23T20:41:07.424609+0000","last_active":"2023-09-23T20:41:08.381395+0000","last_peered":"2023-09-23T20:41:08.381395+0000","last_clean":"2023-09-23T20:41:08.381395+0000","last_became_active":"2023-09-23T20:38:05.622261+0000","last_became_peered":"2023-09-23T20:38:05.622261+0000","last_unstale":"2023-09-23T20:41:08.381395+0000","last_undegraded":"2023-09-23T20:41:08.381395+0000","last_fullsized":"2023-09-23T20:41:08.381395+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:41:07.424492+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:41:07.424492+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:45:21.689123+0000","scrub_duration":0.0018960959999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"86'119","reported_seq":767,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:09.133426+0000","last_change":"2023-09-23T20:29:45.521257+0000","last_active":"2023-09-23T20:41:09.133426+0000","last_peered":"2023-09-23T20:41:09.133426+0000","last_clean":"2023-09-23T20:41:09.133426+0000","last_became_active":"2023-09-23T20:29:45.520546+0000","last_became_peered":"2023-09-23T20:29:45.520546+0000","last_unstale":"2023-09-23T20:41:09.133426+0000","last_undegraded":"2023-09-23T20:41:09.133426+0000","last_fullsized":"2023-09-23T20:41:09.133426+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":119,"log_dups_size":0,"ondisk_log_size":119,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:58:56.085777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032705499999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":523,"num_read_kb":3503,"num_write":136,"num_write_kb":3128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":28,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:09.136362+0000","last_change":"2023-09-23T20:40:19.557009+0000","last_active":"2023-09-23T20:41:09.136362+0000","last_peered":"2023-09-23T20:41:09.136362+0000","last_clean":"2023-09-23T20:41:09.136362+0000","last_became_active":"2023-09-23T20:40:19.550473+0000","last_became_peered":"2023-09-23T20:40:19.550473+0000","last_unstale":"2023-09-23T20:41:09.136362+0000","last_undegraded":"2023-09-23T20:41:09.136362+0000","last_fullsized":"2023-09-23T20:41:09.136362+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:35:01.140258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":414,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:09.132193+0000","last_change":"2023-09-23T20:40:19.530942+0000","last_active":"2023-09-23T20:41:09.132193+0000","last_peered":"2023-09-23T20:41:09.132193+0000","last_clean":"2023-09-23T20:41:09.132193+0000","last_became_active":"2023-09-23T20:40:19.530617+0000","last_became_peered":"2023-09-23T20:40:19.530617+0000","last_unstale":"2023-09-23T20:41:09.132193+0000","last_undegraded":"2023-09-23T20:41:09.132193+0000","last_fullsized":"2023-09-23T20:41:09.132193+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:24:09.231934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.9","version":"459'1906","reported_seq":1919,"reported_epoch":459,"state":"active+clean","last_fresh":"2023-09-23T20:41:21.257269+0000","last_change":"2023-09-23T20:41:04.556637+0000","last_active":"2023-09-23T20:41:21.257269+0000","last_peered":"2023-09-23T20:41:21.257269+0000","last_clean":"2023-09-23T20:41:21.257269+0000","last_became_active":"2023-09-23T20:41:04.555979+0000","last_became_peered":"2023-09-23T20:41:04.555979+0000","last_unstale":"2023-09-23T20:41:21.257269+0000","last_undegraded":"2023-09-23T20:41:21.257269+0000","last_fullsized":"2023-09-23T20:41:21.257269+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":455,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":1906,"log_dups_size":0,"ondisk_log_size":1906,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:45:38.600613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":124911616,"num_objects":1906,"num_object_clones":0,"num_object_copies":5718,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1906,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1906,"num_write_kb":121984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":532,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:09.135886+0000","last_change":"2023-09-23T20:40:22.445201+0000","last_active":"2023-09-23T20:41:09.135886+0000","last_peered":"2023-09-23T20:41:09.135886+0000","last_clean":"2023-09-23T20:41:09.135886+0000","last_became_active":"2023-09-23T20:40:22.444703+0000","last_became_peered":"2023-09-23T20:40:22.444703+0000","last_unstale":"2023-09-23T20:41:09.135886+0000","last_undegraded":"2023-09-23T20:41:09.135886+0000","last_fullsized":"2023-09-23T20:41:09.135886+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:45.785450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:45.785450+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"86'106","reported_seq":1178,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:09.136128+0000","last_change":"2023-09-23T20:38:37.476512+0000","last_active":"2023-09-23T20:41:09.136128+0000","last_peered":"2023-09-23T20:41:09.136128+0000","last_clean":"2023-09-23T20:41:09.136128+0000","last_became_active":"2023-09-23T20:38:37.475980+0000","last_became_peered":"2023-09-23T20:38:37.475980+0000","last_unstale":"2023-09-23T20:41:09.136128+0000","last_undegraded":"2023-09-23T20:41:09.136128+0000","last_fullsized":"2023-09-23T20:41:09.136128+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":106,"log_dups_size":0,"ondisk_log_size":106,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:58:45.508831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00099302599999999993,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1094,"num_read_kb":1678,"num_write":100,"num_write_kb":404,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":176,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:08.290217+0000","last_change":"2023-09-23T20:38:02.552008+0000","last_active":"2023-09-23T20:41:08.290217+0000","last_peered":"2023-09-23T20:41:08.290217+0000","last_clean":"2023-09-23T20:41:08.290217+0000","last_became_active":"2023-09-23T20:38:02.551684+0000","last_became_peered":"2023-09-23T20:38:02.551684+0000","last_unstale":"2023-09-23T20:41:08.290217+0000","last_undegraded":"2023-09-23T20:41:08.290217+0000","last_fullsized":"2023-09-23T20:41:08.290217+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:40:37.147538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":420,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:09.133342+0000","last_change":"2023-09-23T20:29:45.533406+0000","last_active":"2023-09-23T20:41:09.133342+0000","last_peered":"2023-09-23T20:41:09.133342+0000","last_clean":"2023-09-23T20:41:09.133342+0000","last_became_active":"2023-09-23T20:29:45.526001+0000","last_became_peered":"2023-09-23T20:29:45.526001+0000","last_unstale":"2023-09-23T20:41:09.133342+0000","last_undegraded":"2023-09-23T20:41:09.133342+0000","last_fullsized":"2023-09-23T20:41:09.133342+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:43.787251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.a","version":"459'1804","reported_seq":1816,"reported_epoch":459,"state":"active+clean","last_fresh":"2023-09-23T20:41:20.459948+0000","last_change":"2023-09-23T20:41:04.556831+0000","last_active":"2023-09-23T20:41:20.459948+0000","last_peered":"2023-09-23T20:41:20.459948+0000","last_clean":"2023-09-23T20:41:20.459948+0000","last_became_active":"2023-09-23T20:41:04.556573+0000","last_became_peered":"2023-09-23T20:41:04.556573+0000","last_unstale":"2023-09-23T20:41:20.459948+0000","last_undegraded":"2023-09-23T20:41:20.459948+0000","last_fullsized":"2023-09-23T20:41:20.459948+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":455,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":1804,"log_dups_size":0,"ondisk_log_size":1804,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:55:53.953463+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":118226944,"num_objects":1804,"num_object_clones":0,"num_object_copies":5412,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1804,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1804,"num_write_kb":115456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":931,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:08.768989+0000","last_change":"2023-09-23T20:35:02.260708+0000","last_active":"2023-09-23T20:41:08.768989+0000","last_peered":"2023-09-23T20:41:08.768989+0000","last_clean":"2023-09-23T20:41:08.768989+0000","last_became_active":"2023-09-23T20:35:02.260348+0000","last_became_peered":"2023-09-23T20:35:02.260348+0000","last_unstale":"2023-09-23T20:41:08.768989+0000","last_undegraded":"2023-09-23T20:41:08.768989+0000","last_fullsized":"2023-09-23T20:41:08.768989+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:43.390908+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:43.390908+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:09:14.144070+0000","scrub_duration":0.0032721,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"86'130","reported_seq":775,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:08.619332+0000","last_change":"2023-09-23T20:25:39.637458+0000","last_active":"2023-09-23T20:41:08.619332+0000","last_peered":"2023-09-23T20:41:08.619332+0000","last_clean":"2023-09-23T20:41:08.619332+0000","last_became_active":"2023-09-23T20:25:39.636773+0000","last_became_peered":"2023-09-23T20:25:39.636773+0000","last_unstale":"2023-09-23T20:41:08.619332+0000","last_undegraded":"2023-09-23T20:41:08.619332+0000","last_fullsized":"2023-09-23T20:41:08.619332+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:00:32.810564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011300049999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":512,"num_read_kb":1820,"num_write":133,"num_write_kb":1347,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":428,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:09.132095+0000","last_change":"2023-09-23T20:38:02.591335+0000","last_active":"2023-09-23T20:41:09.132095+0000","last_peered":"2023-09-23T20:41:09.132095+0000","last_clean":"2023-09-23T20:41:09.132095+0000","last_became_active":"2023-09-23T20:38:02.591015+0000","last_became_peered":"2023-09-23T20:38:02.591015+0000","last_unstale":"2023-09-23T20:41:09.132095+0000","last_undegraded":"2023-09-23T20:41:09.132095+0000","last_fullsized":"2023-09-23T20:41:09.132095+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:55:32.272195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":420,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:09.133002+0000","last_change":"2023-09-23T20:40:19.544333+0000","last_active":"2023-09-23T20:41:09.133002+0000","last_peered":"2023-09-23T20:41:09.133002+0000","last_clean":"2023-09-23T20:41:09.133002+0000","last_became_active":"2023-09-23T20:40:19.543987+0000","last_became_peered":"2023-09-23T20:40:19.543987+0000","last_unstale":"2023-09-23T20:41:09.133002+0000","last_undegraded":"2023-09-23T20:41:09.133002+0000","last_fullsized":"2023-09-23T20:41:09.133002+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:12:59.203296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.8","version":"459'1539","reported_seq":1552,"reported_epoch":459,"state":"active+clean","last_fresh":"2023-09-23T20:41:18.898662+0000","last_change":"2023-09-23T20:41:04.550745+0000","last_active":"2023-09-23T20:41:18.898662+0000","last_peered":"2023-09-23T20:41:18.898662+0000","last_clean":"2023-09-23T20:41:18.898662+0000","last_became_active":"2023-09-23T20:41:04.550493+0000","last_became_peered":"2023-09-23T20:41:04.550493+0000","last_unstale":"2023-09-23T20:41:18.898662+0000","last_undegraded":"2023-09-23T20:41:18.898662+0000","last_fullsized":"2023-09-23T20:41:18.898662+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":455,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":1539,"log_dups_size":0,"ondisk_log_size":1539,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:28:41.080518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":100859904,"num_objects":1539,"num_object_clones":0,"num_object_copies":4617,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1539,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1539,"num_write_kb":98496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"86'194","reported_seq":811,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:09.135872+0000","last_change":"2023-09-23T20:40:51.458056+0000","last_active":"2023-09-23T20:41:09.135872+0000","last_peered":"2023-09-23T20:41:09.135872+0000","last_clean":"2023-09-23T20:41:09.135872+0000","last_became_active":"2023-09-23T20:40:51.457465+0000","last_became_peered":"2023-09-23T20:40:51.457465+0000","last_unstale":"2023-09-23T20:41:09.135872+0000","last_undegraded":"2023-09-23T20:41:09.135872+0000","last_fullsized":"2023-09-23T20:41:09.135872+0000","mapping_epoch":450,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":451,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":194,"log_dups_size":0,"ondisk_log_size":194,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:18:36.482553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00037995999999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1024,"num_read_kb":1200,"num_write":173,"num_write_kb":1276,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":921,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:08.768789+0000","last_change":"2023-09-23T20:38:05.602508+0000","last_active":"2023-09-23T20:41:08.768789+0000","last_peered":"2023-09-23T20:41:08.768789+0000","last_clean":"2023-09-23T20:41:08.768789+0000","last_became_active":"2023-09-23T20:38:05.602129+0000","last_became_peered":"2023-09-23T20:38:05.602129+0000","last_unstale":"2023-09-23T20:41:08.768789+0000","last_undegraded":"2023-09-23T20:41:08.768789+0000","last_fullsized":"2023-09-23T20:41:08.768789+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:45.343742+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:45.343742+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:48.910343+0000","scrub_duration":0.001475802,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":28,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:09.135847+0000","last_change":"2023-09-23T20:40:19.542142+0000","last_active":"2023-09-23T20:41:09.135847+0000","last_peered":"2023-09-23T20:41:09.135847+0000","last_clean":"2023-09-23T20:41:09.135847+0000","last_became_active":"2023-09-23T20:40:19.541870+0000","last_became_peered":"2023-09-23T20:40:19.541870+0000","last_unstale":"2023-09-23T20:41:09.135847+0000","last_undegraded":"2023-09-23T20:41:09.135847+0000","last_fullsized":"2023-09-23T20:41:09.135847+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:01.518149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":64,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:08.381338+0000","last_change":"2023-09-23T20:38:02.573200+0000","last_active":"2023-09-23T20:41:08.381338+0000","last_peered":"2023-09-23T20:41:08.381338+0000","last_clean":"2023-09-23T20:41:08.381338+0000","last_became_active":"2023-09-23T20:38:02.572945+0000","last_became_peered":"2023-09-23T20:38:02.572945+0000","last_unstale":"2023-09-23T20:41:08.381338+0000","last_undegraded":"2023-09-23T20:41:08.381338+0000","last_fullsized":"2023-09-23T20:41:08.381338+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:57.246533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.b","version":"459'1837","reported_seq":1850,"reported_epoch":459,"state":"active+clean","last_fresh":"2023-09-23T20:41:21.264498+0000","last_change":"2023-09-23T20:41:04.557718+0000","last_active":"2023-09-23T20:41:21.264498+0000","last_peered":"2023-09-23T20:41:21.264498+0000","last_clean":"2023-09-23T20:41:21.264498+0000","last_became_active":"2023-09-23T20:41:04.557420+0000","last_became_peered":"2023-09-23T20:41:04.557420+0000","last_unstale":"2023-09-23T20:41:21.264498+0000","last_undegraded":"2023-09-23T20:41:21.264498+0000","last_fullsized":"2023-09-23T20:41:21.264498+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":455,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":1837,"log_dups_size":0,"ondisk_log_size":1837,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:40:23.719326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":120389632,"num_objects":1837,"num_object_clones":0,"num_object_copies":5511,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1837,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1837,"num_write_kb":117568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":789,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:09.132585+0000","last_change":"2023-09-23T20:38:05.615711+0000","last_active":"2023-09-23T20:41:09.132585+0000","last_peered":"2023-09-23T20:41:09.132585+0000","last_clean":"2023-09-23T20:41:09.132585+0000","last_became_active":"2023-09-23T20:38:05.615390+0000","last_became_peered":"2023-09-23T20:38:05.615390+0000","last_unstale":"2023-09-23T20:41:09.132585+0000","last_undegraded":"2023-09-23T20:41:09.132585+0000","last_fullsized":"2023-09-23T20:41:09.132585+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:43.582146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:43.582146+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:45:11.917871+0000","scrub_duration":0.0016219730000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"86'80","reported_seq":711,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:08.768596+0000","last_change":"2023-09-23T20:29:45.536796+0000","last_active":"2023-09-23T20:41:08.768596+0000","last_peered":"2023-09-23T20:41:08.768596+0000","last_clean":"2023-09-23T20:41:08.768596+0000","last_became_active":"2023-09-23T20:29:45.536439+0000","last_became_peered":"2023-09-23T20:29:45.536439+0000","last_unstale":"2023-09-23T20:41:08.768596+0000","last_undegraded":"2023-09-23T20:41:08.768596+0000","last_fullsized":"2023-09-23T20:41:08.768596+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":80,"log_dups_size":0,"ondisk_log_size":80,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:30:57.793132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035115,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":395,"num_read_kb":1370,"num_write":66,"num_write_kb":1061,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":464,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:07.553350+0000","last_change":"2023-09-23T20:34:59.984286+0000","last_active":"2023-09-23T20:41:07.553350+0000","last_peered":"2023-09-23T20:41:07.553350+0000","last_clean":"2023-09-23T20:41:07.553350+0000","last_became_active":"2023-09-23T20:34:59.983950+0000","last_became_peered":"2023-09-23T20:34:59.983950+0000","last_unstale":"2023-09-23T20:41:07.553350+0000","last_undegraded":"2023-09-23T20:41:07.553350+0000","last_fullsized":"2023-09-23T20:41:07.553350+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:31:53.557035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":458,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:07.553793+0000","last_change":"2023-09-23T20:29:45.690815+0000","last_active":"2023-09-23T20:41:07.553793+0000","last_peered":"2023-09-23T20:41:07.553793+0000","last_clean":"2023-09-23T20:41:07.553793+0000","last_became_active":"2023-09-23T20:29:45.690478+0000","last_became_peered":"2023-09-23T20:29:45.690478+0000","last_unstale":"2023-09-23T20:41:07.553793+0000","last_undegraded":"2023-09-23T20:41:07.553793+0000","last_fullsized":"2023-09-23T20:41:07.553793+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:38.288193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.e","version":"459'1396","reported_seq":1407,"reported_epoch":459,"state":"active+clean","last_fresh":"2023-09-23T20:41:17.364569+0000","last_change":"2023-09-23T20:41:04.574876+0000","last_active":"2023-09-23T20:41:17.364569+0000","last_peered":"2023-09-23T20:41:17.364569+0000","last_clean":"2023-09-23T20:41:17.364569+0000","last_became_active":"2023-09-23T20:41:04.574595+0000","last_became_peered":"2023-09-23T20:41:04.574595+0000","last_unstale":"2023-09-23T20:41:17.364569+0000","last_undegraded":"2023-09-23T20:41:17.364569+0000","last_fullsized":"2023-09-23T20:41:17.364569+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":455,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":1396,"log_dups_size":0,"ondisk_log_size":1396,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:31:00.147966+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":91488256,"num_objects":1396,"num_object_clones":0,"num_object_copies":4188,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1396,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1396,"num_write_kb":89344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":570,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:08.381163+0000","last_change":"2023-09-23T20:41:08.380084+0000","last_active":"2023-09-23T20:41:08.381163+0000","last_peered":"2023-09-23T20:41:08.381163+0000","last_clean":"2023-09-23T20:41:08.381163+0000","last_became_active":"2023-09-23T20:38:05.610881+0000","last_became_peered":"2023-09-23T20:38:05.610881+0000","last_unstale":"2023-09-23T20:41:08.381163+0000","last_undegraded":"2023-09-23T20:41:08.381163+0000","last_fullsized":"2023-09-23T20:41:08.381163+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:41:08.379959+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:41:08.379959+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:19:39.547923+0000","scrub_duration":0.0019158420000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"86'102","reported_seq":1053,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:08.768623+0000","last_change":"2023-09-23T20:32:36.314672+0000","last_active":"2023-09-23T20:41:08.768623+0000","last_peered":"2023-09-23T20:41:08.768623+0000","last_clean":"2023-09-23T20:41:08.768623+0000","last_became_active":"2023-09-23T20:32:36.314231+0000","last_became_peered":"2023-09-23T20:32:36.314231+0000","last_unstale":"2023-09-23T20:41:08.768623+0000","last_undegraded":"2023-09-23T20:41:08.768623+0000","last_fullsized":"2023-09-23T20:41:08.768623+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":102,"log_dups_size":0,"ondisk_log_size":102,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:11:40.514709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030656500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":963,"num_read_kb":1234,"num_write":60,"num_write_kb":225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":404,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:09.133512+0000","last_change":"2023-09-23T20:40:19.544384+0000","last_active":"2023-09-23T20:41:09.133512+0000","last_peered":"2023-09-23T20:41:09.133512+0000","last_clean":"2023-09-23T20:41:09.133512+0000","last_became_active":"2023-09-23T20:40:19.544079+0000","last_became_peered":"2023-09-23T20:40:19.544079+0000","last_unstale":"2023-09-23T20:41:09.133512+0000","last_undegraded":"2023-09-23T20:41:09.133512+0000","last_fullsized":"2023-09-23T20:41:09.133512+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:00:19.498631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":414,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:07.553669+0000","last_change":"2023-09-23T20:40:19.532976+0000","last_active":"2023-09-23T20:41:07.553669+0000","last_peered":"2023-09-23T20:41:07.553669+0000","last_clean":"2023-09-23T20:41:07.553669+0000","last_became_active":"2023-09-23T20:40:19.532639+0000","last_became_peered":"2023-09-23T20:40:19.532639+0000","last_unstale":"2023-09-23T20:41:07.553669+0000","last_undegraded":"2023-09-23T20:41:07.553669+0000","last_fullsized":"2023-09-23T20:41:07.553669+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:02:35.301884+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"459'1477","reported_seq":1489,"reported_epoch":459,"state":"active+clean","last_fresh":"2023-09-23T20:41:17.360639+0000","last_change":"2023-09-23T20:41:04.558941+0000","last_active":"2023-09-23T20:41:17.360639+0000","last_peered":"2023-09-23T20:41:17.360639+0000","last_clean":"2023-09-23T20:41:17.360639+0000","last_became_active":"2023-09-23T20:41:04.558482+0000","last_became_peered":"2023-09-23T20:41:04.558482+0000","last_unstale":"2023-09-23T20:41:17.360639+0000","last_undegraded":"2023-09-23T20:41:17.360639+0000","last_fullsized":"2023-09-23T20:41:17.360639+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":455,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":1477,"log_dups_size":0,"ondisk_log_size":1477,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:36:14.259949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":96796672,"num_objects":1477,"num_object_clones":0,"num_object_copies":4431,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1477,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1477,"num_write_kb":94528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":737,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:08.290510+0000","last_change":"2023-09-23T20:38:05.590361+0000","last_active":"2023-09-23T20:41:08.290510+0000","last_peered":"2023-09-23T20:41:08.290510+0000","last_clean":"2023-09-23T20:41:08.290510+0000","last_became_active":"2023-09-23T20:38:05.589985+0000","last_became_peered":"2023-09-23T20:38:05.589985+0000","last_unstale":"2023-09-23T20:41:08.290510+0000","last_undegraded":"2023-09-23T20:41:08.290510+0000","last_fullsized":"2023-09-23T20:41:08.290510+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:03.246271+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:03.246271+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:38:54.351576+0000","scrub_duration":0.0019016700000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":196,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:08.769616+0000","last_change":"2023-09-23T20:32:50.549695+0000","last_active":"2023-09-23T20:41:08.769616+0000","last_peered":"2023-09-23T20:41:08.769616+0000","last_clean":"2023-09-23T20:41:08.769616+0000","last_became_active":"2023-09-23T20:32:50.549431+0000","last_became_peered":"2023-09-23T20:32:50.549431+0000","last_unstale":"2023-09-23T20:41:08.769616+0000","last_undegraded":"2023-09-23T20:41:08.769616+0000","last_fullsized":"2023-09-23T20:41:08.769616+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:45:53.666569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"86'128","reported_seq":870,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:09.133064+0000","last_change":"2023-09-23T20:38:03.541765+0000","last_active":"2023-09-23T20:41:09.133064+0000","last_peered":"2023-09-23T20:41:09.133064+0000","last_clean":"2023-09-23T20:41:09.133064+0000","last_became_active":"2023-09-23T20:38:03.541277+0000","last_became_peered":"2023-09-23T20:38:03.541277+0000","last_unstale":"2023-09-23T20:41:09.133064+0000","last_undegraded":"2023-09-23T20:41:09.133064+0000","last_fullsized":"2023-09-23T20:41:09.133064+0000","mapping_epoch":404,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":405,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:20:53.834554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000293202,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":579,"num_read_kb":1473,"num_write":96,"num_write_kb":1062,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":237,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:08.520206+0000","last_change":"2023-09-23T20:37:43.582105+0000","last_active":"2023-09-23T20:41:08.520206+0000","last_peered":"2023-09-23T20:41:08.520206+0000","last_clean":"2023-09-23T20:41:08.520206+0000","last_became_active":"2023-09-23T20:37:43.581860+0000","last_became_peered":"2023-09-23T20:37:43.581860+0000","last_unstale":"2023-09-23T20:41:08.520206+0000","last_undegraded":"2023-09-23T20:41:08.520206+0000","last_fullsized":"2023-09-23T20:41:08.520206+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:18:38.765486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.c","version":"459'1613","reported_seq":1626,"reported_epoch":459,"state":"active+clean","last_fresh":"2023-09-23T20:41:18.927614+0000","last_change":"2023-09-23T20:41:04.575496+0000","last_active":"2023-09-23T20:41:18.927614+0000","last_peered":"2023-09-23T20:41:18.927614+0000","last_clean":"2023-09-23T20:41:18.927614+0000","last_became_active":"2023-09-23T20:41:04.575242+0000","last_became_peered":"2023-09-23T20:41:04.575242+0000","last_unstale":"2023-09-23T20:41:18.927614+0000","last_undegraded":"2023-09-23T20:41:18.927614+0000","last_fullsized":"2023-09-23T20:41:18.927614+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":455,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":1613,"log_dups_size":0,"ondisk_log_size":1613,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:30:14.611529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":105709568,"num_objects":1613,"num_object_clones":0,"num_object_copies":4839,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1613,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1613,"num_write_kb":103232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"90'161","reported_seq":671,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:08.520608+0000","last_change":"2023-09-23T20:40:22.427435+0000","last_active":"2023-09-23T20:41:08.520608+0000","last_peered":"2023-09-23T20:41:08.520608+0000","last_clean":"2023-09-23T20:41:08.520608+0000","last_became_active":"2023-09-23T20:40:22.427051+0000","last_became_peered":"2023-09-23T20:40:22.427051+0000","last_unstale":"2023-09-23T20:41:08.520608+0000","last_undegraded":"2023-09-23T20:41:08.520608+0000","last_fullsized":"2023-09-23T20:41:08.520608+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:34:54.504554+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:34:54.504554+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:42.001652+0000","scrub_duration":0.011176864,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"86'117","reported_seq":691,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:08.381352+0000","last_change":"2023-09-23T20:38:28.446107+0000","last_active":"2023-09-23T20:41:08.381352+0000","last_peered":"2023-09-23T20:41:08.381352+0000","last_clean":"2023-09-23T20:41:08.381352+0000","last_became_active":"2023-09-23T20:38:28.445667+0000","last_became_peered":"2023-09-23T20:38:28.445667+0000","last_unstale":"2023-09-23T20:41:08.381352+0000","last_undegraded":"2023-09-23T20:41:08.381352+0000","last_fullsized":"2023-09-23T20:41:08.381352+0000","mapping_epoch":411,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":412,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:58:47.430360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046891400000000002,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":893,"num_read_kb":922,"num_write":82,"num_write_kb":4193,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":458,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:07.553865+0000","last_change":"2023-09-23T20:32:33.277827+0000","last_active":"2023-09-23T20:41:07.553865+0000","last_peered":"2023-09-23T20:41:07.553865+0000","last_clean":"2023-09-23T20:41:07.553865+0000","last_became_active":"2023-09-23T20:32:33.236087+0000","last_became_peered":"2023-09-23T20:32:33.236087+0000","last_unstale":"2023-09-23T20:41:07.553865+0000","last_undegraded":"2023-09-23T20:41:07.553865+0000","last_fullsized":"2023-09-23T20:41:07.553865+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:20:58.176854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":196,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:08.289896+0000","last_change":"2023-09-23T20:40:19.530602+0000","last_active":"2023-09-23T20:41:08.289896+0000","last_peered":"2023-09-23T20:41:08.289896+0000","last_clean":"2023-09-23T20:41:08.289896+0000","last_became_active":"2023-09-23T20:40:19.530256+0000","last_became_peered":"2023-09-23T20:40:19.530256+0000","last_unstale":"2023-09-23T20:41:08.289896+0000","last_undegraded":"2023-09-23T20:41:08.289896+0000","last_fullsized":"2023-09-23T20:41:08.289896+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:18:45.658867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"90'161","reported_seq":728,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:07.553026+0000","last_change":"2023-09-23T20:38:05.630221+0000","last_active":"2023-09-23T20:41:07.553026+0000","last_peered":"2023-09-23T20:41:07.553026+0000","last_clean":"2023-09-23T20:41:07.553026+0000","last_became_active":"2023-09-23T20:38:05.629839+0000","last_became_peered":"2023-09-23T20:38:05.629839+0000","last_unstale":"2023-09-23T20:41:07.553026+0000","last_undegraded":"2023-09-23T20:41:07.553026+0000","last_fullsized":"2023-09-23T20:41:07.553026+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:06.487351+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:06.487351+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:25:51.325529+0000","scrub_duration":0.001964379,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":426,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:07.553625+0000","last_change":"2023-09-23T20:32:33.237234+0000","last_active":"2023-09-23T20:41:07.553625+0000","last_peered":"2023-09-23T20:41:07.553625+0000","last_clean":"2023-09-23T20:41:07.553625+0000","last_became_active":"2023-09-23T20:32:33.236881+0000","last_became_peered":"2023-09-23T20:32:33.236881+0000","last_unstale":"2023-09-23T20:41:07.553625+0000","last_undegraded":"2023-09-23T20:41:07.553625+0000","last_fullsized":"2023-09-23T20:41:07.553625+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:56:17.384831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":314,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:08.619932+0000","last_change":"2023-09-23T20:29:45.538314+0000","last_active":"2023-09-23T20:41:08.619932+0000","last_peered":"2023-09-23T20:41:08.619932+0000","last_clean":"2023-09-23T20:41:08.619932+0000","last_became_active":"2023-09-23T20:29:45.536847+0000","last_became_peered":"2023-09-23T20:29:45.536847+0000","last_unstale":"2023-09-23T20:41:08.619932+0000","last_undegraded":"2023-09-23T20:41:08.619932+0000","last_fullsized":"2023-09-23T20:41:08.619932+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:51.948721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"90'161","reported_seq":530,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:08.520922+0000","last_change":"2023-09-23T20:41:06.499511+0000","last_active":"2023-09-23T20:41:08.520922+0000","last_peered":"2023-09-23T20:41:08.520922+0000","last_clean":"2023-09-23T20:41:08.520922+0000","last_became_active":"2023-09-23T20:38:05.601874+0000","last_became_peered":"2023-09-23T20:38:05.601874+0000","last_unstale":"2023-09-23T20:41:08.520922+0000","last_undegraded":"2023-09-23T20:41:08.520922+0000","last_fullsized":"2023-09-23T20:41:08.520922+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:41:06.499396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:41:06.499396+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:31:34.059077+0000","scrub_duration":0.0019670310000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":57,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:09.136061+0000","last_change":"2023-09-23T20:38:37.481551+0000","last_active":"2023-09-23T20:41:09.136061+0000","last_peered":"2023-09-23T20:41:09.136061+0000","last_clean":"2023-09-23T20:41:09.136061+0000","last_became_active":"2023-09-23T20:38:37.458687+0000","last_became_peered":"2023-09-23T20:38:37.458687+0000","last_unstale":"2023-09-23T20:41:09.136061+0000","last_undegraded":"2023-09-23T20:41:09.136061+0000","last_fullsized":"2023-09-23T20:41:09.136061+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:22:15.033243+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":115,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:08.290053+0000","last_change":"2023-09-23T20:34:59.987575+0000","last_active":"2023-09-23T20:41:08.290053+0000","last_peered":"2023-09-23T20:41:08.290053+0000","last_clean":"2023-09-23T20:41:08.290053+0000","last_became_active":"2023-09-23T20:34:59.987229+0000","last_became_peered":"2023-09-23T20:34:59.987229+0000","last_unstale":"2023-09-23T20:41:08.290053+0000","last_undegraded":"2023-09-23T20:41:08.290053+0000","last_fullsized":"2023-09-23T20:41:08.290053+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:34:01.820159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":57,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:09.136453+0000","last_change":"2023-09-23T20:38:37.457634+0000","last_active":"2023-09-23T20:41:09.136453+0000","last_peered":"2023-09-23T20:41:09.136453+0000","last_clean":"2023-09-23T20:41:09.136453+0000","last_became_active":"2023-09-23T20:38:37.452208+0000","last_became_peered":"2023-09-23T20:38:37.452208+0000","last_unstale":"2023-09-23T20:41:09.136453+0000","last_undegraded":"2023-09-23T20:41:09.136453+0000","last_fullsized":"2023-09-23T20:41:09.136453+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:09:41.754569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":115,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:08.289720+0000","last_change":"2023-09-23T20:34:59.987416+0000","last_active":"2023-09-23T20:41:08.289720+0000","last_peered":"2023-09-23T20:41:08.289720+0000","last_clean":"2023-09-23T20:41:08.289720+0000","last_became_active":"2023-09-23T20:34:59.987117+0000","last_became_peered":"2023-09-23T20:34:59.987117+0000","last_unstale":"2023-09-23T20:41:08.289720+0000","last_undegraded":"2023-09-23T20:41:08.289720+0000","last_fullsized":"2023-09-23T20:41:08.289720+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:36:16.318623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"90'161","reported_seq":714,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:08.769024+0000","last_change":"2023-09-23T20:38:05.591992+0000","last_active":"2023-09-23T20:41:08.769024+0000","last_peered":"2023-09-23T20:41:08.769024+0000","last_clean":"2023-09-23T20:41:08.769024+0000","last_became_active":"2023-09-23T20:38:05.591701+0000","last_became_peered":"2023-09-23T20:38:05.591701+0000","last_unstale":"2023-09-23T20:41:08.769024+0000","last_undegraded":"2023-09-23T20:41:08.769024+0000","last_fullsized":"2023-09-23T20:41:08.769024+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:02.384133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:02.384133+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:41.722542+0000","scrub_duration":0.001769098,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":279,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:08.521032+0000","last_change":"2023-09-23T20:38:02.542719+0000","last_active":"2023-09-23T20:41:08.521032+0000","last_peered":"2023-09-23T20:41:08.521032+0000","last_clean":"2023-09-23T20:41:08.521032+0000","last_became_active":"2023-09-23T20:38:02.542385+0000","last_became_peered":"2023-09-23T20:38:02.542385+0000","last_unstale":"2023-09-23T20:41:08.521032+0000","last_undegraded":"2023-09-23T20:41:08.521032+0000","last_fullsized":"2023-09-23T20:41:08.521032+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:36:23.673757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":246,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:08.520519+0000","last_change":"2023-09-23T20:37:43.584253+0000","last_active":"2023-09-23T20:41:08.520519+0000","last_peered":"2023-09-23T20:41:08.520519+0000","last_clean":"2023-09-23T20:41:08.520519+0000","last_became_active":"2023-09-23T20:37:43.580069+0000","last_became_peered":"2023-09-23T20:37:43.580069+0000","last_unstale":"2023-09-23T20:41:08.520519+0000","last_undegraded":"2023-09-23T20:41:08.520519+0000","last_fullsized":"2023-09-23T20:41:08.520519+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:39:11.482581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"90'161","reported_seq":551,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:09.132673+0000","last_change":"2023-09-23T20:36:01.505098+0000","last_active":"2023-09-23T20:41:09.132673+0000","last_peered":"2023-09-23T20:41:09.132673+0000","last_clean":"2023-09-23T20:41:09.132673+0000","last_became_active":"2023-09-23T20:35:26.644707+0000","last_became_peered":"2023-09-23T20:35:26.644707+0000","last_unstale":"2023-09-23T20:41:09.132673+0000","last_undegraded":"2023-09-23T20:41:09.132673+0000","last_fullsized":"2023-09-23T20:41:09.132673+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:01.504973+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:01.504973+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:02:10.880809+0000","scrub_duration":0.00202344,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":504,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:07.554120+0000","last_change":"2023-09-23T20:40:19.569032+0000","last_active":"2023-09-23T20:41:07.554120+0000","last_peered":"2023-09-23T20:41:07.554120+0000","last_clean":"2023-09-23T20:41:07.554120+0000","last_became_active":"2023-09-23T20:40:19.538476+0000","last_became_peered":"2023-09-23T20:40:19.538476+0000","last_unstale":"2023-09-23T20:41:07.554120+0000","last_undegraded":"2023-09-23T20:41:07.554120+0000","last_fullsized":"2023-09-23T20:41:07.554120+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:31:58.998168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3324,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":176,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:08.289772+0000","last_change":"2023-09-23T20:38:02.538333+0000","last_active":"2023-09-23T20:41:08.289772+0000","last_peered":"2023-09-23T20:41:08.289772+0000","last_clean":"2023-09-23T20:41:08.289772+0000","last_became_active":"2023-09-23T20:38:02.538008+0000","last_became_peered":"2023-09-23T20:38:02.538008+0000","last_unstale":"2023-09-23T20:41:08.289772+0000","last_undegraded":"2023-09-23T20:41:08.289772+0000","last_fullsized":"2023-09-23T20:41:08.289772+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:02:38.789530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":80,"num_bytes_recovered":18486,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26691,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":381350,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":423698,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":1741946880,"num_objects":26580,"num_object_clones":0,"num_object_copies":79740,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26580,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":26580,"num_write_kb":1701120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":5133172736,"data_stored":5133172736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":26580,"ondisk_log_size":26580,"up":48,"acting":48,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5983,"num_read_kb":13200,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":68,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6082,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201638,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1130692,"kb_used_data":750460,"kb_used_omap":0,"kb_used_meta":380160,"kb_avail":92614460,"statfs":{"total":95995035648,"available":94837207040,"internally_reserved":0,"allocated":768471040,"data_stored":767022953,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":389283840},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:40:54 2023","interfaces":[{"interface":"back","average":{"1min":1.423,"5min":1.968,"15min":2.1520000000000001},"min":{"1min":0.27200000000000002,"5min":0.24399999999999999,"15min":0.20799999999999999},"max":{"1min":5.6310000000000002,"5min":18.315000000000001,"15min":48.798999999999999},"last":0.28100000000000003},{"interface":"front","average":{"1min":1.069,"5min":1.2909999999999999,"15min":1.974},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":5.3330000000000002,"5min":9.4429999999999996,"15min":48.247},"last":1.476}]},{"osd":1,"last update":"Sat Sep 23 20:41:06 2023","interfaces":[{"interface":"back","average":{"1min":0.91400000000000003,"5min":2.2770000000000001,"15min":2.0960000000000001},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.254},"max":{"1min":4.2460000000000004,"5min":49.965000000000003,"15min":49.965000000000003},"last":0.84499999999999997},{"interface":"front","average":{"1min":1.147,"5min":2.323,"15min":2.1160000000000001},"min":{"1min":0.29799999999999999,"5min":0.26900000000000002,"15min":0.248},"max":{"1min":4.2999999999999998,"5min":53.451999999999998,"15min":53.451999999999998},"last":1.0840000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:40:54 2023","interfaces":[{"interface":"back","average":{"1min":1.339,"5min":1.736,"15min":2.0840000000000001},"min":{"1min":0.32000000000000001,"5min":0.28599999999999998,"15min":0.27600000000000002},"max":{"1min":4.5039999999999996,"5min":16.965,"15min":16.965},"last":1.3779999999999999},{"interface":"front","average":{"1min":1.4630000000000001,"5min":2.1960000000000002,"15min":2.4489999999999998},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.26600000000000001},"max":{"1min":6.4329999999999998,"5min":23.414000000000001,"15min":23.494},"last":1.232}]},{"osd":3,"last update":"Sat Sep 23 20:40:21 2023","interfaces":[{"interface":"back","average":{"1min":1.105,"5min":2.2480000000000002,"15min":2.4470000000000001},"min":{"1min":0.35999999999999999,"5min":0.311,"15min":0.26200000000000001},"max":{"1min":4.2149999999999999,"5min":23.352,"15min":23.352},"last":1.1990000000000001},{"interface":"front","average":{"1min":1.139,"5min":2.2709999999999999,"15min":2.423},"min":{"1min":0.46500000000000002,"5min":0.34399999999999997,"15min":0.32500000000000001},"max":{"1min":4.2359999999999998,"5min":20.969999999999999,"15min":20.969999999999999},"last":0.98999999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:40:21 2023","interfaces":[{"interface":"back","average":{"1min":1.046,"5min":1.085,"15min":1.2170000000000001},"min":{"1min":0.35799999999999998,"5min":0.25600000000000001,"15min":0.23200000000000001},"max":{"1min":4.1639999999999997,"5min":18.364999999999998,"15min":18.364999999999998},"last":0.40000000000000002},{"interface":"front","average":{"1min":0.94999999999999996,"5min":1.296,"15min":1.417},"min":{"1min":0.39100000000000001,"5min":0.28399999999999997,"15min":0.27600000000000002},"max":{"1min":2.605,"5min":23.617000000000001,"15min":23.617000000000001},"last":1.339}]},{"osd":5,"last update":"Sat Sep 23 20:40:21 2023","interfaces":[{"interface":"back","average":{"1min":1.0780000000000001,"5min":1.1499999999999999,"15min":1.2509999999999999},"min":{"1min":0.30099999999999999,"5min":0.29199999999999998,"15min":0.26600000000000001},"max":{"1min":4.3780000000000001,"5min":12.516,"15min":13.241},"last":1.133},{"interface":"front","average":{"1min":0.76400000000000001,"5min":1.286,"15min":1.3540000000000001},"min":{"1min":0.38400000000000001,"5min":0.31900000000000001,"15min":0.315},"max":{"1min":3.1030000000000002,"5min":18.390000000000001,"15min":18.831},"last":0.95599999999999996}]},{"osd":6,"last update":"Sat Sep 23 20:40:21 2023","interfaces":[{"interface":"back","average":{"1min":1.2629999999999999,"5min":1.4350000000000001,"15min":1.329},"min":{"1min":0.44,"5min":0.312,"15min":0.30399999999999999},"max":{"1min":4.0540000000000003,"5min":18.417000000000002,"15min":18.417000000000002},"last":0.495},{"interface":"front","average":{"1min":1.1659999999999999,"5min":1.113,"15min":1.252},"min":{"1min":0.38200000000000001,"5min":0.32700000000000001,"15min":0.318},"max":{"1min":6.4340000000000002,"5min":7.266,"15min":9.9120000000000008},"last":1.026}]}]},{"osd":6,"up_from":47,"seq":201863463277,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":693496,"kb_used_data":298800,"kb_used_omap":0,"kb_used_meta":394688,"kb_avail":93051656,"statfs":{"total":95995035648,"available":95284895744,"internally_reserved":0,"allocated":305971200,"data_stored":304534730,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":404160512},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:40:50 2023","interfaces":[{"interface":"back","average":{"1min":1.1910000000000001,"5min":1.7470000000000001,"15min":1.9159999999999999},"min":{"1min":0.28100000000000003,"5min":0.23300000000000001,"15min":0.221},"max":{"1min":4.1399999999999997,"5min":17.492999999999999,"15min":53.701999999999998},"last":0.30199999999999999},{"interface":"front","average":{"1min":1.395,"5min":2.306,"15min":2.3399999999999999},"min":{"1min":0.253,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":6.0570000000000004,"5min":60.328000000000003,"15min":60.328000000000003},"last":1.623}]},{"osd":1,"last update":"Sat Sep 23 20:41:09 2023","interfaces":[{"interface":"back","average":{"1min":0.66200000000000003,"5min":1.726,"15min":2.222},"min":{"1min":0.253,"5min":0.253,"15min":0.249},"max":{"1min":2.8250000000000002,"5min":64.274000000000001,"15min":64.274000000000001},"last":3.4780000000000002},{"interface":"front","average":{"1min":1.173,"5min":1.2170000000000001,"15min":2.3290000000000002},"min":{"1min":0.32900000000000001,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":4.29,"5min":6.8460000000000001,"15min":34.490000000000002},"last":1.611}]},{"osd":2,"last update":"Sat Sep 23 20:40:50 2023","interfaces":[{"interface":"back","average":{"1min":2.097,"5min":1.6419999999999999,"15min":2.133},"min":{"1min":0.31,"5min":0.30299999999999999,"15min":0.27700000000000002},"max":{"1min":9.9049999999999994,"5min":13.010999999999999,"15min":34.317999999999998},"last":1.329},{"interface":"front","average":{"1min":2.1080000000000001,"5min":1.585,"15min":2.0569999999999999},"min":{"1min":0.311,"5min":0.311,"15min":0.30399999999999999},"max":{"1min":9.9269999999999996,"5min":12.917999999999999,"15min":34.292999999999999},"last":1.3480000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:40:36 2023","interfaces":[{"interface":"back","average":{"1min":2.0099999999999998,"5min":1.8759999999999999,"15min":1.9910000000000001},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":5.9130000000000003,"5min":8.9670000000000005,"15min":9.4239999999999995},"last":0.89100000000000001},{"interface":"front","average":{"1min":1.921,"5min":1.968,"15min":2.1659999999999999},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.317},"max":{"1min":5.8380000000000001,"5min":11.913,"15min":12.875999999999999},"last":0.57599999999999996}]},{"osd":4,"last update":"Sat Sep 23 20:41:14 2023","interfaces":[{"interface":"back","average":{"1min":0.63800000000000001,"5min":0.85499999999999998,"15min":0.83599999999999997},"min":{"1min":0.30299999999999999,"5min":0.28000000000000003,"15min":0.251},"max":{"1min":4.0830000000000002,"5min":23.904,"15min":23.904},"last":0.61599999999999999},{"interface":"front","average":{"1min":0.68500000000000005,"5min":1.0780000000000001,"15min":1.1140000000000001},"min":{"1min":0.309,"5min":0.29199999999999998,"15min":0.27000000000000002},"max":{"1min":2.7160000000000002,"5min":11.515000000000001,"15min":11.515000000000001},"last":0.49299999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:41:14 2023","interfaces":[{"interface":"back","average":{"1min":0.45800000000000002,"5min":0.76900000000000002,"15min":0.80900000000000005},"min":{"1min":0.34000000000000002,"5min":0.318,"15min":0.28699999999999998},"max":{"1min":0.78500000000000003,"5min":6.6340000000000003,"15min":8.4499999999999993},"last":0.54200000000000004},{"interface":"front","average":{"1min":0.76600000000000001,"5min":0.95699999999999996,"15min":1.179},"min":{"1min":0.308,"5min":0.308,"15min":0.30599999999999999},"max":{"1min":4.7910000000000004,"5min":6.7320000000000002,"15min":18.192},"last":0.60499999999999998}]},{"osd":7,"last update":"Sat Sep 23 20:40:36 2023","interfaces":[{"interface":"back","average":{"1min":1.1240000000000001,"5min":0.88300000000000001,"15min":1.121},"min":{"1min":0.40400000000000003,"5min":0.376,"15min":0.28599999999999998},"max":{"1min":5.2729999999999997,"5min":5.2729999999999997,"15min":10.217000000000001},"last":0.58599999999999997},{"interface":"front","average":{"1min":0.97899999999999998,"5min":1.048,"15min":1.3120000000000001},"min":{"1min":0.45700000000000002,"5min":0.36799999999999999,"15min":0.30299999999999999},"max":{"1min":2.2909999999999999,"5min":8.8119999999999994,"15min":9.7360000000000007},"last":0.51900000000000002}]}]},{"osd":1,"up_from":266,"seq":1142461300876,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1411900,"kb_used_data":957620,"kb_used_omap":8,"kb_used_meta":454263,"kb_avail":92333252,"statfs":{"total":95995035648,"available":94549250048,"internally_reserved":0,"allocated":980602880,"data_stored":978117919,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8206,"internal_metadata":465166322},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:41:07 2023","interfaces":[{"interface":"back","average":{"1min":2.1469999999999998,"5min":2.0030000000000001,"15min":2.4980000000000002},"min":{"1min":0.215,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":16.292000000000002,"5min":16.292000000000002,"15min":52.689999999999998},"last":0.86899999999999999},{"interface":"front","average":{"1min":2.1419999999999999,"5min":2.6560000000000001,"15min":3.1949999999999998},"min":{"1min":0.20100000000000001,"5min":0.20100000000000001,"15min":0.19900000000000001},"max":{"1min":16.332999999999998,"5min":17.169,"15min":52.662999999999997},"last":0.85199999999999998}]},{"osd":2,"last update":"Sat Sep 23 20:41:07 2023","interfaces":[{"interface":"back","average":{"1min":1.8220000000000001,"5min":2.5139999999999998,"15min":3.8610000000000002},"min":{"1min":0.216,"5min":0.216,"15min":0.20899999999999999},"max":{"1min":10.404,"5min":32.255000000000003,"15min":32.255000000000003},"last":1.1719999999999999},{"interface":"front","average":{"1min":1.611,"5min":1.9610000000000001,"15min":2.2839999999999998},"min":{"1min":0.24399999999999999,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":5.9699999999999998,"5min":13.404999999999999,"15min":13.404999999999999},"last":0.67300000000000004}]},{"osd":3,"last update":"Sat Sep 23 20:41:07 2023","interfaces":[{"interface":"back","average":{"1min":1.7929999999999999,"5min":1.8460000000000001,"15min":3.4039999999999999},"min":{"1min":0.33800000000000002,"5min":0.247,"15min":0.247},"max":{"1min":16.245999999999999,"5min":16.245999999999999,"15min":80.745999999999995},"last":1.651},{"interface":"front","average":{"1min":2.343,"5min":2.202,"15min":3.7650000000000001},"min":{"1min":0.34300000000000003,"5min":0.27500000000000002,"15min":0.22700000000000001},"max":{"1min":13.497999999999999,"5min":14.048999999999999,"15min":82.988},"last":1.012}]},{"osd":4,"last update":"Sat Sep 23 20:41:07 2023","interfaces":[{"interface":"back","average":{"1min":0.96899999999999997,"5min":1.75,"15min":2.1349999999999998},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30299999999999999},"max":{"1min":3.149,"5min":7.8399999999999999,"15min":24.327999999999999},"last":0.34899999999999998},{"interface":"front","average":{"1min":1.1240000000000001,"5min":1.3300000000000001,"15min":1.704},"min":{"1min":0.32600000000000001,"5min":0.311,"15min":0.311},"max":{"1min":4.3319999999999999,"5min":9.3339999999999996,"15min":11.574999999999999},"last":0.374}]},{"osd":5,"last update":"Sat Sep 23 20:41:07 2023","interfaces":[{"interface":"back","average":{"1min":2.02,"5min":2.129,"15min":2.3199999999999998},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.33200000000000002},"max":{"1min":13.550000000000001,"5min":33.401000000000003,"15min":33.401000000000003},"last":0.89600000000000002},{"interface":"front","average":{"1min":1.008,"5min":2.2069999999999999,"15min":2.355},"min":{"1min":0.377,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":3.238,"5min":33.417000000000002,"15min":33.417000000000002},"last":0.45000000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:41:07 2023","interfaces":[{"interface":"back","average":{"1min":2.0329999999999999,"5min":2.0049999999999999,"15min":3.8090000000000002},"min":{"1min":0.43099999999999999,"5min":0.42599999999999999,"15min":0.35699999999999998},"max":{"1min":13.593999999999999,"5min":13.593999999999999,"15min":72.682000000000002},"last":0.97799999999999998},{"interface":"front","average":{"1min":1.0669999999999999,"5min":1.7789999999999999,"15min":3.6869999999999998},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":3.2050000000000001,"5min":7.883,"15min":72.563999999999993},"last":0.40000000000000002}]},{"osd":7,"last update":"Sat Sep 23 20:41:07 2023","interfaces":[{"interface":"back","average":{"1min":2.1970000000000001,"5min":1.97,"15min":2.5030000000000001},"min":{"1min":0.40300000000000002,"5min":0.39500000000000002,"15min":0.39300000000000002},"max":{"1min":16.359000000000002,"5min":16.359000000000002,"15min":18.437000000000001},"last":0.94299999999999995},{"interface":"front","average":{"1min":1.1830000000000001,"5min":1.8859999999999999,"15min":2.3809999999999998},"min":{"1min":0.44800000000000001,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":3.27,"5min":7.9089999999999998,"15min":24.524000000000001},"last":0.42699999999999999}]}]},{"osd":0,"up_from":94,"seq":403726926079,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1107796,"kb_used_data":470988,"kb_used_omap":7,"kb_used_meta":636792,"kb_avail":92637356,"statfs":{"total":95995035648,"available":94860652544,"internally_reserved":0,"allocated":482291712,"data_stored":479960425,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":652075025},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:41:16 2023","interfaces":[{"interface":"back","average":{"1min":2.9329999999999998,"5min":1.8460000000000001,"15min":3.0049999999999999},"min":{"1min":0.24299999999999999,"5min":0.17199999999999999,"15min":0.17199999999999999},"max":{"1min":24.010000000000002,"5min":24.010000000000002,"15min":42.000999999999998},"last":1.3560000000000001},{"interface":"front","average":{"1min":2.7469999999999999,"5min":1.9359999999999999,"15min":3.2229999999999999},"min":{"1min":0.29499999999999998,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":14.279999999999999,"5min":14.279999999999999,"15min":23.036000000000001},"last":1.379}]},{"osd":2,"last update":"Sat Sep 23 20:40:50 2023","interfaces":[{"interface":"back","average":{"1min":3.0609999999999999,"5min":2.657,"15min":2.9849999999999999},"min":{"1min":0.29199999999999998,"5min":0.218,"15min":0.185},"max":{"1min":13.871,"5min":62.726999999999997,"15min":77.448999999999998},"last":0.29799999999999999},{"interface":"front","average":{"1min":2.6600000000000001,"5min":2.875,"15min":2.8919999999999999},"min":{"1min":0.27800000000000002,"5min":0.247,"15min":0.188},"max":{"1min":13.356999999999999,"5min":62.689,"15min":81.724999999999994},"last":0.34599999999999997}]},{"osd":3,"last update":"Sat Sep 23 20:40:23 2023","interfaces":[{"interface":"back","average":{"1min":2.399,"5min":2.7450000000000001,"15min":3.133},"min":{"1min":0.34699999999999998,"5min":0.24399999999999999,"15min":0.23400000000000001},"max":{"1min":9.0039999999999996,"5min":51.536000000000001,"15min":81.400999999999996},"last":2.1520000000000001},{"interface":"front","average":{"1min":2.8839999999999999,"5min":3.1389999999999998,"15min":3.4590000000000001},"min":{"1min":0.245,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":14.237,"5min":44.811999999999998,"15min":81.137},"last":2.0470000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:40:50 2023","interfaces":[{"interface":"back","average":{"1min":2.7970000000000002,"5min":1.9019999999999999,"15min":2.1920000000000002},"min":{"1min":0.432,"5min":0.28899999999999998,"15min":0.25900000000000001},"max":{"1min":6.8970000000000002,"5min":9.6110000000000007,"15min":23.876999999999999},"last":0.40799999999999997},{"interface":"front","average":{"1min":2.9929999999999999,"5min":1.615,"15min":1.9770000000000001},"min":{"1min":0.36299999999999999,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":9.7789999999999999,"5min":9.7789999999999999,"15min":23.681999999999999},"last":1.8779999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:40:50 2023","interfaces":[{"interface":"back","average":{"1min":3.0649999999999999,"5min":1.6519999999999999,"15min":2.1819999999999999},"min":{"1min":0.41499999999999998,"5min":0.371,"15min":0.34200000000000003},"max":{"1min":9.8629999999999995,"5min":9.8629999999999995,"15min":24.198},"last":1.9470000000000001},{"interface":"front","average":{"1min":2.8439999999999999,"5min":2.4449999999999998,"15min":2.3170000000000002},"min":{"1min":0.38900000000000001,"5min":0.318,"15min":0.25600000000000001},"max":{"1min":12.929,"5min":26.228000000000002,"15min":26.228000000000002},"last":0.42299999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:40:50 2023","interfaces":[{"interface":"back","average":{"1min":3.6520000000000001,"5min":1.9650000000000001,"15min":2.2549999999999999},"min":{"1min":0.39800000000000002,"5min":0.34499999999999997,"15min":0.318},"max":{"1min":24.056000000000001,"5min":24.056000000000001,"15min":24.408000000000001},"last":1.9139999999999999},{"interface":"front","average":{"1min":2.1219999999999999,"5min":2.2770000000000001,"15min":2.282},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.29399999999999998},"max":{"1min":8.0549999999999997,"5min":26.268000000000001,"15min":26.268000000000001},"last":0.52600000000000002}]},{"osd":7,"last update":"Sat Sep 23 20:40:50 2023","interfaces":[{"interface":"back","average":{"1min":2.4239999999999999,"5min":2.3180000000000001,"15min":2.4220000000000002},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.30599999999999999},"max":{"1min":8.1189999999999998,"5min":26.177,"15min":26.177},"last":0.57399999999999995},{"interface":"front","average":{"1min":2.9910000000000001,"5min":1.966,"15min":2.569},"min":{"1min":0.53700000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":6.8979999999999997,"5min":9.7200000000000006,"15min":24.669},"last":0.55200000000000005}]}]},{"osd":2,"up_from":104,"seq":446676599026,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":953808,"kb_used_data":280136,"kb_used_omap":11,"kb_used_meta":673652,"kb_avail":92791344,"statfs":{"total":95995035648,"available":95018336256,"internally_reserved":0,"allocated":286859264,"data_stored":284529815,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11878,"internal_metadata":689820058},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:41:03 2023","interfaces":[{"interface":"back","average":{"1min":0.97199999999999998,"5min":1.327,"15min":2.617},"min":{"1min":0.25,"5min":0.20300000000000001,"15min":0.17999999999999999},"max":{"1min":5.875,"5min":12.007,"15min":82.424999999999997},"last":0.53300000000000003},{"interface":"front","average":{"1min":1.583,"5min":1.7949999999999999,"15min":2.6339999999999999},"min":{"1min":0.27000000000000002,"5min":0.19,"15min":0.19},"max":{"1min":9.2379999999999995,"5min":9.5600000000000005,"15min":82.524000000000001},"last":0.32500000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:41:03 2023","interfaces":[{"interface":"back","average":{"1min":1.234,"5min":2.4780000000000002,"15min":3.3820000000000001},"min":{"1min":0.27100000000000002,"5min":0.245,"15min":0.221},"max":{"1min":7.8250000000000002,"5min":46.405000000000001,"15min":46.405000000000001},"last":1.4119999999999999},{"interface":"front","average":{"1min":1.105,"5min":1.9430000000000001,"15min":2.7749999999999999},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":5.5270000000000001,"5min":46.459000000000003,"15min":46.459000000000003},"last":2.7480000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:40:22 2023","interfaces":[{"interface":"back","average":{"1min":2.851,"5min":2.613,"15min":2.4769999999999999},"min":{"1min":0.28499999999999998,"5min":0.27100000000000002,"15min":0.19800000000000001},"max":{"1min":11.304,"5min":19.562999999999999,"15min":19.562999999999999},"last":1.2030000000000001},{"interface":"front","average":{"1min":1.7909999999999999,"5min":1.6160000000000001,"15min":2.5089999999999999},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.23899999999999999},"max":{"1min":8.9369999999999994,"5min":13.148999999999999,"15min":30.704999999999998},"last":0.58499999999999996}]},{"osd":4,"last update":"Sat Sep 23 20:41:03 2023","interfaces":[{"interface":"back","average":{"1min":1.345,"5min":1.4399999999999999,"15min":1.917},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.27500000000000002},"max":{"1min":5.6150000000000002,"5min":6.6849999999999996,"15min":16.797999999999998},"last":0.61199999999999999},{"interface":"front","average":{"1min":1.1799999999999999,"5min":1.601,"15min":2.0899999999999999},"min":{"1min":0.379,"5min":0.379,"15min":0.27600000000000002},"max":{"1min":3.758,"5min":19.440999999999999,"15min":19.440999999999999},"last":0.41599999999999998}]},{"osd":5,"last update":"Sat Sep 23 20:41:03 2023","interfaces":[{"interface":"back","average":{"1min":1.4730000000000001,"5min":1.488,"15min":2.403},"min":{"1min":0.43099999999999999,"5min":0.36599999999999999,"15min":0.309},"max":{"1min":5.6580000000000004,"5min":6.6319999999999997,"15min":95.393000000000001},"last":0.56799999999999995},{"interface":"front","average":{"1min":1.1419999999999999,"5min":1.613,"15min":2.6669999999999998},"min":{"1min":0.44800000000000001,"5min":0.41399999999999998,"15min":0.32300000000000001},"max":{"1min":3.8279999999999998,"5min":19.513000000000002,"15min":95.414000000000001},"last":0.47899999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:41:03 2023","interfaces":[{"interface":"back","average":{"1min":1.1279999999999999,"5min":1.585,"15min":2.177},"min":{"1min":0.432,"5min":0.42399999999999999,"15min":0.32100000000000001},"max":{"1min":3.7949999999999999,"5min":19.486000000000001,"15min":19.486000000000001},"last":0.51500000000000001},{"interface":"front","average":{"1min":0.94199999999999995,"5min":1.05,"15min":1.9259999999999999},"min":{"1min":0.42099999999999999,"5min":0.36899999999999999,"15min":0.316},"max":{"1min":5.5789999999999997,"5min":9.3010000000000002,"15min":30.670999999999999},"last":0.68500000000000005}]},{"osd":7,"last update":"Sat Sep 23 20:41:03 2023","interfaces":[{"interface":"back","average":{"1min":1.2030000000000001,"5min":1.038,"15min":1.9650000000000001},"min":{"1min":0.47399999999999998,"5min":0.39300000000000002,"15min":0.35199999999999998},"max":{"1min":7.5860000000000003,"5min":7.5860000000000003,"15min":30.741},"last":0.64100000000000001},{"interface":"front","average":{"1min":1.831,"5min":1.6870000000000001,"15min":2.1739999999999999},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":5.8529999999999998,"5min":6.7220000000000004,"15min":16.719000000000001},"last":0.66500000000000004}]}]},{"osd":3,"up_from":109,"seq":468151435500,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1635324,"kb_used_data":802292,"kb_used_omap":4,"kb_used_meta":833019,"kb_avail":92109828,"statfs":{"total":95995035648,"available":94320463872,"internally_reserved":0,"allocated":821547008,"data_stored":819234360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4514,"internal_metadata":853012062},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:40:26 2023","interfaces":[{"interface":"back","average":{"1min":1.409,"5min":1.7350000000000001,"15min":2.544},"min":{"1min":0.21199999999999999,"5min":0.21199999999999999,"15min":0.186},"max":{"1min":7.6879999999999997,"5min":14.638,"15min":67.305999999999997},"last":0.24299999999999999},{"interface":"front","average":{"1min":2.4540000000000002,"5min":1.865,"15min":2.415},"min":{"1min":0.29499999999999998,"5min":0.26600000000000001,"15min":0.19500000000000001},"max":{"1min":8.8439999999999994,"5min":10.140000000000001,"15min":67.403999999999996},"last":0.29399999999999998}]},{"osd":1,"last update":"Sat Sep 23 20:41:05 2023","interfaces":[{"interface":"back","average":{"1min":1.244,"5min":1.212,"15min":2.2519999999999998},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":4.8099999999999996,"5min":9.5299999999999994,"15min":16.189},"last":0.33800000000000002},{"interface":"front","average":{"1min":2.262,"5min":2.0369999999999999,"15min":2.617},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":8.8030000000000008,"5min":12.702999999999999,"15min":32.603999999999999},"last":0.26500000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:40:26 2023","interfaces":[{"interface":"back","average":{"1min":2.3140000000000001,"5min":1.837,"15min":2.5030000000000001},"min":{"1min":0.26600000000000001,"5min":0.246,"15min":0.22800000000000001},"max":{"1min":6.3200000000000003,"5min":10.747999999999999,"15min":32.857999999999997},"last":1.05},{"interface":"front","average":{"1min":1.516,"5min":2.0259999999999998,"15min":2.4980000000000002},"min":{"1min":0.31,"5min":0.28100000000000003,"15min":0.20899999999999999},"max":{"1min":8.5530000000000008,"5min":22.32,"15min":24.146999999999998},"last":0.628}]},{"osd":4,"last update":"Sat Sep 23 20:40:26 2023","interfaces":[{"interface":"back","average":{"1min":0.82199999999999995,"5min":1.0860000000000001,"15min":1.786},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.29799999999999999},"max":{"1min":2.5899999999999999,"5min":9.3160000000000007,"15min":46.042999999999999},"last":0.55700000000000005},{"interface":"front","average":{"1min":1.6359999999999999,"5min":1.879,"15min":2.4340000000000002},"min":{"1min":0.33200000000000002,"5min":0.33000000000000002,"15min":0.28100000000000003},"max":{"1min":5.141,"5min":13.571999999999999,"15min":46.579999999999998},"last":0.5}]},{"osd":5,"last update":"Sat Sep 23 20:40:26 2023","interfaces":[{"interface":"back","average":{"1min":0.82099999999999995,"5min":1.167,"15min":1.998},"min":{"1min":0.32700000000000001,"5min":0.32300000000000001,"15min":0.31900000000000001},"max":{"1min":2.9889999999999999,"5min":9.4130000000000003,"15min":45.292000000000002},"last":0.48099999999999998},{"interface":"front","average":{"1min":1.6859999999999999,"5min":1.482,"15min":1.968},"min":{"1min":0.38500000000000001,"5min":0.34399999999999997,"15min":0.313},"max":{"1min":5.0060000000000002,"5min":9.0839999999999996,"15min":44.679000000000002},"last":0.44}]},{"osd":6,"last update":"Sat Sep 23 20:40:26 2023","interfaces":[{"interface":"back","average":{"1min":1.8640000000000001,"5min":1.3959999999999999,"15min":1.964},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":7.3630000000000004,"5min":9.0470000000000006,"15min":17.524999999999999},"last":0.46500000000000002},{"interface":"front","average":{"1min":1.4750000000000001,"5min":1.8839999999999999,"15min":2.165},"min":{"1min":0.434,"5min":0.39800000000000002,"15min":0.33700000000000002},"max":{"1min":4.8819999999999997,"5min":13.500999999999999,"15min":19.524000000000001},"last":0.53600000000000003}]},{"osd":7,"last update":"Sat Sep 23 20:40:26 2023","interfaces":[{"interface":"back","average":{"1min":1.5109999999999999,"5min":2.1829999999999998,"15min":2.3959999999999999},"min":{"1min":0.433,"5min":0.39900000000000002,"15min":0.34999999999999998},"max":{"1min":4.319,"5min":13.544,"15min":19.571999999999999},"last":0.45100000000000001},{"interface":"front","average":{"1min":0.996,"5min":1.274,"15min":2.0720000000000001},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.371},"max":{"1min":4.157,"5min":9.3719999999999999,"15min":45.387999999999998},"last":0.52200000000000002}]}]},{"osd":4,"up_from":30,"seq":128849019254,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":717944,"kb_used_data":343792,"kb_used_omap":0,"kb_used_meta":374144,"kb_avail":93027208,"statfs":{"total":95995035648,"available":95259860992,"internally_reserved":0,"allocated":352043008,"data_stored":350606272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":383123456},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:40:49 2023","interfaces":[{"interface":"back","average":{"1min":2.1949999999999998,"5min":1.6739999999999999,"15min":1.724},"min":{"1min":0.31,"5min":0.24099999999999999,"15min":0.23699999999999999},"max":{"1min":11.553000000000001,"5min":24.114999999999998,"15min":24.114999999999998},"last":0.55800000000000005},{"interface":"front","average":{"1min":2.2789999999999999,"5min":1.8080000000000001,"15min":1.895},"min":{"1min":0.28899999999999998,"5min":0.246,"15min":0.246},"max":{"1min":11.536,"5min":24.135999999999999,"15min":24.135999999999999},"last":0.90700000000000003}]},{"osd":1,"last update":"Sat Sep 23 20:41:10 2023","interfaces":[{"interface":"back","average":{"1min":1.2589999999999999,"5min":1.27,"15min":1.6910000000000001},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":10.363,"5min":18.190999999999999,"15min":18.190999999999999},"last":0.51400000000000001},{"interface":"front","average":{"1min":1.365,"5min":1.3420000000000001,"15min":1.647},"min":{"1min":0.33400000000000002,"5min":0.28499999999999998,"15min":0.28199999999999997},"max":{"1min":10.426,"5min":18.173999999999999,"15min":18.173999999999999},"last":0.58799999999999997}]},{"osd":2,"last update":"Sat Sep 23 20:41:05 2023","interfaces":[{"interface":"back","average":{"1min":0.99099999999999999,"5min":1.264,"15min":1.821},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.26600000000000001},"max":{"1min":4.4640000000000004,"5min":7.75,"15min":32.363},"last":3.3050000000000002},{"interface":"front","average":{"1min":0.82899999999999996,"5min":1.6779999999999999,"15min":2.2120000000000002},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.27600000000000002},"max":{"1min":2.4590000000000001,"5min":11.917999999999999,"15min":34.042999999999999},"last":2.6960000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:40:17 2023","interfaces":[{"interface":"back","average":{"1min":2.4209999999999998,"5min":1.8480000000000001,"15min":2.113},"min":{"1min":0.40500000000000003,"5min":0.372,"15min":0.29099999999999998},"max":{"1min":12.147,"5min":12.147,"15min":32.332999999999998},"last":0.61799999999999999},{"interface":"front","average":{"1min":1.8280000000000001,"5min":1.8360000000000001,"15min":2.206},"min":{"1min":0.371,"5min":0.371,"15min":0.30299999999999999},"max":{"1min":11.914,"5min":11.914,"15min":32.399999999999999},"last":1.0720000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:40:47 2023","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.68200000000000005,"15min":1.51},"min":{"1min":0.28100000000000003,"5min":0.26400000000000001,"15min":0.247},"max":{"1min":2.202,"5min":4.1260000000000003,"15min":82.866},"last":0.54800000000000004},{"interface":"front","average":{"1min":1.006,"5min":0.96999999999999997,"15min":1.4810000000000001},"min":{"1min":0.32200000000000001,"5min":0.28999999999999998,"15min":0.27800000000000002},"max":{"1min":5.2670000000000003,"5min":7.0129999999999999,"15min":82.546000000000006},"last":0.96799999999999997}]},{"osd":6,"last update":"Sat Sep 23 20:41:05 2023","interfaces":[{"interface":"back","average":{"1min":0.67300000000000004,"5min":0.96299999999999997,"15min":1.258},"min":{"1min":0.35799999999999998,"5min":0.32200000000000001,"15min":0.27600000000000002},"max":{"1min":2.7450000000000001,"5min":14.731999999999999,"15min":15.252000000000001},"last":0.93999999999999995},{"interface":"front","average":{"1min":0.64500000000000002,"5min":0.88200000000000001,"15min":1.1240000000000001},"min":{"1min":0.314,"5min":0.309,"15min":0.27200000000000002},"max":{"1min":2.5539999999999998,"5min":7.2850000000000001,"15min":11.199},"last":0.83899999999999997}]},{"osd":7,"last update":"Sat Sep 23 20:40:17 2023","interfaces":[{"interface":"back","average":{"1min":1.0389999999999999,"5min":1.143,"15min":1.5369999999999999},"min":{"1min":0.39100000000000001,"5min":0.318,"15min":0.318},"max":{"1min":4.21,"5min":14.534000000000001,"15min":16.427},"last":1.732},{"interface":"front","average":{"1min":1.472,"5min":0.84599999999999997,"15min":1.3180000000000001},"min":{"1min":0.36399999999999999,"5min":0.32700000000000001,"15min":0.307},"max":{"1min":7.9390000000000001,"5min":7.9390000000000001,"15min":19.018999999999998},"last":0.755}]}]},{"osd":5,"up_from":39,"seq":167503724915,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2136820,"kb_used_data":1723756,"kb_used_omap":0,"kb_used_meta":413056,"kb_avail":91608332,"statfs":{"total":95995035648,"available":93806931968,"internally_reserved":0,"allocated":1765126144,"data_stored":1763690134,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":422969344},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:40:53 2023","interfaces":[{"interface":"back","average":{"1min":1.3640000000000001,"5min":1.397,"15min":1.758},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.20699999999999999},"max":{"1min":5.8330000000000002,"5min":7.819,"15min":18.16},"last":0.27300000000000002},{"interface":"front","average":{"1min":1.3149999999999999,"5min":1.4770000000000001,"15min":1.665},"min":{"1min":0.29899999999999999,"5min":0.29499999999999998,"15min":0.245},"max":{"1min":5.157,"5min":7.9859999999999998,"15min":9.0570000000000004},"last":0.437}]},{"osd":1,"last update":"Sat Sep 23 20:41:09 2023","interfaces":[{"interface":"back","average":{"1min":1.391,"5min":1.95,"15min":1.8740000000000001},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.253},"max":{"1min":6.1139999999999999,"5min":36.588999999999999,"15min":36.588999999999999},"last":61.005000000000003},{"interface":"front","average":{"1min":1.236,"5min":1.587,"15min":1.675},"min":{"1min":0.30199999999999999,"5min":0.27700000000000002,"15min":0.23499999999999999},"max":{"1min":5.7800000000000002,"5min":36.920999999999999,"15min":36.920999999999999},"last":59.688000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:40:53 2023","interfaces":[{"interface":"back","average":{"1min":2.0840000000000001,"5min":1.9470000000000001,"15min":1.9590000000000001},"min":{"1min":0.28000000000000003,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":9.2119999999999997,"5min":37.011000000000003,"15min":37.011000000000003},"last":0.35299999999999998},{"interface":"front","average":{"1min":1.5,"5min":1.74,"15min":1.978},"min":{"1min":0.32800000000000001,"5min":0.28999999999999998,"15min":0.255},"max":{"1min":5.4889999999999999,"5min":36.835000000000001,"15min":36.835000000000001},"last":0.32300000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:41:15 2023","interfaces":[{"interface":"back","average":{"1min":0.86699999999999999,"5min":1.5229999999999999,"15min":2.1539999999999999},"min":{"1min":0.36099999999999999,"5min":0.34699999999999998,"15min":0.25700000000000001},"max":{"1min":3.512,"5min":6.9669999999999996,"15min":36.716000000000001},"last":0.40699999999999997},{"interface":"front","average":{"1min":1.2989999999999999,"5min":2.0699999999999998,"15min":2.0150000000000001},"min":{"1min":0.371,"5min":0.34000000000000002,"15min":0.28799999999999998},"max":{"1min":5.8940000000000001,"5min":18.504000000000001,"15min":18.504000000000001},"last":0.39500000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:40:44 2023","interfaces":[{"interface":"back","average":{"1min":1.2689999999999999,"5min":0.90600000000000003,"15min":1.3140000000000001},"min":{"1min":0.29999999999999999,"5min":0.255,"15min":0.255},"max":{"1min":6.3780000000000001,"5min":6.9180000000000001,"15min":28.015999999999998},"last":0.33700000000000002},{"interface":"front","average":{"1min":1.5760000000000001,"5min":0.93700000000000006,"15min":1.0940000000000001},"min":{"1min":0.32500000000000001,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":5.7469999999999999,"5min":6.8380000000000001,"15min":12.300000000000001},"last":0.42699999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:40:58 2023","interfaces":[{"interface":"back","average":{"1min":1.1160000000000001,"5min":1.089,"15min":1.1970000000000001},"min":{"1min":0.34599999999999997,"5min":0.318,"15min":0.30199999999999999},"max":{"1min":6.0739999999999998,"5min":6.9960000000000004,"15min":12.890000000000001},"last":0.372},{"interface":"front","average":{"1min":1.339,"5min":1.075,"15min":1.117},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":5.8609999999999998,"5min":6.7930000000000001,"15min":16.265999999999998},"last":0.45100000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:41:15 2023","interfaces":[{"interface":"back","average":{"1min":1.1799999999999999,"5min":0.93700000000000006,"15min":1.1279999999999999},"min":{"1min":0.34499999999999997,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":6.4720000000000004,"5min":6.4720000000000004,"15min":36.621000000000002},"last":0.47299999999999998},{"interface":"front","average":{"1min":1.319,"5min":1.0469999999999999,"15min":1.129},"min":{"1min":0.375,"5min":0.33200000000000002,"15min":0.30099999999999999},"max":{"1min":7.3700000000000001,"5min":7.3700000000000001,"15min":37.331000000000003},"last":0.46600000000000003}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4722,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1360,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8206,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7156,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3154,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":483196928,"data_stored":483196928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":985464832,"data_stored":985464832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":286982144,"data_stored":286982144,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":834928640,"data_stored":834928640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":352124928,"data_stored":352124928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1056374784,"data_stored":1056374784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":297074688,"data_stored":297074688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":837025792,"data_stored":837025792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:41:24.163 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-09-23T20:41:24.163 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:41:24.164 INFO:tasks.thrashosds.thrasher:Growing pool .mgr 2023-09-23T20:41:24.164 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:41:24.164 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:41:24.184 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:41:24.225 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:24.280 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:41:24.287 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:24.287 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:41:24.389 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:41:24.430 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:24.448 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:41:24.482 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:41:24.486 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:24.486 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:41:24.587 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:41:24.623 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:24.669 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:41:24.676 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:24.677 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:41:24.777 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:41:24.818 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:24 smithi149 ceph-mon[170394]: from='client.36468 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:41:24.821 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:24.877 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:41:24.880 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:24.880 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:41:24.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:24 smithi018 ceph-mon[179647]: from='client.36468 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:41:24.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:24 smithi018 ceph-mon[184144]: from='client.36468 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:41:24.981 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:41:25.024 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:25.084 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:41:25.087 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:25.087 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:41:25.133 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 16 16 36239 36223 141.452 89.5625 0.0137993 0.00706567 2023-09-23T20:41:25.184 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:41:25.188 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:41:25.274 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:25.361 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:41:25.370 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:41:25 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:41:25] "GET /metrics HTTP/1.1" 200 36513 "" "Prometheus/2.43.0" 2023-09-23T20:41:25.370 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:25.371 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:41:25.471 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:41:25.515 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:25.554 INFO:teuthology.orchestra.run.smithi018.stderr:noscrub is set 2023-09-23T20:41:25.583 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:41:25.589 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:25.589 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:41:25.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:25 smithi018 ceph-mon[184144]: pgmap v1134: 119 pgs: 119 active+clean; 1.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 147 MiB/s wr, 2.35k op/s 2023-09-23T20:41:25.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:25 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2710061888' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:41:25.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:25 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:41:25.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:25 smithi018 ceph-mon[179647]: pgmap v1134: 119 pgs: 119 active+clean; 1.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 147 MiB/s wr, 2.35k op/s 2023-09-23T20:41:25.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:25 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2710061888' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:41:25.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:25 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:41:25.690 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:41:25.769 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:25.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:25 smithi149 ceph-mon[170394]: pgmap v1134: 119 pgs: 119 active+clean; 1.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 147 MiB/s wr, 2.35k op/s 2023-09-23T20:41:25.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:25 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2710061888' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:41:25.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:25 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:41:26.135 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 17 16 37832 37816 138.971 99.5625 0.0122139 0.00718917 2023-09-23T20:41:26.254 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:41:26.261 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:26.262 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:41:26.272 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:41:26.363 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:41:26.414 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:26.466 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:41:26.480 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:26.480 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:41:26.581 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:41:26.598 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2023-09-23T20:41:26.598 INFO:teuthology.orchestra.run.smithi018.stdout: "ops": [], 2023-09-23T20:41:26.598 INFO:teuthology.orchestra.run.smithi018.stdout: "complaint_time": 30, 2023-09-23T20:41:26.598 INFO:teuthology.orchestra.run.smithi018.stdout: "num_blocked_ops": 0 2023-09-23T20:41:26.598 INFO:teuthology.orchestra.run.smithi018.stdout:} 2023-09-23T20:41:26.629 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:26.631 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:26 smithi018 ceph-mon[179647]: 1.1b scrub starts 2023-09-23T20:41:26.631 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:26 smithi018 ceph-mon[179647]: 1.1b scrub ok 2023-09-23T20:41:26.631 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:26 smithi018 ceph-mon[179647]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:41:26.631 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:26 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:41:26.631 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:26 smithi018 ceph-mon[179647]: osdmap e460: 8 total, 8 up, 8 in 2023-09-23T20:41:26.632 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:26 smithi018 ceph-mon[179647]: failed to encode map e460 with expected crc 2023-09-23T20:41:26.632 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:26 smithi018 ceph-mon[179647]: failed to encode map e460 with expected crc 2023-09-23T20:41:26.632 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:26 smithi018 ceph-mon[179647]: failed to encode map e460 with expected crc 2023-09-23T20:41:26.632 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:26 smithi018 ceph-mon[179647]: failed to encode map e460 with expected crc 2023-09-23T20:41:26.632 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:26 smithi018 ceph-mon[184144]: 1.1b scrub starts 2023-09-23T20:41:26.632 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:26 smithi018 ceph-mon[184144]: 1.1b scrub ok 2023-09-23T20:41:26.632 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:26 smithi018 ceph-mon[184144]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:41:26.632 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:26 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:41:26.632 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:26 smithi018 ceph-mon[184144]: osdmap e460: 8 total, 8 up, 8 in 2023-09-23T20:41:26.633 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:26 smithi018 ceph-mon[184144]: failed to encode map e460 with expected crc 2023-09-23T20:41:26.633 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:26 smithi018 ceph-mon[184144]: failed to encode map e460 with expected crc 2023-09-23T20:41:26.633 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:26 smithi018 ceph-mon[184144]: failed to encode map e460 with expected crc 2023-09-23T20:41:26.633 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:26 smithi018 ceph-mon[184144]: failed to encode map e460 with expected crc 2023-09-23T20:41:26.695 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:41:26.699 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:26 smithi149 ceph-mon[170394]: 1.1b scrub starts 2023-09-23T20:41:26.699 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:26 smithi149 ceph-mon[170394]: 1.1b scrub ok 2023-09-23T20:41:26.699 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:26 smithi149 ceph-mon[170394]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:41:26.699 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:26 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:41:26.699 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:26 smithi149 ceph-mon[170394]: osdmap e460: 8 total, 8 up, 8 in 2023-09-23T20:41:26.700 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:26 smithi149 ceph-mon[170394]: failed to encode map e460 with expected crc 2023-09-23T20:41:26.700 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:26 smithi149 ceph-mon[170394]: failed to encode map e460 with expected crc 2023-09-23T20:41:26.700 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:26 smithi149 ceph-mon[170394]: failed to encode map e460 with expected crc 2023-09-23T20:41:26.700 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:26 smithi149 ceph-mon[170394]: failed to encode map e460 with expected crc 2023-09-23T20:41:26.700 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:26.701 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:41:26.802 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:41:26.903 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:27.122 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:41:27.128 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:27.128 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:41:27.135 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 18 16 39462 39446 136.91 101.875 0.0115454 0.00729926 2023-09-23T20:41:27.229 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T20:41:27.229 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:41:27.332 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:27.429 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:41:27.439 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:27.439 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:41:27.540 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:41:27.611 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:27.840 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:41:27.846 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:27.847 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:41:27.847 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:27 smithi018 ceph-mon[184144]: pgmap v1136: 119 pgs: 119 active+clean; 2.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 129 MiB/s wr, 2.06k op/s 2023-09-23T20:41:27.847 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:27 smithi018 ceph-mon[179647]: pgmap v1136: 119 pgs: 119 active+clean; 2.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 129 MiB/s wr, 2.06k op/s 2023-09-23T20:41:27.948 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:41:27.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:27 smithi149 ceph-mon[170394]: pgmap v1136: 119 pgs: 119 active+clean; 2.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 129 MiB/s wr, 2.06k op/s 2023-09-23T20:41:28.005 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:28.086 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:41:28.094 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:28.095 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:41:28.106 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:41:28.114 INFO:teuthology.orchestra.run.smithi018.stderr:dumped all 2023-09-23T20:41:28.135 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 19 16 40806 40790 134.126 84 0.0134425 0.00745171 2023-09-23T20:41:28.196 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:41:28.239 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:28.283 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:41:28.289 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:28.289 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:41:28.346 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T20:41:28.390 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:41:28.493 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:28.597 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:41:28.619 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:28.620 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:41:28.651 INFO:teuthology.orchestra.run.smithi018.stdout:{"pg_ready":true,"pg_map":{"version":1136,"stamp":"2023-09-23T20:41:26.103394+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2212226873,"num_objects":33713,"num_object_clones":0,"num_object_copies":101139,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":33713,"num_whiteouts":0,"num_read":6027,"num_read_kb":13283,"num_write":34615,"num_write_kb":2168849,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":149,"num_bytes_recovered":8830792,"num_keys_recovered":40,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39708,"ondisk_log_size":39708,"up":357,"acting":357,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":363,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":11353476,"kb_used_data":7150084,"kb_used_omap":32,"kb_used_meta":4203295,"kb_avail":738607740,"statfs":{"total":767960285184,"available":756334325760,"internally_reserved":0,"allocated":7321686016,"data_stored":7306470368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32773,"internal_metadata":4304175099},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[6,4,6,4],"upper_bound":16},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":1348638509,"num_objects":20579,"num_object_clones":0,"num_object_copies":61737,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":20579,"num_whiteouts":0,"num_read":-22,"num_read_kb":-41,"num_write":20568,"num_write_kb":1316947,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":-80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.005423"},"pg_stats":[{"pgid":"1.18","version":"90'161","reported_seq":442,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:09.135550+0000","last_change":"2023-09-23T20:40:56.483418+0000","last_active":"2023-09-23T20:41:09.135550+0000","last_peered":"2023-09-23T20:41:09.135550+0000","last_clean":"2023-09-23T20:41:09.135550+0000","last_became_active":"2023-09-23T20:40:56.482886+0000","last_became_peered":"2023-09-23T20:40:56.482886+0000","last_unstale":"2023-09-23T20:41:09.135550+0000","last_undegraded":"2023-09-23T20:41:09.135550+0000","last_fullsized":"2023-09-23T20:41:09.135550+0000","mapping_epoch":453,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":454,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:35:03.551143+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:35:03.551143+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":343193,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"69'1","reported_seq":157,"reported_epoch":459,"state":"active+clean","last_fresh":"2023-09-23T20:41:11.585369+0000","last_change":"2023-09-23T20:32:33.233853+0000","last_active":"2023-09-23T20:41:11.585369+0000","last_peered":"2023-09-23T20:41:11.585369+0000","last_clean":"2023-09-23T20:41:11.585369+0000","last_became_active":"2023-09-23T20:32:33.233546+0000","last_became_peered":"2023-09-23T20:32:33.233546+0000","last_unstale":"2023-09-23T20:41:11.585369+0000","last_undegraded":"2023-09-23T20:41:11.585369+0000","last_fullsized":"2023-09-23T20:41:11.585369+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:50:27.130808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":272,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:08.520708+0000","last_change":"2023-09-23T20:37:43.579611+0000","last_active":"2023-09-23T20:41:08.520708+0000","last_peered":"2023-09-23T20:41:08.520708+0000","last_clean":"2023-09-23T20:41:08.520708+0000","last_became_active":"2023-09-23T20:37:43.579323+0000","last_became_peered":"2023-09-23T20:37:43.579323+0000","last_unstale":"2023-09-23T20:41:08.520708+0000","last_undegraded":"2023-09-23T20:41:08.520708+0000","last_fullsized":"2023-09-23T20:41:08.520708+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:17:32.311427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"90'161","reported_seq":542,"reported_epoch":459,"state":"active+clean","last_fresh":"2023-09-23T20:41:25.432850+0000","last_change":"2023-09-23T20:41:25.432814+0000","last_active":"2023-09-23T20:41:25.432850+0000","last_peered":"2023-09-23T20:41:25.432850+0000","last_clean":"2023-09-23T20:41:25.432850+0000","last_became_active":"2023-09-23T20:40:22.444637+0000","last_became_peered":"2023-09-23T20:40:22.444637+0000","last_unstale":"2023-09-23T20:41:25.432850+0000","last_undegraded":"2023-09-23T20:41:25.432850+0000","last_fullsized":"2023-09-23T20:41:25.432850+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:41:25.432747+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:36:05.442288+0000","last_clean_scrub_stamp":"2023-09-23T20:41:25.432747+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:50:06.434043+0000","scrub_duration":0.010139941,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":427,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:09.132135+0000","last_change":"2023-09-23T20:34:59.983035+0000","last_active":"2023-09-23T20:41:09.132135+0000","last_peered":"2023-09-23T20:41:09.132135+0000","last_clean":"2023-09-23T20:41:09.132135+0000","last_became_active":"2023-09-23T20:34:59.982459+0000","last_became_peered":"2023-09-23T20:34:59.982459+0000","last_unstale":"2023-09-23T20:41:09.132135+0000","last_undegraded":"2023-09-23T20:41:09.132135+0000","last_fullsized":"2023-09-23T20:41:09.132135+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:16:48.510783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":429,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:07.553949+0000","last_change":"2023-09-23T20:25:02.957185+0000","last_active":"2023-09-23T20:41:07.553949+0000","last_peered":"2023-09-23T20:41:07.553949+0000","last_clean":"2023-09-23T20:41:07.553949+0000","last_became_active":"2023-09-23T20:25:02.956689+0000","last_became_peered":"2023-09-23T20:25:02.956689+0000","last_unstale":"2023-09-23T20:41:07.553949+0000","last_undegraded":"2023-09-23T20:41:07.553949+0000","last_fullsized":"2023-09-23T20:41:07.553949+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:04:28.524015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"90'161","reported_seq":539,"reported_epoch":459,"state":"active+clean","last_fresh":"2023-09-23T20:41:11.584810+0000","last_change":"2023-09-23T20:36:01.292103+0000","last_active":"2023-09-23T20:41:11.584810+0000","last_peered":"2023-09-23T20:41:11.584810+0000","last_clean":"2023-09-23T20:41:11.584810+0000","last_became_active":"2023-09-23T20:35:26.639912+0000","last_became_peered":"2023-09-23T20:35:26.639912+0000","last_unstale":"2023-09-23T20:41:11.584810+0000","last_undegraded":"2023-09-23T20:41:11.584810+0000","last_fullsized":"2023-09-23T20:41:11.584810+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:01.292032+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:01.292032+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:04:53.926640+0000","scrub_duration":0.0032983159999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":252,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:08.520302+0000","last_change":"2023-09-23T20:37:43.584414+0000","last_active":"2023-09-23T20:41:08.520302+0000","last_peered":"2023-09-23T20:41:08.520302+0000","last_clean":"2023-09-23T20:41:08.520302+0000","last_became_active":"2023-09-23T20:37:43.584116+0000","last_became_peered":"2023-09-23T20:37:43.584116+0000","last_unstale":"2023-09-23T20:41:08.520302+0000","last_undegraded":"2023-09-23T20:41:08.520302+0000","last_fullsized":"2023-09-23T20:41:08.520302+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:17:54.258172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":74,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:08.381235+0000","last_change":"2023-09-23T20:38:37.457427+0000","last_active":"2023-09-23T20:41:08.381235+0000","last_peered":"2023-09-23T20:41:08.381235+0000","last_clean":"2023-09-23T20:41:08.381235+0000","last_became_active":"2023-09-23T20:38:37.457099+0000","last_became_peered":"2023-09-23T20:38:37.457099+0000","last_unstale":"2023-09-23T20:41:08.381235+0000","last_undegraded":"2023-09-23T20:41:08.381235+0000","last_fullsized":"2023-09-23T20:41:08.381235+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:31:27.084994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"90'161","reported_seq":671,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:08.769306+0000","last_change":"2023-09-23T20:40:22.449754+0000","last_active":"2023-09-23T20:41:08.769306+0000","last_peered":"2023-09-23T20:41:08.769306+0000","last_clean":"2023-09-23T20:41:08.769306+0000","last_became_active":"2023-09-23T20:40:22.448838+0000","last_became_peered":"2023-09-23T20:40:22.448838+0000","last_unstale":"2023-09-23T20:41:08.769306+0000","last_undegraded":"2023-09-23T20:41:08.769306+0000","last_fullsized":"2023-09-23T20:41:08.769306+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:03.411025+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:03.411025+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:22:07.948856+0000","scrub_duration":0.001802922,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":462,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:07.554042+0000","last_change":"2023-09-23T20:40:19.538844+0000","last_active":"2023-09-23T20:41:07.554042+0000","last_peered":"2023-09-23T20:41:07.554042+0000","last_clean":"2023-09-23T20:41:07.554042+0000","last_became_active":"2023-09-23T20:40:19.538434+0000","last_became_peered":"2023-09-23T20:40:19.538434+0000","last_unstale":"2023-09-23T20:41:07.554042+0000","last_undegraded":"2023-09-23T20:41:07.554042+0000","last_fullsized":"2023-09-23T20:41:07.554042+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:53:59.418427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":183,"reported_epoch":459,"state":"active+clean","last_fresh":"2023-09-23T20:41:11.600968+0000","last_change":"2023-09-23T20:38:02.579343+0000","last_active":"2023-09-23T20:41:11.600968+0000","last_peered":"2023-09-23T20:41:11.600968+0000","last_clean":"2023-09-23T20:41:11.600968+0000","last_became_active":"2023-09-23T20:38:02.579047+0000","last_became_peered":"2023-09-23T20:38:02.579047+0000","last_unstale":"2023-09-23T20:41:11.600968+0000","last_undegraded":"2023-09-23T20:41:11.600968+0000","last_fullsized":"2023-09-23T20:41:11.600968+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:44:03.166556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.14","version":"90'161","reported_seq":788,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:08.619762+0000","last_change":"2023-09-23T20:35:02.236398+0000","last_active":"2023-09-23T20:41:08.619762+0000","last_peered":"2023-09-23T20:41:08.619762+0000","last_clean":"2023-09-23T20:41:08.619762+0000","last_became_active":"2023-09-23T20:35:02.235988+0000","last_became_peered":"2023-09-23T20:35:02.235988+0000","last_unstale":"2023-09-23T20:41:08.619762+0000","last_undegraded":"2023-09-23T20:41:08.619762+0000","last_fullsized":"2023-09-23T20:41:08.619762+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:47.479012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:47.479012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:13.519648+0000","scrub_duration":0.0020136590000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":28,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:09.136481+0000","last_change":"2023-09-23T20:40:19.545472+0000","last_active":"2023-09-23T20:41:09.136481+0000","last_peered":"2023-09-23T20:41:09.136481+0000","last_clean":"2023-09-23T20:41:09.136481+0000","last_became_active":"2023-09-23T20:40:19.543853+0000","last_became_peered":"2023-09-23T20:40:19.543853+0000","last_unstale":"2023-09-23T20:41:09.136481+0000","last_undegraded":"2023-09-23T20:41:09.136481+0000","last_fullsized":"2023-09-23T20:41:09.136481+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:08.014744+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":283,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:08.520523+0000","last_change":"2023-09-23T20:38:02.542034+0000","last_active":"2023-09-23T20:41:08.520523+0000","last_peered":"2023-09-23T20:41:08.520523+0000","last_clean":"2023-09-23T20:41:08.520523+0000","last_became_active":"2023-09-23T20:38:02.541575+0000","last_became_peered":"2023-09-23T20:38:02.541575+0000","last_unstale":"2023-09-23T20:41:08.520523+0000","last_undegraded":"2023-09-23T20:41:08.520523+0000","last_fullsized":"2023-09-23T20:41:08.520523+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:50:54.591529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"90'161","reported_seq":622,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:11.210079+0000","last_change":"2023-09-23T20:41:11.210009+0000","last_active":"2023-09-23T20:41:11.210079+0000","last_peered":"2023-09-23T20:41:11.210079+0000","last_clean":"2023-09-23T20:41:11.210079+0000","last_became_active":"2023-09-23T20:40:22.444543+0000","last_became_peered":"2023-09-23T20:40:22.444543+0000","last_unstale":"2023-09-23T20:41:11.210079+0000","last_undegraded":"2023-09-23T20:41:11.210079+0000","last_fullsized":"2023-09-23T20:41:11.210079+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:41:11.209916+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:41:11.209916+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:30:05.067685+0000","scrub_duration":0.0025146090000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":486,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:08.769513+0000","last_change":"2023-09-23T20:32:33.270617+0000","last_active":"2023-09-23T20:41:08.769513+0000","last_peered":"2023-09-23T20:41:08.769513+0000","last_clean":"2023-09-23T20:41:08.769513+0000","last_became_active":"2023-09-23T20:32:33.227902+0000","last_became_peered":"2023-09-23T20:32:33.227902+0000","last_unstale":"2023-09-23T20:41:08.769513+0000","last_undegraded":"2023-09-23T20:41:08.769513+0000","last_fullsized":"2023-09-23T20:41:08.769513+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:30:26.847831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":360,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":428,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:09.132505+0000","last_change":"2023-09-23T20:38:02.599068+0000","last_active":"2023-09-23T20:41:09.132505+0000","last_peered":"2023-09-23T20:41:09.132505+0000","last_clean":"2023-09-23T20:41:09.132505+0000","last_became_active":"2023-09-23T20:38:02.597602+0000","last_became_peered":"2023-09-23T20:38:02.597602+0000","last_unstale":"2023-09-23T20:41:09.132505+0000","last_undegraded":"2023-09-23T20:41:09.132505+0000","last_fullsized":"2023-09-23T20:41:09.132505+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:34:06.337029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"90'161","reported_seq":583,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:09.132772+0000","last_change":"2023-09-23T20:36:02.548387+0000","last_active":"2023-09-23T20:41:09.132772+0000","last_peered":"2023-09-23T20:41:09.132772+0000","last_clean":"2023-09-23T20:41:09.132772+0000","last_became_active":"2023-09-23T20:35:26.644787+0000","last_became_peered":"2023-09-23T20:35:26.644787+0000","last_unstale":"2023-09-23T20:41:09.132772+0000","last_undegraded":"2023-09-23T20:41:09.132772+0000","last_fullsized":"2023-09-23T20:41:09.132772+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:02.548285+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:02.548285+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:37:15.216553+0000","scrub_duration":0.0015514459999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":207,"reported_epoch":459,"state":"active+clean","last_fresh":"2023-09-23T20:41:11.585999+0000","last_change":"2023-09-23T20:38:02.560646+0000","last_active":"2023-09-23T20:41:11.585999+0000","last_peered":"2023-09-23T20:41:11.585999+0000","last_clean":"2023-09-23T20:41:11.585999+0000","last_became_active":"2023-09-23T20:38:02.539814+0000","last_became_peered":"2023-09-23T20:38:02.539814+0000","last_unstale":"2023-09-23T20:41:11.585999+0000","last_undegraded":"2023-09-23T20:41:11.585999+0000","last_fullsized":"2023-09-23T20:41:11.585999+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:00:09.435023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":187,"reported_epoch":459,"state":"active+clean","last_fresh":"2023-09-23T20:41:11.586228+0000","last_change":"2023-09-23T20:38:02.579222+0000","last_active":"2023-09-23T20:41:11.586228+0000","last_peered":"2023-09-23T20:41:11.586228+0000","last_clean":"2023-09-23T20:41:11.586228+0000","last_became_active":"2023-09-23T20:38:02.578813+0000","last_became_peered":"2023-09-23T20:38:02.578813+0000","last_unstale":"2023-09-23T20:41:11.586228+0000","last_undegraded":"2023-09-23T20:41:11.586228+0000","last_fullsized":"2023-09-23T20:41:11.586228+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:57:31.080027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"90'161","reported_seq":725,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:07.553128+0000","last_change":"2023-09-23T20:40:22.427437+0000","last_active":"2023-09-23T20:41:07.553128+0000","last_peered":"2023-09-23T20:41:07.553128+0000","last_clean":"2023-09-23T20:41:07.553128+0000","last_became_active":"2023-09-23T20:40:22.427090+0000","last_became_peered":"2023-09-23T20:40:22.427090+0000","last_unstale":"2023-09-23T20:41:07.553128+0000","last_undegraded":"2023-09-23T20:41:07.553128+0000","last_fullsized":"2023-09-23T20:41:07.553128+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:43.578666+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:43.578666+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:08:43.995672+0000","scrub_duration":0.0021890379999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":485,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:08.769039+0000","last_change":"2023-09-23T20:32:33.242943+0000","last_active":"2023-09-23T20:41:08.769039+0000","last_peered":"2023-09-23T20:41:08.769039+0000","last_clean":"2023-09-23T20:41:08.769039+0000","last_became_active":"2023-09-23T20:32:33.225468+0000","last_became_peered":"2023-09-23T20:32:33.225468+0000","last_unstale":"2023-09-23T20:41:08.769039+0000","last_undegraded":"2023-09-23T20:41:08.769039+0000","last_fullsized":"2023-09-23T20:41:08.769039+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:40:47.867573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":194,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:08.520407+0000","last_change":"2023-09-23T20:37:43.571836+0000","last_active":"2023-09-23T20:41:08.520407+0000","last_peered":"2023-09-23T20:41:08.520407+0000","last_clean":"2023-09-23T20:41:08.520407+0000","last_became_active":"2023-09-23T20:37:43.571511+0000","last_became_peered":"2023-09-23T20:37:43.571511+0000","last_unstale":"2023-09-23T20:41:08.520407+0000","last_undegraded":"2023-09-23T20:41:08.520407+0000","last_fullsized":"2023-09-23T20:41:08.520407+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:59:25.507917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"90'161","reported_seq":651,"reported_epoch":459,"state":"active+clean","last_fresh":"2023-09-23T20:41:22.390249+0000","last_change":"2023-09-23T20:41:22.390219+0000","last_active":"2023-09-23T20:41:22.390249+0000","last_peered":"2023-09-23T20:41:22.390249+0000","last_clean":"2023-09-23T20:41:22.390249+0000","last_became_active":"2023-09-23T20:40:22.427114+0000","last_became_peered":"2023-09-23T20:40:22.427114+0000","last_unstale":"2023-09-23T20:41:22.390249+0000","last_undegraded":"2023-09-23T20:41:22.390249+0000","last_fullsized":"2023-09-23T20:41:22.390249+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:41:22.390153+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:41:22.390153+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:17:27.488965+0000","scrub_duration":0.00186133,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":427,"reported_epoch":459,"state":"active+clean","last_fresh":"2023-09-23T20:41:11.599626+0000","last_change":"2023-09-23T20:25:02.953993+0000","last_active":"2023-09-23T20:41:11.599626+0000","last_peered":"2023-09-23T20:41:11.599626+0000","last_clean":"2023-09-23T20:41:11.599626+0000","last_became_active":"2023-09-23T20:25:02.953672+0000","last_became_peered":"2023-09-23T20:25:02.953672+0000","last_unstale":"2023-09-23T20:41:11.599626+0000","last_undegraded":"2023-09-23T20:41:11.599626+0000","last_fullsized":"2023-09-23T20:41:11.599626+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:26:08.603577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":28,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:09.136428+0000","last_change":"2023-09-23T20:40:19.561512+0000","last_active":"2023-09-23T20:41:09.136428+0000","last_peered":"2023-09-23T20:41:09.136428+0000","last_clean":"2023-09-23T20:41:09.136428+0000","last_became_active":"2023-09-23T20:40:19.561123+0000","last_became_peered":"2023-09-23T20:40:19.561123+0000","last_unstale":"2023-09-23T20:41:09.136428+0000","last_undegraded":"2023-09-23T20:41:09.136428+0000","last_fullsized":"2023-09-23T20:41:09.136428+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:50:02.093978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"90'161","reported_seq":744,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:09.132686+0000","last_change":"2023-09-23T20:35:02.238923+0000","last_active":"2023-09-23T20:41:09.132686+0000","last_peered":"2023-09-23T20:41:09.132686+0000","last_clean":"2023-09-23T20:41:09.132686+0000","last_became_active":"2023-09-23T20:35:02.238293+0000","last_became_peered":"2023-09-23T20:35:02.238293+0000","last_unstale":"2023-09-23T20:41:09.132686+0000","last_undegraded":"2023-09-23T20:41:09.132686+0000","last_fullsized":"2023-09-23T20:41:09.132686+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:42.544308+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:42.544308+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:51:22.281932+0000","scrub_duration":0.0029958179999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":462,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:08.768930+0000","last_change":"2023-09-23T20:38:02.539774+0000","last_active":"2023-09-23T20:41:08.768930+0000","last_peered":"2023-09-23T20:41:08.768930+0000","last_clean":"2023-09-23T20:41:08.768930+0000","last_became_active":"2023-09-23T20:38:02.539436+0000","last_became_peered":"2023-09-23T20:38:02.539436+0000","last_unstale":"2023-09-23T20:41:08.768930+0000","last_undegraded":"2023-09-23T20:41:08.768930+0000","last_fullsized":"2023-09-23T20:41:08.768930+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:50:57.375803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":436,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:07.554026+0000","last_change":"2023-09-23T20:25:02.957010+0000","last_active":"2023-09-23T20:41:07.554026+0000","last_peered":"2023-09-23T20:41:07.554026+0000","last_clean":"2023-09-23T20:41:07.554026+0000","last_became_active":"2023-09-23T20:25:02.956267+0000","last_became_peered":"2023-09-23T20:25:02.956267+0000","last_unstale":"2023-09-23T20:41:07.554026+0000","last_undegraded":"2023-09-23T20:41:07.554026+0000","last_fullsized":"2023-09-23T20:41:07.554026+0000","mapping_epoch":165,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":166,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:10:55.299422+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"90'161","reported_seq":574,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:09.136215+0000","last_change":"2023-09-23T20:41:08.258019+0000","last_active":"2023-09-23T20:41:09.136215+0000","last_peered":"2023-09-23T20:41:09.136215+0000","last_clean":"2023-09-23T20:41:09.136215+0000","last_became_active":"2023-09-23T20:40:22.430717+0000","last_became_peered":"2023-09-23T20:40:22.430717+0000","last_unstale":"2023-09-23T20:41:09.136215+0000","last_undegraded":"2023-09-23T20:41:09.136215+0000","last_fullsized":"2023-09-23T20:41:09.136215+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:41:08.257952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:41:08.257952+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:47:51.982877+0000","scrub_duration":0.00158735,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":94,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:08.380892+0000","last_change":"2023-09-23T20:40:19.564378+0000","last_active":"2023-09-23T20:41:08.380892+0000","last_peered":"2023-09-23T20:41:08.380892+0000","last_clean":"2023-09-23T20:41:08.380892+0000","last_became_active":"2023-09-23T20:40:19.532991+0000","last_became_peered":"2023-09-23T20:40:19.532991+0000","last_unstale":"2023-09-23T20:41:08.380892+0000","last_undegraded":"2023-09-23T20:41:08.380892+0000","last_fullsized":"2023-09-23T20:41:08.380892+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:22:42.832115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":252,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:08.521100+0000","last_change":"2023-09-23T20:37:43.571083+0000","last_active":"2023-09-23T20:41:08.521100+0000","last_peered":"2023-09-23T20:41:08.521100+0000","last_clean":"2023-09-23T20:41:08.521100+0000","last_became_active":"2023-09-23T20:37:43.570306+0000","last_became_peered":"2023-09-23T20:37:43.570306+0000","last_unstale":"2023-09-23T20:41:08.521100+0000","last_undegraded":"2023-09-23T20:41:08.521100+0000","last_fullsized":"2023-09-23T20:41:08.521100+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:42:02.396718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.5","version":"459'1811","reported_seq":1824,"reported_epoch":459,"state":"active+clean","last_fresh":"2023-09-23T20:41:21.259952+0000","last_change":"2023-09-23T20:41:04.556343+0000","last_active":"2023-09-23T20:41:21.259952+0000","last_peered":"2023-09-23T20:41:21.259952+0000","last_clean":"2023-09-23T20:41:21.259952+0000","last_became_active":"2023-09-23T20:41:04.556043+0000","last_became_peered":"2023-09-23T20:41:04.556043+0000","last_unstale":"2023-09-23T20:41:21.259952+0000","last_undegraded":"2023-09-23T20:41:21.259952+0000","last_fullsized":"2023-09-23T20:41:21.259952+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":455,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":1811,"log_dups_size":0,"ondisk_log_size":1811,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:33:41.345407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":118685696,"num_objects":1811,"num_object_clones":0,"num_object_copies":5433,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1811,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1811,"num_write_kb":115904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.d","version":"90'161","reported_seq":751,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:08.769410+0000","last_change":"2023-09-23T20:34:59.496438+0000","last_active":"2023-09-23T20:41:08.769410+0000","last_peered":"2023-09-23T20:41:08.769410+0000","last_clean":"2023-09-23T20:41:08.769410+0000","last_became_active":"2023-09-23T20:33:38.970527+0000","last_became_peered":"2023-09-23T20:33:38.970527+0000","last_unstale":"2023-09-23T20:41:08.769410+0000","last_undegraded":"2023-09-23T20:41:08.769410+0000","last_fullsized":"2023-09-23T20:41:08.769410+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:34:59.496375+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:34:59.496375+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:46:18.244734+0000","scrub_duration":0.0017843189999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":462,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:08.769526+0000","last_change":"2023-09-23T20:32:33.283752+0000","last_active":"2023-09-23T20:41:08.769526+0000","last_peered":"2023-09-23T20:41:08.769526+0000","last_clean":"2023-09-23T20:41:08.769526+0000","last_became_active":"2023-09-23T20:32:33.230503+0000","last_became_peered":"2023-09-23T20:32:33.230503+0000","last_unstale":"2023-09-23T20:41:08.769526+0000","last_undegraded":"2023-09-23T20:41:08.769526+0000","last_fullsized":"2023-09-23T20:41:08.769526+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:29.084410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":28,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:09.135687+0000","last_change":"2023-09-23T20:40:19.545668+0000","last_active":"2023-09-23T20:41:09.135687+0000","last_peered":"2023-09-23T20:41:09.135687+0000","last_clean":"2023-09-23T20:41:09.135687+0000","last_became_active":"2023-09-23T20:40:19.543857+0000","last_became_peered":"2023-09-23T20:40:19.543857+0000","last_unstale":"2023-09-23T20:41:09.135687+0000","last_undegraded":"2023-09-23T20:41:09.135687+0000","last_fullsized":"2023-09-23T20:41:09.135687+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:27:09.470605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"459'2177","reported_seq":2190,"reported_epoch":459,"state":"active+clean","last_fresh":"2023-09-23T20:41:24.541611+0000","last_change":"2023-09-23T20:41:04.575200+0000","last_active":"2023-09-23T20:41:24.541611+0000","last_peered":"2023-09-23T20:41:24.541611+0000","last_clean":"2023-09-23T20:41:24.541611+0000","last_became_active":"2023-09-23T20:41:04.574824+0000","last_became_peered":"2023-09-23T20:41:04.574824+0000","last_unstale":"2023-09-23T20:41:24.541611+0000","last_undegraded":"2023-09-23T20:41:24.541611+0000","last_fullsized":"2023-09-23T20:41:24.541611+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":455,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":2177,"log_dups_size":0,"ondisk_log_size":2177,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:50:46.714383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":142671872,"num_objects":2177,"num_object_clones":0,"num_object_copies":6531,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2177,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2177,"num_write_kb":139328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.c","version":"90'161","reported_seq":718,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:09.506481+0000","last_change":"2023-09-23T20:41:09.506481+0000","last_active":"2023-09-23T20:41:09.506481+0000","last_peered":"2023-09-23T20:41:09.506481+0000","last_clean":"2023-09-23T20:41:09.506481+0000","last_became_active":"2023-09-23T20:38:28.442229+0000","last_became_peered":"2023-09-23T20:38:28.442229+0000","last_unstale":"2023-09-23T20:41:09.506481+0000","last_undegraded":"2023-09-23T20:41:09.506481+0000","last_fullsized":"2023-09-23T20:41:09.506481+0000","mapping_epoch":411,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":412,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:41:09.506338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:41:09.506338+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:19:46.265011+0000","scrub_duration":0.01112002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":497,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:08.769693+0000","last_change":"2023-09-23T20:34:59.987981+0000","last_active":"2023-09-23T20:41:08.769693+0000","last_peered":"2023-09-23T20:41:08.769693+0000","last_clean":"2023-09-23T20:41:08.769693+0000","last_became_active":"2023-09-23T20:34:59.985368+0000","last_became_peered":"2023-09-23T20:34:59.985368+0000","last_unstale":"2023-09-23T20:41:08.769693+0000","last_undegraded":"2023-09-23T20:41:08.769693+0000","last_fullsized":"2023-09-23T20:41:08.769693+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:12:43.082493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":157,"reported_epoch":459,"state":"active+clean","last_fresh":"2023-09-23T20:41:11.585047+0000","last_change":"2023-09-23T20:32:33.232142+0000","last_active":"2023-09-23T20:41:11.585047+0000","last_peered":"2023-09-23T20:41:11.585047+0000","last_clean":"2023-09-23T20:41:11.585047+0000","last_became_active":"2023-09-23T20:32:33.231683+0000","last_became_peered":"2023-09-23T20:32:33.231683+0000","last_unstale":"2023-09-23T20:41:11.585047+0000","last_undegraded":"2023-09-23T20:41:11.585047+0000","last_fullsized":"2023-09-23T20:41:11.585047+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:02:58.336348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"459'2315","reported_seq":2327,"reported_epoch":459,"state":"active+clean","last_fresh":"2023-09-23T20:41:25.474090+0000","last_change":"2023-09-23T20:41:04.557945+0000","last_active":"2023-09-23T20:41:25.474090+0000","last_peered":"2023-09-23T20:41:25.474090+0000","last_clean":"2023-09-23T20:41:25.474090+0000","last_became_active":"2023-09-23T20:41:04.557435+0000","last_became_peered":"2023-09-23T20:41:04.557435+0000","last_unstale":"2023-09-23T20:41:25.474090+0000","last_undegraded":"2023-09-23T20:41:25.474090+0000","last_fullsized":"2023-09-23T20:41:25.474090+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":455,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":2315,"log_dups_size":0,"ondisk_log_size":2315,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:09:02.910592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":151715840,"num_objects":2315,"num_object_clones":0,"num_object_copies":6945,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2315,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2315,"num_write_kb":148160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.f","version":"90'161","reported_seq":614,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:08.381437+0000","last_change":"2023-09-23T20:38:40.491902+0000","last_active":"2023-09-23T20:41:08.381437+0000","last_peered":"2023-09-23T20:41:08.381437+0000","last_clean":"2023-09-23T20:41:08.381437+0000","last_became_active":"2023-09-23T20:38:40.491563+0000","last_became_peered":"2023-09-23T20:38:40.491563+0000","last_unstale":"2023-09-23T20:41:08.381437+0000","last_undegraded":"2023-09-23T20:41:08.381437+0000","last_fullsized":"2023-09-23T20:41:08.381437+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:46.217489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:46.217489+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:39:25.008796+0000","scrub_duration":0.001456429,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":487,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:08.769471+0000","last_change":"2023-09-23T20:40:19.677655+0000","last_active":"2023-09-23T20:41:08.769471+0000","last_peered":"2023-09-23T20:41:08.769471+0000","last_clean":"2023-09-23T20:41:08.769471+0000","last_became_active":"2023-09-23T20:40:19.551189+0000","last_became_peered":"2023-09-23T20:40:19.551189+0000","last_unstale":"2023-09-23T20:41:08.769471+0000","last_undegraded":"2023-09-23T20:41:08.769471+0000","last_fullsized":"2023-09-23T20:41:08.769471+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:11:01.559576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":64,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:08.380895+0000","last_change":"2023-09-23T20:38:02.590613+0000","last_active":"2023-09-23T20:41:08.380895+0000","last_peered":"2023-09-23T20:41:08.380895+0000","last_clean":"2023-09-23T20:41:08.380895+0000","last_became_active":"2023-09-23T20:38:02.590284+0000","last_became_peered":"2023-09-23T20:38:02.590284+0000","last_unstale":"2023-09-23T20:41:08.380895+0000","last_undegraded":"2023-09-23T20:41:08.380895+0000","last_fullsized":"2023-09-23T20:41:08.380895+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:40:07.825054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"459'2276","reported_seq":2288,"reported_epoch":459,"state":"active+clean","last_fresh":"2023-09-23T20:41:25.467208+0000","last_change":"2023-09-23T20:41:04.573199+0000","last_active":"2023-09-23T20:41:25.467208+0000","last_peered":"2023-09-23T20:41:25.467208+0000","last_clean":"2023-09-23T20:41:25.467208+0000","last_became_active":"2023-09-23T20:41:04.572850+0000","last_became_peered":"2023-09-23T20:41:04.572850+0000","last_unstale":"2023-09-23T20:41:25.467208+0000","last_undegraded":"2023-09-23T20:41:25.467208+0000","last_fullsized":"2023-09-23T20:41:25.467208+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":455,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":2276,"log_dups_size":0,"ondisk_log_size":2276,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:53:49.894113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":149159936,"num_objects":2276,"num_object_clones":0,"num_object_copies":6828,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2276,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2276,"num_write_kb":145664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.e","version":"90'161","reported_seq":560,"reported_epoch":459,"state":"active+clean","last_fresh":"2023-09-23T20:41:11.585550+0000","last_change":"2023-09-23T20:36:00.322453+0000","last_active":"2023-09-23T20:41:11.585550+0000","last_peered":"2023-09-23T20:41:11.585550+0000","last_clean":"2023-09-23T20:41:11.585550+0000","last_became_active":"2023-09-23T20:35:02.233536+0000","last_became_peered":"2023-09-23T20:35:02.233536+0000","last_unstale":"2023-09-23T20:41:11.585550+0000","last_undegraded":"2023-09-23T20:41:11.585550+0000","last_fullsized":"2023-09-23T20:41:11.585550+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:00.322381+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:00.322381+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:02:15.489069+0000","scrub_duration":0.0017321859999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":53,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:09.136104+0000","last_change":"2023-09-23T20:38:37.456683+0000","last_active":"2023-09-23T20:41:09.136104+0000","last_peered":"2023-09-23T20:41:09.136104+0000","last_clean":"2023-09-23T20:41:09.136104+0000","last_became_active":"2023-09-23T20:38:37.456289+0000","last_became_peered":"2023-09-23T20:38:37.456289+0000","last_unstale":"2023-09-23T20:41:09.136104+0000","last_undegraded":"2023-09-23T20:41:09.136104+0000","last_fullsized":"2023-09-23T20:41:09.136104+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:03:10.507333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":445,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:07.553544+0000","last_change":"2023-09-23T20:32:33.236122+0000","last_active":"2023-09-23T20:41:07.553544+0000","last_peered":"2023-09-23T20:41:07.553544+0000","last_clean":"2023-09-23T20:41:07.553544+0000","last_became_active":"2023-09-23T20:32:33.235384+0000","last_became_peered":"2023-09-23T20:32:33.235384+0000","last_unstale":"2023-09-23T20:41:07.553544+0000","last_undegraded":"2023-09-23T20:41:07.553544+0000","last_fullsized":"2023-09-23T20:41:07.553544+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:56.843733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"459'2085","reported_seq":2097,"reported_epoch":459,"state":"active+clean","last_fresh":"2023-09-23T20:41:23.124216+0000","last_change":"2023-09-23T20:41:04.558473+0000","last_active":"2023-09-23T20:41:23.124216+0000","last_peered":"2023-09-23T20:41:23.124216+0000","last_clean":"2023-09-23T20:41:23.124216+0000","last_became_active":"2023-09-23T20:41:04.558173+0000","last_became_peered":"2023-09-23T20:41:04.558173+0000","last_unstale":"2023-09-23T20:41:23.124216+0000","last_undegraded":"2023-09-23T20:41:23.124216+0000","last_fullsized":"2023-09-23T20:41:23.124216+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":455,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":2085,"log_dups_size":0,"ondisk_log_size":2085,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:17:53.184877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":136642560,"num_objects":2085,"num_object_clones":0,"num_object_copies":6255,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2085,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2085,"num_write_kb":133440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":864,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:08.768866+0000","last_change":"2023-09-23T20:35:02.523308+0000","last_active":"2023-09-23T20:41:08.768866+0000","last_peered":"2023-09-23T20:41:08.768866+0000","last_clean":"2023-09-23T20:41:08.768866+0000","last_became_active":"2023-09-23T20:33:38.979989+0000","last_became_peered":"2023-09-23T20:33:38.979989+0000","last_unstale":"2023-09-23T20:41:08.768866+0000","last_undegraded":"2023-09-23T20:41:08.768866+0000","last_fullsized":"2023-09-23T20:41:08.768866+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:35:02.523203+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:35:02.523203+0000","last_clean_scrub_stamp":"2023-09-23T20:35:02.523203+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:02:41.472958+0000","scrub_duration":0.0085927989999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":278,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:08.520614+0000","last_change":"2023-09-23T20:37:43.583731+0000","last_active":"2023-09-23T20:41:08.520614+0000","last_peered":"2023-09-23T20:41:08.520614+0000","last_clean":"2023-09-23T20:41:08.520614+0000","last_became_active":"2023-09-23T20:37:43.583066+0000","last_became_peered":"2023-09-23T20:37:43.583066+0000","last_unstale":"2023-09-23T20:41:08.520614+0000","last_undegraded":"2023-09-23T20:41:08.520614+0000","last_fullsized":"2023-09-23T20:41:08.520614+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:03:45.211900+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":177,"reported_epoch":459,"state":"active+clean","last_fresh":"2023-09-23T20:41:11.585919+0000","last_change":"2023-09-23T20:38:02.579814+0000","last_active":"2023-09-23T20:41:11.585919+0000","last_peered":"2023-09-23T20:41:11.585919+0000","last_clean":"2023-09-23T20:41:11.585919+0000","last_became_active":"2023-09-23T20:38:02.579253+0000","last_became_peered":"2023-09-23T20:38:02.579253+0000","last_unstale":"2023-09-23T20:41:11.585919+0000","last_undegraded":"2023-09-23T20:41:11.585919+0000","last_fullsized":"2023-09-23T20:41:11.585919+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:05:07.298198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"459'2182","reported_seq":2195,"reported_epoch":459,"state":"active+clean","last_fresh":"2023-09-23T20:41:24.518957+0000","last_change":"2023-09-23T20:41:04.573101+0000","last_active":"2023-09-23T20:41:24.518957+0000","last_peered":"2023-09-23T20:41:24.518957+0000","last_clean":"2023-09-23T20:41:24.518957+0000","last_became_active":"2023-09-23T20:41:04.572728+0000","last_became_peered":"2023-09-23T20:41:04.572728+0000","last_unstale":"2023-09-23T20:41:24.518957+0000","last_undegraded":"2023-09-23T20:41:24.518957+0000","last_fullsized":"2023-09-23T20:41:24.518957+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":455,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":2182,"log_dups_size":0,"ondisk_log_size":2182,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:21:10.721593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":142999552,"num_objects":2182,"num_object_clones":0,"num_object_copies":6546,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2182,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2182,"num_write_kb":139648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":570,"reported_epoch":459,"state":"active+clean","last_fresh":"2023-09-23T20:41:16.281729+0000","last_change":"2023-09-23T20:41:16.281681+0000","last_active":"2023-09-23T20:41:16.281729+0000","last_peered":"2023-09-23T20:41:16.281729+0000","last_clean":"2023-09-23T20:41:16.281729+0000","last_became_active":"2023-09-23T20:40:22.433023+0000","last_became_peered":"2023-09-23T20:40:22.433023+0000","last_unstale":"2023-09-23T20:41:16.281729+0000","last_undegraded":"2023-09-23T20:41:16.281729+0000","last_fullsized":"2023-09-23T20:41:16.281729+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:41:16.281575+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:41:16.281575+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:51:51.938276+0000","scrub_duration":0.0055773209999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":64,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:08.380898+0000","last_change":"2023-09-23T20:38:02.578703+0000","last_active":"2023-09-23T20:41:08.380898+0000","last_peered":"2023-09-23T20:41:08.380898+0000","last_clean":"2023-09-23T20:41:08.380898+0000","last_became_active":"2023-09-23T20:38:02.578414+0000","last_became_peered":"2023-09-23T20:38:02.578414+0000","last_unstale":"2023-09-23T20:41:08.380898+0000","last_undegraded":"2023-09-23T20:41:08.380898+0000","last_fullsized":"2023-09-23T20:41:08.380898+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:43:59.779295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":126,"reported_epoch":459,"state":"active+clean","last_fresh":"2023-09-23T20:41:11.585031+0000","last_change":"2023-09-23T20:37:31.296606+0000","last_active":"2023-09-23T20:41:11.585031+0000","last_peered":"2023-09-23T20:41:11.585031+0000","last_clean":"2023-09-23T20:41:11.585031+0000","last_became_active":"2023-09-23T20:37:31.295924+0000","last_became_peered":"2023-09-23T20:37:31.295924+0000","last_unstale":"2023-09-23T20:41:11.585031+0000","last_undegraded":"2023-09-23T20:41:11.585031+0000","last_fullsized":"2023-09-23T20:41:11.585031+0000","mapping_epoch":391,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":392,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:27:50.948510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"459'1980","reported_seq":1992,"reported_epoch":459,"state":"active+clean","last_fresh":"2023-09-23T20:41:22.345244+0000","last_change":"2023-09-23T20:41:04.555980+0000","last_active":"2023-09-23T20:41:22.345244+0000","last_peered":"2023-09-23T20:41:22.345244+0000","last_clean":"2023-09-23T20:41:22.345244+0000","last_became_active":"2023-09-23T20:41:04.555688+0000","last_became_peered":"2023-09-23T20:41:04.555688+0000","last_unstale":"2023-09-23T20:41:22.345244+0000","last_undegraded":"2023-09-23T20:41:22.345244+0000","last_fullsized":"2023-09-23T20:41:22.345244+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":455,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":1980,"log_dups_size":0,"ondisk_log_size":1980,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:28:42.819404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":129761280,"num_objects":1980,"num_object_clones":0,"num_object_copies":5940,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1980,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1980,"num_write_kb":126720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.b","version":"90'161","reported_seq":761,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:09.135450+0000","last_change":"2023-09-23T20:40:22.445808+0000","last_active":"2023-09-23T20:41:09.135450+0000","last_peered":"2023-09-23T20:41:09.135450+0000","last_clean":"2023-09-23T20:41:09.135450+0000","last_became_active":"2023-09-23T20:40:22.445456+0000","last_became_peered":"2023-09-23T20:40:22.445456+0000","last_unstale":"2023-09-23T20:41:09.135450+0000","last_undegraded":"2023-09-23T20:41:09.135450+0000","last_fullsized":"2023-09-23T20:41:09.135450+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:05.595042+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:05.595042+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:39:23.678154+0000","scrub_duration":0.007271675,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":116,"reported_epoch":459,"state":"active+clean","last_fresh":"2023-09-23T20:41:11.584762+0000","last_change":"2023-09-23T20:34:59.983122+0000","last_active":"2023-09-23T20:41:11.584762+0000","last_peered":"2023-09-23T20:41:11.584762+0000","last_clean":"2023-09-23T20:41:11.584762+0000","last_became_active":"2023-09-23T20:34:59.982809+0000","last_became_peered":"2023-09-23T20:34:59.982809+0000","last_unstale":"2023-09-23T20:41:11.584762+0000","last_undegraded":"2023-09-23T20:41:11.584762+0000","last_fullsized":"2023-09-23T20:41:11.584762+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:49:40.839723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":177,"reported_epoch":459,"state":"active+clean","last_fresh":"2023-09-23T20:41:11.585715+0000","last_change":"2023-09-23T20:38:02.543866+0000","last_active":"2023-09-23T20:41:11.585715+0000","last_peered":"2023-09-23T20:41:11.585715+0000","last_clean":"2023-09-23T20:41:11.585715+0000","last_became_active":"2023-09-23T20:38:02.543563+0000","last_became_peered":"2023-09-23T20:38:02.543563+0000","last_unstale":"2023-09-23T20:41:11.585715+0000","last_undegraded":"2023-09-23T20:41:11.585715+0000","last_fullsized":"2023-09-23T20:41:11.585715+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:33.050536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"459'2180","reported_seq":2193,"reported_epoch":459,"state":"active+clean","last_fresh":"2023-09-23T20:41:23.976018+0000","last_change":"2023-09-23T20:41:04.573300+0000","last_active":"2023-09-23T20:41:23.976018+0000","last_peered":"2023-09-23T20:41:23.976018+0000","last_clean":"2023-09-23T20:41:23.976018+0000","last_became_active":"2023-09-23T20:41:04.572919+0000","last_became_peered":"2023-09-23T20:41:04.572919+0000","last_unstale":"2023-09-23T20:41:23.976018+0000","last_undegraded":"2023-09-23T20:41:23.976018+0000","last_fullsized":"2023-09-23T20:41:23.976018+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":455,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":2180,"log_dups_size":0,"ondisk_log_size":2180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:57:46.465243+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":142868480,"num_objects":2180,"num_object_clones":0,"num_object_copies":6540,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2180,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2180,"num_write_kb":139520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":573,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:09.135936+0000","last_change":"2023-09-23T20:41:07.289523+0000","last_active":"2023-09-23T20:41:09.135936+0000","last_peered":"2023-09-23T20:41:09.135936+0000","last_clean":"2023-09-23T20:41:09.135936+0000","last_became_active":"2023-09-23T20:38:40.494049+0000","last_became_peered":"2023-09-23T20:38:40.494049+0000","last_unstale":"2023-09-23T20:41:09.135936+0000","last_undegraded":"2023-09-23T20:41:09.135936+0000","last_fullsized":"2023-09-23T20:41:09.135936+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:41:07.289420+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:41:07.289420+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:12:25.551623+0000","scrub_duration":0.0015464249999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":252,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:08.520299+0000","last_change":"2023-09-23T20:37:43.570610+0000","last_active":"2023-09-23T20:41:08.520299+0000","last_peered":"2023-09-23T20:41:08.520299+0000","last_clean":"2023-09-23T20:41:08.520299+0000","last_became_active":"2023-09-23T20:37:43.570216+0000","last_became_peered":"2023-09-23T20:37:43.570216+0000","last_unstale":"2023-09-23T20:41:08.520299+0000","last_undegraded":"2023-09-23T20:41:08.520299+0000","last_fullsized":"2023-09-23T20:41:08.520299+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:39:26.555329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":283,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:09.133072+0000","last_change":"2023-09-23T20:32:33.230582+0000","last_active":"2023-09-23T20:41:09.133072+0000","last_peered":"2023-09-23T20:41:09.133072+0000","last_clean":"2023-09-23T20:41:09.133072+0000","last_became_active":"2023-09-23T20:32:33.230257+0000","last_became_peered":"2023-09-23T20:32:33.230257+0000","last_unstale":"2023-09-23T20:41:09.133072+0000","last_undegraded":"2023-09-23T20:41:09.133072+0000","last_fullsized":"2023-09-23T20:41:09.133072+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:11:45.333096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.f","version":"459'2314","reported_seq":2324,"reported_epoch":459,"state":"active+clean","last_fresh":"2023-09-23T20:41:25.462353+0000","last_change":"2023-09-23T20:41:04.557353+0000","last_active":"2023-09-23T20:41:25.462353+0000","last_peered":"2023-09-23T20:41:25.462353+0000","last_clean":"2023-09-23T20:41:25.462353+0000","last_became_active":"2023-09-23T20:41:04.557037+0000","last_became_peered":"2023-09-23T20:41:04.557037+0000","last_unstale":"2023-09-23T20:41:25.462353+0000","last_undegraded":"2023-09-23T20:41:25.462353+0000","last_fullsized":"2023-09-23T20:41:25.462353+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":455,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":2314,"log_dups_size":0,"ondisk_log_size":2314,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:23:30.329193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":151650304,"num_objects":2314,"num_object_clones":0,"num_object_copies":6942,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2314,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2314,"num_write_kb":148096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":647,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:08.381395+0000","last_change":"2023-09-23T20:41:07.424609+0000","last_active":"2023-09-23T20:41:08.381395+0000","last_peered":"2023-09-23T20:41:08.381395+0000","last_clean":"2023-09-23T20:41:08.381395+0000","last_became_active":"2023-09-23T20:38:05.622261+0000","last_became_peered":"2023-09-23T20:38:05.622261+0000","last_unstale":"2023-09-23T20:41:08.381395+0000","last_undegraded":"2023-09-23T20:41:08.381395+0000","last_fullsized":"2023-09-23T20:41:08.381395+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:41:07.424492+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:41:07.424492+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:45:21.689123+0000","scrub_duration":0.0018960959999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"86'119","reported_seq":767,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:09.133426+0000","last_change":"2023-09-23T20:29:45.521257+0000","last_active":"2023-09-23T20:41:09.133426+0000","last_peered":"2023-09-23T20:41:09.133426+0000","last_clean":"2023-09-23T20:41:09.133426+0000","last_became_active":"2023-09-23T20:29:45.520546+0000","last_became_peered":"2023-09-23T20:29:45.520546+0000","last_unstale":"2023-09-23T20:41:09.133426+0000","last_undegraded":"2023-09-23T20:41:09.133426+0000","last_fullsized":"2023-09-23T20:41:09.133426+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":119,"log_dups_size":0,"ondisk_log_size":119,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:58:56.085777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032705499999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":523,"num_read_kb":3503,"num_write":136,"num_write_kb":3128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":28,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:09.136362+0000","last_change":"2023-09-23T20:40:19.557009+0000","last_active":"2023-09-23T20:41:09.136362+0000","last_peered":"2023-09-23T20:41:09.136362+0000","last_clean":"2023-09-23T20:41:09.136362+0000","last_became_active":"2023-09-23T20:40:19.550473+0000","last_became_peered":"2023-09-23T20:40:19.550473+0000","last_unstale":"2023-09-23T20:41:09.136362+0000","last_undegraded":"2023-09-23T20:41:09.136362+0000","last_fullsized":"2023-09-23T20:41:09.136362+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:35:01.140258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":414,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:09.132193+0000","last_change":"2023-09-23T20:40:19.530942+0000","last_active":"2023-09-23T20:41:09.132193+0000","last_peered":"2023-09-23T20:41:09.132193+0000","last_clean":"2023-09-23T20:41:09.132193+0000","last_became_active":"2023-09-23T20:40:19.530617+0000","last_became_peered":"2023-09-23T20:40:19.530617+0000","last_unstale":"2023-09-23T20:41:09.132193+0000","last_undegraded":"2023-09-23T20:41:09.132193+0000","last_fullsized":"2023-09-23T20:41:09.132193+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:24:09.231934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.9","version":"459'1906","reported_seq":1919,"reported_epoch":459,"state":"active+clean","last_fresh":"2023-09-23T20:41:21.257269+0000","last_change":"2023-09-23T20:41:04.556637+0000","last_active":"2023-09-23T20:41:21.257269+0000","last_peered":"2023-09-23T20:41:21.257269+0000","last_clean":"2023-09-23T20:41:21.257269+0000","last_became_active":"2023-09-23T20:41:04.555979+0000","last_became_peered":"2023-09-23T20:41:04.555979+0000","last_unstale":"2023-09-23T20:41:21.257269+0000","last_undegraded":"2023-09-23T20:41:21.257269+0000","last_fullsized":"2023-09-23T20:41:21.257269+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":455,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":1906,"log_dups_size":0,"ondisk_log_size":1906,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:45:38.600613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":124911616,"num_objects":1906,"num_object_clones":0,"num_object_copies":5718,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1906,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1906,"num_write_kb":121984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":532,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:09.135886+0000","last_change":"2023-09-23T20:40:22.445201+0000","last_active":"2023-09-23T20:41:09.135886+0000","last_peered":"2023-09-23T20:41:09.135886+0000","last_clean":"2023-09-23T20:41:09.135886+0000","last_became_active":"2023-09-23T20:40:22.444703+0000","last_became_peered":"2023-09-23T20:40:22.444703+0000","last_unstale":"2023-09-23T20:41:09.135886+0000","last_undegraded":"2023-09-23T20:41:09.135886+0000","last_fullsized":"2023-09-23T20:41:09.135886+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:45.785450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:45.785450+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"86'106","reported_seq":1178,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:09.136128+0000","last_change":"2023-09-23T20:38:37.476512+0000","last_active":"2023-09-23T20:41:09.136128+0000","last_peered":"2023-09-23T20:41:09.136128+0000","last_clean":"2023-09-23T20:41:09.136128+0000","last_became_active":"2023-09-23T20:38:37.475980+0000","last_became_peered":"2023-09-23T20:38:37.475980+0000","last_unstale":"2023-09-23T20:41:09.136128+0000","last_undegraded":"2023-09-23T20:41:09.136128+0000","last_fullsized":"2023-09-23T20:41:09.136128+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":106,"log_dups_size":0,"ondisk_log_size":106,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:58:45.508831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00099302599999999993,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1094,"num_read_kb":1678,"num_write":100,"num_write_kb":404,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":177,"reported_epoch":459,"state":"active+clean","last_fresh":"2023-09-23T20:41:11.586098+0000","last_change":"2023-09-23T20:38:02.552008+0000","last_active":"2023-09-23T20:41:11.586098+0000","last_peered":"2023-09-23T20:41:11.586098+0000","last_clean":"2023-09-23T20:41:11.586098+0000","last_became_active":"2023-09-23T20:38:02.551684+0000","last_became_peered":"2023-09-23T20:38:02.551684+0000","last_unstale":"2023-09-23T20:41:11.586098+0000","last_undegraded":"2023-09-23T20:41:11.586098+0000","last_fullsized":"2023-09-23T20:41:11.586098+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:40:37.147538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":420,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:09.133342+0000","last_change":"2023-09-23T20:29:45.533406+0000","last_active":"2023-09-23T20:41:09.133342+0000","last_peered":"2023-09-23T20:41:09.133342+0000","last_clean":"2023-09-23T20:41:09.133342+0000","last_became_active":"2023-09-23T20:29:45.526001+0000","last_became_peered":"2023-09-23T20:29:45.526001+0000","last_unstale":"2023-09-23T20:41:09.133342+0000","last_undegraded":"2023-09-23T20:41:09.133342+0000","last_fullsized":"2023-09-23T20:41:09.133342+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:43.787251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.a","version":"459'2269","reported_seq":2280,"reported_epoch":459,"state":"active+clean","last_fresh":"2023-09-23T20:41:25.472203+0000","last_change":"2023-09-23T20:41:04.556831+0000","last_active":"2023-09-23T20:41:25.472203+0000","last_peered":"2023-09-23T20:41:25.472203+0000","last_clean":"2023-09-23T20:41:25.472203+0000","last_became_active":"2023-09-23T20:41:04.556573+0000","last_became_peered":"2023-09-23T20:41:04.556573+0000","last_unstale":"2023-09-23T20:41:25.472203+0000","last_undegraded":"2023-09-23T20:41:25.472203+0000","last_fullsized":"2023-09-23T20:41:25.472203+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":455,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":2269,"log_dups_size":0,"ondisk_log_size":2269,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:55:53.953463+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":148701184,"num_objects":2269,"num_object_clones":0,"num_object_copies":6807,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2269,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2269,"num_write_kb":145216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":931,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:08.768989+0000","last_change":"2023-09-23T20:35:02.260708+0000","last_active":"2023-09-23T20:41:08.768989+0000","last_peered":"2023-09-23T20:41:08.768989+0000","last_clean":"2023-09-23T20:41:08.768989+0000","last_became_active":"2023-09-23T20:35:02.260348+0000","last_became_peered":"2023-09-23T20:35:02.260348+0000","last_unstale":"2023-09-23T20:41:08.768989+0000","last_undegraded":"2023-09-23T20:41:08.768989+0000","last_fullsized":"2023-09-23T20:41:08.768989+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:43.390908+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:43.390908+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:09:14.144070+0000","scrub_duration":0.0032721,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"86'130","reported_seq":775,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:08.619332+0000","last_change":"2023-09-23T20:25:39.637458+0000","last_active":"2023-09-23T20:41:08.619332+0000","last_peered":"2023-09-23T20:41:08.619332+0000","last_clean":"2023-09-23T20:41:08.619332+0000","last_became_active":"2023-09-23T20:25:39.636773+0000","last_became_peered":"2023-09-23T20:25:39.636773+0000","last_unstale":"2023-09-23T20:41:08.619332+0000","last_undegraded":"2023-09-23T20:41:08.619332+0000","last_fullsized":"2023-09-23T20:41:08.619332+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:00:32.810564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011300049999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":512,"num_read_kb":1820,"num_write":133,"num_write_kb":1347,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":428,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:09.132095+0000","last_change":"2023-09-23T20:38:02.591335+0000","last_active":"2023-09-23T20:41:09.132095+0000","last_peered":"2023-09-23T20:41:09.132095+0000","last_clean":"2023-09-23T20:41:09.132095+0000","last_became_active":"2023-09-23T20:38:02.591015+0000","last_became_peered":"2023-09-23T20:38:02.591015+0000","last_unstale":"2023-09-23T20:41:09.132095+0000","last_undegraded":"2023-09-23T20:41:09.132095+0000","last_fullsized":"2023-09-23T20:41:09.132095+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:55:32.272195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":420,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:09.133002+0000","last_change":"2023-09-23T20:40:19.544333+0000","last_active":"2023-09-23T20:41:09.133002+0000","last_peered":"2023-09-23T20:41:09.133002+0000","last_clean":"2023-09-23T20:41:09.133002+0000","last_became_active":"2023-09-23T20:40:19.543987+0000","last_became_peered":"2023-09-23T20:40:19.543987+0000","last_unstale":"2023-09-23T20:41:09.133002+0000","last_undegraded":"2023-09-23T20:41:09.133002+0000","last_fullsized":"2023-09-23T20:41:09.133002+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:12:59.203296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.8","version":"459'2137","reported_seq":2150,"reported_epoch":459,"state":"active+clean","last_fresh":"2023-09-23T20:41:23.995369+0000","last_change":"2023-09-23T20:41:04.550745+0000","last_active":"2023-09-23T20:41:23.995369+0000","last_peered":"2023-09-23T20:41:23.995369+0000","last_clean":"2023-09-23T20:41:23.995369+0000","last_became_active":"2023-09-23T20:41:04.550493+0000","last_became_peered":"2023-09-23T20:41:04.550493+0000","last_unstale":"2023-09-23T20:41:23.995369+0000","last_undegraded":"2023-09-23T20:41:23.995369+0000","last_fullsized":"2023-09-23T20:41:23.995369+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":455,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":2137,"log_dups_size":0,"ondisk_log_size":2137,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:28:41.080518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":140050432,"num_objects":2137,"num_object_clones":0,"num_object_copies":6411,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2137,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2137,"num_write_kb":136768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"86'194","reported_seq":811,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:09.135872+0000","last_change":"2023-09-23T20:40:51.458056+0000","last_active":"2023-09-23T20:41:09.135872+0000","last_peered":"2023-09-23T20:41:09.135872+0000","last_clean":"2023-09-23T20:41:09.135872+0000","last_became_active":"2023-09-23T20:40:51.457465+0000","last_became_peered":"2023-09-23T20:40:51.457465+0000","last_unstale":"2023-09-23T20:41:09.135872+0000","last_undegraded":"2023-09-23T20:41:09.135872+0000","last_fullsized":"2023-09-23T20:41:09.135872+0000","mapping_epoch":450,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":451,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":194,"log_dups_size":0,"ondisk_log_size":194,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:18:36.482553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00037995999999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1024,"num_read_kb":1200,"num_write":173,"num_write_kb":1276,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":921,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:08.768789+0000","last_change":"2023-09-23T20:38:05.602508+0000","last_active":"2023-09-23T20:41:08.768789+0000","last_peered":"2023-09-23T20:41:08.768789+0000","last_clean":"2023-09-23T20:41:08.768789+0000","last_became_active":"2023-09-23T20:38:05.602129+0000","last_became_peered":"2023-09-23T20:38:05.602129+0000","last_unstale":"2023-09-23T20:41:08.768789+0000","last_undegraded":"2023-09-23T20:41:08.768789+0000","last_fullsized":"2023-09-23T20:41:08.768789+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:45.343742+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:45.343742+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:48.910343+0000","scrub_duration":0.001475802,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":28,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:09.135847+0000","last_change":"2023-09-23T20:40:19.542142+0000","last_active":"2023-09-23T20:41:09.135847+0000","last_peered":"2023-09-23T20:41:09.135847+0000","last_clean":"2023-09-23T20:41:09.135847+0000","last_became_active":"2023-09-23T20:40:19.541870+0000","last_became_peered":"2023-09-23T20:40:19.541870+0000","last_unstale":"2023-09-23T20:41:09.135847+0000","last_undegraded":"2023-09-23T20:41:09.135847+0000","last_fullsized":"2023-09-23T20:41:09.135847+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:01.518149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":64,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:08.381338+0000","last_change":"2023-09-23T20:38:02.573200+0000","last_active":"2023-09-23T20:41:08.381338+0000","last_peered":"2023-09-23T20:41:08.381338+0000","last_clean":"2023-09-23T20:41:08.381338+0000","last_became_active":"2023-09-23T20:38:02.572945+0000","last_became_peered":"2023-09-23T20:38:02.572945+0000","last_unstale":"2023-09-23T20:41:08.381338+0000","last_undegraded":"2023-09-23T20:41:08.381338+0000","last_fullsized":"2023-09-23T20:41:08.381338+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:57.246533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.b","version":"459'1837","reported_seq":1850,"reported_epoch":459,"state":"active+clean","last_fresh":"2023-09-23T20:41:21.264498+0000","last_change":"2023-09-23T20:41:04.557718+0000","last_active":"2023-09-23T20:41:21.264498+0000","last_peered":"2023-09-23T20:41:21.264498+0000","last_clean":"2023-09-23T20:41:21.264498+0000","last_became_active":"2023-09-23T20:41:04.557420+0000","last_became_peered":"2023-09-23T20:41:04.557420+0000","last_unstale":"2023-09-23T20:41:21.264498+0000","last_undegraded":"2023-09-23T20:41:21.264498+0000","last_fullsized":"2023-09-23T20:41:21.264498+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":455,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":1837,"log_dups_size":0,"ondisk_log_size":1837,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:40:23.719326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":120389632,"num_objects":1837,"num_object_clones":0,"num_object_copies":5511,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1837,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1837,"num_write_kb":117568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":789,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:09.132585+0000","last_change":"2023-09-23T20:38:05.615711+0000","last_active":"2023-09-23T20:41:09.132585+0000","last_peered":"2023-09-23T20:41:09.132585+0000","last_clean":"2023-09-23T20:41:09.132585+0000","last_became_active":"2023-09-23T20:38:05.615390+0000","last_became_peered":"2023-09-23T20:38:05.615390+0000","last_unstale":"2023-09-23T20:41:09.132585+0000","last_undegraded":"2023-09-23T20:41:09.132585+0000","last_fullsized":"2023-09-23T20:41:09.132585+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:43.582146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:43.582146+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:45:11.917871+0000","scrub_duration":0.0016219730000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"86'80","reported_seq":711,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:08.768596+0000","last_change":"2023-09-23T20:29:45.536796+0000","last_active":"2023-09-23T20:41:08.768596+0000","last_peered":"2023-09-23T20:41:08.768596+0000","last_clean":"2023-09-23T20:41:08.768596+0000","last_became_active":"2023-09-23T20:29:45.536439+0000","last_became_peered":"2023-09-23T20:29:45.536439+0000","last_unstale":"2023-09-23T20:41:08.768596+0000","last_undegraded":"2023-09-23T20:41:08.768596+0000","last_fullsized":"2023-09-23T20:41:08.768596+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":80,"log_dups_size":0,"ondisk_log_size":80,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:30:57.793132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035115,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":395,"num_read_kb":1370,"num_write":66,"num_write_kb":1061,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":464,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:07.553350+0000","last_change":"2023-09-23T20:34:59.984286+0000","last_active":"2023-09-23T20:41:07.553350+0000","last_peered":"2023-09-23T20:41:07.553350+0000","last_clean":"2023-09-23T20:41:07.553350+0000","last_became_active":"2023-09-23T20:34:59.983950+0000","last_became_peered":"2023-09-23T20:34:59.983950+0000","last_unstale":"2023-09-23T20:41:07.553350+0000","last_undegraded":"2023-09-23T20:41:07.553350+0000","last_fullsized":"2023-09-23T20:41:07.553350+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:31:53.557035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":458,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:07.553793+0000","last_change":"2023-09-23T20:29:45.690815+0000","last_active":"2023-09-23T20:41:07.553793+0000","last_peered":"2023-09-23T20:41:07.553793+0000","last_clean":"2023-09-23T20:41:07.553793+0000","last_became_active":"2023-09-23T20:29:45.690478+0000","last_became_peered":"2023-09-23T20:29:45.690478+0000","last_unstale":"2023-09-23T20:41:07.553793+0000","last_undegraded":"2023-09-23T20:41:07.553793+0000","last_fullsized":"2023-09-23T20:41:07.553793+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:38.288193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.e","version":"459'1981","reported_seq":1993,"reported_epoch":459,"state":"active+clean","last_fresh":"2023-09-23T20:41:22.361151+0000","last_change":"2023-09-23T20:41:04.574876+0000","last_active":"2023-09-23T20:41:22.361151+0000","last_peered":"2023-09-23T20:41:22.361151+0000","last_clean":"2023-09-23T20:41:22.361151+0000","last_became_active":"2023-09-23T20:41:04.574595+0000","last_became_peered":"2023-09-23T20:41:04.574595+0000","last_unstale":"2023-09-23T20:41:22.361151+0000","last_undegraded":"2023-09-23T20:41:22.361151+0000","last_fullsized":"2023-09-23T20:41:22.361151+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":455,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":1981,"log_dups_size":0,"ondisk_log_size":1981,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:31:00.147966+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":129826816,"num_objects":1981,"num_object_clones":0,"num_object_copies":5943,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1981,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1981,"num_write_kb":126784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":570,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:08.381163+0000","last_change":"2023-09-23T20:41:08.380084+0000","last_active":"2023-09-23T20:41:08.381163+0000","last_peered":"2023-09-23T20:41:08.381163+0000","last_clean":"2023-09-23T20:41:08.381163+0000","last_became_active":"2023-09-23T20:38:05.610881+0000","last_became_peered":"2023-09-23T20:38:05.610881+0000","last_unstale":"2023-09-23T20:41:08.381163+0000","last_undegraded":"2023-09-23T20:41:08.381163+0000","last_fullsized":"2023-09-23T20:41:08.381163+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:41:08.379959+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:41:08.379959+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:19:39.547923+0000","scrub_duration":0.0019158420000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"86'102","reported_seq":1053,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:08.768623+0000","last_change":"2023-09-23T20:32:36.314672+0000","last_active":"2023-09-23T20:41:08.768623+0000","last_peered":"2023-09-23T20:41:08.768623+0000","last_clean":"2023-09-23T20:41:08.768623+0000","last_became_active":"2023-09-23T20:32:36.314231+0000","last_became_peered":"2023-09-23T20:32:36.314231+0000","last_unstale":"2023-09-23T20:41:08.768623+0000","last_undegraded":"2023-09-23T20:41:08.768623+0000","last_fullsized":"2023-09-23T20:41:08.768623+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":102,"log_dups_size":0,"ondisk_log_size":102,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:11:40.514709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030656500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":963,"num_read_kb":1234,"num_write":60,"num_write_kb":225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":404,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:09.133512+0000","last_change":"2023-09-23T20:40:19.544384+0000","last_active":"2023-09-23T20:41:09.133512+0000","last_peered":"2023-09-23T20:41:09.133512+0000","last_clean":"2023-09-23T20:41:09.133512+0000","last_became_active":"2023-09-23T20:40:19.544079+0000","last_became_peered":"2023-09-23T20:40:19.544079+0000","last_unstale":"2023-09-23T20:41:09.133512+0000","last_undegraded":"2023-09-23T20:41:09.133512+0000","last_fullsized":"2023-09-23T20:41:09.133512+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:00:19.498631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":414,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:07.553669+0000","last_change":"2023-09-23T20:40:19.532976+0000","last_active":"2023-09-23T20:41:07.553669+0000","last_peered":"2023-09-23T20:41:07.553669+0000","last_clean":"2023-09-23T20:41:07.553669+0000","last_became_active":"2023-09-23T20:40:19.532639+0000","last_became_peered":"2023-09-23T20:40:19.532639+0000","last_unstale":"2023-09-23T20:41:07.553669+0000","last_undegraded":"2023-09-23T20:41:07.553669+0000","last_fullsized":"2023-09-23T20:41:07.553669+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:02:35.301884+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"459'2076","reported_seq":2088,"reported_epoch":459,"state":"active+clean","last_fresh":"2023-09-23T20:41:22.352380+0000","last_change":"2023-09-23T20:41:04.558941+0000","last_active":"2023-09-23T20:41:22.352380+0000","last_peered":"2023-09-23T20:41:22.352380+0000","last_clean":"2023-09-23T20:41:22.352380+0000","last_became_active":"2023-09-23T20:41:04.558482+0000","last_became_peered":"2023-09-23T20:41:04.558482+0000","last_unstale":"2023-09-23T20:41:22.352380+0000","last_undegraded":"2023-09-23T20:41:22.352380+0000","last_fullsized":"2023-09-23T20:41:22.352380+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":455,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":2076,"log_dups_size":0,"ondisk_log_size":2076,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:36:14.259949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":136052736,"num_objects":2076,"num_object_clones":0,"num_object_copies":6228,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2076,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2076,"num_write_kb":132864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":738,"reported_epoch":459,"state":"active+clean","last_fresh":"2023-09-23T20:41:11.585504+0000","last_change":"2023-09-23T20:38:05.590361+0000","last_active":"2023-09-23T20:41:11.585504+0000","last_peered":"2023-09-23T20:41:11.585504+0000","last_clean":"2023-09-23T20:41:11.585504+0000","last_became_active":"2023-09-23T20:38:05.589985+0000","last_became_peered":"2023-09-23T20:38:05.589985+0000","last_unstale":"2023-09-23T20:41:11.585504+0000","last_undegraded":"2023-09-23T20:41:11.585504+0000","last_fullsized":"2023-09-23T20:41:11.585504+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:03.246271+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:03.246271+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:38:54.351576+0000","scrub_duration":0.0019016700000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":196,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:08.769616+0000","last_change":"2023-09-23T20:32:50.549695+0000","last_active":"2023-09-23T20:41:08.769616+0000","last_peered":"2023-09-23T20:41:08.769616+0000","last_clean":"2023-09-23T20:41:08.769616+0000","last_became_active":"2023-09-23T20:32:50.549431+0000","last_became_peered":"2023-09-23T20:32:50.549431+0000","last_unstale":"2023-09-23T20:41:08.769616+0000","last_undegraded":"2023-09-23T20:41:08.769616+0000","last_fullsized":"2023-09-23T20:41:08.769616+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:45:53.666569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"86'128","reported_seq":870,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:09.133064+0000","last_change":"2023-09-23T20:38:03.541765+0000","last_active":"2023-09-23T20:41:09.133064+0000","last_peered":"2023-09-23T20:41:09.133064+0000","last_clean":"2023-09-23T20:41:09.133064+0000","last_became_active":"2023-09-23T20:38:03.541277+0000","last_became_peered":"2023-09-23T20:38:03.541277+0000","last_unstale":"2023-09-23T20:41:09.133064+0000","last_undegraded":"2023-09-23T20:41:09.133064+0000","last_fullsized":"2023-09-23T20:41:09.133064+0000","mapping_epoch":404,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":405,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:20:53.834554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000293202,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":579,"num_read_kb":1473,"num_write":96,"num_write_kb":1062,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":237,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:08.520206+0000","last_change":"2023-09-23T20:37:43.582105+0000","last_active":"2023-09-23T20:41:08.520206+0000","last_peered":"2023-09-23T20:41:08.520206+0000","last_clean":"2023-09-23T20:41:08.520206+0000","last_became_active":"2023-09-23T20:37:43.581860+0000","last_became_peered":"2023-09-23T20:37:43.581860+0000","last_unstale":"2023-09-23T20:41:08.520206+0000","last_undegraded":"2023-09-23T20:41:08.520206+0000","last_fullsized":"2023-09-23T20:41:08.520206+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:18:38.765486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.c","version":"459'2160","reported_seq":2172,"reported_epoch":459,"state":"active+clean","last_fresh":"2023-09-23T20:41:23.993399+0000","last_change":"2023-09-23T20:41:04.575496+0000","last_active":"2023-09-23T20:41:23.993399+0000","last_peered":"2023-09-23T20:41:23.993399+0000","last_clean":"2023-09-23T20:41:23.993399+0000","last_became_active":"2023-09-23T20:41:04.575242+0000","last_became_peered":"2023-09-23T20:41:04.575242+0000","last_unstale":"2023-09-23T20:41:23.993399+0000","last_undegraded":"2023-09-23T20:41:23.993399+0000","last_fullsized":"2023-09-23T20:41:23.993399+0000","mapping_epoch":455,"log_start":"0'0","ondisk_log_start":"0'0","created":455,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":2160,"log_dups_size":0,"ondisk_log_size":2160,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:30:14.611529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":141557760,"num_objects":2160,"num_object_clones":0,"num_object_copies":6480,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2160,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2160,"num_write_kb":138240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"90'161","reported_seq":671,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:08.520608+0000","last_change":"2023-09-23T20:40:22.427435+0000","last_active":"2023-09-23T20:41:08.520608+0000","last_peered":"2023-09-23T20:41:08.520608+0000","last_clean":"2023-09-23T20:41:08.520608+0000","last_became_active":"2023-09-23T20:40:22.427051+0000","last_became_peered":"2023-09-23T20:40:22.427051+0000","last_unstale":"2023-09-23T20:41:08.520608+0000","last_undegraded":"2023-09-23T20:41:08.520608+0000","last_fullsized":"2023-09-23T20:41:08.520608+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:34:54.504554+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:34:54.504554+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:42.001652+0000","scrub_duration":0.011176864,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"86'117","reported_seq":691,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:08.381352+0000","last_change":"2023-09-23T20:38:28.446107+0000","last_active":"2023-09-23T20:41:08.381352+0000","last_peered":"2023-09-23T20:41:08.381352+0000","last_clean":"2023-09-23T20:41:08.381352+0000","last_became_active":"2023-09-23T20:38:28.445667+0000","last_became_peered":"2023-09-23T20:38:28.445667+0000","last_unstale":"2023-09-23T20:41:08.381352+0000","last_undegraded":"2023-09-23T20:41:08.381352+0000","last_fullsized":"2023-09-23T20:41:08.381352+0000","mapping_epoch":411,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":412,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:58:47.430360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046891400000000002,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":893,"num_read_kb":922,"num_write":82,"num_write_kb":4193,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":458,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:07.553865+0000","last_change":"2023-09-23T20:32:33.277827+0000","last_active":"2023-09-23T20:41:07.553865+0000","last_peered":"2023-09-23T20:41:07.553865+0000","last_clean":"2023-09-23T20:41:07.553865+0000","last_became_active":"2023-09-23T20:32:33.236087+0000","last_became_peered":"2023-09-23T20:32:33.236087+0000","last_unstale":"2023-09-23T20:41:07.553865+0000","last_undegraded":"2023-09-23T20:41:07.553865+0000","last_fullsized":"2023-09-23T20:41:07.553865+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:20:58.176854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":197,"reported_epoch":459,"state":"active+clean","last_fresh":"2023-09-23T20:41:11.585131+0000","last_change":"2023-09-23T20:40:19.530602+0000","last_active":"2023-09-23T20:41:11.585131+0000","last_peered":"2023-09-23T20:41:11.585131+0000","last_clean":"2023-09-23T20:41:11.585131+0000","last_became_active":"2023-09-23T20:40:19.530256+0000","last_became_peered":"2023-09-23T20:40:19.530256+0000","last_unstale":"2023-09-23T20:41:11.585131+0000","last_undegraded":"2023-09-23T20:41:11.585131+0000","last_fullsized":"2023-09-23T20:41:11.585131+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:18:45.658867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"90'161","reported_seq":728,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:07.553026+0000","last_change":"2023-09-23T20:38:05.630221+0000","last_active":"2023-09-23T20:41:07.553026+0000","last_peered":"2023-09-23T20:41:07.553026+0000","last_clean":"2023-09-23T20:41:07.553026+0000","last_became_active":"2023-09-23T20:38:05.629839+0000","last_became_peered":"2023-09-23T20:38:05.629839+0000","last_unstale":"2023-09-23T20:41:07.553026+0000","last_undegraded":"2023-09-23T20:41:07.553026+0000","last_fullsized":"2023-09-23T20:41:07.553026+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:06.487351+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:06.487351+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:25:51.325529+0000","scrub_duration":0.001964379,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":426,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:07.553625+0000","last_change":"2023-09-23T20:32:33.237234+0000","last_active":"2023-09-23T20:41:07.553625+0000","last_peered":"2023-09-23T20:41:07.553625+0000","last_clean":"2023-09-23T20:41:07.553625+0000","last_became_active":"2023-09-23T20:32:33.236881+0000","last_became_peered":"2023-09-23T20:32:33.236881+0000","last_unstale":"2023-09-23T20:41:07.553625+0000","last_undegraded":"2023-09-23T20:41:07.553625+0000","last_fullsized":"2023-09-23T20:41:07.553625+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:56:17.384831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":314,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:08.619932+0000","last_change":"2023-09-23T20:29:45.538314+0000","last_active":"2023-09-23T20:41:08.619932+0000","last_peered":"2023-09-23T20:41:08.619932+0000","last_clean":"2023-09-23T20:41:08.619932+0000","last_became_active":"2023-09-23T20:29:45.536847+0000","last_became_peered":"2023-09-23T20:29:45.536847+0000","last_unstale":"2023-09-23T20:41:08.619932+0000","last_undegraded":"2023-09-23T20:41:08.619932+0000","last_fullsized":"2023-09-23T20:41:08.619932+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:51.948721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"90'161","reported_seq":530,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:08.520922+0000","last_change":"2023-09-23T20:41:06.499511+0000","last_active":"2023-09-23T20:41:08.520922+0000","last_peered":"2023-09-23T20:41:08.520922+0000","last_clean":"2023-09-23T20:41:08.520922+0000","last_became_active":"2023-09-23T20:38:05.601874+0000","last_became_peered":"2023-09-23T20:38:05.601874+0000","last_unstale":"2023-09-23T20:41:08.520922+0000","last_undegraded":"2023-09-23T20:41:08.520922+0000","last_fullsized":"2023-09-23T20:41:08.520922+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:41:06.499396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:41:06.499396+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:31:34.059077+0000","scrub_duration":0.0019670310000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":57,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:09.136061+0000","last_change":"2023-09-23T20:38:37.481551+0000","last_active":"2023-09-23T20:41:09.136061+0000","last_peered":"2023-09-23T20:41:09.136061+0000","last_clean":"2023-09-23T20:41:09.136061+0000","last_became_active":"2023-09-23T20:38:37.458687+0000","last_became_peered":"2023-09-23T20:38:37.458687+0000","last_unstale":"2023-09-23T20:41:09.136061+0000","last_undegraded":"2023-09-23T20:41:09.136061+0000","last_fullsized":"2023-09-23T20:41:09.136061+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:22:15.033243+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":116,"reported_epoch":459,"state":"active+clean","last_fresh":"2023-09-23T20:41:11.585781+0000","last_change":"2023-09-23T20:34:59.987575+0000","last_active":"2023-09-23T20:41:11.585781+0000","last_peered":"2023-09-23T20:41:11.585781+0000","last_clean":"2023-09-23T20:41:11.585781+0000","last_became_active":"2023-09-23T20:34:59.987229+0000","last_became_peered":"2023-09-23T20:34:59.987229+0000","last_unstale":"2023-09-23T20:41:11.585781+0000","last_undegraded":"2023-09-23T20:41:11.585781+0000","last_fullsized":"2023-09-23T20:41:11.585781+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:34:01.820159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":57,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:09.136453+0000","last_change":"2023-09-23T20:38:37.457634+0000","last_active":"2023-09-23T20:41:09.136453+0000","last_peered":"2023-09-23T20:41:09.136453+0000","last_clean":"2023-09-23T20:41:09.136453+0000","last_became_active":"2023-09-23T20:38:37.452208+0000","last_became_peered":"2023-09-23T20:38:37.452208+0000","last_unstale":"2023-09-23T20:41:09.136453+0000","last_undegraded":"2023-09-23T20:41:09.136453+0000","last_fullsized":"2023-09-23T20:41:09.136453+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:09:41.754569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":116,"reported_epoch":459,"state":"active+clean","last_fresh":"2023-09-23T20:41:11.585176+0000","last_change":"2023-09-23T20:34:59.987416+0000","last_active":"2023-09-23T20:41:11.585176+0000","last_peered":"2023-09-23T20:41:11.585176+0000","last_clean":"2023-09-23T20:41:11.585176+0000","last_became_active":"2023-09-23T20:34:59.987117+0000","last_became_peered":"2023-09-23T20:34:59.987117+0000","last_unstale":"2023-09-23T20:41:11.585176+0000","last_undegraded":"2023-09-23T20:41:11.585176+0000","last_fullsized":"2023-09-23T20:41:11.585176+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:36:16.318623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"90'161","reported_seq":714,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:08.769024+0000","last_change":"2023-09-23T20:38:05.591992+0000","last_active":"2023-09-23T20:41:08.769024+0000","last_peered":"2023-09-23T20:41:08.769024+0000","last_clean":"2023-09-23T20:41:08.769024+0000","last_became_active":"2023-09-23T20:38:05.591701+0000","last_became_peered":"2023-09-23T20:38:05.591701+0000","last_unstale":"2023-09-23T20:41:08.769024+0000","last_undegraded":"2023-09-23T20:41:08.769024+0000","last_fullsized":"2023-09-23T20:41:08.769024+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:02.384133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:02.384133+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:41.722542+0000","scrub_duration":0.001769098,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":279,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:08.521032+0000","last_change":"2023-09-23T20:38:02.542719+0000","last_active":"2023-09-23T20:41:08.521032+0000","last_peered":"2023-09-23T20:41:08.521032+0000","last_clean":"2023-09-23T20:41:08.521032+0000","last_became_active":"2023-09-23T20:38:02.542385+0000","last_became_peered":"2023-09-23T20:38:02.542385+0000","last_unstale":"2023-09-23T20:41:08.521032+0000","last_undegraded":"2023-09-23T20:41:08.521032+0000","last_fullsized":"2023-09-23T20:41:08.521032+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:36:23.673757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":246,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:08.520519+0000","last_change":"2023-09-23T20:37:43.584253+0000","last_active":"2023-09-23T20:41:08.520519+0000","last_peered":"2023-09-23T20:41:08.520519+0000","last_clean":"2023-09-23T20:41:08.520519+0000","last_became_active":"2023-09-23T20:37:43.580069+0000","last_became_peered":"2023-09-23T20:37:43.580069+0000","last_unstale":"2023-09-23T20:41:08.520519+0000","last_undegraded":"2023-09-23T20:41:08.520519+0000","last_fullsized":"2023-09-23T20:41:08.520519+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:39:11.482581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"90'161","reported_seq":551,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:09.132673+0000","last_change":"2023-09-23T20:36:01.505098+0000","last_active":"2023-09-23T20:41:09.132673+0000","last_peered":"2023-09-23T20:41:09.132673+0000","last_clean":"2023-09-23T20:41:09.132673+0000","last_became_active":"2023-09-23T20:35:26.644707+0000","last_became_peered":"2023-09-23T20:35:26.644707+0000","last_unstale":"2023-09-23T20:41:09.132673+0000","last_undegraded":"2023-09-23T20:41:09.132673+0000","last_fullsized":"2023-09-23T20:41:09.132673+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:01.504973+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:01.504973+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:02:10.880809+0000","scrub_duration":0.00202344,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":504,"reported_epoch":458,"state":"active+clean","last_fresh":"2023-09-23T20:41:07.554120+0000","last_change":"2023-09-23T20:40:19.569032+0000","last_active":"2023-09-23T20:41:07.554120+0000","last_peered":"2023-09-23T20:41:07.554120+0000","last_clean":"2023-09-23T20:41:07.554120+0000","last_became_active":"2023-09-23T20:40:19.538476+0000","last_became_peered":"2023-09-23T20:40:19.538476+0000","last_unstale":"2023-09-23T20:41:07.554120+0000","last_undegraded":"2023-09-23T20:41:07.554120+0000","last_fullsized":"2023-09-23T20:41:07.554120+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:31:58.998168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3324,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":177,"reported_epoch":459,"state":"active+clean","last_fresh":"2023-09-23T20:41:11.584747+0000","last_change":"2023-09-23T20:38:02.538333+0000","last_active":"2023-09-23T20:41:11.584747+0000","last_peered":"2023-09-23T20:41:11.584747+0000","last_clean":"2023-09-23T20:41:11.584747+0000","last_became_active":"2023-09-23T20:38:02.538008+0000","last_became_peered":"2023-09-23T20:38:02.538008+0000","last_unstale":"2023-09-23T20:41:11.584747+0000","last_undegraded":"2023-09-23T20:41:11.584747+0000","last_fullsized":"2023-09-23T20:41:11.584747+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:02:38.789530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":80,"num_bytes_recovered":18486,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26691,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":381350,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":423698,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":2207645696,"num_objects":33686,"num_object_clones":0,"num_object_copies":101058,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":33686,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":33686,"num_write_kb":2155904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":6720716800,"data_stored":6720716800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":33686,"ondisk_log_size":33686,"up":48,"acting":48,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5983,"num_read_kb":13200,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":68,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6082,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201639,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1486660,"kb_used_data":1101116,"kb_used_omap":0,"kb_used_meta":385536,"kb_avail":92258492,"statfs":{"total":95995035648,"available":94472695808,"internally_reserved":0,"allocated":1127542784,"data_stored":1126094697,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":394788864},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:40:54 2023","interfaces":[{"interface":"back","average":{"1min":1.423,"5min":1.968,"15min":2.1520000000000001},"min":{"1min":0.27200000000000002,"5min":0.24399999999999999,"15min":0.20799999999999999},"max":{"1min":5.6310000000000002,"5min":18.315000000000001,"15min":48.798999999999999},"last":0.55800000000000005},{"interface":"front","average":{"1min":1.069,"5min":1.2909999999999999,"15min":1.974},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":5.3330000000000002,"5min":9.4429999999999996,"15min":48.247},"last":0.30099999999999999}]},{"osd":1,"last update":"Sat Sep 23 20:41:06 2023","interfaces":[{"interface":"back","average":{"1min":0.91400000000000003,"5min":2.2770000000000001,"15min":2.0960000000000001},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.254},"max":{"1min":4.2460000000000004,"5min":49.965000000000003,"15min":49.965000000000003},"last":0.314},{"interface":"front","average":{"1min":1.147,"5min":2.323,"15min":2.1160000000000001},"min":{"1min":0.29799999999999999,"5min":0.26900000000000002,"15min":0.248},"max":{"1min":4.2999999999999998,"5min":53.451999999999998,"15min":53.451999999999998},"last":2.452}]},{"osd":2,"last update":"Sat Sep 23 20:40:54 2023","interfaces":[{"interface":"back","average":{"1min":1.339,"5min":1.736,"15min":2.0840000000000001},"min":{"1min":0.32000000000000001,"5min":0.28599999999999998,"15min":0.27600000000000002},"max":{"1min":4.5039999999999996,"5min":16.965,"15min":16.965},"last":0.33800000000000002},{"interface":"front","average":{"1min":1.4630000000000001,"5min":2.1960000000000002,"15min":2.4489999999999998},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.26600000000000001},"max":{"1min":6.4329999999999998,"5min":23.414000000000001,"15min":23.494},"last":2.1110000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:40:21 2023","interfaces":[{"interface":"back","average":{"1min":1.105,"5min":2.2480000000000002,"15min":2.4470000000000001},"min":{"1min":0.35999999999999999,"5min":0.311,"15min":0.26200000000000001},"max":{"1min":4.2149999999999999,"5min":23.352,"15min":23.352},"last":1.425},{"interface":"front","average":{"1min":1.139,"5min":2.2709999999999999,"15min":2.423},"min":{"1min":0.46500000000000002,"5min":0.34399999999999997,"15min":0.32500000000000001},"max":{"1min":4.2359999999999998,"5min":20.969999999999999,"15min":20.969999999999999},"last":1.607}]},{"osd":4,"last update":"Sat Sep 23 20:40:21 2023","interfaces":[{"interface":"back","average":{"1min":1.046,"5min":1.085,"15min":1.2170000000000001},"min":{"1min":0.35799999999999998,"5min":0.25600000000000001,"15min":0.23200000000000001},"max":{"1min":4.1639999999999997,"5min":18.364999999999998,"15min":18.364999999999998},"last":0.34799999999999998},{"interface":"front","average":{"1min":0.94999999999999996,"5min":1.296,"15min":1.417},"min":{"1min":0.39100000000000001,"5min":0.28399999999999997,"15min":0.27600000000000002},"max":{"1min":2.605,"5min":23.617000000000001,"15min":23.617000000000001},"last":0.46700000000000003}]},{"osd":5,"last update":"Sat Sep 23 20:40:21 2023","interfaces":[{"interface":"back","average":{"1min":1.0780000000000001,"5min":1.1499999999999999,"15min":1.2509999999999999},"min":{"1min":0.30099999999999999,"5min":0.29199999999999998,"15min":0.26600000000000001},"max":{"1min":4.3780000000000001,"5min":12.516,"15min":13.241},"last":0.36899999999999999},{"interface":"front","average":{"1min":0.76400000000000001,"5min":1.286,"15min":1.3540000000000001},"min":{"1min":0.38400000000000001,"5min":0.31900000000000001,"15min":0.315},"max":{"1min":3.1030000000000002,"5min":18.390000000000001,"15min":18.831},"last":0.48599999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:40:21 2023","interfaces":[{"interface":"back","average":{"1min":1.2629999999999999,"5min":1.4350000000000001,"15min":1.329},"min":{"1min":0.44,"5min":0.312,"15min":0.30399999999999999},"max":{"1min":4.0540000000000003,"5min":18.417000000000002,"15min":18.417000000000002},"last":0.38300000000000001},{"interface":"front","average":{"1min":1.1659999999999999,"5min":1.113,"15min":1.252},"min":{"1min":0.38200000000000001,"5min":0.32700000000000001,"15min":0.318},"max":{"1min":6.4340000000000002,"5min":7.266,"15min":9.9120000000000008},"last":0.69399999999999995}]}]},{"osd":6,"up_from":47,"seq":201863463278,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":800376,"kb_used_data":395824,"kb_used_omap":0,"kb_used_meta":404544,"kb_avail":92944776,"statfs":{"total":95995035648,"available":95175450624,"internally_reserved":0,"allocated":405323776,"data_stored":403887306,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":414253056},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:40:50 2023","interfaces":[{"interface":"back","average":{"1min":1.1910000000000001,"5min":1.7470000000000001,"15min":1.9159999999999999},"min":{"1min":0.28100000000000003,"5min":0.23300000000000001,"15min":0.221},"max":{"1min":4.1399999999999997,"5min":17.492999999999999,"15min":53.701999999999998},"last":0.42699999999999999},{"interface":"front","average":{"1min":1.395,"5min":2.306,"15min":2.3399999999999999},"min":{"1min":0.253,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":6.0570000000000004,"5min":60.328000000000003,"15min":60.328000000000003},"last":0.54900000000000004}]},{"osd":1,"last update":"Sat Sep 23 20:41:09 2023","interfaces":[{"interface":"back","average":{"1min":0.66200000000000003,"5min":1.726,"15min":2.222},"min":{"1min":0.253,"5min":0.253,"15min":0.249},"max":{"1min":2.8250000000000002,"5min":64.274000000000001,"15min":64.274000000000001},"last":0.311},{"interface":"front","average":{"1min":1.173,"5min":1.2170000000000001,"15min":2.3290000000000002},"min":{"1min":0.32900000000000001,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":4.29,"5min":6.8460000000000001,"15min":34.490000000000002},"last":0.79100000000000004}]},{"osd":2,"last update":"Sat Sep 23 20:40:50 2023","interfaces":[{"interface":"back","average":{"1min":2.097,"5min":1.6419999999999999,"15min":2.133},"min":{"1min":0.31,"5min":0.30299999999999999,"15min":0.27700000000000002},"max":{"1min":9.9049999999999994,"5min":13.010999999999999,"15min":34.317999999999998},"last":0.35199999999999998},{"interface":"front","average":{"1min":2.1080000000000001,"5min":1.585,"15min":2.0569999999999999},"min":{"1min":0.311,"5min":0.311,"15min":0.30399999999999999},"max":{"1min":9.9269999999999996,"5min":12.917999999999999,"15min":34.292999999999999},"last":0.34000000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:40:36 2023","interfaces":[{"interface":"back","average":{"1min":2.0099999999999998,"5min":1.8759999999999999,"15min":1.9910000000000001},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":5.9130000000000003,"5min":8.9670000000000005,"15min":9.4239999999999995},"last":1.9410000000000001},{"interface":"front","average":{"1min":1.921,"5min":1.968,"15min":2.1659999999999999},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.317},"max":{"1min":5.8380000000000001,"5min":11.913,"15min":12.875999999999999},"last":0.52900000000000003}]},{"osd":4,"last update":"Sat Sep 23 20:41:14 2023","interfaces":[{"interface":"back","average":{"1min":0.63800000000000001,"5min":0.85499999999999998,"15min":0.83599999999999997},"min":{"1min":0.30299999999999999,"5min":0.28000000000000003,"15min":0.251},"max":{"1min":4.0830000000000002,"5min":23.904,"15min":23.904},"last":0.29699999999999999},{"interface":"front","average":{"1min":0.68500000000000005,"5min":1.0780000000000001,"15min":1.1140000000000001},"min":{"1min":0.309,"5min":0.29199999999999998,"15min":0.27000000000000002},"max":{"1min":2.7160000000000002,"5min":11.515000000000001,"15min":11.515000000000001},"last":0.37}]},{"osd":5,"last update":"Sat Sep 23 20:41:14 2023","interfaces":[{"interface":"back","average":{"1min":0.45800000000000002,"5min":0.76900000000000002,"15min":0.80900000000000005},"min":{"1min":0.34000000000000002,"5min":0.318,"15min":0.28699999999999998},"max":{"1min":0.78500000000000003,"5min":6.6340000000000003,"15min":8.4499999999999993},"last":0.51500000000000001},{"interface":"front","average":{"1min":0.76600000000000001,"5min":0.95699999999999996,"15min":1.179},"min":{"1min":0.308,"5min":0.308,"15min":0.30599999999999999},"max":{"1min":4.7910000000000004,"5min":6.7320000000000002,"15min":18.192},"last":0.32400000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:40:36 2023","interfaces":[{"interface":"back","average":{"1min":1.1240000000000001,"5min":0.88300000000000001,"15min":1.121},"min":{"1min":0.40400000000000003,"5min":0.376,"15min":0.28599999999999998},"max":{"1min":5.2729999999999997,"5min":5.2729999999999997,"15min":10.217000000000001},"last":0.40000000000000002},{"interface":"front","average":{"1min":0.97899999999999998,"5min":1.048,"15min":1.3120000000000001},"min":{"1min":0.45700000000000002,"5min":0.36799999999999999,"15min":0.30299999999999999},"max":{"1min":2.2909999999999999,"5min":8.8119999999999994,"15min":9.7360000000000007},"last":1.905}]}]},{"osd":1,"up_from":266,"seq":1142461300877,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1725052,"kb_used_data":1274868,"kb_used_omap":8,"kb_used_meta":450167,"kb_avail":92020100,"statfs":{"total":95995035648,"available":94228582400,"internally_reserved":0,"allocated":1305464832,"data_stored":1302979871,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8206,"internal_metadata":460972018},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,1,2],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:41:07 2023","interfaces":[{"interface":"back","average":{"1min":2.1469999999999998,"5min":2.0030000000000001,"15min":2.4980000000000002},"min":{"1min":0.215,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":16.292000000000002,"5min":16.292000000000002,"15min":52.689999999999998},"last":3.5449999999999999},{"interface":"front","average":{"1min":2.1419999999999999,"5min":2.6560000000000001,"15min":3.1949999999999998},"min":{"1min":0.20100000000000001,"5min":0.20100000000000001,"15min":0.19900000000000001},"max":{"1min":16.332999999999998,"5min":17.169,"15min":52.662999999999997},"last":1.913}]},{"osd":2,"last update":"Sat Sep 23 20:41:07 2023","interfaces":[{"interface":"back","average":{"1min":1.8220000000000001,"5min":2.5139999999999998,"15min":3.8610000000000002},"min":{"1min":0.216,"5min":0.216,"15min":0.20899999999999999},"max":{"1min":10.404,"5min":32.255000000000003,"15min":32.255000000000003},"last":1.3620000000000001},{"interface":"front","average":{"1min":1.611,"5min":1.9610000000000001,"15min":2.2839999999999998},"min":{"1min":0.24399999999999999,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":5.9699999999999998,"5min":13.404999999999999,"15min":13.404999999999999},"last":2.4820000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:41:07 2023","interfaces":[{"interface":"back","average":{"1min":1.7929999999999999,"5min":1.8460000000000001,"15min":3.4039999999999999},"min":{"1min":0.33800000000000002,"5min":0.247,"15min":0.247},"max":{"1min":16.245999999999999,"5min":16.245999999999999,"15min":80.745999999999995},"last":2.3919999999999999},{"interface":"front","average":{"1min":2.343,"5min":2.202,"15min":3.7650000000000001},"min":{"1min":0.34300000000000003,"5min":0.27500000000000002,"15min":0.22700000000000001},"max":{"1min":13.497999999999999,"5min":14.048999999999999,"15min":82.988},"last":2.2290000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:41:07 2023","interfaces":[{"interface":"back","average":{"1min":0.96899999999999997,"5min":1.75,"15min":2.1349999999999998},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30299999999999999},"max":{"1min":3.149,"5min":7.8399999999999999,"15min":24.327999999999999},"last":2.7599999999999998},{"interface":"front","average":{"1min":1.1240000000000001,"5min":1.3300000000000001,"15min":1.704},"min":{"1min":0.32600000000000001,"5min":0.311,"15min":0.311},"max":{"1min":4.3319999999999999,"5min":9.3339999999999996,"15min":11.574999999999999},"last":2.444}]},{"osd":5,"last update":"Sat Sep 23 20:41:07 2023","interfaces":[{"interface":"back","average":{"1min":2.02,"5min":2.129,"15min":2.3199999999999998},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.33200000000000002},"max":{"1min":13.550000000000001,"5min":33.401000000000003,"15min":33.401000000000003},"last":1.948},{"interface":"front","average":{"1min":1.008,"5min":2.2069999999999999,"15min":2.355},"min":{"1min":0.377,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":3.238,"5min":33.417000000000002,"15min":33.417000000000002},"last":2.8319999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:41:07 2023","interfaces":[{"interface":"back","average":{"1min":2.0329999999999999,"5min":2.0049999999999999,"15min":3.8090000000000002},"min":{"1min":0.43099999999999999,"5min":0.42599999999999999,"15min":0.35699999999999998},"max":{"1min":13.593999999999999,"5min":13.593999999999999,"15min":72.682000000000002},"last":1.9950000000000001},{"interface":"front","average":{"1min":1.0669999999999999,"5min":1.7789999999999999,"15min":3.6869999999999998},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":3.2050000000000001,"5min":7.883,"15min":72.563999999999993},"last":2.8639999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:41:07 2023","interfaces":[{"interface":"back","average":{"1min":2.1970000000000001,"5min":1.97,"15min":2.5030000000000001},"min":{"1min":0.40300000000000002,"5min":0.39500000000000002,"15min":0.39300000000000002},"max":{"1min":16.359000000000002,"5min":16.359000000000002,"15min":18.437000000000001},"last":1.974},{"interface":"front","average":{"1min":1.1830000000000001,"5min":1.8859999999999999,"15min":2.3809999999999998},"min":{"1min":0.44800000000000001,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":3.27,"5min":7.9089999999999998,"15min":24.524000000000001},"last":2.798}]}]},{"osd":0,"up_from":94,"seq":403726926079,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1107796,"kb_used_data":470988,"kb_used_omap":7,"kb_used_meta":636792,"kb_avail":92637356,"statfs":{"total":95995035648,"available":94860652544,"internally_reserved":0,"allocated":482291712,"data_stored":479960425,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":652075025},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:41:16 2023","interfaces":[{"interface":"back","average":{"1min":2.9329999999999998,"5min":1.8460000000000001,"15min":3.0049999999999999},"min":{"1min":0.24299999999999999,"5min":0.17199999999999999,"15min":0.17199999999999999},"max":{"1min":24.010000000000002,"5min":24.010000000000002,"15min":42.000999999999998},"last":1.3560000000000001},{"interface":"front","average":{"1min":2.7469999999999999,"5min":1.9359999999999999,"15min":3.2229999999999999},"min":{"1min":0.29499999999999998,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":14.279999999999999,"5min":14.279999999999999,"15min":23.036000000000001},"last":1.379}]},{"osd":2,"last update":"Sat Sep 23 20:40:50 2023","interfaces":[{"interface":"back","average":{"1min":3.0609999999999999,"5min":2.657,"15min":2.9849999999999999},"min":{"1min":0.29199999999999998,"5min":0.218,"15min":0.185},"max":{"1min":13.871,"5min":62.726999999999997,"15min":77.448999999999998},"last":0.29799999999999999},{"interface":"front","average":{"1min":2.6600000000000001,"5min":2.875,"15min":2.8919999999999999},"min":{"1min":0.27800000000000002,"5min":0.247,"15min":0.188},"max":{"1min":13.356999999999999,"5min":62.689,"15min":81.724999999999994},"last":0.34599999999999997}]},{"osd":3,"last update":"Sat Sep 23 20:40:23 2023","interfaces":[{"interface":"back","average":{"1min":2.399,"5min":2.7450000000000001,"15min":3.133},"min":{"1min":0.34699999999999998,"5min":0.24399999999999999,"15min":0.23400000000000001},"max":{"1min":9.0039999999999996,"5min":51.536000000000001,"15min":81.400999999999996},"last":2.1520000000000001},{"interface":"front","average":{"1min":2.8839999999999999,"5min":3.1389999999999998,"15min":3.4590000000000001},"min":{"1min":0.245,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":14.237,"5min":44.811999999999998,"15min":81.137},"last":2.0470000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:40:50 2023","interfaces":[{"interface":"back","average":{"1min":2.7970000000000002,"5min":1.9019999999999999,"15min":2.1920000000000002},"min":{"1min":0.432,"5min":0.28899999999999998,"15min":0.25900000000000001},"max":{"1min":6.8970000000000002,"5min":9.6110000000000007,"15min":23.876999999999999},"last":0.40799999999999997},{"interface":"front","average":{"1min":2.9929999999999999,"5min":1.615,"15min":1.9770000000000001},"min":{"1min":0.36299999999999999,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":9.7789999999999999,"5min":9.7789999999999999,"15min":23.681999999999999},"last":1.8779999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:40:50 2023","interfaces":[{"interface":"back","average":{"1min":3.0649999999999999,"5min":1.6519999999999999,"15min":2.1819999999999999},"min":{"1min":0.41499999999999998,"5min":0.371,"15min":0.34200000000000003},"max":{"1min":9.8629999999999995,"5min":9.8629999999999995,"15min":24.198},"last":1.9470000000000001},{"interface":"front","average":{"1min":2.8439999999999999,"5min":2.4449999999999998,"15min":2.3170000000000002},"min":{"1min":0.38900000000000001,"5min":0.318,"15min":0.25600000000000001},"max":{"1min":12.929,"5min":26.228000000000002,"15min":26.228000000000002},"last":0.42299999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:40:50 2023","interfaces":[{"interface":"back","average":{"1min":3.6520000000000001,"5min":1.9650000000000001,"15min":2.2549999999999999},"min":{"1min":0.39800000000000002,"5min":0.34499999999999997,"15min":0.318},"max":{"1min":24.056000000000001,"5min":24.056000000000001,"15min":24.408000000000001},"last":1.9139999999999999},{"interface":"front","average":{"1min":2.1219999999999999,"5min":2.2770000000000001,"15min":2.282},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.29399999999999998},"max":{"1min":8.0549999999999997,"5min":26.268000000000001,"15min":26.268000000000001},"last":0.52600000000000002}]},{"osd":7,"last update":"Sat Sep 23 20:40:50 2023","interfaces":[{"interface":"back","average":{"1min":2.4239999999999999,"5min":2.3180000000000001,"15min":2.4220000000000002},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.30599999999999999},"max":{"1min":8.1189999999999998,"5min":26.177,"15min":26.177},"last":0.57399999999999995},{"interface":"front","average":{"1min":2.9910000000000001,"5min":1.966,"15min":2.569},"min":{"1min":0.53700000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":6.8979999999999997,"5min":9.7200000000000006,"15min":24.669},"last":0.55200000000000005}]}]},{"osd":2,"up_from":104,"seq":446676599027,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1091792,"kb_used_data":395976,"kb_used_omap":11,"kb_used_meta":695796,"kb_avail":92653360,"statfs":{"total":95995035648,"available":94877040640,"internally_reserved":0,"allocated":405479424,"data_stored":403149975,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11878,"internal_metadata":712495514},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:41:03 2023","interfaces":[{"interface":"back","average":{"1min":0.97199999999999998,"5min":1.327,"15min":2.617},"min":{"1min":0.25,"5min":0.20300000000000001,"15min":0.17999999999999999},"max":{"1min":5.875,"5min":12.007,"15min":82.424999999999997},"last":0.312},{"interface":"front","average":{"1min":1.583,"5min":1.7949999999999999,"15min":2.6339999999999999},"min":{"1min":0.27000000000000002,"5min":0.19,"15min":0.19},"max":{"1min":9.2379999999999995,"5min":9.5600000000000005,"15min":82.524000000000001},"last":2.2269999999999999}]},{"osd":1,"last update":"Sat Sep 23 20:41:03 2023","interfaces":[{"interface":"back","average":{"1min":1.234,"5min":2.4780000000000002,"15min":3.3820000000000001},"min":{"1min":0.27100000000000002,"5min":0.245,"15min":0.221},"max":{"1min":7.8250000000000002,"5min":46.405000000000001,"15min":46.405000000000001},"last":1.145},{"interface":"front","average":{"1min":1.105,"5min":1.9430000000000001,"15min":2.7749999999999999},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":5.5270000000000001,"5min":46.459000000000003,"15min":46.459000000000003},"last":0.35199999999999998}]},{"osd":3,"last update":"Sat Sep 23 20:40:22 2023","interfaces":[{"interface":"back","average":{"1min":2.851,"5min":2.613,"15min":2.4769999999999999},"min":{"1min":0.28499999999999998,"5min":0.27100000000000002,"15min":0.19800000000000001},"max":{"1min":11.304,"5min":19.562999999999999,"15min":19.562999999999999},"last":0.38100000000000001},{"interface":"front","average":{"1min":1.7909999999999999,"5min":1.6160000000000001,"15min":2.5089999999999999},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.23899999999999999},"max":{"1min":8.9369999999999994,"5min":13.148999999999999,"15min":30.704999999999998},"last":1.3129999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:41:03 2023","interfaces":[{"interface":"back","average":{"1min":1.345,"5min":1.4399999999999999,"15min":1.917},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.27500000000000002},"max":{"1min":5.6150000000000002,"5min":6.6849999999999996,"15min":16.797999999999998},"last":2.2650000000000001},{"interface":"front","average":{"1min":1.1799999999999999,"5min":1.601,"15min":2.0899999999999999},"min":{"1min":0.379,"5min":0.379,"15min":0.27600000000000002},"max":{"1min":3.758,"5min":19.440999999999999,"15min":19.440999999999999},"last":0.47699999999999998}]},{"osd":5,"last update":"Sat Sep 23 20:41:03 2023","interfaces":[{"interface":"back","average":{"1min":1.4730000000000001,"5min":1.488,"15min":2.403},"min":{"1min":0.43099999999999999,"5min":0.36599999999999999,"15min":0.309},"max":{"1min":5.6580000000000004,"5min":6.6319999999999997,"15min":95.393000000000001},"last":2.3290000000000002},{"interface":"front","average":{"1min":1.1419999999999999,"5min":1.613,"15min":2.6669999999999998},"min":{"1min":0.44800000000000001,"5min":0.41399999999999998,"15min":0.32300000000000001},"max":{"1min":3.8279999999999998,"5min":19.513000000000002,"15min":95.414000000000001},"last":0.76200000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:41:03 2023","interfaces":[{"interface":"back","average":{"1min":1.1279999999999999,"5min":1.585,"15min":2.177},"min":{"1min":0.432,"5min":0.42399999999999999,"15min":0.32100000000000001},"max":{"1min":3.7949999999999999,"5min":19.486000000000001,"15min":19.486000000000001},"last":0.505},{"interface":"front","average":{"1min":0.94199999999999995,"5min":1.05,"15min":1.9259999999999999},"min":{"1min":0.42099999999999999,"5min":0.36899999999999999,"15min":0.316},"max":{"1min":5.5789999999999997,"5min":9.3010000000000002,"15min":30.670999999999999},"last":0.55200000000000005}]},{"osd":7,"last update":"Sat Sep 23 20:41:03 2023","interfaces":[{"interface":"back","average":{"1min":1.2030000000000001,"5min":1.038,"15min":1.9650000000000001},"min":{"1min":0.47399999999999998,"5min":0.39300000000000002,"15min":0.35199999999999998},"max":{"1min":7.5860000000000003,"5min":7.5860000000000003,"15min":30.741},"last":1.4730000000000001},{"interface":"front","average":{"1min":1.831,"5min":1.6870000000000001,"15min":2.1739999999999999},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":5.8529999999999998,"5min":6.7220000000000004,"15min":16.719000000000001},"last":2.2989999999999999}]}]},{"osd":3,"up_from":109,"seq":468151435501,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1926972,"kb_used_data":1093940,"kb_used_omap":4,"kb_used_meta":833019,"kb_avail":91818180,"statfs":{"total":95995035648,"available":94021816320,"internally_reserved":0,"allocated":1120194560,"data_stored":1117881912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4514,"internal_metadata":853012062},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,2,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:40:26 2023","interfaces":[{"interface":"back","average":{"1min":1.409,"5min":1.7350000000000001,"15min":2.544},"min":{"1min":0.21199999999999999,"5min":0.21199999999999999,"15min":0.186},"max":{"1min":7.6879999999999997,"5min":14.638,"15min":67.305999999999997},"last":1.2350000000000001},{"interface":"front","average":{"1min":2.4540000000000002,"5min":1.865,"15min":2.415},"min":{"1min":0.29499999999999998,"5min":0.26600000000000001,"15min":0.19500000000000001},"max":{"1min":8.8439999999999994,"5min":10.140000000000001,"15min":67.403999999999996},"last":1.179}]},{"osd":1,"last update":"Sat Sep 23 20:41:05 2023","interfaces":[{"interface":"back","average":{"1min":1.244,"5min":1.212,"15min":2.2519999999999998},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":4.8099999999999996,"5min":9.5299999999999994,"15min":16.189},"last":1.532},{"interface":"front","average":{"1min":2.262,"5min":2.0369999999999999,"15min":2.617},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":8.8030000000000008,"5min":12.702999999999999,"15min":32.603999999999999},"last":1.502}]},{"osd":2,"last update":"Sat Sep 23 20:40:26 2023","interfaces":[{"interface":"back","average":{"1min":2.3140000000000001,"5min":1.837,"15min":2.5030000000000001},"min":{"1min":0.26600000000000001,"5min":0.246,"15min":0.22800000000000001},"max":{"1min":6.3200000000000003,"5min":10.747999999999999,"15min":32.857999999999997},"last":1.212},{"interface":"front","average":{"1min":1.516,"5min":2.0259999999999998,"15min":2.4980000000000002},"min":{"1min":0.31,"5min":0.28100000000000003,"15min":0.20899999999999999},"max":{"1min":8.5530000000000008,"5min":22.32,"15min":24.146999999999998},"last":1.5129999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:40:26 2023","interfaces":[{"interface":"back","average":{"1min":0.82199999999999995,"5min":1.0860000000000001,"15min":1.786},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.29799999999999999},"max":{"1min":2.5899999999999999,"5min":9.3160000000000007,"15min":46.042999999999999},"last":1.4299999999999999},{"interface":"front","average":{"1min":1.6359999999999999,"5min":1.879,"15min":2.4340000000000002},"min":{"1min":0.33200000000000002,"5min":0.33000000000000002,"15min":0.28100000000000003},"max":{"1min":5.141,"5min":13.571999999999999,"15min":46.579999999999998},"last":0.64900000000000002}]},{"osd":5,"last update":"Sat Sep 23 20:40:26 2023","interfaces":[{"interface":"back","average":{"1min":0.82099999999999995,"5min":1.167,"15min":1.998},"min":{"1min":0.32700000000000001,"5min":0.32300000000000001,"15min":0.31900000000000001},"max":{"1min":2.9889999999999999,"5min":9.4130000000000003,"15min":45.292000000000002},"last":1.4870000000000001},{"interface":"front","average":{"1min":1.6859999999999999,"5min":1.482,"15min":1.968},"min":{"1min":0.38500000000000001,"5min":0.34399999999999997,"15min":0.313},"max":{"1min":5.0060000000000002,"5min":9.0839999999999996,"15min":44.679000000000002},"last":1.2629999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:40:26 2023","interfaces":[{"interface":"back","average":{"1min":1.8640000000000001,"5min":1.3959999999999999,"15min":1.964},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":7.3630000000000004,"5min":9.0470000000000006,"15min":17.524999999999999},"last":1.319},{"interface":"front","average":{"1min":1.4750000000000001,"5min":1.8839999999999999,"15min":2.165},"min":{"1min":0.434,"5min":0.39800000000000002,"15min":0.33700000000000002},"max":{"1min":4.8819999999999997,"5min":13.500999999999999,"15min":19.524000000000001},"last":1.1060000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:40:26 2023","interfaces":[{"interface":"back","average":{"1min":1.5109999999999999,"5min":2.1829999999999998,"15min":2.3959999999999999},"min":{"1min":0.433,"5min":0.39900000000000002,"15min":0.34999999999999998},"max":{"1min":4.319,"5min":13.544,"15min":19.571999999999999},"last":0.68899999999999995},{"interface":"front","average":{"1min":0.996,"5min":1.274,"15min":2.0720000000000001},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.371},"max":{"1min":4.157,"5min":9.3719999999999999,"15min":45.387999999999998},"last":1.4630000000000001}]}]},{"osd":4,"up_from":30,"seq":128849019255,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":818680,"kb_used_data":438960,"kb_used_omap":0,"kb_used_meta":379712,"kb_avail":92926472,"statfs":{"total":95995035648,"available":95156707328,"internally_reserved":0,"allocated":449495040,"data_stored":448058304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":388825088},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:40:49 2023","interfaces":[{"interface":"back","average":{"1min":2.1949999999999998,"5min":1.6739999999999999,"15min":1.724},"min":{"1min":0.31,"5min":0.24099999999999999,"15min":0.23699999999999999},"max":{"1min":11.553000000000001,"5min":24.114999999999998,"15min":24.114999999999998},"last":0.58599999999999997},{"interface":"front","average":{"1min":2.2789999999999999,"5min":1.8080000000000001,"15min":1.895},"min":{"1min":0.28899999999999998,"5min":0.246,"15min":0.246},"max":{"1min":11.536,"5min":24.135999999999999,"15min":24.135999999999999},"last":0.439}]},{"osd":1,"last update":"Sat Sep 23 20:41:10 2023","interfaces":[{"interface":"back","average":{"1min":1.2589999999999999,"5min":1.27,"15min":1.6910000000000001},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":10.363,"5min":18.190999999999999,"15min":18.190999999999999},"last":0.377},{"interface":"front","average":{"1min":1.365,"5min":1.3420000000000001,"15min":1.647},"min":{"1min":0.33400000000000002,"5min":0.28499999999999998,"15min":0.28199999999999997},"max":{"1min":10.426,"5min":18.173999999999999,"15min":18.173999999999999},"last":0.76700000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:41:05 2023","interfaces":[{"interface":"back","average":{"1min":0.99099999999999999,"5min":1.264,"15min":1.821},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.26600000000000001},"max":{"1min":4.4640000000000004,"5min":7.75,"15min":32.363},"last":0.628},{"interface":"front","average":{"1min":0.82899999999999996,"5min":1.6779999999999999,"15min":2.2120000000000002},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.27600000000000002},"max":{"1min":2.4590000000000001,"5min":11.917999999999999,"15min":34.042999999999999},"last":0.52600000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:41:21 2023","interfaces":[{"interface":"back","average":{"1min":0.75800000000000001,"5min":1.5069999999999999,"15min":2},"min":{"1min":0.372,"5min":0.372,"15min":0.29099999999999998},"max":{"1min":3.5249999999999999,"5min":12.147,"15min":32.332999999999998},"last":0.73399999999999999},{"interface":"front","average":{"1min":0.93999999999999995,"5min":1.571,"15min":2.077},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.30299999999999999},"max":{"1min":3.125,"5min":11.914,"15min":32.399999999999999},"last":0.70099999999999996}]},{"osd":5,"last update":"Sat Sep 23 20:40:47 2023","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.68200000000000005,"15min":1.51},"min":{"1min":0.28100000000000003,"5min":0.26400000000000001,"15min":0.247},"max":{"1min":2.202,"5min":4.1260000000000003,"15min":82.866},"last":0.41099999999999998},{"interface":"front","average":{"1min":1.006,"5min":0.96999999999999997,"15min":1.4810000000000001},"min":{"1min":0.32200000000000001,"5min":0.28999999999999998,"15min":0.27800000000000002},"max":{"1min":5.2670000000000003,"5min":7.0129999999999999,"15min":82.546000000000006},"last":0.55200000000000005}]},{"osd":6,"last update":"Sat Sep 23 20:41:05 2023","interfaces":[{"interface":"back","average":{"1min":0.67300000000000004,"5min":0.96299999999999997,"15min":1.258},"min":{"1min":0.35799999999999998,"5min":0.32200000000000001,"15min":0.27600000000000002},"max":{"1min":2.7450000000000001,"5min":14.731999999999999,"15min":15.252000000000001},"last":0.40000000000000002},{"interface":"front","average":{"1min":0.64500000000000002,"5min":0.88200000000000001,"15min":1.1240000000000001},"min":{"1min":0.314,"5min":0.309,"15min":0.27200000000000002},"max":{"1min":2.5539999999999998,"5min":7.2850000000000001,"15min":11.199},"last":0.51100000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:41:21 2023","interfaces":[{"interface":"back","average":{"1min":0.84799999999999998,"5min":1.1499999999999999,"15min":1.446},"min":{"1min":0.39100000000000001,"5min":0.318,"15min":0.318},"max":{"1min":2.9079999999999999,"5min":14.534000000000001,"15min":16.427},"last":0.48199999999999998},{"interface":"front","average":{"1min":0.57899999999999996,"5min":0.82499999999999996,"15min":1.169},"min":{"1min":0.34599999999999997,"5min":0.33200000000000002,"15min":0.307},"max":{"1min":1.488,"5min":7.9390000000000001,"15min":15.645},"last":0.45800000000000002}]}]},{"osd":5,"up_from":39,"seq":167503724916,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2396148,"kb_used_data":1978412,"kb_used_omap":0,"kb_used_meta":417728,"kb_avail":91349004,"statfs":{"total":95995035648,"available":93541380096,"internally_reserved":0,"allocated":2025893888,"data_stored":2024457878,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":427753472},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,3,0,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:40:53 2023","interfaces":[{"interface":"back","average":{"1min":1.3640000000000001,"5min":1.397,"15min":1.758},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.20699999999999999},"max":{"1min":5.8330000000000002,"5min":7.819,"15min":18.16},"last":0.63500000000000001},{"interface":"front","average":{"1min":1.3149999999999999,"5min":1.4770000000000001,"15min":1.665},"min":{"1min":0.29899999999999999,"5min":0.29499999999999998,"15min":0.245},"max":{"1min":5.157,"5min":7.9859999999999998,"15min":9.0570000000000004},"last":0.55000000000000004}]},{"osd":1,"last update":"Sat Sep 23 20:41:09 2023","interfaces":[{"interface":"back","average":{"1min":1.391,"5min":1.95,"15min":1.8740000000000001},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.253},"max":{"1min":6.1139999999999999,"5min":36.588999999999999,"15min":36.588999999999999},"last":0.27200000000000002},{"interface":"front","average":{"1min":1.236,"5min":1.587,"15min":1.675},"min":{"1min":0.30199999999999999,"5min":0.27700000000000002,"15min":0.23499999999999999},"max":{"1min":5.7800000000000002,"5min":36.920999999999999,"15min":36.920999999999999},"last":0.65200000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:40:53 2023","interfaces":[{"interface":"back","average":{"1min":2.0840000000000001,"5min":1.9470000000000001,"15min":1.9590000000000001},"min":{"1min":0.28000000000000003,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":9.2119999999999997,"5min":37.011000000000003,"15min":37.011000000000003},"last":0.51100000000000001},{"interface":"front","average":{"1min":1.5,"5min":1.74,"15min":1.978},"min":{"1min":0.32800000000000001,"5min":0.28999999999999998,"15min":0.255},"max":{"1min":5.4889999999999999,"5min":36.835000000000001,"15min":36.835000000000001},"last":0.60599999999999998}]},{"osd":3,"last update":"Sat Sep 23 20:41:15 2023","interfaces":[{"interface":"back","average":{"1min":0.86699999999999999,"5min":1.5229999999999999,"15min":2.1539999999999999},"min":{"1min":0.36099999999999999,"5min":0.34699999999999998,"15min":0.25700000000000001},"max":{"1min":3.512,"5min":6.9669999999999996,"15min":36.716000000000001},"last":3.9470000000000001},{"interface":"front","average":{"1min":1.2989999999999999,"5min":2.0699999999999998,"15min":2.0150000000000001},"min":{"1min":0.371,"5min":0.34000000000000002,"15min":0.28799999999999998},"max":{"1min":5.8940000000000001,"5min":18.504000000000001,"15min":18.504000000000001},"last":2.5190000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:40:44 2023","interfaces":[{"interface":"back","average":{"1min":1.2689999999999999,"5min":0.90600000000000003,"15min":1.3140000000000001},"min":{"1min":0.29999999999999999,"5min":0.255,"15min":0.255},"max":{"1min":6.3780000000000001,"5min":6.9180000000000001,"15min":28.015999999999998},"last":0.32200000000000001},{"interface":"front","average":{"1min":1.5760000000000001,"5min":0.93700000000000006,"15min":1.0940000000000001},"min":{"1min":0.32500000000000001,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":5.7469999999999999,"5min":6.8380000000000001,"15min":12.300000000000001},"last":0.42799999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:40:58 2023","interfaces":[{"interface":"back","average":{"1min":1.1160000000000001,"5min":1.089,"15min":1.1970000000000001},"min":{"1min":0.34599999999999997,"5min":0.318,"15min":0.30199999999999999},"max":{"1min":6.0739999999999998,"5min":6.9960000000000004,"15min":12.890000000000001},"last":0.34300000000000003},{"interface":"front","average":{"1min":1.339,"5min":1.075,"15min":1.117},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":5.8609999999999998,"5min":6.7930000000000001,"15min":16.265999999999998},"last":0.56200000000000006}]},{"osd":7,"last update":"Sat Sep 23 20:41:15 2023","interfaces":[{"interface":"back","average":{"1min":1.1799999999999999,"5min":0.93700000000000006,"15min":1.1279999999999999},"min":{"1min":0.34499999999999997,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":6.4720000000000004,"5min":6.4720000000000004,"15min":36.621000000000002},"last":0.36399999999999999},{"interface":"front","average":{"1min":1.319,"5min":1.0469999999999999,"15min":1.129},"min":{"1min":0.375,"5min":0.33200000000000002,"15min":0.30099999999999999},"max":{"1min":7.3700000000000001,"5min":7.3700000000000001,"15min":37.331000000000003},"last":0.47299999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4722,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1360,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8206,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7156,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3154,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":483196928,"data_stored":483196928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1304428544,"data_stored":1304428544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":403111936,"data_stored":403111936,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1131347968,"data_stored":1131347968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":452591616,"data_stored":452591616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1358888960,"data_stored":1358888960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":406913024,"data_stored":406913024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1180237824,"data_stored":1180237824,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:41:28.659 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-09-23T20:41:28.659 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd pool set .mgr pg_num 41 2023-09-23T20:41:28.721 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:41:28.722 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:41:28.766 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:28.810 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:41:28.811 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:41:28.816 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:28.816 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:41:28.882 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:41:28.917 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:41:28.929 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:41:28.959 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:28.983 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:41:29.014 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:41:29.018 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:29.019 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:41:29.039 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:41:29.086 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:41:29.120 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:41:29.129 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:41:29.135 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:2023-09-23T20:41:29.135858+0000 min lat: 0.00110414 max lat: 0.151505 avg lat: 0.00760355 2023-09-23T20:41:29.135 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T20:41:29.135 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 20 16 42087 42071 131.423 80.0625 0.00804966 0.00760355 2023-09-23T20:41:29.169 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:29.184 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:41:29.202 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:41:29.202 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:41:29.202 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:41:29.202 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:41:29.202 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:41:29.202 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:41:29.202 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:41:29.202 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:41:29.221 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:41:29.226 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:29.227 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:41:29.327 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:41:29.360 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:29.425 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:41:29.430 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:29.430 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:41:29.531 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:41:29.584 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:29.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:29 smithi018 ceph-mon[184144]: from='client.36477 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:41:29.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:29 smithi018 ceph-mon[184144]: pgmap v1137: 119 pgs: 119 active+clean; 2.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 117 MiB/s wr, 1.87k op/s 2023-09-23T20:41:29.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:29 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3181833071' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:41:29.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:29 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:41:29.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:29 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:41:29.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:29 smithi018 ceph-mon[179647]: from='client.36477 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:41:29.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:29 smithi018 ceph-mon[179647]: pgmap v1137: 119 pgs: 119 active+clean; 2.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 117 MiB/s wr, 1.87k op/s 2023-09-23T20:41:29.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:29 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3181833071' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:41:29.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:29 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:41:29.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:29 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:41:29.643 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:41:29.650 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:29 smithi149 ceph-mon[170394]: from='client.36477 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:41:29.650 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:29 smithi149 ceph-mon[170394]: pgmap v1137: 119 pgs: 119 active+clean; 2.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 117 MiB/s wr, 1.87k op/s 2023-09-23T20:41:29.651 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:29 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3181833071' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:41:29.651 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:29 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:41:29.651 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:29 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:41:29.651 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:29.652 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:41:29.722 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:41:29.752 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:41:29.823 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:29.929 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:41:29.977 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:41:29.984 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:29.985 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:41:30.085 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:41:30.122 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:30.135 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 21 16 43591 43575 129.641 94 0.013022 0.00770877 2023-09-23T20:41:30.185 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:41:30.191 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:30.191 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:41:30.292 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:41:30.365 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:30.425 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:41:30.430 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:30.430 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:41:30.535 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:41:30.588 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:30.652 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:41:30.655 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:30.655 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:41:30.756 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:41:30.834 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:31.136 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 22 16 45104 45088 128.046 94.5625 0.00731279 0.00780467 2023-09-23T20:41:31.297 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:41:31.302 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:31.302 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:41:31.403 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:41:31.463 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:31.591 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:41:31.604 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:31.605 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:41:31.674 INFO:teuthology.orchestra.run.smithi018.stdout:ERROR: (22) Invalid argument 2023-09-23T20:41:31.675 INFO:teuthology.orchestra.run.smithi018.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:41:31.706 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:41:31.750 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:31.761 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:41:31.800 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:41:31.805 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:31.806 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:41:31.824 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:31 smithi018 ceph-mon[179647]: pgmap v1138: 119 pgs: 119 active+clean; 2.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 109 MiB/s wr, 1.75k op/s 2023-09-23T20:41:31.824 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:31 smithi018 ceph-mon[184144]: pgmap v1138: 119 pgs: 119 active+clean; 2.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 109 MiB/s wr, 1.75k op/s 2023-09-23T20:41:31.907 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:41:31.950 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:31 smithi149 ceph-mon[170394]: pgmap v1138: 119 pgs: 119 active+clean; 2.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 109 MiB/s wr, 1.75k op/s 2023-09-23T20:41:31.953 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:32.001 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:41:32.007 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:32.007 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:41:32.108 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:41:32.136 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 23 16 46455 46439 126.144 84.4375 0.0106128 0.00792167 2023-09-23T20:41:32.189 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:32.275 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:41:32.281 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:32.282 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:41:32.355 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T20:41:32.383 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:41:32.436 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:32.488 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:41:32.493 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:32.494 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:41:32.595 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:41:32.620 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:32 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1281525317' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:41:32.620 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:32 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:41:32.633 INFO:teuthology.orchestra.run.smithi018.stderr:nodeep-scrub is set 2023-09-23T20:41:32.680 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:32.719 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:32 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1281525317' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:41:32.719 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:32 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:41:32.741 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:41:32.753 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:32.754 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:41:32.800 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:41:32.855 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:41:32.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:32 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1281525317' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:41:32.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:32 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:41:32.924 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:33.137 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 24 16 47740 47724 124.235 80.3125 0.0124866 0.0080446 2023-09-23T20:41:33.481 INFO:teuthology.orchestra.run.smithi018.stderr:Error EINVAL: specified pg_num 41 < pg_num_max 32 2023-09-23T20:41:33.735 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:41:33.740 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:33.741 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:41:33.841 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:41:33.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:33 smithi018 ceph-mon[179647]: pgmap v1139: 119 pgs: 119 active+clean; 2.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 101 MiB/s wr, 1.61k op/s 2023-09-23T20:41:33.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:33 smithi018 ceph-mon[179647]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:41:33.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:33 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:41:33.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:33 smithi018 ceph-mon[179647]: osdmap e461: 8 total, 8 up, 8 in 2023-09-23T20:41:33.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:33 smithi018 ceph-mon[179647]: failed to encode map e461 with expected crc 2023-09-23T20:41:33.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:33 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/722668914' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "41"}]: dispatch 2023-09-23T20:41:33.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:33 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "41"}]: dispatch 2023-09-23T20:41:33.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:33 smithi018 ceph-mon[184144]: pgmap v1139: 119 pgs: 119 active+clean; 2.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 101 MiB/s wr, 1.61k op/s 2023-09-23T20:41:33.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:33 smithi018 ceph-mon[184144]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:41:33.894 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:33 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:41:33.894 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:33 smithi018 ceph-mon[184144]: osdmap e461: 8 total, 8 up, 8 in 2023-09-23T20:41:33.894 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:33 smithi018 ceph-mon[184144]: failed to encode map e461 with expected crc 2023-09-23T20:41:33.894 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:33 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/722668914' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "41"}]: dispatch 2023-09-23T20:41:33.894 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:33 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "41"}]: dispatch 2023-09-23T20:41:33.905 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:33.977 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:33 smithi149 ceph-mon[170394]: pgmap v1139: 119 pgs: 119 active+clean; 2.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 101 MiB/s wr, 1.61k op/s 2023-09-23T20:41:33.977 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:33 smithi149 ceph-mon[170394]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:41:33.977 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:33 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:41:33.977 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:33 smithi149 ceph-mon[170394]: osdmap e461: 8 total, 8 up, 8 in 2023-09-23T20:41:33.977 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:33 smithi149 ceph-mon[170394]: failed to encode map e461 with expected crc 2023-09-23T20:41:33.978 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:33 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/722668914' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "41"}]: dispatch 2023-09-23T20:41:33.978 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:33 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "41"}]: dispatch 2023-09-23T20:41:34.137 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 25 16 49212 49196 122.946 92 0.0102123 0.00812799 2023-09-23T20:41:34.521 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:41:34.530 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:34.531 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:41:34.632 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:41:34.683 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:34.684 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-09-23T20:41:34.742 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:41:34.742 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:41:34.750 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:34.750 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:41:34.851 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:41:34.901 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:34.965 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:41:34.980 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:34.981 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:41:35.081 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:41:35.137 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 26 16 50723 50707 121.849 94.4375 0.0130141 0.00820189 2023-09-23T20:41:35.147 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:35.221 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:41:35.226 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:35.227 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:41:35.319 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:41:34 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:41:34] "GET /metrics HTTP/1.1" 200 36522 "" "Prometheus/2.43.0" 2023-09-23T20:41:35.327 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:41:35.383 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:35.441 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:41:35.449 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:35.450 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:41:35.551 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:41:35.623 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:35.630 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:35 smithi018 ceph-mon[179647]: pgmap v1141: 119 pgs: 119 active+clean; 2.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 88 MiB/s wr, 1.40k op/s 2023-09-23T20:41:35.630 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:35 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:41:35.631 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:35 smithi018 ceph-mon[184144]: pgmap v1141: 119 pgs: 119 active+clean; 2.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 88 MiB/s wr, 1.40k op/s 2023-09-23T20:41:35.631 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:35 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:41:35.675 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:41:35.683 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:35.683 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:41:35.784 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:41:35.846 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:35.905 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:41:35.909 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:35 smithi149 ceph-mon[170394]: pgmap v1141: 119 pgs: 119 active+clean; 2.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 88 MiB/s wr, 1.40k op/s 2023-09-23T20:41:35.909 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:35 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:41:35.910 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:35.910 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:41:36.011 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:41:36.072 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:36.139 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 27 16 51995 51979 120.272 79.5 0.0194833 0.00830857 2023-09-23T20:41:36.142 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:41:36.147 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:36.148 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:41:36.248 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:41:36.303 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:36.378 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:41:36.388 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:36.430 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:41:36.430 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:41:36.466 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:41:36.506 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:41:36.530 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T20:41:36.531 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:41:36.545 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:41:36.589 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:41:36.601 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2023-09-23T20:41:36.601 INFO:teuthology.orchestra.run.smithi018.stdout: "ops": [ 2023-09-23T20:41:36.601 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:41:36.602 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.36450.0:52527 9.9 9:99353b01:::benchmark_data_smithi018_381516_object52526:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e461)", 2023-09-23T20:41:36.602 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:41:36.593218+0000", 2023-09-23T20:41:36.602 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.0061345760000000001, 2023-09-23T20:41:36.602 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.006176422, 2023-09-23T20:41:36.602 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:41:36.602 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:41:36.602 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:41:36.602 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.36450", 2023-09-23T20:41:36.602 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:41:36.602 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 52527 2023-09-23T20:41:36.602 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:41:36.602 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:41:36.603 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:41:36.603 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:41:36.603 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:41:36.593218+0000", 2023-09-23T20:41:36.603 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:41:36.603 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:41:36.603 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:41:36.603 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:41:36.603 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:41:36.593218+0000", 2023-09-23T20:41:36.603 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:41:36.603 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:41:36.603 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:41:36.604 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:41:36.604 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:41:36.593215+0000", 2023-09-23T20:41:36.604 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:41:36.604 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:41:36.604 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:41:36.604 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:41:36.604 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:41:36.593247+0000", 2023-09-23T20:41:36.604 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.1155999999999998e-05 2023-09-23T20:41:36.604 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:41:36.604 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:41:36.604 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:41:36.604 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:41:36.593264+0000", 2023-09-23T20:41:36.605 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.7011e-05 2023-09-23T20:41:36.605 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:41:36.605 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:41:36.605 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:41:36.605 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:41:36.593278+0000", 2023-09-23T20:41:36.605 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.4093e-05 2023-09-23T20:41:36.605 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:41:36.605 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:41:36.605 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:41:36.605 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:41:36.593337+0000", 2023-09-23T20:41:36.605 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.9298e-05 2023-09-23T20:41:36.605 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:41:36.605 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:41:36.606 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:41:36.606 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:41:36.593505+0000", 2023-09-23T20:41:36.606 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000167999 2023-09-23T20:41:36.606 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:41:36.606 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:41:36.606 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 5,7", 2023-09-23T20:41:36.606 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:41:36.593658+0000", 2023-09-23T20:41:36.606 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00015331700000000001 2023-09-23T20:41:36.606 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:41:36.606 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:41:36.606 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:41:36.607 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:41:36.598120+0000", 2023-09-23T20:41:36.607 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.004461563 2023-09-23T20:41:36.607 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:41:36.607 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:41:36.607 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:41:36.607 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:41:36.607 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:41:36.607 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.36450.0:52528 9.b 9:d766d94b:::benchmark_data_smithi018_381516_object52527:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e461)", 2023-09-23T20:41:36.607 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:41:36.593300+0000", 2023-09-23T20:41:36.607 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.006052048, 2023-09-23T20:41:36.607 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0062079520000000001, 2023-09-23T20:41:36.607 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:41:36.608 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:41:36.608 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:41:36.608 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.36450", 2023-09-23T20:41:36.608 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:41:36.608 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 52528 2023-09-23T20:41:36.608 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:41:36.608 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:41:36.608 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:41:36.608 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:41:36.608 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:41:36.593300+0000", 2023-09-23T20:41:36.608 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:41:36.608 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:41:36.609 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:41:36.609 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:41:36.609 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:41:36.593300+0000", 2023-09-23T20:41:36.609 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:41:36.609 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:41:36.609 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:41:36.609 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:41:36.609 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:41:36.593299+0000", 2023-09-23T20:41:36.609 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:41:36.609 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:41:36.609 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:41:36.609 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:41:36.610 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:41:36.593332+0000", 2023-09-23T20:41:36.610 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.2047e-05 2023-09-23T20:41:36.610 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:41:36.610 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:41:36.610 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:41:36.610 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:41:36.593342+0000", 2023-09-23T20:41:36.610 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0258000000000001e-05 2023-09-23T20:41:36.610 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:41:36.610 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:41:36.610 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:41:36.610 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:41:36.593349+0000", 2023-09-23T20:41:36.611 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.8580000000000002e-06 2023-09-23T20:41:36.611 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:41:36.611 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:41:36.611 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:41:36.611 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:41:36.595666+0000", 2023-09-23T20:41:36.611 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0023170249999999999 2023-09-23T20:41:36.611 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:41:36.611 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:41:36.611 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:41:36.611 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:41:36.595836+0000", 2023-09-23T20:41:36.611 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00017007900000000001 2023-09-23T20:41:36.611 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:41:36.612 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:41:36.612 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 5,6", 2023-09-23T20:41:36.612 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:41:36.595976+0000", 2023-09-23T20:41:36.612 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000140257 2023-09-23T20:41:36.612 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:41:36.612 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:41:36.612 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:41:36.612 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:41:36.612 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:41:36.612 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.36450.0:52529 9.5 9:ac9e066d:::benchmark_data_smithi018_381516_object52528:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e461)", 2023-09-23T20:41:36.612 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:41:36.593367+0000", 2023-09-23T20:41:36.612 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.0059853800000000002, 2023-09-23T20:41:36.613 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0062312549999999998, 2023-09-23T20:41:36.613 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:41:36.613 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:41:36.613 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:41:36.613 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.36450", 2023-09-23T20:41:36.613 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:41:36.613 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 52529 2023-09-23T20:41:36.613 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:41:36.613 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:41:36.613 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:41:36.613 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:41:36.613 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:41:36.593367+0000", 2023-09-23T20:41:36.614 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:41:36.614 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:41:36.614 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:41:36.614 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:41:36.614 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:41:36.593367+0000", 2023-09-23T20:41:36.614 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:41:36.614 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:41:36.614 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:41:36.614 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:41:36.614 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:41:36.593366+0000", 2023-09-23T20:41:36.614 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.999999 2023-09-23T20:41:36.615 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:41:36.615 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:41:36.615 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:41:36.615 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:41:36.593391+0000", 2023-09-23T20:41:36.615 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.4545999999999998e-05 2023-09-23T20:41:36.615 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:41:36.615 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:41:36.615 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:41:36.615 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:41:36.593399+0000", 2023-09-23T20:41:36.615 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.8109999999999992e-06 2023-09-23T20:41:36.615 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:41:36.615 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:41:36.616 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:41:36.616 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:41:36.593406+0000", 2023-09-23T20:41:36.616 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.4479999999999999e-06 2023-09-23T20:41:36.616 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:41:36.616 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:41:36.616 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:41:36.616 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:41:36.593456+0000", 2023-09-23T20:41:36.616 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.0238999999999998e-05 2023-09-23T20:41:36.616 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:41:36.616 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:41:36.616 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:41:36.616 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:41:36.593596+0000", 2023-09-23T20:41:36.617 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000139994 2023-09-23T20:41:36.617 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:41:36.617 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:41:36.617 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 5,7", 2023-09-23T20:41:36.617 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:41:36.593737+0000", 2023-09-23T20:41:36.617 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00014050400000000001 2023-09-23T20:41:36.617 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:41:36.617 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:41:36.617 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:41:36.617 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:41:36.596180+0000", 2023-09-23T20:41:36.617 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0024434510000000001 2023-09-23T20:41:36.617 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:41:36.618 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:41:36.618 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:41:36.618 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:41:36.598669+0000", 2023-09-23T20:41:36.618 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.002488545 2023-09-23T20:41:36.618 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:41:36.618 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:41:36.618 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:41:36.618 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:41:36.618 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:41:36.618 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.36450.0:52533 9.9 9:96638cad:::benchmark_data_smithi018_381516_object52532:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e461)", 2023-09-23T20:41:36.618 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:41:36.593425+0000", 2023-09-23T20:41:36.619 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.0059268380000000002, 2023-09-23T20:41:36.619 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0062692959999999997, 2023-09-23T20:41:36.619 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:41:36.619 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:41:36.619 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:41:36.619 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.36450", 2023-09-23T20:41:36.619 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:41:36.619 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 52533 2023-09-23T20:41:36.619 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:41:36.619 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:41:36.619 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:41:36.619 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:41:36.620 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:41:36.593425+0000", 2023-09-23T20:41:36.620 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:41:36.620 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:41:36.620 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:41:36.620 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:41:36.620 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:41:36.593425+0000", 2023-09-23T20:41:36.620 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:41:36.620 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:41:36.620 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:41:36.620 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:41:36.620 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:41:36.593425+0000", 2023-09-23T20:41:36.620 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:41:36.621 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:41:36.621 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:41:36.621 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:41:36.621 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:41:36.593451+0000", 2023-09-23T20:41:36.621 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.6092999999999999e-05 2023-09-23T20:41:36.621 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:41:36.621 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:41:36.621 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:41:36.621 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:41:36.593459+0000", 2023-09-23T20:41:36.621 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.0450000000000001e-06 2023-09-23T20:41:36.621 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:41:36.621 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:41:36.622 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:41:36.622 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:41:36.593465+0000", 2023-09-23T20:41:36.622 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.4309999999999999e-06 2023-09-23T20:41:36.622 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:41:36.622 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:41:36.622 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:41:36.622 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:41:36.595950+0000", 2023-09-23T20:41:36.622 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.002484591 2023-09-23T20:41:36.622 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:41:36.622 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:41:36.622 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:41:36.622 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:41:36.596077+0000", 2023-09-23T20:41:36.623 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00012727600000000001 2023-09-23T20:41:36.623 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:41:36.623 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:41:36.623 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 5,7", 2023-09-23T20:41:36.623 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:41:36.596201+0000", 2023-09-23T20:41:36.623 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00012344999999999999 2023-09-23T20:41:36.623 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:41:36.623 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:41:36.623 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:41:36.623 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:41:36.623 INFO:teuthology.orchestra.run.smithi018.stdout: ], 2023-09-23T20:41:36.623 INFO:teuthology.orchestra.run.smithi018.stdout: "num_ops": 4 2023-09-23T20:41:36.624 INFO:teuthology.orchestra.run.smithi018.stdout:} 2023-09-23T20:41:36.645 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:36.654 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:41:36.699 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:41:36.717 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:41:36.722 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:36.723 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:41:36.756 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:41:36.775 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:41:36.775 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:41:36.775 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:41:36.775 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:41:36.776 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:41:36.776 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:41:36.776 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:41:36.776 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:41:36.823 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:41:36.883 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:36.895 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:36 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3000288902' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:41:36.895 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:36 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3000288902' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:41:36.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:36 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3000288902' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:41:36.990 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:41:37.002 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:37.002 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:41:37.103 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:41:37.142 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 28 16 53148 53132 118.538 72.0625 0.0119459 0.00843119 2023-09-23T20:41:37.175 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:37.212 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T20:41:37.345 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:41:37.358 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:37.359 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:41:37.460 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:41:37.509 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:37.514 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:41:37.564 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:41:37.569 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:37.569 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:41:37.670 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:41:37.729 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:37.844 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:41:37.850 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:37.851 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:41:37.852 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:37 smithi149 ceph-mon[170394]: pgmap v1142: 119 pgs: 119 active+clean; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 92 MiB/s wr, 1.48k op/s 2023-09-23T20:41:37.921 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:37 smithi018 ceph-mon[179647]: pgmap v1142: 119 pgs: 119 active+clean; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 92 MiB/s wr, 1.48k op/s 2023-09-23T20:41:37.923 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:37 smithi018 ceph-mon[184144]: pgmap v1142: 119 pgs: 119 active+clean; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 92 MiB/s wr, 1.48k op/s 2023-09-23T20:41:37.951 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:41:38.021 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:38.142 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 29 16 54318 54302 116.972 73.125 0.00925939 0.00854432 2023-09-23T20:41:38.285 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:41:38.288 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:38.289 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:41:38.389 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:41:38.448 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:38.517 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:41:38.525 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:38.526 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:41:38.582 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:41:38.627 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:41:38.714 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:38.777 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:41:38.786 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:38.787 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:41:38.888 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:41:38.952 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:39.055 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:41:39.061 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:39.061 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:41:39.142 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 30 16 55533 55517 115.605 75.9375 0.0143673 0.00864604 2023-09-23T20:41:39.162 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:41:39.215 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:39.308 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:41:39.345 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:41:39.353 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:39.353 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:41:39.454 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:41:39.546 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:39.685 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 7, 3, 2, 4, 6, 5] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T20:41:39.685 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:41:39.686 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-09-23T20:41:39.686 INFO:tasks.thrashosds.thrasher:Setting osd 0 primary_affinity to 1.000000 2023-09-23T20:41:39.686 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 0 1 2023-09-23T20:41:39.734 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:41:39.739 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:39.739 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:41:39.840 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:41:39.893 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:39.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:39 smithi149 ceph-mon[170394]: pgmap v1143: 119 pgs: 119 active+clean; 3.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 89 MiB/s wr, 1.43k op/s 2023-09-23T20:41:40.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:39 smithi018 ceph-mon[179647]: pgmap v1143: 119 pgs: 119 active+clean; 3.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 89 MiB/s wr, 1.43k op/s 2023-09-23T20:41:40.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:39 smithi018 ceph-mon[184144]: pgmap v1143: 119 pgs: 119 active+clean; 3.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 89 MiB/s wr, 1.43k op/s 2023-09-23T20:41:40.142 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 31 16 56595 56579 114.017 66.375 0.00350629 0.0087623 2023-09-23T20:41:40.154 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:41:40.157 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:40.157 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:41:40.258 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:41:40.340 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:40.627 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:41:40.634 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:40.635 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:41:40.726 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:41:40.735 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:41:40.737 INFO:teuthology.orchestra.run.smithi018.stderr:noscrub is unset 2023-09-23T20:41:40.848 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:40.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:40 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1180991944' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:41:40.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:40 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:41:40.894 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:40 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1180991944' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:41:40.894 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:40 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:41:40.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:40 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1180991944' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:41:40.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:40 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:41:41.063 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2023-09-23T20:41:41.063 INFO:teuthology.orchestra.run.smithi018.stdout: "ops": [], 2023-09-23T20:41:41.063 INFO:teuthology.orchestra.run.smithi018.stdout: "complaint_time": 30, 2023-09-23T20:41:41.064 INFO:teuthology.orchestra.run.smithi018.stdout: "num_blocked_ops": 0 2023-09-23T20:41:41.064 INFO:teuthology.orchestra.run.smithi018.stdout:} 2023-09-23T20:41:41.142 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 32 16 57657 57641 112.528 66.375 0.0169894 0.00888217 2023-09-23T20:41:41.155 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:41:41.160 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:41.160 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:41:41.262 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:41:41.326 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:41.377 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:41:41.383 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:41.383 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:41:41.484 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:41:41.552 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:41.695 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:41 smithi149 ceph-mon[170394]: pgmap v1144: 119 pgs: 119 active+clean; 3.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s 2023-09-23T20:41:41.695 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:41 smithi149 ceph-mon[170394]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:41:41.767 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:41 smithi018 ceph-mon[184144]: pgmap v1144: 119 pgs: 119 active+clean; 3.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s 2023-09-23T20:41:41.768 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:41 smithi018 ceph-mon[184144]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:41:41.768 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:41 smithi018 ceph-mon[184144]: failed to encode map e462 with expected crc 2023-09-23T20:41:41.768 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:41 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:41:41.768 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:41 smithi018 ceph-mon[184144]: osdmap e462: 8 total, 8 up, 8 in 2023-09-23T20:41:41.768 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:41 smithi018 ceph-mon[184144]: failed to encode map e462 with expected crc 2023-09-23T20:41:41.768 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:41 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:41:41.768 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:41 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:41:41.768 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:41 smithi018 ceph-mon[179647]: pgmap v1144: 119 pgs: 119 active+clean; 3.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s 2023-09-23T20:41:41.769 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:41 smithi018 ceph-mon[179647]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:41:41.769 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:41 smithi018 ceph-mon[179647]: failed to encode map e462 with expected crc 2023-09-23T20:41:41.769 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:41 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:41:41.769 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:41 smithi018 ceph-mon[179647]: osdmap e462: 8 total, 8 up, 8 in 2023-09-23T20:41:41.769 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:41 smithi018 ceph-mon[179647]: failed to encode map e462 with expected crc 2023-09-23T20:41:41.769 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:41 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:41:41.769 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:41 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:41:41.971 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:41 smithi149 ceph-mon[170394]: failed to encode map e462 with expected crc 2023-09-23T20:41:41.971 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:41 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:41:41.971 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:41 smithi149 ceph-mon[170394]: osdmap e462: 8 total, 8 up, 8 in 2023-09-23T20:41:41.971 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:41 smithi149 ceph-mon[170394]: failed to encode map e462 with expected crc 2023-09-23T20:41:41.971 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:41 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:41:41.971 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:41 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:41:42.143 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 33 16 58894 58878 111.46 77.3125 0.00571607 0.00896757 2023-09-23T20:41:42.151 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:41:42.160 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:42.160 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:41:42.208 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T20:41:42.222 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:41:42.261 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:41:42.294 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:42.341 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:41:42.349 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:42.350 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:41:42.450 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:41:42.498 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:42.550 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:41:42.555 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:42.555 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:41:42.656 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:41:42.755 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:42.822 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:41:42.827 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:42.827 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:41:42.928 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:41:42.963 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:43.012 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:41:43.018 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:43.018 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:41:43.102 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:41:43.120 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:41:43.143 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 34 16 59943 59927 110.11 65.5625 0.0219699 0.00907757 2023-09-23T20:41:43.161 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:41:43.197 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:43.201 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:41:43.253 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T20:41:43.257 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:41:43.297 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:41:43.314 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:41:43.323 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:43.324 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:41:43.348 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:41:43.404 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:41:43.425 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:41:43.440 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:41:43.461 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:41:43.462 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:41:43.462 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:41:43.462 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:41:43.462 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:41:43.462 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:41:43.462 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:41:43.462 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:41:43.478 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:43.530 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:41:43.536 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:43.537 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:41:43.637 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:41:43.667 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:43.714 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:41:43.719 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:43.719 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:41:43.820 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:41:43.883 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:43.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:43 smithi018 ceph-mon[179647]: pgmap v1146: 119 pgs: 119 active+clean; 3.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 78 MiB/s wr, 1.24k op/s 2023-09-23T20:41:43.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:43 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/977357769' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:41:43.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:43 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:41:43.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:43 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:41:43.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:43 smithi018 ceph-mon[184144]: pgmap v1146: 119 pgs: 119 active+clean; 3.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 78 MiB/s wr, 1.24k op/s 2023-09-23T20:41:43.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:43 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/977357769' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:41:43.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:43 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:41:43.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:43 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:41:43.947 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:41:43.953 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:43.954 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:41:44.055 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:41:44.148 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 35 16 60999 60983 108.841 66 0.0209599 0.00918275 2023-09-23T20:41:44.152 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:44.202 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:41:44.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:43 smithi149 ceph-mon[170394]: pgmap v1146: 119 pgs: 119 active+clean; 3.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 78 MiB/s wr, 1.24k op/s 2023-09-23T20:41:44.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:43 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/977357769' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:41:44.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:43 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:41:44.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:43 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:41:44.492 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:41:44.500 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:44.500 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:41:44.601 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:41:44.676 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:44.700 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:41:44.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:44 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:41:44.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:44 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:41:44.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:44 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:41:44.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:44 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:41:44.895 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:44 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:41:44.895 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:44 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:41:44.895 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:44 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:41:44.895 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:44 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:41:45.146 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 36 16 62116 62100 107.757 69.8125 0.00762047 0.00927599 2023-09-23T20:41:45.171 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:41:45.176 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:45.177 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:41:45.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:44 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:41:45.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:44 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:41:45.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:44 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:41:45.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:44 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:41:45.278 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:41:45.335 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:41:44 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:41:44] "GET /metrics HTTP/1.1" 200 36524 "" "Prometheus/2.43.0" 2023-09-23T20:41:45.341 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:45.407 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:41:45.414 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:45.414 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:41:45.515 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:41:45.581 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:45.767 INFO:teuthology.orchestra.run.smithi018.stderr:set osd.0 primary-affinity to 1 (8655362) 2023-09-23T20:41:46.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:45 smithi018 ceph-mon[184144]: pgmap v1147: 119 pgs: 119 active+clean; 3.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2023-09-23T20:41:46.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:45 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/252407728' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T20:41:46.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:45 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T20:41:46.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:45 smithi018 ceph-mon[179647]: pgmap v1147: 119 pgs: 119 active+clean; 3.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2023-09-23T20:41:46.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:45 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/252407728' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T20:41:46.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:45 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T20:41:46.146 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 37 16 63300 63284 106.845 74 0.00461615 0.00935388 2023-09-23T20:41:46.179 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:41:46.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:45 smithi149 ceph-mon[170394]: pgmap v1147: 119 pgs: 119 active+clean; 3.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2023-09-23T20:41:46.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:45 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/252407728' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T20:41:46.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:45 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T20:41:46.734 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:41:46.745 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:46.746 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:41:46.846 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:41:46.931 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:47.111 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:46 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2023-09-23T20:41:47.111 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:46 smithi018 ceph-mon[179647]: osdmap e463: 8 total, 8 up, 8 in 2023-09-23T20:41:47.111 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:46 smithi018 ceph-mon[179647]: failed to encode map e463 with expected crc 2023-09-23T20:41:47.112 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:46 smithi018 ceph-mon[179647]: failed to encode map e463 with expected crc 2023-09-23T20:41:47.112 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:46 smithi018 ceph-mon[179647]: failed to encode map e463 with expected crc 2023-09-23T20:41:47.112 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:46 smithi018 ceph-mon[179647]: failed to encode map e463 with expected crc 2023-09-23T20:41:47.112 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:46 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2023-09-23T20:41:47.112 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:46 smithi018 ceph-mon[184144]: osdmap e463: 8 total, 8 up, 8 in 2023-09-23T20:41:47.112 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:46 smithi018 ceph-mon[184144]: failed to encode map e463 with expected crc 2023-09-23T20:41:47.112 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:46 smithi018 ceph-mon[184144]: failed to encode map e463 with expected crc 2023-09-23T20:41:47.112 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:46 smithi018 ceph-mon[184144]: failed to encode map e463 with expected crc 2023-09-23T20:41:47.112 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:46 smithi018 ceph-mon[184144]: failed to encode map e463 with expected crc 2023-09-23T20:41:47.146 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 38 16 64538 64522 106.07 77.375 0.0133633 0.0094228 2023-09-23T20:41:47.218 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:46 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2023-09-23T20:41:47.219 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:46 smithi149 ceph-mon[170394]: osdmap e463: 8 total, 8 up, 8 in 2023-09-23T20:41:47.219 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:46 smithi149 ceph-mon[170394]: failed to encode map e463 with expected crc 2023-09-23T20:41:47.219 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:46 smithi149 ceph-mon[170394]: failed to encode map e463 with expected crc 2023-09-23T20:41:47.219 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:46 smithi149 ceph-mon[170394]: failed to encode map e463 with expected crc 2023-09-23T20:41:47.219 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:46 smithi149 ceph-mon[170394]: failed to encode map e463 with expected crc 2023-09-23T20:41:47.325 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:41:47.330 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:47.331 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:41:47.431 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:41:47.479 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:47.539 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:41:47.545 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:47.545 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:41:47.646 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:41:47.710 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:47.765 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:47 smithi149 ceph-mon[170394]: pgmap v1149: 119 pgs: 119 active+clean; 3.7 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2023-09-23T20:41:47.776 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:41:47.782 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:47.783 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:41:47.883 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:41:47.945 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:47.999 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:41:48.005 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:48.005 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:41:48.034 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:47 smithi018 ceph-mon[184144]: pgmap v1149: 119 pgs: 119 active+clean; 3.7 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2023-09-23T20:41:48.035 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:47 smithi018 ceph-mon[179647]: pgmap v1149: 119 pgs: 119 active+clean; 3.7 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2023-09-23T20:41:48.087 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:41:48.106 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:41:48.150 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 39 16 65737 65721 105.272 74.9375 0.0208984 0.00949408 2023-09-23T20:41:48.175 INFO:teuthology.orchestra.run.smithi018.stdout:ERROR: (22) Invalid argument 2023-09-23T20:41:48.175 INFO:teuthology.orchestra.run.smithi018.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:41:48.235 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:48.324 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:41:48.331 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:48.332 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:41:48.432 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:41:48.486 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:48.554 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:41:48.559 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:48.559 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:41:48.660 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:41:48.705 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:48.752 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:41:48.756 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:48.756 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:41:48.783 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T20:41:48.858 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:41:48.870 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:48 smithi018 ceph-mon[184144]: 1.1 scrub starts 2023-09-23T20:41:48.871 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:48 smithi018 ceph-mon[184144]: 1.1 scrub ok 2023-09-23T20:41:48.873 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:48 smithi018 ceph-mon[179647]: 1.1 scrub starts 2023-09-23T20:41:48.873 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:48 smithi018 ceph-mon[179647]: 1.1 scrub ok 2023-09-23T20:41:48.953 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:49.148 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:2023-09-23T20:41:49.148823+0000 min lat: 0.00110414 max lat: 0.151505 avg lat: 0.00958059 2023-09-23T20:41:49.148 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T20:41:49.148 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 40 16 66825 66809 104.336 68 0.0189112 0.00958059 2023-09-23T20:41:49.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:48 smithi149 ceph-mon[170394]: 1.1 scrub starts 2023-09-23T20:41:49.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:48 smithi149 ceph-mon[170394]: 1.1 scrub ok 2023-09-23T20:41:49.423 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:41:49.433 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:49.433 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:41:49.534 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:41:49.618 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:49.779 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:49 smithi018 ceph-mon[184144]: pgmap v1150: 119 pgs: 119 active+clean; 3.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-09-23T20:41:49.779 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:49 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/4164034082' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:41:49.782 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:49 smithi018 ceph-mon[179647]: pgmap v1150: 119 pgs: 119 active+clean; 3.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-09-23T20:41:49.782 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:49 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/4164034082' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:41:49.809 INFO:teuthology.orchestra.run.smithi018.stderr:nodeep-scrub is unset 2023-09-23T20:41:49.858 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:41:50.148 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 41 16 67900 67884 103.43 67.1875 0.0181972 0.00966333 2023-09-23T20:41:50.206 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:41:50.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:49 smithi149 ceph-mon[170394]: pgmap v1150: 119 pgs: 119 active+clean; 3.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-09-23T20:41:50.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:49 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/4164034082' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:41:50.218 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:50.218 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:41:50.235 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:41:50.274 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:41:50.316 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:41:50.318 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:41:50.355 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:41:50.366 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:50.387 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:41:50.422 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:41:50.427 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:41:50.429 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:50.429 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:41:50.460 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:41:50.499 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:41:50.515 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:41:50.515 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:41:50.515 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:41:50.515 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:41:50.515 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:41:50.515 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:41:50.515 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:41:50.515 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:41:50.530 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:41:50.594 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:50.657 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:41:50.662 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:50.663 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:41:50.764 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:41:50.842 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:50.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:50 smithi149 ceph-mon[170394]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:41:50.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:50 smithi149 ceph-mon[170394]: Cluster is now healthy 2023-09-23T20:41:50.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:50 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/4164034082' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:41:50.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:50 smithi149 ceph-mon[170394]: osdmap e464: 8 total, 8 up, 8 in 2023-09-23T20:41:50.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:50 smithi149 ceph-mon[170394]: failed to encode map e464 with expected crc 2023-09-23T20:41:50.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:50 smithi149 ceph-mon[170394]: failed to encode map e464 with expected crc 2023-09-23T20:41:50.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:50 smithi149 ceph-mon[170394]: failed to encode map e464 with expected crc 2023-09-23T20:41:50.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:50 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3367778114' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:41:51.043 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:50 smithi018 ceph-mon[184144]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:41:51.044 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:50 smithi018 ceph-mon[184144]: Cluster is now healthy 2023-09-23T20:41:51.044 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:50 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/4164034082' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:41:51.044 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:50 smithi018 ceph-mon[184144]: osdmap e464: 8 total, 8 up, 8 in 2023-09-23T20:41:51.044 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:50 smithi018 ceph-mon[184144]: failed to encode map e464 with expected crc 2023-09-23T20:41:51.044 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:50 smithi018 ceph-mon[184144]: failed to encode map e464 with expected crc 2023-09-23T20:41:51.044 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:50 smithi018 ceph-mon[184144]: failed to encode map e464 with expected crc 2023-09-23T20:41:51.044 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:50 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3367778114' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:41:51.046 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:41:51.047 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:50 smithi018 ceph-mon[179647]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:41:51.047 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:50 smithi018 ceph-mon[179647]: Cluster is now healthy 2023-09-23T20:41:51.047 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:50 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/4164034082' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:41:51.047 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:50 smithi018 ceph-mon[179647]: osdmap e464: 8 total, 8 up, 8 in 2023-09-23T20:41:51.047 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:50 smithi018 ceph-mon[179647]: failed to encode map e464 with expected crc 2023-09-23T20:41:51.047 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:50 smithi018 ceph-mon[179647]: failed to encode map e464 with expected crc 2023-09-23T20:41:51.047 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:50 smithi018 ceph-mon[179647]: failed to encode map e464 with expected crc 2023-09-23T20:41:51.047 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:50 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3367778114' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:41:51.052 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:51.052 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:41:51.148 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 42 16 69049 69033 102.677 71.8125 0.0125837 0.00973489 2023-09-23T20:41:51.153 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:41:51.238 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:51.351 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:41:51.358 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:51.359 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:41:51.459 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:41:51.527 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:51.615 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:41:51.622 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:51.625 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:41:51.726 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:41:51.803 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:51.872 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:41:51.878 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:51.878 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:51 smithi018 ceph-mon[184144]: pgmap v1152: 119 pgs: 119 active+clean; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s 2023-09-23T20:41:51.879 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:41:51.879 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:51 smithi018 ceph-mon[179647]: pgmap v1152: 119 pgs: 119 active+clean; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s 2023-09-23T20:41:51.980 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:41:52.044 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:52.108 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:41:52.120 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:52.120 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:51 smithi149 ceph-mon[170394]: pgmap v1152: 119 pgs: 119 active+clean; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s 2023-09-23T20:41:52.121 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:41:52.148 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 43 16 70210 70194 101.977 72.5625 0.00927005 0.00980116 2023-09-23T20:41:52.222 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:41:52.293 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:52.422 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T20:41:52.439 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 7, 3, 2, 4, 6, 5] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T20:41:52.440 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:41:52.440 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-09-23T20:41:52.440 INFO:tasks.thrashosds.thrasher:Setting osd 0 primary_affinity to 0.000000 2023-09-23T20:41:52.440 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 0 0 2023-09-23T20:41:52.894 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2023-09-23T20:41:52.894 INFO:teuthology.orchestra.run.smithi018.stdout: "ops": [], 2023-09-23T20:41:52.895 INFO:teuthology.orchestra.run.smithi018.stdout: "num_ops": 0 2023-09-23T20:41:52.895 INFO:teuthology.orchestra.run.smithi018.stdout:} 2023-09-23T20:41:52.911 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:41:52.918 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:52.919 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:41:53.019 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:41:53.066 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:53.126 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:41:53.127 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:41:53.133 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:53.134 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:41:53.150 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 44 16 71228 71212 101.104 63.625 0.0114301 0.00988571 2023-09-23T20:41:53.234 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:41:53.280 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:53.337 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:41:53.342 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:53.343 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:41:53.443 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:41:53.530 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:53.600 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:41:53.606 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:53.607 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:41:53.664 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:41:53.707 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:41:53.721 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T20:41:53.803 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:53.872 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:41:53.882 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:53.883 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:41:53.984 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:41:54.025 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:54.078 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:41:54.086 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:54.086 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:41:54.087 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:53 smithi149 ceph-mon[170394]: pgmap v1153: 119 pgs: 119 active+clean; 4.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s 2023-09-23T20:41:54.087 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:53 smithi149 ceph-mon[170394]: 1.18 scrub starts 2023-09-23T20:41:54.087 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:53 smithi149 ceph-mon[170394]: 1.18 scrub ok 2023-09-23T20:41:54.091 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:53 smithi018 ceph-mon[179647]: pgmap v1153: 119 pgs: 119 active+clean; 4.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s 2023-09-23T20:41:54.091 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:53 smithi018 ceph-mon[179647]: 1.18 scrub starts 2023-09-23T20:41:54.091 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:53 smithi018 ceph-mon[179647]: 1.18 scrub ok 2023-09-23T20:41:54.091 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:53 smithi018 ceph-mon[184144]: pgmap v1153: 119 pgs: 119 active+clean; 4.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s 2023-09-23T20:41:54.091 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:53 smithi018 ceph-mon[184144]: 1.18 scrub starts 2023-09-23T20:41:54.091 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:53 smithi018 ceph-mon[184144]: 1.18 scrub ok 2023-09-23T20:41:54.149 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 45 16 72219 72203 100.233 61.9375 0.010425 0.00997209 2023-09-23T20:41:54.187 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:41:54.270 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:54.346 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:41:54.352 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:54.353 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:41:54.353 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:41:54.374 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:41:54.455 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:41:54.520 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:54.663 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:41:54.667 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:54.667 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:41:54.766 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:41:54.769 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:41:54.810 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:54.866 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:41:54.872 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:54.873 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:41:54.932 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:54 smithi018 ceph-mon[179647]: pgmap v1154: 119 pgs: 119 active+clean; 4.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 69 MiB/s wr, 1.09k op/s 2023-09-23T20:41:54.932 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:54 smithi018 ceph-mon[184144]: pgmap v1154: 119 pgs: 119 active+clean; 4.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 69 MiB/s wr, 1.09k op/s 2023-09-23T20:41:54.973 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:41:55.046 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:55.150 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 46 16 73254 73238 99.4608 64.6875 0.0151002 0.0100482 2023-09-23T20:41:55.164 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:41:55.170 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:55.171 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:41:55.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:54 smithi149 ceph-mon[170394]: pgmap v1154: 119 pgs: 119 active+clean; 4.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 69 MiB/s wr, 1.09k op/s 2023-09-23T20:41:55.224 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:41:54 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:41:54] "GET /metrics HTTP/1.1" 200 36524 "" "Prometheus/2.43.0" 2023-09-23T20:41:55.272 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:41:55.348 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:55.682 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:41:55.693 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:55.694 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:41:55.795 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:41:55.833 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:55.894 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:41:55.900 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:55.900 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:41:56.001 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:41:56.072 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:56.151 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 47 16 74385 74369 98.8466 70.6875 0.00925811 0.0101117 2023-09-23T20:41:56.252 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:41:56.265 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:56.265 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:41:56.366 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:41:56.438 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:56.563 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:41:56.567 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:56.568 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:41:56.668 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:41:56.742 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:57.151 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 48 16 75503 75487 98.2431 69.875 0.0113222 0.0101738 2023-09-23T20:41:57.392 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:41:57.400 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:57.400 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:41:57.501 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:41:57.577 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:57.692 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:41:57.697 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:57.698 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:41:57.698 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:57 smithi018 ceph-mon[184144]: pgmap v1155: 119 pgs: 119 active+clean; 4.4 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-09-23T20:41:57.699 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:57 smithi018 ceph-mon[179647]: pgmap v1155: 119 pgs: 119 active+clean; 4.4 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-09-23T20:41:57.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:57 smithi149 ceph-mon[170394]: pgmap v1155: 119 pgs: 119 active+clean; 4.4 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2023-09-23T20:41:57.798 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:41:57.862 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:57.923 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:41:57.931 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:57.931 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:41:58.032 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:41:58.102 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:58.151 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 49 16 76587 76571 97.6207 67.75 0.00828463 0.0102389 2023-09-23T20:41:58.274 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:41:58.282 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:58.282 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:41:58.383 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:41:58.450 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:58.473 INFO:teuthology.orchestra.run.smithi018.stderr:set osd.0 primary-affinity to 0 (802) 2023-09-23T20:41:58.516 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:41:58.521 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:58.521 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:41:58.622 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:41:58.651 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:58.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:58 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3404904042' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.0}]: dispatch 2023-09-23T20:41:58.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:58 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3404904042' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.0}]: dispatch 2023-09-23T20:41:58.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:58 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3404904042' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.0}]: dispatch 2023-09-23T20:41:59.151 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 50 16 77025 77009 96.2162 27.375 0.00312315 0.0102589 2023-09-23T20:41:59.307 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:41:59.313 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:59.313 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:41:59.414 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:41:59.446 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:59.532 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:41:59.536 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:41:59.537 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:41:59.638 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:41:59.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:59 smithi018 ceph-mon[179647]: pgmap v1156: 119 pgs: 119 active+clean; 4.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s 2023-09-23T20:41:59.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:59 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3404904042' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.0}]': finished 2023-09-23T20:41:59.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:59 smithi018 ceph-mon[179647]: osdmap e465: 8 total, 8 up, 8 in 2023-09-23T20:41:59.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:59 smithi018 ceph-mon[179647]: failed to encode map e465 with expected crc 2023-09-23T20:41:59.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:59 smithi018 ceph-mon[179647]: failed to encode map e465 with expected crc 2023-09-23T20:41:59.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:41:59 smithi018 ceph-mon[179647]: failed to encode map e465 with expected crc 2023-09-23T20:41:59.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:59 smithi018 ceph-mon[184144]: pgmap v1156: 119 pgs: 119 active+clean; 4.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s 2023-09-23T20:41:59.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:59 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3404904042' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.0}]': finished 2023-09-23T20:41:59.643 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:59 smithi018 ceph-mon[184144]: osdmap e465: 8 total, 8 up, 8 in 2023-09-23T20:41:59.643 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:59 smithi018 ceph-mon[184144]: failed to encode map e465 with expected crc 2023-09-23T20:41:59.643 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:59 smithi018 ceph-mon[184144]: failed to encode map e465 with expected crc 2023-09-23T20:41:59.643 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:41:59 smithi018 ceph-mon[184144]: failed to encode map e465 with expected crc 2023-09-23T20:41:59.691 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:41:59.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:59 smithi149 ceph-mon[170394]: pgmap v1156: 119 pgs: 119 active+clean; 4.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s 2023-09-23T20:41:59.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:59 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3404904042' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.0}]': finished 2023-09-23T20:41:59.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:59 smithi149 ceph-mon[170394]: osdmap e465: 8 total, 8 up, 8 in 2023-09-23T20:41:59.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:59 smithi149 ceph-mon[170394]: failed to encode map e465 with expected crc 2023-09-23T20:41:59.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:59 smithi149 ceph-mon[170394]: failed to encode map e465 with expected crc 2023-09-23T20:41:59.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:41:59 smithi149 ceph-mon[170394]: failed to encode map e465 with expected crc 2023-09-23T20:42:00.151 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 51 16 77793 77777 95.2711 48 0.0149961 0.0104918 2023-09-23T20:42:00.719 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:00 smithi149 ceph-mon[170394]: failed to encode map e466 with expected crc 2023-09-23T20:42:00.719 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:00 smithi149 ceph-mon[170394]: failed to encode map e466 with expected crc 2023-09-23T20:42:00.720 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:00 smithi149 ceph-mon[170394]: failed to encode map e466 with expected crc 2023-09-23T20:42:00.720 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:00 smithi149 ceph-mon[170394]: osdmap e466: 8 total, 8 up, 8 in 2023-09-23T20:42:00.720 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:00 smithi149 ceph-mon[170394]: failed to encode map e466 with expected crc 2023-09-23T20:42:00.720 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:00 smithi149 ceph-mon[170394]: failed to encode map e466 with expected crc 2023-09-23T20:42:00.720 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:00 smithi149 ceph-mon[170394]: failed to encode map e466 with expected crc 2023-09-23T20:42:00.720 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:00 smithi149 ceph-mon[170394]: failed to encode map e466 with expected crc 2023-09-23T20:42:00.720 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:00 smithi149 ceph-mon[170394]: failed to encode map e466 with expected crc 2023-09-23T20:42:00.720 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:00 smithi149 ceph-mon[170394]: failed to encode map e466 with expected crc 2023-09-23T20:42:00.720 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:00 smithi149 ceph-mon[170394]: failed to encode map e466 with expected crc 2023-09-23T20:42:00.720 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:00 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:42:00.720 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:00 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:42:00.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:00 smithi018 ceph-mon[184144]: failed to encode map e466 with expected crc 2023-09-23T20:42:00.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:00 smithi018 ceph-mon[184144]: failed to encode map e466 with expected crc 2023-09-23T20:42:00.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:00 smithi018 ceph-mon[184144]: failed to encode map e466 with expected crc 2023-09-23T20:42:00.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:00 smithi018 ceph-mon[184144]: osdmap e466: 8 total, 8 up, 8 in 2023-09-23T20:42:00.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:00 smithi018 ceph-mon[184144]: failed to encode map e466 with expected crc 2023-09-23T20:42:00.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:00 smithi018 ceph-mon[184144]: failed to encode map e466 with expected crc 2023-09-23T20:42:00.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:00 smithi018 ceph-mon[184144]: failed to encode map e466 with expected crc 2023-09-23T20:42:00.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:00 smithi018 ceph-mon[184144]: failed to encode map e466 with expected crc 2023-09-23T20:42:00.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:00 smithi018 ceph-mon[184144]: failed to encode map e466 with expected crc 2023-09-23T20:42:00.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:00 smithi018 ceph-mon[184144]: failed to encode map e466 with expected crc 2023-09-23T20:42:00.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:00 smithi018 ceph-mon[184144]: failed to encode map e466 with expected crc 2023-09-23T20:42:00.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:00 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:42:00.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:00 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:42:00.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:00 smithi018 ceph-mon[179647]: failed to encode map e466 with expected crc 2023-09-23T20:42:00.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:00 smithi018 ceph-mon[179647]: failed to encode map e466 with expected crc 2023-09-23T20:42:00.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:00 smithi018 ceph-mon[179647]: failed to encode map e466 with expected crc 2023-09-23T20:42:00.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:00 smithi018 ceph-mon[179647]: osdmap e466: 8 total, 8 up, 8 in 2023-09-23T20:42:00.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:00 smithi018 ceph-mon[179647]: failed to encode map e466 with expected crc 2023-09-23T20:42:00.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:00 smithi018 ceph-mon[179647]: failed to encode map e466 with expected crc 2023-09-23T20:42:00.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:00 smithi018 ceph-mon[179647]: failed to encode map e466 with expected crc 2023-09-23T20:42:00.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:00 smithi018 ceph-mon[179647]: failed to encode map e466 with expected crc 2023-09-23T20:42:00.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:00 smithi018 ceph-mon[179647]: failed to encode map e466 with expected crc 2023-09-23T20:42:00.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:00 smithi018 ceph-mon[179647]: failed to encode map e466 with expected crc 2023-09-23T20:42:00.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:00 smithi018 ceph-mon[179647]: failed to encode map e466 with expected crc 2023-09-23T20:42:00.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:00 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:42:00.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:00 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:42:01.151 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 52 16 78941 78925 94.8188 71.75 0.00428916 0.0105403 2023-09-23T20:42:01.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:01 smithi149 ceph-mon[170394]: pgmap v1159: 119 pgs: 1 peering, 118 active+clean; 4.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s 2023-09-23T20:42:01.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:01 smithi149 ceph-mon[170394]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T20:42:01.808 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:01 smithi018 ceph-mon[184144]: pgmap v1159: 119 pgs: 1 peering, 118 active+clean; 4.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s 2023-09-23T20:42:01.809 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:01 smithi018 ceph-mon[184144]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T20:42:01.809 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:01 smithi018 ceph-mon[179647]: pgmap v1159: 119 pgs: 1 peering, 118 active+clean; 4.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s 2023-09-23T20:42:01.809 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:01 smithi018 ceph-mon[179647]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T20:42:02.151 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 53 16 80012 79996 94.2928 66.9375 0.01549 0.0106009 2023-09-23T20:42:02.837 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:42:02.846 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:02.847 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:42:02.947 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:42:03.012 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:03.077 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:42:03.086 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:03.086 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:42:03.154 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 54 16 81141 81125 93.849 70.5625 0.0182839 0.0106505 2023-09-23T20:42:03.187 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:42:03.272 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:03.710 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:03 smithi018 ceph-mon[179647]: pgmap v1160: 119 pgs: 1 peering, 118 active+clean; 4.7 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 56 MiB/s wr, 902 op/s 2023-09-23T20:42:03.710 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:03 smithi018 ceph-mon[184144]: pgmap v1160: 119 pgs: 1 peering, 118 active+clean; 4.7 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 56 MiB/s wr, 902 op/s 2023-09-23T20:42:03.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:03 smithi149 ceph-mon[170394]: pgmap v1160: 119 pgs: 1 peering, 118 active+clean; 4.7 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 56 MiB/s wr, 902 op/s 2023-09-23T20:42:03.880 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:42:03.885 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:03.885 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:42:03.934 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2023-09-23T20:42:03.934 INFO:teuthology.orchestra.run.smithi018.stdout: "ops": [], 2023-09-23T20:42:03.935 INFO:teuthology.orchestra.run.smithi018.stdout: "complaint_time": 30, 2023-09-23T20:42:03.935 INFO:teuthology.orchestra.run.smithi018.stdout: "num_blocked_ops": 0 2023-09-23T20:42:03.935 INFO:teuthology.orchestra.run.smithi018.stdout:} 2023-09-23T20:42:03.986 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:42:04.041 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:04.066 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:42:04.099 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:42:04.105 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:04.106 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:42:04.108 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:42:04.153 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:42:04.154 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 55 16 82142 82126 93.2803 62.5625 0.0159322 0.0107154 2023-09-23T20:42:04.188 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:42:04.207 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:42:04.242 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:42:04.274 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:42:04.284 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:04.322 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:42:04.355 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:42:04.358 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:42:04.358 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:42:04.358 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:42:04.359 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:42:04.359 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:42:04.359 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:42:04.359 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:42:04.359 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:42:04.447 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:42:04.461 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:04.462 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:42:04.560 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:04 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/596032446' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:42:04.561 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:04 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/596032446' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:42:04.562 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T20:42:04.563 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:42:04.664 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:04.853 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:42:04.862 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:04.863 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:42:04.964 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:42:04.977 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:04 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/596032446' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:42:05.035 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:05.097 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:42:05.104 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:05.105 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:42:05.145 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:42:04 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:42:04] "GET /metrics HTTP/1.1" 200 36515 "" "Prometheus/2.43.0" 2023-09-23T20:42:05.154 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 56 16 83071 83055 92.6515 58.0625 0.0114207 0.0107879 2023-09-23T20:42:05.206 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:42:05.275 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:05.334 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:42:05.345 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:05.346 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:42:05.446 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:42:05.457 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:42:05.519 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:05.535 INFO:teuthology.orchestra.run.smithi018.stderr:noscrub is set 2023-09-23T20:42:05.687 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:42:05.693 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:05.693 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:42:05.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:05 smithi149 ceph-mon[170394]: pgmap v1161: 119 pgs: 119 active+clean; 4.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 62 MiB/s wr, 994 op/s; 7.9 KiB/s, 0 objects/s recovering 2023-09-23T20:42:05.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:05 smithi149 ceph-mon[170394]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T20:42:05.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:05 smithi149 ceph-mon[170394]: Cluster is now healthy 2023-09-23T20:42:05.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:05 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2009558546' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:42:05.794 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:42:05.858 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:05.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:05 smithi018 ceph-mon[184144]: pgmap v1161: 119 pgs: 119 active+clean; 4.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 62 MiB/s wr, 994 op/s; 7.9 KiB/s, 0 objects/s recovering 2023-09-23T20:42:05.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:05 smithi018 ceph-mon[184144]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T20:42:05.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:05 smithi018 ceph-mon[184144]: Cluster is now healthy 2023-09-23T20:42:05.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:05 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2009558546' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:42:05.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:05 smithi018 ceph-mon[179647]: pgmap v1161: 119 pgs: 119 active+clean; 4.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 62 MiB/s wr, 994 op/s; 7.9 KiB/s, 0 objects/s recovering 2023-09-23T20:42:05.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:05 smithi018 ceph-mon[179647]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T20:42:05.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:05 smithi018 ceph-mon[179647]: Cluster is now healthy 2023-09-23T20:42:05.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:05 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2009558546' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:42:05.929 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:42:05.936 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:05.936 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:42:06.038 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:42:06.110 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:06.156 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 57 16 84118 84102 92.1712 65.4375 0.0137194 0.0108439 2023-09-23T20:42:06.174 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:42:06.183 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:06.183 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:42:06.284 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:42:06.348 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:06.405 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:42:06.412 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:06.412 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:42:06.513 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:42:06.564 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:06 smithi018 ceph-mon[179647]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:42:06.564 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:06 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2009558546' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:42:06.564 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:06 smithi018 ceph-mon[179647]: osdmap e467: 8 total, 8 up, 8 in 2023-09-23T20:42:06.565 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:06 smithi018 ceph-mon[179647]: failed to encode map e467 with expected crc 2023-09-23T20:42:06.565 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:06 smithi018 ceph-mon[179647]: failed to encode map e467 with expected crc 2023-09-23T20:42:06.565 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:06 smithi018 ceph-mon[179647]: failed to encode map e467 with expected crc 2023-09-23T20:42:06.567 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:06 smithi018 ceph-mon[184144]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:42:06.568 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:06 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2009558546' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:42:06.568 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:06 smithi018 ceph-mon[184144]: osdmap e467: 8 total, 8 up, 8 in 2023-09-23T20:42:06.568 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:06 smithi018 ceph-mon[184144]: failed to encode map e467 with expected crc 2023-09-23T20:42:06.568 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:06 smithi018 ceph-mon[184144]: failed to encode map e467 with expected crc 2023-09-23T20:42:06.568 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:06 smithi018 ceph-mon[184144]: failed to encode map e467 with expected crc 2023-09-23T20:42:06.570 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:06.718 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:06 smithi149 ceph-mon[170394]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:42:06.718 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:06 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2009558546' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:42:06.718 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:06 smithi149 ceph-mon[170394]: osdmap e467: 8 total, 8 up, 8 in 2023-09-23T20:42:06.718 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:06 smithi149 ceph-mon[170394]: failed to encode map e467 with expected crc 2023-09-23T20:42:06.718 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:06 smithi149 ceph-mon[170394]: failed to encode map e467 with expected crc 2023-09-23T20:42:06.718 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:06 smithi149 ceph-mon[170394]: failed to encode map e467 with expected crc 2023-09-23T20:42:06.738 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:42:06.743 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:06.744 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:42:06.844 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:42:06.928 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:06.939 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:42:07.037 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:42:07.044 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:07.044 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:42:07.145 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:42:07.182 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 58 16 85162 85146 91.7069 65.25 0.0236862 0.0108999 2023-09-23T20:42:07.273 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:07.346 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:42:07.351 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:07.351 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:42:07.452 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:42:07.515 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:07 smithi018 ceph-mon[184144]: pgmap v1163: 119 pgs: 119 active+clean; 5.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s; 8.3 KiB/s, 0 objects/s recovering 2023-09-23T20:42:07.515 INFO:teuthology.orchestra.run.smithi018.stdout:ERROR: (22) Invalid argument 2023-09-23T20:42:07.515 INFO:teuthology.orchestra.run.smithi018.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:42:07.545 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:07.607 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:42:07.617 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:07.618 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:42:07.719 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:42:07.777 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:07 smithi018 ceph-mon[179647]: pgmap v1163: 119 pgs: 119 active+clean; 5.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s; 8.3 KiB/s, 0 objects/s recovering 2023-09-23T20:42:07.780 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:07.936 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:42:07.942 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:07.943 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:42:07.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:07 smithi149 ceph-mon[170394]: pgmap v1163: 119 pgs: 119 active+clean; 5.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s; 8.3 KiB/s, 0 objects/s recovering 2023-09-23T20:42:07.995 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:42:08.000 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T20:42:08.025 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T20:42:08.044 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:42:08.125 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:08.156 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 59 16 86097 86081 91.1432 58.4375 0.0237001 0.0109665 2023-09-23T20:42:08.200 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:42:08.207 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:08.207 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:42:08.308 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:42:08.349 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:08.352 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 7, 3, 2, 4, 6, 5] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T20:42:08.352 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:42:08.352 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '1000', 'type': 'test-reweight-by-utilization'} 2023-09-23T20:42:08.353 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd test-reweight-by-utilization 1000 1.0 2023-09-23T20:42:08.402 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:42:08.411 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:08.411 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:42:08.512 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:42:08.554 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:08.609 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:42:08.615 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:08.615 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:42:08.716 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:42:08.754 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:08.817 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:42:08.832 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:08.833 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:42:08.934 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:42:09.007 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:09.158 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:2023-09-23T20:42:09.159180+0000 min lat: 0.00110414 max lat: 1.04101 avg lat: 0.0110388 2023-09-23T20:42:09.159 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T20:42:09.159 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 60 16 86990 86974 90.5515 55.8125 0.00753662 0.0110388 2023-09-23T20:42:09.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:09 smithi018 ceph-mon[184144]: pgmap v1164: 119 pgs: 119 active+clean; 5.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 60 MiB/s wr, 966 op/s; 7.3 KiB/s, 0 objects/s recovering 2023-09-23T20:42:09.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:09 smithi018 ceph-mon[179647]: pgmap v1164: 119 pgs: 119 active+clean; 5.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 60 MiB/s wr, 966 op/s; 7.3 KiB/s, 0 objects/s recovering 2023-09-23T20:42:09.968 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:09 smithi149 ceph-mon[170394]: pgmap v1164: 119 pgs: 119 active+clean; 5.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 60 MiB/s wr, 966 op/s; 7.3 KiB/s, 0 objects/s recovering 2023-09-23T20:42:10.159 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 61 16 88078 88062 90.1818 68 0.0229071 0.0110837 2023-09-23T20:42:10.571 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:42:10.578 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:10.578 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:42:10.679 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:42:10.713 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:10.758 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:42:10.762 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:10.762 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:42:10.863 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:42:10.952 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:11.159 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 62 16 89114 89098 89.7716 64.75 0.0121625 0.0111348 2023-09-23T20:42:11.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:11 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:42:11.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:11 smithi018 ceph-mon[184144]: pgmap v1165: 119 pgs: 119 active+clean; 5.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 62 MiB/s wr, 993 op/s; 6.3 KiB/s, 0 objects/s recovering 2023-09-23T20:42:11.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:11 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:42:11.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:11 smithi018 ceph-mon[179647]: pgmap v1165: 119 pgs: 119 active+clean; 5.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 62 MiB/s wr, 993 op/s; 6.3 KiB/s, 0 objects/s recovering 2023-09-23T20:42:11.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:11 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:42:11.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:11 smithi149 ceph-mon[170394]: pgmap v1165: 119 pgs: 119 active+clean; 5.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 62 MiB/s wr, 993 op/s; 6.3 KiB/s, 0 objects/s recovering 2023-09-23T20:42:11.760 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:42:11.767 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:11.768 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:42:11.845 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:42:11.845 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:42:11.848 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:42:11.869 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:42:11.915 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:11.987 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:42:11.995 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:11.995 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:42:12.096 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:42:12.159 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 63 16 90149 90133 89.3735 64.6875 0.00948516 0.0111838 2023-09-23T20:42:12.159 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:12.221 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:42:12.228 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:12.229 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:42:12.330 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:42:12.399 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:12.472 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:42:12.481 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:12.481 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:42:12.582 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:42:12.624 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:13.038 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:42:13.046 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:13.046 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:42:13.085 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:42:13.127 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:42:13.147 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:42:13.159 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 64 16 91121 91105 88.9264 60.75 0.0149809 0.0112398 2023-09-23T20:42:13.160 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:42:13.203 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:42:13.231 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:13.239 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:42:13.286 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:42:13.337 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:42:13.386 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:42:13.401 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:42:13.401 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:42:13.401 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:42:13.401 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:42:13.401 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:42:13.401 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:42:13.401 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:42:13.401 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:42:13.538 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:42:13.544 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:13 smithi018 ceph-mon[184144]: pgmap v1166: 119 pgs: 119 active+clean; 5.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s; 6.4 KiB/s, 0 objects/s recovering 2023-09-23T20:42:13.544 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:13 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/4073264958' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:42:13.544 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:13 smithi018 ceph-mon[179647]: pgmap v1166: 119 pgs: 119 active+clean; 5.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s; 6.4 KiB/s, 0 objects/s recovering 2023-09-23T20:42:13.544 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:13 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/4073264958' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:42:13.546 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:13.547 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:42:13.647 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:42:13.710 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:13.760 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:42:13.762 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:13 smithi149 ceph-mon[170394]: pgmap v1166: 119 pgs: 119 active+clean; 5.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s; 6.4 KiB/s, 0 objects/s recovering 2023-09-23T20:42:13.763 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:13 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/4073264958' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:42:13.763 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:13.764 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:42:13.864 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:42:13.901 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:13.957 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:42:13.969 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:13.970 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:42:14.070 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:42:14.157 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:14.159 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 65 16 91987 91971 88.3911 54.125 0.0132279 0.0113087 2023-09-23T20:42:14.230 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:42:14.239 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:14.239 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:42:14.340 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:42:14.366 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:14.400 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:42:14.403 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:14.403 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:42:14.504 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:42:14.586 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:14.665 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:42:14.670 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:14.671 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:42:14.772 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:42:14.826 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:14.882 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:42:14.891 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:14.891 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:42:14.992 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:42:15.059 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:15.128 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:42:15.141 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:15.142 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:42:15.159 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 66 16 92842 92826 87.8616 53.4375 0.0302113 0.0113756 2023-09-23T20:42:15.243 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:42:15.300 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:42:14 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:42:14] "GET /metrics HTTP/1.1" 200 36515 "" "Prometheus/2.43.0" 2023-09-23T20:42:15.306 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:15.648 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:15 smithi018 ceph-mon[179647]: pgmap v1167: 119 pgs: 119 active+clean; 5.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-09-23T20:42:15.648 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:15 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:42:15.648 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:15 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:42:15.648 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:15 smithi018 ceph-mon[184144]: pgmap v1167: 119 pgs: 119 active+clean; 5.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-09-23T20:42:15.648 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:15 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:42:15.648 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:15 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:42:15.969 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:15 smithi149 ceph-mon[170394]: pgmap v1167: 119 pgs: 119 active+clean; 5.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-09-23T20:42:15.969 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:15 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:42:15.969 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:15 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:42:16.159 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 67 16 93893 93877 87.5307 65.6875 0.0152177 0.0114202 2023-09-23T20:42:17.159 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 68 16 94932 94916 87.1985 64.9375 0.0109135 0.0114639 2023-09-23T20:42:17.843 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:42:17.853 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:17 smithi018 ceph-mon[179647]: pgmap v1168: 119 pgs: 119 active+clean; 5.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s 2023-09-23T20:42:17.854 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:17 smithi018 ceph-mon[184144]: pgmap v1168: 119 pgs: 119 active+clean; 5.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s 2023-09-23T20:42:17.854 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:17.855 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:42:17.956 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:42:17.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:17 smithi149 ceph-mon[170394]: pgmap v1168: 119 pgs: 119 active+clean; 5.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s 2023-09-23T20:42:17.994 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:18.055 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:42:18.062 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:18.063 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:42:18.163 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 69 16 95824 95808 86.7412 55.75 0.0116803 0.0115241 2023-09-23T20:42:18.164 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:42:18.222 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:19.161 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 70 16 96799 96783 86.3727 60.9375 0.0101707 0.0115723 2023-09-23T20:42:19.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:19 smithi018 ceph-mon[184144]: pgmap v1169: 119 pgs: 119 active+clean; 5.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 58 MiB/s wr, 921 op/s 2023-09-23T20:42:19.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:19 smithi018 ceph-mon[179647]: pgmap v1169: 119 pgs: 119 active+clean; 5.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 58 MiB/s wr, 921 op/s 2023-09-23T20:42:19.961 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:42:19.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:19 smithi149 ceph-mon[170394]: pgmap v1169: 119 pgs: 119 active+clean; 5.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 58 MiB/s wr, 921 op/s 2023-09-23T20:42:19.973 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:19.974 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:42:20.074 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:42:20.142 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:20.161 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 71 16 97824 97808 86.0585 64.0625 0.0154545 0.011615 2023-09-23T20:42:20.199 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:42:20.208 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:20.209 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:42:20.309 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:42:20.383 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:20.445 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:42:20.456 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:20.457 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:42:20.558 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:42:20.591 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:42:20.622 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:20.674 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:42:20.681 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:20.681 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:42:20.782 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:42:20.893 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:21.057 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:42:21.067 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:21.067 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:42:21.105 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2023-09-23T20:42:21.105 INFO:teuthology.orchestra.run.smithi018.stdout: "ops": [ 2023-09-23T20:42:21.105 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:42:21.105 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.36450.0:98587 9.2 9:4a490304:::benchmark_data_smithi018_381516_object98586:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e467)", 2023-09-23T20:42:21.105 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:42:21.067014+0000", 2023-09-23T20:42:21.105 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.025287, 2023-09-23T20:42:21.105 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.025331764, 2023-09-23T20:42:21.105 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:42:21.105 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:42:21.105 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:42:21.106 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.36450", 2023-09-23T20:42:21.106 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:42:21.106 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 98587 2023-09-23T20:42:21.106 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:42:21.106 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:42:21.106 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:42:21.106 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:42:21.106 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:42:21.067014+0000", 2023-09-23T20:42:21.106 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:42:21.106 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:42:21.106 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:42:21.107 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:42:21.107 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:42:21.067014+0000", 2023-09-23T20:42:21.107 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:42:21.107 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:42:21.107 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:42:21.107 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:42:21.107 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:42:21.067012+0000", 2023-09-23T20:42:21.107 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:42:21.107 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:42:21.107 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:42:21.107 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:42:21.107 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:42:21.067038+0000", 2023-09-23T20:42:21.108 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.5755999999999999e-05 2023-09-23T20:42:21.108 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:42:21.108 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:42:21.108 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:42:21.108 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:42:21.067053+0000", 2023-09-23T20:42:21.108 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.5119999999999999e-05 2023-09-23T20:42:21.108 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:42:21.108 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:42:21.108 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:42:21.108 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:42:21.067062+0000", 2023-09-23T20:42:21.108 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.8740000000000001e-06 2023-09-23T20:42:21.109 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:42:21.109 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:42:21.109 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:42:21.109 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:42:21.067184+0000", 2023-09-23T20:42:21.109 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00012189900000000001 2023-09-23T20:42:21.109 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:42:21.109 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:42:21.109 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:42:21.109 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:42:21.067332+0000", 2023-09-23T20:42:21.109 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00014771000000000001 2023-09-23T20:42:21.109 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:42:21.109 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:42:21.110 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,2", 2023-09-23T20:42:21.110 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:42:21.067492+0000", 2023-09-23T20:42:21.110 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00016008799999999999 2023-09-23T20:42:21.110 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:42:21.110 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:42:21.110 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:42:21.110 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:42:21.082644+0000", 2023-09-23T20:42:21.110 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.015151714 2023-09-23T20:42:21.110 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:42:21.110 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:42:21.110 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:42:21.111 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:42:21.084945+0000", 2023-09-23T20:42:21.111 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.002301572 2023-09-23T20:42:21.111 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:42:21.111 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:42:21.111 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:42:21.111 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:42:21.111 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:42:21.111 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.36450.0:98588 9.2 9:43a361b2:::benchmark_data_smithi018_381516_object98587:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e467)", 2023-09-23T20:42:21.111 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:42:21.067080+0000", 2023-09-23T20:42:21.111 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.025220926000000001, 2023-09-23T20:42:21.111 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.025385584999999999, 2023-09-23T20:42:21.112 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:42:21.112 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:42:21.112 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:42:21.112 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.36450", 2023-09-23T20:42:21.112 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:42:21.112 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 98588 2023-09-23T20:42:21.112 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:42:21.112 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:42:21.112 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:42:21.112 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:42:21.112 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:42:21.067080+0000", 2023-09-23T20:42:21.113 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:42:21.113 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:42:21.113 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:42:21.113 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:42:21.113 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:42:21.067080+0000", 2023-09-23T20:42:21.113 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:42:21.113 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:42:21.113 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:42:21.113 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:42:21.113 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:42:21.067080+0000", 2023-09-23T20:42:21.113 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:42:21.113 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:42:21.114 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:42:21.114 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:42:21.114 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:42:21.067100+0000", 2023-09-23T20:42:21.114 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.0716999999999998e-05 2023-09-23T20:42:21.114 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:42:21.114 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:42:21.114 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:42:21.114 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:42:21.067108+0000", 2023-09-23T20:42:21.114 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.7449999999999995e-06 2023-09-23T20:42:21.114 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:42:21.114 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:42:21.115 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:42:21.115 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:42:21.067114+0000", 2023-09-23T20:42:21.115 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.8479999999999997e-06 2023-09-23T20:42:21.115 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:42:21.115 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:42:21.115 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:42:21.115 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:42:21.070764+0000", 2023-09-23T20:42:21.115 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0036496779999999999 2023-09-23T20:42:21.115 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:42:21.115 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:42:21.115 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:42:21.116 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:42:21.070950+0000", 2023-09-23T20:42:21.116 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000186019 2023-09-23T20:42:21.116 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:42:21.116 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:42:21.116 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,2", 2023-09-23T20:42:21.116 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:42:21.071102+0000", 2023-09-23T20:42:21.116 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00015220500000000001 2023-09-23T20:42:21.116 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:42:21.116 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:42:21.116 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:42:21.116 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:42:21.082676+0000", 2023-09-23T20:42:21.117 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.011573783000000001 2023-09-23T20:42:21.117 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:42:21.117 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:42:21.117 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:42:21.117 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:42:21.084950+0000", 2023-09-23T20:42:21.117 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0022742069999999999 2023-09-23T20:42:21.117 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:42:21.117 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:42:21.117 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:42:21.117 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:42:21.117 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:42:21.117 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_repop(client.36450.0:98597 9.4 e467/455 9:2bbe4b1f:::benchmark_data_smithi018_381516_object98596:head v 467'6206, mlcod=467'6204)", 2023-09-23T20:42:21.118 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:42:21.089885+0000", 2023-09-23T20:42:21.118 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.0024159939999999999, 2023-09-23T20:42:21.118 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.002680305, 2023-09-23T20:42:21.118 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:42:21.118 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "started", 2023-09-23T20:42:21.118 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:42:21.118 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:42:21.118 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:42:21.118 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:42:21.089885+0000", 2023-09-23T20:42:21.118 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:42:21.118 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:42:21.118 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:42:21.119 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:42:21.119 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:42:21.089885+0000", 2023-09-23T20:42:21.119 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:42:21.119 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:42:21.119 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:42:21.119 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:42:21.119 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:42:21.089884+0000", 2023-09-23T20:42:21.119 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.999999 2023-09-23T20:42:21.119 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:42:21.119 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:42:21.119 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:42:21.119 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:42:21.089911+0000", 2023-09-23T20:42:21.120 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.6424e-05 2023-09-23T20:42:21.120 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:42:21.120 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:42:21.120 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:42:21.120 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:42:21.089928+0000", 2023-09-23T20:42:21.120 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.7764000000000001e-05 2023-09-23T20:42:21.120 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:42:21.120 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:42:21.120 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:42:21.120 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:42:21.089937+0000", 2023-09-23T20:42:21.120 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.0450000000000006e-06 2023-09-23T20:42:21.120 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:42:21.121 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:42:21.121 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:42:21.121 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:42:21.090022+0000", 2023-09-23T20:42:21.121 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.4379e-05 2023-09-23T20:42:21.121 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:42:21.121 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:42:21.121 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:42:21.121 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:42:21.090052+0000", 2023-09-23T20:42:21.121 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.0422000000000001e-05 2023-09-23T20:42:21.121 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:42:21.121 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:42:21.121 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:42:21.122 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:42:21.122 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:42:21.122 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.36450.0:98583 9.2 9:4f36c408:::benchmark_data_smithi018_381516_object98582:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e467)", 2023-09-23T20:42:21.122 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:42:21.058737+0000", 2023-09-23T20:42:21.122 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.033563779000000002, 2023-09-23T20:42:21.122 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.033902241, 2023-09-23T20:42:21.122 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:42:21.122 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:42:21.122 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:42:21.122 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.36450", 2023-09-23T20:42:21.122 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:42:21.123 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 98583 2023-09-23T20:42:21.123 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:42:21.123 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:42:21.123 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:42:21.123 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:42:21.123 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:42:21.058737+0000", 2023-09-23T20:42:21.123 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:42:21.123 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:42:21.123 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:42:21.123 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:42:21.123 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:42:21.058737+0000", 2023-09-23T20:42:21.123 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:42:21.123 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:42:21.124 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:42:21.124 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:42:21.124 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:42:21.058736+0000", 2023-09-23T20:42:21.124 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:42:21.124 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:42:21.124 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:42:21.124 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:42:21.124 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:42:21.058764+0000", 2023-09-23T20:42:21.124 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.8705999999999999e-05 2023-09-23T20:42:21.124 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:42:21.124 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:42:21.124 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:42:21.125 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:42:21.058786+0000", 2023-09-23T20:42:21.125 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.1858999999999999e-05 2023-09-23T20:42:21.125 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:42:21.125 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:42:21.125 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:42:21.125 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:42:21.058994+0000", 2023-09-23T20:42:21.125 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00020824799999999999 2023-09-23T20:42:21.125 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:42:21.125 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:42:21.125 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:42:21.125 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:42:21.059728+0000", 2023-09-23T20:42:21.126 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00073384299999999997 2023-09-23T20:42:21.126 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:42:21.126 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:42:21.126 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:42:21.126 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:42:21.059899+0000", 2023-09-23T20:42:21.126 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00017068699999999999 2023-09-23T20:42:21.126 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:42:21.126 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:42:21.126 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,2", 2023-09-23T20:42:21.126 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:42:21.060059+0000", 2023-09-23T20:42:21.126 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000160421 2023-09-23T20:42:21.126 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:42:21.127 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:42:21.127 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:42:21.127 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:42:21.063502+0000", 2023-09-23T20:42:21.127 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0034427239999999999 2023-09-23T20:42:21.127 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:42:21.127 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:42:21.127 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:42:21.127 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:42:21.063738+0000", 2023-09-23T20:42:21.127 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00023604799999999999 2023-09-23T20:42:21.127 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:42:21.127 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:42:21.128 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:42:21.128 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:42:21.128 INFO:teuthology.orchestra.run.smithi018.stdout: ], 2023-09-23T20:42:21.128 INFO:teuthology.orchestra.run.smithi018.stdout: "num_ops": 4 2023-09-23T20:42:21.128 INFO:teuthology.orchestra.run.smithi018.stdout:} 2023-09-23T20:42:21.161 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 72 16 98646 98630 85.5769 51.375 0.0398208 0.0116799 2023-09-23T20:42:21.168 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:42:21.272 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:21.373 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:42:21.377 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:21.378 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:42:21.479 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:42:21.531 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:21.541 INFO:teuthology.orchestra.run.smithi018.stderr:nodeep-scrub is set 2023-09-23T20:42:21.582 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:42:21.589 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:21.590 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:42:21.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:21 smithi018 ceph-mon[179647]: pgmap v1170: 119 pgs: 119 active+clean; 5.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 62 MiB/s wr, 994 op/s 2023-09-23T20:42:21.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:21 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/4268342287' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:42:21.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:21 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:42:21.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:21 smithi018 ceph-mon[184144]: pgmap v1170: 119 pgs: 119 active+clean; 5.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 62 MiB/s wr, 994 op/s 2023-09-23T20:42:21.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:21 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/4268342287' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:42:21.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:21 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:42:21.690 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:42:21.756 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:21.775 INFO:teuthology.orchestra.run.smithi018.stdout:moved 0 / 357 (0%) 2023-09-23T20:42:21.775 INFO:teuthology.orchestra.run.smithi018.stdout:avg 44.625 2023-09-23T20:42:21.775 INFO:teuthology.orchestra.run.smithi018.stdout:stddev 7.06996 -> 7.06996 (expected baseline 6.24875) 2023-09-23T20:42:21.775 INFO:teuthology.orchestra.run.smithi018.stdout:min osd.2 with 30 -> 30 pgs (0.672269 -> 0.672269 * mean) 2023-09-23T20:42:21.775 INFO:teuthology.orchestra.run.smithi018.stdout:max osd.1 with 52 -> 52 pgs (1.16527 -> 1.16527 * mean) 2023-09-23T20:42:21.775 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:42:21.775 INFO:teuthology.orchestra.run.smithi018.stdout:oload 1000 2023-09-23T20:42:21.775 INFO:teuthology.orchestra.run.smithi018.stdout:max_change 1 2023-09-23T20:42:21.775 INFO:teuthology.orchestra.run.smithi018.stdout:max_change_osds 4 2023-09-23T20:42:21.775 INFO:teuthology.orchestra.run.smithi018.stdout:average_utilization 0.0310 2023-09-23T20:42:21.776 INFO:teuthology.orchestra.run.smithi018.stdout:overload_utilization 0.3104 2023-09-23T20:42:21.782 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T20:42:21.805 INFO:teuthology.orchestra.run.smithi018.stderr:no change 2023-09-23T20:42:21.822 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:42:21.831 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:21.831 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:21 smithi149 ceph-mon[170394]: pgmap v1170: 119 pgs: 119 active+clean; 5.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 62 MiB/s wr, 994 op/s 2023-09-23T20:42:21.831 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:21 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/4268342287' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:42:21.831 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:21 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:42:21.832 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:42:21.932 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:42:21.987 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:22.038 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:42:22.044 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:22.044 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:42:22.145 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:42:22.161 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 73 16 99498 99482 85.1336 53.25 0.0229103 0.0117413 2023-09-23T20:42:22.241 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:22.241 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '1000', 'type': 'reweight-by-utilization'} 2023-09-23T20:42:22.241 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd reweight-by-utilization 1000 3.0 2023-09-23T20:42:22.409 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:42:22.409 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:42:22.418 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:22.418 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:42:22.519 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:42:22.593 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:22.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:22 smithi018 ceph-mon[184144]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:42:22.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:22 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:42:22.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:22 smithi018 ceph-mon[184144]: osdmap e468: 8 total, 8 up, 8 in 2023-09-23T20:42:22.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:22 smithi018 ceph-mon[184144]: failed to encode map e468 with expected crc 2023-09-23T20:42:22.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:22 smithi018 ceph-mon[184144]: from='client.36549 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:42:22.647 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:22 smithi018 ceph-mon[179647]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:42:22.647 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:22 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:42:22.647 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:22 smithi018 ceph-mon[179647]: osdmap e468: 8 total, 8 up, 8 in 2023-09-23T20:42:22.647 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:22 smithi018 ceph-mon[179647]: failed to encode map e468 with expected crc 2023-09-23T20:42:22.647 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:22 smithi018 ceph-mon[179647]: from='client.36549 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:42:22.785 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:42:22.796 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:22.797 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:42:22.898 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:42:22.957 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:22.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:22 smithi149 ceph-mon[170394]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:42:22.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:22 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:42:22.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:22 smithi149 ceph-mon[170394]: osdmap e468: 8 total, 8 up, 8 in 2023-09-23T20:42:22.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:22 smithi149 ceph-mon[170394]: failed to encode map e468 with expected crc 2023-09-23T20:42:22.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:22 smithi149 ceph-mon[170394]: from='client.36549 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:42:23.023 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:42:23.026 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:42:23.051 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:23.051 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:42:23.152 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:42:23.161 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 74 16 100345 100329 84.6986 52.9375 0.0188874 0.0118019 2023-09-23T20:42:23.204 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:23.274 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:42:23.281 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:23.281 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:42:23.382 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:42:23.403 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:42:23.459 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:23.526 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:42:23.531 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:23.532 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:42:23.633 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:42:23.689 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:23.710 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:23 smithi018 ceph-mon[179647]: pgmap v1172: 119 pgs: 119 active+clean; 5.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 61 MiB/s wr, 972 op/s 2023-09-23T20:42:23.710 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:23 smithi018 ceph-mon[184144]: pgmap v1172: 119 pgs: 119 active+clean; 5.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 61 MiB/s wr, 972 op/s 2023-09-23T20:42:23.789 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:42:23.798 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:23.799 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:42:23.900 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:42:23.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:23 smithi149 ceph-mon[170394]: pgmap v1172: 119 pgs: 119 active+clean; 5.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 61 MiB/s wr, 972 op/s 2023-09-23T20:42:23.989 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:24.162 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 75 16 101241 101225 84.3158 56 0.0111555 0.0118548 2023-09-23T20:42:24.195 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:42:24.201 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:24.201 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:42:24.302 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:42:24.383 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:24.475 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T20:42:24.524 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:42:24.536 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:24.536 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:42:24.637 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:42:24.758 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:24.882 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:42:24.920 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:42:24.956 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:42:24.996 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:42:25.030 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:42:25.065 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:42:25.067 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:25.067 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:42:25.069 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:42:25.109 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:42:25.142 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:42:24 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:42:24] "GET /metrics HTTP/1.1" 200 36515 "" "Prometheus/2.43.0" 2023-09-23T20:42:25.142 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:42:25.148 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:42:25.148 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:42:25.148 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:42:25.148 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:42:25.148 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:42:25.148 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:42:25.148 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:42:25.148 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:42:25.165 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 76 16 102084 102068 83.8965 52.6875 0.023882 0.0119147 2023-09-23T20:42:25.168 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:42:25.279 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:25.567 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:25 smithi018 ceph-mon[179647]: pgmap v1173: 119 pgs: 119 active+clean; 6.1 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 59 MiB/s wr, 946 op/s 2023-09-23T20:42:25.568 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:25 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2971458655' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:42:25.710 INFO:teuthology.orchestra.run.smithi018.stdout:ERROR: (22) Invalid argument 2023-09-23T20:42:25.710 INFO:teuthology.orchestra.run.smithi018.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:42:25.769 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:42:25.785 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:25.785 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:42:25.842 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:25 smithi018 ceph-mon[184144]: pgmap v1173: 119 pgs: 119 active+clean; 6.1 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 59 MiB/s wr, 946 op/s 2023-09-23T20:42:25.842 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:25 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2971458655' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:42:25.887 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:42:25.898 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:42:25.965 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:25.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:25 smithi149 ceph-mon[170394]: pgmap v1173: 119 pgs: 119 active+clean; 6.1 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 59 MiB/s wr, 946 op/s 2023-09-23T20:42:25.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:25 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2971458655' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:42:26.165 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 77 16 102964 102948 83.5213 55 0.0205295 0.0119679 2023-09-23T20:42:26.198 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:42:26.205 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:26.205 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:42:26.306 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:42:26.362 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T20:42:26.374 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:26.432 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:42:26.440 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:26.441 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:42:26.542 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:42:26.579 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:26.646 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:42:26.649 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:26.650 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:42:26.751 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:42:26.822 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:26.849 INFO:teuthology.orchestra.run.smithi018.stdout:moved 0 / 357 (0%) 2023-09-23T20:42:26.850 INFO:teuthology.orchestra.run.smithi018.stdout:avg 44.625 2023-09-23T20:42:26.850 INFO:teuthology.orchestra.run.smithi018.stdout:stddev 7.06996 -> 7.06996 (expected baseline 6.24875) 2023-09-23T20:42:26.850 INFO:teuthology.orchestra.run.smithi018.stdout:min osd.2 with 30 -> 30 pgs (0.672269 -> 0.672269 * mean) 2023-09-23T20:42:26.850 INFO:teuthology.orchestra.run.smithi018.stdout:max osd.1 with 52 -> 52 pgs (1.16527 -> 1.16527 * mean) 2023-09-23T20:42:26.850 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:42:26.850 INFO:teuthology.orchestra.run.smithi018.stdout:oload 1000 2023-09-23T20:42:26.850 INFO:teuthology.orchestra.run.smithi018.stdout:max_change 3 2023-09-23T20:42:26.850 INFO:teuthology.orchestra.run.smithi018.stdout:max_change_osds 4 2023-09-23T20:42:26.850 INFO:teuthology.orchestra.run.smithi018.stdout:average_utilization 0.0321 2023-09-23T20:42:26.850 INFO:teuthology.orchestra.run.smithi018.stdout:overload_utilization 0.3213 2023-09-23T20:42:26.858 INFO:teuthology.orchestra.run.smithi018.stderr:no change 2023-09-23T20:42:27.168 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 78 16 103818 103802 83.1348 53.375 0.0086867 0.0120245 2023-09-23T20:42:27.380 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:42:27.394 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:27.394 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:42:27.422 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '110', 'type': 'test-reweight-by-utilization'} 2023-09-23T20:42:27.422 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd test-reweight-by-utilization 110 1.0 2023-09-23T20:42:27.495 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:42:27.574 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:27.638 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:42:27.666 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:42:27.672 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:27.673 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:42:27.773 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:42:27.842 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:42:27.858 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:27 smithi018 ceph-mon[179647]: pgmap v1174: 119 pgs: 119 active+clean; 6.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 58 MiB/s wr, 927 op/s 2023-09-23T20:42:27.858 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:27 smithi018 ceph-mon[179647]: from='client.56221 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:42:27.858 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:27 smithi018 ceph-mon[184144]: pgmap v1174: 119 pgs: 119 active+clean; 6.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 58 MiB/s wr, 927 op/s 2023-09-23T20:42:27.858 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:27 smithi018 ceph-mon[184144]: from='client.56221 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:42:27.871 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:27.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:27 smithi149 ceph-mon[170394]: pgmap v1174: 119 pgs: 119 active+clean; 6.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 58 MiB/s wr, 927 op/s 2023-09-23T20:42:27.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:27 smithi149 ceph-mon[170394]: from='client.56221 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:42:28.016 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:42:28.026 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:28.026 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:42:28.127 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:42:28.165 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 79 16 104652 104636 82.7424 52.125 0.0171906 0.0120799 2023-09-23T20:42:28.232 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:28.338 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:42:28.346 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:28.346 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:42:28.447 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:42:28.550 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:28.581 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:42:28.861 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:42:28.871 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:28.872 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:42:28.973 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:42:29.041 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:42:29.056 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:29.165 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:2023-09-23T20:42:29.165993+0000 min lat: 0.00110414 max lat: 1.04101 avg lat: 0.0121427 2023-09-23T20:42:29.166 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T20:42:29.166 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 80 16 105443 105427 82.3262 49.4375 0.0191483 0.0121427 2023-09-23T20:42:29.209 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:42:29.214 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:29.214 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:42:29.315 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:42:29.362 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:29.430 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:42:29.434 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:29.435 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:42:29.535 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:42:29.610 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:29.628 INFO:teuthology.orchestra.run.smithi018.stderr:noscrub is unset 2023-09-23T20:42:29.666 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:29 smithi018 ceph-mon[184144]: pgmap v1175: 119 pgs: 119 active+clean; 6.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 57 MiB/s wr, 916 op/s 2023-09-23T20:42:29.666 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:29 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3664727302' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:42:29.666 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:29 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:42:29.667 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:29 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:42:29.667 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:29 smithi018 ceph-mon[179647]: pgmap v1175: 119 pgs: 119 active+clean; 6.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 57 MiB/s wr, 916 op/s 2023-09-23T20:42:29.667 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:29 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3664727302' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:42:29.667 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:29 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:42:29.667 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:29 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:42:29.846 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:42:29.860 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:29.860 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:42:29.961 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:42:29.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:29 smithi149 ceph-mon[170394]: pgmap v1175: 119 pgs: 119 active+clean; 6.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 57 MiB/s wr, 916 op/s 2023-09-23T20:42:29.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:29 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3664727302' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:42:29.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:29 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:42:29.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:29 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:42:30.016 INFO:teuthology.orchestra.run.smithi018.stdout:ERROR: (22) Invalid argument 2023-09-23T20:42:30.016 INFO:teuthology.orchestra.run.smithi018.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:42:30.046 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:30.167 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 81 16 106325 106309 81.9887 55.125 0.0160457 0.0121918 2023-09-23T20:42:30.446 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:42:30.452 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:30.453 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:42:30.554 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:42:30.660 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:30.807 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:42:30.814 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:30 smithi018 ceph-mon[179647]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:42:30.814 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:30 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:42:30.814 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:30 smithi018 ceph-mon[179647]: osdmap e469: 8 total, 8 up, 8 in 2023-09-23T20:42:30.814 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:30 smithi018 ceph-mon[179647]: failed to encode map e469 with expected crc 2023-09-23T20:42:30.815 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:30 smithi018 ceph-mon[184144]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:42:30.815 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:30 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:42:30.815 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:30 smithi018 ceph-mon[184144]: osdmap e469: 8 total, 8 up, 8 in 2023-09-23T20:42:30.815 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:30 smithi018 ceph-mon[184144]: failed to encode map e469 with expected crc 2023-09-23T20:42:30.816 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:30.816 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:42:30.864 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T20:42:30.910 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:30 smithi149 ceph-mon[170394]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:42:30.910 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:30 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:42:30.910 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:30 smithi149 ceph-mon[170394]: osdmap e469: 8 total, 8 up, 8 in 2023-09-23T20:42:30.910 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:30 smithi149 ceph-mon[170394]: failed to encode map e469 with expected crc 2023-09-23T20:42:30.917 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:42:30.998 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:31.068 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:42:31.076 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:31.077 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:42:31.167 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 82 16 107219 107203 81.6703 55.875 0.0121472 0.0122394 2023-09-23T20:42:31.178 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:42:31.280 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:31.443 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:42:31.456 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:31.457 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:42:31.558 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:42:31.627 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:31.687 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:42:31.766 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:42:31.777 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:31 smithi018 ceph-mon[184144]: pgmap v1177: 119 pgs: 119 active+clean; 6.4 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 53 MiB/s wr, 852 op/s 2023-09-23T20:42:31.783 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:31.783 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:31 smithi018 ceph-mon[179647]: pgmap v1177: 119 pgs: 119 active+clean; 6.4 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 53 MiB/s wr, 852 op/s 2023-09-23T20:42:31.784 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:42:31.884 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:42:31.930 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:42:31.931 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:31 smithi149 ceph-mon[170394]: pgmap v1177: 119 pgs: 119 active+clean; 6.4 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 53 MiB/s wr, 852 op/s 2023-09-23T20:42:31.944 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:31.958 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:42:31.971 INFO:teuthology.orchestra.run.smithi018.stdout:moved 58 / 357 (16.2465%) 2023-09-23T20:42:31.971 INFO:teuthology.orchestra.run.smithi018.stdout:avg 44.625 2023-09-23T20:42:31.971 INFO:teuthology.orchestra.run.smithi018.stdout:stddev 7.06996 -> 5.19465 (expected baseline 6.24875) 2023-09-23T20:42:31.972 INFO:teuthology.orchestra.run.smithi018.stdout:min osd.2 with 30 -> 44 pgs (0.672269 -> 0.985994 * mean) 2023-09-23T20:42:31.972 INFO:teuthology.orchestra.run.smithi018.stdout:max osd.1 with 52 -> 44 pgs (1.16527 -> 0.985994 * mean) 2023-09-23T20:42:31.972 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:42:31.972 INFO:teuthology.orchestra.run.smithi018.stdout:oload 110 2023-09-23T20:42:31.972 INFO:teuthology.orchestra.run.smithi018.stdout:max_change 1 2023-09-23T20:42:31.972 INFO:teuthology.orchestra.run.smithi018.stdout:max_change_osds 4 2023-09-23T20:42:31.972 INFO:teuthology.orchestra.run.smithi018.stdout:average_utilization 0.0329 2023-09-23T20:42:31.972 INFO:teuthology.orchestra.run.smithi018.stdout:overload_utilization 0.0362 2023-09-23T20:42:31.972 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1 weight 1.0000 -> 0.7130 2023-09-23T20:42:31.972 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5 weight 1.0000 -> 0.7155 2023-09-23T20:42:31.972 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3 weight 1.0000 -> 0.7168 2023-09-23T20:42:31.972 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7 weight 1.0000 -> 0.7442 2023-09-23T20:42:31.976 INFO:teuthology.orchestra.run.smithi018.stderr:no change 2023-09-23T20:42:31.995 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:42:32.012 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:42:32.021 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:32.021 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:42:32.025 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:42:32.074 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:42:32.111 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:42:32.121 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:42:32.142 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:42:31 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: 2023-09-23T20:42:31.969+0000 7f953be6a700 -1 mgr.server reply reply (4) Interrupted system call no change 2023-09-23T20:42:32.154 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:42:32.167 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 83 16 108042 108026 81.3057 51.4375 0.0217281 0.0122941 2023-09-23T20:42:32.183 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:32.184 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:42:32.198 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:42:32.198 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:42:32.199 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:42:32.199 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:42:32.199 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:42:32.199 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:42:32.199 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:42:32.199 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:42:32.546 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:42:32.557 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:32.558 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:42:32.658 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:42:32.748 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:32.821 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:32 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1762577303' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:42:32.822 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:32 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1762577303' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:42:32.846 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:42:32.851 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:32.851 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:42:32.912 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '1000', 'type': 'reweight-by-utilization'} 2023-09-23T20:42:32.913 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd reweight-by-utilization 1000 1.0 2023-09-23T20:42:32.952 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:42:32.970 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:32 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1762577303' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:42:33.050 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:33.148 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:42:33.156 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:33.156 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:42:33.168 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 84 16 108926 108910 80.9955 55.25 0.0220813 0.0123409 2023-09-23T20:42:33.209 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T20:42:33.257 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:42:33.312 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:33.377 INFO:teuthology.orchestra.run.smithi149.stdout:{ 2023-09-23T20:42:33.377 INFO:teuthology.orchestra.run.smithi149.stdout: "ops": [], 2023-09-23T20:42:33.377 INFO:teuthology.orchestra.run.smithi149.stdout: "num_ops": 0 2023-09-23T20:42:33.377 INFO:teuthology.orchestra.run.smithi149.stdout:} 2023-09-23T20:42:33.403 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:42:33.407 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:33.407 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:42:33.508 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:42:33.578 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:33.775 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:42:33.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:33 smithi149 ceph-mon[170394]: from='client.46340 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:42:33.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:33 smithi149 ceph-mon[170394]: pgmap v1178: 119 pgs: 119 active+clean; 6.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 54 MiB/s wr, 869 op/s 2023-09-23T20:42:34.034 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:33 smithi018 ceph-mon[179647]: from='client.46340 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:42:34.034 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:33 smithi018 ceph-mon[179647]: pgmap v1178: 119 pgs: 119 active+clean; 6.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 54 MiB/s wr, 869 op/s 2023-09-23T20:42:34.034 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:33 smithi018 ceph-mon[184144]: from='client.46340 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:42:34.035 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:33 smithi018 ceph-mon[184144]: pgmap v1178: 119 pgs: 119 active+clean; 6.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 54 MiB/s wr, 869 op/s 2023-09-23T20:42:34.061 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:42:34.072 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:34.073 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:42:34.097 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T20:42:34.171 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 85 16 109783 109767 80.6698 53.5625 0.0196021 0.0123911 2023-09-23T20:42:34.173 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:42:34.257 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:42:34.266 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:34.345 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:42:34.362 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:34.363 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:42:34.464 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:42:34.527 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:34.664 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:42:34.667 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:34.668 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:42:34.768 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:42:34.836 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:34.891 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:42:34.911 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:34.911 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:42:34.967 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:42:34.994 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:42:35.012 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:42:35.153 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:35.171 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 86 16 110608 110592 80.3315 51.5625 0.0144465 0.0124433 2023-09-23T20:42:35.385 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:42:35.388 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:35.388 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:42:35.389 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:42:34 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:42:34] "GET /metrics HTTP/1.1" 200 36516 "" "Prometheus/2.43.0" 2023-09-23T20:42:35.490 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:42:35.537 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:35.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:35 smithi149 ceph-mon[170394]: pgmap v1179: 119 pgs: 119 active+clean; 6.6 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 54 MiB/s wr, 857 op/s 2023-09-23T20:42:35.765 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:42:35.774 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:35.775 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:42:35.861 INFO:teuthology.orchestra.run.smithi018.stdout:moved 0 / 357 (0%) 2023-09-23T20:42:35.861 INFO:teuthology.orchestra.run.smithi018.stdout:avg 44.625 2023-09-23T20:42:35.861 INFO:teuthology.orchestra.run.smithi018.stdout:stddev 7.06996 -> 7.06996 (expected baseline 6.24875) 2023-09-23T20:42:35.862 INFO:teuthology.orchestra.run.smithi018.stdout:min osd.2 with 30 -> 30 pgs (0.672269 -> 0.672269 * mean) 2023-09-23T20:42:35.862 INFO:teuthology.orchestra.run.smithi018.stdout:max osd.1 with 52 -> 52 pgs (1.16527 -> 1.16527 * mean) 2023-09-23T20:42:35.862 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:42:35.862 INFO:teuthology.orchestra.run.smithi018.stdout:oload 1000 2023-09-23T20:42:35.862 INFO:teuthology.orchestra.run.smithi018.stdout:max_change 1 2023-09-23T20:42:35.862 INFO:teuthology.orchestra.run.smithi018.stdout:max_change_osds 4 2023-09-23T20:42:35.862 INFO:teuthology.orchestra.run.smithi018.stdout:average_utilization 0.0338 2023-09-23T20:42:35.862 INFO:teuthology.orchestra.run.smithi018.stdout:overload_utilization 0.3381 2023-09-23T20:42:35.867 INFO:teuthology.orchestra.run.smithi018.stderr:no change 2023-09-23T20:42:35.876 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:42:35.898 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:42:35.947 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:36.097 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:35 smithi018 ceph-mon[184144]: pgmap v1179: 119 pgs: 119 active+clean; 6.6 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 54 MiB/s wr, 857 op/s 2023-09-23T20:42:36.101 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:35 smithi018 ceph-mon[179647]: pgmap v1179: 119 pgs: 119 active+clean; 6.6 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 54 MiB/s wr, 857 op/s 2023-09-23T20:42:36.132 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:42:36.142 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:36.142 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:42:36.171 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 87 16 111407 111391 79.9822 49.9375 0.0139963 0.0124981 2023-09-23T20:42:36.243 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:42:36.291 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:36.362 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:42:36.368 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:36.368 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:42:36.469 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:42:36.547 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:36.605 INFO:teuthology.orchestra.run.smithi149.stdout:{ 2023-09-23T20:42:36.605 INFO:teuthology.orchestra.run.smithi149.stdout: "ops": [], 2023-09-23T20:42:36.605 INFO:teuthology.orchestra.run.smithi149.stdout: "complaint_time": 30, 2023-09-23T20:42:36.605 INFO:teuthology.orchestra.run.smithi149.stdout: "num_blocked_ops": 0 2023-09-23T20:42:36.606 INFO:teuthology.orchestra.run.smithi149.stdout:} 2023-09-23T20:42:36.732 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:42:36.740 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:36.740 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:42:36.774 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '110', 'type': 'reweight-by-utilization'} 2023-09-23T20:42:36.774 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd reweight-by-utilization 110 0.05 2023-09-23T20:42:36.841 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:42:36.947 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:37.056 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T20:42:37.171 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 88 16 112200 112184 79.6366 49.5625 0.0152396 0.0125521 2023-09-23T20:42:37.186 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:42:37.193 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:37.193 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:42:37.294 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:42:37.364 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:37.443 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:42:37.460 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:37.460 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:42:37.561 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:42:37.631 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:37.735 INFO:teuthology.orchestra.run.smithi018.stderr:nodeep-scrub is unset 2023-09-23T20:42:37.754 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:42:37.764 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:37.764 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:42:37.848 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:42:37.865 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:42:37.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:37 smithi018 ceph-mon[184144]: from='client.46346 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:42:37.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:37 smithi018 ceph-mon[184144]: pgmap v1180: 119 pgs: 119 active+clean; 6.7 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 53 MiB/s wr, 852 op/s 2023-09-23T20:42:37.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:37 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/850065810' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:42:37.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:37 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:42:37.896 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:37 smithi018 ceph-mon[179647]: from='client.46346 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:42:37.896 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:37 smithi018 ceph-mon[179647]: pgmap v1180: 119 pgs: 119 active+clean; 6.7 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 53 MiB/s wr, 852 op/s 2023-09-23T20:42:37.896 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:37 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/850065810' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:42:37.896 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:37 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:42:37.897 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:37.904 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:42:37.978 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:42:37.995 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:37.996 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:42:37.996 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:37 smithi149 ceph-mon[170394]: from='client.46346 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:42:37.996 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:37 smithi149 ceph-mon[170394]: pgmap v1180: 119 pgs: 119 active+clean; 6.7 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 53 MiB/s wr, 852 op/s 2023-09-23T20:42:37.996 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:37 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/850065810' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:42:37.996 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:37 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:42:38.013 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:42:38.052 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:42:38.083 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:42:38.097 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:42:38.122 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:42:38.150 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:38.161 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:42:38.171 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 89 16 113010 112994 79.3107 50.625 0.0205139 0.0126042 2023-09-23T20:42:38.196 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:42:38.218 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:42:38.224 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:38.224 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:42:38.229 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:42:38.267 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:42:38.273 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:42:38.273 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:42:38.273 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:42:38.274 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:42:38.274 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:42:38.274 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:42:38.274 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:42:38.274 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:42:38.325 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:42:38.418 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:38.840 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:42:38.849 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:38.849 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:42:38.872 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:38 smithi149 ceph-mon[170394]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:42:38.872 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:38 smithi149 ceph-mon[170394]: Cluster is now healthy 2023-09-23T20:42:38.873 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:38 smithi149 ceph-mon[170394]: failed to encode map e470 with expected crc 2023-09-23T20:42:38.873 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:38 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:42:38.873 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:38 smithi149 ceph-mon[170394]: osdmap e470: 8 total, 8 up, 8 in 2023-09-23T20:42:38.873 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:38 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/520805291' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:42:38.950 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:42:39.035 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:39.062 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:38 smithi018 ceph-mon[184144]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:42:39.062 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:38 smithi018 ceph-mon[184144]: Cluster is now healthy 2023-09-23T20:42:39.062 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:38 smithi018 ceph-mon[184144]: failed to encode map e470 with expected crc 2023-09-23T20:42:39.062 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:38 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:42:39.062 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:38 smithi018 ceph-mon[184144]: osdmap e470: 8 total, 8 up, 8 in 2023-09-23T20:42:39.062 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:38 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/520805291' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:42:39.063 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:38 smithi018 ceph-mon[179647]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:42:39.063 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:38 smithi018 ceph-mon[179647]: Cluster is now healthy 2023-09-23T20:42:39.063 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:38 smithi018 ceph-mon[179647]: failed to encode map e470 with expected crc 2023-09-23T20:42:39.064 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:38 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:42:39.064 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:38 smithi018 ceph-mon[179647]: osdmap e470: 8 total, 8 up, 8 in 2023-09-23T20:42:39.064 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:38 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/520805291' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:42:39.172 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Total time run: 90.0077 2023-09-23T20:42:39.172 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Total writes made: 113831 2023-09-23T20:42:39.172 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Write size: 65536 2023-09-23T20:42:39.172 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Object size: 65536 2023-09-23T20:42:39.172 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Bandwidth (MB/sec): 79.0425 2023-09-23T20:42:39.172 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Stddev Bandwidth: 35.1147 2023-09-23T20:42:39.172 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Max bandwidth (MB/sec): 197.312 2023-09-23T20:42:39.172 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Min bandwidth (MB/sec): 27.375 2023-09-23T20:42:39.172 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Average IOPS: 1264 2023-09-23T20:42:39.173 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Stddev IOPS: 561.835 2023-09-23T20:42:39.173 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Max IOPS: 3157 2023-09-23T20:42:39.173 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Min IOPS: 438 2023-09-23T20:42:39.173 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Average Latency(s): 0.0126475 2023-09-23T20:42:39.173 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Stddev Latency(s): 0.0138865 2023-09-23T20:42:39.173 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Max latency(s): 1.04101 2023-09-23T20:42:39.173 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Min latency(s): 0.00110414 2023-09-23T20:42:39.178 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Cleaning up (deleting benchmark objects) 2023-09-23T20:42:39.361 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:42:39.372 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:39.373 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:42:39.473 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:42:39.547 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:39.623 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:42:39.632 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:39.633 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:42:39.734 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:42:39.810 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:39.883 INFO:teuthology.orchestra.run.smithi149.stdout:ERROR: (22) Invalid argument 2023-09-23T20:42:39.884 INFO:teuthology.orchestra.run.smithi149.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:42:39.887 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:42:39.891 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:39.891 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:42:39.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:39 smithi018 ceph-mon[184144]: pgmap v1182: 119 pgs: 119 active+clean; 6.8 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 52 MiB/s wr, 834 op/s 2023-09-23T20:42:39.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:39 smithi018 ceph-mon[179647]: pgmap v1182: 119 pgs: 119 active+clean; 6.8 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 52 MiB/s wr, 834 op/s 2023-09-23T20:42:39.992 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:42:40.039 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:40.039 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:39 smithi149 ceph-mon[170394]: pgmap v1182: 119 pgs: 119 active+clean; 6.8 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 52 MiB/s wr, 834 op/s 2023-09-23T20:42:40.098 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:42:40.106 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:40.106 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:42:40.207 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:42:40.256 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T20:42:40.270 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:40.294 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T20:42:40.326 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:42:40.336 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:40.336 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:42:40.437 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:42:40.540 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:40.628 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:42:40.634 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:40.634 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:42:40.735 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:42:40.773 INFO:teuthology.orchestra.run.smithi018.stdout:moved 11 / 357 (3.08123%) 2023-09-23T20:42:40.773 INFO:teuthology.orchestra.run.smithi018.stdout:avg 44.625 2023-09-23T20:42:40.773 INFO:teuthology.orchestra.run.smithi018.stdout:stddev 7.06996 -> 6.90901 (expected baseline 6.24875) 2023-09-23T20:42:40.773 INFO:teuthology.orchestra.run.smithi018.stdout:min osd.2 with 30 -> 31 pgs (0.672269 -> 0.694678 * mean) 2023-09-23T20:42:40.774 INFO:teuthology.orchestra.run.smithi018.stdout:max osd.1 with 52 -> 50 pgs (1.16527 -> 1.12045 * mean) 2023-09-23T20:42:40.774 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:42:40.774 INFO:teuthology.orchestra.run.smithi018.stdout:oload 110 2023-09-23T20:42:40.774 INFO:teuthology.orchestra.run.smithi018.stdout:max_change 0.05 2023-09-23T20:42:40.774 INFO:teuthology.orchestra.run.smithi018.stdout:max_change_osds 4 2023-09-23T20:42:40.774 INFO:teuthology.orchestra.run.smithi018.stdout:average_utilization 0.0347 2023-09-23T20:42:40.774 INFO:teuthology.orchestra.run.smithi018.stdout:overload_utilization 0.0381 2023-09-23T20:42:40.774 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5 weight 1.0000 -> 0.9500 2023-09-23T20:42:40.774 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1 weight 1.0000 -> 0.9500 2023-09-23T20:42:40.774 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7 weight 1.0000 -> 0.9500 2023-09-23T20:42:40.775 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3 weight 1.0000 -> 0.9500 2023-09-23T20:42:40.806 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:40.809 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:42:40.859 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:42:40.863 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:40.864 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:42:40.964 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:42:40.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:40 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'1':'62260','3':'62260','5':'62260','7':'62260'}"}]: dispatch 2023-09-23T20:42:40.993 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:41.026 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:42:41.029 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:40 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'1':'62260','3':'62260','5':'62260','7':'62260'}"}]: dispatch 2023-09-23T20:42:41.029 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:40 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'1':'62260','3':'62260','5':'62260','7':'62260'}"}]: dispatch 2023-09-23T20:42:41.030 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:41.031 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:42:41.072 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:42:41.132 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:42:41.159 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:41.198 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:42:41.201 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:41.201 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:42:41.219 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:42:41.302 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:42:41.330 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:41.366 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:42:41.368 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:41.369 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:42:41.470 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:42:41.498 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:41.790 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:42:41.792 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:41.793 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:42:41.861 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:41 smithi149 ceph-mon[170394]: from='client.56251 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:42:41.861 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:41 smithi149 ceph-mon[170394]: pgmap v1183: 119 pgs: 119 active+clean; 6.9 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 53 MiB/s wr, 844 op/s 2023-09-23T20:42:41.861 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:41 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'1':'62260','3':'62260','5':'62260','7':'62260'}"}]': finished 2023-09-23T20:42:41.862 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:41 smithi149 ceph-mon[170394]: osdmap e471: 8 total, 8 up, 8 in 2023-09-23T20:42:41.862 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:41 smithi149 ceph-mon[170394]: failed to encode map e471 with expected crc 2023-09-23T20:42:41.862 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:41 smithi149 ceph-mon[170394]: failed to encode map e471 with expected crc 2023-09-23T20:42:41.862 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:41 smithi149 ceph-mon[170394]: failed to encode map e471 with expected crc 2023-09-23T20:42:41.862 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:41 smithi149 ceph-mon[170394]: failed to encode map e471 with expected crc 2023-09-23T20:42:41.862 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:41 smithi149 ceph-mon[170394]: failed to encode map e471 with expected crc 2023-09-23T20:42:41.862 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:41 smithi149 ceph-mon[170394]: failed to encode map e471 with expected crc 2023-09-23T20:42:41.862 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:41 smithi149 ceph-mon[170394]: failed to encode map e472 with expected crc 2023-09-23T20:42:41.862 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:41 smithi149 ceph-mon[170394]: osdmap e472: 8 total, 8 up, 8 in 2023-09-23T20:42:41.863 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:41 smithi149 ceph-mon[170394]: failed to encode map e472 with expected crc 2023-09-23T20:42:41.863 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:41 smithi149 ceph-mon[170394]: failed to encode map e472 with expected crc 2023-09-23T20:42:41.863 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:41 smithi149 ceph-mon[170394]: failed to encode map e472 with expected crc 2023-09-23T20:42:41.863 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:41 smithi149 ceph-mon[170394]: failed to encode map e472 with expected crc 2023-09-23T20:42:41.895 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:42:41.913 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:42:41.924 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:41.974 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:42:41.977 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:41.978 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:42:42.079 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:42:42.107 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:42.113 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:41 smithi018 ceph-mon[184144]: from='client.56251 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:42:42.113 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:41 smithi018 ceph-mon[184144]: pgmap v1183: 119 pgs: 119 active+clean; 6.9 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 53 MiB/s wr, 844 op/s 2023-09-23T20:42:42.113 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:41 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'1':'62260','3':'62260','5':'62260','7':'62260'}"}]': finished 2023-09-23T20:42:42.114 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:41 smithi018 ceph-mon[184144]: osdmap e471: 8 total, 8 up, 8 in 2023-09-23T20:42:42.114 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:41 smithi018 ceph-mon[184144]: failed to encode map e471 with expected crc 2023-09-23T20:42:42.114 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:41 smithi018 ceph-mon[184144]: failed to encode map e471 with expected crc 2023-09-23T20:42:42.114 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:41 smithi018 ceph-mon[184144]: failed to encode map e471 with expected crc 2023-09-23T20:42:42.114 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:41 smithi018 ceph-mon[184144]: failed to encode map e471 with expected crc 2023-09-23T20:42:42.114 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:41 smithi018 ceph-mon[184144]: failed to encode map e471 with expected crc 2023-09-23T20:42:42.114 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:41 smithi018 ceph-mon[184144]: failed to encode map e471 with expected crc 2023-09-23T20:42:42.114 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:41 smithi018 ceph-mon[184144]: failed to encode map e472 with expected crc 2023-09-23T20:42:42.114 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:41 smithi018 ceph-mon[184144]: osdmap e472: 8 total, 8 up, 8 in 2023-09-23T20:42:42.114 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:41 smithi018 ceph-mon[184144]: failed to encode map e472 with expected crc 2023-09-23T20:42:42.114 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:41 smithi018 ceph-mon[184144]: failed to encode map e472 with expected crc 2023-09-23T20:42:42.115 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:41 smithi018 ceph-mon[184144]: failed to encode map e472 with expected crc 2023-09-23T20:42:42.115 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:41 smithi018 ceph-mon[184144]: failed to encode map e472 with expected crc 2023-09-23T20:42:42.115 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:41 smithi018 ceph-mon[179647]: from='client.56251 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T20:42:42.115 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:41 smithi018 ceph-mon[179647]: pgmap v1183: 119 pgs: 119 active+clean; 6.9 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 53 MiB/s wr, 844 op/s 2023-09-23T20:42:42.115 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:41 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'1':'62260','3':'62260','5':'62260','7':'62260'}"}]': finished 2023-09-23T20:42:42.115 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:41 smithi018 ceph-mon[179647]: osdmap e471: 8 total, 8 up, 8 in 2023-09-23T20:42:42.115 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:41 smithi018 ceph-mon[179647]: failed to encode map e471 with expected crc 2023-09-23T20:42:42.115 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:41 smithi018 ceph-mon[179647]: failed to encode map e471 with expected crc 2023-09-23T20:42:42.115 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:41 smithi018 ceph-mon[179647]: failed to encode map e471 with expected crc 2023-09-23T20:42:42.115 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:41 smithi018 ceph-mon[179647]: failed to encode map e471 with expected crc 2023-09-23T20:42:42.115 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:41 smithi018 ceph-mon[179647]: failed to encode map e471 with expected crc 2023-09-23T20:42:42.116 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:41 smithi018 ceph-mon[179647]: failed to encode map e471 with expected crc 2023-09-23T20:42:42.116 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:41 smithi018 ceph-mon[179647]: failed to encode map e472 with expected crc 2023-09-23T20:42:42.116 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:41 smithi018 ceph-mon[179647]: osdmap e472: 8 total, 8 up, 8 in 2023-09-23T20:42:42.116 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:41 smithi018 ceph-mon[179647]: failed to encode map e472 with expected crc 2023-09-23T20:42:42.116 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:41 smithi018 ceph-mon[179647]: failed to encode map e472 with expected crc 2023-09-23T20:42:42.116 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:41 smithi018 ceph-mon[179647]: failed to encode map e472 with expected crc 2023-09-23T20:42:42.116 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:41 smithi018 ceph-mon[179647]: failed to encode map e472 with expected crc 2023-09-23T20:42:42.193 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:42:42.196 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:42.197 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:42:42.286 INFO:teuthology.orchestra.run.smithi149.stdout:ERROR: (22) Invalid argument 2023-09-23T20:42:42.287 INFO:teuthology.orchestra.run.smithi149.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:42:42.298 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:42:42.328 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:42.564 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:42:42.570 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:42.571 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:42:42.671 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:42:42.738 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:42.740 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T20:42:42.969 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:42 smithi149 ceph-mon[170394]: failed to encode map e472 with expected crc 2023-09-23T20:42:42.969 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:42 smithi149 ceph-mon[170394]: failed to encode map e472 with expected crc 2023-09-23T20:42:42.969 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:42 smithi149 ceph-mon[170394]: failed to encode map e473 with expected crc 2023-09-23T20:42:42.969 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:42 smithi149 ceph-mon[170394]: failed to encode map e473 with expected crc 2023-09-23T20:42:42.969 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:42 smithi149 ceph-mon[170394]: osdmap e473: 8 total, 8 up, 8 in 2023-09-23T20:42:42.969 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:42 smithi149 ceph-mon[170394]: failed to encode map e473 with expected crc 2023-09-23T20:42:42.969 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:42 smithi149 ceph-mon[170394]: failed to encode map e473 with expected crc 2023-09-23T20:42:42.969 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:42 smithi149 ceph-mon[170394]: 9.d starting backfill to osd.4 from (0'0,0'0] MAX to 470'7234 2023-09-23T20:42:43.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:42 smithi018 ceph-mon[179647]: failed to encode map e472 with expected crc 2023-09-23T20:42:43.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:42 smithi018 ceph-mon[179647]: failed to encode map e472 with expected crc 2023-09-23T20:42:43.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:42 smithi018 ceph-mon[179647]: failed to encode map e473 with expected crc 2023-09-23T20:42:43.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:42 smithi018 ceph-mon[179647]: failed to encode map e473 with expected crc 2023-09-23T20:42:43.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:42 smithi018 ceph-mon[179647]: osdmap e473: 8 total, 8 up, 8 in 2023-09-23T20:42:43.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:42 smithi018 ceph-mon[179647]: failed to encode map e473 with expected crc 2023-09-23T20:42:43.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:42 smithi018 ceph-mon[179647]: failed to encode map e473 with expected crc 2023-09-23T20:42:43.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:42 smithi018 ceph-mon[179647]: 9.d starting backfill to osd.4 from (0'0,0'0] MAX to 470'7234 2023-09-23T20:42:43.144 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:42 smithi018 ceph-mon[184144]: failed to encode map e472 with expected crc 2023-09-23T20:42:43.144 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:42 smithi018 ceph-mon[184144]: failed to encode map e472 with expected crc 2023-09-23T20:42:43.144 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:42 smithi018 ceph-mon[184144]: failed to encode map e473 with expected crc 2023-09-23T20:42:43.144 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:42 smithi018 ceph-mon[184144]: failed to encode map e473 with expected crc 2023-09-23T20:42:43.144 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:42 smithi018 ceph-mon[184144]: osdmap e473: 8 total, 8 up, 8 in 2023-09-23T20:42:43.145 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:42 smithi018 ceph-mon[184144]: failed to encode map e473 with expected crc 2023-09-23T20:42:43.145 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:42 smithi018 ceph-mon[184144]: failed to encode map e473 with expected crc 2023-09-23T20:42:43.145 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:42 smithi018 ceph-mon[184144]: 9.d starting backfill to osd.4 from (0'0,0'0] MAX to 470'7234 2023-09-23T20:42:43.490 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:42:44.008 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:42:44.017 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:44.018 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:42:44.118 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:42:44.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:43 smithi018 ceph-mon[184144]: pgmap v1186: 119 pgs: 119 active+clean; 6.9 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 39 MiB/s wr, 692 op/s 2023-09-23T20:42:44.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:43 smithi018 ceph-mon[184144]: failed to encode map e473 with expected crc 2023-09-23T20:42:44.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:43 smithi018 ceph-mon[184144]: 9.8 starting backfill to osd.1 from (0'0,0'0] MAX to 470'7349 2023-09-23T20:42:44.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:43 smithi018 ceph-mon[184144]: failed to encode map e473 with expected crc 2023-09-23T20:42:44.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:43 smithi018 ceph-mon[184144]: 9.5 starting backfill to osd.1 from (0'0,0'0] MAX to 470'7073 2023-09-23T20:42:44.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:43 smithi018 ceph-mon[184144]: 9.8 starting backfill to osd.7 from (0'0,0'0] MAX to 470'7349 2023-09-23T20:42:44.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:43 smithi018 ceph-mon[184144]: osdmap e474: 8 total, 8 up, 8 in 2023-09-23T20:42:44.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:43 smithi018 ceph-mon[184144]: failed to encode map e474 with expected crc 2023-09-23T20:42:44.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:43 smithi018 ceph-mon[184144]: failed to encode map e474 with expected crc 2023-09-23T20:42:44.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:43 smithi018 ceph-mon[184144]: failed to encode map e474 with expected crc 2023-09-23T20:42:44.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:43 smithi018 ceph-mon[179647]: pgmap v1186: 119 pgs: 119 active+clean; 6.9 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 39 MiB/s wr, 692 op/s 2023-09-23T20:42:44.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:43 smithi018 ceph-mon[179647]: failed to encode map e473 with expected crc 2023-09-23T20:42:44.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:43 smithi018 ceph-mon[179647]: 9.8 starting backfill to osd.1 from (0'0,0'0] MAX to 470'7349 2023-09-23T20:42:44.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:43 smithi018 ceph-mon[179647]: failed to encode map e473 with expected crc 2023-09-23T20:42:44.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:43 smithi018 ceph-mon[179647]: 9.5 starting backfill to osd.1 from (0'0,0'0] MAX to 470'7073 2023-09-23T20:42:44.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:43 smithi018 ceph-mon[179647]: 9.8 starting backfill to osd.7 from (0'0,0'0] MAX to 470'7349 2023-09-23T20:42:44.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:43 smithi018 ceph-mon[179647]: osdmap e474: 8 total, 8 up, 8 in 2023-09-23T20:42:44.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:43 smithi018 ceph-mon[179647]: failed to encode map e474 with expected crc 2023-09-23T20:42:44.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:43 smithi018 ceph-mon[179647]: failed to encode map e474 with expected crc 2023-09-23T20:42:44.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:43 smithi018 ceph-mon[179647]: failed to encode map e474 with expected crc 2023-09-23T20:42:44.156 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:43 smithi149 ceph-mon[170394]: pgmap v1186: 119 pgs: 119 active+clean; 6.9 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 39 MiB/s wr, 692 op/s 2023-09-23T20:42:44.157 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:43 smithi149 ceph-mon[170394]: failed to encode map e473 with expected crc 2023-09-23T20:42:44.157 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:43 smithi149 ceph-mon[170394]: 9.8 starting backfill to osd.1 from (0'0,0'0] MAX to 470'7349 2023-09-23T20:42:44.157 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:43 smithi149 ceph-mon[170394]: failed to encode map e473 with expected crc 2023-09-23T20:42:44.157 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:43 smithi149 ceph-mon[170394]: 9.5 starting backfill to osd.1 from (0'0,0'0] MAX to 470'7073 2023-09-23T20:42:44.157 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:43 smithi149 ceph-mon[170394]: 9.8 starting backfill to osd.7 from (0'0,0'0] MAX to 470'7349 2023-09-23T20:42:44.157 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:43 smithi149 ceph-mon[170394]: osdmap e474: 8 total, 8 up, 8 in 2023-09-23T20:42:44.157 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:43 smithi149 ceph-mon[170394]: failed to encode map e474 with expected crc 2023-09-23T20:42:44.157 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:43 smithi149 ceph-mon[170394]: failed to encode map e474 with expected crc 2023-09-23T20:42:44.157 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:43 smithi149 ceph-mon[170394]: failed to encode map e474 with expected crc 2023-09-23T20:42:44.179 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:44.645 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:42:44.659 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:44.659 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:42:44.760 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:42:44.877 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:44.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:44 smithi018 ceph-mon[179647]: failed to encode map e474 with expected crc 2023-09-23T20:42:44.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:44 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:42:44.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:44 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:42:44.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:44 smithi018 ceph-mon[179647]: osdmap e475: 8 total, 8 up, 8 in 2023-09-23T20:42:44.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:44 smithi018 ceph-mon[179647]: failed to encode map e475 with expected crc 2023-09-23T20:42:44.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:44 smithi018 ceph-mon[179647]: failed to encode map e475 with expected crc 2023-09-23T20:42:44.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:44 smithi018 ceph-mon[179647]: failed to encode map e475 with expected crc 2023-09-23T20:42:44.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:44 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:42:44.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:44 smithi018 ceph-mon[184144]: failed to encode map e474 with expected crc 2023-09-23T20:42:44.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:44 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:42:44.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:44 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:42:44.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:44 smithi018 ceph-mon[184144]: osdmap e475: 8 total, 8 up, 8 in 2023-09-23T20:42:44.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:44 smithi018 ceph-mon[184144]: failed to encode map e475 with expected crc 2023-09-23T20:42:44.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:44 smithi018 ceph-mon[184144]: failed to encode map e475 with expected crc 2023-09-23T20:42:44.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:44 smithi018 ceph-mon[184144]: failed to encode map e475 with expected crc 2023-09-23T20:42:44.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:44 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:42:45.218 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:44 smithi149 ceph-mon[170394]: failed to encode map e474 with expected crc 2023-09-23T20:42:45.218 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:44 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:42:45.218 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:44 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:42:45.218 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:44 smithi149 ceph-mon[170394]: osdmap e475: 8 total, 8 up, 8 in 2023-09-23T20:42:45.218 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:44 smithi149 ceph-mon[170394]: failed to encode map e475 with expected crc 2023-09-23T20:42:45.218 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:44 smithi149 ceph-mon[170394]: failed to encode map e475 with expected crc 2023-09-23T20:42:45.218 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:44 smithi149 ceph-mon[170394]: failed to encode map e475 with expected crc 2023-09-23T20:42:45.218 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:44 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:42:45.387 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:42:45.392 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:45.392 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:42:45.400 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:42:44 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:42:44] "GET /metrics HTTP/1.1" 200 36602 "" "Prometheus/2.43.0" 2023-09-23T20:42:45.493 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:42:45.555 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:45.642 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:42:45.648 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:45.648 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:42:45.716 INFO:teuthology.orchestra.run.smithi149.stdout:ERROR: (22) Invalid argument 2023-09-23T20:42:45.717 INFO:teuthology.orchestra.run.smithi149.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:42:45.749 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:42:45.817 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:45.824 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:45 smithi018 ceph-mon[179647]: pgmap v1189: 119 pgs: 2 active+undersized+remapped, 1 remapped+peering, 116 active+clean; 6.9 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 10 KiB/s rd, 21 MiB/s wr, 695 op/s; 1.0 MiB/s, 2 objects/s recovering 2023-09-23T20:42:45.824 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:45 smithi018 ceph-mon[179647]: failed to encode map e475 with expected crc 2023-09-23T20:42:45.824 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:45 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2165934538' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:42:45.824 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:45 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1311248871' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:42:45.847 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:42:45.858 INFO:teuthology.orchestra.run.smithi018.stderr:noscrub is set 2023-09-23T20:42:45.883 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:42:45.892 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:42:45.897 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:45 smithi149 ceph-mon[170394]: pgmap v1189: 119 pgs: 2 active+undersized+remapped, 1 remapped+peering, 116 active+clean; 6.9 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 10 KiB/s rd, 21 MiB/s wr, 695 op/s; 1.0 MiB/s, 2 objects/s recovering 2023-09-23T20:42:45.897 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:45 smithi149 ceph-mon[170394]: failed to encode map e475 with expected crc 2023-09-23T20:42:45.898 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:45 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2165934538' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:42:45.898 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:45 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1311248871' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:42:45.898 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:45.899 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:42:45.932 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:42:45.982 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:42:45.999 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:42:46.027 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:42:46.062 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:46.065 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:42:46.105 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:42:46.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:45 smithi018 ceph-mon[184144]: pgmap v1189: 119 pgs: 2 active+undersized+remapped, 1 remapped+peering, 116 active+clean; 6.9 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 10 KiB/s rd, 21 MiB/s wr, 695 op/s; 1.0 MiB/s, 2 objects/s recovering 2023-09-23T20:42:46.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:45 smithi018 ceph-mon[184144]: failed to encode map e475 with expected crc 2023-09-23T20:42:46.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:45 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2165934538' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:42:46.144 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:45 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1311248871' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:42:46.146 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:42:46.162 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:42:46.162 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:42:46.162 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:42:46.162 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:42:46.162 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:42:46.162 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:42:46.162 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:42:46.163 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:42:46.187 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T20:42:46.200 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:42:46.205 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:46.206 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:42:46.223 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 7, 3, 2, 4, 6, 5] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T20:42:46.223 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:42:46.223 INFO:tasks.thrashosds.thrasher:Growing pool .mgr 2023-09-23T20:42:46.223 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:42:46.307 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:42:46.397 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:46.513 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:42:46.526 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:46.526 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:42:46.626 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:42:46.698 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:46.879 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:42:46.882 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:46.883 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:42:46.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:46 smithi018 ceph-mon[179647]: 1.8 scrub starts 2023-09-23T20:42:46.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:46 smithi018 ceph-mon[179647]: 1.8 scrub ok 2023-09-23T20:42:46.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:46 smithi018 ceph-mon[179647]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:42:46.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:46 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2165934538' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:42:46.895 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:46 smithi018 ceph-mon[179647]: osdmap e476: 8 total, 8 up, 8 in 2023-09-23T20:42:46.895 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:46 smithi018 ceph-mon[179647]: failed to encode map e476 with expected crc 2023-09-23T20:42:46.895 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:46 smithi018 ceph-mon[184144]: 1.8 scrub starts 2023-09-23T20:42:46.895 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:46 smithi018 ceph-mon[184144]: 1.8 scrub ok 2023-09-23T20:42:46.895 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:46 smithi018 ceph-mon[184144]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:42:46.895 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:46 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2165934538' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:42:46.895 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:46 smithi018 ceph-mon[184144]: osdmap e476: 8 total, 8 up, 8 in 2023-09-23T20:42:46.895 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:46 smithi018 ceph-mon[184144]: failed to encode map e476 with expected crc 2023-09-23T20:42:46.984 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:42:47.055 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:47.104 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:42:47.108 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:47.109 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:42:47.157 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:42:47.209 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:42:47.219 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:46 smithi149 ceph-mon[170394]: 1.8 scrub starts 2023-09-23T20:42:47.219 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:46 smithi149 ceph-mon[170394]: 1.8 scrub ok 2023-09-23T20:42:47.219 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:46 smithi149 ceph-mon[170394]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:42:47.219 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:46 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2165934538' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:42:47.219 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:46 smithi149 ceph-mon[170394]: osdmap e476: 8 total, 8 up, 8 in 2023-09-23T20:42:47.219 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:46 smithi149 ceph-mon[170394]: failed to encode map e476 with expected crc 2023-09-23T20:42:47.265 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:47.315 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:42:47.368 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:42:47.375 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:47.376 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:42:47.476 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:42:47.571 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:47.638 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:42:47.643 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:47.644 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:42:47.744 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:42:47.810 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:47.981 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:42:47.988 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:47.989 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:42:48.090 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:42:48.140 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:47 smithi018 ceph-mon[184144]: pgmap v1192: 119 pgs: 1 active+remapped+backfill_wait, 2 active+undersized+remapped, 1 remapped+peering, 115 active+clean; 6.8 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 10 KiB/s rd, 7.4 MiB/s wr, 662 op/s; 6780/333798 objects misplaced (2.031%); 1.0 MiB/s, 2 objects/s recovering 2023-09-23T20:42:48.144 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:47 smithi018 ceph-mon[179647]: pgmap v1192: 119 pgs: 1 active+remapped+backfill_wait, 2 active+undersized+remapped, 1 remapped+peering, 115 active+clean; 6.8 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 10 KiB/s rd, 7.4 MiB/s wr, 662 op/s; 6780/333798 objects misplaced (2.031%); 1.0 MiB/s, 2 objects/s recovering 2023-09-23T20:42:48.186 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:48.202 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:47 smithi149 ceph-mon[170394]: pgmap v1192: 119 pgs: 1 active+remapped+backfill_wait, 2 active+undersized+remapped, 1 remapped+peering, 115 active+clean; 6.8 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 10 KiB/s rd, 7.4 MiB/s wr, 662 op/s; 6780/333798 objects misplaced (2.031%); 1.0 MiB/s, 2 objects/s recovering 2023-09-23T20:42:48.256 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T20:42:48.265 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:48.266 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:42:48.266 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:42:48.367 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:42:48.467 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:48.562 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:42:48.569 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:48.569 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:42:48.609 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:42:48.670 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:42:48.813 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:48.879 INFO:teuthology.orchestra.run.smithi149.stdout:{ 2023-09-23T20:42:48.879 INFO:teuthology.orchestra.run.smithi149.stdout: "size": 20, 2023-09-23T20:42:48.879 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 600, 2023-09-23T20:42:48.879 INFO:teuthology.orchestra.run.smithi149.stdout: "ops": [ 2023-09-23T20:42:48.879 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.880 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:51043 8.0 8:0f697c15:::benchmark_data_smithi018_341490_object51042:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e374)", 2023-09-23T20:42:48.880 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:36:42.561436+0000", 2023-09-23T20:42:48.880 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 366.31434743199998, 2023-09-23T20:42:48.880 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.14033216600000001, 2023-09-23T20:42:48.880 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:42:48.880 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:42:48.880 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:42:48.880 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:42:48.880 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:42:48.880 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 51043 2023-09-23T20:42:48.880 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.881 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:42:48.881 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.881 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:42:48.881 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.561436+0000", 2023-09-23T20:42:48.881 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:42:48.881 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.881 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.881 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:42:48.881 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.561436+0000", 2023-09-23T20:42:48.881 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:42:48.881 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.882 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.882 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:42:48.882 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.561435+0000", 2023-09-23T20:42:48.882 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:42:48.882 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.882 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.882 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:42:48.882 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.561495+0000", 2023-09-23T20:42:48.882 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.0056999999999998e-05 2023-09-23T20:42:48.882 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.882 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.883 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:42:48.883 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.561507+0000", 2023-09-23T20:42:48.883 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1627999999999999e-05 2023-09-23T20:42:48.883 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.883 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.883 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:42:48.883 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.561513+0000", 2023-09-23T20:42:48.883 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.5969999999999997e-06 2023-09-23T20:42:48.883 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.883 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.883 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:42:48.884 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.561542+0000", 2023-09-23T20:42:48.884 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.8665000000000001e-05 2023-09-23T20:42:48.884 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.884 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.884 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:42:48.884 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.561627+0000", 2023-09-23T20:42:48.884 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.4596000000000001e-05 2023-09-23T20:42:48.884 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.885 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.885 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:42:48.885 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.561720+0000", 2023-09-23T20:42:48.885 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.3153999999999996e-05 2023-09-23T20:42:48.885 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.885 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.885 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:42:48.885 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.563303+0000", 2023-09-23T20:42:48.886 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0015831689999999999 2023-09-23T20:42:48.886 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.886 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.886 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:42:48.886 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.563545+0000", 2023-09-23T20:42:48.886 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00024228800000000001 2023-09-23T20:42:48.886 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.887 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.887 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:42:48.887 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.701697+0000", 2023-09-23T20:42:48.887 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.138151417 2023-09-23T20:42:48.887 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.887 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.887 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:42:48.887 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.701739+0000", 2023-09-23T20:42:48.888 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.2009999999999999e-05 2023-09-23T20:42:48.888 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.888 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.888 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:42:48.888 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.701768+0000", 2023-09-23T20:42:48.888 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.9332e-05 2023-09-23T20:42:48.888 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:42:48.889 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:42:48.889 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:42:48.889 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.889 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.889 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_repop(client.35994.0:167663 8.d e444/440 8:b711b4f4:::benchmark_data_smithi018_341490_object78340:head v 444'10486, mlcod=378'5269)", 2023-09-23T20:42:48.889 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:40:24.984398+0000", 2023-09-23T20:42:48.889 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 143.89138564800001, 2023-09-23T20:42:48.889 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.10260506799999999, 2023-09-23T20:42:48.890 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:42:48.890 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:42:48.890 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:42:48.890 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.890 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:42:48.890 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:40:24.984398+0000", 2023-09-23T20:42:48.890 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:42:48.890 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.891 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.891 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:42:48.891 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:40:24.984398+0000", 2023-09-23T20:42:48.891 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:42:48.891 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.891 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.891 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:42:48.891 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:40:24.984397+0000", 2023-09-23T20:42:48.891 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:42:48.891 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.891 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.892 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:42:48.892 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:40:24.984403+0000", 2023-09-23T20:42:48.892 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.2439999999999998e-06 2023-09-23T20:42:48.892 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.892 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.892 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:42:48.892 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:40:24.984410+0000", 2023-09-23T20:42:48.892 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.8650000000000003e-06 2023-09-23T20:42:48.892 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.892 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.892 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:42:48.892 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:40:24.984415+0000", 2023-09-23T20:42:48.893 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.1790000000000001e-06 2023-09-23T20:42:48.893 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.893 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.893 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:42:48.893 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:40:25.051221+0000", 2023-09-23T20:42:48.893 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.066806563999999999 2023-09-23T20:42:48.893 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.893 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.893 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:42:48.893 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:40:25.051240+0000", 2023-09-23T20:42:48.893 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.931e-05 2023-09-23T20:42:48.893 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.894 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.894 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:42:48.894 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:40:25.086883+0000", 2023-09-23T20:42:48.894 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.035642979999999998 2023-09-23T20:42:48.894 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.894 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.894 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:42:48.894 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:40:25.087003+0000", 2023-09-23T20:42:48.894 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000119336 2023-09-23T20:42:48.894 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:42:48.894 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:42:48.895 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:42:48.895 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.895 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.895 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:29549 9.7 9:ea1cf7ea:::benchmark_data_smithi018_381516_object29548:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e459)", 2023-09-23T20:42:48.895 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:41:21.052150+0000", 2023-09-23T20:42:48.895 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 87.823632837999995, 2023-09-23T20:42:48.895 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.14682632800000001, 2023-09-23T20:42:48.895 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:42:48.895 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:42:48.895 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:42:48.895 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:42:48.895 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:42:48.895 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 29549 2023-09-23T20:42:48.896 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.896 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:42:48.896 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.896 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:42:48.896 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.052150+0000", 2023-09-23T20:42:48.896 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:42:48.896 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.896 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.896 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:42:48.896 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.052150+0000", 2023-09-23T20:42:48.896 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:42:48.896 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.897 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.897 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:42:48.897 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.052149+0000", 2023-09-23T20:42:48.897 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:42:48.897 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.897 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.897 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:42:48.897 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.052237+0000", 2023-09-23T20:42:48.897 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.7569000000000001e-05 2023-09-23T20:42:48.897 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.897 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.897 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:42:48.898 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.052246+0000", 2023-09-23T20:42:48.898 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.5289999999999997e-06 2023-09-23T20:42:48.898 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.898 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.898 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:42:48.898 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.052252+0000", 2023-09-23T20:42:48.898 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.5450000000000003e-06 2023-09-23T20:42:48.898 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.898 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.898 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:42:48.899 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.052294+0000", 2023-09-23T20:42:48.899 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.2228999999999998e-05 2023-09-23T20:42:48.899 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.899 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.899 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:42:48.899 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.052411+0000", 2023-09-23T20:42:48.899 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00011702899999999999 2023-09-23T20:42:48.899 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.899 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.899 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 3,7", 2023-09-23T20:42:48.899 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.052532+0000", 2023-09-23T20:42:48.899 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00012102799999999999 2023-09-23T20:42:48.900 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.900 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.900 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:42:48.900 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.055293+0000", 2023-09-23T20:42:48.900 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.002760424 2023-09-23T20:42:48.900 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.900 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.900 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:42:48.900 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.056788+0000", 2023-09-23T20:42:48.900 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00149522 2023-09-23T20:42:48.900 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.900 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.900 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:42:48.901 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.198887+0000", 2023-09-23T20:42:48.901 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.142099535 2023-09-23T20:42:48.901 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.901 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.901 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:42:48.901 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.198937+0000", 2023-09-23T20:42:48.901 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.9252999999999998e-05 2023-09-23T20:42:48.901 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.901 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.901 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:42:48.901 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.198977+0000", 2023-09-23T20:42:48.901 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.0151000000000002e-05 2023-09-23T20:42:48.902 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:42:48.902 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:42:48.902 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:42:48.902 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.902 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.902 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:29552 9.f 9:fef69f4f:::benchmark_data_smithi018_381516_object29551:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e459)", 2023-09-23T20:42:48.902 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:41:21.053398+0000", 2023-09-23T20:42:48.902 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 87.822384858000007, 2023-09-23T20:42:48.902 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.14557930599999999, 2023-09-23T20:42:48.902 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:42:48.902 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:42:48.903 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:42:48.903 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:42:48.903 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:42:48.903 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 29552 2023-09-23T20:42:48.903 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.903 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:42:48.903 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.903 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:42:48.903 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.053398+0000", 2023-09-23T20:42:48.903 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:42:48.903 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.903 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.904 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:42:48.904 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.053398+0000", 2023-09-23T20:42:48.904 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:42:48.904 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.904 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.904 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:42:48.904 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.053398+0000", 2023-09-23T20:42:48.904 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:42:48.904 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.904 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.904 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:42:48.904 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.053439+0000", 2023-09-23T20:42:48.905 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.1603000000000001e-05 2023-09-23T20:42:48.905 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.905 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.905 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:42:48.905 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.053453+0000", 2023-09-23T20:42:48.905 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.3842000000000001e-05 2023-09-23T20:42:48.905 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.905 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.905 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:42:48.905 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.053462+0000", 2023-09-23T20:42:48.905 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.9190000000000005e-06 2023-09-23T20:42:48.905 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.905 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.906 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:42:48.906 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.053503+0000", 2023-09-23T20:42:48.906 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.1069000000000003e-05 2023-09-23T20:42:48.906 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.906 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.906 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:42:48.906 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.053629+0000", 2023-09-23T20:42:48.906 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00012554299999999999 2023-09-23T20:42:48.906 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.906 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.906 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:42:48.906 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.053773+0000", 2023-09-23T20:42:48.907 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000144427 2023-09-23T20:42:48.907 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.907 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.907 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:42:48.907 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.056678+0000", 2023-09-23T20:42:48.907 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0029051200000000002 2023-09-23T20:42:48.907 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.907 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.907 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:42:48.907 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.060074+0000", 2023-09-23T20:42:48.907 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0033957610000000002 2023-09-23T20:42:48.907 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.908 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.908 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:42:48.908 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.198920+0000", 2023-09-23T20:42:48.908 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.13884589899999999 2023-09-23T20:42:48.908 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.908 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.908 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:42:48.908 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.198949+0000", 2023-09-23T20:42:48.908 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.9641e-05 2023-09-23T20:42:48.908 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.908 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.908 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:42:48.909 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.198978+0000", 2023-09-23T20:42:48.909 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.8285000000000001e-05 2023-09-23T20:42:48.909 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:42:48.909 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:42:48.909 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:42:48.909 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.909 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.909 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:29557 9.7 9:e1a9e246:::benchmark_data_smithi018_381516_object29556:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e459)", 2023-09-23T20:42:48.909 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:41:21.057117+0000", 2023-09-23T20:42:48.909 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 87.818666385, 2023-09-23T20:42:48.909 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.14586998200000001, 2023-09-23T20:42:48.909 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:42:48.910 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:42:48.910 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:42:48.910 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:42:48.910 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:42:48.910 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 29557 2023-09-23T20:42:48.910 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.910 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:42:48.910 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.910 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:42:48.910 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.057117+0000", 2023-09-23T20:42:48.910 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:42:48.911 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.911 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.911 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:42:48.911 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.057117+0000", 2023-09-23T20:42:48.911 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:42:48.911 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.911 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.911 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:42:48.911 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.057115+0000", 2023-09-23T20:42:48.911 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:42:48.911 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.911 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.912 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:42:48.912 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.057187+0000", 2023-09-23T20:42:48.912 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.1619999999999995e-05 2023-09-23T20:42:48.912 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.912 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.912 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:42:48.912 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.057200+0000", 2023-09-23T20:42:48.912 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.3261000000000001e-05 2023-09-23T20:42:48.912 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.912 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.912 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:42:48.912 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.057207+0000", 2023-09-23T20:42:48.913 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.6309999999999997e-06 2023-09-23T20:42:48.913 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.913 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.913 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:42:48.913 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.057249+0000", 2023-09-23T20:42:48.913 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.1993999999999998e-05 2023-09-23T20:42:48.913 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.913 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.913 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:42:48.913 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.057363+0000", 2023-09-23T20:42:48.913 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000113553 2023-09-23T20:42:48.913 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.914 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.914 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 3,7", 2023-09-23T20:42:48.914 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.057458+0000", 2023-09-23T20:42:48.914 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.5340000000000005e-05 2023-09-23T20:42:48.914 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.914 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.914 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:42:48.914 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.058584+0000", 2023-09-23T20:42:48.914 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0011261489999999999 2023-09-23T20:42:48.914 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.914 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.915 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:42:48.915 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.061821+0000", 2023-09-23T20:42:48.915 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0032369619999999999 2023-09-23T20:42:48.915 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.915 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.915 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:42:48.915 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.202885+0000", 2023-09-23T20:42:48.915 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.14106375800000001 2023-09-23T20:42:48.915 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.915 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.915 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:42:48.916 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.202939+0000", 2023-09-23T20:42:48.916 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.4626000000000002e-05 2023-09-23T20:42:48.916 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.916 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.916 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:42:48.916 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.202987+0000", 2023-09-23T20:42:48.916 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.7528000000000003e-05 2023-09-23T20:42:48.916 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:42:48.916 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:42:48.916 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:42:48.916 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.916 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.917 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:29569 9.7 9:edf50775:::benchmark_data_smithi018_381516_object29568:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e459)", 2023-09-23T20:42:48.917 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:41:21.063339+0000", 2023-09-23T20:42:48.917 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 87.812444056999993, 2023-09-23T20:42:48.917 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.13984607700000001, 2023-09-23T20:42:48.917 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:42:48.917 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:42:48.917 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:42:48.917 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:42:48.917 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:42:48.917 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 29569 2023-09-23T20:42:48.917 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.918 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:42:48.918 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.918 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:42:48.918 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.063339+0000", 2023-09-23T20:42:48.918 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:42:48.918 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.918 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.918 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:42:48.918 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.063339+0000", 2023-09-23T20:42:48.918 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:42:48.918 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.918 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.919 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:42:48.919 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.063338+0000", 2023-09-23T20:42:48.919 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:42:48.919 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.919 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.919 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:42:48.919 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.063399+0000", 2023-09-23T20:42:48.919 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.0869000000000003e-05 2023-09-23T20:42:48.919 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.919 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.919 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:42:48.920 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.063417+0000", 2023-09-23T20:42:48.920 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.7456999999999999e-05 2023-09-23T20:42:48.920 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.920 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.920 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:42:48.920 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.063423+0000", 2023-09-23T20:42:48.920 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.4139999999999999e-06 2023-09-23T20:42:48.920 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.920 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.920 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:42:48.920 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.063463+0000", 2023-09-23T20:42:48.920 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.9626999999999999e-05 2023-09-23T20:42:48.921 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.921 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.921 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:42:48.921 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.063572+0000", 2023-09-23T20:42:48.921 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000109697 2023-09-23T20:42:48.921 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.921 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.921 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 3,7", 2023-09-23T20:42:48.921 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.063672+0000", 2023-09-23T20:42:48.921 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.9401000000000006e-05 2023-09-23T20:42:48.921 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.921 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.922 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:42:48.922 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.065628+0000", 2023-09-23T20:42:48.922 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.001955816 2023-09-23T20:42:48.922 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.922 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.922 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:42:48.922 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.065657+0000", 2023-09-23T20:42:48.922 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.9050000000000001e-05 2023-09-23T20:42:48.922 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.922 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.922 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:42:48.923 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.203114+0000", 2023-09-23T20:42:48.923 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.13745722199999999 2023-09-23T20:42:48.923 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.923 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.923 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:42:48.923 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.203160+0000", 2023-09-23T20:42:48.923 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.6292999999999999e-05 2023-09-23T20:42:48.923 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.923 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.923 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:42:48.923 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.203185+0000", 2023-09-23T20:42:48.923 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.5196999999999999e-05 2023-09-23T20:42:48.924 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:42:48.924 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:42:48.924 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:42:48.924 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.924 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.924 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:29570 9.7 9:ed5a7646:::benchmark_data_smithi018_381516_object29569:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e459)", 2023-09-23T20:42:48.924 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:41:21.063661+0000", 2023-09-23T20:42:48.924 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 87.812122789, 2023-09-23T20:42:48.924 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.14287293200000001, 2023-09-23T20:42:48.924 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:42:48.924 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:42:48.925 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:42:48.925 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:42:48.925 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:42:48.925 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 29570 2023-09-23T20:42:48.925 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.925 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:42:48.925 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.925 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:42:48.925 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.063661+0000", 2023-09-23T20:42:48.925 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:42:48.925 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.925 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.926 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:42:48.926 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.063661+0000", 2023-09-23T20:42:48.926 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:42:48.926 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.926 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.926 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:42:48.926 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.063660+0000", 2023-09-23T20:42:48.926 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:42:48.926 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.926 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.926 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:42:48.926 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.063718+0000", 2023-09-23T20:42:48.927 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.8522e-05 2023-09-23T20:42:48.927 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.927 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.927 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:42:48.927 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.063730+0000", 2023-09-23T20:42:48.927 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1586999999999999e-05 2023-09-23T20:42:48.927 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.927 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.927 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:42:48.927 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.063734+0000", 2023-09-23T20:42:48.927 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.3889999999999998e-06 2023-09-23T20:42:48.927 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.928 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.928 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:42:48.928 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.064722+0000", 2023-09-23T20:42:48.928 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00098758900000000009 2023-09-23T20:42:48.928 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.928 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.928 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:42:48.928 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.064854+0000", 2023-09-23T20:42:48.928 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00013245499999999999 2023-09-23T20:42:48.928 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.928 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.928 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 3,7", 2023-09-23T20:42:48.929 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.064964+0000", 2023-09-23T20:42:48.929 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000109548 2023-09-23T20:42:48.929 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.929 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.929 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:42:48.929 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.065828+0000", 2023-09-23T20:42:48.929 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00086438499999999996 2023-09-23T20:42:48.929 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.929 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.929 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:42:48.929 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.066131+0000", 2023-09-23T20:42:48.930 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00030242099999999998 2023-09-23T20:42:48.930 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.930 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.930 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:42:48.930 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.206430+0000", 2023-09-23T20:42:48.930 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.14029873500000001 2023-09-23T20:42:48.930 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.930 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.930 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:42:48.930 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.206487+0000", 2023-09-23T20:42:48.930 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.7812000000000001e-05 2023-09-23T20:42:48.930 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.931 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.931 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:42:48.931 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.206533+0000", 2023-09-23T20:42:48.931 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.6122999999999999e-05 2023-09-23T20:42:48.931 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:42:48.931 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:42:48.931 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:42:48.931 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.931 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.931 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:76985 9.5 9:ababdd75:::benchmark_data_smithi018_381516_object76984:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e465)", 2023-09-23T20:42:48.931 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:41:58.522099+0000", 2023-09-23T20:42:48.931 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 50.353684381000001, 2023-09-23T20:42:48.932 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.96136469999999996, 2023-09-23T20:42:48.932 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:42:48.932 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:42:48.932 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:42:48.932 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:42:48.932 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:42:48.932 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 76985 2023-09-23T20:42:48.932 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.932 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:42:48.932 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.932 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:42:48.933 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.522099+0000", 2023-09-23T20:42:48.933 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:42:48.933 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.933 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.933 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:42:48.933 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.522099+0000", 2023-09-23T20:42:48.933 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:42:48.933 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.933 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.933 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:42:48.933 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.522098+0000", 2023-09-23T20:42:48.933 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:42:48.934 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.934 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.934 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:42:48.934 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.522133+0000", 2023-09-23T20:42:48.934 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.4041000000000002e-05 2023-09-23T20:42:48.934 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.934 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.934 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:42:48.934 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.522141+0000", 2023-09-23T20:42:48.934 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.7900000000000005e-06 2023-09-23T20:42:48.934 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.934 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.935 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:42:48.935 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.522146+0000", 2023-09-23T20:42:48.935 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.3340000000000003e-06 2023-09-23T20:42:48.935 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.935 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.935 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:42:48.935 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.522199+0000", 2023-09-23T20:42:48.935 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.3347000000000003e-05 2023-09-23T20:42:48.935 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.935 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.935 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:42:48.935 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.522206+0000", 2023-09-23T20:42:48.936 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.6209999999999998e-06 2023-09-23T20:42:48.936 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.936 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.936 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:42:48.936 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.471807+0000", 2023-09-23T20:42:48.936 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.94960129400000004 2023-09-23T20:42:48.936 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.936 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.936 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:42:48.936 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.474386+0000", 2023-09-23T20:42:48.936 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0025794749999999999 2023-09-23T20:42:48.937 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.937 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.937 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,7", 2023-09-23T20:42:48.937 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.474496+0000", 2023-09-23T20:42:48.937 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000109733 2023-09-23T20:42:48.937 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.937 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.937 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:42:48.937 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.483272+0000", 2023-09-23T20:42:48.937 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0087758269999999999 2023-09-23T20:42:48.937 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.937 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.938 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:42:48.938 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.483367+0000", 2023-09-23T20:42:48.938 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.5458999999999995e-05 2023-09-23T20:42:48.938 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.938 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.938 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:42:48.938 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.483375+0000", 2023-09-23T20:42:48.938 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.7910000000000005e-06 2023-09-23T20:42:48.938 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.938 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.938 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:42:48.939 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.483424+0000", 2023-09-23T20:42:48.939 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.8497e-05 2023-09-23T20:42:48.939 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.939 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.939 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:42:48.939 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.483464+0000", 2023-09-23T20:42:48.939 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.9960999999999999e-05 2023-09-23T20:42:48.939 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:42:48.939 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:42:48.939 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:42:48.939 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.939 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.940 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:76994 9.5 9:a38d7184:::benchmark_data_smithi018_381516_object76993:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e465)", 2023-09-23T20:42:48.940 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:41:58.527883+0000", 2023-09-23T20:42:48.940 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 50.347900439, 2023-09-23T20:42:48.940 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.95854079599999997, 2023-09-23T20:42:48.940 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:42:48.940 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:42:48.940 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:42:48.940 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:42:48.940 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:42:48.940 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 76994 2023-09-23T20:42:48.940 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.940 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:42:48.941 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.941 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:42:48.941 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.527883+0000", 2023-09-23T20:42:48.941 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:42:48.941 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.941 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.941 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:42:48.941 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.527883+0000", 2023-09-23T20:42:48.941 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:42:48.941 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.941 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.942 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:42:48.942 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.527882+0000", 2023-09-23T20:42:48.942 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:42:48.942 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.942 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.942 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:42:48.942 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.527986+0000", 2023-09-23T20:42:48.942 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000104201 2023-09-23T20:42:48.942 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.942 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.942 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:42:48.943 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.528001+0000", 2023-09-23T20:42:48.943 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.5187999999999999e-05 2023-09-23T20:42:48.943 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.943 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.943 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:42:48.943 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.528009+0000", 2023-09-23T20:42:48.943 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.4259999999999997e-06 2023-09-23T20:42:48.943 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.943 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.943 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:42:48.943 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.528056+0000", 2023-09-23T20:42:48.944 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.727e-05 2023-09-23T20:42:48.944 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.944 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.944 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:42:48.944 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.528062+0000", 2023-09-23T20:42:48.944 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.1929999999999998e-06 2023-09-23T20:42:48.944 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.944 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.944 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:42:48.944 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.477679+0000", 2023-09-23T20:42:48.944 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.94961691500000001 2023-09-23T20:42:48.944 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.945 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.945 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:42:48.945 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.477800+0000", 2023-09-23T20:42:48.945 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00012139800000000001 2023-09-23T20:42:48.945 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.945 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.945 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,7", 2023-09-23T20:42:48.945 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.477938+0000", 2023-09-23T20:42:48.945 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00013737499999999999 2023-09-23T20:42:48.945 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.945 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.946 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:42:48.946 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.483301+0000", 2023-09-23T20:42:48.946 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0053636430000000004 2023-09-23T20:42:48.946 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.946 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.946 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:42:48.946 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.483541+0000", 2023-09-23T20:42:48.946 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00023936699999999999 2023-09-23T20:42:48.946 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.946 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.946 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:42:48.946 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.486265+0000", 2023-09-23T20:42:48.947 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.002724363 2023-09-23T20:42:48.947 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.947 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.947 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:42:48.947 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.486368+0000", 2023-09-23T20:42:48.947 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000103058 2023-09-23T20:42:48.947 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.947 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.947 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:42:48.947 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.486424+0000", 2023-09-23T20:42:48.947 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.5560000000000003e-05 2023-09-23T20:42:48.947 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:42:48.948 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:42:48.948 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:42:48.948 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.948 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.948 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:77025 9.5 9:a8adfb4b:::benchmark_data_smithi018_381516_object77024:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e465)", 2023-09-23T20:42:48.948 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:41:58.574546+0000", 2023-09-23T20:42:48.948 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 50.301236821000003, 2023-09-23T20:42:48.948 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.91796858599999998, 2023-09-23T20:42:48.948 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:42:48.948 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:42:48.948 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:42:48.949 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:42:48.949 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:42:48.949 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 77025 2023-09-23T20:42:48.949 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.949 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:42:48.949 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.949 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:42:48.949 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.574546+0000", 2023-09-23T20:42:48.949 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:42:48.949 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.949 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.949 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:42:48.950 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.574546+0000", 2023-09-23T20:42:48.950 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:42:48.950 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.950 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.950 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:42:48.950 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.574546+0000", 2023-09-23T20:42:48.950 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:42:48.950 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.950 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.950 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:42:48.950 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.574601+0000", 2023-09-23T20:42:48.951 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.4879000000000001e-05 2023-09-23T20:42:48.951 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.951 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.951 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:42:48.951 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.574612+0000", 2023-09-23T20:42:48.951 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1201e-05 2023-09-23T20:42:48.951 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.951 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.951 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:42:48.951 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.574616+0000", 2023-09-23T20:42:48.951 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.1370000000000004e-06 2023-09-23T20:42:48.952 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.952 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.952 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:42:48.952 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.574650+0000", 2023-09-23T20:42:48.952 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.4007000000000002e-05 2023-09-23T20:42:48.952 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.952 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.952 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:42:48.952 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.574655+0000", 2023-09-23T20:42:48.952 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.0340000000000004e-06 2023-09-23T20:42:48.952 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.952 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.953 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:42:48.953 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.480178+0000", 2023-09-23T20:42:48.953 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.90552307700000001 2023-09-23T20:42:48.953 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.953 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.953 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:42:48.953 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.480276+0000", 2023-09-23T20:42:48.953 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.8187e-05 2023-09-23T20:42:48.953 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.953 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.953 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,7", 2023-09-23T20:42:48.953 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.480425+0000", 2023-09-23T20:42:48.954 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000148228 2023-09-23T20:42:48.954 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.954 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.954 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:42:48.954 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.483500+0000", 2023-09-23T20:42:48.954 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0030750740000000001 2023-09-23T20:42:48.954 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.954 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.954 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:42:48.954 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.483545+0000", 2023-09-23T20:42:48.954 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.5864e-05 2023-09-23T20:42:48.955 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.955 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.955 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:42:48.955 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.492442+0000", 2023-09-23T20:42:48.955 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0088969189999999997 2023-09-23T20:42:48.955 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.955 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.955 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:42:48.955 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.492481+0000", 2023-09-23T20:42:48.955 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.8887999999999998e-05 2023-09-23T20:42:48.955 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.955 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.956 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:42:48.956 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.492515+0000", 2023-09-23T20:42:48.956 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.3810999999999999e-05 2023-09-23T20:42:48.956 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:42:48.956 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:42:48.956 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:42:48.956 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.956 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.956 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:91635 9.f 9:f296a7d6:::benchmark_data_smithi018_381516_object91634:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e467)", 2023-09-23T20:42:48.956 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:42:13.701301+0000", 2023-09-23T20:42:48.956 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 35.174482081999997, 2023-09-23T20:42:48.957 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.128139787, 2023-09-23T20:42:48.957 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:42:48.957 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:42:48.957 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:42:48.957 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:42:48.957 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:42:48.957 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 91635 2023-09-23T20:42:48.957 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.957 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:42:48.957 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.957 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:42:48.957 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.701301+0000", 2023-09-23T20:42:48.958 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:42:48.958 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.958 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.958 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:42:48.958 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.701301+0000", 2023-09-23T20:42:48.958 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:42:48.958 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.958 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.958 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:42:48.958 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.701301+0000", 2023-09-23T20:42:48.958 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:42:48.959 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.959 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.959 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:42:48.959 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.701346+0000", 2023-09-23T20:42:48.959 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.5599999999999997e-05 2023-09-23T20:42:48.959 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.959 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.959 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:42:48.959 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.701359+0000", 2023-09-23T20:42:48.959 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.3251000000000001e-05 2023-09-23T20:42:48.959 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.960 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.960 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:42:48.960 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.701365+0000", 2023-09-23T20:42:48.960 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.8309999999999997e-06 2023-09-23T20:42:48.960 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.960 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.960 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:42:48.960 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.706931+0000", 2023-09-23T20:42:48.960 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0055660900000000001 2023-09-23T20:42:48.960 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.960 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.960 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:42:48.961 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.707048+0000", 2023-09-23T20:42:48.961 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000116663 2023-09-23T20:42:48.961 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.961 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.961 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:42:48.961 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.714996+0000", 2023-09-23T20:42:48.961 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0079476500000000005 2023-09-23T20:42:48.961 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.961 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.961 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:42:48.961 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.719222+0000", 2023-09-23T20:42:48.962 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.004226769 2023-09-23T20:42:48.962 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.962 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.962 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:42:48.962 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.726451+0000", 2023-09-23T20:42:48.962 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0072287560000000002 2023-09-23T20:42:48.962 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.962 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.962 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:42:48.962 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.829335+0000", 2023-09-23T20:42:48.962 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.102883501 2023-09-23T20:42:48.963 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.963 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.963 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:42:48.963 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.829386+0000", 2023-09-23T20:42:48.963 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.1496999999999998e-05 2023-09-23T20:42:48.963 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.963 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.963 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:42:48.963 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.829441+0000", 2023-09-23T20:42:48.963 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.4868999999999999e-05 2023-09-23T20:42:48.963 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:42:48.964 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:42:48.964 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:42:48.964 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.964 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.964 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:91642 9.7 9:e21f89c5:::benchmark_data_smithi018_381516_object91641:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e467)", 2023-09-23T20:42:48.964 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:42:13.710744+0000", 2023-09-23T20:42:48.964 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 35.165039581999999, 2023-09-23T20:42:48.964 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.13617485700000001, 2023-09-23T20:42:48.964 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:42:48.964 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:42:48.964 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:42:48.964 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:42:48.965 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:42:48.965 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 91642 2023-09-23T20:42:48.965 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.965 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:42:48.965 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.965 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:42:48.965 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.710744+0000", 2023-09-23T20:42:48.965 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:42:48.965 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.965 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.965 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:42:48.966 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.710744+0000", 2023-09-23T20:42:48.966 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:42:48.966 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.966 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.966 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:42:48.966 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.710742+0000", 2023-09-23T20:42:48.966 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:42:48.966 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.966 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.966 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:42:48.966 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.710818+0000", 2023-09-23T20:42:48.967 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.6044999999999994e-05 2023-09-23T20:42:48.967 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.967 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.967 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:42:48.967 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.710834+0000", 2023-09-23T20:42:48.967 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.5733e-05 2023-09-23T20:42:48.967 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.967 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.967 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:42:48.967 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.710841+0000", 2023-09-23T20:42:48.967 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.0559999999999999e-06 2023-09-23T20:42:48.967 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.968 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.968 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:42:48.968 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.719091+0000", 2023-09-23T20:42:48.968 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0082496040000000007 2023-09-23T20:42:48.968 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.968 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.968 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:42:48.968 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.719366+0000", 2023-09-23T20:42:48.968 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00027542799999999998 2023-09-23T20:42:48.968 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.968 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.968 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 3,7", 2023-09-23T20:42:48.969 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.719569+0000", 2023-09-23T20:42:48.969 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00020253299999999999 2023-09-23T20:42:48.969 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.969 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.969 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:42:48.969 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.729681+0000", 2023-09-23T20:42:48.969 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.010112571000000001 2023-09-23T20:42:48.969 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.969 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.969 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:42:48.969 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.740599+0000", 2023-09-23T20:42:48.970 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.010917425 2023-09-23T20:42:48.970 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.970 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.970 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:42:48.970 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.846806+0000", 2023-09-23T20:42:48.970 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.10620702 2023-09-23T20:42:48.970 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.970 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.970 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:42:48.970 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.846863+0000", 2023-09-23T20:42:48.970 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.7924999999999999e-05 2023-09-23T20:42:48.970 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.971 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.971 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:42:48.971 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.846919+0000", 2023-09-23T20:42:48.971 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.5124999999999998e-05 2023-09-23T20:42:48.971 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:42:48.971 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:42:48.971 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:42:48.971 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.971 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.971 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:91649 9.7 9:ed097aa6:::benchmark_data_smithi018_381516_object91648:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e467)", 2023-09-23T20:42:48.971 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:42:13.719094+0000", 2023-09-23T20:42:48.971 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 35.156688938000002, 2023-09-23T20:42:48.972 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.12801558599999999, 2023-09-23T20:42:48.972 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:42:48.972 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:42:48.972 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:42:48.972 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:42:48.972 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:42:48.972 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 91649 2023-09-23T20:42:48.972 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.972 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:42:48.972 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.972 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:42:48.972 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.719094+0000", 2023-09-23T20:42:48.973 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:42:48.973 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.973 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.973 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:42:48.973 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.719094+0000", 2023-09-23T20:42:48.973 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:42:48.973 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.973 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.973 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:42:48.974 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.719093+0000", 2023-09-23T20:42:48.974 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:42:48.974 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.974 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.974 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:42:48.974 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.719128+0000", 2023-09-23T20:42:48.974 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.5079000000000001e-05 2023-09-23T20:42:48.974 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.974 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.974 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:42:48.974 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.719139+0000", 2023-09-23T20:42:48.975 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0761000000000001e-05 2023-09-23T20:42:48.975 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.975 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.975 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:42:48.975 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.719144+0000", 2023-09-23T20:42:48.975 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.2920000000000003e-06 2023-09-23T20:42:48.975 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.975 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.975 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:42:48.975 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.722939+0000", 2023-09-23T20:42:48.975 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.003794136 2023-09-23T20:42:48.975 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.976 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.976 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:42:48.976 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.723030+0000", 2023-09-23T20:42:48.976 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.1193999999999995e-05 2023-09-23T20:42:48.976 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.976 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.976 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 3,7", 2023-09-23T20:42:48.976 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.723143+0000", 2023-09-23T20:42:48.976 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000113268 2023-09-23T20:42:48.976 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.976 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.977 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:42:48.977 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.729715+0000", 2023-09-23T20:42:48.977 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0065721130000000001 2023-09-23T20:42:48.977 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.977 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.977 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:42:48.977 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.740629+0000", 2023-09-23T20:42:48.977 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.010913997 2023-09-23T20:42:48.977 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.977 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.977 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:42:48.978 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.846966+0000", 2023-09-23T20:42:48.978 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.106336736 2023-09-23T20:42:48.978 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.978 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.978 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:42:48.978 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.847061+0000", 2023-09-23T20:42:48.978 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.4692999999999996e-05 2023-09-23T20:42:48.978 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.978 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.978 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:42:48.978 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.847110+0000", 2023-09-23T20:42:48.978 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.9382000000000003e-05 2023-09-23T20:42:48.979 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:42:48.979 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:42:48.979 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:42:48.979 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.979 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.979 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:91651 9.f 9:f2c174a9:::benchmark_data_smithi018_381516_object91650:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e467)", 2023-09-23T20:42:48.979 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:42:13.719151+0000", 2023-09-23T20:42:48.979 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 35.156632807999998, 2023-09-23T20:42:48.979 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.12779057599999999, 2023-09-23T20:42:48.979 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:42:48.979 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:42:48.979 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:42:48.980 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:42:48.980 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:42:48.980 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 91651 2023-09-23T20:42:48.980 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.980 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:42:48.980 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.980 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:42:48.980 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.719151+0000", 2023-09-23T20:42:48.980 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:42:48.980 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.980 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.981 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:42:48.981 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.719151+0000", 2023-09-23T20:42:48.981 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:42:48.981 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.981 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.981 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:42:48.981 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.719150+0000", 2023-09-23T20:42:48.981 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:42:48.981 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.981 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.981 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:42:48.981 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.719179+0000", 2023-09-23T20:42:48.981 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.9420999999999998e-05 2023-09-23T20:42:48.982 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.982 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.982 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:42:48.982 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.719187+0000", 2023-09-23T20:42:48.982 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.2320000000000004e-06 2023-09-23T20:42:48.982 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.982 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.982 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:42:48.982 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.719191+0000", 2023-09-23T20:42:48.982 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.3220000000000003e-06 2023-09-23T20:42:48.982 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.983 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.983 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:42:48.983 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.722930+0000", 2023-09-23T20:42:48.983 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0037388640000000002 2023-09-23T20:42:48.983 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.983 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.983 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:42:48.983 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.723047+0000", 2023-09-23T20:42:48.983 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00011688699999999999 2023-09-23T20:42:48.983 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.983 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.984 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:42:48.984 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.723182+0000", 2023-09-23T20:42:48.984 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00013561199999999999 2023-09-23T20:42:48.984 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.984 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.984 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:42:48.984 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.729712+0000", 2023-09-23T20:42:48.984 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0065292340000000001 2023-09-23T20:42:48.984 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.984 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.984 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:42:48.984 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.729718+0000", 2023-09-23T20:42:48.985 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.3829999999999997e-06 2023-09-23T20:42:48.985 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.985 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.985 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:42:48.985 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.846858+0000", 2023-09-23T20:42:48.985 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.117139934 2023-09-23T20:42:48.985 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.985 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.985 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:42:48.985 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.846905+0000", 2023-09-23T20:42:48.985 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.7187000000000001e-05 2023-09-23T20:42:48.986 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.986 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.986 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:42:48.986 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.846941+0000", 2023-09-23T20:42:48.986 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.5975e-05 2023-09-23T20:42:48.986 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:42:48.986 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:42:48.986 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:42:48.986 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.986 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.987 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:91653 9.7 9:ea65aee7:::benchmark_data_smithi018_381516_object91652:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e467)", 2023-09-23T20:42:48.987 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:42:13.723355+0000", 2023-09-23T20:42:48.987 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 35.152428346999997, 2023-09-23T20:42:48.987 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.123887294, 2023-09-23T20:42:48.987 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:42:48.987 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:42:48.987 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:42:48.987 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:42:48.987 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:42:48.987 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 91653 2023-09-23T20:42:48.987 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.988 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:42:48.988 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.988 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:42:48.988 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.723355+0000", 2023-09-23T20:42:48.988 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:42:48.988 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.988 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.988 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:42:48.988 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.723355+0000", 2023-09-23T20:42:48.988 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:42:48.988 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.988 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.989 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:42:48.989 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.723353+0000", 2023-09-23T20:42:48.989 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:42:48.989 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.989 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.989 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:42:48.989 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.723423+0000", 2023-09-23T20:42:48.989 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.9944000000000007e-05 2023-09-23T20:42:48.989 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.989 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.989 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:42:48.989 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.723437+0000", 2023-09-23T20:42:48.990 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.3225e-05 2023-09-23T20:42:48.990 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.990 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.990 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:42:48.990 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.723444+0000", 2023-09-23T20:42:48.990 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.2949999999999996e-06 2023-09-23T20:42:48.990 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.990 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.990 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:42:48.990 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.726600+0000", 2023-09-23T20:42:48.990 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0031559629999999999 2023-09-23T20:42:48.991 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.991 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.991 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:42:48.991 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.726742+0000", 2023-09-23T20:42:48.991 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00014221500000000001 2023-09-23T20:42:48.991 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.991 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.991 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 3,7", 2023-09-23T20:42:48.991 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.726930+0000", 2023-09-23T20:42:48.991 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00018778999999999999 2023-09-23T20:42:48.991 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.991 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.992 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:42:48.992 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.732785+0000", 2023-09-23T20:42:48.992 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0058547709999999999 2023-09-23T20:42:48.992 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.992 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.992 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:42:48.992 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.740849+0000", 2023-09-23T20:42:48.992 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0080647329999999993 2023-09-23T20:42:48.992 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.992 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.992 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:42:48.993 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.847159+0000", 2023-09-23T20:42:48.993 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.10631001299999999 2023-09-23T20:42:48.993 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.993 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.993 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:42:48.993 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.847205+0000", 2023-09-23T20:42:48.993 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.5303000000000003e-05 2023-09-23T20:42:48.993 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.993 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.993 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:42:48.993 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.847242+0000", 2023-09-23T20:42:48.993 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.7539000000000001e-05 2023-09-23T20:42:48.994 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:42:48.994 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:42:48.994 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:42:48.994 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.994 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.994 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:92127 9.7 9:eb5270d9:::benchmark_data_smithi018_381516_object92126:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e467)", 2023-09-23T20:42:48.994 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:42:14.303475+0000", 2023-09-23T20:42:48.994 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 34.572308307999997, 2023-09-23T20:42:48.994 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.13503338400000001, 2023-09-23T20:42:48.994 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:42:48.995 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:42:48.995 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:42:48.995 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:42:48.995 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:42:48.995 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 92127 2023-09-23T20:42:48.995 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.995 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:42:48.995 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.995 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:42:48.995 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.303475+0000", 2023-09-23T20:42:48.995 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:42:48.996 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.996 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.996 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:42:48.996 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.303475+0000", 2023-09-23T20:42:48.996 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:42:48.996 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.996 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.996 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:42:48.996 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.303475+0000", 2023-09-23T20:42:48.996 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:42:48.996 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.996 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.997 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:42:48.997 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.303510+0000", 2023-09-23T20:42:48.997 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.5318999999999998e-05 2023-09-23T20:42:48.997 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.997 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.997 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:42:48.997 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.303516+0000", 2023-09-23T20:42:48.997 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.4320000000000004e-06 2023-09-23T20:42:48.997 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.997 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.997 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:42:48.998 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.303520+0000", 2023-09-23T20:42:48.998 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.0280000000000002e-06 2023-09-23T20:42:48.998 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.998 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.998 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:42:48.998 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.303550+0000", 2023-09-23T20:42:48.998 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.9938e-05 2023-09-23T20:42:48.998 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.998 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.998 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:42:48.998 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.303677+0000", 2023-09-23T20:42:48.999 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00012659800000000001 2023-09-23T20:42:48.999 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.999 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.999 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 3,7", 2023-09-23T20:42:48.999 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.303805+0000", 2023-09-23T20:42:48.999 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00012804799999999999 2023-09-23T20:42:48.999 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:48.999 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:48.999 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:42:48.999 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.306643+0000", 2023-09-23T20:42:48.999 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.002837759 2023-09-23T20:42:48.999 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:49.000 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:49.000 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:42:49.000 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.317326+0000", 2023-09-23T20:42:49.000 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.010683002 2023-09-23T20:42:49.000 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:49.000 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:49.000 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:42:49.000 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.438439+0000", 2023-09-23T20:42:49.000 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.121113448 2023-09-23T20:42:49.000 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:49.000 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:49.001 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:42:49.001 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.438478+0000", 2023-09-23T20:42:49.001 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.8921999999999998e-05 2023-09-23T20:42:49.001 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:49.001 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:49.001 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:42:49.001 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.438508+0000", 2023-09-23T20:42:49.001 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.029e-05 2023-09-23T20:42:49.001 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:42:49.001 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:42:49.001 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:42:49.001 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:49.002 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:49.002 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:92129 9.f 9:fdf451a5:::benchmark_data_smithi018_381516_object92128:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e467)", 2023-09-23T20:42:49.002 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:42:14.307177+0000", 2023-09-23T20:42:49.002 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 34.568606535000001, 2023-09-23T20:42:49.002 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.138573006, 2023-09-23T20:42:49.002 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:42:49.002 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:42:49.002 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:42:49.002 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:42:49.002 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:42:49.003 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 92129 2023-09-23T20:42:49.003 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:49.003 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:42:49.003 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:49.003 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:42:49.003 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.307177+0000", 2023-09-23T20:42:49.003 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:42:49.003 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:49.003 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:49.003 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:42:49.003 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.307177+0000", 2023-09-23T20:42:49.004 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:42:49.004 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:49.004 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:49.004 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:42:49.004 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.307175+0000", 2023-09-23T20:42:49.004 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:42:49.004 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:49.004 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:49.004 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:42:49.004 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.307245+0000", 2023-09-23T20:42:49.004 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.0261999999999999e-05 2023-09-23T20:42:49.005 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:49.005 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:49.005 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:42:49.005 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.307253+0000", 2023-09-23T20:42:49.005 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.7219999999999999e-06 2023-09-23T20:42:49.005 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:49.005 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:49.005 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:42:49.005 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.307257+0000", 2023-09-23T20:42:49.005 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.0430000000000001e-06 2023-09-23T20:42:49.005 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:49.005 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:49.006 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:42:49.006 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.307299+0000", 2023-09-23T20:42:49.006 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.2128000000000001e-05 2023-09-23T20:42:49.006 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:49.006 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:49.006 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:42:49.006 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.307431+0000", 2023-09-23T20:42:49.006 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00013203499999999999 2023-09-23T20:42:49.006 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:49.006 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:49.006 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:42:49.007 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.307559+0000", 2023-09-23T20:42:49.007 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000127632 2023-09-23T20:42:49.007 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:49.007 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:49.007 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:42:49.007 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.317355+0000", 2023-09-23T20:42:49.007 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0097961349999999992 2023-09-23T20:42:49.007 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:49.007 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:49.007 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:42:49.007 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.320352+0000", 2023-09-23T20:42:49.007 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0029971569999999999 2023-09-23T20:42:49.008 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:49.008 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:49.008 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:42:49.008 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.445677+0000", 2023-09-23T20:42:49.008 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.12532515999999999 2023-09-23T20:42:49.008 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:49.008 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:49.008 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:42:49.008 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.445717+0000", 2023-09-23T20:42:49.008 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.9616000000000002e-05 2023-09-23T20:42:49.008 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:49.009 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:49.009 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:42:49.009 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.445750+0000", 2023-09-23T20:42:49.009 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.2719999999999998e-05 2023-09-23T20:42:49.009 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:42:49.009 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:42:49.009 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:42:49.009 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:49.009 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:49.009 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:92140 9.f 9:f3d16644:::benchmark_data_smithi018_381516_object92139:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e467)", 2023-09-23T20:42:49.009 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:42:14.313964+0000", 2023-09-23T20:42:49.009 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 34.561819126000003, 2023-09-23T20:42:49.010 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.13195174000000001, 2023-09-23T20:42:49.010 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:42:49.010 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:42:49.010 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:42:49.010 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:42:49.010 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:42:49.010 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 92140 2023-09-23T20:42:49.010 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:49.010 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:42:49.010 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:49.010 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:42:49.010 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.313964+0000", 2023-09-23T20:42:49.011 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:42:49.011 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:49.011 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:49.011 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:42:49.011 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.313964+0000", 2023-09-23T20:42:49.011 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:42:49.011 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:49.011 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:49.011 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:42:49.011 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.313964+0000", 2023-09-23T20:42:49.011 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:42:49.012 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:49.012 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:49.012 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:42:49.012 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.314026+0000", 2023-09-23T20:42:49.012 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.2818999999999996e-05 2023-09-23T20:42:49.012 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:49.012 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:49.012 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:42:49.012 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.314035+0000", 2023-09-23T20:42:49.012 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.7059999999999993e-06 2023-09-23T20:42:49.012 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:49.012 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:49.013 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:42:49.013 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.314040+0000", 2023-09-23T20:42:49.013 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.2660000000000001e-06 2023-09-23T20:42:49.013 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:49.013 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:49.013 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:42:49.013 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.314084+0000", 2023-09-23T20:42:49.013 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.3204000000000002e-05 2023-09-23T20:42:49.013 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:49.013 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:49.013 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:42:49.014 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.314195+0000", 2023-09-23T20:42:49.014 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000111585 2023-09-23T20:42:49.014 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:49.014 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:49.014 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:42:49.014 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.314336+0000", 2023-09-23T20:42:49.014 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000140977 2023-09-23T20:42:49.014 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:49.014 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:49.014 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:42:49.014 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.320388+0000", 2023-09-23T20:42:49.015 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.006051519 2023-09-23T20:42:49.015 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:49.015 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:49.015 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:42:49.015 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.320393+0000", 2023-09-23T20:42:49.015 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.7189999999999998e-06 2023-09-23T20:42:49.015 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:49.015 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:49.015 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:42:49.015 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.445855+0000", 2023-09-23T20:42:49.015 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.125461666 2023-09-23T20:42:49.015 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:49.016 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:49.016 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:42:49.016 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.445885+0000", 2023-09-23T20:42:49.016 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.0454e-05 2023-09-23T20:42:49.016 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:49.016 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:49.016 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:42:49.016 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.445916+0000", 2023-09-23T20:42:49.016 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.0446e-05 2023-09-23T20:42:49.016 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:42:49.016 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:42:49.016 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:42:49.017 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:49.017 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:49.017 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:92144 9.7 9:ec1b46c9:::benchmark_data_smithi018_381516_object92143:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e467)", 2023-09-23T20:42:49.017 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:42:14.317205+0000", 2023-09-23T20:42:49.017 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 34.558577962000001, 2023-09-23T20:42:49.017 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.12863032999999999, 2023-09-23T20:42:49.017 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:42:49.017 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:42:49.017 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:42:49.017 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:42:49.017 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:42:49.017 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 92144 2023-09-23T20:42:49.018 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:49.018 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:42:49.018 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:49.018 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:42:49.018 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.317205+0000", 2023-09-23T20:42:49.018 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:42:49.018 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:49.018 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:49.018 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:42:49.018 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.317205+0000", 2023-09-23T20:42:49.018 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:42:49.019 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:49.019 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:49.019 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:42:49.019 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.317204+0000", 2023-09-23T20:42:49.019 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:42:49.019 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:49.019 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:49.019 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:42:49.019 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.317280+0000", 2023-09-23T20:42:49.019 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.6215000000000001e-05 2023-09-23T20:42:49.019 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:49.019 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:49.020 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:42:49.020 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.317291+0000", 2023-09-23T20:42:49.020 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0546e-05 2023-09-23T20:42:49.020 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:49.020 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:49.020 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:42:49.020 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.317297+0000", 2023-09-23T20:42:49.020 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.2650000000000002e-06 2023-09-23T20:42:49.020 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:49.020 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:49.020 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:42:49.020 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.317374+0000", 2023-09-23T20:42:49.021 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.6600999999999994e-05 2023-09-23T20:42:49.021 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:49.021 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:49.021 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:42:49.021 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.317479+0000", 2023-09-23T20:42:49.021 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000105531 2023-09-23T20:42:49.021 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:49.021 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:49.021 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 3,7", 2023-09-23T20:42:49.021 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.317604+0000", 2023-09-23T20:42:49.021 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000124745 2023-09-23T20:42:49.022 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:49.022 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:49.022 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:42:49.022 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.321102+0000", 2023-09-23T20:42:49.022 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0034977849999999998 2023-09-23T20:42:49.022 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:49.022 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:49.022 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:42:49.022 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.321402+0000", 2023-09-23T20:42:49.022 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00029973499999999998 2023-09-23T20:42:49.022 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:49.022 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:49.023 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:42:49.023 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.445781+0000", 2023-09-23T20:42:49.023 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.124378979 2023-09-23T20:42:49.023 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:49.023 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:49.023 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:42:49.023 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.445816+0000", 2023-09-23T20:42:49.023 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.5222000000000003e-05 2023-09-23T20:42:49.023 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:49.023 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:49.023 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:42:49.024 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.445836+0000", 2023-09-23T20:42:49.024 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.9952000000000001e-05 2023-09-23T20:42:49.024 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:42:49.024 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:42:49.024 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:42:49.024 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:49.024 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:49.024 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:92152 9.f 9:fe7b9bff:::benchmark_data_smithi018_381516_object92151:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e467)", 2023-09-23T20:42:49.024 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:42:14.329392+0000", 2023-09-23T20:42:49.024 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 34.546391671999999, 2023-09-23T20:42:49.024 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.11662318300000001, 2023-09-23T20:42:49.024 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:42:49.025 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:42:49.025 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:42:49.025 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:42:49.025 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:42:49.025 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 92152 2023-09-23T20:42:49.025 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:49.025 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:42:49.025 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:49.025 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:42:49.025 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.329392+0000", 2023-09-23T20:42:49.025 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:42:49.026 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:49.026 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:49.026 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:42:49.026 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.329392+0000", 2023-09-23T20:42:49.026 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:42:49.026 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:49.026 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:49.026 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:42:49.026 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.329390+0000", 2023-09-23T20:42:49.026 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:42:49.027 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:49.027 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:49.027 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:42:49.027 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.329444+0000", 2023-09-23T20:42:49.027 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.3837e-05 2023-09-23T20:42:49.027 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:49.027 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:49.027 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:42:49.027 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.329454+0000", 2023-09-23T20:42:49.027 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.8789999999999998e-06 2023-09-23T20:42:49.027 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:49.027 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:49.028 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:42:49.028 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.329458+0000", 2023-09-23T20:42:49.028 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.0629999999999999e-06 2023-09-23T20:42:49.028 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:49.028 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:49.028 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:42:49.028 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.329492+0000", 2023-09-23T20:42:49.028 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.3692000000000002e-05 2023-09-23T20:42:49.028 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:49.028 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:49.028 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:42:49.029 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.329586+0000", 2023-09-23T20:42:49.029 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.3844000000000005e-05 2023-09-23T20:42:49.029 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:49.029 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:49.029 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:42:49.029 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.329684+0000", 2023-09-23T20:42:49.029 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.8581999999999999e-05 2023-09-23T20:42:49.029 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:49.029 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:49.029 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:42:49.029 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.332231+0000", 2023-09-23T20:42:49.029 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0025466989999999999 2023-09-23T20:42:49.030 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:49.030 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:49.030 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:42:49.030 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.332432+0000", 2023-09-23T20:42:49.030 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00020079700000000001 2023-09-23T20:42:49.030 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:49.030 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:49.030 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:42:49.030 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.445947+0000", 2023-09-23T20:42:49.030 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.11351505200000001 2023-09-23T20:42:49.030 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:49.031 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:49.031 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:42:49.031 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.445983+0000", 2023-09-23T20:42:49.031 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.6483999999999998e-05 2023-09-23T20:42:49.031 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:49.031 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:49.031 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:42:49.031 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.446015+0000", 2023-09-23T20:42:49.031 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.1401e-05 2023-09-23T20:42:49.031 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:42:49.031 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:42:49.031 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:42:49.032 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:42:49.032 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:42:49.032 INFO:teuthology.orchestra.run.smithi149.stdout:} 2023-09-23T20:42:49.033 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:49.033 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:42:49.033 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:42:49.134 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:42:49.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:48 smithi018 ceph-mon[179647]: pgmap v1193: 119 pgs: 2 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+undersized+remapped, 115 active+clean; 6.7 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 1.3 MiB/s wr, 549 op/s; 26067/330120 objects misplaced (7.896%); 13 MiB/s, 196 objects/s recovering 2023-09-23T20:42:49.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:48 smithi018 ceph-mon[184144]: pgmap v1193: 119 pgs: 2 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+undersized+remapped, 115 active+clean; 6.7 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 1.3 MiB/s wr, 549 op/s; 26067/330120 objects misplaced (7.896%); 13 MiB/s, 196 objects/s recovering 2023-09-23T20:42:49.208 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:48 smithi149 ceph-mon[170394]: pgmap v1193: 119 pgs: 2 active+remapped+backfilling, 1 active+remapped+backfill_wait, 1 active+undersized+remapped, 115 active+clean; 6.7 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 1.3 MiB/s wr, 549 op/s; 26067/330120 objects misplaced (7.896%); 13 MiB/s, 196 objects/s recovering 2023-09-23T20:42:49.213 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:49.266 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:42:49.325 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:42:49.349 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:42:49.351 INFO:teuthology.orchestra.run.smithi018.stderr:dumped all 2023-09-23T20:42:49.368 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:49.369 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:42:49.410 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T20:42:49.470 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:42:49.599 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:49.724 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:42:49.738 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:49.738 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:42:49.839 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:42:49.891 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:49.957 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:42:49.962 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:49.962 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:42:50.001 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:42:50.007 INFO:teuthology.orchestra.run.smithi018.stdout:{"pg_ready":true,"pg_map":{"version":1193,"stamp":"2023-09-23T20:42:48.136265+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":7214471115,"num_objects":110040,"num_object_clones":0,"num_object_copies":330120,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":26067,"num_objects_unfound":0,"num_objects_dirty":110040,"num_whiteouts":0,"num_read":5984,"num_read_kb":13201,"num_write":118559,"num_write_kb":7297910,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1261,"num_bytes_recovered":85018124,"num_keys_recovered":40,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":106998,"ondisk_log_size":106998,"up":357,"acting":356,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":362,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":25965696,"kb_used_data":21402240,"kb_used_omap":32,"kb_used_meta":4563231,"kb_avail":723995520,"statfs":{"total":767960285184,"available":741371412480,"internally_reserved":0,"allocated":21915893760,"data_stored":21900416707,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32773,"internal_metadata":4672749563},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":9,"apply_latency_ms":9,"commit_latency_ns":9000000,"apply_latency_ns":9000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-189136896,"num_objects":-2886,"num_object_clones":0,"num_object_copies":-8658,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":26067,"num_objects_unfound":0,"num_objects_dirty":-2886,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3120,"num_write_kb":7488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1116,"num_bytes_recovered":76546048,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"5.672807"},"pg_stats":[{"pgid":"1.18","version":"90'161","reported_seq":468,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T20:42:43.490935+0000","last_change":"2023-09-23T20:41:52.496116+0000","last_active":"2023-09-23T20:42:43.490935+0000","last_peered":"2023-09-23T20:42:43.490935+0000","last_clean":"2023-09-23T20:42:43.490935+0000","last_became_active":"2023-09-23T20:40:56.482886+0000","last_became_peered":"2023-09-23T20:40:56.482886+0000","last_unstale":"2023-09-23T20:42:43.490935+0000","last_undegraded":"2023-09-23T20:42:43.490935+0000","last_fullsized":"2023-09-23T20:42:43.490935+0000","mapping_epoch":453,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":454,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:41:52.496014+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:41:52.496014+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:43:46.350508+0000","scrub_duration":0.011825843000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"69'1","reported_seq":172,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T20:42:43.483600+0000","last_change":"2023-09-23T20:32:33.233853+0000","last_active":"2023-09-23T20:42:43.483600+0000","last_peered":"2023-09-23T20:42:43.483600+0000","last_clean":"2023-09-23T20:42:43.483600+0000","last_became_active":"2023-09-23T20:32:33.233546+0000","last_became_peered":"2023-09-23T20:32:33.233546+0000","last_unstale":"2023-09-23T20:42:43.483600+0000","last_undegraded":"2023-09-23T20:42:43.483600+0000","last_fullsized":"2023-09-23T20:42:43.483600+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:50:27.130808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":288,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T20:42:43.487170+0000","last_change":"2023-09-23T20:37:43.579611+0000","last_active":"2023-09-23T20:42:43.487170+0000","last_peered":"2023-09-23T20:42:43.487170+0000","last_clean":"2023-09-23T20:42:43.487170+0000","last_became_active":"2023-09-23T20:37:43.579323+0000","last_became_peered":"2023-09-23T20:37:43.579323+0000","last_unstale":"2023-09-23T20:42:43.487170+0000","last_undegraded":"2023-09-23T20:42:43.487170+0000","last_fullsized":"2023-09-23T20:42:43.487170+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:17:32.311427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"90'161","reported_seq":558,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T20:42:43.487877+0000","last_change":"2023-09-23T20:41:25.432814+0000","last_active":"2023-09-23T20:42:43.487877+0000","last_peered":"2023-09-23T20:42:43.487877+0000","last_clean":"2023-09-23T20:42:43.487877+0000","last_became_active":"2023-09-23T20:40:22.444637+0000","last_became_peered":"2023-09-23T20:40:22.444637+0000","last_unstale":"2023-09-23T20:42:43.487877+0000","last_undegraded":"2023-09-23T20:42:43.487877+0000","last_fullsized":"2023-09-23T20:42:43.487877+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:41:25.432747+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:36:05.442288+0000","last_clean_scrub_stamp":"2023-09-23T20:41:25.432747+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:50:06.434043+0000","scrub_duration":0.010139941,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":444,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T20:42:43.486916+0000","last_change":"2023-09-23T20:34:59.983035+0000","last_active":"2023-09-23T20:42:43.486916+0000","last_peered":"2023-09-23T20:42:43.486916+0000","last_clean":"2023-09-23T20:42:43.486916+0000","last_became_active":"2023-09-23T20:34:59.982459+0000","last_became_peered":"2023-09-23T20:34:59.982459+0000","last_unstale":"2023-09-23T20:42:43.486916+0000","last_undegraded":"2023-09-23T20:42:43.486916+0000","last_fullsized":"2023-09-23T20:42:43.486916+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:16:48.510783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":142,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T20:42:44.513732+0000","last_change":"2023-09-23T20:41:59.470187+0000","last_active":"2023-09-23T20:42:44.513732+0000","last_peered":"2023-09-23T20:42:44.513732+0000","last_clean":"2023-09-23T20:42:44.513732+0000","last_became_active":"2023-09-23T20:41:59.469864+0000","last_became_peered":"2023-09-23T20:41:59.469864+0000","last_unstale":"2023-09-23T20:42:44.513732+0000","last_undegraded":"2023-09-23T20:42:44.513732+0000","last_fullsized":"2023-09-23T20:42:44.513732+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:59.986014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.1a","version":"90'161","reported_seq":554,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T20:42:43.482036+0000","last_change":"2023-09-23T20:36:01.292103+0000","last_active":"2023-09-23T20:42:43.482036+0000","last_peered":"2023-09-23T20:42:43.482036+0000","last_clean":"2023-09-23T20:42:43.482036+0000","last_became_active":"2023-09-23T20:35:26.639912+0000","last_became_peered":"2023-09-23T20:35:26.639912+0000","last_unstale":"2023-09-23T20:42:43.482036+0000","last_undegraded":"2023-09-23T20:42:43.482036+0000","last_fullsized":"2023-09-23T20:42:43.482036+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:01.292032+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:01.292032+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:04:53.926640+0000","scrub_duration":0.0032983159999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":268,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-09-23T20:42:42.473289+0000","last_change":"2023-09-23T20:37:43.584414+0000","last_active":"2023-09-23T20:42:42.473289+0000","last_peered":"2023-09-23T20:42:42.473289+0000","last_clean":"2023-09-23T20:42:42.473289+0000","last_became_active":"2023-09-23T20:37:43.584116+0000","last_became_peered":"2023-09-23T20:37:43.584116+0000","last_unstale":"2023-09-23T20:42:42.473289+0000","last_undegraded":"2023-09-23T20:42:42.473289+0000","last_fullsized":"2023-09-23T20:42:42.473289+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:17:54.258172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":88,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T20:42:41.471032+0000","last_change":"2023-09-23T20:38:37.457427+0000","last_active":"2023-09-23T20:42:41.471032+0000","last_peered":"2023-09-23T20:42:41.471032+0000","last_clean":"2023-09-23T20:42:41.471032+0000","last_became_active":"2023-09-23T20:38:37.457099+0000","last_became_peered":"2023-09-23T20:38:37.457099+0000","last_unstale":"2023-09-23T20:42:41.471032+0000","last_undegraded":"2023-09-23T20:42:41.471032+0000","last_fullsized":"2023-09-23T20:42:41.471032+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:31:27.084994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"90'161","reported_seq":688,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T20:42:44.505599+0000","last_change":"2023-09-23T20:40:22.449754+0000","last_active":"2023-09-23T20:42:44.505599+0000","last_peered":"2023-09-23T20:42:44.505599+0000","last_clean":"2023-09-23T20:42:44.505599+0000","last_became_active":"2023-09-23T20:40:22.448838+0000","last_became_peered":"2023-09-23T20:40:22.448838+0000","last_unstale":"2023-09-23T20:42:44.505599+0000","last_undegraded":"2023-09-23T20:42:44.505599+0000","last_fullsized":"2023-09-23T20:42:44.505599+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:03.411025+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:03.411025+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:22:07.948856+0000","scrub_duration":0.001802922,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":457,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T20:42:43.481266+0000","last_change":"2023-09-23T20:41:59.464578+0000","last_active":"2023-09-23T20:42:43.481266+0000","last_peered":"2023-09-23T20:42:43.481266+0000","last_clean":"2023-09-23T20:42:43.481266+0000","last_became_active":"2023-09-23T20:41:59.464270+0000","last_became_peered":"2023-09-23T20:41:59.464270+0000","last_unstale":"2023-09-23T20:42:43.481266+0000","last_undegraded":"2023-09-23T20:42:43.481266+0000","last_fullsized":"2023-09-23T20:42:43.481266+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:27:54.869013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":200,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T20:42:44.499049+0000","last_change":"2023-09-23T20:38:02.579343+0000","last_active":"2023-09-23T20:42:44.499049+0000","last_peered":"2023-09-23T20:42:44.499049+0000","last_clean":"2023-09-23T20:42:44.499049+0000","last_became_active":"2023-09-23T20:38:02.579047+0000","last_became_peered":"2023-09-23T20:38:02.579047+0000","last_unstale":"2023-09-23T20:42:44.499049+0000","last_undegraded":"2023-09-23T20:42:44.499049+0000","last_fullsized":"2023-09-23T20:42:44.499049+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:44:03.166556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.14","version":"90'161","reported_seq":806,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T20:42:44.499439+0000","last_change":"2023-09-23T20:35:02.236398+0000","last_active":"2023-09-23T20:42:44.499439+0000","last_peered":"2023-09-23T20:42:44.499439+0000","last_clean":"2023-09-23T20:42:44.499439+0000","last_became_active":"2023-09-23T20:35:02.235988+0000","last_became_peered":"2023-09-23T20:35:02.235988+0000","last_unstale":"2023-09-23T20:42:44.499439+0000","last_undegraded":"2023-09-23T20:42:44.499439+0000","last_fullsized":"2023-09-23T20:42:44.499439+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:47.479012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:47.479012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:13.519648+0000","scrub_duration":0.0020136590000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":44,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T20:42:43.487728+0000","last_change":"2023-09-23T20:40:19.545472+0000","last_active":"2023-09-23T20:42:43.487728+0000","last_peered":"2023-09-23T20:42:43.487728+0000","last_clean":"2023-09-23T20:42:43.487728+0000","last_became_active":"2023-09-23T20:40:19.543853+0000","last_became_peered":"2023-09-23T20:40:19.543853+0000","last_unstale":"2023-09-23T20:42:43.487728+0000","last_undegraded":"2023-09-23T20:42:43.487728+0000","last_fullsized":"2023-09-23T20:42:43.487728+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:08.014744+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":299,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-09-23T20:42:42.471396+0000","last_change":"2023-09-23T20:38:02.542034+0000","last_active":"2023-09-23T20:42:42.471396+0000","last_peered":"2023-09-23T20:42:42.471396+0000","last_clean":"2023-09-23T20:42:42.471396+0000","last_became_active":"2023-09-23T20:38:02.541575+0000","last_became_peered":"2023-09-23T20:38:02.541575+0000","last_unstale":"2023-09-23T20:42:42.471396+0000","last_undegraded":"2023-09-23T20:42:42.471396+0000","last_fullsized":"2023-09-23T20:42:42.471396+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:50:54.591529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"90'161","reported_seq":638,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T20:42:43.481354+0000","last_change":"2023-09-23T20:41:11.210009+0000","last_active":"2023-09-23T20:42:43.481354+0000","last_peered":"2023-09-23T20:42:43.481354+0000","last_clean":"2023-09-23T20:42:43.481354+0000","last_became_active":"2023-09-23T20:40:22.444543+0000","last_became_peered":"2023-09-23T20:40:22.444543+0000","last_unstale":"2023-09-23T20:42:43.481354+0000","last_undegraded":"2023-09-23T20:42:43.481354+0000","last_fullsized":"2023-09-23T20:42:43.481354+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:41:11.209916+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:41:11.209916+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:30:05.067685+0000","scrub_duration":0.0025146090000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":503,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T20:42:44.507009+0000","last_change":"2023-09-23T20:32:33.270617+0000","last_active":"2023-09-23T20:42:44.507009+0000","last_peered":"2023-09-23T20:42:44.507009+0000","last_clean":"2023-09-23T20:42:44.507009+0000","last_became_active":"2023-09-23T20:32:33.227902+0000","last_became_peered":"2023-09-23T20:32:33.227902+0000","last_unstale":"2023-09-23T20:42:44.507009+0000","last_undegraded":"2023-09-23T20:42:44.507009+0000","last_fullsized":"2023-09-23T20:42:44.507009+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:30:26.847831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":360,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":377,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T20:42:41.472613+0000","last_change":"2023-09-23T20:42:41.472218+0000","last_active":"2023-09-23T20:42:41.472613+0000","last_peered":"2023-09-23T20:42:41.472613+0000","last_clean":"2023-09-23T20:42:41.472613+0000","last_became_active":"2023-09-23T20:42:41.471882+0000","last_became_peered":"2023-09-23T20:42:41.471882+0000","last_unstale":"2023-09-23T20:42:41.472613+0000","last_undegraded":"2023-09-23T20:42:41.472613+0000","last_fullsized":"2023-09-23T20:42:41.472613+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:26:00.814907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,0],"acting":[6,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.16","version":"90'161","reported_seq":600,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T20:42:44.501723+0000","last_change":"2023-09-23T20:36:02.548387+0000","last_active":"2023-09-23T20:42:44.501723+0000","last_peered":"2023-09-23T20:42:44.501723+0000","last_clean":"2023-09-23T20:42:44.501723+0000","last_became_active":"2023-09-23T20:35:26.644787+0000","last_became_peered":"2023-09-23T20:35:26.644787+0000","last_unstale":"2023-09-23T20:42:44.501723+0000","last_undegraded":"2023-09-23T20:42:44.501723+0000","last_fullsized":"2023-09-23T20:42:44.501723+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:02.548285+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:02.548285+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:37:15.216553+0000","scrub_duration":0.0015514459999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":222,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T20:42:43.483863+0000","last_change":"2023-09-23T20:38:02.560646+0000","last_active":"2023-09-23T20:42:43.483863+0000","last_peered":"2023-09-23T20:42:43.483863+0000","last_clean":"2023-09-23T20:42:43.483863+0000","last_became_active":"2023-09-23T20:38:02.539814+0000","last_became_peered":"2023-09-23T20:38:02.539814+0000","last_unstale":"2023-09-23T20:42:43.483863+0000","last_undegraded":"2023-09-23T20:42:43.483863+0000","last_fullsized":"2023-09-23T20:42:43.483863+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:00:09.435023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":202,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T20:42:43.483348+0000","last_change":"2023-09-23T20:38:02.579222+0000","last_active":"2023-09-23T20:42:43.483348+0000","last_peered":"2023-09-23T20:42:43.483348+0000","last_clean":"2023-09-23T20:42:43.483348+0000","last_became_active":"2023-09-23T20:38:02.578813+0000","last_became_peered":"2023-09-23T20:38:02.578813+0000","last_unstale":"2023-09-23T20:42:43.483348+0000","last_undegraded":"2023-09-23T20:42:43.483348+0000","last_fullsized":"2023-09-23T20:42:43.483348+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:57:31.080027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"90'161","reported_seq":714,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T20:42:43.481643+0000","last_change":"2023-09-23T20:41:59.463614+0000","last_active":"2023-09-23T20:42:43.481643+0000","last_peered":"2023-09-23T20:42:43.481643+0000","last_clean":"2023-09-23T20:42:43.481643+0000","last_became_active":"2023-09-23T20:41:59.463229+0000","last_became_peered":"2023-09-23T20:41:59.463229+0000","last_unstale":"2023-09-23T20:42:43.481643+0000","last_undegraded":"2023-09-23T20:42:43.481643+0000","last_fullsized":"2023-09-23T20:42:43.481643+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:43.578666+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:43.578666+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:36:52.767038+0000","scrub_duration":0.0021890379999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":502,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T20:42:44.505593+0000","last_change":"2023-09-23T20:32:33.242943+0000","last_active":"2023-09-23T20:42:44.505593+0000","last_peered":"2023-09-23T20:42:44.505593+0000","last_clean":"2023-09-23T20:42:44.505593+0000","last_became_active":"2023-09-23T20:32:33.225468+0000","last_became_peered":"2023-09-23T20:32:33.225468+0000","last_unstale":"2023-09-23T20:42:44.505593+0000","last_undegraded":"2023-09-23T20:42:44.505593+0000","last_fullsized":"2023-09-23T20:42:44.505593+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:40:47.867573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":210,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-09-23T20:42:42.471457+0000","last_change":"2023-09-23T20:37:43.571836+0000","last_active":"2023-09-23T20:42:42.471457+0000","last_peered":"2023-09-23T20:42:42.471457+0000","last_clean":"2023-09-23T20:42:42.471457+0000","last_became_active":"2023-09-23T20:37:43.571511+0000","last_became_peered":"2023-09-23T20:37:43.571511+0000","last_unstale":"2023-09-23T20:42:42.471457+0000","last_undegraded":"2023-09-23T20:42:42.471457+0000","last_fullsized":"2023-09-23T20:42:42.471457+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:59:25.507917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"90'161","reported_seq":667,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T20:42:43.491250+0000","last_change":"2023-09-23T20:41:22.390219+0000","last_active":"2023-09-23T20:42:43.491250+0000","last_peered":"2023-09-23T20:42:43.491250+0000","last_clean":"2023-09-23T20:42:43.491250+0000","last_became_active":"2023-09-23T20:40:22.427114+0000","last_became_peered":"2023-09-23T20:40:22.427114+0000","last_unstale":"2023-09-23T20:42:43.491250+0000","last_undegraded":"2023-09-23T20:42:43.491250+0000","last_fullsized":"2023-09-23T20:42:43.491250+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:41:22.390153+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:41:22.390153+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:17:27.488965+0000","scrub_duration":0.00186133,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":141,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T20:42:44.505342+0000","last_change":"2023-09-23T20:41:59.465625+0000","last_active":"2023-09-23T20:42:44.505342+0000","last_peered":"2023-09-23T20:42:44.505342+0000","last_clean":"2023-09-23T20:42:44.505342+0000","last_became_active":"2023-09-23T20:41:59.465309+0000","last_became_peered":"2023-09-23T20:41:59.465309+0000","last_unstale":"2023-09-23T20:42:44.505342+0000","last_undegraded":"2023-09-23T20:42:44.505342+0000","last_fullsized":"2023-09-23T20:42:44.505342+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:55:51.159392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":44,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T20:42:43.481037+0000","last_change":"2023-09-23T20:40:19.561512+0000","last_active":"2023-09-23T20:42:43.481037+0000","last_peered":"2023-09-23T20:42:43.481037+0000","last_clean":"2023-09-23T20:42:43.481037+0000","last_became_active":"2023-09-23T20:40:19.561123+0000","last_became_peered":"2023-09-23T20:40:19.561123+0000","last_unstale":"2023-09-23T20:42:43.481037+0000","last_undegraded":"2023-09-23T20:42:43.481037+0000","last_fullsized":"2023-09-23T20:42:43.481037+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:50:02.093978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"90'161","reported_seq":761,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T20:42:43.489122+0000","last_change":"2023-09-23T20:35:02.238923+0000","last_active":"2023-09-23T20:42:43.489122+0000","last_peered":"2023-09-23T20:42:43.489122+0000","last_clean":"2023-09-23T20:42:43.489122+0000","last_became_active":"2023-09-23T20:35:02.238293+0000","last_became_peered":"2023-09-23T20:35:02.238293+0000","last_unstale":"2023-09-23T20:42:43.489122+0000","last_undegraded":"2023-09-23T20:42:43.489122+0000","last_fullsized":"2023-09-23T20:42:43.489122+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:42.544308+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:42.544308+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:51:22.281932+0000","scrub_duration":0.0029958179999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":479,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T20:42:44.505898+0000","last_change":"2023-09-23T20:38:02.539774+0000","last_active":"2023-09-23T20:42:44.505898+0000","last_peered":"2023-09-23T20:42:44.505898+0000","last_clean":"2023-09-23T20:42:44.505898+0000","last_became_active":"2023-09-23T20:38:02.539436+0000","last_became_peered":"2023-09-23T20:38:02.539436+0000","last_unstale":"2023-09-23T20:42:44.505898+0000","last_undegraded":"2023-09-23T20:42:44.505898+0000","last_fullsized":"2023-09-23T20:42:44.505898+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:50:57.375803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":147,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-09-23T20:42:42.471113+0000","last_change":"2023-09-23T20:41:59.468921+0000","last_active":"2023-09-23T20:42:42.471113+0000","last_peered":"2023-09-23T20:42:42.471113+0000","last_clean":"2023-09-23T20:42:42.471113+0000","last_became_active":"2023-09-23T20:41:59.468511+0000","last_became_peered":"2023-09-23T20:41:59.468511+0000","last_unstale":"2023-09-23T20:42:42.471113+0000","last_undegraded":"2023-09-23T20:42:42.471113+0000","last_fullsized":"2023-09-23T20:42:42.471113+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:50:16.849066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"90'161","reported_seq":590,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T20:42:43.480931+0000","last_change":"2023-09-23T20:41:08.258019+0000","last_active":"2023-09-23T20:42:43.480931+0000","last_peered":"2023-09-23T20:42:43.480931+0000","last_clean":"2023-09-23T20:42:43.480931+0000","last_became_active":"2023-09-23T20:40:22.430717+0000","last_became_peered":"2023-09-23T20:40:22.430717+0000","last_unstale":"2023-09-23T20:42:43.480931+0000","last_undegraded":"2023-09-23T20:42:43.480931+0000","last_fullsized":"2023-09-23T20:42:43.480931+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:41:08.257952+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:41:08.257952+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:47:51.982877+0000","scrub_duration":0.00158735,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":128,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T20:42:41.477270+0000","last_change":"2023-09-23T20:42:41.476946+0000","last_active":"2023-09-23T20:42:41.477270+0000","last_peered":"2023-09-23T20:42:41.477270+0000","last_clean":"2023-09-23T20:42:41.477270+0000","last_became_active":"2023-09-23T20:42:41.472591+0000","last_became_peered":"2023-09-23T20:42:41.472591+0000","last_unstale":"2023-09-23T20:42:41.477270+0000","last_undegraded":"2023-09-23T20:42:41.477270+0000","last_fullsized":"2023-09-23T20:42:41.477270+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:54:00.487927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":268,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-09-23T20:42:42.475142+0000","last_change":"2023-09-23T20:37:43.571083+0000","last_active":"2023-09-23T20:42:42.475142+0000","last_peered":"2023-09-23T20:42:42.475142+0000","last_clean":"2023-09-23T20:42:42.475142+0000","last_became_active":"2023-09-23T20:37:43.570306+0000","last_became_peered":"2023-09-23T20:37:43.570306+0000","last_unstale":"2023-09-23T20:42:42.475142+0000","last_undegraded":"2023-09-23T20:42:42.475142+0000","last_fullsized":"2023-09-23T20:42:42.475142+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:42:02.396718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.5","version":"475'7206","reported_seq":7256,"reported_epoch":475,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T20:42:45.475499+0000","last_change":"2023-09-23T20:42:42.595017+0000","last_active":"2023-09-23T20:42:45.475499+0000","last_peered":"2023-09-23T20:42:45.475499+0000","last_clean":"2023-09-23T20:42:40.765834+0000","last_became_active":"2023-09-23T20:42:42.584419+0000","last_became_peered":"2023-09-23T20:42:42.584419+0000","last_unstale":"2023-09-23T20:42:45.475499+0000","last_undegraded":"2023-09-23T20:42:45.475499+0000","last_fullsized":"2023-09-23T20:42:45.475499+0000","mapping_epoch":472,"log_start":"458'200","ondisk_log_start":"458'200","created":455,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":7006,"log_dups_size":0,"ondisk_log_size":7006,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:56:35.792239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":444334080,"num_objects":6780,"num_object_clones":0,"num_object_copies":20340,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":6780,"num_objects_unfound":0,"num_objects_dirty":6780,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7206,"num_write_kb":447552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,7],"avail_no_missing":["5","0","7"],"object_location_counts":[{"shards":"0,5,7","objects":6780}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"90'161","reported_seq":768,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T20:42:44.505747+0000","last_change":"2023-09-23T20:34:59.496438+0000","last_active":"2023-09-23T20:42:44.505747+0000","last_peered":"2023-09-23T20:42:44.505747+0000","last_clean":"2023-09-23T20:42:44.505747+0000","last_became_active":"2023-09-23T20:33:38.970527+0000","last_became_peered":"2023-09-23T20:33:38.970527+0000","last_unstale":"2023-09-23T20:42:44.505747+0000","last_undegraded":"2023-09-23T20:42:44.505747+0000","last_fullsized":"2023-09-23T20:42:44.505747+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:34:59.496375+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:34:59.496375+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:46:18.244734+0000","scrub_duration":0.0017843189999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":479,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T20:42:44.513001+0000","last_change":"2023-09-23T20:32:33.283752+0000","last_active":"2023-09-23T20:42:44.513001+0000","last_peered":"2023-09-23T20:42:44.513001+0000","last_clean":"2023-09-23T20:42:44.513001+0000","last_became_active":"2023-09-23T20:32:33.230503+0000","last_became_peered":"2023-09-23T20:32:33.230503+0000","last_unstale":"2023-09-23T20:42:44.513001+0000","last_undegraded":"2023-09-23T20:42:44.513001+0000","last_fullsized":"2023-09-23T20:42:44.513001+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:29.084410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":44,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T20:42:43.491196+0000","last_change":"2023-09-23T20:40:19.545668+0000","last_active":"2023-09-23T20:42:43.491196+0000","last_peered":"2023-09-23T20:42:43.491196+0000","last_clean":"2023-09-23T20:42:43.491196+0000","last_became_active":"2023-09-23T20:40:19.543857+0000","last_became_peered":"2023-09-23T20:40:19.543857+0000","last_unstale":"2023-09-23T20:42:43.491196+0000","last_undegraded":"2023-09-23T20:42:43.491196+0000","last_fullsized":"2023-09-23T20:42:43.491196+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:27:09.470605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"475'7348","reported_seq":7377,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T20:42:44.614377+0000","last_change":"2023-09-23T20:41:04.575200+0000","last_active":"2023-09-23T20:42:44.614377+0000","last_peered":"2023-09-23T20:42:44.614377+0000","last_clean":"2023-09-23T20:42:44.614377+0000","last_became_active":"2023-09-23T20:41:04.574824+0000","last_became_peered":"2023-09-23T20:41:04.574824+0000","last_unstale":"2023-09-23T20:42:44.614377+0000","last_undegraded":"2023-09-23T20:42:44.614377+0000","last_fullsized":"2023-09-23T20:42:44.614377+0000","mapping_epoch":455,"log_start":"459'1400","ondisk_log_start":"459'1400","created":455,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":5948,"log_dups_size":0,"ondisk_log_size":5948,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:50:46.714383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":456261632,"num_objects":6962,"num_object_clones":0,"num_object_copies":20886,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6962,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7348,"num_write_kb":457920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.c","version":"90'161","reported_seq":734,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-09-23T20:42:42.471546+0000","last_change":"2023-09-23T20:41:09.506481+0000","last_active":"2023-09-23T20:42:42.471546+0000","last_peered":"2023-09-23T20:42:42.471546+0000","last_clean":"2023-09-23T20:42:42.471546+0000","last_became_active":"2023-09-23T20:38:28.442229+0000","last_became_peered":"2023-09-23T20:38:28.442229+0000","last_unstale":"2023-09-23T20:42:42.471546+0000","last_undegraded":"2023-09-23T20:42:42.471546+0000","last_fullsized":"2023-09-23T20:42:42.471546+0000","mapping_epoch":411,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":412,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:41:09.506338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:41:09.506338+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:19:46.265011+0000","scrub_duration":0.01112002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":514,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T20:42:44.513652+0000","last_change":"2023-09-23T20:34:59.987981+0000","last_active":"2023-09-23T20:42:44.513652+0000","last_peered":"2023-09-23T20:42:44.513652+0000","last_clean":"2023-09-23T20:42:44.513652+0000","last_became_active":"2023-09-23T20:34:59.985368+0000","last_became_peered":"2023-09-23T20:34:59.985368+0000","last_unstale":"2023-09-23T20:42:44.513652+0000","last_undegraded":"2023-09-23T20:42:44.513652+0000","last_fullsized":"2023-09-23T20:42:44.513652+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:12:43.082493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":172,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T20:42:43.482443+0000","last_change":"2023-09-23T20:32:33.232142+0000","last_active":"2023-09-23T20:42:43.482443+0000","last_peered":"2023-09-23T20:42:43.482443+0000","last_clean":"2023-09-23T20:42:43.482443+0000","last_became_active":"2023-09-23T20:32:33.231683+0000","last_became_peered":"2023-09-23T20:32:33.231683+0000","last_unstale":"2023-09-23T20:42:43.482443+0000","last_undegraded":"2023-09-23T20:42:43.482443+0000","last_fullsized":"2023-09-23T20:42:43.482443+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:02:58.336348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"475'7344","reported_seq":7369,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T20:42:45.539677+0000","last_change":"2023-09-23T20:41:04.557945+0000","last_active":"2023-09-23T20:42:45.539677+0000","last_peered":"2023-09-23T20:42:45.539677+0000","last_clean":"2023-09-23T20:42:45.539677+0000","last_became_active":"2023-09-23T20:41:04.557435+0000","last_became_peered":"2023-09-23T20:41:04.557435+0000","last_unstale":"2023-09-23T20:42:45.539677+0000","last_undegraded":"2023-09-23T20:42:45.539677+0000","last_fullsized":"2023-09-23T20:42:45.539677+0000","mapping_epoch":455,"log_start":"458'300","ondisk_log_start":"458'300","created":455,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":7044,"log_dups_size":0,"ondisk_log_size":7044,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:09:02.910592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":447741952,"num_objects":6832,"num_object_clones":0,"num_object_copies":20496,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6832,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7344,"num_write_kb":453632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.f","version":"90'161","reported_seq":628,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T20:42:41.471241+0000","last_change":"2023-09-23T20:38:40.491902+0000","last_active":"2023-09-23T20:42:41.471241+0000","last_peered":"2023-09-23T20:42:41.471241+0000","last_clean":"2023-09-23T20:42:41.471241+0000","last_became_active":"2023-09-23T20:38:40.491563+0000","last_became_peered":"2023-09-23T20:38:40.491563+0000","last_unstale":"2023-09-23T20:42:41.471241+0000","last_undegraded":"2023-09-23T20:42:41.471241+0000","last_fullsized":"2023-09-23T20:42:41.471241+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:46.217489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:46.217489+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:39:25.008796+0000","scrub_duration":0.001456429,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":504,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T20:42:44.505830+0000","last_change":"2023-09-23T20:40:19.677655+0000","last_active":"2023-09-23T20:42:44.505830+0000","last_peered":"2023-09-23T20:42:44.505830+0000","last_clean":"2023-09-23T20:42:44.505830+0000","last_became_active":"2023-09-23T20:40:19.551189+0000","last_became_peered":"2023-09-23T20:40:19.551189+0000","last_unstale":"2023-09-23T20:42:44.505830+0000","last_undegraded":"2023-09-23T20:42:44.505830+0000","last_fullsized":"2023-09-23T20:42:44.505830+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:11:01.559576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":78,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T20:42:41.470115+0000","last_change":"2023-09-23T20:38:02.590613+0000","last_active":"2023-09-23T20:42:41.470115+0000","last_peered":"2023-09-23T20:42:41.470115+0000","last_clean":"2023-09-23T20:42:41.470115+0000","last_became_active":"2023-09-23T20:38:02.590284+0000","last_became_peered":"2023-09-23T20:38:02.590284+0000","last_unstale":"2023-09-23T20:42:41.470115+0000","last_undegraded":"2023-09-23T20:42:41.470115+0000","last_fullsized":"2023-09-23T20:42:41.470115+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:40:07.825054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"475'7308","reported_seq":7336,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T20:42:45.469330+0000","last_change":"2023-09-23T20:41:04.573199+0000","last_active":"2023-09-23T20:42:45.469330+0000","last_peered":"2023-09-23T20:42:45.469330+0000","last_clean":"2023-09-23T20:42:45.469330+0000","last_became_active":"2023-09-23T20:41:04.572850+0000","last_became_peered":"2023-09-23T20:41:04.572850+0000","last_unstale":"2023-09-23T20:42:45.469330+0000","last_undegraded":"2023-09-23T20:42:45.469330+0000","last_fullsized":"2023-09-23T20:42:45.469330+0000","mapping_epoch":455,"log_start":"458'300","ondisk_log_start":"458'300","created":455,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":7008,"log_dups_size":0,"ondisk_log_size":7008,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:53:49.894113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":445775872,"num_objects":6802,"num_object_clones":0,"num_object_copies":20406,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6802,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7308,"num_write_kb":451520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.e","version":"90'161","reported_seq":575,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T20:42:43.485026+0000","last_change":"2023-09-23T20:36:00.322453+0000","last_active":"2023-09-23T20:42:43.485026+0000","last_peered":"2023-09-23T20:42:43.485026+0000","last_clean":"2023-09-23T20:42:43.485026+0000","last_became_active":"2023-09-23T20:35:02.233536+0000","last_became_peered":"2023-09-23T20:35:02.233536+0000","last_unstale":"2023-09-23T20:42:43.485026+0000","last_undegraded":"2023-09-23T20:42:43.485026+0000","last_fullsized":"2023-09-23T20:42:43.485026+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:00.322381+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:00.322381+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:02:15.489069+0000","scrub_duration":0.0017321859999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":69,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T20:42:43.492908+0000","last_change":"2023-09-23T20:38:37.456683+0000","last_active":"2023-09-23T20:42:43.492908+0000","last_peered":"2023-09-23T20:42:43.492908+0000","last_clean":"2023-09-23T20:42:43.492908+0000","last_became_active":"2023-09-23T20:38:37.456289+0000","last_became_peered":"2023-09-23T20:38:37.456289+0000","last_unstale":"2023-09-23T20:42:43.492908+0000","last_undegraded":"2023-09-23T20:42:43.492908+0000","last_fullsized":"2023-09-23T20:42:43.492908+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:03:10.507333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":307,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T20:42:43.489042+0000","last_change":"2023-09-23T20:41:59.459663+0000","last_active":"2023-09-23T20:42:43.489042+0000","last_peered":"2023-09-23T20:42:43.489042+0000","last_clean":"2023-09-23T20:42:43.489042+0000","last_became_active":"2023-09-23T20:41:59.459326+0000","last_became_peered":"2023-09-23T20:41:59.459326+0000","last_unstale":"2023-09-23T20:42:43.489042+0000","last_undegraded":"2023-09-23T20:42:43.489042+0000","last_fullsized":"2023-09-23T20:42:43.489042+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:59:41.487932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.1","version":"473'7213","reported_seq":7238,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-09-23T20:42:43.177113+0000","last_change":"2023-09-23T20:41:04.558473+0000","last_active":"2023-09-23T20:42:43.177113+0000","last_peered":"2023-09-23T20:42:43.177113+0000","last_clean":"2023-09-23T20:42:43.177113+0000","last_became_active":"2023-09-23T20:41:04.558173+0000","last_became_peered":"2023-09-23T20:41:04.558173+0000","last_unstale":"2023-09-23T20:42:43.177113+0000","last_undegraded":"2023-09-23T20:42:43.177113+0000","last_fullsized":"2023-09-23T20:42:43.177113+0000","mapping_epoch":455,"log_start":"458'363","ondisk_log_start":"458'363","created":455,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":6850,"log_dups_size":0,"ondisk_log_size":6850,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:17:53.184877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":455540736,"num_objects":6951,"num_object_clones":0,"num_object_copies":20853,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6951,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7213,"num_write_kb":453248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":881,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T20:42:44.506249+0000","last_change":"2023-09-23T20:35:02.523308+0000","last_active":"2023-09-23T20:42:44.506249+0000","last_peered":"2023-09-23T20:42:44.506249+0000","last_clean":"2023-09-23T20:42:44.506249+0000","last_became_active":"2023-09-23T20:33:38.979989+0000","last_became_peered":"2023-09-23T20:33:38.979989+0000","last_unstale":"2023-09-23T20:42:44.506249+0000","last_undegraded":"2023-09-23T20:42:44.506249+0000","last_fullsized":"2023-09-23T20:42:44.506249+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:35:02.523203+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:35:02.523203+0000","last_clean_scrub_stamp":"2023-09-23T20:35:02.523203+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:02:41.472958+0000","scrub_duration":0.0085927989999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":294,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-09-23T20:42:42.471060+0000","last_change":"2023-09-23T20:37:43.583731+0000","last_active":"2023-09-23T20:42:42.471060+0000","last_peered":"2023-09-23T20:42:42.471060+0000","last_clean":"2023-09-23T20:42:42.471060+0000","last_became_active":"2023-09-23T20:37:43.583066+0000","last_became_peered":"2023-09-23T20:37:43.583066+0000","last_unstale":"2023-09-23T20:42:42.471060+0000","last_undegraded":"2023-09-23T20:42:42.471060+0000","last_fullsized":"2023-09-23T20:42:42.471060+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:03:45.211900+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":192,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T20:42:43.483786+0000","last_change":"2023-09-23T20:38:02.579814+0000","last_active":"2023-09-23T20:42:43.483786+0000","last_peered":"2023-09-23T20:42:43.483786+0000","last_clean":"2023-09-23T20:42:43.483786+0000","last_became_active":"2023-09-23T20:38:02.579253+0000","last_became_peered":"2023-09-23T20:38:02.579253+0000","last_unstale":"2023-09-23T20:42:43.483786+0000","last_undegraded":"2023-09-23T20:42:43.483786+0000","last_fullsized":"2023-09-23T20:42:43.483786+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:05:07.298198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"475'7325","reported_seq":7354,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T20:42:44.645476+0000","last_change":"2023-09-23T20:41:04.573101+0000","last_active":"2023-09-23T20:42:44.645476+0000","last_peered":"2023-09-23T20:42:44.645476+0000","last_clean":"2023-09-23T20:42:44.645476+0000","last_became_active":"2023-09-23T20:41:04.572728+0000","last_became_peered":"2023-09-23T20:41:04.572728+0000","last_unstale":"2023-09-23T20:42:44.645476+0000","last_undegraded":"2023-09-23T20:42:44.645476+0000","last_fullsized":"2023-09-23T20:42:44.645476+0000","mapping_epoch":455,"log_start":"459'1400","ondisk_log_start":"459'1400","created":455,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":5925,"log_dups_size":0,"ondisk_log_size":5925,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:21:10.721593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":454754304,"num_objects":6939,"num_object_clones":0,"num_object_copies":20817,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6939,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7325,"num_write_kb":456448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":573,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T20:42:45.492384+0000","last_change":"2023-09-23T20:42:45.492384+0000","last_active":"2023-09-23T20:42:45.492384+0000","last_peered":"2023-09-23T20:42:45.492384+0000","last_clean":"2023-09-23T20:42:45.492384+0000","last_became_active":"2023-09-23T20:42:44.508223+0000","last_became_peered":"2023-09-23T20:42:44.508223+0000","last_unstale":"2023-09-23T20:42:45.492384+0000","last_undegraded":"2023-09-23T20:42:45.492384+0000","last_fullsized":"2023-09-23T20:42:45.492384+0000","mapping_epoch":474,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":475,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:42:45.492220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:42:45.492220+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:43:53.211255+0000","scrub_duration":0.015259412,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":78,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T20:42:41.470192+0000","last_change":"2023-09-23T20:38:02.578703+0000","last_active":"2023-09-23T20:42:41.470192+0000","last_peered":"2023-09-23T20:42:41.470192+0000","last_clean":"2023-09-23T20:42:41.470192+0000","last_became_active":"2023-09-23T20:38:02.578414+0000","last_became_peered":"2023-09-23T20:38:02.578414+0000","last_unstale":"2023-09-23T20:42:41.470192+0000","last_undegraded":"2023-09-23T20:42:41.470192+0000","last_fullsized":"2023-09-23T20:42:41.470192+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:43:59.779295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":141,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T20:42:43.484904+0000","last_change":"2023-09-23T20:37:31.296606+0000","last_active":"2023-09-23T20:42:43.484904+0000","last_peered":"2023-09-23T20:42:43.484904+0000","last_clean":"2023-09-23T20:42:43.484904+0000","last_became_active":"2023-09-23T20:37:31.295924+0000","last_became_peered":"2023-09-23T20:37:31.295924+0000","last_unstale":"2023-09-23T20:42:43.484904+0000","last_undegraded":"2023-09-23T20:42:43.484904+0000","last_fullsized":"2023-09-23T20:42:43.484904+0000","mapping_epoch":391,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":392,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:27:50.948510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"476'7468","reported_seq":7497,"reported_epoch":476,"state":"active+clean","last_fresh":"2023-09-23T20:42:47.397341+0000","last_change":"2023-09-23T20:41:04.555980+0000","last_active":"2023-09-23T20:42:47.397341+0000","last_peered":"2023-09-23T20:42:47.397341+0000","last_clean":"2023-09-23T20:42:47.397341+0000","last_became_active":"2023-09-23T20:41:04.555688+0000","last_became_peered":"2023-09-23T20:41:04.555688+0000","last_unstale":"2023-09-23T20:42:47.397341+0000","last_undegraded":"2023-09-23T20:42:47.397341+0000","last_fullsized":"2023-09-23T20:42:47.397341+0000","mapping_epoch":455,"log_start":"459'1756","ondisk_log_start":"459'1756","created":455,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":5712,"log_dups_size":0,"ondisk_log_size":5712,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:28:42.819404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":442499072,"num_objects":6752,"num_object_clones":0,"num_object_copies":20256,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6752,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7468,"num_write_kb":455040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.b","version":"90'161","reported_seq":777,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T20:42:43.489175+0000","last_change":"2023-09-23T20:40:22.445808+0000","last_active":"2023-09-23T20:42:43.489175+0000","last_peered":"2023-09-23T20:42:43.489175+0000","last_clean":"2023-09-23T20:42:43.489175+0000","last_became_active":"2023-09-23T20:40:22.445456+0000","last_became_peered":"2023-09-23T20:40:22.445456+0000","last_unstale":"2023-09-23T20:42:43.489175+0000","last_undegraded":"2023-09-23T20:42:43.489175+0000","last_fullsized":"2023-09-23T20:42:43.489175+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:05.595042+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:05.595042+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:39:23.678154+0000","scrub_duration":0.007271675,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":131,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T20:42:43.482205+0000","last_change":"2023-09-23T20:34:59.983122+0000","last_active":"2023-09-23T20:42:43.482205+0000","last_peered":"2023-09-23T20:42:43.482205+0000","last_clean":"2023-09-23T20:42:43.482205+0000","last_became_active":"2023-09-23T20:34:59.982809+0000","last_became_peered":"2023-09-23T20:34:59.982809+0000","last_unstale":"2023-09-23T20:42:43.482205+0000","last_undegraded":"2023-09-23T20:42:43.482205+0000","last_fullsized":"2023-09-23T20:42:43.482205+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:49:40.839723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":202,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T20:42:43.485214+0000","last_change":"2023-09-23T20:42:41.471863+0000","last_active":"2023-09-23T20:42:43.485214+0000","last_peered":"2023-09-23T20:42:43.485214+0000","last_clean":"2023-09-23T20:42:43.485214+0000","last_became_active":"2023-09-23T20:42:41.471546+0000","last_became_peered":"2023-09-23T20:42:41.471546+0000","last_unstale":"2023-09-23T20:42:43.485214+0000","last_undegraded":"2023-09-23T20:42:43.485214+0000","last_fullsized":"2023-09-23T20:42:43.485214+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:46:02.873050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"474'7329","reported_seq":7357,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T20:42:44.027998+0000","last_change":"2023-09-23T20:41:04.573300+0000","last_active":"2023-09-23T20:42:44.027998+0000","last_peered":"2023-09-23T20:42:44.027998+0000","last_clean":"2023-09-23T20:42:44.027998+0000","last_became_active":"2023-09-23T20:41:04.572919+0000","last_became_peered":"2023-09-23T20:41:04.572919+0000","last_unstale":"2023-09-23T20:42:44.027998+0000","last_undegraded":"2023-09-23T20:42:44.027998+0000","last_fullsized":"2023-09-23T20:42:44.027998+0000","mapping_epoch":455,"log_start":"459'1694","ondisk_log_start":"459'1694","created":455,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":5635,"log_dups_size":0,"ondisk_log_size":5635,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:57:46.465243+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":457768960,"num_objects":6985,"num_object_clones":0,"num_object_copies":20955,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6985,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7329,"num_write_kb":458048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":589,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T20:42:43.480770+0000","last_change":"2023-09-23T20:41:07.289523+0000","last_active":"2023-09-23T20:42:43.480770+0000","last_peered":"2023-09-23T20:42:43.480770+0000","last_clean":"2023-09-23T20:42:43.480770+0000","last_became_active":"2023-09-23T20:38:40.494049+0000","last_became_peered":"2023-09-23T20:38:40.494049+0000","last_unstale":"2023-09-23T20:42:43.480770+0000","last_undegraded":"2023-09-23T20:42:43.480770+0000","last_fullsized":"2023-09-23T20:42:43.480770+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:41:07.289420+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:41:07.289420+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:12:25.551623+0000","scrub_duration":0.0015464249999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":268,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-09-23T20:42:42.473271+0000","last_change":"2023-09-23T20:37:43.570610+0000","last_active":"2023-09-23T20:42:42.473271+0000","last_peered":"2023-09-23T20:42:42.473271+0000","last_clean":"2023-09-23T20:42:42.473271+0000","last_became_active":"2023-09-23T20:37:43.570216+0000","last_became_peered":"2023-09-23T20:37:43.570216+0000","last_unstale":"2023-09-23T20:42:42.473271+0000","last_undegraded":"2023-09-23T20:42:42.473271+0000","last_fullsized":"2023-09-23T20:42:42.473271+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:39:26.555329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":299,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T20:42:43.488213+0000","last_change":"2023-09-23T20:32:33.230582+0000","last_active":"2023-09-23T20:42:43.488213+0000","last_peered":"2023-09-23T20:42:43.488213+0000","last_clean":"2023-09-23T20:42:43.488213+0000","last_became_active":"2023-09-23T20:32:33.230257+0000","last_became_peered":"2023-09-23T20:32:33.230257+0000","last_unstale":"2023-09-23T20:42:43.488213+0000","last_undegraded":"2023-09-23T20:42:43.488213+0000","last_fullsized":"2023-09-23T20:42:43.488213+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:11:45.333096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.f","version":"475'7440","reported_seq":7468,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T20:42:45.414808+0000","last_change":"2023-09-23T20:41:04.557353+0000","last_active":"2023-09-23T20:42:45.414808+0000","last_peered":"2023-09-23T20:42:45.414808+0000","last_clean":"2023-09-23T20:42:45.414808+0000","last_became_active":"2023-09-23T20:41:04.557037+0000","last_became_peered":"2023-09-23T20:41:04.557037+0000","last_unstale":"2023-09-23T20:42:45.414808+0000","last_undegraded":"2023-09-23T20:42:45.414808+0000","last_fullsized":"2023-09-23T20:42:45.414808+0000","mapping_epoch":455,"log_start":"458'400","ondisk_log_start":"458'400","created":455,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":7040,"log_dups_size":0,"ondisk_log_size":7040,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:23:30.329193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":455213056,"num_objects":6946,"num_object_clones":0,"num_object_copies":20838,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6946,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7440,"num_write_kb":460352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":661,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T20:42:41.471161+0000","last_change":"2023-09-23T20:41:07.424609+0000","last_active":"2023-09-23T20:42:41.471161+0000","last_peered":"2023-09-23T20:42:41.471161+0000","last_clean":"2023-09-23T20:42:41.471161+0000","last_became_active":"2023-09-23T20:38:05.622261+0000","last_became_peered":"2023-09-23T20:38:05.622261+0000","last_unstale":"2023-09-23T20:42:41.471161+0000","last_undegraded":"2023-09-23T20:42:41.471161+0000","last_fullsized":"2023-09-23T20:42:41.471161+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:41:07.424492+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:41:07.424492+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:45:21.689123+0000","scrub_duration":0.0018960959999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"86'119","reported_seq":784,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T20:42:43.487440+0000","last_change":"2023-09-23T20:29:45.521257+0000","last_active":"2023-09-23T20:42:43.487440+0000","last_peered":"2023-09-23T20:42:43.487440+0000","last_clean":"2023-09-23T20:42:43.487440+0000","last_became_active":"2023-09-23T20:29:45.520546+0000","last_became_peered":"2023-09-23T20:29:45.520546+0000","last_unstale":"2023-09-23T20:42:43.487440+0000","last_undegraded":"2023-09-23T20:42:43.487440+0000","last_fullsized":"2023-09-23T20:42:43.487440+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":119,"log_dups_size":0,"ondisk_log_size":119,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:58:56.085777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032705499999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":523,"num_read_kb":3503,"num_write":136,"num_write_kb":3128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":44,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T20:42:43.480962+0000","last_change":"2023-09-23T20:40:19.557009+0000","last_active":"2023-09-23T20:42:43.480962+0000","last_peered":"2023-09-23T20:42:43.480962+0000","last_clean":"2023-09-23T20:42:43.480962+0000","last_became_active":"2023-09-23T20:40:19.550473+0000","last_became_peered":"2023-09-23T20:40:19.550473+0000","last_unstale":"2023-09-23T20:42:43.480962+0000","last_undegraded":"2023-09-23T20:42:43.480962+0000","last_fullsized":"2023-09-23T20:42:43.480962+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:35:01.140258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":431,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T20:42:43.487666+0000","last_change":"2023-09-23T20:40:19.530942+0000","last_active":"2023-09-23T20:42:43.487666+0000","last_peered":"2023-09-23T20:42:43.487666+0000","last_clean":"2023-09-23T20:42:43.487666+0000","last_became_active":"2023-09-23T20:40:19.530617+0000","last_became_peered":"2023-09-23T20:40:19.530617+0000","last_unstale":"2023-09-23T20:42:43.487666+0000","last_undegraded":"2023-09-23T20:42:43.487666+0000","last_fullsized":"2023-09-23T20:42:43.487666+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:24:09.231934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.9","version":"476'7422","reported_seq":7466,"reported_epoch":476,"state":"active+clean","last_fresh":"2023-09-23T20:42:47.381308+0000","last_change":"2023-09-23T20:41:59.507136+0000","last_active":"2023-09-23T20:42:47.381308+0000","last_peered":"2023-09-23T20:42:47.381308+0000","last_clean":"2023-09-23T20:42:47.381308+0000","last_became_active":"2023-09-23T20:41:59.477838+0000","last_became_peered":"2023-09-23T20:41:59.477838+0000","last_unstale":"2023-09-23T20:42:47.381308+0000","last_undegraded":"2023-09-23T20:42:47.381308+0000","last_fullsized":"2023-09-23T20:42:47.381308+0000","mapping_epoch":465,"log_start":"459'1634","ondisk_log_start":"459'1634","created":455,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":5788,"log_dups_size":0,"ondisk_log_size":5788,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:22:31.172548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":443678720,"num_objects":6770,"num_object_clones":0,"num_object_copies":20310,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6770,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7422,"num_write_kb":454144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":558,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T20:42:43.487352+0000","last_change":"2023-09-23T20:41:47.556551+0000","last_active":"2023-09-23T20:42:43.487352+0000","last_peered":"2023-09-23T20:42:43.487352+0000","last_clean":"2023-09-23T20:42:43.487352+0000","last_became_active":"2023-09-23T20:40:22.444703+0000","last_became_peered":"2023-09-23T20:40:22.444703+0000","last_unstale":"2023-09-23T20:42:43.487352+0000","last_undegraded":"2023-09-23T20:42:43.487352+0000","last_fullsized":"2023-09-23T20:42:43.487352+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:41:47.556463+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:41:47.556463+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:06:28.247430+0000","scrub_duration":0.0099000080000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"86'106","reported_seq":1194,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T20:42:43.480852+0000","last_change":"2023-09-23T20:38:37.476512+0000","last_active":"2023-09-23T20:42:43.480852+0000","last_peered":"2023-09-23T20:42:43.480852+0000","last_clean":"2023-09-23T20:42:43.480852+0000","last_became_active":"2023-09-23T20:38:37.475980+0000","last_became_peered":"2023-09-23T20:38:37.475980+0000","last_unstale":"2023-09-23T20:42:43.480852+0000","last_undegraded":"2023-09-23T20:42:43.480852+0000","last_fullsized":"2023-09-23T20:42:43.480852+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":106,"log_dups_size":0,"ondisk_log_size":106,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:58:45.508831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00099302599999999993,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1094,"num_read_kb":1678,"num_write":100,"num_write_kb":404,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":192,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T20:42:43.483210+0000","last_change":"2023-09-23T20:38:02.552008+0000","last_active":"2023-09-23T20:42:43.483210+0000","last_peered":"2023-09-23T20:42:43.483210+0000","last_clean":"2023-09-23T20:42:43.483210+0000","last_became_active":"2023-09-23T20:38:02.551684+0000","last_became_peered":"2023-09-23T20:38:02.551684+0000","last_unstale":"2023-09-23T20:42:43.483210+0000","last_undegraded":"2023-09-23T20:42:43.483210+0000","last_fullsized":"2023-09-23T20:42:43.483210+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:40:37.147538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":437,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T20:42:43.487365+0000","last_change":"2023-09-23T20:29:45.533406+0000","last_active":"2023-09-23T20:42:43.487365+0000","last_peered":"2023-09-23T20:42:43.487365+0000","last_clean":"2023-09-23T20:42:43.487365+0000","last_became_active":"2023-09-23T20:29:45.526001+0000","last_became_peered":"2023-09-23T20:29:45.526001+0000","last_unstale":"2023-09-23T20:42:43.487365+0000","last_undegraded":"2023-09-23T20:42:43.487365+0000","last_fullsized":"2023-09-23T20:42:43.487365+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:43.787251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.a","version":"475'7343","reported_seq":7371,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T20:42:45.528176+0000","last_change":"2023-09-23T20:41:04.556831+0000","last_active":"2023-09-23T20:42:45.528176+0000","last_peered":"2023-09-23T20:42:45.528176+0000","last_clean":"2023-09-23T20:42:45.528176+0000","last_became_active":"2023-09-23T20:41:04.556573+0000","last_became_peered":"2023-09-23T20:41:04.556573+0000","last_unstale":"2023-09-23T20:42:45.528176+0000","last_undegraded":"2023-09-23T20:42:45.528176+0000","last_fullsized":"2023-09-23T20:42:45.528176+0000","mapping_epoch":455,"log_start":"458'300","ondisk_log_start":"458'300","created":455,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":7043,"log_dups_size":0,"ondisk_log_size":7043,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:55:53.953463+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":449380352,"num_objects":6857,"num_object_clones":0,"num_object_copies":20571,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6857,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7343,"num_write_kb":454400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":836,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T20:42:44.521284+0000","last_change":"2023-09-23T20:42:44.521284+0000","last_active":"2023-09-23T20:42:44.521284+0000","last_peered":"2023-09-23T20:42:44.521284+0000","last_clean":"2023-09-23T20:42:44.521284+0000","last_became_active":"2023-09-23T20:42:44.520995+0000","last_became_peered":"2023-09-23T20:42:44.520995+0000","last_unstale":"2023-09-23T20:42:44.521284+0000","last_undegraded":"2023-09-23T20:42:44.521284+0000","last_fullsized":"2023-09-23T20:42:44.521284+0000","mapping_epoch":474,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":475,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:43.390908+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:43.390908+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:05:21.148891+0000","scrub_duration":0.0032721,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,2],"acting":[4,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1","version":"86'130","reported_seq":793,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T20:42:44.499007+0000","last_change":"2023-09-23T20:25:39.637458+0000","last_active":"2023-09-23T20:42:44.499007+0000","last_peered":"2023-09-23T20:42:44.499007+0000","last_clean":"2023-09-23T20:42:44.499007+0000","last_became_active":"2023-09-23T20:25:39.636773+0000","last_became_peered":"2023-09-23T20:25:39.636773+0000","last_unstale":"2023-09-23T20:42:44.499007+0000","last_undegraded":"2023-09-23T20:42:44.499007+0000","last_fullsized":"2023-09-23T20:42:44.499007+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:00:32.810564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011300049999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":512,"num_read_kb":1820,"num_write":133,"num_write_kb":1347,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":445,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T20:42:43.486890+0000","last_change":"2023-09-23T20:38:02.591335+0000","last_active":"2023-09-23T20:42:43.486890+0000","last_peered":"2023-09-23T20:42:43.486890+0000","last_clean":"2023-09-23T20:42:43.486890+0000","last_became_active":"2023-09-23T20:38:02.591015+0000","last_became_peered":"2023-09-23T20:38:02.591015+0000","last_unstale":"2023-09-23T20:42:43.486890+0000","last_undegraded":"2023-09-23T20:42:43.486890+0000","last_fullsized":"2023-09-23T20:42:43.486890+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:55:32.272195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":408,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T20:42:43.481197+0000","last_change":"2023-09-23T20:42:41.471865+0000","last_active":"2023-09-23T20:42:43.481197+0000","last_peered":"2023-09-23T20:42:43.481197+0000","last_clean":"2023-09-23T20:42:43.481197+0000","last_became_active":"2023-09-23T20:42:41.471542+0000","last_became_peered":"2023-09-23T20:42:41.471542+0000","last_unstale":"2023-09-23T20:42:43.481197+0000","last_undegraded":"2023-09-23T20:42:43.481197+0000","last_fullsized":"2023-09-23T20:42:43.481197+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:10:03.529951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.8","version":"476'7517","reported_seq":8772,"reported_epoch":476,"state":"active+remapped+backfilling","last_fresh":"2023-09-23T20:42:46.313155+0000","last_change":"2023-09-23T20:42:42.604651+0000","last_active":"2023-09-23T20:42:46.313155+0000","last_peered":"2023-09-23T20:42:46.313155+0000","last_clean":"2023-09-23T20:42:40.741538+0000","last_became_active":"2023-09-23T20:42:42.570860+0000","last_became_peered":"2023-09-23T20:42:42.570860+0000","last_unstale":"2023-09-23T20:42:46.313155+0000","last_undegraded":"2023-09-23T20:42:46.313155+0000","last_fullsized":"2023-09-23T20:42:46.313155+0000","mapping_epoch":472,"log_start":"459'1400","ondisk_log_start":"459'1400","created":455,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":6117,"log_dups_size":0,"ondisk_log_size":6117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:49:31.363949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":457637888,"num_objects":6983,"num_object_clones":0,"num_object_copies":20949,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":13160,"num_objects_unfound":0,"num_objects_dirty":6983,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7517,"num_write_kb":464000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":405,"num_bytes_recovered":26542080,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[0,3,5],"avail_no_missing":["0","3","5"],"object_location_counts":[{"shards":"0,3,5","objects":6982},{"shards":"1","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"86'194","reported_seq":827,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T20:42:43.487790+0000","last_change":"2023-09-23T20:40:51.458056+0000","last_active":"2023-09-23T20:42:43.487790+0000","last_peered":"2023-09-23T20:42:43.487790+0000","last_clean":"2023-09-23T20:42:43.487790+0000","last_became_active":"2023-09-23T20:40:51.457465+0000","last_became_peered":"2023-09-23T20:40:51.457465+0000","last_unstale":"2023-09-23T20:42:43.487790+0000","last_undegraded":"2023-09-23T20:42:43.487790+0000","last_fullsized":"2023-09-23T20:42:43.487790+0000","mapping_epoch":450,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":451,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":194,"log_dups_size":0,"ondisk_log_size":194,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:18:36.482553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00037995999999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1024,"num_read_kb":1200,"num_write":173,"num_write_kb":1276,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":938,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T20:42:44.505189+0000","last_change":"2023-09-23T20:38:05.602508+0000","last_active":"2023-09-23T20:42:44.505189+0000","last_peered":"2023-09-23T20:42:44.505189+0000","last_clean":"2023-09-23T20:42:44.505189+0000","last_became_active":"2023-09-23T20:38:05.602129+0000","last_became_peered":"2023-09-23T20:38:05.602129+0000","last_unstale":"2023-09-23T20:42:44.505189+0000","last_undegraded":"2023-09-23T20:42:44.505189+0000","last_fullsized":"2023-09-23T20:42:44.505189+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:45.343742+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:45.343742+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:48.910343+0000","scrub_duration":0.001475802,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":44,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T20:42:43.480688+0000","last_change":"2023-09-23T20:40:19.542142+0000","last_active":"2023-09-23T20:42:43.480688+0000","last_peered":"2023-09-23T20:42:43.480688+0000","last_clean":"2023-09-23T20:42:43.480688+0000","last_became_active":"2023-09-23T20:40:19.541870+0000","last_became_peered":"2023-09-23T20:40:19.541870+0000","last_unstale":"2023-09-23T20:42:43.480688+0000","last_undegraded":"2023-09-23T20:42:43.480688+0000","last_fullsized":"2023-09-23T20:42:43.480688+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:01.518149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":78,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T20:42:41.470594+0000","last_change":"2023-09-23T20:38:02.573200+0000","last_active":"2023-09-23T20:42:41.470594+0000","last_peered":"2023-09-23T20:42:41.470594+0000","last_clean":"2023-09-23T20:42:41.470594+0000","last_became_active":"2023-09-23T20:38:02.572945+0000","last_became_peered":"2023-09-23T20:38:02.572945+0000","last_unstale":"2023-09-23T20:42:41.470594+0000","last_undegraded":"2023-09-23T20:42:41.470594+0000","last_fullsized":"2023-09-23T20:42:41.470594+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:57.246533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.b","version":"473'7176","reported_seq":7212,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-09-23T20:42:43.139130+0000","last_change":"2023-09-23T20:41:59.493277+0000","last_active":"2023-09-23T20:42:43.139130+0000","last_peered":"2023-09-23T20:42:43.139130+0000","last_clean":"2023-09-23T20:42:43.139130+0000","last_became_active":"2023-09-23T20:41:59.462570+0000","last_became_peered":"2023-09-23T20:41:59.462570+0000","last_unstale":"2023-09-23T20:42:43.139130+0000","last_undegraded":"2023-09-23T20:42:43.139130+0000","last_fullsized":"2023-09-23T20:42:43.139130+0000","mapping_epoch":465,"log_start":"458'334","ondisk_log_start":"458'334","created":455,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":6842,"log_dups_size":0,"ondisk_log_size":6842,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:51:24.675595+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":452722688,"num_objects":6908,"num_object_clones":0,"num_object_copies":20724,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6908,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7176,"num_write_kb":450688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":806,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T20:42:43.489069+0000","last_change":"2023-09-23T20:38:05.615711+0000","last_active":"2023-09-23T20:42:43.489069+0000","last_peered":"2023-09-23T20:42:43.489069+0000","last_clean":"2023-09-23T20:42:43.489069+0000","last_became_active":"2023-09-23T20:38:05.615390+0000","last_became_peered":"2023-09-23T20:38:05.615390+0000","last_unstale":"2023-09-23T20:42:43.489069+0000","last_undegraded":"2023-09-23T20:42:43.489069+0000","last_fullsized":"2023-09-23T20:42:43.489069+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:43.582146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:43.582146+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:45:11.917871+0000","scrub_duration":0.0016219730000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"86'80","reported_seq":728,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T20:42:44.505005+0000","last_change":"2023-09-23T20:29:45.536796+0000","last_active":"2023-09-23T20:42:44.505005+0000","last_peered":"2023-09-23T20:42:44.505005+0000","last_clean":"2023-09-23T20:42:44.505005+0000","last_became_active":"2023-09-23T20:29:45.536439+0000","last_became_peered":"2023-09-23T20:29:45.536439+0000","last_unstale":"2023-09-23T20:42:44.505005+0000","last_undegraded":"2023-09-23T20:42:44.505005+0000","last_fullsized":"2023-09-23T20:42:44.505005+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":80,"log_dups_size":0,"ondisk_log_size":80,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:30:57.793132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035115,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":395,"num_read_kb":1370,"num_write":66,"num_write_kb":1061,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":369,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T20:42:43.482128+0000","last_change":"2023-09-23T20:41:59.461601+0000","last_active":"2023-09-23T20:42:43.482128+0000","last_peered":"2023-09-23T20:42:43.482128+0000","last_clean":"2023-09-23T20:42:43.482128+0000","last_became_active":"2023-09-23T20:41:59.461336+0000","last_became_peered":"2023-09-23T20:41:59.461336+0000","last_unstale":"2023-09-23T20:42:43.482128+0000","last_undegraded":"2023-09-23T20:42:43.482128+0000","last_fullsized":"2023-09-23T20:42:43.482128+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:15:23.259553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,3],"acting":[4,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":273,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T20:42:43.487533+0000","last_change":"2023-09-23T20:41:59.462980+0000","last_active":"2023-09-23T20:42:43.487533+0000","last_peered":"2023-09-23T20:42:43.487533+0000","last_clean":"2023-09-23T20:42:43.487533+0000","last_became_active":"2023-09-23T20:41:59.462696+0000","last_became_peered":"2023-09-23T20:41:59.462696+0000","last_unstale":"2023-09-23T20:42:43.487533+0000","last_undegraded":"2023-09-23T20:42:43.487533+0000","last_fullsized":"2023-09-23T20:42:43.487533+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:17:05.399919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.e","version":"476'7428","reported_seq":7457,"reported_epoch":476,"state":"active+clean","last_fresh":"2023-09-23T20:42:47.354789+0000","last_change":"2023-09-23T20:41:04.574876+0000","last_active":"2023-09-23T20:42:47.354789+0000","last_peered":"2023-09-23T20:42:47.354789+0000","last_clean":"2023-09-23T20:42:47.354789+0000","last_became_active":"2023-09-23T20:41:04.574595+0000","last_became_peered":"2023-09-23T20:41:04.574595+0000","last_unstale":"2023-09-23T20:42:47.354789+0000","last_undegraded":"2023-09-23T20:42:47.354789+0000","last_fullsized":"2023-09-23T20:42:47.354789+0000","mapping_epoch":455,"log_start":"459'1741","ondisk_log_start":"459'1741","created":455,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":5687,"log_dups_size":0,"ondisk_log_size":5687,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:31:00.147966+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":445120512,"num_objects":6792,"num_object_clones":0,"num_object_copies":20376,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6792,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7428,"num_write_kb":455040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":584,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T20:42:41.470718+0000","last_change":"2023-09-23T20:41:08.380084+0000","last_active":"2023-09-23T20:42:41.470718+0000","last_peered":"2023-09-23T20:42:41.470718+0000","last_clean":"2023-09-23T20:42:41.470718+0000","last_became_active":"2023-09-23T20:38:05.610881+0000","last_became_peered":"2023-09-23T20:38:05.610881+0000","last_unstale":"2023-09-23T20:42:41.470718+0000","last_undegraded":"2023-09-23T20:42:41.470718+0000","last_fullsized":"2023-09-23T20:42:41.470718+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:41:08.379959+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:41:08.379959+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:19:39.547923+0000","scrub_duration":0.0019158420000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"86'102","reported_seq":1070,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T20:42:44.505910+0000","last_change":"2023-09-23T20:32:36.314672+0000","last_active":"2023-09-23T20:42:44.505910+0000","last_peered":"2023-09-23T20:42:44.505910+0000","last_clean":"2023-09-23T20:42:44.505910+0000","last_became_active":"2023-09-23T20:32:36.314231+0000","last_became_peered":"2023-09-23T20:32:36.314231+0000","last_unstale":"2023-09-23T20:42:44.505910+0000","last_undegraded":"2023-09-23T20:42:44.505910+0000","last_fullsized":"2023-09-23T20:42:44.505910+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":102,"log_dups_size":0,"ondisk_log_size":102,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:11:40.514709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030656500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":963,"num_read_kb":1234,"num_write":60,"num_write_kb":225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":421,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T20:42:43.487513+0000","last_change":"2023-09-23T20:40:19.544384+0000","last_active":"2023-09-23T20:42:43.487513+0000","last_peered":"2023-09-23T20:42:43.487513+0000","last_clean":"2023-09-23T20:42:43.487513+0000","last_became_active":"2023-09-23T20:40:19.544079+0000","last_became_peered":"2023-09-23T20:40:19.544079+0000","last_unstale":"2023-09-23T20:42:43.487513+0000","last_undegraded":"2023-09-23T20:42:43.487513+0000","last_fullsized":"2023-09-23T20:42:43.487513+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:00:19.498631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":408,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T20:42:43.487516+0000","last_change":"2023-09-23T20:41:59.466175+0000","last_active":"2023-09-23T20:42:43.487516+0000","last_peered":"2023-09-23T20:42:43.487516+0000","last_clean":"2023-09-23T20:42:43.487516+0000","last_became_active":"2023-09-23T20:41:59.465861+0000","last_became_peered":"2023-09-23T20:41:59.465861+0000","last_unstale":"2023-09-23T20:42:43.487516+0000","last_undegraded":"2023-09-23T20:42:43.487516+0000","last_fullsized":"2023-09-23T20:42:43.487516+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:59:17.054848+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.d","version":"476'7475","reported_seq":8919,"reported_epoch":476,"state":"active+remapped+backfilling","last_fresh":"2023-09-23T20:42:47.432358+0000","last_change":"2023-09-23T20:42:44.549147+0000","last_active":"2023-09-23T20:42:47.432358+0000","last_peered":"2023-09-23T20:42:47.432358+0000","last_clean":"2023-09-23T20:42:40.766572+0000","last_became_active":"2023-09-23T20:42:42.556044+0000","last_became_peered":"2023-09-23T20:42:42.556044+0000","last_unstale":"2023-09-23T20:42:47.432358+0000","last_undegraded":"2023-09-23T20:42:47.432358+0000","last_fullsized":"2023-09-23T20:42:47.432358+0000","mapping_epoch":472,"log_start":"459'1779","ondisk_log_start":"459'1779","created":455,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":5696,"log_dups_size":0,"ondisk_log_size":5696,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:28:08.995525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":446300184,"num_objects":6811,"num_object_clones":0,"num_object_copies":20433,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":6127,"num_objects_unfound":0,"num_objects_dirty":6811,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7475,"num_write_kb":457089,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":699,"num_bytes_recovered":45809664,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,1],"avail_no_missing":["7","1","5"],"object_location_counts":[{"shards":"1,5,7","objects":6811}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":753,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T20:42:43.485410+0000","last_change":"2023-09-23T20:38:05.590361+0000","last_active":"2023-09-23T20:42:43.485410+0000","last_peered":"2023-09-23T20:42:43.485410+0000","last_clean":"2023-09-23T20:42:43.485410+0000","last_became_active":"2023-09-23T20:38:05.589985+0000","last_became_peered":"2023-09-23T20:38:05.589985+0000","last_unstale":"2023-09-23T20:42:43.485410+0000","last_undegraded":"2023-09-23T20:42:43.485410+0000","last_fullsized":"2023-09-23T20:42:43.485410+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:03.246271+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:03.246271+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:38:54.351576+0000","scrub_duration":0.0019016700000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":213,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T20:42:44.513130+0000","last_change":"2023-09-23T20:32:50.549695+0000","last_active":"2023-09-23T20:42:44.513130+0000","last_peered":"2023-09-23T20:42:44.513130+0000","last_clean":"2023-09-23T20:42:44.513130+0000","last_became_active":"2023-09-23T20:32:50.549431+0000","last_became_peered":"2023-09-23T20:32:50.549431+0000","last_unstale":"2023-09-23T20:42:44.513130+0000","last_undegraded":"2023-09-23T20:42:44.513130+0000","last_fullsized":"2023-09-23T20:42:44.513130+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:45:53.666569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"86'128","reported_seq":886,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T20:42:43.487399+0000","last_change":"2023-09-23T20:38:03.541765+0000","last_active":"2023-09-23T20:42:43.487399+0000","last_peered":"2023-09-23T20:42:43.487399+0000","last_clean":"2023-09-23T20:42:43.487399+0000","last_became_active":"2023-09-23T20:38:03.541277+0000","last_became_peered":"2023-09-23T20:38:03.541277+0000","last_unstale":"2023-09-23T20:42:43.487399+0000","last_undegraded":"2023-09-23T20:42:43.487399+0000","last_fullsized":"2023-09-23T20:42:43.487399+0000","mapping_epoch":404,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":405,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:20:53.834554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000293202,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":579,"num_read_kb":1473,"num_write":96,"num_write_kb":1062,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":253,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-09-23T20:42:42.472819+0000","last_change":"2023-09-23T20:37:43.582105+0000","last_active":"2023-09-23T20:42:42.472819+0000","last_peered":"2023-09-23T20:42:42.472819+0000","last_clean":"2023-09-23T20:42:42.472819+0000","last_became_active":"2023-09-23T20:37:43.581860+0000","last_became_peered":"2023-09-23T20:37:43.581860+0000","last_unstale":"2023-09-23T20:42:42.472819+0000","last_undegraded":"2023-09-23T20:42:42.472819+0000","last_fullsized":"2023-09-23T20:42:42.472819+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:18:38.765486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.c","version":"474'7310","reported_seq":7338,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T20:42:44.030349+0000","last_change":"2023-09-23T20:41:04.575496+0000","last_active":"2023-09-23T20:42:44.030349+0000","last_peered":"2023-09-23T20:42:44.030349+0000","last_clean":"2023-09-23T20:42:44.030349+0000","last_became_active":"2023-09-23T20:41:04.575242+0000","last_became_peered":"2023-09-23T20:41:04.575242+0000","last_unstale":"2023-09-23T20:42:44.030349+0000","last_undegraded":"2023-09-23T20:42:44.030349+0000","last_fullsized":"2023-09-23T20:42:44.030349+0000","mapping_epoch":455,"log_start":"459'1675","ondisk_log_start":"459'1675","created":455,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":5635,"log_dups_size":0,"ondisk_log_size":5635,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:30:14.611529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":454950912,"num_objects":6942,"num_object_clones":0,"num_object_copies":20826,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6942,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7310,"num_write_kb":456064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"90'161","reported_seq":687,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-09-23T20:42:42.471635+0000","last_change":"2023-09-23T20:40:22.427435+0000","last_active":"2023-09-23T20:42:42.471635+0000","last_peered":"2023-09-23T20:42:42.471635+0000","last_clean":"2023-09-23T20:42:42.471635+0000","last_became_active":"2023-09-23T20:40:22.427051+0000","last_became_peered":"2023-09-23T20:40:22.427051+0000","last_unstale":"2023-09-23T20:42:42.471635+0000","last_undegraded":"2023-09-23T20:42:42.471635+0000","last_fullsized":"2023-09-23T20:42:42.471635+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:34:54.504554+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:34:54.504554+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:42.001652+0000","scrub_duration":0.011176864,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"86'117","reported_seq":733,"reported_epoch":473,"state":"active+undersized+remapped","last_fresh":"2023-09-23T20:42:42.515914+0000","last_change":"2023-09-23T20:42:42.516100+0000","last_active":"2023-09-23T20:42:42.515914+0000","last_peered":"2023-09-23T20:42:42.515914+0000","last_clean":"2023-09-23T20:42:37.726242+0000","last_became_active":"2023-09-23T20:42:42.477174+0000","last_became_peered":"2023-09-23T20:42:42.477174+0000","last_unstale":"2023-09-23T20:42:42.515914+0000","last_undegraded":"2023-09-23T20:42:42.515914+0000","last_fullsized":"2023-09-23T20:42:42.469313+0000","mapping_epoch":472,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":412,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:07.316838+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046891400000000002,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":893,"num_read_kb":922,"num_write":82,"num_write_kb":4193,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":12582912,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2],"avail_no_missing":["6","2","3","7"],"object_location_counts":[{"shards":"2,6","objects":1}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":44,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T20:42:43.487250+0000","last_change":"2023-09-23T20:41:59.459879+0000","last_active":"2023-09-23T20:42:43.487250+0000","last_peered":"2023-09-23T20:42:43.487250+0000","last_clean":"2023-09-23T20:42:43.487250+0000","last_became_active":"2023-09-23T20:41:59.459519+0000","last_became_peered":"2023-09-23T20:41:59.459519+0000","last_unstale":"2023-09-23T20:42:43.487250+0000","last_undegraded":"2023-09-23T20:42:43.487250+0000","last_fullsized":"2023-09-23T20:42:43.487250+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:26:35.636792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":212,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T20:42:43.482378+0000","last_change":"2023-09-23T20:40:19.530602+0000","last_active":"2023-09-23T20:42:43.482378+0000","last_peered":"2023-09-23T20:42:43.482378+0000","last_clean":"2023-09-23T20:42:43.482378+0000","last_became_active":"2023-09-23T20:40:19.530256+0000","last_became_peered":"2023-09-23T20:40:19.530256+0000","last_unstale":"2023-09-23T20:42:43.482378+0000","last_undegraded":"2023-09-23T20:42:43.482378+0000","last_fullsized":"2023-09-23T20:42:43.482378+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:18:45.658867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"90'161","reported_seq":677,"reported_epoch":472,"state":"active+clean","last_fresh":"2023-09-23T20:42:41.470035+0000","last_change":"2023-09-23T20:41:59.463315+0000","last_active":"2023-09-23T20:42:41.470035+0000","last_peered":"2023-09-23T20:42:41.470035+0000","last_clean":"2023-09-23T20:42:41.470035+0000","last_became_active":"2023-09-23T20:41:59.463011+0000","last_became_peered":"2023-09-23T20:41:59.463011+0000","last_unstale":"2023-09-23T20:42:41.470035+0000","last_undegraded":"2023-09-23T20:42:41.470035+0000","last_fullsized":"2023-09-23T20:42:41.470035+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:06.487351+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:06.487351+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:29:37.260966+0000","scrub_duration":0.001964379,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":289,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T20:42:43.483029+0000","last_change":"2023-09-23T20:41:59.466836+0000","last_active":"2023-09-23T20:42:43.483029+0000","last_peered":"2023-09-23T20:42:43.483029+0000","last_clean":"2023-09-23T20:42:43.483029+0000","last_became_active":"2023-09-23T20:41:59.466479+0000","last_became_peered":"2023-09-23T20:41:59.466479+0000","last_unstale":"2023-09-23T20:42:43.483029+0000","last_undegraded":"2023-09-23T20:42:43.483029+0000","last_fullsized":"2023-09-23T20:42:43.483029+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:46:22.157025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":331,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T20:42:44.499594+0000","last_change":"2023-09-23T20:29:45.538314+0000","last_active":"2023-09-23T20:42:44.499594+0000","last_peered":"2023-09-23T20:42:44.499594+0000","last_clean":"2023-09-23T20:42:44.499594+0000","last_became_active":"2023-09-23T20:29:45.536847+0000","last_became_peered":"2023-09-23T20:29:45.536847+0000","last_unstale":"2023-09-23T20:42:44.499594+0000","last_undegraded":"2023-09-23T20:42:44.499594+0000","last_fullsized":"2023-09-23T20:42:44.499594+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:51.948721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"90'161","reported_seq":546,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-09-23T20:42:42.470695+0000","last_change":"2023-09-23T20:41:06.499511+0000","last_active":"2023-09-23T20:42:42.470695+0000","last_peered":"2023-09-23T20:42:42.470695+0000","last_clean":"2023-09-23T20:42:42.470695+0000","last_became_active":"2023-09-23T20:38:05.601874+0000","last_became_peered":"2023-09-23T20:38:05.601874+0000","last_unstale":"2023-09-23T20:42:42.470695+0000","last_undegraded":"2023-09-23T20:42:42.470695+0000","last_fullsized":"2023-09-23T20:42:42.470695+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:41:06.499396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:41:06.499396+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:31:34.059077+0000","scrub_duration":0.0019670310000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":73,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T20:42:43.492828+0000","last_change":"2023-09-23T20:38:37.481551+0000","last_active":"2023-09-23T20:42:43.492828+0000","last_peered":"2023-09-23T20:42:43.492828+0000","last_clean":"2023-09-23T20:42:43.492828+0000","last_became_active":"2023-09-23T20:38:37.458687+0000","last_became_peered":"2023-09-23T20:38:37.458687+0000","last_unstale":"2023-09-23T20:42:43.492828+0000","last_undegraded":"2023-09-23T20:42:43.492828+0000","last_fullsized":"2023-09-23T20:42:43.492828+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:22:15.033243+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":131,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T20:42:43.482886+0000","last_change":"2023-09-23T20:34:59.987575+0000","last_active":"2023-09-23T20:42:43.482886+0000","last_peered":"2023-09-23T20:42:43.482886+0000","last_clean":"2023-09-23T20:42:43.482886+0000","last_became_active":"2023-09-23T20:34:59.987229+0000","last_became_peered":"2023-09-23T20:34:59.987229+0000","last_unstale":"2023-09-23T20:42:43.482886+0000","last_undegraded":"2023-09-23T20:42:43.482886+0000","last_fullsized":"2023-09-23T20:42:43.482886+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:34:01.820159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":73,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T20:42:43.481118+0000","last_change":"2023-09-23T20:38:37.457634+0000","last_active":"2023-09-23T20:42:43.481118+0000","last_peered":"2023-09-23T20:42:43.481118+0000","last_clean":"2023-09-23T20:42:43.481118+0000","last_became_active":"2023-09-23T20:38:37.452208+0000","last_became_peered":"2023-09-23T20:38:37.452208+0000","last_unstale":"2023-09-23T20:42:43.481118+0000","last_undegraded":"2023-09-23T20:42:43.481118+0000","last_fullsized":"2023-09-23T20:42:43.481118+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:09:41.754569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":131,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T20:42:43.483425+0000","last_change":"2023-09-23T20:34:59.987416+0000","last_active":"2023-09-23T20:42:43.483425+0000","last_peered":"2023-09-23T20:42:43.483425+0000","last_clean":"2023-09-23T20:42:43.483425+0000","last_became_active":"2023-09-23T20:34:59.987117+0000","last_became_peered":"2023-09-23T20:34:59.987117+0000","last_unstale":"2023-09-23T20:42:43.483425+0000","last_undegraded":"2023-09-23T20:42:43.483425+0000","last_fullsized":"2023-09-23T20:42:43.483425+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:36:16.318623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"90'161","reported_seq":731,"reported_epoch":475,"state":"active+clean","last_fresh":"2023-09-23T20:42:44.506842+0000","last_change":"2023-09-23T20:38:05.591992+0000","last_active":"2023-09-23T20:42:44.506842+0000","last_peered":"2023-09-23T20:42:44.506842+0000","last_clean":"2023-09-23T20:42:44.506842+0000","last_became_active":"2023-09-23T20:38:05.591701+0000","last_became_peered":"2023-09-23T20:38:05.591701+0000","last_unstale":"2023-09-23T20:42:44.506842+0000","last_undegraded":"2023-09-23T20:42:44.506842+0000","last_fullsized":"2023-09-23T20:42:44.506842+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:02.384133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:02.384133+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:41.722542+0000","scrub_duration":0.001769098,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":295,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-09-23T20:42:42.470961+0000","last_change":"2023-09-23T20:38:02.542719+0000","last_active":"2023-09-23T20:42:42.470961+0000","last_peered":"2023-09-23T20:42:42.470961+0000","last_clean":"2023-09-23T20:42:42.470961+0000","last_became_active":"2023-09-23T20:38:02.542385+0000","last_became_peered":"2023-09-23T20:38:02.542385+0000","last_unstale":"2023-09-23T20:42:42.470961+0000","last_undegraded":"2023-09-23T20:42:42.470961+0000","last_fullsized":"2023-09-23T20:42:42.470961+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:36:23.673757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":262,"reported_epoch":473,"state":"active+clean","last_fresh":"2023-09-23T20:42:42.471303+0000","last_change":"2023-09-23T20:37:43.584253+0000","last_active":"2023-09-23T20:42:42.471303+0000","last_peered":"2023-09-23T20:42:42.471303+0000","last_clean":"2023-09-23T20:42:42.471303+0000","last_became_active":"2023-09-23T20:37:43.580069+0000","last_became_peered":"2023-09-23T20:37:43.580069+0000","last_unstale":"2023-09-23T20:42:42.471303+0000","last_undegraded":"2023-09-23T20:42:42.471303+0000","last_fullsized":"2023-09-23T20:42:42.471303+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:39:11.482581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"90'161","reported_seq":567,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T20:42:43.488005+0000","last_change":"2023-09-23T20:36:01.505098+0000","last_active":"2023-09-23T20:42:43.488005+0000","last_peered":"2023-09-23T20:42:43.488005+0000","last_clean":"2023-09-23T20:42:43.488005+0000","last_became_active":"2023-09-23T20:35:26.644707+0000","last_became_peered":"2023-09-23T20:35:26.644707+0000","last_unstale":"2023-09-23T20:42:43.488005+0000","last_undegraded":"2023-09-23T20:42:43.488005+0000","last_fullsized":"2023-09-23T20:42:43.488005+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:01.504973+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:01.504973+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:02:10.880809+0000","scrub_duration":0.00202344,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":492,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T20:42:43.481121+0000","last_change":"2023-09-23T20:41:59.463600+0000","last_active":"2023-09-23T20:42:43.481121+0000","last_peered":"2023-09-23T20:42:43.481121+0000","last_clean":"2023-09-23T20:42:43.481121+0000","last_became_active":"2023-09-23T20:41:59.463237+0000","last_became_peered":"2023-09-23T20:41:59.463237+0000","last_unstale":"2023-09-23T20:42:43.481121+0000","last_undegraded":"2023-09-23T20:42:43.481121+0000","last_fullsized":"2023-09-23T20:42:43.481121+0000","mapping_epoch":465,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":466,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:18:27.896483+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2770,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":192,"reported_epoch":474,"state":"active+clean","last_fresh":"2023-09-23T20:42:43.482021+0000","last_change":"2023-09-23T20:38:02.538333+0000","last_active":"2023-09-23T20:42:43.482021+0000","last_peered":"2023-09-23T20:42:43.482021+0000","last_clean":"2023-09-23T20:42:43.482021+0000","last_became_active":"2023-09-23T20:38:02.538008+0000","last_became_peered":"2023-09-23T20:38:02.538008+0000","last_unstale":"2023-09-23T20:42:43.482021+0000","last_undegraded":"2023-09-23T20:42:43.482021+0000","last_fullsized":"2023-09-23T20:42:43.482021+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:02:38.789530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":80,"num_bytes_recovered":17932,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26691,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":7209680920,"num_objects":110012,"num_object_clones":0,"num_object_copies":330036,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":26067,"num_objects_unfound":0,"num_objects_dirty":110012,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":117652,"num_write_kb":7285185,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1105,"num_bytes_recovered":72417280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":21696294912,"data_stored":21696282696,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":100976,"ondisk_log_size":100976,"up":48,"acting":48,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5983,"num_read_kb":13200,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":76,"num_bytes_recovered":12582912,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6082,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":23,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201656,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4395492,"kb_used_data":3946076,"kb_used_omap":0,"kb_used_meta":449408,"kb_avail":89349660,"statfs":{"total":95995035648,"available":91494051840,"internally_reserved":0,"allocated":4040781824,"data_stored":4039308324,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":460193792},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:41:56 2023","interfaces":[{"interface":"back","average":{"1min":1.2430000000000001,"5min":1.829,"15min":2.1110000000000002},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.20799999999999999},"max":{"1min":5.2309999999999999,"5min":18.315000000000001,"15min":48.798999999999999},"last":6.2869999999999999},{"interface":"front","average":{"1min":1.55,"5min":1.3540000000000001,"15min":1.841},"min":{"1min":0.28999999999999998,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":6.5720000000000001,"5min":9.4429999999999996,"15min":48.247},"last":0.57699999999999996}]},{"osd":1,"last update":"Sat Sep 23 20:42:06 2023","interfaces":[{"interface":"back","average":{"1min":0.82999999999999996,"5min":1.611,"15min":2.0339999999999998},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":5.1689999999999996,"5min":29.254999999999999,"15min":49.965000000000003},"last":2.137},{"interface":"front","average":{"1min":2.1139999999999999,"5min":1.8400000000000001,"15min":2.1240000000000001},"min":{"1min":0.27100000000000002,"5min":0.26900000000000002,"15min":0.248},"max":{"1min":6.3029999999999999,"5min":16.882000000000001,"15min":53.451999999999998},"last":0.32300000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:41:56 2023","interfaces":[{"interface":"back","average":{"1min":1.71,"5min":1.754,"15min":1.9810000000000001},"min":{"1min":0.33800000000000002,"5min":0.28599999999999998,"15min":0.27600000000000002},"max":{"1min":6.1529999999999996,"5min":16.965,"15min":16.965},"last":0.35699999999999998},{"interface":"front","average":{"1min":1.466,"5min":2.2109999999999999,"15min":2.3340000000000001},"min":{"1min":0.35499999999999998,"5min":0.30299999999999999,"15min":0.26600000000000001},"max":{"1min":8.1739999999999995,"5min":23.414000000000001,"15min":23.494},"last":1.1879999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:42:25 2023","interfaces":[{"interface":"back","average":{"1min":2.0169999999999999,"5min":2.0339999999999998,"15min":2.2799999999999998},"min":{"1min":0.41199999999999998,"5min":0.311,"15min":0.26200000000000001},"max":{"1min":8.3849999999999998,"5min":23.352,"15min":23.352},"last":5.694},{"interface":"front","average":{"1min":2.3439999999999999,"5min":2.1579999999999999,"15min":2.2989999999999999},"min":{"1min":0.439,"5min":0.439,"15min":0.33100000000000002},"max":{"1min":8.782,"5min":20.969999999999999,"15min":20.969999999999999},"last":5.7880000000000003}]},{"osd":4,"last update":"Sat Sep 23 20:42:25 2023","interfaces":[{"interface":"back","average":{"1min":1.202,"5min":1.125,"15min":1.1439999999999999},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.23200000000000001},"max":{"1min":4.8209999999999997,"5min":18.364999999999998,"15min":18.364999999999998},"last":7.2779999999999996},{"interface":"front","average":{"1min":1.345,"5min":1.3300000000000001,"15min":1.2150000000000001},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.27600000000000002},"max":{"1min":6.7999999999999998,"5min":23.617000000000001,"15min":23.617000000000001},"last":5.9870000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:42:25 2023","interfaces":[{"interface":"back","average":{"1min":1.4550000000000001,"5min":1.0920000000000001,"15min":1.0620000000000001},"min":{"1min":0.30499999999999999,"5min":0.29499999999999998,"15min":0.26600000000000001},"max":{"1min":8.2189999999999994,"5min":12.516,"15min":13.241},"last":1.226},{"interface":"front","average":{"1min":1.1759999999999999,"5min":1.1439999999999999,"15min":1.1839999999999999},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.315},"max":{"1min":5.2610000000000001,"5min":18.390000000000001,"15min":18.831},"last":1.4179999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:42:25 2023","interfaces":[{"interface":"back","average":{"1min":1.538,"5min":1.379,"15min":1.1950000000000001},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.30399999999999999},"max":{"1min":5.3840000000000003,"5min":18.417000000000002,"15min":18.417000000000002},"last":0.501},{"interface":"front","average":{"1min":1.349,"5min":1.1839999999999999,"15min":1.097},"min":{"1min":0.372,"5min":0.32700000000000001,"15min":0.318},"max":{"1min":7.5439999999999996,"5min":7.5439999999999996,"15min":8.0069999999999997},"last":0.51800000000000002}]}]},{"osd":6,"up_from":47,"seq":201863463294,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1763504,"kb_used_data":1351400,"kb_used_omap":0,"kb_used_meta":412096,"kb_avail":91981648,"statfs":{"total":95995035648,"available":94189207552,"internally_reserved":0,"allocated":1383833600,"data_stored":1382388050,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":421986304},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:41:51 2023","interfaces":[{"interface":"back","average":{"1min":1.1279999999999999,"5min":1.6839999999999999,"15min":1.8759999999999999},"min":{"1min":0.30199999999999999,"5min":0.23300000000000001,"15min":0.22800000000000001},"max":{"1min":6.3819999999999997,"5min":17.492999999999999,"15min":53.701999999999998},"last":0.57199999999999995},{"interface":"front","average":{"1min":1.216,"5min":2.2650000000000001,"15min":2.3340000000000001},"min":{"1min":0.36399999999999999,"5min":0.23999999999999999,"15min":0.22700000000000001},"max":{"1min":3.8700000000000001,"5min":60.328000000000003,"15min":60.328000000000003},"last":0.48499999999999999}]},{"osd":1,"last update":"Sat Sep 23 20:42:09 2023","interfaces":[{"interface":"back","average":{"1min":1.6519999999999999,"5min":1.794,"15min":2.1480000000000001},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":8.9770000000000003,"5min":64.274000000000001,"15min":64.274000000000001},"last":0.86799999999999999},{"interface":"front","average":{"1min":1.6040000000000001,"5min":1.345,"15min":2.2029999999999998},"min":{"1min":0.378,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":4.1950000000000003,"5min":6.8460000000000001,"15min":34.490000000000002},"last":0.60599999999999998}]},{"osd":2,"last update":"Sat Sep 23 20:41:51 2023","interfaces":[{"interface":"back","average":{"1min":1.5189999999999999,"5min":1.6499999999999999,"15min":2.0899999999999999},"min":{"1min":0.35199999999999998,"5min":0.30299999999999999,"15min":0.27700000000000002},"max":{"1min":4.6580000000000004,"5min":13.010999999999999,"15min":34.317999999999998},"last":0.81699999999999995},{"interface":"front","average":{"1min":1.4279999999999999,"5min":1.595,"15min":2.0139999999999998},"min":{"1min":0.34000000000000002,"5min":0.311,"15min":0.30399999999999999},"max":{"1min":4.6769999999999996,"5min":12.917999999999999,"15min":34.292999999999999},"last":0.66700000000000004}]},{"osd":3,"last update":"Sat Sep 23 20:42:41 2023","interfaces":[{"interface":"back","average":{"1min":2.605,"5min":1.8779999999999999,"15min":1.8740000000000001},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":6.6210000000000004,"5min":8.7799999999999994,"15min":8.9670000000000005},"last":0.68200000000000005},{"interface":"front","average":{"1min":2.7599999999999998,"5min":2.036,"15min":2.04},"min":{"1min":0.34899999999999998,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":11.808,"5min":11.913,"15min":12.875999999999999},"last":0.63300000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:42:14 2023","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.85899999999999999,"15min":0.77200000000000002},"min":{"1min":0.29699999999999999,"5min":0.28000000000000003,"15min":0.251},"max":{"1min":1.2789999999999999,"5min":23.904,"15min":23.904},"last":0.5},{"interface":"front","average":{"1min":1.0089999999999999,"5min":1.0509999999999999,"15min":1.0509999999999999},"min":{"1min":0.33200000000000002,"5min":0.29199999999999998,"15min":0.27000000000000002},"max":{"1min":2.9380000000000002,"5min":11.515000000000001,"15min":11.515000000000001},"last":0.44500000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:42:14 2023","interfaces":[{"interface":"back","average":{"1min":0.89100000000000001,"5min":0.78000000000000003,"15min":0.78800000000000003},"min":{"1min":0.35699999999999998,"5min":0.32200000000000001,"15min":0.28699999999999998},"max":{"1min":2.9340000000000002,"5min":6.6340000000000003,"15min":8.4499999999999993},"last":0.47199999999999998},{"interface":"front","average":{"1min":1.4530000000000001,"5min":1.0309999999999999,"15min":1.087},"min":{"1min":0.32400000000000001,"5min":0.308,"15min":0.30599999999999999},"max":{"1min":4.7539999999999996,"5min":6.6909999999999998,"15min":9.9299999999999997},"last":0.79300000000000004}]},{"osd":7,"last update":"Sat Sep 23 20:42:41 2023","interfaces":[{"interface":"back","average":{"1min":0.67000000000000004,"5min":0.755,"15min":0.87},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.28599999999999998},"max":{"1min":3.0299999999999998,"5min":5.2729999999999997,"15min":8.5060000000000002},"last":0.54200000000000004},{"interface":"front","average":{"1min":1.859,"5min":1.0600000000000001,"15min":1.1519999999999999},"min":{"1min":0.35699999999999998,"5min":0.34699999999999998,"15min":0.30299999999999999},"max":{"1min":7.1929999999999996,"5min":7.1929999999999996,"15min":8.8119999999999994},"last":0.73999999999999999}]}]},{"osd":1,"up_from":266,"seq":1142461300893,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4544236,"kb_used_data":4013604,"kb_used_omap":8,"kb_used_meta":530551,"kb_avail":89200916,"statfs":{"total":95995035648,"available":91341737984,"internally_reserved":0,"allocated":4109930496,"data_stored":4107397982,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8206,"internal_metadata":543285234},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:42:10 2023","interfaces":[{"interface":"back","average":{"1min":1.6519999999999999,"5min":1.889,"15min":2.452},"min":{"1min":0.20000000000000001,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":5.8200000000000003,"5min":16.292000000000002,"15min":52.689999999999998},"last":0.69799999999999995},{"interface":"front","average":{"1min":1.4610000000000001,"5min":2.5329999999999999,"15min":3.0790000000000002},"min":{"1min":0.221,"5min":0.20100000000000001,"15min":0.19900000000000001},"max":{"1min":6.1719999999999997,"5min":17.169,"15min":52.662999999999997},"last":0.28499999999999998}]},{"osd":2,"last update":"Sat Sep 23 20:42:10 2023","interfaces":[{"interface":"back","average":{"1min":1.7150000000000001,"5min":2.4980000000000002,"15min":3.633},"min":{"1min":0.25,"5min":0.216,"15min":0.20899999999999999},"max":{"1min":4.6420000000000003,"5min":32.255000000000003,"15min":32.255000000000003},"last":1.099},{"interface":"front","average":{"1min":1.1200000000000001,"5min":2.0030000000000001,"15min":2.2189999999999999},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":5.8949999999999996,"5min":13.404999999999999,"15min":13.404999999999999},"last":0.58199999999999996}]},{"osd":3,"last update":"Sat Sep 23 20:42:10 2023","interfaces":[{"interface":"back","average":{"1min":1.651,"5min":1.8859999999999999,"15min":3.2570000000000001},"min":{"1min":0.29099999999999998,"5min":0.247,"15min":0.247},"max":{"1min":6.3280000000000003,"5min":16.245999999999999,"15min":80.745999999999995},"last":0.85699999999999998},{"interface":"front","average":{"1min":1.944,"5min":2.3069999999999999,"15min":3.5720000000000001},"min":{"1min":0.28199999999999997,"5min":0.27500000000000002,"15min":0.22700000000000001},"max":{"1min":6.2030000000000003,"5min":14.048999999999999,"15min":82.988},"last":0.53100000000000003}]},{"osd":4,"last update":"Sat Sep 23 20:42:10 2023","interfaces":[{"interface":"back","average":{"1min":1.581,"5min":1.698,"15min":2.101},"min":{"1min":0.34899999999999998,"5min":0.30599999999999999,"15min":0.30299999999999999},"max":{"1min":4.3040000000000003,"5min":7.8399999999999999,"15min":24.327999999999999},"last":1.1719999999999999},{"interface":"front","average":{"1min":1.6879999999999999,"5min":1.498,"15min":1.6819999999999999},"min":{"1min":0.34200000000000003,"5min":0.32600000000000001,"15min":0.311},"max":{"1min":5.8639999999999999,"5min":9.3339999999999996,"15min":11.574999999999999},"last":0.62}]},{"osd":5,"last update":"Sat Sep 23 20:42:10 2023","interfaces":[{"interface":"back","average":{"1min":2.125,"5min":1.913,"15min":2.2749999999999999},"min":{"1min":0.42499999999999999,"5min":0.36699999999999999,"15min":0.33200000000000002},"max":{"1min":7.9770000000000003,"5min":13.550000000000001,"15min":33.401000000000003},"last":0.44900000000000001},{"interface":"front","average":{"1min":2.0139999999999998,"5min":1.9419999999999999,"15min":2.3500000000000001},"min":{"1min":0.42099999999999999,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":6.657,"5min":7.9340000000000002,"15min":33.417000000000002},"last":1.2150000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:42:10 2023","interfaces":[{"interface":"back","average":{"1min":1.8280000000000001,"5min":2.0390000000000001,"15min":3.4340000000000002},"min":{"1min":0.441,"5min":0.42599999999999999,"15min":0.35699999999999998},"max":{"1min":6.3810000000000002,"5min":13.593999999999999,"15min":72.682000000000002},"last":0.49399999999999999},{"interface":"front","average":{"1min":2.1059999999999999,"5min":1.829,"15min":3.399},"min":{"1min":0.40000000000000002,"5min":0.317,"15min":0.317},"max":{"1min":6.3860000000000001,"5min":7.883,"15min":72.563999999999993},"last":1.254}]},{"osd":7,"last update":"Sat Sep 23 20:42:10 2023","interfaces":[{"interface":"back","average":{"1min":2.0539999999999998,"5min":2.0579999999999998,"15min":2.4220000000000002},"min":{"1min":0.47999999999999998,"5min":0.39500000000000002,"15min":0.39300000000000002},"max":{"1min":6.3170000000000002,"5min":16.359000000000002,"15min":18.437000000000001},"last":2.5409999999999999},{"interface":"front","average":{"1min":2.0939999999999999,"5min":1.9630000000000001,"15min":2.363},"min":{"1min":0.42699999999999999,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":6.4340000000000002,"5min":7.9089999999999998,"15min":24.524000000000001},"last":2.5550000000000002}]}]},{"osd":0,"up_from":94,"seq":403726926096,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2448188,"kb_used_data":1762996,"kb_used_omap":7,"kb_used_meta":685176,"kb_avail":91296964,"statfs":{"total":95995035648,"available":93488091136,"internally_reserved":0,"allocated":1805307904,"data_stored":1802936909,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":701620241},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:42:20 2023","interfaces":[{"interface":"back","average":{"1min":1.895,"5min":1.8480000000000001,"15min":2.8719999999999999},"min":{"1min":0.184,"5min":0.184,"15min":0.17199999999999999},"max":{"1min":8.7799999999999994,"5min":24.010000000000002,"15min":42.000999999999998},"last":0.223},{"interface":"front","average":{"1min":2.5299999999999998,"5min":1.9450000000000001,"15min":3.0950000000000002},"min":{"1min":0.22700000000000001,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":11.117000000000001,"5min":14.279999999999999,"15min":23.036000000000001},"last":0.30399999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:41:51 2023","interfaces":[{"interface":"back","average":{"1min":0.80200000000000005,"5min":2.6099999999999999,"15min":2.75},"min":{"1min":0.26700000000000002,"5min":0.22600000000000001,"15min":0.185},"max":{"1min":3.5529999999999999,"5min":62.726999999999997,"15min":77.448999999999998},"last":48.841000000000001},{"interface":"front","average":{"1min":0.88900000000000001,"5min":2.8450000000000002,"15min":2.8599999999999999},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.188},"max":{"1min":2.5550000000000002,"5min":62.689,"15min":81.724999999999994},"last":46.183}]},{"osd":3,"last update":"Sat Sep 23 20:42:25 2023","interfaces":[{"interface":"back","average":{"1min":2.2309999999999999,"5min":2.75,"15min":3.0550000000000002},"min":{"1min":0.317,"5min":0.27600000000000002,"15min":0.23400000000000001},"max":{"1min":9.1959999999999997,"5min":51.536000000000001,"15min":81.400999999999996},"last":0.25700000000000001},{"interface":"front","average":{"1min":2.1030000000000002,"5min":2.9849999999999999,"15min":3.355},"min":{"1min":0.25700000000000001,"5min":0.245,"15min":0.21199999999999999},"max":{"1min":9.968,"5min":44.811999999999998,"15min":81.137},"last":0.34399999999999997}]},{"osd":4,"last update":"Sat Sep 23 20:41:51 2023","interfaces":[{"interface":"back","average":{"1min":0.84799999999999998,"5min":1.8600000000000001,"15min":1.9370000000000001},"min":{"1min":0.33500000000000002,"5min":0.28899999999999998,"15min":0.25900000000000001},"max":{"1min":2.6259999999999999,"5min":9.6110000000000007,"15min":12.587999999999999},"last":0.37},{"interface":"front","average":{"1min":1.0780000000000001,"5min":1.623,"15min":1.857},"min":{"1min":0.29199999999999998,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":3.2559999999999998,"5min":9.7789999999999999,"15min":23.001000000000001},"last":0.39200000000000002}]},{"osd":5,"last update":"Sat Sep 23 20:41:51 2023","interfaces":[{"interface":"back","average":{"1min":1.427,"5min":1.6890000000000001,"15min":1.988},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":7.2640000000000002,"5min":9.8629999999999995,"15min":22.969999999999999},"last":0.46999999999999997},{"interface":"front","average":{"1min":1.296,"5min":2.448,"15min":2.2690000000000001},"min":{"1min":0.35799999999999998,"5min":0.318,"15min":0.25600000000000001},"max":{"1min":5.5229999999999997,"5min":26.228000000000002,"15min":26.228000000000002},"last":0.53200000000000003}]},{"osd":6,"last update":"Sat Sep 23 20:41:51 2023","interfaces":[{"interface":"back","average":{"1min":1.3999999999999999,"5min":1.96,"15min":2.1019999999999999},"min":{"1min":0.371,"5min":0.34499999999999997,"15min":0.318},"max":{"1min":7.4400000000000004,"5min":24.056000000000001,"15min":24.056000000000001},"last":0.443},{"interface":"front","average":{"1min":1.0620000000000001,"5min":2.27,"15min":2.1179999999999999},"min":{"1min":0.376,"5min":0.34100000000000003,"15min":0.29399999999999998},"max":{"1min":2.8570000000000002,"5min":26.268000000000001,"15min":26.268000000000001},"last":0.51000000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:41:51 2023","interfaces":[{"interface":"back","average":{"1min":1.4419999999999999,"5min":2.387,"15min":2.2959999999999998},"min":{"1min":0.41899999999999998,"5min":0.36799999999999999,"15min":0.30599999999999999},"max":{"1min":5.4779999999999998,"5min":26.177,"15min":26.177},"last":0.47999999999999998},{"interface":"front","average":{"1min":1.1459999999999999,"5min":1.9990000000000001,"15min":2.2930000000000001},"min":{"1min":0.42399999999999999,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":2.9660000000000002,"5min":9.7200000000000006,"15min":24.669},"last":0.40400000000000003}]}]},{"osd":2,"up_from":104,"seq":446676599044,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2022608,"kb_used_data":1312840,"kb_used_omap":11,"kb_used_meta":709748,"kb_avail":91722544,"statfs":{"total":95995035648,"available":93923885056,"internally_reserved":0,"allocated":1344348160,"data_stored":1341964120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11878,"internal_metadata":726782362},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:42:04 2023","interfaces":[{"interface":"back","average":{"1min":0.71799999999999997,"5min":1.298,"15min":2.4780000000000002},"min":{"1min":0.24399999999999999,"5min":0.20399999999999999,"15min":0.17999999999999999},"max":{"1min":2.6230000000000002,"5min":12.007,"15min":82.424999999999997},"last":1.619},{"interface":"front","average":{"1min":1.4350000000000001,"5min":1.8089999999999999,"15min":2.5899999999999999},"min":{"1min":0.28299999999999997,"5min":0.23000000000000001,"15min":0.19},"max":{"1min":5.415,"5min":9.5600000000000005,"15min":82.524000000000001},"last":0.40999999999999998}]},{"osd":1,"last update":"Sat Sep 23 20:42:04 2023","interfaces":[{"interface":"back","average":{"1min":2.6200000000000001,"5min":2.6989999999999998,"15min":3.234},"min":{"1min":0.26500000000000001,"5min":0.245,"15min":0.221},"max":{"1min":12.981,"5min":46.405000000000001,"15min":46.405000000000001},"last":0.76900000000000002},{"interface":"front","average":{"1min":1.9930000000000001,"5min":2.1160000000000001,"15min":2.7090000000000001},"min":{"1min":0.33100000000000002,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":7.7439999999999998,"5min":46.459000000000003,"15min":46.459000000000003},"last":1.7030000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:42:24 2023","interfaces":[{"interface":"back","average":{"1min":2.3109999999999999,"5min":2.4740000000000002,"15min":2.4870000000000001},"min":{"1min":0.372,"5min":0.25900000000000001,"15min":0.19800000000000001},"max":{"1min":8.9280000000000008,"5min":19.562999999999999,"15min":19.562999999999999},"last":0.89300000000000002},{"interface":"front","average":{"1min":2.0179999999999998,"5min":1.462,"15min":2.419},"min":{"1min":0.315,"5min":0.26400000000000001,"15min":0.23899999999999999},"max":{"1min":11.375,"5min":11.375,"15min":30.704999999999998},"last":1.679}]},{"osd":4,"last update":"Sat Sep 23 20:42:04 2023","interfaces":[{"interface":"back","average":{"1min":1.288,"5min":1.492,"15min":1.8700000000000001},"min":{"1min":0.34599999999999997,"5min":0.33300000000000002,"15min":0.27500000000000002},"max":{"1min":5.508,"5min":6.6849999999999996,"15min":16.797999999999998},"last":0.45000000000000001},{"interface":"front","average":{"1min":1.3109999999999999,"5min":1.6639999999999999,"15min":2.02},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.27600000000000002},"max":{"1min":4.6200000000000001,"5min":19.440999999999999,"15min":19.440999999999999},"last":0.80700000000000005}]},{"osd":5,"last update":"Sat Sep 23 20:42:04 2023","interfaces":[{"interface":"back","average":{"1min":1.3140000000000001,"5min":1.54,"15min":2.3010000000000002},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.314},"max":{"1min":5.468,"5min":6.6319999999999997,"15min":95.393000000000001},"last":0.60499999999999998},{"interface":"front","average":{"1min":1.3149999999999999,"5min":1.6619999999999999,"15min":2.5249999999999999},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.32300000000000001},"max":{"1min":4.6520000000000001,"5min":19.513000000000002,"15min":95.414000000000001},"last":2.0990000000000002}]},{"osd":6,"last update":"Sat Sep 23 20:42:04 2023","interfaces":[{"interface":"back","average":{"1min":1.4590000000000001,"5min":1.6819999999999999,"15min":2.1400000000000001},"min":{"1min":0.377,"5min":0.377,"15min":0.32100000000000001},"max":{"1min":4.5750000000000002,"5min":19.486000000000001,"15min":19.486000000000001},"last":0.92400000000000004},{"interface":"front","average":{"1min":0.72199999999999998,"5min":1.0309999999999999,"15min":1.792},"min":{"1min":0.38600000000000001,"5min":0.36899999999999999,"15min":0.316},"max":{"1min":2.048,"5min":9.3010000000000002,"15min":30.670999999999999},"last":1.653}]},{"osd":7,"last update":"Sat Sep 23 20:42:04 2023","interfaces":[{"interface":"back","average":{"1min":0.82299999999999995,"5min":1.0409999999999999,"15min":1.8129999999999999},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":3.738,"5min":7.5860000000000003,"15min":30.741},"last":2.7429999999999999},{"interface":"front","average":{"1min":1.419,"5min":1.7390000000000001,"15min":2.0939999999999999},"min":{"1min":0.441,"5min":0.38,"15min":0.38},"max":{"1min":5.6120000000000001,"5min":6.7220000000000004,"15min":16.719000000000001},"last":1.6339999999999999}]}]},{"osd":3,"up_from":109,"seq":468151435517,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4503204,"kb_used_data":3584028,"kb_used_omap":4,"kb_used_meta":919163,"kb_avail":89241948,"statfs":{"total":95995035648,"available":91383754752,"internally_reserved":0,"allocated":3670044672,"data_stored":3667685544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4514,"internal_metadata":941223518},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:42:34 2023","interfaces":[{"interface":"back","average":{"1min":2.2389999999999999,"5min":1.956,"15min":2.387},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.186},"max":{"1min":8.3260000000000005,"5min":22.32,"15min":22.32},"last":2.4399999999999999},{"interface":"front","average":{"1min":2.859,"5min":2.0979999999999999,"15min":2.363},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.19500000000000001},"max":{"1min":12.814,"5min":22.073,"15min":22.073},"last":4.3390000000000004}]},{"osd":1,"last update":"Sat Sep 23 20:42:05 2023","interfaces":[{"interface":"back","average":{"1min":1.9239999999999999,"5min":1.3939999999999999,"15min":2.1739999999999999},"min":{"1min":0.26500000000000001,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":8.3650000000000002,"5min":8.3650000000000002,"15min":16.189},"last":1.48},{"interface":"front","average":{"1min":2.0209999999999999,"5min":2.117,"15min":2.5350000000000001},"min":{"1min":0.24399999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":6.4930000000000003,"5min":12.702999999999999,"15min":32.603999999999999},"last":4.3730000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:42:34 2023","interfaces":[{"interface":"back","average":{"1min":2.9340000000000002,"5min":3.0310000000000001,"15min":2.8159999999999998},"min":{"1min":0.29199999999999998,"5min":0.246,"15min":0.22800000000000001},"max":{"1min":9.0109999999999992,"5min":68.206999999999994,"15min":68.206999999999994},"last":0.98699999999999999},{"interface":"front","average":{"1min":1.9570000000000001,"5min":2.9929999999999999,"15min":2.665},"min":{"1min":0.308,"5min":0.28100000000000003,"15min":0.20899999999999999},"max":{"1min":4.7130000000000001,"5min":67.736999999999995,"15min":67.736999999999995},"last":0.41099999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:42:34 2023","interfaces":[{"interface":"back","average":{"1min":1.419,"5min":1.573,"15min":1.798},"min":{"1min":0.34499999999999997,"5min":0.30499999999999999,"15min":0.29799999999999999},"max":{"1min":6.0309999999999997,"5min":22.157,"15min":46.042999999999999},"last":1.518},{"interface":"front","average":{"1min":2.2730000000000001,"5min":2.3090000000000002,"15min":2.3069999999999999},"min":{"1min":0.38900000000000001,"5min":0.33000000000000002,"15min":0.28100000000000003},"max":{"1min":8.6910000000000007,"5min":15.868,"15min":46.579999999999998},"last":0.51100000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:42:34 2023","interfaces":[{"interface":"back","average":{"1min":2.1920000000000002,"5min":1.768,"15min":1.99},"min":{"1min":0.39000000000000001,"5min":0.32700000000000001,"15min":0.31900000000000001},"max":{"1min":7.8090000000000002,"5min":22.289999999999999,"15min":45.292000000000002},"last":5.2290000000000001},{"interface":"front","average":{"1min":1.9399999999999999,"5min":1.6080000000000001,"15min":1.9770000000000001},"min":{"1min":0.435,"5min":0.38500000000000001,"15min":0.32000000000000001},"max":{"1min":9.2319999999999993,"5min":9.2319999999999993,"15min":44.679000000000002},"last":0.54500000000000004}]},{"osd":6,"last update":"Sat Sep 23 20:42:34 2023","interfaces":[{"interface":"back","average":{"1min":2.109,"5min":1.5649999999999999,"15min":1.865},"min":{"1min":0.441,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":6.2229999999999999,"5min":7.3630000000000004,"15min":17.524999999999999},"last":4.3049999999999997},{"interface":"front","average":{"1min":2.3410000000000002,"5min":2.2650000000000001,"15min":2.1549999999999998},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.33700000000000002},"max":{"1min":7.6020000000000003,"5min":15.936,"15min":19.524000000000001},"last":4.5510000000000002}]},{"osd":7,"last update":"Sat Sep 23 20:42:34 2023","interfaces":[{"interface":"back","average":{"1min":3.0790000000000002,"5min":2.6309999999999998,"15min":2.4420000000000002},"min":{"1min":0.41499999999999998,"5min":0.39900000000000002,"15min":0.36899999999999999},"max":{"1min":7.9450000000000003,"5min":15.976000000000001,"15min":19.571999999999999},"last":0.61299999999999999},{"interface":"front","average":{"1min":2.1640000000000001,"5min":1.8340000000000001,"15min":2.0230000000000001},"min":{"1min":0.45400000000000001,"5min":0.40999999999999998,"15min":0.371},"max":{"1min":7.5609999999999999,"5min":22.202999999999999,"15min":45.387999999999998},"last":1.55}]}]},{"osd":4,"up_from":30,"seq":128849019271,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1748488,"kb_used_data":1355200,"kb_used_omap":0,"kb_used_meta":393280,"kb_avail":91996664,"statfs":{"total":95995035648,"available":94204583936,"internally_reserved":0,"allocated":1387724800,"data_stored":1386270810,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":402718720},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:41:50 2023","interfaces":[{"interface":"back","average":{"1min":1.792,"5min":1.5369999999999999,"15min":1.6930000000000001},"min":{"1min":0.26200000000000001,"5min":0.24099999999999999,"15min":0.23699999999999999},"max":{"1min":8.5370000000000008,"5min":15.553000000000001,"15min":24.114999999999998},"last":1.51},{"interface":"front","average":{"1min":1.7030000000000001,"5min":1.6759999999999999,"15min":1.831},"min":{"1min":0.39600000000000002,"5min":0.25,"15min":0.246},"max":{"1min":8.5269999999999992,"5min":15.686,"15min":24.135999999999999},"last":4.2510000000000003}]},{"osd":1,"last update":"Sat Sep 23 20:42:10 2023","interfaces":[{"interface":"back","average":{"1min":1.512,"5min":1.3089999999999999,"15min":1.6579999999999999},"min":{"1min":0.28899999999999998,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":6.5789999999999997,"5min":18.190999999999999,"15min":18.190999999999999},"last":1.329},{"interface":"front","average":{"1min":1.6910000000000001,"5min":1.407,"15min":1.625},"min":{"1min":0.33000000000000002,"5min":0.28499999999999998,"15min":0.28199999999999997},"max":{"1min":6.5620000000000003,"5min":18.173999999999999,"15min":18.173999999999999},"last":1.345}]},{"osd":2,"last update":"Sat Sep 23 20:42:06 2023","interfaces":[{"interface":"back","average":{"1min":2.3130000000000002,"5min":1.486,"15min":1.8300000000000001},"min":{"1min":0.311,"5min":0.28399999999999997,"15min":0.26600000000000001},"max":{"1min":13.051,"5min":13.051,"15min":32.363},"last":0.34499999999999997},{"interface":"front","average":{"1min":2.113,"5min":1.863,"15min":2.133},"min":{"1min":0.30299999999999999,"5min":0.28399999999999997,"15min":0.27600000000000002},"max":{"1min":10.945,"5min":11.917999999999999,"15min":34.042999999999999},"last":5.0540000000000003}]},{"osd":3,"last update":"Sat Sep 23 20:42:21 2023","interfaces":[{"interface":"back","average":{"1min":2.3919999999999999,"5min":1.6779999999999999,"15min":1.992},"min":{"1min":0.442,"5min":0.372,"15min":0.29099999999999998},"max":{"1min":7.1710000000000003,"5min":12.147,"15min":32.332999999999998},"last":0.42199999999999999},{"interface":"front","average":{"1min":2.5270000000000001,"5min":1.6679999999999999,"15min":2.081},"min":{"1min":0.41799999999999998,"5min":0.35499999999999998,"15min":0.30299999999999999},"max":{"1min":13.35,"5min":13.35,"15min":32.399999999999999},"last":1.143}]},{"osd":5,"last update":"Sat Sep 23 20:41:50 2023","interfaces":[{"interface":"back","average":{"1min":0.89400000000000002,"5min":0.73199999999999998,"15min":1.236},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.247},"max":{"1min":2.3140000000000001,"5min":4.1260000000000003,"15min":82.866},"last":9.7260000000000009},{"interface":"front","average":{"1min":0.626,"5min":0.94899999999999995,"15min":1.3120000000000001},"min":{"1min":0.314,"5min":0.314,"15min":0.27800000000000002},"max":{"1min":1.5860000000000001,"5min":7.0129999999999999,"15min":82.546000000000006},"last":3.8300000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:42:06 2023","interfaces":[{"interface":"back","average":{"1min":1.6830000000000001,"5min":1.1160000000000001,"15min":1.1459999999999999},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.27600000000000002},"max":{"1min":7.2539999999999996,"5min":14.731999999999999,"15min":14.731999999999999},"last":3.6150000000000002},{"interface":"front","average":{"1min":0.88200000000000001,"5min":0.88400000000000001,"15min":1.0549999999999999},"min":{"1min":0.36799999999999999,"5min":0.314,"15min":0.29899999999999999},"max":{"1min":3.9569999999999999,"5min":7.2850000000000001,"15min":11.199},"last":0.44}]},{"osd":7,"last update":"Sat Sep 23 20:42:21 2023","interfaces":[{"interface":"back","average":{"1min":2.4340000000000002,"5min":1.2490000000000001,"15min":1.3200000000000001},"min":{"1min":0.35099999999999998,"5min":0.318,"15min":0.318},"max":{"1min":22.184999999999999,"5min":22.184999999999999,"15min":22.184999999999999},"last":5.4569999999999999},{"interface":"front","average":{"1min":2.3809999999999998,"5min":1.1020000000000001,"15min":1.1080000000000001},"min":{"1min":0.33900000000000002,"5min":0.33200000000000002,"15min":0.307},"max":{"1min":21.754000000000001,"5min":21.754000000000001,"15min":21.754000000000001},"last":2.3620000000000001}]}]},{"osd":5,"up_from":39,"seq":167503724932,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4539976,"kb_used_data":4076096,"kb_used_omap":0,"kb_used_meta":463808,"kb_avail":89205176,"statfs":{"total":95995035648,"available":91346100224,"internally_reserved":0,"allocated":4173922304,"data_stored":4172464968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":474939392},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:41:56 2023","interfaces":[{"interface":"back","average":{"1min":1.0600000000000001,"5min":1.335,"15min":1.673},"min":{"1min":0.23100000000000001,"5min":0.22500000000000001,"15min":0.20699999999999999},"max":{"1min":3.1819999999999999,"5min":7.819,"15min":18.16},"last":1.0209999999999999},{"interface":"front","average":{"1min":0.97999999999999998,"5min":1.3520000000000001,"15min":1.599},"min":{"1min":0.29999999999999999,"5min":0.29899999999999999,"15min":0.248},"max":{"1min":5.3499999999999996,"5min":7.9859999999999998,"15min":9.0570000000000004},"last":1.0069999999999999}]},{"osd":1,"last update":"Sat Sep 23 20:42:09 2023","interfaces":[{"interface":"back","average":{"1min":5.7199999999999998,"5min":2.3780000000000001,"15min":2.1190000000000002},"min":{"1min":0.27200000000000002,"5min":0.26500000000000001,"15min":0.253},"max":{"1min":61.005000000000003,"5min":61.005000000000003,"15min":61.005000000000003},"last":1.7310000000000001},{"interface":"front","average":{"1min":5.101,"5min":2.02,"15min":1.911},"min":{"1min":0.28100000000000003,"5min":0.27700000000000002,"15min":0.23499999999999999},"max":{"1min":59.688000000000002,"5min":59.688000000000002,"15min":59.688000000000002},"last":2.1429999999999998}]},{"osd":2,"last update":"Sat Sep 23 20:41:56 2023","interfaces":[{"interface":"back","average":{"1min":1.6359999999999999,"5min":1.625,"15min":1.9350000000000001},"min":{"1min":0.35299999999999998,"5min":0.28000000000000003,"15min":0.26500000000000001},"max":{"1min":6.282,"5min":10.743,"15min":37.011000000000003},"last":1.966},{"interface":"front","average":{"1min":0.96499999999999997,"5min":1.3200000000000001,"15min":1.9019999999999999},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.255},"max":{"1min":5.415,"5min":8.7560000000000002,"15min":36.835000000000001},"last":5.0190000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:42:18 2023","interfaces":[{"interface":"back","average":{"1min":2.3799999999999999,"5min":1.651,"15min":2.145},"min":{"1min":0.441,"5min":0.34699999999999998,"15min":0.25700000000000001},"max":{"1min":6.0389999999999997,"5min":6.9669999999999996,"15min":36.716000000000001},"last":7.4320000000000004},{"interface":"front","average":{"1min":2.8149999999999999,"5min":2.2349999999999999,"15min":2.085},"min":{"1min":0.38600000000000001,"5min":0.34000000000000002,"15min":0.28799999999999998},"max":{"1min":6.2480000000000002,"5min":18.504000000000001,"15min":18.504000000000001},"last":1.9990000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:41:47 2023","interfaces":[{"interface":"back","average":{"1min":0.61699999999999999,"5min":0.86699999999999999,"15min":1.129},"min":{"1min":0.27300000000000002,"5min":0.255,"15min":0.255},"max":{"1min":2.843,"5min":6.9180000000000001,"15min":28.015999999999998},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.879,"5min":0.95499999999999996,"15min":0.97399999999999998},"min":{"1min":0.30299999999999999,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":3.3359999999999999,"5min":6.8380000000000001,"15min":12.300000000000001},"last":0.57699999999999996}]},{"osd":6,"last update":"Sat Sep 23 20:41:59 2023","interfaces":[{"interface":"back","average":{"1min":1.3400000000000001,"5min":1.129,"15min":1.111},"min":{"1min":0.317,"5min":0.317,"15min":0.30199999999999999},"max":{"1min":7.867,"5min":7.867,"15min":12.890000000000001},"last":7.383},{"interface":"front","average":{"1min":1.6100000000000001,"5min":1.2030000000000001,"15min":1.1120000000000001},"min":{"1min":0.34100000000000003,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":8.7449999999999992,"5min":8.7449999999999992,"15min":16.265999999999998},"last":3.5939999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:42:18 2023","interfaces":[{"interface":"back","average":{"1min":1.4059999999999999,"5min":1.0369999999999999,"15min":1.069},"min":{"1min":0.36399999999999999,"5min":0.33400000000000002,"15min":0.28499999999999998},"max":{"1min":3.9860000000000002,"5min":6.4720000000000004,"15min":36.621000000000002},"last":0.621},{"interface":"front","average":{"1min":2.0259999999999998,"5min":1.095,"15min":1.1830000000000001},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.30099999999999999},"max":{"1min":5.9359999999999999,"5min":7.3700000000000001,"15min":37.331000000000003},"last":1.502}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4722,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1360,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8206,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7156,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3154,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1790705664,"data_stored":1790705664,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4101902336,"data_stored":4101898264,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1329266688,"data_stored":1329266688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3654287360,"data_stored":3654287360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1383071744,"data_stored":1383071744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4048359424,"data_stored":4048355352,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1367736320,"data_stored":1367736320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4020965376,"data_stored":4020961304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:42:50.016 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-09-23T20:42:50.016 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd pool set .mgr pg_num 51 2023-09-23T20:42:50.063 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:42:50.173 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:49 smithi018 ceph-mon[184144]: from='client.36597 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:42:50.173 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:49 smithi018 ceph-mon[179647]: from='client.36597 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:42:50.181 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:50.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:49 smithi149 ceph-mon[170394]: from='client.36597 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:42:50.337 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:42:50.483 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:42:50.490 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:50.490 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:42:50.591 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:42:50.705 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:50.934 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:42:50.942 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:50.943 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:42:51.044 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:42:51.132 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:51.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:50 smithi149 ceph-mon[170394]: pgmap v1194: 119 pgs: 2 active+remapped+backfilling, 1 active+remapped+backfill_wait, 116 active+clean; 6.6 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 128 KiB/s wr, 614 op/s; 26067/326049 objects misplaced (7.995%); 10 MiB/s, 165 objects/s recovering 2023-09-23T20:42:51.254 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:50 smithi018 ceph-mon[179647]: pgmap v1194: 119 pgs: 2 active+remapped+backfilling, 1 active+remapped+backfill_wait, 116 active+clean; 6.6 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 128 KiB/s wr, 614 op/s; 26067/326049 objects misplaced (7.995%); 10 MiB/s, 165 objects/s recovering 2023-09-23T20:42:51.254 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:50 smithi018 ceph-mon[184144]: pgmap v1194: 119 pgs: 2 active+remapped+backfilling, 1 active+remapped+backfill_wait, 116 active+clean; 6.6 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 128 KiB/s wr, 614 op/s; 26067/326049 objects misplaced (7.995%); 10 MiB/s, 165 objects/s recovering 2023-09-23T20:42:51.617 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:42:51.630 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:42:51.633 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:51.634 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:42:51.734 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:42:51.804 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:51.978 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:42:51.985 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:51.985 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:42:52.086 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:42:52.171 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:52.267 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:42:52.287 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:52.287 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:42:52.388 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:42:52.495 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:52.516 INFO:teuthology.orchestra.run.smithi018.stderr:nodeep-scrub is set 2023-09-23T20:42:52.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:52 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3181005286' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:42:52.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:52 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:42:52.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:52 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:42:52.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:52 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:42:52.726 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:42:52.772 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:42:52.822 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:42:52.856 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:42:52.897 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:52 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3181005286' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:42:52.897 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:52 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:42:52.898 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:52 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:42:52.898 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:52 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:42:52.898 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:52 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3181005286' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:42:52.898 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:52 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:42:52.898 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:52 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:42:52.898 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:52 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:42:52.912 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:42:52.951 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:42:53.005 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:42:53.022 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:42:53.024 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:53.024 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:42:53.042 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:42:53.060 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:42:53.060 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:42:53.060 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:42:53.060 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:42:53.060 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:42:53.060 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:42:53.061 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:42:53.061 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:42:53.125 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:42:53.171 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:53.303 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:42:53.310 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:53.311 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:42:53.411 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:42:53.470 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:53.600 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:42:53.610 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:53.610 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:42:53.654 INFO:teuthology.orchestra.run.smithi149.stdout:ERROR: (22) Invalid argument 2023-09-23T20:42:53.654 INFO:teuthology.orchestra.run.smithi149.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:42:53.691 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:53 smithi149 ceph-mon[170394]: pgmap v1195: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 6.6 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 18 KiB/s wr, 633 op/s; 24659/322032 objects misplaced (7.657%); 12 MiB/s, 187 objects/s recovering 2023-09-23T20:42:53.691 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:53 smithi149 ceph-mon[170394]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:42:53.691 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:53 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:42:53.691 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:53 smithi149 ceph-mon[170394]: osdmap e477: 8 total, 8 up, 8 in 2023-09-23T20:42:53.691 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:53 smithi149 ceph-mon[170394]: failed to encode map e477 with expected crc 2023-09-23T20:42:53.691 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:53 smithi149 ceph-mon[170394]: failed to encode map e477 with expected crc 2023-09-23T20:42:53.691 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:53 smithi149 ceph-mon[170394]: failed to encode map e477 with expected crc 2023-09-23T20:42:53.692 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:53 smithi149 ceph-mon[170394]: failed to encode map e477 with expected crc 2023-09-23T20:42:53.692 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:53 smithi149 ceph-mon[170394]: failed to encode map e477 with expected crc 2023-09-23T20:42:53.692 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:53 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2388552646' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:42:53.711 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:42:53.752 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:53 smithi018 ceph-mon[184144]: pgmap v1195: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 6.6 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 18 KiB/s wr, 633 op/s; 24659/322032 objects misplaced (7.657%); 12 MiB/s, 187 objects/s recovering 2023-09-23T20:42:53.753 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:53 smithi018 ceph-mon[184144]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:42:53.753 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:53 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:42:53.753 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:53 smithi018 ceph-mon[184144]: osdmap e477: 8 total, 8 up, 8 in 2023-09-23T20:42:53.753 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:53 smithi018 ceph-mon[184144]: failed to encode map e477 with expected crc 2023-09-23T20:42:53.753 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:53 smithi018 ceph-mon[184144]: failed to encode map e477 with expected crc 2023-09-23T20:42:53.753 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:53 smithi018 ceph-mon[184144]: failed to encode map e477 with expected crc 2023-09-23T20:42:53.753 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:53 smithi018 ceph-mon[184144]: failed to encode map e477 with expected crc 2023-09-23T20:42:53.753 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:53 smithi018 ceph-mon[184144]: failed to encode map e477 with expected crc 2023-09-23T20:42:53.753 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:53 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2388552646' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:42:53.754 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:53 smithi018 ceph-mon[179647]: pgmap v1195: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 6.6 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 18 KiB/s wr, 633 op/s; 24659/322032 objects misplaced (7.657%); 12 MiB/s, 187 objects/s recovering 2023-09-23T20:42:53.754 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:53 smithi018 ceph-mon[179647]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:42:53.754 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:53 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:42:53.754 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:53 smithi018 ceph-mon[179647]: osdmap e477: 8 total, 8 up, 8 in 2023-09-23T20:42:53.754 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:53 smithi018 ceph-mon[179647]: failed to encode map e477 with expected crc 2023-09-23T20:42:53.754 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:53 smithi018 ceph-mon[179647]: failed to encode map e477 with expected crc 2023-09-23T20:42:53.754 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:53 smithi018 ceph-mon[179647]: failed to encode map e477 with expected crc 2023-09-23T20:42:53.754 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:53 smithi018 ceph-mon[179647]: failed to encode map e477 with expected crc 2023-09-23T20:42:53.754 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:53 smithi018 ceph-mon[179647]: failed to encode map e477 with expected crc 2023-09-23T20:42:53.754 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:53 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2388552646' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:42:53.763 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:53.934 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:42:53.941 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:53.942 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:42:54.042 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:42:54.093 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:54.220 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:42:54.231 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:54.231 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:42:54.279 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T20:42:54.332 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:42:54.407 INFO:teuthology.orchestra.run.smithi018.stderr:Error EINVAL: specified pg_num 51 < pg_num_max 32 2023-09-23T20:42:54.438 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:54.574 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:42:54.583 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:54.583 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:42:54.684 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:42:54.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:54 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1805946307' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "51"}]: dispatch 2023-09-23T20:42:54.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:54 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "51"}]: dispatch 2023-09-23T20:42:54.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:54 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:42:54.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:54 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:42:54.803 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:54.874 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:54 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1805946307' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "51"}]: dispatch 2023-09-23T20:42:54.874 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:54 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "51"}]: dispatch 2023-09-23T20:42:54.874 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:54 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:42:54.874 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:54 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:42:54.875 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:54 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1805946307' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "51"}]: dispatch 2023-09-23T20:42:54.875 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:54 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "51"}]: dispatch 2023-09-23T20:42:54.875 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:54 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:42:54.875 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:54 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:42:54.890 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:42:54.903 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:54.903 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:42:54.907 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-09-23T20:42:55.004 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:42:55.048 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:55.098 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:42:55.141 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:42:54 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:42:54] "GET /metrics HTTP/1.1" 200 36602 "" "Prometheus/2.43.0" 2023-09-23T20:42:55.147 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:42:55.158 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:55.159 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:42:55.259 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:42:55.269 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T20:42:55.330 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:55.422 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:42:55.433 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:55.433 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:42:55.534 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:42:55.603 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:55 smithi018 ceph-mon[184144]: pgmap v1197: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 6.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 18 KiB/s wr, 693 op/s; 23544/316518 objects misplaced (7.438%); 18 MiB/s, 289 objects/s recovering 2023-09-23T20:42:55.603 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:55 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:42:55.603 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:55 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:42:55.603 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:55 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:42:55.603 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:55 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:42:55.609 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:55 smithi018 ceph-mon[179647]: pgmap v1197: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 6.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 18 KiB/s wr, 693 op/s; 23544/316518 objects misplaced (7.438%); 18 MiB/s, 289 objects/s recovering 2023-09-23T20:42:55.609 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:55 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:42:55.609 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:55 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:42:55.609 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:55 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:42:55.609 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:55 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:42:55.609 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:55.638 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:42:55.696 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:42:55.715 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:55.716 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:42:55.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:55 smithi149 ceph-mon[170394]: pgmap v1197: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 6.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 18 KiB/s wr, 693 op/s; 23544/316518 objects misplaced (7.438%); 18 MiB/s, 289 objects/s recovering 2023-09-23T20:42:55.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:55 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:42:55.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:55 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:42:55.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:55 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:42:55.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:55 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:42:55.816 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:42:55.903 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:55.999 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:42:56.006 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:56.006 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:42:56.107 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:42:56.117 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:42:56.180 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:56.256 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:42:56.266 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:56.266 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:42:56.367 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:42:56.469 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:56.544 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:42:56.553 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:56.554 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:42:56.655 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:42:56.675 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:42:56.725 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:56.739 INFO:teuthology.orchestra.run.smithi149.stdout:ERROR: (22) Invalid argument 2023-09-23T20:42:56.739 INFO:teuthology.orchestra.run.smithi149.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:42:56.864 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:42:56.868 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:56.868 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:42:56.969 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:42:57.062 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:57.121 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:42:57.130 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:57.130 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:42:57.206 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T20:42:57.231 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:42:57.327 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:57.494 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:42:57.504 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:57.504 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:42:57.605 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:42:57.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:57 smithi018 ceph-mon[184144]: pgmap v1198: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 6.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 15 KiB/s wr, 736 op/s; 23319/311685 objects misplaced (7.482%); 15 MiB/s, 240 objects/s recovering 2023-09-23T20:42:57.646 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:57 smithi018 ceph-mon[179647]: pgmap v1198: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 6.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 15 KiB/s wr, 736 op/s; 23319/311685 objects misplaced (7.482%); 15 MiB/s, 240 objects/s recovering 2023-09-23T20:42:57.680 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:57.750 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:42:57.762 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:57.763 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:42:57.863 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:42:57.953 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:57 smithi149 ceph-mon[170394]: pgmap v1198: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 6.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 15 KiB/s wr, 736 op/s; 23319/311685 objects misplaced (7.482%); 15 MiB/s, 240 objects/s recovering 2023-09-23T20:42:57.956 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:58.017 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:42:58.025 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:58.025 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:42:58.088 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:42:58.126 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:42:58.213 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:58.587 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:42:58.589 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:58.590 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:42:58.592 INFO:teuthology.orchestra.run.smithi018.stderr:noscrub is unset 2023-09-23T20:42:58.607 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:42:58.652 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:42:58.691 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:42:58.701 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:42:58.752 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:42:58.789 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:58.801 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:42:58.848 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:42:58.884 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:42:58.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:58 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1683015235' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:42:58.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:58 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:42:58.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:58 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1683015235' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:42:58.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:58 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:42:58.926 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:42:58.928 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:42:58.928 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:42:58.929 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:42:58.929 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:42:58.929 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:42:58.929 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:42:58.929 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:42:58.929 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:42:58.942 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:58 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1683015235' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:42:58.942 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:58 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:42:59.033 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:42:59.038 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:59.039 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:42:59.139 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:42:59.238 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:59.309 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:42:59.314 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:59.314 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:42:59.415 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:42:59.478 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:59.544 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:42:59.548 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:59.548 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:42:59.649 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:42:59.682 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:59 smithi018 ceph-mon[184144]: pgmap v1199: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 6.3 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 732 op/s; 21246/308121 objects misplaced (6.895%); 15 MiB/s, 238 objects/s recovering 2023-09-23T20:42:59.683 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:59 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2843620515' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:42:59.683 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:59 smithi018 ceph-mon[184144]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:42:59.683 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:59 smithi018 ceph-mon[184144]: failed to encode map e478 with expected crc 2023-09-23T20:42:59.683 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:59 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:42:59.683 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:59 smithi018 ceph-mon[184144]: osdmap e478: 8 total, 8 up, 8 in 2023-09-23T20:42:59.684 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:59 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:42:59.684 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:42:59 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:42:59.685 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:59.685 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:59 smithi018 ceph-mon[179647]: pgmap v1199: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 6.3 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 732 op/s; 21246/308121 objects misplaced (6.895%); 15 MiB/s, 238 objects/s recovering 2023-09-23T20:42:59.685 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:59 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2843620515' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:42:59.685 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:59 smithi018 ceph-mon[179647]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:42:59.686 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:59 smithi018 ceph-mon[179647]: failed to encode map e478 with expected crc 2023-09-23T20:42:59.686 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:59 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:42:59.686 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:59 smithi018 ceph-mon[179647]: osdmap e478: 8 total, 8 up, 8 in 2023-09-23T20:42:59.686 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:59 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:42:59.686 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:42:59 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:42:59.759 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:42:59.764 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:42:59.764 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:42:59.865 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:42:59.909 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 7, 3, 2, 4, 6, 5] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T20:42:59.909 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:42:59.909 INFO:tasks.thrashosds.thrasher:inject_pause on osd.3 2023-09-23T20:42:59.909 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T20:42:59.909 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T20:42:59.909 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 config set filestore_inject_stall 3 2023-09-23T20:42:59.919 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:42:59.967 INFO:teuthology.orchestra.run.smithi149.stdout:{ 2023-09-23T20:42:59.967 INFO:teuthology.orchestra.run.smithi149.stdout: "size": 20, 2023-09-23T20:42:59.967 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 600, 2023-09-23T20:42:59.967 INFO:teuthology.orchestra.run.smithi149.stdout: "ops": [ 2023-09-23T20:42:59.967 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:59.967 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:84069 8.9 8:9824f39b:::benchmark_data_smithi018_341490_object84068:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e378)", 2023-09-23T20:42:59.967 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:37:13.327117+0000", 2023-09-23T20:42:59.967 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 346.63624819099999, 2023-09-23T20:42:59.967 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.3519245670000002, 2023-09-23T20:42:59.968 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:42:59.968 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:42:59.968 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:42:59.968 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:42:59.968 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:42:59.968 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 84069 2023-09-23T20:42:59.968 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:59.968 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:42:59.968 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:59.968 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:42:59.969 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.327117+0000", 2023-09-23T20:42:59.969 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:42:59.969 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:59.969 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:59.969 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:42:59.969 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.327117+0000", 2023-09-23T20:42:59.969 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:42:59.969 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:59.969 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:59.969 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:42:59.970 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.327116+0000", 2023-09-23T20:42:59.970 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:42:59.970 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:59.970 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:59.970 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:42:59.970 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.327148+0000", 2023-09-23T20:42:59.970 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.1741e-05 2023-09-23T20:42:59.970 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:59.970 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:59.971 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:42:59.971 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.327160+0000", 2023-09-23T20:42:59.971 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.185e-05 2023-09-23T20:42:59.971 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:59.971 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:59.971 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:42:59.971 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.327165+0000", 2023-09-23T20:42:59.971 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.1749999999999996e-06 2023-09-23T20:42:59.971 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:59.971 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:59.972 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:42:59.972 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.330726+0000", 2023-09-23T20:42:59.972 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00356121 2023-09-23T20:42:59.972 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:59.972 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:59.972 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:42:59.972 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.330874+0000", 2023-09-23T20:42:59.972 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000148097 2023-09-23T20:42:59.972 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:59.972 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:59.973 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:42:59.973 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.331021+0000", 2023-09-23T20:42:59.973 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00014648499999999999 2023-09-23T20:42:59.973 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:59.973 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:59.973 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:42:59.973 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.334676+0000", 2023-09-23T20:42:59.973 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0036551869999999998 2023-09-23T20:42:59.973 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:59.973 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:59.974 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:42:59.974 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.335154+0000", 2023-09-23T20:42:59.974 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00047863799999999998 2023-09-23T20:42:59.974 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:59.974 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:59.974 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:42:59.974 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.368539+0000", 2023-09-23T20:42:59.974 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.033384692000000001 2023-09-23T20:42:59.974 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:59.975 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:59.975 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:42:59.975 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.368550+0000", 2023-09-23T20:42:59.975 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1178e-05 2023-09-23T20:42:59.975 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:59.975 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:59.975 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:42:59.975 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:14.362533+0000", 2023-09-23T20:42:59.975 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.993982649 2023-09-23T20:42:59.975 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:59.976 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:59.976 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:42:59.976 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:14.362546+0000", 2023-09-23T20:42:59.976 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.2660000000000001e-05 2023-09-23T20:42:59.976 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:59.976 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:59.976 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:42:59.976 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:15.678962+0000", 2023-09-23T20:42:59.976 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.316415911 2023-09-23T20:42:59.976 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:59.976 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:59.977 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:42:59.977 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:15.679041+0000", 2023-09-23T20:42:59.977 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.9655999999999998e-05 2023-09-23T20:42:59.977 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:42:59.977 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:42:59.977 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:42:59.977 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:59.977 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:59.977 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:84081 8.9 8:97fd4d67:::benchmark_data_smithi018_341490_object84080:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e378)", 2023-09-23T20:42:59.977 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:37:13.341077+0000", 2023-09-23T20:42:59.978 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 346.62228812000001, 2023-09-23T20:42:59.978 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.3380652359999998, 2023-09-23T20:42:59.978 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:42:59.978 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:42:59.978 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:42:59.978 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:42:59.978 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:42:59.978 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 84081 2023-09-23T20:42:59.978 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:59.979 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:42:59.979 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:59.979 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:42:59.979 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.341077+0000", 2023-09-23T20:42:59.979 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:42:59.979 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:59.979 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:59.979 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:42:59.979 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.341077+0000", 2023-09-23T20:42:59.979 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:42:59.979 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:59.980 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:59.980 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:42:59.980 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.341075+0000", 2023-09-23T20:42:59.980 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999981 2023-09-23T20:42:59.980 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:59.980 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:59.980 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:42:59.980 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.341116+0000", 2023-09-23T20:42:59.980 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.1675000000000003e-05 2023-09-23T20:42:59.980 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:59.980 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:59.981 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:42:59.981 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.341135+0000", 2023-09-23T20:42:59.981 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.8110000000000001e-05 2023-09-23T20:42:59.981 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:59.981 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:59.981 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:42:59.981 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.341141+0000", 2023-09-23T20:42:59.981 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.849e-06 2023-09-23T20:42:59.981 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:59.981 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:59.982 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:42:59.982 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.341194+0000", 2023-09-23T20:42:59.982 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.2553000000000003e-05 2023-09-23T20:42:59.982 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:59.982 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:59.982 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:42:59.982 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.341313+0000", 2023-09-23T20:42:59.982 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00011935899999999999 2023-09-23T20:42:59.982 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:59.982 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:59.983 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:42:59.983 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.341466+0000", 2023-09-23T20:42:59.983 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00015230300000000001 2023-09-23T20:42:59.983 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:59.983 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:59.983 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:42:59.983 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.344704+0000", 2023-09-23T20:42:59.983 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0032382510000000002 2023-09-23T20:42:59.983 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:59.983 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:59.984 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:42:59.984 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.348055+0000", 2023-09-23T20:42:59.984 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.003350872 2023-09-23T20:42:59.984 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:59.984 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:59.984 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:42:59.984 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.368594+0000", 2023-09-23T20:42:59.984 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.020538948000000001 2023-09-23T20:42:59.984 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:59.984 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:59.984 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:42:59.985 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.368601+0000", 2023-09-23T20:42:59.985 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.6529999999999994e-06 2023-09-23T20:42:59.985 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:59.985 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:59.985 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:42:59.985 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:14.362580+0000", 2023-09-23T20:42:59.985 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.99397867299999998 2023-09-23T20:42:59.985 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:59.985 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:59.985 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:42:59.986 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:14.362589+0000", 2023-09-23T20:42:59.986 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.9509999999999995e-06 2023-09-23T20:42:59.986 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:59.986 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:59.986 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:42:59.986 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:15.679073+0000", 2023-09-23T20:42:59.986 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.316483598 2023-09-23T20:42:59.986 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:59.986 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:59.986 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:42:59.987 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:15.679142+0000", 2023-09-23T20:42:59.987 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.9389999999999998e-05 2023-09-23T20:42:59.987 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:42:59.987 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:42:59.987 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:42:59.987 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:59.987 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:59.987 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:105775 8.9 8:97ec68b7:::benchmark_data_smithi018_341490_object16452:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e411)", 2023-09-23T20:42:59.987 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:38:27.408390+0000", 2023-09-23T20:42:59.987 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 272.55497432800001, 2023-09-23T20:42:59.987 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.1501579759999991, 2023-09-23T20:42:59.988 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:42:59.988 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:42:59.988 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:42:59.988 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:42:59.988 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:42:59.988 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 105775 2023-09-23T20:42:59.988 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:59.988 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:42:59.988 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:59.988 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:42:59.989 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.408390+0000", 2023-09-23T20:42:59.989 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:42:59.989 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:59.989 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:59.989 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:42:59.989 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.408390+0000", 2023-09-23T20:42:59.989 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:42:59.989 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:59.989 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:59.989 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:42:59.989 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.408389+0000", 2023-09-23T20:42:59.990 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:42:59.990 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:59.990 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:59.990 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:42:59.990 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.408397+0000", 2023-09-23T20:42:59.990 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.7009999999999996e-06 2023-09-23T20:42:59.990 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:59.990 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:59.990 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:42:59.990 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.408411+0000", 2023-09-23T20:42:59.990 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.4228e-05 2023-09-23T20:42:59.991 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:59.991 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:59.991 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:42:59.991 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.408419+0000", 2023-09-23T20:42:59.991 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.5599999999999994e-06 2023-09-23T20:42:59.991 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:59.991 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:59.991 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:42:59.991 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.414829+0000", 2023-09-23T20:42:59.991 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0064094740000000001 2023-09-23T20:42:59.991 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:59.991 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:59.992 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:42:59.992 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.414843+0000", 2023-09-23T20:42:59.992 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.4561e-05 2023-09-23T20:42:59.992 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:59.992 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:59.992 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:42:59.992 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.556066+0000", 2023-09-23T20:42:59.992 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.1412221710000008 2023-09-23T20:42:59.992 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:59.992 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:59.992 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:42:59.992 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.558548+0000", 2023-09-23T20:42:59.992 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0024828070000000001 2023-09-23T20:42:59.993 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:42:59.993 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:42:59.993 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:42:59.993 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:59.993 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:59.993 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:105788 8.9 8:94f5ae14:::benchmark_data_smithi018_341490_object16465:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e411)", 2023-09-23T20:42:59.993 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:38:27.415518+0000", 2023-09-23T20:42:59.993 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 272.54784641200001, 2023-09-23T20:42:59.993 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 10.492815438999999, 2023-09-23T20:42:59.993 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:42:59.993 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:42:59.993 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:42:59.994 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:42:59.994 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:42:59.994 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 105788 2023-09-23T20:42:59.994 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:59.994 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:42:59.994 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:59.994 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:42:59.994 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.415518+0000", 2023-09-23T20:42:59.994 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:42:59.994 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:59.994 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:59.995 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:42:59.995 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.415518+0000", 2023-09-23T20:42:59.995 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:42:59.995 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:59.995 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:59.995 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:42:59.995 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.415517+0000", 2023-09-23T20:42:59.995 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:42:59.995 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:59.995 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:59.995 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:42:59.995 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.415524+0000", 2023-09-23T20:42:59.996 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.0530000000000001e-06 2023-09-23T20:42:59.996 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:59.996 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:59.996 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:42:59.996 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.415539+0000", 2023-09-23T20:42:59.996 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.4847e-05 2023-09-23T20:42:59.996 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:59.996 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:59.996 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:42:59.996 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.415548+0000", 2023-09-23T20:42:59.996 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.1600000000000004e-06 2023-09-23T20:42:59.996 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:59.997 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:59.997 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:42:59.997 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.423501+0000", 2023-09-23T20:42:59.997 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0079528459999999995 2023-09-23T20:42:59.997 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:59.997 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:59.997 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:42:59.997 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.423513+0000", 2023-09-23T20:42:59.997 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.2523e-05 2023-09-23T20:42:59.997 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:59.997 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:59.997 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:42:59.997 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.559074+0000", 2023-09-23T20:42:59.998 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.1355603290000005 2023-09-23T20:42:59.998 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:59.998 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:59.998 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for degraded object", 2023-09-23T20:42:59.998 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.567834+0000", 2023-09-23T20:42:59.998 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0087603060000000007 2023-09-23T20:42:59.998 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:59.998 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:59.998 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:42:59.998 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.866509+0000", 2023-09-23T20:42:59.998 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.2986752469999998 2023-09-23T20:42:59.998 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:59.999 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:59.999 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:42:59.999 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.869596+0000", 2023-09-23T20:42:59.999 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0030866829999999998 2023-09-23T20:42:59.999 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:59.999 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:59.999 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:42:59.999 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.875831+0000", 2023-09-23T20:42:59.999 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0062352570000000001 2023-09-23T20:42:59.999 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:42:59.999 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:42:59.999 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:43:00.000 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.893196+0000", 2023-09-23T20:43:00.000 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.017365463000000001 2023-09-23T20:43:00.000 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.000 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.000 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:43:00.000 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.898820+0000", 2023-09-23T20:43:00.000 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0056230400000000002 2023-09-23T20:43:00.000 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.000 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.000 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:43:00.000 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.899854+0000", 2023-09-23T20:43:00.000 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0010348670000000001 2023-09-23T20:43:00.000 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.001 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.001 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:43:00.001 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.901497+0000", 2023-09-23T20:43:00.001 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0016428160000000001 2023-09-23T20:43:00.001 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.001 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.001 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:43:00.001 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.908334+0000", 2023-09-23T20:43:00.001 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0068366219999999997 2023-09-23T20:43:00.001 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:43:00.001 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:43:00.001 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:43:00.001 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.002 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.002 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:105815 8.19 8:9f25a704:::benchmark_data_smithi018_341490_object16492:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e411)", 2023-09-23T20:43:00.002 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:38:27.621206+0000", 2023-09-23T20:43:00.002 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 272.342159192, 2023-09-23T20:43:00.002 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 10.285912701000001, 2023-09-23T20:43:00.002 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:43:00.002 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:43:00.002 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:43:00.002 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:43:00.002 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:43:00.003 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 105815 2023-09-23T20:43:00.003 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.003 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:43:00.003 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.003 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:43:00.003 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.621206+0000", 2023-09-23T20:43:00.003 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:43:00.003 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.003 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.003 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:43:00.003 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.621206+0000", 2023-09-23T20:43:00.003 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:43:00.004 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.004 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.004 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:43:00.004 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.621204+0000", 2023-09-23T20:43:00.004 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999981 2023-09-23T20:43:00.004 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.004 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.004 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:43:00.004 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.621213+0000", 2023-09-23T20:43:00.004 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.6399999999999992e-06 2023-09-23T20:43:00.004 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.004 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.005 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:43:00.005 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.621236+0000", 2023-09-23T20:43:00.005 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.2016000000000001e-05 2023-09-23T20:43:00.005 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.005 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.005 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:43:00.005 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.621244+0000", 2023-09-23T20:43:00.005 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.7339999999999997e-06 2023-09-23T20:43:00.005 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.005 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.005 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:43:00.006 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.621311+0000", 2023-09-23T20:43:00.006 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.7037000000000002e-05 2023-09-23T20:43:00.006 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.006 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.006 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:43:00.006 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.621338+0000", 2023-09-23T20:43:00.006 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.6506999999999999e-05 2023-09-23T20:43:00.006 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.006 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.006 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:43:00.006 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.559059+0000", 2023-09-23T20:43:00.007 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.9377208670000003 2023-09-23T20:43:00.007 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.007 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.007 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for degraded object", 2023-09-23T20:43:00.007 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.566949+0000", 2023-09-23T20:43:00.007 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0078899190000000004 2023-09-23T20:43:00.007 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.007 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.007 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:43:00.007 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.857769+0000", 2023-09-23T20:43:00.007 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.290820509 2023-09-23T20:43:00.007 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.007 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.008 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:43:00.008 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.860457+0000", 2023-09-23T20:43:00.008 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0026880620000000002 2023-09-23T20:43:00.008 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.008 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.008 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:43:00.008 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.865930+0000", 2023-09-23T20:43:00.008 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0054732979999999997 2023-09-23T20:43:00.008 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.008 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.008 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:43:00.008 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.881728+0000", 2023-09-23T20:43:00.009 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.015797813000000001 2023-09-23T20:43:00.009 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.009 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.009 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:43:00.009 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.897756+0000", 2023-09-23T20:43:00.009 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.016027501 2023-09-23T20:43:00.009 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.009 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.009 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:43:00.009 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.898850+0000", 2023-09-23T20:43:00.009 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.001094292 2023-09-23T20:43:00.010 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.010 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.010 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:43:00.010 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.900364+0000", 2023-09-23T20:43:00.010 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0015141250000000001 2023-09-23T20:43:00.010 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.010 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.010 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:43:00.010 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.907118+0000", 2023-09-23T20:43:00.010 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0067541490000000001 2023-09-23T20:43:00.010 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:43:00.011 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:43:00.011 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:43:00.011 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.011 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.011 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:105818 8.19 8:9e8d5bb0:::benchmark_data_smithi018_341490_object16495:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e411)", 2023-09-23T20:43:00.011 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:38:27.629200+0000", 2023-09-23T20:43:00.011 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 272.334165111, 2023-09-23T20:43:00.011 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 10.286819241, 2023-09-23T20:43:00.011 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:43:00.011 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:43:00.011 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:43:00.012 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:43:00.012 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:43:00.012 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 105818 2023-09-23T20:43:00.012 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.012 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:43:00.012 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.012 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:43:00.012 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.629200+0000", 2023-09-23T20:43:00.012 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:43:00.012 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.013 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.013 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:43:00.013 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.629200+0000", 2023-09-23T20:43:00.013 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:43:00.013 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.013 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.013 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:43:00.013 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.629198+0000", 2023-09-23T20:43:00.013 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999981 2023-09-23T20:43:00.013 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.013 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.014 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:43:00.014 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.629208+0000", 2023-09-23T20:43:00.014 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.9119999999999993e-06 2023-09-23T20:43:00.014 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.014 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.014 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:43:00.014 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.629231+0000", 2023-09-23T20:43:00.014 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.3258999999999999e-05 2023-09-23T20:43:00.014 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.014 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.014 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:43:00.015 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.629240+0000", 2023-09-23T20:43:00.015 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.5719999999999992e-06 2023-09-23T20:43:00.015 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.015 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.015 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:43:00.015 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.629296+0000", 2023-09-23T20:43:00.015 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.5918999999999998e-05 2023-09-23T20:43:00.015 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.015 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.015 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:43:00.015 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.629305+0000", 2023-09-23T20:43:00.015 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.0909999999999999e-06 2023-09-23T20:43:00.016 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.016 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.016 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:43:00.016 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.567292+0000", 2023-09-23T20:43:00.016 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.937986414 2023-09-23T20:43:00.016 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.016 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.016 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for degraded object", 2023-09-23T20:43:00.016 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.576261+0000", 2023-09-23T20:43:00.016 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0089692720000000004 2023-09-23T20:43:00.016 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.016 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.017 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:43:00.017 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.882072+0000", 2023-09-23T20:43:00.017 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.3058112469999998 2023-09-23T20:43:00.017 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.017 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.017 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:43:00.017 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.884583+0000", 2023-09-23T20:43:00.017 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0025109939999999999 2023-09-23T20:43:00.017 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.017 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.017 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:43:00.018 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.889724+0000", 2023-09-23T20:43:00.018 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0051411909999999998 2023-09-23T20:43:00.018 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.018 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.018 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:43:00.018 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.894676+0000", 2023-09-23T20:43:00.018 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0049515119999999999 2023-09-23T20:43:00.018 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.018 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.018 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:43:00.018 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.908009+0000", 2023-09-23T20:43:00.019 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.013332899 2023-09-23T20:43:00.019 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.019 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.019 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:43:00.019 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.908929+0000", 2023-09-23T20:43:00.019 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00092006300000000004 2023-09-23T20:43:00.019 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.019 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.019 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:43:00.019 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.910490+0000", 2023-09-23T20:43:00.019 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.001560931 2023-09-23T20:43:00.020 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.020 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.020 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:43:00.020 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.916019+0000", 2023-09-23T20:43:00.020 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0055290570000000004 2023-09-23T20:43:00.020 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:43:00.020 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:43:00.020 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:43:00.020 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.020 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.020 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "MOSDPGPushReply(8.19 413/410 [PushReplyOp(8:9f25a704:::benchmark_data_smithi018_341490_object16492:head)])", 2023-09-23T20:43:00.021 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:38:35.569213+0000", 2023-09-23T20:43:00.021 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 264.39415182400001, 2023-09-23T20:43:00.021 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.2881636309999998, 2023-09-23T20:43:00.021 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:43:00.021 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:43:00.021 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:43:00.021 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.021 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:43:00.021 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.569213+0000", 2023-09-23T20:43:00.021 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:43:00.021 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.021 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.022 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:43:00.022 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.569213+0000", 2023-09-23T20:43:00.022 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:43:00.022 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.022 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.022 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:43:00.022 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.569213+0000", 2023-09-23T20:43:00.022 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:43:00.022 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.022 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.022 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:43:00.022 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.569217+0000", 2023-09-23T20:43:00.023 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.5480000000000003e-06 2023-09-23T20:43:00.023 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.023 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.023 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:43:00.023 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.569231+0000", 2023-09-23T20:43:00.023 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.3505e-05 2023-09-23T20:43:00.023 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.023 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.023 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:43:00.023 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.569241+0000", 2023-09-23T20:43:00.023 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0695e-05 2023-09-23T20:43:00.023 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.024 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.024 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:43:00.024 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.845386+0000", 2023-09-23T20:43:00.024 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.2761449049999998 2023-09-23T20:43:00.024 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.024 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.024 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:43:00.024 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.857377+0000", 2023-09-23T20:43:00.024 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.011990341999999999 2023-09-23T20:43:00.024 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:43:00.024 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:43:00.024 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:43:00.025 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.025 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.025 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "MOSDPGPushReply(8.9 413/411 [PushReplyOp(8:94f5ae14:::benchmark_data_smithi018_341490_object16465:head)])", 2023-09-23T20:43:00.025 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:38:35.570981+0000", 2023-09-23T20:43:00.025 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 264.39238363999999, 2023-09-23T20:43:00.025 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.295177131, 2023-09-23T20:43:00.025 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:43:00.025 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:43:00.025 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:43:00.025 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.025 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:43:00.025 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.570981+0000", 2023-09-23T20:43:00.026 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:43:00.026 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.026 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.026 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:43:00.026 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.570981+0000", 2023-09-23T20:43:00.026 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:43:00.026 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.026 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.026 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:43:00.026 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.570981+0000", 2023-09-23T20:43:00.026 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:43:00.027 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.027 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.027 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:43:00.027 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.570985+0000", 2023-09-23T20:43:00.027 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.2690000000000001e-06 2023-09-23T20:43:00.027 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.027 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.027 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:43:00.027 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.570995+0000", 2023-09-23T20:43:00.027 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.9799999999999993e-06 2023-09-23T20:43:00.027 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.027 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.027 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:43:00.028 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.570999+0000", 2023-09-23T20:43:00.028 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.5739999999999997e-06 2023-09-23T20:43:00.028 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.028 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.028 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:43:00.028 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.852442+0000", 2023-09-23T20:43:00.028 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.2814426320000001 2023-09-23T20:43:00.028 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.028 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.028 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:43:00.029 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.866158+0000", 2023-09-23T20:43:00.029 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.013716178000000001 2023-09-23T20:43:00.029 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:43:00.029 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:43:00.029 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:43:00.029 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.029 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.029 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "MOSDPGPushReply(8.19 413/410 [PushReplyOp(8:9e8d5bb0:::benchmark_data_smithi018_341490_object16495:head)])", 2023-09-23T20:43:00.029 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:38:35.577963+0000", 2023-09-23T20:43:00.029 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 264.38540148300001, 2023-09-23T20:43:00.029 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.303767224, 2023-09-23T20:43:00.029 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:43:00.030 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:43:00.030 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:43:00.030 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.030 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:43:00.030 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.577963+0000", 2023-09-23T20:43:00.030 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:43:00.030 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.030 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.030 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:43:00.030 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.577963+0000", 2023-09-23T20:43:00.030 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:43:00.031 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.031 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.031 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:43:00.031 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.577963+0000", 2023-09-23T20:43:00.031 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:43:00.031 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.031 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.031 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:43:00.031 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.577968+0000", 2023-09-23T20:43:00.031 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.13e-06 2023-09-23T20:43:00.031 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.031 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.032 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:43:00.032 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.577980+0000", 2023-09-23T20:43:00.032 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1749000000000001e-05 2023-09-23T20:43:00.032 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.032 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.032 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:43:00.032 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.577986+0000", 2023-09-23T20:43:00.032 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.8210000000000004e-06 2023-09-23T20:43:00.032 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.032 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.032 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:43:00.032 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.869571+0000", 2023-09-23T20:43:00.032 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.2915846719999999 2023-09-23T20:43:00.033 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.033 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.033 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:43:00.033 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.881731+0000", 2023-09-23T20:43:00.033 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.012159467 2023-09-23T20:43:00.033 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:43:00.033 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:43:00.033 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:43:00.033 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.033 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.033 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:106043 8.9 8:949649ba:::benchmark_data_smithi018_341490_object16720:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e414)", 2023-09-23T20:43:00.033 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:38:35.582779+0000", 2023-09-23T20:43:00.034 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 264.38058578900001, 2023-09-23T20:43:00.034 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.7561004659999999, 2023-09-23T20:43:00.034 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:43:00.034 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:43:00.034 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:43:00.034 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:43:00.034 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:43:00.034 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 106043 2023-09-23T20:43:00.034 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.034 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:43:00.034 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.035 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:43:00.035 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.582779+0000", 2023-09-23T20:43:00.035 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:43:00.035 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.035 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.035 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:43:00.035 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.582779+0000", 2023-09-23T20:43:00.035 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:43:00.035 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.035 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.035 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:43:00.036 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.582778+0000", 2023-09-23T20:43:00.036 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:43:00.036 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.036 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.036 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:43:00.036 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.582784+0000", 2023-09-23T20:43:00.036 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.0050000000000001e-06 2023-09-23T20:43:00.036 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.036 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.036 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:43:00.036 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.582798+0000", 2023-09-23T20:43:00.036 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.3788000000000001e-05 2023-09-23T20:43:00.037 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.037 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.037 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:43:00.037 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.582804+0000", 2023-09-23T20:43:00.037 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.6760000000000001e-06 2023-09-23T20:43:00.037 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.037 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.037 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:43:00.037 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.238120+0000", 2023-09-23T20:43:00.037 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.655315308 2023-09-23T20:43:00.037 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.037 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.038 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for degraded object", 2023-09-23T20:43:00.038 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.246651+0000", 2023-09-23T20:43:00.038 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0085312070000000007 2023-09-23T20:43:00.038 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.038 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.038 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:43:00.038 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.290876+0000", 2023-09-23T20:43:00.038 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.044225546999999997 2023-09-23T20:43:00.038 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.038 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.039 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:43:00.039 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.294002+0000", 2023-09-23T20:43:00.039 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0031256779999999998 2023-09-23T20:43:00.039 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.039 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.039 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:43:00.039 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.300293+0000", 2023-09-23T20:43:00.039 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0062908959999999998 2023-09-23T20:43:00.039 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.039 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.039 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:43:00.040 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.328008+0000", 2023-09-23T20:43:00.040 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.027715281000000001 2023-09-23T20:43:00.040 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.040 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.040 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:43:00.040 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.329088+0000", 2023-09-23T20:43:00.040 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.001079775 2023-09-23T20:43:00.040 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.040 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.040 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:43:00.040 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.330303+0000", 2023-09-23T20:43:00.041 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0012148199999999999 2023-09-23T20:43:00.041 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.041 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.041 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:43:00.041 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.332003+0000", 2023-09-23T20:43:00.041 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0016998580000000001 2023-09-23T20:43:00.041 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.041 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.041 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:43:00.041 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.338879+0000", 2023-09-23T20:43:00.041 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0068769119999999998 2023-09-23T20:43:00.042 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:43:00.042 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:43:00.042 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:43:00.042 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.042 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.042 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "MOSDPGPush(8.19 427/410 [PushOp(8:9f37acf0:::benchmark_data_smithi018_341490_object84819:head, version: 394'5338, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(8:9f37acf0:::benchmark_data_smithi018_341490_object84819:head@394'5338, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-09-23T20:43:00.042 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:39:26.795989+0000", 2023-09-23T20:43:00.042 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 213.167375623, 2023-09-23T20:43:00.042 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 18.381641347999999, 2023-09-23T20:43:00.042 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:43:00.042 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "started", 2023-09-23T20:43:00.043 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:43:00.043 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.043 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:43:00.043 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:26.795989+0000", 2023-09-23T20:43:00.043 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:43:00.043 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.043 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.043 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:43:00.043 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:26.795989+0000", 2023-09-23T20:43:00.043 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:43:00.043 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.044 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.044 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:43:00.044 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:26.795988+0000", 2023-09-23T20:43:00.044 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:43:00.044 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.044 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.044 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:43:00.044 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:26.796034+0000", 2023-09-23T20:43:00.044 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.5374999999999998e-05 2023-09-23T20:43:00.044 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.044 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.045 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:43:00.045 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:26.796055+0000", 2023-09-23T20:43:00.045 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.126e-05 2023-09-23T20:43:00.045 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.045 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.045 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:43:00.045 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:26.796067+0000", 2023-09-23T20:43:00.045 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.2043e-05 2023-09-23T20:43:00.045 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.045 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.045 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:43:00.046 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:45.167841+0000", 2023-09-23T20:43:00.046 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 18.371773717 2023-09-23T20:43:00.046 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.046 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.046 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:43:00.046 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:45.168079+0000", 2023-09-23T20:43:00.046 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000237839 2023-09-23T20:43:00.046 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.046 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.046 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:43:00.046 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:45.177631+0000", 2023-09-23T20:43:00.047 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0095519390000000006 2023-09-23T20:43:00.047 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:43:00.047 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:43:00.047 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:43:00.047 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.047 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.047 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "MOSDPGPush(8.19 435/410 [PushOp(8:9a33faf0:::benchmark_data_smithi018_341490_object85471:head, version: 395'5385, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(8:9a33faf0:::benchmark_data_smithi018_341490_object85471:head@395'5385, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-09-23T20:43:00.047 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:39:53.671016+0000", 2023-09-23T20:43:00.047 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 186.292348764, 2023-09-23T20:43:00.047 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.82055882099999999, 2023-09-23T20:43:00.047 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:43:00.047 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "started", 2023-09-23T20:43:00.048 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:43:00.048 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.048 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:43:00.048 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:53.671016+0000", 2023-09-23T20:43:00.048 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:43:00.048 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.048 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.048 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:43:00.048 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:53.671016+0000", 2023-09-23T20:43:00.048 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:43:00.048 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.048 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.048 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:43:00.049 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:53.671015+0000", 2023-09-23T20:43:00.049 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:43:00.049 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.049 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.049 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:43:00.049 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:53.671046+0000", 2023-09-23T20:43:00.049 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.0802000000000001e-05 2023-09-23T20:43:00.049 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.049 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.049 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:43:00.049 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:53.671061+0000", 2023-09-23T20:43:00.049 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.5446000000000001e-05 2023-09-23T20:43:00.050 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.050 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.050 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:43:00.050 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:53.671069+0000", 2023-09-23T20:43:00.050 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.9519999999999994e-06 2023-09-23T20:43:00.050 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.050 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.050 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:43:00.050 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:54.483764+0000", 2023-09-23T20:43:00.050 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.812694429 2023-09-23T20:43:00.051 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.051 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.051 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:43:00.051 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:54.483953+0000", 2023-09-23T20:43:00.051 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00018980099999999999 2023-09-23T20:43:00.051 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.051 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.051 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:43:00.051 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:54.491575+0000", 2023-09-23T20:43:00.051 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0076214999999999998 2023-09-23T20:43:00.051 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:43:00.051 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:43:00.051 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:43:00.052 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.052 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.052 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:76949 9.b 9:dc3199d0:::benchmark_data_smithi018_381516_object76948:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e465)", 2023-09-23T20:43:00.052 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:41:58.481021+0000", 2023-09-23T20:43:00.052 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 61.482343813999996, 2023-09-23T20:43:00.052 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.012473631, 2023-09-23T20:43:00.052 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:43:00.052 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:43:00.052 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:43:00.052 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:43:00.052 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:43:00.052 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 76949 2023-09-23T20:43:00.053 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.053 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:43:00.053 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.053 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:43:00.053 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.481021+0000", 2023-09-23T20:43:00.053 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:43:00.053 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.053 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.053 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:43:00.053 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.481021+0000", 2023-09-23T20:43:00.053 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:43:00.054 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.054 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.054 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:43:00.054 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.481020+0000", 2023-09-23T20:43:00.054 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:43:00.054 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.054 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.054 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:43:00.054 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.481102+0000", 2023-09-23T20:43:00.054 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.2538000000000002e-05 2023-09-23T20:43:00.054 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.055 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.055 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:43:00.055 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.481122+0000", 2023-09-23T20:43:00.055 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.9556e-05 2023-09-23T20:43:00.055 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.055 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.055 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:43:00.055 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.481129+0000", 2023-09-23T20:43:00.055 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.7449999999999995e-06 2023-09-23T20:43:00.055 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.055 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.055 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:43:00.056 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.481713+0000", 2023-09-23T20:43:00.056 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000583167 2023-09-23T20:43:00.056 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.056 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.056 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:43:00.056 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.481726+0000", 2023-09-23T20:43:00.056 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.3291e-05 2023-09-23T20:43:00.056 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.056 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.056 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:43:00.056 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.467637+0000", 2023-09-23T20:43:00.056 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.98591135500000004 2023-09-23T20:43:00.057 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.057 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.057 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:43:00.057 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.470079+0000", 2023-09-23T20:43:00.057 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0024414089999999999 2023-09-23T20:43:00.057 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.057 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.057 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:43:00.057 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.470267+0000", 2023-09-23T20:43:00.057 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00018824299999999999 2023-09-23T20:43:00.057 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.057 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.057 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:43:00.058 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.481273+0000", 2023-09-23T20:43:00.058 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.011006395 2023-09-23T20:43:00.058 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.058 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.058 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:43:00.058 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.493371+0000", 2023-09-23T20:43:00.058 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.012098138 2023-09-23T20:43:00.058 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.058 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.058 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:43:00.059 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.493433+0000", 2023-09-23T20:43:00.059 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.1437000000000001e-05 2023-09-23T20:43:00.059 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.059 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.059 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:43:00.059 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.493472+0000", 2023-09-23T20:43:00.059 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.9038000000000001e-05 2023-09-23T20:43:00.059 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.059 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.059 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:43:00.059 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.493495+0000", 2023-09-23T20:43:00.060 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.2778000000000001e-05 2023-09-23T20:43:00.060 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:43:00.060 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:43:00.060 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:43:00.060 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.060 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.060 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:76967 9.b 9:d898a916:::benchmark_data_smithi018_381516_object76966:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e465)", 2023-09-23T20:43:00.060 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:41:58.507229+0000", 2023-09-23T20:43:00.060 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 61.456135404000001, 2023-09-23T20:43:00.060 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.98635601299999998, 2023-09-23T20:43:00.060 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:43:00.061 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:43:00.061 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:43:00.061 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:43:00.061 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:43:00.061 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 76967 2023-09-23T20:43:00.061 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.061 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:43:00.061 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.061 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:43:00.061 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.507229+0000", 2023-09-23T20:43:00.061 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:43:00.062 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.062 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.062 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:43:00.062 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.507229+0000", 2023-09-23T20:43:00.062 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:43:00.062 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.062 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.062 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:43:00.062 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.507228+0000", 2023-09-23T20:43:00.062 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999981 2023-09-23T20:43:00.063 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.063 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.063 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:43:00.063 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.507285+0000", 2023-09-23T20:43:00.063 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.7531000000000002e-05 2023-09-23T20:43:00.063 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.063 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.063 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:43:00.063 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.507307+0000", 2023-09-23T20:43:00.063 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.2039e-05 2023-09-23T20:43:00.063 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.064 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.064 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:43:00.064 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.507314+0000", 2023-09-23T20:43:00.064 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.7089999999999996e-06 2023-09-23T20:43:00.064 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.064 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.064 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:43:00.064 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.507629+0000", 2023-09-23T20:43:00.064 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00031477799999999999 2023-09-23T20:43:00.064 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.064 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.065 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:43:00.065 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.507641+0000", 2023-09-23T20:43:00.065 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.258e-05 2023-09-23T20:43:00.065 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.065 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.065 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:43:00.065 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.472754+0000", 2023-09-23T20:43:00.065 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.96511288200000001 2023-09-23T20:43:00.065 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.065 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.065 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:43:00.066 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.472887+0000", 2023-09-23T20:43:00.066 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000133132 2023-09-23T20:43:00.066 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.066 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.066 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:43:00.066 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.472997+0000", 2023-09-23T20:43:00.066 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000109805 2023-09-23T20:43:00.066 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.066 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.066 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:43:00.066 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.484044+0000", 2023-09-23T20:43:00.067 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.011047250999999999 2023-09-23T20:43:00.067 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.067 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.067 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:43:00.067 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.493403+0000", 2023-09-23T20:43:00.067 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0093591230000000004 2023-09-23T20:43:00.067 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.067 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.067 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:43:00.067 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.493533+0000", 2023-09-23T20:43:00.067 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000129271 2023-09-23T20:43:00.068 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.068 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.068 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:43:00.068 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.493558+0000", 2023-09-23T20:43:00.068 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.5477e-05 2023-09-23T20:43:00.068 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.068 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.068 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:43:00.068 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.493585+0000", 2023-09-23T20:43:00.068 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.7274e-05 2023-09-23T20:43:00.068 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:43:00.068 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:43:00.068 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:43:00.069 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.069 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.069 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:76968 9.b 9:d3229d21:::benchmark_data_smithi018_381516_object76967:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e465)", 2023-09-23T20:43:00.069 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:41:58.507386+0000", 2023-09-23T20:43:00.069 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 61.455978471000002, 2023-09-23T20:43:00.069 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.98627873399999999, 2023-09-23T20:43:00.069 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:43:00.069 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:43:00.069 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:43:00.069 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:43:00.069 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:43:00.069 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 76968 2023-09-23T20:43:00.070 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.070 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:43:00.070 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.070 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:43:00.070 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.507386+0000", 2023-09-23T20:43:00.070 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:43:00.070 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.070 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.070 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:43:00.070 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.507386+0000", 2023-09-23T20:43:00.070 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:43:00.071 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.071 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.071 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:43:00.071 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.507386+0000", 2023-09-23T20:43:00.071 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:43:00.071 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.071 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.071 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:43:00.071 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.507447+0000", 2023-09-23T20:43:00.071 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.1439999999999995e-05 2023-09-23T20:43:00.071 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.071 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.072 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:43:00.072 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.507457+0000", 2023-09-23T20:43:00.072 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.4679999999999998e-06 2023-09-23T20:43:00.072 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.072 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.072 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:43:00.072 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.507463+0000", 2023-09-23T20:43:00.072 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.8440000000000003e-06 2023-09-23T20:43:00.072 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.072 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.072 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:43:00.072 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.507659+0000", 2023-09-23T20:43:00.073 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000196691 2023-09-23T20:43:00.073 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.073 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.073 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:43:00.073 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.507663+0000", 2023-09-23T20:43:00.073 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.0040000000000001e-06 2023-09-23T20:43:00.073 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.073 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.073 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:43:00.073 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.475638+0000", 2023-09-23T20:43:00.073 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.96797460700000004 2023-09-23T20:43:00.074 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.074 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.074 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:43:00.074 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.475775+0000", 2023-09-23T20:43:00.074 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00013739899999999999 2023-09-23T20:43:00.074 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.074 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.074 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:43:00.074 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.475934+0000", 2023-09-23T20:43:00.074 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00015842499999999999 2023-09-23T20:43:00.074 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.074 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.075 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:43:00.075 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.484048+0000", 2023-09-23T20:43:00.075 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0081147540000000001 2023-09-23T20:43:00.075 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.075 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.075 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:43:00.075 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.493516+0000", 2023-09-23T20:43:00.075 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0094679240000000008 2023-09-23T20:43:00.075 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.075 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.075 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:43:00.075 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.493605+0000", 2023-09-23T20:43:00.075 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.8725000000000002e-05 2023-09-23T20:43:00.076 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.076 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.076 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:43:00.076 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.493633+0000", 2023-09-23T20:43:00.076 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.8e-05 2023-09-23T20:43:00.076 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.076 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.076 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:43:00.076 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.493665+0000", 2023-09-23T20:43:00.076 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.2017000000000002e-05 2023-09-23T20:43:00.076 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:43:00.076 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:43:00.076 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:43:00.077 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.077 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.077 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:76972 9.b 9:d979209f:::benchmark_data_smithi018_381516_object76971:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e465)", 2023-09-23T20:43:00.077 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:41:58.507469+0000", 2023-09-23T20:43:00.077 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 61.455895871999999, 2023-09-23T20:43:00.077 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.98628621599999999, 2023-09-23T20:43:00.077 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:43:00.077 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:43:00.077 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:43:00.077 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:43:00.078 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:43:00.078 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 76972 2023-09-23T20:43:00.078 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.078 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:43:00.078 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.078 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:43:00.078 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.507469+0000", 2023-09-23T20:43:00.078 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:43:00.078 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.078 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.078 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:43:00.078 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.507469+0000", 2023-09-23T20:43:00.079 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:43:00.079 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.079 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.079 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:43:00.079 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.507468+0000", 2023-09-23T20:43:00.079 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:43:00.079 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.079 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.079 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:43:00.079 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.507525+0000", 2023-09-23T20:43:00.079 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.6431000000000002e-05 2023-09-23T20:43:00.079 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.080 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.080 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:43:00.080 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.507533+0000", 2023-09-23T20:43:00.080 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.2400000000000007e-06 2023-09-23T20:43:00.080 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.080 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.080 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:43:00.080 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.507538+0000", 2023-09-23T20:43:00.080 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.2129999999999999e-06 2023-09-23T20:43:00.080 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.080 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.080 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:43:00.081 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.507674+0000", 2023-09-23T20:43:00.081 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00013521599999999999 2023-09-23T20:43:00.081 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.081 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.081 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:43:00.081 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.507677+0000", 2023-09-23T20:43:00.081 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.3409999999999999e-06 2023-09-23T20:43:00.081 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.081 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.081 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:43:00.081 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.478437+0000", 2023-09-23T20:43:00.082 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.97076035400000005 2023-09-23T20:43:00.082 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.082 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.082 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:43:00.082 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.478570+0000", 2023-09-23T20:43:00.082 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00013248599999999999 2023-09-23T20:43:00.082 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.082 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.082 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:43:00.082 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.478661+0000", 2023-09-23T20:43:00.082 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.1013000000000004e-05 2023-09-23T20:43:00.082 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.083 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.083 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:43:00.083 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.486788+0000", 2023-09-23T20:43:00.083 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0081275240000000006 2023-09-23T20:43:00.083 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.083 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.083 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:43:00.083 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.493688+0000", 2023-09-23T20:43:00.083 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0068995159999999996 2023-09-23T20:43:00.083 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.083 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.084 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:43:00.084 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.493704+0000", 2023-09-23T20:43:00.084 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.6441999999999999e-05 2023-09-23T20:43:00.084 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.084 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.084 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:43:00.084 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.493734+0000", 2023-09-23T20:43:00.084 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.9258999999999999e-05 2023-09-23T20:43:00.084 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.084 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.084 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:43:00.084 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.493755+0000", 2023-09-23T20:43:00.085 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.1636000000000001e-05 2023-09-23T20:43:00.085 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:43:00.085 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:43:00.085 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:43:00.085 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.085 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.085 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:76982 9.b 9:db90636b:::benchmark_data_smithi018_381516_object76981:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e465)", 2023-09-23T20:43:00.085 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:41:58.524610+0000", 2023-09-23T20:43:00.085 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 61.438755161000003, 2023-09-23T20:43:00.085 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.97237829799999997, 2023-09-23T20:43:00.086 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:43:00.086 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:43:00.086 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:43:00.086 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:43:00.086 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:43:00.086 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 76982 2023-09-23T20:43:00.086 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.086 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:43:00.086 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.086 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:43:00.086 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.524610+0000", 2023-09-23T20:43:00.087 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:43:00.087 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.087 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.087 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:43:00.087 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.524610+0000", 2023-09-23T20:43:00.087 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:43:00.087 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.087 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.087 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:43:00.087 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.524608+0000", 2023-09-23T20:43:00.087 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:43:00.087 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.088 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.088 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:43:00.088 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.524642+0000", 2023-09-23T20:43:00.088 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.4152000000000001e-05 2023-09-23T20:43:00.088 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.088 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.088 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:43:00.088 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.524659+0000", 2023-09-23T20:43:00.088 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.7108e-05 2023-09-23T20:43:00.088 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.088 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.088 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:43:00.089 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.524666+0000", 2023-09-23T20:43:00.089 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.0489999999999998e-06 2023-09-23T20:43:00.089 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.089 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.089 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:43:00.089 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.524731+0000", 2023-09-23T20:43:00.089 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.4169000000000002e-05 2023-09-23T20:43:00.089 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.089 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.089 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:43:00.089 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.524743+0000", 2023-09-23T20:43:00.089 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.2434e-05 2023-09-23T20:43:00.089 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.090 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.090 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:43:00.090 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.481305+0000", 2023-09-23T20:43:00.090 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.95656189599999997 2023-09-23T20:43:00.090 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.090 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.090 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:43:00.090 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.481433+0000", 2023-09-23T20:43:00.090 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000128059 2023-09-23T20:43:00.090 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.090 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.091 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:43:00.091 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.481584+0000", 2023-09-23T20:43:00.091 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00015048899999999999 2023-09-23T20:43:00.091 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.091 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.091 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:43:00.091 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.493780+0000", 2023-09-23T20:43:00.091 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.012196716 2023-09-23T20:43:00.091 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.091 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.091 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:43:00.091 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.493806+0000", 2023-09-23T20:43:00.092 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.6188999999999999e-05 2023-09-23T20:43:00.092 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.092 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.092 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:43:00.092 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.496920+0000", 2023-09-23T20:43:00.092 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.003113111 2023-09-23T20:43:00.092 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.092 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.092 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:43:00.092 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.496963+0000", 2023-09-23T20:43:00.092 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.3648e-05 2023-09-23T20:43:00.093 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.093 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.093 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:43:00.093 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.496988+0000", 2023-09-23T20:43:00.093 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.4782000000000001e-05 2023-09-23T20:43:00.093 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:43:00.093 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:43:00.093 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:43:00.093 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.093 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.093 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:77009 9.b 9:db9e621f:::benchmark_data_smithi018_381516_object77008:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e465)", 2023-09-23T20:43:00.094 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:41:58.547295+0000", 2023-09-23T20:43:00.094 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 61.416069755000002, 2023-09-23T20:43:00.094 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.94976649700000004, 2023-09-23T20:43:00.094 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:43:00.094 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:43:00.094 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:43:00.094 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:43:00.094 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:43:00.094 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 77009 2023-09-23T20:43:00.094 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.094 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:43:00.095 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.095 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:43:00.095 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.547295+0000", 2023-09-23T20:43:00.095 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:43:00.095 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.095 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.095 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:43:00.095 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.547295+0000", 2023-09-23T20:43:00.095 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:43:00.095 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.095 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.095 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:43:00.096 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.547293+0000", 2023-09-23T20:43:00.096 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999976 2023-09-23T20:43:00.096 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.096 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.096 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:43:00.096 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.547385+0000", 2023-09-23T20:43:00.096 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.2566999999999997e-05 2023-09-23T20:43:00.096 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.096 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.096 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:43:00.096 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.547409+0000", 2023-09-23T20:43:00.096 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.3802999999999999e-05 2023-09-23T20:43:00.097 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.097 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.097 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:43:00.097 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.547418+0000", 2023-09-23T20:43:00.097 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.7509999999999997e-06 2023-09-23T20:43:00.097 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.097 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.097 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:43:00.097 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.547486+0000", 2023-09-23T20:43:00.097 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.7928000000000004e-05 2023-09-23T20:43:00.097 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.097 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.097 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:43:00.098 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.547499+0000", 2023-09-23T20:43:00.098 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.3556e-05 2023-09-23T20:43:00.098 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.098 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.098 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:43:00.098 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.484068+0000", 2023-09-23T20:43:00.098 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.93656838200000003 2023-09-23T20:43:00.098 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.098 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.098 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:43:00.098 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.484183+0000", 2023-09-23T20:43:00.099 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000115436 2023-09-23T20:43:00.099 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.099 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.099 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:43:00.099 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.484306+0000", 2023-09-23T20:43:00.099 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00012274399999999999 2023-09-23T20:43:00.099 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.099 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.099 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:43:00.099 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.493822+0000", 2023-09-23T20:43:00.099 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.009516142 2023-09-23T20:43:00.099 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.100 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.100 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:43:00.100 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.493837+0000", 2023-09-23T20:43:00.100 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.4698000000000001e-05 2023-09-23T20:43:00.100 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.100 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.100 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:43:00.100 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.496999+0000", 2023-09-23T20:43:00.100 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0031623770000000001 2023-09-23T20:43:00.100 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.100 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.100 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:43:00.100 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.497034+0000", 2023-09-23T20:43:00.101 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.4620999999999999e-05 2023-09-23T20:43:00.101 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.101 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.101 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:43:00.101 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.497062+0000", 2023-09-23T20:43:00.101 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.7654e-05 2023-09-23T20:43:00.101 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:43:00.101 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:43:00.101 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:43:00.101 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.101 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.102 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:77014 9.b 9:dc5d982b:::benchmark_data_smithi018_381516_object77013:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e465)", 2023-09-23T20:43:00.102 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:41:58.554929+0000", 2023-09-23T20:43:00.102 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 61.408436092999999, 2023-09-23T20:43:00.102 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.94218858999999999, 2023-09-23T20:43:00.102 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:43:00.102 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:43:00.102 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:43:00.102 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:43:00.102 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:43:00.102 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 77014 2023-09-23T20:43:00.102 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.103 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:43:00.103 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.103 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:43:00.103 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.554929+0000", 2023-09-23T20:43:00.103 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:43:00.103 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.103 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.103 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:43:00.103 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.554929+0000", 2023-09-23T20:43:00.103 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:43:00.103 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.103 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.104 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:43:00.104 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.554927+0000", 2023-09-23T20:43:00.104 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999981 2023-09-23T20:43:00.104 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.104 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.104 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:43:00.104 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.554989+0000", 2023-09-23T20:43:00.104 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.2597999999999999e-05 2023-09-23T20:43:00.104 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.104 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.104 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:43:00.105 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.555008+0000", 2023-09-23T20:43:00.105 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.8382000000000001e-05 2023-09-23T20:43:00.105 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.105 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.105 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:43:00.105 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.555014+0000", 2023-09-23T20:43:00.105 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.336e-06 2023-09-23T20:43:00.105 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.105 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.105 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:43:00.105 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.555098+0000", 2023-09-23T20:43:00.106 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.4109000000000004e-05 2023-09-23T20:43:00.106 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.106 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.106 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:43:00.106 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.555119+0000", 2023-09-23T20:43:00.106 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.0341000000000001e-05 2023-09-23T20:43:00.106 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.106 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.106 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:43:00.106 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.486855+0000", 2023-09-23T20:43:00.106 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.93173672500000004 2023-09-23T20:43:00.107 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.107 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.107 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:43:00.107 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.486994+0000", 2023-09-23T20:43:00.107 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000138243 2023-09-23T20:43:00.107 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.107 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.107 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:43:00.107 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.487116+0000", 2023-09-23T20:43:00.107 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000122268 2023-09-23T20:43:00.107 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.107 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.108 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:43:00.108 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.493852+0000", 2023-09-23T20:43:00.108 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0067365419999999999 2023-09-23T20:43:00.108 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.108 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.108 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:43:00.108 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.493874+0000", 2023-09-23T20:43:00.108 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.1189999999999999e-05 2023-09-23T20:43:00.108 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.108 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.108 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:43:00.108 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.497069+0000", 2023-09-23T20:43:00.109 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0031949600000000002 2023-09-23T20:43:00.109 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.109 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.109 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:43:00.109 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.497094+0000", 2023-09-23T20:43:00.109 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.5075999999999998e-05 2023-09-23T20:43:00.109 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.109 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.109 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:43:00.109 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.497117+0000", 2023-09-23T20:43:00.109 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.3669e-05 2023-09-23T20:43:00.109 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:43:00.109 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:43:00.110 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:43:00.110 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.110 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.110 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:77023 9.b 9:d3d436f2:::benchmark_data_smithi018_381516_object77022:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e465)", 2023-09-23T20:43:00.110 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:41:58.571188+0000", 2023-09-23T20:43:00.110 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 61.39217687, 2023-09-23T20:43:00.110 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.92609987900000001, 2023-09-23T20:43:00.110 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:43:00.110 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:43:00.110 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:43:00.110 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:43:00.111 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:43:00.111 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 77023 2023-09-23T20:43:00.111 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.111 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:43:00.111 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.111 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:43:00.111 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.571188+0000", 2023-09-23T20:43:00.111 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:43:00.111 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.111 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.111 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:43:00.112 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.571188+0000", 2023-09-23T20:43:00.112 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:43:00.112 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.112 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.112 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:43:00.112 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.571186+0000", 2023-09-23T20:43:00.112 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999981 2023-09-23T20:43:00.112 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.112 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.112 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:43:00.112 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.571248+0000", 2023-09-23T20:43:00.112 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.1997999999999998e-05 2023-09-23T20:43:00.113 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.113 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.113 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:43:00.113 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.571266+0000", 2023-09-23T20:43:00.113 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.7788999999999999e-05 2023-09-23T20:43:00.113 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.113 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.113 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:43:00.113 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.571273+0000", 2023-09-23T20:43:00.113 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.5910000000000001e-06 2023-09-23T20:43:00.113 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.114 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.114 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:43:00.114 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.571335+0000", 2023-09-23T20:43:00.114 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.2872000000000004e-05 2023-09-23T20:43:00.114 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.114 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.114 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:43:00.114 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.571346+0000", 2023-09-23T20:43:00.114 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0069e-05 2023-09-23T20:43:00.114 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.114 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.115 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:43:00.115 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.489960+0000", 2023-09-23T20:43:00.115 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.918614017 2023-09-23T20:43:00.115 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.115 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.115 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:43:00.115 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.490095+0000", 2023-09-23T20:43:00.115 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00013572700000000001 2023-09-23T20:43:00.115 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.115 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.115 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:43:00.116 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.490256+0000", 2023-09-23T20:43:00.116 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00016105000000000001 2023-09-23T20:43:00.116 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.116 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.116 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:43:00.116 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.493899+0000", 2023-09-23T20:43:00.116 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0036422210000000002 2023-09-23T20:43:00.116 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.116 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.116 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:43:00.116 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.497124+0000", 2023-09-23T20:43:00.116 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0032253220000000001 2023-09-23T20:43:00.117 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.117 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.117 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:43:00.117 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.497214+0000", 2023-09-23T20:43:00.117 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.9739000000000003e-05 2023-09-23T20:43:00.117 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.117 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.117 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:43:00.117 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.497254+0000", 2023-09-23T20:43:00.117 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.0278999999999998e-05 2023-09-23T20:43:00.117 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:00.117 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:00.118 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:43:00.118 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.497288+0000", 2023-09-23T20:43:00.118 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.3927999999999998e-05 2023-09-23T20:43:00.118 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:43:00.118 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:43:00.118 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:43:00.118 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:43:00.118 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:43:00.118 INFO:teuthology.orchestra.run.smithi149.stdout:} 2023-09-23T20:43:00.119 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:00.119 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:59 smithi149 ceph-mon[170394]: pgmap v1199: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 6.3 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 732 op/s; 21246/308121 objects misplaced (6.895%); 15 MiB/s, 238 objects/s recovering 2023-09-23T20:43:00.119 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:59 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2843620515' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:43:00.119 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:59 smithi149 ceph-mon[170394]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:43:00.119 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:59 smithi149 ceph-mon[170394]: failed to encode map e478 with expected crc 2023-09-23T20:43:00.120 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:59 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:43:00.120 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:59 smithi149 ceph-mon[170394]: osdmap e478: 8 total, 8 up, 8 in 2023-09-23T20:43:00.120 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:59 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:43:00.120 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:42:59 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:43:00.120 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:43:00.121 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:43:00.221 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:43:00.296 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:00.363 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:43:00.382 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:00.382 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:43:00.483 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:43:00.485 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T20:43:00.550 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:00.660 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:43:00.667 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:43:00.671 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:00.672 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:43:00.772 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:43:00.833 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:01.096 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T20:43:01.259 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:43:01.402 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:43:01.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:01 smithi018 ceph-mon[184144]: pgmap v1201: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 6.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 760 op/s; 21246/303786 objects misplaced (6.994%); 16 MiB/s, 249 objects/s recovering 2023-09-23T20:43:01.895 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:01 smithi018 ceph-mon[179647]: pgmap v1201: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 6.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 760 op/s; 21246/303786 objects misplaced (6.994%); 16 MiB/s, 249 objects/s recovering 2023-09-23T20:43:01.917 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:01 smithi149 ceph-mon[170394]: pgmap v1201: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 6.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 760 op/s; 21246/303786 objects misplaced (6.994%); 16 MiB/s, 249 objects/s recovering 2023-09-23T20:43:01.971 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:43:01.975 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:01.975 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:43:02.076 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:43:02.124 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:02.824 INFO:teuthology.orchestra.run.smithi149.stdout:{ 2023-09-23T20:43:02.824 INFO:teuthology.orchestra.run.smithi149.stdout: "ops": [ 2023-09-23T20:43:02.824 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:02.825 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:130834 9.d 9:b76fb38a:::benchmark_data_smithi018_381516_object17000:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e478)", 2023-09-23T20:43:02.825 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:43:02.814785+0000", 2023-09-23T20:43:02.825 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 0.0020934130000000001, 2023-09-23T20:43:02.825 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0021268469999999999, 2023-09-23T20:43:02.825 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:43:02.825 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:43:02.825 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:43:02.825 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:43:02.825 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:43:02.826 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 130834 2023-09-23T20:43:02.826 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:02.826 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:43:02.826 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:02.826 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:43:02.826 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:02.814785+0000", 2023-09-23T20:43:02.826 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:43:02.826 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:02.826 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:02.826 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:43:02.827 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:02.814785+0000", 2023-09-23T20:43:02.827 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:43:02.827 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:02.827 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:02.827 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:43:02.827 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:02.814784+0000", 2023-09-23T20:43:02.827 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:43:02.827 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:02.827 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:02.827 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:43:02.828 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:02.814787+0000", 2023-09-23T20:43:02.828 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.6400000000000001e-06 2023-09-23T20:43:02.828 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:02.828 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:02.828 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:43:02.828 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:02.814793+0000", 2023-09-23T20:43:02.828 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.3439999999999997e-06 2023-09-23T20:43:02.828 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:02.828 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:02.829 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:43:02.829 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:02.814798+0000", 2023-09-23T20:43:02.829 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.3109999999999999e-06 2023-09-23T20:43:02.829 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:02.829 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:02.829 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:43:02.829 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:02.815040+0000", 2023-09-23T20:43:02.829 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000242649 2023-09-23T20:43:02.829 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:02.829 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:02.830 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:43:02.830 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:02.815157+0000", 2023-09-23T20:43:02.830 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00011685500000000001 2023-09-23T20:43:02.830 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:02.830 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:02.830 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 1,4,5", 2023-09-23T20:43:02.830 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:02.815290+0000", 2023-09-23T20:43:02.830 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00013237200000000001 2023-09-23T20:43:02.830 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:43:02.830 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:43:02.831 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:43:02.831 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:02.831 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:02.831 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:130837 9.3 9.a5d5a383 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e478)", 2023-09-23T20:43:02.831 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:43:02.814803+0000", 2023-09-23T20:43:02.831 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 0.0020748580000000002, 2023-09-23T20:43:02.831 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.002187371, 2023-09-23T20:43:02.831 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:43:02.831 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "queued for pg", 2023-09-23T20:43:02.832 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:43:02.832 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:43:02.832 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:43:02.832 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 130837 2023-09-23T20:43:02.832 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:02.832 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:43:02.832 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:02.832 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:43:02.832 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:02.814803+0000", 2023-09-23T20:43:02.832 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:43:02.833 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:02.833 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:02.833 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:43:02.833 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:02.814803+0000", 2023-09-23T20:43:02.833 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:43:02.833 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:02.833 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:02.833 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:43:02.833 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:02.814803+0000", 2023-09-23T20:43:02.833 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:43:02.834 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:02.834 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:02.834 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:43:02.834 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:02.814806+0000", 2023-09-23T20:43:02.834 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.5380000000000001e-06 2023-09-23T20:43:02.834 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:02.834 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:02.834 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:43:02.834 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:02.814813+0000", 2023-09-23T20:43:02.834 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.233e-06 2023-09-23T20:43:02.835 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:02.835 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:02.835 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:43:02.835 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:02.814817+0000", 2023-09-23T20:43:02.835 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.3980000000000001e-06 2023-09-23T20:43:02.835 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:43:02.835 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:43:02.835 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:43:02.835 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:02.835 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:02.836 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_repop(client.36450.0:130835 9.8 e478/472 9:1529a070:::benchmark_data_smithi018_381516_object17001:head v 478'8301, mlcod=478'8299)", 2023-09-23T20:43:02.836 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:43:02.815610+0000", 2023-09-23T20:43:02.836 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 0.001268019, 2023-09-23T20:43:02.836 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0014361000000000001, 2023-09-23T20:43:02.836 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:43:02.836 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "started", 2023-09-23T20:43:02.836 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:43:02.836 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:02.836 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:43:02.836 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:02.815610+0000", 2023-09-23T20:43:02.837 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:43:02.837 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:02.837 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:02.837 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:43:02.837 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:02.815610+0000", 2023-09-23T20:43:02.837 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:43:02.837 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:02.837 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:02.837 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:43:02.837 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:02.815610+0000", 2023-09-23T20:43:02.838 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:43:02.838 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:02.838 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:02.838 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:43:02.838 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:02.815615+0000", 2023-09-23T20:43:02.838 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.7379999999999999e-06 2023-09-23T20:43:02.838 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:02.838 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:02.838 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:43:02.838 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:02.815625+0000", 2023-09-23T20:43:02.839 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.2110000000000004e-06 2023-09-23T20:43:02.839 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:02.839 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:02.839 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:43:02.839 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:02.815630+0000", 2023-09-23T20:43:02.839 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.1440000000000002e-06 2023-09-23T20:43:02.839 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:02.839 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:02.839 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:43:02.839 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:02.815682+0000", 2023-09-23T20:43:02.840 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.2475000000000002e-05 2023-09-23T20:43:02.840 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:02.840 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:02.840 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:43:02.840 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:02.815703+0000", 2023-09-23T20:43:02.840 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.0458999999999998e-05 2023-09-23T20:43:02.840 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:43:02.840 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:43:02.840 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:43:02.840 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:02.841 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:02.841 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:130833 9.3 9:cd16e5cb:::benchmark_data_smithi018_381516_object16999:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e478)", 2023-09-23T20:43:02.841 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:43:02.814754+0000", 2023-09-23T20:43:02.841 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 0.002124234, 2023-09-23T20:43:02.841 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.002353394, 2023-09-23T20:43:02.841 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:43:02.841 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:43:02.841 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:43:02.841 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:43:02.841 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:43:02.842 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 130833 2023-09-23T20:43:02.842 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:02.842 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:43:02.842 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:02.842 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:43:02.842 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:02.814754+0000", 2023-09-23T20:43:02.842 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:43:02.842 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:02.842 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:02.842 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:43:02.843 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:02.814754+0000", 2023-09-23T20:43:02.843 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:43:02.843 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:02.843 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:02.843 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:43:02.843 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:02.814753+0000", 2023-09-23T20:43:02.843 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:43:02.843 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:02.843 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:02.843 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:43:02.844 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:02.814758+0000", 2023-09-23T20:43:02.844 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.6310000000000002e-06 2023-09-23T20:43:02.844 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:02.844 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:02.844 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:43:02.844 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:02.814767+0000", 2023-09-23T20:43:02.844 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.7280000000000007e-06 2023-09-23T20:43:02.844 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:02.844 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:02.844 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:43:02.845 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:02.814773+0000", 2023-09-23T20:43:02.845 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.8039999999999998e-06 2023-09-23T20:43:02.845 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:02.845 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:02.845 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:43:02.845 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:02.814851+0000", 2023-09-23T20:43:02.845 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.8069000000000001e-05 2023-09-23T20:43:02.845 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:02.845 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:02.845 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:43:02.845 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:02.814950+0000", 2023-09-23T20:43:02.846 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.9092999999999995e-05 2023-09-23T20:43:02.846 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:43:02.846 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:43:02.846 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:43:02.846 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:02.815020+0000", 2023-09-23T20:43:02.846 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.9332999999999995e-05 2023-09-23T20:43:02.846 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:43:02.846 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:43:02.846 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:43:02.846 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:43:02.847 INFO:teuthology.orchestra.run.smithi149.stdout: ], 2023-09-23T20:43:02.847 INFO:teuthology.orchestra.run.smithi149.stdout: "num_ops": 4 2023-09-23T20:43:02.847 INFO:teuthology.orchestra.run.smithi149.stdout:} 2023-09-23T20:43:02.847 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:43:02.850 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:02.850 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:43:02.951 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:43:03.054 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:03.571 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:43:03.611 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:03 smithi018 ceph-mon[184144]: pgmap v1202: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 6.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 681 op/s; 19719/299415 objects misplaced (6.586%); 9.9 MiB/s, 157 objects/s recovering 2023-09-23T20:43:03.613 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:03 smithi018 ceph-mon[179647]: pgmap v1202: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 6.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 681 op/s; 19719/299415 objects misplaced (6.586%); 9.9 MiB/s, 157 objects/s recovering 2023-09-23T20:43:03.628 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:43:03.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:03 smithi149 ceph-mon[170394]: pgmap v1202: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 6.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 681 op/s; 19719/299415 objects misplaced (6.586%); 9.9 MiB/s, 157 objects/s recovering 2023-09-23T20:43:04.678 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2023-09-23T20:43:04.678 INFO:teuthology.orchestra.run.smithi018.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-09-23T20:43:04.678 INFO:teuthology.orchestra.run.smithi018.stdout:} 2023-09-23T20:43:05.241 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:43:05.254 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:05.255 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:43:05.294 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:43:04 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:43:04] "GET /metrics HTTP/1.1" 200 36598 "" "Prometheus/2.43.0" 2023-09-23T20:43:05.348 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T20:43:05.355 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:43:05.489 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:05.565 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:43:05.578 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:05.578 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:43:05.614 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:05 smithi018 ceph-mon[179647]: pgmap v1203: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 6.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 775 op/s; 18860/293232 objects misplaced (6.432%); 14 MiB/s, 222 objects/s recovering 2023-09-23T20:43:05.679 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:43:05.737 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:05.868 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:05 smithi018 ceph-mon[184144]: pgmap v1203: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 6.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 775 op/s; 18860/293232 objects misplaced (6.432%); 14 MiB/s, 222 objects/s recovering 2023-09-23T20:43:05.870 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:43:05.880 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:05.881 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:43:05.969 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:05 smithi149 ceph-mon[170394]: pgmap v1203: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 6.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 775 op/s; 18860/293232 objects misplaced (6.432%); 14 MiB/s, 222 objects/s recovering 2023-09-23T20:43:05.981 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:43:06.082 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:06.163 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:43:06.168 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:06.168 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:43:06.189 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:43:06.202 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:43:06.238 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:43:06.269 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:43:06.276 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:43:06.303 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:43:06.315 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:06.341 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:43:06.383 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:43:06.423 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:43:06.487 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:43:06.489 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:43:06.489 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:43:06.489 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:43:06.489 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:43:06.489 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:43:06.489 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:43:06.489 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:43:06.489 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:43:06.644 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:06 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2515507549' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:43:06.644 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:06 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1507311257' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:43:06.644 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:06 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:43:06.669 INFO:teuthology.orchestra.run.smithi018.stderr:nodeep-scrub is unset 2023-09-23T20:43:06.870 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:43:06.877 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:06.878 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:43:06.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:06 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2515507549' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:43:06.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:06 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1507311257' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:43:06.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:06 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:43:06.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:06 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2515507549' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:43:06.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:06 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1507311257' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:43:06.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:06 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:43:06.978 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:43:07.047 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:07.192 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:43:07.199 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:07.199 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:43:07.300 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:43:07.354 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:07.647 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:07 smithi149 ceph-mon[170394]: pgmap v1204: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 5.9 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 795 op/s; 18626/287799 objects misplaced (6.472%); 14 MiB/s, 222 objects/s recovering 2023-09-23T20:43:07.647 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:07 smithi149 ceph-mon[170394]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:43:07.647 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:07 smithi149 ceph-mon[170394]: Cluster is now healthy 2023-09-23T20:43:07.647 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:07 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:43:07.647 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:07 smithi149 ceph-mon[170394]: osdmap e479: 8 total, 8 up, 8 in 2023-09-23T20:43:07.648 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:07 smithi149 ceph-mon[170394]: failed to encode map e479 with expected crc 2023-09-23T20:43:07.648 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:07 smithi149 ceph-mon[170394]: failed to encode map e479 with expected crc 2023-09-23T20:43:07.770 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:43:07.782 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:07.783 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:43:07.884 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:43:07.937 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:07 smithi018 ceph-mon[184144]: pgmap v1204: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 5.9 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 795 op/s; 18626/287799 objects misplaced (6.472%); 14 MiB/s, 222 objects/s recovering 2023-09-23T20:43:07.937 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:07 smithi018 ceph-mon[184144]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:43:07.938 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:07 smithi018 ceph-mon[184144]: Cluster is now healthy 2023-09-23T20:43:07.938 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:07 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:43:07.938 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:07 smithi018 ceph-mon[184144]: osdmap e479: 8 total, 8 up, 8 in 2023-09-23T20:43:07.938 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:07 smithi018 ceph-mon[184144]: failed to encode map e479 with expected crc 2023-09-23T20:43:07.938 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:07 smithi018 ceph-mon[184144]: failed to encode map e479 with expected crc 2023-09-23T20:43:07.938 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:07 smithi018 ceph-mon[179647]: pgmap v1204: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 5.9 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 795 op/s; 18626/287799 objects misplaced (6.472%); 14 MiB/s, 222 objects/s recovering 2023-09-23T20:43:07.938 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:07 smithi018 ceph-mon[179647]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:43:07.938 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:07 smithi018 ceph-mon[179647]: Cluster is now healthy 2023-09-23T20:43:07.939 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:07 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:43:07.939 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:07 smithi018 ceph-mon[179647]: osdmap e479: 8 total, 8 up, 8 in 2023-09-23T20:43:07.939 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:07 smithi018 ceph-mon[179647]: failed to encode map e479 with expected crc 2023-09-23T20:43:07.939 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:07 smithi018 ceph-mon[179647]: failed to encode map e479 with expected crc 2023-09-23T20:43:07.946 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:08.025 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:43:08.034 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:08.035 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:43:08.135 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:43:08.187 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:08.251 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:43:08.257 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:08.257 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:43:08.358 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:43:08.460 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:08.531 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:43:08.541 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:08.583 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:43:08.684 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:43:08.791 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:08.959 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:43:08.966 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:08.967 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:43:09.063 INFO:teuthology.orchestra.run.smithi149.stdout:ERROR: (22) Invalid argument 2023-09-23T20:43:09.063 INFO:teuthology.orchestra.run.smithi149.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:43:09.067 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:43:09.131 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:09.183 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:43:09.185 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:43:09.189 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:09.190 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:43:09.249 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T20:43:09.291 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:43:09.424 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:09.473 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T20:43:09.543 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:43:09.558 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:09.559 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:43:09.659 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:43:09.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:09 smithi149 ceph-mon[170394]: pgmap v1206: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 5.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 786 op/s; 16441/284022 objects misplaced (5.789%); 16 MiB/s, 250 objects/s recovering 2023-09-23T20:43:09.729 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:09 smithi018 ceph-mon[184144]: pgmap v1206: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 5.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 786 op/s; 16441/284022 objects misplaced (5.789%); 16 MiB/s, 250 objects/s recovering 2023-09-23T20:43:09.729 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:09 smithi018 ceph-mon[179647]: pgmap v1206: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 5.8 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 786 op/s; 16441/284022 objects misplaced (5.789%); 16 MiB/s, 250 objects/s recovering 2023-09-23T20:43:09.733 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:09.793 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:43:09.802 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:09.802 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:43:09.903 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:43:09.969 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:10.013 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:43:10.038 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:43:10.049 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:10.049 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:43:10.069 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:43:10.150 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:43:10.197 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 7, 3, 2, 4, 6, 5] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T20:43:10.198 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:43:10.198 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-09-23T20:43:10.198 INFO:tasks.thrashosds.thrasher:Setting osd 5 primary_affinity to 0.078650 2023-09-23T20:43:10.198 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 5 0.07865020022639246 2023-09-23T20:43:10.238 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:10.304 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:43:10.320 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:43:10.327 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:10.328 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:43:10.428 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:43:10.513 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:10.829 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:43:10.840 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:10.841 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:43:10.942 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:43:11.027 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:11.159 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:43:11.161 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:11.162 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:43:11.262 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:43:11.349 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:11.452 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:43:11.473 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:11.473 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:43:11.574 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:43:11.690 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:11.708 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:11 smithi018 ceph-mon[179647]: pgmap v1207: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 5.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 811 op/s; 16441/279435 objects misplaced (5.884%); 15 MiB/s, 239 objects/s recovering 2023-09-23T20:43:11.710 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:11 smithi018 ceph-mon[184144]: pgmap v1207: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 5.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 811 op/s; 16441/279435 objects misplaced (5.884%); 15 MiB/s, 239 objects/s recovering 2023-09-23T20:43:11.852 INFO:teuthology.orchestra.run.smithi149.stdout:ERROR: (22) Invalid argument 2023-09-23T20:43:11.852 INFO:teuthology.orchestra.run.smithi149.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:43:11.884 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:11 smithi149 ceph-mon[170394]: pgmap v1207: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 5.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 811 op/s; 16441/279435 objects misplaced (5.884%); 15 MiB/s, 239 objects/s recovering 2023-09-23T20:43:11.897 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:43:11.956 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:43:11.963 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:11.963 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:43:12.064 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:43:12.161 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:12.292 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T20:43:12.336 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:43:12.345 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:12.345 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:43:12.397 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:43:12.438 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:43:12.446 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:43:12.474 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:43:12.494 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:12.514 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:43:12.553 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:43:12.558 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:12.558 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:43:12.571 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:43:12.604 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:43:12.653 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:43:12.659 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:43:12.699 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:43:12.703 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:12.704 INFO:teuthology.orchestra.run.smithi018.stderr:noscrub is set 2023-09-23T20:43:12.744 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:43:12.744 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:43:12.745 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:43:12.745 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:43:12.745 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:43:12.745 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:43:12.745 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:43:12.745 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:43:12.758 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:43:12.763 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:12.764 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:43:12.865 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:43:12.896 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:12 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2177706562' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:43:12.896 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:12 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2775520413' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:43:12.896 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:12 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:43:12.896 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:12 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2177706562' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:43:12.897 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:12 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2775520413' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:43:12.897 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:12 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:43:12.931 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:12 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2177706562' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:43:12.931 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:12 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2775520413' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:43:12.931 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:12 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:43:12.936 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:13.000 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:43:13.011 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:13.012 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:43:13.113 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:43:13.210 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:13.817 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:43:13.821 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:13.821 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:43:13.922 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:43:13.937 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:13 smithi018 ceph-mon[179647]: pgmap v1208: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 808 op/s; 15008/275157 objects misplaced (5.454%); 15 MiB/s, 241 objects/s recovering 2023-09-23T20:43:13.937 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:13 smithi018 ceph-mon[179647]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:43:13.938 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:13 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:43:13.938 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:13 smithi018 ceph-mon[179647]: osdmap e480: 8 total, 8 up, 8 in 2023-09-23T20:43:13.938 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:13 smithi018 ceph-mon[179647]: failed to encode map e480 with expected crc 2023-09-23T20:43:13.938 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:13 smithi018 ceph-mon[179647]: failed to encode map e480 with expected crc 2023-09-23T20:43:13.938 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:13 smithi018 ceph-mon[179647]: failed to encode map e480 with expected crc 2023-09-23T20:43:13.938 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:13 smithi018 ceph-mon[184144]: pgmap v1208: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 808 op/s; 15008/275157 objects misplaced (5.454%); 15 MiB/s, 241 objects/s recovering 2023-09-23T20:43:13.938 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:13 smithi018 ceph-mon[184144]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:43:13.938 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:13 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:43:13.938 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:13 smithi018 ceph-mon[184144]: osdmap e480: 8 total, 8 up, 8 in 2023-09-23T20:43:13.939 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:13 smithi018 ceph-mon[184144]: failed to encode map e480 with expected crc 2023-09-23T20:43:13.939 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:13 smithi018 ceph-mon[184144]: failed to encode map e480 with expected crc 2023-09-23T20:43:13.939 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:13 smithi018 ceph-mon[184144]: failed to encode map e480 with expected crc 2023-09-23T20:43:13.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:13 smithi149 ceph-mon[170394]: pgmap v1208: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 808 op/s; 15008/275157 objects misplaced (5.454%); 15 MiB/s, 241 objects/s recovering 2023-09-23T20:43:13.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:13 smithi149 ceph-mon[170394]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:43:13.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:13 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:43:13.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:13 smithi149 ceph-mon[170394]: osdmap e480: 8 total, 8 up, 8 in 2023-09-23T20:43:13.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:13 smithi149 ceph-mon[170394]: failed to encode map e480 with expected crc 2023-09-23T20:43:13.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:13 smithi149 ceph-mon[170394]: failed to encode map e480 with expected crc 2023-09-23T20:43:13.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:13 smithi149 ceph-mon[170394]: failed to encode map e480 with expected crc 2023-09-23T20:43:13.983 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:14.058 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:43:14.062 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:14.063 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:43:14.164 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:43:14.252 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:43:14.265 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:14.353 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:43:14.368 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:14.369 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:43:14.470 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:43:14.557 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:14.632 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:43:14.648 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:14.649 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:43:14.750 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:43:14.814 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:14.849 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:14 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:43:14.850 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:14 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:43:14.892 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:43:14.897 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:14.898 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:43:14.968 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:14 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:43:14.999 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:43:15.061 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:15.141 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:43:14 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:43:14] "GET /metrics HTTP/1.1" 200 36596 "" "Prometheus/2.43.0" 2023-09-23T20:43:15.141 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:43:15.147 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:15.148 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:43:15.248 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:43:15.267 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T20:43:15.329 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:15.408 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:43:15.411 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:15.412 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:43:15.512 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:43:15.632 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:15.727 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:43:15.732 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:15.732 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:43:15.747 INFO:teuthology.orchestra.run.smithi018.stderr:set osd.5 primary-affinity to 0.0786502 (851542) 2023-09-23T20:43:15.833 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:43:15.862 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:15.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:15 smithi018 ceph-mon[184144]: pgmap v1210: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 5.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 771 op/s; 14144/269283 objects misplaced (5.252%); 19 MiB/s, 312 objects/s recovering 2023-09-23T20:43:15.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:15 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3771306415' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.07865020022639246}]: dispatch 2023-09-23T20:43:15.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:15 smithi018 ceph-mon[179647]: pgmap v1210: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 5.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 771 op/s; 14144/269283 objects misplaced (5.252%); 19 MiB/s, 312 objects/s recovering 2023-09-23T20:43:15.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:15 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3771306415' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.07865020022639246}]: dispatch 2023-09-23T20:43:15.901 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:43:15.904 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:15.904 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:43:15.914 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2023-09-23T20:43:15.915 INFO:teuthology.orchestra.run.smithi018.stdout: "ops": [ 2023-09-23T20:43:15.915 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:15.915 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "MOSDPGPush(9.8 481/472 [PushOp(9:1768f1c1:::benchmark_data_smithi018_381516_object52873:head, version: 461'3265, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(9:1768f1c1:::benchmark_data_smithi018_381516_object52873:head@461'3265, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-09-23T20:43:15.915 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:43:15.913544+0000", 2023-09-23T20:43:15.915 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.000279296, 2023-09-23T20:43:15.915 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00032246800000000002, 2023-09-23T20:43:15.915 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:43:15.915 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "started", 2023-09-23T20:43:15.915 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:43:15.916 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:15.916 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:43:15.916 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.913544+0000", 2023-09-23T20:43:15.916 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:43:15.916 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:15.916 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:15.916 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:43:15.916 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.913544+0000", 2023-09-23T20:43:15.916 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:43:15.916 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:15.917 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:15.917 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:43:15.917 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.913543+0000", 2023-09-23T20:43:15.917 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:43:15.917 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:15.917 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:15.917 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:43:15.917 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.913571+0000", 2023-09-23T20:43:15.917 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.7690000000000001e-05 2023-09-23T20:43:15.917 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:15.917 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:15.918 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:43:15.918 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.913591+0000", 2023-09-23T20:43:15.918 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.0129e-05 2023-09-23T20:43:15.918 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:15.918 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:15.918 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:43:15.918 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.913608+0000", 2023-09-23T20:43:15.918 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.6577000000000001e-05 2023-09-23T20:43:15.918 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:15.919 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:15.919 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:43:15.919 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.913664+0000", 2023-09-23T20:43:15.919 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.5711000000000002e-05 2023-09-23T20:43:15.919 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:15.919 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:15.919 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:43:15.919 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.913672+0000", 2023-09-23T20:43:15.919 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.8950000000000004e-06 2023-09-23T20:43:15.919 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:15.920 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:15.920 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:43:15.920 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.913866+0000", 2023-09-23T20:43:15.920 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00019405299999999999 2023-09-23T20:43:15.920 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:43:15.920 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:43:15.920 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:43:15.920 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:15.920 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:15.920 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "pg_backfill(progress 9.8 e 481/481 lb 9:17685ed8:::benchmark_data_smithi018_381516_object90047:head)", 2023-09-23T20:43:15.921 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:43:15.913630+0000", 2023-09-23T20:43:15.921 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.00019363599999999999, 2023-09-23T20:43:15.921 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00030794600000000002, 2023-09-23T20:43:15.921 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:43:15.921 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "started", 2023-09-23T20:43:15.921 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:43:15.921 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:15.921 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:43:15.921 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.913630+0000", 2023-09-23T20:43:15.921 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:43:15.922 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:15.922 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:15.922 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:43:15.922 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.913630+0000", 2023-09-23T20:43:15.922 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:43:15.922 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:15.922 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:15.922 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:43:15.922 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.913629+0000", 2023-09-23T20:43:15.922 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:43:15.923 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:15.923 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:15.923 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:43:15.923 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.913636+0000", 2023-09-23T20:43:15.923 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.9330000000000004e-06 2023-09-23T20:43:15.923 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:15.923 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:15.923 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:43:15.923 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.913645+0000", 2023-09-23T20:43:15.924 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.3839999999999997e-06 2023-09-23T20:43:15.924 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:15.924 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:15.924 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:43:15.924 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.913651+0000", 2023-09-23T20:43:15.924 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.4620000000000001e-06 2023-09-23T20:43:15.924 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:15.924 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:15.924 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:43:15.924 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.913886+0000", 2023-09-23T20:43:15.925 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00023530600000000001 2023-09-23T20:43:15.925 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:15.925 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:15.925 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:43:15.925 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.913900+0000", 2023-09-23T20:43:15.925 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3421000000000001e-05 2023-09-23T20:43:15.925 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:15.925 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:15.925 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:43:15.925 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.913938+0000", 2023-09-23T20:43:15.926 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.7870999999999997e-05 2023-09-23T20:43:15.926 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:43:15.926 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:43:15.926 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:43:15.926 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:15.926 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:15.926 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.36450.0:141029 9.a 9.97c404ea (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e481)", 2023-09-23T20:43:15.926 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:43:15.738002+0000", 2023-09-23T20:43:15.926 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.17582161599999999, 2023-09-23T20:43:15.926 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.17599398699999999, 2023-09-23T20:43:15.927 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:43:15.927 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "delayed", 2023-09-23T20:43:15.927 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:43:15.927 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.36450", 2023-09-23T20:43:15.927 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:43:15.927 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 141029 2023-09-23T20:43:15.927 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:15.927 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:43:15.927 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:15.927 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:43:15.928 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738002+0000", 2023-09-23T20:43:15.928 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:43:15.928 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:15.928 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:15.928 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:43:15.928 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738002+0000", 2023-09-23T20:43:15.928 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:43:15.928 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:15.928 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:15.928 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:43:15.929 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738000+0000", 2023-09-23T20:43:15.929 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:43:15.929 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:15.929 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:15.929 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:43:15.929 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738007+0000", 2023-09-23T20:43:15.929 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.4760000000000001e-06 2023-09-23T20:43:15.929 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:15.929 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:15.929 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:43:15.930 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738025+0000", 2023-09-23T20:43:15.930 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.7952e-05 2023-09-23T20:43:15.930 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:15.930 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:15.930 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:43:15.930 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738035+0000", 2023-09-23T20:43:15.930 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.6120000000000005e-06 2023-09-23T20:43:15.930 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:15.930 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:15.931 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:43:15.931 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.744123+0000", 2023-09-23T20:43:15.931 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0060876410000000004 2023-09-23T20:43:15.931 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:15.931 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:15.931 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op must wait for map", 2023-09-23T20:43:15.931 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.744131+0000", 2023-09-23T20:43:15.931 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.8800000000000008e-06 2023-09-23T20:43:15.931 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:15.931 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:15.932 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:43:15.932 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.754919+0000", 2023-09-23T20:43:15.932 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.010788483999999999 2023-09-23T20:43:15.932 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:15.932 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:15.932 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:43:15.932 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.754929+0000", 2023-09-23T20:43:15.932 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0309000000000001e-05 2023-09-23T20:43:15.932 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:43:15.932 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:43:15.933 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:43:15.933 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:15.933 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:15.933 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.36450.0:141040 9.a 9.64c4a4ca (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e481)", 2023-09-23T20:43:15.933 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:43:15.738057+0000", 2023-09-23T20:43:15.933 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.175766004, 2023-09-23T20:43:15.933 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.176011054, 2023-09-23T20:43:15.933 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:43:15.933 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "delayed", 2023-09-23T20:43:15.933 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:43:15.934 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.36450", 2023-09-23T20:43:15.934 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:43:15.934 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 141040 2023-09-23T20:43:15.934 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:15.934 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:43:15.934 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:15.934 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:43:15.934 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738057+0000", 2023-09-23T20:43:15.934 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:43:15.934 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:15.935 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:15.935 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:43:15.935 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738057+0000", 2023-09-23T20:43:15.935 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:43:15.935 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:15.935 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:15.935 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:43:15.935 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738057+0000", 2023-09-23T20:43:15.935 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:43:15.935 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:15.936 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:15.936 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:43:15.936 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738060+0000", 2023-09-23T20:43:15.936 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.9699999999999999e-06 2023-09-23T20:43:15.936 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:15.936 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:15.936 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:43:15.936 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738066+0000", 2023-09-23T20:43:15.936 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.5019999999999998e-06 2023-09-23T20:43:15.936 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:15.937 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:15.937 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:43:15.937 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738072+0000", 2023-09-23T20:43:15.937 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.2209999999999996e-06 2023-09-23T20:43:15.937 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:15.937 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:15.937 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:43:15.937 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.744157+0000", 2023-09-23T20:43:15.937 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0060856 2023-09-23T20:43:15.938 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:15.938 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:15.938 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting_for_map not empty", 2023-09-23T20:43:15.938 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.744162+0000", 2023-09-23T20:43:15.938 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.1610000000000002e-06 2023-09-23T20:43:15.938 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:15.938 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:15.938 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:43:15.938 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.754951+0000", 2023-09-23T20:43:15.938 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.010789127000000001 2023-09-23T20:43:15.939 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:15.939 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:15.939 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:43:15.939 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.754959+0000", 2023-09-23T20:43:15.939 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.3189999999999997e-06 2023-09-23T20:43:15.939 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:43:15.939 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:43:15.939 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:43:15.939 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:15.940 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:15.940 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.36450.0:141043 9.a 9.a95f95ea (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e481)", 2023-09-23T20:43:15.940 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:43:15.738077+0000", 2023-09-23T20:43:15.940 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.17574636699999999, 2023-09-23T20:43:15.940 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.17605021600000001, 2023-09-23T20:43:15.940 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:43:15.940 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "delayed", 2023-09-23T20:43:15.940 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:43:15.940 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.36450", 2023-09-23T20:43:15.940 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:43:15.941 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 141043 2023-09-23T20:43:15.941 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:15.941 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:43:15.941 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:15.941 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:43:15.941 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738077+0000", 2023-09-23T20:43:15.941 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:43:15.941 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:15.941 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:15.941 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:43:15.942 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738077+0000", 2023-09-23T20:43:15.942 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:43:15.942 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:15.942 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:15.942 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:43:15.942 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738077+0000", 2023-09-23T20:43:15.942 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:43:15.942 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:15.942 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:15.942 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:43:15.943 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738079+0000", 2023-09-23T20:43:15.943 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.0320000000000002e-06 2023-09-23T20:43:15.943 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:15.943 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:15.943 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:43:15.943 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738084+0000", 2023-09-23T20:43:15.943 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.4310000000000002e-06 2023-09-23T20:43:15.943 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:15.943 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:15.944 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:43:15.944 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738089+0000", 2023-09-23T20:43:15.944 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.6789999999999996e-06 2023-09-23T20:43:15.944 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:15.944 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:15.944 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:43:15.944 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.744187+0000", 2023-09-23T20:43:15.944 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0060979859999999997 2023-09-23T20:43:15.944 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:15.944 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:15.945 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting_for_map not empty", 2023-09-23T20:43:15.945 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.744194+0000", 2023-09-23T20:43:15.945 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.7090000000000002e-06 2023-09-23T20:43:15.945 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:15.945 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:15.945 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:43:15.945 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.754979+0000", 2023-09-23T20:43:15.945 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.010784141000000001 2023-09-23T20:43:15.945 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:15.945 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:15.946 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:43:15.946 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.754986+0000", 2023-09-23T20:43:15.946 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.0339999999999999e-06 2023-09-23T20:43:15.946 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:43:15.946 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:43:15.946 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:43:15.946 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:15.946 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:15.946 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.36450.0:141045 9.f 9.1fdbaa5f (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e481)", 2023-09-23T20:43:15.947 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:43:15.738093+0000", 2023-09-23T20:43:15.947 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.17573051000000001, 2023-09-23T20:43:15.947 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.17610632900000001, 2023-09-23T20:43:15.947 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:43:15.947 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "delayed", 2023-09-23T20:43:15.947 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:43:15.947 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.36450", 2023-09-23T20:43:15.947 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:43:15.947 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 141045 2023-09-23T20:43:15.947 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:15.948 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:43:15.948 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:15.948 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:43:15.948 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738093+0000", 2023-09-23T20:43:15.948 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:43:15.948 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:15.948 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:15.948 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:43:15.948 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738093+0000", 2023-09-23T20:43:15.948 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:43:15.949 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:15.949 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:15.949 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:43:15.949 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738092+0000", 2023-09-23T20:43:15.949 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:43:15.949 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:15.949 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:15.949 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:43:15.949 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738095+0000", 2023-09-23T20:43:15.949 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.063e-06 2023-09-23T20:43:15.950 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:15.950 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:15.950 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:43:15.950 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738100+0000", 2023-09-23T20:43:15.950 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.2469999999999999e-06 2023-09-23T20:43:15.950 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:15.950 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:15.950 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:43:15.950 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738104+0000", 2023-09-23T20:43:15.950 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.3649999999999997e-06 2023-09-23T20:43:15.951 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:15.951 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:15.951 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:43:15.951 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738271+0000", 2023-09-23T20:43:15.951 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00016673799999999999 2023-09-23T20:43:15.951 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:15.951 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:15.951 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op must wait for map", 2023-09-23T20:43:15.951 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738278+0000", 2023-09-23T20:43:15.952 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.0659999999999998e-06 2023-09-23T20:43:15.952 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:15.952 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:15.952 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:43:15.952 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.770314+0000", 2023-09-23T20:43:15.952 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.032036012000000003 2023-09-23T20:43:15.952 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:15.952 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:15.952 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:43:15.952 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.770322+0000", 2023-09-23T20:43:15.953 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.9330000000000001e-06 2023-09-23T20:43:15.953 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:43:15.953 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:43:15.953 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:43:15.953 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:15.953 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:15.953 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.36450.0:141047 9.a 9.718b97aa (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e481)", 2023-09-23T20:43:15.953 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:43:15.747358+0000", 2023-09-23T20:43:15.953 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.16646530400000001, 2023-09-23T20:43:15.954 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.16691388300000001, 2023-09-23T20:43:15.954 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:43:15.954 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "delayed", 2023-09-23T20:43:15.954 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:43:15.954 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.36450", 2023-09-23T20:43:15.954 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:43:15.954 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 141047 2023-09-23T20:43:15.954 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:15.954 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:43:15.954 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:15.955 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:43:15.955 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.747358+0000", 2023-09-23T20:43:15.955 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:43:15.955 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:15.955 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:15.955 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:43:15.955 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.747358+0000", 2023-09-23T20:43:15.955 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:43:15.955 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:15.956 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:15.956 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:43:15.956 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.747356+0000", 2023-09-23T20:43:15.956 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:43:15.956 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:15.956 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:15.956 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:43:15.956 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.747364+0000", 2023-09-23T20:43:15.956 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.6569999999999997e-06 2023-09-23T20:43:15.956 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:15.956 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:15.957 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:43:15.957 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.747379+0000", 2023-09-23T20:43:15.957 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.5153999999999999e-05 2023-09-23T20:43:15.957 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:15.957 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:15.957 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:43:15.957 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.747388+0000", 2023-09-23T20:43:15.957 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.7059999999999993e-06 2023-09-23T20:43:15.957 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:15.957 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:15.957 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:43:15.958 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.748082+0000", 2023-09-23T20:43:15.958 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00069463100000000002 2023-09-23T20:43:15.958 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:15.958 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:15.958 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting_for_map not empty", 2023-09-23T20:43:15.958 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.748090+0000", 2023-09-23T20:43:15.958 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.1349999999999992e-06 2023-09-23T20:43:15.958 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:15.958 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:15.959 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:43:15.959 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.755006+0000", 2023-09-23T20:43:15.959 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0069154289999999998 2023-09-23T20:43:15.959 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:15.959 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:15.959 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:43:15.959 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.755013+0000", 2023-09-23T20:43:15.959 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.7909999999999999e-06 2023-09-23T20:43:15.959 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:43:15.959 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:43:15.960 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:43:15.960 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:43:15.960 INFO:teuthology.orchestra.run.smithi018.stdout: ], 2023-09-23T20:43:15.960 INFO:teuthology.orchestra.run.smithi018.stdout: "num_ops": 7 2023-09-23T20:43:15.960 INFO:teuthology.orchestra.run.smithi018.stdout:} 2023-09-23T20:43:15.960 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:43:16.006 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:43:16.032 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:15 smithi149 ceph-mon[170394]: pgmap v1210: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 5.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 771 op/s; 14144/269283 objects misplaced (5.252%); 19 MiB/s, 312 objects/s recovering 2023-09-23T20:43:16.032 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:15 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3771306415' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.07865020022639246}]: dispatch 2023-09-23T20:43:16.035 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:16.073 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:43:16.076 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:16.077 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:43:16.177 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:43:16.207 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:16.245 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:43:16.248 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:16.249 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:43:16.350 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:43:16.378 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:16.417 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:43:16.420 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:16.421 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:43:16.464 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T20:43:16.521 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:43:16.555 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:16.598 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:43:16.601 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:43:16.606 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:16.607 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:43:16.707 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:43:16.736 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:16.748 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:43:16.792 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:43:16.797 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:16.798 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:43:16.871 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:16 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3771306415' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.07865020022639246}]': finished 2023-09-23T20:43:16.871 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:16 smithi018 ceph-mon[179647]: osdmap e481: 8 total, 8 up, 8 in 2023-09-23T20:43:16.871 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:16 smithi018 ceph-mon[179647]: failed to encode map e481 with expected crc 2023-09-23T20:43:16.872 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:16 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3771306415' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.07865020022639246}]': finished 2023-09-23T20:43:16.872 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:16 smithi018 ceph-mon[184144]: osdmap e481: 8 total, 8 up, 8 in 2023-09-23T20:43:16.872 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:16 smithi018 ceph-mon[184144]: failed to encode map e481 with expected crc 2023-09-23T20:43:16.898 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:43:16.967 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:17.102 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:43:17.109 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:17.109 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:43:17.210 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:43:17.219 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:16 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3771306415' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 0.07865020022639246}]': finished 2023-09-23T20:43:17.219 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:16 smithi149 ceph-mon[170394]: osdmap e481: 8 total, 8 up, 8 in 2023-09-23T20:43:17.219 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:16 smithi149 ceph-mon[170394]: failed to encode map e481 with expected crc 2023-09-23T20:43:17.287 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:17.342 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:43:17.355 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:17.356 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:43:17.457 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:43:17.525 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:17.595 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:43:17.601 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:17.601 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:43:17.702 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:43:17.757 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:17.827 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:43:17.836 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:17.837 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:43:17.890 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:43:17.937 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:43:17.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:17 smithi149 ceph-mon[170394]: pgmap v1212: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 5.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 831 op/s; 13874/264051 objects misplaced (5.254%); 9.8 MiB/s, 156 objects/s recovering 2023-09-23T20:43:17.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:17 smithi149 ceph-mon[170394]: failed to encode map e482 with expected crc 2023-09-23T20:43:17.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:17 smithi149 ceph-mon[170394]: osdmap e482: 8 total, 8 up, 8 in 2023-09-23T20:43:17.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:17 smithi149 ceph-mon[170394]: failed to encode map e482 with expected crc 2023-09-23T20:43:17.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:17 smithi149 ceph-mon[170394]: failed to encode map e482 with expected crc 2023-09-23T20:43:17.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:17 smithi149 ceph-mon[170394]: failed to encode map e482 with expected crc 2023-09-23T20:43:17.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:17 smithi149 ceph-mon[170394]: failed to encode map e482 with expected crc 2023-09-23T20:43:17.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:17 smithi149 ceph-mon[170394]: failed to encode map e482 with expected crc 2023-09-23T20:43:17.968 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:17 smithi149 ceph-mon[170394]: failed to encode map e482 with expected crc 2023-09-23T20:43:17.968 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:17 smithi149 ceph-mon[170394]: failed to encode map e482 with expected crc 2023-09-23T20:43:17.968 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:17 smithi149 ceph-mon[170394]: failed to encode map e482 with expected crc 2023-09-23T20:43:17.968 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:17 smithi149 ceph-mon[170394]: failed to encode map e482 with expected crc 2023-09-23T20:43:17.968 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:17 smithi149 ceph-mon[170394]: 9.5 continuing backfill to osd.1 from (459'1300,480'8637] MIN to 480'8637 2023-09-23T20:43:18.017 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:17 smithi018 ceph-mon[179647]: pgmap v1212: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 5.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 831 op/s; 13874/264051 objects misplaced (5.254%); 9.8 MiB/s, 156 objects/s recovering 2023-09-23T20:43:18.017 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:17 smithi018 ceph-mon[179647]: failed to encode map e482 with expected crc 2023-09-23T20:43:18.017 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:17 smithi018 ceph-mon[179647]: osdmap e482: 8 total, 8 up, 8 in 2023-09-23T20:43:18.017 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:17 smithi018 ceph-mon[179647]: failed to encode map e482 with expected crc 2023-09-23T20:43:18.017 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:17 smithi018 ceph-mon[179647]: failed to encode map e482 with expected crc 2023-09-23T20:43:18.017 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:17 smithi018 ceph-mon[179647]: failed to encode map e482 with expected crc 2023-09-23T20:43:18.018 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:17 smithi018 ceph-mon[179647]: failed to encode map e482 with expected crc 2023-09-23T20:43:18.018 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:17 smithi018 ceph-mon[179647]: failed to encode map e482 with expected crc 2023-09-23T20:43:18.018 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:17 smithi018 ceph-mon[179647]: failed to encode map e482 with expected crc 2023-09-23T20:43:18.018 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:17 smithi018 ceph-mon[179647]: failed to encode map e482 with expected crc 2023-09-23T20:43:18.018 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:17 smithi018 ceph-mon[179647]: failed to encode map e482 with expected crc 2023-09-23T20:43:18.018 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:17 smithi018 ceph-mon[179647]: failed to encode map e482 with expected crc 2023-09-23T20:43:18.018 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:17 smithi018 ceph-mon[179647]: 9.5 continuing backfill to osd.1 from (459'1300,480'8637] MIN to 480'8637 2023-09-23T20:43:18.019 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:17 smithi018 ceph-mon[184144]: pgmap v1212: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 5.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 831 op/s; 13874/264051 objects misplaced (5.254%); 9.8 MiB/s, 156 objects/s recovering 2023-09-23T20:43:18.019 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:17 smithi018 ceph-mon[184144]: failed to encode map e482 with expected crc 2023-09-23T20:43:18.019 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:17 smithi018 ceph-mon[184144]: osdmap e482: 8 total, 8 up, 8 in 2023-09-23T20:43:18.019 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:17 smithi018 ceph-mon[184144]: failed to encode map e482 with expected crc 2023-09-23T20:43:18.019 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:17 smithi018 ceph-mon[184144]: failed to encode map e482 with expected crc 2023-09-23T20:43:18.019 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:17 smithi018 ceph-mon[184144]: failed to encode map e482 with expected crc 2023-09-23T20:43:18.019 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:17 smithi018 ceph-mon[184144]: failed to encode map e482 with expected crc 2023-09-23T20:43:18.019 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:17 smithi018 ceph-mon[184144]: failed to encode map e482 with expected crc 2023-09-23T20:43:18.019 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:17 smithi018 ceph-mon[184144]: failed to encode map e482 with expected crc 2023-09-23T20:43:18.019 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:17 smithi018 ceph-mon[184144]: failed to encode map e482 with expected crc 2023-09-23T20:43:18.019 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:17 smithi018 ceph-mon[184144]: failed to encode map e482 with expected crc 2023-09-23T20:43:18.020 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:17 smithi018 ceph-mon[184144]: failed to encode map e482 with expected crc 2023-09-23T20:43:18.020 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:17 smithi018 ceph-mon[184144]: 9.5 continuing backfill to osd.1 from (459'1300,480'8637] MIN to 480'8637 2023-09-23T20:43:18.031 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:18.130 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:43:18.138 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:18.138 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:43:18.239 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:43:18.313 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:18.382 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:43:18.391 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:18.391 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:43:18.492 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:43:18.574 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:18.987 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:43:18.987 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:43:18.987 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:43:18.987 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:43:19.003 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:43:19.008 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:43:19.011 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:19.012 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:43:19.031 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:43:19.051 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:43:19.072 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:43:19.074 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:43:19.074 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:43:19.074 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:43:19.074 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:43:19.074 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:43:19.074 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:43:19.074 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:43:19.074 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:43:19.113 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:43:19.139 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:19.174 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:43:19.178 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:19.178 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:43:19.279 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:43:19.305 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:19.340 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:43:19.343 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:19.344 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:43:19.444 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:43:19.481 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:19.787 INFO:teuthology.orchestra.run.smithi018.stderr:nodeep-scrub is set 2023-09-23T20:43:20.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:19 smithi018 ceph-mon[179647]: pgmap v1214: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 21 KiB/s rd, 55 KiB/s wr, 817 op/s; 11001/260721 objects misplaced (4.219%); 29 MiB/s, 457 objects/s recovering 2023-09-23T20:43:20.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:19 smithi018 ceph-mon[179647]: osdmap e483: 8 total, 8 up, 8 in 2023-09-23T20:43:20.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:19 smithi018 ceph-mon[179647]: failed to encode map e483 with expected crc 2023-09-23T20:43:20.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:19 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1247067903' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:43:20.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:19 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2119192344' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:43:20.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:19 smithi018 ceph-mon[184144]: pgmap v1214: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 21 KiB/s rd, 55 KiB/s wr, 817 op/s; 11001/260721 objects misplaced (4.219%); 29 MiB/s, 457 objects/s recovering 2023-09-23T20:43:20.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:19 smithi018 ceph-mon[184144]: osdmap e483: 8 total, 8 up, 8 in 2023-09-23T20:43:20.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:19 smithi018 ceph-mon[184144]: failed to encode map e483 with expected crc 2023-09-23T20:43:20.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:19 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1247067903' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:43:20.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:19 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2119192344' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:43:20.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:19 smithi149 ceph-mon[170394]: pgmap v1214: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 21 KiB/s rd, 55 KiB/s wr, 817 op/s; 11001/260721 objects misplaced (4.219%); 29 MiB/s, 457 objects/s recovering 2023-09-23T20:43:20.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:19 smithi149 ceph-mon[170394]: osdmap e483: 8 total, 8 up, 8 in 2023-09-23T20:43:20.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:19 smithi149 ceph-mon[170394]: failed to encode map e483 with expected crc 2023-09-23T20:43:20.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:19 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1247067903' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:43:20.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:19 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2119192344' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:43:21.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:20 smithi018 ceph-mon[184144]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:43:21.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:20 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2119192344' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:43:21.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:20 smithi018 ceph-mon[184144]: osdmap e484: 8 total, 8 up, 8 in 2023-09-23T20:43:21.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:20 smithi018 ceph-mon[184144]: failed to encode map e484 with expected crc 2023-09-23T20:43:21.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:20 smithi018 ceph-mon[184144]: failed to encode map e484 with expected crc 2023-09-23T20:43:21.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:20 smithi018 ceph-mon[179647]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:43:21.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:20 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2119192344' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:43:21.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:20 smithi018 ceph-mon[179647]: osdmap e484: 8 total, 8 up, 8 in 2023-09-23T20:43:21.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:20 smithi018 ceph-mon[179647]: failed to encode map e484 with expected crc 2023-09-23T20:43:21.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:20 smithi018 ceph-mon[179647]: failed to encode map e484 with expected crc 2023-09-23T20:43:21.219 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:20 smithi149 ceph-mon[170394]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:43:21.219 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:20 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2119192344' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:43:21.219 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:20 smithi149 ceph-mon[170394]: osdmap e484: 8 total, 8 up, 8 in 2023-09-23T20:43:21.219 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:20 smithi149 ceph-mon[170394]: failed to encode map e484 with expected crc 2023-09-23T20:43:21.219 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:20 smithi149 ceph-mon[170394]: failed to encode map e484 with expected crc 2023-09-23T20:43:21.367 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 7, 3, 2, 4, 6, 5] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T20:43:21.367 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:43:21.367 INFO:tasks.thrashosds.thrasher:inject_pause on osd.3 2023-09-23T20:43:21.367 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T20:43:21.368 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T20:43:21.368 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 config set filestore_inject_stall 3 2023-09-23T20:43:22.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:21 smithi018 ceph-mon[184144]: pgmap v1217: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 5.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 42 KiB/s rd, 112 KiB/s wr, 578 op/s; 11001/256821 objects misplaced (4.284%); 28 MiB/s, 444 objects/s recovering 2023-09-23T20:43:22.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:21 smithi018 ceph-mon[179647]: pgmap v1217: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 5.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 42 KiB/s rd, 112 KiB/s wr, 578 op/s; 11001/256821 objects misplaced (4.284%); 28 MiB/s, 444 objects/s recovering 2023-09-23T20:43:22.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:21 smithi149 ceph-mon[170394]: pgmap v1217: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 5.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 42 KiB/s rd, 112 KiB/s wr, 578 op/s; 11001/256821 objects misplaced (4.284%); 28 MiB/s, 444 objects/s recovering 2023-09-23T20:43:22.222 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:43:22.227 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:22.227 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:43:22.328 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:43:22.389 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:22.455 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:43:22.459 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:22.460 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:43:22.560 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:43:22.630 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:22.705 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:43:22.716 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:22.716 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:43:22.817 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:43:22.879 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:24.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:23 smithi018 ceph-mon[184144]: pgmap v1218: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 5.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 31 KiB/s rd, 82 KiB/s wr, 488 op/s; 9464/255708 objects misplaced (3.701%); 30 MiB/s, 485 objects/s recovering 2023-09-23T20:43:24.144 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:23 smithi018 ceph-mon[179647]: pgmap v1218: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 5.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 31 KiB/s rd, 82 KiB/s wr, 488 op/s; 9464/255708 objects misplaced (3.701%); 30 MiB/s, 485 objects/s recovering 2023-09-23T20:43:24.220 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:23 smithi149 ceph-mon[170394]: pgmap v1218: 119 pgs: 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 116 active+clean; 5.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 31 KiB/s rd, 82 KiB/s wr, 488 op/s; 9464/255708 objects misplaced (3.701%); 30 MiB/s, 485 objects/s recovering 2023-09-23T20:43:24.779 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:43:24.785 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:24.785 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:43:24.886 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:43:24.938 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:24.984 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:43:24.992 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:24.992 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:43:25.093 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:43:25.141 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:43:24 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:43:24] "GET /metrics HTTP/1.1" 200 36596 "" "Prometheus/2.43.0" 2023-09-23T20:43:25.155 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:25.695 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:43:25.704 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:25.705 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:43:25.806 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:43:25.909 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:25.981 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:25 smithi018 ceph-mon[179647]: pgmap v1219: 119 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 117 active+clean; 5.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 25 KiB/s rd, 67 KiB/s wr, 680 op/s; 9294/248736 objects misplaced (3.736%); 21 MiB/s, 328 objects/s recovering 2023-09-23T20:43:25.981 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:25 smithi018 ceph-mon[184144]: pgmap v1219: 119 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 117 active+clean; 5.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 25 KiB/s rd, 67 KiB/s wr, 680 op/s; 9294/248736 objects misplaced (3.736%); 21 MiB/s, 328 objects/s recovering 2023-09-23T20:43:25.996 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:43:26.001 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:26.002 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:43:26.071 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:43:26.103 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:43:26.154 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:26.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:25 smithi149 ceph-mon[170394]: pgmap v1219: 119 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 117 active+clean; 5.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 25 KiB/s rd, 67 KiB/s wr, 680 op/s; 9294/248736 objects misplaced (3.736%); 21 MiB/s, 328 objects/s recovering 2023-09-23T20:43:26.523 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T20:43:26.893 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:43:26.909 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:26.909 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:43:26.968 INFO:teuthology.orchestra.run.smithi018.stdout:ERROR: (22) Invalid argument 2023-09-23T20:43:26.968 INFO:teuthology.orchestra.run.smithi018.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:43:27.010 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:43:27.076 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:27.156 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:43:27.167 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:27.167 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:43:27.268 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:43:27.339 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:27.574 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:43:27.579 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:27.579 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:43:27.680 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:43:27.740 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:43:27.746 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T20:43:27.769 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:27.862 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:43:27.871 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:27.871 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:43:27.939 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:43:27.972 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:43:28.147 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:27 smithi018 ceph-mon[184144]: pgmap v1220: 119 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 117 active+clean; 5.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 7.9 KiB/s rd, 21 KiB/s wr, 678 op/s; 8541/244587 objects misplaced (3.492%); 13 MiB/s, 204 objects/s recovering 2023-09-23T20:43:28.147 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:27 smithi018 ceph-mon[179647]: pgmap v1220: 119 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 117 active+clean; 5.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 7.9 KiB/s rd, 21 KiB/s wr, 678 op/s; 8541/244587 objects misplaced (3.492%); 13 MiB/s, 204 objects/s recovering 2023-09-23T20:43:28.159 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:28.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:27 smithi149 ceph-mon[170394]: pgmap v1220: 119 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 117 active+clean; 5.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 7.9 KiB/s rd, 21 KiB/s wr, 678 op/s; 8541/244587 objects misplaced (3.492%); 13 MiB/s, 204 objects/s recovering 2023-09-23T20:43:28.275 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:43:28.288 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:28.289 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:43:28.389 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:43:28.487 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:28.807 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:43:28.817 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:28.817 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:43:28.918 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:43:28.989 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:29.200 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:43:29.209 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:29.210 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:43:29.311 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:43:29.329 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:43:29.337 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2023-09-23T20:43:29.337 INFO:teuthology.orchestra.run.smithi018.stdout: "success": "" 2023-09-23T20:43:29.337 INFO:teuthology.orchestra.run.smithi018.stdout:} 2023-09-23T20:43:29.402 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:29.894 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:29 smithi018 ceph-mon[184144]: pgmap v1221: 119 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 117 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 4.5 KiB/s rd, 12 KiB/s wr, 750 op/s; 8277/238623 objects misplaced (3.469%); 11 MiB/s, 174 objects/s recovering 2023-09-23T20:43:29.895 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:29 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:43:29.895 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:29 smithi018 ceph-mon[179647]: pgmap v1221: 119 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 117 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 4.5 KiB/s rd, 12 KiB/s wr, 750 op/s; 8277/238623 objects misplaced (3.469%); 11 MiB/s, 174 objects/s recovering 2023-09-23T20:43:29.895 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:29 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:43:29.899 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:43:29.911 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:29.911 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:43:30.012 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:43:30.057 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:30.099 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:43:30.105 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:30.105 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:43:30.206 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:43:30.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:29 smithi149 ceph-mon[170394]: pgmap v1221: 119 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 117 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 4.5 KiB/s rd, 12 KiB/s wr, 750 op/s; 8277/238623 objects misplaced (3.469%); 11 MiB/s, 174 objects/s recovering 2023-09-23T20:43:30.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:29 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:43:30.231 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:30.264 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:43:30.267 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:30.268 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:43:30.268 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:43:30.369 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:43:30.419 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:30.491 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:43:30.503 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:30.504 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:43:30.604 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:43:30.693 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:30.761 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:43:30.767 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:30.767 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:43:30.868 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:43:30.910 INFO:teuthology.orchestra.run.smithi018.stderr:noscrub is unset 2023-09-23T20:43:30.930 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:30.995 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:43:31.001 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:31.002 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:43:31.102 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:43:31.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:30 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/535018270' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:43:31.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:30 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:43:31.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:30 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/535018270' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:43:31.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:30 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:43:31.177 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:30 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/535018270' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:43:31.177 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:30 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:43:31.180 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:31.237 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:43:31.244 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:31.244 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:43:31.345 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:43:31.413 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:31.754 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:43:31.768 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:31.768 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:43:31.869 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:43:31.947 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:31 smithi018 ceph-mon[179647]: pgmap v1222: 119 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 117 active+clean; 4.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 792 op/s; 8277/232158 objects misplaced (3.565%); 9.8 MiB/s, 157 objects/s recovering 2023-09-23T20:43:31.947 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:31 smithi018 ceph-mon[179647]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:43:31.947 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:31 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:43:31.947 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:31 smithi018 ceph-mon[179647]: osdmap e485: 8 total, 8 up, 8 in 2023-09-23T20:43:31.947 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:31 smithi018 ceph-mon[179647]: failed to encode map e485 with expected crc 2023-09-23T20:43:31.947 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:31 smithi018 ceph-mon[179647]: failed to encode map e485 with expected crc 2023-09-23T20:43:31.947 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:31 smithi018 ceph-mon[179647]: failed to encode map e485 with expected crc 2023-09-23T20:43:31.947 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:31 smithi018 ceph-mon[179647]: failed to encode map e485 with expected crc 2023-09-23T20:43:31.948 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:31 smithi018 ceph-mon[179647]: 1.17 scrub starts 2023-09-23T20:43:31.948 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:31 smithi018 ceph-mon[179647]: 1.1b scrub starts 2023-09-23T20:43:31.948 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:31 smithi018 ceph-mon[184144]: pgmap v1222: 119 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 117 active+clean; 4.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 792 op/s; 8277/232158 objects misplaced (3.565%); 9.8 MiB/s, 157 objects/s recovering 2023-09-23T20:43:31.948 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:31 smithi018 ceph-mon[184144]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:43:31.948 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:31 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:43:31.948 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:31 smithi018 ceph-mon[184144]: osdmap e485: 8 total, 8 up, 8 in 2023-09-23T20:43:31.948 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:31 smithi018 ceph-mon[184144]: failed to encode map e485 with expected crc 2023-09-23T20:43:31.948 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:31 smithi018 ceph-mon[184144]: failed to encode map e485 with expected crc 2023-09-23T20:43:31.948 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:31 smithi018 ceph-mon[184144]: failed to encode map e485 with expected crc 2023-09-23T20:43:31.948 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:31 smithi018 ceph-mon[184144]: failed to encode map e485 with expected crc 2023-09-23T20:43:31.949 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:31 smithi018 ceph-mon[184144]: 1.17 scrub starts 2023-09-23T20:43:31.949 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:31 smithi018 ceph-mon[184144]: 1.1b scrub starts 2023-09-23T20:43:31.950 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:32.037 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:43:32.045 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:32.045 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:43:32.143 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:43:32.147 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:43:32.183 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:43:32.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:31 smithi149 ceph-mon[170394]: pgmap v1222: 119 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 117 active+clean; 4.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 792 op/s; 8277/232158 objects misplaced (3.565%); 9.8 MiB/s, 157 objects/s recovering 2023-09-23T20:43:32.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:31 smithi149 ceph-mon[170394]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:43:32.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:31 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:43:32.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:31 smithi149 ceph-mon[170394]: osdmap e485: 8 total, 8 up, 8 in 2023-09-23T20:43:32.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:31 smithi149 ceph-mon[170394]: failed to encode map e485 with expected crc 2023-09-23T20:43:32.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:31 smithi149 ceph-mon[170394]: failed to encode map e485 with expected crc 2023-09-23T20:43:32.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:31 smithi149 ceph-mon[170394]: failed to encode map e485 with expected crc 2023-09-23T20:43:32.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:31 smithi149 ceph-mon[170394]: failed to encode map e485 with expected crc 2023-09-23T20:43:32.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:31 smithi149 ceph-mon[170394]: 1.17 scrub starts 2023-09-23T20:43:32.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:31 smithi149 ceph-mon[170394]: 1.1b scrub starts 2023-09-23T20:43:32.224 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:32.234 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:43:32.269 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:43:32.309 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:43:32.357 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:43:32.382 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:43:32.391 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:32.392 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:43:32.392 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:43:32.432 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:43:32.458 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:43:32.458 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:43:32.459 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:43:32.459 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:43:32.459 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:43:32.459 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:43:32.459 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:43:32.459 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:43:32.493 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:43:32.499 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2023-09-23T20:43:32.499 INFO:teuthology.orchestra.run.smithi018.stdout: "size": 20, 2023-09-23T20:43:32.500 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 600, 2023-09-23T20:43:32.500 INFO:teuthology.orchestra.run.smithi018.stdout: "ops": [ 2023-09-23T20:43:32.500 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.500 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:156899 7.1 7:894832cf:::benchmark_data_smithi018_308601_object62347:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e354)", 2023-09-23T20:43:32.500 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:34:59.972856+0000", 2023-09-23T20:43:32.500 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 512.51072791700005, 2023-09-23T20:43:32.500 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1521833800000001, 2023-09-23T20:43:32.500 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:43:32.500 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:43:32.500 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:43:32.500 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:43:32.501 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:43:32.501 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 156899 2023-09-23T20:43:32.501 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.501 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:43:32.501 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.501 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:43:32.501 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.972856+0000", 2023-09-23T20:43:32.501 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:43:32.501 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.501 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.501 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:43:32.501 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.972856+0000", 2023-09-23T20:43:32.502 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:43:32.502 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.502 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.502 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:43:32.502 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.972854+0000", 2023-09-23T20:43:32.502 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:43:32.502 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.502 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.502 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:43:32.502 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.972863+0000", 2023-09-23T20:43:32.502 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.738e-06 2023-09-23T20:43:32.503 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.503 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.503 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:43:32.503 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.972876+0000", 2023-09-23T20:43:32.503 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3362e-05 2023-09-23T20:43:32.503 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.503 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.503 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:43:32.503 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.972888+0000", 2023-09-23T20:43:32.503 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1856e-05 2023-09-23T20:43:32.503 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.503 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.503 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:43:32.504 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.975112+0000", 2023-09-23T20:43:32.504 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0022238980000000002 2023-09-23T20:43:32.504 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.504 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.504 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:43:32.504 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.975128+0000", 2023-09-23T20:43:32.504 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.6325999999999999e-05 2023-09-23T20:43:32.504 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.504 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.504 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:43:32.504 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.115938+0000", 2023-09-23T20:43:32.505 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1408093079999999 2023-09-23T20:43:32.505 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.505 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.505 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:43:32.505 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.116085+0000", 2023-09-23T20:43:32.505 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00014754199999999999 2023-09-23T20:43:32.505 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.505 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.505 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 3,4,6", 2023-09-23T20:43:32.505 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.116220+0000", 2023-09-23T20:43:32.505 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000135308 2023-09-23T20:43:32.505 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.506 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.506 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:43:32.506 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.122330+0000", 2023-09-23T20:43:32.506 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.006109377 2023-09-23T20:43:32.506 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.506 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.506 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:43:32.506 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.124884+0000", 2023-09-23T20:43:32.506 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0025545559999999999 2023-09-23T20:43:32.506 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.506 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.506 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:43:32.507 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.124904+0000", 2023-09-23T20:43:32.507 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.9967999999999999e-05 2023-09-23T20:43:32.507 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.507 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.507 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:43:32.507 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.124921+0000", 2023-09-23T20:43:32.507 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.7166000000000001e-05 2023-09-23T20:43:32.507 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.507 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.507 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:43:32.507 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.124957+0000", 2023-09-23T20:43:32.508 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.5417000000000003e-05 2023-09-23T20:43:32.508 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.508 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.508 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:43:32.508 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.125039+0000", 2023-09-23T20:43:32.508 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.2295999999999999e-05 2023-09-23T20:43:32.508 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:43:32.508 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:43:32.508 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:43:32.508 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.508 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.508 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:156906 7.1 7:8761197e:::benchmark_data_smithi018_308601_object62354:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e354)", 2023-09-23T20:43:32.509 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:34:59.972906+0000", 2023-09-23T20:43:32.509 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 512.51067740999997, 2023-09-23T20:43:32.509 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.204579289, 2023-09-23T20:43:32.509 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:43:32.509 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:43:32.509 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:43:32.509 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:43:32.509 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:43:32.509 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 156906 2023-09-23T20:43:32.509 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.509 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:43:32.509 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.510 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:43:32.510 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.972906+0000", 2023-09-23T20:43:32.510 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:43:32.510 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.510 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.510 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:43:32.510 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.972906+0000", 2023-09-23T20:43:32.510 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:43:32.510 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.510 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.510 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:43:32.510 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.972906+0000", 2023-09-23T20:43:32.511 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:43:32.511 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.511 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.511 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:43:32.511 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.972909+0000", 2023-09-23T20:43:32.511 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.6569999999999999e-06 2023-09-23T20:43:32.511 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.511 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.511 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:43:32.511 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.972917+0000", 2023-09-23T20:43:32.511 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.6389999999999992e-06 2023-09-23T20:43:32.511 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.512 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.512 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:43:32.512 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.972923+0000", 2023-09-23T20:43:32.512 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.6810000000000003e-06 2023-09-23T20:43:32.512 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.512 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.512 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:43:32.512 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.975155+0000", 2023-09-23T20:43:32.512 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.002232445 2023-09-23T20:43:32.512 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.512 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.512 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:43:32.513 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.975164+0000", 2023-09-23T20:43:32.513 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.6309999999999992e-06 2023-09-23T20:43:32.513 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.513 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.513 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:43:32.513 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.146302+0000", 2023-09-23T20:43:32.513 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.171138427 2023-09-23T20:43:32.513 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.513 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.513 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:43:32.513 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.146426+0000", 2023-09-23T20:43:32.514 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00012366800000000001 2023-09-23T20:43:32.514 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.514 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.514 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 3,4,6", 2023-09-23T20:43:32.514 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.146558+0000", 2023-09-23T20:43:32.514 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00013182499999999999 2023-09-23T20:43:32.514 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.514 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.514 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:43:32.514 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.150022+0000", 2023-09-23T20:43:32.514 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0034646070000000002 2023-09-23T20:43:32.514 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.515 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.515 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:43:32.515 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.150058+0000", 2023-09-23T20:43:32.515 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.5277000000000001e-05 2023-09-23T20:43:32.515 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.515 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.515 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:43:32.515 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.166990+0000", 2023-09-23T20:43:32.515 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.016932365000000001 2023-09-23T20:43:32.515 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.515 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.515 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:43:32.516 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.177360+0000", 2023-09-23T20:43:32.516 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.010369682 2023-09-23T20:43:32.516 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.516 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.516 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:43:32.516 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.177412+0000", 2023-09-23T20:43:32.516 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.2039000000000001e-05 2023-09-23T20:43:32.516 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.516 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.516 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:43:32.516 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.177486+0000", 2023-09-23T20:43:32.517 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.3830000000000003e-05 2023-09-23T20:43:32.517 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:43:32.517 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:43:32.517 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:43:32.517 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.517 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.517 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:156911 7.1 7:8bb3fc92:::benchmark_data_smithi018_308601_object62359:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e354)", 2023-09-23T20:43:32.517 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:34:59.972928+0000", 2023-09-23T20:43:32.517 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 512.51065559100005, 2023-09-23T20:43:32.517 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.204718468, 2023-09-23T20:43:32.517 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:43:32.517 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:43:32.518 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:43:32.518 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:43:32.518 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:43:32.518 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 156911 2023-09-23T20:43:32.518 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.518 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:43:32.518 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.518 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:43:32.518 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.972928+0000", 2023-09-23T20:43:32.518 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:43:32.518 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.519 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.519 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:43:32.519 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.972928+0000", 2023-09-23T20:43:32.519 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:43:32.519 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.519 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.519 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:43:32.519 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.972928+0000", 2023-09-23T20:43:32.519 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:43:32.519 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.519 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.519 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:43:32.520 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.972930+0000", 2023-09-23T20:43:32.520 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.5359999999999999e-06 2023-09-23T20:43:32.520 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.520 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.520 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:43:32.520 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.972936+0000", 2023-09-23T20:43:32.520 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.1190000000000002e-06 2023-09-23T20:43:32.520 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.520 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.520 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:43:32.520 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.972942+0000", 2023-09-23T20:43:32.521 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.5369999999999997e-06 2023-09-23T20:43:32.521 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.521 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.521 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:43:32.521 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.975188+0000", 2023-09-23T20:43:32.521 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.002246521 2023-09-23T20:43:32.521 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.521 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.521 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:43:32.521 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.975196+0000", 2023-09-23T20:43:32.521 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.0900000000000005e-06 2023-09-23T20:43:32.522 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.522 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.522 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:43:32.522 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.150087+0000", 2023-09-23T20:43:32.522 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.174890802 2023-09-23T20:43:32.522 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.522 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.522 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:43:32.522 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.150236+0000", 2023-09-23T20:43:32.522 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00014853699999999999 2023-09-23T20:43:32.522 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.522 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.523 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 3,4,6", 2023-09-23T20:43:32.523 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.150378+0000", 2023-09-23T20:43:32.523 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00014263000000000001 2023-09-23T20:43:32.523 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.523 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.523 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:43:32.523 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.157143+0000", 2023-09-23T20:43:32.523 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0067646909999999998 2023-09-23T20:43:32.523 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.523 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.523 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:43:32.524 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.157179+0000", 2023-09-23T20:43:32.524 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.6099000000000001e-05 2023-09-23T20:43:32.524 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.524 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.524 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:43:32.524 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.167022+0000", 2023-09-23T20:43:32.524 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0098431739999999997 2023-09-23T20:43:32.524 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.524 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.524 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:43:32.524 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.177564+0000", 2023-09-23T20:43:32.524 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.010541867999999999 2023-09-23T20:43:32.525 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.525 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.525 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:43:32.525 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.177590+0000", 2023-09-23T20:43:32.525 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.62e-05 2023-09-23T20:43:32.525 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.525 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.525 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:43:32.525 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.177647+0000", 2023-09-23T20:43:32.525 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.6063999999999997e-05 2023-09-23T20:43:32.525 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:43:32.525 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:43:32.526 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:43:32.526 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.526 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.526 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:156944 7.1 7:848d83ce:::benchmark_data_smithi018_308601_object62392:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e354)", 2023-09-23T20:43:32.526 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:34:59.972947+0000", 2023-09-23T20:43:32.526 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 512.510636901, 2023-09-23T20:43:32.526 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.207883992, 2023-09-23T20:43:32.526 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:43:32.526 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:43:32.526 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:43:32.526 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:43:32.526 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:43:32.527 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 156944 2023-09-23T20:43:32.527 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.527 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:43:32.527 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.527 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:43:32.527 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.972947+0000", 2023-09-23T20:43:32.527 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:43:32.527 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.527 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.527 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:43:32.527 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.972947+0000", 2023-09-23T20:43:32.527 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:43:32.528 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.528 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.528 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:43:32.528 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.972946+0000", 2023-09-23T20:43:32.528 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:43:32.528 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.528 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.528 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:43:32.528 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.972949+0000", 2023-09-23T20:43:32.528 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.503e-06 2023-09-23T20:43:32.528 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.528 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.529 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:43:32.529 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.972956+0000", 2023-09-23T20:43:32.529 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.9870000000000002e-06 2023-09-23T20:43:32.529 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.529 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.529 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:43:32.529 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.972962+0000", 2023-09-23T20:43:32.529 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.7339999999999996e-06 2023-09-23T20:43:32.529 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.529 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.529 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:43:32.530 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.975220+0000", 2023-09-23T20:43:32.530 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0022582489999999999 2023-09-23T20:43:32.530 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.530 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.530 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:43:32.530 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.975229+0000", 2023-09-23T20:43:32.530 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.3540000000000006e-06 2023-09-23T20:43:32.530 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.530 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.530 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:43:32.530 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.153614+0000", 2023-09-23T20:43:32.531 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1783843549999999 2023-09-23T20:43:32.531 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.531 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.531 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:43:32.531 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.153771+0000", 2023-09-23T20:43:32.531 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000156935 2023-09-23T20:43:32.531 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.531 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.531 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 3,4,6", 2023-09-23T20:43:32.531 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.153909+0000", 2023-09-23T20:43:32.531 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00013803099999999999 2023-09-23T20:43:32.531 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.532 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.532 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:43:32.532 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.166810+0000", 2023-09-23T20:43:32.532 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.012901347000000001 2023-09-23T20:43:32.532 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.532 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.532 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:43:32.532 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.166863+0000", 2023-09-23T20:43:32.532 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.3046e-05 2023-09-23T20:43:32.532 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.532 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.533 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:43:32.533 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.177657+0000", 2023-09-23T20:43:32.533 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.010794419 2023-09-23T20:43:32.533 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.533 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.533 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:43:32.533 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.180631+0000", 2023-09-23T20:43:32.533 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0029738070000000002 2023-09-23T20:43:32.533 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.533 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.533 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:43:32.533 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.180693+0000", 2023-09-23T20:43:32.534 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.1752000000000001e-05 2023-09-23T20:43:32.534 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.534 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.534 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:43:32.534 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.180831+0000", 2023-09-23T20:43:32.534 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00013783600000000001 2023-09-23T20:43:32.534 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:43:32.534 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:43:32.534 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:43:32.534 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.534 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.535 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:156954 7.1 7:8fbb61c7:::benchmark_data_smithi018_308601_object62402:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e354)", 2023-09-23T20:43:32.535 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:34:59.972967+0000", 2023-09-23T20:43:32.535 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 512.51061706400003, 2023-09-23T20:43:32.535 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.208111494, 2023-09-23T20:43:32.535 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:43:32.535 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:43:32.535 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:43:32.535 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:43:32.535 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:43:32.535 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 156954 2023-09-23T20:43:32.535 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.535 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:43:32.536 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.536 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:43:32.536 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.972967+0000", 2023-09-23T20:43:32.536 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:43:32.536 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.536 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.536 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:43:32.536 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.972967+0000", 2023-09-23T20:43:32.536 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:43:32.536 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.536 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.537 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:43:32.537 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.972966+0000", 2023-09-23T20:43:32.537 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:43:32.537 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.537 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.537 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:43:32.537 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.972969+0000", 2023-09-23T20:43:32.537 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.548e-06 2023-09-23T20:43:32.537 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.537 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.537 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:43:32.537 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.972974+0000", 2023-09-23T20:43:32.538 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.6389999999999997e-06 2023-09-23T20:43:32.538 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.538 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.538 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:43:32.538 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.972979+0000", 2023-09-23T20:43:32.538 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.2639999999999999e-06 2023-09-23T20:43:32.538 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.538 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.538 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:43:32.538 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.975256+0000", 2023-09-23T20:43:32.538 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0022768879999999999 2023-09-23T20:43:32.539 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.539 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.539 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:43:32.539 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.975264+0000", 2023-09-23T20:43:32.539 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.0260000000000007e-06 2023-09-23T20:43:32.539 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.539 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.539 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:43:32.539 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.157204+0000", 2023-09-23T20:43:32.539 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.181939297 2023-09-23T20:43:32.539 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.539 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.540 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:43:32.540 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.157336+0000", 2023-09-23T20:43:32.540 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00013253599999999999 2023-09-23T20:43:32.540 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.540 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.540 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 3,4,6", 2023-09-23T20:43:32.540 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.157473+0000", 2023-09-23T20:43:32.540 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000137219 2023-09-23T20:43:32.540 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.540 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.540 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:43:32.540 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.166957+0000", 2023-09-23T20:43:32.541 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0094833369999999997 2023-09-23T20:43:32.541 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.541 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.541 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:43:32.541 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.174176+0000", 2023-09-23T20:43:32.541 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.007219306 2023-09-23T20:43:32.541 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.541 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.541 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:43:32.541 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.177664+0000", 2023-09-23T20:43:32.541 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0034882519999999998 2023-09-23T20:43:32.541 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.542 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.542 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:43:32.542 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.180887+0000", 2023-09-23T20:43:32.542 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0032224430000000002 2023-09-23T20:43:32.542 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.542 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.542 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:43:32.542 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.180953+0000", 2023-09-23T20:43:32.542 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.5775e-05 2023-09-23T20:43:32.542 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.542 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.543 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:43:32.543 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.181078+0000", 2023-09-23T20:43:32.543 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00012554600000000001 2023-09-23T20:43:32.543 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:43:32.543 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:43:32.543 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:43:32.543 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.543 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.543 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35994.0:84076 8.a 8:51aef8e7:::benchmark_data_smithi018_341490_object84075:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e378)", 2023-09-23T20:43:32.543 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:37:13.334851+0000", 2023-09-23T20:43:32.543 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 379.14873225999997, 2023-09-23T20:43:32.543 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.367426676, 2023-09-23T20:43:32.544 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:43:32.544 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:43:32.544 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:43:32.544 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35994", 2023-09-23T20:43:32.544 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:43:32.544 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 84076 2023-09-23T20:43:32.544 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.544 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:43:32.544 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.544 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:43:32.544 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.334851+0000", 2023-09-23T20:43:32.544 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:43:32.545 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.545 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.545 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:43:32.545 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.334851+0000", 2023-09-23T20:43:32.545 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:43:32.545 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.545 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.545 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:43:32.545 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.334851+0000", 2023-09-23T20:43:32.545 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:43:32.545 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.545 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.546 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:43:32.546 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.334872+0000", 2023-09-23T20:43:32.546 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.1464999999999999e-05 2023-09-23T20:43:32.546 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.546 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.546 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:43:32.546 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.334880+0000", 2023-09-23T20:43:32.546 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.7489999999999998e-06 2023-09-23T20:43:32.546 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.546 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.546 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:43:32.547 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.334886+0000", 2023-09-23T20:43:32.547 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.626e-06 2023-09-23T20:43:32.547 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.547 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.547 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:43:32.547 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.338890+0000", 2023-09-23T20:43:32.547 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0040048039999999998 2023-09-23T20:43:32.547 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.547 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.547 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:43:32.547 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.339048+0000", 2023-09-23T20:43:32.547 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000157548 2023-09-23T20:43:32.548 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.548 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.548 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 2,5", 2023-09-23T20:43:32.548 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.339203+0000", 2023-09-23T20:43:32.548 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00015535000000000001 2023-09-23T20:43:32.548 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.548 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.548 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:43:32.548 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.350871+0000", 2023-09-23T20:43:32.548 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0116682 2023-09-23T20:43:32.548 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.548 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.549 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:43:32.549 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.354938+0000", 2023-09-23T20:43:32.549 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0040660749999999997 2023-09-23T20:43:32.549 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.549 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.549 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:43:32.549 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.361832+0000", 2023-09-23T20:43:32.549 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0068944369999999998 2023-09-23T20:43:32.549 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.549 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.549 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:43:32.549 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.361845+0000", 2023-09-23T20:43:32.550 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2889e-05 2023-09-23T20:43:32.550 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.550 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.550 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:43:32.550 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:14.353806+0000", 2023-09-23T20:43:32.550 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.99196156400000002 2023-09-23T20:43:32.550 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.550 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.550 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:43:32.550 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:14.353825+0000", 2023-09-23T20:43:32.550 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.8437999999999999e-05 2023-09-23T20:43:32.551 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.551 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.551 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:43:32.551 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:15.688941+0000", 2023-09-23T20:43:32.551 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.335116411 2023-09-23T20:43:32.551 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.551 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.551 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for degraded object", 2023-09-23T20:43:32.551 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:15.689523+0000", 2023-09-23T20:43:32.551 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00058162999999999997 2023-09-23T20:43:32.551 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.551 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.552 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:43:32.552 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:15.702170+0000", 2023-09-23T20:43:32.552 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.012646734999999999 2023-09-23T20:43:32.552 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.552 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.552 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:43:32.552 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:15.702278+0000", 2023-09-23T20:43:32.552 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00010842 2023-09-23T20:43:32.552 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:43:32.552 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:43:32.552 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:43:32.552 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.553 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.553 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35994.0:84079 8.a 8:53d5899b:::benchmark_data_smithi018_341490_object84078:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e378)", 2023-09-23T20:43:32.553 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:37:13.344056+0000", 2023-09-23T20:43:32.553 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 379.13952793499999, 2023-09-23T20:43:32.553 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.3675740109999999, 2023-09-23T20:43:32.553 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:43:32.553 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:43:32.553 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:43:32.553 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35994", 2023-09-23T20:43:32.553 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:43:32.553 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 84079 2023-09-23T20:43:32.554 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.554 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:43:32.554 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.554 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:43:32.554 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.344056+0000", 2023-09-23T20:43:32.554 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:43:32.554 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.554 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.554 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:43:32.554 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.344056+0000", 2023-09-23T20:43:32.554 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:43:32.555 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.555 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.555 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:43:32.555 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.344053+0000", 2023-09-23T20:43:32.555 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999976 2023-09-23T20:43:32.555 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.555 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.555 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:43:32.555 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.344081+0000", 2023-09-23T20:43:32.555 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.7042999999999998e-05 2023-09-23T20:43:32.555 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.555 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.556 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:43:32.556 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.344099+0000", 2023-09-23T20:43:32.556 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.8505e-05 2023-09-23T20:43:32.556 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.556 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.556 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:43:32.556 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.344109+0000", 2023-09-23T20:43:32.556 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.6350000000000001e-06 2023-09-23T20:43:32.556 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.556 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.556 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:43:32.556 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.350901+0000", 2023-09-23T20:43:32.557 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0067922520000000004 2023-09-23T20:43:32.557 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.557 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.557 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:43:32.557 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.351025+0000", 2023-09-23T20:43:32.557 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00012372899999999999 2023-09-23T20:43:32.557 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.557 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.557 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 2,5", 2023-09-23T20:43:32.557 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.351144+0000", 2023-09-23T20:43:32.557 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00011948 2023-09-23T20:43:32.557 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.558 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.558 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:43:32.558 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.354963+0000", 2023-09-23T20:43:32.558 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0038190049999999999 2023-09-23T20:43:32.558 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.558 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.558 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:43:32.558 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.361878+0000", 2023-09-23T20:43:32.558 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0069147660000000001 2023-09-23T20:43:32.558 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.558 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.559 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:43:32.559 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.361889+0000", 2023-09-23T20:43:32.559 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0721e-05 2023-09-23T20:43:32.559 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.559 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.559 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:43:32.559 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:14.353869+0000", 2023-09-23T20:43:32.559 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.991980046 2023-09-23T20:43:32.559 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.559 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.559 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:43:32.559 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:14.353879+0000", 2023-09-23T20:43:32.560 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0455000000000001e-05 2023-09-23T20:43:32.560 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.560 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.560 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:43:32.560 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:15.692953+0000", 2023-09-23T20:43:32.560 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3390743249999999 2023-09-23T20:43:32.560 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.560 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.560 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for degraded object", 2023-09-23T20:43:32.560 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:15.693411+0000", 2023-09-23T20:43:32.560 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00045711999999999999 2023-09-23T20:43:32.561 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.561 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.561 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:43:32.561 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:15.711539+0000", 2023-09-23T20:43:32.561 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.018128634000000001 2023-09-23T20:43:32.561 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.561 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.561 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:43:32.561 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:15.711630+0000", 2023-09-23T20:43:32.561 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.0568000000000004e-05 2023-09-23T20:43:32.561 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:43:32.562 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:43:32.562 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:43:32.562 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.562 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.562 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35994.0:84089 8.a 8:5f99afa4:::benchmark_data_smithi018_341490_object84088:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e379)", 2023-09-23T20:43:32.562 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:37:13.363463+0000", 2023-09-23T20:43:32.562 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 379.12012021200002, 2023-09-23T20:43:32.562 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.3832335360000001, 2023-09-23T20:43:32.562 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:43:32.562 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:43:32.562 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:43:32.562 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35994", 2023-09-23T20:43:32.563 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:43:32.563 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 84089 2023-09-23T20:43:32.563 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.563 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:43:32.563 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.563 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:43:32.563 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.363463+0000", 2023-09-23T20:43:32.563 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:43:32.563 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.563 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.563 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:43:32.564 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.363463+0000", 2023-09-23T20:43:32.564 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:43:32.564 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.564 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.564 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:43:32.564 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.363462+0000", 2023-09-23T20:43:32.564 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:43:32.564 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.564 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.564 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:43:32.564 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.363487+0000", 2023-09-23T20:43:32.564 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.5588999999999998e-05 2023-09-23T20:43:32.565 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.565 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.565 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:43:32.565 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.363504+0000", 2023-09-23T20:43:32.565 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.7118000000000001e-05 2023-09-23T20:43:32.565 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.565 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.565 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:43:32.565 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.363513+0000", 2023-09-23T20:43:32.565 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.337e-06 2023-09-23T20:43:32.565 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.566 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.566 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:43:32.566 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.363563+0000", 2023-09-23T20:43:32.566 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.0002e-05 2023-09-23T20:43:32.566 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.566 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.566 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:43:32.566 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.363572+0000", 2023-09-23T20:43:32.566 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.9269999999999994e-06 2023-09-23T20:43:32.566 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.566 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.566 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:43:32.567 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:14.353905+0000", 2023-09-23T20:43:32.567 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.99033294699999996 2023-09-23T20:43:32.567 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.567 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.567 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:43:32.567 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:14.353913+0000", 2023-09-23T20:43:32.567 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.4440000000000008e-06 2023-09-23T20:43:32.567 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.567 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.567 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:43:32.567 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:15.711657+0000", 2023-09-23T20:43:32.568 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3577443149999999 2023-09-23T20:43:32.568 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.568 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.568 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:43:32.568 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:15.711807+0000", 2023-09-23T20:43:32.568 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00014948199999999999 2023-09-23T20:43:32.568 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.568 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.568 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 2,3", 2023-09-23T20:43:32.568 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:15.711983+0000", 2023-09-23T20:43:32.568 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00017621999999999999 2023-09-23T20:43:32.569 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.569 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.569 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:43:32.569 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:15.727869+0000", 2023-09-23T20:43:32.569 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.015886246 2023-09-23T20:43:32.569 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.569 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.569 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:43:32.569 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:15.727928+0000", 2023-09-23T20:43:32.569 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.8464000000000001e-05 2023-09-23T20:43:32.569 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.569 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.570 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:43:32.570 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:15.743251+0000", 2023-09-23T20:43:32.570 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.015322794000000001 2023-09-23T20:43:32.570 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.570 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.570 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:43:32.570 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:15.743305+0000", 2023-09-23T20:43:32.570 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.4292000000000001e-05 2023-09-23T20:43:32.570 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.570 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.570 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:43:32.571 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:15.746697+0000", 2023-09-23T20:43:32.571 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0033921620000000002 2023-09-23T20:43:32.571 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:43:32.571 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:43:32.571 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:43:32.571 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.571 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.571 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35994.0:84091 8.3 8:c9851101:::benchmark_data_smithi018_341490_object84090:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e379)", 2023-09-23T20:43:32.571 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:37:13.366065+0000", 2023-09-23T20:43:32.571 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 379.11751821600001, 2023-09-23T20:43:32.571 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.2838538150000001, 2023-09-23T20:43:32.571 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:43:32.572 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:43:32.572 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:43:32.572 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35994", 2023-09-23T20:43:32.572 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:43:32.572 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 84091 2023-09-23T20:43:32.572 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.572 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:43:32.572 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.572 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:43:32.572 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.366065+0000", 2023-09-23T20:43:32.572 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:43:32.572 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.573 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.573 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:43:32.573 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.366065+0000", 2023-09-23T20:43:32.573 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:43:32.573 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.573 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.573 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:43:32.573 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.366065+0000", 2023-09-23T20:43:32.573 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:43:32.573 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.573 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.573 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:43:32.574 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.366089+0000", 2023-09-23T20:43:32.574 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.3904e-05 2023-09-23T20:43:32.574 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.574 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.574 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:43:32.574 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.366098+0000", 2023-09-23T20:43:32.574 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.9949999999999994e-06 2023-09-23T20:43:32.574 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.574 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.574 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:43:32.574 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.366117+0000", 2023-09-23T20:43:32.575 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.8924e-05 2023-09-23T20:43:32.575 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.575 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.575 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:43:32.575 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.366220+0000", 2023-09-23T20:43:32.575 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000103059 2023-09-23T20:43:32.575 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.575 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.575 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:43:32.575 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.366232+0000", 2023-09-23T20:43:32.575 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.206e-05 2023-09-23T20:43:32.575 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.576 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.576 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:43:32.576 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:14.349352+0000", 2023-09-23T20:43:32.576 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.98312001199999999 2023-09-23T20:43:32.576 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.576 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.576 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:43:32.576 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:14.349365+0000", 2023-09-23T20:43:32.576 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2999e-05 2023-09-23T20:43:32.576 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.576 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.577 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:43:32.577 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:15.629266+0000", 2023-09-23T20:43:32.577 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2799014390000001 2023-09-23T20:43:32.577 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.577 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.577 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:43:32.577 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:15.629429+0000", 2023-09-23T20:43:32.577 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00016256800000000001 2023-09-23T20:43:32.577 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.577 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.577 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 2,6", 2023-09-23T20:43:32.577 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:15.629608+0000", 2023-09-23T20:43:32.578 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00017943599999999999 2023-09-23T20:43:32.578 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.578 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.578 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:43:32.578 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:15.646076+0000", 2023-09-23T20:43:32.578 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.016468046 2023-09-23T20:43:32.578 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.578 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.578 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:43:32.578 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:15.646127+0000", 2023-09-23T20:43:32.578 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.0583e-05 2023-09-23T20:43:32.578 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.579 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.579 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:43:32.579 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:15.646140+0000", 2023-09-23T20:43:32.579 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3519e-05 2023-09-23T20:43:32.579 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.579 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.579 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:43:32.579 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:15.646219+0000", 2023-09-23T20:43:32.579 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.8881000000000006e-05 2023-09-23T20:43:32.579 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.579 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.579 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:43:32.580 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:15.649919+0000", 2023-09-23T20:43:32.580 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0036999530000000002 2023-09-23T20:43:32.580 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:43:32.580 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:43:32.580 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:43:32.580 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.580 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.580 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35994.0:84092 8.a 8:5d9db4b0:::benchmark_data_smithi018_341490_object84091:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e379)", 2023-09-23T20:43:32.580 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:37:13.366132+0000", 2023-09-23T20:43:32.580 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 379.11745192400002, 2023-09-23T20:43:32.580 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.3842849990000001, 2023-09-23T20:43:32.581 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:43:32.581 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:43:32.581 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:43:32.581 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35994", 2023-09-23T20:43:32.581 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:43:32.581 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 84092 2023-09-23T20:43:32.581 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.581 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:43:32.581 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.581 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:43:32.581 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.366132+0000", 2023-09-23T20:43:32.581 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:43:32.582 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.582 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.582 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:43:32.582 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.366132+0000", 2023-09-23T20:43:32.582 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:43:32.582 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.582 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.582 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:43:32.582 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.366131+0000", 2023-09-23T20:43:32.582 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:43:32.582 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.583 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.583 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:43:32.583 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.366154+0000", 2023-09-23T20:43:32.583 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.3272e-05 2023-09-23T20:43:32.583 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.583 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.583 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:43:32.583 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.366161+0000", 2023-09-23T20:43:32.583 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.046e-06 2023-09-23T20:43:32.583 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.583 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.583 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:43:32.584 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.366168+0000", 2023-09-23T20:43:32.584 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.3620000000000002e-06 2023-09-23T20:43:32.584 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.584 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.584 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:43:32.584 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.366275+0000", 2023-09-23T20:43:32.584 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000107376 2023-09-23T20:43:32.584 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.584 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.584 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:43:32.584 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.366286+0000", 2023-09-23T20:43:32.584 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1239e-05 2023-09-23T20:43:32.585 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.585 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.585 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:43:32.585 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:14.353936+0000", 2023-09-23T20:43:32.585 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.98764978299999995 2023-09-23T20:43:32.585 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.585 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.585 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:43:32.585 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:14.353944+0000", 2023-09-23T20:43:32.585 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.1340000000000004e-06 2023-09-23T20:43:32.585 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.586 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.586 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:43:32.586 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:15.714442+0000", 2023-09-23T20:43:32.586 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3604973929999999 2023-09-23T20:43:32.586 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.586 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.586 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:43:32.586 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:15.714618+0000", 2023-09-23T20:43:32.586 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00017675500000000001 2023-09-23T20:43:32.586 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.586 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.586 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 2,3", 2023-09-23T20:43:32.587 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:15.714894+0000", 2023-09-23T20:43:32.587 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00027531799999999998 2023-09-23T20:43:32.587 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.587 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.587 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:43:32.587 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:15.727877+0000", 2023-09-23T20:43:32.587 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.012983112999999999 2023-09-23T20:43:32.587 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.587 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.587 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:43:32.587 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:15.727983+0000", 2023-09-23T20:43:32.588 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000106573 2023-09-23T20:43:32.588 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.588 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.588 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:43:32.588 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:15.746758+0000", 2023-09-23T20:43:32.588 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.018774036000000001 2023-09-23T20:43:32.588 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.588 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.588 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:43:32.588 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:15.746834+0000", 2023-09-23T20:43:32.588 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.6039999999999997e-05 2023-09-23T20:43:32.588 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.589 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.589 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:43:32.589 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:15.750417+0000", 2023-09-23T20:43:32.589 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0035831880000000002 2023-09-23T20:43:32.589 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:43:32.589 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:43:32.589 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:43:32.589 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.589 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.589 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35994.0:84104 8.a 8:510db2a2:::benchmark_data_smithi018_341490_object84103:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e379)", 2023-09-23T20:43:32.589 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:37:13.384561+0000", 2023-09-23T20:43:32.589 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 379.09902218600001, 2023-09-23T20:43:32.590 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.368910107, 2023-09-23T20:43:32.590 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:43:32.590 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:43:32.590 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:43:32.590 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35994", 2023-09-23T20:43:32.590 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:43:32.590 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 84104 2023-09-23T20:43:32.590 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.590 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:43:32.590 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.590 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:43:32.591 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.384561+0000", 2023-09-23T20:43:32.591 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:43:32.591 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.591 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.591 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:43:32.591 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.384561+0000", 2023-09-23T20:43:32.591 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:43:32.591 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.591 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.591 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:43:32.591 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.384559+0000", 2023-09-23T20:43:32.591 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999976 2023-09-23T20:43:32.592 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.592 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.592 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:43:32.592 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.384588+0000", 2023-09-23T20:43:32.592 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.9560000000000002e-05 2023-09-23T20:43:32.592 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.592 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.592 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:43:32.592 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.384600+0000", 2023-09-23T20:43:32.592 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1868999999999999e-05 2023-09-23T20:43:32.592 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.593 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.593 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:43:32.593 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.384608+0000", 2023-09-23T20:43:32.593 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.0290000000000005e-06 2023-09-23T20:43:32.593 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.593 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.593 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:43:32.593 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.384667+0000", 2023-09-23T20:43:32.593 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.8554999999999998e-05 2023-09-23T20:43:32.593 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.593 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.593 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:43:32.593 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.384679+0000", 2023-09-23T20:43:32.594 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2193e-05 2023-09-23T20:43:32.594 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.594 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.594 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:43:32.594 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:14.353967+0000", 2023-09-23T20:43:32.594 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.96928828600000005 2023-09-23T20:43:32.594 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.594 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.594 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:43:32.594 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:14.353976+0000", 2023-09-23T20:43:32.594 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.4509999999999992e-06 2023-09-23T20:43:32.595 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.595 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.595 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:43:32.595 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:15.718028+0000", 2023-09-23T20:43:32.595 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3640520330000001 2023-09-23T20:43:32.595 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.595 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.595 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:43:32.595 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:15.718486+0000", 2023-09-23T20:43:32.595 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00045779600000000001 2023-09-23T20:43:32.595 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.595 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.596 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 2,3", 2023-09-23T20:43:32.596 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:15.718646+0000", 2023-09-23T20:43:32.596 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000160545 2023-09-23T20:43:32.596 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.596 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.596 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:43:32.596 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:15.727882+0000", 2023-09-23T20:43:32.596 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0092361939999999997 2023-09-23T20:43:32.596 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.596 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.596 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:43:32.596 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:15.730716+0000", 2023-09-23T20:43:32.597 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0028335600000000002 2023-09-23T20:43:32.597 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.597 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.597 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:43:32.597 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:15.750486+0000", 2023-09-23T20:43:32.597 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.019770167000000002 2023-09-23T20:43:32.597 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.597 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.597 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:43:32.597 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:15.750549+0000", 2023-09-23T20:43:32.597 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.2545999999999994e-05 2023-09-23T20:43:32.597 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.598 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.598 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:43:32.598 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:15.753472+0000", 2023-09-23T20:43:32.598 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0029228790000000002 2023-09-23T20:43:32.598 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:43:32.598 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:43:32.598 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:43:32.598 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.598 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.598 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35994.0:84111 8.0 8:0d10ef15:::benchmark_data_smithi018_341490_object84110:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e379)", 2023-09-23T20:43:32.598 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:37:13.393609+0000", 2023-09-23T20:43:32.599 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 379.08997507800001, 2023-09-23T20:43:32.599 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.2585651910000002, 2023-09-23T20:43:32.599 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:43:32.599 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:43:32.599 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:43:32.599 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35994", 2023-09-23T20:43:32.599 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:43:32.599 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 84111 2023-09-23T20:43:32.599 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.599 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:43:32.599 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.599 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:43:32.600 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.393609+0000", 2023-09-23T20:43:32.600 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:43:32.600 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.600 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.600 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:43:32.600 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.393609+0000", 2023-09-23T20:43:32.600 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:43:32.600 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.600 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.600 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:43:32.600 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.393608+0000", 2023-09-23T20:43:32.600 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:43:32.601 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.601 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.601 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:43:32.601 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.393626+0000", 2023-09-23T20:43:32.601 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.7727000000000002e-05 2023-09-23T20:43:32.601 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.601 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.601 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:43:32.601 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.393631+0000", 2023-09-23T20:43:32.601 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.9339999999999997e-06 2023-09-23T20:43:32.601 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.601 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.602 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:43:32.602 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.393635+0000", 2023-09-23T20:43:32.602 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.924e-06 2023-09-23T20:43:32.602 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.602 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.602 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:43:32.602 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.393697+0000", 2023-09-23T20:43:32.602 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.1972999999999998e-05 2023-09-23T20:43:32.602 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.602 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.602 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:43:32.603 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.393711+0000", 2023-09-23T20:43:32.603 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.4250000000000001e-05 2023-09-23T20:43:32.603 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.603 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.603 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:43:32.603 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:14.347541+0000", 2023-09-23T20:43:32.603 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.95382987600000002 2023-09-23T20:43:32.603 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.603 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.603 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:43:32.603 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:14.347552+0000", 2023-09-23T20:43:32.603 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1e-05 2023-09-23T20:43:32.604 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.604 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.604 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:43:32.604 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:15.603057+0000", 2023-09-23T20:43:32.604 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2555049060000001 2023-09-23T20:43:32.604 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.604 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.604 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:43:32.604 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:15.605409+0000", 2023-09-23T20:43:32.604 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0023523680000000001 2023-09-23T20:43:32.604 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.604 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.605 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:43:32.605 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:15.605629+0000", 2023-09-23T20:43:32.605 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00021984899999999999 2023-09-23T20:43:32.605 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.605 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.605 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:43:32.605 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:15.615897+0000", 2023-09-23T20:43:32.605 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.010267716 2023-09-23T20:43:32.605 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.605 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.605 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:43:32.606 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:15.631851+0000", 2023-09-23T20:43:32.606 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.015954349 2023-09-23T20:43:32.606 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.606 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.606 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:43:32.606 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:15.648581+0000", 2023-09-23T20:43:32.606 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.016730098999999998 2023-09-23T20:43:32.606 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.606 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.606 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:43:32.606 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:15.648651+0000", 2023-09-23T20:43:32.607 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.0188e-05 2023-09-23T20:43:32.607 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.607 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.607 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:43:32.607 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:15.652174+0000", 2023-09-23T20:43:32.607 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0035223810000000002 2023-09-23T20:43:32.607 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:43:32.607 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:43:32.607 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:43:32.607 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.607 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.607 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35994.0:84113 8.3 8:ca914450:::benchmark_data_smithi018_341490_object84112:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e379)", 2023-09-23T20:43:32.608 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:37:13.401619+0000", 2023-09-23T20:43:32.608 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 379.08196436700001, 2023-09-23T20:43:32.608 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.2600310069999998, 2023-09-23T20:43:32.608 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:43:32.608 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:43:32.608 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:43:32.608 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35994", 2023-09-23T20:43:32.608 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:43:32.608 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 84113 2023-09-23T20:43:32.608 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.608 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:43:32.609 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.609 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:43:32.609 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.401619+0000", 2023-09-23T20:43:32.609 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:43:32.609 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.609 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.609 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:43:32.609 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.401619+0000", 2023-09-23T20:43:32.609 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:43:32.609 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.609 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.609 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:43:32.610 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.401618+0000", 2023-09-23T20:43:32.610 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:43:32.610 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.610 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.610 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:43:32.610 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.401642+0000", 2023-09-23T20:43:32.610 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.3949000000000001e-05 2023-09-23T20:43:32.610 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.610 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.610 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:43:32.610 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.401654+0000", 2023-09-23T20:43:32.610 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2651e-05 2023-09-23T20:43:32.611 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.611 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.611 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:43:32.611 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.401661+0000", 2023-09-23T20:43:32.611 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.2389999999999999e-06 2023-09-23T20:43:32.611 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.611 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.611 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:43:32.611 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.401714+0000", 2023-09-23T20:43:32.611 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.3087000000000002e-05 2023-09-23T20:43:32.611 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.612 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.612 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:43:32.612 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:13.401721+0000", 2023-09-23T20:43:32.612 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.8339999999999999e-06 2023-09-23T20:43:32.612 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.612 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.612 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:43:32.612 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:14.349394+0000", 2023-09-23T20:43:32.612 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.94767251600000002 2023-09-23T20:43:32.612 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.612 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.612 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:43:32.613 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:14.349402+0000", 2023-09-23T20:43:32.613 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.3110000000000006e-06 2023-09-23T20:43:32.613 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.613 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.613 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:43:32.613 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:15.649982+0000", 2023-09-23T20:43:32.613 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3005794319999999 2023-09-23T20:43:32.613 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.613 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.613 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:43:32.613 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:15.650187+0000", 2023-09-23T20:43:32.614 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000205181 2023-09-23T20:43:32.614 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.614 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.614 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 2,6", 2023-09-23T20:43:32.614 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:15.650404+0000", 2023-09-23T20:43:32.614 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000217035 2023-09-23T20:43:32.614 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.614 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.614 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:43:32.614 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:15.653830+0000", 2023-09-23T20:43:32.614 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0034262009999999998 2023-09-23T20:43:32.614 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.615 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.615 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:43:32.615 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:15.658265+0000", 2023-09-23T20:43:32.615 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0044345979999999997 2023-09-23T20:43:32.615 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.615 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.615 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:43:32.615 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:15.658288+0000", 2023-09-23T20:43:32.615 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.3549000000000001e-05 2023-09-23T20:43:32.615 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.615 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.616 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:43:32.616 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:15.658336+0000", 2023-09-23T20:43:32.616 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.7372e-05 2023-09-23T20:43:32.616 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.616 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.616 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:43:32.616 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:37:15.661650+0000", 2023-09-23T20:43:32.616 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0033146180000000001 2023-09-23T20:43:32.616 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:43:32.616 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:43:32.616 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:43:32.616 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.617 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.617 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.36450.0:29567 9.4 9:2575c9a5:::benchmark_data_smithi018_381516_object29566:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e459)", 2023-09-23T20:43:32.617 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:41:21.062247+0000", 2023-09-23T20:43:32.617 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 131.42133637000001, 2023-09-23T20:43:32.617 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.140840776, 2023-09-23T20:43:32.617 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:43:32.617 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:43:32.617 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:43:32.617 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.36450", 2023-09-23T20:43:32.617 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:43:32.617 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 29567 2023-09-23T20:43:32.617 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.618 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:43:32.618 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.618 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:43:32.618 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:41:21.062247+0000", 2023-09-23T20:43:32.618 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:43:32.618 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.618 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.618 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:43:32.618 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:41:21.062247+0000", 2023-09-23T20:43:32.618 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:43:32.618 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.618 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.619 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:43:32.619 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:41:21.062246+0000", 2023-09-23T20:43:32.619 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:43:32.619 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.619 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.619 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:43:32.619 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:41:21.062264+0000", 2023-09-23T20:43:32.619 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.8488e-05 2023-09-23T20:43:32.619 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.619 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.619 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:43:32.619 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:41:21.062282+0000", 2023-09-23T20:43:32.620 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.7356000000000001e-05 2023-09-23T20:43:32.620 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.620 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.620 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:43:32.620 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:41:21.062288+0000", 2023-09-23T20:43:32.620 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.93e-06 2023-09-23T20:43:32.620 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.620 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.620 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:43:32.620 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:41:21.062328+0000", 2023-09-23T20:43:32.620 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.9935000000000003e-05 2023-09-23T20:43:32.620 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.621 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.621 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:43:32.621 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:41:21.062436+0000", 2023-09-23T20:43:32.621 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00010854 2023-09-23T20:43:32.621 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.621 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.621 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:43:32.621 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:41:21.062523+0000", 2023-09-23T20:43:32.621 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.6953999999999995e-05 2023-09-23T20:43:32.621 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.621 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.621 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:43:32.622 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:41:21.063560+0000", 2023-09-23T20:43:32.622 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0010364160000000001 2023-09-23T20:43:32.622 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.622 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.622 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:43:32.622 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:41:21.064173+0000", 2023-09-23T20:43:32.622 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000613892 2023-09-23T20:43:32.622 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.622 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.622 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:43:32.623 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:41:21.202986+0000", 2023-09-23T20:43:32.623 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.13881212600000001 2023-09-23T20:43:32.623 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.623 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.623 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:43:32.623 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:41:21.203040+0000", 2023-09-23T20:43:32.623 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.4239e-05 2023-09-23T20:43:32.623 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.623 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.623 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:43:32.623 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:41:21.203088+0000", 2023-09-23T20:43:32.623 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.8257000000000002e-05 2023-09-23T20:43:32.624 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:43:32.624 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:43:32.624 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:43:32.624 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.624 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.624 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.36450.0:92148 9.4 9:2b76a244:::benchmark_data_smithi018_381516_object92147:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e467)", 2023-09-23T20:43:32.624 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:42:14.325091+0000", 2023-09-23T20:43:32.624 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 78.158492401999993, 2023-09-23T20:43:32.624 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.11363255, 2023-09-23T20:43:32.624 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:43:32.624 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:43:32.625 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:43:32.625 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.36450", 2023-09-23T20:43:32.625 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:43:32.625 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 92148 2023-09-23T20:43:32.625 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.625 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:43:32.625 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.625 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:43:32.625 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:42:14.325091+0000", 2023-09-23T20:43:32.625 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:43:32.625 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.625 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.626 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:43:32.626 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:42:14.325091+0000", 2023-09-23T20:43:32.626 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:43:32.626 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.626 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.626 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:43:32.626 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:42:14.325090+0000", 2023-09-23T20:43:32.626 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:43:32.626 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.626 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.626 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:43:32.626 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:42:14.325113+0000", 2023-09-23T20:43:32.627 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.3218999999999999e-05 2023-09-23T20:43:32.627 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.627 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.627 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:43:32.627 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:42:14.325127+0000", 2023-09-23T20:43:32.627 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3905e-05 2023-09-23T20:43:32.627 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.627 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.627 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:43:32.627 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:42:14.325136+0000", 2023-09-23T20:43:32.627 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.9150000000000002e-06 2023-09-23T20:43:32.627 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.628 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.628 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:43:32.628 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:42:14.325185+0000", 2023-09-23T20:43:32.628 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.9564999999999997e-05 2023-09-23T20:43:32.628 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.628 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.628 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:43:32.628 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:42:14.325296+0000", 2023-09-23T20:43:32.628 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00011083500000000001 2023-09-23T20:43:32.628 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.628 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.628 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:43:32.629 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:42:14.325397+0000", 2023-09-23T20:43:32.629 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00010084799999999999 2023-09-23T20:43:32.629 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.629 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.629 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:43:32.629 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:42:14.327980+0000", 2023-09-23T20:43:32.629 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0025830150000000001 2023-09-23T20:43:32.629 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.629 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.629 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:43:32.629 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:42:14.329049+0000", 2023-09-23T20:43:32.630 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0010695959999999999 2023-09-23T20:43:32.630 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.630 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.630 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:43:32.630 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:42:14.438632+0000", 2023-09-23T20:43:32.630 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.109582601 2023-09-23T20:43:32.630 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.630 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.630 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:43:32.630 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:42:14.438684+0000", 2023-09-23T20:43:32.630 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.1681000000000002e-05 2023-09-23T20:43:32.630 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.631 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.631 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:43:32.631 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:42:14.438724+0000", 2023-09-23T20:43:32.631 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.0067000000000001e-05 2023-09-23T20:43:32.631 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:43:32.631 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:43:32.631 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:43:32.631 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.631 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.631 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.36450.0:141029 9.a 9:572023e9:::benchmark_data_smithi018_381516_object27195:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e481)", 2023-09-23T20:43:32.631 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:43:15.738002+0000", 2023-09-23T20:43:32.631 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 16.745581991999998, 2023-09-23T20:43:32.632 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0527425420000001, 2023-09-23T20:43:32.632 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:43:32.632 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:43:32.632 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:43:32.632 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.36450", 2023-09-23T20:43:32.632 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:43:32.632 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 141029 2023-09-23T20:43:32.632 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.632 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:43:32.632 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.632 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:43:32.633 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738002+0000", 2023-09-23T20:43:32.633 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:43:32.633 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.633 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.633 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:43:32.633 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738002+0000", 2023-09-23T20:43:32.633 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:43:32.633 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.633 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.633 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:43:32.633 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738000+0000", 2023-09-23T20:43:32.633 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:43:32.634 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.634 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.634 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:43:32.634 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738007+0000", 2023-09-23T20:43:32.634 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.4760000000000001e-06 2023-09-23T20:43:32.634 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.634 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.634 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:43:32.634 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738025+0000", 2023-09-23T20:43:32.634 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.7952e-05 2023-09-23T20:43:32.634 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.635 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.635 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:43:32.635 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738035+0000", 2023-09-23T20:43:32.635 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.6120000000000005e-06 2023-09-23T20:43:32.635 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.635 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.635 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:43:32.635 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.744123+0000", 2023-09-23T20:43:32.635 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0060876410000000004 2023-09-23T20:43:32.635 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.635 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.635 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op must wait for map", 2023-09-23T20:43:32.636 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.744131+0000", 2023-09-23T20:43:32.636 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.8800000000000008e-06 2023-09-23T20:43:32.636 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.636 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.636 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:43:32.636 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.754919+0000", 2023-09-23T20:43:32.636 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.010788483999999999 2023-09-23T20:43:32.636 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.636 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.636 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:43:32.636 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.754929+0000", 2023-09-23T20:43:32.637 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0309000000000001e-05 2023-09-23T20:43:32.637 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.637 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.637 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:43:32.637 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:16.790652+0000", 2023-09-23T20:43:32.637 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0357231570000001 2023-09-23T20:43:32.637 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.637 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.637 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:43:32.637 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:16.790744+0000", 2023-09-23T20:43:32.637 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.1743000000000006e-05 2023-09-23T20:43:32.637 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:43:32.638 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:43:32.638 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:43:32.638 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.638 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.638 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.36450.0:141040 9.a 9:53252326:::benchmark_data_smithi018_381516_object27206:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e481)", 2023-09-23T20:43:32.638 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:43:15.738057+0000", 2023-09-23T20:43:32.638 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 16.745526380000001, 2023-09-23T20:43:32.638 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0530090860000001, 2023-09-23T20:43:32.638 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:43:32.638 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:43:32.638 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:43:32.638 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.36450", 2023-09-23T20:43:32.639 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:43:32.639 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 141040 2023-09-23T20:43:32.639 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.639 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:43:32.639 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.639 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:43:32.639 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738057+0000", 2023-09-23T20:43:32.639 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:43:32.639 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.639 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.639 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:43:32.640 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738057+0000", 2023-09-23T20:43:32.640 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:43:32.640 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.640 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.640 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:43:32.640 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738057+0000", 2023-09-23T20:43:32.640 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:43:32.640 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.640 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.640 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:43:32.640 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738060+0000", 2023-09-23T20:43:32.640 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.9699999999999999e-06 2023-09-23T20:43:32.641 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.641 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.641 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:43:32.641 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738066+0000", 2023-09-23T20:43:32.641 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.5019999999999998e-06 2023-09-23T20:43:32.641 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.641 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.641 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:43:32.641 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738072+0000", 2023-09-23T20:43:32.641 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.2209999999999996e-06 2023-09-23T20:43:32.641 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.641 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.642 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:43:32.642 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.744157+0000", 2023-09-23T20:43:32.642 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0060856 2023-09-23T20:43:32.642 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.642 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.642 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting_for_map not empty", 2023-09-23T20:43:32.642 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.744162+0000", 2023-09-23T20:43:32.642 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.1610000000000002e-06 2023-09-23T20:43:32.642 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.642 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.642 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:43:32.642 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.754951+0000", 2023-09-23T20:43:32.643 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.010789127000000001 2023-09-23T20:43:32.643 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.643 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.643 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:43:32.643 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.754959+0000", 2023-09-23T20:43:32.643 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.3189999999999997e-06 2023-09-23T20:43:32.643 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.643 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.643 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:43:32.643 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:16.790999+0000", 2023-09-23T20:43:32.643 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.03604052 2023-09-23T20:43:32.643 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.644 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.644 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:43:32.644 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:16.791066+0000", 2023-09-23T20:43:32.644 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.7101999999999996e-05 2023-09-23T20:43:32.644 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:43:32.644 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:43:32.644 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:43:32.644 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.644 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.644 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.36450.0:141043 9.a 9:57a9fa95:::benchmark_data_smithi018_381516_object27209:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e481)", 2023-09-23T20:43:32.644 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:43:15.738077+0000", 2023-09-23T20:43:32.645 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 16.745506743, 2023-09-23T20:43:32.645 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0909135969999999, 2023-09-23T20:43:32.645 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:43:32.645 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:43:32.645 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:43:32.645 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.36450", 2023-09-23T20:43:32.645 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:43:32.645 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 141043 2023-09-23T20:43:32.645 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.645 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:43:32.645 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.645 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:43:32.646 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738077+0000", 2023-09-23T20:43:32.646 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:43:32.646 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.646 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.646 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:43:32.646 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738077+0000", 2023-09-23T20:43:32.646 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:43:32.646 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.646 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.646 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:43:32.646 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738077+0000", 2023-09-23T20:43:32.647 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:43:32.647 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.647 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.647 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:43:32.647 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738079+0000", 2023-09-23T20:43:32.647 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.0320000000000002e-06 2023-09-23T20:43:32.647 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.647 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.647 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:43:32.647 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738084+0000", 2023-09-23T20:43:32.647 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.4310000000000002e-06 2023-09-23T20:43:32.648 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.648 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.648 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:43:32.648 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738089+0000", 2023-09-23T20:43:32.648 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.6789999999999996e-06 2023-09-23T20:43:32.648 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.648 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.648 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:43:32.648 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.744187+0000", 2023-09-23T20:43:32.648 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0060979859999999997 2023-09-23T20:43:32.648 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.648 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.649 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting_for_map not empty", 2023-09-23T20:43:32.649 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.744194+0000", 2023-09-23T20:43:32.649 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.7090000000000002e-06 2023-09-23T20:43:32.649 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.649 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.649 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:43:32.649 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.754979+0000", 2023-09-23T20:43:32.649 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.010784141000000001 2023-09-23T20:43:32.649 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.649 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.649 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:43:32.650 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.754986+0000", 2023-09-23T20:43:32.650 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.0339999999999999e-06 2023-09-23T20:43:32.650 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.650 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.650 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:43:32.650 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:16.791090+0000", 2023-09-23T20:43:32.650 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.036103939 2023-09-23T20:43:32.650 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.650 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.650 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for missing object", 2023-09-23T20:43:32.650 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:16.791184+0000", 2023-09-23T20:43:32.650 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.4289000000000005e-05 2023-09-23T20:43:32.651 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.651 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.651 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:43:32.651 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:16.828905+0000", 2023-09-23T20:43:32.651 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.037721372000000003 2023-09-23T20:43:32.651 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.651 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.651 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:43:32.651 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:16.828991+0000", 2023-09-23T20:43:32.651 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.5331e-05 2023-09-23T20:43:32.651 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:43:32.651 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:43:32.652 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:43:32.652 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.652 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.652 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.36450.0:141045 9.f 9:fa55dbf8:::benchmark_data_smithi018_381516_object27211:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e481)", 2023-09-23T20:43:32.652 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:43:15.738093+0000", 2023-09-23T20:43:32.652 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 16.745490885999999, 2023-09-23T20:43:32.652 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0403492489999999, 2023-09-23T20:43:32.652 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:43:32.652 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:43:32.652 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:43:32.652 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.36450", 2023-09-23T20:43:32.653 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:43:32.653 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 141045 2023-09-23T20:43:32.653 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.653 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:43:32.653 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.653 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:43:32.653 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738093+0000", 2023-09-23T20:43:32.653 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:43:32.653 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.653 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.653 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:43:32.653 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738093+0000", 2023-09-23T20:43:32.654 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:43:32.654 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.654 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.654 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:43:32.654 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738092+0000", 2023-09-23T20:43:32.654 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:43:32.654 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.654 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.654 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:43:32.654 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738095+0000", 2023-09-23T20:43:32.654 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.063e-06 2023-09-23T20:43:32.655 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.655 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.655 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:43:32.655 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738100+0000", 2023-09-23T20:43:32.655 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.2469999999999999e-06 2023-09-23T20:43:32.655 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.655 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.655 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:43:32.655 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738104+0000", 2023-09-23T20:43:32.655 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.3649999999999997e-06 2023-09-23T20:43:32.655 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.655 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.656 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:43:32.656 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738271+0000", 2023-09-23T20:43:32.656 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00016673799999999999 2023-09-23T20:43:32.656 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.656 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.656 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op must wait for map", 2023-09-23T20:43:32.656 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738278+0000", 2023-09-23T20:43:32.656 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.0659999999999998e-06 2023-09-23T20:43:32.656 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.656 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.656 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:43:32.657 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.770314+0000", 2023-09-23T20:43:32.657 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.032036012000000003 2023-09-23T20:43:32.657 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.657 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.657 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:43:32.657 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.770322+0000", 2023-09-23T20:43:32.657 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.9330000000000001e-06 2023-09-23T20:43:32.657 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.657 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.657 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:43:32.657 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:16.776028+0000", 2023-09-23T20:43:32.657 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0057059290000001 2023-09-23T20:43:32.658 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.658 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.658 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:43:32.658 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:16.778442+0000", 2023-09-23T20:43:32.658 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0024142080000000002 2023-09-23T20:43:32.658 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:43:32.658 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:43:32.658 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:43:32.658 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.658 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.658 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.36450.0:141047 9.a 9:55e9d18e:::benchmark_data_smithi018_381516_object27213:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e481)", 2023-09-23T20:43:32.658 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:43:15.747358+0000", 2023-09-23T20:43:32.659 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 16.73622568, 2023-09-23T20:43:32.659 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.088479043, 2023-09-23T20:43:32.659 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:43:32.659 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:43:32.659 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:43:32.659 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.36450", 2023-09-23T20:43:32.659 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:43:32.659 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 141047 2023-09-23T20:43:32.659 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.659 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:43:32.660 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.660 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:43:32.660 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.747358+0000", 2023-09-23T20:43:32.660 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:43:32.660 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.660 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.660 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:43:32.660 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.747358+0000", 2023-09-23T20:43:32.660 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:43:32.660 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.660 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.660 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:43:32.661 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.747356+0000", 2023-09-23T20:43:32.661 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:43:32.661 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.661 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.661 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:43:32.661 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.747364+0000", 2023-09-23T20:43:32.661 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.6569999999999997e-06 2023-09-23T20:43:32.661 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.661 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.661 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:43:32.661 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.747379+0000", 2023-09-23T20:43:32.662 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.5153999999999999e-05 2023-09-23T20:43:32.662 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.662 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.662 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:43:32.662 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.747388+0000", 2023-09-23T20:43:32.662 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.7059999999999993e-06 2023-09-23T20:43:32.662 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.662 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.662 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:43:32.662 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.748082+0000", 2023-09-23T20:43:32.662 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00069463100000000002 2023-09-23T20:43:32.663 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.663 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.663 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting_for_map not empty", 2023-09-23T20:43:32.663 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.748090+0000", 2023-09-23T20:43:32.663 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.1349999999999992e-06 2023-09-23T20:43:32.663 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.663 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.663 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:43:32.663 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.755006+0000", 2023-09-23T20:43:32.663 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0069154289999999998 2023-09-23T20:43:32.663 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.663 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.664 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:43:32.664 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.755013+0000", 2023-09-23T20:43:32.664 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.7909999999999999e-06 2023-09-23T20:43:32.664 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.664 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.664 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:43:32.664 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:16.791216+0000", 2023-09-23T20:43:32.664 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.036203188 2023-09-23T20:43:32.664 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.664 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.664 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:43:32.665 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:16.791332+0000", 2023-09-23T20:43:32.665 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000115798 2023-09-23T20:43:32.665 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.665 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.665 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 5,7", 2023-09-23T20:43:32.665 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:16.791928+0000", 2023-09-23T20:43:32.665 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00059646299999999998 2023-09-23T20:43:32.665 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.665 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.665 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:43:32.665 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:16.822354+0000", 2023-09-23T20:43:32.665 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.030425812999999999 2023-09-23T20:43:32.666 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.666 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.666 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:43:32.666 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:16.822834+0000", 2023-09-23T20:43:32.666 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00048018300000000002 2023-09-23T20:43:32.666 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.666 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.666 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:43:32.666 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:16.835654+0000", 2023-09-23T20:43:32.666 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.012820066 2023-09-23T20:43:32.666 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.667 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.667 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:43:32.667 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:16.835711+0000", 2023-09-23T20:43:32.667 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.7300999999999999e-05 2023-09-23T20:43:32.667 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:32.667 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:32.667 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:43:32.667 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:16.835837+0000", 2023-09-23T20:43:32.667 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00012553400000000001 2023-09-23T20:43:32.667 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:43:32.667 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:43:32.667 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:43:32.668 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:43:32.668 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:43:32.668 INFO:teuthology.orchestra.run.smithi018.stdout:} 2023-09-23T20:43:32.669 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:32.669 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:32.669 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:43:32.669 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:43:32.770 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:43:32.830 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:32.916 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:43:32.920 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:32.920 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:43:33.021 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:43:33.082 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:32 smithi149 ceph-mon[170394]: 1.10 scrub starts 2023-09-23T20:43:33.082 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:32 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1979431532' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:43:33.082 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:32 smithi149 ceph-mon[170394]: pgmap v1224: 119 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 117 active+clean; 4.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 839 op/s; 7375/230502 objects misplaced (3.200%); 7.1 MiB/s, 113 objects/s recovering 2023-09-23T20:43:33.082 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:32 smithi149 ceph-mon[170394]: 1.17 scrub starts 2023-09-23T20:43:33.087 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:33.165 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:43:33.169 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:33.169 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:43:33.175 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T20:43:33.178 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:32 smithi018 ceph-mon[179647]: 1.10 scrub starts 2023-09-23T20:43:33.178 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:32 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1979431532' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:43:33.178 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:32 smithi018 ceph-mon[179647]: pgmap v1224: 119 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 117 active+clean; 4.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 839 op/s; 7375/230502 objects misplaced (3.200%); 7.1 MiB/s, 113 objects/s recovering 2023-09-23T20:43:33.178 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:32 smithi018 ceph-mon[179647]: 1.17 scrub starts 2023-09-23T20:43:33.179 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:32 smithi018 ceph-mon[184144]: 1.10 scrub starts 2023-09-23T20:43:33.179 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:32 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1979431532' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:43:33.179 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:32 smithi018 ceph-mon[184144]: pgmap v1224: 119 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 117 active+clean; 4.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 839 op/s; 7375/230502 objects misplaced (3.200%); 7.1 MiB/s, 113 objects/s recovering 2023-09-23T20:43:33.179 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:32 smithi018 ceph-mon[184144]: 1.17 scrub starts 2023-09-23T20:43:33.270 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:43:33.375 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:33.467 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:43:33.474 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:33.475 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:43:33.576 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:43:33.654 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:33.720 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:43:33.725 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:33.726 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:43:33.827 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:43:33.837 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T20:43:33.888 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:33.993 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:43:33.998 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:33.998 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:43:34.043 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:43:34.099 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:43:34.133 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:34.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:33 smithi018 ceph-mon[184144]: 1.1 scrub starts 2023-09-23T20:43:34.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:33 smithi018 ceph-mon[184144]: 1.17 scrub starts 2023-09-23T20:43:34.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:33 smithi018 ceph-mon[184144]: 1.1b scrub starts 2023-09-23T20:43:34.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:33 smithi018 ceph-mon[179647]: 1.1 scrub starts 2023-09-23T20:43:34.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:33 smithi018 ceph-mon[179647]: 1.17 scrub starts 2023-09-23T20:43:34.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:33 smithi018 ceph-mon[179647]: 1.1b scrub starts 2023-09-23T20:43:34.204 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:43:34.215 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:34.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:33 smithi149 ceph-mon[170394]: 1.1 scrub starts 2023-09-23T20:43:34.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:33 smithi149 ceph-mon[170394]: 1.17 scrub starts 2023-09-23T20:43:34.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:33 smithi149 ceph-mon[170394]: 1.1b scrub starts 2023-09-23T20:43:34.217 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:43:34.317 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:43:34.372 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:34.429 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:43:34.438 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:34.438 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:43:34.539 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:43:34.623 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:34.690 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:43:34.699 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:34.699 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:43:34.800 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:43:34.860 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:34.950 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 7, 3, 2, 4, 6, 5] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T20:43:34.950 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:43:34.950 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-09-23T20:43:34.950 INFO:tasks.thrashosds.thrasher:Setting osd 0 primary_affinity to 1.000000 2023-09-23T20:43:34.951 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 0 1 2023-09-23T20:43:34.971 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:43:34.977 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:34.977 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:43:34.983 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:43:35.078 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:43:35.183 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:34 smithi018 ceph-mon[179647]: 1.18 scrub starts 2023-09-23T20:43:35.184 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:34 smithi018 ceph-mon[179647]: pgmap v1225: 119 pgs: 1 active+clean+scrubbing, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 116 active+clean; 4.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 865 op/s; 7375/222765 objects misplaced (3.311%); 6.1 MiB/s, 97 objects/s recovering 2023-09-23T20:43:35.184 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:34 smithi018 ceph-mon[179647]: 1.1b scrub starts 2023-09-23T20:43:35.184 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:43:35 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:43:34] "GET /metrics HTTP/1.1" 200 36613 "" "Prometheus/2.43.0" 2023-09-23T20:43:35.184 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:34 smithi018 ceph-mon[184144]: 1.18 scrub starts 2023-09-23T20:43:35.184 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:34 smithi018 ceph-mon[184144]: pgmap v1225: 119 pgs: 1 active+clean+scrubbing, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 116 active+clean; 4.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 865 op/s; 7375/222765 objects misplaced (3.311%); 6.1 MiB/s, 97 objects/s recovering 2023-09-23T20:43:35.184 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:34 smithi018 ceph-mon[184144]: 1.1b scrub starts 2023-09-23T20:43:35.191 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:43:35.192 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:35.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:34 smithi149 ceph-mon[170394]: 1.18 scrub starts 2023-09-23T20:43:35.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:34 smithi149 ceph-mon[170394]: pgmap v1225: 119 pgs: 1 active+clean+scrubbing, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 116 active+clean; 4.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 865 op/s; 7375/222765 objects misplaced (3.311%); 6.1 MiB/s, 97 objects/s recovering 2023-09-23T20:43:35.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:34 smithi149 ceph-mon[170394]: 1.1b scrub starts 2023-09-23T20:43:35.427 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:43:35.432 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:35.432 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:43:35.533 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:43:35.649 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:35.822 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:43:35.832 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:35.832 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:43:35.933 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:43:35.986 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:36.052 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:43:36.059 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:36.059 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:43:36.060 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2023-09-23T20:43:36.060 INFO:teuthology.orchestra.run.smithi018.stdout: "ops": [ 2023-09-23T20:43:36.060 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:36.060 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_repop(client.36450.0:156807 9.5 e485/481 9:aee0f397:::benchmark_data_smithi018_381516_object42973:head v 485'9634, mlcod=485'9632)", 2023-09-23T20:43:36.061 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:43:36.049399+0000", 2023-09-23T20:43:36.061 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.0031775409999999999, 2023-09-23T20:43:36.061 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00321449, 2023-09-23T20:43:36.061 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:43:36.061 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "started", 2023-09-23T20:43:36.061 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:43:36.061 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:36.061 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:43:36.061 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:36.049399+0000", 2023-09-23T20:43:36.061 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:43:36.061 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:36.061 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:36.062 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:43:36.062 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:36.049399+0000", 2023-09-23T20:43:36.062 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:43:36.062 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:36.062 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:36.062 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:43:36.062 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:36.049399+0000", 2023-09-23T20:43:36.062 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:43:36.062 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:36.062 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:36.062 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:43:36.063 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:36.049403+0000", 2023-09-23T20:43:36.063 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.1710000000000004e-06 2023-09-23T20:43:36.063 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:36.063 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:36.063 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:43:36.063 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:36.049412+0000", 2023-09-23T20:43:36.063 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.5089999999999999e-06 2023-09-23T20:43:36.063 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:36.063 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:36.063 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:43:36.063 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:36.049417+0000", 2023-09-23T20:43:36.063 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.9389999999999996e-06 2023-09-23T20:43:36.064 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:36.064 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:36.064 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:43:36.064 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:36.049461+0000", 2023-09-23T20:43:36.064 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.3733000000000003e-05 2023-09-23T20:43:36.064 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:36.064 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:36.064 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:43:36.064 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:36.049492+0000", 2023-09-23T20:43:36.064 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.0642999999999998e-05 2023-09-23T20:43:36.064 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:43:36.064 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:43:36.065 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:43:36.065 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:36.065 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:36.065 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_repop(client.36450.0:156810 9.b e485/465 9:d13ad804:::benchmark_data_smithi018_381516_object42976:head v 485'9736, mlcod=485'9732)", 2023-09-23T20:43:36.065 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:43:36.049441+0000", 2023-09-23T20:43:36.065 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.003135756, 2023-09-23T20:43:36.065 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.003235164, 2023-09-23T20:43:36.065 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:43:36.065 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "started", 2023-09-23T20:43:36.065 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:43:36.065 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:36.066 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:43:36.066 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:36.049441+0000", 2023-09-23T20:43:36.066 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:43:36.066 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:36.066 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:36.066 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:43:36.066 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:36.049441+0000", 2023-09-23T20:43:36.066 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:43:36.066 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:36.066 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:36.066 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:43:36.066 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:36.049440+0000", 2023-09-23T20:43:36.067 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:43:36.067 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:36.067 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:36.067 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:43:36.067 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:36.049444+0000", 2023-09-23T20:43:36.067 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.8530000000000002e-06 2023-09-23T20:43:36.067 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:36.067 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:36.067 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:43:36.067 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:36.049450+0000", 2023-09-23T20:43:36.067 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.2169999999999999e-06 2023-09-23T20:43:36.067 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:36.068 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:36.068 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:43:36.068 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:36.049455+0000", 2023-09-23T20:43:36.068 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.4229999999999998e-06 2023-09-23T20:43:36.068 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:36.068 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:36.068 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:43:36.068 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:36.049506+0000", 2023-09-23T20:43:36.068 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.0937999999999999e-05 2023-09-23T20:43:36.068 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:36.068 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:36.068 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:43:36.069 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:36.049533+0000", 2023-09-23T20:43:36.069 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.7797000000000002e-05 2023-09-23T20:43:36.069 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:43:36.069 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:43:36.069 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:43:36.069 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:36.069 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:36.069 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_repop(client.36450.0:156813 9.b e485/465)", 2023-09-23T20:43:36.069 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:43:36.049465+0000", 2023-09-23T20:43:36.069 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.0031108720000000002, 2023-09-23T20:43:36.069 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0032453959999999998, 2023-09-23T20:43:36.069 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:43:36.070 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "queued for pg", 2023-09-23T20:43:36.070 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:43:36.070 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:36.070 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:43:36.070 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:36.049465+0000", 2023-09-23T20:43:36.070 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:43:36.070 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:36.070 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:36.070 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:43:36.070 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:36.049465+0000", 2023-09-23T20:43:36.070 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:43:36.071 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:36.071 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:36.071 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:43:36.071 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:36.049465+0000", 2023-09-23T20:43:36.071 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967296 2023-09-23T20:43:36.071 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:36.071 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:36.071 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:43:36.071 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:36.049468+0000", 2023-09-23T20:43:36.071 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.1070000000000001e-06 2023-09-23T20:43:36.071 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:36.071 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:36.072 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:43:36.072 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:36.049473+0000", 2023-09-23T20:43:36.072 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.0259999999999998e-06 2023-09-23T20:43:36.072 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:36.072 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:36.072 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:43:36.072 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:36.049477+0000", 2023-09-23T20:43:36.072 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.946e-06 2023-09-23T20:43:36.072 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:43:36.072 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:43:36.072 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:43:36.072 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:36.073 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:36.073 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_repop(client.36450.0:156815 9.5 e485/481)", 2023-09-23T20:43:36.073 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:43:36.050448+0000", 2023-09-23T20:43:36.073 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.0021283109999999999, 2023-09-23T20:43:36.073 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.002290065, 2023-09-23T20:43:36.073 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:43:36.073 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "queued for pg", 2023-09-23T20:43:36.073 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:43:36.073 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:36.073 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:43:36.073 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:36.050448+0000", 2023-09-23T20:43:36.073 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:43:36.074 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:36.074 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:36.074 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:43:36.074 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:36.050448+0000", 2023-09-23T20:43:36.074 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:43:36.074 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:36.074 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:36.074 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:43:36.074 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:36.050447+0000", 2023-09-23T20:43:36.074 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:43:36.074 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:36.075 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:36.075 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:43:36.075 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:36.050454+0000", 2023-09-23T20:43:36.075 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.2500000000000003e-06 2023-09-23T20:43:36.075 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:36.075 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:36.075 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:43:36.075 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:36.050467+0000", 2023-09-23T20:43:36.075 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3784000000000001e-05 2023-09-23T20:43:36.075 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:36.075 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:36.075 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:43:36.076 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:36.050475+0000", 2023-09-23T20:43:36.076 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.3370000000000002e-06 2023-09-23T20:43:36.076 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:43:36.076 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:43:36.076 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:43:36.076 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:36.076 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:36.076 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.36450.0:156822 9.8 9:1a48daf7:::benchmark_data_smithi018_381516_object42988:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e485)", 2023-09-23T20:43:36.076 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:43:36.051009+0000", 2023-09-23T20:43:36.076 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.001567237, 2023-09-23T20:43:36.076 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.001760691, 2023-09-23T20:43:36.076 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:43:36.077 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:43:36.077 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:43:36.077 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.36450", 2023-09-23T20:43:36.077 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:43:36.077 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 156822 2023-09-23T20:43:36.077 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:36.077 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:43:36.077 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:36.077 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:43:36.077 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:36.051009+0000", 2023-09-23T20:43:36.077 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:43:36.077 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:36.078 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:36.078 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:43:36.078 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:36.051009+0000", 2023-09-23T20:43:36.078 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:43:36.078 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:36.078 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:36.078 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:43:36.078 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:36.051008+0000", 2023-09-23T20:43:36.078 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:43:36.078 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:36.078 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:36.078 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:43:36.079 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:36.051013+0000", 2023-09-23T20:43:36.079 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.13e-06 2023-09-23T20:43:36.079 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:36.079 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:36.079 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:43:36.079 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:36.051030+0000", 2023-09-23T20:43:36.079 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.7557000000000001e-05 2023-09-23T20:43:36.079 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:36.079 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:36.079 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:43:36.079 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:36.051039+0000", 2023-09-23T20:43:36.079 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.9409999999999996e-06 2023-09-23T20:43:36.080 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:36.080 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:36.080 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:43:36.080 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:36.051091+0000", 2023-09-23T20:43:36.080 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.1588000000000001e-05 2023-09-23T20:43:36.080 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:36.080 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:36.080 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:43:36.080 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:36.051244+0000", 2023-09-23T20:43:36.080 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00015310200000000001 2023-09-23T20:43:36.080 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:36.080 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:36.081 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,3,5,7", 2023-09-23T20:43:36.081 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:36.051393+0000", 2023-09-23T20:43:36.081 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00014934599999999999 2023-09-23T20:43:36.081 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:43:36.081 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:43:36.081 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:43:36.081 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:36.081 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:36.081 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.36450.0:156808 9.8 9:122b4acb:::benchmark_data_smithi018_381516_object42974:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e485)", 2023-09-23T20:43:36.081 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:43:36.040600+0000", 2023-09-23T20:43:36.081 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.011976084, 2023-09-23T20:43:36.081 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.012238242, 2023-09-23T20:43:36.082 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:43:36.082 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:43:36.082 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:43:36.082 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.36450", 2023-09-23T20:43:36.082 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:43:36.082 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 156808 2023-09-23T20:43:36.082 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:36.082 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:43:36.082 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:36.082 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:43:36.082 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:36.040600+0000", 2023-09-23T20:43:36.083 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:43:36.083 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:36.083 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:36.083 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:43:36.083 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:36.040600+0000", 2023-09-23T20:43:36.083 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:43:36.083 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:36.083 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:36.083 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:43:36.083 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:36.040599+0000", 2023-09-23T20:43:36.083 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:43:36.083 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:36.084 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:36.084 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:43:36.084 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:36.040605+0000", 2023-09-23T20:43:36.084 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.4479999999999999e-06 2023-09-23T20:43:36.084 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:36.084 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:36.084 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:43:36.084 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:36.040619+0000", 2023-09-23T20:43:36.084 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3424000000000001e-05 2023-09-23T20:43:36.084 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:36.084 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:36.084 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:43:36.085 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:36.040625+0000", 2023-09-23T20:43:36.085 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.2910000000000004e-06 2023-09-23T20:43:36.085 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:36.085 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:36.085 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:43:36.085 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:36.040683+0000", 2023-09-23T20:43:36.085 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.8148e-05 2023-09-23T20:43:36.085 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:36.085 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:36.085 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:43:36.085 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:36.040881+0000", 2023-09-23T20:43:36.085 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000197264 2023-09-23T20:43:36.086 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:36.086 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:36.086 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,3,5,7", 2023-09-23T20:43:36.086 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:36.041094+0000", 2023-09-23T20:43:36.086 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00021387000000000001 2023-09-23T20:43:36.086 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:36.086 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:36.086 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:43:36.086 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:36.049847+0000", 2023-09-23T20:43:36.086 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0087530300000000002 2023-09-23T20:43:36.086 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:36.086 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:36.087 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:43:36.087 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:36.049891+0000", 2023-09-23T20:43:36.087 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.3562000000000001e-05 2023-09-23T20:43:36.087 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:36.087 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:36.087 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:43:36.087 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:36.049899+0000", 2023-09-23T20:43:36.087 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.9570000000000002e-06 2023-09-23T20:43:36.087 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:36.087 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:36.087 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:43:36.087 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:36.049972+0000", 2023-09-23T20:43:36.088 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.3418999999999996e-05 2023-09-23T20:43:36.088 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:43:36.088 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:43:36.088 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:43:36.088 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:36.088 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:36.088 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.36450.0:156812 9.8 9:1804c059:::benchmark_data_smithi018_381516_object42978:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e485)", 2023-09-23T20:43:36.088 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:43:36.042500+0000", 2023-09-23T20:43:36.088 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.010076379, 2023-09-23T20:43:36.088 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.01040642, 2023-09-23T20:43:36.088 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:43:36.089 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:43:36.089 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:43:36.089 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.36450", 2023-09-23T20:43:36.089 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:43:36.089 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 156812 2023-09-23T20:43:36.089 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:36.089 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:43:36.089 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:36.089 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:43:36.089 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:36.042500+0000", 2023-09-23T20:43:36.089 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:43:36.089 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:36.089 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:36.090 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:43:36.090 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:36.042500+0000", 2023-09-23T20:43:36.090 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:43:36.090 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:36.090 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:36.090 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:43:36.090 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:36.042499+0000", 2023-09-23T20:43:36.090 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.999999 2023-09-23T20:43:36.090 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:36.090 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:36.090 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:43:36.090 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:36.042505+0000", 2023-09-23T20:43:36.091 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.7100000000000004e-06 2023-09-23T20:43:36.091 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:36.091 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:36.091 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:43:36.091 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:36.042516+0000", 2023-09-23T20:43:36.091 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0947e-05 2023-09-23T20:43:36.091 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:36.091 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:36.091 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:43:36.091 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:36.042523+0000", 2023-09-23T20:43:36.091 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.7800000000000001e-06 2023-09-23T20:43:36.091 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:36.092 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:36.092 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:43:36.092 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:36.045297+0000", 2023-09-23T20:43:36.092 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.002773508 2023-09-23T20:43:36.092 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:36.092 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:36.092 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:43:36.092 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:36.045471+0000", 2023-09-23T20:43:36.092 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00017389 2023-09-23T20:43:36.092 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:36.092 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:36.093 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,3,5,7", 2023-09-23T20:43:36.093 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:36.045653+0000", 2023-09-23T20:43:36.093 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00018180299999999999 2023-09-23T20:43:36.093 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:36.093 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:36.093 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:43:36.093 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:36.049905+0000", 2023-09-23T20:43:36.093 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0042521399999999997 2023-09-23T20:43:36.093 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:36.093 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:36.093 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:43:36.093 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:36.049940+0000", 2023-09-23T20:43:36.094 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.5555000000000001e-05 2023-09-23T20:43:36.094 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:43:36.094 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:43:36.094 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:43:36.094 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:43:36.094 INFO:teuthology.orchestra.run.smithi018.stdout: ], 2023-09-23T20:43:36.094 INFO:teuthology.orchestra.run.smithi018.stdout: "num_ops": 7 2023-09-23T20:43:36.094 INFO:teuthology.orchestra.run.smithi018.stdout:} 2023-09-23T20:43:36.094 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:35 smithi018 ceph-mon[179647]: 1.17 scrub starts 2023-09-23T20:43:36.095 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:35 smithi018 ceph-mon[179647]: 1.1b scrub starts 2023-09-23T20:43:36.095 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:35 smithi018 ceph-mon[184144]: 1.17 scrub starts 2023-09-23T20:43:36.095 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:35 smithi018 ceph-mon[184144]: 1.1b scrub starts 2023-09-23T20:43:36.160 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:43:36.176 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:43:36.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:35 smithi149 ceph-mon[170394]: 1.17 scrub starts 2023-09-23T20:43:36.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:35 smithi149 ceph-mon[170394]: 1.1b scrub starts 2023-09-23T20:43:36.295 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:36.570 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:43:36.581 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:36.581 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:43:36.610 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T20:43:36.615 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:43:36.683 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:43:36.776 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:36.903 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:43:36.916 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:36.917 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:43:37.018 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:43:37.112 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:37.295 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:43:37.306 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:37.306 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:43:37.407 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:43:37.497 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:37.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:37 smithi018 ceph-mon[184144]: pgmap v1226: 119 pgs: 1 active+remapped+backfilling, 1 active+clean+scrubbing, 1 active+remapped+backfill_wait, 116 active+clean; 4.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 862 op/s; 6748/218691 objects misplaced (3.086%); 5.8 MiB/s, 92 objects/s recovering 2023-09-23T20:43:37.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:37 smithi018 ceph-mon[184144]: 1.a scrub starts 2023-09-23T20:43:37.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:37 smithi018 ceph-mon[179647]: pgmap v1226: 119 pgs: 1 active+remapped+backfilling, 1 active+clean+scrubbing, 1 active+remapped+backfill_wait, 116 active+clean; 4.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 862 op/s; 6748/218691 objects misplaced (3.086%); 5.8 MiB/s, 92 objects/s recovering 2023-09-23T20:43:37.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:37 smithi018 ceph-mon[179647]: 1.a scrub starts 2023-09-23T20:43:37.969 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:37 smithi149 ceph-mon[170394]: pgmap v1226: 119 pgs: 1 active+remapped+backfilling, 1 active+clean+scrubbing, 1 active+remapped+backfill_wait, 116 active+clean; 4.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 862 op/s; 6748/218691 objects misplaced (3.086%); 5.8 MiB/s, 92 objects/s recovering 2023-09-23T20:43:37.969 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:37 smithi149 ceph-mon[170394]: 1.a scrub starts 2023-09-23T20:43:38.529 INFO:teuthology.orchestra.run.smithi018.stderr:nodeep-scrub is unset 2023-09-23T20:43:38.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:38 smithi018 ceph-mon[179647]: 1.10 scrub starts 2023-09-23T20:43:38.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:38 smithi018 ceph-mon[179647]: 1.17 scrub starts 2023-09-23T20:43:38.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:38 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1792243966' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:43:38.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:38 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:43:38.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:38 smithi018 ceph-mon[184144]: 1.10 scrub starts 2023-09-23T20:43:38.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:38 smithi018 ceph-mon[184144]: 1.17 scrub starts 2023-09-23T20:43:38.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:38 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1792243966' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:43:38.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:38 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:43:38.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:38 smithi149 ceph-mon[170394]: 1.10 scrub starts 2023-09-23T20:43:38.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:38 smithi149 ceph-mon[170394]: 1.17 scrub starts 2023-09-23T20:43:38.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:38 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1792243966' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:43:38.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:38 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:43:39.100 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:43:39.109 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:39.110 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:43:39.210 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:43:39.239 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:39.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:39 smithi018 ceph-mon[179647]: 1.10 scrub starts 2023-09-23T20:43:39.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:39 smithi018 ceph-mon[179647]: pgmap v1227: 119 pgs: 1 active+remapped+backfill_wait, 3 active+clean+scrubbing, 1 active+remapped+backfilling, 114 active+clean; 4.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 832 op/s; 6528/213636 objects misplaced (3.056%); 5.8 MiB/s, 92 objects/s recovering 2023-09-23T20:43:39.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:39 smithi018 ceph-mon[179647]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:43:39.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:39 smithi018 ceph-mon[179647]: Cluster is now healthy 2023-09-23T20:43:39.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:39 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:43:39.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:39 smithi018 ceph-mon[179647]: osdmap e486: 8 total, 8 up, 8 in 2023-09-23T20:43:39.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:39 smithi018 ceph-mon[179647]: failed to encode map e486 with expected crc 2023-09-23T20:43:39.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:39 smithi018 ceph-mon[179647]: failed to encode map e486 with expected crc 2023-09-23T20:43:39.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:39 smithi018 ceph-mon[179647]: failed to encode map e486 with expected crc 2023-09-23T20:43:39.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:39 smithi018 ceph-mon[179647]: failed to encode map e486 with expected crc 2023-09-23T20:43:39.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:39 smithi018 ceph-mon[184144]: 1.10 scrub starts 2023-09-23T20:43:39.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:39 smithi018 ceph-mon[184144]: pgmap v1227: 119 pgs: 1 active+remapped+backfill_wait, 3 active+clean+scrubbing, 1 active+remapped+backfilling, 114 active+clean; 4.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 832 op/s; 6528/213636 objects misplaced (3.056%); 5.8 MiB/s, 92 objects/s recovering 2023-09-23T20:43:39.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:39 smithi018 ceph-mon[184144]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:43:39.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:39 smithi018 ceph-mon[184144]: Cluster is now healthy 2023-09-23T20:43:39.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:39 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:43:39.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:39 smithi018 ceph-mon[184144]: osdmap e486: 8 total, 8 up, 8 in 2023-09-23T20:43:39.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:39 smithi018 ceph-mon[184144]: failed to encode map e486 with expected crc 2023-09-23T20:43:39.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:39 smithi018 ceph-mon[184144]: failed to encode map e486 with expected crc 2023-09-23T20:43:39.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:39 smithi018 ceph-mon[184144]: failed to encode map e486 with expected crc 2023-09-23T20:43:39.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:39 smithi018 ceph-mon[184144]: failed to encode map e486 with expected crc 2023-09-23T20:43:39.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:39 smithi149 ceph-mon[170394]: 1.10 scrub starts 2023-09-23T20:43:39.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:39 smithi149 ceph-mon[170394]: pgmap v1227: 119 pgs: 1 active+remapped+backfill_wait, 3 active+clean+scrubbing, 1 active+remapped+backfilling, 114 active+clean; 4.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 832 op/s; 6528/213636 objects misplaced (3.056%); 5.8 MiB/s, 92 objects/s recovering 2023-09-23T20:43:39.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:39 smithi149 ceph-mon[170394]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:43:39.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:39 smithi149 ceph-mon[170394]: Cluster is now healthy 2023-09-23T20:43:39.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:39 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:43:39.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:39 smithi149 ceph-mon[170394]: osdmap e486: 8 total, 8 up, 8 in 2023-09-23T20:43:39.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:39 smithi149 ceph-mon[170394]: failed to encode map e486 with expected crc 2023-09-23T20:43:39.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:39 smithi149 ceph-mon[170394]: failed to encode map e486 with expected crc 2023-09-23T20:43:39.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:39 smithi149 ceph-mon[170394]: failed to encode map e486 with expected crc 2023-09-23T20:43:39.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:39 smithi149 ceph-mon[170394]: failed to encode map e486 with expected crc 2023-09-23T20:43:40.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:40 smithi018 ceph-mon[179647]: 1.17 scrub starts 2023-09-23T20:43:40.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:40 smithi018 ceph-mon[179647]: 1.1b scrub starts 2023-09-23T20:43:40.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:40 smithi018 ceph-mon[184144]: 1.17 scrub starts 2023-09-23T20:43:40.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:40 smithi018 ceph-mon[184144]: 1.1b scrub starts 2023-09-23T20:43:40.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:40 smithi149 ceph-mon[170394]: 1.17 scrub starts 2023-09-23T20:43:40.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:40 smithi149 ceph-mon[170394]: 1.1b scrub starts 2023-09-23T20:43:41.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:41 smithi018 ceph-mon[184144]: pgmap v1229: 119 pgs: 1 active+remapped+backfill_wait, 3 active+clean+scrubbing, 1 active+remapped+backfilling, 114 active+clean; 4.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 857 op/s; 6528/207432 objects misplaced (3.147%); 3.2 MiB/s, 50 objects/s recovering 2023-09-23T20:43:41.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:41 smithi018 ceph-mon[184144]: 1.17 scrub starts 2023-09-23T20:43:41.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:41 smithi018 ceph-mon[179647]: pgmap v1229: 119 pgs: 1 active+remapped+backfill_wait, 3 active+clean+scrubbing, 1 active+remapped+backfilling, 114 active+clean; 4.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 857 op/s; 6528/207432 objects misplaced (3.147%); 3.2 MiB/s, 50 objects/s recovering 2023-09-23T20:43:41.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:41 smithi018 ceph-mon[179647]: 1.17 scrub starts 2023-09-23T20:43:41.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:41 smithi149 ceph-mon[170394]: pgmap v1229: 119 pgs: 1 active+remapped+backfill_wait, 3 active+clean+scrubbing, 1 active+remapped+backfilling, 114 active+clean; 4.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 857 op/s; 6528/207432 objects misplaced (3.147%); 3.2 MiB/s, 50 objects/s recovering 2023-09-23T20:43:41.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:41 smithi149 ceph-mon[170394]: 1.17 scrub starts 2023-09-23T20:43:42.212 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:43:42.224 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:42.225 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:43:42.326 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:43:42.369 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:42.464 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:43:42.480 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:42.481 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:43:42.582 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:43:42.608 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:42.673 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:43:42.686 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:42 smithi149 ceph-mon[170394]: 1.17 scrub starts 2023-09-23T20:43:42.687 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:42.687 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:43:42.788 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:43:42.830 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:42.873 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:43:42.880 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:42.880 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:43:42.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:42 smithi018 ceph-mon[184144]: 1.17 scrub starts 2023-09-23T20:43:42.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:42 smithi018 ceph-mon[179647]: 1.17 scrub starts 2023-09-23T20:43:42.981 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:43:43.025 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:43.085 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:43:43.098 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:43.098 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:43:43.199 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:43:43.281 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:43.287 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:43:43.336 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:43:43.341 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:43.341 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:43:43.442 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:43:43.556 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:43.885 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:43 smithi018 ceph-mon[184144]: 1.10 scrub starts 2023-09-23T20:43:43.886 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:43 smithi018 ceph-mon[184144]: pgmap v1230: 119 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 3 active+clean+scrubbing, 114 active+clean; 4.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 818 op/s; 5711/205941 objects misplaced (2.773%); 5.9 MiB/s, 94 objects/s recovering 2023-09-23T20:43:43.886 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:43 smithi018 ceph-mon[184144]: 1.17 scrub starts 2023-09-23T20:43:43.887 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:43 smithi018 ceph-mon[179647]: 1.10 scrub starts 2023-09-23T20:43:43.887 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:43 smithi018 ceph-mon[179647]: pgmap v1230: 119 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 3 active+clean+scrubbing, 114 active+clean; 4.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 818 op/s; 5711/205941 objects misplaced (2.773%); 5.9 MiB/s, 94 objects/s recovering 2023-09-23T20:43:43.887 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:43 smithi018 ceph-mon[179647]: 1.17 scrub starts 2023-09-23T20:43:43.966 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:43:43.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:43 smithi149 ceph-mon[170394]: 1.10 scrub starts 2023-09-23T20:43:43.968 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:43 smithi149 ceph-mon[170394]: pgmap v1230: 119 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 3 active+clean+scrubbing, 114 active+clean; 4.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 818 op/s; 5711/205941 objects misplaced (2.773%); 5.9 MiB/s, 94 objects/s recovering 2023-09-23T20:43:43.968 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:43 smithi149 ceph-mon[170394]: 1.17 scrub starts 2023-09-23T20:43:43.975 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:43.975 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:43:44.076 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:43:44.159 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:44.343 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T20:43:44.434 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:43:44.437 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:44.438 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:43:44.540 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:43:44.586 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:44.627 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:43:44.643 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:43:44.651 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:44.652 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:43:44.695 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:43:44.742 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:43:44.752 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:43:44.789 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:43:44.792 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:44.814 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:43:44.849 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:43:44.856 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:43:44.865 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:44 smithi149 ceph-mon[170394]: 1.1 scrub starts 2023-09-23T20:43:44.865 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:44 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:43:44.865 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:44 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:43:44.866 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:44.867 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:43:44.890 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:43:44.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:44 smithi018 ceph-mon[184144]: 1.1 scrub starts 2023-09-23T20:43:44.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:44 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:43:44.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:44 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:43:44.895 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:44 smithi018 ceph-mon[179647]: 1.1 scrub starts 2023-09-23T20:43:44.895 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:44 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:43:44.895 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:44 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:43:44.927 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:43:44.929 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:43:44.929 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:43:44.929 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:43:44.929 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:43:44.929 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:43:44.929 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:43:44.929 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:43:44.930 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:43:44.967 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:43:45.036 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:45.102 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:43:45.109 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:45.110 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:43:45.210 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:43:45.246 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:45.348 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:43:45.355 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:45.356 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:43:45.395 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:43:44 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:43:44] "GET /metrics HTTP/1.1" 200 36599 "" "Prometheus/2.43.0" 2023-09-23T20:43:45.457 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:43:45.517 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:45.636 INFO:teuthology.orchestra.run.smithi018.stderr:set osd.0 primary-affinity to 1 (8655362) 2023-09-23T20:43:45.732 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:45 smithi018 ceph-mon[179647]: 1.10 scrub starts 2023-09-23T20:43:45.733 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:45 smithi018 ceph-mon[179647]: pgmap v1231: 119 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 117 active+clean; 4.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 847 op/s; 5711/197328 objects misplaced (2.894%); 5.9 MiB/s, 94 objects/s recovering 2023-09-23T20:43:45.733 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:45 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2677415047' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:43:45.733 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:45 smithi018 ceph-mon[179647]: 1.1b scrub starts 2023-09-23T20:43:45.733 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:45 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1590153189' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T20:43:45.733 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:45 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T20:43:45.734 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:45 smithi018 ceph-mon[184144]: 1.10 scrub starts 2023-09-23T20:43:45.734 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:45 smithi018 ceph-mon[184144]: pgmap v1231: 119 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 117 active+clean; 4.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 847 op/s; 5711/197328 objects misplaced (2.894%); 5.9 MiB/s, 94 objects/s recovering 2023-09-23T20:43:45.734 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:45 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2677415047' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:43:45.734 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:45 smithi018 ceph-mon[184144]: 1.1b scrub starts 2023-09-23T20:43:45.735 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:45 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1590153189' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T20:43:45.735 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:45 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T20:43:45.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:45 smithi149 ceph-mon[170394]: 1.10 scrub starts 2023-09-23T20:43:45.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:45 smithi149 ceph-mon[170394]: pgmap v1231: 119 pgs: 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 117 active+clean; 4.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 847 op/s; 5711/197328 objects misplaced (2.894%); 5.9 MiB/s, 94 objects/s recovering 2023-09-23T20:43:45.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:45 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2677415047' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:43:45.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:45 smithi149 ceph-mon[170394]: 1.1b scrub starts 2023-09-23T20:43:45.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:45 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1590153189' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T20:43:45.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:45 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T20:43:46.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:46 smithi018 ceph-mon[184144]: 1.10 scrub starts 2023-09-23T20:43:46.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:46 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2023-09-23T20:43:46.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:46 smithi018 ceph-mon[184144]: osdmap e487: 8 total, 8 up, 8 in 2023-09-23T20:43:46.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:46 smithi018 ceph-mon[184144]: failed to encode map e487 with expected crc 2023-09-23T20:43:46.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:46 smithi018 ceph-mon[184144]: failed to encode map e487 with expected crc 2023-09-23T20:43:46.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:46 smithi018 ceph-mon[184144]: failed to encode map e487 with expected crc 2023-09-23T20:43:46.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:46 smithi018 ceph-mon[184144]: failed to encode map e487 with expected crc 2023-09-23T20:43:46.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:46 smithi018 ceph-mon[184144]: 1.18 scrub starts 2023-09-23T20:43:46.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:46 smithi018 ceph-mon[184144]: osdmap e488: 8 total, 8 up, 8 in 2023-09-23T20:43:46.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:46 smithi018 ceph-mon[179647]: 1.10 scrub starts 2023-09-23T20:43:46.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:46 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2023-09-23T20:43:46.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:46 smithi018 ceph-mon[179647]: osdmap e487: 8 total, 8 up, 8 in 2023-09-23T20:43:46.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:46 smithi018 ceph-mon[179647]: failed to encode map e487 with expected crc 2023-09-23T20:43:46.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:46 smithi018 ceph-mon[179647]: failed to encode map e487 with expected crc 2023-09-23T20:43:46.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:46 smithi018 ceph-mon[179647]: failed to encode map e487 with expected crc 2023-09-23T20:43:46.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:46 smithi018 ceph-mon[179647]: failed to encode map e487 with expected crc 2023-09-23T20:43:46.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:46 smithi018 ceph-mon[179647]: 1.18 scrub starts 2023-09-23T20:43:46.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:46 smithi018 ceph-mon[179647]: osdmap e488: 8 total, 8 up, 8 in 2023-09-23T20:43:46.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:46 smithi149 ceph-mon[170394]: 1.10 scrub starts 2023-09-23T20:43:46.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:46 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2023-09-23T20:43:46.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:46 smithi149 ceph-mon[170394]: osdmap e487: 8 total, 8 up, 8 in 2023-09-23T20:43:46.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:46 smithi149 ceph-mon[170394]: failed to encode map e487 with expected crc 2023-09-23T20:43:46.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:46 smithi149 ceph-mon[170394]: failed to encode map e487 with expected crc 2023-09-23T20:43:46.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:46 smithi149 ceph-mon[170394]: failed to encode map e487 with expected crc 2023-09-23T20:43:46.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:46 smithi149 ceph-mon[170394]: failed to encode map e487 with expected crc 2023-09-23T20:43:46.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:46 smithi149 ceph-mon[170394]: 1.18 scrub starts 2023-09-23T20:43:46.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:46 smithi149 ceph-mon[170394]: osdmap e488: 8 total, 8 up, 8 in 2023-09-23T20:43:47.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:47 smithi018 ceph-mon[179647]: pgmap v1233: 119 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 117 active+clean; 3.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 856 op/s; 5084/193071 objects misplaced (2.633%); 7.7 MiB/s, 122 objects/s recovering 2023-09-23T20:43:47.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:47 smithi018 ceph-mon[179647]: failed to encode map e488 with expected crc 2023-09-23T20:43:47.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:47 smithi018 ceph-mon[179647]: failed to encode map e488 with expected crc 2023-09-23T20:43:47.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:47 smithi018 ceph-mon[179647]: 9.8 continuing backfill to osd.1 from (461'3000,486'10410] 9:18c77fc6:::benchmark_data_smithi018_381516_object89540:head to 486'10410 2023-09-23T20:43:47.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:47 smithi018 ceph-mon[179647]: 9.8 continuing backfill to osd.7 from (461'3000,486'10410] 9:18c77fc6:::benchmark_data_smithi018_381516_object89540:head to 486'10410 2023-09-23T20:43:47.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:47 smithi018 ceph-mon[179647]: failed to encode map e488 with expected crc 2023-09-23T20:43:47.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:47 smithi018 ceph-mon[179647]: failed to encode map e488 with expected crc 2023-09-23T20:43:47.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:47 smithi018 ceph-mon[179647]: failed to encode map e488 with expected crc 2023-09-23T20:43:47.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:47 smithi018 ceph-mon[179647]: osdmap e489: 8 total, 8 up, 8 in 2023-09-23T20:43:47.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:47 smithi018 ceph-mon[184144]: pgmap v1233: 119 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 117 active+clean; 3.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 856 op/s; 5084/193071 objects misplaced (2.633%); 7.7 MiB/s, 122 objects/s recovering 2023-09-23T20:43:47.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:47 smithi018 ceph-mon[184144]: failed to encode map e488 with expected crc 2023-09-23T20:43:47.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:47 smithi018 ceph-mon[184144]: failed to encode map e488 with expected crc 2023-09-23T20:43:47.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:47 smithi018 ceph-mon[184144]: 9.8 continuing backfill to osd.1 from (461'3000,486'10410] 9:18c77fc6:::benchmark_data_smithi018_381516_object89540:head to 486'10410 2023-09-23T20:43:47.894 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:47 smithi018 ceph-mon[184144]: 9.8 continuing backfill to osd.7 from (461'3000,486'10410] 9:18c77fc6:::benchmark_data_smithi018_381516_object89540:head to 486'10410 2023-09-23T20:43:47.894 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:47 smithi018 ceph-mon[184144]: failed to encode map e488 with expected crc 2023-09-23T20:43:47.894 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:47 smithi018 ceph-mon[184144]: failed to encode map e488 with expected crc 2023-09-23T20:43:47.894 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:47 smithi018 ceph-mon[184144]: failed to encode map e488 with expected crc 2023-09-23T20:43:47.894 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:47 smithi018 ceph-mon[184144]: osdmap e489: 8 total, 8 up, 8 in 2023-09-23T20:43:47.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:47 smithi149 ceph-mon[170394]: pgmap v1233: 119 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 117 active+clean; 3.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 856 op/s; 5084/193071 objects misplaced (2.633%); 7.7 MiB/s, 122 objects/s recovering 2023-09-23T20:43:47.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:47 smithi149 ceph-mon[170394]: failed to encode map e488 with expected crc 2023-09-23T20:43:47.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:47 smithi149 ceph-mon[170394]: failed to encode map e488 with expected crc 2023-09-23T20:43:47.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:47 smithi149 ceph-mon[170394]: 9.8 continuing backfill to osd.1 from (461'3000,486'10410] 9:18c77fc6:::benchmark_data_smithi018_381516_object89540:head to 486'10410 2023-09-23T20:43:47.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:47 smithi149 ceph-mon[170394]: 9.8 continuing backfill to osd.7 from (461'3000,486'10410] 9:18c77fc6:::benchmark_data_smithi018_381516_object89540:head to 486'10410 2023-09-23T20:43:47.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:47 smithi149 ceph-mon[170394]: failed to encode map e488 with expected crc 2023-09-23T20:43:47.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:47 smithi149 ceph-mon[170394]: failed to encode map e488 with expected crc 2023-09-23T20:43:47.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:47 smithi149 ceph-mon[170394]: failed to encode map e488 with expected crc 2023-09-23T20:43:47.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:47 smithi149 ceph-mon[170394]: osdmap e489: 8 total, 8 up, 8 in 2023-09-23T20:43:48.413 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:43:48.418 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:48.418 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:43:48.519 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:43:48.591 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:48.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:48 smithi018 ceph-mon[179647]: failed to encode map e489 with expected crc 2023-09-23T20:43:48.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:48 smithi018 ceph-mon[179647]: 9.5 continuing backfill to osd.1 from (460'2600,486'10127] 9:aac00bb4:::benchmark_data_smithi018_381516_object88723:head to 486'10127 2023-09-23T20:43:48.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:48 smithi018 ceph-mon[179647]: failed to encode map e489 with expected crc 2023-09-23T20:43:48.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:48 smithi018 ceph-mon[179647]: failed to encode map e489 with expected crc 2023-09-23T20:43:48.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:48 smithi018 ceph-mon[179647]: failed to encode map e489 with expected crc 2023-09-23T20:43:48.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:48 smithi018 ceph-mon[184144]: failed to encode map e489 with expected crc 2023-09-23T20:43:48.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:48 smithi018 ceph-mon[184144]: 9.5 continuing backfill to osd.1 from (460'2600,486'10127] 9:aac00bb4:::benchmark_data_smithi018_381516_object88723:head to 486'10127 2023-09-23T20:43:48.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:48 smithi018 ceph-mon[184144]: failed to encode map e489 with expected crc 2023-09-23T20:43:48.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:48 smithi018 ceph-mon[184144]: failed to encode map e489 with expected crc 2023-09-23T20:43:48.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:48 smithi018 ceph-mon[184144]: failed to encode map e489 with expected crc 2023-09-23T20:43:48.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:48 smithi149 ceph-mon[170394]: failed to encode map e489 with expected crc 2023-09-23T20:43:48.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:48 smithi149 ceph-mon[170394]: 9.5 continuing backfill to osd.1 from (460'2600,486'10127] 9:aac00bb4:::benchmark_data_smithi018_381516_object88723:head to 486'10127 2023-09-23T20:43:48.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:48 smithi149 ceph-mon[170394]: failed to encode map e489 with expected crc 2023-09-23T20:43:48.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:48 smithi149 ceph-mon[170394]: failed to encode map e489 with expected crc 2023-09-23T20:43:48.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:48 smithi149 ceph-mon[170394]: failed to encode map e489 with expected crc 2023-09-23T20:43:49.661 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:43:49.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:49 smithi018 ceph-mon[184144]: pgmap v1236: 119 pgs: 1 remapped+peering, 17 peering, 1 active+remapped+backfilling, 100 active+clean; 3.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 6.8 KiB/s rd, 18 KiB/s wr, 951 op/s; 1214/188913 objects misplaced (0.643%); 5.3 MiB/s, 85 objects/s recovering 2023-09-23T20:43:49.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:49 smithi018 ceph-mon[184144]: Health check failed: Reduced data availability: 8 pgs inactive, 15 pgs peering (PG_AVAILABILITY) 2023-09-23T20:43:49.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:49 smithi018 ceph-mon[179647]: pgmap v1236: 119 pgs: 1 remapped+peering, 17 peering, 1 active+remapped+backfilling, 100 active+clean; 3.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 6.8 KiB/s rd, 18 KiB/s wr, 951 op/s; 1214/188913 objects misplaced (0.643%); 5.3 MiB/s, 85 objects/s recovering 2023-09-23T20:43:49.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:49 smithi018 ceph-mon[179647]: Health check failed: Reduced data availability: 8 pgs inactive, 15 pgs peering (PG_AVAILABILITY) 2023-09-23T20:43:49.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:49 smithi149 ceph-mon[170394]: pgmap v1236: 119 pgs: 1 remapped+peering, 17 peering, 1 active+remapped+backfilling, 100 active+clean; 3.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 6.8 KiB/s rd, 18 KiB/s wr, 951 op/s; 1214/188913 objects misplaced (0.643%); 5.3 MiB/s, 85 objects/s recovering 2023-09-23T20:43:49.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:49 smithi149 ceph-mon[170394]: Health check failed: Reduced data availability: 8 pgs inactive, 15 pgs peering (PG_AVAILABILITY) 2023-09-23T20:43:50.483 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:43:50.487 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:50.488 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:43:50.588 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:43:50.673 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:50.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:50 smithi018 ceph-mon[184144]: 1.1b scrub starts 2023-09-23T20:43:50.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:50 smithi018 ceph-mon[179647]: 1.1b scrub starts 2023-09-23T20:43:50.968 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:50 smithi149 ceph-mon[170394]: 1.1b scrub starts 2023-09-23T20:43:51.170 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:43:51.172 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:51.172 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:43:51.172 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:43:51.273 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:43:51.327 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:51.387 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:43:51.393 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:51.393 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:43:51.494 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:43:51.550 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:51.607 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:43:51.610 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:51.611 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:43:51.695 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:43:51.712 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:43:51.760 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:51.809 INFO:teuthology.orchestra.run.smithi018.stdout:ERROR: (22) Invalid argument 2023-09-23T20:43:51.809 INFO:teuthology.orchestra.run.smithi018.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:43:51.820 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:43:51.828 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:51.828 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:43:51.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:51 smithi018 ceph-mon[179647]: pgmap v1237: 119 pgs: 1 remapped+peering, 17 peering, 1 active+remapped+backfilling, 100 active+clean; 3.8 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 6.8 KiB/s rd, 18 KiB/s wr, 691 op/s; 1214/184977 objects misplaced (0.656%); 5.3 MiB/s, 85 objects/s recovering 2023-09-23T20:43:51.895 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:51 smithi018 ceph-mon[184144]: pgmap v1237: 119 pgs: 1 remapped+peering, 17 peering, 1 active+remapped+backfilling, 100 active+clean; 3.8 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 6.8 KiB/s rd, 18 KiB/s wr, 691 op/s; 1214/184977 objects misplaced (0.656%); 5.3 MiB/s, 85 objects/s recovering 2023-09-23T20:43:51.929 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:43:51.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:51 smithi149 ceph-mon[170394]: pgmap v1237: 119 pgs: 1 remapped+peering, 17 peering, 1 active+remapped+backfilling, 100 active+clean; 3.8 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 6.8 KiB/s rd, 18 KiB/s wr, 691 op/s; 1214/184977 objects misplaced (0.656%); 5.3 MiB/s, 85 objects/s recovering 2023-09-23T20:43:52.035 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:52.384 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:43:52.396 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:52.396 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:43:52.497 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:43:52.574 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:52.607 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T20:43:52.706 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:43:52.715 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:52.716 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:43:52.817 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:43:52.887 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:52 smithi018 ceph-mon[184144]: 1.a scrub starts 2023-09-23T20:43:52.887 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:52 smithi018 ceph-mon[184144]: 1.a scrub ok 2023-09-23T20:43:52.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:52 smithi018 ceph-mon[179647]: 1.a scrub starts 2023-09-23T20:43:52.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:52 smithi018 ceph-mon[179647]: 1.a scrub ok 2023-09-23T20:43:52.906 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:52.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:52 smithi149 ceph-mon[170394]: 1.a scrub starts 2023-09-23T20:43:52.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:52 smithi149 ceph-mon[170394]: 1.a scrub ok 2023-09-23T20:43:52.987 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:43:52.991 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:52.991 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:43:53.092 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:43:53.172 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:53.393 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:43:53.401 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:53.401 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:43:53.502 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:43:53.561 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:53.622 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:43:53.628 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:53.629 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:43:53.729 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:43:53.762 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:53.816 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:43:53.824 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:53.824 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:43:53.825 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:53 smithi149 ceph-mon[170394]: pgmap v1238: 119 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 117 active+clean; 3.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 6.2 KiB/s rd, 17 KiB/s wr, 595 op/s; 3142/182241 objects misplaced (1.724%); 12 MiB/s, 192 objects/s recovering 2023-09-23T20:43:53.825 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:53 smithi149 ceph-mon[170394]: 1.1b deep-scrub starts 2023-09-23T20:43:53.825 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:53 smithi149 ceph-mon[170394]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 8 pgs inactive, 15 pgs peering) 2023-09-23T20:43:53.825 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:53 smithi149 ceph-mon[170394]: Cluster is now healthy 2023-09-23T20:43:53.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:53 smithi018 ceph-mon[179647]: pgmap v1238: 119 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 117 active+clean; 3.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 6.2 KiB/s rd, 17 KiB/s wr, 595 op/s; 3142/182241 objects misplaced (1.724%); 12 MiB/s, 192 objects/s recovering 2023-09-23T20:43:53.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:53 smithi018 ceph-mon[179647]: 1.1b deep-scrub starts 2023-09-23T20:43:53.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:53 smithi018 ceph-mon[179647]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 8 pgs inactive, 15 pgs peering) 2023-09-23T20:43:53.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:53 smithi018 ceph-mon[179647]: Cluster is now healthy 2023-09-23T20:43:53.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:53 smithi018 ceph-mon[184144]: pgmap v1238: 119 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 117 active+clean; 3.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 6.2 KiB/s rd, 17 KiB/s wr, 595 op/s; 3142/182241 objects misplaced (1.724%); 12 MiB/s, 192 objects/s recovering 2023-09-23T20:43:53.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:53 smithi018 ceph-mon[184144]: 1.1b deep-scrub starts 2023-09-23T20:43:53.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:53 smithi018 ceph-mon[184144]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 8 pgs inactive, 15 pgs peering) 2023-09-23T20:43:53.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:53 smithi018 ceph-mon[184144]: Cluster is now healthy 2023-09-23T20:43:53.909 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:43:53.925 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:43:53.961 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:54.023 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:43:54.030 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:54.031 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:43:54.131 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:43:54.180 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:54.243 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:43:54.249 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:54.250 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:43:54.351 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:43:54.399 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:54.416 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:43:54.457 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:43:54.462 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:43:54.464 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:54.464 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:43:54.501 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:43:54.546 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:43:54.565 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:43:54.579 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:43:54.607 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:54.630 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:43:54.672 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:43:54.675 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:43:54.686 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:54.687 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:43:54.704 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:43:54.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:54 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/4091010368' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:43:54.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:54 smithi149 ceph-mon[170394]: 1.18 scrub starts 2023-09-23T20:43:54.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:54 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1518794500' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:43:54.719 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:43:54.720 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:43:54.720 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:43:54.720 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:43:54.720 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:43:54.720 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:43:54.720 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:43:54.720 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:43:54.725 INFO:teuthology.orchestra.run.smithi018.stderr:noscrub is set 2023-09-23T20:43:54.788 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:43:54.881 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:54.960 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:43:54.969 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:54 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/4091010368' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:43:54.969 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:54 smithi018 ceph-mon[184144]: 1.18 scrub starts 2023-09-23T20:43:54.969 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:54 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1518794500' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:43:54.970 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:54 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/4091010368' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:43:54.970 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:54 smithi018 ceph-mon[179647]: 1.18 scrub starts 2023-09-23T20:43:54.970 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:54 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1518794500' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:43:54.970 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:54.971 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:43:55.071 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:43:55.141 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:55.392 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:43:54 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:43:54] "GET /metrics HTTP/1.1" 200 36599 "" "Prometheus/2.43.0" 2023-09-23T20:43:55.418 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:43:55.429 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:55.429 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:43:55.530 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:43:55.576 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:55.658 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:43:55.669 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:55.670 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:43:55.716 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:55 smithi018 ceph-mon[179647]: pgmap v1239: 119 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 117 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 127 B/s wr, 725 op/s; 3142/175641 objects misplaced (1.789%); 5.8 MiB/s, 95 objects/s recovering 2023-09-23T20:43:55.716 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:55 smithi018 ceph-mon[179647]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:43:55.716 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:55 smithi018 ceph-mon[179647]: failed to encode map e490 with expected crc 2023-09-23T20:43:55.716 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:55 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/4091010368' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:43:55.716 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:55 smithi018 ceph-mon[179647]: osdmap e490: 8 total, 8 up, 8 in 2023-09-23T20:43:55.717 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:55 smithi018 ceph-mon[184144]: pgmap v1239: 119 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 117 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 127 B/s wr, 725 op/s; 3142/175641 objects misplaced (1.789%); 5.8 MiB/s, 95 objects/s recovering 2023-09-23T20:43:55.718 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:55 smithi018 ceph-mon[184144]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:43:55.718 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:55 smithi018 ceph-mon[184144]: failed to encode map e490 with expected crc 2023-09-23T20:43:55.718 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:55 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/4091010368' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:43:55.718 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:55 smithi018 ceph-mon[184144]: osdmap e490: 8 total, 8 up, 8 in 2023-09-23T20:43:55.770 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:43:55.810 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:55.893 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:43:55.903 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:55.903 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:55 smithi149 ceph-mon[170394]: pgmap v1239: 119 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 117 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 127 B/s wr, 725 op/s; 3142/175641 objects misplaced (1.789%); 5.8 MiB/s, 95 objects/s recovering 2023-09-23T20:43:55.903 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:55 smithi149 ceph-mon[170394]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:43:55.904 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:55 smithi149 ceph-mon[170394]: failed to encode map e490 with expected crc 2023-09-23T20:43:55.904 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:55 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/4091010368' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:43:55.904 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:55 smithi149 ceph-mon[170394]: osdmap e490: 8 total, 8 up, 8 in 2023-09-23T20:43:55.904 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:43:56.005 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:43:56.063 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:56.138 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:43:56.143 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:56.144 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:43:56.211 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 7, 3, 2, 4, 6, 5] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T20:43:56.211 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd reweight 0 1 2023-09-23T20:43:56.244 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:43:56.317 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:57.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:56 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:43:57.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:56 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:43:57.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:56 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:43:58.059 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:57 smithi018 ceph-mon[184144]: pgmap v1241: 119 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 117 active+clean; 3.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 732 op/s; 3142/172557 objects misplaced (1.821%); 5.4 MiB/s, 86 objects/s recovering 2023-09-23T20:43:58.059 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:57 smithi018 ceph-mon[179647]: pgmap v1241: 119 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 117 active+clean; 3.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 732 op/s; 3142/172557 objects misplaced (1.821%); 5.4 MiB/s, 86 objects/s recovering 2023-09-23T20:43:58.126 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:43:58.140 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:58.140 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:43:58.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:57 smithi149 ceph-mon[170394]: pgmap v1241: 119 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 117 active+clean; 3.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 732 op/s; 3142/172557 objects misplaced (1.821%); 5.4 MiB/s, 86 objects/s recovering 2023-09-23T20:43:58.242 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:43:58.346 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:58.436 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:43:58.451 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:58.451 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:43:58.452 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:43:58.553 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:43:58.610 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:58.666 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:43:58.673 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:58.674 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:43:58.774 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:43:58.895 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:59.060 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:43:59.070 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:59.070 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:43:59.171 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:43:59.208 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2023-09-23T20:43:59.209 INFO:teuthology.orchestra.run.smithi018.stdout: "size": 20, 2023-09-23T20:43:59.209 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 600, 2023-09-23T20:43:59.209 INFO:teuthology.orchestra.run.smithi018.stdout: "ops": [ 2023-09-23T20:43:59.209 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.209 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:156873 7.4 7:2413b99c:::benchmark_data_smithi018_308601_object62321:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e352)", 2023-09-23T20:43:59.209 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:34:58.961681+0000", 2023-09-23T20:43:59.209 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 540.23955004300001, 2023-09-23T20:43:59.209 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2142744990000001, 2023-09-23T20:43:59.209 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:43:59.209 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:43:59.209 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:43:59.210 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:43:59.210 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:43:59.210 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 156873 2023-09-23T20:43:59.210 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.210 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:43:59.210 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.210 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:43:59.210 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:58.961681+0000", 2023-09-23T20:43:59.210 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:43:59.210 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.210 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.211 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:43:59.211 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:58.961681+0000", 2023-09-23T20:43:59.211 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:43:59.211 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.211 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.211 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:43:59.211 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:58.961679+0000", 2023-09-23T20:43:59.211 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:43:59.211 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.211 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.211 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:43:59.211 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:58.961685+0000", 2023-09-23T20:43:59.212 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.7289999999999997e-06 2023-09-23T20:43:59.212 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.212 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.212 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:43:59.212 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:58.961698+0000", 2023-09-23T20:43:59.212 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3456e-05 2023-09-23T20:43:59.212 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.212 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.212 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:43:59.212 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:58.961707+0000", 2023-09-23T20:43:59.212 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.371e-06 2023-09-23T20:43:59.212 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.213 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.213 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:43:59.213 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:58.961766+0000", 2023-09-23T20:43:59.213 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.8736000000000001e-05 2023-09-23T20:43:59.213 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.213 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.213 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:43:59.213 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:58.961919+0000", 2023-09-23T20:43:59.213 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000153473 2023-09-23T20:43:59.213 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.213 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.213 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,6", 2023-09-23T20:43:59.213 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:58.962066+0000", 2023-09-23T20:43:59.214 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00014647799999999999 2023-09-23T20:43:59.214 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.214 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.214 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:43:59.214 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:58.982292+0000", 2023-09-23T20:43:59.214 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.020226839 2023-09-23T20:43:59.214 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.214 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.214 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:43:59.214 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:58.982307+0000", 2023-09-23T20:43:59.214 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.4323e-05 2023-09-23T20:43:59.215 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.215 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.215 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:43:59.215 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.150194+0000", 2023-09-23T20:43:59.215 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.167887355 2023-09-23T20:43:59.215 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.215 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.215 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for missing object", 2023-09-23T20:43:59.215 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.150339+0000", 2023-09-23T20:43:59.215 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00014458400000000001 2023-09-23T20:43:59.215 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.215 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.216 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:43:59.216 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.175879+0000", 2023-09-23T20:43:59.216 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.025540245999999999 2023-09-23T20:43:59.216 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.216 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.216 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:43:59.216 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.175955+0000", 2023-09-23T20:43:59.216 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.6230999999999996e-05 2023-09-23T20:43:59.216 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:43:59.216 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:43:59.216 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:43:59.216 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.217 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.217 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:156874 7.4 7:2210d725:::benchmark_data_smithi018_308601_object62322:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e352)", 2023-09-23T20:43:59.217 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:34:58.965887+0000", 2023-09-23T20:43:59.217 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 540.23534407700004, 2023-09-23T20:43:59.217 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.22316603, 2023-09-23T20:43:59.217 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:43:59.217 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:43:59.217 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:43:59.217 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:43:59.217 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:43:59.217 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 156874 2023-09-23T20:43:59.217 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.218 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:43:59.218 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.218 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:43:59.218 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:58.965887+0000", 2023-09-23T20:43:59.218 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:43:59.218 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.218 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.218 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:43:59.218 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:58.965887+0000", 2023-09-23T20:43:59.218 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:43:59.218 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.218 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.219 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:43:59.219 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:58.965885+0000", 2023-09-23T20:43:59.219 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:43:59.219 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.219 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.219 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:43:59.219 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:58.965895+0000", 2023-09-23T20:43:59.219 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.9150000000000008e-06 2023-09-23T20:43:59.219 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.219 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.219 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:43:59.219 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:58.965908+0000", 2023-09-23T20:43:59.220 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3349e-05 2023-09-23T20:43:59.220 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.220 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.220 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:43:59.220 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:58.965918+0000", 2023-09-23T20:43:59.220 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.6139999999999998e-06 2023-09-23T20:43:59.220 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.220 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.220 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:43:59.220 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:58.966970+0000", 2023-09-23T20:43:59.220 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.001052545 2023-09-23T20:43:59.220 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.221 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.221 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:43:59.221 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:58.967159+0000", 2023-09-23T20:43:59.221 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000189087 2023-09-23T20:43:59.221 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.221 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.221 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,6", 2023-09-23T20:43:59.221 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:58.967370+0000", 2023-09-23T20:43:59.221 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000210795 2023-09-23T20:43:59.221 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.221 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.221 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:43:59.222 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:58.982341+0000", 2023-09-23T20:43:59.222 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.014970255 2023-09-23T20:43:59.222 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.222 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.222 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:43:59.222 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:58.982350+0000", 2023-09-23T20:43:59.222 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.6029999999999994e-06 2023-09-23T20:43:59.222 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.222 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.222 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:43:59.222 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.151266+0000", 2023-09-23T20:43:59.222 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1689161379999999 2023-09-23T20:43:59.223 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.223 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.223 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for missing object", 2023-09-23T20:43:59.223 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.151368+0000", 2023-09-23T20:43:59.223 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00010191699999999999 2023-09-23T20:43:59.223 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.223 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.223 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:43:59.223 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.188980+0000", 2023-09-23T20:43:59.223 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.037611376000000002 2023-09-23T20:43:59.223 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.223 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.224 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:43:59.224 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.189053+0000", 2023-09-23T20:43:59.224 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.3058000000000004e-05 2023-09-23T20:43:59.224 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:43:59.224 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:43:59.224 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:43:59.224 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.224 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.224 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:156880 7.9 7:97f9ea4b:::benchmark_data_smithi018_308601_object62328:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e353)", 2023-09-23T20:43:59.224 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:34:58.989864+0000", 2023-09-23T20:43:59.224 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 540.21136667200005, 2023-09-23T20:43:59.224 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.1443719099999998, 2023-09-23T20:43:59.225 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:43:59.225 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:43:59.225 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:43:59.225 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:43:59.225 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:43:59.225 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 156880 2023-09-23T20:43:59.225 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.225 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:43:59.225 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.225 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:43:59.225 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:58.989864+0000", 2023-09-23T20:43:59.225 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:43:59.226 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.226 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.226 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:43:59.226 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:58.989864+0000", 2023-09-23T20:43:59.226 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:43:59.226 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.226 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.226 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:43:59.226 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:58.989862+0000", 2023-09-23T20:43:59.226 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:43:59.226 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.227 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.227 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:43:59.227 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:58.989870+0000", 2023-09-23T20:43:59.227 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.7959999999999996e-06 2023-09-23T20:43:59.227 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.227 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.227 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:43:59.227 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:58.989888+0000", 2023-09-23T20:43:59.227 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.8008000000000001e-05 2023-09-23T20:43:59.227 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.227 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.227 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:43:59.228 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:58.989898+0000", 2023-09-23T20:43:59.228 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.4930000000000004e-06 2023-09-23T20:43:59.228 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.228 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.228 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:43:59.228 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:58.994377+0000", 2023-09-23T20:43:59.228 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0044797750000000001 2023-09-23T20:43:59.228 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.228 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.228 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:43:59.228 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:58.994389+0000", 2023-09-23T20:43:59.228 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1224e-05 2023-09-23T20:43:59.229 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.229 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.229 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:43:59.229 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.969523+0000", 2023-09-23T20:43:59.229 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.97513417700000005 2023-09-23T20:43:59.229 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.229 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.229 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:43:59.229 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.969537+0000", 2023-09-23T20:43:59.229 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.4653e-05 2023-09-23T20:43:59.229 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.229 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.230 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:43:59.230 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.107362+0000", 2023-09-23T20:43:59.230 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.13782403 2023-09-23T20:43:59.230 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.230 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.230 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:43:59.230 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.107518+0000", 2023-09-23T20:43:59.230 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00015667200000000001 2023-09-23T20:43:59.230 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.230 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.230 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,4,7", 2023-09-23T20:43:59.231 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.107667+0000", 2023-09-23T20:43:59.231 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00014836800000000001 2023-09-23T20:43:59.231 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.231 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.231 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:43:59.231 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.111586+0000", 2023-09-23T20:43:59.231 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0039193520000000001 2023-09-23T20:43:59.231 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.231 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.231 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:43:59.231 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.111613+0000", 2023-09-23T20:43:59.232 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.6597999999999998e-05 2023-09-23T20:43:59.232 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.232 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.232 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:43:59.232 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.115895+0000", 2023-09-23T20:43:59.232 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.004282526 2023-09-23T20:43:59.232 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.232 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.232 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:43:59.232 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.134027+0000", 2023-09-23T20:43:59.232 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.018131807 2023-09-23T20:43:59.232 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.233 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.233 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:43:59.233 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.134094+0000", 2023-09-23T20:43:59.233 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.7551000000000004e-05 2023-09-23T20:43:59.233 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.233 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.233 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:43:59.233 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.134236+0000", 2023-09-23T20:43:59.233 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00014152500000000001 2023-09-23T20:43:59.233 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:43:59.233 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:43:59.234 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:43:59.234 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.234 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.234 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:156884 7.9 7:90c28959:::benchmark_data_smithi018_308601_object62332:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e353)", 2023-09-23T20:43:59.234 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:34:58.995119+0000", 2023-09-23T20:43:59.234 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 540.20611157999997, 2023-09-23T20:43:59.234 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.146166928, 2023-09-23T20:43:59.234 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:43:59.234 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:43:59.234 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:43:59.234 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:43:59.234 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:43:59.235 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 156884 2023-09-23T20:43:59.235 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.235 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:43:59.235 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.235 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:43:59.235 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:58.995119+0000", 2023-09-23T20:43:59.235 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:43:59.235 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.235 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.235 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:43:59.235 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:58.995119+0000", 2023-09-23T20:43:59.236 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:43:59.236 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.236 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.236 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:43:59.236 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:58.995118+0000", 2023-09-23T20:43:59.236 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.999999 2023-09-23T20:43:59.236 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.236 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.236 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:43:59.236 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:58.995124+0000", 2023-09-23T20:43:59.236 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.4960000000000004e-06 2023-09-23T20:43:59.236 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.237 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.237 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:43:59.237 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:58.995138+0000", 2023-09-23T20:43:59.237 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.4413000000000001e-05 2023-09-23T20:43:59.237 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.237 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.237 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:43:59.237 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:58.995146+0000", 2023-09-23T20:43:59.237 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.0150000000000003e-06 2023-09-23T20:43:59.237 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.237 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.237 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:43:59.238 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:58.995225+0000", 2023-09-23T20:43:59.238 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.9101e-05 2023-09-23T20:43:59.238 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.238 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.238 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:43:59.238 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:58.995235+0000", 2023-09-23T20:43:59.238 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.9699999999999994e-06 2023-09-23T20:43:59.238 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.238 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.238 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:43:59.238 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.969600+0000", 2023-09-23T20:43:59.238 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.97436495300000003 2023-09-23T20:43:59.239 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.239 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.239 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:43:59.239 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.969609+0000", 2023-09-23T20:43:59.239 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.5699999999999993e-06 2023-09-23T20:43:59.239 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.239 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.239 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:43:59.239 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.111637+0000", 2023-09-23T20:43:59.239 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.142028147 2023-09-23T20:43:59.239 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.239 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.240 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:43:59.240 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.111784+0000", 2023-09-23T20:43:59.240 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000146883 2023-09-23T20:43:59.240 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.240 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.240 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,4,7", 2023-09-23T20:43:59.240 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.111947+0000", 2023-09-23T20:43:59.240 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00016295400000000001 2023-09-23T20:43:59.240 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.240 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.240 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:43:59.240 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.119811+0000", 2023-09-23T20:43:59.241 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0078639850000000004 2023-09-23T20:43:59.241 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.241 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.241 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:43:59.241 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.119872+0000", 2023-09-23T20:43:59.241 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.1822000000000005e-05 2023-09-23T20:43:59.241 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.241 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.241 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:43:59.241 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.119937+0000", 2023-09-23T20:43:59.241 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.5036000000000005e-05 2023-09-23T20:43:59.242 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.242 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.242 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:43:59.242 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.141091+0000", 2023-09-23T20:43:59.242 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.021153885000000001 2023-09-23T20:43:59.242 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.242 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.242 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:43:59.242 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.141158+0000", 2023-09-23T20:43:59.242 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.6433000000000006e-05 2023-09-23T20:43:59.242 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.242 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.243 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:43:59.243 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.141286+0000", 2023-09-23T20:43:59.243 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00012822300000000001 2023-09-23T20:43:59.243 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:43:59.243 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:43:59.243 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:43:59.243 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.243 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.243 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:156891 7.9 7:900a49cc:::benchmark_data_smithi018_308601_object62339:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e353)", 2023-09-23T20:43:59.243 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:34:59.007075+0000", 2023-09-23T20:43:59.243 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 540.19415566500004, 2023-09-23T20:43:59.243 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.1390987259999998, 2023-09-23T20:43:59.244 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:43:59.244 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:43:59.244 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:43:59.244 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:43:59.244 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:43:59.244 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 156891 2023-09-23T20:43:59.244 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.244 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:43:59.244 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.244 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:43:59.244 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.007075+0000", 2023-09-23T20:43:59.244 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:43:59.245 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.245 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.245 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:43:59.245 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.007075+0000", 2023-09-23T20:43:59.245 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:43:59.245 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.245 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.245 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:43:59.245 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.007073+0000", 2023-09-23T20:43:59.245 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:43:59.245 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.246 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.246 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:43:59.246 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.007079+0000", 2023-09-23T20:43:59.246 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.9619999999999998e-06 2023-09-23T20:43:59.246 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.246 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.246 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:43:59.246 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.007090+0000", 2023-09-23T20:43:59.246 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0694e-05 2023-09-23T20:43:59.246 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.246 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.246 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:43:59.247 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.007097+0000", 2023-09-23T20:43:59.247 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.6729999999999992e-06 2023-09-23T20:43:59.247 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.247 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.247 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:43:59.247 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.012873+0000", 2023-09-23T20:43:59.247 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0057759999999999999 2023-09-23T20:43:59.247 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.247 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.247 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:43:59.247 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.012890+0000", 2023-09-23T20:43:59.247 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.6566999999999999e-05 2023-09-23T20:43:59.248 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.248 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.248 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:43:59.248 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.969636+0000", 2023-09-23T20:43:59.248 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.95674641400000004 2023-09-23T20:43:59.248 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.248 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.248 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:43:59.248 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.969644+0000", 2023-09-23T20:43:59.248 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.2989999999999999e-06 2023-09-23T20:43:59.248 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.248 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.249 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:43:59.249 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.115914+0000", 2023-09-23T20:43:59.249 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1462701470000001 2023-09-23T20:43:59.249 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.249 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.249 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:43:59.249 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.116062+0000", 2023-09-23T20:43:59.249 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00014790800000000001 2023-09-23T20:43:59.249 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.249 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.249 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,4,7", 2023-09-23T20:43:59.249 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.116218+0000", 2023-09-23T20:43:59.250 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000156725 2023-09-23T20:43:59.250 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.250 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.250 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:43:59.250 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.119974+0000", 2023-09-23T20:43:59.250 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0037559519999999999 2023-09-23T20:43:59.250 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.250 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.250 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:43:59.250 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.120032+0000", 2023-09-23T20:43:59.250 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.7485999999999998e-05 2023-09-23T20:43:59.251 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.251 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.251 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:43:59.251 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.126273+0000", 2023-09-23T20:43:59.251 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0062413069999999998 2023-09-23T20:43:59.251 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.251 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.251 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:43:59.251 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.146012+0000", 2023-09-23T20:43:59.251 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.019739014999999999 2023-09-23T20:43:59.251 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.251 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.252 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:43:59.252 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.146065+0000", 2023-09-23T20:43:59.252 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.2939000000000003e-05 2023-09-23T20:43:59.252 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.252 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.252 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:43:59.252 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.146174+0000", 2023-09-23T20:43:59.252 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000108713 2023-09-23T20:43:59.252 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:43:59.252 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:43:59.252 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:43:59.252 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.253 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.253 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:156904 7.4 7:256a86e5:::benchmark_data_smithi018_308601_object62352:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e353)", 2023-09-23T20:43:59.253 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:34:59.030138+0000", 2023-09-23T20:43:59.253 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 540.17109221099997, 2023-09-23T20:43:59.253 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1549561939999999, 2023-09-23T20:43:59.253 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:43:59.253 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:43:59.253 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:43:59.253 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:43:59.253 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:43:59.253 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 156904 2023-09-23T20:43:59.254 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.254 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:43:59.254 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.254 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:43:59.254 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.030138+0000", 2023-09-23T20:43:59.254 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:43:59.254 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.254 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.254 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:43:59.254 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.030138+0000", 2023-09-23T20:43:59.254 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:43:59.255 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.255 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.255 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:43:59.255 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.030138+0000", 2023-09-23T20:43:59.255 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:43:59.255 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.255 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.255 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:43:59.255 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.030141+0000", 2023-09-23T20:43:59.255 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.8260000000000001e-06 2023-09-23T20:43:59.255 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.256 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.256 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:43:59.256 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.030147+0000", 2023-09-23T20:43:59.256 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.3749999999999999e-06 2023-09-23T20:43:59.256 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.256 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.256 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:43:59.256 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.030152+0000", 2023-09-23T20:43:59.256 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.0340000000000004e-06 2023-09-23T20:43:59.256 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.256 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.257 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:43:59.257 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.078211+0000", 2023-09-23T20:43:59.257 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.048058233999999998 2023-09-23T20:43:59.257 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.257 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.257 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:43:59.257 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.078229+0000", 2023-09-23T20:43:59.257 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.8342000000000002e-05 2023-09-23T20:43:59.257 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.257 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.257 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:43:59.258 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.151421+0000", 2023-09-23T20:43:59.258 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0731922810000001 2023-09-23T20:43:59.258 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.258 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.258 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:43:59.258 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.151535+0000", 2023-09-23T20:43:59.258 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000113695 2023-09-23T20:43:59.258 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.258 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.258 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,4", 2023-09-23T20:43:59.258 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.151626+0000", 2023-09-23T20:43:59.259 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.0983999999999995e-05 2023-09-23T20:43:59.259 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.259 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.259 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:43:59.259 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.158726+0000", 2023-09-23T20:43:59.259 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0070999100000000001 2023-09-23T20:43:59.259 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.259 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.259 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:43:59.259 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.159871+0000", 2023-09-23T20:43:59.260 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.001145177 2023-09-23T20:43:59.260 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.260 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.260 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:43:59.260 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.183778+0000", 2023-09-23T20:43:59.260 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.023906849000000001 2023-09-23T20:43:59.260 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.260 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.260 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:43:59.260 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.183826+0000", 2023-09-23T20:43:59.260 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.8149999999999998e-05 2023-09-23T20:43:59.261 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.261 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.261 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:43:59.261 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.185095+0000", 2023-09-23T20:43:59.261 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0012687359999999999 2023-09-23T20:43:59.261 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:43:59.261 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:43:59.261 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:43:59.261 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.261 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.261 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:156909 7.9 7:9eb097ac:::benchmark_data_smithi018_308601_object62357:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e353)", 2023-09-23T20:43:59.261 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:34:59.038642+0000", 2023-09-23T20:43:59.262 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 540.16258823199996, 2023-09-23T20:43:59.262 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.1077584620000001, 2023-09-23T20:43:59.262 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:43:59.262 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:43:59.262 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:43:59.262 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:43:59.262 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:43:59.262 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 156909 2023-09-23T20:43:59.262 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.262 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:43:59.262 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.263 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:43:59.263 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.038642+0000", 2023-09-23T20:43:59.263 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:43:59.263 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.263 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.263 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:43:59.263 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.038642+0000", 2023-09-23T20:43:59.263 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:43:59.263 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.263 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.263 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:43:59.263 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.038641+0000", 2023-09-23T20:43:59.264 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:43:59.264 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.264 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.264 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:43:59.264 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.038649+0000", 2023-09-23T20:43:59.264 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.7000000000000008e-06 2023-09-23T20:43:59.264 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.264 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.264 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:43:59.264 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.038665+0000", 2023-09-23T20:43:59.264 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.6545000000000002e-05 2023-09-23T20:43:59.264 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.265 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.265 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:43:59.265 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.038674+0000", 2023-09-23T20:43:59.265 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.1440000000000003e-06 2023-09-23T20:43:59.265 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.265 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.265 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:43:59.265 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.038726+0000", 2023-09-23T20:43:59.265 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.2135000000000001e-05 2023-09-23T20:43:59.265 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.265 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.265 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:43:59.266 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.038737+0000", 2023-09-23T20:43:59.266 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1823e-05 2023-09-23T20:43:59.266 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.266 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.266 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:43:59.266 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.969669+0000", 2023-09-23T20:43:59.266 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.93093119400000002 2023-09-23T20:43:59.266 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.266 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.266 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:43:59.266 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.969676+0000", 2023-09-23T20:43:59.267 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.9480000000000002e-06 2023-09-23T20:43:59.267 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.267 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.267 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:43:59.267 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.120069+0000", 2023-09-23T20:43:59.267 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1503932180000001 2023-09-23T20:43:59.267 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.267 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.267 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:43:59.267 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.120165+0000", 2023-09-23T20:43:59.267 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.5729000000000005e-05 2023-09-23T20:43:59.268 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.268 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.268 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,4,7", 2023-09-23T20:43:59.268 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.120247+0000", 2023-09-23T20:43:59.268 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.2736000000000002e-05 2023-09-23T20:43:59.268 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.268 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.268 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:43:59.268 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.126258+0000", 2023-09-23T20:43:59.268 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0060109810000000003 2023-09-23T20:43:59.268 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.269 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.269 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:43:59.269 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.126315+0000", 2023-09-23T20:43:59.269 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.6663999999999998e-05 2023-09-23T20:43:59.269 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.269 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.269 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:43:59.269 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.126326+0000", 2023-09-23T20:43:59.269 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0812000000000001e-05 2023-09-23T20:43:59.269 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.270 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.270 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:43:59.270 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.146229+0000", 2023-09-23T20:43:59.270 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.019902835000000001 2023-09-23T20:43:59.270 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.270 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.270 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:43:59.270 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.146283+0000", 2023-09-23T20:43:59.270 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.3922000000000003e-05 2023-09-23T20:43:59.270 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.270 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.271 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:43:59.271 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.146401+0000", 2023-09-23T20:43:59.271 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000118396 2023-09-23T20:43:59.271 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:43:59.271 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:43:59.271 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:43:59.271 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.271 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.271 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:156910 7.4 7:20d2ed44:::benchmark_data_smithi018_308601_object62358:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e353)", 2023-09-23T20:43:59.271 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:34:59.038685+0000", 2023-09-23T20:43:59.271 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 540.16254543800005, 2023-09-23T20:43:59.271 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1477232370000001, 2023-09-23T20:43:59.272 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:43:59.272 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:43:59.272 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:43:59.272 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:43:59.272 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:43:59.272 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 156910 2023-09-23T20:43:59.272 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.272 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:43:59.272 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.272 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:43:59.272 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.038685+0000", 2023-09-23T20:43:59.272 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:43:59.273 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.273 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.273 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:43:59.273 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.038685+0000", 2023-09-23T20:43:59.273 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:43:59.273 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.273 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.273 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:43:59.273 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.038685+0000", 2023-09-23T20:43:59.273 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:43:59.273 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.273 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.274 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:43:59.274 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.038687+0000", 2023-09-23T20:43:59.274 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.5620000000000002e-06 2023-09-23T20:43:59.274 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.274 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.274 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:43:59.274 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.038692+0000", 2023-09-23T20:43:59.274 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.5560000000000001e-06 2023-09-23T20:43:59.274 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.274 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.274 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:43:59.275 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.038696+0000", 2023-09-23T20:43:59.275 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.6040000000000001e-06 2023-09-23T20:43:59.275 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.275 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.275 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:43:59.275 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.078252+0000", 2023-09-23T20:43:59.275 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.039556674999999999 2023-09-23T20:43:59.275 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.275 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.275 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:43:59.275 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.078261+0000", 2023-09-23T20:43:59.275 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.3999999999999992e-06 2023-09-23T20:43:59.276 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.276 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.276 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:43:59.276 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.154318+0000", 2023-09-23T20:43:59.276 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.076057134 2023-09-23T20:43:59.276 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.276 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.276 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:43:59.276 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.154417+0000", 2023-09-23T20:43:59.276 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.9007999999999998e-05 2023-09-23T20:43:59.276 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.277 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.277 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,4", 2023-09-23T20:43:59.277 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.154500+0000", 2023-09-23T20:43:59.277 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.3410000000000003e-05 2023-09-23T20:43:59.277 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.277 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.277 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:43:59.277 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.158731+0000", 2023-09-23T20:43:59.277 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0042308340000000002 2023-09-23T20:43:59.277 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.277 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.277 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:43:59.277 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.159915+0000", 2023-09-23T20:43:59.278 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0011838739999999999 2023-09-23T20:43:59.278 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.278 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.278 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:43:59.278 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.185179+0000", 2023-09-23T20:43:59.278 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.025264134000000001 2023-09-23T20:43:59.278 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.278 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.278 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:43:59.278 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.185248+0000", 2023-09-23T20:43:59.278 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.8959000000000002e-05 2023-09-23T20:43:59.279 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.279 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.279 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:43:59.279 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.186408+0000", 2023-09-23T20:43:59.279 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.001160424 2023-09-23T20:43:59.279 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:43:59.279 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:43:59.279 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:43:59.279 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.279 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.279 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:156918 7.9 7:969f6c77:::benchmark_data_smithi018_308601_object62366:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e353)", 2023-09-23T20:43:59.279 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:34:59.046909+0000", 2023-09-23T20:43:59.280 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 540.15432201600004, 2023-09-23T20:43:59.280 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.0997079030000001, 2023-09-23T20:43:59.280 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:43:59.280 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:43:59.280 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:43:59.280 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:43:59.280 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:43:59.280 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 156918 2023-09-23T20:43:59.280 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.280 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:43:59.280 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.280 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:43:59.281 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.046909+0000", 2023-09-23T20:43:59.281 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:43:59.281 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.281 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.281 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:43:59.281 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.046909+0000", 2023-09-23T20:43:59.281 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:43:59.281 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.281 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.281 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:43:59.281 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.046907+0000", 2023-09-23T20:43:59.281 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:43:59.282 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.282 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.282 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:43:59.282 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.046914+0000", 2023-09-23T20:43:59.282 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.657e-06 2023-09-23T20:43:59.282 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.282 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.282 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:43:59.282 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.046929+0000", 2023-09-23T20:43:59.282 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.4713e-05 2023-09-23T20:43:59.282 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.282 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.283 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:43:59.283 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.046936+0000", 2023-09-23T20:43:59.283 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.5900000000000002e-06 2023-09-23T20:43:59.283 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.283 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.283 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:43:59.283 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.046998+0000", 2023-09-23T20:43:59.283 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.1655000000000005e-05 2023-09-23T20:43:59.283 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.283 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.283 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:43:59.284 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.047008+0000", 2023-09-23T20:43:59.284 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0223e-05 2023-09-23T20:43:59.284 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.284 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.284 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:43:59.284 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.969700+0000", 2023-09-23T20:43:59.284 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.92269210300000004 2023-09-23T20:43:59.284 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.284 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.284 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:43:59.284 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.969707+0000", 2023-09-23T20:43:59.285 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.861e-06 2023-09-23T20:43:59.285 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.285 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.285 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:43:59.285 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.123042+0000", 2023-09-23T20:43:59.285 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.153334992 2023-09-23T20:43:59.285 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.285 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.285 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:43:59.285 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.123185+0000", 2023-09-23T20:43:59.285 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00014291400000000001 2023-09-23T20:43:59.285 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.286 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.286 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,4,7", 2023-09-23T20:43:59.286 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.123329+0000", 2023-09-23T20:43:59.286 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000144344 2023-09-23T20:43:59.286 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.286 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.286 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:43:59.286 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.126389+0000", 2023-09-23T20:43:59.286 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0030597329999999998 2023-09-23T20:43:59.286 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.286 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.287 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:43:59.287 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.130226+0000", 2023-09-23T20:43:59.287 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0038367789999999998 2023-09-23T20:43:59.287 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.287 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.287 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:43:59.287 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.133972+0000", 2023-09-23T20:43:59.287 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0037460340000000001 2023-09-23T20:43:59.287 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.287 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.287 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:43:59.287 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.146449+0000", 2023-09-23T20:43:59.288 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.012477626 2023-09-23T20:43:59.288 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.288 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.288 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:43:59.288 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.146505+0000", 2023-09-23T20:43:59.288 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.5926e-05 2023-09-23T20:43:59.288 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.288 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.288 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:43:59.288 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.146617+0000", 2023-09-23T20:43:59.288 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00011121899999999999 2023-09-23T20:43:59.289 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:43:59.289 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:43:59.289 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:43:59.289 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.289 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.289 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:156927 7.4 7:2c3ea5a8:::benchmark_data_smithi018_308601_object62375:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e353)", 2023-09-23T20:43:59.289 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:34:59.064897+0000", 2023-09-23T20:43:59.289 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 540.13633402899995, 2023-09-23T20:43:59.289 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.12259446, 2023-09-23T20:43:59.289 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:43:59.289 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:43:59.289 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:43:59.290 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:43:59.290 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:43:59.290 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 156927 2023-09-23T20:43:59.290 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.290 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:43:59.290 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.290 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:43:59.290 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.064897+0000", 2023-09-23T20:43:59.290 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:43:59.290 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.290 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.291 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:43:59.291 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.064897+0000", 2023-09-23T20:43:59.291 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:43:59.291 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.291 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.291 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:43:59.291 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.064895+0000", 2023-09-23T20:43:59.291 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:43:59.291 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.291 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.291 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:43:59.292 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.064903+0000", 2023-09-23T20:43:59.292 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.2300000000000008e-06 2023-09-23T20:43:59.292 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.292 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.292 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:43:59.292 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.064916+0000", 2023-09-23T20:43:59.292 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3355e-05 2023-09-23T20:43:59.292 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.292 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.292 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:43:59.292 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.064925+0000", 2023-09-23T20:43:59.292 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.8799999999999997e-06 2023-09-23T20:43:59.293 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.293 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.293 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:43:59.293 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.078285+0000", 2023-09-23T20:43:59.293 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.013359861000000001 2023-09-23T20:43:59.293 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.293 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.293 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:43:59.293 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.078293+0000", 2023-09-23T20:43:59.293 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.1489999999999994e-06 2023-09-23T20:43:59.293 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.294 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.294 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:43:59.294 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.156712+0000", 2023-09-23T20:43:59.294 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0784186039999999 2023-09-23T20:43:59.294 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.294 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.294 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:43:59.294 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.156823+0000", 2023-09-23T20:43:59.294 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000111264 2023-09-23T20:43:59.294 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.294 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.295 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,4", 2023-09-23T20:43:59.295 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.156902+0000", 2023-09-23T20:43:59.295 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.9296999999999997e-05 2023-09-23T20:43:59.295 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.295 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.295 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:43:59.295 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.159936+0000", 2023-09-23T20:43:59.295 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.003034003 2023-09-23T20:43:59.295 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.295 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.295 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:43:59.296 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.160017+0000", 2023-09-23T20:43:59.296 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.0453999999999999e-05 2023-09-23T20:43:59.296 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.296 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.296 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:43:59.296 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.186490+0000", 2023-09-23T20:43:59.296 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.026473625000000001 2023-09-23T20:43:59.296 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.296 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.296 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:43:59.296 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.186551+0000", 2023-09-23T20:43:59.296 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.0473000000000002e-05 2023-09-23T20:43:59.297 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.297 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.297 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:43:59.297 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.187491+0000", 2023-09-23T20:43:59.297 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000940229 2023-09-23T20:43:59.297 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:43:59.297 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:43:59.297 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:43:59.297 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.297 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.297 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:156946 7.9 7:9c9b215c:::benchmark_data_smithi018_308601_object62394:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e353)", 2023-09-23T20:43:59.297 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:34:59.100567+0000", 2023-09-23T20:43:59.298 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 540.10066375999997, 2023-09-23T20:43:59.298 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.0462718, 2023-09-23T20:43:59.298 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:43:59.298 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:43:59.298 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:43:59.298 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:43:59.298 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:43:59.298 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 156946 2023-09-23T20:43:59.298 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.298 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:43:59.298 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.299 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:43:59.299 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.100567+0000", 2023-09-23T20:43:59.299 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:43:59.299 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.299 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.299 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:43:59.299 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.100567+0000", 2023-09-23T20:43:59.299 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:43:59.299 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.299 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.299 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:43:59.299 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.100566+0000", 2023-09-23T20:43:59.300 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:43:59.300 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.300 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.300 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:43:59.300 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.100571+0000", 2023-09-23T20:43:59.300 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.3759999999999999e-06 2023-09-23T20:43:59.300 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.300 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.300 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:43:59.300 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.100584+0000", 2023-09-23T20:43:59.300 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2679e-05 2023-09-23T20:43:59.300 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.300 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.301 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:43:59.301 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.100590+0000", 2023-09-23T20:43:59.301 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.2929999999999997e-06 2023-09-23T20:43:59.301 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.301 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.301 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:43:59.301 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.100652+0000", 2023-09-23T20:43:59.301 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.2476000000000003e-05 2023-09-23T20:43:59.301 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.301 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.301 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:43:59.301 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.100667+0000", 2023-09-23T20:43:59.302 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.4270999999999999e-05 2023-09-23T20:43:59.302 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.302 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.302 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:43:59.302 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.971028+0000", 2023-09-23T20:43:59.302 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.87036148599999996 2023-09-23T20:43:59.302 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.302 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.302 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:43:59.302 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:34:59.971042+0000", 2023-09-23T20:43:59.302 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3366999999999999e-05 2023-09-23T20:43:59.303 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.303 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.303 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:43:59.303 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.126433+0000", 2023-09-23T20:43:59.303 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1553918599999999 2023-09-23T20:43:59.303 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.303 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.303 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:43:59.303 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.126572+0000", 2023-09-23T20:43:59.303 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00013825299999999999 2023-09-23T20:43:59.303 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.304 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.304 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,4,7", 2023-09-23T20:43:59.304 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.126718+0000", 2023-09-23T20:43:59.304 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00014583900000000001 2023-09-23T20:43:59.304 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.304 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.304 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:43:59.304 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.134301+0000", 2023-09-23T20:43:59.304 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0075839439999999996 2023-09-23T20:43:59.304 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.304 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.304 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:43:59.305 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.134357+0000", 2023-09-23T20:43:59.305 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.5522000000000001e-05 2023-09-23T20:43:59.305 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.305 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.305 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:43:59.305 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.134367+0000", 2023-09-23T20:43:59.305 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0118e-05 2023-09-23T20:43:59.305 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.305 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.305 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:43:59.305 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.146664+0000", 2023-09-23T20:43:59.305 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.012296848000000001 2023-09-23T20:43:59.306 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.306 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.306 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:43:59.306 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.146717+0000", 2023-09-23T20:43:59.306 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.2800999999999998e-05 2023-09-23T20:43:59.306 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.306 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.306 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:43:59.306 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.146839+0000", 2023-09-23T20:43:59.306 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000121931 2023-09-23T20:43:59.306 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:43:59.307 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:43:59.307 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:43:59.307 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.307 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.307 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:156966 7.9 7:9ee3fd12:::benchmark_data_smithi018_308601_object62414:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e354)", 2023-09-23T20:43:59.307 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:35:00.195629+0000", 2023-09-23T20:43:59.307 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 539.00560206800003, 2023-09-23T20:43:59.307 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.95484696700000005, 2023-09-23T20:43:59.307 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:43:59.307 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:43:59.307 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:43:59.308 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:43:59.308 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:43:59.308 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 156966 2023-09-23T20:43:59.308 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.308 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:43:59.308 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.308 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:43:59.308 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.195629+0000", 2023-09-23T20:43:59.308 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:43:59.308 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.308 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.308 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:43:59.309 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.195629+0000", 2023-09-23T20:43:59.309 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:43:59.309 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.309 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.309 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:43:59.309 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.195627+0000", 2023-09-23T20:43:59.309 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:43:59.309 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.309 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.309 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:43:59.309 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.195635+0000", 2023-09-23T20:43:59.309 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.8909999999999995e-06 2023-09-23T20:43:59.310 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.310 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.310 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:43:59.310 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.195652+0000", 2023-09-23T20:43:59.310 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.6968000000000001e-05 2023-09-23T20:43:59.310 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.310 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.310 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:43:59.310 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.195659+0000", 2023-09-23T20:43:59.310 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.8019999999999992e-06 2023-09-23T20:43:59.310 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.311 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.311 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:43:59.311 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.197096+0000", 2023-09-23T20:43:59.311 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.001436751 2023-09-23T20:43:59.311 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.311 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.311 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:43:59.311 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.197108+0000", 2023-09-23T20:43:59.311 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1445e-05 2023-09-23T20:43:59.311 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.312 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.312 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:43:59.312 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.130266+0000", 2023-09-23T20:43:59.312 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.933158288 2023-09-23T20:43:59.312 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.312 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.312 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:43:59.312 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.130363+0000", 2023-09-23T20:43:59.312 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.7326999999999998e-05 2023-09-23T20:43:59.312 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.312 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.313 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,4,7", 2023-09-23T20:43:59.313 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.130454+0000", 2023-09-23T20:43:59.313 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.0653000000000001e-05 2023-09-23T20:43:59.313 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.313 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.313 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:43:59.313 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.134376+0000", 2023-09-23T20:43:59.313 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0039225140000000002 2023-09-23T20:43:59.313 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.313 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.313 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:43:59.313 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.134421+0000", 2023-09-23T20:43:59.314 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.4378000000000001e-05 2023-09-23T20:43:59.314 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.314 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.314 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:43:59.314 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.134474+0000", 2023-09-23T20:43:59.314 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.3263999999999997e-05 2023-09-23T20:43:59.314 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.314 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.314 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:43:59.314 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.150283+0000", 2023-09-23T20:43:59.314 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.015809519000000001 2023-09-23T20:43:59.314 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.315 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.315 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:43:59.315 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.150341+0000", 2023-09-23T20:43:59.315 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.7182999999999998e-05 2023-09-23T20:43:59.315 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.315 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.315 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:43:59.315 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.150476+0000", 2023-09-23T20:43:59.315 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000134893 2023-09-23T20:43:59.315 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:43:59.315 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:43:59.316 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:43:59.316 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.316 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.316 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:156978 7.9 7:90ac27fe:::benchmark_data_smithi018_308601_object62426:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e354)", 2023-09-23T20:43:59.316 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:35:00.215473+0000", 2023-09-23T20:43:59.316 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 538.98575796099999, 2023-09-23T20:43:59.316 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.93521590300000002, 2023-09-23T20:43:59.316 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:43:59.316 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:43:59.316 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:43:59.316 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:43:59.316 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:43:59.317 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 156978 2023-09-23T20:43:59.317 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.317 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:43:59.317 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.317 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:43:59.317 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.215473+0000", 2023-09-23T20:43:59.317 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:43:59.317 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.317 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.317 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:43:59.317 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.215473+0000", 2023-09-23T20:43:59.317 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:43:59.318 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.318 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.318 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:43:59.318 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.215471+0000", 2023-09-23T20:43:59.318 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:43:59.318 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.318 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.318 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:43:59.318 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.215478+0000", 2023-09-23T20:43:59.318 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.6710000000000002e-06 2023-09-23T20:43:59.318 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.319 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.319 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:43:59.319 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.215494+0000", 2023-09-23T20:43:59.319 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.6480000000000001e-05 2023-09-23T20:43:59.319 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.319 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.319 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:43:59.319 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.215503+0000", 2023-09-23T20:43:59.319 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.3030000000000005e-06 2023-09-23T20:43:59.319 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.319 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.319 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:43:59.320 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.217858+0000", 2023-09-23T20:43:59.320 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0023550530000000002 2023-09-23T20:43:59.320 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.320 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.320 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:43:59.320 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.217873+0000", 2023-09-23T20:43:59.320 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.428e-05 2023-09-23T20:43:59.320 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.320 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.320 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:43:59.320 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.134499+0000", 2023-09-23T20:43:59.321 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.91662635100000001 2023-09-23T20:43:59.321 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.321 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.321 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:43:59.321 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.134590+0000", 2023-09-23T20:43:59.321 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.0661999999999994e-05 2023-09-23T20:43:59.321 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.321 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.321 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,4,7", 2023-09-23T20:43:59.321 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.134659+0000", 2023-09-23T20:43:59.321 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.9627000000000003e-05 2023-09-23T20:43:59.321 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.322 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.322 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:43:59.322 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.140958+0000", 2023-09-23T20:43:59.322 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0062990199999999998 2023-09-23T20:43:59.322 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.322 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.322 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:43:59.322 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.140971+0000", 2023-09-23T20:43:59.322 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2649e-05 2023-09-23T20:43:59.322 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.322 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.322 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:43:59.323 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.141033+0000", 2023-09-23T20:43:59.323 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.1580000000000003e-05 2023-09-23T20:43:59.323 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.323 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.323 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:43:59.323 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.150526+0000", 2023-09-23T20:43:59.323 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0094936429999999995 2023-09-23T20:43:59.323 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.323 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.323 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:43:59.323 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.150571+0000", 2023-09-23T20:43:59.324 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.4948999999999999e-05 2023-09-23T20:43:59.324 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.324 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.324 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:43:59.324 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.150689+0000", 2023-09-23T20:43:59.324 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00011736800000000001 2023-09-23T20:43:59.324 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:43:59.324 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:43:59.324 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:43:59.324 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.324 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.324 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35700.0:156997 7.9 7:926c23dd:::benchmark_data_smithi018_308601_object62445:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e354)", 2023-09-23T20:43:59.325 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:35:00.331744+0000", 2023-09-23T20:43:59.325 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 538.86948649999999, 2023-09-23T20:43:59.325 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.83941280200000001, 2023-09-23T20:43:59.325 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:43:59.325 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:43:59.325 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:43:59.325 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35700", 2023-09-23T20:43:59.325 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:43:59.325 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 156997 2023-09-23T20:43:59.325 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.325 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:43:59.325 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.326 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:43:59.326 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.331744+0000", 2023-09-23T20:43:59.326 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:43:59.326 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.326 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.326 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:43:59.326 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.331744+0000", 2023-09-23T20:43:59.326 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:43:59.326 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.326 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.326 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:43:59.326 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.331743+0000", 2023-09-23T20:43:59.327 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:43:59.327 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.327 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.327 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:43:59.327 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.331749+0000", 2023-09-23T20:43:59.327 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.1140000000000002e-06 2023-09-23T20:43:59.327 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.327 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.327 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:43:59.327 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.331762+0000", 2023-09-23T20:43:59.327 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3275000000000001e-05 2023-09-23T20:43:59.328 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.328 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.328 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:43:59.328 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.331768+0000", 2023-09-23T20:43:59.328 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.7930000000000003e-06 2023-09-23T20:43:59.328 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.328 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.328 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:43:59.328 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.331839+0000", 2023-09-23T20:43:59.328 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.0685999999999994e-05 2023-09-23T20:43:59.328 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.328 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.329 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:43:59.329 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:00.331853+0000", 2023-09-23T20:43:59.329 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.4698999999999999e-05 2023-09-23T20:43:59.329 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.329 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.329 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:43:59.329 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.137742+0000", 2023-09-23T20:43:59.329 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.80588849600000001 2023-09-23T20:43:59.329 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.329 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.329 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:43:59.329 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.137885+0000", 2023-09-23T20:43:59.330 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00014333 2023-09-23T20:43:59.330 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.330 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.330 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,4,7", 2023-09-23T20:43:59.330 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.138005+0000", 2023-09-23T20:43:59.330 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000120022 2023-09-23T20:43:59.330 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.330 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.330 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:43:59.330 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.141352+0000", 2023-09-23T20:43:59.330 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.003346768 2023-09-23T20:43:59.330 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.331 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.331 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:43:59.331 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.141417+0000", 2023-09-23T20:43:59.331 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.4555999999999997e-05 2023-09-23T20:43:59.331 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.331 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.331 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:43:59.331 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.141425+0000", 2023-09-23T20:43:59.331 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.3869999999999995e-06 2023-09-23T20:43:59.331 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.331 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.332 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:43:59.332 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.170973+0000", 2023-09-23T20:43:59.332 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.02954824 2023-09-23T20:43:59.332 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.332 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.332 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:43:59.332 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.171028+0000", 2023-09-23T20:43:59.332 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.5040999999999997e-05 2023-09-23T20:43:59.332 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.332 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.332 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:43:59.333 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:35:01.171157+0000", 2023-09-23T20:43:59.333 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00012873999999999999 2023-09-23T20:43:59.333 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:43:59.333 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:43:59.333 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:43:59.333 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.333 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.333 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35994.0:105775 8.9 8.ed1637e9 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e410)", 2023-09-23T20:43:59.333 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:38:26.448011+0000", 2023-09-23T20:43:59.333 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 332.75321981799999, 2023-09-23T20:43:59.333 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.95335516099999995, 2023-09-23T20:43:59.333 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:43:59.334 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:43:59.334 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:43:59.334 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35994", 2023-09-23T20:43:59.334 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:43:59.334 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 105775 2023-09-23T20:43:59.334 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.334 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:43:59.334 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.334 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:43:59.334 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.448011+0000", 2023-09-23T20:43:59.334 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:43:59.335 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.335 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.335 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:43:59.335 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.448011+0000", 2023-09-23T20:43:59.335 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:43:59.335 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.335 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.335 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:43:59.335 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.448010+0000", 2023-09-23T20:43:59.335 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:43:59.335 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.336 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.336 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:43:59.336 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.448014+0000", 2023-09-23T20:43:59.336 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.2040000000000002e-06 2023-09-23T20:43:59.336 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.336 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.336 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:43:59.336 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.448022+0000", 2023-09-23T20:43:59.336 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.5199999999999997e-06 2023-09-23T20:43:59.336 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.336 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.336 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:43:59.337 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.448027+0000", 2023-09-23T20:43:59.337 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.2460000000000003e-06 2023-09-23T20:43:59.337 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.337 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.337 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:43:59.337 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.457490+0000", 2023-09-23T20:43:59.337 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0094626720000000001 2023-09-23T20:43:59.337 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.337 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.337 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op must wait for map", 2023-09-23T20:43:59.337 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.457499+0000", 2023-09-23T20:43:59.337 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.2050000000000008e-06 2023-09-23T20:43:59.338 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.338 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.338 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:43:59.338 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.789612+0000", 2023-09-23T20:43:59.338 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.33211301500000001 2023-09-23T20:43:59.338 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.338 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.338 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:43:59.338 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.789620+0000", 2023-09-23T20:43:59.338 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.0479999999999999e-06 2023-09-23T20:43:59.338 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.339 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.339 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:43:59.339 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.401354+0000", 2023-09-23T20:43:59.339 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.61173345599999995 2023-09-23T20:43:59.339 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.339 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.339 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:43:59.339 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.401366+0000", 2023-09-23T20:43:59.339 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2254e-05 2023-09-23T20:43:59.339 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:43:59.339 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:43:59.339 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:43:59.340 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.340 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.340 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35994.0:105788 8.9 8.2875af29 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e410)", 2023-09-23T20:43:59.340 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:38:26.448040+0000", 2023-09-23T20:43:59.340 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 332.75319067700002, 2023-09-23T20:43:59.340 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.95336172100000005, 2023-09-23T20:43:59.340 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:43:59.340 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:43:59.340 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:43:59.340 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35994", 2023-09-23T20:43:59.340 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:43:59.340 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 105788 2023-09-23T20:43:59.341 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.341 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:43:59.341 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.341 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:43:59.341 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.448040+0000", 2023-09-23T20:43:59.341 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:43:59.341 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.341 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.341 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:43:59.341 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.448040+0000", 2023-09-23T20:43:59.341 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:43:59.341 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.342 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.342 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:43:59.342 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.448040+0000", 2023-09-23T20:43:59.342 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:43:59.342 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.342 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.342 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:43:59.342 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.448042+0000", 2023-09-23T20:43:59.342 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.4789999999999999e-06 2023-09-23T20:43:59.342 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.342 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.343 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:43:59.343 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.448050+0000", 2023-09-23T20:43:59.343 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.503e-06 2023-09-23T20:43:59.343 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.343 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.343 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:43:59.343 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.448055+0000", 2023-09-23T20:43:59.343 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.0849999999999996e-06 2023-09-23T20:43:59.343 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.343 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.343 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:43:59.344 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.457556+0000", 2023-09-23T20:43:59.344 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0095011509999999993 2023-09-23T20:43:59.344 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.344 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.344 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting_for_map not empty", 2023-09-23T20:43:59.344 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.457563+0000", 2023-09-23T20:43:59.344 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.4760000000000001e-06 2023-09-23T20:43:59.344 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.344 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.344 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:43:59.344 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.789633+0000", 2023-09-23T20:43:59.344 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.33206993600000001 2023-09-23T20:43:59.345 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.345 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.345 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:43:59.345 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.789638+0000", 2023-09-23T20:43:59.345 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.3630000000000004e-06 2023-09-23T20:43:59.346 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.346 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.346 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:43:59.346 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.401390+0000", 2023-09-23T20:43:59.346 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.61175234999999994 2023-09-23T20:43:59.347 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.347 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.347 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:43:59.347 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.401402+0000", 2023-09-23T20:43:59.347 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1780000000000001e-05 2023-09-23T20:43:59.347 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:43:59.347 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:43:59.347 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:43:59.347 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.347 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.347 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.36450.0:159258 9.8 9:18f5e2a5:::benchmark_data_smithi018_381516_object45424:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e486)", 2023-09-23T20:43:59.348 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:43:38.942054+0000", 2023-09-23T20:43:59.348 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 20.259176735, 2023-09-23T20:43:59.348 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.065835723999999998, 2023-09-23T20:43:59.348 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:43:59.348 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:43:59.348 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:43:59.348 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.36450", 2023-09-23T20:43:59.348 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:43:59.348 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 159258 2023-09-23T20:43:59.348 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.348 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:43:59.349 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.349 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:43:59.349 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:38.942054+0000", 2023-09-23T20:43:59.349 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:43:59.349 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.349 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.349 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:43:59.349 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:38.942054+0000", 2023-09-23T20:43:59.349 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:43:59.349 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.349 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.349 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:43:59.350 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:38.942053+0000", 2023-09-23T20:43:59.350 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:43:59.350 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.350 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.350 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:43:59.350 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:38.942059+0000", 2023-09-23T20:43:59.350 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.2790000000000004e-06 2023-09-23T20:43:59.350 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.350 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.350 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:43:59.350 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:38.942068+0000", 2023-09-23T20:43:59.350 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.5510000000000005e-06 2023-09-23T20:43:59.351 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.351 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.351 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:43:59.351 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:38.942076+0000", 2023-09-23T20:43:59.351 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.824e-06 2023-09-23T20:43:59.351 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.351 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.351 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:43:59.351 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:38.942560+0000", 2023-09-23T20:43:59.351 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00048387199999999999 2023-09-23T20:43:59.351 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.352 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.352 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:43:59.352 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:38.942711+0000", 2023-09-23T20:43:59.352 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000150946 2023-09-23T20:43:59.352 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.352 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.352 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,3,5,7", 2023-09-23T20:43:59.352 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:38.942886+0000", 2023-09-23T20:43:59.352 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00017465100000000001 2023-09-23T20:43:59.352 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.352 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.352 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:43:59.353 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:38.946235+0000", 2023-09-23T20:43:59.353 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0033494829999999999 2023-09-23T20:43:59.353 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.353 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.353 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:43:59.353 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:38.946453+0000", 2023-09-23T20:43:59.353 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00021817700000000001 2023-09-23T20:43:59.353 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.353 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.353 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:43:59.353 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:38.946901+0000", 2023-09-23T20:43:59.353 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00044799800000000002 2023-09-23T20:43:59.354 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.354 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.354 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:43:59.354 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:38.949493+0000", 2023-09-23T20:43:59.354 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0025915970000000002 2023-09-23T20:43:59.354 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.354 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.354 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:43:59.354 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:39.007643+0000", 2023-09-23T20:43:59.354 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.058150065000000001 2023-09-23T20:43:59.354 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.355 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.355 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:43:59.355 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:39.007705+0000", 2023-09-23T20:43:59.355 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.2292000000000006e-05 2023-09-23T20:43:59.355 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.355 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.355 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:43:59.355 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:39.007890+0000", 2023-09-23T20:43:59.355 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000184413 2023-09-23T20:43:59.355 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:43:59.355 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:43:59.356 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:43:59.356 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.356 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.356 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.36450.0:159453 9.8 9:16bc9847:::benchmark_data_smithi018_381516_object45619:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e486)", 2023-09-23T20:43:59.356 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:43:39.189308+0000", 2023-09-23T20:43:59.356 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 20.011922963, 2023-09-23T20:43:59.356 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.119838766, 2023-09-23T20:43:59.356 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:43:59.356 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:43:59.356 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:43:59.356 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.36450", 2023-09-23T20:43:59.357 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:43:59.357 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 159453 2023-09-23T20:43:59.357 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.357 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:43:59.357 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.357 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:43:59.357 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:39.189308+0000", 2023-09-23T20:43:59.357 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:43:59.357 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.357 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.357 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:43:59.357 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:39.189308+0000", 2023-09-23T20:43:59.358 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:43:59.358 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.358 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.358 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:43:59.358 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:39.189306+0000", 2023-09-23T20:43:59.358 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:43:59.358 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.358 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.358 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:43:59.358 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:39.189313+0000", 2023-09-23T20:43:59.358 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.579e-06 2023-09-23T20:43:59.359 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.359 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.359 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:43:59.359 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:39.189326+0000", 2023-09-23T20:43:59.359 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.327e-05 2023-09-23T20:43:59.359 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.359 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.359 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:43:59.359 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:39.189334+0000", 2023-09-23T20:43:59.359 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.4169999999999992e-06 2023-09-23T20:43:59.359 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.359 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.359 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:43:59.360 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:39.189381+0000", 2023-09-23T20:43:59.360 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.6687000000000002e-05 2023-09-23T20:43:59.360 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.360 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.360 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:43:59.360 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:39.189511+0000", 2023-09-23T20:43:59.360 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00012972799999999999 2023-09-23T20:43:59.360 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.360 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.360 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,3,5,7", 2023-09-23T20:43:59.360 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:39.189643+0000", 2023-09-23T20:43:59.361 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00013187299999999999 2023-09-23T20:43:59.361 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.361 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.361 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:43:59.361 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:39.193529+0000", 2023-09-23T20:43:59.361 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0038860980000000002 2023-09-23T20:43:59.361 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.361 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.361 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:43:59.361 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:39.193536+0000", 2023-09-23T20:43:59.361 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.0249999999999997e-06 2023-09-23T20:43:59.361 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.362 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.362 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:43:59.362 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:39.193567+0000", 2023-09-23T20:43:59.362 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.1257000000000002e-05 2023-09-23T20:43:59.362 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.362 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.362 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:43:59.362 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:39.198872+0000", 2023-09-23T20:43:59.362 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.005304586 2023-09-23T20:43:59.362 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.362 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.363 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:43:59.363 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:39.308968+0000", 2023-09-23T20:43:59.363 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.110096237 2023-09-23T20:43:59.363 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.363 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.363 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:43:59.363 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:39.309017+0000", 2023-09-23T20:43:59.363 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.8903999999999998e-05 2023-09-23T20:43:59.363 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.363 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.363 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:43:59.363 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:39.309146+0000", 2023-09-23T20:43:59.364 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00012967499999999999 2023-09-23T20:43:59.364 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:43:59.364 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:43:59.364 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:43:59.364 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.364 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.364 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.36450.0:159456 9.8 9:1b85c752:::benchmark_data_smithi018_381516_object45622:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e486)", 2023-09-23T20:43:59.364 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:43:39.193325+0000", 2023-09-23T20:43:59.364 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 20.007906071000001, 2023-09-23T20:43:59.364 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.116019725, 2023-09-23T20:43:59.364 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:43:59.364 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:43:59.365 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:43:59.365 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.36450", 2023-09-23T20:43:59.365 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:43:59.365 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 159456 2023-09-23T20:43:59.365 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.365 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:43:59.365 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.365 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:43:59.365 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:39.193325+0000", 2023-09-23T20:43:59.365 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:43:59.365 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.366 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.366 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:43:59.366 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:39.193325+0000", 2023-09-23T20:43:59.366 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:43:59.366 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.366 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.366 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:43:59.366 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:39.193324+0000", 2023-09-23T20:43:59.366 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.999999 2023-09-23T20:43:59.366 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.367 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.367 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:43:59.367 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:39.193328+0000", 2023-09-23T20:43:59.367 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.9280000000000001e-06 2023-09-23T20:43:59.367 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.367 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.367 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:43:59.367 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:39.193337+0000", 2023-09-23T20:43:59.367 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.3529999999999995e-06 2023-09-23T20:43:59.367 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.367 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.367 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:43:59.368 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:39.193342+0000", 2023-09-23T20:43:59.368 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.1379999999999998e-06 2023-09-23T20:43:59.368 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.368 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.368 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:43:59.368 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:39.193582+0000", 2023-09-23T20:43:59.368 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00023963000000000001 2023-09-23T20:43:59.368 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.368 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.368 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:43:59.368 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:39.193665+0000", 2023-09-23T20:43:59.368 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.3502000000000001e-05 2023-09-23T20:43:59.369 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.369 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.369 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,3,5,7", 2023-09-23T20:43:59.369 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:39.193747+0000", 2023-09-23T20:43:59.369 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.2020000000000004e-05 2023-09-23T20:43:59.369 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.369 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.369 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:43:59.369 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:39.196673+0000", 2023-09-23T20:43:59.369 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0029255980000000002 2023-09-23T20:43:59.369 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.369 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.370 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:43:59.370 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:39.197321+0000", 2023-09-23T20:43:59.370 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00064826800000000002 2023-09-23T20:43:59.370 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.370 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.370 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:43:59.370 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:39.197782+0000", 2023-09-23T20:43:59.370 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00046122900000000001 2023-09-23T20:43:59.370 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.370 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.370 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:43:59.371 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:39.198999+0000", 2023-09-23T20:43:59.371 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0012170569999999999 2023-09-23T20:43:59.371 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.371 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.371 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:43:59.371 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:39.309185+0000", 2023-09-23T20:43:59.371 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.11018600000000001 2023-09-23T20:43:59.371 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.371 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.371 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:43:59.371 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:39.309230+0000", 2023-09-23T20:43:59.371 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.5037999999999998e-05 2023-09-23T20:43:59.372 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.372 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.372 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:43:59.372 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:39.309344+0000", 2023-09-23T20:43:59.372 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00011400099999999999 2023-09-23T20:43:59.372 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:43:59.372 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:43:59.372 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:43:59.372 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.372 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.372 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.36450.0:172000 9.8 9:11359877:::benchmark_data_smithi018_381516_object58166:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e490)", 2023-09-23T20:43:59.373 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:43:55.320893+0000", 2023-09-23T20:43:59.373 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 3.8803375879999997, 2023-09-23T20:43:59.373 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.064814561000000007, 2023-09-23T20:43:59.373 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:43:59.373 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:43:59.373 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:43:59.373 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.36450", 2023-09-23T20:43:59.373 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:43:59.373 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 172000 2023-09-23T20:43:59.373 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.373 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:43:59.373 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.374 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:43:59.374 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:55.320893+0000", 2023-09-23T20:43:59.374 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:43:59.374 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.374 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.374 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:43:59.374 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:55.320893+0000", 2023-09-23T20:43:59.374 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:43:59.374 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.374 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.374 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:43:59.374 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:55.320893+0000", 2023-09-23T20:43:59.375 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:43:59.375 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.375 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.375 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:43:59.375 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:55.320896+0000", 2023-09-23T20:43:59.375 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.9699999999999999e-06 2023-09-23T20:43:59.375 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.375 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.375 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:43:59.375 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:55.320902+0000", 2023-09-23T20:43:59.375 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.6800000000000004e-06 2023-09-23T20:43:59.375 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.376 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.376 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:43:59.376 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:55.320908+0000", 2023-09-23T20:43:59.376 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.1099999999999999e-06 2023-09-23T20:43:59.376 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.376 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.376 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:43:59.376 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:55.325234+0000", 2023-09-23T20:43:59.376 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0043259320000000002 2023-09-23T20:43:59.376 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.376 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.376 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:43:59.377 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:55.325384+0000", 2023-09-23T20:43:59.377 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000149529 2023-09-23T20:43:59.377 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.377 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.377 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,3,5,7", 2023-09-23T20:43:59.377 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:55.325523+0000", 2023-09-23T20:43:59.377 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000138892 2023-09-23T20:43:59.377 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.377 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.377 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:43:59.377 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:55.329427+0000", 2023-09-23T20:43:59.378 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0039038150000000001 2023-09-23T20:43:59.378 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.378 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.378 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:43:59.378 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:55.331384+0000", 2023-09-23T20:43:59.378 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0019575700000000001 2023-09-23T20:43:59.378 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.378 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.378 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:43:59.378 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:55.332436+0000", 2023-09-23T20:43:59.378 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0010522050000000001 2023-09-23T20:43:59.379 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.379 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.379 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:43:59.379 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:55.341701+0000", 2023-09-23T20:43:59.379 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0092646529999999994 2023-09-23T20:43:59.379 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.379 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.379 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:43:59.379 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:55.385519+0000", 2023-09-23T20:43:59.379 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.043818358000000002 2023-09-23T20:43:59.379 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.380 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.380 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:43:59.380 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:55.385569+0000", 2023-09-23T20:43:59.380 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.9471000000000001e-05 2023-09-23T20:43:59.380 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:43:59.380 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:43:59.380 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:43:59.380 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:55.385708+0000", 2023-09-23T20:43:59.380 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00013881399999999999 2023-09-23T20:43:59.380 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:43:59.380 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:43:59.380 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:43:59.381 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:43:59.381 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:43:59.381 INFO:teuthology.orchestra.run.smithi018.stdout:} 2023-09-23T20:43:59.382 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:59.382 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:59.382 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:43:59.383 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:43:59.484 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:43:59.579 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:43:59.755 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:59 smithi018 ceph-mon[184144]: pgmap v1242: 119 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 117 active+clean; 3.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 770 op/s; 2239/165759 objects misplaced (1.351%); 7.0 MiB/s, 111 objects/s recovering 2023-09-23T20:43:59.860 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:59 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:43:59.860 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:43:59 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:43:59.861 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:59 smithi018 ceph-mon[179647]: pgmap v1242: 119 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 117 active+clean; 3.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 770 op/s; 2239/165759 objects misplaced (1.351%); 7.0 MiB/s, 111 objects/s recovering 2023-09-23T20:43:59.861 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:59 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:43:59.861 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:43:59 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:43:59.862 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:43:59.871 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:43:59.872 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:43:59.884 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:43:59.892 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T20:43:59.972 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:44:00.013 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:59 smithi149 ceph-mon[170394]: pgmap v1242: 119 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 117 active+clean; 3.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 770 op/s; 2239/165759 objects misplaced (1.351%); 7.0 MiB/s, 111 objects/s recovering 2023-09-23T20:44:00.013 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:59 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:44:00.013 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:43:59 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:44:00.016 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:00.062 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:44:00.069 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:00.069 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:44:00.170 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:44:00.176 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T20:44:00.230 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:00.334 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:44:00.340 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:00.340 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:44:00.441 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:44:00.632 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:00.748 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:44:00.755 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:00.755 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:44:00.856 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:44:00.899 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:00 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.15", "id": [3, 2]}]: dispatch 2023-09-23T20:44:00.900 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:00.902 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:00 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.15", "id": [3, 2]}]: dispatch 2023-09-23T20:44:00.958 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:44:00.965 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:00.965 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:44:01.027 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:44:01.045 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:44:01.066 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:44:01.171 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:01.218 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:00 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.15", "id": [3, 2]}]: dispatch 2023-09-23T20:44:01.819 INFO:teuthology.orchestra.run.smithi018.stderr:reweighted osd.0 to 1 (10000) 2023-09-23T20:44:02.035 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:44:02.042 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:02.043 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:44:02.043 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:01 smithi018 ceph-mon[184144]: pgmap v1243: 119 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 117 active+clean; 3.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 820 op/s; 2239/160320 objects misplaced (1.397%); 7.0 MiB/s, 111 objects/s recovering 2023-09-23T20:44:02.043 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:01 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.15", "id": [3, 2]}]': finished 2023-09-23T20:44:02.043 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:01 smithi018 ceph-mon[184144]: osdmap e491: 8 total, 8 up, 8 in 2023-09-23T20:44:02.043 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:01 smithi018 ceph-mon[184144]: failed to encode map e491 with expected crc 2023-09-23T20:44:02.043 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:01 smithi018 ceph-mon[184144]: failed to encode map e491 with expected crc 2023-09-23T20:44:02.043 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:01 smithi018 ceph-mon[184144]: failed to encode map e491 with expected crc 2023-09-23T20:44:02.043 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:01 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2714287693' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T20:44:02.044 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:01 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:44:02.044 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:01 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:44:02.044 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:01 smithi018 ceph-mon[179647]: pgmap v1243: 119 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 117 active+clean; 3.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 820 op/s; 2239/160320 objects misplaced (1.397%); 7.0 MiB/s, 111 objects/s recovering 2023-09-23T20:44:02.044 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:01 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.15", "id": [3, 2]}]': finished 2023-09-23T20:44:02.044 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:01 smithi018 ceph-mon[179647]: osdmap e491: 8 total, 8 up, 8 in 2023-09-23T20:44:02.044 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:01 smithi018 ceph-mon[179647]: failed to encode map e491 with expected crc 2023-09-23T20:44:02.044 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:01 smithi018 ceph-mon[179647]: failed to encode map e491 with expected crc 2023-09-23T20:44:02.045 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:01 smithi018 ceph-mon[179647]: failed to encode map e491 with expected crc 2023-09-23T20:44:02.045 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:01 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2714287693' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T20:44:02.045 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:01 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:44:02.045 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:01 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:44:02.053 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:44:02.144 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:44:02.183 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:01 smithi149 ceph-mon[170394]: pgmap v1243: 119 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 117 active+clean; 3.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 820 op/s; 2239/160320 objects misplaced (1.397%); 7.0 MiB/s, 111 objects/s recovering 2023-09-23T20:44:02.183 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:01 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.15", "id": [3, 2]}]': finished 2023-09-23T20:44:02.183 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:01 smithi149 ceph-mon[170394]: osdmap e491: 8 total, 8 up, 8 in 2023-09-23T20:44:02.183 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:01 smithi149 ceph-mon[170394]: failed to encode map e491 with expected crc 2023-09-23T20:44:02.184 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:01 smithi149 ceph-mon[170394]: failed to encode map e491 with expected crc 2023-09-23T20:44:02.184 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:01 smithi149 ceph-mon[170394]: failed to encode map e491 with expected crc 2023-09-23T20:44:02.184 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:01 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2714287693' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T20:44:02.184 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:01 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:44:02.184 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:01 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:44:02.188 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:02.248 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:44:02.254 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:02.254 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:44:02.355 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:44:02.441 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:03.130 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:44:03.137 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:03.138 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:44:03.138 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:02 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2714287693' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T20:44:03.139 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:02 smithi018 ceph-mon[184144]: osdmap e492: 8 total, 8 up, 8 in 2023-09-23T20:44:03.139 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:02 smithi018 ceph-mon[184144]: failed to encode map e492 with expected crc 2023-09-23T20:44:03.139 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:02 smithi018 ceph-mon[184144]: failed to encode map e492 with expected crc 2023-09-23T20:44:03.139 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:02 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2714287693' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T20:44:03.140 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:02 smithi018 ceph-mon[179647]: osdmap e492: 8 total, 8 up, 8 in 2023-09-23T20:44:03.140 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:02 smithi018 ceph-mon[179647]: failed to encode map e492 with expected crc 2023-09-23T20:44:03.140 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:02 smithi018 ceph-mon[179647]: failed to encode map e492 with expected crc 2023-09-23T20:44:03.218 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:02 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2714287693' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T20:44:03.218 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:02 smithi149 ceph-mon[170394]: osdmap e492: 8 total, 8 up, 8 in 2023-09-23T20:44:03.218 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:02 smithi149 ceph-mon[170394]: failed to encode map e492 with expected crc 2023-09-23T20:44:03.218 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:02 smithi149 ceph-mon[170394]: failed to encode map e492 with expected crc 2023-09-23T20:44:03.239 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:44:03.296 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:03.415 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:44:03.436 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:03.436 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:44:03.537 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:44:03.579 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd reweight 1 1 2023-09-23T20:44:03.584 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:03.637 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:44:03.645 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:03.646 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:44:03.747 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:44:03.788 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:03.815 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:03 smithi018 ceph-mon[179647]: pgmap v1246: 119 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 117 active+clean; 3.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 803 op/s; 1429/157623 objects misplaced (0.907%); 5.9 MiB/s, 95 objects/s recovering 2023-09-23T20:44:03.815 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:03 smithi018 ceph-mon[184144]: pgmap v1246: 119 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 117 active+clean; 3.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 803 op/s; 1429/157623 objects misplaced (0.907%); 5.9 MiB/s, 95 objects/s recovering 2023-09-23T20:44:03.837 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:44:03.842 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:03 smithi149 ceph-mon[170394]: pgmap v1246: 119 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 117 active+clean; 3.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 803 op/s; 1429/157623 objects misplaced (0.907%); 5.9 MiB/s, 95 objects/s recovering 2023-09-23T20:44:03.843 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:03.844 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:44:03.945 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:44:04.050 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:04.630 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:44:04.635 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:04.636 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:44:04.736 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:44:04.791 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:04.856 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:44:04.865 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:04.865 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:44:04.966 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:44:05.018 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:05.148 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:44:05.152 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:05.152 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:44:05.253 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:44:05.280 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:05.313 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:44:05.316 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:05.317 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:44:05.391 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:44:04 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:44:04] "GET /metrics HTTP/1.1" 200 36604 "" "Prometheus/2.43.0" 2023-09-23T20:44:05.418 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:44:05.444 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:05.480 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:44:05.483 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:05.483 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:44:05.584 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:44:05.624 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:05.864 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:44:05.867 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:05.867 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:44:05.968 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:44:06.027 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:06.079 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:44:06.084 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:06.085 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:44:06.146 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:05 smithi018 ceph-mon[179647]: pgmap v1247: 119 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 117 active+clean; 3.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 844 op/s; 1429/152271 objects misplaced (0.938%); 5.5 MiB/s, 88 objects/s recovering 2023-09-23T20:44:06.146 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:05 smithi018 ceph-mon[179647]: osdmap e493: 8 total, 8 up, 8 in 2023-09-23T20:44:06.146 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:05 smithi018 ceph-mon[179647]: failed to encode map e493 with expected crc 2023-09-23T20:44:06.146 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:05 smithi018 ceph-mon[179647]: failed to encode map e493 with expected crc 2023-09-23T20:44:06.146 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:05 smithi018 ceph-mon[179647]: failed to encode map e493 with expected crc 2023-09-23T20:44:06.146 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:05 smithi018 ceph-mon[179647]: failed to encode map e493 with expected crc 2023-09-23T20:44:06.146 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:05 smithi018 ceph-mon[179647]: failed to encode map e493 with expected crc 2023-09-23T20:44:06.146 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:05 smithi018 ceph-mon[179647]: failed to encode map e493 with expected crc 2023-09-23T20:44:06.147 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:05 smithi018 ceph-mon[184144]: pgmap v1247: 119 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 117 active+clean; 3.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 844 op/s; 1429/152271 objects misplaced (0.938%); 5.5 MiB/s, 88 objects/s recovering 2023-09-23T20:44:06.147 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:05 smithi018 ceph-mon[184144]: osdmap e493: 8 total, 8 up, 8 in 2023-09-23T20:44:06.147 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:05 smithi018 ceph-mon[184144]: failed to encode map e493 with expected crc 2023-09-23T20:44:06.148 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:05 smithi018 ceph-mon[184144]: failed to encode map e493 with expected crc 2023-09-23T20:44:06.148 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:05 smithi018 ceph-mon[184144]: failed to encode map e493 with expected crc 2023-09-23T20:44:06.148 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:05 smithi018 ceph-mon[184144]: failed to encode map e493 with expected crc 2023-09-23T20:44:06.148 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:05 smithi018 ceph-mon[184144]: failed to encode map e493 with expected crc 2023-09-23T20:44:06.148 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:05 smithi018 ceph-mon[184144]: failed to encode map e493 with expected crc 2023-09-23T20:44:06.185 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:44:06.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:05 smithi149 ceph-mon[170394]: pgmap v1247: 119 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 117 active+clean; 3.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 844 op/s; 1429/152271 objects misplaced (0.938%); 5.5 MiB/s, 88 objects/s recovering 2023-09-23T20:44:06.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:05 smithi149 ceph-mon[170394]: osdmap e493: 8 total, 8 up, 8 in 2023-09-23T20:44:06.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:05 smithi149 ceph-mon[170394]: failed to encode map e493 with expected crc 2023-09-23T20:44:06.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:05 smithi149 ceph-mon[170394]: failed to encode map e493 with expected crc 2023-09-23T20:44:06.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:05 smithi149 ceph-mon[170394]: failed to encode map e493 with expected crc 2023-09-23T20:44:06.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:05 smithi149 ceph-mon[170394]: failed to encode map e493 with expected crc 2023-09-23T20:44:06.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:05 smithi149 ceph-mon[170394]: failed to encode map e493 with expected crc 2023-09-23T20:44:06.218 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:05 smithi149 ceph-mon[170394]: failed to encode map e493 with expected crc 2023-09-23T20:44:06.247 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:06.277 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:44:06.468 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:44:06.477 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:06.478 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:44:06.579 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:44:06.611 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:06.659 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:44:06.665 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:06.665 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:44:06.766 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:44:06.861 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:06.906 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2023-09-23T20:44:06.906 INFO:teuthology.orchestra.run.smithi018.stdout: "ops": [], 2023-09-23T20:44:06.906 INFO:teuthology.orchestra.run.smithi018.stdout: "num_ops": 0 2023-09-23T20:44:06.907 INFO:teuthology.orchestra.run.smithi018.stdout:} 2023-09-23T20:44:07.077 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:44:07.118 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:44:07.136 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:44:07.142 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:07.143 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:44:07.145 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:06 smithi018 ceph-mon[184144]: failed to encode map e494 with expected crc 2023-09-23T20:44:07.145 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:06 smithi018 ceph-mon[184144]: osdmap e494: 8 total, 8 up, 8 in 2023-09-23T20:44:07.145 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:06 smithi018 ceph-mon[184144]: failed to encode map e494 with expected crc 2023-09-23T20:44:07.145 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:06 smithi018 ceph-mon[184144]: failed to encode map e494 with expected crc 2023-09-23T20:44:07.145 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:06 smithi018 ceph-mon[184144]: failed to encode map e494 with expected crc 2023-09-23T20:44:07.145 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:06 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/48814973' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:44:07.145 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:06 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:44:07.150 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:06 smithi018 ceph-mon[179647]: failed to encode map e494 with expected crc 2023-09-23T20:44:07.150 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:06 smithi018 ceph-mon[179647]: osdmap e494: 8 total, 8 up, 8 in 2023-09-23T20:44:07.150 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:06 smithi018 ceph-mon[179647]: failed to encode map e494 with expected crc 2023-09-23T20:44:07.151 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:06 smithi018 ceph-mon[179647]: failed to encode map e494 with expected crc 2023-09-23T20:44:07.151 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:06 smithi018 ceph-mon[179647]: failed to encode map e494 with expected crc 2023-09-23T20:44:07.151 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:06 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/48814973' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:44:07.151 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:06 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:44:07.162 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:44:07.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:06 smithi149 ceph-mon[170394]: failed to encode map e494 with expected crc 2023-09-23T20:44:07.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:06 smithi149 ceph-mon[170394]: osdmap e494: 8 total, 8 up, 8 in 2023-09-23T20:44:07.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:06 smithi149 ceph-mon[170394]: failed to encode map e494 with expected crc 2023-09-23T20:44:07.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:06 smithi149 ceph-mon[170394]: failed to encode map e494 with expected crc 2023-09-23T20:44:07.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:06 smithi149 ceph-mon[170394]: failed to encode map e494 with expected crc 2023-09-23T20:44:07.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:06 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/48814973' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:44:07.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:06 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:44:07.217 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:44:07.243 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:44:07.265 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:44:07.304 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:07.305 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:44:07.346 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:44:07.383 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:44:07.404 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:44:07.404 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:44:07.404 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:44:07.404 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:44:07.404 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:44:07.404 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:44:07.404 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:44:07.404 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:44:07.569 INFO:teuthology.orchestra.run.smithi018.stderr:nodeep-scrub is set 2023-09-23T20:44:07.652 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T20:44:08.002 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:44:08.005 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:07 smithi018 ceph-mon[179647]: pgmap v1250: 119 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 117 active+clean; 3.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 664 op/s; 1429/149568 objects misplaced (0.955%); 4.0 MiB/s, 63 objects/s recovering 2023-09-23T20:44:08.005 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:07 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/623367654' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:44:08.005 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:07 smithi018 ceph-mon[179647]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:44:08.005 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:07 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:44:08.005 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:07 smithi018 ceph-mon[179647]: osdmap e495: 8 total, 8 up, 8 in 2023-09-23T20:44:08.005 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:07 smithi018 ceph-mon[179647]: failed to encode map e495 with expected crc 2023-09-23T20:44:08.006 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:08.007 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:44:08.012 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:07 smithi018 ceph-mon[184144]: pgmap v1250: 119 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 117 active+clean; 3.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 664 op/s; 1429/149568 objects misplaced (0.955%); 4.0 MiB/s, 63 objects/s recovering 2023-09-23T20:44:08.012 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:07 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/623367654' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:44:08.013 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:07 smithi018 ceph-mon[184144]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:44:08.013 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:07 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:44:08.013 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:07 smithi018 ceph-mon[184144]: osdmap e495: 8 total, 8 up, 8 in 2023-09-23T20:44:08.013 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:07 smithi018 ceph-mon[184144]: failed to encode map e495 with expected crc 2023-09-23T20:44:08.107 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:44:08.203 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:08.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:07 smithi149 ceph-mon[170394]: pgmap v1250: 119 pgs: 1 active+remapped+backfilling, 1 active+remapped+backfill_wait, 117 active+clean; 3.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 664 op/s; 1429/149568 objects misplaced (0.955%); 4.0 MiB/s, 63 objects/s recovering 2023-09-23T20:44:08.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:07 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/623367654' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:44:08.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:07 smithi149 ceph-mon[170394]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:44:08.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:07 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:44:08.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:07 smithi149 ceph-mon[170394]: osdmap e495: 8 total, 8 up, 8 in 2023-09-23T20:44:08.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:07 smithi149 ceph-mon[170394]: failed to encode map e495 with expected crc 2023-09-23T20:44:08.381 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:44:08.405 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:08.406 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:44:08.507 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:44:08.552 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:08.610 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:44:08.614 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:08.614 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:44:08.715 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:44:08.757 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:08.802 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:44:08.806 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:08.806 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:44:08.906 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:44:08.980 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:44:08.997 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:09.164 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:44:09.169 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:09.169 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:44:09.270 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:44:09.370 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:09.445 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:44:09.454 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:09.454 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:44:09.555 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:44:09.564 INFO:teuthology.orchestra.run.smithi018.stderr:reweighted osd.1 to 1 (10000) 2023-09-23T20:44:09.567 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:09 smithi018 ceph-mon[179647]: pgmap v1252: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 2.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 743 op/s; 216/144240 objects misplaced (0.150%); 9.8 MiB/s, 157 objects/s recovering 2023-09-23T20:44:09.567 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:09 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:44:09.567 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:09 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:44:09.568 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:09 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1880679898' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T20:44:09.568 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:09 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T20:44:09.568 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:09 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:44:09.568 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:09 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:44:09.568 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:09 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:44:09.568 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:09 smithi018 ceph-mon[184144]: pgmap v1252: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 2.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 743 op/s; 216/144240 objects misplaced (0.150%); 9.8 MiB/s, 157 objects/s recovering 2023-09-23T20:44:09.569 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:09 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:44:09.569 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:09 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:44:09.570 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:09 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1880679898' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T20:44:09.570 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:09 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T20:44:09.570 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:09 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:44:09.570 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:09 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:44:09.570 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:09 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:44:09.628 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:09.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:09 smithi149 ceph-mon[170394]: pgmap v1252: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 2.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 743 op/s; 216/144240 objects misplaced (0.150%); 9.8 MiB/s, 157 objects/s recovering 2023-09-23T20:44:09.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:09 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:44:09.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:09 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:44:09.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:09 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1880679898' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T20:44:09.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:09 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T20:44:09.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:09 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:44:09.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:09 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:44:09.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:09 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:44:10.117 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd reweight 7 1 2023-09-23T20:44:10.232 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:44:10.236 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:10.236 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:44:10.337 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:44:10.366 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:10.404 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:44:10.406 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:10.407 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:44:10.433 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T20:44:10.507 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:44:10.536 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:10.575 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:44:10.578 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:10.578 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:44:10.679 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:44:10.704 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:10 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T20:44:10.705 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:10 smithi149 ceph-mon[170394]: osdmap e496: 8 total, 8 up, 8 in 2023-09-23T20:44:10.705 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:10 smithi149 ceph-mon[170394]: failed to encode map e496 with expected crc 2023-09-23T20:44:10.705 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:10 smithi149 ceph-mon[170394]: failed to encode map e496 with expected crc 2023-09-23T20:44:10.705 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:10 smithi149 ceph-mon[170394]: failed to encode map e496 with expected crc 2023-09-23T20:44:10.705 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:10 smithi149 ceph-mon[170394]: failed to encode map e496 with expected crc 2023-09-23T20:44:10.707 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:10.753 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:44:10.759 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:10.760 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:44:10.837 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:44:10.861 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:44:10.866 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:10 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T20:44:10.866 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:10 smithi018 ceph-mon[184144]: osdmap e496: 8 total, 8 up, 8 in 2023-09-23T20:44:10.867 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:10 smithi018 ceph-mon[184144]: failed to encode map e496 with expected crc 2023-09-23T20:44:10.867 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:10 smithi018 ceph-mon[184144]: failed to encode map e496 with expected crc 2023-09-23T20:44:10.867 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:10 smithi018 ceph-mon[184144]: failed to encode map e496 with expected crc 2023-09-23T20:44:10.867 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:10 smithi018 ceph-mon[184144]: failed to encode map e496 with expected crc 2023-09-23T20:44:10.867 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:10 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T20:44:10.867 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:10 smithi018 ceph-mon[179647]: osdmap e496: 8 total, 8 up, 8 in 2023-09-23T20:44:10.867 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:10 smithi018 ceph-mon[179647]: failed to encode map e496 with expected crc 2023-09-23T20:44:10.867 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:10 smithi018 ceph-mon[179647]: failed to encode map e496 with expected crc 2023-09-23T20:44:10.867 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:10 smithi018 ceph-mon[179647]: failed to encode map e496 with expected crc 2023-09-23T20:44:10.867 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:10 smithi018 ceph-mon[179647]: failed to encode map e496 with expected crc 2023-09-23T20:44:10.873 INFO:teuthology.orchestra.run.smithi018.stdout:ERROR: (22) Invalid argument 2023-09-23T20:44:10.873 INFO:teuthology.orchestra.run.smithi018.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:44:10.886 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:10.908 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:44:10.918 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:44:10.921 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:10.921 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:44:11.022 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:44:11.051 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:11.407 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T20:44:11.435 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:44:11.438 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:11.439 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:44:11.480 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:44:11.540 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:44:11.573 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:11.609 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:44:11.612 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:11.612 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:44:11.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:11 smithi018 ceph-mon[184144]: pgmap v1254: 119 pgs: 2 peering, 1 active+remapped+backfilling, 116 active+clean; 2.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 580 op/s; 216/140262 objects misplaced (0.154%); 11 MiB/s, 176 objects/s recovering 2023-09-23T20:44:11.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:11 smithi018 ceph-mon[184144]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T20:44:11.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:11 smithi018 ceph-mon[184144]: failed to encode map e497 with expected crc 2023-09-23T20:44:11.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:11 smithi018 ceph-mon[184144]: osdmap e497: 8 total, 8 up, 8 in 2023-09-23T20:44:11.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:11 smithi018 ceph-mon[184144]: failed to encode map e497 with expected crc 2023-09-23T20:44:11.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:11 smithi018 ceph-mon[184144]: failed to encode map e497 with expected crc 2023-09-23T20:44:11.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:11 smithi018 ceph-mon[184144]: failed to encode map e497 with expected crc 2023-09-23T20:44:11.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:11 smithi018 ceph-mon[184144]: failed to encode map e497 with expected crc 2023-09-23T20:44:11.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:11 smithi018 ceph-mon[179647]: pgmap v1254: 119 pgs: 2 peering, 1 active+remapped+backfilling, 116 active+clean; 2.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 580 op/s; 216/140262 objects misplaced (0.154%); 11 MiB/s, 176 objects/s recovering 2023-09-23T20:44:11.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:11 smithi018 ceph-mon[179647]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T20:44:11.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:11 smithi018 ceph-mon[179647]: failed to encode map e497 with expected crc 2023-09-23T20:44:11.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:11 smithi018 ceph-mon[179647]: osdmap e497: 8 total, 8 up, 8 in 2023-09-23T20:44:11.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:11 smithi018 ceph-mon[179647]: failed to encode map e497 with expected crc 2023-09-23T20:44:11.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:11 smithi018 ceph-mon[179647]: failed to encode map e497 with expected crc 2023-09-23T20:44:11.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:11 smithi018 ceph-mon[179647]: failed to encode map e497 with expected crc 2023-09-23T20:44:11.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:11 smithi018 ceph-mon[179647]: failed to encode map e497 with expected crc 2023-09-23T20:44:11.713 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:44:11.740 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:11.773 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:44:11.776 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:11.777 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:44:11.873 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:44:11.878 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:44:11.904 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:11 smithi149 ceph-mon[170394]: pgmap v1254: 119 pgs: 2 peering, 1 active+remapped+backfilling, 116 active+clean; 2.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 580 op/s; 216/140262 objects misplaced (0.154%); 11 MiB/s, 176 objects/s recovering 2023-09-23T20:44:11.904 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:11 smithi149 ceph-mon[170394]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T20:44:11.904 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:11 smithi149 ceph-mon[170394]: failed to encode map e497 with expected crc 2023-09-23T20:44:11.904 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:11 smithi149 ceph-mon[170394]: osdmap e497: 8 total, 8 up, 8 in 2023-09-23T20:44:11.905 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:11 smithi149 ceph-mon[170394]: failed to encode map e497 with expected crc 2023-09-23T20:44:11.905 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:11 smithi149 ceph-mon[170394]: failed to encode map e497 with expected crc 2023-09-23T20:44:11.905 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:11 smithi149 ceph-mon[170394]: failed to encode map e497 with expected crc 2023-09-23T20:44:11.905 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:11 smithi149 ceph-mon[170394]: failed to encode map e497 with expected crc 2023-09-23T20:44:11.910 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:11.961 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:44:11.967 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:11.967 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:44:12.068 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:44:12.190 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:12.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:12 smithi018 ceph-mon[184144]: osdmap e498: 8 total, 8 up, 8 in 2023-09-23T20:44:12.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:12 smithi018 ceph-mon[184144]: failed to encode map e498 with expected crc 2023-09-23T20:44:12.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:12 smithi018 ceph-mon[184144]: 9.d starting backfill to osd.1 from (461'3179,482'9018] MIN to 495'11447 2023-09-23T20:44:12.645 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:12 smithi018 ceph-mon[179647]: osdmap e498: 8 total, 8 up, 8 in 2023-09-23T20:44:12.645 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:12 smithi018 ceph-mon[179647]: failed to encode map e498 with expected crc 2023-09-23T20:44:12.645 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:12 smithi018 ceph-mon[179647]: 9.d starting backfill to osd.1 from (461'3179,482'9018] MIN to 495'11447 2023-09-23T20:44:12.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:12 smithi149 ceph-mon[170394]: osdmap e498: 8 total, 8 up, 8 in 2023-09-23T20:44:12.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:12 smithi149 ceph-mon[170394]: failed to encode map e498 with expected crc 2023-09-23T20:44:12.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:12 smithi149 ceph-mon[170394]: 9.d starting backfill to osd.1 from (461'3179,482'9018] MIN to 495'11447 2023-09-23T20:44:12.985 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:44:12.988 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:12.989 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:44:13.089 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:44:13.171 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:13.267 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:44:13.274 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:13.275 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:44:13.375 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:44:13.446 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:44:13.488 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:13.638 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:44:13.644 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:13.644 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:44:13.745 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:44:13.804 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:13 smithi018 ceph-mon[184144]: failed to encode map e498 with expected crc 2023-09-23T20:44:13.804 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:13 smithi018 ceph-mon[184144]: pgmap v1257: 119 pgs: 2 peering, 117 active+clean; 2.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 466 op/s; 2.8 MiB/s, 44 objects/s recovering 2023-09-23T20:44:13.808 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:13 smithi018 ceph-mon[179647]: failed to encode map e498 with expected crc 2023-09-23T20:44:13.808 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:13 smithi018 ceph-mon[179647]: pgmap v1257: 119 pgs: 2 peering, 117 active+clean; 2.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 466 op/s; 2.8 MiB/s, 44 objects/s recovering 2023-09-23T20:44:13.822 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:13.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:13 smithi149 ceph-mon[170394]: failed to encode map e498 with expected crc 2023-09-23T20:44:13.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:13 smithi149 ceph-mon[170394]: pgmap v1257: 119 pgs: 2 peering, 117 active+clean; 2.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 466 op/s; 2.8 MiB/s, 44 objects/s recovering 2023-09-23T20:44:14.183 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:44:14.194 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:14.195 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:44:14.296 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:44:14.372 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:14.601 INFO:teuthology.orchestra.run.smithi018.stderr:reweighted osd.7 to 1 (10000) 2023-09-23T20:44:14.603 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:44:14.609 INFO:teuthology.orchestra.run.smithi018.stderr:noscrub is unset 2023-09-23T20:44:14.611 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:14.611 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:44:14.712 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:44:14.775 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:44:14.796 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:14.812 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:44:14.844 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:14 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1346800781' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T20:44:14.844 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:14 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T20:44:14.845 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:14 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3889229990' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:44:14.845 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:14 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:44:14.845 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:14 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:44:14.845 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:44:14.845 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:14 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1346800781' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T20:44:14.845 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:14 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T20:44:14.845 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:14 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3889229990' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:44:14.845 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:14 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:44:14.846 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:14 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:44:14.870 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:44:14.897 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:44:14.927 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:44:14.949 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:44:14.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:14 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1346800781' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T20:44:14.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:14 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T20:44:14.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:14 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3889229990' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:44:14.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:14 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:44:14.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:14 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:44:14.975 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:44:14.977 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:44:14.977 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:44:14.977 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:44:14.977 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:44:14.977 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:44:14.977 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:44:14.978 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:44:14.978 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:44:15.112 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:44:15.115 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:44:14 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:44:14] "GET /metrics HTTP/1.1" 200 36584 "" "Prometheus/2.43.0" 2023-09-23T20:44:15.117 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:15.118 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:44:15.219 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:44:15.248 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:15.290 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:44:15.296 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:15.297 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:44:15.398 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:44:15.423 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:15.458 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:44:15.461 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:15.462 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:44:15.562 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:44:15.595 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:15.667 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd reweight 3 1 2023-09-23T20:44:15.692 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:44:15.695 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:15.695 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:44:15.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:15 smithi149 ceph-mon[170394]: pgmap v1258: 119 pgs: 1 active+remapped+backfilling, 2 peering, 116 active+clean; 2.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 517 op/s; 2749/134916 objects misplaced (2.038%); 2.6 MiB/s, 41 objects/s recovering 2023-09-23T20:44:15.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:15 smithi149 ceph-mon[170394]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:44:15.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:15 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T20:44:15.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:15 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3889229990' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:44:15.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:15 smithi149 ceph-mon[170394]: osdmap e499: 8 total, 8 up, 8 in 2023-09-23T20:44:15.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:15 smithi149 ceph-mon[170394]: failed to encode map e499 with expected crc 2023-09-23T20:44:15.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:15 smithi149 ceph-mon[170394]: failed to encode map e499 with expected crc 2023-09-23T20:44:15.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:15 smithi149 ceph-mon[170394]: failed to encode map e499 with expected crc 2023-09-23T20:44:15.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:15 smithi149 ceph-mon[170394]: failed to encode map e499 with expected crc 2023-09-23T20:44:15.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:15 smithi149 ceph-mon[170394]: 1.1b scrub starts 2023-09-23T20:44:15.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:15 smithi149 ceph-mon[170394]: 1.1b scrub ok 2023-09-23T20:44:15.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:15 smithi149 ceph-mon[170394]: 1.17 scrub starts 2023-09-23T20:44:15.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:15 smithi149 ceph-mon[170394]: 1.17 scrub ok 2023-09-23T20:44:15.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:15 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3644390898' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:44:15.796 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:44:15.823 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:15.858 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:44:15.860 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:15.861 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:44:15.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:15 smithi018 ceph-mon[179647]: pgmap v1258: 119 pgs: 1 active+remapped+backfilling, 2 peering, 116 active+clean; 2.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 517 op/s; 2749/134916 objects misplaced (2.038%); 2.6 MiB/s, 41 objects/s recovering 2023-09-23T20:44:15.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:15 smithi018 ceph-mon[179647]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:44:15.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:15 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T20:44:15.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:15 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3889229990' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:44:15.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:15 smithi018 ceph-mon[179647]: osdmap e499: 8 total, 8 up, 8 in 2023-09-23T20:44:15.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:15 smithi018 ceph-mon[179647]: failed to encode map e499 with expected crc 2023-09-23T20:44:15.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:15 smithi018 ceph-mon[179647]: failed to encode map e499 with expected crc 2023-09-23T20:44:15.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:15 smithi018 ceph-mon[179647]: failed to encode map e499 with expected crc 2023-09-23T20:44:15.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:15 smithi018 ceph-mon[179647]: failed to encode map e499 with expected crc 2023-09-23T20:44:15.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:15 smithi018 ceph-mon[179647]: 1.1b scrub starts 2023-09-23T20:44:15.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:15 smithi018 ceph-mon[179647]: 1.1b scrub ok 2023-09-23T20:44:15.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:15 smithi018 ceph-mon[179647]: 1.17 scrub starts 2023-09-23T20:44:15.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:15 smithi018 ceph-mon[179647]: 1.17 scrub ok 2023-09-23T20:44:15.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:15 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3644390898' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:44:15.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:15 smithi018 ceph-mon[184144]: pgmap v1258: 119 pgs: 1 active+remapped+backfilling, 2 peering, 116 active+clean; 2.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 517 op/s; 2749/134916 objects misplaced (2.038%); 2.6 MiB/s, 41 objects/s recovering 2023-09-23T20:44:15.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:15 smithi018 ceph-mon[184144]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:44:15.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:15 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T20:44:15.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:15 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3889229990' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:44:15.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:15 smithi018 ceph-mon[184144]: osdmap e499: 8 total, 8 up, 8 in 2023-09-23T20:44:15.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:15 smithi018 ceph-mon[184144]: failed to encode map e499 with expected crc 2023-09-23T20:44:15.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:15 smithi018 ceph-mon[184144]: failed to encode map e499 with expected crc 2023-09-23T20:44:15.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:15 smithi018 ceph-mon[184144]: failed to encode map e499 with expected crc 2023-09-23T20:44:15.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:15 smithi018 ceph-mon[184144]: failed to encode map e499 with expected crc 2023-09-23T20:44:15.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:15 smithi018 ceph-mon[184144]: 1.1b scrub starts 2023-09-23T20:44:15.894 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:15 smithi018 ceph-mon[184144]: 1.1b scrub ok 2023-09-23T20:44:15.894 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:15 smithi018 ceph-mon[184144]: 1.17 scrub starts 2023-09-23T20:44:15.894 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:15 smithi018 ceph-mon[184144]: 1.17 scrub ok 2023-09-23T20:44:15.894 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:15 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3644390898' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:44:15.961 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:44:15.990 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:16.024 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:44:16.027 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:16.028 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:44:16.030 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2023-09-23T20:44:16.030 INFO:teuthology.orchestra.run.smithi018.stdout: "size": 20, 2023-09-23T20:44:16.030 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 600, 2023-09-23T20:44:16.030 INFO:teuthology.orchestra.run.smithi018.stdout: "ops": [ 2023-09-23T20:44:16.031 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.031 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35994.0:51024 8.f 8:f455c68e:::benchmark_data_smithi018_341490_object51023:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e374)", 2023-09-23T20:44:16.031 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:36:42.547256+0000", 2023-09-23T20:44:16.031 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 453.47840754100002, 2023-09-23T20:44:16.031 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.15709083700000001, 2023-09-23T20:44:16.031 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:44:16.031 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:44:16.031 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:44:16.032 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35994", 2023-09-23T20:44:16.032 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:44:16.032 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 51024 2023-09-23T20:44:16.032 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.032 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:44:16.032 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.032 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:44:16.032 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:36:42.547256+0000", 2023-09-23T20:44:16.032 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:44:16.033 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.033 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.033 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:44:16.033 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:36:42.547256+0000", 2023-09-23T20:44:16.033 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:44:16.033 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.033 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.033 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:44:16.034 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:36:42.547255+0000", 2023-09-23T20:44:16.034 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:44:16.034 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.034 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.034 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:44:16.034 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:36:42.547278+0000", 2023-09-23T20:44:16.034 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.3464000000000001e-05 2023-09-23T20:44:16.034 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.035 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.035 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:44:16.035 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:36:42.547298+0000", 2023-09-23T20:44:16.035 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.9731000000000001e-05 2023-09-23T20:44:16.035 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.035 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.035 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:44:16.035 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:36:42.547308+0000", 2023-09-23T20:44:16.036 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.5759999999999995e-06 2023-09-23T20:44:16.036 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.036 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.036 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:44:16.036 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:36:42.547369+0000", 2023-09-23T20:44:16.036 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.1072000000000001e-05 2023-09-23T20:44:16.036 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.036 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.036 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:44:16.037 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:36:42.547539+0000", 2023-09-23T20:44:16.037 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00017053399999999999 2023-09-23T20:44:16.037 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.037 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.037 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:44:16.037 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:36:42.547677+0000", 2023-09-23T20:44:16.037 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00013801500000000001 2023-09-23T20:44:16.037 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.037 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.038 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:44:16.038 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:36:42.549518+0000", 2023-09-23T20:44:16.038 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.001841028 2023-09-23T20:44:16.038 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.038 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.038 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:16.038 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:36:42.550140+0000", 2023-09-23T20:44:16.038 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00062157399999999995 2023-09-23T20:44:16.039 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.039 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.039 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:16.039 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:36:42.704243+0000", 2023-09-23T20:44:16.039 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.154103238 2023-09-23T20:44:16.039 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.039 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.039 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:44:16.039 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:36:42.704307+0000", 2023-09-23T20:44:16.040 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.4120000000000003e-05 2023-09-23T20:44:16.040 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.040 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.040 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:44:16.040 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:36:42.704347+0000", 2023-09-23T20:44:16.040 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.0099999999999999e-05 2023-09-23T20:44:16.040 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:44:16.040 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:44:16.040 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:44:16.041 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.041 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.041 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35994.0:51038 8.f 8:f070768c:::benchmark_data_smithi018_341490_object51037:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e374)", 2023-09-23T20:44:16.041 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:36:42.557492+0000", 2023-09-23T20:44:16.041 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 453.468171993, 2023-09-23T20:44:16.041 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.14697069099999999, 2023-09-23T20:44:16.041 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:44:16.041 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:44:16.041 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:44:16.042 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35994", 2023-09-23T20:44:16.042 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:44:16.042 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 51038 2023-09-23T20:44:16.042 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.042 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:44:16.042 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.042 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:44:16.042 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:36:42.557492+0000", 2023-09-23T20:44:16.043 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:44:16.043 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.043 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.043 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:44:16.043 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:36:42.557492+0000", 2023-09-23T20:44:16.043 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:44:16.043 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.043 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.043 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:44:16.043 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:36:42.557491+0000", 2023-09-23T20:44:16.044 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.999999 2023-09-23T20:44:16.044 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.044 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.044 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:44:16.044 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:36:42.557508+0000", 2023-09-23T20:44:16.044 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.751e-05 2023-09-23T20:44:16.044 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.044 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.044 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:44:16.045 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:36:42.557523+0000", 2023-09-23T20:44:16.045 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.4435e-05 2023-09-23T20:44:16.045 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.045 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.045 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:44:16.045 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:36:42.557531+0000", 2023-09-23T20:44:16.045 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.4849999999999992e-06 2023-09-23T20:44:16.045 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.045 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.046 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:44:16.046 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:36:42.557591+0000", 2023-09-23T20:44:16.046 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.9945999999999998e-05 2023-09-23T20:44:16.046 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.046 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.046 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:44:16.046 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:36:42.557779+0000", 2023-09-23T20:44:16.047 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00018756799999999999 2023-09-23T20:44:16.047 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.047 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.047 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:44:16.047 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:36:42.557983+0000", 2023-09-23T20:44:16.047 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00020390100000000001 2023-09-23T20:44:16.047 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.047 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.047 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:44:16.048 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:36:42.559708+0000", 2023-09-23T20:44:16.048 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0017257520000000001 2023-09-23T20:44:16.048 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.048 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.048 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:16.048 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:36:42.559949+0000", 2023-09-23T20:44:16.048 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000240809 2023-09-23T20:44:16.048 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.049 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.049 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:16.049 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:36:42.704388+0000", 2023-09-23T20:44:16.049 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.14443837900000001 2023-09-23T20:44:16.049 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.049 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.049 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:44:16.049 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:36:42.704427+0000", 2023-09-23T20:44:16.049 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.9910000000000002e-05 2023-09-23T20:44:16.050 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.050 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.050 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:44:16.050 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:36:42.704463+0000", 2023-09-23T20:44:16.050 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.5175000000000001e-05 2023-09-23T20:44:16.050 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:44:16.050 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:44:16.050 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:44:16.051 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.051 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.051 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35994.0:105795 8.f 8:fbd22ea9:::benchmark_data_smithi018_341490_object16472:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e409)", 2023-09-23T20:44:16.051 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:38:26.417976+0000", 2023-09-23T20:44:16.051 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 349.60768760500002, 2023-09-23T20:44:16.051 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.35500995099999999, 2023-09-23T20:44:16.051 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:44:16.051 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:44:16.051 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:44:16.052 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35994", 2023-09-23T20:44:16.052 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:44:16.052 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 105795 2023-09-23T20:44:16.052 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.052 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:44:16.052 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.052 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:44:16.052 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.417976+0000", 2023-09-23T20:44:16.052 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:44:16.053 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.053 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.053 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:44:16.053 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.417976+0000", 2023-09-23T20:44:16.053 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:44:16.053 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.053 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.053 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:44:16.053 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.417974+0000", 2023-09-23T20:44:16.054 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:44:16.054 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.054 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.054 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:44:16.054 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.417985+0000", 2023-09-23T20:44:16.054 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0900000000000001e-05 2023-09-23T20:44:16.054 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.054 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.055 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:44:16.055 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.418013+0000", 2023-09-23T20:44:16.055 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.7627000000000002e-05 2023-09-23T20:44:16.055 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.055 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.055 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:44:16.055 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.418025+0000", 2023-09-23T20:44:16.055 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2130000000000001e-05 2023-09-23T20:44:16.055 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.056 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.056 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:44:16.056 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.418124+0000", 2023-09-23T20:44:16.056 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.8815999999999997e-05 2023-09-23T20:44:16.056 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.056 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.056 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:44:16.056 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.418247+0000", 2023-09-23T20:44:16.057 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000123249 2023-09-23T20:44:16.057 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.057 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.057 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:44:16.057 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.418326+0000", 2023-09-23T20:44:16.057 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.9097000000000005e-05 2023-09-23T20:44:16.057 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.057 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.057 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:44:16.058 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.423030+0000", 2023-09-23T20:44:16.058 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.004704387 2023-09-23T20:44:16.058 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.058 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.058 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:16.058 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.517001+0000", 2023-09-23T20:44:16.058 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.093970674000000004 2023-09-23T20:44:16.059 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.059 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.059 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:16.059 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.772873+0000", 2023-09-23T20:44:16.059 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.25587215499999999 2023-09-23T20:44:16.059 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.059 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.059 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:44:16.059 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.772933+0000", 2023-09-23T20:44:16.059 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.0099999999999997e-05 2023-09-23T20:44:16.060 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.060 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.060 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:44:16.060 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.772986+0000", 2023-09-23T20:44:16.060 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.2918999999999999e-05 2023-09-23T20:44:16.060 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:44:16.060 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:44:16.060 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:44:16.060 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.060 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.060 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35994.0:167877 8.f 8:f4c71872:::benchmark_data_smithi018_341490_object78554:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e444)", 2023-09-23T20:44:16.061 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:40:25.311045+0000", 2023-09-23T20:44:16.061 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 230.71461855999999, 2023-09-23T20:44:16.061 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.077145796000000003, 2023-09-23T20:44:16.061 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:44:16.061 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:44:16.061 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:44:16.061 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35994", 2023-09-23T20:44:16.061 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:44:16.061 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 167877 2023-09-23T20:44:16.061 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.061 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:44:16.062 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.062 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:44:16.062 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:40:25.311045+0000", 2023-09-23T20:44:16.062 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:44:16.062 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.062 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.062 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:44:16.062 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:40:25.311045+0000", 2023-09-23T20:44:16.062 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:44:16.062 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.062 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.062 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:44:16.063 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:40:25.311043+0000", 2023-09-23T20:44:16.063 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:44:16.063 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.063 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.063 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:44:16.063 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:40:25.311052+0000", 2023-09-23T20:44:16.063 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.5699999999999993e-06 2023-09-23T20:44:16.063 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.063 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.063 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:44:16.063 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:40:25.311072+0000", 2023-09-23T20:44:16.063 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.9944000000000001e-05 2023-09-23T20:44:16.064 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.064 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.064 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:44:16.064 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:40:25.311082+0000", 2023-09-23T20:44:16.064 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.9219999999999992e-06 2023-09-23T20:44:16.064 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.064 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.064 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:44:16.064 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:40:25.313521+0000", 2023-09-23T20:44:16.064 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0024389569999999998 2023-09-23T20:44:16.064 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.064 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.065 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:44:16.065 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:40:25.313663+0000", 2023-09-23T20:44:16.065 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00014258900000000001 2023-09-23T20:44:16.065 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.065 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.065 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:44:16.065 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:40:25.313784+0000", 2023-09-23T20:44:16.065 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00012035 2023-09-23T20:44:16.065 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.065 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.065 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:16.066 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:40:25.352458+0000", 2023-09-23T20:44:16.066 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.038674279999999998 2023-09-23T20:44:16.066 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.066 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.066 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:44:16.066 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:40:25.361576+0000", 2023-09-23T20:44:16.066 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0091178769999999999 2023-09-23T20:44:16.066 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.066 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.066 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:16.066 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:40:25.388099+0000", 2023-09-23T20:44:16.066 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.026522761999999998 2023-09-23T20:44:16.067 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.067 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.067 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:44:16.067 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:40:25.388145+0000", 2023-09-23T20:44:16.067 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.6424999999999997e-05 2023-09-23T20:44:16.067 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.067 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.067 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:44:16.067 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:40:25.388191+0000", 2023-09-23T20:44:16.067 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.6063999999999998e-05 2023-09-23T20:44:16.067 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:44:16.067 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:44:16.068 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:44:16.068 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.068 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.068 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.36450.0:141027 9.6 9:6dcbc3cc:::benchmark_data_smithi018_381516_object27193:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e481)", 2023-09-23T20:44:16.068 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:43:15.747787+0000", 2023-09-23T20:44:16.068 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 60.277877154000002, 2023-09-23T20:44:16.068 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.040264882, 2023-09-23T20:44:16.068 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:44:16.068 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:44:16.068 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:44:16.068 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.36450", 2023-09-23T20:44:16.069 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:44:16.069 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 141027 2023-09-23T20:44:16.069 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.069 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:44:16.069 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.069 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:44:16.069 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.747787+0000", 2023-09-23T20:44:16.069 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:44:16.069 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.069 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.069 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:44:16.069 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.747787+0000", 2023-09-23T20:44:16.069 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:44:16.070 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.070 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.070 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:44:16.070 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.747785+0000", 2023-09-23T20:44:16.070 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:44:16.070 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.070 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.070 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:44:16.070 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.747801+0000", 2023-09-23T20:44:16.070 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.5854000000000001e-05 2023-09-23T20:44:16.070 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.071 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.071 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:44:16.071 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.747818+0000", 2023-09-23T20:44:16.071 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.7499e-05 2023-09-23T20:44:16.071 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.071 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.071 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:44:16.071 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.747829+0000", 2023-09-23T20:44:16.071 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0508e-05 2023-09-23T20:44:16.071 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.071 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.071 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:44:16.072 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.747912+0000", 2023-09-23T20:44:16.072 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.3052000000000004e-05 2023-09-23T20:44:16.072 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.072 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.072 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:44:16.072 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.747926+0000", 2023-09-23T20:44:16.072 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3577000000000001e-05 2023-09-23T20:44:16.072 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.072 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.072 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:44:16.072 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:16.785351+0000", 2023-09-23T20:44:16.072 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.037425635 2023-09-23T20:44:16.073 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.073 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.073 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:44:16.073 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:16.788052+0000", 2023-09-23T20:44:16.073 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0027005010000000001 2023-09-23T20:44:16.073 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:44:16.073 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:44:16.073 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:44:16.073 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.073 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.073 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.36450.0:141056 9.6 9:6159416c:::benchmark_data_smithi018_381516_object27222:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e481)", 2023-09-23T20:44:16.074 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:43:15.776179+0000", 2023-09-23T20:44:16.074 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 60.249485126000003, 2023-09-23T20:44:16.074 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.023410698, 2023-09-23T20:44:16.074 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:44:16.074 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:44:16.074 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:44:16.074 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.36450", 2023-09-23T20:44:16.074 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:44:16.074 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 141056 2023-09-23T20:44:16.074 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.074 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:44:16.075 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.075 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:44:16.075 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.776179+0000", 2023-09-23T20:44:16.075 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:44:16.075 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.075 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.075 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:44:16.075 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.776179+0000", 2023-09-23T20:44:16.075 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:44:16.075 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.075 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.076 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:44:16.076 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.776177+0000", 2023-09-23T20:44:16.076 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:44:16.076 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.076 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.076 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:44:16.076 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.776187+0000", 2023-09-23T20:44:16.076 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0278000000000001e-05 2023-09-23T20:44:16.076 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.076 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.076 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:44:16.076 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.776211+0000", 2023-09-23T20:44:16.077 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.3674000000000001e-05 2023-09-23T20:44:16.077 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.077 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.077 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:44:16.077 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.776222+0000", 2023-09-23T20:44:16.077 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1599e-05 2023-09-23T20:44:16.077 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.077 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.077 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:44:16.077 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.776301+0000", 2023-09-23T20:44:16.077 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.8295999999999997e-05 2023-09-23T20:44:16.077 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.077 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.078 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:44:16.078 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.776315+0000", 2023-09-23T20:44:16.078 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3906999999999999e-05 2023-09-23T20:44:16.078 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.078 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.078 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:44:16.078 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:16.788351+0000", 2023-09-23T20:44:16.078 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0120362730000001 2023-09-23T20:44:16.078 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.078 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.078 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:44:16.078 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:16.789277+0000", 2023-09-23T20:44:16.079 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00092556400000000003 2023-09-23T20:44:16.079 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.079 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.079 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,5", 2023-09-23T20:44:16.079 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:16.789391+0000", 2023-09-23T20:44:16.079 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000114228 2023-09-23T20:44:16.079 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.079 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.079 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:16.079 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:16.797045+0000", 2023-09-23T20:44:16.079 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0076541880000000001 2023-09-23T20:44:16.079 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.080 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.080 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:44:16.080 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:16.798259+0000", 2023-09-23T20:44:16.080 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.001214033 2023-09-23T20:44:16.080 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.080 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.080 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:16.080 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:16.799494+0000", 2023-09-23T20:44:16.080 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0012353520000000001 2023-09-23T20:44:16.080 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.080 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.080 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:44:16.081 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:16.799549+0000", 2023-09-23T20:44:16.081 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.4401999999999999e-05 2023-09-23T20:44:16.081 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.081 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.081 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:44:16.081 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:16.799589+0000", 2023-09-23T20:44:16.081 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.0774e-05 2023-09-23T20:44:16.081 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:44:16.081 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:44:16.081 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:44:16.081 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.082 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.082 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.36450.0:141066 9.6 9:6eed4220:::benchmark_data_smithi018_381516_object27232:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e481)", 2023-09-23T20:44:16.082 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:43:15.809374+0000", 2023-09-23T20:44:16.082 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 60.216290262000001, 2023-09-23T20:44:16.082 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0041945379999999, 2023-09-23T20:44:16.082 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:44:16.082 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:44:16.082 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:44:16.082 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.36450", 2023-09-23T20:44:16.082 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:44:16.082 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 141066 2023-09-23T20:44:16.082 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.083 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:44:16.083 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.083 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:44:16.083 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.809374+0000", 2023-09-23T20:44:16.083 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:44:16.083 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.083 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.083 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:44:16.083 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.809374+0000", 2023-09-23T20:44:16.083 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:44:16.083 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.084 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.084 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:44:16.084 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.809372+0000", 2023-09-23T20:44:16.084 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:44:16.084 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.084 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.084 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:44:16.084 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.809383+0000", 2023-09-23T20:44:16.084 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1195999999999999e-05 2023-09-23T20:44:16.084 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.084 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.084 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:44:16.085 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.809407+0000", 2023-09-23T20:44:16.085 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.3937000000000001e-05 2023-09-23T20:44:16.085 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.085 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.085 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:44:16.085 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.809419+0000", 2023-09-23T20:44:16.085 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2525e-05 2023-09-23T20:44:16.085 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.085 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.085 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:44:16.085 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.809490+0000", 2023-09-23T20:44:16.085 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.1048000000000001e-05 2023-09-23T20:44:16.086 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.086 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.086 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:44:16.086 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.809508+0000", 2023-09-23T20:44:16.086 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.7263000000000001e-05 2023-09-23T20:44:16.086 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.086 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.086 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:44:16.086 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:16.793445+0000", 2023-09-23T20:44:16.086 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.98393740500000004 2023-09-23T20:44:16.086 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.087 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.087 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:44:16.087 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:16.793568+0000", 2023-09-23T20:44:16.087 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000123492 2023-09-23T20:44:16.087 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.087 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.087 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,5", 2023-09-23T20:44:16.087 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:16.793699+0000", 2023-09-23T20:44:16.087 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000130455 2023-09-23T20:44:16.087 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.087 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.087 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:44:16.088 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:16.798264+0000", 2023-09-23T20:44:16.088 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0045648709999999999 2023-09-23T20:44:16.088 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.088 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.088 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:16.088 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:16.800278+0000", 2023-09-23T20:44:16.088 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0020140480000000001 2023-09-23T20:44:16.088 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.088 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.088 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:16.088 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:16.813461+0000", 2023-09-23T20:44:16.089 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.01318304 2023-09-23T20:44:16.089 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.089 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.089 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:44:16.089 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:16.813519+0000", 2023-09-23T20:44:16.089 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.7908000000000002e-05 2023-09-23T20:44:16.089 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.089 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.089 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:44:16.089 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:16.813568+0000", 2023-09-23T20:44:16.089 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.9450000000000003e-05 2023-09-23T20:44:16.090 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:44:16.090 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:44:16.090 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:44:16.090 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.090 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.090 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.36450.0:155725 9.6 9:62f578fb:::benchmark_data_smithi018_381516_object41891:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e485)", 2023-09-23T20:44:16.090 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:43:34.624102+0000", 2023-09-23T20:44:16.090 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 41.401562358, 2023-09-23T20:44:16.090 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.066191748999999994, 2023-09-23T20:44:16.090 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:44:16.090 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:44:16.090 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:44:16.091 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.36450", 2023-09-23T20:44:16.091 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:44:16.091 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 155725 2023-09-23T20:44:16.091 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.091 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:44:16.091 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.091 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:44:16.091 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:34.624102+0000", 2023-09-23T20:44:16.091 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:44:16.091 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.091 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.092 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:44:16.092 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:34.624102+0000", 2023-09-23T20:44:16.092 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:44:16.092 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.092 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.092 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:44:16.092 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:34.624101+0000", 2023-09-23T20:44:16.092 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.999999 2023-09-23T20:44:16.092 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.092 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.092 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:44:16.092 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:34.624106+0000", 2023-09-23T20:44:16.093 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.5659999999999997e-06 2023-09-23T20:44:16.093 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.093 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.093 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:44:16.093 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:34.624117+0000", 2023-09-23T20:44:16.093 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0631e-05 2023-09-23T20:44:16.093 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.093 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.093 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:44:16.093 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:34.624125+0000", 2023-09-23T20:44:16.093 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.2900000000000002e-06 2023-09-23T20:44:16.093 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.094 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.094 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:44:16.094 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:34.631532+0000", 2023-09-23T20:44:16.094 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0074064140000000001 2023-09-23T20:44:16.094 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.094 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.094 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:44:16.094 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:34.631674+0000", 2023-09-23T20:44:16.094 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00014208100000000001 2023-09-23T20:44:16.094 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.094 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.095 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,5", 2023-09-23T20:44:16.095 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:34.631806+0000", 2023-09-23T20:44:16.095 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00013258599999999999 2023-09-23T20:44:16.095 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.095 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.095 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:44:16.095 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:34.636556+0000", 2023-09-23T20:44:16.095 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0047501610000000001 2023-09-23T20:44:16.095 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.095 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.095 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:16.095 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:34.637060+0000", 2023-09-23T20:44:16.096 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00050393200000000001 2023-09-23T20:44:16.096 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.096 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.096 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:16.096 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:34.690202+0000", 2023-09-23T20:44:16.096 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.053141314000000002 2023-09-23T20:44:16.096 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.096 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.096 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:44:16.096 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:34.690250+0000", 2023-09-23T20:44:16.096 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.8643999999999997e-05 2023-09-23T20:44:16.096 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.097 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.097 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:44:16.097 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:34.690293+0000", 2023-09-23T20:44:16.097 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.2967999999999999e-05 2023-09-23T20:44:16.097 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:44:16.097 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:44:16.097 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:44:16.097 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.097 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.097 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.36450.0:159450 9.6 9:6ea5db69:::benchmark_data_smithi018_381516_object45616:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e486)", 2023-09-23T20:44:16.097 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:43:39.185200+0000", 2023-09-23T20:44:16.097 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 36.8404636, 2023-09-23T20:44:16.098 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.122052284, 2023-09-23T20:44:16.098 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:44:16.098 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:44:16.098 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:44:16.098 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.36450", 2023-09-23T20:44:16.098 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:44:16.098 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 159450 2023-09-23T20:44:16.098 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.098 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:44:16.098 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.098 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:44:16.099 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:39.185200+0000", 2023-09-23T20:44:16.099 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:44:16.099 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.099 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.099 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:44:16.099 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:39.185200+0000", 2023-09-23T20:44:16.099 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:44:16.099 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.099 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.099 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:44:16.099 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:39.185198+0000", 2023-09-23T20:44:16.099 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:44:16.100 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.100 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.100 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:44:16.100 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:39.185208+0000", 2023-09-23T20:44:16.100 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0006999999999999e-05 2023-09-23T20:44:16.100 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.100 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.100 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:44:16.100 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:39.185227+0000", 2023-09-23T20:44:16.100 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.9023999999999999e-05 2023-09-23T20:44:16.100 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.100 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.101 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:44:16.101 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:39.185237+0000", 2023-09-23T20:44:16.101 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.9350000000000006e-06 2023-09-23T20:44:16.101 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.101 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.101 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:44:16.101 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:39.185301+0000", 2023-09-23T20:44:16.101 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.4024999999999998e-05 2023-09-23T20:44:16.101 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.101 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.101 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:44:16.102 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:39.185452+0000", 2023-09-23T20:44:16.102 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00015056299999999999 2023-09-23T20:44:16.102 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.102 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.102 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,5", 2023-09-23T20:44:16.102 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:39.185569+0000", 2023-09-23T20:44:16.102 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000117163 2023-09-23T20:44:16.102 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.102 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.102 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:44:16.102 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:39.192821+0000", 2023-09-23T20:44:16.102 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0072523989999999997 2023-09-23T20:44:16.103 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.103 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.103 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:16.103 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:39.197225+0000", 2023-09-23T20:44:16.103 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0044031419999999996 2023-09-23T20:44:16.103 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.103 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.103 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:16.103 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:39.307164+0000", 2023-09-23T20:44:16.103 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.109939202 2023-09-23T20:44:16.103 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.103 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.104 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:44:16.104 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:39.307215+0000", 2023-09-23T20:44:16.104 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.1008999999999999e-05 2023-09-23T20:44:16.104 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.104 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.104 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:44:16.104 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:39.307253+0000", 2023-09-23T20:44:16.104 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.7781000000000003e-05 2023-09-23T20:44:16.104 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:44:16.104 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:44:16.104 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:44:16.104 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.105 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.105 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.36450.0:159452 9.6 9:63f08544:::benchmark_data_smithi018_381516_object45618:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e486)", 2023-09-23T20:44:16.105 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:43:39.189307+0000", 2023-09-23T20:44:16.105 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 36.836356481999999, 2023-09-23T20:44:16.105 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.11804816999999999, 2023-09-23T20:44:16.105 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:44:16.105 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:44:16.105 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:44:16.105 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.36450", 2023-09-23T20:44:16.105 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:44:16.105 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 159452 2023-09-23T20:44:16.105 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.106 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:44:16.106 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.106 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:44:16.106 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:39.189307+0000", 2023-09-23T20:44:16.106 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:44:16.106 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.106 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.106 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:44:16.106 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:39.189307+0000", 2023-09-23T20:44:16.106 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:44:16.106 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.106 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.107 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:44:16.107 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:39.189306+0000", 2023-09-23T20:44:16.107 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:44:16.107 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.107 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.107 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:44:16.107 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:39.189313+0000", 2023-09-23T20:44:16.107 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.6760000000000001e-06 2023-09-23T20:44:16.107 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.107 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.107 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:44:16.108 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:39.189326+0000", 2023-09-23T20:44:16.108 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3189e-05 2023-09-23T20:44:16.108 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.108 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.108 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:44:16.108 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:39.189335+0000", 2023-09-23T20:44:16.108 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.8710000000000003e-06 2023-09-23T20:44:16.108 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.108 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.108 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:44:16.108 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:39.189387+0000", 2023-09-23T20:44:16.108 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.1674e-05 2023-09-23T20:44:16.109 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.109 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.109 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:44:16.109 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:39.189471+0000", 2023-09-23T20:44:16.109 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.4553000000000002e-05 2023-09-23T20:44:16.109 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.109 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.109 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,5", 2023-09-23T20:44:16.109 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:39.189541+0000", 2023-09-23T20:44:16.109 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.0141000000000005e-05 2023-09-23T20:44:16.109 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.109 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.110 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:44:16.110 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:39.192909+0000", 2023-09-23T20:44:16.110 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0033677640000000001 2023-09-23T20:44:16.110 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.110 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.110 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:16.110 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:39.197657+0000", 2023-09-23T20:44:16.110 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0047484229999999999 2023-09-23T20:44:16.110 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.110 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.110 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:16.110 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:39.307292+0000", 2023-09-23T20:44:16.111 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.10963426699999999 2023-09-23T20:44:16.111 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.111 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.111 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:44:16.111 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:39.307325+0000", 2023-09-23T20:44:16.111 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.3784e-05 2023-09-23T20:44:16.111 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.111 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.111 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:44:16.111 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:39.307356+0000", 2023-09-23T20:44:16.111 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.0128999999999999e-05 2023-09-23T20:44:16.111 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:44:16.112 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:44:16.112 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:44:16.112 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.112 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.112 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.36450.0:162579 9.6 9:654b2841:::benchmark_data_smithi018_381516_object48745:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e486)", 2023-09-23T20:44:16.112 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:43:42.542988+0000", 2023-09-23T20:44:16.112 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 33.482676208999997, 2023-09-23T20:44:16.112 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.082878170000000001, 2023-09-23T20:44:16.112 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:44:16.112 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:44:16.112 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:44:16.112 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.36450", 2023-09-23T20:44:16.113 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:44:16.113 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 162579 2023-09-23T20:44:16.113 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.113 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:44:16.113 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.113 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:44:16.113 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:42.542988+0000", 2023-09-23T20:44:16.113 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:44:16.113 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.113 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.113 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:44:16.113 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:42.542988+0000", 2023-09-23T20:44:16.114 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:44:16.114 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.114 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.114 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:44:16.114 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:42.542986+0000", 2023-09-23T20:44:16.114 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:44:16.114 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.114 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.114 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:44:16.114 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:42.542994+0000", 2023-09-23T20:44:16.114 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.6410000000000002e-06 2023-09-23T20:44:16.114 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.115 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.115 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:44:16.115 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:42.543008+0000", 2023-09-23T20:44:16.115 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.4294000000000001e-05 2023-09-23T20:44:16.115 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.115 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.115 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:44:16.115 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:42.543017+0000", 2023-09-23T20:44:16.115 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.0040000000000005e-06 2023-09-23T20:44:16.115 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.115 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.115 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:44:16.116 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:42.543070+0000", 2023-09-23T20:44:16.116 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.2811e-05 2023-09-23T20:44:16.116 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.116 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.116 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:44:16.116 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:42.543160+0000", 2023-09-23T20:44:16.116 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.967e-05 2023-09-23T20:44:16.116 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.116 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.116 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,5", 2023-09-23T20:44:16.116 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:42.543231+0000", 2023-09-23T20:44:16.116 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.1675999999999996e-05 2023-09-23T20:44:16.117 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.117 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.117 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:44:16.117 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:42.547121+0000", 2023-09-23T20:44:16.117 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0038898510000000002 2023-09-23T20:44:16.117 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.117 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.117 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:16.117 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:42.548701+0000", 2023-09-23T20:44:16.117 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0015795480000000001 2023-09-23T20:44:16.117 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.118 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.118 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:16.118 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:42.625764+0000", 2023-09-23T20:44:16.118 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.077063440999999996 2023-09-23T20:44:16.118 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.118 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.118 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:44:16.118 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:42.625824+0000", 2023-09-23T20:44:16.118 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.9468000000000001e-05 2023-09-23T20:44:16.118 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.118 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.118 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:44:16.119 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:42.625866+0000", 2023-09-23T20:44:16.119 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.2256000000000003e-05 2023-09-23T20:44:16.119 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:44:16.119 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:44:16.119 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:44:16.119 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.119 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.119 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.36450.0:178639 9.6 9:617a0824:::benchmark_data_smithi018_381516_object64805:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e492)", 2023-09-23T20:44:16.119 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:44:04.152481+0000", 2023-09-23T20:44:16.119 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 11.873183234000001, 2023-09-23T20:44:16.119 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.117540705, 2023-09-23T20:44:16.120 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:44:16.120 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:44:16.120 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:44:16.120 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.36450", 2023-09-23T20:44:16.120 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:44:16.120 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 178639 2023-09-23T20:44:16.120 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.120 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:44:16.120 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.120 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:44:16.120 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:04.152481+0000", 2023-09-23T20:44:16.120 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:44:16.121 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.121 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.121 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:44:16.121 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:04.152481+0000", 2023-09-23T20:44:16.121 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:44:16.121 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.121 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.121 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:44:16.121 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:04.152479+0000", 2023-09-23T20:44:16.121 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:44:16.121 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.121 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.121 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:44:16.122 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:04.152486+0000", 2023-09-23T20:44:16.122 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.5950000000000001e-06 2023-09-23T20:44:16.122 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.122 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.122 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:44:16.122 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:04.152501+0000", 2023-09-23T20:44:16.122 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.4949e-05 2023-09-23T20:44:16.122 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.122 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.122 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:44:16.122 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:04.152510+0000", 2023-09-23T20:44:16.122 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.8279999999999992e-06 2023-09-23T20:44:16.123 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.123 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.123 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:44:16.123 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:04.152583+0000", 2023-09-23T20:44:16.123 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.3170000000000006e-05 2023-09-23T20:44:16.123 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.123 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.123 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:44:16.123 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:04.152701+0000", 2023-09-23T20:44:16.123 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000118091 2023-09-23T20:44:16.123 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.124 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.124 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,5", 2023-09-23T20:44:16.124 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:04.152819+0000", 2023-09-23T20:44:16.124 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00011758500000000001 2023-09-23T20:44:16.124 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.124 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.124 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:44:16.124 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:04.162116+0000", 2023-09-23T20:44:16.124 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0092967589999999999 2023-09-23T20:44:16.124 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.124 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.124 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:16.124 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:04.168138+0000", 2023-09-23T20:44:16.125 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0060219740000000003 2023-09-23T20:44:16.125 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.125 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.125 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:16.125 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:04.269927+0000", 2023-09-23T20:44:16.125 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.101789769 2023-09-23T20:44:16.125 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.125 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.125 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:44:16.125 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:04.269981+0000", 2023-09-23T20:44:16.125 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.3467999999999997e-05 2023-09-23T20:44:16.125 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.126 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.126 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:44:16.126 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:04.270021+0000", 2023-09-23T20:44:16.126 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.0448000000000003e-05 2023-09-23T20:44:16.126 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:44:16.126 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:44:16.126 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:44:16.126 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.126 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.126 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.36450.0:178641 9.6 9:6654a987:::benchmark_data_smithi018_381516_object64807:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e492)", 2023-09-23T20:44:16.126 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:44:04.152527+0000", 2023-09-23T20:44:16.126 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 11.873136544999999, 2023-09-23T20:44:16.127 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.12654283999999999, 2023-09-23T20:44:16.127 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:44:16.127 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:44:16.127 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:44:16.127 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.36450", 2023-09-23T20:44:16.127 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:44:16.127 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 178641 2023-09-23T20:44:16.127 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.127 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:44:16.127 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.127 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:44:16.127 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:04.152527+0000", 2023-09-23T20:44:16.128 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:44:16.128 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.128 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.128 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:44:16.128 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:04.152527+0000", 2023-09-23T20:44:16.128 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:44:16.128 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.128 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.128 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:44:16.128 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:04.152527+0000", 2023-09-23T20:44:16.128 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:44:16.128 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.129 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.129 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:44:16.129 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:04.152530+0000", 2023-09-23T20:44:16.129 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.0920000000000002e-06 2023-09-23T20:44:16.129 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.129 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.129 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:44:16.129 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:04.152537+0000", 2023-09-23T20:44:16.129 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.4269999999999996e-06 2023-09-23T20:44:16.129 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.129 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.129 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:44:16.130 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:04.152542+0000", 2023-09-23T20:44:16.130 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.7169999999999996e-06 2023-09-23T20:44:16.130 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.130 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.130 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:44:16.130 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:04.157254+0000", 2023-09-23T20:44:16.130 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0047119370000000002 2023-09-23T20:44:16.130 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.130 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.130 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:44:16.130 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:04.157373+0000", 2023-09-23T20:44:16.130 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000118764 2023-09-23T20:44:16.131 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.131 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.131 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,5", 2023-09-23T20:44:16.131 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:04.157487+0000", 2023-09-23T20:44:16.131 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000114089 2023-09-23T20:44:16.131 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.131 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.131 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:44:16.131 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:04.165069+0000", 2023-09-23T20:44:16.131 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.007582208 2023-09-23T20:44:16.131 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.131 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.132 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:16.132 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:04.170028+0000", 2023-09-23T20:44:16.132 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0049585159999999996 2023-09-23T20:44:16.132 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.132 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.132 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:16.132 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:04.278971+0000", 2023-09-23T20:44:16.132 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.10894280200000001 2023-09-23T20:44:16.132 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.132 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.132 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:44:16.132 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:04.279027+0000", 2023-09-23T20:44:16.133 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.6715999999999997e-05 2023-09-23T20:44:16.133 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.133 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.133 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:44:16.133 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:04.279070+0000", 2023-09-23T20:44:16.133 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.2957999999999998e-05 2023-09-23T20:44:16.133 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:44:16.133 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:44:16.133 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:44:16.133 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.133 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.133 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.36450.0:178662 9.6 9:6fd2349c:::benchmark_data_smithi018_381516_object64828:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e492)", 2023-09-23T20:44:16.134 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:44:04.171816+0000", 2023-09-23T20:44:16.134 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 11.853847696999999, 2023-09-23T20:44:16.134 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.107375846, 2023-09-23T20:44:16.134 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:44:16.134 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:44:16.134 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:44:16.134 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.36450", 2023-09-23T20:44:16.134 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:44:16.134 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 178662 2023-09-23T20:44:16.134 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.134 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:44:16.134 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.135 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:44:16.135 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:04.171816+0000", 2023-09-23T20:44:16.135 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:44:16.135 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.135 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.135 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:44:16.135 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:04.171816+0000", 2023-09-23T20:44:16.135 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:44:16.135 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.135 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.135 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:44:16.136 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:04.171815+0000", 2023-09-23T20:44:16.136 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:44:16.136 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.136 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.136 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:44:16.136 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:04.171822+0000", 2023-09-23T20:44:16.136 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.8539999999999997e-06 2023-09-23T20:44:16.136 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.136 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.136 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:44:16.136 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:04.171839+0000", 2023-09-23T20:44:16.136 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.6058000000000001e-05 2023-09-23T20:44:16.137 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.137 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.137 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:44:16.137 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:04.171848+0000", 2023-09-23T20:44:16.137 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.5719999999999992e-06 2023-09-23T20:44:16.137 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.137 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.137 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:44:16.137 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:04.171897+0000", 2023-09-23T20:44:16.137 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.8795999999999998e-05 2023-09-23T20:44:16.137 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.138 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.138 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:44:16.138 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:04.171992+0000", 2023-09-23T20:44:16.138 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.5443000000000001e-05 2023-09-23T20:44:16.138 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.138 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.138 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,5", 2023-09-23T20:44:16.138 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:04.172071+0000", 2023-09-23T20:44:16.138 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.8274000000000003e-05 2023-09-23T20:44:16.138 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.138 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.138 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:44:16.139 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:04.175873+0000", 2023-09-23T20:44:16.139 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0038023150000000001 2023-09-23T20:44:16.139 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.139 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.139 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:16.139 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:04.176377+0000", 2023-09-23T20:44:16.139 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00050367999999999995 2023-09-23T20:44:16.139 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.139 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.139 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:16.139 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:04.279127+0000", 2023-09-23T20:44:16.139 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.102749938 2023-09-23T20:44:16.140 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.140 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.140 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:44:16.140 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:04.279159+0000", 2023-09-23T20:44:16.140 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.2492e-05 2023-09-23T20:44:16.140 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.140 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.140 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:44:16.140 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:04.279192+0000", 2023-09-23T20:44:16.140 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.3018000000000001e-05 2023-09-23T20:44:16.140 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:44:16.140 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:44:16.141 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:44:16.141 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.141 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.141 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.36450.0:179610 9.6 9:60aed7d8:::benchmark_data_smithi018_381516_object65776:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e494)", 2023-09-23T20:44:16.141 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:44:06.197104+0000", 2023-09-23T20:44:16.141 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 9.8285596099999992, 2023-09-23T20:44:16.141 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0451108, 2023-09-23T20:44:16.141 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:44:16.141 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:44:16.141 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:44:16.141 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.36450", 2023-09-23T20:44:16.141 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:44:16.142 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 179610 2023-09-23T20:44:16.142 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.142 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:44:16.142 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.142 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:44:16.142 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:06.197104+0000", 2023-09-23T20:44:16.142 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:44:16.142 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.142 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.142 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:44:16.142 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:06.197104+0000", 2023-09-23T20:44:16.142 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:44:16.143 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.143 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.143 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:44:16.143 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:06.197103+0000", 2023-09-23T20:44:16.143 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:44:16.143 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.143 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.143 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:44:16.143 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:06.197110+0000", 2023-09-23T20:44:16.143 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.5959999999999998e-06 2023-09-23T20:44:16.143 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.143 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.144 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:44:16.144 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:06.197126+0000", 2023-09-23T20:44:16.144 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.5926e-05 2023-09-23T20:44:16.144 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.144 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.144 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:44:16.144 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:06.197135+0000", 2023-09-23T20:44:16.144 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.7660000000000004e-06 2023-09-23T20:44:16.144 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.144 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.144 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:44:16.145 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:06.212866+0000", 2023-09-23T20:44:16.145 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.015731163999999999 2023-09-23T20:44:16.145 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.145 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.145 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:44:16.145 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:06.213015+0000", 2023-09-23T20:44:16.145 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00014915300000000001 2023-09-23T20:44:16.145 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.145 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.145 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,5", 2023-09-23T20:44:16.145 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:06.213103+0000", 2023-09-23T20:44:16.145 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.7865e-05 2023-09-23T20:44:16.145 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.146 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.146 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:44:16.146 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:06.223082+0000", 2023-09-23T20:44:16.146 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0099787309999999994 2023-09-23T20:44:16.146 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.146 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.146 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:16.146 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:06.223121+0000", 2023-09-23T20:44:16.146 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.9087e-05 2023-09-23T20:44:16.146 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.146 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.147 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:16.147 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:06.242113+0000", 2023-09-23T20:44:16.147 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.018991838 2023-09-23T20:44:16.147 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.147 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.147 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:44:16.147 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:06.242169+0000", 2023-09-23T20:44:16.147 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.6045000000000003e-05 2023-09-23T20:44:16.147 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.147 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.147 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:44:16.147 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:06.242215+0000", 2023-09-23T20:44:16.148 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.6267999999999998e-05 2023-09-23T20:44:16.148 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:44:16.148 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:44:16.148 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:44:16.148 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.148 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.148 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.36450.0:179612 9.6 9:69ef5475:::benchmark_data_smithi018_381516_object65778:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e494)", 2023-09-23T20:44:16.148 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:44:06.197200+0000", 2023-09-23T20:44:16.148 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 9.8284641530000005, 2023-09-23T20:44:16.148 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.045203440999999997, 2023-09-23T20:44:16.148 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:44:16.148 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:44:16.149 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:44:16.149 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.36450", 2023-09-23T20:44:16.149 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:44:16.149 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 179612 2023-09-23T20:44:16.149 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.149 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:44:16.149 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.149 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:44:16.149 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:06.197200+0000", 2023-09-23T20:44:16.149 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:44:16.149 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.149 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.150 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:44:16.150 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:06.197200+0000", 2023-09-23T20:44:16.150 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:44:16.150 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.150 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.150 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:44:16.150 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:06.197199+0000", 2023-09-23T20:44:16.150 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.999999 2023-09-23T20:44:16.150 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.150 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.150 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:44:16.150 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:06.197205+0000", 2023-09-23T20:44:16.151 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.3709999999999996e-06 2023-09-23T20:44:16.151 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.151 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.151 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:44:16.151 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:06.197221+0000", 2023-09-23T20:44:16.151 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.6032000000000002e-05 2023-09-23T20:44:16.151 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.151 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.151 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:44:16.151 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:06.197230+0000", 2023-09-23T20:44:16.151 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.1849999999999993e-06 2023-09-23T20:44:16.151 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.152 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.152 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:44:16.152 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:06.218055+0000", 2023-09-23T20:44:16.152 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.020824504000000001 2023-09-23T20:44:16.152 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.152 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.152 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:44:16.152 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:06.218159+0000", 2023-09-23T20:44:16.152 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000104409 2023-09-23T20:44:16.152 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.152 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.152 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,5", 2023-09-23T20:44:16.152 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:06.218243+0000", 2023-09-23T20:44:16.153 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.3603999999999994e-05 2023-09-23T20:44:16.153 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.153 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.153 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:44:16.153 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:06.223269+0000", 2023-09-23T20:44:16.153 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0050260859999999999 2023-09-23T20:44:16.153 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.153 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.153 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:16.153 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:06.223428+0000", 2023-09-23T20:44:16.153 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000158954 2023-09-23T20:44:16.154 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.154 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.154 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:16.154 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:06.242307+0000", 2023-09-23T20:44:16.154 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.018879571000000001 2023-09-23T20:44:16.154 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.154 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.154 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:44:16.154 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:06.242359+0000", 2023-09-23T20:44:16.154 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.1149000000000001e-05 2023-09-23T20:44:16.154 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.154 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.155 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:44:16.155 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:06.242403+0000", 2023-09-23T20:44:16.155 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.4595000000000002e-05 2023-09-23T20:44:16.155 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:44:16.155 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:44:16.155 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:44:16.155 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.155 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.155 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.36450.0:179621 9.6 9:605f26ce:::benchmark_data_smithi018_381516_object65787:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e494)", 2023-09-23T20:44:16.155 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:44:06.209053+0000", 2023-09-23T20:44:16.155 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 9.8166108209999994, 2023-09-23T20:44:16.155 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.036416023999999998, 2023-09-23T20:44:16.156 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:44:16.156 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:44:16.156 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:44:16.156 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.36450", 2023-09-23T20:44:16.156 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:44:16.156 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 179621 2023-09-23T20:44:16.156 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.156 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:44:16.156 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.156 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:44:16.156 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:06.209053+0000", 2023-09-23T20:44:16.156 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:44:16.157 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.157 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.157 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:44:16.157 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:06.209053+0000", 2023-09-23T20:44:16.157 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:44:16.157 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.157 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.157 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:44:16.157 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:06.209051+0000", 2023-09-23T20:44:16.157 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:44:16.157 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.157 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.157 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:44:16.158 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:06.209060+0000", 2023-09-23T20:44:16.158 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.4689999999999997e-06 2023-09-23T20:44:16.158 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.158 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.158 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:44:16.158 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:06.209074+0000", 2023-09-23T20:44:16.158 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.4301000000000001e-05 2023-09-23T20:44:16.158 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.158 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.158 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:44:16.158 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:06.209083+0000", 2023-09-23T20:44:16.159 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.7749999999999998e-06 2023-09-23T20:44:16.159 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.159 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.159 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:44:16.159 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:06.223461+0000", 2023-09-23T20:44:16.159 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.014378461 2023-09-23T20:44:16.159 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.159 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.159 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:44:16.159 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:06.223563+0000", 2023-09-23T20:44:16.159 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000101184 2023-09-23T20:44:16.159 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.160 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.160 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,5", 2023-09-23T20:44:16.160 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:06.223676+0000", 2023-09-23T20:44:16.160 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000113576 2023-09-23T20:44:16.160 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.160 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.160 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:16.160 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:06.233735+0000", 2023-09-23T20:44:16.160 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.010059257 2023-09-23T20:44:16.160 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.160 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.160 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:44:16.161 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:06.233746+0000", 2023-09-23T20:44:16.161 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1022999999999999e-05 2023-09-23T20:44:16.161 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.161 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.161 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:16.161 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:06.245372+0000", 2023-09-23T20:44:16.161 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.011625163000000001 2023-09-23T20:44:16.161 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.161 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.161 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:44:16.161 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:06.245425+0000", 2023-09-23T20:44:16.161 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.3671000000000002e-05 2023-09-23T20:44:16.162 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.162 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.162 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:44:16.162 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:06.245469+0000", 2023-09-23T20:44:16.162 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.3936000000000001e-05 2023-09-23T20:44:16.162 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:44:16.162 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:44:16.162 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:44:16.162 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.162 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.162 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.36450.0:179713 9.6 9:64b2f644:::benchmark_data_smithi018_381516_object65879:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e494)", 2023-09-23T20:44:16.162 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:44:06.342582+0000", 2023-09-23T20:44:16.163 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 9.6830820170000003, 2023-09-23T20:44:16.163 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.043839305000000002, 2023-09-23T20:44:16.163 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:44:16.163 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:44:16.163 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:44:16.163 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.36450", 2023-09-23T20:44:16.163 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:44:16.163 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 179713 2023-09-23T20:44:16.163 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.163 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:44:16.163 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.163 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:44:16.164 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:06.342582+0000", 2023-09-23T20:44:16.164 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:44:16.164 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.164 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.164 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:44:16.164 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:06.342582+0000", 2023-09-23T20:44:16.164 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:44:16.164 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.164 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.164 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:44:16.164 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:06.342581+0000", 2023-09-23T20:44:16.164 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:44:16.165 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.165 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.165 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:44:16.165 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:06.342584+0000", 2023-09-23T20:44:16.165 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.9210000000000001e-06 2023-09-23T20:44:16.165 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.165 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.165 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:44:16.165 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:06.342597+0000", 2023-09-23T20:44:16.165 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2860000000000001e-05 2023-09-23T20:44:16.165 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.166 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.166 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:44:16.166 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:06.342603+0000", 2023-09-23T20:44:16.166 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.4430000000000002e-06 2023-09-23T20:44:16.166 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.166 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.166 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:44:16.166 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:06.353970+0000", 2023-09-23T20:44:16.166 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.011367396 2023-09-23T20:44:16.166 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.166 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.166 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:44:16.167 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:06.354100+0000", 2023-09-23T20:44:16.167 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00012970399999999999 2023-09-23T20:44:16.167 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.167 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.167 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,5", 2023-09-23T20:44:16.167 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:06.354218+0000", 2023-09-23T20:44:16.167 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000118455 2023-09-23T20:44:16.167 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.167 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.167 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:16.167 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:06.360038+0000", 2023-09-23T20:44:16.168 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0058195959999999998 2023-09-23T20:44:16.168 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.168 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.168 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:44:16.168 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:06.360121+0000", 2023-09-23T20:44:16.168 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.3523999999999995e-05 2023-09-23T20:44:16.168 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.168 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.168 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:16.168 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:06.386274+0000", 2023-09-23T20:44:16.168 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.026152920999999999 2023-09-23T20:44:16.168 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.169 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.169 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:44:16.169 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:06.386379+0000", 2023-09-23T20:44:16.169 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000104409 2023-09-23T20:44:16.169 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.169 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.169 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:44:16.169 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:06.386421+0000", 2023-09-23T20:44:16.169 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.2517e-05 2023-09-23T20:44:16.169 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:44:16.169 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:44:16.169 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:44:16.170 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.170 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.170 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.36450.0:179759 9.6 9:6a87048d:::benchmark_data_smithi018_381516_object65925:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e494)", 2023-09-23T20:44:16.170 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:44:06.425540+0000", 2023-09-23T20:44:16.170 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 9.6001244129999996, 2023-09-23T20:44:16.170 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.037034071000000002, 2023-09-23T20:44:16.170 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:44:16.170 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:44:16.170 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:44:16.170 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.36450", 2023-09-23T20:44:16.170 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:44:16.170 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 179759 2023-09-23T20:44:16.171 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.171 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:44:16.171 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.171 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:44:16.171 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:06.425540+0000", 2023-09-23T20:44:16.171 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:44:16.171 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.171 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.171 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:44:16.171 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:06.425540+0000", 2023-09-23T20:44:16.171 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:44:16.172 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.172 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.172 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:44:16.172 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:06.425538+0000", 2023-09-23T20:44:16.172 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:44:16.172 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.172 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.172 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:44:16.172 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:06.425546+0000", 2023-09-23T20:44:16.172 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.5569999999999996e-06 2023-09-23T20:44:16.172 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.172 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.173 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:44:16.173 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:06.425566+0000", 2023-09-23T20:44:16.173 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.9707999999999998e-05 2023-09-23T20:44:16.173 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.173 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.173 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:44:16.173 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:06.425577+0000", 2023-09-23T20:44:16.173 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0908e-05 2023-09-23T20:44:16.173 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.173 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.173 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:44:16.173 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:06.425662+0000", 2023-09-23T20:44:16.174 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.5358999999999994e-05 2023-09-23T20:44:16.174 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.174 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.174 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:44:16.174 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:06.425809+0000", 2023-09-23T20:44:16.174 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000146389 2023-09-23T20:44:16.174 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.174 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.174 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,5", 2023-09-23T20:44:16.174 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:06.425932+0000", 2023-09-23T20:44:16.174 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00012353099999999999 2023-09-23T20:44:16.174 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.175 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.175 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:16.175 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:06.442078+0000", 2023-09-23T20:44:16.175 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.016146126 2023-09-23T20:44:16.175 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.175 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.175 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:44:16.175 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:06.447026+0000", 2023-09-23T20:44:16.175 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0049474690000000003 2023-09-23T20:44:16.175 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.175 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.175 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:16.176 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:06.462476+0000", 2023-09-23T20:44:16.176 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.015449987 2023-09-23T20:44:16.176 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.176 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.176 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:44:16.176 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:06.462532+0000", 2023-09-23T20:44:16.176 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.5927999999999997e-05 2023-09-23T20:44:16.176 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.176 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.176 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:44:16.176 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:06.462574+0000", 2023-09-23T20:44:16.176 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.1811000000000003e-05 2023-09-23T20:44:16.177 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:44:16.177 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:44:16.177 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:44:16.177 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.177 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.177 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.36450.0:180241 9.6 9:66da16f5:::benchmark_data_smithi018_381516_object66407:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e494)", 2023-09-23T20:44:16.177 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:44:07.125089+0000", 2023-09-23T20:44:16.177 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 8.9005747450000001, 2023-09-23T20:44:16.177 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.036096597000000001, 2023-09-23T20:44:16.177 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:44:16.177 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:44:16.177 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:44:16.178 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.36450", 2023-09-23T20:44:16.178 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:44:16.178 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 180241 2023-09-23T20:44:16.178 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.178 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:44:16.178 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.178 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:44:16.178 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:07.125089+0000", 2023-09-23T20:44:16.178 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:44:16.178 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.178 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.178 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:44:16.179 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:07.125089+0000", 2023-09-23T20:44:16.179 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:44:16.179 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.179 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.179 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:44:16.179 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:07.125087+0000", 2023-09-23T20:44:16.179 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:44:16.179 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.179 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.179 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:44:16.179 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:07.125096+0000", 2023-09-23T20:44:16.179 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.7620000000000001e-06 2023-09-23T20:44:16.180 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.180 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.180 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:44:16.180 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:07.125116+0000", 2023-09-23T20:44:16.180 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.0060000000000001e-05 2023-09-23T20:44:16.180 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.180 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.180 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:44:16.180 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:07.125126+0000", 2023-09-23T20:44:16.180 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0084e-05 2023-09-23T20:44:16.180 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.180 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.180 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:44:16.181 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:07.125185+0000", 2023-09-23T20:44:16.181 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.8894999999999998e-05 2023-09-23T20:44:16.181 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.181 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.181 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:44:16.181 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:07.125324+0000", 2023-09-23T20:44:16.181 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00013910699999999999 2023-09-23T20:44:16.181 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.181 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.181 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,5", 2023-09-23T20:44:16.181 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:07.125438+0000", 2023-09-23T20:44:16.181 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000114139 2023-09-23T20:44:16.182 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.182 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.182 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:16.182 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:07.130558+0000", 2023-09-23T20:44:16.182 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0051199119999999999 2023-09-23T20:44:16.182 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.182 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.182 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:44:16.182 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:07.130691+0000", 2023-09-23T20:44:16.182 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000132664 2023-09-23T20:44:16.182 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.182 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.183 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:16.183 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:07.161090+0000", 2023-09-23T20:44:16.183 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.030399189 2023-09-23T20:44:16.183 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.183 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.183 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:44:16.183 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:07.161142+0000", 2023-09-23T20:44:16.183 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.1940000000000001e-05 2023-09-23T20:44:16.183 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:16.183 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:16.183 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:44:16.183 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:07.161186+0000", 2023-09-23T20:44:16.184 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.3871000000000001e-05 2023-09-23T20:44:16.184 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:44:16.184 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:44:16.184 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:44:16.184 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:44:16.184 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:44:16.184 INFO:teuthology.orchestra.run.smithi018.stdout:} 2023-09-23T20:44:16.184 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:44:16.209 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:16.244 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:44:16.247 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:16.247 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:44:16.348 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:44:16.376 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:16.399 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T20:44:16.401 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:44:16.413 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:44:16.417 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:16.417 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:44:16.518 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:44:16.545 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:16.583 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:44:16.586 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:16.586 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:44:16.636 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:16 smithi018 ceph-mon[184144]: osdmap e500: 8 total, 8 up, 8 in 2023-09-23T20:44:16.636 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:16 smithi018 ceph-mon[184144]: failed to encode map e500 with expected crc 2023-09-23T20:44:16.636 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:16 smithi018 ceph-mon[184144]: failed to encode map e500 with expected crc 2023-09-23T20:44:16.636 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:16 smithi018 ceph-mon[184144]: failed to encode map e500 with expected crc 2023-09-23T20:44:16.636 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:16 smithi018 ceph-mon[184144]: failed to encode map e500 with expected crc 2023-09-23T20:44:16.637 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:16 smithi018 ceph-mon[184144]: failed to encode map e500 with expected crc 2023-09-23T20:44:16.637 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:16 smithi018 ceph-mon[184144]: 1.1 scrub starts 2023-09-23T20:44:16.637 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:16 smithi018 ceph-mon[184144]: 1.1 scrub ok 2023-09-23T20:44:16.637 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:16 smithi018 ceph-mon[184144]: 1.18 scrub starts 2023-09-23T20:44:16.637 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:16 smithi018 ceph-mon[179647]: osdmap e500: 8 total, 8 up, 8 in 2023-09-23T20:44:16.637 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:16 smithi018 ceph-mon[179647]: failed to encode map e500 with expected crc 2023-09-23T20:44:16.637 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:16 smithi018 ceph-mon[179647]: failed to encode map e500 with expected crc 2023-09-23T20:44:16.637 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:16 smithi018 ceph-mon[179647]: failed to encode map e500 with expected crc 2023-09-23T20:44:16.637 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:16 smithi018 ceph-mon[179647]: failed to encode map e500 with expected crc 2023-09-23T20:44:16.637 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:16 smithi018 ceph-mon[179647]: failed to encode map e500 with expected crc 2023-09-23T20:44:16.637 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:16 smithi018 ceph-mon[179647]: 1.1 scrub starts 2023-09-23T20:44:16.638 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:16 smithi018 ceph-mon[179647]: 1.1 scrub ok 2023-09-23T20:44:16.638 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:16 smithi018 ceph-mon[179647]: 1.18 scrub starts 2023-09-23T20:44:16.687 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:44:16.718 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:16.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:16 smithi149 ceph-mon[170394]: osdmap e500: 8 total, 8 up, 8 in 2023-09-23T20:44:16.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:16 smithi149 ceph-mon[170394]: failed to encode map e500 with expected crc 2023-09-23T20:44:16.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:16 smithi149 ceph-mon[170394]: failed to encode map e500 with expected crc 2023-09-23T20:44:16.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:16 smithi149 ceph-mon[170394]: failed to encode map e500 with expected crc 2023-09-23T20:44:16.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:16 smithi149 ceph-mon[170394]: failed to encode map e500 with expected crc 2023-09-23T20:44:16.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:16 smithi149 ceph-mon[170394]: failed to encode map e500 with expected crc 2023-09-23T20:44:16.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:16 smithi149 ceph-mon[170394]: 1.1 scrub starts 2023-09-23T20:44:16.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:16 smithi149 ceph-mon[170394]: 1.1 scrub ok 2023-09-23T20:44:16.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:16 smithi149 ceph-mon[170394]: 1.18 scrub starts 2023-09-23T20:44:17.147 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T20:44:17.151 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:44:17.167 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:17.168 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:44:17.208 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:44:17.268 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:44:17.368 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:17.450 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:44:17.459 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:17.460 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:44:17.560 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:44:17.611 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:17 smithi018 ceph-mon[179647]: pgmap v1261: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 2.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 417 op/s; 2749/133200 objects misplaced (2.064%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T20:44:17.611 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:17 smithi018 ceph-mon[179647]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T20:44:17.611 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:17 smithi018 ceph-mon[179647]: failed to encode map e501 with expected crc 2023-09-23T20:44:17.611 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:17 smithi018 ceph-mon[179647]: osdmap e501: 8 total, 8 up, 8 in 2023-09-23T20:44:17.611 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:17 smithi018 ceph-mon[179647]: failed to encode map e501 with expected crc 2023-09-23T20:44:17.611 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:17 smithi018 ceph-mon[179647]: failed to encode map e501 with expected crc 2023-09-23T20:44:17.611 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:17 smithi018 ceph-mon[179647]: 9.5 starting backfill to osd.7 from (462'3810,495'11137] MIN to 498'11296 2023-09-23T20:44:17.611 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:17 smithi018 ceph-mon[179647]: failed to encode map e501 with expected crc 2023-09-23T20:44:17.666 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:17.766 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:44:17.767 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:17.768 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:44:17.794 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:44:17.865 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:17 smithi018 ceph-mon[184144]: pgmap v1261: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 2.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 417 op/s; 2749/133200 objects misplaced (2.064%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T20:44:17.865 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:17 smithi018 ceph-mon[184144]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T20:44:17.865 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:17 smithi018 ceph-mon[184144]: failed to encode map e501 with expected crc 2023-09-23T20:44:17.865 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:17 smithi018 ceph-mon[184144]: osdmap e501: 8 total, 8 up, 8 in 2023-09-23T20:44:17.865 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:17 smithi018 ceph-mon[184144]: failed to encode map e501 with expected crc 2023-09-23T20:44:17.865 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:17 smithi018 ceph-mon[184144]: failed to encode map e501 with expected crc 2023-09-23T20:44:17.866 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:17 smithi018 ceph-mon[184144]: 9.5 starting backfill to osd.7 from (462'3810,495'11137] MIN to 498'11296 2023-09-23T20:44:17.866 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:17 smithi018 ceph-mon[184144]: failed to encode map e501 with expected crc 2023-09-23T20:44:17.868 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:44:17.896 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:17 smithi149 ceph-mon[170394]: pgmap v1261: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 2.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 417 op/s; 2749/133200 objects misplaced (2.064%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T20:44:17.897 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:17 smithi149 ceph-mon[170394]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T20:44:17.897 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:17 smithi149 ceph-mon[170394]: failed to encode map e501 with expected crc 2023-09-23T20:44:17.897 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:17 smithi149 ceph-mon[170394]: osdmap e501: 8 total, 8 up, 8 in 2023-09-23T20:44:17.897 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:17 smithi149 ceph-mon[170394]: failed to encode map e501 with expected crc 2023-09-23T20:44:17.897 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:17 smithi149 ceph-mon[170394]: failed to encode map e501 with expected crc 2023-09-23T20:44:17.897 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:17 smithi149 ceph-mon[170394]: 9.5 starting backfill to osd.7 from (462'3810,495'11137] MIN to 498'11296 2023-09-23T20:44:17.897 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:17 smithi149 ceph-mon[170394]: failed to encode map e501 with expected crc 2023-09-23T20:44:17.900 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:17.943 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:44:17.947 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:17.947 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:44:18.048 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:44:18.095 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:18.323 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:44:18.326 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:18.326 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:44:18.408 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:44:18.427 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:44:18.455 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:18.490 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:44:18.494 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:18.494 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:44:18.595 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:44:18.623 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:18 smithi018 ceph-mon[184144]: osdmap e502: 8 total, 8 up, 8 in 2023-09-23T20:44:18.623 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:18 smithi018 ceph-mon[184144]: failed to encode map e502 with expected crc 2023-09-23T20:44:18.624 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:18 smithi018 ceph-mon[184144]: failed to encode map e502 with expected crc 2023-09-23T20:44:18.624 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:18 smithi018 ceph-mon[184144]: failed to encode map e502 with expected crc 2023-09-23T20:44:18.624 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:18 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2207557701' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T20:44:18.624 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:18 smithi018 ceph-mon[179647]: osdmap e502: 8 total, 8 up, 8 in 2023-09-23T20:44:18.624 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:18 smithi018 ceph-mon[179647]: failed to encode map e502 with expected crc 2023-09-23T20:44:18.624 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:18 smithi018 ceph-mon[179647]: failed to encode map e502 with expected crc 2023-09-23T20:44:18.624 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:18 smithi018 ceph-mon[179647]: failed to encode map e502 with expected crc 2023-09-23T20:44:18.624 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:18 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2207557701' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T20:44:18.626 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:18.637 INFO:teuthology.orchestra.run.smithi018.stderr:reweighted osd.3 to 1 (10000) 2023-09-23T20:44:18.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:18 smithi149 ceph-mon[170394]: osdmap e502: 8 total, 8 up, 8 in 2023-09-23T20:44:18.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:18 smithi149 ceph-mon[170394]: failed to encode map e502 with expected crc 2023-09-23T20:44:18.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:18 smithi149 ceph-mon[170394]: failed to encode map e502 with expected crc 2023-09-23T20:44:18.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:18 smithi149 ceph-mon[170394]: failed to encode map e502 with expected crc 2023-09-23T20:44:18.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:18 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2207557701' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T20:44:18.880 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:44:18.884 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:18.885 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:44:18.985 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:44:19.020 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:19.057 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:44:19.061 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:19.062 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:44:19.163 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:44:19.171 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd reweight 2 1 2023-09-23T20:44:19.178 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:44:19.189 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:19.223 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:44:19.225 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:19.226 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:44:19.262 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2023-09-23T20:44:19.262 INFO:teuthology.orchestra.run.smithi018.stdout: "ops": [ 2023-09-23T20:44:19.263 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.263 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.36450.0:185417 9.8 9.6b5d6608 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e503)", 2023-09-23T20:44:19.263 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:44:18.825896+0000", 2023-09-23T20:44:19.263 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.43478684400000001, 2023-09-23T20:44:19.263 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.43481931899999998, 2023-09-23T20:44:19.263 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:44:19.263 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "delayed", 2023-09-23T20:44:19.263 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:44:19.263 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.36450", 2023-09-23T20:44:19.263 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:44:19.263 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 185417 2023-09-23T20:44:19.264 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.264 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:44:19.264 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.264 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:44:19.264 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.825896+0000", 2023-09-23T20:44:19.264 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:44:19.264 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.264 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.264 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:44:19.264 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.825896+0000", 2023-09-23T20:44:19.264 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:44:19.264 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.265 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.265 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:44:19.265 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.825895+0000", 2023-09-23T20:44:19.265 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.999999 2023-09-23T20:44:19.265 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.265 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.265 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:44:19.265 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.825899+0000", 2023-09-23T20:44:19.265 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.6700000000000002e-06 2023-09-23T20:44:19.265 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.265 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.265 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:44:19.266 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.825911+0000", 2023-09-23T20:44:19.266 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1284000000000001e-05 2023-09-23T20:44:19.266 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.266 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.266 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:44:19.266 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.825916+0000", 2023-09-23T20:44:19.266 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.5600000000000001e-06 2023-09-23T20:44:19.266 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.266 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.266 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:44:19.267 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.825968+0000", 2023-09-23T20:44:19.267 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.1684999999999997e-05 2023-09-23T20:44:19.267 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.267 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.267 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:44:19.267 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.825984+0000", 2023-09-23T20:44:19.267 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.6416e-05 2023-09-23T20:44:19.267 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:44:19.267 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:44:19.267 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:44:19.267 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.268 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.268 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.36450.0:185451 9.8 9.41c6ff38 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e503)", 2023-09-23T20:44:19.268 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:44:19.037165+0000", 2023-09-23T20:44:19.268 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.22351706800000001, 2023-09-23T20:44:19.268 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.22359942899999999, 2023-09-23T20:44:19.268 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:44:19.268 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "delayed", 2023-09-23T20:44:19.268 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:44:19.268 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.36450", 2023-09-23T20:44:19.268 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:44:19.268 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 185451 2023-09-23T20:44:19.268 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.269 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:44:19.269 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.269 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:44:19.269 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:19.037165+0000", 2023-09-23T20:44:19.269 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:44:19.269 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.269 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.269 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:44:19.269 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:19.037165+0000", 2023-09-23T20:44:19.269 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:44:19.269 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.269 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.269 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:44:19.270 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:19.037163+0000", 2023-09-23T20:44:19.270 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:44:19.270 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.270 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.270 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:44:19.270 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:19.037174+0000", 2023-09-23T20:44:19.270 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0869999999999999e-05 2023-09-23T20:44:19.270 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.270 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.270 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:44:19.270 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:19.037193+0000", 2023-09-23T20:44:19.270 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.8388000000000001e-05 2023-09-23T20:44:19.271 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.271 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.271 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:44:19.271 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:19.037205+0000", 2023-09-23T20:44:19.271 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2757e-05 2023-09-23T20:44:19.271 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.271 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.271 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:44:19.271 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:19.037283+0000", 2023-09-23T20:44:19.271 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.7362999999999998e-05 2023-09-23T20:44:19.271 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.271 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.272 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:44:19.272 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:19.037292+0000", 2023-09-23T20:44:19.272 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.1400000000000006e-06 2023-09-23T20:44:19.272 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:44:19.272 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:44:19.272 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:44:19.272 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.272 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.272 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.36450.0:185489 9.8 9.7681d278 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e503)", 2023-09-23T20:44:19.272 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:44:19.129445+0000", 2023-09-23T20:44:19.272 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.131237886, 2023-09-23T20:44:19.272 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.13137254300000001, 2023-09-23T20:44:19.273 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:44:19.273 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "delayed", 2023-09-23T20:44:19.273 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:44:19.273 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.36450", 2023-09-23T20:44:19.273 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:44:19.273 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 185489 2023-09-23T20:44:19.273 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.273 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:44:19.273 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.273 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:44:19.273 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:19.129445+0000", 2023-09-23T20:44:19.274 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:44:19.274 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.274 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.274 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:44:19.274 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:19.129445+0000", 2023-09-23T20:44:19.274 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:44:19.274 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.274 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.274 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:44:19.274 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:19.129443+0000", 2023-09-23T20:44:19.274 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:44:19.275 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.275 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.275 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:44:19.275 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:19.129450+0000", 2023-09-23T20:44:19.275 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.3799999999999999e-06 2023-09-23T20:44:19.275 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.275 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.275 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:44:19.275 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:19.129462+0000", 2023-09-23T20:44:19.275 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2466000000000001e-05 2023-09-23T20:44:19.275 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.275 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.276 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:44:19.276 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:19.129469+0000", 2023-09-23T20:44:19.276 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.8340000000000001e-06 2023-09-23T20:44:19.276 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.276 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.276 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:44:19.276 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:19.129535+0000", 2023-09-23T20:44:19.276 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.5968000000000003e-05 2023-09-23T20:44:19.276 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.276 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.276 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:44:19.276 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:19.129550+0000", 2023-09-23T20:44:19.277 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.4571e-05 2023-09-23T20:44:19.277 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:44:19.277 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:44:19.277 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:44:19.277 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.277 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.277 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.36450.0:185452 9.8 9.1e2dff98 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e503)", 2023-09-23T20:44:19.277 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:44:19.043273+0000", 2023-09-23T20:44:19.277 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.21740954500000001, 2023-09-23T20:44:19.277 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.21758282600000001, 2023-09-23T20:44:19.277 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:44:19.277 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "delayed", 2023-09-23T20:44:19.277 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:44:19.278 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.36450", 2023-09-23T20:44:19.278 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:44:19.278 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 185452 2023-09-23T20:44:19.278 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.278 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:44:19.278 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.278 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:44:19.278 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:19.043273+0000", 2023-09-23T20:44:19.278 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:44:19.278 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.278 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.279 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:44:19.279 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:19.043273+0000", 2023-09-23T20:44:19.279 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:44:19.279 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.279 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.279 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:44:19.279 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:19.043272+0000", 2023-09-23T20:44:19.279 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:44:19.279 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.279 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.279 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:44:19.279 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:19.043279+0000", 2023-09-23T20:44:19.280 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.4440000000000002e-06 2023-09-23T20:44:19.280 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.280 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.280 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:44:19.280 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:19.043301+0000", 2023-09-23T20:44:19.280 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.1735000000000001e-05 2023-09-23T20:44:19.280 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.280 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.280 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:44:19.280 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:19.043310+0000", 2023-09-23T20:44:19.280 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.3989999999999993e-06 2023-09-23T20:44:19.280 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.281 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.281 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:44:19.281 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:19.043363+0000", 2023-09-23T20:44:19.281 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.2679999999999997e-05 2023-09-23T20:44:19.281 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.281 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.281 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:44:19.281 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:19.043384+0000", 2023-09-23T20:44:19.281 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.0678000000000001e-05 2023-09-23T20:44:19.281 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:44:19.282 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:44:19.282 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:44:19.282 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.282 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.282 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.36450.0:185319 9.8 9.8673f918 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e503)", 2023-09-23T20:44:19.282 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:44:18.697532+0000", 2023-09-23T20:44:19.282 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.56315010200000004, 2023-09-23T20:44:19.282 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.563362747, 2023-09-23T20:44:19.282 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:44:19.283 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "delayed", 2023-09-23T20:44:19.283 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:44:19.283 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.36450", 2023-09-23T20:44:19.283 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:44:19.283 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 185319 2023-09-23T20:44:19.283 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.283 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:44:19.283 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.283 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:44:19.283 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.697532+0000", 2023-09-23T20:44:19.283 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:44:19.284 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.284 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.284 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:44:19.284 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.697532+0000", 2023-09-23T20:44:19.285 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:44:19.285 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.285 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.285 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:44:19.285 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.697532+0000", 2023-09-23T20:44:19.285 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:44:19.285 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.285 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.285 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:44:19.285 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.697535+0000", 2023-09-23T20:44:19.286 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.1709999999999998e-06 2023-09-23T20:44:19.286 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.286 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.286 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:44:19.286 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.697542+0000", 2023-09-23T20:44:19.286 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.7880000000000001e-06 2023-09-23T20:44:19.286 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.286 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.286 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:44:19.286 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.697547+0000", 2023-09-23T20:44:19.286 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.0159999999999999e-06 2023-09-23T20:44:19.287 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.287 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.287 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:44:19.287 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.698190+0000", 2023-09-23T20:44:19.287 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00064296299999999998 2023-09-23T20:44:19.287 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.287 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.287 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:44:19.287 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.698201+0000", 2023-09-23T20:44:19.287 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0898e-05 2023-09-23T20:44:19.287 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:44:19.287 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:44:19.288 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:44:19.288 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.288 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.288 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.36450.0:185425 9.8 9.55322cb8 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e503)", 2023-09-23T20:44:19.288 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:44:18.834145+0000", 2023-09-23T20:44:19.288 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.426537412, 2023-09-23T20:44:19.288 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.426787688, 2023-09-23T20:44:19.288 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:44:19.288 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "delayed", 2023-09-23T20:44:19.288 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:44:19.288 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.36450", 2023-09-23T20:44:19.289 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:44:19.289 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 185425 2023-09-23T20:44:19.289 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.289 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:44:19.289 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.289 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:44:19.289 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.834145+0000", 2023-09-23T20:44:19.289 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:44:19.289 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.289 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.289 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:44:19.289 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.834145+0000", 2023-09-23T20:44:19.290 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:44:19.290 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.290 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.290 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:44:19.290 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.834144+0000", 2023-09-23T20:44:19.290 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.999999 2023-09-23T20:44:19.290 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.290 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.290 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:44:19.290 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.834150+0000", 2023-09-23T20:44:19.290 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.6799999999999998e-06 2023-09-23T20:44:19.291 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.291 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.291 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:44:19.291 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.834166+0000", 2023-09-23T20:44:19.291 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.6228000000000001e-05 2023-09-23T20:44:19.291 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.291 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.291 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:44:19.291 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.834176+0000", 2023-09-23T20:44:19.291 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0032e-05 2023-09-23T20:44:19.291 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.292 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.292 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:44:19.292 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.834242+0000", 2023-09-23T20:44:19.292 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.5899e-05 2023-09-23T20:44:19.292 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.292 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.292 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:44:19.292 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.834252+0000", 2023-09-23T20:44:19.292 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0525e-05 2023-09-23T20:44:19.292 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:44:19.292 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:44:19.292 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:44:19.293 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.293 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.293 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.36450.0:185321 9.8 9.f3369438 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e503)", 2023-09-23T20:44:19.293 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:44:18.708465+0000", 2023-09-23T20:44:19.293 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.55221790299999995, 2023-09-23T20:44:19.293 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.55250586999999995, 2023-09-23T20:44:19.293 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:44:19.293 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "delayed", 2023-09-23T20:44:19.293 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:44:19.293 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.36450", 2023-09-23T20:44:19.293 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:44:19.294 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 185321 2023-09-23T20:44:19.294 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.294 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:44:19.294 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.294 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:44:19.294 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.708465+0000", 2023-09-23T20:44:19.294 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:44:19.294 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.294 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.294 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:44:19.294 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.708465+0000", 2023-09-23T20:44:19.294 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:44:19.295 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.295 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.295 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:44:19.295 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.708463+0000", 2023-09-23T20:44:19.295 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:44:19.295 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.295 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.295 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:44:19.295 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.708482+0000", 2023-09-23T20:44:19.295 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.9258000000000001e-05 2023-09-23T20:44:19.295 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.296 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.296 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:44:19.296 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.708500+0000", 2023-09-23T20:44:19.296 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.7708000000000001e-05 2023-09-23T20:44:19.296 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.296 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.296 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:44:19.296 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.708510+0000", 2023-09-23T20:44:19.296 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.6190000000000006e-06 2023-09-23T20:44:19.296 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.296 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.296 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:44:19.297 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.708611+0000", 2023-09-23T20:44:19.297 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000101296 2023-09-23T20:44:19.297 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.297 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.297 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:44:19.297 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.708623+0000", 2023-09-23T20:44:19.297 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.201e-05 2023-09-23T20:44:19.297 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:44:19.297 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:44:19.297 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:44:19.297 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.297 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.298 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.36450.0:185429 9.8 9.f4d7f968 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e503)", 2023-09-23T20:44:19.298 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:44:18.837932+0000", 2023-09-23T20:44:19.298 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.42275016199999998, 2023-09-23T20:44:19.298 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.42307798099999999, 2023-09-23T20:44:19.298 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:44:19.298 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "delayed", 2023-09-23T20:44:19.298 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:44:19.298 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.36450", 2023-09-23T20:44:19.298 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:44:19.298 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 185429 2023-09-23T20:44:19.298 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.298 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:44:19.299 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.299 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:44:19.299 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.837932+0000", 2023-09-23T20:44:19.299 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:44:19.299 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.299 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.299 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:44:19.299 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.837932+0000", 2023-09-23T20:44:19.299 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:44:19.299 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.299 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.299 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:44:19.299 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.837931+0000", 2023-09-23T20:44:19.300 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.999999 2023-09-23T20:44:19.300 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.300 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.300 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:44:19.300 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.837936+0000", 2023-09-23T20:44:19.300 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.0159999999999999e-06 2023-09-23T20:44:19.300 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.300 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.300 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:44:19.300 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.837948+0000", 2023-09-23T20:44:19.300 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1338999999999999e-05 2023-09-23T20:44:19.300 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.301 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.301 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:44:19.301 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.837953+0000", 2023-09-23T20:44:19.301 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.5090000000000001e-06 2023-09-23T20:44:19.301 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.301 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.301 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:44:19.301 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.838010+0000", 2023-09-23T20:44:19.301 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.6254e-05 2023-09-23T20:44:19.301 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.301 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.302 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:44:19.302 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.838022+0000", 2023-09-23T20:44:19.302 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2218000000000001e-05 2023-09-23T20:44:19.302 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:44:19.302 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:44:19.302 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:44:19.302 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.302 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.302 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.36450.0:185330 9.8 9.d7918778 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e503)", 2023-09-23T20:44:19.302 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:44:18.717304+0000", 2023-09-23T20:44:19.303 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.54337830099999995, 2023-09-23T20:44:19.303 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.54374353900000005, 2023-09-23T20:44:19.303 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:44:19.303 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "delayed", 2023-09-23T20:44:19.303 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:44:19.303 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.36450", 2023-09-23T20:44:19.303 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:44:19.303 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 185330 2023-09-23T20:44:19.303 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.303 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:44:19.303 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.303 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:44:19.304 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.717304+0000", 2023-09-23T20:44:19.304 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:44:19.304 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.304 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.304 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:44:19.304 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.717304+0000", 2023-09-23T20:44:19.304 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:44:19.304 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.304 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.304 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:44:19.304 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.717303+0000", 2023-09-23T20:44:19.305 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.999999 2023-09-23T20:44:19.305 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.305 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.305 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:44:19.305 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.717309+0000", 2023-09-23T20:44:19.305 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.6620000000000002e-06 2023-09-23T20:44:19.305 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.305 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.305 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:44:19.305 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.717319+0000", 2023-09-23T20:44:19.305 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.8970000000000003e-06 2023-09-23T20:44:19.306 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.306 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.306 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:44:19.306 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.717326+0000", 2023-09-23T20:44:19.306 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.5190000000000003e-06 2023-09-23T20:44:19.306 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.306 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.306 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:44:19.306 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.717379+0000", 2023-09-23T20:44:19.306 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.2589000000000001e-05 2023-09-23T20:44:19.306 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.306 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.306 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:44:19.307 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.717394+0000", 2023-09-23T20:44:19.307 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.4843e-05 2023-09-23T20:44:19.307 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:44:19.307 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:44:19.307 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:44:19.307 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.307 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.307 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.36450.0:185405 9.8 9.6fb15688 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e503)", 2023-09-23T20:44:19.307 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:44:18.813440+0000", 2023-09-23T20:44:19.307 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.44724222699999999, 2023-09-23T20:44:19.307 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.44764432300000001, 2023-09-23T20:44:19.307 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:44:19.307 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "delayed", 2023-09-23T20:44:19.308 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:44:19.308 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.36450", 2023-09-23T20:44:19.308 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:44:19.308 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 185405 2023-09-23T20:44:19.308 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.308 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:44:19.308 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.308 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:44:19.308 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.813440+0000", 2023-09-23T20:44:19.308 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:44:19.308 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.308 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.309 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:44:19.309 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.813440+0000", 2023-09-23T20:44:19.309 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:44:19.309 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.309 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.309 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:44:19.309 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.813440+0000", 2023-09-23T20:44:19.309 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:44:19.309 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.309 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.309 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:44:19.309 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.813442+0000", 2023-09-23T20:44:19.310 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.2989999999999999e-06 2023-09-23T20:44:19.310 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.310 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.310 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:44:19.310 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.813450+0000", 2023-09-23T20:44:19.310 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.7689999999999996e-06 2023-09-23T20:44:19.310 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.310 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.310 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:44:19.310 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.813455+0000", 2023-09-23T20:44:19.310 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.4229999999999996e-06 2023-09-23T20:44:19.310 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.311 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.311 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:44:19.311 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.813525+0000", 2023-09-23T20:44:19.311 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.9319999999999994e-05 2023-09-23T20:44:19.311 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.311 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.311 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:44:19.311 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.813536+0000", 2023-09-23T20:44:19.311 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0865e-05 2023-09-23T20:44:19.311 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:44:19.311 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:44:19.311 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:44:19.312 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.312 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.312 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.36450.0:185407 9.8 9.40f2f428 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e503)", 2023-09-23T20:44:19.312 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:44:18.816212+0000", 2023-09-23T20:44:19.312 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.44447000199999998, 2023-09-23T20:44:19.312 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.44490930899999998, 2023-09-23T20:44:19.312 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:44:19.312 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "delayed", 2023-09-23T20:44:19.312 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:44:19.312 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.36450", 2023-09-23T20:44:19.312 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:44:19.312 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 185407 2023-09-23T20:44:19.313 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.313 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:44:19.313 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.313 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:44:19.313 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.816212+0000", 2023-09-23T20:44:19.313 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:44:19.313 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.313 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.313 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:44:19.313 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.816212+0000", 2023-09-23T20:44:19.313 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:44:19.314 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.314 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.314 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:44:19.314 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.816211+0000", 2023-09-23T20:44:19.314 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.999999 2023-09-23T20:44:19.314 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.314 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.314 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:44:19.314 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.816217+0000", 2023-09-23T20:44:19.314 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.4809999999999997e-06 2023-09-23T20:44:19.314 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.314 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.315 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:44:19.315 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.816230+0000", 2023-09-23T20:44:19.315 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2943e-05 2023-09-23T20:44:19.315 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.315 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.315 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:44:19.315 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.816237+0000", 2023-09-23T20:44:19.315 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.4800000000000004e-06 2023-09-23T20:44:19.315 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.315 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.315 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:44:19.315 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.816296+0000", 2023-09-23T20:44:19.316 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.8839999999999999e-05 2023-09-23T20:44:19.316 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.316 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.316 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:44:19.316 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.816310+0000", 2023-09-23T20:44:19.316 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3747999999999999e-05 2023-09-23T20:44:19.316 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:44:19.316 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:44:19.316 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:44:19.316 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.316 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.317 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.36450.0:185334 9.8 9.99d653f8 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e503)", 2023-09-23T20:44:19.317 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:44:18.722946+0000", 2023-09-23T20:44:19.317 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.53773692299999998, 2023-09-23T20:44:19.317 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.53821371500000004, 2023-09-23T20:44:19.317 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:44:19.317 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "delayed", 2023-09-23T20:44:19.317 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:44:19.317 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.36450", 2023-09-23T20:44:19.317 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:44:19.317 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 185334 2023-09-23T20:44:19.317 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.317 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:44:19.318 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.318 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:44:19.318 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.722946+0000", 2023-09-23T20:44:19.318 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:44:19.318 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.318 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.318 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:44:19.318 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.722946+0000", 2023-09-23T20:44:19.318 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:44:19.318 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.318 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.319 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:44:19.319 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.722944+0000", 2023-09-23T20:44:19.319 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:44:19.319 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.319 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.319 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:44:19.319 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.722951+0000", 2023-09-23T20:44:19.319 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.0520000000000004e-06 2023-09-23T20:44:19.319 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.319 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.319 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:44:19.319 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.722976+0000", 2023-09-23T20:44:19.319 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.5139000000000001e-05 2023-09-23T20:44:19.320 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.320 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.320 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:44:19.320 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.722985+0000", 2023-09-23T20:44:19.320 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.5869999999999993e-06 2023-09-23T20:44:19.320 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.320 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.320 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:44:19.320 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.724318+0000", 2023-09-23T20:44:19.320 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.001333356 2023-09-23T20:44:19.320 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.320 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.321 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:44:19.321 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.724329+0000", 2023-09-23T20:44:19.321 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0661e-05 2023-09-23T20:44:19.321 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:44:19.321 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:44:19.321 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:44:19.321 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.321 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.321 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.36450.0:185337 9.8 9.430bfff8 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e503)", 2023-09-23T20:44:19.321 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:44:18.723328+0000", 2023-09-23T20:44:19.321 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.537354624, 2023-09-23T20:44:19.321 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.53786843100000004, 2023-09-23T20:44:19.322 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:44:19.322 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "delayed", 2023-09-23T20:44:19.322 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:44:19.322 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.36450", 2023-09-23T20:44:19.322 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:44:19.322 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 185337 2023-09-23T20:44:19.322 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.322 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:44:19.322 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.322 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:44:19.322 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.723328+0000", 2023-09-23T20:44:19.322 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:44:19.322 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.323 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.323 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:44:19.323 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.723328+0000", 2023-09-23T20:44:19.323 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:44:19.323 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.323 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.323 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:44:19.323 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.723327+0000", 2023-09-23T20:44:19.323 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.999999 2023-09-23T20:44:19.323 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.323 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.323 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:44:19.324 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.723332+0000", 2023-09-23T20:44:19.324 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.7079999999999996e-06 2023-09-23T20:44:19.324 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.324 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.324 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:44:19.324 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.723344+0000", 2023-09-23T20:44:19.324 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2031000000000001e-05 2023-09-23T20:44:19.324 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.324 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.324 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:44:19.324 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.723352+0000", 2023-09-23T20:44:19.324 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.8800000000000008e-06 2023-09-23T20:44:19.325 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.325 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.325 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:44:19.325 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.724351+0000", 2023-09-23T20:44:19.325 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00099955399999999994 2023-09-23T20:44:19.325 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.325 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.325 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:44:19.325 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.724359+0000", 2023-09-23T20:44:19.325 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.5449999999999998e-06 2023-09-23T20:44:19.325 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:44:19.325 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:44:19.326 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:44:19.326 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.326 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.326 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.36450.0:185447 9.8 9.43c58ee8 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e503)", 2023-09-23T20:44:19.326 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:44:19.023152+0000", 2023-09-23T20:44:19.326 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.23753070700000001, 2023-09-23T20:44:19.326 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.238082291, 2023-09-23T20:44:19.326 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:44:19.326 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "delayed", 2023-09-23T20:44:19.326 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:44:19.326 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.36450", 2023-09-23T20:44:19.327 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:44:19.327 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 185447 2023-09-23T20:44:19.327 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.327 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:44:19.327 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.327 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:44:19.327 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:19.023152+0000", 2023-09-23T20:44:19.327 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:44:19.327 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.327 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.327 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:44:19.327 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:19.023152+0000", 2023-09-23T20:44:19.328 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:44:19.328 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.328 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.328 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:44:19.328 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:19.023151+0000", 2023-09-23T20:44:19.328 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.999999 2023-09-23T20:44:19.328 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.328 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.328 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:44:19.328 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:19.023156+0000", 2023-09-23T20:44:19.328 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.1939999999999997e-06 2023-09-23T20:44:19.329 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.329 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.329 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:44:19.329 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:19.023168+0000", 2023-09-23T20:44:19.329 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2553e-05 2023-09-23T20:44:19.329 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.329 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.329 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:44:19.329 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:19.023175+0000", 2023-09-23T20:44:19.329 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.2870000000000001e-06 2023-09-23T20:44:19.329 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.330 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.330 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:44:19.330 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:19.023224+0000", 2023-09-23T20:44:19.330 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.9270000000000001e-05 2023-09-23T20:44:19.330 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.330 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.330 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:44:19.330 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:19.023234+0000", 2023-09-23T20:44:19.330 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.008e-05 2023-09-23T20:44:19.330 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:44:19.330 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:44:19.330 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:44:19.331 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.331 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.331 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.36450.0:185449 9.8 9.5dae5428 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e503)", 2023-09-23T20:44:19.331 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:44:19.029317+0000", 2023-09-23T20:44:19.331 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.23136509399999999, 2023-09-23T20:44:19.331 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.231956784, 2023-09-23T20:44:19.331 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:44:19.331 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "delayed", 2023-09-23T20:44:19.331 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:44:19.331 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.36450", 2023-09-23T20:44:19.331 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:44:19.332 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 185449 2023-09-23T20:44:19.332 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.332 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:44:19.332 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.332 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:44:19.332 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:19.029317+0000", 2023-09-23T20:44:19.332 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:44:19.332 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.332 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.332 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:44:19.332 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:19.029317+0000", 2023-09-23T20:44:19.333 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:44:19.333 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.333 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.333 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:44:19.333 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:19.029316+0000", 2023-09-23T20:44:19.333 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:44:19.333 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.333 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.333 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:44:19.333 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:19.029324+0000", 2023-09-23T20:44:19.333 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.4780000000000008e-06 2023-09-23T20:44:19.333 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.334 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.334 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:44:19.334 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:19.029341+0000", 2023-09-23T20:44:19.334 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.7090000000000001e-05 2023-09-23T20:44:19.334 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.334 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.334 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:44:19.334 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:19.029351+0000", 2023-09-23T20:44:19.334 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.594e-06 2023-09-23T20:44:19.334 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.334 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.334 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:44:19.335 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:19.029422+0000", 2023-09-23T20:44:19.335 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.0914000000000006e-05 2023-09-23T20:44:19.335 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.335 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.335 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:44:19.335 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:19.029437+0000", 2023-09-23T20:44:19.335 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.521e-05 2023-09-23T20:44:19.335 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:44:19.335 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:44:19.335 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:44:19.335 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.335 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.336 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.36450.0:185485 9.8 9.b82b0a18 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e503)", 2023-09-23T20:44:19.336 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:44:19.116998+0000", 2023-09-23T20:44:19.336 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.14368439199999999, 2023-09-23T20:44:19.336 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.14431338399999999, 2023-09-23T20:44:19.336 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:44:19.336 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "delayed", 2023-09-23T20:44:19.336 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:44:19.336 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.36450", 2023-09-23T20:44:19.336 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:44:19.336 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 185485 2023-09-23T20:44:19.336 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.336 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:44:19.337 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.337 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:44:19.337 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:19.116998+0000", 2023-09-23T20:44:19.337 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:44:19.337 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.337 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.337 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:44:19.337 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:19.116998+0000", 2023-09-23T20:44:19.337 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:44:19.337 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.337 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.337 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:44:19.338 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:19.116996+0000", 2023-09-23T20:44:19.338 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:44:19.338 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.338 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.338 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:44:19.338 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:19.117004+0000", 2023-09-23T20:44:19.338 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.6340000000000001e-06 2023-09-23T20:44:19.338 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.338 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.338 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:44:19.338 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:19.117021+0000", 2023-09-23T20:44:19.338 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.7272e-05 2023-09-23T20:44:19.339 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.339 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.339 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:44:19.339 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:19.117031+0000", 2023-09-23T20:44:19.339 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.3200000000000006e-06 2023-09-23T20:44:19.339 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.339 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.339 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:44:19.339 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:19.117115+0000", 2023-09-23T20:44:19.339 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.4190000000000005e-05 2023-09-23T20:44:19.339 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:44:19.340 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:44:19.340 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:44:19.340 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:19.117127+0000", 2023-09-23T20:44:19.340 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2024e-05 2023-09-23T20:44:19.340 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:44:19.340 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:44:19.340 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:44:19.340 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:44:19.340 INFO:teuthology.orchestra.run.smithi018.stdout: ], 2023-09-23T20:44:19.340 INFO:teuthology.orchestra.run.smithi018.stdout: "num_ops": 16 2023-09-23T20:44:19.340 INFO:teuthology.orchestra.run.smithi018.stdout:} 2023-09-23T20:44:19.341 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:44:19.368 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:19.402 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:44:19.405 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:19.406 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:44:19.506 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:44:19.532 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:19.565 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:44:19.568 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:19.568 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:44:19.669 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:44:19.695 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:19.728 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:44:19.731 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:19.731 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:44:19.787 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:19 smithi018 ceph-mon[184144]: pgmap v1264: 119 pgs: 1 active+clean+remapped, 2 remapped+peering, 116 active+clean; 2.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 450 op/s; 11 MiB/s, 180 objects/s recovering 2023-09-23T20:44:19.787 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:19 smithi018 ceph-mon[184144]: failed to encode map e503 with expected crc 2023-09-23T20:44:19.787 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:19 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2207557701' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T20:44:19.787 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:19 smithi018 ceph-mon[184144]: osdmap e503: 8 total, 8 up, 8 in 2023-09-23T20:44:19.787 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:19 smithi018 ceph-mon[184144]: failed to encode map e503 with expected crc 2023-09-23T20:44:19.787 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:19 smithi018 ceph-mon[184144]: failed to encode map e503 with expected crc 2023-09-23T20:44:19.787 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:19 smithi018 ceph-mon[179647]: pgmap v1264: 119 pgs: 1 active+clean+remapped, 2 remapped+peering, 116 active+clean; 2.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 450 op/s; 11 MiB/s, 180 objects/s recovering 2023-09-23T20:44:19.788 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:19 smithi018 ceph-mon[179647]: failed to encode map e503 with expected crc 2023-09-23T20:44:19.788 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:19 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2207557701' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T20:44:19.788 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:19 smithi018 ceph-mon[179647]: osdmap e503: 8 total, 8 up, 8 in 2023-09-23T20:44:19.788 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:19 smithi018 ceph-mon[179647]: failed to encode map e503 with expected crc 2023-09-23T20:44:19.788 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:19 smithi018 ceph-mon[179647]: failed to encode map e503 with expected crc 2023-09-23T20:44:19.832 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:44:19.864 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:19.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:19 smithi149 ceph-mon[170394]: pgmap v1264: 119 pgs: 1 active+clean+remapped, 2 remapped+peering, 116 active+clean; 2.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 450 op/s; 11 MiB/s, 180 objects/s recovering 2023-09-23T20:44:19.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:19 smithi149 ceph-mon[170394]: failed to encode map e503 with expected crc 2023-09-23T20:44:19.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:19 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2207557701' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T20:44:19.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:19 smithi149 ceph-mon[170394]: osdmap e503: 8 total, 8 up, 8 in 2023-09-23T20:44:19.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:19 smithi149 ceph-mon[170394]: failed to encode map e503 with expected crc 2023-09-23T20:44:19.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:19 smithi149 ceph-mon[170394]: failed to encode map e503 with expected crc 2023-09-23T20:44:19.983 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T20:44:20.009 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:44:20.012 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:20.012 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:44:20.113 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:44:20.144 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:20.244 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:44:20.247 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:20.247 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:44:20.348 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:44:20.379 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:20.637 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:20 smithi018 ceph-mon[179647]: 1.12 scrub starts 2023-09-23T20:44:20.638 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:20 smithi018 ceph-mon[179647]: 1.12 scrub ok 2023-09-23T20:44:20.638 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:20 smithi018 ceph-mon[179647]: failed to encode map e504 with expected crc 2023-09-23T20:44:20.638 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:20 smithi018 ceph-mon[179647]: osdmap e504: 8 total, 8 up, 8 in 2023-09-23T20:44:20.638 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:20 smithi018 ceph-mon[179647]: failed to encode map e504 with expected crc 2023-09-23T20:44:20.638 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:20 smithi018 ceph-mon[179647]: 1.18 scrub starts 2023-09-23T20:44:20.638 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:20 smithi018 ceph-mon[179647]: 1.18 scrub ok 2023-09-23T20:44:20.665 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:44:20.678 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:20.679 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:44:20.779 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:44:20.809 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:20.847 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:44:20.847 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:44:20.850 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:20.850 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:44:20.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:20 smithi018 ceph-mon[184144]: 1.12 scrub starts 2023-09-23T20:44:20.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:20 smithi018 ceph-mon[184144]: 1.12 scrub ok 2023-09-23T20:44:20.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:20 smithi018 ceph-mon[184144]: failed to encode map e504 with expected crc 2023-09-23T20:44:20.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:20 smithi018 ceph-mon[184144]: osdmap e504: 8 total, 8 up, 8 in 2023-09-23T20:44:20.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:20 smithi018 ceph-mon[184144]: failed to encode map e504 with expected crc 2023-09-23T20:44:20.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:20 smithi018 ceph-mon[184144]: 1.18 scrub starts 2023-09-23T20:44:20.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:20 smithi018 ceph-mon[184144]: 1.18 scrub ok 2023-09-23T20:44:20.951 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:44:20.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:20 smithi149 ceph-mon[170394]: 1.12 scrub starts 2023-09-23T20:44:20.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:20 smithi149 ceph-mon[170394]: 1.12 scrub ok 2023-09-23T20:44:20.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:20 smithi149 ceph-mon[170394]: failed to encode map e504 with expected crc 2023-09-23T20:44:20.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:20 smithi149 ceph-mon[170394]: osdmap e504: 8 total, 8 up, 8 in 2023-09-23T20:44:20.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:20 smithi149 ceph-mon[170394]: failed to encode map e504 with expected crc 2023-09-23T20:44:20.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:20 smithi149 ceph-mon[170394]: 1.18 scrub starts 2023-09-23T20:44:20.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:20 smithi149 ceph-mon[170394]: 1.18 scrub ok 2023-09-23T20:44:20.983 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:21.019 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:44:21.022 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:21.023 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:44:21.123 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:44:21.159 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:21.207 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:44:21.214 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:21.215 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:44:21.246 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:44:21.315 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:44:21.318 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:44:21.356 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:21.357 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:44:21.392 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:44:21.421 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:44:21.425 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:21.425 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:44:21.438 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:44:21.494 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:44:21.526 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:44:21.541 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:44:21.576 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:21.578 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:44:21.625 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:44:21.635 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:21.636 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:44:21.642 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:44:21.649 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:44:21.649 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:44:21.649 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:44:21.649 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:44:21.649 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:44:21.649 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:44:21.649 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:44:21.649 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:44:21.709 INFO:teuthology.orchestra.run.smithi018.stderr:nodeep-scrub is unset 2023-09-23T20:44:21.737 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:44:21.782 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:21.832 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:44:21.840 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:21.841 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:44:21.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:21 smithi018 ceph-mon[179647]: 1.10 scrub starts 2023-09-23T20:44:21.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:21 smithi018 ceph-mon[179647]: 1.10 scrub ok 2023-09-23T20:44:21.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:21 smithi018 ceph-mon[179647]: pgmap v1267: 119 pgs: 1 unknown, 1 peering, 1 active+clean+remapped, 2 remapped+peering, 114 active+clean; 2.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 351 op/s; 12 MiB/s, 182 objects/s recovering 2023-09-23T20:44:21.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:21 smithi018 ceph-mon[179647]: osdmap e505: 8 total, 8 up, 8 in 2023-09-23T20:44:21.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:21 smithi018 ceph-mon[179647]: 9.8 starting backfill to osd.3 from (463'4000,492'11326] MIN to 502'11761 2023-09-23T20:44:21.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:21 smithi018 ceph-mon[179647]: failed to encode map e505 with expected crc 2023-09-23T20:44:21.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:21 smithi018 ceph-mon[179647]: failed to encode map e505 with expected crc 2023-09-23T20:44:21.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:21 smithi018 ceph-mon[179647]: failed to encode map e505 with expected crc 2023-09-23T20:44:21.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:21 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1406368814' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:44:21.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:21 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:44:21.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:21 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3172097570' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:44:21.894 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:21 smithi018 ceph-mon[184144]: 1.10 scrub starts 2023-09-23T20:44:21.894 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:21 smithi018 ceph-mon[184144]: 1.10 scrub ok 2023-09-23T20:44:21.894 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:21 smithi018 ceph-mon[184144]: pgmap v1267: 119 pgs: 1 unknown, 1 peering, 1 active+clean+remapped, 2 remapped+peering, 114 active+clean; 2.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 351 op/s; 12 MiB/s, 182 objects/s recovering 2023-09-23T20:44:21.894 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:21 smithi018 ceph-mon[184144]: osdmap e505: 8 total, 8 up, 8 in 2023-09-23T20:44:21.894 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:21 smithi018 ceph-mon[184144]: 9.8 starting backfill to osd.3 from (463'4000,492'11326] MIN to 502'11761 2023-09-23T20:44:21.894 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:21 smithi018 ceph-mon[184144]: failed to encode map e505 with expected crc 2023-09-23T20:44:21.894 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:21 smithi018 ceph-mon[184144]: failed to encode map e505 with expected crc 2023-09-23T20:44:21.894 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:21 smithi018 ceph-mon[184144]: failed to encode map e505 with expected crc 2023-09-23T20:44:21.895 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:21 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1406368814' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:44:21.895 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:21 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:44:21.895 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:21 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3172097570' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:44:21.941 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:44:21.968 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:21 smithi149 ceph-mon[170394]: 1.10 scrub starts 2023-09-23T20:44:21.968 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:21 smithi149 ceph-mon[170394]: 1.10 scrub ok 2023-09-23T20:44:21.968 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:21 smithi149 ceph-mon[170394]: pgmap v1267: 119 pgs: 1 unknown, 1 peering, 1 active+clean+remapped, 2 remapped+peering, 114 active+clean; 2.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 351 op/s; 12 MiB/s, 182 objects/s recovering 2023-09-23T20:44:21.968 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:21 smithi149 ceph-mon[170394]: osdmap e505: 8 total, 8 up, 8 in 2023-09-23T20:44:21.968 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:21 smithi149 ceph-mon[170394]: 9.8 starting backfill to osd.3 from (463'4000,492'11326] MIN to 502'11761 2023-09-23T20:44:21.968 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:21 smithi149 ceph-mon[170394]: failed to encode map e505 with expected crc 2023-09-23T20:44:21.968 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:21 smithi149 ceph-mon[170394]: failed to encode map e505 with expected crc 2023-09-23T20:44:21.969 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:21 smithi149 ceph-mon[170394]: failed to encode map e505 with expected crc 2023-09-23T20:44:21.969 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:21 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1406368814' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:44:21.969 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:21 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:44:21.969 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:21 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3172097570' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:44:21.985 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:22.048 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:44:22.055 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:22.056 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:44:22.156 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:44:22.233 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:22.516 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:44:22.529 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:22.529 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:44:22.630 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:44:22.750 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:22.886 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:44:22.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:22 smithi018 ceph-mon[179647]: failed to encode map e505 with expected crc 2023-09-23T20:44:22.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:22 smithi018 ceph-mon[179647]: failed to encode map e505 with expected crc 2023-09-23T20:44:22.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:22 smithi018 ceph-mon[179647]: failed to encode map e505 with expected crc 2023-09-23T20:44:22.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:22 smithi018 ceph-mon[179647]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:44:22.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:22 smithi018 ceph-mon[179647]: Cluster is now healthy 2023-09-23T20:44:22.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:22 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:44:22.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:22 smithi018 ceph-mon[179647]: osdmap e506: 8 total, 8 up, 8 in 2023-09-23T20:44:22.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:22 smithi018 ceph-mon[179647]: failed to encode map e506 with expected crc 2023-09-23T20:44:22.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:22 smithi018 ceph-mon[179647]: failed to encode map e506 with expected crc 2023-09-23T20:44:22.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:22 smithi018 ceph-mon[179647]: failed to encode map e506 with expected crc 2023-09-23T20:44:22.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:22 smithi018 ceph-mon[184144]: failed to encode map e505 with expected crc 2023-09-23T20:44:22.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:22 smithi018 ceph-mon[184144]: failed to encode map e505 with expected crc 2023-09-23T20:44:22.894 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:22 smithi018 ceph-mon[184144]: failed to encode map e505 with expected crc 2023-09-23T20:44:22.894 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:22 smithi018 ceph-mon[184144]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:44:22.894 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:22 smithi018 ceph-mon[184144]: Cluster is now healthy 2023-09-23T20:44:22.894 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:22 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:44:22.894 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:22 smithi018 ceph-mon[184144]: osdmap e506: 8 total, 8 up, 8 in 2023-09-23T20:44:22.894 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:22 smithi018 ceph-mon[184144]: failed to encode map e506 with expected crc 2023-09-23T20:44:22.895 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:22 smithi018 ceph-mon[184144]: failed to encode map e506 with expected crc 2023-09-23T20:44:22.895 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:22 smithi018 ceph-mon[184144]: failed to encode map e506 with expected crc 2023-09-23T20:44:22.901 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:22.902 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:44:22.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:22 smithi149 ceph-mon[170394]: failed to encode map e505 with expected crc 2023-09-23T20:44:22.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:22 smithi149 ceph-mon[170394]: failed to encode map e505 with expected crc 2023-09-23T20:44:22.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:22 smithi149 ceph-mon[170394]: failed to encode map e505 with expected crc 2023-09-23T20:44:22.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:22 smithi149 ceph-mon[170394]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:44:22.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:22 smithi149 ceph-mon[170394]: Cluster is now healthy 2023-09-23T20:44:22.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:22 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:44:22.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:22 smithi149 ceph-mon[170394]: osdmap e506: 8 total, 8 up, 8 in 2023-09-23T20:44:22.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:22 smithi149 ceph-mon[170394]: failed to encode map e506 with expected crc 2023-09-23T20:44:22.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:22 smithi149 ceph-mon[170394]: failed to encode map e506 with expected crc 2023-09-23T20:44:22.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:22 smithi149 ceph-mon[170394]: failed to encode map e506 with expected crc 2023-09-23T20:44:23.002 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:44:23.041 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:23.086 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:44:23.090 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:23.091 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:44:23.191 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:44:23.231 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:23.286 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:44:23.290 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:23.291 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:44:23.391 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:44:23.429 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:23.496 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:44:23.499 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:23.500 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:44:23.601 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:44:23.665 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:23.703 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:44:23.714 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:23.715 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:44:23.810 INFO:teuthology.orchestra.run.smithi018.stdout:ERROR: (22) Invalid argument 2023-09-23T20:44:23.810 INFO:teuthology.orchestra.run.smithi018.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:44:23.815 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:44:23.852 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:23 smithi018 ceph-mon[179647]: failed to encode map e506 with expected crc 2023-09-23T20:44:23.852 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:23 smithi018 ceph-mon[179647]: pgmap v1270: 119 pgs: 1 active+remapped+backfilling, 1 peering, 1 active+clean+remapped, 1 remapped+peering, 115 active+clean; 2.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 212 op/s; 2686/126861 objects misplaced (2.117%); 3.0 MiB/s, 34 objects/s recovering 2023-09-23T20:44:23.852 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:23 smithi018 ceph-mon[184144]: failed to encode map e506 with expected crc 2023-09-23T20:44:23.852 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:23 smithi018 ceph-mon[184144]: pgmap v1270: 119 pgs: 1 active+remapped+backfilling, 1 peering, 1 active+clean+remapped, 1 remapped+peering, 115 active+clean; 2.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 212 op/s; 2686/126861 objects misplaced (2.117%); 3.0 MiB/s, 34 objects/s recovering 2023-09-23T20:44:23.887 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:23.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:23 smithi149 ceph-mon[170394]: failed to encode map e506 with expected crc 2023-09-23T20:44:23.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:23 smithi149 ceph-mon[170394]: pgmap v1270: 119 pgs: 1 active+remapped+backfilling, 1 peering, 1 active+clean+remapped, 1 remapped+peering, 115 active+clean; 2.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 212 op/s; 2686/126861 objects misplaced (2.117%); 3.0 MiB/s, 34 objects/s recovering 2023-09-23T20:44:24.237 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:44:24.241 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:24.242 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:44:24.246 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T20:44:24.342 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:44:24.421 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:24.513 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:44:24.519 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:24.520 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:44:24.621 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:44:24.622 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:44:24.626 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T20:44:24.656 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:24.697 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:44:24.701 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:24.701 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:44:24.726 INFO:teuthology.orchestra.run.smithi018.stderr:reweighted osd.2 to 1 (10000) 2023-09-23T20:44:24.802 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:44:24.850 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:24 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3175525298' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T20:44:24.850 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:24 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T20:44:24.850 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:24 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3175525298' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T20:44:24.851 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:24 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T20:44:24.936 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:24.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:24 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3175525298' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T20:44:24.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:24 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T20:44:25.141 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:44:24 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:44:24] "GET /metrics HTTP/1.1" 200 36584 "" "Prometheus/2.43.0" 2023-09-23T20:44:25.270 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:44:25.293 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:25.294 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:44:25.328 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd reweight 4 1 2023-09-23T20:44:25.395 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:44:25.424 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:25.464 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:44:25.468 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:25.468 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:44:25.499 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:44:25.568 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:44:25.613 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:44:25.666 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:25.694 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:44:25.867 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:44:25.874 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:25.875 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:44:25.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:25 smithi149 ceph-mon[170394]: pgmap v1271: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 2.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 300 op/s; 2686/124002 objects misplaced (2.166%); 1.4 MiB/s, 23 objects/s recovering 2023-09-23T20:44:25.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:25 smithi149 ceph-mon[170394]: failed to encode map e507 with expected crc 2023-09-23T20:44:25.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:25 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T20:44:25.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:25 smithi149 ceph-mon[170394]: osdmap e507: 8 total, 8 up, 8 in 2023-09-23T20:44:25.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:25 smithi149 ceph-mon[170394]: 1.2 scrub starts 2023-09-23T20:44:25.976 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:44:26.005 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:26.037 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:25 smithi018 ceph-mon[184144]: pgmap v1271: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 2.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 300 op/s; 2686/124002 objects misplaced (2.166%); 1.4 MiB/s, 23 objects/s recovering 2023-09-23T20:44:26.038 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:25 smithi018 ceph-mon[184144]: failed to encode map e507 with expected crc 2023-09-23T20:44:26.038 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:25 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T20:44:26.038 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:25 smithi018 ceph-mon[184144]: osdmap e507: 8 total, 8 up, 8 in 2023-09-23T20:44:26.038 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:25 smithi018 ceph-mon[184144]: 1.2 scrub starts 2023-09-23T20:44:26.038 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:25 smithi018 ceph-mon[179647]: pgmap v1271: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 2.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 300 op/s; 2686/124002 objects misplaced (2.166%); 1.4 MiB/s, 23 objects/s recovering 2023-09-23T20:44:26.038 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:25 smithi018 ceph-mon[179647]: failed to encode map e507 with expected crc 2023-09-23T20:44:26.038 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:25 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T20:44:26.038 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:25 smithi018 ceph-mon[179647]: osdmap e507: 8 total, 8 up, 8 in 2023-09-23T20:44:26.039 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:25 smithi018 ceph-mon[179647]: 1.2 scrub starts 2023-09-23T20:44:26.043 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:44:26.047 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:26.047 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:44:26.148 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:44:26.180 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:26.235 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:44:26.238 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:26.239 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:44:26.340 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:44:26.411 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:26.579 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:44:26.586 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:26.587 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:44:26.634 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:44:26.688 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:44:26.754 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:27.091 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:44:27.094 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:27.094 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:44:27.195 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:44:27.226 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:27.722 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:44:27.728 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:27 smithi018 ceph-mon[179647]: 1.2 scrub starts 2023-09-23T20:44:27.728 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:27 smithi018 ceph-mon[179647]: pgmap v1273: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 2.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 404 op/s; 2686/121593 objects misplaced (2.209%); 1.3 MiB/s, 21 objects/s recovering 2023-09-23T20:44:27.728 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:27 smithi018 ceph-mon[179647]: osdmap e508: 8 total, 8 up, 8 in 2023-09-23T20:44:27.728 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:27 smithi018 ceph-mon[179647]: failed to encode map e508 with expected crc 2023-09-23T20:44:27.728 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:27 smithi018 ceph-mon[184144]: 1.2 scrub starts 2023-09-23T20:44:27.728 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:27 smithi018 ceph-mon[184144]: pgmap v1273: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 2.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 404 op/s; 2686/121593 objects misplaced (2.209%); 1.3 MiB/s, 21 objects/s recovering 2023-09-23T20:44:27.728 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:27 smithi018 ceph-mon[184144]: osdmap e508: 8 total, 8 up, 8 in 2023-09-23T20:44:27.728 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:27 smithi018 ceph-mon[184144]: failed to encode map e508 with expected crc 2023-09-23T20:44:27.729 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:27.729 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:44:27.830 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:44:27.938 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:27.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:27 smithi149 ceph-mon[170394]: 1.2 scrub starts 2023-09-23T20:44:27.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:27 smithi149 ceph-mon[170394]: pgmap v1273: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 2.5 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 404 op/s; 2686/121593 objects misplaced (2.209%); 1.3 MiB/s, 21 objects/s recovering 2023-09-23T20:44:27.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:27 smithi149 ceph-mon[170394]: osdmap e508: 8 total, 8 up, 8 in 2023-09-23T20:44:27.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:27 smithi149 ceph-mon[170394]: failed to encode map e508 with expected crc 2023-09-23T20:44:28.247 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:44:28.252 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:28.253 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:44:28.353 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:44:28.394 INFO:teuthology.orchestra.run.smithi018.stdout:ERROR: (22) Invalid argument 2023-09-23T20:44:28.394 INFO:teuthology.orchestra.run.smithi018.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:44:28.428 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:28.559 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:28 smithi018 ceph-mon[179647]: 1.2 scrub starts 2023-09-23T20:44:28.559 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:28 smithi018 ceph-mon[179647]: 1.2 scrub ok 2023-09-23T20:44:28.559 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:28 smithi018 ceph-mon[179647]: osdmap e509: 8 total, 8 up, 8 in 2023-09-23T20:44:28.559 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:28 smithi018 ceph-mon[179647]: failed to encode map e509 with expected crc 2023-09-23T20:44:28.559 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:28 smithi018 ceph-mon[179647]: failed to encode map e509 with expected crc 2023-09-23T20:44:28.559 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:28 smithi018 ceph-mon[179647]: failed to encode map e509 with expected crc 2023-09-23T20:44:28.560 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:28 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3713002339' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:44:28.586 INFO:teuthology.orchestra.run.smithi018.stderr:noscrub is set 2023-09-23T20:44:28.727 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:44:28.737 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:28.737 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:44:28.811 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:28 smithi018 ceph-mon[184144]: 1.2 scrub starts 2023-09-23T20:44:28.811 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:28 smithi018 ceph-mon[184144]: 1.2 scrub ok 2023-09-23T20:44:28.811 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:28 smithi018 ceph-mon[184144]: osdmap e509: 8 total, 8 up, 8 in 2023-09-23T20:44:28.811 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:28 smithi018 ceph-mon[184144]: failed to encode map e509 with expected crc 2023-09-23T20:44:28.811 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:28 smithi018 ceph-mon[184144]: failed to encode map e509 with expected crc 2023-09-23T20:44:28.812 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:28 smithi018 ceph-mon[184144]: failed to encode map e509 with expected crc 2023-09-23T20:44:28.812 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:28 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3713002339' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:44:28.838 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:44:28.951 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:28.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:28 smithi149 ceph-mon[170394]: 1.2 scrub starts 2023-09-23T20:44:28.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:28 smithi149 ceph-mon[170394]: 1.2 scrub ok 2023-09-23T20:44:28.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:28 smithi149 ceph-mon[170394]: osdmap e509: 8 total, 8 up, 8 in 2023-09-23T20:44:28.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:28 smithi149 ceph-mon[170394]: failed to encode map e509 with expected crc 2023-09-23T20:44:28.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:28 smithi149 ceph-mon[170394]: failed to encode map e509 with expected crc 2023-09-23T20:44:28.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:28 smithi149 ceph-mon[170394]: failed to encode map e509 with expected crc 2023-09-23T20:44:28.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:28 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3713002339' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:44:28.974 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:44:28.974 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:44:28.975 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:44:28.975 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:44:28.975 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:44:29.006 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:44:29.049 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:44:29.071 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:44:29.082 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:29.083 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:44:29.092 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:44:29.100 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:44:29.100 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:44:29.100 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:44:29.100 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:44:29.100 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:44:29.100 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:44:29.100 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:44:29.100 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:44:29.184 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:44:29.220 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:29.275 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:44:29.280 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:29.281 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:44:29.381 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:44:29.456 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T20:44:29.457 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:29.626 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:44:29.631 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:29.631 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:44:29.637 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:29 smithi018 ceph-mon[179647]: pgmap v1276: 119 pgs: 1 active+clean+remapped, 118 active+clean; 2.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 588 op/s; 3.1 MiB/s, 51 objects/s recovering 2023-09-23T20:44:29.637 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:29 smithi018 ceph-mon[179647]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:44:29.638 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:29 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3713002339' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:44:29.638 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:29 smithi018 ceph-mon[179647]: osdmap e510: 8 total, 8 up, 8 in 2023-09-23T20:44:29.638 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:29 smithi018 ceph-mon[179647]: failed to encode map e510 with expected crc 2023-09-23T20:44:29.638 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:29 smithi018 ceph-mon[179647]: failed to encode map e510 with expected crc 2023-09-23T20:44:29.638 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:29 smithi018 ceph-mon[179647]: failed to encode map e510 with expected crc 2023-09-23T20:44:29.638 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:29 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/786417776' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:44:29.638 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:29 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:44:29.638 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:29 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:44:29.638 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:29 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2688790441' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T20:44:29.639 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:29 smithi018 ceph-mon[184144]: pgmap v1276: 119 pgs: 1 active+clean+remapped, 118 active+clean; 2.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 588 op/s; 3.1 MiB/s, 51 objects/s recovering 2023-09-23T20:44:29.639 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:29 smithi018 ceph-mon[184144]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:44:29.639 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:29 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3713002339' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:44:29.639 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:29 smithi018 ceph-mon[184144]: osdmap e510: 8 total, 8 up, 8 in 2023-09-23T20:44:29.639 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:29 smithi018 ceph-mon[184144]: failed to encode map e510 with expected crc 2023-09-23T20:44:29.639 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:29 smithi018 ceph-mon[184144]: failed to encode map e510 with expected crc 2023-09-23T20:44:29.639 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:29 smithi018 ceph-mon[184144]: failed to encode map e510 with expected crc 2023-09-23T20:44:29.639 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:29 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/786417776' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:44:29.639 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:29 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:44:29.639 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:29 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:44:29.639 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:29 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2688790441' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T20:44:29.732 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:44:29.797 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:29.856 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:44:29.866 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:29.867 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:44:29.967 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:44:29.969 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:29 smithi149 ceph-mon[170394]: pgmap v1276: 119 pgs: 1 active+clean+remapped, 118 active+clean; 2.4 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 588 op/s; 3.1 MiB/s, 51 objects/s recovering 2023-09-23T20:44:29.969 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:29 smithi149 ceph-mon[170394]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:44:29.969 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:29 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3713002339' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:44:29.969 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:29 smithi149 ceph-mon[170394]: osdmap e510: 8 total, 8 up, 8 in 2023-09-23T20:44:29.969 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:29 smithi149 ceph-mon[170394]: failed to encode map e510 with expected crc 2023-09-23T20:44:29.969 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:29 smithi149 ceph-mon[170394]: failed to encode map e510 with expected crc 2023-09-23T20:44:29.969 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:29 smithi149 ceph-mon[170394]: failed to encode map e510 with expected crc 2023-09-23T20:44:29.969 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:29 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/786417776' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:44:29.969 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:29 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:44:29.970 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:29 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:44:29.970 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:29 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2688790441' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T20:44:30.019 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:30.082 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:44:30.088 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:30.088 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:44:30.138 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:44:30.189 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:44:30.234 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:30.288 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:44:30.295 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:30.295 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:44:30.396 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:44:30.472 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:30.512 INFO:teuthology.orchestra.run.smithi018.stderr:reweighted osd.4 to 1 (10000) 2023-09-23T20:44:30.537 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:44:30.548 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:30.548 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:44:30.649 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:44:30.703 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:31.655 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T20:44:31.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:31 smithi149 ceph-mon[170394]: pgmap v1278: 119 pgs: 119 active+clean; 2.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 722 op/s; 3.5 MiB/s, 56 objects/s recovering 2023-09-23T20:44:31.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:31 smithi149 ceph-mon[170394]: failed to encode map e511 with expected crc 2023-09-23T20:44:31.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:31 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2688790441' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T20:44:31.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:31 smithi149 ceph-mon[170394]: osdmap e511: 8 total, 8 up, 8 in 2023-09-23T20:44:31.725 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:44:31.785 INFO:teuthology.orchestra.run.smithi149.stdout:{ 2023-09-23T20:44:31.785 INFO:teuthology.orchestra.run.smithi149.stdout: "ops": [], 2023-09-23T20:44:31.785 INFO:teuthology.orchestra.run.smithi149.stdout: "num_ops": 0 2023-09-23T20:44:31.785 INFO:teuthology.orchestra.run.smithi149.stdout:} 2023-09-23T20:44:31.798 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:31 smithi018 ceph-mon[179647]: pgmap v1278: 119 pgs: 119 active+clean; 2.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 722 op/s; 3.5 MiB/s, 56 objects/s recovering 2023-09-23T20:44:31.798 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:31 smithi018 ceph-mon[179647]: failed to encode map e511 with expected crc 2023-09-23T20:44:31.798 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:31 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2688790441' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T20:44:31.799 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:31 smithi018 ceph-mon[179647]: osdmap e511: 8 total, 8 up, 8 in 2023-09-23T20:44:31.806 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:31 smithi018 ceph-mon[184144]: pgmap v1278: 119 pgs: 119 active+clean; 2.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 722 op/s; 3.5 MiB/s, 56 objects/s recovering 2023-09-23T20:44:31.806 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:31 smithi018 ceph-mon[184144]: failed to encode map e511 with expected crc 2023-09-23T20:44:31.806 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:31 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2688790441' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T20:44:31.806 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:31 smithi018 ceph-mon[184144]: osdmap e511: 8 total, 8 up, 8 in 2023-09-23T20:44:32.294 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T20:44:32.938 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:44:32.944 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:32.944 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:44:33.002 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd reweight 6 1 2023-09-23T20:44:33.045 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:44:33.088 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:44:33.092 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:33.149 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:44:33.154 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:33.155 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:44:33.233 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:44:33.251 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:44:33.256 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:44:33.290 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:33.395 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:44:33.407 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:33.408 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:44:33.508 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:44:33.589 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:33.640 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:33 smithi149 ceph-mon[170394]: pgmap v1280: 119 pgs: 119 active+clean; 2.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 523 op/s 2023-09-23T20:44:33.721 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:44:33.731 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:33.731 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:44:33.832 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:44:33.882 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:33 smithi018 ceph-mon[179647]: pgmap v1280: 119 pgs: 119 active+clean; 2.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 523 op/s 2023-09-23T20:44:33.886 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:33.886 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:33 smithi018 ceph-mon[184144]: pgmap v1280: 119 pgs: 119 active+clean; 2.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 523 op/s 2023-09-23T20:44:34.118 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:44:34.121 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:34.121 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:44:34.222 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:44:34.285 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:34.456 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:44:34.473 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:34.473 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:44:34.574 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:44:34.648 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:34.744 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:44:34.754 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:34.754 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:44:34.762 INFO:teuthology.orchestra.run.smithi149.stdout:{ 2023-09-23T20:44:34.762 INFO:teuthology.orchestra.run.smithi149.stdout: "ops": [], 2023-09-23T20:44:34.762 INFO:teuthology.orchestra.run.smithi149.stdout: "complaint_time": 30, 2023-09-23T20:44:34.762 INFO:teuthology.orchestra.run.smithi149.stdout: "num_blocked_ops": 0 2023-09-23T20:44:34.762 INFO:teuthology.orchestra.run.smithi149.stdout:} 2023-09-23T20:44:34.855 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:44:34.925 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:35.161 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T20:44:35.206 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:44:34 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:44:34] "GET /metrics HTTP/1.1" 200 36547 "" "Prometheus/2.43.0" 2023-09-23T20:44:35.293 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:44:35.314 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:35.315 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:44:35.415 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:44:35.496 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:35.749 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:35 smithi018 ceph-mon[184144]: pgmap v1281: 119 pgs: 119 active+clean; 2.1 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 706 op/s 2023-09-23T20:44:35.749 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:35 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:44:35.750 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:35 smithi018 ceph-mon[179647]: pgmap v1281: 119 pgs: 119 active+clean; 2.1 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 706 op/s 2023-09-23T20:44:35.750 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:35 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:44:35.878 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:44:35.888 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:35.889 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:44:35.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:35 smithi149 ceph-mon[170394]: pgmap v1281: 119 pgs: 119 active+clean; 2.1 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 706 op/s 2023-09-23T20:44:35.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:35 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:44:35.989 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:44:36.023 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:44:36.029 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:44:36.060 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:36.175 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:44:36.181 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:36.181 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:44:36.282 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:44:36.338 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:36.399 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:44:36.418 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:36.419 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:44:36.520 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:44:36.576 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:36.893 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:44:36.900 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:36.900 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:44:37.001 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:44:37.068 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:37.121 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:44:37.124 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:37.125 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:44:37.225 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:44:37.309 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:37.420 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:44:37.431 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:37.432 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:44:37.501 INFO:teuthology.orchestra.run.smithi149.stdout:{ 2023-09-23T20:44:37.501 INFO:teuthology.orchestra.run.smithi149.stdout: "size": 20, 2023-09-23T20:44:37.501 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 600, 2023-09-23T20:44:37.501 INFO:teuthology.orchestra.run.smithi149.stdout: "ops": [ 2023-09-23T20:44:37.501 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.501 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35700.0:156899 7.1 7.f34c1291 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e353)", 2023-09-23T20:44:37.501 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:34:59.025405+0000", 2023-09-23T20:44:37.501 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 578.47195868100005, 2023-09-23T20:44:37.501 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.95359936199999995, 2023-09-23T20:44:37.502 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:44:37.502 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:44:37.502 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:44:37.502 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35700", 2023-09-23T20:44:37.502 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:44:37.502 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 156899 2023-09-23T20:44:37.502 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.502 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:44:37.502 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.502 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:44:37.503 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:59.025405+0000", 2023-09-23T20:44:37.503 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:37.503 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.503 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.503 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:44:37.503 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:59.025405+0000", 2023-09-23T20:44:37.503 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:37.503 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.503 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.503 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:44:37.504 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:59.025404+0000", 2023-09-23T20:44:37.504 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:44:37.504 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.504 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.504 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:44:37.504 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:59.025410+0000", 2023-09-23T20:44:37.504 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.2659999999999998e-06 2023-09-23T20:44:37.504 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.504 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.504 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:44:37.504 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:59.025427+0000", 2023-09-23T20:44:37.505 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.6750000000000001e-05 2023-09-23T20:44:37.505 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.505 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.505 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:44:37.505 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:59.025434+0000", 2023-09-23T20:44:37.505 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.0430000000000002e-06 2023-09-23T20:44:37.505 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.505 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.505 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:44:37.505 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:59.025493+0000", 2023-09-23T20:44:37.506 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.9514e-05 2023-09-23T20:44:37.506 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.506 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.506 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:44:37.506 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:59.025503+0000", 2023-09-23T20:44:37.506 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0042e-05 2023-09-23T20:44:37.506 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.506 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.506 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:44:37.506 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:59.978991+0000", 2023-09-23T20:44:37.507 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.953488055 2023-09-23T20:44:37.507 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.507 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.507 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:44:37.507 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:59.979004+0000", 2023-09-23T20:44:37.507 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.307e-05 2023-09-23T20:44:37.507 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:37.507 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:44:37.507 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:37.507 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.507 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.508 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35700.0:156906 7.1 7.7e9886e1 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e353)", 2023-09-23T20:44:37.508 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:34:59.029901+0000", 2023-09-23T20:44:37.508 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 578.46746280699995, 2023-09-23T20:44:37.508 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.94915789500000003, 2023-09-23T20:44:37.508 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:44:37.508 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:44:37.508 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:44:37.508 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35700", 2023-09-23T20:44:37.508 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:44:37.508 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 156906 2023-09-23T20:44:37.509 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.509 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:44:37.509 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.509 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:44:37.509 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:59.029901+0000", 2023-09-23T20:44:37.509 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:37.509 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.509 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.509 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:44:37.509 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:59.029901+0000", 2023-09-23T20:44:37.509 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:37.509 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.510 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.510 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:44:37.510 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:59.029900+0000", 2023-09-23T20:44:37.510 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:44:37.510 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.510 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.510 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:44:37.510 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:59.029909+0000", 2023-09-23T20:44:37.510 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.7830000000000004e-06 2023-09-23T20:44:37.510 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.510 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.511 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:44:37.511 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:59.029919+0000", 2023-09-23T20:44:37.511 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0606e-05 2023-09-23T20:44:37.511 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.511 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.511 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:44:37.511 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:59.029923+0000", 2023-09-23T20:44:37.511 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.1899999999999997e-06 2023-09-23T20:44:37.511 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.511 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.511 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:44:37.511 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:59.029966+0000", 2023-09-23T20:44:37.512 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.3061e-05 2023-09-23T20:44:37.512 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.512 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.512 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:44:37.512 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:59.029979+0000", 2023-09-23T20:44:37.512 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.273e-05 2023-09-23T20:44:37.512 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.512 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.512 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:44:37.512 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:59.979028+0000", 2023-09-23T20:44:37.512 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.94904833600000005 2023-09-23T20:44:37.512 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.513 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.513 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:44:37.513 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:59.979059+0000", 2023-09-23T20:44:37.513 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.1163e-05 2023-09-23T20:44:37.513 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:37.513 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:44:37.513 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:37.513 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.513 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.513 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35700.0:156911 7.1 7.493fcdd1 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e353)", 2023-09-23T20:44:37.513 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:34:59.038682+0000", 2023-09-23T20:44:37.513 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 578.45868146800001, 2023-09-23T20:44:37.514 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.94040694800000002, 2023-09-23T20:44:37.514 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:44:37.514 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:44:37.514 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:44:37.514 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35700", 2023-09-23T20:44:37.514 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:44:37.514 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 156911 2023-09-23T20:44:37.514 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.514 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:44:37.514 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.514 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:44:37.514 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:59.038682+0000", 2023-09-23T20:44:37.515 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:37.515 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.515 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.515 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:44:37.515 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:59.038682+0000", 2023-09-23T20:44:37.515 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:37.515 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.515 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.515 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:44:37.515 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:59.038681+0000", 2023-09-23T20:44:37.515 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:44:37.515 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.516 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.516 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:44:37.516 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:59.038687+0000", 2023-09-23T20:44:37.516 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.3629999999999999e-06 2023-09-23T20:44:37.516 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.516 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.516 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:44:37.516 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:59.038702+0000", 2023-09-23T20:44:37.516 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.4983999999999999e-05 2023-09-23T20:44:37.516 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.516 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.516 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:44:37.517 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:59.038708+0000", 2023-09-23T20:44:37.517 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.9710000000000001e-06 2023-09-23T20:44:37.517 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.517 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.517 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:44:37.517 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:59.038768+0000", 2023-09-23T20:44:37.517 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.0174999999999999e-05 2023-09-23T20:44:37.517 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.517 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.517 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:44:37.517 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:59.038780+0000", 2023-09-23T20:44:37.517 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1742e-05 2023-09-23T20:44:37.518 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.518 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.518 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:44:37.518 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:59.979080+0000", 2023-09-23T20:44:37.518 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.94029980999999996 2023-09-23T20:44:37.518 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.518 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.518 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:44:37.518 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:59.979089+0000", 2023-09-23T20:44:37.518 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.0710000000000001e-06 2023-09-23T20:44:37.518 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:37.519 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:44:37.519 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:37.519 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.519 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.519 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35700.0:156944 7.1 7.73c1b121 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e353)", 2023-09-23T20:44:37.519 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:34:59.099576+0000", 2023-09-23T20:44:37.519 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 578.39778770199996, 2023-09-23T20:44:37.519 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.87954308699999995, 2023-09-23T20:44:37.519 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:44:37.519 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:44:37.519 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:44:37.520 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35700", 2023-09-23T20:44:37.520 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3781442691", 2023-09-23T20:44:37.520 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 156944 2023-09-23T20:44:37.520 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.520 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:44:37.520 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.520 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:44:37.520 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:59.099576+0000", 2023-09-23T20:44:37.520 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:37.520 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.520 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.521 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:44:37.521 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:59.099576+0000", 2023-09-23T20:44:37.521 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:37.521 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.521 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.521 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:44:37.521 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:59.099575+0000", 2023-09-23T20:44:37.521 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:44:37.521 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.521 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.521 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:44:37.522 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:59.099581+0000", 2023-09-23T20:44:37.522 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.7270000000000001e-06 2023-09-23T20:44:37.522 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.522 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.522 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:44:37.522 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:59.099594+0000", 2023-09-23T20:44:37.522 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.2323e-05 2023-09-23T20:44:37.522 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.522 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.522 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:44:37.522 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:59.099601+0000", 2023-09-23T20:44:37.522 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.1550000000000001e-06 2023-09-23T20:44:37.523 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.523 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.523 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:44:37.523 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:59.099672+0000", 2023-09-23T20:44:37.523 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.1493000000000001e-05 2023-09-23T20:44:37.523 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.523 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.523 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:44:37.523 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:59.099685+0000", 2023-09-23T20:44:37.523 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.2201999999999999e-05 2023-09-23T20:44:37.523 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.523 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.524 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:44:37.524 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:59.979110+0000", 2023-09-23T20:44:37.524 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.87942509400000002 2023-09-23T20:44:37.524 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.524 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.524 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:44:37.524 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:34:59.979119+0000", 2023-09-23T20:44:37.524 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.3719999999999994e-06 2023-09-23T20:44:37.524 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:37.524 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:44:37.524 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:37.524 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.525 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.525 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:105803 8.12 8:4d1c432a:::benchmark_data_smithi018_341490_object16480:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e410)", 2023-09-23T20:44:37.525 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:38:26.699836+0000", 2023-09-23T20:44:37.525 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 370.79752810000002, 2023-09-23T20:44:37.525 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.85041256099999996, 2023-09-23T20:44:37.525 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:44:37.525 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:44:37.525 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:44:37.525 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:44:37.525 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:44:37.525 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 105803 2023-09-23T20:44:37.525 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.526 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:44:37.526 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.526 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:44:37.526 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:26.699836+0000", 2023-09-23T20:44:37.526 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:37.526 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.526 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.526 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:44:37.526 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:26.699836+0000", 2023-09-23T20:44:37.526 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:37.526 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.526 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.527 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:44:37.527 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:26.699832+0000", 2023-09-23T20:44:37.527 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999967 2023-09-23T20:44:37.527 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.527 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.527 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:44:37.527 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:26.699849+0000", 2023-09-23T20:44:37.527 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.7223000000000001e-05 2023-09-23T20:44:37.527 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.527 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.527 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:44:37.528 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:26.699870+0000", 2023-09-23T20:44:37.528 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.1005e-05 2023-09-23T20:44:37.528 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.528 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.528 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:44:37.528 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:26.699884+0000", 2023-09-23T20:44:37.528 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.3349e-05 2023-09-23T20:44:37.528 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.528 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.528 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:44:37.528 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:26.703306+0000", 2023-09-23T20:44:37.528 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0034221799999999999 2023-09-23T20:44:37.529 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.529 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.529 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:44:37.529 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:26.703327+0000", 2023-09-23T20:44:37.529 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.0890000000000002e-05 2023-09-23T20:44:37.529 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.529 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.529 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:44:37.529 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.504227+0000", 2023-09-23T20:44:37.529 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.800900163 2023-09-23T20:44:37.529 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.529 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.530 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for degraded object", 2023-09-23T20:44:37.530 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.507106+0000", 2023-09-23T20:44:37.530 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0028788500000000001 2023-09-23T20:44:37.530 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.530 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.530 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:44:37.530 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.543770+0000", 2023-09-23T20:44:37.530 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.036664344000000001 2023-09-23T20:44:37.530 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.530 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.530 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:44:37.530 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.543848+0000", 2023-09-23T20:44:37.531 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.8217e-05 2023-09-23T20:44:37.531 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.531 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.531 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:44:37.531 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.543938+0000", 2023-09-23T20:44:37.531 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.9777000000000005e-05 2023-09-23T20:44:37.531 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.531 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.531 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:44:37.531 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.545834+0000", 2023-09-23T20:44:37.531 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0018956579999999999 2023-09-23T20:44:37.531 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.532 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.532 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:37.532 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.546152+0000", 2023-09-23T20:44:37.532 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000318311 2023-09-23T20:44:37.532 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.532 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.532 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:37.532 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.549292+0000", 2023-09-23T20:44:37.532 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0031395870000000001 2023-09-23T20:44:37.532 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.532 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.532 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:44:37.533 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.549336+0000", 2023-09-23T20:44:37.533 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.3982e-05 2023-09-23T20:44:37.533 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.533 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.533 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:44:37.533 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.550248+0000", 2023-09-23T20:44:37.533 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00091245400000000004 2023-09-23T20:44:37.533 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:37.533 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:44:37.533 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:37.533 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.533 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.534 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:106049 8.14 8:2acb269a:::benchmark_data_smithi018_341490_object16726:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e416)", 2023-09-23T20:44:37.534 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:38:37.916629+0000", 2023-09-23T20:44:37.534 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 359.58073488500003, 2023-09-23T20:44:37.534 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.3484781940000001, 2023-09-23T20:44:37.534 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:44:37.534 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:44:37.534 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:44:37.534 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:44:37.534 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:44:37.534 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 106049 2023-09-23T20:44:37.534 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.535 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:44:37.535 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.535 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:44:37.535 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.916629+0000", 2023-09-23T20:44:37.535 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:37.535 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.535 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.535 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:44:37.535 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.916629+0000", 2023-09-23T20:44:37.535 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:37.535 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.535 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.536 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:44:37.536 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.916627+0000", 2023-09-23T20:44:37.536 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999981 2023-09-23T20:44:37.536 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.536 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.536 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:44:37.536 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.916636+0000", 2023-09-23T20:44:37.536 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.0790000000000007e-06 2023-09-23T20:44:37.536 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.536 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.536 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:44:37.536 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.916657+0000", 2023-09-23T20:44:37.537 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.0982e-05 2023-09-23T20:44:37.537 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.537 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.537 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:44:37.537 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.916665+0000", 2023-09-23T20:44:37.537 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.3289999999999994e-06 2023-09-23T20:44:37.537 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.537 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.537 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:44:37.537 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.916720+0000", 2023-09-23T20:44:37.537 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.5334999999999998e-05 2023-09-23T20:44:37.537 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.538 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.538 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:44:37.538 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.916731+0000", 2023-09-23T20:44:37.538 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0329000000000001e-05 2023-09-23T20:44:37.538 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.538 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.538 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:44:37.538 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.258803+0000", 2023-09-23T20:44:37.538 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.342072444 2023-09-23T20:44:37.538 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.538 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.539 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:44:37.539 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.258947+0000", 2023-09-23T20:44:37.539 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00014386600000000001 2023-09-23T20:44:37.539 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.539 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.539 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 2,5", 2023-09-23T20:44:37.539 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.259044+0000", 2023-09-23T20:44:37.539 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.7132000000000004e-05 2023-09-23T20:44:37.539 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.539 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.539 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:37.539 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.261960+0000", 2023-09-23T20:44:37.540 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.002915866 2023-09-23T20:44:37.540 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.540 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.540 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:44:37.540 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.261969+0000", 2023-09-23T20:44:37.540 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.2299999999999997e-06 2023-09-23T20:44:37.540 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.540 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.540 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:37.540 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.264756+0000", 2023-09-23T20:44:37.540 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0027866610000000002 2023-09-23T20:44:37.541 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.541 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.541 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:44:37.541 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.264798+0000", 2023-09-23T20:44:37.541 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.2058999999999998e-05 2023-09-23T20:44:37.541 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.541 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.541 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:44:37.541 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.265107+0000", 2023-09-23T20:44:37.541 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000308943 2023-09-23T20:44:37.541 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:37.542 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:44:37.542 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:37.542 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.542 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.542 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:106054 8.12 8:4dc1e34a:::benchmark_data_smithi018_341490_object16731:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e416)", 2023-09-23T20:44:37.542 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:38:37.939136+0000", 2023-09-23T20:44:37.542 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 359.558227407, 2023-09-23T20:44:37.542 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.3347287539999999, 2023-09-23T20:44:37.542 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:44:37.542 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:44:37.542 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:44:37.542 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:44:37.543 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:44:37.543 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 106054 2023-09-23T20:44:37.543 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.543 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:44:37.543 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.543 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:44:37.543 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.939136+0000", 2023-09-23T20:44:37.543 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:37.543 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.543 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.543 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:44:37.544 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.939136+0000", 2023-09-23T20:44:37.544 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:37.544 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.544 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.544 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:44:37.544 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.939135+0000", 2023-09-23T20:44:37.544 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:44:37.544 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.544 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.544 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:44:37.544 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.939141+0000", 2023-09-23T20:44:37.544 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.0850000000000002e-06 2023-09-23T20:44:37.545 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.545 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.545 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:44:37.545 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.939155+0000", 2023-09-23T20:44:37.545 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.3912e-05 2023-09-23T20:44:37.545 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.545 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.545 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:44:37.545 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.939160+0000", 2023-09-23T20:44:37.545 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.6289999999999998e-06 2023-09-23T20:44:37.545 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.545 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.546 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:44:37.546 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.939210+0000", 2023-09-23T20:44:37.546 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.9122999999999997e-05 2023-09-23T20:44:37.546 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.546 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.546 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:44:37.546 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.939218+0000", 2023-09-23T20:44:37.546 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.2309999999999996e-06 2023-09-23T20:44:37.546 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.546 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.546 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:44:37.546 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.260728+0000", 2023-09-23T20:44:37.547 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.321510314 2023-09-23T20:44:37.547 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.547 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.547 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:44:37.547 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.260841+0000", 2023-09-23T20:44:37.547 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000112938 2023-09-23T20:44:37.547 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.547 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.547 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:44:37.547 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.260955+0000", 2023-09-23T20:44:37.547 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000113584 2023-09-23T20:44:37.547 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.548 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.548 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:44:37.548 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.266252+0000", 2023-09-23T20:44:37.548 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0052969810000000001 2023-09-23T20:44:37.548 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.548 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.548 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:37.548 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.273531+0000", 2023-09-23T20:44:37.548 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0072792580000000003 2023-09-23T20:44:37.548 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.548 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.548 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:37.549 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.273639+0000", 2023-09-23T20:44:37.549 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000107676 2023-09-23T20:44:37.549 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.549 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.549 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:44:37.549 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.273672+0000", 2023-09-23T20:44:37.549 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.3679000000000001e-05 2023-09-23T20:44:37.549 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.549 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.549 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:44:37.549 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.273865+0000", 2023-09-23T20:44:37.549 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00019271100000000001 2023-09-23T20:44:37.550 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:37.550 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:44:37.550 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:37.550 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.550 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.550 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:106060 8.4 8:25362c51:::benchmark_data_smithi018_341490_object16737:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e416)", 2023-09-23T20:44:37.550 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:38:38.009403+0000", 2023-09-23T20:44:37.550 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 359.48796027600002, 2023-09-23T20:44:37.550 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.262631214, 2023-09-23T20:44:37.550 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:44:37.550 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:44:37.550 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:44:37.551 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:44:37.551 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:44:37.551 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 106060 2023-09-23T20:44:37.551 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.551 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:44:37.551 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.551 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:44:37.551 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.009403+0000", 2023-09-23T20:44:37.551 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:37.551 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.551 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.551 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:44:37.552 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.009403+0000", 2023-09-23T20:44:37.552 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:37.552 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.552 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.552 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:44:37.552 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.009402+0000", 2023-09-23T20:44:37.552 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:44:37.552 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.552 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.552 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:44:37.552 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.009407+0000", 2023-09-23T20:44:37.553 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.3259999999999997e-06 2023-09-23T20:44:37.553 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.553 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.553 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:44:37.553 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.009419+0000", 2023-09-23T20:44:37.553 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.2813000000000001e-05 2023-09-23T20:44:37.553 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.553 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.553 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:44:37.553 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.009425+0000", 2023-09-23T20:44:37.553 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.2839999999999997e-06 2023-09-23T20:44:37.553 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.554 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.554 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:44:37.554 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.009487+0000", 2023-09-23T20:44:37.554 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.2145999999999998e-05 2023-09-23T20:44:37.554 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.554 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.554 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:44:37.554 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.009500+0000", 2023-09-23T20:44:37.554 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.3382e-05 2023-09-23T20:44:37.554 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.554 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.554 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:44:37.555 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.261986+0000", 2023-09-23T20:44:37.555 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.2524858210000001 2023-09-23T20:44:37.555 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.555 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.555 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:44:37.555 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.262103+0000", 2023-09-23T20:44:37.555 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000116661 2023-09-23T20:44:37.555 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.555 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.555 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 2,5", 2023-09-23T20:44:37.555 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.262187+0000", 2023-09-23T20:44:37.555 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.4349000000000002e-05 2023-09-23T20:44:37.556 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.556 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.556 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:44:37.556 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.266297+0000", 2023-09-23T20:44:37.556 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0041101369999999998 2023-09-23T20:44:37.556 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.556 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.556 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:37.556 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.266347+0000", 2023-09-23T20:44:37.556 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.9561999999999997e-05 2023-09-23T20:44:37.556 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.556 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.557 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:37.557 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.271596+0000", 2023-09-23T20:44:37.557 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0052490719999999996 2023-09-23T20:44:37.557 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.557 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.557 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:44:37.557 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.271648+0000", 2023-09-23T20:44:37.557 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.2011000000000001e-05 2023-09-23T20:44:37.557 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.557 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.557 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:44:37.557 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.272035+0000", 2023-09-23T20:44:37.558 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000386702 2023-09-23T20:44:37.558 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:37.558 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:44:37.558 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:37.558 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.558 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.558 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:106077 8.2 8:405ecd88:::benchmark_data_smithi018_341490_object16754:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e416)", 2023-09-23T20:44:37.558 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:38:38.055215+0000", 2023-09-23T20:44:37.558 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 359.442148159, 2023-09-23T20:44:37.558 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.218947317, 2023-09-23T20:44:37.558 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:44:37.558 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:44:37.559 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:44:37.559 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:44:37.559 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:44:37.559 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 106077 2023-09-23T20:44:37.559 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.559 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:44:37.559 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.559 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:44:37.559 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.055215+0000", 2023-09-23T20:44:37.559 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:37.559 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.560 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.560 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:44:37.560 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.055215+0000", 2023-09-23T20:44:37.560 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:37.560 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.560 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.560 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:44:37.560 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.055214+0000", 2023-09-23T20:44:37.560 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:44:37.560 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.560 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.560 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:44:37.561 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.055221+0000", 2023-09-23T20:44:37.561 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.3150000000000003e-06 2023-09-23T20:44:37.561 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.561 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.561 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:44:37.561 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.055238+0000", 2023-09-23T20:44:37.561 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.6225000000000001e-05 2023-09-23T20:44:37.561 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.561 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.561 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:44:37.561 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.055244+0000", 2023-09-23T20:44:37.562 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.6780000000000003e-06 2023-09-23T20:44:37.562 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.562 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.562 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:44:37.562 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.055311+0000", 2023-09-23T20:44:37.562 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.6406000000000001e-05 2023-09-23T20:44:37.562 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.562 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.562 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:44:37.562 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.055328+0000", 2023-09-23T20:44:37.563 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.7190999999999999e-05 2023-09-23T20:44:37.563 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.563 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.563 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:44:37.563 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.259906+0000", 2023-09-23T20:44:37.563 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.2045777580000001 2023-09-23T20:44:37.563 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.563 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.563 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:44:37.563 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.259995+0000", 2023-09-23T20:44:37.563 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.9866999999999999e-05 2023-09-23T20:44:37.564 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.564 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.564 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:44:37.564 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.260085+0000", 2023-09-23T20:44:37.564 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.9098999999999995e-05 2023-09-23T20:44:37.564 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.564 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.564 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:44:37.564 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.266243+0000", 2023-09-23T20:44:37.564 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0061579579999999998 2023-09-23T20:44:37.564 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.564 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.565 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:37.565 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.273553+0000", 2023-09-23T20:44:37.565 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0073100300000000003 2023-09-23T20:44:37.565 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.565 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.565 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:37.565 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.273929+0000", 2023-09-23T20:44:37.565 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00037609299999999998 2023-09-23T20:44:37.565 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.565 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.565 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:44:37.565 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.273960+0000", 2023-09-23T20:44:37.566 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.1851999999999999e-05 2023-09-23T20:44:37.566 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.566 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.566 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:44:37.566 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.274163+0000", 2023-09-23T20:44:37.566 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000202303 2023-09-23T20:44:37.566 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:37.566 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:44:37.566 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:37.566 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.566 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.566 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:106082 8.2 8:43f6991f:::benchmark_data_smithi018_341490_object16759:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e416)", 2023-09-23T20:44:37.567 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:38:38.078328+0000", 2023-09-23T20:44:37.567 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 359.41903523500002, 2023-09-23T20:44:37.567 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.200252715, 2023-09-23T20:44:37.567 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:44:37.567 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:44:37.567 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:44:37.567 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:44:37.567 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:44:37.567 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 106082 2023-09-23T20:44:37.567 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.567 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:44:37.568 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.568 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:44:37.568 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.078328+0000", 2023-09-23T20:44:37.568 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:37.568 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.568 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.568 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:44:37.568 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.078328+0000", 2023-09-23T20:44:37.568 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:37.568 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.568 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.568 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:44:37.569 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.078327+0000", 2023-09-23T20:44:37.569 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999981 2023-09-23T20:44:37.569 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.569 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.569 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:44:37.569 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.078334+0000", 2023-09-23T20:44:37.569 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.4769999999999997e-06 2023-09-23T20:44:37.569 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.569 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.569 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:44:37.569 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.078353+0000", 2023-09-23T20:44:37.569 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.8457e-05 2023-09-23T20:44:37.570 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.570 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.570 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:44:37.570 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.078359+0000", 2023-09-23T20:44:37.570 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.6359999999999997e-06 2023-09-23T20:44:37.570 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.570 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.570 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:44:37.570 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.078428+0000", 2023-09-23T20:44:37.570 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.8532e-05 2023-09-23T20:44:37.570 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.570 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.571 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:44:37.571 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.078441+0000", 2023-09-23T20:44:37.571 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.2943e-05 2023-09-23T20:44:37.571 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.571 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.571 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:44:37.571 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.268537+0000", 2023-09-23T20:44:37.571 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.190096729 2023-09-23T20:44:37.571 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.571 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.571 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:44:37.571 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.268680+0000", 2023-09-23T20:44:37.572 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00014273500000000001 2023-09-23T20:44:37.572 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.572 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.572 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:44:37.572 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.268813+0000", 2023-09-23T20:44:37.572 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00013306700000000001 2023-09-23T20:44:37.572 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.572 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.572 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:44:37.572 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.272998+0000", 2023-09-23T20:44:37.572 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0041846360000000003 2023-09-23T20:44:37.572 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.573 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.573 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:37.573 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.274587+0000", 2023-09-23T20:44:37.573 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0015889490000000001 2023-09-23T20:44:37.573 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.573 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.573 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:37.573 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.278215+0000", 2023-09-23T20:44:37.573 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0036286130000000002 2023-09-23T20:44:37.573 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.573 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.573 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:44:37.574 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.278259+0000", 2023-09-23T20:44:37.574 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.3995000000000001e-05 2023-09-23T20:44:37.574 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.574 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.574 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:44:37.574 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.278581+0000", 2023-09-23T20:44:37.574 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00032174500000000002 2023-09-23T20:44:37.574 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:37.574 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:44:37.574 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:37.574 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.574 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.575 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:106090 8.4 8:202238a2:::benchmark_data_smithi018_341490_object16767:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e416)", 2023-09-23T20:44:37.575 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:38:38.108972+0000", 2023-09-23T20:44:37.575 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 359.38839154700003, 2023-09-23T20:44:37.575 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1634425150000001, 2023-09-23T20:44:37.575 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:44:37.575 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:44:37.575 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:44:37.575 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:44:37.575 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:44:37.575 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 106090 2023-09-23T20:44:37.575 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.575 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:44:37.576 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.576 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:44:37.576 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.108972+0000", 2023-09-23T20:44:37.576 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:37.576 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.576 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.576 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:44:37.576 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.108972+0000", 2023-09-23T20:44:37.576 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:37.576 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.576 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.576 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:44:37.577 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.108971+0000", 2023-09-23T20:44:37.577 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:44:37.577 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.577 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.577 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:44:37.577 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.108976+0000", 2023-09-23T20:44:37.577 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.4110000000000004e-06 2023-09-23T20:44:37.577 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.577 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.577 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:44:37.577 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.108989+0000", 2023-09-23T20:44:37.577 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.3166000000000001e-05 2023-09-23T20:44:37.578 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.578 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.578 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:44:37.578 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.108995+0000", 2023-09-23T20:44:37.578 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.4480000000000002e-06 2023-09-23T20:44:37.578 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.578 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.578 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:44:37.578 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.109033+0000", 2023-09-23T20:44:37.578 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.8402000000000003e-05 2023-09-23T20:44:37.578 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.578 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.579 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:44:37.579 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.109042+0000", 2023-09-23T20:44:37.579 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.5790000000000004e-06 2023-09-23T20:44:37.579 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.579 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.579 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:44:37.579 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.263413+0000", 2023-09-23T20:44:37.579 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1543713289999999 2023-09-23T20:44:37.579 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.579 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.579 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:44:37.579 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.263504+0000", 2023-09-23T20:44:37.580 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.0820000000000001e-05 2023-09-23T20:44:37.580 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.580 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.580 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 2,5", 2023-09-23T20:44:37.580 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.263583+0000", 2023-09-23T20:44:37.580 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.9167999999999998e-05 2023-09-23T20:44:37.580 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.580 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.580 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:37.580 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.266401+0000", 2023-09-23T20:44:37.580 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0028183430000000001 2023-09-23T20:44:37.580 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.581 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.581 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:44:37.581 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.266409+0000", 2023-09-23T20:44:37.581 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.0190000000000006e-06 2023-09-23T20:44:37.581 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.581 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.581 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:37.581 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.272098+0000", 2023-09-23T20:44:37.581 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0056886230000000003 2023-09-23T20:44:37.581 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.581 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.582 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:44:37.582 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.272162+0000", 2023-09-23T20:44:37.582 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.3738999999999994e-05 2023-09-23T20:44:37.582 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.582 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.582 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:44:37.582 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.272415+0000", 2023-09-23T20:44:37.583 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000252768 2023-09-23T20:44:37.583 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:37.583 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:44:37.583 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:37.583 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.583 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.583 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:106105 8.12 8:4ed2b003:::benchmark_data_smithi018_341490_object16782:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e416)", 2023-09-23T20:44:37.583 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:38:38.247356+0000", 2023-09-23T20:44:37.583 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 359.25000766599999, 2023-09-23T20:44:37.583 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.027019473, 2023-09-23T20:44:37.583 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:44:37.583 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:44:37.584 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:44:37.584 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:44:37.584 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:44:37.584 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 106105 2023-09-23T20:44:37.584 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.584 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:44:37.584 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.584 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:44:37.584 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.247356+0000", 2023-09-23T20:44:37.584 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:37.584 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.584 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.585 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:44:37.585 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.247356+0000", 2023-09-23T20:44:37.585 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:37.585 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.585 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.585 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:44:37.585 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.247355+0000", 2023-09-23T20:44:37.585 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:44:37.585 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.585 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.585 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:44:37.585 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.247360+0000", 2023-09-23T20:44:37.586 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.5809999999999996e-06 2023-09-23T20:44:37.586 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.586 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.586 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:44:37.586 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.247374+0000", 2023-09-23T20:44:37.586 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.3869999999999999e-05 2023-09-23T20:44:37.586 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.586 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.586 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:44:37.586 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.247380+0000", 2023-09-23T20:44:37.586 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.5670000000000002e-06 2023-09-23T20:44:37.587 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.587 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.587 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:44:37.587 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.247458+0000", 2023-09-23T20:44:37.587 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.7976e-05 2023-09-23T20:44:37.587 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.587 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.587 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:44:37.587 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.247467+0000", 2023-09-23T20:44:37.587 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.8230000000000006e-06 2023-09-23T20:44:37.587 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.587 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.588 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:44:37.588 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.262332+0000", 2023-09-23T20:44:37.588 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.014864722 2023-09-23T20:44:37.588 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.588 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.588 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:44:37.588 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.262420+0000", 2023-09-23T20:44:37.588 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.7597999999999997e-05 2023-09-23T20:44:37.588 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.588 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.588 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:44:37.588 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.262520+0000", 2023-09-23T20:44:37.589 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.9769e-05 2023-09-23T20:44:37.589 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.589 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.589 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:44:37.589 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.266258+0000", 2023-09-23T20:44:37.589 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.003738285 2023-09-23T20:44:37.589 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.589 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.589 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:37.589 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.273571+0000", 2023-09-23T20:44:37.589 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0073127560000000001 2023-09-23T20:44:37.589 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.590 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.590 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:37.590 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.274186+0000", 2023-09-23T20:44:37.590 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00061492000000000003 2023-09-23T20:44:37.590 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.590 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.590 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:44:37.590 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.274218+0000", 2023-09-23T20:44:37.590 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.2221000000000002e-05 2023-09-23T20:44:37.590 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.590 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.590 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:44:37.590 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.274375+0000", 2023-09-23T20:44:37.591 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000157708 2023-09-23T20:44:37.591 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:37.591 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:44:37.591 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:37.591 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.591 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.591 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:106118 8.4 8:27850836:::benchmark_data_smithi018_341490_object16795:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e416)", 2023-09-23T20:44:37.591 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:38:38.309923+0000", 2023-09-23T20:44:37.591 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 359.18744043700002, 2023-09-23T20:44:37.591 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.96280872399999995, 2023-09-23T20:44:37.591 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:44:37.593 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:44:37.593 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:44:37.593 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:44:37.593 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:44:37.593 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 106118 2023-09-23T20:44:37.593 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.593 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:44:37.593 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.594 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:44:37.594 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.309923+0000", 2023-09-23T20:44:37.594 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:37.594 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.594 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.594 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:44:37.594 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.309923+0000", 2023-09-23T20:44:37.594 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:37.594 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.594 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.594 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:44:37.594 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.309922+0000", 2023-09-23T20:44:37.595 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:44:37.595 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.595 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.595 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:44:37.595 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.309927+0000", 2023-09-23T20:44:37.595 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.2959999999999998e-06 2023-09-23T20:44:37.595 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.595 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.595 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:44:37.595 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.309941+0000", 2023-09-23T20:44:37.595 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.3375e-05 2023-09-23T20:44:37.595 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.596 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.596 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:44:37.596 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.309947+0000", 2023-09-23T20:44:37.596 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.7440000000000001e-06 2023-09-23T20:44:37.596 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.596 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.596 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:44:37.596 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.309999+0000", 2023-09-23T20:44:37.596 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.1266e-05 2023-09-23T20:44:37.596 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.596 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.596 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:44:37.597 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.310006+0000", 2023-09-23T20:44:37.597 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.6499999999999996e-06 2023-09-23T20:44:37.597 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.597 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.597 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:44:37.597 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.264766+0000", 2023-09-23T20:44:37.597 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.95475926600000005 2023-09-23T20:44:37.597 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.597 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.597 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:44:37.597 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.264896+0000", 2023-09-23T20:44:37.597 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000130514 2023-09-23T20:44:37.598 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.598 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.598 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 2,5", 2023-09-23T20:44:37.598 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.265017+0000", 2023-09-23T20:44:37.598 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000120581 2023-09-23T20:44:37.598 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.598 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.598 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:44:37.598 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.266595+0000", 2023-09-23T20:44:37.598 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.001578217 2023-09-23T20:44:37.598 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.598 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.599 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:37.599 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.267280+0000", 2023-09-23T20:44:37.599 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00068537800000000005 2023-09-23T20:44:37.599 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.599 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.599 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:37.599 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.272466+0000", 2023-09-23T20:44:37.599 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0051855180000000001 2023-09-23T20:44:37.599 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.599 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.599 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:44:37.599 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.272522+0000", 2023-09-23T20:44:37.600 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.5914e-05 2023-09-23T20:44:37.600 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.600 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.600 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:44:37.600 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.272732+0000", 2023-09-23T20:44:37.600 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00021019499999999999 2023-09-23T20:44:37.600 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:37.600 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:44:37.600 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:37.600 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.600 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.600 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:106123 8.2 8:435cd3c6:::benchmark_data_smithi018_341490_object16800:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e416)", 2023-09-23T20:44:37.601 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:38:38.326800+0000", 2023-09-23T20:44:37.601 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 359.17056347400001, 2023-09-23T20:44:37.601 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.95465794400000004, 2023-09-23T20:44:37.601 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:44:37.601 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:44:37.601 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:44:37.601 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:44:37.601 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:44:37.601 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 106123 2023-09-23T20:44:37.601 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.601 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:44:37.602 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.602 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:44:37.602 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.326800+0000", 2023-09-23T20:44:37.602 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:37.602 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.602 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.602 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:44:37.602 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.326800+0000", 2023-09-23T20:44:37.602 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:37.602 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.602 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.602 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:44:37.603 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.326799+0000", 2023-09-23T20:44:37.603 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:44:37.603 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.603 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.603 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:44:37.603 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.326805+0000", 2023-09-23T20:44:37.603 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.6259999999999997e-06 2023-09-23T20:44:37.603 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.603 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.603 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:44:37.603 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.326820+0000", 2023-09-23T20:44:37.604 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.4508000000000001e-05 2023-09-23T20:44:37.604 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.604 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.604 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:44:37.604 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.326826+0000", 2023-09-23T20:44:37.604 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.1709999999999999e-06 2023-09-23T20:44:37.604 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.604 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.604 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:44:37.604 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.326899+0000", 2023-09-23T20:44:37.604 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.2838999999999998e-05 2023-09-23T20:44:37.605 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.605 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.605 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:44:37.605 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.326913+0000", 2023-09-23T20:44:37.605 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.4022999999999999e-05 2023-09-23T20:44:37.605 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.605 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.605 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:44:37.605 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.270145+0000", 2023-09-23T20:44:37.605 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.94323180900000003 2023-09-23T20:44:37.605 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.606 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.606 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:44:37.606 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.270297+0000", 2023-09-23T20:44:37.606 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00015283500000000001 2023-09-23T20:44:37.606 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.606 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.606 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:44:37.606 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.270461+0000", 2023-09-23T20:44:37.606 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00016353799999999999 2023-09-23T20:44:37.606 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.606 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.606 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:44:37.607 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.273062+0000", 2023-09-23T20:44:37.607 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0026012800000000001 2023-09-23T20:44:37.607 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.607 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.607 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:37.607 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.276398+0000", 2023-09-23T20:44:37.607 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.003335321 2023-09-23T20:44:37.607 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.607 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.607 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:37.607 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.281194+0000", 2023-09-23T20:44:37.607 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0047966939999999998 2023-09-23T20:44:37.608 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.608 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.608 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:44:37.608 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.281226+0000", 2023-09-23T20:44:37.608 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.1965000000000003e-05 2023-09-23T20:44:37.608 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.608 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.608 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:44:37.608 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.281458+0000", 2023-09-23T20:44:37.608 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000231876 2023-09-23T20:44:37.608 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:37.608 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:44:37.609 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:37.609 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.609 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.609 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:106135 8.14 8:2e4b5b89:::benchmark_data_smithi018_341490_object16812:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e416)", 2023-09-23T20:44:37.609 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:38:38.358297+0000", 2023-09-23T20:44:37.609 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 359.13906630700001, 2023-09-23T20:44:37.609 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.91161518500000005, 2023-09-23T20:44:37.609 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:44:37.609 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:44:37.609 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:44:37.609 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:44:37.609 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:44:37.610 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 106135 2023-09-23T20:44:37.610 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.610 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:44:37.610 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.610 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:44:37.610 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.358297+0000", 2023-09-23T20:44:37.610 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:37.610 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.610 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.610 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:44:37.610 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.358297+0000", 2023-09-23T20:44:37.610 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:37.611 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.611 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.611 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:44:37.611 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.358294+0000", 2023-09-23T20:44:37.611 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999971 2023-09-23T20:44:37.611 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.611 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.611 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:44:37.611 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.358302+0000", 2023-09-23T20:44:37.611 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.0039999999999999e-06 2023-09-23T20:44:37.611 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.611 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.612 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:44:37.612 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.358329+0000", 2023-09-23T20:44:37.612 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.6804999999999998e-05 2023-09-23T20:44:37.612 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.612 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.612 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:44:37.612 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.358337+0000", 2023-09-23T20:44:37.612 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.276e-06 2023-09-23T20:44:37.612 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.612 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.612 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:44:37.612 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.358408+0000", 2023-09-23T20:44:37.613 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.0525000000000007e-05 2023-09-23T20:44:37.613 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.613 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.613 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:44:37.613 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.358419+0000", 2023-09-23T20:44:37.613 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.083e-05 2023-09-23T20:44:37.613 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.613 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.613 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:44:37.613 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.260630+0000", 2023-09-23T20:44:37.613 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.90221140300000002 2023-09-23T20:44:37.613 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.614 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.614 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:44:37.614 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.260747+0000", 2023-09-23T20:44:37.614 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000116555 2023-09-23T20:44:37.614 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.614 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.614 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 2,5", 2023-09-23T20:44:37.614 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.260886+0000", 2023-09-23T20:44:37.614 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00013910600000000001 2023-09-23T20:44:37.614 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.614 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.614 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:37.615 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.263408+0000", 2023-09-23T20:44:37.615 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0025218670000000001 2023-09-23T20:44:37.615 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.615 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.615 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:44:37.615 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.265119+0000", 2023-09-23T20:44:37.615 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.001711661 2023-09-23T20:44:37.615 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.615 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.615 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:37.615 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.269557+0000", 2023-09-23T20:44:37.615 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0044376390000000002 2023-09-23T20:44:37.616 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.616 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.616 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:44:37.616 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.269618+0000", 2023-09-23T20:44:37.616 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.0798999999999998e-05 2023-09-23T20:44:37.616 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.616 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.616 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:44:37.616 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.269913+0000", 2023-09-23T20:44:37.616 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00029461999999999999 2023-09-23T20:44:37.616 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:37.616 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:44:37.616 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:37.617 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.617 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.617 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_repop(client.36450.0:118950 9.c e476/455 9:3e7512c3:::benchmark_data_smithi018_381516_object5116:head v 476'7456, mlcod=476'7452)", 2023-09-23T20:44:37.617 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:42:47.232089+0000", 2023-09-23T20:44:37.617 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 110.265274297, 2023-09-23T20:44:37.617 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.035653698999999997, 2023-09-23T20:44:37.617 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:44:37.617 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:44:37.617 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:44:37.617 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.617 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:44:37.618 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:47.232089+0000", 2023-09-23T20:44:37.618 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:37.618 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.618 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.618 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:44:37.618 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:47.232089+0000", 2023-09-23T20:44:37.618 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:37.618 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.618 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.618 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:44:37.618 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:47.232089+0000", 2023-09-23T20:44:37.618 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967296 2023-09-23T20:44:37.619 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.619 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.619 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:44:37.619 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:47.232096+0000", 2023-09-23T20:44:37.619 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.1899999999999998e-06 2023-09-23T20:44:37.619 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.619 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.619 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:44:37.619 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:47.232101+0000", 2023-09-23T20:44:37.619 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.9699999999999998e-06 2023-09-23T20:44:37.619 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.619 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.620 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:44:37.620 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:47.232104+0000", 2023-09-23T20:44:37.620 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.9689999999999999e-06 2023-09-23T20:44:37.620 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.620 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.620 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:44:37.620 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:47.238269+0000", 2023-09-23T20:44:37.620 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0061642750000000003 2023-09-23T20:44:37.620 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.620 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.620 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:44:37.620 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:47.238287+0000", 2023-09-23T20:44:37.621 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.8706e-05 2023-09-23T20:44:37.621 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.621 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.621 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:44:37.621 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:47.267692+0000", 2023-09-23T20:44:37.621 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.029404652999999999 2023-09-23T20:44:37.621 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.621 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.621 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:44:37.621 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:47.267743+0000", 2023-09-23T20:44:37.621 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.1097000000000002e-05 2023-09-23T20:44:37.622 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:37.622 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:44:37.622 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:37.622 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.622 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.622 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_repop(client.36450.0:131373 9.c e478/455 9:3bcbfe7e:::benchmark_data_smithi018_381516_object17539:head v 478'8232, mlcod=478'8230)", 2023-09-23T20:44:37.622 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:43:03.468208+0000", 2023-09-23T20:44:37.622 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 94.029155168000003, 2023-09-23T20:44:37.622 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.054668136999999999, 2023-09-23T20:44:37.622 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:44:37.622 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:44:37.622 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:44:37.623 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.623 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:44:37.623 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:03.468208+0000", 2023-09-23T20:44:37.623 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:37.623 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.623 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.623 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:44:37.623 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:03.468208+0000", 2023-09-23T20:44:37.623 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:37.623 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.623 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.623 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:44:37.624 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:03.468208+0000", 2023-09-23T20:44:37.624 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:44:37.624 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.624 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.624 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:44:37.624 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:03.468214+0000", 2023-09-23T20:44:37.624 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.2620000000000004e-06 2023-09-23T20:44:37.624 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.624 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.624 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:44:37.624 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:03.468224+0000", 2023-09-23T20:44:37.625 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.7270000000000002e-06 2023-09-23T20:44:37.625 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.625 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.625 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:44:37.625 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:03.468230+0000", 2023-09-23T20:44:37.625 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.4989999999999999e-06 2023-09-23T20:44:37.625 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.625 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.625 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:44:37.625 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:03.471230+0000", 2023-09-23T20:44:37.625 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0029999390000000001 2023-09-23T20:44:37.626 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.626 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.626 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:44:37.626 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:03.471267+0000", 2023-09-23T20:44:37.626 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.6650000000000003e-05 2023-09-23T20:44:37.626 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.626 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.626 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:44:37.626 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:03.522841+0000", 2023-09-23T20:44:37.626 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.051573594 2023-09-23T20:44:37.626 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.627 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.627 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:44:37.627 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:03.522877+0000", 2023-09-23T20:44:37.627 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.6053000000000002e-05 2023-09-23T20:44:37.627 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:37.627 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:44:37.627 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:37.627 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.627 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.627 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_repop(client.36450.0:131376 9.d e478/472 9:bb4a56ee:::benchmark_data_smithi018_381516_object17542:head v 478'8296, mlcod=478'8294)", 2023-09-23T20:44:37.627 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:43:03.469304+0000", 2023-09-23T20:44:37.627 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 94.028059987999995, 2023-09-23T20:44:37.628 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.056257667999999997, 2023-09-23T20:44:37.628 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:44:37.628 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:44:37.628 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:44:37.628 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.628 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:44:37.628 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:03.469304+0000", 2023-09-23T20:44:37.628 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:37.628 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.628 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.628 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:44:37.628 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:03.469304+0000", 2023-09-23T20:44:37.629 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:37.629 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.629 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.629 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:44:37.629 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:03.469303+0000", 2023-09-23T20:44:37.629 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:44:37.629 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.629 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.629 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:44:37.629 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:03.469309+0000", 2023-09-23T20:44:37.629 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.3240000000000002e-06 2023-09-23T20:44:37.629 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.630 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.630 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:44:37.630 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:03.469332+0000", 2023-09-23T20:44:37.630 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.3458000000000001e-05 2023-09-23T20:44:37.630 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.630 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.630 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:44:37.630 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:03.469338+0000", 2023-09-23T20:44:37.630 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.2940000000000002e-06 2023-09-23T20:44:37.630 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.630 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.630 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:44:37.631 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:03.469641+0000", 2023-09-23T20:44:37.631 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000302863 2023-09-23T20:44:37.631 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.631 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.631 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:44:37.631 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:03.469658+0000", 2023-09-23T20:44:37.631 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.6481999999999999e-05 2023-09-23T20:44:37.631 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.631 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.631 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:44:37.631 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:03.525530+0000", 2023-09-23T20:44:37.631 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.055872217000000002 2023-09-23T20:44:37.632 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.632 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.632 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:44:37.632 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:03.525561+0000", 2023-09-23T20:44:37.632 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.1401e-05 2023-09-23T20:44:37.632 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:37.632 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:44:37.632 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:37.632 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.632 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.632 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_repop(client.36450.0:131378 9.d e478/472 9:bdf5ae4f:::benchmark_data_smithi018_381516_object17544:head v 478'8297, mlcod=478'8294)", 2023-09-23T20:44:37.632 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:43:03.472694+0000", 2023-09-23T20:44:37.633 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 94.024669196000005, 2023-09-23T20:44:37.633 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.055612288000000003, 2023-09-23T20:44:37.633 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:44:37.633 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:44:37.633 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:44:37.633 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.633 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:44:37.633 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:03.472694+0000", 2023-09-23T20:44:37.633 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:37.633 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.633 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.633 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:44:37.634 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:03.472694+0000", 2023-09-23T20:44:37.634 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:37.634 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.634 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.634 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:44:37.634 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:03.472694+0000", 2023-09-23T20:44:37.634 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:44:37.634 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.634 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.634 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:44:37.634 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:03.472700+0000", 2023-09-23T20:44:37.634 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.4790000000000001e-06 2023-09-23T20:44:37.635 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.635 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.635 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:44:37.635 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:03.472710+0000", 2023-09-23T20:44:37.635 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.7249999999999992e-06 2023-09-23T20:44:37.635 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.635 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.635 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:44:37.635 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:03.472716+0000", 2023-09-23T20:44:37.635 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.4639999999999997e-06 2023-09-23T20:44:37.635 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.635 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.636 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:44:37.636 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:03.472784+0000", 2023-09-23T20:44:37.636 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.8369999999999998e-05 2023-09-23T20:44:37.636 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.636 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.636 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:44:37.636 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:03.472803+0000", 2023-09-23T20:44:37.636 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.8640000000000001e-05 2023-09-23T20:44:37.636 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.636 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.636 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:44:37.636 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:03.528280+0000", 2023-09-23T20:44:37.637 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.055476909999999997 2023-09-23T20:44:37.637 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.637 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.637 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:44:37.637 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:03.528307+0000", 2023-09-23T20:44:37.637 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.7211000000000001e-05 2023-09-23T20:44:37.637 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:37.637 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:44:37.637 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:37.637 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.637 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.637 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "pg_backfill(progress 9.d e 478/478 lb 9:b79deb11:::benchmark_data_smithi018_381516_object108753:head)", 2023-09-23T20:44:37.638 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:43:03.480481+0000", 2023-09-23T20:44:37.638 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 94.016882518000003, 2023-09-23T20:44:37.638 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.045085508000000003, 2023-09-23T20:44:37.638 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:44:37.638 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "started", 2023-09-23T20:44:37.638 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:44:37.638 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.638 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:44:37.638 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:03.480481+0000", 2023-09-23T20:44:37.638 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:37.638 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.638 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.639 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:44:37.639 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:03.480481+0000", 2023-09-23T20:44:37.639 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:37.639 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.639 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.639 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:44:37.639 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:03.480480+0000", 2023-09-23T20:44:37.639 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:44:37.639 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.639 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.639 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:44:37.639 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:03.480492+0000", 2023-09-23T20:44:37.640 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.2109e-05 2023-09-23T20:44:37.640 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.640 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.640 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:44:37.640 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:03.522791+0000", 2023-09-23T20:44:37.640 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.042299032 2023-09-23T20:44:37.640 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.640 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.640 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:44:37.640 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:03.522806+0000", 2023-09-23T20:44:37.640 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.5023e-05 2023-09-23T20:44:37.640 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.641 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.641 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:44:37.641 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:03.525482+0000", 2023-09-23T20:44:37.641 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0026760640000000001 2023-09-23T20:44:37.641 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.641 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.641 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:44:37.641 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:03.525503+0000", 2023-09-23T20:44:37.641 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.0358000000000001e-05 2023-09-23T20:44:37.641 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:37.641 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:37.641 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:44:37.641 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:03.525567+0000", 2023-09-23T20:44:37.642 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.3927e-05 2023-09-23T20:44:37.642 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:37.642 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:44:37.642 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:37.642 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:37.642 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:44:37.642 INFO:teuthology.orchestra.run.smithi149.stdout:} 2023-09-23T20:44:37.642 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:44:37.643 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:44:37.643 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:37 smithi018 ceph-mon[184144]: pgmap v1282: 119 pgs: 119 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 712 op/s 2023-09-23T20:44:37.644 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:37 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3264974194' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:44:37.644 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:37 smithi018 ceph-mon[179647]: pgmap v1282: 119 pgs: 119 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 712 op/s 2023-09-23T20:44:37.644 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:37 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3264974194' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:44:37.644 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:44:37.662 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:44:37.709 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:44:37.717 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:37.753 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:44:37.787 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:44:37.798 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:44:37.805 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:37.805 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:44:37.837 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:44:37.878 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:44:37.889 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:44:37.889 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:44:37.889 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:44:37.889 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:44:37.889 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:44:37.890 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:44:37.890 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:44:37.890 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:44:37.906 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:44:37.958 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:37 smithi149 ceph-mon[170394]: pgmap v1282: 119 pgs: 119 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 712 op/s 2023-09-23T20:44:37.958 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:37 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3264974194' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:44:37.976 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:37.991 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T20:44:38.292 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:44:38.316 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:38.317 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:44:38.418 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:44:38.500 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:38.563 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:44:38.572 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:38.573 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:44:38.606 INFO:teuthology.orchestra.run.smithi018.stderr:reweighted osd.6 to 1 (10000) 2023-09-23T20:44:38.607 INFO:teuthology.orchestra.run.smithi018.stderr:nodeep-scrub is set 2023-09-23T20:44:38.673 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:44:38.722 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:38.744 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:44:38.781 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:44:38.786 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:38.787 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:44:38.787 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:38 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3887378669' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:44:38.787 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:38 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:44:38.787 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:38 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2226590039' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T20:44:38.888 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:44:38.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:38 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3887378669' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:44:38.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:38 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:44:38.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:38 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2226590039' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T20:44:38.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:38 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3887378669' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:44:38.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:38 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:44:38.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:38 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2226590039' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T20:44:38.963 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:39.276 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:44:39.281 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:39.281 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:44:39.345 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd reweight 5 1 2023-09-23T20:44:39.382 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:44:39.446 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:39.591 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:39 smithi018 ceph-mon[179647]: pgmap v1283: 119 pgs: 119 active+clean; 1.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 799 op/s 2023-09-23T20:44:39.592 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:39 smithi018 ceph-mon[179647]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:44:39.592 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:39 smithi018 ceph-mon[179647]: failed to encode map e512 with expected crc 2023-09-23T20:44:39.592 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:39 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:44:39.592 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:39 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2226590039' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T20:44:39.592 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:39 smithi018 ceph-mon[179647]: osdmap e512: 8 total, 8 up, 8 in 2023-09-23T20:44:39.702 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:44:39.708 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:39.709 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:44:39.809 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:44:39.860 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:39 smithi018 ceph-mon[184144]: pgmap v1283: 119 pgs: 119 active+clean; 1.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 799 op/s 2023-09-23T20:44:39.860 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:39 smithi018 ceph-mon[184144]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:44:39.861 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:39 smithi018 ceph-mon[184144]: failed to encode map e512 with expected crc 2023-09-23T20:44:39.861 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:39 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:44:39.861 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:39 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2226590039' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T20:44:39.861 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:39 smithi018 ceph-mon[184144]: osdmap e512: 8 total, 8 up, 8 in 2023-09-23T20:44:39.875 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:39.954 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:44:39.957 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:39.958 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:44:39.968 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:39 smithi149 ceph-mon[170394]: pgmap v1283: 119 pgs: 119 active+clean; 1.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 799 op/s 2023-09-23T20:44:39.968 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:39 smithi149 ceph-mon[170394]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:44:39.968 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:39 smithi149 ceph-mon[170394]: failed to encode map e512 with expected crc 2023-09-23T20:44:39.968 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:39 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:44:39.968 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:39 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2226590039' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T20:44:39.969 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:39 smithi149 ceph-mon[170394]: osdmap e512: 8 total, 8 up, 8 in 2023-09-23T20:44:40.058 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:44:40.101 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:40.244 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:44:40.314 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:44:40.321 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:44:40.322 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:40.323 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:44:40.424 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:44:40.487 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:40.545 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:44:40.551 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:40.551 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:44:40.627 INFO:teuthology.orchestra.run.smithi149.stdout:ERROR: (22) Invalid argument 2023-09-23T20:44:40.627 INFO:teuthology.orchestra.run.smithi149.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:44:40.653 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:44:40.744 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:41.063 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:44:41.069 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:41.070 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:44:41.103 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T20:44:41.171 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:44:41.256 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:41.351 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T20:44:41.382 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:44:41.389 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:41.389 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:44:41.490 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:44:41.536 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:41.617 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:44:41.631 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:41.631 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:44:41.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:41 smithi018 ceph-mon[184144]: pgmap v1285: 119 pgs: 119 active+clean; 1.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 896 op/s 2023-09-23T20:44:41.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:41 smithi018 ceph-mon[179647]: pgmap v1285: 119 pgs: 119 active+clean; 1.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 896 op/s 2023-09-23T20:44:41.732 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:44:41.785 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:44:41.817 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:41 smithi149 ceph-mon[170394]: pgmap v1285: 119 pgs: 119 active+clean; 1.8 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 896 op/s 2023-09-23T20:44:41.818 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:41.884 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:44:41.893 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:41.894 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:44:41.921 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:44:41.994 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:44:42.090 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:42.193 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:44:42.201 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:42.201 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:44:42.302 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:44:42.388 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:42.491 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:44:42.495 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:42.496 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:44:42.597 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:44:42.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:42 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3727909481' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T20:44:42.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:42 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T20:44:42.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:42 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3727909481' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T20:44:42.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:42 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T20:44:42.645 INFO:teuthology.orchestra.run.smithi018.stderr:reweighted osd.5 to 1 (10000) 2023-09-23T20:44:42.666 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:42.721 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:42 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3727909481' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T20:44:42.721 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:42 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T20:44:42.807 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:44:42.895 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:44:42.898 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:42.898 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:44:42.999 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:44:43.031 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:43.335 INFO:teuthology.orchestra.run.smithi149.stdout:ERROR: (22) Invalid argument 2023-09-23T20:44:43.335 INFO:teuthology.orchestra.run.smithi149.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:44:43.366 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:44:43.369 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:43.370 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:44:43.418 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-09-23T20:44:43.419 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:44:43.471 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:44:43.502 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:43.635 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:43 smithi018 ceph-mon[184144]: pgmap v1286: 119 pgs: 119 active+clean; 1.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 881 op/s 2023-09-23T20:44:43.635 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:43 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T20:44:43.635 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:43 smithi018 ceph-mon[184144]: osdmap e513: 8 total, 8 up, 8 in 2023-09-23T20:44:43.635 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:43 smithi018 ceph-mon[184144]: failed to encode map e513 with expected crc 2023-09-23T20:44:43.636 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:43 smithi018 ceph-mon[179647]: pgmap v1286: 119 pgs: 119 active+clean; 1.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 881 op/s 2023-09-23T20:44:43.636 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:43 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T20:44:43.636 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:43 smithi018 ceph-mon[179647]: osdmap e513: 8 total, 8 up, 8 in 2023-09-23T20:44:43.636 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:43 smithi018 ceph-mon[179647]: failed to encode map e513 with expected crc 2023-09-23T20:44:43.697 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T20:44:43.698 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:44:43.702 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:43.702 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:44:43.722 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:43 smithi149 ceph-mon[170394]: pgmap v1286: 119 pgs: 119 active+clean; 1.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 881 op/s 2023-09-23T20:44:43.722 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:43 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T20:44:43.722 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:43 smithi149 ceph-mon[170394]: osdmap e513: 8 total, 8 up, 8 in 2023-09-23T20:44:43.722 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:43 smithi149 ceph-mon[170394]: failed to encode map e513 with expected crc 2023-09-23T20:44:43.803 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:44:43.830 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:43.863 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:44:43.865 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:43.866 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:44:43.966 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:44:43.992 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:44.026 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:44:44.030 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:44.030 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:44:44.036 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:44:44.057 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:44:44.078 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:44:44.098 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:44:44.116 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:44:44.131 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:44:44.135 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:44:44.156 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:44:44.166 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:44.176 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:44:44.178 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:44:44.178 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:44:44.179 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:44:44.179 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:44:44.179 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:44:44.179 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:44:44.179 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:44:44.179 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:44:44.203 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:44:44.206 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:44.206 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:44:44.302 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:44:44.307 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:44:44.337 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:44.498 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:44:44.501 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:44.501 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:44:44.552 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:44:44.603 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:44:44.632 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:44.651 INFO:teuthology.orchestra.run.smithi018.stderr:noscrub is unset 2023-09-23T20:44:44.676 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:44:44.678 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:44.679 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:44:44.780 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:44:44.810 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:44.870 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:44 smithi149 ceph-mon[170394]: failed to encode map e514 with expected crc 2023-09-23T20:44:44.870 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:44 smithi149 ceph-mon[170394]: osdmap e514: 8 total, 8 up, 8 in 2023-09-23T20:44:44.870 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:44 smithi149 ceph-mon[170394]: failed to encode map e514 with expected crc 2023-09-23T20:44:44.871 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:44 smithi149 ceph-mon[170394]: failed to encode map e514 with expected crc 2023-09-23T20:44:44.871 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:44 smithi149 ceph-mon[170394]: failed to encode map e514 with expected crc 2023-09-23T20:44:44.871 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:44 smithi149 ceph-mon[170394]: failed to encode map e514 with expected crc 2023-09-23T20:44:44.871 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:44 smithi149 ceph-mon[170394]: failed to encode map e514 with expected crc 2023-09-23T20:44:44.871 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:44 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1501136596' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:44:44.871 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:44 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3078359630' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:44:44.871 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:44 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:44:44.871 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:44 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:44:44.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:44 smithi018 ceph-mon[179647]: failed to encode map e514 with expected crc 2023-09-23T20:44:44.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:44 smithi018 ceph-mon[179647]: osdmap e514: 8 total, 8 up, 8 in 2023-09-23T20:44:44.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:44 smithi018 ceph-mon[179647]: failed to encode map e514 with expected crc 2023-09-23T20:44:44.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:44 smithi018 ceph-mon[179647]: failed to encode map e514 with expected crc 2023-09-23T20:44:44.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:44 smithi018 ceph-mon[179647]: failed to encode map e514 with expected crc 2023-09-23T20:44:44.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:44 smithi018 ceph-mon[179647]: failed to encode map e514 with expected crc 2023-09-23T20:44:44.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:44 smithi018 ceph-mon[179647]: failed to encode map e514 with expected crc 2023-09-23T20:44:44.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:44 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1501136596' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:44:44.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:44 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3078359630' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:44:44.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:44 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:44:44.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:44 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:44:44.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:44 smithi018 ceph-mon[184144]: failed to encode map e514 with expected crc 2023-09-23T20:44:44.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:44 smithi018 ceph-mon[184144]: osdmap e514: 8 total, 8 up, 8 in 2023-09-23T20:44:44.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:44 smithi018 ceph-mon[184144]: failed to encode map e514 with expected crc 2023-09-23T20:44:44.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:44 smithi018 ceph-mon[184144]: failed to encode map e514 with expected crc 2023-09-23T20:44:44.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:44 smithi018 ceph-mon[184144]: failed to encode map e514 with expected crc 2023-09-23T20:44:44.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:44 smithi018 ceph-mon[184144]: failed to encode map e514 with expected crc 2023-09-23T20:44:44.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:44 smithi018 ceph-mon[184144]: failed to encode map e514 with expected crc 2023-09-23T20:44:44.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:44 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1501136596' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:44:44.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:44 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3078359630' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:44:44.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:44 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:44:44.894 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:44 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:44:44.996 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:44:45.000 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:45.000 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:44:45.101 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:44:45.146 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:45.230 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:44:44 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:44:44] "GET /metrics HTTP/1.1" 200 36518 "" "Prometheus/2.43.0" 2023-09-23T20:44:45.275 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:44:45.287 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:45.288 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:44:45.389 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:44:45.461 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:45.549 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:44:45.557 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:45.557 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:44:45.658 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:44:45.727 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:45.743 INFO:teuthology.orchestra.run.smithi149.stdout:{ 2023-09-23T20:44:45.743 INFO:teuthology.orchestra.run.smithi149.stdout: "size": 20, 2023-09-23T20:44:45.743 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 600, 2023-09-23T20:44:45.743 INFO:teuthology.orchestra.run.smithi149.stdout: "ops": [ 2023-09-23T20:44:45.743 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.743 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:51043 8.0 8:0f697c15:::benchmark_data_smithi018_341490_object51042:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e374)", 2023-09-23T20:44:45.744 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:36:42.561436+0000", 2023-09-23T20:44:45.744 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 483.17779375499998, 2023-09-23T20:44:45.744 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.14033216600000001, 2023-09-23T20:44:45.744 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:44:45.744 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:44:45.744 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:44:45.744 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:44:45.744 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:44:45.744 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 51043 2023-09-23T20:44:45.744 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.744 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:44:45.744 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.745 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:44:45.745 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.561436+0000", 2023-09-23T20:44:45.745 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:45.745 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.745 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.745 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:44:45.745 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.561436+0000", 2023-09-23T20:44:45.745 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:45.745 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.745 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.745 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:44:45.745 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.561435+0000", 2023-09-23T20:44:45.746 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:44:45.746 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.746 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.746 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:44:45.746 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.561495+0000", 2023-09-23T20:44:45.746 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.0056999999999998e-05 2023-09-23T20:44:45.746 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.746 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.746 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:44:45.746 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.561507+0000", 2023-09-23T20:44:45.746 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1627999999999999e-05 2023-09-23T20:44:45.746 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.747 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.747 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:44:45.747 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.561513+0000", 2023-09-23T20:44:45.747 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.5969999999999997e-06 2023-09-23T20:44:45.747 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.747 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.747 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:44:45.747 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.561542+0000", 2023-09-23T20:44:45.747 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.8665000000000001e-05 2023-09-23T20:44:45.747 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.747 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.747 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:44:45.748 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.561627+0000", 2023-09-23T20:44:45.748 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.4596000000000001e-05 2023-09-23T20:44:45.748 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.748 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.748 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:44:45.748 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.561720+0000", 2023-09-23T20:44:45.748 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.3153999999999996e-05 2023-09-23T20:44:45.748 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.748 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.748 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:44:45.748 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.563303+0000", 2023-09-23T20:44:45.748 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0015831689999999999 2023-09-23T20:44:45.749 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.749 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.749 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:45.749 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.563545+0000", 2023-09-23T20:44:45.749 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00024228800000000001 2023-09-23T20:44:45.749 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.749 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.749 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:45.749 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.701697+0000", 2023-09-23T20:44:45.749 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.138151417 2023-09-23T20:44:45.749 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.749 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.750 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:44:45.750 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.701739+0000", 2023-09-23T20:44:45.750 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.2009999999999999e-05 2023-09-23T20:44:45.750 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.750 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.750 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:44:45.750 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.701768+0000", 2023-09-23T20:44:45.750 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.9332e-05 2023-09-23T20:44:45.750 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:45.750 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:44:45.750 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:45.751 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.751 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.751 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:29549 9.7 9:ea1cf7ea:::benchmark_data_smithi018_381516_object29548:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e459)", 2023-09-23T20:44:45.751 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:41:21.052150+0000", 2023-09-23T20:44:45.751 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 204.68707916100001, 2023-09-23T20:44:45.751 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.14682632800000001, 2023-09-23T20:44:45.751 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:44:45.751 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:44:45.751 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:44:45.751 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:44:45.751 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:44:45.752 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 29549 2023-09-23T20:44:45.752 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.752 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:44:45.752 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.752 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:44:45.752 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.052150+0000", 2023-09-23T20:44:45.752 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:45.752 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.752 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.752 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:44:45.752 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.052150+0000", 2023-09-23T20:44:45.752 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:45.753 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.753 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.753 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:44:45.753 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.052149+0000", 2023-09-23T20:44:45.753 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:44:45.753 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.753 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.754 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:44:45.754 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.052237+0000", 2023-09-23T20:44:45.754 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.7569000000000001e-05 2023-09-23T20:44:45.754 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.754 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.754 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:44:45.754 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.052246+0000", 2023-09-23T20:44:45.754 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.5289999999999997e-06 2023-09-23T20:44:45.754 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.754 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.754 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:44:45.754 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.052252+0000", 2023-09-23T20:44:45.755 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.5450000000000003e-06 2023-09-23T20:44:45.755 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.755 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.755 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:44:45.755 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.052294+0000", 2023-09-23T20:44:45.755 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.2228999999999998e-05 2023-09-23T20:44:45.755 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.756 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.756 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:44:45.756 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.052411+0000", 2023-09-23T20:44:45.756 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00011702899999999999 2023-09-23T20:44:45.756 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.756 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.756 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 3,7", 2023-09-23T20:44:45.757 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.052532+0000", 2023-09-23T20:44:45.757 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00012102799999999999 2023-09-23T20:44:45.757 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.757 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.757 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:44:45.757 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.055293+0000", 2023-09-23T20:44:45.757 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.002760424 2023-09-23T20:44:45.757 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.757 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.757 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:45.757 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.056788+0000", 2023-09-23T20:44:45.757 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00149522 2023-09-23T20:44:45.758 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.758 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.758 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:45.758 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.198887+0000", 2023-09-23T20:44:45.758 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.142099535 2023-09-23T20:44:45.758 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.758 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.758 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:44:45.758 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.198937+0000", 2023-09-23T20:44:45.758 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.9252999999999998e-05 2023-09-23T20:44:45.758 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.759 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.759 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:44:45.759 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.198977+0000", 2023-09-23T20:44:45.759 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.0151000000000002e-05 2023-09-23T20:44:45.759 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:45.759 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:44:45.759 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:45.759 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.759 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.759 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:29552 9.f 9:fef69f4f:::benchmark_data_smithi018_381516_object29551:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e459)", 2023-09-23T20:44:45.759 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:41:21.053398+0000", 2023-09-23T20:44:45.759 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 204.685831181, 2023-09-23T20:44:45.759 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.14557930599999999, 2023-09-23T20:44:45.760 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:44:45.760 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:44:45.760 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:44:45.760 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:44:45.760 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:44:45.760 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 29552 2023-09-23T20:44:45.760 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.760 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:44:45.760 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.760 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:44:45.760 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.053398+0000", 2023-09-23T20:44:45.760 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:45.761 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.761 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.761 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:44:45.761 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.053398+0000", 2023-09-23T20:44:45.761 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:45.761 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.761 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.761 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:44:45.761 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.053398+0000", 2023-09-23T20:44:45.761 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:44:45.761 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.761 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.762 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:44:45.762 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.053439+0000", 2023-09-23T20:44:45.762 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.1603000000000001e-05 2023-09-23T20:44:45.762 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.762 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.762 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:44:45.762 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.053453+0000", 2023-09-23T20:44:45.762 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.3842000000000001e-05 2023-09-23T20:44:45.762 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.762 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.762 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:44:45.762 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.053462+0000", 2023-09-23T20:44:45.763 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.9190000000000005e-06 2023-09-23T20:44:45.763 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.763 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.763 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:44:45.763 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.053503+0000", 2023-09-23T20:44:45.763 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.1069000000000003e-05 2023-09-23T20:44:45.763 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.763 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.763 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:44:45.763 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.053629+0000", 2023-09-23T20:44:45.763 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00012554299999999999 2023-09-23T20:44:45.763 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.764 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.764 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:44:45.764 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.053773+0000", 2023-09-23T20:44:45.764 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000144427 2023-09-23T20:44:45.764 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.764 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.764 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:44:45.764 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.056678+0000", 2023-09-23T20:44:45.764 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0029051200000000002 2023-09-23T20:44:45.764 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.764 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.764 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:45.764 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.060074+0000", 2023-09-23T20:44:45.765 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0033957610000000002 2023-09-23T20:44:45.765 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.765 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.765 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:45.765 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.198920+0000", 2023-09-23T20:44:45.765 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.13884589899999999 2023-09-23T20:44:45.765 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.765 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.765 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:44:45.765 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.198949+0000", 2023-09-23T20:44:45.765 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.9641e-05 2023-09-23T20:44:45.765 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.766 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.766 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:44:45.766 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.198978+0000", 2023-09-23T20:44:45.766 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.8285000000000001e-05 2023-09-23T20:44:45.766 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:45.766 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:44:45.766 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:45.766 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.766 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.766 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:29557 9.7 9:e1a9e246:::benchmark_data_smithi018_381516_object29556:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e459)", 2023-09-23T20:44:45.766 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:41:21.057117+0000", 2023-09-23T20:44:45.766 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 204.68211270800001, 2023-09-23T20:44:45.767 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.14586998200000001, 2023-09-23T20:44:45.767 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:44:45.767 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:44:45.767 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:44:45.767 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:44:45.767 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:44:45.767 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 29557 2023-09-23T20:44:45.767 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.767 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:44:45.767 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.767 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:44:45.767 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.057117+0000", 2023-09-23T20:44:45.767 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:45.768 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.768 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.768 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:44:45.768 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.057117+0000", 2023-09-23T20:44:45.768 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:45.768 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.768 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.768 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:44:45.768 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.057115+0000", 2023-09-23T20:44:45.768 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:44:45.768 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.768 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.769 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:44:45.769 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.057187+0000", 2023-09-23T20:44:45.769 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.1619999999999995e-05 2023-09-23T20:44:45.769 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.769 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.769 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:44:45.769 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.057200+0000", 2023-09-23T20:44:45.769 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.3261000000000001e-05 2023-09-23T20:44:45.769 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.769 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.769 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:44:45.769 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.057207+0000", 2023-09-23T20:44:45.770 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.6309999999999997e-06 2023-09-23T20:44:45.770 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.770 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.770 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:44:45.770 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.057249+0000", 2023-09-23T20:44:45.770 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.1993999999999998e-05 2023-09-23T20:44:45.770 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.770 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.770 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:44:45.770 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.057363+0000", 2023-09-23T20:44:45.770 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000113553 2023-09-23T20:44:45.770 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.771 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.771 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 3,7", 2023-09-23T20:44:45.771 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.057458+0000", 2023-09-23T20:44:45.771 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.5340000000000005e-05 2023-09-23T20:44:45.771 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.771 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.771 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:44:45.771 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.058584+0000", 2023-09-23T20:44:45.771 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0011261489999999999 2023-09-23T20:44:45.771 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.771 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.771 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:45.772 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.061821+0000", 2023-09-23T20:44:45.772 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0032369619999999999 2023-09-23T20:44:45.772 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.772 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.772 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:45.772 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.202885+0000", 2023-09-23T20:44:45.772 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.14106375800000001 2023-09-23T20:44:45.772 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.773 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.773 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:44:45.773 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.202939+0000", 2023-09-23T20:44:45.773 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.4626000000000002e-05 2023-09-23T20:44:45.773 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.773 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.773 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:44:45.773 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.202987+0000", 2023-09-23T20:44:45.773 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.7528000000000003e-05 2023-09-23T20:44:45.773 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:45.773 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:44:45.773 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:45.774 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.774 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.774 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:29569 9.7 9:edf50775:::benchmark_data_smithi018_381516_object29568:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e459)", 2023-09-23T20:44:45.774 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:41:21.063339+0000", 2023-09-23T20:44:45.774 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 204.67589038, 2023-09-23T20:44:45.774 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.13984607700000001, 2023-09-23T20:44:45.774 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:44:45.774 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:44:45.774 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:44:45.774 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:44:45.774 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:44:45.774 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 29569 2023-09-23T20:44:45.775 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.775 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:44:45.775 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.775 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:44:45.775 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.063339+0000", 2023-09-23T20:44:45.775 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:45.775 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.775 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.775 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:44:45.775 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.063339+0000", 2023-09-23T20:44:45.775 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:45.775 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.775 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.776 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:44:45.776 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.063338+0000", 2023-09-23T20:44:45.776 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:44:45.776 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.776 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.776 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:44:45.776 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.063399+0000", 2023-09-23T20:44:45.776 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.0869000000000003e-05 2023-09-23T20:44:45.776 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.776 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.776 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:44:45.776 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.063417+0000", 2023-09-23T20:44:45.777 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.7456999999999999e-05 2023-09-23T20:44:45.777 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.777 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.777 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:44:45.777 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.063423+0000", 2023-09-23T20:44:45.777 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.4139999999999999e-06 2023-09-23T20:44:45.777 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.777 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.777 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:44:45.777 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.063463+0000", 2023-09-23T20:44:45.777 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.9626999999999999e-05 2023-09-23T20:44:45.777 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.778 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.778 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:44:45.778 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.063572+0000", 2023-09-23T20:44:45.778 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000109697 2023-09-23T20:44:45.778 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.778 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.778 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 3,7", 2023-09-23T20:44:45.778 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.063672+0000", 2023-09-23T20:44:45.778 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.9401000000000006e-05 2023-09-23T20:44:45.778 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.778 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.778 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:44:45.779 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.065628+0000", 2023-09-23T20:44:45.779 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.001955816 2023-09-23T20:44:45.779 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.779 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.779 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:45.779 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.065657+0000", 2023-09-23T20:44:45.779 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.9050000000000001e-05 2023-09-23T20:44:45.779 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.779 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.779 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:45.779 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.203114+0000", 2023-09-23T20:44:45.779 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.13745722199999999 2023-09-23T20:44:45.780 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.780 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.780 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:44:45.780 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.203160+0000", 2023-09-23T20:44:45.780 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.6292999999999999e-05 2023-09-23T20:44:45.780 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.780 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.780 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:44:45.780 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.203185+0000", 2023-09-23T20:44:45.780 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.5196999999999999e-05 2023-09-23T20:44:45.781 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:45.781 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:44:45.781 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:45.781 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.781 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.781 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:29570 9.7 9:ed5a7646:::benchmark_data_smithi018_381516_object29569:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e459)", 2023-09-23T20:44:45.781 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:41:21.063661+0000", 2023-09-23T20:44:45.782 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 204.67556911200001, 2023-09-23T20:44:45.782 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.14287293200000001, 2023-09-23T20:44:45.782 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:44:45.782 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:44:45.782 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:44:45.782 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:44:45.782 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:44:45.782 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 29570 2023-09-23T20:44:45.782 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.782 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:44:45.783 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.783 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:44:45.783 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.063661+0000", 2023-09-23T20:44:45.783 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:45.783 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.783 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.783 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:44:45.783 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.063661+0000", 2023-09-23T20:44:45.783 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:45.783 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.783 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.784 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:44:45.784 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.063660+0000", 2023-09-23T20:44:45.784 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:44:45.784 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.784 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.784 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:44:45.784 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.063718+0000", 2023-09-23T20:44:45.784 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.8522e-05 2023-09-23T20:44:45.784 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.784 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.784 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:44:45.784 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.063730+0000", 2023-09-23T20:44:45.785 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1586999999999999e-05 2023-09-23T20:44:45.785 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.785 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.785 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:44:45.785 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.063734+0000", 2023-09-23T20:44:45.785 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.3889999999999998e-06 2023-09-23T20:44:45.785 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.785 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.785 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:44:45.785 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.064722+0000", 2023-09-23T20:44:45.785 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00098758900000000009 2023-09-23T20:44:45.785 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.786 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.786 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:44:45.786 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.064854+0000", 2023-09-23T20:44:45.786 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00013245499999999999 2023-09-23T20:44:45.786 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.786 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.786 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 3,7", 2023-09-23T20:44:45.786 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.064964+0000", 2023-09-23T20:44:45.787 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000109548 2023-09-23T20:44:45.787 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.787 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.787 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:44:45.787 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.065828+0000", 2023-09-23T20:44:45.787 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00086438499999999996 2023-09-23T20:44:45.787 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.787 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.787 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:45.787 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.066131+0000", 2023-09-23T20:44:45.787 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00030242099999999998 2023-09-23T20:44:45.788 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.788 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.788 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:45.788 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.206430+0000", 2023-09-23T20:44:45.788 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.14029873500000001 2023-09-23T20:44:45.788 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.788 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.788 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:44:45.788 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.206487+0000", 2023-09-23T20:44:45.788 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.7812000000000001e-05 2023-09-23T20:44:45.788 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.788 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.789 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:44:45.789 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.206533+0000", 2023-09-23T20:44:45.789 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.6122999999999999e-05 2023-09-23T20:44:45.789 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:45.789 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:44:45.789 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:45.789 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.789 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.789 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:76985 9.5 9:ababdd75:::benchmark_data_smithi018_381516_object76984:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e465)", 2023-09-23T20:44:45.789 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:41:58.522099+0000", 2023-09-23T20:44:45.789 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 167.217130704, 2023-09-23T20:44:45.789 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.96136469999999996, 2023-09-23T20:44:45.789 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:44:45.790 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:44:45.790 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:44:45.790 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:44:45.790 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:44:45.790 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 76985 2023-09-23T20:44:45.790 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.790 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:44:45.790 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.790 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:44:45.790 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.522099+0000", 2023-09-23T20:44:45.790 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:45.791 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.791 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.791 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:44:45.791 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.522099+0000", 2023-09-23T20:44:45.791 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:45.791 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.791 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.791 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:44:45.791 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.522098+0000", 2023-09-23T20:44:45.791 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:44:45.791 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.791 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.792 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:44:45.792 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.522133+0000", 2023-09-23T20:44:45.792 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.4041000000000002e-05 2023-09-23T20:44:45.792 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.792 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.792 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:44:45.792 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.522141+0000", 2023-09-23T20:44:45.792 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.7900000000000005e-06 2023-09-23T20:44:45.792 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.792 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.792 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:44:45.792 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.522146+0000", 2023-09-23T20:44:45.792 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.3340000000000003e-06 2023-09-23T20:44:45.793 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.793 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.793 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:44:45.793 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.522199+0000", 2023-09-23T20:44:45.793 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.3347000000000003e-05 2023-09-23T20:44:45.793 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.793 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.793 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:44:45.793 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.522206+0000", 2023-09-23T20:44:45.793 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.6209999999999998e-06 2023-09-23T20:44:45.793 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.793 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.794 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:44:45.794 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.471807+0000", 2023-09-23T20:44:45.794 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.94960129400000004 2023-09-23T20:44:45.794 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.794 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.794 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:44:45.794 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.474386+0000", 2023-09-23T20:44:45.794 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0025794749999999999 2023-09-23T20:44:45.794 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.794 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.794 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,7", 2023-09-23T20:44:45.794 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.474496+0000", 2023-09-23T20:44:45.795 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000109733 2023-09-23T20:44:45.795 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.795 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.795 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:45.795 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.483272+0000", 2023-09-23T20:44:45.795 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0087758269999999999 2023-09-23T20:44:45.795 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.795 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.795 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:45.795 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.483367+0000", 2023-09-23T20:44:45.795 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.5458999999999995e-05 2023-09-23T20:44:45.795 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.796 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.796 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:44:45.796 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.483375+0000", 2023-09-23T20:44:45.796 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.7910000000000005e-06 2023-09-23T20:44:45.796 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.796 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.796 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:44:45.796 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.483424+0000", 2023-09-23T20:44:45.796 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.8497e-05 2023-09-23T20:44:45.796 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.796 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.796 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:44:45.797 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.483464+0000", 2023-09-23T20:44:45.797 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.9960999999999999e-05 2023-09-23T20:44:45.797 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:45.797 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:44:45.797 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:45.797 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.797 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.797 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:76994 9.5 9:a38d7184:::benchmark_data_smithi018_381516_object76993:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e465)", 2023-09-23T20:44:45.797 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:41:58.527883+0000", 2023-09-23T20:44:45.797 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 167.21134676200001, 2023-09-23T20:44:45.797 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.95854079599999997, 2023-09-23T20:44:45.797 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:44:45.798 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:44:45.798 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:44:45.798 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:44:45.798 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:44:45.798 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 76994 2023-09-23T20:44:45.798 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.798 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:44:45.798 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.798 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:44:45.798 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.527883+0000", 2023-09-23T20:44:45.798 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:45.798 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.799 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.799 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:44:45.799 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.527883+0000", 2023-09-23T20:44:45.799 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:45.799 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.799 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.799 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:44:45.799 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.527882+0000", 2023-09-23T20:44:45.799 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:44:45.799 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.799 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.799 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:44:45.800 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.527986+0000", 2023-09-23T20:44:45.800 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000104201 2023-09-23T20:44:45.800 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.800 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.800 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:44:45.800 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.528001+0000", 2023-09-23T20:44:45.800 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.5187999999999999e-05 2023-09-23T20:44:45.800 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.800 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.800 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:44:45.801 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.528009+0000", 2023-09-23T20:44:45.801 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.4259999999999997e-06 2023-09-23T20:44:45.801 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.801 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.801 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:44:45.801 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.528056+0000", 2023-09-23T20:44:45.801 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.727e-05 2023-09-23T20:44:45.801 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.801 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.801 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:44:45.801 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.528062+0000", 2023-09-23T20:44:45.801 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.1929999999999998e-06 2023-09-23T20:44:45.802 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.802 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.802 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:44:45.802 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.477679+0000", 2023-09-23T20:44:45.802 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.94961691500000001 2023-09-23T20:44:45.802 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.802 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.802 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:44:45.802 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.477800+0000", 2023-09-23T20:44:45.802 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00012139800000000001 2023-09-23T20:44:45.803 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.803 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.803 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,7", 2023-09-23T20:44:45.803 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.477938+0000", 2023-09-23T20:44:45.803 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00013737499999999999 2023-09-23T20:44:45.803 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.803 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.803 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:45.803 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.483301+0000", 2023-09-23T20:44:45.803 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0053636430000000004 2023-09-23T20:44:45.803 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.803 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.804 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:44:45.804 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.483541+0000", 2023-09-23T20:44:45.804 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00023936699999999999 2023-09-23T20:44:45.804 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.804 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.804 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:45.804 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.486265+0000", 2023-09-23T20:44:45.804 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.002724363 2023-09-23T20:44:45.804 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.804 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.804 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:44:45.804 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.486368+0000", 2023-09-23T20:44:45.804 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000103058 2023-09-23T20:44:45.805 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.805 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.805 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:44:45.805 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.486424+0000", 2023-09-23T20:44:45.805 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.5560000000000003e-05 2023-09-23T20:44:45.805 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:45.805 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:44:45.805 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:45.805 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.805 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.805 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:77025 9.5 9:a8adfb4b:::benchmark_data_smithi018_381516_object77024:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e465)", 2023-09-23T20:44:45.805 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:41:58.574546+0000", 2023-09-23T20:44:45.806 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 167.16468314400001, 2023-09-23T20:44:45.806 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.91796858599999998, 2023-09-23T20:44:45.806 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:44:45.806 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:44:45.806 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:44:45.806 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:44:45.806 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:44:45.806 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 77025 2023-09-23T20:44:45.806 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.806 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:44:45.806 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.806 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:44:45.807 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.574546+0000", 2023-09-23T20:44:45.807 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:45.807 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.807 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.807 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:44:45.807 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.574546+0000", 2023-09-23T20:44:45.807 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:45.807 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.807 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.807 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:44:45.807 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.574546+0000", 2023-09-23T20:44:45.808 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:44:45.808 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.808 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.808 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:44:45.808 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.574601+0000", 2023-09-23T20:44:45.808 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.4879000000000001e-05 2023-09-23T20:44:45.808 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.808 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.808 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:44:45.809 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.574612+0000", 2023-09-23T20:44:45.809 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1201e-05 2023-09-23T20:44:45.809 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.809 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.809 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:44:45.809 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.574616+0000", 2023-09-23T20:44:45.810 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.1370000000000004e-06 2023-09-23T20:44:45.810 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.810 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.810 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:44:45.810 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.574650+0000", 2023-09-23T20:44:45.811 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.4007000000000002e-05 2023-09-23T20:44:45.811 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.811 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.811 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:44:45.811 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.574655+0000", 2023-09-23T20:44:45.811 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.0340000000000004e-06 2023-09-23T20:44:45.811 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.812 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.812 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:44:45.812 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.480178+0000", 2023-09-23T20:44:45.812 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.90552307700000001 2023-09-23T20:44:45.812 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.812 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.812 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:44:45.813 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.480276+0000", 2023-09-23T20:44:45.813 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.8187e-05 2023-09-23T20:44:45.813 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.813 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.813 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,7", 2023-09-23T20:44:45.813 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.480425+0000", 2023-09-23T20:44:45.814 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000148228 2023-09-23T20:44:45.814 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.814 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.814 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:45.814 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.483500+0000", 2023-09-23T20:44:45.814 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0030750740000000001 2023-09-23T20:44:45.814 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.814 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.814 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:44:45.814 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.483545+0000", 2023-09-23T20:44:45.814 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.5864e-05 2023-09-23T20:44:45.814 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.815 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.815 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:45.815 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.492442+0000", 2023-09-23T20:44:45.815 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0088969189999999997 2023-09-23T20:44:45.815 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.815 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.815 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:44:45.815 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.492481+0000", 2023-09-23T20:44:45.815 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.8887999999999998e-05 2023-09-23T20:44:45.815 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.815 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.815 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:44:45.816 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.492515+0000", 2023-09-23T20:44:45.816 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.3810999999999999e-05 2023-09-23T20:44:45.816 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:45.816 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:44:45.816 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:45.816 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.816 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.816 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:91635 9.f 9:f296a7d6:::benchmark_data_smithi018_381516_object91634:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e467)", 2023-09-23T20:44:45.816 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:42:13.701301+0000", 2023-09-23T20:44:45.816 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 152.037928405, 2023-09-23T20:44:45.816 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.128139787, 2023-09-23T20:44:45.816 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:44:45.817 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:44:45.817 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:44:45.817 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:44:45.817 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:44:45.817 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 91635 2023-09-23T20:44:45.817 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.817 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:44:45.817 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.817 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:44:45.817 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.701301+0000", 2023-09-23T20:44:45.817 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:45.817 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.817 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.818 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:44:45.818 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.701301+0000", 2023-09-23T20:44:45.818 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:45.818 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.818 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.818 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:44:45.818 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.701301+0000", 2023-09-23T20:44:45.818 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:44:45.818 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.818 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.818 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:44:45.818 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.701346+0000", 2023-09-23T20:44:45.819 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.5599999999999997e-05 2023-09-23T20:44:45.819 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.819 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.819 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:44:45.819 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.701359+0000", 2023-09-23T20:44:45.819 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.3251000000000001e-05 2023-09-23T20:44:45.819 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.819 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.819 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:44:45.819 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.701365+0000", 2023-09-23T20:44:45.819 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.8309999999999997e-06 2023-09-23T20:44:45.819 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.820 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.820 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:44:45.820 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.706931+0000", 2023-09-23T20:44:45.820 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0055660900000000001 2023-09-23T20:44:45.820 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.820 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.820 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:44:45.820 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.707048+0000", 2023-09-23T20:44:45.820 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000116663 2023-09-23T20:44:45.820 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.820 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.820 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:44:45.821 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.714996+0000", 2023-09-23T20:44:45.821 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0079476500000000005 2023-09-23T20:44:45.821 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.821 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.821 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:45.821 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.719222+0000", 2023-09-23T20:44:45.821 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.004226769 2023-09-23T20:44:45.821 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.821 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.821 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:44:45.821 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.726451+0000", 2023-09-23T20:44:45.821 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0072287560000000002 2023-09-23T20:44:45.821 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.822 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.822 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:45.822 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.829335+0000", 2023-09-23T20:44:45.822 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.102883501 2023-09-23T20:44:45.822 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.822 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.822 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:44:45.822 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.829386+0000", 2023-09-23T20:44:45.822 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.1496999999999998e-05 2023-09-23T20:44:45.822 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.822 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.822 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:44:45.823 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.829441+0000", 2023-09-23T20:44:45.823 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.4868999999999999e-05 2023-09-23T20:44:45.823 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:45.823 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:44:45.823 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:45.823 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.823 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.823 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:91642 9.7 9:e21f89c5:::benchmark_data_smithi018_381516_object91641:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e467)", 2023-09-23T20:44:45.823 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:42:13.710744+0000", 2023-09-23T20:44:45.823 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 152.028485905, 2023-09-23T20:44:45.823 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.13617485700000001, 2023-09-23T20:44:45.823 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:44:45.824 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:44:45.824 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:44:45.824 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:44:45.824 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:44:45.824 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 91642 2023-09-23T20:44:45.824 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.824 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:44:45.824 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.824 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:44:45.824 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.710744+0000", 2023-09-23T20:44:45.824 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:45.825 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.825 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.825 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:44:45.825 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.710744+0000", 2023-09-23T20:44:45.825 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:45.825 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.825 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.825 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:44:45.825 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.710742+0000", 2023-09-23T20:44:45.825 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:44:45.825 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.825 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.826 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:44:45.826 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.710818+0000", 2023-09-23T20:44:45.826 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.6044999999999994e-05 2023-09-23T20:44:45.826 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.826 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.826 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:44:45.826 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.710834+0000", 2023-09-23T20:44:45.826 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.5733e-05 2023-09-23T20:44:45.827 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.827 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.827 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:44:45.827 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.710841+0000", 2023-09-23T20:44:45.827 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.0559999999999999e-06 2023-09-23T20:44:45.827 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.827 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.827 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:44:45.827 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.719091+0000", 2023-09-23T20:44:45.827 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0082496040000000007 2023-09-23T20:44:45.827 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.828 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.828 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:44:45.828 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.719366+0000", 2023-09-23T20:44:45.828 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00027542799999999998 2023-09-23T20:44:45.828 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.828 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.828 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 3,7", 2023-09-23T20:44:45.828 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.719569+0000", 2023-09-23T20:44:45.828 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00020253299999999999 2023-09-23T20:44:45.828 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.828 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.828 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:44:45.828 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.729681+0000", 2023-09-23T20:44:45.829 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.010112571000000001 2023-09-23T20:44:45.829 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.829 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.829 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:45.829 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.740599+0000", 2023-09-23T20:44:45.829 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.010917425 2023-09-23T20:44:45.829 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.829 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.829 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:45.829 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.846806+0000", 2023-09-23T20:44:45.829 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.10620702 2023-09-23T20:44:45.829 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.830 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.830 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:44:45.830 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.846863+0000", 2023-09-23T20:44:45.830 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.7924999999999999e-05 2023-09-23T20:44:45.830 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.830 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.830 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:44:45.830 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.846919+0000", 2023-09-23T20:44:45.830 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.5124999999999998e-05 2023-09-23T20:44:45.830 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:45.830 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:44:45.830 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:45.831 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.831 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.831 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:91649 9.7 9:ed097aa6:::benchmark_data_smithi018_381516_object91648:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e467)", 2023-09-23T20:44:45.831 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:42:13.719094+0000", 2023-09-23T20:44:45.831 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 152.02013526100001, 2023-09-23T20:44:45.831 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.12801558599999999, 2023-09-23T20:44:45.831 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:44:45.831 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:44:45.831 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:44:45.831 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:44:45.831 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:44:45.831 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 91649 2023-09-23T20:44:45.832 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.832 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:44:45.832 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.832 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:44:45.832 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.719094+0000", 2023-09-23T20:44:45.832 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:45.832 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.832 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.832 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:44:45.832 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.719094+0000", 2023-09-23T20:44:45.832 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:45.832 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.833 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.833 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:44:45.833 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.719093+0000", 2023-09-23T20:44:45.833 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:44:45.833 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.833 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.833 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:44:45.833 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.719128+0000", 2023-09-23T20:44:45.833 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.5079000000000001e-05 2023-09-23T20:44:45.833 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.833 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.833 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:44:45.834 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.719139+0000", 2023-09-23T20:44:45.834 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0761000000000001e-05 2023-09-23T20:44:45.834 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.834 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.834 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:44:45.834 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.719144+0000", 2023-09-23T20:44:45.834 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.2920000000000003e-06 2023-09-23T20:44:45.834 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.834 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.834 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:44:45.834 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.722939+0000", 2023-09-23T20:44:45.834 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.003794136 2023-09-23T20:44:45.835 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.835 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.835 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:44:45.835 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.723030+0000", 2023-09-23T20:44:45.835 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.1193999999999995e-05 2023-09-23T20:44:45.835 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.835 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.835 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 3,7", 2023-09-23T20:44:45.835 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.723143+0000", 2023-09-23T20:44:45.835 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000113268 2023-09-23T20:44:45.835 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.835 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.836 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:44:45.836 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.729715+0000", 2023-09-23T20:44:45.836 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0065721130000000001 2023-09-23T20:44:45.836 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.836 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.836 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:45.836 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.740629+0000", 2023-09-23T20:44:45.836 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.010913997 2023-09-23T20:44:45.836 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.836 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.836 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:45.836 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.846966+0000", 2023-09-23T20:44:45.837 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.106336736 2023-09-23T20:44:45.837 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.837 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.837 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:44:45.837 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.847061+0000", 2023-09-23T20:44:45.837 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.4692999999999996e-05 2023-09-23T20:44:45.837 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.837 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.837 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:44:45.837 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.847110+0000", 2023-09-23T20:44:45.837 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.9382000000000003e-05 2023-09-23T20:44:45.837 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:45.838 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:44:45.838 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:45.838 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.838 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.838 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:91651 9.f 9:f2c174a9:::benchmark_data_smithi018_381516_object91650:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e467)", 2023-09-23T20:44:45.838 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:42:13.719151+0000", 2023-09-23T20:44:45.838 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 152.02007913099999, 2023-09-23T20:44:45.838 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.12779057599999999, 2023-09-23T20:44:45.838 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:44:45.838 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:44:45.838 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:44:45.839 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:44:45.839 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:44:45.839 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 91651 2023-09-23T20:44:45.839 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.839 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:44:45.839 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.839 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:44:45.839 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.719151+0000", 2023-09-23T20:44:45.839 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:45.839 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.839 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.839 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:44:45.840 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.719151+0000", 2023-09-23T20:44:45.840 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:45.840 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.840 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.840 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:44:45.840 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.719150+0000", 2023-09-23T20:44:45.840 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:44:45.840 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.840 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.840 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:44:45.840 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.719179+0000", 2023-09-23T20:44:45.840 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.9420999999999998e-05 2023-09-23T20:44:45.840 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.841 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.841 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:44:45.841 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.719187+0000", 2023-09-23T20:44:45.841 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.2320000000000004e-06 2023-09-23T20:44:45.841 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.841 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.841 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:44:45.841 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.719191+0000", 2023-09-23T20:44:45.841 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.3220000000000003e-06 2023-09-23T20:44:45.841 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.841 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.841 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:44:45.842 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.722930+0000", 2023-09-23T20:44:45.842 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0037388640000000002 2023-09-23T20:44:45.842 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.842 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.842 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:44:45.842 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.723047+0000", 2023-09-23T20:44:45.842 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00011688699999999999 2023-09-23T20:44:45.842 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.842 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.842 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:44:45.842 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.723182+0000", 2023-09-23T20:44:45.842 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00013561199999999999 2023-09-23T20:44:45.843 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.843 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.843 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:45.843 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.729712+0000", 2023-09-23T20:44:45.843 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0065292340000000001 2023-09-23T20:44:45.843 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.843 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.843 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:44:45.843 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.729718+0000", 2023-09-23T20:44:45.843 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.3829999999999997e-06 2023-09-23T20:44:45.843 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.843 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.844 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:45.844 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.846858+0000", 2023-09-23T20:44:45.844 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.117139934 2023-09-23T20:44:45.844 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.844 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.844 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:44:45.844 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.846905+0000", 2023-09-23T20:44:45.844 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.7187000000000001e-05 2023-09-23T20:44:45.844 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.844 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.844 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:44:45.844 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.846941+0000", 2023-09-23T20:44:45.845 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.5975e-05 2023-09-23T20:44:45.845 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:45.845 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:44:45.845 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:45.845 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.845 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.845 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:92127 9.7 9:eb5270d9:::benchmark_data_smithi018_381516_object92126:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e467)", 2023-09-23T20:44:45.845 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:42:14.303475+0000", 2023-09-23T20:44:45.845 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 151.43575463100001, 2023-09-23T20:44:45.845 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.13503338400000001, 2023-09-23T20:44:45.845 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:44:45.845 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:44:45.846 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:44:45.846 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:44:45.846 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:44:45.846 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 92127 2023-09-23T20:44:45.846 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.846 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:44:45.846 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.846 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:44:45.846 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.303475+0000", 2023-09-23T20:44:45.846 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:45.846 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.846 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.846 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:44:45.847 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.303475+0000", 2023-09-23T20:44:45.847 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:45.847 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.847 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.847 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:44:45.847 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.303475+0000", 2023-09-23T20:44:45.847 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:44:45.847 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.847 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.847 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:44:45.847 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.303510+0000", 2023-09-23T20:44:45.847 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.5318999999999998e-05 2023-09-23T20:44:45.848 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.848 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.848 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:44:45.848 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.303516+0000", 2023-09-23T20:44:45.848 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.4320000000000004e-06 2023-09-23T20:44:45.848 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.848 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.848 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:44:45.848 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.303520+0000", 2023-09-23T20:44:45.848 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.0280000000000002e-06 2023-09-23T20:44:45.848 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.848 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.848 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:44:45.849 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.303550+0000", 2023-09-23T20:44:45.849 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.9938e-05 2023-09-23T20:44:45.849 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.849 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.849 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:44:45.849 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.303677+0000", 2023-09-23T20:44:45.849 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00012659800000000001 2023-09-23T20:44:45.849 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.849 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.849 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 3,7", 2023-09-23T20:44:45.849 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.303805+0000", 2023-09-23T20:44:45.849 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00012804799999999999 2023-09-23T20:44:45.850 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.850 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.850 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:44:45.850 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.306643+0000", 2023-09-23T20:44:45.850 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.002837759 2023-09-23T20:44:45.850 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.850 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.850 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:45.850 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.317326+0000", 2023-09-23T20:44:45.850 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.010683002 2023-09-23T20:44:45.850 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.850 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.851 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:45.851 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.438439+0000", 2023-09-23T20:44:45.851 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.121113448 2023-09-23T20:44:45.851 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.851 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.851 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:44:45.851 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.438478+0000", 2023-09-23T20:44:45.851 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.8921999999999998e-05 2023-09-23T20:44:45.851 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.851 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.851 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:44:45.851 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.438508+0000", 2023-09-23T20:44:45.852 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.029e-05 2023-09-23T20:44:45.852 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:45.852 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:44:45.852 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:45.852 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.852 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.852 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:92129 9.f 9:fdf451a5:::benchmark_data_smithi018_381516_object92128:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e467)", 2023-09-23T20:44:45.852 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:42:14.307177+0000", 2023-09-23T20:44:45.852 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 151.43205285799999, 2023-09-23T20:44:45.852 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.138573006, 2023-09-23T20:44:45.852 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:44:45.852 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:44:45.853 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:44:45.853 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:44:45.853 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:44:45.853 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 92129 2023-09-23T20:44:45.853 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.853 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:44:45.853 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.853 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:44:45.853 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.307177+0000", 2023-09-23T20:44:45.853 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:45.853 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.853 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.853 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:44:45.854 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.307177+0000", 2023-09-23T20:44:45.854 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:45.854 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.854 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.854 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:44:45.854 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.307175+0000", 2023-09-23T20:44:45.854 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:44:45.854 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.854 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.854 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:44:45.854 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.307245+0000", 2023-09-23T20:44:45.855 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.0261999999999999e-05 2023-09-23T20:44:45.855 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.855 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.855 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:44:45.855 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.307253+0000", 2023-09-23T20:44:45.855 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.7219999999999999e-06 2023-09-23T20:44:45.855 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.855 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.855 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:44:45.855 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.307257+0000", 2023-09-23T20:44:45.855 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.0430000000000001e-06 2023-09-23T20:44:45.855 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.856 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.856 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:44:45.856 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.307299+0000", 2023-09-23T20:44:45.856 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.2128000000000001e-05 2023-09-23T20:44:45.856 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.856 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.856 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:44:45.856 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.307431+0000", 2023-09-23T20:44:45.856 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00013203499999999999 2023-09-23T20:44:45.856 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.856 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.856 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:44:45.857 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.307559+0000", 2023-09-23T20:44:45.857 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000127632 2023-09-23T20:44:45.857 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.857 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.857 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:45.857 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.317355+0000", 2023-09-23T20:44:45.857 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0097961349999999992 2023-09-23T20:44:45.857 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.857 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.857 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:44:45.857 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.320352+0000", 2023-09-23T20:44:45.857 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0029971569999999999 2023-09-23T20:44:45.858 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.858 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.858 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:45.858 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.445677+0000", 2023-09-23T20:44:45.858 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.12532515999999999 2023-09-23T20:44:45.858 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.858 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.858 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:44:45.858 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.445717+0000", 2023-09-23T20:44:45.858 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.9616000000000002e-05 2023-09-23T20:44:45.858 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.858 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.859 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:44:45.859 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.445750+0000", 2023-09-23T20:44:45.859 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.2719999999999998e-05 2023-09-23T20:44:45.859 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:45.859 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:44:45.859 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:45.859 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.859 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.859 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:92140 9.f 9:f3d16644:::benchmark_data_smithi018_381516_object92139:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e467)", 2023-09-23T20:44:45.859 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:42:14.313964+0000", 2023-09-23T20:44:45.859 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 151.42526544899999, 2023-09-23T20:44:45.859 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.13195174000000001, 2023-09-23T20:44:45.860 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:44:45.860 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:44:45.860 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:44:45.860 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:44:45.860 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:44:45.860 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 92140 2023-09-23T20:44:45.860 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.860 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:44:45.860 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.860 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:44:45.860 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.313964+0000", 2023-09-23T20:44:45.860 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:45.861 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.861 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.861 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:44:45.861 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.313964+0000", 2023-09-23T20:44:45.861 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:45.861 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.861 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.861 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:44:45.861 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.313964+0000", 2023-09-23T20:44:45.861 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:44:45.861 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.861 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.861 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:44:45.862 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.314026+0000", 2023-09-23T20:44:45.862 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.2818999999999996e-05 2023-09-23T20:44:45.862 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.862 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.862 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:44:45.862 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.314035+0000", 2023-09-23T20:44:45.862 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.7059999999999993e-06 2023-09-23T20:44:45.862 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.862 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.862 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:44:45.862 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.314040+0000", 2023-09-23T20:44:45.863 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.2660000000000001e-06 2023-09-23T20:44:45.863 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.863 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.863 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:44:45.863 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.314084+0000", 2023-09-23T20:44:45.863 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.3204000000000002e-05 2023-09-23T20:44:45.863 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.863 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.863 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:44:45.863 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.314195+0000", 2023-09-23T20:44:45.863 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000111585 2023-09-23T20:44:45.863 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.863 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.864 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:44:45.864 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.314336+0000", 2023-09-23T20:44:45.864 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000140977 2023-09-23T20:44:45.864 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.864 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.864 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:45.864 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.320388+0000", 2023-09-23T20:44:45.864 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.006051519 2023-09-23T20:44:45.864 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.864 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.864 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:44:45.864 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.320393+0000", 2023-09-23T20:44:45.865 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.7189999999999998e-06 2023-09-23T20:44:45.865 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.865 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.865 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:45.865 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.445855+0000", 2023-09-23T20:44:45.865 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.125461666 2023-09-23T20:44:45.865 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.865 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.865 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:44:45.865 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.445885+0000", 2023-09-23T20:44:45.865 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.0454e-05 2023-09-23T20:44:45.865 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.866 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.866 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:44:45.866 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.445916+0000", 2023-09-23T20:44:45.866 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.0446e-05 2023-09-23T20:44:45.866 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:45.866 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:44:45.866 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:45.866 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.866 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.866 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:92144 9.7 9:ec1b46c9:::benchmark_data_smithi018_381516_object92143:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e467)", 2023-09-23T20:44:45.866 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:42:14.317205+0000", 2023-09-23T20:44:45.866 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 151.42202428499999, 2023-09-23T20:44:45.867 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.12863032999999999, 2023-09-23T20:44:45.867 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:44:45.867 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:44:45.867 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:44:45.867 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:44:45.867 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:44:45.867 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 92144 2023-09-23T20:44:45.867 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.867 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:44:45.867 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.867 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:44:45.868 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.317205+0000", 2023-09-23T20:44:45.868 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:45.868 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.868 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.868 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:44:45.868 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.317205+0000", 2023-09-23T20:44:45.868 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:45.868 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.868 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.868 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:44:45.868 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.317204+0000", 2023-09-23T20:44:45.868 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:44:45.869 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.869 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.869 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:44:45.869 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.317280+0000", 2023-09-23T20:44:45.869 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.6215000000000001e-05 2023-09-23T20:44:45.869 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.869 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.869 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:44:45.869 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.317291+0000", 2023-09-23T20:44:45.869 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0546e-05 2023-09-23T20:44:45.869 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.869 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.869 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:44:45.870 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.317297+0000", 2023-09-23T20:44:45.870 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.2650000000000002e-06 2023-09-23T20:44:45.870 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.870 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.870 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:44:45.870 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.317374+0000", 2023-09-23T20:44:45.870 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.6600999999999994e-05 2023-09-23T20:44:45.870 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.870 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.870 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:44:45.870 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.317479+0000", 2023-09-23T20:44:45.871 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000105531 2023-09-23T20:44:45.871 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.871 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.871 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 3,7", 2023-09-23T20:44:45.871 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.317604+0000", 2023-09-23T20:44:45.871 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000124745 2023-09-23T20:44:45.871 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.871 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.871 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:44:45.871 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.321102+0000", 2023-09-23T20:44:45.871 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0034977849999999998 2023-09-23T20:44:45.871 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.872 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.872 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:45.872 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.321402+0000", 2023-09-23T20:44:45.872 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00029973499999999998 2023-09-23T20:44:45.872 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.872 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.872 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:45.872 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.445781+0000", 2023-09-23T20:44:45.872 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.124378979 2023-09-23T20:44:45.872 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.872 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.872 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:44:45.872 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.445816+0000", 2023-09-23T20:44:45.873 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.5222000000000003e-05 2023-09-23T20:44:45.873 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.873 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.873 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:44:45.873 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.445836+0000", 2023-09-23T20:44:45.873 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.9952000000000001e-05 2023-09-23T20:44:45.873 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:45.873 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:44:45.873 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:45.873 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.873 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.873 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:141052 9.5 9:a7a766e0:::benchmark_data_smithi018_381516_object27218:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e481)", 2023-09-23T20:44:45.874 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:43:15.774779+0000", 2023-09-23T20:44:45.874 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 89.964450870999997, 2023-09-23T20:44:45.874 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.023693841, 2023-09-23T20:44:45.874 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:44:45.874 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:44:45.874 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:44:45.874 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:44:45.874 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:44:45.874 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 141052 2023-09-23T20:44:45.874 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.874 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:44:45.875 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.875 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:44:45.875 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.774779+0000", 2023-09-23T20:44:45.875 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:45.875 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.875 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.875 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:44:45.875 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.774779+0000", 2023-09-23T20:44:45.875 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:45.875 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.875 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.875 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:44:45.875 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.774778+0000", 2023-09-23T20:44:45.876 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:44:45.876 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.876 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.876 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:44:45.876 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.774783+0000", 2023-09-23T20:44:45.876 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.7660000000000004e-06 2023-09-23T20:44:45.876 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.876 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.876 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:44:45.876 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.774794+0000", 2023-09-23T20:44:45.876 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0744000000000001e-05 2023-09-23T20:44:45.876 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.876 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.877 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:44:45.877 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.774799+0000", 2023-09-23T20:44:45.877 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.9810000000000003e-06 2023-09-23T20:44:45.877 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.877 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.877 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:44:45.877 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.774870+0000", 2023-09-23T20:44:45.877 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.1316000000000006e-05 2023-09-23T20:44:45.877 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.877 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.877 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:44:45.877 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.774881+0000", 2023-09-23T20:44:45.877 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0291e-05 2023-09-23T20:44:45.878 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.878 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.878 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:44:45.878 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.785209+0000", 2023-09-23T20:44:45.878 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0103286419999999 2023-09-23T20:44:45.878 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.878 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.878 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:44:45.878 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.785375+0000", 2023-09-23T20:44:45.878 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00016602399999999999 2023-09-23T20:44:45.878 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.878 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.879 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,1,7", 2023-09-23T20:44:45.879 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.785518+0000", 2023-09-23T20:44:45.879 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00014225800000000001 2023-09-23T20:44:45.879 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.879 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.879 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:44:45.879 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.793627+0000", 2023-09-23T20:44:45.879 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0081096780000000004 2023-09-23T20:44:45.879 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.879 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.879 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:45.879 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.793665+0000", 2023-09-23T20:44:45.880 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.7286000000000002e-05 2023-09-23T20:44:45.880 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.880 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.880 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:45.880 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.797943+0000", 2023-09-23T20:44:45.880 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0042786869999999998 2023-09-23T20:44:45.880 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.880 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.880 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:45.880 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.798292+0000", 2023-09-23T20:44:45.880 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00034836099999999999 2023-09-23T20:44:45.880 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.881 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.881 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:44:45.881 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.798353+0000", 2023-09-23T20:44:45.881 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.1019e-05 2023-09-23T20:44:45.881 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.881 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.881 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:44:45.881 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.798473+0000", 2023-09-23T20:44:45.881 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000119888 2023-09-23T20:44:45.881 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:45.881 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:44:45.882 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:45.882 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.882 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.882 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:141057 9.5 9:aa98a8f8:::benchmark_data_smithi018_381516_object27223:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e481)", 2023-09-23T20:44:45.882 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:43:15.777430+0000", 2023-09-23T20:44:45.882 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 89.961799481, 2023-09-23T20:44:45.882 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0450975220000001, 2023-09-23T20:44:45.882 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:44:45.882 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:44:45.882 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:44:45.883 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:44:45.883 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:44:45.883 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 141057 2023-09-23T20:44:45.883 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.883 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:44:45.883 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.883 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:44:45.883 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.777430+0000", 2023-09-23T20:44:45.883 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:45.883 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.883 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.883 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:44:45.884 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.777430+0000", 2023-09-23T20:44:45.884 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:45.884 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.884 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.884 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:44:45.884 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.777428+0000", 2023-09-23T20:44:45.884 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999981 2023-09-23T20:44:45.884 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.884 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.884 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:44:45.884 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.777438+0000", 2023-09-23T20:44:45.884 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.025e-05 2023-09-23T20:44:45.885 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.885 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.885 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:44:45.885 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.777451+0000", 2023-09-23T20:44:45.885 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.2218000000000001e-05 2023-09-23T20:44:45.885 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.885 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.885 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:44:45.885 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.777456+0000", 2023-09-23T20:44:45.885 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.9019999999999998e-06 2023-09-23T20:44:45.885 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.885 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.886 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:44:45.886 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.777492+0000", 2023-09-23T20:44:45.886 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.6758000000000003e-05 2023-09-23T20:44:45.886 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.886 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.886 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:44:45.886 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.777498+0000", 2023-09-23T20:44:45.886 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.9529999999999996e-06 2023-09-23T20:44:45.886 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.886 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.886 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:44:45.886 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.793684+0000", 2023-09-23T20:44:45.887 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0161855 2023-09-23T20:44:45.887 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.887 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.887 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:44:45.887 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.793806+0000", 2023-09-23T20:44:45.887 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00012180600000000001 2023-09-23T20:44:45.887 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.887 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.887 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,1,7", 2023-09-23T20:44:45.887 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.793923+0000", 2023-09-23T20:44:45.887 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00011699100000000001 2023-09-23T20:44:45.887 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.888 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.888 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:44:45.888 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.803080+0000", 2023-09-23T20:44:45.888 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0091576539999999994 2023-09-23T20:44:45.888 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.888 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.888 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:45.888 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.809899+0000", 2023-09-23T20:44:45.888 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0068191249999999997 2023-09-23T20:44:45.888 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.888 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.888 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:45.889 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.817155+0000", 2023-09-23T20:44:45.889 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0072551860000000003 2023-09-23T20:44:45.889 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.889 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.889 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:45.889 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.822362+0000", 2023-09-23T20:44:45.889 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0052070450000000004 2023-09-23T20:44:45.889 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.889 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.889 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:44:45.889 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.822413+0000", 2023-09-23T20:44:45.889 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.1240999999999999e-05 2023-09-23T20:44:45.890 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.890 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.890 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:44:45.890 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.822528+0000", 2023-09-23T20:44:45.890 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000114843 2023-09-23T20:44:45.890 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:45.890 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:44:45.890 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:45.890 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.890 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.890 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:141060 9.5 9:a222c8d8:::benchmark_data_smithi018_381516_object27226:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e481)", 2023-09-23T20:44:45.890 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:43:15.784280+0000", 2023-09-23T20:44:45.891 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 89.954949263000003, 2023-09-23T20:44:45.891 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0384760930000001, 2023-09-23T20:44:45.891 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:44:45.891 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:44:45.891 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:44:45.891 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:44:45.891 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:44:45.891 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 141060 2023-09-23T20:44:45.891 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.891 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:44:45.891 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.891 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:44:45.892 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.784280+0000", 2023-09-23T20:44:45.892 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:45.892 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.892 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.892 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:44:45.892 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.784280+0000", 2023-09-23T20:44:45.892 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:45.892 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.892 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.892 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:44:45.892 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.784278+0000", 2023-09-23T20:44:45.892 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999981 2023-09-23T20:44:45.893 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.893 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.893 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:44:45.893 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.784293+0000", 2023-09-23T20:44:45.893 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.5048999999999999e-05 2023-09-23T20:44:45.893 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.893 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.893 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:44:45.893 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.784311+0000", 2023-09-23T20:44:45.893 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.7530999999999999e-05 2023-09-23T20:44:45.893 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.893 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.894 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:44:45.894 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.784329+0000", 2023-09-23T20:44:45.894 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.8470000000000001e-05 2023-09-23T20:44:45.894 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.894 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.894 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:44:45.894 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.784394+0000", 2023-09-23T20:44:45.894 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.4487999999999996e-05 2023-09-23T20:44:45.894 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.894 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.894 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:44:45.894 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.784404+0000", 2023-09-23T20:44:45.895 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0478e-05 2023-09-23T20:44:45.895 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.895 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.895 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:44:45.895 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.798720+0000", 2023-09-23T20:44:45.895 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.014316145 2023-09-23T20:44:45.895 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.895 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.895 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:44:45.895 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.798848+0000", 2023-09-23T20:44:45.895 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000127448 2023-09-23T20:44:45.895 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.896 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.896 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,1,7", 2023-09-23T20:44:45.896 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.798989+0000", 2023-09-23T20:44:45.896 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00014109800000000001 2023-09-23T20:44:45.896 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.896 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.896 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:44:45.896 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.809908+0000", 2023-09-23T20:44:45.896 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.01091902 2023-09-23T20:44:45.896 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.896 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.896 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:45.897 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.812374+0000", 2023-09-23T20:44:45.897 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0024658710000000001 2023-09-23T20:44:45.897 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.897 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.897 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:45.897 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.817185+0000", 2023-09-23T20:44:45.897 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0048108229999999997 2023-09-23T20:44:45.897 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.897 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.897 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:45.897 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.822622+0000", 2023-09-23T20:44:45.897 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0054370920000000001 2023-09-23T20:44:45.898 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.898 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.898 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:44:45.898 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.822660+0000", 2023-09-23T20:44:45.898 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.8250000000000001e-05 2023-09-23T20:44:45.898 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:45.898 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:45.898 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:44:45.898 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.822756+0000", 2023-09-23T20:44:45.898 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.6471999999999994e-05 2023-09-23T20:44:45.898 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:45.898 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:44:45.899 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:45.899 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:45.899 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:44:45.899 INFO:teuthology.orchestra.run.smithi149.stdout:} 2023-09-23T20:44:45.899 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:44:45.900 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:45 smithi018 ceph-mon[184144]: pgmap v1289: 119 pgs: 1 remapped+peering, 2 peering, 116 active+clean; 1.5 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 970 op/s 2023-09-23T20:44:45.900 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:45 smithi018 ceph-mon[184144]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:44:45.900 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:45 smithi018 ceph-mon[184144]: 9.8 starting backfill to osd.5 from (463'4000,492'11326] MIN to 512'12901 2023-09-23T20:44:45.900 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:45 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:44:45.900 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:45 smithi018 ceph-mon[184144]: osdmap e515: 8 total, 8 up, 8 in 2023-09-23T20:44:45.900 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:45.901 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:44:45.901 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:45 smithi018 ceph-mon[179647]: pgmap v1289: 119 pgs: 1 remapped+peering, 2 peering, 116 active+clean; 1.5 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 970 op/s 2023-09-23T20:44:45.901 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:45 smithi018 ceph-mon[179647]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:44:45.901 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:45 smithi018 ceph-mon[179647]: 9.8 starting backfill to osd.5 from (463'4000,492'11326] MIN to 512'12901 2023-09-23T20:44:45.901 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:45 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:44:45.902 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:45 smithi018 ceph-mon[179647]: osdmap e515: 8 total, 8 up, 8 in 2023-09-23T20:44:45.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:45 smithi149 ceph-mon[170394]: pgmap v1289: 119 pgs: 1 remapped+peering, 2 peering, 116 active+clean; 1.5 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 970 op/s 2023-09-23T20:44:45.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:45 smithi149 ceph-mon[170394]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:44:45.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:45 smithi149 ceph-mon[170394]: 9.8 starting backfill to osd.5 from (463'4000,492'11326] MIN to 512'12901 2023-09-23T20:44:45.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:45 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:44:45.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:45 smithi149 ceph-mon[170394]: osdmap e515: 8 total, 8 up, 8 in 2023-09-23T20:44:46.002 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:44:46.079 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:46.191 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:44:46.195 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:46.195 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:44:46.296 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:44:46.354 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:46.539 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:44:46.582 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:44:46.602 INFO:teuthology.orchestra.run.smithi018.stderr:dumped all 2023-09-23T20:44:46.894 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:46 smithi018 ceph-mon[184144]: failed to encode map e515 with expected crc 2023-09-23T20:44:46.895 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:46 smithi018 ceph-mon[184144]: failed to encode map e515 with expected crc 2023-09-23T20:44:46.895 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:46 smithi018 ceph-mon[184144]: failed to encode map e515 with expected crc 2023-09-23T20:44:46.895 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:46 smithi018 ceph-mon[184144]: failed to encode map e515 with expected crc 2023-09-23T20:44:46.895 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:46 smithi018 ceph-mon[184144]: osdmap e516: 8 total, 8 up, 8 in 2023-09-23T20:44:46.895 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:46 smithi018 ceph-mon[184144]: failed to encode map e516 with expected crc 2023-09-23T20:44:46.895 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:46 smithi018 ceph-mon[184144]: failed to encode map e516 with expected crc 2023-09-23T20:44:46.896 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:46 smithi018 ceph-mon[179647]: failed to encode map e515 with expected crc 2023-09-23T20:44:46.896 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:46 smithi018 ceph-mon[179647]: failed to encode map e515 with expected crc 2023-09-23T20:44:46.896 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:46 smithi018 ceph-mon[179647]: failed to encode map e515 with expected crc 2023-09-23T20:44:46.896 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:46 smithi018 ceph-mon[179647]: failed to encode map e515 with expected crc 2023-09-23T20:44:46.897 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:46 smithi018 ceph-mon[179647]: osdmap e516: 8 total, 8 up, 8 in 2023-09-23T20:44:46.897 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:46 smithi018 ceph-mon[179647]: failed to encode map e516 with expected crc 2023-09-23T20:44:46.897 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:46 smithi018 ceph-mon[179647]: failed to encode map e516 with expected crc 2023-09-23T20:44:46.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:46 smithi149 ceph-mon[170394]: failed to encode map e515 with expected crc 2023-09-23T20:44:46.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:46 smithi149 ceph-mon[170394]: failed to encode map e515 with expected crc 2023-09-23T20:44:46.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:46 smithi149 ceph-mon[170394]: failed to encode map e515 with expected crc 2023-09-23T20:44:46.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:46 smithi149 ceph-mon[170394]: failed to encode map e515 with expected crc 2023-09-23T20:44:46.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:46 smithi149 ceph-mon[170394]: osdmap e516: 8 total, 8 up, 8 in 2023-09-23T20:44:46.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:46 smithi149 ceph-mon[170394]: failed to encode map e516 with expected crc 2023-09-23T20:44:46.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:46 smithi149 ceph-mon[170394]: failed to encode map e516 with expected crc 2023-09-23T20:44:47.131 INFO:teuthology.orchestra.run.smithi018.stdout:{"pg_ready":true,"pg_map":{"version":1292,"stamp":"2023-09-23T20:44:46.199945+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1643059147,"num_objects":25027,"num_object_clones":0,"num_object_copies":75081,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":25027,"num_whiteouts":0,"num_read":5984,"num_read_kb":13201,"num_write":203572,"num_write_kb":7297910,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16628,"num_bytes_recovered":1094993998,"num_keys_recovered":40,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":108282,"ondisk_log_size":108282,"up":357,"acting":357,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":362,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":10257820,"kb_used_data":5386652,"kb_used_omap":32,"kb_used_meta":4870943,"kb_avail":739703396,"statfs":{"total":767960285184,"available":757456277504,"internally_reserved":0,"allocated":5515931648,"data_stored":5499740580,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32773,"internal_metadata":4987846651},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-186318848,"num_objects":-2843,"num_object_clones":0,"num_object_copies":-8529,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-2843,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2843,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"4.005975"},"pg_stats":[{"pgid":"1.18","version":"90'161","reported_seq":558,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-23T20:44:38.623704+0000","last_change":"2023-09-23T20:44:19.677357+0000","last_active":"2023-09-23T20:44:38.623704+0000","last_peered":"2023-09-23T20:44:38.623704+0000","last_clean":"2023-09-23T20:44:38.623704+0000","last_became_active":"2023-09-23T20:43:16.766305+0000","last_became_peered":"2023-09-23T20:43:16.766305+0000","last_unstale":"2023-09-23T20:44:38.623704+0000","last_undegraded":"2023-09-23T20:44:38.623704+0000","last_fullsized":"2023-09-23T20:44:38.623704+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:19.677209+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:19.677209+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:14:06.510115+0000","scrub_duration":0.0023292260000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"69'1","reported_seq":213,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-09-23T20:44:44.899495+0000","last_change":"2023-09-23T20:32:33.233853+0000","last_active":"2023-09-23T20:44:44.899495+0000","last_peered":"2023-09-23T20:44:44.899495+0000","last_clean":"2023-09-23T20:44:44.899495+0000","last_became_active":"2023-09-23T20:32:33.233546+0000","last_became_peered":"2023-09-23T20:32:33.233546+0000","last_unstale":"2023-09-23T20:44:44.899495+0000","last_undegraded":"2023-09-23T20:44:44.899495+0000","last_fullsized":"2023-09-23T20:44:44.899495+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:50:27.130808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":328,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-23T20:44:42.656990+0000","last_change":"2023-09-23T20:37:43.579611+0000","last_active":"2023-09-23T20:44:42.656990+0000","last_peered":"2023-09-23T20:44:42.656990+0000","last_clean":"2023-09-23T20:44:42.656990+0000","last_became_active":"2023-09-23T20:37:43.579323+0000","last_became_peered":"2023-09-23T20:37:43.579323+0000","last_unstale":"2023-09-23T20:44:42.656990+0000","last_undegraded":"2023-09-23T20:44:42.656990+0000","last_fullsized":"2023-09-23T20:44:42.656990+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:17:32.311427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"90'161","reported_seq":637,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-09-23T20:44:44.899155+0000","last_change":"2023-09-23T20:44:14.647447+0000","last_active":"2023-09-23T20:44:44.899155+0000","last_peered":"2023-09-23T20:44:44.899155+0000","last_clean":"2023-09-23T20:44:44.899155+0000","last_became_active":"2023-09-23T20:43:16.756116+0000","last_became_peered":"2023-09-23T20:43:16.756116+0000","last_unstale":"2023-09-23T20:44:44.899155+0000","last_undegraded":"2023-09-23T20:44:44.899155+0000","last_fullsized":"2023-09-23T20:44:44.899155+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:14.647386+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:36:05.442288+0000","last_clean_scrub_stamp":"2023-09-23T20:44:14.647386+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:25:25.151100+0000","scrub_duration":0.0097381210000000006,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":483,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-23T20:44:38.614762+0000","last_change":"2023-09-23T20:34:59.983035+0000","last_active":"2023-09-23T20:44:38.614762+0000","last_peered":"2023-09-23T20:44:38.614762+0000","last_clean":"2023-09-23T20:44:38.614762+0000","last_became_active":"2023-09-23T20:34:59.982459+0000","last_became_peered":"2023-09-23T20:34:59.982459+0000","last_unstale":"2023-09-23T20:44:38.614762+0000","last_undegraded":"2023-09-23T20:44:38.614762+0000","last_fullsized":"2023-09-23T20:44:38.614762+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:16:48.510783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":469,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-23T20:44:30.507837+0000","last_change":"2023-09-23T20:43:46.521939+0000","last_active":"2023-09-23T20:44:30.507837+0000","last_peered":"2023-09-23T20:44:30.507837+0000","last_clean":"2023-09-23T20:44:30.507837+0000","last_became_active":"2023-09-23T20:43:46.521185+0000","last_became_peered":"2023-09-23T20:43:46.521185+0000","last_unstale":"2023-09-23T20:44:30.507837+0000","last_undegraded":"2023-09-23T20:44:30.507837+0000","last_fullsized":"2023-09-23T20:44:30.507837+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:59:25.045627+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"90'161","reported_seq":595,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-09-23T20:44:44.899326+0000","last_change":"2023-09-23T20:36:01.292103+0000","last_active":"2023-09-23T20:44:44.899326+0000","last_peered":"2023-09-23T20:44:44.899326+0000","last_clean":"2023-09-23T20:44:44.899326+0000","last_became_active":"2023-09-23T20:35:26.639912+0000","last_became_peered":"2023-09-23T20:35:26.639912+0000","last_unstale":"2023-09-23T20:44:44.899326+0000","last_undegraded":"2023-09-23T20:44:44.899326+0000","last_fullsized":"2023-09-23T20:44:44.899326+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:01.292032+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:01.292032+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:04:53.926640+0000","scrub_duration":0.0032983159999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":308,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-23T20:44:42.659861+0000","last_change":"2023-09-23T20:37:43.584414+0000","last_active":"2023-09-23T20:44:42.659861+0000","last_peered":"2023-09-23T20:44:42.659861+0000","last_clean":"2023-09-23T20:44:42.659861+0000","last_became_active":"2023-09-23T20:37:43.584116+0000","last_became_peered":"2023-09-23T20:37:43.584116+0000","last_unstale":"2023-09-23T20:44:42.659861+0000","last_undegraded":"2023-09-23T20:44:42.659861+0000","last_fullsized":"2023-09-23T20:44:42.659861+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:17:54.258172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":128,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-23T20:44:38.602901+0000","last_change":"2023-09-23T20:38:37.457427+0000","last_active":"2023-09-23T20:44:38.602901+0000","last_peered":"2023-09-23T20:44:38.602901+0000","last_clean":"2023-09-23T20:44:38.602901+0000","last_became_active":"2023-09-23T20:38:37.457099+0000","last_became_peered":"2023-09-23T20:38:37.457099+0000","last_unstale":"2023-09-23T20:44:38.602901+0000","last_undegraded":"2023-09-23T20:44:38.602901+0000","last_fullsized":"2023-09-23T20:44:38.602901+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:31:27.084994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"90'161","reported_seq":724,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-23T20:44:30.504912+0000","last_change":"2023-09-23T20:40:22.449754+0000","last_active":"2023-09-23T20:44:30.504912+0000","last_peered":"2023-09-23T20:44:30.504912+0000","last_clean":"2023-09-23T20:44:30.504912+0000","last_became_active":"2023-09-23T20:40:22.448838+0000","last_became_peered":"2023-09-23T20:40:22.448838+0000","last_unstale":"2023-09-23T20:44:30.504912+0000","last_undegraded":"2023-09-23T20:44:30.504912+0000","last_fullsized":"2023-09-23T20:44:30.504912+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:03.411025+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:03.411025+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:22:07.948856+0000","scrub_duration":0.001802922,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":502,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-23T20:44:30.508262+0000","last_change":"2023-09-23T20:43:46.522325+0000","last_active":"2023-09-23T20:44:30.508262+0000","last_peered":"2023-09-23T20:44:30.508262+0000","last_clean":"2023-09-23T20:44:30.508262+0000","last_became_active":"2023-09-23T20:43:46.521996+0000","last_became_peered":"2023-09-23T20:43:46.521996+0000","last_unstale":"2023-09-23T20:44:30.508262+0000","last_undegraded":"2023-09-23T20:44:30.508262+0000","last_fullsized":"2023-09-23T20:44:30.508262+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:51:17.629956+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":238,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-23T20:44:38.623835+0000","last_change":"2023-09-23T20:38:02.579343+0000","last_active":"2023-09-23T20:44:38.623835+0000","last_peered":"2023-09-23T20:44:38.623835+0000","last_clean":"2023-09-23T20:44:38.623835+0000","last_became_active":"2023-09-23T20:38:02.579047+0000","last_became_peered":"2023-09-23T20:38:02.579047+0000","last_unstale":"2023-09-23T20:44:38.623835+0000","last_undegraded":"2023-09-23T20:44:38.623835+0000","last_fullsized":"2023-09-23T20:44:38.623835+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:44:03.166556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.14","version":"90'161","reported_seq":844,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-23T20:44:38.623034+0000","last_change":"2023-09-23T20:35:02.236398+0000","last_active":"2023-09-23T20:44:38.623034+0000","last_peered":"2023-09-23T20:44:38.623034+0000","last_clean":"2023-09-23T20:44:38.623034+0000","last_became_active":"2023-09-23T20:35:02.235988+0000","last_became_peered":"2023-09-23T20:35:02.235988+0000","last_unstale":"2023-09-23T20:44:38.623034+0000","last_undegraded":"2023-09-23T20:44:38.623034+0000","last_fullsized":"2023-09-23T20:44:38.623034+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:47.479012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:47.479012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:13.519648+0000","scrub_duration":0.0020136590000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":348,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-23T20:44:30.504961+0000","last_change":"2023-09-23T20:43:16.780478+0000","last_active":"2023-09-23T20:44:30.504961+0000","last_peered":"2023-09-23T20:44:30.504961+0000","last_clean":"2023-09-23T20:44:30.504961+0000","last_became_active":"2023-09-23T20:43:16.780183+0000","last_became_peered":"2023-09-23T20:43:16.780183+0000","last_unstale":"2023-09-23T20:44:30.504961+0000","last_undegraded":"2023-09-23T20:44:30.504961+0000","last_fullsized":"2023-09-23T20:44:30.504961+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:38:30.102066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":340,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-23T20:44:43.631710+0000","last_change":"2023-09-23T20:38:02.542034+0000","last_active":"2023-09-23T20:44:43.631710+0000","last_peered":"2023-09-23T20:44:43.631710+0000","last_clean":"2023-09-23T20:44:43.631710+0000","last_became_active":"2023-09-23T20:38:02.541575+0000","last_became_peered":"2023-09-23T20:38:02.541575+0000","last_unstale":"2023-09-23T20:44:43.631710+0000","last_undegraded":"2023-09-23T20:44:43.631710+0000","last_fullsized":"2023-09-23T20:44:43.631710+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:50:54.591529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"90'161","reported_seq":637,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-23T20:44:30.508072+0000","last_change":"2023-09-23T20:44:14.669973+0000","last_active":"2023-09-23T20:44:30.508072+0000","last_peered":"2023-09-23T20:44:30.508072+0000","last_clean":"2023-09-23T20:44:30.508072+0000","last_became_active":"2023-09-23T20:43:46.519592+0000","last_became_peered":"2023-09-23T20:43:46.519592+0000","last_unstale":"2023-09-23T20:44:30.508072+0000","last_undegraded":"2023-09-23T20:44:30.508072+0000","last_fullsized":"2023-09-23T20:44:30.508072+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:14.669831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:14.669831+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:56:28.076617+0000","scrub_duration":0.0083441460000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":539,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-23T20:44:30.505227+0000","last_change":"2023-09-23T20:32:33.270617+0000","last_active":"2023-09-23T20:44:30.505227+0000","last_peered":"2023-09-23T20:44:30.505227+0000","last_clean":"2023-09-23T20:44:30.505227+0000","last_became_active":"2023-09-23T20:32:33.227902+0000","last_became_peered":"2023-09-23T20:32:33.227902+0000","last_unstale":"2023-09-23T20:44:30.505227+0000","last_undegraded":"2023-09-23T20:44:30.505227+0000","last_fullsized":"2023-09-23T20:44:30.505227+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:30:26.847831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":360,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":27,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-23T20:44:38.625044+0000","last_change":"2023-09-23T20:44:10.611381+0000","last_active":"2023-09-23T20:44:38.625044+0000","last_peered":"2023-09-23T20:44:38.625044+0000","last_clean":"2023-09-23T20:44:38.625044+0000","last_became_active":"2023-09-23T20:44:10.611091+0000","last_became_peered":"2023-09-23T20:44:10.611091+0000","last_unstale":"2023-09-23T20:44:38.625044+0000","last_undegraded":"2023-09-23T20:44:38.625044+0000","last_fullsized":"2023-09-23T20:44:38.625044+0000","mapping_epoch":496,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":497,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:00:30.162478+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"90'161","reported_seq":638,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-23T20:44:38.613930+0000","last_change":"2023-09-23T20:36:02.548387+0000","last_active":"2023-09-23T20:44:38.613930+0000","last_peered":"2023-09-23T20:44:38.613930+0000","last_clean":"2023-09-23T20:44:38.613930+0000","last_became_active":"2023-09-23T20:35:26.644787+0000","last_became_peered":"2023-09-23T20:35:26.644787+0000","last_unstale":"2023-09-23T20:44:38.613930+0000","last_undegraded":"2023-09-23T20:44:38.613930+0000","last_fullsized":"2023-09-23T20:44:38.613930+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:02.548285+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:02.548285+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:37:15.216553+0000","scrub_duration":0.0015514459999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":263,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-09-23T20:44:44.899658+0000","last_change":"2023-09-23T20:38:02.560646+0000","last_active":"2023-09-23T20:44:44.899658+0000","last_peered":"2023-09-23T20:44:44.899658+0000","last_clean":"2023-09-23T20:44:44.899658+0000","last_became_active":"2023-09-23T20:38:02.539814+0000","last_became_peered":"2023-09-23T20:38:02.539814+0000","last_unstale":"2023-09-23T20:44:44.899658+0000","last_undegraded":"2023-09-23T20:44:44.899658+0000","last_fullsized":"2023-09-23T20:44:44.899658+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:00:09.435023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":243,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-09-23T20:44:44.899720+0000","last_change":"2023-09-23T20:38:02.579222+0000","last_active":"2023-09-23T20:44:44.899720+0000","last_peered":"2023-09-23T20:44:44.899720+0000","last_clean":"2023-09-23T20:44:44.899720+0000","last_became_active":"2023-09-23T20:38:02.578813+0000","last_became_peered":"2023-09-23T20:38:02.578813+0000","last_unstale":"2023-09-23T20:44:44.899720+0000","last_undegraded":"2023-09-23T20:44:44.899720+0000","last_fullsized":"2023-09-23T20:44:44.899720+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:57:31.080027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"90'161","reported_seq":765,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-23T20:44:30.506640+0000","last_change":"2023-09-23T20:43:46.544727+0000","last_active":"2023-09-23T20:44:30.506640+0000","last_peered":"2023-09-23T20:44:30.506640+0000","last_clean":"2023-09-23T20:44:30.506640+0000","last_became_active":"2023-09-23T20:43:46.543952+0000","last_became_peered":"2023-09-23T20:43:46.543952+0000","last_unstale":"2023-09-23T20:44:30.506640+0000","last_undegraded":"2023-09-23T20:44:30.506640+0000","last_fullsized":"2023-09-23T20:44:30.506640+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:43.578666+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:43.578666+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:11:16.611920+0000","scrub_duration":0.0021890379999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":538,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-23T20:44:30.504592+0000","last_change":"2023-09-23T20:32:33.242943+0000","last_active":"2023-09-23T20:44:30.504592+0000","last_peered":"2023-09-23T20:44:30.504592+0000","last_clean":"2023-09-23T20:44:30.504592+0000","last_became_active":"2023-09-23T20:32:33.225468+0000","last_became_peered":"2023-09-23T20:32:33.225468+0000","last_unstale":"2023-09-23T20:44:30.504592+0000","last_undegraded":"2023-09-23T20:44:30.504592+0000","last_fullsized":"2023-09-23T20:44:30.504592+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:40:47.867573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":251,"reported_epoch":514,"state":"active+clean","last_fresh":"2023-09-23T20:44:43.630733+0000","last_change":"2023-09-23T20:37:43.571836+0000","last_active":"2023-09-23T20:44:43.630733+0000","last_peered":"2023-09-23T20:44:43.630733+0000","last_clean":"2023-09-23T20:44:43.630733+0000","last_became_active":"2023-09-23T20:37:43.571511+0000","last_became_peered":"2023-09-23T20:37:43.571511+0000","last_unstale":"2023-09-23T20:44:43.630733+0000","last_undegraded":"2023-09-23T20:44:43.630733+0000","last_fullsized":"2023-09-23T20:44:43.630733+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:59:25.507917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"90'161","reported_seq":792,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-23T20:44:30.503158+0000","last_change":"2023-09-23T20:44:20.041766+0000","last_active":"2023-09-23T20:44:30.503158+0000","last_peered":"2023-09-23T20:44:30.503158+0000","last_clean":"2023-09-23T20:44:30.503158+0000","last_became_active":"2023-09-23T20:43:16.765015+0000","last_became_peered":"2023-09-23T20:43:16.765015+0000","last_unstale":"2023-09-23T20:44:30.503158+0000","last_undegraded":"2023-09-23T20:44:30.503158+0000","last_fullsized":"2023-09-23T20:44:30.503158+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:20.041690+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:20.041690+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:33:30.150109+0000","scrub_duration":0.0026180330000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":467,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-23T20:44:30.506399+0000","last_change":"2023-09-23T20:43:46.521294+0000","last_active":"2023-09-23T20:44:30.506399+0000","last_peered":"2023-09-23T20:44:30.506399+0000","last_clean":"2023-09-23T20:44:30.506399+0000","last_became_active":"2023-09-23T20:43:46.521001+0000","last_became_peered":"2023-09-23T20:43:46.521001+0000","last_unstale":"2023-09-23T20:44:30.506399+0000","last_undegraded":"2023-09-23T20:44:30.506399+0000","last_fullsized":"2023-09-23T20:44:30.506399+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:01:35.126716+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":279,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-23T20:44:30.505299+0000","last_change":"2023-09-23T20:44:01.841565+0000","last_active":"2023-09-23T20:44:30.505299+0000","last_peered":"2023-09-23T20:44:30.505299+0000","last_clean":"2023-09-23T20:44:30.505299+0000","last_became_active":"2023-09-23T20:44:01.841255+0000","last_became_peered":"2023-09-23T20:44:01.841255+0000","last_unstale":"2023-09-23T20:44:30.505299+0000","last_undegraded":"2023-09-23T20:44:30.505299+0000","last_fullsized":"2023-09-23T20:44:30.505299+0000","mapping_epoch":491,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":492,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:18:44.817589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.13","version":"90'161","reported_seq":800,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-23T20:44:38.613934+0000","last_change":"2023-09-23T20:35:02.238923+0000","last_active":"2023-09-23T20:44:38.613934+0000","last_peered":"2023-09-23T20:44:38.613934+0000","last_clean":"2023-09-23T20:44:38.613934+0000","last_became_active":"2023-09-23T20:35:02.238293+0000","last_became_peered":"2023-09-23T20:35:02.238293+0000","last_unstale":"2023-09-23T20:44:38.613934+0000","last_undegraded":"2023-09-23T20:44:38.613934+0000","last_fullsized":"2023-09-23T20:44:38.613934+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:42.544308+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:42.544308+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:51:22.281932+0000","scrub_duration":0.0029958179999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":515,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-23T20:44:30.503481+0000","last_change":"2023-09-23T20:38:02.539774+0000","last_active":"2023-09-23T20:44:30.503481+0000","last_peered":"2023-09-23T20:44:30.503481+0000","last_clean":"2023-09-23T20:44:30.503481+0000","last_became_active":"2023-09-23T20:38:02.539436+0000","last_became_peered":"2023-09-23T20:38:02.539436+0000","last_unstale":"2023-09-23T20:44:30.503481+0000","last_undegraded":"2023-09-23T20:44:30.503481+0000","last_fullsized":"2023-09-23T20:44:30.503481+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:50:57.375803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":476,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-23T20:44:30.507909+0000","last_change":"2023-09-23T20:43:46.521735+0000","last_active":"2023-09-23T20:44:30.507909+0000","last_peered":"2023-09-23T20:44:30.507909+0000","last_clean":"2023-09-23T20:44:30.507909+0000","last_became_active":"2023-09-23T20:43:46.520762+0000","last_became_peered":"2023-09-23T20:43:46.520762+0000","last_unstale":"2023-09-23T20:44:30.507909+0000","last_undegraded":"2023-09-23T20:44:30.507909+0000","last_fullsized":"2023-09-23T20:44:30.507909+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:26:48.841930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"90'161","reported_seq":596,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-23T20:44:38.625569+0000","last_change":"2023-09-23T20:44:18.952808+0000","last_active":"2023-09-23T20:44:38.625569+0000","last_peered":"2023-09-23T20:44:38.625569+0000","last_clean":"2023-09-23T20:44:38.625569+0000","last_became_active":"2023-09-23T20:43:16.758652+0000","last_became_peered":"2023-09-23T20:43:16.758652+0000","last_unstale":"2023-09-23T20:44:38.625569+0000","last_undegraded":"2023-09-23T20:44:38.625569+0000","last_fullsized":"2023-09-23T20:44:38.625569+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:18.952693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:18.952693+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:12:19.496465+0000","scrub_duration":0.001699365,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":170,"reported_epoch":513,"state":"peering","last_fresh":"2023-09-23T20:44:42.663680+0000","last_change":"2023-09-23T20:44:42.663999+0000","last_active":"2023-09-23T20:44:42.663639+0000","last_peered":"2023-09-23T20:44:38.602018+0000","last_clean":"2023-09-23T20:44:38.602018+0000","last_became_active":"2023-09-23T20:42:41.472591+0000","last_became_peered":"2023-09-23T20:42:41.472591+0000","last_unstale":"2023-09-23T20:44:42.663680+0000","last_undegraded":"2023-09-23T20:44:42.663680+0000","last_fullsized":"2023-09-23T20:44:42.663680+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:38:28.624882+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[0,3,5],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":308,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-23T20:44:42.658052+0000","last_change":"2023-09-23T20:37:43.571083+0000","last_active":"2023-09-23T20:44:42.658052+0000","last_peered":"2023-09-23T20:44:42.658052+0000","last_clean":"2023-09-23T20:44:42.658052+0000","last_became_active":"2023-09-23T20:37:43.570306+0000","last_became_peered":"2023-09-23T20:37:43.570306+0000","last_unstale":"2023-09-23T20:44:42.658052+0000","last_undegraded":"2023-09-23T20:44:42.658052+0000","last_fullsized":"2023-09-23T20:44:42.658052+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:42:02.396718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.5","version":"512'12357","reported_seq":20909,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-23T20:44:41.267639+0000","last_change":"2023-09-23T20:44:20.680609+0000","last_active":"2023-09-23T20:44:41.267639+0000","last_peered":"2023-09-23T20:44:41.267639+0000","last_clean":"2023-09-23T20:44:41.267639+0000","last_became_active":"2023-09-23T20:44:20.638636+0000","last_became_peered":"2023-09-23T20:44:20.638636+0000","last_unstale":"2023-09-23T20:44:41.267639+0000","last_undegraded":"2023-09-23T20:44:41.267639+0000","last_fullsized":"2023-09-23T20:44:41.267639+0000","mapping_epoch":504,"log_start":"466'4810","ondisk_log_start":"466'4810","created":455,"last_epoch_clean":505,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":7547,"log_dups_size":0,"ondisk_log_size":7547,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:11:11.154742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":106758144,"num_objects":1629,"num_object_clones":0,"num_object_copies":4887,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1629,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12357,"num_write_kb":447552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4204,"num_bytes_recovered":275316736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.d","version":"90'161","reported_seq":804,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-23T20:44:30.505049+0000","last_change":"2023-09-23T20:34:59.496438+0000","last_active":"2023-09-23T20:44:30.505049+0000","last_peered":"2023-09-23T20:44:30.505049+0000","last_clean":"2023-09-23T20:44:30.505049+0000","last_became_active":"2023-09-23T20:33:38.970527+0000","last_became_peered":"2023-09-23T20:33:38.970527+0000","last_unstale":"2023-09-23T20:44:30.505049+0000","last_undegraded":"2023-09-23T20:44:30.505049+0000","last_fullsized":"2023-09-23T20:44:30.505049+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:34:59.496375+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:34:59.496375+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:46:18.244734+0000","scrub_duration":0.0017843189999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":515,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-23T20:44:30.504898+0000","last_change":"2023-09-23T20:32:33.283752+0000","last_active":"2023-09-23T20:44:30.504898+0000","last_peered":"2023-09-23T20:44:30.504898+0000","last_clean":"2023-09-23T20:44:30.504898+0000","last_became_active":"2023-09-23T20:32:33.230503+0000","last_became_peered":"2023-09-23T20:32:33.230503+0000","last_unstale":"2023-09-23T20:44:30.504898+0000","last_undegraded":"2023-09-23T20:44:30.504898+0000","last_fullsized":"2023-09-23T20:44:30.504898+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:29.084410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":326,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-23T20:44:30.503454+0000","last_change":"2023-09-23T20:43:16.765253+0000","last_active":"2023-09-23T20:44:30.503454+0000","last_peered":"2023-09-23T20:44:30.503454+0000","last_clean":"2023-09-23T20:44:30.503454+0000","last_became_active":"2023-09-23T20:43:16.764978+0000","last_became_peered":"2023-09-23T20:43:16.764978+0000","last_unstale":"2023-09-23T20:44:30.503454+0000","last_undegraded":"2023-09-23T20:44:30.503454+0000","last_fullsized":"2023-09-23T20:44:30.503454+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:07:23.539372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.4","version":"513'12772","reported_seq":12840,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-23T20:44:42.823709+0000","last_change":"2023-09-23T20:41:04.575200+0000","last_active":"2023-09-23T20:44:42.823709+0000","last_peered":"2023-09-23T20:44:42.823709+0000","last_clean":"2023-09-23T20:44:42.823709+0000","last_became_active":"2023-09-23T20:41:04.574824+0000","last_became_peered":"2023-09-23T20:41:04.574824+0000","last_unstale":"2023-09-23T20:44:42.823709+0000","last_undegraded":"2023-09-23T20:44:42.823709+0000","last_fullsized":"2023-09-23T20:44:42.823709+0000","mapping_epoch":455,"log_start":"470'7183","ondisk_log_start":"470'7183","created":455,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":5589,"log_dups_size":0,"ondisk_log_size":5589,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:50:46.714383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":100794368,"num_objects":1538,"num_object_clones":0,"num_object_copies":4614,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1538,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12772,"num_write_kb":457920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.c","version":"90'161","reported_seq":775,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-23T20:44:42.657240+0000","last_change":"2023-09-23T20:41:09.506481+0000","last_active":"2023-09-23T20:44:42.657240+0000","last_peered":"2023-09-23T20:44:42.657240+0000","last_clean":"2023-09-23T20:44:42.657240+0000","last_became_active":"2023-09-23T20:38:28.442229+0000","last_became_peered":"2023-09-23T20:38:28.442229+0000","last_unstale":"2023-09-23T20:44:42.657240+0000","last_undegraded":"2023-09-23T20:44:42.657240+0000","last_fullsized":"2023-09-23T20:44:42.657240+0000","mapping_epoch":411,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":412,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:41:09.506338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:41:09.506338+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:19:46.265011+0000","scrub_duration":0.01112002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":550,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-23T20:44:30.505048+0000","last_change":"2023-09-23T20:34:59.987981+0000","last_active":"2023-09-23T20:44:30.505048+0000","last_peered":"2023-09-23T20:44:30.505048+0000","last_clean":"2023-09-23T20:44:30.505048+0000","last_became_active":"2023-09-23T20:34:59.985368+0000","last_became_peered":"2023-09-23T20:34:59.985368+0000","last_unstale":"2023-09-23T20:44:30.505048+0000","last_undegraded":"2023-09-23T20:44:30.505048+0000","last_fullsized":"2023-09-23T20:44:30.505048+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:12:43.082493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":213,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-09-23T20:44:44.899444+0000","last_change":"2023-09-23T20:32:33.232142+0000","last_active":"2023-09-23T20:44:44.899444+0000","last_peered":"2023-09-23T20:44:44.899444+0000","last_clean":"2023-09-23T20:44:44.899444+0000","last_became_active":"2023-09-23T20:32:33.231683+0000","last_became_peered":"2023-09-23T20:32:33.231683+0000","last_unstale":"2023-09-23T20:44:44.899444+0000","last_undegraded":"2023-09-23T20:44:44.899444+0000","last_fullsized":"2023-09-23T20:44:44.899444+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:02:58.336348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"512'12719","reported_seq":12796,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-23T20:44:42.518777+0000","last_change":"2023-09-23T20:43:16.854027+0000","last_active":"2023-09-23T20:44:42.518777+0000","last_peered":"2023-09-23T20:44:42.518777+0000","last_clean":"2023-09-23T20:44:42.518777+0000","last_became_active":"2023-09-23T20:43:16.766133+0000","last_became_peered":"2023-09-23T20:43:16.766133+0000","last_unstale":"2023-09-23T20:44:42.518777+0000","last_undegraded":"2023-09-23T20:44:42.518777+0000","last_fullsized":"2023-09-23T20:44:42.518777+0000","mapping_epoch":481,"log_start":"469'6985","ondisk_log_start":"469'6985","created":455,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":5734,"log_dups_size":0,"ondisk_log_size":5734,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:42:32.213805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":95485952,"num_objects":1457,"num_object_clones":0,"num_object_copies":4371,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1457,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12719,"num_write_kb":453632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.f","version":"90'161","reported_seq":668,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-23T20:44:38.603223+0000","last_change":"2023-09-23T20:38:40.491902+0000","last_active":"2023-09-23T20:44:38.603223+0000","last_peered":"2023-09-23T20:44:38.603223+0000","last_clean":"2023-09-23T20:44:38.603223+0000","last_became_active":"2023-09-23T20:38:40.491563+0000","last_became_peered":"2023-09-23T20:38:40.491563+0000","last_unstale":"2023-09-23T20:44:38.603223+0000","last_undegraded":"2023-09-23T20:44:38.603223+0000","last_fullsized":"2023-09-23T20:44:38.603223+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:46.217489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:46.217489+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:39:25.008796+0000","scrub_duration":0.001456429,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":540,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-23T20:44:30.505143+0000","last_change":"2023-09-23T20:40:19.677655+0000","last_active":"2023-09-23T20:44:30.505143+0000","last_peered":"2023-09-23T20:44:30.505143+0000","last_clean":"2023-09-23T20:44:30.505143+0000","last_became_active":"2023-09-23T20:40:19.551189+0000","last_became_peered":"2023-09-23T20:40:19.551189+0000","last_unstale":"2023-09-23T20:44:30.505143+0000","last_undegraded":"2023-09-23T20:44:30.505143+0000","last_fullsized":"2023-09-23T20:44:30.505143+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:11:01.559576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":118,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-23T20:44:38.602480+0000","last_change":"2023-09-23T20:38:02.590613+0000","last_active":"2023-09-23T20:44:38.602480+0000","last_peered":"2023-09-23T20:44:38.602480+0000","last_clean":"2023-09-23T20:44:38.602480+0000","last_became_active":"2023-09-23T20:38:02.590284+0000","last_became_peered":"2023-09-23T20:38:02.590284+0000","last_unstale":"2023-09-23T20:44:38.602480+0000","last_undegraded":"2023-09-23T20:44:38.602480+0000","last_fullsized":"2023-09-23T20:44:38.602480+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:40:07.825054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"513'12602","reported_seq":12676,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-23T20:44:42.866064+0000","last_change":"2023-09-23T20:43:16.788249+0000","last_active":"2023-09-23T20:44:42.866064+0000","last_peered":"2023-09-23T20:44:42.866064+0000","last_clean":"2023-09-23T20:44:42.866064+0000","last_became_active":"2023-09-23T20:43:16.763555+0000","last_became_peered":"2023-09-23T20:43:16.763555+0000","last_unstale":"2023-09-23T20:44:42.866064+0000","last_undegraded":"2023-09-23T20:44:42.866064+0000","last_fullsized":"2023-09-23T20:44:42.866064+0000","mapping_epoch":481,"log_start":"460'2904","ondisk_log_start":"460'2904","created":455,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":9698,"log_dups_size":0,"ondisk_log_size":9698,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:19:57.856019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":98828288,"num_objects":1508,"num_object_clones":0,"num_object_copies":4524,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1508,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12602,"num_write_kb":451520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[2,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.e","version":"90'161","reported_seq":616,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-09-23T20:44:44.899785+0000","last_change":"2023-09-23T20:36:00.322453+0000","last_active":"2023-09-23T20:44:44.899785+0000","last_peered":"2023-09-23T20:44:44.899785+0000","last_clean":"2023-09-23T20:44:44.899785+0000","last_became_active":"2023-09-23T20:35:02.233536+0000","last_became_peered":"2023-09-23T20:35:02.233536+0000","last_unstale":"2023-09-23T20:44:44.899785+0000","last_undegraded":"2023-09-23T20:44:44.899785+0000","last_fullsized":"2023-09-23T20:44:44.899785+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:00.322381+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:00.322381+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:02:15.489069+0000","scrub_duration":0.0017321859999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":473,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-23T20:44:42.657739+0000","last_change":"2023-09-23T20:43:16.756437+0000","last_active":"2023-09-23T20:44:42.657739+0000","last_peered":"2023-09-23T20:44:42.657739+0000","last_clean":"2023-09-23T20:44:42.657739+0000","last_became_active":"2023-09-23T20:43:16.756193+0000","last_became_peered":"2023-09-23T20:43:16.756193+0000","last_unstale":"2023-09-23T20:44:42.657739+0000","last_undegraded":"2023-09-23T20:44:42.657739+0000","last_fullsized":"2023-09-23T20:44:42.657739+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:33:25.632781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":485,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-23T20:44:30.510507+0000","last_change":"2023-09-23T20:43:46.541524+0000","last_active":"2023-09-23T20:44:30.510507+0000","last_peered":"2023-09-23T20:44:30.510507+0000","last_clean":"2023-09-23T20:44:30.510507+0000","last_became_active":"2023-09-23T20:43:46.541051+0000","last_became_peered":"2023-09-23T20:43:46.541051+0000","last_unstale":"2023-09-23T20:44:30.510507+0000","last_undegraded":"2023-09-23T20:44:30.510507+0000","last_fullsized":"2023-09-23T20:44:30.510507+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:59:36.632592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"513'12601","reported_seq":12667,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-23T20:44:42.796848+0000","last_change":"2023-09-23T20:41:04.558473+0000","last_active":"2023-09-23T20:44:42.796848+0000","last_peered":"2023-09-23T20:44:42.796848+0000","last_clean":"2023-09-23T20:44:42.796848+0000","last_became_active":"2023-09-23T20:41:04.558173+0000","last_became_peered":"2023-09-23T20:41:04.558173+0000","last_unstale":"2023-09-23T20:44:42.796848+0000","last_undegraded":"2023-09-23T20:44:42.796848+0000","last_fullsized":"2023-09-23T20:44:42.796848+0000","mapping_epoch":455,"log_start":"467'5763","ondisk_log_start":"467'5763","created":455,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":6838,"log_dups_size":0,"ondisk_log_size":6838,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:17:53.184877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":102432768,"num_objects":1563,"num_object_clones":0,"num_object_copies":4689,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1563,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12601,"num_write_kb":453248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":917,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-23T20:44:30.503343+0000","last_change":"2023-09-23T20:35:02.523308+0000","last_active":"2023-09-23T20:44:30.503343+0000","last_peered":"2023-09-23T20:44:30.503343+0000","last_clean":"2023-09-23T20:44:30.503343+0000","last_became_active":"2023-09-23T20:33:38.979989+0000","last_became_peered":"2023-09-23T20:33:38.979989+0000","last_unstale":"2023-09-23T20:44:30.503343+0000","last_undegraded":"2023-09-23T20:44:30.503343+0000","last_fullsized":"2023-09-23T20:44:30.503343+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:35:02.523203+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:35:02.523203+0000","last_clean_scrub_stamp":"2023-09-23T20:35:02.523203+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:02:41.472958+0000","scrub_duration":0.0085927989999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":335,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-23T20:44:42.657279+0000","last_change":"2023-09-23T20:37:43.583731+0000","last_active":"2023-09-23T20:44:42.657279+0000","last_peered":"2023-09-23T20:44:42.657279+0000","last_clean":"2023-09-23T20:44:42.657279+0000","last_became_active":"2023-09-23T20:37:43.583066+0000","last_became_peered":"2023-09-23T20:37:43.583066+0000","last_unstale":"2023-09-23T20:44:42.657279+0000","last_undegraded":"2023-09-23T20:44:42.657279+0000","last_fullsized":"2023-09-23T20:44:42.657279+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:03:45.211900+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":233,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-09-23T20:44:44.899560+0000","last_change":"2023-09-23T20:38:02.579814+0000","last_active":"2023-09-23T20:44:44.899560+0000","last_peered":"2023-09-23T20:44:44.899560+0000","last_clean":"2023-09-23T20:44:44.899560+0000","last_became_active":"2023-09-23T20:38:02.579253+0000","last_became_peered":"2023-09-23T20:38:02.579253+0000","last_unstale":"2023-09-23T20:44:44.899560+0000","last_undegraded":"2023-09-23T20:44:44.899560+0000","last_fullsized":"2023-09-23T20:44:44.899560+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:05:07.298198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"513'12732","reported_seq":12800,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-23T20:44:42.780369+0000","last_change":"2023-09-23T20:41:04.573101+0000","last_active":"2023-09-23T20:44:42.780369+0000","last_peered":"2023-09-23T20:44:42.780369+0000","last_clean":"2023-09-23T20:44:42.780369+0000","last_became_active":"2023-09-23T20:41:04.572728+0000","last_became_peered":"2023-09-23T20:41:04.572728+0000","last_unstale":"2023-09-23T20:44:42.780369+0000","last_undegraded":"2023-09-23T20:44:42.780369+0000","last_fullsized":"2023-09-23T20:44:42.780369+0000","mapping_epoch":455,"log_start":"470'7125","ondisk_log_start":"470'7125","created":455,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":5607,"log_dups_size":0,"ondisk_log_size":5607,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:21:10.721593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":100401152,"num_objects":1532,"num_object_clones":0,"num_object_copies":4596,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1532,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12732,"num_write_kb":456448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":611,"reported_epoch":513,"state":"peering","last_fresh":"2023-09-23T20:44:42.662146+0000","last_change":"2023-09-23T20:44:42.662314+0000","last_active":"2023-09-23T20:44:42.662146+0000","last_peered":"2023-09-23T20:44:38.624021+0000","last_clean":"2023-09-23T20:44:38.624021+0000","last_became_active":"2023-09-23T20:42:44.508223+0000","last_became_peered":"2023-09-23T20:42:44.508223+0000","last_unstale":"2023-09-23T20:44:42.662146+0000","last_undegraded":"2023-09-23T20:44:42.662146+0000","last_fullsized":"2023-09-23T20:44:42.662146+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":475,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:42:45.492220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:42:45.492220+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:02:39.158558+0000","scrub_duration":0.015259412,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[3,5,6],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":118,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-23T20:44:38.602554+0000","last_change":"2023-09-23T20:38:02.578703+0000","last_active":"2023-09-23T20:44:38.602554+0000","last_peered":"2023-09-23T20:44:38.602554+0000","last_clean":"2023-09-23T20:44:38.602554+0000","last_became_active":"2023-09-23T20:38:02.578414+0000","last_became_peered":"2023-09-23T20:38:02.578414+0000","last_unstale":"2023-09-23T20:44:38.602554+0000","last_undegraded":"2023-09-23T20:44:38.602554+0000","last_fullsized":"2023-09-23T20:44:38.602554+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:43:59.779295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":182,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-09-23T20:44:44.899559+0000","last_change":"2023-09-23T20:37:31.296606+0000","last_active":"2023-09-23T20:44:44.899559+0000","last_peered":"2023-09-23T20:44:44.899559+0000","last_clean":"2023-09-23T20:44:44.899559+0000","last_became_active":"2023-09-23T20:37:31.295924+0000","last_became_peered":"2023-09-23T20:37:31.295924+0000","last_unstale":"2023-09-23T20:44:44.899559+0000","last_undegraded":"2023-09-23T20:44:44.899559+0000","last_fullsized":"2023-09-23T20:44:44.899559+0000","mapping_epoch":391,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":392,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:27:50.948510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"512'12648","reported_seq":12713,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-23T20:44:42.501311+0000","last_change":"2023-09-23T20:41:04.555980+0000","last_active":"2023-09-23T20:44:42.501311+0000","last_peered":"2023-09-23T20:44:42.501311+0000","last_clean":"2023-09-23T20:44:42.501311+0000","last_became_active":"2023-09-23T20:41:04.555688+0000","last_became_peered":"2023-09-23T20:41:04.555688+0000","last_unstale":"2023-09-23T20:44:42.501311+0000","last_undegraded":"2023-09-23T20:44:42.501311+0000","last_fullsized":"2023-09-23T20:44:42.501311+0000","mapping_epoch":455,"log_start":"469'6925","ondisk_log_start":"469'6925","created":455,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":5723,"log_dups_size":0,"ondisk_log_size":5723,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:28:42.819404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":103022592,"num_objects":1572,"num_object_clones":0,"num_object_copies":4716,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1572,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12648,"num_write_kb":455040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.b","version":"90'161","reported_seq":816,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-23T20:44:38.614022+0000","last_change":"2023-09-23T20:40:22.445808+0000","last_active":"2023-09-23T20:44:38.614022+0000","last_peered":"2023-09-23T20:44:38.614022+0000","last_clean":"2023-09-23T20:44:38.614022+0000","last_became_active":"2023-09-23T20:40:22.445456+0000","last_became_peered":"2023-09-23T20:40:22.445456+0000","last_unstale":"2023-09-23T20:44:38.614022+0000","last_undegraded":"2023-09-23T20:44:38.614022+0000","last_fullsized":"2023-09-23T20:44:38.614022+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:05.595042+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:05.595042+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:39:23.678154+0000","scrub_duration":0.007271675,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":172,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-09-23T20:44:44.899263+0000","last_change":"2023-09-23T20:34:59.983122+0000","last_active":"2023-09-23T20:44:44.899263+0000","last_peered":"2023-09-23T20:44:44.899263+0000","last_clean":"2023-09-23T20:44:44.899263+0000","last_became_active":"2023-09-23T20:34:59.982809+0000","last_became_peered":"2023-09-23T20:34:59.982809+0000","last_unstale":"2023-09-23T20:44:44.899263+0000","last_undegraded":"2023-09-23T20:44:44.899263+0000","last_fullsized":"2023-09-23T20:44:44.899263+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:49:40.839723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":253,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-09-23T20:44:44.899909+0000","last_change":"2023-09-23T20:44:10.606995+0000","last_active":"2023-09-23T20:44:44.899909+0000","last_peered":"2023-09-23T20:44:44.899909+0000","last_clean":"2023-09-23T20:44:44.899909+0000","last_became_active":"2023-09-23T20:44:10.606718+0000","last_became_peered":"2023-09-23T20:44:10.606718+0000","last_unstale":"2023-09-23T20:44:44.899909+0000","last_undegraded":"2023-09-23T20:44:44.899909+0000","last_fullsized":"2023-09-23T20:44:44.899909+0000","mapping_epoch":496,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":497,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:13:12.496245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"513'12741","reported_seq":12810,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-23T20:44:42.896615+0000","last_change":"2023-09-23T20:41:04.573300+0000","last_active":"2023-09-23T20:44:42.896615+0000","last_peered":"2023-09-23T20:44:42.896615+0000","last_clean":"2023-09-23T20:44:42.896615+0000","last_became_active":"2023-09-23T20:41:04.572919+0000","last_became_peered":"2023-09-23T20:41:04.572919+0000","last_unstale":"2023-09-23T20:44:42.896615+0000","last_undegraded":"2023-09-23T20:44:42.896615+0000","last_fullsized":"2023-09-23T20:44:42.896615+0000","mapping_epoch":455,"log_start":"469'6794","ondisk_log_start":"469'6794","created":455,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":5947,"log_dups_size":0,"ondisk_log_size":5947,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:57:46.465243+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":103088128,"num_objects":1573,"num_object_clones":0,"num_object_copies":4719,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1573,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12741,"num_write_kb":458048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":538,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-23T20:44:38.625347+0000","last_change":"2023-09-23T20:43:51.693618+0000","last_active":"2023-09-23T20:44:38.625347+0000","last_peered":"2023-09-23T20:44:38.625347+0000","last_clean":"2023-09-23T20:44:38.625347+0000","last_became_active":"2023-09-23T20:43:16.765186+0000","last_became_peered":"2023-09-23T20:43:16.765186+0000","last_unstale":"2023-09-23T20:44:38.625347+0000","last_undegraded":"2023-09-23T20:44:38.625347+0000","last_fullsized":"2023-09-23T20:44:38.625347+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:43:51.693459+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:43:51.693459+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:33:30.208355+0000","scrub_duration":0.0016145269999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":309,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-23T20:44:42.659773+0000","last_change":"2023-09-23T20:37:43.570610+0000","last_active":"2023-09-23T20:44:42.659773+0000","last_peered":"2023-09-23T20:44:42.659773+0000","last_clean":"2023-09-23T20:44:42.659773+0000","last_became_active":"2023-09-23T20:37:43.570216+0000","last_became_peered":"2023-09-23T20:37:43.570216+0000","last_unstale":"2023-09-23T20:44:42.659773+0000","last_undegraded":"2023-09-23T20:44:42.659773+0000","last_fullsized":"2023-09-23T20:44:42.659773+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:39:26.555329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":338,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-23T20:44:38.614938+0000","last_change":"2023-09-23T20:32:33.230582+0000","last_active":"2023-09-23T20:44:38.614938+0000","last_peered":"2023-09-23T20:44:38.614938+0000","last_clean":"2023-09-23T20:44:38.614938+0000","last_became_active":"2023-09-23T20:32:33.230257+0000","last_became_peered":"2023-09-23T20:32:33.230257+0000","last_unstale":"2023-09-23T20:44:38.614938+0000","last_undegraded":"2023-09-23T20:44:38.614938+0000","last_fullsized":"2023-09-23T20:44:38.614938+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:11:45.333096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.f","version":"513'12860","reported_seq":12934,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-23T20:44:42.805133+0000","last_change":"2023-09-23T20:43:16.778725+0000","last_active":"2023-09-23T20:44:42.805133+0000","last_peered":"2023-09-23T20:44:42.805133+0000","last_clean":"2023-09-23T20:44:42.805133+0000","last_became_active":"2023-09-23T20:43:16.759443+0000","last_became_peered":"2023-09-23T20:43:16.759443+0000","last_unstale":"2023-09-23T20:44:42.805133+0000","last_undegraded":"2023-09-23T20:44:42.805133+0000","last_fullsized":"2023-09-23T20:44:42.805133+0000","mapping_epoch":481,"log_start":"470'7284","ondisk_log_start":"470'7284","created":455,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":5576,"log_dups_size":0,"ondisk_log_size":5576,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:55:32.141361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":100007936,"num_objects":1526,"num_object_clones":0,"num_object_copies":4578,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1526,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12860,"num_write_kb":460352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":701,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-23T20:44:38.603134+0000","last_change":"2023-09-23T20:41:07.424609+0000","last_active":"2023-09-23T20:44:38.603134+0000","last_peered":"2023-09-23T20:44:38.603134+0000","last_clean":"2023-09-23T20:44:38.603134+0000","last_became_active":"2023-09-23T20:38:05.622261+0000","last_became_peered":"2023-09-23T20:38:05.622261+0000","last_unstale":"2023-09-23T20:44:38.603134+0000","last_undegraded":"2023-09-23T20:44:38.603134+0000","last_fullsized":"2023-09-23T20:44:38.603134+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:41:07.424492+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:41:07.424492+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:45:21.689123+0000","scrub_duration":0.0018960959999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"86'119","reported_seq":823,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-23T20:44:38.617118+0000","last_change":"2023-09-23T20:29:45.521257+0000","last_active":"2023-09-23T20:44:38.617118+0000","last_peered":"2023-09-23T20:44:38.617118+0000","last_clean":"2023-09-23T20:44:38.617118+0000","last_became_active":"2023-09-23T20:29:45.520546+0000","last_became_peered":"2023-09-23T20:29:45.520546+0000","last_unstale":"2023-09-23T20:44:38.617118+0000","last_undegraded":"2023-09-23T20:44:38.617118+0000","last_fullsized":"2023-09-23T20:44:38.617118+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":119,"log_dups_size":0,"ondisk_log_size":119,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:58:56.085777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032705499999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":523,"num_read_kb":3503,"num_write":136,"num_write_kb":3128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":306,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-23T20:44:42.657070+0000","last_change":"2023-09-23T20:43:16.756494+0000","last_active":"2023-09-23T20:44:42.657070+0000","last_peered":"2023-09-23T20:44:42.657070+0000","last_clean":"2023-09-23T20:44:42.657070+0000","last_became_active":"2023-09-23T20:43:16.756131+0000","last_became_peered":"2023-09-23T20:43:16.756131+0000","last_unstale":"2023-09-23T20:44:42.657070+0000","last_undegraded":"2023-09-23T20:44:42.657070+0000","last_fullsized":"2023-09-23T20:44:42.657070+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:10:19.219533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":470,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-23T20:44:38.625350+0000","last_change":"2023-09-23T20:40:19.530942+0000","last_active":"2023-09-23T20:44:38.625350+0000","last_peered":"2023-09-23T20:44:38.625350+0000","last_clean":"2023-09-23T20:44:38.625350+0000","last_became_active":"2023-09-23T20:40:19.530617+0000","last_became_peered":"2023-09-23T20:40:19.530617+0000","last_unstale":"2023-09-23T20:44:38.625350+0000","last_undegraded":"2023-09-23T20:44:38.625350+0000","last_fullsized":"2023-09-23T20:44:38.625350+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:24:09.231934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.9","version":"512'12602","reported_seq":12687,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-23T20:44:41.328109+0000","last_change":"2023-09-23T20:43:46.544365+0000","last_active":"2023-09-23T20:44:41.328109+0000","last_peered":"2023-09-23T20:44:41.328109+0000","last_clean":"2023-09-23T20:44:41.328109+0000","last_became_active":"2023-09-23T20:43:46.519248+0000","last_became_peered":"2023-09-23T20:43:46.519248+0000","last_unstale":"2023-09-23T20:44:41.328109+0000","last_undegraded":"2023-09-23T20:44:41.328109+0000","last_fullsized":"2023-09-23T20:44:41.328109+0000","mapping_epoch":487,"log_start":"466'5034","ondisk_log_start":"466'5034","created":455,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":7568,"log_dups_size":0,"ondisk_log_size":7568,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:49:43.541062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":104202240,"num_objects":1590,"num_object_clones":0,"num_object_copies":4770,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1590,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12602,"num_write_kb":454144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":557,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-23T20:44:30.507009+0000","last_change":"2023-09-23T20:44:15.656984+0000","last_active":"2023-09-23T20:44:30.507009+0000","last_peered":"2023-09-23T20:44:30.507009+0000","last_clean":"2023-09-23T20:44:30.507009+0000","last_became_active":"2023-09-23T20:43:46.512210+0000","last_became_peered":"2023-09-23T20:43:46.512210+0000","last_unstale":"2023-09-23T20:44:30.507009+0000","last_undegraded":"2023-09-23T20:44:30.507009+0000","last_fullsized":"2023-09-23T20:44:30.507009+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:15.656880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:15.656880+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:37:56.453466+0000","scrub_duration":0.0016711180000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"86'106","reported_seq":763,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-23T20:44:38.625493+0000","last_change":"2023-09-23T20:43:16.759138+0000","last_active":"2023-09-23T20:44:38.625493+0000","last_peered":"2023-09-23T20:44:38.625493+0000","last_clean":"2023-09-23T20:44:38.625493+0000","last_became_active":"2023-09-23T20:43:16.758015+0000","last_became_peered":"2023-09-23T20:43:16.758015+0000","last_unstale":"2023-09-23T20:44:38.625493+0000","last_undegraded":"2023-09-23T20:44:38.625493+0000","last_fullsized":"2023-09-23T20:44:38.625493+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":106,"log_dups_size":0,"ondisk_log_size":106,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:08:02.852472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00099302599999999993,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1094,"num_read_kb":1678,"num_write":100,"num_write_kb":404,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":233,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-09-23T20:44:44.899824+0000","last_change":"2023-09-23T20:38:02.552008+0000","last_active":"2023-09-23T20:44:44.899824+0000","last_peered":"2023-09-23T20:44:44.899824+0000","last_clean":"2023-09-23T20:44:44.899824+0000","last_became_active":"2023-09-23T20:38:02.551684+0000","last_became_peered":"2023-09-23T20:38:02.551684+0000","last_unstale":"2023-09-23T20:44:44.899824+0000","last_undegraded":"2023-09-23T20:44:44.899824+0000","last_fullsized":"2023-09-23T20:44:44.899824+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:40:37.147538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":476,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-23T20:44:38.617038+0000","last_change":"2023-09-23T20:29:45.533406+0000","last_active":"2023-09-23T20:44:38.617038+0000","last_peered":"2023-09-23T20:44:38.617038+0000","last_clean":"2023-09-23T20:44:38.617038+0000","last_became_active":"2023-09-23T20:29:45.526001+0000","last_became_peered":"2023-09-23T20:29:45.526001+0000","last_unstale":"2023-09-23T20:44:38.617038+0000","last_undegraded":"2023-09-23T20:44:38.617038+0000","last_fullsized":"2023-09-23T20:44:38.617038+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:43.787251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.a","version":"513'12671","reported_seq":12748,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-23T20:44:42.878222+0000","last_change":"2023-09-23T20:43:16.852909+0000","last_active":"2023-09-23T20:44:42.878222+0000","last_peered":"2023-09-23T20:44:42.878222+0000","last_clean":"2023-09-23T20:44:42.878222+0000","last_became_active":"2023-09-23T20:43:16.770100+0000","last_became_peered":"2023-09-23T20:43:16.770100+0000","last_unstale":"2023-09-23T20:44:42.878222+0000","last_undegraded":"2023-09-23T20:44:42.878222+0000","last_fullsized":"2023-09-23T20:44:42.878222+0000","mapping_epoch":481,"log_start":"470'7106","ondisk_log_start":"470'7106","created":455,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":5565,"log_dups_size":0,"ondisk_log_size":5565,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:13:16.136666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":100204544,"num_objects":1529,"num_object_clones":0,"num_object_copies":4587,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1529,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12671,"num_write_kb":454400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,7],"acting":[1,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":475,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-23T20:44:30.504502+0000","last_change":"2023-09-23T20:44:26.978243+0000","last_active":"2023-09-23T20:44:30.504502+0000","last_peered":"2023-09-23T20:44:30.504502+0000","last_clean":"2023-09-23T20:44:30.504502+0000","last_became_active":"2023-09-23T20:44:18.638989+0000","last_became_peered":"2023-09-23T20:44:18.638989+0000","last_unstale":"2023-09-23T20:44:30.504502+0000","last_undegraded":"2023-09-23T20:44:30.504502+0000","last_fullsized":"2023-09-23T20:44:30.504502+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:26.978179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:26.978179+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:25:38.977011+0000","scrub_duration":0.0013962829999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"86'130","reported_seq":831,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-23T20:44:38.622499+0000","last_change":"2023-09-23T20:25:39.637458+0000","last_active":"2023-09-23T20:44:38.622499+0000","last_peered":"2023-09-23T20:44:38.622499+0000","last_clean":"2023-09-23T20:44:38.622499+0000","last_became_active":"2023-09-23T20:25:39.636773+0000","last_became_peered":"2023-09-23T20:25:39.636773+0000","last_unstale":"2023-09-23T20:44:38.622499+0000","last_undegraded":"2023-09-23T20:44:38.622499+0000","last_fullsized":"2023-09-23T20:44:38.622499+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:00:32.810564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011300049999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":512,"num_read_kb":1820,"num_write":133,"num_write_kb":1347,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":484,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-23T20:44:38.613528+0000","last_change":"2023-09-23T20:38:02.591335+0000","last_active":"2023-09-23T20:44:38.613528+0000","last_peered":"2023-09-23T20:44:38.613528+0000","last_clean":"2023-09-23T20:44:38.613528+0000","last_became_active":"2023-09-23T20:38:02.591015+0000","last_became_peered":"2023-09-23T20:38:02.591015+0000","last_unstale":"2023-09-23T20:44:38.613528+0000","last_undegraded":"2023-09-23T20:44:38.613528+0000","last_fullsized":"2023-09-23T20:44:38.613528+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:55:32.272195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":27,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-23T20:44:38.613697+0000","last_change":"2023-09-23T20:44:10.555583+0000","last_active":"2023-09-23T20:44:38.613697+0000","last_peered":"2023-09-23T20:44:38.613697+0000","last_clean":"2023-09-23T20:44:38.613697+0000","last_became_active":"2023-09-23T20:44:10.555281+0000","last_became_peered":"2023-09-23T20:44:10.555281+0000","last_unstale":"2023-09-23T20:44:38.613697+0000","last_undegraded":"2023-09-23T20:44:38.613697+0000","last_fullsized":"2023-09-23T20:44:38.613697+0000","mapping_epoch":496,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":497,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:43:11.942147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.8","version":"512'12901","reported_seq":29003,"reported_epoch":514,"state":"remapped+peering","last_fresh":"2023-09-23T20:44:43.632710+0000","last_change":"2023-09-23T20:44:43.632710+0000","last_active":"2023-09-23T20:44:42.658971+0000","last_peered":"2023-09-23T20:44:42.651025+0000","last_clean":"2023-09-23T20:44:42.651025+0000","last_became_active":"2023-09-23T20:44:27.550683+0000","last_became_peered":"2023-09-23T20:44:27.550683+0000","last_unstale":"2023-09-23T20:44:43.632710+0000","last_undegraded":"2023-09-23T20:44:43.632710+0000","last_fullsized":"2023-09-23T20:44:43.632710+0000","mapping_epoch":514,"log_start":"469'6927","ondisk_log_start":"469'6927","created":455,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":5974,"log_dups_size":0,"ondisk_log_size":5974,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:49:59.355532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":104792064,"num_objects":1599,"num_object_clones":0,"num_object_copies":4797,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1599,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12901,"num_write_kb":464000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5435,"num_bytes_recovered":356188160,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[7],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"86'194","reported_seq":869,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-23T20:44:38.622937+0000","last_change":"2023-09-23T20:43:16.765865+0000","last_active":"2023-09-23T20:44:38.622937+0000","last_peered":"2023-09-23T20:44:38.622937+0000","last_clean":"2023-09-23T20:44:38.622937+0000","last_became_active":"2023-09-23T20:43:16.765289+0000","last_became_peered":"2023-09-23T20:43:16.765289+0000","last_unstale":"2023-09-23T20:44:38.622937+0000","last_undegraded":"2023-09-23T20:44:38.622937+0000","last_fullsized":"2023-09-23T20:44:38.622937+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":194,"log_dups_size":0,"ondisk_log_size":194,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:15:16.461900+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00037995999999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1024,"num_read_kb":1200,"num_write":173,"num_write_kb":1276,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":974,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-23T20:44:30.503360+0000","last_change":"2023-09-23T20:38:05.602508+0000","last_active":"2023-09-23T20:44:30.503360+0000","last_peered":"2023-09-23T20:44:30.503360+0000","last_clean":"2023-09-23T20:44:30.503360+0000","last_became_active":"2023-09-23T20:38:05.602129+0000","last_became_peered":"2023-09-23T20:38:05.602129+0000","last_unstale":"2023-09-23T20:44:30.503360+0000","last_undegraded":"2023-09-23T20:44:30.503360+0000","last_fullsized":"2023-09-23T20:44:30.503360+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:45.343742+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:45.343742+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:48.910343+0000","scrub_duration":0.001475802,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":88,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-23T20:44:38.602674+0000","last_change":"2023-09-23T20:43:16.758213+0000","last_active":"2023-09-23T20:44:38.602674+0000","last_peered":"2023-09-23T20:44:38.602674+0000","last_clean":"2023-09-23T20:44:38.602674+0000","last_became_active":"2023-09-23T20:43:16.757869+0000","last_became_peered":"2023-09-23T20:43:16.757869+0000","last_unstale":"2023-09-23T20:44:38.602674+0000","last_undegraded":"2023-09-23T20:44:38.602674+0000","last_fullsized":"2023-09-23T20:44:38.602674+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:43:10.811749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":118,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-23T20:44:38.602200+0000","last_change":"2023-09-23T20:38:02.573200+0000","last_active":"2023-09-23T20:44:38.602200+0000","last_peered":"2023-09-23T20:44:38.602200+0000","last_clean":"2023-09-23T20:44:38.602200+0000","last_became_active":"2023-09-23T20:38:02.572945+0000","last_became_peered":"2023-09-23T20:38:02.572945+0000","last_unstale":"2023-09-23T20:44:38.602200+0000","last_undegraded":"2023-09-23T20:44:38.602200+0000","last_fullsized":"2023-09-23T20:44:38.602200+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:57.246533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.b","version":"512'12402","reported_seq":12485,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-23T20:44:41.348050+0000","last_change":"2023-09-23T20:43:46.540991+0000","last_active":"2023-09-23T20:44:41.348050+0000","last_peered":"2023-09-23T20:44:41.348050+0000","last_clean":"2023-09-23T20:44:41.348050+0000","last_became_active":"2023-09-23T20:43:46.512772+0000","last_became_peered":"2023-09-23T20:43:46.512772+0000","last_unstale":"2023-09-23T20:44:41.348050+0000","last_undegraded":"2023-09-23T20:44:41.348050+0000","last_fullsized":"2023-09-23T20:44:41.348050+0000","mapping_epoch":487,"log_start":"466'4834","ondisk_log_start":"466'4834","created":455,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":7568,"log_dups_size":0,"ondisk_log_size":7568,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:46:02.578833+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":110231552,"num_objects":1682,"num_object_clones":0,"num_object_copies":5046,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1682,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12402,"num_write_kb":450688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":845,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-23T20:44:38.613860+0000","last_change":"2023-09-23T20:38:05.615711+0000","last_active":"2023-09-23T20:44:38.613860+0000","last_peered":"2023-09-23T20:44:38.613860+0000","last_clean":"2023-09-23T20:44:38.613860+0000","last_became_active":"2023-09-23T20:38:05.615390+0000","last_became_peered":"2023-09-23T20:38:05.615390+0000","last_unstale":"2023-09-23T20:44:38.613860+0000","last_undegraded":"2023-09-23T20:44:38.613860+0000","last_fullsized":"2023-09-23T20:44:38.613860+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:43.582146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:43.582146+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:45:11.917871+0000","scrub_duration":0.0016219730000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"86'80","reported_seq":764,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-23T20:44:30.503219+0000","last_change":"2023-09-23T20:29:45.536796+0000","last_active":"2023-09-23T20:44:30.503219+0000","last_peered":"2023-09-23T20:44:30.503219+0000","last_clean":"2023-09-23T20:44:30.503219+0000","last_became_active":"2023-09-23T20:29:45.536439+0000","last_became_peered":"2023-09-23T20:29:45.536439+0000","last_unstale":"2023-09-23T20:44:30.503219+0000","last_undegraded":"2023-09-23T20:44:30.503219+0000","last_fullsized":"2023-09-23T20:44:30.503219+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":80,"log_dups_size":0,"ondisk_log_size":80,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:30:57.793132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035115,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":395,"num_read_kb":1370,"num_write":66,"num_write_kb":1061,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":504,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-23T20:44:30.506843+0000","last_change":"2023-09-23T20:43:46.544563+0000","last_active":"2023-09-23T20:44:30.506843+0000","last_peered":"2023-09-23T20:44:30.506843+0000","last_clean":"2023-09-23T20:44:30.506843+0000","last_became_active":"2023-09-23T20:43:46.543958+0000","last_became_peered":"2023-09-23T20:43:46.543958+0000","last_unstale":"2023-09-23T20:44:30.506843+0000","last_undegraded":"2023-09-23T20:44:30.506843+0000","last_fullsized":"2023-09-23T20:44:30.506843+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:46:36.921752+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":499,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-23T20:44:30.507670+0000","last_change":"2023-09-23T20:43:46.520386+0000","last_active":"2023-09-23T20:44:30.507670+0000","last_peered":"2023-09-23T20:44:30.507670+0000","last_clean":"2023-09-23T20:44:30.507670+0000","last_became_active":"2023-09-23T20:43:46.519757+0000","last_became_peered":"2023-09-23T20:43:46.519757+0000","last_unstale":"2023-09-23T20:44:30.507670+0000","last_undegraded":"2023-09-23T20:44:30.507670+0000","last_fullsized":"2023-09-23T20:44:30.507670+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:23:14.662734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.e","version":"512'12669","reported_seq":12733,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-23T20:44:42.525479+0000","last_change":"2023-09-23T20:41:04.574876+0000","last_active":"2023-09-23T20:44:42.525479+0000","last_peered":"2023-09-23T20:44:42.525479+0000","last_clean":"2023-09-23T20:44:42.525479+0000","last_became_active":"2023-09-23T20:41:04.574595+0000","last_became_peered":"2023-09-23T20:41:04.574595+0000","last_unstale":"2023-09-23T20:44:42.525479+0000","last_undegraded":"2023-09-23T20:44:42.525479+0000","last_fullsized":"2023-09-23T20:44:42.525479+0000","mapping_epoch":455,"log_start":"469'7008","ondisk_log_start":"469'7008","created":455,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":5661,"log_dups_size":0,"ondisk_log_size":5661,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:31:00.147966+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":101646336,"num_objects":1551,"num_object_clones":0,"num_object_copies":4653,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1551,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12669,"num_write_kb":455040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":624,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-23T20:44:38.601984+0000","last_change":"2023-09-23T20:41:08.380084+0000","last_active":"2023-09-23T20:44:38.601984+0000","last_peered":"2023-09-23T20:44:38.601984+0000","last_clean":"2023-09-23T20:44:38.601984+0000","last_became_active":"2023-09-23T20:38:05.610881+0000","last_became_peered":"2023-09-23T20:38:05.610881+0000","last_unstale":"2023-09-23T20:44:38.601984+0000","last_undegraded":"2023-09-23T20:44:38.601984+0000","last_fullsized":"2023-09-23T20:44:38.601984+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:41:08.379959+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:41:08.379959+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:19:39.547923+0000","scrub_duration":0.0019158420000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"86'102","reported_seq":1106,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-23T20:44:30.505203+0000","last_change":"2023-09-23T20:32:36.314672+0000","last_active":"2023-09-23T20:44:30.505203+0000","last_peered":"2023-09-23T20:44:30.505203+0000","last_clean":"2023-09-23T20:44:30.505203+0000","last_became_active":"2023-09-23T20:32:36.314231+0000","last_became_peered":"2023-09-23T20:32:36.314231+0000","last_unstale":"2023-09-23T20:44:30.505203+0000","last_undegraded":"2023-09-23T20:44:30.505203+0000","last_fullsized":"2023-09-23T20:44:30.505203+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":102,"log_dups_size":0,"ondisk_log_size":102,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:11:40.514709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030656500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":963,"num_read_kb":1234,"num_write":60,"num_write_kb":225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":460,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-23T20:44:38.617197+0000","last_change":"2023-09-23T20:40:19.544384+0000","last_active":"2023-09-23T20:44:38.617197+0000","last_peered":"2023-09-23T20:44:38.617197+0000","last_clean":"2023-09-23T20:44:38.617197+0000","last_became_active":"2023-09-23T20:40:19.544079+0000","last_became_peered":"2023-09-23T20:40:19.544079+0000","last_unstale":"2023-09-23T20:44:38.617197+0000","last_undegraded":"2023-09-23T20:44:38.617197+0000","last_fullsized":"2023-09-23T20:44:38.617197+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:00:19.498631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":454,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-23T20:44:30.510726+0000","last_change":"2023-09-23T20:43:46.541173+0000","last_active":"2023-09-23T20:44:30.510726+0000","last_peered":"2023-09-23T20:44:30.510726+0000","last_clean":"2023-09-23T20:44:30.510726+0000","last_became_active":"2023-09-23T20:43:46.540726+0000","last_became_peered":"2023-09-23T20:43:46.540726+0000","last_unstale":"2023-09-23T20:44:30.510726+0000","last_undegraded":"2023-09-23T20:44:30.510726+0000","last_fullsized":"2023-09-23T20:44:30.510726+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:49:04.137885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"512'12671","reported_seq":26432,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-23T20:44:42.501105+0000","last_change":"2023-09-23T20:44:18.688723+0000","last_active":"2023-09-23T20:44:42.501105+0000","last_peered":"2023-09-23T20:44:42.501105+0000","last_clean":"2023-09-23T20:44:42.501105+0000","last_became_active":"2023-09-23T20:44:18.636162+0000","last_became_peered":"2023-09-23T20:44:18.636162+0000","last_unstale":"2023-09-23T20:44:42.501105+0000","last_undegraded":"2023-09-23T20:44:42.501105+0000","last_fullsized":"2023-09-23T20:44:42.501105+0000","mapping_epoch":502,"log_start":"469'6948","ondisk_log_start":"469'6948","created":455,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":5723,"log_dups_size":0,"ondisk_log_size":5723,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:49:45.956725+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":105775128,"num_objects":1615,"num_object_clones":0,"num_object_copies":4845,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1615,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":12671,"num_write_kb":457089,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6816,"num_bytes_recovered":446627864,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":794,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-09-23T20:44:44.899853+0000","last_change":"2023-09-23T20:38:05.590361+0000","last_active":"2023-09-23T20:44:44.899853+0000","last_peered":"2023-09-23T20:44:44.899853+0000","last_clean":"2023-09-23T20:44:44.899853+0000","last_became_active":"2023-09-23T20:38:05.589985+0000","last_became_peered":"2023-09-23T20:38:05.589985+0000","last_unstale":"2023-09-23T20:44:44.899853+0000","last_undegraded":"2023-09-23T20:44:44.899853+0000","last_fullsized":"2023-09-23T20:44:44.899853+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:03.246271+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:03.246271+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:38:54.351576+0000","scrub_duration":0.0019016700000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":249,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-23T20:44:30.505048+0000","last_change":"2023-09-23T20:32:50.549695+0000","last_active":"2023-09-23T20:44:30.505048+0000","last_peered":"2023-09-23T20:44:30.505048+0000","last_clean":"2023-09-23T20:44:30.505048+0000","last_became_active":"2023-09-23T20:32:50.549431+0000","last_became_peered":"2023-09-23T20:32:50.549431+0000","last_unstale":"2023-09-23T20:44:30.505048+0000","last_undegraded":"2023-09-23T20:44:30.505048+0000","last_fullsized":"2023-09-23T20:44:30.505048+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:45:53.666569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"86'128","reported_seq":925,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-23T20:44:38.614144+0000","last_change":"2023-09-23T20:38:03.541765+0000","last_active":"2023-09-23T20:44:38.614144+0000","last_peered":"2023-09-23T20:44:38.614144+0000","last_clean":"2023-09-23T20:44:38.614144+0000","last_became_active":"2023-09-23T20:38:03.541277+0000","last_became_peered":"2023-09-23T20:38:03.541277+0000","last_unstale":"2023-09-23T20:44:38.614144+0000","last_undegraded":"2023-09-23T20:44:38.614144+0000","last_fullsized":"2023-09-23T20:44:38.614144+0000","mapping_epoch":404,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":405,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:20:53.834554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000293202,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":579,"num_read_kb":1473,"num_write":96,"num_write_kb":1062,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":294,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-23T20:44:42.659697+0000","last_change":"2023-09-23T20:37:43.582105+0000","last_active":"2023-09-23T20:44:42.659697+0000","last_peered":"2023-09-23T20:44:42.659697+0000","last_clean":"2023-09-23T20:44:42.659697+0000","last_became_active":"2023-09-23T20:37:43.581860+0000","last_became_peered":"2023-09-23T20:37:43.581860+0000","last_unstale":"2023-09-23T20:44:42.659697+0000","last_undegraded":"2023-09-23T20:44:42.659697+0000","last_fullsized":"2023-09-23T20:44:42.659697+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:18:38.765486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.c","version":"513'12717","reported_seq":12786,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-23T20:44:42.893383+0000","last_change":"2023-09-23T20:41:04.575496+0000","last_active":"2023-09-23T20:44:42.893383+0000","last_peered":"2023-09-23T20:44:42.893383+0000","last_clean":"2023-09-23T20:44:42.893383+0000","last_became_active":"2023-09-23T20:41:04.575242+0000","last_became_peered":"2023-09-23T20:41:04.575242+0000","last_unstale":"2023-09-23T20:44:42.893383+0000","last_undegraded":"2023-09-23T20:44:42.893383+0000","last_fullsized":"2023-09-23T20:44:42.893383+0000","mapping_epoch":455,"log_start":"469'6775","ondisk_log_start":"469'6775","created":455,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":5942,"log_dups_size":0,"ondisk_log_size":5942,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:30:14.611529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":100597760,"num_objects":1535,"num_object_clones":0,"num_object_copies":4605,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1535,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12717,"num_write_kb":456064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"90'161","reported_seq":728,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-23T20:44:42.657419+0000","last_change":"2023-09-23T20:40:22.427435+0000","last_active":"2023-09-23T20:44:42.657419+0000","last_peered":"2023-09-23T20:44:42.657419+0000","last_clean":"2023-09-23T20:44:42.657419+0000","last_became_active":"2023-09-23T20:40:22.427051+0000","last_became_peered":"2023-09-23T20:40:22.427051+0000","last_unstale":"2023-09-23T20:44:42.657419+0000","last_undegraded":"2023-09-23T20:44:42.657419+0000","last_fullsized":"2023-09-23T20:44:42.657419+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:34:54.504554+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:34:54.504554+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:42.001652+0000","scrub_duration":0.011176864,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"86'117","reported_seq":821,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-23T20:44:38.603047+0000","last_change":"2023-09-23T20:44:18.635224+0000","last_active":"2023-09-23T20:44:38.603047+0000","last_peered":"2023-09-23T20:44:38.603047+0000","last_clean":"2023-09-23T20:44:38.603047+0000","last_became_active":"2023-09-23T20:44:18.634703+0000","last_became_peered":"2023-09-23T20:44:18.634703+0000","last_unstale":"2023-09-23T20:44:38.603047+0000","last_undegraded":"2023-09-23T20:44:38.603047+0000","last_fullsized":"2023-09-23T20:44:38.603047+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:59:38.030626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046891400000000002,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":893,"num_read_kb":922,"num_write":82,"num_write_kb":4193,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":16777216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":499,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-23T20:44:30.507754+0000","last_change":"2023-09-23T20:43:46.521505+0000","last_active":"2023-09-23T20:44:30.507754+0000","last_peered":"2023-09-23T20:44:30.507754+0000","last_clean":"2023-09-23T20:44:30.507754+0000","last_became_active":"2023-09-23T20:43:46.520727+0000","last_became_peered":"2023-09-23T20:43:46.520727+0000","last_unstale":"2023-09-23T20:44:30.507754+0000","last_undegraded":"2023-09-23T20:44:30.507754+0000","last_fullsized":"2023-09-23T20:44:30.507754+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:17:06.164741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":253,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-09-23T20:44:44.899381+0000","last_change":"2023-09-23T20:40:19.530602+0000","last_active":"2023-09-23T20:44:44.899381+0000","last_peered":"2023-09-23T20:44:44.899381+0000","last_clean":"2023-09-23T20:44:44.899381+0000","last_became_active":"2023-09-23T20:40:19.530256+0000","last_became_peered":"2023-09-23T20:40:19.530256+0000","last_unstale":"2023-09-23T20:44:44.899381+0000","last_undegraded":"2023-09-23T20:44:44.899381+0000","last_fullsized":"2023-09-23T20:44:44.899381+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:18:45.658867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"90'161","reported_seq":768,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-23T20:44:30.506556+0000","last_change":"2023-09-23T20:43:46.544750+0000","last_active":"2023-09-23T20:44:30.506556+0000","last_peered":"2023-09-23T20:44:30.506556+0000","last_clean":"2023-09-23T20:44:30.506556+0000","last_became_active":"2023-09-23T20:43:46.544395+0000","last_became_peered":"2023-09-23T20:43:46.544395+0000","last_unstale":"2023-09-23T20:44:30.506556+0000","last_undegraded":"2023-09-23T20:44:30.506556+0000","last_fullsized":"2023-09-23T20:44:30.506556+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:06.487351+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:06.487351+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:56.866168+0000","scrub_duration":0.001964379,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":466,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-23T20:44:30.510656+0000","last_change":"2023-09-23T20:43:46.541379+0000","last_active":"2023-09-23T20:44:30.510656+0000","last_peered":"2023-09-23T20:44:30.510656+0000","last_clean":"2023-09-23T20:44:30.510656+0000","last_became_active":"2023-09-23T20:43:46.540732+0000","last_became_peered":"2023-09-23T20:43:46.540732+0000","last_unstale":"2023-09-23T20:44:30.510656+0000","last_undegraded":"2023-09-23T20:44:30.510656+0000","last_fullsized":"2023-09-23T20:44:30.510656+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:31:40.605772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":369,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-23T20:44:38.623216+0000","last_change":"2023-09-23T20:29:45.538314+0000","last_active":"2023-09-23T20:44:38.623216+0000","last_peered":"2023-09-23T20:44:38.623216+0000","last_clean":"2023-09-23T20:44:38.623216+0000","last_became_active":"2023-09-23T20:29:45.536847+0000","last_became_peered":"2023-09-23T20:29:45.536847+0000","last_unstale":"2023-09-23T20:44:38.623216+0000","last_undegraded":"2023-09-23T20:44:38.623216+0000","last_fullsized":"2023-09-23T20:44:38.623216+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:51.948721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"90'161","reported_seq":587,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-23T20:44:42.657462+0000","last_change":"2023-09-23T20:41:06.499511+0000","last_active":"2023-09-23T20:44:42.657462+0000","last_peered":"2023-09-23T20:44:42.657462+0000","last_clean":"2023-09-23T20:44:42.657462+0000","last_became_active":"2023-09-23T20:38:05.601874+0000","last_became_peered":"2023-09-23T20:38:05.601874+0000","last_unstale":"2023-09-23T20:44:42.657462+0000","last_undegraded":"2023-09-23T20:44:42.657462+0000","last_fullsized":"2023-09-23T20:44:42.657462+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:41:06.499396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:41:06.499396+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:31:34.059077+0000","scrub_duration":0.0019670310000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":408,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-09-23T20:44:44.899182+0000","last_change":"2023-09-23T20:43:16.758761+0000","last_active":"2023-09-23T20:44:44.899182+0000","last_peered":"2023-09-23T20:44:44.899182+0000","last_clean":"2023-09-23T20:44:44.899182+0000","last_became_active":"2023-09-23T20:43:16.758407+0000","last_became_peered":"2023-09-23T20:43:16.758407+0000","last_unstale":"2023-09-23T20:44:44.899182+0000","last_undegraded":"2023-09-23T20:44:44.899182+0000","last_fullsized":"2023-09-23T20:44:44.899182+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:59:27.070685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":172,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-09-23T20:44:44.899415+0000","last_change":"2023-09-23T20:34:59.987575+0000","last_active":"2023-09-23T20:44:44.899415+0000","last_peered":"2023-09-23T20:44:44.899415+0000","last_clean":"2023-09-23T20:44:44.899415+0000","last_became_active":"2023-09-23T20:34:59.987229+0000","last_became_peered":"2023-09-23T20:34:59.987229+0000","last_unstale":"2023-09-23T20:44:44.899415+0000","last_undegraded":"2023-09-23T20:44:44.899415+0000","last_fullsized":"2023-09-23T20:44:44.899415+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:34:01.820159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":416,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-09-23T20:44:44.899650+0000","last_change":"2023-09-23T20:43:16.759691+0000","last_active":"2023-09-23T20:44:44.899650+0000","last_peered":"2023-09-23T20:44:44.899650+0000","last_clean":"2023-09-23T20:44:44.899650+0000","last_became_active":"2023-09-23T20:43:16.759309+0000","last_became_peered":"2023-09-23T20:43:16.759309+0000","last_unstale":"2023-09-23T20:44:44.899650+0000","last_undegraded":"2023-09-23T20:44:44.899650+0000","last_fullsized":"2023-09-23T20:44:44.899650+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:58:52.893019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":172,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-09-23T20:44:44.899377+0000","last_change":"2023-09-23T20:34:59.987416+0000","last_active":"2023-09-23T20:44:44.899377+0000","last_peered":"2023-09-23T20:44:44.899377+0000","last_clean":"2023-09-23T20:44:44.899377+0000","last_became_active":"2023-09-23T20:34:59.987117+0000","last_became_peered":"2023-09-23T20:34:59.987117+0000","last_unstale":"2023-09-23T20:44:44.899377+0000","last_undegraded":"2023-09-23T20:44:44.899377+0000","last_fullsized":"2023-09-23T20:44:44.899377+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:36:16.318623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"90'161","reported_seq":767,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-23T20:44:30.504971+0000","last_change":"2023-09-23T20:38:05.591992+0000","last_active":"2023-09-23T20:44:30.504971+0000","last_peered":"2023-09-23T20:44:30.504971+0000","last_clean":"2023-09-23T20:44:30.504971+0000","last_became_active":"2023-09-23T20:38:05.591701+0000","last_became_peered":"2023-09-23T20:38:05.591701+0000","last_unstale":"2023-09-23T20:44:30.504971+0000","last_undegraded":"2023-09-23T20:44:30.504971+0000","last_fullsized":"2023-09-23T20:44:30.504971+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:02.384133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:02.384133+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:41.722542+0000","scrub_duration":0.001769098,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":336,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-23T20:44:42.657681+0000","last_change":"2023-09-23T20:38:02.542719+0000","last_active":"2023-09-23T20:44:42.657681+0000","last_peered":"2023-09-23T20:44:42.657681+0000","last_clean":"2023-09-23T20:44:42.657681+0000","last_became_active":"2023-09-23T20:38:02.542385+0000","last_became_peered":"2023-09-23T20:38:02.542385+0000","last_unstale":"2023-09-23T20:44:42.657681+0000","last_undegraded":"2023-09-23T20:44:42.657681+0000","last_fullsized":"2023-09-23T20:44:42.657681+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:36:23.673757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":302,"reported_epoch":513,"state":"active+clean","last_fresh":"2023-09-23T20:44:42.656937+0000","last_change":"2023-09-23T20:37:43.584253+0000","last_active":"2023-09-23T20:44:42.656937+0000","last_peered":"2023-09-23T20:44:42.656937+0000","last_clean":"2023-09-23T20:44:42.656937+0000","last_became_active":"2023-09-23T20:37:43.580069+0000","last_became_peered":"2023-09-23T20:37:43.580069+0000","last_unstale":"2023-09-23T20:44:42.656937+0000","last_undegraded":"2023-09-23T20:44:42.656937+0000","last_fullsized":"2023-09-23T20:44:42.656937+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:39:11.482581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"90'161","reported_seq":606,"reported_epoch":512,"state":"active+clean","last_fresh":"2023-09-23T20:44:38.621447+0000","last_change":"2023-09-23T20:36:01.505098+0000","last_active":"2023-09-23T20:44:38.621447+0000","last_peered":"2023-09-23T20:44:38.621447+0000","last_clean":"2023-09-23T20:44:38.621447+0000","last_became_active":"2023-09-23T20:35:26.644707+0000","last_became_peered":"2023-09-23T20:35:26.644707+0000","last_unstale":"2023-09-23T20:44:38.621447+0000","last_undegraded":"2023-09-23T20:44:38.621447+0000","last_fullsized":"2023-09-23T20:44:38.621447+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:01.504973+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:01.504973+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:02:10.880809+0000","scrub_duration":0.00202344,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":544,"reported_epoch":511,"state":"active+clean","last_fresh":"2023-09-23T20:44:30.508350+0000","last_change":"2023-09-23T20:43:46.511657+0000","last_active":"2023-09-23T20:44:30.508350+0000","last_peered":"2023-09-23T20:44:30.508350+0000","last_clean":"2023-09-23T20:44:30.508350+0000","last_became_active":"2023-09-23T20:43:46.511326+0000","last_became_peered":"2023-09-23T20:43:46.511326+0000","last_unstale":"2023-09-23T20:44:30.508350+0000","last_undegraded":"2023-09-23T20:44:30.508350+0000","last_fullsized":"2023-09-23T20:44:30.508350+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:36:46.139261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3324,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":233,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-09-23T20:44:44.899208+0000","last_change":"2023-09-23T20:38:02.538333+0000","last_active":"2023-09-23T20:44:44.899208+0000","last_peered":"2023-09-23T20:44:44.899208+0000","last_clean":"2023-09-23T20:44:44.899208+0000","last_became_active":"2023-09-23T20:38:02.538008+0000","last_became_peered":"2023-09-23T20:38:02.538008+0000","last_unstale":"2023-09-23T20:44:44.899208+0000","last_undegraded":"2023-09-23T20:44:44.899208+0000","last_fullsized":"2023-09-23T20:44:44.899208+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:02:38.789530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":86,"num_bytes_recovered":18486,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26691,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":1638268952,"num_objects":24999,"num_object_clones":0,"num_object_copies":74997,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24999,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":202665,"num_write_kb":7285185,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16458,"num_bytes_recovered":1078198296,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":5400444928,"data_stored":5400428640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":102260,"ondisk_log_size":102260,"up":48,"acting":48,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5983,"num_read_kb":13200,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":84,"num_bytes_recovered":16777216,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6082,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201679,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1526956,"kb_used_data":1031908,"kb_used_omap":0,"kb_used_meta":494912,"kb_avail":92218196,"statfs":{"total":95995035648,"available":94431432704,"internally_reserved":0,"allocated":1056673792,"data_stored":1055158628,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":506789888},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:43:58 2023","interfaces":[{"interface":"back","average":{"1min":2.5600000000000001,"5min":1.643,"15min":2.089},"min":{"1min":0.23899999999999999,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":15.1,"5min":15.1,"15min":18.315000000000001},"last":3.125},{"interface":"front","average":{"1min":2.7040000000000002,"5min":1.587,"15min":1.8069999999999999},"min":{"1min":0.33300000000000002,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":15.327999999999999,"5min":15.327999999999999,"15min":19.355},"last":0.32900000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:44:08 2023","interfaces":[{"interface":"back","average":{"1min":2.714,"5min":1.452,"15min":2.105},"min":{"1min":0.26900000000000002,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":15.131,"5min":15.131,"15min":49.965000000000003},"last":2.5609999999999999},{"interface":"front","average":{"1min":3.6850000000000001,"5min":2.266,"15min":2.339},"min":{"1min":0.36299999999999999,"5min":0.26700000000000002,"15min":0.248},"max":{"1min":15.18,"5min":15.18,"15min":53.451999999999998},"last":0.39700000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:43:58 2023","interfaces":[{"interface":"back","average":{"1min":3.1859999999999999,"5min":2.0600000000000001,"15min":2.1880000000000002},"min":{"1min":0.39800000000000002,"5min":0.27900000000000003,"15min":0.27600000000000002},"max":{"1min":15.087,"5min":15.087,"15min":16.965},"last":2.6459999999999999},{"interface":"front","average":{"1min":2.2320000000000002,"5min":1.716,"15min":2.4279999999999999},"min":{"1min":0.35899999999999999,"5min":0.30299999999999999,"15min":0.26600000000000001},"max":{"1min":15.193,"5min":15.193,"15min":23.494},"last":2.7130000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:44:28 2023","interfaces":[{"interface":"back","average":{"1min":2.6840000000000002,"5min":1.772,"15min":2.371},"min":{"1min":0.41399999999999998,"5min":0.32500000000000001,"15min":0.29199999999999998},"max":{"1min":15.115,"5min":15.115,"15min":23.352},"last":1.6859999999999999},{"interface":"front","average":{"1min":3.3570000000000002,"5min":2.1389999999999998,"15min":2.4489999999999998},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.33100000000000002},"max":{"1min":15.144,"5min":15.144,"15min":20.969999999999999},"last":1.706}]},{"osd":4,"last update":"Sat Sep 23 20:44:28 2023","interfaces":[{"interface":"back","average":{"1min":2.0110000000000001,"5min":1.391,"15min":1.2669999999999999},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.23200000000000001},"max":{"1min":12.260999999999999,"5min":12.260999999999999,"15min":18.364999999999998},"last":0.38200000000000001},{"interface":"front","average":{"1min":2.145,"5min":1.5640000000000001,"15min":1.3320000000000001},"min":{"1min":0.315,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":6.9939999999999998,"5min":6.9939999999999998,"15min":23.617000000000001},"last":0.61399999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:44:28 2023","interfaces":[{"interface":"back","average":{"1min":2.5270000000000001,"5min":1.6040000000000001,"15min":1.1970000000000001},"min":{"1min":0.36899999999999999,"5min":0.30099999999999999,"15min":0.26600000000000001},"max":{"1min":15.005000000000001,"5min":15.005000000000001,"15min":15.005000000000001},"last":0.42599999999999999},{"interface":"front","average":{"1min":1.9990000000000001,"5min":1.556,"15min":1.3120000000000001},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":12.212,"5min":12.731,"15min":18.831},"last":0.441}]},{"osd":6,"last update":"Sat Sep 23 20:44:28 2023","interfaces":[{"interface":"back","average":{"1min":3.6379999999999999,"5min":1.837,"15min":1.4359999999999999},"min":{"1min":0.34599999999999997,"5min":0.33700000000000002,"15min":0.30399999999999999},"max":{"1min":15.038,"5min":15.038,"15min":18.417000000000002},"last":0.40999999999999998},{"interface":"front","average":{"1min":2.8969999999999998,"5min":1.6659999999999999,"15min":1.274},"min":{"1min":0.47999999999999998,"5min":0.36499999999999999,"15min":0.318},"max":{"1min":15.369999999999999,"5min":15.369999999999999,"15min":15.369999999999999},"last":0.65100000000000002}]}]},{"osd":6,"up_from":47,"seq":201863463318,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":735772,"kb_used_data":316948,"kb_used_omap":0,"kb_used_meta":418816,"kb_avail":93009380,"statfs":{"total":95995035648,"available":95241605120,"internally_reserved":0,"allocated":324554752,"data_stored":323055277,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":428867584},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:43:54 2023","interfaces":[{"interface":"back","average":{"1min":2.2799999999999998,"5min":1.375,"15min":1.9790000000000001},"min":{"1min":0.24099999999999999,"5min":0.23699999999999999,"15min":0.22800000000000001},"max":{"1min":12.552,"5min":12.552,"15min":53.701999999999998},"last":7.7720000000000002},{"interface":"front","average":{"1min":2.149,"5min":1.5589999999999999,"15min":2.3490000000000002},"min":{"1min":0.31,"5min":0.23300000000000001,"15min":0.22700000000000001},"max":{"1min":9.2699999999999996,"5min":9.2699999999999996,"15min":60.328000000000003},"last":0.438}]},{"osd":1,"last update":"Sat Sep 23 20:44:17 2023","interfaces":[{"interface":"back","average":{"1min":3.3330000000000002,"5min":1.744,"15min":2.1309999999999998},"min":{"1min":0.317,"5min":0.249,"15min":0.249},"max":{"1min":9.2520000000000007,"5min":9.2520000000000007,"15min":64.274000000000001},"last":0.748},{"interface":"front","average":{"1min":2.8940000000000001,"5min":1.8100000000000001,"15min":2.0739999999999998},"min":{"1min":0.42799999999999999,"5min":0.32000000000000001,"15min":0.28000000000000003},"max":{"1min":17.786000000000001,"5min":17.786000000000001,"15min":34.490000000000002},"last":7.8849999999999998}]},{"osd":2,"last update":"Sat Sep 23 20:43:54 2023","interfaces":[{"interface":"back","average":{"1min":2.5659999999999998,"5min":1.9419999999999999,"15min":2.202},"min":{"1min":0.34999999999999998,"5min":0.27800000000000002,"15min":0.27700000000000002},"max":{"1min":8.0370000000000008,"5min":9.9049999999999994,"15min":34.317999999999998},"last":0.78100000000000003},{"interface":"front","average":{"1min":2.0019999999999998,"5min":1.766,"15min":2.1080000000000001},"min":{"1min":0.40699999999999997,"5min":0.311,"15min":0.30399999999999999},"max":{"1min":8.7309999999999999,"5min":9.9269999999999996,"15min":34.292999999999999},"last":0.38600000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:43:44 2023","interfaces":[{"interface":"back","average":{"1min":1.835,"5min":1.843,"15min":1.923},"min":{"1min":0.42599999999999999,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":5.1109999999999998,"5min":6.7690000000000001,"15min":8.9670000000000005},"last":0.499},{"interface":"front","average":{"1min":2.4380000000000002,"5min":1.917,"15min":2.1440000000000001},"min":{"1min":0.40300000000000002,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":5.843,"5min":11.808,"15min":12.875999999999999},"last":0.48499999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:44:17 2023","interfaces":[{"interface":"back","average":{"1min":1.27,"5min":0.997,"15min":0.85199999999999998},"min":{"1min":0.33600000000000002,"5min":0.25600000000000001,"15min":0.251},"max":{"1min":6.5800000000000001,"5min":6.5800000000000001,"15min":23.904},"last":0.41799999999999998},{"interface":"front","average":{"1min":1.4019999999999999,"5min":1.1519999999999999,"15min":1.1240000000000001},"min":{"1min":0.34999999999999998,"5min":0.309,"15min":0.27000000000000002},"max":{"1min":5.6260000000000003,"5min":6.7359999999999998,"15min":11.515000000000001},"last":0.46600000000000003}]},{"osd":5,"last update":"Sat Sep 23 20:44:17 2023","interfaces":[{"interface":"back","average":{"1min":1.381,"5min":0.97099999999999997,"15min":0.86499999999999999},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":6.5460000000000003,"5min":6.6340000000000003,"15min":8.4499999999999993},"last":7.8529999999999998},{"interface":"front","average":{"1min":1.5509999999999999,"5min":1.4870000000000001,"15min":1.1970000000000001},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":5.4969999999999999,"5min":7.1319999999999997,"15min":9.9299999999999997},"last":0.51300000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:43:44 2023","interfaces":[{"interface":"back","average":{"1min":1.657,"5min":0.96999999999999997,"15min":0.94299999999999995},"min":{"1min":0.46999999999999997,"5min":0.33200000000000002,"15min":0.28599999999999998},"max":{"1min":8.8350000000000009,"5min":8.8350000000000009,"15min":8.8350000000000009},"last":7.8250000000000002},{"interface":"front","average":{"1min":2.1619999999999999,"5min":1.371,"15min":1.242},"min":{"1min":0.38800000000000001,"5min":0.34699999999999998,"15min":0.30299999999999999},"max":{"1min":5.9180000000000001,"5min":7.1929999999999996,"15min":8.8119999999999994},"last":0.82099999999999995}]}]},{"osd":1,"up_from":266,"seq":1142461300917,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1765284,"kb_used_data":1150940,"kb_used_omap":8,"kb_used_meta":614327,"kb_avail":91979868,"statfs":{"total":95995035648,"available":94187384832,"internally_reserved":0,"allocated":1178562560,"data_stored":1175898698,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8206,"internal_metadata":629071858},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:44:15 2023","interfaces":[{"interface":"back","average":{"1min":3.6110000000000002,"5min":2.206,"15min":2.5329999999999999},"min":{"1min":0.214,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":15.532999999999999,"5min":16.292000000000002,"15min":52.689999999999998},"last":2.4329999999999998},{"interface":"front","average":{"1min":3.2290000000000001,"5min":2.2970000000000002,"15min":3.0179999999999998},"min":{"1min":0.22600000000000001,"5min":0.20100000000000001,"15min":0.19900000000000001},"max":{"1min":14.445,"5min":16.332999999999998,"15min":52.662999999999997},"last":2.5880000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:44:15 2023","interfaces":[{"interface":"back","average":{"1min":3.0390000000000001,"5min":2.4140000000000001,"15min":3.3370000000000002},"min":{"1min":0.28999999999999998,"5min":0.216,"15min":0.20899999999999999},"max":{"1min":8.6519999999999992,"5min":14.148,"15min":32.255000000000003},"last":2.3980000000000001},{"interface":"front","average":{"1min":2.9049999999999998,"5min":1.9810000000000001,"15min":2.2909999999999999},"min":{"1min":0.26000000000000001,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":10.041,"5min":10.041,"15min":13.404999999999999},"last":2.097}]},{"osd":3,"last update":"Sat Sep 23 20:44:15 2023","interfaces":[{"interface":"back","average":{"1min":2.5830000000000002,"5min":2.089,"15min":3.097},"min":{"1min":0.36899999999999999,"5min":0.247,"15min":0.247},"max":{"1min":7.2469999999999999,"5min":16.245999999999999,"15min":80.745999999999995},"last":2.5699999999999998},{"interface":"front","average":{"1min":3.5779999999999998,"5min":2.645,"15min":3.3450000000000002},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.22700000000000001},"max":{"1min":11.523,"5min":13.497999999999999,"15min":82.988},"last":2.798}]},{"osd":4,"last update":"Sat Sep 23 20:44:15 2023","interfaces":[{"interface":"back","average":{"1min":2.3250000000000002,"5min":2.202,"15min":2.25},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":6.5970000000000004,"5min":20.634,"15min":24.327999999999999},"last":2.7320000000000002},{"interface":"front","average":{"1min":2.0190000000000001,"5min":1.982,"15min":1.748},"min":{"1min":0.34699999999999998,"5min":0.32600000000000001,"15min":0.311},"max":{"1min":6.0869999999999997,"5min":20.835999999999999,"15min":20.835999999999999},"last":1.9139999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:44:15 2023","interfaces":[{"interface":"back","average":{"1min":3.2040000000000002,"5min":2.4319999999999999,"15min":2.3029999999999999},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":14.366,"5min":14.366,"15min":33.401000000000003},"last":0.40000000000000002},{"interface":"front","average":{"1min":2.7469999999999999,"5min":2.3519999999999999,"15min":2.4820000000000002},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.32500000000000001},"max":{"1min":7.3300000000000001,"5min":10.958,"15min":33.417000000000002},"last":2.629}]},{"osd":6,"last update":"Sat Sep 23 20:44:15 2023","interfaces":[{"interface":"back","average":{"1min":3.0030000000000001,"5min":2.3639999999999999,"15min":2.839},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35699999999999998},"max":{"1min":14.414999999999999,"5min":14.414999999999999,"15min":72.682000000000002},"last":0.46899999999999997},{"interface":"front","average":{"1min":3.2829999999999999,"5min":2.3620000000000001,"15min":2.9860000000000002},"min":{"1min":0.38500000000000001,"5min":0.317,"15min":0.317},"max":{"1min":9.1470000000000002,"5min":9.4090000000000007,"15min":72.563999999999993},"last":2.8290000000000002}]},{"osd":7,"last update":"Sat Sep 23 20:44:15 2023","interfaces":[{"interface":"back","average":{"1min":2.98,"5min":2.4470000000000001,"15min":2.3700000000000001},"min":{"1min":0.43099999999999999,"5min":0.37,"15min":0.37},"max":{"1min":9.2409999999999997,"5min":16.359000000000002,"15min":18.437000000000001},"last":0.435},{"interface":"front","average":{"1min":2.6459999999999999,"5min":2.4460000000000002,"15min":2.4910000000000001},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":7.1959999999999997,"5min":10.34,"15min":24.524000000000001},"last":2.7629999999999999}]}]},{"osd":0,"up_from":94,"seq":403726926119,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1154796,"kb_used_data":437604,"kb_used_omap":7,"kb_used_meta":717176,"kb_avail":92590356,"statfs":{"total":95995035648,"available":94812524544,"internally_reserved":0,"allocated":448106496,"data_stored":445604420,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":734388241},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:44:24 2023","interfaces":[{"interface":"back","average":{"1min":3.2120000000000002,"5min":2.4100000000000001,"15min":2.7210000000000001},"min":{"1min":0.184,"5min":0.184,"15min":0.17199999999999999},"max":{"1min":18.052,"5min":24.010000000000002,"15min":42.000999999999998},"last":3.661},{"interface":"front","average":{"1min":3.0939999999999999,"5min":2.605,"15min":2.9129999999999998},"min":{"1min":0.33800000000000002,"5min":0.218,"15min":0.20399999999999999},"max":{"1min":18.097000000000001,"5min":18.097000000000001,"15min":23.036000000000001},"last":3.98}]},{"osd":2,"last update":"Sat Sep 23 20:43:58 2023","interfaces":[{"interface":"back","average":{"1min":2.5169999999999999,"5min":2.5699999999999998,"15min":3.0190000000000001},"min":{"1min":0.248,"5min":0.23400000000000001,"15min":0.185},"max":{"1min":10.199,"5min":48.841000000000001,"15min":77.448999999999998},"last":0.371},{"interface":"front","average":{"1min":1.9590000000000001,"5min":2.3239999999999998,"15min":2.9489999999999998},"min":{"1min":0.22,"5min":0.22,"15min":0.188},"max":{"1min":6.6580000000000004,"5min":46.183,"15min":81.724999999999994},"last":2.2850000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:44:31 2023","interfaces":[{"interface":"back","average":{"1min":3.6960000000000002,"5min":2.7290000000000001,"15min":3.1640000000000001},"min":{"1min":0.30599999999999999,"5min":0.248,"15min":0.23400000000000001},"max":{"1min":15.048,"5min":15.048,"15min":81.400999999999996},"last":3.4420000000000002},{"interface":"front","average":{"1min":2.1549999999999998,"5min":2.3159999999999998,"15min":3.3439999999999999},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.21199999999999999},"max":{"1min":9.6379999999999999,"5min":14.237,"15min":81.137},"last":4.0540000000000003}]},{"osd":4,"last update":"Sat Sep 23 20:43:58 2023","interfaces":[{"interface":"back","average":{"1min":1.385,"5min":1.925,"15min":1.9750000000000001},"min":{"1min":0.32000000000000001,"5min":0.31900000000000001,"15min":0.25900000000000001},"max":{"1min":4.5279999999999996,"5min":6.8970000000000002,"15min":12.587999999999999},"last":0.433},{"interface":"front","average":{"1min":1.7310000000000001,"5min":1.823,"15min":1.917},"min":{"1min":0.28899999999999998,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":6.5890000000000004,"5min":9.8480000000000008,"15min":23.001000000000001},"last":6.6130000000000004}]},{"osd":5,"last update":"Sat Sep 23 20:43:58 2023","interfaces":[{"interface":"back","average":{"1min":2.077,"5min":2.2370000000000001,"15min":2.121},"min":{"1min":0.30599999999999999,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":6.6200000000000001,"5min":10.239000000000001,"15min":22.969999999999999},"last":7.4800000000000004},{"interface":"front","average":{"1min":2.5529999999999999,"5min":2.2010000000000001,"15min":2.3719999999999999},"min":{"1min":0.34300000000000003,"5min":0.318,"15min":0.25600000000000001},"max":{"1min":6.75,"5min":12.929,"15min":26.228000000000002},"last":2.1989999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:43:58 2023","interfaces":[{"interface":"back","average":{"1min":2.1659999999999999,"5min":2.1909999999999998,"15min":2.1680000000000001},"min":{"1min":0.433,"5min":0.34499999999999997,"15min":0.32300000000000001},"max":{"1min":7.6150000000000002,"5min":24.056000000000001,"15min":24.056000000000001},"last":6.681},{"interface":"front","average":{"1min":2.2799999999999998,"5min":1.79,"15min":2.1539999999999999},"min":{"1min":0.36399999999999999,"5min":0.34100000000000003,"15min":0.29399999999999998},"max":{"1min":6.7119999999999997,"5min":8.0549999999999997,"15min":26.268000000000001},"last":2.2290000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:43:58 2023","interfaces":[{"interface":"back","average":{"1min":2.9220000000000002,"5min":2.0899999999999999,"15min":2.3969999999999998},"min":{"1min":0.433,"5min":0.36799999999999999,"15min":0.30599999999999999},"max":{"1min":9.4250000000000007,"5min":9.4250000000000007,"15min":26.177},"last":2.2509999999999999},{"interface":"front","average":{"1min":2.669,"5min":2.4100000000000001,"15min":2.4529999999999998},"min":{"1min":0.46899999999999997,"5min":0.39200000000000002,"15min":0.34200000000000003},"max":{"1min":15.006,"5min":15.006,"15min":24.669},"last":0.40600000000000003}]}]},{"osd":2,"up_from":104,"seq":446676599067,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1048180,"kb_used_data":306412,"kb_used_omap":11,"kb_used_meta":741748,"kb_avail":92696972,"statfs":{"total":95995035648,"available":94921699328,"internally_reserved":0,"allocated":313765888,"data_stored":311261926,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11878,"internal_metadata":759550362},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:44:05 2023","interfaces":[{"interface":"back","average":{"1min":1.887,"5min":1.3859999999999999,"15min":2.4430000000000001},"min":{"1min":0.20499999999999999,"5min":0.17299999999999999,"15min":0.17299999999999999},"max":{"1min":11.07,"5min":11.07,"15min":82.424999999999997},"last":0.58299999999999996},{"interface":"front","average":{"1min":2.6739999999999999,"5min":2.012,"15min":2.7149999999999999},"min":{"1min":0.27700000000000002,"5min":0.23000000000000001,"15min":0.19},"max":{"1min":16.678999999999998,"5min":16.678999999999998,"15min":82.524000000000001},"last":3.9049999999999998}]},{"osd":1,"last update":"Sat Sep 23 20:44:05 2023","interfaces":[{"interface":"back","average":{"1min":1.4430000000000001,"5min":2.02,"15min":2.839},"min":{"1min":0.248,"5min":0.248,"15min":0.221},"max":{"1min":3.8069999999999999,"5min":12.981,"15min":46.405000000000001},"last":4.2149999999999999},{"interface":"front","average":{"1min":2.0590000000000002,"5min":1.9119999999999999,"15min":2.641},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":11.683999999999999,"5min":11.683999999999999,"15min":46.459000000000003},"last":0.25800000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:44:31 2023","interfaces":[{"interface":"back","average":{"1min":2.1800000000000002,"5min":2.2010000000000001,"15min":2.5990000000000002},"min":{"1min":0.253,"5min":0.253,"15min":0.19800000000000001},"max":{"1min":12.847,"5min":12.847,"15min":19.562999999999999},"last":4.4139999999999997},{"interface":"front","average":{"1min":0.91400000000000003,"5min":1.425,"15min":2.1720000000000002},"min":{"1min":0.26500000000000001,"5min":0.26400000000000001,"15min":0.249},"max":{"1min":6.3700000000000001,"5min":11.375,"15min":30.704999999999998},"last":3.8610000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:44:05 2023","interfaces":[{"interface":"back","average":{"1min":1.7509999999999999,"5min":1.6359999999999999,"15min":1.911},"min":{"1min":0.34300000000000003,"5min":0.29799999999999999,"15min":0.27500000000000002},"max":{"1min":6.8719999999999999,"5min":12.025,"15min":16.797999999999998},"last":4.016},{"interface":"front","average":{"1min":1.702,"5min":1.603,"15min":2.0659999999999998},"min":{"1min":0.38100000000000001,"5min":0.30599999999999999,"15min":0.30499999999999999},"max":{"1min":7.1150000000000002,"5min":9.1379999999999999,"15min":19.440999999999999},"last":4.2519999999999998}]},{"osd":5,"last update":"Sat Sep 23 20:44:05 2023","interfaces":[{"interface":"back","average":{"1min":2.194,"5min":1.863,"15min":2.452},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":6.3179999999999996,"5min":12.07,"15min":95.393000000000001},"last":3.956},{"interface":"front","average":{"1min":1.7929999999999999,"5min":1.6220000000000001,"15min":2.5659999999999998},"min":{"1min":0.377,"5min":0.37,"15min":0.34799999999999998},"max":{"1min":5.1909999999999998,"5min":7.8179999999999996,"15min":95.414000000000001},"last":4.2830000000000004}]},{"osd":6,"last update":"Sat Sep 23 20:44:05 2023","interfaces":[{"interface":"back","average":{"1min":2.258,"5min":1.9059999999999999,"15min":2.246},"min":{"1min":0.39800000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":11.894,"5min":14.047000000000001,"15min":19.486000000000001},"last":4.3150000000000004},{"interface":"front","average":{"1min":2.069,"5min":1.323,"15min":1.7190000000000001},"min":{"1min":0.48099999999999998,"5min":0.34200000000000003,"15min":0.316},"max":{"1min":11.747,"5min":11.747,"15min":30.670999999999999},"last":0.51600000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:44:05 2023","interfaces":[{"interface":"back","average":{"1min":1.4399999999999999,"5min":1.2929999999999999,"15min":1.7270000000000001},"min":{"1min":0.39200000000000002,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":6.1849999999999996,"5min":8.4949999999999992,"15min":30.741},"last":0.55500000000000005},{"interface":"front","average":{"1min":2.6360000000000001,"5min":2.081,"15min":2.2469999999999999},"min":{"1min":0.44700000000000001,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":6.2910000000000004,"5min":12.099,"15min":16.719000000000001},"last":3.984}]}]},{"osd":3,"up_from":109,"seq":468151435541,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1755808,"kb_used_data":796440,"kb_used_omap":4,"kb_used_meta":959355,"kb_avail":91989344,"statfs":{"total":95995035648,"available":94197088256,"internally_reserved":0,"allocated":815554560,"data_stored":813067338,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4514,"internal_metadata":982380126},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:44:39 2023","interfaces":[{"interface":"back","average":{"1min":2.71,"5min":2.4169999999999998,"15min":2.4929999999999999},"min":{"1min":0.222,"5min":0.20899999999999999,"15min":0.191},"max":{"1min":8.5419999999999998,"5min":22.32,"15min":22.32},"last":5.5339999999999998},{"interface":"front","average":{"1min":2.105,"5min":2.6749999999999998,"15min":2.4929999999999999},"min":{"1min":0.249,"5min":0.22900000000000001,"15min":0.19500000000000001},"max":{"1min":6.2160000000000002,"5min":22.805,"15min":22.805},"last":5.4930000000000003}]},{"osd":1,"last update":"Sat Sep 23 20:44:11 2023","interfaces":[{"interface":"back","average":{"1min":2.823,"5min":2.1960000000000002,"15min":2.2290000000000001},"min":{"1min":0.31,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":10.022,"5min":17.529,"15min":17.529},"last":5.423},{"interface":"front","average":{"1min":1.742,"5min":2.3700000000000001,"15min":2.4500000000000002},"min":{"1min":0.33300000000000002,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":7.1029999999999998,"5min":14.19,"15min":32.603999999999999},"last":3.9510000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:44:39 2023","interfaces":[{"interface":"back","average":{"1min":2.2970000000000002,"5min":3.5840000000000001,"15min":3.0019999999999998},"min":{"1min":0.28599999999999998,"5min":0.26600000000000001,"15min":0.23799999999999999},"max":{"1min":7.7859999999999996,"5min":68.206999999999994,"15min":68.206999999999994},"last":4.0570000000000004},{"interface":"front","average":{"1min":2.4470000000000001,"5min":2.7130000000000001,"15min":2.7450000000000001},"min":{"1min":0.251,"5min":0.245,"15min":0.20899999999999999},"max":{"1min":9.1760000000000002,"5min":67.736999999999995,"15min":67.736999999999995},"last":2.3370000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:44:39 2023","interfaces":[{"interface":"back","average":{"1min":2.2879999999999998,"5min":2.008,"15min":1.786},"min":{"1min":0.35799999999999998,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":6.7889999999999997,"5min":22.157,"15min":22.157},"last":5.468},{"interface":"front","average":{"1min":2.113,"5min":2.3290000000000002,"15min":2.3279999999999998},"min":{"1min":0.36499999999999999,"5min":0.33200000000000002,"15min":0.28100000000000003},"max":{"1min":7.9020000000000001,"5min":15.868,"15min":19.635000000000002},"last":2.375}]},{"osd":5,"last update":"Sat Sep 23 20:44:39 2023","interfaces":[{"interface":"back","average":{"1min":2.1840000000000002,"5min":2.2970000000000002,"15min":1.992},"min":{"1min":0.33800000000000002,"5min":0.32700000000000001,"15min":0.31900000000000001},"max":{"1min":6.7469999999999999,"5min":22.289999999999999,"15min":22.289999999999999},"last":5.5019999999999998},{"interface":"front","average":{"1min":1.8089999999999999,"5min":1.8440000000000001,"15min":1.841},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.32000000000000001},"max":{"1min":4.6559999999999997,"5min":12.952,"15min":12.952},"last":3.411}]},{"osd":6,"last update":"Sat Sep 23 20:44:39 2023","interfaces":[{"interface":"back","average":{"1min":1.845,"5min":2.1360000000000001,"15min":1.956},"min":{"1min":0.36599999999999999,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":6.5350000000000001,"5min":12.917,"15min":17.524999999999999},"last":3.4420000000000002},{"interface":"front","average":{"1min":1.988,"5min":2.3300000000000001,"15min":2.2869999999999999},"min":{"1min":0.38100000000000001,"5min":0.35099999999999998,"15min":0.33700000000000002},"max":{"1min":6.8479999999999999,"5min":15.936,"15min":19.524000000000001},"last":3.2370000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:44:39 2023","interfaces":[{"interface":"back","average":{"1min":2.7850000000000001,"5min":2.7509999999999999,"15min":2.6429999999999998},"min":{"1min":0.46999999999999997,"5min":0.41499999999999998,"15min":0.36899999999999999},"max":{"1min":8.4429999999999996,"5min":15.976000000000001,"15min":19.571999999999999},"last":3.3220000000000001},{"interface":"front","average":{"1min":2.4750000000000001,"5min":2.262,"15min":1.9850000000000001},"min":{"1min":0.46300000000000002,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":8.5289999999999999,"5min":22.202999999999999,"15min":22.202999999999999},"last":10.853999999999999}]}]},{"osd":4,"up_from":30,"seq":128849019295,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":841076,"kb_used_data":418668,"kb_used_omap":0,"kb_used_meta":422400,"kb_avail":92904076,"statfs":{"total":95995035648,"available":95133773824,"internally_reserved":0,"allocated":428716032,"data_stored":427208063,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":432537600},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:43:57 2023","interfaces":[{"interface":"back","average":{"1min":2.5609999999999999,"5min":1.8129999999999999,"15min":1.746},"min":{"1min":0.251,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":11.709,"5min":11.709,"15min":24.114999999999998},"last":0.68300000000000005},{"interface":"front","average":{"1min":2.8330000000000002,"5min":2.1440000000000001,"15min":1.9710000000000001},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.246},"max":{"1min":13.74,"5min":13.74,"15min":24.135999999999999},"last":0.63200000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:44:11 2023","interfaces":[{"interface":"back","average":{"1min":2.306,"5min":1.8979999999999999,"15min":1.7749999999999999},"min":{"1min":0.26300000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":8.7460000000000004,"5min":10.363,"15min":18.190999999999999},"last":0.61099999999999999},{"interface":"front","average":{"1min":1.355,"5min":1.5569999999999999,"15min":1.613},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":4.8659999999999997,"5min":10.426,"15min":18.173999999999999},"last":0.64600000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:44:11 2023","interfaces":[{"interface":"back","average":{"1min":2.6040000000000001,"5min":1.756,"15min":1.885},"min":{"1min":0.29499999999999998,"5min":0.28399999999999997,"15min":0.26600000000000001},"max":{"1min":9.3320000000000007,"5min":13.051,"15min":32.363},"last":0.53700000000000003},{"interface":"front","average":{"1min":2.0150000000000001,"5min":2.0539999999999998,"15min":2.2090000000000001},"min":{"1min":0.41199999999999998,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":7.6159999999999997,"5min":11.917999999999999,"15min":34.042999999999999},"last":0.65900000000000003}]},{"osd":3,"last update":"Sat Sep 23 20:44:27 2023","interfaces":[{"interface":"back","average":{"1min":1.877,"5min":2.04,"15min":1.9950000000000001},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":7.7480000000000002,"5min":12.147,"15min":14.515000000000001},"last":0.59299999999999997},{"interface":"front","average":{"1min":2.4260000000000002,"5min":2.0990000000000002,"15min":2.0699999999999998},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.30299999999999999},"max":{"1min":10.493,"5min":13.35,"15min":15.991},"last":0.71699999999999997}]},{"osd":5,"last update":"Sat Sep 23 20:43:57 2023","interfaces":[{"interface":"back","average":{"1min":2.1789999999999998,"5min":1.3540000000000001,"15min":1.3440000000000001},"min":{"1min":0.34399999999999997,"5min":0.28000000000000003,"15min":0.25800000000000001},"max":{"1min":7.5810000000000004,"5min":9.7260000000000009,"15min":82.866},"last":0.66800000000000004},{"interface":"front","average":{"1min":1.363,"5min":1.2729999999999999,"15min":1.3180000000000001},"min":{"1min":0.311,"5min":0.311,"15min":0.28999999999999998},"max":{"1min":6.601,"5min":7.0129999999999999,"15min":82.546000000000006},"last":0.51400000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:44:11 2023","interfaces":[{"interface":"back","average":{"1min":1.0920000000000001,"5min":1.5449999999999999,"15min":1.214},"min":{"1min":0.34300000000000003,"5min":0.29499999999999998,"15min":0.27600000000000002},"max":{"1min":3.96,"5min":14.731999999999999,"15min":14.731999999999999},"last":0.57299999999999995},{"interface":"front","average":{"1min":1.8959999999999999,"5min":1.2989999999999999,"15min":1.1180000000000001},"min":{"1min":0.34300000000000003,"5min":0.314,"15min":0.29899999999999999},"max":{"1min":6.3440000000000003,"5min":7.6150000000000002,"15min":11.199},"last":0.55300000000000005}]},{"osd":7,"last update":"Sat Sep 23 20:44:27 2023","interfaces":[{"interface":"back","average":{"1min":2.0630000000000002,"5min":2.0979999999999999,"15min":1.5649999999999999},"min":{"1min":0.374,"5min":0.35099999999999998,"15min":0.318},"max":{"1min":7.7629999999999999,"5min":22.184999999999999,"15min":22.184999999999999},"last":0.69599999999999995},{"interface":"front","average":{"1min":1.1240000000000001,"5min":1.724,"15min":1.2150000000000001},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.307},"max":{"1min":6.3689999999999998,"5min":21.754000000000001,"15min":21.754000000000001},"last":0.70699999999999996}]}]},{"osd":5,"up_from":39,"seq":167503724956,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1429948,"kb_used_data":927732,"kb_used_omap":0,"kb_used_meta":502208,"kb_avail":92315204,"statfs":{"total":95995035648,"available":94530768896,"internally_reserved":0,"allocated":949997568,"data_stored":948486230,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":514260992},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:44:00 2023","interfaces":[{"interface":"back","average":{"1min":2.145,"5min":1.7050000000000001,"15min":1.7989999999999999},"min":{"1min":0.23999999999999999,"5min":0.22500000000000001,"15min":0.20699999999999999},"max":{"1min":15.018000000000001,"5min":15.018000000000001,"15min":18.16},"last":0.58499999999999996},{"interface":"front","average":{"1min":2.7130000000000001,"5min":1.6870000000000001,"15min":1.7410000000000001},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":15.034000000000001,"5min":15.034000000000001,"15min":15.034000000000001},"last":0.52800000000000002}]},{"osd":1,"last update":"Sat Sep 23 20:44:14 2023","interfaces":[{"interface":"back","average":{"1min":2.7349999999999999,"5min":3.0299999999999998,"15min":2.2690000000000001},"min":{"1min":0.379,"5min":0.26500000000000001,"15min":0.253},"max":{"1min":9.4199999999999999,"5min":61.005000000000003,"15min":61.005000000000003},"last":0.59099999999999997},{"interface":"front","average":{"1min":1.339,"5min":2.4129999999999998,"15min":2.0019999999999998},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.23499999999999999},"max":{"1min":6.7350000000000003,"5min":59.688000000000002,"15min":59.688000000000002},"last":0.42799999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:44:00 2023","interfaces":[{"interface":"back","average":{"1min":2.3330000000000002,"5min":2.032,"15min":2.0950000000000002},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.26500000000000001},"max":{"1min":8.3719999999999999,"5min":9.2119999999999997,"15min":37.011000000000003},"last":0.625},{"interface":"front","average":{"1min":2.7240000000000002,"5min":1.9359999999999999,"15min":2.0659999999999998},"min":{"1min":0.314,"5min":0.28299999999999997,"15min":0.255},"max":{"1min":9.8260000000000005,"5min":10.565,"15min":36.835000000000001},"last":0.71399999999999997}]},{"osd":3,"last update":"Sat Sep 23 20:44:20 2023","interfaces":[{"interface":"back","average":{"1min":1.7769999999999999,"5min":2.375,"15min":2.375},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.29599999999999999},"max":{"1min":5.6619999999999999,"5min":11.032,"15min":36.716000000000001},"last":0.53800000000000003},{"interface":"front","average":{"1min":1.5900000000000001,"5min":2.238,"15min":2.2530000000000001},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":7.4100000000000001,"5min":13.266,"15min":18.504000000000001},"last":0.56799999999999995}]},{"osd":4,"last update":"Sat Sep 23 20:43:50 2023","interfaces":[{"interface":"back","average":{"1min":3.8220000000000001,"5min":1.786,"15min":1.292},"min":{"1min":0.309,"5min":0.27300000000000002,"15min":0.255},"max":{"1min":13.391999999999999,"5min":13.391999999999999,"15min":28.015999999999998},"last":0.45500000000000002},{"interface":"front","average":{"1min":2.1760000000000002,"5min":1.5669999999999999,"15min":1.1220000000000001},"min":{"1min":0.34699999999999998,"5min":0.30299999999999999,"15min":0.28299999999999997},"max":{"1min":7.3799999999999999,"5min":7.3799999999999999,"15min":12.300000000000001},"last":0.54900000000000004}]},{"osd":6,"last update":"Sat Sep 23 20:44:07 2023","interfaces":[{"interface":"back","average":{"1min":3.3999999999999999,"5min":2.04,"15min":1.3320000000000001},"min":{"1min":0.39100000000000001,"5min":0.317,"15min":0.30199999999999999},"max":{"1min":13.734,"5min":13.734,"15min":13.734},"last":0.38100000000000001},{"interface":"front","average":{"1min":1.375,"5min":1.577,"15min":1.1950000000000001},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":5.274,"5min":8.7449999999999992,"15min":16.265999999999998},"last":0.50600000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:44:20 2023","interfaces":[{"interface":"back","average":{"1min":1.4610000000000001,"5min":1.7330000000000001,"15min":1.1339999999999999},"min":{"1min":0.372,"5min":0.34499999999999997,"15min":0.28499999999999998},"max":{"1min":4.3250000000000002,"5min":16.366,"15min":16.366},"last":0.47399999999999998},{"interface":"front","average":{"1min":1.4950000000000001,"5min":1.6439999999999999,"15min":1.173},"min":{"1min":0.38900000000000001,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":7.524,"5min":7.915,"15min":13.122},"last":0.48799999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4722,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1360,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8206,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7156,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3154,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":430702592,"data_stored":430702592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1171591168,"data_stored":1171587096,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":302317568,"data_stored":302317568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":808648704,"data_stored":808648704,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":416747520,"data_stored":416743448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":942477312,"data_stored":942473240,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":307888128,"data_stored":307888128,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1020071936,"data_stored":1020067864,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:44:47.139 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:44:47.321 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T20:44:47.367 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:44:47.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:47 smithi149 ceph-mon[170394]: pgmap v1292: 119 pgs: 1 remapped+peering, 2 peering, 116 active+clean; 1.5 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 709 op/s 2023-09-23T20:44:47.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:47 smithi149 ceph-mon[170394]: from='client.46493 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:44:47.968 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:47 smithi149 ceph-mon[170394]: failed to encode map e517 with expected crc 2023-09-23T20:44:47.968 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:47 smithi149 ceph-mon[170394]: osdmap e517: 8 total, 8 up, 8 in 2023-09-23T20:44:48.092 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:47 smithi018 ceph-mon[179647]: pgmap v1292: 119 pgs: 1 remapped+peering, 2 peering, 116 active+clean; 1.5 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 709 op/s 2023-09-23T20:44:48.092 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:47 smithi018 ceph-mon[179647]: from='client.46493 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:44:48.092 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:47 smithi018 ceph-mon[179647]: failed to encode map e517 with expected crc 2023-09-23T20:44:48.093 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:47 smithi018 ceph-mon[179647]: osdmap e517: 8 total, 8 up, 8 in 2023-09-23T20:44:48.094 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:47 smithi018 ceph-mon[184144]: pgmap v1292: 119 pgs: 1 remapped+peering, 2 peering, 116 active+clean; 1.5 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 709 op/s 2023-09-23T20:44:48.094 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:47 smithi018 ceph-mon[184144]: from='client.46493 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:44:48.094 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:47 smithi018 ceph-mon[184144]: failed to encode map e517 with expected crc 2023-09-23T20:44:48.094 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:47 smithi018 ceph-mon[184144]: osdmap e517: 8 total, 8 up, 8 in 2023-09-23T20:44:48.379 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:44:48.396 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:48.396 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:44:48.424 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T20:44:48.429 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:44:48.497 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:44:48.577 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:48.596 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:44:48.631 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:44:48.634 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:48.635 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:44:48.736 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:44:48.786 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:48.835 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:44:48.851 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:48.852 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:44:48.952 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:44:48.986 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:49.024 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:44:49.028 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:49.028 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:44:49.129 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:44:49.196 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:44:49.204 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:49.252 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:44:49.286 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:44:49.319 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:44:49.322 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:44:49.329 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:49.330 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:44:49.412 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:44:49.431 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:44:49.436 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:44:49.476 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:44:49.482 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:49.513 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:44:49.548 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:44:49.552 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:44:49.552 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:44:49.552 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:44:49.553 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:44:49.553 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:44:49.553 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:44:49.553 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:44:49.553 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:44:49.688 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:44:49.695 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:49.696 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:44:49.709 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:49 smithi149 ceph-mon[170394]: pgmap v1294: 119 pgs: 1 remapped+peering, 1 peering, 117 active+clean; 1.5 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 367 op/s 2023-09-23T20:44:49.709 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:49 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2767256717' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:44:49.790 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:49 smithi018 ceph-mon[184144]: pgmap v1294: 119 pgs: 1 remapped+peering, 1 peering, 117 active+clean; 1.5 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 367 op/s 2023-09-23T20:44:49.791 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:49 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2767256717' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:44:49.795 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:49 smithi018 ceph-mon[179647]: pgmap v1294: 119 pgs: 1 remapped+peering, 1 peering, 117 active+clean; 1.5 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 367 op/s 2023-09-23T20:44:49.796 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:49 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2767256717' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:44:49.797 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:44:49.855 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:49.978 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:44:49.987 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:49.988 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:44:50.089 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:44:50.138 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:50.203 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:44:50.214 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:50.214 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:44:50.316 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:44:50.396 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:50.550 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:44:50.555 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:50.555 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:44:50.656 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:44:50.671 INFO:teuthology.orchestra.run.smithi149.stdout:ERROR: (22) Invalid argument 2023-09-23T20:44:50.671 INFO:teuthology.orchestra.run.smithi149.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:44:50.747 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:50.846 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:44:50.861 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:50.861 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:44:50.962 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:44:51.006 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:51.061 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:44:51.066 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:51.067 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:44:51.092 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:44:51.100 INFO:teuthology.orchestra.run.smithi018.stderr:dumped all 2023-09-23T20:44:51.147 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T20:44:51.168 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:44:51.203 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:51.509 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:44:51.511 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:51.512 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:44:51.559 INFO:teuthology.orchestra.run.smithi018.stdout:{"pg_ready":true,"pg_map":{"version":1296,"stamp":"2023-09-23T20:44:50.727997+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1472731083,"num_objects":22428,"num_object_clones":0,"num_object_copies":67284,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":329,"num_objects_unfound":0,"num_objects_dirty":22428,"num_whiteouts":0,"num_read":5984,"num_read_kb":13201,"num_write":206171,"num_write_kb":7297910,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":17055,"num_bytes_recovered":1122650190,"num_keys_recovered":40,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":108881,"ondisk_log_size":108881,"up":357,"acting":357,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":361,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":9656692,"kb_used_data":4766964,"kb_used_omap":32,"kb_used_meta":4889311,"kb_avail":740304524,"statfs":{"total":767960285184,"available":758071832576,"internally_reserved":0,"allocated":4881371136,"data_stored":4865085629,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32773,"internal_metadata":5006655483},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,2,7,1,0,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-170328064,"num_objects":-2599,"num_object_clones":0,"num_object_copies":-7797,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":329,"num_objects_unfound":0,"num_objects_dirty":-2599,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2599,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":427,"num_bytes_recovered":27656192,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.093120"},"pg_stats":[{"pgid":"1.18","version":"90'161","reported_seq":562,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-23T20:44:45.669689+0000","last_change":"2023-09-23T20:44:19.677357+0000","last_active":"2023-09-23T20:44:45.669689+0000","last_peered":"2023-09-23T20:44:45.669689+0000","last_clean":"2023-09-23T20:44:45.669689+0000","last_became_active":"2023-09-23T20:43:16.766305+0000","last_became_peered":"2023-09-23T20:43:16.766305+0000","last_unstale":"2023-09-23T20:44:45.669689+0000","last_undegraded":"2023-09-23T20:44:45.669689+0000","last_fullsized":"2023-09-23T20:44:45.669689+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:19.677209+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:19.677209+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:14:06.510115+0000","scrub_duration":0.0023292260000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"69'1","reported_seq":214,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-23T20:44:45.686097+0000","last_change":"2023-09-23T20:32:33.233853+0000","last_active":"2023-09-23T20:44:45.686097+0000","last_peered":"2023-09-23T20:44:45.686097+0000","last_clean":"2023-09-23T20:44:45.686097+0000","last_became_active":"2023-09-23T20:32:33.233546+0000","last_became_peered":"2023-09-23T20:32:33.233546+0000","last_unstale":"2023-09-23T20:44:45.686097+0000","last_undegraded":"2023-09-23T20:44:45.686097+0000","last_fullsized":"2023-09-23T20:44:45.686097+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:50:27.130808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":331,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-23T20:44:45.673992+0000","last_change":"2023-09-23T20:37:43.579611+0000","last_active":"2023-09-23T20:44:45.673992+0000","last_peered":"2023-09-23T20:44:45.673992+0000","last_clean":"2023-09-23T20:44:45.673992+0000","last_became_active":"2023-09-23T20:37:43.579323+0000","last_became_peered":"2023-09-23T20:37:43.579323+0000","last_unstale":"2023-09-23T20:44:45.673992+0000","last_undegraded":"2023-09-23T20:44:45.673992+0000","last_fullsized":"2023-09-23T20:44:45.673992+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:17:32.311427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"90'161","reported_seq":638,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-23T20:44:45.686989+0000","last_change":"2023-09-23T20:44:14.647447+0000","last_active":"2023-09-23T20:44:45.686989+0000","last_peered":"2023-09-23T20:44:45.686989+0000","last_clean":"2023-09-23T20:44:45.686989+0000","last_became_active":"2023-09-23T20:43:16.756116+0000","last_became_peered":"2023-09-23T20:43:16.756116+0000","last_unstale":"2023-09-23T20:44:45.686989+0000","last_undegraded":"2023-09-23T20:44:45.686989+0000","last_fullsized":"2023-09-23T20:44:45.686989+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:14.647386+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:36:05.442288+0000","last_clean_scrub_stamp":"2023-09-23T20:44:14.647386+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:25:25.151100+0000","scrub_duration":0.0097381210000000006,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":487,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-23T20:44:45.679861+0000","last_change":"2023-09-23T20:34:59.983035+0000","last_active":"2023-09-23T20:44:45.679861+0000","last_peered":"2023-09-23T20:44:45.679861+0000","last_clean":"2023-09-23T20:44:45.679861+0000","last_became_active":"2023-09-23T20:34:59.982459+0000","last_became_peered":"2023-09-23T20:34:59.982459+0000","last_unstale":"2023-09-23T20:44:45.679861+0000","last_undegraded":"2023-09-23T20:44:45.679861+0000","last_fullsized":"2023-09-23T20:44:45.679861+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:16:48.510783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":473,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-09-23T20:44:44.664810+0000","last_change":"2023-09-23T20:43:46.521939+0000","last_active":"2023-09-23T20:44:44.664810+0000","last_peered":"2023-09-23T20:44:44.664810+0000","last_clean":"2023-09-23T20:44:44.664810+0000","last_became_active":"2023-09-23T20:43:46.521185+0000","last_became_peered":"2023-09-23T20:43:46.521185+0000","last_unstale":"2023-09-23T20:44:44.664810+0000","last_undegraded":"2023-09-23T20:44:44.664810+0000","last_fullsized":"2023-09-23T20:44:44.664810+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:59:25.045627+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"90'161","reported_seq":596,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-23T20:44:45.684601+0000","last_change":"2023-09-23T20:36:01.292103+0000","last_active":"2023-09-23T20:44:45.684601+0000","last_peered":"2023-09-23T20:44:45.684601+0000","last_clean":"2023-09-23T20:44:45.684601+0000","last_became_active":"2023-09-23T20:35:26.639912+0000","last_became_peered":"2023-09-23T20:35:26.639912+0000","last_unstale":"2023-09-23T20:44:45.684601+0000","last_undegraded":"2023-09-23T20:44:45.684601+0000","last_fullsized":"2023-09-23T20:44:45.684601+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:01.292032+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:01.292032+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:04:53.926640+0000","scrub_duration":0.0032983159999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":311,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-23T20:44:45.675099+0000","last_change":"2023-09-23T20:37:43.584414+0000","last_active":"2023-09-23T20:44:45.675099+0000","last_peered":"2023-09-23T20:44:45.675099+0000","last_clean":"2023-09-23T20:44:45.675099+0000","last_became_active":"2023-09-23T20:37:43.584116+0000","last_became_peered":"2023-09-23T20:37:43.584116+0000","last_unstale":"2023-09-23T20:44:45.675099+0000","last_undegraded":"2023-09-23T20:44:45.675099+0000","last_fullsized":"2023-09-23T20:44:45.675099+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:17:54.258172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":132,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-23T20:44:45.673468+0000","last_change":"2023-09-23T20:38:37.457427+0000","last_active":"2023-09-23T20:44:45.673468+0000","last_peered":"2023-09-23T20:44:45.673468+0000","last_clean":"2023-09-23T20:44:45.673468+0000","last_became_active":"2023-09-23T20:38:37.457099+0000","last_became_peered":"2023-09-23T20:38:37.457099+0000","last_unstale":"2023-09-23T20:44:45.673468+0000","last_undegraded":"2023-09-23T20:44:45.673468+0000","last_fullsized":"2023-09-23T20:44:45.673468+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:31:27.084994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"90'161","reported_seq":729,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-23T20:44:45.677149+0000","last_change":"2023-09-23T20:40:22.449754+0000","last_active":"2023-09-23T20:44:45.677149+0000","last_peered":"2023-09-23T20:44:45.677149+0000","last_clean":"2023-09-23T20:44:45.677149+0000","last_became_active":"2023-09-23T20:40:22.448838+0000","last_became_peered":"2023-09-23T20:40:22.448838+0000","last_unstale":"2023-09-23T20:44:45.677149+0000","last_undegraded":"2023-09-23T20:44:45.677149+0000","last_fullsized":"2023-09-23T20:44:45.677149+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:03.411025+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:03.411025+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:22:07.948856+0000","scrub_duration":0.001802922,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":506,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-09-23T20:44:44.664876+0000","last_change":"2023-09-23T20:43:46.522325+0000","last_active":"2023-09-23T20:44:44.664876+0000","last_peered":"2023-09-23T20:44:44.664876+0000","last_clean":"2023-09-23T20:44:44.664876+0000","last_became_active":"2023-09-23T20:43:46.521996+0000","last_became_peered":"2023-09-23T20:43:46.521996+0000","last_unstale":"2023-09-23T20:44:44.664876+0000","last_undegraded":"2023-09-23T20:44:44.664876+0000","last_fullsized":"2023-09-23T20:44:44.664876+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:51:17.629956+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":242,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-23T20:44:45.669810+0000","last_change":"2023-09-23T20:38:02.579343+0000","last_active":"2023-09-23T20:44:45.669810+0000","last_peered":"2023-09-23T20:44:45.669810+0000","last_clean":"2023-09-23T20:44:45.669810+0000","last_became_active":"2023-09-23T20:38:02.579047+0000","last_became_peered":"2023-09-23T20:38:02.579047+0000","last_unstale":"2023-09-23T20:44:45.669810+0000","last_undegraded":"2023-09-23T20:44:45.669810+0000","last_fullsized":"2023-09-23T20:44:45.669810+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:44:03.166556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.14","version":"90'161","reported_seq":848,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-23T20:44:45.669833+0000","last_change":"2023-09-23T20:35:02.236398+0000","last_active":"2023-09-23T20:44:45.669833+0000","last_peered":"2023-09-23T20:44:45.669833+0000","last_clean":"2023-09-23T20:44:45.669833+0000","last_became_active":"2023-09-23T20:35:02.235988+0000","last_became_peered":"2023-09-23T20:35:02.235988+0000","last_unstale":"2023-09-23T20:44:45.669833+0000","last_undegraded":"2023-09-23T20:44:45.669833+0000","last_fullsized":"2023-09-23T20:44:45.669833+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:47.479012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:47.479012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:13.519648+0000","scrub_duration":0.0020136590000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":353,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-23T20:44:45.681976+0000","last_change":"2023-09-23T20:43:16.780478+0000","last_active":"2023-09-23T20:44:45.681976+0000","last_peered":"2023-09-23T20:44:45.681976+0000","last_clean":"2023-09-23T20:44:45.681976+0000","last_became_active":"2023-09-23T20:43:16.780183+0000","last_became_peered":"2023-09-23T20:43:16.780183+0000","last_unstale":"2023-09-23T20:44:45.681976+0000","last_undegraded":"2023-09-23T20:44:45.681976+0000","last_fullsized":"2023-09-23T20:44:45.681976+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:38:30.102066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":343,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-23T20:44:45.675890+0000","last_change":"2023-09-23T20:38:02.542034+0000","last_active":"2023-09-23T20:44:45.675890+0000","last_peered":"2023-09-23T20:44:45.675890+0000","last_clean":"2023-09-23T20:44:45.675890+0000","last_became_active":"2023-09-23T20:38:02.541575+0000","last_became_peered":"2023-09-23T20:38:02.541575+0000","last_unstale":"2023-09-23T20:44:45.675890+0000","last_undegraded":"2023-09-23T20:44:45.675890+0000","last_fullsized":"2023-09-23T20:44:45.675890+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:50:54.591529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"90'161","reported_seq":641,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-09-23T20:44:44.664655+0000","last_change":"2023-09-23T20:44:14.669973+0000","last_active":"2023-09-23T20:44:44.664655+0000","last_peered":"2023-09-23T20:44:44.664655+0000","last_clean":"2023-09-23T20:44:44.664655+0000","last_became_active":"2023-09-23T20:43:46.519592+0000","last_became_peered":"2023-09-23T20:43:46.519592+0000","last_unstale":"2023-09-23T20:44:44.664655+0000","last_undegraded":"2023-09-23T20:44:44.664655+0000","last_fullsized":"2023-09-23T20:44:44.664655+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:14.669831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:14.669831+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:56:28.076617+0000","scrub_duration":0.0083441460000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":544,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-23T20:44:45.677511+0000","last_change":"2023-09-23T20:32:33.270617+0000","last_active":"2023-09-23T20:44:45.677511+0000","last_peered":"2023-09-23T20:44:45.677511+0000","last_clean":"2023-09-23T20:44:45.677511+0000","last_became_active":"2023-09-23T20:32:33.227902+0000","last_became_peered":"2023-09-23T20:32:33.227902+0000","last_unstale":"2023-09-23T20:44:45.677511+0000","last_undegraded":"2023-09-23T20:44:45.677511+0000","last_fullsized":"2023-09-23T20:44:45.677511+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:30:26.847831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":360,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":31,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-23T20:44:45.686890+0000","last_change":"2023-09-23T20:44:10.611381+0000","last_active":"2023-09-23T20:44:45.686890+0000","last_peered":"2023-09-23T20:44:45.686890+0000","last_clean":"2023-09-23T20:44:45.686890+0000","last_became_active":"2023-09-23T20:44:10.611091+0000","last_became_peered":"2023-09-23T20:44:10.611091+0000","last_unstale":"2023-09-23T20:44:45.686890+0000","last_undegraded":"2023-09-23T20:44:45.686890+0000","last_fullsized":"2023-09-23T20:44:45.686890+0000","mapping_epoch":496,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":497,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:00:30.162478+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"90'161","reported_seq":642,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-23T20:44:45.680929+0000","last_change":"2023-09-23T20:36:02.548387+0000","last_active":"2023-09-23T20:44:45.680929+0000","last_peered":"2023-09-23T20:44:45.680929+0000","last_clean":"2023-09-23T20:44:45.680929+0000","last_became_active":"2023-09-23T20:35:26.644787+0000","last_became_peered":"2023-09-23T20:35:26.644787+0000","last_unstale":"2023-09-23T20:44:45.680929+0000","last_undegraded":"2023-09-23T20:44:45.680929+0000","last_fullsized":"2023-09-23T20:44:45.680929+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:02.548285+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:02.548285+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:37:15.216553+0000","scrub_duration":0.0015514459999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":264,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-23T20:44:45.686178+0000","last_change":"2023-09-23T20:38:02.560646+0000","last_active":"2023-09-23T20:44:45.686178+0000","last_peered":"2023-09-23T20:44:45.686178+0000","last_clean":"2023-09-23T20:44:45.686178+0000","last_became_active":"2023-09-23T20:38:02.539814+0000","last_became_peered":"2023-09-23T20:38:02.539814+0000","last_unstale":"2023-09-23T20:44:45.686178+0000","last_undegraded":"2023-09-23T20:44:45.686178+0000","last_fullsized":"2023-09-23T20:44:45.686178+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:00:09.435023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":244,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-23T20:44:45.687558+0000","last_change":"2023-09-23T20:38:02.579222+0000","last_active":"2023-09-23T20:44:45.687558+0000","last_peered":"2023-09-23T20:44:45.687558+0000","last_clean":"2023-09-23T20:44:45.687558+0000","last_became_active":"2023-09-23T20:38:02.578813+0000","last_became_peered":"2023-09-23T20:38:02.578813+0000","last_unstale":"2023-09-23T20:44:45.687558+0000","last_undegraded":"2023-09-23T20:44:45.687558+0000","last_fullsized":"2023-09-23T20:44:45.687558+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:57:31.080027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"90'161","reported_seq":769,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-09-23T20:44:44.664397+0000","last_change":"2023-09-23T20:43:46.544727+0000","last_active":"2023-09-23T20:44:44.664397+0000","last_peered":"2023-09-23T20:44:44.664397+0000","last_clean":"2023-09-23T20:44:44.664397+0000","last_became_active":"2023-09-23T20:43:46.543952+0000","last_became_peered":"2023-09-23T20:43:46.543952+0000","last_unstale":"2023-09-23T20:44:44.664397+0000","last_undegraded":"2023-09-23T20:44:44.664397+0000","last_fullsized":"2023-09-23T20:44:44.664397+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:43.578666+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:43.578666+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:11:16.611920+0000","scrub_duration":0.0021890379999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":543,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-23T20:44:45.677363+0000","last_change":"2023-09-23T20:32:33.242943+0000","last_active":"2023-09-23T20:44:45.677363+0000","last_peered":"2023-09-23T20:44:45.677363+0000","last_clean":"2023-09-23T20:44:45.677363+0000","last_became_active":"2023-09-23T20:32:33.225468+0000","last_became_peered":"2023-09-23T20:32:33.225468+0000","last_unstale":"2023-09-23T20:44:45.677363+0000","last_undegraded":"2023-09-23T20:44:45.677363+0000","last_fullsized":"2023-09-23T20:44:45.677363+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:40:47.867573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":254,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-23T20:44:45.674846+0000","last_change":"2023-09-23T20:37:43.571836+0000","last_active":"2023-09-23T20:44:45.674846+0000","last_peered":"2023-09-23T20:44:45.674846+0000","last_clean":"2023-09-23T20:44:45.674846+0000","last_became_active":"2023-09-23T20:37:43.571511+0000","last_became_peered":"2023-09-23T20:37:43.571511+0000","last_unstale":"2023-09-23T20:44:45.674846+0000","last_undegraded":"2023-09-23T20:44:45.674846+0000","last_fullsized":"2023-09-23T20:44:45.674846+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:59:25.507917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"90'161","reported_seq":797,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-23T20:44:45.676584+0000","last_change":"2023-09-23T20:44:20.041766+0000","last_active":"2023-09-23T20:44:45.676584+0000","last_peered":"2023-09-23T20:44:45.676584+0000","last_clean":"2023-09-23T20:44:45.676584+0000","last_became_active":"2023-09-23T20:43:16.765015+0000","last_became_peered":"2023-09-23T20:43:16.765015+0000","last_unstale":"2023-09-23T20:44:45.676584+0000","last_undegraded":"2023-09-23T20:44:45.676584+0000","last_fullsized":"2023-09-23T20:44:45.676584+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:20.041690+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:20.041690+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:33:30.150109+0000","scrub_duration":0.0026180330000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":471,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-09-23T20:44:44.664908+0000","last_change":"2023-09-23T20:43:46.521294+0000","last_active":"2023-09-23T20:44:44.664908+0000","last_peered":"2023-09-23T20:44:44.664908+0000","last_clean":"2023-09-23T20:44:44.664908+0000","last_became_active":"2023-09-23T20:43:46.521001+0000","last_became_peered":"2023-09-23T20:43:46.521001+0000","last_unstale":"2023-09-23T20:44:44.664908+0000","last_undegraded":"2023-09-23T20:44:44.664908+0000","last_fullsized":"2023-09-23T20:44:44.664908+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:01:35.126716+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":284,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-23T20:44:45.677623+0000","last_change":"2023-09-23T20:44:01.841565+0000","last_active":"2023-09-23T20:44:45.677623+0000","last_peered":"2023-09-23T20:44:45.677623+0000","last_clean":"2023-09-23T20:44:45.677623+0000","last_became_active":"2023-09-23T20:44:01.841255+0000","last_became_peered":"2023-09-23T20:44:01.841255+0000","last_unstale":"2023-09-23T20:44:45.677623+0000","last_undegraded":"2023-09-23T20:44:45.677623+0000","last_fullsized":"2023-09-23T20:44:45.677623+0000","mapping_epoch":491,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":492,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:18:44.817589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.13","version":"90'161","reported_seq":804,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-23T20:44:45.680602+0000","last_change":"2023-09-23T20:35:02.238923+0000","last_active":"2023-09-23T20:44:45.680602+0000","last_peered":"2023-09-23T20:44:45.680602+0000","last_clean":"2023-09-23T20:44:45.680602+0000","last_became_active":"2023-09-23T20:35:02.238293+0000","last_became_peered":"2023-09-23T20:35:02.238293+0000","last_unstale":"2023-09-23T20:44:45.680602+0000","last_undegraded":"2023-09-23T20:44:45.680602+0000","last_fullsized":"2023-09-23T20:44:45.680602+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:42.544308+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:42.544308+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:51:22.281932+0000","scrub_duration":0.0029958179999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":520,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-23T20:44:45.678414+0000","last_change":"2023-09-23T20:38:02.539774+0000","last_active":"2023-09-23T20:44:45.678414+0000","last_peered":"2023-09-23T20:44:45.678414+0000","last_clean":"2023-09-23T20:44:45.678414+0000","last_became_active":"2023-09-23T20:38:02.539436+0000","last_became_peered":"2023-09-23T20:38:02.539436+0000","last_unstale":"2023-09-23T20:44:45.678414+0000","last_undegraded":"2023-09-23T20:44:45.678414+0000","last_fullsized":"2023-09-23T20:44:45.678414+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:50:57.375803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":480,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-09-23T20:44:44.664886+0000","last_change":"2023-09-23T20:43:46.521735+0000","last_active":"2023-09-23T20:44:44.664886+0000","last_peered":"2023-09-23T20:44:44.664886+0000","last_clean":"2023-09-23T20:44:44.664886+0000","last_became_active":"2023-09-23T20:43:46.520762+0000","last_became_peered":"2023-09-23T20:43:46.520762+0000","last_unstale":"2023-09-23T20:44:44.664886+0000","last_undegraded":"2023-09-23T20:44:44.664886+0000","last_fullsized":"2023-09-23T20:44:44.664886+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:26:48.841930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"90'161","reported_seq":600,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-23T20:44:45.687583+0000","last_change":"2023-09-23T20:44:18.952808+0000","last_active":"2023-09-23T20:44:45.687583+0000","last_peered":"2023-09-23T20:44:45.687583+0000","last_clean":"2023-09-23T20:44:45.687583+0000","last_became_active":"2023-09-23T20:43:16.758652+0000","last_became_peered":"2023-09-23T20:43:16.758652+0000","last_unstale":"2023-09-23T20:44:45.687583+0000","last_undegraded":"2023-09-23T20:44:45.687583+0000","last_fullsized":"2023-09-23T20:44:45.687583+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:18.952693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:18.952693+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:12:19.496465+0000","scrub_duration":0.001699365,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":192,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-23T20:44:45.673941+0000","last_change":"2023-09-23T20:44:43.644290+0000","last_active":"2023-09-23T20:44:45.673941+0000","last_peered":"2023-09-23T20:44:45.673941+0000","last_clean":"2023-09-23T20:44:45.673941+0000","last_became_active":"2023-09-23T20:44:43.638313+0000","last_became_peered":"2023-09-23T20:44:43.638313+0000","last_unstale":"2023-09-23T20:44:45.673941+0000","last_undegraded":"2023-09-23T20:44:45.673941+0000","last_fullsized":"2023-09-23T20:44:45.673941+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:32:52.892603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":311,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-23T20:44:45.685406+0000","last_change":"2023-09-23T20:37:43.571083+0000","last_active":"2023-09-23T20:44:45.685406+0000","last_peered":"2023-09-23T20:44:45.685406+0000","last_clean":"2023-09-23T20:44:45.685406+0000","last_became_active":"2023-09-23T20:37:43.570306+0000","last_became_peered":"2023-09-23T20:37:43.570306+0000","last_unstale":"2023-09-23T20:44:45.685406+0000","last_undegraded":"2023-09-23T20:44:45.685406+0000","last_fullsized":"2023-09-23T20:44:45.685406+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:42:02.396718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.5","version":"516'12488","reported_seq":21044,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-23T20:44:46.360649+0000","last_change":"2023-09-23T20:44:20.680609+0000","last_active":"2023-09-23T20:44:46.360649+0000","last_peered":"2023-09-23T20:44:46.360649+0000","last_clean":"2023-09-23T20:44:46.360649+0000","last_became_active":"2023-09-23T20:44:20.638636+0000","last_became_peered":"2023-09-23T20:44:20.638636+0000","last_unstale":"2023-09-23T20:44:46.360649+0000","last_undegraded":"2023-09-23T20:44:46.360649+0000","last_fullsized":"2023-09-23T20:44:46.360649+0000","mapping_epoch":504,"log_start":"466'4910","ondisk_log_start":"466'4910","created":455,"last_epoch_clean":505,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":7578,"log_dups_size":0,"ondisk_log_size":7578,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:11:11.154742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":98172928,"num_objects":1498,"num_object_clones":0,"num_object_copies":4494,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1498,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12488,"num_write_kb":447552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4204,"num_bytes_recovered":275316736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.d","version":"90'161","reported_seq":809,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-23T20:44:45.677275+0000","last_change":"2023-09-23T20:34:59.496438+0000","last_active":"2023-09-23T20:44:45.677275+0000","last_peered":"2023-09-23T20:44:45.677275+0000","last_clean":"2023-09-23T20:44:45.677275+0000","last_became_active":"2023-09-23T20:33:38.970527+0000","last_became_peered":"2023-09-23T20:33:38.970527+0000","last_unstale":"2023-09-23T20:44:45.677275+0000","last_undegraded":"2023-09-23T20:44:45.677275+0000","last_fullsized":"2023-09-23T20:44:45.677275+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:34:59.496375+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:34:59.496375+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:46:18.244734+0000","scrub_duration":0.0017843189999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":520,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-23T20:44:45.678543+0000","last_change":"2023-09-23T20:32:33.283752+0000","last_active":"2023-09-23T20:44:45.678543+0000","last_peered":"2023-09-23T20:44:45.678543+0000","last_clean":"2023-09-23T20:44:45.678543+0000","last_became_active":"2023-09-23T20:32:33.230503+0000","last_became_peered":"2023-09-23T20:32:33.230503+0000","last_unstale":"2023-09-23T20:44:45.678543+0000","last_undegraded":"2023-09-23T20:44:45.678543+0000","last_fullsized":"2023-09-23T20:44:45.678543+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:29.084410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":331,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-23T20:44:45.677028+0000","last_change":"2023-09-23T20:43:16.765253+0000","last_active":"2023-09-23T20:44:45.677028+0000","last_peered":"2023-09-23T20:44:45.677028+0000","last_clean":"2023-09-23T20:44:45.677028+0000","last_became_active":"2023-09-23T20:43:16.764978+0000","last_became_peered":"2023-09-23T20:43:16.764978+0000","last_unstale":"2023-09-23T20:44:45.677028+0000","last_undegraded":"2023-09-23T20:44:45.677028+0000","last_fullsized":"2023-09-23T20:44:45.677028+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:07:23.539372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.4","version":"517'12958","reported_seq":13030,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T20:44:48.472398+0000","last_change":"2023-09-23T20:41:04.575200+0000","last_active":"2023-09-23T20:44:48.472398+0000","last_peered":"2023-09-23T20:44:48.472398+0000","last_clean":"2023-09-23T20:44:48.472398+0000","last_became_active":"2023-09-23T20:41:04.574824+0000","last_became_peered":"2023-09-23T20:41:04.574824+0000","last_unstale":"2023-09-23T20:44:48.472398+0000","last_undegraded":"2023-09-23T20:44:48.472398+0000","last_fullsized":"2023-09-23T20:44:48.472398+0000","mapping_epoch":455,"log_start":"475'7383","ondisk_log_start":"475'7383","created":455,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":5575,"log_dups_size":0,"ondisk_log_size":5575,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:50:46.714383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":88604672,"num_objects":1352,"num_object_clones":0,"num_object_copies":4056,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1352,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12958,"num_write_kb":457920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.c","version":"90'161","reported_seq":778,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-23T20:44:45.674992+0000","last_change":"2023-09-23T20:41:09.506481+0000","last_active":"2023-09-23T20:44:45.674992+0000","last_peered":"2023-09-23T20:44:45.674992+0000","last_clean":"2023-09-23T20:44:45.674992+0000","last_became_active":"2023-09-23T20:38:28.442229+0000","last_became_peered":"2023-09-23T20:38:28.442229+0000","last_unstale":"2023-09-23T20:44:45.674992+0000","last_undegraded":"2023-09-23T20:44:45.674992+0000","last_fullsized":"2023-09-23T20:44:45.674992+0000","mapping_epoch":411,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":412,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:41:09.506338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:41:09.506338+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:19:46.265011+0000","scrub_duration":0.01112002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":555,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-23T20:44:45.681979+0000","last_change":"2023-09-23T20:34:59.987981+0000","last_active":"2023-09-23T20:44:45.681979+0000","last_peered":"2023-09-23T20:44:45.681979+0000","last_clean":"2023-09-23T20:44:45.681979+0000","last_became_active":"2023-09-23T20:34:59.985368+0000","last_became_peered":"2023-09-23T20:34:59.985368+0000","last_unstale":"2023-09-23T20:44:45.681979+0000","last_undegraded":"2023-09-23T20:44:45.681979+0000","last_fullsized":"2023-09-23T20:44:45.681979+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:12:43.082493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":214,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-23T20:44:45.684983+0000","last_change":"2023-09-23T20:32:33.232142+0000","last_active":"2023-09-23T20:44:45.684983+0000","last_peered":"2023-09-23T20:44:45.684983+0000","last_clean":"2023-09-23T20:44:45.684983+0000","last_became_active":"2023-09-23T20:32:33.231683+0000","last_became_peered":"2023-09-23T20:32:33.231683+0000","last_unstale":"2023-09-23T20:44:45.684983+0000","last_undegraded":"2023-09-23T20:44:45.684983+0000","last_fullsized":"2023-09-23T20:44:45.684983+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:02:58.336348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"517'12834","reported_seq":12916,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T20:44:47.525615+0000","last_change":"2023-09-23T20:43:16.854027+0000","last_active":"2023-09-23T20:44:47.525615+0000","last_peered":"2023-09-23T20:44:47.525615+0000","last_clean":"2023-09-23T20:44:47.525615+0000","last_became_active":"2023-09-23T20:43:16.766133+0000","last_became_peered":"2023-09-23T20:43:16.766133+0000","last_unstale":"2023-09-23T20:44:47.525615+0000","last_undegraded":"2023-09-23T20:44:47.525615+0000","last_fullsized":"2023-09-23T20:44:47.525615+0000","mapping_epoch":481,"log_start":"470'7085","ondisk_log_start":"470'7085","created":455,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":5749,"log_dups_size":0,"ondisk_log_size":5749,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:42:32.213805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":87949312,"num_objects":1342,"num_object_clones":0,"num_object_copies":4026,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1342,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12834,"num_write_kb":453632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.f","version":"90'161","reported_seq":672,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-23T20:44:45.674409+0000","last_change":"2023-09-23T20:38:40.491902+0000","last_active":"2023-09-23T20:44:45.674409+0000","last_peered":"2023-09-23T20:44:45.674409+0000","last_clean":"2023-09-23T20:44:45.674409+0000","last_became_active":"2023-09-23T20:38:40.491563+0000","last_became_peered":"2023-09-23T20:38:40.491563+0000","last_unstale":"2023-09-23T20:44:45.674409+0000","last_undegraded":"2023-09-23T20:44:45.674409+0000","last_fullsized":"2023-09-23T20:44:45.674409+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:46.217489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:46.217489+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:39:25.008796+0000","scrub_duration":0.001456429,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":545,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-23T20:44:45.677359+0000","last_change":"2023-09-23T20:40:19.677655+0000","last_active":"2023-09-23T20:44:45.677359+0000","last_peered":"2023-09-23T20:44:45.677359+0000","last_clean":"2023-09-23T20:44:45.677359+0000","last_became_active":"2023-09-23T20:40:19.551189+0000","last_became_peered":"2023-09-23T20:40:19.551189+0000","last_unstale":"2023-09-23T20:44:45.677359+0000","last_undegraded":"2023-09-23T20:44:45.677359+0000","last_fullsized":"2023-09-23T20:44:45.677359+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:11:01.559576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":122,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-23T20:44:45.673201+0000","last_change":"2023-09-23T20:38:02.590613+0000","last_active":"2023-09-23T20:44:45.673201+0000","last_peered":"2023-09-23T20:44:45.673201+0000","last_clean":"2023-09-23T20:44:45.673201+0000","last_became_active":"2023-09-23T20:38:02.590284+0000","last_became_peered":"2023-09-23T20:38:02.590284+0000","last_unstale":"2023-09-23T20:44:45.673201+0000","last_undegraded":"2023-09-23T20:44:45.673201+0000","last_fullsized":"2023-09-23T20:44:45.673201+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:40:07.825054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"517'12755","reported_seq":12833,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T20:44:48.074688+0000","last_change":"2023-09-23T20:43:16.788249+0000","last_active":"2023-09-23T20:44:48.074688+0000","last_peered":"2023-09-23T20:44:48.074688+0000","last_clean":"2023-09-23T20:44:48.074688+0000","last_became_active":"2023-09-23T20:43:16.763555+0000","last_became_peered":"2023-09-23T20:43:16.763555+0000","last_unstale":"2023-09-23T20:44:48.074688+0000","last_undegraded":"2023-09-23T20:44:48.074688+0000","last_fullsized":"2023-09-23T20:44:48.074688+0000","mapping_epoch":481,"log_start":"461'3004","ondisk_log_start":"461'3004","created":455,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":9751,"log_dups_size":0,"ondisk_log_size":9751,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:19:57.856019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":88801280,"num_objects":1355,"num_object_clones":0,"num_object_copies":4065,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1355,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12755,"num_write_kb":451520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[2,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.e","version":"90'161","reported_seq":617,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-23T20:44:45.691480+0000","last_change":"2023-09-23T20:36:00.322453+0000","last_active":"2023-09-23T20:44:45.691480+0000","last_peered":"2023-09-23T20:44:45.691480+0000","last_clean":"2023-09-23T20:44:45.691480+0000","last_became_active":"2023-09-23T20:35:02.233536+0000","last_became_peered":"2023-09-23T20:35:02.233536+0000","last_unstale":"2023-09-23T20:44:45.691480+0000","last_undegraded":"2023-09-23T20:44:45.691480+0000","last_fullsized":"2023-09-23T20:44:45.691480+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:00.322381+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:00.322381+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:02:15.489069+0000","scrub_duration":0.0017321859999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":476,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-23T20:44:45.675360+0000","last_change":"2023-09-23T20:43:16.756437+0000","last_active":"2023-09-23T20:44:45.675360+0000","last_peered":"2023-09-23T20:44:45.675360+0000","last_clean":"2023-09-23T20:44:45.675360+0000","last_became_active":"2023-09-23T20:43:16.756193+0000","last_became_peered":"2023-09-23T20:43:16.756193+0000","last_unstale":"2023-09-23T20:44:45.675360+0000","last_undegraded":"2023-09-23T20:44:45.675360+0000","last_fullsized":"2023-09-23T20:44:45.675360+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:33:25.632781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":489,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-09-23T20:44:44.665486+0000","last_change":"2023-09-23T20:43:46.541524+0000","last_active":"2023-09-23T20:44:44.665486+0000","last_peered":"2023-09-23T20:44:44.665486+0000","last_clean":"2023-09-23T20:44:44.665486+0000","last_became_active":"2023-09-23T20:43:46.541051+0000","last_became_peered":"2023-09-23T20:43:46.541051+0000","last_unstale":"2023-09-23T20:44:44.665486+0000","last_undegraded":"2023-09-23T20:44:44.665486+0000","last_fullsized":"2023-09-23T20:44:44.665486+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:59:36.632592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"517'12760","reported_seq":12830,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T20:44:48.201561+0000","last_change":"2023-09-23T20:41:04.558473+0000","last_active":"2023-09-23T20:44:48.201561+0000","last_peered":"2023-09-23T20:44:48.201561+0000","last_clean":"2023-09-23T20:44:48.201561+0000","last_became_active":"2023-09-23T20:41:04.558173+0000","last_became_peered":"2023-09-23T20:41:04.558173+0000","last_unstale":"2023-09-23T20:44:48.201561+0000","last_undegraded":"2023-09-23T20:44:48.201561+0000","last_fullsized":"2023-09-23T20:44:48.201561+0000","mapping_epoch":455,"log_start":"467'5763","ondisk_log_start":"467'5763","created":455,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":6997,"log_dups_size":0,"ondisk_log_size":6997,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:17:53.184877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":92012544,"num_objects":1404,"num_object_clones":0,"num_object_copies":4212,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1404,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12760,"num_write_kb":453248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":922,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-23T20:44:45.678286+0000","last_change":"2023-09-23T20:35:02.523308+0000","last_active":"2023-09-23T20:44:45.678286+0000","last_peered":"2023-09-23T20:44:45.678286+0000","last_clean":"2023-09-23T20:44:45.678286+0000","last_became_active":"2023-09-23T20:33:38.979989+0000","last_became_peered":"2023-09-23T20:33:38.979989+0000","last_unstale":"2023-09-23T20:44:45.678286+0000","last_undegraded":"2023-09-23T20:44:45.678286+0000","last_fullsized":"2023-09-23T20:44:45.678286+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:35:02.523203+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:35:02.523203+0000","last_clean_scrub_stamp":"2023-09-23T20:35:02.523203+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:02:41.472958+0000","scrub_duration":0.0085927989999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":338,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-23T20:44:45.674707+0000","last_change":"2023-09-23T20:37:43.583731+0000","last_active":"2023-09-23T20:44:45.674707+0000","last_peered":"2023-09-23T20:44:45.674707+0000","last_clean":"2023-09-23T20:44:45.674707+0000","last_became_active":"2023-09-23T20:37:43.583066+0000","last_became_peered":"2023-09-23T20:37:43.583066+0000","last_unstale":"2023-09-23T20:44:45.674707+0000","last_undegraded":"2023-09-23T20:44:45.674707+0000","last_fullsized":"2023-09-23T20:44:45.674707+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:03:45.211900+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":234,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-23T20:44:45.686095+0000","last_change":"2023-09-23T20:38:02.579814+0000","last_active":"2023-09-23T20:44:45.686095+0000","last_peered":"2023-09-23T20:44:45.686095+0000","last_clean":"2023-09-23T20:44:45.686095+0000","last_became_active":"2023-09-23T20:38:02.579253+0000","last_became_peered":"2023-09-23T20:38:02.579253+0000","last_unstale":"2023-09-23T20:44:45.686095+0000","last_undegraded":"2023-09-23T20:44:45.686095+0000","last_fullsized":"2023-09-23T20:44:45.686095+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:05:07.298198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"517'12902","reported_seq":12974,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T20:44:48.502116+0000","last_change":"2023-09-23T20:41:04.573101+0000","last_active":"2023-09-23T20:44:48.502116+0000","last_peered":"2023-09-23T20:44:48.502116+0000","last_clean":"2023-09-23T20:44:48.502116+0000","last_became_active":"2023-09-23T20:41:04.572728+0000","last_became_peered":"2023-09-23T20:41:04.572728+0000","last_unstale":"2023-09-23T20:44:48.502116+0000","last_undegraded":"2023-09-23T20:44:48.502116+0000","last_fullsized":"2023-09-23T20:44:48.502116+0000","mapping_epoch":455,"log_start":"475'7325","ondisk_log_start":"475'7325","created":455,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":5577,"log_dups_size":0,"ondisk_log_size":5577,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:21:10.721593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":89260032,"num_objects":1362,"num_object_clones":0,"num_object_copies":4086,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1362,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12902,"num_write_kb":456448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":631,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T20:44:46.725779+0000","last_change":"2023-09-23T20:44:46.725779+0000","last_active":"2023-09-23T20:44:46.725779+0000","last_peered":"2023-09-23T20:44:46.725779+0000","last_clean":"2023-09-23T20:44:46.725779+0000","last_became_active":"2023-09-23T20:44:46.725433+0000","last_became_peered":"2023-09-23T20:44:46.725433+0000","last_unstale":"2023-09-23T20:44:46.725779+0000","last_undegraded":"2023-09-23T20:44:46.725779+0000","last_fullsized":"2023-09-23T20:44:46.725779+0000","mapping_epoch":516,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":517,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:42:45.492220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:42:45.492220+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:28:35.202407+0000","scrub_duration":0.015259412,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":122,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-23T20:44:45.673251+0000","last_change":"2023-09-23T20:38:02.578703+0000","last_active":"2023-09-23T20:44:45.673251+0000","last_peered":"2023-09-23T20:44:45.673251+0000","last_clean":"2023-09-23T20:44:45.673251+0000","last_became_active":"2023-09-23T20:38:02.578414+0000","last_became_peered":"2023-09-23T20:38:02.578414+0000","last_unstale":"2023-09-23T20:44:45.673251+0000","last_undegraded":"2023-09-23T20:44:45.673251+0000","last_fullsized":"2023-09-23T20:44:45.673251+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:43:59.779295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":183,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-23T20:44:45.686081+0000","last_change":"2023-09-23T20:37:31.296606+0000","last_active":"2023-09-23T20:44:45.686081+0000","last_peered":"2023-09-23T20:44:45.686081+0000","last_clean":"2023-09-23T20:44:45.686081+0000","last_became_active":"2023-09-23T20:37:31.295924+0000","last_became_peered":"2023-09-23T20:37:31.295924+0000","last_unstale":"2023-09-23T20:44:45.686081+0000","last_undegraded":"2023-09-23T20:44:45.686081+0000","last_fullsized":"2023-09-23T20:44:45.686081+0000","mapping_epoch":391,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":392,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:27:50.948510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"517'12775","reported_seq":12845,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T20:44:47.487079+0000","last_change":"2023-09-23T20:41:04.555980+0000","last_active":"2023-09-23T20:44:47.487079+0000","last_peered":"2023-09-23T20:44:47.487079+0000","last_clean":"2023-09-23T20:44:47.487079+0000","last_became_active":"2023-09-23T20:41:04.555688+0000","last_became_peered":"2023-09-23T20:41:04.555688+0000","last_unstale":"2023-09-23T20:44:47.487079+0000","last_undegraded":"2023-09-23T20:44:47.487079+0000","last_fullsized":"2023-09-23T20:44:47.487079+0000","mapping_epoch":455,"log_start":"469'7025","ondisk_log_start":"469'7025","created":455,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":5750,"log_dups_size":0,"ondisk_log_size":5750,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:28:42.819404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":94699520,"num_objects":1445,"num_object_clones":0,"num_object_copies":4335,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1445,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12775,"num_write_kb":455040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.b","version":"90'161","reported_seq":820,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-23T20:44:45.681217+0000","last_change":"2023-09-23T20:40:22.445808+0000","last_active":"2023-09-23T20:44:45.681217+0000","last_peered":"2023-09-23T20:44:45.681217+0000","last_clean":"2023-09-23T20:44:45.681217+0000","last_became_active":"2023-09-23T20:40:22.445456+0000","last_became_peered":"2023-09-23T20:40:22.445456+0000","last_unstale":"2023-09-23T20:44:45.681217+0000","last_undegraded":"2023-09-23T20:44:45.681217+0000","last_fullsized":"2023-09-23T20:44:45.681217+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:05.595042+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:05.595042+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:39:23.678154+0000","scrub_duration":0.007271675,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":173,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-23T20:44:45.684747+0000","last_change":"2023-09-23T20:34:59.983122+0000","last_active":"2023-09-23T20:44:45.684747+0000","last_peered":"2023-09-23T20:44:45.684747+0000","last_clean":"2023-09-23T20:44:45.684747+0000","last_became_active":"2023-09-23T20:34:59.982809+0000","last_became_peered":"2023-09-23T20:34:59.982809+0000","last_unstale":"2023-09-23T20:44:45.684747+0000","last_undegraded":"2023-09-23T20:44:45.684747+0000","last_fullsized":"2023-09-23T20:44:45.684747+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:49:40.839723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":254,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-23T20:44:45.691699+0000","last_change":"2023-09-23T20:44:10.606995+0000","last_active":"2023-09-23T20:44:45.691699+0000","last_peered":"2023-09-23T20:44:45.691699+0000","last_clean":"2023-09-23T20:44:45.691699+0000","last_became_active":"2023-09-23T20:44:10.606718+0000","last_became_peered":"2023-09-23T20:44:10.606718+0000","last_unstale":"2023-09-23T20:44:45.691699+0000","last_undegraded":"2023-09-23T20:44:45.691699+0000","last_fullsized":"2023-09-23T20:44:45.691699+0000","mapping_epoch":496,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":497,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:13:12.496245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"517'12965","reported_seq":13037,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T20:44:49.295142+0000","last_change":"2023-09-23T20:41:04.573300+0000","last_active":"2023-09-23T20:44:49.295142+0000","last_peered":"2023-09-23T20:44:49.295142+0000","last_clean":"2023-09-23T20:44:49.295142+0000","last_became_active":"2023-09-23T20:41:04.572919+0000","last_became_peered":"2023-09-23T20:41:04.572919+0000","last_unstale":"2023-09-23T20:44:49.295142+0000","last_undegraded":"2023-09-23T20:44:49.295142+0000","last_fullsized":"2023-09-23T20:44:49.295142+0000","mapping_epoch":455,"log_start":"469'6894","ondisk_log_start":"469'6894","created":455,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":6071,"log_dups_size":0,"ondisk_log_size":6071,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:57:46.465243+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":88408064,"num_objects":1349,"num_object_clones":0,"num_object_copies":4047,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1349,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12965,"num_write_kb":458048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":542,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-23T20:44:45.670236+0000","last_change":"2023-09-23T20:43:51.693618+0000","last_active":"2023-09-23T20:44:45.670236+0000","last_peered":"2023-09-23T20:44:45.670236+0000","last_clean":"2023-09-23T20:44:45.670236+0000","last_became_active":"2023-09-23T20:43:16.765186+0000","last_became_peered":"2023-09-23T20:43:16.765186+0000","last_unstale":"2023-09-23T20:44:45.670236+0000","last_undegraded":"2023-09-23T20:44:45.670236+0000","last_fullsized":"2023-09-23T20:44:45.670236+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:43:51.693459+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:43:51.693459+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:33:30.208355+0000","scrub_duration":0.0016145269999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":312,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-23T20:44:45.675008+0000","last_change":"2023-09-23T20:37:43.570610+0000","last_active":"2023-09-23T20:44:45.675008+0000","last_peered":"2023-09-23T20:44:45.675008+0000","last_clean":"2023-09-23T20:44:45.675008+0000","last_became_active":"2023-09-23T20:37:43.570216+0000","last_became_peered":"2023-09-23T20:37:43.570216+0000","last_unstale":"2023-09-23T20:44:45.675008+0000","last_undegraded":"2023-09-23T20:44:45.675008+0000","last_fullsized":"2023-09-23T20:44:45.675008+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:39:26.555329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":342,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-23T20:44:45.680643+0000","last_change":"2023-09-23T20:32:33.230582+0000","last_active":"2023-09-23T20:44:45.680643+0000","last_peered":"2023-09-23T20:44:45.680643+0000","last_clean":"2023-09-23T20:44:45.680643+0000","last_became_active":"2023-09-23T20:32:33.230257+0000","last_became_peered":"2023-09-23T20:32:33.230257+0000","last_unstale":"2023-09-23T20:44:45.680643+0000","last_undegraded":"2023-09-23T20:44:45.680643+0000","last_fullsized":"2023-09-23T20:44:45.680643+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:11:45.333096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.f","version":"517'13033","reported_seq":13111,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T20:44:48.459101+0000","last_change":"2023-09-23T20:43:16.778725+0000","last_active":"2023-09-23T20:44:48.459101+0000","last_peered":"2023-09-23T20:44:48.459101+0000","last_clean":"2023-09-23T20:44:48.459101+0000","last_became_active":"2023-09-23T20:43:16.759443+0000","last_became_peered":"2023-09-23T20:43:16.759443+0000","last_unstale":"2023-09-23T20:44:48.459101+0000","last_undegraded":"2023-09-23T20:44:48.459101+0000","last_fullsized":"2023-09-23T20:44:48.459101+0000","mapping_epoch":481,"log_start":"474'7384","ondisk_log_start":"474'7384","created":455,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":5649,"log_dups_size":0,"ondisk_log_size":5649,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:55:32.141361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":88670208,"num_objects":1353,"num_object_clones":0,"num_object_copies":4059,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1353,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13033,"num_write_kb":460352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":705,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-23T20:44:45.674393+0000","last_change":"2023-09-23T20:41:07.424609+0000","last_active":"2023-09-23T20:44:45.674393+0000","last_peered":"2023-09-23T20:44:45.674393+0000","last_clean":"2023-09-23T20:44:45.674393+0000","last_became_active":"2023-09-23T20:38:05.622261+0000","last_became_peered":"2023-09-23T20:38:05.622261+0000","last_unstale":"2023-09-23T20:44:45.674393+0000","last_undegraded":"2023-09-23T20:44:45.674393+0000","last_fullsized":"2023-09-23T20:44:45.674393+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:41:07.424492+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:41:07.424492+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:45:21.689123+0000","scrub_duration":0.0018960959999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"86'119","reported_seq":827,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-23T20:44:45.681091+0000","last_change":"2023-09-23T20:29:45.521257+0000","last_active":"2023-09-23T20:44:45.681091+0000","last_peered":"2023-09-23T20:44:45.681091+0000","last_clean":"2023-09-23T20:44:45.681091+0000","last_became_active":"2023-09-23T20:29:45.520546+0000","last_became_peered":"2023-09-23T20:29:45.520546+0000","last_unstale":"2023-09-23T20:44:45.681091+0000","last_undegraded":"2023-09-23T20:44:45.681091+0000","last_fullsized":"2023-09-23T20:44:45.681091+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":119,"log_dups_size":0,"ondisk_log_size":119,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:58:56.085777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032705499999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":523,"num_read_kb":3503,"num_write":136,"num_write_kb":3128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":309,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-23T20:44:45.674083+0000","last_change":"2023-09-23T20:43:16.756494+0000","last_active":"2023-09-23T20:44:45.674083+0000","last_peered":"2023-09-23T20:44:45.674083+0000","last_clean":"2023-09-23T20:44:45.674083+0000","last_became_active":"2023-09-23T20:43:16.756131+0000","last_became_peered":"2023-09-23T20:43:16.756131+0000","last_unstale":"2023-09-23T20:44:45.674083+0000","last_undegraded":"2023-09-23T20:44:45.674083+0000","last_fullsized":"2023-09-23T20:44:45.674083+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:10:19.219533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":474,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-23T20:44:45.687225+0000","last_change":"2023-09-23T20:40:19.530942+0000","last_active":"2023-09-23T20:44:45.687225+0000","last_peered":"2023-09-23T20:44:45.687225+0000","last_clean":"2023-09-23T20:44:45.687225+0000","last_became_active":"2023-09-23T20:40:19.530617+0000","last_became_peered":"2023-09-23T20:40:19.530617+0000","last_unstale":"2023-09-23T20:44:45.687225+0000","last_undegraded":"2023-09-23T20:44:45.687225+0000","last_fullsized":"2023-09-23T20:44:45.687225+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:24:09.231934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.9","version":"516'12765","reported_seq":12851,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-23T20:44:46.291827+0000","last_change":"2023-09-23T20:43:46.544365+0000","last_active":"2023-09-23T20:44:46.291827+0000","last_peered":"2023-09-23T20:44:46.291827+0000","last_clean":"2023-09-23T20:44:46.291827+0000","last_became_active":"2023-09-23T20:43:46.519248+0000","last_became_peered":"2023-09-23T20:43:46.519248+0000","last_unstale":"2023-09-23T20:44:46.291827+0000","last_undegraded":"2023-09-23T20:44:46.291827+0000","last_fullsized":"2023-09-23T20:44:46.291827+0000","mapping_epoch":487,"log_start":"466'5234","ondisk_log_start":"466'5234","created":455,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":7531,"log_dups_size":0,"ondisk_log_size":7531,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:49:43.541062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":93519872,"num_objects":1427,"num_object_clones":0,"num_object_copies":4281,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1427,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12765,"num_write_kb":454144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":561,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-09-23T20:44:44.664772+0000","last_change":"2023-09-23T20:44:15.656984+0000","last_active":"2023-09-23T20:44:44.664772+0000","last_peered":"2023-09-23T20:44:44.664772+0000","last_clean":"2023-09-23T20:44:44.664772+0000","last_became_active":"2023-09-23T20:43:46.512210+0000","last_became_peered":"2023-09-23T20:43:46.512210+0000","last_unstale":"2023-09-23T20:44:44.664772+0000","last_undegraded":"2023-09-23T20:44:44.664772+0000","last_fullsized":"2023-09-23T20:44:44.664772+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:15.656880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:15.656880+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:37:56.453466+0000","scrub_duration":0.0016711180000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"86'106","reported_seq":767,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-23T20:44:45.687446+0000","last_change":"2023-09-23T20:43:16.759138+0000","last_active":"2023-09-23T20:44:45.687446+0000","last_peered":"2023-09-23T20:44:45.687446+0000","last_clean":"2023-09-23T20:44:45.687446+0000","last_became_active":"2023-09-23T20:43:16.758015+0000","last_became_peered":"2023-09-23T20:43:16.758015+0000","last_unstale":"2023-09-23T20:44:45.687446+0000","last_undegraded":"2023-09-23T20:44:45.687446+0000","last_fullsized":"2023-09-23T20:44:45.687446+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":106,"log_dups_size":0,"ondisk_log_size":106,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:08:02.852472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00099302599999999993,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1094,"num_read_kb":1678,"num_write":100,"num_write_kb":404,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":234,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-23T20:44:45.687436+0000","last_change":"2023-09-23T20:38:02.552008+0000","last_active":"2023-09-23T20:44:45.687436+0000","last_peered":"2023-09-23T20:44:45.687436+0000","last_clean":"2023-09-23T20:44:45.687436+0000","last_became_active":"2023-09-23T20:38:02.551684+0000","last_became_peered":"2023-09-23T20:38:02.551684+0000","last_unstale":"2023-09-23T20:44:45.687436+0000","last_undegraded":"2023-09-23T20:44:45.687436+0000","last_fullsized":"2023-09-23T20:44:45.687436+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:40:37.147538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":480,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-23T20:44:45.681047+0000","last_change":"2023-09-23T20:29:45.533406+0000","last_active":"2023-09-23T20:44:45.681047+0000","last_peered":"2023-09-23T20:44:45.681047+0000","last_clean":"2023-09-23T20:44:45.681047+0000","last_became_active":"2023-09-23T20:29:45.526001+0000","last_became_peered":"2023-09-23T20:29:45.526001+0000","last_unstale":"2023-09-23T20:44:45.681047+0000","last_undegraded":"2023-09-23T20:44:45.681047+0000","last_fullsized":"2023-09-23T20:44:45.681047+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:43.787251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.a","version":"517'12837","reported_seq":12918,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T20:44:48.439704+0000","last_change":"2023-09-23T20:43:16.852909+0000","last_active":"2023-09-23T20:44:48.439704+0000","last_peered":"2023-09-23T20:44:48.439704+0000","last_clean":"2023-09-23T20:44:48.439704+0000","last_became_active":"2023-09-23T20:43:16.770100+0000","last_became_peered":"2023-09-23T20:43:16.770100+0000","last_unstale":"2023-09-23T20:44:48.439704+0000","last_undegraded":"2023-09-23T20:44:48.439704+0000","last_fullsized":"2023-09-23T20:44:48.439704+0000","mapping_epoch":481,"log_start":"473'7206","ondisk_log_start":"473'7206","created":455,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":5631,"log_dups_size":0,"ondisk_log_size":5631,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:13:16.136666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":89325568,"num_objects":1363,"num_object_clones":0,"num_object_copies":4089,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1363,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12837,"num_write_kb":454400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,7],"acting":[1,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":480,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-23T20:44:45.676834+0000","last_change":"2023-09-23T20:44:26.978243+0000","last_active":"2023-09-23T20:44:45.676834+0000","last_peered":"2023-09-23T20:44:45.676834+0000","last_clean":"2023-09-23T20:44:45.676834+0000","last_became_active":"2023-09-23T20:44:18.638989+0000","last_became_peered":"2023-09-23T20:44:18.638989+0000","last_unstale":"2023-09-23T20:44:45.676834+0000","last_undegraded":"2023-09-23T20:44:45.676834+0000","last_fullsized":"2023-09-23T20:44:45.676834+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:26.978179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:26.978179+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:25:38.977011+0000","scrub_duration":0.0013962829999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"86'130","reported_seq":835,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-23T20:44:45.669336+0000","last_change":"2023-09-23T20:25:39.637458+0000","last_active":"2023-09-23T20:44:45.669336+0000","last_peered":"2023-09-23T20:44:45.669336+0000","last_clean":"2023-09-23T20:44:45.669336+0000","last_became_active":"2023-09-23T20:25:39.636773+0000","last_became_peered":"2023-09-23T20:25:39.636773+0000","last_unstale":"2023-09-23T20:44:45.669336+0000","last_undegraded":"2023-09-23T20:44:45.669336+0000","last_fullsized":"2023-09-23T20:44:45.669336+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:00:32.810564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011300049999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":512,"num_read_kb":1820,"num_write":133,"num_write_kb":1347,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":488,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-23T20:44:45.686022+0000","last_change":"2023-09-23T20:38:02.591335+0000","last_active":"2023-09-23T20:44:45.686022+0000","last_peered":"2023-09-23T20:44:45.686022+0000","last_clean":"2023-09-23T20:44:45.686022+0000","last_became_active":"2023-09-23T20:38:02.591015+0000","last_became_peered":"2023-09-23T20:38:02.591015+0000","last_unstale":"2023-09-23T20:44:45.686022+0000","last_undegraded":"2023-09-23T20:44:45.686022+0000","last_fullsized":"2023-09-23T20:44:45.686022+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:55:32.272195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":31,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-23T20:44:45.679956+0000","last_change":"2023-09-23T20:44:10.555583+0000","last_active":"2023-09-23T20:44:45.679956+0000","last_peered":"2023-09-23T20:44:45.679956+0000","last_clean":"2023-09-23T20:44:45.679956+0000","last_became_active":"2023-09-23T20:44:10.555281+0000","last_became_peered":"2023-09-23T20:44:10.555281+0000","last_unstale":"2023-09-23T20:44:45.679956+0000","last_undegraded":"2023-09-23T20:44:45.679956+0000","last_fullsized":"2023-09-23T20:44:45.679956+0000","mapping_epoch":496,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":497,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:43:11.942147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.8","version":"517'13119","reported_seq":30078,"reported_epoch":517,"state":"active+remapped+backfilling","last_fresh":"2023-09-23T20:44:49.127075+0000","last_change":"2023-09-23T20:44:45.164739+0000","last_active":"2023-09-23T20:44:49.127075+0000","last_peered":"2023-09-23T20:44:49.127075+0000","last_clean":"2023-09-23T20:44:42.651025+0000","last_became_active":"2023-09-23T20:44:45.078718+0000","last_became_peered":"2023-09-23T20:44:45.078718+0000","last_unstale":"2023-09-23T20:44:49.127075+0000","last_undegraded":"2023-09-23T20:44:49.127075+0000","last_fullsized":"2023-09-23T20:44:49.127075+0000","mapping_epoch":514,"log_start":"469'7027","ondisk_log_start":"469'7027","created":455,"last_epoch_clean":509,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":6092,"log_dups_size":0,"ondisk_log_size":6092,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:13:18.812327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90505216,"num_objects":1381,"num_object_clones":0,"num_object_copies":4143,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":329,"num_objects_unfound":0,"num_objects_dirty":1381,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13119,"num_write_kb":464000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5858,"num_bytes_recovered":383844352,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,7],"avail_no_missing":["3","0","1","7"],"object_location_counts":[{"shards":"0,3,7","objects":1381}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"86'194","reported_seq":873,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-23T20:44:45.669911+0000","last_change":"2023-09-23T20:43:16.765865+0000","last_active":"2023-09-23T20:44:45.669911+0000","last_peered":"2023-09-23T20:44:45.669911+0000","last_clean":"2023-09-23T20:44:45.669911+0000","last_became_active":"2023-09-23T20:43:16.765289+0000","last_became_peered":"2023-09-23T20:43:16.765289+0000","last_unstale":"2023-09-23T20:44:45.669911+0000","last_undegraded":"2023-09-23T20:44:45.669911+0000","last_fullsized":"2023-09-23T20:44:45.669911+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":194,"log_dups_size":0,"ondisk_log_size":194,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:15:16.461900+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00037995999999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1024,"num_read_kb":1200,"num_write":173,"num_write_kb":1276,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":979,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-23T20:44:45.676890+0000","last_change":"2023-09-23T20:38:05.602508+0000","last_active":"2023-09-23T20:44:45.676890+0000","last_peered":"2023-09-23T20:44:45.676890+0000","last_clean":"2023-09-23T20:44:45.676890+0000","last_became_active":"2023-09-23T20:38:05.602129+0000","last_became_peered":"2023-09-23T20:38:05.602129+0000","last_unstale":"2023-09-23T20:44:45.676890+0000","last_undegraded":"2023-09-23T20:44:45.676890+0000","last_fullsized":"2023-09-23T20:44:45.676890+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:45.343742+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:45.343742+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:48.910343+0000","scrub_duration":0.001475802,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":92,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-23T20:44:45.673288+0000","last_change":"2023-09-23T20:43:16.758213+0000","last_active":"2023-09-23T20:44:45.673288+0000","last_peered":"2023-09-23T20:44:45.673288+0000","last_clean":"2023-09-23T20:44:45.673288+0000","last_became_active":"2023-09-23T20:43:16.757869+0000","last_became_peered":"2023-09-23T20:43:16.757869+0000","last_unstale":"2023-09-23T20:44:45.673288+0000","last_undegraded":"2023-09-23T20:44:45.673288+0000","last_fullsized":"2023-09-23T20:44:45.673288+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:43:10.811749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":122,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-23T20:44:45.677546+0000","last_change":"2023-09-23T20:38:02.573200+0000","last_active":"2023-09-23T20:44:45.677546+0000","last_peered":"2023-09-23T20:44:45.677546+0000","last_clean":"2023-09-23T20:44:45.677546+0000","last_became_active":"2023-09-23T20:38:02.572945+0000","last_became_peered":"2023-09-23T20:38:02.572945+0000","last_unstale":"2023-09-23T20:44:45.677546+0000","last_undegraded":"2023-09-23T20:44:45.677546+0000","last_fullsized":"2023-09-23T20:44:45.677546+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:57.246533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.b","version":"516'12540","reported_seq":12627,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-23T20:44:46.352722+0000","last_change":"2023-09-23T20:43:46.540991+0000","last_active":"2023-09-23T20:44:46.352722+0000","last_peered":"2023-09-23T20:44:46.352722+0000","last_clean":"2023-09-23T20:44:46.352722+0000","last_became_active":"2023-09-23T20:43:46.512772+0000","last_became_peered":"2023-09-23T20:43:46.512772+0000","last_unstale":"2023-09-23T20:44:46.352722+0000","last_undegraded":"2023-09-23T20:44:46.352722+0000","last_fullsized":"2023-09-23T20:44:46.352722+0000","mapping_epoch":487,"log_start":"466'5034","ondisk_log_start":"466'5034","created":455,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":7506,"log_dups_size":0,"ondisk_log_size":7506,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:46:02.578833+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":101187584,"num_objects":1544,"num_object_clones":0,"num_object_copies":4632,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1544,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12540,"num_write_kb":450688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":849,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-23T20:44:45.680360+0000","last_change":"2023-09-23T20:38:05.615711+0000","last_active":"2023-09-23T20:44:45.680360+0000","last_peered":"2023-09-23T20:44:45.680360+0000","last_clean":"2023-09-23T20:44:45.680360+0000","last_became_active":"2023-09-23T20:38:05.615390+0000","last_became_peered":"2023-09-23T20:38:05.615390+0000","last_unstale":"2023-09-23T20:44:45.680360+0000","last_undegraded":"2023-09-23T20:44:45.680360+0000","last_fullsized":"2023-09-23T20:44:45.680360+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:43.582146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:43.582146+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:45:11.917871+0000","scrub_duration":0.0016219730000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"86'80","reported_seq":769,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-23T20:44:45.676684+0000","last_change":"2023-09-23T20:29:45.536796+0000","last_active":"2023-09-23T20:44:45.676684+0000","last_peered":"2023-09-23T20:44:45.676684+0000","last_clean":"2023-09-23T20:44:45.676684+0000","last_became_active":"2023-09-23T20:29:45.536439+0000","last_became_peered":"2023-09-23T20:29:45.536439+0000","last_unstale":"2023-09-23T20:44:45.676684+0000","last_undegraded":"2023-09-23T20:44:45.676684+0000","last_fullsized":"2023-09-23T20:44:45.676684+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":80,"log_dups_size":0,"ondisk_log_size":80,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:30:57.793132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035115,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":395,"num_read_kb":1370,"num_write":66,"num_write_kb":1061,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":508,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-09-23T20:44:44.664563+0000","last_change":"2023-09-23T20:43:46.544563+0000","last_active":"2023-09-23T20:44:44.664563+0000","last_peered":"2023-09-23T20:44:44.664563+0000","last_clean":"2023-09-23T20:44:44.664563+0000","last_became_active":"2023-09-23T20:43:46.543958+0000","last_became_peered":"2023-09-23T20:43:46.543958+0000","last_unstale":"2023-09-23T20:44:44.664563+0000","last_undegraded":"2023-09-23T20:44:44.664563+0000","last_fullsized":"2023-09-23T20:44:44.664563+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:46:36.921752+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":503,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-09-23T20:44:44.664631+0000","last_change":"2023-09-23T20:43:46.520386+0000","last_active":"2023-09-23T20:44:44.664631+0000","last_peered":"2023-09-23T20:44:44.664631+0000","last_clean":"2023-09-23T20:44:44.664631+0000","last_became_active":"2023-09-23T20:43:46.519757+0000","last_became_peered":"2023-09-23T20:43:46.519757+0000","last_unstale":"2023-09-23T20:44:44.664631+0000","last_undegraded":"2023-09-23T20:44:44.664631+0000","last_fullsized":"2023-09-23T20:44:44.664631+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:23:14.662734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.e","version":"517'12803","reported_seq":12873,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T20:44:47.538994+0000","last_change":"2023-09-23T20:41:04.574876+0000","last_active":"2023-09-23T20:44:47.538994+0000","last_peered":"2023-09-23T20:44:47.538994+0000","last_clean":"2023-09-23T20:44:47.538994+0000","last_became_active":"2023-09-23T20:41:04.574595+0000","last_became_peered":"2023-09-23T20:41:04.574595+0000","last_unstale":"2023-09-23T20:44:47.538994+0000","last_undegraded":"2023-09-23T20:44:47.538994+0000","last_fullsized":"2023-09-23T20:44:47.538994+0000","mapping_epoch":455,"log_start":"470'7108","ondisk_log_start":"470'7108","created":455,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":5695,"log_dups_size":0,"ondisk_log_size":5695,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:31:00.147966+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":92864512,"num_objects":1417,"num_object_clones":0,"num_object_copies":4251,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1417,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12803,"num_write_kb":455040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":628,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-23T20:44:45.673746+0000","last_change":"2023-09-23T20:41:08.380084+0000","last_active":"2023-09-23T20:44:45.673746+0000","last_peered":"2023-09-23T20:44:45.673746+0000","last_clean":"2023-09-23T20:44:45.673746+0000","last_became_active":"2023-09-23T20:38:05.610881+0000","last_became_peered":"2023-09-23T20:38:05.610881+0000","last_unstale":"2023-09-23T20:44:45.673746+0000","last_undegraded":"2023-09-23T20:44:45.673746+0000","last_fullsized":"2023-09-23T20:44:45.673746+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:41:08.379959+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:41:08.379959+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:19:39.547923+0000","scrub_duration":0.0019158420000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"86'102","reported_seq":1111,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-23T20:44:45.677433+0000","last_change":"2023-09-23T20:32:36.314672+0000","last_active":"2023-09-23T20:44:45.677433+0000","last_peered":"2023-09-23T20:44:45.677433+0000","last_clean":"2023-09-23T20:44:45.677433+0000","last_became_active":"2023-09-23T20:32:36.314231+0000","last_became_peered":"2023-09-23T20:32:36.314231+0000","last_unstale":"2023-09-23T20:44:45.677433+0000","last_undegraded":"2023-09-23T20:44:45.677433+0000","last_fullsized":"2023-09-23T20:44:45.677433+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":102,"log_dups_size":0,"ondisk_log_size":102,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:11:40.514709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030656500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":963,"num_read_kb":1234,"num_write":60,"num_write_kb":225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":464,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-23T20:44:45.681126+0000","last_change":"2023-09-23T20:40:19.544384+0000","last_active":"2023-09-23T20:44:45.681126+0000","last_peered":"2023-09-23T20:44:45.681126+0000","last_clean":"2023-09-23T20:44:45.681126+0000","last_became_active":"2023-09-23T20:40:19.544079+0000","last_became_peered":"2023-09-23T20:40:19.544079+0000","last_unstale":"2023-09-23T20:44:45.681126+0000","last_undegraded":"2023-09-23T20:44:45.681126+0000","last_fullsized":"2023-09-23T20:44:45.681126+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:00:19.498631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":458,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-09-23T20:44:44.665630+0000","last_change":"2023-09-23T20:43:46.541173+0000","last_active":"2023-09-23T20:44:44.665630+0000","last_peered":"2023-09-23T20:44:44.665630+0000","last_clean":"2023-09-23T20:44:44.665630+0000","last_became_active":"2023-09-23T20:43:46.540726+0000","last_became_peered":"2023-09-23T20:43:46.540726+0000","last_unstale":"2023-09-23T20:44:44.665630+0000","last_undegraded":"2023-09-23T20:44:44.665630+0000","last_fullsized":"2023-09-23T20:44:44.665630+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:49:04.137885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"517'12809","reported_seq":26575,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T20:44:47.496204+0000","last_change":"2023-09-23T20:44:18.688723+0000","last_active":"2023-09-23T20:44:47.496204+0000","last_peered":"2023-09-23T20:44:47.496204+0000","last_clean":"2023-09-23T20:44:47.496204+0000","last_became_active":"2023-09-23T20:44:18.636162+0000","last_became_peered":"2023-09-23T20:44:18.636162+0000","last_unstale":"2023-09-23T20:44:47.496204+0000","last_undegraded":"2023-09-23T20:44:47.496204+0000","last_fullsized":"2023-09-23T20:44:47.496204+0000","mapping_epoch":502,"log_start":"470'7148","ondisk_log_start":"470'7148","created":455,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":5661,"log_dups_size":0,"ondisk_log_size":5661,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:49:45.956725+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":96731160,"num_objects":1477,"num_object_clones":0,"num_object_copies":4431,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1477,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":12809,"num_write_kb":457089,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6816,"num_bytes_recovered":446627864,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":795,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-23T20:44:45.686639+0000","last_change":"2023-09-23T20:38:05.590361+0000","last_active":"2023-09-23T20:44:45.686639+0000","last_peered":"2023-09-23T20:44:45.686639+0000","last_clean":"2023-09-23T20:44:45.686639+0000","last_became_active":"2023-09-23T20:38:05.589985+0000","last_became_peered":"2023-09-23T20:38:05.589985+0000","last_unstale":"2023-09-23T20:44:45.686639+0000","last_undegraded":"2023-09-23T20:44:45.686639+0000","last_fullsized":"2023-09-23T20:44:45.686639+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:03.246271+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:03.246271+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:38:54.351576+0000","scrub_duration":0.0019016700000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":254,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-23T20:44:45.678707+0000","last_change":"2023-09-23T20:32:50.549695+0000","last_active":"2023-09-23T20:44:45.678707+0000","last_peered":"2023-09-23T20:44:45.678707+0000","last_clean":"2023-09-23T20:44:45.678707+0000","last_became_active":"2023-09-23T20:32:50.549431+0000","last_became_peered":"2023-09-23T20:32:50.549431+0000","last_unstale":"2023-09-23T20:44:45.678707+0000","last_undegraded":"2023-09-23T20:44:45.678707+0000","last_fullsized":"2023-09-23T20:44:45.678707+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:45:53.666569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"86'128","reported_seq":929,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-23T20:44:45.681161+0000","last_change":"2023-09-23T20:38:03.541765+0000","last_active":"2023-09-23T20:44:45.681161+0000","last_peered":"2023-09-23T20:44:45.681161+0000","last_clean":"2023-09-23T20:44:45.681161+0000","last_became_active":"2023-09-23T20:38:03.541277+0000","last_became_peered":"2023-09-23T20:38:03.541277+0000","last_unstale":"2023-09-23T20:44:45.681161+0000","last_undegraded":"2023-09-23T20:44:45.681161+0000","last_fullsized":"2023-09-23T20:44:45.681161+0000","mapping_epoch":404,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":405,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:20:53.834554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000293202,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":579,"num_read_kb":1473,"num_write":96,"num_write_kb":1062,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":297,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-23T20:44:45.674920+0000","last_change":"2023-09-23T20:37:43.582105+0000","last_active":"2023-09-23T20:44:45.674920+0000","last_peered":"2023-09-23T20:44:45.674920+0000","last_clean":"2023-09-23T20:44:45.674920+0000","last_became_active":"2023-09-23T20:37:43.581860+0000","last_became_peered":"2023-09-23T20:37:43.581860+0000","last_unstale":"2023-09-23T20:44:45.674920+0000","last_undegraded":"2023-09-23T20:44:45.674920+0000","last_fullsized":"2023-09-23T20:44:45.674920+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:18:38.765486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.c","version":"517'12921","reported_seq":12993,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T20:44:49.346592+0000","last_change":"2023-09-23T20:41:04.575496+0000","last_active":"2023-09-23T20:44:49.346592+0000","last_peered":"2023-09-23T20:44:49.346592+0000","last_clean":"2023-09-23T20:44:49.346592+0000","last_became_active":"2023-09-23T20:41:04.575242+0000","last_became_peered":"2023-09-23T20:41:04.575242+0000","last_unstale":"2023-09-23T20:44:49.346592+0000","last_undegraded":"2023-09-23T20:44:49.346592+0000","last_fullsized":"2023-09-23T20:44:49.346592+0000","mapping_epoch":455,"log_start":"469'6875","ondisk_log_start":"469'6875","created":455,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":6046,"log_dups_size":0,"ondisk_log_size":6046,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:30:14.611529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":87228416,"num_objects":1331,"num_object_clones":0,"num_object_copies":3993,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1331,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12921,"num_write_kb":456064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"90'161","reported_seq":731,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-23T20:44:45.675153+0000","last_change":"2023-09-23T20:40:22.427435+0000","last_active":"2023-09-23T20:44:45.675153+0000","last_peered":"2023-09-23T20:44:45.675153+0000","last_clean":"2023-09-23T20:44:45.675153+0000","last_became_active":"2023-09-23T20:40:22.427051+0000","last_became_peered":"2023-09-23T20:40:22.427051+0000","last_unstale":"2023-09-23T20:44:45.675153+0000","last_undegraded":"2023-09-23T20:44:45.675153+0000","last_fullsized":"2023-09-23T20:44:45.675153+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:34:54.504554+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:34:54.504554+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:42.001652+0000","scrub_duration":0.011176864,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"86'117","reported_seq":825,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-23T20:44:45.674346+0000","last_change":"2023-09-23T20:44:18.635224+0000","last_active":"2023-09-23T20:44:45.674346+0000","last_peered":"2023-09-23T20:44:45.674346+0000","last_clean":"2023-09-23T20:44:45.674346+0000","last_became_active":"2023-09-23T20:44:18.634703+0000","last_became_peered":"2023-09-23T20:44:18.634703+0000","last_unstale":"2023-09-23T20:44:45.674346+0000","last_undegraded":"2023-09-23T20:44:45.674346+0000","last_fullsized":"2023-09-23T20:44:45.674346+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:59:38.030626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046891400000000002,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":893,"num_read_kb":922,"num_write":82,"num_write_kb":4193,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":16777216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":503,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-09-23T20:44:44.664720+0000","last_change":"2023-09-23T20:43:46.521505+0000","last_active":"2023-09-23T20:44:44.664720+0000","last_peered":"2023-09-23T20:44:44.664720+0000","last_clean":"2023-09-23T20:44:44.664720+0000","last_became_active":"2023-09-23T20:43:46.520727+0000","last_became_peered":"2023-09-23T20:43:46.520727+0000","last_unstale":"2023-09-23T20:44:44.664720+0000","last_undegraded":"2023-09-23T20:44:44.664720+0000","last_fullsized":"2023-09-23T20:44:44.664720+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:17:06.164741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":254,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-23T20:44:45.684918+0000","last_change":"2023-09-23T20:40:19.530602+0000","last_active":"2023-09-23T20:44:45.684918+0000","last_peered":"2023-09-23T20:44:45.684918+0000","last_clean":"2023-09-23T20:44:45.684918+0000","last_became_active":"2023-09-23T20:40:19.530256+0000","last_became_peered":"2023-09-23T20:40:19.530256+0000","last_unstale":"2023-09-23T20:44:45.684918+0000","last_undegraded":"2023-09-23T20:44:45.684918+0000","last_fullsized":"2023-09-23T20:44:45.684918+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:18:45.658867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"90'161","reported_seq":772,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-09-23T20:44:44.664389+0000","last_change":"2023-09-23T20:43:46.544750+0000","last_active":"2023-09-23T20:44:44.664389+0000","last_peered":"2023-09-23T20:44:44.664389+0000","last_clean":"2023-09-23T20:44:44.664389+0000","last_became_active":"2023-09-23T20:43:46.544395+0000","last_became_peered":"2023-09-23T20:43:46.544395+0000","last_unstale":"2023-09-23T20:44:44.664389+0000","last_undegraded":"2023-09-23T20:44:44.664389+0000","last_fullsized":"2023-09-23T20:44:44.664389+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:06.487351+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:06.487351+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:56.866168+0000","scrub_duration":0.001964379,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":470,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-09-23T20:44:44.665589+0000","last_change":"2023-09-23T20:43:46.541379+0000","last_active":"2023-09-23T20:44:44.665589+0000","last_peered":"2023-09-23T20:44:44.665589+0000","last_clean":"2023-09-23T20:44:44.665589+0000","last_became_active":"2023-09-23T20:43:46.540732+0000","last_became_peered":"2023-09-23T20:43:46.540732+0000","last_unstale":"2023-09-23T20:44:44.665589+0000","last_undegraded":"2023-09-23T20:44:44.665589+0000","last_fullsized":"2023-09-23T20:44:44.665589+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:31:40.605772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":373,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-23T20:44:45.670152+0000","last_change":"2023-09-23T20:29:45.538314+0000","last_active":"2023-09-23T20:44:45.670152+0000","last_peered":"2023-09-23T20:44:45.670152+0000","last_clean":"2023-09-23T20:44:45.670152+0000","last_became_active":"2023-09-23T20:29:45.536847+0000","last_became_peered":"2023-09-23T20:29:45.536847+0000","last_unstale":"2023-09-23T20:44:45.670152+0000","last_undegraded":"2023-09-23T20:44:45.670152+0000","last_fullsized":"2023-09-23T20:44:45.670152+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:51.948721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"90'161","reported_seq":590,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-23T20:44:45.673930+0000","last_change":"2023-09-23T20:41:06.499511+0000","last_active":"2023-09-23T20:44:45.673930+0000","last_peered":"2023-09-23T20:44:45.673930+0000","last_clean":"2023-09-23T20:44:45.673930+0000","last_became_active":"2023-09-23T20:38:05.601874+0000","last_became_peered":"2023-09-23T20:38:05.601874+0000","last_unstale":"2023-09-23T20:44:45.673930+0000","last_undegraded":"2023-09-23T20:44:45.673930+0000","last_fullsized":"2023-09-23T20:44:45.673930+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:41:06.499396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:41:06.499396+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:31:34.059077+0000","scrub_duration":0.0019670310000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":409,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-23T20:44:45.686255+0000","last_change":"2023-09-23T20:43:16.758761+0000","last_active":"2023-09-23T20:44:45.686255+0000","last_peered":"2023-09-23T20:44:45.686255+0000","last_clean":"2023-09-23T20:44:45.686255+0000","last_became_active":"2023-09-23T20:43:16.758407+0000","last_became_peered":"2023-09-23T20:43:16.758407+0000","last_unstale":"2023-09-23T20:44:45.686255+0000","last_undegraded":"2023-09-23T20:44:45.686255+0000","last_fullsized":"2023-09-23T20:44:45.686255+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:59:27.070685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":173,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-23T20:44:45.687135+0000","last_change":"2023-09-23T20:34:59.987575+0000","last_active":"2023-09-23T20:44:45.687135+0000","last_peered":"2023-09-23T20:44:45.687135+0000","last_clean":"2023-09-23T20:44:45.687135+0000","last_became_active":"2023-09-23T20:34:59.987229+0000","last_became_peered":"2023-09-23T20:34:59.987229+0000","last_unstale":"2023-09-23T20:44:45.687135+0000","last_undegraded":"2023-09-23T20:44:45.687135+0000","last_fullsized":"2023-09-23T20:44:45.687135+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:34:01.820159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":417,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-23T20:44:45.686234+0000","last_change":"2023-09-23T20:43:16.759691+0000","last_active":"2023-09-23T20:44:45.686234+0000","last_peered":"2023-09-23T20:44:45.686234+0000","last_clean":"2023-09-23T20:44:45.686234+0000","last_became_active":"2023-09-23T20:43:16.759309+0000","last_became_peered":"2023-09-23T20:43:16.759309+0000","last_unstale":"2023-09-23T20:44:45.686234+0000","last_undegraded":"2023-09-23T20:44:45.686234+0000","last_fullsized":"2023-09-23T20:44:45.686234+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:58:52.893019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":173,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-23T20:44:45.685873+0000","last_change":"2023-09-23T20:34:59.987416+0000","last_active":"2023-09-23T20:44:45.685873+0000","last_peered":"2023-09-23T20:44:45.685873+0000","last_clean":"2023-09-23T20:44:45.685873+0000","last_became_active":"2023-09-23T20:34:59.987117+0000","last_became_peered":"2023-09-23T20:34:59.987117+0000","last_unstale":"2023-09-23T20:44:45.685873+0000","last_undegraded":"2023-09-23T20:44:45.685873+0000","last_fullsized":"2023-09-23T20:44:45.685873+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:36:16.318623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"90'161","reported_seq":772,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-23T20:44:45.677705+0000","last_change":"2023-09-23T20:38:05.591992+0000","last_active":"2023-09-23T20:44:45.677705+0000","last_peered":"2023-09-23T20:44:45.677705+0000","last_clean":"2023-09-23T20:44:45.677705+0000","last_became_active":"2023-09-23T20:38:05.591701+0000","last_became_peered":"2023-09-23T20:38:05.591701+0000","last_unstale":"2023-09-23T20:44:45.677705+0000","last_undegraded":"2023-09-23T20:44:45.677705+0000","last_fullsized":"2023-09-23T20:44:45.677705+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:02.384133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:02.384133+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:41.722542+0000","scrub_duration":0.001769098,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":339,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-23T20:44:45.681965+0000","last_change":"2023-09-23T20:38:02.542719+0000","last_active":"2023-09-23T20:44:45.681965+0000","last_peered":"2023-09-23T20:44:45.681965+0000","last_clean":"2023-09-23T20:44:45.681965+0000","last_became_active":"2023-09-23T20:38:02.542385+0000","last_became_peered":"2023-09-23T20:38:02.542385+0000","last_unstale":"2023-09-23T20:44:45.681965+0000","last_undegraded":"2023-09-23T20:44:45.681965+0000","last_fullsized":"2023-09-23T20:44:45.681965+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:36:23.673757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":305,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-23T20:44:45.675823+0000","last_change":"2023-09-23T20:37:43.584253+0000","last_active":"2023-09-23T20:44:45.675823+0000","last_peered":"2023-09-23T20:44:45.675823+0000","last_clean":"2023-09-23T20:44:45.675823+0000","last_became_active":"2023-09-23T20:37:43.580069+0000","last_became_peered":"2023-09-23T20:37:43.580069+0000","last_unstale":"2023-09-23T20:44:45.675823+0000","last_undegraded":"2023-09-23T20:44:45.675823+0000","last_fullsized":"2023-09-23T20:44:45.675823+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:39:11.482581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"90'161","reported_seq":610,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-23T20:44:45.681753+0000","last_change":"2023-09-23T20:36:01.505098+0000","last_active":"2023-09-23T20:44:45.681753+0000","last_peered":"2023-09-23T20:44:45.681753+0000","last_clean":"2023-09-23T20:44:45.681753+0000","last_became_active":"2023-09-23T20:35:26.644707+0000","last_became_peered":"2023-09-23T20:35:26.644707+0000","last_unstale":"2023-09-23T20:44:45.681753+0000","last_undegraded":"2023-09-23T20:44:45.681753+0000","last_fullsized":"2023-09-23T20:44:45.681753+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:01.504973+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:01.504973+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:02:10.880809+0000","scrub_duration":0.00202344,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":548,"reported_epoch":515,"state":"active+clean","last_fresh":"2023-09-23T20:44:44.664972+0000","last_change":"2023-09-23T20:43:46.511657+0000","last_active":"2023-09-23T20:44:44.664972+0000","last_peered":"2023-09-23T20:44:44.664972+0000","last_clean":"2023-09-23T20:44:44.664972+0000","last_became_active":"2023-09-23T20:43:46.511326+0000","last_became_peered":"2023-09-23T20:43:46.511326+0000","last_unstale":"2023-09-23T20:44:44.664972+0000","last_undegraded":"2023-09-23T20:44:44.664972+0000","last_fullsized":"2023-09-23T20:44:44.664972+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:36:46.139261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3324,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":234,"reported_epoch":516,"state":"active+clean","last_fresh":"2023-09-23T20:44:45.684588+0000","last_change":"2023-09-23T20:38:02.538333+0000","last_active":"2023-09-23T20:44:45.684588+0000","last_peered":"2023-09-23T20:44:45.684588+0000","last_clean":"2023-09-23T20:44:45.684588+0000","last_became_active":"2023-09-23T20:38:02.538008+0000","last_became_peered":"2023-09-23T20:38:02.538008+0000","last_unstale":"2023-09-23T20:44:45.684588+0000","last_undegraded":"2023-09-23T20:44:45.684588+0000","last_fullsized":"2023-09-23T20:44:45.684588+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:02:38.789530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":90,"num_bytes_recovered":18486,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26691,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":1467940888,"num_objects":22400,"num_object_clones":0,"num_object_copies":67200,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":329,"num_objects_unfound":0,"num_objects_dirty":22400,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":205264,"num_write_kb":7285185,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16881,"num_bytes_recovered":1105854488,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":4741808128,"data_stored":4741791840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":102859,"ondisk_log_size":102859,"up":48,"acting":48,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5983,"num_read_kb":13200,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":84,"num_bytes_recovered":16777216,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6082,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201680,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1440284,"kb_used_data":940116,"kb_used_omap":0,"kb_used_meta":500032,"kb_avail":92304868,"statfs":{"total":95995035648,"available":94520184832,"internally_reserved":0,"allocated":962678784,"data_stored":961155313,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":512032768},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:43:58 2023","interfaces":[{"interface":"back","average":{"1min":2.5600000000000001,"5min":1.643,"15min":2.089},"min":{"1min":0.23899999999999999,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":15.1,"5min":15.1,"15min":18.315000000000001},"last":0.73499999999999999},{"interface":"front","average":{"1min":2.7040000000000002,"5min":1.587,"15min":1.8069999999999999},"min":{"1min":0.33300000000000002,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":15.327999999999999,"5min":15.327999999999999,"15min":19.355},"last":0.67600000000000005}]},{"osd":1,"last update":"Sat Sep 23 20:44:08 2023","interfaces":[{"interface":"back","average":{"1min":2.714,"5min":1.452,"15min":2.105},"min":{"1min":0.26900000000000002,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":15.131,"5min":15.131,"15min":49.965000000000003},"last":0.70199999999999996},{"interface":"front","average":{"1min":3.6850000000000001,"5min":2.266,"15min":2.339},"min":{"1min":0.36299999999999999,"5min":0.26700000000000002,"15min":0.248},"max":{"1min":15.18,"5min":15.18,"15min":53.451999999999998},"last":0.748}]},{"osd":2,"last update":"Sat Sep 23 20:43:58 2023","interfaces":[{"interface":"back","average":{"1min":3.1859999999999999,"5min":2.0600000000000001,"15min":2.1880000000000002},"min":{"1min":0.39800000000000002,"5min":0.27900000000000003,"15min":0.27600000000000002},"max":{"1min":15.087,"5min":15.087,"15min":16.965},"last":0.64300000000000002},{"interface":"front","average":{"1min":2.2320000000000002,"5min":1.716,"15min":2.4279999999999999},"min":{"1min":0.35899999999999999,"5min":0.30299999999999999,"15min":0.26600000000000001},"max":{"1min":15.193,"5min":15.193,"15min":23.494},"last":0.69099999999999995}]},{"osd":3,"last update":"Sat Sep 23 20:44:28 2023","interfaces":[{"interface":"back","average":{"1min":2.6840000000000002,"5min":1.772,"15min":2.371},"min":{"1min":0.41399999999999998,"5min":0.32500000000000001,"15min":0.29199999999999998},"max":{"1min":15.115,"5min":15.115,"15min":23.352},"last":0.75900000000000001},{"interface":"front","average":{"1min":3.3570000000000002,"5min":2.1389999999999998,"15min":2.4489999999999998},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.33100000000000002},"max":{"1min":15.144,"5min":15.144,"15min":20.969999999999999},"last":0.71699999999999997}]},{"osd":4,"last update":"Sat Sep 23 20:44:28 2023","interfaces":[{"interface":"back","average":{"1min":2.0110000000000001,"5min":1.391,"15min":1.2669999999999999},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.23200000000000001},"max":{"1min":12.260999999999999,"5min":12.260999999999999,"15min":18.364999999999998},"last":0.443},{"interface":"front","average":{"1min":2.145,"5min":1.5640000000000001,"15min":1.3320000000000001},"min":{"1min":0.315,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":6.9939999999999998,"5min":6.9939999999999998,"15min":23.617000000000001},"last":0.501}]},{"osd":5,"last update":"Sat Sep 23 20:44:28 2023","interfaces":[{"interface":"back","average":{"1min":2.5270000000000001,"5min":1.6040000000000001,"15min":1.1970000000000001},"min":{"1min":0.36899999999999999,"5min":0.30099999999999999,"15min":0.26600000000000001},"max":{"1min":15.005000000000001,"5min":15.005000000000001,"15min":15.005000000000001},"last":0.47299999999999998},{"interface":"front","average":{"1min":1.9990000000000001,"5min":1.556,"15min":1.3120000000000001},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":12.212,"5min":12.731,"15min":18.831},"last":0.48399999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:44:28 2023","interfaces":[{"interface":"back","average":{"1min":3.6379999999999999,"5min":1.837,"15min":1.4359999999999999},"min":{"1min":0.34599999999999997,"5min":0.33700000000000002,"15min":0.30399999999999999},"max":{"1min":15.038,"5min":15.038,"15min":18.417000000000002},"last":0.51900000000000002},{"interface":"front","average":{"1min":2.8969999999999998,"5min":1.6659999999999999,"15min":1.274},"min":{"1min":0.47999999999999998,"5min":0.36499999999999999,"15min":0.318},"max":{"1min":15.369999999999999,"5min":15.369999999999999,"15min":15.369999999999999},"last":0.45900000000000002}]}]},{"osd":6,"up_from":47,"seq":201863463319,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":712908,"kb_used_data":289668,"kb_used_omap":0,"kb_used_meta":423232,"kb_avail":93032244,"statfs":{"total":95995035648,"available":95265017856,"internally_reserved":0,"allocated":296620032,"data_stored":295112250,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":433389568},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:43:54 2023","interfaces":[{"interface":"back","average":{"1min":2.2799999999999998,"5min":1.375,"15min":1.9790000000000001},"min":{"1min":0.24099999999999999,"5min":0.23699999999999999,"15min":0.22800000000000001},"max":{"1min":12.552,"5min":12.552,"15min":53.701999999999998},"last":0.28100000000000003},{"interface":"front","average":{"1min":2.149,"5min":1.5589999999999999,"15min":2.3490000000000002},"min":{"1min":0.31,"5min":0.23300000000000001,"15min":0.22700000000000001},"max":{"1min":9.2699999999999996,"5min":9.2699999999999996,"15min":60.328000000000003},"last":0.81399999999999995}]},{"osd":1,"last update":"Sat Sep 23 20:44:17 2023","interfaces":[{"interface":"back","average":{"1min":3.3330000000000002,"5min":1.744,"15min":2.1309999999999998},"min":{"1min":0.317,"5min":0.249,"15min":0.249},"max":{"1min":9.2520000000000007,"5min":9.2520000000000007,"15min":64.274000000000001},"last":1.224},{"interface":"front","average":{"1min":2.8940000000000001,"5min":1.8100000000000001,"15min":2.0739999999999998},"min":{"1min":0.42799999999999999,"5min":0.32000000000000001,"15min":0.28000000000000003},"max":{"1min":17.786000000000001,"5min":17.786000000000001,"15min":34.490000000000002},"last":0.49099999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:43:54 2023","interfaces":[{"interface":"back","average":{"1min":2.5659999999999998,"5min":1.9419999999999999,"15min":2.202},"min":{"1min":0.34999999999999998,"5min":0.27800000000000002,"15min":0.27700000000000002},"max":{"1min":8.0370000000000008,"5min":9.9049999999999994,"15min":34.317999999999998},"last":1.258},{"interface":"front","average":{"1min":2.0019999999999998,"5min":1.766,"15min":2.1080000000000001},"min":{"1min":0.40699999999999997,"5min":0.311,"15min":0.30399999999999999},"max":{"1min":8.7309999999999999,"5min":9.9269999999999996,"15min":34.292999999999999},"last":0.77700000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:44:46 2023","interfaces":[{"interface":"back","average":{"1min":2.7330000000000001,"5min":2.024,"15min":1.974},"min":{"1min":0.47999999999999998,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":11.664,"5min":11.664,"15min":11.664},"last":2.9079999999999999},{"interface":"front","average":{"1min":2.3039999999999998,"5min":2.0299999999999998,"15min":2.1459999999999999},"min":{"1min":0.42099999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":8.2560000000000002,"5min":11.808,"15min":12.875999999999999},"last":3.085}]},{"osd":4,"last update":"Sat Sep 23 20:44:17 2023","interfaces":[{"interface":"back","average":{"1min":1.27,"5min":0.997,"15min":0.85199999999999998},"min":{"1min":0.33600000000000002,"5min":0.25600000000000001,"15min":0.251},"max":{"1min":6.5800000000000001,"5min":6.5800000000000001,"15min":23.904},"last":1.4330000000000001},{"interface":"front","average":{"1min":1.4019999999999999,"5min":1.1519999999999999,"15min":1.1240000000000001},"min":{"1min":0.34999999999999998,"5min":0.309,"15min":0.27000000000000002},"max":{"1min":5.6260000000000003,"5min":6.7359999999999998,"15min":11.515000000000001},"last":0.40600000000000003}]},{"osd":5,"last update":"Sat Sep 23 20:44:17 2023","interfaces":[{"interface":"back","average":{"1min":1.381,"5min":0.97099999999999997,"15min":0.86499999999999999},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":6.5460000000000003,"5min":6.6340000000000003,"15min":8.4499999999999993},"last":0.54100000000000004},{"interface":"front","average":{"1min":1.5509999999999999,"5min":1.4870000000000001,"15min":1.1970000000000001},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":5.4969999999999999,"5min":7.1319999999999997,"15min":9.9299999999999997},"last":0.34999999999999998}]},{"osd":7,"last update":"Sat Sep 23 20:44:46 2023","interfaces":[{"interface":"back","average":{"1min":1.4199999999999999,"5min":1.079,"15min":0.94799999999999995},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":7.8250000000000002,"5min":8.8350000000000009,"15min":8.8350000000000009},"last":0.63300000000000001},{"interface":"front","average":{"1min":2.6440000000000001,"5min":1.661,"15min":1.3009999999999999},"min":{"1min":0.45900000000000002,"5min":0.34699999999999998,"15min":0.30299999999999999},"max":{"1min":7.1239999999999997,"5min":7.1929999999999996,"15min":8.8119999999999994},"last":0.503}]}]},{"osd":1,"up_from":266,"seq":1142461300918,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1680344,"kb_used_data":1060048,"kb_used_omap":8,"kb_used_meta":620087,"kb_avail":92064808,"statfs":{"total":95995035648,"available":94274363392,"internally_reserved":0,"allocated":1085489152,"data_stored":1082805967,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8206,"internal_metadata":634970098},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,4,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:44:15 2023","interfaces":[{"interface":"back","average":{"1min":3.6110000000000002,"5min":2.206,"15min":2.5329999999999999},"min":{"1min":0.214,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":15.532999999999999,"5min":16.292000000000002,"15min":52.689999999999998},"last":1.71},{"interface":"front","average":{"1min":3.2290000000000001,"5min":2.2970000000000002,"15min":3.0179999999999998},"min":{"1min":0.22600000000000001,"5min":0.20100000000000001,"15min":0.19900000000000001},"max":{"1min":14.445,"5min":16.332999999999998,"15min":52.662999999999997},"last":0.32600000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:44:15 2023","interfaces":[{"interface":"back","average":{"1min":3.0390000000000001,"5min":2.4140000000000001,"15min":3.3370000000000002},"min":{"1min":0.28999999999999998,"5min":0.216,"15min":0.20899999999999999},"max":{"1min":8.6519999999999992,"5min":14.148,"15min":32.255000000000003},"last":6.9009999999999998},{"interface":"front","average":{"1min":2.9049999999999998,"5min":1.9810000000000001,"15min":2.2909999999999999},"min":{"1min":0.26000000000000001,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":10.041,"5min":10.041,"15min":13.404999999999999},"last":1.7509999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:44:15 2023","interfaces":[{"interface":"back","average":{"1min":2.5830000000000002,"5min":2.089,"15min":3.097},"min":{"1min":0.36899999999999999,"5min":0.247,"15min":0.247},"max":{"1min":7.2469999999999999,"5min":16.245999999999999,"15min":80.745999999999995},"last":5.8760000000000003},{"interface":"front","average":{"1min":3.5779999999999998,"5min":2.645,"15min":3.3450000000000002},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.22700000000000001},"max":{"1min":11.523,"5min":13.497999999999999,"15min":82.988},"last":0.40300000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:44:15 2023","interfaces":[{"interface":"back","average":{"1min":2.3250000000000002,"5min":2.202,"15min":2.25},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":6.5970000000000004,"5min":20.634,"15min":24.327999999999999},"last":5.1159999999999997},{"interface":"front","average":{"1min":2.0190000000000001,"5min":1.982,"15min":1.748},"min":{"1min":0.34699999999999998,"5min":0.32600000000000001,"15min":0.311},"max":{"1min":6.0869999999999997,"5min":20.835999999999999,"15min":20.835999999999999},"last":1.7889999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:44:15 2023","interfaces":[{"interface":"back","average":{"1min":3.2040000000000002,"5min":2.4319999999999999,"15min":2.3029999999999999},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":14.366,"5min":14.366,"15min":33.401000000000003},"last":0.438},{"interface":"front","average":{"1min":2.7469999999999999,"5min":2.3519999999999999,"15min":2.4820000000000002},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.32500000000000001},"max":{"1min":7.3300000000000001,"5min":10.958,"15min":33.417000000000002},"last":5.0659999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:44:15 2023","interfaces":[{"interface":"back","average":{"1min":3.0030000000000001,"5min":2.3639999999999999,"15min":2.839},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35699999999999998},"max":{"1min":14.414999999999999,"5min":14.414999999999999,"15min":72.682000000000002},"last":0.46500000000000002},{"interface":"front","average":{"1min":3.2829999999999999,"5min":2.3620000000000001,"15min":2.9860000000000002},"min":{"1min":0.38500000000000001,"5min":0.317,"15min":0.317},"max":{"1min":9.1470000000000002,"5min":9.4090000000000007,"15min":72.563999999999993},"last":5.2119999999999997}]},{"osd":7,"last update":"Sat Sep 23 20:44:15 2023","interfaces":[{"interface":"back","average":{"1min":2.98,"5min":2.4470000000000001,"15min":2.3700000000000001},"min":{"1min":0.43099999999999999,"5min":0.37,"15min":0.37},"max":{"1min":9.2409999999999997,"5min":16.359000000000002,"15min":18.437000000000001},"last":0.48899999999999999},{"interface":"front","average":{"1min":2.6459999999999999,"5min":2.4460000000000002,"15min":2.4910000000000001},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":7.1959999999999997,"5min":10.34,"15min":24.524000000000001},"last":5.1760000000000002}]}]},{"osd":0,"up_from":94,"seq":403726926120,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1114972,"kb_used_data":397780,"kb_used_omap":7,"kb_used_meta":717176,"kb_avail":92630180,"statfs":{"total":95995035648,"available":94853304320,"internally_reserved":0,"allocated":407326720,"data_stored":404808995,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":734388241},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:44:24 2023","interfaces":[{"interface":"back","average":{"1min":3.2120000000000002,"5min":2.4100000000000001,"15min":2.7210000000000001},"min":{"1min":0.184,"5min":0.184,"15min":0.17199999999999999},"max":{"1min":18.052,"5min":24.010000000000002,"15min":42.000999999999998},"last":3.661},{"interface":"front","average":{"1min":3.0939999999999999,"5min":2.605,"15min":2.9129999999999998},"min":{"1min":0.33800000000000002,"5min":0.218,"15min":0.20399999999999999},"max":{"1min":18.097000000000001,"5min":18.097000000000001,"15min":23.036000000000001},"last":3.98}]},{"osd":2,"last update":"Sat Sep 23 20:43:58 2023","interfaces":[{"interface":"back","average":{"1min":2.5169999999999999,"5min":2.5699999999999998,"15min":3.0190000000000001},"min":{"1min":0.248,"5min":0.23400000000000001,"15min":0.185},"max":{"1min":10.199,"5min":48.841000000000001,"15min":77.448999999999998},"last":0.371},{"interface":"front","average":{"1min":1.9590000000000001,"5min":2.3239999999999998,"15min":2.9489999999999998},"min":{"1min":0.22,"5min":0.22,"15min":0.188},"max":{"1min":6.6580000000000004,"5min":46.183,"15min":81.724999999999994},"last":2.2850000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:44:31 2023","interfaces":[{"interface":"back","average":{"1min":3.6960000000000002,"5min":2.7290000000000001,"15min":3.1640000000000001},"min":{"1min":0.30599999999999999,"5min":0.248,"15min":0.23400000000000001},"max":{"1min":15.048,"5min":15.048,"15min":81.400999999999996},"last":3.4420000000000002},{"interface":"front","average":{"1min":2.1549999999999998,"5min":2.3159999999999998,"15min":3.3439999999999999},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.21199999999999999},"max":{"1min":9.6379999999999999,"5min":14.237,"15min":81.137},"last":4.0540000000000003}]},{"osd":4,"last update":"Sat Sep 23 20:43:58 2023","interfaces":[{"interface":"back","average":{"1min":1.385,"5min":1.925,"15min":1.9750000000000001},"min":{"1min":0.32000000000000001,"5min":0.31900000000000001,"15min":0.25900000000000001},"max":{"1min":4.5279999999999996,"5min":6.8970000000000002,"15min":12.587999999999999},"last":0.433},{"interface":"front","average":{"1min":1.7310000000000001,"5min":1.823,"15min":1.917},"min":{"1min":0.28899999999999998,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":6.5890000000000004,"5min":9.8480000000000008,"15min":23.001000000000001},"last":6.6130000000000004}]},{"osd":5,"last update":"Sat Sep 23 20:43:58 2023","interfaces":[{"interface":"back","average":{"1min":2.077,"5min":2.2370000000000001,"15min":2.121},"min":{"1min":0.30599999999999999,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":6.6200000000000001,"5min":10.239000000000001,"15min":22.969999999999999},"last":7.4800000000000004},{"interface":"front","average":{"1min":2.5529999999999999,"5min":2.2010000000000001,"15min":2.3719999999999999},"min":{"1min":0.34300000000000003,"5min":0.318,"15min":0.25600000000000001},"max":{"1min":6.75,"5min":12.929,"15min":26.228000000000002},"last":2.1989999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:43:58 2023","interfaces":[{"interface":"back","average":{"1min":2.1659999999999999,"5min":2.1909999999999998,"15min":2.1680000000000001},"min":{"1min":0.433,"5min":0.34499999999999997,"15min":0.32300000000000001},"max":{"1min":7.6150000000000002,"5min":24.056000000000001,"15min":24.056000000000001},"last":6.681},{"interface":"front","average":{"1min":2.2799999999999998,"5min":1.79,"15min":2.1539999999999999},"min":{"1min":0.36399999999999999,"5min":0.34100000000000003,"15min":0.29399999999999998},"max":{"1min":6.7119999999999997,"5min":8.0549999999999997,"15min":26.268000000000001},"last":2.2290000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:43:58 2023","interfaces":[{"interface":"back","average":{"1min":2.9220000000000002,"5min":2.0899999999999999,"15min":2.3969999999999998},"min":{"1min":0.433,"5min":0.36799999999999999,"15min":0.30599999999999999},"max":{"1min":9.4250000000000007,"5min":9.4250000000000007,"15min":26.177},"last":2.2509999999999999},{"interface":"front","average":{"1min":2.669,"5min":2.4100000000000001,"15min":2.4529999999999998},"min":{"1min":0.46899999999999997,"5min":0.39200000000000002,"15min":0.34200000000000003},"max":{"1min":15.006,"5min":15.006,"15min":24.669},"last":0.40600000000000003}]}]},{"osd":2,"up_from":104,"seq":446676599068,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1020840,"kb_used_data":279072,"kb_used_omap":11,"kb_used_meta":741748,"kb_avail":92724312,"statfs":{"total":95995035648,"available":94949695488,"internally_reserved":0,"allocated":285769728,"data_stored":283246443,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11878,"internal_metadata":759550362},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:44:05 2023","interfaces":[{"interface":"back","average":{"1min":1.887,"5min":1.3859999999999999,"15min":2.4430000000000001},"min":{"1min":0.20499999999999999,"5min":0.17299999999999999,"15min":0.17299999999999999},"max":{"1min":11.07,"5min":11.07,"15min":82.424999999999997},"last":1.149},{"interface":"front","average":{"1min":2.6739999999999999,"5min":2.012,"15min":2.7149999999999999},"min":{"1min":0.27700000000000002,"5min":0.23000000000000001,"15min":0.19},"max":{"1min":16.678999999999998,"5min":16.678999999999998,"15min":82.524000000000001},"last":7.3760000000000003}]},{"osd":1,"last update":"Sat Sep 23 20:44:05 2023","interfaces":[{"interface":"back","average":{"1min":1.4430000000000001,"5min":2.02,"15min":2.839},"min":{"1min":0.248,"5min":0.248,"15min":0.221},"max":{"1min":3.8069999999999999,"5min":12.981,"15min":46.405000000000001},"last":12.019},{"interface":"front","average":{"1min":2.0590000000000002,"5min":1.9119999999999999,"15min":2.641},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":11.683999999999999,"5min":11.683999999999999,"15min":46.459000000000003},"last":1.488}]},{"osd":3,"last update":"Sat Sep 23 20:44:31 2023","interfaces":[{"interface":"back","average":{"1min":2.1800000000000002,"5min":2.2010000000000001,"15min":2.5990000000000002},"min":{"1min":0.253,"5min":0.253,"15min":0.19800000000000001},"max":{"1min":12.847,"5min":12.847,"15min":19.562999999999999},"last":10.999000000000001},{"interface":"front","average":{"1min":0.91400000000000003,"5min":1.425,"15min":2.1720000000000002},"min":{"1min":0.26500000000000001,"5min":0.26400000000000001,"15min":0.249},"max":{"1min":6.3700000000000001,"5min":11.375,"15min":30.704999999999998},"last":1.1890000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:44:05 2023","interfaces":[{"interface":"back","average":{"1min":1.7509999999999999,"5min":1.6359999999999999,"15min":1.911},"min":{"1min":0.34300000000000003,"5min":0.29799999999999999,"15min":0.27500000000000002},"max":{"1min":6.8719999999999999,"5min":12.025,"15min":16.797999999999998},"last":7.4299999999999997},{"interface":"front","average":{"1min":1.702,"5min":1.603,"15min":2.0659999999999998},"min":{"1min":0.38100000000000001,"5min":0.30599999999999999,"15min":0.30499999999999999},"max":{"1min":7.1150000000000002,"5min":9.1379999999999999,"15min":19.440999999999999},"last":10.948}]},{"osd":5,"last update":"Sat Sep 23 20:44:05 2023","interfaces":[{"interface":"back","average":{"1min":2.194,"5min":1.863,"15min":2.452},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":6.3179999999999996,"5min":12.07,"15min":95.393000000000001},"last":7.5049999999999999},{"interface":"front","average":{"1min":1.7929999999999999,"5min":1.6220000000000001,"15min":2.5659999999999998},"min":{"1min":0.377,"5min":0.37,"15min":0.34799999999999998},"max":{"1min":5.1909999999999998,"5min":7.8179999999999996,"15min":95.414000000000001},"last":11.061999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:44:05 2023","interfaces":[{"interface":"back","average":{"1min":2.258,"5min":1.9059999999999999,"15min":2.246},"min":{"1min":0.39800000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":11.894,"5min":14.047000000000001,"15min":19.486000000000001},"last":11.031000000000001},{"interface":"front","average":{"1min":2.069,"5min":1.323,"15min":1.7190000000000001},"min":{"1min":0.48099999999999998,"5min":0.34200000000000003,"15min":0.316},"max":{"1min":11.747,"5min":11.747,"15min":30.670999999999999},"last":0.45600000000000002}]},{"osd":7,"last update":"Sat Sep 23 20:44:05 2023","interfaces":[{"interface":"back","average":{"1min":1.4399999999999999,"5min":1.2929999999999999,"15min":1.7270000000000001},"min":{"1min":0.39200000000000002,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":6.1849999999999996,"5min":8.4949999999999992,"15min":30.741},"last":0.41999999999999998},{"interface":"front","average":{"1min":2.6360000000000001,"5min":2.081,"15min":2.2469999999999999},"min":{"1min":0.44700000000000001,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":6.2910000000000004,"5min":12.099,"15min":16.719000000000001},"last":7.4660000000000002}]}]},{"osd":3,"up_from":109,"seq":468151435542,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1654612,"kb_used_data":691148,"kb_used_omap":4,"kb_used_meta":963451,"kb_avail":92090540,"statfs":{"total":95995035648,"available":94300712960,"internally_reserved":0,"allocated":707735552,"data_stored":705229007,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4514,"internal_metadata":986574430},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,2,0,0,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:44:39 2023","interfaces":[{"interface":"back","average":{"1min":2.71,"5min":2.4169999999999998,"15min":2.4929999999999999},"min":{"1min":0.222,"5min":0.20899999999999999,"15min":0.191},"max":{"1min":8.5419999999999998,"5min":22.32,"15min":22.32},"last":4.2249999999999996},{"interface":"front","average":{"1min":2.105,"5min":2.6749999999999998,"15min":2.4929999999999999},"min":{"1min":0.249,"5min":0.22900000000000001,"15min":0.19500000000000001},"max":{"1min":6.2160000000000002,"5min":22.805,"15min":22.805},"last":4.8300000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:44:11 2023","interfaces":[{"interface":"back","average":{"1min":2.823,"5min":2.1960000000000002,"15min":2.2290000000000001},"min":{"1min":0.31,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":10.022,"5min":17.529,"15min":17.529},"last":4.4420000000000002},{"interface":"front","average":{"1min":1.742,"5min":2.3700000000000001,"15min":2.4500000000000002},"min":{"1min":0.33300000000000002,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":7.1029999999999998,"5min":14.19,"15min":32.603999999999999},"last":5.6799999999999997}]},{"osd":2,"last update":"Sat Sep 23 20:44:39 2023","interfaces":[{"interface":"back","average":{"1min":2.2970000000000002,"5min":3.5840000000000001,"15min":3.0019999999999998},"min":{"1min":0.28599999999999998,"5min":0.26600000000000001,"15min":0.23799999999999999},"max":{"1min":7.7859999999999996,"5min":68.206999999999994,"15min":68.206999999999994},"last":6.9180000000000001},{"interface":"front","average":{"1min":2.4470000000000001,"5min":2.7130000000000001,"15min":2.7450000000000001},"min":{"1min":0.251,"5min":0.245,"15min":0.20899999999999999},"max":{"1min":9.1760000000000002,"5min":67.736999999999995,"15min":67.736999999999995},"last":4.7140000000000004}]},{"osd":4,"last update":"Sat Sep 23 20:44:39 2023","interfaces":[{"interface":"back","average":{"1min":2.2879999999999998,"5min":2.008,"15min":1.786},"min":{"1min":0.35799999999999998,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":6.7889999999999997,"5min":22.157,"15min":22.157},"last":4.3849999999999998},{"interface":"front","average":{"1min":2.113,"5min":2.3290000000000002,"15min":2.3279999999999998},"min":{"1min":0.36499999999999999,"5min":0.33200000000000002,"15min":0.28100000000000003},"max":{"1min":7.9020000000000001,"5min":15.868,"15min":19.635000000000002},"last":4.5819999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:44:39 2023","interfaces":[{"interface":"back","average":{"1min":2.1840000000000002,"5min":2.2970000000000002,"15min":1.992},"min":{"1min":0.33800000000000002,"5min":0.32700000000000001,"15min":0.31900000000000001},"max":{"1min":6.7469999999999999,"5min":22.289999999999999,"15min":22.289999999999999},"last":4.4119999999999999},{"interface":"front","average":{"1min":1.8089999999999999,"5min":1.8440000000000001,"15min":1.841},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.32000000000000001},"max":{"1min":4.6559999999999997,"5min":12.952,"15min":12.952},"last":4.8620000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:44:39 2023","interfaces":[{"interface":"back","average":{"1min":1.845,"5min":2.1360000000000001,"15min":1.956},"min":{"1min":0.36599999999999999,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":6.5350000000000001,"5min":12.917,"15min":17.524999999999999},"last":4.8970000000000002},{"interface":"front","average":{"1min":1.988,"5min":2.3300000000000001,"15min":2.2869999999999999},"min":{"1min":0.38100000000000001,"5min":0.35099999999999998,"15min":0.33700000000000002},"max":{"1min":6.8479999999999999,"5min":15.936,"15min":19.524000000000001},"last":4.6280000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:44:39 2023","interfaces":[{"interface":"back","average":{"1min":2.7850000000000001,"5min":2.7509999999999999,"15min":2.6429999999999998},"min":{"1min":0.46999999999999997,"5min":0.41499999999999998,"15min":0.36899999999999999},"max":{"1min":8.4429999999999996,"5min":15.976000000000001,"15min":19.571999999999999},"last":4.6630000000000003},{"interface":"front","average":{"1min":2.4750000000000001,"5min":2.262,"15min":1.9850000000000001},"min":{"1min":0.46300000000000002,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":8.5289999999999999,"5min":22.202999999999999,"15min":22.202999999999999},"last":4.6440000000000001}]}]},{"osd":4,"up_from":30,"seq":128849019296,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":792528,"kb_used_data":365576,"kb_used_omap":0,"kb_used_meta":426944,"kb_avail":92952624,"statfs":{"total":95995035648,"available":95183486976,"internally_reserved":0,"allocated":374349824,"data_stored":372837704,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":437190656},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:43:57 2023","interfaces":[{"interface":"back","average":{"1min":2.5609999999999999,"5min":1.8129999999999999,"15min":1.746},"min":{"1min":0.251,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":11.709,"5min":11.709,"15min":24.114999999999998},"last":0.32400000000000001},{"interface":"front","average":{"1min":2.8330000000000002,"5min":2.1440000000000001,"15min":1.9710000000000001},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.246},"max":{"1min":13.74,"5min":13.74,"15min":24.135999999999999},"last":0.43099999999999999}]},{"osd":1,"last update":"Sat Sep 23 20:44:11 2023","interfaces":[{"interface":"back","average":{"1min":2.306,"5min":1.8979999999999999,"15min":1.7749999999999999},"min":{"1min":0.26300000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":8.7460000000000004,"5min":10.363,"15min":18.190999999999999},"last":0.53200000000000003},{"interface":"front","average":{"1min":1.355,"5min":1.5569999999999999,"15min":1.613},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":4.8659999999999997,"5min":10.426,"15min":18.173999999999999},"last":0.38800000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:44:11 2023","interfaces":[{"interface":"back","average":{"1min":2.6040000000000001,"5min":1.756,"15min":1.885},"min":{"1min":0.29499999999999998,"5min":0.28399999999999997,"15min":0.26600000000000001},"max":{"1min":9.3320000000000007,"5min":13.051,"15min":32.363},"last":0.35799999999999998},{"interface":"front","average":{"1min":2.0150000000000001,"5min":2.0539999999999998,"15min":2.2090000000000001},"min":{"1min":0.41199999999999998,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":7.6159999999999997,"5min":11.917999999999999,"15min":34.042999999999999},"last":2.847}]},{"osd":3,"last update":"Sat Sep 23 20:44:27 2023","interfaces":[{"interface":"back","average":{"1min":1.877,"5min":2.04,"15min":1.9950000000000001},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":7.7480000000000002,"5min":12.147,"15min":14.515000000000001},"last":0.89400000000000002},{"interface":"front","average":{"1min":2.4260000000000002,"5min":2.0990000000000002,"15min":2.0699999999999998},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.30299999999999999},"max":{"1min":10.493,"5min":13.35,"15min":15.991},"last":17.169}]},{"osd":5,"last update":"Sat Sep 23 20:43:57 2023","interfaces":[{"interface":"back","average":{"1min":2.1789999999999998,"5min":1.3540000000000001,"15min":1.3440000000000001},"min":{"1min":0.34399999999999997,"5min":0.28000000000000003,"15min":0.25800000000000001},"max":{"1min":7.5810000000000004,"5min":9.7260000000000009,"15min":82.866},"last":0.56799999999999995},{"interface":"front","average":{"1min":1.363,"5min":1.2729999999999999,"15min":1.3180000000000001},"min":{"1min":0.311,"5min":0.311,"15min":0.28999999999999998},"max":{"1min":6.601,"5min":7.0129999999999999,"15min":82.546000000000006},"last":0.51400000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:44:11 2023","interfaces":[{"interface":"back","average":{"1min":1.0920000000000001,"5min":1.5449999999999999,"15min":1.214},"min":{"1min":0.34300000000000003,"5min":0.29499999999999998,"15min":0.27600000000000002},"max":{"1min":3.96,"5min":14.731999999999999,"15min":14.731999999999999},"last":0.46800000000000003},{"interface":"front","average":{"1min":1.8959999999999999,"5min":1.2989999999999999,"15min":1.1180000000000001},"min":{"1min":0.34300000000000003,"5min":0.314,"15min":0.29899999999999999},"max":{"1min":6.3440000000000003,"5min":7.6150000000000002,"15min":11.199},"last":0.621}]},{"osd":7,"last update":"Sat Sep 23 20:44:27 2023","interfaces":[{"interface":"back","average":{"1min":2.0630000000000002,"5min":2.0979999999999999,"15min":1.5649999999999999},"min":{"1min":0.374,"5min":0.35099999999999998,"15min":0.318},"max":{"1min":7.7629999999999999,"5min":22.184999999999999,"15min":22.184999999999999},"last":0.48999999999999999},{"interface":"front","average":{"1min":1.1240000000000001,"5min":1.724,"15min":1.2150000000000001},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.307},"max":{"1min":6.3689999999999998,"5min":21.754000000000001,"15min":21.754000000000001},"last":0.59499999999999997}]}]},{"osd":5,"up_from":39,"seq":167503724957,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1240204,"kb_used_data":743556,"kb_used_omap":0,"kb_used_meta":496640,"kb_avail":92504948,"statfs":{"total":95995035648,"available":94725066752,"internally_reserved":0,"allocated":761401344,"data_stored":759889950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":508559360},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:44:00 2023","interfaces":[{"interface":"back","average":{"1min":2.145,"5min":1.7050000000000001,"15min":1.7989999999999999},"min":{"1min":0.23999999999999999,"5min":0.22500000000000001,"15min":0.20699999999999999},"max":{"1min":15.018000000000001,"5min":15.018000000000001,"15min":18.16},"last":0.97799999999999998},{"interface":"front","average":{"1min":2.7130000000000001,"5min":1.6870000000000001,"15min":1.7410000000000001},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":15.034000000000001,"5min":15.034000000000001,"15min":15.034000000000001},"last":0.996}]},{"osd":1,"last update":"Sat Sep 23 20:44:14 2023","interfaces":[{"interface":"back","average":{"1min":2.7349999999999999,"5min":3.0299999999999998,"15min":2.2690000000000001},"min":{"1min":0.379,"5min":0.26500000000000001,"15min":0.253},"max":{"1min":9.4199999999999999,"5min":61.005000000000003,"15min":61.005000000000003},"last":0.35599999999999998},{"interface":"front","average":{"1min":1.339,"5min":2.4129999999999998,"15min":2.0019999999999998},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.23499999999999999},"max":{"1min":6.7350000000000003,"5min":59.688000000000002,"15min":59.688000000000002},"last":0.44600000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:44:00 2023","interfaces":[{"interface":"back","average":{"1min":2.3330000000000002,"5min":2.032,"15min":2.0950000000000002},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.26500000000000001},"max":{"1min":8.3719999999999999,"5min":9.2119999999999997,"15min":37.011000000000003},"last":0.51700000000000002},{"interface":"front","average":{"1min":2.7240000000000002,"5min":1.9359999999999999,"15min":2.0659999999999998},"min":{"1min":0.314,"5min":0.28299999999999997,"15min":0.255},"max":{"1min":9.8260000000000005,"5min":10.565,"15min":36.835000000000001},"last":0.81699999999999995}]},{"osd":3,"last update":"Sat Sep 23 20:44:20 2023","interfaces":[{"interface":"back","average":{"1min":1.7769999999999999,"5min":2.375,"15min":2.375},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.29599999999999999},"max":{"1min":5.6619999999999999,"5min":11.032,"15min":36.716000000000001},"last":0.55500000000000005},{"interface":"front","average":{"1min":1.5900000000000001,"5min":2.238,"15min":2.2530000000000001},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":7.4100000000000001,"5min":13.266,"15min":18.504000000000001},"last":0.83699999999999997}]},{"osd":4,"last update":"Sat Sep 23 20:43:50 2023","interfaces":[{"interface":"back","average":{"1min":3.8220000000000001,"5min":1.786,"15min":1.292},"min":{"1min":0.309,"5min":0.27300000000000002,"15min":0.255},"max":{"1min":13.391999999999999,"5min":13.391999999999999,"15min":28.015999999999998},"last":0.39300000000000002},{"interface":"front","average":{"1min":2.1760000000000002,"5min":1.5669999999999999,"15min":1.1220000000000001},"min":{"1min":0.34699999999999998,"5min":0.30299999999999999,"15min":0.28299999999999997},"max":{"1min":7.3799999999999999,"5min":7.3799999999999999,"15min":12.300000000000001},"last":0.56899999999999995}]},{"osd":6,"last update":"Sat Sep 23 20:44:07 2023","interfaces":[{"interface":"back","average":{"1min":3.3999999999999999,"5min":2.04,"15min":1.3320000000000001},"min":{"1min":0.39100000000000001,"5min":0.317,"15min":0.30199999999999999},"max":{"1min":13.734,"5min":13.734,"15min":13.734},"last":0.58999999999999997},{"interface":"front","average":{"1min":1.375,"5min":1.577,"15min":1.1950000000000001},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":5.274,"5min":8.7449999999999992,"15min":16.265999999999998},"last":0.60399999999999998}]},{"osd":7,"last update":"Sat Sep 23 20:44:20 2023","interfaces":[{"interface":"back","average":{"1min":1.4610000000000001,"5min":1.7330000000000001,"15min":1.1339999999999999},"min":{"1min":0.372,"5min":0.34499999999999997,"15min":0.28499999999999998},"max":{"1min":4.3250000000000002,"5min":16.366,"15min":16.366},"last":0.48599999999999999},{"interface":"front","average":{"1min":1.4950000000000001,"5min":1.6439999999999999,"15min":1.173},"min":{"1min":0.38900000000000001,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":7.524,"5min":7.915,"15min":13.122},"last":0.432}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4722,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1360,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8206,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7156,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3154,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":392232960,"data_stored":392232960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1057755136,"data_stored":1057751064,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":271712256,"data_stored":271712256,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":700121088,"data_stored":700121088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":359206912,"data_stored":359202840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":751767552,"data_stored":751763480,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":277479424,"data_stored":277479424,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":931532800,"data_stored":931528728,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:44:51.660 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-23T20:44:51.660 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:44:51.692 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:51.727 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:44:51.730 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:51.730 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:44:51.738 INFO:teuthology.orchestra.run.smithi018.stderr:nodeep-scrub is unset 2023-09-23T20:44:51.753 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:44:51.831 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:44:51.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:51 smithi018 ceph-mon[179647]: pgmap v1295: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 1.4 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 432 op/s; 329/67284 objects misplaced (0.489%); 4.4 MiB/s, 71 objects/s recovering 2023-09-23T20:44:51.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:51 smithi018 ceph-mon[179647]: osdmap e518: 8 total, 8 up, 8 in 2023-09-23T20:44:51.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:51 smithi018 ceph-mon[179647]: failed to encode map e518 with expected crc 2023-09-23T20:44:51.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:51 smithi018 ceph-mon[179647]: failed to encode map e518 with expected crc 2023-09-23T20:44:51.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:51 smithi018 ceph-mon[179647]: failed to encode map e518 with expected crc 2023-09-23T20:44:51.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:51 smithi018 ceph-mon[179647]: failed to encode map e518 with expected crc 2023-09-23T20:44:51.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:51 smithi018 ceph-mon[179647]: failed to encode map e518 with expected crc 2023-09-23T20:44:51.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:51 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/932502547' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:44:51.896 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:51 smithi018 ceph-mon[184144]: pgmap v1295: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 1.4 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 432 op/s; 329/67284 objects misplaced (0.489%); 4.4 MiB/s, 71 objects/s recovering 2023-09-23T20:44:51.896 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:51 smithi018 ceph-mon[184144]: osdmap e518: 8 total, 8 up, 8 in 2023-09-23T20:44:51.896 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:51 smithi018 ceph-mon[184144]: failed to encode map e518 with expected crc 2023-09-23T20:44:51.896 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:51 smithi018 ceph-mon[184144]: failed to encode map e518 with expected crc 2023-09-23T20:44:51.896 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:51 smithi018 ceph-mon[184144]: failed to encode map e518 with expected crc 2023-09-23T20:44:51.896 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:51 smithi018 ceph-mon[184144]: failed to encode map e518 with expected crc 2023-09-23T20:44:51.896 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:51 smithi018 ceph-mon[184144]: failed to encode map e518 with expected crc 2023-09-23T20:44:51.896 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:51 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/932502547' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:44:51.953 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:51.987 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:51 smithi149 ceph-mon[170394]: pgmap v1295: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 1.4 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 432 op/s; 329/67284 objects misplaced (0.489%); 4.4 MiB/s, 71 objects/s recovering 2023-09-23T20:44:51.987 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:51 smithi149 ceph-mon[170394]: osdmap e518: 8 total, 8 up, 8 in 2023-09-23T20:44:51.987 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:51 smithi149 ceph-mon[170394]: failed to encode map e518 with expected crc 2023-09-23T20:44:51.988 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:51 smithi149 ceph-mon[170394]: failed to encode map e518 with expected crc 2023-09-23T20:44:51.988 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:51 smithi149 ceph-mon[170394]: failed to encode map e518 with expected crc 2023-09-23T20:44:51.988 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:51 smithi149 ceph-mon[170394]: failed to encode map e518 with expected crc 2023-09-23T20:44:51.988 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:51 smithi149 ceph-mon[170394]: failed to encode map e518 with expected crc 2023-09-23T20:44:51.988 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:51 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/932502547' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:44:52.133 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:44:52.141 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:52.141 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:44:52.215 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:44:52.242 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:44:52.313 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:52.393 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:44:52.397 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:52.398 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:44:52.498 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:44:52.554 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:44:52.577 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:52.725 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:44:52.728 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:52.728 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:44:52.829 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:44:52.850 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:52 smithi018 ceph-mon[179647]: from='client.56410 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:44:52.850 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:52 smithi018 ceph-mon[179647]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:44:52.851 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:52 smithi018 ceph-mon[179647]: Cluster is now healthy 2023-09-23T20:44:52.851 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:52 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/932502547' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:44:52.851 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:52 smithi018 ceph-mon[179647]: osdmap e519: 8 total, 8 up, 8 in 2023-09-23T20:44:52.851 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:52 smithi018 ceph-mon[179647]: failed to encode map e519 with expected crc 2023-09-23T20:44:52.851 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:52 smithi018 ceph-mon[179647]: failed to encode map e519 with expected crc 2023-09-23T20:44:52.851 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:52 smithi018 ceph-mon[179647]: failed to encode map e519 with expected crc 2023-09-23T20:44:52.852 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:52 smithi018 ceph-mon[184144]: from='client.56410 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:44:52.852 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:52 smithi018 ceph-mon[184144]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:44:52.852 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:52 smithi018 ceph-mon[184144]: Cluster is now healthy 2023-09-23T20:44:52.852 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:52 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/932502547' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:44:52.852 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:52 smithi018 ceph-mon[184144]: osdmap e519: 8 total, 8 up, 8 in 2023-09-23T20:44:52.852 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:52 smithi018 ceph-mon[184144]: failed to encode map e519 with expected crc 2023-09-23T20:44:52.852 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:52 smithi018 ceph-mon[184144]: failed to encode map e519 with expected crc 2023-09-23T20:44:52.853 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:52 smithi018 ceph-mon[184144]: failed to encode map e519 with expected crc 2023-09-23T20:44:52.892 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:52.975 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:44:52.981 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:52.982 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:44:53.083 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:44:53.110 INFO:teuthology.orchestra.run.smithi149.stdout:ERROR: (22) Invalid argument 2023-09-23T20:44:53.110 INFO:teuthology.orchestra.run.smithi149.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:44:53.145 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:52 smithi149 ceph-mon[170394]: from='client.56410 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:44:53.145 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:52 smithi149 ceph-mon[170394]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:44:53.145 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:52 smithi149 ceph-mon[170394]: Cluster is now healthy 2023-09-23T20:44:53.145 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:52 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/932502547' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:44:53.145 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:52 smithi149 ceph-mon[170394]: osdmap e519: 8 total, 8 up, 8 in 2023-09-23T20:44:53.145 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:52 smithi149 ceph-mon[170394]: failed to encode map e519 with expected crc 2023-09-23T20:44:53.146 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:52 smithi149 ceph-mon[170394]: failed to encode map e519 with expected crc 2023-09-23T20:44:53.146 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:52 smithi149 ceph-mon[170394]: failed to encode map e519 with expected crc 2023-09-23T20:44:53.159 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:53.489 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:44:53.493 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:53.493 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:44:53.498 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:44:53.572 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T20:44:53.594 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:44:53.675 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:53.746 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:53 smithi018 ceph-mon[179647]: pgmap v1298: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 1.3 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 547 op/s; 329/65211 objects misplaced (0.505%); 4.4 MiB/s, 71 objects/s recovering 2023-09-23T20:44:53.757 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:44:53.762 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:53.763 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:44:53.863 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:44:53.921 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:53 smithi149 ceph-mon[170394]: pgmap v1298: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 1.3 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 547 op/s; 329/65211 objects misplaced (0.505%); 4.4 MiB/s, 71 objects/s recovering 2023-09-23T20:44:53.925 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:53.981 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:44:53.985 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:53.986 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:44:54.012 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:53 smithi018 ceph-mon[184144]: pgmap v1298: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 1.3 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 547 op/s; 329/65211 objects misplaced (0.505%); 4.4 MiB/s, 71 objects/s recovering 2023-09-23T20:44:54.087 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:44:54.155 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:54.193 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T20:44:54.197 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:44:54.202 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:54.203 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:44:54.303 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:44:54.347 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:54.382 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:44:54.400 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:44:54.404 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:54.404 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:44:54.505 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:44:54.531 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:54.625 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:44:54.625 INFO:teuthology.orchestra.run.smithi018.stdout:{"fsid":"a3b3b17e-5a4c-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_OK","checks":{},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1571,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":519,"num_osds":8,"num_up_osds":8,"osd_up_since":1695500984,"num_in_osds":8,"osd_in_since":1695501516,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":118},{"state_name":"active+remapped+backfilling","count":1}],"num_pgs":119,"num_pools":5,"num_objects":21737,"data_bytes":1427445707,"bytes_used":9828900864,"bytes_avail":758131384320,"bytes_total":767960285184,"misplaced_objects":329,"misplaced_total":65211,"misplaced_ratio":0.0050451610924537272,"recovering_objects_per_sec":71,"recovering_bytes_per_sec":4605742,"recovering_keys_per_sec":0,"num_objects_recovered":427,"num_bytes_recovered":27656192,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":547},"fsmap":{"epoch":9,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi018.hduiol","status":"up:active","gid":24566}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.18:8443/","prometheus":"http://172.21.15.18:9283/"}},"servicemap":{"epoch":23,"modified":"2023-09-23T20:34:43.862209+0000","services":{}},"progress_events":{"8e7eff18-0e45-4130-a6e1-662b3902f0d2":{"message":"Upgrade to 18.2.0-460-gbb0daeff (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2023-09-23T20:44:54.825 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:44:54.831 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:54.831 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:44:54.896 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:54 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3733877978' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T20:44:54.896 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:54 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3733877978' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T20:44:54.932 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:44:55.009 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:55.151 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:44:55.166 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:55.167 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:44:55.212 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-09-23T20:44:55.212 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:44:55.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:54 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3733877978' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T20:44:55.268 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:44:55.294 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:44:54 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:44:54] "GET /metrics HTTP/1.1" 200 36518 "" "Prometheus/2.43.0" 2023-09-23T20:44:55.346 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:55.666 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:44:55.810 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:44:55.829 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:44:55.836 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:55.836 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:44:55.858 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:44:55.899 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:44:55.937 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:44:55.953 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:44:55.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:55 smithi149 ceph-mon[170394]: pgmap v1299: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 1.2 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 654 op/s; 329/59025 objects misplaced (0.557%); 3.5 MiB/s, 56 objects/s recovering 2023-09-23T20:44:55.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:55 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:44:55.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:55 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/202189382' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:44:55.989 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:55.994 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:44:56.035 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:44:56.069 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:44:56.074 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:44:56.083 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:56.084 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:44:56.111 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:44:56.113 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:44:56.114 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:44:56.114 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:44:56.114 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:44:56.114 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:44:56.114 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:44:56.114 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:44:56.114 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:44:56.145 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:55 smithi018 ceph-mon[179647]: pgmap v1299: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 1.2 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 654 op/s; 329/59025 objects misplaced (0.557%); 3.5 MiB/s, 56 objects/s recovering 2023-09-23T20:44:56.145 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:55 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:44:56.146 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:55 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/202189382' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:44:56.146 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:55 smithi018 ceph-mon[184144]: pgmap v1299: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 1.2 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 654 op/s; 329/59025 objects misplaced (0.557%); 3.5 MiB/s, 56 objects/s recovering 2023-09-23T20:44:56.146 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:55 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:44:56.146 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:55 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/202189382' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:44:56.184 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:44:56.227 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:56.319 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:44:56.327 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:56.327 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:44:56.428 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:44:56.474 INFO:teuthology.orchestra.run.smithi149.stdout:{ 2023-09-23T20:44:56.474 INFO:teuthology.orchestra.run.smithi149.stdout: "size": 20, 2023-09-23T20:44:56.474 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 600, 2023-09-23T20:44:56.474 INFO:teuthology.orchestra.run.smithi149.stdout: "ops": [ 2023-09-23T20:44:56.474 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.474 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:84069 8.9 8:9824f39b:::benchmark_data_smithi018_341490_object84068:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e378)", 2023-09-23T20:44:56.475 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:37:13.327117+0000", 2023-09-23T20:44:56.475 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 463.14320664600001, 2023-09-23T20:44:56.475 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.3519245670000002, 2023-09-23T20:44:56.475 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:44:56.475 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:44:56.475 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:44:56.475 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:44:56.475 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:44:56.475 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 84069 2023-09-23T20:44:56.475 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.475 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:44:56.475 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.476 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:44:56.476 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.327117+0000", 2023-09-23T20:44:56.476 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:56.476 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.476 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.476 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:44:56.476 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.327117+0000", 2023-09-23T20:44:56.476 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:56.476 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.476 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.476 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:44:56.476 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.327116+0000", 2023-09-23T20:44:56.477 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:44:56.477 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.477 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.477 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:44:56.477 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.327148+0000", 2023-09-23T20:44:56.477 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.1741e-05 2023-09-23T20:44:56.477 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.477 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.477 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:44:56.477 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.327160+0000", 2023-09-23T20:44:56.477 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.185e-05 2023-09-23T20:44:56.477 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.477 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.478 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:44:56.478 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.327165+0000", 2023-09-23T20:44:56.478 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.1749999999999996e-06 2023-09-23T20:44:56.478 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.478 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.478 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:44:56.478 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.330726+0000", 2023-09-23T20:44:56.478 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00356121 2023-09-23T20:44:56.478 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.478 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.478 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:44:56.478 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.330874+0000", 2023-09-23T20:44:56.479 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000148097 2023-09-23T20:44:56.479 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.479 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.479 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:44:56.479 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.331021+0000", 2023-09-23T20:44:56.479 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00014648499999999999 2023-09-23T20:44:56.479 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.479 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.479 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:44:56.479 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.334676+0000", 2023-09-23T20:44:56.479 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0036551869999999998 2023-09-23T20:44:56.479 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.480 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.480 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:56.480 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.335154+0000", 2023-09-23T20:44:56.480 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00047863799999999998 2023-09-23T20:44:56.480 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.480 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.480 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:44:56.480 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.368539+0000", 2023-09-23T20:44:56.480 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.033384692000000001 2023-09-23T20:44:56.480 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.480 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.480 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:44:56.481 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.368550+0000", 2023-09-23T20:44:56.481 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1178e-05 2023-09-23T20:44:56.481 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.481 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.481 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:44:56.481 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:14.362533+0000", 2023-09-23T20:44:56.481 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.993982649 2023-09-23T20:44:56.481 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.481 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.481 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:44:56.481 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:14.362546+0000", 2023-09-23T20:44:56.481 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.2660000000000001e-05 2023-09-23T20:44:56.482 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.482 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.482 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:44:56.482 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:15.678962+0000", 2023-09-23T20:44:56.482 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.316415911 2023-09-23T20:44:56.482 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.482 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.482 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:44:56.482 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:15.679041+0000", 2023-09-23T20:44:56.482 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.9655999999999998e-05 2023-09-23T20:44:56.482 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:56.483 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:44:56.483 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:56.483 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.483 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.483 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:84081 8.9 8:97fd4d67:::benchmark_data_smithi018_341490_object84080:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e378)", 2023-09-23T20:44:56.483 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:37:13.341077+0000", 2023-09-23T20:44:56.483 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 463.12924657500002, 2023-09-23T20:44:56.483 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.3380652359999998, 2023-09-23T20:44:56.483 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:44:56.483 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:44:56.483 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:44:56.483 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:44:56.484 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:44:56.484 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 84081 2023-09-23T20:44:56.484 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.484 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:44:56.484 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.484 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:44:56.484 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.341077+0000", 2023-09-23T20:44:56.484 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:56.484 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.484 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.484 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:44:56.484 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.341077+0000", 2023-09-23T20:44:56.485 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:56.485 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.485 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.485 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:44:56.485 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.341075+0000", 2023-09-23T20:44:56.485 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999981 2023-09-23T20:44:56.485 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.485 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.485 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:44:56.485 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.341116+0000", 2023-09-23T20:44:56.485 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.1675000000000003e-05 2023-09-23T20:44:56.486 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.486 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.486 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:44:56.486 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.341135+0000", 2023-09-23T20:44:56.486 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.8110000000000001e-05 2023-09-23T20:44:56.486 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.486 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.486 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:44:56.486 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.341141+0000", 2023-09-23T20:44:56.486 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.849e-06 2023-09-23T20:44:56.486 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.486 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.487 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:44:56.487 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.341194+0000", 2023-09-23T20:44:56.487 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.2553000000000003e-05 2023-09-23T20:44:56.487 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.487 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.487 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:44:56.487 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.341313+0000", 2023-09-23T20:44:56.487 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00011935899999999999 2023-09-23T20:44:56.487 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.487 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.487 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:44:56.487 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.341466+0000", 2023-09-23T20:44:56.488 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00015230300000000001 2023-09-23T20:44:56.488 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.488 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.488 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:56.488 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.344704+0000", 2023-09-23T20:44:56.488 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0032382510000000002 2023-09-23T20:44:56.488 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.488 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.488 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:44:56.488 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.348055+0000", 2023-09-23T20:44:56.488 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.003350872 2023-09-23T20:44:56.488 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.489 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.489 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:44:56.489 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.368594+0000", 2023-09-23T20:44:56.489 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.020538948000000001 2023-09-23T20:44:56.489 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.489 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.489 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:44:56.489 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.368601+0000", 2023-09-23T20:44:56.489 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.6529999999999994e-06 2023-09-23T20:44:56.489 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.489 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.489 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:44:56.490 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:14.362580+0000", 2023-09-23T20:44:56.490 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.99397867299999998 2023-09-23T20:44:56.490 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.490 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.490 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:44:56.490 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:14.362589+0000", 2023-09-23T20:44:56.490 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.9509999999999995e-06 2023-09-23T20:44:56.490 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.490 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.490 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:44:56.490 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:15.679073+0000", 2023-09-23T20:44:56.490 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.316483598 2023-09-23T20:44:56.491 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.491 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.491 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:44:56.491 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:15.679142+0000", 2023-09-23T20:44:56.491 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.9389999999999998e-05 2023-09-23T20:44:56.491 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:56.491 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:44:56.491 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:56.491 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.491 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.491 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:105775 8.9 8:97ec68b7:::benchmark_data_smithi018_341490_object16452:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e411)", 2023-09-23T20:44:56.491 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:38:27.408390+0000", 2023-09-23T20:44:56.492 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 389.06193278299997, 2023-09-23T20:44:56.492 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.1501579759999991, 2023-09-23T20:44:56.492 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:44:56.492 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:44:56.492 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:44:56.492 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:44:56.492 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:44:56.492 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 105775 2023-09-23T20:44:56.492 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.492 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:44:56.492 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.492 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:44:56.493 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.408390+0000", 2023-09-23T20:44:56.493 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:56.493 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.493 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.493 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:44:56.493 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.408390+0000", 2023-09-23T20:44:56.493 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:56.493 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.493 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.493 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:44:56.493 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.408389+0000", 2023-09-23T20:44:56.494 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:44:56.494 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.494 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.494 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:44:56.494 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.408397+0000", 2023-09-23T20:44:56.494 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.7009999999999996e-06 2023-09-23T20:44:56.494 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.494 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.494 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:44:56.494 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.408411+0000", 2023-09-23T20:44:56.494 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.4228e-05 2023-09-23T20:44:56.494 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.495 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.495 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:44:56.495 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.408419+0000", 2023-09-23T20:44:56.495 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.5599999999999994e-06 2023-09-23T20:44:56.495 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.495 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.495 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:44:56.495 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.414829+0000", 2023-09-23T20:44:56.495 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0064094740000000001 2023-09-23T20:44:56.495 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.495 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.495 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:44:56.496 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.414843+0000", 2023-09-23T20:44:56.496 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.4561e-05 2023-09-23T20:44:56.496 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.496 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.496 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:44:56.496 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.556066+0000", 2023-09-23T20:44:56.496 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.1412221710000008 2023-09-23T20:44:56.496 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.496 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.496 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:44:56.496 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.558548+0000", 2023-09-23T20:44:56.496 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0024828070000000001 2023-09-23T20:44:56.497 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:56.497 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:44:56.497 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:56.497 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.497 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.497 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:105788 8.9 8:94f5ae14:::benchmark_data_smithi018_341490_object16465:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e411)", 2023-09-23T20:44:56.497 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:38:27.415518+0000", 2023-09-23T20:44:56.497 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 389.05480486699997, 2023-09-23T20:44:56.497 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 10.492815438999999, 2023-09-23T20:44:56.497 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:44:56.497 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:44:56.498 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:44:56.498 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:44:56.498 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:44:56.498 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 105788 2023-09-23T20:44:56.498 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.498 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:44:56.498 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.498 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:44:56.498 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.415518+0000", 2023-09-23T20:44:56.498 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:56.498 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.498 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.499 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:44:56.499 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.415518+0000", 2023-09-23T20:44:56.499 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:56.499 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.499 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.499 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:44:56.499 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.415517+0000", 2023-09-23T20:44:56.499 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:44:56.499 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.499 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.499 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:44:56.500 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.415524+0000", 2023-09-23T20:44:56.500 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.0530000000000001e-06 2023-09-23T20:44:56.500 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.500 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.500 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:44:56.500 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.415539+0000", 2023-09-23T20:44:56.500 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.4847e-05 2023-09-23T20:44:56.500 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.500 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.500 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:44:56.500 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.415548+0000", 2023-09-23T20:44:56.500 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.1600000000000004e-06 2023-09-23T20:44:56.500 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.501 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.501 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:44:56.501 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.423501+0000", 2023-09-23T20:44:56.501 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0079528459999999995 2023-09-23T20:44:56.501 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.501 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.501 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:44:56.501 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.423513+0000", 2023-09-23T20:44:56.501 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.2523e-05 2023-09-23T20:44:56.501 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.501 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.502 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:44:56.502 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.559074+0000", 2023-09-23T20:44:56.502 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.1355603290000005 2023-09-23T20:44:56.502 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.502 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.502 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for degraded object", 2023-09-23T20:44:56.502 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.567834+0000", 2023-09-23T20:44:56.502 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0087603060000000007 2023-09-23T20:44:56.502 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.502 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.502 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:44:56.503 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.866509+0000", 2023-09-23T20:44:56.503 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.2986752469999998 2023-09-23T20:44:56.503 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.503 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.503 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:44:56.503 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.869596+0000", 2023-09-23T20:44:56.503 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0030866829999999998 2023-09-23T20:44:56.503 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.503 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.503 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:44:56.503 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.875831+0000", 2023-09-23T20:44:56.503 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0062352570000000001 2023-09-23T20:44:56.504 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.504 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.504 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:44:56.504 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.893196+0000", 2023-09-23T20:44:56.504 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.017365463000000001 2023-09-23T20:44:56.504 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.504 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.504 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:56.504 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.898820+0000", 2023-09-23T20:44:56.504 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0056230400000000002 2023-09-23T20:44:56.504 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.504 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.505 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:56.505 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.899854+0000", 2023-09-23T20:44:56.505 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0010348670000000001 2023-09-23T20:44:56.505 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.505 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.505 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:44:56.505 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.901497+0000", 2023-09-23T20:44:56.505 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0016428160000000001 2023-09-23T20:44:56.505 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.505 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.505 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:44:56.505 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.908334+0000", 2023-09-23T20:44:56.506 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0068366219999999997 2023-09-23T20:44:56.506 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:56.506 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:44:56.506 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:56.506 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.506 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.506 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:105815 8.19 8:9f25a704:::benchmark_data_smithi018_341490_object16492:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e411)", 2023-09-23T20:44:56.506 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:38:27.621206+0000", 2023-09-23T20:44:56.506 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 388.84911764700001, 2023-09-23T20:44:56.506 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 10.285912701000001, 2023-09-23T20:44:56.506 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:44:56.507 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:44:56.507 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:44:56.507 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:44:56.507 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:44:56.507 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 105815 2023-09-23T20:44:56.507 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.507 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:44:56.507 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.507 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:44:56.507 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.621206+0000", 2023-09-23T20:44:56.507 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:56.507 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.507 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.508 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:44:56.508 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.621206+0000", 2023-09-23T20:44:56.508 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:56.508 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.508 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.508 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:44:56.508 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.621204+0000", 2023-09-23T20:44:56.508 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999981 2023-09-23T20:44:56.508 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.508 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.508 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:44:56.508 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.621213+0000", 2023-09-23T20:44:56.508 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.6399999999999992e-06 2023-09-23T20:44:56.509 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.509 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.509 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:44:56.509 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.621236+0000", 2023-09-23T20:44:56.509 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.2016000000000001e-05 2023-09-23T20:44:56.509 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.509 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.509 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:44:56.509 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.621244+0000", 2023-09-23T20:44:56.509 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.7339999999999997e-06 2023-09-23T20:44:56.509 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.509 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.509 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:44:56.510 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.621311+0000", 2023-09-23T20:44:56.510 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.7037000000000002e-05 2023-09-23T20:44:56.510 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.510 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.510 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:44:56.510 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.621338+0000", 2023-09-23T20:44:56.510 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.6506999999999999e-05 2023-09-23T20:44:56.510 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.510 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.510 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:44:56.510 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.559059+0000", 2023-09-23T20:44:56.511 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.9377208670000003 2023-09-23T20:44:56.511 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.511 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.511 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for degraded object", 2023-09-23T20:44:56.511 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.566949+0000", 2023-09-23T20:44:56.511 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0078899190000000004 2023-09-23T20:44:56.511 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.511 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.511 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:44:56.511 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.857769+0000", 2023-09-23T20:44:56.511 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.290820509 2023-09-23T20:44:56.511 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.512 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.512 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:44:56.512 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.860457+0000", 2023-09-23T20:44:56.512 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0026880620000000002 2023-09-23T20:44:56.512 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.512 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.512 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:44:56.512 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.865930+0000", 2023-09-23T20:44:56.512 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0054732979999999997 2023-09-23T20:44:56.512 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.512 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.512 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:44:56.512 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.881728+0000", 2023-09-23T20:44:56.513 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.015797813000000001 2023-09-23T20:44:56.513 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.513 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.513 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:56.513 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.897756+0000", 2023-09-23T20:44:56.513 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.016027501 2023-09-23T20:44:56.513 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.513 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.513 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:56.513 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.898850+0000", 2023-09-23T20:44:56.513 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.001094292 2023-09-23T20:44:56.514 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.514 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.514 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:44:56.514 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.900364+0000", 2023-09-23T20:44:56.514 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0015141250000000001 2023-09-23T20:44:56.514 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.514 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.514 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:44:56.514 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.907118+0000", 2023-09-23T20:44:56.514 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0067541490000000001 2023-09-23T20:44:56.514 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:56.514 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:44:56.515 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:56.515 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.515 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.515 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:105818 8.19 8:9e8d5bb0:::benchmark_data_smithi018_341490_object16495:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e411)", 2023-09-23T20:44:56.515 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:38:27.629200+0000", 2023-09-23T20:44:56.515 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 388.84112356600002, 2023-09-23T20:44:56.515 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 10.286819241, 2023-09-23T20:44:56.515 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:44:56.515 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:44:56.515 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:44:56.515 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:44:56.515 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:44:56.516 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 105818 2023-09-23T20:44:56.516 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.516 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:44:56.516 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.516 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:44:56.516 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.629200+0000", 2023-09-23T20:44:56.516 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:56.516 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.516 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.516 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:44:56.516 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.629200+0000", 2023-09-23T20:44:56.516 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:56.517 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.517 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.517 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:44:56.517 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.629198+0000", 2023-09-23T20:44:56.517 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999981 2023-09-23T20:44:56.517 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.517 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.517 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:44:56.517 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.629208+0000", 2023-09-23T20:44:56.517 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.9119999999999993e-06 2023-09-23T20:44:56.517 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.517 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.518 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:44:56.518 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.629231+0000", 2023-09-23T20:44:56.518 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.3258999999999999e-05 2023-09-23T20:44:56.518 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.518 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.518 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:44:56.518 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.629240+0000", 2023-09-23T20:44:56.518 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.5719999999999992e-06 2023-09-23T20:44:56.518 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.518 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.518 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:44:56.519 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.629296+0000", 2023-09-23T20:44:56.519 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.5918999999999998e-05 2023-09-23T20:44:56.519 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.519 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.519 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:44:56.519 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.629305+0000", 2023-09-23T20:44:56.519 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.0909999999999999e-06 2023-09-23T20:44:56.519 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.519 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.519 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:44:56.519 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.567292+0000", 2023-09-23T20:44:56.519 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.937986414 2023-09-23T20:44:56.520 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.520 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.520 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for degraded object", 2023-09-23T20:44:56.520 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.576261+0000", 2023-09-23T20:44:56.520 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0089692720000000004 2023-09-23T20:44:56.520 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.520 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.520 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:44:56.520 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.882072+0000", 2023-09-23T20:44:56.520 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.3058112469999998 2023-09-23T20:44:56.520 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.520 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.520 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:44:56.521 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.884583+0000", 2023-09-23T20:44:56.521 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0025109939999999999 2023-09-23T20:44:56.521 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.521 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.521 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:44:56.521 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.889724+0000", 2023-09-23T20:44:56.521 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0051411909999999998 2023-09-23T20:44:56.521 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.521 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.521 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:44:56.521 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.894676+0000", 2023-09-23T20:44:56.521 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0049515119999999999 2023-09-23T20:44:56.522 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.522 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.522 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:56.522 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.908009+0000", 2023-09-23T20:44:56.522 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.013332899 2023-09-23T20:44:56.522 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.522 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.522 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:56.522 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.908929+0000", 2023-09-23T20:44:56.522 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00092006300000000004 2023-09-23T20:44:56.522 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.523 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.523 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:44:56.523 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.910490+0000", 2023-09-23T20:44:56.523 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.001560931 2023-09-23T20:44:56.523 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.523 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.523 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:44:56.523 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.916019+0000", 2023-09-23T20:44:56.523 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0055290570000000004 2023-09-23T20:44:56.523 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:56.523 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:44:56.524 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:56.524 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.524 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.524 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "MOSDPGPushReply(8.19 413/410 [PushReplyOp(8:9f25a704:::benchmark_data_smithi018_341490_object16492:head)])", 2023-09-23T20:44:56.524 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:38:35.569213+0000", 2023-09-23T20:44:56.524 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 380.90111027900002, 2023-09-23T20:44:56.524 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.2881636309999998, 2023-09-23T20:44:56.524 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:44:56.524 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:44:56.524 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:44:56.524 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.525 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:44:56.525 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.569213+0000", 2023-09-23T20:44:56.525 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:56.525 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.525 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.525 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:44:56.525 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.569213+0000", 2023-09-23T20:44:56.525 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:56.525 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.525 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.525 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:44:56.526 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.569213+0000", 2023-09-23T20:44:56.526 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:44:56.526 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.526 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.526 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:44:56.526 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.569217+0000", 2023-09-23T20:44:56.526 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.5480000000000003e-06 2023-09-23T20:44:56.526 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.526 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.526 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:44:56.526 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.569231+0000", 2023-09-23T20:44:56.526 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.3505e-05 2023-09-23T20:44:56.527 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.527 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.527 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:44:56.527 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.569241+0000", 2023-09-23T20:44:56.527 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0695e-05 2023-09-23T20:44:56.527 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.527 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.527 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:44:56.527 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.845386+0000", 2023-09-23T20:44:56.527 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.2761449049999998 2023-09-23T20:44:56.527 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.527 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.528 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:44:56.528 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.857377+0000", 2023-09-23T20:44:56.528 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.011990341999999999 2023-09-23T20:44:56.528 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:56.528 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:44:56.528 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:56.528 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.528 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.528 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "MOSDPGPushReply(8.9 413/411 [PushReplyOp(8:94f5ae14:::benchmark_data_smithi018_341490_object16465:head)])", 2023-09-23T20:44:56.528 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:38:35.570981+0000", 2023-09-23T20:44:56.528 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 380.89934209500001, 2023-09-23T20:44:56.529 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.295177131, 2023-09-23T20:44:56.529 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:44:56.529 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:44:56.529 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:44:56.529 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.529 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:44:56.529 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.570981+0000", 2023-09-23T20:44:56.529 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:56.529 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.529 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.529 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:44:56.529 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.570981+0000", 2023-09-23T20:44:56.530 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:56.530 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.530 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.530 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:44:56.530 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.570981+0000", 2023-09-23T20:44:56.530 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:44:56.530 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.530 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.530 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:44:56.530 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.570985+0000", 2023-09-23T20:44:56.530 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.2690000000000001e-06 2023-09-23T20:44:56.530 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.531 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.531 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:44:56.531 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.570995+0000", 2023-09-23T20:44:56.531 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.9799999999999993e-06 2023-09-23T20:44:56.531 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.531 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.531 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:44:56.531 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.570999+0000", 2023-09-23T20:44:56.531 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.5739999999999997e-06 2023-09-23T20:44:56.531 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.531 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.531 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:44:56.532 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.852442+0000", 2023-09-23T20:44:56.532 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.2814426320000001 2023-09-23T20:44:56.532 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.532 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.532 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:44:56.532 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.866158+0000", 2023-09-23T20:44:56.532 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.013716178000000001 2023-09-23T20:44:56.532 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:56.532 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:44:56.532 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:56.532 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.532 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.533 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "MOSDPGPushReply(8.19 413/410 [PushReplyOp(8:9e8d5bb0:::benchmark_data_smithi018_341490_object16495:head)])", 2023-09-23T20:44:56.533 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:38:35.577963+0000", 2023-09-23T20:44:56.533 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 380.89235993800003, 2023-09-23T20:44:56.533 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.303767224, 2023-09-23T20:44:56.533 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:44:56.533 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:44:56.533 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:44:56.533 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.533 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:44:56.533 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.577963+0000", 2023-09-23T20:44:56.533 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:56.534 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.534 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.534 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:44:56.534 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.577963+0000", 2023-09-23T20:44:56.534 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:56.534 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.534 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.534 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:44:56.534 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.577963+0000", 2023-09-23T20:44:56.534 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:44:56.534 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.534 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.535 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:44:56.535 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.577968+0000", 2023-09-23T20:44:56.535 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.13e-06 2023-09-23T20:44:56.535 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.535 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.535 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:44:56.535 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.577980+0000", 2023-09-23T20:44:56.535 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1749000000000001e-05 2023-09-23T20:44:56.535 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.535 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.535 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:44:56.535 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.577986+0000", 2023-09-23T20:44:56.536 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.8210000000000004e-06 2023-09-23T20:44:56.536 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.536 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.536 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:44:56.536 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.869571+0000", 2023-09-23T20:44:56.536 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.2915846719999999 2023-09-23T20:44:56.536 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.536 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.536 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:44:56.536 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.881731+0000", 2023-09-23T20:44:56.536 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.012159467 2023-09-23T20:44:56.536 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:56.537 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:44:56.537 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:56.537 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.537 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.537 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:106043 8.9 8:949649ba:::benchmark_data_smithi018_341490_object16720:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e414)", 2023-09-23T20:44:56.537 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:38:35.582779+0000", 2023-09-23T20:44:56.537 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 380.88754424400003, 2023-09-23T20:44:56.537 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.7561004659999999, 2023-09-23T20:44:56.537 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:44:56.537 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:44:56.537 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:44:56.537 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:44:56.538 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:44:56.538 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 106043 2023-09-23T20:44:56.538 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.538 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:44:56.538 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.538 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:44:56.538 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.582779+0000", 2023-09-23T20:44:56.538 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:56.538 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.538 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.538 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:44:56.538 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.582779+0000", 2023-09-23T20:44:56.539 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:56.539 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.539 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.539 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:44:56.539 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.582778+0000", 2023-09-23T20:44:56.539 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:44:56.539 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.539 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.539 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:44:56.539 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.582784+0000", 2023-09-23T20:44:56.539 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.0050000000000001e-06 2023-09-23T20:44:56.540 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.540 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.540 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:44:56.540 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.582798+0000", 2023-09-23T20:44:56.540 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.3788000000000001e-05 2023-09-23T20:44:56.540 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.540 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.540 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:44:56.540 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.582804+0000", 2023-09-23T20:44:56.540 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.6760000000000001e-06 2023-09-23T20:44:56.540 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.541 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.541 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:44:56.541 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.238120+0000", 2023-09-23T20:44:56.541 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.655315308 2023-09-23T20:44:56.541 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.541 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.541 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for degraded object", 2023-09-23T20:44:56.541 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.246651+0000", 2023-09-23T20:44:56.541 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0085312070000000007 2023-09-23T20:44:56.541 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.541 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.541 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:44:56.542 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.290876+0000", 2023-09-23T20:44:56.542 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.044225546999999997 2023-09-23T20:44:56.542 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.542 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.542 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:44:56.542 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.294002+0000", 2023-09-23T20:44:56.542 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0031256779999999998 2023-09-23T20:44:56.542 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.542 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.542 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:44:56.542 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.300293+0000", 2023-09-23T20:44:56.543 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0062908959999999998 2023-09-23T20:44:56.543 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.543 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.543 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:44:56.543 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.328008+0000", 2023-09-23T20:44:56.543 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.027715281000000001 2023-09-23T20:44:56.543 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.543 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.543 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:56.543 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.329088+0000", 2023-09-23T20:44:56.543 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.001079775 2023-09-23T20:44:56.543 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.544 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.544 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:56.544 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.330303+0000", 2023-09-23T20:44:56.544 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0012148199999999999 2023-09-23T20:44:56.544 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.544 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.544 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:44:56.544 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.332003+0000", 2023-09-23T20:44:56.544 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0016998580000000001 2023-09-23T20:44:56.544 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.544 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.544 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:44:56.545 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.338879+0000", 2023-09-23T20:44:56.545 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0068769119999999998 2023-09-23T20:44:56.545 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:56.545 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:44:56.545 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:56.545 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.545 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.545 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "MOSDPGPush(8.19 427/410 [PushOp(8:9f37acf0:::benchmark_data_smithi018_341490_object84819:head, version: 394'5338, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(8:9f37acf0:::benchmark_data_smithi018_341490_object84819:head@394'5338, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-09-23T20:44:56.545 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:39:26.795989+0000", 2023-09-23T20:44:56.545 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 329.67433407800002, 2023-09-23T20:44:56.545 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 18.381641347999999, 2023-09-23T20:44:56.545 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:44:56.546 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "started", 2023-09-23T20:44:56.546 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:44:56.546 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.546 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:44:56.546 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:26.795989+0000", 2023-09-23T20:44:56.546 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:56.546 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.546 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.546 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:44:56.546 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:26.795989+0000", 2023-09-23T20:44:56.546 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:56.547 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.547 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.547 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:44:56.547 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:26.795988+0000", 2023-09-23T20:44:56.547 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:44:56.547 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.547 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.547 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:44:56.547 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:26.796034+0000", 2023-09-23T20:44:56.547 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.5374999999999998e-05 2023-09-23T20:44:56.547 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.548 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.548 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:44:56.548 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:26.796055+0000", 2023-09-23T20:44:56.548 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.126e-05 2023-09-23T20:44:56.548 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.548 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.548 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:44:56.548 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:26.796067+0000", 2023-09-23T20:44:56.548 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.2043e-05 2023-09-23T20:44:56.548 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.548 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.548 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:44:56.549 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:45.167841+0000", 2023-09-23T20:44:56.549 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 18.371773717 2023-09-23T20:44:56.549 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.549 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.549 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:44:56.549 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:45.168079+0000", 2023-09-23T20:44:56.549 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000237839 2023-09-23T20:44:56.549 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.549 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.549 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:44:56.549 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:45.177631+0000", 2023-09-23T20:44:56.549 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0095519390000000006 2023-09-23T20:44:56.550 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:56.550 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:44:56.550 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:56.550 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.550 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.550 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "MOSDPGPush(8.19 435/410 [PushOp(8:9a33faf0:::benchmark_data_smithi018_341490_object85471:head, version: 395'5385, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(8:9a33faf0:::benchmark_data_smithi018_341490_object85471:head@395'5385, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-09-23T20:44:56.550 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:39:53.671016+0000", 2023-09-23T20:44:56.550 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 302.79930721900001, 2023-09-23T20:44:56.550 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.82055882099999999, 2023-09-23T20:44:56.550 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:44:56.550 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "started", 2023-09-23T20:44:56.551 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:44:56.551 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.551 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:44:56.551 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:53.671016+0000", 2023-09-23T20:44:56.551 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:56.551 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.551 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.551 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:44:56.551 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:53.671016+0000", 2023-09-23T20:44:56.551 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:56.551 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.551 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.552 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:44:56.552 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:53.671015+0000", 2023-09-23T20:44:56.552 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:44:56.552 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.552 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.552 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:44:56.552 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:53.671046+0000", 2023-09-23T20:44:56.552 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.0802000000000001e-05 2023-09-23T20:44:56.552 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.552 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.552 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:44:56.552 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:53.671061+0000", 2023-09-23T20:44:56.553 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.5446000000000001e-05 2023-09-23T20:44:56.553 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.553 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.553 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:44:56.553 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:53.671069+0000", 2023-09-23T20:44:56.553 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.9519999999999994e-06 2023-09-23T20:44:56.553 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.553 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.553 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:44:56.553 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:54.483764+0000", 2023-09-23T20:44:56.553 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.812694429 2023-09-23T20:44:56.554 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.554 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.554 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:44:56.554 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:54.483953+0000", 2023-09-23T20:44:56.554 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00018980099999999999 2023-09-23T20:44:56.554 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.554 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.554 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:44:56.554 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:54.491575+0000", 2023-09-23T20:44:56.554 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0076214999999999998 2023-09-23T20:44:56.554 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:56.554 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:44:56.555 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:56.555 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.555 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.555 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:76949 9.b 9:dc3199d0:::benchmark_data_smithi018_381516_object76948:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e465)", 2023-09-23T20:44:56.555 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:41:58.481021+0000", 2023-09-23T20:44:56.555 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 177.98930226900001, 2023-09-23T20:44:56.555 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.012473631, 2023-09-23T20:44:56.555 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:44:56.555 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:44:56.555 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:44:56.555 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:44:56.555 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:44:56.556 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 76949 2023-09-23T20:44:56.556 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.556 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:44:56.556 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.556 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:44:56.556 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.481021+0000", 2023-09-23T20:44:56.556 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:56.556 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.556 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.556 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:44:56.556 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.481021+0000", 2023-09-23T20:44:56.557 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:56.557 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.557 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.557 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:44:56.557 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.481020+0000", 2023-09-23T20:44:56.557 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:44:56.557 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.557 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.557 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:44:56.557 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.481102+0000", 2023-09-23T20:44:56.557 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.2538000000000002e-05 2023-09-23T20:44:56.557 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.557 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.558 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:44:56.558 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.481122+0000", 2023-09-23T20:44:56.558 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.9556e-05 2023-09-23T20:44:56.558 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.558 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.558 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:44:56.558 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.481129+0000", 2023-09-23T20:44:56.558 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.7449999999999995e-06 2023-09-23T20:44:56.558 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.558 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.558 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:44:56.559 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.481713+0000", 2023-09-23T20:44:56.559 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000583167 2023-09-23T20:44:56.559 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.559 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.559 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:44:56.559 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.481726+0000", 2023-09-23T20:44:56.559 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.3291e-05 2023-09-23T20:44:56.559 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.559 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.559 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:44:56.559 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.467637+0000", 2023-09-23T20:44:56.559 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.98591135500000004 2023-09-23T20:44:56.559 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.560 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.560 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:44:56.560 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.470079+0000", 2023-09-23T20:44:56.560 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0024414089999999999 2023-09-23T20:44:56.560 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.560 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.560 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:44:56.560 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.470267+0000", 2023-09-23T20:44:56.560 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00018824299999999999 2023-09-23T20:44:56.560 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.560 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.560 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:44:56.561 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.481273+0000", 2023-09-23T20:44:56.561 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.011006395 2023-09-23T20:44:56.561 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.561 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.561 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:56.561 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.493371+0000", 2023-09-23T20:44:56.561 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.012098138 2023-09-23T20:44:56.561 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.561 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.561 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:56.561 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.493433+0000", 2023-09-23T20:44:56.561 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.1437000000000001e-05 2023-09-23T20:44:56.562 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.562 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.562 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:44:56.562 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.493472+0000", 2023-09-23T20:44:56.562 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.9038000000000001e-05 2023-09-23T20:44:56.562 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.562 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.562 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:44:56.562 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.493495+0000", 2023-09-23T20:44:56.562 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.2778000000000001e-05 2023-09-23T20:44:56.562 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:56.562 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:44:56.563 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:56.563 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.563 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.563 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:76967 9.b 9:d898a916:::benchmark_data_smithi018_381516_object76966:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e465)", 2023-09-23T20:44:56.563 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:41:58.507229+0000", 2023-09-23T20:44:56.563 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 177.963093859, 2023-09-23T20:44:56.563 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.98635601299999998, 2023-09-23T20:44:56.563 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:44:56.563 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:44:56.563 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:44:56.563 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:44:56.564 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:44:56.564 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 76967 2023-09-23T20:44:56.564 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.564 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:44:56.564 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.564 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:44:56.564 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.507229+0000", 2023-09-23T20:44:56.564 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:56.564 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.564 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.564 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:44:56.564 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.507229+0000", 2023-09-23T20:44:56.565 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:56.565 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.565 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.565 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:44:56.565 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.507228+0000", 2023-09-23T20:44:56.565 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999981 2023-09-23T20:44:56.565 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.565 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.565 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:44:56.565 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.507285+0000", 2023-09-23T20:44:56.565 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.7531000000000002e-05 2023-09-23T20:44:56.565 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.566 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.566 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:44:56.566 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.507307+0000", 2023-09-23T20:44:56.566 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.2039e-05 2023-09-23T20:44:56.566 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.566 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.566 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:44:56.566 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.507314+0000", 2023-09-23T20:44:56.566 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.7089999999999996e-06 2023-09-23T20:44:56.566 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.566 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.567 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:44:56.567 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.507629+0000", 2023-09-23T20:44:56.567 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00031477799999999999 2023-09-23T20:44:56.567 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.567 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.567 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:44:56.567 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.507641+0000", 2023-09-23T20:44:56.567 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.258e-05 2023-09-23T20:44:56.567 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.567 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.567 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:44:56.567 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.472754+0000", 2023-09-23T20:44:56.568 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.96511288200000001 2023-09-23T20:44:56.568 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.568 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.568 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:44:56.568 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.472887+0000", 2023-09-23T20:44:56.568 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000133132 2023-09-23T20:44:56.568 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.568 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.568 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:44:56.568 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.472997+0000", 2023-09-23T20:44:56.568 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000109805 2023-09-23T20:44:56.568 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.569 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.569 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:44:56.569 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.484044+0000", 2023-09-23T20:44:56.569 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.011047250999999999 2023-09-23T20:44:56.569 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.569 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.569 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:56.569 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.493403+0000", 2023-09-23T20:44:56.569 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0093591230000000004 2023-09-23T20:44:56.569 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.569 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.570 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:56.570 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.493533+0000", 2023-09-23T20:44:56.570 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000129271 2023-09-23T20:44:56.570 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.570 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.570 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:44:56.570 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.493558+0000", 2023-09-23T20:44:56.570 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.5477e-05 2023-09-23T20:44:56.570 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.570 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.570 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:44:56.570 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.493585+0000", 2023-09-23T20:44:56.571 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.7274e-05 2023-09-23T20:44:56.571 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:56.571 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:44:56.571 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:56.571 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.571 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.571 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:76968 9.b 9:d3229d21:::benchmark_data_smithi018_381516_object76967:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e465)", 2023-09-23T20:44:56.571 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:41:58.507386+0000", 2023-09-23T20:44:56.571 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 177.962936926, 2023-09-23T20:44:56.571 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.98627873399999999, 2023-09-23T20:44:56.572 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:44:56.572 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:44:56.572 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:44:56.572 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:44:56.572 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:44:56.572 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 76968 2023-09-23T20:44:56.572 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.572 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:44:56.572 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.572 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:44:56.572 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.507386+0000", 2023-09-23T20:44:56.572 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:56.573 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.573 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.573 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:44:56.573 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.507386+0000", 2023-09-23T20:44:56.573 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:56.573 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.573 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.573 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:44:56.573 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.507386+0000", 2023-09-23T20:44:56.573 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:44:56.573 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.574 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.574 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:44:56.574 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.507447+0000", 2023-09-23T20:44:56.574 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.1439999999999995e-05 2023-09-23T20:44:56.574 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.574 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.574 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:44:56.574 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.507457+0000", 2023-09-23T20:44:56.574 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.4679999999999998e-06 2023-09-23T20:44:56.574 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.574 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.574 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:44:56.575 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.507463+0000", 2023-09-23T20:44:56.575 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.8440000000000003e-06 2023-09-23T20:44:56.575 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.575 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.575 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:44:56.575 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.507659+0000", 2023-09-23T20:44:56.575 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000196691 2023-09-23T20:44:56.575 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.575 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.575 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:44:56.575 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.507663+0000", 2023-09-23T20:44:56.575 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.0040000000000001e-06 2023-09-23T20:44:56.576 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.576 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.576 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:44:56.576 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.475638+0000", 2023-09-23T20:44:56.576 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.96797460700000004 2023-09-23T20:44:56.576 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.576 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.576 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:44:56.576 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.475775+0000", 2023-09-23T20:44:56.576 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00013739899999999999 2023-09-23T20:44:56.576 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.576 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.577 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:44:56.577 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.475934+0000", 2023-09-23T20:44:56.577 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00015842499999999999 2023-09-23T20:44:56.577 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.577 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.577 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:44:56.577 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.484048+0000", 2023-09-23T20:44:56.577 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0081147540000000001 2023-09-23T20:44:56.577 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.577 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.577 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:56.577 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.493516+0000", 2023-09-23T20:44:56.578 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0094679240000000008 2023-09-23T20:44:56.578 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.578 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.578 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:56.578 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.493605+0000", 2023-09-23T20:44:56.578 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.8725000000000002e-05 2023-09-23T20:44:56.578 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.578 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.578 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:44:56.578 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.493633+0000", 2023-09-23T20:44:56.578 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.8e-05 2023-09-23T20:44:56.578 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.579 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.579 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:44:56.579 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.493665+0000", 2023-09-23T20:44:56.579 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.2017000000000002e-05 2023-09-23T20:44:56.579 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:56.579 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:44:56.579 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:56.579 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.579 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.579 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:76972 9.b 9:d979209f:::benchmark_data_smithi018_381516_object76971:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e465)", 2023-09-23T20:44:56.579 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:41:58.507469+0000", 2023-09-23T20:44:56.579 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 177.962854327, 2023-09-23T20:44:56.580 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.98628621599999999, 2023-09-23T20:44:56.580 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:44:56.580 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:44:56.580 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:44:56.580 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:44:56.580 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:44:56.580 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 76972 2023-09-23T20:44:56.580 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.580 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:44:56.580 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.580 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:44:56.580 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.507469+0000", 2023-09-23T20:44:56.581 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:56.581 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.581 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.581 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:44:56.581 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.507469+0000", 2023-09-23T20:44:56.581 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:56.581 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.581 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.581 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:44:56.581 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.507468+0000", 2023-09-23T20:44:56.581 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:44:56.581 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.582 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.582 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:44:56.582 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.507525+0000", 2023-09-23T20:44:56.582 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.6431000000000002e-05 2023-09-23T20:44:56.582 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.582 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.582 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:44:56.582 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.507533+0000", 2023-09-23T20:44:56.582 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.2400000000000007e-06 2023-09-23T20:44:56.582 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.582 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.582 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:44:56.583 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.507538+0000", 2023-09-23T20:44:56.583 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.2129999999999999e-06 2023-09-23T20:44:56.583 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.583 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.583 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:44:56.583 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.507674+0000", 2023-09-23T20:44:56.583 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00013521599999999999 2023-09-23T20:44:56.583 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.583 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.583 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:44:56.583 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.507677+0000", 2023-09-23T20:44:56.584 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.3409999999999999e-06 2023-09-23T20:44:56.584 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.584 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.584 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:44:56.584 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.478437+0000", 2023-09-23T20:44:56.584 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.97076035400000005 2023-09-23T20:44:56.584 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.584 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.584 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:44:56.584 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.478570+0000", 2023-09-23T20:44:56.584 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00013248599999999999 2023-09-23T20:44:56.584 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.585 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.585 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:44:56.585 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.478661+0000", 2023-09-23T20:44:56.585 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.1013000000000004e-05 2023-09-23T20:44:56.585 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.585 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.585 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:44:56.585 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.486788+0000", 2023-09-23T20:44:56.585 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0081275240000000006 2023-09-23T20:44:56.585 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.585 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.586 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:56.586 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.493688+0000", 2023-09-23T20:44:56.586 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0068995159999999996 2023-09-23T20:44:56.586 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.586 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.586 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:56.586 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.493704+0000", 2023-09-23T20:44:56.586 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.6441999999999999e-05 2023-09-23T20:44:56.586 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.586 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.586 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:44:56.587 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.493734+0000", 2023-09-23T20:44:56.587 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.9258999999999999e-05 2023-09-23T20:44:56.587 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.587 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.587 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:44:56.587 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.493755+0000", 2023-09-23T20:44:56.587 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.1636000000000001e-05 2023-09-23T20:44:56.587 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:56.587 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:44:56.587 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:56.587 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.587 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.588 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:76982 9.b 9:db90636b:::benchmark_data_smithi018_381516_object76981:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e465)", 2023-09-23T20:44:56.588 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:41:58.524610+0000", 2023-09-23T20:44:56.588 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 177.94571361600001, 2023-09-23T20:44:56.588 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.97237829799999997, 2023-09-23T20:44:56.588 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:44:56.588 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:44:56.588 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:44:56.588 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:44:56.588 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:44:56.588 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 76982 2023-09-23T20:44:56.588 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.589 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:44:56.589 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.589 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:44:56.589 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.524610+0000", 2023-09-23T20:44:56.589 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:56.589 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.589 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.589 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:44:56.589 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.524610+0000", 2023-09-23T20:44:56.589 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:56.589 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.590 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.590 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:44:56.590 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.524608+0000", 2023-09-23T20:44:56.590 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:44:56.590 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.590 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.590 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:44:56.590 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.524642+0000", 2023-09-23T20:44:56.590 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.4152000000000001e-05 2023-09-23T20:44:56.590 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.590 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.591 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:44:56.591 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.524659+0000", 2023-09-23T20:44:56.591 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.7108e-05 2023-09-23T20:44:56.591 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.591 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.591 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:44:56.591 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.524666+0000", 2023-09-23T20:44:56.591 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.0489999999999998e-06 2023-09-23T20:44:56.591 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.591 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.591 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:44:56.591 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.524731+0000", 2023-09-23T20:44:56.592 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.4169000000000002e-05 2023-09-23T20:44:56.592 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.592 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.592 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:44:56.592 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.524743+0000", 2023-09-23T20:44:56.592 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.2434e-05 2023-09-23T20:44:56.592 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.592 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.592 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:44:56.592 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.481305+0000", 2023-09-23T20:44:56.592 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.95656189599999997 2023-09-23T20:44:56.592 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.593 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.593 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:44:56.593 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.481433+0000", 2023-09-23T20:44:56.593 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000128059 2023-09-23T20:44:56.593 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.593 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.593 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:44:56.593 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.481584+0000", 2023-09-23T20:44:56.593 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00015048899999999999 2023-09-23T20:44:56.593 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.593 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.593 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:56.594 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.493780+0000", 2023-09-23T20:44:56.594 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.012196716 2023-09-23T20:44:56.594 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.594 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.594 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:56.594 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.493806+0000", 2023-09-23T20:44:56.594 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.6188999999999999e-05 2023-09-23T20:44:56.594 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.594 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.594 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:44:56.594 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.496920+0000", 2023-09-23T20:44:56.595 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.003113111 2023-09-23T20:44:56.595 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.595 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.595 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:44:56.595 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.496963+0000", 2023-09-23T20:44:56.595 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.3648e-05 2023-09-23T20:44:56.595 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.595 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.595 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:44:56.595 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.496988+0000", 2023-09-23T20:44:56.595 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.4782000000000001e-05 2023-09-23T20:44:56.595 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:56.596 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:44:56.596 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:56.596 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.596 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.596 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:77009 9.b 9:db9e621f:::benchmark_data_smithi018_381516_object77008:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e465)", 2023-09-23T20:44:56.596 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:41:58.547295+0000", 2023-09-23T20:44:56.596 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 177.92302821000001, 2023-09-23T20:44:56.596 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.94976649700000004, 2023-09-23T20:44:56.596 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:44:56.596 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:44:56.596 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:44:56.596 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:44:56.597 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:44:56.597 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 77009 2023-09-23T20:44:56.597 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.597 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:44:56.597 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.597 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:44:56.597 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.547295+0000", 2023-09-23T20:44:56.597 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:56.597 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.597 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.597 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:44:56.597 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.547295+0000", 2023-09-23T20:44:56.598 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:56.598 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.598 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.598 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:44:56.598 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.547293+0000", 2023-09-23T20:44:56.598 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999976 2023-09-23T20:44:56.598 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.598 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.598 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:44:56.598 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.547385+0000", 2023-09-23T20:44:56.598 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.2566999999999997e-05 2023-09-23T20:44:56.598 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.599 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.599 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:44:56.599 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.547409+0000", 2023-09-23T20:44:56.599 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.3802999999999999e-05 2023-09-23T20:44:56.599 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.599 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.599 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:44:56.599 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.547418+0000", 2023-09-23T20:44:56.599 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.7509999999999997e-06 2023-09-23T20:44:56.599 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.599 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.600 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:44:56.600 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.547486+0000", 2023-09-23T20:44:56.600 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.7928000000000004e-05 2023-09-23T20:44:56.600 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.600 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.600 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:44:56.600 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.547499+0000", 2023-09-23T20:44:56.600 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.3556e-05 2023-09-23T20:44:56.600 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.600 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.600 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:44:56.600 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.484068+0000", 2023-09-23T20:44:56.601 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.93656838200000003 2023-09-23T20:44:56.601 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.601 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.601 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:44:56.601 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.484183+0000", 2023-09-23T20:44:56.601 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000115436 2023-09-23T20:44:56.601 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.601 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.601 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:44:56.601 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.484306+0000", 2023-09-23T20:44:56.601 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00012274399999999999 2023-09-23T20:44:56.602 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.602 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.602 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:56.602 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.493822+0000", 2023-09-23T20:44:56.602 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.009516142 2023-09-23T20:44:56.602 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.602 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.602 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:56.602 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.493837+0000", 2023-09-23T20:44:56.602 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.4698000000000001e-05 2023-09-23T20:44:56.602 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.602 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.603 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:44:56.603 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.496999+0000", 2023-09-23T20:44:56.603 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0031623770000000001 2023-09-23T20:44:56.603 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.603 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.603 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:44:56.603 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.497034+0000", 2023-09-23T20:44:56.603 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.4620999999999999e-05 2023-09-23T20:44:56.603 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.603 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.603 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:44:56.603 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.497062+0000", 2023-09-23T20:44:56.604 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.7654e-05 2023-09-23T20:44:56.604 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:56.604 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:44:56.604 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:56.604 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.604 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.604 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:77014 9.b 9:dc5d982b:::benchmark_data_smithi018_381516_object77013:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e465)", 2023-09-23T20:44:56.604 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:41:58.554929+0000", 2023-09-23T20:44:56.604 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 177.91539454799999, 2023-09-23T20:44:56.604 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.94218858999999999, 2023-09-23T20:44:56.604 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:44:56.604 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:44:56.605 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:44:56.605 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:44:56.605 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:44:56.605 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 77014 2023-09-23T20:44:56.605 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.605 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:44:56.605 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.605 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:44:56.605 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.554929+0000", 2023-09-23T20:44:56.605 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:56.605 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.605 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.606 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:44:56.606 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.554929+0000", 2023-09-23T20:44:56.606 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:56.606 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.606 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.606 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:44:56.606 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.554927+0000", 2023-09-23T20:44:56.606 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999981 2023-09-23T20:44:56.606 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.606 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.606 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:44:56.607 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.554989+0000", 2023-09-23T20:44:56.607 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.2597999999999999e-05 2023-09-23T20:44:56.607 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.607 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.607 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:44:56.607 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.555008+0000", 2023-09-23T20:44:56.607 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.8382000000000001e-05 2023-09-23T20:44:56.607 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.607 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.607 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:44:56.607 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.555014+0000", 2023-09-23T20:44:56.607 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.336e-06 2023-09-23T20:44:56.608 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.608 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.608 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:44:56.608 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.555098+0000", 2023-09-23T20:44:56.608 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.4109000000000004e-05 2023-09-23T20:44:56.608 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.608 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.608 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:44:56.608 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.555119+0000", 2023-09-23T20:44:56.608 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.0341000000000001e-05 2023-09-23T20:44:56.608 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.608 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.609 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:44:56.609 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.486855+0000", 2023-09-23T20:44:56.609 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.93173672500000004 2023-09-23T20:44:56.609 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.609 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.609 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:44:56.609 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.486994+0000", 2023-09-23T20:44:56.609 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000138243 2023-09-23T20:44:56.609 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.609 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.609 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:44:56.609 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.487116+0000", 2023-09-23T20:44:56.610 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000122268 2023-09-23T20:44:56.610 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.610 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.610 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:56.610 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.493852+0000", 2023-09-23T20:44:56.610 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0067365419999999999 2023-09-23T20:44:56.610 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.610 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.610 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:56.610 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.493874+0000", 2023-09-23T20:44:56.610 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.1189999999999999e-05 2023-09-23T20:44:56.611 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.611 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.611 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:44:56.611 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.497069+0000", 2023-09-23T20:44:56.611 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0031949600000000002 2023-09-23T20:44:56.611 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.611 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.611 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:44:56.611 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.497094+0000", 2023-09-23T20:44:56.611 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.5075999999999998e-05 2023-09-23T20:44:56.611 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.611 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.612 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:44:56.612 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.497117+0000", 2023-09-23T20:44:56.612 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.3669e-05 2023-09-23T20:44:56.612 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:56.612 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:44:56.612 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:56.612 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.612 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.612 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:77023 9.b 9:d3d436f2:::benchmark_data_smithi018_381516_object77022:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e465)", 2023-09-23T20:44:56.612 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:41:58.571188+0000", 2023-09-23T20:44:56.612 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 177.899135325, 2023-09-23T20:44:56.613 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.92609987900000001, 2023-09-23T20:44:56.613 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:44:56.613 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:44:56.613 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:44:56.613 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:44:56.613 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:44:56.613 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 77023 2023-09-23T20:44:56.613 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.613 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:44:56.613 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.613 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:44:56.613 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.571188+0000", 2023-09-23T20:44:56.614 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:56.614 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.614 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.614 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:44:56.614 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.571188+0000", 2023-09-23T20:44:56.614 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:56.614 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.614 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.614 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:44:56.614 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.571186+0000", 2023-09-23T20:44:56.614 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999981 2023-09-23T20:44:56.614 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.615 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.615 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:44:56.615 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.571248+0000", 2023-09-23T20:44:56.615 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.1997999999999998e-05 2023-09-23T20:44:56.615 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.615 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.615 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:44:56.615 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.571266+0000", 2023-09-23T20:44:56.615 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.7788999999999999e-05 2023-09-23T20:44:56.615 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.615 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.615 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:44:56.616 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.571273+0000", 2023-09-23T20:44:56.616 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.5910000000000001e-06 2023-09-23T20:44:56.616 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.616 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.616 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:44:56.616 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.571335+0000", 2023-09-23T20:44:56.616 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.2872000000000004e-05 2023-09-23T20:44:56.616 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.616 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.616 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:44:56.616 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.571346+0000", 2023-09-23T20:44:56.616 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0069e-05 2023-09-23T20:44:56.617 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.617 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.617 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:44:56.617 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.489960+0000", 2023-09-23T20:44:56.617 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.918614017 2023-09-23T20:44:56.617 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.617 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.617 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:44:56.617 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.490095+0000", 2023-09-23T20:44:56.617 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00013572700000000001 2023-09-23T20:44:56.617 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.617 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.618 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:44:56.618 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.490256+0000", 2023-09-23T20:44:56.618 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00016105000000000001 2023-09-23T20:44:56.618 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.618 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.618 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:56.618 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.493899+0000", 2023-09-23T20:44:56.618 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0036422210000000002 2023-09-23T20:44:56.618 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.618 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.618 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:44:56.619 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.497124+0000", 2023-09-23T20:44:56.619 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0032253220000000001 2023-09-23T20:44:56.619 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.619 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.619 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:44:56.619 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.497214+0000", 2023-09-23T20:44:56.619 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.9739000000000003e-05 2023-09-23T20:44:56.619 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.619 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.619 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:44:56.619 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.497254+0000", 2023-09-23T20:44:56.619 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.0278999999999998e-05 2023-09-23T20:44:56.620 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:56.620 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:56.620 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:44:56.620 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.497288+0000", 2023-09-23T20:44:56.620 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.3927999999999998e-05 2023-09-23T20:44:56.620 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:56.620 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:44:56.620 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:56.620 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:56.620 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:44:56.620 INFO:teuthology.orchestra.run.smithi149.stdout:} 2023-09-23T20:44:56.621 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:56.704 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:44:56.711 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:56.712 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:44:56.812 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:44:56.834 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:44:56.860 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:56.925 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T20:44:56.925 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:44:56.932 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:56.932 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:44:57.033 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:44:57.100 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:57.189 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:44:57.194 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:57.195 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:44:57.295 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:44:57.373 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:57.661 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:44:57.666 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:57.667 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:44:57.694 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:44:57.768 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:44:57.853 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:58.050 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:44:58.062 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:57 smithi018 ceph-mon[179647]: pgmap v1300: 119 pgs: 119 active+clean; 1.2 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 606 op/s; 3.3 MiB/s, 53 objects/s recovering 2023-09-23T20:44:58.062 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:58.063 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:44:58.063 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:57 smithi018 ceph-mon[184144]: pgmap v1300: 119 pgs: 119 active+clean; 1.2 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 606 op/s; 3.3 MiB/s, 53 objects/s recovering 2023-09-23T20:44:58.156 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:57 smithi149 ceph-mon[170394]: pgmap v1300: 119 pgs: 119 active+clean; 1.2 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 606 op/s; 3.3 MiB/s, 53 objects/s recovering 2023-09-23T20:44:58.163 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:44:58.255 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:58.350 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:44:58.355 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:58.355 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:44:58.456 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:44:58.505 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:58.734 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:44:58.844 INFO:teuthology.orchestra.run.smithi018.stderr:noscrub is set 2023-09-23T20:44:59.060 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:58 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/4202037913' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:44:59.061 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:58 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:44:59.065 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:58 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/4202037913' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:44:59.065 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:58 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:44:59.073 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:44:59.077 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:58 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/4202037913' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:44:59.077 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:58 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:44:59.077 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:44:59.078 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:44:59.179 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:44:59.255 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:44:59.273 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:44:59.275 INFO:teuthology.orchestra.run.smithi018.stderr:dumped all 2023-09-23T20:44:59.693 INFO:teuthology.orchestra.run.smithi149.stdout:{ 2023-09-23T20:44:59.693 INFO:teuthology.orchestra.run.smithi149.stdout: "ops": [ 2023-09-23T20:44:59.693 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:59.693 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:213534 9.e 9:78b2bc03:::benchmark_data_smithi018_381516_object99700:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e520)", 2023-09-23T20:44:59.693 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:44:59.685875+0000", 2023-09-23T20:44:59.693 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 0.0040578230000000003, 2023-09-23T20:44:59.693 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.004088169, 2023-09-23T20:44:59.693 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:44:59.693 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:44:59.693 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:44:59.694 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:44:59.694 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:44:59.694 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 213534 2023-09-23T20:44:59.694 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:59.694 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:44:59.694 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:59.694 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:44:59.694 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:59.685875+0000", 2023-09-23T20:44:59.694 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:59.694 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:59.694 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:59.694 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:44:59.695 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:59.685875+0000", 2023-09-23T20:44:59.695 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:59.695 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:59.695 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:59.695 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:44:59.695 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:59.685874+0000", 2023-09-23T20:44:59.695 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:44:59.695 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:59.695 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:59.695 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:44:59.695 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:59.685880+0000", 2023-09-23T20:44:59.695 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.28e-06 2023-09-23T20:44:59.696 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:59.696 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:59.696 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:44:59.696 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:59.685895+0000", 2023-09-23T20:44:59.696 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.5196e-05 2023-09-23T20:44:59.696 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:59.696 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:59.696 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:44:59.696 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:59.685902+0000", 2023-09-23T20:44:59.696 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.3599999999999998e-06 2023-09-23T20:44:59.696 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:59.697 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:59.697 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:44:59.697 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:59.685947+0000", 2023-09-23T20:44:59.697 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.4826000000000001e-05 2023-09-23T20:44:59.697 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:59.697 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:59.697 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:44:59.697 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:59.686086+0000", 2023-09-23T20:44:59.697 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00013899600000000001 2023-09-23T20:44:59.697 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:59.697 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:59.697 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 1,4", 2023-09-23T20:44:59.697 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:59.686222+0000", 2023-09-23T20:44:59.698 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000135826 2023-09-23T20:44:59.698 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:59.698 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:44:59.698 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:59.698 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:59.698 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:59.698 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:213538 9.d 9:b319850e:::benchmark_data_smithi018_381516_object99704:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e520)", 2023-09-23T20:44:59.698 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:44:59.689509+0000", 2023-09-23T20:44:59.698 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 0.00042391100000000001, 2023-09-23T20:44:59.698 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00054413400000000005, 2023-09-23T20:44:59.698 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:44:59.699 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:44:59.699 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:44:59.699 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:44:59.699 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:44:59.699 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 213538 2023-09-23T20:44:59.699 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:59.699 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:44:59.699 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:59.699 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:44:59.699 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:59.689509+0000", 2023-09-23T20:44:59.699 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:59.699 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:59.700 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:59.700 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:44:59.700 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:59.689509+0000", 2023-09-23T20:44:59.700 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:59.700 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:59.700 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:59.700 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:44:59.700 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:59.689508+0000", 2023-09-23T20:44:59.700 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:44:59.700 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:59.700 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:59.700 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:44:59.700 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:59.689514+0000", 2023-09-23T20:44:59.701 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.5740000000000001e-06 2023-09-23T20:44:59.701 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:59.701 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:59.701 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:44:59.701 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:59.689527+0000", 2023-09-23T20:44:59.701 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.2216e-05 2023-09-23T20:44:59.701 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:59.701 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:59.701 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:44:59.701 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:59.689534+0000", 2023-09-23T20:44:59.701 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.9929999999999998e-06 2023-09-23T20:44:59.701 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:59.702 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:59.702 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:44:59.702 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:59.689591+0000", 2023-09-23T20:44:59.702 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.7488999999999998e-05 2023-09-23T20:44:59.702 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:59.702 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:59.702 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:44:59.702 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:59.689718+0000", 2023-09-23T20:44:59.702 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00012687500000000001 2023-09-23T20:44:59.703 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:59.703 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:59.703 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 1,5", 2023-09-23T20:44:59.703 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:59.689849+0000", 2023-09-23T20:44:59.703 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00013106899999999999 2023-09-23T20:44:59.703 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:59.703 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:44:59.703 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:59.703 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:59.703 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:59.703 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:213539 9.e 9:74505b07:::benchmark_data_smithi018_381516_object99705:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e520)", 2023-09-23T20:44:59.703 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:44:59.689553+0000", 2023-09-23T20:44:59.704 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 0.000379427, 2023-09-23T20:44:59.704 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00058108599999999997, 2023-09-23T20:44:59.704 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:44:59.704 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:44:59.704 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:44:59.704 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:44:59.704 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:44:59.704 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 213539 2023-09-23T20:44:59.704 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:59.704 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:44:59.704 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:59.704 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:44:59.705 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:59.689553+0000", 2023-09-23T20:44:59.705 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:59.705 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:59.705 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:59.705 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:44:59.705 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:59.689553+0000", 2023-09-23T20:44:59.705 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:59.705 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:59.705 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:59.705 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:44:59.705 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:59.689553+0000", 2023-09-23T20:44:59.706 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:44:59.706 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:59.706 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:59.706 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:44:59.706 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:59.689556+0000", 2023-09-23T20:44:59.706 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.4769999999999999e-06 2023-09-23T20:44:59.706 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:59.706 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:59.706 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:44:59.706 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:59.689563+0000", 2023-09-23T20:44:59.706 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.9770000000000003e-06 2023-09-23T20:44:59.706 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:59.707 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:59.707 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:44:59.707 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:59.689568+0000", 2023-09-23T20:44:59.707 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.8659999999999997e-06 2023-09-23T20:44:59.707 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:59.707 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:59.707 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:44:59.707 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:59.689618+0000", 2023-09-23T20:44:59.707 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.9863e-05 2023-09-23T20:44:59.707 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:59.707 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:59.708 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:44:59.708 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:59.689724+0000", 2023-09-23T20:44:59.708 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000105961 2023-09-23T20:44:59.708 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:59.708 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:59.708 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 1,4", 2023-09-23T20:44:59.708 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:59.689849+0000", 2023-09-23T20:44:59.708 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00012482699999999999 2023-09-23T20:44:59.708 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:59.708 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:44:59.708 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:59.708 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:59.709 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:59.709 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:213543 9.d 9.d035321d (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e520)", 2023-09-23T20:44:59.709 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:44:59.689594+0000", 2023-09-23T20:44:59.709 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 0.00033858, 2023-09-23T20:44:59.709 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00061654099999999996, 2023-09-23T20:44:59.709 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:44:59.709 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "queued for pg", 2023-09-23T20:44:59.709 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:44:59.709 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:44:59.709 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:44:59.709 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 213543 2023-09-23T20:44:59.709 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:59.710 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:44:59.710 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:59.710 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:44:59.710 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:59.689594+0000", 2023-09-23T20:44:59.710 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:59.710 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:59.710 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:59.710 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:44:59.710 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:59.689594+0000", 2023-09-23T20:44:59.710 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:59.710 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:59.710 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:59.711 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:44:59.711 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:59.689594+0000", 2023-09-23T20:44:59.711 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:44:59.711 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:59.711 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:59.711 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:44:59.711 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:59.689598+0000", 2023-09-23T20:44:59.711 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.7099999999999998e-06 2023-09-23T20:44:59.711 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:59.711 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:59.711 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:44:59.711 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:59.689608+0000", 2023-09-23T20:44:59.712 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.8689999999999999e-06 2023-09-23T20:44:59.712 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:59.712 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:59.712 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:44:59.712 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:59.689613+0000", 2023-09-23T20:44:59.712 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.6e-06 2023-09-23T20:44:59.712 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:59.712 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:44:59.712 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:59.712 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:59.712 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:59.712 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_repop(client.36450.0:213544 9.a e520/481 9:5c440c0c:::benchmark_data_smithi018_381516_object99710:head v 520'13331, mlcod=520'13329)", 2023-09-23T20:44:59.713 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:44:59.689638+0000", 2023-09-23T20:44:59.713 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 0.00029535999999999999, 2023-09-23T20:44:59.713 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00062874000000000003, 2023-09-23T20:44:59.713 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:44:59.713 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "started", 2023-09-23T20:44:59.713 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:44:59.713 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:59.713 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:44:59.713 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:59.689638+0000", 2023-09-23T20:44:59.713 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:59.713 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:59.713 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:59.713 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:44:59.714 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:59.689638+0000", 2023-09-23T20:44:59.714 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:44:59.714 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:59.714 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:59.714 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:44:59.714 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:59.689637+0000", 2023-09-23T20:44:59.714 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:44:59.714 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:59.714 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:59.714 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:44:59.714 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:59.689642+0000", 2023-09-23T20:44:59.715 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.8579999999999999e-06 2023-09-23T20:44:59.715 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:59.715 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:59.715 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:44:59.715 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:59.689647+0000", 2023-09-23T20:44:59.715 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.4099999999999999e-06 2023-09-23T20:44:59.715 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:59.715 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:59.715 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:44:59.715 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:59.689650+0000", 2023-09-23T20:44:59.715 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.2150000000000001e-06 2023-09-23T20:44:59.715 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:59.716 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:59.716 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:44:59.716 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:59.689683+0000", 2023-09-23T20:44:59.716 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.2768000000000002e-05 2023-09-23T20:44:59.716 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:44:59.716 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:44:59.716 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:44:59.716 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:59.689704+0000", 2023-09-23T20:44:59.716 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.0675000000000001e-05 2023-09-23T20:44:59.716 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:59.716 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:44:59.716 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:59.717 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:44:59.717 INFO:teuthology.orchestra.run.smithi149.stdout: ], 2023-09-23T20:44:59.717 INFO:teuthology.orchestra.run.smithi149.stdout: "num_ops": 5 2023-09-23T20:44:59.717 INFO:teuthology.orchestra.run.smithi149.stdout:} 2023-09-23T20:45:00.086 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:59 smithi018 ceph-mon[179647]: pgmap v1301: 119 pgs: 119 active+clean; 1.1 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 645 op/s 2023-09-23T20:45:00.086 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:59 smithi018 ceph-mon[179647]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:45:00.086 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:59 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:45:00.086 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:59 smithi018 ceph-mon[179647]: osdmap e520: 8 total, 8 up, 8 in 2023-09-23T20:45:00.087 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:59 smithi018 ceph-mon[179647]: failed to encode map e520 with expected crc 2023-09-23T20:45:00.087 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:59 smithi018 ceph-mon[179647]: failed to encode map e520 with expected crc 2023-09-23T20:45:00.087 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:59 smithi018 ceph-mon[179647]: failed to encode map e520 with expected crc 2023-09-23T20:45:00.087 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:44:59 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:45:00.087 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:59 smithi018 ceph-mon[184144]: pgmap v1301: 119 pgs: 119 active+clean; 1.1 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 645 op/s 2023-09-23T20:45:00.087 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:59 smithi018 ceph-mon[184144]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:45:00.087 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:59 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:45:00.087 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:59 smithi018 ceph-mon[184144]: osdmap e520: 8 total, 8 up, 8 in 2023-09-23T20:45:00.087 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:59 smithi018 ceph-mon[184144]: failed to encode map e520 with expected crc 2023-09-23T20:45:00.087 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:59 smithi018 ceph-mon[184144]: failed to encode map e520 with expected crc 2023-09-23T20:45:00.088 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:59 smithi018 ceph-mon[184144]: failed to encode map e520 with expected crc 2023-09-23T20:45:00.088 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:44:59 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:45:00.137 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:59 smithi149 ceph-mon[170394]: pgmap v1301: 119 pgs: 119 active+clean; 1.1 GiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 645 op/s 2023-09-23T20:45:00.137 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:59 smithi149 ceph-mon[170394]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:45:00.137 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:59 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:45:00.137 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:59 smithi149 ceph-mon[170394]: osdmap e520: 8 total, 8 up, 8 in 2023-09-23T20:45:00.138 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:59 smithi149 ceph-mon[170394]: failed to encode map e520 with expected crc 2023-09-23T20:45:00.138 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:59 smithi149 ceph-mon[170394]: failed to encode map e520 with expected crc 2023-09-23T20:45:00.138 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:59 smithi149 ceph-mon[170394]: failed to encode map e520 with expected crc 2023-09-23T20:45:00.138 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:44:59 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:45:00.139 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T20:45:00.235 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:45:00.247 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:00.248 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:45:00.349 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:45:00.413 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:00.946 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:45:01.144 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:00 smithi018 ceph-mon[184144]: failed to encode map e520 with expected crc 2023-09-23T20:45:01.145 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:00 smithi018 ceph-mon[184144]: failed to encode map e520 with expected crc 2023-09-23T20:45:01.145 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:00 smithi018 ceph-mon[184144]: from='client.56437 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:45:01.145 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:00 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 5]}]: dispatch 2023-09-23T20:45:01.145 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:00 smithi018 ceph-mon[179647]: failed to encode map e520 with expected crc 2023-09-23T20:45:01.145 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:00 smithi018 ceph-mon[179647]: failed to encode map e520 with expected crc 2023-09-23T20:45:01.145 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:00 smithi018 ceph-mon[179647]: from='client.56437 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:45:01.145 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:00 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 5]}]: dispatch 2023-09-23T20:45:01.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:00 smithi149 ceph-mon[170394]: failed to encode map e520 with expected crc 2023-09-23T20:45:01.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:00 smithi149 ceph-mon[170394]: failed to encode map e520 with expected crc 2023-09-23T20:45:01.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:00 smithi149 ceph-mon[170394]: from='client.56437 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:45:01.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:00 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 5]}]: dispatch 2023-09-23T20:45:01.836 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:45:01.939 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:01.940 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:45:01.940 INFO:teuthology.orchestra.run.smithi018.stdout:{"pg_ready":true,"pg_map":{"version":1302,"stamp":"2023-09-23T20:44:58.813088+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1061885899,"num_objects":16159,"num_object_clones":0,"num_object_copies":48477,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":16159,"num_whiteouts":0,"num_read":5984,"num_read_kb":13201,"num_write":212440,"num_write_kb":7297910,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":17055,"num_bytes_recovered":1122650190,"num_keys_recovered":40,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":108750,"ondisk_log_size":108750,"up":357,"acting":357,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":361,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":8584604,"kb_used_data":3641308,"kb_used_omap":32,"kb_used_meta":4943071,"kb_avail":741376612,"statfs":{"total":767960285184,"available":759169650688,"internally_reserved":0,"allocated":3728699392,"data_stored":3712383141,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32773,"internal_metadata":5061705723},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,4,2,2,2,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-410845184,"num_objects":-6269,"num_object_clones":0,"num_object_copies":-18807,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-329,"num_objects_unfound":0,"num_objects_dirty":-6269,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6269,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.085091"},"pg_stats":[{"pgid":"1.18","version":"90'161","reported_seq":564,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-23T20:44:50.747771+0000","last_change":"2023-09-23T20:44:19.677357+0000","last_active":"2023-09-23T20:44:50.747771+0000","last_peered":"2023-09-23T20:44:50.747771+0000","last_clean":"2023-09-23T20:44:50.747771+0000","last_became_active":"2023-09-23T20:43:16.766305+0000","last_became_peered":"2023-09-23T20:43:16.766305+0000","last_unstale":"2023-09-23T20:44:50.747771+0000","last_undegraded":"2023-09-23T20:44:50.747771+0000","last_fullsized":"2023-09-23T20:44:50.747771+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:19.677209+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:19.677209+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:14:06.510115+0000","scrub_duration":0.0023292260000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"69'1","reported_seq":216,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-23T20:44:50.754118+0000","last_change":"2023-09-23T20:32:33.233853+0000","last_active":"2023-09-23T20:44:50.754118+0000","last_peered":"2023-09-23T20:44:50.754118+0000","last_clean":"2023-09-23T20:44:50.754118+0000","last_became_active":"2023-09-23T20:32:33.233546+0000","last_became_peered":"2023-09-23T20:32:33.233546+0000","last_unstale":"2023-09-23T20:44:50.754118+0000","last_undegraded":"2023-09-23T20:44:50.754118+0000","last_fullsized":"2023-09-23T20:44:50.754118+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:50:27.130808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":333,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-23T20:44:50.742315+0000","last_change":"2023-09-23T20:37:43.579611+0000","last_active":"2023-09-23T20:44:50.742315+0000","last_peered":"2023-09-23T20:44:50.742315+0000","last_clean":"2023-09-23T20:44:50.742315+0000","last_became_active":"2023-09-23T20:37:43.579323+0000","last_became_peered":"2023-09-23T20:37:43.579323+0000","last_unstale":"2023-09-23T20:44:50.742315+0000","last_undegraded":"2023-09-23T20:44:50.742315+0000","last_fullsized":"2023-09-23T20:44:50.742315+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:17:32.311427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"90'161","reported_seq":640,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-23T20:44:50.753776+0000","last_change":"2023-09-23T20:44:14.647447+0000","last_active":"2023-09-23T20:44:50.753776+0000","last_peered":"2023-09-23T20:44:50.753776+0000","last_clean":"2023-09-23T20:44:50.753776+0000","last_became_active":"2023-09-23T20:43:16.756116+0000","last_became_peered":"2023-09-23T20:43:16.756116+0000","last_unstale":"2023-09-23T20:44:50.753776+0000","last_undegraded":"2023-09-23T20:44:50.753776+0000","last_fullsized":"2023-09-23T20:44:50.753776+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:14.647386+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:36:05.442288+0000","last_clean_scrub_stamp":"2023-09-23T20:44:14.647386+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:25:25.151100+0000","scrub_duration":0.0097381210000000006,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":489,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-23T20:44:50.739246+0000","last_change":"2023-09-23T20:34:59.983035+0000","last_active":"2023-09-23T20:44:50.739246+0000","last_peered":"2023-09-23T20:44:50.739246+0000","last_clean":"2023-09-23T20:44:50.739246+0000","last_became_active":"2023-09-23T20:34:59.982459+0000","last_became_peered":"2023-09-23T20:34:59.982459+0000","last_unstale":"2023-09-23T20:44:50.739246+0000","last_undegraded":"2023-09-23T20:44:50.739246+0000","last_fullsized":"2023-09-23T20:44:50.739246+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:16:48.510783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":476,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-23T20:44:50.745881+0000","last_change":"2023-09-23T20:43:46.521939+0000","last_active":"2023-09-23T20:44:50.745881+0000","last_peered":"2023-09-23T20:44:50.745881+0000","last_clean":"2023-09-23T20:44:50.745881+0000","last_became_active":"2023-09-23T20:43:46.521185+0000","last_became_peered":"2023-09-23T20:43:46.521185+0000","last_unstale":"2023-09-23T20:44:50.745881+0000","last_undegraded":"2023-09-23T20:44:50.745881+0000","last_fullsized":"2023-09-23T20:44:50.745881+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:59:25.045627+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"90'161","reported_seq":598,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-23T20:44:50.753710+0000","last_change":"2023-09-23T20:36:01.292103+0000","last_active":"2023-09-23T20:44:50.753710+0000","last_peered":"2023-09-23T20:44:50.753710+0000","last_clean":"2023-09-23T20:44:50.753710+0000","last_became_active":"2023-09-23T20:35:26.639912+0000","last_became_peered":"2023-09-23T20:35:26.639912+0000","last_unstale":"2023-09-23T20:44:50.753710+0000","last_undegraded":"2023-09-23T20:44:50.753710+0000","last_fullsized":"2023-09-23T20:44:50.753710+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:01.292032+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:01.292032+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:04:53.926640+0000","scrub_duration":0.0032983159999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":313,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-23T20:44:50.741656+0000","last_change":"2023-09-23T20:37:43.584414+0000","last_active":"2023-09-23T20:44:50.741656+0000","last_peered":"2023-09-23T20:44:50.741656+0000","last_clean":"2023-09-23T20:44:50.741656+0000","last_became_active":"2023-09-23T20:37:43.584116+0000","last_became_peered":"2023-09-23T20:37:43.584116+0000","last_unstale":"2023-09-23T20:44:50.741656+0000","last_undegraded":"2023-09-23T20:44:50.741656+0000","last_fullsized":"2023-09-23T20:44:50.741656+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:17:54.258172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":134,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-23T20:44:50.750412+0000","last_change":"2023-09-23T20:38:37.457427+0000","last_active":"2023-09-23T20:44:50.750412+0000","last_peered":"2023-09-23T20:44:50.750412+0000","last_clean":"2023-09-23T20:44:50.750412+0000","last_became_active":"2023-09-23T20:38:37.457099+0000","last_became_peered":"2023-09-23T20:38:37.457099+0000","last_unstale":"2023-09-23T20:44:50.750412+0000","last_undegraded":"2023-09-23T20:44:50.750412+0000","last_fullsized":"2023-09-23T20:44:50.750412+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:31:27.084994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"90'161","reported_seq":731,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-23T20:44:50.744475+0000","last_change":"2023-09-23T20:40:22.449754+0000","last_active":"2023-09-23T20:44:50.744475+0000","last_peered":"2023-09-23T20:44:50.744475+0000","last_clean":"2023-09-23T20:44:50.744475+0000","last_became_active":"2023-09-23T20:40:22.448838+0000","last_became_peered":"2023-09-23T20:40:22.448838+0000","last_unstale":"2023-09-23T20:44:50.744475+0000","last_undegraded":"2023-09-23T20:44:50.744475+0000","last_fullsized":"2023-09-23T20:44:50.744475+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:03.411025+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:03.411025+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:22:07.948856+0000","scrub_duration":0.001802922,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":509,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-23T20:44:50.746153+0000","last_change":"2023-09-23T20:43:46.522325+0000","last_active":"2023-09-23T20:44:50.746153+0000","last_peered":"2023-09-23T20:44:50.746153+0000","last_clean":"2023-09-23T20:44:50.746153+0000","last_became_active":"2023-09-23T20:43:46.521996+0000","last_became_peered":"2023-09-23T20:43:46.521996+0000","last_unstale":"2023-09-23T20:44:50.746153+0000","last_undegraded":"2023-09-23T20:44:50.746153+0000","last_fullsized":"2023-09-23T20:44:50.746153+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:51:17.629956+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":244,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-23T20:44:50.747901+0000","last_change":"2023-09-23T20:38:02.579343+0000","last_active":"2023-09-23T20:44:50.747901+0000","last_peered":"2023-09-23T20:44:50.747901+0000","last_clean":"2023-09-23T20:44:50.747901+0000","last_became_active":"2023-09-23T20:38:02.579047+0000","last_became_peered":"2023-09-23T20:38:02.579047+0000","last_unstale":"2023-09-23T20:44:50.747901+0000","last_undegraded":"2023-09-23T20:44:50.747901+0000","last_fullsized":"2023-09-23T20:44:50.747901+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:44:03.166556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.14","version":"90'161","reported_seq":850,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-23T20:44:50.747751+0000","last_change":"2023-09-23T20:35:02.236398+0000","last_active":"2023-09-23T20:44:50.747751+0000","last_peered":"2023-09-23T20:44:50.747751+0000","last_clean":"2023-09-23T20:44:50.747751+0000","last_became_active":"2023-09-23T20:35:02.235988+0000","last_became_peered":"2023-09-23T20:35:02.235988+0000","last_unstale":"2023-09-23T20:44:50.747751+0000","last_undegraded":"2023-09-23T20:44:50.747751+0000","last_fullsized":"2023-09-23T20:44:50.747751+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:47.479012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:47.479012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:13.519648+0000","scrub_duration":0.0020136590000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":355,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-23T20:44:50.744785+0000","last_change":"2023-09-23T20:43:16.780478+0000","last_active":"2023-09-23T20:44:50.744785+0000","last_peered":"2023-09-23T20:44:50.744785+0000","last_clean":"2023-09-23T20:44:50.744785+0000","last_became_active":"2023-09-23T20:43:16.780183+0000","last_became_peered":"2023-09-23T20:43:16.780183+0000","last_unstale":"2023-09-23T20:44:50.744785+0000","last_undegraded":"2023-09-23T20:44:50.744785+0000","last_fullsized":"2023-09-23T20:44:50.744785+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:38:30.102066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":345,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-23T20:44:50.744149+0000","last_change":"2023-09-23T20:38:02.542034+0000","last_active":"2023-09-23T20:44:50.744149+0000","last_peered":"2023-09-23T20:44:50.744149+0000","last_clean":"2023-09-23T20:44:50.744149+0000","last_became_active":"2023-09-23T20:38:02.541575+0000","last_became_peered":"2023-09-23T20:38:02.541575+0000","last_unstale":"2023-09-23T20:44:50.744149+0000","last_undegraded":"2023-09-23T20:44:50.744149+0000","last_fullsized":"2023-09-23T20:44:50.744149+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:50:54.591529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"90'161","reported_seq":644,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-23T20:44:50.746015+0000","last_change":"2023-09-23T20:44:14.669973+0000","last_active":"2023-09-23T20:44:50.746015+0000","last_peered":"2023-09-23T20:44:50.746015+0000","last_clean":"2023-09-23T20:44:50.746015+0000","last_became_active":"2023-09-23T20:43:46.519592+0000","last_became_peered":"2023-09-23T20:43:46.519592+0000","last_unstale":"2023-09-23T20:44:50.746015+0000","last_undegraded":"2023-09-23T20:44:50.746015+0000","last_fullsized":"2023-09-23T20:44:50.746015+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:14.669831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:14.669831+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:56:28.076617+0000","scrub_duration":0.0083441460000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":546,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-23T20:44:50.744882+0000","last_change":"2023-09-23T20:32:33.270617+0000","last_active":"2023-09-23T20:44:50.744882+0000","last_peered":"2023-09-23T20:44:50.744882+0000","last_clean":"2023-09-23T20:44:50.744882+0000","last_became_active":"2023-09-23T20:32:33.227902+0000","last_became_peered":"2023-09-23T20:32:33.227902+0000","last_unstale":"2023-09-23T20:44:50.744882+0000","last_undegraded":"2023-09-23T20:44:50.744882+0000","last_fullsized":"2023-09-23T20:44:50.744882+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:30:26.847831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":360,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":33,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-23T20:44:50.742068+0000","last_change":"2023-09-23T20:44:10.611381+0000","last_active":"2023-09-23T20:44:50.742068+0000","last_peered":"2023-09-23T20:44:50.742068+0000","last_clean":"2023-09-23T20:44:50.742068+0000","last_became_active":"2023-09-23T20:44:10.611091+0000","last_became_peered":"2023-09-23T20:44:10.611091+0000","last_unstale":"2023-09-23T20:44:50.742068+0000","last_undegraded":"2023-09-23T20:44:50.742068+0000","last_fullsized":"2023-09-23T20:44:50.742068+0000","mapping_epoch":496,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":497,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:00:30.162478+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"90'161","reported_seq":644,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-23T20:44:50.743175+0000","last_change":"2023-09-23T20:36:02.548387+0000","last_active":"2023-09-23T20:44:50.743175+0000","last_peered":"2023-09-23T20:44:50.743175+0000","last_clean":"2023-09-23T20:44:50.743175+0000","last_became_active":"2023-09-23T20:35:26.644787+0000","last_became_peered":"2023-09-23T20:35:26.644787+0000","last_unstale":"2023-09-23T20:44:50.743175+0000","last_undegraded":"2023-09-23T20:44:50.743175+0000","last_fullsized":"2023-09-23T20:44:50.743175+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:02.548285+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:02.548285+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:37:15.216553+0000","scrub_duration":0.0015514459999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":266,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-23T20:44:50.754661+0000","last_change":"2023-09-23T20:38:02.560646+0000","last_active":"2023-09-23T20:44:50.754661+0000","last_peered":"2023-09-23T20:44:50.754661+0000","last_clean":"2023-09-23T20:44:50.754661+0000","last_became_active":"2023-09-23T20:38:02.539814+0000","last_became_peered":"2023-09-23T20:38:02.539814+0000","last_unstale":"2023-09-23T20:44:50.754661+0000","last_undegraded":"2023-09-23T20:44:50.754661+0000","last_fullsized":"2023-09-23T20:44:50.754661+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:00:09.435023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":246,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-23T20:44:50.754397+0000","last_change":"2023-09-23T20:38:02.579222+0000","last_active":"2023-09-23T20:44:50.754397+0000","last_peered":"2023-09-23T20:44:50.754397+0000","last_clean":"2023-09-23T20:44:50.754397+0000","last_became_active":"2023-09-23T20:38:02.578813+0000","last_became_peered":"2023-09-23T20:38:02.578813+0000","last_unstale":"2023-09-23T20:44:50.754397+0000","last_undegraded":"2023-09-23T20:44:50.754397+0000","last_fullsized":"2023-09-23T20:44:50.754397+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:57:31.080027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"90'161","reported_seq":772,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-23T20:44:50.745439+0000","last_change":"2023-09-23T20:43:46.544727+0000","last_active":"2023-09-23T20:44:50.745439+0000","last_peered":"2023-09-23T20:44:50.745439+0000","last_clean":"2023-09-23T20:44:50.745439+0000","last_became_active":"2023-09-23T20:43:46.543952+0000","last_became_peered":"2023-09-23T20:43:46.543952+0000","last_unstale":"2023-09-23T20:44:50.745439+0000","last_undegraded":"2023-09-23T20:44:50.745439+0000","last_fullsized":"2023-09-23T20:44:50.745439+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:43.578666+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:43.578666+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:11:16.611920+0000","scrub_duration":0.0021890379999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":545,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-23T20:44:50.744142+0000","last_change":"2023-09-23T20:32:33.242943+0000","last_active":"2023-09-23T20:44:50.744142+0000","last_peered":"2023-09-23T20:44:50.744142+0000","last_clean":"2023-09-23T20:44:50.744142+0000","last_became_active":"2023-09-23T20:32:33.225468+0000","last_became_peered":"2023-09-23T20:32:33.225468+0000","last_unstale":"2023-09-23T20:44:50.744142+0000","last_undegraded":"2023-09-23T20:44:50.744142+0000","last_fullsized":"2023-09-23T20:44:50.744142+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:40:47.867573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":256,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-23T20:44:50.744294+0000","last_change":"2023-09-23T20:37:43.571836+0000","last_active":"2023-09-23T20:44:50.744294+0000","last_peered":"2023-09-23T20:44:50.744294+0000","last_clean":"2023-09-23T20:44:50.744294+0000","last_became_active":"2023-09-23T20:37:43.571511+0000","last_became_peered":"2023-09-23T20:37:43.571511+0000","last_unstale":"2023-09-23T20:44:50.744294+0000","last_undegraded":"2023-09-23T20:44:50.744294+0000","last_fullsized":"2023-09-23T20:44:50.744294+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:59:25.507917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"90'161","reported_seq":799,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-23T20:44:50.744273+0000","last_change":"2023-09-23T20:44:20.041766+0000","last_active":"2023-09-23T20:44:50.744273+0000","last_peered":"2023-09-23T20:44:50.744273+0000","last_clean":"2023-09-23T20:44:50.744273+0000","last_became_active":"2023-09-23T20:43:16.765015+0000","last_became_peered":"2023-09-23T20:43:16.765015+0000","last_unstale":"2023-09-23T20:44:50.744273+0000","last_undegraded":"2023-09-23T20:44:50.744273+0000","last_fullsized":"2023-09-23T20:44:50.744273+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:20.041690+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:20.041690+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:33:30.150109+0000","scrub_duration":0.0026180330000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":474,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-23T20:44:50.745426+0000","last_change":"2023-09-23T20:43:46.521294+0000","last_active":"2023-09-23T20:44:50.745426+0000","last_peered":"2023-09-23T20:44:50.745426+0000","last_clean":"2023-09-23T20:44:50.745426+0000","last_became_active":"2023-09-23T20:43:46.521001+0000","last_became_peered":"2023-09-23T20:43:46.521001+0000","last_unstale":"2023-09-23T20:44:50.745426+0000","last_undegraded":"2023-09-23T20:44:50.745426+0000","last_fullsized":"2023-09-23T20:44:50.745426+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:01:35.126716+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":286,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-23T20:44:50.745040+0000","last_change":"2023-09-23T20:44:01.841565+0000","last_active":"2023-09-23T20:44:50.745040+0000","last_peered":"2023-09-23T20:44:50.745040+0000","last_clean":"2023-09-23T20:44:50.745040+0000","last_became_active":"2023-09-23T20:44:01.841255+0000","last_became_peered":"2023-09-23T20:44:01.841255+0000","last_unstale":"2023-09-23T20:44:50.745040+0000","last_undegraded":"2023-09-23T20:44:50.745040+0000","last_fullsized":"2023-09-23T20:44:50.745040+0000","mapping_epoch":491,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":492,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:18:44.817589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.13","version":"90'161","reported_seq":806,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-23T20:44:50.740089+0000","last_change":"2023-09-23T20:35:02.238923+0000","last_active":"2023-09-23T20:44:50.740089+0000","last_peered":"2023-09-23T20:44:50.740089+0000","last_clean":"2023-09-23T20:44:50.740089+0000","last_became_active":"2023-09-23T20:35:02.238293+0000","last_became_peered":"2023-09-23T20:35:02.238293+0000","last_unstale":"2023-09-23T20:44:50.740089+0000","last_undegraded":"2023-09-23T20:44:50.740089+0000","last_fullsized":"2023-09-23T20:44:50.740089+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:42.544308+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:42.544308+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:51:22.281932+0000","scrub_duration":0.0029958179999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":522,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-23T20:44:50.747522+0000","last_change":"2023-09-23T20:38:02.539774+0000","last_active":"2023-09-23T20:44:50.747522+0000","last_peered":"2023-09-23T20:44:50.747522+0000","last_clean":"2023-09-23T20:44:50.747522+0000","last_became_active":"2023-09-23T20:38:02.539436+0000","last_became_peered":"2023-09-23T20:38:02.539436+0000","last_unstale":"2023-09-23T20:44:50.747522+0000","last_undegraded":"2023-09-23T20:44:50.747522+0000","last_fullsized":"2023-09-23T20:44:50.747522+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:50:57.375803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":483,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-23T20:44:50.745920+0000","last_change":"2023-09-23T20:43:46.521735+0000","last_active":"2023-09-23T20:44:50.745920+0000","last_peered":"2023-09-23T20:44:50.745920+0000","last_clean":"2023-09-23T20:44:50.745920+0000","last_became_active":"2023-09-23T20:43:46.520762+0000","last_became_peered":"2023-09-23T20:43:46.520762+0000","last_unstale":"2023-09-23T20:44:50.745920+0000","last_undegraded":"2023-09-23T20:44:50.745920+0000","last_fullsized":"2023-09-23T20:44:50.745920+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:26:48.841930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"90'161","reported_seq":602,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-23T20:44:50.742736+0000","last_change":"2023-09-23T20:44:18.952808+0000","last_active":"2023-09-23T20:44:50.742736+0000","last_peered":"2023-09-23T20:44:50.742736+0000","last_clean":"2023-09-23T20:44:50.742736+0000","last_became_active":"2023-09-23T20:43:16.758652+0000","last_became_peered":"2023-09-23T20:43:16.758652+0000","last_unstale":"2023-09-23T20:44:50.742736+0000","last_undegraded":"2023-09-23T20:44:50.742736+0000","last_fullsized":"2023-09-23T20:44:50.742736+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:18.952693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:18.952693+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:12:19.496465+0000","scrub_duration":0.001699365,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":194,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-23T20:44:50.748798+0000","last_change":"2023-09-23T20:44:43.644290+0000","last_active":"2023-09-23T20:44:50.748798+0000","last_peered":"2023-09-23T20:44:50.748798+0000","last_clean":"2023-09-23T20:44:50.748798+0000","last_became_active":"2023-09-23T20:44:43.638313+0000","last_became_peered":"2023-09-23T20:44:43.638313+0000","last_unstale":"2023-09-23T20:44:50.748798+0000","last_undegraded":"2023-09-23T20:44:50.748798+0000","last_fullsized":"2023-09-23T20:44:50.748798+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:32:52.892603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":313,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-23T20:44:50.743407+0000","last_change":"2023-09-23T20:37:43.571083+0000","last_active":"2023-09-23T20:44:50.743407+0000","last_peered":"2023-09-23T20:44:50.743407+0000","last_clean":"2023-09-23T20:44:50.743407+0000","last_became_active":"2023-09-23T20:37:43.570306+0000","last_became_peered":"2023-09-23T20:37:43.570306+0000","last_unstale":"2023-09-23T20:44:50.743407+0000","last_undegraded":"2023-09-23T20:44:50.743407+0000","last_fullsized":"2023-09-23T20:44:50.743407+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:42:02.396718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.5","version":"519'12933","reported_seq":21492,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-23T20:44:56.354190+0000","last_change":"2023-09-23T20:44:20.680609+0000","last_active":"2023-09-23T20:44:56.354190+0000","last_peered":"2023-09-23T20:44:56.354190+0000","last_clean":"2023-09-23T20:44:56.354190+0000","last_became_active":"2023-09-23T20:44:20.638636+0000","last_became_peered":"2023-09-23T20:44:20.638636+0000","last_unstale":"2023-09-23T20:44:56.354190+0000","last_undegraded":"2023-09-23T20:44:56.354190+0000","last_fullsized":"2023-09-23T20:44:56.354190+0000","mapping_epoch":504,"log_start":"467'5410","ondisk_log_start":"467'5410","created":455,"last_epoch_clean":505,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":7523,"log_dups_size":0,"ondisk_log_size":7523,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:11:11.154742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":69009408,"num_objects":1053,"num_object_clones":0,"num_object_copies":3159,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1053,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12933,"num_write_kb":447552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4204,"num_bytes_recovered":275316736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.d","version":"90'161","reported_seq":811,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-23T20:44:50.744620+0000","last_change":"2023-09-23T20:34:59.496438+0000","last_active":"2023-09-23T20:44:50.744620+0000","last_peered":"2023-09-23T20:44:50.744620+0000","last_clean":"2023-09-23T20:44:50.744620+0000","last_became_active":"2023-09-23T20:33:38.970527+0000","last_became_peered":"2023-09-23T20:33:38.970527+0000","last_unstale":"2023-09-23T20:44:50.744620+0000","last_undegraded":"2023-09-23T20:44:50.744620+0000","last_fullsized":"2023-09-23T20:44:50.744620+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:34:59.496375+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:34:59.496375+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:46:18.244734+0000","scrub_duration":0.0017843189999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":522,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-23T20:44:50.744700+0000","last_change":"2023-09-23T20:32:33.283752+0000","last_active":"2023-09-23T20:44:50.744700+0000","last_peered":"2023-09-23T20:44:50.744700+0000","last_clean":"2023-09-23T20:44:50.744700+0000","last_became_active":"2023-09-23T20:32:33.230503+0000","last_became_peered":"2023-09-23T20:32:33.230503+0000","last_unstale":"2023-09-23T20:44:50.744700+0000","last_undegraded":"2023-09-23T20:44:50.744700+0000","last_fullsized":"2023-09-23T20:44:50.744700+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:29.084410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":333,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-23T20:44:50.745540+0000","last_change":"2023-09-23T20:43:16.765253+0000","last_active":"2023-09-23T20:44:50.745540+0000","last_peered":"2023-09-23T20:44:50.745540+0000","last_clean":"2023-09-23T20:44:50.745540+0000","last_became_active":"2023-09-23T20:43:16.764978+0000","last_became_peered":"2023-09-23T20:43:16.764978+0000","last_unstale":"2023-09-23T20:44:50.745540+0000","last_undegraded":"2023-09-23T20:44:50.745540+0000","last_fullsized":"2023-09-23T20:44:50.745540+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:07:23.539372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.4","version":"519'13371","reported_seq":13444,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-23T20:44:58.465559+0000","last_change":"2023-09-23T20:41:04.575200+0000","last_active":"2023-09-23T20:44:58.465559+0000","last_peered":"2023-09-23T20:44:58.465559+0000","last_clean":"2023-09-23T20:44:58.465559+0000","last_became_active":"2023-09-23T20:41:04.574824+0000","last_became_peered":"2023-09-23T20:41:04.574824+0000","last_unstale":"2023-09-23T20:44:58.465559+0000","last_undegraded":"2023-09-23T20:44:58.465559+0000","last_fullsized":"2023-09-23T20:44:58.465559+0000","mapping_epoch":455,"log_start":"477'7783","ondisk_log_start":"477'7783","created":455,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":5588,"log_dups_size":0,"ondisk_log_size":5588,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:50:46.714383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":61538304,"num_objects":939,"num_object_clones":0,"num_object_copies":2817,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":939,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13371,"num_write_kb":457920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.c","version":"90'161","reported_seq":780,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-23T20:44:50.744447+0000","last_change":"2023-09-23T20:41:09.506481+0000","last_active":"2023-09-23T20:44:50.744447+0000","last_peered":"2023-09-23T20:44:50.744447+0000","last_clean":"2023-09-23T20:44:50.744447+0000","last_became_active":"2023-09-23T20:38:28.442229+0000","last_became_peered":"2023-09-23T20:38:28.442229+0000","last_unstale":"2023-09-23T20:44:50.744447+0000","last_undegraded":"2023-09-23T20:44:50.744447+0000","last_fullsized":"2023-09-23T20:44:50.744447+0000","mapping_epoch":411,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":412,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:41:09.506338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:41:09.506338+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:19:46.265011+0000","scrub_duration":0.01112002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":557,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-23T20:44:50.744825+0000","last_change":"2023-09-23T20:34:59.987981+0000","last_active":"2023-09-23T20:44:50.744825+0000","last_peered":"2023-09-23T20:44:50.744825+0000","last_clean":"2023-09-23T20:44:50.744825+0000","last_became_active":"2023-09-23T20:34:59.985368+0000","last_became_peered":"2023-09-23T20:34:59.985368+0000","last_unstale":"2023-09-23T20:44:50.744825+0000","last_undegraded":"2023-09-23T20:44:50.744825+0000","last_fullsized":"2023-09-23T20:44:50.744825+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:12:43.082493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":216,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-23T20:44:50.754902+0000","last_change":"2023-09-23T20:32:33.232142+0000","last_active":"2023-09-23T20:44:50.754902+0000","last_peered":"2023-09-23T20:44:50.754902+0000","last_clean":"2023-09-23T20:44:50.754902+0000","last_became_active":"2023-09-23T20:32:33.231683+0000","last_became_peered":"2023-09-23T20:32:33.231683+0000","last_unstale":"2023-09-23T20:44:50.754902+0000","last_undegraded":"2023-09-23T20:44:50.754902+0000","last_fullsized":"2023-09-23T20:44:50.754902+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:02:58.336348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"519'13270","reported_seq":13354,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-23T20:44:57.486126+0000","last_change":"2023-09-23T20:43:16.854027+0000","last_active":"2023-09-23T20:44:57.486126+0000","last_peered":"2023-09-23T20:44:57.486126+0000","last_clean":"2023-09-23T20:44:57.486126+0000","last_became_active":"2023-09-23T20:43:16.766133+0000","last_became_peered":"2023-09-23T20:43:16.766133+0000","last_unstale":"2023-09-23T20:44:57.486126+0000","last_undegraded":"2023-09-23T20:44:57.486126+0000","last_fullsized":"2023-09-23T20:44:57.486126+0000","mapping_epoch":481,"log_start":"476'7585","ondisk_log_start":"476'7585","created":455,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":5685,"log_dups_size":0,"ondisk_log_size":5685,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:42:32.213805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":59375616,"num_objects":906,"num_object_clones":0,"num_object_copies":2718,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":906,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13270,"num_write_kb":453632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.f","version":"90'161","reported_seq":674,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-23T20:44:50.750751+0000","last_change":"2023-09-23T20:38:40.491902+0000","last_active":"2023-09-23T20:44:50.750751+0000","last_peered":"2023-09-23T20:44:50.750751+0000","last_clean":"2023-09-23T20:44:50.750751+0000","last_became_active":"2023-09-23T20:38:40.491563+0000","last_became_peered":"2023-09-23T20:38:40.491563+0000","last_unstale":"2023-09-23T20:44:50.750751+0000","last_undegraded":"2023-09-23T20:44:50.750751+0000","last_fullsized":"2023-09-23T20:44:50.750751+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:46.217489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:46.217489+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:39:25.008796+0000","scrub_duration":0.001456429,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":547,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-23T20:44:50.744702+0000","last_change":"2023-09-23T20:40:19.677655+0000","last_active":"2023-09-23T20:44:50.744702+0000","last_peered":"2023-09-23T20:44:50.744702+0000","last_clean":"2023-09-23T20:44:50.744702+0000","last_became_active":"2023-09-23T20:40:19.551189+0000","last_became_peered":"2023-09-23T20:40:19.551189+0000","last_unstale":"2023-09-23T20:44:50.744702+0000","last_undegraded":"2023-09-23T20:44:50.744702+0000","last_fullsized":"2023-09-23T20:44:50.744702+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:11:01.559576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":124,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-23T20:44:50.748924+0000","last_change":"2023-09-23T20:38:02.590613+0000","last_active":"2023-09-23T20:44:50.748924+0000","last_peered":"2023-09-23T20:44:50.748924+0000","last_clean":"2023-09-23T20:44:50.748924+0000","last_became_active":"2023-09-23T20:38:02.590284+0000","last_became_peered":"2023-09-23T20:38:02.590284+0000","last_unstale":"2023-09-23T20:44:50.748924+0000","last_undegraded":"2023-09-23T20:44:50.748924+0000","last_fullsized":"2023-09-23T20:44:50.748924+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:40:07.825054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"519'13160","reported_seq":13240,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-23T20:44:58.059624+0000","last_change":"2023-09-23T20:43:16.788249+0000","last_active":"2023-09-23T20:44:58.059624+0000","last_peered":"2023-09-23T20:44:58.059624+0000","last_clean":"2023-09-23T20:44:58.059624+0000","last_became_active":"2023-09-23T20:43:16.763555+0000","last_became_peered":"2023-09-23T20:43:16.763555+0000","last_unstale":"2023-09-23T20:44:58.059624+0000","last_undegraded":"2023-09-23T20:44:58.059624+0000","last_fullsized":"2023-09-23T20:44:58.059624+0000","mapping_epoch":481,"log_start":"461'3404","ondisk_log_start":"461'3404","created":455,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":9756,"log_dups_size":0,"ondisk_log_size":9756,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:19:57.856019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":62259200,"num_objects":950,"num_object_clones":0,"num_object_copies":2850,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":950,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13160,"num_write_kb":451520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[2,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.e","version":"90'161","reported_seq":619,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-23T20:44:50.756708+0000","last_change":"2023-09-23T20:36:00.322453+0000","last_active":"2023-09-23T20:44:50.756708+0000","last_peered":"2023-09-23T20:44:50.756708+0000","last_clean":"2023-09-23T20:44:50.756708+0000","last_became_active":"2023-09-23T20:35:02.233536+0000","last_became_peered":"2023-09-23T20:35:02.233536+0000","last_unstale":"2023-09-23T20:44:50.756708+0000","last_undegraded":"2023-09-23T20:44:50.756708+0000","last_fullsized":"2023-09-23T20:44:50.756708+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:00.322381+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:00.322381+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:02:15.489069+0000","scrub_duration":0.0017321859999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":478,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-23T20:44:50.744859+0000","last_change":"2023-09-23T20:43:16.756437+0000","last_active":"2023-09-23T20:44:50.744859+0000","last_peered":"2023-09-23T20:44:50.744859+0000","last_clean":"2023-09-23T20:44:50.744859+0000","last_became_active":"2023-09-23T20:43:16.756193+0000","last_became_peered":"2023-09-23T20:43:16.756193+0000","last_unstale":"2023-09-23T20:44:50.744859+0000","last_undegraded":"2023-09-23T20:44:50.744859+0000","last_fullsized":"2023-09-23T20:44:50.744859+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:33:25.632781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":492,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-23T20:44:50.746780+0000","last_change":"2023-09-23T20:43:46.541524+0000","last_active":"2023-09-23T20:44:50.746780+0000","last_peered":"2023-09-23T20:44:50.746780+0000","last_clean":"2023-09-23T20:44:50.746780+0000","last_became_active":"2023-09-23T20:43:46.541051+0000","last_became_peered":"2023-09-23T20:43:46.541051+0000","last_unstale":"2023-09-23T20:44:50.746780+0000","last_undegraded":"2023-09-23T20:44:50.746780+0000","last_fullsized":"2023-09-23T20:44:50.746780+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:59:36.632592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"519'13184","reported_seq":13256,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-23T20:44:58.190037+0000","last_change":"2023-09-23T20:41:04.558473+0000","last_active":"2023-09-23T20:44:58.190037+0000","last_peered":"2023-09-23T20:44:58.190037+0000","last_clean":"2023-09-23T20:44:58.190037+0000","last_became_active":"2023-09-23T20:41:04.558173+0000","last_became_peered":"2023-09-23T20:41:04.558173+0000","last_unstale":"2023-09-23T20:44:58.190037+0000","last_undegraded":"2023-09-23T20:44:58.190037+0000","last_fullsized":"2023-09-23T20:44:58.190037+0000","mapping_epoch":455,"log_start":"468'6163","ondisk_log_start":"468'6163","created":455,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":7021,"log_dups_size":0,"ondisk_log_size":7021,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:17:53.184877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":64225280,"num_objects":980,"num_object_clones":0,"num_object_copies":2940,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":980,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13184,"num_write_kb":453248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":924,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-23T20:44:50.747439+0000","last_change":"2023-09-23T20:35:02.523308+0000","last_active":"2023-09-23T20:44:50.747439+0000","last_peered":"2023-09-23T20:44:50.747439+0000","last_clean":"2023-09-23T20:44:50.747439+0000","last_became_active":"2023-09-23T20:33:38.979989+0000","last_became_peered":"2023-09-23T20:33:38.979989+0000","last_unstale":"2023-09-23T20:44:50.747439+0000","last_undegraded":"2023-09-23T20:44:50.747439+0000","last_fullsized":"2023-09-23T20:44:50.747439+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:35:02.523203+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:35:02.523203+0000","last_clean_scrub_stamp":"2023-09-23T20:35:02.523203+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:02:41.472958+0000","scrub_duration":0.0085927989999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":340,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-23T20:44:50.742023+0000","last_change":"2023-09-23T20:37:43.583731+0000","last_active":"2023-09-23T20:44:50.742023+0000","last_peered":"2023-09-23T20:44:50.742023+0000","last_clean":"2023-09-23T20:44:50.742023+0000","last_became_active":"2023-09-23T20:37:43.583066+0000","last_became_peered":"2023-09-23T20:37:43.583066+0000","last_unstale":"2023-09-23T20:44:50.742023+0000","last_undegraded":"2023-09-23T20:44:50.742023+0000","last_fullsized":"2023-09-23T20:44:50.742023+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:03:45.211900+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":236,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-23T20:44:50.754585+0000","last_change":"2023-09-23T20:38:02.579814+0000","last_active":"2023-09-23T20:44:50.754585+0000","last_peered":"2023-09-23T20:44:50.754585+0000","last_clean":"2023-09-23T20:44:50.754585+0000","last_became_active":"2023-09-23T20:38:02.579253+0000","last_became_peered":"2023-09-23T20:38:02.579253+0000","last_unstale":"2023-09-23T20:44:50.754585+0000","last_undegraded":"2023-09-23T20:44:50.754585+0000","last_fullsized":"2023-09-23T20:44:50.754585+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:05:07.298198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"519'13340","reported_seq":13414,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-23T20:44:58.487226+0000","last_change":"2023-09-23T20:41:04.573101+0000","last_active":"2023-09-23T20:44:58.487226+0000","last_peered":"2023-09-23T20:44:58.487226+0000","last_clean":"2023-09-23T20:44:58.487226+0000","last_became_active":"2023-09-23T20:41:04.572728+0000","last_became_peered":"2023-09-23T20:41:04.572728+0000","last_unstale":"2023-09-23T20:44:58.487226+0000","last_undegraded":"2023-09-23T20:44:58.487226+0000","last_fullsized":"2023-09-23T20:44:58.487226+0000","mapping_epoch":455,"log_start":"477'7725","ondisk_log_start":"477'7725","created":455,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":5615,"log_dups_size":0,"ondisk_log_size":5615,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:21:10.721593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":60555264,"num_objects":924,"num_object_clones":0,"num_object_copies":2772,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":924,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13340,"num_write_kb":456448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":634,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-23T20:44:51.776746+0000","last_change":"2023-09-23T20:44:46.725779+0000","last_active":"2023-09-23T20:44:51.776746+0000","last_peered":"2023-09-23T20:44:51.776746+0000","last_clean":"2023-09-23T20:44:51.776746+0000","last_became_active":"2023-09-23T20:44:46.725433+0000","last_became_peered":"2023-09-23T20:44:46.725433+0000","last_unstale":"2023-09-23T20:44:51.776746+0000","last_undegraded":"2023-09-23T20:44:51.776746+0000","last_fullsized":"2023-09-23T20:44:51.776746+0000","mapping_epoch":516,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":517,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:42:45.492220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:42:45.492220+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:28:35.202407+0000","scrub_duration":0.015259412,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":124,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-23T20:44:50.749004+0000","last_change":"2023-09-23T20:38:02.578703+0000","last_active":"2023-09-23T20:44:50.749004+0000","last_peered":"2023-09-23T20:44:50.749004+0000","last_clean":"2023-09-23T20:44:50.749004+0000","last_became_active":"2023-09-23T20:38:02.578414+0000","last_became_peered":"2023-09-23T20:38:02.578414+0000","last_unstale":"2023-09-23T20:44:50.749004+0000","last_undegraded":"2023-09-23T20:44:50.749004+0000","last_fullsized":"2023-09-23T20:44:50.749004+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:43:59.779295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":185,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-23T20:44:50.753763+0000","last_change":"2023-09-23T20:37:31.296606+0000","last_active":"2023-09-23T20:44:50.753763+0000","last_peered":"2023-09-23T20:44:50.753763+0000","last_clean":"2023-09-23T20:44:50.753763+0000","last_became_active":"2023-09-23T20:37:31.295924+0000","last_became_peered":"2023-09-23T20:37:31.295924+0000","last_unstale":"2023-09-23T20:44:50.753763+0000","last_undegraded":"2023-09-23T20:44:50.753763+0000","last_fullsized":"2023-09-23T20:44:50.753763+0000","mapping_epoch":391,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":392,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:27:50.948510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"519'13218","reported_seq":13290,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-23T20:44:57.538435+0000","last_change":"2023-09-23T20:41:04.555980+0000","last_active":"2023-09-23T20:44:57.538435+0000","last_peered":"2023-09-23T20:44:57.538435+0000","last_clean":"2023-09-23T20:44:57.538435+0000","last_became_active":"2023-09-23T20:41:04.555688+0000","last_became_peered":"2023-09-23T20:41:04.555688+0000","last_unstale":"2023-09-23T20:44:57.538435+0000","last_undegraded":"2023-09-23T20:44:57.538435+0000","last_fullsized":"2023-09-23T20:44:57.538435+0000","mapping_epoch":455,"log_start":"476'7525","ondisk_log_start":"476'7525","created":455,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":5693,"log_dups_size":0,"ondisk_log_size":5693,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:28:42.819404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":65667072,"num_objects":1002,"num_object_clones":0,"num_object_copies":3006,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1002,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13218,"num_write_kb":455040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.b","version":"90'161","reported_seq":822,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-23T20:44:50.740240+0000","last_change":"2023-09-23T20:40:22.445808+0000","last_active":"2023-09-23T20:44:50.740240+0000","last_peered":"2023-09-23T20:44:50.740240+0000","last_clean":"2023-09-23T20:44:50.740240+0000","last_became_active":"2023-09-23T20:40:22.445456+0000","last_became_peered":"2023-09-23T20:40:22.445456+0000","last_unstale":"2023-09-23T20:44:50.740240+0000","last_undegraded":"2023-09-23T20:44:50.740240+0000","last_fullsized":"2023-09-23T20:44:50.740240+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:05.595042+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:05.595042+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:39:23.678154+0000","scrub_duration":0.007271675,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":175,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-23T20:44:50.754585+0000","last_change":"2023-09-23T20:34:59.983122+0000","last_active":"2023-09-23T20:44:50.754585+0000","last_peered":"2023-09-23T20:44:50.754585+0000","last_clean":"2023-09-23T20:44:50.754585+0000","last_became_active":"2023-09-23T20:34:59.982809+0000","last_became_peered":"2023-09-23T20:34:59.982809+0000","last_unstale":"2023-09-23T20:44:50.754585+0000","last_undegraded":"2023-09-23T20:44:50.754585+0000","last_fullsized":"2023-09-23T20:44:50.754585+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:49:40.839723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":256,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-23T20:44:50.756879+0000","last_change":"2023-09-23T20:44:10.606995+0000","last_active":"2023-09-23T20:44:50.756879+0000","last_peered":"2023-09-23T20:44:50.756879+0000","last_clean":"2023-09-23T20:44:50.756879+0000","last_became_active":"2023-09-23T20:44:10.606718+0000","last_became_peered":"2023-09-23T20:44:10.606718+0000","last_unstale":"2023-09-23T20:44:50.756879+0000","last_undegraded":"2023-09-23T20:44:50.756879+0000","last_fullsized":"2023-09-23T20:44:50.756879+0000","mapping_epoch":496,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":497,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:13:12.496245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"519'13177","reported_seq":13251,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-23T20:44:54.326386+0000","last_change":"2023-09-23T20:41:04.573300+0000","last_active":"2023-09-23T20:44:54.326386+0000","last_peered":"2023-09-23T20:44:54.326386+0000","last_clean":"2023-09-23T20:44:54.326386+0000","last_became_active":"2023-09-23T20:41:04.572919+0000","last_became_peered":"2023-09-23T20:41:04.572919+0000","last_unstale":"2023-09-23T20:44:54.326386+0000","last_undegraded":"2023-09-23T20:44:54.326386+0000","last_fullsized":"2023-09-23T20:44:54.326386+0000","mapping_epoch":455,"log_start":"470'7094","ondisk_log_start":"470'7094","created":455,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":6083,"log_dups_size":0,"ondisk_log_size":6083,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:57:46.465243+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":74514432,"num_objects":1137,"num_object_clones":0,"num_object_copies":3411,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1137,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13177,"num_write_kb":458048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":544,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-23T20:44:50.747429+0000","last_change":"2023-09-23T20:43:51.693618+0000","last_active":"2023-09-23T20:44:50.747429+0000","last_peered":"2023-09-23T20:44:50.747429+0000","last_clean":"2023-09-23T20:44:50.747429+0000","last_became_active":"2023-09-23T20:43:16.765186+0000","last_became_peered":"2023-09-23T20:43:16.765186+0000","last_unstale":"2023-09-23T20:44:50.747429+0000","last_undegraded":"2023-09-23T20:44:50.747429+0000","last_fullsized":"2023-09-23T20:44:50.747429+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:43:51.693459+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:43:51.693459+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:33:30.208355+0000","scrub_duration":0.0016145269999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":314,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-23T20:44:50.741618+0000","last_change":"2023-09-23T20:37:43.570610+0000","last_active":"2023-09-23T20:44:50.741618+0000","last_peered":"2023-09-23T20:44:50.741618+0000","last_clean":"2023-09-23T20:44:50.741618+0000","last_became_active":"2023-09-23T20:37:43.570216+0000","last_became_peered":"2023-09-23T20:37:43.570216+0000","last_unstale":"2023-09-23T20:44:50.741618+0000","last_undegraded":"2023-09-23T20:44:50.741618+0000","last_fullsized":"2023-09-23T20:44:50.741618+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:39:26.555329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":344,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-23T20:44:50.739569+0000","last_change":"2023-09-23T20:32:33.230582+0000","last_active":"2023-09-23T20:44:50.739569+0000","last_peered":"2023-09-23T20:44:50.739569+0000","last_clean":"2023-09-23T20:44:50.739569+0000","last_became_active":"2023-09-23T20:32:33.230257+0000","last_became_peered":"2023-09-23T20:32:33.230257+0000","last_unstale":"2023-09-23T20:44:50.739569+0000","last_undegraded":"2023-09-23T20:44:50.739569+0000","last_fullsized":"2023-09-23T20:44:50.739569+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:11:45.333096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.f","version":"519'13449","reported_seq":13528,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-23T20:44:58.486870+0000","last_change":"2023-09-23T20:43:16.778725+0000","last_active":"2023-09-23T20:44:58.486870+0000","last_peered":"2023-09-23T20:44:58.486870+0000","last_clean":"2023-09-23T20:44:58.486870+0000","last_became_active":"2023-09-23T20:43:16.759443+0000","last_became_peered":"2023-09-23T20:43:16.759443+0000","last_unstale":"2023-09-23T20:44:58.486870+0000","last_undegraded":"2023-09-23T20:44:58.486870+0000","last_fullsized":"2023-09-23T20:44:58.486870+0000","mapping_epoch":481,"log_start":"477'7884","ondisk_log_start":"477'7884","created":455,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":5565,"log_dups_size":0,"ondisk_log_size":5565,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:55:32.141361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":61407232,"num_objects":937,"num_object_clones":0,"num_object_copies":2811,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":937,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13449,"num_write_kb":460352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":707,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-23T20:44:50.750653+0000","last_change":"2023-09-23T20:41:07.424609+0000","last_active":"2023-09-23T20:44:50.750653+0000","last_peered":"2023-09-23T20:44:50.750653+0000","last_clean":"2023-09-23T20:44:50.750653+0000","last_became_active":"2023-09-23T20:38:05.622261+0000","last_became_peered":"2023-09-23T20:38:05.622261+0000","last_unstale":"2023-09-23T20:44:50.750653+0000","last_undegraded":"2023-09-23T20:44:50.750653+0000","last_fullsized":"2023-09-23T20:44:50.750653+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:41:07.424492+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:41:07.424492+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:45:21.689123+0000","scrub_duration":0.0018960959999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"86'119","reported_seq":829,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-23T20:44:50.740473+0000","last_change":"2023-09-23T20:29:45.521257+0000","last_active":"2023-09-23T20:44:50.740473+0000","last_peered":"2023-09-23T20:44:50.740473+0000","last_clean":"2023-09-23T20:44:50.740473+0000","last_became_active":"2023-09-23T20:29:45.520546+0000","last_became_peered":"2023-09-23T20:29:45.520546+0000","last_unstale":"2023-09-23T20:44:50.740473+0000","last_undegraded":"2023-09-23T20:44:50.740473+0000","last_fullsized":"2023-09-23T20:44:50.740473+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":119,"log_dups_size":0,"ondisk_log_size":119,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:58:56.085777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032705499999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":523,"num_read_kb":3503,"num_write":136,"num_write_kb":3128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":311,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-23T20:44:50.742387+0000","last_change":"2023-09-23T20:43:16.756494+0000","last_active":"2023-09-23T20:44:50.742387+0000","last_peered":"2023-09-23T20:44:50.742387+0000","last_clean":"2023-09-23T20:44:50.742387+0000","last_became_active":"2023-09-23T20:43:16.756131+0000","last_became_peered":"2023-09-23T20:43:16.756131+0000","last_unstale":"2023-09-23T20:44:50.742387+0000","last_undegraded":"2023-09-23T20:44:50.742387+0000","last_fullsized":"2023-09-23T20:44:50.742387+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:10:19.219533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":476,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-23T20:44:50.742357+0000","last_change":"2023-09-23T20:40:19.530942+0000","last_active":"2023-09-23T20:44:50.742357+0000","last_peered":"2023-09-23T20:44:50.742357+0000","last_clean":"2023-09-23T20:44:50.742357+0000","last_became_active":"2023-09-23T20:40:19.530617+0000","last_became_peered":"2023-09-23T20:40:19.530617+0000","last_unstale":"2023-09-23T20:44:50.742357+0000","last_undegraded":"2023-09-23T20:44:50.742357+0000","last_fullsized":"2023-09-23T20:44:50.742357+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:24:09.231934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.9","version":"519'13179","reported_seq":13271,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-23T20:44:56.292830+0000","last_change":"2023-09-23T20:43:46.544365+0000","last_active":"2023-09-23T20:44:56.292830+0000","last_peered":"2023-09-23T20:44:56.292830+0000","last_clean":"2023-09-23T20:44:56.292830+0000","last_became_active":"2023-09-23T20:43:46.519248+0000","last_became_peered":"2023-09-23T20:43:46.519248+0000","last_unstale":"2023-09-23T20:44:56.292830+0000","last_undegraded":"2023-09-23T20:44:56.292830+0000","last_fullsized":"2023-09-23T20:44:56.292830+0000","mapping_epoch":487,"log_start":"467'5634","ondisk_log_start":"467'5634","created":455,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":7545,"log_dups_size":0,"ondisk_log_size":7545,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:49:43.541062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":66387968,"num_objects":1013,"num_object_clones":0,"num_object_copies":3039,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1013,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13179,"num_write_kb":454144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":564,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-23T20:44:50.745808+0000","last_change":"2023-09-23T20:44:15.656984+0000","last_active":"2023-09-23T20:44:50.745808+0000","last_peered":"2023-09-23T20:44:50.745808+0000","last_clean":"2023-09-23T20:44:50.745808+0000","last_became_active":"2023-09-23T20:43:46.512210+0000","last_became_peered":"2023-09-23T20:43:46.512210+0000","last_unstale":"2023-09-23T20:44:50.745808+0000","last_undegraded":"2023-09-23T20:44:50.745808+0000","last_fullsized":"2023-09-23T20:44:50.745808+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:15.656880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:15.656880+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:37:56.453466+0000","scrub_duration":0.0016711180000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"86'106","reported_seq":769,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-23T20:44:50.742601+0000","last_change":"2023-09-23T20:43:16.759138+0000","last_active":"2023-09-23T20:44:50.742601+0000","last_peered":"2023-09-23T20:44:50.742601+0000","last_clean":"2023-09-23T20:44:50.742601+0000","last_became_active":"2023-09-23T20:43:16.758015+0000","last_became_peered":"2023-09-23T20:43:16.758015+0000","last_unstale":"2023-09-23T20:44:50.742601+0000","last_undegraded":"2023-09-23T20:44:50.742601+0000","last_fullsized":"2023-09-23T20:44:50.742601+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":106,"log_dups_size":0,"ondisk_log_size":106,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:08:02.852472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00099302599999999993,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1094,"num_read_kb":1678,"num_write":100,"num_write_kb":404,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":236,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-23T20:44:50.754226+0000","last_change":"2023-09-23T20:38:02.552008+0000","last_active":"2023-09-23T20:44:50.754226+0000","last_peered":"2023-09-23T20:44:50.754226+0000","last_clean":"2023-09-23T20:44:50.754226+0000","last_became_active":"2023-09-23T20:38:02.551684+0000","last_became_peered":"2023-09-23T20:38:02.551684+0000","last_unstale":"2023-09-23T20:44:50.754226+0000","last_undegraded":"2023-09-23T20:44:50.754226+0000","last_fullsized":"2023-09-23T20:44:50.754226+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:40:37.147538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":482,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-23T20:44:50.740378+0000","last_change":"2023-09-23T20:29:45.533406+0000","last_active":"2023-09-23T20:44:50.740378+0000","last_peered":"2023-09-23T20:44:50.740378+0000","last_clean":"2023-09-23T20:44:50.740378+0000","last_became_active":"2023-09-23T20:29:45.526001+0000","last_became_peered":"2023-09-23T20:29:45.526001+0000","last_unstale":"2023-09-23T20:44:50.740378+0000","last_undegraded":"2023-09-23T20:44:50.740378+0000","last_fullsized":"2023-09-23T20:44:50.740378+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:43.787251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.a","version":"519'13281","reported_seq":13364,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-23T20:44:58.493079+0000","last_change":"2023-09-23T20:43:16.852909+0000","last_active":"2023-09-23T20:44:58.493079+0000","last_peered":"2023-09-23T20:44:58.493079+0000","last_clean":"2023-09-23T20:44:58.493079+0000","last_became_active":"2023-09-23T20:43:16.770100+0000","last_became_peered":"2023-09-23T20:43:16.770100+0000","last_unstale":"2023-09-23T20:44:58.493079+0000","last_undegraded":"2023-09-23T20:44:58.493079+0000","last_fullsized":"2023-09-23T20:44:58.493079+0000","mapping_epoch":481,"log_start":"477'7706","ondisk_log_start":"477'7706","created":455,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":5575,"log_dups_size":0,"ondisk_log_size":5575,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:13:16.136666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":60227584,"num_objects":919,"num_object_clones":0,"num_object_copies":2757,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":919,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13281,"num_write_kb":454400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,7],"acting":[1,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":482,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-23T20:44:50.744061+0000","last_change":"2023-09-23T20:44:26.978243+0000","last_active":"2023-09-23T20:44:50.744061+0000","last_peered":"2023-09-23T20:44:50.744061+0000","last_clean":"2023-09-23T20:44:50.744061+0000","last_became_active":"2023-09-23T20:44:18.638989+0000","last_became_peered":"2023-09-23T20:44:18.638989+0000","last_unstale":"2023-09-23T20:44:50.744061+0000","last_undegraded":"2023-09-23T20:44:50.744061+0000","last_fullsized":"2023-09-23T20:44:50.744061+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:26.978179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:26.978179+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:25:38.977011+0000","scrub_duration":0.0013962829999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"86'130","reported_seq":837,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-23T20:44:50.747355+0000","last_change":"2023-09-23T20:25:39.637458+0000","last_active":"2023-09-23T20:44:50.747355+0000","last_peered":"2023-09-23T20:44:50.747355+0000","last_clean":"2023-09-23T20:44:50.747355+0000","last_became_active":"2023-09-23T20:25:39.636773+0000","last_became_peered":"2023-09-23T20:25:39.636773+0000","last_unstale":"2023-09-23T20:44:50.747355+0000","last_undegraded":"2023-09-23T20:44:50.747355+0000","last_fullsized":"2023-09-23T20:44:50.747355+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:00:32.810564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011300049999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":512,"num_read_kb":1820,"num_write":133,"num_write_kb":1347,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":490,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-23T20:44:50.738778+0000","last_change":"2023-09-23T20:38:02.591335+0000","last_active":"2023-09-23T20:44:50.738778+0000","last_peered":"2023-09-23T20:44:50.738778+0000","last_clean":"2023-09-23T20:44:50.738778+0000","last_became_active":"2023-09-23T20:38:02.591015+0000","last_became_peered":"2023-09-23T20:38:02.591015+0000","last_unstale":"2023-09-23T20:44:50.738778+0000","last_undegraded":"2023-09-23T20:44:50.738778+0000","last_fullsized":"2023-09-23T20:44:50.738778+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:55:32.272195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":33,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-23T20:44:50.739402+0000","last_change":"2023-09-23T20:44:10.555583+0000","last_active":"2023-09-23T20:44:50.739402+0000","last_peered":"2023-09-23T20:44:50.739402+0000","last_clean":"2023-09-23T20:44:50.739402+0000","last_became_active":"2023-09-23T20:44:10.555281+0000","last_became_peered":"2023-09-23T20:44:10.555281+0000","last_unstale":"2023-09-23T20:44:50.739402+0000","last_undegraded":"2023-09-23T20:44:50.739402+0000","last_fullsized":"2023-09-23T20:44:50.739402+0000","mapping_epoch":496,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":497,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:43:11.942147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.8","version":"519'13337","reported_seq":30313,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-23T20:44:54.321778+0000","last_change":"2023-09-23T20:44:51.765604+0000","last_active":"2023-09-23T20:44:54.321778+0000","last_peered":"2023-09-23T20:44:54.321778+0000","last_clean":"2023-09-23T20:44:54.321778+0000","last_became_active":"2023-09-23T20:44:51.738188+0000","last_became_peered":"2023-09-23T20:44:51.738188+0000","last_unstale":"2023-09-23T20:44:54.321778+0000","last_undegraded":"2023-09-23T20:44:54.321778+0000","last_fullsized":"2023-09-23T20:44:54.321778+0000","mapping_epoch":518,"log_start":"470'7327","ondisk_log_start":"470'7327","created":455,"last_epoch_clean":519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":6010,"log_dups_size":0,"ondisk_log_size":6010,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:58:46.821782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":76218368,"num_objects":1163,"num_object_clones":0,"num_object_copies":3489,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1163,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13337,"num_write_kb":464000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5858,"num_bytes_recovered":383844352,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"86'194","reported_seq":875,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-23T20:44:50.747721+0000","last_change":"2023-09-23T20:43:16.765865+0000","last_active":"2023-09-23T20:44:50.747721+0000","last_peered":"2023-09-23T20:44:50.747721+0000","last_clean":"2023-09-23T20:44:50.747721+0000","last_became_active":"2023-09-23T20:43:16.765289+0000","last_became_peered":"2023-09-23T20:43:16.765289+0000","last_unstale":"2023-09-23T20:44:50.747721+0000","last_undegraded":"2023-09-23T20:44:50.747721+0000","last_fullsized":"2023-09-23T20:44:50.747721+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":194,"log_dups_size":0,"ondisk_log_size":194,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:15:16.461900+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00037995999999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1024,"num_read_kb":1200,"num_write":173,"num_write_kb":1276,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":981,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-23T20:44:50.745410+0000","last_change":"2023-09-23T20:38:05.602508+0000","last_active":"2023-09-23T20:44:50.745410+0000","last_peered":"2023-09-23T20:44:50.745410+0000","last_clean":"2023-09-23T20:44:50.745410+0000","last_became_active":"2023-09-23T20:38:05.602129+0000","last_became_peered":"2023-09-23T20:38:05.602129+0000","last_unstale":"2023-09-23T20:44:50.745410+0000","last_undegraded":"2023-09-23T20:44:50.745410+0000","last_fullsized":"2023-09-23T20:44:50.745410+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:45.343742+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:45.343742+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:48.910343+0000","scrub_duration":0.001475802,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":94,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-23T20:44:50.749075+0000","last_change":"2023-09-23T20:43:16.758213+0000","last_active":"2023-09-23T20:44:50.749075+0000","last_peered":"2023-09-23T20:44:50.749075+0000","last_clean":"2023-09-23T20:44:50.749075+0000","last_became_active":"2023-09-23T20:43:16.757869+0000","last_became_peered":"2023-09-23T20:43:16.757869+0000","last_unstale":"2023-09-23T20:44:50.749075+0000","last_undegraded":"2023-09-23T20:44:50.749075+0000","last_fullsized":"2023-09-23T20:44:50.749075+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:43:10.811749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":124,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-23T20:44:50.749345+0000","last_change":"2023-09-23T20:38:02.573200+0000","last_active":"2023-09-23T20:44:50.749345+0000","last_peered":"2023-09-23T20:44:50.749345+0000","last_clean":"2023-09-23T20:44:50.749345+0000","last_became_active":"2023-09-23T20:38:02.572945+0000","last_became_peered":"2023-09-23T20:38:02.572945+0000","last_unstale":"2023-09-23T20:44:50.749345+0000","last_undegraded":"2023-09-23T20:44:50.749345+0000","last_fullsized":"2023-09-23T20:44:50.749345+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:57.246533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.b","version":"519'13008","reported_seq":13098,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-23T20:44:56.287245+0000","last_change":"2023-09-23T20:43:46.540991+0000","last_active":"2023-09-23T20:44:56.287245+0000","last_peered":"2023-09-23T20:44:56.287245+0000","last_clean":"2023-09-23T20:44:56.287245+0000","last_became_active":"2023-09-23T20:43:46.512772+0000","last_became_peered":"2023-09-23T20:43:46.512772+0000","last_unstale":"2023-09-23T20:44:56.287245+0000","last_undegraded":"2023-09-23T20:44:56.287245+0000","last_fullsized":"2023-09-23T20:44:56.287245+0000","mapping_epoch":487,"log_start":"467'5434","ondisk_log_start":"467'5434","created":455,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":7574,"log_dups_size":0,"ondisk_log_size":7574,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:46:02.578833+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70516736,"num_objects":1076,"num_object_clones":0,"num_object_copies":3228,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1076,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13008,"num_write_kb":450688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":851,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-23T20:44:50.739998+0000","last_change":"2023-09-23T20:38:05.615711+0000","last_active":"2023-09-23T20:44:50.739998+0000","last_peered":"2023-09-23T20:44:50.739998+0000","last_clean":"2023-09-23T20:44:50.739998+0000","last_became_active":"2023-09-23T20:38:05.615390+0000","last_became_peered":"2023-09-23T20:38:05.615390+0000","last_unstale":"2023-09-23T20:44:50.739998+0000","last_undegraded":"2023-09-23T20:44:50.739998+0000","last_fullsized":"2023-09-23T20:44:50.739998+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:43.582146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:43.582146+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:45:11.917871+0000","scrub_duration":0.0016219730000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"86'80","reported_seq":771,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-23T20:44:50.744374+0000","last_change":"2023-09-23T20:29:45.536796+0000","last_active":"2023-09-23T20:44:50.744374+0000","last_peered":"2023-09-23T20:44:50.744374+0000","last_clean":"2023-09-23T20:44:50.744374+0000","last_became_active":"2023-09-23T20:29:45.536439+0000","last_became_peered":"2023-09-23T20:29:45.536439+0000","last_unstale":"2023-09-23T20:44:50.744374+0000","last_undegraded":"2023-09-23T20:44:50.744374+0000","last_fullsized":"2023-09-23T20:44:50.744374+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":80,"log_dups_size":0,"ondisk_log_size":80,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:30:57.793132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035115,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":395,"num_read_kb":1370,"num_write":66,"num_write_kb":1061,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":511,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-23T20:44:50.745635+0000","last_change":"2023-09-23T20:43:46.544563+0000","last_active":"2023-09-23T20:44:50.745635+0000","last_peered":"2023-09-23T20:44:50.745635+0000","last_clean":"2023-09-23T20:44:50.745635+0000","last_became_active":"2023-09-23T20:43:46.543958+0000","last_became_peered":"2023-09-23T20:43:46.543958+0000","last_unstale":"2023-09-23T20:44:50.745635+0000","last_undegraded":"2023-09-23T20:44:50.745635+0000","last_fullsized":"2023-09-23T20:44:50.745635+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:46:36.921752+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":506,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-23T20:44:50.745723+0000","last_change":"2023-09-23T20:43:46.520386+0000","last_active":"2023-09-23T20:44:50.745723+0000","last_peered":"2023-09-23T20:44:50.745723+0000","last_clean":"2023-09-23T20:44:50.745723+0000","last_became_active":"2023-09-23T20:43:46.519757+0000","last_became_peered":"2023-09-23T20:43:46.519757+0000","last_unstale":"2023-09-23T20:44:50.745723+0000","last_undegraded":"2023-09-23T20:44:50.745723+0000","last_fullsized":"2023-09-23T20:44:50.745723+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:23:14.662734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.e","version":"519'13251","reported_seq":13323,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-23T20:44:57.506585+0000","last_change":"2023-09-23T20:41:04.574876+0000","last_active":"2023-09-23T20:44:57.506585+0000","last_peered":"2023-09-23T20:44:57.506585+0000","last_clean":"2023-09-23T20:44:57.506585+0000","last_became_active":"2023-09-23T20:41:04.574595+0000","last_became_peered":"2023-09-23T20:41:04.574595+0000","last_unstale":"2023-09-23T20:44:57.506585+0000","last_undegraded":"2023-09-23T20:44:57.506585+0000","last_fullsized":"2023-09-23T20:44:57.506585+0000","mapping_epoch":455,"log_start":"476'7508","ondisk_log_start":"476'7508","created":455,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":5743,"log_dups_size":0,"ondisk_log_size":5743,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:31:00.147966+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":63504384,"num_objects":969,"num_object_clones":0,"num_object_copies":2907,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":969,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13251,"num_write_kb":455040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":630,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-23T20:44:50.749385+0000","last_change":"2023-09-23T20:41:08.380084+0000","last_active":"2023-09-23T20:44:50.749385+0000","last_peered":"2023-09-23T20:44:50.749385+0000","last_clean":"2023-09-23T20:44:50.749385+0000","last_became_active":"2023-09-23T20:38:05.610881+0000","last_became_peered":"2023-09-23T20:38:05.610881+0000","last_unstale":"2023-09-23T20:44:50.749385+0000","last_undegraded":"2023-09-23T20:44:50.749385+0000","last_fullsized":"2023-09-23T20:44:50.749385+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:41:08.379959+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:41:08.379959+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:19:39.547923+0000","scrub_duration":0.0019158420000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"86'102","reported_seq":1113,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-23T20:44:50.744794+0000","last_change":"2023-09-23T20:32:36.314672+0000","last_active":"2023-09-23T20:44:50.744794+0000","last_peered":"2023-09-23T20:44:50.744794+0000","last_clean":"2023-09-23T20:44:50.744794+0000","last_became_active":"2023-09-23T20:32:36.314231+0000","last_became_peered":"2023-09-23T20:32:36.314231+0000","last_unstale":"2023-09-23T20:44:50.744794+0000","last_undegraded":"2023-09-23T20:44:50.744794+0000","last_fullsized":"2023-09-23T20:44:50.744794+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":102,"log_dups_size":0,"ondisk_log_size":102,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:11:40.514709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030656500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":963,"num_read_kb":1234,"num_write":60,"num_write_kb":225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":466,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-23T20:44:50.740566+0000","last_change":"2023-09-23T20:40:19.544384+0000","last_active":"2023-09-23T20:44:50.740566+0000","last_peered":"2023-09-23T20:44:50.740566+0000","last_clean":"2023-09-23T20:44:50.740566+0000","last_became_active":"2023-09-23T20:40:19.544079+0000","last_became_peered":"2023-09-23T20:40:19.544079+0000","last_unstale":"2023-09-23T20:44:50.740566+0000","last_undegraded":"2023-09-23T20:44:50.740566+0000","last_fullsized":"2023-09-23T20:44:50.740566+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:00:19.498631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":461,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-23T20:44:50.746916+0000","last_change":"2023-09-23T20:43:46.541173+0000","last_active":"2023-09-23T20:44:50.746916+0000","last_peered":"2023-09-23T20:44:50.746916+0000","last_clean":"2023-09-23T20:44:50.746916+0000","last_became_active":"2023-09-23T20:43:46.540726+0000","last_became_peered":"2023-09-23T20:43:46.540726+0000","last_unstale":"2023-09-23T20:44:50.746916+0000","last_undegraded":"2023-09-23T20:44:50.746916+0000","last_fullsized":"2023-09-23T20:44:50.746916+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:49:04.137885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"519'13273","reported_seq":27041,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-23T20:44:57.495480+0000","last_change":"2023-09-23T20:44:18.688723+0000","last_active":"2023-09-23T20:44:57.495480+0000","last_peered":"2023-09-23T20:44:57.495480+0000","last_clean":"2023-09-23T20:44:57.495480+0000","last_became_active":"2023-09-23T20:44:18.636162+0000","last_became_peered":"2023-09-23T20:44:18.636162+0000","last_unstale":"2023-09-23T20:44:57.495480+0000","last_undegraded":"2023-09-23T20:44:57.495480+0000","last_fullsized":"2023-09-23T20:44:57.495480+0000","mapping_epoch":502,"log_start":"476'7548","ondisk_log_start":"476'7548","created":455,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":5725,"log_dups_size":0,"ondisk_log_size":5725,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:49:45.956725+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":66322456,"num_objects":1013,"num_object_clones":0,"num_object_copies":3039,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1013,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":13273,"num_write_kb":457089,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6816,"num_bytes_recovered":446627864,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":797,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-23T20:44:50.756425+0000","last_change":"2023-09-23T20:38:05.590361+0000","last_active":"2023-09-23T20:44:50.756425+0000","last_peered":"2023-09-23T20:44:50.756425+0000","last_clean":"2023-09-23T20:44:50.756425+0000","last_became_active":"2023-09-23T20:38:05.589985+0000","last_became_peered":"2023-09-23T20:38:05.589985+0000","last_unstale":"2023-09-23T20:44:50.756425+0000","last_undegraded":"2023-09-23T20:44:50.756425+0000","last_fullsized":"2023-09-23T20:44:50.756425+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:03.246271+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:03.246271+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:38:54.351576+0000","scrub_duration":0.0019016700000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":256,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-23T20:44:50.744881+0000","last_change":"2023-09-23T20:32:50.549695+0000","last_active":"2023-09-23T20:44:50.744881+0000","last_peered":"2023-09-23T20:44:50.744881+0000","last_clean":"2023-09-23T20:44:50.744881+0000","last_became_active":"2023-09-23T20:32:50.549431+0000","last_became_peered":"2023-09-23T20:32:50.549431+0000","last_unstale":"2023-09-23T20:44:50.744881+0000","last_undegraded":"2023-09-23T20:44:50.744881+0000","last_fullsized":"2023-09-23T20:44:50.744881+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:45:53.666569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"86'128","reported_seq":932,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-23T20:44:51.783176+0000","last_change":"2023-09-23T20:38:03.541765+0000","last_active":"2023-09-23T20:44:51.783176+0000","last_peered":"2023-09-23T20:44:51.783176+0000","last_clean":"2023-09-23T20:44:51.783176+0000","last_became_active":"2023-09-23T20:38:03.541277+0000","last_became_peered":"2023-09-23T20:38:03.541277+0000","last_unstale":"2023-09-23T20:44:51.783176+0000","last_undegraded":"2023-09-23T20:44:51.783176+0000","last_fullsized":"2023-09-23T20:44:51.783176+0000","mapping_epoch":404,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":405,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:20:53.834554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000293202,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":579,"num_read_kb":1473,"num_write":96,"num_write_kb":1062,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":299,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-23T20:44:50.741547+0000","last_change":"2023-09-23T20:37:43.582105+0000","last_active":"2023-09-23T20:44:50.741547+0000","last_peered":"2023-09-23T20:44:50.741547+0000","last_clean":"2023-09-23T20:44:50.741547+0000","last_became_active":"2023-09-23T20:37:43.581860+0000","last_became_peered":"2023-09-23T20:37:43.581860+0000","last_unstale":"2023-09-23T20:44:50.741547+0000","last_undegraded":"2023-09-23T20:44:50.741547+0000","last_fullsized":"2023-09-23T20:44:50.741547+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:18:38.765486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.c","version":"519'13102","reported_seq":13176,"reported_epoch":519,"state":"active+clean","last_fresh":"2023-09-23T20:44:54.289356+0000","last_change":"2023-09-23T20:41:04.575496+0000","last_active":"2023-09-23T20:44:54.289356+0000","last_peered":"2023-09-23T20:44:54.289356+0000","last_clean":"2023-09-23T20:44:54.289356+0000","last_became_active":"2023-09-23T20:41:04.575242+0000","last_became_peered":"2023-09-23T20:41:04.575242+0000","last_unstale":"2023-09-23T20:44:54.289356+0000","last_undegraded":"2023-09-23T20:44:54.289356+0000","last_fullsized":"2023-09-23T20:44:54.289356+0000","mapping_epoch":455,"log_start":"470'7075","ondisk_log_start":"470'7075","created":455,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":6027,"log_dups_size":0,"ondisk_log_size":6027,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:30:14.611529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":75366400,"num_objects":1150,"num_object_clones":0,"num_object_copies":3450,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1150,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13102,"num_write_kb":456064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"90'161","reported_seq":733,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-23T20:44:50.744620+0000","last_change":"2023-09-23T20:40:22.427435+0000","last_active":"2023-09-23T20:44:50.744620+0000","last_peered":"2023-09-23T20:44:50.744620+0000","last_clean":"2023-09-23T20:44:50.744620+0000","last_became_active":"2023-09-23T20:40:22.427051+0000","last_became_peered":"2023-09-23T20:40:22.427051+0000","last_unstale":"2023-09-23T20:44:50.744620+0000","last_undegraded":"2023-09-23T20:44:50.744620+0000","last_fullsized":"2023-09-23T20:44:50.744620+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:34:54.504554+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:34:54.504554+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:42.001652+0000","scrub_duration":0.011176864,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"86'117","reported_seq":827,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-23T20:44:50.750560+0000","last_change":"2023-09-23T20:44:18.635224+0000","last_active":"2023-09-23T20:44:50.750560+0000","last_peered":"2023-09-23T20:44:50.750560+0000","last_clean":"2023-09-23T20:44:50.750560+0000","last_became_active":"2023-09-23T20:44:18.634703+0000","last_became_peered":"2023-09-23T20:44:18.634703+0000","last_unstale":"2023-09-23T20:44:50.750560+0000","last_undegraded":"2023-09-23T20:44:50.750560+0000","last_fullsized":"2023-09-23T20:44:50.750560+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:59:38.030626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046891400000000002,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":893,"num_read_kb":922,"num_write":82,"num_write_kb":4193,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":16777216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":506,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-23T20:44:50.745833+0000","last_change":"2023-09-23T20:43:46.521505+0000","last_active":"2023-09-23T20:44:50.745833+0000","last_peered":"2023-09-23T20:44:50.745833+0000","last_clean":"2023-09-23T20:44:50.745833+0000","last_became_active":"2023-09-23T20:43:46.520727+0000","last_became_peered":"2023-09-23T20:43:46.520727+0000","last_unstale":"2023-09-23T20:44:50.745833+0000","last_undegraded":"2023-09-23T20:44:50.745833+0000","last_fullsized":"2023-09-23T20:44:50.745833+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:17:06.164741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":256,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-23T20:44:50.754831+0000","last_change":"2023-09-23T20:40:19.530602+0000","last_active":"2023-09-23T20:44:50.754831+0000","last_peered":"2023-09-23T20:44:50.754831+0000","last_clean":"2023-09-23T20:44:50.754831+0000","last_became_active":"2023-09-23T20:40:19.530256+0000","last_became_peered":"2023-09-23T20:40:19.530256+0000","last_unstale":"2023-09-23T20:44:50.754831+0000","last_undegraded":"2023-09-23T20:44:50.754831+0000","last_fullsized":"2023-09-23T20:44:50.754831+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:18:45.658867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"90'161","reported_seq":775,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-23T20:44:50.745322+0000","last_change":"2023-09-23T20:43:46.544750+0000","last_active":"2023-09-23T20:44:50.745322+0000","last_peered":"2023-09-23T20:44:50.745322+0000","last_clean":"2023-09-23T20:44:50.745322+0000","last_became_active":"2023-09-23T20:43:46.544395+0000","last_became_peered":"2023-09-23T20:43:46.544395+0000","last_unstale":"2023-09-23T20:44:50.745322+0000","last_undegraded":"2023-09-23T20:44:50.745322+0000","last_fullsized":"2023-09-23T20:44:50.745322+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:06.487351+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:06.487351+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:56.866168+0000","scrub_duration":0.001964379,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":473,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-23T20:44:50.746894+0000","last_change":"2023-09-23T20:43:46.541379+0000","last_active":"2023-09-23T20:44:50.746894+0000","last_peered":"2023-09-23T20:44:50.746894+0000","last_clean":"2023-09-23T20:44:50.746894+0000","last_became_active":"2023-09-23T20:43:46.540732+0000","last_became_peered":"2023-09-23T20:43:46.540732+0000","last_unstale":"2023-09-23T20:44:50.746894+0000","last_undegraded":"2023-09-23T20:44:50.746894+0000","last_fullsized":"2023-09-23T20:44:50.746894+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:31:40.605772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":375,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-23T20:44:50.747942+0000","last_change":"2023-09-23T20:29:45.538314+0000","last_active":"2023-09-23T20:44:50.747942+0000","last_peered":"2023-09-23T20:44:50.747942+0000","last_clean":"2023-09-23T20:44:50.747942+0000","last_became_active":"2023-09-23T20:29:45.536847+0000","last_became_peered":"2023-09-23T20:29:45.536847+0000","last_unstale":"2023-09-23T20:44:50.747942+0000","last_undegraded":"2023-09-23T20:44:50.747942+0000","last_fullsized":"2023-09-23T20:44:50.747942+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:51.948721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"90'161","reported_seq":592,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-23T20:44:50.742580+0000","last_change":"2023-09-23T20:41:06.499511+0000","last_active":"2023-09-23T20:44:50.742580+0000","last_peered":"2023-09-23T20:44:50.742580+0000","last_clean":"2023-09-23T20:44:50.742580+0000","last_became_active":"2023-09-23T20:38:05.601874+0000","last_became_peered":"2023-09-23T20:38:05.601874+0000","last_unstale":"2023-09-23T20:44:50.742580+0000","last_undegraded":"2023-09-23T20:44:50.742580+0000","last_fullsized":"2023-09-23T20:44:50.742580+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:41:06.499396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:41:06.499396+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:31:34.059077+0000","scrub_duration":0.0019670310000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":411,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-23T20:44:50.754734+0000","last_change":"2023-09-23T20:43:16.758761+0000","last_active":"2023-09-23T20:44:50.754734+0000","last_peered":"2023-09-23T20:44:50.754734+0000","last_clean":"2023-09-23T20:44:50.754734+0000","last_became_active":"2023-09-23T20:43:16.758407+0000","last_became_peered":"2023-09-23T20:43:16.758407+0000","last_unstale":"2023-09-23T20:44:50.754734+0000","last_undegraded":"2023-09-23T20:44:50.754734+0000","last_fullsized":"2023-09-23T20:44:50.754734+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:59:27.070685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":175,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-23T20:44:50.753906+0000","last_change":"2023-09-23T20:34:59.987575+0000","last_active":"2023-09-23T20:44:50.753906+0000","last_peered":"2023-09-23T20:44:50.753906+0000","last_clean":"2023-09-23T20:44:50.753906+0000","last_became_active":"2023-09-23T20:34:59.987229+0000","last_became_peered":"2023-09-23T20:34:59.987229+0000","last_unstale":"2023-09-23T20:44:50.753906+0000","last_undegraded":"2023-09-23T20:44:50.753906+0000","last_fullsized":"2023-09-23T20:44:50.753906+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:34:01.820159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":419,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-23T20:44:50.756031+0000","last_change":"2023-09-23T20:43:16.759691+0000","last_active":"2023-09-23T20:44:50.756031+0000","last_peered":"2023-09-23T20:44:50.756031+0000","last_clean":"2023-09-23T20:44:50.756031+0000","last_became_active":"2023-09-23T20:43:16.759309+0000","last_became_peered":"2023-09-23T20:43:16.759309+0000","last_unstale":"2023-09-23T20:44:50.756031+0000","last_undegraded":"2023-09-23T20:44:50.756031+0000","last_fullsized":"2023-09-23T20:44:50.756031+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:58:52.893019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":175,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-23T20:44:50.753871+0000","last_change":"2023-09-23T20:34:59.987416+0000","last_active":"2023-09-23T20:44:50.753871+0000","last_peered":"2023-09-23T20:44:50.753871+0000","last_clean":"2023-09-23T20:44:50.753871+0000","last_became_active":"2023-09-23T20:34:59.987117+0000","last_became_peered":"2023-09-23T20:34:59.987117+0000","last_unstale":"2023-09-23T20:44:50.753871+0000","last_undegraded":"2023-09-23T20:44:50.753871+0000","last_fullsized":"2023-09-23T20:44:50.753871+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:36:16.318623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"90'161","reported_seq":774,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-23T20:44:50.744329+0000","last_change":"2023-09-23T20:38:05.591992+0000","last_active":"2023-09-23T20:44:50.744329+0000","last_peered":"2023-09-23T20:44:50.744329+0000","last_clean":"2023-09-23T20:44:50.744329+0000","last_became_active":"2023-09-23T20:38:05.591701+0000","last_became_peered":"2023-09-23T20:38:05.591701+0000","last_unstale":"2023-09-23T20:44:50.744329+0000","last_undegraded":"2023-09-23T20:44:50.744329+0000","last_fullsized":"2023-09-23T20:44:50.744329+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:02.384133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:02.384133+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:41.722542+0000","scrub_duration":0.001769098,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":341,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-23T20:44:50.742764+0000","last_change":"2023-09-23T20:38:02.542719+0000","last_active":"2023-09-23T20:44:50.742764+0000","last_peered":"2023-09-23T20:44:50.742764+0000","last_clean":"2023-09-23T20:44:50.742764+0000","last_became_active":"2023-09-23T20:38:02.542385+0000","last_became_peered":"2023-09-23T20:38:02.542385+0000","last_unstale":"2023-09-23T20:44:50.742764+0000","last_undegraded":"2023-09-23T20:44:50.742764+0000","last_fullsized":"2023-09-23T20:44:50.742764+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:36:23.673757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":307,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-23T20:44:50.744032+0000","last_change":"2023-09-23T20:37:43.584253+0000","last_active":"2023-09-23T20:44:50.744032+0000","last_peered":"2023-09-23T20:44:50.744032+0000","last_clean":"2023-09-23T20:44:50.744032+0000","last_became_active":"2023-09-23T20:37:43.580069+0000","last_became_peered":"2023-09-23T20:37:43.580069+0000","last_unstale":"2023-09-23T20:44:50.744032+0000","last_undegraded":"2023-09-23T20:44:50.744032+0000","last_fullsized":"2023-09-23T20:44:50.744032+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:39:11.482581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"90'161","reported_seq":612,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-23T20:44:50.739441+0000","last_change":"2023-09-23T20:36:01.505098+0000","last_active":"2023-09-23T20:44:50.739441+0000","last_peered":"2023-09-23T20:44:50.739441+0000","last_clean":"2023-09-23T20:44:50.739441+0000","last_became_active":"2023-09-23T20:35:26.644707+0000","last_became_peered":"2023-09-23T20:35:26.644707+0000","last_unstale":"2023-09-23T20:44:50.739441+0000","last_undegraded":"2023-09-23T20:44:50.739441+0000","last_fullsized":"2023-09-23T20:44:50.739441+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:01.504973+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:01.504973+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:02:10.880809+0000","scrub_duration":0.00202344,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":551,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-23T20:44:50.746199+0000","last_change":"2023-09-23T20:43:46.511657+0000","last_active":"2023-09-23T20:44:50.746199+0000","last_peered":"2023-09-23T20:44:50.746199+0000","last_clean":"2023-09-23T20:44:50.746199+0000","last_became_active":"2023-09-23T20:43:46.511326+0000","last_became_peered":"2023-09-23T20:43:46.511326+0000","last_unstale":"2023-09-23T20:44:50.746199+0000","last_undegraded":"2023-09-23T20:44:50.746199+0000","last_fullsized":"2023-09-23T20:44:50.746199+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:36:46.139261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3324,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":236,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-23T20:44:50.753635+0000","last_change":"2023-09-23T20:38:02.538333+0000","last_active":"2023-09-23T20:44:50.753635+0000","last_peered":"2023-09-23T20:44:50.753635+0000","last_clean":"2023-09-23T20:44:50.753635+0000","last_became_active":"2023-09-23T20:38:02.538008+0000","last_became_peered":"2023-09-23T20:38:02.538008+0000","last_unstale":"2023-09-23T20:44:50.753635+0000","last_undegraded":"2023-09-23T20:44:50.753635+0000","last_fullsized":"2023-09-23T20:44:50.753635+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:02:38.789530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":90,"num_bytes_recovered":18486,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26691,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":1057095704,"num_objects":16131,"num_object_clones":0,"num_object_copies":48393,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":16131,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":211533,"num_write_kb":7285185,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16881,"num_bytes_recovered":1105854488,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3611901952,"data_stored":3611885664,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":102728,"ondisk_log_size":102728,"up":48,"acting":48,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5983,"num_read_kb":13200,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":84,"num_bytes_recovered":16777216,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6082,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201682,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1166764,"kb_used_data":663908,"kb_used_omap":0,"kb_used_meta":502784,"kb_avail":92578388,"statfs":{"total":95995035648,"available":94800269312,"internally_reserved":0,"allocated":679841792,"data_stored":678318199,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":514850816},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:43:58 2023","interfaces":[{"interface":"back","average":{"1min":2.5600000000000001,"5min":1.643,"15min":2.089},"min":{"1min":0.23899999999999999,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":15.1,"5min":15.1,"15min":18.315000000000001},"last":0.24199999999999999},{"interface":"front","average":{"1min":2.7040000000000002,"5min":1.587,"15min":1.8069999999999999},"min":{"1min":0.33300000000000002,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":15.327999999999999,"5min":15.327999999999999,"15min":19.355},"last":2.4079999999999999}]},{"osd":1,"last update":"Sat Sep 23 20:44:08 2023","interfaces":[{"interface":"back","average":{"1min":2.714,"5min":1.452,"15min":2.105},"min":{"1min":0.26900000000000002,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":15.131,"5min":15.131,"15min":49.965000000000003},"last":1.526},{"interface":"front","average":{"1min":3.6850000000000001,"5min":2.266,"15min":2.339},"min":{"1min":0.36299999999999999,"5min":0.26700000000000002,"15min":0.248},"max":{"1min":15.18,"5min":15.18,"15min":53.451999999999998},"last":2.4460000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:43:58 2023","interfaces":[{"interface":"back","average":{"1min":3.1859999999999999,"5min":2.0600000000000001,"15min":2.1880000000000002},"min":{"1min":0.39800000000000002,"5min":0.27900000000000003,"15min":0.27600000000000002},"max":{"1min":15.087,"5min":15.087,"15min":16.965},"last":2.4740000000000002},{"interface":"front","average":{"1min":2.2320000000000002,"5min":1.716,"15min":2.4279999999999999},"min":{"1min":0.35899999999999999,"5min":0.30299999999999999,"15min":0.26600000000000001},"max":{"1min":15.193,"5min":15.193,"15min":23.494},"last":2.1179999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:44:28 2023","interfaces":[{"interface":"back","average":{"1min":2.6840000000000002,"5min":1.772,"15min":2.371},"min":{"1min":0.41399999999999998,"5min":0.32500000000000001,"15min":0.29199999999999998},"max":{"1min":15.115,"5min":15.115,"15min":23.352},"last":5.9169999999999998},{"interface":"front","average":{"1min":3.3570000000000002,"5min":2.1389999999999998,"15min":2.4489999999999998},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.33100000000000002},"max":{"1min":15.144,"5min":15.144,"15min":20.969999999999999},"last":4.242}]},{"osd":4,"last update":"Sat Sep 23 20:44:28 2023","interfaces":[{"interface":"back","average":{"1min":2.0110000000000001,"5min":1.391,"15min":1.2669999999999999},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.23200000000000001},"max":{"1min":12.260999999999999,"5min":12.260999999999999,"15min":18.364999999999998},"last":2.2949999999999999},{"interface":"front","average":{"1min":2.145,"5min":1.5640000000000001,"15min":1.3320000000000001},"min":{"1min":0.315,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":6.9939999999999998,"5min":6.9939999999999998,"15min":23.617000000000001},"last":2.524}]},{"osd":5,"last update":"Sat Sep 23 20:44:28 2023","interfaces":[{"interface":"back","average":{"1min":2.5270000000000001,"5min":1.6040000000000001,"15min":1.1970000000000001},"min":{"1min":0.36899999999999999,"5min":0.30099999999999999,"15min":0.26600000000000001},"max":{"1min":15.005000000000001,"5min":15.005000000000001,"15min":15.005000000000001},"last":5.9619999999999997},{"interface":"front","average":{"1min":1.9990000000000001,"5min":1.556,"15min":1.3120000000000001},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":12.212,"5min":12.731,"15min":18.831},"last":9.0839999999999996}]},{"osd":6,"last update":"Sat Sep 23 20:44:28 2023","interfaces":[{"interface":"back","average":{"1min":3.6379999999999999,"5min":1.837,"15min":1.4359999999999999},"min":{"1min":0.34599999999999997,"5min":0.33700000000000002,"15min":0.30399999999999999},"max":{"1min":15.038,"5min":15.038,"15min":18.417000000000002},"last":0.44900000000000001},{"interface":"front","average":{"1min":2.8969999999999998,"5min":1.6659999999999999,"15min":1.274},"min":{"1min":0.47999999999999998,"5min":0.36499999999999999,"15min":0.318},"max":{"1min":15.369999999999999,"5min":15.369999999999999,"15min":15.369999999999999},"last":2.4990000000000001}]}]},{"osd":6,"up_from":47,"seq":201863463321,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":624476,"kb_used_data":196180,"kb_used_omap":0,"kb_used_meta":428288,"kb_avail":93120676,"statfs":{"total":95995035648,"available":95355572224,"internally_reserved":0,"allocated":200888320,"data_stored":199380416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":438566912},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:44:55 2023","interfaces":[{"interface":"back","average":{"1min":1.883,"5min":1.5720000000000001,"15min":1.986},"min":{"1min":0.219,"5min":0.219,"15min":0.219},"max":{"1min":7.7720000000000002,"5min":12.552,"15min":53.701999999999998},"last":2.4079999999999999},{"interface":"front","average":{"1min":2.343,"5min":1.7430000000000001,"15min":2.3730000000000002},"min":{"1min":0.313,"5min":0.23300000000000001,"15min":0.22700000000000001},"max":{"1min":10.895,"5min":10.895,"15min":60.328000000000003},"last":2.6619999999999999}]},{"osd":1,"last update":"Sat Sep 23 20:44:17 2023","interfaces":[{"interface":"back","average":{"1min":3.3330000000000002,"5min":1.744,"15min":2.1309999999999998},"min":{"1min":0.317,"5min":0.249,"15min":0.249},"max":{"1min":9.2520000000000007,"5min":9.2520000000000007,"15min":64.274000000000001},"last":0.28899999999999998},{"interface":"front","average":{"1min":2.8940000000000001,"5min":1.8100000000000001,"15min":2.0739999999999998},"min":{"1min":0.42799999999999999,"5min":0.32000000000000001,"15min":0.28000000000000003},"max":{"1min":17.786000000000001,"5min":17.786000000000001,"15min":34.490000000000002},"last":7.4400000000000004}]},{"osd":2,"last update":"Sat Sep 23 20:44:55 2023","interfaces":[{"interface":"back","average":{"1min":2.5409999999999999,"5min":2.2370000000000001,"15min":2.1739999999999999},"min":{"1min":0.34899999999999998,"5min":0.27800000000000002,"15min":0.27700000000000002},"max":{"1min":6.6130000000000004,"5min":9.9049999999999994,"15min":34.317999999999998},"last":5.1029999999999998},{"interface":"front","average":{"1min":2.3079999999999998,"5min":2.0129999999999999,"15min":2.069},"min":{"1min":0.373,"5min":0.311,"15min":0.30399999999999999},"max":{"1min":6.5620000000000003,"5min":9.9269999999999996,"15min":34.292999999999999},"last":5.1239999999999997}]},{"osd":3,"last update":"Sat Sep 23 20:44:46 2023","interfaces":[{"interface":"back","average":{"1min":2.7330000000000001,"5min":2.024,"15min":1.974},"min":{"1min":0.47999999999999998,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":11.664,"5min":11.664,"15min":11.664},"last":0.45200000000000001},{"interface":"front","average":{"1min":2.3039999999999998,"5min":2.0299999999999998,"15min":2.1459999999999999},"min":{"1min":0.42099999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":8.2560000000000002,"5min":11.808,"15min":12.875999999999999},"last":2.6989999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:44:17 2023","interfaces":[{"interface":"back","average":{"1min":1.27,"5min":0.997,"15min":0.85199999999999998},"min":{"1min":0.33600000000000002,"5min":0.25600000000000001,"15min":0.251},"max":{"1min":6.5800000000000001,"5min":6.5800000000000001,"15min":23.904},"last":0.37},{"interface":"front","average":{"1min":1.4019999999999999,"5min":1.1519999999999999,"15min":1.1240000000000001},"min":{"1min":0.34999999999999998,"5min":0.309,"15min":0.27000000000000002},"max":{"1min":5.6260000000000003,"5min":6.7359999999999998,"15min":11.515000000000001},"last":0.32300000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:44:17 2023","interfaces":[{"interface":"back","average":{"1min":1.381,"5min":0.97099999999999997,"15min":0.86499999999999999},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":6.5460000000000003,"5min":6.6340000000000003,"15min":8.4499999999999993},"last":0.41799999999999998},{"interface":"front","average":{"1min":1.5509999999999999,"5min":1.4870000000000001,"15min":1.1970000000000001},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":5.4969999999999999,"5min":7.1319999999999997,"15min":9.9299999999999997},"last":0.38900000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:44:46 2023","interfaces":[{"interface":"back","average":{"1min":1.4199999999999999,"5min":1.079,"15min":0.94799999999999995},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":7.8250000000000002,"5min":8.8350000000000009,"15min":8.8350000000000009},"last":0.46200000000000002},{"interface":"front","average":{"1min":2.6440000000000001,"5min":1.661,"15min":1.3009999999999999},"min":{"1min":0.45900000000000002,"5min":0.34699999999999998,"15min":0.30299999999999999},"max":{"1min":7.1239999999999997,"5min":7.1929999999999996,"15min":8.8119999999999994},"last":0.35999999999999999}]}]},{"osd":1,"up_from":266,"seq":1142461300920,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1442480,"kb_used_data":773992,"kb_used_omap":8,"kb_used_meta":668471,"kb_avail":92302672,"statfs":{"total":95995035648,"available":94517936128,"internally_reserved":0,"allocated":792567808,"data_stored":789877066,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8206,"internal_metadata":684515314},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,4,1,0,2,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:44:15 2023","interfaces":[{"interface":"back","average":{"1min":3.6110000000000002,"5min":2.206,"15min":2.5329999999999999},"min":{"1min":0.214,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":15.532999999999999,"5min":16.292000000000002,"15min":52.689999999999998},"last":5.0220000000000002},{"interface":"front","average":{"1min":3.2290000000000001,"5min":2.2970000000000002,"15min":3.0179999999999998},"min":{"1min":0.22600000000000001,"5min":0.20100000000000001,"15min":0.19900000000000001},"max":{"1min":14.445,"5min":16.332999999999998,"15min":52.662999999999997},"last":20.346}]},{"osd":2,"last update":"Sat Sep 23 20:44:15 2023","interfaces":[{"interface":"back","average":{"1min":3.0390000000000001,"5min":2.4140000000000001,"15min":3.3370000000000002},"min":{"1min":0.28999999999999998,"5min":0.216,"15min":0.20899999999999999},"max":{"1min":8.6519999999999992,"5min":14.148,"15min":32.255000000000003},"last":0.253},{"interface":"front","average":{"1min":2.9049999999999998,"5min":1.9810000000000001,"15min":2.2909999999999999},"min":{"1min":0.26000000000000001,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":10.041,"5min":10.041,"15min":13.404999999999999},"last":1.4079999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:44:15 2023","interfaces":[{"interface":"back","average":{"1min":2.5830000000000002,"5min":2.089,"15min":3.097},"min":{"1min":0.36899999999999999,"5min":0.247,"15min":0.247},"max":{"1min":7.2469999999999999,"5min":16.245999999999999,"15min":80.745999999999995},"last":1.371},{"interface":"front","average":{"1min":3.5779999999999998,"5min":2.645,"15min":3.3450000000000002},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.22700000000000001},"max":{"1min":11.523,"5min":13.497999999999999,"15min":82.988},"last":0.33600000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:44:15 2023","interfaces":[{"interface":"back","average":{"1min":2.3250000000000002,"5min":2.202,"15min":2.25},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":6.5970000000000004,"5min":20.634,"15min":24.327999999999999},"last":1.5309999999999999},{"interface":"front","average":{"1min":2.0190000000000001,"5min":1.982,"15min":1.748},"min":{"1min":0.34699999999999998,"5min":0.32600000000000001,"15min":0.311},"max":{"1min":6.0869999999999997,"5min":20.835999999999999,"15min":20.835999999999999},"last":1.4319999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:44:15 2023","interfaces":[{"interface":"back","average":{"1min":3.2040000000000002,"5min":2.4319999999999999,"15min":2.3029999999999999},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":14.366,"5min":14.366,"15min":33.401000000000003},"last":6.3869999999999996},{"interface":"front","average":{"1min":2.7469999999999999,"5min":2.3519999999999999,"15min":2.4820000000000002},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.32500000000000001},"max":{"1min":7.3300000000000001,"5min":10.958,"15min":33.417000000000002},"last":1.732}]},{"osd":6,"last update":"Sat Sep 23 20:44:15 2023","interfaces":[{"interface":"back","average":{"1min":3.0030000000000001,"5min":2.3639999999999999,"15min":2.839},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35699999999999998},"max":{"1min":14.414999999999999,"5min":14.414999999999999,"15min":72.682000000000002},"last":0.42399999999999999},{"interface":"front","average":{"1min":3.2829999999999999,"5min":2.3620000000000001,"15min":2.9860000000000002},"min":{"1min":0.38500000000000001,"5min":0.317,"15min":0.317},"max":{"1min":9.1470000000000002,"5min":9.4090000000000007,"15min":72.563999999999993},"last":5.4569999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:44:15 2023","interfaces":[{"interface":"back","average":{"1min":2.98,"5min":2.4470000000000001,"15min":2.3700000000000001},"min":{"1min":0.43099999999999999,"5min":0.37,"15min":0.37},"max":{"1min":9.2409999999999997,"5min":16.359000000000002,"15min":18.437000000000001},"last":6.431},{"interface":"front","average":{"1min":2.6459999999999999,"5min":2.4460000000000002,"15min":2.4910000000000001},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":7.1959999999999997,"5min":10.34,"15min":24.524000000000001},"last":5.4260000000000002}]}]},{"osd":0,"up_from":94,"seq":403726926122,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":993020,"kb_used_data":283956,"kb_used_omap":7,"kb_used_meta":708984,"kb_avail":92752132,"statfs":{"total":95995035648,"available":94978183168,"internally_reserved":0,"allocated":290770944,"data_stored":288245634,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":725999633},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:44:24 2023","interfaces":[{"interface":"back","average":{"1min":3.2120000000000002,"5min":2.4100000000000001,"15min":2.7210000000000001},"min":{"1min":0.184,"5min":0.184,"15min":0.17199999999999999},"max":{"1min":18.052,"5min":24.010000000000002,"15min":42.000999999999998},"last":3.7290000000000001},{"interface":"front","average":{"1min":3.0939999999999999,"5min":2.605,"15min":2.9129999999999998},"min":{"1min":0.33800000000000002,"5min":0.218,"15min":0.20399999999999999},"max":{"1min":18.097000000000001,"5min":18.097000000000001,"15min":23.036000000000001},"last":3.7549999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:43:58 2023","interfaces":[{"interface":"back","average":{"1min":2.5169999999999999,"5min":2.5699999999999998,"15min":3.0190000000000001},"min":{"1min":0.248,"5min":0.23400000000000001,"15min":0.185},"max":{"1min":10.199,"5min":48.841000000000001,"15min":77.448999999999998},"last":0.39400000000000002},{"interface":"front","average":{"1min":1.9590000000000001,"5min":2.3239999999999998,"15min":2.9489999999999998},"min":{"1min":0.22,"5min":0.22,"15min":0.188},"max":{"1min":6.6580000000000004,"5min":46.183,"15min":81.724999999999994},"last":3.641}]},{"osd":3,"last update":"Sat Sep 23 20:44:31 2023","interfaces":[{"interface":"back","average":{"1min":3.6960000000000002,"5min":2.7290000000000001,"15min":3.1640000000000001},"min":{"1min":0.30599999999999999,"5min":0.248,"15min":0.23400000000000001},"max":{"1min":15.048,"5min":15.048,"15min":81.400999999999996},"last":0.35299999999999998},{"interface":"front","average":{"1min":2.1549999999999998,"5min":2.3159999999999998,"15min":3.3439999999999999},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.21199999999999999},"max":{"1min":9.6379999999999999,"5min":14.237,"15min":81.137},"last":3.7000000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:43:58 2023","interfaces":[{"interface":"back","average":{"1min":1.385,"5min":1.925,"15min":1.9750000000000001},"min":{"1min":0.32000000000000001,"5min":0.31900000000000001,"15min":0.25900000000000001},"max":{"1min":4.5279999999999996,"5min":6.8970000000000002,"15min":12.587999999999999},"last":0.42799999999999999},{"interface":"front","average":{"1min":1.7310000000000001,"5min":1.823,"15min":1.917},"min":{"1min":0.28899999999999998,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":6.5890000000000004,"5min":9.8480000000000008,"15min":23.001000000000001},"last":0.58799999999999997}]},{"osd":5,"last update":"Sat Sep 23 20:43:58 2023","interfaces":[{"interface":"back","average":{"1min":2.077,"5min":2.2370000000000001,"15min":2.121},"min":{"1min":0.30599999999999999,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":6.6200000000000001,"5min":10.239000000000001,"15min":22.969999999999999},"last":1.169},{"interface":"front","average":{"1min":2.5529999999999999,"5min":2.2010000000000001,"15min":2.3719999999999999},"min":{"1min":0.34300000000000003,"5min":0.318,"15min":0.25600000000000001},"max":{"1min":6.75,"5min":12.929,"15min":26.228000000000002},"last":0.46100000000000002}]},{"osd":6,"last update":"Sat Sep 23 20:43:58 2023","interfaces":[{"interface":"back","average":{"1min":2.1659999999999999,"5min":2.1909999999999998,"15min":2.1680000000000001},"min":{"1min":0.433,"5min":0.34499999999999997,"15min":0.32300000000000001},"max":{"1min":7.6150000000000002,"5min":24.056000000000001,"15min":24.056000000000001},"last":0.61399999999999999},{"interface":"front","average":{"1min":2.2799999999999998,"5min":1.79,"15min":2.1539999999999999},"min":{"1min":0.36399999999999999,"5min":0.34100000000000003,"15min":0.29399999999999998},"max":{"1min":6.7119999999999997,"5min":8.0549999999999997,"15min":26.268000000000001},"last":0.53500000000000003}]},{"osd":7,"last update":"Sat Sep 23 20:43:58 2023","interfaces":[{"interface":"back","average":{"1min":2.9220000000000002,"5min":2.0899999999999999,"15min":2.3969999999999998},"min":{"1min":0.433,"5min":0.36799999999999999,"15min":0.30599999999999999},"max":{"1min":9.4250000000000007,"5min":9.4250000000000007,"15min":26.177},"last":0.502},{"interface":"front","average":{"1min":2.669,"5min":2.4100000000000001,"15min":2.4529999999999998},"min":{"1min":0.46899999999999997,"5min":0.39200000000000002,"15min":0.34200000000000003},"max":{"1min":15.006,"5min":15.006,"15min":24.669},"last":0.47899999999999998}]}]},{"osd":2,"up_from":104,"seq":446676599070,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":942400,"kb_used_data":196536,"kb_used_omap":11,"kb_used_meta":745844,"kb_avail":92802752,"statfs":{"total":95995035648,"available":95030018048,"internally_reserved":0,"allocated":201252864,"data_stored":198722022,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11878,"internal_metadata":763744666},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:44:05 2023","interfaces":[{"interface":"back","average":{"1min":1.887,"5min":1.3859999999999999,"15min":2.4430000000000001},"min":{"1min":0.20499999999999999,"5min":0.17299999999999999,"15min":0.17299999999999999},"max":{"1min":11.07,"5min":11.07,"15min":82.424999999999997},"last":3.4289999999999998},{"interface":"front","average":{"1min":2.6739999999999999,"5min":2.012,"15min":2.7149999999999999},"min":{"1min":0.27700000000000002,"5min":0.23000000000000001,"15min":0.19},"max":{"1min":16.678999999999998,"5min":16.678999999999998,"15min":82.524000000000001},"last":5.5049999999999999}]},{"osd":1,"last update":"Sat Sep 23 20:44:05 2023","interfaces":[{"interface":"back","average":{"1min":1.4430000000000001,"5min":2.02,"15min":2.839},"min":{"1min":0.248,"5min":0.248,"15min":0.221},"max":{"1min":3.8069999999999999,"5min":12.981,"15min":46.405000000000001},"last":2.2799999999999998},{"interface":"front","average":{"1min":2.0590000000000002,"5min":1.9119999999999999,"15min":2.641},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":11.683999999999999,"5min":11.683999999999999,"15min":46.459000000000003},"last":5.4370000000000003}]},{"osd":3,"last update":"Sat Sep 23 20:44:31 2023","interfaces":[{"interface":"back","average":{"1min":2.1800000000000002,"5min":2.2010000000000001,"15min":2.5990000000000002},"min":{"1min":0.253,"5min":0.253,"15min":0.19800000000000001},"max":{"1min":12.847,"5min":12.847,"15min":19.562999999999999},"last":4.5410000000000004},{"interface":"front","average":{"1min":0.91400000000000003,"5min":1.425,"15min":2.1720000000000002},"min":{"1min":0.26500000000000001,"5min":0.26400000000000001,"15min":0.249},"max":{"1min":6.3700000000000001,"5min":11.375,"15min":30.704999999999998},"last":3.464}]},{"osd":4,"last update":"Sat Sep 23 20:44:05 2023","interfaces":[{"interface":"back","average":{"1min":1.7509999999999999,"5min":1.6359999999999999,"15min":1.911},"min":{"1min":0.34300000000000003,"5min":0.29799999999999999,"15min":0.27500000000000002},"max":{"1min":6.8719999999999999,"5min":12.025,"15min":16.797999999999998},"last":5.5609999999999999},{"interface":"front","average":{"1min":1.702,"5min":1.603,"15min":2.0659999999999998},"min":{"1min":0.38100000000000001,"5min":0.30599999999999999,"15min":0.30499999999999999},"max":{"1min":7.1150000000000002,"5min":9.1379999999999999,"15min":19.440999999999999},"last":2.444}]},{"osd":5,"last update":"Sat Sep 23 20:44:05 2023","interfaces":[{"interface":"back","average":{"1min":2.194,"5min":1.863,"15min":2.452},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":6.3179999999999996,"5min":12.07,"15min":95.393000000000001},"last":5.5350000000000001},{"interface":"front","average":{"1min":1.7929999999999999,"5min":1.6220000000000001,"15min":2.5659999999999998},"min":{"1min":0.377,"5min":0.37,"15min":0.34799999999999998},"max":{"1min":5.1909999999999998,"5min":7.8179999999999996,"15min":95.414000000000001},"last":2.4060000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:44:05 2023","interfaces":[{"interface":"back","average":{"1min":2.258,"5min":1.9059999999999999,"15min":2.246},"min":{"1min":0.39800000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":11.894,"5min":14.047000000000001,"15min":19.486000000000001},"last":2.3700000000000001},{"interface":"front","average":{"1min":2.069,"5min":1.323,"15min":1.7190000000000001},"min":{"1min":0.48099999999999998,"5min":0.34200000000000003,"15min":0.316},"max":{"1min":11.747,"5min":11.747,"15min":30.670999999999999},"last":3.5110000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:44:05 2023","interfaces":[{"interface":"back","average":{"1min":1.4399999999999999,"5min":1.2929999999999999,"15min":1.7270000000000001},"min":{"1min":0.39200000000000002,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":6.1849999999999996,"5min":8.4949999999999992,"15min":30.741},"last":3.488},{"interface":"front","average":{"1min":2.6360000000000001,"5min":2.081,"15min":2.2469999999999999},"min":{"1min":0.44700000000000001,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":6.2910000000000004,"5min":12.099,"15min":16.719000000000001},"last":5.5970000000000004}]}]},{"osd":3,"up_from":109,"seq":468151435543,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1533804,"kb_used_data":582628,"kb_used_omap":4,"kb_used_meta":951163,"kb_avail":92211348,"statfs":{"total":95995035648,"available":94424420352,"internally_reserved":0,"allocated":596611072,"data_stored":594096970,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4514,"internal_metadata":973991518},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:44:39 2023","interfaces":[{"interface":"back","average":{"1min":2.71,"5min":2.4169999999999998,"15min":2.4929999999999999},"min":{"1min":0.222,"5min":0.20899999999999999,"15min":0.191},"max":{"1min":8.5419999999999998,"5min":22.32,"15min":22.32},"last":0.34399999999999997},{"interface":"front","average":{"1min":2.105,"5min":2.6749999999999998,"15min":2.4929999999999999},"min":{"1min":0.249,"5min":0.22900000000000001,"15min":0.19500000000000001},"max":{"1min":6.2160000000000002,"5min":22.805,"15min":22.805},"last":0.45200000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:44:11 2023","interfaces":[{"interface":"back","average":{"1min":2.823,"5min":2.1960000000000002,"15min":2.2290000000000001},"min":{"1min":0.31,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":10.022,"5min":17.529,"15min":17.529},"last":0.39800000000000002},{"interface":"front","average":{"1min":1.742,"5min":2.3700000000000001,"15min":2.4500000000000002},"min":{"1min":0.33300000000000002,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":7.1029999999999998,"5min":14.19,"15min":32.603999999999999},"last":0.32600000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:44:39 2023","interfaces":[{"interface":"back","average":{"1min":2.2970000000000002,"5min":3.5840000000000001,"15min":3.0019999999999998},"min":{"1min":0.28599999999999998,"5min":0.26600000000000001,"15min":0.23799999999999999},"max":{"1min":7.7859999999999996,"5min":68.206999999999994,"15min":68.206999999999994},"last":0.377},{"interface":"front","average":{"1min":2.4470000000000001,"5min":2.7130000000000001,"15min":2.7450000000000001},"min":{"1min":0.251,"5min":0.245,"15min":0.20899999999999999},"max":{"1min":9.1760000000000002,"5min":67.736999999999995,"15min":67.736999999999995},"last":0.35899999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:44:39 2023","interfaces":[{"interface":"back","average":{"1min":2.2879999999999998,"5min":2.008,"15min":1.786},"min":{"1min":0.35799999999999998,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":6.7889999999999997,"5min":22.157,"15min":22.157},"last":0.44},{"interface":"front","average":{"1min":2.113,"5min":2.3290000000000002,"15min":2.3279999999999998},"min":{"1min":0.36499999999999999,"5min":0.33200000000000002,"15min":0.28100000000000003},"max":{"1min":7.9020000000000001,"5min":15.868,"15min":19.635000000000002},"last":0.42799999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:44:39 2023","interfaces":[{"interface":"back","average":{"1min":2.1840000000000002,"5min":2.2970000000000002,"15min":1.992},"min":{"1min":0.33800000000000002,"5min":0.32700000000000001,"15min":0.31900000000000001},"max":{"1min":6.7469999999999999,"5min":22.289999999999999,"15min":22.289999999999999},"last":0.47199999999999998},{"interface":"front","average":{"1min":1.8089999999999999,"5min":1.8440000000000001,"15min":1.841},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.32000000000000001},"max":{"1min":4.6559999999999997,"5min":12.952,"15min":12.952},"last":0.58299999999999996}]},{"osd":6,"last update":"Sat Sep 23 20:44:39 2023","interfaces":[{"interface":"back","average":{"1min":1.845,"5min":2.1360000000000001,"15min":1.956},"min":{"1min":0.36599999999999999,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":6.5350000000000001,"5min":12.917,"15min":17.524999999999999},"last":0.63400000000000001},{"interface":"front","average":{"1min":1.988,"5min":2.3300000000000001,"15min":2.2869999999999999},"min":{"1min":0.38100000000000001,"5min":0.35099999999999998,"15min":0.33700000000000002},"max":{"1min":6.8479999999999999,"5min":15.936,"15min":19.524000000000001},"last":0.60399999999999998}]},{"osd":7,"last update":"Sat Sep 23 20:44:39 2023","interfaces":[{"interface":"back","average":{"1min":2.7850000000000001,"5min":2.7509999999999999,"15min":2.6429999999999998},"min":{"1min":0.46999999999999997,"5min":0.41499999999999998,"15min":0.36899999999999999},"max":{"1min":8.4429999999999996,"5min":15.976000000000001,"15min":19.571999999999999},"last":0.56899999999999995},{"interface":"front","average":{"1min":2.4750000000000001,"5min":2.262,"15min":1.9850000000000001},"min":{"1min":0.46300000000000002,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":8.5289999999999999,"5min":22.202999999999999,"15min":22.202999999999999},"last":0.49399999999999999}]}]},{"osd":4,"up_from":30,"seq":128849019297,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":748896,"kb_used_data":317400,"kb_used_omap":0,"kb_used_meta":431488,"kb_avail":92996256,"statfs":{"total":95995035648,"available":95228166144,"internally_reserved":0,"allocated":325017600,"data_stored":323505358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":441843712},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:43:57 2023","interfaces":[{"interface":"back","average":{"1min":2.5609999999999999,"5min":1.8129999999999999,"15min":1.746},"min":{"1min":0.251,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":11.709,"5min":11.709,"15min":24.114999999999998},"last":0.39300000000000002},{"interface":"front","average":{"1min":2.8330000000000002,"5min":2.1440000000000001,"15min":1.9710000000000001},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.246},"max":{"1min":13.74,"5min":13.74,"15min":24.135999999999999},"last":0.443}]},{"osd":1,"last update":"Sat Sep 23 20:44:11 2023","interfaces":[{"interface":"back","average":{"1min":2.306,"5min":1.8979999999999999,"15min":1.7749999999999999},"min":{"1min":0.26300000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":8.7460000000000004,"5min":10.363,"15min":18.190999999999999},"last":0.374},{"interface":"front","average":{"1min":1.355,"5min":1.5569999999999999,"15min":1.613},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":4.8659999999999997,"5min":10.426,"15min":18.173999999999999},"last":0.49199999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:44:11 2023","interfaces":[{"interface":"back","average":{"1min":2.6040000000000001,"5min":1.756,"15min":1.885},"min":{"1min":0.29499999999999998,"5min":0.28399999999999997,"15min":0.26600000000000001},"max":{"1min":9.3320000000000007,"5min":13.051,"15min":32.363},"last":0.54200000000000004},{"interface":"front","average":{"1min":2.0150000000000001,"5min":2.0539999999999998,"15min":2.2090000000000001},"min":{"1min":0.41199999999999998,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":7.6159999999999997,"5min":11.917999999999999,"15min":34.042999999999999},"last":0.57999999999999996}]},{"osd":3,"last update":"Sat Sep 23 20:44:27 2023","interfaces":[{"interface":"back","average":{"1min":1.877,"5min":2.04,"15min":1.9950000000000001},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":7.7480000000000002,"5min":12.147,"15min":14.515000000000001},"last":0.56699999999999995},{"interface":"front","average":{"1min":2.4260000000000002,"5min":2.0990000000000002,"15min":2.0699999999999998},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.30299999999999999},"max":{"1min":10.493,"5min":13.35,"15min":15.991},"last":0.58899999999999997}]},{"osd":5,"last update":"Sat Sep 23 20:43:57 2023","interfaces":[{"interface":"back","average":{"1min":2.1789999999999998,"5min":1.3540000000000001,"15min":1.3440000000000001},"min":{"1min":0.34399999999999997,"5min":0.28000000000000003,"15min":0.25800000000000001},"max":{"1min":7.5810000000000004,"5min":9.7260000000000009,"15min":82.866},"last":0.51400000000000001},{"interface":"front","average":{"1min":1.363,"5min":1.2729999999999999,"15min":1.3180000000000001},"min":{"1min":0.311,"5min":0.311,"15min":0.28999999999999998},"max":{"1min":6.601,"5min":7.0129999999999999,"15min":82.546000000000006},"last":0.52900000000000003}]},{"osd":6,"last update":"Sat Sep 23 20:44:11 2023","interfaces":[{"interface":"back","average":{"1min":1.0920000000000001,"5min":1.5449999999999999,"15min":1.214},"min":{"1min":0.34300000000000003,"5min":0.29499999999999998,"15min":0.27600000000000002},"max":{"1min":3.96,"5min":14.731999999999999,"15min":14.731999999999999},"last":0.55800000000000005},{"interface":"front","average":{"1min":1.8959999999999999,"5min":1.2989999999999999,"15min":1.1180000000000001},"min":{"1min":0.34300000000000003,"5min":0.314,"15min":0.29899999999999999},"max":{"1min":6.3440000000000003,"5min":7.6150000000000002,"15min":11.199},"last":0.55000000000000004}]},{"osd":7,"last update":"Sat Sep 23 20:44:27 2023","interfaces":[{"interface":"back","average":{"1min":2.0630000000000002,"5min":2.0979999999999999,"15min":1.5649999999999999},"min":{"1min":0.374,"5min":0.35099999999999998,"15min":0.318},"max":{"1min":7.7629999999999999,"5min":22.184999999999999,"15min":22.184999999999999},"last":313.50099999999998},{"interface":"front","average":{"1min":1.1240000000000001,"5min":1.724,"15min":1.2150000000000001},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.307},"max":{"1min":6.3689999999999998,"5min":21.754000000000001,"15min":21.754000000000001},"last":312.67500000000001}]}]},{"osd":5,"up_from":39,"seq":167503724958,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1132764,"kb_used_data":626708,"kb_used_omap":0,"kb_used_meta":506048,"kb_avail":92612388,"statfs":{"total":95995035648,"available":94835085312,"internally_reserved":0,"allocated":641748992,"data_stored":640237476,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":518193152},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:44:00 2023","interfaces":[{"interface":"back","average":{"1min":2.145,"5min":1.7050000000000001,"15min":1.7989999999999999},"min":{"1min":0.23999999999999999,"5min":0.22500000000000001,"15min":0.20699999999999999},"max":{"1min":15.018000000000001,"5min":15.018000000000001,"15min":18.16},"last":0.69999999999999996},{"interface":"front","average":{"1min":2.7130000000000001,"5min":1.6870000000000001,"15min":1.7410000000000001},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":15.034000000000001,"5min":15.034000000000001,"15min":15.034000000000001},"last":0.432}]},{"osd":1,"last update":"Sat Sep 23 20:44:14 2023","interfaces":[{"interface":"back","average":{"1min":2.7349999999999999,"5min":3.0299999999999998,"15min":2.2690000000000001},"min":{"1min":0.379,"5min":0.26500000000000001,"15min":0.253},"max":{"1min":9.4199999999999999,"5min":61.005000000000003,"15min":61.005000000000003},"last":0.40699999999999997},{"interface":"front","average":{"1min":1.339,"5min":2.4129999999999998,"15min":2.0019999999999998},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.23499999999999999},"max":{"1min":6.7350000000000003,"5min":59.688000000000002,"15min":59.688000000000002},"last":0.71899999999999997}]},{"osd":2,"last update":"Sat Sep 23 20:44:00 2023","interfaces":[{"interface":"back","average":{"1min":2.3330000000000002,"5min":2.032,"15min":2.0950000000000002},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.26500000000000001},"max":{"1min":8.3719999999999999,"5min":9.2119999999999997,"15min":37.011000000000003},"last":0.77300000000000002},{"interface":"front","average":{"1min":2.7240000000000002,"5min":1.9359999999999999,"15min":2.0659999999999998},"min":{"1min":0.314,"5min":0.28299999999999997,"15min":0.255},"max":{"1min":9.8260000000000005,"5min":10.565,"15min":36.835000000000001},"last":0.39200000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:44:20 2023","interfaces":[{"interface":"back","average":{"1min":1.7769999999999999,"5min":2.375,"15min":2.375},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.29599999999999999},"max":{"1min":5.6619999999999999,"5min":11.032,"15min":36.716000000000001},"last":0.51700000000000002},{"interface":"front","average":{"1min":1.5900000000000001,"5min":2.238,"15min":2.2530000000000001},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":7.4100000000000001,"5min":13.266,"15min":18.504000000000001},"last":0.75600000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:44:54 2023","interfaces":[{"interface":"back","average":{"1min":1.1850000000000001,"5min":1.857,"15min":1.2729999999999999},"min":{"1min":0.30199999999999999,"5min":0.27300000000000002,"15min":0.255},"max":{"1min":5.1050000000000004,"5min":13.391999999999999,"15min":28.015999999999998},"last":0.44900000000000001},{"interface":"front","average":{"1min":1.1499999999999999,"5min":1.6319999999999999,"15min":1.1200000000000001},"min":{"1min":0.32000000000000001,"5min":0.30299999999999999,"15min":0.28299999999999997},"max":{"1min":6.8399999999999999,"5min":7.3799999999999999,"15min":12.300000000000001},"last":0.67200000000000004}]},{"osd":6,"last update":"Sat Sep 23 20:44:07 2023","interfaces":[{"interface":"back","average":{"1min":3.3999999999999999,"5min":2.04,"15min":1.3320000000000001},"min":{"1min":0.39100000000000001,"5min":0.317,"15min":0.30199999999999999},"max":{"1min":13.734,"5min":13.734,"15min":13.734},"last":0.47899999999999998},{"interface":"front","average":{"1min":1.375,"5min":1.577,"15min":1.1950000000000001},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":5.274,"5min":8.7449999999999992,"15min":16.265999999999998},"last":0.73799999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:44:20 2023","interfaces":[{"interface":"back","average":{"1min":1.4610000000000001,"5min":1.7330000000000001,"15min":1.1339999999999999},"min":{"1min":0.372,"5min":0.34499999999999997,"15min":0.28499999999999998},"max":{"1min":4.3250000000000002,"5min":16.366,"15min":16.366},"last":294.70999999999998},{"interface":"front","average":{"1min":1.4950000000000001,"5min":1.6439999999999999,"15min":1.173},"min":{"1min":0.38900000000000001,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":7.524,"5min":7.915,"15min":13.122},"last":294.74400000000003}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4722,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1360,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8206,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7156,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3154,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":276758528,"data_stored":276758528,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":769593344,"data_stored":769589272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":188940288,"data_stored":188940288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":589627392,"data_stored":589627392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":311758848,"data_stored":311754776,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":632360960,"data_stored":632356888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":190644224,"data_stored":190644224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":652218368,"data_stored":652214296,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:45:02.069 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:45:02.070 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:01 smithi149 ceph-mon[170394]: pgmap v1303: 119 pgs: 119 active+clean; 970 MiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 736 op/s 2023-09-23T20:45:02.071 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:01 smithi149 ceph-mon[170394]: failed to encode map e521 with expected crc 2023-09-23T20:45:02.071 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:01 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 5]}]': finished 2023-09-23T20:45:02.071 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:01 smithi149 ceph-mon[170394]: osdmap e521: 8 total, 8 up, 8 in 2023-09-23T20:45:02.072 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:45:02.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:01 smithi018 ceph-mon[179647]: pgmap v1303: 119 pgs: 119 active+clean; 970 MiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 736 op/s 2023-09-23T20:45:02.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:01 smithi018 ceph-mon[179647]: failed to encode map e521 with expected crc 2023-09-23T20:45:02.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:01 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 5]}]': finished 2023-09-23T20:45:02.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:01 smithi018 ceph-mon[179647]: osdmap e521: 8 total, 8 up, 8 in 2023-09-23T20:45:02.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:01 smithi018 ceph-mon[184144]: pgmap v1303: 119 pgs: 119 active+clean; 970 MiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 736 op/s 2023-09-23T20:45:02.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:01 smithi018 ceph-mon[184144]: failed to encode map e521 with expected crc 2023-09-23T20:45:02.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:01 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [1, 5]}]': finished 2023-09-23T20:45:02.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:01 smithi018 ceph-mon[184144]: osdmap e521: 8 total, 8 up, 8 in 2023-09-23T20:45:02.155 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:02.344 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T20:45:02.861 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:02 smithi018 ceph-mon[179647]: osdmap e522: 8 total, 8 up, 8 in 2023-09-23T20:45:02.861 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:02 smithi018 ceph-mon[179647]: failed to encode map e522 with expected crc 2023-09-23T20:45:02.861 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:02 smithi018 ceph-mon[179647]: failed to encode map e522 with expected crc 2023-09-23T20:45:02.861 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:02 smithi018 ceph-mon[179647]: failed to encode map e522 with expected crc 2023-09-23T20:45:02.861 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:02 smithi018 ceph-mon[179647]: failed to encode map e522 with expected crc 2023-09-23T20:45:02.929 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:45:02.935 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:02.936 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:45:03.037 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:45:03.114 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:02 smithi149 ceph-mon[170394]: osdmap e522: 8 total, 8 up, 8 in 2023-09-23T20:45:03.114 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:02 smithi149 ceph-mon[170394]: failed to encode map e522 with expected crc 2023-09-23T20:45:03.114 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:02 smithi149 ceph-mon[170394]: failed to encode map e522 with expected crc 2023-09-23T20:45:03.114 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:02 smithi149 ceph-mon[170394]: failed to encode map e522 with expected crc 2023-09-23T20:45:03.114 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:02 smithi149 ceph-mon[170394]: failed to encode map e522 with expected crc 2023-09-23T20:45:03.120 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:03.137 INFO:teuthology.orchestra.run.smithi149.stdout:{ 2023-09-23T20:45:03.137 INFO:teuthology.orchestra.run.smithi149.stdout: "ops": [], 2023-09-23T20:45:03.137 INFO:teuthology.orchestra.run.smithi149.stdout: "complaint_time": 30, 2023-09-23T20:45:03.137 INFO:teuthology.orchestra.run.smithi149.stdout: "num_blocked_ops": 0 2023-09-23T20:45:03.137 INFO:teuthology.orchestra.run.smithi149.stdout:} 2023-09-23T20:45:03.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:02 smithi018 ceph-mon[184144]: osdmap e522: 8 total, 8 up, 8 in 2023-09-23T20:45:03.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:02 smithi018 ceph-mon[184144]: failed to encode map e522 with expected crc 2023-09-23T20:45:03.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:02 smithi018 ceph-mon[184144]: failed to encode map e522 with expected crc 2023-09-23T20:45:03.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:02 smithi018 ceph-mon[184144]: failed to encode map e522 with expected crc 2023-09-23T20:45:03.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:02 smithi018 ceph-mon[184144]: failed to encode map e522 with expected crc 2023-09-23T20:45:03.189 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:45:03.201 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:03.202 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:45:03.302 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:45:03.360 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:03.634 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:45:03.643 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:03.643 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:45:03.722 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T20:45:03.744 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:45:03.840 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:03.863 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:03 smithi018 ceph-mon[179647]: pgmap v1306: 119 pgs: 119 active+clean; 923 MiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 724 op/s 2023-09-23T20:45:03.909 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:45:03.917 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:03 smithi149 ceph-mon[170394]: pgmap v1306: 119 pgs: 119 active+clean; 923 MiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 724 op/s 2023-09-23T20:45:03.917 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:03.918 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:45:04.018 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:45:04.092 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:04.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:03 smithi018 ceph-mon[184144]: pgmap v1306: 119 pgs: 119 active+clean; 923 MiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 724 op/s 2023-09-23T20:45:04.179 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:45:04.186 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:04.187 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:45:04.287 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:45:04.289 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:45:04.397 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:04.488 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:45:04.496 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:04.496 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:45:04.524 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:45:04.597 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:45:04.676 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:05.069 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:45:05.338 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:45:04 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:45:04] "GET /metrics HTTP/1.1" 200 36510 "" "Prometheus/2.43.0" 2023-09-23T20:45:05.549 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:45:05.565 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:05.566 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:45:05.666 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:45:05.744 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:05.882 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:45:05.922 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:45:05.961 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:45:05.982 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:45:05.988 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:05 smithi018 ceph-mon[184144]: pgmap v1307: 119 pgs: 119 active+clean; 751 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 879 op/s 2023-09-23T20:45:05.988 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:05 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:45:05.988 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:05 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/617387331' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:45:05.989 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:05 smithi018 ceph-mon[179647]: pgmap v1307: 119 pgs: 119 active+clean; 751 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 879 op/s 2023-09-23T20:45:05.989 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:05 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:45:05.989 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:05 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/617387331' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:45:05.989 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:05.990 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:45:06.004 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:45:06.060 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:45:06.091 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:45:06.106 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:45:06.134 INFO:teuthology.orchestra.run.smithi149.stdout:{ 2023-09-23T20:45:06.134 INFO:teuthology.orchestra.run.smithi149.stdout: "size": 20, 2023-09-23T20:45:06.134 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 600, 2023-09-23T20:45:06.134 INFO:teuthology.orchestra.run.smithi149.stdout: "ops": [ 2023-09-23T20:45:06.134 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.135 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:84102 8.d 8:b20f6310:::benchmark_data_smithi018_341490_object84101:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e379)", 2023-09-23T20:45:06.135 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:37:13.381094+0000", 2023-09-23T20:45:06.135 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 472.74873567399999, 2023-09-23T20:45:06.135 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.221042502, 2023-09-23T20:45:06.135 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:45:06.135 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:45:06.135 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:45:06.135 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:45:06.135 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:45:06.135 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 84102 2023-09-23T20:45:06.135 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.136 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:45:06.136 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.136 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:45:06.136 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.381094+0000", 2023-09-23T20:45:06.136 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:45:06.136 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.136 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.136 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:45:06.136 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.381094+0000", 2023-09-23T20:45:06.136 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:45:06.136 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.136 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.137 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:45:06.137 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.381093+0000", 2023-09-23T20:45:06.137 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:45:06.137 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.137 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.137 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:45:06.137 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.381160+0000", 2023-09-23T20:45:06.137 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.7119999999999994e-05 2023-09-23T20:45:06.137 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.137 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.137 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:45:06.137 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.381175+0000", 2023-09-23T20:45:06.137 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.5091e-05 2023-09-23T20:45:06.138 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.138 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.138 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:45:06.138 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.381181+0000", 2023-09-23T20:45:06.138 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.1160000000000004e-06 2023-09-23T20:45:06.138 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.138 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.138 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:45:06.138 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.381234+0000", 2023-09-23T20:45:06.138 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.2218e-05 2023-09-23T20:45:06.138 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.139 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.139 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:45:06.139 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.381242+0000", 2023-09-23T20:45:06.139 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.3149999999999992e-06 2023-09-23T20:45:06.139 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.139 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.139 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:45:06.139 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:14.352689+0000", 2023-09-23T20:45:06.139 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.97144663399999998 2023-09-23T20:45:06.139 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.139 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.139 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:45:06.140 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:14.352699+0000", 2023-09-23T20:45:06.140 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0249e-05 2023-09-23T20:45:06.140 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.140 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.140 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:45:06.140 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:15.563152+0000", 2023-09-23T20:45:06.140 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.210452927 2023-09-23T20:45:06.140 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.140 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.140 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:45:06.140 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:15.564607+0000", 2023-09-23T20:45:06.140 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00145528 2023-09-23T20:45:06.140 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.141 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.141 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 2,6", 2023-09-23T20:45:06.141 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:15.564732+0000", 2023-09-23T20:45:06.141 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000125208 2023-09-23T20:45:06.141 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.141 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.141 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:45:06.141 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:15.572216+0000", 2023-09-23T20:45:06.141 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.007483712 2023-09-23T20:45:06.141 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.141 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.141 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:45:06.142 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:15.572291+0000", 2023-09-23T20:45:06.142 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.4611999999999996e-05 2023-09-23T20:45:06.142 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.142 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.142 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:45:06.142 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:15.600899+0000", 2023-09-23T20:45:06.142 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.028608602 2023-09-23T20:45:06.142 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.142 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.142 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:45:06.142 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:15.600945+0000", 2023-09-23T20:45:06.142 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.6074e-05 2023-09-23T20:45:06.143 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.143 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.143 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:45:06.143 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:15.602136+0000", 2023-09-23T20:45:06.143 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.001191147 2023-09-23T20:45:06.143 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:45:06.143 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:45:06.143 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:45:06.143 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.143 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.143 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:84105 8.d 8:b309f62d:::benchmark_data_smithi018_341490_object84104:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e379)", 2023-09-23T20:45:06.143 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:37:13.384749+0000", 2023-09-23T20:45:06.144 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 472.74508105400002, 2023-09-23T20:45:06.144 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.2195194549999999, 2023-09-23T20:45:06.144 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:45:06.144 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:45:06.144 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:45:06.144 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:45:06.144 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:45:06.144 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 84105 2023-09-23T20:45:06.144 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.144 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:45:06.144 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.144 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:45:06.145 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.384749+0000", 2023-09-23T20:45:06.145 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:45:06.145 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.145 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.145 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:45:06.145 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.384749+0000", 2023-09-23T20:45:06.145 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:45:06.145 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.145 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.145 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:45:06.145 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.384748+0000", 2023-09-23T20:45:06.146 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:45:06.146 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.146 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.146 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:45:06.146 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.384787+0000", 2023-09-23T20:45:06.146 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.9078000000000001e-05 2023-09-23T20:45:06.146 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.146 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.146 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:45:06.146 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.384803+0000", 2023-09-23T20:45:06.146 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.6277e-05 2023-09-23T20:45:06.146 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.147 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.147 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:45:06.147 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.384810+0000", 2023-09-23T20:45:06.147 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.9009999999999997e-06 2023-09-23T20:45:06.147 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.147 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.147 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:45:06.147 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.384862+0000", 2023-09-23T20:45:06.147 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.1962000000000002e-05 2023-09-23T20:45:06.147 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.147 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.147 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:45:06.148 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.384872+0000", 2023-09-23T20:45:06.148 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.781e-06 2023-09-23T20:45:06.148 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.148 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.148 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:45:06.148 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:14.352738+0000", 2023-09-23T20:45:06.148 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.96786639799999996 2023-09-23T20:45:06.148 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.148 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.148 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:45:06.148 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:14.352747+0000", 2023-09-23T20:45:06.148 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.9649999999999997e-06 2023-09-23T20:45:06.149 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.149 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.149 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:45:06.149 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:15.566789+0000", 2023-09-23T20:45:06.149 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.2140422710000001 2023-09-23T20:45:06.149 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.149 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.149 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:45:06.149 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:15.566946+0000", 2023-09-23T20:45:06.149 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00015720799999999999 2023-09-23T20:45:06.149 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.149 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.150 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 2,6", 2023-09-23T20:45:06.150 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:15.567141+0000", 2023-09-23T20:45:06.150 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000194215 2023-09-23T20:45:06.150 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.150 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.150 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:45:06.150 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:15.572233+0000", 2023-09-23T20:45:06.150 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.005092343 2023-09-23T20:45:06.150 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.150 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.150 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:45:06.151 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:15.577236+0000", 2023-09-23T20:45:06.151 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0050031850000000003 2023-09-23T20:45:06.151 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.151 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.151 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:45:06.151 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:15.602198+0000", 2023-09-23T20:45:06.151 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.024961555 2023-09-23T20:45:06.151 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.151 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.151 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:45:06.151 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:15.602245+0000", 2023-09-23T20:45:06.151 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.7126999999999998e-05 2023-09-23T20:45:06.152 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.152 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.152 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:45:06.152 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:15.604268+0000", 2023-09-23T20:45:06.152 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0020230959999999998 2023-09-23T20:45:06.152 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:45:06.152 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:45:06.152 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:45:06.152 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.152 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.152 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:76927 9.9 9:9b548e53:::benchmark_data_smithi018_381516_object76926:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e465)", 2023-09-23T20:45:06.152 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:41:58.464989+0000", 2023-09-23T20:45:06.153 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 187.664840295, 2023-09-23T20:45:06.153 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.029900163, 2023-09-23T20:45:06.153 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:45:06.153 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:45:06.153 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:45:06.153 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:45:06.153 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:45:06.153 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 76927 2023-09-23T20:45:06.153 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.153 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:45:06.153 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.154 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:45:06.154 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.464989+0000", 2023-09-23T20:45:06.154 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:45:06.154 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.154 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.154 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:45:06.154 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.464989+0000", 2023-09-23T20:45:06.154 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:45:06.154 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.154 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.154 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:45:06.154 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.464988+0000", 2023-09-23T20:45:06.155 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:45:06.155 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.155 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.155 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:45:06.155 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.465033+0000", 2023-09-23T20:45:06.155 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.5207999999999998e-05 2023-09-23T20:45:06.155 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.155 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.155 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:45:06.155 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.465181+0000", 2023-09-23T20:45:06.155 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000147949 2023-09-23T20:45:06.156 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.156 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.156 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:45:06.156 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.465192+0000", 2023-09-23T20:45:06.156 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0370999999999999e-05 2023-09-23T20:45:06.156 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.156 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.156 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:45:06.156 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.471908+0000", 2023-09-23T20:45:06.156 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0067160800000000001 2023-09-23T20:45:06.156 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.156 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.157 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:45:06.157 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.471918+0000", 2023-09-23T20:45:06.157 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0189e-05 2023-09-23T20:45:06.157 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.157 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.157 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:45:06.157 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.482524+0000", 2023-09-23T20:45:06.157 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.010605714 2023-09-23T20:45:06.157 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.157 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.157 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for degraded object", 2023-09-23T20:45:06.157 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.483138+0000", 2023-09-23T20:45:06.158 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00061383100000000001 2023-09-23T20:45:06.158 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.158 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.158 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:45:06.158 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.494835+0000", 2023-09-23T20:45:06.158 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.01169716 2023-09-23T20:45:06.158 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.158 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.158 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:45:06.158 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.494889+0000", 2023-09-23T20:45:06.158 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.4761e-05 2023-09-23T20:45:06.159 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:45:06.159 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:45:06.159 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:45:06.159 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.159 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.159 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:76942 9.9 9:944a28a3:::benchmark_data_smithi018_381516_object76941:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e465)", 2023-09-23T20:45:06.159 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:41:58.472086+0000", 2023-09-23T20:45:06.159 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 187.65774374599999, 2023-09-23T20:45:06.159 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0269352270000001, 2023-09-23T20:45:06.159 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:45:06.159 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:45:06.159 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:45:06.160 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:45:06.160 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:45:06.160 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 76942 2023-09-23T20:45:06.160 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.160 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:45:06.160 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.160 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:45:06.160 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.472086+0000", 2023-09-23T20:45:06.160 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:45:06.160 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.160 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.160 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:45:06.161 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.472086+0000", 2023-09-23T20:45:06.161 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:45:06.161 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.161 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.161 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:45:06.161 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.472085+0000", 2023-09-23T20:45:06.161 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:45:06.161 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.161 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.161 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:45:06.161 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.472164+0000", 2023-09-23T20:45:06.161 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.9431000000000006e-05 2023-09-23T20:45:06.161 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.161 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.162 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:45:06.162 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.472175+0000", 2023-09-23T20:45:06.162 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0321e-05 2023-09-23T20:45:06.162 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.162 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.162 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:45:06.162 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.472181+0000", 2023-09-23T20:45:06.162 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.3629999999999999e-06 2023-09-23T20:45:06.162 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.162 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.162 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:45:06.162 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.472225+0000", 2023-09-23T20:45:06.163 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.3844000000000002e-05 2023-09-23T20:45:06.163 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.163 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.163 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:45:06.163 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.472234+0000", 2023-09-23T20:45:06.163 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.0100000000000001e-06 2023-09-23T20:45:06.163 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.163 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.163 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:45:06.163 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.483396+0000", 2023-09-23T20:45:06.163 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0111617260000001 2023-09-23T20:45:06.163 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.163 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.164 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:45:06.164 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.485820+0000", 2023-09-23T20:45:06.164 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0024248550000000001 2023-09-23T20:45:06.164 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.164 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.164 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:45:06.164 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.486061+0000", 2023-09-23T20:45:06.164 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00024016599999999999 2023-09-23T20:45:06.164 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.164 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.164 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:45:06.164 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.494551+0000", 2023-09-23T20:45:06.165 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0084907460000000004 2023-09-23T20:45:06.165 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.165 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.165 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:45:06.165 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.494921+0000", 2023-09-23T20:45:06.165 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000369825 2023-09-23T20:45:06.165 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.165 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.165 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:45:06.165 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.498893+0000", 2023-09-23T20:45:06.165 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.003971937 2023-09-23T20:45:06.165 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.165 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.165 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:45:06.166 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.498942+0000", 2023-09-23T20:45:06.166 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.8832999999999998e-05 2023-09-23T20:45:06.166 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.166 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.166 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:45:06.166 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.499021+0000", 2023-09-23T20:45:06.166 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.9107e-05 2023-09-23T20:45:06.166 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:45:06.166 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:45:06.166 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:45:06.166 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.166 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.167 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:76989 9.9 9:9a35c134:::benchmark_data_smithi018_381516_object76988:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e465)", 2023-09-23T20:45:06.167 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:41:58.523243+0000", 2023-09-23T20:45:06.167 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 187.60658643599999, 2023-09-23T20:45:06.167 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.97667275399999998, 2023-09-23T20:45:06.167 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:45:06.167 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:45:06.167 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:45:06.167 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:45:06.167 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:45:06.167 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 76989 2023-09-23T20:45:06.167 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.167 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:45:06.168 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.168 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:45:06.168 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.523243+0000", 2023-09-23T20:45:06.168 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:45:06.168 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.168 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.168 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:45:06.168 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.523243+0000", 2023-09-23T20:45:06.168 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:45:06.168 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.168 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.168 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:45:06.168 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.523242+0000", 2023-09-23T20:45:06.169 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:45:06.169 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.169 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.169 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:45:06.169 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.523361+0000", 2023-09-23T20:45:06.169 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00011913000000000001 2023-09-23T20:45:06.169 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.169 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.169 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:45:06.169 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.523375+0000", 2023-09-23T20:45:06.169 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.4385e-05 2023-09-23T20:45:06.170 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.170 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.170 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:45:06.170 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.523381+0000", 2023-09-23T20:45:06.170 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.8340000000000004e-06 2023-09-23T20:45:06.170 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.170 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.170 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:45:06.170 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.523439+0000", 2023-09-23T20:45:06.170 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.7637999999999999e-05 2023-09-23T20:45:06.170 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.171 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.171 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:45:06.171 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.523448+0000", 2023-09-23T20:45:06.171 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.6350000000000001e-06 2023-09-23T20:45:06.171 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.171 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.171 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:45:06.171 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.489120+0000", 2023-09-23T20:45:06.171 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.96567205899999997 2023-09-23T20:45:06.171 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.171 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.171 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:45:06.171 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.489252+0000", 2023-09-23T20:45:06.172 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00013131300000000001 2023-09-23T20:45:06.172 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.172 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.172 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:45:06.172 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.489398+0000", 2023-09-23T20:45:06.172 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00014658600000000001 2023-09-23T20:45:06.172 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.172 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.172 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:45:06.172 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.494579+0000", 2023-09-23T20:45:06.172 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0051810770000000001 2023-09-23T20:45:06.172 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.173 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.173 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:45:06.173 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.499034+0000", 2023-09-23T20:45:06.173 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0044545080000000003 2023-09-23T20:45:06.173 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.173 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.173 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:45:06.173 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.499764+0000", 2023-09-23T20:45:06.173 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00073013099999999997 2023-09-23T20:45:06.173 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.173 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.173 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:45:06.173 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.499812+0000", 2023-09-23T20:45:06.174 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.7814e-05 2023-09-23T20:45:06.174 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.174 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.174 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:45:06.174 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.499916+0000", 2023-09-23T20:45:06.174 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000104232 2023-09-23T20:45:06.174 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:45:06.174 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:45:06.174 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:45:06.174 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.174 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.174 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:76991 9.9 9:92539b4f:::benchmark_data_smithi018_381516_object76990:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e465)", 2023-09-23T20:45:06.174 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:41:58.527930+0000", 2023-09-23T20:45:06.175 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 187.60189975500001, 2023-09-23T20:45:06.175 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.97865332000000005, 2023-09-23T20:45:06.175 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:45:06.175 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:45:06.175 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:45:06.175 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:45:06.175 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:45:06.175 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 76991 2023-09-23T20:45:06.175 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.175 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:45:06.175 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.176 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:45:06.176 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.527930+0000", 2023-09-23T20:45:06.176 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:45:06.176 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.176 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.176 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:45:06.176 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.527930+0000", 2023-09-23T20:45:06.176 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:45:06.176 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.176 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.176 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:45:06.176 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.527929+0000", 2023-09-23T20:45:06.177 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:45:06.177 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.177 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.177 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:45:06.177 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.528008+0000", 2023-09-23T20:45:06.177 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.9217999999999999e-05 2023-09-23T20:45:06.177 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.177 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.177 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:45:06.177 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.528020+0000", 2023-09-23T20:45:06.177 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1829e-05 2023-09-23T20:45:06.177 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.178 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.178 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:45:06.178 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.528026+0000", 2023-09-23T20:45:06.178 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.0769999999999996e-06 2023-09-23T20:45:06.178 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.178 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.178 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:45:06.178 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.528067+0000", 2023-09-23T20:45:06.178 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.0475000000000001e-05 2023-09-23T20:45:06.178 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.178 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.178 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:45:06.179 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.528073+0000", 2023-09-23T20:45:06.179 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.3790000000000003e-06 2023-09-23T20:45:06.179 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.179 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.179 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:45:06.179 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.491819+0000", 2023-09-23T20:45:06.179 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.96374567099999997 2023-09-23T20:45:06.179 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.179 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.179 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:45:06.179 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.491918+0000", 2023-09-23T20:45:06.179 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.9658e-05 2023-09-23T20:45:06.179 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.180 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.180 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:45:06.180 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.492021+0000", 2023-09-23T20:45:06.180 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00010265200000000001 2023-09-23T20:45:06.180 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.180 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.180 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:45:06.180 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.494942+0000", 2023-09-23T20:45:06.180 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0029211799999999998 2023-09-23T20:45:06.180 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.180 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.180 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:45:06.181 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.499042+0000", 2023-09-23T20:45:06.181 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0040998809999999997 2023-09-23T20:45:06.181 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.181 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.181 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:45:06.181 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.506416+0000", 2023-09-23T20:45:06.181 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0073741270000000003 2023-09-23T20:45:06.181 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.181 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.181 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:45:06.181 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.506477+0000", 2023-09-23T20:45:06.181 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.1086000000000004e-05 2023-09-23T20:45:06.182 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.182 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.182 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:45:06.182 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.506583+0000", 2023-09-23T20:45:06.182 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00010592200000000001 2023-09-23T20:45:06.182 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:45:06.182 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:45:06.182 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:45:06.182 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.182 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.182 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:77005 9.9 9:92384e24:::benchmark_data_smithi018_381516_object77004:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e465)", 2023-09-23T20:45:06.182 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:41:58.545679+0000", 2023-09-23T20:45:06.183 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 187.584150687, 2023-09-23T20:45:06.183 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.96107413500000005, 2023-09-23T20:45:06.183 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:45:06.183 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:45:06.183 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:45:06.183 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:45:06.183 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:45:06.183 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 77005 2023-09-23T20:45:06.183 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.183 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:45:06.183 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.183 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:45:06.183 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.545679+0000", 2023-09-23T20:45:06.184 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:45:06.184 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.184 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.184 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:45:06.184 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.545679+0000", 2023-09-23T20:45:06.184 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:45:06.184 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.184 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.184 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:45:06.184 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.545678+0000", 2023-09-23T20:45:06.184 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:45:06.184 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.185 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.185 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:45:06.185 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.545781+0000", 2023-09-23T20:45:06.185 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000102415 2023-09-23T20:45:06.185 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.185 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.185 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:45:06.185 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.545796+0000", 2023-09-23T20:45:06.185 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.5554999999999999e-05 2023-09-23T20:45:06.185 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.185 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.186 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:45:06.186 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.545800+0000", 2023-09-23T20:45:06.186 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.2089999999999999e-06 2023-09-23T20:45:06.186 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.186 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.186 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:45:06.186 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.545850+0000", 2023-09-23T20:45:06.186 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.9740000000000001e-05 2023-09-23T20:45:06.186 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.186 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.186 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:45:06.187 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.545860+0000", 2023-09-23T20:45:06.187 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.611e-06 2023-09-23T20:45:06.187 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.187 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.187 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:45:06.187 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.494954+0000", 2023-09-23T20:45:06.187 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.94909395200000002 2023-09-23T20:45:06.187 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.187 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.187 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:45:06.187 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.495058+0000", 2023-09-23T20:45:06.187 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000103807 2023-09-23T20:45:06.188 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.188 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.188 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:45:06.188 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.495164+0000", 2023-09-23T20:45:06.188 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000106128 2023-09-23T20:45:06.188 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.188 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.188 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:45:06.188 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.498333+0000", 2023-09-23T20:45:06.188 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0031692030000000002 2023-09-23T20:45:06.188 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.188 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.188 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:45:06.189 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.499048+0000", 2023-09-23T20:45:06.189 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00071484300000000005 2023-09-23T20:45:06.189 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.189 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.189 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:45:06.189 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.506626+0000", 2023-09-23T20:45:06.189 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0075784850000000003 2023-09-23T20:45:06.189 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.189 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.189 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:45:06.189 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.506672+0000", 2023-09-23T20:45:06.189 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.5529999999999999e-05 2023-09-23T20:45:06.190 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.190 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.190 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:45:06.190 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.506753+0000", 2023-09-23T20:45:06.190 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.1328000000000005e-05 2023-09-23T20:45:06.190 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:45:06.190 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:45:06.190 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:45:06.190 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.190 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.190 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:141034 9.7 9:eb6fa991:::benchmark_data_smithi018_381516_object27200:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e481)", 2023-09-23T20:45:06.191 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:43:15.740452+0000", 2023-09-23T20:45:06.191 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 110.389377731, 2023-09-23T20:45:06.191 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0443972239999999, 2023-09-23T20:45:06.191 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:45:06.191 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:45:06.191 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:45:06.191 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:45:06.191 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:45:06.191 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 141034 2023-09-23T20:45:06.191 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.191 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:45:06.191 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.192 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:45:06.192 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.740452+0000", 2023-09-23T20:45:06.192 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:45:06.192 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.192 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.192 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:45:06.192 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.740452+0000", 2023-09-23T20:45:06.192 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:45:06.192 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.192 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.192 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:45:06.192 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.740451+0000", 2023-09-23T20:45:06.193 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:45:06.193 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.193 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.193 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:45:06.193 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.740454+0000", 2023-09-23T20:45:06.193 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.0079999999999998e-06 2023-09-23T20:45:06.193 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.193 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.193 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:45:06.193 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.740461+0000", 2023-09-23T20:45:06.193 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.601e-06 2023-09-23T20:45:06.194 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.194 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.194 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:45:06.194 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.740465+0000", 2023-09-23T20:45:06.194 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.1069999999999998e-06 2023-09-23T20:45:06.194 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.194 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.194 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:45:06.194 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.744721+0000", 2023-09-23T20:45:06.194 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0042555429999999997 2023-09-23T20:45:06.194 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.195 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.195 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op must wait for map", 2023-09-23T20:45:06.195 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.744728+0000", 2023-09-23T20:45:06.195 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.413e-06 2023-09-23T20:45:06.195 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.195 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.195 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:45:06.195 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.752872+0000", 2023-09-23T20:45:06.195 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0081438929999999993 2023-09-23T20:45:06.195 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.195 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.195 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:45:06.195 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.752882+0000", 2023-09-23T20:45:06.196 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0397999999999999e-05 2023-09-23T20:45:06.196 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.196 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.196 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:45:06.196 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.784753+0000", 2023-09-23T20:45:06.196 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0318710520000001 2023-09-23T20:45:06.196 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.196 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.196 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:45:06.196 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.784849+0000", 2023-09-23T20:45:06.196 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.5699999999999995e-05 2023-09-23T20:45:06.196 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:45:06.197 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:45:06.197 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:45:06.197 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.197 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.197 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:141036 9.7 9:eb69e1ed:::benchmark_data_smithi018_381516_object27202:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e481)", 2023-09-23T20:45:06.197 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:43:15.740478+0000", 2023-09-23T20:45:06.197 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 110.389351913, 2023-09-23T20:45:06.197 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0444631559999999, 2023-09-23T20:45:06.197 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:45:06.197 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:45:06.197 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:45:06.197 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:45:06.197 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:45:06.198 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 141036 2023-09-23T20:45:06.198 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.198 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:45:06.198 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.198 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:45:06.198 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.740478+0000", 2023-09-23T20:45:06.198 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:45:06.198 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.198 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.198 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:45:06.198 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.740478+0000", 2023-09-23T20:45:06.199 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:45:06.199 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.199 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.199 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:45:06.199 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.740477+0000", 2023-09-23T20:45:06.199 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:45:06.199 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.199 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.199 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:45:06.199 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.740480+0000", 2023-09-23T20:45:06.199 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.729e-06 2023-09-23T20:45:06.199 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.200 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.200 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:45:06.200 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.740486+0000", 2023-09-23T20:45:06.200 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.1909999999999997e-06 2023-09-23T20:45:06.200 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.200 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.200 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:45:06.200 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.740492+0000", 2023-09-23T20:45:06.200 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.4990000000000002e-06 2023-09-23T20:45:06.200 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.200 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.200 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:45:06.201 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.744777+0000", 2023-09-23T20:45:06.201 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0042849949999999998 2023-09-23T20:45:06.201 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.201 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.201 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting_for_map not empty", 2023-09-23T20:45:06.201 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.744782+0000", 2023-09-23T20:45:06.201 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.4870000000000002e-06 2023-09-23T20:45:06.201 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.201 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.201 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:45:06.202 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.752918+0000", 2023-09-23T20:45:06.202 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0081358779999999992 2023-09-23T20:45:06.202 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.202 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.202 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:45:06.202 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.752926+0000", 2023-09-23T20:45:06.202 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.2749999999999995e-06 2023-09-23T20:45:06.202 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.202 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.202 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:45:06.202 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.784873+0000", 2023-09-23T20:45:06.202 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.03194633 2023-09-23T20:45:06.203 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.203 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.203 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:45:06.203 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.784941+0000", 2023-09-23T20:45:06.203 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.8213000000000006e-05 2023-09-23T20:45:06.203 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:45:06.203 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:45:06.203 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:45:06.203 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.203 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.204 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:141041 9.7 9:e17fa1db:::benchmark_data_smithi018_381516_object27207:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e481)", 2023-09-23T20:45:06.204 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:43:15.740496+0000", 2023-09-23T20:45:06.204 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 110.389333372, 2023-09-23T20:45:06.204 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0815582909999999, 2023-09-23T20:45:06.204 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:45:06.204 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:45:06.204 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:45:06.204 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:45:06.204 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:45:06.204 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 141041 2023-09-23T20:45:06.204 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.205 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:45:06.205 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.205 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:45:06.205 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.740496+0000", 2023-09-23T20:45:06.205 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:45:06.205 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.205 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.205 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:45:06.205 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.740496+0000", 2023-09-23T20:45:06.205 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:45:06.205 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.206 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.206 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:45:06.206 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.740496+0000", 2023-09-23T20:45:06.206 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:45:06.206 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.206 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.206 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:45:06.206 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.740498+0000", 2023-09-23T20:45:06.206 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.1780000000000002e-06 2023-09-23T20:45:06.206 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.206 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.206 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:45:06.207 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.740504+0000", 2023-09-23T20:45:06.207 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.5609999999999998e-06 2023-09-23T20:45:06.207 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.207 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.207 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:45:06.207 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.740507+0000", 2023-09-23T20:45:06.207 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.6950000000000002e-06 2023-09-23T20:45:06.207 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.207 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.207 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:45:06.207 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.744941+0000", 2023-09-23T20:45:06.208 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0044336360000000003 2023-09-23T20:45:06.208 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.208 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.208 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting_for_map not empty", 2023-09-23T20:45:06.208 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.744961+0000", 2023-09-23T20:45:06.208 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.9601000000000001e-05 2023-09-23T20:45:06.208 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.208 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.208 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:45:06.208 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.752986+0000", 2023-09-23T20:45:06.208 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0080249980000000002 2023-09-23T20:45:06.208 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.209 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.209 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:45:06.209 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.752993+0000", 2023-09-23T20:45:06.209 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.7719999999999994e-06 2023-09-23T20:45:06.209 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.209 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.209 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:45:06.209 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.784961+0000", 2023-09-23T20:45:06.209 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0319681039999999 2023-09-23T20:45:06.209 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.209 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.209 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for missing object", 2023-09-23T20:45:06.210 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.785024+0000", 2023-09-23T20:45:06.210 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.2626000000000007e-05 2023-09-23T20:45:06.210 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.210 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.210 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:45:06.210 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.821977+0000", 2023-09-23T20:45:06.210 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.036953184 2023-09-23T20:45:06.210 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.210 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.210 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:45:06.210 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.822055+0000", 2023-09-23T20:45:06.211 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.7239999999999999e-05 2023-09-23T20:45:06.211 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:45:06.211 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:45:06.211 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:45:06.211 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.211 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.211 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:141053 9.7 9:e03873d0:::benchmark_data_smithi018_381516_object27219:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e481)", 2023-09-23T20:45:06.211 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:43:15.775241+0000", 2023-09-23T20:45:06.211 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 110.354588444, 2023-09-23T20:45:06.211 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.047376954, 2023-09-23T20:45:06.211 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:45:06.211 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:45:06.212 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:45:06.212 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:45:06.212 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:45:06.212 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 141053 2023-09-23T20:45:06.212 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.212 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:45:06.212 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.212 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:45:06.212 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.775241+0000", 2023-09-23T20:45:06.212 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:45:06.212 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.212 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.213 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:45:06.213 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.775241+0000", 2023-09-23T20:45:06.213 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:45:06.213 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.213 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.213 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:45:06.213 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.775240+0000", 2023-09-23T20:45:06.213 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:45:06.213 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.213 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.213 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:45:06.213 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.775245+0000", 2023-09-23T20:45:06.214 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.8890000000000001e-06 2023-09-23T20:45:06.214 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.214 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.214 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:45:06.214 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.775257+0000", 2023-09-23T20:45:06.214 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1571e-05 2023-09-23T20:45:06.214 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.214 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.214 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:45:06.214 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.775261+0000", 2023-09-23T20:45:06.214 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.7999999999999998e-06 2023-09-23T20:45:06.214 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.215 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.215 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:45:06.215 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.775328+0000", 2023-09-23T20:45:06.215 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.6329999999999997e-05 2023-09-23T20:45:06.215 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.215 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.215 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:45:06.215 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.775340+0000", 2023-09-23T20:45:06.215 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.2172e-05 2023-09-23T20:45:06.215 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.215 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.215 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:45:06.216 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.785061+0000", 2023-09-23T20:45:06.216 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0097207939999999 2023-09-23T20:45:06.216 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.216 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.216 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:45:06.216 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.785170+0000", 2023-09-23T20:45:06.216 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000109317 2023-09-23T20:45:06.216 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.216 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.216 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 3,5", 2023-09-23T20:45:06.216 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.785802+0000", 2023-09-23T20:45:06.216 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00063209099999999999 2023-09-23T20:45:06.217 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.217 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.217 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:45:06.217 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.816501+0000", 2023-09-23T20:45:06.217 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.030699185 2023-09-23T20:45:06.217 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.217 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.217 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:45:06.217 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.817345+0000", 2023-09-23T20:45:06.217 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00084414799999999997 2023-09-23T20:45:06.217 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.218 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.218 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:45:06.218 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.822536+0000", 2023-09-23T20:45:06.218 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0051903310000000003 2023-09-23T20:45:06.218 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.218 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.218 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:45:06.218 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.822567+0000", 2023-09-23T20:45:06.218 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.1572999999999997e-05 2023-09-23T20:45:06.218 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.218 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.218 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:45:06.218 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.822618+0000", 2023-09-23T20:45:06.219 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.0825000000000002e-05 2023-09-23T20:45:06.219 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:45:06.219 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:45:06.219 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:45:06.219 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.219 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.219 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:141065 9.7 9:e7858487:::benchmark_data_smithi018_381516_object27231:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e481)", 2023-09-23T20:45:06.219 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:43:15.798761+0000", 2023-09-23T20:45:06.219 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 110.331068753, 2023-09-23T20:45:06.219 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0239784860000001, 2023-09-23T20:45:06.219 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:45:06.220 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:45:06.220 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:45:06.220 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:45:06.220 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:45:06.220 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 141065 2023-09-23T20:45:06.220 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.220 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:45:06.220 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.220 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:45:06.220 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.798761+0000", 2023-09-23T20:45:06.220 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:45:06.220 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.221 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.221 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:45:06.221 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.798761+0000", 2023-09-23T20:45:06.221 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:45:06.221 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.221 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.221 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:45:06.221 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.798760+0000", 2023-09-23T20:45:06.221 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:45:06.221 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.221 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.221 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:45:06.222 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.798764+0000", 2023-09-23T20:45:06.222 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.7869999999999999e-06 2023-09-23T20:45:06.222 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.222 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.222 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:45:06.222 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.798772+0000", 2023-09-23T20:45:06.222 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.2810000000000008e-06 2023-09-23T20:45:06.222 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.222 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.222 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:45:06.222 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.798776+0000", 2023-09-23T20:45:06.223 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.2899999999999996e-06 2023-09-23T20:45:06.223 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.223 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.223 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:45:06.223 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.798827+0000", 2023-09-23T20:45:06.223 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.0853000000000003e-05 2023-09-23T20:45:06.223 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.223 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.223 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:45:06.223 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.798836+0000", 2023-09-23T20:45:06.223 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.9439999999999994e-06 2023-09-23T20:45:06.224 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.224 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.224 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:45:06.224 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.809959+0000", 2023-09-23T20:45:06.224 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0111224510000001 2023-09-23T20:45:06.224 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.224 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.224 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:45:06.224 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.810079+0000", 2023-09-23T20:45:06.224 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00012024200000000001 2023-09-23T20:45:06.224 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.224 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.225 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 3,5", 2023-09-23T20:45:06.225 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.810209+0000", 2023-09-23T20:45:06.225 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00012967299999999999 2023-09-23T20:45:06.225 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.225 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.225 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:45:06.225 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.816508+0000", 2023-09-23T20:45:06.225 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0062991899999999997 2023-09-23T20:45:06.225 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.225 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.225 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:45:06.225 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.821373+0000", 2023-09-23T20:45:06.226 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0048652419999999997 2023-09-23T20:45:06.226 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.226 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.226 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:45:06.226 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.822639+0000", 2023-09-23T20:45:06.226 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0012657650000000001 2023-09-23T20:45:06.226 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.226 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.226 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:45:06.226 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.822671+0000", 2023-09-23T20:45:06.226 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.2335999999999997e-05 2023-09-23T20:45:06.227 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.227 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.227 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:45:06.227 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.822739+0000", 2023-09-23T20:45:06.227 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.8224000000000003e-05 2023-09-23T20:45:06.227 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:45:06.227 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:45:06.227 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:45:06.227 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.227 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.227 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:185078 9.d 9:b3b89e5b:::benchmark_data_smithi018_381516_object71244:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e501)", 2023-09-23T20:45:06.228 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:44:17.632886+0000", 2023-09-23T20:45:06.228 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 48.496943784999999, 2023-09-23T20:45:06.228 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0561142219999999, 2023-09-23T20:45:06.228 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:45:06.228 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:45:06.228 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:45:06.228 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:45:06.228 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:45:06.228 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 185078 2023-09-23T20:45:06.228 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.228 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:45:06.228 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.228 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:45:06.229 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:17.632886+0000", 2023-09-23T20:45:06.229 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:45:06.229 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.229 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.229 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:45:06.229 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:17.632886+0000", 2023-09-23T20:45:06.229 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:45:06.229 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.229 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.229 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:45:06.229 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:17.632885+0000", 2023-09-23T20:45:06.229 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:45:06.230 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.230 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.230 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:45:06.230 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:17.632890+0000", 2023-09-23T20:45:06.230 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.1120000000000004e-06 2023-09-23T20:45:06.230 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.230 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.230 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:45:06.230 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:17.632900+0000", 2023-09-23T20:45:06.230 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0541e-05 2023-09-23T20:45:06.230 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.230 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.231 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:45:06.231 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:17.632906+0000", 2023-09-23T20:45:06.231 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.1939999999999997e-06 2023-09-23T20:45:06.231 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.231 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.231 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:45:06.231 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:17.634328+0000", 2023-09-23T20:45:06.231 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0014227879999999999 2023-09-23T20:45:06.231 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.231 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.231 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:45:06.232 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:17.634341+0000", 2023-09-23T20:45:06.232 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.2524000000000001e-05 2023-09-23T20:45:06.232 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.232 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.232 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:45:06.232 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:18.658019+0000", 2023-09-23T20:45:06.232 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.023678552 2023-09-23T20:45:06.232 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.232 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.232 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:45:06.232 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:18.658147+0000", 2023-09-23T20:45:06.233 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000128088 2023-09-23T20:45:06.233 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.233 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.233 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 1,5", 2023-09-23T20:45:06.233 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:18.658233+0000", 2023-09-23T20:45:06.233 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.5655000000000006e-05 2023-09-23T20:45:06.233 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.233 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.233 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:45:06.233 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:18.667554+0000", 2023-09-23T20:45:06.233 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0093206199999999999 2023-09-23T20:45:06.233 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.234 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.234 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:45:06.234 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:18.688813+0000", 2023-09-23T20:45:06.234 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.021259726999999999 2023-09-23T20:45:06.234 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.234 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.234 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:45:06.234 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:18.688941+0000", 2023-09-23T20:45:06.234 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00012763799999999999 2023-09-23T20:45:06.234 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.234 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.234 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:45:06.235 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:18.688974+0000", 2023-09-23T20:45:06.237 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.2818000000000003e-05 2023-09-23T20:45:06.237 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.238 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.238 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:45:06.238 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:18.689000+0000", 2023-09-23T20:45:06.238 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.6100000000000001e-05 2023-09-23T20:45:06.238 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:45:06.238 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:45:06.238 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:45:06.238 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.238 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.238 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:185094 9.d 9:b6a792e5:::benchmark_data_smithi018_381516_object71260:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e502)", 2023-09-23T20:45:06.238 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:44:17.668103+0000", 2023-09-23T20:45:06.238 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 48.461726284000001, 2023-09-23T20:45:06.239 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.020989529, 2023-09-23T20:45:06.239 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:45:06.239 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:45:06.239 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:45:06.239 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:45:06.239 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:45:06.239 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 185094 2023-09-23T20:45:06.239 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.239 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:45:06.239 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.239 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:45:06.239 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:17.668103+0000", 2023-09-23T20:45:06.240 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:45:06.240 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.240 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.240 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:45:06.240 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:17.668103+0000", 2023-09-23T20:45:06.240 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:45:06.240 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.240 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.240 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:45:06.240 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:17.668101+0000", 2023-09-23T20:45:06.240 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999981 2023-09-23T20:45:06.241 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.241 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.241 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:45:06.241 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:17.668111+0000", 2023-09-23T20:45:06.241 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.7170000000000003e-06 2023-09-23T20:45:06.241 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.241 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.241 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:45:06.241 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:17.668138+0000", 2023-09-23T20:45:06.241 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.6786000000000001e-05 2023-09-23T20:45:06.241 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.241 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.242 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:45:06.242 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:17.668149+0000", 2023-09-23T20:45:06.242 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1185999999999999e-05 2023-09-23T20:45:06.242 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.242 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.242 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:45:06.242 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:17.668255+0000", 2023-09-23T20:45:06.242 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000105659 2023-09-23T20:45:06.242 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.242 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.242 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:45:06.242 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:17.668269+0000", 2023-09-23T20:45:06.243 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.4499e-05 2023-09-23T20:45:06.243 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.243 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.243 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:45:06.243 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:18.660713+0000", 2023-09-23T20:45:06.243 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.99244370500000001 2023-09-23T20:45:06.243 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.243 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.243 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:45:06.243 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:18.660830+0000", 2023-09-23T20:45:06.243 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000116749 2023-09-23T20:45:06.243 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.244 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.244 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 1,5", 2023-09-23T20:45:06.244 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:18.660911+0000", 2023-09-23T20:45:06.244 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.1076999999999996e-05 2023-09-23T20:45:06.244 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.244 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.244 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:45:06.244 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:18.671038+0000", 2023-09-23T20:45:06.244 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.010126753 2023-09-23T20:45:06.244 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.244 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.244 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:45:06.245 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:18.688861+0000", 2023-09-23T20:45:06.245 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.017823008000000001 2023-09-23T20:45:06.245 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.245 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.245 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:45:06.245 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:18.689042+0000", 2023-09-23T20:45:06.245 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000181774 2023-09-23T20:45:06.245 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.245 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.245 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:45:06.245 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:18.689071+0000", 2023-09-23T20:45:06.245 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.845e-05 2023-09-23T20:45:06.246 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.246 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.246 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:45:06.246 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:18.689093+0000", 2023-09-23T20:45:06.246 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.2059e-05 2023-09-23T20:45:06.246 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:45:06.246 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:45:06.246 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:45:06.246 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.246 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.246 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:185104 9.d 9:b62087dc:::benchmark_data_smithi018_381516_object71270:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e502)", 2023-09-23T20:45:06.246 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:44:17.675664+0000", 2023-09-23T20:45:06.247 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 48.454165392, 2023-09-23T20:45:06.247 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.013501465, 2023-09-23T20:45:06.247 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:45:06.247 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:45:06.247 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:45:06.247 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:45:06.247 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:45:06.247 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 185104 2023-09-23T20:45:06.247 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.247 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:45:06.247 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.247 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:45:06.248 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:17.675664+0000", 2023-09-23T20:45:06.248 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:45:06.248 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.248 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.248 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:45:06.248 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:17.675664+0000", 2023-09-23T20:45:06.248 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:45:06.248 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.248 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.248 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:45:06.248 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:17.675663+0000", 2023-09-23T20:45:06.248 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:45:06.249 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.249 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.249 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:45:06.249 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:17.675668+0000", 2023-09-23T20:45:06.249 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.1590000000000001e-06 2023-09-23T20:45:06.249 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.249 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.249 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:45:06.249 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:17.675680+0000", 2023-09-23T20:45:06.249 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1627999999999999e-05 2023-09-23T20:45:06.249 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.249 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.250 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:45:06.250 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:17.675686+0000", 2023-09-23T20:45:06.250 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.336e-06 2023-09-23T20:45:06.250 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.250 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.250 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:45:06.250 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:17.675741+0000", 2023-09-23T20:45:06.250 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.5118999999999999e-05 2023-09-23T20:45:06.250 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.250 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.250 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:45:06.250 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:17.675751+0000", 2023-09-23T20:45:06.251 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0273e-05 2023-09-23T20:45:06.251 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.251 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.251 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:45:06.251 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:18.663046+0000", 2023-09-23T20:45:06.251 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.98729466099999996 2023-09-23T20:45:06.251 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.251 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.251 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:45:06.251 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:18.663140+0000", 2023-09-23T20:45:06.251 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.4197999999999995e-05 2023-09-23T20:45:06.251 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.252 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.252 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 1,5", 2023-09-23T20:45:06.252 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:18.663221+0000", 2023-09-23T20:45:06.252 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.1042000000000001e-05 2023-09-23T20:45:06.252 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.252 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.252 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:45:06.252 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:18.671047+0000", 2023-09-23T20:45:06.252 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0078258500000000005 2023-09-23T20:45:06.252 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.252 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.252 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:45:06.253 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:18.688896+0000", 2023-09-23T20:45:06.253 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.017848636000000001 2023-09-23T20:45:06.253 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.253 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.253 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:45:06.253 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:18.689121+0000", 2023-09-23T20:45:06.253 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00022569800000000001 2023-09-23T20:45:06.253 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.253 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.253 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:45:06.253 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:18.689146+0000", 2023-09-23T20:45:06.253 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.4720999999999999e-05 2023-09-23T20:45:06.254 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.254 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.254 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:45:06.254 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:18.689166+0000", 2023-09-23T20:45:06.254 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.9443999999999999e-05 2023-09-23T20:45:06.254 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:45:06.254 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:45:06.254 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:45:06.254 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.254 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.254 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:185118 9.d 9:b4f497e5:::benchmark_data_smithi018_381516_object71284:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e502)", 2023-09-23T20:45:06.254 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:44:17.691676+0000", 2023-09-23T20:45:06.255 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 48.438154040000001, 2023-09-23T20:45:06.255 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.99755927600000005, 2023-09-23T20:45:06.255 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:45:06.255 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:45:06.255 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:45:06.255 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:45:06.255 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:45:06.255 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 185118 2023-09-23T20:45:06.255 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.255 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:45:06.255 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.255 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:45:06.256 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:17.691676+0000", 2023-09-23T20:45:06.256 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:45:06.256 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.256 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.256 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:45:06.256 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:17.691676+0000", 2023-09-23T20:45:06.256 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:45:06.256 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.256 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.256 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:45:06.256 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:17.691675+0000", 2023-09-23T20:45:06.256 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:45:06.257 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.257 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.257 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:45:06.257 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:17.691678+0000", 2023-09-23T20:45:06.257 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.6759999999999999e-06 2023-09-23T20:45:06.257 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.257 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.257 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:45:06.257 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:17.691684+0000", 2023-09-23T20:45:06.257 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.1129999999999997e-06 2023-09-23T20:45:06.257 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.257 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.258 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:45:06.258 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:17.691688+0000", 2023-09-23T20:45:06.258 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.605e-06 2023-09-23T20:45:06.258 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.258 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.258 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:45:06.258 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:17.691755+0000", 2023-09-23T20:45:06.258 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.6821000000000003e-05 2023-09-23T20:45:06.258 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.258 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.258 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:45:06.258 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:17.691764+0000", 2023-09-23T20:45:06.259 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.0920000000000004e-06 2023-09-23T20:45:06.259 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.259 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.259 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:45:06.259 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:18.665566+0000", 2023-09-23T20:45:06.259 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.97380192099999996 2023-09-23T20:45:06.259 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.259 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.259 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:45:06.259 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:18.665669+0000", 2023-09-23T20:45:06.259 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000102705 2023-09-23T20:45:06.259 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.260 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.260 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 1,5", 2023-09-23T20:45:06.260 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:18.665742+0000", 2023-09-23T20:45:06.260 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.2551000000000004e-05 2023-09-23T20:45:06.260 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.260 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.260 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:45:06.260 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:18.674880+0000", 2023-09-23T20:45:06.260 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0091383439999999996 2023-09-23T20:45:06.260 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.260 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.260 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:45:06.260 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:18.688918+0000", 2023-09-23T20:45:06.261 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.014038030999999999 2023-09-23T20:45:06.261 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.261 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.261 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:45:06.261 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:18.689192+0000", 2023-09-23T20:45:06.261 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0002744 2023-09-23T20:45:06.261 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.261 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.261 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:45:06.261 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:18.689216+0000", 2023-09-23T20:45:06.261 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.3363999999999999e-05 2023-09-23T20:45:06.261 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.262 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.262 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:45:06.262 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:18.689235+0000", 2023-09-23T20:45:06.262 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.9128999999999999e-05 2023-09-23T20:45:06.262 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:45:06.262 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:45:06.262 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:45:06.262 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.262 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.262 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:185119 9.d 9:b5ed0d02:::benchmark_data_smithi018_381516_object71285:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e502)", 2023-09-23T20:45:06.262 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:44:17.695956+0000", 2023-09-23T20:45:06.263 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 48.433873898000002, 2023-09-23T20:45:06.263 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.99344711699999999, 2023-09-23T20:45:06.263 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:45:06.263 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:45:06.263 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:45:06.263 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:45:06.263 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:45:06.263 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 185119 2023-09-23T20:45:06.263 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.263 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:45:06.263 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.263 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:45:06.264 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:17.695956+0000", 2023-09-23T20:45:06.264 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:45:06.264 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.264 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.264 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:45:06.264 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:17.695956+0000", 2023-09-23T20:45:06.264 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:45:06.264 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.264 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.264 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:45:06.264 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:17.695955+0000", 2023-09-23T20:45:06.264 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:45:06.265 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.265 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.265 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:45:06.265 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:17.695960+0000", 2023-09-23T20:45:06.265 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.7870000000000001e-06 2023-09-23T20:45:06.265 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.265 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.265 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:45:06.265 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:17.695969+0000", 2023-09-23T20:45:06.265 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.5480000000000007e-06 2023-09-23T20:45:06.265 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.265 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.266 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:45:06.266 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:17.695975+0000", 2023-09-23T20:45:06.266 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.9939999999999997e-06 2023-09-23T20:45:06.266 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.266 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.266 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:45:06.266 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:17.696021+0000", 2023-09-23T20:45:06.266 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.5930000000000002e-05 2023-09-23T20:45:06.266 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.266 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.266 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:45:06.267 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:17.696031+0000", 2023-09-23T20:45:06.267 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.4919999999999999e-06 2023-09-23T20:45:06.267 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.267 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.267 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:45:06.267 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:18.667599+0000", 2023-09-23T20:45:06.267 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.97156826500000004 2023-09-23T20:45:06.267 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.267 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.267 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:45:06.267 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:18.667687+0000", 2023-09-23T20:45:06.267 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.8027000000000003e-05 2023-09-23T20:45:06.268 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.268 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.268 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 1,5", 2023-09-23T20:45:06.268 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:18.667760+0000", 2023-09-23T20:45:06.268 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.3023999999999997e-05 2023-09-23T20:45:06.268 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.268 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.268 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:45:06.268 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:18.674888+0000", 2023-09-23T20:45:06.268 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0071280880000000003 2023-09-23T20:45:06.268 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.269 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.269 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:45:06.269 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:18.689255+0000", 2023-09-23T20:45:06.269 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.014366556000000001 2023-09-23T20:45:06.269 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.269 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.269 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:45:06.269 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:18.689347+0000", 2023-09-23T20:45:06.269 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.2168000000000003e-05 2023-09-23T20:45:06.269 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.269 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.269 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:45:06.270 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:18.689374+0000", 2023-09-23T20:45:06.270 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.7549999999999999e-05 2023-09-23T20:45:06.270 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.270 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.270 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:45:06.270 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:18.689403+0000", 2023-09-23T20:45:06.270 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.8544999999999999e-05 2023-09-23T20:45:06.270 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:45:06.270 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:45:06.270 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:45:06.270 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.271 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.271 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:185146 9.d 9:b2fc9462:::benchmark_data_smithi018_381516_object71312:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e502)", 2023-09-23T20:45:06.271 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:44:17.733852+0000", 2023-09-23T20:45:06.271 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 48.395977424999998, 2023-09-23T20:45:06.271 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.95566239200000003, 2023-09-23T20:45:06.271 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:45:06.271 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:45:06.271 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:45:06.271 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:45:06.271 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:45:06.271 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 185146 2023-09-23T20:45:06.272 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.272 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:45:06.272 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.272 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:45:06.272 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:17.733852+0000", 2023-09-23T20:45:06.272 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:45:06.272 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.272 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.272 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:45:06.272 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:17.733852+0000", 2023-09-23T20:45:06.272 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:45:06.273 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.273 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.273 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:45:06.273 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:17.733851+0000", 2023-09-23T20:45:06.273 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:45:06.273 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.273 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.273 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:45:06.273 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:17.733856+0000", 2023-09-23T20:45:06.273 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.7620000000000003e-06 2023-09-23T20:45:06.273 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.273 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.274 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:45:06.274 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:17.733882+0000", 2023-09-23T20:45:06.274 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.6594999999999999e-05 2023-09-23T20:45:06.274 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.274 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.274 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:45:06.274 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:17.733887+0000", 2023-09-23T20:45:06.274 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.707e-06 2023-09-23T20:45:06.274 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.274 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.274 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:45:06.275 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:17.733917+0000", 2023-09-23T20:45:06.275 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.9641999999999998e-05 2023-09-23T20:45:06.275 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.275 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.275 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:45:06.275 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:17.733927+0000", 2023-09-23T20:45:06.275 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0237e-05 2023-09-23T20:45:06.275 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.275 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.275 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:45:06.275 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:18.669286+0000", 2023-09-23T20:45:06.275 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.93535897800000001 2023-09-23T20:45:06.275 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.276 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.276 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:45:06.276 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:18.669417+0000", 2023-09-23T20:45:06.276 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000130681 2023-09-23T20:45:06.276 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.276 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.276 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 1,5", 2023-09-23T20:45:06.276 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:18.669556+0000", 2023-09-23T20:45:06.276 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00013978800000000001 2023-09-23T20:45:06.276 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.276 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.277 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:45:06.277 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:18.678879+0000", 2023-09-23T20:45:06.277 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0093222779999999998 2023-09-23T20:45:06.277 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.277 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.277 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:45:06.277 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:18.689434+0000", 2023-09-23T20:45:06.277 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.010555314 2023-09-23T20:45:06.277 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.277 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.277 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:45:06.277 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:18.689465+0000", 2023-09-23T20:45:06.278 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.0781000000000002e-05 2023-09-23T20:45:06.278 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.278 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.278 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:45:06.278 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:18.689495+0000", 2023-09-23T20:45:06.278 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.9865999999999998e-05 2023-09-23T20:45:06.278 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.278 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.278 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:45:06.278 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:18.689515+0000", 2023-09-23T20:45:06.279 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.9998999999999999e-05 2023-09-23T20:45:06.279 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:45:06.279 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:45:06.279 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:45:06.279 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.279 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.279 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:185150 9.d 9:b4742237:::benchmark_data_smithi018_381516_object71316:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e502)", 2023-09-23T20:45:06.279 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:44:17.744069+0000", 2023-09-23T20:45:06.279 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 48.385760449999999, 2023-09-23T20:45:06.279 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.945572937, 2023-09-23T20:45:06.279 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:45:06.279 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:45:06.280 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:45:06.280 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:45:06.280 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:45:06.280 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 185150 2023-09-23T20:45:06.280 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.280 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:45:06.280 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.280 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:45:06.280 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:17.744069+0000", 2023-09-23T20:45:06.280 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:45:06.280 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.281 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.281 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:45:06.281 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:17.744069+0000", 2023-09-23T20:45:06.281 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:45:06.281 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.281 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.281 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:45:06.281 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:17.744069+0000", 2023-09-23T20:45:06.281 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:45:06.281 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.281 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.281 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:45:06.282 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:17.744072+0000", 2023-09-23T20:45:06.282 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.9409999999999999e-06 2023-09-23T20:45:06.282 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.282 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.282 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:45:06.282 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:17.744081+0000", 2023-09-23T20:45:06.282 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.6569999999999992e-06 2023-09-23T20:45:06.282 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.282 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.282 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:45:06.282 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:17.744086+0000", 2023-09-23T20:45:06.282 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.7339999999999999e-06 2023-09-23T20:45:06.283 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.283 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.283 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:45:06.283 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:17.744122+0000", 2023-09-23T20:45:06.283 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.5547999999999999e-05 2023-09-23T20:45:06.283 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.283 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.283 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:45:06.283 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:17.744131+0000", 2023-09-23T20:45:06.283 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.5079999999999994e-06 2023-09-23T20:45:06.283 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.283 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.284 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:45:06.284 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:18.671089+0000", 2023-09-23T20:45:06.284 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.92695763099999995 2023-09-23T20:45:06.284 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.284 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.284 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:45:06.284 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:18.671195+0000", 2023-09-23T20:45:06.284 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00010632999999999999 2023-09-23T20:45:06.284 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.284 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.284 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 1,5", 2023-09-23T20:45:06.284 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:18.671292+0000", 2023-09-23T20:45:06.285 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.6521999999999995e-05 2023-09-23T20:45:06.285 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.285 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.285 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:45:06.285 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:18.678889+0000", 2023-09-23T20:45:06.285 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0075970090000000001 2023-09-23T20:45:06.285 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.285 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.285 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:45:06.285 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:18.689543+0000", 2023-09-23T20:45:06.285 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.010654716 2023-09-23T20:45:06.285 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.286 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.286 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:45:06.286 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:18.689596+0000", 2023-09-23T20:45:06.286 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.2240999999999997e-05 2023-09-23T20:45:06.286 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.286 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.286 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:45:06.286 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:18.689621+0000", 2023-09-23T20:45:06.286 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.5037000000000001e-05 2023-09-23T20:45:06.286 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.286 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.286 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:45:06.287 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:18.689642+0000", 2023-09-23T20:45:06.287 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.1500000000000001e-05 2023-09-23T20:45:06.287 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:45:06.287 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:45:06.287 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:45:06.287 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.287 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.287 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:185176 9.d 9:b2b2acf7:::benchmark_data_smithi018_381516_object71342:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e502)", 2023-09-23T20:45:06.287 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:44:17.783438+0000", 2023-09-23T20:45:06.287 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 48.346391574999998, 2023-09-23T20:45:06.287 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.90630178900000002, 2023-09-23T20:45:06.287 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:45:06.287 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:45:06.288 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:45:06.288 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:45:06.288 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:45:06.288 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 185176 2023-09-23T20:45:06.288 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.288 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:45:06.288 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.288 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:45:06.288 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:17.783438+0000", 2023-09-23T20:45:06.288 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:45:06.288 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.288 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.289 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:45:06.289 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:17.783438+0000", 2023-09-23T20:45:06.289 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:45:06.289 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.289 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.289 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:45:06.289 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:17.783437+0000", 2023-09-23T20:45:06.289 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:45:06.289 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.289 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.289 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:45:06.289 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:17.783443+0000", 2023-09-23T20:45:06.290 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.8560000000000003e-06 2023-09-23T20:45:06.290 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.290 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.290 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:45:06.290 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:17.783455+0000", 2023-09-23T20:45:06.290 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.2285e-05 2023-09-23T20:45:06.290 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.290 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.290 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:45:06.290 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:17.783461+0000", 2023-09-23T20:45:06.290 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.6189999999999999e-06 2023-09-23T20:45:06.291 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.291 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.291 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:45:06.291 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:17.783493+0000", 2023-09-23T20:45:06.291 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.2659e-05 2023-09-23T20:45:06.291 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.291 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.291 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:45:06.291 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:17.783499+0000", 2023-09-23T20:45:06.291 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.9959999999999999e-06 2023-09-23T20:45:06.291 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.291 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.292 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:45:06.292 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:18.673266+0000", 2023-09-23T20:45:06.292 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.88976690800000002 2023-09-23T20:45:06.292 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.292 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.292 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:45:06.292 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:18.673377+0000", 2023-09-23T20:45:06.292 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000111025 2023-09-23T20:45:06.292 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.292 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.292 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 1,5", 2023-09-23T20:45:06.292 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:18.673450+0000", 2023-09-23T20:45:06.293 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.3075000000000001e-05 2023-09-23T20:45:06.293 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.293 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.293 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:45:06.293 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:18.682110+0000", 2023-09-23T20:45:06.293 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0086597629999999991 2023-09-23T20:45:06.293 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.293 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.293 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:45:06.293 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:18.689571+0000", 2023-09-23T20:45:06.293 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0074609680000000001 2023-09-23T20:45:06.293 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.294 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.294 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:45:06.294 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:18.689695+0000", 2023-09-23T20:45:06.294 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00012444999999999999 2023-09-23T20:45:06.294 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.294 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.294 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:45:06.294 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:18.689719+0000", 2023-09-23T20:45:06.294 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.3634000000000001e-05 2023-09-23T20:45:06.294 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:45:06.294 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:45:06.295 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:45:06.295 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:18.689740+0000", 2023-09-23T20:45:06.295 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.0670999999999999e-05 2023-09-23T20:45:06.295 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:45:06.295 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:45:06.295 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:45:06.295 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:45:06.295 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:45:06.295 INFO:teuthology.orchestra.run.smithi149.stdout:} 2023-09-23T20:45:06.296 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:45:06.296 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:05 smithi149 ceph-mon[170394]: pgmap v1307: 119 pgs: 119 active+clean; 751 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 879 op/s 2023-09-23T20:45:06.296 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:05 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:45:06.296 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:05 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/617387331' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:45:06.296 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:06.297 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:45:06.297 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:45:06.297 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:45:06.297 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:45:06.297 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:45:06.297 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:45:06.297 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:45:06.298 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:45:06.298 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:45:06.332 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:45:06.529 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:45:06.538 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:06.538 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:45:06.564 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T20:45:06.639 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:45:06.719 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:06.799 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:45:06.811 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:06.812 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:45:06.912 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:45:06.983 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:07.115 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:45:07.129 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:07.129 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:45:07.230 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:45:07.305 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:07.906 INFO:teuthology.orchestra.run.smithi018.stderr:nodeep-scrub is set 2023-09-23T20:45:08.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:07 smithi018 ceph-mon[179647]: pgmap v1308: 119 pgs: 119 active+clean; 698 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 680 op/s 2023-09-23T20:45:08.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:07 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3205532824' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:45:08.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:07 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:45:08.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:07 smithi018 ceph-mon[184144]: pgmap v1308: 119 pgs: 119 active+clean; 698 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 680 op/s 2023-09-23T20:45:08.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:07 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3205532824' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:45:08.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:07 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:45:08.216 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:45:07 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a[153650]: logger=cleanup t=2023-09-23T20:45:07.928647632Z level=info msg="Completed cleanup jobs" duration=13.931304ms 2023-09-23T20:45:08.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:07 smithi149 ceph-mon[170394]: pgmap v1308: 119 pgs: 119 active+clean; 698 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 680 op/s 2023-09-23T20:45:08.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:07 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3205532824' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:45:08.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:07 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:45:08.773 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:45:09.146 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:08 smithi018 ceph-mon[179647]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:45:09.146 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:08 smithi018 ceph-mon[179647]: failed to encode map e523 with expected crc 2023-09-23T20:45:09.146 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:08 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:45:09.146 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:08 smithi018 ceph-mon[179647]: osdmap e523: 8 total, 8 up, 8 in 2023-09-23T20:45:09.147 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:08 smithi018 ceph-mon[184144]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:45:09.147 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:08 smithi018 ceph-mon[184144]: failed to encode map e523 with expected crc 2023-09-23T20:45:09.147 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:08 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:45:09.147 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:08 smithi018 ceph-mon[184144]: osdmap e523: 8 total, 8 up, 8 in 2023-09-23T20:45:09.219 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:08 smithi149 ceph-mon[170394]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:45:09.219 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:08 smithi149 ceph-mon[170394]: failed to encode map e523 with expected crc 2023-09-23T20:45:09.219 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:08 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:45:09.219 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:08 smithi149 ceph-mon[170394]: osdmap e523: 8 total, 8 up, 8 in 2023-09-23T20:45:09.909 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:09 smithi018 ceph-mon[184144]: pgmap v1310: 119 pgs: 119 active+clean; 561 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 885 op/s 2023-09-23T20:45:09.910 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:09 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:45:10.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:09 smithi149 ceph-mon[170394]: pgmap v1310: 119 pgs: 119 active+clean; 561 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 885 op/s 2023-09-23T20:45:10.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:09 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:45:10.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:09 smithi018 ceph-mon[179647]: pgmap v1310: 119 pgs: 119 active+clean; 561 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 885 op/s 2023-09-23T20:45:10.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:09 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:45:10.696 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:45:10.702 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:10.703 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:45:10.803 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:45:10.867 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:10.972 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:45:10.981 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:10.981 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:45:11.082 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:45:11.141 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:11.392 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:45:12.082 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:45:12.102 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:12.103 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:45:12.204 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:45:12.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:11 smithi149 ceph-mon[170394]: pgmap v1311: 119 pgs: 119 active+clean; 423 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 954 op/s 2023-09-23T20:45:12.256 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:11 smithi018 ceph-mon[184144]: pgmap v1311: 119 pgs: 119 active+clean; 423 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 954 op/s 2023-09-23T20:45:12.256 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:11 smithi018 ceph-mon[179647]: pgmap v1311: 119 pgs: 119 active+clean; 423 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 954 op/s 2023-09-23T20:45:12.269 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:12.363 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:45:13.185 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:45:13.189 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:13.190 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:45:13.290 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:45:13.360 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:14.005 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:45:14.012 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:14.013 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:45:14.087 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:13 smithi018 ceph-mon[184144]: pgmap v1312: 119 pgs: 119 active+clean; 368 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 886 op/s 2023-09-23T20:45:14.088 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:13 smithi018 ceph-mon[179647]: pgmap v1312: 119 pgs: 119 active+clean; 368 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 886 op/s 2023-09-23T20:45:14.114 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:45:14.137 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T20:45:14.215 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:14.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:13 smithi149 ceph-mon[170394]: pgmap v1312: 119 pgs: 119 active+clean; 368 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 886 op/s 2023-09-23T20:45:14.310 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:45:14.329 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:14.329 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:45:14.430 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:45:14.546 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:14.666 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:45:14.681 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:14.681 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:45:14.693 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:45:14.722 INFO:teuthology.orchestra.run.smithi018.stderr:dumped all 2023-09-23T20:45:14.783 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:45:14.823 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:14.898 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:45:14.904 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:14.904 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:45:15.005 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:45:15.091 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:14 smithi149 ceph-mon[170394]: pgmap v1313: 119 pgs: 119 active+clean; 189 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 898 op/s 2023-09-23T20:45:15.091 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:14 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:45:15.091 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:14 smithi149 ceph-mon[170394]: from='client.36792 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:45:15.112 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:15.183 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:45:15.189 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:15.189 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:45:15.290 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:45:15.311 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:14 smithi018 ceph-mon[179647]: pgmap v1313: 119 pgs: 119 active+clean; 189 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 898 op/s 2023-09-23T20:45:15.311 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:14 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:45:15.311 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:14 smithi018 ceph-mon[179647]: from='client.36792 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:45:15.312 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:45:14 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:45:14] "GET /metrics HTTP/1.1" 200 36506 "" "Prometheus/2.43.0" 2023-09-23T20:45:15.312 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:14 smithi018 ceph-mon[184144]: pgmap v1313: 119 pgs: 119 active+clean; 189 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 898 op/s 2023-09-23T20:45:15.312 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:14 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:45:15.312 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:14 smithi018 ceph-mon[184144]: from='client.36792 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:45:15.382 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:15.446 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:45:15.453 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:15.453 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:45:15.546 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Removed 113831 objects 2023-09-23T20:45:15.546 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Clean up completed and total clean up time :156.367 2023-09-23T20:45:15.555 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:45:15.620 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:17.100 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:16 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:45:17.100 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:16 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:45:17.100 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:16 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:45:17.100 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:16 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:45:17.210 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:45:17.215 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:17.216 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:45:17.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:16 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:45:17.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:16 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:45:17.317 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:45:17.354 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:17.370 INFO:teuthology.orchestra.run.smithi018.stdout:{"pg_ready":true,"pg_map":{"version":1313,"stamp":"2023-09-23T20:45:14.224670+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":197990347,"num_objects":2977,"num_object_clones":0,"num_object_copies":8931,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2977,"num_whiteouts":0,"num_read":5984,"num_read_kb":13201,"num_write":225622,"num_write_kb":7297910,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":17055,"num_bytes_recovered":1122650190,"num_keys_recovered":40,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":109232,"ondisk_log_size":109232,"up":357,"acting":357,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":361,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":5958592,"kb_used_data":1008384,"kb_used_omap":32,"kb_used_meta":4949471,"kb_avail":744002624,"statfs":{"total":767960285184,"available":761858686976,"internally_reserved":0,"allocated":1032585216,"data_stored":1016234627,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32773,"internal_metadata":5068259323},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":7,"apply_latency_ms":7,"commit_latency_ns":7000000,"apply_latency_ns":7000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-589496320,"num_objects":-8995,"num_object_clones":0,"num_object_copies":-26985,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-8995,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8995,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.005642"},"pg_stats":[{"pgid":"1.18","version":"90'161","reported_seq":568,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.864220+0000","last_change":"2023-09-23T20:44:19.677357+0000","last_active":"2023-09-23T20:45:01.864220+0000","last_peered":"2023-09-23T20:45:01.864220+0000","last_clean":"2023-09-23T20:45:01.864220+0000","last_became_active":"2023-09-23T20:43:16.766305+0000","last_became_peered":"2023-09-23T20:43:16.766305+0000","last_unstale":"2023-09-23T20:45:01.864220+0000","last_undegraded":"2023-09-23T20:45:01.864220+0000","last_fullsized":"2023-09-23T20:45:01.864220+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:19.677209+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:19.677209+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:14:06.510115+0000","scrub_duration":0.0023292260000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"69'1","reported_seq":220,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.853993+0000","last_change":"2023-09-23T20:32:33.233853+0000","last_active":"2023-09-23T20:45:01.853993+0000","last_peered":"2023-09-23T20:45:01.853993+0000","last_clean":"2023-09-23T20:45:01.853993+0000","last_became_active":"2023-09-23T20:32:33.233546+0000","last_became_peered":"2023-09-23T20:32:33.233546+0000","last_unstale":"2023-09-23T20:45:01.853993+0000","last_undegraded":"2023-09-23T20:45:01.853993+0000","last_fullsized":"2023-09-23T20:45:01.853993+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:50:27.130808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":337,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.857252+0000","last_change":"2023-09-23T20:37:43.579611+0000","last_active":"2023-09-23T20:45:01.857252+0000","last_peered":"2023-09-23T20:45:01.857252+0000","last_clean":"2023-09-23T20:45:01.857252+0000","last_became_active":"2023-09-23T20:37:43.579323+0000","last_became_peered":"2023-09-23T20:37:43.579323+0000","last_unstale":"2023-09-23T20:45:01.857252+0000","last_undegraded":"2023-09-23T20:45:01.857252+0000","last_fullsized":"2023-09-23T20:45:01.857252+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:17:32.311427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"90'161","reported_seq":644,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.853969+0000","last_change":"2023-09-23T20:44:14.647447+0000","last_active":"2023-09-23T20:45:01.853969+0000","last_peered":"2023-09-23T20:45:01.853969+0000","last_clean":"2023-09-23T20:45:01.853969+0000","last_became_active":"2023-09-23T20:43:16.756116+0000","last_became_peered":"2023-09-23T20:43:16.756116+0000","last_unstale":"2023-09-23T20:45:01.853969+0000","last_undegraded":"2023-09-23T20:45:01.853969+0000","last_fullsized":"2023-09-23T20:45:01.853969+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:14.647386+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:36:05.442288+0000","last_clean_scrub_stamp":"2023-09-23T20:44:14.647386+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:25:25.151100+0000","scrub_duration":0.0097381210000000006,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":493,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.860051+0000","last_change":"2023-09-23T20:34:59.983035+0000","last_active":"2023-09-23T20:45:01.860051+0000","last_peered":"2023-09-23T20:45:01.860051+0000","last_clean":"2023-09-23T20:45:01.860051+0000","last_became_active":"2023-09-23T20:34:59.982459+0000","last_became_peered":"2023-09-23T20:34:59.982459+0000","last_unstale":"2023-09-23T20:45:01.860051+0000","last_undegraded":"2023-09-23T20:45:01.860051+0000","last_fullsized":"2023-09-23T20:45:01.860051+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:16:48.510783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":480,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.869841+0000","last_change":"2023-09-23T20:43:46.521939+0000","last_active":"2023-09-23T20:45:01.869841+0000","last_peered":"2023-09-23T20:45:01.869841+0000","last_clean":"2023-09-23T20:45:01.869841+0000","last_became_active":"2023-09-23T20:43:46.521185+0000","last_became_peered":"2023-09-23T20:43:46.521185+0000","last_unstale":"2023-09-23T20:45:01.869841+0000","last_undegraded":"2023-09-23T20:45:01.869841+0000","last_fullsized":"2023-09-23T20:45:01.869841+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:59:25.045627+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"90'161","reported_seq":602,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.853468+0000","last_change":"2023-09-23T20:36:01.292103+0000","last_active":"2023-09-23T20:45:01.853468+0000","last_peered":"2023-09-23T20:45:01.853468+0000","last_clean":"2023-09-23T20:45:01.853468+0000","last_became_active":"2023-09-23T20:35:26.639912+0000","last_became_peered":"2023-09-23T20:35:26.639912+0000","last_unstale":"2023-09-23T20:45:01.853468+0000","last_undegraded":"2023-09-23T20:45:01.853468+0000","last_fullsized":"2023-09-23T20:45:01.853468+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:01.292032+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:01.292032+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:04:53.926640+0000","scrub_duration":0.0032983159999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":318,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.863818+0000","last_change":"2023-09-23T20:37:43.584414+0000","last_active":"2023-09-23T20:45:01.863818+0000","last_peered":"2023-09-23T20:45:01.863818+0000","last_clean":"2023-09-23T20:45:01.863818+0000","last_became_active":"2023-09-23T20:37:43.584116+0000","last_became_peered":"2023-09-23T20:37:43.584116+0000","last_unstale":"2023-09-23T20:45:01.863818+0000","last_undegraded":"2023-09-23T20:45:01.863818+0000","last_fullsized":"2023-09-23T20:45:01.863818+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:17:54.258172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":138,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.874185+0000","last_change":"2023-09-23T20:38:37.457427+0000","last_active":"2023-09-23T20:45:01.874185+0000","last_peered":"2023-09-23T20:45:01.874185+0000","last_clean":"2023-09-23T20:45:01.874185+0000","last_became_active":"2023-09-23T20:38:37.457099+0000","last_became_peered":"2023-09-23T20:38:37.457099+0000","last_unstale":"2023-09-23T20:45:01.874185+0000","last_undegraded":"2023-09-23T20:45:01.874185+0000","last_fullsized":"2023-09-23T20:45:01.874185+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:31:27.084994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"90'161","reported_seq":735,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.862330+0000","last_change":"2023-09-23T20:40:22.449754+0000","last_active":"2023-09-23T20:45:01.862330+0000","last_peered":"2023-09-23T20:45:01.862330+0000","last_clean":"2023-09-23T20:45:01.862330+0000","last_became_active":"2023-09-23T20:40:22.448838+0000","last_became_peered":"2023-09-23T20:40:22.448838+0000","last_unstale":"2023-09-23T20:45:01.862330+0000","last_undegraded":"2023-09-23T20:45:01.862330+0000","last_fullsized":"2023-09-23T20:45:01.862330+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:03.411025+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:03.411025+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:22:07.948856+0000","scrub_duration":0.001802922,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":513,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.870254+0000","last_change":"2023-09-23T20:43:46.522325+0000","last_active":"2023-09-23T20:45:01.870254+0000","last_peered":"2023-09-23T20:45:01.870254+0000","last_clean":"2023-09-23T20:45:01.870254+0000","last_became_active":"2023-09-23T20:43:46.521996+0000","last_became_peered":"2023-09-23T20:43:46.521996+0000","last_unstale":"2023-09-23T20:45:01.870254+0000","last_undegraded":"2023-09-23T20:45:01.870254+0000","last_fullsized":"2023-09-23T20:45:01.870254+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:51:17.629956+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":248,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.864368+0000","last_change":"2023-09-23T20:38:02.579343+0000","last_active":"2023-09-23T20:45:01.864368+0000","last_peered":"2023-09-23T20:45:01.864368+0000","last_clean":"2023-09-23T20:45:01.864368+0000","last_became_active":"2023-09-23T20:38:02.579047+0000","last_became_peered":"2023-09-23T20:38:02.579047+0000","last_unstale":"2023-09-23T20:45:01.864368+0000","last_undegraded":"2023-09-23T20:45:01.864368+0000","last_fullsized":"2023-09-23T20:45:01.864368+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:44:03.166556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.14","version":"90'161","reported_seq":854,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.864613+0000","last_change":"2023-09-23T20:35:02.236398+0000","last_active":"2023-09-23T20:45:01.864613+0000","last_peered":"2023-09-23T20:45:01.864613+0000","last_clean":"2023-09-23T20:45:01.864613+0000","last_became_active":"2023-09-23T20:35:02.235988+0000","last_became_peered":"2023-09-23T20:35:02.235988+0000","last_unstale":"2023-09-23T20:45:01.864613+0000","last_undegraded":"2023-09-23T20:45:01.864613+0000","last_fullsized":"2023-09-23T20:45:01.864613+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:47.479012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:47.479012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:13.519648+0000","scrub_duration":0.0020136590000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":359,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.861964+0000","last_change":"2023-09-23T20:43:16.780478+0000","last_active":"2023-09-23T20:45:01.861964+0000","last_peered":"2023-09-23T20:45:01.861964+0000","last_clean":"2023-09-23T20:45:01.861964+0000","last_became_active":"2023-09-23T20:43:16.780183+0000","last_became_peered":"2023-09-23T20:43:16.780183+0000","last_unstale":"2023-09-23T20:45:01.861964+0000","last_undegraded":"2023-09-23T20:45:01.861964+0000","last_fullsized":"2023-09-23T20:45:01.861964+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:38:30.102066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":349,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.856672+0000","last_change":"2023-09-23T20:38:02.542034+0000","last_active":"2023-09-23T20:45:01.856672+0000","last_peered":"2023-09-23T20:45:01.856672+0000","last_clean":"2023-09-23T20:45:01.856672+0000","last_became_active":"2023-09-23T20:38:02.541575+0000","last_became_peered":"2023-09-23T20:38:02.541575+0000","last_unstale":"2023-09-23T20:45:01.856672+0000","last_undegraded":"2023-09-23T20:45:01.856672+0000","last_fullsized":"2023-09-23T20:45:01.856672+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:50:54.591529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"90'161","reported_seq":648,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.870056+0000","last_change":"2023-09-23T20:44:14.669973+0000","last_active":"2023-09-23T20:45:01.870056+0000","last_peered":"2023-09-23T20:45:01.870056+0000","last_clean":"2023-09-23T20:45:01.870056+0000","last_became_active":"2023-09-23T20:43:46.519592+0000","last_became_peered":"2023-09-23T20:43:46.519592+0000","last_unstale":"2023-09-23T20:45:01.870056+0000","last_undegraded":"2023-09-23T20:45:01.870056+0000","last_fullsized":"2023-09-23T20:45:01.870056+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:14.669831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:14.669831+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:56:28.076617+0000","scrub_duration":0.0083441460000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":550,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.864808+0000","last_change":"2023-09-23T20:32:33.270617+0000","last_active":"2023-09-23T20:45:01.864808+0000","last_peered":"2023-09-23T20:45:01.864808+0000","last_clean":"2023-09-23T20:45:01.864808+0000","last_became_active":"2023-09-23T20:32:33.227902+0000","last_became_peered":"2023-09-23T20:32:33.227902+0000","last_unstale":"2023-09-23T20:45:01.864808+0000","last_undegraded":"2023-09-23T20:45:01.864808+0000","last_fullsized":"2023-09-23T20:45:01.864808+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:30:26.847831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":360,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":37,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.860211+0000","last_change":"2023-09-23T20:44:10.611381+0000","last_active":"2023-09-23T20:45:01.860211+0000","last_peered":"2023-09-23T20:45:01.860211+0000","last_clean":"2023-09-23T20:45:01.860211+0000","last_became_active":"2023-09-23T20:44:10.611091+0000","last_became_peered":"2023-09-23T20:44:10.611091+0000","last_unstale":"2023-09-23T20:45:01.860211+0000","last_undegraded":"2023-09-23T20:45:01.860211+0000","last_fullsized":"2023-09-23T20:45:01.860211+0000","mapping_epoch":496,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":497,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:00:30.162478+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"90'161","reported_seq":649,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-09-23T20:45:07.931294+0000","last_change":"2023-09-23T20:36:02.548387+0000","last_active":"2023-09-23T20:45:07.931294+0000","last_peered":"2023-09-23T20:45:07.931294+0000","last_clean":"2023-09-23T20:45:07.931294+0000","last_became_active":"2023-09-23T20:35:26.644787+0000","last_became_peered":"2023-09-23T20:35:26.644787+0000","last_unstale":"2023-09-23T20:45:07.931294+0000","last_undegraded":"2023-09-23T20:45:07.931294+0000","last_fullsized":"2023-09-23T20:45:07.931294+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:02.548285+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:02.548285+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:37:15.216553+0000","scrub_duration":0.0015514459999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":270,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.868058+0000","last_change":"2023-09-23T20:38:02.560646+0000","last_active":"2023-09-23T20:45:01.868058+0000","last_peered":"2023-09-23T20:45:01.868058+0000","last_clean":"2023-09-23T20:45:01.868058+0000","last_became_active":"2023-09-23T20:38:02.539814+0000","last_became_peered":"2023-09-23T20:38:02.539814+0000","last_unstale":"2023-09-23T20:45:01.868058+0000","last_undegraded":"2023-09-23T20:45:01.868058+0000","last_fullsized":"2023-09-23T20:45:01.868058+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:00:09.435023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":250,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.854485+0000","last_change":"2023-09-23T20:38:02.579222+0000","last_active":"2023-09-23T20:45:01.854485+0000","last_peered":"2023-09-23T20:45:01.854485+0000","last_clean":"2023-09-23T20:45:01.854485+0000","last_became_active":"2023-09-23T20:38:02.578813+0000","last_became_peered":"2023-09-23T20:38:02.578813+0000","last_unstale":"2023-09-23T20:45:01.854485+0000","last_undegraded":"2023-09-23T20:45:01.854485+0000","last_fullsized":"2023-09-23T20:45:01.854485+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:57:31.080027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"90'161","reported_seq":776,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.872703+0000","last_change":"2023-09-23T20:43:46.544727+0000","last_active":"2023-09-23T20:45:01.872703+0000","last_peered":"2023-09-23T20:45:01.872703+0000","last_clean":"2023-09-23T20:45:01.872703+0000","last_became_active":"2023-09-23T20:43:46.543952+0000","last_became_peered":"2023-09-23T20:43:46.543952+0000","last_unstale":"2023-09-23T20:45:01.872703+0000","last_undegraded":"2023-09-23T20:45:01.872703+0000","last_fullsized":"2023-09-23T20:45:01.872703+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:43.578666+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:43.578666+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:11:16.611920+0000","scrub_duration":0.0021890379999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":549,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.861928+0000","last_change":"2023-09-23T20:32:33.242943+0000","last_active":"2023-09-23T20:45:01.861928+0000","last_peered":"2023-09-23T20:45:01.861928+0000","last_clean":"2023-09-23T20:45:01.861928+0000","last_became_active":"2023-09-23T20:32:33.225468+0000","last_became_peered":"2023-09-23T20:32:33.225468+0000","last_unstale":"2023-09-23T20:45:01.861928+0000","last_undegraded":"2023-09-23T20:45:01.861928+0000","last_fullsized":"2023-09-23T20:45:01.861928+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:40:47.867573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":269,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.877123+0000","last_change":"2023-09-23T20:45:01.875931+0000","last_active":"2023-09-23T20:45:01.877123+0000","last_peered":"2023-09-23T20:45:01.877123+0000","last_clean":"2023-09-23T20:45:01.877123+0000","last_became_active":"2023-09-23T20:45:01.872839+0000","last_became_peered":"2023-09-23T20:45:01.872839+0000","last_unstale":"2023-09-23T20:45:01.877123+0000","last_undegraded":"2023-09-23T20:45:01.877123+0000","last_fullsized":"2023-09-23T20:45:01.877123+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:15:09.584507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"90'161","reported_seq":803,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.861461+0000","last_change":"2023-09-23T20:44:20.041766+0000","last_active":"2023-09-23T20:45:01.861461+0000","last_peered":"2023-09-23T20:45:01.861461+0000","last_clean":"2023-09-23T20:45:01.861461+0000","last_became_active":"2023-09-23T20:43:16.765015+0000","last_became_peered":"2023-09-23T20:43:16.765015+0000","last_unstale":"2023-09-23T20:45:01.861461+0000","last_undegraded":"2023-09-23T20:45:01.861461+0000","last_fullsized":"2023-09-23T20:45:01.861461+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:20.041690+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:20.041690+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:33:30.150109+0000","scrub_duration":0.0026180330000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":478,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.869359+0000","last_change":"2023-09-23T20:43:46.521294+0000","last_active":"2023-09-23T20:45:01.869359+0000","last_peered":"2023-09-23T20:45:01.869359+0000","last_clean":"2023-09-23T20:45:01.869359+0000","last_became_active":"2023-09-23T20:43:46.521001+0000","last_became_peered":"2023-09-23T20:43:46.521001+0000","last_unstale":"2023-09-23T20:45:01.869359+0000","last_undegraded":"2023-09-23T20:45:01.869359+0000","last_fullsized":"2023-09-23T20:45:01.869359+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:01:35.126716+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":290,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.864922+0000","last_change":"2023-09-23T20:44:01.841565+0000","last_active":"2023-09-23T20:45:01.864922+0000","last_peered":"2023-09-23T20:45:01.864922+0000","last_clean":"2023-09-23T20:45:01.864922+0000","last_became_active":"2023-09-23T20:44:01.841255+0000","last_became_peered":"2023-09-23T20:44:01.841255+0000","last_unstale":"2023-09-23T20:45:01.864922+0000","last_undegraded":"2023-09-23T20:45:01.864922+0000","last_fullsized":"2023-09-23T20:45:01.864922+0000","mapping_epoch":491,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":492,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:18:44.817589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.13","version":"90'161","reported_seq":810,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.860176+0000","last_change":"2023-09-23T20:35:02.238923+0000","last_active":"2023-09-23T20:45:01.860176+0000","last_peered":"2023-09-23T20:45:01.860176+0000","last_clean":"2023-09-23T20:45:01.860176+0000","last_became_active":"2023-09-23T20:35:02.238293+0000","last_became_peered":"2023-09-23T20:35:02.238293+0000","last_unstale":"2023-09-23T20:45:01.860176+0000","last_undegraded":"2023-09-23T20:45:01.860176+0000","last_fullsized":"2023-09-23T20:45:01.860176+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:42.544308+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:42.544308+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:51:22.281932+0000","scrub_duration":0.0029958179999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":526,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.861681+0000","last_change":"2023-09-23T20:38:02.539774+0000","last_active":"2023-09-23T20:45:01.861681+0000","last_peered":"2023-09-23T20:45:01.861681+0000","last_clean":"2023-09-23T20:45:01.861681+0000","last_became_active":"2023-09-23T20:38:02.539436+0000","last_became_peered":"2023-09-23T20:38:02.539436+0000","last_unstale":"2023-09-23T20:45:01.861681+0000","last_undegraded":"2023-09-23T20:45:01.861681+0000","last_fullsized":"2023-09-23T20:45:01.861681+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:50:57.375803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":487,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.869906+0000","last_change":"2023-09-23T20:43:46.521735+0000","last_active":"2023-09-23T20:45:01.869906+0000","last_peered":"2023-09-23T20:45:01.869906+0000","last_clean":"2023-09-23T20:45:01.869906+0000","last_became_active":"2023-09-23T20:43:46.520762+0000","last_became_peered":"2023-09-23T20:43:46.520762+0000","last_unstale":"2023-09-23T20:45:01.869906+0000","last_undegraded":"2023-09-23T20:45:01.869906+0000","last_fullsized":"2023-09-23T20:45:01.869906+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:26:48.841930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"90'161","reported_seq":606,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.860851+0000","last_change":"2023-09-23T20:44:18.952808+0000","last_active":"2023-09-23T20:45:01.860851+0000","last_peered":"2023-09-23T20:45:01.860851+0000","last_clean":"2023-09-23T20:45:01.860851+0000","last_became_active":"2023-09-23T20:43:16.758652+0000","last_became_peered":"2023-09-23T20:43:16.758652+0000","last_unstale":"2023-09-23T20:45:01.860851+0000","last_undegraded":"2023-09-23T20:45:01.860851+0000","last_fullsized":"2023-09-23T20:45:01.860851+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:18.952693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:18.952693+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:12:19.496465+0000","scrub_duration":0.001699365,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":198,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.873520+0000","last_change":"2023-09-23T20:44:43.644290+0000","last_active":"2023-09-23T20:45:01.873520+0000","last_peered":"2023-09-23T20:45:01.873520+0000","last_clean":"2023-09-23T20:45:01.873520+0000","last_became_active":"2023-09-23T20:44:43.638313+0000","last_became_peered":"2023-09-23T20:44:43.638313+0000","last_unstale":"2023-09-23T20:45:01.873520+0000","last_undegraded":"2023-09-23T20:45:01.873520+0000","last_fullsized":"2023-09-23T20:45:01.873520+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:32:52.892603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":318,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.859194+0000","last_change":"2023-09-23T20:37:43.571083+0000","last_active":"2023-09-23T20:45:01.859194+0000","last_peered":"2023-09-23T20:45:01.859194+0000","last_clean":"2023-09-23T20:45:01.859194+0000","last_became_active":"2023-09-23T20:37:43.570306+0000","last_became_peered":"2023-09-23T20:37:43.570306+0000","last_unstale":"2023-09-23T20:45:01.859194+0000","last_undegraded":"2023-09-23T20:45:01.859194+0000","last_fullsized":"2023-09-23T20:45:01.859194+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:42:02.396718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.5","version":"523'13784","reported_seq":22347,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-09-23T20:45:11.340932+0000","last_change":"2023-09-23T20:44:20.680609+0000","last_active":"2023-09-23T20:45:11.340932+0000","last_peered":"2023-09-23T20:45:11.340932+0000","last_clean":"2023-09-23T20:45:11.340932+0000","last_became_active":"2023-09-23T20:44:20.638636+0000","last_became_peered":"2023-09-23T20:44:20.638636+0000","last_unstale":"2023-09-23T20:45:11.340932+0000","last_undegraded":"2023-09-23T20:45:11.340932+0000","last_fullsized":"2023-09-23T20:45:11.340932+0000","mapping_epoch":504,"log_start":"468'6210","ondisk_log_start":"468'6210","created":455,"last_epoch_clean":505,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":7574,"log_dups_size":0,"ondisk_log_size":7574,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:11:11.154742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13238272,"num_objects":202,"num_object_clones":0,"num_object_copies":606,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":202,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13784,"num_write_kb":447552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4204,"num_bytes_recovered":275316736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.d","version":"90'161","reported_seq":815,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.864565+0000","last_change":"2023-09-23T20:34:59.496438+0000","last_active":"2023-09-23T20:45:01.864565+0000","last_peered":"2023-09-23T20:45:01.864565+0000","last_clean":"2023-09-23T20:45:01.864565+0000","last_became_active":"2023-09-23T20:33:38.970527+0000","last_became_peered":"2023-09-23T20:33:38.970527+0000","last_unstale":"2023-09-23T20:45:01.864565+0000","last_undegraded":"2023-09-23T20:45:01.864565+0000","last_fullsized":"2023-09-23T20:45:01.864565+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:34:59.496375+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:34:59.496375+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:46:18.244734+0000","scrub_duration":0.0017843189999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":526,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.862076+0000","last_change":"2023-09-23T20:32:33.283752+0000","last_active":"2023-09-23T20:45:01.862076+0000","last_peered":"2023-09-23T20:45:01.862076+0000","last_clean":"2023-09-23T20:45:01.862076+0000","last_became_active":"2023-09-23T20:32:33.230503+0000","last_became_peered":"2023-09-23T20:32:33.230503+0000","last_unstale":"2023-09-23T20:45:01.862076+0000","last_undegraded":"2023-09-23T20:45:01.862076+0000","last_fullsized":"2023-09-23T20:45:01.862076+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:29.084410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":337,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.861693+0000","last_change":"2023-09-23T20:43:16.765253+0000","last_active":"2023-09-23T20:45:01.861693+0000","last_peered":"2023-09-23T20:45:01.861693+0000","last_clean":"2023-09-23T20:45:01.861693+0000","last_became_active":"2023-09-23T20:43:16.764978+0000","last_became_peered":"2023-09-23T20:43:16.764978+0000","last_unstale":"2023-09-23T20:45:01.861693+0000","last_undegraded":"2023-09-23T20:45:01.861693+0000","last_fullsized":"2023-09-23T20:45:01.861693+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:07:23.539372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.4","version":"523'14212","reported_seq":14290,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-09-23T20:45:13.478496+0000","last_change":"2023-09-23T20:41:04.575200+0000","last_active":"2023-09-23T20:45:13.478496+0000","last_peered":"2023-09-23T20:45:13.478496+0000","last_clean":"2023-09-23T20:45:13.478496+0000","last_became_active":"2023-09-23T20:41:04.574824+0000","last_became_peered":"2023-09-23T20:41:04.574824+0000","last_unstale":"2023-09-23T20:45:13.478496+0000","last_undegraded":"2023-09-23T20:45:13.478496+0000","last_fullsized":"2023-09-23T20:45:13.478496+0000","mapping_epoch":455,"log_start":"479'8483","ondisk_log_start":"479'8483","created":455,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":5729,"log_dups_size":0,"ondisk_log_size":5729,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:50:46.714383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":6422528,"num_objects":98,"num_object_clones":0,"num_object_copies":294,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":98,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14212,"num_write_kb":457920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.c","version":"90'161","reported_seq":784,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.862950+0000","last_change":"2023-09-23T20:41:09.506481+0000","last_active":"2023-09-23T20:45:01.862950+0000","last_peered":"2023-09-23T20:45:01.862950+0000","last_clean":"2023-09-23T20:45:01.862950+0000","last_became_active":"2023-09-23T20:38:28.442229+0000","last_became_peered":"2023-09-23T20:38:28.442229+0000","last_unstale":"2023-09-23T20:45:01.862950+0000","last_undegraded":"2023-09-23T20:45:01.862950+0000","last_fullsized":"2023-09-23T20:45:01.862950+0000","mapping_epoch":411,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":412,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:41:09.506338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:41:09.506338+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:19:46.265011+0000","scrub_duration":0.01112002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":561,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.862045+0000","last_change":"2023-09-23T20:34:59.987981+0000","last_active":"2023-09-23T20:45:01.862045+0000","last_peered":"2023-09-23T20:45:01.862045+0000","last_clean":"2023-09-23T20:45:01.862045+0000","last_became_active":"2023-09-23T20:34:59.985368+0000","last_became_peered":"2023-09-23T20:34:59.985368+0000","last_unstale":"2023-09-23T20:45:01.862045+0000","last_undegraded":"2023-09-23T20:45:01.862045+0000","last_fullsized":"2023-09-23T20:45:01.862045+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:12:43.082493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":220,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.853755+0000","last_change":"2023-09-23T20:32:33.232142+0000","last_active":"2023-09-23T20:45:01.853755+0000","last_peered":"2023-09-23T20:45:01.853755+0000","last_clean":"2023-09-23T20:45:01.853755+0000","last_became_active":"2023-09-23T20:32:33.231683+0000","last_became_peered":"2023-09-23T20:32:33.231683+0000","last_unstale":"2023-09-23T20:45:01.853755+0000","last_undegraded":"2023-09-23T20:45:01.853755+0000","last_fullsized":"2023-09-23T20:45:01.853755+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:02:58.336348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"523'14036","reported_seq":14124,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-09-23T20:45:12.535209+0000","last_change":"2023-09-23T20:43:16.854027+0000","last_active":"2023-09-23T20:45:12.535209+0000","last_peered":"2023-09-23T20:45:12.535209+0000","last_clean":"2023-09-23T20:45:12.535209+0000","last_became_active":"2023-09-23T20:43:16.766133+0000","last_became_peered":"2023-09-23T20:43:16.766133+0000","last_unstale":"2023-09-23T20:45:12.535209+0000","last_undegraded":"2023-09-23T20:45:12.535209+0000","last_fullsized":"2023-09-23T20:45:12.535209+0000","mapping_epoch":481,"log_start":"478'8285","ondisk_log_start":"478'8285","created":455,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":5751,"log_dups_size":0,"ondisk_log_size":5751,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:42:32.213805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":9175040,"num_objects":140,"num_object_clones":0,"num_object_copies":420,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":140,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14036,"num_write_kb":453632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.f","version":"90'161","reported_seq":678,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.874398+0000","last_change":"2023-09-23T20:38:40.491902+0000","last_active":"2023-09-23T20:45:01.874398+0000","last_peered":"2023-09-23T20:45:01.874398+0000","last_clean":"2023-09-23T20:45:01.874398+0000","last_became_active":"2023-09-23T20:38:40.491563+0000","last_became_peered":"2023-09-23T20:38:40.491563+0000","last_unstale":"2023-09-23T20:45:01.874398+0000","last_undegraded":"2023-09-23T20:45:01.874398+0000","last_fullsized":"2023-09-23T20:45:01.874398+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:46.217489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:46.217489+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:39:25.008796+0000","scrub_duration":0.001456429,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":551,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.864652+0000","last_change":"2023-09-23T20:40:19.677655+0000","last_active":"2023-09-23T20:45:01.864652+0000","last_peered":"2023-09-23T20:45:01.864652+0000","last_clean":"2023-09-23T20:45:01.864652+0000","last_became_active":"2023-09-23T20:40:19.551189+0000","last_became_peered":"2023-09-23T20:40:19.551189+0000","last_unstale":"2023-09-23T20:45:01.864652+0000","last_undegraded":"2023-09-23T20:45:01.864652+0000","last_fullsized":"2023-09-23T20:45:01.864652+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:11:01.559576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":128,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.873594+0000","last_change":"2023-09-23T20:38:02.590613+0000","last_active":"2023-09-23T20:45:01.873594+0000","last_peered":"2023-09-23T20:45:01.873594+0000","last_clean":"2023-09-23T20:45:01.873594+0000","last_became_active":"2023-09-23T20:38:02.590284+0000","last_became_peered":"2023-09-23T20:38:02.590284+0000","last_unstale":"2023-09-23T20:45:01.873594+0000","last_undegraded":"2023-09-23T20:45:01.873594+0000","last_fullsized":"2023-09-23T20:45:01.873594+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:40:07.825054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"523'13971","reported_seq":14055,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-09-23T20:45:13.090594+0000","last_change":"2023-09-23T20:43:16.788249+0000","last_active":"2023-09-23T20:45:13.090594+0000","last_peered":"2023-09-23T20:45:13.090594+0000","last_clean":"2023-09-23T20:45:13.090594+0000","last_became_active":"2023-09-23T20:43:16.763555+0000","last_became_peered":"2023-09-23T20:43:16.763555+0000","last_unstale":"2023-09-23T20:45:13.090594+0000","last_undegraded":"2023-09-23T20:45:13.090594+0000","last_fullsized":"2023-09-23T20:45:13.090594+0000","mapping_epoch":481,"log_start":"464'4204","ondisk_log_start":"464'4204","created":455,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":9767,"log_dups_size":0,"ondisk_log_size":9767,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:19:57.856019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":9109504,"num_objects":139,"num_object_clones":0,"num_object_copies":417,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":139,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13971,"num_write_kb":451520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[2,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.e","version":"90'161","reported_seq":623,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.867508+0000","last_change":"2023-09-23T20:36:00.322453+0000","last_active":"2023-09-23T20:45:01.867508+0000","last_peered":"2023-09-23T20:45:01.867508+0000","last_clean":"2023-09-23T20:45:01.867508+0000","last_became_active":"2023-09-23T20:35:02.233536+0000","last_became_peered":"2023-09-23T20:35:02.233536+0000","last_unstale":"2023-09-23T20:45:01.867508+0000","last_undegraded":"2023-09-23T20:45:01.867508+0000","last_fullsized":"2023-09-23T20:45:01.867508+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:00.322381+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:00.322381+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:02:15.489069+0000","scrub_duration":0.0017321859999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":482,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.863283+0000","last_change":"2023-09-23T20:43:16.756437+0000","last_active":"2023-09-23T20:45:01.863283+0000","last_peered":"2023-09-23T20:45:01.863283+0000","last_clean":"2023-09-23T20:45:01.863283+0000","last_became_active":"2023-09-23T20:43:16.756193+0000","last_became_peered":"2023-09-23T20:43:16.756193+0000","last_unstale":"2023-09-23T20:45:01.863283+0000","last_undegraded":"2023-09-23T20:45:01.863283+0000","last_fullsized":"2023-09-23T20:45:01.863283+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:33:25.632781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":496,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.873323+0000","last_change":"2023-09-23T20:43:46.541524+0000","last_active":"2023-09-23T20:45:01.873323+0000","last_peered":"2023-09-23T20:45:01.873323+0000","last_clean":"2023-09-23T20:45:01.873323+0000","last_became_active":"2023-09-23T20:43:46.541051+0000","last_became_peered":"2023-09-23T20:43:46.541051+0000","last_unstale":"2023-09-23T20:45:01.873323+0000","last_undegraded":"2023-09-23T20:45:01.873323+0000","last_fullsized":"2023-09-23T20:45:01.873323+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:59:36.632592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"523'14051","reported_seq":14127,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-09-23T20:45:13.254840+0000","last_change":"2023-09-23T20:41:04.558473+0000","last_active":"2023-09-23T20:45:13.254840+0000","last_peered":"2023-09-23T20:45:13.254840+0000","last_clean":"2023-09-23T20:45:13.254840+0000","last_became_active":"2023-09-23T20:41:04.558173+0000","last_became_peered":"2023-09-23T20:41:04.558173+0000","last_unstale":"2023-09-23T20:45:13.254840+0000","last_undegraded":"2023-09-23T20:45:13.254840+0000","last_fullsized":"2023-09-23T20:45:13.254840+0000","mapping_epoch":455,"log_start":"470'7063","ondisk_log_start":"470'7063","created":455,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":6988,"log_dups_size":0,"ondisk_log_size":6988,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:17:53.184877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":7405568,"num_objects":113,"num_object_clones":0,"num_object_copies":339,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":113,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14051,"num_write_kb":453248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":928,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.861615+0000","last_change":"2023-09-23T20:35:02.523308+0000","last_active":"2023-09-23T20:45:01.861615+0000","last_peered":"2023-09-23T20:45:01.861615+0000","last_clean":"2023-09-23T20:45:01.861615+0000","last_became_active":"2023-09-23T20:33:38.979989+0000","last_became_peered":"2023-09-23T20:33:38.979989+0000","last_unstale":"2023-09-23T20:45:01.861615+0000","last_undegraded":"2023-09-23T20:45:01.861615+0000","last_fullsized":"2023-09-23T20:45:01.861615+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:35:02.523203+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:35:02.523203+0000","last_clean_scrub_stamp":"2023-09-23T20:35:02.523203+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:02:41.472958+0000","scrub_duration":0.0085927989999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":344,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.857632+0000","last_change":"2023-09-23T20:37:43.583731+0000","last_active":"2023-09-23T20:45:01.857632+0000","last_peered":"2023-09-23T20:45:01.857632+0000","last_clean":"2023-09-23T20:45:01.857632+0000","last_became_active":"2023-09-23T20:37:43.583066+0000","last_became_peered":"2023-09-23T20:37:43.583066+0000","last_unstale":"2023-09-23T20:45:01.857632+0000","last_undegraded":"2023-09-23T20:45:01.857632+0000","last_fullsized":"2023-09-23T20:45:01.857632+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:03:45.211900+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":240,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.868003+0000","last_change":"2023-09-23T20:38:02.579814+0000","last_active":"2023-09-23T20:45:01.868003+0000","last_peered":"2023-09-23T20:45:01.868003+0000","last_clean":"2023-09-23T20:45:01.868003+0000","last_became_active":"2023-09-23T20:38:02.579253+0000","last_became_peered":"2023-09-23T20:38:02.579253+0000","last_unstale":"2023-09-23T20:45:01.868003+0000","last_undegraded":"2023-09-23T20:45:01.868003+0000","last_fullsized":"2023-09-23T20:45:01.868003+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:05:07.298198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"523'14162","reported_seq":14241,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-09-23T20:45:13.451080+0000","last_change":"2023-09-23T20:41:04.573101+0000","last_active":"2023-09-23T20:45:13.451080+0000","last_peered":"2023-09-23T20:45:13.451080+0000","last_clean":"2023-09-23T20:45:13.451080+0000","last_became_active":"2023-09-23T20:41:04.572728+0000","last_became_peered":"2023-09-23T20:41:04.572728+0000","last_unstale":"2023-09-23T20:45:13.451080+0000","last_undegraded":"2023-09-23T20:45:13.451080+0000","last_fullsized":"2023-09-23T20:45:13.451080+0000","mapping_epoch":455,"log_start":"479'8425","ondisk_log_start":"479'8425","created":455,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":5737,"log_dups_size":0,"ondisk_log_size":5737,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:21:10.721593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":6684672,"num_objects":102,"num_object_clones":0,"num_object_copies":306,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":102,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14162,"num_write_kb":456448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":638,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.864522+0000","last_change":"2023-09-23T20:44:46.725779+0000","last_active":"2023-09-23T20:45:01.864522+0000","last_peered":"2023-09-23T20:45:01.864522+0000","last_clean":"2023-09-23T20:45:01.864522+0000","last_became_active":"2023-09-23T20:44:46.725433+0000","last_became_peered":"2023-09-23T20:44:46.725433+0000","last_unstale":"2023-09-23T20:45:01.864522+0000","last_undegraded":"2023-09-23T20:45:01.864522+0000","last_fullsized":"2023-09-23T20:45:01.864522+0000","mapping_epoch":516,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":517,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:42:45.492220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:42:45.492220+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:28:35.202407+0000","scrub_duration":0.015259412,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":128,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.873641+0000","last_change":"2023-09-23T20:38:02.578703+0000","last_active":"2023-09-23T20:45:01.873641+0000","last_peered":"2023-09-23T20:45:01.873641+0000","last_clean":"2023-09-23T20:45:01.873641+0000","last_became_active":"2023-09-23T20:38:02.578414+0000","last_became_peered":"2023-09-23T20:38:02.578414+0000","last_unstale":"2023-09-23T20:45:01.873641+0000","last_undegraded":"2023-09-23T20:45:01.873641+0000","last_fullsized":"2023-09-23T20:45:01.873641+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:43:59.779295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":189,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.854604+0000","last_change":"2023-09-23T20:37:31.296606+0000","last_active":"2023-09-23T20:45:01.854604+0000","last_peered":"2023-09-23T20:45:01.854604+0000","last_clean":"2023-09-23T20:45:01.854604+0000","last_became_active":"2023-09-23T20:37:31.295924+0000","last_became_peered":"2023-09-23T20:37:31.295924+0000","last_unstale":"2023-09-23T20:45:01.854604+0000","last_undegraded":"2023-09-23T20:45:01.854604+0000","last_fullsized":"2023-09-23T20:45:01.854604+0000","mapping_epoch":391,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":392,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:27:50.948510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"523'14058","reported_seq":14134,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-09-23T20:45:12.554368+0000","last_change":"2023-09-23T20:41:04.555980+0000","last_active":"2023-09-23T20:45:12.554368+0000","last_peered":"2023-09-23T20:45:12.554368+0000","last_clean":"2023-09-23T20:45:12.554368+0000","last_became_active":"2023-09-23T20:41:04.555688+0000","last_became_peered":"2023-09-23T20:41:04.555688+0000","last_unstale":"2023-09-23T20:45:12.554368+0000","last_undegraded":"2023-09-23T20:45:12.554368+0000","last_fullsized":"2023-09-23T20:45:12.554368+0000","mapping_epoch":455,"log_start":"478'8325","ondisk_log_start":"478'8325","created":455,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":5733,"log_dups_size":0,"ondisk_log_size":5733,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:28:42.819404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":10616832,"num_objects":162,"num_object_clones":0,"num_object_copies":486,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":162,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14058,"num_write_kb":455040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.b","version":"90'161","reported_seq":826,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.860393+0000","last_change":"2023-09-23T20:40:22.445808+0000","last_active":"2023-09-23T20:45:01.860393+0000","last_peered":"2023-09-23T20:45:01.860393+0000","last_clean":"2023-09-23T20:45:01.860393+0000","last_became_active":"2023-09-23T20:40:22.445456+0000","last_became_peered":"2023-09-23T20:40:22.445456+0000","last_unstale":"2023-09-23T20:45:01.860393+0000","last_undegraded":"2023-09-23T20:45:01.860393+0000","last_fullsized":"2023-09-23T20:45:01.860393+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:05.595042+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:05.595042+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:39:23.678154+0000","scrub_duration":0.007271675,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":179,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.853533+0000","last_change":"2023-09-23T20:34:59.983122+0000","last_active":"2023-09-23T20:45:01.853533+0000","last_peered":"2023-09-23T20:45:01.853533+0000","last_clean":"2023-09-23T20:45:01.853533+0000","last_became_active":"2023-09-23T20:34:59.982809+0000","last_became_peered":"2023-09-23T20:34:59.982809+0000","last_unstale":"2023-09-23T20:45:01.853533+0000","last_undegraded":"2023-09-23T20:45:01.853533+0000","last_fullsized":"2023-09-23T20:45:01.853533+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:49:40.839723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":260,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.867603+0000","last_change":"2023-09-23T20:44:10.606995+0000","last_active":"2023-09-23T20:45:01.867603+0000","last_peered":"2023-09-23T20:45:01.867603+0000","last_clean":"2023-09-23T20:45:01.867603+0000","last_became_active":"2023-09-23T20:44:10.606718+0000","last_became_peered":"2023-09-23T20:44:10.606718+0000","last_unstale":"2023-09-23T20:45:01.867603+0000","last_undegraded":"2023-09-23T20:45:01.867603+0000","last_fullsized":"2023-09-23T20:45:01.867603+0000","mapping_epoch":496,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":497,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:13:12.496245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"523'13941","reported_seq":14019,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-09-23T20:45:09.377569+0000","last_change":"2023-09-23T20:41:04.573300+0000","last_active":"2023-09-23T20:45:09.377569+0000","last_peered":"2023-09-23T20:45:09.377569+0000","last_clean":"2023-09-23T20:45:09.377569+0000","last_became_active":"2023-09-23T20:41:04.572919+0000","last_became_peered":"2023-09-23T20:41:04.572919+0000","last_unstale":"2023-09-23T20:45:09.377569+0000","last_undegraded":"2023-09-23T20:45:09.377569+0000","last_fullsized":"2023-09-23T20:45:09.377569+0000","mapping_epoch":455,"log_start":"477'7894","ondisk_log_start":"477'7894","created":455,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":6047,"log_dups_size":0,"ondisk_log_size":6047,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:57:46.465243+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":24444928,"num_objects":373,"num_object_clones":0,"num_object_copies":1119,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":373,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13941,"num_write_kb":458048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":548,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.864326+0000","last_change":"2023-09-23T20:43:51.693618+0000","last_active":"2023-09-23T20:45:01.864326+0000","last_peered":"2023-09-23T20:45:01.864326+0000","last_clean":"2023-09-23T20:45:01.864326+0000","last_became_active":"2023-09-23T20:43:16.765186+0000","last_became_peered":"2023-09-23T20:43:16.765186+0000","last_unstale":"2023-09-23T20:45:01.864326+0000","last_undegraded":"2023-09-23T20:45:01.864326+0000","last_fullsized":"2023-09-23T20:45:01.864326+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:43:51.693459+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:43:51.693459+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:33:30.208355+0000","scrub_duration":0.0016145269999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":318,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.863818+0000","last_change":"2023-09-23T20:37:43.570610+0000","last_active":"2023-09-23T20:45:01.863818+0000","last_peered":"2023-09-23T20:45:01.863818+0000","last_clean":"2023-09-23T20:45:01.863818+0000","last_became_active":"2023-09-23T20:37:43.570216+0000","last_became_peered":"2023-09-23T20:37:43.570216+0000","last_unstale":"2023-09-23T20:45:01.863818+0000","last_undegraded":"2023-09-23T20:45:01.863818+0000","last_fullsized":"2023-09-23T20:45:01.863818+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:39:26.555329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":348,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.861040+0000","last_change":"2023-09-23T20:32:33.230582+0000","last_active":"2023-09-23T20:45:01.861040+0000","last_peered":"2023-09-23T20:45:01.861040+0000","last_clean":"2023-09-23T20:45:01.861040+0000","last_became_active":"2023-09-23T20:32:33.230257+0000","last_became_peered":"2023-09-23T20:32:33.230257+0000","last_unstale":"2023-09-23T20:45:01.861040+0000","last_undegraded":"2023-09-23T20:45:01.861040+0000","last_fullsized":"2023-09-23T20:45:01.861040+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:11:45.333096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.f","version":"523'14300","reported_seq":14384,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-09-23T20:45:13.492399+0000","last_change":"2023-09-23T20:43:16.778725+0000","last_active":"2023-09-23T20:45:13.492399+0000","last_peered":"2023-09-23T20:45:13.492399+0000","last_clean":"2023-09-23T20:45:13.492399+0000","last_became_active":"2023-09-23T20:43:16.759443+0000","last_became_peered":"2023-09-23T20:43:16.759443+0000","last_unstale":"2023-09-23T20:45:13.492399+0000","last_undegraded":"2023-09-23T20:45:13.492399+0000","last_fullsized":"2023-09-23T20:45:13.492399+0000","mapping_epoch":481,"log_start":"479'8584","ondisk_log_start":"479'8584","created":455,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":5716,"log_dups_size":0,"ondisk_log_size":5716,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:55:32.141361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":5636096,"num_objects":86,"num_object_clones":0,"num_object_copies":258,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":86,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14300,"num_write_kb":460352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":711,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.874371+0000","last_change":"2023-09-23T20:41:07.424609+0000","last_active":"2023-09-23T20:45:01.874371+0000","last_peered":"2023-09-23T20:45:01.874371+0000","last_clean":"2023-09-23T20:45:01.874371+0000","last_became_active":"2023-09-23T20:38:05.622261+0000","last_became_peered":"2023-09-23T20:38:05.622261+0000","last_unstale":"2023-09-23T20:45:01.874371+0000","last_undegraded":"2023-09-23T20:45:01.874371+0000","last_fullsized":"2023-09-23T20:45:01.874371+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:41:07.424492+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:41:07.424492+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:45:21.689123+0000","scrub_duration":0.0018960959999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"86'119","reported_seq":833,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.868557+0000","last_change":"2023-09-23T20:29:45.521257+0000","last_active":"2023-09-23T20:45:01.868557+0000","last_peered":"2023-09-23T20:45:01.868557+0000","last_clean":"2023-09-23T20:45:01.868557+0000","last_became_active":"2023-09-23T20:29:45.520546+0000","last_became_peered":"2023-09-23T20:29:45.520546+0000","last_unstale":"2023-09-23T20:45:01.868557+0000","last_undegraded":"2023-09-23T20:45:01.868557+0000","last_fullsized":"2023-09-23T20:45:01.868557+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":119,"log_dups_size":0,"ondisk_log_size":119,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:58:56.085777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032705499999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":523,"num_read_kb":3503,"num_write":136,"num_write_kb":3128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":315,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.857325+0000","last_change":"2023-09-23T20:43:16.756494+0000","last_active":"2023-09-23T20:45:01.857325+0000","last_peered":"2023-09-23T20:45:01.857325+0000","last_clean":"2023-09-23T20:45:01.857325+0000","last_became_active":"2023-09-23T20:43:16.756131+0000","last_became_peered":"2023-09-23T20:43:16.756131+0000","last_unstale":"2023-09-23T20:45:01.857325+0000","last_undegraded":"2023-09-23T20:45:01.857325+0000","last_fullsized":"2023-09-23T20:45:01.857325+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:10:19.219533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":480,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.860537+0000","last_change":"2023-09-23T20:40:19.530942+0000","last_active":"2023-09-23T20:45:01.860537+0000","last_peered":"2023-09-23T20:45:01.860537+0000","last_clean":"2023-09-23T20:45:01.860537+0000","last_became_active":"2023-09-23T20:40:19.530617+0000","last_became_peered":"2023-09-23T20:40:19.530617+0000","last_unstale":"2023-09-23T20:45:01.860537+0000","last_undegraded":"2023-09-23T20:45:01.860537+0000","last_fullsized":"2023-09-23T20:45:01.860537+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:24:09.231934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.9","version":"523'13987","reported_seq":14083,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-09-23T20:45:11.389469+0000","last_change":"2023-09-23T20:43:46.544365+0000","last_active":"2023-09-23T20:45:11.389469+0000","last_peered":"2023-09-23T20:45:11.389469+0000","last_clean":"2023-09-23T20:45:11.389469+0000","last_became_active":"2023-09-23T20:43:46.519248+0000","last_became_peered":"2023-09-23T20:43:46.519248+0000","last_unstale":"2023-09-23T20:45:11.389469+0000","last_undegraded":"2023-09-23T20:45:11.389469+0000","last_fullsized":"2023-09-23T20:45:11.389469+0000","mapping_epoch":487,"log_start":"468'6434","ondisk_log_start":"468'6434","created":455,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":7553,"log_dups_size":0,"ondisk_log_size":7553,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:49:43.541062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13434880,"num_objects":205,"num_object_clones":0,"num_object_copies":615,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":205,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13987,"num_write_kb":454144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":568,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.873045+0000","last_change":"2023-09-23T20:44:15.656984+0000","last_active":"2023-09-23T20:45:01.873045+0000","last_peered":"2023-09-23T20:45:01.873045+0000","last_clean":"2023-09-23T20:45:01.873045+0000","last_became_active":"2023-09-23T20:43:46.512210+0000","last_became_peered":"2023-09-23T20:43:46.512210+0000","last_unstale":"2023-09-23T20:45:01.873045+0000","last_undegraded":"2023-09-23T20:45:01.873045+0000","last_fullsized":"2023-09-23T20:45:01.873045+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:15.656880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:15.656880+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:37:56.453466+0000","scrub_duration":0.0016711180000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"86'106","reported_seq":773,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.860716+0000","last_change":"2023-09-23T20:43:16.759138+0000","last_active":"2023-09-23T20:45:01.860716+0000","last_peered":"2023-09-23T20:45:01.860716+0000","last_clean":"2023-09-23T20:45:01.860716+0000","last_became_active":"2023-09-23T20:43:16.758015+0000","last_became_peered":"2023-09-23T20:43:16.758015+0000","last_unstale":"2023-09-23T20:45:01.860716+0000","last_undegraded":"2023-09-23T20:45:01.860716+0000","last_fullsized":"2023-09-23T20:45:01.860716+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":106,"log_dups_size":0,"ondisk_log_size":106,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:08:02.852472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00099302599999999993,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1094,"num_read_kb":1678,"num_write":100,"num_write_kb":404,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":240,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.854368+0000","last_change":"2023-09-23T20:38:02.552008+0000","last_active":"2023-09-23T20:45:01.854368+0000","last_peered":"2023-09-23T20:45:01.854368+0000","last_clean":"2023-09-23T20:45:01.854368+0000","last_became_active":"2023-09-23T20:38:02.551684+0000","last_became_peered":"2023-09-23T20:38:02.551684+0000","last_unstale":"2023-09-23T20:45:01.854368+0000","last_undegraded":"2023-09-23T20:45:01.854368+0000","last_fullsized":"2023-09-23T20:45:01.854368+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:40:37.147538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":486,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.868482+0000","last_change":"2023-09-23T20:29:45.533406+0000","last_active":"2023-09-23T20:45:01.868482+0000","last_peered":"2023-09-23T20:45:01.868482+0000","last_clean":"2023-09-23T20:45:01.868482+0000","last_became_active":"2023-09-23T20:29:45.526001+0000","last_became_peered":"2023-09-23T20:29:45.526001+0000","last_unstale":"2023-09-23T20:45:01.868482+0000","last_undegraded":"2023-09-23T20:45:01.868482+0000","last_fullsized":"2023-09-23T20:45:01.868482+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:43.787251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.a","version":"523'14091","reported_seq":14178,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-09-23T20:45:13.542653+0000","last_change":"2023-09-23T20:43:16.852909+0000","last_active":"2023-09-23T20:45:13.542653+0000","last_peered":"2023-09-23T20:45:13.542653+0000","last_clean":"2023-09-23T20:45:13.542653+0000","last_became_active":"2023-09-23T20:43:16.770100+0000","last_became_peered":"2023-09-23T20:43:16.770100+0000","last_unstale":"2023-09-23T20:45:13.542653+0000","last_undegraded":"2023-09-23T20:45:13.542653+0000","last_fullsized":"2023-09-23T20:45:13.542653+0000","mapping_epoch":481,"log_start":"479'8406","ondisk_log_start":"479'8406","created":455,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":5685,"log_dups_size":0,"ondisk_log_size":5685,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:13:16.136666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":7143424,"num_objects":109,"num_object_clones":0,"num_object_copies":327,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":109,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14091,"num_write_kb":454400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,7],"acting":[1,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":486,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.861773+0000","last_change":"2023-09-23T20:44:26.978243+0000","last_active":"2023-09-23T20:45:01.861773+0000","last_peered":"2023-09-23T20:45:01.861773+0000","last_clean":"2023-09-23T20:45:01.861773+0000","last_became_active":"2023-09-23T20:44:18.638989+0000","last_became_peered":"2023-09-23T20:44:18.638989+0000","last_unstale":"2023-09-23T20:45:01.861773+0000","last_undegraded":"2023-09-23T20:45:01.861773+0000","last_fullsized":"2023-09-23T20:45:01.861773+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:26.978179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:26.978179+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:25:38.977011+0000","scrub_duration":0.0013962829999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"86'130","reported_seq":841,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.864189+0000","last_change":"2023-09-23T20:25:39.637458+0000","last_active":"2023-09-23T20:45:01.864189+0000","last_peered":"2023-09-23T20:45:01.864189+0000","last_clean":"2023-09-23T20:45:01.864189+0000","last_became_active":"2023-09-23T20:25:39.636773+0000","last_became_peered":"2023-09-23T20:25:39.636773+0000","last_unstale":"2023-09-23T20:45:01.864189+0000","last_undegraded":"2023-09-23T20:45:01.864189+0000","last_fullsized":"2023-09-23T20:45:01.864189+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:00:32.810564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011300049999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":512,"num_read_kb":1820,"num_write":133,"num_write_kb":1347,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":494,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.869703+0000","last_change":"2023-09-23T20:38:02.591335+0000","last_active":"2023-09-23T20:45:01.869703+0000","last_peered":"2023-09-23T20:45:01.869703+0000","last_clean":"2023-09-23T20:45:01.869703+0000","last_became_active":"2023-09-23T20:38:02.591015+0000","last_became_peered":"2023-09-23T20:38:02.591015+0000","last_unstale":"2023-09-23T20:45:01.869703+0000","last_undegraded":"2023-09-23T20:45:01.869703+0000","last_fullsized":"2023-09-23T20:45:01.869703+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:55:32.272195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":37,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.860891+0000","last_change":"2023-09-23T20:44:10.555583+0000","last_active":"2023-09-23T20:45:01.860891+0000","last_peered":"2023-09-23T20:45:01.860891+0000","last_clean":"2023-09-23T20:45:01.860891+0000","last_became_active":"2023-09-23T20:44:10.555281+0000","last_became_peered":"2023-09-23T20:44:10.555281+0000","last_unstale":"2023-09-23T20:45:01.860891+0000","last_undegraded":"2023-09-23T20:45:01.860891+0000","last_fullsized":"2023-09-23T20:45:01.860891+0000","mapping_epoch":496,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":497,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:43:11.942147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.8","version":"523'14141","reported_seq":31119,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-09-23T20:45:09.373646+0000","last_change":"2023-09-23T20:44:51.765604+0000","last_active":"2023-09-23T20:45:09.373646+0000","last_peered":"2023-09-23T20:45:09.373646+0000","last_clean":"2023-09-23T20:45:09.373646+0000","last_became_active":"2023-09-23T20:44:51.738188+0000","last_became_peered":"2023-09-23T20:44:51.738188+0000","last_unstale":"2023-09-23T20:45:09.373646+0000","last_undegraded":"2023-09-23T20:45:09.373646+0000","last_fullsized":"2023-09-23T20:45:09.373646+0000","mapping_epoch":518,"log_start":"478'8127","ondisk_log_start":"478'8127","created":455,"last_epoch_clean":519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":6014,"log_dups_size":0,"ondisk_log_size":6014,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:58:46.821782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":23527424,"num_objects":359,"num_object_clones":0,"num_object_copies":1077,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":359,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14141,"num_write_kb":464000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5858,"num_bytes_recovered":383844352,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"86'194","reported_seq":879,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.864645+0000","last_change":"2023-09-23T20:43:16.765865+0000","last_active":"2023-09-23T20:45:01.864645+0000","last_peered":"2023-09-23T20:45:01.864645+0000","last_clean":"2023-09-23T20:45:01.864645+0000","last_became_active":"2023-09-23T20:43:16.765289+0000","last_became_peered":"2023-09-23T20:43:16.765289+0000","last_unstale":"2023-09-23T20:45:01.864645+0000","last_undegraded":"2023-09-23T20:45:01.864645+0000","last_fullsized":"2023-09-23T20:45:01.864645+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":194,"log_dups_size":0,"ondisk_log_size":194,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:15:16.461900+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00037995999999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1024,"num_read_kb":1200,"num_write":173,"num_write_kb":1276,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":985,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.861606+0000","last_change":"2023-09-23T20:38:05.602508+0000","last_active":"2023-09-23T20:45:01.861606+0000","last_peered":"2023-09-23T20:45:01.861606+0000","last_clean":"2023-09-23T20:45:01.861606+0000","last_became_active":"2023-09-23T20:38:05.602129+0000","last_became_peered":"2023-09-23T20:38:05.602129+0000","last_unstale":"2023-09-23T20:45:01.861606+0000","last_undegraded":"2023-09-23T20:45:01.861606+0000","last_fullsized":"2023-09-23T20:45:01.861606+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:45.343742+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:45.343742+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:48.910343+0000","scrub_duration":0.001475802,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":98,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.873681+0000","last_change":"2023-09-23T20:43:16.758213+0000","last_active":"2023-09-23T20:45:01.873681+0000","last_peered":"2023-09-23T20:45:01.873681+0000","last_clean":"2023-09-23T20:45:01.873681+0000","last_became_active":"2023-09-23T20:43:16.757869+0000","last_became_peered":"2023-09-23T20:43:16.757869+0000","last_unstale":"2023-09-23T20:45:01.873681+0000","last_undegraded":"2023-09-23T20:45:01.873681+0000","last_fullsized":"2023-09-23T20:45:01.873681+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:43:10.811749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":128,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.874089+0000","last_change":"2023-09-23T20:38:02.573200+0000","last_active":"2023-09-23T20:45:01.874089+0000","last_peered":"2023-09-23T20:45:01.874089+0000","last_clean":"2023-09-23T20:45:01.874089+0000","last_became_active":"2023-09-23T20:38:02.572945+0000","last_became_peered":"2023-09-23T20:38:02.572945+0000","last_unstale":"2023-09-23T20:45:01.874089+0000","last_undegraded":"2023-09-23T20:45:01.874089+0000","last_fullsized":"2023-09-23T20:45:01.874089+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:57.246533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.b","version":"523'13842","reported_seq":13936,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-09-23T20:45:11.374361+0000","last_change":"2023-09-23T20:43:46.540991+0000","last_active":"2023-09-23T20:45:11.374361+0000","last_peered":"2023-09-23T20:45:11.374361+0000","last_clean":"2023-09-23T20:45:11.374361+0000","last_became_active":"2023-09-23T20:43:46.512772+0000","last_became_peered":"2023-09-23T20:43:46.512772+0000","last_unstale":"2023-09-23T20:45:11.374361+0000","last_undegraded":"2023-09-23T20:45:11.374361+0000","last_fullsized":"2023-09-23T20:45:11.374361+0000","mapping_epoch":487,"log_start":"468'6334","ondisk_log_start":"468'6334","created":455,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":7508,"log_dups_size":0,"ondisk_log_size":7508,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:46:02.578833+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":15859712,"num_objects":242,"num_object_clones":0,"num_object_copies":726,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":242,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13842,"num_write_kb":450688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":855,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.860097+0000","last_change":"2023-09-23T20:38:05.615711+0000","last_active":"2023-09-23T20:45:01.860097+0000","last_peered":"2023-09-23T20:45:01.860097+0000","last_clean":"2023-09-23T20:45:01.860097+0000","last_became_active":"2023-09-23T20:38:05.615390+0000","last_became_peered":"2023-09-23T20:38:05.615390+0000","last_unstale":"2023-09-23T20:45:01.860097+0000","last_undegraded":"2023-09-23T20:45:01.860097+0000","last_fullsized":"2023-09-23T20:45:01.860097+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:43.582146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:43.582146+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:45:11.917871+0000","scrub_duration":0.0016219730000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"86'80","reported_seq":775,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.861507+0000","last_change":"2023-09-23T20:29:45.536796+0000","last_active":"2023-09-23T20:45:01.861507+0000","last_peered":"2023-09-23T20:45:01.861507+0000","last_clean":"2023-09-23T20:45:01.861507+0000","last_became_active":"2023-09-23T20:29:45.536439+0000","last_became_peered":"2023-09-23T20:29:45.536439+0000","last_unstale":"2023-09-23T20:45:01.861507+0000","last_undegraded":"2023-09-23T20:45:01.861507+0000","last_fullsized":"2023-09-23T20:45:01.861507+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":80,"log_dups_size":0,"ondisk_log_size":80,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:30:57.793132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035115,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":395,"num_read_kb":1370,"num_write":66,"num_write_kb":1061,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":515,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.872890+0000","last_change":"2023-09-23T20:43:46.544563+0000","last_active":"2023-09-23T20:45:01.872890+0000","last_peered":"2023-09-23T20:45:01.872890+0000","last_clean":"2023-09-23T20:45:01.872890+0000","last_became_active":"2023-09-23T20:43:46.543958+0000","last_became_peered":"2023-09-23T20:43:46.543958+0000","last_unstale":"2023-09-23T20:45:01.872890+0000","last_undegraded":"2023-09-23T20:45:01.872890+0000","last_fullsized":"2023-09-23T20:45:01.872890+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:46:36.921752+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":510,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.869675+0000","last_change":"2023-09-23T20:43:46.520386+0000","last_active":"2023-09-23T20:45:01.869675+0000","last_peered":"2023-09-23T20:45:01.869675+0000","last_clean":"2023-09-23T20:45:01.869675+0000","last_became_active":"2023-09-23T20:43:46.519757+0000","last_became_peered":"2023-09-23T20:43:46.519757+0000","last_unstale":"2023-09-23T20:45:01.869675+0000","last_undegraded":"2023-09-23T20:45:01.869675+0000","last_fullsized":"2023-09-23T20:45:01.869675+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:23:14.662734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.e","version":"523'14084","reported_seq":14160,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-09-23T20:45:12.565331+0000","last_change":"2023-09-23T20:41:04.574876+0000","last_active":"2023-09-23T20:45:12.565331+0000","last_peered":"2023-09-23T20:45:12.565331+0000","last_clean":"2023-09-23T20:45:12.565331+0000","last_became_active":"2023-09-23T20:41:04.574595+0000","last_became_peered":"2023-09-23T20:41:04.574595+0000","last_unstale":"2023-09-23T20:45:12.565331+0000","last_undegraded":"2023-09-23T20:45:12.565331+0000","last_fullsized":"2023-09-23T20:45:12.565331+0000","mapping_epoch":455,"log_start":"479'8408","ondisk_log_start":"479'8408","created":455,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":5676,"log_dups_size":0,"ondisk_log_size":5676,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:31:00.147966+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8912896,"num_objects":136,"num_object_clones":0,"num_object_copies":408,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":136,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14084,"num_write_kb":455040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":634,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.874124+0000","last_change":"2023-09-23T20:41:08.380084+0000","last_active":"2023-09-23T20:45:01.874124+0000","last_peered":"2023-09-23T20:45:01.874124+0000","last_clean":"2023-09-23T20:45:01.874124+0000","last_became_active":"2023-09-23T20:38:05.610881+0000","last_became_peered":"2023-09-23T20:38:05.610881+0000","last_unstale":"2023-09-23T20:45:01.874124+0000","last_undegraded":"2023-09-23T20:45:01.874124+0000","last_fullsized":"2023-09-23T20:45:01.874124+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:41:08.379959+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:41:08.379959+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:19:39.547923+0000","scrub_duration":0.0019158420000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"86'102","reported_seq":1117,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.864729+0000","last_change":"2023-09-23T20:32:36.314672+0000","last_active":"2023-09-23T20:45:01.864729+0000","last_peered":"2023-09-23T20:45:01.864729+0000","last_clean":"2023-09-23T20:45:01.864729+0000","last_became_active":"2023-09-23T20:32:36.314231+0000","last_became_peered":"2023-09-23T20:32:36.314231+0000","last_unstale":"2023-09-23T20:45:01.864729+0000","last_undegraded":"2023-09-23T20:45:01.864729+0000","last_fullsized":"2023-09-23T20:45:01.864729+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":102,"log_dups_size":0,"ondisk_log_size":102,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:11:40.514709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030656500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":963,"num_read_kb":1234,"num_write":60,"num_write_kb":225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":470,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.868631+0000","last_change":"2023-09-23T20:40:19.544384+0000","last_active":"2023-09-23T20:45:01.868631+0000","last_peered":"2023-09-23T20:45:01.868631+0000","last_clean":"2023-09-23T20:45:01.868631+0000","last_became_active":"2023-09-23T20:40:19.544079+0000","last_became_peered":"2023-09-23T20:40:19.544079+0000","last_unstale":"2023-09-23T20:45:01.868631+0000","last_undegraded":"2023-09-23T20:45:01.868631+0000","last_fullsized":"2023-09-23T20:45:01.868631+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:00:19.498631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":465,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.873537+0000","last_change":"2023-09-23T20:43:46.541173+0000","last_active":"2023-09-23T20:45:01.873537+0000","last_peered":"2023-09-23T20:45:01.873537+0000","last_clean":"2023-09-23T20:45:01.873537+0000","last_became_active":"2023-09-23T20:43:46.540726+0000","last_became_peered":"2023-09-23T20:43:46.540726+0000","last_unstale":"2023-09-23T20:45:01.873537+0000","last_undegraded":"2023-09-23T20:45:01.873537+0000","last_fullsized":"2023-09-23T20:45:01.873537+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:49:04.137885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"523'14113","reported_seq":27885,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-09-23T20:45:12.510085+0000","last_change":"2023-09-23T20:44:18.688723+0000","last_active":"2023-09-23T20:45:12.510085+0000","last_peered":"2023-09-23T20:45:12.510085+0000","last_clean":"2023-09-23T20:45:12.510085+0000","last_became_active":"2023-09-23T20:44:18.636162+0000","last_became_peered":"2023-09-23T20:44:18.636162+0000","last_unstale":"2023-09-23T20:45:12.510085+0000","last_undegraded":"2023-09-23T20:45:12.510085+0000","last_fullsized":"2023-09-23T20:45:12.510085+0000","mapping_epoch":502,"log_start":"478'8448","ondisk_log_start":"478'8448","created":455,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":5665,"log_dups_size":0,"ondisk_log_size":5665,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:49:45.956725+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":11272216,"num_objects":173,"num_object_clones":0,"num_object_copies":519,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":173,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":14113,"num_write_kb":457089,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6816,"num_bytes_recovered":446627864,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":801,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.867453+0000","last_change":"2023-09-23T20:38:05.590361+0000","last_active":"2023-09-23T20:45:01.867453+0000","last_peered":"2023-09-23T20:45:01.867453+0000","last_clean":"2023-09-23T20:45:01.867453+0000","last_became_active":"2023-09-23T20:38:05.589985+0000","last_became_peered":"2023-09-23T20:38:05.589985+0000","last_unstale":"2023-09-23T20:45:01.867453+0000","last_undegraded":"2023-09-23T20:45:01.867453+0000","last_fullsized":"2023-09-23T20:45:01.867453+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:03.246271+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:03.246271+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:38:54.351576+0000","scrub_duration":0.0019016700000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":260,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.862187+0000","last_change":"2023-09-23T20:32:50.549695+0000","last_active":"2023-09-23T20:45:01.862187+0000","last_peered":"2023-09-23T20:45:01.862187+0000","last_clean":"2023-09-23T20:45:01.862187+0000","last_became_active":"2023-09-23T20:32:50.549431+0000","last_became_peered":"2023-09-23T20:32:50.549431+0000","last_unstale":"2023-09-23T20:45:01.862187+0000","last_undegraded":"2023-09-23T20:45:01.862187+0000","last_fullsized":"2023-09-23T20:45:01.862187+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:45:53.666569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"86'128","reported_seq":936,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.865625+0000","last_change":"2023-09-23T20:38:03.541765+0000","last_active":"2023-09-23T20:45:01.865625+0000","last_peered":"2023-09-23T20:45:01.865625+0000","last_clean":"2023-09-23T20:45:01.865625+0000","last_became_active":"2023-09-23T20:38:03.541277+0000","last_became_peered":"2023-09-23T20:38:03.541277+0000","last_unstale":"2023-09-23T20:45:01.865625+0000","last_undegraded":"2023-09-23T20:45:01.865625+0000","last_fullsized":"2023-09-23T20:45:01.865625+0000","mapping_epoch":404,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":405,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:20:53.834554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000293202,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":579,"num_read_kb":1473,"num_write":96,"num_write_kb":1062,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":303,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.863712+0000","last_change":"2023-09-23T20:37:43.582105+0000","last_active":"2023-09-23T20:45:01.863712+0000","last_peered":"2023-09-23T20:45:01.863712+0000","last_clean":"2023-09-23T20:45:01.863712+0000","last_became_active":"2023-09-23T20:37:43.581860+0000","last_became_peered":"2023-09-23T20:37:43.581860+0000","last_unstale":"2023-09-23T20:45:01.863712+0000","last_undegraded":"2023-09-23T20:45:01.863712+0000","last_fullsized":"2023-09-23T20:45:01.863712+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:18:38.765486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.c","version":"523'13942","reported_seq":14020,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-09-23T20:45:09.348488+0000","last_change":"2023-09-23T20:41:04.575496+0000","last_active":"2023-09-23T20:45:09.348488+0000","last_peered":"2023-09-23T20:45:09.348488+0000","last_clean":"2023-09-23T20:45:09.348488+0000","last_became_active":"2023-09-23T20:41:04.575242+0000","last_became_peered":"2023-09-23T20:41:04.575242+0000","last_unstale":"2023-09-23T20:45:09.348488+0000","last_undegraded":"2023-09-23T20:45:09.348488+0000","last_fullsized":"2023-09-23T20:45:09.348488+0000","mapping_epoch":455,"log_start":"477'7875","ondisk_log_start":"477'7875","created":455,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":6067,"log_dups_size":0,"ondisk_log_size":6067,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:30:14.611529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":20316160,"num_objects":310,"num_object_clones":0,"num_object_copies":930,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":310,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13942,"num_write_kb":456064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"90'161","reported_seq":737,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.863066+0000","last_change":"2023-09-23T20:40:22.427435+0000","last_active":"2023-09-23T20:45:01.863066+0000","last_peered":"2023-09-23T20:45:01.863066+0000","last_clean":"2023-09-23T20:45:01.863066+0000","last_became_active":"2023-09-23T20:40:22.427051+0000","last_became_peered":"2023-09-23T20:40:22.427051+0000","last_unstale":"2023-09-23T20:45:01.863066+0000","last_undegraded":"2023-09-23T20:45:01.863066+0000","last_fullsized":"2023-09-23T20:45:01.863066+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:34:54.504554+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:34:54.504554+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:42.001652+0000","scrub_duration":0.011176864,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"86'117","reported_seq":831,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.874299+0000","last_change":"2023-09-23T20:44:18.635224+0000","last_active":"2023-09-23T20:45:01.874299+0000","last_peered":"2023-09-23T20:45:01.874299+0000","last_clean":"2023-09-23T20:45:01.874299+0000","last_became_active":"2023-09-23T20:44:18.634703+0000","last_became_peered":"2023-09-23T20:44:18.634703+0000","last_unstale":"2023-09-23T20:45:01.874299+0000","last_undegraded":"2023-09-23T20:45:01.874299+0000","last_fullsized":"2023-09-23T20:45:01.874299+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:59:38.030626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046891400000000002,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":893,"num_read_kb":922,"num_write":82,"num_write_kb":4193,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":16777216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":510,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.869762+0000","last_change":"2023-09-23T20:43:46.521505+0000","last_active":"2023-09-23T20:45:01.869762+0000","last_peered":"2023-09-23T20:45:01.869762+0000","last_clean":"2023-09-23T20:45:01.869762+0000","last_became_active":"2023-09-23T20:43:46.520727+0000","last_became_peered":"2023-09-23T20:43:46.520727+0000","last_unstale":"2023-09-23T20:45:01.869762+0000","last_undegraded":"2023-09-23T20:45:01.869762+0000","last_fullsized":"2023-09-23T20:45:01.869762+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:17:06.164741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":260,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.853692+0000","last_change":"2023-09-23T20:40:19.530602+0000","last_active":"2023-09-23T20:45:01.853692+0000","last_peered":"2023-09-23T20:45:01.853692+0000","last_clean":"2023-09-23T20:45:01.853692+0000","last_became_active":"2023-09-23T20:40:19.530256+0000","last_became_peered":"2023-09-23T20:40:19.530256+0000","last_unstale":"2023-09-23T20:45:01.853692+0000","last_undegraded":"2023-09-23T20:45:01.853692+0000","last_fullsized":"2023-09-23T20:45:01.853692+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:18:45.658867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"90'161","reported_seq":779,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.872622+0000","last_change":"2023-09-23T20:43:46.544750+0000","last_active":"2023-09-23T20:45:01.872622+0000","last_peered":"2023-09-23T20:45:01.872622+0000","last_clean":"2023-09-23T20:45:01.872622+0000","last_became_active":"2023-09-23T20:43:46.544395+0000","last_became_peered":"2023-09-23T20:43:46.544395+0000","last_unstale":"2023-09-23T20:45:01.872622+0000","last_undegraded":"2023-09-23T20:45:01.872622+0000","last_fullsized":"2023-09-23T20:45:01.872622+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:06.487351+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:06.487351+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:56.866168+0000","scrub_duration":0.001964379,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":477,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.873464+0000","last_change":"2023-09-23T20:43:46.541379+0000","last_active":"2023-09-23T20:45:01.873464+0000","last_peered":"2023-09-23T20:45:01.873464+0000","last_clean":"2023-09-23T20:45:01.873464+0000","last_became_active":"2023-09-23T20:43:46.540732+0000","last_became_peered":"2023-09-23T20:43:46.540732+0000","last_unstale":"2023-09-23T20:45:01.873464+0000","last_undegraded":"2023-09-23T20:45:01.873464+0000","last_fullsized":"2023-09-23T20:45:01.873464+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:31:40.605772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":379,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.864774+0000","last_change":"2023-09-23T20:29:45.538314+0000","last_active":"2023-09-23T20:45:01.864774+0000","last_peered":"2023-09-23T20:45:01.864774+0000","last_clean":"2023-09-23T20:45:01.864774+0000","last_became_active":"2023-09-23T20:29:45.536847+0000","last_became_peered":"2023-09-23T20:29:45.536847+0000","last_unstale":"2023-09-23T20:45:01.864774+0000","last_undegraded":"2023-09-23T20:45:01.864774+0000","last_fullsized":"2023-09-23T20:45:01.864774+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:51.948721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"90'161","reported_seq":596,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.857119+0000","last_change":"2023-09-23T20:41:06.499511+0000","last_active":"2023-09-23T20:45:01.857119+0000","last_peered":"2023-09-23T20:45:01.857119+0000","last_clean":"2023-09-23T20:45:01.857119+0000","last_became_active":"2023-09-23T20:38:05.601874+0000","last_became_peered":"2023-09-23T20:38:05.601874+0000","last_unstale":"2023-09-23T20:45:01.857119+0000","last_undegraded":"2023-09-23T20:45:01.857119+0000","last_fullsized":"2023-09-23T20:45:01.857119+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:41:06.499396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:41:06.499396+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:31:34.059077+0000","scrub_duration":0.0019670310000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":415,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.868060+0000","last_change":"2023-09-23T20:43:16.758761+0000","last_active":"2023-09-23T20:45:01.868060+0000","last_peered":"2023-09-23T20:45:01.868060+0000","last_clean":"2023-09-23T20:45:01.868060+0000","last_became_active":"2023-09-23T20:43:16.758407+0000","last_became_peered":"2023-09-23T20:43:16.758407+0000","last_unstale":"2023-09-23T20:45:01.868060+0000","last_undegraded":"2023-09-23T20:45:01.868060+0000","last_fullsized":"2023-09-23T20:45:01.868060+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:59:27.070685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":179,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.854089+0000","last_change":"2023-09-23T20:34:59.987575+0000","last_active":"2023-09-23T20:45:01.854089+0000","last_peered":"2023-09-23T20:45:01.854089+0000","last_clean":"2023-09-23T20:45:01.854089+0000","last_became_active":"2023-09-23T20:34:59.987229+0000","last_became_peered":"2023-09-23T20:34:59.987229+0000","last_unstale":"2023-09-23T20:45:01.854089+0000","last_undegraded":"2023-09-23T20:45:01.854089+0000","last_fullsized":"2023-09-23T20:45:01.854089+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:34:01.820159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":423,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.854693+0000","last_change":"2023-09-23T20:43:16.759691+0000","last_active":"2023-09-23T20:45:01.854693+0000","last_peered":"2023-09-23T20:45:01.854693+0000","last_clean":"2023-09-23T20:45:01.854693+0000","last_became_active":"2023-09-23T20:43:16.759309+0000","last_became_peered":"2023-09-23T20:43:16.759309+0000","last_unstale":"2023-09-23T20:45:01.854693+0000","last_undegraded":"2023-09-23T20:45:01.854693+0000","last_fullsized":"2023-09-23T20:45:01.854693+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:58:52.893019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":179,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.853876+0000","last_change":"2023-09-23T20:34:59.987416+0000","last_active":"2023-09-23T20:45:01.853876+0000","last_peered":"2023-09-23T20:45:01.853876+0000","last_clean":"2023-09-23T20:45:01.853876+0000","last_became_active":"2023-09-23T20:34:59.987117+0000","last_became_peered":"2023-09-23T20:34:59.987117+0000","last_unstale":"2023-09-23T20:45:01.853876+0000","last_undegraded":"2023-09-23T20:45:01.853876+0000","last_fullsized":"2023-09-23T20:45:01.853876+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:36:16.318623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"90'161","reported_seq":778,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.861780+0000","last_change":"2023-09-23T20:38:05.591992+0000","last_active":"2023-09-23T20:45:01.861780+0000","last_peered":"2023-09-23T20:45:01.861780+0000","last_clean":"2023-09-23T20:45:01.861780+0000","last_became_active":"2023-09-23T20:38:05.591701+0000","last_became_peered":"2023-09-23T20:38:05.591701+0000","last_unstale":"2023-09-23T20:45:01.861780+0000","last_undegraded":"2023-09-23T20:45:01.861780+0000","last_fullsized":"2023-09-23T20:45:01.861780+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:02.384133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:02.384133+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:41.722542+0000","scrub_duration":0.001769098,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":345,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.857318+0000","last_change":"2023-09-23T20:38:02.542719+0000","last_active":"2023-09-23T20:45:01.857318+0000","last_peered":"2023-09-23T20:45:01.857318+0000","last_clean":"2023-09-23T20:45:01.857318+0000","last_became_active":"2023-09-23T20:38:02.542385+0000","last_became_peered":"2023-09-23T20:38:02.542385+0000","last_unstale":"2023-09-23T20:45:01.857318+0000","last_undegraded":"2023-09-23T20:45:01.857318+0000","last_fullsized":"2023-09-23T20:45:01.857318+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:36:23.673757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":312,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.856603+0000","last_change":"2023-09-23T20:37:43.584253+0000","last_active":"2023-09-23T20:45:01.856603+0000","last_peered":"2023-09-23T20:45:01.856603+0000","last_clean":"2023-09-23T20:45:01.856603+0000","last_became_active":"2023-09-23T20:37:43.580069+0000","last_became_peered":"2023-09-23T20:37:43.580069+0000","last_unstale":"2023-09-23T20:45:01.856603+0000","last_undegraded":"2023-09-23T20:45:01.856603+0000","last_fullsized":"2023-09-23T20:45:01.856603+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:39:11.482581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"90'161","reported_seq":616,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.867161+0000","last_change":"2023-09-23T20:36:01.505098+0000","last_active":"2023-09-23T20:45:01.867161+0000","last_peered":"2023-09-23T20:45:01.867161+0000","last_clean":"2023-09-23T20:45:01.867161+0000","last_became_active":"2023-09-23T20:35:26.644707+0000","last_became_peered":"2023-09-23T20:35:26.644707+0000","last_unstale":"2023-09-23T20:45:01.867161+0000","last_undegraded":"2023-09-23T20:45:01.867161+0000","last_fullsized":"2023-09-23T20:45:01.867161+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:01.504973+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:01.504973+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:02:10.880809+0000","scrub_duration":0.00202344,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":555,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.870333+0000","last_change":"2023-09-23T20:43:46.511657+0000","last_active":"2023-09-23T20:45:01.870333+0000","last_peered":"2023-09-23T20:45:01.870333+0000","last_clean":"2023-09-23T20:45:01.870333+0000","last_became_active":"2023-09-23T20:43:46.511326+0000","last_became_peered":"2023-09-23T20:43:46.511326+0000","last_unstale":"2023-09-23T20:45:01.870333+0000","last_undegraded":"2023-09-23T20:45:01.870333+0000","last_fullsized":"2023-09-23T20:45:01.870333+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:36:46.139261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3324,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":240,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.853435+0000","last_change":"2023-09-23T20:38:02.538333+0000","last_active":"2023-09-23T20:45:01.853435+0000","last_peered":"2023-09-23T20:45:01.853435+0000","last_clean":"2023-09-23T20:45:01.853435+0000","last_became_active":"2023-09-23T20:38:02.538008+0000","last_became_peered":"2023-09-23T20:38:02.538008+0000","last_unstale":"2023-09-23T20:45:01.853435+0000","last_undegraded":"2023-09-23T20:45:01.853435+0000","last_fullsized":"2023-09-23T20:45:01.853435+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:02:38.789530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":90,"num_bytes_recovered":18486,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26691,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":193200152,"num_objects":2949,"num_object_clones":0,"num_object_copies":8847,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2949,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":224715,"num_write_kb":7285185,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16881,"num_bytes_recovered":1105854488,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":905129984,"data_stored":905117768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":103210,"ondisk_log_size":103210,"up":48,"acting":48,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5983,"num_read_kb":13200,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":84,"num_bytes_recovered":16777216,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6082,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201685,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":658636,"kb_used_data":159172,"kb_used_omap":0,"kb_used_meta":499456,"kb_avail":93086516,"statfs":{"total":95995035648,"available":95320592384,"internally_reserved":0,"allocated":162992128,"data_stored":161468378,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":511442944},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:44:59 2023","interfaces":[{"interface":"back","average":{"1min":2.8599999999999999,"5min":2.0510000000000002,"15min":2.1640000000000001},"min":{"1min":0.24199999999999999,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":10.683999999999999,"5min":15.1,"15min":18.315000000000001},"last":0.43099999999999999},{"interface":"front","average":{"1min":2.282,"5min":1.8799999999999999,"15min":1.897},"min":{"1min":0.31900000000000001,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":11.045999999999999,"5min":15.327999999999999,"15min":19.355},"last":0.28299999999999997}]},{"osd":1,"last update":"Sat Sep 23 20:45:08 2023","interfaces":[{"interface":"back","average":{"1min":1.002,"5min":1.468,"15min":2.0539999999999998},"min":{"1min":0.26100000000000001,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":2.5609999999999999,"5min":15.131,"15min":49.965000000000003},"last":0.39100000000000001},{"interface":"front","average":{"1min":2.7730000000000001,"5min":2.6419999999999999,"15min":2.3919999999999999},"min":{"1min":0.39700000000000002,"5min":0.26700000000000002,"15min":0.248},"max":{"1min":11.276,"5min":15.18,"15min":53.451999999999998},"last":1.3380000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:44:59 2023","interfaces":[{"interface":"back","average":{"1min":2.4889999999999999,"5min":2.3119999999999998,"15min":2.286},"min":{"1min":0.30499999999999999,"5min":0.27900000000000003,"15min":0.27800000000000002},"max":{"1min":8.0020000000000007,"5min":15.087,"15min":16.965},"last":0.34100000000000003},{"interface":"front","average":{"1min":2.6520000000000001,"5min":2.0230000000000001,"15min":2.4670000000000001},"min":{"1min":0.371,"5min":0.30299999999999999,"15min":0.27400000000000002},"max":{"1min":13.5,"5min":15.193,"15min":23.494},"last":0.49199999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:44:28 2023","interfaces":[{"interface":"back","average":{"1min":2.6840000000000002,"5min":1.772,"15min":2.371},"min":{"1min":0.41399999999999998,"5min":0.32500000000000001,"15min":0.29199999999999998},"max":{"1min":15.115,"5min":15.115,"15min":23.352},"last":1.665},{"interface":"front","average":{"1min":3.3570000000000002,"5min":2.1389999999999998,"15min":2.4489999999999998},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.33100000000000002},"max":{"1min":15.144,"5min":15.144,"15min":20.969999999999999},"last":1.6240000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:44:28 2023","interfaces":[{"interface":"back","average":{"1min":2.0110000000000001,"5min":1.391,"15min":1.2669999999999999},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.23200000000000001},"max":{"1min":12.260999999999999,"5min":12.260999999999999,"15min":18.364999999999998},"last":0.57099999999999995},{"interface":"front","average":{"1min":2.145,"5min":1.5640000000000001,"15min":1.3320000000000001},"min":{"1min":0.315,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":6.9939999999999998,"5min":6.9939999999999998,"15min":23.617000000000001},"last":0.66200000000000003}]},{"osd":5,"last update":"Sat Sep 23 20:44:28 2023","interfaces":[{"interface":"back","average":{"1min":2.5270000000000001,"5min":1.6040000000000001,"15min":1.1970000000000001},"min":{"1min":0.36899999999999999,"5min":0.30099999999999999,"15min":0.26600000000000001},"max":{"1min":15.005000000000001,"5min":15.005000000000001,"15min":15.005000000000001},"last":0.45700000000000002},{"interface":"front","average":{"1min":1.9990000000000001,"5min":1.556,"15min":1.3120000000000001},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":12.212,"5min":12.731,"15min":18.831},"last":1.456}]},{"osd":6,"last update":"Sat Sep 23 20:44:28 2023","interfaces":[{"interface":"back","average":{"1min":3.6379999999999999,"5min":1.837,"15min":1.4359999999999999},"min":{"1min":0.34599999999999997,"5min":0.33700000000000002,"15min":0.30399999999999999},"max":{"1min":15.038,"5min":15.038,"15min":18.417000000000002},"last":0.61599999999999999},{"interface":"front","average":{"1min":2.8969999999999998,"5min":1.6659999999999999,"15min":1.274},"min":{"1min":0.47999999999999998,"5min":0.36499999999999999,"15min":0.318},"max":{"1min":15.369999999999999,"5min":15.369999999999999,"15min":15.369999999999999},"last":0.69399999999999995}]}]},{"osd":6,"up_from":47,"seq":201863463324,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":455036,"kb_used_data":33268,"kb_used_omap":0,"kb_used_meta":421760,"kb_avail":93290116,"statfs":{"total":95995035648,"available":95529078784,"internally_reserved":0,"allocated":34066432,"data_stored":32558371,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":431882240},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:44:55 2023","interfaces":[{"interface":"back","average":{"1min":1.883,"5min":1.5720000000000001,"15min":1.986},"min":{"1min":0.219,"5min":0.219,"15min":0.219},"max":{"1min":7.7720000000000002,"5min":12.552,"15min":53.701999999999998},"last":1.4390000000000001},{"interface":"front","average":{"1min":2.343,"5min":1.7430000000000001,"15min":2.3730000000000002},"min":{"1min":0.313,"5min":0.23300000000000001,"15min":0.22700000000000001},"max":{"1min":10.895,"5min":10.895,"15min":60.328000000000003},"last":1.502}]},{"osd":1,"last update":"Sat Sep 23 20:44:17 2023","interfaces":[{"interface":"back","average":{"1min":3.3330000000000002,"5min":1.744,"15min":2.1309999999999998},"min":{"1min":0.317,"5min":0.249,"15min":0.249},"max":{"1min":9.2520000000000007,"5min":9.2520000000000007,"15min":64.274000000000001},"last":3.9009999999999998},{"interface":"front","average":{"1min":2.8940000000000001,"5min":1.8100000000000001,"15min":2.0739999999999998},"min":{"1min":0.42799999999999999,"5min":0.32000000000000001,"15min":0.28000000000000003},"max":{"1min":17.786000000000001,"5min":17.786000000000001,"15min":34.490000000000002},"last":4.2279999999999998}]},{"osd":2,"last update":"Sat Sep 23 20:44:55 2023","interfaces":[{"interface":"back","average":{"1min":2.5409999999999999,"5min":2.2370000000000001,"15min":2.1739999999999999},"min":{"1min":0.34899999999999998,"5min":0.27800000000000002,"15min":0.27700000000000002},"max":{"1min":6.6130000000000004,"5min":9.9049999999999994,"15min":34.317999999999998},"last":1.478},{"interface":"front","average":{"1min":2.3079999999999998,"5min":2.0129999999999999,"15min":2.069},"min":{"1min":0.373,"5min":0.311,"15min":0.30399999999999999},"max":{"1min":6.5620000000000003,"5min":9.9269999999999996,"15min":34.292999999999999},"last":1.4650000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:44:46 2023","interfaces":[{"interface":"back","average":{"1min":2.7330000000000001,"5min":2.024,"15min":1.974},"min":{"1min":0.47999999999999998,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":11.664,"5min":11.664,"15min":11.664},"last":1.55},{"interface":"front","average":{"1min":2.3039999999999998,"5min":2.0299999999999998,"15min":2.1459999999999999},"min":{"1min":0.42099999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":8.2560000000000002,"5min":11.808,"15min":12.875999999999999},"last":1.5700000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:44:17 2023","interfaces":[{"interface":"back","average":{"1min":1.27,"5min":0.997,"15min":0.85199999999999998},"min":{"1min":0.33600000000000002,"5min":0.25600000000000001,"15min":0.251},"max":{"1min":6.5800000000000001,"5min":6.5800000000000001,"15min":23.904},"last":1.407},{"interface":"front","average":{"1min":1.4019999999999999,"5min":1.1519999999999999,"15min":1.1240000000000001},"min":{"1min":0.34999999999999998,"5min":0.309,"15min":0.27000000000000002},"max":{"1min":5.6260000000000003,"5min":6.7359999999999998,"15min":11.515000000000001},"last":1.528}]},{"osd":5,"last update":"Sat Sep 23 20:44:17 2023","interfaces":[{"interface":"back","average":{"1min":1.381,"5min":0.97099999999999997,"15min":0.86499999999999999},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":6.5460000000000003,"5min":6.6340000000000003,"15min":8.4499999999999993},"last":1.8120000000000001},{"interface":"front","average":{"1min":1.5509999999999999,"5min":1.4870000000000001,"15min":1.1970000000000001},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":5.4969999999999999,"5min":7.1319999999999997,"15min":9.9299999999999997},"last":2.129}]},{"osd":7,"last update":"Sat Sep 23 20:44:46 2023","interfaces":[{"interface":"back","average":{"1min":1.4199999999999999,"5min":1.079,"15min":0.94799999999999995},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":7.8250000000000002,"5min":8.8350000000000009,"15min":8.8350000000000009},"last":1.631},{"interface":"front","average":{"1min":2.6440000000000001,"5min":1.661,"15min":1.3009999999999999},"min":{"1min":0.45900000000000002,"5min":0.34699999999999998,"15min":0.30299999999999999},"max":{"1min":7.1239999999999997,"5min":7.1929999999999996,"15min":8.8119999999999994},"last":1.591}]}]},{"osd":1,"up_from":266,"seq":1142461300923,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":914072,"kb_used_data":235536,"kb_used_omap":8,"kb_used_meta":678327,"kb_avail":92831080,"statfs":{"total":95995035648,"available":95059025920,"internally_reserved":0,"allocated":241188864,"data_stored":238490530,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8206,"internal_metadata":694607858},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:44:15 2023","interfaces":[{"interface":"back","average":{"1min":3.6110000000000002,"5min":2.206,"15min":2.5329999999999999},"min":{"1min":0.214,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":15.532999999999999,"5min":16.292000000000002,"15min":52.689999999999998},"last":0.372},{"interface":"front","average":{"1min":3.2290000000000001,"5min":2.2970000000000002,"15min":3.0179999999999998},"min":{"1min":0.22600000000000001,"5min":0.20100000000000001,"15min":0.19900000000000001},"max":{"1min":14.445,"5min":16.332999999999998,"15min":52.662999999999997},"last":2.0299999999999998}]},{"osd":2,"last update":"Sat Sep 23 20:44:15 2023","interfaces":[{"interface":"back","average":{"1min":3.0390000000000001,"5min":2.4140000000000001,"15min":3.3370000000000002},"min":{"1min":0.28999999999999998,"5min":0.216,"15min":0.20899999999999999},"max":{"1min":8.6519999999999992,"5min":14.148,"15min":32.255000000000003},"last":3.3090000000000002},{"interface":"front","average":{"1min":2.9049999999999998,"5min":1.9810000000000001,"15min":2.2909999999999999},"min":{"1min":0.26000000000000001,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":10.041,"5min":10.041,"15min":13.404999999999999},"last":0.32100000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:44:15 2023","interfaces":[{"interface":"back","average":{"1min":2.5830000000000002,"5min":2.089,"15min":3.097},"min":{"1min":0.36899999999999999,"5min":0.247,"15min":0.247},"max":{"1min":7.2469999999999999,"5min":16.245999999999999,"15min":80.745999999999995},"last":1.6259999999999999},{"interface":"front","average":{"1min":3.5779999999999998,"5min":2.645,"15min":3.3450000000000002},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.22700000000000001},"max":{"1min":11.523,"5min":13.497999999999999,"15min":82.988},"last":3.0920000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:44:15 2023","interfaces":[{"interface":"back","average":{"1min":2.3250000000000002,"5min":2.202,"15min":2.25},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":6.5970000000000004,"5min":20.634,"15min":24.327999999999999},"last":4.6040000000000001},{"interface":"front","average":{"1min":2.0190000000000001,"5min":1.982,"15min":1.748},"min":{"1min":0.34699999999999998,"5min":0.32600000000000001,"15min":0.311},"max":{"1min":6.0869999999999997,"5min":20.835999999999999,"15min":20.835999999999999},"last":10.468999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:44:15 2023","interfaces":[{"interface":"back","average":{"1min":3.2040000000000002,"5min":2.4319999999999999,"15min":2.3029999999999999},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":14.366,"5min":14.366,"15min":33.401000000000003},"last":5.9560000000000004},{"interface":"front","average":{"1min":2.7469999999999999,"5min":2.3519999999999999,"15min":2.4820000000000002},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.32500000000000001},"max":{"1min":7.3300000000000001,"5min":10.958,"15min":33.417000000000002},"last":4.1139999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:44:15 2023","interfaces":[{"interface":"back","average":{"1min":3.0030000000000001,"5min":2.3639999999999999,"15min":2.839},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35699999999999998},"max":{"1min":14.414999999999999,"5min":14.414999999999999,"15min":72.682000000000002},"last":5.9969999999999999},{"interface":"front","average":{"1min":3.2829999999999999,"5min":2.3620000000000001,"15min":2.9860000000000002},"min":{"1min":0.38500000000000001,"5min":0.317,"15min":0.317},"max":{"1min":9.1470000000000002,"5min":9.4090000000000007,"15min":72.563999999999993},"last":7.6210000000000004}]},{"osd":7,"last update":"Sat Sep 23 20:44:15 2023","interfaces":[{"interface":"back","average":{"1min":2.98,"5min":2.4470000000000001,"15min":2.3700000000000001},"min":{"1min":0.43099999999999999,"5min":0.37,"15min":0.37},"max":{"1min":9.2409999999999997,"5min":16.359000000000002,"15min":18.437000000000001},"last":6.0259999999999998},{"interface":"front","average":{"1min":2.6459999999999999,"5min":2.4460000000000002,"15min":2.4910000000000001},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":7.1959999999999997,"5min":10.34,"15min":24.524000000000001},"last":5.2329999999999997}]}]},{"osd":0,"up_from":94,"seq":403726926125,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":792804,"kb_used_data":71452,"kb_used_omap":7,"kb_used_meta":721272,"kb_avail":92952348,"statfs":{"total":95995035648,"available":95183204352,"internally_reserved":0,"allocated":73166848,"data_stored":70633946,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":738582545},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:44:24 2023","interfaces":[{"interface":"back","average":{"1min":3.2120000000000002,"5min":2.4100000000000001,"15min":2.7210000000000001},"min":{"1min":0.184,"5min":0.184,"15min":0.17199999999999999},"max":{"1min":18.052,"5min":24.010000000000002,"15min":42.000999999999998},"last":0.35099999999999998},{"interface":"front","average":{"1min":3.0939999999999999,"5min":2.605,"15min":2.9129999999999998},"min":{"1min":0.33800000000000002,"5min":0.218,"15min":0.20399999999999999},"max":{"1min":18.097000000000001,"5min":18.097000000000001,"15min":23.036000000000001},"last":0.72699999999999998}]},{"osd":2,"last update":"Sat Sep 23 20:45:01 2023","interfaces":[{"interface":"back","average":{"1min":3.242,"5min":3.052,"15min":3.085},"min":{"1min":0.27000000000000002,"5min":0.248,"15min":0.185},"max":{"1min":15.108000000000001,"5min":48.841000000000001,"15min":77.448999999999998},"last":2.258},{"interface":"front","average":{"1min":3.5419999999999998,"5min":2.722,"15min":3.0470000000000002},"min":{"1min":0.28599999999999998,"5min":0.22,"15min":0.188},"max":{"1min":17.013000000000002,"5min":46.183,"15min":81.724999999999994},"last":3.0139999999999998}]},{"osd":3,"last update":"Sat Sep 23 20:44:31 2023","interfaces":[{"interface":"back","average":{"1min":3.6960000000000002,"5min":2.7290000000000001,"15min":3.1640000000000001},"min":{"1min":0.30599999999999999,"5min":0.248,"15min":0.23400000000000001},"max":{"1min":15.048,"5min":15.048,"15min":81.400999999999996},"last":2.431},{"interface":"front","average":{"1min":2.1549999999999998,"5min":2.3159999999999998,"15min":3.3439999999999999},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.21199999999999999},"max":{"1min":9.6379999999999999,"5min":14.237,"15min":81.137},"last":0.65600000000000003}]},{"osd":4,"last update":"Sat Sep 23 20:45:01 2023","interfaces":[{"interface":"back","average":{"1min":1.968,"5min":2.052,"15min":1.996},"min":{"1min":0.35999999999999999,"5min":0.32000000000000001,"15min":0.25900000000000001},"max":{"1min":10.368,"5min":10.368,"15min":12.587999999999999},"last":2.0499999999999998},{"interface":"front","average":{"1min":1.8049999999999999,"5min":1.9650000000000001,"15min":1.9099999999999999},"min":{"1min":0.34599999999999997,"5min":0.28899999999999998,"15min":0.27400000000000002},"max":{"1min":7.6740000000000004,"5min":9.8480000000000008,"15min":23.001000000000001},"last":0.627}]},{"osd":5,"last update":"Sat Sep 23 20:45:01 2023","interfaces":[{"interface":"back","average":{"1min":2.0819999999999999,"5min":2.4590000000000001,"15min":2.0880000000000001},"min":{"1min":0.308,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":7.7290000000000001,"5min":10.239000000000001,"15min":22.969999999999999},"last":0.45300000000000001},{"interface":"front","average":{"1min":2.964,"5min":2.496,"15min":2.3780000000000001},"min":{"1min":0.442,"5min":0.34300000000000003,"15min":0.25600000000000001},"max":{"1min":12.601000000000001,"5min":12.929,"15min":26.228000000000002},"last":2.9169999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:45:01 2023","interfaces":[{"interface":"back","average":{"1min":2.0760000000000001,"5min":2.375,"15min":2.1720000000000002},"min":{"1min":0.39100000000000001,"5min":0.371,"15min":0.32300000000000001},"max":{"1min":7.7969999999999997,"5min":24.056000000000001,"15min":24.056000000000001},"last":0.48799999999999999},{"interface":"front","average":{"1min":3.153,"5min":2.0880000000000001,"15min":2.2309999999999999},"min":{"1min":0.46999999999999997,"5min":0.34100000000000003,"15min":0.29399999999999998},"max":{"1min":12.698,"5min":12.698,"15min":26.268000000000001},"last":2.9860000000000002}]},{"osd":7,"last update":"Sat Sep 23 20:45:01 2023","interfaces":[{"interface":"back","average":{"1min":2.9910000000000001,"5min":2.3580000000000001,"15min":2.3860000000000001},"min":{"1min":0.497,"5min":0.36799999999999999,"15min":0.32200000000000001},"max":{"1min":12.651999999999999,"5min":12.651999999999999,"15min":26.177},"last":2.956},{"interface":"front","average":{"1min":2.4729999999999999,"5min":2.6389999999999998,"15min":2.3799999999999999},"min":{"1min":0.40600000000000003,"5min":0.40400000000000003,"15min":0.34200000000000003},"max":{"1min":10.32,"5min":15.006,"15min":22.422999999999998},"last":2.6379999999999999}]}]},{"osd":2,"up_from":104,"seq":446676599073,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":779180,"kb_used_data":37412,"kb_used_omap":11,"kb_used_meta":741748,"kb_avail":92965972,"statfs":{"total":95995035648,"available":95197155328,"internally_reserved":0,"allocated":38309888,"data_stored":35767747,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11878,"internal_metadata":759550362},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:45:09 2023","interfaces":[{"interface":"back","average":{"1min":1.696,"5min":1.2989999999999999,"15min":2.456},"min":{"1min":0.20499999999999999,"5min":0.17299999999999999,"15min":0.17299999999999999},"max":{"1min":7.4480000000000004,"5min":11.07,"15min":82.424999999999997},"last":0.35199999999999998},{"interface":"front","average":{"1min":2.972,"5min":2.1480000000000001,"15min":2.8140000000000001},"min":{"1min":0.26100000000000001,"5min":0.23599999999999999,"15min":0.19},"max":{"1min":11.244,"5min":16.678999999999998,"15min":82.524000000000001},"last":0.32600000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:45:09 2023","interfaces":[{"interface":"back","average":{"1min":3.4609999999999999,"5min":2.218,"15min":2.746},"min":{"1min":0.314,"5min":0.248,"15min":0.221},"max":{"1min":12.797000000000001,"5min":12.981,"15min":46.405000000000001},"last":2.7090000000000001},{"interface":"front","average":{"1min":2.5720000000000001,"5min":2.125,"15min":2.6139999999999999},"min":{"1min":0.23200000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":13.215999999999999,"5min":13.215999999999999,"15min":46.459000000000003},"last":0.57299999999999995}]},{"osd":3,"last update":"Sat Sep 23 20:44:31 2023","interfaces":[{"interface":"back","average":{"1min":2.1800000000000002,"5min":2.2010000000000001,"15min":2.5990000000000002},"min":{"1min":0.253,"5min":0.253,"15min":0.19800000000000001},"max":{"1min":12.847,"5min":12.847,"15min":19.562999999999999},"last":2.786},{"interface":"front","average":{"1min":0.91400000000000003,"5min":1.425,"15min":2.1720000000000002},"min":{"1min":0.26500000000000001,"5min":0.26400000000000001,"15min":0.249},"max":{"1min":6.3700000000000001,"5min":11.375,"15min":30.704999999999998},"last":0.42599999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:45:09 2023","interfaces":[{"interface":"back","average":{"1min":2.4929999999999999,"5min":1.8169999999999999,"15min":1.976},"min":{"1min":0.32000000000000001,"5min":0.29799999999999999,"15min":0.27500000000000002},"max":{"1min":11.141999999999999,"5min":12.025,"15min":16.797999999999998},"last":4.5839999999999996},{"interface":"front","average":{"1min":2.593,"5min":1.831,"15min":2.0840000000000001},"min":{"1min":0.40500000000000003,"5min":0.30599999999999999,"15min":0.30499999999999999},"max":{"1min":10.948,"5min":10.948,"15min":19.440999999999999},"last":4.4029999999999996}]},{"osd":5,"last update":"Sat Sep 23 20:45:09 2023","interfaces":[{"interface":"back","average":{"1min":2.6459999999999999,"5min":2.0470000000000002,"15min":2.1179999999999999},"min":{"1min":0.39600000000000002,"5min":0.311,"15min":0.311},"max":{"1min":11.183,"5min":12.07,"15min":16.766999999999999},"last":0.41199999999999998},{"interface":"front","average":{"1min":2.7559999999999998,"5min":1.831,"15min":2.2269999999999999},"min":{"1min":0.373,"5min":0.37,"15min":0.34799999999999998},"max":{"1min":11.061999999999999,"5min":11.061999999999999,"15min":19.513000000000002},"last":4.444}]},{"osd":6,"last update":"Sat Sep 23 20:45:09 2023","interfaces":[{"interface":"back","average":{"1min":2.7069999999999999,"5min":2.121,"15min":2.262},"min":{"1min":0.39100000000000001,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":11.031000000000001,"5min":14.047000000000001,"15min":19.486000000000001},"last":2.8900000000000001},{"interface":"front","average":{"1min":1.415,"5min":1.304,"15min":1.6930000000000001},"min":{"1min":0.44500000000000001,"5min":0.34200000000000003,"15min":0.316},"max":{"1min":8.5920000000000005,"5min":11.747,"15min":30.670999999999999},"last":2.931}]},{"osd":7,"last update":"Sat Sep 23 20:45:09 2023","interfaces":[{"interface":"back","average":{"1min":1.341,"5min":1.347,"15min":1.714},"min":{"1min":0.41999999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":3.9220000000000002,"5min":8.4949999999999992,"15min":30.741},"last":1.474},{"interface":"front","average":{"1min":2.71,"5min":2.2730000000000001,"15min":2.294},"min":{"1min":0.35799999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":11.218,"5min":12.099,"15min":16.719000000000001},"last":2.2170000000000001}]}]},{"osd":3,"up_from":109,"seq":468151435546,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1139608,"kb_used_data":180240,"kb_used_omap":4,"kb_used_meta":959355,"kb_avail":92605544,"statfs":{"total":95995035648,"available":94828077056,"internally_reserved":0,"allocated":184565760,"data_stored":182040359,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4514,"internal_metadata":982380126},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:44:39 2023","interfaces":[{"interface":"back","average":{"1min":2.71,"5min":2.4169999999999998,"15min":2.4929999999999999},"min":{"1min":0.222,"5min":0.20899999999999999,"15min":0.191},"max":{"1min":8.5419999999999998,"5min":22.32,"15min":22.32},"last":5.3070000000000004},{"interface":"front","average":{"1min":2.105,"5min":2.6749999999999998,"15min":2.4929999999999999},"min":{"1min":0.249,"5min":0.22900000000000001,"15min":0.19500000000000001},"max":{"1min":6.2160000000000002,"5min":22.805,"15min":22.805},"last":1.7410000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:44:11 2023","interfaces":[{"interface":"back","average":{"1min":2.823,"5min":2.1960000000000002,"15min":2.2290000000000001},"min":{"1min":0.31,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":10.022,"5min":17.529,"15min":17.529},"last":5.0250000000000004},{"interface":"front","average":{"1min":1.742,"5min":2.3700000000000001,"15min":2.4500000000000002},"min":{"1min":0.33300000000000002,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":7.1029999999999998,"5min":14.19,"15min":32.603999999999999},"last":0.47599999999999998}]},{"osd":2,"last update":"Sat Sep 23 20:44:39 2023","interfaces":[{"interface":"back","average":{"1min":2.2970000000000002,"5min":3.5840000000000001,"15min":3.0019999999999998},"min":{"1min":0.28599999999999998,"5min":0.26600000000000001,"15min":0.23799999999999999},"max":{"1min":7.7859999999999996,"5min":68.206999999999994,"15min":68.206999999999994},"last":1.004},{"interface":"front","average":{"1min":2.4470000000000001,"5min":2.7130000000000001,"15min":2.7450000000000001},"min":{"1min":0.251,"5min":0.245,"15min":0.20899999999999999},"max":{"1min":9.1760000000000002,"5min":67.736999999999995,"15min":67.736999999999995},"last":0.33800000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:44:39 2023","interfaces":[{"interface":"back","average":{"1min":2.2879999999999998,"5min":2.008,"15min":1.786},"min":{"1min":0.35799999999999998,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":6.7889999999999997,"5min":22.157,"15min":22.157},"last":4.9960000000000004},{"interface":"front","average":{"1min":2.113,"5min":2.3290000000000002,"15min":2.3279999999999998},"min":{"1min":0.36499999999999999,"5min":0.33200000000000002,"15min":0.28100000000000003},"max":{"1min":7.9020000000000001,"5min":15.868,"15min":19.635000000000002},"last":0.45900000000000002}]},{"osd":5,"last update":"Sat Sep 23 20:44:39 2023","interfaces":[{"interface":"back","average":{"1min":2.1840000000000002,"5min":2.2970000000000002,"15min":1.992},"min":{"1min":0.33800000000000002,"5min":0.32700000000000001,"15min":0.31900000000000001},"max":{"1min":6.7469999999999999,"5min":22.289999999999999,"15min":22.289999999999999},"last":4.9580000000000002},{"interface":"front","average":{"1min":1.8089999999999999,"5min":1.8440000000000001,"15min":1.841},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.32000000000000001},"max":{"1min":4.6559999999999997,"5min":12.952,"15min":12.952},"last":2.395}]},{"osd":6,"last update":"Sat Sep 23 20:44:39 2023","interfaces":[{"interface":"back","average":{"1min":1.845,"5min":2.1360000000000001,"15min":1.956},"min":{"1min":0.36599999999999999,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":6.5350000000000001,"5min":12.917,"15min":17.524999999999999},"last":0.55200000000000005},{"interface":"front","average":{"1min":1.988,"5min":2.3300000000000001,"15min":2.2869999999999999},"min":{"1min":0.38100000000000001,"5min":0.35099999999999998,"15min":0.33700000000000002},"max":{"1min":6.8479999999999999,"5min":15.936,"15min":19.524000000000001},"last":3.7930000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:44:39 2023","interfaces":[{"interface":"back","average":{"1min":2.7850000000000001,"5min":2.7509999999999999,"15min":2.6429999999999998},"min":{"1min":0.46999999999999997,"5min":0.41499999999999998,"15min":0.36899999999999999},"max":{"1min":8.4429999999999996,"5min":15.976000000000001,"15min":19.571999999999999},"last":0.42699999999999999},{"interface":"front","average":{"1min":2.4750000000000001,"5min":2.262,"15min":1.9850000000000001},"min":{"1min":0.46300000000000002,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":8.5289999999999999,"5min":22.202999999999999,"15min":22.202999999999999},"last":6.165}]}]},{"osd":4,"up_from":30,"seq":128849019300,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":557564,"kb_used_data":132596,"kb_used_omap":0,"kb_used_meta":424960,"kb_avail":93187588,"statfs":{"total":95995035648,"available":95424090112,"internally_reserved":0,"allocated":135778304,"data_stored":134269977,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":435159040},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:45:00 2023","interfaces":[{"interface":"back","average":{"1min":2.0289999999999999,"5min":2.109,"15min":1.77},"min":{"1min":0.26100000000000001,"5min":0.251,"15min":0.24099999999999999},"max":{"1min":7.9029999999999996,"5min":11.709,"15min":24.114999999999998},"last":0.41899999999999998},{"interface":"front","average":{"1min":2.343,"5min":2.3839999999999999,"15min":1.98},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.246},"max":{"1min":9.8580000000000005,"5min":13.74,"15min":24.135999999999999},"last":0.25800000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:44:11 2023","interfaces":[{"interface":"back","average":{"1min":2.306,"5min":1.8979999999999999,"15min":1.7749999999999999},"min":{"1min":0.26300000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":8.7460000000000004,"5min":10.363,"15min":18.190999999999999},"last":0.495},{"interface":"front","average":{"1min":1.355,"5min":1.5569999999999999,"15min":1.613},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":4.8659999999999997,"5min":10.426,"15min":18.173999999999999},"last":0.45700000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:44:11 2023","interfaces":[{"interface":"back","average":{"1min":2.6040000000000001,"5min":1.756,"15min":1.885},"min":{"1min":0.29499999999999998,"5min":0.28399999999999997,"15min":0.26600000000000001},"max":{"1min":9.3320000000000007,"5min":13.051,"15min":32.363},"last":3.0190000000000001},{"interface":"front","average":{"1min":2.0150000000000001,"5min":2.0539999999999998,"15min":2.2090000000000001},"min":{"1min":0.41199999999999998,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":7.6159999999999997,"5min":11.917999999999999,"15min":34.042999999999999},"last":3.706}]},{"osd":3,"last update":"Sat Sep 23 20:44:27 2023","interfaces":[{"interface":"back","average":{"1min":1.877,"5min":2.04,"15min":1.9950000000000001},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":7.7480000000000002,"5min":12.147,"15min":14.515000000000001},"last":2.0139999999999998},{"interface":"front","average":{"1min":2.4260000000000002,"5min":2.0990000000000002,"15min":2.0699999999999998},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.30299999999999999},"max":{"1min":10.493,"5min":13.35,"15min":15.991},"last":2.7490000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:45:00 2023","interfaces":[{"interface":"back","average":{"1min":1.504,"5min":1.486,"15min":1.0569999999999999},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":10.202,"5min":10.202,"15min":10.202},"last":0.55300000000000005},{"interface":"front","average":{"1min":1.895,"5min":1.389,"15min":1.0940000000000001},"min":{"1min":0.316,"5min":0.311,"15min":0.28999999999999998},"max":{"1min":8.1440000000000001,"5min":8.1440000000000001,"15min":8.1440000000000001},"last":0.47499999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:44:11 2023","interfaces":[{"interface":"back","average":{"1min":1.0920000000000001,"5min":1.5449999999999999,"15min":1.214},"min":{"1min":0.34300000000000003,"5min":0.29499999999999998,"15min":0.27600000000000002},"max":{"1min":3.96,"5min":14.731999999999999,"15min":14.731999999999999},"last":0.94999999999999996},{"interface":"front","average":{"1min":1.8959999999999999,"5min":1.2989999999999999,"15min":1.1180000000000001},"min":{"1min":0.34300000000000003,"5min":0.314,"15min":0.29899999999999999},"max":{"1min":6.3440000000000003,"5min":7.6150000000000002,"15min":11.199},"last":1.5129999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:44:27 2023","interfaces":[{"interface":"back","average":{"1min":2.0630000000000002,"5min":2.0979999999999999,"15min":1.5649999999999999},"min":{"1min":0.374,"5min":0.35099999999999998,"15min":0.318},"max":{"1min":7.7629999999999999,"5min":22.184999999999999,"15min":22.184999999999999},"last":0.82499999999999996},{"interface":"front","average":{"1min":1.1240000000000001,"5min":1.724,"15min":1.2150000000000001},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.307},"max":{"1min":6.3689999999999998,"5min":21.754000000000001,"15min":21.754000000000001},"last":1.546}]}]},{"osd":5,"up_from":39,"seq":167503724961,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":661692,"kb_used_data":158708,"kb_used_omap":0,"kb_used_meta":502592,"kb_avail":93083460,"statfs":{"total":95995035648,"available":95317463040,"internally_reserved":0,"allocated":162516992,"data_stored":161005319,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":514654208},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:45:02 2023","interfaces":[{"interface":"back","average":{"1min":2.222,"5min":1.8500000000000001,"15min":1.887},"min":{"1min":0.26300000000000001,"5min":0.22500000000000001,"15min":0.20699999999999999},"max":{"1min":11.25,"5min":15.018000000000001,"15min":18.16},"last":0.47899999999999998},{"interface":"front","average":{"1min":1.5589999999999999,"5min":1.726,"15min":1.766},"min":{"1min":0.307,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":6.71,"5min":15.034000000000001,"15min":15.034000000000001},"last":3.3919999999999999}]},{"osd":1,"last update":"Sat Sep 23 20:44:14 2023","interfaces":[{"interface":"back","average":{"1min":2.7349999999999999,"5min":3.0299999999999998,"15min":2.2690000000000001},"min":{"1min":0.379,"5min":0.26500000000000001,"15min":0.253},"max":{"1min":9.4199999999999999,"5min":61.005000000000003,"15min":61.005000000000003},"last":3.6200000000000001},{"interface":"front","average":{"1min":1.339,"5min":2.4129999999999998,"15min":2.0019999999999998},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.23499999999999999},"max":{"1min":6.7350000000000003,"5min":59.688000000000002,"15min":59.688000000000002},"last":2.6120000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:45:02 2023","interfaces":[{"interface":"back","average":{"1min":2.2919999999999998,"5min":2.2610000000000001,"15min":2.2010000000000001},"min":{"1min":0.34599999999999997,"5min":0.27100000000000002,"15min":0.26500000000000001},"max":{"1min":12.226000000000001,"5min":12.226000000000001,"15min":37.011000000000003},"last":1.5249999999999999},{"interface":"front","average":{"1min":1.246,"5min":1.893,"15min":2.0390000000000001},"min":{"1min":0.33000000000000002,"5min":0.28299999999999997,"15min":0.255},"max":{"1min":3.5259999999999998,"5min":10.565,"15min":36.835000000000001},"last":3.427}]},{"osd":3,"last update":"Sat Sep 23 20:44:20 2023","interfaces":[{"interface":"back","average":{"1min":1.7769999999999999,"5min":2.375,"15min":2.375},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.29599999999999999},"max":{"1min":5.6619999999999999,"5min":11.032,"15min":36.716000000000001},"last":3.0110000000000001},{"interface":"front","average":{"1min":1.5900000000000001,"5min":2.238,"15min":2.2530000000000001},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":7.4100000000000001,"5min":13.266,"15min":18.504000000000001},"last":2.665}]},{"osd":4,"last update":"Sat Sep 23 20:44:54 2023","interfaces":[{"interface":"back","average":{"1min":1.1850000000000001,"5min":1.857,"15min":1.2729999999999999},"min":{"1min":0.30199999999999999,"5min":0.27300000000000002,"15min":0.255},"max":{"1min":5.1050000000000004,"5min":13.391999999999999,"15min":28.015999999999998},"last":2.9870000000000001},{"interface":"front","average":{"1min":1.1499999999999999,"5min":1.6319999999999999,"15min":1.1200000000000001},"min":{"1min":0.32000000000000001,"5min":0.30299999999999999,"15min":0.28299999999999997},"max":{"1min":6.8399999999999999,"5min":7.3799999999999999,"15min":12.300000000000001},"last":2.6429999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:45:08 2023","interfaces":[{"interface":"back","average":{"1min":1.9350000000000001,"5min":2.161,"15min":1.319},"min":{"1min":0.38100000000000001,"5min":0.317,"15min":0.30199999999999999},"max":{"1min":6.952,"5min":13.734,"15min":13.734},"last":2.9510000000000001},{"interface":"front","average":{"1min":1.1220000000000001,"5min":1.5660000000000001,"15min":1.0980000000000001},"min":{"1min":0.35299999999999998,"5min":0.317,"15min":0.317},"max":{"1min":5.2629999999999999,"5min":8.7449999999999992,"15min":8.7449999999999992},"last":0.63700000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:44:20 2023","interfaces":[{"interface":"back","average":{"1min":1.4610000000000001,"5min":1.7330000000000001,"15min":1.1339999999999999},"min":{"1min":0.372,"5min":0.34499999999999997,"15min":0.28499999999999998},"max":{"1min":4.3250000000000002,"5min":16.366,"15min":16.366},"last":0.44800000000000001},{"interface":"front","average":{"1min":1.4950000000000001,"5min":1.6439999999999999,"15min":1.173},"min":{"1min":0.38900000000000001,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":7.524,"5min":7.915,"15min":13.122},"last":6.976}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4722,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1360,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8206,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7156,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3154,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":57344000,"data_stored":57344000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":215289856,"data_stored":215285784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":26673152,"data_stored":26673152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":175177728,"data_stored":175177728,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":121765888,"data_stored":121765888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":153423872,"data_stored":153419800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":22609920,"data_stored":22609920,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":132845568,"data_stored":132841496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:45:17.378 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-09-23T20:45:17.378 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:45:17.379 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_4 2023-09-23T20:45:17.379 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:45:17.613 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:45:17.617 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:17.617 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:45:17.718 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:45:17.751 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:17.788 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:45:17.791 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:17.792 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:45:17.852 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:45:17.873 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:45:17.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:17 smithi018 ceph-mon[184144]: pgmap v1314: 119 pgs: 119 active+clean; 135 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 900 op/s 2023-09-23T20:45:17.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:17 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2982809441' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:45:17.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:17 smithi018 ceph-mon[179647]: pgmap v1314: 119 pgs: 119 active+clean; 135 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 900 op/s 2023-09-23T20:45:17.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:17 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2982809441' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:45:17.893 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:45:17.905 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:45:17.921 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:17.928 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:45:17.930 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:45:17.930 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2023-09-23T20:45:17.931 INFO:teuthology.orchestra.run.smithi018.stdout: "ops": [], 2023-09-23T20:45:17.931 INFO:teuthology.orchestra.run.smithi018.stdout: "num_ops": 0 2023-09-23T20:45:17.931 INFO:teuthology.orchestra.run.smithi018.stdout:} 2023-09-23T20:45:17.956 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:45:17.959 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:45:17.959 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:17.959 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:45:17.979 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:45:17.998 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:45:18.020 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:45:18.021 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:45:18.021 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:45:18.021 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:45:18.022 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:45:18.022 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:45:18.022 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:45:18.022 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:45:18.022 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:45:18.060 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:45:18.092 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:18.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:17 smithi149 ceph-mon[170394]: pgmap v1314: 119 pgs: 119 active+clean; 135 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 900 op/s 2023-09-23T20:45:18.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:17 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2982809441' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:45:18.455 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:45:18.460 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:18.461 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:45:18.561 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:45:18.591 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:18.627 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:45:18.630 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:18.630 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:45:18.691 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T20:45:18.731 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:45:18.757 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:18.790 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:45:18.793 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:18.794 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:45:18.895 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:45:18.921 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:18.954 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:45:18.956 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:18.957 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:45:18.961 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:45:19.058 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:45:19.087 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:19.178 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:45:19.181 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:19.181 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:45:19.282 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:45:19.320 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:19.599 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:45:19.689 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:45:19.692 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:19.693 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:45:19.794 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:45:19.824 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:19.860 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:45:19.863 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:19.864 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:45:19.964 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:45:19.992 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:20.006 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:19 smithi018 ceph-mon[179647]: pgmap v1315: 119 pgs: 119 active+clean; 47 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 822 op/s 2023-09-23T20:45:20.006 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:19 smithi018 ceph-mon[184144]: pgmap v1315: 119 pgs: 119 active+clean; 47 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 822 op/s 2023-09-23T20:45:20.026 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:45:20.028 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:20.029 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:45:20.130 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:45:20.165 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:20.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:19 smithi149 ceph-mon[170394]: pgmap v1315: 119 pgs: 119 active+clean; 47 MiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 822 op/s 2023-09-23T20:45:20.219 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:45:20.222 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:20.223 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:45:20.324 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:45:20.357 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:20.393 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:45:20.396 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:20.396 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:45:20.497 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:45:20.527 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:20.560 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:45:20.563 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:20.564 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:45:20.585 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:45:20.664 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:45:20.690 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:20.723 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:45:20.726 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:20.727 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:45:20.795 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:45:20.797 INFO:teuthology.orchestra.run.smithi018.stderr:dumped all 2023-09-23T20:45:20.827 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:45:20.858 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:20.883 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:20 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/9036109' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:45:20.883 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:20 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:45:20.883 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:20 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/9036109' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:45:20.883 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:20 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:45:20.950 INFO:teuthology.orchestra.run.smithi018.stderr:noscrub is unset 2023-09-23T20:45:21.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:20 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/9036109' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:45:21.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:20 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:45:21.295 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:45:21.298 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:21.299 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:45:21.399 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:45:21.429 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:21.838 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:45:21.841 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:21.841 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:45:21.867 INFO:teuthology.orchestra.run.smithi018.stdout:{"pg_ready":true,"pg_map":{"version":1316,"stamp":"2023-09-23T20:45:20.226448+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790195,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":5984,"num_read_kb":13201,"num_write":228571,"num_write_kb":7297910,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":17055,"num_bytes_recovered":1122650190,"num_keys_recovered":40,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":109281,"ondisk_log_size":109281,"up":357,"acting":357,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":361,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":5271028,"kb_used_data":295924,"kb_used_omap":32,"kb_used_meta":4975007,"kb_avail":744690188,"statfs":{"total":767960285184,"available":762562752512,"internally_reserved":0,"allocated":303026176,"data_stored":286687803,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32773,"internal_metadata":5094408187},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":7,"apply_latency_ms":7,"commit_latency_ns":7000000,"apply_latency_ns":7000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-583860248,"num_objects":-8910,"num_object_clones":0,"num_object_copies":-26730,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-8910,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8910,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003545"},"pg_stats":[{"pgid":"1.18","version":"90'161","reported_seq":568,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.864220+0000","last_change":"2023-09-23T20:44:19.677357+0000","last_active":"2023-09-23T20:45:01.864220+0000","last_peered":"2023-09-23T20:45:01.864220+0000","last_clean":"2023-09-23T20:45:01.864220+0000","last_became_active":"2023-09-23T20:43:16.766305+0000","last_became_peered":"2023-09-23T20:43:16.766305+0000","last_unstale":"2023-09-23T20:45:01.864220+0000","last_undegraded":"2023-09-23T20:45:01.864220+0000","last_fullsized":"2023-09-23T20:45:01.864220+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:19.677209+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:19.677209+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:14:06.510115+0000","scrub_duration":0.0023292260000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"69'1","reported_seq":220,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.853993+0000","last_change":"2023-09-23T20:32:33.233853+0000","last_active":"2023-09-23T20:45:01.853993+0000","last_peered":"2023-09-23T20:45:01.853993+0000","last_clean":"2023-09-23T20:45:01.853993+0000","last_became_active":"2023-09-23T20:32:33.233546+0000","last_became_peered":"2023-09-23T20:32:33.233546+0000","last_unstale":"2023-09-23T20:45:01.853993+0000","last_undegraded":"2023-09-23T20:45:01.853993+0000","last_fullsized":"2023-09-23T20:45:01.853993+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:50:27.130808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":338,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-09-23T20:45:07.909292+0000","last_change":"2023-09-23T20:37:43.579611+0000","last_active":"2023-09-23T20:45:07.909292+0000","last_peered":"2023-09-23T20:45:07.909292+0000","last_clean":"2023-09-23T20:45:07.909292+0000","last_became_active":"2023-09-23T20:37:43.579323+0000","last_became_peered":"2023-09-23T20:37:43.579323+0000","last_unstale":"2023-09-23T20:45:07.909292+0000","last_undegraded":"2023-09-23T20:45:07.909292+0000","last_fullsized":"2023-09-23T20:45:07.909292+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:17:32.311427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"90'161","reported_seq":644,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.853969+0000","last_change":"2023-09-23T20:44:14.647447+0000","last_active":"2023-09-23T20:45:01.853969+0000","last_peered":"2023-09-23T20:45:01.853969+0000","last_clean":"2023-09-23T20:45:01.853969+0000","last_became_active":"2023-09-23T20:43:16.756116+0000","last_became_peered":"2023-09-23T20:43:16.756116+0000","last_unstale":"2023-09-23T20:45:01.853969+0000","last_undegraded":"2023-09-23T20:45:01.853969+0000","last_fullsized":"2023-09-23T20:45:01.853969+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:14.647386+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:36:05.442288+0000","last_clean_scrub_stamp":"2023-09-23T20:44:14.647386+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:25:25.151100+0000","scrub_duration":0.0097381210000000006,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":493,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.860051+0000","last_change":"2023-09-23T20:34:59.983035+0000","last_active":"2023-09-23T20:45:01.860051+0000","last_peered":"2023-09-23T20:45:01.860051+0000","last_clean":"2023-09-23T20:45:01.860051+0000","last_became_active":"2023-09-23T20:34:59.982459+0000","last_became_peered":"2023-09-23T20:34:59.982459+0000","last_unstale":"2023-09-23T20:45:01.860051+0000","last_undegraded":"2023-09-23T20:45:01.860051+0000","last_fullsized":"2023-09-23T20:45:01.860051+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:16:48.510783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":480,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.869841+0000","last_change":"2023-09-23T20:43:46.521939+0000","last_active":"2023-09-23T20:45:01.869841+0000","last_peered":"2023-09-23T20:45:01.869841+0000","last_clean":"2023-09-23T20:45:01.869841+0000","last_became_active":"2023-09-23T20:43:46.521185+0000","last_became_peered":"2023-09-23T20:43:46.521185+0000","last_unstale":"2023-09-23T20:45:01.869841+0000","last_undegraded":"2023-09-23T20:45:01.869841+0000","last_fullsized":"2023-09-23T20:45:01.869841+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:59:25.045627+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"90'161","reported_seq":602,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.853468+0000","last_change":"2023-09-23T20:36:01.292103+0000","last_active":"2023-09-23T20:45:01.853468+0000","last_peered":"2023-09-23T20:45:01.853468+0000","last_clean":"2023-09-23T20:45:01.853468+0000","last_became_active":"2023-09-23T20:35:26.639912+0000","last_became_peered":"2023-09-23T20:35:26.639912+0000","last_unstale":"2023-09-23T20:45:01.853468+0000","last_undegraded":"2023-09-23T20:45:01.853468+0000","last_fullsized":"2023-09-23T20:45:01.853468+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:01.292032+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:01.292032+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:04:53.926640+0000","scrub_duration":0.0032983159999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":318,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.863818+0000","last_change":"2023-09-23T20:37:43.584414+0000","last_active":"2023-09-23T20:45:01.863818+0000","last_peered":"2023-09-23T20:45:01.863818+0000","last_clean":"2023-09-23T20:45:01.863818+0000","last_became_active":"2023-09-23T20:37:43.584116+0000","last_became_peered":"2023-09-23T20:37:43.584116+0000","last_unstale":"2023-09-23T20:45:01.863818+0000","last_undegraded":"2023-09-23T20:45:01.863818+0000","last_fullsized":"2023-09-23T20:45:01.863818+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:17:54.258172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":138,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.874185+0000","last_change":"2023-09-23T20:38:37.457427+0000","last_active":"2023-09-23T20:45:01.874185+0000","last_peered":"2023-09-23T20:45:01.874185+0000","last_clean":"2023-09-23T20:45:01.874185+0000","last_became_active":"2023-09-23T20:38:37.457099+0000","last_became_peered":"2023-09-23T20:38:37.457099+0000","last_unstale":"2023-09-23T20:45:01.874185+0000","last_undegraded":"2023-09-23T20:45:01.874185+0000","last_fullsized":"2023-09-23T20:45:01.874185+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:31:27.084994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"90'161","reported_seq":735,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.862330+0000","last_change":"2023-09-23T20:40:22.449754+0000","last_active":"2023-09-23T20:45:01.862330+0000","last_peered":"2023-09-23T20:45:01.862330+0000","last_clean":"2023-09-23T20:45:01.862330+0000","last_became_active":"2023-09-23T20:40:22.448838+0000","last_became_peered":"2023-09-23T20:40:22.448838+0000","last_unstale":"2023-09-23T20:45:01.862330+0000","last_undegraded":"2023-09-23T20:45:01.862330+0000","last_fullsized":"2023-09-23T20:45:01.862330+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:03.411025+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:03.411025+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:22:07.948856+0000","scrub_duration":0.001802922,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":513,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.870254+0000","last_change":"2023-09-23T20:43:46.522325+0000","last_active":"2023-09-23T20:45:01.870254+0000","last_peered":"2023-09-23T20:45:01.870254+0000","last_clean":"2023-09-23T20:45:01.870254+0000","last_became_active":"2023-09-23T20:43:46.521996+0000","last_became_peered":"2023-09-23T20:43:46.521996+0000","last_unstale":"2023-09-23T20:45:01.870254+0000","last_undegraded":"2023-09-23T20:45:01.870254+0000","last_fullsized":"2023-09-23T20:45:01.870254+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:51:17.629956+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":248,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.864368+0000","last_change":"2023-09-23T20:38:02.579343+0000","last_active":"2023-09-23T20:45:01.864368+0000","last_peered":"2023-09-23T20:45:01.864368+0000","last_clean":"2023-09-23T20:45:01.864368+0000","last_became_active":"2023-09-23T20:38:02.579047+0000","last_became_peered":"2023-09-23T20:38:02.579047+0000","last_unstale":"2023-09-23T20:45:01.864368+0000","last_undegraded":"2023-09-23T20:45:01.864368+0000","last_fullsized":"2023-09-23T20:45:01.864368+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:44:03.166556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.14","version":"90'161","reported_seq":854,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.864613+0000","last_change":"2023-09-23T20:35:02.236398+0000","last_active":"2023-09-23T20:45:01.864613+0000","last_peered":"2023-09-23T20:45:01.864613+0000","last_clean":"2023-09-23T20:45:01.864613+0000","last_became_active":"2023-09-23T20:35:02.235988+0000","last_became_peered":"2023-09-23T20:35:02.235988+0000","last_unstale":"2023-09-23T20:45:01.864613+0000","last_undegraded":"2023-09-23T20:45:01.864613+0000","last_fullsized":"2023-09-23T20:45:01.864613+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:47.479012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:47.479012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:13.519648+0000","scrub_duration":0.0020136590000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":359,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.861964+0000","last_change":"2023-09-23T20:43:16.780478+0000","last_active":"2023-09-23T20:45:01.861964+0000","last_peered":"2023-09-23T20:45:01.861964+0000","last_clean":"2023-09-23T20:45:01.861964+0000","last_became_active":"2023-09-23T20:43:16.780183+0000","last_became_peered":"2023-09-23T20:43:16.780183+0000","last_unstale":"2023-09-23T20:45:01.861964+0000","last_undegraded":"2023-09-23T20:45:01.861964+0000","last_fullsized":"2023-09-23T20:45:01.861964+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:38:30.102066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":349,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.856672+0000","last_change":"2023-09-23T20:38:02.542034+0000","last_active":"2023-09-23T20:45:01.856672+0000","last_peered":"2023-09-23T20:45:01.856672+0000","last_clean":"2023-09-23T20:45:01.856672+0000","last_became_active":"2023-09-23T20:38:02.541575+0000","last_became_peered":"2023-09-23T20:38:02.541575+0000","last_unstale":"2023-09-23T20:45:01.856672+0000","last_undegraded":"2023-09-23T20:45:01.856672+0000","last_fullsized":"2023-09-23T20:45:01.856672+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:50:54.591529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"90'161","reported_seq":648,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.870056+0000","last_change":"2023-09-23T20:44:14.669973+0000","last_active":"2023-09-23T20:45:01.870056+0000","last_peered":"2023-09-23T20:45:01.870056+0000","last_clean":"2023-09-23T20:45:01.870056+0000","last_became_active":"2023-09-23T20:43:46.519592+0000","last_became_peered":"2023-09-23T20:43:46.519592+0000","last_unstale":"2023-09-23T20:45:01.870056+0000","last_undegraded":"2023-09-23T20:45:01.870056+0000","last_fullsized":"2023-09-23T20:45:01.870056+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:14.669831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:14.669831+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:56:28.076617+0000","scrub_duration":0.0083441460000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":550,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.864808+0000","last_change":"2023-09-23T20:32:33.270617+0000","last_active":"2023-09-23T20:45:01.864808+0000","last_peered":"2023-09-23T20:45:01.864808+0000","last_clean":"2023-09-23T20:45:01.864808+0000","last_became_active":"2023-09-23T20:32:33.227902+0000","last_became_peered":"2023-09-23T20:32:33.227902+0000","last_unstale":"2023-09-23T20:45:01.864808+0000","last_undegraded":"2023-09-23T20:45:01.864808+0000","last_fullsized":"2023-09-23T20:45:01.864808+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:30:26.847831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":360,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":37,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.860211+0000","last_change":"2023-09-23T20:44:10.611381+0000","last_active":"2023-09-23T20:45:01.860211+0000","last_peered":"2023-09-23T20:45:01.860211+0000","last_clean":"2023-09-23T20:45:01.860211+0000","last_became_active":"2023-09-23T20:44:10.611091+0000","last_became_peered":"2023-09-23T20:44:10.611091+0000","last_unstale":"2023-09-23T20:45:01.860211+0000","last_undegraded":"2023-09-23T20:45:01.860211+0000","last_fullsized":"2023-09-23T20:45:01.860211+0000","mapping_epoch":496,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":497,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:00:30.162478+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"90'161","reported_seq":649,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-09-23T20:45:07.931294+0000","last_change":"2023-09-23T20:36:02.548387+0000","last_active":"2023-09-23T20:45:07.931294+0000","last_peered":"2023-09-23T20:45:07.931294+0000","last_clean":"2023-09-23T20:45:07.931294+0000","last_became_active":"2023-09-23T20:35:26.644787+0000","last_became_peered":"2023-09-23T20:35:26.644787+0000","last_unstale":"2023-09-23T20:45:07.931294+0000","last_undegraded":"2023-09-23T20:45:07.931294+0000","last_fullsized":"2023-09-23T20:45:07.931294+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:02.548285+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:02.548285+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:37:15.216553+0000","scrub_duration":0.0015514459999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":270,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.868058+0000","last_change":"2023-09-23T20:38:02.560646+0000","last_active":"2023-09-23T20:45:01.868058+0000","last_peered":"2023-09-23T20:45:01.868058+0000","last_clean":"2023-09-23T20:45:01.868058+0000","last_became_active":"2023-09-23T20:38:02.539814+0000","last_became_peered":"2023-09-23T20:38:02.539814+0000","last_unstale":"2023-09-23T20:45:01.868058+0000","last_undegraded":"2023-09-23T20:45:01.868058+0000","last_fullsized":"2023-09-23T20:45:01.868058+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:00:09.435023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":250,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.854485+0000","last_change":"2023-09-23T20:38:02.579222+0000","last_active":"2023-09-23T20:45:01.854485+0000","last_peered":"2023-09-23T20:45:01.854485+0000","last_clean":"2023-09-23T20:45:01.854485+0000","last_became_active":"2023-09-23T20:38:02.578813+0000","last_became_peered":"2023-09-23T20:38:02.578813+0000","last_unstale":"2023-09-23T20:45:01.854485+0000","last_undegraded":"2023-09-23T20:45:01.854485+0000","last_fullsized":"2023-09-23T20:45:01.854485+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:57:31.080027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"90'161","reported_seq":776,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.872703+0000","last_change":"2023-09-23T20:43:46.544727+0000","last_active":"2023-09-23T20:45:01.872703+0000","last_peered":"2023-09-23T20:45:01.872703+0000","last_clean":"2023-09-23T20:45:01.872703+0000","last_became_active":"2023-09-23T20:43:46.543952+0000","last_became_peered":"2023-09-23T20:43:46.543952+0000","last_unstale":"2023-09-23T20:45:01.872703+0000","last_undegraded":"2023-09-23T20:45:01.872703+0000","last_fullsized":"2023-09-23T20:45:01.872703+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:43.578666+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:43.578666+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:11:16.611920+0000","scrub_duration":0.0021890379999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":549,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.861928+0000","last_change":"2023-09-23T20:32:33.242943+0000","last_active":"2023-09-23T20:45:01.861928+0000","last_peered":"2023-09-23T20:45:01.861928+0000","last_clean":"2023-09-23T20:45:01.861928+0000","last_became_active":"2023-09-23T20:32:33.225468+0000","last_became_peered":"2023-09-23T20:32:33.225468+0000","last_unstale":"2023-09-23T20:45:01.861928+0000","last_undegraded":"2023-09-23T20:45:01.861928+0000","last_fullsized":"2023-09-23T20:45:01.861928+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:40:47.867573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":269,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.877123+0000","last_change":"2023-09-23T20:45:01.875931+0000","last_active":"2023-09-23T20:45:01.877123+0000","last_peered":"2023-09-23T20:45:01.877123+0000","last_clean":"2023-09-23T20:45:01.877123+0000","last_became_active":"2023-09-23T20:45:01.872839+0000","last_became_peered":"2023-09-23T20:45:01.872839+0000","last_unstale":"2023-09-23T20:45:01.877123+0000","last_undegraded":"2023-09-23T20:45:01.877123+0000","last_fullsized":"2023-09-23T20:45:01.877123+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:15:09.584507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"90'161","reported_seq":803,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.861461+0000","last_change":"2023-09-23T20:44:20.041766+0000","last_active":"2023-09-23T20:45:01.861461+0000","last_peered":"2023-09-23T20:45:01.861461+0000","last_clean":"2023-09-23T20:45:01.861461+0000","last_became_active":"2023-09-23T20:43:16.765015+0000","last_became_peered":"2023-09-23T20:43:16.765015+0000","last_unstale":"2023-09-23T20:45:01.861461+0000","last_undegraded":"2023-09-23T20:45:01.861461+0000","last_fullsized":"2023-09-23T20:45:01.861461+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:20.041690+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:20.041690+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:33:30.150109+0000","scrub_duration":0.0026180330000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":478,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.869359+0000","last_change":"2023-09-23T20:43:46.521294+0000","last_active":"2023-09-23T20:45:01.869359+0000","last_peered":"2023-09-23T20:45:01.869359+0000","last_clean":"2023-09-23T20:45:01.869359+0000","last_became_active":"2023-09-23T20:43:46.521001+0000","last_became_peered":"2023-09-23T20:43:46.521001+0000","last_unstale":"2023-09-23T20:45:01.869359+0000","last_undegraded":"2023-09-23T20:45:01.869359+0000","last_fullsized":"2023-09-23T20:45:01.869359+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:01:35.126716+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":290,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.864922+0000","last_change":"2023-09-23T20:44:01.841565+0000","last_active":"2023-09-23T20:45:01.864922+0000","last_peered":"2023-09-23T20:45:01.864922+0000","last_clean":"2023-09-23T20:45:01.864922+0000","last_became_active":"2023-09-23T20:44:01.841255+0000","last_became_peered":"2023-09-23T20:44:01.841255+0000","last_unstale":"2023-09-23T20:45:01.864922+0000","last_undegraded":"2023-09-23T20:45:01.864922+0000","last_fullsized":"2023-09-23T20:45:01.864922+0000","mapping_epoch":491,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":492,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:18:44.817589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.13","version":"90'161","reported_seq":810,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.860176+0000","last_change":"2023-09-23T20:35:02.238923+0000","last_active":"2023-09-23T20:45:01.860176+0000","last_peered":"2023-09-23T20:45:01.860176+0000","last_clean":"2023-09-23T20:45:01.860176+0000","last_became_active":"2023-09-23T20:35:02.238293+0000","last_became_peered":"2023-09-23T20:35:02.238293+0000","last_unstale":"2023-09-23T20:45:01.860176+0000","last_undegraded":"2023-09-23T20:45:01.860176+0000","last_fullsized":"2023-09-23T20:45:01.860176+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:42.544308+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:42.544308+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:51:22.281932+0000","scrub_duration":0.0029958179999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":526,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.861681+0000","last_change":"2023-09-23T20:38:02.539774+0000","last_active":"2023-09-23T20:45:01.861681+0000","last_peered":"2023-09-23T20:45:01.861681+0000","last_clean":"2023-09-23T20:45:01.861681+0000","last_became_active":"2023-09-23T20:38:02.539436+0000","last_became_peered":"2023-09-23T20:38:02.539436+0000","last_unstale":"2023-09-23T20:45:01.861681+0000","last_undegraded":"2023-09-23T20:45:01.861681+0000","last_fullsized":"2023-09-23T20:45:01.861681+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:50:57.375803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":487,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.869906+0000","last_change":"2023-09-23T20:43:46.521735+0000","last_active":"2023-09-23T20:45:01.869906+0000","last_peered":"2023-09-23T20:45:01.869906+0000","last_clean":"2023-09-23T20:45:01.869906+0000","last_became_active":"2023-09-23T20:43:46.520762+0000","last_became_peered":"2023-09-23T20:43:46.520762+0000","last_unstale":"2023-09-23T20:45:01.869906+0000","last_undegraded":"2023-09-23T20:45:01.869906+0000","last_fullsized":"2023-09-23T20:45:01.869906+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:26:48.841930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"90'161","reported_seq":606,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.860851+0000","last_change":"2023-09-23T20:44:18.952808+0000","last_active":"2023-09-23T20:45:01.860851+0000","last_peered":"2023-09-23T20:45:01.860851+0000","last_clean":"2023-09-23T20:45:01.860851+0000","last_became_active":"2023-09-23T20:43:16.758652+0000","last_became_peered":"2023-09-23T20:43:16.758652+0000","last_unstale":"2023-09-23T20:45:01.860851+0000","last_undegraded":"2023-09-23T20:45:01.860851+0000","last_fullsized":"2023-09-23T20:45:01.860851+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:18.952693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:18.952693+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:12:19.496465+0000","scrub_duration":0.001699365,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":198,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.873520+0000","last_change":"2023-09-23T20:44:43.644290+0000","last_active":"2023-09-23T20:45:01.873520+0000","last_peered":"2023-09-23T20:45:01.873520+0000","last_clean":"2023-09-23T20:45:01.873520+0000","last_became_active":"2023-09-23T20:44:43.638313+0000","last_became_peered":"2023-09-23T20:44:43.638313+0000","last_unstale":"2023-09-23T20:45:01.873520+0000","last_undegraded":"2023-09-23T20:45:01.873520+0000","last_fullsized":"2023-09-23T20:45:01.873520+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:32:52.892603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":318,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.859194+0000","last_change":"2023-09-23T20:37:43.571083+0000","last_active":"2023-09-23T20:45:01.859194+0000","last_peered":"2023-09-23T20:45:01.859194+0000","last_clean":"2023-09-23T20:45:01.859194+0000","last_became_active":"2023-09-23T20:37:43.570306+0000","last_became_peered":"2023-09-23T20:37:43.570306+0000","last_unstale":"2023-09-23T20:45:01.859194+0000","last_undegraded":"2023-09-23T20:45:01.859194+0000","last_fullsized":"2023-09-23T20:45:01.859194+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:42:02.396718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.5","version":"523'13986","reported_seq":22549,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-09-23T20:45:15.517431+0000","last_change":"2023-09-23T20:44:20.680609+0000","last_active":"2023-09-23T20:45:15.517431+0000","last_peered":"2023-09-23T20:45:15.517431+0000","last_clean":"2023-09-23T20:45:15.517431+0000","last_became_active":"2023-09-23T20:44:20.638636+0000","last_became_peered":"2023-09-23T20:44:20.638636+0000","last_unstale":"2023-09-23T20:45:15.517431+0000","last_undegraded":"2023-09-23T20:45:15.517431+0000","last_fullsized":"2023-09-23T20:45:15.517431+0000","mapping_epoch":504,"log_start":"468'6410","ondisk_log_start":"468'6410","created":455,"last_epoch_clean":505,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":7576,"log_dups_size":0,"ondisk_log_size":7576,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:11:11.154742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13986,"num_write_kb":447552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4204,"num_bytes_recovered":275316736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.d","version":"90'161","reported_seq":815,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.864565+0000","last_change":"2023-09-23T20:34:59.496438+0000","last_active":"2023-09-23T20:45:01.864565+0000","last_peered":"2023-09-23T20:45:01.864565+0000","last_clean":"2023-09-23T20:45:01.864565+0000","last_became_active":"2023-09-23T20:33:38.970527+0000","last_became_peered":"2023-09-23T20:33:38.970527+0000","last_unstale":"2023-09-23T20:45:01.864565+0000","last_undegraded":"2023-09-23T20:45:01.864565+0000","last_fullsized":"2023-09-23T20:45:01.864565+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:34:59.496375+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:34:59.496375+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:46:18.244734+0000","scrub_duration":0.0017843189999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":526,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.862076+0000","last_change":"2023-09-23T20:32:33.283752+0000","last_active":"2023-09-23T20:45:01.862076+0000","last_peered":"2023-09-23T20:45:01.862076+0000","last_clean":"2023-09-23T20:45:01.862076+0000","last_became_active":"2023-09-23T20:32:33.230503+0000","last_became_peered":"2023-09-23T20:32:33.230503+0000","last_unstale":"2023-09-23T20:45:01.862076+0000","last_undegraded":"2023-09-23T20:45:01.862076+0000","last_fullsized":"2023-09-23T20:45:01.862076+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:29.084410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":337,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.861693+0000","last_change":"2023-09-23T20:43:16.765253+0000","last_active":"2023-09-23T20:45:01.861693+0000","last_peered":"2023-09-23T20:45:01.861693+0000","last_clean":"2023-09-23T20:45:01.861693+0000","last_became_active":"2023-09-23T20:43:16.764978+0000","last_became_peered":"2023-09-23T20:43:16.764978+0000","last_unstale":"2023-09-23T20:45:01.861693+0000","last_undegraded":"2023-09-23T20:45:01.861693+0000","last_fullsized":"2023-09-23T20:45:01.861693+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:07:23.539372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.4","version":"523'14310","reported_seq":14389,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-09-23T20:45:15.543518+0000","last_change":"2023-09-23T20:41:04.575200+0000","last_active":"2023-09-23T20:45:15.543518+0000","last_peered":"2023-09-23T20:45:15.543518+0000","last_clean":"2023-09-23T20:45:15.543518+0000","last_became_active":"2023-09-23T20:41:04.574824+0000","last_became_peered":"2023-09-23T20:41:04.574824+0000","last_unstale":"2023-09-23T20:45:15.543518+0000","last_undegraded":"2023-09-23T20:45:15.543518+0000","last_fullsized":"2023-09-23T20:45:15.543518+0000","mapping_epoch":455,"log_start":"479'8583","ondisk_log_start":"479'8583","created":455,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":5727,"log_dups_size":0,"ondisk_log_size":5727,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:50:46.714383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14310,"num_write_kb":457920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.c","version":"90'161","reported_seq":784,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.862950+0000","last_change":"2023-09-23T20:41:09.506481+0000","last_active":"2023-09-23T20:45:01.862950+0000","last_peered":"2023-09-23T20:45:01.862950+0000","last_clean":"2023-09-23T20:45:01.862950+0000","last_became_active":"2023-09-23T20:38:28.442229+0000","last_became_peered":"2023-09-23T20:38:28.442229+0000","last_unstale":"2023-09-23T20:45:01.862950+0000","last_undegraded":"2023-09-23T20:45:01.862950+0000","last_fullsized":"2023-09-23T20:45:01.862950+0000","mapping_epoch":411,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":412,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:41:09.506338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:41:09.506338+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:19:46.265011+0000","scrub_duration":0.01112002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":561,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.862045+0000","last_change":"2023-09-23T20:34:59.987981+0000","last_active":"2023-09-23T20:45:01.862045+0000","last_peered":"2023-09-23T20:45:01.862045+0000","last_clean":"2023-09-23T20:45:01.862045+0000","last_became_active":"2023-09-23T20:34:59.985368+0000","last_became_peered":"2023-09-23T20:34:59.985368+0000","last_unstale":"2023-09-23T20:45:01.862045+0000","last_undegraded":"2023-09-23T20:45:01.862045+0000","last_fullsized":"2023-09-23T20:45:01.862045+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:12:43.082493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":220,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.853755+0000","last_change":"2023-09-23T20:32:33.232142+0000","last_active":"2023-09-23T20:45:01.853755+0000","last_peered":"2023-09-23T20:45:01.853755+0000","last_clean":"2023-09-23T20:45:01.853755+0000","last_became_active":"2023-09-23T20:32:33.231683+0000","last_became_peered":"2023-09-23T20:32:33.231683+0000","last_unstale":"2023-09-23T20:45:01.853755+0000","last_undegraded":"2023-09-23T20:45:01.853755+0000","last_fullsized":"2023-09-23T20:45:01.853755+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:02:58.336348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"523'14176","reported_seq":14264,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-09-23T20:45:15.481917+0000","last_change":"2023-09-23T20:43:16.854027+0000","last_active":"2023-09-23T20:45:15.481917+0000","last_peered":"2023-09-23T20:45:15.481917+0000","last_clean":"2023-09-23T20:45:15.481917+0000","last_became_active":"2023-09-23T20:43:16.766133+0000","last_became_peered":"2023-09-23T20:43:16.766133+0000","last_unstale":"2023-09-23T20:45:15.481917+0000","last_undegraded":"2023-09-23T20:45:15.481917+0000","last_fullsized":"2023-09-23T20:45:15.481917+0000","mapping_epoch":481,"log_start":"479'8485","ondisk_log_start":"479'8485","created":455,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":5691,"log_dups_size":0,"ondisk_log_size":5691,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:42:32.213805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14176,"num_write_kb":453632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.f","version":"90'161","reported_seq":678,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.874398+0000","last_change":"2023-09-23T20:38:40.491902+0000","last_active":"2023-09-23T20:45:01.874398+0000","last_peered":"2023-09-23T20:45:01.874398+0000","last_clean":"2023-09-23T20:45:01.874398+0000","last_became_active":"2023-09-23T20:38:40.491563+0000","last_became_peered":"2023-09-23T20:38:40.491563+0000","last_unstale":"2023-09-23T20:45:01.874398+0000","last_undegraded":"2023-09-23T20:45:01.874398+0000","last_fullsized":"2023-09-23T20:45:01.874398+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:46.217489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:46.217489+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:39:25.008796+0000","scrub_duration":0.001456429,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":551,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.864652+0000","last_change":"2023-09-23T20:40:19.677655+0000","last_active":"2023-09-23T20:45:01.864652+0000","last_peered":"2023-09-23T20:45:01.864652+0000","last_clean":"2023-09-23T20:45:01.864652+0000","last_became_active":"2023-09-23T20:40:19.551189+0000","last_became_peered":"2023-09-23T20:40:19.551189+0000","last_unstale":"2023-09-23T20:45:01.864652+0000","last_undegraded":"2023-09-23T20:45:01.864652+0000","last_fullsized":"2023-09-23T20:45:01.864652+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:11:01.559576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":128,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.873594+0000","last_change":"2023-09-23T20:38:02.590613+0000","last_active":"2023-09-23T20:45:01.873594+0000","last_peered":"2023-09-23T20:45:01.873594+0000","last_clean":"2023-09-23T20:45:01.873594+0000","last_became_active":"2023-09-23T20:38:02.590284+0000","last_became_peered":"2023-09-23T20:38:02.590284+0000","last_unstale":"2023-09-23T20:45:01.873594+0000","last_undegraded":"2023-09-23T20:45:01.873594+0000","last_fullsized":"2023-09-23T20:45:01.873594+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:40:07.825054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"523'14110","reported_seq":14194,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-09-23T20:45:15.483714+0000","last_change":"2023-09-23T20:43:16.788249+0000","last_active":"2023-09-23T20:45:15.483714+0000","last_peered":"2023-09-23T20:45:15.483714+0000","last_clean":"2023-09-23T20:45:15.483714+0000","last_became_active":"2023-09-23T20:43:16.763555+0000","last_became_peered":"2023-09-23T20:43:16.763555+0000","last_unstale":"2023-09-23T20:45:15.483714+0000","last_undegraded":"2023-09-23T20:45:15.483714+0000","last_fullsized":"2023-09-23T20:45:15.483714+0000","mapping_epoch":481,"log_start":"464'4404","ondisk_log_start":"464'4404","created":455,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":9706,"log_dups_size":0,"ondisk_log_size":9706,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:19:57.856019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14110,"num_write_kb":451520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[2,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.e","version":"90'161","reported_seq":623,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.867508+0000","last_change":"2023-09-23T20:36:00.322453+0000","last_active":"2023-09-23T20:45:01.867508+0000","last_peered":"2023-09-23T20:45:01.867508+0000","last_clean":"2023-09-23T20:45:01.867508+0000","last_became_active":"2023-09-23T20:35:02.233536+0000","last_became_peered":"2023-09-23T20:35:02.233536+0000","last_unstale":"2023-09-23T20:45:01.867508+0000","last_undegraded":"2023-09-23T20:45:01.867508+0000","last_fullsized":"2023-09-23T20:45:01.867508+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:00.322381+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:00.322381+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:02:15.489069+0000","scrub_duration":0.0017321859999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":482,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.863283+0000","last_change":"2023-09-23T20:43:16.756437+0000","last_active":"2023-09-23T20:45:01.863283+0000","last_peered":"2023-09-23T20:45:01.863283+0000","last_clean":"2023-09-23T20:45:01.863283+0000","last_became_active":"2023-09-23T20:43:16.756193+0000","last_became_peered":"2023-09-23T20:43:16.756193+0000","last_unstale":"2023-09-23T20:45:01.863283+0000","last_undegraded":"2023-09-23T20:45:01.863283+0000","last_fullsized":"2023-09-23T20:45:01.863283+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:33:25.632781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":496,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.873323+0000","last_change":"2023-09-23T20:43:46.541524+0000","last_active":"2023-09-23T20:45:01.873323+0000","last_peered":"2023-09-23T20:45:01.873323+0000","last_clean":"2023-09-23T20:45:01.873323+0000","last_became_active":"2023-09-23T20:43:46.541051+0000","last_became_peered":"2023-09-23T20:43:46.541051+0000","last_unstale":"2023-09-23T20:45:01.873323+0000","last_undegraded":"2023-09-23T20:45:01.873323+0000","last_fullsized":"2023-09-23T20:45:01.873323+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:59:36.632592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"523'14164","reported_seq":14240,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-09-23T20:45:15.505315+0000","last_change":"2023-09-23T20:41:04.558473+0000","last_active":"2023-09-23T20:45:15.505315+0000","last_peered":"2023-09-23T20:45:15.505315+0000","last_clean":"2023-09-23T20:45:15.505315+0000","last_became_active":"2023-09-23T20:41:04.558173+0000","last_became_peered":"2023-09-23T20:41:04.558173+0000","last_unstale":"2023-09-23T20:45:15.505315+0000","last_undegraded":"2023-09-23T20:45:15.505315+0000","last_fullsized":"2023-09-23T20:45:15.505315+0000","mapping_epoch":455,"log_start":"470'7163","ondisk_log_start":"470'7163","created":455,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":7001,"log_dups_size":0,"ondisk_log_size":7001,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:17:53.184877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14164,"num_write_kb":453248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":928,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.861615+0000","last_change":"2023-09-23T20:35:02.523308+0000","last_active":"2023-09-23T20:45:01.861615+0000","last_peered":"2023-09-23T20:45:01.861615+0000","last_clean":"2023-09-23T20:45:01.861615+0000","last_became_active":"2023-09-23T20:33:38.979989+0000","last_became_peered":"2023-09-23T20:33:38.979989+0000","last_unstale":"2023-09-23T20:45:01.861615+0000","last_undegraded":"2023-09-23T20:45:01.861615+0000","last_fullsized":"2023-09-23T20:45:01.861615+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:35:02.523203+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:35:02.523203+0000","last_clean_scrub_stamp":"2023-09-23T20:35:02.523203+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:02:41.472958+0000","scrub_duration":0.0085927989999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":344,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.857632+0000","last_change":"2023-09-23T20:37:43.583731+0000","last_active":"2023-09-23T20:45:01.857632+0000","last_peered":"2023-09-23T20:45:01.857632+0000","last_clean":"2023-09-23T20:45:01.857632+0000","last_became_active":"2023-09-23T20:37:43.583066+0000","last_became_peered":"2023-09-23T20:37:43.583066+0000","last_unstale":"2023-09-23T20:45:01.857632+0000","last_undegraded":"2023-09-23T20:45:01.857632+0000","last_fullsized":"2023-09-23T20:45:01.857632+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:03:45.211900+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":240,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.868003+0000","last_change":"2023-09-23T20:38:02.579814+0000","last_active":"2023-09-23T20:45:01.868003+0000","last_peered":"2023-09-23T20:45:01.868003+0000","last_clean":"2023-09-23T20:45:01.868003+0000","last_became_active":"2023-09-23T20:38:02.579253+0000","last_became_peered":"2023-09-23T20:38:02.579253+0000","last_unstale":"2023-09-23T20:45:01.868003+0000","last_undegraded":"2023-09-23T20:45:01.868003+0000","last_fullsized":"2023-09-23T20:45:01.868003+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:05:07.298198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"523'14264","reported_seq":14343,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-09-23T20:45:15.536205+0000","last_change":"2023-09-23T20:41:04.573101+0000","last_active":"2023-09-23T20:45:15.536205+0000","last_peered":"2023-09-23T20:45:15.536205+0000","last_clean":"2023-09-23T20:45:15.536205+0000","last_became_active":"2023-09-23T20:41:04.572728+0000","last_became_peered":"2023-09-23T20:41:04.572728+0000","last_unstale":"2023-09-23T20:45:15.536205+0000","last_undegraded":"2023-09-23T20:45:15.536205+0000","last_fullsized":"2023-09-23T20:45:15.536205+0000","mapping_epoch":455,"log_start":"479'8525","ondisk_log_start":"479'8525","created":455,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":5739,"log_dups_size":0,"ondisk_log_size":5739,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:21:10.721593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14264,"num_write_kb":456448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":638,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.864522+0000","last_change":"2023-09-23T20:44:46.725779+0000","last_active":"2023-09-23T20:45:01.864522+0000","last_peered":"2023-09-23T20:45:01.864522+0000","last_clean":"2023-09-23T20:45:01.864522+0000","last_became_active":"2023-09-23T20:44:46.725433+0000","last_became_peered":"2023-09-23T20:44:46.725433+0000","last_unstale":"2023-09-23T20:45:01.864522+0000","last_undegraded":"2023-09-23T20:45:01.864522+0000","last_fullsized":"2023-09-23T20:45:01.864522+0000","mapping_epoch":516,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":517,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:42:45.492220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:42:45.492220+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:28:35.202407+0000","scrub_duration":0.015259412,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":128,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.873641+0000","last_change":"2023-09-23T20:38:02.578703+0000","last_active":"2023-09-23T20:45:01.873641+0000","last_peered":"2023-09-23T20:45:01.873641+0000","last_clean":"2023-09-23T20:45:01.873641+0000","last_became_active":"2023-09-23T20:38:02.578414+0000","last_became_peered":"2023-09-23T20:38:02.578414+0000","last_unstale":"2023-09-23T20:45:01.873641+0000","last_undegraded":"2023-09-23T20:45:01.873641+0000","last_fullsized":"2023-09-23T20:45:01.873641+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:43:59.779295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":189,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.854604+0000","last_change":"2023-09-23T20:37:31.296606+0000","last_active":"2023-09-23T20:45:01.854604+0000","last_peered":"2023-09-23T20:45:01.854604+0000","last_clean":"2023-09-23T20:45:01.854604+0000","last_became_active":"2023-09-23T20:37:31.295924+0000","last_became_peered":"2023-09-23T20:37:31.295924+0000","last_unstale":"2023-09-23T20:45:01.854604+0000","last_undegraded":"2023-09-23T20:45:01.854604+0000","last_fullsized":"2023-09-23T20:45:01.854604+0000","mapping_epoch":391,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":392,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:27:50.948510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"523'14220","reported_seq":14296,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-09-23T20:45:15.517216+0000","last_change":"2023-09-23T20:41:04.555980+0000","last_active":"2023-09-23T20:45:15.517216+0000","last_peered":"2023-09-23T20:45:15.517216+0000","last_clean":"2023-09-23T20:45:15.517216+0000","last_became_active":"2023-09-23T20:41:04.555688+0000","last_became_peered":"2023-09-23T20:41:04.555688+0000","last_unstale":"2023-09-23T20:45:15.517216+0000","last_undegraded":"2023-09-23T20:45:15.517216+0000","last_fullsized":"2023-09-23T20:45:15.517216+0000","mapping_epoch":455,"log_start":"479'8525","ondisk_log_start":"479'8525","created":455,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":5695,"log_dups_size":0,"ondisk_log_size":5695,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:28:42.819404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14220,"num_write_kb":455040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.b","version":"90'161","reported_seq":826,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.860393+0000","last_change":"2023-09-23T20:40:22.445808+0000","last_active":"2023-09-23T20:45:01.860393+0000","last_peered":"2023-09-23T20:45:01.860393+0000","last_clean":"2023-09-23T20:45:01.860393+0000","last_became_active":"2023-09-23T20:40:22.445456+0000","last_became_peered":"2023-09-23T20:40:22.445456+0000","last_unstale":"2023-09-23T20:45:01.860393+0000","last_undegraded":"2023-09-23T20:45:01.860393+0000","last_fullsized":"2023-09-23T20:45:01.860393+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:05.595042+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:05.595042+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:39:23.678154+0000","scrub_duration":0.007271675,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":179,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.853533+0000","last_change":"2023-09-23T20:34:59.983122+0000","last_active":"2023-09-23T20:45:01.853533+0000","last_peered":"2023-09-23T20:45:01.853533+0000","last_clean":"2023-09-23T20:45:01.853533+0000","last_became_active":"2023-09-23T20:34:59.982809+0000","last_became_peered":"2023-09-23T20:34:59.982809+0000","last_unstale":"2023-09-23T20:45:01.853533+0000","last_undegraded":"2023-09-23T20:45:01.853533+0000","last_fullsized":"2023-09-23T20:45:01.853533+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:49:40.839723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":260,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.867603+0000","last_change":"2023-09-23T20:44:10.606995+0000","last_active":"2023-09-23T20:45:01.867603+0000","last_peered":"2023-09-23T20:45:01.867603+0000","last_clean":"2023-09-23T20:45:01.867603+0000","last_became_active":"2023-09-23T20:44:10.606718+0000","last_became_peered":"2023-09-23T20:44:10.606718+0000","last_unstale":"2023-09-23T20:45:01.867603+0000","last_undegraded":"2023-09-23T20:45:01.867603+0000","last_fullsized":"2023-09-23T20:45:01.867603+0000","mapping_epoch":496,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":497,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:13:12.496245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"523'14314","reported_seq":14393,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-09-23T20:45:15.530839+0000","last_change":"2023-09-23T20:41:04.573300+0000","last_active":"2023-09-23T20:45:15.530839+0000","last_peered":"2023-09-23T20:45:15.530839+0000","last_clean":"2023-09-23T20:45:15.530839+0000","last_became_active":"2023-09-23T20:41:04.572919+0000","last_became_peered":"2023-09-23T20:41:04.572919+0000","last_unstale":"2023-09-23T20:45:15.530839+0000","last_undegraded":"2023-09-23T20:45:15.530839+0000","last_fullsized":"2023-09-23T20:45:15.530839+0000","mapping_epoch":455,"log_start":"478'8294","ondisk_log_start":"478'8294","created":455,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":6020,"log_dups_size":0,"ondisk_log_size":6020,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:57:46.465243+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14314,"num_write_kb":458048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":548,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.864326+0000","last_change":"2023-09-23T20:43:51.693618+0000","last_active":"2023-09-23T20:45:01.864326+0000","last_peered":"2023-09-23T20:45:01.864326+0000","last_clean":"2023-09-23T20:45:01.864326+0000","last_became_active":"2023-09-23T20:43:16.765186+0000","last_became_peered":"2023-09-23T20:43:16.765186+0000","last_unstale":"2023-09-23T20:45:01.864326+0000","last_undegraded":"2023-09-23T20:45:01.864326+0000","last_fullsized":"2023-09-23T20:45:01.864326+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:43:51.693459+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:43:51.693459+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:33:30.208355+0000","scrub_duration":0.0016145269999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":318,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.863818+0000","last_change":"2023-09-23T20:37:43.570610+0000","last_active":"2023-09-23T20:45:01.863818+0000","last_peered":"2023-09-23T20:45:01.863818+0000","last_clean":"2023-09-23T20:45:01.863818+0000","last_became_active":"2023-09-23T20:37:43.570216+0000","last_became_peered":"2023-09-23T20:37:43.570216+0000","last_unstale":"2023-09-23T20:45:01.863818+0000","last_undegraded":"2023-09-23T20:45:01.863818+0000","last_fullsized":"2023-09-23T20:45:01.863818+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:39:26.555329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":348,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.861040+0000","last_change":"2023-09-23T20:32:33.230582+0000","last_active":"2023-09-23T20:45:01.861040+0000","last_peered":"2023-09-23T20:45:01.861040+0000","last_clean":"2023-09-23T20:45:01.861040+0000","last_became_active":"2023-09-23T20:32:33.230257+0000","last_became_peered":"2023-09-23T20:32:33.230257+0000","last_unstale":"2023-09-23T20:45:01.861040+0000","last_undegraded":"2023-09-23T20:45:01.861040+0000","last_fullsized":"2023-09-23T20:45:01.861040+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:11:45.333096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.f","version":"523'14386","reported_seq":14470,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-09-23T20:45:15.525488+0000","last_change":"2023-09-23T20:43:16.778725+0000","last_active":"2023-09-23T20:45:15.525488+0000","last_peered":"2023-09-23T20:45:15.525488+0000","last_clean":"2023-09-23T20:45:15.525488+0000","last_became_active":"2023-09-23T20:43:16.759443+0000","last_became_peered":"2023-09-23T20:43:16.759443+0000","last_unstale":"2023-09-23T20:45:15.525488+0000","last_undegraded":"2023-09-23T20:45:15.525488+0000","last_fullsized":"2023-09-23T20:45:15.525488+0000","mapping_epoch":481,"log_start":"479'8684","ondisk_log_start":"479'8684","created":455,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":5702,"log_dups_size":0,"ondisk_log_size":5702,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:55:32.141361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14386,"num_write_kb":460352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":711,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.874371+0000","last_change":"2023-09-23T20:41:07.424609+0000","last_active":"2023-09-23T20:45:01.874371+0000","last_peered":"2023-09-23T20:45:01.874371+0000","last_clean":"2023-09-23T20:45:01.874371+0000","last_became_active":"2023-09-23T20:38:05.622261+0000","last_became_peered":"2023-09-23T20:38:05.622261+0000","last_unstale":"2023-09-23T20:45:01.874371+0000","last_undegraded":"2023-09-23T20:45:01.874371+0000","last_fullsized":"2023-09-23T20:45:01.874371+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:41:07.424492+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:41:07.424492+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:45:21.689123+0000","scrub_duration":0.0018960959999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"86'119","reported_seq":833,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.868557+0000","last_change":"2023-09-23T20:29:45.521257+0000","last_active":"2023-09-23T20:45:01.868557+0000","last_peered":"2023-09-23T20:45:01.868557+0000","last_clean":"2023-09-23T20:45:01.868557+0000","last_became_active":"2023-09-23T20:29:45.520546+0000","last_became_peered":"2023-09-23T20:29:45.520546+0000","last_unstale":"2023-09-23T20:45:01.868557+0000","last_undegraded":"2023-09-23T20:45:01.868557+0000","last_fullsized":"2023-09-23T20:45:01.868557+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":119,"log_dups_size":0,"ondisk_log_size":119,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:58:56.085777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032705499999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":523,"num_read_kb":3503,"num_write":136,"num_write_kb":3128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":315,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.857325+0000","last_change":"2023-09-23T20:43:16.756494+0000","last_active":"2023-09-23T20:45:01.857325+0000","last_peered":"2023-09-23T20:45:01.857325+0000","last_clean":"2023-09-23T20:45:01.857325+0000","last_became_active":"2023-09-23T20:43:16.756131+0000","last_became_peered":"2023-09-23T20:43:16.756131+0000","last_unstale":"2023-09-23T20:45:01.857325+0000","last_undegraded":"2023-09-23T20:45:01.857325+0000","last_fullsized":"2023-09-23T20:45:01.857325+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:10:19.219533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":480,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.860537+0000","last_change":"2023-09-23T20:40:19.530942+0000","last_active":"2023-09-23T20:45:01.860537+0000","last_peered":"2023-09-23T20:45:01.860537+0000","last_clean":"2023-09-23T20:45:01.860537+0000","last_became_active":"2023-09-23T20:40:19.530617+0000","last_became_peered":"2023-09-23T20:40:19.530617+0000","last_unstale":"2023-09-23T20:45:01.860537+0000","last_undegraded":"2023-09-23T20:45:01.860537+0000","last_fullsized":"2023-09-23T20:45:01.860537+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:24:09.231934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.9","version":"523'14192","reported_seq":14288,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-09-23T20:45:15.542089+0000","last_change":"2023-09-23T20:43:46.544365+0000","last_active":"2023-09-23T20:45:15.542089+0000","last_peered":"2023-09-23T20:45:15.542089+0000","last_clean":"2023-09-23T20:45:15.542089+0000","last_became_active":"2023-09-23T20:43:46.519248+0000","last_became_peered":"2023-09-23T20:43:46.519248+0000","last_unstale":"2023-09-23T20:45:15.542089+0000","last_undegraded":"2023-09-23T20:45:15.542089+0000","last_fullsized":"2023-09-23T20:45:15.542089+0000","mapping_epoch":487,"log_start":"468'6634","ondisk_log_start":"468'6634","created":455,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":7558,"log_dups_size":0,"ondisk_log_size":7558,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:49:43.541062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14192,"num_write_kb":454144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":568,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.873045+0000","last_change":"2023-09-23T20:44:15.656984+0000","last_active":"2023-09-23T20:45:01.873045+0000","last_peered":"2023-09-23T20:45:01.873045+0000","last_clean":"2023-09-23T20:45:01.873045+0000","last_became_active":"2023-09-23T20:43:46.512210+0000","last_became_peered":"2023-09-23T20:43:46.512210+0000","last_unstale":"2023-09-23T20:45:01.873045+0000","last_undegraded":"2023-09-23T20:45:01.873045+0000","last_fullsized":"2023-09-23T20:45:01.873045+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:15.656880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:15.656880+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:37:56.453466+0000","scrub_duration":0.0016711180000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"86'106","reported_seq":773,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.860716+0000","last_change":"2023-09-23T20:43:16.759138+0000","last_active":"2023-09-23T20:45:01.860716+0000","last_peered":"2023-09-23T20:45:01.860716+0000","last_clean":"2023-09-23T20:45:01.860716+0000","last_became_active":"2023-09-23T20:43:16.758015+0000","last_became_peered":"2023-09-23T20:43:16.758015+0000","last_unstale":"2023-09-23T20:45:01.860716+0000","last_undegraded":"2023-09-23T20:45:01.860716+0000","last_fullsized":"2023-09-23T20:45:01.860716+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":106,"log_dups_size":0,"ondisk_log_size":106,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:08:02.852472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00099302599999999993,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1094,"num_read_kb":1678,"num_write":100,"num_write_kb":404,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":240,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.854368+0000","last_change":"2023-09-23T20:38:02.552008+0000","last_active":"2023-09-23T20:45:01.854368+0000","last_peered":"2023-09-23T20:45:01.854368+0000","last_clean":"2023-09-23T20:45:01.854368+0000","last_became_active":"2023-09-23T20:38:02.551684+0000","last_became_peered":"2023-09-23T20:38:02.551684+0000","last_unstale":"2023-09-23T20:45:01.854368+0000","last_undegraded":"2023-09-23T20:45:01.854368+0000","last_fullsized":"2023-09-23T20:45:01.854368+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:40:37.147538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":486,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.868482+0000","last_change":"2023-09-23T20:29:45.533406+0000","last_active":"2023-09-23T20:45:01.868482+0000","last_peered":"2023-09-23T20:45:01.868482+0000","last_clean":"2023-09-23T20:45:01.868482+0000","last_became_active":"2023-09-23T20:29:45.526001+0000","last_became_peered":"2023-09-23T20:29:45.526001+0000","last_unstale":"2023-09-23T20:45:01.868482+0000","last_undegraded":"2023-09-23T20:45:01.868482+0000","last_fullsized":"2023-09-23T20:45:01.868482+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:43.787251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.a","version":"523'14200","reported_seq":14287,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-09-23T20:45:15.443986+0000","last_change":"2023-09-23T20:43:16.852909+0000","last_active":"2023-09-23T20:45:15.443986+0000","last_peered":"2023-09-23T20:45:15.443986+0000","last_clean":"2023-09-23T20:45:15.443986+0000","last_became_active":"2023-09-23T20:43:16.770100+0000","last_became_peered":"2023-09-23T20:43:16.770100+0000","last_unstale":"2023-09-23T20:45:15.443986+0000","last_undegraded":"2023-09-23T20:45:15.443986+0000","last_fullsized":"2023-09-23T20:45:15.443986+0000","mapping_epoch":481,"log_start":"479'8506","ondisk_log_start":"479'8506","created":455,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":5694,"log_dups_size":0,"ondisk_log_size":5694,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:13:16.136666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14200,"num_write_kb":454400,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,7],"acting":[1,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":486,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.861773+0000","last_change":"2023-09-23T20:44:26.978243+0000","last_active":"2023-09-23T20:45:01.861773+0000","last_peered":"2023-09-23T20:45:01.861773+0000","last_clean":"2023-09-23T20:45:01.861773+0000","last_became_active":"2023-09-23T20:44:18.638989+0000","last_became_peered":"2023-09-23T20:44:18.638989+0000","last_unstale":"2023-09-23T20:45:01.861773+0000","last_undegraded":"2023-09-23T20:45:01.861773+0000","last_fullsized":"2023-09-23T20:45:01.861773+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:26.978179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:26.978179+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:25:38.977011+0000","scrub_duration":0.0013962829999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"86'130","reported_seq":841,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.864189+0000","last_change":"2023-09-23T20:25:39.637458+0000","last_active":"2023-09-23T20:45:01.864189+0000","last_peered":"2023-09-23T20:45:01.864189+0000","last_clean":"2023-09-23T20:45:01.864189+0000","last_became_active":"2023-09-23T20:25:39.636773+0000","last_became_peered":"2023-09-23T20:25:39.636773+0000","last_unstale":"2023-09-23T20:45:01.864189+0000","last_undegraded":"2023-09-23T20:45:01.864189+0000","last_fullsized":"2023-09-23T20:45:01.864189+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:00:32.810564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011300049999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":512,"num_read_kb":1820,"num_write":133,"num_write_kb":1347,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":494,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.869703+0000","last_change":"2023-09-23T20:38:02.591335+0000","last_active":"2023-09-23T20:45:01.869703+0000","last_peered":"2023-09-23T20:45:01.869703+0000","last_clean":"2023-09-23T20:45:01.869703+0000","last_became_active":"2023-09-23T20:38:02.591015+0000","last_became_peered":"2023-09-23T20:38:02.591015+0000","last_unstale":"2023-09-23T20:45:01.869703+0000","last_undegraded":"2023-09-23T20:45:01.869703+0000","last_fullsized":"2023-09-23T20:45:01.869703+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:55:32.272195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":37,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.860891+0000","last_change":"2023-09-23T20:44:10.555583+0000","last_active":"2023-09-23T20:45:01.860891+0000","last_peered":"2023-09-23T20:45:01.860891+0000","last_clean":"2023-09-23T20:45:01.860891+0000","last_became_active":"2023-09-23T20:44:10.555281+0000","last_became_peered":"2023-09-23T20:44:10.555281+0000","last_unstale":"2023-09-23T20:45:01.860891+0000","last_undegraded":"2023-09-23T20:45:01.860891+0000","last_fullsized":"2023-09-23T20:45:01.860891+0000","mapping_epoch":496,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":497,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:43:11.942147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.8","version":"523'14500","reported_seq":31480,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-09-23T20:45:15.517041+0000","last_change":"2023-09-23T20:44:51.765604+0000","last_active":"2023-09-23T20:45:15.517041+0000","last_peered":"2023-09-23T20:45:15.517041+0000","last_clean":"2023-09-23T20:45:15.517041+0000","last_became_active":"2023-09-23T20:44:51.738188+0000","last_became_peered":"2023-09-23T20:44:51.738188+0000","last_unstale":"2023-09-23T20:45:15.517041+0000","last_undegraded":"2023-09-23T20:45:15.517041+0000","last_fullsized":"2023-09-23T20:45:15.517041+0000","mapping_epoch":518,"log_start":"478'8427","ondisk_log_start":"478'8427","created":455,"last_epoch_clean":519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":6073,"log_dups_size":0,"ondisk_log_size":6073,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:58:46.821782+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14500,"num_write_kb":464000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5858,"num_bytes_recovered":383844352,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"86'194","reported_seq":879,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.864645+0000","last_change":"2023-09-23T20:43:16.765865+0000","last_active":"2023-09-23T20:45:01.864645+0000","last_peered":"2023-09-23T20:45:01.864645+0000","last_clean":"2023-09-23T20:45:01.864645+0000","last_became_active":"2023-09-23T20:43:16.765289+0000","last_became_peered":"2023-09-23T20:43:16.765289+0000","last_unstale":"2023-09-23T20:45:01.864645+0000","last_undegraded":"2023-09-23T20:45:01.864645+0000","last_fullsized":"2023-09-23T20:45:01.864645+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":194,"log_dups_size":0,"ondisk_log_size":194,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:15:16.461900+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00037995999999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1024,"num_read_kb":1200,"num_write":173,"num_write_kb":1276,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":985,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.861606+0000","last_change":"2023-09-23T20:38:05.602508+0000","last_active":"2023-09-23T20:45:01.861606+0000","last_peered":"2023-09-23T20:45:01.861606+0000","last_clean":"2023-09-23T20:45:01.861606+0000","last_became_active":"2023-09-23T20:38:05.602129+0000","last_became_peered":"2023-09-23T20:38:05.602129+0000","last_unstale":"2023-09-23T20:45:01.861606+0000","last_undegraded":"2023-09-23T20:45:01.861606+0000","last_fullsized":"2023-09-23T20:45:01.861606+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:45.343742+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:45.343742+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:48.910343+0000","scrub_duration":0.001475802,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":98,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.873681+0000","last_change":"2023-09-23T20:43:16.758213+0000","last_active":"2023-09-23T20:45:01.873681+0000","last_peered":"2023-09-23T20:45:01.873681+0000","last_clean":"2023-09-23T20:45:01.873681+0000","last_became_active":"2023-09-23T20:43:16.757869+0000","last_became_peered":"2023-09-23T20:43:16.757869+0000","last_unstale":"2023-09-23T20:45:01.873681+0000","last_undegraded":"2023-09-23T20:45:01.873681+0000","last_fullsized":"2023-09-23T20:45:01.873681+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:43:10.811749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":128,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.874089+0000","last_change":"2023-09-23T20:38:02.573200+0000","last_active":"2023-09-23T20:45:01.874089+0000","last_peered":"2023-09-23T20:45:01.874089+0000","last_clean":"2023-09-23T20:45:01.874089+0000","last_became_active":"2023-09-23T20:38:02.572945+0000","last_became_peered":"2023-09-23T20:38:02.572945+0000","last_unstale":"2023-09-23T20:45:01.874089+0000","last_undegraded":"2023-09-23T20:45:01.874089+0000","last_fullsized":"2023-09-23T20:45:01.874089+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:57.246533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.b","version":"523'14084","reported_seq":14178,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-09-23T20:45:15.525830+0000","last_change":"2023-09-23T20:43:46.540991+0000","last_active":"2023-09-23T20:45:15.525830+0000","last_peered":"2023-09-23T20:45:15.525830+0000","last_clean":"2023-09-23T20:45:15.525830+0000","last_became_active":"2023-09-23T20:43:46.512772+0000","last_became_peered":"2023-09-23T20:43:46.512772+0000","last_unstale":"2023-09-23T20:45:15.525830+0000","last_undegraded":"2023-09-23T20:45:15.525830+0000","last_fullsized":"2023-09-23T20:45:15.525830+0000","mapping_epoch":487,"log_start":"469'6534","ondisk_log_start":"469'6534","created":455,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":7550,"log_dups_size":0,"ondisk_log_size":7550,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:46:02.578833+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14084,"num_write_kb":450688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":855,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.860097+0000","last_change":"2023-09-23T20:38:05.615711+0000","last_active":"2023-09-23T20:45:01.860097+0000","last_peered":"2023-09-23T20:45:01.860097+0000","last_clean":"2023-09-23T20:45:01.860097+0000","last_became_active":"2023-09-23T20:38:05.615390+0000","last_became_peered":"2023-09-23T20:38:05.615390+0000","last_unstale":"2023-09-23T20:45:01.860097+0000","last_undegraded":"2023-09-23T20:45:01.860097+0000","last_fullsized":"2023-09-23T20:45:01.860097+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:43.582146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:43.582146+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:45:11.917871+0000","scrub_duration":0.0016219730000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"86'80","reported_seq":775,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.861507+0000","last_change":"2023-09-23T20:29:45.536796+0000","last_active":"2023-09-23T20:45:01.861507+0000","last_peered":"2023-09-23T20:45:01.861507+0000","last_clean":"2023-09-23T20:45:01.861507+0000","last_became_active":"2023-09-23T20:29:45.536439+0000","last_became_peered":"2023-09-23T20:29:45.536439+0000","last_unstale":"2023-09-23T20:45:01.861507+0000","last_undegraded":"2023-09-23T20:45:01.861507+0000","last_fullsized":"2023-09-23T20:45:01.861507+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":80,"log_dups_size":0,"ondisk_log_size":80,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:30:57.793132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035115,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":395,"num_read_kb":1370,"num_write":66,"num_write_kb":1061,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":515,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.872890+0000","last_change":"2023-09-23T20:43:46.544563+0000","last_active":"2023-09-23T20:45:01.872890+0000","last_peered":"2023-09-23T20:45:01.872890+0000","last_clean":"2023-09-23T20:45:01.872890+0000","last_became_active":"2023-09-23T20:43:46.543958+0000","last_became_peered":"2023-09-23T20:43:46.543958+0000","last_unstale":"2023-09-23T20:45:01.872890+0000","last_undegraded":"2023-09-23T20:45:01.872890+0000","last_fullsized":"2023-09-23T20:45:01.872890+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:46:36.921752+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":510,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.869675+0000","last_change":"2023-09-23T20:43:46.520386+0000","last_active":"2023-09-23T20:45:01.869675+0000","last_peered":"2023-09-23T20:45:01.869675+0000","last_clean":"2023-09-23T20:45:01.869675+0000","last_became_active":"2023-09-23T20:43:46.519757+0000","last_became_peered":"2023-09-23T20:43:46.519757+0000","last_unstale":"2023-09-23T20:45:01.869675+0000","last_undegraded":"2023-09-23T20:45:01.869675+0000","last_fullsized":"2023-09-23T20:45:01.869675+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:23:14.662734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.e","version":"523'14220","reported_seq":14296,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-09-23T20:45:15.517185+0000","last_change":"2023-09-23T20:41:04.574876+0000","last_active":"2023-09-23T20:45:15.517185+0000","last_peered":"2023-09-23T20:45:15.517185+0000","last_clean":"2023-09-23T20:45:15.517185+0000","last_became_active":"2023-09-23T20:41:04.574595+0000","last_became_peered":"2023-09-23T20:41:04.574595+0000","last_unstale":"2023-09-23T20:45:15.517185+0000","last_undegraded":"2023-09-23T20:45:15.517185+0000","last_fullsized":"2023-09-23T20:45:15.517185+0000","mapping_epoch":455,"log_start":"479'8508","ondisk_log_start":"479'8508","created":455,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":5712,"log_dups_size":0,"ondisk_log_size":5712,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:31:00.147966+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14220,"num_write_kb":455040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":634,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.874124+0000","last_change":"2023-09-23T20:41:08.380084+0000","last_active":"2023-09-23T20:45:01.874124+0000","last_peered":"2023-09-23T20:45:01.874124+0000","last_clean":"2023-09-23T20:45:01.874124+0000","last_became_active":"2023-09-23T20:38:05.610881+0000","last_became_peered":"2023-09-23T20:38:05.610881+0000","last_unstale":"2023-09-23T20:45:01.874124+0000","last_undegraded":"2023-09-23T20:45:01.874124+0000","last_fullsized":"2023-09-23T20:45:01.874124+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:41:08.379959+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:41:08.379959+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:19:39.547923+0000","scrub_duration":0.0019158420000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"86'102","reported_seq":1117,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.864729+0000","last_change":"2023-09-23T20:32:36.314672+0000","last_active":"2023-09-23T20:45:01.864729+0000","last_peered":"2023-09-23T20:45:01.864729+0000","last_clean":"2023-09-23T20:45:01.864729+0000","last_became_active":"2023-09-23T20:32:36.314231+0000","last_became_peered":"2023-09-23T20:32:36.314231+0000","last_unstale":"2023-09-23T20:45:01.864729+0000","last_undegraded":"2023-09-23T20:45:01.864729+0000","last_fullsized":"2023-09-23T20:45:01.864729+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":102,"log_dups_size":0,"ondisk_log_size":102,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:11:40.514709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030656500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":963,"num_read_kb":1234,"num_write":60,"num_write_kb":225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":470,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.868631+0000","last_change":"2023-09-23T20:40:19.544384+0000","last_active":"2023-09-23T20:45:01.868631+0000","last_peered":"2023-09-23T20:45:01.868631+0000","last_clean":"2023-09-23T20:45:01.868631+0000","last_became_active":"2023-09-23T20:40:19.544079+0000","last_became_peered":"2023-09-23T20:40:19.544079+0000","last_unstale":"2023-09-23T20:45:01.868631+0000","last_undegraded":"2023-09-23T20:45:01.868631+0000","last_fullsized":"2023-09-23T20:45:01.868631+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:00:19.498631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":465,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.873537+0000","last_change":"2023-09-23T20:43:46.541173+0000","last_active":"2023-09-23T20:45:01.873537+0000","last_peered":"2023-09-23T20:45:01.873537+0000","last_clean":"2023-09-23T20:45:01.873537+0000","last_became_active":"2023-09-23T20:43:46.540726+0000","last_became_peered":"2023-09-23T20:43:46.540726+0000","last_unstale":"2023-09-23T20:45:01.873537+0000","last_undegraded":"2023-09-23T20:45:01.873537+0000","last_fullsized":"2023-09-23T20:45:01.873537+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:49:04.137885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"523'14286","reported_seq":28058,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-09-23T20:45:15.546129+0000","last_change":"2023-09-23T20:44:18.688723+0000","last_active":"2023-09-23T20:45:15.546129+0000","last_peered":"2023-09-23T20:45:15.546129+0000","last_clean":"2023-09-23T20:45:15.546129+0000","last_became_active":"2023-09-23T20:44:18.636162+0000","last_became_peered":"2023-09-23T20:44:18.636162+0000","last_unstale":"2023-09-23T20:45:15.546129+0000","last_undegraded":"2023-09-23T20:45:15.546129+0000","last_fullsized":"2023-09-23T20:45:15.546129+0000","mapping_epoch":502,"log_start":"479'8548","ondisk_log_start":"479'8548","created":455,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":5738,"log_dups_size":0,"ondisk_log_size":5738,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:49:45.956725+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":14286,"num_write_kb":457089,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6816,"num_bytes_recovered":446627864,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":801,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.867453+0000","last_change":"2023-09-23T20:38:05.590361+0000","last_active":"2023-09-23T20:45:01.867453+0000","last_peered":"2023-09-23T20:45:01.867453+0000","last_clean":"2023-09-23T20:45:01.867453+0000","last_became_active":"2023-09-23T20:38:05.589985+0000","last_became_peered":"2023-09-23T20:38:05.589985+0000","last_unstale":"2023-09-23T20:45:01.867453+0000","last_undegraded":"2023-09-23T20:45:01.867453+0000","last_fullsized":"2023-09-23T20:45:01.867453+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:03.246271+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:03.246271+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:38:54.351576+0000","scrub_duration":0.0019016700000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":260,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.862187+0000","last_change":"2023-09-23T20:32:50.549695+0000","last_active":"2023-09-23T20:45:01.862187+0000","last_peered":"2023-09-23T20:45:01.862187+0000","last_clean":"2023-09-23T20:45:01.862187+0000","last_became_active":"2023-09-23T20:32:50.549431+0000","last_became_peered":"2023-09-23T20:32:50.549431+0000","last_unstale":"2023-09-23T20:45:01.862187+0000","last_undegraded":"2023-09-23T20:45:01.862187+0000","last_fullsized":"2023-09-23T20:45:01.862187+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:45:53.666569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"86'128","reported_seq":936,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.865625+0000","last_change":"2023-09-23T20:38:03.541765+0000","last_active":"2023-09-23T20:45:01.865625+0000","last_peered":"2023-09-23T20:45:01.865625+0000","last_clean":"2023-09-23T20:45:01.865625+0000","last_became_active":"2023-09-23T20:38:03.541277+0000","last_became_peered":"2023-09-23T20:38:03.541277+0000","last_unstale":"2023-09-23T20:45:01.865625+0000","last_undegraded":"2023-09-23T20:45:01.865625+0000","last_fullsized":"2023-09-23T20:45:01.865625+0000","mapping_epoch":404,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":405,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:20:53.834554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000293202,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":579,"num_read_kb":1473,"num_write":96,"num_write_kb":1062,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":303,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.863712+0000","last_change":"2023-09-23T20:37:43.582105+0000","last_active":"2023-09-23T20:45:01.863712+0000","last_peered":"2023-09-23T20:45:01.863712+0000","last_clean":"2023-09-23T20:45:01.863712+0000","last_became_active":"2023-09-23T20:37:43.581860+0000","last_became_peered":"2023-09-23T20:37:43.581860+0000","last_unstale":"2023-09-23T20:45:01.863712+0000","last_undegraded":"2023-09-23T20:45:01.863712+0000","last_fullsized":"2023-09-23T20:45:01.863712+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:18:38.765486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.c","version":"523'14252","reported_seq":14330,"reported_epoch":523,"state":"active+clean","last_fresh":"2023-09-23T20:45:15.530161+0000","last_change":"2023-09-23T20:41:04.575496+0000","last_active":"2023-09-23T20:45:15.530161+0000","last_peered":"2023-09-23T20:45:15.530161+0000","last_clean":"2023-09-23T20:45:15.530161+0000","last_became_active":"2023-09-23T20:41:04.575242+0000","last_became_peered":"2023-09-23T20:41:04.575242+0000","last_unstale":"2023-09-23T20:45:15.530161+0000","last_undegraded":"2023-09-23T20:45:15.530161+0000","last_fullsized":"2023-09-23T20:45:15.530161+0000","mapping_epoch":455,"log_start":"478'8175","ondisk_log_start":"478'8175","created":455,"last_epoch_clean":456,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:41:03.527576+0000","last_clean_scrub_stamp":"2023-09-23T20:41:03.527576+0000","objects_scrubbed":0,"log_size":6077,"log_dups_size":0,"ondisk_log_size":6077,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:30:14.611529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14252,"num_write_kb":456064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"90'161","reported_seq":737,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.863066+0000","last_change":"2023-09-23T20:40:22.427435+0000","last_active":"2023-09-23T20:45:01.863066+0000","last_peered":"2023-09-23T20:45:01.863066+0000","last_clean":"2023-09-23T20:45:01.863066+0000","last_became_active":"2023-09-23T20:40:22.427051+0000","last_became_peered":"2023-09-23T20:40:22.427051+0000","last_unstale":"2023-09-23T20:45:01.863066+0000","last_undegraded":"2023-09-23T20:45:01.863066+0000","last_fullsized":"2023-09-23T20:45:01.863066+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:34:54.504554+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:34:54.504554+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:42.001652+0000","scrub_duration":0.011176864,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"86'117","reported_seq":831,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.874299+0000","last_change":"2023-09-23T20:44:18.635224+0000","last_active":"2023-09-23T20:45:01.874299+0000","last_peered":"2023-09-23T20:45:01.874299+0000","last_clean":"2023-09-23T20:45:01.874299+0000","last_became_active":"2023-09-23T20:44:18.634703+0000","last_became_peered":"2023-09-23T20:44:18.634703+0000","last_unstale":"2023-09-23T20:45:01.874299+0000","last_undegraded":"2023-09-23T20:45:01.874299+0000","last_fullsized":"2023-09-23T20:45:01.874299+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:59:38.030626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046891400000000002,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":893,"num_read_kb":922,"num_write":82,"num_write_kb":4193,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":16777216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":510,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.869762+0000","last_change":"2023-09-23T20:43:46.521505+0000","last_active":"2023-09-23T20:45:01.869762+0000","last_peered":"2023-09-23T20:45:01.869762+0000","last_clean":"2023-09-23T20:45:01.869762+0000","last_became_active":"2023-09-23T20:43:46.520727+0000","last_became_peered":"2023-09-23T20:43:46.520727+0000","last_unstale":"2023-09-23T20:45:01.869762+0000","last_undegraded":"2023-09-23T20:45:01.869762+0000","last_fullsized":"2023-09-23T20:45:01.869762+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:17:06.164741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":260,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.853692+0000","last_change":"2023-09-23T20:40:19.530602+0000","last_active":"2023-09-23T20:45:01.853692+0000","last_peered":"2023-09-23T20:45:01.853692+0000","last_clean":"2023-09-23T20:45:01.853692+0000","last_became_active":"2023-09-23T20:40:19.530256+0000","last_became_peered":"2023-09-23T20:40:19.530256+0000","last_unstale":"2023-09-23T20:45:01.853692+0000","last_undegraded":"2023-09-23T20:45:01.853692+0000","last_fullsized":"2023-09-23T20:45:01.853692+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:18:45.658867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"90'161","reported_seq":779,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.872622+0000","last_change":"2023-09-23T20:43:46.544750+0000","last_active":"2023-09-23T20:45:01.872622+0000","last_peered":"2023-09-23T20:45:01.872622+0000","last_clean":"2023-09-23T20:45:01.872622+0000","last_became_active":"2023-09-23T20:43:46.544395+0000","last_became_peered":"2023-09-23T20:43:46.544395+0000","last_unstale":"2023-09-23T20:45:01.872622+0000","last_undegraded":"2023-09-23T20:45:01.872622+0000","last_fullsized":"2023-09-23T20:45:01.872622+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:06.487351+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:06.487351+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:56.866168+0000","scrub_duration":0.001964379,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":477,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.873464+0000","last_change":"2023-09-23T20:43:46.541379+0000","last_active":"2023-09-23T20:45:01.873464+0000","last_peered":"2023-09-23T20:45:01.873464+0000","last_clean":"2023-09-23T20:45:01.873464+0000","last_became_active":"2023-09-23T20:43:46.540732+0000","last_became_peered":"2023-09-23T20:43:46.540732+0000","last_unstale":"2023-09-23T20:45:01.873464+0000","last_undegraded":"2023-09-23T20:45:01.873464+0000","last_fullsized":"2023-09-23T20:45:01.873464+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:31:40.605772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":379,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.864774+0000","last_change":"2023-09-23T20:29:45.538314+0000","last_active":"2023-09-23T20:45:01.864774+0000","last_peered":"2023-09-23T20:45:01.864774+0000","last_clean":"2023-09-23T20:45:01.864774+0000","last_became_active":"2023-09-23T20:29:45.536847+0000","last_became_peered":"2023-09-23T20:29:45.536847+0000","last_unstale":"2023-09-23T20:45:01.864774+0000","last_undegraded":"2023-09-23T20:45:01.864774+0000","last_fullsized":"2023-09-23T20:45:01.864774+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:51.948721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"90'161","reported_seq":596,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.857119+0000","last_change":"2023-09-23T20:41:06.499511+0000","last_active":"2023-09-23T20:45:01.857119+0000","last_peered":"2023-09-23T20:45:01.857119+0000","last_clean":"2023-09-23T20:45:01.857119+0000","last_became_active":"2023-09-23T20:38:05.601874+0000","last_became_peered":"2023-09-23T20:38:05.601874+0000","last_unstale":"2023-09-23T20:45:01.857119+0000","last_undegraded":"2023-09-23T20:45:01.857119+0000","last_fullsized":"2023-09-23T20:45:01.857119+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:41:06.499396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:41:06.499396+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:31:34.059077+0000","scrub_duration":0.0019670310000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":415,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.868060+0000","last_change":"2023-09-23T20:43:16.758761+0000","last_active":"2023-09-23T20:45:01.868060+0000","last_peered":"2023-09-23T20:45:01.868060+0000","last_clean":"2023-09-23T20:45:01.868060+0000","last_became_active":"2023-09-23T20:43:16.758407+0000","last_became_peered":"2023-09-23T20:43:16.758407+0000","last_unstale":"2023-09-23T20:45:01.868060+0000","last_undegraded":"2023-09-23T20:45:01.868060+0000","last_fullsized":"2023-09-23T20:45:01.868060+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:59:27.070685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":179,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.854089+0000","last_change":"2023-09-23T20:34:59.987575+0000","last_active":"2023-09-23T20:45:01.854089+0000","last_peered":"2023-09-23T20:45:01.854089+0000","last_clean":"2023-09-23T20:45:01.854089+0000","last_became_active":"2023-09-23T20:34:59.987229+0000","last_became_peered":"2023-09-23T20:34:59.987229+0000","last_unstale":"2023-09-23T20:45:01.854089+0000","last_undegraded":"2023-09-23T20:45:01.854089+0000","last_fullsized":"2023-09-23T20:45:01.854089+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:34:01.820159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":423,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.854693+0000","last_change":"2023-09-23T20:43:16.759691+0000","last_active":"2023-09-23T20:45:01.854693+0000","last_peered":"2023-09-23T20:45:01.854693+0000","last_clean":"2023-09-23T20:45:01.854693+0000","last_became_active":"2023-09-23T20:43:16.759309+0000","last_became_peered":"2023-09-23T20:43:16.759309+0000","last_unstale":"2023-09-23T20:45:01.854693+0000","last_undegraded":"2023-09-23T20:45:01.854693+0000","last_fullsized":"2023-09-23T20:45:01.854693+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:58:52.893019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":179,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.853876+0000","last_change":"2023-09-23T20:34:59.987416+0000","last_active":"2023-09-23T20:45:01.853876+0000","last_peered":"2023-09-23T20:45:01.853876+0000","last_clean":"2023-09-23T20:45:01.853876+0000","last_became_active":"2023-09-23T20:34:59.987117+0000","last_became_peered":"2023-09-23T20:34:59.987117+0000","last_unstale":"2023-09-23T20:45:01.853876+0000","last_undegraded":"2023-09-23T20:45:01.853876+0000","last_fullsized":"2023-09-23T20:45:01.853876+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:36:16.318623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"90'161","reported_seq":778,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.861780+0000","last_change":"2023-09-23T20:38:05.591992+0000","last_active":"2023-09-23T20:45:01.861780+0000","last_peered":"2023-09-23T20:45:01.861780+0000","last_clean":"2023-09-23T20:45:01.861780+0000","last_became_active":"2023-09-23T20:38:05.591701+0000","last_became_peered":"2023-09-23T20:38:05.591701+0000","last_unstale":"2023-09-23T20:45:01.861780+0000","last_undegraded":"2023-09-23T20:45:01.861780+0000","last_fullsized":"2023-09-23T20:45:01.861780+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:02.384133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:02.384133+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:41.722542+0000","scrub_duration":0.001769098,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":345,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.857318+0000","last_change":"2023-09-23T20:38:02.542719+0000","last_active":"2023-09-23T20:45:01.857318+0000","last_peered":"2023-09-23T20:45:01.857318+0000","last_clean":"2023-09-23T20:45:01.857318+0000","last_became_active":"2023-09-23T20:38:02.542385+0000","last_became_peered":"2023-09-23T20:38:02.542385+0000","last_unstale":"2023-09-23T20:45:01.857318+0000","last_undegraded":"2023-09-23T20:45:01.857318+0000","last_fullsized":"2023-09-23T20:45:01.857318+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:36:23.673757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":312,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.856603+0000","last_change":"2023-09-23T20:37:43.584253+0000","last_active":"2023-09-23T20:45:01.856603+0000","last_peered":"2023-09-23T20:45:01.856603+0000","last_clean":"2023-09-23T20:45:01.856603+0000","last_became_active":"2023-09-23T20:37:43.580069+0000","last_became_peered":"2023-09-23T20:37:43.580069+0000","last_unstale":"2023-09-23T20:45:01.856603+0000","last_undegraded":"2023-09-23T20:45:01.856603+0000","last_fullsized":"2023-09-23T20:45:01.856603+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:39:11.482581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"90'161","reported_seq":616,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.867161+0000","last_change":"2023-09-23T20:36:01.505098+0000","last_active":"2023-09-23T20:45:01.867161+0000","last_peered":"2023-09-23T20:45:01.867161+0000","last_clean":"2023-09-23T20:45:01.867161+0000","last_became_active":"2023-09-23T20:35:26.644707+0000","last_became_peered":"2023-09-23T20:35:26.644707+0000","last_unstale":"2023-09-23T20:45:01.867161+0000","last_undegraded":"2023-09-23T20:45:01.867161+0000","last_fullsized":"2023-09-23T20:45:01.867161+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:01.504973+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:01.504973+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:02:10.880809+0000","scrub_duration":0.00202344,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":555,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.870333+0000","last_change":"2023-09-23T20:43:46.511657+0000","last_active":"2023-09-23T20:45:01.870333+0000","last_peered":"2023-09-23T20:45:01.870333+0000","last_clean":"2023-09-23T20:45:01.870333+0000","last_became_active":"2023-09-23T20:43:46.511326+0000","last_became_peered":"2023-09-23T20:43:46.511326+0000","last_unstale":"2023-09-23T20:45:01.870333+0000","last_undegraded":"2023-09-23T20:45:01.870333+0000","last_fullsized":"2023-09-23T20:45:01.870333+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:36:46.139261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3324,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":240,"reported_epoch":522,"state":"active+clean","last_fresh":"2023-09-23T20:45:01.853435+0000","last_change":"2023-09-23T20:38:02.538333+0000","last_active":"2023-09-23T20:45:01.853435+0000","last_peered":"2023-09-23T20:45:01.853435+0000","last_clean":"2023-09-23T20:45:01.853435+0000","last_became_active":"2023-09-23T20:38:02.538008+0000","last_became_peered":"2023-09-23T20:38:02.538008+0000","last_unstale":"2023-09-23T20:45:01.853435+0000","last_undegraded":"2023-09-23T20:45:01.853435+0000","last_fullsized":"2023-09-23T20:45:01.853435+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:02:38.789530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":90,"num_bytes_recovered":18486,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26691,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":227664,"num_write_kb":7285185,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16881,"num_bytes_recovered":1105854488,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":233046016,"data_stored":233046016,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":103259,"ondisk_log_size":103259,"up":48,"acting":48,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5983,"num_read_kb":13200,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":84,"num_bytes_recovered":16777216,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6082,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201686,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":547848,"kb_used_data":43200,"kb_used_omap":0,"kb_used_meta":504640,"kb_avail":93197304,"statfs":{"total":95995035648,"available":95434039296,"internally_reserved":0,"allocated":44236800,"data_stored":42717122,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":516751360},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:44:59 2023","interfaces":[{"interface":"back","average":{"1min":2.8599999999999999,"5min":2.0510000000000002,"15min":2.1640000000000001},"min":{"1min":0.24199999999999999,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":10.683999999999999,"5min":15.1,"15min":18.315000000000001},"last":0.57599999999999996},{"interface":"front","average":{"1min":2.282,"5min":1.8799999999999999,"15min":1.897},"min":{"1min":0.31900000000000001,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":11.045999999999999,"5min":15.327999999999999,"15min":19.355},"last":0.56100000000000005}]},{"osd":1,"last update":"Sat Sep 23 20:45:08 2023","interfaces":[{"interface":"back","average":{"1min":1.002,"5min":1.468,"15min":2.0539999999999998},"min":{"1min":0.26100000000000001,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":2.5609999999999999,"5min":15.131,"15min":49.965000000000003},"last":0.60899999999999999},{"interface":"front","average":{"1min":2.7730000000000001,"5min":2.6419999999999999,"15min":2.3919999999999999},"min":{"1min":0.39700000000000002,"5min":0.26700000000000002,"15min":0.248},"max":{"1min":11.276,"5min":15.18,"15min":53.451999999999998},"last":0.59799999999999998}]},{"osd":2,"last update":"Sat Sep 23 20:44:59 2023","interfaces":[{"interface":"back","average":{"1min":2.4889999999999999,"5min":2.3119999999999998,"15min":2.286},"min":{"1min":0.30499999999999999,"5min":0.27900000000000003,"15min":0.27800000000000002},"max":{"1min":8.0020000000000007,"5min":15.087,"15min":16.965},"last":0.52100000000000002},{"interface":"front","average":{"1min":2.6520000000000001,"5min":2.0230000000000001,"15min":2.4670000000000001},"min":{"1min":0.371,"5min":0.30299999999999999,"15min":0.27400000000000002},"max":{"1min":13.5,"5min":15.193,"15min":23.494},"last":0.58599999999999997}]},{"osd":3,"last update":"Sat Sep 23 20:44:28 2023","interfaces":[{"interface":"back","average":{"1min":2.6840000000000002,"5min":1.772,"15min":2.371},"min":{"1min":0.41399999999999998,"5min":0.32500000000000001,"15min":0.29199999999999998},"max":{"1min":15.115,"5min":15.115,"15min":23.352},"last":0.623},{"interface":"front","average":{"1min":3.3570000000000002,"5min":2.1389999999999998,"15min":2.4489999999999998},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.33100000000000002},"max":{"1min":15.144,"5min":15.144,"15min":20.969999999999999},"last":0.63200000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:44:28 2023","interfaces":[{"interface":"back","average":{"1min":2.0110000000000001,"5min":1.391,"15min":1.2669999999999999},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.23200000000000001},"max":{"1min":12.260999999999999,"5min":12.260999999999999,"15min":18.364999999999998},"last":0.41699999999999998},{"interface":"front","average":{"1min":2.145,"5min":1.5640000000000001,"15min":1.3320000000000001},"min":{"1min":0.315,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":6.9939999999999998,"5min":6.9939999999999998,"15min":23.617000000000001},"last":0.376}]},{"osd":5,"last update":"Sat Sep 23 20:44:28 2023","interfaces":[{"interface":"back","average":{"1min":2.5270000000000001,"5min":1.6040000000000001,"15min":1.1970000000000001},"min":{"1min":0.36899999999999999,"5min":0.30099999999999999,"15min":0.26600000000000001},"max":{"1min":15.005000000000001,"5min":15.005000000000001,"15min":15.005000000000001},"last":0.39600000000000002},{"interface":"front","average":{"1min":1.9990000000000001,"5min":1.556,"15min":1.3120000000000001},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":12.212,"5min":12.731,"15min":18.831},"last":0.48799999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:44:28 2023","interfaces":[{"interface":"back","average":{"1min":3.6379999999999999,"5min":1.837,"15min":1.4359999999999999},"min":{"1min":0.34599999999999997,"5min":0.33700000000000002,"15min":0.30399999999999999},"max":{"1min":15.038,"5min":15.038,"15min":18.417000000000002},"last":0.53200000000000003},{"interface":"front","average":{"1min":2.8969999999999998,"5min":1.6659999999999999,"15min":1.274},"min":{"1min":0.47999999999999998,"5min":0.36499999999999999,"15min":0.318},"max":{"1min":15.369999999999999,"5min":15.369999999999999,"15min":15.369999999999999},"last":0.436}]}]},{"osd":6,"up_from":47,"seq":201863463325,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":435964,"kb_used_data":9844,"kb_used_omap":0,"kb_used_meta":426112,"kb_avail":93309188,"statfs":{"total":95995035648,"available":95548608512,"internally_reserved":0,"allocated":10080256,"data_stored":8572195,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":436338688},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:44:55 2023","interfaces":[{"interface":"back","average":{"1min":1.883,"5min":1.5720000000000001,"15min":1.986},"min":{"1min":0.219,"5min":0.219,"15min":0.219},"max":{"1min":7.7720000000000002,"5min":12.552,"15min":53.701999999999998},"last":0.497},{"interface":"front","average":{"1min":2.343,"5min":1.7430000000000001,"15min":2.3730000000000002},"min":{"1min":0.313,"5min":0.23300000000000001,"15min":0.22700000000000001},"max":{"1min":10.895,"5min":10.895,"15min":60.328000000000003},"last":0.63}]},{"osd":1,"last update":"Sat Sep 23 20:45:17 2023","interfaces":[{"interface":"back","average":{"1min":1.9390000000000001,"5min":1.8999999999999999,"15min":2.077},"min":{"1min":0.28899999999999998,"5min":0.249,"15min":0.249},"max":{"1min":7.9189999999999996,"5min":9.2520000000000007,"15min":64.274000000000001},"last":0.59299999999999997},{"interface":"front","average":{"1min":2.198,"5min":2.0019999999999998,"15min":1.988},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.28000000000000003},"max":{"1min":7.8849999999999998,"5min":17.786000000000001,"15min":34.490000000000002},"last":0.72399999999999998}]},{"osd":2,"last update":"Sat Sep 23 20:44:55 2023","interfaces":[{"interface":"back","average":{"1min":2.5409999999999999,"5min":2.2370000000000001,"15min":2.1739999999999999},"min":{"1min":0.34899999999999998,"5min":0.27800000000000002,"15min":0.27700000000000002},"max":{"1min":6.6130000000000004,"5min":9.9049999999999994,"15min":34.317999999999998},"last":0.53900000000000003},{"interface":"front","average":{"1min":2.3079999999999998,"5min":2.0129999999999999,"15min":2.069},"min":{"1min":0.373,"5min":0.311,"15min":0.30399999999999999},"max":{"1min":6.5620000000000003,"5min":9.9269999999999996,"15min":34.292999999999999},"last":0.81499999999999995}]},{"osd":3,"last update":"Sat Sep 23 20:44:46 2023","interfaces":[{"interface":"back","average":{"1min":2.7330000000000001,"5min":2.024,"15min":1.974},"min":{"1min":0.47999999999999998,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":11.664,"5min":11.664,"15min":11.664},"last":0.60999999999999999},{"interface":"front","average":{"1min":2.3039999999999998,"5min":2.0299999999999998,"15min":2.1459999999999999},"min":{"1min":0.42099999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":8.2560000000000002,"5min":11.808,"15min":12.875999999999999},"last":0.66000000000000003}]},{"osd":4,"last update":"Sat Sep 23 20:45:17 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.93100000000000005,"15min":0.78800000000000003},"min":{"1min":0.29499999999999998,"5min":0.25600000000000001,"15min":0.251},"max":{"1min":1.4330000000000001,"5min":6.5800000000000001,"15min":23.904},"last":0.76300000000000001},{"interface":"front","average":{"1min":0.89700000000000002,"5min":1.032,"15min":1.022},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.27000000000000002},"max":{"1min":2.7839999999999998,"5min":5.6260000000000003,"15min":11.515000000000001},"last":0.58399999999999996}]},{"osd":5,"last update":"Sat Sep 23 20:45:17 2023","interfaces":[{"interface":"back","average":{"1min":1.5489999999999999,"5min":1.0369999999999999,"15min":0.85099999999999998},"min":{"1min":0.26400000000000001,"5min":0.253,"15min":0.253},"max":{"1min":7.8529999999999998,"5min":7.8529999999999998,"15min":7.8529999999999998},"last":0.79000000000000004},{"interface":"front","average":{"1min":1.2549999999999999,"5min":1.488,"15min":1.103},"min":{"1min":0.34699999999999998,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":5.141,"5min":7.1319999999999997,"15min":7.7300000000000004},"last":0.70999999999999996}]},{"osd":7,"last update":"Sat Sep 23 20:44:46 2023","interfaces":[{"interface":"back","average":{"1min":1.4199999999999999,"5min":1.079,"15min":0.94799999999999995},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":7.8250000000000002,"5min":8.8350000000000009,"15min":8.8350000000000009},"last":0.84199999999999997},{"interface":"front","average":{"1min":2.6440000000000001,"5min":1.661,"15min":1.3009999999999999},"min":{"1min":0.45900000000000002,"5min":0.34699999999999998,"15min":0.30299999999999999},"max":{"1min":7.1239999999999997,"5min":7.1929999999999996,"15min":8.8119999999999994},"last":0.68500000000000005}]}]},{"osd":1,"up_from":266,"seq":1142461300924,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":822676,"kb_used_data":140236,"kb_used_omap":8,"kb_used_meta":682423,"kb_avail":92922476,"statfs":{"total":95995035648,"available":95152615424,"internally_reserved":0,"allocated":143601664,"data_stored":140907402,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8206,"internal_metadata":698802162},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:44:15 2023","interfaces":[{"interface":"back","average":{"1min":3.6110000000000002,"5min":2.206,"15min":2.5329999999999999},"min":{"1min":0.214,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":15.532999999999999,"5min":16.292000000000002,"15min":52.689999999999998},"last":5.1849999999999996},{"interface":"front","average":{"1min":3.2290000000000001,"5min":2.2970000000000002,"15min":3.0179999999999998},"min":{"1min":0.22600000000000001,"5min":0.20100000000000001,"15min":0.19900000000000001},"max":{"1min":14.445,"5min":16.332999999999998,"15min":52.662999999999997},"last":5.6580000000000004}]},{"osd":2,"last update":"Sat Sep 23 20:44:15 2023","interfaces":[{"interface":"back","average":{"1min":3.0390000000000001,"5min":2.4140000000000001,"15min":3.3370000000000002},"min":{"1min":0.28999999999999998,"5min":0.216,"15min":0.20899999999999999},"max":{"1min":8.6519999999999992,"5min":14.148,"15min":32.255000000000003},"last":0.40500000000000003},{"interface":"front","average":{"1min":2.9049999999999998,"5min":1.9810000000000001,"15min":2.2909999999999999},"min":{"1min":0.26000000000000001,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":10.041,"5min":10.041,"15min":13.404999999999999},"last":0.48199999999999998}]},{"osd":3,"last update":"Sat Sep 23 20:44:15 2023","interfaces":[{"interface":"back","average":{"1min":2.5830000000000002,"5min":2.089,"15min":3.097},"min":{"1min":0.36899999999999999,"5min":0.247,"15min":0.247},"max":{"1min":7.2469999999999999,"5min":16.245999999999999,"15min":80.745999999999995},"last":5.3120000000000003},{"interface":"front","average":{"1min":3.5779999999999998,"5min":2.645,"15min":3.3450000000000002},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.22700000000000001},"max":{"1min":11.523,"5min":13.497999999999999,"15min":82.988},"last":5.5940000000000003}]},{"osd":4,"last update":"Sat Sep 23 20:44:15 2023","interfaces":[{"interface":"back","average":{"1min":2.3250000000000002,"5min":2.202,"15min":2.25},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":6.5970000000000004,"5min":20.634,"15min":24.327999999999999},"last":0.86199999999999999},{"interface":"front","average":{"1min":2.0190000000000001,"5min":1.982,"15min":1.748},"min":{"1min":0.34699999999999998,"5min":0.32600000000000001,"15min":0.311},"max":{"1min":6.0869999999999997,"5min":20.835999999999999,"15min":20.835999999999999},"last":1.956}]},{"osd":5,"last update":"Sat Sep 23 20:44:15 2023","interfaces":[{"interface":"back","average":{"1min":3.2040000000000002,"5min":2.4319999999999999,"15min":2.3029999999999999},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":14.366,"5min":14.366,"15min":33.401000000000003},"last":5.5629999999999997},{"interface":"front","average":{"1min":2.7469999999999999,"5min":2.3519999999999999,"15min":2.4820000000000002},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.32500000000000001},"max":{"1min":7.3300000000000001,"5min":10.958,"15min":33.417000000000002},"last":2.7090000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:44:15 2023","interfaces":[{"interface":"back","average":{"1min":3.0030000000000001,"5min":2.3639999999999999,"15min":2.839},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35699999999999998},"max":{"1min":14.414999999999999,"5min":14.414999999999999,"15min":72.682000000000002},"last":5.516},{"interface":"front","average":{"1min":3.2829999999999999,"5min":2.3620000000000001,"15min":2.9860000000000002},"min":{"1min":0.38500000000000001,"5min":0.317,"15min":0.317},"max":{"1min":9.1470000000000002,"5min":9.4090000000000007,"15min":72.563999999999993},"last":2.6549999999999998}]},{"osd":7,"last update":"Sat Sep 23 20:44:15 2023","interfaces":[{"interface":"back","average":{"1min":2.98,"5min":2.4470000000000001,"15min":2.3700000000000001},"min":{"1min":0.43099999999999999,"5min":0.37,"15min":0.37},"max":{"1min":9.2409999999999997,"5min":16.359000000000002,"15min":18.437000000000001},"last":5.6269999999999998},{"interface":"front","average":{"1min":2.6459999999999999,"5min":2.4460000000000002,"15min":2.4910000000000001},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":7.1959999999999997,"5min":10.34,"15min":24.524000000000001},"last":2.6869999999999998}]}]},{"osd":0,"up_from":94,"seq":403726926126,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":715876,"kb_used_data":6876,"kb_used_omap":7,"kb_used_meta":708984,"kb_avail":93029276,"statfs":{"total":95995035648,"available":95261978624,"internally_reserved":0,"allocated":7041024,"data_stored":4508122,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":725999633},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:44:24 2023","interfaces":[{"interface":"back","average":{"1min":3.2120000000000002,"5min":2.4100000000000001,"15min":2.7210000000000001},"min":{"1min":0.184,"5min":0.184,"15min":0.17199999999999999},"max":{"1min":18.052,"5min":24.010000000000002,"15min":42.000999999999998},"last":4.4950000000000001},{"interface":"front","average":{"1min":3.0939999999999999,"5min":2.605,"15min":2.9129999999999998},"min":{"1min":0.33800000000000002,"5min":0.218,"15min":0.20399999999999999},"max":{"1min":18.097000000000001,"5min":18.097000000000001,"15min":23.036000000000001},"last":0.625}]},{"osd":2,"last update":"Sat Sep 23 20:45:01 2023","interfaces":[{"interface":"back","average":{"1min":3.242,"5min":3.052,"15min":3.085},"min":{"1min":0.27000000000000002,"5min":0.248,"15min":0.185},"max":{"1min":15.108000000000001,"5min":48.841000000000001,"15min":77.448999999999998},"last":3.714},{"interface":"front","average":{"1min":3.5419999999999998,"5min":2.722,"15min":3.0470000000000002},"min":{"1min":0.28599999999999998,"5min":0.22,"15min":0.188},"max":{"1min":17.013000000000002,"5min":46.183,"15min":81.724999999999994},"last":7.3170000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:44:31 2023","interfaces":[{"interface":"back","average":{"1min":3.6960000000000002,"5min":2.7290000000000001,"15min":3.1640000000000001},"min":{"1min":0.30599999999999999,"5min":0.248,"15min":0.23400000000000001},"max":{"1min":15.048,"5min":15.048,"15min":81.400999999999996},"last":3.8700000000000001},{"interface":"front","average":{"1min":2.1549999999999998,"5min":2.3159999999999998,"15min":3.3439999999999999},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.21199999999999999},"max":{"1min":9.6379999999999999,"5min":14.237,"15min":81.137},"last":1.8109999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:45:01 2023","interfaces":[{"interface":"back","average":{"1min":1.968,"5min":2.052,"15min":1.996},"min":{"1min":0.35999999999999999,"5min":0.32000000000000001,"15min":0.25900000000000001},"max":{"1min":10.368,"5min":10.368,"15min":12.587999999999999},"last":3.7690000000000001},{"interface":"front","average":{"1min":1.8049999999999999,"5min":1.9650000000000001,"15min":1.9099999999999999},"min":{"1min":0.34599999999999997,"5min":0.28899999999999998,"15min":0.27400000000000002},"max":{"1min":7.6740000000000004,"5min":9.8480000000000008,"15min":23.001000000000001},"last":0.65700000000000003}]},{"osd":5,"last update":"Sat Sep 23 20:45:01 2023","interfaces":[{"interface":"back","average":{"1min":2.0819999999999999,"5min":2.4590000000000001,"15min":2.0880000000000001},"min":{"1min":0.308,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":7.7290000000000001,"5min":10.239000000000001,"15min":22.969999999999999},"last":0.55600000000000005},{"interface":"front","average":{"1min":2.964,"5min":2.496,"15min":2.3780000000000001},"min":{"1min":0.442,"5min":0.34300000000000003,"15min":0.25600000000000001},"max":{"1min":12.601000000000001,"5min":12.929,"15min":26.228000000000002},"last":0.49199999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:45:01 2023","interfaces":[{"interface":"back","average":{"1min":2.0760000000000001,"5min":2.375,"15min":2.1720000000000002},"min":{"1min":0.39100000000000001,"5min":0.371,"15min":0.32300000000000001},"max":{"1min":7.7969999999999997,"5min":24.056000000000001,"15min":24.056000000000001},"last":0.59399999999999997},{"interface":"front","average":{"1min":3.153,"5min":2.0880000000000001,"15min":2.2309999999999999},"min":{"1min":0.46999999999999997,"5min":0.34100000000000003,"15min":0.29399999999999998},"max":{"1min":12.698,"5min":12.698,"15min":26.268000000000001},"last":0.45300000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:45:01 2023","interfaces":[{"interface":"back","average":{"1min":2.9910000000000001,"5min":2.3580000000000001,"15min":2.3860000000000001},"min":{"1min":0.497,"5min":0.36799999999999999,"15min":0.32200000000000001},"max":{"1min":12.651999999999999,"5min":12.651999999999999,"15min":26.177},"last":1.593},{"interface":"front","average":{"1min":2.4729999999999999,"5min":2.6389999999999998,"15min":2.3799999999999999},"min":{"1min":0.40600000000000003,"5min":0.40400000000000003,"15min":0.34200000000000003},"max":{"1min":10.32,"5min":15.006,"15min":22.422999999999998},"last":3.9039999999999999}]}]},{"osd":2,"up_from":104,"seq":446676599074,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":752684,"kb_used_data":10916,"kb_used_omap":11,"kb_used_meta":741748,"kb_avail":92992468,"statfs":{"total":95995035648,"available":95224287232,"internally_reserved":0,"allocated":11177984,"data_stored":8635843,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11878,"internal_metadata":759550362},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:45:09 2023","interfaces":[{"interface":"back","average":{"1min":1.696,"5min":1.2989999999999999,"15min":2.456},"min":{"1min":0.20499999999999999,"5min":0.17299999999999999,"15min":0.17299999999999999},"max":{"1min":7.4480000000000004,"5min":11.07,"15min":82.424999999999997},"last":0.45400000000000001},{"interface":"front","average":{"1min":2.972,"5min":2.1480000000000001,"15min":2.8140000000000001},"min":{"1min":0.26100000000000001,"5min":0.23599999999999999,"15min":0.19},"max":{"1min":11.244,"5min":16.678999999999998,"15min":82.524000000000001},"last":0.36799999999999999}]},{"osd":1,"last update":"Sat Sep 23 20:45:09 2023","interfaces":[{"interface":"back","average":{"1min":3.4609999999999999,"5min":2.218,"15min":2.746},"min":{"1min":0.314,"5min":0.248,"15min":0.221},"max":{"1min":12.797000000000001,"5min":12.981,"15min":46.405000000000001},"last":0.32300000000000001},{"interface":"front","average":{"1min":2.5720000000000001,"5min":2.125,"15min":2.6139999999999999},"min":{"1min":0.23200000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":13.215999999999999,"5min":13.215999999999999,"15min":46.459000000000003},"last":0.39600000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:44:31 2023","interfaces":[{"interface":"back","average":{"1min":2.1800000000000002,"5min":2.2010000000000001,"15min":2.5990000000000002},"min":{"1min":0.253,"5min":0.253,"15min":0.19800000000000001},"max":{"1min":12.847,"5min":12.847,"15min":19.562999999999999},"last":0.38200000000000001},{"interface":"front","average":{"1min":0.91400000000000003,"5min":1.425,"15min":2.1720000000000002},"min":{"1min":0.26500000000000001,"5min":0.26400000000000001,"15min":0.249},"max":{"1min":6.3700000000000001,"5min":11.375,"15min":30.704999999999998},"last":0.434}]},{"osd":4,"last update":"Sat Sep 23 20:45:09 2023","interfaces":[{"interface":"back","average":{"1min":2.4929999999999999,"5min":1.8169999999999999,"15min":1.976},"min":{"1min":0.32000000000000001,"5min":0.29799999999999999,"15min":0.27500000000000002},"max":{"1min":11.141999999999999,"5min":12.025,"15min":16.797999999999998},"last":0.56399999999999995},{"interface":"front","average":{"1min":2.593,"5min":1.831,"15min":2.0840000000000001},"min":{"1min":0.40500000000000003,"5min":0.30599999999999999,"15min":0.30499999999999999},"max":{"1min":10.948,"5min":10.948,"15min":19.440999999999999},"last":0.442}]},{"osd":5,"last update":"Sat Sep 23 20:45:09 2023","interfaces":[{"interface":"back","average":{"1min":2.6459999999999999,"5min":2.0470000000000002,"15min":2.1179999999999999},"min":{"1min":0.39600000000000002,"5min":0.311,"15min":0.311},"max":{"1min":11.183,"5min":12.07,"15min":16.766999999999999},"last":0.53400000000000003},{"interface":"front","average":{"1min":2.7559999999999998,"5min":1.831,"15min":2.2269999999999999},"min":{"1min":0.373,"5min":0.37,"15min":0.34799999999999998},"max":{"1min":11.061999999999999,"5min":11.061999999999999,"15min":19.513000000000002},"last":0.57199999999999995}]},{"osd":6,"last update":"Sat Sep 23 20:45:09 2023","interfaces":[{"interface":"back","average":{"1min":2.7069999999999999,"5min":2.121,"15min":2.262},"min":{"1min":0.39100000000000001,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":11.031000000000001,"5min":14.047000000000001,"15min":19.486000000000001},"last":0.54400000000000004},{"interface":"front","average":{"1min":1.415,"5min":1.304,"15min":1.6930000000000001},"min":{"1min":0.44500000000000001,"5min":0.34200000000000003,"15min":0.316},"max":{"1min":8.5920000000000005,"5min":11.747,"15min":30.670999999999999},"last":0.52200000000000002}]},{"osd":7,"last update":"Sat Sep 23 20:45:09 2023","interfaces":[{"interface":"back","average":{"1min":1.341,"5min":1.347,"15min":1.714},"min":{"1min":0.41999999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":3.9220000000000002,"5min":8.4949999999999992,"15min":30.741},"last":0.47499999999999998},{"interface":"front","average":{"1min":2.71,"5min":2.2730000000000001,"15min":2.294},"min":{"1min":0.35799999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":11.218,"5min":12.099,"15min":16.719000000000001},"last":0.58299999999999996}]}]},{"osd":3,"up_from":109,"seq":468151435548,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":976088,"kb_used_data":6864,"kb_used_omap":4,"kb_used_meta":969211,"kb_avail":92769064,"statfs":{"total":95995035648,"available":94995521536,"internally_reserved":0,"allocated":7028736,"data_stored":4503335,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4514,"internal_metadata":992472670},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:44:39 2023","interfaces":[{"interface":"back","average":{"1min":2.71,"5min":2.4169999999999998,"15min":2.4929999999999999},"min":{"1min":0.222,"5min":0.20899999999999999,"15min":0.191},"max":{"1min":8.5419999999999998,"5min":22.32,"15min":22.32},"last":0.53700000000000003},{"interface":"front","average":{"1min":2.105,"5min":2.6749999999999998,"15min":2.4929999999999999},"min":{"1min":0.249,"5min":0.22900000000000001,"15min":0.19500000000000001},"max":{"1min":6.2160000000000002,"5min":22.805,"15min":22.805},"last":0.54500000000000004}]},{"osd":1,"last update":"Sat Sep 23 20:45:12 2023","interfaces":[{"interface":"back","average":{"1min":3.1800000000000002,"5min":2.669,"15min":2.2349999999999999},"min":{"1min":0.21099999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":15.177,"5min":17.529,"15min":17.529},"last":0.501},{"interface":"front","average":{"1min":2.931,"5min":2.488,"15min":2.4279999999999999},"min":{"1min":0.29399999999999998,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":9.6600000000000001,"5min":14.19,"15min":32.603999999999999},"last":0.61299999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:44:39 2023","interfaces":[{"interface":"back","average":{"1min":2.2970000000000002,"5min":3.5840000000000001,"15min":3.0019999999999998},"min":{"1min":0.28599999999999998,"5min":0.26600000000000001,"15min":0.23799999999999999},"max":{"1min":7.7859999999999996,"5min":68.206999999999994,"15min":68.206999999999994},"last":0.58799999999999997},{"interface":"front","average":{"1min":2.4470000000000001,"5min":2.7130000000000001,"15min":2.7450000000000001},"min":{"1min":0.251,"5min":0.245,"15min":0.20899999999999999},"max":{"1min":9.1760000000000002,"5min":67.736999999999995,"15min":67.736999999999995},"last":0.46700000000000003}]},{"osd":4,"last update":"Sat Sep 23 20:44:39 2023","interfaces":[{"interface":"back","average":{"1min":2.2879999999999998,"5min":2.008,"15min":1.786},"min":{"1min":0.35799999999999998,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":6.7889999999999997,"5min":22.157,"15min":22.157},"last":1.214},{"interface":"front","average":{"1min":2.113,"5min":2.3290000000000002,"15min":2.3279999999999998},"min":{"1min":0.36499999999999999,"5min":0.33200000000000002,"15min":0.28100000000000003},"max":{"1min":7.9020000000000001,"5min":15.868,"15min":19.635000000000002},"last":1.254}]},{"osd":5,"last update":"Sat Sep 23 20:44:39 2023","interfaces":[{"interface":"back","average":{"1min":2.1840000000000002,"5min":2.2970000000000002,"15min":1.992},"min":{"1min":0.33800000000000002,"5min":0.32700000000000001,"15min":0.31900000000000001},"max":{"1min":6.7469999999999999,"5min":22.289999999999999,"15min":22.289999999999999},"last":0.72099999999999997},{"interface":"front","average":{"1min":1.8089999999999999,"5min":1.8440000000000001,"15min":1.841},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.32000000000000001},"max":{"1min":4.6559999999999997,"5min":12.952,"15min":12.952},"last":0.78500000000000003}]},{"osd":6,"last update":"Sat Sep 23 20:44:39 2023","interfaces":[{"interface":"back","average":{"1min":1.845,"5min":2.1360000000000001,"15min":1.956},"min":{"1min":0.36599999999999999,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":6.5350000000000001,"5min":12.917,"15min":17.524999999999999},"last":0.75900000000000001},{"interface":"front","average":{"1min":1.988,"5min":2.3300000000000001,"15min":2.2869999999999999},"min":{"1min":0.38100000000000001,"5min":0.35099999999999998,"15min":0.33700000000000002},"max":{"1min":6.8479999999999999,"5min":15.936,"15min":19.524000000000001},"last":1.23}]},{"osd":7,"last update":"Sat Sep 23 20:44:39 2023","interfaces":[{"interface":"back","average":{"1min":2.7850000000000001,"5min":2.7509999999999999,"15min":2.6429999999999998},"min":{"1min":0.46999999999999997,"5min":0.41499999999999998,"15min":0.36899999999999999},"max":{"1min":8.4429999999999996,"5min":15.976000000000001,"15min":19.571999999999999},"last":1.198},{"interface":"front","average":{"1min":2.4750000000000001,"5min":2.262,"15min":1.9850000000000001},"min":{"1min":0.46300000000000002,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":8.5289999999999999,"5min":22.202999999999999,"15min":22.202999999999999},"last":0.69199999999999995}]}]},{"osd":4,"up_from":30,"seq":128849019301,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":501884,"kb_used_data":72308,"kb_used_omap":0,"kb_used_meta":429568,"kb_avail":93243268,"statfs":{"total":95995035648,"available":95481106432,"internally_reserved":0,"allocated":74043392,"data_stored":72535065,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":439877632},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:45:00 2023","interfaces":[{"interface":"back","average":{"1min":2.0289999999999999,"5min":2.109,"15min":1.77},"min":{"1min":0.26100000000000001,"5min":0.251,"15min":0.24099999999999999},"max":{"1min":7.9029999999999996,"5min":11.709,"15min":24.114999999999998},"last":6.109},{"interface":"front","average":{"1min":2.343,"5min":2.3839999999999999,"15min":1.98},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.246},"max":{"1min":9.8580000000000005,"5min":13.74,"15min":24.135999999999999},"last":4.226}]},{"osd":1,"last update":"Sat Sep 23 20:45:12 2023","interfaces":[{"interface":"back","average":{"1min":2.6760000000000002,"5min":2.2410000000000001,"15min":1.819},"min":{"1min":0.374,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":9.673,"5min":10.363,"15min":18.190999999999999},"last":3.992},{"interface":"front","average":{"1min":1.9890000000000001,"5min":1.778,"15min":1.611},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":8.0730000000000004,"5min":10.426,"15min":18.173999999999999},"last":0.41499999999999998}]},{"osd":2,"last update":"Sat Sep 23 20:45:12 2023","interfaces":[{"interface":"back","average":{"1min":1.593,"5min":1.8069999999999999,"15min":1.788},"min":{"1min":0.35799999999999998,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":7.8559999999999999,"5min":13.051,"15min":13.051},"last":0.65100000000000002},{"interface":"front","average":{"1min":2.7650000000000001,"5min":2.1659999999999999,"15min":2.105},"min":{"1min":0.40799999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":6.8479999999999999,"5min":10.945,"15min":14.614000000000001},"last":0.72499999999999998}]},{"osd":3,"last update":"Sat Sep 23 20:44:27 2023","interfaces":[{"interface":"back","average":{"1min":1.877,"5min":2.04,"15min":1.9950000000000001},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":7.7480000000000002,"5min":12.147,"15min":14.515000000000001},"last":1.591},{"interface":"front","average":{"1min":2.4260000000000002,"5min":2.0990000000000002,"15min":2.0699999999999998},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.30299999999999999},"max":{"1min":10.493,"5min":13.35,"15min":15.991},"last":0.54000000000000004}]},{"osd":5,"last update":"Sat Sep 23 20:45:00 2023","interfaces":[{"interface":"back","average":{"1min":1.504,"5min":1.486,"15min":1.0569999999999999},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":10.202,"5min":10.202,"15min":10.202},"last":4.1310000000000002},{"interface":"front","average":{"1min":1.895,"5min":1.389,"15min":1.0940000000000001},"min":{"1min":0.316,"5min":0.311,"15min":0.28999999999999998},"max":{"1min":8.1440000000000001,"5min":8.1440000000000001,"15min":8.1440000000000001},"last":0.45600000000000002}]},{"osd":6,"last update":"Sat Sep 23 20:45:12 2023","interfaces":[{"interface":"back","average":{"1min":2.0409999999999999,"5min":1.609,"15min":1.2350000000000001},"min":{"1min":0.33900000000000002,"5min":0.29499999999999998,"15min":0.27600000000000002},"max":{"1min":10.840999999999999,"5min":10.840999999999999,"15min":14.731999999999999},"last":0.48699999999999999},{"interface":"front","average":{"1min":1.6879999999999999,"5min":1.4319999999999999,"15min":1.0840000000000001},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":4.9820000000000002,"5min":7.6150000000000002,"15min":7.6150000000000002},"last":4.165}]},{"osd":7,"last update":"Sat Sep 23 20:44:27 2023","interfaces":[{"interface":"back","average":{"1min":2.0630000000000002,"5min":2.0979999999999999,"15min":1.5649999999999999},"min":{"1min":0.374,"5min":0.35099999999999998,"15min":0.318},"max":{"1min":7.7629999999999999,"5min":22.184999999999999,"15min":22.184999999999999},"last":0.56999999999999995},{"interface":"front","average":{"1min":1.1240000000000001,"5min":1.724,"15min":1.2150000000000001},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.307},"max":{"1min":6.3689999999999998,"5min":21.754000000000001,"15min":21.754000000000001},"last":4.0419999999999998}]}]},{"osd":5,"up_from":39,"seq":167503724962,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":518008,"kb_used_data":5680,"kb_used_omap":0,"kb_used_meta":512320,"kb_avail":93227144,"statfs":{"total":95995035648,"available":95464595456,"internally_reserved":0,"allocated":5816320,"data_stored":4308719,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":524615680},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:45:02 2023","interfaces":[{"interface":"back","average":{"1min":2.222,"5min":1.8500000000000001,"15min":1.887},"min":{"1min":0.26300000000000001,"5min":0.22500000000000001,"15min":0.20699999999999999},"max":{"1min":11.25,"5min":15.018000000000001,"15min":18.16},"last":0.28699999999999998},{"interface":"front","average":{"1min":1.5589999999999999,"5min":1.726,"15min":1.766},"min":{"1min":0.307,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":6.71,"5min":15.034000000000001,"15min":15.034000000000001},"last":3.3260000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:44:14 2023","interfaces":[{"interface":"back","average":{"1min":2.7349999999999999,"5min":3.0299999999999998,"15min":2.2690000000000001},"min":{"1min":0.379,"5min":0.26500000000000001,"15min":0.253},"max":{"1min":9.4199999999999999,"5min":61.005000000000003,"15min":61.005000000000003},"last":2.4399999999999999},{"interface":"front","average":{"1min":1.339,"5min":2.4129999999999998,"15min":2.0019999999999998},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.23499999999999999},"max":{"1min":6.7350000000000003,"5min":59.688000000000002,"15min":59.688000000000002},"last":2.3399999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:45:02 2023","interfaces":[{"interface":"back","average":{"1min":2.2919999999999998,"5min":2.2610000000000001,"15min":2.2010000000000001},"min":{"1min":0.34599999999999997,"5min":0.27100000000000002,"15min":0.26500000000000001},"max":{"1min":12.226000000000001,"5min":12.226000000000001,"15min":37.011000000000003},"last":1.496},{"interface":"front","average":{"1min":1.246,"5min":1.893,"15min":2.0390000000000001},"min":{"1min":0.33000000000000002,"5min":0.28299999999999997,"15min":0.255},"max":{"1min":3.5259999999999998,"5min":10.565,"15min":36.835000000000001},"last":3.3530000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:44:20 2023","interfaces":[{"interface":"back","average":{"1min":1.7769999999999999,"5min":2.375,"15min":2.375},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.29599999999999999},"max":{"1min":5.6619999999999999,"5min":11.032,"15min":36.716000000000001},"last":0.67700000000000005},{"interface":"front","average":{"1min":1.5900000000000001,"5min":2.238,"15min":2.2530000000000001},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":7.4100000000000001,"5min":13.266,"15min":18.504000000000001},"last":0.35599999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:44:54 2023","interfaces":[{"interface":"back","average":{"1min":1.1850000000000001,"5min":1.857,"15min":1.2729999999999999},"min":{"1min":0.30199999999999999,"5min":0.27300000000000002,"15min":0.255},"max":{"1min":5.1050000000000004,"5min":13.391999999999999,"15min":28.015999999999998},"last":0.70499999999999996},{"interface":"front","average":{"1min":1.1499999999999999,"5min":1.6319999999999999,"15min":1.1200000000000001},"min":{"1min":0.32000000000000001,"5min":0.30299999999999999,"15min":0.28299999999999997},"max":{"1min":6.8399999999999999,"5min":7.3799999999999999,"15min":12.300000000000001},"last":0.58499999999999996}]},{"osd":6,"last update":"Sat Sep 23 20:45:08 2023","interfaces":[{"interface":"back","average":{"1min":1.9350000000000001,"5min":2.161,"15min":1.319},"min":{"1min":0.38100000000000001,"5min":0.317,"15min":0.30199999999999999},"max":{"1min":6.952,"5min":13.734,"15min":13.734},"last":0.64900000000000002},{"interface":"front","average":{"1min":1.1220000000000001,"5min":1.5660000000000001,"15min":1.0980000000000001},"min":{"1min":0.35299999999999998,"5min":0.317,"15min":0.317},"max":{"1min":5.2629999999999999,"5min":8.7449999999999992,"15min":8.7449999999999992},"last":0.61699999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:44:20 2023","interfaces":[{"interface":"back","average":{"1min":1.4610000000000001,"5min":1.7330000000000001,"15min":1.1339999999999999},"min":{"1min":0.372,"5min":0.34499999999999997,"15min":0.28499999999999998},"max":{"1min":4.3250000000000002,"5min":16.366,"15min":16.366},"last":0.86499999999999999},{"interface":"front","average":{"1min":1.4950000000000001,"5min":1.6439999999999999,"15min":1.173},"min":{"1min":0.38900000000000001,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":7.524,"5min":7.915,"15min":13.122},"last":3.2879999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4722,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1360,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8206,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7156,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3154,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":136708096,"data_stored":136708096,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":62128128,"data_stored":62128128,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":34209792,"data_stored":34209792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:45:21.874 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-09-23T20:45:21.875 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd pool set unique_pool_4 pg_num 26 2023-09-23T20:45:21.942 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:45:21.974 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:22.011 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:45:22.014 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:22.015 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:45:22.115 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:45:22.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:21 smithi018 ceph-mon[179647]: pgmap v1316: 119 pgs: 119 active+clean; 4.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 742 op/s 2023-09-23T20:45:22.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:21 smithi018 ceph-mon[179647]: from='client.46550 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:45:22.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:21 smithi018 ceph-mon[179647]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:45:22.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:21 smithi018 ceph-mon[179647]: failed to encode map e524 with expected crc 2023-09-23T20:45:22.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:21 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:45:22.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:21 smithi018 ceph-mon[179647]: osdmap e524: 8 total, 8 up, 8 in 2023-09-23T20:45:22.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:21 smithi018 ceph-mon[184144]: pgmap v1316: 119 pgs: 119 active+clean; 4.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 742 op/s 2023-09-23T20:45:22.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:21 smithi018 ceph-mon[184144]: from='client.46550 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:45:22.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:21 smithi018 ceph-mon[184144]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:45:22.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:21 smithi018 ceph-mon[184144]: failed to encode map e524 with expected crc 2023-09-23T20:45:22.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:21 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:45:22.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:21 smithi018 ceph-mon[184144]: osdmap e524: 8 total, 8 up, 8 in 2023-09-23T20:45:22.145 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:22.190 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:45:22.195 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:22.195 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:45:22.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:21 smithi149 ceph-mon[170394]: pgmap v1316: 119 pgs: 119 active+clean; 4.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 742 op/s 2023-09-23T20:45:22.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:21 smithi149 ceph-mon[170394]: from='client.46550 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:45:22.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:21 smithi149 ceph-mon[170394]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:45:22.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:21 smithi149 ceph-mon[170394]: failed to encode map e524 with expected crc 2023-09-23T20:45:22.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:21 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:45:22.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:21 smithi149 ceph-mon[170394]: osdmap e524: 8 total, 8 up, 8 in 2023-09-23T20:45:22.263 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:45:22.265 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2023-09-23T20:45:22.265 INFO:teuthology.orchestra.run.smithi018.stdout: "ops": [], 2023-09-23T20:45:22.265 INFO:teuthology.orchestra.run.smithi018.stdout: "complaint_time": 30, 2023-09-23T20:45:22.265 INFO:teuthology.orchestra.run.smithi018.stdout: "num_blocked_ops": 0 2023-09-23T20:45:22.265 INFO:teuthology.orchestra.run.smithi018.stdout:} 2023-09-23T20:45:22.296 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:45:22.328 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:22.365 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:45:22.368 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:22.369 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:45:22.470 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:45:22.498 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:22.534 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:45:22.536 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:22.537 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:45:22.638 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:45:22.674 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:22.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:22 smithi018 ceph-mon[179647]: failed to encode map e524 with expected crc 2023-09-23T20:45:22.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:22 smithi018 ceph-mon[179647]: failed to encode map e524 with expected crc 2023-09-23T20:45:22.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:22 smithi018 ceph-mon[184144]: failed to encode map e524 with expected crc 2023-09-23T20:45:22.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:22 smithi018 ceph-mon[184144]: failed to encode map e524 with expected crc 2023-09-23T20:45:22.911 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:45:23.145 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:45:23.150 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:23.150 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:45:23.170 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T20:45:23.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:22 smithi149 ceph-mon[170394]: failed to encode map e524 with expected crc 2023-09-23T20:45:23.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:22 smithi149 ceph-mon[170394]: failed to encode map e524 with expected crc 2023-09-23T20:45:23.252 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:45:23.283 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:23.319 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:45:23.321 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:23.322 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:45:23.423 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:45:23.451 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:23.488 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:45:23.491 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:23.492 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:45:23.592 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:45:23.626 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:23.659 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:45:23.664 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:23.664 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:45:23.765 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:45:23.796 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:23.904 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T20:45:24.008 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:23 smithi018 ceph-mon[179647]: pgmap v1318: 119 pgs: 119 active+clean; 4.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 582 op/s 2023-09-23T20:45:24.008 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:23 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:45:24.008 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:23 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:45:24.008 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:23 smithi018 ceph-mon[184144]: pgmap v1318: 119 pgs: 119 active+clean; 4.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 582 op/s 2023-09-23T20:45:24.008 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:23 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:45:24.009 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:23 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:45:24.132 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:45:24.135 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:24.135 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:45:24.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:23 smithi149 ceph-mon[170394]: pgmap v1318: 119 pgs: 119 active+clean; 4.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 582 op/s 2023-09-23T20:45:24.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:23 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:45:24.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:23 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:45:24.236 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:45:24.268 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:24.304 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:45:24.307 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:24.307 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:45:24.311 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:45:24.329 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:45:24.335 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:45:24.352 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:45:24.379 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:45:24.402 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:45:24.407 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:45:24.422 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:45:24.437 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:24.442 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:45:24.463 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:45:24.464 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:45:24.464 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:45:24.465 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:45:24.465 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:45:24.465 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:45:24.465 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:45:24.465 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:45:24.465 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:45:24.930 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:45:24.933 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:24.933 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:45:25.035 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:45:25.067 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:25.080 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:45:24 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:45:24] "GET /metrics HTTP/1.1" 200 36506 "" "Prometheus/2.43.0" 2023-09-23T20:45:25.081 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:24 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1348724334' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:45:25.081 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:24 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:45:25.081 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:24 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:45:25.081 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:24 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:45:25.082 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:24 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1348724334' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:45:25.082 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:24 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:45:25.082 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:24 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:45:25.082 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:24 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:45:25.103 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:45:25.106 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:25.107 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:45:25.208 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:45:25.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:24 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1348724334' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:45:25.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:24 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:45:25.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:24 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:45:25.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:24 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:45:25.238 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:25.273 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:45:25.276 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:25.277 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:45:25.377 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:45:25.405 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:25.439 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:45:25.442 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:25.443 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:45:25.543 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:45:25.577 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:25.617 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:45:25.619 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:25.620 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:45:25.681 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:45:25.721 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:45:25.753 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:25.812 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:45:25.815 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:25.816 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:45:25.916 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:45:25.946 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:25.982 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:45:25.985 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:25.986 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:45:26.086 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:45:26.113 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:25 smithi018 ceph-mon[184144]: pgmap v1319: 119 pgs: 119 active+clean; 4.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 294 op/s 2023-09-23T20:45:26.113 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:25 smithi018 ceph-mon[179647]: pgmap v1319: 119 pgs: 119 active+clean; 4.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 294 op/s 2023-09-23T20:45:26.115 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:26.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:25 smithi149 ceph-mon[170394]: pgmap v1319: 119 pgs: 119 active+clean; 4.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 294 op/s 2023-09-23T20:45:26.250 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:45:26.252 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:26.252 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:45:26.351 INFO:teuthology.orchestra.run.smithi018.stdout:ERROR: (22) Invalid argument 2023-09-23T20:45:26.351 INFO:teuthology.orchestra.run.smithi018.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:45:26.353 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:45:26.385 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:26.420 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:45:26.423 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:26.423 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:45:26.524 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:45:26.551 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:26.585 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:45:26.588 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:26.588 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:45:26.689 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:45:26.718 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:26.861 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:45:26.865 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:26.866 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:45:26.873 INFO:teuthology.orchestra.run.smithi018.stderr:set pool 9 pg_num to 26 2023-09-23T20:45:26.882 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:26 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2953103559' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T20:45:26.883 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:26 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2953103559' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T20:45:26.947 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T20:45:26.966 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:45:26.993 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:45:26.994 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:27.027 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:45:27.030 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:27.030 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:45:27.131 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:45:27.155 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:26 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2953103559' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T20:45:27.157 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:27.189 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:45:27.192 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:27.192 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:45:27.293 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:45:27.321 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:27.359 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:45:27.362 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:27.362 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:45:27.463 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:45:27.475 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_4 2023-09-23T20:45:27.475 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd pool rm unique_pool_4 unique_pool_4 --yes-i-really-really-mean-it 2023-09-23T20:45:27.498 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:27.585 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:45:27.588 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:27.589 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:45:27.689 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:45:27.719 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:27.760 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:45:27.764 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:27.764 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:45:27.865 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:45:27.868 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:27 smithi018 ceph-mon[184144]: pgmap v1320: 119 pgs: 119 active+clean; 4.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 208 op/s 2023-09-23T20:45:27.868 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:27 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2953103559' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"}]': finished 2023-09-23T20:45:27.868 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:27 smithi018 ceph-mon[184144]: osdmap e525: 8 total, 8 up, 8 in 2023-09-23T20:45:27.868 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:27 smithi018 ceph-mon[184144]: failed to encode map e525 with expected crc 2023-09-23T20:45:27.894 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:27.920 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:45:27.923 INFO:teuthology.orchestra.run.smithi018.stderr:nodeep-scrub is unset 2023-09-23T20:45:27.931 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:45:27.933 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:27.934 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:45:28.035 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:45:28.064 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:28.111 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:45:28.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:27 smithi018 ceph-mon[179647]: pgmap v1320: 119 pgs: 119 active+clean; 4.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 208 op/s 2023-09-23T20:45:28.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:27 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2953103559' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"}]': finished 2023-09-23T20:45:28.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:27 smithi018 ceph-mon[179647]: osdmap e525: 8 total, 8 up, 8 in 2023-09-23T20:45:28.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:27 smithi018 ceph-mon[179647]: failed to encode map e525 with expected crc 2023-09-23T20:45:28.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:27 smithi149 ceph-mon[170394]: pgmap v1320: 119 pgs: 119 active+clean; 4.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 208 op/s 2023-09-23T20:45:28.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:27 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2953103559' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"}]': finished 2023-09-23T20:45:28.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:27 smithi149 ceph-mon[170394]: osdmap e525: 8 total, 8 up, 8 in 2023-09-23T20:45:28.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:27 smithi149 ceph-mon[170394]: failed to encode map e525 with expected crc 2023-09-23T20:45:28.451 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:45:28.454 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:28.455 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:45:28.556 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:45:28.588 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:28.689 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:45:28.877 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:45:28.880 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:28.881 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:45:28.982 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:45:29.039 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:29.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:28 smithi018 ceph-mon[184144]: failed to encode map e525 with expected crc 2023-09-23T20:45:29.144 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:28 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/4256844657' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:45:29.144 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:28 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:45:29.144 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:28 smithi018 ceph-mon[184144]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:45:29.144 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:28 smithi018 ceph-mon[184144]: Cluster is now healthy 2023-09-23T20:45:29.144 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:28 smithi018 ceph-mon[184144]: failed to encode map e526 with expected crc 2023-09-23T20:45:29.144 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:28 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:45:29.144 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:28 smithi018 ceph-mon[184144]: osdmap e526: 8 total, 8 up, 8 in 2023-09-23T20:45:29.144 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:28 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-09-23T20:45:29.153 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:28 smithi018 ceph-mon[179647]: failed to encode map e525 with expected crc 2023-09-23T20:45:29.153 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:28 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/4256844657' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:45:29.153 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:28 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:45:29.153 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:28 smithi018 ceph-mon[179647]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:45:29.153 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:28 smithi018 ceph-mon[179647]: Cluster is now healthy 2023-09-23T20:45:29.153 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:28 smithi018 ceph-mon[179647]: failed to encode map e526 with expected crc 2023-09-23T20:45:29.154 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:28 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:45:29.154 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:28 smithi018 ceph-mon[179647]: osdmap e526: 8 total, 8 up, 8 in 2023-09-23T20:45:29.154 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:28 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-09-23T20:45:29.210 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:45:29.215 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:29.216 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:45:29.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:28 smithi149 ceph-mon[170394]: failed to encode map e525 with expected crc 2023-09-23T20:45:29.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:28 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/4256844657' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:45:29.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:28 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:45:29.218 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:28 smithi149 ceph-mon[170394]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:45:29.218 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:28 smithi149 ceph-mon[170394]: Cluster is now healthy 2023-09-23T20:45:29.218 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:28 smithi149 ceph-mon[170394]: failed to encode map e526 with expected crc 2023-09-23T20:45:29.218 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:28 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:45:29.218 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:28 smithi149 ceph-mon[170394]: osdmap e526: 8 total, 8 up, 8 in 2023-09-23T20:45:29.218 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:28 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-09-23T20:45:29.317 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:45:29.439 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:29.857 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:45:29.860 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:29.860 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:45:29.961 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:45:30.061 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:29 smithi018 ceph-mon[184144]: pgmap v1323: 119 pgs: 119 active+clean; 4.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail 2023-09-23T20:45:30.061 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:29 smithi018 ceph-mon[184144]: failed to encode map e527 with expected crc 2023-09-23T20:45:30.061 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:29 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "26"}]': finished 2023-09-23T20:45:30.061 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:29 smithi018 ceph-mon[184144]: osdmap e527: 8 total, 8 up, 8 in 2023-09-23T20:45:30.061 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:29 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:45:30.062 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:29 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:45:30.062 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:29 smithi018 ceph-mon[179647]: pgmap v1323: 119 pgs: 119 active+clean; 4.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail 2023-09-23T20:45:30.062 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:29 smithi018 ceph-mon[179647]: failed to encode map e527 with expected crc 2023-09-23T20:45:30.062 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:29 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "26"}]': finished 2023-09-23T20:45:30.062 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:29 smithi018 ceph-mon[179647]: osdmap e527: 8 total, 8 up, 8 in 2023-09-23T20:45:30.062 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:29 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:45:30.062 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:29 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:45:30.066 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:30.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:29 smithi149 ceph-mon[170394]: pgmap v1323: 119 pgs: 119 active+clean; 4.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail 2023-09-23T20:45:30.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:29 smithi149 ceph-mon[170394]: failed to encode map e527 with expected crc 2023-09-23T20:45:30.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:29 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "26"}]': finished 2023-09-23T20:45:30.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:29 smithi149 ceph-mon[170394]: osdmap e527: 8 total, 8 up, 8 in 2023-09-23T20:45:30.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:29 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:45:30.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:29 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:45:30.464 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T20:45:30.524 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:45:30.527 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:30.528 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:45:30.628 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:45:30.659 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:30.701 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:45:30.705 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:30.705 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:45:30.806 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:45:30.833 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:30.933 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:30 smithi018 ceph-mon[184144]: failed to encode map e528 with expected crc 2023-09-23T20:45:30.934 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:30 smithi018 ceph-mon[184144]: osdmap e528: 8 total, 8 up, 8 in 2023-09-23T20:45:30.934 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:30 smithi018 ceph-mon[184144]: failed to encode map e528 with expected crc 2023-09-23T20:45:30.934 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:30 smithi018 ceph-mon[184144]: failed to encode map e528 with expected crc 2023-09-23T20:45:30.934 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:30 smithi018 ceph-mon[184144]: failed to encode map e528 with expected crc 2023-09-23T20:45:30.934 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:30 smithi018 ceph-mon[184144]: failed to encode map e528 with expected crc 2023-09-23T20:45:30.934 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:30 smithi018 ceph-mon[184144]: 9.5 scrub starts 2023-09-23T20:45:30.934 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:30 smithi018 ceph-mon[184144]: 9.5 scrub ok 2023-09-23T20:45:30.934 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:30 smithi018 ceph-mon[184144]: pgmap v1326: 129 pgs: 2 peering, 10 unknown, 117 active+clean; 4.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail 2023-09-23T20:45:30.934 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:30 smithi018 ceph-mon[179647]: failed to encode map e528 with expected crc 2023-09-23T20:45:30.935 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:30 smithi018 ceph-mon[179647]: osdmap e528: 8 total, 8 up, 8 in 2023-09-23T20:45:30.935 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:30 smithi018 ceph-mon[179647]: failed to encode map e528 with expected crc 2023-09-23T20:45:30.935 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:30 smithi018 ceph-mon[179647]: failed to encode map e528 with expected crc 2023-09-23T20:45:30.935 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:30 smithi018 ceph-mon[179647]: failed to encode map e528 with expected crc 2023-09-23T20:45:30.935 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:30 smithi018 ceph-mon[179647]: failed to encode map e528 with expected crc 2023-09-23T20:45:30.935 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:30 smithi018 ceph-mon[179647]: 9.5 scrub starts 2023-09-23T20:45:30.935 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:30 smithi018 ceph-mon[179647]: 9.5 scrub ok 2023-09-23T20:45:30.935 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:30 smithi018 ceph-mon[179647]: pgmap v1326: 129 pgs: 2 peering, 10 unknown, 117 active+clean; 4.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail 2023-09-23T20:45:31.003 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:45:31.011 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:31.011 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:45:31.025 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:45:31.030 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2023-09-23T20:45:31.030 INFO:teuthology.orchestra.run.smithi018.stdout: "ops": [], 2023-09-23T20:45:31.030 INFO:teuthology.orchestra.run.smithi018.stdout: "num_ops": 0 2023-09-23T20:45:31.030 INFO:teuthology.orchestra.run.smithi018.stdout:} 2023-09-23T20:45:31.046 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:45:31.068 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:45:31.094 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:45:31.112 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:45:31.123 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:45:31.143 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:31.150 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:45:31.180 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:45:31.199 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:45:31.201 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:45:31.201 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:45:31.201 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:45:31.201 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:45:31.201 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:45:31.202 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:45:31.202 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:45:31.202 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:45:31.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:30 smithi149 ceph-mon[170394]: failed to encode map e528 with expected crc 2023-09-23T20:45:31.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:30 smithi149 ceph-mon[170394]: osdmap e528: 8 total, 8 up, 8 in 2023-09-23T20:45:31.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:30 smithi149 ceph-mon[170394]: failed to encode map e528 with expected crc 2023-09-23T20:45:31.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:30 smithi149 ceph-mon[170394]: failed to encode map e528 with expected crc 2023-09-23T20:45:31.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:30 smithi149 ceph-mon[170394]: failed to encode map e528 with expected crc 2023-09-23T20:45:31.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:30 smithi149 ceph-mon[170394]: failed to encode map e528 with expected crc 2023-09-23T20:45:31.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:30 smithi149 ceph-mon[170394]: 9.5 scrub starts 2023-09-23T20:45:31.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:30 smithi149 ceph-mon[170394]: 9.5 scrub ok 2023-09-23T20:45:31.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:30 smithi149 ceph-mon[170394]: pgmap v1326: 129 pgs: 2 peering, 10 unknown, 117 active+clean; 4.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail 2023-09-23T20:45:31.621 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T20:45:31.699 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:45:31.701 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:31.702 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:45:31.803 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:45:31.834 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:31.880 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:45:31.885 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:31.886 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:45:31.987 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:45:32.019 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:32.055 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:45:32.058 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:32.059 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:45:32.088 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:45:32.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:31 smithi018 ceph-mon[179647]: 9.6 deep-scrub starts 2023-09-23T20:45:32.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:31 smithi018 ceph-mon[179647]: 9.6 deep-scrub ok 2023-09-23T20:45:32.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:31 smithi018 ceph-mon[179647]: 9.7 scrub starts 2023-09-23T20:45:32.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:31 smithi018 ceph-mon[179647]: 9.7 scrub ok 2023-09-23T20:45:32.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:31 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/149101482' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:45:32.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:31 smithi018 ceph-mon[179647]: 9.9 scrub starts 2023-09-23T20:45:32.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:31 smithi018 ceph-mon[179647]: 9.9 scrub ok 2023-09-23T20:45:32.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:31 smithi018 ceph-mon[179647]: 9.1 scrub starts 2023-09-23T20:45:32.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:31 smithi018 ceph-mon[179647]: 9.1 scrub ok 2023-09-23T20:45:32.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:31 smithi018 ceph-mon[179647]: 9.2 scrub starts 2023-09-23T20:45:32.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:31 smithi018 ceph-mon[184144]: 9.6 deep-scrub starts 2023-09-23T20:45:32.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:31 smithi018 ceph-mon[184144]: 9.6 deep-scrub ok 2023-09-23T20:45:32.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:31 smithi018 ceph-mon[184144]: 9.7 scrub starts 2023-09-23T20:45:32.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:31 smithi018 ceph-mon[184144]: 9.7 scrub ok 2023-09-23T20:45:32.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:31 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/149101482' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:45:32.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:31 smithi018 ceph-mon[184144]: 9.9 scrub starts 2023-09-23T20:45:32.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:31 smithi018 ceph-mon[184144]: 9.9 scrub ok 2023-09-23T20:45:32.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:31 smithi018 ceph-mon[184144]: 9.1 scrub starts 2023-09-23T20:45:32.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:31 smithi018 ceph-mon[184144]: 9.1 scrub ok 2023-09-23T20:45:32.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:31 smithi018 ceph-mon[184144]: 9.2 scrub starts 2023-09-23T20:45:32.160 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:45:32.186 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:31 smithi149 ceph-mon[170394]: 9.6 deep-scrub starts 2023-09-23T20:45:32.186 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:31 smithi149 ceph-mon[170394]: 9.6 deep-scrub ok 2023-09-23T20:45:32.187 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:31 smithi149 ceph-mon[170394]: 9.7 scrub starts 2023-09-23T20:45:32.187 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:31 smithi149 ceph-mon[170394]: 9.7 scrub ok 2023-09-23T20:45:32.187 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:31 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/149101482' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:45:32.187 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:31 smithi149 ceph-mon[170394]: 9.9 scrub starts 2023-09-23T20:45:32.187 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:31 smithi149 ceph-mon[170394]: 9.9 scrub ok 2023-09-23T20:45:32.187 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:31 smithi149 ceph-mon[170394]: 9.1 scrub starts 2023-09-23T20:45:32.187 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:31 smithi149 ceph-mon[170394]: 9.1 scrub ok 2023-09-23T20:45:32.187 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:31 smithi149 ceph-mon[170394]: 9.2 scrub starts 2023-09-23T20:45:32.189 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:32.224 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:45:32.227 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:32.227 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:45:32.328 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:45:32.355 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:32.389 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:45:32.392 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:32.392 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:45:32.476 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 7, 3, 2, 4, 6, 5] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T20:45:32.478 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:45:32.479 INFO:tasks.thrashosds.thrasher:inject_pause on osd.0 2023-09-23T20:45:32.479 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T20:45:32.479 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T20:45:32.479 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 config set filestore_inject_stall 3 2023-09-23T20:45:32.493 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:45:32.519 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:32.522 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:45:32.551 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:45:32.554 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:32.554 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:45:32.655 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:45:32.681 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:32.714 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:45:32.716 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:32.717 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:45:32.817 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:45:32.845 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:32.879 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:45:32.883 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:32.883 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:45:32.945 INFO:teuthology.orchestra.run.smithi018.stderr:pool 'unique_pool_4' removed 2023-09-23T20:45:32.985 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:45:33.101 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:33.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:32 smithi018 ceph-mon[184144]: 9.2 scrub ok 2023-09-23T20:45:33.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:32 smithi018 ceph-mon[184144]: 9.3 scrub starts 2023-09-23T20:45:33.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:32 smithi018 ceph-mon[184144]: 9.3 scrub ok 2023-09-23T20:45:33.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:32 smithi018 ceph-mon[184144]: pgmap v1327: 129 pgs: 2 peering, 8 unknown, 119 active+clean; 4.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail 2023-09-23T20:45:33.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:32 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/394047534' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T20:45:33.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:32 smithi018 ceph-mon[184144]: 9.8 scrub starts 2023-09-23T20:45:33.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:32 smithi018 ceph-mon[179647]: 9.2 scrub ok 2023-09-23T20:45:33.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:32 smithi018 ceph-mon[179647]: 9.3 scrub starts 2023-09-23T20:45:33.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:32 smithi018 ceph-mon[179647]: 9.3 scrub ok 2023-09-23T20:45:33.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:32 smithi018 ceph-mon[179647]: pgmap v1327: 129 pgs: 2 peering, 8 unknown, 119 active+clean; 4.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail 2023-09-23T20:45:33.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:32 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/394047534' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T20:45:33.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:32 smithi018 ceph-mon[179647]: 9.8 scrub starts 2023-09-23T20:45:33.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:32 smithi149 ceph-mon[170394]: 9.2 scrub ok 2023-09-23T20:45:33.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:32 smithi149 ceph-mon[170394]: 9.3 scrub starts 2023-09-23T20:45:33.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:32 smithi149 ceph-mon[170394]: 9.3 scrub ok 2023-09-23T20:45:33.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:32 smithi149 ceph-mon[170394]: pgmap v1327: 129 pgs: 2 peering, 8 unknown, 119 active+clean; 4.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail 2023-09-23T20:45:33.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:32 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/394047534' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T20:45:33.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:32 smithi149 ceph-mon[170394]: 9.8 scrub starts 2023-09-23T20:45:33.660 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:45:33.663 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:33.664 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:45:33.688 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:45:33.731 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-09-23T20:45:33.731 INFO:tasks.radosbench:Beginning radosbench... 2023-09-23T20:45:33.731 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_5 2023-09-23T20:45:33.732 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd pool create unique_pool_5 16 2023-09-23T20:45:33.765 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:45:33.800 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:33.987 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:45:33.991 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:33.992 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:45:34.086 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:45:34.093 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:45:34.125 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:34.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:33 smithi018 ceph-mon[184144]: 9.8 scrub ok 2023-09-23T20:45:34.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:33 smithi018 ceph-mon[184144]: failed to encode map e529 with expected crc 2023-09-23T20:45:34.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:33 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/394047534' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T20:45:34.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:33 smithi018 ceph-mon[184144]: osdmap e529: 8 total, 8 up, 8 in 2023-09-23T20:45:34.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:33 smithi018 ceph-mon[179647]: 9.8 scrub ok 2023-09-23T20:45:34.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:33 smithi018 ceph-mon[179647]: failed to encode map e529 with expected crc 2023-09-23T20:45:34.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:33 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/394047534' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T20:45:34.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:33 smithi018 ceph-mon[179647]: osdmap e529: 8 total, 8 up, 8 in 2023-09-23T20:45:34.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:33 smithi149 ceph-mon[170394]: 9.8 scrub ok 2023-09-23T20:45:34.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:33 smithi149 ceph-mon[170394]: failed to encode map e529 with expected crc 2023-09-23T20:45:34.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:33 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/394047534' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T20:45:34.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:33 smithi149 ceph-mon[170394]: osdmap e529: 8 total, 8 up, 8 in 2023-09-23T20:45:34.307 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:45:34.312 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:34.313 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:45:34.413 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:45:34.446 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:34.482 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:45:34.485 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:34.486 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:45:34.587 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:45:34.616 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:34.651 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:45:34.654 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:34.655 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:45:34.696 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:45:34.756 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:45:34.789 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:34.866 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2023-09-23T20:45:34.866 INFO:teuthology.orchestra.run.smithi018.stdout: "ops": [], 2023-09-23T20:45:34.866 INFO:teuthology.orchestra.run.smithi018.stdout: "complaint_time": 30, 2023-09-23T20:45:34.866 INFO:teuthology.orchestra.run.smithi018.stdout: "num_blocked_ops": 0 2023-09-23T20:45:34.866 INFO:teuthology.orchestra.run.smithi018.stdout:} 2023-09-23T20:45:34.926 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:45:34.956 INFO:teuthology.orchestra.run.smithi018.stderr:noscrub is set 2023-09-23T20:45:35.184 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:34 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/856101437' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:45:35.184 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:34 smithi018 ceph-mon[184144]: pgmap v1329: 103 pgs: 103 active+clean; 4.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail 2023-09-23T20:45:35.184 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:45:34 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:45:34] "GET /metrics HTTP/1.1" 200 36506 "" "Prometheus/2.43.0" 2023-09-23T20:45:35.184 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:34 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/856101437' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:45:35.184 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:34 smithi018 ceph-mon[179647]: pgmap v1329: 103 pgs: 103 active+clean; 4.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail 2023-09-23T20:45:35.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:34 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/856101437' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:45:35.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:34 smithi149 ceph-mon[170394]: pgmap v1329: 103 pgs: 103 active+clean; 4.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail 2023-09-23T20:45:35.376 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:45:35.381 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:35.381 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:45:35.482 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:45:35.504 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T20:45:35.512 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:35.547 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:45:35.550 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:35.551 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:45:35.651 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:45:35.682 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:36.036 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:35 smithi018 ceph-mon[184144]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:45:36.036 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:35 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/856101437' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:45:36.036 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:35 smithi018 ceph-mon[184144]: osdmap e530: 8 total, 8 up, 8 in 2023-09-23T20:45:36.036 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:35 smithi018 ceph-mon[184144]: failed to encode map e530 with expected crc 2023-09-23T20:45:36.036 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:35 smithi018 ceph-mon[184144]: failed to encode map e530 with expected crc 2023-09-23T20:45:36.036 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:35 smithi018 ceph-mon[184144]: failed to encode map e530 with expected crc 2023-09-23T20:45:36.037 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:35 smithi018 ceph-mon[179647]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:45:36.037 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:35 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/856101437' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:45:36.038 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:35 smithi018 ceph-mon[179647]: osdmap e530: 8 total, 8 up, 8 in 2023-09-23T20:45:36.038 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:35 smithi018 ceph-mon[179647]: failed to encode map e530 with expected crc 2023-09-23T20:45:36.038 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:35 smithi018 ceph-mon[179647]: failed to encode map e530 with expected crc 2023-09-23T20:45:36.038 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:35 smithi018 ceph-mon[179647]: failed to encode map e530 with expected crc 2023-09-23T20:45:36.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:35 smithi149 ceph-mon[170394]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:45:36.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:35 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/856101437' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:45:36.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:35 smithi149 ceph-mon[170394]: osdmap e530: 8 total, 8 up, 8 in 2023-09-23T20:45:36.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:35 smithi149 ceph-mon[170394]: failed to encode map e530 with expected crc 2023-09-23T20:45:36.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:35 smithi149 ceph-mon[170394]: failed to encode map e530 with expected crc 2023-09-23T20:45:36.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:35 smithi149 ceph-mon[170394]: failed to encode map e530 with expected crc 2023-09-23T20:45:36.257 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:45:36.260 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:36.261 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:45:36.361 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:45:36.390 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:36.442 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2023-09-23T20:45:36.442 INFO:teuthology.orchestra.run.smithi018.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-09-23T20:45:36.443 INFO:teuthology.orchestra.run.smithi018.stdout:} 2023-09-23T20:45:36.534 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:45:36.537 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:36.537 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:45:36.638 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:45:36.670 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:36.708 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:45:36.711 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:36.711 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:45:36.812 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:45:36.841 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:37.029 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:45:37.032 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:37.033 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:45:37.134 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:45:37.166 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:37.204 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:45:37.206 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:36 smithi149 ceph-mon[170394]: failed to encode map e530 with expected crc 2023-09-23T20:45:37.206 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:36 smithi149 ceph-mon[170394]: pgmap v1331: 103 pgs: 103 active+clean; 4.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail 2023-09-23T20:45:37.207 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:37.207 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:45:37.308 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:45:37.317 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:45:37.342 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:36 smithi018 ceph-mon[184144]: failed to encode map e530 with expected crc 2023-09-23T20:45:37.342 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:36 smithi018 ceph-mon[184144]: pgmap v1331: 103 pgs: 103 active+clean; 4.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail 2023-09-23T20:45:37.342 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:36 smithi018 ceph-mon[179647]: failed to encode map e530 with expected crc 2023-09-23T20:45:37.342 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:36 smithi018 ceph-mon[179647]: pgmap v1331: 103 pgs: 103 active+clean; 4.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail 2023-09-23T20:45:37.346 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:37.356 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:45:37.385 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:45:37.413 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:45:37.441 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:45:37.452 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:45:37.468 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:45:37.471 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:45:37.472 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:37.472 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:45:37.491 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:45:37.511 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:45:37.512 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:45:37.512 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:45:37.512 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:45:37.513 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:45:37.513 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:45:37.513 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:45:37.513 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:45:37.513 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:45:37.573 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:45:37.608 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:37.647 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:45:37.651 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:37.652 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:45:37.753 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:45:37.783 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:38.038 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T20:45:38.084 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:45:38.088 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:38.088 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:45:38.189 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:45:38.218 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:38.252 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:45:38.255 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:38.256 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:45:38.327 INFO:teuthology.orchestra.run.smithi018.stderr:pool 'unique_pool_5' created 2023-09-23T20:45:38.356 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:45:38.389 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:38.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:38 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3917292069' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:45:38.517 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:38 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3917292069' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:45:38.518 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:38 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3917292069' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:45:38.910 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd pool application enable unique_pool_5 rados --yes-i-really-mean-it || true 2023-09-23T20:45:38.919 INFO:teuthology.orchestra.run.smithi018.stdout:ERROR: (22) Invalid argument 2023-09-23T20:45:38.919 INFO:teuthology.orchestra.run.smithi018.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:45:38.931 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:45:38.934 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:38.934 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:45:39.035 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:45:39.063 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:39.098 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:45:39.100 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:39.101 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:45:39.202 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:45:39.229 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:39.336 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:45:39.337 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:45:39.340 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:39.341 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:45:39.401 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T20:45:39.442 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:45:39.473 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:39.510 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:45:39.514 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:39.514 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:45:39.542 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:39 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3778151101' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-09-23T20:45:39.542 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:39 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-09-23T20:45:39.542 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:39 smithi018 ceph-mon[179647]: pgmap v1332: 103 pgs: 103 active+clean; 4.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail 2023-09-23T20:45:39.542 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:39 smithi018 ceph-mon[179647]: failed to encode map e531 with expected crc 2023-09-23T20:45:39.543 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:39 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2023-09-23T20:45:39.543 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:39 smithi018 ceph-mon[179647]: osdmap e531: 8 total, 8 up, 8 in 2023-09-23T20:45:39.543 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:39 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3778151101' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-09-23T20:45:39.543 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:39 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-09-23T20:45:39.544 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:39 smithi018 ceph-mon[184144]: pgmap v1332: 103 pgs: 103 active+clean; 4.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail 2023-09-23T20:45:39.544 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:39 smithi018 ceph-mon[184144]: failed to encode map e531 with expected crc 2023-09-23T20:45:39.544 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:39 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2023-09-23T20:45:39.544 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:39 smithi018 ceph-mon[184144]: osdmap e531: 8 total, 8 up, 8 in 2023-09-23T20:45:39.615 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:45:39.640 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:39 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3778151101' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-09-23T20:45:39.640 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:39 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-09-23T20:45:39.640 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:39 smithi149 ceph-mon[170394]: pgmap v1332: 103 pgs: 103 active+clean; 4.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail 2023-09-23T20:45:39.640 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:39 smithi149 ceph-mon[170394]: failed to encode map e531 with expected crc 2023-09-23T20:45:39.641 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:39 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2023-09-23T20:45:39.641 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:39 smithi149 ceph-mon[170394]: osdmap e531: 8 total, 8 up, 8 in 2023-09-23T20:45:39.642 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:39.677 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:45:39.679 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:39.680 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:45:39.781 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:45:39.807 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:39.840 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:45:39.842 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:39.843 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:45:39.944 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:45:39.971 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:40.003 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:45:40.005 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:40.006 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:45:40.087 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:45:40.107 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:45:40.137 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:40.170 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:45:40.318 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:45:40.322 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:40.323 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:45:40.380 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:45:40.424 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:45:40.450 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:40 smithi018 ceph-mon[184144]: failed to encode map e532 with expected crc 2023-09-23T20:45:40.451 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:40 smithi018 ceph-mon[184144]: osdmap e532: 8 total, 8 up, 8 in 2023-09-23T20:45:40.451 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:40 smithi018 ceph-mon[184144]: failed to encode map e532 with expected crc 2023-09-23T20:45:40.451 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:40 smithi018 ceph-mon[184144]: failed to encode map e532 with expected crc 2023-09-23T20:45:40.451 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:40 smithi018 ceph-mon[184144]: failed to encode map e532 with expected crc 2023-09-23T20:45:40.451 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:40 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:45:40.451 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:40 smithi018 ceph-mon[179647]: failed to encode map e532 with expected crc 2023-09-23T20:45:40.451 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:40 smithi018 ceph-mon[179647]: osdmap e532: 8 total, 8 up, 8 in 2023-09-23T20:45:40.451 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:40 smithi018 ceph-mon[179647]: failed to encode map e532 with expected crc 2023-09-23T20:45:40.451 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:40 smithi018 ceph-mon[179647]: failed to encode map e532 with expected crc 2023-09-23T20:45:40.452 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:40 smithi018 ceph-mon[179647]: failed to encode map e532 with expected crc 2023-09-23T20:45:40.452 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:40 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:45:40.452 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:40.487 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:45:40.489 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:40.490 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:45:40.590 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:45:40.615 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:40 smithi149 ceph-mon[170394]: failed to encode map e532 with expected crc 2023-09-23T20:45:40.615 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:40 smithi149 ceph-mon[170394]: osdmap e532: 8 total, 8 up, 8 in 2023-09-23T20:45:40.615 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:40 smithi149 ceph-mon[170394]: failed to encode map e532 with expected crc 2023-09-23T20:45:40.615 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:40 smithi149 ceph-mon[170394]: failed to encode map e532 with expected crc 2023-09-23T20:45:40.615 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:40 smithi149 ceph-mon[170394]: failed to encode map e532 with expected crc 2023-09-23T20:45:40.615 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:40 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:45:40.617 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:40.651 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:45:40.653 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:40.654 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:45:40.755 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:45:40.781 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:40.814 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:45:40.817 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:40.817 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:45:40.918 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:45:40.949 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:40.991 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:45:40.995 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:40.995 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:45:41.096 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:45:41.125 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:41.161 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:45:41.164 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:41.164 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:45:41.265 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:45:41.294 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:41.328 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:45:41.331 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:41.332 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:45:41.339 INFO:teuthology.orchestra.run.smithi018.stderr:nodeep-scrub is set 2023-09-23T20:45:41.355 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:45:41.433 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:45:41.465 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:41.510 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:41 smithi018 ceph-mon[184144]: pgmap v1335: 119 pgs: 1 creating+peering, 15 unknown, 103 active+clean; 4.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail 2023-09-23T20:45:41.510 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:41 smithi018 ceph-mon[184144]: failed to encode map e533 with expected crc 2023-09-23T20:45:41.510 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:41 smithi018 ceph-mon[184144]: osdmap e533: 8 total, 8 up, 8 in 2023-09-23T20:45:41.510 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:41 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1934041554' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:45:41.510 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:41 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:45:41.511 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:41 smithi018 ceph-mon[179647]: pgmap v1335: 119 pgs: 1 creating+peering, 15 unknown, 103 active+clean; 4.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail 2023-09-23T20:45:41.511 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:41 smithi018 ceph-mon[179647]: failed to encode map e533 with expected crc 2023-09-23T20:45:41.511 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:41 smithi018 ceph-mon[179647]: osdmap e533: 8 total, 8 up, 8 in 2023-09-23T20:45:41.511 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:41 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1934041554' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:45:41.511 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:41 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:45:41.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:41 smithi149 ceph-mon[170394]: pgmap v1335: 119 pgs: 1 creating+peering, 15 unknown, 103 active+clean; 4.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail 2023-09-23T20:45:41.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:41 smithi149 ceph-mon[170394]: failed to encode map e533 with expected crc 2023-09-23T20:45:41.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:41 smithi149 ceph-mon[170394]: osdmap e533: 8 total, 8 up, 8 in 2023-09-23T20:45:41.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:41 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1934041554' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:45:41.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:41 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:45:42.051 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 7, 3, 2, 4, 6, 5] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T20:45:42.052 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:45:42.052 INFO:tasks.thrashosds.thrasher:inject_pause on osd.6 2023-09-23T20:45:42.052 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T20:45:42.052 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T20:45:42.052 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 config set filestore_inject_stall 3 2023-09-23T20:45:42.081 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:45:42.085 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:42.085 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:45:42.186 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:45:42.212 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:42.248 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:45:42.251 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:42.252 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:45:42.352 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:45:42.394 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:42.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:42 smithi149 ceph-mon[170394]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:45:42.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:42 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:45:42.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:42 smithi149 ceph-mon[170394]: osdmap e534: 8 total, 8 up, 8 in 2023-09-23T20:45:42.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:42 smithi149 ceph-mon[170394]: failed to encode map e534 with expected crc 2023-09-23T20:45:42.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:42 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1583143745' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T20:45:42.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:42 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T20:45:42.480 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:45:42.483 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:42.483 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:45:42.566 INFO:teuthology.orchestra.run.smithi018.stderr:enabled application 'rados' on pool 'unique_pool_5' 2023-09-23T20:45:42.584 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:45:42.596 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:42 smithi018 ceph-mon[179647]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:45:42.596 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:42 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:45:42.596 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:42 smithi018 ceph-mon[179647]: osdmap e534: 8 total, 8 up, 8 in 2023-09-23T20:45:42.596 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:42 smithi018 ceph-mon[179647]: failed to encode map e534 with expected crc 2023-09-23T20:45:42.596 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:42 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1583143745' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T20:45:42.597 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:42 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T20:45:42.597 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:42 smithi018 ceph-mon[184144]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:45:42.597 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:42 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:45:42.597 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:42 smithi018 ceph-mon[184144]: osdmap e534: 8 total, 8 up, 8 in 2023-09-23T20:45:42.597 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:42 smithi018 ceph-mon[184144]: failed to encode map e534 with expected crc 2023-09-23T20:45:42.597 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:42 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1583143745' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T20:45:42.597 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:42 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T20:45:42.612 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:42.680 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:45:42.791 INFO:teuthology.orchestra.run.smithi018.stdout:ERROR: (22) Invalid argument 2023-09-23T20:45:42.791 INFO:teuthology.orchestra.run.smithi018.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:45:42.898 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:45:42.900 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:42.901 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:45:43.002 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:45:43.031 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:43.067 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:45:43.069 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:43.070 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:45:43.170 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:45:43.200 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:43.337 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:45:43.340 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:43.341 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:45:43.385 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:45:43.405 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:45:43.427 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:45:43.441 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:45:43.446 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:45:43.469 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:45:43.472 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:43.477 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T20:45:43.496 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:45:43.521 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:45:43.540 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:45:43.541 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:45:43.541 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:45:43.541 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:45:43.541 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:45:43.541 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:45:43.542 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:45:43.542 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:45:43.542 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:45:43.678 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:43 smithi018 ceph-mon[184144]: failed to encode map e534 with expected crc 2023-09-23T20:45:43.678 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:43 smithi018 ceph-mon[184144]: failed to encode map e534 with expected crc 2023-09-23T20:45:43.678 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:43 smithi018 ceph-mon[184144]: failed to encode map e534 with expected crc 2023-09-23T20:45:43.678 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:43 smithi018 ceph-mon[184144]: pgmap v1338: 119 pgs: 1 creating+peering, 10 unknown, 108 active+clean; 4.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail 2023-09-23T20:45:43.678 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:43 smithi018 ceph-mon[184144]: failed to encode map e535 with expected crc 2023-09-23T20:45:43.678 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:43 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T20:45:43.679 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:43 smithi018 ceph-mon[184144]: osdmap e535: 8 total, 8 up, 8 in 2023-09-23T20:45:43.679 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:43 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2027093496' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:45:43.679 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:43 smithi018 ceph-mon[179647]: failed to encode map e534 with expected crc 2023-09-23T20:45:43.679 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:43 smithi018 ceph-mon[179647]: failed to encode map e534 with expected crc 2023-09-23T20:45:43.679 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:43 smithi018 ceph-mon[179647]: failed to encode map e534 with expected crc 2023-09-23T20:45:43.679 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:43 smithi018 ceph-mon[179647]: pgmap v1338: 119 pgs: 1 creating+peering, 10 unknown, 108 active+clean; 4.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail 2023-09-23T20:45:43.679 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:43 smithi018 ceph-mon[179647]: failed to encode map e535 with expected crc 2023-09-23T20:45:43.679 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:43 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T20:45:43.679 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:43 smithi018 ceph-mon[179647]: osdmap e535: 8 total, 8 up, 8 in 2023-09-23T20:45:43.679 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:43 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2027093496' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:45:43.785 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:45:43.788 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:43.789 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:45:43.812 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:43 smithi149 ceph-mon[170394]: failed to encode map e534 with expected crc 2023-09-23T20:45:43.813 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:43 smithi149 ceph-mon[170394]: failed to encode map e534 with expected crc 2023-09-23T20:45:43.813 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:43 smithi149 ceph-mon[170394]: failed to encode map e534 with expected crc 2023-09-23T20:45:43.813 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:43 smithi149 ceph-mon[170394]: pgmap v1338: 119 pgs: 1 creating+peering, 10 unknown, 108 active+clean; 4.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail 2023-09-23T20:45:43.813 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:43 smithi149 ceph-mon[170394]: failed to encode map e535 with expected crc 2023-09-23T20:45:43.813 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:43 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T20:45:43.814 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:43 smithi149 ceph-mon[170394]: osdmap e535: 8 total, 8 up, 8 in 2023-09-23T20:45:43.814 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:43 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2027093496' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:45:43.890 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:45:43.921 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:43.957 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:45:43.960 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:43.960 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:45:44.061 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:45:44.066 INFO:teuthology.orchestra.run.smithi149.stdout:{ 2023-09-23T20:45:44.066 INFO:teuthology.orchestra.run.smithi149.stdout: "success": "" 2023-09-23T20:45:44.066 INFO:teuthology.orchestra.run.smithi149.stdout:} 2023-09-23T20:45:44.092 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:44.130 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:45:44.133 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:44.133 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:45:44.234 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:45:44.261 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:44.268 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T20:45:44.347 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:45:44.452 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:45:44.455 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:44.455 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:45:44.556 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:45:44.584 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:44.618 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:45:44.621 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:44.621 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:45:44.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:44 smithi149 ceph-mon[170394]: failed to encode map e535 with expected crc 2023-09-23T20:45:44.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:44 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:45:44.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:44 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:45:44.722 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:45:44.748 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:44.781 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:45:44.783 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:44.784 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:45:44.867 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:44 smithi018 ceph-mon[179647]: failed to encode map e535 with expected crc 2023-09-23T20:45:44.867 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:44 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:45:44.867 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:44 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:45:44.867 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:44 smithi018 ceph-mon[184144]: failed to encode map e535 with expected crc 2023-09-23T20:45:44.868 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:44 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:45:44.868 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:44 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:45:44.884 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:45:44.889 DEBUG:teuthology.orchestra.run.smithi018:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_5 bench 90 write' 2023-09-23T20:45:44.912 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:44.914 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-09-23T20:45:44.914 INFO:teuthology.orchestra.run:waiting for 3000 2023-09-23T20:45:44.977 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:hints = 1 2023-09-23T20:45:44.977 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-09-23T20:45:44.978 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Object prefix: benchmark_data_smithi018_422431 2023-09-23T20:45:44.978 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T20:45:44.978 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 0 0 0 0 0 0 - 0 2023-09-23T20:45:45.141 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:45:44 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:45:44] "GET /metrics HTTP/1.1" 200 36502 "" "Prometheus/2.43.0" 2023-09-23T20:45:45.159 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:45:45.166 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:45.167 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:45:45.219 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:45:45.268 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:45:45.326 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:45.375 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:45:45.380 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:45.381 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:45:45.481 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:45:45.546 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:45.621 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:45:45.626 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:45.626 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:45:45.706 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:45 smithi018 ceph-mon[179647]: pgmap v1340: 119 pgs: 119 active+clean; 4.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail 2023-09-23T20:45:45.706 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:45 smithi018 ceph-mon[184144]: pgmap v1340: 119 pgs: 119 active+clean; 4.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail 2023-09-23T20:45:45.727 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:45:45.769 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:45.778 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2023-09-23T20:45:45.778 INFO:teuthology.orchestra.run.smithi018.stdout: "ops": [], 2023-09-23T20:45:45.779 INFO:teuthology.orchestra.run.smithi018.stdout: "complaint_time": 30, 2023-09-23T20:45:45.779 INFO:teuthology.orchestra.run.smithi018.stdout: "num_blocked_ops": 0 2023-09-23T20:45:45.779 INFO:teuthology.orchestra.run.smithi018.stdout:} 2023-09-23T20:45:45.826 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:45:45.830 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:45 smithi149 ceph-mon[170394]: pgmap v1340: 119 pgs: 119 active+clean; 4.6 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail 2023-09-23T20:45:45.831 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:45.832 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:45:45.932 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:45:45.977 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:45.978 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 1 15 3383 3368 210.397 210.5 0.00399346 0.00473948 2023-09-23T20:45:46.037 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:45:46.042 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:46.043 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:45:46.087 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:45:46.143 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:45:46.193 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:46.252 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:45:46.257 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:46.257 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:45:46.358 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:45:46.401 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:46.412 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T20:45:46.457 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:45:46.461 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:46.462 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:45:46.563 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:45:46.711 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:46.779 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:45:46.783 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:46.784 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:45:46.886 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:45:46.928 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:46.979 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 2 16 6279 6263 195.637 180.938 0.00310457 0.00510187 2023-09-23T20:45:46.983 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:45:46.987 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:46.988 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:45:47.059 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:45:47.089 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:45:47.166 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:47.238 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:45:47.247 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:47.248 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:45:47.351 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:45:47.393 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:47.448 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:45:47.452 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:47.452 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:45:47.549 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:45:47.553 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:45:47.592 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:47.643 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:45:47.647 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:47 smithi149 ceph-mon[170394]: pgmap v1341: 119 pgs: 119 active+clean; 25 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 3.3 MiB/s wr, 53 op/s 2023-09-23T20:45:47.648 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:47.649 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:45:47.749 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:45:47.800 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:47.853 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:47 smithi018 ceph-mon[179647]: pgmap v1341: 119 pgs: 119 active+clean; 25 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 3.3 MiB/s wr, 53 op/s 2023-09-23T20:45:47.854 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:47 smithi018 ceph-mon[184144]: pgmap v1341: 119 pgs: 119 active+clean; 25 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 3.3 MiB/s wr, 53 op/s 2023-09-23T20:45:47.863 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:45:47.868 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:47.868 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:45:47.969 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:45:47.979 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 3 16 9022 9006 187.564 171.438 0.00330668 0.00532265 2023-09-23T20:45:48.018 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:48.073 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:45:48.079 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:48.079 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:45:48.180 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:45:48.243 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:48.359 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:45:48.364 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:48.364 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:45:48.465 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:45:48.520 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:48.609 INFO:teuthology.orchestra.run.smithi018.stderr:noscrub is unset 2023-09-23T20:45:48.665 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:45:48.672 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:48.672 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:45:48.726 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:48 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1236982355' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:45:48.727 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:48 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1236982355' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:45:48.773 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:45:48.853 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:48.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:48 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1236982355' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:45:48.979 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 4 16 11642 11626 181.607 163.75 0.00435575 0.00549993 2023-09-23T20:45:49.096 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:45:49.134 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:45:49.141 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:45:49.152 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:49.152 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:45:49.175 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:45:49.217 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:45:49.253 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:45:49.260 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:45:49.296 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:45:49.298 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:49.335 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:45:49.352 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:45:49.356 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:49.357 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:45:49.381 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:45:49.382 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:45:49.382 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:45:49.382 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:45:49.382 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:45:49.382 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:45:49.383 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:45:49.383 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:45:49.383 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:45:49.429 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 7, 3, 2, 4, 6, 5] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T20:45:49.429 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:45:49.429 INFO:tasks.thrashosds.thrasher:Growing pool .mgr 2023-09-23T20:45:49.430 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:45:49.458 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:45:49.594 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:49.676 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:45:49.680 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:49 smithi018 ceph-mon[179647]: pgmap v1342: 119 pgs: 119 active+clean; 221 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 27 MiB/s wr, 438 op/s 2023-09-23T20:45:49.680 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:49 smithi018 ceph-mon[179647]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:45:49.680 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:49 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1236982355' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:45:49.680 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:49 smithi018 ceph-mon[179647]: osdmap e536: 8 total, 8 up, 8 in 2023-09-23T20:45:49.681 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:49 smithi018 ceph-mon[179647]: failed to encode map e536 with expected crc 2023-09-23T20:45:49.681 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:49 smithi018 ceph-mon[179647]: failed to encode map e536 with expected crc 2023-09-23T20:45:49.681 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:49 smithi018 ceph-mon[179647]: failed to encode map e536 with expected crc 2023-09-23T20:45:49.681 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:49 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/475951424' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:45:49.681 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:49 smithi018 ceph-mon[184144]: pgmap v1342: 119 pgs: 119 active+clean; 221 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 27 MiB/s wr, 438 op/s 2023-09-23T20:45:49.681 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:49 smithi018 ceph-mon[184144]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:45:49.681 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:49 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1236982355' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:45:49.681 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:49 smithi018 ceph-mon[184144]: osdmap e536: 8 total, 8 up, 8 in 2023-09-23T20:45:49.681 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:49 smithi018 ceph-mon[184144]: failed to encode map e536 with expected crc 2023-09-23T20:45:49.681 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:49 smithi018 ceph-mon[184144]: failed to encode map e536 with expected crc 2023-09-23T20:45:49.682 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:49 smithi018 ceph-mon[184144]: failed to encode map e536 with expected crc 2023-09-23T20:45:49.682 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:49 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/475951424' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:45:49.682 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:49.682 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:45:49.699 INFO:teuthology.orchestra.run.smithi018.stdout:ERROR: (22) Invalid argument 2023-09-23T20:45:49.699 INFO:teuthology.orchestra.run.smithi018.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:45:49.784 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:45:49.853 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:49.969 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:49 smithi149 ceph-mon[170394]: pgmap v1342: 119 pgs: 119 active+clean; 221 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 27 MiB/s wr, 438 op/s 2023-09-23T20:45:49.969 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:49 smithi149 ceph-mon[170394]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:45:49.969 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:49 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1236982355' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:45:49.969 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:49 smithi149 ceph-mon[170394]: osdmap e536: 8 total, 8 up, 8 in 2023-09-23T20:45:49.969 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:49 smithi149 ceph-mon[170394]: failed to encode map e536 with expected crc 2023-09-23T20:45:49.969 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:49 smithi149 ceph-mon[170394]: failed to encode map e536 with expected crc 2023-09-23T20:45:49.969 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:49 smithi149 ceph-mon[170394]: failed to encode map e536 with expected crc 2023-09-23T20:45:49.969 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:49 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/475951424' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:45:49.979 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 5 16 14040 14024 175.259 149.875 0.00330878 0.00569893 2023-09-23T20:45:50.522 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:45:50.526 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:50.527 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:45:50.552 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T20:45:50.628 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:45:50.738 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:50.803 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:45:50.812 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:50.812 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:45:50.836 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:45:50.913 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:45:50.979 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 6 15 16605 16590 172.776 160.375 0.00565728 0.00577955 2023-09-23T20:45:51.000 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:51.118 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:45:51.123 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:51.124 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:45:51.224 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:45:51.271 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:51.285 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T20:45:51.326 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:45:51.332 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:51.333 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:45:51.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:51 smithi018 ceph-mon[184144]: pgmap v1344: 119 pgs: 119 active+clean; 489 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 60 MiB/s wr, 967 op/s 2023-09-23T20:45:51.394 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:51 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:45:51.394 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:51 smithi018 ceph-mon[179647]: pgmap v1344: 119 pgs: 119 active+clean; 489 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 60 MiB/s wr, 967 op/s 2023-09-23T20:45:51.394 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:51 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:45:51.433 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:45:51.509 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:51.685 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:45:51.690 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:51.691 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:45:51.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:51 smithi149 ceph-mon[170394]: pgmap v1344: 119 pgs: 119 active+clean; 489 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 60 MiB/s wr, 967 op/s 2023-09-23T20:45:51.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:51 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:45:51.745 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:45:51.792 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:45:51.868 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:51.981 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 7 16 18778 18762 167.439 135.75 0.00705705 0.00596752 2023-09-23T20:45:51.986 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:45:51.994 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:51.994 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:45:52.095 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:45:52.143 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:52.204 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:45:52.208 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:52.208 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:45:52.271 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:45:52.308 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:45:52.342 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:52.400 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:45:52.406 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:52.407 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:45:52.481 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:45:52.507 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:45:52.662 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:52.981 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 8 16 20806 20790 162.353 126.75 0.00228111 0.00615261 2023-09-23T20:45:52.998 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:45:53.000 INFO:teuthology.orchestra.run.smithi018.stderr:dumped all 2023-09-23T20:45:53.579 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:53 smithi018 ceph-mon[179647]: pgmap v1345: 119 pgs: 119 active+clean; 745 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 77 MiB/s wr, 1.22k op/s 2023-09-23T20:45:53.885 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:53 smithi018 ceph-mon[184144]: pgmap v1345: 119 pgs: 119 active+clean; 745 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 77 MiB/s wr, 1.22k op/s 2023-09-23T20:45:53.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:53 smithi149 ceph-mon[170394]: pgmap v1345: 119 pgs: 119 active+clean; 745 MiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 77 MiB/s wr, 1.22k op/s 2023-09-23T20:45:53.981 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 9 16 23074 23058 160.063 141.75 0.00587779 0.00624238 2023-09-23T20:45:54.231 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:45:54.236 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:54.237 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:45:54.338 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:45:54.396 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:54.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:54 smithi018 ceph-mon[179647]: from='client.46637 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:45:54.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:54 smithi018 ceph-mon[184144]: from='client.46637 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:45:54.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:54 smithi149 ceph-mon[170394]: from='client.46637 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:45:54.981 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 10 16 25455 25439 158.937 148.812 0.00692618 0.00628632 2023-09-23T20:45:55.391 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:45:54 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:45:54] "GET /metrics HTTP/1.1" 200 36502 "" "Prometheus/2.43.0" 2023-09-23T20:45:55.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:55 smithi018 ceph-mon[184144]: pgmap v1346: 119 pgs: 119 active+clean; 1.2 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 124 MiB/s wr, 1.99k op/s 2023-09-23T20:45:55.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:55 smithi018 ceph-mon[179647]: pgmap v1346: 119 pgs: 119 active+clean; 1.2 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 124 MiB/s wr, 1.99k op/s 2023-09-23T20:45:55.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:55 smithi149 ceph-mon[170394]: pgmap v1346: 119 pgs: 119 active+clean; 1.2 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 124 MiB/s wr, 1.99k op/s 2023-09-23T20:45:55.981 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 11 16 27568 27552 156.493 132.062 0.00379442 0.00638492 2023-09-23T20:45:56.341 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:45:56.347 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:56.347 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:45:56.448 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:45:56.503 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:56.568 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:45:56.571 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:56.572 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:45:56.673 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:45:56.735 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:56.804 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:45:56.810 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:56.810 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:45:56.840 INFO:teuthology.orchestra.run.smithi018.stdout:{"pg_ready":true,"pg_map":{"version":1345,"stamp":"2023-09-23T20:45:52.234952+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":781260723,"num_objects":11876,"num_object_clones":0,"num_object_copies":35628,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":11876,"num_whiteouts":0,"num_read":5983,"num_read_kb":13200,"num_write":12755,"num_write_kb":770997,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":174,"num_bytes_recovered":16795702,"num_keys_recovered":40,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":17870,"ondisk_log_size":17870,"up":357,"acting":357,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":359,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":7480020,"kb_used_data":2334804,"kb_used_omap":32,"kb_used_meta":5145119,"kb_avail":742481196,"statfs":{"total":767960285184,"available":760300744704,"internally_reserved":0,"allocated":2390839296,"data_stored":2374374279,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32773,"internal_metadata":5268602875},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[5,2,4,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":776470528,"num_objects":11848,"num_object_clones":0,"num_object_copies":35544,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":11848,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11848,"num_write_kb":758272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.673989"},"pg_stats":[{"pgid":"1.18","version":"90'161","reported_seq":581,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:45:43.105873+0000","last_change":"2023-09-23T20:44:19.677357+0000","last_active":"2023-09-23T20:45:43.105873+0000","last_peered":"2023-09-23T20:45:43.105873+0000","last_clean":"2023-09-23T20:45:43.105873+0000","last_became_active":"2023-09-23T20:43:16.766305+0000","last_became_peered":"2023-09-23T20:43:16.766305+0000","last_unstale":"2023-09-23T20:45:43.105873+0000","last_undegraded":"2023-09-23T20:45:43.105873+0000","last_fullsized":"2023-09-23T20:45:43.105873+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:19.677209+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:19.677209+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:14:06.510115+0000","scrub_duration":0.0023292260000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"69'1","reported_seq":233,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-09-23T20:45:42.564945+0000","last_change":"2023-09-23T20:32:33.233853+0000","last_active":"2023-09-23T20:45:42.564945+0000","last_peered":"2023-09-23T20:45:42.564945+0000","last_clean":"2023-09-23T20:45:42.564945+0000","last_became_active":"2023-09-23T20:32:33.233546+0000","last_became_peered":"2023-09-23T20:32:33.233546+0000","last_unstale":"2023-09-23T20:45:42.564945+0000","last_undegraded":"2023-09-23T20:45:42.564945+0000","last_fullsized":"2023-09-23T20:45:42.564945+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:50:27.130808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":351,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-09-23T20:45:43.508016+0000","last_change":"2023-09-23T20:37:43.579611+0000","last_active":"2023-09-23T20:45:43.508016+0000","last_peered":"2023-09-23T20:45:43.508016+0000","last_clean":"2023-09-23T20:45:43.508016+0000","last_became_active":"2023-09-23T20:37:43.579323+0000","last_became_peered":"2023-09-23T20:37:43.579323+0000","last_unstale":"2023-09-23T20:45:43.508016+0000","last_undegraded":"2023-09-23T20:45:43.508016+0000","last_fullsized":"2023-09-23T20:45:43.508016+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:17:32.311427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"90'161","reported_seq":657,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-09-23T20:45:42.564863+0000","last_change":"2023-09-23T20:44:14.647447+0000","last_active":"2023-09-23T20:45:42.564863+0000","last_peered":"2023-09-23T20:45:42.564863+0000","last_clean":"2023-09-23T20:45:42.564863+0000","last_became_active":"2023-09-23T20:43:16.756116+0000","last_became_peered":"2023-09-23T20:43:16.756116+0000","last_unstale":"2023-09-23T20:45:42.564863+0000","last_undegraded":"2023-09-23T20:45:42.564863+0000","last_fullsized":"2023-09-23T20:45:42.564863+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:14.647386+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:36:05.442288+0000","last_clean_scrub_stamp":"2023-09-23T20:44:14.647386+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:25:25.151100+0000","scrub_duration":0.0097381210000000006,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":505,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:45:41.333687+0000","last_change":"2023-09-23T20:34:59.983035+0000","last_active":"2023-09-23T20:45:41.333687+0000","last_peered":"2023-09-23T20:45:41.333687+0000","last_clean":"2023-09-23T20:45:41.333687+0000","last_became_active":"2023-09-23T20:34:59.982459+0000","last_became_peered":"2023-09-23T20:34:59.982459+0000","last_unstale":"2023-09-23T20:45:41.333687+0000","last_undegraded":"2023-09-23T20:45:41.333687+0000","last_fullsized":"2023-09-23T20:45:41.333687+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:16:48.510783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":494,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-09-23T20:45:46.423460+0000","last_change":"2023-09-23T20:43:46.521939+0000","last_active":"2023-09-23T20:45:46.423460+0000","last_peered":"2023-09-23T20:45:46.423460+0000","last_clean":"2023-09-23T20:45:46.423460+0000","last_became_active":"2023-09-23T20:43:46.521185+0000","last_became_peered":"2023-09-23T20:43:46.521185+0000","last_unstale":"2023-09-23T20:45:46.423460+0000","last_undegraded":"2023-09-23T20:45:46.423460+0000","last_fullsized":"2023-09-23T20:45:46.423460+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:59:25.045627+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"90'161","reported_seq":615,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-09-23T20:45:42.564704+0000","last_change":"2023-09-23T20:36:01.292103+0000","last_active":"2023-09-23T20:45:42.564704+0000","last_peered":"2023-09-23T20:45:42.564704+0000","last_clean":"2023-09-23T20:45:42.564704+0000","last_became_active":"2023-09-23T20:35:26.639912+0000","last_became_peered":"2023-09-23T20:35:26.639912+0000","last_unstale":"2023-09-23T20:45:42.564704+0000","last_undegraded":"2023-09-23T20:45:42.564704+0000","last_fullsized":"2023-09-23T20:45:42.564704+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:01.292032+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:01.292032+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:04:53.926640+0000","scrub_duration":0.0032983159999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":331,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-09-23T20:45:43.507224+0000","last_change":"2023-09-23T20:37:43.584414+0000","last_active":"2023-09-23T20:45:43.507224+0000","last_peered":"2023-09-23T20:45:43.507224+0000","last_clean":"2023-09-23T20:45:43.507224+0000","last_became_active":"2023-09-23T20:37:43.584116+0000","last_became_peered":"2023-09-23T20:37:43.584116+0000","last_unstale":"2023-09-23T20:45:43.507224+0000","last_undegraded":"2023-09-23T20:45:43.507224+0000","last_fullsized":"2023-09-23T20:45:43.507224+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:17:54.258172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":150,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:45:42.051307+0000","last_change":"2023-09-23T20:38:37.457427+0000","last_active":"2023-09-23T20:45:42.051307+0000","last_peered":"2023-09-23T20:45:42.051307+0000","last_clean":"2023-09-23T20:45:42.051307+0000","last_became_active":"2023-09-23T20:38:37.457099+0000","last_became_peered":"2023-09-23T20:38:37.457099+0000","last_unstale":"2023-09-23T20:45:42.051307+0000","last_undegraded":"2023-09-23T20:45:42.051307+0000","last_fullsized":"2023-09-23T20:45:42.051307+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:31:27.084994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"90'161","reported_seq":747,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:45:41.914659+0000","last_change":"2023-09-23T20:40:22.449754+0000","last_active":"2023-09-23T20:45:41.914659+0000","last_peered":"2023-09-23T20:45:41.914659+0000","last_clean":"2023-09-23T20:45:41.914659+0000","last_became_active":"2023-09-23T20:40:22.448838+0000","last_became_peered":"2023-09-23T20:40:22.448838+0000","last_unstale":"2023-09-23T20:45:41.914659+0000","last_undegraded":"2023-09-23T20:45:41.914659+0000","last_fullsized":"2023-09-23T20:45:41.914659+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:03.411025+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:03.411025+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:22:07.948856+0000","scrub_duration":0.001802922,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":526,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-09-23T20:45:43.505198+0000","last_change":"2023-09-23T20:43:46.522325+0000","last_active":"2023-09-23T20:45:43.505198+0000","last_peered":"2023-09-23T20:45:43.505198+0000","last_clean":"2023-09-23T20:45:43.505198+0000","last_became_active":"2023-09-23T20:43:46.521996+0000","last_became_peered":"2023-09-23T20:43:46.521996+0000","last_unstale":"2023-09-23T20:45:43.505198+0000","last_undegraded":"2023-09-23T20:45:43.505198+0000","last_fullsized":"2023-09-23T20:45:43.505198+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:51:17.629956+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":260,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:45:42.356361+0000","last_change":"2023-09-23T20:38:02.579343+0000","last_active":"2023-09-23T20:45:42.356361+0000","last_peered":"2023-09-23T20:45:42.356361+0000","last_clean":"2023-09-23T20:45:42.356361+0000","last_became_active":"2023-09-23T20:38:02.579047+0000","last_became_peered":"2023-09-23T20:38:02.579047+0000","last_unstale":"2023-09-23T20:45:42.356361+0000","last_undegraded":"2023-09-23T20:45:42.356361+0000","last_fullsized":"2023-09-23T20:45:42.356361+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:44:03.166556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.14","version":"90'161","reported_seq":866,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:45:42.355515+0000","last_change":"2023-09-23T20:35:02.236398+0000","last_active":"2023-09-23T20:45:42.355515+0000","last_peered":"2023-09-23T20:45:42.355515+0000","last_clean":"2023-09-23T20:45:42.355515+0000","last_became_active":"2023-09-23T20:35:02.235988+0000","last_became_peered":"2023-09-23T20:35:02.235988+0000","last_unstale":"2023-09-23T20:45:42.355515+0000","last_undegraded":"2023-09-23T20:45:42.355515+0000","last_fullsized":"2023-09-23T20:45:42.355515+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:47.479012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:47.479012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:13.519648+0000","scrub_duration":0.0020136590000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":371,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:45:41.914974+0000","last_change":"2023-09-23T20:43:16.780478+0000","last_active":"2023-09-23T20:45:41.914974+0000","last_peered":"2023-09-23T20:45:41.914974+0000","last_clean":"2023-09-23T20:45:41.914974+0000","last_became_active":"2023-09-23T20:43:16.780183+0000","last_became_peered":"2023-09-23T20:43:16.780183+0000","last_unstale":"2023-09-23T20:45:41.914974+0000","last_undegraded":"2023-09-23T20:45:41.914974+0000","last_fullsized":"2023-09-23T20:45:41.914974+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:38:30.102066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":362,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-09-23T20:45:43.507879+0000","last_change":"2023-09-23T20:38:02.542034+0000","last_active":"2023-09-23T20:45:43.507879+0000","last_peered":"2023-09-23T20:45:43.507879+0000","last_clean":"2023-09-23T20:45:43.507879+0000","last_became_active":"2023-09-23T20:38:02.541575+0000","last_became_peered":"2023-09-23T20:38:02.541575+0000","last_unstale":"2023-09-23T20:45:43.507879+0000","last_undegraded":"2023-09-23T20:45:43.507879+0000","last_fullsized":"2023-09-23T20:45:43.507879+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:50:54.591529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"90'161","reported_seq":661,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-09-23T20:45:43.504982+0000","last_change":"2023-09-23T20:44:14.669973+0000","last_active":"2023-09-23T20:45:43.504982+0000","last_peered":"2023-09-23T20:45:43.504982+0000","last_clean":"2023-09-23T20:45:43.504982+0000","last_became_active":"2023-09-23T20:43:46.519592+0000","last_became_peered":"2023-09-23T20:43:46.519592+0000","last_unstale":"2023-09-23T20:45:43.504982+0000","last_undegraded":"2023-09-23T20:45:43.504982+0000","last_fullsized":"2023-09-23T20:45:43.504982+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:14.669831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:14.669831+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:56:28.076617+0000","scrub_duration":0.0083441460000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":562,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:45:41.914962+0000","last_change":"2023-09-23T20:32:33.270617+0000","last_active":"2023-09-23T20:45:41.914962+0000","last_peered":"2023-09-23T20:45:41.914962+0000","last_clean":"2023-09-23T20:45:41.914962+0000","last_became_active":"2023-09-23T20:32:33.227902+0000","last_became_peered":"2023-09-23T20:32:33.227902+0000","last_unstale":"2023-09-23T20:45:41.914962+0000","last_undegraded":"2023-09-23T20:45:41.914962+0000","last_fullsized":"2023-09-23T20:45:41.914962+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:30:26.847831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":360,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":49,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:45:41.333728+0000","last_change":"2023-09-23T20:44:10.611381+0000","last_active":"2023-09-23T20:45:41.333728+0000","last_peered":"2023-09-23T20:45:41.333728+0000","last_clean":"2023-09-23T20:45:41.333728+0000","last_became_active":"2023-09-23T20:44:10.611091+0000","last_became_peered":"2023-09-23T20:44:10.611091+0000","last_unstale":"2023-09-23T20:45:41.333728+0000","last_undegraded":"2023-09-23T20:45:41.333728+0000","last_fullsized":"2023-09-23T20:45:41.333728+0000","mapping_epoch":496,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":497,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:00:30.162478+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"90'161","reported_seq":661,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:45:41.334567+0000","last_change":"2023-09-23T20:36:02.548387+0000","last_active":"2023-09-23T20:45:41.334567+0000","last_peered":"2023-09-23T20:45:41.334567+0000","last_clean":"2023-09-23T20:45:41.334567+0000","last_became_active":"2023-09-23T20:35:26.644787+0000","last_became_peered":"2023-09-23T20:35:26.644787+0000","last_unstale":"2023-09-23T20:45:41.334567+0000","last_undegraded":"2023-09-23T20:45:41.334567+0000","last_fullsized":"2023-09-23T20:45:41.334567+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:02.548285+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:02.548285+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:37:15.216553+0000","scrub_duration":0.0015514459999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":283,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-09-23T20:45:42.565220+0000","last_change":"2023-09-23T20:38:02.560646+0000","last_active":"2023-09-23T20:45:42.565220+0000","last_peered":"2023-09-23T20:45:42.565220+0000","last_clean":"2023-09-23T20:45:42.565220+0000","last_became_active":"2023-09-23T20:38:02.539814+0000","last_became_peered":"2023-09-23T20:38:02.539814+0000","last_unstale":"2023-09-23T20:45:42.565220+0000","last_undegraded":"2023-09-23T20:45:42.565220+0000","last_fullsized":"2023-09-23T20:45:42.565220+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:00:09.435023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":263,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-09-23T20:45:42.565117+0000","last_change":"2023-09-23T20:38:02.579222+0000","last_active":"2023-09-23T20:45:42.565117+0000","last_peered":"2023-09-23T20:45:42.565117+0000","last_clean":"2023-09-23T20:45:42.565117+0000","last_became_active":"2023-09-23T20:38:02.578813+0000","last_became_peered":"2023-09-23T20:38:02.578813+0000","last_unstale":"2023-09-23T20:45:42.565117+0000","last_undegraded":"2023-09-23T20:45:42.565117+0000","last_fullsized":"2023-09-23T20:45:42.565117+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:57:31.080027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"90'161","reported_seq":789,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-09-23T20:45:43.506708+0000","last_change":"2023-09-23T20:43:46.544727+0000","last_active":"2023-09-23T20:45:43.506708+0000","last_peered":"2023-09-23T20:45:43.506708+0000","last_clean":"2023-09-23T20:45:43.506708+0000","last_became_active":"2023-09-23T20:43:46.543952+0000","last_became_peered":"2023-09-23T20:43:46.543952+0000","last_unstale":"2023-09-23T20:45:43.506708+0000","last_undegraded":"2023-09-23T20:45:43.506708+0000","last_fullsized":"2023-09-23T20:45:43.506708+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:43.578666+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:43.578666+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:11:16.611920+0000","scrub_duration":0.0021890379999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":561,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:45:41.914510+0000","last_change":"2023-09-23T20:32:33.242943+0000","last_active":"2023-09-23T20:45:41.914510+0000","last_peered":"2023-09-23T20:45:41.914510+0000","last_clean":"2023-09-23T20:45:41.914510+0000","last_became_active":"2023-09-23T20:32:33.225468+0000","last_became_peered":"2023-09-23T20:32:33.225468+0000","last_unstale":"2023-09-23T20:45:41.914510+0000","last_undegraded":"2023-09-23T20:45:41.914510+0000","last_fullsized":"2023-09-23T20:45:41.914510+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:40:47.867573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":282,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-09-23T20:45:43.508964+0000","last_change":"2023-09-23T20:45:01.875931+0000","last_active":"2023-09-23T20:45:43.508964+0000","last_peered":"2023-09-23T20:45:43.508964+0000","last_clean":"2023-09-23T20:45:43.508964+0000","last_became_active":"2023-09-23T20:45:01.872839+0000","last_became_peered":"2023-09-23T20:45:01.872839+0000","last_unstale":"2023-09-23T20:45:43.508964+0000","last_undegraded":"2023-09-23T20:45:43.508964+0000","last_fullsized":"2023-09-23T20:45:43.508964+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:15:09.584507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"90'161","reported_seq":815,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:45:41.914150+0000","last_change":"2023-09-23T20:44:20.041766+0000","last_active":"2023-09-23T20:45:41.914150+0000","last_peered":"2023-09-23T20:45:41.914150+0000","last_clean":"2023-09-23T20:45:41.914150+0000","last_became_active":"2023-09-23T20:43:16.765015+0000","last_became_peered":"2023-09-23T20:43:16.765015+0000","last_unstale":"2023-09-23T20:45:41.914150+0000","last_undegraded":"2023-09-23T20:45:41.914150+0000","last_fullsized":"2023-09-23T20:45:41.914150+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:20.041690+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:20.041690+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:33:30.150109+0000","scrub_duration":0.0026180330000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":491,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-09-23T20:45:43.506465+0000","last_change":"2023-09-23T20:43:46.521294+0000","last_active":"2023-09-23T20:45:43.506465+0000","last_peered":"2023-09-23T20:45:43.506465+0000","last_clean":"2023-09-23T20:45:43.506465+0000","last_became_active":"2023-09-23T20:43:46.521001+0000","last_became_peered":"2023-09-23T20:43:46.521001+0000","last_unstale":"2023-09-23T20:45:43.506465+0000","last_undegraded":"2023-09-23T20:45:43.506465+0000","last_fullsized":"2023-09-23T20:45:43.506465+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:01:35.126716+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":302,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:45:41.915056+0000","last_change":"2023-09-23T20:44:01.841565+0000","last_active":"2023-09-23T20:45:41.915056+0000","last_peered":"2023-09-23T20:45:41.915056+0000","last_clean":"2023-09-23T20:45:41.915056+0000","last_became_active":"2023-09-23T20:44:01.841255+0000","last_became_peered":"2023-09-23T20:44:01.841255+0000","last_unstale":"2023-09-23T20:45:41.915056+0000","last_undegraded":"2023-09-23T20:45:41.915056+0000","last_fullsized":"2023-09-23T20:45:41.915056+0000","mapping_epoch":491,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":492,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:18:44.817589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.13","version":"90'161","reported_seq":822,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:45:41.334353+0000","last_change":"2023-09-23T20:35:02.238923+0000","last_active":"2023-09-23T20:45:41.334353+0000","last_peered":"2023-09-23T20:45:41.334353+0000","last_clean":"2023-09-23T20:45:41.334353+0000","last_became_active":"2023-09-23T20:35:02.238293+0000","last_became_peered":"2023-09-23T20:35:02.238293+0000","last_unstale":"2023-09-23T20:45:41.334353+0000","last_undegraded":"2023-09-23T20:45:41.334353+0000","last_fullsized":"2023-09-23T20:45:41.334353+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:42.544308+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:42.544308+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:51:22.281932+0000","scrub_duration":0.0029958179999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":538,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:45:41.914443+0000","last_change":"2023-09-23T20:38:02.539774+0000","last_active":"2023-09-23T20:45:41.914443+0000","last_peered":"2023-09-23T20:45:41.914443+0000","last_clean":"2023-09-23T20:45:41.914443+0000","last_became_active":"2023-09-23T20:38:02.539436+0000","last_became_peered":"2023-09-23T20:38:02.539436+0000","last_unstale":"2023-09-23T20:45:41.914443+0000","last_undegraded":"2023-09-23T20:45:41.914443+0000","last_fullsized":"2023-09-23T20:45:41.914443+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:50:57.375803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":500,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-09-23T20:45:43.505925+0000","last_change":"2023-09-23T20:43:46.521735+0000","last_active":"2023-09-23T20:45:43.505925+0000","last_peered":"2023-09-23T20:45:43.505925+0000","last_clean":"2023-09-23T20:45:43.505925+0000","last_became_active":"2023-09-23T20:43:46.520762+0000","last_became_peered":"2023-09-23T20:43:46.520762+0000","last_unstale":"2023-09-23T20:45:43.505925+0000","last_undegraded":"2023-09-23T20:45:43.505925+0000","last_fullsized":"2023-09-23T20:45:43.505925+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:26:48.841930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"90'161","reported_seq":619,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:45:41.334101+0000","last_change":"2023-09-23T20:44:18.952808+0000","last_active":"2023-09-23T20:45:41.334101+0000","last_peered":"2023-09-23T20:45:41.334101+0000","last_clean":"2023-09-23T20:45:41.334101+0000","last_became_active":"2023-09-23T20:43:16.758652+0000","last_became_peered":"2023-09-23T20:43:16.758652+0000","last_unstale":"2023-09-23T20:45:41.334101+0000","last_undegraded":"2023-09-23T20:45:41.334101+0000","last_fullsized":"2023-09-23T20:45:41.334101+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:18.952693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:18.952693+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:12:19.496465+0000","scrub_duration":0.001699365,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":210,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:45:42.051737+0000","last_change":"2023-09-23T20:44:43.644290+0000","last_active":"2023-09-23T20:45:42.051737+0000","last_peered":"2023-09-23T20:45:42.051737+0000","last_clean":"2023-09-23T20:45:42.051737+0000","last_became_active":"2023-09-23T20:44:43.638313+0000","last_became_peered":"2023-09-23T20:44:43.638313+0000","last_unstale":"2023-09-23T20:45:42.051737+0000","last_undegraded":"2023-09-23T20:45:42.051737+0000","last_fullsized":"2023-09-23T20:45:42.051737+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:32:52.892603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":331,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-09-23T20:45:43.508934+0000","last_change":"2023-09-23T20:37:43.571083+0000","last_active":"2023-09-23T20:45:43.508934+0000","last_peered":"2023-09-23T20:45:43.508934+0000","last_clean":"2023-09-23T20:45:43.508934+0000","last_became_active":"2023-09-23T20:37:43.570306+0000","last_became_peered":"2023-09-23T20:37:43.570306+0000","last_unstale":"2023-09-23T20:45:43.508934+0000","last_undegraded":"2023-09-23T20:45:43.508934+0000","last_fullsized":"2023-09-23T20:45:43.508934+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:42:02.396718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.6","version":"536'1089","reported_seq":1103,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-09-23T20:45:51.389368+0000","last_change":"2023-09-23T20:45:39.488709+0000","last_active":"2023-09-23T20:45:51.389368+0000","last_peered":"2023-09-23T20:45:51.389368+0000","last_clean":"2023-09-23T20:45:51.389368+0000","last_became_active":"2023-09-23T20:45:39.488360+0000","last_became_peered":"2023-09-23T20:45:39.488360+0000","last_unstale":"2023-09-23T20:45:51.389368+0000","last_undegraded":"2023-09-23T20:45:51.389368+0000","last_fullsized":"2023-09-23T20:45:51.389368+0000","mapping_epoch":531,"log_start":"0'0","ondisk_log_start":"0'0","created":531,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":1089,"log_dups_size":0,"ondisk_log_size":1089,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:51:44.738010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":71368704,"num_objects":1089,"num_object_clones":0,"num_object_copies":3267,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1089,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1089,"num_write_kb":69696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.d","version":"90'161","reported_seq":827,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:45:41.914832+0000","last_change":"2023-09-23T20:34:59.496438+0000","last_active":"2023-09-23T20:45:41.914832+0000","last_peered":"2023-09-23T20:45:41.914832+0000","last_clean":"2023-09-23T20:45:41.914832+0000","last_became_active":"2023-09-23T20:33:38.970527+0000","last_became_peered":"2023-09-23T20:33:38.970527+0000","last_unstale":"2023-09-23T20:45:41.914832+0000","last_undegraded":"2023-09-23T20:45:41.914832+0000","last_fullsized":"2023-09-23T20:45:41.914832+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:34:59.496375+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:34:59.496375+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:46:18.244734+0000","scrub_duration":0.0017843189999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":538,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:45:41.915106+0000","last_change":"2023-09-23T20:32:33.283752+0000","last_active":"2023-09-23T20:45:41.915106+0000","last_peered":"2023-09-23T20:45:41.915106+0000","last_clean":"2023-09-23T20:45:41.915106+0000","last_became_active":"2023-09-23T20:32:33.230503+0000","last_became_peered":"2023-09-23T20:32:33.230503+0000","last_unstale":"2023-09-23T20:45:41.915106+0000","last_undegraded":"2023-09-23T20:45:41.915106+0000","last_fullsized":"2023-09-23T20:45:41.915106+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:29.084410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":349,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:45:41.914424+0000","last_change":"2023-09-23T20:43:16.765253+0000","last_active":"2023-09-23T20:45:41.914424+0000","last_peered":"2023-09-23T20:45:41.914424+0000","last_clean":"2023-09-23T20:45:41.914424+0000","last_became_active":"2023-09-23T20:43:16.764978+0000","last_became_peered":"2023-09-23T20:43:16.764978+0000","last_unstale":"2023-09-23T20:45:41.914424+0000","last_undegraded":"2023-09-23T20:45:41.914424+0000","last_fullsized":"2023-09-23T20:45:41.914424+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:07:23.539372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.7","version":"535'645","reported_seq":657,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-09-23T20:45:48.251834+0000","last_change":"2023-09-23T20:45:39.488542+0000","last_active":"2023-09-23T20:45:48.251834+0000","last_peered":"2023-09-23T20:45:48.251834+0000","last_clean":"2023-09-23T20:45:48.251834+0000","last_became_active":"2023-09-23T20:45:39.488259+0000","last_became_peered":"2023-09-23T20:45:39.488259+0000","last_unstale":"2023-09-23T20:45:48.251834+0000","last_undegraded":"2023-09-23T20:45:48.251834+0000","last_fullsized":"2023-09-23T20:45:48.251834+0000","mapping_epoch":531,"log_start":"0'0","ondisk_log_start":"0'0","created":531,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":645,"log_dups_size":0,"ondisk_log_size":645,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:39:05.833662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":42270720,"num_objects":645,"num_object_clones":0,"num_object_copies":1935,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":645,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":645,"num_write_kb":41280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.c","version":"90'161","reported_seq":797,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-09-23T20:45:43.509049+0000","last_change":"2023-09-23T20:41:09.506481+0000","last_active":"2023-09-23T20:45:43.509049+0000","last_peered":"2023-09-23T20:45:43.509049+0000","last_clean":"2023-09-23T20:45:43.509049+0000","last_became_active":"2023-09-23T20:38:28.442229+0000","last_became_peered":"2023-09-23T20:38:28.442229+0000","last_unstale":"2023-09-23T20:45:43.509049+0000","last_undegraded":"2023-09-23T20:45:43.509049+0000","last_fullsized":"2023-09-23T20:45:43.509049+0000","mapping_epoch":411,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":412,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:41:09.506338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:41:09.506338+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:19:46.265011+0000","scrub_duration":0.01112002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":573,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:45:41.915019+0000","last_change":"2023-09-23T20:34:59.987981+0000","last_active":"2023-09-23T20:45:41.915019+0000","last_peered":"2023-09-23T20:45:41.915019+0000","last_clean":"2023-09-23T20:45:41.915019+0000","last_became_active":"2023-09-23T20:34:59.985368+0000","last_became_peered":"2023-09-23T20:34:59.985368+0000","last_unstale":"2023-09-23T20:45:41.915019+0000","last_undegraded":"2023-09-23T20:45:41.915019+0000","last_fullsized":"2023-09-23T20:45:41.915019+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:12:43.082493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":233,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-09-23T20:45:42.564876+0000","last_change":"2023-09-23T20:32:33.232142+0000","last_active":"2023-09-23T20:45:42.564876+0000","last_peered":"2023-09-23T20:45:42.564876+0000","last_clean":"2023-09-23T20:45:42.564876+0000","last_became_active":"2023-09-23T20:32:33.231683+0000","last_became_peered":"2023-09-23T20:32:33.231683+0000","last_unstale":"2023-09-23T20:45:42.564876+0000","last_undegraded":"2023-09-23T20:45:42.564876+0000","last_fullsized":"2023-09-23T20:45:42.564876+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:02:58.336348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"536'1094","reported_seq":1108,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-09-23T20:45:51.403173+0000","last_change":"2023-09-23T20:45:39.381562+0000","last_active":"2023-09-23T20:45:51.403173+0000","last_peered":"2023-09-23T20:45:51.403173+0000","last_clean":"2023-09-23T20:45:51.403173+0000","last_became_active":"2023-09-23T20:45:39.380887+0000","last_became_peered":"2023-09-23T20:45:39.380887+0000","last_unstale":"2023-09-23T20:45:51.403173+0000","last_undegraded":"2023-09-23T20:45:51.403173+0000","last_fullsized":"2023-09-23T20:45:51.403173+0000","mapping_epoch":531,"log_start":"0'0","ondisk_log_start":"0'0","created":531,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":1094,"log_dups_size":0,"ondisk_log_size":1094,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:41:52.967029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":71696384,"num_objects":1094,"num_object_clones":0,"num_object_copies":3282,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1094,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1094,"num_write_kb":70016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"90'161","reported_seq":690,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:45:42.052260+0000","last_change":"2023-09-23T20:38:40.491902+0000","last_active":"2023-09-23T20:45:42.052260+0000","last_peered":"2023-09-23T20:45:42.052260+0000","last_clean":"2023-09-23T20:45:42.052260+0000","last_became_active":"2023-09-23T20:38:40.491563+0000","last_became_peered":"2023-09-23T20:38:40.491563+0000","last_unstale":"2023-09-23T20:45:42.052260+0000","last_undegraded":"2023-09-23T20:45:42.052260+0000","last_fullsized":"2023-09-23T20:45:42.052260+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:46.217489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:46.217489+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:39:25.008796+0000","scrub_duration":0.001456429,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":563,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:45:41.914836+0000","last_change":"2023-09-23T20:40:19.677655+0000","last_active":"2023-09-23T20:45:41.914836+0000","last_peered":"2023-09-23T20:45:41.914836+0000","last_clean":"2023-09-23T20:45:41.914836+0000","last_became_active":"2023-09-23T20:40:19.551189+0000","last_became_peered":"2023-09-23T20:40:19.551189+0000","last_unstale":"2023-09-23T20:45:41.914836+0000","last_undegraded":"2023-09-23T20:45:41.914836+0000","last_fullsized":"2023-09-23T20:45:41.914836+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:11:01.559576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":140,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:45:42.051603+0000","last_change":"2023-09-23T20:38:02.590613+0000","last_active":"2023-09-23T20:45:42.051603+0000","last_peered":"2023-09-23T20:45:42.051603+0000","last_clean":"2023-09-23T20:45:42.051603+0000","last_became_active":"2023-09-23T20:38:02.590284+0000","last_became_peered":"2023-09-23T20:38:02.590284+0000","last_unstale":"2023-09-23T20:45:42.051603+0000","last_undegraded":"2023-09-23T20:45:42.051603+0000","last_fullsized":"2023-09-23T20:45:42.051603+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:40:07.825054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.5","version":"535'567","reported_seq":580,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-09-23T20:45:48.089212+0000","last_change":"2023-09-23T20:45:39.380986+0000","last_active":"2023-09-23T20:45:48.089212+0000","last_peered":"2023-09-23T20:45:48.089212+0000","last_clean":"2023-09-23T20:45:48.089212+0000","last_became_active":"2023-09-23T20:45:39.380650+0000","last_became_peered":"2023-09-23T20:45:39.380650+0000","last_unstale":"2023-09-23T20:45:48.089212+0000","last_undegraded":"2023-09-23T20:45:48.089212+0000","last_fullsized":"2023-09-23T20:45:48.089212+0000","mapping_epoch":531,"log_start":"0'0","ondisk_log_start":"0'0","created":531,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":567,"log_dups_size":0,"ondisk_log_size":567,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:24:24.301239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":37158912,"num_objects":567,"num_object_clones":0,"num_object_copies":1701,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":567,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":567,"num_write_kb":36288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.e","version":"90'161","reported_seq":636,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-09-23T20:45:42.565423+0000","last_change":"2023-09-23T20:36:00.322453+0000","last_active":"2023-09-23T20:45:42.565423+0000","last_peered":"2023-09-23T20:45:42.565423+0000","last_clean":"2023-09-23T20:45:42.565423+0000","last_became_active":"2023-09-23T20:35:02.233536+0000","last_became_peered":"2023-09-23T20:35:02.233536+0000","last_unstale":"2023-09-23T20:45:42.565423+0000","last_undegraded":"2023-09-23T20:45:42.565423+0000","last_fullsized":"2023-09-23T20:45:42.565423+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:00.322381+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:00.322381+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:02:15.489069+0000","scrub_duration":0.0017321859999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":496,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-09-23T20:45:43.509187+0000","last_change":"2023-09-23T20:43:16.756437+0000","last_active":"2023-09-23T20:45:43.509187+0000","last_peered":"2023-09-23T20:45:43.509187+0000","last_clean":"2023-09-23T20:45:43.509187+0000","last_became_active":"2023-09-23T20:43:16.756193+0000","last_became_peered":"2023-09-23T20:43:16.756193+0000","last_unstale":"2023-09-23T20:45:43.509187+0000","last_undegraded":"2023-09-23T20:45:43.509187+0000","last_fullsized":"2023-09-23T20:45:43.509187+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:33:25.632781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":509,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-09-23T20:45:43.507447+0000","last_change":"2023-09-23T20:43:46.541524+0000","last_active":"2023-09-23T20:45:43.507447+0000","last_peered":"2023-09-23T20:45:43.507447+0000","last_clean":"2023-09-23T20:45:43.507447+0000","last_became_active":"2023-09-23T20:43:46.541051+0000","last_became_peered":"2023-09-23T20:43:46.541051+0000","last_unstale":"2023-09-23T20:45:43.507447+0000","last_undegraded":"2023-09-23T20:45:43.507447+0000","last_fullsized":"2023-09-23T20:45:43.507447+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:59:36.632592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"535'623","reported_seq":636,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-09-23T20:45:48.527007+0000","last_change":"2023-09-23T20:45:39.331236+0000","last_active":"2023-09-23T20:45:48.527007+0000","last_peered":"2023-09-23T20:45:48.527007+0000","last_clean":"2023-09-23T20:45:48.527007+0000","last_became_active":"2023-09-23T20:45:39.330923+0000","last_became_peered":"2023-09-23T20:45:39.330923+0000","last_unstale":"2023-09-23T20:45:48.527007+0000","last_undegraded":"2023-09-23T20:45:48.527007+0000","last_fullsized":"2023-09-23T20:45:48.527007+0000","mapping_epoch":531,"log_start":"0'0","ondisk_log_start":"0'0","created":531,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":623,"log_dups_size":0,"ondisk_log_size":623,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:48:20.711262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":40828928,"num_objects":623,"num_object_clones":0,"num_object_copies":1869,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":623,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":623,"num_write_kb":39872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":940,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:45:41.914274+0000","last_change":"2023-09-23T20:35:02.523308+0000","last_active":"2023-09-23T20:45:41.914274+0000","last_peered":"2023-09-23T20:45:41.914274+0000","last_clean":"2023-09-23T20:45:41.914274+0000","last_became_active":"2023-09-23T20:33:38.979989+0000","last_became_peered":"2023-09-23T20:33:38.979989+0000","last_unstale":"2023-09-23T20:45:41.914274+0000","last_undegraded":"2023-09-23T20:45:41.914274+0000","last_fullsized":"2023-09-23T20:45:41.914274+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:35:02.523203+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:35:02.523203+0000","last_clean_scrub_stamp":"2023-09-23T20:35:02.523203+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:02:41.472958+0000","scrub_duration":0.0085927989999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":357,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-09-23T20:45:43.508682+0000","last_change":"2023-09-23T20:37:43.583731+0000","last_active":"2023-09-23T20:45:43.508682+0000","last_peered":"2023-09-23T20:45:43.508682+0000","last_clean":"2023-09-23T20:45:43.508682+0000","last_became_active":"2023-09-23T20:37:43.583066+0000","last_became_peered":"2023-09-23T20:37:43.583066+0000","last_unstale":"2023-09-23T20:45:43.508682+0000","last_undegraded":"2023-09-23T20:45:43.508682+0000","last_fullsized":"2023-09-23T20:45:43.508682+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:03:45.211900+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":253,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-09-23T20:45:42.565151+0000","last_change":"2023-09-23T20:38:02.579814+0000","last_active":"2023-09-23T20:45:42.565151+0000","last_peered":"2023-09-23T20:45:42.565151+0000","last_clean":"2023-09-23T20:45:42.565151+0000","last_became_active":"2023-09-23T20:38:02.579253+0000","last_became_peered":"2023-09-23T20:38:02.579253+0000","last_unstale":"2023-09-23T20:45:42.565151+0000","last_undegraded":"2023-09-23T20:45:42.565151+0000","last_fullsized":"2023-09-23T20:45:42.565151+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:05:07.298198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"535'617","reported_seq":630,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-09-23T20:45:48.099476+0000","last_change":"2023-09-23T20:45:39.488901+0000","last_active":"2023-09-23T20:45:48.099476+0000","last_peered":"2023-09-23T20:45:48.099476+0000","last_clean":"2023-09-23T20:45:48.099476+0000","last_became_active":"2023-09-23T20:45:39.488499+0000","last_became_peered":"2023-09-23T20:45:39.488499+0000","last_unstale":"2023-09-23T20:45:48.099476+0000","last_undegraded":"2023-09-23T20:45:48.099476+0000","last_fullsized":"2023-09-23T20:45:48.099476+0000","mapping_epoch":531,"log_start":"0'0","ondisk_log_start":"0'0","created":531,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":617,"log_dups_size":0,"ondisk_log_size":617,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:25:10.137770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":40435712,"num_objects":617,"num_object_clones":0,"num_object_copies":1851,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":617,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":617,"num_write_kb":39488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":650,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:45:42.355933+0000","last_change":"2023-09-23T20:44:46.725779+0000","last_active":"2023-09-23T20:45:42.355933+0000","last_peered":"2023-09-23T20:45:42.355933+0000","last_clean":"2023-09-23T20:45:42.355933+0000","last_became_active":"2023-09-23T20:44:46.725433+0000","last_became_peered":"2023-09-23T20:44:46.725433+0000","last_unstale":"2023-09-23T20:45:42.355933+0000","last_undegraded":"2023-09-23T20:45:42.355933+0000","last_fullsized":"2023-09-23T20:45:42.355933+0000","mapping_epoch":516,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":517,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:42:45.492220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:42:45.492220+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:28:35.202407+0000","scrub_duration":0.015259412,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":140,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:45:42.051716+0000","last_change":"2023-09-23T20:38:02.578703+0000","last_active":"2023-09-23T20:45:42.051716+0000","last_peered":"2023-09-23T20:45:42.051716+0000","last_clean":"2023-09-23T20:45:42.051716+0000","last_became_active":"2023-09-23T20:38:02.578414+0000","last_became_peered":"2023-09-23T20:38:02.578414+0000","last_unstale":"2023-09-23T20:45:42.051716+0000","last_undegraded":"2023-09-23T20:45:42.051716+0000","last_fullsized":"2023-09-23T20:45:42.051716+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:43:59.779295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":202,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-09-23T20:45:42.565217+0000","last_change":"2023-09-23T20:37:31.296606+0000","last_active":"2023-09-23T20:45:42.565217+0000","last_peered":"2023-09-23T20:45:42.565217+0000","last_clean":"2023-09-23T20:45:42.565217+0000","last_became_active":"2023-09-23T20:37:31.295924+0000","last_became_peered":"2023-09-23T20:37:31.295924+0000","last_unstale":"2023-09-23T20:45:42.565217+0000","last_undegraded":"2023-09-23T20:45:42.565217+0000","last_fullsized":"2023-09-23T20:45:42.565217+0000","mapping_epoch":391,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":392,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:27:50.948510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.0","version":"535'593","reported_seq":605,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-09-23T20:45:48.259164+0000","last_change":"2023-09-23T20:45:39.331645+0000","last_active":"2023-09-23T20:45:48.259164+0000","last_peered":"2023-09-23T20:45:48.259164+0000","last_clean":"2023-09-23T20:45:48.259164+0000","last_became_active":"2023-09-23T20:45:39.331293+0000","last_became_peered":"2023-09-23T20:45:39.331293+0000","last_unstale":"2023-09-23T20:45:48.259164+0000","last_undegraded":"2023-09-23T20:45:48.259164+0000","last_fullsized":"2023-09-23T20:45:48.259164+0000","mapping_epoch":531,"log_start":"0'0","ondisk_log_start":"0'0","created":531,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":593,"log_dups_size":0,"ondisk_log_size":593,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:29:42.789115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":38862848,"num_objects":593,"num_object_clones":0,"num_object_copies":1779,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":593,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":593,"num_write_kb":37952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.b","version":"90'161","reported_seq":839,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:45:41.334382+0000","last_change":"2023-09-23T20:40:22.445808+0000","last_active":"2023-09-23T20:45:41.334382+0000","last_peered":"2023-09-23T20:45:41.334382+0000","last_clean":"2023-09-23T20:45:41.334382+0000","last_became_active":"2023-09-23T20:40:22.445456+0000","last_became_peered":"2023-09-23T20:40:22.445456+0000","last_unstale":"2023-09-23T20:45:41.334382+0000","last_undegraded":"2023-09-23T20:45:41.334382+0000","last_fullsized":"2023-09-23T20:45:41.334382+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:05.595042+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:05.595042+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:39:23.678154+0000","scrub_duration":0.007271675,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":192,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-09-23T20:45:42.564731+0000","last_change":"2023-09-23T20:34:59.983122+0000","last_active":"2023-09-23T20:45:42.564731+0000","last_peered":"2023-09-23T20:45:42.564731+0000","last_clean":"2023-09-23T20:45:42.564731+0000","last_became_active":"2023-09-23T20:34:59.982809+0000","last_became_peered":"2023-09-23T20:34:59.982809+0000","last_unstale":"2023-09-23T20:45:42.564731+0000","last_undegraded":"2023-09-23T20:45:42.564731+0000","last_fullsized":"2023-09-23T20:45:42.564731+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:49:40.839723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":273,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-09-23T20:45:42.565581+0000","last_change":"2023-09-23T20:44:10.606995+0000","last_active":"2023-09-23T20:45:42.565581+0000","last_peered":"2023-09-23T20:45:42.565581+0000","last_clean":"2023-09-23T20:45:42.565581+0000","last_became_active":"2023-09-23T20:44:10.606718+0000","last_became_peered":"2023-09-23T20:44:10.606718+0000","last_unstale":"2023-09-23T20:45:42.565581+0000","last_undegraded":"2023-09-23T20:45:42.565581+0000","last_fullsized":"2023-09-23T20:45:42.565581+0000","mapping_epoch":496,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":497,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:13:12.496245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"536'1023","reported_seq":1036,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-09-23T20:45:50.700469+0000","last_change":"2023-09-23T20:45:39.333840+0000","last_active":"2023-09-23T20:45:50.700469+0000","last_peered":"2023-09-23T20:45:50.700469+0000","last_clean":"2023-09-23T20:45:50.700469+0000","last_became_active":"2023-09-23T20:45:39.333454+0000","last_became_peered":"2023-09-23T20:45:39.333454+0000","last_unstale":"2023-09-23T20:45:50.700469+0000","last_undegraded":"2023-09-23T20:45:50.700469+0000","last_fullsized":"2023-09-23T20:45:50.700469+0000","mapping_epoch":531,"log_start":"0'0","ondisk_log_start":"0'0","created":531,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":1023,"log_dups_size":0,"ondisk_log_size":1023,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:23:21.002722+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":67043328,"num_objects":1023,"num_object_clones":0,"num_object_copies":3069,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1023,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1023,"num_write_kb":65472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":561,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:45:42.355952+0000","last_change":"2023-09-23T20:43:51.693618+0000","last_active":"2023-09-23T20:45:42.355952+0000","last_peered":"2023-09-23T20:45:42.355952+0000","last_clean":"2023-09-23T20:45:42.355952+0000","last_became_active":"2023-09-23T20:43:16.765186+0000","last_became_peered":"2023-09-23T20:43:16.765186+0000","last_unstale":"2023-09-23T20:45:42.355952+0000","last_undegraded":"2023-09-23T20:45:42.355952+0000","last_fullsized":"2023-09-23T20:45:42.355952+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:43:51.693459+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:43:51.693459+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:33:30.208355+0000","scrub_duration":0.0016145269999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":331,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-09-23T20:45:43.507139+0000","last_change":"2023-09-23T20:37:43.570610+0000","last_active":"2023-09-23T20:45:43.507139+0000","last_peered":"2023-09-23T20:45:43.507139+0000","last_clean":"2023-09-23T20:45:43.507139+0000","last_became_active":"2023-09-23T20:37:43.570216+0000","last_became_peered":"2023-09-23T20:37:43.570216+0000","last_unstale":"2023-09-23T20:45:43.507139+0000","last_undegraded":"2023-09-23T20:45:43.507139+0000","last_fullsized":"2023-09-23T20:45:43.507139+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:39:26.555329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":361,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:45:41.334709+0000","last_change":"2023-09-23T20:32:33.230582+0000","last_active":"2023-09-23T20:45:41.334709+0000","last_peered":"2023-09-23T20:45:41.334709+0000","last_clean":"2023-09-23T20:45:41.334709+0000","last_became_active":"2023-09-23T20:32:33.230257+0000","last_became_peered":"2023-09-23T20:32:33.230257+0000","last_unstale":"2023-09-23T20:45:41.334709+0000","last_undegraded":"2023-09-23T20:45:41.334709+0000","last_fullsized":"2023-09-23T20:45:41.334709+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:11:45.333096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"535'492","reported_seq":504,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-09-23T20:45:47.560342+0000","last_change":"2023-09-23T20:45:39.332694+0000","last_active":"2023-09-23T20:45:47.560342+0000","last_peered":"2023-09-23T20:45:47.560342+0000","last_clean":"2023-09-23T20:45:47.560342+0000","last_became_active":"2023-09-23T20:45:39.332422+0000","last_became_peered":"2023-09-23T20:45:39.332422+0000","last_unstale":"2023-09-23T20:45:47.560342+0000","last_undegraded":"2023-09-23T20:45:47.560342+0000","last_fullsized":"2023-09-23T20:45:47.560342+0000","mapping_epoch":531,"log_start":"0'0","ondisk_log_start":"0'0","created":531,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":492,"log_dups_size":0,"ondisk_log_size":492,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:50:39.768517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":32243712,"num_objects":492,"num_object_clones":0,"num_object_copies":1476,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":492,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":492,"num_write_kb":31488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":723,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:45:42.052195+0000","last_change":"2023-09-23T20:41:07.424609+0000","last_active":"2023-09-23T20:45:42.052195+0000","last_peered":"2023-09-23T20:45:42.052195+0000","last_clean":"2023-09-23T20:45:42.052195+0000","last_became_active":"2023-09-23T20:38:05.622261+0000","last_became_peered":"2023-09-23T20:38:05.622261+0000","last_unstale":"2023-09-23T20:45:42.052195+0000","last_undegraded":"2023-09-23T20:45:42.052195+0000","last_fullsized":"2023-09-23T20:45:42.052195+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:41:07.424492+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:41:07.424492+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:45:21.689123+0000","scrub_duration":0.0018960959999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"86'119","reported_seq":845,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:45:41.334430+0000","last_change":"2023-09-23T20:29:45.521257+0000","last_active":"2023-09-23T20:45:41.334430+0000","last_peered":"2023-09-23T20:45:41.334430+0000","last_clean":"2023-09-23T20:45:41.334430+0000","last_became_active":"2023-09-23T20:29:45.520546+0000","last_became_peered":"2023-09-23T20:29:45.520546+0000","last_unstale":"2023-09-23T20:45:41.334430+0000","last_undegraded":"2023-09-23T20:45:41.334430+0000","last_fullsized":"2023-09-23T20:45:41.334430+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":119,"log_dups_size":0,"ondisk_log_size":119,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:58:56.085777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032705499999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":523,"num_read_kb":3503,"num_write":136,"num_write_kb":3128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":329,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-09-23T20:45:43.508097+0000","last_change":"2023-09-23T20:43:16.756494+0000","last_active":"2023-09-23T20:45:43.508097+0000","last_peered":"2023-09-23T20:45:43.508097+0000","last_clean":"2023-09-23T20:45:43.508097+0000","last_became_active":"2023-09-23T20:43:16.756131+0000","last_became_peered":"2023-09-23T20:43:16.756131+0000","last_unstale":"2023-09-23T20:45:43.508097+0000","last_undegraded":"2023-09-23T20:45:43.508097+0000","last_fullsized":"2023-09-23T20:45:43.508097+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:10:19.219533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":492,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:45:41.333863+0000","last_change":"2023-09-23T20:40:19.530942+0000","last_active":"2023-09-23T20:45:41.333863+0000","last_peered":"2023-09-23T20:45:41.333863+0000","last_clean":"2023-09-23T20:45:41.333863+0000","last_became_active":"2023-09-23T20:40:19.530617+0000","last_became_peered":"2023-09-23T20:40:19.530617+0000","last_unstale":"2023-09-23T20:45:41.333863+0000","last_undegraded":"2023-09-23T20:45:41.333863+0000","last_fullsized":"2023-09-23T20:45:41.333863+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:24:09.231934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"535'601","reported_seq":613,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-09-23T20:45:48.253791+0000","last_change":"2023-09-23T20:45:39.489110+0000","last_active":"2023-09-23T20:45:48.253791+0000","last_peered":"2023-09-23T20:45:48.253791+0000","last_clean":"2023-09-23T20:45:48.253791+0000","last_became_active":"2023-09-23T20:45:39.488817+0000","last_became_peered":"2023-09-23T20:45:39.488817+0000","last_unstale":"2023-09-23T20:45:48.253791+0000","last_undegraded":"2023-09-23T20:45:48.253791+0000","last_fullsized":"2023-09-23T20:45:48.253791+0000","mapping_epoch":531,"log_start":"0'0","ondisk_log_start":"0'0","created":531,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":601,"log_dups_size":0,"ondisk_log_size":601,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:07:21.467971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":39387136,"num_objects":601,"num_object_clones":0,"num_object_copies":1803,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":601,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":601,"num_write_kb":38464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":582,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-09-23T20:45:48.592269+0000","last_change":"2023-09-23T20:44:15.656984+0000","last_active":"2023-09-23T20:45:48.592269+0000","last_peered":"2023-09-23T20:45:48.592269+0000","last_clean":"2023-09-23T20:45:48.592269+0000","last_became_active":"2023-09-23T20:43:46.512210+0000","last_became_peered":"2023-09-23T20:43:46.512210+0000","last_unstale":"2023-09-23T20:45:48.592269+0000","last_undegraded":"2023-09-23T20:45:48.592269+0000","last_fullsized":"2023-09-23T20:45:48.592269+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:15.656880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:15.656880+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:37:56.453466+0000","scrub_duration":0.0016711180000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"86'106","reported_seq":786,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:45:41.334023+0000","last_change":"2023-09-23T20:43:16.759138+0000","last_active":"2023-09-23T20:45:41.334023+0000","last_peered":"2023-09-23T20:45:41.334023+0000","last_clean":"2023-09-23T20:45:41.334023+0000","last_became_active":"2023-09-23T20:43:16.758015+0000","last_became_peered":"2023-09-23T20:43:16.758015+0000","last_unstale":"2023-09-23T20:45:41.334023+0000","last_undegraded":"2023-09-23T20:45:41.334023+0000","last_fullsized":"2023-09-23T20:45:41.334023+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":106,"log_dups_size":0,"ondisk_log_size":106,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:08:02.852472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00099302599999999993,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1094,"num_read_kb":1678,"num_write":100,"num_write_kb":404,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":253,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-09-23T20:45:42.565063+0000","last_change":"2023-09-23T20:38:02.552008+0000","last_active":"2023-09-23T20:45:42.565063+0000","last_peered":"2023-09-23T20:45:42.565063+0000","last_clean":"2023-09-23T20:45:42.565063+0000","last_became_active":"2023-09-23T20:38:02.551684+0000","last_became_peered":"2023-09-23T20:38:02.551684+0000","last_unstale":"2023-09-23T20:45:42.565063+0000","last_undegraded":"2023-09-23T20:45:42.565063+0000","last_fullsized":"2023-09-23T20:45:42.565063+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:40:37.147538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":498,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:45:41.334446+0000","last_change":"2023-09-23T20:29:45.533406+0000","last_active":"2023-09-23T20:45:41.334446+0000","last_peered":"2023-09-23T20:45:41.334446+0000","last_clean":"2023-09-23T20:45:41.334446+0000","last_became_active":"2023-09-23T20:29:45.526001+0000","last_became_peered":"2023-09-23T20:29:45.526001+0000","last_unstale":"2023-09-23T20:45:41.334446+0000","last_undegraded":"2023-09-23T20:45:41.334446+0000","last_fullsized":"2023-09-23T20:45:41.334446+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:43.787251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.9","version":"535'586","reported_seq":599,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-09-23T20:45:48.097565+0000","last_change":"2023-09-23T20:45:39.488721+0000","last_active":"2023-09-23T20:45:48.097565+0000","last_peered":"2023-09-23T20:45:48.097565+0000","last_clean":"2023-09-23T20:45:48.097565+0000","last_became_active":"2023-09-23T20:45:39.488320+0000","last_became_peered":"2023-09-23T20:45:39.488320+0000","last_unstale":"2023-09-23T20:45:48.097565+0000","last_undegraded":"2023-09-23T20:45:48.097565+0000","last_fullsized":"2023-09-23T20:45:48.097565+0000","mapping_epoch":531,"log_start":"0'0","ondisk_log_start":"0'0","created":531,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":586,"log_dups_size":0,"ondisk_log_size":586,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:46:30.536848+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":38404096,"num_objects":586,"num_object_clones":0,"num_object_copies":1758,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":586,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":586,"num_write_kb":37504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":498,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:45:41.914426+0000","last_change":"2023-09-23T20:44:26.978243+0000","last_active":"2023-09-23T20:45:41.914426+0000","last_peered":"2023-09-23T20:45:41.914426+0000","last_clean":"2023-09-23T20:45:41.914426+0000","last_became_active":"2023-09-23T20:44:18.638989+0000","last_became_peered":"2023-09-23T20:44:18.638989+0000","last_unstale":"2023-09-23T20:45:41.914426+0000","last_undegraded":"2023-09-23T20:45:41.914426+0000","last_fullsized":"2023-09-23T20:45:41.914426+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:26.978179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:26.978179+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:25:38.977011+0000","scrub_duration":0.0013962829999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"86'130","reported_seq":853,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:45:42.355762+0000","last_change":"2023-09-23T20:25:39.637458+0000","last_active":"2023-09-23T20:45:42.355762+0000","last_peered":"2023-09-23T20:45:42.355762+0000","last_clean":"2023-09-23T20:45:42.355762+0000","last_became_active":"2023-09-23T20:25:39.636773+0000","last_became_peered":"2023-09-23T20:25:39.636773+0000","last_unstale":"2023-09-23T20:45:42.355762+0000","last_undegraded":"2023-09-23T20:45:42.355762+0000","last_fullsized":"2023-09-23T20:45:42.355762+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:00:32.810564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011300049999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":512,"num_read_kb":1820,"num_write":133,"num_write_kb":1347,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":506,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:45:41.333700+0000","last_change":"2023-09-23T20:38:02.591335+0000","last_active":"2023-09-23T20:45:41.333700+0000","last_peered":"2023-09-23T20:45:41.333700+0000","last_clean":"2023-09-23T20:45:41.333700+0000","last_became_active":"2023-09-23T20:38:02.591015+0000","last_became_peered":"2023-09-23T20:38:02.591015+0000","last_unstale":"2023-09-23T20:45:41.333700+0000","last_undegraded":"2023-09-23T20:45:41.333700+0000","last_fullsized":"2023-09-23T20:45:41.333700+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:55:32.272195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":49,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:45:41.334626+0000","last_change":"2023-09-23T20:44:10.555583+0000","last_active":"2023-09-23T20:45:41.334626+0000","last_peered":"2023-09-23T20:45:41.334626+0000","last_clean":"2023-09-23T20:45:41.334626+0000","last_became_active":"2023-09-23T20:44:10.555281+0000","last_became_peered":"2023-09-23T20:44:10.555281+0000","last_unstale":"2023-09-23T20:45:41.334626+0000","last_undegraded":"2023-09-23T20:45:41.334626+0000","last_fullsized":"2023-09-23T20:45:41.334626+0000","mapping_epoch":496,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":497,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:43:11.942147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"535'618","reported_seq":630,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-09-23T20:45:48.253978+0000","last_change":"2023-09-23T20:45:39.331096+0000","last_active":"2023-09-23T20:45:48.253978+0000","last_peered":"2023-09-23T20:45:48.253978+0000","last_clean":"2023-09-23T20:45:48.253978+0000","last_became_active":"2023-09-23T20:45:39.330752+0000","last_became_peered":"2023-09-23T20:45:39.330752+0000","last_unstale":"2023-09-23T20:45:48.253978+0000","last_undegraded":"2023-09-23T20:45:48.253978+0000","last_fullsized":"2023-09-23T20:45:48.253978+0000","mapping_epoch":531,"log_start":"0'0","ondisk_log_start":"0'0","created":531,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":618,"log_dups_size":0,"ondisk_log_size":618,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:39:01.627762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":40501248,"num_objects":618,"num_object_clones":0,"num_object_copies":1854,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":618,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":618,"num_write_kb":39552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"86'194","reported_seq":892,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:45:42.356218+0000","last_change":"2023-09-23T20:43:16.765865+0000","last_active":"2023-09-23T20:45:42.356218+0000","last_peered":"2023-09-23T20:45:42.356218+0000","last_clean":"2023-09-23T20:45:42.356218+0000","last_became_active":"2023-09-23T20:43:16.765289+0000","last_became_peered":"2023-09-23T20:43:16.765289+0000","last_unstale":"2023-09-23T20:45:42.356218+0000","last_undegraded":"2023-09-23T20:45:42.356218+0000","last_fullsized":"2023-09-23T20:45:42.356218+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":194,"log_dups_size":0,"ondisk_log_size":194,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:15:16.461900+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00037995999999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1024,"num_read_kb":1200,"num_write":173,"num_write_kb":1276,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":997,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:45:41.914281+0000","last_change":"2023-09-23T20:38:05.602508+0000","last_active":"2023-09-23T20:45:41.914281+0000","last_peered":"2023-09-23T20:45:41.914281+0000","last_clean":"2023-09-23T20:45:41.914281+0000","last_became_active":"2023-09-23T20:38:05.602129+0000","last_became_peered":"2023-09-23T20:38:05.602129+0000","last_unstale":"2023-09-23T20:45:41.914281+0000","last_undegraded":"2023-09-23T20:45:41.914281+0000","last_fullsized":"2023-09-23T20:45:41.914281+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:45.343742+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:45.343742+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:48.910343+0000","scrub_duration":0.001475802,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":110,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:45:42.051814+0000","last_change":"2023-09-23T20:43:16.758213+0000","last_active":"2023-09-23T20:45:42.051814+0000","last_peered":"2023-09-23T20:45:42.051814+0000","last_clean":"2023-09-23T20:45:42.051814+0000","last_became_active":"2023-09-23T20:43:16.757869+0000","last_became_peered":"2023-09-23T20:43:16.757869+0000","last_unstale":"2023-09-23T20:45:42.051814+0000","last_undegraded":"2023-09-23T20:45:42.051814+0000","last_fullsized":"2023-09-23T20:45:42.051814+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:43:10.811749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":140,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:45:42.052082+0000","last_change":"2023-09-23T20:38:02.573200+0000","last_active":"2023-09-23T20:45:42.052082+0000","last_peered":"2023-09-23T20:45:42.052082+0000","last_clean":"2023-09-23T20:45:42.052082+0000","last_became_active":"2023-09-23T20:38:02.572945+0000","last_became_peered":"2023-09-23T20:38:02.572945+0000","last_unstale":"2023-09-23T20:45:42.052082+0000","last_undegraded":"2023-09-23T20:45:42.052082+0000","last_fullsized":"2023-09-23T20:45:42.052082+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:57.246533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.8","version":"536'1077","reported_seq":1091,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-09-23T20:45:51.405802+0000","last_change":"2023-09-23T20:45:39.332075+0000","last_active":"2023-09-23T20:45:51.405802+0000","last_peered":"2023-09-23T20:45:51.405802+0000","last_clean":"2023-09-23T20:45:51.405802+0000","last_became_active":"2023-09-23T20:45:39.331755+0000","last_became_peered":"2023-09-23T20:45:39.331755+0000","last_unstale":"2023-09-23T20:45:51.405802+0000","last_undegraded":"2023-09-23T20:45:51.405802+0000","last_fullsized":"2023-09-23T20:45:51.405802+0000","mapping_epoch":531,"log_start":"0'0","ondisk_log_start":"0'0","created":531,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":1077,"log_dups_size":0,"ondisk_log_size":1077,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:11:17.895002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70582272,"num_objects":1077,"num_object_clones":0,"num_object_copies":3231,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1077,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1077,"num_write_kb":68928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":867,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:45:41.334286+0000","last_change":"2023-09-23T20:38:05.615711+0000","last_active":"2023-09-23T20:45:41.334286+0000","last_peered":"2023-09-23T20:45:41.334286+0000","last_clean":"2023-09-23T20:45:41.334286+0000","last_became_active":"2023-09-23T20:38:05.615390+0000","last_became_peered":"2023-09-23T20:38:05.615390+0000","last_unstale":"2023-09-23T20:45:41.334286+0000","last_undegraded":"2023-09-23T20:45:41.334286+0000","last_fullsized":"2023-09-23T20:45:41.334286+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:43.582146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:43.582146+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:45:11.917871+0000","scrub_duration":0.0016219730000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"86'80","reported_seq":787,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:45:41.914221+0000","last_change":"2023-09-23T20:29:45.536796+0000","last_active":"2023-09-23T20:45:41.914221+0000","last_peered":"2023-09-23T20:45:41.914221+0000","last_clean":"2023-09-23T20:45:41.914221+0000","last_became_active":"2023-09-23T20:29:45.536439+0000","last_became_peered":"2023-09-23T20:29:45.536439+0000","last_unstale":"2023-09-23T20:45:41.914221+0000","last_undegraded":"2023-09-23T20:45:41.914221+0000","last_fullsized":"2023-09-23T20:45:41.914221+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":80,"log_dups_size":0,"ondisk_log_size":80,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:30:57.793132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035115,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":395,"num_read_kb":1370,"num_write":66,"num_write_kb":1061,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":529,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-09-23T20:45:48.592208+0000","last_change":"2023-09-23T20:43:46.544563+0000","last_active":"2023-09-23T20:45:48.592208+0000","last_peered":"2023-09-23T20:45:48.592208+0000","last_clean":"2023-09-23T20:45:48.592208+0000","last_became_active":"2023-09-23T20:43:46.543958+0000","last_became_peered":"2023-09-23T20:43:46.543958+0000","last_unstale":"2023-09-23T20:45:48.592208+0000","last_undegraded":"2023-09-23T20:45:48.592208+0000","last_fullsized":"2023-09-23T20:45:48.592208+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:46:36.921752+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":523,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-09-23T20:45:43.505630+0000","last_change":"2023-09-23T20:43:46.520386+0000","last_active":"2023-09-23T20:45:43.505630+0000","last_peered":"2023-09-23T20:45:43.505630+0000","last_clean":"2023-09-23T20:45:43.505630+0000","last_became_active":"2023-09-23T20:43:46.519757+0000","last_became_peered":"2023-09-23T20:43:46.519757+0000","last_unstale":"2023-09-23T20:45:43.505630+0000","last_undegraded":"2023-09-23T20:45:43.505630+0000","last_fullsized":"2023-09-23T20:45:43.505630+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:23:14.662734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"535'592","reported_seq":604,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-09-23T20:45:48.252570+0000","last_change":"2023-09-23T20:45:39.331637+0000","last_active":"2023-09-23T20:45:48.252570+0000","last_peered":"2023-09-23T20:45:48.252570+0000","last_clean":"2023-09-23T20:45:48.252570+0000","last_became_active":"2023-09-23T20:45:39.331378+0000","last_became_peered":"2023-09-23T20:45:39.331378+0000","last_unstale":"2023-09-23T20:45:48.252570+0000","last_undegraded":"2023-09-23T20:45:48.252570+0000","last_fullsized":"2023-09-23T20:45:48.252570+0000","mapping_epoch":531,"log_start":"0'0","ondisk_log_start":"0'0","created":531,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":592,"log_dups_size":0,"ondisk_log_size":592,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:43:34.423232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":38797312,"num_objects":592,"num_object_clones":0,"num_object_copies":1776,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":592,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":592,"num_write_kb":37888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":646,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:45:42.051962+0000","last_change":"2023-09-23T20:41:08.380084+0000","last_active":"2023-09-23T20:45:42.051962+0000","last_peered":"2023-09-23T20:45:42.051962+0000","last_clean":"2023-09-23T20:45:42.051962+0000","last_became_active":"2023-09-23T20:38:05.610881+0000","last_became_peered":"2023-09-23T20:38:05.610881+0000","last_unstale":"2023-09-23T20:45:42.051962+0000","last_undegraded":"2023-09-23T20:45:42.051962+0000","last_fullsized":"2023-09-23T20:45:42.051962+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:41:08.379959+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:41:08.379959+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:19:39.547923+0000","scrub_duration":0.0019158420000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"86'102","reported_seq":1129,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:45:41.914935+0000","last_change":"2023-09-23T20:32:36.314672+0000","last_active":"2023-09-23T20:45:41.914935+0000","last_peered":"2023-09-23T20:45:41.914935+0000","last_clean":"2023-09-23T20:45:41.914935+0000","last_became_active":"2023-09-23T20:32:36.314231+0000","last_became_peered":"2023-09-23T20:32:36.314231+0000","last_unstale":"2023-09-23T20:45:41.914935+0000","last_undegraded":"2023-09-23T20:45:41.914935+0000","last_fullsized":"2023-09-23T20:45:41.914935+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":102,"log_dups_size":0,"ondisk_log_size":102,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:11:40.514709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030656500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":963,"num_read_kb":1234,"num_write":60,"num_write_kb":225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":482,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:45:41.334605+0000","last_change":"2023-09-23T20:40:19.544384+0000","last_active":"2023-09-23T20:45:41.334605+0000","last_peered":"2023-09-23T20:45:41.334605+0000","last_clean":"2023-09-23T20:45:41.334605+0000","last_became_active":"2023-09-23T20:40:19.544079+0000","last_became_peered":"2023-09-23T20:40:19.544079+0000","last_unstale":"2023-09-23T20:45:41.334605+0000","last_undegraded":"2023-09-23T20:45:41.334605+0000","last_fullsized":"2023-09-23T20:45:41.334605+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:00:19.498631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":479,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-09-23T20:45:48.594277+0000","last_change":"2023-09-23T20:43:46.541173+0000","last_active":"2023-09-23T20:45:48.594277+0000","last_peered":"2023-09-23T20:45:48.594277+0000","last_clean":"2023-09-23T20:45:48.594277+0000","last_became_active":"2023-09-23T20:43:46.540726+0000","last_became_peered":"2023-09-23T20:43:46.540726+0000","last_unstale":"2023-09-23T20:45:48.594277+0000","last_undegraded":"2023-09-23T20:45:48.594277+0000","last_fullsized":"2023-09-23T20:45:48.594277+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:49:04.137885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.e","version":"536'1023","reported_seq":1036,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-09-23T20:45:50.702422+0000","last_change":"2023-09-23T20:45:39.330822+0000","last_active":"2023-09-23T20:45:50.702422+0000","last_peered":"2023-09-23T20:45:50.702422+0000","last_clean":"2023-09-23T20:45:50.702422+0000","last_became_active":"2023-09-23T20:45:39.330513+0000","last_became_peered":"2023-09-23T20:45:39.330513+0000","last_unstale":"2023-09-23T20:45:50.702422+0000","last_undegraded":"2023-09-23T20:45:50.702422+0000","last_fullsized":"2023-09-23T20:45:50.702422+0000","mapping_epoch":531,"log_start":"0'0","ondisk_log_start":"0'0","created":531,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":1023,"log_dups_size":0,"ondisk_log_size":1023,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:35:59.769584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":67043328,"num_objects":1023,"num_object_clones":0,"num_object_copies":3069,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1023,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1023,"num_write_kb":65472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":814,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-09-23T20:45:42.565482+0000","last_change":"2023-09-23T20:38:05.590361+0000","last_active":"2023-09-23T20:45:42.565482+0000","last_peered":"2023-09-23T20:45:42.565482+0000","last_clean":"2023-09-23T20:45:42.565482+0000","last_became_active":"2023-09-23T20:38:05.589985+0000","last_became_peered":"2023-09-23T20:38:05.589985+0000","last_unstale":"2023-09-23T20:45:42.565482+0000","last_undegraded":"2023-09-23T20:45:42.565482+0000","last_fullsized":"2023-09-23T20:45:42.565482+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:03.246271+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:03.246271+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:38:54.351576+0000","scrub_duration":0.0019016700000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":272,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:45:41.915170+0000","last_change":"2023-09-23T20:32:50.549695+0000","last_active":"2023-09-23T20:45:41.915170+0000","last_peered":"2023-09-23T20:45:41.915170+0000","last_clean":"2023-09-23T20:45:41.915170+0000","last_became_active":"2023-09-23T20:32:50.549431+0000","last_became_peered":"2023-09-23T20:32:50.549431+0000","last_unstale":"2023-09-23T20:45:41.915170+0000","last_undegraded":"2023-09-23T20:45:41.915170+0000","last_fullsized":"2023-09-23T20:45:41.915170+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:45:53.666569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"86'128","reported_seq":948,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:45:41.334675+0000","last_change":"2023-09-23T20:38:03.541765+0000","last_active":"2023-09-23T20:45:41.334675+0000","last_peered":"2023-09-23T20:45:41.334675+0000","last_clean":"2023-09-23T20:45:41.334675+0000","last_became_active":"2023-09-23T20:38:03.541277+0000","last_became_peered":"2023-09-23T20:38:03.541277+0000","last_unstale":"2023-09-23T20:45:41.334675+0000","last_undegraded":"2023-09-23T20:45:41.334675+0000","last_fullsized":"2023-09-23T20:45:41.334675+0000","mapping_epoch":404,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":405,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:20:53.834554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000293202,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":579,"num_read_kb":1473,"num_write":96,"num_write_kb":1062,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":317,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-09-23T20:45:44.432480+0000","last_change":"2023-09-23T20:37:43.582105+0000","last_active":"2023-09-23T20:45:44.432480+0000","last_peered":"2023-09-23T20:45:44.432480+0000","last_clean":"2023-09-23T20:45:44.432480+0000","last_became_active":"2023-09-23T20:37:43.581860+0000","last_became_peered":"2023-09-23T20:37:43.581860+0000","last_unstale":"2023-09-23T20:45:44.432480+0000","last_undegraded":"2023-09-23T20:45:44.432480+0000","last_fullsized":"2023-09-23T20:45:44.432480+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:18:38.765486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"535'608","reported_seq":620,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-09-23T20:45:48.241327+0000","last_change":"2023-09-23T20:45:39.381283+0000","last_active":"2023-09-23T20:45:48.241327+0000","last_peered":"2023-09-23T20:45:48.241327+0000","last_clean":"2023-09-23T20:45:48.241327+0000","last_became_active":"2023-09-23T20:45:39.380945+0000","last_became_peered":"2023-09-23T20:45:39.380945+0000","last_unstale":"2023-09-23T20:45:48.241327+0000","last_undegraded":"2023-09-23T20:45:48.241327+0000","last_fullsized":"2023-09-23T20:45:48.241327+0000","mapping_epoch":531,"log_start":"0'0","ondisk_log_start":"0'0","created":531,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":608,"log_dups_size":0,"ondisk_log_size":608,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:04:01.529754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":39845888,"num_objects":608,"num_object_clones":0,"num_object_copies":1824,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":608,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":608,"num_write_kb":38912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.4","version":"90'161","reported_seq":750,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-09-23T20:45:43.509067+0000","last_change":"2023-09-23T20:40:22.427435+0000","last_active":"2023-09-23T20:45:43.509067+0000","last_peered":"2023-09-23T20:45:43.509067+0000","last_clean":"2023-09-23T20:45:43.509067+0000","last_became_active":"2023-09-23T20:40:22.427051+0000","last_became_peered":"2023-09-23T20:40:22.427051+0000","last_unstale":"2023-09-23T20:45:43.509067+0000","last_undegraded":"2023-09-23T20:45:43.509067+0000","last_fullsized":"2023-09-23T20:45:43.509067+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:34:54.504554+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:34:54.504554+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:42.001652+0000","scrub_duration":0.011176864,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"86'117","reported_seq":843,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:45:42.052190+0000","last_change":"2023-09-23T20:44:18.635224+0000","last_active":"2023-09-23T20:45:42.052190+0000","last_peered":"2023-09-23T20:45:42.052190+0000","last_clean":"2023-09-23T20:45:42.052190+0000","last_became_active":"2023-09-23T20:44:18.634703+0000","last_became_peered":"2023-09-23T20:44:18.634703+0000","last_unstale":"2023-09-23T20:45:42.052190+0000","last_undegraded":"2023-09-23T20:45:42.052190+0000","last_fullsized":"2023-09-23T20:45:42.052190+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:59:38.030626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046891400000000002,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":893,"num_read_kb":922,"num_write":82,"num_write_kb":4193,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":16777216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":523,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-09-23T20:45:43.505738+0000","last_change":"2023-09-23T20:43:46.521505+0000","last_active":"2023-09-23T20:45:43.505738+0000","last_peered":"2023-09-23T20:45:43.505738+0000","last_clean":"2023-09-23T20:45:43.505738+0000","last_became_active":"2023-09-23T20:43:46.520727+0000","last_became_peered":"2023-09-23T20:43:46.520727+0000","last_unstale":"2023-09-23T20:45:43.505738+0000","last_undegraded":"2023-09-23T20:45:43.505738+0000","last_fullsized":"2023-09-23T20:45:43.505738+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:17:06.164741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":273,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-09-23T20:45:42.564831+0000","last_change":"2023-09-23T20:40:19.530602+0000","last_active":"2023-09-23T20:45:42.564831+0000","last_peered":"2023-09-23T20:45:42.564831+0000","last_clean":"2023-09-23T20:45:42.564831+0000","last_became_active":"2023-09-23T20:40:19.530256+0000","last_became_peered":"2023-09-23T20:40:19.530256+0000","last_unstale":"2023-09-23T20:45:42.564831+0000","last_undegraded":"2023-09-23T20:45:42.564831+0000","last_fullsized":"2023-09-23T20:45:42.564831+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:18:45.658867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"90'161","reported_seq":792,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-09-23T20:45:43.506628+0000","last_change":"2023-09-23T20:43:46.544750+0000","last_active":"2023-09-23T20:45:43.506628+0000","last_peered":"2023-09-23T20:45:43.506628+0000","last_clean":"2023-09-23T20:45:43.506628+0000","last_became_active":"2023-09-23T20:43:46.544395+0000","last_became_peered":"2023-09-23T20:43:46.544395+0000","last_unstale":"2023-09-23T20:45:43.506628+0000","last_undegraded":"2023-09-23T20:45:43.506628+0000","last_fullsized":"2023-09-23T20:45:43.506628+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:06.487351+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:06.487351+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:56.866168+0000","scrub_duration":0.001964379,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":490,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-09-23T20:45:43.507590+0000","last_change":"2023-09-23T20:43:46.541379+0000","last_active":"2023-09-23T20:45:43.507590+0000","last_peered":"2023-09-23T20:45:43.507590+0000","last_clean":"2023-09-23T20:45:43.507590+0000","last_became_active":"2023-09-23T20:43:46.540732+0000","last_became_peered":"2023-09-23T20:43:46.540732+0000","last_unstale":"2023-09-23T20:45:43.507590+0000","last_undegraded":"2023-09-23T20:45:43.507590+0000","last_fullsized":"2023-09-23T20:45:43.507590+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:31:40.605772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":392,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:45:43.106046+0000","last_change":"2023-09-23T20:29:45.538314+0000","last_active":"2023-09-23T20:45:43.106046+0000","last_peered":"2023-09-23T20:45:43.106046+0000","last_clean":"2023-09-23T20:45:43.106046+0000","last_became_active":"2023-09-23T20:29:45.536847+0000","last_became_peered":"2023-09-23T20:29:45.536847+0000","last_unstale":"2023-09-23T20:45:43.106046+0000","last_undegraded":"2023-09-23T20:45:43.106046+0000","last_fullsized":"2023-09-23T20:45:43.106046+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:51.948721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"90'161","reported_seq":610,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-09-23T20:45:44.432573+0000","last_change":"2023-09-23T20:41:06.499511+0000","last_active":"2023-09-23T20:45:44.432573+0000","last_peered":"2023-09-23T20:45:44.432573+0000","last_clean":"2023-09-23T20:45:44.432573+0000","last_became_active":"2023-09-23T20:38:05.601874+0000","last_became_peered":"2023-09-23T20:38:05.601874+0000","last_unstale":"2023-09-23T20:45:44.432573+0000","last_undegraded":"2023-09-23T20:45:44.432573+0000","last_fullsized":"2023-09-23T20:45:44.432573+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:41:06.499396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:41:06.499396+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:31:34.059077+0000","scrub_duration":0.0019670310000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":428,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-09-23T20:45:42.565315+0000","last_change":"2023-09-23T20:43:16.758761+0000","last_active":"2023-09-23T20:45:42.565315+0000","last_peered":"2023-09-23T20:45:42.565315+0000","last_clean":"2023-09-23T20:45:42.565315+0000","last_became_active":"2023-09-23T20:43:16.758407+0000","last_became_peered":"2023-09-23T20:43:16.758407+0000","last_unstale":"2023-09-23T20:45:42.565315+0000","last_undegraded":"2023-09-23T20:45:42.565315+0000","last_fullsized":"2023-09-23T20:45:42.565315+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:59:27.070685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":192,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-09-23T20:45:42.564911+0000","last_change":"2023-09-23T20:34:59.987575+0000","last_active":"2023-09-23T20:45:42.564911+0000","last_peered":"2023-09-23T20:45:42.564911+0000","last_clean":"2023-09-23T20:45:42.564911+0000","last_became_active":"2023-09-23T20:34:59.987229+0000","last_became_peered":"2023-09-23T20:34:59.987229+0000","last_unstale":"2023-09-23T20:45:42.564911+0000","last_undegraded":"2023-09-23T20:45:42.564911+0000","last_fullsized":"2023-09-23T20:45:42.564911+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:34:01.820159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":436,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-09-23T20:45:42.565332+0000","last_change":"2023-09-23T20:43:16.759691+0000","last_active":"2023-09-23T20:45:42.565332+0000","last_peered":"2023-09-23T20:45:42.565332+0000","last_clean":"2023-09-23T20:45:42.565332+0000","last_became_active":"2023-09-23T20:43:16.759309+0000","last_became_peered":"2023-09-23T20:43:16.759309+0000","last_unstale":"2023-09-23T20:45:42.565332+0000","last_undegraded":"2023-09-23T20:45:42.565332+0000","last_fullsized":"2023-09-23T20:45:42.565332+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:58:52.893019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":192,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-09-23T20:45:42.564780+0000","last_change":"2023-09-23T20:34:59.987416+0000","last_active":"2023-09-23T20:45:42.564780+0000","last_peered":"2023-09-23T20:45:42.564780+0000","last_clean":"2023-09-23T20:45:42.564780+0000","last_became_active":"2023-09-23T20:34:59.987117+0000","last_became_peered":"2023-09-23T20:34:59.987117+0000","last_unstale":"2023-09-23T20:45:42.564780+0000","last_undegraded":"2023-09-23T20:45:42.564780+0000","last_fullsized":"2023-09-23T20:45:42.564780+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:36:16.318623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"90'161","reported_seq":790,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:45:41.914088+0000","last_change":"2023-09-23T20:38:05.591992+0000","last_active":"2023-09-23T20:45:41.914088+0000","last_peered":"2023-09-23T20:45:41.914088+0000","last_clean":"2023-09-23T20:45:41.914088+0000","last_became_active":"2023-09-23T20:38:05.591701+0000","last_became_peered":"2023-09-23T20:38:05.591701+0000","last_unstale":"2023-09-23T20:45:41.914088+0000","last_undegraded":"2023-09-23T20:45:41.914088+0000","last_fullsized":"2023-09-23T20:45:41.914088+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:02.384133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:02.384133+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:41.722542+0000","scrub_duration":0.001769098,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":359,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-09-23T20:45:44.432590+0000","last_change":"2023-09-23T20:38:02.542719+0000","last_active":"2023-09-23T20:45:44.432590+0000","last_peered":"2023-09-23T20:45:44.432590+0000","last_clean":"2023-09-23T20:45:44.432590+0000","last_became_active":"2023-09-23T20:38:02.542385+0000","last_became_peered":"2023-09-23T20:38:02.542385+0000","last_unstale":"2023-09-23T20:45:44.432590+0000","last_undegraded":"2023-09-23T20:45:44.432590+0000","last_fullsized":"2023-09-23T20:45:44.432590+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:36:23.673757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":325,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-09-23T20:45:43.507796+0000","last_change":"2023-09-23T20:37:43.584253+0000","last_active":"2023-09-23T20:45:43.507796+0000","last_peered":"2023-09-23T20:45:43.507796+0000","last_clean":"2023-09-23T20:45:43.507796+0000","last_became_active":"2023-09-23T20:37:43.580069+0000","last_became_peered":"2023-09-23T20:37:43.580069+0000","last_unstale":"2023-09-23T20:45:43.507796+0000","last_undegraded":"2023-09-23T20:45:43.507796+0000","last_fullsized":"2023-09-23T20:45:43.507796+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:39:11.482581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"90'161","reported_seq":629,"reported_epoch":534,"state":"active+clean","last_fresh":"2023-09-23T20:45:41.334664+0000","last_change":"2023-09-23T20:36:01.505098+0000","last_active":"2023-09-23T20:45:41.334664+0000","last_peered":"2023-09-23T20:45:41.334664+0000","last_clean":"2023-09-23T20:45:41.334664+0000","last_became_active":"2023-09-23T20:35:26.644707+0000","last_became_peered":"2023-09-23T20:35:26.644707+0000","last_unstale":"2023-09-23T20:45:41.334664+0000","last_undegraded":"2023-09-23T20:45:41.334664+0000","last_fullsized":"2023-09-23T20:45:41.334664+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:01.504973+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:01.504973+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:02:10.880809+0000","scrub_duration":0.00202344,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":569,"reported_epoch":536,"state":"active+clean","last_fresh":"2023-09-23T20:45:48.592970+0000","last_change":"2023-09-23T20:43:46.511657+0000","last_active":"2023-09-23T20:45:48.592970+0000","last_peered":"2023-09-23T20:45:48.592970+0000","last_clean":"2023-09-23T20:45:48.592970+0000","last_became_active":"2023-09-23T20:43:46.511326+0000","last_became_peered":"2023-09-23T20:43:46.511326+0000","last_unstale":"2023-09-23T20:45:48.592970+0000","last_undegraded":"2023-09-23T20:45:48.592970+0000","last_fullsized":"2023-09-23T20:45:48.592970+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:36:46.139261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3324,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":253,"reported_epoch":535,"state":"active+clean","last_fresh":"2023-09-23T20:45:42.564619+0000","last_change":"2023-09-23T20:38:02.538333+0000","last_active":"2023-09-23T20:45:42.564619+0000","last_peered":"2023-09-23T20:45:42.564619+0000","last_clean":"2023-09-23T20:45:42.564619+0000","last_became_active":"2023-09-23T20:38:02.538008+0000","last_became_peered":"2023-09-23T20:38:02.538008+0000","last_unstale":"2023-09-23T20:45:42.564619+0000","last_undegraded":"2023-09-23T20:45:42.564619+0000","last_fullsized":"2023-09-23T20:45:42.564619+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:02:38.789530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":776470528,"num_objects":11848,"num_object_clones":0,"num_object_copies":35544,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":11848,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11848,"num_write_kb":758272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2597257216,"data_stored":2597257216,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":11848,"ondisk_log_size":11848,"up":48,"acting":48,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":90,"num_bytes_recovered":18486,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26691,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5983,"num_read_kb":13200,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":84,"num_bytes_recovered":16777216,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6082,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201692,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":625900,"kb_used_data":100836,"kb_used_omap":0,"kb_used_meta":525056,"kb_avail":93119252,"statfs":{"total":95995035648,"available":95354114048,"internally_reserved":0,"allocated":103256064,"data_stored":101731517,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":537657344},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:44:59 2023","interfaces":[{"interface":"back","average":{"1min":2.8599999999999999,"5min":2.0510000000000002,"15min":2.1640000000000001},"min":{"1min":0.24199999999999999,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":10.683999999999999,"5min":15.1,"15min":18.315000000000001},"last":0.66500000000000004},{"interface":"front","average":{"1min":2.282,"5min":1.8799999999999999,"15min":1.897},"min":{"1min":0.31900000000000001,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":11.045999999999999,"5min":15.327999999999999,"15min":19.355},"last":0.44}]},{"osd":1,"last update":"Sat Sep 23 20:45:08 2023","interfaces":[{"interface":"back","average":{"1min":1.002,"5min":1.468,"15min":2.0539999999999998},"min":{"1min":0.26100000000000001,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":2.5609999999999999,"5min":15.131,"15min":49.965000000000003},"last":0.63100000000000001},{"interface":"front","average":{"1min":2.7730000000000001,"5min":2.6419999999999999,"15min":2.3919999999999999},"min":{"1min":0.39700000000000002,"5min":0.26700000000000002,"15min":0.248},"max":{"1min":11.276,"5min":15.18,"15min":53.451999999999998},"last":0.40000000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:44:59 2023","interfaces":[{"interface":"back","average":{"1min":2.4889999999999999,"5min":2.3119999999999998,"15min":2.286},"min":{"1min":0.30499999999999999,"5min":0.27900000000000003,"15min":0.27800000000000002},"max":{"1min":8.0020000000000007,"5min":15.087,"15min":16.965},"last":0.496},{"interface":"front","average":{"1min":2.6520000000000001,"5min":2.0230000000000001,"15min":2.4670000000000001},"min":{"1min":0.371,"5min":0.30299999999999999,"15min":0.27400000000000002},"max":{"1min":13.5,"5min":15.193,"15min":23.494},"last":0.77300000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:45:30 2023","interfaces":[{"interface":"back","average":{"1min":1.3520000000000001,"5min":1.821,"15min":2.306},"min":{"1min":0.39600000000000002,"5min":0.32500000000000001,"15min":0.311},"max":{"1min":5.9169999999999998,"5min":15.115,"15min":23.352},"last":0.45900000000000002},{"interface":"front","average":{"1min":1.6160000000000001,"5min":2.2349999999999999,"15min":2.4860000000000002},"min":{"1min":0.43099999999999999,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":5.4669999999999996,"5min":15.144,"15min":20.969999999999999},"last":0.54300000000000004}]},{"osd":4,"last update":"Sat Sep 23 20:45:30 2023","interfaces":[{"interface":"back","average":{"1min":0.76100000000000001,"5min":1.3340000000000001,"15min":1.079},"min":{"1min":0.33000000000000002,"5min":0.27000000000000002,"15min":0.23200000000000001},"max":{"1min":2.2949999999999999,"5min":12.260999999999999,"15min":18.364999999999998},"last":0.70699999999999996},{"interface":"front","average":{"1min":1.7170000000000001,"5min":1.7170000000000001,"15min":1.3100000000000001},"min":{"1min":0.376,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":7.2469999999999999,"5min":7.2469999999999999,"15min":23.617000000000001},"last":0.65400000000000003}]},{"osd":5,"last update":"Sat Sep 23 20:45:30 2023","interfaces":[{"interface":"back","average":{"1min":1.2549999999999999,"5min":1.6399999999999999,"15min":1.1639999999999999},"min":{"1min":0.33500000000000002,"5min":0.30499999999999999,"15min":0.26600000000000001},"max":{"1min":5.9619999999999997,"5min":15.005000000000001,"15min":15.005000000000001},"last":0.52900000000000003},{"interface":"front","average":{"1min":2.024,"5min":1.8080000000000001,"15min":1.252},"min":{"1min":0.377,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":10.641999999999999,"5min":12.731,"15min":18.390000000000001},"last":0.58799999999999997}]},{"osd":6,"last update":"Sat Sep 23 20:45:30 2023","interfaces":[{"interface":"back","average":{"1min":1.337,"5min":1.851,"15min":1.321},"min":{"1min":0.317,"5min":0.317,"15min":0.30399999999999999},"max":{"1min":5.4269999999999996,"5min":15.038,"15min":18.417000000000002},"last":0.68600000000000005},{"interface":"front","average":{"1min":1.3149999999999999,"5min":1.696,"15min":1.2509999999999999},"min":{"1min":0.36599999999999999,"5min":0.36499999999999999,"15min":0.318},"max":{"1min":5.5199999999999996,"5min":15.369999999999999,"15min":15.369999999999999},"last":0.57699999999999996}]}]},{"osd":6,"up_from":47,"seq":201863463331,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":715104,"kb_used_data":267864,"kb_used_omap":0,"kb_used_meta":447232,"kb_avail":93030048,"statfs":{"total":95995035648,"available":95262769152,"internally_reserved":0,"allocated":274292736,"data_stored":272779806,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":457965568},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,3],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:44:55 2023","interfaces":[{"interface":"back","average":{"1min":1.883,"5min":1.5720000000000001,"15min":1.986},"min":{"1min":0.219,"5min":0.219,"15min":0.219},"max":{"1min":7.7720000000000002,"5min":12.552,"15min":53.701999999999998},"last":0.69899999999999995},{"interface":"front","average":{"1min":2.343,"5min":1.7430000000000001,"15min":2.3730000000000002},"min":{"1min":0.313,"5min":0.23300000000000001,"15min":0.22700000000000001},"max":{"1min":10.895,"5min":10.895,"15min":60.328000000000003},"last":0.45500000000000002}]},{"osd":1,"last update":"Sat Sep 23 20:45:17 2023","interfaces":[{"interface":"back","average":{"1min":1.9390000000000001,"5min":1.8999999999999999,"15min":2.077},"min":{"1min":0.28899999999999998,"5min":0.249,"15min":0.249},"max":{"1min":7.9189999999999996,"5min":9.2520000000000007,"15min":64.274000000000001},"last":1.3160000000000001},{"interface":"front","average":{"1min":2.198,"5min":2.0019999999999998,"15min":1.988},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.28000000000000003},"max":{"1min":7.8849999999999998,"5min":17.786000000000001,"15min":34.490000000000002},"last":0.81100000000000005}]},{"osd":2,"last update":"Sat Sep 23 20:44:55 2023","interfaces":[{"interface":"back","average":{"1min":2.5409999999999999,"5min":2.2370000000000001,"15min":2.1739999999999999},"min":{"1min":0.34899999999999998,"5min":0.27800000000000002,"15min":0.27700000000000002},"max":{"1min":6.6130000000000004,"5min":9.9049999999999994,"15min":34.317999999999998},"last":1.347},{"interface":"front","average":{"1min":2.3079999999999998,"5min":2.0129999999999999,"15min":2.069},"min":{"1min":0.373,"5min":0.311,"15min":0.30399999999999999},"max":{"1min":6.5620000000000003,"5min":9.9269999999999996,"15min":34.292999999999999},"last":0.60599999999999998}]},{"osd":3,"last update":"Sat Sep 23 20:45:48 2023","interfaces":[{"interface":"back","average":{"1min":0.99199999999999999,"5min":1.821,"15min":1.911},"min":{"1min":0.45200000000000001,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":2.8090000000000002,"5min":11.664,"15min":11.664},"last":1.4339999999999999},{"interface":"front","average":{"1min":1.153,"5min":1.8759999999999999,"15min":2.0859999999999999},"min":{"1min":0.378,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":2.8969999999999998,"5min":11.808,"15min":12.875999999999999},"last":0.73499999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:45:17 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.93100000000000005,"15min":0.78800000000000003},"min":{"1min":0.29499999999999998,"5min":0.25600000000000001,"15min":0.251},"max":{"1min":1.4330000000000001,"5min":6.5800000000000001,"15min":23.904},"last":0.78500000000000003},{"interface":"front","average":{"1min":0.89700000000000002,"5min":1.032,"15min":1.022},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.27000000000000002},"max":{"1min":2.7839999999999998,"5min":5.6260000000000003,"15min":11.515000000000001},"last":0.34399999999999997}]},{"osd":5,"last update":"Sat Sep 23 20:45:17 2023","interfaces":[{"interface":"back","average":{"1min":1.5489999999999999,"5min":1.0369999999999999,"15min":0.85099999999999998},"min":{"1min":0.26400000000000001,"5min":0.253,"15min":0.253},"max":{"1min":7.8529999999999998,"5min":7.8529999999999998,"15min":7.8529999999999998},"last":0.65700000000000003},{"interface":"front","average":{"1min":1.2549999999999999,"5min":1.488,"15min":1.103},"min":{"1min":0.34699999999999998,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":5.141,"5min":7.1319999999999997,"15min":7.7300000000000004},"last":0.59299999999999997}]},{"osd":7,"last update":"Sat Sep 23 20:45:48 2023","interfaces":[{"interface":"back","average":{"1min":0.68700000000000006,"5min":0.99099999999999999,"15min":0.85699999999999998},"min":{"1min":0.32200000000000001,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":1.631,"5min":8.8350000000000009,"15min":8.8350000000000009},"last":0.746},{"interface":"front","average":{"1min":0.79500000000000004,"5min":1.625,"15min":1.204},"min":{"1min":0.35999999999999999,"5min":0.34699999999999998,"15min":0.30299999999999999},"max":{"1min":2.8300000000000001,"5min":7.1929999999999996,"15min":8.8119999999999994},"last":0.623}]}]},{"osd":1,"up_from":266,"seq":1142461300930,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":834000,"kb_used_data":141704,"kb_used_omap":8,"kb_used_meta":692279,"kb_avail":92911152,"statfs":{"total":95995035648,"available":95141019648,"internally_reserved":0,"allocated":145104896,"data_stored":142383959,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8206,"internal_metadata":708894706},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:45:20 2023","interfaces":[{"interface":"back","average":{"1min":2.8450000000000002,"5min":2.5009999999999999,"15min":2.5670000000000002},"min":{"1min":0.221,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":10.289,"5min":16.292000000000002,"15min":52.689999999999998},"last":0.60899999999999999},{"interface":"front","average":{"1min":4.8440000000000003,"5min":2.7829999999999999,"15min":3.1280000000000001},"min":{"1min":0.27500000000000002,"5min":0.20100000000000001,"15min":0.19900000000000001},"max":{"1min":20.346,"5min":20.346,"15min":52.662999999999997},"last":0.70399999999999996}]},{"osd":2,"last update":"Sat Sep 23 20:45:20 2023","interfaces":[{"interface":"back","average":{"1min":4.2670000000000003,"5min":2.722,"15min":3.2799999999999998},"min":{"1min":0.23699999999999999,"5min":0.216,"15min":0.20899999999999999},"max":{"1min":19.026,"5min":19.026,"15min":32.255000000000003},"last":1.5609999999999999},{"interface":"front","average":{"1min":1.873,"5min":1.974,"15min":2.2770000000000001},"min":{"1min":0.32100000000000001,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":8.9760000000000009,"5min":10.041,"15min":13.404999999999999},"last":0.72599999999999998}]},{"osd":3,"last update":"Sat Sep 23 20:45:20 2023","interfaces":[{"interface":"back","average":{"1min":2.681,"5min":2.2919999999999998,"15min":3.0179999999999998},"min":{"1min":0.34999999999999998,"5min":0.29099999999999998,"15min":0.247},"max":{"1min":9.0299999999999994,"5min":16.245999999999999,"15min":80.745999999999995},"last":0.64700000000000002},{"interface":"front","average":{"1min":3.7890000000000001,"5min":2.8719999999999999,"15min":3.274},"min":{"1min":0.33600000000000002,"5min":0.26400000000000001,"15min":0.22700000000000001},"max":{"1min":11.534000000000001,"5min":13.497999999999999,"15min":82.988},"last":0.98599999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:45:20 2023","interfaces":[{"interface":"back","average":{"1min":3.7389999999999999,"5min":2.5350000000000001,"15min":2.3610000000000002},"min":{"1min":0.36199999999999999,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":11.791,"5min":20.634,"15min":24.327999999999999},"last":0.55000000000000004},{"interface":"front","average":{"1min":2.1589999999999998,"5min":2.0030000000000001,"15min":1.7569999999999999},"min":{"1min":0.44700000000000001,"5min":0.32600000000000001,"15min":0.311},"max":{"1min":10.468999999999999,"5min":20.835999999999999,"15min":20.835999999999999},"last":0.68400000000000005}]},{"osd":5,"last update":"Sat Sep 23 20:45:20 2023","interfaces":[{"interface":"back","average":{"1min":3.1760000000000002,"5min":2.6680000000000001,"15min":2.3279999999999998},"min":{"1min":0.40000000000000002,"5min":0.308,"15min":0.308},"max":{"1min":7.9290000000000003,"5min":14.366,"15min":33.401000000000003},"last":0.82099999999999995},{"interface":"front","average":{"1min":3.871,"5min":2.6099999999999999,"15min":2.601},"min":{"1min":0.40699999999999997,"5min":0.33100000000000002,"15min":0.32500000000000001},"max":{"1min":11.750999999999999,"5min":11.750999999999999,"15min":33.417000000000002},"last":0.65900000000000003}]},{"osd":6,"last update":"Sat Sep 23 20:45:20 2023","interfaces":[{"interface":"back","average":{"1min":2.8159999999999998,"5min":2.5270000000000001,"15min":2.5299999999999998},"min":{"1min":0.42399999999999999,"5min":0.35799999999999998,"15min":0.35699999999999998},"max":{"1min":7.9000000000000004,"5min":14.414999999999999,"15min":72.682000000000002},"last":0.755},{"interface":"front","average":{"1min":4.4340000000000002,"5min":2.8599999999999999,"15min":2.8530000000000002},"min":{"1min":0.45100000000000001,"5min":0.317,"15min":0.317},"max":{"1min":11.832000000000001,"5min":11.832000000000001,"15min":72.563999999999993},"last":0.58799999999999997}]},{"osd":7,"last update":"Sat Sep 23 20:45:20 2023","interfaces":[{"interface":"back","average":{"1min":3.3300000000000001,"5min":2.6739999999999999,"15min":2.3730000000000002},"min":{"1min":0.40899999999999997,"5min":0.37,"15min":0.37},"max":{"1min":8.7189999999999994,"5min":16.359000000000002,"15min":18.437000000000001},"last":0.79000000000000004},{"interface":"front","average":{"1min":4.4820000000000002,"5min":2.8799999999999999,"15min":2.633},"min":{"1min":0.49399999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":11.864000000000001,"5min":11.864000000000001,"15min":24.524000000000001},"last":0.625}]}]},{"osd":0,"up_from":94,"seq":403726926133,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1428972,"kb_used_data":671588,"kb_used_omap":7,"kb_used_meta":757368,"kb_avail":92316180,"statfs":{"total":95995035648,"available":94531768320,"internally_reserved":0,"allocated":687706112,"data_stored":685142796,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":775544849},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,2,0,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:45:24 2023","interfaces":[{"interface":"back","average":{"1min":1.756,"5min":2.419,"15min":2.5790000000000002},"min":{"1min":0.19400000000000001,"5min":0.184,"15min":0.17199999999999999},"max":{"1min":9.1509999999999998,"5min":24.010000000000002,"15min":42.000999999999998},"last":0.20799999999999999},{"interface":"front","average":{"1min":2.3740000000000001,"5min":2.7669999999999999,"15min":2.774},"min":{"1min":0.23799999999999999,"5min":0.22700000000000001,"15min":0.20399999999999999},"max":{"1min":11.292999999999999,"5min":18.097000000000001,"15min":23.036000000000001},"last":3.875}]},{"osd":2,"last update":"Sat Sep 23 20:45:01 2023","interfaces":[{"interface":"back","average":{"1min":3.242,"5min":3.052,"15min":3.085},"min":{"1min":0.27000000000000002,"5min":0.248,"15min":0.185},"max":{"1min":15.108000000000001,"5min":48.841000000000001,"15min":77.448999999999998},"last":0.99399999999999999},{"interface":"front","average":{"1min":3.5419999999999998,"5min":2.722,"15min":3.0470000000000002},"min":{"1min":0.28599999999999998,"5min":0.22,"15min":0.188},"max":{"1min":17.013000000000002,"5min":46.183,"15min":81.724999999999994},"last":5.423}]},{"osd":3,"last update":"Sat Sep 23 20:45:35 2023","interfaces":[{"interface":"back","average":{"1min":2.7669999999999999,"5min":2.802,"15min":3.2120000000000002},"min":{"1min":0.28499999999999998,"5min":0.248,"15min":0.23400000000000001},"max":{"1min":10.406000000000001,"5min":15.048,"15min":81.400999999999996},"last":4.0369999999999999},{"interface":"front","average":{"1min":2.2669999999999999,"5min":2.1930000000000001,"15min":3.3690000000000002},"min":{"1min":0.29499999999999998,"5min":0.23799999999999999,"15min":0.21199999999999999},"max":{"1min":9.2270000000000003,"5min":10.739000000000001,"15min":81.137},"last":2.5150000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:45:01 2023","interfaces":[{"interface":"back","average":{"1min":1.968,"5min":2.052,"15min":1.996},"min":{"1min":0.35999999999999999,"5min":0.32000000000000001,"15min":0.25900000000000001},"max":{"1min":10.368,"5min":10.368,"15min":12.587999999999999},"last":0.58299999999999996},{"interface":"front","average":{"1min":1.8049999999999999,"5min":1.9650000000000001,"15min":1.9099999999999999},"min":{"1min":0.34599999999999997,"5min":0.28899999999999998,"15min":0.27400000000000002},"max":{"1min":7.6740000000000004,"5min":9.8480000000000008,"15min":23.001000000000001},"last":0.313}]},{"osd":5,"last update":"Sat Sep 23 20:45:01 2023","interfaces":[{"interface":"back","average":{"1min":2.0819999999999999,"5min":2.4590000000000001,"15min":2.0880000000000001},"min":{"1min":0.308,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":7.7290000000000001,"5min":10.239000000000001,"15min":22.969999999999999},"last":0.51000000000000001},{"interface":"front","average":{"1min":2.964,"5min":2.496,"15min":2.3780000000000001},"min":{"1min":0.442,"5min":0.34300000000000003,"15min":0.25600000000000001},"max":{"1min":12.601000000000001,"5min":12.929,"15min":26.228000000000002},"last":0.46400000000000002}]},{"osd":6,"last update":"Sat Sep 23 20:45:01 2023","interfaces":[{"interface":"back","average":{"1min":2.0760000000000001,"5min":2.375,"15min":2.1720000000000002},"min":{"1min":0.39100000000000001,"5min":0.371,"15min":0.32300000000000001},"max":{"1min":7.7969999999999997,"5min":24.056000000000001,"15min":24.056000000000001},"last":0.374},{"interface":"front","average":{"1min":3.153,"5min":2.0880000000000001,"15min":2.2309999999999999},"min":{"1min":0.46999999999999997,"5min":0.34100000000000003,"15min":0.29399999999999998},"max":{"1min":12.698,"5min":12.698,"15min":26.268000000000001},"last":0.41599999999999998}]},{"osd":7,"last update":"Sat Sep 23 20:45:01 2023","interfaces":[{"interface":"back","average":{"1min":2.9910000000000001,"5min":2.3580000000000001,"15min":2.3860000000000001},"min":{"1min":0.497,"5min":0.36799999999999999,"15min":0.32200000000000001},"max":{"1min":12.651999999999999,"5min":12.651999999999999,"15min":26.177},"last":0.495},{"interface":"front","average":{"1min":2.4729999999999999,"5min":2.6389999999999998,"15min":2.3799999999999999},"min":{"1min":0.40600000000000003,"5min":0.40400000000000003,"15min":0.34200000000000003},"max":{"1min":10.32,"5min":15.006,"15min":22.422999999999998},"last":0.54800000000000004}]}]},{"osd":2,"up_from":104,"seq":446676599080,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":925668,"kb_used_data":175708,"kb_used_omap":11,"kb_used_meta":749940,"kb_avail":92819484,"statfs":{"total":95995035648,"available":95047151616,"internally_reserved":0,"allocated":179924992,"data_stored":177359543,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11878,"internal_metadata":767938970},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:45:09 2023","interfaces":[{"interface":"back","average":{"1min":1.696,"5min":1.2989999999999999,"15min":2.456},"min":{"1min":0.20499999999999999,"5min":0.17299999999999999,"15min":0.17299999999999999},"max":{"1min":7.4480000000000004,"5min":11.07,"15min":82.424999999999997},"last":0.44600000000000001},{"interface":"front","average":{"1min":2.972,"5min":2.1480000000000001,"15min":2.8140000000000001},"min":{"1min":0.26100000000000001,"5min":0.23599999999999999,"15min":0.19},"max":{"1min":11.244,"5min":16.678999999999998,"15min":82.524000000000001},"last":0.56899999999999995}]},{"osd":1,"last update":"Sat Sep 23 20:45:09 2023","interfaces":[{"interface":"back","average":{"1min":3.4609999999999999,"5min":2.218,"15min":2.746},"min":{"1min":0.314,"5min":0.248,"15min":0.221},"max":{"1min":12.797000000000001,"5min":12.981,"15min":46.405000000000001},"last":0.41399999999999998},{"interface":"front","average":{"1min":2.5720000000000001,"5min":2.125,"15min":2.6139999999999999},"min":{"1min":0.23200000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":13.215999999999999,"5min":13.215999999999999,"15min":46.459000000000003},"last":0.52500000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:45:33 2023","interfaces":[{"interface":"back","average":{"1min":2.52,"5min":2.1349999999999998,"15min":2.5670000000000002},"min":{"1min":0.25600000000000001,"5min":0.253,"15min":0.253},"max":{"1min":10.999000000000001,"5min":12.847,"15min":19.562999999999999},"last":0.57999999999999996},{"interface":"front","average":{"1min":1.129,"5min":1.292,"15min":2.141},"min":{"1min":0.32000000000000001,"5min":0.26500000000000001,"15min":0.26400000000000001},"max":{"1min":3.8610000000000002,"5min":11.375,"15min":30.704999999999998},"last":0.55800000000000005}]},{"osd":4,"last update":"Sat Sep 23 20:45:09 2023","interfaces":[{"interface":"back","average":{"1min":2.4929999999999999,"5min":1.8169999999999999,"15min":1.976},"min":{"1min":0.32000000000000001,"5min":0.29799999999999999,"15min":0.27500000000000002},"max":{"1min":11.141999999999999,"5min":12.025,"15min":16.797999999999998},"last":0.61399999999999999},{"interface":"front","average":{"1min":2.593,"5min":1.831,"15min":2.0840000000000001},"min":{"1min":0.40500000000000003,"5min":0.30599999999999999,"15min":0.30499999999999999},"max":{"1min":10.948,"5min":10.948,"15min":19.440999999999999},"last":0.69499999999999995}]},{"osd":5,"last update":"Sat Sep 23 20:45:09 2023","interfaces":[{"interface":"back","average":{"1min":2.6459999999999999,"5min":2.0470000000000002,"15min":2.1179999999999999},"min":{"1min":0.39600000000000002,"5min":0.311,"15min":0.311},"max":{"1min":11.183,"5min":12.07,"15min":16.766999999999999},"last":0.84299999999999997},{"interface":"front","average":{"1min":2.7559999999999998,"5min":1.831,"15min":2.2269999999999999},"min":{"1min":0.373,"5min":0.37,"15min":0.34799999999999998},"max":{"1min":11.061999999999999,"5min":11.061999999999999,"15min":19.513000000000002},"last":0.63}]},{"osd":6,"last update":"Sat Sep 23 20:45:09 2023","interfaces":[{"interface":"back","average":{"1min":2.7069999999999999,"5min":2.121,"15min":2.262},"min":{"1min":0.39100000000000001,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":11.031000000000001,"5min":14.047000000000001,"15min":19.486000000000001},"last":0.66500000000000004},{"interface":"front","average":{"1min":1.415,"5min":1.304,"15min":1.6930000000000001},"min":{"1min":0.44500000000000001,"5min":0.34200000000000003,"15min":0.316},"max":{"1min":8.5920000000000005,"5min":11.747,"15min":30.670999999999999},"last":0.71299999999999997}]},{"osd":7,"last update":"Sat Sep 23 20:45:09 2023","interfaces":[{"interface":"back","average":{"1min":1.341,"5min":1.347,"15min":1.714},"min":{"1min":0.41999999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":3.9220000000000002,"5min":8.4949999999999992,"15min":30.741},"last":0.77000000000000002},{"interface":"front","average":{"1min":2.71,"5min":2.2730000000000001,"15min":2.294},"min":{"1min":0.35799999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":11.218,"5min":12.099,"15min":16.719000000000001},"last":0.88100000000000001}]}]},{"osd":3,"up_from":109,"seq":468151435554,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1264156,"kb_used_data":264596,"kb_used_omap":4,"kb_used_meta":999547,"kb_avail":92480996,"statfs":{"total":95995035648,"available":94700539904,"internally_reserved":0,"allocated":270946304,"data_stored":268394196,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4514,"internal_metadata":1023536734},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:45:41 2023","interfaces":[{"interface":"back","average":{"1min":1.419,"5min":2.419,"15min":2.4649999999999999},"min":{"1min":0.249,"5min":0.20899999999999999,"15min":0.191},"max":{"1min":5.5339999999999998,"5min":22.32,"15min":22.32},"last":7.9050000000000002},{"interface":"front","average":{"1min":2.746,"5min":2.7330000000000001,"15min":2.5510000000000002},"min":{"1min":0.313,"5min":0.22900000000000001,"15min":0.19500000000000001},"max":{"1min":17.623000000000001,"5min":22.805,"15min":22.805},"last":2.1520000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:45:12 2023","interfaces":[{"interface":"back","average":{"1min":3.1800000000000002,"5min":2.669,"15min":2.2349999999999999},"min":{"1min":0.21099999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":15.177,"5min":17.529,"15min":17.529},"last":7.6509999999999998},{"interface":"front","average":{"1min":2.931,"5min":2.488,"15min":2.4279999999999999},"min":{"1min":0.29399999999999998,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":9.6600000000000001,"5min":14.19,"15min":32.603999999999999},"last":2.1160000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:45:41 2023","interfaces":[{"interface":"back","average":{"1min":2.9830000000000001,"5min":3.7170000000000001,"15min":3.0169999999999999},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":12.473000000000001,"5min":68.206999999999994,"15min":68.206999999999994},"last":0.45900000000000002},{"interface":"front","average":{"1min":2.5529999999999999,"5min":2.9199999999999999,"15min":2.7050000000000001},"min":{"1min":0.29999999999999999,"5min":0.245,"15min":0.245},"max":{"1min":17.919,"5min":67.736999999999995,"15min":67.736999999999995},"last":0.51200000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:45:41 2023","interfaces":[{"interface":"back","average":{"1min":1.5109999999999999,"5min":2.1459999999999999,"15min":1.7350000000000001},"min":{"1min":0.41599999999999998,"5min":0.317,"15min":0.30499999999999999},"max":{"1min":5.468,"5min":22.157,"15min":22.157},"last":0.52700000000000002},{"interface":"front","average":{"1min":1.407,"5min":2.2829999999999999,"15min":2.2650000000000001},"min":{"1min":0.372,"5min":0.35399999999999998,"15min":0.29399999999999998},"max":{"1min":4.8339999999999996,"5min":15.868,"15min":19.635000000000002},"last":0.47999999999999998}]},{"osd":5,"last update":"Sat Sep 23 20:45:41 2023","interfaces":[{"interface":"back","average":{"1min":1.349,"5min":2.4020000000000001,"15min":1.927},"min":{"1min":0.45500000000000002,"5min":0.33800000000000002,"15min":0.32200000000000001},"max":{"1min":5.5019999999999998,"5min":22.289999999999999,"15min":22.289999999999999},"last":0.57799999999999996},{"interface":"front","average":{"1min":2.702,"5min":2.0470000000000002,"15min":1.8819999999999999},"min":{"1min":0.47699999999999998,"5min":0.34100000000000003,"15min":0.32000000000000001},"max":{"1min":8.6129999999999995,"5min":12.952,"15min":12.952},"last":0.41899999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:45:41 2023","interfaces":[{"interface":"back","average":{"1min":2.4809999999999999,"5min":2.2599999999999998,"15min":1.974},"min":{"1min":0.44800000000000001,"5min":0.36599999999999999,"15min":0.34300000000000003},"max":{"1min":8.6790000000000003,"5min":12.917,"15min":17.524999999999999},"last":0.65900000000000003},{"interface":"front","average":{"1min":1.7929999999999999,"5min":2.3940000000000001,"15min":2.2320000000000002},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.33700000000000002},"max":{"1min":6.7960000000000003,"5min":15.936,"15min":19.524000000000001},"last":0.54800000000000004}]},{"osd":7,"last update":"Sat Sep 23 20:45:41 2023","interfaces":[{"interface":"back","average":{"1min":1.7,"5min":2.7890000000000001,"15min":2.5310000000000001},"min":{"1min":0.42699999999999999,"5min":0.41499999999999998,"15min":0.36899999999999999},"max":{"1min":4.8630000000000004,"5min":15.976000000000001,"15min":19.571999999999999},"last":0.59199999999999997},{"interface":"front","average":{"1min":2.1179999999999999,"5min":2.4870000000000001,"15min":1.97},"min":{"1min":0.46899999999999997,"5min":0.42799999999999999,"15min":0.40999999999999998},"max":{"1min":10.853999999999999,"5min":22.202999999999999,"15min":22.202999999999999},"last":0.62}]}]},{"osd":4,"up_from":30,"seq":128849019308,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":883240,"kb_used_data":439904,"kb_used_omap":0,"kb_used_meta":443328,"kb_avail":92861912,"statfs":{"total":95995035648,"available":95090597888,"internally_reserved":0,"allocated":450461696,"data_stored":448948468,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":453967872},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:45:00 2023","interfaces":[{"interface":"back","average":{"1min":2.0289999999999999,"5min":2.109,"15min":1.77},"min":{"1min":0.26100000000000001,"5min":0.251,"15min":0.24099999999999999},"max":{"1min":7.9029999999999996,"5min":11.709,"15min":24.114999999999998},"last":0.96199999999999997},{"interface":"front","average":{"1min":2.343,"5min":2.3839999999999999,"15min":1.98},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.246},"max":{"1min":9.8580000000000005,"5min":13.74,"15min":24.135999999999999},"last":0.57899999999999996}]},{"osd":1,"last update":"Sat Sep 23 20:45:12 2023","interfaces":[{"interface":"back","average":{"1min":2.6760000000000002,"5min":2.2410000000000001,"15min":1.819},"min":{"1min":0.374,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":9.673,"5min":10.363,"15min":18.190999999999999},"last":0.496},{"interface":"front","average":{"1min":1.9890000000000001,"5min":1.778,"15min":1.611},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":8.0730000000000004,"5min":10.426,"15min":18.173999999999999},"last":1.625}]},{"osd":2,"last update":"Sat Sep 23 20:45:12 2023","interfaces":[{"interface":"back","average":{"1min":1.593,"5min":1.8069999999999999,"15min":1.788},"min":{"1min":0.35799999999999998,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":7.8559999999999999,"5min":13.051,"15min":13.051},"last":1.0589999999999999},{"interface":"front","average":{"1min":2.7650000000000001,"5min":2.1659999999999999,"15min":2.105},"min":{"1min":0.40799999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":6.8479999999999999,"5min":10.945,"15min":14.614000000000001},"last":1.552}]},{"osd":3,"last update":"Sat Sep 23 20:45:28 2023","interfaces":[{"interface":"back","average":{"1min":2.0790000000000002,"5min":1.972,"15min":2.0339999999999998},"min":{"1min":0.40600000000000003,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":8.6180000000000003,"5min":8.6180000000000003,"15min":14.515000000000001},"last":1.093},{"interface":"front","average":{"1min":2.2389999999999999,"5min":2.181,"15min":2.0920000000000001},"min":{"1min":0.40600000000000003,"5min":0.33000000000000002,"15min":0.30299999999999999},"max":{"1min":17.169,"5min":17.169,"15min":17.169},"last":1.026}]},{"osd":5,"last update":"Sat Sep 23 20:45:00 2023","interfaces":[{"interface":"back","average":{"1min":1.504,"5min":1.486,"15min":1.0569999999999999},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":10.202,"5min":10.202,"15min":10.202},"last":0.46200000000000002},{"interface":"front","average":{"1min":1.895,"5min":1.389,"15min":1.0940000000000001},"min":{"1min":0.316,"5min":0.311,"15min":0.28999999999999998},"max":{"1min":8.1440000000000001,"5min":8.1440000000000001,"15min":8.1440000000000001},"last":0.50900000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:45:12 2023","interfaces":[{"interface":"back","average":{"1min":2.0409999999999999,"5min":1.609,"15min":1.2350000000000001},"min":{"1min":0.33900000000000002,"5min":0.29499999999999998,"15min":0.27600000000000002},"max":{"1min":10.840999999999999,"5min":10.840999999999999,"15min":14.731999999999999},"last":0.54400000000000004},{"interface":"front","average":{"1min":1.6879999999999999,"5min":1.4319999999999999,"15min":1.0840000000000001},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":4.9820000000000002,"5min":7.6150000000000002,"15min":7.6150000000000002},"last":0.42099999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:45:28 2023","interfaces":[{"interface":"back","average":{"1min":17.373999999999999,"5min":5.3650000000000002,"15min":2.532},"min":{"1min":0.46400000000000002,"5min":0.35099999999999998,"15min":0.318},"max":{"1min":313.50099999999998,"5min":313.50099999999998,"15min":313.50099999999998},"last":1.7150000000000001},{"interface":"front","average":{"1min":16.062999999999999,"5min":4.6429999999999998,"15min":2.1019999999999999},"min":{"1min":0.39800000000000002,"5min":0.33200000000000002,"15min":0.32700000000000001},"max":{"1min":312.67500000000001,"5min":312.67500000000001,"15min":312.67500000000001},"last":0.52500000000000002}]}]},{"osd":5,"up_from":39,"seq":167503724969,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":802980,"kb_used_data":272604,"kb_used_omap":0,"kb_used_meta":530368,"kb_avail":92942172,"statfs":{"total":95995035648,"available":95172784128,"internally_reserved":0,"allocated":279146496,"data_stored":277633994,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":543096832},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:45:02 2023","interfaces":[{"interface":"back","average":{"1min":2.222,"5min":1.8500000000000001,"15min":1.887},"min":{"1min":0.26300000000000001,"5min":0.22500000000000001,"15min":0.20699999999999999},"max":{"1min":11.25,"5min":15.018000000000001,"15min":18.16},"last":0.85899999999999999},{"interface":"front","average":{"1min":1.5589999999999999,"5min":1.726,"15min":1.766},"min":{"1min":0.307,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":6.71,"5min":15.034000000000001,"15min":15.034000000000001},"last":0.83499999999999996}]},{"osd":1,"last update":"Sat Sep 23 20:45:16 2023","interfaces":[{"interface":"back","average":{"1min":2.3039999999999998,"5min":3.1549999999999998,"15min":2.2850000000000001},"min":{"1min":0.35599999999999998,"5min":0.26500000000000001,"15min":0.253},"max":{"1min":6.2610000000000001,"5min":61.005000000000003,"15min":61.005000000000003},"last":0.40699999999999997},{"interface":"front","average":{"1min":1.76,"5min":2.5150000000000001,"15min":2.0150000000000001},"min":{"1min":0.40799999999999997,"5min":0.27700000000000002,"15min":0.23499999999999999},"max":{"1min":6.6139999999999999,"5min":59.688000000000002,"15min":59.688000000000002},"last":1.407}]},{"osd":2,"last update":"Sat Sep 23 20:45:02 2023","interfaces":[{"interface":"back","average":{"1min":2.2919999999999998,"5min":2.2610000000000001,"15min":2.2010000000000001},"min":{"1min":0.34599999999999997,"5min":0.27100000000000002,"15min":0.26500000000000001},"max":{"1min":12.226000000000001,"5min":12.226000000000001,"15min":37.011000000000003},"last":1.3560000000000001},{"interface":"front","average":{"1min":1.246,"5min":1.893,"15min":2.0390000000000001},"min":{"1min":0.33000000000000002,"5min":0.28299999999999997,"15min":0.255},"max":{"1min":3.5259999999999998,"5min":10.565,"15min":36.835000000000001},"last":0.79300000000000004}]},{"osd":3,"last update":"Sat Sep 23 20:45:24 2023","interfaces":[{"interface":"back","average":{"1min":2.157,"5min":2.3599999999999999,"15min":2.3540000000000001},"min":{"1min":0.44500000000000001,"5min":0.33800000000000002,"15min":0.29599999999999999},"max":{"1min":7.218,"5min":11.032,"15min":36.716000000000001},"last":2.1549999999999998},{"interface":"front","average":{"1min":1.788,"5min":2.29,"15min":2.2629999999999999},"min":{"1min":0.35599999999999998,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":7.8869999999999996,"5min":13.266,"15min":18.504000000000001},"last":1.4399999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:44:54 2023","interfaces":[{"interface":"back","average":{"1min":1.1850000000000001,"5min":1.857,"15min":1.2729999999999999},"min":{"1min":0.30199999999999999,"5min":0.27300000000000002,"15min":0.255},"max":{"1min":5.1050000000000004,"5min":13.391999999999999,"15min":28.015999999999998},"last":0.59399999999999997},{"interface":"front","average":{"1min":1.1499999999999999,"5min":1.6319999999999999,"15min":1.1200000000000001},"min":{"1min":0.32000000000000001,"5min":0.30299999999999999,"15min":0.28299999999999997},"max":{"1min":6.8399999999999999,"5min":7.3799999999999999,"15min":12.300000000000001},"last":1.468}]},{"osd":6,"last update":"Sat Sep 23 20:45:08 2023","interfaces":[{"interface":"back","average":{"1min":1.9350000000000001,"5min":2.161,"15min":1.319},"min":{"1min":0.38100000000000001,"5min":0.317,"15min":0.30199999999999999},"max":{"1min":6.952,"5min":13.734,"15min":13.734},"last":2.0960000000000001},{"interface":"front","average":{"1min":1.1220000000000001,"5min":1.5660000000000001,"15min":1.0980000000000001},"min":{"1min":0.35299999999999998,"5min":0.317,"15min":0.317},"max":{"1min":5.2629999999999999,"5min":8.7449999999999992,"15min":8.7449999999999992},"last":1.5289999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:45:24 2023","interfaces":[{"interface":"back","average":{"1min":15.821999999999999,"5min":4.6180000000000003,"15min":2.0510000000000002},"min":{"1min":0.44500000000000001,"5min":0.34499999999999997,"15min":0.28499999999999998},"max":{"1min":294.70999999999998,"5min":294.70999999999998,"15min":294.70999999999998},"last":1.4990000000000001},{"interface":"front","average":{"1min":16.821000000000002,"5min":4.8049999999999997,"15min":2.1469999999999998},"min":{"1min":0.432,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":294.74400000000003,"5min":294.74400000000003,"15min":294.74400000000003},"last":2.2599999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4722,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1360,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8206,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7156,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3154,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":717422592,"data_stored":717422592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":127074304,"data_stored":127074304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":228851712,"data_stored":228851712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":264372224,"data_stored":264372224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":526647296,"data_stored":526647296,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":326500352,"data_stored":326500352,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":278528000,"data_stored":278528000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":127860736,"data_stored":127860736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:45:56.847 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-09-23T20:45:56.848 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd pool set .mgr pg_num 61 2023-09-23T20:45:56.911 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:45:56.978 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:56.982 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 12 16 29190 29174 151.9 101.375 0.00667935 0.00657699 2023-09-23T20:45:57.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:57 smithi018 ceph-mon[179647]: pgmap v1347: 119 pgs: 119 active+clean; 1.3 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 131 MiB/s wr, 2.09k op/s 2023-09-23T20:45:57.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:57 smithi018 ceph-mon[184144]: pgmap v1347: 119 pgs: 119 active+clean; 1.3 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 131 MiB/s wr, 2.09k op/s 2023-09-23T20:45:57.968 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:57 smithi149 ceph-mon[170394]: pgmap v1347: 119 pgs: 119 active+clean; 1.3 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 131 MiB/s wr, 2.09k op/s 2023-09-23T20:45:57.982 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 13 16 31092 31076 149.36 118.875 0.00886426 0.00668903 2023-09-23T20:45:58.462 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:45:58.468 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:58.468 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:45:58.569 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:45:58.596 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:58.661 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:45:58.668 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:58.669 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:45:58.770 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:45:58.824 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:58.896 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:45:58.902 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:58.902 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:45:58.983 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 14 15 32924 32909 146.86 114.562 0.014679 0.00680319 2023-09-23T20:45:59.003 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:45:59.062 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:59.248 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:45:59.253 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:59.253 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:45:59.354 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:45:59.376 INFO:teuthology.orchestra.run.smithi018.stdout:ERROR: (22) Invalid argument 2023-09-23T20:45:59.376 INFO:teuthology.orchestra.run.smithi018.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:45:59.394 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:59.401 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:45:59.439 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:45:59.445 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:45:59.450 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:59.450 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:45:59.551 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:45:59.599 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:59.657 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:45:59.661 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:59.662 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:45:59.662 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:59 smithi149 ceph-mon[170394]: pgmap v1348: 119 pgs: 119 active+clean; 1.6 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 140 MiB/s wr, 2.24k op/s 2023-09-23T20:45:59.662 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:45:59 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:45:59.763 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:45:59.798 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:59 smithi018 ceph-mon[184144]: pgmap v1348: 119 pgs: 119 active+clean; 1.6 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 140 MiB/s wr, 2.24k op/s 2023-09-23T20:45:59.798 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:45:59 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:45:59.798 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:59 smithi018 ceph-mon[179647]: pgmap v1348: 119 pgs: 119 active+clean; 1.6 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 140 MiB/s wr, 2.24k op/s 2023-09-23T20:45:59.798 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:45:59 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:45:59.813 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:45:59.871 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T20:45:59.876 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:45:59.881 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:45:59.882 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:45:59.983 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:45:59.985 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 15 16 34500 34484 143.614 98.4375 0.0135422 0.00695812 2023-09-23T20:46:00.025 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:00.068 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:46:00.072 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:00.072 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:46:00.173 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:46:00.215 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:00.274 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:46:00.280 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:00.281 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:46:00.381 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:46:00.454 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:00.530 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:46:00.536 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:00.536 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:46:00.616 INFO:teuthology.orchestra.run.smithi018.stderr:nodeep-scrub is unset 2023-09-23T20:46:00.637 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:46:00.697 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:00.780 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:46:00.785 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:00.786 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:46:00.786 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:00 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/584330429' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:46:00.786 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:00 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:46:00.788 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:00 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/584330429' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:46:00.788 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:00 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:46:00.887 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:46:00.932 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:00 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/584330429' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:46:00.933 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:00 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:46:00.941 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:00.985 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 16 16 36082 36066 140.818 98.875 0.00454982 0.00709549 2023-09-23T20:46:01.008 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:46:01.014 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:01.015 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:46:01.116 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:46:01.194 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:01.316 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:46:01.327 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:01.328 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:46:01.428 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:46:01.467 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:01.529 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:46:01.537 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:01.537 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:46:01.605 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:01 smithi018 ceph-mon[184144]: pgmap v1349: 119 pgs: 119 active+clean; 1.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 121 MiB/s wr, 1.94k op/s 2023-09-23T20:46:01.605 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:01 smithi018 ceph-mon[184144]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:46:01.605 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:01 smithi018 ceph-mon[184144]: Cluster is now healthy 2023-09-23T20:46:01.605 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:01 smithi018 ceph-mon[184144]: failed to encode map e537 with expected crc 2023-09-23T20:46:01.605 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:01 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:46:01.606 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:01 smithi018 ceph-mon[184144]: osdmap e537: 8 total, 8 up, 8 in 2023-09-23T20:46:01.638 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:46:01.698 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:01.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:01 smithi149 ceph-mon[170394]: pgmap v1349: 119 pgs: 119 active+clean; 1.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 121 MiB/s wr, 1.94k op/s 2023-09-23T20:46:01.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:01 smithi149 ceph-mon[170394]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:46:01.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:01 smithi149 ceph-mon[170394]: Cluster is now healthy 2023-09-23T20:46:01.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:01 smithi149 ceph-mon[170394]: failed to encode map e537 with expected crc 2023-09-23T20:46:01.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:01 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:46:01.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:01 smithi149 ceph-mon[170394]: osdmap e537: 8 total, 8 up, 8 in 2023-09-23T20:46:01.765 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:46:01.773 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:01.773 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:46:01.866 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:46:01.875 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:46:01.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:01 smithi018 ceph-mon[179647]: pgmap v1349: 119 pgs: 119 active+clean; 1.9 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 121 MiB/s wr, 1.94k op/s 2023-09-23T20:46:01.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:01 smithi018 ceph-mon[179647]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:46:01.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:01 smithi018 ceph-mon[179647]: Cluster is now healthy 2023-09-23T20:46:01.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:01 smithi018 ceph-mon[179647]: failed to encode map e537 with expected crc 2023-09-23T20:46:01.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:01 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:46:01.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:01 smithi018 ceph-mon[179647]: osdmap e537: 8 total, 8 up, 8 in 2023-09-23T20:46:01.965 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:01.985 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 17 16 37633 37617 138.237 96.9375 0.0158735 0.00722948 2023-09-23T20:46:02.012 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:46:02.040 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:46:02.045 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:02.046 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:46:02.057 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:46:02.090 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:46:02.117 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:46:02.147 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:46:02.153 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:46:02.190 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:46:02.222 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:02.225 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:46:02.283 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:46:02.289 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:46:02.289 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:46:02.289 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:46:02.289 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:46:02.289 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:46:02.290 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:46:02.290 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:46:02.290 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:46:02.297 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:46:02.307 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:02.308 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:46:02.408 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:46:02.450 INFO:teuthology.orchestra.run.smithi018.stderr:Error EINVAL: specified pg_num 61 < pg_num_max 32 2023-09-23T20:46:02.459 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:02.522 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:46:02.529 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:02.530 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:46:02.630 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:46:02.695 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:02.718 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:02 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3739409398' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:46:02.719 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:02 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2036941395' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "61"}]: dispatch 2023-09-23T20:46:02.719 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:02 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "61"}]: dispatch 2023-09-23T20:46:02.765 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:46:02.768 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:02.769 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:46:02.869 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:46:02.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:02 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3739409398' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:46:02.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:02 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2036941395' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "61"}]: dispatch 2023-09-23T20:46:02.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:02 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "61"}]: dispatch 2023-09-23T20:46:02.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:02 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3739409398' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:46:02.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:02 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2036941395' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "61"}]: dispatch 2023-09-23T20:46:02.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:02 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "61"}]: dispatch 2023-09-23T20:46:02.932 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:02.989 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 18 16 39159 39143 135.845 95.375 0.0172314 0.00735594 2023-09-23T20:46:03.148 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:46:03.153 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:03.154 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:46:03.254 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:46:03.308 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:03.360 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:46:03.368 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:03.368 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:46:03.463 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T20:46:03.587 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:46:03.640 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:03.696 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:46:03.704 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:03.704 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:46:03.805 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:46:03.878 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:03 smithi018 ceph-mon[179647]: pgmap v1351: 119 pgs: 119 active+clean; 2.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 141 MiB/s wr, 2.26k op/s 2023-09-23T20:46:03.878 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:03 smithi018 ceph-mon[184144]: pgmap v1351: 119 pgs: 119 active+clean; 2.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 141 MiB/s wr, 2.26k op/s 2023-09-23T20:46:03.888 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:03.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:03 smithi149 ceph-mon[170394]: pgmap v1351: 119 pgs: 119 active+clean; 2.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 141 MiB/s wr, 2.26k op/s 2023-09-23T20:46:03.988 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 19 16 40629 40613 133.528 91.875 0.00722532 0.00748293 2023-09-23T20:46:04.990 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:2023-09-23T20:46:04.988215+0000 min lat: 0.00123458 max lat: 0.108138 avg lat: 0.0075428 2023-09-23T20:46:04.990 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T20:46:04.990 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 20 16 42436 42420 132.498 112.938 0.00828622 0.0075428 2023-09-23T20:46:05.391 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:46:04 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:46:04] "GET /metrics HTTP/1.1" 200 36572 "" "Prometheus/2.43.0" 2023-09-23T20:46:05.693 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:46:05.703 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:05.703 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:46:05.749 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:05 smithi018 ceph-mon[179647]: pgmap v1352: 119 pgs: 119 active+clean; 2.2 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 104 MiB/s wr, 1.66k op/s 2023-09-23T20:46:05.749 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:05 smithi018 ceph-mon[184144]: pgmap v1352: 119 pgs: 119 active+clean; 2.2 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 104 MiB/s wr, 1.66k op/s 2023-09-23T20:46:05.750 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-09-23T20:46:05.803 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:46:05.877 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:05.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:05 smithi149 ceph-mon[170394]: pgmap v1352: 119 pgs: 119 active+clean; 2.2 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 104 MiB/s wr, 1.66k op/s 2023-09-23T20:46:05.988 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 21 16 43918 43902 130.6 92.625 0.00769674 0.00765162 2023-09-23T20:46:06.077 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:46:06.083 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:06.084 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:46:06.185 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:46:06.228 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:06.284 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:46:06.293 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:06.293 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:46:06.394 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:46:06.411 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:46:06.442 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:06.495 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:46:06.500 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:06.501 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:46:06.598 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2023-09-23T20:46:06.598 INFO:teuthology.orchestra.run.smithi018.stdout: "ops": [], 2023-09-23T20:46:06.598 INFO:teuthology.orchestra.run.smithi018.stdout: "complaint_time": 30, 2023-09-23T20:46:06.598 INFO:teuthology.orchestra.run.smithi018.stdout: "num_blocked_ops": 0 2023-09-23T20:46:06.598 INFO:teuthology.orchestra.run.smithi018.stdout:} 2023-09-23T20:46:06.601 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:46:06.676 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:06.746 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:46:06.755 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:06.755 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:46:06.856 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:46:06.923 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:06.990 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 22 16 45415 45399 128.906 93.5625 0.0146369 0.0077521 2023-09-23T20:46:07.229 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:46:07.234 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:07.234 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:46:07.335 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:46:07.387 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:07.449 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:46:07.457 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:07.457 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:46:07.558 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:46:07.593 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T20:46:07.656 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:07.736 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:46:07.751 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:07.752 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:07 smithi018 ceph-mon[179647]: pgmap v1353: 119 pgs: 119 active+clean; 2.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 124 MiB/s wr, 1.98k op/s 2023-09-23T20:46:07.752 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:07 smithi018 ceph-mon[184144]: pgmap v1353: 119 pgs: 119 active+clean; 2.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 124 MiB/s wr, 1.98k op/s 2023-09-23T20:46:07.753 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:46:07.853 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:46:07.966 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:07.968 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:07 smithi149 ceph-mon[170394]: pgmap v1353: 119 pgs: 119 active+clean; 2.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 124 MiB/s wr, 1.98k op/s 2023-09-23T20:46:07.989 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 23 16 46888 46872 127.304 92.0625 0.00813026 0.00785036 2023-09-23T20:46:08.044 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:46:08.048 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:08.048 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:46:08.126 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:46:08.150 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:46:08.194 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:08.250 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:46:08.254 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:08.254 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:46:08.355 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:46:08.396 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:08.458 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:46:08.465 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:08.465 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:46:08.566 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:46:08.578 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:46:08.611 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:08.651 INFO:teuthology.orchestra.run.smithi018.stderr:noscrub is set 2023-09-23T20:46:08.661 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:46:08.665 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:08.665 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:46:08.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:08 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2743961297' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:46:08.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:08 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:46:08.766 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:46:08.852 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:08.990 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 24 16 48181 48165 125.367 80.8125 0.00666141 0.00797208 2023-09-23T20:46:09.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:08 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2743961297' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:46:09.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:08 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:46:09.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:08 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2743961297' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:46:09.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:08 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:46:09.274 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:46:09.284 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:09.285 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:46:09.385 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:46:09.440 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:09.498 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:46:09.502 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:09.503 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:46:09.604 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:46:09.681 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:09.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:09 smithi149 ceph-mon[170394]: pgmap v1354: 119 pgs: 119 active+clean; 2.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 113 MiB/s wr, 1.80k op/s 2023-09-23T20:46:09.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:09 smithi149 ceph-mon[170394]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:46:09.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:09 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:46:09.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:09 smithi149 ceph-mon[170394]: osdmap e538: 8 total, 8 up, 8 in 2023-09-23T20:46:09.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:09 smithi149 ceph-mon[170394]: failed to encode map e538 with expected crc 2023-09-23T20:46:09.990 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 25 16 49740 49724 124.25 97.4375 0.0108652 0.00804291 2023-09-23T20:46:10.042 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:46:10.051 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:10.051 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:46:10.051 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:09 smithi018 ceph-mon[179647]: pgmap v1354: 119 pgs: 119 active+clean; 2.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 113 MiB/s wr, 1.80k op/s 2023-09-23T20:46:10.052 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:09 smithi018 ceph-mon[179647]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:46:10.052 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:09 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:46:10.052 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:09 smithi018 ceph-mon[179647]: osdmap e538: 8 total, 8 up, 8 in 2023-09-23T20:46:10.052 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:09 smithi018 ceph-mon[179647]: failed to encode map e538 with expected crc 2023-09-23T20:46:10.052 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:09 smithi018 ceph-mon[184144]: pgmap v1354: 119 pgs: 119 active+clean; 2.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 113 MiB/s wr, 1.80k op/s 2023-09-23T20:46:10.052 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:09 smithi018 ceph-mon[184144]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:46:10.052 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:09 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:46:10.052 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:09 smithi018 ceph-mon[184144]: osdmap e538: 8 total, 8 up, 8 in 2023-09-23T20:46:10.052 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:09 smithi018 ceph-mon[184144]: failed to encode map e538 with expected crc 2023-09-23T20:46:10.082 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:46:10.152 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:46:10.241 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:10.751 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 7, 3, 2, 4, 6, 5] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T20:46:10.751 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:46:10.751 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_5 2023-09-23T20:46:10.752 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:46:10.990 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 26 16 51156 51140 122.875 88.5 0.00415388 0.00812086 2023-09-23T20:46:11.234 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:46:11.237 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:11.238 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:46:11.338 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:46:11.385 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:11.443 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:46:11.449 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:11.450 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:46:11.551 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:46:11.611 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:11.677 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:46:11.684 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:11.685 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:46:11.785 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:46:11.823 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:11 smithi018 ceph-mon[184144]: pgmap v1356: 119 pgs: 119 active+clean; 2.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s 2023-09-23T20:46:11.823 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:11.823 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:11 smithi018 ceph-mon[179647]: pgmap v1356: 119 pgs: 119 active+clean; 2.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s 2023-09-23T20:46:11.881 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:46:11.886 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:11.886 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:46:11.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:11 smithi149 ceph-mon[170394]: pgmap v1356: 119 pgs: 119 active+clean; 2.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s 2023-09-23T20:46:11.987 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:46:11.990 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 27 16 52440 52424 121.297 80.25 0.00536971 0.00823893 2023-09-23T20:46:12.040 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:12.104 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:46:12.113 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:12.114 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:46:12.124 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T20:46:12.214 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:46:12.238 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:46:12.339 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:12.750 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:46:12.761 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:12.762 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:46:12.863 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:46:12.905 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:12.966 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:46:12.972 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:12.972 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:46:12.992 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 28 16 53688 53672 119.751 78 0.0108514 0.0083462 2023-09-23T20:46:13.039 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2023-09-23T20:46:13.039 INFO:teuthology.orchestra.run.smithi018.stdout: "size": 20, 2023-09-23T20:46:13.039 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 600, 2023-09-23T20:46:13.039 INFO:teuthology.orchestra.run.smithi018.stdout: "ops": [ 2023-09-23T20:46:13.039 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.039 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35994.0:105785 8.10 8:0b1076e5:::benchmark_data_smithi018_341490_object16462:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e410)", 2023-09-23T20:46:13.039 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:38:26.443871+0000", 2023-09-23T20:46:13.040 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 466.59199431299999, 2023-09-23T20:46:13.040 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1767037899999999, 2023-09-23T20:46:13.040 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:46:13.040 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:46:13.040 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:46:13.040 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35994", 2023-09-23T20:46:13.040 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:46:13.040 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 105785 2023-09-23T20:46:13.040 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.040 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:46:13.040 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.040 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:46:13.041 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.443871+0000", 2023-09-23T20:46:13.041 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:46:13.041 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.041 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.041 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:46:13.041 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.443871+0000", 2023-09-23T20:46:13.041 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:46:13.041 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.041 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.041 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:46:13.041 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.443871+0000", 2023-09-23T20:46:13.041 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:46:13.041 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.042 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.042 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:46:13.042 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.443875+0000", 2023-09-23T20:46:13.042 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.9509999999999999e-06 2023-09-23T20:46:13.042 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.042 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.042 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:46:13.042 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.443882+0000", 2023-09-23T20:46:13.042 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.8390000000000007e-06 2023-09-23T20:46:13.042 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.042 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.043 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:46:13.043 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.443889+0000", 2023-09-23T20:46:13.043 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.5230000000000001e-06 2023-09-23T20:46:13.043 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.043 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.043 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:46:13.043 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.713126+0000", 2023-09-23T20:46:13.043 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.269236743 2023-09-23T20:46:13.043 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.043 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.043 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:46:13.043 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.713135+0000", 2023-09-23T20:46:13.044 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.0790000000000007e-06 2023-09-23T20:46:13.044 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.044 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.044 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:46:13.044 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.610121+0000", 2023-09-23T20:46:13.044 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.89698586199999997 2023-09-23T20:46:13.044 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.044 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.044 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:46:13.044 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.610236+0000", 2023-09-23T20:46:13.044 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000114833 2023-09-23T20:46:13.044 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.045 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.045 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,4", 2023-09-23T20:46:13.045 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.610355+0000", 2023-09-23T20:46:13.045 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00011921099999999999 2023-09-23T20:46:13.045 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.045 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.045 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:46:13.045 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.615886+0000", 2023-09-23T20:46:13.045 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0055312039999999996 2023-09-23T20:46:13.045 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.045 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.045 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:46:13.046 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.618509+0000", 2023-09-23T20:46:13.046 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.002623434 2023-09-23T20:46:13.046 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.046 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.046 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:46:13.046 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.619136+0000", 2023-09-23T20:46:13.046 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00062698799999999998 2023-09-23T20:46:13.046 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.046 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.046 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:46:13.046 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.619190+0000", 2023-09-23T20:46:13.047 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.3492000000000002e-05 2023-09-23T20:46:13.047 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.047 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.047 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:46:13.047 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.620575+0000", 2023-09-23T20:46:13.047 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.001385231 2023-09-23T20:46:13.047 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:46:13.047 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:46:13.047 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:46:13.047 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.047 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.047 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35994.0:105786 8.10 8:0f8943c0:::benchmark_data_smithi018_341490_object16463:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e410)", 2023-09-23T20:46:13.047 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:38:26.443896+0000", 2023-09-23T20:46:13.048 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 466.59196939100002, 2023-09-23T20:46:13.048 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1781387480000001, 2023-09-23T20:46:13.048 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:46:13.048 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:46:13.048 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:46:13.048 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35994", 2023-09-23T20:46:13.048 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:46:13.048 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 105786 2023-09-23T20:46:13.048 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.048 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:46:13.048 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.048 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:46:13.049 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.443896+0000", 2023-09-23T20:46:13.049 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:46:13.049 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.049 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.049 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:46:13.049 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.443896+0000", 2023-09-23T20:46:13.049 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:46:13.049 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.049 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.049 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:46:13.049 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.443896+0000", 2023-09-23T20:46:13.049 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:46:13.050 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.050 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.050 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:46:13.050 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.443899+0000", 2023-09-23T20:46:13.050 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.337e-06 2023-09-23T20:46:13.050 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.050 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.050 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:46:13.050 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.443905+0000", 2023-09-23T20:46:13.050 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.2260000000000002e-06 2023-09-23T20:46:13.050 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.051 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.051 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:46:13.051 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.443911+0000", 2023-09-23T20:46:13.051 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.135e-06 2023-09-23T20:46:13.051 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.051 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.051 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:46:13.051 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.713155+0000", 2023-09-23T20:46:13.051 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.26924421900000001 2023-09-23T20:46:13.051 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.051 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.051 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:46:13.052 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.713161+0000", 2023-09-23T20:46:13.052 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.7799999999999997e-06 2023-09-23T20:46:13.052 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.052 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.052 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:46:13.052 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.611522+0000", 2023-09-23T20:46:13.052 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.89836117699999996 2023-09-23T20:46:13.052 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.052 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.052 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:46:13.052 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.611650+0000", 2023-09-23T20:46:13.052 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000128677 2023-09-23T20:46:13.052 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.053 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.053 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,4", 2023-09-23T20:46:13.053 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.611781+0000", 2023-09-23T20:46:13.053 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00013079300000000001 2023-09-23T20:46:13.053 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.053 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.053 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:46:13.053 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.618515+0000", 2023-09-23T20:46:13.053 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0067336820000000004 2023-09-23T20:46:13.053 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.053 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.053 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:46:13.053 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.619100+0000", 2023-09-23T20:46:13.054 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00058515599999999998 2023-09-23T20:46:13.054 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.054 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.054 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:46:13.054 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.620654+0000", 2023-09-23T20:46:13.054 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.001553476 2023-09-23T20:46:13.054 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.054 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.054 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:46:13.054 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.620709+0000", 2023-09-23T20:46:13.054 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.5781e-05 2023-09-23T20:46:13.054 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.055 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.055 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:46:13.055 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.622035+0000", 2023-09-23T20:46:13.055 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0013256679999999999 2023-09-23T20:46:13.055 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:46:13.055 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:46:13.055 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:46:13.055 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.055 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.055 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35994.0:105787 8.0 8:015f4eb0:::benchmark_data_smithi018_341490_object16464:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e410)", 2023-09-23T20:46:13.055 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:38:26.443916+0000", 2023-09-23T20:46:13.055 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 466.591949865, 2023-09-23T20:46:13.056 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1854093080000001, 2023-09-23T20:46:13.056 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:46:13.056 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:46:13.056 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:46:13.056 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35994", 2023-09-23T20:46:13.056 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:46:13.056 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 105787 2023-09-23T20:46:13.056 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.056 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:46:13.056 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.056 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:46:13.056 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.443916+0000", 2023-09-23T20:46:13.056 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:46:13.057 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.057 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.057 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:46:13.057 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.443916+0000", 2023-09-23T20:46:13.057 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:46:13.057 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.057 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.057 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:46:13.057 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.443915+0000", 2023-09-23T20:46:13.057 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:46:13.057 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.057 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.057 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:46:13.058 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.443918+0000", 2023-09-23T20:46:13.058 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.509e-06 2023-09-23T20:46:13.058 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.058 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.058 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:46:13.058 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.443924+0000", 2023-09-23T20:46:13.058 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.6740000000000002e-06 2023-09-23T20:46:13.058 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.058 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.058 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:46:13.058 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.443929+0000", 2023-09-23T20:46:13.058 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.5199999999999997e-06 2023-09-23T20:46:13.059 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.059 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.059 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:46:13.059 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.713182+0000", 2023-09-23T20:46:13.059 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.26925305700000002 2023-09-23T20:46:13.059 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.059 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.059 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:46:13.059 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.713190+0000", 2023-09-23T20:46:13.059 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.7980000000000006e-06 2023-09-23T20:46:13.059 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.059 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.059 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:46:13.060 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.613650+0000", 2023-09-23T20:46:13.060 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.90046006099999998 2023-09-23T20:46:13.060 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.060 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.060 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:46:13.060 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.613744+0000", 2023-09-23T20:46:13.060 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.4349000000000001e-05 2023-09-23T20:46:13.060 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.060 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.060 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,4", 2023-09-23T20:46:13.060 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.613821+0000", 2023-09-23T20:46:13.060 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.6915000000000005e-05 2023-09-23T20:46:13.061 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.061 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.061 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:46:13.061 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.619293+0000", 2023-09-23T20:46:13.061 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0054714159999999998 2023-09-23T20:46:13.061 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.061 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.061 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:46:13.061 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.627285+0000", 2023-09-23T20:46:13.061 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0079923000000000008 2023-09-23T20:46:13.061 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.061 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.062 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:46:13.062 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.627385+0000", 2023-09-23T20:46:13.062 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.9766000000000007e-05 2023-09-23T20:46:13.062 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.062 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.062 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:46:13.062 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.627433+0000", 2023-09-23T20:46:13.062 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.7963999999999997e-05 2023-09-23T20:46:13.062 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.062 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.062 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:46:13.062 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.629325+0000", 2023-09-23T20:46:13.063 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0018923880000000001 2023-09-23T20:46:13.063 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:46:13.063 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:46:13.063 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:46:13.063 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.063 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.063 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35994.0:105791 8.10 8:0d4c47c5:::benchmark_data_smithi018_341490_object16468:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e410)", 2023-09-23T20:46:13.063 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:38:26.443934+0000", 2023-09-23T20:46:13.063 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 466.59193168299998, 2023-09-23T20:46:13.063 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1850523630000001, 2023-09-23T20:46:13.063 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:46:13.063 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:46:13.063 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:46:13.064 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35994", 2023-09-23T20:46:13.064 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:46:13.064 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 105791 2023-09-23T20:46:13.064 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.064 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:46:13.064 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.064 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:46:13.064 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.443934+0000", 2023-09-23T20:46:13.064 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:46:13.064 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.064 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.064 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:46:13.064 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.443934+0000", 2023-09-23T20:46:13.065 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:46:13.065 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.065 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.065 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:46:13.065 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.443934+0000", 2023-09-23T20:46:13.065 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:46:13.065 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.065 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.065 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:46:13.065 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.443936+0000", 2023-09-23T20:46:13.065 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.1469999999999999e-06 2023-09-23T20:46:13.065 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.065 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.066 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:46:13.066 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.443942+0000", 2023-09-23T20:46:13.066 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.9050000000000002e-06 2023-09-23T20:46:13.066 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.066 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.066 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:46:13.066 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.443947+0000", 2023-09-23T20:46:13.066 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.8699999999999997e-06 2023-09-23T20:46:13.066 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.066 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.066 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:46:13.066 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.713210+0000", 2023-09-23T20:46:13.067 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.269262891 2023-09-23T20:46:13.067 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.067 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.067 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:46:13.067 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.713216+0000", 2023-09-23T20:46:13.067 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.0769999999999996e-06 2023-09-23T20:46:13.067 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.067 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.067 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:46:13.067 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.612639+0000", 2023-09-23T20:46:13.067 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.89942242299999997 2023-09-23T20:46:13.067 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.068 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.068 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:46:13.068 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.612710+0000", 2023-09-23T20:46:13.068 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.0743999999999999e-05 2023-09-23T20:46:13.068 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.068 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.068 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,4", 2023-09-23T20:46:13.068 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.612781+0000", 2023-09-23T20:46:13.068 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.0965999999999998e-05 2023-09-23T20:46:13.068 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.068 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.069 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:46:13.069 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.618520+0000", 2023-09-23T20:46:13.069 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0057398170000000004 2023-09-23T20:46:13.069 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.069 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.069 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:46:13.069 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.627222+0000", 2023-09-23T20:46:13.069 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0087017199999999996 2023-09-23T20:46:13.069 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.069 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.069 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:46:13.069 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.627275+0000", 2023-09-23T20:46:13.070 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.3081000000000002e-05 2023-09-23T20:46:13.070 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.070 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.070 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:46:13.070 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.627336+0000", 2023-09-23T20:46:13.070 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.0630000000000001e-05 2023-09-23T20:46:13.070 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.070 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.070 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:46:13.070 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.628986+0000", 2023-09-23T20:46:13.070 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0016504309999999999 2023-09-23T20:46:13.071 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:46:13.071 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:46:13.071 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:46:13.071 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.071 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.071 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35994.0:105792 8.0 8:02af189b:::benchmark_data_smithi018_341490_object16469:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e410)", 2023-09-23T20:46:13.071 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:38:26.443952+0000", 2023-09-23T20:46:13.071 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 466.59191341100001, 2023-09-23T20:46:13.071 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1871479650000001, 2023-09-23T20:46:13.071 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:46:13.071 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:46:13.071 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:46:13.072 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35994", 2023-09-23T20:46:13.072 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:46:13.072 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 105792 2023-09-23T20:46:13.072 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.072 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:46:13.072 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.072 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:46:13.072 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.443952+0000", 2023-09-23T20:46:13.072 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:46:13.072 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.072 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.072 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:46:13.073 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.443952+0000", 2023-09-23T20:46:13.073 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:46:13.073 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.073 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.073 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:46:13.073 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.443952+0000", 2023-09-23T20:46:13.073 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:46:13.073 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.073 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.073 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:46:13.073 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.443954+0000", 2023-09-23T20:46:13.073 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.6089999999999999e-06 2023-09-23T20:46:13.073 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.074 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.074 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:46:13.074 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.443961+0000", 2023-09-23T20:46:13.074 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.3029999999999996e-06 2023-09-23T20:46:13.074 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.074 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.074 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:46:13.074 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.443966+0000", 2023-09-23T20:46:13.074 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.9729999999999997e-06 2023-09-23T20:46:13.074 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.074 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.074 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:46:13.075 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.713239+0000", 2023-09-23T20:46:13.075 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.26927288300000002 2023-09-23T20:46:13.075 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.075 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.075 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:46:13.075 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.713246+0000", 2023-09-23T20:46:13.075 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.1690000000000003e-06 2023-09-23T20:46:13.075 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.075 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.075 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:46:13.075 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.614811+0000", 2023-09-23T20:46:13.075 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.90156508599999996 2023-09-23T20:46:13.076 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.076 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.076 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:46:13.076 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.614929+0000", 2023-09-23T20:46:13.076 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000117846 2023-09-23T20:46:13.076 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.076 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.076 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,4", 2023-09-23T20:46:13.076 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.615050+0000", 2023-09-23T20:46:13.076 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00012100900000000001 2023-09-23T20:46:13.076 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.076 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.076 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:46:13.077 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.619299+0000", 2023-09-23T20:46:13.077 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0042493160000000004 2023-09-23T20:46:13.077 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.077 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.077 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:46:13.077 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.627319+0000", 2023-09-23T20:46:13.077 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0080197559999999994 2023-09-23T20:46:13.077 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.077 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.077 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:46:13.077 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.629386+0000", 2023-09-23T20:46:13.077 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0020676959999999999 2023-09-23T20:46:13.077 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.078 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.078 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:46:13.078 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.629448+0000", 2023-09-23T20:46:13.078 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.1155999999999995e-05 2023-09-23T20:46:13.078 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.078 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.078 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:46:13.078 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.631100+0000", 2023-09-23T20:46:13.078 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.001652518 2023-09-23T20:46:13.078 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:46:13.078 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:46:13.078 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:46:13.079 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.079 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.079 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35994.0:105793 8.18 8:193032ff:::benchmark_data_smithi018_341490_object16470:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e410)", 2023-09-23T20:46:13.079 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:38:26.443971+0000", 2023-09-23T20:46:13.079 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 466.59189416999999, 2023-09-23T20:46:13.079 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.173527269, 2023-09-23T20:46:13.079 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:46:13.079 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:46:13.079 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:46:13.079 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35994", 2023-09-23T20:46:13.079 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:46:13.079 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 105793 2023-09-23T20:46:13.080 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.080 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:46:13.080 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.080 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:46:13.080 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.443971+0000", 2023-09-23T20:46:13.080 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:46:13.080 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.080 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.080 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:46:13.080 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.443971+0000", 2023-09-23T20:46:13.080 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:46:13.080 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.080 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.081 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:46:13.081 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.443971+0000", 2023-09-23T20:46:13.081 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:46:13.081 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.081 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.081 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:46:13.081 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.443979+0000", 2023-09-23T20:46:13.081 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.4270000000000008e-06 2023-09-23T20:46:13.081 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.081 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.081 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:46:13.081 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.443986+0000", 2023-09-23T20:46:13.082 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.5520000000000001e-06 2023-09-23T20:46:13.082 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.082 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.082 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:46:13.082 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.443991+0000", 2023-09-23T20:46:13.082 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.2429999999999996e-06 2023-09-23T20:46:13.082 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.082 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.082 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:46:13.082 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.713266+0000", 2023-09-23T20:46:13.082 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.26927488199999999 2023-09-23T20:46:13.082 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.083 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.083 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:46:13.083 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.713273+0000", 2023-09-23T20:46:13.083 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.4309999999999999e-06 2023-09-23T20:46:13.083 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.083 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.083 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:46:13.083 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.605188+0000", 2023-09-23T20:46:13.083 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.89191558699999995 2023-09-23T20:46:13.083 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.083 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.083 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:46:13.084 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.605348+0000", 2023-09-23T20:46:13.084 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00016024400000000001 2023-09-23T20:46:13.084 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.084 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.084 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 0,7", 2023-09-23T20:46:13.084 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.605486+0000", 2023-09-23T20:46:13.084 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00013745700000000001 2023-09-23T20:46:13.084 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.084 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.084 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:46:13.084 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.610094+0000", 2023-09-23T20:46:13.084 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0046085149999999997 2023-09-23T20:46:13.084 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.085 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.085 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:46:13.085 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.612624+0000", 2023-09-23T20:46:13.085 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0025298859999999999 2023-09-23T20:46:13.085 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.085 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.085 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:46:13.085 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.615895+0000", 2023-09-23T20:46:13.085 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0032704919999999998 2023-09-23T20:46:13.085 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.085 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.085 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:46:13.086 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.615938+0000", 2023-09-23T20:46:13.086 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.3510999999999997e-05 2023-09-23T20:46:13.086 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.086 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.086 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:46:13.086 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.617499+0000", 2023-09-23T20:46:13.086 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.001560475 2023-09-23T20:46:13.086 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:46:13.086 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:46:13.086 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:46:13.086 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.087 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.087 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35994.0:105794 8.0 8:06d1e15f:::benchmark_data_smithi018_341490_object16471:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e410)", 2023-09-23T20:46:13.087 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:38:26.443996+0000", 2023-09-23T20:46:13.087 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 466.59186988099998, 2023-09-23T20:46:13.087 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1894021189999999, 2023-09-23T20:46:13.087 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:46:13.087 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:46:13.087 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:46:13.087 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35994", 2023-09-23T20:46:13.087 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:46:13.087 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 105794 2023-09-23T20:46:13.087 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.087 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:46:13.088 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.088 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:46:13.088 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.443996+0000", 2023-09-23T20:46:13.088 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:46:13.088 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.088 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.088 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:46:13.088 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.443996+0000", 2023-09-23T20:46:13.088 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:46:13.088 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.088 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.088 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:46:13.088 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.443995+0000", 2023-09-23T20:46:13.089 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:46:13.089 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.089 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.089 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:46:13.089 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.443998+0000", 2023-09-23T20:46:13.089 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.7020000000000001e-06 2023-09-23T20:46:13.089 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.089 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.089 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:46:13.089 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.444004+0000", 2023-09-23T20:46:13.089 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.9479999999999996e-06 2023-09-23T20:46:13.089 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.089 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.090 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:46:13.090 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.444009+0000", 2023-09-23T20:46:13.090 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.502e-06 2023-09-23T20:46:13.090 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.090 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.090 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:46:13.090 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.713294+0000", 2023-09-23T20:46:13.090 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.26928485600000002 2023-09-23T20:46:13.090 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.090 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.090 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:46:13.090 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.713302+0000", 2023-09-23T20:46:13.091 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.4440000000000002e-06 2023-09-23T20:46:13.091 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.091 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.091 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:46:13.091 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.615633+0000", 2023-09-23T20:46:13.091 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.90233140000000001 2023-09-23T20:46:13.091 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.091 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.091 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:46:13.091 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.615759+0000", 2023-09-23T20:46:13.091 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00012542599999999999 2023-09-23T20:46:13.091 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.091 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.092 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,4", 2023-09-23T20:46:13.092 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.617918+0000", 2023-09-23T20:46:13.092 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0021595719999999998 2023-09-23T20:46:13.092 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.092 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.092 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:46:13.092 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.627197+0000", 2023-09-23T20:46:13.092 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0092789759999999995 2023-09-23T20:46:13.092 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.092 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.092 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:46:13.092 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.627353+0000", 2023-09-23T20:46:13.092 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00015556599999999999 2023-09-23T20:46:13.093 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.093 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.093 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:46:13.093 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.631674+0000", 2023-09-23T20:46:13.093 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0043214890000000004 2023-09-23T20:46:13.093 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.093 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.093 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:46:13.093 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.631719+0000", 2023-09-23T20:46:13.093 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.5314e-05 2023-09-23T20:46:13.093 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.093 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.093 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:46:13.094 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.633398+0000", 2023-09-23T20:46:13.094 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.001678368 2023-09-23T20:46:13.094 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:46:13.094 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:46:13.094 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:46:13.094 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.094 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.094 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.35994.0:105797 8.18 8:1843db78:::benchmark_data_smithi018_341490_object16474:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e410)", 2023-09-23T20:46:13.094 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:38:26.444014+0000", 2023-09-23T20:46:13.094 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 466.59185131200002, 2023-09-23T20:46:13.094 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.17448456, 2023-09-23T20:46:13.094 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:46:13.095 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:46:13.095 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:46:13.095 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.35994", 2023-09-23T20:46:13.095 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:46:13.095 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 105797 2023-09-23T20:46:13.095 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.095 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:46:13.095 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.095 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:46:13.095 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.444014+0000", 2023-09-23T20:46:13.095 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:46:13.095 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.095 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.096 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:46:13.096 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.444014+0000", 2023-09-23T20:46:13.096 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:46:13.096 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.096 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.096 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:46:13.096 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.444014+0000", 2023-09-23T20:46:13.096 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:46:13.096 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.096 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.096 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:46:13.096 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.444016+0000", 2023-09-23T20:46:13.096 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.097e-06 2023-09-23T20:46:13.097 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.097 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.097 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:46:13.097 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.444022+0000", 2023-09-23T20:46:13.097 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.8200000000000002e-06 2023-09-23T20:46:13.097 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.097 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.097 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:46:13.097 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.444027+0000", 2023-09-23T20:46:13.097 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.4360000000000001e-06 2023-09-23T20:46:13.097 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.097 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.097 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:46:13.098 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.713323+0000", 2023-09-23T20:46:13.098 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.26929555599999999 2023-09-23T20:46:13.098 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.098 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.098 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:46:13.098 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:26.713329+0000", 2023-09-23T20:46:13.098 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.9819999999999996e-06 2023-09-23T20:46:13.098 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.098 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.098 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:46:13.098 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.608546+0000", 2023-09-23T20:46:13.098 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.895217021 2023-09-23T20:46:13.099 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.099 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.099 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:46:13.099 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.608698+0000", 2023-09-23T20:46:13.099 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00015241000000000001 2023-09-23T20:46:13.099 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.099 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.099 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 0,7", 2023-09-23T20:46:13.099 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.608843+0000", 2023-09-23T20:46:13.099 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00014468799999999999 2023-09-23T20:46:13.099 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.099 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.100 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:46:13.100 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.613634+0000", 2023-09-23T20:46:13.100 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0047907979999999998 2023-09-23T20:46:13.100 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.100 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.100 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:46:13.100 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.615841+0000", 2023-09-23T20:46:13.100 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0022071690000000001 2023-09-23T20:46:13.100 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.100 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.100 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:46:13.100 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.617508+0000", 2023-09-23T20:46:13.100 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0016669390000000001 2023-09-23T20:46:13.101 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.101 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.101 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:46:13.101 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.617562+0000", 2023-09-23T20:46:13.101 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.3891999999999998e-05 2023-09-23T20:46:13.101 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.101 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.101 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:46:13.101 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:38:27.618499+0000", 2023-09-23T20:46:13.101 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00093712400000000001 2023-09-23T20:46:13.101 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:46:13.101 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:46:13.101 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:46:13.102 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.102 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.102 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.36450.0:185319 9.8 9.8673f918 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e503)", 2023-09-23T20:46:13.102 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:44:18.697532+0000", 2023-09-23T20:46:13.102 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 114.33833322, 2023-09-23T20:46:13.102 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.93022128999999998, 2023-09-23T20:46:13.102 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:46:13.102 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:46:13.102 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:46:13.102 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.36450", 2023-09-23T20:46:13.102 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:46:13.102 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 185319 2023-09-23T20:46:13.103 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.103 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:46:13.103 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.103 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:46:13.103 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.697532+0000", 2023-09-23T20:46:13.103 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:46:13.103 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.103 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.103 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:46:13.103 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.697532+0000", 2023-09-23T20:46:13.103 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:46:13.103 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.103 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.104 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:46:13.104 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.697532+0000", 2023-09-23T20:46:13.104 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:46:13.104 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.104 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.104 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:46:13.104 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.697535+0000", 2023-09-23T20:46:13.104 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.1709999999999998e-06 2023-09-23T20:46:13.104 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.104 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.104 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:46:13.104 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.697542+0000", 2023-09-23T20:46:13.105 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.7880000000000001e-06 2023-09-23T20:46:13.105 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.105 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.105 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:46:13.105 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.697547+0000", 2023-09-23T20:46:13.105 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.0159999999999999e-06 2023-09-23T20:46:13.105 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.105 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.105 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:46:13.105 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.698190+0000", 2023-09-23T20:46:13.105 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00064296299999999998 2023-09-23T20:46:13.105 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.106 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.106 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:46:13.106 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.698201+0000", 2023-09-23T20:46:13.106 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0898e-05 2023-09-23T20:46:13.106 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.106 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.106 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:46:13.106 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:19.627739+0000", 2023-09-23T20:46:13.106 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.92953824600000001 2023-09-23T20:46:13.106 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.106 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.106 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:46:13.107 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:19.627754+0000", 2023-09-23T20:46:13.107 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.4634000000000001e-05 2023-09-23T20:46:13.107 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:46:13.107 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:46:13.107 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:46:13.107 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.107 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.107 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.36450.0:185321 9.8 9.f3369438 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e503)", 2023-09-23T20:46:13.107 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:44:18.708465+0000", 2023-09-23T20:46:13.107 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 114.327401021, 2023-09-23T20:46:13.107 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.91932099599999995, 2023-09-23T20:46:13.107 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:46:13.107 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:46:13.107 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:46:13.108 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.36450", 2023-09-23T20:46:13.108 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:46:13.108 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 185321 2023-09-23T20:46:13.108 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.108 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:46:13.108 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.108 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:46:13.108 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.708465+0000", 2023-09-23T20:46:13.108 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:46:13.108 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.108 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.108 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:46:13.109 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.708465+0000", 2023-09-23T20:46:13.109 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:46:13.109 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.109 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.109 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:46:13.109 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.708463+0000", 2023-09-23T20:46:13.109 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:46:13.109 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.109 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.109 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:46:13.109 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.708482+0000", 2023-09-23T20:46:13.109 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.9258000000000001e-05 2023-09-23T20:46:13.109 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.110 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.110 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:46:13.110 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.708500+0000", 2023-09-23T20:46:13.110 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.7708000000000001e-05 2023-09-23T20:46:13.110 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.110 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.110 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:46:13.110 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.708510+0000", 2023-09-23T20:46:13.110 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.6190000000000006e-06 2023-09-23T20:46:13.110 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.110 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.110 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:46:13.111 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.708611+0000", 2023-09-23T20:46:13.111 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000101296 2023-09-23T20:46:13.111 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.111 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.111 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:46:13.111 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.708623+0000", 2023-09-23T20:46:13.111 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.201e-05 2023-09-23T20:46:13.111 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.111 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.111 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:46:13.111 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:19.627776+0000", 2023-09-23T20:46:13.111 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.919153161 2023-09-23T20:46:13.112 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.112 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.112 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:46:13.112 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:19.627786+0000", 2023-09-23T20:46:13.112 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.4369999999999996e-06 2023-09-23T20:46:13.112 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:46:13.112 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:46:13.112 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:46:13.112 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.112 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.112 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.36450.0:185330 9.8 9.d7918778 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e503)", 2023-09-23T20:46:13.112 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:44:18.717304+0000", 2023-09-23T20:46:13.112 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 114.31856141900001, 2023-09-23T20:46:13.113 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.91053041599999995, 2023-09-23T20:46:13.113 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:46:13.113 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:46:13.113 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:46:13.113 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.36450", 2023-09-23T20:46:13.113 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:46:13.113 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 185330 2023-09-23T20:46:13.113 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.113 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:46:13.113 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.113 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:46:13.113 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.717304+0000", 2023-09-23T20:46:13.114 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:46:13.114 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.114 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.114 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:46:13.114 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.717304+0000", 2023-09-23T20:46:13.114 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:46:13.114 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.114 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.114 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:46:13.114 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.717303+0000", 2023-09-23T20:46:13.115 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.999999 2023-09-23T20:46:13.115 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.115 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.115 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:46:13.115 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.717309+0000", 2023-09-23T20:46:13.115 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.6620000000000002e-06 2023-09-23T20:46:13.115 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.115 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.115 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:46:13.115 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.717319+0000", 2023-09-23T20:46:13.116 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.8970000000000003e-06 2023-09-23T20:46:13.116 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.116 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.116 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:46:13.116 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.717326+0000", 2023-09-23T20:46:13.116 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.5190000000000003e-06 2023-09-23T20:46:13.116 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.116 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.116 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:46:13.116 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.717379+0000", 2023-09-23T20:46:13.116 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.2589000000000001e-05 2023-09-23T20:46:13.116 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.116 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.117 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:46:13.117 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.717394+0000", 2023-09-23T20:46:13.117 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.4843e-05 2023-09-23T20:46:13.117 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.117 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.117 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:46:13.117 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:19.627825+0000", 2023-09-23T20:46:13.117 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.91043133300000001 2023-09-23T20:46:13.117 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.117 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.117 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:46:13.117 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:19.627835+0000", 2023-09-23T20:46:13.117 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.5319999999999995e-06 2023-09-23T20:46:13.118 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:46:13.118 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:46:13.118 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:46:13.118 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.118 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.118 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.36450.0:185334 9.8 9.99d653f8 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e503)", 2023-09-23T20:46:13.118 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:44:18.722946+0000", 2023-09-23T20:46:13.118 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 114.312920041, 2023-09-23T20:46:13.118 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.90492348099999997, 2023-09-23T20:46:13.118 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:46:13.118 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:46:13.118 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:46:13.119 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.36450", 2023-09-23T20:46:13.119 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:46:13.119 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 185334 2023-09-23T20:46:13.119 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.119 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:46:13.119 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.119 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:46:13.119 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.722946+0000", 2023-09-23T20:46:13.119 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:46:13.119 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.119 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.119 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:46:13.119 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.722946+0000", 2023-09-23T20:46:13.120 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:46:13.120 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.120 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.120 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:46:13.120 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.722944+0000", 2023-09-23T20:46:13.120 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:46:13.120 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.120 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.120 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:46:13.120 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.722951+0000", 2023-09-23T20:46:13.120 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.0520000000000004e-06 2023-09-23T20:46:13.120 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.121 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.121 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:46:13.121 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.722976+0000", 2023-09-23T20:46:13.121 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.5139000000000001e-05 2023-09-23T20:46:13.121 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.121 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.121 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:46:13.121 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.722985+0000", 2023-09-23T20:46:13.121 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.5869999999999993e-06 2023-09-23T20:46:13.121 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.121 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.121 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:46:13.122 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.724318+0000", 2023-09-23T20:46:13.122 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.001333356 2023-09-23T20:46:13.122 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.122 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.122 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:46:13.122 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.724329+0000", 2023-09-23T20:46:13.122 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0661e-05 2023-09-23T20:46:13.122 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.122 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.122 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:46:13.122 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:19.627859+0000", 2023-09-23T20:46:13.122 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.90353019099999998 2023-09-23T20:46:13.123 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.123 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.123 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:46:13.123 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:19.627869+0000", 2023-09-23T20:46:13.123 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.9019999999999994e-06 2023-09-23T20:46:13.123 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:46:13.123 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:46:13.123 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:46:13.123 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.123 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.123 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.36450.0:185337 9.8 9.430bfff8 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e503)", 2023-09-23T20:46:13.123 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:44:18.723328+0000", 2023-09-23T20:46:13.124 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 114.312537742, 2023-09-23T20:46:13.124 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.90457610099999997, 2023-09-23T20:46:13.124 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:46:13.124 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:46:13.124 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:46:13.124 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.36450", 2023-09-23T20:46:13.124 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:46:13.124 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 185337 2023-09-23T20:46:13.124 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.124 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:46:13.124 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.124 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:46:13.125 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.723328+0000", 2023-09-23T20:46:13.125 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:46:13.125 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.125 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.125 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:46:13.125 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.723328+0000", 2023-09-23T20:46:13.125 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:46:13.125 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.125 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.125 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:46:13.125 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.723327+0000", 2023-09-23T20:46:13.125 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.999999 2023-09-23T20:46:13.126 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.126 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.126 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:46:13.126 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.723332+0000", 2023-09-23T20:46:13.126 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.7079999999999996e-06 2023-09-23T20:46:13.126 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.126 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.126 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:46:13.126 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.723344+0000", 2023-09-23T20:46:13.126 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2031000000000001e-05 2023-09-23T20:46:13.126 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.127 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.127 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:46:13.127 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.723352+0000", 2023-09-23T20:46:13.127 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.8800000000000008e-06 2023-09-23T20:46:13.127 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.127 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.127 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:46:13.127 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.724351+0000", 2023-09-23T20:46:13.127 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00099955399999999994 2023-09-23T20:46:13.127 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.127 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.127 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:46:13.128 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.724359+0000", 2023-09-23T20:46:13.128 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.5449999999999998e-06 2023-09-23T20:46:13.128 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.128 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.128 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:46:13.128 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:19.627894+0000", 2023-09-23T20:46:13.128 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.90353528100000002 2023-09-23T20:46:13.128 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.128 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.128 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:46:13.128 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:19.627904+0000", 2023-09-23T20:46:13.128 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.8810000000000008e-06 2023-09-23T20:46:13.128 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:46:13.129 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:46:13.129 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:46:13.129 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.129 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.129 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.36450.0:185405 9.8 9.6fb15688 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e503)", 2023-09-23T20:46:13.129 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:44:18.813440+0000", 2023-09-23T20:46:13.129 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 114.222425345, 2023-09-23T20:46:13.129 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.81449727900000002, 2023-09-23T20:46:13.129 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:46:13.129 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:46:13.129 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:46:13.129 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.36450", 2023-09-23T20:46:13.129 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:46:13.130 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 185405 2023-09-23T20:46:13.130 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.130 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:46:13.130 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.130 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:46:13.130 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.813440+0000", 2023-09-23T20:46:13.130 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:46:13.130 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.130 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.130 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:46:13.130 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.813440+0000", 2023-09-23T20:46:13.130 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:46:13.131 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.131 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.131 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:46:13.131 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.813440+0000", 2023-09-23T20:46:13.131 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:46:13.131 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.131 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.131 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:46:13.131 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.813442+0000", 2023-09-23T20:46:13.131 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.2989999999999999e-06 2023-09-23T20:46:13.131 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.131 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.132 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:46:13.132 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.813450+0000", 2023-09-23T20:46:13.132 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.7689999999999996e-06 2023-09-23T20:46:13.132 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.132 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.132 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:46:13.132 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.813455+0000", 2023-09-23T20:46:13.132 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.4229999999999996e-06 2023-09-23T20:46:13.132 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.132 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.132 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:46:13.132 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.813525+0000", 2023-09-23T20:46:13.133 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.9319999999999994e-05 2023-09-23T20:46:13.133 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.133 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.133 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:46:13.133 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.813536+0000", 2023-09-23T20:46:13.133 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0865e-05 2023-09-23T20:46:13.133 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.133 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.133 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:46:13.133 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:19.627928+0000", 2023-09-23T20:46:13.133 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.81439254999999999 2023-09-23T20:46:13.133 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.134 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.134 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:46:13.134 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:19.627938+0000", 2023-09-23T20:46:13.134 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.4239999999999999e-06 2023-09-23T20:46:13.134 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:46:13.134 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:46:13.134 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:46:13.134 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.134 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.134 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.36450.0:185407 9.8 9.40f2f428 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e503)", 2023-09-23T20:46:13.134 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:44:18.816212+0000", 2023-09-23T20:46:13.134 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 114.21965312, 2023-09-23T20:46:13.135 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.81175977700000002, 2023-09-23T20:46:13.135 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:46:13.135 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:46:13.135 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:46:13.135 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.36450", 2023-09-23T20:46:13.135 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:46:13.135 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 185407 2023-09-23T20:46:13.135 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.135 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:46:13.135 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.135 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:46:13.135 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.816212+0000", 2023-09-23T20:46:13.135 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:46:13.136 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.136 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.136 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:46:13.136 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.816212+0000", 2023-09-23T20:46:13.136 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:46:13.136 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.136 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.136 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:46:13.136 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.816211+0000", 2023-09-23T20:46:13.136 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.999999 2023-09-23T20:46:13.136 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.136 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.137 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:46:13.137 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.816217+0000", 2023-09-23T20:46:13.137 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.4809999999999997e-06 2023-09-23T20:46:13.137 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.137 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.137 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:46:13.137 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.816230+0000", 2023-09-23T20:46:13.137 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2943e-05 2023-09-23T20:46:13.137 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.137 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.137 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:46:13.137 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.816237+0000", 2023-09-23T20:46:13.137 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.4800000000000004e-06 2023-09-23T20:46:13.138 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.138 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.138 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:46:13.138 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.816296+0000", 2023-09-23T20:46:13.138 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.8839999999999999e-05 2023-09-23T20:46:13.138 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.138 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.138 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:46:13.138 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.816310+0000", 2023-09-23T20:46:13.138 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3747999999999999e-05 2023-09-23T20:46:13.138 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.138 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.138 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:46:13.139 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:19.627962+0000", 2023-09-23T20:46:13.139 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.81165225699999999 2023-09-23T20:46:13.139 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.139 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.139 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:46:13.139 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:19.627972+0000", 2023-09-23T20:46:13.139 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0132000000000001e-05 2023-09-23T20:46:13.139 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:46:13.139 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:46:13.139 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:46:13.139 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.139 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.139 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.36450.0:185417 9.8 9.6b5d6608 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e503)", 2023-09-23T20:46:13.140 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:44:18.825896+0000", 2023-09-23T20:46:13.140 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 114.209969962, 2023-09-23T20:46:13.140 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.802110306, 2023-09-23T20:46:13.140 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:46:13.140 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:46:13.140 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:46:13.140 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.36450", 2023-09-23T20:46:13.140 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:46:13.140 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 185417 2023-09-23T20:46:13.140 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.140 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:46:13.140 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.140 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:46:13.141 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.825896+0000", 2023-09-23T20:46:13.141 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:46:13.141 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.141 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.141 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:46:13.141 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.825896+0000", 2023-09-23T20:46:13.141 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:46:13.141 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.141 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.141 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:46:13.141 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.825895+0000", 2023-09-23T20:46:13.141 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.999999 2023-09-23T20:46:13.141 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.142 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.142 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:46:13.142 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.825899+0000", 2023-09-23T20:46:13.142 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.6700000000000002e-06 2023-09-23T20:46:13.142 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.142 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.142 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:46:13.142 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.825911+0000", 2023-09-23T20:46:13.142 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1284000000000001e-05 2023-09-23T20:46:13.142 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.142 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.142 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:46:13.142 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.825916+0000", 2023-09-23T20:46:13.143 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.5600000000000001e-06 2023-09-23T20:46:13.143 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.143 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.143 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:46:13.143 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.825968+0000", 2023-09-23T20:46:13.143 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.1684999999999997e-05 2023-09-23T20:46:13.143 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.143 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.143 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:46:13.143 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.825984+0000", 2023-09-23T20:46:13.143 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.6416e-05 2023-09-23T20:46:13.143 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.143 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.144 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:46:13.144 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:19.627996+0000", 2023-09-23T20:46:13.144 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.80201148200000005 2023-09-23T20:46:13.144 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.144 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.144 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:46:13.144 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:19.628006+0000", 2023-09-23T20:46:13.144 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0169999999999999e-05 2023-09-23T20:46:13.144 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:46:13.144 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:46:13.144 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:46:13.144 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.144 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.145 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.36450.0:185425 9.8 9.55322cb8 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e503)", 2023-09-23T20:46:13.145 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:44:18.834145+0000", 2023-09-23T20:46:13.145 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 114.20172053, 2023-09-23T20:46:13.145 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.79389570600000003, 2023-09-23T20:46:13.145 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:46:13.145 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:46:13.145 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:46:13.145 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.36450", 2023-09-23T20:46:13.145 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:46:13.145 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 185425 2023-09-23T20:46:13.145 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.145 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:46:13.145 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.146 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:46:13.146 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.834145+0000", 2023-09-23T20:46:13.146 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:46:13.146 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.146 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.146 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:46:13.146 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.834145+0000", 2023-09-23T20:46:13.146 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:46:13.146 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.146 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.146 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:46:13.146 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.834144+0000", 2023-09-23T20:46:13.147 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.999999 2023-09-23T20:46:13.147 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.147 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.147 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:46:13.147 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.834150+0000", 2023-09-23T20:46:13.147 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.6799999999999998e-06 2023-09-23T20:46:13.147 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.147 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.147 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:46:13.147 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.834166+0000", 2023-09-23T20:46:13.147 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.6228000000000001e-05 2023-09-23T20:46:13.147 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.147 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.148 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:46:13.148 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.834176+0000", 2023-09-23T20:46:13.148 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0032e-05 2023-09-23T20:46:13.148 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.148 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.148 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:46:13.148 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.834242+0000", 2023-09-23T20:46:13.148 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.5899e-05 2023-09-23T20:46:13.148 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.148 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.148 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:46:13.148 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.834252+0000", 2023-09-23T20:46:13.148 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0525e-05 2023-09-23T20:46:13.149 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.149 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.149 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:46:13.149 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:19.628031+0000", 2023-09-23T20:46:13.149 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.79377829300000002 2023-09-23T20:46:13.149 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.149 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.149 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:46:13.149 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:19.628041+0000", 2023-09-23T20:46:13.149 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0186999999999999e-05 2023-09-23T20:46:13.149 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:46:13.149 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:46:13.150 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:46:13.150 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.150 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.150 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.36450.0:185429 9.8 9.f4d7f968 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e503)", 2023-09-23T20:46:13.150 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:44:18.837932+0000", 2023-09-23T20:46:13.150 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 114.19793328, 2023-09-23T20:46:13.150 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.79014410400000001, 2023-09-23T20:46:13.150 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:46:13.150 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:46:13.150 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:46:13.150 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.36450", 2023-09-23T20:46:13.150 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:46:13.151 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 185429 2023-09-23T20:46:13.151 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.151 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:46:13.151 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.151 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:46:13.151 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.837932+0000", 2023-09-23T20:46:13.151 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:46:13.151 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.151 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.151 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:46:13.151 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.837932+0000", 2023-09-23T20:46:13.151 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:46:13.152 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.152 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.152 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:46:13.152 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.837931+0000", 2023-09-23T20:46:13.152 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.999999 2023-09-23T20:46:13.152 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.152 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.152 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:46:13.152 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.837936+0000", 2023-09-23T20:46:13.152 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.0159999999999999e-06 2023-09-23T20:46:13.152 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.152 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.152 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:46:13.153 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.837948+0000", 2023-09-23T20:46:13.153 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1338999999999999e-05 2023-09-23T20:46:13.153 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.153 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.153 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:46:13.153 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.837953+0000", 2023-09-23T20:46:13.153 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.5090000000000001e-06 2023-09-23T20:46:13.153 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.153 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.153 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:46:13.153 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.838010+0000", 2023-09-23T20:46:13.153 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.6254e-05 2023-09-23T20:46:13.154 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.154 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.154 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:46:13.154 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.838022+0000", 2023-09-23T20:46:13.154 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2218000000000001e-05 2023-09-23T20:46:13.154 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.154 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.154 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:46:13.154 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:19.628066+0000", 2023-09-23T20:46:13.154 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.79004441299999995 2023-09-23T20:46:13.154 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.154 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.155 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:46:13.155 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:19.628076+0000", 2023-09-23T20:46:13.155 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0233e-05 2023-09-23T20:46:13.155 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:46:13.155 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:46:13.155 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:46:13.155 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.155 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.155 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.36450.0:185447 9.8 9.43c58ee8 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e503)", 2023-09-23T20:46:13.155 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:44:19.023152+0000", 2023-09-23T20:46:13.155 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 114.01271382500001, 2023-09-23T20:46:13.155 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.60496012899999996, 2023-09-23T20:46:13.155 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:46:13.156 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:46:13.156 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:46:13.156 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.36450", 2023-09-23T20:46:13.156 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:46:13.156 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 185447 2023-09-23T20:46:13.156 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.156 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:46:13.156 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.156 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:46:13.156 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:19.023152+0000", 2023-09-23T20:46:13.156 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:46:13.156 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.157 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.157 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:46:13.157 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:19.023152+0000", 2023-09-23T20:46:13.157 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:46:13.157 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.157 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.157 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:46:13.157 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:19.023151+0000", 2023-09-23T20:46:13.157 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.999999 2023-09-23T20:46:13.157 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.157 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.157 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:46:13.157 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:19.023156+0000", 2023-09-23T20:46:13.157 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.1939999999999997e-06 2023-09-23T20:46:13.158 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.158 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.158 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:46:13.158 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:19.023168+0000", 2023-09-23T20:46:13.158 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2553e-05 2023-09-23T20:46:13.158 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.158 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.158 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:46:13.158 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:19.023175+0000", 2023-09-23T20:46:13.158 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.2870000000000001e-06 2023-09-23T20:46:13.158 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.158 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.159 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:46:13.159 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:19.023224+0000", 2023-09-23T20:46:13.159 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.9270000000000001e-05 2023-09-23T20:46:13.159 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.159 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.159 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:46:13.159 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:19.023234+0000", 2023-09-23T20:46:13.159 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.008e-05 2023-09-23T20:46:13.159 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.159 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.159 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:46:13.159 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:19.628102+0000", 2023-09-23T20:46:13.159 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.60486762800000005 2023-09-23T20:46:13.160 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.160 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.160 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:46:13.160 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:19.628112+0000", 2023-09-23T20:46:13.160 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0179e-05 2023-09-23T20:46:13.160 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:46:13.160 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:46:13.160 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:46:13.160 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.160 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.160 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.36450.0:185449 9.8 9.5dae5428 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e503)", 2023-09-23T20:46:13.160 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:44:19.029317+0000", 2023-09-23T20:46:13.161 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 114.006548212, 2023-09-23T20:46:13.161 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.59882766799999998, 2023-09-23T20:46:13.161 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:46:13.161 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:46:13.161 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:46:13.161 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.36450", 2023-09-23T20:46:13.161 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:46:13.161 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 185449 2023-09-23T20:46:13.161 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.161 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:46:13.161 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.161 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:46:13.161 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:19.029317+0000", 2023-09-23T20:46:13.162 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:46:13.162 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.162 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.162 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:46:13.162 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:19.029317+0000", 2023-09-23T20:46:13.162 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:46:13.162 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.162 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.162 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:46:13.162 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:19.029316+0000", 2023-09-23T20:46:13.162 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:46:13.162 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.163 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.163 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:46:13.163 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:19.029324+0000", 2023-09-23T20:46:13.163 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.4780000000000008e-06 2023-09-23T20:46:13.163 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.163 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.163 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:46:13.163 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:19.029341+0000", 2023-09-23T20:46:13.163 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.7090000000000001e-05 2023-09-23T20:46:13.163 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.163 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.163 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:46:13.164 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:19.029351+0000", 2023-09-23T20:46:13.164 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.594e-06 2023-09-23T20:46:13.164 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.164 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.164 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:46:13.164 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:19.029422+0000", 2023-09-23T20:46:13.164 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.0914000000000006e-05 2023-09-23T20:46:13.164 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.164 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.164 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:46:13.164 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:19.029437+0000", 2023-09-23T20:46:13.164 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.521e-05 2023-09-23T20:46:13.164 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.165 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.165 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:46:13.165 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:19.628135+0000", 2023-09-23T20:46:13.165 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.59869824900000002 2023-09-23T20:46:13.165 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:46:13.165 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:46:13.165 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:46:13.165 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:19.628145+0000", 2023-09-23T20:46:13.165 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.6209999999999999e-06 2023-09-23T20:46:13.165 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:46:13.165 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:46:13.165 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:46:13.166 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:46:13.166 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:46:13.166 INFO:teuthology.orchestra.run.smithi018.stdout:} 2023-09-23T20:46:13.166 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:46:13.218 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:13.294 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:46:13.300 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:13.301 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:46:13.401 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:46:13.468 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:13.547 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:46:13.552 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:13.553 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:46:13.653 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:46:13.714 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:13.777 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:46:13.784 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:13.784 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:46:13.885 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:46:13.887 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:46:13.924 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:46:13.926 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:13 smithi149 ceph-mon[170394]: pgmap v1357: 119 pgs: 119 active+clean; 3.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 95 MiB/s wr, 1.53k op/s 2023-09-23T20:46:13.936 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:13.959 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:46:13.985 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:46:13.989 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:13.989 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:46:13.990 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 29 16 55010 54994 118.471 82.625 0.0176133 0.00843379 2023-09-23T20:46:13.998 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:13 smithi018 ceph-mon[179647]: pgmap v1357: 119 pgs: 119 active+clean; 3.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 95 MiB/s wr, 1.53k op/s 2023-09-23T20:46:13.998 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:13 smithi018 ceph-mon[184144]: pgmap v1357: 119 pgs: 119 active+clean; 3.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 95 MiB/s wr, 1.53k op/s 2023-09-23T20:46:13.999 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:46:14.034 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:46:14.076 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:46:14.090 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:46:14.116 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:46:14.136 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:14.153 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:46:14.155 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:46:14.155 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:46:14.155 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:46:14.155 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:46:14.155 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:46:14.155 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:46:14.155 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:46:14.155 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:46:14.194 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:46:14.203 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:14.204 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:46:14.304 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:46:14.373 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:14.432 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:46:14.448 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:46:14.458 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:14.458 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:46:14.499 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T20:46:14.559 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:46:14.622 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:14.713 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:46:14.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:14 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3756738628' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:46:14.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:14 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:46:14.720 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:14.720 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:46:14.821 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:46:14.874 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:14.940 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:46:14.945 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:14.946 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:46:14.993 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 30 16 56276 56260 117.149 79.125 0.0105876 0.00853005 2023-09-23T20:46:14.995 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:14 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3756738628' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:46:14.995 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:14 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:46:14.996 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:14 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3756738628' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:46:14.996 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:14 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:46:14.996 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:46:14 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:46:14] "GET /metrics HTTP/1.1" 200 36572 "" "Prometheus/2.43.0" 2023-09-23T20:46:15.046 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:46:15.084 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:15.153 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:46:15.159 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:15.159 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:46:15.260 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:46:15.307 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:46:15.327 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:15.498 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:46:15.506 INFO:teuthology.orchestra.run.smithi018.stderr:dumped all 2023-09-23T20:46:15.843 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:46:15.852 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:15 smithi018 ceph-mon[179647]: pgmap v1358: 119 pgs: 119 active+clean; 3.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 94 MiB/s wr, 1.50k op/s 2023-09-23T20:46:15.853 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:15 smithi018 ceph-mon[184144]: pgmap v1358: 119 pgs: 119 active+clean; 3.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 94 MiB/s wr, 1.50k op/s 2023-09-23T20:46:15.854 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:15.854 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:46:15.863 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:15 smithi149 ceph-mon[170394]: pgmap v1358: 119 pgs: 119 active+clean; 3.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 94 MiB/s wr, 1.50k op/s 2023-09-23T20:46:15.956 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:46:15.993 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 31 16 57530 57514 115.898 78.375 0.0029605 0.00861459 2023-09-23T20:46:15.999 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:16.218 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:46:16.224 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:16.224 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:46:16.325 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:46:16.403 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:16.532 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:46:16.536 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:16.537 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:46:16.637 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:46:16.719 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:16.759 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:46:16.886 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:46:16.890 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:16 smithi149 ceph-mon[170394]: from='client.46655 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:46:16.891 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:16.891 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:46:16.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:16 smithi018 ceph-mon[179647]: from='client.46655 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:46:16.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:16 smithi018 ceph-mon[184144]: from='client.46655 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:46:16.993 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:46:16.994 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 32 16 58789 58773 114.736 78.6875 0.00825248 0.00871118 2023-09-23T20:46:17.044 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:17.057 INFO:teuthology.orchestra.run.smithi149.stdout:ERROR: (22) Invalid argument 2023-09-23T20:46:17.058 INFO:teuthology.orchestra.run.smithi149.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:46:17.109 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:46:17.114 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:17.114 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:46:17.169 INFO:teuthology.orchestra.run.smithi018.stdout:{"pg_ready":true,"pg_map":{"version":1358,"stamp":"2023-09-23T20:46:14.249900+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3383695283,"num_objects":51586,"num_object_clones":0,"num_object_copies":154758,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":51586,"num_whiteouts":0,"num_read":5983,"num_read_kb":13200,"num_write":52465,"num_write_kb":3312437,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":174,"num_bytes_recovered":16795702,"num_keys_recovered":40,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":57580,"ondisk_log_size":57580,"up":357,"acting":357,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":357,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":15044800,"kb_used_data":9814592,"kb_used_omap":32,"kb_used_meta":5230111,"kb_avail":734916416,"statfs":{"total":767960285184,"available":752554409984,"internally_reserved":0,"allocated":10050142208,"data_stored":10033665430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32773,"internal_metadata":5355634683},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[9,1,8,9,4,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":985595904,"num_objects":15039,"num_object_clones":0,"num_object_copies":45117,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":15039,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15039,"num_write_kb":962496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.004702"},"pg_stats":[{"pgid":"1.18","version":"90'161","reported_seq":584,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-09-23T20:46:00.623325+0000","last_change":"2023-09-23T20:44:19.677357+0000","last_active":"2023-09-23T20:46:00.623325+0000","last_peered":"2023-09-23T20:46:00.623325+0000","last_clean":"2023-09-23T20:46:00.623325+0000","last_became_active":"2023-09-23T20:43:16.766305+0000","last_became_peered":"2023-09-23T20:43:16.766305+0000","last_unstale":"2023-09-23T20:46:00.623325+0000","last_undegraded":"2023-09-23T20:46:00.623325+0000","last_fullsized":"2023-09-23T20:46:00.623325+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:19.677209+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:19.677209+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:14:06.510115+0000","scrub_duration":0.0023292260000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"69'1","reported_seq":235,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-09-23T20:46:00.613941+0000","last_change":"2023-09-23T20:32:33.233853+0000","last_active":"2023-09-23T20:46:00.613941+0000","last_peered":"2023-09-23T20:46:00.613941+0000","last_clean":"2023-09-23T20:46:00.613941+0000","last_became_active":"2023-09-23T20:32:33.233546+0000","last_became_peered":"2023-09-23T20:32:33.233546+0000","last_unstale":"2023-09-23T20:46:00.613941+0000","last_undegraded":"2023-09-23T20:46:00.613941+0000","last_fullsized":"2023-09-23T20:46:00.613941+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:50:27.130808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":353,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-09-23T20:46:00.625109+0000","last_change":"2023-09-23T20:37:43.579611+0000","last_active":"2023-09-23T20:46:00.625109+0000","last_peered":"2023-09-23T20:46:00.625109+0000","last_clean":"2023-09-23T20:46:00.625109+0000","last_became_active":"2023-09-23T20:37:43.579323+0000","last_became_peered":"2023-09-23T20:37:43.579323+0000","last_unstale":"2023-09-23T20:46:00.625109+0000","last_undegraded":"2023-09-23T20:46:00.625109+0000","last_fullsized":"2023-09-23T20:46:00.625109+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:17:32.311427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"90'161","reported_seq":659,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-09-23T20:46:00.613637+0000","last_change":"2023-09-23T20:44:14.647447+0000","last_active":"2023-09-23T20:46:00.613637+0000","last_peered":"2023-09-23T20:46:00.613637+0000","last_clean":"2023-09-23T20:46:00.613637+0000","last_became_active":"2023-09-23T20:43:16.756116+0000","last_became_peered":"2023-09-23T20:43:16.756116+0000","last_unstale":"2023-09-23T20:46:00.613637+0000","last_undegraded":"2023-09-23T20:46:00.613637+0000","last_fullsized":"2023-09-23T20:46:00.613637+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:14.647386+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:36:05.442288+0000","last_clean_scrub_stamp":"2023-09-23T20:44:14.647386+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:25:25.151100+0000","scrub_duration":0.0097381210000000006,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":508,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-09-23T20:46:00.619755+0000","last_change":"2023-09-23T20:34:59.983035+0000","last_active":"2023-09-23T20:46:00.619755+0000","last_peered":"2023-09-23T20:46:00.619755+0000","last_clean":"2023-09-23T20:46:00.619755+0000","last_became_active":"2023-09-23T20:34:59.982459+0000","last_became_peered":"2023-09-23T20:34:59.982459+0000","last_unstale":"2023-09-23T20:46:00.619755+0000","last_undegraded":"2023-09-23T20:46:00.619755+0000","last_fullsized":"2023-09-23T20:46:00.619755+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:16:48.510783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":496,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-09-23T20:46:00.618819+0000","last_change":"2023-09-23T20:43:46.521939+0000","last_active":"2023-09-23T20:46:00.618819+0000","last_peered":"2023-09-23T20:46:00.618819+0000","last_clean":"2023-09-23T20:46:00.618819+0000","last_became_active":"2023-09-23T20:43:46.521185+0000","last_became_peered":"2023-09-23T20:43:46.521185+0000","last_unstale":"2023-09-23T20:46:00.618819+0000","last_undegraded":"2023-09-23T20:46:00.618819+0000","last_fullsized":"2023-09-23T20:46:00.618819+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:59:25.045627+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"90'161","reported_seq":617,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-09-23T20:46:00.613540+0000","last_change":"2023-09-23T20:36:01.292103+0000","last_active":"2023-09-23T20:46:00.613540+0000","last_peered":"2023-09-23T20:46:00.613540+0000","last_clean":"2023-09-23T20:46:00.613540+0000","last_became_active":"2023-09-23T20:35:26.639912+0000","last_became_peered":"2023-09-23T20:35:26.639912+0000","last_unstale":"2023-09-23T20:46:00.613540+0000","last_undegraded":"2023-09-23T20:46:00.613540+0000","last_fullsized":"2023-09-23T20:46:00.613540+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:01.292032+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:01.292032+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:04:53.926640+0000","scrub_duration":0.0032983159999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":333,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-09-23T20:46:00.625250+0000","last_change":"2023-09-23T20:37:43.584414+0000","last_active":"2023-09-23T20:46:00.625250+0000","last_peered":"2023-09-23T20:46:00.625250+0000","last_clean":"2023-09-23T20:46:00.625250+0000","last_became_active":"2023-09-23T20:37:43.584116+0000","last_became_peered":"2023-09-23T20:37:43.584116+0000","last_unstale":"2023-09-23T20:46:00.625250+0000","last_undegraded":"2023-09-23T20:46:00.625250+0000","last_fullsized":"2023-09-23T20:46:00.625250+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:17:54.258172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":153,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-09-23T20:46:00.619213+0000","last_change":"2023-09-23T20:38:37.457427+0000","last_active":"2023-09-23T20:46:00.619213+0000","last_peered":"2023-09-23T20:46:00.619213+0000","last_clean":"2023-09-23T20:46:00.619213+0000","last_became_active":"2023-09-23T20:38:37.457099+0000","last_became_peered":"2023-09-23T20:38:37.457099+0000","last_unstale":"2023-09-23T20:46:00.619213+0000","last_undegraded":"2023-09-23T20:46:00.619213+0000","last_fullsized":"2023-09-23T20:46:00.619213+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:31:27.084994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"90'161","reported_seq":750,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-09-23T20:46:00.618733+0000","last_change":"2023-09-23T20:40:22.449754+0000","last_active":"2023-09-23T20:46:00.618733+0000","last_peered":"2023-09-23T20:46:00.618733+0000","last_clean":"2023-09-23T20:46:00.618733+0000","last_became_active":"2023-09-23T20:40:22.448838+0000","last_became_peered":"2023-09-23T20:40:22.448838+0000","last_unstale":"2023-09-23T20:46:00.618733+0000","last_undegraded":"2023-09-23T20:46:00.618733+0000","last_fullsized":"2023-09-23T20:46:00.618733+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:03.411025+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:03.411025+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:22:07.948856+0000","scrub_duration":0.001802922,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":529,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-09-23T20:46:08.660255+0000","last_change":"2023-09-23T20:43:46.522325+0000","last_active":"2023-09-23T20:46:08.660255+0000","last_peered":"2023-09-23T20:46:08.660255+0000","last_clean":"2023-09-23T20:46:08.660255+0000","last_became_active":"2023-09-23T20:43:46.521996+0000","last_became_peered":"2023-09-23T20:43:46.521996+0000","last_unstale":"2023-09-23T20:46:08.660255+0000","last_undegraded":"2023-09-23T20:46:08.660255+0000","last_fullsized":"2023-09-23T20:46:08.660255+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:51:17.629956+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":264,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-09-23T20:46:03.111122+0000","last_change":"2023-09-23T20:38:02.579343+0000","last_active":"2023-09-23T20:46:03.111122+0000","last_peered":"2023-09-23T20:46:03.111122+0000","last_clean":"2023-09-23T20:46:03.111122+0000","last_became_active":"2023-09-23T20:38:02.579047+0000","last_became_peered":"2023-09-23T20:38:02.579047+0000","last_unstale":"2023-09-23T20:46:03.111122+0000","last_undegraded":"2023-09-23T20:46:03.111122+0000","last_fullsized":"2023-09-23T20:46:03.111122+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:44:03.166556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.14","version":"90'161","reported_seq":869,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-09-23T20:46:00.623906+0000","last_change":"2023-09-23T20:35:02.236398+0000","last_active":"2023-09-23T20:46:00.623906+0000","last_peered":"2023-09-23T20:46:00.623906+0000","last_clean":"2023-09-23T20:46:00.623906+0000","last_became_active":"2023-09-23T20:35:02.235988+0000","last_became_peered":"2023-09-23T20:35:02.235988+0000","last_unstale":"2023-09-23T20:46:00.623906+0000","last_undegraded":"2023-09-23T20:46:00.623906+0000","last_fullsized":"2023-09-23T20:46:00.623906+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:47.479012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:47.479012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:13.519648+0000","scrub_duration":0.0020136590000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":374,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-09-23T20:46:00.619357+0000","last_change":"2023-09-23T20:43:16.780478+0000","last_active":"2023-09-23T20:46:00.619357+0000","last_peered":"2023-09-23T20:46:00.619357+0000","last_clean":"2023-09-23T20:46:00.619357+0000","last_became_active":"2023-09-23T20:43:16.780183+0000","last_became_peered":"2023-09-23T20:43:16.780183+0000","last_unstale":"2023-09-23T20:46:00.619357+0000","last_undegraded":"2023-09-23T20:46:00.619357+0000","last_fullsized":"2023-09-23T20:46:00.619357+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:38:30.102066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":364,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-09-23T20:46:00.624970+0000","last_change":"2023-09-23T20:38:02.542034+0000","last_active":"2023-09-23T20:46:00.624970+0000","last_peered":"2023-09-23T20:46:00.624970+0000","last_clean":"2023-09-23T20:46:00.624970+0000","last_became_active":"2023-09-23T20:38:02.541575+0000","last_became_peered":"2023-09-23T20:38:02.541575+0000","last_unstale":"2023-09-23T20:46:00.624970+0000","last_undegraded":"2023-09-23T20:46:00.624970+0000","last_fullsized":"2023-09-23T20:46:00.624970+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:50:54.591529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"90'161","reported_seq":664,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-09-23T20:46:08.659997+0000","last_change":"2023-09-23T20:44:14.669973+0000","last_active":"2023-09-23T20:46:08.659997+0000","last_peered":"2023-09-23T20:46:08.659997+0000","last_clean":"2023-09-23T20:46:08.659997+0000","last_became_active":"2023-09-23T20:43:46.519592+0000","last_became_peered":"2023-09-23T20:43:46.519592+0000","last_unstale":"2023-09-23T20:46:08.659997+0000","last_undegraded":"2023-09-23T20:46:08.659997+0000","last_fullsized":"2023-09-23T20:46:08.659997+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:14.669831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:14.669831+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:56:28.076617+0000","scrub_duration":0.0083441460000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":565,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-09-23T20:46:00.618977+0000","last_change":"2023-09-23T20:32:33.270617+0000","last_active":"2023-09-23T20:46:00.618977+0000","last_peered":"2023-09-23T20:46:00.618977+0000","last_clean":"2023-09-23T20:46:00.618977+0000","last_became_active":"2023-09-23T20:32:33.227902+0000","last_became_peered":"2023-09-23T20:32:33.227902+0000","last_unstale":"2023-09-23T20:46:00.618977+0000","last_undegraded":"2023-09-23T20:46:00.618977+0000","last_fullsized":"2023-09-23T20:46:00.618977+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:30:26.847831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":360,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":52,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-09-23T20:46:00.620194+0000","last_change":"2023-09-23T20:44:10.611381+0000","last_active":"2023-09-23T20:46:00.620194+0000","last_peered":"2023-09-23T20:46:00.620194+0000","last_clean":"2023-09-23T20:46:00.620194+0000","last_became_active":"2023-09-23T20:44:10.611091+0000","last_became_peered":"2023-09-23T20:44:10.611091+0000","last_unstale":"2023-09-23T20:46:00.620194+0000","last_undegraded":"2023-09-23T20:46:00.620194+0000","last_fullsized":"2023-09-23T20:46:00.620194+0000","mapping_epoch":496,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":497,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:00:30.162478+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"90'161","reported_seq":664,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-09-23T20:46:00.621305+0000","last_change":"2023-09-23T20:36:02.548387+0000","last_active":"2023-09-23T20:46:00.621305+0000","last_peered":"2023-09-23T20:46:00.621305+0000","last_clean":"2023-09-23T20:46:00.621305+0000","last_became_active":"2023-09-23T20:35:26.644787+0000","last_became_peered":"2023-09-23T20:35:26.644787+0000","last_unstale":"2023-09-23T20:46:00.621305+0000","last_undegraded":"2023-09-23T20:46:00.621305+0000","last_fullsized":"2023-09-23T20:46:00.621305+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:02.548285+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:02.548285+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:37:15.216553+0000","scrub_duration":0.0015514459999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":285,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-09-23T20:46:00.615495+0000","last_change":"2023-09-23T20:38:02.560646+0000","last_active":"2023-09-23T20:46:00.615495+0000","last_peered":"2023-09-23T20:46:00.615495+0000","last_clean":"2023-09-23T20:46:00.615495+0000","last_became_active":"2023-09-23T20:38:02.539814+0000","last_became_peered":"2023-09-23T20:38:02.539814+0000","last_unstale":"2023-09-23T20:46:00.615495+0000","last_undegraded":"2023-09-23T20:46:00.615495+0000","last_fullsized":"2023-09-23T20:46:00.615495+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:00:09.435023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":265,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-09-23T20:46:00.614162+0000","last_change":"2023-09-23T20:38:02.579222+0000","last_active":"2023-09-23T20:46:00.614162+0000","last_peered":"2023-09-23T20:46:00.614162+0000","last_clean":"2023-09-23T20:46:00.614162+0000","last_became_active":"2023-09-23T20:38:02.578813+0000","last_became_peered":"2023-09-23T20:38:02.578813+0000","last_unstale":"2023-09-23T20:46:00.614162+0000","last_undegraded":"2023-09-23T20:46:00.614162+0000","last_fullsized":"2023-09-23T20:46:00.614162+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:57:31.080027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"90'161","reported_seq":792,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-09-23T20:46:06.437446+0000","last_change":"2023-09-23T20:43:46.544727+0000","last_active":"2023-09-23T20:46:06.437446+0000","last_peered":"2023-09-23T20:46:06.437446+0000","last_clean":"2023-09-23T20:46:06.437446+0000","last_became_active":"2023-09-23T20:43:46.543952+0000","last_became_peered":"2023-09-23T20:43:46.543952+0000","last_unstale":"2023-09-23T20:46:06.437446+0000","last_undegraded":"2023-09-23T20:46:06.437446+0000","last_fullsized":"2023-09-23T20:46:06.437446+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:43.578666+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:43.578666+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:11:16.611920+0000","scrub_duration":0.0021890379999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":564,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-09-23T20:46:00.616616+0000","last_change":"2023-09-23T20:32:33.242943+0000","last_active":"2023-09-23T20:46:00.616616+0000","last_peered":"2023-09-23T20:46:00.616616+0000","last_clean":"2023-09-23T20:46:00.616616+0000","last_became_active":"2023-09-23T20:32:33.225468+0000","last_became_peered":"2023-09-23T20:32:33.225468+0000","last_unstale":"2023-09-23T20:46:00.616616+0000","last_undegraded":"2023-09-23T20:46:00.616616+0000","last_fullsized":"2023-09-23T20:46:00.616616+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:40:47.867573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":284,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-09-23T20:46:00.623936+0000","last_change":"2023-09-23T20:45:01.875931+0000","last_active":"2023-09-23T20:46:00.623936+0000","last_peered":"2023-09-23T20:46:00.623936+0000","last_clean":"2023-09-23T20:46:00.623936+0000","last_became_active":"2023-09-23T20:45:01.872839+0000","last_became_peered":"2023-09-23T20:45:01.872839+0000","last_unstale":"2023-09-23T20:46:00.623936+0000","last_undegraded":"2023-09-23T20:46:00.623936+0000","last_fullsized":"2023-09-23T20:46:00.623936+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:15:09.584507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"90'161","reported_seq":818,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-09-23T20:46:00.614528+0000","last_change":"2023-09-23T20:44:20.041766+0000","last_active":"2023-09-23T20:46:00.614528+0000","last_peered":"2023-09-23T20:46:00.614528+0000","last_clean":"2023-09-23T20:46:00.614528+0000","last_became_active":"2023-09-23T20:43:16.765015+0000","last_became_peered":"2023-09-23T20:43:16.765015+0000","last_unstale":"2023-09-23T20:46:00.614528+0000","last_undegraded":"2023-09-23T20:46:00.614528+0000","last_fullsized":"2023-09-23T20:46:00.614528+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:20.041690+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:20.041690+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:33:30.150109+0000","scrub_duration":0.0026180330000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":494,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-09-23T20:46:06.437498+0000","last_change":"2023-09-23T20:43:46.521294+0000","last_active":"2023-09-23T20:46:06.437498+0000","last_peered":"2023-09-23T20:46:06.437498+0000","last_clean":"2023-09-23T20:46:06.437498+0000","last_became_active":"2023-09-23T20:43:46.521001+0000","last_became_peered":"2023-09-23T20:43:46.521001+0000","last_unstale":"2023-09-23T20:46:06.437498+0000","last_undegraded":"2023-09-23T20:46:06.437498+0000","last_fullsized":"2023-09-23T20:46:06.437498+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:01:35.126716+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":305,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-09-23T20:46:00.620081+0000","last_change":"2023-09-23T20:44:01.841565+0000","last_active":"2023-09-23T20:46:00.620081+0000","last_peered":"2023-09-23T20:46:00.620081+0000","last_clean":"2023-09-23T20:46:00.620081+0000","last_became_active":"2023-09-23T20:44:01.841255+0000","last_became_peered":"2023-09-23T20:44:01.841255+0000","last_unstale":"2023-09-23T20:46:00.620081+0000","last_undegraded":"2023-09-23T20:46:00.620081+0000","last_fullsized":"2023-09-23T20:46:00.620081+0000","mapping_epoch":491,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":492,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:18:44.817589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.13","version":"90'161","reported_seq":826,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-09-23T20:46:08.662925+0000","last_change":"2023-09-23T20:35:02.238923+0000","last_active":"2023-09-23T20:46:08.662925+0000","last_peered":"2023-09-23T20:46:08.662925+0000","last_clean":"2023-09-23T20:46:08.662925+0000","last_became_active":"2023-09-23T20:35:02.238293+0000","last_became_peered":"2023-09-23T20:35:02.238293+0000","last_unstale":"2023-09-23T20:46:08.662925+0000","last_undegraded":"2023-09-23T20:46:08.662925+0000","last_fullsized":"2023-09-23T20:46:08.662925+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:42.544308+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:42.544308+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:51:22.281932+0000","scrub_duration":0.0029958179999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":541,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-09-23T20:46:00.618485+0000","last_change":"2023-09-23T20:38:02.539774+0000","last_active":"2023-09-23T20:46:00.618485+0000","last_peered":"2023-09-23T20:46:00.618485+0000","last_clean":"2023-09-23T20:46:00.618485+0000","last_became_active":"2023-09-23T20:38:02.539436+0000","last_became_peered":"2023-09-23T20:38:02.539436+0000","last_unstale":"2023-09-23T20:46:00.618485+0000","last_undegraded":"2023-09-23T20:46:00.618485+0000","last_fullsized":"2023-09-23T20:46:00.618485+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:50:57.375803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":503,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-09-23T20:46:08.660057+0000","last_change":"2023-09-23T20:43:46.521735+0000","last_active":"2023-09-23T20:46:08.660057+0000","last_peered":"2023-09-23T20:46:08.660057+0000","last_clean":"2023-09-23T20:46:08.660057+0000","last_became_active":"2023-09-23T20:43:46.520762+0000","last_became_peered":"2023-09-23T20:43:46.520762+0000","last_unstale":"2023-09-23T20:46:08.660057+0000","last_undegraded":"2023-09-23T20:46:08.660057+0000","last_fullsized":"2023-09-23T20:46:08.660057+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:26:48.841930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"90'161","reported_seq":622,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-09-23T20:46:00.620742+0000","last_change":"2023-09-23T20:44:18.952808+0000","last_active":"2023-09-23T20:46:00.620742+0000","last_peered":"2023-09-23T20:46:00.620742+0000","last_clean":"2023-09-23T20:46:00.620742+0000","last_became_active":"2023-09-23T20:43:16.758652+0000","last_became_peered":"2023-09-23T20:43:16.758652+0000","last_unstale":"2023-09-23T20:46:00.620742+0000","last_undegraded":"2023-09-23T20:46:00.620742+0000","last_fullsized":"2023-09-23T20:46:00.620742+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:18.952693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:18.952693+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:12:19.496465+0000","scrub_duration":0.001699365,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":213,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-09-23T20:46:00.620071+0000","last_change":"2023-09-23T20:44:43.644290+0000","last_active":"2023-09-23T20:46:00.620071+0000","last_peered":"2023-09-23T20:46:00.620071+0000","last_clean":"2023-09-23T20:46:00.620071+0000","last_became_active":"2023-09-23T20:44:43.638313+0000","last_became_peered":"2023-09-23T20:44:43.638313+0000","last_unstale":"2023-09-23T20:46:00.620071+0000","last_undegraded":"2023-09-23T20:46:00.620071+0000","last_fullsized":"2023-09-23T20:46:00.620071+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:32:52.892603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":333,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-09-23T20:46:00.625258+0000","last_change":"2023-09-23T20:37:43.571083+0000","last_active":"2023-09-23T20:46:00.625258+0000","last_peered":"2023-09-23T20:46:00.625258+0000","last_clean":"2023-09-23T20:46:00.625258+0000","last_became_active":"2023-09-23T20:37:43.570306+0000","last_became_peered":"2023-09-23T20:37:43.570306+0000","last_unstale":"2023-09-23T20:46:00.625258+0000","last_undegraded":"2023-09-23T20:46:00.625258+0000","last_fullsized":"2023-09-23T20:46:00.625258+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:42:02.396718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.6","version":"538'3248","reported_seq":3264,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-09-23T20:46:11.379002+0000","last_change":"2023-09-23T20:45:39.488709+0000","last_active":"2023-09-23T20:46:11.379002+0000","last_peered":"2023-09-23T20:46:11.379002+0000","last_clean":"2023-09-23T20:46:11.379002+0000","last_became_active":"2023-09-23T20:45:39.488360+0000","last_became_peered":"2023-09-23T20:45:39.488360+0000","last_unstale":"2023-09-23T20:46:11.379002+0000","last_undegraded":"2023-09-23T20:46:11.379002+0000","last_fullsized":"2023-09-23T20:46:11.379002+0000","mapping_epoch":531,"log_start":"0'0","ondisk_log_start":"0'0","created":531,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":3248,"log_dups_size":0,"ondisk_log_size":3248,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:51:44.738010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":212860928,"num_objects":3248,"num_object_clones":0,"num_object_copies":9744,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3248,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3248,"num_write_kb":207872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.d","version":"90'161","reported_seq":830,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-09-23T20:46:00.618826+0000","last_change":"2023-09-23T20:34:59.496438+0000","last_active":"2023-09-23T20:46:00.618826+0000","last_peered":"2023-09-23T20:46:00.618826+0000","last_clean":"2023-09-23T20:46:00.618826+0000","last_became_active":"2023-09-23T20:33:38.970527+0000","last_became_peered":"2023-09-23T20:33:38.970527+0000","last_unstale":"2023-09-23T20:46:00.618826+0000","last_undegraded":"2023-09-23T20:46:00.618826+0000","last_fullsized":"2023-09-23T20:46:00.618826+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:34:59.496375+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:34:59.496375+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:46:18.244734+0000","scrub_duration":0.0017843189999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":541,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-09-23T20:46:00.619762+0000","last_change":"2023-09-23T20:32:33.283752+0000","last_active":"2023-09-23T20:46:00.619762+0000","last_peered":"2023-09-23T20:46:00.619762+0000","last_clean":"2023-09-23T20:46:00.619762+0000","last_became_active":"2023-09-23T20:32:33.230503+0000","last_became_peered":"2023-09-23T20:32:33.230503+0000","last_unstale":"2023-09-23T20:46:00.619762+0000","last_undegraded":"2023-09-23T20:46:00.619762+0000","last_fullsized":"2023-09-23T20:46:00.619762+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:29.084410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":352,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-09-23T20:46:00.616359+0000","last_change":"2023-09-23T20:43:16.765253+0000","last_active":"2023-09-23T20:46:00.616359+0000","last_peered":"2023-09-23T20:46:00.616359+0000","last_clean":"2023-09-23T20:46:00.616359+0000","last_became_active":"2023-09-23T20:43:16.764978+0000","last_became_peered":"2023-09-23T20:43:16.764978+0000","last_unstale":"2023-09-23T20:46:00.616359+0000","last_undegraded":"2023-09-23T20:46:00.616359+0000","last_fullsized":"2023-09-23T20:46:00.616359+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:07:23.539372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.7","version":"538'3098","reported_seq":3113,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-09-23T20:46:10.316705+0000","last_change":"2023-09-23T20:45:39.488542+0000","last_active":"2023-09-23T20:46:10.316705+0000","last_peered":"2023-09-23T20:46:10.316705+0000","last_clean":"2023-09-23T20:46:10.316705+0000","last_became_active":"2023-09-23T20:45:39.488259+0000","last_became_peered":"2023-09-23T20:45:39.488259+0000","last_unstale":"2023-09-23T20:46:10.316705+0000","last_undegraded":"2023-09-23T20:46:10.316705+0000","last_fullsized":"2023-09-23T20:46:10.316705+0000","mapping_epoch":531,"log_start":"0'0","ondisk_log_start":"0'0","created":531,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":3098,"log_dups_size":0,"ondisk_log_size":3098,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:39:05.833662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":203030528,"num_objects":3098,"num_object_clones":0,"num_object_copies":9294,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3098,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3098,"num_write_kb":198272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.c","version":"90'161","reported_seq":799,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-09-23T20:46:00.624026+0000","last_change":"2023-09-23T20:41:09.506481+0000","last_active":"2023-09-23T20:46:00.624026+0000","last_peered":"2023-09-23T20:46:00.624026+0000","last_clean":"2023-09-23T20:46:00.624026+0000","last_became_active":"2023-09-23T20:38:28.442229+0000","last_became_peered":"2023-09-23T20:38:28.442229+0000","last_unstale":"2023-09-23T20:46:00.624026+0000","last_undegraded":"2023-09-23T20:46:00.624026+0000","last_fullsized":"2023-09-23T20:46:00.624026+0000","mapping_epoch":411,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":412,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:41:09.506338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:41:09.506338+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:19:46.265011+0000","scrub_duration":0.01112002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":576,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-09-23T20:46:00.619445+0000","last_change":"2023-09-23T20:34:59.987981+0000","last_active":"2023-09-23T20:46:00.619445+0000","last_peered":"2023-09-23T20:46:00.619445+0000","last_clean":"2023-09-23T20:46:00.619445+0000","last_became_active":"2023-09-23T20:34:59.985368+0000","last_became_peered":"2023-09-23T20:34:59.985368+0000","last_unstale":"2023-09-23T20:46:00.619445+0000","last_undegraded":"2023-09-23T20:46:00.619445+0000","last_fullsized":"2023-09-23T20:46:00.619445+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:12:43.082493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":235,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-09-23T20:46:00.614336+0000","last_change":"2023-09-23T20:32:33.232142+0000","last_active":"2023-09-23T20:46:00.614336+0000","last_peered":"2023-09-23T20:46:00.614336+0000","last_clean":"2023-09-23T20:46:00.614336+0000","last_became_active":"2023-09-23T20:32:33.231683+0000","last_became_peered":"2023-09-23T20:32:33.231683+0000","last_unstale":"2023-09-23T20:46:00.614336+0000","last_undegraded":"2023-09-23T20:46:00.614336+0000","last_fullsized":"2023-09-23T20:46:00.614336+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:02:58.336348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"538'3196","reported_seq":3212,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-09-23T20:46:11.428056+0000","last_change":"2023-09-23T20:45:39.381562+0000","last_active":"2023-09-23T20:46:11.428056+0000","last_peered":"2023-09-23T20:46:11.428056+0000","last_clean":"2023-09-23T20:46:11.428056+0000","last_became_active":"2023-09-23T20:45:39.380887+0000","last_became_peered":"2023-09-23T20:45:39.380887+0000","last_unstale":"2023-09-23T20:46:11.428056+0000","last_undegraded":"2023-09-23T20:46:11.428056+0000","last_fullsized":"2023-09-23T20:46:11.428056+0000","mapping_epoch":531,"log_start":"0'0","ondisk_log_start":"0'0","created":531,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":3196,"log_dups_size":0,"ondisk_log_size":3196,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:41:52.967029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":209453056,"num_objects":3196,"num_object_clones":0,"num_object_copies":9588,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3196,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3196,"num_write_kb":204544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"90'161","reported_seq":693,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-09-23T20:46:00.619606+0000","last_change":"2023-09-23T20:38:40.491902+0000","last_active":"2023-09-23T20:46:00.619606+0000","last_peered":"2023-09-23T20:46:00.619606+0000","last_clean":"2023-09-23T20:46:00.619606+0000","last_became_active":"2023-09-23T20:38:40.491563+0000","last_became_peered":"2023-09-23T20:38:40.491563+0000","last_unstale":"2023-09-23T20:46:00.619606+0000","last_undegraded":"2023-09-23T20:46:00.619606+0000","last_fullsized":"2023-09-23T20:46:00.619606+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:46.217489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:46.217489+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:39:25.008796+0000","scrub_duration":0.001456429,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":566,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-09-23T20:46:00.618872+0000","last_change":"2023-09-23T20:40:19.677655+0000","last_active":"2023-09-23T20:46:00.618872+0000","last_peered":"2023-09-23T20:46:00.618872+0000","last_clean":"2023-09-23T20:46:00.618872+0000","last_became_active":"2023-09-23T20:40:19.551189+0000","last_became_peered":"2023-09-23T20:40:19.551189+0000","last_unstale":"2023-09-23T20:46:00.618872+0000","last_undegraded":"2023-09-23T20:46:00.618872+0000","last_fullsized":"2023-09-23T20:46:00.618872+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:11:01.559576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":143,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-09-23T20:46:00.619516+0000","last_change":"2023-09-23T20:38:02.590613+0000","last_active":"2023-09-23T20:46:00.619516+0000","last_peered":"2023-09-23T20:46:00.619516+0000","last_clean":"2023-09-23T20:46:00.619516+0000","last_became_active":"2023-09-23T20:38:02.590284+0000","last_became_peered":"2023-09-23T20:38:02.590284+0000","last_unstale":"2023-09-23T20:46:00.619516+0000","last_undegraded":"2023-09-23T20:46:00.619516+0000","last_fullsized":"2023-09-23T20:46:00.619516+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:40:07.825054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.5","version":"538'3357","reported_seq":3371,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-09-23T20:46:13.110337+0000","last_change":"2023-09-23T20:45:39.380986+0000","last_active":"2023-09-23T20:46:13.110337+0000","last_peered":"2023-09-23T20:46:13.110337+0000","last_clean":"2023-09-23T20:46:13.110337+0000","last_became_active":"2023-09-23T20:45:39.380650+0000","last_became_peered":"2023-09-23T20:45:39.380650+0000","last_unstale":"2023-09-23T20:46:13.110337+0000","last_undegraded":"2023-09-23T20:46:13.110337+0000","last_fullsized":"2023-09-23T20:46:13.110337+0000","mapping_epoch":531,"log_start":"0'0","ondisk_log_start":"0'0","created":531,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":3357,"log_dups_size":0,"ondisk_log_size":3357,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:24:24.301239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":220004352,"num_objects":3357,"num_object_clones":0,"num_object_copies":10071,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3357,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3357,"num_write_kb":214848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.e","version":"90'161","reported_seq":638,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-09-23T20:46:00.615612+0000","last_change":"2023-09-23T20:36:00.322453+0000","last_active":"2023-09-23T20:46:00.615612+0000","last_peered":"2023-09-23T20:46:00.615612+0000","last_clean":"2023-09-23T20:46:00.615612+0000","last_became_active":"2023-09-23T20:35:02.233536+0000","last_became_peered":"2023-09-23T20:35:02.233536+0000","last_unstale":"2023-09-23T20:46:00.615612+0000","last_undegraded":"2023-09-23T20:46:00.615612+0000","last_fullsized":"2023-09-23T20:46:00.615612+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:00.322381+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:00.322381+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:02:15.489069+0000","scrub_duration":0.0017321859999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":498,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-09-23T20:46:00.624358+0000","last_change":"2023-09-23T20:43:16.756437+0000","last_active":"2023-09-23T20:46:00.624358+0000","last_peered":"2023-09-23T20:46:00.624358+0000","last_clean":"2023-09-23T20:46:00.624358+0000","last_became_active":"2023-09-23T20:43:16.756193+0000","last_became_peered":"2023-09-23T20:43:16.756193+0000","last_unstale":"2023-09-23T20:46:00.624358+0000","last_undegraded":"2023-09-23T20:46:00.624358+0000","last_fullsized":"2023-09-23T20:46:00.624358+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:33:25.632781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":512,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-09-23T20:46:01.429765+0000","last_change":"2023-09-23T20:43:46.541524+0000","last_active":"2023-09-23T20:46:01.429765+0000","last_peered":"2023-09-23T20:46:01.429765+0000","last_clean":"2023-09-23T20:46:01.429765+0000","last_became_active":"2023-09-23T20:43:46.541051+0000","last_became_peered":"2023-09-23T20:43:46.541051+0000","last_unstale":"2023-09-23T20:46:01.429765+0000","last_undegraded":"2023-09-23T20:46:01.429765+0000","last_fullsized":"2023-09-23T20:46:01.429765+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:59:36.632592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"538'3493","reported_seq":3509,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-09-23T20:46:13.553022+0000","last_change":"2023-09-23T20:45:39.331236+0000","last_active":"2023-09-23T20:46:13.553022+0000","last_peered":"2023-09-23T20:46:13.553022+0000","last_clean":"2023-09-23T20:46:13.553022+0000","last_became_active":"2023-09-23T20:45:39.330923+0000","last_became_peered":"2023-09-23T20:45:39.330923+0000","last_unstale":"2023-09-23T20:46:13.553022+0000","last_undegraded":"2023-09-23T20:46:13.553022+0000","last_fullsized":"2023-09-23T20:46:13.553022+0000","mapping_epoch":531,"log_start":"0'0","ondisk_log_start":"0'0","created":531,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":3493,"log_dups_size":0,"ondisk_log_size":3493,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:48:20.711262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":228917248,"num_objects":3493,"num_object_clones":0,"num_object_copies":10479,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3493,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3493,"num_write_kb":223552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":943,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-09-23T20:46:00.618378+0000","last_change":"2023-09-23T20:35:02.523308+0000","last_active":"2023-09-23T20:46:00.618378+0000","last_peered":"2023-09-23T20:46:00.618378+0000","last_clean":"2023-09-23T20:46:00.618378+0000","last_became_active":"2023-09-23T20:33:38.979989+0000","last_became_peered":"2023-09-23T20:33:38.979989+0000","last_unstale":"2023-09-23T20:46:00.618378+0000","last_undegraded":"2023-09-23T20:46:00.618378+0000","last_fullsized":"2023-09-23T20:46:00.618378+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:35:02.523203+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:35:02.523203+0000","last_clean_scrub_stamp":"2023-09-23T20:35:02.523203+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:02:41.472958+0000","scrub_duration":0.0085927989999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":359,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-09-23T20:46:00.633970+0000","last_change":"2023-09-23T20:37:43.583731+0000","last_active":"2023-09-23T20:46:00.633970+0000","last_peered":"2023-09-23T20:46:00.633970+0000","last_clean":"2023-09-23T20:46:00.633970+0000","last_became_active":"2023-09-23T20:37:43.583066+0000","last_became_peered":"2023-09-23T20:37:43.583066+0000","last_unstale":"2023-09-23T20:46:00.633970+0000","last_undegraded":"2023-09-23T20:46:00.633970+0000","last_fullsized":"2023-09-23T20:46:00.633970+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:03:45.211900+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":255,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-09-23T20:46:00.615334+0000","last_change":"2023-09-23T20:38:02.579814+0000","last_active":"2023-09-23T20:46:00.615334+0000","last_peered":"2023-09-23T20:46:00.615334+0000","last_clean":"2023-09-23T20:46:00.615334+0000","last_became_active":"2023-09-23T20:38:02.579253+0000","last_became_peered":"2023-09-23T20:38:02.579253+0000","last_unstale":"2023-09-23T20:46:00.615334+0000","last_undegraded":"2023-09-23T20:46:00.615334+0000","last_fullsized":"2023-09-23T20:46:00.615334+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:05:07.298198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"538'3433","reported_seq":3447,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-09-23T20:46:13.097555+0000","last_change":"2023-09-23T20:45:39.488901+0000","last_active":"2023-09-23T20:46:13.097555+0000","last_peered":"2023-09-23T20:46:13.097555+0000","last_clean":"2023-09-23T20:46:13.097555+0000","last_became_active":"2023-09-23T20:45:39.488499+0000","last_became_peered":"2023-09-23T20:45:39.488499+0000","last_unstale":"2023-09-23T20:46:13.097555+0000","last_undegraded":"2023-09-23T20:46:13.097555+0000","last_fullsized":"2023-09-23T20:46:13.097555+0000","mapping_epoch":531,"log_start":"0'0","ondisk_log_start":"0'0","created":531,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":3433,"log_dups_size":0,"ondisk_log_size":3433,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:25:10.137770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":224985088,"num_objects":3433,"num_object_clones":0,"num_object_copies":10299,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3433,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3433,"num_write_kb":219712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":653,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-09-23T20:46:00.624179+0000","last_change":"2023-09-23T20:44:46.725779+0000","last_active":"2023-09-23T20:46:00.624179+0000","last_peered":"2023-09-23T20:46:00.624179+0000","last_clean":"2023-09-23T20:46:00.624179+0000","last_became_active":"2023-09-23T20:44:46.725433+0000","last_became_peered":"2023-09-23T20:44:46.725433+0000","last_unstale":"2023-09-23T20:46:00.624179+0000","last_undegraded":"2023-09-23T20:46:00.624179+0000","last_fullsized":"2023-09-23T20:46:00.624179+0000","mapping_epoch":516,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":517,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:42:45.492220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:42:45.492220+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:28:35.202407+0000","scrub_duration":0.015259412,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":143,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-09-23T20:46:00.619593+0000","last_change":"2023-09-23T20:38:02.578703+0000","last_active":"2023-09-23T20:46:00.619593+0000","last_peered":"2023-09-23T20:46:00.619593+0000","last_clean":"2023-09-23T20:46:00.619593+0000","last_became_active":"2023-09-23T20:38:02.578414+0000","last_became_peered":"2023-09-23T20:38:02.578414+0000","last_unstale":"2023-09-23T20:46:00.619593+0000","last_undegraded":"2023-09-23T20:46:00.619593+0000","last_fullsized":"2023-09-23T20:46:00.619593+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:43:59.779295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":204,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-09-23T20:46:00.615100+0000","last_change":"2023-09-23T20:37:31.296606+0000","last_active":"2023-09-23T20:46:00.615100+0000","last_peered":"2023-09-23T20:46:00.615100+0000","last_clean":"2023-09-23T20:46:00.615100+0000","last_became_active":"2023-09-23T20:37:31.295924+0000","last_became_peered":"2023-09-23T20:37:31.295924+0000","last_unstale":"2023-09-23T20:46:00.615100+0000","last_undegraded":"2023-09-23T20:46:00.615100+0000","last_fullsized":"2023-09-23T20:46:00.615100+0000","mapping_epoch":391,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":392,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:27:50.948510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.0","version":"538'3146","reported_seq":3161,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-09-23T20:46:10.314405+0000","last_change":"2023-09-23T20:45:39.331645+0000","last_active":"2023-09-23T20:46:10.314405+0000","last_peered":"2023-09-23T20:46:10.314405+0000","last_clean":"2023-09-23T20:46:10.314405+0000","last_became_active":"2023-09-23T20:45:39.331293+0000","last_became_peered":"2023-09-23T20:45:39.331293+0000","last_unstale":"2023-09-23T20:46:10.314405+0000","last_undegraded":"2023-09-23T20:46:10.314405+0000","last_fullsized":"2023-09-23T20:46:10.314405+0000","mapping_epoch":531,"log_start":"0'0","ondisk_log_start":"0'0","created":531,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":3146,"log_dups_size":0,"ondisk_log_size":3146,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:29:42.789115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":206176256,"num_objects":3146,"num_object_clones":0,"num_object_copies":9438,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3146,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3146,"num_write_kb":201344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.b","version":"90'161","reported_seq":842,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-09-23T20:46:00.621630+0000","last_change":"2023-09-23T20:40:22.445808+0000","last_active":"2023-09-23T20:46:00.621630+0000","last_peered":"2023-09-23T20:46:00.621630+0000","last_clean":"2023-09-23T20:46:00.621630+0000","last_became_active":"2023-09-23T20:40:22.445456+0000","last_became_peered":"2023-09-23T20:40:22.445456+0000","last_unstale":"2023-09-23T20:46:00.621630+0000","last_undegraded":"2023-09-23T20:46:00.621630+0000","last_fullsized":"2023-09-23T20:46:00.621630+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:05.595042+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:05.595042+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:39:23.678154+0000","scrub_duration":0.007271675,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":194,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-09-23T20:46:00.614087+0000","last_change":"2023-09-23T20:34:59.983122+0000","last_active":"2023-09-23T20:46:00.614087+0000","last_peered":"2023-09-23T20:46:00.614087+0000","last_clean":"2023-09-23T20:46:00.614087+0000","last_became_active":"2023-09-23T20:34:59.982809+0000","last_became_peered":"2023-09-23T20:34:59.982809+0000","last_unstale":"2023-09-23T20:46:00.614087+0000","last_undegraded":"2023-09-23T20:46:00.614087+0000","last_fullsized":"2023-09-23T20:46:00.614087+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:49:40.839723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":275,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-09-23T20:46:00.615880+0000","last_change":"2023-09-23T20:44:10.606995+0000","last_active":"2023-09-23T20:46:00.615880+0000","last_peered":"2023-09-23T20:46:00.615880+0000","last_clean":"2023-09-23T20:46:00.615880+0000","last_became_active":"2023-09-23T20:44:10.606718+0000","last_became_peered":"2023-09-23T20:44:10.606718+0000","last_unstale":"2023-09-23T20:46:00.615880+0000","last_undegraded":"2023-09-23T20:46:00.615880+0000","last_fullsized":"2023-09-23T20:46:00.615880+0000","mapping_epoch":496,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":497,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:13:12.496245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"538'3203","reported_seq":3218,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-09-23T20:46:10.701220+0000","last_change":"2023-09-23T20:45:39.333840+0000","last_active":"2023-09-23T20:46:10.701220+0000","last_peered":"2023-09-23T20:46:10.701220+0000","last_clean":"2023-09-23T20:46:10.701220+0000","last_became_active":"2023-09-23T20:45:39.333454+0000","last_became_peered":"2023-09-23T20:45:39.333454+0000","last_unstale":"2023-09-23T20:46:10.701220+0000","last_undegraded":"2023-09-23T20:46:10.701220+0000","last_fullsized":"2023-09-23T20:46:10.701220+0000","mapping_epoch":531,"log_start":"0'0","ondisk_log_start":"0'0","created":531,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":3203,"log_dups_size":0,"ondisk_log_size":3203,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:23:21.002722+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":209911808,"num_objects":3203,"num_object_clones":0,"num_object_copies":9609,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3203,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3203,"num_write_kb":204992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":564,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-09-23T20:46:00.623732+0000","last_change":"2023-09-23T20:43:51.693618+0000","last_active":"2023-09-23T20:46:00.623732+0000","last_peered":"2023-09-23T20:46:00.623732+0000","last_clean":"2023-09-23T20:46:00.623732+0000","last_became_active":"2023-09-23T20:43:16.765186+0000","last_became_peered":"2023-09-23T20:43:16.765186+0000","last_unstale":"2023-09-23T20:46:00.623732+0000","last_undegraded":"2023-09-23T20:46:00.623732+0000","last_fullsized":"2023-09-23T20:46:00.623732+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:43:51.693459+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:43:51.693459+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:33:30.208355+0000","scrub_duration":0.0016145269999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":333,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-09-23T20:46:00.625171+0000","last_change":"2023-09-23T20:37:43.570610+0000","last_active":"2023-09-23T20:46:00.625171+0000","last_peered":"2023-09-23T20:46:00.625171+0000","last_clean":"2023-09-23T20:46:00.625171+0000","last_became_active":"2023-09-23T20:37:43.570216+0000","last_became_peered":"2023-09-23T20:37:43.570216+0000","last_unstale":"2023-09-23T20:46:00.625171+0000","last_undegraded":"2023-09-23T20:46:00.625171+0000","last_fullsized":"2023-09-23T20:46:00.625171+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:39:26.555329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":364,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-09-23T20:46:00.621073+0000","last_change":"2023-09-23T20:32:33.230582+0000","last_active":"2023-09-23T20:46:00.621073+0000","last_peered":"2023-09-23T20:46:00.621073+0000","last_clean":"2023-09-23T20:46:00.621073+0000","last_became_active":"2023-09-23T20:32:33.230257+0000","last_became_peered":"2023-09-23T20:32:33.230257+0000","last_unstale":"2023-09-23T20:46:00.621073+0000","last_undegraded":"2023-09-23T20:46:00.621073+0000","last_fullsized":"2023-09-23T20:46:00.621073+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:11:45.333096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"538'3181","reported_seq":3196,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-09-23T20:46:10.322475+0000","last_change":"2023-09-23T20:45:39.332694+0000","last_active":"2023-09-23T20:46:10.322475+0000","last_peered":"2023-09-23T20:46:10.322475+0000","last_clean":"2023-09-23T20:46:10.322475+0000","last_became_active":"2023-09-23T20:45:39.332422+0000","last_became_peered":"2023-09-23T20:45:39.332422+0000","last_unstale":"2023-09-23T20:46:10.322475+0000","last_undegraded":"2023-09-23T20:46:10.322475+0000","last_fullsized":"2023-09-23T20:46:10.322475+0000","mapping_epoch":531,"log_start":"0'0","ondisk_log_start":"0'0","created":531,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":3181,"log_dups_size":0,"ondisk_log_size":3181,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:50:39.768517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":208470016,"num_objects":3181,"num_object_clones":0,"num_object_copies":9543,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3181,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3181,"num_write_kb":203584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":726,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-09-23T20:46:00.619517+0000","last_change":"2023-09-23T20:41:07.424609+0000","last_active":"2023-09-23T20:46:00.619517+0000","last_peered":"2023-09-23T20:46:00.619517+0000","last_clean":"2023-09-23T20:46:00.619517+0000","last_became_active":"2023-09-23T20:38:05.622261+0000","last_became_peered":"2023-09-23T20:38:05.622261+0000","last_unstale":"2023-09-23T20:46:00.619517+0000","last_undegraded":"2023-09-23T20:46:00.619517+0000","last_fullsized":"2023-09-23T20:46:00.619517+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:41:07.424492+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:41:07.424492+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:45:21.689123+0000","scrub_duration":0.0018960959999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"86'119","reported_seq":848,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-09-23T20:46:00.621836+0000","last_change":"2023-09-23T20:29:45.521257+0000","last_active":"2023-09-23T20:46:00.621836+0000","last_peered":"2023-09-23T20:46:00.621836+0000","last_clean":"2023-09-23T20:46:00.621836+0000","last_became_active":"2023-09-23T20:29:45.520546+0000","last_became_peered":"2023-09-23T20:29:45.520546+0000","last_unstale":"2023-09-23T20:46:00.621836+0000","last_undegraded":"2023-09-23T20:46:00.621836+0000","last_fullsized":"2023-09-23T20:46:00.621836+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":119,"log_dups_size":0,"ondisk_log_size":119,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:58:56.085777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032705499999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":523,"num_read_kb":3503,"num_write":136,"num_write_kb":3128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":331,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-09-23T20:46:00.625555+0000","last_change":"2023-09-23T20:43:16.756494+0000","last_active":"2023-09-23T20:46:00.625555+0000","last_peered":"2023-09-23T20:46:00.625555+0000","last_clean":"2023-09-23T20:46:00.625555+0000","last_became_active":"2023-09-23T20:43:16.756131+0000","last_became_peered":"2023-09-23T20:43:16.756131+0000","last_unstale":"2023-09-23T20:46:00.625555+0000","last_undegraded":"2023-09-23T20:46:00.625555+0000","last_fullsized":"2023-09-23T20:46:00.625555+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:10:19.219533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":495,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-09-23T20:46:00.620327+0000","last_change":"2023-09-23T20:40:19.530942+0000","last_active":"2023-09-23T20:46:00.620327+0000","last_peered":"2023-09-23T20:46:00.620327+0000","last_clean":"2023-09-23T20:46:00.620327+0000","last_became_active":"2023-09-23T20:40:19.530617+0000","last_became_peered":"2023-09-23T20:40:19.530617+0000","last_unstale":"2023-09-23T20:46:00.620327+0000","last_undegraded":"2023-09-23T20:46:00.620327+0000","last_fullsized":"2023-09-23T20:46:00.620327+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:24:09.231934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"538'3068","reported_seq":3083,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-09-23T20:46:10.307879+0000","last_change":"2023-09-23T20:45:39.489110+0000","last_active":"2023-09-23T20:46:10.307879+0000","last_peered":"2023-09-23T20:46:10.307879+0000","last_clean":"2023-09-23T20:46:10.307879+0000","last_became_active":"2023-09-23T20:45:39.488817+0000","last_became_peered":"2023-09-23T20:45:39.488817+0000","last_unstale":"2023-09-23T20:46:10.307879+0000","last_undegraded":"2023-09-23T20:46:10.307879+0000","last_fullsized":"2023-09-23T20:46:10.307879+0000","mapping_epoch":531,"log_start":"0'0","ondisk_log_start":"0'0","created":531,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":3068,"log_dups_size":0,"ondisk_log_size":3068,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:07:21.467971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":201064448,"num_objects":3068,"num_object_clones":0,"num_object_copies":9204,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3068,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3068,"num_write_kb":196352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":584,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-09-23T20:46:00.618623+0000","last_change":"2023-09-23T20:44:15.656984+0000","last_active":"2023-09-23T20:46:00.618623+0000","last_peered":"2023-09-23T20:46:00.618623+0000","last_clean":"2023-09-23T20:46:00.618623+0000","last_became_active":"2023-09-23T20:43:46.512210+0000","last_became_peered":"2023-09-23T20:43:46.512210+0000","last_unstale":"2023-09-23T20:46:00.618623+0000","last_undegraded":"2023-09-23T20:46:00.618623+0000","last_fullsized":"2023-09-23T20:46:00.618623+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:15.656880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:15.656880+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:37:56.453466+0000","scrub_duration":0.0016711180000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"86'106","reported_seq":789,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-09-23T20:46:00.620609+0000","last_change":"2023-09-23T20:43:16.759138+0000","last_active":"2023-09-23T20:46:00.620609+0000","last_peered":"2023-09-23T20:46:00.620609+0000","last_clean":"2023-09-23T20:46:00.620609+0000","last_became_active":"2023-09-23T20:43:16.758015+0000","last_became_peered":"2023-09-23T20:43:16.758015+0000","last_unstale":"2023-09-23T20:46:00.620609+0000","last_undegraded":"2023-09-23T20:46:00.620609+0000","last_fullsized":"2023-09-23T20:46:00.620609+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":106,"log_dups_size":0,"ondisk_log_size":106,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:08:02.852472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00099302599999999993,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1094,"num_read_kb":1678,"num_write":100,"num_write_kb":404,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":255,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-09-23T20:46:00.614026+0000","last_change":"2023-09-23T20:38:02.552008+0000","last_active":"2023-09-23T20:46:00.614026+0000","last_peered":"2023-09-23T20:46:00.614026+0000","last_clean":"2023-09-23T20:46:00.614026+0000","last_became_active":"2023-09-23T20:38:02.551684+0000","last_became_peered":"2023-09-23T20:38:02.551684+0000","last_unstale":"2023-09-23T20:46:00.614026+0000","last_undegraded":"2023-09-23T20:46:00.614026+0000","last_fullsized":"2023-09-23T20:46:00.614026+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:40:37.147538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":501,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-09-23T20:46:00.621127+0000","last_change":"2023-09-23T20:29:45.533406+0000","last_active":"2023-09-23T20:46:00.621127+0000","last_peered":"2023-09-23T20:46:00.621127+0000","last_clean":"2023-09-23T20:46:00.621127+0000","last_became_active":"2023-09-23T20:29:45.526001+0000","last_became_peered":"2023-09-23T20:29:45.526001+0000","last_unstale":"2023-09-23T20:46:00.621127+0000","last_undegraded":"2023-09-23T20:46:00.621127+0000","last_fullsized":"2023-09-23T20:46:00.621127+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:43.787251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.9","version":"538'3398","reported_seq":3414,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-09-23T20:46:13.087984+0000","last_change":"2023-09-23T20:45:39.488721+0000","last_active":"2023-09-23T20:46:13.087984+0000","last_peered":"2023-09-23T20:46:13.087984+0000","last_clean":"2023-09-23T20:46:13.087984+0000","last_became_active":"2023-09-23T20:45:39.488320+0000","last_became_peered":"2023-09-23T20:45:39.488320+0000","last_unstale":"2023-09-23T20:46:13.087984+0000","last_undegraded":"2023-09-23T20:46:13.087984+0000","last_fullsized":"2023-09-23T20:46:13.087984+0000","mapping_epoch":531,"log_start":"0'0","ondisk_log_start":"0'0","created":531,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":3398,"log_dups_size":0,"ondisk_log_size":3398,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:46:30.536848+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":222691328,"num_objects":3398,"num_object_clones":0,"num_object_copies":10194,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3398,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3398,"num_write_kb":217472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":501,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-09-23T20:46:00.616394+0000","last_change":"2023-09-23T20:44:26.978243+0000","last_active":"2023-09-23T20:46:00.616394+0000","last_peered":"2023-09-23T20:46:00.616394+0000","last_clean":"2023-09-23T20:46:00.616394+0000","last_became_active":"2023-09-23T20:44:18.638989+0000","last_became_peered":"2023-09-23T20:44:18.638989+0000","last_unstale":"2023-09-23T20:46:00.616394+0000","last_undegraded":"2023-09-23T20:46:00.616394+0000","last_fullsized":"2023-09-23T20:46:00.616394+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:26.978179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:26.978179+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:25:38.977011+0000","scrub_duration":0.0013962829999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"86'130","reported_seq":856,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-09-23T20:46:00.623584+0000","last_change":"2023-09-23T20:25:39.637458+0000","last_active":"2023-09-23T20:46:00.623584+0000","last_peered":"2023-09-23T20:46:00.623584+0000","last_clean":"2023-09-23T20:46:00.623584+0000","last_became_active":"2023-09-23T20:25:39.636773+0000","last_became_peered":"2023-09-23T20:25:39.636773+0000","last_unstale":"2023-09-23T20:46:00.623584+0000","last_undegraded":"2023-09-23T20:46:00.623584+0000","last_fullsized":"2023-09-23T20:46:00.623584+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:00:32.810564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011300049999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":512,"num_read_kb":1820,"num_write":133,"num_write_kb":1347,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":510,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-09-23T20:46:00.619844+0000","last_change":"2023-09-23T20:38:02.591335+0000","last_active":"2023-09-23T20:46:00.619844+0000","last_peered":"2023-09-23T20:46:00.619844+0000","last_clean":"2023-09-23T20:46:00.619844+0000","last_became_active":"2023-09-23T20:38:02.591015+0000","last_became_peered":"2023-09-23T20:38:02.591015+0000","last_unstale":"2023-09-23T20:46:00.619844+0000","last_undegraded":"2023-09-23T20:46:00.619844+0000","last_fullsized":"2023-09-23T20:46:00.619844+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:55:32.272195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":52,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-09-23T20:46:00.620991+0000","last_change":"2023-09-23T20:44:10.555583+0000","last_active":"2023-09-23T20:46:00.620991+0000","last_peered":"2023-09-23T20:46:00.620991+0000","last_clean":"2023-09-23T20:46:00.620991+0000","last_became_active":"2023-09-23T20:44:10.555281+0000","last_became_peered":"2023-09-23T20:44:10.555281+0000","last_unstale":"2023-09-23T20:46:00.620991+0000","last_undegraded":"2023-09-23T20:46:00.620991+0000","last_fullsized":"2023-09-23T20:46:00.620991+0000","mapping_epoch":496,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":497,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:43:11.942147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"538'3162","reported_seq":3177,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-09-23T20:46:10.256601+0000","last_change":"2023-09-23T20:45:39.331096+0000","last_active":"2023-09-23T20:46:10.256601+0000","last_peered":"2023-09-23T20:46:10.256601+0000","last_clean":"2023-09-23T20:46:10.256601+0000","last_became_active":"2023-09-23T20:45:39.330752+0000","last_became_peered":"2023-09-23T20:45:39.330752+0000","last_unstale":"2023-09-23T20:46:10.256601+0000","last_undegraded":"2023-09-23T20:46:10.256601+0000","last_fullsized":"2023-09-23T20:46:10.256601+0000","mapping_epoch":531,"log_start":"0'0","ondisk_log_start":"0'0","created":531,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":3162,"log_dups_size":0,"ondisk_log_size":3162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:39:01.627762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":207224832,"num_objects":3162,"num_object_clones":0,"num_object_copies":9486,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3162,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3162,"num_write_kb":202368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"86'194","reported_seq":895,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-09-23T20:46:00.624629+0000","last_change":"2023-09-23T20:43:16.765865+0000","last_active":"2023-09-23T20:46:00.624629+0000","last_peered":"2023-09-23T20:46:00.624629+0000","last_clean":"2023-09-23T20:46:00.624629+0000","last_became_active":"2023-09-23T20:43:16.765289+0000","last_became_peered":"2023-09-23T20:43:16.765289+0000","last_unstale":"2023-09-23T20:46:00.624629+0000","last_undegraded":"2023-09-23T20:46:00.624629+0000","last_fullsized":"2023-09-23T20:46:00.624629+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":194,"log_dups_size":0,"ondisk_log_size":194,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:15:16.461900+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00037995999999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1024,"num_read_kb":1200,"num_write":173,"num_write_kb":1276,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":1000,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-09-23T20:46:00.614762+0000","last_change":"2023-09-23T20:38:05.602508+0000","last_active":"2023-09-23T20:46:00.614762+0000","last_peered":"2023-09-23T20:46:00.614762+0000","last_clean":"2023-09-23T20:46:00.614762+0000","last_became_active":"2023-09-23T20:38:05.602129+0000","last_became_peered":"2023-09-23T20:38:05.602129+0000","last_unstale":"2023-09-23T20:46:00.614762+0000","last_undegraded":"2023-09-23T20:46:00.614762+0000","last_fullsized":"2023-09-23T20:46:00.614762+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:45.343742+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:45.343742+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:48.910343+0000","scrub_duration":0.001475802,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":113,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-09-23T20:46:00.619667+0000","last_change":"2023-09-23T20:43:16.758213+0000","last_active":"2023-09-23T20:46:00.619667+0000","last_peered":"2023-09-23T20:46:00.619667+0000","last_clean":"2023-09-23T20:46:00.619667+0000","last_became_active":"2023-09-23T20:43:16.757869+0000","last_became_peered":"2023-09-23T20:43:16.757869+0000","last_unstale":"2023-09-23T20:46:00.619667+0000","last_undegraded":"2023-09-23T20:46:00.619667+0000","last_fullsized":"2023-09-23T20:46:00.619667+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:43:10.811749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":143,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-09-23T20:46:00.619597+0000","last_change":"2023-09-23T20:38:02.573200+0000","last_active":"2023-09-23T20:46:00.619597+0000","last_peered":"2023-09-23T20:46:00.619597+0000","last_clean":"2023-09-23T20:46:00.619597+0000","last_became_active":"2023-09-23T20:38:02.572945+0000","last_became_peered":"2023-09-23T20:38:02.572945+0000","last_unstale":"2023-09-23T20:46:00.619597+0000","last_undegraded":"2023-09-23T20:46:00.619597+0000","last_fullsized":"2023-09-23T20:46:00.619597+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:57.246533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.8","version":"538'3249","reported_seq":3265,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-09-23T20:46:11.416497+0000","last_change":"2023-09-23T20:45:39.332075+0000","last_active":"2023-09-23T20:46:11.416497+0000","last_peered":"2023-09-23T20:46:11.416497+0000","last_clean":"2023-09-23T20:46:11.416497+0000","last_became_active":"2023-09-23T20:45:39.331755+0000","last_became_peered":"2023-09-23T20:45:39.331755+0000","last_unstale":"2023-09-23T20:46:11.416497+0000","last_undegraded":"2023-09-23T20:46:11.416497+0000","last_fullsized":"2023-09-23T20:46:11.416497+0000","mapping_epoch":531,"log_start":"0'0","ondisk_log_start":"0'0","created":531,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":3249,"log_dups_size":0,"ondisk_log_size":3249,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:11:17.895002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":212926464,"num_objects":3249,"num_object_clones":0,"num_object_copies":9747,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3249,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3249,"num_write_kb":207936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":870,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-09-23T20:46:00.621424+0000","last_change":"2023-09-23T20:38:05.615711+0000","last_active":"2023-09-23T20:46:00.621424+0000","last_peered":"2023-09-23T20:46:00.621424+0000","last_clean":"2023-09-23T20:46:00.621424+0000","last_became_active":"2023-09-23T20:38:05.615390+0000","last_became_peered":"2023-09-23T20:38:05.615390+0000","last_unstale":"2023-09-23T20:46:00.621424+0000","last_undegraded":"2023-09-23T20:46:00.621424+0000","last_fullsized":"2023-09-23T20:46:00.621424+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:43.582146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:43.582146+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:45:11.917871+0000","scrub_duration":0.0016219730000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"86'80","reported_seq":790,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-09-23T20:46:00.614677+0000","last_change":"2023-09-23T20:29:45.536796+0000","last_active":"2023-09-23T20:46:00.614677+0000","last_peered":"2023-09-23T20:46:00.614677+0000","last_clean":"2023-09-23T20:46:00.614677+0000","last_became_active":"2023-09-23T20:29:45.536439+0000","last_became_peered":"2023-09-23T20:29:45.536439+0000","last_unstale":"2023-09-23T20:46:00.614677+0000","last_undegraded":"2023-09-23T20:46:00.614677+0000","last_fullsized":"2023-09-23T20:46:00.614677+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":80,"log_dups_size":0,"ondisk_log_size":80,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:30:57.793132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035115,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":395,"num_read_kb":1370,"num_write":66,"num_write_kb":1061,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":531,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-09-23T20:46:00.618542+0000","last_change":"2023-09-23T20:43:46.544563+0000","last_active":"2023-09-23T20:46:00.618542+0000","last_peered":"2023-09-23T20:46:00.618542+0000","last_clean":"2023-09-23T20:46:00.618542+0000","last_became_active":"2023-09-23T20:43:46.543958+0000","last_became_peered":"2023-09-23T20:43:46.543958+0000","last_unstale":"2023-09-23T20:46:00.618542+0000","last_undegraded":"2023-09-23T20:46:00.618542+0000","last_fullsized":"2023-09-23T20:46:00.618542+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:46:36.921752+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":526,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-09-23T20:46:08.659840+0000","last_change":"2023-09-23T20:43:46.520386+0000","last_active":"2023-09-23T20:46:08.659840+0000","last_peered":"2023-09-23T20:46:08.659840+0000","last_clean":"2023-09-23T20:46:08.659840+0000","last_became_active":"2023-09-23T20:43:46.519757+0000","last_became_peered":"2023-09-23T20:43:46.519757+0000","last_unstale":"2023-09-23T20:46:08.659840+0000","last_undegraded":"2023-09-23T20:46:08.659840+0000","last_fullsized":"2023-09-23T20:46:08.659840+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:23:14.662734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"538'3061","reported_seq":3076,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-09-23T20:46:10.313085+0000","last_change":"2023-09-23T20:45:39.331637+0000","last_active":"2023-09-23T20:46:10.313085+0000","last_peered":"2023-09-23T20:46:10.313085+0000","last_clean":"2023-09-23T20:46:10.313085+0000","last_became_active":"2023-09-23T20:45:39.331378+0000","last_became_peered":"2023-09-23T20:45:39.331378+0000","last_unstale":"2023-09-23T20:46:10.313085+0000","last_undegraded":"2023-09-23T20:46:10.313085+0000","last_fullsized":"2023-09-23T20:46:10.313085+0000","mapping_epoch":531,"log_start":"0'0","ondisk_log_start":"0'0","created":531,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":3061,"log_dups_size":0,"ondisk_log_size":3061,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:43:34.423232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":200605696,"num_objects":3061,"num_object_clones":0,"num_object_copies":9183,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3061,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3061,"num_write_kb":195904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":649,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-09-23T20:46:00.618997+0000","last_change":"2023-09-23T20:41:08.380084+0000","last_active":"2023-09-23T20:46:00.618997+0000","last_peered":"2023-09-23T20:46:00.618997+0000","last_clean":"2023-09-23T20:46:00.618997+0000","last_became_active":"2023-09-23T20:38:05.610881+0000","last_became_peered":"2023-09-23T20:38:05.610881+0000","last_unstale":"2023-09-23T20:46:00.618997+0000","last_undegraded":"2023-09-23T20:46:00.618997+0000","last_fullsized":"2023-09-23T20:46:00.618997+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:41:08.379959+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:41:08.379959+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:19:39.547923+0000","scrub_duration":0.0019158420000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"86'102","reported_seq":1132,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-09-23T20:46:00.618927+0000","last_change":"2023-09-23T20:32:36.314672+0000","last_active":"2023-09-23T20:46:00.618927+0000","last_peered":"2023-09-23T20:46:00.618927+0000","last_clean":"2023-09-23T20:46:00.618927+0000","last_became_active":"2023-09-23T20:32:36.314231+0000","last_became_peered":"2023-09-23T20:32:36.314231+0000","last_unstale":"2023-09-23T20:46:00.618927+0000","last_undegraded":"2023-09-23T20:46:00.618927+0000","last_fullsized":"2023-09-23T20:46:00.618927+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":102,"log_dups_size":0,"ondisk_log_size":102,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:11:40.514709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030656500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":963,"num_read_kb":1234,"num_write":60,"num_write_kb":225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":485,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-09-23T20:46:00.621703+0000","last_change":"2023-09-23T20:40:19.544384+0000","last_active":"2023-09-23T20:46:00.621703+0000","last_peered":"2023-09-23T20:46:00.621703+0000","last_clean":"2023-09-23T20:46:00.621703+0000","last_became_active":"2023-09-23T20:40:19.544079+0000","last_became_peered":"2023-09-23T20:40:19.544079+0000","last_unstale":"2023-09-23T20:46:00.621703+0000","last_undegraded":"2023-09-23T20:46:00.621703+0000","last_fullsized":"2023-09-23T20:46:00.621703+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:00:19.498631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":481,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-09-23T20:46:00.618990+0000","last_change":"2023-09-23T20:43:46.541173+0000","last_active":"2023-09-23T20:46:00.618990+0000","last_peered":"2023-09-23T20:46:00.618990+0000","last_clean":"2023-09-23T20:46:00.618990+0000","last_became_active":"2023-09-23T20:43:46.540726+0000","last_became_peered":"2023-09-23T20:43:46.540726+0000","last_unstale":"2023-09-23T20:46:00.618990+0000","last_undegraded":"2023-09-23T20:46:00.618990+0000","last_fullsized":"2023-09-23T20:46:00.618990+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:49:04.137885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.e","version":"538'3167","reported_seq":3179,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-09-23T20:46:10.716446+0000","last_change":"2023-09-23T20:45:39.330822+0000","last_active":"2023-09-23T20:46:10.716446+0000","last_peered":"2023-09-23T20:46:10.716446+0000","last_clean":"2023-09-23T20:46:10.716446+0000","last_became_active":"2023-09-23T20:45:39.330513+0000","last_became_peered":"2023-09-23T20:45:39.330513+0000","last_unstale":"2023-09-23T20:46:10.716446+0000","last_undegraded":"2023-09-23T20:46:10.716446+0000","last_fullsized":"2023-09-23T20:46:10.716446+0000","mapping_epoch":531,"log_start":"0'0","ondisk_log_start":"0'0","created":531,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":3167,"log_dups_size":0,"ondisk_log_size":3167,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:35:59.769584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":207552512,"num_objects":3167,"num_object_clones":0,"num_object_copies":9501,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3167,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3167,"num_write_kb":202688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":816,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-09-23T20:46:00.615584+0000","last_change":"2023-09-23T20:38:05.590361+0000","last_active":"2023-09-23T20:46:00.615584+0000","last_peered":"2023-09-23T20:46:00.615584+0000","last_clean":"2023-09-23T20:46:00.615584+0000","last_became_active":"2023-09-23T20:38:05.589985+0000","last_became_peered":"2023-09-23T20:38:05.589985+0000","last_unstale":"2023-09-23T20:46:00.615584+0000","last_undegraded":"2023-09-23T20:46:00.615584+0000","last_fullsized":"2023-09-23T20:46:00.615584+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:03.246271+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:03.246271+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:38:54.351576+0000","scrub_duration":0.0019016700000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":275,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-09-23T20:46:00.619897+0000","last_change":"2023-09-23T20:32:50.549695+0000","last_active":"2023-09-23T20:46:00.619897+0000","last_peered":"2023-09-23T20:46:00.619897+0000","last_clean":"2023-09-23T20:46:00.619897+0000","last_became_active":"2023-09-23T20:32:50.549431+0000","last_became_peered":"2023-09-23T20:32:50.549431+0000","last_unstale":"2023-09-23T20:46:00.619897+0000","last_undegraded":"2023-09-23T20:46:00.619897+0000","last_fullsized":"2023-09-23T20:46:00.619897+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:45:53.666569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"86'128","reported_seq":951,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-09-23T20:46:00.621508+0000","last_change":"2023-09-23T20:38:03.541765+0000","last_active":"2023-09-23T20:46:00.621508+0000","last_peered":"2023-09-23T20:46:00.621508+0000","last_clean":"2023-09-23T20:46:00.621508+0000","last_became_active":"2023-09-23T20:38:03.541277+0000","last_became_peered":"2023-09-23T20:38:03.541277+0000","last_unstale":"2023-09-23T20:46:00.621508+0000","last_undegraded":"2023-09-23T20:46:00.621508+0000","last_fullsized":"2023-09-23T20:46:00.621508+0000","mapping_epoch":404,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":405,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:20:53.834554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000293202,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":579,"num_read_kb":1473,"num_write":96,"num_write_kb":1062,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":319,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-09-23T20:46:00.625081+0000","last_change":"2023-09-23T20:37:43.582105+0000","last_active":"2023-09-23T20:46:00.625081+0000","last_peered":"2023-09-23T20:46:00.625081+0000","last_clean":"2023-09-23T20:46:00.625081+0000","last_became_active":"2023-09-23T20:37:43.581860+0000","last_became_peered":"2023-09-23T20:37:43.581860+0000","last_unstale":"2023-09-23T20:46:00.625081+0000","last_undegraded":"2023-09-23T20:46:00.625081+0000","last_fullsized":"2023-09-23T20:46:00.625081+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:18:38.765486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"538'3098","reported_seq":3113,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-09-23T20:46:10.298894+0000","last_change":"2023-09-23T20:45:39.381283+0000","last_active":"2023-09-23T20:46:10.298894+0000","last_peered":"2023-09-23T20:46:10.298894+0000","last_clean":"2023-09-23T20:46:10.298894+0000","last_became_active":"2023-09-23T20:45:39.380945+0000","last_became_peered":"2023-09-23T20:45:39.380945+0000","last_unstale":"2023-09-23T20:46:10.298894+0000","last_undegraded":"2023-09-23T20:46:10.298894+0000","last_fullsized":"2023-09-23T20:46:10.298894+0000","mapping_epoch":531,"log_start":"0'0","ondisk_log_start":"0'0","created":531,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":3098,"log_dups_size":0,"ondisk_log_size":3098,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:04:01.529754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":203030528,"num_objects":3098,"num_object_clones":0,"num_object_copies":9294,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3098,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3098,"num_write_kb":198272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.4","version":"90'161","reported_seq":753,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-09-23T20:46:04.440880+0000","last_change":"2023-09-23T20:40:22.427435+0000","last_active":"2023-09-23T20:46:04.440880+0000","last_peered":"2023-09-23T20:46:04.440880+0000","last_clean":"2023-09-23T20:46:04.440880+0000","last_became_active":"2023-09-23T20:40:22.427051+0000","last_became_peered":"2023-09-23T20:40:22.427051+0000","last_unstale":"2023-09-23T20:46:04.440880+0000","last_undegraded":"2023-09-23T20:46:04.440880+0000","last_fullsized":"2023-09-23T20:46:04.440880+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:34:54.504554+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:34:54.504554+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:42.001652+0000","scrub_duration":0.011176864,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"86'117","reported_seq":846,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-09-23T20:46:00.619439+0000","last_change":"2023-09-23T20:44:18.635224+0000","last_active":"2023-09-23T20:46:00.619439+0000","last_peered":"2023-09-23T20:46:00.619439+0000","last_clean":"2023-09-23T20:46:00.619439+0000","last_became_active":"2023-09-23T20:44:18.634703+0000","last_became_peered":"2023-09-23T20:44:18.634703+0000","last_unstale":"2023-09-23T20:46:00.619439+0000","last_undegraded":"2023-09-23T20:46:00.619439+0000","last_fullsized":"2023-09-23T20:46:00.619439+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:59:38.030626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046891400000000002,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":893,"num_read_kb":922,"num_write":82,"num_write_kb":4193,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":16777216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":526,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-09-23T20:46:08.659920+0000","last_change":"2023-09-23T20:43:46.521505+0000","last_active":"2023-09-23T20:46:08.659920+0000","last_peered":"2023-09-23T20:46:08.659920+0000","last_clean":"2023-09-23T20:46:08.659920+0000","last_became_active":"2023-09-23T20:43:46.520727+0000","last_became_peered":"2023-09-23T20:43:46.520727+0000","last_unstale":"2023-09-23T20:46:08.659920+0000","last_undegraded":"2023-09-23T20:46:08.659920+0000","last_fullsized":"2023-09-23T20:46:08.659920+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:17:06.164741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":275,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-09-23T20:46:00.614257+0000","last_change":"2023-09-23T20:40:19.530602+0000","last_active":"2023-09-23T20:46:00.614257+0000","last_peered":"2023-09-23T20:46:00.614257+0000","last_clean":"2023-09-23T20:46:00.614257+0000","last_became_active":"2023-09-23T20:40:19.530256+0000","last_became_peered":"2023-09-23T20:40:19.530256+0000","last_unstale":"2023-09-23T20:46:00.614257+0000","last_undegraded":"2023-09-23T20:46:00.614257+0000","last_fullsized":"2023-09-23T20:46:00.614257+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:18:45.658867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"90'161","reported_seq":795,"reported_epoch":538,"state":"active+clean","last_fresh":"2023-09-23T20:46:08.658708+0000","last_change":"2023-09-23T20:43:46.544750+0000","last_active":"2023-09-23T20:46:08.658708+0000","last_peered":"2023-09-23T20:46:08.658708+0000","last_clean":"2023-09-23T20:46:08.658708+0000","last_became_active":"2023-09-23T20:43:46.544395+0000","last_became_peered":"2023-09-23T20:43:46.544395+0000","last_unstale":"2023-09-23T20:46:08.658708+0000","last_undegraded":"2023-09-23T20:46:08.658708+0000","last_fullsized":"2023-09-23T20:46:08.658708+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:06.487351+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:06.487351+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:56.866168+0000","scrub_duration":0.001964379,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":492,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-09-23T20:46:00.618909+0000","last_change":"2023-09-23T20:43:46.541379+0000","last_active":"2023-09-23T20:46:00.618909+0000","last_peered":"2023-09-23T20:46:00.618909+0000","last_clean":"2023-09-23T20:46:00.618909+0000","last_became_active":"2023-09-23T20:43:46.540732+0000","last_became_peered":"2023-09-23T20:43:46.540732+0000","last_unstale":"2023-09-23T20:46:00.618909+0000","last_undegraded":"2023-09-23T20:46:00.618909+0000","last_fullsized":"2023-09-23T20:46:00.618909+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:31:40.605772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":395,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-09-23T20:46:00.624089+0000","last_change":"2023-09-23T20:29:45.538314+0000","last_active":"2023-09-23T20:46:00.624089+0000","last_peered":"2023-09-23T20:46:00.624089+0000","last_clean":"2023-09-23T20:46:00.624089+0000","last_became_active":"2023-09-23T20:29:45.536847+0000","last_became_peered":"2023-09-23T20:29:45.536847+0000","last_unstale":"2023-09-23T20:46:00.624089+0000","last_undegraded":"2023-09-23T20:46:00.624089+0000","last_fullsized":"2023-09-23T20:46:00.624089+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:51.948721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"90'161","reported_seq":612,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-09-23T20:46:00.624491+0000","last_change":"2023-09-23T20:41:06.499511+0000","last_active":"2023-09-23T20:46:00.624491+0000","last_peered":"2023-09-23T20:46:00.624491+0000","last_clean":"2023-09-23T20:46:00.624491+0000","last_became_active":"2023-09-23T20:38:05.601874+0000","last_became_peered":"2023-09-23T20:38:05.601874+0000","last_unstale":"2023-09-23T20:46:00.624491+0000","last_undegraded":"2023-09-23T20:46:00.624491+0000","last_fullsized":"2023-09-23T20:46:00.624491+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:41:06.499396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:41:06.499396+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:31:34.059077+0000","scrub_duration":0.0019670310000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":430,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-09-23T20:46:00.615745+0000","last_change":"2023-09-23T20:43:16.758761+0000","last_active":"2023-09-23T20:46:00.615745+0000","last_peered":"2023-09-23T20:46:00.615745+0000","last_clean":"2023-09-23T20:46:00.615745+0000","last_became_active":"2023-09-23T20:43:16.758407+0000","last_became_peered":"2023-09-23T20:43:16.758407+0000","last_unstale":"2023-09-23T20:46:00.615745+0000","last_undegraded":"2023-09-23T20:46:00.615745+0000","last_fullsized":"2023-09-23T20:46:00.615745+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:59:27.070685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":194,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-09-23T20:46:00.613763+0000","last_change":"2023-09-23T20:34:59.987575+0000","last_active":"2023-09-23T20:46:00.613763+0000","last_peered":"2023-09-23T20:46:00.613763+0000","last_clean":"2023-09-23T20:46:00.613763+0000","last_became_active":"2023-09-23T20:34:59.987229+0000","last_became_peered":"2023-09-23T20:34:59.987229+0000","last_unstale":"2023-09-23T20:46:00.613763+0000","last_undegraded":"2023-09-23T20:46:00.613763+0000","last_fullsized":"2023-09-23T20:46:00.613763+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:34:01.820159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":438,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-09-23T20:46:00.615232+0000","last_change":"2023-09-23T20:43:16.759691+0000","last_active":"2023-09-23T20:46:00.615232+0000","last_peered":"2023-09-23T20:46:00.615232+0000","last_clean":"2023-09-23T20:46:00.615232+0000","last_became_active":"2023-09-23T20:43:16.759309+0000","last_became_peered":"2023-09-23T20:43:16.759309+0000","last_unstale":"2023-09-23T20:46:00.615232+0000","last_undegraded":"2023-09-23T20:46:00.615232+0000","last_fullsized":"2023-09-23T20:46:00.615232+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:58:52.893019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":194,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-09-23T20:46:00.613688+0000","last_change":"2023-09-23T20:34:59.987416+0000","last_active":"2023-09-23T20:46:00.613688+0000","last_peered":"2023-09-23T20:46:00.613688+0000","last_clean":"2023-09-23T20:46:00.613688+0000","last_became_active":"2023-09-23T20:34:59.987117+0000","last_became_peered":"2023-09-23T20:34:59.987117+0000","last_unstale":"2023-09-23T20:46:00.613688+0000","last_undegraded":"2023-09-23T20:46:00.613688+0000","last_fullsized":"2023-09-23T20:46:00.613688+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:36:16.318623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"90'161","reported_seq":793,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-09-23T20:46:00.616764+0000","last_change":"2023-09-23T20:38:05.591992+0000","last_active":"2023-09-23T20:46:00.616764+0000","last_peered":"2023-09-23T20:46:00.616764+0000","last_clean":"2023-09-23T20:46:00.616764+0000","last_became_active":"2023-09-23T20:38:05.591701+0000","last_became_peered":"2023-09-23T20:38:05.591701+0000","last_unstale":"2023-09-23T20:46:00.616764+0000","last_undegraded":"2023-09-23T20:46:00.616764+0000","last_fullsized":"2023-09-23T20:46:00.616764+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:02.384133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:02.384133+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:41.722542+0000","scrub_duration":0.001769098,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":361,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-09-23T20:46:00.624714+0000","last_change":"2023-09-23T20:38:02.542719+0000","last_active":"2023-09-23T20:46:00.624714+0000","last_peered":"2023-09-23T20:46:00.624714+0000","last_clean":"2023-09-23T20:46:00.624714+0000","last_became_active":"2023-09-23T20:38:02.542385+0000","last_became_peered":"2023-09-23T20:38:02.542385+0000","last_unstale":"2023-09-23T20:46:00.624714+0000","last_undegraded":"2023-09-23T20:46:00.624714+0000","last_fullsized":"2023-09-23T20:46:00.624714+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:36:23.673757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":327,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-09-23T20:46:00.624898+0000","last_change":"2023-09-23T20:37:43.584253+0000","last_active":"2023-09-23T20:46:00.624898+0000","last_peered":"2023-09-23T20:46:00.624898+0000","last_clean":"2023-09-23T20:46:00.624898+0000","last_became_active":"2023-09-23T20:37:43.580069+0000","last_became_peered":"2023-09-23T20:37:43.580069+0000","last_unstale":"2023-09-23T20:46:00.624898+0000","last_undegraded":"2023-09-23T20:46:00.624898+0000","last_fullsized":"2023-09-23T20:46:00.624898+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:39:11.482581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"90'161","reported_seq":632,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-09-23T20:46:00.621810+0000","last_change":"2023-09-23T20:36:01.505098+0000","last_active":"2023-09-23T20:46:00.621810+0000","last_peered":"2023-09-23T20:46:00.621810+0000","last_clean":"2023-09-23T20:46:00.621810+0000","last_became_active":"2023-09-23T20:35:26.644707+0000","last_became_peered":"2023-09-23T20:35:26.644707+0000","last_unstale":"2023-09-23T20:46:00.621810+0000","last_undegraded":"2023-09-23T20:46:00.621810+0000","last_fullsized":"2023-09-23T20:46:00.621810+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:01.504973+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:01.504973+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:02:10.880809+0000","scrub_duration":0.00202344,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":571,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-09-23T20:46:00.618251+0000","last_change":"2023-09-23T20:43:46.511657+0000","last_active":"2023-09-23T20:46:00.618251+0000","last_peered":"2023-09-23T20:46:00.618251+0000","last_clean":"2023-09-23T20:46:00.618251+0000","last_became_active":"2023-09-23T20:43:46.511326+0000","last_became_peered":"2023-09-23T20:43:46.511326+0000","last_unstale":"2023-09-23T20:46:00.618251+0000","last_undegraded":"2023-09-23T20:46:00.618251+0000","last_fullsized":"2023-09-23T20:46:00.618251+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:36:46.139261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3324,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":255,"reported_epoch":537,"state":"active+clean","last_fresh":"2023-09-23T20:46:00.614468+0000","last_change":"2023-09-23T20:38:02.538333+0000","last_active":"2023-09-23T20:46:00.614468+0000","last_peered":"2023-09-23T20:46:00.614468+0000","last_clean":"2023-09-23T20:46:00.614468+0000","last_became_active":"2023-09-23T20:38:02.538008+0000","last_became_peered":"2023-09-23T20:38:02.538008+0000","last_unstale":"2023-09-23T20:46:00.614468+0000","last_undegraded":"2023-09-23T20:46:00.614468+0000","last_fullsized":"2023-09-23T20:46:00.614468+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:02:38.789530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":3378905088,"num_objects":51558,"num_object_clones":0,"num_object_copies":154674,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":51558,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":51558,"num_write_kb":3299712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":10082451456,"data_stored":10082451456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":51558,"ondisk_log_size":51558,"up":48,"acting":48,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":90,"num_bytes_recovered":18486,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26691,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5983,"num_read_kb":13200,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":84,"num_bytes_recovered":16777216,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6082,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201697,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1331588,"kb_used_data":803900,"kb_used_omap":0,"kb_used_meta":527680,"kb_avail":92413564,"statfs":{"total":95995035648,"available":94631489536,"internally_reserved":0,"allocated":823193600,"data_stored":821668957,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":540344320},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:46:00 2023","interfaces":[{"interface":"back","average":{"1min":0.71499999999999997,"5min":1.909,"15min":2.1360000000000001},"min":{"1min":0.224,"5min":0.224,"15min":0.224},"max":{"1min":2.5379999999999998,"5min":15.1,"15min":18.315000000000001},"last":1.4970000000000001},{"interface":"front","average":{"1min":1.1220000000000001,"5min":1.891,"15min":1.7969999999999999},"min":{"1min":0.28299999999999997,"5min":0.27800000000000002,"15min":0.23899999999999999},"max":{"1min":5.1619999999999999,"5min":15.327999999999999,"15min":19.355},"last":0.39300000000000002}]},{"osd":1,"last update":"Sat Sep 23 20:46:08 2023","interfaces":[{"interface":"back","average":{"1min":0.95299999999999996,"5min":1.4750000000000001,"15min":2.0009999999999999},"min":{"1min":0.35299999999999998,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":3.4359999999999999,"5min":15.131,"15min":49.965000000000003},"last":3.4239999999999999},{"interface":"front","average":{"1min":1.1259999999999999,"5min":2.6379999999999999,"15min":2.335},"min":{"1min":0.37,"5min":0.26700000000000002,"15min":0.248},"max":{"1min":3.7770000000000001,"5min":15.18,"15min":53.451999999999998},"last":3.7770000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:46:00 2023","interfaces":[{"interface":"back","average":{"1min":1.137,"5min":2.2719999999999998,"15min":2.1659999999999999},"min":{"1min":0.34100000000000003,"5min":0.27900000000000003,"15min":0.27800000000000002},"max":{"1min":3.1080000000000001,"5min":15.087,"15min":16.965},"last":0.42599999999999999},{"interface":"front","average":{"1min":0.83299999999999996,"5min":1.897,"15min":2.3239999999999998},"min":{"1min":0.31,"5min":0.30299999999999999,"15min":0.27400000000000002},"max":{"1min":4.9400000000000004,"5min":15.193,"15min":23.494},"last":3.2629999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:45:30 2023","interfaces":[{"interface":"back","average":{"1min":1.3520000000000001,"5min":1.821,"15min":2.306},"min":{"1min":0.39600000000000002,"5min":0.32500000000000001,"15min":0.311},"max":{"1min":5.9169999999999998,"5min":15.115,"15min":23.352},"last":0.95899999999999996},{"interface":"front","average":{"1min":1.6160000000000001,"5min":2.2349999999999999,"15min":2.4860000000000002},"min":{"1min":0.43099999999999999,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":5.4669999999999996,"5min":15.144,"15min":20.969999999999999},"last":0.91100000000000003}]},{"osd":4,"last update":"Sat Sep 23 20:45:30 2023","interfaces":[{"interface":"back","average":{"1min":0.76100000000000001,"5min":1.3340000000000001,"15min":1.079},"min":{"1min":0.33000000000000002,"5min":0.27000000000000002,"15min":0.23200000000000001},"max":{"1min":2.2949999999999999,"5min":12.260999999999999,"15min":18.364999999999998},"last":0.46100000000000002},{"interface":"front","average":{"1min":1.7170000000000001,"5min":1.7170000000000001,"15min":1.3100000000000001},"min":{"1min":0.376,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":7.2469999999999999,"5min":7.2469999999999999,"15min":23.617000000000001},"last":1.3660000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:45:30 2023","interfaces":[{"interface":"back","average":{"1min":1.2549999999999999,"5min":1.6399999999999999,"15min":1.1639999999999999},"min":{"1min":0.33500000000000002,"5min":0.30499999999999999,"15min":0.26600000000000001},"max":{"1min":5.9619999999999997,"5min":15.005000000000001,"15min":15.005000000000001},"last":0.505},{"interface":"front","average":{"1min":2.024,"5min":1.8080000000000001,"15min":1.252},"min":{"1min":0.377,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":10.641999999999999,"5min":12.731,"15min":18.390000000000001},"last":1.462}]},{"osd":6,"last update":"Sat Sep 23 20:45:30 2023","interfaces":[{"interface":"back","average":{"1min":1.337,"5min":1.851,"15min":1.321},"min":{"1min":0.317,"5min":0.317,"15min":0.30399999999999999},"max":{"1min":5.4269999999999996,"5min":15.038,"15min":18.417000000000002},"last":1.347},{"interface":"front","average":{"1min":1.3149999999999999,"5min":1.696,"15min":1.2509999999999999},"min":{"1min":0.36599999999999999,"5min":0.36499999999999999,"15min":0.318},"max":{"1min":5.5199999999999996,"5min":15.369999999999999,"15min":15.369999999999999},"last":0.61799999999999999}]}]},{"osd":6,"up_from":47,"seq":201863463336,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1840568,"kb_used_data":1388400,"kb_used_omap":0,"kb_used_meta":452160,"kb_avail":91904584,"statfs":{"total":95995035648,"available":94110294016,"internally_reserved":0,"allocated":1421721600,"data_stored":1420208574,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":463011840},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[4],"upper_bound":2},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:46:00 2023","interfaces":[{"interface":"back","average":{"1min":0.74199999999999999,"5min":1.4830000000000001,"15min":1.9390000000000001},"min":{"1min":0.26200000000000001,"5min":0.219,"15min":0.219},"max":{"1min":3.403,"5min":12.552,"15min":53.701999999999998},"last":0.318},{"interface":"front","average":{"1min":0.995,"5min":1.663,"15min":2.2959999999999998},"min":{"1min":0.312,"5min":0.23300000000000001,"15min":0.22700000000000001},"max":{"1min":3.9769999999999999,"5min":10.895,"15min":60.328000000000003},"last":0.56699999999999995}]},{"osd":1,"last update":"Sat Sep 23 20:45:17 2023","interfaces":[{"interface":"back","average":{"1min":1.9390000000000001,"5min":1.8999999999999999,"15min":2.077},"min":{"1min":0.28899999999999998,"5min":0.249,"15min":0.249},"max":{"1min":7.9189999999999996,"5min":9.2520000000000007,"15min":64.274000000000001},"last":0.38300000000000001},{"interface":"front","average":{"1min":2.198,"5min":2.0019999999999998,"15min":1.988},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.28000000000000003},"max":{"1min":7.8849999999999998,"5min":17.786000000000001,"15min":34.490000000000002},"last":3.0790000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:46:00 2023","interfaces":[{"interface":"back","average":{"1min":0.877,"5min":1.9930000000000001,"15min":2.0489999999999999},"min":{"1min":0.40100000000000002,"5min":0.27800000000000002,"15min":0.27700000000000002},"max":{"1min":2.7490000000000001,"5min":9.0150000000000006,"15min":34.317999999999998},"last":0.441},{"interface":"front","average":{"1min":0.70699999999999996,"5min":1.7330000000000001,"15min":1.9490000000000001},"min":{"1min":0.376,"5min":0.34000000000000002,"15min":0.30399999999999999},"max":{"1min":1.4650000000000001,"5min":8.7309999999999999,"15min":34.292999999999999},"last":2.9780000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:45:48 2023","interfaces":[{"interface":"back","average":{"1min":0.99199999999999999,"5min":1.821,"15min":1.911},"min":{"1min":0.45200000000000001,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":2.8090000000000002,"5min":11.664,"15min":11.664},"last":1.78},{"interface":"front","average":{"1min":1.153,"5min":1.8759999999999999,"15min":2.0859999999999999},"min":{"1min":0.378,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":2.8969999999999998,"5min":11.808,"15min":12.875999999999999},"last":4.7510000000000003}]},{"osd":4,"last update":"Sat Sep 23 20:45:17 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.93100000000000005,"15min":0.78800000000000003},"min":{"1min":0.29499999999999998,"5min":0.25600000000000001,"15min":0.251},"max":{"1min":1.4330000000000001,"5min":6.5800000000000001,"15min":23.904},"last":3.052},{"interface":"front","average":{"1min":0.89700000000000002,"5min":1.032,"15min":1.022},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.27000000000000002},"max":{"1min":2.7839999999999998,"5min":5.6260000000000003,"15min":11.515000000000001},"last":0.40999999999999998}]},{"osd":5,"last update":"Sat Sep 23 20:45:17 2023","interfaces":[{"interface":"back","average":{"1min":1.5489999999999999,"5min":1.0369999999999999,"15min":0.85099999999999998},"min":{"1min":0.26400000000000001,"5min":0.253,"15min":0.253},"max":{"1min":7.8529999999999998,"5min":7.8529999999999998,"15min":7.8529999999999998},"last":3.105},{"interface":"front","average":{"1min":1.2549999999999999,"5min":1.488,"15min":1.103},"min":{"1min":0.34699999999999998,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":5.141,"5min":7.1319999999999997,"15min":7.7300000000000004},"last":0.65300000000000002}]},{"osd":7,"last update":"Sat Sep 23 20:45:48 2023","interfaces":[{"interface":"back","average":{"1min":0.68700000000000006,"5min":0.99099999999999999,"15min":0.85699999999999998},"min":{"1min":0.32200000000000001,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":1.631,"5min":8.8350000000000009,"15min":8.8350000000000009},"last":3.0179999999999998},{"interface":"front","average":{"1min":0.79500000000000004,"5min":1.625,"15min":1.204},"min":{"1min":0.35999999999999999,"5min":0.34699999999999998,"15min":0.30299999999999999},"max":{"1min":2.8300000000000001,"5min":7.1929999999999996,"15min":8.8119999999999994},"last":0.746}]}]},{"osd":1,"up_from":266,"seq":1142461300935,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1383468,"kb_used_data":660836,"kb_used_omap":8,"kb_used_meta":722615,"kb_avail":92361684,"statfs":{"total":95995035648,"available":94578364416,"internally_reserved":0,"allocated":676696064,"data_stored":673971324,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8206,"internal_metadata":739958770},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:45:20 2023","interfaces":[{"interface":"back","average":{"1min":2.8450000000000002,"5min":2.5009999999999999,"15min":2.5670000000000002},"min":{"1min":0.221,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":10.289,"5min":16.292000000000002,"15min":52.689999999999998},"last":0.20799999999999999},{"interface":"front","average":{"1min":4.8440000000000003,"5min":2.7829999999999999,"15min":3.1280000000000001},"min":{"1min":0.27500000000000002,"5min":0.20100000000000001,"15min":0.19900000000000001},"max":{"1min":20.346,"5min":20.346,"15min":52.662999999999997},"last":0.254}]},{"osd":2,"last update":"Sat Sep 23 20:45:20 2023","interfaces":[{"interface":"back","average":{"1min":4.2670000000000003,"5min":2.722,"15min":3.2799999999999998},"min":{"1min":0.23699999999999999,"5min":0.216,"15min":0.20899999999999999},"max":{"1min":19.026,"5min":19.026,"15min":32.255000000000003},"last":2.2410000000000001},{"interface":"front","average":{"1min":1.873,"5min":1.974,"15min":2.2770000000000001},"min":{"1min":0.32100000000000001,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":8.9760000000000009,"5min":10.041,"15min":13.404999999999999},"last":0.58399999999999996}]},{"osd":3,"last update":"Sat Sep 23 20:45:20 2023","interfaces":[{"interface":"back","average":{"1min":2.681,"5min":2.2919999999999998,"15min":3.0179999999999998},"min":{"1min":0.34999999999999998,"5min":0.29099999999999998,"15min":0.247},"max":{"1min":9.0299999999999994,"5min":16.245999999999999,"15min":80.745999999999995},"last":0.371},{"interface":"front","average":{"1min":3.7890000000000001,"5min":2.8719999999999999,"15min":3.274},"min":{"1min":0.33600000000000002,"5min":0.26400000000000001,"15min":0.22700000000000001},"max":{"1min":11.534000000000001,"5min":13.497999999999999,"15min":82.988},"last":0.35499999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:45:20 2023","interfaces":[{"interface":"back","average":{"1min":3.7389999999999999,"5min":2.5350000000000001,"15min":2.3610000000000002},"min":{"1min":0.36199999999999999,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":11.791,"5min":20.634,"15min":24.327999999999999},"last":2.3370000000000002},{"interface":"front","average":{"1min":2.1589999999999998,"5min":2.0030000000000001,"15min":1.7569999999999999},"min":{"1min":0.44700000000000001,"5min":0.32600000000000001,"15min":0.311},"max":{"1min":10.468999999999999,"5min":20.835999999999999,"15min":20.835999999999999},"last":0.54700000000000004}]},{"osd":5,"last update":"Sat Sep 23 20:45:20 2023","interfaces":[{"interface":"back","average":{"1min":3.1760000000000002,"5min":2.6680000000000001,"15min":2.3279999999999998},"min":{"1min":0.40000000000000002,"5min":0.308,"15min":0.308},"max":{"1min":7.9290000000000003,"5min":14.366,"15min":33.401000000000003},"last":0.48899999999999999},{"interface":"front","average":{"1min":3.871,"5min":2.6099999999999999,"15min":2.601},"min":{"1min":0.40699999999999997,"5min":0.33100000000000002,"15min":0.32500000000000001},"max":{"1min":11.750999999999999,"5min":11.750999999999999,"15min":33.417000000000002},"last":2.306}]},{"osd":6,"last update":"Sat Sep 23 20:45:20 2023","interfaces":[{"interface":"back","average":{"1min":2.8159999999999998,"5min":2.5270000000000001,"15min":2.5299999999999998},"min":{"1min":0.42399999999999999,"5min":0.35799999999999998,"15min":0.35699999999999998},"max":{"1min":7.9000000000000004,"5min":14.414999999999999,"15min":72.682000000000002},"last":3.2799999999999998},{"interface":"front","average":{"1min":4.4340000000000002,"5min":2.8599999999999999,"15min":2.8530000000000002},"min":{"1min":0.45100000000000001,"5min":0.317,"15min":0.317},"max":{"1min":11.832000000000001,"5min":11.832000000000001,"15min":72.563999999999993},"last":2.3639999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:45:20 2023","interfaces":[{"interface":"back","average":{"1min":3.3300000000000001,"5min":2.6739999999999999,"15min":2.3730000000000002},"min":{"1min":0.40899999999999997,"5min":0.37,"15min":0.37},"max":{"1min":8.7189999999999994,"5min":16.359000000000002,"15min":18.437000000000001},"last":2.4009999999999998},{"interface":"front","average":{"1min":4.4820000000000002,"5min":2.8799999999999999,"15min":2.633},"min":{"1min":0.49399999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":11.864000000000001,"5min":11.864000000000001,"15min":24.524000000000001},"last":2.79}]}]},{"osd":0,"up_from":94,"seq":403726926137,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2841468,"kb_used_data":2052084,"kb_used_omap":7,"kb_used_meta":789368,"kb_avail":90903684,"statfs":{"total":95995035648,"available":93085372416,"internally_reserved":0,"allocated":2101334016,"data_stored":2098770636,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":808312849},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,3,2,2,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:45:24 2023","interfaces":[{"interface":"back","average":{"1min":1.756,"5min":2.419,"15min":2.5790000000000002},"min":{"1min":0.19400000000000001,"5min":0.184,"15min":0.17199999999999999},"max":{"1min":9.1509999999999998,"5min":24.010000000000002,"15min":42.000999999999998},"last":0.78500000000000003},{"interface":"front","average":{"1min":2.3740000000000001,"5min":2.7669999999999999,"15min":2.774},"min":{"1min":0.23799999999999999,"5min":0.22700000000000001,"15min":0.20399999999999999},"max":{"1min":11.292999999999999,"5min":18.097000000000001,"15min":23.036000000000001},"last":0.69199999999999995}]},{"osd":2,"last update":"Sat Sep 23 20:46:01 2023","interfaces":[{"interface":"back","average":{"1min":1.2030000000000001,"5min":2.6800000000000002,"15min":2.7730000000000001},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.193},"max":{"1min":3.714,"5min":48.841000000000001,"15min":62.726999999999997},"last":9.2789999999999999},{"interface":"front","average":{"1min":1.5449999999999999,"5min":2.4990000000000001,"15min":2.7440000000000002},"min":{"1min":0.251,"5min":0.22,"15min":0.188},"max":{"1min":7.3170000000000002,"5min":46.183,"15min":62.689},"last":0.27700000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:45:35 2023","interfaces":[{"interface":"back","average":{"1min":2.7669999999999999,"5min":2.802,"15min":3.2120000000000002},"min":{"1min":0.28499999999999998,"5min":0.248,"15min":0.23400000000000001},"max":{"1min":10.406000000000001,"5min":15.048,"15min":81.400999999999996},"last":0.32600000000000001},{"interface":"front","average":{"1min":2.2669999999999999,"5min":2.1930000000000001,"15min":3.3690000000000002},"min":{"1min":0.29499999999999998,"5min":0.23799999999999999,"15min":0.21199999999999999},"max":{"1min":9.2270000000000003,"5min":10.739000000000001,"15min":81.137},"last":0.751}]},{"osd":4,"last update":"Sat Sep 23 20:46:01 2023","interfaces":[{"interface":"back","average":{"1min":1.036,"5min":1.7,"15min":1.925},"min":{"1min":0.34300000000000003,"5min":0.32000000000000001,"15min":0.25900000000000001},"max":{"1min":3.7690000000000001,"5min":10.368,"15min":12.587999999999999},"last":0.35799999999999998},{"interface":"front","average":{"1min":1.0249999999999999,"5min":1.5720000000000001,"15min":1.8440000000000001},"min":{"1min":0.313,"5min":0.28899999999999998,"15min":0.27400000000000002},"max":{"1min":3.2410000000000001,"5min":9.8480000000000008,"15min":23.001000000000001},"last":1.21}]},{"osd":5,"last update":"Sat Sep 23 20:46:01 2023","interfaces":[{"interface":"back","average":{"1min":1.0029999999999999,"5min":2.0470000000000002,"15min":1.978},"min":{"1min":0.36699999999999999,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":3.492,"5min":10.239000000000001,"15min":22.969999999999999},"last":0.81799999999999995},{"interface":"front","average":{"1min":0.90600000000000003,"5min":2.1080000000000001,"15min":2.2909999999999999},"min":{"1min":0.41199999999999998,"5min":0.34300000000000003,"15min":0.25600000000000001},"max":{"1min":3.161,"5min":12.601000000000001,"15min":26.228000000000002},"last":0.375}]},{"osd":6,"last update":"Sat Sep 23 20:46:01 2023","interfaces":[{"interface":"back","average":{"1min":1.0309999999999999,"5min":1.8500000000000001,"15min":2.0910000000000002},"min":{"1min":0.374,"5min":0.371,"15min":0.32300000000000001},"max":{"1min":3.5470000000000002,"5min":9.9060000000000006,"15min":24.056000000000001},"last":2.0830000000000002},{"interface":"front","average":{"1min":0.90300000000000002,"5min":1.845,"15min":2.1739999999999999},"min":{"1min":0.38200000000000001,"5min":0.36399999999999999,"15min":0.29399999999999998},"max":{"1min":3.0699999999999998,"5min":12.698,"15min":26.268000000000001},"last":0.434}]},{"osd":7,"last update":"Sat Sep 23 20:46:01 2023","interfaces":[{"interface":"back","average":{"1min":1.8859999999999999,"5min":2.25,"15min":2.3769999999999998},"min":{"1min":0.434,"5min":0.41899999999999998,"15min":0.32200000000000001},"max":{"1min":16.553999999999998,"5min":16.553999999999998,"15min":26.177},"last":2.8479999999999999},{"interface":"front","average":{"1min":2.3370000000000002,"5min":2.5089999999999999,"15min":2.3660000000000001},"min":{"1min":0.47699999999999998,"5min":0.40400000000000003,"15min":0.34200000000000003},"max":{"1min":16.649000000000001,"5min":16.649000000000001,"15min":22.422999999999998},"last":2.778}]}]},{"osd":2,"up_from":104,"seq":446676599085,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2043328,"kb_used_data":1283512,"kb_used_omap":11,"kb_used_meta":759796,"kb_avail":91701824,"statfs":{"total":95995035648,"available":93902667776,"internally_reserved":0,"allocated":1314316288,"data_stored":1311747036,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11878,"internal_metadata":778031514},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,2,3,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:46:09 2023","interfaces":[{"interface":"back","average":{"1min":0.89200000000000002,"5min":1.2829999999999999,"15min":2.2949999999999999},"min":{"1min":0.23999999999999999,"5min":0.17299999999999999,"15min":0.17299999999999999},"max":{"1min":4.0620000000000003,"5min":11.07,"15min":82.424999999999997},"last":1.8779999999999999},{"interface":"front","average":{"1min":0.42599999999999999,"5min":1.917,"15min":2.5},"min":{"1min":0.26500000000000001,"5min":0.23599999999999999,"15min":0.19},"max":{"1min":1.0169999999999999,"5min":16.678999999999998,"15min":82.524000000000001},"last":1.5720000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:46:09 2023","interfaces":[{"interface":"back","average":{"1min":0.65100000000000002,"5min":2.101,"15min":2.4660000000000002},"min":{"1min":0.224,"5min":0.224,"15min":0.224},"max":{"1min":2.7090000000000001,"5min":12.981,"15min":46.405000000000001},"last":4.298},{"interface":"front","average":{"1min":0.995,"5min":2.1030000000000002,"15min":2.4809999999999999},"min":{"1min":0.22,"5min":0.22,"15min":0.22},"max":{"1min":3.8079999999999998,"5min":13.215999999999999,"15min":46.459000000000003},"last":2.3530000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:45:33 2023","interfaces":[{"interface":"back","average":{"1min":2.52,"5min":2.1349999999999998,"15min":2.5670000000000002},"min":{"1min":0.25600000000000001,"5min":0.253,"15min":0.253},"max":{"1min":10.999000000000001,"5min":12.847,"15min":19.562999999999999},"last":4.2060000000000004},{"interface":"front","average":{"1min":1.129,"5min":1.292,"15min":2.141},"min":{"1min":0.32000000000000001,"5min":0.26500000000000001,"15min":0.26400000000000001},"max":{"1min":3.8610000000000002,"5min":11.375,"15min":30.704999999999998},"last":2.3959999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:46:09 2023","interfaces":[{"interface":"back","average":{"1min":0.98299999999999998,"5min":1.7450000000000001,"15min":1.776},"min":{"1min":0.34200000000000003,"5min":0.29799999999999999,"15min":0.27500000000000002},"max":{"1min":4.5839999999999996,"5min":12.025,"15min":12.025},"last":0.40100000000000002},{"interface":"front","average":{"1min":0.92700000000000005,"5min":1.78,"15min":1.964},"min":{"1min":0.38800000000000001,"5min":0.30599999999999999,"15min":0.30499999999999999},"max":{"1min":4.4029999999999996,"5min":10.948,"15min":19.440999999999999},"last":1.9490000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:46:09 2023","interfaces":[{"interface":"back","average":{"1min":0.68100000000000005,"5min":1.8879999999999999,"15min":1.861},"min":{"1min":0.33200000000000002,"5min":0.311,"15min":0.311},"max":{"1min":1.6399999999999999,"5min":12.07,"15min":12.07},"last":0.46700000000000003},{"interface":"front","average":{"1min":0.93200000000000005,"5min":1.7889999999999999,"15min":2.0569999999999999},"min":{"1min":0.45200000000000001,"5min":0.37,"15min":0.35299999999999998},"max":{"1min":4.444,"5min":11.061999999999999,"15min":19.513000000000002},"last":1.978}]},{"osd":6,"last update":"Sat Sep 23 20:46:09 2023","interfaces":[{"interface":"back","average":{"1min":0.94599999999999995,"5min":2.0840000000000001,"15min":2.1080000000000001},"min":{"1min":0.46800000000000003,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":2.8900000000000001,"5min":14.047000000000001,"15min":19.486000000000001},"last":2.012},{"interface":"front","average":{"1min":1.2949999999999999,"5min":1.3740000000000001,"15min":1.5940000000000001},"min":{"1min":0.505,"5min":0.34200000000000003,"15min":0.318},"max":{"1min":3.8450000000000002,"5min":11.747,"15min":11.747},"last":2.028}]},{"osd":7,"last update":"Sat Sep 23 20:46:09 2023","interfaces":[{"interface":"back","average":{"1min":1.077,"5min":1.321,"15min":1.5549999999999999},"min":{"1min":0.42999999999999999,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":3.8769999999999998,"5min":8.4949999999999992,"15min":12.734999999999999},"last":2.0720000000000001},{"interface":"front","average":{"1min":0.92300000000000004,"5min":2.0910000000000002,"15min":2.0310000000000001},"min":{"1min":0.40000000000000002,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":2.2170000000000001,"5min":12.099,"15min":12.099},"last":0.439}]}]},{"osd":3,"up_from":109,"seq":468151435558,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1964016,"kb_used_data":976744,"kb_used_omap":4,"kb_used_meta":987259,"kb_avail":91781136,"statfs":{"total":95995035648,"available":93983883264,"internally_reserved":0,"allocated":1000185856,"data_stored":997629977,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4514,"internal_metadata":1010953822},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2],"upper_bound":2},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:45:41 2023","interfaces":[{"interface":"back","average":{"1min":1.419,"5min":2.419,"15min":2.4649999999999999},"min":{"1min":0.249,"5min":0.20899999999999999,"15min":0.191},"max":{"1min":5.5339999999999998,"5min":22.32,"15min":22.32},"last":1.0369999999999999},{"interface":"front","average":{"1min":2.746,"5min":2.7330000000000001,"15min":2.5510000000000002},"min":{"1min":0.313,"5min":0.22900000000000001,"15min":0.19500000000000001},"max":{"1min":17.623000000000001,"5min":22.805,"15min":22.805},"last":1.131}]},{"osd":1,"last update":"Sat Sep 23 20:45:12 2023","interfaces":[{"interface":"back","average":{"1min":3.1800000000000002,"5min":2.669,"15min":2.2349999999999999},"min":{"1min":0.21099999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":15.177,"5min":17.529,"15min":17.529},"last":0.24199999999999999},{"interface":"front","average":{"1min":2.931,"5min":2.488,"15min":2.4279999999999999},"min":{"1min":0.29399999999999998,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":9.6600000000000001,"5min":14.19,"15min":32.603999999999999},"last":2.3940000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:45:41 2023","interfaces":[{"interface":"back","average":{"1min":2.9830000000000001,"5min":3.7170000000000001,"15min":3.0169999999999999},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":12.473000000000001,"5min":68.206999999999994,"15min":68.206999999999994},"last":2.3100000000000001},{"interface":"front","average":{"1min":2.5529999999999999,"5min":2.9199999999999999,"15min":2.7050000000000001},"min":{"1min":0.29999999999999999,"5min":0.245,"15min":0.245},"max":{"1min":17.919,"5min":67.736999999999995,"15min":67.736999999999995},"last":2.0840000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:45:41 2023","interfaces":[{"interface":"back","average":{"1min":1.5109999999999999,"5min":2.1459999999999999,"15min":1.7350000000000001},"min":{"1min":0.41599999999999998,"5min":0.317,"15min":0.30499999999999999},"max":{"1min":5.468,"5min":22.157,"15min":22.157},"last":1.2689999999999999},{"interface":"front","average":{"1min":1.407,"5min":2.2829999999999999,"15min":2.2650000000000001},"min":{"1min":0.372,"5min":0.35399999999999998,"15min":0.29399999999999998},"max":{"1min":4.8339999999999996,"5min":15.868,"15min":19.635000000000002},"last":0.45500000000000002}]},{"osd":5,"last update":"Sat Sep 23 20:45:41 2023","interfaces":[{"interface":"back","average":{"1min":1.349,"5min":2.4020000000000001,"15min":1.927},"min":{"1min":0.45500000000000002,"5min":0.33800000000000002,"15min":0.32200000000000001},"max":{"1min":5.5019999999999998,"5min":22.289999999999999,"15min":22.289999999999999},"last":1.2989999999999999},{"interface":"front","average":{"1min":2.702,"5min":2.0470000000000002,"15min":1.8819999999999999},"min":{"1min":0.47699999999999998,"5min":0.34100000000000003,"15min":0.32000000000000001},"max":{"1min":8.6129999999999995,"5min":12.952,"15min":12.952},"last":1.1759999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:45:41 2023","interfaces":[{"interface":"back","average":{"1min":2.4809999999999999,"5min":2.2599999999999998,"15min":1.974},"min":{"1min":0.44800000000000001,"5min":0.36599999999999999,"15min":0.34300000000000003},"max":{"1min":8.6790000000000003,"5min":12.917,"15min":17.524999999999999},"last":1.5640000000000001},{"interface":"front","average":{"1min":1.7929999999999999,"5min":2.3940000000000001,"15min":2.2320000000000002},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.33700000000000002},"max":{"1min":6.7960000000000003,"5min":15.936,"15min":19.524000000000001},"last":1.5980000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:45:41 2023","interfaces":[{"interface":"back","average":{"1min":1.7,"5min":2.7890000000000001,"15min":2.5310000000000001},"min":{"1min":0.42699999999999999,"5min":0.41499999999999998,"15min":0.36899999999999999},"max":{"1min":4.8630000000000004,"5min":15.976000000000001,"15min":19.571999999999999},"last":0.57899999999999996},{"interface":"front","average":{"1min":2.1179999999999999,"5min":2.4870000000000001,"15min":1.97},"min":{"1min":0.46899999999999997,"5min":0.42799999999999999,"15min":0.40999999999999998},"max":{"1min":10.853999999999999,"5min":22.202999999999999,"15min":22.202999999999999},"last":1.429}]}]},{"osd":4,"up_from":30,"seq":128849019312,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2089784,"kb_used_data":1632176,"kb_used_omap":0,"kb_used_meta":457600,"kb_avail":91655368,"statfs":{"total":95995035648,"available":93855096832,"internally_reserved":0,"allocated":1671348224,"data_stored":1669834932,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":468582400},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,1,4],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:46:00 2023","interfaces":[{"interface":"back","average":{"1min":0.98299999999999998,"5min":1.8660000000000001,"15min":1.7330000000000001},"min":{"1min":0.28899999999999998,"5min":0.251,"15min":0.24099999999999999},"max":{"1min":6.109,"5min":11.709,"15min":24.114999999999998},"last":0.80000000000000004},{"interface":"front","average":{"1min":1.113,"5min":2.1499999999999999,"15min":1.931},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.246},"max":{"1min":4.4349999999999996,"5min":13.74,"15min":24.135999999999999},"last":0.82299999999999995}]},{"osd":1,"last update":"Sat Sep 23 20:45:12 2023","interfaces":[{"interface":"back","average":{"1min":2.6760000000000002,"5min":2.2410000000000001,"15min":1.819},"min":{"1min":0.374,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":9.673,"5min":10.363,"15min":18.190999999999999},"last":3.2709999999999999},{"interface":"front","average":{"1min":1.9890000000000001,"5min":1.778,"15min":1.611},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":8.0730000000000004,"5min":10.426,"15min":18.173999999999999},"last":0.29299999999999998}]},{"osd":2,"last update":"Sat Sep 23 20:45:12 2023","interfaces":[{"interface":"back","average":{"1min":1.593,"5min":1.8069999999999999,"15min":1.788},"min":{"1min":0.35799999999999998,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":7.8559999999999999,"5min":13.051,"15min":13.051},"last":0.25800000000000001},{"interface":"front","average":{"1min":2.7650000000000001,"5min":2.1659999999999999,"15min":2.105},"min":{"1min":0.40799999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":6.8479999999999999,"5min":10.945,"15min":14.614000000000001},"last":0.311}]},{"osd":3,"last update":"Sat Sep 23 20:45:28 2023","interfaces":[{"interface":"back","average":{"1min":2.0790000000000002,"5min":1.972,"15min":2.0339999999999998},"min":{"1min":0.40600000000000003,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":8.6180000000000003,"5min":8.6180000000000003,"15min":14.515000000000001},"last":0.32400000000000001},{"interface":"front","average":{"1min":2.2389999999999999,"5min":2.181,"15min":2.0920000000000001},"min":{"1min":0.40600000000000003,"5min":0.33000000000000002,"15min":0.30299999999999999},"max":{"1min":17.169,"5min":17.169,"15min":17.169},"last":0.34399999999999997}]},{"osd":5,"last update":"Sat Sep 23 20:46:00 2023","interfaces":[{"interface":"back","average":{"1min":0.75800000000000001,"5min":1.5169999999999999,"15min":0.97399999999999998},"min":{"1min":0.316,"5min":0.253,"15min":0.253},"max":{"1min":4.1310000000000002,"5min":10.202,"15min":10.202},"last":3.3130000000000002},{"interface":"front","average":{"1min":1.266,"5min":1.4410000000000001,"15min":1.0680000000000001},"min":{"1min":0.36499999999999999,"5min":0.311,"15min":0.28999999999999998},"max":{"1min":6.5839999999999996,"5min":8.1440000000000001,"15min":8.1440000000000001},"last":1.5820000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:45:12 2023","interfaces":[{"interface":"back","average":{"1min":2.0409999999999999,"5min":1.609,"15min":1.2350000000000001},"min":{"1min":0.33900000000000002,"5min":0.29499999999999998,"15min":0.27600000000000002},"max":{"1min":10.840999999999999,"5min":10.840999999999999,"15min":14.731999999999999},"last":0.28199999999999997},{"interface":"front","average":{"1min":1.6879999999999999,"5min":1.4319999999999999,"15min":1.0840000000000001},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":4.9820000000000002,"5min":7.6150000000000002,"15min":7.6150000000000002},"last":3.3460000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:45:28 2023","interfaces":[{"interface":"back","average":{"1min":17.373999999999999,"5min":5.3650000000000002,"15min":2.532},"min":{"1min":0.46400000000000002,"5min":0.35099999999999998,"15min":0.318},"max":{"1min":313.50099999999998,"5min":313.50099999999998,"15min":313.50099999999998},"last":1.4530000000000001},{"interface":"front","average":{"1min":16.062999999999999,"5min":4.6429999999999998,"15min":2.1019999999999999},"min":{"1min":0.39800000000000002,"5min":0.33200000000000002,"15min":0.32700000000000001},"max":{"1min":312.67500000000001,"5min":312.67500000000001,"15min":312.67500000000001},"last":3.379}]}]},{"osd":5,"up_from":39,"seq":167503724973,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1550580,"kb_used_data":1016940,"kb_used_omap":0,"kb_used_meta":533632,"kb_avail":92194572,"statfs":{"total":95995035648,"available":94407241728,"internally_reserved":0,"allocated":1041346560,"data_stored":1039833994,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":546439168},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:46:04 2023","interfaces":[{"interface":"back","average":{"1min":0.65900000000000003,"5min":1.7090000000000001,"15min":1.744},"min":{"1min":0.24199999999999999,"5min":0.23100000000000001,"15min":0.20699999999999999},"max":{"1min":2.0430000000000001,"5min":15.018000000000001,"15min":18.16},"last":0.24199999999999999},{"interface":"front","average":{"1min":1.45,"5min":1.7529999999999999,"15min":1.6890000000000001},"min":{"1min":0.27900000000000003,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":4.6040000000000001,"5min":15.034000000000001,"15min":15.034000000000001},"last":0.33500000000000002}]},{"osd":1,"last update":"Sat Sep 23 20:45:16 2023","interfaces":[{"interface":"back","average":{"1min":2.3039999999999998,"5min":3.1549999999999998,"15min":2.2850000000000001},"min":{"1min":0.35599999999999998,"5min":0.26500000000000001,"15min":0.253},"max":{"1min":6.2610000000000001,"5min":61.005000000000003,"15min":61.005000000000003},"last":1.085},{"interface":"front","average":{"1min":1.76,"5min":2.5150000000000001,"15min":2.0150000000000001},"min":{"1min":0.40799999999999997,"5min":0.27700000000000002,"15min":0.23499999999999999},"max":{"1min":6.6139999999999999,"5min":59.688000000000002,"15min":59.688000000000002},"last":0.52300000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:46:04 2023","interfaces":[{"interface":"back","average":{"1min":0.88500000000000001,"5min":2.0209999999999999,"15min":2.0249999999999999},"min":{"1min":0.36099999999999999,"5min":0.27100000000000002,"15min":0.26500000000000001},"max":{"1min":1.796,"5min":12.226000000000001,"15min":37.011000000000003},"last":1.78},{"interface":"front","average":{"1min":1.177,"5min":1.829,"15min":1.992},"min":{"1min":0.372,"5min":0.28299999999999997,"15min":0.255},"max":{"1min":3.427,"5min":10.565,"15min":36.835000000000001},"last":1.6499999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:45:24 2023","interfaces":[{"interface":"back","average":{"1min":2.157,"5min":2.3599999999999999,"15min":2.3540000000000001},"min":{"1min":0.44500000000000001,"5min":0.33800000000000002,"15min":0.29599999999999999},"max":{"1min":7.218,"5min":11.032,"15min":36.716000000000001},"last":2.9249999999999998},{"interface":"front","average":{"1min":1.788,"5min":2.29,"15min":2.2629999999999999},"min":{"1min":0.35599999999999998,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":7.8869999999999996,"5min":13.266,"15min":18.504000000000001},"last":0.49399999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:45:58 2023","interfaces":[{"interface":"back","average":{"1min":1.2729999999999999,"5min":1.8580000000000001,"15min":1.147},"min":{"1min":0.33500000000000002,"5min":0.27300000000000002,"15min":0.255},"max":{"1min":7.758,"5min":13.391999999999999,"15min":13.391999999999999},"last":0.41499999999999998},{"interface":"front","average":{"1min":1.4830000000000001,"5min":1.613,"15min":1.083},"min":{"1min":0.41199999999999998,"5min":0.30299999999999999,"15min":0.28299999999999997},"max":{"1min":6.6550000000000002,"5min":7.3799999999999999,"15min":7.3799999999999999},"last":1.3120000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:45:08 2023","interfaces":[{"interface":"back","average":{"1min":1.9350000000000001,"5min":2.161,"15min":1.319},"min":{"1min":0.38100000000000001,"5min":0.317,"15min":0.30199999999999999},"max":{"1min":6.952,"5min":13.734,"15min":13.734},"last":0.63500000000000001},{"interface":"front","average":{"1min":1.1220000000000001,"5min":1.5660000000000001,"15min":1.0980000000000001},"min":{"1min":0.35299999999999998,"5min":0.317,"15min":0.317},"max":{"1min":5.2629999999999999,"5min":8.7449999999999992,"15min":8.7449999999999992},"last":0.62}]},{"osd":7,"last update":"Sat Sep 23 20:45:24 2023","interfaces":[{"interface":"back","average":{"1min":15.821999999999999,"5min":4.6180000000000003,"15min":2.0510000000000002},"min":{"1min":0.44500000000000001,"5min":0.34499999999999997,"15min":0.28499999999999998},"max":{"1min":294.70999999999998,"5min":294.70999999999998,"15min":294.70999999999998},"last":1.2729999999999999},{"interface":"front","average":{"1min":16.821000000000002,"5min":4.8049999999999997,"15min":2.1469999999999998},"min":{"1min":0.432,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":294.74400000000003,"5min":294.74400000000003,"15min":294.74400000000003},"last":0.379}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4722,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1360,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8206,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7156,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3154,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2117926912,"data_stored":2117926912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":678494208,"data_stored":678494208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1326186496,"data_stored":1326186496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":997654528,"data_stored":997654528,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1669595136,"data_stored":1669595136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1036386304,"data_stored":1036386304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1426194432,"data_stored":1426194432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":830013440,"data_stored":830013440,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:46:17.177 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd pool set unique_pool_5 pgp_num 16 2023-09-23T20:46:17.215 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:46:17.264 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:17.486 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:46:17.490 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:17.490 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:46:17.572 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T20:46:17.591 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:46:17.668 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:17.687 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:46:17.770 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:46:17.782 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:17.783 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:46:17.884 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:46:17.946 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:17 smithi018 ceph-mon[179647]: pgmap v1359: 119 pgs: 119 active+clean; 3.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s 2023-09-23T20:46:17.949 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:17 smithi018 ceph-mon[184144]: pgmap v1359: 119 pgs: 119 active+clean; 3.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s 2023-09-23T20:46:17.956 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:17.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:17 smithi149 ceph-mon[170394]: pgmap v1359: 119 pgs: 119 active+clean; 3.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s 2023-09-23T20:46:17.993 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 33 16 59894 59878 113.352 69.0625 0.0109898 0.00881701 2023-09-23T20:46:18.008 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:46:18.035 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:46:18.039 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:18.039 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:46:18.140 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:46:18.198 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:18.269 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:46:18.284 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:18.284 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:46:18.382 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:46:18.386 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:46:18.447 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:18.573 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:46:18.580 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:18.580 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:46:18.681 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:46:18.742 INFO:teuthology.orchestra.run.smithi018.stderr:nodeep-scrub is set 2023-09-23T20:46:18.743 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:18.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:18 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1700983899' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:46:18.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:18 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:46:18.994 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 34 16 60866 60850 111.805 60.75 0.00395789 0.00892027 2023-09-23T20:46:19.054 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:18 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1700983899' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:46:19.055 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:18 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:46:19.055 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:18 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1700983899' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:46:19.055 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:18 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:46:19.800 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:46:19.806 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:19.807 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:46:19.908 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:46:19.965 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:19.994 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 35 16 62026 62010 110.682 72.5 0.0116904 0.00902984 2023-09-23T20:46:20.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:19 smithi018 ceph-mon[184144]: pgmap v1360: 119 pgs: 119 active+clean; 3.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 85 MiB/s wr, 1.35k op/s 2023-09-23T20:46:20.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:19 smithi018 ceph-mon[184144]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:46:20.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:19 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:46:20.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:19 smithi018 ceph-mon[184144]: osdmap e539: 8 total, 8 up, 8 in 2023-09-23T20:46:20.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:19 smithi018 ceph-mon[184144]: failed to encode map e539 with expected crc 2023-09-23T20:46:20.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:19 smithi018 ceph-mon[184144]: failed to encode map e539 with expected crc 2023-09-23T20:46:20.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:19 smithi018 ceph-mon[179647]: pgmap v1360: 119 pgs: 119 active+clean; 3.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 85 MiB/s wr, 1.35k op/s 2023-09-23T20:46:20.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:19 smithi018 ceph-mon[179647]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:46:20.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:19 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:46:20.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:19 smithi018 ceph-mon[179647]: osdmap e539: 8 total, 8 up, 8 in 2023-09-23T20:46:20.144 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:19 smithi018 ceph-mon[179647]: failed to encode map e539 with expected crc 2023-09-23T20:46:20.144 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:19 smithi018 ceph-mon[179647]: failed to encode map e539 with expected crc 2023-09-23T20:46:20.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:19 smithi149 ceph-mon[170394]: pgmap v1360: 119 pgs: 119 active+clean; 3.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 85 MiB/s wr, 1.35k op/s 2023-09-23T20:46:20.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:19 smithi149 ceph-mon[170394]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:46:20.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:19 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:46:20.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:19 smithi149 ceph-mon[170394]: osdmap e539: 8 total, 8 up, 8 in 2023-09-23T20:46:20.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:19 smithi149 ceph-mon[170394]: failed to encode map e539 with expected crc 2023-09-23T20:46:20.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:19 smithi149 ceph-mon[170394]: failed to encode map e539 with expected crc 2023-09-23T20:46:20.712 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:46:20.715 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:20.715 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:46:20.817 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:46:20.884 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:20.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:20 smithi149 ceph-mon[170394]: failed to encode map e539 with expected crc 2023-09-23T20:46:20.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:20 smithi149 ceph-mon[170394]: failed to encode map e539 with expected crc 2023-09-23T20:46:20.994 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 36 16 63267 63251 109.763 77.5625 0.00605577 0.00910595 2023-09-23T20:46:21.037 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:46:21.050 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:21.050 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:46:21.050 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:20 smithi018 ceph-mon[184144]: failed to encode map e539 with expected crc 2023-09-23T20:46:21.051 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:20 smithi018 ceph-mon[184144]: failed to encode map e539 with expected crc 2023-09-23T20:46:21.055 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:20 smithi018 ceph-mon[179647]: failed to encode map e539 with expected crc 2023-09-23T20:46:21.055 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:20 smithi018 ceph-mon[179647]: failed to encode map e539 with expected crc 2023-09-23T20:46:21.151 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:46:21.196 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:21.222 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:46:21.258 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:46:21.302 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:46:21.345 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:46:21.378 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:46:21.426 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:46:21.469 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:46:21.509 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:46:21.516 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:46:21.516 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:46:21.516 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:46:21.517 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:46:21.517 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:46:21.517 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:46:21.517 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:46:21.517 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:46:21.653 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:46:21.659 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:21.660 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:46:21.761 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:46:21.810 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:21 smithi018 ceph-mon[184144]: pgmap v1362: 119 pgs: 119 active+clean; 3.5 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 84 MiB/s wr, 1.35k op/s 2023-09-23T20:46:21.810 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:21 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/588509888' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:46:21.810 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:21 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/4072897841' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T20:46:21.810 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:21 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T20:46:21.814 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:21 smithi018 ceph-mon[179647]: pgmap v1362: 119 pgs: 119 active+clean; 3.5 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 84 MiB/s wr, 1.35k op/s 2023-09-23T20:46:21.814 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:21 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/588509888' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:46:21.814 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:21 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/4072897841' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T20:46:21.814 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:21 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T20:46:21.817 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:21.929 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:46:21.937 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:21.937 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:46:21.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:21 smithi149 ceph-mon[170394]: pgmap v1362: 119 pgs: 119 active+clean; 3.5 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 84 MiB/s wr, 1.35k op/s 2023-09-23T20:46:21.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:21 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/588509888' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:46:21.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:21 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/4072897841' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T20:46:21.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:21 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T20:46:21.994 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 37 16 64463 64447 108.816 74.75 0.00630476 0.00918405 2023-09-23T20:46:22.039 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:46:22.104 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:22.165 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:46:22.170 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:22.170 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:46:22.271 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:46:22.336 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:22.365 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T20:46:22.409 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:46:22.415 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:22.416 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:46:22.517 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:46:22.568 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:22.602 INFO:teuthology.orchestra.run.smithi018.stderr:set pool 10 pgp_num to 16 2023-09-23T20:46:22.698 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:46:22.702 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:22.702 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:46:22.803 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:46:22.857 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:22.928 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:46:22.937 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:22.938 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:46:22.994 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 38 16 65649 65633 107.904 74.125 0.0111292 0.00926184 2023-09-23T20:46:23.009 INFO:teuthology.orchestra.run.smithi149.stdout:{ 2023-09-23T20:46:23.009 INFO:teuthology.orchestra.run.smithi149.stdout: "ops": [], 2023-09-23T20:46:23.009 INFO:teuthology.orchestra.run.smithi149.stdout: "complaint_time": 30, 2023-09-23T20:46:23.009 INFO:teuthology.orchestra.run.smithi149.stdout: "num_blocked_ops": 0 2023-09-23T20:46:23.009 INFO:teuthology.orchestra.run.smithi149.stdout:} 2023-09-23T20:46:23.039 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:46:23.108 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:23.183 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:46:23.187 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:23.187 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:46:23.270 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:46:23.288 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:46:23.352 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:23.833 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:23 smithi018 ceph-mon[184144]: pgmap v1363: 119 pgs: 119 active+clean; 3.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 77 MiB/s wr, 1.24k op/s 2023-09-23T20:46:23.834 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:23 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "16"}]': finished 2023-09-23T20:46:23.834 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:23 smithi018 ceph-mon[184144]: osdmap e540: 8 total, 8 up, 8 in 2023-09-23T20:46:23.834 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:23 smithi018 ceph-mon[184144]: failed to encode map e540 with expected crc 2023-09-23T20:46:23.834 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:23 smithi018 ceph-mon[179647]: pgmap v1363: 119 pgs: 119 active+clean; 3.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 77 MiB/s wr, 1.24k op/s 2023-09-23T20:46:23.834 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:23 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "16"}]': finished 2023-09-23T20:46:23.835 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:23 smithi018 ceph-mon[179647]: osdmap e540: 8 total, 8 up, 8 in 2023-09-23T20:46:23.835 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:23 smithi018 ceph-mon[179647]: failed to encode map e540 with expected crc 2023-09-23T20:46:23.892 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:46:23.896 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:23.897 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:46:23.899 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:23 smithi149 ceph-mon[170394]: pgmap v1363: 119 pgs: 119 active+clean; 3.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 77 MiB/s wr, 1.24k op/s 2023-09-23T20:46:23.899 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:23 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "16"}]': finished 2023-09-23T20:46:23.899 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:23 smithi149 ceph-mon[170394]: osdmap e540: 8 total, 8 up, 8 in 2023-09-23T20:46:23.899 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:23 smithi149 ceph-mon[170394]: failed to encode map e540 with expected crc 2023-09-23T20:46:23.970 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T20:46:23.994 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 39 16 66885 66869 107.118 77.25 0.0136132 0.00933011 2023-09-23T20:46:23.997 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:46:24.027 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:46:24.065 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:24.250 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:46:24.253 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:24.254 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:46:24.355 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:46:24.392 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:24.461 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:46:24.474 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:24.475 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:46:24.575 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:46:24.638 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:24.698 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:46:24.705 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:24.705 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:46:24.706 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:24 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:46:24.806 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:46:24.860 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:24.872 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:46:24.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:24 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:46:24.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:24 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:46:24.946 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:46:24.957 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:24.957 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:46:24.994 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:2023-09-23T20:46:24.994679+0000 min lat: 0.00123458 max lat: 0.175821 avg lat: 0.00942254 2023-09-23T20:46:24.994 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T20:46:24.994 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 40 16 67904 67888 106.032 63.6875 0.00285148 0.00942254 2023-09-23T20:46:25.058 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:46:25.068 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:46:25.125 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:25.298 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:46:25.303 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:25.304 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:46:24 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:46:24] "GET /metrics HTTP/1.1" 200 36572 "" "Prometheus/2.43.0" 2023-09-23T20:46:25.305 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:46:25.405 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:46:25.469 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:25.617 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:25 smithi018 ceph-mon[184144]: pgmap v1365: 119 pgs: 119 active+clean; 3.9 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-09-23T20:46:25.617 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:25 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/4137514850' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:46:25.618 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:25 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:46:25.618 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:25 smithi018 ceph-mon[179647]: pgmap v1365: 119 pgs: 119 active+clean; 3.9 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-09-23T20:46:25.618 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:25 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/4137514850' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:46:25.618 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:25 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:46:25.660 INFO:teuthology.orchestra.run.smithi018.stderr:noscrub is unset 2023-09-23T20:46:25.671 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:46:25.682 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:25.682 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:46:25.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:25 smithi149 ceph-mon[170394]: pgmap v1365: 119 pgs: 119 active+clean; 3.9 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2023-09-23T20:46:25.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:25 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/4137514850' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:46:25.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:25 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:46:25.784 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:46:25.812 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:25.994 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 41 16 68937 68921 105.021 64.5625 0.0188715 0.00951592 2023-09-23T20:46:26.353 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:46:26.360 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:26.361 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:46:26.462 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:46:26.524 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:26.626 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:26 smithi018 ceph-mon[179647]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:46:26.627 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:26 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:46:26.627 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:26 smithi018 ceph-mon[179647]: osdmap e541: 8 total, 8 up, 8 in 2023-09-23T20:46:26.627 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:26 smithi018 ceph-mon[179647]: failed to encode map e541 with expected crc 2023-09-23T20:46:26.643 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:46:26.648 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:26.649 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:46:26.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:26 smithi149 ceph-mon[170394]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:46:26.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:26 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:46:26.717 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:26 smithi149 ceph-mon[170394]: osdmap e541: 8 total, 8 up, 8 in 2023-09-23T20:46:26.718 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:26 smithi149 ceph-mon[170394]: failed to encode map e541 with expected crc 2023-09-23T20:46:26.750 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:46:26.809 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:26.896 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:26 smithi018 ceph-mon[184144]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:46:26.896 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:26 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:46:26.896 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:26 smithi018 ceph-mon[184144]: osdmap e541: 8 total, 8 up, 8 in 2023-09-23T20:46:26.896 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:26 smithi018 ceph-mon[184144]: failed to encode map e541 with expected crc 2023-09-23T20:46:26.994 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 42 16 70024 70008 104.138 67.9375 0.0037625 0.00959265 2023-09-23T20:46:27.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:27 smithi149 ceph-mon[170394]: pgmap v1367: 119 pgs: 119 active+clean; 4.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 75 MiB/s wr, 1.19k op/s 2023-09-23T20:46:27.996 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 43 16 71176 71160 103.387 72 0.0171063 0.00966705 2023-09-23T20:46:28.056 INFO:teuthology.orchestra.run.smithi149.stdout:{ 2023-09-23T20:46:28.057 INFO:teuthology.orchestra.run.smithi149.stdout: "size": 20, 2023-09-23T20:46:28.057 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 600, 2023-09-23T20:46:28.057 INFO:teuthology.orchestra.run.smithi149.stdout: "ops": [ 2023-09-23T20:46:28.057 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.057 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:105803 8.12 8:4d1c432a:::benchmark_data_smithi018_341490_object16480:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e410)", 2023-09-23T20:46:28.057 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:38:26.699836+0000", 2023-09-23T20:46:28.057 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 481.35325264099998, 2023-09-23T20:46:28.057 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.85041256099999996, 2023-09-23T20:46:28.057 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:46:28.057 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:46:28.057 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:46:28.058 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:46:28.058 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:46:28.058 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 105803 2023-09-23T20:46:28.058 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.058 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:46:28.058 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.058 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:46:28.058 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:26.699836+0000", 2023-09-23T20:46:28.058 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:28.058 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.058 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.059 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:46:28.059 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:26.699836+0000", 2023-09-23T20:46:28.059 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:28.059 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.059 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.059 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:46:28.059 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:26.699832+0000", 2023-09-23T20:46:28.059 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999967 2023-09-23T20:46:28.059 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.059 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.059 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:46:28.060 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:26.699849+0000", 2023-09-23T20:46:28.060 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.7223000000000001e-05 2023-09-23T20:46:28.060 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.060 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.060 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:46:28.060 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:26.699870+0000", 2023-09-23T20:46:28.060 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.1005e-05 2023-09-23T20:46:28.060 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.060 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.060 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:46:28.060 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:26.699884+0000", 2023-09-23T20:46:28.060 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.3349e-05 2023-09-23T20:46:28.061 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.061 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.061 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:46:28.061 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:26.703306+0000", 2023-09-23T20:46:28.061 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0034221799999999999 2023-09-23T20:46:28.061 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.061 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.061 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:46:28.061 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:26.703327+0000", 2023-09-23T20:46:28.061 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.0890000000000002e-05 2023-09-23T20:46:28.061 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.061 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.062 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:46:28.062 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.504227+0000", 2023-09-23T20:46:28.062 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.800900163 2023-09-23T20:46:28.062 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.062 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.062 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for degraded object", 2023-09-23T20:46:28.062 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.507106+0000", 2023-09-23T20:46:28.062 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0028788500000000001 2023-09-23T20:46:28.062 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.062 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.062 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:46:28.063 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.543770+0000", 2023-09-23T20:46:28.063 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.036664344000000001 2023-09-23T20:46:28.063 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.063 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.063 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:46:28.063 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.543848+0000", 2023-09-23T20:46:28.063 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.8217e-05 2023-09-23T20:46:28.063 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.063 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.063 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:46:28.063 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.543938+0000", 2023-09-23T20:46:28.063 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.9777000000000005e-05 2023-09-23T20:46:28.064 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.064 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.064 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:46:28.064 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.545834+0000", 2023-09-23T20:46:28.064 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0018956579999999999 2023-09-23T20:46:28.064 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.064 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.064 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:46:28.064 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.546152+0000", 2023-09-23T20:46:28.064 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000318311 2023-09-23T20:46:28.064 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.064 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.065 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:46:28.065 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.549292+0000", 2023-09-23T20:46:28.065 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0031395870000000001 2023-09-23T20:46:28.065 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.065 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.065 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:46:28.065 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.549336+0000", 2023-09-23T20:46:28.065 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.3982e-05 2023-09-23T20:46:28.065 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.065 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.065 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:46:28.066 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.550248+0000", 2023-09-23T20:46:28.066 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00091245400000000004 2023-09-23T20:46:28.066 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:28.066 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:46:28.066 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:28.066 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.066 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.066 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:106049 8.14 8:2acb269a:::benchmark_data_smithi018_341490_object16726:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e416)", 2023-09-23T20:46:28.066 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:38:37.916629+0000", 2023-09-23T20:46:28.066 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 470.13645942599999, 2023-09-23T20:46:28.066 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.3484781940000001, 2023-09-23T20:46:28.066 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:46:28.067 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:46:28.067 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:46:28.067 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:46:28.067 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:46:28.067 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 106049 2023-09-23T20:46:28.067 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.067 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:46:28.067 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.067 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:46:28.067 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.916629+0000", 2023-09-23T20:46:28.067 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:28.068 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.068 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.068 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:46:28.068 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.916629+0000", 2023-09-23T20:46:28.068 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:28.068 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.068 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.068 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:46:28.068 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.916627+0000", 2023-09-23T20:46:28.068 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999981 2023-09-23T20:46:28.068 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.068 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.069 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:46:28.069 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.916636+0000", 2023-09-23T20:46:28.069 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.0790000000000007e-06 2023-09-23T20:46:28.069 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.069 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.069 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:46:28.069 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.916657+0000", 2023-09-23T20:46:28.069 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.0982e-05 2023-09-23T20:46:28.069 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.069 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.069 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:46:28.070 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.916665+0000", 2023-09-23T20:46:28.070 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.3289999999999994e-06 2023-09-23T20:46:28.070 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.070 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.070 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:46:28.070 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.916720+0000", 2023-09-23T20:46:28.070 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.5334999999999998e-05 2023-09-23T20:46:28.070 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.070 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.070 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:46:28.070 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.916731+0000", 2023-09-23T20:46:28.071 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0329000000000001e-05 2023-09-23T20:46:28.071 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.071 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.071 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:46:28.071 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.258803+0000", 2023-09-23T20:46:28.071 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.342072444 2023-09-23T20:46:28.071 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.071 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.071 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:46:28.071 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.258947+0000", 2023-09-23T20:46:28.071 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00014386600000000001 2023-09-23T20:46:28.072 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.072 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.072 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 2,5", 2023-09-23T20:46:28.072 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.259044+0000", 2023-09-23T20:46:28.072 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.7132000000000004e-05 2023-09-23T20:46:28.072 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.072 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.072 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:46:28.072 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.261960+0000", 2023-09-23T20:46:28.072 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.002915866 2023-09-23T20:46:28.072 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.073 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.073 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:46:28.073 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.261969+0000", 2023-09-23T20:46:28.073 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.2299999999999997e-06 2023-09-23T20:46:28.073 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.073 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.073 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:46:28.074 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.264756+0000", 2023-09-23T20:46:28.074 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0027866610000000002 2023-09-23T20:46:28.074 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.075 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.075 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:46:28.075 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.264798+0000", 2023-09-23T20:46:28.075 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.2058999999999998e-05 2023-09-23T20:46:28.075 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.075 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.075 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:46:28.075 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.265107+0000", 2023-09-23T20:46:28.075 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000308943 2023-09-23T20:46:28.075 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:28.075 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:46:28.076 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:28.076 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.076 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.076 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:106054 8.12 8:4dc1e34a:::benchmark_data_smithi018_341490_object16731:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e416)", 2023-09-23T20:46:28.076 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:38:37.939136+0000", 2023-09-23T20:46:28.076 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 470.11395194800002, 2023-09-23T20:46:28.076 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.3347287539999999, 2023-09-23T20:46:28.076 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:46:28.076 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:46:28.076 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:46:28.076 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:46:28.076 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:46:28.077 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 106054 2023-09-23T20:46:28.077 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.077 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:46:28.077 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.077 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:46:28.077 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.939136+0000", 2023-09-23T20:46:28.077 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:28.077 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.077 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.077 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:46:28.077 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.939136+0000", 2023-09-23T20:46:28.078 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:28.078 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.078 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.078 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:46:28.078 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.939135+0000", 2023-09-23T20:46:28.078 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:46:28.078 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.078 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.078 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:46:28.078 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.939141+0000", 2023-09-23T20:46:28.078 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.0850000000000002e-06 2023-09-23T20:46:28.079 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.079 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.079 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:46:28.079 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.939155+0000", 2023-09-23T20:46:28.079 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.3912e-05 2023-09-23T20:46:28.079 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.079 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.079 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:46:28.079 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.939160+0000", 2023-09-23T20:46:28.079 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.6289999999999998e-06 2023-09-23T20:46:28.079 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.079 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.080 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:46:28.080 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.939210+0000", 2023-09-23T20:46:28.080 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.9122999999999997e-05 2023-09-23T20:46:28.080 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.080 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.080 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:46:28.080 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.939218+0000", 2023-09-23T20:46:28.080 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.2309999999999996e-06 2023-09-23T20:46:28.080 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.080 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.080 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:46:28.080 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.260728+0000", 2023-09-23T20:46:28.081 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.321510314 2023-09-23T20:46:28.081 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.081 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.081 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:46:28.081 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.260841+0000", 2023-09-23T20:46:28.081 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000112938 2023-09-23T20:46:28.081 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.081 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.081 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:46:28.081 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.260955+0000", 2023-09-23T20:46:28.081 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000113584 2023-09-23T20:46:28.081 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.082 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.082 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:46:28.082 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.266252+0000", 2023-09-23T20:46:28.082 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0052969810000000001 2023-09-23T20:46:28.082 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.082 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.082 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:46:28.082 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.273531+0000", 2023-09-23T20:46:28.082 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0072792580000000003 2023-09-23T20:46:28.082 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.082 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.083 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:46:28.083 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.273639+0000", 2023-09-23T20:46:28.083 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000107676 2023-09-23T20:46:28.083 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.083 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.083 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:46:28.083 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.273672+0000", 2023-09-23T20:46:28.083 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.3679000000000001e-05 2023-09-23T20:46:28.083 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.083 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.083 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:46:28.083 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.273865+0000", 2023-09-23T20:46:28.084 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00019271100000000001 2023-09-23T20:46:28.084 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:28.084 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:46:28.084 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:28.084 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.084 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.084 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:106060 8.4 8:25362c51:::benchmark_data_smithi018_341490_object16737:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e416)", 2023-09-23T20:46:28.084 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:38:38.009403+0000", 2023-09-23T20:46:28.084 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 470.04368481699998, 2023-09-23T20:46:28.084 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.262631214, 2023-09-23T20:46:28.084 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:46:28.085 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:46:28.085 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:46:28.085 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:46:28.085 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:46:28.085 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 106060 2023-09-23T20:46:28.085 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.085 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:46:28.085 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.085 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:46:28.085 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.009403+0000", 2023-09-23T20:46:28.085 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:28.085 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.086 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.086 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:46:28.086 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.009403+0000", 2023-09-23T20:46:28.086 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:28.086 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.086 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.086 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:46:28.086 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.009402+0000", 2023-09-23T20:46:28.086 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:46:28.086 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.088 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.088 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:46:28.088 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.009407+0000", 2023-09-23T20:46:28.088 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.3259999999999997e-06 2023-09-23T20:46:28.088 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.088 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.088 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:46:28.088 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.009419+0000", 2023-09-23T20:46:28.088 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.2813000000000001e-05 2023-09-23T20:46:28.089 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.090 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.090 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:46:28.090 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.009425+0000", 2023-09-23T20:46:28.090 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.2839999999999997e-06 2023-09-23T20:46:28.090 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.090 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.091 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:46:28.091 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.009487+0000", 2023-09-23T20:46:28.091 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.2145999999999998e-05 2023-09-23T20:46:28.091 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.091 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.092 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:46:28.092 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.009500+0000", 2023-09-23T20:46:28.092 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.3382e-05 2023-09-23T20:46:28.092 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.092 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.092 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:46:28.093 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.261986+0000", 2023-09-23T20:46:28.093 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.2524858210000001 2023-09-23T20:46:28.093 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.093 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.093 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:46:28.093 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.262103+0000", 2023-09-23T20:46:28.093 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000116661 2023-09-23T20:46:28.093 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.094 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.094 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 2,5", 2023-09-23T20:46:28.094 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.262187+0000", 2023-09-23T20:46:28.094 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.4349000000000002e-05 2023-09-23T20:46:28.094 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.094 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.094 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:46:28.095 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.266297+0000", 2023-09-23T20:46:28.095 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0041101369999999998 2023-09-23T20:46:28.095 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.095 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.095 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:46:28.095 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.266347+0000", 2023-09-23T20:46:28.095 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.9561999999999997e-05 2023-09-23T20:46:28.096 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.096 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.096 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:46:28.096 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.271596+0000", 2023-09-23T20:46:28.096 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0052490719999999996 2023-09-23T20:46:28.096 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.096 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.097 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:46:28.097 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.271648+0000", 2023-09-23T20:46:28.097 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.2011000000000001e-05 2023-09-23T20:46:28.097 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.097 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.097 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:46:28.097 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.272035+0000", 2023-09-23T20:46:28.097 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000386702 2023-09-23T20:46:28.097 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:28.097 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:46:28.097 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:28.097 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.098 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.098 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:106077 8.2 8:405ecd88:::benchmark_data_smithi018_341490_object16754:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e416)", 2023-09-23T20:46:28.098 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:38:38.055215+0000", 2023-09-23T20:46:28.098 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 469.99787270000002, 2023-09-23T20:46:28.098 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.218947317, 2023-09-23T20:46:28.098 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:46:28.098 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:46:28.099 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:46:28.099 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:46:28.099 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:46:28.099 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 106077 2023-09-23T20:46:28.099 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.099 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:46:28.099 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.099 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:46:28.099 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.055215+0000", 2023-09-23T20:46:28.099 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:28.099 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.099 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.100 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:46:28.100 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.055215+0000", 2023-09-23T20:46:28.100 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:28.100 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.100 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.100 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:46:28.100 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.055214+0000", 2023-09-23T20:46:28.100 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:46:28.100 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.100 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.100 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:46:28.100 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.055221+0000", 2023-09-23T20:46:28.101 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.3150000000000003e-06 2023-09-23T20:46:28.101 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.101 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.101 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:46:28.101 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.055238+0000", 2023-09-23T20:46:28.101 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.6225000000000001e-05 2023-09-23T20:46:28.101 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.101 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.101 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:46:28.101 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.055244+0000", 2023-09-23T20:46:28.101 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.6780000000000003e-06 2023-09-23T20:46:28.101 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.102 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.102 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:46:28.102 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.055311+0000", 2023-09-23T20:46:28.102 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.6406000000000001e-05 2023-09-23T20:46:28.102 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.102 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.102 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:46:28.102 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.055328+0000", 2023-09-23T20:46:28.102 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.7190999999999999e-05 2023-09-23T20:46:28.102 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.102 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.103 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:46:28.103 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.259906+0000", 2023-09-23T20:46:28.103 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.2045777580000001 2023-09-23T20:46:28.103 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.103 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.103 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:46:28.103 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.259995+0000", 2023-09-23T20:46:28.103 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.9866999999999999e-05 2023-09-23T20:46:28.103 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.103 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.103 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:46:28.103 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.260085+0000", 2023-09-23T20:46:28.104 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.9098999999999995e-05 2023-09-23T20:46:28.104 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.104 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.104 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:46:28.104 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.266243+0000", 2023-09-23T20:46:28.104 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0061579579999999998 2023-09-23T20:46:28.104 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.104 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.104 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:46:28.104 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.273553+0000", 2023-09-23T20:46:28.104 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0073100300000000003 2023-09-23T20:46:28.104 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.105 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.105 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:46:28.105 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.273929+0000", 2023-09-23T20:46:28.105 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00037609299999999998 2023-09-23T20:46:28.105 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.105 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.105 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:46:28.105 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.273960+0000", 2023-09-23T20:46:28.105 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.1851999999999999e-05 2023-09-23T20:46:28.105 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.105 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.105 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:46:28.106 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.274163+0000", 2023-09-23T20:46:28.106 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000202303 2023-09-23T20:46:28.106 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:28.106 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:46:28.106 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:28.106 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.106 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.106 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:106082 8.2 8:43f6991f:::benchmark_data_smithi018_341490_object16759:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e416)", 2023-09-23T20:46:28.106 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:38:38.078328+0000", 2023-09-23T20:46:28.106 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 469.97475977599998, 2023-09-23T20:46:28.107 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.200252715, 2023-09-23T20:46:28.107 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:46:28.107 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:46:28.107 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:46:28.107 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:46:28.107 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:46:28.107 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 106082 2023-09-23T20:46:28.107 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.107 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:46:28.107 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.107 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:46:28.107 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.078328+0000", 2023-09-23T20:46:28.107 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:28.108 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.108 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.108 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:46:28.108 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.078328+0000", 2023-09-23T20:46:28.108 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:28.108 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.108 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.109 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:46:28.109 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.078327+0000", 2023-09-23T20:46:28.109 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999981 2023-09-23T20:46:28.109 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.109 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.109 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:46:28.109 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.078334+0000", 2023-09-23T20:46:28.109 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.4769999999999997e-06 2023-09-23T20:46:28.110 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.110 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.110 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:46:28.110 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.078353+0000", 2023-09-23T20:46:28.110 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.8457e-05 2023-09-23T20:46:28.110 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.110 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.110 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:46:28.110 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.078359+0000", 2023-09-23T20:46:28.111 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.6359999999999997e-06 2023-09-23T20:46:28.111 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.111 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.111 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:46:28.111 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.078428+0000", 2023-09-23T20:46:28.111 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.8532e-05 2023-09-23T20:46:28.111 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.111 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.111 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:46:28.111 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.078441+0000", 2023-09-23T20:46:28.111 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.2943e-05 2023-09-23T20:46:28.112 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.112 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.112 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:46:28.112 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.268537+0000", 2023-09-23T20:46:28.112 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.190096729 2023-09-23T20:46:28.112 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.112 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.112 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:46:28.112 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.268680+0000", 2023-09-23T20:46:28.112 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00014273500000000001 2023-09-23T20:46:28.112 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.112 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.113 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:46:28.113 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.268813+0000", 2023-09-23T20:46:28.113 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00013306700000000001 2023-09-23T20:46:28.113 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.113 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.113 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:46:28.113 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.272998+0000", 2023-09-23T20:46:28.113 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0041846360000000003 2023-09-23T20:46:28.113 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.113 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.113 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:46:28.114 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.274587+0000", 2023-09-23T20:46:28.114 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0015889490000000001 2023-09-23T20:46:28.114 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.114 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.114 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:46:28.114 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.278215+0000", 2023-09-23T20:46:28.114 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0036286130000000002 2023-09-23T20:46:28.114 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.114 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.114 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:46:28.115 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.278259+0000", 2023-09-23T20:46:28.115 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.3995000000000001e-05 2023-09-23T20:46:28.115 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.115 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.115 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:46:28.115 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.278581+0000", 2023-09-23T20:46:28.115 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00032174500000000002 2023-09-23T20:46:28.115 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:28.115 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:46:28.116 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:28.116 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.116 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.116 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:106090 8.4 8:202238a2:::benchmark_data_smithi018_341490_object16767:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e416)", 2023-09-23T20:46:28.116 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:38:38.108972+0000", 2023-09-23T20:46:28.116 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 469.94411608799999, 2023-09-23T20:46:28.116 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1634425150000001, 2023-09-23T20:46:28.116 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:46:28.116 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:46:28.116 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:46:28.116 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:46:28.117 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:46:28.117 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 106090 2023-09-23T20:46:28.117 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.117 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:46:28.117 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.117 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:46:28.117 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.108972+0000", 2023-09-23T20:46:28.117 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:28.117 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.118 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.118 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:46:28.118 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.108972+0000", 2023-09-23T20:46:28.118 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:28.118 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.118 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.118 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:46:28.118 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.108971+0000", 2023-09-23T20:46:28.118 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:46:28.118 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.118 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.119 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:46:28.119 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.108976+0000", 2023-09-23T20:46:28.119 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.4110000000000004e-06 2023-09-23T20:46:28.119 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.119 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.119 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:46:28.119 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.108989+0000", 2023-09-23T20:46:28.119 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.3166000000000001e-05 2023-09-23T20:46:28.119 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.119 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.119 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:46:28.120 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.108995+0000", 2023-09-23T20:46:28.120 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.4480000000000002e-06 2023-09-23T20:46:28.120 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.120 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.120 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:46:28.120 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.109033+0000", 2023-09-23T20:46:28.120 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.8402000000000003e-05 2023-09-23T20:46:28.120 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.120 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.120 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:46:28.120 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.109042+0000", 2023-09-23T20:46:28.120 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.5790000000000004e-06 2023-09-23T20:46:28.121 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.121 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.121 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:46:28.121 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.263413+0000", 2023-09-23T20:46:28.121 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1543713289999999 2023-09-23T20:46:28.121 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.121 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.121 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:46:28.121 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.263504+0000", 2023-09-23T20:46:28.122 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.0820000000000001e-05 2023-09-23T20:46:28.122 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.122 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.122 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 2,5", 2023-09-23T20:46:28.122 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.263583+0000", 2023-09-23T20:46:28.122 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.9167999999999998e-05 2023-09-23T20:46:28.122 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.122 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.123 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:46:28.123 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.266401+0000", 2023-09-23T20:46:28.123 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0028183430000000001 2023-09-23T20:46:28.123 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.123 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.123 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:46:28.123 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.266409+0000", 2023-09-23T20:46:28.123 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.0190000000000006e-06 2023-09-23T20:46:28.123 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.123 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.123 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:46:28.124 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.272098+0000", 2023-09-23T20:46:28.124 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0056886230000000003 2023-09-23T20:46:28.124 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.124 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.124 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:46:28.124 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.272162+0000", 2023-09-23T20:46:28.124 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.3738999999999994e-05 2023-09-23T20:46:28.124 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.124 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.125 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:46:28.125 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.272415+0000", 2023-09-23T20:46:28.125 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000252768 2023-09-23T20:46:28.125 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:28.125 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:46:28.125 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:28.125 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.125 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.125 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:106105 8.12 8:4ed2b003:::benchmark_data_smithi018_341490_object16782:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e416)", 2023-09-23T20:46:28.125 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:38:38.247356+0000", 2023-09-23T20:46:28.126 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 469.80573220700001, 2023-09-23T20:46:28.126 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.027019473, 2023-09-23T20:46:28.126 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:46:28.126 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:46:28.126 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:46:28.126 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:46:28.126 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:46:28.126 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 106105 2023-09-23T20:46:28.126 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.126 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:46:28.126 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.127 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:46:28.127 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.247356+0000", 2023-09-23T20:46:28.127 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:28.127 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.127 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.127 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:46:28.127 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.247356+0000", 2023-09-23T20:46:28.127 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:28.127 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.127 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.127 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:46:28.128 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.247355+0000", 2023-09-23T20:46:28.128 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:46:28.128 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.128 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.128 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:46:28.128 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.247360+0000", 2023-09-23T20:46:28.128 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.5809999999999996e-06 2023-09-23T20:46:28.128 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.128 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.128 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:46:28.128 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.247374+0000", 2023-09-23T20:46:28.128 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.3869999999999999e-05 2023-09-23T20:46:28.129 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.129 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.129 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:46:28.129 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.247380+0000", 2023-09-23T20:46:28.129 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.5670000000000002e-06 2023-09-23T20:46:28.129 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.129 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.129 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:46:28.129 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.247458+0000", 2023-09-23T20:46:28.129 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.7976e-05 2023-09-23T20:46:28.129 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.129 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.130 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:46:28.130 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.247467+0000", 2023-09-23T20:46:28.130 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.8230000000000006e-06 2023-09-23T20:46:28.130 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.130 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.130 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:46:28.130 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.262332+0000", 2023-09-23T20:46:28.130 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.014864722 2023-09-23T20:46:28.130 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.130 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.130 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:46:28.131 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.262420+0000", 2023-09-23T20:46:28.131 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.7597999999999997e-05 2023-09-23T20:46:28.131 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.131 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.131 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:46:28.131 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.262520+0000", 2023-09-23T20:46:28.131 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.9769e-05 2023-09-23T20:46:28.131 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.131 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.131 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:46:28.131 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.266258+0000", 2023-09-23T20:46:28.131 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.003738285 2023-09-23T20:46:28.132 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.132 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.132 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:46:28.132 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.273571+0000", 2023-09-23T20:46:28.132 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0073127560000000001 2023-09-23T20:46:28.132 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.132 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.132 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:46:28.132 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.274186+0000", 2023-09-23T20:46:28.132 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00061492000000000003 2023-09-23T20:46:28.132 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.133 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.133 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:46:28.133 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.274218+0000", 2023-09-23T20:46:28.133 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.2221000000000002e-05 2023-09-23T20:46:28.133 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.133 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.133 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:46:28.133 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.274375+0000", 2023-09-23T20:46:28.133 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000157708 2023-09-23T20:46:28.133 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:28.133 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:46:28.134 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:28.134 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.134 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.134 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:106118 8.4 8:27850836:::benchmark_data_smithi018_341490_object16795:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e416)", 2023-09-23T20:46:28.134 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:38:38.309923+0000", 2023-09-23T20:46:28.134 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 469.74316497799998, 2023-09-23T20:46:28.134 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.96280872399999995, 2023-09-23T20:46:28.134 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:46:28.134 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:46:28.134 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:46:28.134 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:46:28.135 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:46:28.135 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 106118 2023-09-23T20:46:28.135 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.135 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:46:28.135 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.135 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:46:28.135 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.309923+0000", 2023-09-23T20:46:28.135 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:28.135 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.135 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.135 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:46:28.135 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.309923+0000", 2023-09-23T20:46:28.136 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:28.136 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.136 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.136 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:46:28.136 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.309922+0000", 2023-09-23T20:46:28.136 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:46:28.136 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.136 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.136 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:46:28.136 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.309927+0000", 2023-09-23T20:46:28.136 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.2959999999999998e-06 2023-09-23T20:46:28.136 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.137 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.137 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:46:28.137 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.309941+0000", 2023-09-23T20:46:28.137 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.3375e-05 2023-09-23T20:46:28.137 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.137 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.137 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:46:28.137 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.309947+0000", 2023-09-23T20:46:28.138 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.7440000000000001e-06 2023-09-23T20:46:28.138 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.138 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.138 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:46:28.138 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.309999+0000", 2023-09-23T20:46:28.138 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.1266e-05 2023-09-23T20:46:28.138 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.138 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.138 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:46:28.138 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.310006+0000", 2023-09-23T20:46:28.138 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.6499999999999996e-06 2023-09-23T20:46:28.139 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.139 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.139 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:46:28.139 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.264766+0000", 2023-09-23T20:46:28.139 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.95475926600000005 2023-09-23T20:46:28.139 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.139 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.139 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:46:28.139 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.264896+0000", 2023-09-23T20:46:28.139 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000130514 2023-09-23T20:46:28.139 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.140 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.140 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 2,5", 2023-09-23T20:46:28.140 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.265017+0000", 2023-09-23T20:46:28.140 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000120581 2023-09-23T20:46:28.140 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.140 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.140 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:46:28.140 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.266595+0000", 2023-09-23T20:46:28.140 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.001578217 2023-09-23T20:46:28.140 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.140 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.141 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:46:28.141 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.267280+0000", 2023-09-23T20:46:28.141 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00068537800000000005 2023-09-23T20:46:28.141 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.141 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.141 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:46:28.141 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.272466+0000", 2023-09-23T20:46:28.141 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0051855180000000001 2023-09-23T20:46:28.141 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.141 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.142 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:46:28.142 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.272522+0000", 2023-09-23T20:46:28.142 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.5914e-05 2023-09-23T20:46:28.142 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.142 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.143 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:46:28.143 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.272732+0000", 2023-09-23T20:46:28.143 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00021019499999999999 2023-09-23T20:46:28.143 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:28.143 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:46:28.143 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:28.144 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.144 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.144 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:106123 8.2 8:435cd3c6:::benchmark_data_smithi018_341490_object16800:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e416)", 2023-09-23T20:46:28.144 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:38:38.326800+0000", 2023-09-23T20:46:28.144 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 469.72628801500002, 2023-09-23T20:46:28.144 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.95465794400000004, 2023-09-23T20:46:28.145 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:46:28.145 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:46:28.145 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:46:28.145 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:46:28.145 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:46:28.145 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 106123 2023-09-23T20:46:28.146 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.146 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:46:28.146 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.146 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:46:28.146 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.326800+0000", 2023-09-23T20:46:28.147 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:28.147 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.147 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.147 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:46:28.147 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.326800+0000", 2023-09-23T20:46:28.147 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:28.148 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.148 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.148 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:46:28.148 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.326799+0000", 2023-09-23T20:46:28.148 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:46:28.148 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.148 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.148 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:46:28.149 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.326805+0000", 2023-09-23T20:46:28.149 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.6259999999999997e-06 2023-09-23T20:46:28.149 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.149 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.149 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:46:28.149 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.326820+0000", 2023-09-23T20:46:28.149 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.4508000000000001e-05 2023-09-23T20:46:28.149 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.150 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.150 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:46:28.150 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.326826+0000", 2023-09-23T20:46:28.150 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.1709999999999999e-06 2023-09-23T20:46:28.150 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.150 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.150 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:46:28.151 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.326899+0000", 2023-09-23T20:46:28.151 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.2838999999999998e-05 2023-09-23T20:46:28.151 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.151 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.151 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:46:28.151 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.326913+0000", 2023-09-23T20:46:28.151 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.4022999999999999e-05 2023-09-23T20:46:28.152 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.152 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.152 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:46:28.152 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.270145+0000", 2023-09-23T20:46:28.152 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.94323180900000003 2023-09-23T20:46:28.152 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.152 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.152 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:46:28.152 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.270297+0000", 2023-09-23T20:46:28.152 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00015283500000000001 2023-09-23T20:46:28.153 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.153 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.153 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:46:28.153 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.270461+0000", 2023-09-23T20:46:28.153 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00016353799999999999 2023-09-23T20:46:28.153 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.153 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.153 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:46:28.153 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.273062+0000", 2023-09-23T20:46:28.153 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0026012800000000001 2023-09-23T20:46:28.153 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.154 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.154 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:46:28.154 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.276398+0000", 2023-09-23T20:46:28.154 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.003335321 2023-09-23T20:46:28.154 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.154 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.154 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:46:28.154 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.281194+0000", 2023-09-23T20:46:28.154 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0047966939999999998 2023-09-23T20:46:28.154 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.154 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.155 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:46:28.155 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.281226+0000", 2023-09-23T20:46:28.155 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.1965000000000003e-05 2023-09-23T20:46:28.155 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.155 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.155 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:46:28.155 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.281458+0000", 2023-09-23T20:46:28.155 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000231876 2023-09-23T20:46:28.155 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:28.156 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:46:28.156 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:28.156 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.156 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.156 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:106135 8.14 8:2e4b5b89:::benchmark_data_smithi018_341490_object16812:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e416)", 2023-09-23T20:46:28.156 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:38:38.358297+0000", 2023-09-23T20:46:28.156 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 469.69479084800003, 2023-09-23T20:46:28.156 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.91161518500000005, 2023-09-23T20:46:28.157 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:46:28.157 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:46:28.157 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:46:28.157 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:46:28.157 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:46:28.157 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 106135 2023-09-23T20:46:28.157 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.157 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:46:28.158 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.158 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:46:28.158 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.358297+0000", 2023-09-23T20:46:28.158 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:28.158 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.158 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.158 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:46:28.159 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.358297+0000", 2023-09-23T20:46:28.159 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:28.159 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.159 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.159 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:46:28.159 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.358294+0000", 2023-09-23T20:46:28.159 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999971 2023-09-23T20:46:28.160 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.160 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.160 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:46:28.160 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.358302+0000", 2023-09-23T20:46:28.160 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.0039999999999999e-06 2023-09-23T20:46:28.160 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.160 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.160 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:46:28.160 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.358329+0000", 2023-09-23T20:46:28.160 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.6804999999999998e-05 2023-09-23T20:46:28.160 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.161 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.161 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:46:28.161 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.358337+0000", 2023-09-23T20:46:28.161 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.276e-06 2023-09-23T20:46:28.161 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.161 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.162 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:46:28.162 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.358408+0000", 2023-09-23T20:46:28.162 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.0525000000000007e-05 2023-09-23T20:46:28.162 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.162 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.162 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:46:28.163 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.358419+0000", 2023-09-23T20:46:28.163 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.083e-05 2023-09-23T20:46:28.163 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.163 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.163 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:46:28.163 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.260630+0000", 2023-09-23T20:46:28.163 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.90221140300000002 2023-09-23T20:46:28.164 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.164 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.164 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:46:28.164 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.260747+0000", 2023-09-23T20:46:28.164 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000116555 2023-09-23T20:46:28.164 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.164 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.165 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 2,5", 2023-09-23T20:46:28.165 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.260886+0000", 2023-09-23T20:46:28.165 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00013910600000000001 2023-09-23T20:46:28.165 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.165 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.165 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:46:28.166 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.263408+0000", 2023-09-23T20:46:28.166 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0025218670000000001 2023-09-23T20:46:28.166 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.166 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.166 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:46:28.167 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.265119+0000", 2023-09-23T20:46:28.167 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.001711661 2023-09-23T20:46:28.167 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.167 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.167 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:46:28.167 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.269557+0000", 2023-09-23T20:46:28.167 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0044376390000000002 2023-09-23T20:46:28.168 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.168 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.168 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:46:28.168 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.269618+0000", 2023-09-23T20:46:28.168 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.0798999999999998e-05 2023-09-23T20:46:28.168 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.168 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.168 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:46:28.168 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:39.269913+0000", 2023-09-23T20:46:28.168 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00029461999999999999 2023-09-23T20:46:28.169 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:28.169 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:46:28.169 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:28.169 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.169 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.169 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_repop(client.36450.0:131373 9.c e478/455 9:3bcbfe7e:::benchmark_data_smithi018_381516_object17539:head v 478'8232, mlcod=478'8230)", 2023-09-23T20:46:28.169 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:43:03.468208+0000", 2023-09-23T20:46:28.169 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 204.58487970900001, 2023-09-23T20:46:28.169 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.054668136999999999, 2023-09-23T20:46:28.169 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:46:28.169 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:46:28.170 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:46:28.170 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.170 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:46:28.170 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:03.468208+0000", 2023-09-23T20:46:28.170 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:28.170 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.170 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.170 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:46:28.170 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:03.468208+0000", 2023-09-23T20:46:28.170 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:28.170 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.171 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.171 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:46:28.171 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:03.468208+0000", 2023-09-23T20:46:28.171 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:46:28.171 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.171 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.171 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:46:28.171 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:03.468214+0000", 2023-09-23T20:46:28.171 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.2620000000000004e-06 2023-09-23T20:46:28.172 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.172 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.172 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:46:28.172 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:03.468224+0000", 2023-09-23T20:46:28.172 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.7270000000000002e-06 2023-09-23T20:46:28.172 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.172 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.172 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:46:28.173 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:03.468230+0000", 2023-09-23T20:46:28.173 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.4989999999999999e-06 2023-09-23T20:46:28.173 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.173 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.173 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:46:28.173 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:03.471230+0000", 2023-09-23T20:46:28.173 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0029999390000000001 2023-09-23T20:46:28.173 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.173 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.173 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:46:28.173 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:03.471267+0000", 2023-09-23T20:46:28.173 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.6650000000000003e-05 2023-09-23T20:46:28.174 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.174 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.174 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:46:28.174 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:03.522841+0000", 2023-09-23T20:46:28.174 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.051573594 2023-09-23T20:46:28.174 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.174 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.174 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:46:28.174 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:03.522877+0000", 2023-09-23T20:46:28.174 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.6053000000000002e-05 2023-09-23T20:46:28.175 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:28.175 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:46:28.175 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:28.175 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.175 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.175 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_repop(client.36450.0:131376 9.d e478/472 9:bb4a56ee:::benchmark_data_smithi018_381516_object17542:head v 478'8296, mlcod=478'8294)", 2023-09-23T20:46:28.175 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:43:03.469304+0000", 2023-09-23T20:46:28.175 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 204.58378452900001, 2023-09-23T20:46:28.176 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.056257667999999997, 2023-09-23T20:46:28.176 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:46:28.176 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:46:28.176 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:46:28.176 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.176 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:46:28.176 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:03.469304+0000", 2023-09-23T20:46:28.176 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:28.177 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.177 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.177 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:46:28.177 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:03.469304+0000", 2023-09-23T20:46:28.177 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:28.177 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.177 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.177 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:46:28.177 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:03.469303+0000", 2023-09-23T20:46:28.178 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:46:28.178 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.178 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.178 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:46:28.178 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:03.469309+0000", 2023-09-23T20:46:28.178 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.3240000000000002e-06 2023-09-23T20:46:28.178 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.178 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.179 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:46:28.179 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:03.469332+0000", 2023-09-23T20:46:28.179 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.3458000000000001e-05 2023-09-23T20:46:28.179 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.179 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.179 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:46:28.179 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:03.469338+0000", 2023-09-23T20:46:28.180 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.2940000000000002e-06 2023-09-23T20:46:28.180 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.180 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.180 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:46:28.180 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:03.469641+0000", 2023-09-23T20:46:28.180 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000302863 2023-09-23T20:46:28.180 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.180 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.181 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:46:28.181 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:03.469658+0000", 2023-09-23T20:46:28.181 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.6481999999999999e-05 2023-09-23T20:46:28.181 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.181 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.181 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:46:28.181 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:03.525530+0000", 2023-09-23T20:46:28.181 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.055872217000000002 2023-09-23T20:46:28.181 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.181 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.182 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:46:28.182 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:03.525561+0000", 2023-09-23T20:46:28.182 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.1401e-05 2023-09-23T20:46:28.182 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:28.182 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:46:28.182 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:28.182 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.182 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.182 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_repop(client.36450.0:131378 9.d e478/472 9:bdf5ae4f:::benchmark_data_smithi018_381516_object17544:head v 478'8297, mlcod=478'8294)", 2023-09-23T20:46:28.182 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:43:03.472694+0000", 2023-09-23T20:46:28.182 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 204.58039373700001, 2023-09-23T20:46:28.183 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.055612288000000003, 2023-09-23T20:46:28.183 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:46:28.183 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:46:28.183 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:46:28.183 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.183 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:46:28.183 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:03.472694+0000", 2023-09-23T20:46:28.183 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:28.183 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.183 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.183 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:46:28.184 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:03.472694+0000", 2023-09-23T20:46:28.184 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:28.184 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.184 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.184 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:46:28.184 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:03.472694+0000", 2023-09-23T20:46:28.184 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:46:28.184 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.184 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.184 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:46:28.184 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:03.472700+0000", 2023-09-23T20:46:28.184 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.4790000000000001e-06 2023-09-23T20:46:28.185 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.185 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.185 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:46:28.185 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:03.472710+0000", 2023-09-23T20:46:28.185 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.7249999999999992e-06 2023-09-23T20:46:28.185 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.185 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.185 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:46:28.185 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:03.472716+0000", 2023-09-23T20:46:28.185 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.4639999999999997e-06 2023-09-23T20:46:28.185 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.185 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.186 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:46:28.186 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:03.472784+0000", 2023-09-23T20:46:28.186 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.8369999999999998e-05 2023-09-23T20:46:28.186 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.186 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.186 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:46:28.186 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:03.472803+0000", 2023-09-23T20:46:28.186 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.8640000000000001e-05 2023-09-23T20:46:28.186 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.186 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.186 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:46:28.187 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:03.528280+0000", 2023-09-23T20:46:28.187 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.055476909999999997 2023-09-23T20:46:28.187 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.187 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.187 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:46:28.187 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:03.528307+0000", 2023-09-23T20:46:28.187 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.7211000000000001e-05 2023-09-23T20:46:28.187 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:28.187 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:46:28.187 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:28.187 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.187 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.188 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:39257 10.1 10:8f38d3f9:::benchmark_data_smithi018_422431_object39256:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e537)", 2023-09-23T20:46:28.188 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:46:03.053378+0000", 2023-09-23T20:46:28.188 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 24.999710400000001, 2023-09-23T20:46:28.188 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.058852584999999999, 2023-09-23T20:46:28.188 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:46:28.188 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:46:28.188 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:46:28.188 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:46:28.188 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:46:28.188 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 39257 2023-09-23T20:46:28.188 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.188 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:46:28.189 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.189 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:46:28.189 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:03.053378+0000", 2023-09-23T20:46:28.189 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:28.189 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.189 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.189 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:46:28.189 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:03.053378+0000", 2023-09-23T20:46:28.189 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:28.189 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.189 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.190 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:46:28.190 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:03.053377+0000", 2023-09-23T20:46:28.190 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:46:28.190 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.190 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.190 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:46:28.190 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:03.053413+0000", 2023-09-23T20:46:28.190 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.6001999999999999e-05 2023-09-23T20:46:28.191 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.191 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.191 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:46:28.191 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:03.053423+0000", 2023-09-23T20:46:28.191 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0738e-05 2023-09-23T20:46:28.191 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.191 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.191 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:46:28.192 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:03.053429+0000", 2023-09-23T20:46:28.192 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.6189999999999999e-06 2023-09-23T20:46:28.192 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.192 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.192 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:46:28.193 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:03.053501+0000", 2023-09-23T20:46:28.193 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.1629000000000001e-05 2023-09-23T20:46:28.193 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.193 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.193 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:46:28.193 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:03.053608+0000", 2023-09-23T20:46:28.194 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000107731 2023-09-23T20:46:28.194 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.194 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.194 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,7", 2023-09-23T20:46:28.194 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:03.053738+0000", 2023-09-23T20:46:28.194 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000129455 2023-09-23T20:46:28.194 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.195 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.195 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:46:28.195 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:03.055466+0000", 2023-09-23T20:46:28.195 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00172828 2023-09-23T20:46:28.195 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.195 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.195 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:46:28.196 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:03.055672+0000", 2023-09-23T20:46:28.196 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000205632 2023-09-23T20:46:28.196 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.196 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.196 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:46:28.196 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:03.112157+0000", 2023-09-23T20:46:28.196 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.056485636999999998 2023-09-23T20:46:28.197 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.197 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.197 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:46:28.197 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:03.112194+0000", 2023-09-23T20:46:28.197 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.6650999999999999e-05 2023-09-23T20:46:28.197 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.197 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.197 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:46:28.197 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:03.112230+0000", 2023-09-23T20:46:28.197 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.6411000000000001e-05 2023-09-23T20:46:28.197 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:28.198 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:46:28.198 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:28.198 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.198 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.198 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:67870 10.e 10:7aa95661:::benchmark_data_smithi018_422431_object67869:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e540)", 2023-09-23T20:46:28.198 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:46:24.953879+0000", 2023-09-23T20:46:28.198 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 3.0992093459999999, 2023-09-23T20:46:28.198 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.063370259999999998, 2023-09-23T20:46:28.198 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:46:28.198 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:46:28.199 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:46:28.199 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:46:28.199 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:46:28.199 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 67870 2023-09-23T20:46:28.199 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.199 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:46:28.199 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.199 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:46:28.199 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:24.953879+0000", 2023-09-23T20:46:28.199 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:28.199 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.199 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.200 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:46:28.200 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:24.953879+0000", 2023-09-23T20:46:28.200 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:28.200 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.200 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.200 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:46:28.200 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:24.953877+0000", 2023-09-23T20:46:28.200 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999981 2023-09-23T20:46:28.200 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.200 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.200 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:46:28.200 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:24.953934+0000", 2023-09-23T20:46:28.201 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.6767e-05 2023-09-23T20:46:28.201 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.201 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.201 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:46:28.201 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:24.953943+0000", 2023-09-23T20:46:28.201 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.1279999999999997e-06 2023-09-23T20:46:28.201 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.201 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.201 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:46:28.201 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:24.953948+0000", 2023-09-23T20:46:28.201 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.04e-06 2023-09-23T20:46:28.201 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.202 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.202 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:46:28.202 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:24.953993+0000", 2023-09-23T20:46:28.202 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.4841e-05 2023-09-23T20:46:28.202 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.202 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.202 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:46:28.202 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:24.954117+0000", 2023-09-23T20:46:28.203 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000124251 2023-09-23T20:46:28.203 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.203 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.203 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 2,6", 2023-09-23T20:46:28.203 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:24.954239+0000", 2023-09-23T20:46:28.203 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00012233400000000001 2023-09-23T20:46:28.203 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.204 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.204 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:46:28.204 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:24.958999+0000", 2023-09-23T20:46:28.204 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0047599649999999997 2023-09-23T20:46:28.204 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.204 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.204 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:46:28.204 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:24.959274+0000", 2023-09-23T20:46:28.205 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00027472599999999998 2023-09-23T20:46:28.205 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.205 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.205 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:46:28.205 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:25.017149+0000", 2023-09-23T20:46:28.205 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.057875109000000001 2023-09-23T20:46:28.205 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.206 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.206 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:46:28.206 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:25.017206+0000", 2023-09-23T20:46:28.206 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.6402e-05 2023-09-23T20:46:28.206 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.206 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.207 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:46:28.207 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:25.017249+0000", 2023-09-23T20:46:28.207 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.3498999999999998e-05 2023-09-23T20:46:28.207 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:28.207 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:46:28.207 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:28.208 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.208 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.208 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:67874 10.e 10:71847701:::benchmark_data_smithi018_422431_object67873:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e540)", 2023-09-23T20:46:28.208 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:46:24.961025+0000", 2023-09-23T20:46:28.208 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 3.0920633080000002, 2023-09-23T20:46:28.208 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.056572113, 2023-09-23T20:46:28.209 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:46:28.209 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:46:28.209 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:46:28.209 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:46:28.209 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:46:28.209 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 67874 2023-09-23T20:46:28.209 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.210 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:46:28.210 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.210 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:46:28.210 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:24.961025+0000", 2023-09-23T20:46:28.210 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:28.210 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.210 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.211 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:46:28.211 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:24.961025+0000", 2023-09-23T20:46:28.211 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:28.211 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.211 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.211 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:46:28.212 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:24.961024+0000", 2023-09-23T20:46:28.212 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:46:28.212 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.212 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.212 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:46:28.212 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:24.961074+0000", 2023-09-23T20:46:28.213 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.9904000000000002e-05 2023-09-23T20:46:28.213 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.213 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.213 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:46:28.213 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:24.961091+0000", 2023-09-23T20:46:28.213 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.6730000000000001e-05 2023-09-23T20:46:28.213 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.213 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.214 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:46:28.214 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:24.961096+0000", 2023-09-23T20:46:28.214 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.9740000000000001e-06 2023-09-23T20:46:28.214 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.214 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.214 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:46:28.214 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:24.961142+0000", 2023-09-23T20:46:28.214 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.6136e-05 2023-09-23T20:46:28.214 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.214 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.214 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:46:28.215 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:24.961253+0000", 2023-09-23T20:46:28.215 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000111754 2023-09-23T20:46:28.215 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.215 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.215 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 2,6", 2023-09-23T20:46:28.215 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:24.961356+0000", 2023-09-23T20:46:28.215 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000102871 2023-09-23T20:46:28.215 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.215 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.215 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:46:28.215 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:25.017077+0000", 2023-09-23T20:46:28.215 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.055721184999999999 2023-09-23T20:46:28.216 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.216 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.216 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:46:28.216 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:25.017116+0000", 2023-09-23T20:46:28.216 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.8816000000000002e-05 2023-09-23T20:46:28.216 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.216 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.216 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:46:28.216 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:25.017519+0000", 2023-09-23T20:46:28.216 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00040246800000000001 2023-09-23T20:46:28.216 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.217 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.217 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:46:28.217 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:25.017564+0000", 2023-09-23T20:46:28.217 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.5067000000000001e-05 2023-09-23T20:46:28.217 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.217 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.217 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:46:28.217 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:25.017597+0000", 2023-09-23T20:46:28.217 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.3170000000000003e-05 2023-09-23T20:46:28.217 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:28.217 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:46:28.217 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:28.218 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.218 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.218 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:67878 10.e 10:7816dfee:::benchmark_data_smithi018_422431_object67877:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e540)", 2023-09-23T20:46:28.218 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:46:24.961116+0000", 2023-09-23T20:46:28.218 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 3.0919725319999998, 2023-09-23T20:46:28.218 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.061098663999999997, 2023-09-23T20:46:28.218 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:46:28.218 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:46:28.218 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:46:28.218 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:46:28.218 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:46:28.219 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 67878 2023-09-23T20:46:28.219 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.219 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:46:28.219 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.219 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:46:28.219 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:24.961116+0000", 2023-09-23T20:46:28.219 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:28.219 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.219 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.219 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:46:28.219 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:24.961116+0000", 2023-09-23T20:46:28.219 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:28.220 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.220 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.220 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:46:28.220 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:24.961115+0000", 2023-09-23T20:46:28.220 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:46:28.220 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.220 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.220 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:46:28.220 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:24.961137+0000", 2023-09-23T20:46:28.220 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.1891000000000001e-05 2023-09-23T20:46:28.220 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.221 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.221 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:46:28.221 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:24.961144+0000", 2023-09-23T20:46:28.221 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.8549999999999996e-06 2023-09-23T20:46:28.221 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.221 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.221 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:46:28.221 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:24.961148+0000", 2023-09-23T20:46:28.221 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.2409999999999997e-06 2023-09-23T20:46:28.221 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.221 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.221 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:46:28.222 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:25.014191+0000", 2023-09-23T20:46:28.222 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.053042899999999997 2023-09-23T20:46:28.222 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.222 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.222 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:46:28.222 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:25.014354+0000", 2023-09-23T20:46:28.222 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00016308699999999999 2023-09-23T20:46:28.222 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.222 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.222 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 2,6", 2023-09-23T20:46:28.222 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:25.014641+0000", 2023-09-23T20:46:28.222 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00028713600000000001 2023-09-23T20:46:28.223 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.223 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.223 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:46:28.223 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:25.017628+0000", 2023-09-23T20:46:28.223 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0029868730000000001 2023-09-23T20:46:28.223 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.223 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.223 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:46:28.223 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:25.020527+0000", 2023-09-23T20:46:28.223 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0028984480000000001 2023-09-23T20:46:28.223 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.223 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.224 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:46:28.224 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:25.022125+0000", 2023-09-23T20:46:28.224 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0015986360000000001 2023-09-23T20:46:28.224 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.224 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.224 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:46:28.224 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:25.022171+0000", 2023-09-23T20:46:28.224 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.5232000000000003e-05 2023-09-23T20:46:28.224 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.224 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.224 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:46:28.224 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:25.022214+0000", 2023-09-23T20:46:28.225 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.3720000000000002e-05 2023-09-23T20:46:28.225 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:28.225 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:46:28.225 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:28.225 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.225 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.225 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_repop(client.46625.0:67877 10.c e540/531 10:36e17261:::benchmark_data_smithi018_422431_object67876:head v 540'4289, mlcod=540'4286)", 2023-09-23T20:46:28.225 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:46:24.961578+0000", 2023-09-23T20:46:28.225 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 3.091509989, 2023-09-23T20:46:28.225 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.059771141999999999, 2023-09-23T20:46:28.225 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:46:28.225 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:46:28.226 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:46:28.226 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.226 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:46:28.226 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:24.961578+0000", 2023-09-23T20:46:28.226 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:28.226 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.226 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.226 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:46:28.226 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:24.961578+0000", 2023-09-23T20:46:28.226 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:28.226 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.227 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.227 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:46:28.227 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:24.961578+0000", 2023-09-23T20:46:28.227 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:46:28.227 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.227 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.227 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:46:28.227 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:24.961599+0000", 2023-09-23T20:46:28.227 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.1515e-05 2023-09-23T20:46:28.227 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.227 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.227 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:46:28.228 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:24.961613+0000", 2023-09-23T20:46:28.228 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.4076999999999999e-05 2023-09-23T20:46:28.228 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.228 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.228 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:46:28.228 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:24.961618+0000", 2023-09-23T20:46:28.228 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.5109999999999997e-06 2023-09-23T20:46:28.228 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.228 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.228 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:46:28.228 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:24.961659+0000", 2023-09-23T20:46:28.228 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.0911000000000002e-05 2023-09-23T20:46:28.229 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.229 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.229 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:46:28.229 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:24.961676+0000", 2023-09-23T20:46:28.229 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.7507e-05 2023-09-23T20:46:28.229 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.229 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.229 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:46:28.229 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:25.021291+0000", 2023-09-23T20:46:28.229 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.059615159000000001 2023-09-23T20:46:28.229 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.229 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.230 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:46:28.230 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:25.021349+0000", 2023-09-23T20:46:28.230 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.8082999999999999e-05 2023-09-23T20:46:28.230 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:28.230 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:46:28.230 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:28.230 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.230 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.230 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:67883 10.e 10:70271c1a:::benchmark_data_smithi018_422431_object67882:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e540)", 2023-09-23T20:46:28.230 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:46:24.969432+0000", 2023-09-23T20:46:28.230 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 3.083655995, 2023-09-23T20:46:28.231 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.056275044000000003, 2023-09-23T20:46:28.231 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:46:28.231 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:46:28.231 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:46:28.231 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:46:28.231 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:46:28.231 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 67883 2023-09-23T20:46:28.231 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.231 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:46:28.231 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.231 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:46:28.232 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:24.969432+0000", 2023-09-23T20:46:28.232 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:28.232 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.232 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.232 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:46:28.232 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:24.969432+0000", 2023-09-23T20:46:28.232 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:28.232 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.232 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.232 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:46:28.232 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:24.969431+0000", 2023-09-23T20:46:28.232 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:46:28.232 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.233 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.233 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:46:28.233 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:24.969493+0000", 2023-09-23T20:46:28.233 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.1727e-05 2023-09-23T20:46:28.233 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.233 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.233 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:46:28.233 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:25.014104+0000", 2023-09-23T20:46:28.233 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.04461089 2023-09-23T20:46:28.233 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.233 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.233 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:46:28.234 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:25.014117+0000", 2023-09-23T20:46:28.234 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.3644e-05 2023-09-23T20:46:28.234 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.234 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.234 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:46:28.234 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:25.017646+0000", 2023-09-23T20:46:28.234 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0035291820000000001 2023-09-23T20:46:28.234 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.234 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.234 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:46:28.234 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:25.017778+0000", 2023-09-23T20:46:28.235 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00013171799999999999 2023-09-23T20:46:28.235 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.235 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.235 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 2,6", 2023-09-23T20:46:28.235 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:25.017924+0000", 2023-09-23T20:46:28.235 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00014606299999999999 2023-09-23T20:46:28.235 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.235 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.235 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:46:28.235 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:25.020299+0000", 2023-09-23T20:46:28.235 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.002374672 2023-09-23T20:46:28.235 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.236 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.236 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:46:28.236 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:25.021820+0000", 2023-09-23T20:46:28.236 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0015215230000000001 2023-09-23T20:46:28.236 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.236 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.236 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:46:28.236 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:25.025610+0000", 2023-09-23T20:46:28.236 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0037894909999999999 2023-09-23T20:46:28.236 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.236 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.236 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:46:28.237 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:25.025665+0000", 2023-09-23T20:46:28.237 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.4698999999999999e-05 2023-09-23T20:46:28.237 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:28.237 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:28.237 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:46:28.237 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:25.025707+0000", 2023-09-23T20:46:28.237 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.2698999999999999e-05 2023-09-23T20:46:28.237 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:28.237 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:46:28.237 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:28.237 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:28.237 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:46:28.238 INFO:teuthology.orchestra.run.smithi149.stdout:} 2023-09-23T20:46:28.238 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 7, 3, 2, 4, 6, 5] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T20:46:28.238 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:46:28.238 INFO:tasks.thrashosds.thrasher:inject_pause on osd.1 2023-09-23T20:46:28.238 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T20:46:28.238 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T20:46:28.238 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 config set filestore_inject_stall 3 2023-09-23T20:46:28.239 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:27 smithi018 ceph-mon[184144]: pgmap v1367: 119 pgs: 119 active+clean; 4.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 75 MiB/s wr, 1.19k op/s 2023-09-23T20:46:28.240 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:27 smithi018 ceph-mon[179647]: pgmap v1367: 119 pgs: 119 active+clean; 4.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 75 MiB/s wr, 1.19k op/s 2023-09-23T20:46:28.268 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:46:28.282 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:28.282 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:46:28.383 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:46:28.486 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:28.562 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:46:28.574 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:28.574 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:46:28.675 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:46:28.754 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:28.902 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T20:46:28.999 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 44 16 72055 72039 102.279 54.9375 0.0162656 0.00977183 2023-09-23T20:46:29.031 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:46:29.035 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:29.036 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:46:29.137 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:46:29.185 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T20:46:29.194 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:29.209 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:46:29.262 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:46:29.265 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:46:29.270 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:29.271 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:46:29.297 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:46:29.338 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:46:29.350 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:46:29.371 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:46:29.390 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:46:29.426 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:29.428 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:46:29.471 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:46:29.497 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:46:29.507 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:46:29.508 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:29.508 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:46:29.518 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:46:29.519 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:46:29.519 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:46:29.519 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:46:29.519 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:46:29.519 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:46:29.519 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:46:29.519 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:46:29.609 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:46:29.679 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:29 smithi149 ceph-mon[170394]: pgmap v1368: 119 pgs: 119 active+clean; 4.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s 2023-09-23T20:46:29.679 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:29 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/886723132' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:46:29.679 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:29 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:46:29.694 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:29.819 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:46:29.832 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:29.832 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:46:29.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:29 smithi018 ceph-mon[184144]: pgmap v1368: 119 pgs: 119 active+clean; 4.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s 2023-09-23T20:46:29.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:29 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/886723132' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:46:29.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:29 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:46:29.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:29 smithi018 ceph-mon[179647]: pgmap v1368: 119 pgs: 119 active+clean; 4.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s 2023-09-23T20:46:29.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:29 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/886723132' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:46:29.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:29 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:46:29.933 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:46:29.999 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 45 16 72987 72971 101.3 58.25 0.0310516 0.00986543 2023-09-23T20:46:30.034 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:30.101 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:46:30.106 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:30.106 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:46:30.153 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:46:30.207 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:46:30.286 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:30.452 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:46:30.459 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:30.459 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:46:30.549 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:46:30.560 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:46:30.614 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:30.688 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:46:30.695 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:30.695 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:46:30.796 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:46:30.881 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:30.990 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:46:31.002 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:31.003 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:46:31.003 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 46 16 73983 73967 100.444 62.25 0.0123487 0.00995025 2023-09-23T20:46:31.104 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:46:31.143 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2023-09-23T20:46:31.143 INFO:teuthology.orchestra.run.smithi018.stdout: "success": "" 2023-09-23T20:46:31.143 INFO:teuthology.orchestra.run.smithi018.stdout:} 2023-09-23T20:46:31.164 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:31.242 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:46:31.254 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:31.254 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:46:31.355 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:46:31.432 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:31.525 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:46:31.532 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:31.532 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:46:31.633 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:46:31.686 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:31 smithi149 ceph-mon[170394]: pgmap v1369: 119 pgs: 119 active+clean; 4.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 56 MiB/s wr, 890 op/s 2023-09-23T20:46:31.696 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:32.003 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 47 16 74997 74981 99.6553 63.375 0.00334088 0.0100289 2023-09-23T20:46:32.111 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:31 smithi018 ceph-mon[179647]: pgmap v1369: 119 pgs: 119 active+clean; 4.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 56 MiB/s wr, 890 op/s 2023-09-23T20:46:32.112 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:31 smithi018 ceph-mon[184144]: pgmap v1369: 119 pgs: 119 active+clean; 4.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 56 MiB/s wr, 890 op/s 2023-09-23T20:46:32.122 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:46:32.777 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:46:32.781 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:32.781 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:46:32.882 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:46:32.961 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:32.986 INFO:teuthology.orchestra.run.smithi149.stdout:ERROR: (22) Invalid argument 2023-09-23T20:46:32.986 INFO:teuthology.orchestra.run.smithi149.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:46:33.003 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 48 16 75928 75912 98.7916 58.1875 0.0123447 0.0101149 2023-09-23T20:46:33.037 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:46:33.049 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:33.049 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:46:33.150 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:46:33.192 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:46:33.218 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:33.479 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:46:33.493 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:33.494 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:46:33.510 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T20:46:33.595 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:46:33.696 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:33.715 INFO:teuthology.orchestra.run.smithi018.stderr:nodeep-scrub is unset 2023-09-23T20:46:33.757 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:33 smithi018 ceph-mon[179647]: pgmap v1370: 119 pgs: 119 active+clean; 4.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-09-23T20:46:33.757 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:33 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2656388204' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:46:33.757 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:33 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:46:33.758 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:33 smithi018 ceph-mon[184144]: pgmap v1370: 119 pgs: 119 active+clean; 4.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-09-23T20:46:33.759 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:33 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2656388204' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:46:33.759 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:33 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:46:33.774 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:46:33.782 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:33.783 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:46:33.884 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:46:33.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:33 smithi149 ceph-mon[170394]: pgmap v1370: 119 pgs: 119 active+clean; 4.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-09-23T20:46:33.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:33 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2656388204' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:46:33.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:33 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:46:33.980 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:34.003 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 49 16 76862 76846 97.9668 58.375 0.0195986 0.0102025 2023-09-23T20:46:34.064 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:46:34.077 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:34.078 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:46:34.179 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:46:34.237 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:34.312 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:46:34.325 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:34.326 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:46:34.328 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:46:34.427 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:46:34.622 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:34.757 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:46:34.764 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:34.765 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:46:34.865 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:46:34.881 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:34 smithi018 ceph-mon[179647]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:46:34.881 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:34 smithi018 ceph-mon[179647]: Cluster is now healthy 2023-09-23T20:46:34.881 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:34 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:46:34.881 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:34 smithi018 ceph-mon[179647]: osdmap e542: 8 total, 8 up, 8 in 2023-09-23T20:46:34.881 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:34 smithi018 ceph-mon[179647]: failed to encode map e542 with expected crc 2023-09-23T20:46:34.882 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:34 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:46:34.882 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:34 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:46:34.885 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:34 smithi018 ceph-mon[184144]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:46:34.885 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:34 smithi018 ceph-mon[184144]: Cluster is now healthy 2023-09-23T20:46:34.885 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:34 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:46:34.885 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:34 smithi018 ceph-mon[184144]: osdmap e542: 8 total, 8 up, 8 in 2023-09-23T20:46:34.886 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:34 smithi018 ceph-mon[184144]: failed to encode map e542 with expected crc 2023-09-23T20:46:34.886 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:34 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:46:34.886 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:34 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:46:34.911 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:34.974 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:34 smithi149 ceph-mon[170394]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:46:34.974 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:34 smithi149 ceph-mon[170394]: Cluster is now healthy 2023-09-23T20:46:34.975 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:34 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:46:34.975 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:34 smithi149 ceph-mon[170394]: osdmap e542: 8 total, 8 up, 8 in 2023-09-23T20:46:34.975 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:34 smithi149 ceph-mon[170394]: failed to encode map e542 with expected crc 2023-09-23T20:46:34.975 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:34 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:46:34.975 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:34 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:46:35.003 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 50 16 77816 77800 97.2002 59.625 0.00395805 0.0102827 2023-09-23T20:46:35.124 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:46:35.135 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:35.136 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:46:35.142 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:46:34 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:46:34] "GET /metrics HTTP/1.1" 200 36571 "" "Prometheus/2.43.0" 2023-09-23T20:46:35.236 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:46:35.285 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:46:35.321 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:46:35.322 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:35.358 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:46:35.403 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:46:35.413 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:46:35.415 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:35.416 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:46:35.454 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:46:35.502 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:46:35.517 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:46:35.544 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:46:35.581 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:46:35.584 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:35.587 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:46:35.587 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:46:35.587 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:46:35.587 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:46:35.587 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:46:35.587 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:46:35.587 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:46:35.587 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:46:35.704 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:35 smithi149 ceph-mon[170394]: pgmap v1372: 119 pgs: 119 active+clean; 4.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 59 MiB/s wr, 944 op/s 2023-09-23T20:46:35.704 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:35 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/562529854' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:46:35.704 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:35 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:46:35.704 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:35 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:46:35.848 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:46:35.851 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:35 smithi018 ceph-mon[184144]: pgmap v1372: 119 pgs: 119 active+clean; 4.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 59 MiB/s wr, 944 op/s 2023-09-23T20:46:35.851 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:35 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/562529854' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:46:35.851 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:35 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:46:35.851 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:35 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:46:35.852 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:35 smithi018 ceph-mon[179647]: pgmap v1372: 119 pgs: 119 active+clean; 4.5 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 59 MiB/s wr, 944 op/s 2023-09-23T20:46:35.852 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:35 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/562529854' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:46:35.852 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:35 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:46:35.852 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:35 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:46:35.852 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:35.853 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:46:35.954 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:46:36.004 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 51 16 78830 78814 96.5371 63.375 0.0109218 0.0103535 2023-09-23T20:46:36.014 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:36.074 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:46:36.080 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:36.080 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:46:36.181 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:46:36.252 INFO:teuthology.orchestra.run.smithi149.stdout:{ 2023-09-23T20:46:36.252 INFO:teuthology.orchestra.run.smithi149.stdout: "ops": [], 2023-09-23T20:46:36.252 INFO:teuthology.orchestra.run.smithi149.stdout: "complaint_time": 30, 2023-09-23T20:46:36.253 INFO:teuthology.orchestra.run.smithi149.stdout: "num_blocked_ops": 0 2023-09-23T20:46:36.253 INFO:teuthology.orchestra.run.smithi149.stdout:} 2023-09-23T20:46:36.253 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:36.527 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T20:46:36.640 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T20:46:36.655 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:46:36.666 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:36.666 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:46:36.767 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:46:36.823 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:36.853 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 7, 3, 2, 4, 6, 5] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T20:46:36.853 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:46:36.853 INFO:tasks.thrashosds.thrasher:Growing pool .mgr 2023-09-23T20:46:36.853 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:46:36.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:36 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:46:36.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:36 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:46:36.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:36 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:46:36.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:36 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:46:36.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:36 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:46:36.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:36 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:46:36.898 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:46:36.904 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:36.905 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:46:36.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:36 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:46:36.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:36 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:46:36.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:36 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:46:37.004 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 52 16 79821 79805 95.8719 61.9375 0.00492143 0.0104251 2023-09-23T20:46:37.006 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:46:37.051 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:37.109 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:46:37.116 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:37.116 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:46:37.217 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:46:37.298 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:37.369 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:46:37.374 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:37.374 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:46:37.428 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:46:37.430 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:46:37.476 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:46:37.531 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:37.813 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:37 smithi149 ceph-mon[170394]: pgmap v1373: 119 pgs: 119 active+clean; 4.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-09-23T20:46:37.821 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:46:37.825 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:37.826 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:46:37.926 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:46:37.938 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:46:37.996 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:37 smithi018 ceph-mon[184144]: pgmap v1373: 119 pgs: 119 active+clean; 4.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-09-23T20:46:38.000 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:37 smithi018 ceph-mon[179647]: pgmap v1373: 119 pgs: 119 active+clean; 4.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-09-23T20:46:38.005 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 53 16 80802 80786 95.2173 61.3125 0.0080039 0.0104972 2023-09-23T20:46:38.008 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:38.017 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:46:38.984 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:46:38.991 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:38.992 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:46:39.008 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 54 16 81871 81855 94.687 66.8125 0.0236129 0.0105563 2023-09-23T20:46:39.092 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:46:39.162 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:39.255 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:46:39.258 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:39.258 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:46:39.359 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:46:39.384 INFO:teuthology.orchestra.run.smithi149.stdout:{ 2023-09-23T20:46:39.384 INFO:teuthology.orchestra.run.smithi149.stdout: "size": 20, 2023-09-23T20:46:39.384 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 600, 2023-09-23T20:46:39.384 INFO:teuthology.orchestra.run.smithi149.stdout: "ops": [ 2023-09-23T20:46:39.384 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.384 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:51043 8.0 8:0f697c15:::benchmark_data_smithi018_341490_object51042:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e374)", 2023-09-23T20:46:39.384 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:36:42.561436+0000", 2023-09-23T20:46:39.385 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 596.81881340699999, 2023-09-23T20:46:39.385 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.14033216600000001, 2023-09-23T20:46:39.385 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:46:39.385 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:46:39.385 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:46:39.385 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:46:39.385 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:46:39.385 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 51043 2023-09-23T20:46:39.385 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.385 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:46:39.385 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.385 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:46:39.386 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.561436+0000", 2023-09-23T20:46:39.386 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:39.386 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.386 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.386 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:46:39.386 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.561436+0000", 2023-09-23T20:46:39.386 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:39.386 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.386 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.386 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:46:39.386 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.561435+0000", 2023-09-23T20:46:39.386 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:46:39.387 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.387 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.387 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:46:39.387 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.561495+0000", 2023-09-23T20:46:39.387 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.0056999999999998e-05 2023-09-23T20:46:39.387 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.387 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.387 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:46:39.387 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.561507+0000", 2023-09-23T20:46:39.387 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1627999999999999e-05 2023-09-23T20:46:39.387 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.387 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.387 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:46:39.388 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.561513+0000", 2023-09-23T20:46:39.388 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.5969999999999997e-06 2023-09-23T20:46:39.388 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.388 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.388 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:46:39.388 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.561542+0000", 2023-09-23T20:46:39.388 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.8665000000000001e-05 2023-09-23T20:46:39.388 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.388 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.388 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:46:39.388 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.561627+0000", 2023-09-23T20:46:39.388 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.4596000000000001e-05 2023-09-23T20:46:39.389 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.389 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.389 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:46:39.389 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.561720+0000", 2023-09-23T20:46:39.389 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.3153999999999996e-05 2023-09-23T20:46:39.389 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.389 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.389 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:46:39.389 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.563303+0000", 2023-09-23T20:46:39.389 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0015831689999999999 2023-09-23T20:46:39.389 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.389 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.389 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:46:39.390 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.563545+0000", 2023-09-23T20:46:39.390 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00024228800000000001 2023-09-23T20:46:39.390 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.390 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.390 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:46:39.390 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.701697+0000", 2023-09-23T20:46:39.390 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.138151417 2023-09-23T20:46:39.390 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.390 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.390 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:46:39.390 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.701739+0000", 2023-09-23T20:46:39.390 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.2009999999999999e-05 2023-09-23T20:46:39.391 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.391 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.391 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:46:39.391 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:36:42.701768+0000", 2023-09-23T20:46:39.391 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.9332e-05 2023-09-23T20:46:39.391 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:39.391 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:46:39.391 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:39.391 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.391 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.391 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:29549 9.7 9:ea1cf7ea:::benchmark_data_smithi018_381516_object29548:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e459)", 2023-09-23T20:46:39.391 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:41:21.052150+0000", 2023-09-23T20:46:39.391 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 318.328098813, 2023-09-23T20:46:39.392 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.14682632800000001, 2023-09-23T20:46:39.392 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:46:39.392 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:46:39.392 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:46:39.392 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:46:39.392 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:46:39.392 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 29549 2023-09-23T20:46:39.392 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.392 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:46:39.392 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.392 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:46:39.392 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.052150+0000", 2023-09-23T20:46:39.392 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:39.393 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.393 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.393 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:46:39.393 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.052150+0000", 2023-09-23T20:46:39.393 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:39.393 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.393 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.393 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:46:39.393 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.052149+0000", 2023-09-23T20:46:39.393 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:46:39.393 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.393 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.394 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:46:39.394 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.052237+0000", 2023-09-23T20:46:39.394 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.7569000000000001e-05 2023-09-23T20:46:39.394 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.394 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.394 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:46:39.394 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.052246+0000", 2023-09-23T20:46:39.394 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.5289999999999997e-06 2023-09-23T20:46:39.394 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.394 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.394 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:46:39.394 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.052252+0000", 2023-09-23T20:46:39.395 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.5450000000000003e-06 2023-09-23T20:46:39.395 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.395 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.395 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:46:39.395 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.052294+0000", 2023-09-23T20:46:39.395 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.2228999999999998e-05 2023-09-23T20:46:39.395 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.395 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.395 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:46:39.395 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.052411+0000", 2023-09-23T20:46:39.395 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00011702899999999999 2023-09-23T20:46:39.395 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.395 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.396 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 3,7", 2023-09-23T20:46:39.396 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.052532+0000", 2023-09-23T20:46:39.396 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00012102799999999999 2023-09-23T20:46:39.396 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.396 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.396 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:46:39.396 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.055293+0000", 2023-09-23T20:46:39.396 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.002760424 2023-09-23T20:46:39.396 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.396 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.396 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:46:39.396 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.056788+0000", 2023-09-23T20:46:39.396 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00149522 2023-09-23T20:46:39.397 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.397 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.397 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:46:39.397 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.198887+0000", 2023-09-23T20:46:39.397 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.142099535 2023-09-23T20:46:39.397 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.397 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.397 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:46:39.397 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.198937+0000", 2023-09-23T20:46:39.397 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.9252999999999998e-05 2023-09-23T20:46:39.397 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.397 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.397 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:46:39.398 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.198977+0000", 2023-09-23T20:46:39.398 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.0151000000000002e-05 2023-09-23T20:46:39.398 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:39.398 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:46:39.398 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:39.398 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.398 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.398 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:29552 9.f 9:fef69f4f:::benchmark_data_smithi018_381516_object29551:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e459)", 2023-09-23T20:46:39.398 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:41:21.053398+0000", 2023-09-23T20:46:39.398 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 318.32685083299998, 2023-09-23T20:46:39.398 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.14557930599999999, 2023-09-23T20:46:39.398 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:46:39.399 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:46:39.399 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:46:39.399 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:46:39.399 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:46:39.399 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 29552 2023-09-23T20:46:39.399 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.399 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:46:39.399 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.399 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:46:39.399 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.053398+0000", 2023-09-23T20:46:39.399 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:39.399 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.399 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.400 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:46:39.400 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.053398+0000", 2023-09-23T20:46:39.400 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:39.400 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.400 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.400 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:46:39.400 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.053398+0000", 2023-09-23T20:46:39.400 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:46:39.400 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.400 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.400 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:46:39.400 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.053439+0000", 2023-09-23T20:46:39.401 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.1603000000000001e-05 2023-09-23T20:46:39.401 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.401 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.401 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:46:39.401 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.053453+0000", 2023-09-23T20:46:39.401 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.3842000000000001e-05 2023-09-23T20:46:39.401 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.401 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.401 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:46:39.401 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.053462+0000", 2023-09-23T20:46:39.401 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.9190000000000005e-06 2023-09-23T20:46:39.401 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.401 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.402 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:46:39.402 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.053503+0000", 2023-09-23T20:46:39.402 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.1069000000000003e-05 2023-09-23T20:46:39.402 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.402 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.402 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:46:39.402 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.053629+0000", 2023-09-23T20:46:39.402 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00012554299999999999 2023-09-23T20:46:39.402 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.402 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.402 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:46:39.402 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.053773+0000", 2023-09-23T20:46:39.403 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000144427 2023-09-23T20:46:39.403 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.403 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.403 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:46:39.403 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.056678+0000", 2023-09-23T20:46:39.403 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0029051200000000002 2023-09-23T20:46:39.403 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.403 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.403 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:46:39.403 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.060074+0000", 2023-09-23T20:46:39.403 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0033957610000000002 2023-09-23T20:46:39.403 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.403 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.404 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:46:39.404 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.198920+0000", 2023-09-23T20:46:39.404 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.13884589899999999 2023-09-23T20:46:39.404 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.404 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.404 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:46:39.404 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.198949+0000", 2023-09-23T20:46:39.404 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.9641e-05 2023-09-23T20:46:39.404 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.404 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.404 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:46:39.404 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.198978+0000", 2023-09-23T20:46:39.405 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.8285000000000001e-05 2023-09-23T20:46:39.405 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:39.405 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:46:39.405 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:39.405 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.405 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.405 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:29557 9.7 9:e1a9e246:::benchmark_data_smithi018_381516_object29556:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e459)", 2023-09-23T20:46:39.405 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:41:21.057117+0000", 2023-09-23T20:46:39.405 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 318.32313235999999, 2023-09-23T20:46:39.405 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.14586998200000001, 2023-09-23T20:46:39.405 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:46:39.405 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:46:39.405 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:46:39.406 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:46:39.406 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:46:39.406 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 29557 2023-09-23T20:46:39.406 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.406 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:46:39.406 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.406 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:46:39.406 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.057117+0000", 2023-09-23T20:46:39.406 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:39.406 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.406 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.406 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:46:39.407 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.057117+0000", 2023-09-23T20:46:39.407 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:39.407 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.407 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.407 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:46:39.407 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.057115+0000", 2023-09-23T20:46:39.407 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:46:39.407 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.407 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.407 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:46:39.407 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.057187+0000", 2023-09-23T20:46:39.407 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.1619999999999995e-05 2023-09-23T20:46:39.407 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.408 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.408 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:46:39.408 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.057200+0000", 2023-09-23T20:46:39.408 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.3261000000000001e-05 2023-09-23T20:46:39.408 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.408 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.408 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:46:39.408 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.057207+0000", 2023-09-23T20:46:39.408 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.6309999999999997e-06 2023-09-23T20:46:39.408 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.408 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.408 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:46:39.408 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.057249+0000", 2023-09-23T20:46:39.409 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.1993999999999998e-05 2023-09-23T20:46:39.409 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.409 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.409 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:46:39.409 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.057363+0000", 2023-09-23T20:46:39.409 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000113553 2023-09-23T20:46:39.409 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.409 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.409 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 3,7", 2023-09-23T20:46:39.409 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.057458+0000", 2023-09-23T20:46:39.409 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.5340000000000005e-05 2023-09-23T20:46:39.409 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.409 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.410 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:46:39.410 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.058584+0000", 2023-09-23T20:46:39.410 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0011261489999999999 2023-09-23T20:46:39.410 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.410 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.410 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:46:39.410 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.061821+0000", 2023-09-23T20:46:39.410 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0032369619999999999 2023-09-23T20:46:39.410 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.410 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.410 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:46:39.410 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.202885+0000", 2023-09-23T20:46:39.410 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.14106375800000001 2023-09-23T20:46:39.411 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.411 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.411 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:46:39.411 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.202939+0000", 2023-09-23T20:46:39.411 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.4626000000000002e-05 2023-09-23T20:46:39.411 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.411 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.411 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:46:39.411 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.202987+0000", 2023-09-23T20:46:39.411 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.7528000000000003e-05 2023-09-23T20:46:39.411 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:39.411 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:46:39.411 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:39.412 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.412 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.412 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:29569 9.7 9:edf50775:::benchmark_data_smithi018_381516_object29568:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e459)", 2023-09-23T20:46:39.412 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:41:21.063339+0000", 2023-09-23T20:46:39.412 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 318.31691003200001, 2023-09-23T20:46:39.412 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.13984607700000001, 2023-09-23T20:46:39.412 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:46:39.412 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:46:39.412 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:46:39.412 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:46:39.412 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:46:39.412 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 29569 2023-09-23T20:46:39.412 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.413 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:46:39.413 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.413 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:46:39.413 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.063339+0000", 2023-09-23T20:46:39.413 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:39.413 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.413 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.413 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:46:39.413 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.063339+0000", 2023-09-23T20:46:39.413 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:39.413 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.413 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.413 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:46:39.414 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.063338+0000", 2023-09-23T20:46:39.414 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:46:39.414 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.414 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.414 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:46:39.414 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.063399+0000", 2023-09-23T20:46:39.414 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.0869000000000003e-05 2023-09-23T20:46:39.414 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.414 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.414 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:46:39.415 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.063417+0000", 2023-09-23T20:46:39.415 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.7456999999999999e-05 2023-09-23T20:46:39.415 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.415 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.415 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:46:39.415 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.063423+0000", 2023-09-23T20:46:39.415 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.4139999999999999e-06 2023-09-23T20:46:39.415 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.415 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.415 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:46:39.415 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.063463+0000", 2023-09-23T20:46:39.415 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.9626999999999999e-05 2023-09-23T20:46:39.416 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.416 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.416 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:46:39.416 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.063572+0000", 2023-09-23T20:46:39.416 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000109697 2023-09-23T20:46:39.416 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.416 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.416 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 3,7", 2023-09-23T20:46:39.416 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.063672+0000", 2023-09-23T20:46:39.416 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.9401000000000006e-05 2023-09-23T20:46:39.416 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.416 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.417 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:46:39.417 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.065628+0000", 2023-09-23T20:46:39.417 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.001955816 2023-09-23T20:46:39.417 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.417 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.417 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:46:39.417 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.065657+0000", 2023-09-23T20:46:39.417 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.9050000000000001e-05 2023-09-23T20:46:39.417 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.417 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.417 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:46:39.417 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.203114+0000", 2023-09-23T20:46:39.417 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.13745722199999999 2023-09-23T20:46:39.418 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.418 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.418 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:46:39.418 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.203160+0000", 2023-09-23T20:46:39.418 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.6292999999999999e-05 2023-09-23T20:46:39.418 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.418 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.418 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:46:39.418 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.203185+0000", 2023-09-23T20:46:39.418 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.5196999999999999e-05 2023-09-23T20:46:39.418 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:39.418 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:46:39.419 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:39.419 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.419 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.419 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:29570 9.7 9:ed5a7646:::benchmark_data_smithi018_381516_object29569:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e459)", 2023-09-23T20:46:39.419 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:41:21.063661+0000", 2023-09-23T20:46:39.419 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 318.31658876400002, 2023-09-23T20:46:39.419 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.14287293200000001, 2023-09-23T20:46:39.419 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:46:39.419 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:46:39.419 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:46:39.419 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:46:39.419 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:46:39.419 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 29570 2023-09-23T20:46:39.420 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.420 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:46:39.420 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.420 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:46:39.420 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.063661+0000", 2023-09-23T20:46:39.420 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:39.420 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.420 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.420 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:46:39.420 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.063661+0000", 2023-09-23T20:46:39.420 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:39.420 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.421 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.421 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:46:39.421 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.063660+0000", 2023-09-23T20:46:39.421 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:46:39.421 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.421 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.421 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:46:39.421 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.063718+0000", 2023-09-23T20:46:39.421 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.8522e-05 2023-09-23T20:46:39.421 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.421 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.421 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:46:39.422 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.063730+0000", 2023-09-23T20:46:39.422 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1586999999999999e-05 2023-09-23T20:46:39.422 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.422 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.422 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:46:39.422 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.063734+0000", 2023-09-23T20:46:39.422 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.3889999999999998e-06 2023-09-23T20:46:39.422 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.422 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.422 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:46:39.422 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.064722+0000", 2023-09-23T20:46:39.423 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00098758900000000009 2023-09-23T20:46:39.423 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.423 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.423 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:46:39.423 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.064854+0000", 2023-09-23T20:46:39.423 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00013245499999999999 2023-09-23T20:46:39.423 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.423 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.423 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 3,7", 2023-09-23T20:46:39.423 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.064964+0000", 2023-09-23T20:46:39.423 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000109548 2023-09-23T20:46:39.424 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.424 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.424 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:46:39.424 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.065828+0000", 2023-09-23T20:46:39.424 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00086438499999999996 2023-09-23T20:46:39.424 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.424 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.424 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:46:39.424 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.066131+0000", 2023-09-23T20:46:39.424 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00030242099999999998 2023-09-23T20:46:39.424 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.424 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.425 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:46:39.425 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.206430+0000", 2023-09-23T20:46:39.425 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.14029873500000001 2023-09-23T20:46:39.425 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.425 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.425 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:46:39.425 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.206487+0000", 2023-09-23T20:46:39.425 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.7812000000000001e-05 2023-09-23T20:46:39.425 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.425 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.425 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:46:39.425 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.206533+0000", 2023-09-23T20:46:39.426 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.6122999999999999e-05 2023-09-23T20:46:39.426 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:39.426 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:46:39.426 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:39.426 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.426 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.426 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:76985 9.5 9:ababdd75:::benchmark_data_smithi018_381516_object76984:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e465)", 2023-09-23T20:46:39.426 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:41:58.522099+0000", 2023-09-23T20:46:39.426 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 280.85815035600001, 2023-09-23T20:46:39.426 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.96136469999999996, 2023-09-23T20:46:39.426 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:46:39.427 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:46:39.427 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:46:39.427 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:46:39.427 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:46:39.427 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 76985 2023-09-23T20:46:39.427 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.427 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:46:39.427 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.427 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:46:39.427 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.522099+0000", 2023-09-23T20:46:39.427 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:39.427 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.428 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.428 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:46:39.428 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.522099+0000", 2023-09-23T20:46:39.428 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:39.428 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.428 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.428 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:46:39.428 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.522098+0000", 2023-09-23T20:46:39.428 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:46:39.428 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.428 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.428 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:46:39.428 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.522133+0000", 2023-09-23T20:46:39.429 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.4041000000000002e-05 2023-09-23T20:46:39.429 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.429 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.429 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:46:39.429 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.522141+0000", 2023-09-23T20:46:39.429 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.7900000000000005e-06 2023-09-23T20:46:39.429 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.429 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.429 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:46:39.429 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.522146+0000", 2023-09-23T20:46:39.429 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.3340000000000003e-06 2023-09-23T20:46:39.429 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.429 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.430 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:46:39.430 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.522199+0000", 2023-09-23T20:46:39.430 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.3347000000000003e-05 2023-09-23T20:46:39.430 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.430 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.430 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:46:39.430 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.522206+0000", 2023-09-23T20:46:39.430 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.6209999999999998e-06 2023-09-23T20:46:39.430 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.430 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.430 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:46:39.430 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.471807+0000", 2023-09-23T20:46:39.431 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.94960129400000004 2023-09-23T20:46:39.431 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.431 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.431 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:46:39.431 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.474386+0000", 2023-09-23T20:46:39.431 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0025794749999999999 2023-09-23T20:46:39.431 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.431 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.431 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,7", 2023-09-23T20:46:39.431 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.474496+0000", 2023-09-23T20:46:39.431 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000109733 2023-09-23T20:46:39.431 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.431 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.432 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:46:39.432 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.483272+0000", 2023-09-23T20:46:39.432 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0087758269999999999 2023-09-23T20:46:39.432 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.432 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.432 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:46:39.432 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.483367+0000", 2023-09-23T20:46:39.432 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.5458999999999995e-05 2023-09-23T20:46:39.432 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.432 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.432 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:46:39.432 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.483375+0000", 2023-09-23T20:46:39.432 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.7910000000000005e-06 2023-09-23T20:46:39.433 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.433 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.433 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:46:39.433 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.483424+0000", 2023-09-23T20:46:39.433 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.8497e-05 2023-09-23T20:46:39.433 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.433 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.433 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:46:39.433 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.483464+0000", 2023-09-23T20:46:39.433 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.9960999999999999e-05 2023-09-23T20:46:39.433 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:39.434 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:46:39.434 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:39.434 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.434 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.434 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:76994 9.5 9:a38d7184:::benchmark_data_smithi018_381516_object76993:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e465)", 2023-09-23T20:46:39.434 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:41:58.527883+0000", 2023-09-23T20:46:39.434 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 280.85236641400002, 2023-09-23T20:46:39.434 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.95854079599999997, 2023-09-23T20:46:39.434 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:46:39.434 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:46:39.434 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:46:39.435 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:46:39.435 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:46:39.435 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 76994 2023-09-23T20:46:39.435 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.435 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:46:39.435 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.435 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:46:39.435 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.527883+0000", 2023-09-23T20:46:39.435 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:39.435 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.435 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.435 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:46:39.436 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.527883+0000", 2023-09-23T20:46:39.436 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:39.436 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.436 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.436 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:46:39.436 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.527882+0000", 2023-09-23T20:46:39.436 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:46:39.436 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.436 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.436 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:46:39.436 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.527986+0000", 2023-09-23T20:46:39.436 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000104201 2023-09-23T20:46:39.436 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.437 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.437 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:46:39.437 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.528001+0000", 2023-09-23T20:46:39.437 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.5187999999999999e-05 2023-09-23T20:46:39.437 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.437 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.437 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:46:39.437 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.528009+0000", 2023-09-23T20:46:39.437 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.4259999999999997e-06 2023-09-23T20:46:39.437 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.437 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.437 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:46:39.437 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.528056+0000", 2023-09-23T20:46:39.438 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.727e-05 2023-09-23T20:46:39.438 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.438 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.438 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:46:39.438 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.528062+0000", 2023-09-23T20:46:39.438 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.1929999999999998e-06 2023-09-23T20:46:39.438 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.438 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.438 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:46:39.438 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.477679+0000", 2023-09-23T20:46:39.438 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.94961691500000001 2023-09-23T20:46:39.438 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.439 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.439 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:46:39.439 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.477800+0000", 2023-09-23T20:46:39.439 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00012139800000000001 2023-09-23T20:46:39.439 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.439 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.439 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,7", 2023-09-23T20:46:39.439 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.477938+0000", 2023-09-23T20:46:39.439 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00013737499999999999 2023-09-23T20:46:39.439 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.439 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.439 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:46:39.439 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.483301+0000", 2023-09-23T20:46:39.440 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0053636430000000004 2023-09-23T20:46:39.440 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.440 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.440 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:46:39.440 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.483541+0000", 2023-09-23T20:46:39.440 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00023936699999999999 2023-09-23T20:46:39.440 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.440 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.440 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:46:39.440 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.486265+0000", 2023-09-23T20:46:39.440 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.002724363 2023-09-23T20:46:39.440 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.440 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.441 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:46:39.441 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.486368+0000", 2023-09-23T20:46:39.441 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000103058 2023-09-23T20:46:39.441 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.441 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.441 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:46:39.441 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.486424+0000", 2023-09-23T20:46:39.441 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.5560000000000003e-05 2023-09-23T20:46:39.441 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:39.441 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:46:39.441 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:39.441 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.441 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.442 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:77025 9.5 9:a8adfb4b:::benchmark_data_smithi018_381516_object77024:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e465)", 2023-09-23T20:46:39.442 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:41:58.574546+0000", 2023-09-23T20:46:39.442 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 280.80570279599999, 2023-09-23T20:46:39.442 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.91796858599999998, 2023-09-23T20:46:39.442 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:46:39.442 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:46:39.442 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:46:39.442 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:46:39.442 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:46:39.442 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 77025 2023-09-23T20:46:39.442 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.442 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:46:39.443 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.443 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:46:39.443 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.574546+0000", 2023-09-23T20:46:39.443 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:39.443 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.443 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.443 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:46:39.443 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.574546+0000", 2023-09-23T20:46:39.443 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:39.443 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.443 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.443 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:46:39.444 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.574546+0000", 2023-09-23T20:46:39.444 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:46:39.444 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.444 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.444 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:46:39.444 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.574601+0000", 2023-09-23T20:46:39.444 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.4879000000000001e-05 2023-09-23T20:46:39.444 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.444 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.444 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:46:39.444 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.574612+0000", 2023-09-23T20:46:39.444 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1201e-05 2023-09-23T20:46:39.444 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.445 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.445 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:46:39.445 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.574616+0000", 2023-09-23T20:46:39.445 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.1370000000000004e-06 2023-09-23T20:46:39.445 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.445 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.445 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:46:39.445 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.574650+0000", 2023-09-23T20:46:39.445 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.4007000000000002e-05 2023-09-23T20:46:39.445 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.445 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.445 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:46:39.445 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.574655+0000", 2023-09-23T20:46:39.446 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.0340000000000004e-06 2023-09-23T20:46:39.446 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.446 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.446 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:46:39.446 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.480178+0000", 2023-09-23T20:46:39.446 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.90552307700000001 2023-09-23T20:46:39.446 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.446 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.446 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:46:39.446 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.480276+0000", 2023-09-23T20:46:39.446 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.8187e-05 2023-09-23T20:46:39.447 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.447 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.447 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,7", 2023-09-23T20:46:39.447 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.480425+0000", 2023-09-23T20:46:39.447 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000148228 2023-09-23T20:46:39.447 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.447 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.447 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:46:39.447 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.483500+0000", 2023-09-23T20:46:39.447 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0030750740000000001 2023-09-23T20:46:39.447 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.447 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.447 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:46:39.448 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.483545+0000", 2023-09-23T20:46:39.448 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.5864e-05 2023-09-23T20:46:39.448 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.448 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.448 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:46:39.448 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.492442+0000", 2023-09-23T20:46:39.448 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0088969189999999997 2023-09-23T20:46:39.448 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.448 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.448 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:46:39.448 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.492481+0000", 2023-09-23T20:46:39.448 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.8887999999999998e-05 2023-09-23T20:46:39.449 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.449 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.449 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:46:39.449 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.492515+0000", 2023-09-23T20:46:39.449 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.3810999999999999e-05 2023-09-23T20:46:39.449 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:39.449 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:46:39.449 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:39.449 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.449 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.449 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:91635 9.f 9:f296a7d6:::benchmark_data_smithi018_381516_object91634:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e467)", 2023-09-23T20:46:39.449 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:42:13.701301+0000", 2023-09-23T20:46:39.449 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 265.67894805700001, 2023-09-23T20:46:39.450 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.128139787, 2023-09-23T20:46:39.450 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:46:39.450 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:46:39.450 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:46:39.450 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:46:39.450 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:46:39.450 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 91635 2023-09-23T20:46:39.450 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.450 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:46:39.450 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.450 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:46:39.451 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.701301+0000", 2023-09-23T20:46:39.451 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:39.451 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.451 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.451 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:46:39.451 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.701301+0000", 2023-09-23T20:46:39.451 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:39.451 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.451 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.451 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:46:39.451 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.701301+0000", 2023-09-23T20:46:39.451 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:46:39.452 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.452 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.452 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:46:39.452 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.701346+0000", 2023-09-23T20:46:39.452 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.5599999999999997e-05 2023-09-23T20:46:39.452 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.452 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.452 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:46:39.452 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.701359+0000", 2023-09-23T20:46:39.452 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.3251000000000001e-05 2023-09-23T20:46:39.452 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.452 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.452 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:46:39.453 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.701365+0000", 2023-09-23T20:46:39.453 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.8309999999999997e-06 2023-09-23T20:46:39.453 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.453 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.453 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:46:39.453 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.706931+0000", 2023-09-23T20:46:39.453 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0055660900000000001 2023-09-23T20:46:39.453 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.453 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.453 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:46:39.453 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.707048+0000", 2023-09-23T20:46:39.453 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000116663 2023-09-23T20:46:39.454 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.454 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.454 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:46:39.454 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.714996+0000", 2023-09-23T20:46:39.454 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0079476500000000005 2023-09-23T20:46:39.454 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.454 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.454 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:46:39.454 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.719222+0000", 2023-09-23T20:46:39.454 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.004226769 2023-09-23T20:46:39.454 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.455 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.455 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:46:39.455 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.726451+0000", 2023-09-23T20:46:39.455 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0072287560000000002 2023-09-23T20:46:39.455 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.455 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.455 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:46:39.455 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.829335+0000", 2023-09-23T20:46:39.455 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.102883501 2023-09-23T20:46:39.455 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.455 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.455 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:46:39.455 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.829386+0000", 2023-09-23T20:46:39.456 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.1496999999999998e-05 2023-09-23T20:46:39.456 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.456 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.456 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:46:39.456 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.829441+0000", 2023-09-23T20:46:39.456 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.4868999999999999e-05 2023-09-23T20:46:39.456 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:39.456 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:46:39.456 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:39.456 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.456 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.456 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:91642 9.7 9:e21f89c5:::benchmark_data_smithi018_381516_object91641:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e467)", 2023-09-23T20:46:39.457 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:42:13.710744+0000", 2023-09-23T20:46:39.457 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 265.66950555699998, 2023-09-23T20:46:39.457 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.13617485700000001, 2023-09-23T20:46:39.457 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:46:39.457 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:46:39.457 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:46:39.457 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:46:39.457 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:46:39.457 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 91642 2023-09-23T20:46:39.457 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.457 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:46:39.457 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.457 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:46:39.458 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.710744+0000", 2023-09-23T20:46:39.458 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:39.458 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.458 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.458 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:46:39.458 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.710744+0000", 2023-09-23T20:46:39.458 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:39.458 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.458 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.458 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:46:39.458 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.710742+0000", 2023-09-23T20:46:39.458 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:46:39.459 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.459 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.459 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:46:39.459 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.710818+0000", 2023-09-23T20:46:39.459 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.6044999999999994e-05 2023-09-23T20:46:39.459 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.459 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.459 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:46:39.459 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.710834+0000", 2023-09-23T20:46:39.459 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.5733e-05 2023-09-23T20:46:39.459 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.459 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.459 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:46:39.460 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.710841+0000", 2023-09-23T20:46:39.460 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.0559999999999999e-06 2023-09-23T20:46:39.460 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.460 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.460 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:46:39.460 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.719091+0000", 2023-09-23T20:46:39.460 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0082496040000000007 2023-09-23T20:46:39.460 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.460 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.460 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:46:39.460 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.719366+0000", 2023-09-23T20:46:39.460 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00027542799999999998 2023-09-23T20:46:39.460 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.461 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.461 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 3,7", 2023-09-23T20:46:39.461 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.719569+0000", 2023-09-23T20:46:39.461 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00020253299999999999 2023-09-23T20:46:39.461 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.461 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.461 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:46:39.461 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.729681+0000", 2023-09-23T20:46:39.461 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.010112571000000001 2023-09-23T20:46:39.461 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.461 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.461 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:46:39.462 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.740599+0000", 2023-09-23T20:46:39.462 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.010917425 2023-09-23T20:46:39.462 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.462 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.462 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:46:39.462 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.846806+0000", 2023-09-23T20:46:39.462 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.10620702 2023-09-23T20:46:39.462 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.462 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.462 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:46:39.462 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.846863+0000", 2023-09-23T20:46:39.463 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.7924999999999999e-05 2023-09-23T20:46:39.463 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.463 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.463 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:46:39.463 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.846919+0000", 2023-09-23T20:46:39.463 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.5124999999999998e-05 2023-09-23T20:46:39.463 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:39.463 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:46:39.463 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:39.463 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.463 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.463 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:91649 9.7 9:ed097aa6:::benchmark_data_smithi018_381516_object91648:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e467)", 2023-09-23T20:46:39.463 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:42:13.719094+0000", 2023-09-23T20:46:39.464 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 265.66115491300002, 2023-09-23T20:46:39.464 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.12801558599999999, 2023-09-23T20:46:39.464 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:46:39.464 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:46:39.464 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:46:39.464 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:46:39.464 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:46:39.464 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 91649 2023-09-23T20:46:39.464 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.464 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:46:39.464 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.464 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:46:39.464 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.719094+0000", 2023-09-23T20:46:39.465 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:39.465 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.465 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.465 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:46:39.465 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.719094+0000", 2023-09-23T20:46:39.465 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:39.465 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.465 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.465 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:46:39.465 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.719093+0000", 2023-09-23T20:46:39.465 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:46:39.465 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.465 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.466 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:46:39.466 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.719128+0000", 2023-09-23T20:46:39.466 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.5079000000000001e-05 2023-09-23T20:46:39.466 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.466 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.466 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:46:39.466 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.719139+0000", 2023-09-23T20:46:39.466 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0761000000000001e-05 2023-09-23T20:46:39.466 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.466 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.466 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:46:39.467 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.719144+0000", 2023-09-23T20:46:39.467 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.2920000000000003e-06 2023-09-23T20:46:39.467 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.467 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.467 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:46:39.467 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.722939+0000", 2023-09-23T20:46:39.467 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.003794136 2023-09-23T20:46:39.467 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.467 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.467 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:46:39.467 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.723030+0000", 2023-09-23T20:46:39.467 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.1193999999999995e-05 2023-09-23T20:46:39.467 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.468 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.468 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 3,7", 2023-09-23T20:46:39.468 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.723143+0000", 2023-09-23T20:46:39.468 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000113268 2023-09-23T20:46:39.468 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.468 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.468 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:46:39.468 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.729715+0000", 2023-09-23T20:46:39.468 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0065721130000000001 2023-09-23T20:46:39.468 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.468 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.468 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:46:39.468 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.740629+0000", 2023-09-23T20:46:39.469 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.010913997 2023-09-23T20:46:39.469 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.469 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.469 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:46:39.469 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.846966+0000", 2023-09-23T20:46:39.469 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.106336736 2023-09-23T20:46:39.469 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.469 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.469 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:46:39.469 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.847061+0000", 2023-09-23T20:46:39.469 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.4692999999999996e-05 2023-09-23T20:46:39.469 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.469 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.470 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:46:39.470 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.847110+0000", 2023-09-23T20:46:39.470 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.9382000000000003e-05 2023-09-23T20:46:39.470 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:39.470 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:46:39.470 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:39.470 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.470 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.470 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:91651 9.f 9:f2c174a9:::benchmark_data_smithi018_381516_object91650:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e467)", 2023-09-23T20:46:39.470 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:42:13.719151+0000", 2023-09-23T20:46:39.470 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 265.661098783, 2023-09-23T20:46:39.471 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.12779057599999999, 2023-09-23T20:46:39.471 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:46:39.471 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:46:39.471 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:46:39.471 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:46:39.471 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:46:39.471 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 91651 2023-09-23T20:46:39.471 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.471 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:46:39.471 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.471 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:46:39.471 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.719151+0000", 2023-09-23T20:46:39.471 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:39.472 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.472 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.472 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:46:39.472 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.719151+0000", 2023-09-23T20:46:39.472 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:39.472 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.472 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.472 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:46:39.472 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.719150+0000", 2023-09-23T20:46:39.472 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:46:39.472 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.472 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.472 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:46:39.473 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.719179+0000", 2023-09-23T20:46:39.473 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.9420999999999998e-05 2023-09-23T20:46:39.473 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.473 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.473 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:46:39.473 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.719187+0000", 2023-09-23T20:46:39.473 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.2320000000000004e-06 2023-09-23T20:46:39.473 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.473 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.473 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:46:39.473 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.719191+0000", 2023-09-23T20:46:39.473 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.3220000000000003e-06 2023-09-23T20:46:39.474 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.474 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.474 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:46:39.474 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.722930+0000", 2023-09-23T20:46:39.474 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0037388640000000002 2023-09-23T20:46:39.474 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.474 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.474 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:46:39.474 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.723047+0000", 2023-09-23T20:46:39.474 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00011688699999999999 2023-09-23T20:46:39.474 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.474 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.475 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:46:39.475 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.723182+0000", 2023-09-23T20:46:39.475 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00013561199999999999 2023-09-23T20:46:39.475 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.475 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.475 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:46:39.475 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.729712+0000", 2023-09-23T20:46:39.475 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0065292340000000001 2023-09-23T20:46:39.475 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.475 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.475 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:46:39.475 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.729718+0000", 2023-09-23T20:46:39.476 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.3829999999999997e-06 2023-09-23T20:46:39.476 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.476 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.476 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:46:39.476 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.846858+0000", 2023-09-23T20:46:39.476 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.117139934 2023-09-23T20:46:39.476 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.476 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.476 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:46:39.476 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.846905+0000", 2023-09-23T20:46:39.476 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.7187000000000001e-05 2023-09-23T20:46:39.476 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.476 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.477 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:46:39.477 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.846941+0000", 2023-09-23T20:46:39.477 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.5975e-05 2023-09-23T20:46:39.477 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:39.477 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:46:39.477 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:39.477 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.477 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.477 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:92127 9.7 9:eb5270d9:::benchmark_data_smithi018_381516_object92126:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e467)", 2023-09-23T20:46:39.477 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:42:14.303475+0000", 2023-09-23T20:46:39.477 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 265.07677428300002, 2023-09-23T20:46:39.477 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.13503338400000001, 2023-09-23T20:46:39.477 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:46:39.478 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:46:39.478 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:46:39.478 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:46:39.478 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:46:39.478 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 92127 2023-09-23T20:46:39.478 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.478 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:46:39.478 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.478 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:46:39.478 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.303475+0000", 2023-09-23T20:46:39.478 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:39.478 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.479 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.479 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:46:39.479 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.303475+0000", 2023-09-23T20:46:39.479 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:39.479 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.479 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.479 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:46:39.479 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.303475+0000", 2023-09-23T20:46:39.479 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:46:39.479 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.479 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.479 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:46:39.479 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.303510+0000", 2023-09-23T20:46:39.480 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.5318999999999998e-05 2023-09-23T20:46:39.480 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.480 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.480 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:46:39.480 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.303516+0000", 2023-09-23T20:46:39.480 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.4320000000000004e-06 2023-09-23T20:46:39.480 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.480 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.480 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:46:39.480 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.303520+0000", 2023-09-23T20:46:39.480 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.0280000000000002e-06 2023-09-23T20:46:39.480 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.480 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.481 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:46:39.481 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.303550+0000", 2023-09-23T20:46:39.481 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.9938e-05 2023-09-23T20:46:39.481 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.481 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.481 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:46:39.481 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.303677+0000", 2023-09-23T20:46:39.481 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00012659800000000001 2023-09-23T20:46:39.481 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.481 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.481 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 3,7", 2023-09-23T20:46:39.481 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.303805+0000", 2023-09-23T20:46:39.481 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00012804799999999999 2023-09-23T20:46:39.482 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.482 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.482 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:46:39.482 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.306643+0000", 2023-09-23T20:46:39.482 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.002837759 2023-09-23T20:46:39.482 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.482 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.482 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:46:39.482 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.317326+0000", 2023-09-23T20:46:39.482 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.010683002 2023-09-23T20:46:39.482 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.483 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.483 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:46:39.483 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.438439+0000", 2023-09-23T20:46:39.483 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.121113448 2023-09-23T20:46:39.483 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.483 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.483 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:46:39.483 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.438478+0000", 2023-09-23T20:46:39.483 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.8921999999999998e-05 2023-09-23T20:46:39.483 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.483 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.483 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:46:39.484 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.438508+0000", 2023-09-23T20:46:39.484 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.029e-05 2023-09-23T20:46:39.484 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:39.484 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:46:39.484 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:39.484 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.484 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.484 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:92129 9.f 9:fdf451a5:::benchmark_data_smithi018_381516_object92128:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e467)", 2023-09-23T20:46:39.484 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:42:14.307177+0000", 2023-09-23T20:46:39.484 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 265.07307250999997, 2023-09-23T20:46:39.484 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.138573006, 2023-09-23T20:46:39.484 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:46:39.485 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:46:39.485 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:46:39.485 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:46:39.485 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:46:39.485 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 92129 2023-09-23T20:46:39.485 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.485 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:46:39.485 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.485 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:46:39.485 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.307177+0000", 2023-09-23T20:46:39.485 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:39.485 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.486 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.486 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:46:39.486 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.307177+0000", 2023-09-23T20:46:39.486 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:39.486 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.486 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.486 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:46:39.486 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.307175+0000", 2023-09-23T20:46:39.486 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:46:39.486 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.486 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.487 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:46:39.487 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.307245+0000", 2023-09-23T20:46:39.487 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.0261999999999999e-05 2023-09-23T20:46:39.487 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.487 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.487 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:46:39.487 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.307253+0000", 2023-09-23T20:46:39.487 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.7219999999999999e-06 2023-09-23T20:46:39.487 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.487 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.487 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:46:39.487 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.307257+0000", 2023-09-23T20:46:39.487 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.0430000000000001e-06 2023-09-23T20:46:39.488 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.488 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.488 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:46:39.488 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.307299+0000", 2023-09-23T20:46:39.488 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.2128000000000001e-05 2023-09-23T20:46:39.488 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.488 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.488 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:46:39.488 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.307431+0000", 2023-09-23T20:46:39.488 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00013203499999999999 2023-09-23T20:46:39.488 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.488 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.489 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:46:39.489 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.307559+0000", 2023-09-23T20:46:39.489 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000127632 2023-09-23T20:46:39.489 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.489 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.489 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:46:39.489 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.317355+0000", 2023-09-23T20:46:39.489 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0097961349999999992 2023-09-23T20:46:39.489 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.489 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.489 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:46:39.489 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.320352+0000", 2023-09-23T20:46:39.489 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0029971569999999999 2023-09-23T20:46:39.490 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.490 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.490 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:46:39.490 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.445677+0000", 2023-09-23T20:46:39.490 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.12532515999999999 2023-09-23T20:46:39.490 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.490 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.490 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:46:39.490 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.445717+0000", 2023-09-23T20:46:39.490 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.9616000000000002e-05 2023-09-23T20:46:39.490 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.491 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.491 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:46:39.491 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.445750+0000", 2023-09-23T20:46:39.491 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.2719999999999998e-05 2023-09-23T20:46:39.491 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:39.491 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:46:39.491 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:39.491 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.491 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.491 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:92140 9.f 9:f3d16644:::benchmark_data_smithi018_381516_object92139:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e467)", 2023-09-23T20:46:39.491 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:42:14.313964+0000", 2023-09-23T20:46:39.491 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 265.06628510100001, 2023-09-23T20:46:39.492 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.13195174000000001, 2023-09-23T20:46:39.492 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:46:39.492 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:46:39.492 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:46:39.492 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:46:39.492 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:46:39.492 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 92140 2023-09-23T20:46:39.492 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.492 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:46:39.492 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.492 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:46:39.492 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.313964+0000", 2023-09-23T20:46:39.493 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:39.493 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.493 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.493 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:46:39.493 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.313964+0000", 2023-09-23T20:46:39.493 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:39.493 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.493 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.493 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:46:39.493 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.313964+0000", 2023-09-23T20:46:39.493 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:46:39.493 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.494 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.494 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:46:39.494 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.314026+0000", 2023-09-23T20:46:39.494 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.2818999999999996e-05 2023-09-23T20:46:39.494 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.494 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.494 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:46:39.494 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.314035+0000", 2023-09-23T20:46:39.494 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.7059999999999993e-06 2023-09-23T20:46:39.494 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.494 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.495 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:46:39.495 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.314040+0000", 2023-09-23T20:46:39.495 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.2660000000000001e-06 2023-09-23T20:46:39.495 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.495 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.495 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:46:39.495 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.314084+0000", 2023-09-23T20:46:39.495 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.3204000000000002e-05 2023-09-23T20:46:39.495 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.495 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.495 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:46:39.495 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.314195+0000", 2023-09-23T20:46:39.496 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000111585 2023-09-23T20:46:39.496 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.496 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.496 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:46:39.496 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.314336+0000", 2023-09-23T20:46:39.496 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000140977 2023-09-23T20:46:39.496 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.496 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.496 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:46:39.496 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.320388+0000", 2023-09-23T20:46:39.496 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.006051519 2023-09-23T20:46:39.496 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.497 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.497 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:46:39.497 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.320393+0000", 2023-09-23T20:46:39.497 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.7189999999999998e-06 2023-09-23T20:46:39.497 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.497 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.497 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:46:39.497 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.445855+0000", 2023-09-23T20:46:39.497 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.125461666 2023-09-23T20:46:39.497 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.497 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.497 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:46:39.498 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.445885+0000", 2023-09-23T20:46:39.498 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.0454e-05 2023-09-23T20:46:39.498 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.498 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.498 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:46:39.498 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.445916+0000", 2023-09-23T20:46:39.498 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.0446e-05 2023-09-23T20:46:39.498 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:39.498 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:46:39.498 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:39.498 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.499 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.499 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:92144 9.7 9:ec1b46c9:::benchmark_data_smithi018_381516_object92143:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e467)", 2023-09-23T20:46:39.499 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:42:14.317205+0000", 2023-09-23T20:46:39.499 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 265.06304393699997, 2023-09-23T20:46:39.499 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.12863032999999999, 2023-09-23T20:46:39.499 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:46:39.499 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:46:39.499 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:46:39.499 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:46:39.499 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:46:39.499 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 92144 2023-09-23T20:46:39.499 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.500 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:46:39.500 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.500 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:46:39.500 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.317205+0000", 2023-09-23T20:46:39.500 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:39.500 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.500 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.500 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:46:39.500 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.317205+0000", 2023-09-23T20:46:39.500 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:39.500 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.500 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.501 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:46:39.501 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.317204+0000", 2023-09-23T20:46:39.501 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:46:39.501 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.501 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.501 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:46:39.501 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.317280+0000", 2023-09-23T20:46:39.501 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.6215000000000001e-05 2023-09-23T20:46:39.501 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.501 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.501 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:46:39.501 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.317291+0000", 2023-09-23T20:46:39.501 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0546e-05 2023-09-23T20:46:39.502 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.502 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.502 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:46:39.502 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.317297+0000", 2023-09-23T20:46:39.502 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.2650000000000002e-06 2023-09-23T20:46:39.502 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.502 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.502 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:46:39.502 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.317374+0000", 2023-09-23T20:46:39.502 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.6600999999999994e-05 2023-09-23T20:46:39.502 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.502 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.503 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:46:39.503 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.317479+0000", 2023-09-23T20:46:39.503 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000105531 2023-09-23T20:46:39.503 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.503 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.503 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 3,7", 2023-09-23T20:46:39.503 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.317604+0000", 2023-09-23T20:46:39.503 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000124745 2023-09-23T20:46:39.503 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.503 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.503 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:46:39.503 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.321102+0000", 2023-09-23T20:46:39.503 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0034977849999999998 2023-09-23T20:46:39.504 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.504 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.504 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:46:39.504 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.321402+0000", 2023-09-23T20:46:39.504 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00029973499999999998 2023-09-23T20:46:39.504 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.504 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.504 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:46:39.504 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.445781+0000", 2023-09-23T20:46:39.504 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.124378979 2023-09-23T20:46:39.504 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.504 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.505 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:46:39.505 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.445816+0000", 2023-09-23T20:46:39.505 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.5222000000000003e-05 2023-09-23T20:46:39.505 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.505 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.505 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:46:39.505 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.445836+0000", 2023-09-23T20:46:39.505 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.9952000000000001e-05 2023-09-23T20:46:39.505 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:39.505 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:46:39.505 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:39.505 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.505 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.506 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:141052 9.5 9:a7a766e0:::benchmark_data_smithi018_381516_object27218:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e481)", 2023-09-23T20:46:39.506 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:43:15.774779+0000", 2023-09-23T20:46:39.506 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 203.60547052300001, 2023-09-23T20:46:39.506 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.023693841, 2023-09-23T20:46:39.506 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:46:39.506 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:46:39.506 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:46:39.506 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:46:39.506 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:46:39.506 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 141052 2023-09-23T20:46:39.506 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.506 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:46:39.507 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.507 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:46:39.507 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.774779+0000", 2023-09-23T20:46:39.507 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:39.507 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.507 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.507 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:46:39.507 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.774779+0000", 2023-09-23T20:46:39.507 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:39.507 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.507 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.507 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:46:39.508 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.774778+0000", 2023-09-23T20:46:39.508 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:46:39.508 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.508 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.508 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:46:39.508 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.774783+0000", 2023-09-23T20:46:39.508 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.7660000000000004e-06 2023-09-23T20:46:39.508 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.508 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.508 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:46:39.508 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.774794+0000", 2023-09-23T20:46:39.509 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0744000000000001e-05 2023-09-23T20:46:39.509 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.509 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.509 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:46:39.509 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.774799+0000", 2023-09-23T20:46:39.509 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.9810000000000003e-06 2023-09-23T20:46:39.509 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.509 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.509 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:46:39.509 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.774870+0000", 2023-09-23T20:46:39.509 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.1316000000000006e-05 2023-09-23T20:46:39.509 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.509 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.510 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:46:39.510 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.774881+0000", 2023-09-23T20:46:39.510 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0291e-05 2023-09-23T20:46:39.510 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.510 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.510 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:46:39.510 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.785209+0000", 2023-09-23T20:46:39.510 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0103286419999999 2023-09-23T20:46:39.510 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.510 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.510 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:46:39.510 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.785375+0000", 2023-09-23T20:46:39.511 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00016602399999999999 2023-09-23T20:46:39.511 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.511 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.511 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,1,7", 2023-09-23T20:46:39.511 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.785518+0000", 2023-09-23T20:46:39.511 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00014225800000000001 2023-09-23T20:46:39.511 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.511 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.511 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:46:39.511 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.793627+0000", 2023-09-23T20:46:39.511 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0081096780000000004 2023-09-23T20:46:39.511 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.512 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.512 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:46:39.512 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.793665+0000", 2023-09-23T20:46:39.512 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.7286000000000002e-05 2023-09-23T20:46:39.512 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.512 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.512 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:46:39.512 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.797943+0000", 2023-09-23T20:46:39.512 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0042786869999999998 2023-09-23T20:46:39.512 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.512 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.513 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:46:39.513 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.798292+0000", 2023-09-23T20:46:39.513 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00034836099999999999 2023-09-23T20:46:39.513 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.513 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.513 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:46:39.513 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.798353+0000", 2023-09-23T20:46:39.513 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.1019e-05 2023-09-23T20:46:39.513 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.513 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.513 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:46:39.513 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.798473+0000", 2023-09-23T20:46:39.514 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000119888 2023-09-23T20:46:39.514 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:39.514 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:46:39.514 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:39.514 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.514 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.514 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:141057 9.5 9:aa98a8f8:::benchmark_data_smithi018_381516_object27223:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e481)", 2023-09-23T20:46:39.514 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:43:15.777430+0000", 2023-09-23T20:46:39.514 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 203.602819133, 2023-09-23T20:46:39.514 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0450975220000001, 2023-09-23T20:46:39.514 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:46:39.515 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:46:39.515 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:46:39.515 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:46:39.515 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:46:39.515 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 141057 2023-09-23T20:46:39.515 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.515 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:46:39.515 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.515 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:46:39.515 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.777430+0000", 2023-09-23T20:46:39.515 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:39.515 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.516 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.516 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:46:39.516 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.777430+0000", 2023-09-23T20:46:39.516 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:39.516 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.516 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.516 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:46:39.516 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.777428+0000", 2023-09-23T20:46:39.516 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999981 2023-09-23T20:46:39.516 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.516 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.516 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:46:39.517 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.777438+0000", 2023-09-23T20:46:39.517 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.025e-05 2023-09-23T20:46:39.517 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.517 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.517 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:46:39.517 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.777451+0000", 2023-09-23T20:46:39.517 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.2218000000000001e-05 2023-09-23T20:46:39.517 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.517 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.517 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:46:39.517 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.777456+0000", 2023-09-23T20:46:39.517 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.9019999999999998e-06 2023-09-23T20:46:39.518 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.518 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.518 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:46:39.518 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.777492+0000", 2023-09-23T20:46:39.518 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.6758000000000003e-05 2023-09-23T20:46:39.518 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.518 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.518 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:46:39.518 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.777498+0000", 2023-09-23T20:46:39.518 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.9529999999999996e-06 2023-09-23T20:46:39.518 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.519 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.519 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:46:39.519 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.793684+0000", 2023-09-23T20:46:39.519 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0161855 2023-09-23T20:46:39.519 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.519 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.519 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:46:39.519 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.793806+0000", 2023-09-23T20:46:39.519 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00012180600000000001 2023-09-23T20:46:39.519 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.519 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.519 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,1,7", 2023-09-23T20:46:39.520 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.793923+0000", 2023-09-23T20:46:39.520 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00011699100000000001 2023-09-23T20:46:39.520 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.520 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.520 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:46:39.520 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.803080+0000", 2023-09-23T20:46:39.520 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0091576539999999994 2023-09-23T20:46:39.520 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.520 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.520 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:46:39.520 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.809899+0000", 2023-09-23T20:46:39.521 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0068191249999999997 2023-09-23T20:46:39.521 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.521 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.521 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:46:39.521 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.817155+0000", 2023-09-23T20:46:39.521 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0072551860000000003 2023-09-23T20:46:39.521 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.521 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.521 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:46:39.521 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.822362+0000", 2023-09-23T20:46:39.521 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0052070450000000004 2023-09-23T20:46:39.521 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.522 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.522 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:46:39.522 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.822413+0000", 2023-09-23T20:46:39.522 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.1240999999999999e-05 2023-09-23T20:46:39.522 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.522 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.522 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:46:39.522 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.822528+0000", 2023-09-23T20:46:39.522 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000114843 2023-09-23T20:46:39.522 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:39.522 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:46:39.523 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:39.523 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.523 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.523 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:141060 9.5 9:a222c8d8:::benchmark_data_smithi018_381516_object27226:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e481)", 2023-09-23T20:46:39.523 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:43:15.784280+0000", 2023-09-23T20:46:39.523 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 203.59596891499999, 2023-09-23T20:46:39.523 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0384760930000001, 2023-09-23T20:46:39.523 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:46:39.523 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:46:39.523 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:46:39.523 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:46:39.523 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:46:39.524 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 141060 2023-09-23T20:46:39.524 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.524 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:46:39.524 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.524 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:46:39.524 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.784280+0000", 2023-09-23T20:46:39.524 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:39.524 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.524 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.524 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:46:39.524 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.784280+0000", 2023-09-23T20:46:39.524 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:39.524 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.525 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.525 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:46:39.525 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.784278+0000", 2023-09-23T20:46:39.525 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999981 2023-09-23T20:46:39.525 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.525 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.525 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:46:39.525 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.784293+0000", 2023-09-23T20:46:39.525 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.5048999999999999e-05 2023-09-23T20:46:39.525 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.525 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.526 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:46:39.526 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.784311+0000", 2023-09-23T20:46:39.526 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.7530999999999999e-05 2023-09-23T20:46:39.526 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.526 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.526 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:46:39.526 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.784329+0000", 2023-09-23T20:46:39.526 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.8470000000000001e-05 2023-09-23T20:46:39.526 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.526 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.526 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:46:39.526 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.784394+0000", 2023-09-23T20:46:39.527 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.4487999999999996e-05 2023-09-23T20:46:39.527 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.527 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.527 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:46:39.527 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.784404+0000", 2023-09-23T20:46:39.527 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0478e-05 2023-09-23T20:46:39.527 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.527 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.527 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:46:39.527 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.798720+0000", 2023-09-23T20:46:39.527 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.014316145 2023-09-23T20:46:39.527 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.528 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.528 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:46:39.528 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.798848+0000", 2023-09-23T20:46:39.528 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000127448 2023-09-23T20:46:39.528 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.528 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.528 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,1,7", 2023-09-23T20:46:39.528 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.798989+0000", 2023-09-23T20:46:39.528 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00014109800000000001 2023-09-23T20:46:39.528 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.528 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.528 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:46:39.529 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.809908+0000", 2023-09-23T20:46:39.529 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.01091902 2023-09-23T20:46:39.529 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.529 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.529 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:46:39.529 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.812374+0000", 2023-09-23T20:46:39.529 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0024658710000000001 2023-09-23T20:46:39.529 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.529 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.529 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:46:39.529 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.817185+0000", 2023-09-23T20:46:39.529 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0048108229999999997 2023-09-23T20:46:39.529 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.530 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.530 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:46:39.530 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.822622+0000", 2023-09-23T20:46:39.530 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0054370920000000001 2023-09-23T20:46:39.530 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.530 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.530 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:46:39.530 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.822660+0000", 2023-09-23T20:46:39.530 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.8250000000000001e-05 2023-09-23T20:46:39.530 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:39.530 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:39.530 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:46:39.531 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.822756+0000", 2023-09-23T20:46:39.531 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.6471999999999994e-05 2023-09-23T20:46:39.531 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:39.531 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:46:39.531 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:39.531 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:39.531 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:46:39.531 INFO:teuthology.orchestra.run.smithi149.stdout:} 2023-09-23T20:46:39.532 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:39.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:39 smithi149 ceph-mon[170394]: pgmap v1374: 119 pgs: 119 active+clean; 4.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 62 MiB/s wr, 986 op/s 2023-09-23T20:46:39.819 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:46:39.826 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:39.827 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:46:39.927 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T20:46:39.927 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:46:40.001 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:40.008 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 55 16 82893 82877 94.127 63.875 0.0177876 0.0106189 2023-09-23T20:46:40.090 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:46:40.097 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:40.098 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:46:40.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:39 smithi018 ceph-mon[184144]: pgmap v1374: 119 pgs: 119 active+clean; 4.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 62 MiB/s wr, 986 op/s 2023-09-23T20:46:40.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:39 smithi018 ceph-mon[179647]: pgmap v1374: 119 pgs: 119 active+clean; 4.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 62 MiB/s wr, 986 op/s 2023-09-23T20:46:40.198 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:46:40.296 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:40.369 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:46:40.378 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:40.379 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:46:40.480 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:46:40.506 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:40.541 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:46:40.548 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:40.549 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:46:40.650 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:46:40.695 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:46:40.698 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:40.969 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:46:40.978 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:40.979 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:46:41.008 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 56 16 83764 83748 93.4184 54.4375 0.0220871 0.0106991 2023-09-23T20:46:41.079 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:46:41.146 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:41.234 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:46:41.240 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:41.241 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:46:41.341 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:46:41.385 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:41.572 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:46:41.583 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:41.584 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:46:41.685 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:46:41.723 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:46:41.788 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:41.912 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:46:41.917 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:41.917 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:46:41.933 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:41 smithi149 ceph-mon[170394]: pgmap v1375: 119 pgs: 119 active+clean; 4.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 61 MiB/s wr, 983 op/s 2023-09-23T20:46:42.008 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 57 16 84703 84687 92.8093 58.6875 0.0139256 0.0107696 2023-09-23T20:46:42.018 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:46:42.079 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:42.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:41 smithi018 ceph-mon[184144]: pgmap v1375: 119 pgs: 119 active+clean; 4.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 61 MiB/s wr, 983 op/s 2023-09-23T20:46:42.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:41 smithi018 ceph-mon[179647]: pgmap v1375: 119 pgs: 119 active+clean; 4.8 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 61 MiB/s wr, 983 op/s 2023-09-23T20:46:42.176 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:46:42.193 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:42.193 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:46:42.244 INFO:teuthology.orchestra.run.smithi149.stdout:{ 2023-09-23T20:46:42.244 INFO:teuthology.orchestra.run.smithi149.stdout: "ops": [ 2023-09-23T20:46:42.244 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:42.244 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:84846 10.0 10:03c490a1:::benchmark_data_smithi018_422431_object84845:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e542)", 2023-09-23T20:46:42.244 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:46:42.220697+0000", 2023-09-23T20:46:42.244 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 0.020681188, 2023-09-23T20:46:42.244 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.020708863000000001, 2023-09-23T20:46:42.244 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:46:42.244 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:46:42.245 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:46:42.245 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:46:42.245 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:46:42.245 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 84846 2023-09-23T20:46:42.245 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:42.245 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:46:42.245 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:42.245 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:46:42.245 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:42.220697+0000", 2023-09-23T20:46:42.245 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:42.245 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:42.245 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:42.246 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:46:42.246 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:42.220697+0000", 2023-09-23T20:46:42.246 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:42.246 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:42.246 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:42.246 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:46:42.246 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:42.220697+0000", 2023-09-23T20:46:42.246 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:46:42.246 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:42.246 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:42.246 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:46:42.246 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:42.220727+0000", 2023-09-23T20:46:42.247 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.0241999999999999e-05 2023-09-23T20:46:42.247 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:42.247 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:42.247 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:46:42.247 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:42.224242+0000", 2023-09-23T20:46:42.247 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.003514929 2023-09-23T20:46:42.247 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:42.247 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:42.247 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:46:42.247 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:42.224252+0000", 2023-09-23T20:46:42.247 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0359e-05 2023-09-23T20:46:42.247 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:42.248 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:42.248 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:46:42.248 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:42.228235+0000", 2023-09-23T20:46:42.248 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0039826560000000002 2023-09-23T20:46:42.248 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:42.248 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:42.248 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:46:42.248 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:42.228409+0000", 2023-09-23T20:46:42.248 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000174017 2023-09-23T20:46:42.248 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:42.248 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:42.248 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 1,4", 2023-09-23T20:46:42.249 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:42.228567+0000", 2023-09-23T20:46:42.249 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00015837699999999999 2023-09-23T20:46:42.249 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:42.249 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:42.249 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:46:42.249 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:42.231769+0000", 2023-09-23T20:46:42.249 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.003201354 2023-09-23T20:46:42.249 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:42.249 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:42.249 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:46:42.249 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:42.232331+0000", 2023-09-23T20:46:42.249 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000561876 2023-09-23T20:46:42.250 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:42.250 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:46:42.250 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:42.250 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:42.250 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:42.250 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:84853 10.0 10:0a5e3a03:::benchmark_data_smithi018_422431_object84852:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e542)", 2023-09-23T20:46:42.250 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:46:42.232338+0000", 2023-09-23T20:46:42.250 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 0.0090402899999999994, 2023-09-23T20:46:42.250 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0091268170000000006, 2023-09-23T20:46:42.250 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:46:42.250 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:46:42.251 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:46:42.251 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:46:42.251 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:46:42.251 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 84853 2023-09-23T20:46:42.251 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:42.251 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:46:42.251 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:42.251 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:46:42.251 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:42.232338+0000", 2023-09-23T20:46:42.251 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:42.251 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:42.252 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:42.252 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:46:42.252 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:42.232338+0000", 2023-09-23T20:46:42.252 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:42.252 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:42.252 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:42.252 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:46:42.252 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:42.232337+0000", 2023-09-23T20:46:42.252 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:46:42.252 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:42.252 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:42.253 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:46:42.253 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:42.232371+0000", 2023-09-23T20:46:42.253 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.4007999999999997e-05 2023-09-23T20:46:42.253 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:42.253 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:42.253 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:46:42.253 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:42.232381+0000", 2023-09-23T20:46:42.253 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.8320000000000001e-06 2023-09-23T20:46:42.253 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:42.253 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:42.253 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:46:42.253 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:42.232387+0000", 2023-09-23T20:46:42.254 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.5480000000000001e-06 2023-09-23T20:46:42.254 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:42.254 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:42.254 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:46:42.254 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:42.232435+0000", 2023-09-23T20:46:42.254 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.8457e-05 2023-09-23T20:46:42.254 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:42.254 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:42.254 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:46:42.254 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:42.232569+0000", 2023-09-23T20:46:42.254 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00013365999999999999 2023-09-23T20:46:42.255 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:42.255 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:42.255 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 1,4", 2023-09-23T20:46:42.255 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:42.232707+0000", 2023-09-23T20:46:42.255 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00013875000000000001 2023-09-23T20:46:42.255 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:42.255 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:42.255 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:46:42.255 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:42.236600+0000", 2023-09-23T20:46:42.255 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.003892614 2023-09-23T20:46:42.255 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:42.255 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:42.256 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:46:42.256 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:42.239554+0000", 2023-09-23T20:46:42.256 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0029540780000000002 2023-09-23T20:46:42.256 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:42.256 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:46:42.256 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:42.256 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:42.256 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:42.256 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:84859 10.0 10.b5555c10 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e542)", 2023-09-23T20:46:42.256 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:46:42.239716+0000", 2023-09-23T20:46:42.256 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 0.0016622710000000001, 2023-09-23T20:46:42.256 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0017972730000000001, 2023-09-23T20:46:42.257 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:46:42.257 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "queued for pg", 2023-09-23T20:46:42.257 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:46:42.257 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:46:42.257 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:46:42.257 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 84859 2023-09-23T20:46:42.257 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:42.257 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:46:42.257 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:42.257 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:46:42.257 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:42.239716+0000", 2023-09-23T20:46:42.257 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:42.258 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:42.258 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:42.258 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:46:42.258 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:42.239716+0000", 2023-09-23T20:46:42.258 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:42.258 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:42.258 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:42.258 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:46:42.258 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:42.239715+0000", 2023-09-23T20:46:42.258 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:46:42.259 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:42.259 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:42.259 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:46:42.259 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:42.239766+0000", 2023-09-23T20:46:42.259 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.1218000000000003e-05 2023-09-23T20:46:42.259 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:42.259 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:42.259 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:46:42.259 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:42.239778+0000", 2023-09-23T20:46:42.259 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.2106999999999999e-05 2023-09-23T20:46:42.259 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:42.259 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:42.259 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:46:42.260 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:42.239785+0000", 2023-09-23T20:46:42.260 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.2790000000000004e-06 2023-09-23T20:46:42.260 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:42.260 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:46:42.260 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:42.260 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:42.260 INFO:teuthology.orchestra.run.smithi149.stdout: ], 2023-09-23T20:46:42.260 INFO:teuthology.orchestra.run.smithi149.stdout: "num_ops": 3 2023-09-23T20:46:42.260 INFO:teuthology.orchestra.run.smithi149.stdout:} 2023-09-23T20:46:42.295 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:46:42.361 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:42.410 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:46:42.413 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:42.413 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:46:42.514 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:46:42.578 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:42.700 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:46:42.713 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:42.713 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:46:42.748 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T20:46:42.774 INFO:teuthology.orchestra.run.smithi018.stderr:noscrub is set 2023-09-23T20:46:42.814 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:46:42.842 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:42 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3024561025' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:46:42.842 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:42 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:46:42.843 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:42 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3024561025' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:46:42.843 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:42 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:46:42.846 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:42 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3024561025' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:46:42.846 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:42 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:46:42.905 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:43.010 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 58 16 85465 85449 92.0277 47.625 0.0206101 0.0108608 2023-09-23T20:46:43.318 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:46:43.330 INFO:teuthology.orchestra.run.smithi018.stderr:dumped all 2023-09-23T20:46:43.473 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:46:43.488 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:43.489 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:46:43.550 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:46:43.590 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:46:43.632 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:43.836 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:46:43.840 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:43.841 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:46:43.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:43 smithi018 ceph-mon[179647]: pgmap v1376: 119 pgs: 119 active+clean; 5.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 61 MiB/s wr, 977 op/s 2023-09-23T20:46:43.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:43 smithi018 ceph-mon[179647]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:46:43.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:43 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:46:43.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:43 smithi018 ceph-mon[179647]: osdmap e543: 8 total, 8 up, 8 in 2023-09-23T20:46:43.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:43 smithi018 ceph-mon[179647]: failed to encode map e543 with expected crc 2023-09-23T20:46:43.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:43 smithi018 ceph-mon[179647]: failed to encode map e543 with expected crc 2023-09-23T20:46:43.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:43 smithi018 ceph-mon[179647]: failed to encode map e543 with expected crc 2023-09-23T20:46:43.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:43 smithi018 ceph-mon[179647]: failed to encode map e543 with expected crc 2023-09-23T20:46:43.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:43 smithi018 ceph-mon[184144]: pgmap v1376: 119 pgs: 119 active+clean; 5.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 61 MiB/s wr, 977 op/s 2023-09-23T20:46:43.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:43 smithi018 ceph-mon[184144]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:46:43.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:43 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:46:43.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:43 smithi018 ceph-mon[184144]: osdmap e543: 8 total, 8 up, 8 in 2023-09-23T20:46:43.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:43 smithi018 ceph-mon[184144]: failed to encode map e543 with expected crc 2023-09-23T20:46:43.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:43 smithi018 ceph-mon[184144]: failed to encode map e543 with expected crc 2023-09-23T20:46:43.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:43 smithi018 ceph-mon[184144]: failed to encode map e543 with expected crc 2023-09-23T20:46:43.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:43 smithi018 ceph-mon[184144]: failed to encode map e543 with expected crc 2023-09-23T20:46:43.917 INFO:teuthology.orchestra.run.smithi018.stdout:{"pg_ready":true,"pg_map":{"version":1377,"stamp":"2023-09-23T20:46:42.742631+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5436413875,"num_objects":82908,"num_object_clones":0,"num_object_copies":248724,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":82908,"num_whiteouts":0,"num_read":5983,"num_read_kb":13200,"num_write":83787,"num_write_kb":5317045,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":174,"num_bytes_recovered":16795702,"num_keys_recovered":40,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":88902,"ondisk_log_size":88902,"up":357,"acting":357,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":357,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":21329552,"kb_used_data":15925904,"kb_used_omap":32,"kb_used_meta":5403551,"kb_avail":728631664,"statfs":{"total":767960285184,"available":746118823936,"internally_reserved":0,"allocated":16308125696,"data_stored":16291633406,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32773,"internal_metadata":5533237243},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,5,1,6,4,0,0,2],"upper_bound":256},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":563085312,"num_objects":8592,"num_object_clones":0,"num_object_copies":25776,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":8592,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8592,"num_write_kb":549888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.064442"},"pg_stats":[{"pgid":"1.18","version":"90'161","reported_seq":588,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-09-23T20:46:25.639246+0000","last_change":"2023-09-23T20:44:19.677357+0000","last_active":"2023-09-23T20:46:25.639246+0000","last_peered":"2023-09-23T20:46:25.639246+0000","last_clean":"2023-09-23T20:46:25.639246+0000","last_became_active":"2023-09-23T20:43:16.766305+0000","last_became_peered":"2023-09-23T20:43:16.766305+0000","last_unstale":"2023-09-23T20:46:25.639246+0000","last_undegraded":"2023-09-23T20:46:25.639246+0000","last_fullsized":"2023-09-23T20:46:25.639246+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:19.677209+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:19.677209+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:14:06.510115+0000","scrub_duration":0.0023292260000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"69'1","reported_seq":239,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-09-23T20:46:25.636423+0000","last_change":"2023-09-23T20:32:33.233853+0000","last_active":"2023-09-23T20:46:25.636423+0000","last_peered":"2023-09-23T20:46:25.636423+0000","last_clean":"2023-09-23T20:46:25.636423+0000","last_became_active":"2023-09-23T20:32:33.233546+0000","last_became_peered":"2023-09-23T20:32:33.233546+0000","last_unstale":"2023-09-23T20:46:25.636423+0000","last_undegraded":"2023-09-23T20:46:25.636423+0000","last_fullsized":"2023-09-23T20:46:25.636423+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:50:27.130808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":357,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-09-23T20:46:25.646317+0000","last_change":"2023-09-23T20:37:43.579611+0000","last_active":"2023-09-23T20:46:25.646317+0000","last_peered":"2023-09-23T20:46:25.646317+0000","last_clean":"2023-09-23T20:46:25.646317+0000","last_became_active":"2023-09-23T20:37:43.579323+0000","last_became_peered":"2023-09-23T20:37:43.579323+0000","last_unstale":"2023-09-23T20:46:25.646317+0000","last_undegraded":"2023-09-23T20:46:25.646317+0000","last_fullsized":"2023-09-23T20:46:25.646317+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:17:32.311427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"90'161","reported_seq":663,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-09-23T20:46:25.636510+0000","last_change":"2023-09-23T20:44:14.647447+0000","last_active":"2023-09-23T20:46:25.636510+0000","last_peered":"2023-09-23T20:46:25.636510+0000","last_clean":"2023-09-23T20:46:25.636510+0000","last_became_active":"2023-09-23T20:43:16.756116+0000","last_became_peered":"2023-09-23T20:43:16.756116+0000","last_unstale":"2023-09-23T20:46:25.636510+0000","last_undegraded":"2023-09-23T20:46:25.636510+0000","last_fullsized":"2023-09-23T20:46:25.636510+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:14.647386+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:36:05.442288+0000","last_clean_scrub_stamp":"2023-09-23T20:44:14.647386+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:25:25.151100+0000","scrub_duration":0.0097381210000000006,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":512,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-09-23T20:46:25.641466+0000","last_change":"2023-09-23T20:34:59.983035+0000","last_active":"2023-09-23T20:46:25.641466+0000","last_peered":"2023-09-23T20:46:25.641466+0000","last_clean":"2023-09-23T20:46:25.641466+0000","last_became_active":"2023-09-23T20:34:59.982459+0000","last_became_peered":"2023-09-23T20:34:59.982459+0000","last_unstale":"2023-09-23T20:46:25.641466+0000","last_undegraded":"2023-09-23T20:46:25.641466+0000","last_fullsized":"2023-09-23T20:46:25.641466+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:16:48.510783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":500,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-09-23T20:46:25.645024+0000","last_change":"2023-09-23T20:43:46.521939+0000","last_active":"2023-09-23T20:46:25.645024+0000","last_peered":"2023-09-23T20:46:25.645024+0000","last_clean":"2023-09-23T20:46:25.645024+0000","last_became_active":"2023-09-23T20:43:46.521185+0000","last_became_peered":"2023-09-23T20:43:46.521185+0000","last_unstale":"2023-09-23T20:46:25.645024+0000","last_undegraded":"2023-09-23T20:46:25.645024+0000","last_fullsized":"2023-09-23T20:46:25.645024+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:59:25.045627+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"90'161","reported_seq":621,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-09-23T20:46:25.636043+0000","last_change":"2023-09-23T20:36:01.292103+0000","last_active":"2023-09-23T20:46:25.636043+0000","last_peered":"2023-09-23T20:46:25.636043+0000","last_clean":"2023-09-23T20:46:25.636043+0000","last_became_active":"2023-09-23T20:35:26.639912+0000","last_became_peered":"2023-09-23T20:35:26.639912+0000","last_unstale":"2023-09-23T20:46:25.636043+0000","last_undegraded":"2023-09-23T20:46:25.636043+0000","last_fullsized":"2023-09-23T20:46:25.636043+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:01.292032+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:01.292032+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:04:53.926640+0000","scrub_duration":0.0032983159999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":337,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-09-23T20:46:25.645556+0000","last_change":"2023-09-23T20:37:43.584414+0000","last_active":"2023-09-23T20:46:25.645556+0000","last_peered":"2023-09-23T20:46:25.645556+0000","last_clean":"2023-09-23T20:46:25.645556+0000","last_became_active":"2023-09-23T20:37:43.584116+0000","last_became_peered":"2023-09-23T20:37:43.584116+0000","last_unstale":"2023-09-23T20:46:25.645556+0000","last_undegraded":"2023-09-23T20:46:25.645556+0000","last_fullsized":"2023-09-23T20:46:25.645556+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:17:54.258172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":157,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-09-23T20:46:25.639835+0000","last_change":"2023-09-23T20:38:37.457427+0000","last_active":"2023-09-23T20:46:25.639835+0000","last_peered":"2023-09-23T20:46:25.639835+0000","last_clean":"2023-09-23T20:46:25.639835+0000","last_became_active":"2023-09-23T20:38:37.457099+0000","last_became_peered":"2023-09-23T20:38:37.457099+0000","last_unstale":"2023-09-23T20:46:25.639835+0000","last_undegraded":"2023-09-23T20:46:25.639835+0000","last_fullsized":"2023-09-23T20:46:25.639835+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:31:27.084994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"90'161","reported_seq":754,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-09-23T20:46:25.650197+0000","last_change":"2023-09-23T20:40:22.449754+0000","last_active":"2023-09-23T20:46:25.650197+0000","last_peered":"2023-09-23T20:46:25.650197+0000","last_clean":"2023-09-23T20:46:25.650197+0000","last_became_active":"2023-09-23T20:40:22.448838+0000","last_became_peered":"2023-09-23T20:40:22.448838+0000","last_unstale":"2023-09-23T20:46:25.650197+0000","last_undegraded":"2023-09-23T20:46:25.650197+0000","last_fullsized":"2023-09-23T20:46:25.650197+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:03.411025+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:03.411025+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:22:07.948856+0000","scrub_duration":0.001802922,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":533,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-09-23T20:46:25.643929+0000","last_change":"2023-09-23T20:43:46.522325+0000","last_active":"2023-09-23T20:46:25.643929+0000","last_peered":"2023-09-23T20:46:25.643929+0000","last_clean":"2023-09-23T20:46:25.643929+0000","last_became_active":"2023-09-23T20:43:46.521996+0000","last_became_peered":"2023-09-23T20:43:46.521996+0000","last_unstale":"2023-09-23T20:46:25.643929+0000","last_undegraded":"2023-09-23T20:46:25.643929+0000","last_fullsized":"2023-09-23T20:46:25.643929+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:51:17.629956+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":268,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-09-23T20:46:25.639340+0000","last_change":"2023-09-23T20:38:02.579343+0000","last_active":"2023-09-23T20:46:25.639340+0000","last_peered":"2023-09-23T20:46:25.639340+0000","last_clean":"2023-09-23T20:46:25.639340+0000","last_became_active":"2023-09-23T20:38:02.579047+0000","last_became_peered":"2023-09-23T20:38:02.579047+0000","last_unstale":"2023-09-23T20:46:25.639340+0000","last_undegraded":"2023-09-23T20:46:25.639340+0000","last_fullsized":"2023-09-23T20:46:25.639340+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:44:03.166556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.14","version":"90'161","reported_seq":873,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-09-23T20:46:25.639475+0000","last_change":"2023-09-23T20:35:02.236398+0000","last_active":"2023-09-23T20:46:25.639475+0000","last_peered":"2023-09-23T20:46:25.639475+0000","last_clean":"2023-09-23T20:46:25.639475+0000","last_became_active":"2023-09-23T20:35:02.235988+0000","last_became_peered":"2023-09-23T20:35:02.235988+0000","last_unstale":"2023-09-23T20:46:25.639475+0000","last_undegraded":"2023-09-23T20:46:25.639475+0000","last_fullsized":"2023-09-23T20:46:25.639475+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:47.479012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:47.479012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:13.519648+0000","scrub_duration":0.0020136590000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":378,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-09-23T20:46:25.650814+0000","last_change":"2023-09-23T20:43:16.780478+0000","last_active":"2023-09-23T20:46:25.650814+0000","last_peered":"2023-09-23T20:46:25.650814+0000","last_clean":"2023-09-23T20:46:25.650814+0000","last_became_active":"2023-09-23T20:43:16.780183+0000","last_became_peered":"2023-09-23T20:43:16.780183+0000","last_unstale":"2023-09-23T20:46:25.650814+0000","last_undegraded":"2023-09-23T20:46:25.650814+0000","last_fullsized":"2023-09-23T20:46:25.650814+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:38:30.102066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":368,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-09-23T20:46:25.645859+0000","last_change":"2023-09-23T20:38:02.542034+0000","last_active":"2023-09-23T20:46:25.645859+0000","last_peered":"2023-09-23T20:46:25.645859+0000","last_clean":"2023-09-23T20:46:25.645859+0000","last_became_active":"2023-09-23T20:38:02.541575+0000","last_became_peered":"2023-09-23T20:38:02.541575+0000","last_unstale":"2023-09-23T20:46:25.645859+0000","last_undegraded":"2023-09-23T20:46:25.645859+0000","last_fullsized":"2023-09-23T20:46:25.645859+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:50:54.591529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"90'161","reported_seq":668,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-09-23T20:46:25.643682+0000","last_change":"2023-09-23T20:44:14.669973+0000","last_active":"2023-09-23T20:46:25.643682+0000","last_peered":"2023-09-23T20:46:25.643682+0000","last_clean":"2023-09-23T20:46:25.643682+0000","last_became_active":"2023-09-23T20:43:46.519592+0000","last_became_peered":"2023-09-23T20:43:46.519592+0000","last_unstale":"2023-09-23T20:46:25.643682+0000","last_undegraded":"2023-09-23T20:46:25.643682+0000","last_fullsized":"2023-09-23T20:46:25.643682+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:14.669831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:14.669831+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:56:28.076617+0000","scrub_duration":0.0083441460000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":569,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-09-23T20:46:25.650519+0000","last_change":"2023-09-23T20:32:33.270617+0000","last_active":"2023-09-23T20:46:25.650519+0000","last_peered":"2023-09-23T20:46:25.650519+0000","last_clean":"2023-09-23T20:46:25.650519+0000","last_became_active":"2023-09-23T20:32:33.227902+0000","last_became_peered":"2023-09-23T20:32:33.227902+0000","last_unstale":"2023-09-23T20:46:25.650519+0000","last_undegraded":"2023-09-23T20:46:25.650519+0000","last_fullsized":"2023-09-23T20:46:25.650519+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:30:26.847831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":360,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":57,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-09-23T20:46:28.565157+0000","last_change":"2023-09-23T20:44:10.611381+0000","last_active":"2023-09-23T20:46:28.565157+0000","last_peered":"2023-09-23T20:46:28.565157+0000","last_clean":"2023-09-23T20:46:28.565157+0000","last_became_active":"2023-09-23T20:44:10.611091+0000","last_became_peered":"2023-09-23T20:44:10.611091+0000","last_unstale":"2023-09-23T20:46:28.565157+0000","last_undegraded":"2023-09-23T20:46:28.565157+0000","last_fullsized":"2023-09-23T20:46:28.565157+0000","mapping_epoch":496,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":497,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:00:30.162478+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"90'161","reported_seq":668,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-09-23T20:46:25.642065+0000","last_change":"2023-09-23T20:36:02.548387+0000","last_active":"2023-09-23T20:46:25.642065+0000","last_peered":"2023-09-23T20:46:25.642065+0000","last_clean":"2023-09-23T20:46:25.642065+0000","last_became_active":"2023-09-23T20:35:26.644787+0000","last_became_peered":"2023-09-23T20:35:26.644787+0000","last_unstale":"2023-09-23T20:46:25.642065+0000","last_undegraded":"2023-09-23T20:46:25.642065+0000","last_fullsized":"2023-09-23T20:46:25.642065+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:02.548285+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:02.548285+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:37:15.216553+0000","scrub_duration":0.0015514459999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":289,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-09-23T20:46:25.637148+0000","last_change":"2023-09-23T20:38:02.560646+0000","last_active":"2023-09-23T20:46:25.637148+0000","last_peered":"2023-09-23T20:46:25.637148+0000","last_clean":"2023-09-23T20:46:25.637148+0000","last_became_active":"2023-09-23T20:38:02.539814+0000","last_became_peered":"2023-09-23T20:38:02.539814+0000","last_unstale":"2023-09-23T20:46:25.637148+0000","last_undegraded":"2023-09-23T20:46:25.637148+0000","last_fullsized":"2023-09-23T20:46:25.637148+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:00:09.435023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":269,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-09-23T20:46:25.636968+0000","last_change":"2023-09-23T20:38:02.579222+0000","last_active":"2023-09-23T20:46:25.636968+0000","last_peered":"2023-09-23T20:46:25.636968+0000","last_clean":"2023-09-23T20:46:25.636968+0000","last_became_active":"2023-09-23T20:38:02.578813+0000","last_became_peered":"2023-09-23T20:38:02.578813+0000","last_unstale":"2023-09-23T20:46:25.636968+0000","last_undegraded":"2023-09-23T20:46:25.636968+0000","last_fullsized":"2023-09-23T20:46:25.636968+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:57:31.080027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"90'161","reported_seq":796,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-09-23T20:46:25.644047+0000","last_change":"2023-09-23T20:43:46.544727+0000","last_active":"2023-09-23T20:46:25.644047+0000","last_peered":"2023-09-23T20:46:25.644047+0000","last_clean":"2023-09-23T20:46:25.644047+0000","last_became_active":"2023-09-23T20:43:46.543952+0000","last_became_peered":"2023-09-23T20:43:46.543952+0000","last_unstale":"2023-09-23T20:46:25.644047+0000","last_undegraded":"2023-09-23T20:46:25.644047+0000","last_fullsized":"2023-09-23T20:46:25.644047+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:43.578666+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:43.578666+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:11:16.611920+0000","scrub_duration":0.0021890379999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":568,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-09-23T20:46:25.649740+0000","last_change":"2023-09-23T20:32:33.242943+0000","last_active":"2023-09-23T20:46:25.649740+0000","last_peered":"2023-09-23T20:46:25.649740+0000","last_clean":"2023-09-23T20:46:25.649740+0000","last_became_active":"2023-09-23T20:32:33.225468+0000","last_became_peered":"2023-09-23T20:32:33.225468+0000","last_unstale":"2023-09-23T20:46:25.649740+0000","last_undegraded":"2023-09-23T20:46:25.649740+0000","last_fullsized":"2023-09-23T20:46:25.649740+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:40:47.867573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":288,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-09-23T20:46:25.646101+0000","last_change":"2023-09-23T20:45:01.875931+0000","last_active":"2023-09-23T20:46:25.646101+0000","last_peered":"2023-09-23T20:46:25.646101+0000","last_clean":"2023-09-23T20:46:25.646101+0000","last_became_active":"2023-09-23T20:45:01.872839+0000","last_became_peered":"2023-09-23T20:45:01.872839+0000","last_unstale":"2023-09-23T20:46:25.646101+0000","last_undegraded":"2023-09-23T20:46:25.646101+0000","last_fullsized":"2023-09-23T20:46:25.646101+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:15:09.584507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"90'161","reported_seq":822,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-09-23T20:46:25.646544+0000","last_change":"2023-09-23T20:44:20.041766+0000","last_active":"2023-09-23T20:46:25.646544+0000","last_peered":"2023-09-23T20:46:25.646544+0000","last_clean":"2023-09-23T20:46:25.646544+0000","last_became_active":"2023-09-23T20:43:16.765015+0000","last_became_peered":"2023-09-23T20:43:16.765015+0000","last_unstale":"2023-09-23T20:46:25.646544+0000","last_undegraded":"2023-09-23T20:46:25.646544+0000","last_fullsized":"2023-09-23T20:46:25.646544+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:20.041690+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:20.041690+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:33:30.150109+0000","scrub_duration":0.0026180330000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":498,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-09-23T20:46:25.645225+0000","last_change":"2023-09-23T20:43:46.521294+0000","last_active":"2023-09-23T20:46:25.645225+0000","last_peered":"2023-09-23T20:46:25.645225+0000","last_clean":"2023-09-23T20:46:25.645225+0000","last_became_active":"2023-09-23T20:43:46.521001+0000","last_became_peered":"2023-09-23T20:43:46.521001+0000","last_unstale":"2023-09-23T20:46:25.645225+0000","last_undegraded":"2023-09-23T20:46:25.645225+0000","last_fullsized":"2023-09-23T20:46:25.645225+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:01:35.126716+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":309,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-09-23T20:46:25.650629+0000","last_change":"2023-09-23T20:44:01.841565+0000","last_active":"2023-09-23T20:46:25.650629+0000","last_peered":"2023-09-23T20:46:25.650629+0000","last_clean":"2023-09-23T20:46:25.650629+0000","last_became_active":"2023-09-23T20:44:01.841255+0000","last_became_peered":"2023-09-23T20:44:01.841255+0000","last_unstale":"2023-09-23T20:46:25.650629+0000","last_undegraded":"2023-09-23T20:46:25.650629+0000","last_fullsized":"2023-09-23T20:46:25.650629+0000","mapping_epoch":491,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":492,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:18:44.817589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.13","version":"90'161","reported_seq":830,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-09-23T20:46:25.642991+0000","last_change":"2023-09-23T20:35:02.238923+0000","last_active":"2023-09-23T20:46:25.642991+0000","last_peered":"2023-09-23T20:46:25.642991+0000","last_clean":"2023-09-23T20:46:25.642991+0000","last_became_active":"2023-09-23T20:35:02.238293+0000","last_became_peered":"2023-09-23T20:35:02.238293+0000","last_unstale":"2023-09-23T20:46:25.642991+0000","last_undegraded":"2023-09-23T20:46:25.642991+0000","last_fullsized":"2023-09-23T20:46:25.642991+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:42.544308+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:42.544308+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:51:22.281932+0000","scrub_duration":0.0029958179999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":545,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-09-23T20:46:25.649768+0000","last_change":"2023-09-23T20:38:02.539774+0000","last_active":"2023-09-23T20:46:25.649768+0000","last_peered":"2023-09-23T20:46:25.649768+0000","last_clean":"2023-09-23T20:46:25.649768+0000","last_became_active":"2023-09-23T20:38:02.539436+0000","last_became_peered":"2023-09-23T20:38:02.539436+0000","last_unstale":"2023-09-23T20:46:25.649768+0000","last_undegraded":"2023-09-23T20:46:25.649768+0000","last_fullsized":"2023-09-23T20:46:25.649768+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:50:57.375803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":507,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-09-23T20:46:25.645056+0000","last_change":"2023-09-23T20:43:46.521735+0000","last_active":"2023-09-23T20:46:25.645056+0000","last_peered":"2023-09-23T20:46:25.645056+0000","last_clean":"2023-09-23T20:46:25.645056+0000","last_became_active":"2023-09-23T20:43:46.520762+0000","last_became_peered":"2023-09-23T20:43:46.520762+0000","last_unstale":"2023-09-23T20:46:25.645056+0000","last_undegraded":"2023-09-23T20:46:25.645056+0000","last_fullsized":"2023-09-23T20:46:25.645056+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:26:48.841930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"90'161","reported_seq":626,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-09-23T20:46:25.642100+0000","last_change":"2023-09-23T20:44:18.952808+0000","last_active":"2023-09-23T20:46:25.642100+0000","last_peered":"2023-09-23T20:46:25.642100+0000","last_clean":"2023-09-23T20:46:25.642100+0000","last_became_active":"2023-09-23T20:43:16.758652+0000","last_became_peered":"2023-09-23T20:43:16.758652+0000","last_unstale":"2023-09-23T20:46:25.642100+0000","last_undegraded":"2023-09-23T20:46:25.642100+0000","last_fullsized":"2023-09-23T20:46:25.642100+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:18.952693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:18.952693+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:12:19.496465+0000","scrub_duration":0.001699365,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":217,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-09-23T20:46:25.638237+0000","last_change":"2023-09-23T20:44:43.644290+0000","last_active":"2023-09-23T20:46:25.638237+0000","last_peered":"2023-09-23T20:46:25.638237+0000","last_clean":"2023-09-23T20:46:25.638237+0000","last_became_active":"2023-09-23T20:44:43.638313+0000","last_became_peered":"2023-09-23T20:44:43.638313+0000","last_unstale":"2023-09-23T20:46:25.638237+0000","last_undegraded":"2023-09-23T20:46:25.638237+0000","last_fullsized":"2023-09-23T20:46:25.638237+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:32:52.892603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":337,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-09-23T20:46:25.648965+0000","last_change":"2023-09-23T20:37:43.571083+0000","last_active":"2023-09-23T20:46:25.648965+0000","last_peered":"2023-09-23T20:46:25.648965+0000","last_clean":"2023-09-23T20:46:25.648965+0000","last_became_active":"2023-09-23T20:37:43.570306+0000","last_became_peered":"2023-09-23T20:37:43.570306+0000","last_unstale":"2023-09-23T20:46:25.648965+0000","last_undegraded":"2023-09-23T20:46:25.648965+0000","last_fullsized":"2023-09-23T20:46:25.648965+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:42:02.396718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.6","version":"542'5288","reported_seq":5308,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-09-23T20:46:41.445339+0000","last_change":"2023-09-23T20:45:39.488709+0000","last_active":"2023-09-23T20:46:41.445339+0000","last_peered":"2023-09-23T20:46:41.445339+0000","last_clean":"2023-09-23T20:46:41.445339+0000","last_became_active":"2023-09-23T20:45:39.488360+0000","last_became_peered":"2023-09-23T20:45:39.488360+0000","last_unstale":"2023-09-23T20:46:41.445339+0000","last_undegraded":"2023-09-23T20:46:41.445339+0000","last_fullsized":"2023-09-23T20:46:41.445339+0000","mapping_epoch":531,"log_start":"0'0","ondisk_log_start":"0'0","created":531,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":5288,"log_dups_size":0,"ondisk_log_size":5288,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:51:44.738010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":346554368,"num_objects":5288,"num_object_clones":0,"num_object_copies":15864,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5288,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5288,"num_write_kb":338432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.d","version":"90'161","reported_seq":834,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-09-23T20:46:25.650275+0000","last_change":"2023-09-23T20:34:59.496438+0000","last_active":"2023-09-23T20:46:25.650275+0000","last_peered":"2023-09-23T20:46:25.650275+0000","last_clean":"2023-09-23T20:46:25.650275+0000","last_became_active":"2023-09-23T20:33:38.970527+0000","last_became_peered":"2023-09-23T20:33:38.970527+0000","last_unstale":"2023-09-23T20:46:25.650275+0000","last_undegraded":"2023-09-23T20:46:25.650275+0000","last_fullsized":"2023-09-23T20:46:25.650275+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:34:59.496375+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:34:59.496375+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:46:18.244734+0000","scrub_duration":0.0017843189999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":545,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-09-23T20:46:25.651184+0000","last_change":"2023-09-23T20:32:33.283752+0000","last_active":"2023-09-23T20:46:25.651184+0000","last_peered":"2023-09-23T20:46:25.651184+0000","last_clean":"2023-09-23T20:46:25.651184+0000","last_became_active":"2023-09-23T20:32:33.230503+0000","last_became_peered":"2023-09-23T20:32:33.230503+0000","last_unstale":"2023-09-23T20:46:25.651184+0000","last_undegraded":"2023-09-23T20:46:25.651184+0000","last_fullsized":"2023-09-23T20:46:25.651184+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:29.084410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":356,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-09-23T20:46:25.646757+0000","last_change":"2023-09-23T20:43:16.765253+0000","last_active":"2023-09-23T20:46:25.646757+0000","last_peered":"2023-09-23T20:46:25.646757+0000","last_clean":"2023-09-23T20:46:25.646757+0000","last_became_active":"2023-09-23T20:43:16.764978+0000","last_became_peered":"2023-09-23T20:43:16.764978+0000","last_unstale":"2023-09-23T20:46:25.646757+0000","last_undegraded":"2023-09-23T20:46:25.646757+0000","last_fullsized":"2023-09-23T20:46:25.646757+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:07:23.539372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.7","version":"542'5124","reported_seq":5142,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-09-23T20:46:40.347790+0000","last_change":"2023-09-23T20:45:39.488542+0000","last_active":"2023-09-23T20:46:40.347790+0000","last_peered":"2023-09-23T20:46:40.347790+0000","last_clean":"2023-09-23T20:46:40.347790+0000","last_became_active":"2023-09-23T20:45:39.488259+0000","last_became_peered":"2023-09-23T20:45:39.488259+0000","last_unstale":"2023-09-23T20:46:40.347790+0000","last_undegraded":"2023-09-23T20:46:40.347790+0000","last_fullsized":"2023-09-23T20:46:40.347790+0000","mapping_epoch":531,"log_start":"0'0","ondisk_log_start":"0'0","created":531,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":5124,"log_dups_size":0,"ondisk_log_size":5124,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:39:05.833662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":335806464,"num_objects":5124,"num_object_clones":0,"num_object_copies":15372,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5124,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5124,"num_write_kb":327936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.c","version":"90'161","reported_seq":803,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-09-23T20:46:25.646114+0000","last_change":"2023-09-23T20:41:09.506481+0000","last_active":"2023-09-23T20:46:25.646114+0000","last_peered":"2023-09-23T20:46:25.646114+0000","last_clean":"2023-09-23T20:46:25.646114+0000","last_became_active":"2023-09-23T20:38:28.442229+0000","last_became_peered":"2023-09-23T20:38:28.442229+0000","last_unstale":"2023-09-23T20:46:25.646114+0000","last_undegraded":"2023-09-23T20:46:25.646114+0000","last_fullsized":"2023-09-23T20:46:25.646114+0000","mapping_epoch":411,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":412,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:41:09.506338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:41:09.506338+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:19:46.265011+0000","scrub_duration":0.01112002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":580,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-09-23T20:46:25.650888+0000","last_change":"2023-09-23T20:34:59.987981+0000","last_active":"2023-09-23T20:46:25.650888+0000","last_peered":"2023-09-23T20:46:25.650888+0000","last_clean":"2023-09-23T20:46:25.650888+0000","last_became_active":"2023-09-23T20:34:59.985368+0000","last_became_peered":"2023-09-23T20:34:59.985368+0000","last_unstale":"2023-09-23T20:46:25.650888+0000","last_undegraded":"2023-09-23T20:46:25.650888+0000","last_fullsized":"2023-09-23T20:46:25.650888+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:12:43.082493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":239,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-09-23T20:46:25.636444+0000","last_change":"2023-09-23T20:32:33.232142+0000","last_active":"2023-09-23T20:46:25.636444+0000","last_peered":"2023-09-23T20:46:25.636444+0000","last_clean":"2023-09-23T20:46:25.636444+0000","last_became_active":"2023-09-23T20:32:33.231683+0000","last_became_peered":"2023-09-23T20:32:33.231683+0000","last_unstale":"2023-09-23T20:46:25.636444+0000","last_undegraded":"2023-09-23T20:46:25.636444+0000","last_fullsized":"2023-09-23T20:46:25.636444+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:02:58.336348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"542'5171","reported_seq":5191,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-09-23T20:46:41.449675+0000","last_change":"2023-09-23T20:45:39.381562+0000","last_active":"2023-09-23T20:46:41.449675+0000","last_peered":"2023-09-23T20:46:41.449675+0000","last_clean":"2023-09-23T20:46:41.449675+0000","last_became_active":"2023-09-23T20:45:39.380887+0000","last_became_peered":"2023-09-23T20:45:39.380887+0000","last_unstale":"2023-09-23T20:46:41.449675+0000","last_undegraded":"2023-09-23T20:46:41.449675+0000","last_fullsized":"2023-09-23T20:46:41.449675+0000","mapping_epoch":531,"log_start":"0'0","ondisk_log_start":"0'0","created":531,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":5171,"log_dups_size":0,"ondisk_log_size":5171,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:41:52.967029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":338886656,"num_objects":5171,"num_object_clones":0,"num_object_copies":15513,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5171,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5171,"num_write_kb":330944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"90'161","reported_seq":697,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-09-23T20:46:25.640026+0000","last_change":"2023-09-23T20:38:40.491902+0000","last_active":"2023-09-23T20:46:25.640026+0000","last_peered":"2023-09-23T20:46:25.640026+0000","last_clean":"2023-09-23T20:46:25.640026+0000","last_became_active":"2023-09-23T20:38:40.491563+0000","last_became_peered":"2023-09-23T20:38:40.491563+0000","last_unstale":"2023-09-23T20:46:25.640026+0000","last_undegraded":"2023-09-23T20:46:25.640026+0000","last_fullsized":"2023-09-23T20:46:25.640026+0000","mapping_epoch":418,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":419,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:46.217489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:46.217489+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:39:25.008796+0000","scrub_duration":0.001456429,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":570,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-09-23T20:46:25.650369+0000","last_change":"2023-09-23T20:40:19.677655+0000","last_active":"2023-09-23T20:46:25.650369+0000","last_peered":"2023-09-23T20:46:25.650369+0000","last_clean":"2023-09-23T20:46:25.650369+0000","last_became_active":"2023-09-23T20:40:19.551189+0000","last_became_peered":"2023-09-23T20:40:19.551189+0000","last_unstale":"2023-09-23T20:46:25.650369+0000","last_undegraded":"2023-09-23T20:46:25.650369+0000","last_fullsized":"2023-09-23T20:46:25.650369+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:11:01.559576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":147,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-09-23T20:46:25.638371+0000","last_change":"2023-09-23T20:38:02.590613+0000","last_active":"2023-09-23T20:46:25.638371+0000","last_peered":"2023-09-23T20:46:25.638371+0000","last_clean":"2023-09-23T20:46:25.638371+0000","last_became_active":"2023-09-23T20:38:02.590284+0000","last_became_peered":"2023-09-23T20:38:02.590284+0000","last_unstale":"2023-09-23T20:46:25.638371+0000","last_undegraded":"2023-09-23T20:46:25.638371+0000","last_fullsized":"2023-09-23T20:46:25.638371+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:40:07.825054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.5","version":"542'5040","reported_seq":5060,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-09-23T20:46:38.115010+0000","last_change":"2023-09-23T20:45:39.380986+0000","last_active":"2023-09-23T20:46:38.115010+0000","last_peered":"2023-09-23T20:46:38.115010+0000","last_clean":"2023-09-23T20:46:38.115010+0000","last_became_active":"2023-09-23T20:45:39.380650+0000","last_became_peered":"2023-09-23T20:45:39.380650+0000","last_unstale":"2023-09-23T20:46:38.115010+0000","last_undegraded":"2023-09-23T20:46:38.115010+0000","last_fullsized":"2023-09-23T20:46:38.115010+0000","mapping_epoch":531,"log_start":"0'0","ondisk_log_start":"0'0","created":531,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":5040,"log_dups_size":0,"ondisk_log_size":5040,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:24:24.301239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":330301440,"num_objects":5040,"num_object_clones":0,"num_object_copies":15120,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5040,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5040,"num_write_kb":322560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.e","version":"90'161","reported_seq":642,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-09-23T20:46:25.637666+0000","last_change":"2023-09-23T20:36:00.322453+0000","last_active":"2023-09-23T20:46:25.637666+0000","last_peered":"2023-09-23T20:46:25.637666+0000","last_clean":"2023-09-23T20:46:25.637666+0000","last_became_active":"2023-09-23T20:35:02.233536+0000","last_became_peered":"2023-09-23T20:35:02.233536+0000","last_unstale":"2023-09-23T20:46:25.637666+0000","last_undegraded":"2023-09-23T20:46:25.637666+0000","last_fullsized":"2023-09-23T20:46:25.637666+0000","mapping_epoch":356,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":357,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:00.322381+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:00.322381+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:02:15.489069+0000","scrub_duration":0.0017321859999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":502,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-09-23T20:46:25.646341+0000","last_change":"2023-09-23T20:43:16.756437+0000","last_active":"2023-09-23T20:46:25.646341+0000","last_peered":"2023-09-23T20:46:25.646341+0000","last_clean":"2023-09-23T20:46:25.646341+0000","last_became_active":"2023-09-23T20:43:16.756193+0000","last_became_peered":"2023-09-23T20:43:16.756193+0000","last_unstale":"2023-09-23T20:46:25.646341+0000","last_undegraded":"2023-09-23T20:46:25.646341+0000","last_fullsized":"2023-09-23T20:46:25.646341+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:33:25.632781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":516,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-09-23T20:46:25.643818+0000","last_change":"2023-09-23T20:43:46.541524+0000","last_active":"2023-09-23T20:46:25.643818+0000","last_peered":"2023-09-23T20:46:25.643818+0000","last_clean":"2023-09-23T20:46:25.643818+0000","last_became_active":"2023-09-23T20:43:46.541051+0000","last_became_peered":"2023-09-23T20:43:46.541051+0000","last_unstale":"2023-09-23T20:46:25.643818+0000","last_undegraded":"2023-09-23T20:46:25.643818+0000","last_fullsized":"2023-09-23T20:46:25.643818+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:59:36.632592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"542'5211","reported_seq":5231,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-09-23T20:46:38.528667+0000","last_change":"2023-09-23T20:45:39.331236+0000","last_active":"2023-09-23T20:46:38.528667+0000","last_peered":"2023-09-23T20:46:38.528667+0000","last_clean":"2023-09-23T20:46:38.528667+0000","last_became_active":"2023-09-23T20:45:39.330923+0000","last_became_peered":"2023-09-23T20:45:39.330923+0000","last_unstale":"2023-09-23T20:46:38.528667+0000","last_undegraded":"2023-09-23T20:46:38.528667+0000","last_fullsized":"2023-09-23T20:46:38.528667+0000","mapping_epoch":531,"log_start":"0'0","ondisk_log_start":"0'0","created":531,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":5211,"log_dups_size":0,"ondisk_log_size":5211,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:48:20.711262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":341508096,"num_objects":5211,"num_object_clones":0,"num_object_copies":15633,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5211,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5211,"num_write_kb":333504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":947,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-09-23T20:46:25.649681+0000","last_change":"2023-09-23T20:35:02.523308+0000","last_active":"2023-09-23T20:46:25.649681+0000","last_peered":"2023-09-23T20:46:25.649681+0000","last_clean":"2023-09-23T20:46:25.649681+0000","last_became_active":"2023-09-23T20:33:38.979989+0000","last_became_peered":"2023-09-23T20:33:38.979989+0000","last_unstale":"2023-09-23T20:46:25.649681+0000","last_undegraded":"2023-09-23T20:46:25.649681+0000","last_fullsized":"2023-09-23T20:46:25.649681+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:35:02.523203+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:35:02.523203+0000","last_clean_scrub_stamp":"2023-09-23T20:35:02.523203+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:02:41.472958+0000","scrub_duration":0.0085927989999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":363,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-09-23T20:46:25.645726+0000","last_change":"2023-09-23T20:37:43.583731+0000","last_active":"2023-09-23T20:46:25.645726+0000","last_peered":"2023-09-23T20:46:25.645726+0000","last_clean":"2023-09-23T20:46:25.645726+0000","last_became_active":"2023-09-23T20:37:43.583066+0000","last_became_peered":"2023-09-23T20:37:43.583066+0000","last_unstale":"2023-09-23T20:46:25.645726+0000","last_undegraded":"2023-09-23T20:46:25.645726+0000","last_fullsized":"2023-09-23T20:46:25.645726+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:03:45.211900+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":259,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-09-23T20:46:25.637010+0000","last_change":"2023-09-23T20:38:02.579814+0000","last_active":"2023-09-23T20:46:25.637010+0000","last_peered":"2023-09-23T20:46:25.637010+0000","last_clean":"2023-09-23T20:46:25.637010+0000","last_became_active":"2023-09-23T20:38:02.579253+0000","last_became_peered":"2023-09-23T20:38:02.579253+0000","last_unstale":"2023-09-23T20:46:25.637010+0000","last_undegraded":"2023-09-23T20:46:25.637010+0000","last_fullsized":"2023-09-23T20:46:25.637010+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:05:07.298198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"542'5177","reported_seq":5197,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-09-23T20:46:38.095349+0000","last_change":"2023-09-23T20:45:39.488901+0000","last_active":"2023-09-23T20:46:38.095349+0000","last_peered":"2023-09-23T20:46:38.095349+0000","last_clean":"2023-09-23T20:46:38.095349+0000","last_became_active":"2023-09-23T20:45:39.488499+0000","last_became_peered":"2023-09-23T20:45:39.488499+0000","last_unstale":"2023-09-23T20:46:38.095349+0000","last_undegraded":"2023-09-23T20:46:38.095349+0000","last_fullsized":"2023-09-23T20:46:38.095349+0000","mapping_epoch":531,"log_start":"0'0","ondisk_log_start":"0'0","created":531,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":5177,"log_dups_size":0,"ondisk_log_size":5177,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:25:10.137770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":339279872,"num_objects":5177,"num_object_clones":0,"num_object_copies":15531,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5177,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5177,"num_write_kb":331328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":657,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-09-23T20:46:25.639991+0000","last_change":"2023-09-23T20:44:46.725779+0000","last_active":"2023-09-23T20:46:25.639991+0000","last_peered":"2023-09-23T20:46:25.639991+0000","last_clean":"2023-09-23T20:46:25.639991+0000","last_became_active":"2023-09-23T20:44:46.725433+0000","last_became_peered":"2023-09-23T20:44:46.725433+0000","last_unstale":"2023-09-23T20:46:25.639991+0000","last_undegraded":"2023-09-23T20:46:25.639991+0000","last_fullsized":"2023-09-23T20:46:25.639991+0000","mapping_epoch":516,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":517,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:42:45.492220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:42:45.492220+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:28:35.202407+0000","scrub_duration":0.015259412,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":147,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-09-23T20:46:25.638453+0000","last_change":"2023-09-23T20:38:02.578703+0000","last_active":"2023-09-23T20:46:25.638453+0000","last_peered":"2023-09-23T20:46:25.638453+0000","last_clean":"2023-09-23T20:46:25.638453+0000","last_became_active":"2023-09-23T20:38:02.578414+0000","last_became_peered":"2023-09-23T20:38:02.578414+0000","last_unstale":"2023-09-23T20:46:25.638453+0000","last_undegraded":"2023-09-23T20:46:25.638453+0000","last_fullsized":"2023-09-23T20:46:25.638453+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:43:59.779295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":208,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-09-23T20:46:25.636973+0000","last_change":"2023-09-23T20:37:31.296606+0000","last_active":"2023-09-23T20:46:25.636973+0000","last_peered":"2023-09-23T20:46:25.636973+0000","last_clean":"2023-09-23T20:46:25.636973+0000","last_became_active":"2023-09-23T20:37:31.295924+0000","last_became_peered":"2023-09-23T20:37:31.295924+0000","last_unstale":"2023-09-23T20:46:25.636973+0000","last_undegraded":"2023-09-23T20:46:25.636973+0000","last_fullsized":"2023-09-23T20:46:25.636973+0000","mapping_epoch":391,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":392,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:27:50.948510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.0","version":"542'5206","reported_seq":5225,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-09-23T20:46:40.318397+0000","last_change":"2023-09-23T20:45:39.331645+0000","last_active":"2023-09-23T20:46:40.318397+0000","last_peered":"2023-09-23T20:46:40.318397+0000","last_clean":"2023-09-23T20:46:40.318397+0000","last_became_active":"2023-09-23T20:45:39.331293+0000","last_became_peered":"2023-09-23T20:45:39.331293+0000","last_unstale":"2023-09-23T20:46:40.318397+0000","last_undegraded":"2023-09-23T20:46:40.318397+0000","last_fullsized":"2023-09-23T20:46:40.318397+0000","mapping_epoch":531,"log_start":"0'0","ondisk_log_start":"0'0","created":531,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":5206,"log_dups_size":0,"ondisk_log_size":5206,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:29:42.789115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":341180416,"num_objects":5206,"num_object_clones":0,"num_object_copies":15618,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5206,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5206,"num_write_kb":333184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.b","version":"90'161","reported_seq":846,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-09-23T20:46:25.643042+0000","last_change":"2023-09-23T20:40:22.445808+0000","last_active":"2023-09-23T20:46:25.643042+0000","last_peered":"2023-09-23T20:46:25.643042+0000","last_clean":"2023-09-23T20:46:25.643042+0000","last_became_active":"2023-09-23T20:40:22.445456+0000","last_became_peered":"2023-09-23T20:40:22.445456+0000","last_unstale":"2023-09-23T20:46:25.643042+0000","last_undegraded":"2023-09-23T20:46:25.643042+0000","last_fullsized":"2023-09-23T20:46:25.643042+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:05.595042+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:05.595042+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:39:23.678154+0000","scrub_duration":0.007271675,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":198,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-09-23T20:46:25.636202+0000","last_change":"2023-09-23T20:34:59.983122+0000","last_active":"2023-09-23T20:46:25.636202+0000","last_peered":"2023-09-23T20:46:25.636202+0000","last_clean":"2023-09-23T20:46:25.636202+0000","last_became_active":"2023-09-23T20:34:59.982809+0000","last_became_peered":"2023-09-23T20:34:59.982809+0000","last_unstale":"2023-09-23T20:46:25.636202+0000","last_undegraded":"2023-09-23T20:46:25.636202+0000","last_fullsized":"2023-09-23T20:46:25.636202+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:49:40.839723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":279,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-09-23T20:46:25.637956+0000","last_change":"2023-09-23T20:44:10.606995+0000","last_active":"2023-09-23T20:46:25.637956+0000","last_peered":"2023-09-23T20:46:25.637956+0000","last_clean":"2023-09-23T20:46:25.637956+0000","last_became_active":"2023-09-23T20:44:10.606718+0000","last_became_peered":"2023-09-23T20:44:10.606718+0000","last_unstale":"2023-09-23T20:46:25.637956+0000","last_undegraded":"2023-09-23T20:46:25.637956+0000","last_fullsized":"2023-09-23T20:46:25.637956+0000","mapping_epoch":496,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":497,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:13:12.496245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"542'5212","reported_seq":5231,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-09-23T20:46:40.725108+0000","last_change":"2023-09-23T20:45:39.333840+0000","last_active":"2023-09-23T20:46:40.725108+0000","last_peered":"2023-09-23T20:46:40.725108+0000","last_clean":"2023-09-23T20:46:40.725108+0000","last_became_active":"2023-09-23T20:45:39.333454+0000","last_became_peered":"2023-09-23T20:45:39.333454+0000","last_unstale":"2023-09-23T20:46:40.725108+0000","last_undegraded":"2023-09-23T20:46:40.725108+0000","last_fullsized":"2023-09-23T20:46:40.725108+0000","mapping_epoch":531,"log_start":"0'0","ondisk_log_start":"0'0","created":531,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":5212,"log_dups_size":0,"ondisk_log_size":5212,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:23:21.002722+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":341573632,"num_objects":5212,"num_object_clones":0,"num_object_copies":15636,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5212,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5212,"num_write_kb":333568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":568,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-09-23T20:46:25.639906+0000","last_change":"2023-09-23T20:43:51.693618+0000","last_active":"2023-09-23T20:46:25.639906+0000","last_peered":"2023-09-23T20:46:25.639906+0000","last_clean":"2023-09-23T20:46:25.639906+0000","last_became_active":"2023-09-23T20:43:16.765186+0000","last_became_peered":"2023-09-23T20:43:16.765186+0000","last_unstale":"2023-09-23T20:46:25.639906+0000","last_undegraded":"2023-09-23T20:46:25.639906+0000","last_fullsized":"2023-09-23T20:46:25.639906+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:43:51.693459+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:43:51.693459+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:33:30.208355+0000","scrub_duration":0.0016145269999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":338,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-09-23T20:46:25.645469+0000","last_change":"2023-09-23T20:37:43.570610+0000","last_active":"2023-09-23T20:46:25.645469+0000","last_peered":"2023-09-23T20:46:25.645469+0000","last_clean":"2023-09-23T20:46:25.645469+0000","last_became_active":"2023-09-23T20:37:43.570216+0000","last_became_peered":"2023-09-23T20:37:43.570216+0000","last_unstale":"2023-09-23T20:46:25.645469+0000","last_undegraded":"2023-09-23T20:46:25.645469+0000","last_fullsized":"2023-09-23T20:46:25.645469+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:39:26.555329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":368,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-09-23T20:46:25.642294+0000","last_change":"2023-09-23T20:32:33.230582+0000","last_active":"2023-09-23T20:46:25.642294+0000","last_peered":"2023-09-23T20:46:25.642294+0000","last_clean":"2023-09-23T20:46:25.642294+0000","last_became_active":"2023-09-23T20:32:33.230257+0000","last_became_peered":"2023-09-23T20:32:33.230257+0000","last_unstale":"2023-09-23T20:46:25.642294+0000","last_undegraded":"2023-09-23T20:46:25.642294+0000","last_fullsized":"2023-09-23T20:46:25.642294+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:11:45.333096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"542'5276","reported_seq":5294,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-09-23T20:46:40.323736+0000","last_change":"2023-09-23T20:45:39.332694+0000","last_active":"2023-09-23T20:46:40.323736+0000","last_peered":"2023-09-23T20:46:40.323736+0000","last_clean":"2023-09-23T20:46:40.323736+0000","last_became_active":"2023-09-23T20:45:39.332422+0000","last_became_peered":"2023-09-23T20:45:39.332422+0000","last_unstale":"2023-09-23T20:46:40.323736+0000","last_undegraded":"2023-09-23T20:46:40.323736+0000","last_fullsized":"2023-09-23T20:46:40.323736+0000","mapping_epoch":531,"log_start":"0'0","ondisk_log_start":"0'0","created":531,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":5276,"log_dups_size":0,"ondisk_log_size":5276,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:50:39.768517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":345767936,"num_objects":5276,"num_object_clones":0,"num_object_copies":15828,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5276,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5276,"num_write_kb":337664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":730,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-09-23T20:46:25.640014+0000","last_change":"2023-09-23T20:41:07.424609+0000","last_active":"2023-09-23T20:46:25.640014+0000","last_peered":"2023-09-23T20:46:25.640014+0000","last_clean":"2023-09-23T20:46:25.640014+0000","last_became_active":"2023-09-23T20:38:05.622261+0000","last_became_peered":"2023-09-23T20:38:05.622261+0000","last_unstale":"2023-09-23T20:46:25.640014+0000","last_undegraded":"2023-09-23T20:46:25.640014+0000","last_fullsized":"2023-09-23T20:46:25.640014+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:41:07.424492+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:41:07.424492+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:45:21.689123+0000","scrub_duration":0.0018960959999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"86'119","reported_seq":853,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-09-23T20:46:33.565981+0000","last_change":"2023-09-23T20:29:45.521257+0000","last_active":"2023-09-23T20:46:33.565981+0000","last_peered":"2023-09-23T20:46:33.565981+0000","last_clean":"2023-09-23T20:46:33.565981+0000","last_became_active":"2023-09-23T20:29:45.520546+0000","last_became_peered":"2023-09-23T20:29:45.520546+0000","last_unstale":"2023-09-23T20:46:33.565981+0000","last_undegraded":"2023-09-23T20:46:33.565981+0000","last_fullsized":"2023-09-23T20:46:33.565981+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":119,"log_dups_size":0,"ondisk_log_size":119,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:58:56.085777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032705499999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":523,"num_read_kb":3503,"num_write":136,"num_write_kb":3128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":335,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-09-23T20:46:25.646387+0000","last_change":"2023-09-23T20:43:16.756494+0000","last_active":"2023-09-23T20:46:25.646387+0000","last_peered":"2023-09-23T20:46:25.646387+0000","last_clean":"2023-09-23T20:46:25.646387+0000","last_became_active":"2023-09-23T20:43:16.756131+0000","last_became_peered":"2023-09-23T20:43:16.756131+0000","last_unstale":"2023-09-23T20:46:25.646387+0000","last_undegraded":"2023-09-23T20:46:25.646387+0000","last_fullsized":"2023-09-23T20:46:25.646387+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:10:19.219533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":500,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-09-23T20:46:28.565180+0000","last_change":"2023-09-23T20:40:19.530942+0000","last_active":"2023-09-23T20:46:28.565180+0000","last_peered":"2023-09-23T20:46:28.565180+0000","last_clean":"2023-09-23T20:46:28.565180+0000","last_became_active":"2023-09-23T20:40:19.530617+0000","last_became_peered":"2023-09-23T20:40:19.530617+0000","last_unstale":"2023-09-23T20:46:28.565180+0000","last_undegraded":"2023-09-23T20:46:28.565180+0000","last_fullsized":"2023-09-23T20:46:28.565180+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:24:09.231934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"542'5073","reported_seq":5092,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-09-23T20:46:40.297730+0000","last_change":"2023-09-23T20:45:39.489110+0000","last_active":"2023-09-23T20:46:40.297730+0000","last_peered":"2023-09-23T20:46:40.297730+0000","last_clean":"2023-09-23T20:46:40.297730+0000","last_became_active":"2023-09-23T20:45:39.488817+0000","last_became_peered":"2023-09-23T20:45:39.488817+0000","last_unstale":"2023-09-23T20:46:40.297730+0000","last_undegraded":"2023-09-23T20:46:40.297730+0000","last_fullsized":"2023-09-23T20:46:40.297730+0000","mapping_epoch":531,"log_start":"0'0","ondisk_log_start":"0'0","created":531,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":5073,"log_dups_size":0,"ondisk_log_size":5073,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:07:21.467971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":332464128,"num_objects":5073,"num_object_clones":0,"num_object_copies":15219,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5073,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5073,"num_write_kb":324672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":588,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-09-23T20:46:25.644312+0000","last_change":"2023-09-23T20:44:15.656984+0000","last_active":"2023-09-23T20:46:25.644312+0000","last_peered":"2023-09-23T20:46:25.644312+0000","last_clean":"2023-09-23T20:46:25.644312+0000","last_became_active":"2023-09-23T20:43:46.512210+0000","last_became_peered":"2023-09-23T20:43:46.512210+0000","last_unstale":"2023-09-23T20:46:25.644312+0000","last_undegraded":"2023-09-23T20:46:25.644312+0000","last_fullsized":"2023-09-23T20:46:25.644312+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:15.656880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:15.656880+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:37:56.453466+0000","scrub_duration":0.0016711180000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"86'106","reported_seq":793,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-09-23T20:46:25.641972+0000","last_change":"2023-09-23T20:43:16.759138+0000","last_active":"2023-09-23T20:46:25.641972+0000","last_peered":"2023-09-23T20:46:25.641972+0000","last_clean":"2023-09-23T20:46:25.641972+0000","last_became_active":"2023-09-23T20:43:16.758015+0000","last_became_peered":"2023-09-23T20:43:16.758015+0000","last_unstale":"2023-09-23T20:46:25.641972+0000","last_undegraded":"2023-09-23T20:46:25.641972+0000","last_fullsized":"2023-09-23T20:46:25.641972+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":106,"log_dups_size":0,"ondisk_log_size":106,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:08:02.852472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00099302599999999993,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1094,"num_read_kb":1678,"num_write":100,"num_write_kb":404,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":259,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-09-23T20:46:25.636891+0000","last_change":"2023-09-23T20:38:02.552008+0000","last_active":"2023-09-23T20:46:25.636891+0000","last_peered":"2023-09-23T20:46:25.636891+0000","last_clean":"2023-09-23T20:46:25.636891+0000","last_became_active":"2023-09-23T20:38:02.551684+0000","last_became_peered":"2023-09-23T20:38:02.551684+0000","last_unstale":"2023-09-23T20:46:25.636891+0000","last_undegraded":"2023-09-23T20:46:25.636891+0000","last_fullsized":"2023-09-23T20:46:25.636891+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:40:37.147538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":506,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-09-23T20:46:33.565961+0000","last_change":"2023-09-23T20:29:45.533406+0000","last_active":"2023-09-23T20:46:33.565961+0000","last_peered":"2023-09-23T20:46:33.565961+0000","last_clean":"2023-09-23T20:46:33.565961+0000","last_became_active":"2023-09-23T20:29:45.526001+0000","last_became_peered":"2023-09-23T20:29:45.526001+0000","last_unstale":"2023-09-23T20:46:33.565961+0000","last_undegraded":"2023-09-23T20:46:33.565961+0000","last_fullsized":"2023-09-23T20:46:33.565961+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:43.787251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.9","version":"542'5121","reported_seq":5141,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-09-23T20:46:38.093903+0000","last_change":"2023-09-23T20:45:39.488721+0000","last_active":"2023-09-23T20:46:38.093903+0000","last_peered":"2023-09-23T20:46:38.093903+0000","last_clean":"2023-09-23T20:46:38.093903+0000","last_became_active":"2023-09-23T20:45:39.488320+0000","last_became_peered":"2023-09-23T20:45:39.488320+0000","last_unstale":"2023-09-23T20:46:38.093903+0000","last_undegraded":"2023-09-23T20:46:38.093903+0000","last_fullsized":"2023-09-23T20:46:38.093903+0000","mapping_epoch":531,"log_start":"0'0","ondisk_log_start":"0'0","created":531,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":5121,"log_dups_size":0,"ondisk_log_size":5121,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:46:30.536848+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":335609856,"num_objects":5121,"num_object_clones":0,"num_object_copies":15363,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5121,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5121,"num_write_kb":327744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":505,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-09-23T20:46:25.649796+0000","last_change":"2023-09-23T20:44:26.978243+0000","last_active":"2023-09-23T20:46:25.649796+0000","last_peered":"2023-09-23T20:46:25.649796+0000","last_clean":"2023-09-23T20:46:25.649796+0000","last_became_active":"2023-09-23T20:44:18.638989+0000","last_became_peered":"2023-09-23T20:44:18.638989+0000","last_unstale":"2023-09-23T20:46:25.649796+0000","last_undegraded":"2023-09-23T20:46:25.649796+0000","last_fullsized":"2023-09-23T20:46:25.649796+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:26.978179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:26.978179+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:25:38.977011+0000","scrub_duration":0.0013962829999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"86'130","reported_seq":861,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-09-23T20:46:33.125325+0000","last_change":"2023-09-23T20:25:39.637458+0000","last_active":"2023-09-23T20:46:33.125325+0000","last_peered":"2023-09-23T20:46:33.125325+0000","last_clean":"2023-09-23T20:46:33.125325+0000","last_became_active":"2023-09-23T20:25:39.636773+0000","last_became_peered":"2023-09-23T20:25:39.636773+0000","last_unstale":"2023-09-23T20:46:33.125325+0000","last_undegraded":"2023-09-23T20:46:33.125325+0000","last_fullsized":"2023-09-23T20:46:33.125325+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:00:32.810564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011300049999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":512,"num_read_kb":1820,"num_write":133,"num_write_kb":1347,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":514,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-09-23T20:46:25.642585+0000","last_change":"2023-09-23T20:38:02.591335+0000","last_active":"2023-09-23T20:46:25.642585+0000","last_peered":"2023-09-23T20:46:25.642585+0000","last_clean":"2023-09-23T20:46:25.642585+0000","last_became_active":"2023-09-23T20:38:02.591015+0000","last_became_peered":"2023-09-23T20:38:02.591015+0000","last_unstale":"2023-09-23T20:46:25.642585+0000","last_undegraded":"2023-09-23T20:46:25.642585+0000","last_fullsized":"2023-09-23T20:46:25.642585+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:55:32.272195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":57,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-09-23T20:46:25.642228+0000","last_change":"2023-09-23T20:44:10.555583+0000","last_active":"2023-09-23T20:46:25.642228+0000","last_peered":"2023-09-23T20:46:25.642228+0000","last_clean":"2023-09-23T20:46:25.642228+0000","last_became_active":"2023-09-23T20:44:10.555281+0000","last_became_peered":"2023-09-23T20:44:10.555281+0000","last_unstale":"2023-09-23T20:46:25.642228+0000","last_undegraded":"2023-09-23T20:46:25.642228+0000","last_fullsized":"2023-09-23T20:46:25.642228+0000","mapping_epoch":496,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":497,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:43:11.942147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"542'5254","reported_seq":5273,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-09-23T20:46:40.337360+0000","last_change":"2023-09-23T20:45:39.331096+0000","last_active":"2023-09-23T20:46:40.337360+0000","last_peered":"2023-09-23T20:46:40.337360+0000","last_clean":"2023-09-23T20:46:40.337360+0000","last_became_active":"2023-09-23T20:45:39.330752+0000","last_became_peered":"2023-09-23T20:45:39.330752+0000","last_unstale":"2023-09-23T20:46:40.337360+0000","last_undegraded":"2023-09-23T20:46:40.337360+0000","last_fullsized":"2023-09-23T20:46:40.337360+0000","mapping_epoch":531,"log_start":"0'0","ondisk_log_start":"0'0","created":531,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":5254,"log_dups_size":0,"ondisk_log_size":5254,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:39:01.627762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":344326144,"num_objects":5254,"num_object_clones":0,"num_object_copies":15762,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5254,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5254,"num_write_kb":336256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"86'194","reported_seq":899,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-09-23T20:46:25.640286+0000","last_change":"2023-09-23T20:43:16.765865+0000","last_active":"2023-09-23T20:46:25.640286+0000","last_peered":"2023-09-23T20:46:25.640286+0000","last_clean":"2023-09-23T20:46:25.640286+0000","last_became_active":"2023-09-23T20:43:16.765289+0000","last_became_peered":"2023-09-23T20:43:16.765289+0000","last_unstale":"2023-09-23T20:46:25.640286+0000","last_undegraded":"2023-09-23T20:46:25.640286+0000","last_fullsized":"2023-09-23T20:46:25.640286+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":194,"log_dups_size":0,"ondisk_log_size":194,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:15:16.461900+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00037995999999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1024,"num_read_kb":1200,"num_write":173,"num_write_kb":1276,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":1004,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-09-23T20:46:25.646681+0000","last_change":"2023-09-23T20:38:05.602508+0000","last_active":"2023-09-23T20:46:25.646681+0000","last_peered":"2023-09-23T20:46:25.646681+0000","last_clean":"2023-09-23T20:46:25.646681+0000","last_became_active":"2023-09-23T20:38:05.602129+0000","last_became_peered":"2023-09-23T20:38:05.602129+0000","last_unstale":"2023-09-23T20:46:25.646681+0000","last_undegraded":"2023-09-23T20:46:25.646681+0000","last_fullsized":"2023-09-23T20:46:25.646681+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:45.343742+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:45.343742+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:48.910343+0000","scrub_duration":0.001475802,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":117,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-09-23T20:46:25.638459+0000","last_change":"2023-09-23T20:43:16.758213+0000","last_active":"2023-09-23T20:46:25.638459+0000","last_peered":"2023-09-23T20:46:25.638459+0000","last_clean":"2023-09-23T20:46:25.638459+0000","last_became_active":"2023-09-23T20:43:16.757869+0000","last_became_peered":"2023-09-23T20:43:16.757869+0000","last_unstale":"2023-09-23T20:46:25.638459+0000","last_undegraded":"2023-09-23T20:46:25.638459+0000","last_fullsized":"2023-09-23T20:46:25.638459+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:43:10.811749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":147,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-09-23T20:46:25.638738+0000","last_change":"2023-09-23T20:38:02.573200+0000","last_active":"2023-09-23T20:46:25.638738+0000","last_peered":"2023-09-23T20:46:25.638738+0000","last_clean":"2023-09-23T20:46:25.638738+0000","last_became_active":"2023-09-23T20:38:02.572945+0000","last_became_peered":"2023-09-23T20:38:02.572945+0000","last_unstale":"2023-09-23T20:46:25.638738+0000","last_undegraded":"2023-09-23T20:46:25.638738+0000","last_fullsized":"2023-09-23T20:46:25.638738+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:57.246533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.8","version":"542'5210","reported_seq":5230,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-09-23T20:46:41.468001+0000","last_change":"2023-09-23T20:45:39.332075+0000","last_active":"2023-09-23T20:46:41.468001+0000","last_peered":"2023-09-23T20:46:41.468001+0000","last_clean":"2023-09-23T20:46:41.468001+0000","last_became_active":"2023-09-23T20:45:39.331755+0000","last_became_peered":"2023-09-23T20:45:39.331755+0000","last_unstale":"2023-09-23T20:46:41.468001+0000","last_undegraded":"2023-09-23T20:46:41.468001+0000","last_fullsized":"2023-09-23T20:46:41.468001+0000","mapping_epoch":531,"log_start":"0'0","ondisk_log_start":"0'0","created":531,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":5210,"log_dups_size":0,"ondisk_log_size":5210,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:11:17.895002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":341442560,"num_objects":5210,"num_object_clones":0,"num_object_copies":15630,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5210,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5210,"num_write_kb":333440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":875,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-09-23T20:46:28.565231+0000","last_change":"2023-09-23T20:38:05.615711+0000","last_active":"2023-09-23T20:46:28.565231+0000","last_peered":"2023-09-23T20:46:28.565231+0000","last_clean":"2023-09-23T20:46:28.565231+0000","last_became_active":"2023-09-23T20:38:05.615390+0000","last_became_peered":"2023-09-23T20:38:05.615390+0000","last_unstale":"2023-09-23T20:46:28.565231+0000","last_undegraded":"2023-09-23T20:46:28.565231+0000","last_fullsized":"2023-09-23T20:46:28.565231+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:43.582146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:43.582146+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:45:11.917871+0000","scrub_duration":0.0016219730000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"86'80","reported_seq":794,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-09-23T20:46:25.646632+0000","last_change":"2023-09-23T20:29:45.536796+0000","last_active":"2023-09-23T20:46:25.646632+0000","last_peered":"2023-09-23T20:46:25.646632+0000","last_clean":"2023-09-23T20:46:25.646632+0000","last_became_active":"2023-09-23T20:29:45.536439+0000","last_became_peered":"2023-09-23T20:29:45.536439+0000","last_unstale":"2023-09-23T20:46:25.646632+0000","last_undegraded":"2023-09-23T20:46:25.646632+0000","last_fullsized":"2023-09-23T20:46:25.646632+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":80,"log_dups_size":0,"ondisk_log_size":80,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:30:57.793132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035115,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":395,"num_read_kb":1370,"num_write":66,"num_write_kb":1061,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":535,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-09-23T20:46:25.644237+0000","last_change":"2023-09-23T20:43:46.544563+0000","last_active":"2023-09-23T20:46:25.644237+0000","last_peered":"2023-09-23T20:46:25.644237+0000","last_clean":"2023-09-23T20:46:25.644237+0000","last_became_active":"2023-09-23T20:43:46.543958+0000","last_became_peered":"2023-09-23T20:43:46.543958+0000","last_unstale":"2023-09-23T20:46:25.644237+0000","last_undegraded":"2023-09-23T20:46:25.644237+0000","last_fullsized":"2023-09-23T20:46:25.644237+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:46:36.921752+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":530,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-09-23T20:46:25.644453+0000","last_change":"2023-09-23T20:43:46.520386+0000","last_active":"2023-09-23T20:46:25.644453+0000","last_peered":"2023-09-23T20:46:25.644453+0000","last_clean":"2023-09-23T20:46:25.644453+0000","last_became_active":"2023-09-23T20:43:46.519757+0000","last_became_peered":"2023-09-23T20:43:46.519757+0000","last_unstale":"2023-09-23T20:46:25.644453+0000","last_undegraded":"2023-09-23T20:46:25.644453+0000","last_fullsized":"2023-09-23T20:46:25.644453+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:23:14.662734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"542'5096","reported_seq":5115,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-09-23T20:46:40.315993+0000","last_change":"2023-09-23T20:45:39.331637+0000","last_active":"2023-09-23T20:46:40.315993+0000","last_peered":"2023-09-23T20:46:40.315993+0000","last_clean":"2023-09-23T20:46:40.315993+0000","last_became_active":"2023-09-23T20:45:39.331378+0000","last_became_peered":"2023-09-23T20:45:39.331378+0000","last_unstale":"2023-09-23T20:46:40.315993+0000","last_undegraded":"2023-09-23T20:46:40.315993+0000","last_fullsized":"2023-09-23T20:46:40.315993+0000","mapping_epoch":531,"log_start":"0'0","ondisk_log_start":"0'0","created":531,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":5096,"log_dups_size":0,"ondisk_log_size":5096,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:43:34.423232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":333971456,"num_objects":5096,"num_object_clones":0,"num_object_copies":15288,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5096,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5096,"num_write_kb":326144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":653,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-09-23T20:46:25.638762+0000","last_change":"2023-09-23T20:41:08.380084+0000","last_active":"2023-09-23T20:46:25.638762+0000","last_peered":"2023-09-23T20:46:25.638762+0000","last_clean":"2023-09-23T20:46:25.638762+0000","last_became_active":"2023-09-23T20:38:05.610881+0000","last_became_peered":"2023-09-23T20:38:05.610881+0000","last_unstale":"2023-09-23T20:46:25.638762+0000","last_undegraded":"2023-09-23T20:46:25.638762+0000","last_fullsized":"2023-09-23T20:46:25.638762+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:41:08.379959+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:41:08.379959+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:19:39.547923+0000","scrub_duration":0.0019158420000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"86'102","reported_seq":1136,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-09-23T20:46:25.650443+0000","last_change":"2023-09-23T20:32:36.314672+0000","last_active":"2023-09-23T20:46:25.650443+0000","last_peered":"2023-09-23T20:46:25.650443+0000","last_clean":"2023-09-23T20:46:25.650443+0000","last_became_active":"2023-09-23T20:32:36.314231+0000","last_became_peered":"2023-09-23T20:32:36.314231+0000","last_unstale":"2023-09-23T20:46:25.650443+0000","last_undegraded":"2023-09-23T20:46:25.650443+0000","last_fullsized":"2023-09-23T20:46:25.650443+0000","mapping_epoch":315,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":102,"log_dups_size":0,"ondisk_log_size":102,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:11:40.514709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030656500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":963,"num_read_kb":1234,"num_write":60,"num_write_kb":225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":490,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-09-23T20:46:33.566000+0000","last_change":"2023-09-23T20:40:19.544384+0000","last_active":"2023-09-23T20:46:33.566000+0000","last_peered":"2023-09-23T20:46:33.566000+0000","last_clean":"2023-09-23T20:46:33.566000+0000","last_became_active":"2023-09-23T20:40:19.544079+0000","last_became_peered":"2023-09-23T20:40:19.544079+0000","last_unstale":"2023-09-23T20:46:33.566000+0000","last_undegraded":"2023-09-23T20:46:33.566000+0000","last_fullsized":"2023-09-23T20:46:33.566000+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:00:19.498631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":485,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-09-23T20:46:25.644060+0000","last_change":"2023-09-23T20:43:46.541173+0000","last_active":"2023-09-23T20:46:25.644060+0000","last_peered":"2023-09-23T20:46:25.644060+0000","last_clean":"2023-09-23T20:46:25.644060+0000","last_became_active":"2023-09-23T20:43:46.540726+0000","last_became_peered":"2023-09-23T20:43:46.540726+0000","last_unstale":"2023-09-23T20:46:25.644060+0000","last_undegraded":"2023-09-23T20:46:25.644060+0000","last_fullsized":"2023-09-23T20:46:25.644060+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:49:04.137885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.e","version":"542'5231","reported_seq":5250,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-09-23T20:46:40.681196+0000","last_change":"2023-09-23T20:45:39.330822+0000","last_active":"2023-09-23T20:46:40.681196+0000","last_peered":"2023-09-23T20:46:40.681196+0000","last_clean":"2023-09-23T20:46:40.681196+0000","last_became_active":"2023-09-23T20:45:39.330513+0000","last_became_peered":"2023-09-23T20:45:39.330513+0000","last_unstale":"2023-09-23T20:46:40.681196+0000","last_undegraded":"2023-09-23T20:46:40.681196+0000","last_fullsized":"2023-09-23T20:46:40.681196+0000","mapping_epoch":531,"log_start":"0'0","ondisk_log_start":"0'0","created":531,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":5231,"log_dups_size":0,"ondisk_log_size":5231,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:35:59.769584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":342818816,"num_objects":5231,"num_object_clones":0,"num_object_copies":15693,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5231,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5231,"num_write_kb":334784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":820,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-09-23T20:46:25.637434+0000","last_change":"2023-09-23T20:38:05.590361+0000","last_active":"2023-09-23T20:46:25.637434+0000","last_peered":"2023-09-23T20:46:25.637434+0000","last_clean":"2023-09-23T20:46:25.637434+0000","last_became_active":"2023-09-23T20:38:05.589985+0000","last_became_peered":"2023-09-23T20:38:05.589985+0000","last_unstale":"2023-09-23T20:46:25.637434+0000","last_undegraded":"2023-09-23T20:46:25.637434+0000","last_fullsized":"2023-09-23T20:46:25.637434+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:03.246271+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:03.246271+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:38:54.351576+0000","scrub_duration":0.0019016700000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":279,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-09-23T20:46:25.651309+0000","last_change":"2023-09-23T20:32:50.549695+0000","last_active":"2023-09-23T20:46:25.651309+0000","last_peered":"2023-09-23T20:46:25.651309+0000","last_clean":"2023-09-23T20:46:25.651309+0000","last_became_active":"2023-09-23T20:32:50.549431+0000","last_became_peered":"2023-09-23T20:32:50.549431+0000","last_unstale":"2023-09-23T20:46:25.651309+0000","last_undegraded":"2023-09-23T20:46:25.651309+0000","last_fullsized":"2023-09-23T20:46:25.651309+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:45:53.666569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"86'128","reported_seq":955,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-09-23T20:46:25.642288+0000","last_change":"2023-09-23T20:38:03.541765+0000","last_active":"2023-09-23T20:46:25.642288+0000","last_peered":"2023-09-23T20:46:25.642288+0000","last_clean":"2023-09-23T20:46:25.642288+0000","last_became_active":"2023-09-23T20:38:03.541277+0000","last_became_peered":"2023-09-23T20:38:03.541277+0000","last_unstale":"2023-09-23T20:46:25.642288+0000","last_undegraded":"2023-09-23T20:46:25.642288+0000","last_fullsized":"2023-09-23T20:46:25.642288+0000","mapping_epoch":404,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":405,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:20:53.834554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000293202,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":579,"num_read_kb":1473,"num_write":96,"num_write_kb":1062,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":323,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-09-23T20:46:25.645379+0000","last_change":"2023-09-23T20:37:43.582105+0000","last_active":"2023-09-23T20:46:25.645379+0000","last_peered":"2023-09-23T20:46:25.645379+0000","last_clean":"2023-09-23T20:46:25.645379+0000","last_became_active":"2023-09-23T20:37:43.581860+0000","last_became_peered":"2023-09-23T20:37:43.581860+0000","last_unstale":"2023-09-23T20:46:25.645379+0000","last_undegraded":"2023-09-23T20:46:25.645379+0000","last_fullsized":"2023-09-23T20:46:25.645379+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:18:38.765486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"542'5190","reported_seq":5209,"reported_epoch":542,"state":"active+clean","last_fresh":"2023-09-23T20:46:40.347936+0000","last_change":"2023-09-23T20:45:39.381283+0000","last_active":"2023-09-23T20:46:40.347936+0000","last_peered":"2023-09-23T20:46:40.347936+0000","last_clean":"2023-09-23T20:46:40.347936+0000","last_became_active":"2023-09-23T20:45:39.380945+0000","last_became_peered":"2023-09-23T20:45:39.380945+0000","last_unstale":"2023-09-23T20:46:40.347936+0000","last_undegraded":"2023-09-23T20:46:40.347936+0000","last_fullsized":"2023-09-23T20:46:40.347936+0000","mapping_epoch":531,"log_start":"0'0","ondisk_log_start":"0'0","created":531,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":5190,"log_dups_size":0,"ondisk_log_size":5190,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:04:01.529754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":340131840,"num_objects":5190,"num_object_clones":0,"num_object_copies":15570,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5190,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5190,"num_write_kb":332160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.4","version":"90'161","reported_seq":757,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-09-23T20:46:25.646185+0000","last_change":"2023-09-23T20:40:22.427435+0000","last_active":"2023-09-23T20:46:25.646185+0000","last_peered":"2023-09-23T20:46:25.646185+0000","last_clean":"2023-09-23T20:46:25.646185+0000","last_became_active":"2023-09-23T20:40:22.427051+0000","last_became_peered":"2023-09-23T20:40:22.427051+0000","last_unstale":"2023-09-23T20:46:25.646185+0000","last_undegraded":"2023-09-23T20:46:25.646185+0000","last_fullsized":"2023-09-23T20:46:25.646185+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:34:54.504554+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:34:54.504554+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:42.001652+0000","scrub_duration":0.011176864,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"86'117","reported_seq":850,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-09-23T20:46:25.639979+0000","last_change":"2023-09-23T20:44:18.635224+0000","last_active":"2023-09-23T20:46:25.639979+0000","last_peered":"2023-09-23T20:46:25.639979+0000","last_clean":"2023-09-23T20:46:25.639979+0000","last_became_active":"2023-09-23T20:44:18.634703+0000","last_became_peered":"2023-09-23T20:44:18.634703+0000","last_unstale":"2023-09-23T20:46:25.639979+0000","last_undegraded":"2023-09-23T20:46:25.639979+0000","last_fullsized":"2023-09-23T20:46:25.639979+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:59:38.030626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046891400000000002,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":893,"num_read_kb":922,"num_write":82,"num_write_kb":4193,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":16777216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":530,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-09-23T20:46:25.644468+0000","last_change":"2023-09-23T20:43:46.521505+0000","last_active":"2023-09-23T20:46:25.644468+0000","last_peered":"2023-09-23T20:46:25.644468+0000","last_clean":"2023-09-23T20:46:25.644468+0000","last_became_active":"2023-09-23T20:43:46.520727+0000","last_became_peered":"2023-09-23T20:43:46.520727+0000","last_unstale":"2023-09-23T20:46:25.644468+0000","last_undegraded":"2023-09-23T20:46:25.644468+0000","last_fullsized":"2023-09-23T20:46:25.644468+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:17:06.164741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":279,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-09-23T20:46:25.636372+0000","last_change":"2023-09-23T20:40:19.530602+0000","last_active":"2023-09-23T20:46:25.636372+0000","last_peered":"2023-09-23T20:46:25.636372+0000","last_clean":"2023-09-23T20:46:25.636372+0000","last_became_active":"2023-09-23T20:40:19.530256+0000","last_became_peered":"2023-09-23T20:40:19.530256+0000","last_unstale":"2023-09-23T20:46:25.636372+0000","last_undegraded":"2023-09-23T20:46:25.636372+0000","last_fullsized":"2023-09-23T20:46:25.636372+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:18:45.658867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"90'161","reported_seq":799,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-09-23T20:46:25.643962+0000","last_change":"2023-09-23T20:43:46.544750+0000","last_active":"2023-09-23T20:46:25.643962+0000","last_peered":"2023-09-23T20:46:25.643962+0000","last_clean":"2023-09-23T20:46:25.643962+0000","last_became_active":"2023-09-23T20:43:46.544395+0000","last_became_peered":"2023-09-23T20:43:46.544395+0000","last_unstale":"2023-09-23T20:46:25.643962+0000","last_undegraded":"2023-09-23T20:46:25.643962+0000","last_fullsized":"2023-09-23T20:46:25.643962+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:06.487351+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:06.487351+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:56.866168+0000","scrub_duration":0.001964379,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":497,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-09-23T20:46:25.643983+0000","last_change":"2023-09-23T20:43:46.541379+0000","last_active":"2023-09-23T20:46:25.643983+0000","last_peered":"2023-09-23T20:46:25.643983+0000","last_clean":"2023-09-23T20:46:25.643983+0000","last_became_active":"2023-09-23T20:43:46.540732+0000","last_became_peered":"2023-09-23T20:43:46.540732+0000","last_unstale":"2023-09-23T20:46:25.643983+0000","last_undegraded":"2023-09-23T20:46:25.643983+0000","last_fullsized":"2023-09-23T20:46:25.643983+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:31:40.605772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":399,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-09-23T20:46:25.639664+0000","last_change":"2023-09-23T20:29:45.538314+0000","last_active":"2023-09-23T20:46:25.639664+0000","last_peered":"2023-09-23T20:46:25.639664+0000","last_clean":"2023-09-23T20:46:25.639664+0000","last_became_active":"2023-09-23T20:29:45.536847+0000","last_became_peered":"2023-09-23T20:29:45.536847+0000","last_unstale":"2023-09-23T20:46:25.639664+0000","last_undegraded":"2023-09-23T20:46:25.639664+0000","last_fullsized":"2023-09-23T20:46:25.639664+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:51.948721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"90'161","reported_seq":616,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-09-23T20:46:25.646210+0000","last_change":"2023-09-23T20:41:06.499511+0000","last_active":"2023-09-23T20:46:25.646210+0000","last_peered":"2023-09-23T20:46:25.646210+0000","last_clean":"2023-09-23T20:46:25.646210+0000","last_became_active":"2023-09-23T20:38:05.601874+0000","last_became_peered":"2023-09-23T20:38:05.601874+0000","last_unstale":"2023-09-23T20:46:25.646210+0000","last_undegraded":"2023-09-23T20:46:25.646210+0000","last_fullsized":"2023-09-23T20:46:25.646210+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:41:06.499396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:41:06.499396+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:31:34.059077+0000","scrub_duration":0.0019670310000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":434,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-09-23T20:46:25.637291+0000","last_change":"2023-09-23T20:43:16.758761+0000","last_active":"2023-09-23T20:46:25.637291+0000","last_peered":"2023-09-23T20:46:25.637291+0000","last_clean":"2023-09-23T20:46:25.637291+0000","last_became_active":"2023-09-23T20:43:16.758407+0000","last_became_peered":"2023-09-23T20:43:16.758407+0000","last_unstale":"2023-09-23T20:46:25.637291+0000","last_undegraded":"2023-09-23T20:46:25.637291+0000","last_fullsized":"2023-09-23T20:46:25.637291+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:59:27.070685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":198,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-09-23T20:46:25.636626+0000","last_change":"2023-09-23T20:34:59.987575+0000","last_active":"2023-09-23T20:46:25.636626+0000","last_peered":"2023-09-23T20:46:25.636626+0000","last_clean":"2023-09-23T20:46:25.636626+0000","last_became_active":"2023-09-23T20:34:59.987229+0000","last_became_peered":"2023-09-23T20:34:59.987229+0000","last_unstale":"2023-09-23T20:46:25.636626+0000","last_undegraded":"2023-09-23T20:46:25.636626+0000","last_fullsized":"2023-09-23T20:46:25.636626+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:34:01.820159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":442,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-09-23T20:46:25.637133+0000","last_change":"2023-09-23T20:43:16.759691+0000","last_active":"2023-09-23T20:46:25.637133+0000","last_peered":"2023-09-23T20:46:25.637133+0000","last_clean":"2023-09-23T20:46:25.637133+0000","last_became_active":"2023-09-23T20:43:16.759309+0000","last_became_peered":"2023-09-23T20:43:16.759309+0000","last_unstale":"2023-09-23T20:46:25.637133+0000","last_undegraded":"2023-09-23T20:46:25.637133+0000","last_fullsized":"2023-09-23T20:46:25.637133+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:58:52.893019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":198,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-09-23T20:46:25.636176+0000","last_change":"2023-09-23T20:34:59.987416+0000","last_active":"2023-09-23T20:46:25.636176+0000","last_peered":"2023-09-23T20:46:25.636176+0000","last_clean":"2023-09-23T20:46:25.636176+0000","last_became_active":"2023-09-23T20:34:59.987117+0000","last_became_peered":"2023-09-23T20:34:59.987117+0000","last_unstale":"2023-09-23T20:46:25.636176+0000","last_undegraded":"2023-09-23T20:46:25.636176+0000","last_fullsized":"2023-09-23T20:46:25.636176+0000","mapping_epoch":353,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":354,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:36:16.318623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"90'161","reported_seq":797,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-09-23T20:46:25.650119+0000","last_change":"2023-09-23T20:38:05.591992+0000","last_active":"2023-09-23T20:46:25.650119+0000","last_peered":"2023-09-23T20:46:25.650119+0000","last_clean":"2023-09-23T20:46:25.650119+0000","last_became_active":"2023-09-23T20:38:05.591701+0000","last_became_peered":"2023-09-23T20:38:05.591701+0000","last_unstale":"2023-09-23T20:46:25.650119+0000","last_undegraded":"2023-09-23T20:46:25.650119+0000","last_fullsized":"2023-09-23T20:46:25.650119+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:02.384133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:02.384133+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:41.722542+0000","scrub_duration":0.001769098,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":365,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-09-23T20:46:25.646937+0000","last_change":"2023-09-23T20:38:02.542719+0000","last_active":"2023-09-23T20:46:25.646937+0000","last_peered":"2023-09-23T20:46:25.646937+0000","last_clean":"2023-09-23T20:46:25.646937+0000","last_became_active":"2023-09-23T20:38:02.542385+0000","last_became_peered":"2023-09-23T20:38:02.542385+0000","last_unstale":"2023-09-23T20:46:25.646937+0000","last_undegraded":"2023-09-23T20:46:25.646937+0000","last_fullsized":"2023-09-23T20:46:25.646937+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:36:23.673757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":331,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-09-23T20:46:25.645757+0000","last_change":"2023-09-23T20:37:43.584253+0000","last_active":"2023-09-23T20:46:25.645757+0000","last_peered":"2023-09-23T20:46:25.645757+0000","last_clean":"2023-09-23T20:46:25.645757+0000","last_became_active":"2023-09-23T20:37:43.580069+0000","last_became_peered":"2023-09-23T20:37:43.580069+0000","last_unstale":"2023-09-23T20:46:25.645757+0000","last_undegraded":"2023-09-23T20:46:25.645757+0000","last_fullsized":"2023-09-23T20:46:25.645757+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:39:11.482581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"90'161","reported_seq":636,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-09-23T20:46:25.641868+0000","last_change":"2023-09-23T20:36:01.505098+0000","last_active":"2023-09-23T20:46:25.641868+0000","last_peered":"2023-09-23T20:46:25.641868+0000","last_clean":"2023-09-23T20:46:25.641868+0000","last_became_active":"2023-09-23T20:35:26.644707+0000","last_became_peered":"2023-09-23T20:35:26.644707+0000","last_unstale":"2023-09-23T20:46:25.641868+0000","last_undegraded":"2023-09-23T20:46:25.641868+0000","last_fullsized":"2023-09-23T20:46:25.641868+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:01.504973+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:01.504973+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:02:10.880809+0000","scrub_duration":0.00202344,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":575,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-09-23T20:46:25.644030+0000","last_change":"2023-09-23T20:43:46.511657+0000","last_active":"2023-09-23T20:46:25.644030+0000","last_peered":"2023-09-23T20:46:25.644030+0000","last_clean":"2023-09-23T20:46:25.644030+0000","last_became_active":"2023-09-23T20:43:46.511326+0000","last_became_peered":"2023-09-23T20:43:46.511326+0000","last_unstale":"2023-09-23T20:46:25.644030+0000","last_undegraded":"2023-09-23T20:46:25.644030+0000","last_fullsized":"2023-09-23T20:46:25.644030+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:36:46.139261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3324,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":259,"reported_epoch":541,"state":"active+clean","last_fresh":"2023-09-23T20:46:25.635940+0000","last_change":"2023-09-23T20:38:02.538333+0000","last_active":"2023-09-23T20:46:25.635940+0000","last_peered":"2023-09-23T20:46:25.635940+0000","last_clean":"2023-09-23T20:46:25.635940+0000","last_became_active":"2023-09-23T20:38:02.538008+0000","last_became_peered":"2023-09-23T20:38:02.538008+0000","last_unstale":"2023-09-23T20:46:25.635940+0000","last_undegraded":"2023-09-23T20:46:25.635940+0000","last_fullsized":"2023-09-23T20:46:25.635940+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:02:38.789530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":5431623680,"num_objects":82880,"num_object_clones":0,"num_object_copies":248640,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":82880,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":82880,"num_write_kb":5304320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16315973632,"data_stored":16315973632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":82880,"ondisk_log_size":82880,"up":48,"acting":48,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":90,"num_bytes_recovered":18486,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26691,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5983,"num_read_kb":13200,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":84,"num_bytes_recovered":16777216,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6082,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201703,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1887204,"kb_used_data":1338908,"kb_used_omap":0,"kb_used_meta":548288,"kb_avail":91857948,"statfs":{"total":95995035648,"available":94062538752,"internally_reserved":0,"allocated":1371041792,"data_stored":1369517021,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":561446912},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:46:00 2023","interfaces":[{"interface":"back","average":{"1min":0.71499999999999997,"5min":1.909,"15min":2.1360000000000001},"min":{"1min":0.224,"5min":0.224,"15min":0.224},"max":{"1min":2.5379999999999998,"5min":15.1,"15min":18.315000000000001},"last":1.448},{"interface":"front","average":{"1min":1.1220000000000001,"5min":1.891,"15min":1.7969999999999999},"min":{"1min":0.28299999999999997,"5min":0.27800000000000002,"15min":0.23899999999999999},"max":{"1min":5.1619999999999999,"5min":15.327999999999999,"15min":19.355},"last":2.7090000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:46:08 2023","interfaces":[{"interface":"back","average":{"1min":0.95299999999999996,"5min":1.4750000000000001,"15min":2.0009999999999999},"min":{"1min":0.35299999999999998,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":3.4359999999999999,"5min":15.131,"15min":49.965000000000003},"last":0.35399999999999998},{"interface":"front","average":{"1min":1.1259999999999999,"5min":2.6379999999999999,"15min":2.335},"min":{"1min":0.37,"5min":0.26700000000000002,"15min":0.248},"max":{"1min":3.7770000000000001,"5min":15.18,"15min":53.451999999999998},"last":2.7429999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:46:00 2023","interfaces":[{"interface":"back","average":{"1min":1.137,"5min":2.2719999999999998,"15min":2.1659999999999999},"min":{"1min":0.34100000000000003,"5min":0.27900000000000003,"15min":0.27800000000000002},"max":{"1min":3.1080000000000001,"5min":15.087,"15min":16.965},"last":2.8039999999999998},{"interface":"front","average":{"1min":0.83299999999999996,"5min":1.897,"15min":2.3239999999999998},"min":{"1min":0.31,"5min":0.30299999999999999,"15min":0.27400000000000002},"max":{"1min":4.9400000000000004,"5min":15.193,"15min":23.494},"last":1.0069999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:46:31 2023","interfaces":[{"interface":"back","average":{"1min":1.218,"5min":1.8140000000000001,"15min":2.2200000000000002},"min":{"1min":0.34799999999999998,"5min":0.32500000000000001,"15min":0.311},"max":{"1min":3.2229999999999999,"5min":15.115,"15min":23.352},"last":0.91100000000000003},{"interface":"front","average":{"1min":1.7589999999999999,"5min":2.2869999999999999,"15min":2.4169999999999998},"min":{"1min":0.498,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":5.6230000000000002,"5min":15.144,"15min":20.969999999999999},"last":2.8250000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:46:31 2023","interfaces":[{"interface":"back","average":{"1min":1.6739999999999999,"5min":1.484,"15min":1.141},"min":{"1min":0.254,"5min":0.254,"15min":0.23200000000000001},"max":{"1min":6.8689999999999998,"5min":12.260999999999999,"15min":18.364999999999998},"last":0.42999999999999999},{"interface":"front","average":{"1min":1.704,"5min":1.776,"15min":1.3700000000000001},"min":{"1min":0.29399999999999998,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":6.96,"5min":7.2469999999999999,"15min":23.617000000000001},"last":2.637}]},{"osd":5,"last update":"Sat Sep 23 20:46:31 2023","interfaces":[{"interface":"back","average":{"1min":0.96999999999999997,"5min":1.6559999999999999,"15min":1.1779999999999999},"min":{"1min":0.33100000000000002,"5min":0.30499999999999999,"15min":0.26600000000000001},"max":{"1min":3.0139999999999998,"5min":15.005000000000001,"15min":15.005000000000001},"last":0.82399999999999995},{"interface":"front","average":{"1min":1.3149999999999999,"5min":1.873,"15min":1.286},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":6.7939999999999996,"5min":12.731,"15min":18.390000000000001},"last":0.501}]},{"osd":6,"last update":"Sat Sep 23 20:46:31 2023","interfaces":[{"interface":"back","average":{"1min":2.1240000000000001,"5min":2.1080000000000001,"15min":1.4079999999999999},"min":{"1min":0.32400000000000001,"5min":0.317,"15min":0.30399999999999999},"max":{"1min":12.183,"5min":15.038,"15min":18.417000000000002},"last":2.8980000000000001},{"interface":"front","average":{"1min":2.141,"5min":1.889,"15min":1.331},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.318},"max":{"1min":11.404999999999999,"5min":15.369999999999999,"15min":15.369999999999999},"last":2.7759999999999998}]}]},{"osd":6,"up_from":47,"seq":201863463342,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2790232,"kb_used_data":2315600,"kb_used_omap":0,"kb_used_meta":474624,"kb_avail":90954920,"statfs":{"total":95995035648,"available":93137838080,"internally_reserved":0,"allocated":2371174400,"data_stored":2369661246,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":486014976},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,2,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:46:00 2023","interfaces":[{"interface":"back","average":{"1min":0.74199999999999999,"5min":1.4830000000000001,"15min":1.9390000000000001},"min":{"1min":0.26200000000000001,"5min":0.219,"15min":0.219},"max":{"1min":3.403,"5min":12.552,"15min":53.701999999999998},"last":21.018000000000001},{"interface":"front","average":{"1min":0.995,"5min":1.663,"15min":2.2959999999999998},"min":{"1min":0.312,"5min":0.23300000000000001,"15min":0.22700000000000001},"max":{"1min":3.9769999999999999,"5min":10.895,"15min":60.328000000000003},"last":21.082000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:46:18 2023","interfaces":[{"interface":"back","average":{"1min":0.94299999999999995,"5min":1.956,"15min":1.9570000000000001},"min":{"1min":0.29299999999999998,"5min":0.249,"15min":0.249},"max":{"1min":4.1100000000000003,"5min":9.2520000000000007,"15min":64.274000000000001},"last":0.48999999999999999},{"interface":"front","average":{"1min":0.82999999999999996,"5min":1.9339999999999999,"15min":1.8109999999999999},"min":{"1min":0.36499999999999999,"5min":0.29099999999999998,"15min":0.28000000000000003},"max":{"1min":3.0790000000000002,"5min":17.786000000000001,"15min":34.490000000000002},"last":0.34000000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:46:00 2023","interfaces":[{"interface":"back","average":{"1min":0.877,"5min":1.9930000000000001,"15min":2.0489999999999999},"min":{"1min":0.40100000000000002,"5min":0.27800000000000002,"15min":0.27700000000000002},"max":{"1min":2.7490000000000001,"5min":9.0150000000000006,"15min":34.317999999999998},"last":22.591000000000001},{"interface":"front","average":{"1min":0.70699999999999996,"5min":1.7330000000000001,"15min":1.9490000000000001},"min":{"1min":0.376,"5min":0.34000000000000002,"15min":0.30399999999999999},"max":{"1min":1.4650000000000001,"5min":8.7309999999999999,"15min":34.292999999999999},"last":22.614999999999998}]},{"osd":3,"last update":"Sat Sep 23 20:45:48 2023","interfaces":[{"interface":"back","average":{"1min":0.99199999999999999,"5min":1.821,"15min":1.911},"min":{"1min":0.45200000000000001,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":2.8090000000000002,"5min":11.664,"15min":11.664},"last":0.40999999999999998},{"interface":"front","average":{"1min":1.153,"5min":1.8759999999999999,"15min":2.0859999999999999},"min":{"1min":0.378,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":2.8969999999999998,"5min":11.808,"15min":12.875999999999999},"last":0.373}]},{"osd":4,"last update":"Sat Sep 23 20:46:18 2023","interfaces":[{"interface":"back","average":{"1min":0.83099999999999996,"5min":0.96999999999999997,"15min":0.81399999999999995},"min":{"1min":0.30499999999999999,"5min":0.25600000000000001,"15min":0.251},"max":{"1min":3.052,"5min":6.5800000000000001,"15min":23.904},"last":0.59799999999999998},{"interface":"front","average":{"1min":0.91200000000000003,"5min":1.077,"15min":1.014},"min":{"1min":0.34399999999999997,"5min":0.28399999999999997,"15min":0.28199999999999997},"max":{"1min":4.2709999999999999,"5min":5.6260000000000003,"15min":11.515000000000001},"last":1.859}]},{"osd":5,"last update":"Sat Sep 23 20:46:18 2023","interfaces":[{"interface":"back","average":{"1min":0.85599999999999998,"5min":1.1160000000000001,"15min":0.877},"min":{"1min":0.41699999999999998,"5min":0.253,"15min":0.253},"max":{"1min":3.105,"5min":7.8529999999999998,"15min":7.8529999999999998},"last":0.53200000000000003},{"interface":"front","average":{"1min":0.93899999999999995,"5min":1.522,"15min":1.1000000000000001},"min":{"1min":0.35399999999999998,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":4.2320000000000002,"5min":7.1319999999999997,"15min":7.7300000000000004},"last":1.161}]},{"osd":7,"last update":"Sat Sep 23 20:45:48 2023","interfaces":[{"interface":"back","average":{"1min":0.68700000000000006,"5min":0.99099999999999999,"15min":0.85699999999999998},"min":{"1min":0.32200000000000001,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":1.631,"5min":8.8350000000000009,"15min":8.8350000000000009},"last":1.0349999999999999},{"interface":"front","average":{"1min":0.79500000000000004,"5min":1.625,"15min":1.204},"min":{"1min":0.35999999999999999,"5min":0.34699999999999998,"15min":0.30299999999999999},"max":{"1min":2.8300000000000001,"5min":7.1929999999999996,"15min":8.8119999999999994},"last":0.44800000000000001}]}]},{"osd":1,"up_from":266,"seq":1142461300940,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1702352,"kb_used_data":992008,"kb_used_omap":8,"kb_used_meta":710327,"kb_avail":92042800,"statfs":{"total":95995035648,"available":94251827200,"internally_reserved":0,"allocated":1015816192,"data_stored":1013087957,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8206,"internal_metadata":727375858},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:46:20 2023","interfaces":[{"interface":"back","average":{"1min":1.8109999999999999,"5min":2.4340000000000002,"15min":2.532},"min":{"1min":0.20799999999999999,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":9.5280000000000005,"5min":15.532999999999999,"15min":52.689999999999998},"last":0.80700000000000005},{"interface":"front","average":{"1min":1.014,"5min":2.5569999999999999,"15min":2.9820000000000002},"min":{"1min":0.254,"5min":0.221,"15min":0.19900000000000001},"max":{"1min":5.7969999999999997,"5min":20.346,"15min":52.662999999999997},"last":0.70999999999999996}]},{"osd":2,"last update":"Sat Sep 23 20:46:20 2023","interfaces":[{"interface":"back","average":{"1min":2.032,"5min":2.7639999999999998,"15min":3.0739999999999998},"min":{"1min":0.26600000000000001,"5min":0.23499999999999999,"15min":0.20899999999999999},"max":{"1min":4.7779999999999996,"5min":19.026,"15min":32.255000000000003},"last":0.75700000000000001},{"interface":"front","average":{"1min":1.9930000000000001,"5min":2.0499999999999998,"15min":2.27},"min":{"1min":0.42399999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":9.5969999999999995,"5min":10.041,"15min":13.404999999999999},"last":4.3289999999999997}]},{"osd":3,"last update":"Sat Sep 23 20:46:20 2023","interfaces":[{"interface":"back","average":{"1min":0.84499999999999997,"5min":2.1019999999999999,"15min":2.8159999999999998},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.247},"max":{"1min":3.0310000000000001,"5min":10.757,"15min":80.745999999999995},"last":0.36599999999999999},{"interface":"front","average":{"1min":1.1879999999999999,"5min":2.641,"15min":3.0299999999999998},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22700000000000001},"max":{"1min":5.9400000000000004,"5min":11.534000000000001,"15min":82.988},"last":0.44900000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:46:20 2023","interfaces":[{"interface":"back","average":{"1min":1.2849999999999999,"5min":2.5979999999999999,"15min":2.3079999999999998},"min":{"1min":0.32600000000000001,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":4.327,"5min":20.634,"15min":24.327999999999999},"last":0.64600000000000002},{"interface":"front","average":{"1min":0.76200000000000001,"5min":1.931,"15min":1.673},"min":{"1min":0.50700000000000001,"5min":0.34200000000000003,"15min":0.311},"max":{"1min":2.5,"5min":20.835999999999999,"15min":20.835999999999999},"last":0.40699999999999997}]},{"osd":5,"last update":"Sat Sep 23 20:46:20 2023","interfaces":[{"interface":"back","average":{"1min":0.72699999999999998,"5min":2.4089999999999998,"15min":2.1899999999999999},"min":{"1min":0.34699999999999998,"5min":0.308,"15min":0.308},"max":{"1min":2.1240000000000001,"5min":14.366,"15min":33.401000000000003},"last":0.41799999999999998},{"interface":"front","average":{"1min":1.464,"5min":2.7010000000000001,"15min":2.5590000000000002},"min":{"1min":0.39400000000000002,"5min":0.33100000000000002,"15min":0.32500000000000001},"max":{"1min":4.2489999999999997,"5min":11.750999999999999,"15min":33.417000000000002},"last":0.68899999999999995}]},{"osd":6,"last update":"Sat Sep 23 20:46:20 2023","interfaces":[{"interface":"back","average":{"1min":1.264,"5min":2.3730000000000002,"15min":2.117},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":5.835,"5min":14.414999999999999,"15min":14.414999999999999},"last":0.38600000000000001},{"interface":"front","average":{"1min":1.599,"5min":2.9670000000000001,"15min":2.5299999999999998},"min":{"1min":0.48899999999999999,"5min":0.38500000000000001,"15min":0.317},"max":{"1min":4.4009999999999998,"5min":11.832000000000001,"15min":24.376000000000001},"last":0.72499999999999998}]},{"osd":7,"last update":"Sat Sep 23 20:46:20 2023","interfaces":[{"interface":"back","average":{"1min":0.88500000000000001,"5min":2.4119999999999999,"15min":2.2130000000000001},"min":{"1min":0.377,"5min":0.37,"15min":0.37},"max":{"1min":2.4009999999999998,"5min":9.9420000000000002,"15min":18.437000000000001},"last":0.48799999999999999},{"interface":"front","average":{"1min":1.4359999999999999,"5min":2.9300000000000002,"15min":2.5710000000000002},"min":{"1min":0.41899999999999998,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":4.3620000000000001,"5min":11.864000000000001,"15min":24.524000000000001},"last":0.79100000000000004}]}]},{"osd":0,"up_from":94,"seq":403726926143,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4197216,"kb_used_data":3339736,"kb_used_omap":7,"kb_used_meta":857464,"kb_avail":89547936,"statfs":{"total":95995035648,"available":91697086464,"internally_reserved":0,"allocated":3419889664,"data_stored":3417322449,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":878043153},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,0,1,3],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:46:28 2023","interfaces":[{"interface":"back","average":{"1min":1.2549999999999999,"5min":2.0830000000000002,"15min":2.4039999999999999},"min":{"1min":0.20300000000000001,"5min":0.184,"15min":0.17199999999999999},"max":{"1min":7.3339999999999996,"5min":18.052,"15min":42.000999999999998},"last":0.34999999999999998},{"interface":"front","average":{"1min":1.8029999999999999,"5min":2.5779999999999998,"15min":2.597},"min":{"1min":0.26600000000000001,"5min":0.22700000000000001,"15min":0.20399999999999999},"max":{"1min":7.3689999999999998,"5min":18.097000000000001,"15min":23.036000000000001},"last":0.42199999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:46:01 2023","interfaces":[{"interface":"back","average":{"1min":1.2030000000000001,"5min":2.6800000000000002,"15min":2.7730000000000001},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.193},"max":{"1min":3.714,"5min":48.841000000000001,"15min":62.726999999999997},"last":6.2169999999999996},{"interface":"front","average":{"1min":1.5449999999999999,"5min":2.4990000000000001,"15min":2.7440000000000002},"min":{"1min":0.251,"5min":0.22,"15min":0.188},"max":{"1min":7.3170000000000002,"5min":46.183,"15min":62.689},"last":3.9529999999999998}]},{"osd":3,"last update":"Sat Sep 23 20:46:39 2023","interfaces":[{"interface":"back","average":{"1min":5.1189999999999998,"5min":3.4740000000000002,"15min":3.0819999999999999},"min":{"1min":0.31900000000000001,"5min":0.248,"15min":0.23400000000000001},"max":{"1min":55.017000000000003,"5min":55.017000000000003,"15min":55.017000000000003},"last":0.27800000000000002},{"interface":"front","average":{"1min":5.1740000000000004,"5min":2.895,"15min":3.0899999999999999},"min":{"1min":0.27400000000000002,"5min":0.23799999999999999,"15min":0.21199999999999999},"max":{"1min":55.665999999999997,"5min":55.665999999999997,"15min":55.665999999999997},"last":6.1890000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:46:01 2023","interfaces":[{"interface":"back","average":{"1min":1.036,"5min":1.7,"15min":1.925},"min":{"1min":0.34300000000000003,"5min":0.32000000000000001,"15min":0.25900000000000001},"max":{"1min":3.7690000000000001,"5min":10.368,"15min":12.587999999999999},"last":0.318},{"interface":"front","average":{"1min":1.0249999999999999,"5min":1.5720000000000001,"15min":1.8440000000000001},"min":{"1min":0.313,"5min":0.28899999999999998,"15min":0.27400000000000002},"max":{"1min":3.2410000000000001,"5min":9.8480000000000008,"15min":23.001000000000001},"last":6.2949999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:46:01 2023","interfaces":[{"interface":"back","average":{"1min":1.0029999999999999,"5min":2.0470000000000002,"15min":1.978},"min":{"1min":0.36699999999999999,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":3.492,"5min":10.239000000000001,"15min":22.969999999999999},"last":0.308},{"interface":"front","average":{"1min":0.90600000000000003,"5min":2.1080000000000001,"15min":2.2909999999999999},"min":{"1min":0.41199999999999998,"5min":0.34300000000000003,"15min":0.25600000000000001},"max":{"1min":3.161,"5min":12.601000000000001,"15min":26.228000000000002},"last":6.2389999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:46:01 2023","interfaces":[{"interface":"back","average":{"1min":1.0309999999999999,"5min":1.8500000000000001,"15min":2.0910000000000002},"min":{"1min":0.374,"5min":0.371,"15min":0.32300000000000001},"max":{"1min":3.5470000000000002,"5min":9.9060000000000006,"15min":24.056000000000001},"last":6.2610000000000001},{"interface":"front","average":{"1min":0.90300000000000002,"5min":1.845,"15min":2.1739999999999999},"min":{"1min":0.38200000000000001,"5min":0.36399999999999999,"15min":0.29399999999999998},"max":{"1min":3.0699999999999998,"5min":12.698,"15min":26.268000000000001},"last":6.3659999999999997}]},{"osd":7,"last update":"Sat Sep 23 20:46:01 2023","interfaces":[{"interface":"back","average":{"1min":1.8859999999999999,"5min":2.25,"15min":2.3769999999999998},"min":{"1min":0.434,"5min":0.41899999999999998,"15min":0.32200000000000001},"max":{"1min":16.553999999999998,"5min":16.553999999999998,"15min":26.177},"last":6.2850000000000001},{"interface":"front","average":{"1min":2.3370000000000002,"5min":2.5089999999999999,"15min":2.3660000000000001},"min":{"1min":0.47699999999999998,"5min":0.40400000000000003,"15min":0.34200000000000003},"max":{"1min":16.649000000000001,"5min":16.649000000000001,"15min":22.422999999999998},"last":0.40200000000000002}]}]},{"osd":2,"up_from":104,"seq":446676599090,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2721956,"kb_used_data":1944092,"kb_used_omap":11,"kb_used_meta":777844,"kb_avail":91023196,"statfs":{"total":95995035648,"available":93207752704,"internally_reserved":0,"allocated":1990750208,"data_stored":1988177121,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11878,"internal_metadata":796512666},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:46:09 2023","interfaces":[{"interface":"back","average":{"1min":0.89200000000000002,"5min":1.2829999999999999,"15min":2.2949999999999999},"min":{"1min":0.23999999999999999,"5min":0.17299999999999999,"15min":0.17299999999999999},"max":{"1min":4.0620000000000003,"5min":11.07,"15min":82.424999999999997},"last":0.25700000000000001},{"interface":"front","average":{"1min":0.42599999999999999,"5min":1.917,"15min":2.5},"min":{"1min":0.26500000000000001,"5min":0.23599999999999999,"15min":0.19},"max":{"1min":1.0169999999999999,"5min":16.678999999999998,"15min":82.524000000000001},"last":2.3540000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:46:09 2023","interfaces":[{"interface":"back","average":{"1min":0.65100000000000002,"5min":2.101,"15min":2.4660000000000002},"min":{"1min":0.224,"5min":0.224,"15min":0.224},"max":{"1min":2.7090000000000001,"5min":12.981,"15min":46.405000000000001},"last":2.1880000000000002},{"interface":"front","average":{"1min":0.995,"5min":2.1030000000000002,"15min":2.4809999999999999},"min":{"1min":0.22,"5min":0.22,"15min":0.22},"max":{"1min":3.8079999999999998,"5min":13.215999999999999,"15min":46.459000000000003},"last":0.29099999999999998}]},{"osd":3,"last update":"Sat Sep 23 20:46:35 2023","interfaces":[{"interface":"back","average":{"1min":2.9300000000000002,"5min":2.5569999999999999,"15min":2.5699999999999998},"min":{"1min":0.432,"5min":0.253,"15min":0.253},"max":{"1min":30.283000000000001,"5min":30.283000000000001,"15min":30.283000000000001},"last":2.242},{"interface":"front","average":{"1min":3.294,"5min":1.7290000000000001,"15min":2.077},"min":{"1min":0.379,"5min":0.26500000000000001,"15min":0.26400000000000001},"max":{"1min":30.834,"5min":30.834,"15min":30.834},"last":0.439}]},{"osd":4,"last update":"Sat Sep 23 20:46:09 2023","interfaces":[{"interface":"back","average":{"1min":0.98299999999999998,"5min":1.7450000000000001,"15min":1.776},"min":{"1min":0.34200000000000003,"5min":0.29799999999999999,"15min":0.27500000000000002},"max":{"1min":4.5839999999999996,"5min":12.025,"15min":12.025},"last":2.4239999999999999},{"interface":"front","average":{"1min":0.92700000000000005,"5min":1.78,"15min":1.964},"min":{"1min":0.38800000000000001,"5min":0.30599999999999999,"15min":0.30499999999999999},"max":{"1min":4.4029999999999996,"5min":10.948,"15min":19.440999999999999},"last":3.6389999999999998}]},{"osd":5,"last update":"Sat Sep 23 20:46:09 2023","interfaces":[{"interface":"back","average":{"1min":0.68100000000000005,"5min":1.8879999999999999,"15min":1.861},"min":{"1min":0.33200000000000002,"5min":0.311,"15min":0.311},"max":{"1min":1.6399999999999999,"5min":12.07,"15min":12.07},"last":2.5619999999999998},{"interface":"front","average":{"1min":0.93200000000000005,"5min":1.7889999999999999,"15min":2.0569999999999999},"min":{"1min":0.45200000000000001,"5min":0.37,"15min":0.35299999999999998},"max":{"1min":4.444,"5min":11.061999999999999,"15min":19.513000000000002},"last":2.2789999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:46:09 2023","interfaces":[{"interface":"back","average":{"1min":0.94599999999999995,"5min":2.0840000000000001,"15min":2.1080000000000001},"min":{"1min":0.46800000000000003,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":2.8900000000000001,"5min":14.047000000000001,"15min":19.486000000000001},"last":2.5179999999999998},{"interface":"front","average":{"1min":1.2949999999999999,"5min":1.3740000000000001,"15min":1.5940000000000001},"min":{"1min":0.505,"5min":0.34200000000000003,"15min":0.318},"max":{"1min":3.8450000000000002,"5min":11.747,"15min":11.747},"last":0.46200000000000002}]},{"osd":7,"last update":"Sat Sep 23 20:46:09 2023","interfaces":[{"interface":"back","average":{"1min":1.077,"5min":1.321,"15min":1.5549999999999999},"min":{"1min":0.42999999999999999,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":3.8769999999999998,"5min":8.4949999999999992,"15min":12.734999999999999},"last":3.9510000000000001},{"interface":"front","average":{"1min":0.92300000000000004,"5min":2.0910000000000002,"15min":2.0310000000000001},"min":{"1min":0.40000000000000002,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":2.2170000000000001,"5min":12.099,"15min":12.099},"last":3.427}]}]},{"osd":3,"up_from":109,"seq":468151435564,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2652436,"kb_used_data":1643020,"kb_used_omap":4,"kb_used_meta":1009403,"kb_avail":91092716,"statfs":{"total":95995035648,"available":93278941184,"internally_reserved":0,"allocated":1682452480,"data_stored":1679892766,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4514,"internal_metadata":1033629278},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,2],"upper_bound":256},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:45:41 2023","interfaces":[{"interface":"back","average":{"1min":1.419,"5min":2.419,"15min":2.4649999999999999},"min":{"1min":0.249,"5min":0.20899999999999999,"15min":0.191},"max":{"1min":5.5339999999999998,"5min":22.32,"15min":22.32},"last":0.90000000000000002},{"interface":"front","average":{"1min":2.746,"5min":2.7330000000000001,"15min":2.5510000000000002},"min":{"1min":0.313,"5min":0.22900000000000001,"15min":0.19500000000000001},"max":{"1min":17.623000000000001,"5min":22.805,"15min":22.805},"last":0.69499999999999995}]},{"osd":1,"last update":"Sat Sep 23 20:46:15 2023","interfaces":[{"interface":"back","average":{"1min":0.98299999999999998,"5min":2.6160000000000001,"15min":2.0950000000000002},"min":{"1min":0.24199999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":7.6509999999999998,"5min":17.529,"15min":17.529},"last":1.0249999999999999},{"interface":"front","average":{"1min":1.046,"5min":2.2450000000000001,"15min":2.2810000000000001},"min":{"1min":0.312,"5min":0.24399999999999999,"15min":0.23899999999999999},"max":{"1min":4.0030000000000001,"5min":14.19,"15min":32.603999999999999},"last":0.439}]},{"osd":2,"last update":"Sat Sep 23 20:45:41 2023","interfaces":[{"interface":"back","average":{"1min":2.9830000000000001,"5min":3.7170000000000001,"15min":3.0169999999999999},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":12.473000000000001,"5min":68.206999999999994,"15min":68.206999999999994},"last":0.61599999999999999},{"interface":"front","average":{"1min":2.5529999999999999,"5min":2.9199999999999999,"15min":2.7050000000000001},"min":{"1min":0.29999999999999999,"5min":0.245,"15min":0.245},"max":{"1min":17.919,"5min":67.736999999999995,"15min":67.736999999999995},"last":0.77300000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:45:41 2023","interfaces":[{"interface":"back","average":{"1min":1.5109999999999999,"5min":2.1459999999999999,"15min":1.7350000000000001},"min":{"1min":0.41599999999999998,"5min":0.317,"15min":0.30499999999999999},"max":{"1min":5.468,"5min":22.157,"15min":22.157},"last":0.875},{"interface":"front","average":{"1min":1.407,"5min":2.2829999999999999,"15min":2.2650000000000001},"min":{"1min":0.372,"5min":0.35399999999999998,"15min":0.29399999999999998},"max":{"1min":4.8339999999999996,"5min":15.868,"15min":19.635000000000002},"last":0.58199999999999996}]},{"osd":5,"last update":"Sat Sep 23 20:45:41 2023","interfaces":[{"interface":"back","average":{"1min":1.349,"5min":2.4020000000000001,"15min":1.927},"min":{"1min":0.45500000000000002,"5min":0.33800000000000002,"15min":0.32200000000000001},"max":{"1min":5.5019999999999998,"5min":22.289999999999999,"15min":22.289999999999999},"last":0.81499999999999995},{"interface":"front","average":{"1min":2.702,"5min":2.0470000000000002,"15min":1.8819999999999999},"min":{"1min":0.47699999999999998,"5min":0.34100000000000003,"15min":0.32000000000000001},"max":{"1min":8.6129999999999995,"5min":12.952,"15min":12.952},"last":0.64100000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:45:41 2023","interfaces":[{"interface":"back","average":{"1min":2.4809999999999999,"5min":2.2599999999999998,"15min":1.974},"min":{"1min":0.44800000000000001,"5min":0.36599999999999999,"15min":0.34300000000000003},"max":{"1min":8.6790000000000003,"5min":12.917,"15min":17.524999999999999},"last":0.66400000000000003},{"interface":"front","average":{"1min":1.7929999999999999,"5min":2.3940000000000001,"15min":2.2320000000000002},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.33700000000000002},"max":{"1min":6.7960000000000003,"5min":15.936,"15min":19.524000000000001},"last":0.55600000000000005}]},{"osd":7,"last update":"Sat Sep 23 20:45:41 2023","interfaces":[{"interface":"back","average":{"1min":1.7,"5min":2.7890000000000001,"15min":2.5310000000000001},"min":{"1min":0.42699999999999999,"5min":0.41499999999999998,"15min":0.36899999999999999},"max":{"1min":4.8630000000000004,"5min":15.976000000000001,"15min":19.571999999999999},"last":0.52800000000000002},{"interface":"front","average":{"1min":2.1179999999999999,"5min":2.4870000000000001,"15min":1.97},"min":{"1min":0.46899999999999997,"5min":0.42799999999999999,"15min":0.40999999999999998},"max":{"1min":10.853999999999999,"5min":22.202999999999999,"15min":22.202999999999999},"last":0.84699999999999998}]}]},{"osd":4,"up_from":30,"seq":128849019318,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3155800,"kb_used_data":2678032,"kb_used_omap":0,"kb_used_meta":477760,"kb_avail":90589352,"statfs":{"total":95995035648,"available":92763496448,"internally_reserved":0,"allocated":2742304768,"data_stored":2740791348,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":489226240},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:46:00 2023","interfaces":[{"interface":"back","average":{"1min":0.98299999999999998,"5min":1.8660000000000001,"15min":1.7330000000000001},"min":{"1min":0.28899999999999998,"5min":0.251,"15min":0.24099999999999999},"max":{"1min":6.109,"5min":11.709,"15min":24.114999999999998},"last":0.28499999999999998},{"interface":"front","average":{"1min":1.113,"5min":2.1499999999999999,"15min":1.931},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.246},"max":{"1min":4.4349999999999996,"5min":13.74,"15min":24.135999999999999},"last":3.1240000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:46:15 2023","interfaces":[{"interface":"back","average":{"1min":0.84399999999999997,"5min":2.1579999999999999,"15min":1.742},"min":{"1min":0.34799999999999998,"5min":0.26300000000000001,"15min":0.25900000000000001},"max":{"1min":3.2709999999999999,"5min":9.673,"15min":18.190999999999999},"last":0.36199999999999999},{"interface":"front","average":{"1min":1,"5min":1.7050000000000001,"15min":1.544},"min":{"1min":0.29299999999999998,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":4.5149999999999997,"5min":8.0730000000000004,"15min":18.173999999999999},"last":0.38100000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:46:15 2023","interfaces":[{"interface":"back","average":{"1min":1.0469999999999999,"5min":1.8180000000000001,"15min":1.679},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":4.7729999999999997,"5min":13.051,"15min":13.051},"last":0.45500000000000002},{"interface":"front","average":{"1min":1.0429999999999999,"5min":2.2090000000000001,"15min":2.032},"min":{"1min":0.311,"5min":0.30299999999999999,"15min":0.28399999999999997},"max":{"1min":3.5800000000000001,"5min":10.945,"15min":14.614000000000001},"last":3.1890000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:46:29 2023","interfaces":[{"interface":"back","average":{"1min":1.4830000000000001,"5min":2.117,"15min":1.853},"min":{"1min":0.32400000000000001,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":5.2800000000000002,"5min":8.6180000000000003,"15min":12.147},"last":0.50900000000000001},{"interface":"front","average":{"1min":1.46,"5min":2.2850000000000001,"15min":2.0019999999999998},"min":{"1min":0.33900000000000002,"5min":0.33000000000000002,"15min":0.30299999999999999},"max":{"1min":7.4960000000000004,"5min":17.169,"15min":17.169},"last":0.41899999999999998}]},{"osd":5,"last update":"Sat Sep 23 20:46:00 2023","interfaces":[{"interface":"back","average":{"1min":0.75800000000000001,"5min":1.5169999999999999,"15min":0.97399999999999998},"min":{"1min":0.316,"5min":0.253,"15min":0.253},"max":{"1min":4.1310000000000002,"5min":10.202,"15min":10.202},"last":0.33300000000000002},{"interface":"front","average":{"1min":1.266,"5min":1.4410000000000001,"15min":1.0680000000000001},"min":{"1min":0.36499999999999999,"5min":0.311,"15min":0.28999999999999998},"max":{"1min":6.5839999999999996,"5min":8.1440000000000001,"15min":8.1440000000000001},"last":3.0139999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:46:15 2023","interfaces":[{"interface":"back","average":{"1min":0.77200000000000002,"5min":1.6279999999999999,"15min":1.206},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.27600000000000002},"max":{"1min":4.2800000000000002,"5min":10.840999999999999,"15min":14.731999999999999},"last":3.222},{"interface":"front","average":{"1min":0.72799999999999998,"5min":1.4490000000000001,"15min":1.0609999999999999},"min":{"1min":0.32500000000000001,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":3.3460000000000001,"5min":7.6150000000000002,"15min":7.6150000000000002},"last":2.786}]},{"osd":7,"last update":"Sat Sep 23 20:46:29 2023","interfaces":[{"interface":"back","average":{"1min":1.417,"5min":5.4790000000000001,"15min":2.552},"min":{"1min":0.35899999999999999,"5min":0.35099999999999998,"15min":0.318},"max":{"1min":4.54,"5min":313.50099999999998,"15min":313.50099999999998},"last":3.153},{"interface":"front","average":{"1min":1.444,"5min":4.8159999999999998,"15min":2.1440000000000001},"min":{"1min":0.35099999999999998,"5min":0.33200000000000002,"15min":0.32700000000000001},"max":{"1min":4.9100000000000001,"5min":312.67500000000001,"15min":312.67500000000001},"last":2.8210000000000002}]}]},{"osd":5,"up_from":39,"seq":167503724979,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2222356,"kb_used_data":1674508,"kb_used_omap":0,"kb_used_meta":547840,"kb_avail":91522796,"statfs":{"total":95995035648,"available":93719343104,"internally_reserved":0,"allocated":1714696192,"data_stored":1713183498,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":560988160},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:46:04 2023","interfaces":[{"interface":"back","average":{"1min":0.65900000000000003,"5min":1.7090000000000001,"15min":1.744},"min":{"1min":0.24199999999999999,"5min":0.23100000000000001,"15min":0.20699999999999999},"max":{"1min":2.0430000000000001,"5min":15.018000000000001,"15min":18.16},"last":0.22900000000000001},{"interface":"front","average":{"1min":1.45,"5min":1.7529999999999999,"15min":1.6890000000000001},"min":{"1min":0.27900000000000003,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":4.6040000000000001,"5min":15.034000000000001,"15min":15.034000000000001},"last":0.33200000000000002}]},{"osd":1,"last update":"Sat Sep 23 20:46:22 2023","interfaces":[{"interface":"back","average":{"1min":0.90900000000000003,"5min":3.0590000000000002,"15min":2.2090000000000001},"min":{"1min":0.34499999999999997,"5min":0.27200000000000002,"15min":0.253},"max":{"1min":2.7370000000000001,"5min":61.005000000000003,"15min":61.005000000000003},"last":0.94599999999999995},{"interface":"front","average":{"1min":1.5509999999999999,"5min":2.5779999999999998,"15min":2.0139999999999998},"min":{"1min":0.40500000000000003,"5min":0.27700000000000002,"15min":0.23499999999999999},"max":{"1min":5.548,"5min":59.688000000000002,"15min":59.688000000000002},"last":0.34999999999999998}]},{"osd":2,"last update":"Sat Sep 23 20:46:04 2023","interfaces":[{"interface":"back","average":{"1min":0.88500000000000001,"5min":2.0209999999999999,"15min":2.0249999999999999},"min":{"1min":0.36099999999999999,"5min":0.27100000000000002,"15min":0.26500000000000001},"max":{"1min":1.796,"5min":12.226000000000001,"15min":37.011000000000003},"last":2.5310000000000001},{"interface":"front","average":{"1min":1.177,"5min":1.829,"15min":1.992},"min":{"1min":0.372,"5min":0.28299999999999997,"15min":0.255},"max":{"1min":3.427,"5min":10.565,"15min":36.835000000000001},"last":2.476}]},{"osd":3,"last update":"Sat Sep 23 20:46:26 2023","interfaces":[{"interface":"back","average":{"1min":1.208,"5min":2.4279999999999999,"15min":2.226},"min":{"1min":0.43099999999999999,"5min":0.33800000000000002,"15min":0.29599999999999999},"max":{"1min":2.9249999999999998,"5min":11.032,"15min":36.716000000000001},"last":1.056},{"interface":"front","average":{"1min":1.431,"5min":2.3170000000000002,"15min":2.202},"min":{"1min":0.46100000000000002,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":3.9470000000000001,"5min":13.266,"15min":18.504000000000001},"last":0.38900000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:45:58 2023","interfaces":[{"interface":"back","average":{"1min":1.2729999999999999,"5min":1.8580000000000001,"15min":1.147},"min":{"1min":0.33500000000000002,"5min":0.27300000000000002,"15min":0.255},"max":{"1min":7.758,"5min":13.391999999999999,"15min":13.391999999999999},"last":0.97999999999999998},{"interface":"front","average":{"1min":1.4830000000000001,"5min":1.613,"15min":1.083},"min":{"1min":0.41199999999999998,"5min":0.30299999999999999,"15min":0.28299999999999997},"max":{"1min":6.6550000000000002,"5min":7.3799999999999999,"15min":7.3799999999999999},"last":0.29199999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:46:10 2023","interfaces":[{"interface":"back","average":{"1min":1.079,"5min":2.1539999999999999,"15min":1.3160000000000001},"min":{"1min":0.36199999999999999,"5min":0.317,"15min":0.30199999999999999},"max":{"1min":4.75,"5min":13.734,"15min":13.734},"last":1.03},{"interface":"front","average":{"1min":0.90000000000000002,"5min":1.478,"15min":1.097},"min":{"1min":0.51900000000000002,"5min":0.317,"15min":0.317},"max":{"1min":2.3460000000000001,"5min":8.7449999999999992,"15min":8.7449999999999992},"last":0.41399999999999998}]},{"osd":7,"last update":"Sat Sep 23 20:46:26 2023","interfaces":[{"interface":"back","average":{"1min":1.361,"5min":4.6550000000000002,"15min":2.0739999999999998},"min":{"1min":0.44400000000000001,"5min":0.36399999999999999,"15min":0.28499999999999998},"max":{"1min":3.8719999999999999,"5min":294.70999999999998,"15min":294.70999999999998},"last":1.3839999999999999},{"interface":"front","average":{"1min":1.3819999999999999,"5min":4.8170000000000002,"15min":2.1760000000000002},"min":{"1min":0.379,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":3.6640000000000001,"5min":294.74400000000003,"15min":294.74400000000003},"last":0.51400000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4722,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1360,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8206,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7156,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3154,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":3441557504,"data_stored":3441557504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1011154944,"data_stored":1011154944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1995898880,"data_stored":1995898880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1680474112,"data_stored":1680474112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2738225152,"data_stored":2738225152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1709047808,"data_stored":1709047808,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2369585152,"data_stored":2369585152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1370030080,"data_stored":1370030080,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:46:43.926 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-09-23T20:46:43.926 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd pool set .mgr pg_num 71 2023-09-23T20:46:43.942 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:46:43.996 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:43 smithi149 ceph-mon[170394]: pgmap v1376: 119 pgs: 119 active+clean; 5.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 61 MiB/s wr, 977 op/s 2023-09-23T20:46:43.996 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:43 smithi149 ceph-mon[170394]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:46:43.996 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:43 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:46:43.996 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:43 smithi149 ceph-mon[170394]: osdmap e543: 8 total, 8 up, 8 in 2023-09-23T20:46:43.996 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:43 smithi149 ceph-mon[170394]: failed to encode map e543 with expected crc 2023-09-23T20:46:43.997 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:43 smithi149 ceph-mon[170394]: failed to encode map e543 with expected crc 2023-09-23T20:46:43.997 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:43 smithi149 ceph-mon[170394]: failed to encode map e543 with expected crc 2023-09-23T20:46:43.997 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:43 smithi149 ceph-mon[170394]: failed to encode map e543 with expected crc 2023-09-23T20:46:44.003 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:44.011 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 59 16 86345 86329 91.4003 55 0.0183851 0.0109347 2023-09-23T20:46:44.385 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:46:44.434 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:46:44.473 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:46:44.528 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:46:44.579 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:46:44.590 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:46:44.598 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:44.599 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:46:44.612 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:46:44.648 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:46:44.691 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:46:44.699 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:46:44.705 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:46:44.705 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:46:44.706 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:46:44.706 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:46:44.706 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:46:44.706 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:46:44.706 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:46:44.706 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:46:44.758 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:46:44.805 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:44.951 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:44 smithi149 ceph-mon[170394]: failed to encode map e543 with expected crc 2023-09-23T20:46:44.951 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:44 smithi149 ceph-mon[170394]: failed to encode map e543 with expected crc 2023-09-23T20:46:44.951 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:44 smithi149 ceph-mon[170394]: from='client.36942 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:46:44.951 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:44 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3245392926' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:46:44.951 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:44 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:46:44.994 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:44 smithi018 ceph-mon[184144]: failed to encode map e543 with expected crc 2023-09-23T20:46:44.994 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:44 smithi018 ceph-mon[184144]: failed to encode map e543 with expected crc 2023-09-23T20:46:44.994 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:44 smithi018 ceph-mon[184144]: from='client.36942 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:46:44.995 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:44 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3245392926' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:46:44.995 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:44 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:46:44.995 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:44 smithi018 ceph-mon[179647]: failed to encode map e543 with expected crc 2023-09-23T20:46:44.995 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:44 smithi018 ceph-mon[179647]: failed to encode map e543 with expected crc 2023-09-23T20:46:44.995 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:44 smithi018 ceph-mon[179647]: from='client.36942 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:46:44.995 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:44 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3245392926' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:46:44.995 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:44 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:46:45.010 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:2023-09-23T20:46:45.010872+0000 min lat: 0.00123458 max lat: 0.182054 avg lat: 0.0110064 2023-09-23T20:46:45.010 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T20:46:45.010 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 60 16 87240 87224 90.8094 55.9375 0.0162925 0.0110064 2023-09-23T20:46:45.103 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:46:45.107 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:45.107 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:46:45.208 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:46:45.250 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:46:44 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:46:44] "GET /metrics HTTP/1.1" 200 36566 "" "Prometheus/2.43.0" 2023-09-23T20:46:45.304 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:45.504 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:46:45.509 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:45.510 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:46:45.540 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T20:46:45.610 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:46:45.705 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:45.783 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:45 smithi149 ceph-mon[170394]: pgmap v1378: 119 pgs: 119 active+clean; 5.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 61 MiB/s wr, 974 op/s 2023-09-23T20:46:45.800 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:46:45.807 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:45.807 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:46:45.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:45 smithi018 ceph-mon[179647]: pgmap v1378: 119 pgs: 119 active+clean; 5.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 61 MiB/s wr, 974 op/s 2023-09-23T20:46:45.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:45 smithi018 ceph-mon[184144]: pgmap v1378: 119 pgs: 119 active+clean; 5.1 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 61 MiB/s wr, 974 op/s 2023-09-23T20:46:45.908 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:46:45.996 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:46.010 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 61 16 88137 88121 90.2399 56.0625 0.020119 0.0110767 2023-09-23T20:46:46.361 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:46:46.375 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:46.375 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:46:46.476 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:46:46.500 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:46:46.524 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:46.588 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:46:46.591 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:46.592 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:46:46.693 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:46:46.729 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:46.848 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:46:46.855 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:46.855 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:46:46.913 INFO:teuthology.orchestra.run.smithi018.stderr:Error EINVAL: specified pg_num 71 < pg_num_max 32 2023-09-23T20:46:46.956 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:46:47.003 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:47.010 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 62 16 88949 88933 89.6032 50.75 0.0344043 0.0111561 2023-09-23T20:46:47.196 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:46:47.276 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:46:47.282 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:47.282 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:46:47.383 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:46:47.407 INFO:teuthology.orchestra.run.smithi149.stdout:ERROR: (22) Invalid argument 2023-09-23T20:46:47.407 INFO:teuthology.orchestra.run.smithi149.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:46:47.444 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:47.540 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:46:47.546 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:47.546 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:46:47.647 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:46:47.689 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:47.690 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-09-23T20:46:47.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:47 smithi018 ceph-mon[184144]: pgmap v1379: 119 pgs: 119 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 59 MiB/s wr, 947 op/s 2023-09-23T20:46:47.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:47 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/450784069' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "71"}]: dispatch 2023-09-23T20:46:47.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:47 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "71"}]: dispatch 2023-09-23T20:46:47.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:47 smithi018 ceph-mon[179647]: pgmap v1379: 119 pgs: 119 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 59 MiB/s wr, 947 op/s 2023-09-23T20:46:47.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:47 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/450784069' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "71"}]: dispatch 2023-09-23T20:46:47.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:47 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "71"}]: dispatch 2023-09-23T20:46:48.013 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 63 16 89877 89861 89.098 58 0.01068 0.0112188 2023-09-23T20:46:48.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:47 smithi149 ceph-mon[170394]: pgmap v1379: 119 pgs: 119 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 59 MiB/s wr, 947 op/s 2023-09-23T20:46:48.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:47 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/450784069' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "71"}]: dispatch 2023-09-23T20:46:48.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:47 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "71"}]: dispatch 2023-09-23T20:46:48.358 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:46:48.370 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:46:48.386 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:48.387 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:46:48.388 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T20:46:48.489 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:46:48.590 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:48.656 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:46:48.668 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:48.668 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:46:48.769 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:46:48.860 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:49.013 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 64 16 90775 90759 88.583 56.125 0.0180676 0.0112841 2023-09-23T20:46:49.211 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:46:49.217 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:49.218 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:46:49.227 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:46:49.318 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:46:49.392 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:49.486 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:46:49.490 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:49.491 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:46:49.591 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:46:49.664 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:49.764 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:46:49.773 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:49.773 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:46:49.852 INFO:teuthology.orchestra.run.smithi018.stderr:nodeep-scrub is set 2023-09-23T20:46:49.875 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:46:49.944 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:46:49.975 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:49.986 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:46:50.013 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 65 16 91704 91688 88.1136 58.0625 0.0184234 0.0113439 2023-09-23T20:46:50.045 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:46:50.079 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:46:50.113 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:46:50.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:49 smithi018 ceph-mon[184144]: pgmap v1380: 119 pgs: 119 active+clean; 5.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 58 MiB/s wr, 930 op/s 2023-09-23T20:46:50.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:49 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3513780695' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:46:50.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:49 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:46:50.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:49 smithi018 ceph-mon[179647]: pgmap v1380: 119 pgs: 119 active+clean; 5.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 58 MiB/s wr, 930 op/s 2023-09-23T20:46:50.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:49 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3513780695' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:46:50.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:49 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:46:50.153 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:46:50.194 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:46:50.206 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:46:50.211 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:50.212 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:46:50.218 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:49 smithi149 ceph-mon[170394]: pgmap v1380: 119 pgs: 119 active+clean; 5.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 58 MiB/s wr, 930 op/s 2023-09-23T20:46:50.218 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:49 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3513780695' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:46:50.218 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:49 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:46:50.247 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:46:50.248 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:46:50.249 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:46:50.249 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:46:50.249 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:46:50.249 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:46:50.249 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:46:50.249 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:46:50.249 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:46:50.313 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:46:50.392 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:50.633 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:46:50.639 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:50.640 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:46:50.713 INFO:teuthology.orchestra.run.smithi149.stdout:{ 2023-09-23T20:46:50.713 INFO:teuthology.orchestra.run.smithi149.stdout: "size": 20, 2023-09-23T20:46:50.713 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 600, 2023-09-23T20:46:50.713 INFO:teuthology.orchestra.run.smithi149.stdout: "ops": [ 2023-09-23T20:46:50.713 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.714 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:84069 8.9 8:9824f39b:::benchmark_data_smithi018_341490_object84068:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e378)", 2023-09-23T20:46:50.714 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:37:13.327117+0000", 2023-09-23T20:46:50.714 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 577.383114385, 2023-09-23T20:46:50.714 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.3519245670000002, 2023-09-23T20:46:50.714 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:46:50.714 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:46:50.714 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:46:50.714 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:46:50.714 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:46:50.714 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 84069 2023-09-23T20:46:50.715 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.715 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:46:50.715 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.715 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:46:50.715 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.327117+0000", 2023-09-23T20:46:50.715 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:50.715 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.715 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.715 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:46:50.716 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.327117+0000", 2023-09-23T20:46:50.716 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:50.716 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.716 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.716 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:46:50.716 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.327116+0000", 2023-09-23T20:46:50.716 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:46:50.716 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.716 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.716 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:46:50.716 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.327148+0000", 2023-09-23T20:46:50.717 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.1741e-05 2023-09-23T20:46:50.717 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.717 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.717 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:46:50.717 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.327160+0000", 2023-09-23T20:46:50.717 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.185e-05 2023-09-23T20:46:50.717 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.717 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.717 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:46:50.717 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.327165+0000", 2023-09-23T20:46:50.718 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.1749999999999996e-06 2023-09-23T20:46:50.718 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.718 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.718 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:46:50.718 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.330726+0000", 2023-09-23T20:46:50.718 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00356121 2023-09-23T20:46:50.718 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.718 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.718 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:46:50.718 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.330874+0000", 2023-09-23T20:46:50.719 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000148097 2023-09-23T20:46:50.719 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.719 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.719 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:46:50.719 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.331021+0000", 2023-09-23T20:46:50.719 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00014648499999999999 2023-09-23T20:46:50.719 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.719 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.719 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:46:50.720 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.334676+0000", 2023-09-23T20:46:50.720 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0036551869999999998 2023-09-23T20:46:50.720 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.720 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.720 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:46:50.720 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.335154+0000", 2023-09-23T20:46:50.720 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00047863799999999998 2023-09-23T20:46:50.720 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.720 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.720 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:46:50.721 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.368539+0000", 2023-09-23T20:46:50.721 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.033384692000000001 2023-09-23T20:46:50.721 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.721 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.721 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:46:50.721 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.368550+0000", 2023-09-23T20:46:50.721 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1178e-05 2023-09-23T20:46:50.721 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.721 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.721 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:46:50.722 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:14.362533+0000", 2023-09-23T20:46:50.722 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.993982649 2023-09-23T20:46:50.722 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.722 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.722 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:46:50.722 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:14.362546+0000", 2023-09-23T20:46:50.722 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.2660000000000001e-05 2023-09-23T20:46:50.722 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.722 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.722 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:46:50.723 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:15.678962+0000", 2023-09-23T20:46:50.723 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.316415911 2023-09-23T20:46:50.723 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.723 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.723 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:46:50.723 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:15.679041+0000", 2023-09-23T20:46:50.723 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.9655999999999998e-05 2023-09-23T20:46:50.723 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:50.723 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:46:50.723 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:50.724 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.724 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.724 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:84081 8.9 8:97fd4d67:::benchmark_data_smithi018_341490_object84080:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e378)", 2023-09-23T20:46:50.724 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:37:13.341077+0000", 2023-09-23T20:46:50.724 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 577.36915431399996, 2023-09-23T20:46:50.724 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.3380652359999998, 2023-09-23T20:46:50.724 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:46:50.724 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:46:50.724 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:46:50.725 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:46:50.725 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:46:50.725 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 84081 2023-09-23T20:46:50.725 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.725 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:46:50.725 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.725 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:46:50.725 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.341077+0000", 2023-09-23T20:46:50.725 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:50.725 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.726 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.726 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:46:50.726 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.341077+0000", 2023-09-23T20:46:50.726 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:50.726 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.726 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.726 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:46:50.726 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.341075+0000", 2023-09-23T20:46:50.726 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999981 2023-09-23T20:46:50.726 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.727 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.727 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:46:50.727 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.341116+0000", 2023-09-23T20:46:50.727 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.1675000000000003e-05 2023-09-23T20:46:50.727 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.727 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.727 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:46:50.727 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.341135+0000", 2023-09-23T20:46:50.727 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.8110000000000001e-05 2023-09-23T20:46:50.728 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.728 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.728 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:46:50.728 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.341141+0000", 2023-09-23T20:46:50.728 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.849e-06 2023-09-23T20:46:50.728 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.728 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.728 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:46:50.728 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.341194+0000", 2023-09-23T20:46:50.728 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.2553000000000003e-05 2023-09-23T20:46:50.729 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.729 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.729 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:46:50.729 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.341313+0000", 2023-09-23T20:46:50.729 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00011935899999999999 2023-09-23T20:46:50.729 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.729 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.729 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:46:50.729 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.341466+0000", 2023-09-23T20:46:50.729 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00015230300000000001 2023-09-23T20:46:50.729 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.730 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.730 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:46:50.730 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.344704+0000", 2023-09-23T20:46:50.730 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0032382510000000002 2023-09-23T20:46:50.730 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.730 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.730 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:46:50.730 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.348055+0000", 2023-09-23T20:46:50.730 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.003350872 2023-09-23T20:46:50.731 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.731 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.731 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:46:50.731 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.368594+0000", 2023-09-23T20:46:50.731 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.020538948000000001 2023-09-23T20:46:50.731 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.731 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.731 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:46:50.731 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:13.368601+0000", 2023-09-23T20:46:50.731 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.6529999999999994e-06 2023-09-23T20:46:50.732 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.732 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.732 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:46:50.732 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:14.362580+0000", 2023-09-23T20:46:50.732 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.99397867299999998 2023-09-23T20:46:50.732 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.732 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.732 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:46:50.732 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:14.362589+0000", 2023-09-23T20:46:50.732 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.9509999999999995e-06 2023-09-23T20:46:50.733 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.733 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.733 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:46:50.733 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:15.679073+0000", 2023-09-23T20:46:50.733 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.316483598 2023-09-23T20:46:50.733 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.733 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.733 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:46:50.733 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:37:15.679142+0000", 2023-09-23T20:46:50.733 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.9389999999999998e-05 2023-09-23T20:46:50.734 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:50.734 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:46:50.734 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:50.734 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.734 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.734 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:105775 8.9 8:97ec68b7:::benchmark_data_smithi018_341490_object16452:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e411)", 2023-09-23T20:46:50.734 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:38:27.408390+0000", 2023-09-23T20:46:50.734 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 503.30184052200002, 2023-09-23T20:46:50.734 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.1501579759999991, 2023-09-23T20:46:50.734 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:46:50.735 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:46:50.735 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:46:50.735 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:46:50.735 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:46:50.735 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 105775 2023-09-23T20:46:50.735 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.735 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:46:50.735 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.735 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:46:50.736 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.408390+0000", 2023-09-23T20:46:50.736 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:50.736 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.736 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.736 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:46:50.736 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.408390+0000", 2023-09-23T20:46:50.736 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:50.736 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.736 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.736 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:46:50.737 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.408389+0000", 2023-09-23T20:46:50.737 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:46:50.737 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.737 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.737 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:46:50.737 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.408397+0000", 2023-09-23T20:46:50.737 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.7009999999999996e-06 2023-09-23T20:46:50.737 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.737 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.737 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:46:50.738 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.408411+0000", 2023-09-23T20:46:50.738 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.4228e-05 2023-09-23T20:46:50.738 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.738 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.738 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:46:50.738 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.408419+0000", 2023-09-23T20:46:50.738 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.5599999999999994e-06 2023-09-23T20:46:50.738 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.738 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.738 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:46:50.739 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.414829+0000", 2023-09-23T20:46:50.739 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0064094740000000001 2023-09-23T20:46:50.739 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.739 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.739 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:46:50.739 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.414843+0000", 2023-09-23T20:46:50.739 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.4561e-05 2023-09-23T20:46:50.739 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.739 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.740 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:46:50.740 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.556066+0000", 2023-09-23T20:46:50.740 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.1412221710000008 2023-09-23T20:46:50.740 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.740 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.740 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:46:50.740 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.558548+0000", 2023-09-23T20:46:50.740 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0024828070000000001 2023-09-23T20:46:50.740 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:50.740 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:46:50.741 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:50.741 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.741 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.741 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:105788 8.9 8:94f5ae14:::benchmark_data_smithi018_341490_object16465:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e411)", 2023-09-23T20:46:50.741 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:38:27.415518+0000", 2023-09-23T20:46:50.741 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 503.29471260600002, 2023-09-23T20:46:50.741 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 10.492815438999999, 2023-09-23T20:46:50.741 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:46:50.741 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:46:50.742 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:46:50.742 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:46:50.742 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:46:50.742 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 105788 2023-09-23T20:46:50.742 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.742 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:46:50.742 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.742 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:46:50.742 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.415518+0000", 2023-09-23T20:46:50.742 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:50.743 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.743 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.743 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:46:50.743 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.415518+0000", 2023-09-23T20:46:50.743 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:50.743 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.743 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.743 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:46:50.743 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.415517+0000", 2023-09-23T20:46:50.743 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:46:50.743 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.744 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.744 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:46:50.744 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.415524+0000", 2023-09-23T20:46:50.744 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.0530000000000001e-06 2023-09-23T20:46:50.744 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.744 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.744 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:46:50.744 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.415539+0000", 2023-09-23T20:46:50.744 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.4847e-05 2023-09-23T20:46:50.745 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.745 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.745 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:46:50.745 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.415548+0000", 2023-09-23T20:46:50.745 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.1600000000000004e-06 2023-09-23T20:46:50.745 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.745 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.745 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:46:50.745 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.423501+0000", 2023-09-23T20:46:50.745 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0079528459999999995 2023-09-23T20:46:50.746 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.746 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.746 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:46:50.746 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.423513+0000", 2023-09-23T20:46:50.746 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.2523e-05 2023-09-23T20:46:50.746 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.746 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.747 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:46:50.747 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.559074+0000", 2023-09-23T20:46:50.747 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.1355603290000005 2023-09-23T20:46:50.747 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.747 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.747 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for degraded object", 2023-09-23T20:46:50.747 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.567834+0000", 2023-09-23T20:46:50.747 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0087603060000000007 2023-09-23T20:46:50.747 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.748 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.748 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:46:50.748 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.866509+0000", 2023-09-23T20:46:50.748 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.2986752469999998 2023-09-23T20:46:50.748 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.748 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.748 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:46:50.748 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.869596+0000", 2023-09-23T20:46:50.748 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0030866829999999998 2023-09-23T20:46:50.749 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.749 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.749 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:46:50.749 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.875831+0000", 2023-09-23T20:46:50.749 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0062352570000000001 2023-09-23T20:46:50.749 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.749 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.749 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:46:50.750 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.893196+0000", 2023-09-23T20:46:50.750 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.017365463000000001 2023-09-23T20:46:50.750 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.750 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.750 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:46:50.750 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.898820+0000", 2023-09-23T20:46:50.750 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0056230400000000002 2023-09-23T20:46:50.750 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.751 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.751 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:46:50.751 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.899854+0000", 2023-09-23T20:46:50.751 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0010348670000000001 2023-09-23T20:46:50.751 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.751 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.751 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:46:50.751 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.901497+0000", 2023-09-23T20:46:50.751 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0016428160000000001 2023-09-23T20:46:50.752 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.752 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.752 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:46:50.752 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.908334+0000", 2023-09-23T20:46:50.752 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0068366219999999997 2023-09-23T20:46:50.752 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:50.752 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:46:50.752 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:50.752 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.753 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.753 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:105815 8.19 8:9f25a704:::benchmark_data_smithi018_341490_object16492:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e411)", 2023-09-23T20:46:50.753 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:38:27.621206+0000", 2023-09-23T20:46:50.753 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 503.089025386, 2023-09-23T20:46:50.753 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 10.285912701000001, 2023-09-23T20:46:50.753 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:46:50.753 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:46:50.753 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:46:50.753 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:46:50.754 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:46:50.754 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 105815 2023-09-23T20:46:50.754 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.754 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:46:50.754 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.754 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:46:50.754 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.621206+0000", 2023-09-23T20:46:50.754 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:50.754 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.755 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.755 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:46:50.755 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.621206+0000", 2023-09-23T20:46:50.755 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:50.755 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.755 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.755 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:46:50.755 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.621204+0000", 2023-09-23T20:46:50.755 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999981 2023-09-23T20:46:50.755 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.756 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.756 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:46:50.756 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.621213+0000", 2023-09-23T20:46:50.756 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.6399999999999992e-06 2023-09-23T20:46:50.756 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.756 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.756 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:46:50.756 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.621236+0000", 2023-09-23T20:46:50.756 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.2016000000000001e-05 2023-09-23T20:46:50.757 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.757 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.757 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:46:50.757 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.621244+0000", 2023-09-23T20:46:50.757 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.7339999999999997e-06 2023-09-23T20:46:50.757 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.757 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.757 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:46:50.757 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.621311+0000", 2023-09-23T20:46:50.758 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.7037000000000002e-05 2023-09-23T20:46:50.758 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.758 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.758 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:46:50.758 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.621338+0000", 2023-09-23T20:46:50.758 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.6506999999999999e-05 2023-09-23T20:46:50.758 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.758 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.758 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:46:50.758 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.559059+0000", 2023-09-23T20:46:50.759 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.9377208670000003 2023-09-23T20:46:50.759 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.759 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.759 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for degraded object", 2023-09-23T20:46:50.759 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.566949+0000", 2023-09-23T20:46:50.759 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0078899190000000004 2023-09-23T20:46:50.759 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.759 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.759 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:46:50.759 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.857769+0000", 2023-09-23T20:46:50.760 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.290820509 2023-09-23T20:46:50.760 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.760 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.760 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:46:50.760 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.860457+0000", 2023-09-23T20:46:50.760 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0026880620000000002 2023-09-23T20:46:50.760 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.760 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.760 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:46:50.761 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.865930+0000", 2023-09-23T20:46:50.761 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0054732979999999997 2023-09-23T20:46:50.761 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.761 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.761 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:46:50.761 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.881728+0000", 2023-09-23T20:46:50.761 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.015797813000000001 2023-09-23T20:46:50.761 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.761 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.762 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:46:50.762 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.897756+0000", 2023-09-23T20:46:50.762 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.016027501 2023-09-23T20:46:50.762 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.762 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.762 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:46:50.762 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.898850+0000", 2023-09-23T20:46:50.762 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.001094292 2023-09-23T20:46:50.762 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.763 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.763 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:46:50.763 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.900364+0000", 2023-09-23T20:46:50.763 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0015141250000000001 2023-09-23T20:46:50.763 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.763 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.763 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:46:50.763 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.907118+0000", 2023-09-23T20:46:50.763 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0067541490000000001 2023-09-23T20:46:50.763 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:50.764 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:46:50.764 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:50.764 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.764 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.764 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:105818 8.19 8:9e8d5bb0:::benchmark_data_smithi018_341490_object16495:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e411)", 2023-09-23T20:46:50.764 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:38:27.629200+0000", 2023-09-23T20:46:50.764 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 503.08103130500001, 2023-09-23T20:46:50.764 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 10.286819241, 2023-09-23T20:46:50.765 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:46:50.765 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:46:50.765 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:46:50.765 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:46:50.765 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:46:50.765 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 105818 2023-09-23T20:46:50.765 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.765 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:46:50.765 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.766 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:46:50.766 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.629200+0000", 2023-09-23T20:46:50.766 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:50.766 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.766 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.766 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:46:50.766 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.629200+0000", 2023-09-23T20:46:50.766 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:50.767 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.767 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.767 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:46:50.767 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.629198+0000", 2023-09-23T20:46:50.767 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999981 2023-09-23T20:46:50.767 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.767 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.767 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:46:50.767 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.629208+0000", 2023-09-23T20:46:50.767 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.9119999999999993e-06 2023-09-23T20:46:50.768 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.768 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.768 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:46:50.768 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.629231+0000", 2023-09-23T20:46:50.768 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.3258999999999999e-05 2023-09-23T20:46:50.768 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.768 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.768 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:46:50.768 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.629240+0000", 2023-09-23T20:46:50.769 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.5719999999999992e-06 2023-09-23T20:46:50.769 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.769 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.769 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:46:50.769 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.629296+0000", 2023-09-23T20:46:50.769 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.5918999999999998e-05 2023-09-23T20:46:50.769 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.769 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.769 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:46:50.770 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:27.629305+0000", 2023-09-23T20:46:50.770 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.0909999999999999e-06 2023-09-23T20:46:50.770 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.770 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.770 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:46:50.770 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.567292+0000", 2023-09-23T20:46:50.770 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.937986414 2023-09-23T20:46:50.770 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.770 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.771 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for degraded object", 2023-09-23T20:46:50.771 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.576261+0000", 2023-09-23T20:46:50.771 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0089692720000000004 2023-09-23T20:46:50.771 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.771 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.771 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:46:50.771 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.882072+0000", 2023-09-23T20:46:50.771 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.3058112469999998 2023-09-23T20:46:50.771 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.772 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.772 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:46:50.772 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.884583+0000", 2023-09-23T20:46:50.772 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0025109939999999999 2023-09-23T20:46:50.772 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.772 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.772 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:46:50.772 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.889724+0000", 2023-09-23T20:46:50.772 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0051411909999999998 2023-09-23T20:46:50.773 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.773 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.773 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:46:50.773 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.894676+0000", 2023-09-23T20:46:50.773 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0049515119999999999 2023-09-23T20:46:50.773 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.773 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.773 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:46:50.776 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:46:50.776 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.908009+0000", 2023-09-23T20:46:50.776 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.013332899 2023-09-23T20:46:50.776 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.776 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.776 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:46:50.776 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.908929+0000", 2023-09-23T20:46:50.776 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00092006300000000004 2023-09-23T20:46:50.777 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.777 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.777 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:46:50.777 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.910490+0000", 2023-09-23T20:46:50.777 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.001560931 2023-09-23T20:46:50.777 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.777 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.778 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:46:50.778 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.916019+0000", 2023-09-23T20:46:50.778 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0055290570000000004 2023-09-23T20:46:50.778 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:50.778 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:46:50.778 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:50.778 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.778 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.779 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "MOSDPGPushReply(8.19 413/410 [PushReplyOp(8:9f25a704:::benchmark_data_smithi018_341490_object16492:head)])", 2023-09-23T20:46:50.779 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:38:35.569213+0000", 2023-09-23T20:46:50.779 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 495.14101801800001, 2023-09-23T20:46:50.779 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.2881636309999998, 2023-09-23T20:46:50.779 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:46:50.779 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:46:50.779 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:46:50.779 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.779 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:46:50.779 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.569213+0000", 2023-09-23T20:46:50.780 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:50.780 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.780 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.780 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:46:50.780 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.569213+0000", 2023-09-23T20:46:50.780 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:50.780 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.780 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.780 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:46:50.781 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.569213+0000", 2023-09-23T20:46:50.781 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:46:50.781 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.781 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.781 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:46:50.781 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.569217+0000", 2023-09-23T20:46:50.781 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.5480000000000003e-06 2023-09-23T20:46:50.781 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.781 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.782 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:46:50.782 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.569231+0000", 2023-09-23T20:46:50.782 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.3505e-05 2023-09-23T20:46:50.782 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.782 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.782 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:46:50.782 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.569241+0000", 2023-09-23T20:46:50.782 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0695e-05 2023-09-23T20:46:50.782 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.783 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.783 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:46:50.783 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.845386+0000", 2023-09-23T20:46:50.783 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.2761449049999998 2023-09-23T20:46:50.783 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.783 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.783 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:46:50.783 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.857377+0000", 2023-09-23T20:46:50.783 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.011990341999999999 2023-09-23T20:46:50.783 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:50.784 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:46:50.784 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:50.784 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.784 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.784 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "MOSDPGPushReply(8.9 413/411 [PushReplyOp(8:94f5ae14:::benchmark_data_smithi018_341490_object16465:head)])", 2023-09-23T20:46:50.784 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:38:35.570981+0000", 2023-09-23T20:46:50.784 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 495.139249834, 2023-09-23T20:46:50.784 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.295177131, 2023-09-23T20:46:50.784 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:46:50.785 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:46:50.785 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:46:50.785 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.785 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:46:50.785 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.570981+0000", 2023-09-23T20:46:50.785 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:50.785 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.785 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.785 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:46:50.786 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.570981+0000", 2023-09-23T20:46:50.786 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:50.786 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.786 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.786 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:46:50.786 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.570981+0000", 2023-09-23T20:46:50.786 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:46:50.786 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.786 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.787 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:46:50.787 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.570985+0000", 2023-09-23T20:46:50.787 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.2690000000000001e-06 2023-09-23T20:46:50.787 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.787 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.787 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:46:50.787 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.570995+0000", 2023-09-23T20:46:50.787 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.9799999999999993e-06 2023-09-23T20:46:50.787 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.787 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.788 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:46:50.788 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.570999+0000", 2023-09-23T20:46:50.788 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.5739999999999997e-06 2023-09-23T20:46:50.788 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.788 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.788 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:46:50.788 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.852442+0000", 2023-09-23T20:46:50.788 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.2814426320000001 2023-09-23T20:46:50.788 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.789 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.789 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:46:50.789 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.866158+0000", 2023-09-23T20:46:50.789 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.013716178000000001 2023-09-23T20:46:50.789 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:50.789 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:46:50.789 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:50.789 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.789 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.789 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "MOSDPGPushReply(8.19 413/410 [PushReplyOp(8:9e8d5bb0:::benchmark_data_smithi018_341490_object16495:head)])", 2023-09-23T20:46:50.790 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:38:35.577963+0000", 2023-09-23T20:46:50.790 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 495.13226767700002, 2023-09-23T20:46:50.790 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.303767224, 2023-09-23T20:46:50.790 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:46:50.790 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:46:50.790 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:46:50.790 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.790 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:46:50.790 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.577963+0000", 2023-09-23T20:46:50.790 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:50.791 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.791 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.791 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:46:50.791 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.577963+0000", 2023-09-23T20:46:50.791 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:50.791 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.791 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.791 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:46:50.791 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.577963+0000", 2023-09-23T20:46:50.792 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:46:50.792 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.792 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.792 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:46:50.792 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.577968+0000", 2023-09-23T20:46:50.792 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.13e-06 2023-09-23T20:46:50.792 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.792 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.793 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:46:50.793 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.577980+0000", 2023-09-23T20:46:50.793 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1749000000000001e-05 2023-09-23T20:46:50.793 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.793 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.793 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:46:50.793 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.577986+0000", 2023-09-23T20:46:50.793 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.8210000000000004e-06 2023-09-23T20:46:50.793 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.794 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.794 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:46:50.794 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.869571+0000", 2023-09-23T20:46:50.794 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.2915846719999999 2023-09-23T20:46:50.794 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.794 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.794 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:46:50.794 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:37.881731+0000", 2023-09-23T20:46:50.794 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.012159467 2023-09-23T20:46:50.795 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:50.795 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:46:50.795 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:50.795 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.795 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.795 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.35994.0:106043 8.9 8:949649ba:::benchmark_data_smithi018_341490_object16720:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e414)", 2023-09-23T20:46:50.795 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:38:35.582779+0000", 2023-09-23T20:46:50.795 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 495.12745198300001, 2023-09-23T20:46:50.795 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.7561004659999999, 2023-09-23T20:46:50.795 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:46:50.796 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:46:50.796 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:46:50.796 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.35994", 2023-09-23T20:46:50.796 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/2275432307", 2023-09-23T20:46:50.796 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 106043 2023-09-23T20:46:50.796 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.796 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:46:50.796 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.796 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:46:50.796 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.582779+0000", 2023-09-23T20:46:50.797 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:50.797 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.797 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.797 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:46:50.797 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.582779+0000", 2023-09-23T20:46:50.797 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:50.797 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.797 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.797 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:46:50.797 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.582778+0000", 2023-09-23T20:46:50.798 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:46:50.798 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.798 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.798 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:46:50.798 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.582784+0000", 2023-09-23T20:46:50.798 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.0050000000000001e-06 2023-09-23T20:46:50.798 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.798 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.798 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:46:50.799 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.582798+0000", 2023-09-23T20:46:50.799 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.3788000000000001e-05 2023-09-23T20:46:50.799 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.799 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.799 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:46:50.799 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:35.582804+0000", 2023-09-23T20:46:50.799 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.6760000000000001e-06 2023-09-23T20:46:50.799 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.799 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.799 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:46:50.800 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.238120+0000", 2023-09-23T20:46:50.800 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.655315308 2023-09-23T20:46:50.800 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.800 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.800 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for degraded object", 2023-09-23T20:46:50.800 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.246651+0000", 2023-09-23T20:46:50.800 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0085312070000000007 2023-09-23T20:46:50.800 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.801 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.801 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:46:50.801 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.290876+0000", 2023-09-23T20:46:50.801 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.044225546999999997 2023-09-23T20:46:50.801 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.801 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.801 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:46:50.801 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.294002+0000", 2023-09-23T20:46:50.801 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0031256779999999998 2023-09-23T20:46:50.802 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.802 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.802 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:46:50.802 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.300293+0000", 2023-09-23T20:46:50.802 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0062908959999999998 2023-09-23T20:46:50.802 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.802 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.802 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:46:50.803 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.328008+0000", 2023-09-23T20:46:50.803 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.027715281000000001 2023-09-23T20:46:50.803 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.803 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.803 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:46:50.803 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.329088+0000", 2023-09-23T20:46:50.803 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.001079775 2023-09-23T20:46:50.803 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.803 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.803 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:46:50.803 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.330303+0000", 2023-09-23T20:46:50.804 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0012148199999999999 2023-09-23T20:46:50.804 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.804 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.804 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:46:50.804 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.332003+0000", 2023-09-23T20:46:50.804 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0016998580000000001 2023-09-23T20:46:50.804 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.804 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.804 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:46:50.804 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:38:38.338879+0000", 2023-09-23T20:46:50.804 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0068769119999999998 2023-09-23T20:46:50.805 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:50.805 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:46:50.805 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:50.805 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.805 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.805 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "MOSDPGPush(8.19 427/410 [PushOp(8:9f37acf0:::benchmark_data_smithi018_341490_object84819:head, version: 394'5338, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(8:9f37acf0:::benchmark_data_smithi018_341490_object84819:head@394'5338, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-09-23T20:46:50.805 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:39:26.795989+0000", 2023-09-23T20:46:50.805 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 443.914241817, 2023-09-23T20:46:50.805 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 18.381641347999999, 2023-09-23T20:46:50.805 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:46:50.806 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "started", 2023-09-23T20:46:50.806 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:46:50.806 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.806 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:46:50.806 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:26.795989+0000", 2023-09-23T20:46:50.806 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:50.806 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.806 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.806 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:46:50.807 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:26.795989+0000", 2023-09-23T20:46:50.807 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:50.807 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.807 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.807 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:46:50.807 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:26.795988+0000", 2023-09-23T20:46:50.807 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:46:50.807 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.807 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.807 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:46:50.807 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:26.796034+0000", 2023-09-23T20:46:50.808 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.5374999999999998e-05 2023-09-23T20:46:50.808 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.808 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.808 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:46:50.808 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:26.796055+0000", 2023-09-23T20:46:50.808 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.126e-05 2023-09-23T20:46:50.808 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.808 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.808 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:46:50.808 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:26.796067+0000", 2023-09-23T20:46:50.808 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.2043e-05 2023-09-23T20:46:50.809 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.809 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.809 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:46:50.809 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:45.167841+0000", 2023-09-23T20:46:50.809 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 18.371773717 2023-09-23T20:46:50.809 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.809 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.809 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:46:50.809 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:45.168079+0000", 2023-09-23T20:46:50.809 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000237839 2023-09-23T20:46:50.809 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.809 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.810 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:46:50.810 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:45.177631+0000", 2023-09-23T20:46:50.810 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0095519390000000006 2023-09-23T20:46:50.810 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:50.810 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:46:50.810 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:50.810 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.810 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.810 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "MOSDPGPush(8.19 435/410 [PushOp(8:9a33faf0:::benchmark_data_smithi018_341490_object85471:head, version: 395'5385, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(8:9a33faf0:::benchmark_data_smithi018_341490_object85471:head@395'5385, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-09-23T20:46:50.811 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:39:53.671016+0000", 2023-09-23T20:46:50.811 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 417.039214958, 2023-09-23T20:46:50.811 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.82055882099999999, 2023-09-23T20:46:50.811 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:46:50.811 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "started", 2023-09-23T20:46:50.811 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:46:50.811 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.811 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:46:50.811 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:53.671016+0000", 2023-09-23T20:46:50.812 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:50.812 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.812 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.812 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:46:50.812 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:53.671016+0000", 2023-09-23T20:46:50.812 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:50.812 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.812 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.812 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:46:50.812 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:53.671015+0000", 2023-09-23T20:46:50.813 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:46:50.813 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.813 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.813 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:46:50.813 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:53.671046+0000", 2023-09-23T20:46:50.813 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.0802000000000001e-05 2023-09-23T20:46:50.813 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.813 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.813 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:46:50.813 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:53.671061+0000", 2023-09-23T20:46:50.814 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.5446000000000001e-05 2023-09-23T20:46:50.814 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.814 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.814 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:46:50.814 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:53.671069+0000", 2023-09-23T20:46:50.814 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.9519999999999994e-06 2023-09-23T20:46:50.814 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.814 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.814 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:46:50.814 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:54.483764+0000", 2023-09-23T20:46:50.815 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.812694429 2023-09-23T20:46:50.815 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.815 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.815 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:46:50.815 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:54.483953+0000", 2023-09-23T20:46:50.815 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00018980099999999999 2023-09-23T20:46:50.815 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.815 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.815 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:46:50.816 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:54.491575+0000", 2023-09-23T20:46:50.816 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0076214999999999998 2023-09-23T20:46:50.816 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:50.816 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:46:50.816 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:50.816 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.816 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.816 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:76949 9.b 9:dc3199d0:::benchmark_data_smithi018_381516_object76948:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e465)", 2023-09-23T20:46:50.816 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:41:58.481021+0000", 2023-09-23T20:46:50.816 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 292.229210008, 2023-09-23T20:46:50.817 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.012473631, 2023-09-23T20:46:50.817 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:46:50.817 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:46:50.817 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:46:50.817 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:46:50.817 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:46:50.817 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 76949 2023-09-23T20:46:50.817 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.817 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:46:50.817 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.818 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:46:50.818 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.481021+0000", 2023-09-23T20:46:50.818 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:50.818 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.818 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.818 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:46:50.818 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.481021+0000", 2023-09-23T20:46:50.818 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:50.818 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.819 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.819 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:46:50.819 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.481020+0000", 2023-09-23T20:46:50.819 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:46:50.819 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.819 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.819 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:46:50.819 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.481102+0000", 2023-09-23T20:46:50.819 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.2538000000000002e-05 2023-09-23T20:46:50.819 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.819 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.820 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:46:50.820 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.481122+0000", 2023-09-23T20:46:50.820 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.9556e-05 2023-09-23T20:46:50.820 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.820 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.820 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:46:50.820 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.481129+0000", 2023-09-23T20:46:50.820 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.7449999999999995e-06 2023-09-23T20:46:50.820 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.821 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.821 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:46:50.821 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.481713+0000", 2023-09-23T20:46:50.821 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000583167 2023-09-23T20:46:50.821 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.821 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.821 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:46:50.821 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.481726+0000", 2023-09-23T20:46:50.822 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.3291e-05 2023-09-23T20:46:50.822 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.822 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.822 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:46:50.822 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.467637+0000", 2023-09-23T20:46:50.822 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.98591135500000004 2023-09-23T20:46:50.822 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.822 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.822 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:46:50.822 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.470079+0000", 2023-09-23T20:46:50.823 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0024414089999999999 2023-09-23T20:46:50.823 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.823 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.823 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:46:50.823 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.470267+0000", 2023-09-23T20:46:50.823 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00018824299999999999 2023-09-23T20:46:50.823 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.823 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.823 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:46:50.823 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.481273+0000", 2023-09-23T20:46:50.824 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.011006395 2023-09-23T20:46:50.824 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.824 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.824 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:46:50.824 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.493371+0000", 2023-09-23T20:46:50.824 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.012098138 2023-09-23T20:46:50.824 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.824 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.824 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:46:50.824 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.493433+0000", 2023-09-23T20:46:50.825 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.1437000000000001e-05 2023-09-23T20:46:50.825 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.825 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.825 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:46:50.825 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.493472+0000", 2023-09-23T20:46:50.825 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.9038000000000001e-05 2023-09-23T20:46:50.825 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.825 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.825 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:46:50.825 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.493495+0000", 2023-09-23T20:46:50.826 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.2778000000000001e-05 2023-09-23T20:46:50.826 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:50.826 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:46:50.826 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:50.826 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.826 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.826 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:76967 9.b 9:d898a916:::benchmark_data_smithi018_381516_object76966:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e465)", 2023-09-23T20:46:50.826 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:41:58.507229+0000", 2023-09-23T20:46:50.826 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 292.20300159800001, 2023-09-23T20:46:50.827 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.98635601299999998, 2023-09-23T20:46:50.827 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:46:50.827 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:46:50.827 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:46:50.827 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:46:50.827 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:46:50.827 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 76967 2023-09-23T20:46:50.827 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.827 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:46:50.827 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.827 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:46:50.828 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.507229+0000", 2023-09-23T20:46:50.828 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:50.828 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.828 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.828 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:46:50.828 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.507229+0000", 2023-09-23T20:46:50.828 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:50.828 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.828 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.829 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:46:50.829 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.507228+0000", 2023-09-23T20:46:50.829 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999981 2023-09-23T20:46:50.829 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.829 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.829 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:46:50.829 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.507285+0000", 2023-09-23T20:46:50.829 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.7531000000000002e-05 2023-09-23T20:46:50.829 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.830 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.830 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:46:50.830 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.507307+0000", 2023-09-23T20:46:50.830 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.2039e-05 2023-09-23T20:46:50.830 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.830 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.830 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:46:50.830 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.507314+0000", 2023-09-23T20:46:50.831 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.7089999999999996e-06 2023-09-23T20:46:50.831 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.831 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.831 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:46:50.831 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.507629+0000", 2023-09-23T20:46:50.831 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00031477799999999999 2023-09-23T20:46:50.831 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.831 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.832 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:46:50.832 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.507641+0000", 2023-09-23T20:46:50.832 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.258e-05 2023-09-23T20:46:50.832 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.832 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.832 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:46:50.832 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.472754+0000", 2023-09-23T20:46:50.832 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.96511288200000001 2023-09-23T20:46:50.833 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.833 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.833 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:46:50.833 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.472887+0000", 2023-09-23T20:46:50.833 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000133132 2023-09-23T20:46:50.833 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.833 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.833 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:46:50.833 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.472997+0000", 2023-09-23T20:46:50.834 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000109805 2023-09-23T20:46:50.834 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.834 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.834 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:46:50.834 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.484044+0000", 2023-09-23T20:46:50.834 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.011047250999999999 2023-09-23T20:46:50.834 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.834 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.834 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:46:50.835 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.493403+0000", 2023-09-23T20:46:50.835 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0093591230000000004 2023-09-23T20:46:50.835 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.835 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.835 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:46:50.835 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.493533+0000", 2023-09-23T20:46:50.835 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000129271 2023-09-23T20:46:50.835 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.836 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.836 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:46:50.836 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.493558+0000", 2023-09-23T20:46:50.836 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.5477e-05 2023-09-23T20:46:50.836 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.836 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.836 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:46:50.836 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.493585+0000", 2023-09-23T20:46:50.836 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.7274e-05 2023-09-23T20:46:50.837 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:50.837 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:46:50.837 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:50.837 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.837 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.837 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:76968 9.b 9:d3229d21:::benchmark_data_smithi018_381516_object76967:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e465)", 2023-09-23T20:46:50.837 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:41:58.507386+0000", 2023-09-23T20:46:50.837 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 292.20284466499999, 2023-09-23T20:46:50.838 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.98627873399999999, 2023-09-23T20:46:50.838 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:46:50.838 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:46:50.838 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:46:50.838 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:46:50.838 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:46:50.838 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 76968 2023-09-23T20:46:50.838 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.838 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:46:50.839 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.839 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:46:50.839 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.507386+0000", 2023-09-23T20:46:50.839 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:50.839 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.839 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.839 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:46:50.839 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.507386+0000", 2023-09-23T20:46:50.840 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:50.840 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.840 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.840 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:46:50.840 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.507386+0000", 2023-09-23T20:46:50.840 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:46:50.840 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.840 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.840 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:46:50.841 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.507447+0000", 2023-09-23T20:46:50.841 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.1439999999999995e-05 2023-09-23T20:46:50.841 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.841 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.841 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:46:50.841 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.507457+0000", 2023-09-23T20:46:50.841 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.4679999999999998e-06 2023-09-23T20:46:50.841 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.841 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.841 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:46:50.841 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.507463+0000", 2023-09-23T20:46:50.842 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.8440000000000003e-06 2023-09-23T20:46:50.842 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.842 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.842 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:46:50.842 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.507659+0000", 2023-09-23T20:46:50.842 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000196691 2023-09-23T20:46:50.842 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.842 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.842 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:46:50.843 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.507663+0000", 2023-09-23T20:46:50.843 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.0040000000000001e-06 2023-09-23T20:46:50.843 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.843 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.843 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:46:50.843 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.475638+0000", 2023-09-23T20:46:50.843 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.96797460700000004 2023-09-23T20:46:50.843 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.843 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.843 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:46:50.844 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.475775+0000", 2023-09-23T20:46:50.844 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00013739899999999999 2023-09-23T20:46:50.844 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.844 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.844 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:46:50.844 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.475934+0000", 2023-09-23T20:46:50.844 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00015842499999999999 2023-09-23T20:46:50.844 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.844 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.844 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:46:50.845 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.484048+0000", 2023-09-23T20:46:50.845 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0081147540000000001 2023-09-23T20:46:50.845 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.845 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.845 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:46:50.845 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.493516+0000", 2023-09-23T20:46:50.845 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0094679240000000008 2023-09-23T20:46:50.845 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.845 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.846 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:46:50.846 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.493605+0000", 2023-09-23T20:46:50.846 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.8725000000000002e-05 2023-09-23T20:46:50.846 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.846 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.846 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:46:50.846 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.493633+0000", 2023-09-23T20:46:50.846 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.8e-05 2023-09-23T20:46:50.846 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.846 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.847 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:46:50.847 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.493665+0000", 2023-09-23T20:46:50.847 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.2017000000000002e-05 2023-09-23T20:46:50.847 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:50.847 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:46:50.847 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:50.847 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.847 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.847 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:76972 9.b 9:d979209f:::benchmark_data_smithi018_381516_object76971:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e465)", 2023-09-23T20:46:50.848 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:41:58.507469+0000", 2023-09-23T20:46:50.848 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 292.20276206599999, 2023-09-23T20:46:50.848 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.98628621599999999, 2023-09-23T20:46:50.848 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:46:50.848 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:46:50.848 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:46:50.848 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:46:50.848 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:46:50.848 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 76972 2023-09-23T20:46:50.848 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.849 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:46:50.849 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.849 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:46:50.849 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.507469+0000", 2023-09-23T20:46:50.849 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:50.849 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.850 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.850 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:46:50.850 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.507469+0000", 2023-09-23T20:46:50.850 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:50.850 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.850 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.850 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:46:50.851 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.507468+0000", 2023-09-23T20:46:50.851 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:46:50.851 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.851 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.851 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:46:50.851 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.507525+0000", 2023-09-23T20:46:50.851 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.6431000000000002e-05 2023-09-23T20:46:50.851 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.851 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.852 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:46:50.852 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.507533+0000", 2023-09-23T20:46:50.852 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.2400000000000007e-06 2023-09-23T20:46:50.852 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.852 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.852 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:46:50.852 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.507538+0000", 2023-09-23T20:46:50.853 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.2129999999999999e-06 2023-09-23T20:46:50.853 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.853 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.853 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:46:50.853 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.507674+0000", 2023-09-23T20:46:50.853 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00013521599999999999 2023-09-23T20:46:50.853 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.853 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.853 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:46:50.854 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.507677+0000", 2023-09-23T20:46:50.854 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.3409999999999999e-06 2023-09-23T20:46:50.854 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.854 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.854 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:46:50.854 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.478437+0000", 2023-09-23T20:46:50.854 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.97076035400000005 2023-09-23T20:46:50.854 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.855 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.855 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:46:50.855 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.478570+0000", 2023-09-23T20:46:50.855 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00013248599999999999 2023-09-23T20:46:50.855 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.855 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.855 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:46:50.855 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.478661+0000", 2023-09-23T20:46:50.855 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.1013000000000004e-05 2023-09-23T20:46:50.856 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.856 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.856 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:46:50.856 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.486788+0000", 2023-09-23T20:46:50.856 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0081275240000000006 2023-09-23T20:46:50.856 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.856 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.856 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:46:50.857 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.493688+0000", 2023-09-23T20:46:50.857 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0068995159999999996 2023-09-23T20:46:50.857 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.857 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.857 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:46:50.857 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.493704+0000", 2023-09-23T20:46:50.857 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.6441999999999999e-05 2023-09-23T20:46:50.857 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.858 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.858 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:46:50.858 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.493734+0000", 2023-09-23T20:46:50.858 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.9258999999999999e-05 2023-09-23T20:46:50.858 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.858 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.858 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:46:50.858 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.493755+0000", 2023-09-23T20:46:50.858 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.1636000000000001e-05 2023-09-23T20:46:50.859 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:50.859 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:46:50.859 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:50.859 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.859 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.859 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:76982 9.b 9:db90636b:::benchmark_data_smithi018_381516_object76981:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e465)", 2023-09-23T20:46:50.859 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:41:58.524610+0000", 2023-09-23T20:46:50.859 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 292.18562135500002, 2023-09-23T20:46:50.859 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.97237829799999997, 2023-09-23T20:46:50.859 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:46:50.860 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:46:50.860 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:46:50.860 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:46:50.860 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:46:50.860 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 76982 2023-09-23T20:46:50.860 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.860 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:46:50.860 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.860 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:46:50.860 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.524610+0000", 2023-09-23T20:46:50.861 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:50.861 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.861 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.861 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:46:50.861 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.524610+0000", 2023-09-23T20:46:50.861 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:50.861 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.861 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.861 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:46:50.861 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.524608+0000", 2023-09-23T20:46:50.862 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:46:50.862 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.862 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.862 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:46:50.862 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.524642+0000", 2023-09-23T20:46:50.862 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.4152000000000001e-05 2023-09-23T20:46:50.862 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.862 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.862 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:46:50.862 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.524659+0000", 2023-09-23T20:46:50.863 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.7108e-05 2023-09-23T20:46:50.863 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.863 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.863 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:46:50.863 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.524666+0000", 2023-09-23T20:46:50.863 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.0489999999999998e-06 2023-09-23T20:46:50.863 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.863 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.863 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:46:50.863 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.524731+0000", 2023-09-23T20:46:50.863 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.4169000000000002e-05 2023-09-23T20:46:50.864 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.864 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.864 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:46:50.864 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.524743+0000", 2023-09-23T20:46:50.864 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.2434e-05 2023-09-23T20:46:50.864 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.864 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.864 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:46:50.864 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.481305+0000", 2023-09-23T20:46:50.865 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.95656189599999997 2023-09-23T20:46:50.865 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.865 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.865 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:46:50.865 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.481433+0000", 2023-09-23T20:46:50.865 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000128059 2023-09-23T20:46:50.865 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.865 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.865 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:46:50.865 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.481584+0000", 2023-09-23T20:46:50.866 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00015048899999999999 2023-09-23T20:46:50.866 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.866 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.866 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:46:50.866 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.493780+0000", 2023-09-23T20:46:50.866 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.012196716 2023-09-23T20:46:50.866 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.866 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.866 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:46:50.867 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.493806+0000", 2023-09-23T20:46:50.867 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.6188999999999999e-05 2023-09-23T20:46:50.867 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.867 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.867 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:46:50.867 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.496920+0000", 2023-09-23T20:46:50.867 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.003113111 2023-09-23T20:46:50.867 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.867 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.867 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:46:50.868 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.496963+0000", 2023-09-23T20:46:50.868 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.3648e-05 2023-09-23T20:46:50.868 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.868 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.868 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:46:50.868 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.496988+0000", 2023-09-23T20:46:50.868 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.4782000000000001e-05 2023-09-23T20:46:50.868 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:50.868 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:46:50.869 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:50.869 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.869 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.869 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:77009 9.b 9:db9e621f:::benchmark_data_smithi018_381516_object77008:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e465)", 2023-09-23T20:46:50.869 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:41:58.547295+0000", 2023-09-23T20:46:50.869 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 292.16293594899997, 2023-09-23T20:46:50.869 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.94976649700000004, 2023-09-23T20:46:50.869 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:46:50.869 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:46:50.870 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:46:50.870 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:46:50.870 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:46:50.870 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 77009 2023-09-23T20:46:50.870 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.870 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:46:50.870 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.870 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:46:50.870 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.547295+0000", 2023-09-23T20:46:50.871 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:50.871 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.871 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.871 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:46:50.871 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.547295+0000", 2023-09-23T20:46:50.871 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:50.871 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.871 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.871 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:46:50.871 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.547293+0000", 2023-09-23T20:46:50.872 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999976 2023-09-23T20:46:50.872 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.872 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.872 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:46:50.872 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.547385+0000", 2023-09-23T20:46:50.872 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.2566999999999997e-05 2023-09-23T20:46:50.872 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.872 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.872 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:46:50.872 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.547409+0000", 2023-09-23T20:46:50.873 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.3802999999999999e-05 2023-09-23T20:46:50.873 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.873 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.873 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:46:50.873 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.547418+0000", 2023-09-23T20:46:50.873 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.7509999999999997e-06 2023-09-23T20:46:50.873 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.873 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.873 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:46:50.874 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.547486+0000", 2023-09-23T20:46:50.874 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.7928000000000004e-05 2023-09-23T20:46:50.874 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.874 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.874 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:46:50.874 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.547499+0000", 2023-09-23T20:46:50.874 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.3556e-05 2023-09-23T20:46:50.874 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.875 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.875 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:46:50.875 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.484068+0000", 2023-09-23T20:46:50.875 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.93656838200000003 2023-09-23T20:46:50.875 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.875 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.875 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:46:50.875 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.484183+0000", 2023-09-23T20:46:50.875 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000115436 2023-09-23T20:46:50.875 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.875 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.877 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:46:50.877 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.484306+0000", 2023-09-23T20:46:50.877 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00012274399999999999 2023-09-23T20:46:50.877 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.877 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.877 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:46:50.877 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.493822+0000", 2023-09-23T20:46:50.877 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.009516142 2023-09-23T20:46:50.878 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.878 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.878 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:46:50.878 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.493837+0000", 2023-09-23T20:46:50.878 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.4698000000000001e-05 2023-09-23T20:46:50.878 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.878 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.878 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:46:50.878 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.496999+0000", 2023-09-23T20:46:50.878 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0031623770000000001 2023-09-23T20:46:50.879 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.879 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.879 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:46:50.879 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.497034+0000", 2023-09-23T20:46:50.879 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.4620999999999999e-05 2023-09-23T20:46:50.879 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.879 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.879 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:46:50.879 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.497062+0000", 2023-09-23T20:46:50.879 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.7654e-05 2023-09-23T20:46:50.879 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:50.879 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:46:50.880 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:50.880 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.880 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.880 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:77014 9.b 9:dc5d982b:::benchmark_data_smithi018_381516_object77013:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e465)", 2023-09-23T20:46:50.880 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:41:58.554929+0000", 2023-09-23T20:46:50.880 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 292.15530228699998, 2023-09-23T20:46:50.880 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.94218858999999999, 2023-09-23T20:46:50.880 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:46:50.880 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:46:50.880 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:46:50.880 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:46:50.881 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:46:50.881 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 77014 2023-09-23T20:46:50.881 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.881 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:46:50.881 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.881 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:46:50.881 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.554929+0000", 2023-09-23T20:46:50.881 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:50.881 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.881 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.881 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:46:50.881 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.554929+0000", 2023-09-23T20:46:50.882 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:50.882 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.882 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.882 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:46:50.882 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.554927+0000", 2023-09-23T20:46:50.882 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999981 2023-09-23T20:46:50.882 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.882 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.882 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:46:50.882 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.554989+0000", 2023-09-23T20:46:50.882 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.2597999999999999e-05 2023-09-23T20:46:50.882 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.883 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.883 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:46:50.883 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.555008+0000", 2023-09-23T20:46:50.883 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.8382000000000001e-05 2023-09-23T20:46:50.883 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.883 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.883 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:46:50.883 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.555014+0000", 2023-09-23T20:46:50.883 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.336e-06 2023-09-23T20:46:50.883 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.883 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.884 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:46:50.884 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.555098+0000", 2023-09-23T20:46:50.884 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.4109000000000004e-05 2023-09-23T20:46:50.884 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.884 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.884 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:46:50.884 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.555119+0000", 2023-09-23T20:46:50.884 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.0341000000000001e-05 2023-09-23T20:46:50.884 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.884 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.884 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:46:50.884 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.486855+0000", 2023-09-23T20:46:50.885 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.93173672500000004 2023-09-23T20:46:50.885 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.885 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.885 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:46:50.885 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.486994+0000", 2023-09-23T20:46:50.885 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000138243 2023-09-23T20:46:50.885 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.885 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.885 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:46:50.885 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.487116+0000", 2023-09-23T20:46:50.885 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000122268 2023-09-23T20:46:50.885 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.886 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.886 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:46:50.886 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.493852+0000", 2023-09-23T20:46:50.886 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0067365419999999999 2023-09-23T20:46:50.886 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.886 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.886 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:46:50.886 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.493874+0000", 2023-09-23T20:46:50.886 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.1189999999999999e-05 2023-09-23T20:46:50.886 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.886 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.887 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:46:50.887 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.497069+0000", 2023-09-23T20:46:50.887 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0031949600000000002 2023-09-23T20:46:50.887 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.887 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.887 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:46:50.887 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.497094+0000", 2023-09-23T20:46:50.887 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.5075999999999998e-05 2023-09-23T20:46:50.887 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.887 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.887 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:46:50.887 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.497117+0000", 2023-09-23T20:46:50.888 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.3669e-05 2023-09-23T20:46:50.888 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:50.888 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:46:50.888 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:50.888 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.888 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.888 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:77023 9.b 9:d3d436f2:::benchmark_data_smithi018_381516_object77022:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e465)", 2023-09-23T20:46:50.888 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:41:58.571188+0000", 2023-09-23T20:46:50.888 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 292.13904306400002, 2023-09-23T20:46:50.888 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.92609987900000001, 2023-09-23T20:46:50.888 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:46:50.888 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:46:50.888 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:46:50.889 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:46:50.889 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:46:50.889 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 77023 2023-09-23T20:46:50.889 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.889 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:46:50.889 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.889 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:46:50.889 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.571188+0000", 2023-09-23T20:46:50.889 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:50.889 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.889 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.890 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:46:50.890 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.571188+0000", 2023-09-23T20:46:50.890 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:50.890 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.890 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.890 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:46:50.890 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.571186+0000", 2023-09-23T20:46:50.890 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999981 2023-09-23T20:46:50.890 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.890 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.890 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:46:50.890 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.571248+0000", 2023-09-23T20:46:50.891 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.1997999999999998e-05 2023-09-23T20:46:50.891 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.891 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.891 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:46:50.891 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.571266+0000", 2023-09-23T20:46:50.891 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.7788999999999999e-05 2023-09-23T20:46:50.891 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.891 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.891 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:46:50.891 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.571273+0000", 2023-09-23T20:46:50.891 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.5910000000000001e-06 2023-09-23T20:46:50.891 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.892 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.892 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:46:50.892 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.571335+0000", 2023-09-23T20:46:50.892 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.2872000000000004e-05 2023-09-23T20:46:50.892 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.892 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.892 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:46:50.892 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.571346+0000", 2023-09-23T20:46:50.892 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0069e-05 2023-09-23T20:46:50.892 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.892 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.892 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:46:50.893 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.489960+0000", 2023-09-23T20:46:50.893 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.918614017 2023-09-23T20:46:50.893 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.893 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.893 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:46:50.893 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.490095+0000", 2023-09-23T20:46:50.893 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00013572700000000001 2023-09-23T20:46:50.893 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.893 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.893 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:46:50.893 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.490256+0000", 2023-09-23T20:46:50.894 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00016105000000000001 2023-09-23T20:46:50.894 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.894 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.894 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:46:50.894 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.493899+0000", 2023-09-23T20:46:50.894 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0036422210000000002 2023-09-23T20:46:50.894 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.894 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.894 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:46:50.894 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.497124+0000", 2023-09-23T20:46:50.894 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0032253220000000001 2023-09-23T20:46:50.894 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.895 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.895 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:46:50.895 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.497214+0000", 2023-09-23T20:46:50.895 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.9739000000000003e-05 2023-09-23T20:46:50.895 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.895 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.895 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:46:50.895 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.497254+0000", 2023-09-23T20:46:50.895 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.0278999999999998e-05 2023-09-23T20:46:50.895 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:50.895 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:50.895 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:46:50.896 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.497288+0000", 2023-09-23T20:46:50.896 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.3927999999999998e-05 2023-09-23T20:46:50.896 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:50.896 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:46:50.896 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:50.896 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:50.896 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:46:50.896 INFO:teuthology.orchestra.run.smithi149.stdout:} 2023-09-23T20:46:50.897 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:50 smithi018 ceph-mon[184144]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:46:50.897 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:50 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:46:50.897 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:50 smithi018 ceph-mon[184144]: osdmap e544: 8 total, 8 up, 8 in 2023-09-23T20:46:50.897 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:50 smithi018 ceph-mon[184144]: failed to encode map e544 with expected crc 2023-09-23T20:46:50.898 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:50 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3016213196' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:46:50.898 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:50 smithi018 ceph-mon[179647]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:46:50.898 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:50 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:46:50.898 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:50 smithi018 ceph-mon[179647]: osdmap e544: 8 total, 8 up, 8 in 2023-09-23T20:46:50.898 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:50 smithi018 ceph-mon[179647]: failed to encode map e544 with expected crc 2023-09-23T20:46:50.898 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:50 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3016213196' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:46:50.955 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:51.013 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 66 16 92619 92603 87.6451 57.1875 0.011494 0.011404 2023-09-23T20:46:51.097 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:50 smithi149 ceph-mon[170394]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:46:51.097 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:50 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:46:51.097 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:50 smithi149 ceph-mon[170394]: osdmap e544: 8 total, 8 up, 8 in 2023-09-23T20:46:51.097 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:50 smithi149 ceph-mon[170394]: failed to encode map e544 with expected crc 2023-09-23T20:46:51.097 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:50 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3016213196' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:46:51.121 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:46:51.128 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:51.129 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:46:51.230 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:46:51.288 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:51.787 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:46:51.793 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:51.793 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:46:51.894 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:46:51.895 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T20:46:51.992 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:52.013 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 67 16 93622 93606 87.2727 62.6875 0.0118046 0.0114534 2023-09-23T20:46:52.073 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:46:52.082 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:52.082 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:46:52.083 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:51 smithi149 ceph-mon[170394]: pgmap v1382: 119 pgs: 119 active+clean; 5.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 43 MiB/s wr, 690 op/s 2023-09-23T20:46:52.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:51 smithi018 ceph-mon[179647]: pgmap v1382: 119 pgs: 119 active+clean; 5.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 43 MiB/s wr, 690 op/s 2023-09-23T20:46:52.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:51 smithi018 ceph-mon[184144]: pgmap v1382: 119 pgs: 119 active+clean; 5.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 43 MiB/s wr, 690 op/s 2023-09-23T20:46:52.183 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:46:52.250 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:52.252 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T20:46:52.338 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:46:52.346 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:52.347 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:46:52.448 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:46:52.518 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:52.595 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:46:52.602 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:52.603 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:46:52.691 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 7, 3, 2, 4, 6, 5] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T20:46:52.691 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:46:52.692 INFO:tasks.thrashosds.thrasher:Removing osd 4, in_osds are: [0, 1, 7, 3, 2, 4, 6, 5] 2023-09-23T20:46:52.692 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd out 4 2023-09-23T20:46:52.704 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:46:52.746 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:52.792 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:46:52.796 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:46:52.804 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:52.805 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:46:52.898 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:46:52.906 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:46:52.947 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:53.013 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 68 16 94480 94464 86.778 53.625 0.0216781 0.0115186 2023-09-23T20:46:53.058 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:46:53.069 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:53.069 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:46:53.102 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:46:53.170 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:46:53.254 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:53.379 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:46:53.383 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:53.383 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:46:53.484 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:46:53.569 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:53.651 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:46:53.660 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:46:53.661 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:53.662 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:46:53.763 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:46:53.843 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:53.893 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:53 smithi149 ceph-mon[170394]: pgmap v1383: 119 pgs: 119 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 58 MiB/s wr, 933 op/s 2023-09-23T20:46:53.912 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:46:53.921 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:53.922 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:46:54.014 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 69 16 95322 95306 86.2831 52.625 0.00841402 0.0115844 2023-09-23T20:46:54.022 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:46:54.091 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:54.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:53 smithi018 ceph-mon[184144]: pgmap v1383: 119 pgs: 119 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 58 MiB/s wr, 933 op/s 2023-09-23T20:46:54.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:53 smithi018 ceph-mon[179647]: pgmap v1383: 119 pgs: 119 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 58 MiB/s wr, 933 op/s 2023-09-23T20:46:54.166 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:46:54.174 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:54.174 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:46:54.275 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:46:54.304 INFO:teuthology.orchestra.run.smithi149.stdout:{ 2023-09-23T20:46:54.305 INFO:teuthology.orchestra.run.smithi149.stdout: "ops": [ 2023-09-23T20:46:54.305 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:54.305 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:95555 10.c 10:3c73ec28:::benchmark_data_smithi018_422431_object95554:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e544)", 2023-09-23T20:46:54.305 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:46:54.286664+0000", 2023-09-23T20:46:54.305 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 0.017846988000000001, 2023-09-23T20:46:54.305 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.017881662999999999, 2023-09-23T20:46:54.305 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:46:54.305 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:46:54.305 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:46:54.306 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:46:54.306 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:46:54.306 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 95555 2023-09-23T20:46:54.306 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:54.306 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:46:54.306 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:54.306 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:46:54.306 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:54.286664+0000", 2023-09-23T20:46:54.306 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:54.306 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:54.306 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:54.307 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:46:54.307 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:54.286664+0000", 2023-09-23T20:46:54.307 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:54.307 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:54.307 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:54.307 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:46:54.307 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:54.286663+0000", 2023-09-23T20:46:54.307 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:46:54.307 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:54.307 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:54.307 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:46:54.308 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:54.286699+0000", 2023-09-23T20:46:54.308 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.6077e-05 2023-09-23T20:46:54.308 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:54.308 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:54.308 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:46:54.308 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:54.286712+0000", 2023-09-23T20:46:54.308 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.3427e-05 2023-09-23T20:46:54.308 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:54.308 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:54.308 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:46:54.308 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:54.286720+0000", 2023-09-23T20:46:54.308 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.3919999999999997e-06 2023-09-23T20:46:54.309 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:54.309 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:54.309 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:46:54.309 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:54.286774+0000", 2023-09-23T20:46:54.309 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.4601999999999997e-05 2023-09-23T20:46:54.309 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:54.309 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:54.309 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:46:54.309 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:54.286941+0000", 2023-09-23T20:46:54.309 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00016638499999999999 2023-09-23T20:46:54.309 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:54.310 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:54.310 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,4", 2023-09-23T20:46:54.310 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:54.287125+0000", 2023-09-23T20:46:54.310 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000184167 2023-09-23T20:46:54.310 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:54.310 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:54.310 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:46:54.310 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:54.296744+0000", 2023-09-23T20:46:54.310 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0096188969999999995 2023-09-23T20:46:54.310 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:54.310 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:54.311 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:46:54.311 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:54.303738+0000", 2023-09-23T20:46:54.311 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0069941730000000002 2023-09-23T20:46:54.311 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:54.311 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:46:54.311 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:54.311 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:54.311 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:54.311 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:95560 10.c 10:3f845f00:::benchmark_data_smithi018_422431_object95559:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e544)", 2023-09-23T20:46:54.311 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:46:54.288094+0000", 2023-09-23T20:46:54.311 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 0.016417903000000001, 2023-09-23T20:46:54.312 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.016547323999999999, 2023-09-23T20:46:54.312 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:46:54.312 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:46:54.312 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:46:54.312 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:46:54.312 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:46:54.312 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 95560 2023-09-23T20:46:54.312 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:54.312 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:46:54.312 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:54.312 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:46:54.313 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:54.288094+0000", 2023-09-23T20:46:54.313 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:54.313 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:54.313 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:54.313 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:46:54.313 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:54.288094+0000", 2023-09-23T20:46:54.313 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:54.313 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:54.313 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:54.313 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:46:54.313 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:54.288093+0000", 2023-09-23T20:46:54.313 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:46:54.314 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:54.314 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:54.314 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:46:54.314 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:54.288314+0000", 2023-09-23T20:46:54.314 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00022085099999999999 2023-09-23T20:46:54.314 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:54.314 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:54.314 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:46:54.314 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:54.288343+0000", 2023-09-23T20:46:54.314 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.9912000000000001e-05 2023-09-23T20:46:54.314 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:54.315 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:54.315 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:46:54.315 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:54.288351+0000", 2023-09-23T20:46:54.315 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.7619999999999995e-06 2023-09-23T20:46:54.315 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:54.315 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:54.315 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:46:54.315 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:54.290986+0000", 2023-09-23T20:46:54.315 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.002635288 2023-09-23T20:46:54.315 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:54.315 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:54.315 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:46:54.316 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:54.291112+0000", 2023-09-23T20:46:54.316 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00012594700000000001 2023-09-23T20:46:54.316 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:54.316 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:54.316 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,4", 2023-09-23T20:46:54.316 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:54.291261+0000", 2023-09-23T20:46:54.316 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000148799 2023-09-23T20:46:54.316 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:54.316 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:54.317 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:46:54.317 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:54.296750+0000", 2023-09-23T20:46:54.317 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.005488934 2023-09-23T20:46:54.317 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:54.317 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:54.317 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:46:54.317 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:54.303897+0000", 2023-09-23T20:46:54.317 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0071473300000000003 2023-09-23T20:46:54.317 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:54.317 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:46:54.317 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:54.318 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:54.318 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:54.318 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:95569 10.c 10:3e967435:::benchmark_data_smithi018_422431_object95568:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e544)", 2023-09-23T20:46:54.318 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:46:54.299868+0000", 2023-09-23T20:46:54.318 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 0.0046433730000000001, 2023-09-23T20:46:54.318 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0048595899999999996, 2023-09-23T20:46:54.318 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:46:54.318 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:46:54.318 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:46:54.318 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:46:54.318 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:46:54.319 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 95569 2023-09-23T20:46:54.319 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:54.319 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:46:54.319 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:54.319 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:46:54.319 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:54.299868+0000", 2023-09-23T20:46:54.319 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:54.319 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:54.319 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:54.319 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:46:54.319 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:54.299868+0000", 2023-09-23T20:46:54.319 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:46:54.319 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:54.320 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:54.320 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:46:54.320 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:54.299867+0000", 2023-09-23T20:46:54.320 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:46:54.320 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:54.320 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:54.320 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:46:54.320 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:54.299917+0000", 2023-09-23T20:46:54.320 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.9971e-05 2023-09-23T20:46:54.320 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:54.320 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:54.320 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:46:54.321 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:54.299935+0000", 2023-09-23T20:46:54.321 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.7999000000000002e-05 2023-09-23T20:46:54.321 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:54.321 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:54.321 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:46:54.321 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:54.299941+0000", 2023-09-23T20:46:54.321 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.8760000000000002e-06 2023-09-23T20:46:54.321 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:54.321 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:54.321 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:46:54.321 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:54.299977+0000", 2023-09-23T20:46:54.322 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.6433000000000002e-05 2023-09-23T20:46:54.322 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:54.322 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:54.322 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:46:54.322 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:54.300118+0000", 2023-09-23T20:46:54.322 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00014123599999999999 2023-09-23T20:46:54.322 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:46:54.322 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:46:54.322 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,4", 2023-09-23T20:46:54.322 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:46:54.300272+0000", 2023-09-23T20:46:54.322 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00015349100000000001 2023-09-23T20:46:54.322 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:54.323 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:46:54.323 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:54.323 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:46:54.323 INFO:teuthology.orchestra.run.smithi149.stdout: ], 2023-09-23T20:46:54.323 INFO:teuthology.orchestra.run.smithi149.stdout: "num_ops": 3 2023-09-23T20:46:54.323 INFO:teuthology.orchestra.run.smithi149.stdout:} 2023-09-23T20:46:54.323 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:46:54.338 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:55.017 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 70 16 96191 96175 85.8224 54.3125 0.0134862 0.0116473 2023-09-23T20:46:55.257 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:46:54 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:46:54] "GET /metrics HTTP/1.1" 200 36566 "" "Prometheus/2.43.0" 2023-09-23T20:46:55.521 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:46:55.529 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:55.530 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:46:55.570 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T20:46:55.631 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:46:55.742 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:55.836 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:46:55.844 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:55.844 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:46:55.922 INFO:teuthology.orchestra.run.smithi018.stderr:noscrub is unset 2023-09-23T20:46:55.945 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:46:56.021 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 71 16 97003 96987 85.3285 50.75 0.0187076 0.0117147 2023-09-23T20:46:56.028 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:56.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:55 smithi018 ceph-mon[184144]: pgmap v1384: 119 pgs: 119 active+clean; 5.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 56 MiB/s wr, 889 op/s 2023-09-23T20:46:56.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:55 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/759453218' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:46:56.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:55 smithi018 ceph-mon[179647]: pgmap v1384: 119 pgs: 119 active+clean; 5.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 56 MiB/s wr, 889 op/s 2023-09-23T20:46:56.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:55 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/759453218' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:46:56.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:56 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mon-a[179623]: 2023-09-23T20:46:56.008+0000 7f160fef9700 -1 mon.a@0(leader).osd e545 definitely_dead 0 2023-09-23T20:46:56.169 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:46:56.180 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:56.180 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:46:56.218 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:55 smithi149 ceph-mon[170394]: pgmap v1384: 119 pgs: 119 active+clean; 5.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 56 MiB/s wr, 889 op/s 2023-09-23T20:46:56.218 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:55 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/759453218' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:46:56.282 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:46:56.334 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:56.431 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:46:56.514 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:46:56.524 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:56.525 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:46:56.626 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:46:56.715 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:56.789 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:46:56.801 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:56.802 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:46:56.831 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:46:56.862 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:46:56.903 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:46:56.913 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:46:56.957 INFO:teuthology.orchestra.run.smithi018.stderr:marked out osd.4. 2023-09-23T20:46:56.965 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:46:56.970 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:57.005 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:46:57.017 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 72 16 97858 97842 84.8857 53.4375 0.00511835 0.0117685 2023-09-23T20:46:57.026 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:46:57.048 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:46:57.068 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:46:57.069 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:46:57.070 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:46:57.070 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:46:57.070 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:46:57.070 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:46:57.070 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:46:57.070 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:46:57.070 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:46:57.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:56 smithi149 ceph-mon[170394]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:46:57.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:56 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/759453218' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:46:57.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:56 smithi149 ceph-mon[170394]: osdmap e545: 8 total, 8 up, 8 in 2023-09-23T20:46:57.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:56 smithi149 ceph-mon[170394]: failed to encode map e545 with expected crc 2023-09-23T20:46:57.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:56 smithi149 ceph-mon[170394]: failed to encode map e545 with expected crc 2023-09-23T20:46:57.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:56 smithi149 ceph-mon[170394]: failed to encode map e545 with expected crc 2023-09-23T20:46:57.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:56 smithi149 ceph-mon[170394]: failed to encode map e545 with expected crc 2023-09-23T20:46:57.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:56 smithi149 ceph-mon[170394]: failed to encode map e545 with expected crc 2023-09-23T20:46:57.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:56 smithi149 ceph-mon[170394]: failed to encode map e545 with expected crc 2023-09-23T20:46:57.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:56 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1284887485' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["4"]}]: dispatch 2023-09-23T20:46:57.218 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:56 smithi149 ceph-mon[170394]: Client client.admin marked osd.4 out, while it was still marked up 2023-09-23T20:46:57.218 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:56 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/4087088017' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:46:57.310 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:46:57.313 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:57.313 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:46:57.372 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:56 smithi018 ceph-mon[179647]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:46:57.372 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:56 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/759453218' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:46:57.372 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:56 smithi018 ceph-mon[179647]: osdmap e545: 8 total, 8 up, 8 in 2023-09-23T20:46:57.372 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:56 smithi018 ceph-mon[179647]: failed to encode map e545 with expected crc 2023-09-23T20:46:57.373 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:56 smithi018 ceph-mon[179647]: failed to encode map e545 with expected crc 2023-09-23T20:46:57.373 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:56 smithi018 ceph-mon[179647]: failed to encode map e545 with expected crc 2023-09-23T20:46:57.373 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:56 smithi018 ceph-mon[179647]: failed to encode map e545 with expected crc 2023-09-23T20:46:57.373 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:56 smithi018 ceph-mon[179647]: failed to encode map e545 with expected crc 2023-09-23T20:46:57.373 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:56 smithi018 ceph-mon[179647]: failed to encode map e545 with expected crc 2023-09-23T20:46:57.373 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:56 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1284887485' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["4"]}]: dispatch 2023-09-23T20:46:57.373 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:56 smithi018 ceph-mon[179647]: Client client.admin marked osd.4 out, while it was still marked up 2023-09-23T20:46:57.373 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:56 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/4087088017' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:46:57.373 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:56 smithi018 ceph-mon[184144]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:46:57.374 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:56 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/759453218' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:46:57.374 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:56 smithi018 ceph-mon[184144]: osdmap e545: 8 total, 8 up, 8 in 2023-09-23T20:46:57.374 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:56 smithi018 ceph-mon[184144]: failed to encode map e545 with expected crc 2023-09-23T20:46:57.374 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:56 smithi018 ceph-mon[184144]: failed to encode map e545 with expected crc 2023-09-23T20:46:57.374 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:56 smithi018 ceph-mon[184144]: failed to encode map e545 with expected crc 2023-09-23T20:46:57.374 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:56 smithi018 ceph-mon[184144]: failed to encode map e545 with expected crc 2023-09-23T20:46:57.374 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:56 smithi018 ceph-mon[184144]: failed to encode map e545 with expected crc 2023-09-23T20:46:57.374 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:56 smithi018 ceph-mon[184144]: failed to encode map e545 with expected crc 2023-09-23T20:46:57.374 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:56 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1284887485' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["4"]}]: dispatch 2023-09-23T20:46:57.375 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:56 smithi018 ceph-mon[184144]: Client client.admin marked osd.4 out, while it was still marked up 2023-09-23T20:46:57.375 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:56 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/4087088017' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:46:57.414 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:46:57.443 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:57.478 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:46:57.481 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:57.481 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:46:57.582 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:46:57.608 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:57.694 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:46:57.698 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:57.698 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:46:57.762 INFO:teuthology.orchestra.run.smithi149.stdout:ERROR: (22) Invalid argument 2023-09-23T20:46:57.763 INFO:teuthology.orchestra.run.smithi149.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:46:57.799 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:46:57.828 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:57.863 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:46:57.866 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:57.867 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:46:57.941 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:57 smithi018 ceph-mon[179647]: pgmap v1386: 119 pgs: 119 active+clean; 5.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 57 MiB/s wr, 908 op/s 2023-09-23T20:46:57.941 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:57 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1284887485' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["4"]}]': finished 2023-09-23T20:46:57.941 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:57 smithi018 ceph-mon[179647]: osdmap e546: 8 total, 8 up, 7 in 2023-09-23T20:46:57.941 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:57 smithi018 ceph-mon[179647]: failed to encode map e546 with expected crc 2023-09-23T20:46:57.942 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:57 smithi018 ceph-mon[179647]: failed to encode map e546 with expected crc 2023-09-23T20:46:57.942 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:57 smithi018 ceph-mon[179647]: failed to encode map e546 with expected crc 2023-09-23T20:46:57.942 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:57 smithi018 ceph-mon[179647]: failed to encode map e546 with expected crc 2023-09-23T20:46:57.967 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:46:57.999 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:58.017 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 73 16 97858 97842 83.7234 0 - 0.0117685 2023-09-23T20:46:58.032 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:46:58.035 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:58.035 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:46:58.136 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:46:58.164 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:58.196 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:46:58.198 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:57 smithi018 ceph-mon[184144]: pgmap v1386: 119 pgs: 119 active+clean; 5.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 57 MiB/s wr, 908 op/s 2023-09-23T20:46:58.198 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:57 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1284887485' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["4"]}]': finished 2023-09-23T20:46:58.198 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:57 smithi018 ceph-mon[184144]: osdmap e546: 8 total, 8 up, 7 in 2023-09-23T20:46:58.199 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:57 smithi018 ceph-mon[184144]: failed to encode map e546 with expected crc 2023-09-23T20:46:58.199 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:57 smithi018 ceph-mon[184144]: failed to encode map e546 with expected crc 2023-09-23T20:46:58.199 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:57 smithi018 ceph-mon[184144]: failed to encode map e546 with expected crc 2023-09-23T20:46:58.199 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:57 smithi018 ceph-mon[184144]: failed to encode map e546 with expected crc 2023-09-23T20:46:58.199 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:58.200 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:46:58.200 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:57 smithi149 ceph-mon[170394]: pgmap v1386: 119 pgs: 119 active+clean; 5.8 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 57 MiB/s wr, 908 op/s 2023-09-23T20:46:58.200 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:57 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1284887485' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["4"]}]': finished 2023-09-23T20:46:58.200 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:57 smithi149 ceph-mon[170394]: osdmap e546: 8 total, 8 up, 7 in 2023-09-23T20:46:58.200 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:57 smithi149 ceph-mon[170394]: failed to encode map e546 with expected crc 2023-09-23T20:46:58.201 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:57 smithi149 ceph-mon[170394]: failed to encode map e546 with expected crc 2023-09-23T20:46:58.201 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:57 smithi149 ceph-mon[170394]: failed to encode map e546 with expected crc 2023-09-23T20:46:58.201 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:57 smithi149 ceph-mon[170394]: failed to encode map e546 with expected crc 2023-09-23T20:46:58.202 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T20:46:58.300 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:46:58.327 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:58.359 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:46:58.362 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:58.363 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:46:58.463 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:46:58.479 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T20:46:58.491 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:58.526 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:46:58.529 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:58.530 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:46:58.630 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:46:58.657 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:58.694 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:46:58.697 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:58.698 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:46:58.799 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:46:58.806 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:46:58.826 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:58.923 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:46:58.926 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:58.927 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:46:59.018 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 74 16 97858 97842 82.5925 0 - 0.0117685 2023-09-23T20:46:59.028 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:46:59.082 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:58 smithi149 ceph-mon[170394]: failed to encode map e547 with expected crc 2023-09-23T20:46:59.082 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:58 smithi149 ceph-mon[170394]: osdmap e547: 8 total, 8 up, 7 in 2023-09-23T20:46:59.082 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:58 smithi149 ceph-mon[170394]: failed to encode map e547 with expected crc 2023-09-23T20:46:59.083 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:58 smithi149 ceph-mon[170394]: failed to encode map e547 with expected crc 2023-09-23T20:46:59.083 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:58 smithi149 ceph-mon[170394]: failed to encode map e547 with expected crc 2023-09-23T20:46:59.083 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:58 smithi149 ceph-mon[170394]: failed to encode map e547 with expected crc 2023-09-23T20:46:59.083 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:58 smithi149 ceph-mon[170394]: failed to encode map e547 with expected crc 2023-09-23T20:46:59.083 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:58 smithi149 ceph-mon[170394]: osdmap e548: 8 total, 8 up, 7 in 2023-09-23T20:46:59.133 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:59.167 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:46:59.218 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:46:59.225 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:59.226 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:46:59.228 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:58 smithi018 ceph-mon[179647]: failed to encode map e547 with expected crc 2023-09-23T20:46:59.228 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:58 smithi018 ceph-mon[179647]: osdmap e547: 8 total, 8 up, 7 in 2023-09-23T20:46:59.228 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:58 smithi018 ceph-mon[179647]: failed to encode map e547 with expected crc 2023-09-23T20:46:59.228 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:58 smithi018 ceph-mon[179647]: failed to encode map e547 with expected crc 2023-09-23T20:46:59.228 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:58 smithi018 ceph-mon[179647]: failed to encode map e547 with expected crc 2023-09-23T20:46:59.228 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:58 smithi018 ceph-mon[179647]: failed to encode map e547 with expected crc 2023-09-23T20:46:59.228 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:58 smithi018 ceph-mon[179647]: failed to encode map e547 with expected crc 2023-09-23T20:46:59.228 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:58 smithi018 ceph-mon[179647]: osdmap e548: 8 total, 8 up, 7 in 2023-09-23T20:46:59.228 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:58 smithi018 ceph-mon[184144]: failed to encode map e547 with expected crc 2023-09-23T20:46:59.229 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:58 smithi018 ceph-mon[184144]: osdmap e547: 8 total, 8 up, 7 in 2023-09-23T20:46:59.229 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:58 smithi018 ceph-mon[184144]: failed to encode map e547 with expected crc 2023-09-23T20:46:59.229 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:58 smithi018 ceph-mon[184144]: failed to encode map e547 with expected crc 2023-09-23T20:46:59.229 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:58 smithi018 ceph-mon[184144]: failed to encode map e547 with expected crc 2023-09-23T20:46:59.229 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:58 smithi018 ceph-mon[184144]: failed to encode map e547 with expected crc 2023-09-23T20:46:59.229 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:58 smithi018 ceph-mon[184144]: failed to encode map e547 with expected crc 2023-09-23T20:46:59.229 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:58 smithi018 ceph-mon[184144]: osdmap e548: 8 total, 8 up, 7 in 2023-09-23T20:46:59.327 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:46:59.395 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:46:59.642 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:46:59.647 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:46:59.647 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:46:59.739 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:46:59.748 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:46:59.930 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:00.023 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 75 15 98218 98203 81.7863 7.52083 0.0655262 0.0122216 2023-09-23T20:47:00.066 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:47:00.070 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:00.071 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:47:00.071 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:59 smithi018 ceph-mon[179647]: pgmap v1389: 119 pgs: 2 remapped+peering, 117 active+clean; 5.9 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 53 MiB/s wr, 844 op/s 2023-09-23T20:47:00.071 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:59 smithi018 ceph-mon[179647]: failed to encode map e548 with expected crc 2023-09-23T20:47:00.072 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:59 smithi018 ceph-mon[179647]: failed to encode map e548 with expected crc 2023-09-23T20:47:00.072 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:59 smithi018 ceph-mon[179647]: failed to encode map e548 with expected crc 2023-09-23T20:47:00.072 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:59 smithi018 ceph-mon[179647]: failed to encode map e548 with expected crc 2023-09-23T20:47:00.072 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:59 smithi018 ceph-mon[179647]: failed to encode map e548 with expected crc 2023-09-23T20:47:00.072 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:59 smithi018 ceph-mon[179647]: failed to encode map e548 with expected crc 2023-09-23T20:47:00.072 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:59 smithi018 ceph-mon[179647]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-09-23T20:47:00.072 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:59 smithi018 ceph-mon[179647]: 10.1 starting backfill to osd.1 from (0'0,0'0] MAX to 545'6101 2023-09-23T20:47:00.072 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:59 smithi018 ceph-mon[179647]: failed to encode map e548 with expected crc 2023-09-23T20:47:00.072 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:59 smithi018 ceph-mon[179647]: 10.e starting backfill to osd.5 from (0'0,0'0] MAX to 545'6104 2023-09-23T20:47:00.072 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:59 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:47:00.072 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:59 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:47:00.073 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:46:59 smithi018 ceph-mon[179647]: osdmap e549: 8 total, 8 up, 7 in 2023-09-23T20:47:00.076 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:59 smithi018 ceph-mon[184144]: pgmap v1389: 119 pgs: 2 remapped+peering, 117 active+clean; 5.9 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 53 MiB/s wr, 844 op/s 2023-09-23T20:47:00.076 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:59 smithi018 ceph-mon[184144]: failed to encode map e548 with expected crc 2023-09-23T20:47:00.076 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:59 smithi018 ceph-mon[184144]: failed to encode map e548 with expected crc 2023-09-23T20:47:00.076 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:59 smithi018 ceph-mon[184144]: failed to encode map e548 with expected crc 2023-09-23T20:47:00.077 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:59 smithi018 ceph-mon[184144]: failed to encode map e548 with expected crc 2023-09-23T20:47:00.077 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:59 smithi018 ceph-mon[184144]: failed to encode map e548 with expected crc 2023-09-23T20:47:00.077 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:59 smithi018 ceph-mon[184144]: failed to encode map e548 with expected crc 2023-09-23T20:47:00.077 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:59 smithi018 ceph-mon[184144]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-09-23T20:47:00.077 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:59 smithi018 ceph-mon[184144]: 10.1 starting backfill to osd.1 from (0'0,0'0] MAX to 545'6101 2023-09-23T20:47:00.077 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:59 smithi018 ceph-mon[184144]: failed to encode map e548 with expected crc 2023-09-23T20:47:00.077 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:59 smithi018 ceph-mon[184144]: 10.e starting backfill to osd.5 from (0'0,0'0] MAX to 545'6104 2023-09-23T20:47:00.077 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:59 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:47:00.077 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:59 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:47:00.077 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:46:59 smithi018 ceph-mon[184144]: osdmap e549: 8 total, 8 up, 7 in 2023-09-23T20:47:00.172 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:47:00.220 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:00.459 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:59 smithi149 ceph-mon[170394]: pgmap v1389: 119 pgs: 2 remapped+peering, 117 active+clean; 5.9 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 53 MiB/s wr, 844 op/s 2023-09-23T20:47:00.459 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:59 smithi149 ceph-mon[170394]: failed to encode map e548 with expected crc 2023-09-23T20:47:00.460 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:59 smithi149 ceph-mon[170394]: failed to encode map e548 with expected crc 2023-09-23T20:47:00.460 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:59 smithi149 ceph-mon[170394]: failed to encode map e548 with expected crc 2023-09-23T20:47:00.460 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:59 smithi149 ceph-mon[170394]: failed to encode map e548 with expected crc 2023-09-23T20:47:00.460 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:59 smithi149 ceph-mon[170394]: failed to encode map e548 with expected crc 2023-09-23T20:47:00.460 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:59 smithi149 ceph-mon[170394]: failed to encode map e548 with expected crc 2023-09-23T20:47:00.460 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:59 smithi149 ceph-mon[170394]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-09-23T20:47:00.461 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:59 smithi149 ceph-mon[170394]: 10.1 starting backfill to osd.1 from (0'0,0'0] MAX to 545'6101 2023-09-23T20:47:00.461 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:59 smithi149 ceph-mon[170394]: failed to encode map e548 with expected crc 2023-09-23T20:47:00.461 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:59 smithi149 ceph-mon[170394]: 10.e starting backfill to osd.5 from (0'0,0'0] MAX to 545'6104 2023-09-23T20:47:00.461 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:59 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:47:00.461 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:59 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:47:00.461 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:46:59 smithi149 ceph-mon[170394]: osdmap e549: 8 total, 8 up, 7 in 2023-09-23T20:47:00.618 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:47:00.622 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:00.623 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:47:00.723 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:47:00.757 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:00.818 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:47:00.822 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:00.822 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:47:00.923 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:47:00.955 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:47:00.968 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:01.024 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 76 16 98741 98725 81.1389 32.625 0.0314217 0.0123179 2023-09-23T20:47:01.032 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:47:01.040 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:01.040 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:47:01.141 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:47:01.196 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:01.247 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:47:01.254 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:01.255 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:47:01.280 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:01 smithi018 ceph-mon[184144]: failed to encode map e549 with expected crc 2023-09-23T20:47:01.280 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:01 smithi018 ceph-mon[184144]: failed to encode map e549 with expected crc 2023-09-23T20:47:01.280 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:01 smithi018 ceph-mon[184144]: failed to encode map e549 with expected crc 2023-09-23T20:47:01.281 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:01 smithi018 ceph-mon[184144]: failed to encode map e549 with expected crc 2023-09-23T20:47:01.281 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:01 smithi018 ceph-mon[184144]: osdmap e550: 8 total, 8 up, 7 in 2023-09-23T20:47:01.281 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:01 smithi018 ceph-mon[179647]: failed to encode map e549 with expected crc 2023-09-23T20:47:01.281 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:01 smithi018 ceph-mon[179647]: failed to encode map e549 with expected crc 2023-09-23T20:47:01.281 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:01 smithi018 ceph-mon[179647]: failed to encode map e549 with expected crc 2023-09-23T20:47:01.281 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:01 smithi018 ceph-mon[179647]: failed to encode map e549 with expected crc 2023-09-23T20:47:01.281 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:01 smithi018 ceph-mon[179647]: osdmap e550: 8 total, 8 up, 7 in 2023-09-23T20:47:01.356 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:47:01.384 INFO:teuthology.orchestra.run.smithi149.stdout:ERROR: (22) Invalid argument 2023-09-23T20:47:01.384 INFO:teuthology.orchestra.run.smithi149.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:47:01.397 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:01 smithi149 ceph-mon[170394]: failed to encode map e549 with expected crc 2023-09-23T20:47:01.397 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:01 smithi149 ceph-mon[170394]: failed to encode map e549 with expected crc 2023-09-23T20:47:01.397 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:01 smithi149 ceph-mon[170394]: failed to encode map e549 with expected crc 2023-09-23T20:47:01.397 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:01 smithi149 ceph-mon[170394]: failed to encode map e549 with expected crc 2023-09-23T20:47:01.397 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:01 smithi149 ceph-mon[170394]: osdmap e550: 8 total, 8 up, 7 in 2023-09-23T20:47:01.403 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:01.664 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:47:01.670 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:01.671 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:47:01.772 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:47:01.826 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:01.842 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T20:47:01.881 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:47:01.886 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:01.887 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:47:01.988 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:47:02.024 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 77 16 99222 99206 80.4758 30.0625 0.0548232 0.0124166 2023-09-23T20:47:02.101 INFO:teuthology.orchestra.run.smithi018.stderr:nodeep-scrub is unset 2023-09-23T20:47:02.131 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:02.290 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:02 smithi018 ceph-mon[179647]: pgmap v1392: 119 pgs: 7 remapped+peering, 112 active+clean; 5.9 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 25 MiB/s wr, 399 op/s 2023-09-23T20:47:02.290 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:02 smithi018 ceph-mon[179647]: failed to encode map e550 with expected crc 2023-09-23T20:47:02.290 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:02 smithi018 ceph-mon[179647]: failed to encode map e550 with expected crc 2023-09-23T20:47:02.290 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:02 smithi018 ceph-mon[179647]: failed to encode map e550 with expected crc 2023-09-23T20:47:02.290 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:02 smithi018 ceph-mon[179647]: failed to encode map e550 with expected crc 2023-09-23T20:47:02.291 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:02 smithi018 ceph-mon[179647]: failed to encode map e550 with expected crc 2023-09-23T20:47:02.291 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:02 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2426562397' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:47:02.291 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:02 smithi018 ceph-mon[184144]: pgmap v1392: 119 pgs: 7 remapped+peering, 112 active+clean; 5.9 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 25 MiB/s wr, 399 op/s 2023-09-23T20:47:02.291 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:02 smithi018 ceph-mon[184144]: failed to encode map e550 with expected crc 2023-09-23T20:47:02.291 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:02 smithi018 ceph-mon[184144]: failed to encode map e550 with expected crc 2023-09-23T20:47:02.291 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:02 smithi018 ceph-mon[184144]: failed to encode map e550 with expected crc 2023-09-23T20:47:02.291 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:02 smithi018 ceph-mon[184144]: failed to encode map e550 with expected crc 2023-09-23T20:47:02.291 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:02 smithi018 ceph-mon[184144]: failed to encode map e550 with expected crc 2023-09-23T20:47:02.291 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:02 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2426562397' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:47:02.373 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:47:02.381 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:02.382 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:47:02.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:02 smithi149 ceph-mon[170394]: pgmap v1392: 119 pgs: 7 remapped+peering, 112 active+clean; 5.9 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 25 MiB/s wr, 399 op/s 2023-09-23T20:47:02.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:02 smithi149 ceph-mon[170394]: failed to encode map e550 with expected crc 2023-09-23T20:47:02.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:02 smithi149 ceph-mon[170394]: failed to encode map e550 with expected crc 2023-09-23T20:47:02.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:02 smithi149 ceph-mon[170394]: failed to encode map e550 with expected crc 2023-09-23T20:47:02.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:02 smithi149 ceph-mon[170394]: failed to encode map e550 with expected crc 2023-09-23T20:47:02.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:02 smithi149 ceph-mon[170394]: failed to encode map e550 with expected crc 2023-09-23T20:47:02.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:02 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2426562397' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:47:02.482 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:47:02.537 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:02.595 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:47:02.609 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:02.609 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:47:02.674 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 7, 3, 2, 6, 5] out_osds: [4] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T20:47:02.674 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:47:02.674 INFO:tasks.thrashosds.thrasher:Adding osd 4 2023-09-23T20:47:02.674 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd in 4 2023-09-23T20:47:02.710 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:47:02.755 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:02.799 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:47:02.803 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:02.803 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:47:02.859 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:47:02.904 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:47:02.922 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:47:02.982 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:47:03.007 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:47:03.020 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:03.024 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 78 16 99694 99678 79.8226 29.5 0.0980868 0.0125174 2023-09-23T20:47:03.047 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:47:03.081 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:47:03.127 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:47:03.169 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:47:03.202 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:47:03.243 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:47:03.243 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:47:03.243 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:47:03.243 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:47:03.244 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:47:03.244 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:47:03.244 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:47:03.244 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:47:03.259 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:03 smithi018 ceph-mon[184144]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:47:03.259 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:03 smithi018 ceph-mon[184144]: failed to encode map e551 with expected crc 2023-09-23T20:47:03.259 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:03 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2426562397' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:47:03.260 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:03 smithi018 ceph-mon[184144]: osdmap e551: 8 total, 8 up, 7 in 2023-09-23T20:47:03.260 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:03 smithi018 ceph-mon[184144]: pgmap v1395: 119 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 6 peering, 7 remapped+peering, 99 active+clean; 6.0 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 10 KiB/s rd, 23 MiB/s wr, 382 op/s; 24402/295092 objects degraded (8.269%); 18291/295092 objects misplaced (6.198%); 1.5 MiB/s, 2 keys/s, 26 objects/s recovering 2023-09-23T20:47:03.260 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:03 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/4245103712' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:47:03.260 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:03 smithi018 ceph-mon[179647]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:47:03.260 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:03 smithi018 ceph-mon[179647]: failed to encode map e551 with expected crc 2023-09-23T20:47:03.260 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:03 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2426562397' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:47:03.260 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:03 smithi018 ceph-mon[179647]: osdmap e551: 8 total, 8 up, 7 in 2023-09-23T20:47:03.260 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:03 smithi018 ceph-mon[179647]: pgmap v1395: 119 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 6 peering, 7 remapped+peering, 99 active+clean; 6.0 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 10 KiB/s rd, 23 MiB/s wr, 382 op/s; 24402/295092 objects degraded (8.269%); 18291/295092 objects misplaced (6.198%); 1.5 MiB/s, 2 keys/s, 26 objects/s recovering 2023-09-23T20:47:03.260 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:03 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/4245103712' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:47:03.455 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:47:03.461 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:03.461 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:47:03.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:03 smithi149 ceph-mon[170394]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:47:03.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:03 smithi149 ceph-mon[170394]: failed to encode map e551 with expected crc 2023-09-23T20:47:03.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:03 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2426562397' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:47:03.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:03 smithi149 ceph-mon[170394]: osdmap e551: 8 total, 8 up, 7 in 2023-09-23T20:47:03.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:03 smithi149 ceph-mon[170394]: pgmap v1395: 119 pgs: 4 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 6 peering, 7 remapped+peering, 99 active+clean; 6.0 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 10 KiB/s rd, 23 MiB/s wr, 382 op/s; 24402/295092 objects degraded (8.269%); 18291/295092 objects misplaced (6.198%); 1.5 MiB/s, 2 keys/s, 26 objects/s recovering 2023-09-23T20:47:03.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:03 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/4245103712' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:47:03.562 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:47:03.596 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:03.644 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:47:03.648 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:03.648 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:47:03.749 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:47:03.782 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:03.839 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:47:03.848 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:03.849 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:47:03.950 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:47:04.024 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 79 16 100237 100221 79.242 33.9375 0.0263051 0.0126114 2023-09-23T20:47:04.049 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:04.150 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:47:04.154 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:04.154 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:47:04.213 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:04 smithi018 ceph-mon[179647]: 1.5 scrub starts 2023-09-23T20:47:04.213 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:04 smithi018 ceph-mon[179647]: Health check failed: Degraded data redundancy: 24402/295092 objects degraded (8.269%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T20:47:04.214 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:04 smithi018 ceph-mon[184144]: 1.5 scrub starts 2023-09-23T20:47:04.214 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:04 smithi018 ceph-mon[184144]: Health check failed: Degraded data redundancy: 24402/295092 objects degraded (8.269%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T20:47:04.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:04 smithi149 ceph-mon[170394]: 1.5 scrub starts 2023-09-23T20:47:04.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:04 smithi149 ceph-mon[170394]: Health check failed: Degraded data redundancy: 24402/295092 objects degraded (8.269%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T20:47:04.255 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:47:04.379 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:04.514 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:47:04.519 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:47:04.528 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:04.528 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:47:04.629 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:47:04.686 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:04.742 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T20:47:04.753 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:47:04.768 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:04.768 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:47:04.869 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:47:04.906 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:04.959 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:47:04.964 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:04.965 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:47:05.028 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:2023-09-23T20:47:05.027891+0000 min lat: 0.00123458 max lat: 2.483 avg lat: 0.0127048 2023-09-23T20:47:05.028 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T20:47:05.028 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 80 16 100786 100770 78.678 34.3125 0.0093424 0.0127048 2023-09-23T20:47:05.065 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:47:05.105 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:47:04 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:47:04] "GET /metrics HTTP/1.1" 200 36567 "" "Prometheus/2.43.0" 2023-09-23T20:47:05.211 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:05.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:05 smithi149 ceph-mon[170394]: pgmap v1396: 119 pgs: 1 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+remapped, 6 peering, 104 active+clean; 6.0 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 7.9 KiB/s rd, 22 MiB/s wr, 351 op/s; 24402/296679 objects degraded (8.225%); 24392/296679 objects misplaced (8.222%); 2.2 MiB/s, 1 keys/s, 37 objects/s recovering 2023-09-23T20:47:05.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:05 smithi018 ceph-mon[179647]: pgmap v1396: 119 pgs: 1 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+remapped, 6 peering, 104 active+clean; 6.0 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 7.9 KiB/s rd, 22 MiB/s wr, 351 op/s; 24402/296679 objects degraded (8.225%); 24392/296679 objects misplaced (8.222%); 2.2 MiB/s, 1 keys/s, 37 objects/s recovering 2023-09-23T20:47:05.392 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:47:05.396 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:05 smithi018 ceph-mon[184144]: pgmap v1396: 119 pgs: 1 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+remapped, 6 peering, 104 active+clean; 6.0 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 7.9 KiB/s rd, 22 MiB/s wr, 351 op/s; 24402/296679 objects degraded (8.225%); 24392/296679 objects misplaced (8.222%); 2.2 MiB/s, 1 keys/s, 37 objects/s recovering 2023-09-23T20:47:05.402 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:05.402 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:47:05.442 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2023-09-23T20:47:05.442 INFO:teuthology.orchestra.run.smithi018.stdout: "ops": [ 2023-09-23T20:47:05.442 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:47:05.442 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:100953 10.2 10:4031b8b2:::benchmark_data_smithi018_422431_object100952:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e551)", 2023-09-23T20:47:05.442 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:47:05.423688+0000", 2023-09-23T20:47:05.442 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.016616822999999999, 2023-09-23T20:47:05.443 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.016661615000000001, 2023-09-23T20:47:05.443 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:47:05.443 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:47:05.443 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:47:05.443 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:47:05.443 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:47:05.443 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 100953 2023-09-23T20:47:05.443 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:47:05.443 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:47:05.443 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:47:05.443 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:47:05.443 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:05.423688+0000", 2023-09-23T20:47:05.444 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:47:05.444 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:47:05.444 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:47:05.444 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:47:05.444 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:05.423688+0000", 2023-09-23T20:47:05.444 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:47:05.444 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:47:05.444 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:47:05.444 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:47:05.444 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:05.423687+0000", 2023-09-23T20:47:05.444 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:47:05.444 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:47:05.445 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:47:05.445 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:47:05.445 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:05.423717+0000", 2023-09-23T20:47:05.445 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.0048000000000001e-05 2023-09-23T20:47:05.445 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:47:05.445 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:47:05.445 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:47:05.445 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:05.423732+0000", 2023-09-23T20:47:05.445 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.5172e-05 2023-09-23T20:47:05.445 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:47:05.445 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:47:05.445 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:47:05.446 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:05.423741+0000", 2023-09-23T20:47:05.446 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.9269999999999994e-06 2023-09-23T20:47:05.446 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:47:05.446 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:47:05.446 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:47:05.446 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:05.434836+0000", 2023-09-23T20:47:05.446 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.011095107999999999 2023-09-23T20:47:05.446 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:47:05.446 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:47:05.446 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:47:05.446 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:05.435056+0000", 2023-09-23T20:47:05.446 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00022064799999999999 2023-09-23T20:47:05.447 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:47:05.447 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:47:05.447 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:47:05.447 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:05.435270+0000", 2023-09-23T20:47:05.447 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00021404 2023-09-23T20:47:05.447 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:47:05.447 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:47:05.447 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:47:05.447 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:47:05.447 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:47:05.447 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_repop_reply(client.46625.0:100937 10.2 e551/547 ondisk, result = 0)", 2023-09-23T20:47:05.448 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:47:05.423906+0000", 2023-09-23T20:47:05.448 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.016399146999999999, 2023-09-23T20:47:05.448 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.01652783, 2023-09-23T20:47:05.448 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:47:05.448 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "started", 2023-09-23T20:47:05.448 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:47:05.448 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:47:05.448 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:47:05.448 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:05.423906+0000", 2023-09-23T20:47:05.448 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:47:05.448 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:47:05.448 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:47:05.449 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:47:05.449 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:05.423906+0000", 2023-09-23T20:47:05.449 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:47:05.449 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:47:05.449 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:47:05.449 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:47:05.449 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:05.423905+0000", 2023-09-23T20:47:05.449 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.999999 2023-09-23T20:47:05.449 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:47:05.449 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:47:05.449 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:47:05.449 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:05.423910+0000", 2023-09-23T20:47:05.450 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.8040000000000001e-06 2023-09-23T20:47:05.450 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:47:05.450 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:47:05.450 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:47:05.450 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:05.423920+0000", 2023-09-23T20:47:05.450 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.4499999999999993e-06 2023-09-23T20:47:05.450 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:47:05.450 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:47:05.450 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:47:05.450 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:05.423926+0000", 2023-09-23T20:47:05.450 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.1070000000000001e-06 2023-09-23T20:47:05.451 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:47:05.451 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:47:05.451 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:47:05.451 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:05.438604+0000", 2023-09-23T20:47:05.451 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.014678111000000001 2023-09-23T20:47:05.451 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:47:05.451 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:47:05.451 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:47:05.451 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:05.438614+0000", 2023-09-23T20:47:05.451 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0543000000000001e-05 2023-09-23T20:47:05.451 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:47:05.451 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:47:05.452 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:47:05.452 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:47:05.452 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:47:05.452 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_repop_reply(client.46625.0:100941 10.2 e551/547)", 2023-09-23T20:47:05.452 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:47:05.423932+0000", 2023-09-23T20:47:05.452 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.016373663, 2023-09-23T20:47:05.452 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.016563133000000001, 2023-09-23T20:47:05.452 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:47:05.452 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "queued for pg", 2023-09-23T20:47:05.452 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:47:05.452 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:47:05.452 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:47:05.453 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:05.423932+0000", 2023-09-23T20:47:05.453 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:47:05.453 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:47:05.453 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:47:05.453 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:47:05.453 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:05.423932+0000", 2023-09-23T20:47:05.453 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:47:05.453 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:47:05.453 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:47:05.453 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:47:05.453 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:05.423931+0000", 2023-09-23T20:47:05.453 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:47:05.454 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:47:05.454 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:47:05.454 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:47:05.454 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:05.423933+0000", 2023-09-23T20:47:05.454 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.1720000000000001e-06 2023-09-23T20:47:05.454 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:47:05.454 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:47:05.454 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:47:05.454 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:05.423938+0000", 2023-09-23T20:47:05.454 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.668e-06 2023-09-23T20:47:05.454 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:47:05.455 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:47:05.455 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:47:05.455 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:05.423942+0000", 2023-09-23T20:47:05.455 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.6990000000000001e-06 2023-09-23T20:47:05.455 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:47:05.455 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:47:05.455 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:47:05.455 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:47:05.455 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:47:05.455 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:100955 10.2 10.48d1e872 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e551)", 2023-09-23T20:47:05.455 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:47:05.429554+0000", 2023-09-23T20:47:05.455 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.010750736, 2023-09-23T20:47:05.456 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.010991005999999999, 2023-09-23T20:47:05.456 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:47:05.456 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "queued for pg", 2023-09-23T20:47:05.456 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:47:05.456 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:47:05.456 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:47:05.456 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 100955 2023-09-23T20:47:05.456 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:47:05.456 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:47:05.456 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:47:05.456 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:47:05.456 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:05.429554+0000", 2023-09-23T20:47:05.457 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:47:05.457 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:47:05.457 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:47:05.457 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:47:05.457 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:05.429554+0000", 2023-09-23T20:47:05.457 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:47:05.457 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:47:05.457 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:47:05.457 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:47:05.457 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:05.429553+0000", 2023-09-23T20:47:05.457 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:47:05.457 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:47:05.458 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:47:05.458 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:47:05.458 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:05.429574+0000", 2023-09-23T20:47:05.458 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.052e-05 2023-09-23T20:47:05.458 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:47:05.458 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:47:05.458 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:47:05.458 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:05.429587+0000", 2023-09-23T20:47:05.458 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3633e-05 2023-09-23T20:47:05.458 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:47:05.458 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:47:05.459 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:47:05.459 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:05.429597+0000", 2023-09-23T20:47:05.459 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.7389999999999994e-06 2023-09-23T20:47:05.459 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:47:05.459 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:47:05.459 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:47:05.459 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:47:05.459 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:47:05.459 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:100937 10.2 10:48c576ae:::benchmark_data_smithi018_422431_object100936:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e551)", 2023-09-23T20:47:05.459 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:47:05.393289+0000", 2023-09-23T20:47:05.459 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.047015781999999999, 2023-09-23T20:47:05.460 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.047325535000000002, 2023-09-23T20:47:05.460 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:47:05.460 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:47:05.460 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:47:05.460 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:47:05.460 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:47:05.460 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 100937 2023-09-23T20:47:05.460 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:47:05.460 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:47:05.460 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:47:05.460 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:47:05.460 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:05.393289+0000", 2023-09-23T20:47:05.461 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:47:05.461 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:47:05.461 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:47:05.461 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:47:05.461 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:05.393289+0000", 2023-09-23T20:47:05.461 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:47:05.461 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:47:05.461 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:47:05.461 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:47:05.461 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:05.393287+0000", 2023-09-23T20:47:05.461 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:47:05.461 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:47:05.462 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:47:05.462 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:47:05.462 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:05.393315+0000", 2023-09-23T20:47:05.462 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.7651e-05 2023-09-23T20:47:05.462 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:47:05.462 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:47:05.462 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:47:05.462 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:05.393329+0000", 2023-09-23T20:47:05.462 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3556e-05 2023-09-23T20:47:05.462 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:47:05.463 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:47:05.463 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:47:05.463 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:05.393339+0000", 2023-09-23T20:47:05.463 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0025e-05 2023-09-23T20:47:05.463 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:47:05.463 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:47:05.463 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:47:05.463 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:05.396674+0000", 2023-09-23T20:47:05.463 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0033356879999999998 2023-09-23T20:47:05.463 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:47:05.463 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:47:05.464 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:47:05.464 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:05.396904+0000", 2023-09-23T20:47:05.464 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00023006200000000001 2023-09-23T20:47:05.464 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:47:05.464 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:47:05.464 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:47:05.464 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:05.397112+0000", 2023-09-23T20:47:05.464 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000207462 2023-09-23T20:47:05.464 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:47:05.464 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:47:05.464 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:47:05.464 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:05.400242+0000", 2023-09-23T20:47:05.465 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00313051 2023-09-23T20:47:05.465 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:47:05.465 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:47:05.465 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:47:05.465 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:05.406046+0000", 2023-09-23T20:47:05.465 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0058035459999999997 2023-09-23T20:47:05.465 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:47:05.465 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:47:05.465 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:47:05.465 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:05.438622+0000", 2023-09-23T20:47:05.465 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.032575964999999998 2023-09-23T20:47:05.465 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:47:05.466 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:47:05.466 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:47:05.466 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:05.438678+0000", 2023-09-23T20:47:05.466 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.6127e-05 2023-09-23T20:47:05.466 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:47:05.466 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:47:05.466 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:47:05.466 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:47:05.466 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:47:05.466 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:100941 10.2 10:45b17e7f:::benchmark_data_smithi018_422431_object100940:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e551)", 2023-09-23T20:47:05.467 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:47:05.406942+0000", 2023-09-23T20:47:05.467 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.033363425000000002, 2023-09-23T20:47:05.467 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.033782620999999999, 2023-09-23T20:47:05.467 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:47:05.467 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:47:05.467 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:47:05.467 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:47:05.467 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:47:05.467 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 100941 2023-09-23T20:47:05.467 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:47:05.467 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:47:05.468 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:47:05.468 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:47:05.468 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:05.406942+0000", 2023-09-23T20:47:05.468 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:47:05.468 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:47:05.468 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:47:05.468 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:47:05.468 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:05.406942+0000", 2023-09-23T20:47:05.468 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:47:05.468 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:47:05.468 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:47:05.469 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:47:05.469 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:05.406940+0000", 2023-09-23T20:47:05.469 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:47:05.469 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:47:05.469 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:47:05.469 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:47:05.469 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:05.406969+0000", 2023-09-23T20:47:05.469 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.8609e-05 2023-09-23T20:47:05.469 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:47:05.469 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:47:05.469 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:47:05.469 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:05.406985+0000", 2023-09-23T20:47:05.470 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.6639000000000001e-05 2023-09-23T20:47:05.470 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:47:05.470 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:47:05.470 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:47:05.470 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:05.406994+0000", 2023-09-23T20:47:05.470 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.0659999999999993e-06 2023-09-23T20:47:05.470 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:47:05.470 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:47:05.470 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:47:05.470 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:05.407051+0000", 2023-09-23T20:47:05.470 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.6416999999999999e-05 2023-09-23T20:47:05.470 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:47:05.471 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:47:05.471 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:47:05.471 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:05.407261+0000", 2023-09-23T20:47:05.471 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00021024699999999999 2023-09-23T20:47:05.471 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:47:05.471 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:47:05.471 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:47:05.471 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:05.407446+0000", 2023-09-23T20:47:05.471 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00018484 2023-09-23T20:47:05.471 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:47:05.471 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:47:05.471 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:47:05.472 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:05.410953+0000", 2023-09-23T20:47:05.472 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.003507109 2023-09-23T20:47:05.472 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:47:05.472 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:47:05.472 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:47:05.472 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:05.422852+0000", 2023-09-23T20:47:05.472 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.011899104000000001 2023-09-23T20:47:05.472 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:47:05.472 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:47:05.472 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:47:05.472 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:47:05.472 INFO:teuthology.orchestra.run.smithi018.stdout: ], 2023-09-23T20:47:05.473 INFO:teuthology.orchestra.run.smithi018.stdout: "num_ops": 6 2023-09-23T20:47:05.473 INFO:teuthology.orchestra.run.smithi018.stdout:} 2023-09-23T20:47:05.503 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:47:05.541 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:05.591 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:47:05.597 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:05.598 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:47:05.699 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:47:05.808 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:05.924 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:47:05.939 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:05.940 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:47:06.027 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 81 16 101286 101270 78.0927 31.25 0.0152615 0.0127995 2023-09-23T20:47:06.041 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:47:06.098 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T20:47:06.123 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:47:06.147 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:06.275 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:47:06.293 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:47:06.297 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:06.297 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:47:06.394 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:06 smithi018 ceph-mon[179647]: 1.5 scrub starts 2023-09-23T20:47:06.394 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:06 smithi018 ceph-mon[179647]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-09-23T20:47:06.398 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:47:06.405 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:06 smithi018 ceph-mon[184144]: 1.5 scrub starts 2023-09-23T20:47:06.405 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:06 smithi018 ceph-mon[184144]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-09-23T20:47:06.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:06 smithi149 ceph-mon[170394]: 1.5 scrub starts 2023-09-23T20:47:06.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:06 smithi149 ceph-mon[170394]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-09-23T20:47:06.539 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:06.755 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:47:06.761 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:06.762 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:47:06.862 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:47:06.899 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:06.949 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:47:06.956 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:06.956 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:47:07.027 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 82 16 101716 101700 77.4683 26.875 0.0286053 0.0129025 2023-09-23T20:47:07.057 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:47:07.184 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:07.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:07 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mon-a[179623]: 2023-09-23T20:47:07.215+0000 7f160fef9700 -1 mon.a@0(leader).osd e551 definitely_dead 0 2023-09-23T20:47:07.430 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:47:07.443 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:07.443 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:47:07.497 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:47:07.544 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:47:07.575 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:07.624 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:47:07.629 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:07.629 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:47:07.698 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:47:07.730 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:47:07.824 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:07 smithi018 ceph-mon[179647]: pgmap v1397: 119 pgs: 1 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+remapped, 110 active+clean; 6.1 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 6.6 KiB/s rd, 32 MiB/s wr, 517 op/s; 24402/300873 objects degraded (8.110%); 24540/300873 objects misplaced (8.156%); 3.4 MiB/s, 1 keys/s, 57 objects/s recovering 2023-09-23T20:47:07.825 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:07 smithi018 ceph-mon[179647]: 1.5 deep-scrub starts 2023-09-23T20:47:07.825 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:07 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/899914096' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["4"]}]: dispatch 2023-09-23T20:47:07.825 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:07 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["4"]}]: dispatch 2023-09-23T20:47:07.825 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:07 smithi018 ceph-mon[184144]: pgmap v1397: 119 pgs: 1 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+remapped, 110 active+clean; 6.1 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 6.6 KiB/s rd, 32 MiB/s wr, 517 op/s; 24402/300873 objects degraded (8.110%); 24540/300873 objects misplaced (8.156%); 3.4 MiB/s, 1 keys/s, 57 objects/s recovering 2023-09-23T20:47:07.825 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:07 smithi018 ceph-mon[184144]: 1.5 deep-scrub starts 2023-09-23T20:47:07.825 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:07 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/899914096' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["4"]}]: dispatch 2023-09-23T20:47:07.825 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:07 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["4"]}]: dispatch 2023-09-23T20:47:07.889 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:07.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:07 smithi149 ceph-mon[170394]: pgmap v1397: 119 pgs: 1 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+remapped, 110 active+clean; 6.1 GiB data, 20 GiB used, 606 GiB / 626 GiB avail; 6.6 KiB/s rd, 32 MiB/s wr, 517 op/s; 24402/300873 objects degraded (8.110%); 24540/300873 objects misplaced (8.156%); 3.4 MiB/s, 1 keys/s, 57 objects/s recovering 2023-09-23T20:47:07.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:07 smithi149 ceph-mon[170394]: 1.5 deep-scrub starts 2023-09-23T20:47:07.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:07 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/899914096' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["4"]}]: dispatch 2023-09-23T20:47:07.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:07 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["4"]}]: dispatch 2023-09-23T20:47:08.030 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 83 16 102207 102191 76.9026 30.6875 0.0123263 0.0129969 2023-09-23T20:47:08.099 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:47:08.105 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:08.106 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:47:08.166 INFO:teuthology.orchestra.run.smithi018.stderr:marked in osd.4. 2023-09-23T20:47:08.207 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:47:08.262 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:08.454 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:47:08.458 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:08.458 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:47:08.559 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:47:08.586 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:08.619 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:47:08.622 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:08.622 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:47:08.723 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:47:08.749 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:08.782 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:47:08.785 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:08.786 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:47:08.886 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:47:08.916 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:09.030 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 84 16 102300 102284 76.0567 5.8125 0.0131504 0.0130131 2023-09-23T20:47:09.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:09 smithi149 ceph-mon[170394]: 1.5 scrub starts 2023-09-23T20:47:09.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:09 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["4"]}]': finished 2023-09-23T20:47:09.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:09 smithi149 ceph-mon[170394]: osdmap e552: 8 total, 8 up, 8 in 2023-09-23T20:47:09.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:09 smithi149 ceph-mon[170394]: failed to encode map e552 with expected crc 2023-09-23T20:47:09.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:09 smithi149 ceph-mon[170394]: failed to encode map e552 with expected crc 2023-09-23T20:47:09.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:09 smithi149 ceph-mon[170394]: failed to encode map e552 with expected crc 2023-09-23T20:47:09.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:09 smithi149 ceph-mon[170394]: pgmap v1399: 119 pgs: 1 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+remapped, 110 active+clean; 6.2 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 27 MiB/s wr, 428 op/s; 24402/303708 objects degraded (8.035%); 24540/303708 objects misplaced (8.080%); 2.1 MiB/s, 34 objects/s recovering 2023-09-23T20:47:09.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:09 smithi018 ceph-mon[184144]: 1.5 scrub starts 2023-09-23T20:47:09.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:09 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["4"]}]': finished 2023-09-23T20:47:09.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:09 smithi018 ceph-mon[184144]: osdmap e552: 8 total, 8 up, 8 in 2023-09-23T20:47:09.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:09 smithi018 ceph-mon[184144]: failed to encode map e552 with expected crc 2023-09-23T20:47:09.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:09 smithi018 ceph-mon[184144]: failed to encode map e552 with expected crc 2023-09-23T20:47:09.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:09 smithi018 ceph-mon[184144]: failed to encode map e552 with expected crc 2023-09-23T20:47:09.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:09 smithi018 ceph-mon[184144]: pgmap v1399: 119 pgs: 1 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+remapped, 110 active+clean; 6.2 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 27 MiB/s wr, 428 op/s; 24402/303708 objects degraded (8.035%); 24540/303708 objects misplaced (8.080%); 2.1 MiB/s, 34 objects/s recovering 2023-09-23T20:47:09.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:09 smithi018 ceph-mon[179647]: 1.5 scrub starts 2023-09-23T20:47:09.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:09 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["4"]}]': finished 2023-09-23T20:47:09.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:09 smithi018 ceph-mon[179647]: osdmap e552: 8 total, 8 up, 8 in 2023-09-23T20:47:09.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:09 smithi018 ceph-mon[179647]: failed to encode map e552 with expected crc 2023-09-23T20:47:09.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:09 smithi018 ceph-mon[179647]: failed to encode map e552 with expected crc 2023-09-23T20:47:09.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:09 smithi018 ceph-mon[179647]: failed to encode map e552 with expected crc 2023-09-23T20:47:09.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:09 smithi018 ceph-mon[179647]: pgmap v1399: 119 pgs: 1 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 2 active+recovering+undersized+remapped, 110 active+clean; 6.2 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 27 MiB/s wr, 428 op/s; 24402/303708 objects degraded (8.035%); 24540/303708 objects misplaced (8.080%); 2.1 MiB/s, 34 objects/s recovering 2023-09-23T20:47:09.962 INFO:tasks.thrashosds.thrasher:Added osd 4 2023-09-23T20:47:10.030 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 85 16 102396 102380 75.2329 6 0.00399458 0.013167 2023-09-23T20:47:10.245 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:10 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 24402/303708 objects degraded (8.035%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T20:47:10.245 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:10 smithi018 ceph-mon[179647]: failed to encode map e553 with expected crc 2023-09-23T20:47:10.245 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:10 smithi018 ceph-mon[179647]: osdmap e553: 8 total, 8 up, 8 in 2023-09-23T20:47:10.245 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:10 smithi018 ceph-mon[179647]: failed to encode map e553 with expected crc 2023-09-23T20:47:10.246 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:10 smithi018 ceph-mon[179647]: failed to encode map e553 with expected crc 2023-09-23T20:47:10.246 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:10 smithi018 ceph-mon[179647]: failed to encode map e553 with expected crc 2023-09-23T20:47:10.246 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:10 smithi018 ceph-mon[179647]: failed to encode map e553 with expected crc 2023-09-23T20:47:10.246 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:10 smithi018 ceph-mon[179647]: failed to encode map e553 with expected crc 2023-09-23T20:47:10.249 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:10 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 24402/303708 objects degraded (8.035%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T20:47:10.249 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:10 smithi018 ceph-mon[184144]: failed to encode map e553 with expected crc 2023-09-23T20:47:10.249 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:10 smithi018 ceph-mon[184144]: osdmap e553: 8 total, 8 up, 8 in 2023-09-23T20:47:10.249 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:10 smithi018 ceph-mon[184144]: failed to encode map e553 with expected crc 2023-09-23T20:47:10.249 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:10 smithi018 ceph-mon[184144]: failed to encode map e553 with expected crc 2023-09-23T20:47:10.249 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:10 smithi018 ceph-mon[184144]: failed to encode map e553 with expected crc 2023-09-23T20:47:10.250 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:10 smithi018 ceph-mon[184144]: failed to encode map e553 with expected crc 2023-09-23T20:47:10.250 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:10 smithi018 ceph-mon[184144]: failed to encode map e553 with expected crc 2023-09-23T20:47:10.307 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:47:10.314 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:10.315 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:47:10.416 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:47:10.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:10 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 24402/303708 objects degraded (8.035%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T20:47:10.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:10 smithi149 ceph-mon[170394]: failed to encode map e553 with expected crc 2023-09-23T20:47:10.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:10 smithi149 ceph-mon[170394]: osdmap e553: 8 total, 8 up, 8 in 2023-09-23T20:47:10.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:10 smithi149 ceph-mon[170394]: failed to encode map e553 with expected crc 2023-09-23T20:47:10.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:10 smithi149 ceph-mon[170394]: failed to encode map e553 with expected crc 2023-09-23T20:47:10.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:10 smithi149 ceph-mon[170394]: failed to encode map e553 with expected crc 2023-09-23T20:47:10.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:10 smithi149 ceph-mon[170394]: failed to encode map e553 with expected crc 2023-09-23T20:47:10.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:10 smithi149 ceph-mon[170394]: failed to encode map e553 with expected crc 2023-09-23T20:47:10.473 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:10.544 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:47:10.556 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:10.556 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:47:10.657 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:47:10.724 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:10.808 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:47:10.813 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:10.813 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:47:10.914 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:47:10.983 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:11.030 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 86 16 103158 103142 74.912 47.625 0.0263608 0.0133436 2023-09-23T20:47:11.061 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:47:11.072 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:11.072 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:47:11.173 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:47:11.238 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:11.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:11 smithi149 ceph-mon[170394]: failed to encode map e554 with expected crc 2023-09-23T20:47:11.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:11 smithi149 ceph-mon[170394]: osdmap e554: 8 total, 8 up, 8 in 2023-09-23T20:47:11.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:11 smithi149 ceph-mon[170394]: failed to encode map e554 with expected crc 2023-09-23T20:47:11.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:11 smithi149 ceph-mon[170394]: failed to encode map e554 with expected crc 2023-09-23T20:47:11.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:11 smithi149 ceph-mon[170394]: failed to encode map e554 with expected crc 2023-09-23T20:47:11.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:11 smithi149 ceph-mon[170394]: failed to encode map e554 with expected crc 2023-09-23T20:47:11.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:11 smithi149 ceph-mon[170394]: failed to encode map e554 with expected crc 2023-09-23T20:47:11.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:11 smithi149 ceph-mon[170394]: failed to encode map e554 with expected crc 2023-09-23T20:47:11.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:11 smithi149 ceph-mon[170394]: failed to encode map e554 with expected crc 2023-09-23T20:47:11.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:11 smithi149 ceph-mon[170394]: pgmap v1402: 119 pgs: 1 remapped+peering, 1 unknown, 6 peering, 1 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 103 active+clean; 6.2 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 26 MiB/s wr, 415 op/s; 24402/304152 objects degraded (8.023%); 18439/304152 objects misplaced (6.062%); 2.5 MiB/s, 40 objects/s recovering 2023-09-23T20:47:11.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:11 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:47:11.544 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:47:11.548 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:11 smithi018 ceph-mon[184144]: failed to encode map e554 with expected crc 2023-09-23T20:47:11.548 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:11 smithi018 ceph-mon[184144]: osdmap e554: 8 total, 8 up, 8 in 2023-09-23T20:47:11.548 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:11 smithi018 ceph-mon[184144]: failed to encode map e554 with expected crc 2023-09-23T20:47:11.549 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:11 smithi018 ceph-mon[184144]: failed to encode map e554 with expected crc 2023-09-23T20:47:11.549 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:11 smithi018 ceph-mon[184144]: failed to encode map e554 with expected crc 2023-09-23T20:47:11.549 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:11 smithi018 ceph-mon[184144]: failed to encode map e554 with expected crc 2023-09-23T20:47:11.549 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:11 smithi018 ceph-mon[184144]: failed to encode map e554 with expected crc 2023-09-23T20:47:11.549 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:11 smithi018 ceph-mon[184144]: failed to encode map e554 with expected crc 2023-09-23T20:47:11.549 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:11 smithi018 ceph-mon[184144]: failed to encode map e554 with expected crc 2023-09-23T20:47:11.549 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:11 smithi018 ceph-mon[184144]: pgmap v1402: 119 pgs: 1 remapped+peering, 1 unknown, 6 peering, 1 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 103 active+clean; 6.2 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 26 MiB/s wr, 415 op/s; 24402/304152 objects degraded (8.023%); 18439/304152 objects misplaced (6.062%); 2.5 MiB/s, 40 objects/s recovering 2023-09-23T20:47:11.549 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:11 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:47:11.549 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:11 smithi018 ceph-mon[179647]: failed to encode map e554 with expected crc 2023-09-23T20:47:11.550 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:11 smithi018 ceph-mon[179647]: osdmap e554: 8 total, 8 up, 8 in 2023-09-23T20:47:11.550 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:11 smithi018 ceph-mon[179647]: failed to encode map e554 with expected crc 2023-09-23T20:47:11.550 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:11 smithi018 ceph-mon[179647]: failed to encode map e554 with expected crc 2023-09-23T20:47:11.550 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:11 smithi018 ceph-mon[179647]: failed to encode map e554 with expected crc 2023-09-23T20:47:11.550 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:11 smithi018 ceph-mon[179647]: failed to encode map e554 with expected crc 2023-09-23T20:47:11.550 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:11 smithi018 ceph-mon[179647]: failed to encode map e554 with expected crc 2023-09-23T20:47:11.550 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:11 smithi018 ceph-mon[179647]: failed to encode map e554 with expected crc 2023-09-23T20:47:11.550 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:11 smithi018 ceph-mon[179647]: failed to encode map e554 with expected crc 2023-09-23T20:47:11.550 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:11 smithi018 ceph-mon[179647]: pgmap v1402: 119 pgs: 1 remapped+peering, 1 unknown, 6 peering, 1 active+clean+scrubbing, 4 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 103 active+clean; 6.2 GiB data, 20 GiB used, 605 GiB / 626 GiB avail; 26 MiB/s wr, 415 op/s; 24402/304152 objects degraded (8.023%); 18439/304152 objects misplaced (6.062%); 2.5 MiB/s, 40 objects/s recovering 2023-09-23T20:47:11.550 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:11 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:47:11.551 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:11.551 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:47:11.652 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:47:11.717 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:11.769 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:47:11.773 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:11.773 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:47:11.874 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:47:11.912 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:12.031 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 87 16 103725 103709 74.4585 35.4375 0.00386195 0.0133892 2023-09-23T20:47:12.151 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:47:12.154 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:12.154 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:47:12.255 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:47:12.283 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:12.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:12 smithi149 ceph-mon[170394]: failed to encode map e555 with expected crc 2023-09-23T20:47:12.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:12 smithi149 ceph-mon[170394]: osdmap e555: 8 total, 8 up, 8 in 2023-09-23T20:47:12.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:12 smithi149 ceph-mon[170394]: failed to encode map e555 with expected crc 2023-09-23T20:47:12.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:12 smithi149 ceph-mon[170394]: failed to encode map e555 with expected crc 2023-09-23T20:47:12.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:12 smithi149 ceph-mon[170394]: failed to encode map e556 with expected crc 2023-09-23T20:47:12.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:12 smithi149 ceph-mon[170394]: osdmap e556: 8 total, 8 up, 8 in 2023-09-23T20:47:12.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:12 smithi149 ceph-mon[170394]: failed to encode map e556 with expected crc 2023-09-23T20:47:12.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:12 smithi149 ceph-mon[170394]: failed to encode map e556 with expected crc 2023-09-23T20:47:12.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:12 smithi149 ceph-mon[170394]: failed to encode map e556 with expected crc 2023-09-23T20:47:12.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:12 smithi149 ceph-mon[170394]: failed to encode map e556 with expected crc 2023-09-23T20:47:12.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:12 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2683103801' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:47:12.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:12 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:47:12.628 INFO:teuthology.orchestra.run.smithi018.stderr:noscrub is set 2023-09-23T20:47:12.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:12 smithi018 ceph-mon[184144]: failed to encode map e555 with expected crc 2023-09-23T20:47:12.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:12 smithi018 ceph-mon[184144]: osdmap e555: 8 total, 8 up, 8 in 2023-09-23T20:47:12.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:12 smithi018 ceph-mon[184144]: failed to encode map e555 with expected crc 2023-09-23T20:47:12.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:12 smithi018 ceph-mon[184144]: failed to encode map e555 with expected crc 2023-09-23T20:47:12.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:12 smithi018 ceph-mon[184144]: failed to encode map e556 with expected crc 2023-09-23T20:47:12.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:12 smithi018 ceph-mon[184144]: osdmap e556: 8 total, 8 up, 8 in 2023-09-23T20:47:12.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:12 smithi018 ceph-mon[184144]: failed to encode map e556 with expected crc 2023-09-23T20:47:12.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:12 smithi018 ceph-mon[184144]: failed to encode map e556 with expected crc 2023-09-23T20:47:12.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:12 smithi018 ceph-mon[184144]: failed to encode map e556 with expected crc 2023-09-23T20:47:12.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:12 smithi018 ceph-mon[184144]: failed to encode map e556 with expected crc 2023-09-23T20:47:12.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:12 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2683103801' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:47:12.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:12 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:47:12.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:12 smithi018 ceph-mon[179647]: failed to encode map e555 with expected crc 2023-09-23T20:47:12.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:12 smithi018 ceph-mon[179647]: osdmap e555: 8 total, 8 up, 8 in 2023-09-23T20:47:12.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:12 smithi018 ceph-mon[179647]: failed to encode map e555 with expected crc 2023-09-23T20:47:12.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:12 smithi018 ceph-mon[179647]: failed to encode map e555 with expected crc 2023-09-23T20:47:12.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:12 smithi018 ceph-mon[179647]: failed to encode map e556 with expected crc 2023-09-23T20:47:12.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:12 smithi018 ceph-mon[179647]: osdmap e556: 8 total, 8 up, 8 in 2023-09-23T20:47:12.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:12 smithi018 ceph-mon[179647]: failed to encode map e556 with expected crc 2023-09-23T20:47:12.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:12 smithi018 ceph-mon[179647]: failed to encode map e556 with expected crc 2023-09-23T20:47:12.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:12 smithi018 ceph-mon[179647]: failed to encode map e556 with expected crc 2023-09-23T20:47:12.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:12 smithi018 ceph-mon[179647]: failed to encode map e556 with expected crc 2023-09-23T20:47:12.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:12 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2683103801' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:47:12.643 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:12 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:47:12.840 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:47:12.848 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:12.848 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:47:12.949 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:47:12.982 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:13.031 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 88 16 104017 104001 73.82 18.25 0.00418873 0.0135166 2023-09-23T20:47:13.229 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:47:13.231 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:13 smithi018 ceph-mon[184144]: pgmap v1405: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 remapped+peering, 1 unknown, 6 peering, 1 active+clean+scrubbing, 1 active+recovery_wait, 4 active+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 6.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 23 MiB/s wr, 374 op/s; 849/308199 objects degraded (0.275%); 36/308199 objects misplaced (0.012%); 11 MiB/s, 2 keys/s, 183 objects/s recovering 2023-09-23T20:47:13.231 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:13 smithi018 ceph-mon[184144]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:47:13.231 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:13 smithi018 ceph-mon[184144]: failed to encode map e557 with expected crc 2023-09-23T20:47:13.231 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:13 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:47:13.232 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:13 smithi018 ceph-mon[184144]: osdmap e557: 8 total, 8 up, 8 in 2023-09-23T20:47:13.232 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:13 smithi018 ceph-mon[184144]: failed to encode map e557 with expected crc 2023-09-23T20:47:13.232 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:13 smithi018 ceph-mon[184144]: failed to encode map e557 with expected crc 2023-09-23T20:47:13.232 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:13 smithi018 ceph-mon[179647]: pgmap v1405: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 remapped+peering, 1 unknown, 6 peering, 1 active+clean+scrubbing, 1 active+recovery_wait, 4 active+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 6.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 23 MiB/s wr, 374 op/s; 849/308199 objects degraded (0.275%); 36/308199 objects misplaced (0.012%); 11 MiB/s, 2 keys/s, 183 objects/s recovering 2023-09-23T20:47:13.232 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:13 smithi018 ceph-mon[179647]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:47:13.232 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:13 smithi018 ceph-mon[179647]: failed to encode map e557 with expected crc 2023-09-23T20:47:13.232 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:13 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:47:13.232 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:13 smithi018 ceph-mon[179647]: osdmap e557: 8 total, 8 up, 8 in 2023-09-23T20:47:13.233 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:13 smithi018 ceph-mon[179647]: failed to encode map e557 with expected crc 2023-09-23T20:47:13.233 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:13 smithi018 ceph-mon[179647]: failed to encode map e557 with expected crc 2023-09-23T20:47:13.233 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:13.233 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:47:13.334 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:47:13.363 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:13.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:13 smithi149 ceph-mon[170394]: pgmap v1405: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 remapped+peering, 1 unknown, 6 peering, 1 active+clean+scrubbing, 1 active+recovery_wait, 4 active+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 6.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 23 MiB/s wr, 374 op/s; 849/308199 objects degraded (0.275%); 36/308199 objects misplaced (0.012%); 11 MiB/s, 2 keys/s, 183 objects/s recovering 2023-09-23T20:47:13.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:13 smithi149 ceph-mon[170394]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:47:13.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:13 smithi149 ceph-mon[170394]: failed to encode map e557 with expected crc 2023-09-23T20:47:13.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:13 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:47:13.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:13 smithi149 ceph-mon[170394]: osdmap e557: 8 total, 8 up, 8 in 2023-09-23T20:47:13.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:13 smithi149 ceph-mon[170394]: failed to encode map e557 with expected crc 2023-09-23T20:47:13.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:13 smithi149 ceph-mon[170394]: failed to encode map e557 with expected crc 2023-09-23T20:47:14.031 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 89 16 104229 104213 73.1398 13.25 0.00432522 0.013631 2023-09-23T20:47:14.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:14 smithi149 ceph-mon[170394]: failed to encode map e557 with expected crc 2023-09-23T20:47:14.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:14 smithi149 ceph-mon[170394]: failed to encode map e558 with expected crc 2023-09-23T20:47:14.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:14 smithi149 ceph-mon[170394]: osdmap e558: 8 total, 8 up, 8 in 2023-09-23T20:47:14.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:14 smithi149 ceph-mon[170394]: failed to encode map e558 with expected crc 2023-09-23T20:47:14.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:14 smithi149 ceph-mon[170394]: failed to encode map e558 with expected crc 2023-09-23T20:47:14.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:14 smithi018 ceph-mon[179647]: failed to encode map e557 with expected crc 2023-09-23T20:47:14.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:14 smithi018 ceph-mon[179647]: failed to encode map e558 with expected crc 2023-09-23T20:47:14.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:14 smithi018 ceph-mon[179647]: osdmap e558: 8 total, 8 up, 8 in 2023-09-23T20:47:14.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:14 smithi018 ceph-mon[179647]: failed to encode map e558 with expected crc 2023-09-23T20:47:14.642 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:14 smithi018 ceph-mon[179647]: failed to encode map e558 with expected crc 2023-09-23T20:47:14.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:14 smithi018 ceph-mon[184144]: failed to encode map e557 with expected crc 2023-09-23T20:47:14.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:14 smithi018 ceph-mon[184144]: failed to encode map e558 with expected crc 2023-09-23T20:47:14.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:14 smithi018 ceph-mon[184144]: osdmap e558: 8 total, 8 up, 8 in 2023-09-23T20:47:14.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:14 smithi018 ceph-mon[184144]: failed to encode map e558 with expected crc 2023-09-23T20:47:14.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:14 smithi018 ceph-mon[184144]: failed to encode map e558 with expected crc 2023-09-23T20:47:14.963 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 7, 3, 2, 6, 5, 4] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T20:47:14.963 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:47:14.964 INFO:tasks.thrashosds.thrasher:inject_pause on osd.2 2023-09-23T20:47:14.964 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T20:47:14.964 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T20:47:14.964 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 config set filestore_inject_stall 3 2023-09-23T20:47:15.031 INFO:tasks.radosbench.radosbench.0.smithi018.stdout: 90 11 104493 104482 72.5142 16.8125 0.00629777 0.0137601 2023-09-23T20:47:15.218 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:47:15.221 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:15.222 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:47:15.322 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:47:15.347 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:47:14 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:47:14] "GET /metrics HTTP/1.1" 200 36607 "" "Prometheus/2.43.0" 2023-09-23T20:47:15.347 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:15 smithi018 ceph-mon[179647]: failed to encode map e558 with expected crc 2023-09-23T20:47:15.347 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:15 smithi018 ceph-mon[179647]: pgmap v1408: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+clean+scrubbing, 1 active+recovery_wait, 4 active+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 6.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 28 MiB/s wr, 449 op/s; 1151/309552 objects degraded (0.372%); 36/309552 objects misplaced (0.012%); 10 MiB/s, 2 keys/s, 165 objects/s recovering 2023-09-23T20:47:15.347 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:15 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:47:15.347 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:15 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:47:15.347 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:15 smithi018 ceph-mon[179647]: osdmap e559: 8 total, 8 up, 8 in 2023-09-23T20:47:15.348 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:15 smithi018 ceph-mon[179647]: failed to encode map e559 with expected crc 2023-09-23T20:47:15.348 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:15 smithi018 ceph-mon[179647]: failed to encode map e559 with expected crc 2023-09-23T20:47:15.348 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:15 smithi018 ceph-mon[179647]: failed to encode map e559 with expected crc 2023-09-23T20:47:15.348 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:15 smithi018 ceph-mon[179647]: failed to encode map e559 with expected crc 2023-09-23T20:47:15.348 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:15 smithi018 ceph-mon[184144]: failed to encode map e558 with expected crc 2023-09-23T20:47:15.348 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:15 smithi018 ceph-mon[184144]: pgmap v1408: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+clean+scrubbing, 1 active+recovery_wait, 4 active+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 6.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 28 MiB/s wr, 449 op/s; 1151/309552 objects degraded (0.372%); 36/309552 objects misplaced (0.012%); 10 MiB/s, 2 keys/s, 165 objects/s recovering 2023-09-23T20:47:15.348 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:15 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:47:15.348 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:15 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:47:15.348 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:15 smithi018 ceph-mon[184144]: osdmap e559: 8 total, 8 up, 8 in 2023-09-23T20:47:15.348 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:15 smithi018 ceph-mon[184144]: failed to encode map e559 with expected crc 2023-09-23T20:47:15.349 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:15 smithi018 ceph-mon[184144]: failed to encode map e559 with expected crc 2023-09-23T20:47:15.349 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:15 smithi018 ceph-mon[184144]: failed to encode map e559 with expected crc 2023-09-23T20:47:15.349 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:15 smithi018 ceph-mon[184144]: failed to encode map e559 with expected crc 2023-09-23T20:47:15.349 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:15.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:15 smithi149 ceph-mon[170394]: failed to encode map e558 with expected crc 2023-09-23T20:47:15.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:15 smithi149 ceph-mon[170394]: pgmap v1408: 119 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 1 active+clean+scrubbing, 1 active+recovery_wait, 4 active+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 6.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 28 MiB/s wr, 449 op/s; 1151/309552 objects degraded (0.372%); 36/309552 objects misplaced (0.012%); 10 MiB/s, 2 keys/s, 165 objects/s recovering 2023-09-23T20:47:15.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:15 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:47:15.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:15 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:47:15.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:15 smithi149 ceph-mon[170394]: osdmap e559: 8 total, 8 up, 8 in 2023-09-23T20:47:15.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:15 smithi149 ceph-mon[170394]: failed to encode map e559 with expected crc 2023-09-23T20:47:15.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:15 smithi149 ceph-mon[170394]: failed to encode map e559 with expected crc 2023-09-23T20:47:15.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:15 smithi149 ceph-mon[170394]: failed to encode map e559 with expected crc 2023-09-23T20:47:15.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:15 smithi149 ceph-mon[170394]: failed to encode map e559 with expected crc 2023-09-23T20:47:15.861 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:47:15.864 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:15.864 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:47:15.965 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:47:15.976 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2023-09-23T20:47:15.976 INFO:teuthology.orchestra.run.smithi018.stdout: "ops": [], 2023-09-23T20:47:15.976 INFO:teuthology.orchestra.run.smithi018.stdout: "complaint_time": 30, 2023-09-23T20:47:15.976 INFO:teuthology.orchestra.run.smithi018.stdout: "num_blocked_ops": 0 2023-09-23T20:47:15.976 INFO:teuthology.orchestra.run.smithi018.stdout:} 2023-09-23T20:47:15.996 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:16.031 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Total time run: 90.6762 2023-09-23T20:47:16.031 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Total writes made: 104493 2023-09-23T20:47:16.031 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Write size: 65536 2023-09-23T20:47:16.031 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Object size: 65536 2023-09-23T20:47:16.032 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Bandwidth (MB/sec): 72.0234 2023-09-23T20:47:16.032 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Stddev Bandwidth: 41.2998 2023-09-23T20:47:16.032 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Max bandwidth (MB/sec): 210.5 2023-09-23T20:47:16.032 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Min bandwidth (MB/sec): 0 2023-09-23T20:47:16.032 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Average IOPS: 1152 2023-09-23T20:47:16.032 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Stddev IOPS: 660.803 2023-09-23T20:47:16.032 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Max IOPS: 3368 2023-09-23T20:47:16.032 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Min IOPS: 0 2023-09-23T20:47:16.032 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Average Latency(s): 0.0138487 2023-09-23T20:47:16.032 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Stddev Latency(s): 0.0411257 2023-09-23T20:47:16.032 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Max latency(s): 2.483 2023-09-23T20:47:16.033 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Min latency(s): 0.00123458 2023-09-23T20:47:16.037 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:47:16.039 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:16.040 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:47:16.040 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Cleaning up (deleting benchmark objects) 2023-09-23T20:47:16.141 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:47:16.150 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:47:16.192 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:47:16.199 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:16.224 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:47:16.258 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:47:16.261 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:16 smithi018 ceph-mon[184144]: failed to encode map e559 with expected crc 2023-09-23T20:47:16.261 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:16 smithi018 ceph-mon[184144]: Health check update: Degraded data redundancy: 1151/309552 objects degraded (0.372%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T20:47:16.261 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:16 smithi018 ceph-mon[184144]: failed to encode map e560 with expected crc 2023-09-23T20:47:16.261 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:16 smithi018 ceph-mon[184144]: osdmap e560: 8 total, 8 up, 8 in 2023-09-23T20:47:16.261 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:16 smithi018 ceph-mon[184144]: failed to encode map e560 with expected crc 2023-09-23T20:47:16.261 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:16 smithi018 ceph-mon[184144]: failed to encode map e560 with expected crc 2023-09-23T20:47:16.262 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:16 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/4219255140' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:47:16.262 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:16 smithi018 ceph-mon[179647]: failed to encode map e559 with expected crc 2023-09-23T20:47:16.262 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:16 smithi018 ceph-mon[179647]: Health check update: Degraded data redundancy: 1151/309552 objects degraded (0.372%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T20:47:16.262 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:16 smithi018 ceph-mon[179647]: failed to encode map e560 with expected crc 2023-09-23T20:47:16.263 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:16 smithi018 ceph-mon[179647]: osdmap e560: 8 total, 8 up, 8 in 2023-09-23T20:47:16.263 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:16 smithi018 ceph-mon[179647]: failed to encode map e560 with expected crc 2023-09-23T20:47:16.263 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:16 smithi018 ceph-mon[179647]: failed to encode map e560 with expected crc 2023-09-23T20:47:16.263 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:16 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/4219255140' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:47:16.263 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:47:16.273 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:47:16.282 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:16.282 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:47:16.290 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:47:16.320 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:47:16.346 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:47:16.369 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:47:16.370 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:47:16.370 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:47:16.370 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:47:16.370 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:47:16.371 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:47:16.371 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:47:16.371 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:47:16.371 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:47:16.384 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:47:16.411 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:16.449 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:47:16.453 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:16 smithi149 ceph-mon[170394]: failed to encode map e559 with expected crc 2023-09-23T20:47:16.453 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:16 smithi149 ceph-mon[170394]: Health check update: Degraded data redundancy: 1151/309552 objects degraded (0.372%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T20:47:16.453 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:16 smithi149 ceph-mon[170394]: failed to encode map e560 with expected crc 2023-09-23T20:47:16.454 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:16 smithi149 ceph-mon[170394]: osdmap e560: 8 total, 8 up, 8 in 2023-09-23T20:47:16.454 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:16 smithi149 ceph-mon[170394]: failed to encode map e560 with expected crc 2023-09-23T20:47:16.454 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:16 smithi149 ceph-mon[170394]: failed to encode map e560 with expected crc 2023-09-23T20:47:16.455 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:16 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/4219255140' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:47:16.456 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:16.457 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:47:16.559 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:47:16.587 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:16.739 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T20:47:17.036 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:47:17.045 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:17.045 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:47:17.146 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:47:17.232 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:17.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:17 smithi018 ceph-mon[179647]: failed to encode map e560 with expected crc 2023-09-23T20:47:17.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:17 smithi018 ceph-mon[179647]: pgmap v1411: 119 pgs: 1 peering, 1 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 6.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 23 MiB/s wr, 362 op/s; 575/312546 objects degraded (0.184%); 9.5 MiB/s, 156 objects/s recovering 2023-09-23T20:47:17.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:17 smithi018 ceph-mon[179647]: failed to encode map e561 with expected crc 2023-09-23T20:47:17.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:17 smithi018 ceph-mon[179647]: osdmap e561: 8 total, 8 up, 8 in 2023-09-23T20:47:17.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:17 smithi018 ceph-mon[179647]: failed to encode map e561 with expected crc 2023-09-23T20:47:17.394 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:17 smithi018 ceph-mon[179647]: failed to encode map e561 with expected crc 2023-09-23T20:47:17.394 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:17 smithi018 ceph-mon[184144]: failed to encode map e560 with expected crc 2023-09-23T20:47:17.394 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:17 smithi018 ceph-mon[184144]: pgmap v1411: 119 pgs: 1 peering, 1 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 6.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 23 MiB/s wr, 362 op/s; 575/312546 objects degraded (0.184%); 9.5 MiB/s, 156 objects/s recovering 2023-09-23T20:47:17.394 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:17 smithi018 ceph-mon[184144]: failed to encode map e561 with expected crc 2023-09-23T20:47:17.394 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:17 smithi018 ceph-mon[184144]: osdmap e561: 8 total, 8 up, 8 in 2023-09-23T20:47:17.394 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:17 smithi018 ceph-mon[184144]: failed to encode map e561 with expected crc 2023-09-23T20:47:17.395 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:17 smithi018 ceph-mon[184144]: failed to encode map e561 with expected crc 2023-09-23T20:47:17.395 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:47:17.402 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:17.403 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:47:17.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:17 smithi149 ceph-mon[170394]: failed to encode map e560 with expected crc 2023-09-23T20:47:17.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:17 smithi149 ceph-mon[170394]: pgmap v1411: 119 pgs: 1 peering, 1 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 6.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 23 MiB/s wr, 362 op/s; 575/312546 objects degraded (0.184%); 9.5 MiB/s, 156 objects/s recovering 2023-09-23T20:47:17.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:17 smithi149 ceph-mon[170394]: failed to encode map e561 with expected crc 2023-09-23T20:47:17.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:17 smithi149 ceph-mon[170394]: osdmap e561: 8 total, 8 up, 8 in 2023-09-23T20:47:17.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:17 smithi149 ceph-mon[170394]: failed to encode map e561 with expected crc 2023-09-23T20:47:17.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:17 smithi149 ceph-mon[170394]: failed to encode map e561 with expected crc 2023-09-23T20:47:17.504 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:47:17.551 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:17.609 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:47:17.619 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:17.620 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:47:17.701 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:47:17.720 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:47:17.730 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T20:47:17.768 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:17.820 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:47:17.826 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:17.827 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:47:17.927 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:47:18.043 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:18.130 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:47:18.138 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:18.139 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:47:18.145 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2023-09-23T20:47:18.145 INFO:teuthology.orchestra.run.smithi018.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-09-23T20:47:18.145 INFO:teuthology.orchestra.run.smithi018.stdout:} 2023-09-23T20:47:18.240 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:47:18.323 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:19.011 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:47:19.021 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:47:19.036 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:19.036 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:47:19.137 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:47:19.294 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:19.365 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:47:19.373 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:47:19.384 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:19.384 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:47:19.485 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:47:19.548 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:19.608 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:47:19.612 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:19 smithi149 ceph-mon[170394]: pgmap v1413: 119 pgs: 1 peering, 1 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 6.4 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 17 MiB/s wr, 352 op/s; 302/312420 objects degraded (0.097%); 12 MiB/s, 189 objects/s recovering 2023-09-23T20:47:19.613 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:19.613 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:47:19.648 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:19 smithi018 ceph-mon[184144]: pgmap v1413: 119 pgs: 1 peering, 1 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 6.4 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 17 MiB/s wr, 352 op/s; 302/312420 objects degraded (0.097%); 12 MiB/s, 189 objects/s recovering 2023-09-23T20:47:19.649 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:19 smithi018 ceph-mon[179647]: pgmap v1413: 119 pgs: 1 peering, 1 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 6.4 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 17 MiB/s wr, 352 op/s; 302/312420 objects degraded (0.097%); 12 MiB/s, 189 objects/s recovering 2023-09-23T20:47:19.714 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:47:19.766 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:19.791 INFO:teuthology.orchestra.run.smithi018.stdout:ERROR: (22) Invalid argument 2023-09-23T20:47:19.792 INFO:teuthology.orchestra.run.smithi018.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:47:19.829 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:47:19.839 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:19.840 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:47:19.940 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:47:20.005 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:20.078 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:47:20.087 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:20.087 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:47:20.188 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:47:20.288 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:20.399 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:47:20.411 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:20.411 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:47:20.512 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:47:20.513 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T20:47:20.571 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:20.628 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:47:20.630 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:47:20.634 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:20.634 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:47:20.735 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:47:20.820 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:20.952 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:47:20.957 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:20.958 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:47:21.059 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:47:21.137 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:21.309 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:47:21.313 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:21.313 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:47:21.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:21 smithi018 ceph-mon[184144]: pgmap v1414: 119 pgs: 1 peering, 118 active+clean; 6.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 14 MiB/s wr, 298 op/s; 12 MiB/s, 197 objects/s recovering 2023-09-23T20:47:21.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:21 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:47:21.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:21 smithi018 ceph-mon[184144]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 302/312420 objects degraded (0.097%), 1 pg degraded) 2023-09-23T20:47:21.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:21 smithi018 ceph-mon[179647]: pgmap v1414: 119 pgs: 1 peering, 118 active+clean; 6.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 14 MiB/s wr, 298 op/s; 12 MiB/s, 197 objects/s recovering 2023-09-23T20:47:21.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:21 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:47:21.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:21 smithi018 ceph-mon[179647]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 302/312420 objects degraded (0.097%), 1 pg degraded) 2023-09-23T20:47:21.414 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:47:21.483 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:21.574 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:47:21.581 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:21.582 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:47:21.683 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:47:21.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:21 smithi149 ceph-mon[170394]: pgmap v1414: 119 pgs: 1 peering, 118 active+clean; 6.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 14 MiB/s wr, 298 op/s; 12 MiB/s, 197 objects/s recovering 2023-09-23T20:47:21.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:21 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:47:21.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:21 smithi149 ceph-mon[170394]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 302/312420 objects degraded (0.097%), 1 pg degraded) 2023-09-23T20:47:21.770 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:21.847 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:47:21.855 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:21.855 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:47:21.956 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:47:21.993 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:22.046 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:47:22.049 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:22.050 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:47:22.140 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:47:22.150 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:47:22.220 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:22.355 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:47:22.362 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:22.362 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:47:22.463 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:47:22.545 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:22.631 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:47:22.645 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:22.646 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:47:22.682 INFO:teuthology.orchestra.run.smithi018.stderr:nodeep-scrub is set 2023-09-23T20:47:22.728 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:47:22.747 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:47:22.766 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:47:22.801 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:47:22.802 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:22.841 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:47:22.850 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:47:22.851 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:22 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/895233287' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:47:22.851 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:22 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:47:22.852 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:22 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/895233287' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:47:22.852 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:22 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:47:22.854 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:22.854 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:47:22.890 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:47:22.932 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:47:22.955 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:47:22.963 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:47:22.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:22 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/895233287' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:47:22.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:22 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:47:22.997 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:47:23.000 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:47:23.000 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:47:23.000 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:47:23.000 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:47:23.000 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:47:23.000 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:47:23.000 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:47:23.000 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:47:23.028 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:23.425 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:47:23.434 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:23.434 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:47:23.535 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:47:23.575 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:23.642 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:47:23.651 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:23.651 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:47:23.696 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:23 smithi018 ceph-mon[184144]: pgmap v1415: 119 pgs: 119 active+clean; 6.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 133 B/s rd, 11 MiB/s wr, 621 op/s; 9.6 MiB/s, 154 objects/s recovering 2023-09-23T20:47:23.696 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:23 smithi018 ceph-mon[184144]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:47:23.696 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:23 smithi018 ceph-mon[184144]: failed to encode map e562 with expected crc 2023-09-23T20:47:23.696 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:23 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:47:23.696 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:23 smithi018 ceph-mon[184144]: osdmap e562: 8 total, 8 up, 8 in 2023-09-23T20:47:23.697 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:23 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/4158345266' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:47:23.697 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:23 smithi018 ceph-mon[179647]: pgmap v1415: 119 pgs: 119 active+clean; 6.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 133 B/s rd, 11 MiB/s wr, 621 op/s; 9.6 MiB/s, 154 objects/s recovering 2023-09-23T20:47:23.698 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:23 smithi018 ceph-mon[179647]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:47:23.698 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:23 smithi018 ceph-mon[179647]: failed to encode map e562 with expected crc 2023-09-23T20:47:23.698 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:23 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:47:23.698 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:23 smithi018 ceph-mon[179647]: osdmap e562: 8 total, 8 up, 8 in 2023-09-23T20:47:23.698 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:23 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/4158345266' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:47:23.752 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:47:23.792 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:23.855 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:47:23.863 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:23 smithi149 ceph-mon[170394]: pgmap v1415: 119 pgs: 119 active+clean; 6.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 133 B/s rd, 11 MiB/s wr, 621 op/s; 9.6 MiB/s, 154 objects/s recovering 2023-09-23T20:47:23.863 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:23 smithi149 ceph-mon[170394]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:47:23.863 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:23 smithi149 ceph-mon[170394]: failed to encode map e562 with expected crc 2023-09-23T20:47:23.863 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:23 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:47:23.863 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:23 smithi149 ceph-mon[170394]: osdmap e562: 8 total, 8 up, 8 in 2023-09-23T20:47:23.863 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:23 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/4158345266' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:47:23.864 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:23.864 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:47:23.948 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 7, 3, 2, 6, 5, 4] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T20:47:23.948 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd reweight 0 1 2023-09-23T20:47:23.964 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:47:24.024 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:24.080 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:47:24.087 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:24.087 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:47:24.188 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:47:24.258 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:24.262 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2023-09-23T20:47:24.262 INFO:teuthology.orchestra.run.smithi018.stdout: "ops": [ 2023-09-23T20:47:24.262 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:47:24.262 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_repop(client.46625.0:111633 10.5 e562/531 10:af18947a:::benchmark_data_smithi018_422431_object7137:head v 562'7037, mlcod=562'7034)", 2023-09-23T20:47:24.262 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:47:24.258060+0000", 2023-09-23T20:47:24.262 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.003525556, 2023-09-23T20:47:24.262 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.003548266, 2023-09-23T20:47:24.262 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:47:24.262 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "started", 2023-09-23T20:47:24.263 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:47:24.263 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:47:24.263 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:47:24.263 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:24.258060+0000", 2023-09-23T20:47:24.263 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:47:24.263 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:47:24.263 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:47:24.263 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:47:24.263 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:24.258060+0000", 2023-09-23T20:47:24.263 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:47:24.263 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:47:24.264 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:47:24.264 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:47:24.264 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:24.258059+0000", 2023-09-23T20:47:24.264 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.999999 2023-09-23T20:47:24.264 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:47:24.264 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:47:24.264 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:47:24.264 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:24.258067+0000", 2023-09-23T20:47:24.264 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.9789999999999993e-06 2023-09-23T20:47:24.264 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:47:24.264 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:47:24.264 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:47:24.265 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:24.258082+0000", 2023-09-23T20:47:24.265 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.4659e-05 2023-09-23T20:47:24.265 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:47:24.265 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:47:24.265 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:47:24.265 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:24.258089+0000", 2023-09-23T20:47:24.265 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.2420000000000003e-06 2023-09-23T20:47:24.265 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:47:24.265 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:47:24.265 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:47:24.265 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:24.258171+0000", 2023-09-23T20:47:24.266 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.2057000000000004e-05 2023-09-23T20:47:24.266 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:47:24.266 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:47:24.266 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:47:24.266 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:24.258190+0000", 2023-09-23T20:47:24.266 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.9259e-05 2023-09-23T20:47:24.266 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:47:24.266 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:47:24.266 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:47:24.266 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:47:24.266 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:47:24.266 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_repop(client.46625.0:111634 10.5 e562/531)", 2023-09-23T20:47:24.267 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:47:24.258106+0000", 2023-09-23T20:47:24.267 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.0034791819999999999, 2023-09-23T20:47:24.267 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0035424739999999999, 2023-09-23T20:47:24.267 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:47:24.267 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "queued for pg", 2023-09-23T20:47:24.267 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:47:24.267 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:47:24.267 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:47:24.267 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:24.258106+0000", 2023-09-23T20:47:24.267 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:47:24.267 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:47:24.267 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:47:24.268 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:47:24.268 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:24.258106+0000", 2023-09-23T20:47:24.268 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:47:24.268 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:47:24.268 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:47:24.268 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:47:24.268 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:24.258106+0000", 2023-09-23T20:47:24.268 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:47:24.268 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:47:24.268 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:47:24.268 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:47:24.269 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:24.258109+0000", 2023-09-23T20:47:24.269 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.1650000000000002e-06 2023-09-23T20:47:24.269 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:47:24.269 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:47:24.269 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:47:24.269 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:24.258115+0000", 2023-09-23T20:47:24.269 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.6550000000000001e-06 2023-09-23T20:47:24.269 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:47:24.269 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:47:24.269 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:47:24.269 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:24.258119+0000", 2023-09-23T20:47:24.269 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.2289999999999997e-06 2023-09-23T20:47:24.270 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:47:24.270 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:47:24.270 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:47:24.270 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:47:24.270 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:47:24.270 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_repop(client.46625.0:111637 10.d e562/557)", 2023-09-23T20:47:24.270 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:47:24.259768+0000", 2023-09-23T20:47:24.270 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.001817443, 2023-09-23T20:47:24.270 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0019091640000000001, 2023-09-23T20:47:24.270 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:47:24.270 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "queued for pg", 2023-09-23T20:47:24.270 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:47:24.271 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:47:24.271 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:47:24.271 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:24.259768+0000", 2023-09-23T20:47:24.271 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:47:24.271 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:47:24.271 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:47:24.271 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:47:24.271 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:24.259768+0000", 2023-09-23T20:47:24.271 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:47:24.271 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:47:24.271 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:47:24.272 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:47:24.272 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:24.259767+0000", 2023-09-23T20:47:24.272 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.999999 2023-09-23T20:47:24.272 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:47:24.272 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:47:24.272 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:47:24.272 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:24.259776+0000", 2023-09-23T20:47:24.272 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.9260000000000006e-06 2023-09-23T20:47:24.272 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:47:24.272 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:47:24.272 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:47:24.272 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:24.259795+0000", 2023-09-23T20:47:24.273 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.8800999999999998e-05 2023-09-23T20:47:24.273 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:47:24.273 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:47:24.273 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:47:24.273 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:24.259803+0000", 2023-09-23T20:47:24.273 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.8590000000000005e-06 2023-09-23T20:47:24.273 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:47:24.273 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:47:24.273 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:47:24.273 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:47:24.273 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:47:24.273 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:111618 10.8 10:1ba404f6:::benchmark_data_smithi018_422431_object7122:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e562)", 2023-09-23T20:47:24.274 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:47:24.240849+0000", 2023-09-23T20:47:24.274 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.020737024999999999, 2023-09-23T20:47:24.274 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.020875699000000001, 2023-09-23T20:47:24.274 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:47:24.274 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:47:24.274 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:47:24.274 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:47:24.274 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:47:24.274 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 111618 2023-09-23T20:47:24.274 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:47:24.274 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:47:24.275 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:47:24.275 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:47:24.275 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:24.240849+0000", 2023-09-23T20:47:24.275 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:47:24.275 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:47:24.275 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:47:24.275 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:47:24.275 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:24.240849+0000", 2023-09-23T20:47:24.275 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:47:24.275 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:47:24.275 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:47:24.275 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:47:24.275 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:24.240847+0000", 2023-09-23T20:47:24.276 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:47:24.276 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:47:24.276 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:47:24.276 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:47:24.276 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:24.240853+0000", 2023-09-23T20:47:24.276 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.7490000000000004e-06 2023-09-23T20:47:24.276 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:47:24.276 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:47:24.276 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:47:24.276 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:24.240866+0000", 2023-09-23T20:47:24.276 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.309e-05 2023-09-23T20:47:24.276 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:47:24.277 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:47:24.277 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:47:24.277 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:24.240874+0000", 2023-09-23T20:47:24.277 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.9650000000000008e-06 2023-09-23T20:47:24.277 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:47:24.277 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:47:24.277 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:47:24.277 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:24.247601+0000", 2023-09-23T20:47:24.277 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.006727383 2023-09-23T20:47:24.277 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:47:24.277 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:47:24.277 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:47:24.278 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:24.247751+0000", 2023-09-23T20:47:24.278 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000149662 2023-09-23T20:47:24.278 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:47:24.278 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:47:24.278 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 2,7", 2023-09-23T20:47:24.278 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:24.247892+0000", 2023-09-23T20:47:24.278 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00014059599999999999 2023-09-23T20:47:24.278 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:47:24.278 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:47:24.278 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:47:24.279 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:24.256114+0000", 2023-09-23T20:47:24.279 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0082223409999999993 2023-09-23T20:47:24.279 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:47:24.279 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:47:24.279 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:47:24.279 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:24.256122+0000", 2023-09-23T20:47:24.279 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.524e-06 2023-09-23T20:47:24.279 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:47:24.279 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:47:24.279 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:47:24.279 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:47:24.280 INFO:teuthology.orchestra.run.smithi018.stdout: ], 2023-09-23T20:47:24.280 INFO:teuthology.orchestra.run.smithi018.stdout: "num_ops": 4 2023-09-23T20:47:24.280 INFO:teuthology.orchestra.run.smithi018.stdout:} 2023-09-23T20:47:24.307 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:47:24.314 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:24.315 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:47:24.416 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:47:24.520 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:24.696 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T20:47:24.724 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:47:24.739 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:24.740 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:47:24.841 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:47:24.918 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:47:24.926 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:25.005 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:47:24 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:47:24] "GET /metrics HTTP/1.1" 200 36607 "" "Prometheus/2.43.0" 2023-09-23T20:47:25.014 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:47:25.029 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:25.029 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:47:25.130 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:47:25.162 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T20:47:25.198 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:25.299 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:47:25.304 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:25.304 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:47:25.405 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:47:25.448 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:25.496 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:47:25.506 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:25.507 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:47:25.607 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:47:25.694 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:25.698 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:47:25.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:25 smithi149 ceph-mon[170394]: pgmap v1417: 119 pgs: 119 active+clean; 6.1 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 127 B/s rd, 2.6 MiB/s wr, 613 op/s; 4.5 MiB/s, 71 objects/s recovering 2023-09-23T20:47:25.781 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:47:25.796 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:25.797 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:47:25.862 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:25 smithi018 ceph-mon[184144]: pgmap v1417: 119 pgs: 119 active+clean; 6.1 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 127 B/s rd, 2.6 MiB/s wr, 613 op/s; 4.5 MiB/s, 71 objects/s recovering 2023-09-23T20:47:25.862 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:47:25.862 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:25 smithi018 ceph-mon[179647]: pgmap v1417: 119 pgs: 119 active+clean; 6.1 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 127 B/s rd, 2.6 MiB/s wr, 613 op/s; 4.5 MiB/s, 71 objects/s recovering 2023-09-23T20:47:25.898 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:47:25.957 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:26.029 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:47:26.035 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:26.036 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:47:26.137 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:47:26.215 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:26.288 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:47:26.297 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:26.297 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:47:26.398 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:47:26.456 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:26.515 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:47:26.524 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:26.525 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:47:26.626 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:47:26.689 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:26.979 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:47:26.985 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:26.985 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:47:26.991 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:47:26.991 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:47:27.086 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:47:27.171 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:27.298 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:47:27.305 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:27.306 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:47:27.407 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:47:27.477 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:27.551 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:47:27.557 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:27.557 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:47:27.658 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:47:27.672 INFO:teuthology.orchestra.run.smithi018.stderr:reweighted osd.0 to 1 (10000) 2023-09-23T20:47:27.693 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:27.750 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:47:27.751 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:27.752 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:47:27.752 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:27 smithi149 ceph-mon[170394]: pgmap v1418: 119 pgs: 119 active+clean; 6.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 105 B/s rd, 894 KiB/s wr, 736 op/s; 2.0 MiB/s, 31 objects/s recovering 2023-09-23T20:47:27.752 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:27 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/506545366' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T20:47:27.752 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:27 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T20:47:27.853 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:47:27.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:27 smithi018 ceph-mon[179647]: pgmap v1418: 119 pgs: 119 active+clean; 6.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 105 B/s rd, 894 KiB/s wr, 736 op/s; 2.0 MiB/s, 31 objects/s recovering 2023-09-23T20:47:27.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:27 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/506545366' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T20:47:27.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:27 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T20:47:27.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:27 smithi018 ceph-mon[184144]: pgmap v1418: 119 pgs: 119 active+clean; 6.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 105 B/s rd, 894 KiB/s wr, 736 op/s; 2.0 MiB/s, 31 objects/s recovering 2023-09-23T20:47:27.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:27 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/506545366' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T20:47:27.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:27 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T20:47:27.924 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:27.986 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:47:28.002 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:28.002 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:47:28.103 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:47:28.160 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:28.228 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:47:28.234 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:28.235 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:47:28.335 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:47:28.407 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:28.458 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd reweight 1 1 2023-09-23T20:47:28.528 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:47:28.537 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:28.538 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:47:28.639 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:47:28.694 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:28 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T20:47:28.694 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:28 smithi018 ceph-mon[179647]: osdmap e563: 8 total, 8 up, 8 in 2023-09-23T20:47:28.694 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:28 smithi018 ceph-mon[179647]: failed to encode map e563 with expected crc 2023-09-23T20:47:28.694 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:28 smithi018 ceph-mon[179647]: failed to encode map e563 with expected crc 2023-09-23T20:47:28.694 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:28 smithi018 ceph-mon[179647]: failed to encode map e563 with expected crc 2023-09-23T20:47:28.695 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:28 smithi018 ceph-mon[179647]: failed to encode map e563 with expected crc 2023-09-23T20:47:28.768 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:28.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:28 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T20:47:28.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:28 smithi149 ceph-mon[170394]: osdmap e563: 8 total, 8 up, 8 in 2023-09-23T20:47:28.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:28 smithi149 ceph-mon[170394]: failed to encode map e563 with expected crc 2023-09-23T20:47:28.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:28 smithi149 ceph-mon[170394]: failed to encode map e563 with expected crc 2023-09-23T20:47:28.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:28 smithi149 ceph-mon[170394]: failed to encode map e563 with expected crc 2023-09-23T20:47:28.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:28 smithi149 ceph-mon[170394]: failed to encode map e563 with expected crc 2023-09-23T20:47:28.979 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:47:28.988 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:28.989 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:47:28.990 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:28 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T20:47:28.990 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:28 smithi018 ceph-mon[184144]: osdmap e563: 8 total, 8 up, 8 in 2023-09-23T20:47:28.990 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:28 smithi018 ceph-mon[184144]: failed to encode map e563 with expected crc 2023-09-23T20:47:28.990 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:28 smithi018 ceph-mon[184144]: failed to encode map e563 with expected crc 2023-09-23T20:47:28.990 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:28 smithi018 ceph-mon[184144]: failed to encode map e563 with expected crc 2023-09-23T20:47:28.990 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:28 smithi018 ceph-mon[184144]: failed to encode map e563 with expected crc 2023-09-23T20:47:29.090 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:47:29.136 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2023-09-23T20:47:29.136 INFO:teuthology.orchestra.run.smithi018.stdout: "ops": [], 2023-09-23T20:47:29.136 INFO:teuthology.orchestra.run.smithi018.stdout: "complaint_time": 30, 2023-09-23T20:47:29.136 INFO:teuthology.orchestra.run.smithi018.stdout: "num_blocked_ops": 0 2023-09-23T20:47:29.136 INFO:teuthology.orchestra.run.smithi018.stdout:} 2023-09-23T20:47:29.168 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:29.375 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:47:29.379 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:29.380 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:47:29.481 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:47:29.518 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:29.581 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:47:29.584 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:29.585 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:47:29.686 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:47:29.715 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:29.791 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:47:29.793 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:29 smithi149 ceph-mon[170394]: pgmap v1420: 119 pgs: 119 active+clean; 5.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 127 B/s rd, 88 KiB/s wr, 1.02k op/s 2023-09-23T20:47:29.793 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:29 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:47:29.793 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:29 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:47:29.794 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:29.794 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:47:29.837 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T20:47:29.894 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:47:29.905 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:29 smithi018 ceph-mon[184144]: pgmap v1420: 119 pgs: 119 active+clean; 5.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 127 B/s rd, 88 KiB/s wr, 1.02k op/s 2023-09-23T20:47:29.905 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:29 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:47:29.905 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:29 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:47:29.905 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:29 smithi018 ceph-mon[179647]: pgmap v1420: 119 pgs: 119 active+clean; 5.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 127 B/s rd, 88 KiB/s wr, 1.02k op/s 2023-09-23T20:47:29.906 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:29 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:47:29.906 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:29 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:47:29.958 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:29.966 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:47:29.970 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:47:30.019 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:47:30.027 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:30.027 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:47:30.032 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:47:30.079 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:47:30.109 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:47:30.128 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:47:30.145 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:47:30.181 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:30.191 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:47:30.227 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:47:30.228 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:47:30.234 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:30.234 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:47:30.266 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:47:30.276 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:47:30.276 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:47:30.276 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:47:30.276 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:47:30.276 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:47:30.276 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:47:30.276 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:47:30.276 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:47:30.335 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:47:30.428 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:30.753 INFO:teuthology.orchestra.run.smithi018.stderr:noscrub is unset 2023-09-23T20:47:30.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:30 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/151060679' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:47:30.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:30 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:47:30.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:30 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2990280176' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:47:30.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:30 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:47:30.894 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:30 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/151060679' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:47:30.894 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:30 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:47:30.894 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:30 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2990280176' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:47:30.894 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:30 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:47:30.969 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:30 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/151060679' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:47:30.969 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:30 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:47:30.969 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:30 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2990280176' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:47:30.969 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:30 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:47:31.057 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:47:31.063 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:31.064 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:47:31.164 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:47:31.230 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:31.346 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:47:31.353 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:31.353 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:47:31.454 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:47:31.486 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:31.545 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:47:31.556 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:31.557 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:47:31.645 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:47:31.657 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:47:31.719 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:31.780 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:47:31.783 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:31 smithi149 ceph-mon[170394]: pgmap v1421: 119 pgs: 119 active+clean; 5.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 685 op/s 2023-09-23T20:47:31.783 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:31 smithi149 ceph-mon[170394]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:47:31.783 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:31 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:47:31.783 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:31 smithi149 ceph-mon[170394]: osdmap e564: 8 total, 8 up, 8 in 2023-09-23T20:47:31.783 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:31 smithi149 ceph-mon[170394]: failed to encode map e564 with expected crc 2023-09-23T20:47:31.784 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:31.784 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:47:31.885 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:47:31.922 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:31.979 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:31 smithi018 ceph-mon[179647]: pgmap v1421: 119 pgs: 119 active+clean; 5.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 685 op/s 2023-09-23T20:47:31.979 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:31 smithi018 ceph-mon[179647]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:47:31.980 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:31 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:47:31.980 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:31 smithi018 ceph-mon[179647]: osdmap e564: 8 total, 8 up, 8 in 2023-09-23T20:47:31.980 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:31 smithi018 ceph-mon[179647]: failed to encode map e564 with expected crc 2023-09-23T20:47:31.980 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:31 smithi018 ceph-mon[184144]: pgmap v1421: 119 pgs: 119 active+clean; 5.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 685 op/s 2023-09-23T20:47:31.980 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:31 smithi018 ceph-mon[184144]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:47:31.980 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:31 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:47:31.980 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:31 smithi018 ceph-mon[184144]: osdmap e564: 8 total, 8 up, 8 in 2023-09-23T20:47:31.980 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:31 smithi018 ceph-mon[184144]: failed to encode map e564 with expected crc 2023-09-23T20:47:31.983 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:47:31.991 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:31.992 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:47:32.092 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:47:32.162 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:32.458 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:47:32.468 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:32.468 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:47:32.569 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:47:32.653 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:32.759 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:47:32.768 INFO:teuthology.orchestra.run.smithi018.stderr:reweighted osd.1 to 1 (10000) 2023-09-23T20:47:32.778 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:47:32.784 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:32.785 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:47:32.886 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:47:32.895 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:32 smithi018 ceph-mon[179647]: 1.1a scrub starts 2023-09-23T20:47:32.895 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:32 smithi018 ceph-mon[179647]: 1.1a scrub ok 2023-09-23T20:47:32.895 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:32 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1438135887' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T20:47:32.896 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:32 smithi018 ceph-mon[184144]: 1.1a scrub starts 2023-09-23T20:47:32.896 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:32 smithi018 ceph-mon[184144]: 1.1a scrub ok 2023-09-23T20:47:32.896 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:32 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1438135887' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T20:47:32.973 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:33.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:32 smithi149 ceph-mon[170394]: 1.1a scrub starts 2023-09-23T20:47:33.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:32 smithi149 ceph-mon[170394]: 1.1a scrub ok 2023-09-23T20:47:33.218 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:32 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1438135887' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T20:47:33.284 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:47:33.300 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:33.301 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:47:33.402 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:47:33.402 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T20:47:33.419 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd reweight 7 1 2023-09-23T20:47:33.452 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:33.534 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:47:33.538 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:33.538 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:47:33.639 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:47:33.671 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:33.708 INFO:teuthology.orchestra.run.smithi018.stdout:ERROR: (22) Invalid argument 2023-09-23T20:47:33.708 INFO:teuthology.orchestra.run.smithi018.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:47:33.722 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:47:33.728 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:33.728 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:47:33.820 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:47:33.829 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:47:33.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:33 smithi018 ceph-mon[184144]: pgmap v1423: 119 pgs: 119 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 938 op/s 2023-09-23T20:47:33.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:33 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1438135887' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T20:47:33.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:33 smithi018 ceph-mon[184144]: osdmap e565: 8 total, 8 up, 8 in 2023-09-23T20:47:33.893 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:33 smithi018 ceph-mon[184144]: failed to encode map e565 with expected crc 2023-09-23T20:47:33.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:33 smithi018 ceph-mon[179647]: pgmap v1423: 119 pgs: 119 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 938 op/s 2023-09-23T20:47:33.895 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:33 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1438135887' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T20:47:33.895 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:33 smithi018 ceph-mon[179647]: osdmap e565: 8 total, 8 up, 8 in 2023-09-23T20:47:33.895 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:33 smithi018 ceph-mon[179647]: failed to encode map e565 with expected crc 2023-09-23T20:47:33.942 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:34.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:33 smithi149 ceph-mon[170394]: pgmap v1423: 119 pgs: 119 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 938 op/s 2023-09-23T20:47:34.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:33 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1438135887' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T20:47:34.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:33 smithi149 ceph-mon[170394]: osdmap e565: 8 total, 8 up, 8 in 2023-09-23T20:47:34.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:33 smithi149 ceph-mon[170394]: failed to encode map e565 with expected crc 2023-09-23T20:47:34.251 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:47:34.254 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:34.255 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:47:34.308 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T20:47:34.356 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:47:34.407 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:34.469 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:47:34.473 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:34.473 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:47:34.521 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:47:34.574 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:47:34.615 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:34.690 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:47:34.697 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:34.697 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:47:34.778 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:34 smithi018 ceph-mon[179647]: 1.1b scrub starts 2023-09-23T20:47:34.778 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:34 smithi018 ceph-mon[179647]: 1.1b scrub ok 2023-09-23T20:47:34.778 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:34 smithi018 ceph-mon[184144]: 1.1b scrub starts 2023-09-23T20:47:34.778 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:34 smithi018 ceph-mon[184144]: 1.1b scrub ok 2023-09-23T20:47:34.798 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:47:34.880 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:34.948 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:47:34.956 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:47:34.959 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:34.959 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:47:34.969 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:34 smithi149 ceph-mon[170394]: 1.1b scrub starts 2023-09-23T20:47:34.969 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:34 smithi149 ceph-mon[170394]: 1.1b scrub ok 2023-09-23T20:47:35.060 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:47:35.118 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:47:34 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:47:34] "GET /metrics HTTP/1.1" 200 36586 "" "Prometheus/2.43.0" 2023-09-23T20:47:35.127 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:35.244 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:47:35.253 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:35.253 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:47:35.354 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:47:35.445 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:35.746 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:47:35.753 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:35.753 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:47:35.854 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:47:35.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:35 smithi018 ceph-mon[184144]: pgmap v1425: 119 pgs: 119 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 793 op/s 2023-09-23T20:47:35.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:35 smithi018 ceph-mon[184144]: 1.5 scrub starts 2023-09-23T20:47:35.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:35 smithi018 ceph-mon[184144]: 1.5 scrub ok 2023-09-23T20:47:35.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:35 smithi018 ceph-mon[179647]: pgmap v1425: 119 pgs: 119 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 793 op/s 2023-09-23T20:47:35.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:35 smithi018 ceph-mon[179647]: 1.5 scrub starts 2023-09-23T20:47:35.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:35 smithi018 ceph-mon[179647]: 1.5 scrub ok 2023-09-23T20:47:35.896 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:35.958 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:47:35.970 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:35.971 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:47:36.072 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:47:36.126 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:36.218 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:35 smithi149 ceph-mon[170394]: pgmap v1425: 119 pgs: 119 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 793 op/s 2023-09-23T20:47:36.218 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:35 smithi149 ceph-mon[170394]: 1.5 scrub starts 2023-09-23T20:47:36.218 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:35 smithi149 ceph-mon[170394]: 1.5 scrub ok 2023-09-23T20:47:36.271 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:47:36.278 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:36.279 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:47:36.319 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:47:36.380 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:47:36.489 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:36.575 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:47:36.582 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:36.582 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:47:36.683 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:47:36.755 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:36.807 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:47:36.815 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:36.815 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:47:36.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:36 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:47:36.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:36 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:47:36.916 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:47:36.967 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:37.008 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:47:37.046 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:47:37.051 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:47:37.063 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:37.064 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:47:37.064 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:36 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:47:37.084 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:47:37.143 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:47:37.164 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:47:37.185 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:47:37.222 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:47:37.259 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:37.275 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:47:37.337 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:47:37.338 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:47:37.349 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:37.350 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:47:37.370 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:47:37.371 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:47:37.371 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:47:37.371 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:47:37.371 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:47:37.371 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:47:37.371 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:47:37.371 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:47:37.451 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:47:37.585 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:37.694 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:47:37.706 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:37.707 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:47:37.808 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:47:37.873 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:37.880 INFO:teuthology.orchestra.run.smithi018.stderr:nodeep-scrub is unset 2023-09-23T20:47:37.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:37 smithi018 ceph-mon[184144]: pgmap v1426: 119 pgs: 119 active+clean; 5.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 853 op/s 2023-09-23T20:47:37.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:37 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1583686114' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:47:37.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:37 smithi018 ceph-mon[179647]: pgmap v1426: 119 pgs: 119 active+clean; 5.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 853 op/s 2023-09-23T20:47:37.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:37 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1583686114' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:47:38.054 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:47:38.066 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:38.066 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:37 smithi149 ceph-mon[170394]: pgmap v1426: 119 pgs: 119 active+clean; 5.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 853 op/s 2023-09-23T20:47:38.066 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:37 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1583686114' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:47:38.067 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:47:38.168 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:47:38.243 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:38.350 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:47:38.365 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:38.365 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:47:38.466 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:47:38.538 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:38.593 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:47:38.598 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:38.599 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:47:38.700 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:47:38.779 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:38.869 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:47:38.882 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:38.883 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:47:38.884 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:38 smithi018 ceph-mon[179647]: 1.e scrub starts 2023-09-23T20:47:38.884 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:38 smithi018 ceph-mon[179647]: 1.e scrub ok 2023-09-23T20:47:38.885 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:38 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/4071399398' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:47:38.885 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:38 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:47:38.885 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:38 smithi018 ceph-mon[179647]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:47:38.885 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:38 smithi018 ceph-mon[179647]: Cluster is now healthy 2023-09-23T20:47:38.885 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:38 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:47:38.885 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:38 smithi018 ceph-mon[179647]: osdmap e566: 8 total, 8 up, 8 in 2023-09-23T20:47:38.885 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:38 smithi018 ceph-mon[179647]: failed to encode map e566 with expected crc 2023-09-23T20:47:38.885 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:38 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3560648871' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T20:47:38.885 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:38 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T20:47:38.885 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:38 smithi018 ceph-mon[184144]: 1.e scrub starts 2023-09-23T20:47:38.886 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:38 smithi018 ceph-mon[184144]: 1.e scrub ok 2023-09-23T20:47:38.886 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:38 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/4071399398' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:47:38.886 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:38 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:47:38.886 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:38 smithi018 ceph-mon[184144]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:47:38.886 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:38 smithi018 ceph-mon[184144]: Cluster is now healthy 2023-09-23T20:47:38.886 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:38 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:47:38.886 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:38 smithi018 ceph-mon[184144]: osdmap e566: 8 total, 8 up, 8 in 2023-09-23T20:47:38.886 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:38 smithi018 ceph-mon[184144]: failed to encode map e566 with expected crc 2023-09-23T20:47:38.886 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:38 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3560648871' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T20:47:38.886 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:38 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T20:47:38.897 INFO:teuthology.orchestra.run.smithi018.stderr:reweighted osd.7 to 1 (10000) 2023-09-23T20:47:38.984 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:47:39.062 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:39.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:38 smithi149 ceph-mon[170394]: 1.e scrub starts 2023-09-23T20:47:39.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:38 smithi149 ceph-mon[170394]: 1.e scrub ok 2023-09-23T20:47:39.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:38 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/4071399398' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:47:39.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:38 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:47:39.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:38 smithi149 ceph-mon[170394]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:47:39.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:38 smithi149 ceph-mon[170394]: Cluster is now healthy 2023-09-23T20:47:39.218 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:38 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:47:39.218 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:38 smithi149 ceph-mon[170394]: osdmap e566: 8 total, 8 up, 8 in 2023-09-23T20:47:39.218 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:38 smithi149 ceph-mon[170394]: failed to encode map e566 with expected crc 2023-09-23T20:47:39.218 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:38 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3560648871' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T20:47:39.218 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:38 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T20:47:39.239 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:47:39.248 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:39.248 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:47:39.349 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:47:39.367 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2023-09-23T20:47:39.367 INFO:teuthology.orchestra.run.smithi018.stdout: "ops": [], 2023-09-23T20:47:39.367 INFO:teuthology.orchestra.run.smithi018.stdout: "num_ops": 0 2023-09-23T20:47:39.367 INFO:teuthology.orchestra.run.smithi018.stdout:} 2023-09-23T20:47:39.412 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:39.466 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:47:39.476 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:39.476 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:47:39.577 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:47:39.656 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd reweight 3 1 2023-09-23T20:47:39.689 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:39.860 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:39 smithi018 ceph-mon[179647]: pgmap v1428: 119 pgs: 119 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 833 op/s 2023-09-23T20:47:39.860 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:39 smithi018 ceph-mon[179647]: failed to encode map e567 with expected crc 2023-09-23T20:47:39.860 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:39 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T20:47:39.860 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:39 smithi018 ceph-mon[179647]: osdmap e567: 8 total, 8 up, 8 in 2023-09-23T20:47:39.896 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T20:47:39.896 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:47:39.906 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:39.906 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:47:40.007 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:47:40.009 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:47:40.125 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:39 smithi018 ceph-mon[184144]: pgmap v1428: 119 pgs: 119 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 833 op/s 2023-09-23T20:47:40.125 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:39 smithi018 ceph-mon[184144]: failed to encode map e567 with expected crc 2023-09-23T20:47:40.125 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:39 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T20:47:40.125 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:39 smithi018 ceph-mon[184144]: osdmap e567: 8 total, 8 up, 8 in 2023-09-23T20:47:40.131 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:40.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:39 smithi149 ceph-mon[170394]: pgmap v1428: 119 pgs: 119 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 833 op/s 2023-09-23T20:47:40.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:39 smithi149 ceph-mon[170394]: failed to encode map e567 with expected crc 2023-09-23T20:47:40.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:39 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T20:47:40.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:39 smithi149 ceph-mon[170394]: osdmap e567: 8 total, 8 up, 8 in 2023-09-23T20:47:40.231 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:47:40.244 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:40.244 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:47:40.345 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:47:40.386 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:40.457 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:47:40.462 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:40.463 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:47:40.515 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:47:40.564 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:47:40.608 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:40.676 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:47:40.681 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:40.681 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:47:40.782 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:47:40.810 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T20:47:40.826 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:40.894 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:47:40.900 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:40.901 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:47:41.002 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:47:41.113 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:41.125 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:47:41.125 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:47:41.220 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:47:41.225 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:41.226 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:47:41.327 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:47:41.387 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:41.656 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:47:41.679 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:41.681 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:47:41.782 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:47:41.885 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:41.944 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:41 smithi018 ceph-mon[184144]: pgmap v1430: 119 pgs: 119 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 694 op/s 2023-09-23T20:47:41.944 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:41 smithi018 ceph-mon[179647]: pgmap v1430: 119 pgs: 119 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 694 op/s 2023-09-23T20:47:42.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:41 smithi149 ceph-mon[170394]: pgmap v1430: 119 pgs: 119 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 694 op/s 2023-09-23T20:47:42.924 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:47:42.930 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:42.931 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:47:43.032 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:47:43.085 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:43.135 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:47:43.138 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:43.138 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:47:43.239 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:47:43.302 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:43.355 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:47:43.359 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:43.360 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:47:43.461 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:47:43.506 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:43.579 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:47:43.587 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:43.587 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:47:43.631 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:43 smithi018 ceph-mon[184144]: pgmap v1431: 119 pgs: 119 active+clean; 5.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 886 op/s 2023-09-23T20:47:43.631 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:43 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:47:43.632 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:43 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:47:43.632 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:43 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/4237819193' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T20:47:43.632 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:43 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T20:47:43.632 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:43 smithi018 ceph-mon[179647]: pgmap v1431: 119 pgs: 119 active+clean; 5.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 886 op/s 2023-09-23T20:47:43.632 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:43 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:47:43.632 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:43 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:47:43.632 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:43 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/4237819193' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T20:47:43.633 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:43 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T20:47:43.688 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:47:43.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:43 smithi149 ceph-mon[170394]: pgmap v1431: 119 pgs: 119 active+clean; 5.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 886 op/s 2023-09-23T20:47:43.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:43 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:47:43.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:43 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:47:43.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:43 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/4237819193' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T20:47:43.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:43 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T20:47:43.752 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:43.921 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:47:43.937 INFO:teuthology.orchestra.run.smithi018.stderr:reweighted osd.3 to 1 (10000) 2023-09-23T20:47:43.938 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:47:43.944 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:43.944 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:47:44.045 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:47:44.089 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:44.139 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:47:44.145 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:44.145 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:47:44.246 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:47:44.299 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:44.375 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:47:44.381 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:44.381 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:47:44.482 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:47:44.549 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:44.615 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:47:44.617 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:44.618 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:47:44.638 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd reweight 2 1 2023-09-23T20:47:44.718 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:47:44.736 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2023-09-23T20:47:44.736 INFO:teuthology.orchestra.run.smithi018.stdout: "ops": [], 2023-09-23T20:47:44.736 INFO:teuthology.orchestra.run.smithi018.stdout: "complaint_time": 30, 2023-09-23T20:47:44.736 INFO:teuthology.orchestra.run.smithi018.stdout: "num_blocked_ops": 0 2023-09-23T20:47:44.736 INFO:teuthology.orchestra.run.smithi018.stdout:} 2023-09-23T20:47:44.836 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:44.943 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:47:44.951 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:44.952 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:47:45.053 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:47:45.098 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:45.175 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:47:45.181 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:44 smithi149 ceph-mon[170394]: failed to encode map e568 with expected crc 2023-09-23T20:47:45.181 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:44 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T20:47:45.181 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:44 smithi149 ceph-mon[170394]: osdmap e568: 8 total, 8 up, 8 in 2023-09-23T20:47:45.181 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:44 smithi149 ceph-mon[170394]: failed to encode map e568 with expected crc 2023-09-23T20:47:45.181 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:44 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:47:45.182 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:44 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:47:45.182 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:45.183 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:47:45.284 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:47:45.297 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:44 smithi018 ceph-mon[179647]: failed to encode map e568 with expected crc 2023-09-23T20:47:45.297 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:44 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T20:47:45.297 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:44 smithi018 ceph-mon[179647]: osdmap e568: 8 total, 8 up, 8 in 2023-09-23T20:47:45.297 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:44 smithi018 ceph-mon[179647]: failed to encode map e568 with expected crc 2023-09-23T20:47:45.297 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:44 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:47:45.298 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:44 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:47:45.298 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:44 smithi018 ceph-mon[184144]: failed to encode map e568 with expected crc 2023-09-23T20:47:45.298 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:44 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T20:47:45.298 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:44 smithi018 ceph-mon[184144]: osdmap e568: 8 total, 8 up, 8 in 2023-09-23T20:47:45.298 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:44 smithi018 ceph-mon[184144]: failed to encode map e568 with expected crc 2023-09-23T20:47:45.298 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:44 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:47:45.298 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:44 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:47:45.302 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:47:44 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:47:44] "GET /metrics HTTP/1.1" 200 36575 "" "Prometheus/2.43.0" 2023-09-23T20:47:45.330 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:45.403 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:47:45.411 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:45.412 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:47:45.512 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:47:45.605 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:45.787 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:47:45.790 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T20:47:45.807 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:45.808 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:47:45.909 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:47:45.942 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:45.995 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:47:45.998 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:45.999 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:47:46.100 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:47:46.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:45 smithi018 ceph-mon[184144]: pgmap v1433: 119 pgs: 119 active+clean; 5.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 760 op/s 2023-09-23T20:47:46.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:45 smithi018 ceph-mon[179647]: pgmap v1433: 119 pgs: 119 active+clean; 5.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 760 op/s 2023-09-23T20:47:46.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:45 smithi149 ceph-mon[170394]: pgmap v1433: 119 pgs: 119 active+clean; 5.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 760 op/s 2023-09-23T20:47:46.224 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:46.227 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:47:46.258 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:47:46.295 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:47:46.295 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:47:46.304 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:46.304 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:47:46.347 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:47:46.405 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:47:46.415 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:47:46.459 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:47:46.497 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:46.520 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:47:46.580 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:47:46.590 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:47:46.598 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:46.599 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:47:46.602 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:47:46.602 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:47:46.602 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:47:46.602 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:47:46.602 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:47:46.602 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:47:46.602 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:47:46.602 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:47:46.699 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:47:46.805 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:47.009 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:47:47.022 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:47.023 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:47:47.026 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:46 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/4045322303' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:47:47.026 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:46 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:47:47.026 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:46 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:47:47.026 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:46 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/4045322303' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:47:47.026 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:46 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:47:47.027 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:46 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:47:47.029 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:47:47.124 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:47:47.169 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:47.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:46 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/4045322303' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:47:47.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:46 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:47:47.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:46 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:47:47.228 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:47:47.233 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:47.233 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:47:47.242 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:47:47.334 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:47:47.462 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:47.841 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:47:47.845 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:47.845 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:47:47.946 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:47:47.974 INFO:teuthology.orchestra.run.smithi018.stderr:noscrub is set 2023-09-23T20:47:48.022 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:48.144 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:47 smithi018 ceph-mon[184144]: pgmap v1434: 119 pgs: 119 active+clean; 5.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 794 op/s 2023-09-23T20:47:48.144 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:47 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/4101199682' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:47:48.144 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:47 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:47:48.144 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:47 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:47:48.144 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:47 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:47:48.145 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:47 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:47:48.145 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:47 smithi018 ceph-mon[179647]: pgmap v1434: 119 pgs: 119 active+clean; 5.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 794 op/s 2023-09-23T20:47:48.145 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:47 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/4101199682' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:47:48.145 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:47 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:47:48.145 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:47 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:47:48.145 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:47 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:47:48.145 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:47 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:47:48.220 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:47 smithi149 ceph-mon[170394]: pgmap v1434: 119 pgs: 119 active+clean; 5.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 794 op/s 2023-09-23T20:47:48.220 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:47 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/4101199682' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:47:48.220 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:47 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:47:48.220 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:47 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:47:48.220 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:47 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:47:48.220 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:47 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:47:48.538 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:47:48.544 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:48.545 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:47:48.646 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:47:48.704 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:48.777 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:47:48.784 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:48.785 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:47:48.886 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:47:48.997 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:49.114 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:47:49.122 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:48 smithi018 ceph-mon[179647]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:47:49.122 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:48 smithi018 ceph-mon[179647]: failed to encode map e569 with expected crc 2023-09-23T20:47:49.122 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:48 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:47:49.123 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:48 smithi018 ceph-mon[179647]: osdmap e569: 8 total, 8 up, 8 in 2023-09-23T20:47:49.123 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:48 smithi018 ceph-mon[184144]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:47:49.123 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:48 smithi018 ceph-mon[184144]: failed to encode map e569 with expected crc 2023-09-23T20:47:49.123 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:48 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:47:49.123 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:48 smithi018 ceph-mon[184144]: osdmap e569: 8 total, 8 up, 8 in 2023-09-23T20:47:49.124 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:49.125 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:47:49.143 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:47:49.225 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:47:49.357 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:49.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:48 smithi149 ceph-mon[170394]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:47:49.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:48 smithi149 ceph-mon[170394]: failed to encode map e569 with expected crc 2023-09-23T20:47:49.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:48 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:47:49.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:48 smithi149 ceph-mon[170394]: osdmap e569: 8 total, 8 up, 8 in 2023-09-23T20:47:49.556 INFO:teuthology.orchestra.run.smithi018.stdout:ERROR: (22) Invalid argument 2023-09-23T20:47:49.557 INFO:teuthology.orchestra.run.smithi018.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:47:49.567 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:47:49.575 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:49.576 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:47:49.677 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:47:49.750 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:50.007 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:47:50.013 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:50.013 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:47:50.098 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T20:47:50.114 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:47:50.189 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:50.237 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:47:50.244 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:49 smithi149 ceph-mon[170394]: pgmap v1436: 119 pgs: 119 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 940 op/s 2023-09-23T20:47:50.245 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:50.245 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:47:50.329 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:47:50.346 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:47:50.386 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:50.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:49 smithi018 ceph-mon[184144]: pgmap v1436: 119 pgs: 119 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 940 op/s 2023-09-23T20:47:50.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:49 smithi018 ceph-mon[179647]: pgmap v1436: 119 pgs: 119 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 940 op/s 2023-09-23T20:47:50.444 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:47:50.453 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:50.454 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:47:50.555 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:47:50.629 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:50.651 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T20:47:50.690 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:47:50.694 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:50.695 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:47:50.795 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:47:50.834 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:50.889 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:47:50.893 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:50.894 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:47:50.994 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:47:51.027 INFO:teuthology.orchestra.run.smithi018.stderr:reweighted osd.2 to 1 (10000) 2023-09-23T20:47:51.028 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:51.105 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:47:51.427 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:51 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1431777447' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T20:47:51.428 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:51 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T20:47:51.428 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:51 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1431777447' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T20:47:51.428 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:51 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T20:47:51.429 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:51.429 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:47:51.431 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:51 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1431777447' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T20:47:51.431 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:51 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T20:47:51.495 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd reweight 6 1 2023-09-23T20:47:51.533 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:47:51.576 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:47:51.831 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:47:51.831 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:51.832 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:51.832 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:47:51.832 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:47:51.933 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:47:52.222 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:52.279 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:47:52.294 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:52 smithi018 ceph-mon[179647]: pgmap v1437: 119 pgs: 119 active+clean; 4.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 615 op/s 2023-09-23T20:47:52.294 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:52 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T20:47:52.294 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:52 smithi018 ceph-mon[179647]: osdmap e570: 8 total, 8 up, 8 in 2023-09-23T20:47:52.294 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:52 smithi018 ceph-mon[179647]: failed to encode map e570 with expected crc 2023-09-23T20:47:52.294 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:52 smithi018 ceph-mon[179647]: failed to encode map e570 with expected crc 2023-09-23T20:47:52.294 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:52 smithi149 ceph-mon[170394]: pgmap v1437: 119 pgs: 119 active+clean; 4.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 615 op/s 2023-09-23T20:47:52.295 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:52 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T20:47:52.295 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:52 smithi149 ceph-mon[170394]: osdmap e570: 8 total, 8 up, 8 in 2023-09-23T20:47:52.295 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:52 smithi149 ceph-mon[170394]: failed to encode map e570 with expected crc 2023-09-23T20:47:52.295 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:52 smithi149 ceph-mon[170394]: failed to encode map e570 with expected crc 2023-09-23T20:47:52.295 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:52 smithi018 ceph-mon[184144]: pgmap v1437: 119 pgs: 119 active+clean; 4.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 615 op/s 2023-09-23T20:47:52.295 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:52 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T20:47:52.295 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:52 smithi018 ceph-mon[184144]: osdmap e570: 8 total, 8 up, 8 in 2023-09-23T20:47:52.295 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:52 smithi018 ceph-mon[184144]: failed to encode map e570 with expected crc 2023-09-23T20:47:52.296 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:52 smithi018 ceph-mon[184144]: failed to encode map e570 with expected crc 2023-09-23T20:47:52.296 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:52.297 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:47:52.397 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:47:52.517 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:52.522 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:47:52.563 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:47:52.575 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:52.575 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:47:52.578 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:47:52.634 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:47:52.672 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:47:52.676 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:47:52.722 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:47:52.723 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:52.752 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:47:52.775 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:47:52.780 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:52.780 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:47:52.804 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:47:52.858 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:47:52.867 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:47:52.867 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:47:52.867 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:47:52.867 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:47:52.867 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:47:52.868 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:47:52.868 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:47:52.868 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:47:52.881 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:47:52.940 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:53.011 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:47:53.019 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:53.019 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:47:53.120 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:47:53.158 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:53 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/172550773' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:47:53.158 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:53 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/172550773' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:47:53.165 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:53.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:53 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/172550773' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:47:53.339 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:47:53.349 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:53.350 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:47:53.450 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:47:53.514 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:53.668 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:47:53.674 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:53.674 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:47:53.775 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:47:53.862 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:53.933 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:47:53.942 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:53.943 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:47:53.950 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:47:54.044 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:47:54.045 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:54 smithi018 ceph-mon[179647]: pgmap v1439: 119 pgs: 119 active+clean; 4.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 856 op/s 2023-09-23T20:47:54.135 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:54.237 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:47:54.243 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:54.244 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:47:54.345 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:47:54.391 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:54 smithi149 ceph-mon[170394]: pgmap v1439: 119 pgs: 119 active+clean; 4.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 856 op/s 2023-09-23T20:47:54.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:54 smithi018 ceph-mon[184144]: pgmap v1439: 119 pgs: 119 active+clean; 4.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 856 op/s 2023-09-23T20:47:54.392 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:54.457 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:47:54.461 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:54.462 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:47:54.562 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:47:54.593 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2023-09-23T20:47:54.593 INFO:teuthology.orchestra.run.smithi018.stdout: "ops": [ 2023-09-23T20:47:54.593 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:47:54.593 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_repop(client.46625.0:136181 10.7 e570/531 10:e570c331:::benchmark_data_smithi018_422431_object31685:head v 570'8469, mlcod=570'8467)", 2023-09-23T20:47:54.594 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:47:54.587466+0000", 2023-09-23T20:47:54.594 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.0045634489999999998, 2023-09-23T20:47:54.594 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0045932029999999997, 2023-09-23T20:47:54.594 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:47:54.594 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "started", 2023-09-23T20:47:54.594 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:47:54.594 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:47:54.594 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:47:54.594 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:54.587466+0000", 2023-09-23T20:47:54.594 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:47:54.594 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:47:54.595 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:47:54.595 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:47:54.595 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:54.587466+0000", 2023-09-23T20:47:54.595 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:47:54.595 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:47:54.595 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:47:54.595 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:47:54.595 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:54.587465+0000", 2023-09-23T20:47:54.595 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.999999 2023-09-23T20:47:54.595 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:47:54.595 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:47:54.595 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:47:54.596 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:54.587478+0000", 2023-09-23T20:47:54.596 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.326e-05 2023-09-23T20:47:54.596 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:47:54.596 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:47:54.596 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:47:54.596 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:54.587497+0000", 2023-09-23T20:47:54.596 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.895e-05 2023-09-23T20:47:54.596 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:47:54.596 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:47:54.596 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:47:54.596 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:54.587511+0000", 2023-09-23T20:47:54.597 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3284e-05 2023-09-23T20:47:54.597 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:47:54.597 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:47:54.597 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:47:54.597 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:54.587584+0000", 2023-09-23T20:47:54.597 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.3479000000000006e-05 2023-09-23T20:47:54.597 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:47:54.597 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:47:54.597 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:47:54.597 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:54.587608+0000", 2023-09-23T20:47:54.597 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.3845999999999998e-05 2023-09-23T20:47:54.597 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:47:54.598 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:47:54.598 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:47:54.598 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:47:54.598 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:47:54.598 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_repop(client.46625.0:136179 10.4 e570/558 10:2e72053a:::benchmark_data_smithi018_422431_object31683:head v 570'8423, mlcod=570'8420)", 2023-09-23T20:47:54.598 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:47:54.589755+0000", 2023-09-23T20:47:54.598 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.0022749720000000001, 2023-09-23T20:47:54.598 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0023811940000000001, 2023-09-23T20:47:54.598 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:47:54.598 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "started", 2023-09-23T20:47:54.598 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:47:54.599 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:47:54.599 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:47:54.599 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:54.589755+0000", 2023-09-23T20:47:54.599 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:47:54.599 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:47:54.599 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:47:54.599 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:47:54.599 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:54.589755+0000", 2023-09-23T20:47:54.599 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:47:54.599 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:47:54.599 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:47:54.600 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:47:54.600 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:54.589754+0000", 2023-09-23T20:47:54.600 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.999999 2023-09-23T20:47:54.600 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:47:54.600 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:47:54.600 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:47:54.600 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:54.589763+0000", 2023-09-23T20:47:54.600 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.2359999999999993e-06 2023-09-23T20:47:54.600 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:47:54.600 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:47:54.600 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:47:54.600 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:54.589776+0000", 2023-09-23T20:47:54.601 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3302000000000001e-05 2023-09-23T20:47:54.601 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:47:54.601 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:47:54.601 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:47:54.601 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:54.589784+0000", 2023-09-23T20:47:54.601 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.2879999999999993e-06 2023-09-23T20:47:54.601 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:47:54.601 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:47:54.601 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:47:54.601 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:54.589869+0000", 2023-09-23T20:47:54.601 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.4351999999999995e-05 2023-09-23T20:47:54.601 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:47:54.602 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:47:54.602 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:47:54.602 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:54.589890+0000", 2023-09-23T20:47:54.602 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.1129e-05 2023-09-23T20:47:54.602 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:47:54.602 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:47:54.602 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:47:54.602 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:47:54.602 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:47:54.602 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_repop(client.46625.0:136183 10.7 e570/531 10:ee6cb280:::benchmark_data_smithi018_422431_object31687:head v 570'8470, mlcod=570'8467)", 2023-09-23T20:47:54.602 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:47:54.591194+0000", 2023-09-23T20:47:54.603 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 0.00083613600000000002, 2023-09-23T20:47:54.603 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00099556899999999997, 2023-09-23T20:47:54.603 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:47:54.603 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "started", 2023-09-23T20:47:54.603 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:47:54.603 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:47:54.603 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:47:54.603 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:54.591194+0000", 2023-09-23T20:47:54.603 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:47:54.603 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:47:54.603 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:47:54.603 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:47:54.604 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:54.591194+0000", 2023-09-23T20:47:54.604 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:47:54.604 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:47:54.604 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:47:54.604 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:47:54.604 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:54.591193+0000", 2023-09-23T20:47:54.604 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.999999 2023-09-23T20:47:54.604 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:47:54.604 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:47:54.604 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:47:54.604 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:54.591201+0000", 2023-09-23T20:47:54.604 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.8880000000000003e-06 2023-09-23T20:47:54.605 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:47:54.605 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:47:54.605 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:47:54.605 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:54.591214+0000", 2023-09-23T20:47:54.605 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2577e-05 2023-09-23T20:47:54.605 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:47:54.605 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:47:54.605 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:47:54.605 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:54.591223+0000", 2023-09-23T20:47:54.605 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.5690000000000005e-06 2023-09-23T20:47:54.605 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:47:54.606 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:47:54.606 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:47:54.606 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:54.591534+0000", 2023-09-23T20:47:54.606 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00031162099999999999 2023-09-23T20:47:54.606 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:47:54.606 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:47:54.606 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:47:54.606 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:54.591554+0000", 2023-09-23T20:47:54.606 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.9921000000000001e-05 2023-09-23T20:47:54.606 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:47:54.606 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:47:54.607 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:47:54.607 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:47:54.607 INFO:teuthology.orchestra.run.smithi018.stdout: ], 2023-09-23T20:47:54.607 INFO:teuthology.orchestra.run.smithi018.stdout: "num_ops": 3 2023-09-23T20:47:54.607 INFO:teuthology.orchestra.run.smithi018.stdout:} 2023-09-23T20:47:54.686 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:54.799 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:47:54.807 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:54.808 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:47:54.908 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:47:54.953 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:55.005 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:47:55.008 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:55.009 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:47:55.063 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:47:55 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:47:55] "GET /metrics HTTP/1.1" 200 36575 "" "Prometheus/2.43.0" 2023-09-23T20:47:55.110 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:47:55.173 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:55.219 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:55 smithi149 ceph-mon[170394]: pgmap v1440: 119 pgs: 119 active+clean; 4.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 706 op/s 2023-09-23T20:47:55.359 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T20:47:55.360 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:55 smithi018 ceph-mon[184144]: pgmap v1440: 119 pgs: 119 active+clean; 4.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 706 op/s 2023-09-23T20:47:55.360 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:55 smithi018 ceph-mon[179647]: pgmap v1440: 119 pgs: 119 active+clean; 4.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 706 op/s 2023-09-23T20:47:55.403 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:47:55.413 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:55.413 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:47:55.514 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:47:55.618 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:55.770 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:47:55.801 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:47:55.805 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:55.805 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:47:55.906 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:47:55.948 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:55.995 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:47:56.000 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:56.001 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:47:56.102 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:47:56.136 INFO:teuthology.orchestra.run.smithi018.stderr:nodeep-scrub is set 2023-09-23T20:47:56.198 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:56.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:56 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/87995530' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:47:56.281 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:56 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/87995530' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:47:56.281 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:56 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/87995530' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:47:56.289 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:47:56.293 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:56.293 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:47:56.394 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:47:56.448 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:56.499 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:47:56.505 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:56.505 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:47:56.577 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:47:56.606 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:47:56.706 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:56.796 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:47:56.812 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:56.812 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:47:56.913 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:47:56.970 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:57.036 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:47:57.044 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:57.044 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:47:57.145 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:47:57.212 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:47:57.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:57 smithi149 ceph-mon[170394]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:47:57.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:57 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/87995530' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:47:57.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:57 smithi149 ceph-mon[170394]: osdmap e571: 8 total, 8 up, 8 in 2023-09-23T20:47:57.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:57 smithi149 ceph-mon[170394]: failed to encode map e571 with expected crc 2023-09-23T20:47:57.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:57 smithi149 ceph-mon[170394]: failed to encode map e571 with expected crc 2023-09-23T20:47:57.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:57 smithi149 ceph-mon[170394]: failed to encode map e571 with expected crc 2023-09-23T20:47:57.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:57 smithi149 ceph-mon[170394]: failed to encode map e571 with expected crc 2023-09-23T20:47:57.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:57 smithi149 ceph-mon[170394]: failed to encode map e571 with expected crc 2023-09-23T20:47:57.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:57 smithi149 ceph-mon[170394]: failed to encode map e571 with expected crc 2023-09-23T20:47:57.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:57 smithi149 ceph-mon[170394]: pgmap v1442: 119 pgs: 119 active+clean; 4.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 811 op/s 2023-09-23T20:47:57.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:57 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1836931974' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T20:47:57.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:57 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T20:47:57.272 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:57.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:57 smithi018 ceph-mon[179647]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:47:57.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:57 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/87995530' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:47:57.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:57 smithi018 ceph-mon[179647]: osdmap e571: 8 total, 8 up, 8 in 2023-09-23T20:47:57.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:57 smithi018 ceph-mon[179647]: failed to encode map e571 with expected crc 2023-09-23T20:47:57.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:57 smithi018 ceph-mon[179647]: failed to encode map e571 with expected crc 2023-09-23T20:47:57.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:57 smithi018 ceph-mon[179647]: failed to encode map e571 with expected crc 2023-09-23T20:47:57.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:57 smithi018 ceph-mon[179647]: failed to encode map e571 with expected crc 2023-09-23T20:47:57.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:57 smithi018 ceph-mon[179647]: failed to encode map e571 with expected crc 2023-09-23T20:47:57.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:57 smithi018 ceph-mon[179647]: failed to encode map e571 with expected crc 2023-09-23T20:47:57.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:57 smithi018 ceph-mon[179647]: pgmap v1442: 119 pgs: 119 active+clean; 4.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 811 op/s 2023-09-23T20:47:57.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:57 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1836931974' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T20:47:57.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:57 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T20:47:57.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:57 smithi018 ceph-mon[184144]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:47:57.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:57 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/87995530' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:47:57.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:57 smithi018 ceph-mon[184144]: osdmap e571: 8 total, 8 up, 8 in 2023-09-23T20:47:57.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:57 smithi018 ceph-mon[184144]: failed to encode map e571 with expected crc 2023-09-23T20:47:57.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:57 smithi018 ceph-mon[184144]: failed to encode map e571 with expected crc 2023-09-23T20:47:57.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:57 smithi018 ceph-mon[184144]: failed to encode map e571 with expected crc 2023-09-23T20:47:57.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:57 smithi018 ceph-mon[184144]: failed to encode map e571 with expected crc 2023-09-23T20:47:57.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:57 smithi018 ceph-mon[184144]: failed to encode map e571 with expected crc 2023-09-23T20:47:57.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:57 smithi018 ceph-mon[184144]: failed to encode map e571 with expected crc 2023-09-23T20:47:57.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:57 smithi018 ceph-mon[184144]: pgmap v1442: 119 pgs: 119 active+clean; 4.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 811 op/s 2023-09-23T20:47:57.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:57 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1836931974' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T20:47:57.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:57 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T20:47:57.493 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:47:57.499 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:57.499 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:47:57.600 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:47:57.690 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:57.715 INFO:teuthology.orchestra.run.smithi018.stderr:reweighted osd.6 to 1 (10000) 2023-09-23T20:47:57.763 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:47:57.777 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:57.778 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:47:57.878 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:47:57.933 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:57.992 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:47:57.997 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:57.997 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:47:58.098 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:47:58.150 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:58.178 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd reweight 5 1 2023-09-23T20:47:58.208 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:47:58.216 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:58.216 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:47:58.317 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:47:58.418 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:58.537 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T20:47:58.972 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:58 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T20:47:58.972 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:58 smithi149 ceph-mon[170394]: osdmap e572: 8 total, 8 up, 8 in 2023-09-23T20:47:58.972 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:58 smithi149 ceph-mon[170394]: failed to encode map e572 with expected crc 2023-09-23T20:47:58.972 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:58 smithi149 ceph-mon[170394]: failed to encode map e572 with expected crc 2023-09-23T20:47:59.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:58 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T20:47:59.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:58 smithi018 ceph-mon[184144]: osdmap e572: 8 total, 8 up, 8 in 2023-09-23T20:47:59.144 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:58 smithi018 ceph-mon[184144]: failed to encode map e572 with expected crc 2023-09-23T20:47:59.144 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:58 smithi018 ceph-mon[184144]: failed to encode map e572 with expected crc 2023-09-23T20:47:59.144 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:58 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T20:47:59.144 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:58 smithi018 ceph-mon[179647]: osdmap e572: 8 total, 8 up, 8 in 2023-09-23T20:47:59.144 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:58 smithi018 ceph-mon[179647]: failed to encode map e572 with expected crc 2023-09-23T20:47:59.144 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:58 smithi018 ceph-mon[179647]: failed to encode map e572 with expected crc 2023-09-23T20:47:59.232 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:47:59.244 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:59.244 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:47:59.345 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:47:59.399 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:59.446 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:47:59.450 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:59.451 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:47:59.551 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:47:59.592 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:59.661 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:47:59.666 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:47:59.667 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:47:59.767 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:47:59.821 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:47:59.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:59 smithi149 ceph-mon[170394]: pgmap v1444: 119 pgs: 119 active+clean; 4.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 772 op/s 2023-09-23T20:47:59.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:59 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:47:59.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:47:59 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:47:59.991 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:59 smithi018 ceph-mon[184144]: pgmap v1444: 119 pgs: 119 active+clean; 4.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 772 op/s 2023-09-23T20:47:59.992 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:59 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:47:59.992 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:47:59 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:47:59.992 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:59 smithi018 ceph-mon[179647]: pgmap v1444: 119 pgs: 119 active+clean; 4.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 772 op/s 2023-09-23T20:47:59.992 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:59 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:47:59.992 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:47:59 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:47:59.995 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:47:59.999 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:00.000 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:48:00.101 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:48:00.151 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:00.525 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2023-09-23T20:48:00.525 INFO:teuthology.orchestra.run.smithi018.stdout: "ops": [], 2023-09-23T20:48:00.525 INFO:teuthology.orchestra.run.smithi018.stdout: "complaint_time": 30, 2023-09-23T20:48:00.526 INFO:teuthology.orchestra.run.smithi018.stdout: "num_blocked_ops": 0 2023-09-23T20:48:00.526 INFO:teuthology.orchestra.run.smithi018.stdout:} 2023-09-23T20:48:00.537 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:48:00.553 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:00.554 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:48:00.655 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:48:00.709 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:00.766 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:48:00.772 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:00.773 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:48:00.789 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:48:00.874 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:48:00.906 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:48:00.947 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:48:00.977 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:00.986 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:48:01.048 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:48:01.087 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:48:01.125 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:48:01.158 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:48:01.165 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:48:01.166 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:01.167 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:48:01.203 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:48:01.213 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:48:01.213 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:48:01.213 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:48:01.213 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:48:01.213 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:48:01.213 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:48:01.213 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:48:01.213 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:48:01.268 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:48:01.284 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T20:48:01.320 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:48:01.335 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:01.390 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:48:01.398 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:01.398 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:48:01.499 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:48:01.538 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:01.601 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:48:01.608 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:01.608 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:48:01.701 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:01 smithi018 ceph-mon[179647]: pgmap v1445: 119 pgs: 119 active+clean; 4.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 637 op/s 2023-09-23T20:48:01.702 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:01 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1265467362' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:48:01.709 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:48:01.757 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:01.822 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:48:01.830 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:01.831 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:48:01.932 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:48:01.955 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:01 smithi018 ceph-mon[184144]: pgmap v1445: 119 pgs: 119 active+clean; 4.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 637 op/s 2023-09-23T20:48:01.955 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:01 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1265467362' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:48:01.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:01 smithi149 ceph-mon[170394]: pgmap v1445: 119 pgs: 119 active+clean; 4.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 637 op/s 2023-09-23T20:48:01.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:01 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1265467362' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:48:01.978 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:02.042 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:48:02.046 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:02.047 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:48:02.147 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:48:02.203 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:02.260 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:48:02.266 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:02.267 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:48:02.367 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:48:02.414 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:02.491 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:48:02.497 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:02.497 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:48:02.598 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:48:02.675 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:02.791 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:48:02.797 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:02.797 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:48:02.898 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:48:02.942 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:02.992 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:48:02.996 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:02.996 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:48:03.059 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:48:03.097 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:48:03.147 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:03.202 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:48:03.211 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:03.212 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:48:03.313 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:48:03.431 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:03.570 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:48:03.582 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:03.583 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:48:03.683 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:48:03.797 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:03.861 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:48:03.872 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:03 smithi018 ceph-mon[179647]: pgmap v1446: 119 pgs: 119 active+clean; 4.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 893 op/s 2023-09-23T20:48:03.872 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:03 smithi018 ceph-mon[184144]: pgmap v1446: 119 pgs: 119 active+clean; 4.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 893 op/s 2023-09-23T20:48:03.884 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:48:03.891 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:03.892 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:48:03.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:03 smithi149 ceph-mon[170394]: pgmap v1446: 119 pgs: 119 active+clean; 4.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 893 op/s 2023-09-23T20:48:03.992 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:48:04.128 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:04.253 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:48:04.264 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:04.264 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:48:04.365 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:48:04.451 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:04.550 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:48:04.563 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:04.563 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:48:04.664 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:48:04.701 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:04.723 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:04 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/934930854' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:48:04.723 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:04 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1713484277' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T20:48:04.724 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:04 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T20:48:04.724 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:04 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/934930854' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:48:04.724 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:04 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1713484277' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T20:48:04.724 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:04 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T20:48:04.750 INFO:teuthology.orchestra.run.smithi018.stderr:noscrub is unset 2023-09-23T20:48:04.763 INFO:teuthology.orchestra.run.smithi018.stderr:reweighted osd.5 to 1 (10000) 2023-09-23T20:48:04.767 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:48:04.776 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:04.777 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:48:04.877 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:48:04.923 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:04.973 INFO:teuthology.orchestra.run.smithi018.stdout:ERROR: (22) Invalid argument 2023-09-23T20:48:04.973 INFO:teuthology.orchestra.run.smithi018.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:48:04.975 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:04 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/934930854' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:48:04.975 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:04 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1713484277' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T20:48:04.975 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:04 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T20:48:04.989 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:48:04.992 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:48:04.999 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:48:04 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:48:04] "GET /metrics HTTP/1.1" 200 36575 "" "Prometheus/2.43.0" 2023-09-23T20:48:05.000 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:05.000 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:48:05.101 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:48:05.161 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:05.231 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:48:05.237 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:05.237 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:48:05.338 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:48:05.423 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:05.539 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:48:05.546 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:05.547 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:48:05.648 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:48:05.720 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd reweight 4 1 2023-09-23T20:48:05.728 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:05.777 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T20:48:05.800 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:05 smithi018 ceph-mon[179647]: pgmap v1447: 119 pgs: 119 active+clean; 4.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 719 op/s 2023-09-23T20:48:05.800 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:05 smithi018 ceph-mon[179647]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:48:05.800 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:05 smithi018 ceph-mon[179647]: failed to encode map e573 with expected crc 2023-09-23T20:48:05.800 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:05 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/934930854' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:48:05.800 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:05 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T20:48:05.800 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:05 smithi018 ceph-mon[179647]: osdmap e573: 8 total, 8 up, 8 in 2023-09-23T20:48:05.801 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:05 smithi018 ceph-mon[184144]: pgmap v1447: 119 pgs: 119 active+clean; 4.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 719 op/s 2023-09-23T20:48:05.801 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:05 smithi018 ceph-mon[184144]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:48:05.801 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:05 smithi018 ceph-mon[184144]: failed to encode map e573 with expected crc 2023-09-23T20:48:05.801 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:05 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/934930854' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:48:05.801 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:05 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T20:48:05.801 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:05 smithi018 ceph-mon[184144]: osdmap e573: 8 total, 8 up, 8 in 2023-09-23T20:48:05.995 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:48:06.005 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:06.005 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:48:06.106 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:48:06.142 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:05 smithi149 ceph-mon[170394]: pgmap v1447: 119 pgs: 119 active+clean; 4.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 719 op/s 2023-09-23T20:48:06.142 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:05 smithi149 ceph-mon[170394]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:48:06.142 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:05 smithi149 ceph-mon[170394]: failed to encode map e573 with expected crc 2023-09-23T20:48:06.142 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:05 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/934930854' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:48:06.143 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:05 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T20:48:06.143 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:05 smithi149 ceph-mon[170394]: osdmap e573: 8 total, 8 up, 8 in 2023-09-23T20:48:06.146 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:06.219 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:48:06.235 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:06.236 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:48:06.337 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:48:06.421 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:06.526 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:48:06.649 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:48:06.661 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:06.662 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:48:06.763 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:48:06.819 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:06.883 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:48:06.890 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:06.891 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:48:06.926 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:48:06.992 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:48:07.045 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:07.114 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:48:07.119 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:07.120 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:48:07.221 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:48:07.283 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:07.432 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:48:07.433 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:48:07.437 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:07.437 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:48:07.462 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:48:07.493 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:48:07.538 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:48:07.541 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:48:07.592 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:48:07.602 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:07.641 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:48:07.686 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:48:07.688 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:48:07.710 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:07.711 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:48:07.730 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:48:07.749 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:48:07.749 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:48:07.749 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:48:07.749 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:48:07.750 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:48:07.750 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:48:07.750 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:48:07.750 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:48:07.812 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:48:07.882 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:08.011 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:07 smithi018 ceph-mon[179647]: pgmap v1449: 119 pgs: 119 active+clean; 4.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 861 op/s 2023-09-23T20:48:08.011 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:07 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/841796851' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:48:08.011 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:07 smithi018 ceph-mon[184144]: pgmap v1449: 119 pgs: 119 active+clean; 4.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 861 op/s 2023-09-23T20:48:08.011 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:07 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/841796851' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:48:08.021 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T20:48:08.158 INFO:teuthology.orchestra.run.smithi149.stdout:{ 2023-09-23T20:48:08.158 INFO:teuthology.orchestra.run.smithi149.stdout: "ops": [ 2023-09-23T20:48:08.158 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:08.158 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_repop(client.46625.0:147213 10.0 e573/559 10:0cec98aa:::benchmark_data_smithi018_422431_object42717:head v 573'9169, mlcod=573'9166)", 2023-09-23T20:48:08.159 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:48:08.154426+0000", 2023-09-23T20:48:08.159 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 0.0031526340000000001, 2023-09-23T20:48:08.159 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0031751840000000002, 2023-09-23T20:48:08.159 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:48:08.159 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "started", 2023-09-23T20:48:08.159 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:48:08.159 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:08.159 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:48:08.159 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:08.154426+0000", 2023-09-23T20:48:08.159 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:48:08.159 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:08.160 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:08.160 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:48:08.160 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:08.154426+0000", 2023-09-23T20:48:08.160 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:48:08.160 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:08.160 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:08.160 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:48:08.160 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:08.154425+0000", 2023-09-23T20:48:08.160 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:48:08.160 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:08.160 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:08.160 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:48:08.161 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:08.154432+0000", 2023-09-23T20:48:08.161 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.2580000000000001e-06 2023-09-23T20:48:08.161 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:08.161 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:08.161 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:48:08.161 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:08.154440+0000", 2023-09-23T20:48:08.161 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.4630000000000001e-06 2023-09-23T20:48:08.161 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:08.161 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:08.161 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:48:08.161 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:08.154444+0000", 2023-09-23T20:48:08.161 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.2509999999999996e-06 2023-09-23T20:48:08.162 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:08.162 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:08.162 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:48:08.162 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:08.154483+0000", 2023-09-23T20:48:08.162 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.8989000000000002e-05 2023-09-23T20:48:08.162 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:08.162 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:08.162 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:48:08.162 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:08.154506+0000", 2023-09-23T20:48:08.162 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.2115000000000001e-05 2023-09-23T20:48:08.162 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:48:08.162 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:48:08.163 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:48:08.163 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:08.163 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:08.163 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_repop(client.46625.0:147215 10.0 e573/559 10:0ecbc286:::benchmark_data_smithi018_422431_object42719:head v 573'9170, mlcod=573'9166)", 2023-09-23T20:48:08.163 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:48:08.156367+0000", 2023-09-23T20:48:08.163 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 0.0012115240000000001, 2023-09-23T20:48:08.163 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0012740620000000001, 2023-09-23T20:48:08.163 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:48:08.163 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "started", 2023-09-23T20:48:08.163 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:48:08.163 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:08.163 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:48:08.164 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:08.156367+0000", 2023-09-23T20:48:08.164 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:48:08.164 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:08.164 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:08.164 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:48:08.164 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:08.156367+0000", 2023-09-23T20:48:08.164 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:48:08.164 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:08.164 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:08.164 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:48:08.164 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:08.156367+0000", 2023-09-23T20:48:08.164 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:48:08.165 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:08.165 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:08.165 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:48:08.165 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:08.156370+0000", 2023-09-23T20:48:08.165 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.8659999999999999e-06 2023-09-23T20:48:08.165 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:08.165 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:08.165 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:48:08.165 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:08.156382+0000", 2023-09-23T20:48:08.165 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1727e-05 2023-09-23T20:48:08.165 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:08.165 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:08.166 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:48:08.166 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:08.156389+0000", 2023-09-23T20:48:08.166 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.3239999999999999e-06 2023-09-23T20:48:08.166 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:08.166 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:08.166 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:48:08.166 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:08.157063+0000", 2023-09-23T20:48:08.166 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000674199 2023-09-23T20:48:08.166 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:08.166 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:08.166 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:48:08.167 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:08.157084+0000", 2023-09-23T20:48:08.167 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.1237999999999999e-05 2023-09-23T20:48:08.167 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:48:08.167 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:48:08.167 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:48:08.167 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:08.167 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:08.167 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:147212 10.e 10:799a25d0:::benchmark_data_smithi018_422431_object42716:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e573)", 2023-09-23T20:48:08.167 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:48:08.148361+0000", 2023-09-23T20:48:08.167 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 0.0092180649999999992, 2023-09-23T20:48:08.167 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.009338219, 2023-09-23T20:48:08.167 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:48:08.168 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:48:08.168 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:48:08.168 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:48:08.168 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:48:08.168 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 147212 2023-09-23T20:48:08.168 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:08.168 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:48:08.168 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:08.168 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:48:08.168 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:08.148361+0000", 2023-09-23T20:48:08.168 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:48:08.168 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:08.169 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:08.169 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:48:08.169 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:08.148361+0000", 2023-09-23T20:48:08.169 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:48:08.169 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:08.169 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:08.169 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:48:08.169 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:08.148359+0000", 2023-09-23T20:48:08.169 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:48:08.169 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:08.169 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:08.169 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:48:08.170 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:08.148366+0000", 2023-09-23T20:48:08.170 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.0380000000000002e-06 2023-09-23T20:48:08.170 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:08.170 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:08.170 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:48:08.170 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:08.148378+0000", 2023-09-23T20:48:08.170 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.2075999999999999e-05 2023-09-23T20:48:08.170 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:08.170 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:08.170 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:48:08.170 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:08.148385+0000", 2023-09-23T20:48:08.171 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.9349999999999997e-06 2023-09-23T20:48:08.171 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:08.171 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:08.171 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:48:08.171 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:08.151278+0000", 2023-09-23T20:48:08.171 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.002892553 2023-09-23T20:48:08.171 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:08.171 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:08.171 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:48:08.171 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:08.151422+0000", 2023-09-23T20:48:08.171 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00014402300000000001 2023-09-23T20:48:08.171 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:08.172 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:08.172 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 2,6", 2023-09-23T20:48:08.172 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:08.151512+0000", 2023-09-23T20:48:08.172 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.0387e-05 2023-09-23T20:48:08.172 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:08.172 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:08.172 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:48:08.172 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:08.154872+0000", 2023-09-23T20:48:08.172 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0033591789999999999 2023-09-23T20:48:08.172 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:48:08.172 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:48:08.173 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:48:08.173 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:08.173 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:08.173 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_repop(client.46625.0:147208 10.2 e573/560 10:4eb5bf9b:::benchmark_data_smithi018_422431_object42712:head v 573'9374, mlcod=573'9371)", 2023-09-23T20:48:08.173 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:48:08.154114+0000", 2023-09-23T20:48:08.173 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 0.0034647039999999999, 2023-09-23T20:48:08.173 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.003671087, 2023-09-23T20:48:08.173 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:48:08.173 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "started", 2023-09-23T20:48:08.173 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:48:08.173 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:08.173 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:48:08.174 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:08.154114+0000", 2023-09-23T20:48:08.174 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:48:08.174 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:08.174 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:08.174 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:48:08.174 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:08.154114+0000", 2023-09-23T20:48:08.174 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:48:08.174 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:08.174 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:08.174 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:48:08.174 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:08.154113+0000", 2023-09-23T20:48:08.175 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:48:08.175 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:08.175 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:08.175 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:48:08.175 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:08.154120+0000", 2023-09-23T20:48:08.175 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.3829999999999997e-06 2023-09-23T20:48:08.175 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:08.175 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:08.175 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:48:08.175 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:08.154130+0000", 2023-09-23T20:48:08.175 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0645000000000001e-05 2023-09-23T20:48:08.175 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:08.176 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:08.176 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:48:08.176 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:08.154136+0000", 2023-09-23T20:48:08.176 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.879e-06 2023-09-23T20:48:08.176 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:08.176 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:08.176 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:48:08.176 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:08.154181+0000", 2023-09-23T20:48:08.176 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.4475999999999999e-05 2023-09-23T20:48:08.176 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:08.176 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:08.177 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:48:08.177 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:08.154197+0000", 2023-09-23T20:48:08.177 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.6294e-05 2023-09-23T20:48:08.177 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:48:08.177 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:48:08.177 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:48:08.177 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:48:08.177 INFO:teuthology.orchestra.run.smithi149.stdout: ], 2023-09-23T20:48:08.177 INFO:teuthology.orchestra.run.smithi149.stdout: "num_ops": 4 2023-09-23T20:48:08.177 INFO:teuthology.orchestra.run.smithi149.stdout:} 2023-09-23T20:48:08.186 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:07 smithi149 ceph-mon[170394]: pgmap v1449: 119 pgs: 119 active+clean; 4.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 861 op/s 2023-09-23T20:48:08.186 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:07 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/841796851' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:48:08.307 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:48:08.320 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:08.320 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:48:08.421 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:48:08.467 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:08.529 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:48:08.537 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:08.537 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:48:08.570 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T20:48:08.638 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:48:08.707 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:08.784 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:48:08.795 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:08.795 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:48:08.896 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:48:08.980 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:09.043 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:48:09.056 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:09.056 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:48:09.093 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:48:09.158 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:48:09.225 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:09.326 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:48:09.330 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:09.330 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:48:09.403 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:48:09.430 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:48:09.491 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:09.552 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:48:09.562 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:09.562 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:48:09.663 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:48:09.741 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:09.796 INFO:teuthology.orchestra.run.smithi018.stderr:reweighted osd.4 to 1 (10000) 2023-09-23T20:48:09.894 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:09 smithi018 ceph-mon[184144]: pgmap v1450: 119 pgs: 119 active+clean; 3.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 813 op/s 2023-09-23T20:48:09.894 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:09 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3187593477' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T20:48:09.894 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:09 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T20:48:09.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:09 smithi018 ceph-mon[179647]: pgmap v1450: 119 pgs: 119 active+clean; 3.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 813 op/s 2023-09-23T20:48:09.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:09 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3187593477' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T20:48:09.894 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:09 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T20:48:10.007 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:48:10.013 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:10.014 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:48:10.115 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:48:10.185 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:10.214 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:09 smithi149 ceph-mon[170394]: pgmap v1450: 119 pgs: 119 active+clean; 3.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 813 op/s 2023-09-23T20:48:10.214 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:09 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3187593477' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T20:48:10.214 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:09 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T20:48:10.268 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:48:10.277 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:10.278 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:48:10.328 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-09-23T20:48:10.328 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:48:10.378 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:48:10.450 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:10.482 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:48:10.620 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:10.621 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:48:10.621 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:48:10.722 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:48:10.772 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:10.896 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:10 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T20:48:10.896 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:10 smithi018 ceph-mon[184144]: osdmap e574: 8 total, 8 up, 8 in 2023-09-23T20:48:10.897 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:10 smithi018 ceph-mon[184144]: failed to encode map e574 with expected crc 2023-09-23T20:48:10.897 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:10 smithi018 ceph-mon[184144]: failed to encode map e574 with expected crc 2023-09-23T20:48:10.897 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:10 smithi018 ceph-mon[184144]: failed to encode map e574 with expected crc 2023-09-23T20:48:10.897 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:10 smithi018 ceph-mon[184144]: failed to encode map e574 with expected crc 2023-09-23T20:48:10.897 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:10 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T20:48:10.897 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:10 smithi018 ceph-mon[179647]: osdmap e574: 8 total, 8 up, 8 in 2023-09-23T20:48:10.897 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:10 smithi018 ceph-mon[179647]: failed to encode map e574 with expected crc 2023-09-23T20:48:10.897 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:10 smithi018 ceph-mon[179647]: failed to encode map e574 with expected crc 2023-09-23T20:48:10.897 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:10 smithi018 ceph-mon[179647]: failed to encode map e574 with expected crc 2023-09-23T20:48:10.898 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:10 smithi018 ceph-mon[179647]: failed to encode map e574 with expected crc 2023-09-23T20:48:10.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:10 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T20:48:10.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:10 smithi149 ceph-mon[170394]: osdmap e574: 8 total, 8 up, 8 in 2023-09-23T20:48:10.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:10 smithi149 ceph-mon[170394]: failed to encode map e574 with expected crc 2023-09-23T20:48:10.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:10 smithi149 ceph-mon[170394]: failed to encode map e574 with expected crc 2023-09-23T20:48:10.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:10 smithi149 ceph-mon[170394]: failed to encode map e574 with expected crc 2023-09-23T20:48:10.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:10 smithi149 ceph-mon[170394]: failed to encode map e574 with expected crc 2023-09-23T20:48:11.017 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:48:11.022 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:11.022 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:48:11.123 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:48:11.209 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:11.288 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:48:11.296 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:48:11.300 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:11.300 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:48:11.401 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:48:11.465 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:48:11.465 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:11.597 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:48:11.608 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:11.609 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:48:11.709 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:48:11.765 INFO:teuthology.orchestra.run.smithi149.stdout:{ 2023-09-23T20:48:11.766 INFO:teuthology.orchestra.run.smithi149.stdout: "ops": [], 2023-09-23T20:48:11.766 INFO:teuthology.orchestra.run.smithi149.stdout: "complaint_time": 30, 2023-09-23T20:48:11.766 INFO:teuthology.orchestra.run.smithi149.stdout: "num_blocked_ops": 0 2023-09-23T20:48:11.766 INFO:teuthology.orchestra.run.smithi149.stdout:} 2023-09-23T20:48:11.792 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:11.842 INFO:teuthology.orchestra.run.smithi018.stderr:nodeep-scrub is unset 2023-09-23T20:48:12.063 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:48:12.066 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:11 smithi018 ceph-mon[179647]: failed to encode map e574 with expected crc 2023-09-23T20:48:12.067 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:11 smithi018 ceph-mon[179647]: pgmap v1452: 119 pgs: 119 active+clean; 3.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 637 op/s 2023-09-23T20:48:12.067 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:11 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2573081411' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:48:12.067 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:12.068 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:48:12.069 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:11 smithi018 ceph-mon[184144]: failed to encode map e574 with expected crc 2023-09-23T20:48:12.069 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:11 smithi018 ceph-mon[184144]: pgmap v1452: 119 pgs: 119 active+clean; 3.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 637 op/s 2023-09-23T20:48:12.070 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:11 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2573081411' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:48:12.114 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:11 smithi149 ceph-mon[170394]: failed to encode map e574 with expected crc 2023-09-23T20:48:12.114 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:11 smithi149 ceph-mon[170394]: pgmap v1452: 119 pgs: 119 active+clean; 3.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 637 op/s 2023-09-23T20:48:12.114 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:11 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2573081411' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:48:12.152 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T20:48:12.168 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:48:12.265 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:12.336 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:48:12.340 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:12.340 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:48:12.441 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:48:12.537 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:12.616 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:48:12.624 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:12.625 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:48:12.725 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:48:12.786 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:12 smithi149 ceph-mon[170394]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:48:12.786 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:12 smithi149 ceph-mon[170394]: Cluster is now healthy 2023-09-23T20:48:12.786 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:12 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2573081411' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:48:12.786 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:12 smithi149 ceph-mon[170394]: osdmap e575: 8 total, 8 up, 8 in 2023-09-23T20:48:12.787 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:12 smithi149 ceph-mon[170394]: failed to encode map e575 with expected crc 2023-09-23T20:48:12.826 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:12.906 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:48:12.914 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:12.914 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:48:12.951 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:48:13.015 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:48:13.071 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:13.118 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:12 smithi018 ceph-mon[179647]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:48:13.118 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:12 smithi018 ceph-mon[179647]: Cluster is now healthy 2023-09-23T20:48:13.118 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:12 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2573081411' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:48:13.119 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:12 smithi018 ceph-mon[179647]: osdmap e575: 8 total, 8 up, 8 in 2023-09-23T20:48:13.119 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:12 smithi018 ceph-mon[179647]: failed to encode map e575 with expected crc 2023-09-23T20:48:13.119 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:12 smithi018 ceph-mon[184144]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:48:13.119 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:12 smithi018 ceph-mon[184144]: Cluster is now healthy 2023-09-23T20:48:13.119 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:12 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2573081411' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:48:13.119 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:12 smithi018 ceph-mon[184144]: osdmap e575: 8 total, 8 up, 8 in 2023-09-23T20:48:13.119 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:12 smithi018 ceph-mon[184144]: failed to encode map e575 with expected crc 2023-09-23T20:48:13.485 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:48:13.532 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:48:13.556 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:48:13.560 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:13.561 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:48:13.576 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:48:13.625 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:48:13.661 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:48:13.682 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:48:13.718 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:48:13.757 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:48:13.779 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:13.794 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:48:13.796 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:48:13.796 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:48:13.796 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:48:13.796 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:48:13.797 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:48:13.797 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:48:13.797 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:48:13.797 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:48:13.810 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:13 smithi149 ceph-mon[170394]: pgmap v1454: 119 pgs: 119 active+clean; 3.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 932 op/s 2023-09-23T20:48:13.810 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:13 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1577367973' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:48:13.862 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:48:13.870 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:13.870 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:48:13.971 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:48:14.025 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:48:14.028 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:14.041 INFO:teuthology.orchestra.run.smithi018.stderr:dumped all 2023-09-23T20:48:14.061 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:13 smithi018 ceph-mon[179647]: pgmap v1454: 119 pgs: 119 active+clean; 3.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 932 op/s 2023-09-23T20:48:14.061 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:13 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1577367973' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:48:14.062 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:13 smithi018 ceph-mon[184144]: pgmap v1454: 119 pgs: 119 active+clean; 3.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 932 op/s 2023-09-23T20:48:14.062 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:13 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1577367973' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:48:14.379 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T20:48:14.546 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:48:14.556 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:14.556 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:48:14.570 INFO:teuthology.orchestra.run.smithi018.stdout:{"pg_ready":true,"pg_map":{"version":1454,"stamp":"2023-09-23T20:48:12.313334+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3946977227,"num_objects":60182,"num_object_clones":0,"num_object_copies":180546,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":60182,"num_whiteouts":0,"num_read":5984,"num_read_kb":13201,"num_write":149741,"num_write_kb":6700278,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2468,"num_bytes_recovered":164443868,"num_keys_recovered":60,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":115726,"ondisk_log_size":115726,"up":357,"acting":357,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":357,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":17283696,"kb_used_data":11661872,"kb_used_omap":32,"kb_used_meta":5621599,"kb_avail":732677520,"statfs":{"total":767960285184,"available":750261780480,"internally_reserved":0,"allocated":11941756928,"data_stored":11924618954,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32773,"internal_metadata":5756518395},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,2,2,8,9,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":10,"apply_latency_ms":10,"commit_latency_ns":10000000,"apply_latency_ns":10000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-464125952,"num_objects":-7082,"num_object_clones":0,"num_object_copies":-21246,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-7082,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7082,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"7.593470"},"pg_stats":[{"pgid":"1.18","version":"90'161","reported_seq":620,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-23T20:48:04.738677+0000","last_change":"2023-09-23T20:44:19.677357+0000","last_active":"2023-09-23T20:48:04.738677+0000","last_peered":"2023-09-23T20:48:04.738677+0000","last_clean":"2023-09-23T20:48:04.738677+0000","last_became_active":"2023-09-23T20:43:16.766305+0000","last_became_peered":"2023-09-23T20:43:16.766305+0000","last_unstale":"2023-09-23T20:48:04.738677+0000","last_undegraded":"2023-09-23T20:48:04.738677+0000","last_fullsized":"2023-09-23T20:48:04.738677+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:19.677209+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:19.677209+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:14:06.510115+0000","scrub_duration":0.0023292260000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"69'1","reported_seq":31,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-23T20:48:04.737263+0000","last_change":"2023-09-23T20:47:09.186765+0000","last_active":"2023-09-23T20:48:04.737263+0000","last_peered":"2023-09-23T20:48:04.737263+0000","last_clean":"2023-09-23T20:48:04.737263+0000","last_became_active":"2023-09-23T20:47:09.186366+0000","last_became_peered":"2023-09-23T20:47:09.186366+0000","last_unstale":"2023-09-23T20:48:04.737263+0000","last_undegraded":"2023-09-23T20:48:04.737263+0000","last_fullsized":"2023-09-23T20:48:04.737263+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:10:04.707663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":407,"reported_epoch":572,"state":"active+clean","last_fresh":"2023-09-23T20:47:57.714593+0000","last_change":"2023-09-23T20:47:09.197577+0000","last_active":"2023-09-23T20:47:57.714593+0000","last_peered":"2023-09-23T20:47:57.714593+0000","last_clean":"2023-09-23T20:47:57.714593+0000","last_became_active":"2023-09-23T20:47:09.197280+0000","last_became_peered":"2023-09-23T20:47:09.197280+0000","last_unstale":"2023-09-23T20:47:57.714593+0000","last_undegraded":"2023-09-23T20:47:57.714593+0000","last_fullsized":"2023-09-23T20:47:57.714593+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:55:04.346224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"90'161","reported_seq":577,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-23T20:48:04.737950+0000","last_change":"2023-09-23T20:47:33.523278+0000","last_active":"2023-09-23T20:48:04.737950+0000","last_peered":"2023-09-23T20:48:04.737950+0000","last_clean":"2023-09-23T20:48:04.737950+0000","last_became_active":"2023-09-23T20:47:11.622087+0000","last_became_peered":"2023-09-23T20:47:11.622087+0000","last_unstale":"2023-09-23T20:48:04.737950+0000","last_undegraded":"2023-09-23T20:48:04.737950+0000","last_fullsized":"2023-09-23T20:48:04.737950+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:47:33.523187+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:36:05.442288+0000","last_clean_scrub_stamp":"2023-09-23T20:47:33.523187+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:32:18.410526+0000","scrub_duration":0.013532127,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":562,"reported_epoch":572,"state":"active+clean","last_fresh":"2023-09-23T20:47:57.693479+0000","last_change":"2023-09-23T20:47:09.173501+0000","last_active":"2023-09-23T20:47:57.693479+0000","last_peered":"2023-09-23T20:47:57.693479+0000","last_clean":"2023-09-23T20:47:57.693479+0000","last_became_active":"2023-09-23T20:47:09.173160+0000","last_became_peered":"2023-09-23T20:47:09.173160+0000","last_unstale":"2023-09-23T20:47:57.693479+0000","last_undegraded":"2023-09-23T20:47:57.693479+0000","last_fullsized":"2023-09-23T20:47:57.693479+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:44:36.132093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":533,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-23T20:48:04.735365+0000","last_change":"2023-09-23T20:43:46.521939+0000","last_active":"2023-09-23T20:48:04.735365+0000","last_peered":"2023-09-23T20:48:04.735365+0000","last_clean":"2023-09-23T20:48:04.735365+0000","last_became_active":"2023-09-23T20:43:46.521185+0000","last_became_peered":"2023-09-23T20:43:46.521185+0000","last_unstale":"2023-09-23T20:48:04.735365+0000","last_undegraded":"2023-09-23T20:48:04.735365+0000","last_fullsized":"2023-09-23T20:48:04.735365+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:59:25.045627+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"90'161","reported_seq":462,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-23T20:48:04.737121+0000","last_change":"2023-09-23T20:47:31.475001+0000","last_active":"2023-09-23T20:48:04.737121+0000","last_peered":"2023-09-23T20:48:04.737121+0000","last_clean":"2023-09-23T20:48:04.737121+0000","last_became_active":"2023-09-23T20:47:11.617942+0000","last_became_peered":"2023-09-23T20:47:11.617942+0000","last_unstale":"2023-09-23T20:48:04.737121+0000","last_undegraded":"2023-09-23T20:48:04.737121+0000","last_fullsized":"2023-09-23T20:48:04.737121+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:47:31.474914+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:47:31.474914+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:52:05.169219+0000","scrub_duration":0.0026262870000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":369,"reported_epoch":572,"state":"active+clean","last_fresh":"2023-09-23T20:47:57.708144+0000","last_change":"2023-09-23T20:37:43.584414+0000","last_active":"2023-09-23T20:47:57.708144+0000","last_peered":"2023-09-23T20:47:57.708144+0000","last_clean":"2023-09-23T20:47:57.708144+0000","last_became_active":"2023-09-23T20:37:43.584116+0000","last_became_peered":"2023-09-23T20:37:43.584116+0000","last_unstale":"2023-09-23T20:47:57.708144+0000","last_undegraded":"2023-09-23T20:47:57.708144+0000","last_fullsized":"2023-09-23T20:47:57.708144+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:17:54.258172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":189,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-23T20:48:04.726219+0000","last_change":"2023-09-23T20:38:37.457427+0000","last_active":"2023-09-23T20:48:04.726219+0000","last_peered":"2023-09-23T20:48:04.726219+0000","last_clean":"2023-09-23T20:48:04.726219+0000","last_became_active":"2023-09-23T20:38:37.457099+0000","last_became_peered":"2023-09-23T20:38:37.457099+0000","last_unstale":"2023-09-23T20:48:04.726219+0000","last_undegraded":"2023-09-23T20:48:04.726219+0000","last_fullsized":"2023-09-23T20:48:04.726219+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:31:27.084994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"90'161","reported_seq":786,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-23T20:48:04.740426+0000","last_change":"2023-09-23T20:40:22.449754+0000","last_active":"2023-09-23T20:48:04.740426+0000","last_peered":"2023-09-23T20:48:04.740426+0000","last_clean":"2023-09-23T20:48:04.740426+0000","last_became_active":"2023-09-23T20:40:22.448838+0000","last_became_peered":"2023-09-23T20:40:22.448838+0000","last_unstale":"2023-09-23T20:48:04.740426+0000","last_undegraded":"2023-09-23T20:48:04.740426+0000","last_fullsized":"2023-09-23T20:48:04.740426+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:03.411025+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:03.411025+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:22:07.948856+0000","scrub_duration":0.001802922,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":565,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-23T20:48:04.743333+0000","last_change":"2023-09-23T20:43:46.522325+0000","last_active":"2023-09-23T20:48:04.743333+0000","last_peered":"2023-09-23T20:48:04.743333+0000","last_clean":"2023-09-23T20:48:04.743333+0000","last_became_active":"2023-09-23T20:43:46.521996+0000","last_became_peered":"2023-09-23T20:43:46.521996+0000","last_unstale":"2023-09-23T20:48:04.743333+0000","last_undegraded":"2023-09-23T20:48:04.743333+0000","last_fullsized":"2023-09-23T20:48:04.743333+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:51:17.629956+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":299,"reported_epoch":572,"state":"active+clean","last_fresh":"2023-09-23T20:47:57.692074+0000","last_change":"2023-09-23T20:38:02.579343+0000","last_active":"2023-09-23T20:47:57.692074+0000","last_peered":"2023-09-23T20:47:57.692074+0000","last_clean":"2023-09-23T20:47:57.692074+0000","last_became_active":"2023-09-23T20:38:02.579047+0000","last_became_peered":"2023-09-23T20:38:02.579047+0000","last_unstale":"2023-09-23T20:47:57.692074+0000","last_undegraded":"2023-09-23T20:47:57.692074+0000","last_fullsized":"2023-09-23T20:47:57.692074+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:44:03.166556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.14","version":"90'161","reported_seq":939,"reported_epoch":572,"state":"active+clean","last_fresh":"2023-09-23T20:47:57.696106+0000","last_change":"2023-09-23T20:47:11.618616+0000","last_active":"2023-09-23T20:47:57.696106+0000","last_peered":"2023-09-23T20:47:57.696106+0000","last_clean":"2023-09-23T20:47:57.696106+0000","last_became_active":"2023-09-23T20:47:11.618237+0000","last_became_peered":"2023-09-23T20:47:11.618237+0000","last_unstale":"2023-09-23T20:47:57.696106+0000","last_undegraded":"2023-09-23T20:47:57.696106+0000","last_fullsized":"2023-09-23T20:47:57.696106+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:47.479012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:47.479012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:06:10.817025+0000","scrub_duration":0.0020136590000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":410,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-23T20:48:04.741523+0000","last_change":"2023-09-23T20:43:16.780478+0000","last_active":"2023-09-23T20:48:04.741523+0000","last_peered":"2023-09-23T20:48:04.741523+0000","last_clean":"2023-09-23T20:48:04.741523+0000","last_became_active":"2023-09-23T20:43:16.780183+0000","last_became_peered":"2023-09-23T20:43:16.780183+0000","last_unstale":"2023-09-23T20:48:04.741523+0000","last_undegraded":"2023-09-23T20:48:04.741523+0000","last_fullsized":"2023-09-23T20:48:04.741523+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:38:30.102066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":400,"reported_epoch":572,"state":"active+clean","last_fresh":"2023-09-23T20:47:57.708454+0000","last_change":"2023-09-23T20:38:02.542034+0000","last_active":"2023-09-23T20:47:57.708454+0000","last_peered":"2023-09-23T20:47:57.708454+0000","last_clean":"2023-09-23T20:47:57.708454+0000","last_became_active":"2023-09-23T20:38:02.541575+0000","last_became_peered":"2023-09-23T20:38:02.541575+0000","last_unstale":"2023-09-23T20:47:57.708454+0000","last_undegraded":"2023-09-23T20:47:57.708454+0000","last_fullsized":"2023-09-23T20:47:57.708454+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:50:54.591529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"90'161","reported_seq":700,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-23T20:48:04.743128+0000","last_change":"2023-09-23T20:44:14.669973+0000","last_active":"2023-09-23T20:48:04.743128+0000","last_peered":"2023-09-23T20:48:04.743128+0000","last_clean":"2023-09-23T20:48:04.743128+0000","last_became_active":"2023-09-23T20:43:46.519592+0000","last_became_peered":"2023-09-23T20:43:46.519592+0000","last_unstale":"2023-09-23T20:48:04.743128+0000","last_undegraded":"2023-09-23T20:48:04.743128+0000","last_fullsized":"2023-09-23T20:48:04.743128+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:14.669831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:14.669831+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:56:28.076617+0000","scrub_duration":0.0083441460000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":633,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-23T20:48:04.740660+0000","last_change":"2023-09-23T20:47:10.837423+0000","last_active":"2023-09-23T20:48:04.740660+0000","last_peered":"2023-09-23T20:48:04.740660+0000","last_clean":"2023-09-23T20:48:04.740660+0000","last_became_active":"2023-09-23T20:47:09.204324+0000","last_became_peered":"2023-09-23T20:47:09.204324+0000","last_unstale":"2023-09-23T20:48:04.740660+0000","last_undegraded":"2023-09-23T20:48:04.740660+0000","last_fullsized":"2023-09-23T20:48:04.740660+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":554,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:19.108914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":540,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":88,"reported_epoch":572,"state":"active+clean","last_fresh":"2023-09-23T20:47:57.693619+0000","last_change":"2023-09-23T20:44:10.611381+0000","last_active":"2023-09-23T20:47:57.693619+0000","last_peered":"2023-09-23T20:47:57.693619+0000","last_clean":"2023-09-23T20:47:57.693619+0000","last_became_active":"2023-09-23T20:44:10.611091+0000","last_became_peered":"2023-09-23T20:44:10.611091+0000","last_unstale":"2023-09-23T20:47:57.693619+0000","last_undegraded":"2023-09-23T20:47:57.693619+0000","last_fullsized":"2023-09-23T20:47:57.693619+0000","mapping_epoch":496,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":497,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:00:30.162478+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"90'161","reported_seq":700,"reported_epoch":572,"state":"active+clean","last_fresh":"2023-09-23T20:47:57.694538+0000","last_change":"2023-09-23T20:36:02.548387+0000","last_active":"2023-09-23T20:47:57.694538+0000","last_peered":"2023-09-23T20:47:57.694538+0000","last_clean":"2023-09-23T20:47:57.694538+0000","last_became_active":"2023-09-23T20:35:26.644787+0000","last_became_peered":"2023-09-23T20:35:26.644787+0000","last_unstale":"2023-09-23T20:47:57.694538+0000","last_undegraded":"2023-09-23T20:47:57.694538+0000","last_fullsized":"2023-09-23T20:47:57.694538+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:02.548285+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:02.548285+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:37:15.216553+0000","scrub_duration":0.0015514459999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":41,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-23T20:48:04.738532+0000","last_change":"2023-09-23T20:47:09.340979+0000","last_active":"2023-09-23T20:48:04.738532+0000","last_peered":"2023-09-23T20:48:04.738532+0000","last_clean":"2023-09-23T20:48:04.738532+0000","last_became_active":"2023-09-23T20:47:09.231222+0000","last_became_peered":"2023-09-23T20:47:09.231222+0000","last_unstale":"2023-09-23T20:48:04.738532+0000","last_undegraded":"2023-09-23T20:48:04.738532+0000","last_fullsized":"2023-09-23T20:48:04.738532+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:07:50.620006+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":31,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-23T20:48:04.738120+0000","last_change":"2023-09-23T20:47:09.186315+0000","last_active":"2023-09-23T20:48:04.738120+0000","last_peered":"2023-09-23T20:48:04.738120+0000","last_clean":"2023-09-23T20:48:04.738120+0000","last_became_active":"2023-09-23T20:47:09.184580+0000","last_became_peered":"2023-09-23T20:47:09.184580+0000","last_unstale":"2023-09-23T20:48:04.738120+0000","last_undegraded":"2023-09-23T20:48:04.738120+0000","last_fullsized":"2023-09-23T20:48:04.738120+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:44:35.553948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"90'161","reported_seq":828,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-23T20:48:04.734585+0000","last_change":"2023-09-23T20:43:46.544727+0000","last_active":"2023-09-23T20:48:04.734585+0000","last_peered":"2023-09-23T20:48:04.734585+0000","last_clean":"2023-09-23T20:48:04.734585+0000","last_became_active":"2023-09-23T20:43:46.543952+0000","last_became_peered":"2023-09-23T20:43:46.543952+0000","last_unstale":"2023-09-23T20:48:04.734585+0000","last_undegraded":"2023-09-23T20:48:04.734585+0000","last_fullsized":"2023-09-23T20:48:04.734585+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:43.578666+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:43.578666+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:11:16.611920+0000","scrub_duration":0.0021890379999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":632,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-23T20:48:04.740533+0000","last_change":"2023-09-23T20:47:09.242696+0000","last_active":"2023-09-23T20:48:04.740533+0000","last_peered":"2023-09-23T20:48:04.740533+0000","last_clean":"2023-09-23T20:48:04.740533+0000","last_became_active":"2023-09-23T20:47:09.176728+0000","last_became_peered":"2023-09-23T20:47:09.176728+0000","last_unstale":"2023-09-23T20:48:04.740533+0000","last_undegraded":"2023-09-23T20:48:04.740533+0000","last_fullsized":"2023-09-23T20:48:04.740533+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:46:41.529834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":204,"num_keys_recovered":60,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":320,"reported_epoch":572,"state":"active+clean","last_fresh":"2023-09-23T20:47:57.713738+0000","last_change":"2023-09-23T20:45:01.875931+0000","last_active":"2023-09-23T20:47:57.713738+0000","last_peered":"2023-09-23T20:47:57.713738+0000","last_clean":"2023-09-23T20:47:57.713738+0000","last_became_active":"2023-09-23T20:45:01.872839+0000","last_became_peered":"2023-09-23T20:45:01.872839+0000","last_unstale":"2023-09-23T20:47:57.713738+0000","last_undegraded":"2023-09-23T20:47:57.713738+0000","last_fullsized":"2023-09-23T20:47:57.713738+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:15:09.584507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"90'161","reported_seq":854,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-23T20:48:04.740186+0000","last_change":"2023-09-23T20:44:20.041766+0000","last_active":"2023-09-23T20:48:04.740186+0000","last_peered":"2023-09-23T20:48:04.740186+0000","last_clean":"2023-09-23T20:48:04.740186+0000","last_became_active":"2023-09-23T20:43:16.765015+0000","last_became_peered":"2023-09-23T20:43:16.765015+0000","last_unstale":"2023-09-23T20:48:04.740186+0000","last_undegraded":"2023-09-23T20:48:04.740186+0000","last_fullsized":"2023-09-23T20:48:04.740186+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:20.041690+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:20.041690+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:33:30.150109+0000","scrub_duration":0.0026180330000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":530,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-23T20:48:04.744320+0000","last_change":"2023-09-23T20:43:46.521294+0000","last_active":"2023-09-23T20:48:04.744320+0000","last_peered":"2023-09-23T20:48:04.744320+0000","last_clean":"2023-09-23T20:48:04.744320+0000","last_became_active":"2023-09-23T20:43:46.521001+0000","last_became_peered":"2023-09-23T20:43:46.521001+0000","last_unstale":"2023-09-23T20:48:04.744320+0000","last_undegraded":"2023-09-23T20:48:04.744320+0000","last_fullsized":"2023-09-23T20:48:04.744320+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:01:35.126716+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":341,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-23T20:48:04.740732+0000","last_change":"2023-09-23T20:44:01.841565+0000","last_active":"2023-09-23T20:48:04.740732+0000","last_peered":"2023-09-23T20:48:04.740732+0000","last_clean":"2023-09-23T20:48:04.740732+0000","last_became_active":"2023-09-23T20:44:01.841255+0000","last_became_peered":"2023-09-23T20:44:01.841255+0000","last_unstale":"2023-09-23T20:48:04.740732+0000","last_undegraded":"2023-09-23T20:48:04.740732+0000","last_fullsized":"2023-09-23T20:48:04.740732+0000","mapping_epoch":491,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":492,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:18:44.817589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.13","version":"90'161","reported_seq":895,"reported_epoch":572,"state":"active+clean","last_fresh":"2023-09-23T20:47:57.695118+0000","last_change":"2023-09-23T20:47:11.619371+0000","last_active":"2023-09-23T20:47:57.695118+0000","last_peered":"2023-09-23T20:47:57.695118+0000","last_clean":"2023-09-23T20:47:57.695118+0000","last_became_active":"2023-09-23T20:47:11.619006+0000","last_became_peered":"2023-09-23T20:47:11.619006+0000","last_unstale":"2023-09-23T20:47:57.695118+0000","last_undegraded":"2023-09-23T20:47:57.695118+0000","last_fullsized":"2023-09-23T20:47:57.695118+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:42.544308+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:42.544308+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:47:24.414807+0000","scrub_duration":0.0029958179999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":597,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-23T20:48:04.740387+0000","last_change":"2023-09-23T20:47:09.176613+0000","last_active":"2023-09-23T20:48:04.740387+0000","last_peered":"2023-09-23T20:48:04.740387+0000","last_clean":"2023-09-23T20:48:04.740387+0000","last_became_active":"2023-09-23T20:47:09.176187+0000","last_became_peered":"2023-09-23T20:47:09.176187+0000","last_unstale":"2023-09-23T20:48:04.740387+0000","last_undegraded":"2023-09-23T20:48:04.740387+0000","last_fullsized":"2023-09-23T20:48:04.740387+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:23:01.590989+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":539,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-23T20:48:04.735430+0000","last_change":"2023-09-23T20:43:46.521735+0000","last_active":"2023-09-23T20:48:04.735430+0000","last_peered":"2023-09-23T20:48:04.735430+0000","last_clean":"2023-09-23T20:48:04.735430+0000","last_became_active":"2023-09-23T20:43:46.520762+0000","last_became_peered":"2023-09-23T20:43:46.520762+0000","last_unstale":"2023-09-23T20:48:04.735430+0000","last_undegraded":"2023-09-23T20:48:04.735430+0000","last_fullsized":"2023-09-23T20:48:04.735430+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:26:48.841930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"90'161","reported_seq":658,"reported_epoch":572,"state":"active+clean","last_fresh":"2023-09-23T20:47:58.652283+0000","last_change":"2023-09-23T20:44:18.952808+0000","last_active":"2023-09-23T20:47:58.652283+0000","last_peered":"2023-09-23T20:47:58.652283+0000","last_clean":"2023-09-23T20:47:58.652283+0000","last_became_active":"2023-09-23T20:43:16.758652+0000","last_became_peered":"2023-09-23T20:43:16.758652+0000","last_unstale":"2023-09-23T20:47:58.652283+0000","last_undegraded":"2023-09-23T20:47:58.652283+0000","last_fullsized":"2023-09-23T20:47:58.652283+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:18.952693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:18.952693+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:12:19.496465+0000","scrub_duration":0.001699365,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":249,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-23T20:48:04.725875+0000","last_change":"2023-09-23T20:44:43.644290+0000","last_active":"2023-09-23T20:48:04.725875+0000","last_peered":"2023-09-23T20:48:04.725875+0000","last_clean":"2023-09-23T20:48:04.725875+0000","last_became_active":"2023-09-23T20:44:43.638313+0000","last_became_peered":"2023-09-23T20:44:43.638313+0000","last_unstale":"2023-09-23T20:48:04.725875+0000","last_undegraded":"2023-09-23T20:48:04.725875+0000","last_fullsized":"2023-09-23T20:48:04.725875+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:32:52.892603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":369,"reported_epoch":572,"state":"active+clean","last_fresh":"2023-09-23T20:47:57.720047+0000","last_change":"2023-09-23T20:37:43.571083+0000","last_active":"2023-09-23T20:47:57.720047+0000","last_peered":"2023-09-23T20:47:57.720047+0000","last_clean":"2023-09-23T20:47:57.720047+0000","last_became_active":"2023-09-23T20:37:43.570306+0000","last_became_peered":"2023-09-23T20:37:43.570306+0000","last_unstale":"2023-09-23T20:47:57.720047+0000","last_undegraded":"2023-09-23T20:47:57.720047+0000","last_fullsized":"2023-09-23T20:47:57.720047+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:42:02.396718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.6","version":"574'9412","reported_seq":10066,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-09-23T20:48:11.547894+0000","last_change":"2023-09-23T20:47:12.636171+0000","last_active":"2023-09-23T20:48:11.547894+0000","last_peered":"2023-09-23T20:48:11.547894+0000","last_clean":"2023-09-23T20:48:11.547894+0000","last_became_active":"2023-09-23T20:47:12.628915+0000","last_became_peered":"2023-09-23T20:47:12.628915+0000","last_unstale":"2023-09-23T20:48:11.547894+0000","last_undegraded":"2023-09-23T20:48:11.547894+0000","last_fullsized":"2023-09-23T20:48:11.547894+0000","mapping_epoch":556,"log_start":"537'2890","ondisk_log_start":"537'2890","created":531,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":6522,"log_dups_size":0,"ondisk_log_size":6522,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:52:38.896682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":243269632,"num_objects":3712,"num_object_clones":0,"num_object_copies":11136,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3712,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9412,"num_write_kb":419968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":287,"num_bytes_recovered":18808832,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.d","version":"90'161","reported_seq":906,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-23T20:48:04.740505+0000","last_change":"2023-09-23T20:47:11.619510+0000","last_active":"2023-09-23T20:48:04.740505+0000","last_peered":"2023-09-23T20:48:04.740505+0000","last_clean":"2023-09-23T20:48:04.740505+0000","last_became_active":"2023-09-23T20:47:11.619154+0000","last_became_peered":"2023-09-23T20:47:11.619154+0000","last_unstale":"2023-09-23T20:48:04.740505+0000","last_undegraded":"2023-09-23T20:48:04.740505+0000","last_fullsized":"2023-09-23T20:48:04.740505+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:34:59.496375+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:34:59.496375+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:40:52.712777+0000","scrub_duration":0.0017843189999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":605,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-23T20:48:04.741750+0000","last_change":"2023-09-23T20:47:09.207851+0000","last_active":"2023-09-23T20:48:04.741750+0000","last_peered":"2023-09-23T20:48:04.741750+0000","last_clean":"2023-09-23T20:48:04.741750+0000","last_became_active":"2023-09-23T20:47:09.175836+0000","last_became_peered":"2023-09-23T20:47:09.175836+0000","last_unstale":"2023-09-23T20:48:04.741750+0000","last_undegraded":"2023-09-23T20:48:04.741750+0000","last_fullsized":"2023-09-23T20:48:04.741750+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:43:13.091809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":388,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-23T20:48:04.740434+0000","last_change":"2023-09-23T20:43:16.765253+0000","last_active":"2023-09-23T20:48:04.740434+0000","last_peered":"2023-09-23T20:48:04.740434+0000","last_clean":"2023-09-23T20:48:04.740434+0000","last_became_active":"2023-09-23T20:43:16.764978+0000","last_became_peered":"2023-09-23T20:43:16.764978+0000","last_unstale":"2023-09-23T20:48:04.740434+0000","last_undegraded":"2023-09-23T20:48:04.740434+0000","last_fullsized":"2023-09-23T20:48:04.740434+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:07:23.539372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.7","version":"574'9234","reported_seq":9285,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-09-23T20:48:10.426968+0000","last_change":"2023-09-23T20:45:39.488542+0000","last_active":"2023-09-23T20:48:10.426968+0000","last_peered":"2023-09-23T20:48:10.426968+0000","last_clean":"2023-09-23T20:48:10.426968+0000","last_became_active":"2023-09-23T20:45:39.488259+0000","last_became_peered":"2023-09-23T20:45:39.488259+0000","last_unstale":"2023-09-23T20:48:10.426968+0000","last_undegraded":"2023-09-23T20:48:10.426968+0000","last_fullsized":"2023-09-23T20:48:10.426968+0000","mapping_epoch":531,"log_start":"537'2826","ondisk_log_start":"537'2826","created":531,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":6408,"log_dups_size":0,"ondisk_log_size":6408,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:39:05.833662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":245104640,"num_objects":3740,"num_object_clones":0,"num_object_copies":11220,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3740,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9234,"num_write_kb":415168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.c","version":"90'161","reported_seq":835,"reported_epoch":572,"state":"active+clean","last_fresh":"2023-09-23T20:47:57.713828+0000","last_change":"2023-09-23T20:41:09.506481+0000","last_active":"2023-09-23T20:47:57.713828+0000","last_peered":"2023-09-23T20:47:57.713828+0000","last_clean":"2023-09-23T20:47:57.713828+0000","last_became_active":"2023-09-23T20:38:28.442229+0000","last_became_peered":"2023-09-23T20:38:28.442229+0000","last_unstale":"2023-09-23T20:47:57.713828+0000","last_undegraded":"2023-09-23T20:47:57.713828+0000","last_fullsized":"2023-09-23T20:47:57.713828+0000","mapping_epoch":411,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":412,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:41:09.506338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:41:09.506338+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:19:46.265011+0000","scrub_duration":0.01112002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":643,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-23T20:48:04.741546+0000","last_change":"2023-09-23T20:47:10.848436+0000","last_active":"2023-09-23T20:48:04.741546+0000","last_peered":"2023-09-23T20:48:04.741546+0000","last_clean":"2023-09-23T20:48:04.741546+0000","last_became_active":"2023-09-23T20:47:09.196769+0000","last_became_peered":"2023-09-23T20:47:09.196769+0000","last_unstale":"2023-09-23T20:48:04.741546+0000","last_undegraded":"2023-09-23T20:48:04.741546+0000","last_fullsized":"2023-09-23T20:48:04.741546+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":554,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:59:38.105009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":31,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-23T20:48:04.737670+0000","last_change":"2023-09-23T20:47:09.183860+0000","last_active":"2023-09-23T20:48:04.737670+0000","last_peered":"2023-09-23T20:48:04.737670+0000","last_clean":"2023-09-23T20:48:04.737670+0000","last_became_active":"2023-09-23T20:47:09.183545+0000","last_became_peered":"2023-09-23T20:47:09.183545+0000","last_unstale":"2023-09-23T20:48:04.737670+0000","last_undegraded":"2023-09-23T20:48:04.737670+0000","last_fullsized":"2023-09-23T20:48:04.737670+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:54:03.275528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"574'9261","reported_seq":9889,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-09-23T20:48:11.545135+0000","last_change":"2023-09-23T20:47:14.635782+0000","last_active":"2023-09-23T20:48:11.545135+0000","last_peered":"2023-09-23T20:48:11.545135+0000","last_clean":"2023-09-23T20:48:11.545135+0000","last_became_active":"2023-09-23T20:47:14.627887+0000","last_became_peered":"2023-09-23T20:47:14.627887+0000","last_unstale":"2023-09-23T20:48:11.545135+0000","last_undegraded":"2023-09-23T20:48:11.545135+0000","last_fullsized":"2023-09-23T20:48:11.545135+0000","mapping_epoch":558,"log_start":"537'2693","ondisk_log_start":"537'2693","created":531,"last_epoch_clean":559,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":6568,"log_dups_size":0,"ondisk_log_size":6568,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:13:49.501508+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":240189440,"num_objects":3665,"num_object_clones":0,"num_object_copies":10995,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3665,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9261,"num_write_kb":413632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":274,"num_bytes_recovered":17956864,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"90'161","reported_seq":769,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-23T20:48:04.726415+0000","last_change":"2023-09-23T20:47:11.622303+0000","last_active":"2023-09-23T20:48:04.726415+0000","last_peered":"2023-09-23T20:48:04.726415+0000","last_clean":"2023-09-23T20:48:04.726415+0000","last_became_active":"2023-09-23T20:47:11.621953+0000","last_became_peered":"2023-09-23T20:47:11.621953+0000","last_unstale":"2023-09-23T20:48:04.726415+0000","last_undegraded":"2023-09-23T20:48:04.726415+0000","last_fullsized":"2023-09-23T20:48:04.726415+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:46.217489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:46.217489+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:14:40.478522+0000","scrub_duration":0.001456429,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":602,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-23T20:48:04.740562+0000","last_change":"2023-09-23T20:40:19.677655+0000","last_active":"2023-09-23T20:48:04.740562+0000","last_peered":"2023-09-23T20:48:04.740562+0000","last_clean":"2023-09-23T20:48:04.740562+0000","last_became_active":"2023-09-23T20:40:19.551189+0000","last_became_peered":"2023-09-23T20:40:19.551189+0000","last_unstale":"2023-09-23T20:48:04.740562+0000","last_undegraded":"2023-09-23T20:48:04.740562+0000","last_fullsized":"2023-09-23T20:48:04.740562+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:11:01.559576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":179,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-23T20:48:04.726327+0000","last_change":"2023-09-23T20:38:02.590613+0000","last_active":"2023-09-23T20:48:04.726327+0000","last_peered":"2023-09-23T20:48:04.726327+0000","last_clean":"2023-09-23T20:48:04.726327+0000","last_became_active":"2023-09-23T20:38:02.590284+0000","last_became_peered":"2023-09-23T20:38:02.590284+0000","last_unstale":"2023-09-23T20:48:04.726327+0000","last_undegraded":"2023-09-23T20:48:04.726327+0000","last_fullsized":"2023-09-23T20:48:04.726327+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:40:07.825054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.5","version":"573'9252","reported_seq":9303,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-23T20:48:08.240091+0000","last_change":"2023-09-23T20:45:39.380986+0000","last_active":"2023-09-23T20:48:08.240091+0000","last_peered":"2023-09-23T20:48:08.240091+0000","last_clean":"2023-09-23T20:48:08.240091+0000","last_became_active":"2023-09-23T20:45:39.380650+0000","last_became_peered":"2023-09-23T20:45:39.380650+0000","last_unstale":"2023-09-23T20:48:08.240091+0000","last_undegraded":"2023-09-23T20:48:08.240091+0000","last_fullsized":"2023-09-23T20:48:08.240091+0000","mapping_epoch":531,"log_start":"535'400","ondisk_log_start":"535'400","created":531,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":8852,"log_dups_size":0,"ondisk_log_size":8852,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:24:24.301239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":257687552,"num_objects":3932,"num_object_clones":0,"num_object_copies":11796,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3932,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9252,"num_write_kb":421888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.e","version":"90'161","reported_seq":604,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-23T20:48:04.737935+0000","last_change":"2023-09-23T20:47:37.608426+0000","last_active":"2023-09-23T20:48:04.737935+0000","last_peered":"2023-09-23T20:48:04.737935+0000","last_clean":"2023-09-23T20:48:04.737935+0000","last_became_active":"2023-09-23T20:47:11.617187+0000","last_became_peered":"2023-09-23T20:47:11.617187+0000","last_unstale":"2023-09-23T20:48:04.737935+0000","last_undegraded":"2023-09-23T20:48:04.737935+0000","last_fullsized":"2023-09-23T20:48:04.737935+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:47:37.608309+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:47:37.608309+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:22:25.482297+0000","scrub_duration":0.011600924,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":534,"reported_epoch":572,"state":"active+clean","last_fresh":"2023-09-23T20:47:57.714080+0000","last_change":"2023-09-23T20:43:16.756437+0000","last_active":"2023-09-23T20:47:57.714080+0000","last_peered":"2023-09-23T20:47:57.714080+0000","last_clean":"2023-09-23T20:47:57.714080+0000","last_became_active":"2023-09-23T20:43:16.756193+0000","last_became_peered":"2023-09-23T20:43:16.756193+0000","last_unstale":"2023-09-23T20:47:57.714080+0000","last_undegraded":"2023-09-23T20:47:57.714080+0000","last_fullsized":"2023-09-23T20:47:57.714080+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:33:25.632781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":566,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-23T20:48:04.745064+0000","last_change":"2023-09-23T20:47:09.197427+0000","last_active":"2023-09-23T20:48:04.745064+0000","last_peered":"2023-09-23T20:48:04.745064+0000","last_clean":"2023-09-23T20:48:04.745064+0000","last_became_active":"2023-09-23T20:47:09.196542+0000","last_became_peered":"2023-09-23T20:47:09.196542+0000","last_unstale":"2023-09-23T20:48:04.745064+0000","last_undegraded":"2023-09-23T20:48:04.745064+0000","last_fullsized":"2023-09-23T20:48:04.745064+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:28:38.926202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"573'9405","reported_seq":10445,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-23T20:48:08.632564+0000","last_change":"2023-09-23T20:47:16.635196+0000","last_active":"2023-09-23T20:48:08.632564+0000","last_peered":"2023-09-23T20:48:08.632564+0000","last_clean":"2023-09-23T20:48:08.632564+0000","last_became_active":"2023-09-23T20:47:16.628907+0000","last_became_peered":"2023-09-23T20:47:16.628907+0000","last_unstale":"2023-09-23T20:48:08.632564+0000","last_undegraded":"2023-09-23T20:48:08.632564+0000","last_fullsized":"2023-09-23T20:48:08.632564+0000","mapping_epoch":560,"log_start":"537'2724","ondisk_log_start":"537'2724","created":531,"last_epoch_clean":561,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":6681,"log_dups_size":0,"ondisk_log_size":6681,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:30:22.186026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":254214144,"num_objects":3879,"num_object_clones":0,"num_object_copies":11637,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3879,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9405,"num_write_kb":425088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":477,"num_bytes_recovered":31260672,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":979,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-23T20:48:04.740306+0000","last_change":"2023-09-23T20:35:02.523308+0000","last_active":"2023-09-23T20:48:04.740306+0000","last_peered":"2023-09-23T20:48:04.740306+0000","last_clean":"2023-09-23T20:48:04.740306+0000","last_became_active":"2023-09-23T20:33:38.979989+0000","last_became_peered":"2023-09-23T20:33:38.979989+0000","last_unstale":"2023-09-23T20:48:04.740306+0000","last_undegraded":"2023-09-23T20:48:04.740306+0000","last_fullsized":"2023-09-23T20:48:04.740306+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:35:02.523203+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:35:02.523203+0000","last_clean_scrub_stamp":"2023-09-23T20:35:02.523203+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:02:41.472958+0000","scrub_duration":0.0085927989999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":421,"reported_epoch":572,"state":"active+clean","last_fresh":"2023-09-23T20:47:57.713941+0000","last_change":"2023-09-23T20:47:09.240200+0000","last_active":"2023-09-23T20:47:57.713941+0000","last_peered":"2023-09-23T20:47:57.713941+0000","last_clean":"2023-09-23T20:47:57.713941+0000","last_became_active":"2023-09-23T20:47:09.195204+0000","last_became_peered":"2023-09-23T20:47:09.195204+0000","last_unstale":"2023-09-23T20:47:57.713941+0000","last_undegraded":"2023-09-23T20:47:57.713941+0000","last_fullsized":"2023-09-23T20:47:57.713941+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:03.328250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":31,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-23T20:48:04.738478+0000","last_change":"2023-09-23T20:47:09.231714+0000","last_active":"2023-09-23T20:48:04.738478+0000","last_peered":"2023-09-23T20:48:04.738478+0000","last_clean":"2023-09-23T20:48:04.738478+0000","last_became_active":"2023-09-23T20:47:09.230918+0000","last_became_peered":"2023-09-23T20:47:09.230918+0000","last_unstale":"2023-09-23T20:48:04.738478+0000","last_undegraded":"2023-09-23T20:48:04.738478+0000","last_fullsized":"2023-09-23T20:48:04.738478+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:24.583656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"573'9393","reported_seq":9445,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-23T20:48:08.194693+0000","last_change":"2023-09-23T20:45:39.488901+0000","last_active":"2023-09-23T20:48:08.194693+0000","last_peered":"2023-09-23T20:48:08.194693+0000","last_clean":"2023-09-23T20:48:08.194693+0000","last_became_active":"2023-09-23T20:45:39.488499+0000","last_became_peered":"2023-09-23T20:45:39.488499+0000","last_unstale":"2023-09-23T20:48:08.194693+0000","last_undegraded":"2023-09-23T20:48:08.194693+0000","last_fullsized":"2023-09-23T20:48:08.194693+0000","mapping_epoch":531,"log_start":"535'500","ondisk_log_start":"535'500","created":531,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":8893,"log_dups_size":0,"ondisk_log_size":8893,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:25:10.137770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":259194880,"num_objects":3955,"num_object_clones":0,"num_object_copies":11865,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3955,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9393,"num_write_kb":427136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":689,"reported_epoch":572,"state":"active+clean","last_fresh":"2023-09-23T20:47:57.692152+0000","last_change":"2023-09-23T20:44:46.725779+0000","last_active":"2023-09-23T20:47:57.692152+0000","last_peered":"2023-09-23T20:47:57.692152+0000","last_clean":"2023-09-23T20:47:57.692152+0000","last_became_active":"2023-09-23T20:44:46.725433+0000","last_became_peered":"2023-09-23T20:44:46.725433+0000","last_unstale":"2023-09-23T20:47:57.692152+0000","last_undegraded":"2023-09-23T20:47:57.692152+0000","last_fullsized":"2023-09-23T20:47:57.692152+0000","mapping_epoch":516,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":517,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:42:45.492220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:42:45.492220+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:28:35.202407+0000","scrub_duration":0.015259412,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":199,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-23T20:48:04.726412+0000","last_change":"2023-09-23T20:47:09.178919+0000","last_active":"2023-09-23T20:48:04.726412+0000","last_peered":"2023-09-23T20:48:04.726412+0000","last_clean":"2023-09-23T20:48:04.726412+0000","last_became_active":"2023-09-23T20:47:09.178607+0000","last_became_peered":"2023-09-23T20:47:09.178607+0000","last_unstale":"2023-09-23T20:48:04.726412+0000","last_undegraded":"2023-09-23T20:48:04.726412+0000","last_fullsized":"2023-09-23T20:48:04.726412+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:23:45.196935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":31,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-23T20:48:04.738501+0000","last_change":"2023-09-23T20:47:09.186661+0000","last_active":"2023-09-23T20:48:04.738501+0000","last_peered":"2023-09-23T20:48:04.738501+0000","last_clean":"2023-09-23T20:48:04.738501+0000","last_became_active":"2023-09-23T20:47:09.186311+0000","last_became_peered":"2023-09-23T20:47:09.186311+0000","last_unstale":"2023-09-23T20:48:04.738501+0000","last_undegraded":"2023-09-23T20:48:04.738501+0000","last_fullsized":"2023-09-23T20:48:04.738501+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:18:51.382502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.0","version":"574'9287","reported_seq":10526,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-09-23T20:48:10.402305+0000","last_change":"2023-09-23T20:47:15.653243+0000","last_active":"2023-09-23T20:48:10.402305+0000","last_peered":"2023-09-23T20:48:10.402305+0000","last_clean":"2023-09-23T20:48:10.402305+0000","last_became_active":"2023-09-23T20:47:15.626592+0000","last_became_peered":"2023-09-23T20:47:15.626592+0000","last_unstale":"2023-09-23T20:48:10.402305+0000","last_undegraded":"2023-09-23T20:48:10.402305+0000","last_fullsized":"2023-09-23T20:48:10.402305+0000","mapping_epoch":559,"log_start":"537'2849","ondisk_log_start":"537'2849","created":531,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":6438,"log_dups_size":0,"ondisk_log_size":6438,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:57:31.932707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":242679808,"num_objects":3703,"num_object_clones":0,"num_object_copies":11109,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3703,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9287,"num_write_kb":415680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":578,"num_bytes_recovered":37879808,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.b","version":"90'161","reported_seq":878,"reported_epoch":572,"state":"active+clean","last_fresh":"2023-09-23T20:47:57.695198+0000","last_change":"2023-09-23T20:40:22.445808+0000","last_active":"2023-09-23T20:47:57.695198+0000","last_peered":"2023-09-23T20:47:57.695198+0000","last_clean":"2023-09-23T20:47:57.695198+0000","last_became_active":"2023-09-23T20:40:22.445456+0000","last_became_peered":"2023-09-23T20:40:22.445456+0000","last_unstale":"2023-09-23T20:47:57.695198+0000","last_undegraded":"2023-09-23T20:47:57.695198+0000","last_fullsized":"2023-09-23T20:47:57.695198+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:05.595042+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:05.595042+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:39:23.678154+0000","scrub_duration":0.007271675,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":31,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-23T20:48:04.737542+0000","last_change":"2023-09-23T20:47:09.183852+0000","last_active":"2023-09-23T20:48:04.737542+0000","last_peered":"2023-09-23T20:48:04.737542+0000","last_clean":"2023-09-23T20:48:04.737542+0000","last_became_active":"2023-09-23T20:47:09.183575+0000","last_became_peered":"2023-09-23T20:47:09.183575+0000","last_unstale":"2023-09-23T20:48:04.737542+0000","last_undegraded":"2023-09-23T20:48:04.737542+0000","last_fullsized":"2023-09-23T20:48:04.737542+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:40:30.918482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":31,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-23T20:48:04.738002+0000","last_change":"2023-09-23T20:47:09.186271+0000","last_active":"2023-09-23T20:48:04.738002+0000","last_peered":"2023-09-23T20:48:04.738002+0000","last_clean":"2023-09-23T20:48:04.738002+0000","last_became_active":"2023-09-23T20:47:09.184357+0000","last_became_peered":"2023-09-23T20:47:09.184357+0000","last_unstale":"2023-09-23T20:48:04.738002+0000","last_undegraded":"2023-09-23T20:48:04.738002+0000","last_fullsized":"2023-09-23T20:48:04.738002+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:48:26.339811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"574'9379","reported_seq":9472,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-09-23T20:48:10.748423+0000","last_change":"2023-09-23T20:47:10.208583+0000","last_active":"2023-09-23T20:48:10.748423+0000","last_peered":"2023-09-23T20:48:10.748423+0000","last_clean":"2023-09-23T20:48:10.748423+0000","last_became_active":"2023-09-23T20:47:10.171839+0000","last_became_peered":"2023-09-23T20:47:10.171839+0000","last_unstale":"2023-09-23T20:48:10.748423+0000","last_undegraded":"2023-09-23T20:48:10.748423+0000","last_fullsized":"2023-09-23T20:48:10.748423+0000","mapping_epoch":553,"log_start":"538'3524","ondisk_log_start":"538'3524","created":531,"last_epoch_clean":554,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":5855,"log_dups_size":0,"ondisk_log_size":5855,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:48:36.327572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":244121600,"num_objects":3725,"num_object_clones":0,"num_object_copies":11175,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3725,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9379,"num_write_kb":419328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":786432,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":600,"reported_epoch":572,"state":"active+clean","last_fresh":"2023-09-23T20:47:57.697993+0000","last_change":"2023-09-23T20:43:51.693618+0000","last_active":"2023-09-23T20:47:57.697993+0000","last_peered":"2023-09-23T20:47:57.697993+0000","last_clean":"2023-09-23T20:47:57.697993+0000","last_became_active":"2023-09-23T20:43:16.765186+0000","last_became_peered":"2023-09-23T20:43:16.765186+0000","last_unstale":"2023-09-23T20:47:57.697993+0000","last_undegraded":"2023-09-23T20:47:57.697993+0000","last_fullsized":"2023-09-23T20:47:57.697993+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:43:51.693459+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:43:51.693459+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:33:30.208355+0000","scrub_duration":0.0016145269999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":369,"reported_epoch":572,"state":"active+clean","last_fresh":"2023-09-23T20:47:57.708066+0000","last_change":"2023-09-23T20:37:43.570610+0000","last_active":"2023-09-23T20:47:57.708066+0000","last_peered":"2023-09-23T20:47:57.708066+0000","last_clean":"2023-09-23T20:47:57.708066+0000","last_became_active":"2023-09-23T20:37:43.570216+0000","last_became_peered":"2023-09-23T20:37:43.570216+0000","last_unstale":"2023-09-23T20:47:57.708066+0000","last_undegraded":"2023-09-23T20:47:57.708066+0000","last_fullsized":"2023-09-23T20:47:57.708066+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:39:26.555329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":418,"reported_epoch":572,"state":"active+clean","last_fresh":"2023-09-23T20:47:57.697352+0000","last_change":"2023-09-23T20:47:09.172573+0000","last_active":"2023-09-23T20:47:57.697352+0000","last_peered":"2023-09-23T20:47:57.697352+0000","last_clean":"2023-09-23T20:47:57.697352+0000","last_became_active":"2023-09-23T20:47:09.172298+0000","last_became_peered":"2023-09-23T20:47:09.172298+0000","last_unstale":"2023-09-23T20:47:57.697352+0000","last_undegraded":"2023-09-23T20:47:57.697352+0000","last_fullsized":"2023-09-23T20:47:57.697352+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:22:09.361284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"574'9437","reported_seq":10069,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-09-23T20:48:10.361264+0000","last_change":"2023-09-23T20:47:11.626133+0000","last_active":"2023-09-23T20:48:10.361264+0000","last_peered":"2023-09-23T20:48:10.361264+0000","last_clean":"2023-09-23T20:48:10.361264+0000","last_became_active":"2023-09-23T20:47:11.619197+0000","last_became_peered":"2023-09-23T20:47:11.619197+0000","last_unstale":"2023-09-23T20:48:10.361264+0000","last_undegraded":"2023-09-23T20:48:10.361264+0000","last_fullsized":"2023-09-23T20:48:10.361264+0000","mapping_epoch":555,"log_start":"537'2993","ondisk_log_start":"537'2993","created":531,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":6444,"log_dups_size":0,"ondisk_log_size":6444,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:33:18.084175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":249757696,"num_objects":3811,"num_object_clones":0,"num_object_copies":11433,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3811,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9437,"num_write_kb":423936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":277,"num_bytes_recovered":18153472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":762,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-23T20:48:04.726384+0000","last_change":"2023-09-23T20:41:07.424609+0000","last_active":"2023-09-23T20:48:04.726384+0000","last_peered":"2023-09-23T20:48:04.726384+0000","last_clean":"2023-09-23T20:48:04.726384+0000","last_became_active":"2023-09-23T20:38:05.622261+0000","last_became_peered":"2023-09-23T20:38:05.622261+0000","last_unstale":"2023-09-23T20:48:04.726384+0000","last_undegraded":"2023-09-23T20:48:04.726384+0000","last_fullsized":"2023-09-23T20:48:04.726384+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:41:07.424492+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:41:07.424492+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:45:21.689123+0000","scrub_duration":0.0018960959999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"86'119","reported_seq":884,"reported_epoch":572,"state":"active+clean","last_fresh":"2023-09-23T20:47:57.693854+0000","last_change":"2023-09-23T20:29:45.521257+0000","last_active":"2023-09-23T20:47:57.693854+0000","last_peered":"2023-09-23T20:47:57.693854+0000","last_clean":"2023-09-23T20:47:57.693854+0000","last_became_active":"2023-09-23T20:29:45.520546+0000","last_became_peered":"2023-09-23T20:29:45.520546+0000","last_unstale":"2023-09-23T20:47:57.693854+0000","last_undegraded":"2023-09-23T20:47:57.693854+0000","last_fullsized":"2023-09-23T20:47:57.693854+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":119,"log_dups_size":0,"ondisk_log_size":119,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:58:56.085777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032705499999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":523,"num_read_kb":3503,"num_write":136,"num_write_kb":3128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":367,"reported_epoch":572,"state":"active+clean","last_fresh":"2023-09-23T20:47:57.714678+0000","last_change":"2023-09-23T20:43:16.756494+0000","last_active":"2023-09-23T20:47:57.714678+0000","last_peered":"2023-09-23T20:47:57.714678+0000","last_clean":"2023-09-23T20:47:57.714678+0000","last_became_active":"2023-09-23T20:43:16.756131+0000","last_became_peered":"2023-09-23T20:43:16.756131+0000","last_unstale":"2023-09-23T20:47:57.714678+0000","last_undegraded":"2023-09-23T20:47:57.714678+0000","last_fullsized":"2023-09-23T20:47:57.714678+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:10:19.219533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":549,"reported_epoch":572,"state":"active+clean","last_fresh":"2023-09-23T20:47:57.693754+0000","last_change":"2023-09-23T20:47:09.194947+0000","last_active":"2023-09-23T20:47:57.693754+0000","last_peered":"2023-09-23T20:47:57.693754+0000","last_clean":"2023-09-23T20:47:57.693754+0000","last_became_active":"2023-09-23T20:47:09.194592+0000","last_became_peered":"2023-09-23T20:47:09.194592+0000","last_unstale":"2023-09-23T20:47:57.693754+0000","last_undegraded":"2023-09-23T20:47:57.693754+0000","last_fullsized":"2023-09-23T20:47:57.693754+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:34:15.008492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"574'9098","reported_seq":9149,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-09-23T20:48:10.419363+0000","last_change":"2023-09-23T20:45:39.489110+0000","last_active":"2023-09-23T20:48:10.419363+0000","last_peered":"2023-09-23T20:48:10.419363+0000","last_clean":"2023-09-23T20:48:10.419363+0000","last_became_active":"2023-09-23T20:45:39.488817+0000","last_became_peered":"2023-09-23T20:45:39.488817+0000","last_unstale":"2023-09-23T20:48:10.419363+0000","last_undegraded":"2023-09-23T20:48:10.419363+0000","last_fullsized":"2023-09-23T20:48:10.419363+0000","mapping_epoch":531,"log_start":"537'2625","ondisk_log_start":"537'2625","created":531,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":6473,"log_dups_size":0,"ondisk_log_size":6473,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:07:21.467971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":238551040,"num_objects":3640,"num_object_clones":0,"num_object_copies":10920,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3640,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9098,"num_write_kb":407616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":621,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-23T20:48:04.734860+0000","last_change":"2023-09-23T20:44:15.656984+0000","last_active":"2023-09-23T20:48:04.734860+0000","last_peered":"2023-09-23T20:48:04.734860+0000","last_clean":"2023-09-23T20:48:04.734860+0000","last_became_active":"2023-09-23T20:43:46.512210+0000","last_became_peered":"2023-09-23T20:43:46.512210+0000","last_unstale":"2023-09-23T20:48:04.734860+0000","last_undegraded":"2023-09-23T20:48:04.734860+0000","last_fullsized":"2023-09-23T20:48:04.734860+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:15.656880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:15.656880+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:37:56.453466+0000","scrub_duration":0.0016711180000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"86'106","reported_seq":825,"reported_epoch":572,"state":"active+clean","last_fresh":"2023-09-23T20:47:57.694038+0000","last_change":"2023-09-23T20:43:16.759138+0000","last_active":"2023-09-23T20:47:57.694038+0000","last_peered":"2023-09-23T20:47:57.694038+0000","last_clean":"2023-09-23T20:47:57.694038+0000","last_became_active":"2023-09-23T20:43:16.758015+0000","last_became_peered":"2023-09-23T20:43:16.758015+0000","last_unstale":"2023-09-23T20:47:57.694038+0000","last_undegraded":"2023-09-23T20:47:57.694038+0000","last_fullsized":"2023-09-23T20:47:57.694038+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":106,"log_dups_size":0,"ondisk_log_size":106,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:08:02.852472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00099302599999999993,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1094,"num_read_kb":1678,"num_write":100,"num_write_kb":404,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":31,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-23T20:48:04.738208+0000","last_change":"2023-09-23T20:47:09.177872+0000","last_active":"2023-09-23T20:48:04.738208+0000","last_peered":"2023-09-23T20:48:04.738208+0000","last_clean":"2023-09-23T20:48:04.738208+0000","last_became_active":"2023-09-23T20:47:09.176689+0000","last_became_peered":"2023-09-23T20:47:09.176689+0000","last_unstale":"2023-09-23T20:48:04.738208+0000","last_undegraded":"2023-09-23T20:48:04.738208+0000","last_fullsized":"2023-09-23T20:48:04.738208+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:14:01.590318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":537,"reported_epoch":572,"state":"active+clean","last_fresh":"2023-09-23T20:47:57.693753+0000","last_change":"2023-09-23T20:29:45.533406+0000","last_active":"2023-09-23T20:47:57.693753+0000","last_peered":"2023-09-23T20:47:57.693753+0000","last_clean":"2023-09-23T20:47:57.693753+0000","last_became_active":"2023-09-23T20:29:45.526001+0000","last_became_peered":"2023-09-23T20:29:45.526001+0000","last_unstale":"2023-09-23T20:47:57.693753+0000","last_undegraded":"2023-09-23T20:47:57.693753+0000","last_fullsized":"2023-09-23T20:47:57.693753+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:43.787251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.9","version":"573'9265","reported_seq":9317,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-23T20:48:08.186407+0000","last_change":"2023-09-23T20:45:39.488721+0000","last_active":"2023-09-23T20:48:08.186407+0000","last_peered":"2023-09-23T20:48:08.186407+0000","last_clean":"2023-09-23T20:48:08.186407+0000","last_became_active":"2023-09-23T20:45:39.488320+0000","last_became_peered":"2023-09-23T20:45:39.488320+0000","last_unstale":"2023-09-23T20:48:08.186407+0000","last_undegraded":"2023-09-23T20:48:08.186407+0000","last_fullsized":"2023-09-23T20:48:08.186407+0000","mapping_epoch":531,"log_start":"535'400","ondisk_log_start":"535'400","created":531,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":8865,"log_dups_size":0,"ondisk_log_size":8865,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:46:30.536848+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":258277376,"num_objects":3941,"num_object_clones":0,"num_object_copies":11823,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3941,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9265,"num_write_kb":422592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":577,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-23T20:48:04.740477+0000","last_change":"2023-09-23T20:47:11.618652+0000","last_active":"2023-09-23T20:48:04.740477+0000","last_peered":"2023-09-23T20:48:04.740477+0000","last_clean":"2023-09-23T20:48:04.740477+0000","last_became_active":"2023-09-23T20:47:11.618266+0000","last_became_peered":"2023-09-23T20:47:11.618266+0000","last_unstale":"2023-09-23T20:48:04.740477+0000","last_undegraded":"2023-09-23T20:48:04.740477+0000","last_fullsized":"2023-09-23T20:48:04.740477+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:26.978179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:26.978179+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:29:29.000683+0000","scrub_duration":0.0013962829999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"86'130","reported_seq":892,"reported_epoch":572,"state":"active+clean","last_fresh":"2023-09-23T20:47:57.697117+0000","last_change":"2023-09-23T20:25:39.637458+0000","last_active":"2023-09-23T20:47:57.697117+0000","last_peered":"2023-09-23T20:47:57.697117+0000","last_clean":"2023-09-23T20:47:57.697117+0000","last_became_active":"2023-09-23T20:25:39.636773+0000","last_became_peered":"2023-09-23T20:25:39.636773+0000","last_unstale":"2023-09-23T20:47:57.697117+0000","last_undegraded":"2023-09-23T20:47:57.697117+0000","last_fullsized":"2023-09-23T20:47:57.697117+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:00:32.810564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011300049999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":512,"num_read_kb":1820,"num_write":133,"num_write_kb":1347,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":546,"reported_epoch":572,"state":"active+clean","last_fresh":"2023-09-23T20:47:57.693378+0000","last_change":"2023-09-23T20:38:02.591335+0000","last_active":"2023-09-23T20:47:57.693378+0000","last_peered":"2023-09-23T20:47:57.693378+0000","last_clean":"2023-09-23T20:47:57.693378+0000","last_became_active":"2023-09-23T20:38:02.591015+0000","last_became_peered":"2023-09-23T20:38:02.591015+0000","last_unstale":"2023-09-23T20:47:57.693378+0000","last_undegraded":"2023-09-23T20:47:57.693378+0000","last_fullsized":"2023-09-23T20:47:57.693378+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:55:32.272195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":88,"reported_epoch":572,"state":"active+clean","last_fresh":"2023-09-23T20:47:57.697265+0000","last_change":"2023-09-23T20:44:10.555583+0000","last_active":"2023-09-23T20:47:57.697265+0000","last_peered":"2023-09-23T20:47:57.697265+0000","last_clean":"2023-09-23T20:47:57.697265+0000","last_became_active":"2023-09-23T20:44:10.555281+0000","last_became_peered":"2023-09-23T20:44:10.555281+0000","last_unstale":"2023-09-23T20:47:57.697265+0000","last_undegraded":"2023-09-23T20:47:57.697265+0000","last_fullsized":"2023-09-23T20:47:57.697265+0000","mapping_epoch":496,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":497,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:43:11.942147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"574'9368","reported_seq":9419,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-09-23T20:48:10.400752+0000","last_change":"2023-09-23T20:45:39.331096+0000","last_active":"2023-09-23T20:48:10.400752+0000","last_peered":"2023-09-23T20:48:10.400752+0000","last_clean":"2023-09-23T20:48:10.400752+0000","last_became_active":"2023-09-23T20:45:39.330752+0000","last_became_peered":"2023-09-23T20:45:39.330752+0000","last_unstale":"2023-09-23T20:48:10.400752+0000","last_undegraded":"2023-09-23T20:48:10.400752+0000","last_fullsized":"2023-09-23T20:48:10.400752+0000","mapping_epoch":531,"log_start":"537'2971","ondisk_log_start":"537'2971","created":531,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":6397,"log_dups_size":0,"ondisk_log_size":6397,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:39:01.627762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":245104640,"num_objects":3740,"num_object_clones":0,"num_object_copies":11220,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3740,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9368,"num_write_kb":419456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"86'194","reported_seq":931,"reported_epoch":572,"state":"active+clean","last_fresh":"2023-09-23T20:47:57.696245+0000","last_change":"2023-09-23T20:43:16.765865+0000","last_active":"2023-09-23T20:47:57.696245+0000","last_peered":"2023-09-23T20:47:57.696245+0000","last_clean":"2023-09-23T20:47:57.696245+0000","last_became_active":"2023-09-23T20:43:16.765289+0000","last_became_peered":"2023-09-23T20:43:16.765289+0000","last_unstale":"2023-09-23T20:47:57.696245+0000","last_undegraded":"2023-09-23T20:47:57.696245+0000","last_fullsized":"2023-09-23T20:47:57.696245+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":194,"log_dups_size":0,"ondisk_log_size":194,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:15:16.461900+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00037995999999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1024,"num_read_kb":1200,"num_write":173,"num_write_kb":1276,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":1036,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-23T20:48:04.740288+0000","last_change":"2023-09-23T20:38:05.602508+0000","last_active":"2023-09-23T20:48:04.740288+0000","last_peered":"2023-09-23T20:48:04.740288+0000","last_clean":"2023-09-23T20:48:04.740288+0000","last_became_active":"2023-09-23T20:38:05.602129+0000","last_became_peered":"2023-09-23T20:38:05.602129+0000","last_unstale":"2023-09-23T20:48:04.740288+0000","last_undegraded":"2023-09-23T20:48:04.740288+0000","last_fullsized":"2023-09-23T20:48:04.740288+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:45.343742+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:45.343742+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:48.910343+0000","scrub_duration":0.001475802,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":149,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-23T20:48:04.726489+0000","last_change":"2023-09-23T20:43:16.758213+0000","last_active":"2023-09-23T20:48:04.726489+0000","last_peered":"2023-09-23T20:48:04.726489+0000","last_clean":"2023-09-23T20:48:04.726489+0000","last_became_active":"2023-09-23T20:43:16.757869+0000","last_became_peered":"2023-09-23T20:43:16.757869+0000","last_unstale":"2023-09-23T20:48:04.726489+0000","last_undegraded":"2023-09-23T20:48:04.726489+0000","last_fullsized":"2023-09-23T20:48:04.726489+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:43:10.811749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":199,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-23T20:48:04.726013+0000","last_change":"2023-09-23T20:47:09.202302+0000","last_active":"2023-09-23T20:48:04.726013+0000","last_peered":"2023-09-23T20:48:04.726013+0000","last_clean":"2023-09-23T20:48:04.726013+0000","last_became_active":"2023-09-23T20:47:09.201797+0000","last_became_peered":"2023-09-23T20:47:09.201797+0000","last_unstale":"2023-09-23T20:48:04.726013+0000","last_undegraded":"2023-09-23T20:48:04.726013+0000","last_fullsized":"2023-09-23T20:48:04.726013+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:34:41.055705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.8","version":"574'9282","reported_seq":9334,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-09-23T20:48:11.529862+0000","last_change":"2023-09-23T20:45:39.332075+0000","last_active":"2023-09-23T20:48:11.529862+0000","last_peered":"2023-09-23T20:48:11.529862+0000","last_clean":"2023-09-23T20:48:11.529862+0000","last_became_active":"2023-09-23T20:45:39.331755+0000","last_became_peered":"2023-09-23T20:45:39.331755+0000","last_unstale":"2023-09-23T20:48:11.529862+0000","last_undegraded":"2023-09-23T20:48:11.529862+0000","last_fullsized":"2023-09-23T20:48:11.529862+0000","mapping_epoch":531,"log_start":"537'2661","ondisk_log_start":"537'2661","created":531,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":6621,"log_dups_size":0,"ondisk_log_size":6621,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:11:17.895002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":234094592,"num_objects":3572,"num_object_clones":0,"num_object_copies":10716,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3572,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9282,"num_write_kb":411328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":940,"reported_epoch":572,"state":"active+clean","last_fresh":"2023-09-23T20:47:57.694250+0000","last_change":"2023-09-23T20:47:11.622262+0000","last_active":"2023-09-23T20:47:57.694250+0000","last_peered":"2023-09-23T20:47:57.694250+0000","last_clean":"2023-09-23T20:47:57.694250+0000","last_became_active":"2023-09-23T20:47:11.621920+0000","last_became_peered":"2023-09-23T20:47:11.621920+0000","last_unstale":"2023-09-23T20:47:57.694250+0000","last_undegraded":"2023-09-23T20:47:57.694250+0000","last_fullsized":"2023-09-23T20:47:57.694250+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:43.582146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:43.582146+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:01:15.153543+0000","scrub_duration":0.0016219730000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"86'80","reported_seq":826,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-23T20:48:04.740284+0000","last_change":"2023-09-23T20:29:45.536796+0000","last_active":"2023-09-23T20:48:04.740284+0000","last_peered":"2023-09-23T20:48:04.740284+0000","last_clean":"2023-09-23T20:48:04.740284+0000","last_became_active":"2023-09-23T20:29:45.536439+0000","last_became_peered":"2023-09-23T20:29:45.536439+0000","last_unstale":"2023-09-23T20:48:04.740284+0000","last_undegraded":"2023-09-23T20:48:04.740284+0000","last_fullsized":"2023-09-23T20:48:04.740284+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":80,"log_dups_size":0,"ondisk_log_size":80,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:30:57.793132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035115,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":395,"num_read_kb":1370,"num_write":66,"num_write_kb":1061,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":586,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-23T20:48:04.734770+0000","last_change":"2023-09-23T20:47:09.198326+0000","last_active":"2023-09-23T20:48:04.734770+0000","last_peered":"2023-09-23T20:48:04.734770+0000","last_clean":"2023-09-23T20:48:04.734770+0000","last_became_active":"2023-09-23T20:47:09.198059+0000","last_became_peered":"2023-09-23T20:47:09.198059+0000","last_unstale":"2023-09-23T20:48:04.734770+0000","last_undegraded":"2023-09-23T20:48:04.734770+0000","last_fullsized":"2023-09-23T20:48:04.734770+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:38:53.903706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":562,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-23T20:48:04.735217+0000","last_change":"2023-09-23T20:43:46.520386+0000","last_active":"2023-09-23T20:48:04.735217+0000","last_peered":"2023-09-23T20:48:04.735217+0000","last_clean":"2023-09-23T20:48:04.735217+0000","last_became_active":"2023-09-23T20:43:46.519757+0000","last_became_peered":"2023-09-23T20:43:46.519757+0000","last_unstale":"2023-09-23T20:48:04.735217+0000","last_undegraded":"2023-09-23T20:48:04.735217+0000","last_fullsized":"2023-09-23T20:48:04.735217+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:23:14.662734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"574'9182","reported_seq":9859,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-09-23T20:48:10.417185+0000","last_change":"2023-09-23T20:47:13.668055+0000","last_active":"2023-09-23T20:48:10.417185+0000","last_peered":"2023-09-23T20:48:10.417185+0000","last_clean":"2023-09-23T20:48:10.417185+0000","last_became_active":"2023-09-23T20:47:13.626170+0000","last_became_peered":"2023-09-23T20:47:13.626170+0000","last_unstale":"2023-09-23T20:48:10.417185+0000","last_undegraded":"2023-09-23T20:48:10.417185+0000","last_fullsized":"2023-09-23T20:48:10.417185+0000","mapping_epoch":557,"log_start":"537'2757","ondisk_log_start":"537'2757","created":531,"last_epoch_clean":558,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":6425,"log_dups_size":0,"ondisk_log_size":6425,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:32.805547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":243335192,"num_objects":3714,"num_object_clones":0,"num_object_copies":11142,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3714,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9182,"num_write_kb":412609,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":298,"num_bytes_recovered":19529728,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":685,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-23T20:48:04.726131+0000","last_change":"2023-09-23T20:41:08.380084+0000","last_active":"2023-09-23T20:48:04.726131+0000","last_peered":"2023-09-23T20:48:04.726131+0000","last_clean":"2023-09-23T20:48:04.726131+0000","last_became_active":"2023-09-23T20:38:05.610881+0000","last_became_peered":"2023-09-23T20:38:05.610881+0000","last_unstale":"2023-09-23T20:48:04.726131+0000","last_undegraded":"2023-09-23T20:48:04.726131+0000","last_fullsized":"2023-09-23T20:48:04.726131+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:41:08.379959+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:41:08.379959+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:19:39.547923+0000","scrub_duration":0.0019158420000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"86'102","reported_seq":1244,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-23T20:48:04.740611+0000","last_change":"2023-09-23T20:47:14.628795+0000","last_active":"2023-09-23T20:48:04.740611+0000","last_peered":"2023-09-23T20:48:04.740611+0000","last_clean":"2023-09-23T20:48:04.740611+0000","last_became_active":"2023-09-23T20:47:14.628341+0000","last_became_peered":"2023-09-23T20:47:14.628341+0000","last_unstale":"2023-09-23T20:48:04.740611+0000","last_undegraded":"2023-09-23T20:48:04.740611+0000","last_fullsized":"2023-09-23T20:48:04.740611+0000","mapping_epoch":558,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":559,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":102,"log_dups_size":0,"ondisk_log_size":102,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:42:16.291849+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030656500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":963,"num_read_kb":1234,"num_write":60,"num_write_kb":225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":521,"reported_epoch":572,"state":"active+clean","last_fresh":"2023-09-23T20:47:57.693947+0000","last_change":"2023-09-23T20:40:19.544384+0000","last_active":"2023-09-23T20:47:57.693947+0000","last_peered":"2023-09-23T20:47:57.693947+0000","last_clean":"2023-09-23T20:47:57.693947+0000","last_became_active":"2023-09-23T20:40:19.544079+0000","last_became_peered":"2023-09-23T20:40:19.544079+0000","last_unstale":"2023-09-23T20:47:57.693947+0000","last_undegraded":"2023-09-23T20:47:57.693947+0000","last_fullsized":"2023-09-23T20:47:57.693947+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:00:19.498631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":518,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-23T20:48:04.745324+0000","last_change":"2023-09-23T20:43:46.541173+0000","last_active":"2023-09-23T20:48:04.745324+0000","last_peered":"2023-09-23T20:48:04.745324+0000","last_clean":"2023-09-23T20:48:04.745324+0000","last_became_active":"2023-09-23T20:43:46.540726+0000","last_became_peered":"2023-09-23T20:43:46.540726+0000","last_unstale":"2023-09-23T20:48:04.745324+0000","last_undegraded":"2023-09-23T20:48:04.745324+0000","last_fullsized":"2023-09-23T20:48:04.745324+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:49:04.137885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.e","version":"574'9309","reported_seq":9483,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-09-23T20:48:10.768740+0000","last_change":"2023-09-23T20:47:10.174122+0000","last_active":"2023-09-23T20:48:10.768740+0000","last_peered":"2023-09-23T20:48:10.768740+0000","last_clean":"2023-09-23T20:48:10.768740+0000","last_became_active":"2023-09-23T20:47:10.167795+0000","last_became_peered":"2023-09-23T20:47:10.167795+0000","last_unstale":"2023-09-23T20:48:10.768740+0000","last_undegraded":"2023-09-23T20:48:10.768740+0000","last_fullsized":"2023-09-23T20:48:10.768740+0000","mapping_epoch":553,"log_start":"538'3444","ondisk_log_start":"538'3444","created":531,"last_epoch_clean":554,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":5865,"log_dups_size":0,"ondisk_log_size":5865,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:44:03.693340+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":241762304,"num_objects":3689,"num_object_clones":0,"num_object_copies":11067,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3689,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9309,"num_write_kb":415936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":50,"num_bytes_recovered":3276800,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":571,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-23T20:48:04.738672+0000","last_change":"2023-09-23T20:47:34.561810+0000","last_active":"2023-09-23T20:48:04.738672+0000","last_peered":"2023-09-23T20:48:04.738672+0000","last_clean":"2023-09-23T20:48:04.738672+0000","last_became_active":"2023-09-23T20:47:11.624953+0000","last_became_peered":"2023-09-23T20:47:11.624953+0000","last_unstale":"2023-09-23T20:48:04.738672+0000","last_undegraded":"2023-09-23T20:48:04.738672+0000","last_fullsized":"2023-09-23T20:48:04.738672+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:47:34.561747+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:47:34.561747+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:11:25.458764+0000","scrub_duration":0.0087949929999999992,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":311,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-23T20:48:04.741759+0000","last_change":"2023-09-23T20:32:50.549695+0000","last_active":"2023-09-23T20:48:04.741759+0000","last_peered":"2023-09-23T20:48:04.741759+0000","last_clean":"2023-09-23T20:48:04.741759+0000","last_became_active":"2023-09-23T20:32:50.549431+0000","last_became_peered":"2023-09-23T20:32:50.549431+0000","last_unstale":"2023-09-23T20:48:04.741759+0000","last_undegraded":"2023-09-23T20:48:04.741759+0000","last_fullsized":"2023-09-23T20:48:04.741759+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:45:53.666569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"86'128","reported_seq":1053,"reported_epoch":572,"state":"active+clean","last_fresh":"2023-09-23T20:47:57.694891+0000","last_change":"2023-09-23T20:47:11.621846+0000","last_active":"2023-09-23T20:47:57.694891+0000","last_peered":"2023-09-23T20:47:57.694891+0000","last_clean":"2023-09-23T20:47:57.694891+0000","last_became_active":"2023-09-23T20:47:11.621284+0000","last_became_peered":"2023-09-23T20:47:11.621284+0000","last_unstale":"2023-09-23T20:47:57.694891+0000","last_undegraded":"2023-09-23T20:47:57.694891+0000","last_fullsized":"2023-09-23T20:47:57.694891+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:43:50.658512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000293202,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":579,"num_read_kb":1473,"num_write":96,"num_write_kb":1062,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":355,"reported_epoch":572,"state":"active+clean","last_fresh":"2023-09-23T20:47:57.707967+0000","last_change":"2023-09-23T20:37:43.582105+0000","last_active":"2023-09-23T20:47:57.707967+0000","last_peered":"2023-09-23T20:47:57.707967+0000","last_clean":"2023-09-23T20:47:57.707967+0000","last_became_active":"2023-09-23T20:37:43.581860+0000","last_became_peered":"2023-09-23T20:37:43.581860+0000","last_unstale":"2023-09-23T20:47:57.707967+0000","last_undegraded":"2023-09-23T20:47:57.707967+0000","last_fullsized":"2023-09-23T20:47:57.707967+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:18:38.765486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"574'9270","reported_seq":9321,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-09-23T20:48:10.373163+0000","last_change":"2023-09-23T20:45:39.381283+0000","last_active":"2023-09-23T20:48:10.373163+0000","last_peered":"2023-09-23T20:48:10.373163+0000","last_clean":"2023-09-23T20:48:10.373163+0000","last_became_active":"2023-09-23T20:45:39.380945+0000","last_became_peered":"2023-09-23T20:45:39.380945+0000","last_unstale":"2023-09-23T20:48:10.373163+0000","last_undegraded":"2023-09-23T20:48:10.373163+0000","last_fullsized":"2023-09-23T20:48:10.373163+0000","mapping_epoch":531,"log_start":"537'2873","ondisk_log_start":"537'2873","created":531,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":6397,"log_dups_size":0,"ondisk_log_size":6397,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:04:01.529754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":244842496,"num_objects":3736,"num_object_clones":0,"num_object_copies":11208,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3736,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9270,"num_write_kb":416192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.4","version":"90'161","reported_seq":789,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-23T20:48:04.738521+0000","last_change":"2023-09-23T20:40:22.427435+0000","last_active":"2023-09-23T20:48:04.738521+0000","last_peered":"2023-09-23T20:48:04.738521+0000","last_clean":"2023-09-23T20:48:04.738521+0000","last_became_active":"2023-09-23T20:40:22.427051+0000","last_became_peered":"2023-09-23T20:40:22.427051+0000","last_unstale":"2023-09-23T20:48:04.738521+0000","last_undegraded":"2023-09-23T20:48:04.738521+0000","last_fullsized":"2023-09-23T20:48:04.738521+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:34:54.504554+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:34:54.504554+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:42.001652+0000","scrub_duration":0.011176864,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"86'117","reported_seq":882,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-23T20:48:04.726364+0000","last_change":"2023-09-23T20:44:18.635224+0000","last_active":"2023-09-23T20:48:04.726364+0000","last_peered":"2023-09-23T20:48:04.726364+0000","last_clean":"2023-09-23T20:48:04.726364+0000","last_became_active":"2023-09-23T20:44:18.634703+0000","last_became_peered":"2023-09-23T20:44:18.634703+0000","last_unstale":"2023-09-23T20:48:04.726364+0000","last_undegraded":"2023-09-23T20:48:04.726364+0000","last_fullsized":"2023-09-23T20:48:04.726364+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:59:38.030626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046891400000000002,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":893,"num_read_kb":922,"num_write":82,"num_write_kb":4193,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":16777216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":588,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-23T20:48:04.735297+0000","last_change":"2023-09-23T20:47:09.317116+0000","last_active":"2023-09-23T20:48:04.735297+0000","last_peered":"2023-09-23T20:48:04.735297+0000","last_clean":"2023-09-23T20:48:04.735297+0000","last_became_active":"2023-09-23T20:47:09.205930+0000","last_became_peered":"2023-09-23T20:47:09.205930+0000","last_unstale":"2023-09-23T20:48:04.735297+0000","last_undegraded":"2023-09-23T20:48:04.735297+0000","last_fullsized":"2023-09-23T20:48:04.735297+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:04:55.528625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":31,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-23T20:48:04.737608+0000","last_change":"2023-09-23T20:47:09.192117+0000","last_active":"2023-09-23T20:48:04.737608+0000","last_peered":"2023-09-23T20:48:04.737608+0000","last_clean":"2023-09-23T20:48:04.737608+0000","last_became_active":"2023-09-23T20:47:09.191820+0000","last_became_peered":"2023-09-23T20:47:09.191820+0000","last_unstale":"2023-09-23T20:48:04.737608+0000","last_undegraded":"2023-09-23T20:48:04.737608+0000","last_fullsized":"2023-09-23T20:48:04.737608+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:08.785173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"90'161","reported_seq":831,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-23T20:48:04.734506+0000","last_change":"2023-09-23T20:43:46.544750+0000","last_active":"2023-09-23T20:48:04.734506+0000","last_peered":"2023-09-23T20:48:04.734506+0000","last_clean":"2023-09-23T20:48:04.734506+0000","last_became_active":"2023-09-23T20:43:46.544395+0000","last_became_peered":"2023-09-23T20:43:46.544395+0000","last_unstale":"2023-09-23T20:48:04.734506+0000","last_undegraded":"2023-09-23T20:48:04.734506+0000","last_fullsized":"2023-09-23T20:48:04.734506+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:06.487351+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:06.487351+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:56.866168+0000","scrub_duration":0.001964379,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":547,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-23T20:48:04.745252+0000","last_change":"2023-09-23T20:47:09.197601+0000","last_active":"2023-09-23T20:48:04.745252+0000","last_peered":"2023-09-23T20:48:04.745252+0000","last_clean":"2023-09-23T20:48:04.745252+0000","last_became_active":"2023-09-23T20:47:09.196967+0000","last_became_peered":"2023-09-23T20:47:09.196967+0000","last_unstale":"2023-09-23T20:48:04.745252+0000","last_undegraded":"2023-09-23T20:48:04.745252+0000","last_fullsized":"2023-09-23T20:48:04.745252+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:06:56.826785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":430,"reported_epoch":572,"state":"active+clean","last_fresh":"2023-09-23T20:47:57.696293+0000","last_change":"2023-09-23T20:29:45.538314+0000","last_active":"2023-09-23T20:47:57.696293+0000","last_peered":"2023-09-23T20:47:57.696293+0000","last_clean":"2023-09-23T20:47:57.696293+0000","last_became_active":"2023-09-23T20:29:45.536847+0000","last_became_peered":"2023-09-23T20:29:45.536847+0000","last_unstale":"2023-09-23T20:47:57.696293+0000","last_undegraded":"2023-09-23T20:47:57.696293+0000","last_fullsized":"2023-09-23T20:47:57.696293+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:51.948721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"90'161","reported_seq":648,"reported_epoch":572,"state":"active+clean","last_fresh":"2023-09-23T20:47:57.710272+0000","last_change":"2023-09-23T20:41:06.499511+0000","last_active":"2023-09-23T20:47:57.710272+0000","last_peered":"2023-09-23T20:47:57.710272+0000","last_clean":"2023-09-23T20:47:57.710272+0000","last_became_active":"2023-09-23T20:38:05.601874+0000","last_became_peered":"2023-09-23T20:38:05.601874+0000","last_unstale":"2023-09-23T20:47:57.710272+0000","last_undegraded":"2023-09-23T20:47:57.710272+0000","last_fullsized":"2023-09-23T20:47:57.710272+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:41:06.499396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:41:06.499396+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:31:34.059077+0000","scrub_duration":0.0019670310000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":35,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-23T20:48:04.738652+0000","last_change":"2023-09-23T20:47:09.332980+0000","last_active":"2023-09-23T20:48:04.738652+0000","last_peered":"2023-09-23T20:48:04.738652+0000","last_clean":"2023-09-23T20:48:04.738652+0000","last_became_active":"2023-09-23T20:47:09.230554+0000","last_became_peered":"2023-09-23T20:47:09.230554+0000","last_unstale":"2023-09-23T20:48:04.738652+0000","last_undegraded":"2023-09-23T20:48:04.738652+0000","last_fullsized":"2023-09-23T20:48:04.738652+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:06:21.882823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":31,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-23T20:48:04.738799+0000","last_change":"2023-09-23T20:47:09.186579+0000","last_active":"2023-09-23T20:48:04.738799+0000","last_peered":"2023-09-23T20:48:04.738799+0000","last_clean":"2023-09-23T20:48:04.738799+0000","last_became_active":"2023-09-23T20:47:09.186227+0000","last_became_peered":"2023-09-23T20:47:09.186227+0000","last_unstale":"2023-09-23T20:48:04.738799+0000","last_undegraded":"2023-09-23T20:48:04.738799+0000","last_fullsized":"2023-09-23T20:48:04.738799+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:45:39.485308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":35,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-23T20:48:04.738588+0000","last_change":"2023-09-23T20:47:09.316824+0000","last_active":"2023-09-23T20:48:04.738588+0000","last_peered":"2023-09-23T20:48:04.738588+0000","last_clean":"2023-09-23T20:48:04.738588+0000","last_became_active":"2023-09-23T20:47:09.195100+0000","last_became_peered":"2023-09-23T20:47:09.195100+0000","last_unstale":"2023-09-23T20:48:04.738588+0000","last_undegraded":"2023-09-23T20:48:04.738588+0000","last_fullsized":"2023-09-23T20:48:04.738588+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:29:53.273554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":31,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-23T20:48:04.737192+0000","last_change":"2023-09-23T20:47:09.181240+0000","last_active":"2023-09-23T20:48:04.737192+0000","last_peered":"2023-09-23T20:48:04.737192+0000","last_clean":"2023-09-23T20:48:04.737192+0000","last_became_active":"2023-09-23T20:47:09.181022+0000","last_became_peered":"2023-09-23T20:47:09.181022+0000","last_unstale":"2023-09-23T20:48:04.737192+0000","last_undegraded":"2023-09-23T20:48:04.737192+0000","last_fullsized":"2023-09-23T20:48:04.737192+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:38:59.191781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"90'161","reported_seq":829,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-23T20:48:04.740812+0000","last_change":"2023-09-23T20:38:05.591992+0000","last_active":"2023-09-23T20:48:04.740812+0000","last_peered":"2023-09-23T20:48:04.740812+0000","last_clean":"2023-09-23T20:48:04.740812+0000","last_became_active":"2023-09-23T20:38:05.591701+0000","last_became_peered":"2023-09-23T20:38:05.591701+0000","last_unstale":"2023-09-23T20:48:04.740812+0000","last_undegraded":"2023-09-23T20:48:04.740812+0000","last_fullsized":"2023-09-23T20:48:04.740812+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:02.384133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:02.384133+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:41.722542+0000","scrub_duration":0.001769098,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":397,"reported_epoch":572,"state":"active+clean","last_fresh":"2023-09-23T20:47:57.716850+0000","last_change":"2023-09-23T20:38:02.542719+0000","last_active":"2023-09-23T20:47:57.716850+0000","last_peered":"2023-09-23T20:47:57.716850+0000","last_clean":"2023-09-23T20:47:57.716850+0000","last_became_active":"2023-09-23T20:38:02.542385+0000","last_became_peered":"2023-09-23T20:38:02.542385+0000","last_unstale":"2023-09-23T20:47:57.716850+0000","last_undegraded":"2023-09-23T20:47:57.716850+0000","last_fullsized":"2023-09-23T20:47:57.716850+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:36:23.673757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":363,"reported_epoch":572,"state":"active+clean","last_fresh":"2023-09-23T20:47:57.708376+0000","last_change":"2023-09-23T20:37:43.584253+0000","last_active":"2023-09-23T20:47:57.708376+0000","last_peered":"2023-09-23T20:47:57.708376+0000","last_clean":"2023-09-23T20:47:57.708376+0000","last_became_active":"2023-09-23T20:37:43.580069+0000","last_became_peered":"2023-09-23T20:37:43.580069+0000","last_unstale":"2023-09-23T20:47:57.708376+0000","last_undegraded":"2023-09-23T20:47:57.708376+0000","last_fullsized":"2023-09-23T20:47:57.708376+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:39:11.482581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"90'161","reported_seq":702,"reported_epoch":572,"state":"active+clean","last_fresh":"2023-09-23T20:47:58.652367+0000","last_change":"2023-09-23T20:47:11.621758+0000","last_active":"2023-09-23T20:47:58.652367+0000","last_peered":"2023-09-23T20:47:58.652367+0000","last_clean":"2023-09-23T20:47:58.652367+0000","last_became_active":"2023-09-23T20:47:11.621345+0000","last_became_peered":"2023-09-23T20:47:11.621345+0000","last_unstale":"2023-09-23T20:47:58.652367+0000","last_undegraded":"2023-09-23T20:47:58.652367+0000","last_fullsized":"2023-09-23T20:47:58.652367+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:01.504973+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:01.504973+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:31:17.112393+0000","scrub_duration":0.00202344,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":608,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-23T20:48:04.743425+0000","last_change":"2023-09-23T20:43:46.511657+0000","last_active":"2023-09-23T20:48:04.743425+0000","last_peered":"2023-09-23T20:48:04.743425+0000","last_clean":"2023-09-23T20:48:04.743425+0000","last_became_active":"2023-09-23T20:43:46.511326+0000","last_became_peered":"2023-09-23T20:43:46.511326+0000","last_unstale":"2023-09-23T20:48:04.743425+0000","last_undegraded":"2023-09-23T20:48:04.743425+0000","last_fullsized":"2023-09-23T20:48:04.743425+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:36:46.139261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3324,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":31,"reported_epoch":573,"state":"active+clean","last_fresh":"2023-09-23T20:48:04.738645+0000","last_change":"2023-09-23T20:47:09.183755+0000","last_active":"2023-09-23T20:48:04.738645+0000","last_peered":"2023-09-23T20:48:04.738645+0000","last_clean":"2023-09-23T20:48:04.738645+0000","last_became_active":"2023-09-23T20:47:09.183448+0000","last_became_peered":"2023-09-23T20:47:09.183448+0000","last_unstale":"2023-09-23T20:48:04.738645+0000","last_undegraded":"2023-09-23T20:48:04.738645+0000","last_fullsized":"2023-09-23T20:48:04.738645+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:18:38.303313+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":3942187032,"num_objects":60154,"num_object_clones":0,"num_object_copies":180462,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":60154,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":148834,"num_write_kb":6687553,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2253,"num_bytes_recovered":147652608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":11811696640,"data_stored":11811684424,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":109704,"ondisk_log_size":109704,"up":48,"acting":48,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":101,"num_bytes_recovered":14044,"num_keys_recovered":60,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26691,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5983,"num_read_kb":13200,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":114,"num_bytes_recovered":16777216,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6082,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201721,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1542484,"kb_used_data":975948,"kb_used_omap":0,"kb_used_meta":566528,"kb_avail":92202668,"statfs":{"total":95995035648,"available":94415532032,"internally_reserved":0,"allocated":999370752,"data_stored":997799631,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":580124672},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:48:06 2023","interfaces":[{"interface":"back","average":{"1min":1.5569999999999999,"5min":1.972,"15min":2.1899999999999999},"min":{"1min":0.224,"5min":0.224,"15min":0.224},"max":{"1min":5.6970000000000001,"5min":15.1,"15min":18.315000000000001},"last":1.4930000000000001},{"interface":"front","average":{"1min":2.1539999999999999,"5min":2.1379999999999999,"15min":1.903},"min":{"1min":0.28000000000000003,"5min":0.25800000000000001,"15min":0.23899999999999999},"max":{"1min":6.5780000000000003,"5min":15.327999999999999,"15min":19.355},"last":1.734}]},{"osd":1,"last update":"Sat Sep 23 20:47:11 2023","interfaces":[{"interface":"back","average":{"1min":2.3580000000000001,"5min":1.7809999999999999,"15min":1.9750000000000001},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.24399999999999999},"max":{"1min":14.249000000000001,"5min":15.131,"15min":49.965000000000003},"last":1.4299999999999999},{"interface":"front","average":{"1min":2.5430000000000001,"5min":2.7240000000000002,"15min":2.4350000000000001},"min":{"1min":0.30199999999999999,"5min":0.26700000000000002,"15min":0.248},"max":{"1min":10.471,"5min":15.18,"15min":53.451999999999998},"last":0.79500000000000004}]},{"osd":2,"last update":"Sat Sep 23 20:48:06 2023","interfaces":[{"interface":"back","average":{"1min":2.0779999999999998,"5min":2.153,"15min":2.1899999999999999},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.27800000000000002},"max":{"1min":5.4770000000000003,"5min":15.087,"15min":16.965},"last":0.82799999999999996},{"interface":"front","average":{"1min":2.206,"5min":2.0249999999999999,"15min":2.339},"min":{"1min":0.45200000000000001,"5min":0.31,"15min":0.27400000000000002},"max":{"1min":6.6379999999999999,"5min":15.193,"15min":23.494},"last":0.68200000000000005}]},{"osd":3,"last update":"Sat Sep 23 20:47:35 2023","interfaces":[{"interface":"back","average":{"1min":2.153,"5min":1.841,"15min":2.2589999999999999},"min":{"1min":0.33800000000000002,"5min":0.32500000000000001,"15min":0.311},"max":{"1min":6.734,"5min":15.115,"15min":23.352},"last":0.57499999999999996},{"interface":"front","average":{"1min":2.2040000000000002,"5min":2.2589999999999999,"15min":2.4550000000000001},"min":{"1min":0.443,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":6.5369999999999999,"5min":15.144,"15min":20.969999999999999},"last":0.877}]},{"osd":4,"last update":"Sat Sep 23 20:47:35 2023","interfaces":[{"interface":"back","average":{"1min":1.264,"5min":1.496,"15min":1.1739999999999999},"min":{"1min":0.27900000000000003,"5min":0.254,"15min":0.23200000000000001},"max":{"1min":6.548,"5min":12.260999999999999,"15min":18.364999999999998},"last":1.4059999999999999},{"interface":"front","average":{"1min":1.6160000000000001,"5min":1.8300000000000001,"15min":1.4350000000000001},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":6.0419999999999998,"5min":7.2469999999999999,"15min":23.617000000000001},"last":0.85599999999999998}]},{"osd":5,"last update":"Sat Sep 23 20:47:35 2023","interfaces":[{"interface":"back","average":{"1min":1.101,"5min":1.585,"15min":1.2190000000000001},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.26600000000000001},"max":{"1min":6.367,"5min":15.005000000000001,"15min":15.005000000000001},"last":0.48399999999999999},{"interface":"front","average":{"1min":1.504,"5min":1.9379999999999999,"15min":1.3480000000000001},"min":{"1min":0.36099999999999999,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":6.6280000000000001,"5min":12.731,"15min":18.390000000000001},"last":1.373}]},{"osd":6,"last update":"Sat Sep 23 20:47:35 2023","interfaces":[{"interface":"back","average":{"1min":1.4730000000000001,"5min":2.0950000000000002,"15min":1.47},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":6.4720000000000004,"5min":15.038,"15min":18.417000000000002},"last":5.0060000000000002},{"interface":"front","average":{"1min":1.861,"5min":1.9910000000000001,"15min":1.4139999999999999},"min":{"1min":0.35899999999999999,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":6.0910000000000002,"5min":15.369999999999999,"15min":15.369999999999999},"last":0.997}]}]},{"osd":6,"up_from":47,"seq":201863463360,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2174220,"kb_used_data":1676804,"kb_used_omap":0,"kb_used_meta":497408,"kb_avail":91570932,"statfs":{"total":95995035648,"available":93768634368,"internally_reserved":0,"allocated":1717047296,"data_stored":1715483720,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":509345792},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,0,3],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:48:03 2023","interfaces":[{"interface":"back","average":{"1min":2.9780000000000002,"5min":2.0299999999999998,"15min":1.9419999999999999},"min":{"1min":0.25,"5min":0.219,"15min":0.219},"max":{"1min":15.574999999999999,"5min":21.018000000000001,"15min":34.143999999999998},"last":4.1360000000000001},{"interface":"front","average":{"1min":1.698,"5min":2.1909999999999998,"15min":2.2429999999999999},"min":{"1min":0.255,"5min":0.255,"15min":0.22700000000000001},"max":{"1min":4.8840000000000003,"5min":21.082000000000001,"15min":60.328000000000003},"last":11.557}]},{"osd":1,"last update":"Sat Sep 23 20:47:19 2023","interfaces":[{"interface":"back","average":{"1min":1.9039999999999999,"5min":2.0070000000000001,"15min":1.9610000000000001},"min":{"1min":0.34499999999999997,"5min":0.26400000000000001,"15min":0.249},"max":{"1min":5.5640000000000001,"5min":9.2520000000000007,"15min":64.274000000000001},"last":0.26300000000000001},{"interface":"front","average":{"1min":1.198,"5min":1.8520000000000001,"15min":1.782},"min":{"1min":0.32200000000000001,"5min":0.29099999999999998,"15min":0.28000000000000003},"max":{"1min":8.8070000000000004,"5min":17.786000000000001,"15min":34.490000000000002},"last":3.8849999999999998}]},{"osd":2,"last update":"Sat Sep 23 20:48:03 2023","interfaces":[{"interface":"back","average":{"1min":2.4609999999999999,"5min":2.4380000000000002,"15min":2.242},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.27700000000000002},"max":{"1min":6.1529999999999996,"5min":22.591000000000001,"15min":34.317999999999998},"last":7.6779999999999999},{"interface":"front","average":{"1min":2.4039999999999999,"5min":2.0350000000000001,"15min":2.0539999999999998},"min":{"1min":0.45400000000000001,"5min":0.33900000000000002,"15min":0.30399999999999999},"max":{"1min":7.8710000000000004,"5min":22.614999999999998,"15min":34.292999999999999},"last":5.2619999999999996}]},{"osd":3,"last update":"Sat Sep 23 20:47:54 2023","interfaces":[{"interface":"back","average":{"1min":2.0659999999999998,"5min":1.946,"15min":1.9339999999999999},"min":{"1min":0.307,"5min":0.307,"15min":0.28399999999999997},"max":{"1min":9.6859999999999999,"5min":11.664,"15min":11.664},"last":4.0940000000000003},{"interface":"front","average":{"1min":2.2959999999999998,"5min":2.1859999999999999,"15min":2.1539999999999999},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.30499999999999999},"max":{"1min":9.7810000000000006,"5min":10.125999999999999,"15min":12.875999999999999},"last":11.6}]},{"osd":4,"last update":"Sat Sep 23 20:47:19 2023","interfaces":[{"interface":"back","average":{"1min":1.0529999999999999,"5min":1.0700000000000001,"15min":0.85099999999999998},"min":{"1min":0.28699999999999998,"5min":0.25600000000000001,"15min":0.253},"max":{"1min":10.183999999999999,"5min":10.183999999999999,"15min":23.904},"last":0.33900000000000002},{"interface":"front","average":{"1min":1.6639999999999999,"5min":1.208,"15min":1.0920000000000001},"min":{"1min":0.32500000000000001,"5min":0.28399999999999997,"15min":0.28199999999999997},"max":{"1min":7.2300000000000004,"5min":7.2300000000000004,"15min":11.515000000000001},"last":0.40699999999999997}]},{"osd":5,"last update":"Sat Sep 23 20:47:19 2023","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":1.0569999999999999,"15min":0.88500000000000001},"min":{"1min":0.30499999999999999,"5min":0.253,"15min":0.253},"max":{"1min":1.8979999999999999,"5min":7.8529999999999998,"15min":7.8529999999999998},"last":0.38700000000000001},{"interface":"front","average":{"1min":2.3290000000000002,"5min":1.6970000000000001,"15min":1.222},"min":{"1min":0.41999999999999998,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":9.3559999999999999,"5min":9.3559999999999999,"15min":9.3559999999999999},"last":0.45000000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:47:54 2023","interfaces":[{"interface":"back","average":{"1min":1.421,"5min":1.331,"15min":0.98499999999999999},"min":{"1min":0.35199999999999998,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":7.1379999999999999,"5min":8.9309999999999992,"15min":8.9309999999999992},"last":0.497},{"interface":"front","average":{"1min":2.0459999999999998,"5min":2.0150000000000001,"15min":1.3939999999999999},"min":{"1min":0.434,"5min":0.35999999999999999,"15min":0.34499999999999997},"max":{"1min":9.7530000000000001,"5min":9.8680000000000003,"15min":9.8680000000000003},"last":0.47099999999999997}]}]},{"osd":1,"up_from":266,"seq":1142461300958,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1494860,"kb_used_data":748420,"kb_used_omap":8,"kb_used_meta":746423,"kb_avail":92250292,"statfs":{"total":95995035648,"available":94464299008,"internally_reserved":0,"allocated":766382080,"data_stored":763533959,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8206,"internal_metadata":764338162},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:47:23 2023","interfaces":[{"interface":"back","average":{"1min":1.248,"5min":2.3530000000000002,"15min":2.46},"min":{"1min":0.252,"5min":0.20599999999999999,"15min":0.20000000000000001},"max":{"1min":4.3399999999999999,"5min":15.532999999999999,"15min":52.689999999999998},"last":1.768},{"interface":"front","average":{"1min":2.2999999999999998,"5min":2.7250000000000001,"15min":3.0350000000000001},"min":{"1min":0.27900000000000003,"5min":0.22600000000000001,"15min":0.19900000000000001},"max":{"1min":10.061,"5min":20.346,"15min":52.662999999999997},"last":0.78600000000000003}]},{"osd":2,"last update":"Sat Sep 23 20:47:23 2023","interfaces":[{"interface":"back","average":{"1min":1.647,"5min":2.75,"15min":3.0230000000000001},"min":{"1min":0.24099999999999999,"5min":0.23499999999999999,"15min":0.20899999999999999},"max":{"1min":7.8719999999999999,"5min":19.026,"15min":32.255000000000003},"last":0.25700000000000001},{"interface":"front","average":{"1min":1.8420000000000001,"5min":2.194,"15min":2.262},"min":{"1min":0.29999999999999999,"5min":0.26000000000000001,"15min":0.20899999999999999},"max":{"1min":9.9440000000000008,"5min":10.041,"15min":13.404999999999999},"last":0.497}]},{"osd":3,"last update":"Sat Sep 23 20:47:23 2023","interfaces":[{"interface":"back","average":{"1min":1.958,"5min":2.1629999999999998,"15min":2.8050000000000002},"min":{"1min":0.33900000000000002,"5min":0.28799999999999998,"15min":0.247},"max":{"1min":17.366,"5min":17.366,"15min":80.745999999999995},"last":2.859},{"interface":"front","average":{"1min":2.4769999999999999,"5min":2.7480000000000002,"15min":3.093},"min":{"1min":0.30399999999999999,"5min":0.22900000000000001,"15min":0.22700000000000001},"max":{"1min":10.502000000000001,"5min":11.534000000000001,"15min":82.988},"last":1.0129999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:47:23 2023","interfaces":[{"interface":"back","average":{"1min":1.7490000000000001,"5min":2.6320000000000001,"15min":2.2930000000000001},"min":{"1min":0.34000000000000002,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":8.5440000000000005,"5min":20.634,"15min":24.327999999999999},"last":0.56499999999999995},{"interface":"front","average":{"1min":1.476,"5min":1.8879999999999999,"15min":1.6890000000000001},"min":{"1min":0.35699999999999998,"5min":0.34699999999999998,"15min":0.311},"max":{"1min":4.2939999999999996,"5min":20.835999999999999,"15min":20.835999999999999},"last":0.47199999999999998}]},{"osd":5,"last update":"Sat Sep 23 20:47:23 2023","interfaces":[{"interface":"back","average":{"1min":2.3919999999999999,"5min":2.4630000000000001,"15min":2.2669999999999999},"min":{"1min":0.39100000000000001,"5min":0.308,"15min":0.308},"max":{"1min":9.9779999999999998,"5min":14.366,"15min":33.401000000000003},"last":0.98199999999999998},{"interface":"front","average":{"1min":2.0710000000000002,"5min":2.7130000000000001,"15min":2.5310000000000001},"min":{"1min":0.377,"5min":0.33100000000000002,"15min":0.32500000000000001},"max":{"1min":9.6739999999999995,"5min":11.750999999999999,"15min":33.417000000000002},"last":0.42799999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:47:23 2023","interfaces":[{"interface":"back","average":{"1min":2.2370000000000001,"5min":2.4550000000000001,"15min":2.1579999999999999},"min":{"1min":0.38600000000000001,"5min":0.316,"15min":0.316},"max":{"1min":9.9339999999999993,"5min":14.414999999999999,"15min":14.414999999999999},"last":0.94699999999999995},{"interface":"front","average":{"1min":2.4390000000000001,"5min":3.0329999999999999,"15min":2.528},"min":{"1min":0.46300000000000002,"5min":0.38500000000000001,"15min":0.317},"max":{"1min":8.4969999999999999,"5min":11.832000000000001,"15min":24.376000000000001},"last":0.61899999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:47:23 2023","interfaces":[{"interface":"back","average":{"1min":2.5830000000000002,"5min":2.5179999999999998,"15min":2.3079999999999998},"min":{"1min":0.48799999999999999,"5min":0.37,"15min":0.37},"max":{"1min":10.010999999999999,"5min":10.010999999999999,"15min":18.437000000000001},"last":0.90900000000000003},{"interface":"front","average":{"1min":2.2189999999999999,"5min":2.9550000000000001,"15min":2.552},"min":{"1min":0.49099999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":8.0540000000000003,"5min":11.864000000000001,"15min":24.524000000000001},"last":0.59799999999999998}]}]},{"osd":0,"up_from":94,"seq":403726926161,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3280740,"kb_used_data":2365916,"kb_used_omap":7,"kb_used_meta":914808,"kb_avail":90464412,"statfs":{"total":95995035648,"available":92635557888,"internally_reserved":0,"allocated":2422697984,"data_stored":2420010523,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":936763409},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,0,2,1,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:47:30 2023","interfaces":[{"interface":"back","average":{"1min":8.9149999999999991,"5min":3.4870000000000001,"15min":2.9049999999999998},"min":{"1min":0.20100000000000001,"5min":0.184,"15min":0.17199999999999999},"max":{"1min":152.387,"5min":152.387,"15min":152.387},"last":2.609},{"interface":"front","average":{"1min":8.7140000000000004,"5min":3.8149999999999999,"15min":3.0920000000000001},"min":{"1min":0.23899999999999999,"5min":0.23799999999999999,"15min":0.20399999999999999},"max":{"1min":152.84100000000001,"5min":152.84100000000001,"15min":152.84100000000001},"last":2.2250000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:48:01 2023","interfaces":[{"interface":"back","average":{"1min":4.4219999999999997,"5min":2.9350000000000001,"15min":2.9950000000000001},"min":{"1min":0.254,"5min":0.23499999999999999,"15min":0.193},"max":{"1min":14.538,"5min":15.108000000000001,"15min":62.726999999999997},"last":1.48},{"interface":"front","average":{"1min":2.8250000000000002,"5min":2.7149999999999999,"15min":2.9990000000000001},"min":{"1min":0.27100000000000002,"5min":0.217,"15min":0.217},"max":{"1min":8.3689999999999998,"5min":17.013000000000002,"15min":62.689},"last":9.5609999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:47:39 2023","interfaces":[{"interface":"back","average":{"1min":2.9119999999999999,"5min":3.6099999999999999,"15min":3.165},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":13.367000000000001,"5min":55.017000000000003,"15min":55.017000000000003},"last":6.4550000000000001},{"interface":"front","average":{"1min":2.5760000000000001,"5min":2.9900000000000002,"15min":3.1989999999999998},"min":{"1min":0.32000000000000001,"5min":0.23799999999999999,"15min":0.21199999999999999},"max":{"1min":12.051,"5min":55.665999999999997,"15min":55.665999999999997},"last":2.1960000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:48:01 2023","interfaces":[{"interface":"back","average":{"1min":2.9860000000000002,"5min":2.0139999999999998,"15min":2.1299999999999999},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.25900000000000001},"max":{"1min":11.75,"5min":11.75,"15min":12.587999999999999},"last":6.4109999999999996},{"interface":"front","average":{"1min":3.609,"5min":2.125,"15min":2.093},"min":{"1min":0.31,"5min":0.28899999999999998,"15min":0.27400000000000002},"max":{"1min":12.356999999999999,"5min":12.356999999999999,"15min":23.001000000000001},"last":2.2770000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:48:01 2023","interfaces":[{"interface":"back","average":{"1min":3.4590000000000001,"5min":2.097,"15min":2.169},"min":{"1min":0.34300000000000003,"5min":0.30599999999999999,"15min":0.26500000000000001},"max":{"1min":9.3339999999999996,"5min":9.3339999999999996,"15min":22.969999999999999},"last":2.2509999999999999},{"interface":"front","average":{"1min":2.9300000000000002,"5min":2.3359999999999999,"15min":2.492},"min":{"1min":0.39000000000000001,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":10.089,"5min":12.601000000000001,"15min":26.228000000000002},"last":0.36899999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:48:01 2023","interfaces":[{"interface":"back","average":{"1min":3.6920000000000002,"5min":2.3039999999999998,"15min":2.3439999999999999},"min":{"1min":0.41799999999999998,"5min":0.374,"15min":0.32300000000000001},"max":{"1min":9.2750000000000004,"5min":9.2750000000000004,"15min":24.056000000000001},"last":4.0439999999999996},{"interface":"front","average":{"1min":3.4020000000000001,"5min":2.395,"15min":2.3780000000000001},"min":{"1min":0.40999999999999998,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":12.427,"5min":12.698,"15min":26.268000000000001},"last":0.32300000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:48:01 2023","interfaces":[{"interface":"back","average":{"1min":3.4140000000000001,"5min":2.7749999999999999,"15min":2.5870000000000002},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.36099999999999999},"max":{"1min":12.486000000000001,"5min":16.553999999999998,"15min":26.177},"last":0.39400000000000002},{"interface":"front","average":{"1min":3.3879999999999999,"5min":2.6179999999999999,"15min":2.5179999999999998},"min":{"1min":0.45300000000000001,"5min":0.40200000000000002,"15min":0.34200000000000003},"max":{"1min":10.183999999999999,"5min":16.649000000000001,"15min":22.422999999999998},"last":1.5089999999999999}]}]},{"osd":2,"up_from":104,"seq":446676599108,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2312016,"kb_used_data":1511944,"kb_used_omap":11,"kb_used_meta":799988,"kb_avail":91433136,"statfs":{"total":95995035648,"available":93627531264,"internally_reserved":0,"allocated":1548230656,"data_stored":1545551216,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11878,"internal_metadata":819188122},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:47:10 2023","interfaces":[{"interface":"back","average":{"1min":1.8879999999999999,"5min":1.5169999999999999,"15min":2.2650000000000001},"min":{"1min":0.22800000000000001,"5min":0.17299999999999999,"15min":0.17299999999999999},"max":{"1min":15.324999999999999,"5min":15.324999999999999,"15min":82.424999999999997},"last":1.1499999999999999},{"interface":"front","average":{"1min":1.867,"5min":2.0030000000000001,"15min":2.444},"min":{"1min":0.245,"5min":0.23599999999999999,"15min":0.19},"max":{"1min":15.342000000000001,"5min":16.678999999999998,"15min":82.524000000000001},"last":1.071}]},{"osd":1,"last update":"Sat Sep 23 20:47:10 2023","interfaces":[{"interface":"back","average":{"1min":2.0649999999999999,"5min":1.99,"15min":2.5},"min":{"1min":0.219,"5min":0.219,"15min":0.219},"max":{"1min":21.690000000000001,"5min":21.690000000000001,"15min":46.405000000000001},"last":0.41599999999999998},{"interface":"front","average":{"1min":1.923,"5min":2.089,"15min":2.52},"min":{"1min":0.25700000000000001,"5min":0.22,"15min":0.22},"max":{"1min":8.4369999999999994,"5min":13.215999999999999,"15min":46.459000000000003},"last":0.79000000000000004}]},{"osd":3,"last update":"Sat Sep 23 20:47:37 2023","interfaces":[{"interface":"back","average":{"1min":2.8010000000000002,"5min":2.6549999999999998,"15min":2.6400000000000001},"min":{"1min":0.34100000000000003,"5min":0.253,"15min":0.253},"max":{"1min":9.9580000000000002,"5min":30.283000000000001,"15min":30.283000000000001},"last":1.093},{"interface":"front","average":{"1min":1.1399999999999999,"5min":1.5529999999999999,"15min":2.0619999999999998},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":4.3600000000000003,"5min":30.834,"15min":30.834},"last":0.91400000000000003}]},{"osd":4,"last update":"Sat Sep 23 20:47:10 2023","interfaces":[{"interface":"back","average":{"1min":2.214,"5min":1.9299999999999999,"15min":1.802},"min":{"1min":0.39400000000000002,"5min":0.29799999999999999,"15min":0.27500000000000002},"max":{"1min":7.1449999999999996,"5min":12.025,"15min":12.025},"last":0.58999999999999997},{"interface":"front","average":{"1min":1.9710000000000001,"5min":1.9119999999999999,"15min":1.9910000000000001},"min":{"1min":0.35699999999999998,"5min":0.30599999999999999,"15min":0.30499999999999999},"max":{"1min":6.5419999999999998,"5min":10.948,"15min":19.440999999999999},"last":0.53000000000000003}]},{"osd":5,"last update":"Sat Sep 23 20:47:10 2023","interfaces":[{"interface":"back","average":{"1min":1.347,"5min":1.895,"15min":1.8340000000000001},"min":{"1min":0.36399999999999999,"5min":0.311,"15min":0.311},"max":{"1min":6.8719999999999999,"5min":12.07,"15min":12.07},"last":0.84299999999999997},{"interface":"front","average":{"1min":1.3879999999999999,"5min":1.804,"15min":2.0209999999999999},"min":{"1min":0.40000000000000002,"5min":0.37,"15min":0.35299999999999998},"max":{"1min":3.7589999999999999,"5min":11.061999999999999,"15min":19.513000000000002},"last":0.80800000000000005}]},{"osd":6,"last update":"Sat Sep 23 20:47:10 2023","interfaces":[{"interface":"back","average":{"1min":2.133,"5min":2.2189999999999999,"15min":2.1030000000000002},"min":{"1min":0.42199999999999999,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":7.4290000000000003,"5min":14.047000000000001,"15min":19.486000000000001},"last":0.496},{"interface":"front","average":{"1min":1.7949999999999999,"5min":1.589,"15min":1.621},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.318},"max":{"1min":6.4580000000000002,"5min":11.747,"15min":11.747},"last":0.68700000000000006}]},{"osd":7,"last update":"Sat Sep 23 20:47:10 2023","interfaces":[{"interface":"back","average":{"1min":2.0790000000000002,"5min":1.573,"15min":1.6120000000000001},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.34699999999999998},"max":{"1min":7.718,"5min":8.4949999999999992,"15min":12.734999999999999},"last":0.71599999999999997},{"interface":"front","average":{"1min":1.8240000000000001,"5min":2.1720000000000002,"15min":2.0150000000000001},"min":{"1min":0.439,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":6.9539999999999997,"5min":12.099,"15min":12.099},"last":1.8779999999999999}]}]},{"osd":3,"up_from":109,"seq":468151435582,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2275072,"kb_used_data":1242616,"kb_used_omap":4,"kb_used_meta":1032443,"kb_avail":91470080,"statfs":{"total":95995035648,"available":93665361920,"internally_reserved":0,"allocated":1272438784,"data_stored":1269773415,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4514,"internal_metadata":1057222238},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:47:46 2023","interfaces":[{"interface":"back","average":{"1min":3.6509999999999998,"5min":2.6219999999999999,"15min":2.6019999999999999},"min":{"1min":0.23599999999999999,"5min":0.217,"15min":0.191},"max":{"1min":21.792999999999999,"5min":21.792999999999999,"15min":22.32},"last":0.434},{"interface":"front","average":{"1min":2.7069999999999999,"5min":2.7530000000000001,"15min":2.633},"min":{"1min":0.20200000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":14.845000000000001,"5min":22.805,"15min":22.805},"last":0.45000000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:47:16 2023","interfaces":[{"interface":"back","average":{"1min":1.119,"5min":2.4550000000000001,"15min":2.008},"min":{"1min":0.28199999999999997,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":4.9349999999999996,"5min":17.529,"15min":17.529},"last":0.64500000000000002},{"interface":"front","average":{"1min":1.9330000000000001,"5min":2.2269999999999999,"15min":2.2949999999999999},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":7.3780000000000001,"5min":14.19,"15min":32.603999999999999},"last":0.48799999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:47:46 2023","interfaces":[{"interface":"back","average":{"1min":3.0139999999999998,"5min":2.976,"15min":3.0619999999999998},"min":{"1min":0.34599999999999997,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":7.9210000000000003,"5min":19.478000000000002,"15min":68.206999999999994},"last":0.73999999999999999},{"interface":"front","average":{"1min":1.7829999999999999,"5min":2.016,"15min":2.556},"min":{"1min":0.31,"5min":0.245,"15min":0.245},"max":{"1min":8.0220000000000002,"5min":17.919,"15min":67.736999999999995},"last":1.0509999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:47:46 2023","interfaces":[{"interface":"back","average":{"1min":1.9119999999999999,"5min":1.851,"15min":1.6890000000000001},"min":{"1min":0.48899999999999999,"5min":0.317,"15min":0.30499999999999999},"max":{"1min":9.0259999999999998,"5min":9.0259999999999998,"15min":22.157},"last":1.889},{"interface":"front","average":{"1min":1.4099999999999999,"5min":1.871,"15min":2.077},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.29399999999999998},"max":{"1min":5.024,"5min":8.1910000000000007,"15min":15.868},"last":3.5659999999999998}]},{"osd":5,"last update":"Sat Sep 23 20:47:46 2023","interfaces":[{"interface":"back","average":{"1min":1.9570000000000001,"5min":2.1160000000000001,"15min":1.911},"min":{"1min":0.46200000000000002,"5min":0.33800000000000002,"15min":0.32200000000000001},"max":{"1min":8.9849999999999994,"5min":8.9849999999999994,"15min":22.289999999999999},"last":0.51600000000000001},{"interface":"front","average":{"1min":1.7929999999999999,"5min":2.0369999999999999,"15min":1.829},"min":{"1min":0.379,"5min":0.34100000000000003,"15min":0.32000000000000001},"max":{"1min":6.1390000000000002,"5min":12.952,"15min":12.952},"last":2.9940000000000002}]},{"osd":6,"last update":"Sat Sep 23 20:47:46 2023","interfaces":[{"interface":"back","average":{"1min":2.5819999999999999,"5min":2.6509999999999998,"15min":2.004},"min":{"1min":0.44400000000000001,"5min":0.36599999999999999,"15min":0.34300000000000003},"max":{"1min":7.0389999999999997,"5min":12.917,"15min":12.917},"last":1.6859999999999999},{"interface":"front","average":{"1min":1.698,"5min":2.1440000000000001,"15min":2.1349999999999998},"min":{"1min":0.36699999999999999,"5min":0.34599999999999997,"15min":0.33700000000000002},"max":{"1min":6.4930000000000003,"5min":8.9320000000000004,"15min":15.936},"last":0.39400000000000002}]},{"osd":7,"last update":"Sat Sep 23 20:47:46 2023","interfaces":[{"interface":"back","average":{"1min":1.768,"5min":2.262,"15min":2.302},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.36899999999999999},"max":{"1min":7.6429999999999998,"5min":11.885999999999999,"15min":15.976000000000001},"last":0.53200000000000003},{"interface":"front","average":{"1min":2.1360000000000001,"5min":2.1619999999999999,"15min":1.968},"min":{"1min":0.51100000000000001,"5min":0.42799999999999999,"15min":0.40999999999999998},"max":{"1min":9.0549999999999997,"5min":10.853999999999999,"15min":22.202999999999999},"last":4.2119999999999997}]}]},{"osd":4,"up_from":30,"seq":128849019336,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2427020,"kb_used_data":1920644,"kb_used_omap":0,"kb_used_meta":506368,"kb_avail":91318132,"statfs":{"total":95995035648,"available":93509767168,"internally_reserved":0,"allocated":1966739456,"data_stored":1965175614,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":518520832},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,3,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:48:04 2023","interfaces":[{"interface":"back","average":{"1min":2.1320000000000001,"5min":1.8500000000000001,"15min":1.8420000000000001},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24099999999999999},"max":{"1min":7.1120000000000001,"5min":11.709,"15min":24.114999999999998},"last":0.219},{"interface":"front","average":{"1min":2.4329999999999998,"5min":2.1480000000000001,"15min":2.0870000000000002},"min":{"1min":0.29799999999999999,"5min":0.25800000000000001,"15min":0.246},"max":{"1min":6.8639999999999999,"5min":13.74,"15min":24.135999999999999},"last":0.77300000000000002}]},{"osd":1,"last update":"Sat Sep 23 20:47:15 2023","interfaces":[{"interface":"back","average":{"1min":1.7669999999999999,"5min":2.2090000000000001,"15min":1.764},"min":{"1min":0.317,"5min":0.26300000000000001,"15min":0.25900000000000001},"max":{"1min":6.8600000000000003,"5min":9.673,"15min":18.190999999999999},"last":2.3540000000000001},{"interface":"front","average":{"1min":1.4590000000000001,"5min":1.659,"15min":1.5429999999999999},"min":{"1min":0.29799999999999999,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":4.9560000000000004,"5min":8.0730000000000004,"15min":18.173999999999999},"last":1.2729999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:47:15 2023","interfaces":[{"interface":"back","average":{"1min":2.0630000000000002,"5min":1.768,"15min":1.6970000000000001},"min":{"1min":0.30299999999999999,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":5.2400000000000002,"5min":9.3320000000000007,"15min":13.051},"last":3.8450000000000002},{"interface":"front","average":{"1min":2.367,"5min":2.2599999999999998,"15min":2.1040000000000001},"min":{"1min":0.35699999999999998,"5min":0.311,"15min":0.28399999999999997},"max":{"1min":8.7230000000000008,"5min":8.7230000000000008,"15min":14.614000000000001},"last":1.135}]},{"osd":3,"last update":"Sat Sep 23 20:47:31 2023","interfaces":[{"interface":"back","average":{"1min":1.8560000000000001,"5min":2.0089999999999999,"15min":1.901},"min":{"1min":0.318,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":5.0270000000000001,"5min":8.6180000000000003,"15min":12.147},"last":0.28499999999999998},{"interface":"front","average":{"1min":2.101,"5min":2.2000000000000002,"15min":2.0339999999999998},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.30299999999999999},"max":{"1min":6.3159999999999998,"5min":17.169,"15min":17.169},"last":0.32300000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:48:04 2023","interfaces":[{"interface":"back","average":{"1min":0.79300000000000004,"5min":1.458,"15min":1.0740000000000001},"min":{"1min":0.28399999999999997,"5min":0.253,"15min":0.253},"max":{"1min":3.5790000000000002,"5min":10.202,"15min":10.202},"last":0.499},{"interface":"front","average":{"1min":1.1479999999999999,"5min":1.5369999999999999,"15min":1.1850000000000001},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":4.7430000000000003,"5min":8.1440000000000001,"15min":8.1440000000000001},"last":0.80500000000000005}]},{"osd":6,"last update":"Sat Sep 23 20:47:15 2023","interfaces":[{"interface":"back","average":{"1min":2.1779999999999999,"5min":1.7270000000000001,"15min":1.3069999999999999},"min":{"1min":0.30399999999999999,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":8.7799999999999994,"5min":10.840999999999999,"15min":14.731999999999999},"last":1.167},{"interface":"front","average":{"1min":2.1869999999999998,"5min":1.71,"15min":1.157},"min":{"1min":0.32200000000000001,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":5.7910000000000004,"5min":7.6150000000000002,"15min":7.6150000000000002},"last":2.3159999999999998}]},{"osd":7,"last update":"Sat Sep 23 20:47:31 2023","interfaces":[{"interface":"back","average":{"1min":2.4089999999999998,"5min":5.4740000000000002,"15min":2.6669999999999998},"min":{"1min":0.377,"5min":0.35199999999999998,"15min":0.318},"max":{"1min":6.3920000000000003,"5min":313.50099999999998,"15min":313.50099999999998},"last":0.82899999999999996},{"interface":"front","average":{"1min":1.8340000000000001,"5min":4.7060000000000004,"15min":2.2290000000000001},"min":{"1min":0.35099999999999998,"5min":0.33200000000000002,"15min":0.32700000000000001},"max":{"1min":6.1829999999999998,"5min":312.67500000000001,"15min":312.67500000000001},"last":0.52800000000000002}]}]},{"osd":5,"up_from":39,"seq":167503724997,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1777284,"kb_used_data":1219580,"kb_used_omap":0,"kb_used_meta":557632,"kb_avail":91967868,"statfs":{"total":95995035648,"available":94175096832,"internally_reserved":0,"allocated":1248849920,"data_stored":1247290876,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":571015168},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:48:10 2023","interfaces":[{"interface":"back","average":{"1min":2.5129999999999999,"5min":1.8700000000000001,"15min":1.891},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.20699999999999999},"max":{"1min":6.673,"5min":15.018000000000001,"15min":18.16},"last":1.155},{"interface":"front","average":{"1min":2.258,"5min":1.915,"15min":1.8300000000000001},"min":{"1min":0.30099999999999999,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":8.6050000000000004,"5min":15.034000000000001,"15min":15.034000000000001},"last":1.4370000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:47:23 2023","interfaces":[{"interface":"back","average":{"1min":2.3490000000000002,"5min":2.3849999999999998,"15min":2.286},"min":{"1min":0.32200000000000001,"5min":0.29799999999999999,"15min":0.253},"max":{"1min":6.859,"5min":9.7230000000000008,"15min":61.005000000000003},"last":6.9850000000000003},{"interface":"front","average":{"1min":2.375,"5min":2.0329999999999999,"15min":2.1070000000000002},"min":{"1min":0.315,"5min":0.27700000000000002,"15min":0.23499999999999999},"max":{"1min":16.219000000000001,"5min":16.219000000000001,"15min":59.688000000000002},"last":0.40000000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:48:10 2023","interfaces":[{"interface":"back","average":{"1min":2.6160000000000001,"5min":2.1400000000000001,"15min":2.1579999999999999},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26500000000000001},"max":{"1min":7.194,"5min":12.226000000000001,"15min":37.011000000000003},"last":0.307},{"interface":"front","average":{"1min":2.5219999999999998,"5min":1.867,"15min":2.0350000000000001},"min":{"1min":0.39600000000000002,"5min":0.314,"15min":0.255},"max":{"1min":5.9790000000000001,"5min":9.8260000000000005,"15min":36.835000000000001},"last":0.59199999999999997}]},{"osd":3,"last update":"Sat Sep 23 20:47:27 2023","interfaces":[{"interface":"back","average":{"1min":1.7470000000000001,"5min":2.3010000000000002,"15min":2.23},"min":{"1min":0.38200000000000001,"5min":0.33800000000000002,"15min":0.29599999999999999},"max":{"1min":6.3449999999999998,"5min":11.032,"15min":36.716000000000001},"last":6.9470000000000001},{"interface":"front","average":{"1min":1.8300000000000001,"5min":2.1200000000000001,"15min":2.246},"min":{"1min":0.38900000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":6.7910000000000004,"5min":13.266,"15min":18.504000000000001},"last":3.2559999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:48:00 2023","interfaces":[{"interface":"back","average":{"1min":1.593,"5min":1.8180000000000001,"15min":1.2270000000000001},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.255},"max":{"1min":6.3170000000000002,"5min":13.391999999999999,"15min":13.391999999999999},"last":0.33300000000000002},{"interface":"front","average":{"1min":1.5129999999999999,"5min":1.6899999999999999,"15min":1.2390000000000001},"min":{"1min":0.34200000000000003,"5min":0.29199999999999998,"15min":0.28299999999999997},"max":{"1min":6.6600000000000001,"5min":7.3799999999999999,"15min":7.3799999999999999},"last":0.35299999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:47:10 2023","interfaces":[{"interface":"back","average":{"1min":1.522,"5min":2.1899999999999999,"15min":1.359},"min":{"1min":0.374,"5min":0.33100000000000002,"15min":0.30199999999999999},"max":{"1min":6.3710000000000004,"5min":13.734,"15min":13.734},"last":0.443},{"interface":"front","average":{"1min":1.893,"5min":1.5349999999999999,"15min":1.167},"min":{"1min":0.41399999999999998,"5min":0.317,"15min":0.317},"max":{"1min":6.2670000000000003,"5min":7.0700000000000003,"15min":8.7449999999999992},"last":0.51000000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:47:27 2023","interfaces":[{"interface":"back","average":{"1min":1.421,"5min":4.6580000000000004,"15min":2.1259999999999999},"min":{"1min":0.379,"5min":0.372,"15min":0.28499999999999998},"max":{"1min":6.2990000000000004,"5min":294.70999999999998,"15min":294.70999999999998},"last":5.0910000000000002},{"interface":"front","average":{"1min":1.111,"5min":4.6340000000000003,"15min":2.218},"min":{"1min":0.40500000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":4.7169999999999996,"5min":294.74400000000003,"15min":294.74400000000003},"last":0.63400000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4722,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1360,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8206,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7156,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3154,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2413957120,"data_stored":2413953048,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":754974720,"data_stored":754974720,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1521876992,"data_stored":1521876992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1251278848,"data_stored":1251278848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1955205120,"data_stored":1955201048,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1229127680,"data_stored":1229127680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1702170624,"data_stored":1702166552,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":983105536,"data_stored":983105536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:48:14.579 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:48:14.657 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:48:14.798 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:14.892 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:48:14.896 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:14.897 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:48:14.997 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:48:15.084 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:14 smithi018 ceph-mon[179647]: from='client.46832 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:48:15.084 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:14 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:48:15.084 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:14 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:48:15.084 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:48:14 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:48:14] "GET /metrics HTTP/1.1" 200 36573 "" "Prometheus/2.43.0" 2023-09-23T20:48:15.085 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:14 smithi018 ceph-mon[184144]: from='client.46832 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:48:15.085 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:14 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:48:15.085 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:14 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:48:15.090 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:15.158 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:48:15.166 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:15.166 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:48:15.192 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:48:15.209 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:14 smithi149 ceph-mon[170394]: from='client.46832 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:48:15.209 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:14 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:48:15.209 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:14 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:48:15.267 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:48:15.347 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:15.562 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:48:15.568 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:15.569 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:48:15.669 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:48:15.737 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:15.816 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:48:15.823 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:15.824 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:48:15.900 INFO:teuthology.orchestra.run.smithi149.stdout:ERROR: (22) Invalid argument 2023-09-23T20:48:15.900 INFO:teuthology.orchestra.run.smithi149.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:48:15.925 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:48:15.931 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:15 smithi149 ceph-mon[170394]: pgmap v1455: 119 pgs: 119 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 724 op/s 2023-09-23T20:48:15.948 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:48:15.996 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:15 smithi018 ceph-mon[179647]: pgmap v1455: 119 pgs: 119 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 724 op/s 2023-09-23T20:48:15.996 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:15 smithi018 ceph-mon[184144]: pgmap v1455: 119 pgs: 119 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 724 op/s 2023-09-23T20:48:15.997 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:16.236 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:48:16.239 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:16.240 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:48:16.275 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:48:16.341 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:48:16.343 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T20:48:16.430 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:16.561 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:48:16.574 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:16.575 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:48:16.675 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:48:16.759 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:17.120 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:48:17.437 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:48:17.445 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:17.445 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:48:17.546 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:48:17.613 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:17.683 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:48:17.692 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:17.693 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:48:17.794 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:48:17.846 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:17.970 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:17 smithi149 ceph-mon[170394]: pgmap v1456: 119 pgs: 119 active+clean; 3.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 815 op/s 2023-09-23T20:48:18.042 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:48:18.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:17 smithi018 ceph-mon[179647]: pgmap v1456: 119 pgs: 119 active+clean; 3.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 815 op/s 2023-09-23T20:48:18.144 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:17 smithi018 ceph-mon[184144]: pgmap v1456: 119 pgs: 119 active+clean; 3.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 815 op/s 2023-09-23T20:48:18.401 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:48:18.407 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:18.407 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:48:18.508 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:48:18.555 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:18.899 INFO:teuthology.orchestra.run.smithi018.stderr:noscrub is set 2023-09-23T20:48:18.902 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:48:18.929 INFO:teuthology.orchestra.run.smithi018.stderr:dumped all 2023-09-23T20:48:19.126 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:18 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/869476157' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:48:19.126 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:18 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:48:19.126 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:18 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/869476157' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:48:19.126 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:18 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:48:19.218 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:18 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/869476157' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:48:19.218 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:18 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:48:19.653 INFO:teuthology.orchestra.run.smithi018.stdout:{"pg_ready":true,"pg_map":{"version":1458,"stamp":"2023-09-23T20:48:18.868582+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3613398987,"num_objects":55092,"num_object_clones":0,"num_object_copies":165276,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":55092,"num_whiteouts":0,"num_read":5984,"num_read_kb":13201,"num_write":154831,"num_write_kb":6700278,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2468,"num_bytes_recovered":164443868,"num_keys_recovered":60,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":115716,"ondisk_log_size":115716,"up":357,"acting":357,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":357,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":16363924,"kb_used_data":10750484,"kb_used_omap":32,"kb_used_meta":5613151,"kb_avail":733597292,"statfs":{"total":767960285184,"available":751203627008,"internally_reserved":0,"allocated":11008495616,"data_stored":10991346193,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32773,"internal_metadata":5747867643},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,0,2,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":10,"apply_latency_ms":10,"commit_latency_ns":10000000,"apply_latency_ns":10000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-532217856,"num_objects":-8121,"num_object_clones":0,"num_object_copies":-24363,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-8121,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8121,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.555689"},"pg_stats":[{"pgid":"1.18","version":"90'161","reported_seq":622,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-09-23T20:48:09.771673+0000","last_change":"2023-09-23T20:44:19.677357+0000","last_active":"2023-09-23T20:48:09.771673+0000","last_peered":"2023-09-23T20:48:09.771673+0000","last_clean":"2023-09-23T20:48:09.771673+0000","last_became_active":"2023-09-23T20:43:16.766305+0000","last_became_peered":"2023-09-23T20:43:16.766305+0000","last_unstale":"2023-09-23T20:48:09.771673+0000","last_undegraded":"2023-09-23T20:48:09.771673+0000","last_fullsized":"2023-09-23T20:48:09.771673+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:19.677209+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:19.677209+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:14:06.510115+0000","scrub_duration":0.0023292260000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"69'1","reported_seq":32,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-09-23T20:48:09.789678+0000","last_change":"2023-09-23T20:47:09.186765+0000","last_active":"2023-09-23T20:48:09.789678+0000","last_peered":"2023-09-23T20:48:09.789678+0000","last_clean":"2023-09-23T20:48:09.789678+0000","last_became_active":"2023-09-23T20:47:09.186366+0000","last_became_peered":"2023-09-23T20:47:09.186366+0000","last_unstale":"2023-09-23T20:48:09.789678+0000","last_undegraded":"2023-09-23T20:48:09.789678+0000","last_fullsized":"2023-09-23T20:48:09.789678+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:10:04.707663+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":409,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-09-23T20:48:09.767037+0000","last_change":"2023-09-23T20:47:09.197577+0000","last_active":"2023-09-23T20:48:09.767037+0000","last_peered":"2023-09-23T20:48:09.767037+0000","last_clean":"2023-09-23T20:48:09.767037+0000","last_became_active":"2023-09-23T20:47:09.197280+0000","last_became_peered":"2023-09-23T20:47:09.197280+0000","last_unstale":"2023-09-23T20:48:09.767037+0000","last_undegraded":"2023-09-23T20:48:09.767037+0000","last_fullsized":"2023-09-23T20:48:09.767037+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:55:04.346224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"90'161","reported_seq":578,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-09-23T20:48:09.789771+0000","last_change":"2023-09-23T20:47:33.523278+0000","last_active":"2023-09-23T20:48:09.789771+0000","last_peered":"2023-09-23T20:48:09.789771+0000","last_clean":"2023-09-23T20:48:09.789771+0000","last_became_active":"2023-09-23T20:47:11.622087+0000","last_became_peered":"2023-09-23T20:47:11.622087+0000","last_unstale":"2023-09-23T20:48:09.789771+0000","last_undegraded":"2023-09-23T20:48:09.789771+0000","last_fullsized":"2023-09-23T20:48:09.789771+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:47:33.523187+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:36:05.442288+0000","last_clean_scrub_stamp":"2023-09-23T20:47:33.523187+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:32:18.410526+0000","scrub_duration":0.013532127,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":564,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-09-23T20:48:09.814397+0000","last_change":"2023-09-23T20:47:09.173501+0000","last_active":"2023-09-23T20:48:09.814397+0000","last_peered":"2023-09-23T20:48:09.814397+0000","last_clean":"2023-09-23T20:48:09.814397+0000","last_became_active":"2023-09-23T20:47:09.173160+0000","last_became_peered":"2023-09-23T20:47:09.173160+0000","last_unstale":"2023-09-23T20:48:09.814397+0000","last_undegraded":"2023-09-23T20:48:09.814397+0000","last_fullsized":"2023-09-23T20:48:09.814397+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:44:36.132093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":534,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-09-23T20:48:09.775700+0000","last_change":"2023-09-23T20:43:46.521939+0000","last_active":"2023-09-23T20:48:09.775700+0000","last_peered":"2023-09-23T20:48:09.775700+0000","last_clean":"2023-09-23T20:48:09.775700+0000","last_became_active":"2023-09-23T20:43:46.521185+0000","last_became_peered":"2023-09-23T20:43:46.521185+0000","last_unstale":"2023-09-23T20:48:09.775700+0000","last_undegraded":"2023-09-23T20:48:09.775700+0000","last_fullsized":"2023-09-23T20:48:09.775700+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:59:25.045627+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"90'161","reported_seq":463,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-09-23T20:48:09.789567+0000","last_change":"2023-09-23T20:47:31.475001+0000","last_active":"2023-09-23T20:48:09.789567+0000","last_peered":"2023-09-23T20:48:09.789567+0000","last_clean":"2023-09-23T20:48:09.789567+0000","last_became_active":"2023-09-23T20:47:11.617942+0000","last_became_peered":"2023-09-23T20:47:11.617942+0000","last_unstale":"2023-09-23T20:48:09.789567+0000","last_undegraded":"2023-09-23T20:48:09.789567+0000","last_fullsized":"2023-09-23T20:48:09.789567+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:47:31.474914+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:47:31.474914+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:52:05.169219+0000","scrub_duration":0.0026262870000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":371,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-09-23T20:48:09.766443+0000","last_change":"2023-09-23T20:37:43.584414+0000","last_active":"2023-09-23T20:48:09.766443+0000","last_peered":"2023-09-23T20:48:09.766443+0000","last_clean":"2023-09-23T20:48:09.766443+0000","last_became_active":"2023-09-23T20:37:43.584116+0000","last_became_peered":"2023-09-23T20:37:43.584116+0000","last_unstale":"2023-09-23T20:48:09.766443+0000","last_undegraded":"2023-09-23T20:48:09.766443+0000","last_fullsized":"2023-09-23T20:48:09.766443+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:17:54.258172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":190,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-09-23T20:48:09.798415+0000","last_change":"2023-09-23T20:38:37.457427+0000","last_active":"2023-09-23T20:48:09.798415+0000","last_peered":"2023-09-23T20:48:09.798415+0000","last_clean":"2023-09-23T20:48:09.798415+0000","last_became_active":"2023-09-23T20:38:37.457099+0000","last_became_peered":"2023-09-23T20:38:37.457099+0000","last_unstale":"2023-09-23T20:48:09.798415+0000","last_undegraded":"2023-09-23T20:48:09.798415+0000","last_fullsized":"2023-09-23T20:48:09.798415+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:31:27.084994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"90'161","reported_seq":787,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-09-23T20:48:09.783568+0000","last_change":"2023-09-23T20:40:22.449754+0000","last_active":"2023-09-23T20:48:09.783568+0000","last_peered":"2023-09-23T20:48:09.783568+0000","last_clean":"2023-09-23T20:48:09.783568+0000","last_became_active":"2023-09-23T20:40:22.448838+0000","last_became_peered":"2023-09-23T20:40:22.448838+0000","last_unstale":"2023-09-23T20:48:09.783568+0000","last_undegraded":"2023-09-23T20:48:09.783568+0000","last_fullsized":"2023-09-23T20:48:09.783568+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:03.411025+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:03.411025+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:22:07.948856+0000","scrub_duration":0.001802922,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":566,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-09-23T20:48:09.775170+0000","last_change":"2023-09-23T20:43:46.522325+0000","last_active":"2023-09-23T20:48:09.775170+0000","last_peered":"2023-09-23T20:48:09.775170+0000","last_clean":"2023-09-23T20:48:09.775170+0000","last_became_active":"2023-09-23T20:43:46.521996+0000","last_became_peered":"2023-09-23T20:43:46.521996+0000","last_unstale":"2023-09-23T20:48:09.775170+0000","last_undegraded":"2023-09-23T20:48:09.775170+0000","last_fullsized":"2023-09-23T20:48:09.775170+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:51:17.629956+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":301,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-09-23T20:48:09.771814+0000","last_change":"2023-09-23T20:38:02.579343+0000","last_active":"2023-09-23T20:48:09.771814+0000","last_peered":"2023-09-23T20:48:09.771814+0000","last_clean":"2023-09-23T20:48:09.771814+0000","last_became_active":"2023-09-23T20:38:02.579047+0000","last_became_peered":"2023-09-23T20:38:02.579047+0000","last_unstale":"2023-09-23T20:48:09.771814+0000","last_undegraded":"2023-09-23T20:48:09.771814+0000","last_fullsized":"2023-09-23T20:48:09.771814+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:44:03.166556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.14","version":"90'161","reported_seq":941,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-09-23T20:48:09.771930+0000","last_change":"2023-09-23T20:47:11.618616+0000","last_active":"2023-09-23T20:48:09.771930+0000","last_peered":"2023-09-23T20:48:09.771930+0000","last_clean":"2023-09-23T20:48:09.771930+0000","last_became_active":"2023-09-23T20:47:11.618237+0000","last_became_peered":"2023-09-23T20:47:11.618237+0000","last_unstale":"2023-09-23T20:48:09.771930+0000","last_undegraded":"2023-09-23T20:48:09.771930+0000","last_fullsized":"2023-09-23T20:48:09.771930+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:47.479012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:47.479012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:06:10.817025+0000","scrub_duration":0.0020136590000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":411,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-09-23T20:48:09.784042+0000","last_change":"2023-09-23T20:43:16.780478+0000","last_active":"2023-09-23T20:48:09.784042+0000","last_peered":"2023-09-23T20:48:09.784042+0000","last_clean":"2023-09-23T20:48:09.784042+0000","last_became_active":"2023-09-23T20:43:16.780183+0000","last_became_peered":"2023-09-23T20:43:16.780183+0000","last_unstale":"2023-09-23T20:48:09.784042+0000","last_undegraded":"2023-09-23T20:48:09.784042+0000","last_fullsized":"2023-09-23T20:48:09.784042+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:38:30.102066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":402,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-09-23T20:48:09.766910+0000","last_change":"2023-09-23T20:38:02.542034+0000","last_active":"2023-09-23T20:48:09.766910+0000","last_peered":"2023-09-23T20:48:09.766910+0000","last_clean":"2023-09-23T20:48:09.766910+0000","last_became_active":"2023-09-23T20:38:02.541575+0000","last_became_peered":"2023-09-23T20:38:02.541575+0000","last_unstale":"2023-09-23T20:48:09.766910+0000","last_undegraded":"2023-09-23T20:48:09.766910+0000","last_fullsized":"2023-09-23T20:48:09.766910+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:50:54.591529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"90'161","reported_seq":701,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-09-23T20:48:09.774970+0000","last_change":"2023-09-23T20:44:14.669973+0000","last_active":"2023-09-23T20:48:09.774970+0000","last_peered":"2023-09-23T20:48:09.774970+0000","last_clean":"2023-09-23T20:48:09.774970+0000","last_became_active":"2023-09-23T20:43:46.519592+0000","last_became_peered":"2023-09-23T20:43:46.519592+0000","last_unstale":"2023-09-23T20:48:09.774970+0000","last_undegraded":"2023-09-23T20:48:09.774970+0000","last_fullsized":"2023-09-23T20:48:09.774970+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:14.669831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:14.669831+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:56:28.076617+0000","scrub_duration":0.0083441460000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":634,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-09-23T20:48:09.783850+0000","last_change":"2023-09-23T20:47:10.837423+0000","last_active":"2023-09-23T20:48:09.783850+0000","last_peered":"2023-09-23T20:48:09.783850+0000","last_clean":"2023-09-23T20:48:09.783850+0000","last_became_active":"2023-09-23T20:47:09.204324+0000","last_became_peered":"2023-09-23T20:47:09.204324+0000","last_unstale":"2023-09-23T20:48:09.783850+0000","last_undegraded":"2023-09-23T20:48:09.783850+0000","last_fullsized":"2023-09-23T20:48:09.783850+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":554,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:19.108914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":540,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":90,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-09-23T20:48:09.814990+0000","last_change":"2023-09-23T20:44:10.611381+0000","last_active":"2023-09-23T20:48:09.814990+0000","last_peered":"2023-09-23T20:48:09.814990+0000","last_clean":"2023-09-23T20:48:09.814990+0000","last_became_active":"2023-09-23T20:44:10.611091+0000","last_became_peered":"2023-09-23T20:44:10.611091+0000","last_unstale":"2023-09-23T20:48:09.814990+0000","last_undegraded":"2023-09-23T20:48:09.814990+0000","last_fullsized":"2023-09-23T20:48:09.814990+0000","mapping_epoch":496,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":497,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:00:30.162478+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"90'161","reported_seq":702,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-09-23T20:48:09.815159+0000","last_change":"2023-09-23T20:36:02.548387+0000","last_active":"2023-09-23T20:48:09.815159+0000","last_peered":"2023-09-23T20:48:09.815159+0000","last_clean":"2023-09-23T20:48:09.815159+0000","last_became_active":"2023-09-23T20:35:26.644787+0000","last_became_peered":"2023-09-23T20:35:26.644787+0000","last_unstale":"2023-09-23T20:48:09.815159+0000","last_undegraded":"2023-09-23T20:48:09.815159+0000","last_fullsized":"2023-09-23T20:48:09.815159+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:02.548285+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:02.548285+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:37:15.216553+0000","scrub_duration":0.0015514459999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":42,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-09-23T20:48:09.789792+0000","last_change":"2023-09-23T20:47:09.340979+0000","last_active":"2023-09-23T20:48:09.789792+0000","last_peered":"2023-09-23T20:48:09.789792+0000","last_clean":"2023-09-23T20:48:09.789792+0000","last_became_active":"2023-09-23T20:47:09.231222+0000","last_became_peered":"2023-09-23T20:47:09.231222+0000","last_unstale":"2023-09-23T20:48:09.789792+0000","last_undegraded":"2023-09-23T20:48:09.789792+0000","last_fullsized":"2023-09-23T20:48:09.789792+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:07:50.620006+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":32,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-09-23T20:48:09.789982+0000","last_change":"2023-09-23T20:47:09.186315+0000","last_active":"2023-09-23T20:48:09.789982+0000","last_peered":"2023-09-23T20:48:09.789982+0000","last_clean":"2023-09-23T20:48:09.789982+0000","last_became_active":"2023-09-23T20:47:09.184580+0000","last_became_peered":"2023-09-23T20:47:09.184580+0000","last_unstale":"2023-09-23T20:48:09.789982+0000","last_undegraded":"2023-09-23T20:48:09.789982+0000","last_fullsized":"2023-09-23T20:48:09.789982+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:44:35.553948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"90'161","reported_seq":829,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-09-23T20:48:09.779819+0000","last_change":"2023-09-23T20:43:46.544727+0000","last_active":"2023-09-23T20:48:09.779819+0000","last_peered":"2023-09-23T20:48:09.779819+0000","last_clean":"2023-09-23T20:48:09.779819+0000","last_became_active":"2023-09-23T20:43:46.543952+0000","last_became_peered":"2023-09-23T20:43:46.543952+0000","last_unstale":"2023-09-23T20:48:09.779819+0000","last_undegraded":"2023-09-23T20:48:09.779819+0000","last_fullsized":"2023-09-23T20:48:09.779819+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:43.578666+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:43.578666+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:11:16.611920+0000","scrub_duration":0.0021890379999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":633,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-09-23T20:48:09.783480+0000","last_change":"2023-09-23T20:47:09.242696+0000","last_active":"2023-09-23T20:48:09.783480+0000","last_peered":"2023-09-23T20:48:09.783480+0000","last_clean":"2023-09-23T20:48:09.783480+0000","last_became_active":"2023-09-23T20:47:09.176728+0000","last_became_peered":"2023-09-23T20:47:09.176728+0000","last_unstale":"2023-09-23T20:48:09.783480+0000","last_undegraded":"2023-09-23T20:48:09.783480+0000","last_fullsized":"2023-09-23T20:48:09.783480+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:46:41.529834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":204,"num_keys_recovered":60,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":322,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-09-23T20:48:09.770696+0000","last_change":"2023-09-23T20:45:01.875931+0000","last_active":"2023-09-23T20:48:09.770696+0000","last_peered":"2023-09-23T20:48:09.770696+0000","last_clean":"2023-09-23T20:48:09.770696+0000","last_became_active":"2023-09-23T20:45:01.872839+0000","last_became_peered":"2023-09-23T20:45:01.872839+0000","last_unstale":"2023-09-23T20:48:09.770696+0000","last_undegraded":"2023-09-23T20:48:09.770696+0000","last_fullsized":"2023-09-23T20:48:09.770696+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:15:09.584507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"90'161","reported_seq":855,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-09-23T20:48:09.783166+0000","last_change":"2023-09-23T20:44:20.041766+0000","last_active":"2023-09-23T20:48:09.783166+0000","last_peered":"2023-09-23T20:48:09.783166+0000","last_clean":"2023-09-23T20:48:09.783166+0000","last_became_active":"2023-09-23T20:43:16.765015+0000","last_became_peered":"2023-09-23T20:43:16.765015+0000","last_unstale":"2023-09-23T20:48:09.783166+0000","last_undegraded":"2023-09-23T20:48:09.783166+0000","last_fullsized":"2023-09-23T20:48:09.783166+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:20.041690+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:20.041690+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:33:30.150109+0000","scrub_duration":0.0026180330000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":531,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-09-23T20:48:09.783103+0000","last_change":"2023-09-23T20:43:46.521294+0000","last_active":"2023-09-23T20:48:09.783103+0000","last_peered":"2023-09-23T20:48:09.783103+0000","last_clean":"2023-09-23T20:48:09.783103+0000","last_became_active":"2023-09-23T20:43:46.521001+0000","last_became_peered":"2023-09-23T20:43:46.521001+0000","last_unstale":"2023-09-23T20:48:09.783103+0000","last_undegraded":"2023-09-23T20:48:09.783103+0000","last_fullsized":"2023-09-23T20:48:09.783103+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:01:35.126716+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":342,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-09-23T20:48:09.783864+0000","last_change":"2023-09-23T20:44:01.841565+0000","last_active":"2023-09-23T20:48:09.783864+0000","last_peered":"2023-09-23T20:48:09.783864+0000","last_clean":"2023-09-23T20:48:09.783864+0000","last_became_active":"2023-09-23T20:44:01.841255+0000","last_became_peered":"2023-09-23T20:44:01.841255+0000","last_unstale":"2023-09-23T20:48:09.783864+0000","last_undegraded":"2023-09-23T20:48:09.783864+0000","last_fullsized":"2023-09-23T20:48:09.783864+0000","mapping_epoch":491,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":492,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:18:44.817589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,2],"acting":[7,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.13","version":"90'161","reported_seq":897,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-09-23T20:48:09.816392+0000","last_change":"2023-09-23T20:47:11.619371+0000","last_active":"2023-09-23T20:48:09.816392+0000","last_peered":"2023-09-23T20:48:09.816392+0000","last_clean":"2023-09-23T20:48:09.816392+0000","last_became_active":"2023-09-23T20:47:11.619006+0000","last_became_peered":"2023-09-23T20:47:11.619006+0000","last_unstale":"2023-09-23T20:48:09.816392+0000","last_undegraded":"2023-09-23T20:48:09.816392+0000","last_fullsized":"2023-09-23T20:48:09.816392+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:42.544308+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:42.544308+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:47:24.414807+0000","scrub_duration":0.0029958179999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":598,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-09-23T20:48:09.783453+0000","last_change":"2023-09-23T20:47:09.176613+0000","last_active":"2023-09-23T20:48:09.783453+0000","last_peered":"2023-09-23T20:48:09.783453+0000","last_clean":"2023-09-23T20:48:09.783453+0000","last_became_active":"2023-09-23T20:47:09.176187+0000","last_became_peered":"2023-09-23T20:47:09.176187+0000","last_unstale":"2023-09-23T20:48:09.783453+0000","last_undegraded":"2023-09-23T20:48:09.783453+0000","last_fullsized":"2023-09-23T20:48:09.783453+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:23:01.590989+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":540,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-09-23T20:48:09.775765+0000","last_change":"2023-09-23T20:43:46.521735+0000","last_active":"2023-09-23T20:48:09.775765+0000","last_peered":"2023-09-23T20:48:09.775765+0000","last_clean":"2023-09-23T20:48:09.775765+0000","last_became_active":"2023-09-23T20:43:46.520762+0000","last_became_peered":"2023-09-23T20:43:46.520762+0000","last_unstale":"2023-09-23T20:48:09.775765+0000","last_undegraded":"2023-09-23T20:48:09.775765+0000","last_fullsized":"2023-09-23T20:48:09.775765+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:26:48.841930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"90'161","reported_seq":660,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-09-23T20:48:09.815562+0000","last_change":"2023-09-23T20:44:18.952808+0000","last_active":"2023-09-23T20:48:09.815562+0000","last_peered":"2023-09-23T20:48:09.815562+0000","last_clean":"2023-09-23T20:48:09.815562+0000","last_became_active":"2023-09-23T20:43:16.758652+0000","last_became_peered":"2023-09-23T20:43:16.758652+0000","last_unstale":"2023-09-23T20:48:09.815562+0000","last_undegraded":"2023-09-23T20:48:09.815562+0000","last_fullsized":"2023-09-23T20:48:09.815562+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:18.952693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:18.952693+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:12:19.496465+0000","scrub_duration":0.001699365,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":250,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-09-23T20:48:09.793077+0000","last_change":"2023-09-23T20:44:43.644290+0000","last_active":"2023-09-23T20:48:09.793077+0000","last_peered":"2023-09-23T20:48:09.793077+0000","last_clean":"2023-09-23T20:48:09.793077+0000","last_became_active":"2023-09-23T20:44:43.638313+0000","last_became_peered":"2023-09-23T20:44:43.638313+0000","last_unstale":"2023-09-23T20:48:09.793077+0000","last_undegraded":"2023-09-23T20:48:09.793077+0000","last_fullsized":"2023-09-23T20:48:09.793077+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:32:52.892603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":371,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-09-23T20:48:09.767551+0000","last_change":"2023-09-23T20:37:43.571083+0000","last_active":"2023-09-23T20:48:09.767551+0000","last_peered":"2023-09-23T20:48:09.767551+0000","last_clean":"2023-09-23T20:48:09.767551+0000","last_became_active":"2023-09-23T20:37:43.570306+0000","last_became_peered":"2023-09-23T20:37:43.570306+0000","last_unstale":"2023-09-23T20:48:09.767551+0000","last_undegraded":"2023-09-23T20:48:09.767551+0000","last_fullsized":"2023-09-23T20:48:09.767551+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:42:02.396718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.6","version":"575'9672","reported_seq":10327,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-09-23T20:48:16.551398+0000","last_change":"2023-09-23T20:47:12.636171+0000","last_active":"2023-09-23T20:48:16.551398+0000","last_peered":"2023-09-23T20:48:16.551398+0000","last_clean":"2023-09-23T20:48:16.551398+0000","last_became_active":"2023-09-23T20:47:12.628915+0000","last_became_peered":"2023-09-23T20:47:12.628915+0000","last_unstale":"2023-09-23T20:48:16.551398+0000","last_undegraded":"2023-09-23T20:48:16.551398+0000","last_fullsized":"2023-09-23T20:48:16.551398+0000","mapping_epoch":556,"log_start":"538'3090","ondisk_log_start":"538'3090","created":531,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":6582,"log_dups_size":0,"ondisk_log_size":6582,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:52:38.896682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":226230272,"num_objects":3452,"num_object_clones":0,"num_object_copies":10356,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3452,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9672,"num_write_kb":419968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":287,"num_bytes_recovered":18808832,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.d","version":"90'161","reported_seq":907,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-09-23T20:48:09.783652+0000","last_change":"2023-09-23T20:47:11.619510+0000","last_active":"2023-09-23T20:48:09.783652+0000","last_peered":"2023-09-23T20:48:09.783652+0000","last_clean":"2023-09-23T20:48:09.783652+0000","last_became_active":"2023-09-23T20:47:11.619154+0000","last_became_peered":"2023-09-23T20:47:11.619154+0000","last_unstale":"2023-09-23T20:48:09.783652+0000","last_undegraded":"2023-09-23T20:48:09.783652+0000","last_fullsized":"2023-09-23T20:48:09.783652+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:34:59.496375+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:34:59.496375+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:40:52.712777+0000","scrub_duration":0.0017843189999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":606,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-09-23T20:48:09.784223+0000","last_change":"2023-09-23T20:47:09.207851+0000","last_active":"2023-09-23T20:48:09.784223+0000","last_peered":"2023-09-23T20:48:09.784223+0000","last_clean":"2023-09-23T20:48:09.784223+0000","last_became_active":"2023-09-23T20:47:09.175836+0000","last_became_peered":"2023-09-23T20:47:09.175836+0000","last_unstale":"2023-09-23T20:48:09.784223+0000","last_undegraded":"2023-09-23T20:48:09.784223+0000","last_fullsized":"2023-09-23T20:48:09.784223+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:43:13.091809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":389,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-09-23T20:48:09.783362+0000","last_change":"2023-09-23T20:43:16.765253+0000","last_active":"2023-09-23T20:48:09.783362+0000","last_peered":"2023-09-23T20:48:09.783362+0000","last_clean":"2023-09-23T20:48:09.783362+0000","last_became_active":"2023-09-23T20:43:16.764978+0000","last_became_peered":"2023-09-23T20:43:16.764978+0000","last_unstale":"2023-09-23T20:48:09.783362+0000","last_undegraded":"2023-09-23T20:48:09.783362+0000","last_fullsized":"2023-09-23T20:48:09.783362+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:07:23.539372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.7","version":"575'9489","reported_seq":9541,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-09-23T20:48:15.393777+0000","last_change":"2023-09-23T20:45:39.488542+0000","last_active":"2023-09-23T20:48:15.393777+0000","last_peered":"2023-09-23T20:48:15.393777+0000","last_clean":"2023-09-23T20:48:15.393777+0000","last_became_active":"2023-09-23T20:45:39.488259+0000","last_became_peered":"2023-09-23T20:45:39.488259+0000","last_unstale":"2023-09-23T20:48:15.393777+0000","last_undegraded":"2023-09-23T20:48:15.393777+0000","last_fullsized":"2023-09-23T20:48:15.393777+0000","mapping_epoch":531,"log_start":"538'3026","ondisk_log_start":"538'3026","created":531,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":6463,"log_dups_size":0,"ondisk_log_size":6463,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:39:05.833662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":228392960,"num_objects":3485,"num_object_clones":0,"num_object_copies":10455,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3485,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9489,"num_write_kb":415168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.c","version":"90'161","reported_seq":837,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-09-23T20:48:09.770776+0000","last_change":"2023-09-23T20:41:09.506481+0000","last_active":"2023-09-23T20:48:09.770776+0000","last_peered":"2023-09-23T20:48:09.770776+0000","last_clean":"2023-09-23T20:48:09.770776+0000","last_became_active":"2023-09-23T20:38:28.442229+0000","last_became_peered":"2023-09-23T20:38:28.442229+0000","last_unstale":"2023-09-23T20:48:09.770776+0000","last_undegraded":"2023-09-23T20:48:09.770776+0000","last_fullsized":"2023-09-23T20:48:09.770776+0000","mapping_epoch":411,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":412,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:41:09.506338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:41:09.506338+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:19:46.265011+0000","scrub_duration":0.01112002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":644,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-09-23T20:48:09.784105+0000","last_change":"2023-09-23T20:47:10.848436+0000","last_active":"2023-09-23T20:48:09.784105+0000","last_peered":"2023-09-23T20:48:09.784105+0000","last_clean":"2023-09-23T20:48:09.784105+0000","last_became_active":"2023-09-23T20:47:09.196769+0000","last_became_peered":"2023-09-23T20:47:09.196769+0000","last_unstale":"2023-09-23T20:48:09.784105+0000","last_undegraded":"2023-09-23T20:48:09.784105+0000","last_fullsized":"2023-09-23T20:48:09.784105+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":554,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:59:38.105009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":32,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-09-23T20:48:09.789719+0000","last_change":"2023-09-23T20:47:09.183860+0000","last_active":"2023-09-23T20:48:09.789719+0000","last_peered":"2023-09-23T20:48:09.789719+0000","last_clean":"2023-09-23T20:48:09.789719+0000","last_became_active":"2023-09-23T20:47:09.183545+0000","last_became_peered":"2023-09-23T20:47:09.183545+0000","last_unstale":"2023-09-23T20:48:09.789719+0000","last_undegraded":"2023-09-23T20:48:09.789719+0000","last_fullsized":"2023-09-23T20:48:09.789719+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:54:03.275528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,3],"acting":[4,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"575'9522","reported_seq":10151,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-09-23T20:48:16.546878+0000","last_change":"2023-09-23T20:47:14.635782+0000","last_active":"2023-09-23T20:48:16.546878+0000","last_peered":"2023-09-23T20:48:16.546878+0000","last_clean":"2023-09-23T20:48:16.546878+0000","last_became_active":"2023-09-23T20:47:14.627887+0000","last_became_peered":"2023-09-23T20:47:14.627887+0000","last_unstale":"2023-09-23T20:48:16.546878+0000","last_undegraded":"2023-09-23T20:48:16.546878+0000","last_fullsized":"2023-09-23T20:48:16.546878+0000","mapping_epoch":558,"log_start":"538'2993","ondisk_log_start":"538'2993","created":531,"last_epoch_clean":559,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":6529,"log_dups_size":0,"ondisk_log_size":6529,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:13:49.501508+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":223084544,"num_objects":3404,"num_object_clones":0,"num_object_copies":10212,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3404,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9522,"num_write_kb":413632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":274,"num_bytes_recovered":17956864,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"90'161","reported_seq":770,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-09-23T20:48:09.798818+0000","last_change":"2023-09-23T20:47:11.622303+0000","last_active":"2023-09-23T20:48:09.798818+0000","last_peered":"2023-09-23T20:48:09.798818+0000","last_clean":"2023-09-23T20:48:09.798818+0000","last_became_active":"2023-09-23T20:47:11.621953+0000","last_became_peered":"2023-09-23T20:47:11.621953+0000","last_unstale":"2023-09-23T20:48:09.798818+0000","last_undegraded":"2023-09-23T20:48:09.798818+0000","last_fullsized":"2023-09-23T20:48:09.798818+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:46.217489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:46.217489+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:14:40.478522+0000","scrub_duration":0.001456429,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":603,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-09-23T20:48:09.783741+0000","last_change":"2023-09-23T20:40:19.677655+0000","last_active":"2023-09-23T20:48:09.783741+0000","last_peered":"2023-09-23T20:48:09.783741+0000","last_clean":"2023-09-23T20:48:09.783741+0000","last_became_active":"2023-09-23T20:40:19.551189+0000","last_became_peered":"2023-09-23T20:40:19.551189+0000","last_unstale":"2023-09-23T20:48:09.783741+0000","last_undegraded":"2023-09-23T20:48:09.783741+0000","last_fullsized":"2023-09-23T20:48:09.783741+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:11:01.559576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":180,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-09-23T20:48:09.795477+0000","last_change":"2023-09-23T20:38:02.590613+0000","last_active":"2023-09-23T20:48:09.795477+0000","last_peered":"2023-09-23T20:48:09.795477+0000","last_clean":"2023-09-23T20:48:09.795477+0000","last_became_active":"2023-09-23T20:38:02.590284+0000","last_became_peered":"2023-09-23T20:38:02.590284+0000","last_unstale":"2023-09-23T20:48:09.795477+0000","last_undegraded":"2023-09-23T20:48:09.795477+0000","last_fullsized":"2023-09-23T20:48:09.795477+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:40:07.825054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.5","version":"575'9768","reported_seq":9820,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-09-23T20:48:18.260045+0000","last_change":"2023-09-23T20:45:39.380986+0000","last_active":"2023-09-23T20:48:18.260045+0000","last_peered":"2023-09-23T20:48:18.260045+0000","last_clean":"2023-09-23T20:48:18.260045+0000","last_became_active":"2023-09-23T20:45:39.380650+0000","last_became_peered":"2023-09-23T20:45:39.380650+0000","last_unstale":"2023-09-23T20:48:18.260045+0000","last_undegraded":"2023-09-23T20:48:18.260045+0000","last_fullsized":"2023-09-23T20:48:18.260045+0000","mapping_epoch":531,"log_start":"536'900","ondisk_log_start":"536'900","created":531,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":8868,"log_dups_size":0,"ondisk_log_size":8868,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:24:24.301239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":223870976,"num_objects":3416,"num_object_clones":0,"num_object_copies":10248,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3416,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9768,"num_write_kb":421888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.e","version":"90'161","reported_seq":605,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-09-23T20:48:09.790855+0000","last_change":"2023-09-23T20:47:37.608426+0000","last_active":"2023-09-23T20:48:09.790855+0000","last_peered":"2023-09-23T20:48:09.790855+0000","last_clean":"2023-09-23T20:48:09.790855+0000","last_became_active":"2023-09-23T20:47:11.617187+0000","last_became_peered":"2023-09-23T20:47:11.617187+0000","last_unstale":"2023-09-23T20:48:09.790855+0000","last_undegraded":"2023-09-23T20:48:09.790855+0000","last_fullsized":"2023-09-23T20:48:09.790855+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:47:37.608309+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:47:37.608309+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:22:25.482297+0000","scrub_duration":0.011600924,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":536,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-09-23T20:48:09.771083+0000","last_change":"2023-09-23T20:43:16.756437+0000","last_active":"2023-09-23T20:48:09.771083+0000","last_peered":"2023-09-23T20:48:09.771083+0000","last_clean":"2023-09-23T20:48:09.771083+0000","last_became_active":"2023-09-23T20:43:16.756193+0000","last_became_peered":"2023-09-23T20:43:16.756193+0000","last_unstale":"2023-09-23T20:48:09.771083+0000","last_undegraded":"2023-09-23T20:48:09.771083+0000","last_fullsized":"2023-09-23T20:48:09.771083+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:33:25.632781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":568,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-09-23T20:48:11.812634+0000","last_change":"2023-09-23T20:47:09.197427+0000","last_active":"2023-09-23T20:48:11.812634+0000","last_peered":"2023-09-23T20:48:11.812634+0000","last_clean":"2023-09-23T20:48:11.812634+0000","last_became_active":"2023-09-23T20:47:09.196542+0000","last_became_peered":"2023-09-23T20:47:09.196542+0000","last_unstale":"2023-09-23T20:48:11.812634+0000","last_undegraded":"2023-09-23T20:48:11.812634+0000","last_fullsized":"2023-09-23T20:48:11.812634+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:28:38.926202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"575'9918","reported_seq":10960,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-09-23T20:48:18.629476+0000","last_change":"2023-09-23T20:47:16.635196+0000","last_active":"2023-09-23T20:48:18.629476+0000","last_peered":"2023-09-23T20:48:18.629476+0000","last_clean":"2023-09-23T20:48:18.629476+0000","last_became_active":"2023-09-23T20:47:16.628907+0000","last_became_peered":"2023-09-23T20:47:16.628907+0000","last_unstale":"2023-09-23T20:48:18.629476+0000","last_undegraded":"2023-09-23T20:48:18.629476+0000","last_fullsized":"2023-09-23T20:48:18.629476+0000","mapping_epoch":560,"log_start":"538'3224","ondisk_log_start":"538'3224","created":531,"last_epoch_clean":561,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":6694,"log_dups_size":0,"ondisk_log_size":6694,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:30:22.186026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":220594176,"num_objects":3366,"num_object_clones":0,"num_object_copies":10098,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3366,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9918,"num_write_kb":425088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":477,"num_bytes_recovered":31260672,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":980,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-09-23T20:48:09.783825+0000","last_change":"2023-09-23T20:35:02.523308+0000","last_active":"2023-09-23T20:48:09.783825+0000","last_peered":"2023-09-23T20:48:09.783825+0000","last_clean":"2023-09-23T20:48:09.783825+0000","last_became_active":"2023-09-23T20:33:38.979989+0000","last_became_peered":"2023-09-23T20:33:38.979989+0000","last_unstale":"2023-09-23T20:48:09.783825+0000","last_undegraded":"2023-09-23T20:48:09.783825+0000","last_fullsized":"2023-09-23T20:48:09.783825+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":333,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:35:02.523203+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:35:02.523203+0000","last_clean_scrub_stamp":"2023-09-23T20:35:02.523203+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:02:41.472958+0000","scrub_duration":0.0085927989999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":423,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-09-23T20:48:09.767126+0000","last_change":"2023-09-23T20:47:09.240200+0000","last_active":"2023-09-23T20:48:09.767126+0000","last_peered":"2023-09-23T20:48:09.767126+0000","last_clean":"2023-09-23T20:48:09.767126+0000","last_became_active":"2023-09-23T20:47:09.195204+0000","last_became_peered":"2023-09-23T20:47:09.195204+0000","last_unstale":"2023-09-23T20:48:09.767126+0000","last_undegraded":"2023-09-23T20:48:09.767126+0000","last_fullsized":"2023-09-23T20:48:09.767126+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:03.328250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":32,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-09-23T20:48:09.789709+0000","last_change":"2023-09-23T20:47:09.231714+0000","last_active":"2023-09-23T20:48:09.789709+0000","last_peered":"2023-09-23T20:48:09.789709+0000","last_clean":"2023-09-23T20:48:09.789709+0000","last_became_active":"2023-09-23T20:47:09.230918+0000","last_became_peered":"2023-09-23T20:47:09.230918+0000","last_unstale":"2023-09-23T20:48:09.789709+0000","last_undegraded":"2023-09-23T20:48:09.789709+0000","last_fullsized":"2023-09-23T20:48:09.789709+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:24.583656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"575'9909","reported_seq":9963,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-09-23T20:48:18.233982+0000","last_change":"2023-09-23T20:45:39.488901+0000","last_active":"2023-09-23T20:48:18.233982+0000","last_peered":"2023-09-23T20:48:18.233982+0000","last_clean":"2023-09-23T20:48:18.233982+0000","last_became_active":"2023-09-23T20:45:39.488499+0000","last_became_peered":"2023-09-23T20:45:39.488499+0000","last_unstale":"2023-09-23T20:48:18.233982+0000","last_undegraded":"2023-09-23T20:48:18.233982+0000","last_fullsized":"2023-09-23T20:48:18.233982+0000","mapping_epoch":531,"log_start":"536'1000","ondisk_log_start":"536'1000","created":531,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":8909,"log_dups_size":0,"ondisk_log_size":8909,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:25:10.137770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":225378304,"num_objects":3439,"num_object_clones":0,"num_object_copies":10317,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3439,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9909,"num_write_kb":427136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":691,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-09-23T20:48:09.771948+0000","last_change":"2023-09-23T20:44:46.725779+0000","last_active":"2023-09-23T20:48:09.771948+0000","last_peered":"2023-09-23T20:48:09.771948+0000","last_clean":"2023-09-23T20:48:09.771948+0000","last_became_active":"2023-09-23T20:44:46.725433+0000","last_became_peered":"2023-09-23T20:44:46.725433+0000","last_unstale":"2023-09-23T20:48:09.771948+0000","last_undegraded":"2023-09-23T20:48:09.771948+0000","last_fullsized":"2023-09-23T20:48:09.771948+0000","mapping_epoch":516,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":517,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:42:45.492220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:42:45.492220+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:28:35.202407+0000","scrub_duration":0.015259412,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":200,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-09-23T20:48:09.795491+0000","last_change":"2023-09-23T20:47:09.178919+0000","last_active":"2023-09-23T20:48:09.795491+0000","last_peered":"2023-09-23T20:48:09.795491+0000","last_clean":"2023-09-23T20:48:09.795491+0000","last_became_active":"2023-09-23T20:47:09.178607+0000","last_became_peered":"2023-09-23T20:47:09.178607+0000","last_unstale":"2023-09-23T20:48:09.795491+0000","last_undegraded":"2023-09-23T20:48:09.795491+0000","last_fullsized":"2023-09-23T20:48:09.795491+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:23:45.196935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":32,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-09-23T20:48:09.790294+0000","last_change":"2023-09-23T20:47:09.186661+0000","last_active":"2023-09-23T20:48:09.790294+0000","last_peered":"2023-09-23T20:48:09.790294+0000","last_clean":"2023-09-23T20:48:09.790294+0000","last_became_active":"2023-09-23T20:47:09.186311+0000","last_became_peered":"2023-09-23T20:47:09.186311+0000","last_unstale":"2023-09-23T20:48:09.790294+0000","last_undegraded":"2023-09-23T20:48:09.790294+0000","last_fullsized":"2023-09-23T20:48:09.790294+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:18:51.382502+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.0","version":"575'9546","reported_seq":10786,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-09-23T20:48:15.433668+0000","last_change":"2023-09-23T20:47:15.653243+0000","last_active":"2023-09-23T20:48:15.433668+0000","last_peered":"2023-09-23T20:48:15.433668+0000","last_clean":"2023-09-23T20:48:15.433668+0000","last_became_active":"2023-09-23T20:47:15.626592+0000","last_became_peered":"2023-09-23T20:47:15.626592+0000","last_unstale":"2023-09-23T20:48:15.433668+0000","last_undegraded":"2023-09-23T20:48:15.433668+0000","last_fullsized":"2023-09-23T20:48:15.433668+0000","mapping_epoch":559,"log_start":"538'3149","ondisk_log_start":"538'3149","created":531,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":6397,"log_dups_size":0,"ondisk_log_size":6397,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:57:31.932707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":225705984,"num_objects":3444,"num_object_clones":0,"num_object_copies":10332,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3444,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9546,"num_write_kb":415680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":578,"num_bytes_recovered":37879808,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.b","version":"90'161","reported_seq":880,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-09-23T20:48:09.816465+0000","last_change":"2023-09-23T20:40:22.445808+0000","last_active":"2023-09-23T20:48:09.816465+0000","last_peered":"2023-09-23T20:48:09.816465+0000","last_clean":"2023-09-23T20:48:09.816465+0000","last_became_active":"2023-09-23T20:40:22.445456+0000","last_became_peered":"2023-09-23T20:40:22.445456+0000","last_unstale":"2023-09-23T20:48:09.816465+0000","last_undegraded":"2023-09-23T20:48:09.816465+0000","last_fullsized":"2023-09-23T20:48:09.816465+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:05.595042+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:05.595042+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:39:23.678154+0000","scrub_duration":0.007271675,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":32,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-09-23T20:48:09.789567+0000","last_change":"2023-09-23T20:47:09.183852+0000","last_active":"2023-09-23T20:48:09.789567+0000","last_peered":"2023-09-23T20:48:09.789567+0000","last_clean":"2023-09-23T20:48:09.789567+0000","last_became_active":"2023-09-23T20:47:09.183575+0000","last_became_peered":"2023-09-23T20:47:09.183575+0000","last_unstale":"2023-09-23T20:48:09.789567+0000","last_undegraded":"2023-09-23T20:48:09.789567+0000","last_fullsized":"2023-09-23T20:48:09.789567+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:40:30.918482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":32,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-09-23T20:48:09.790929+0000","last_change":"2023-09-23T20:47:09.186271+0000","last_active":"2023-09-23T20:48:09.790929+0000","last_peered":"2023-09-23T20:48:09.790929+0000","last_clean":"2023-09-23T20:48:09.790929+0000","last_became_active":"2023-09-23T20:47:09.184357+0000","last_became_peered":"2023-09-23T20:47:09.184357+0000","last_unstale":"2023-09-23T20:48:09.790929+0000","last_undegraded":"2023-09-23T20:48:09.790929+0000","last_fullsized":"2023-09-23T20:48:09.790929+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:48:26.339811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"575'9635","reported_seq":9729,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-09-23T20:48:15.738480+0000","last_change":"2023-09-23T20:47:10.208583+0000","last_active":"2023-09-23T20:48:15.738480+0000","last_peered":"2023-09-23T20:48:15.738480+0000","last_clean":"2023-09-23T20:48:15.738480+0000","last_became_active":"2023-09-23T20:47:10.171839+0000","last_became_peered":"2023-09-23T20:47:10.171839+0000","last_unstale":"2023-09-23T20:48:15.738480+0000","last_undegraded":"2023-09-23T20:48:15.738480+0000","last_fullsized":"2023-09-23T20:48:15.738480+0000","mapping_epoch":553,"log_start":"538'3824","ondisk_log_start":"538'3824","created":531,"last_epoch_clean":554,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":5811,"log_dups_size":0,"ondisk_log_size":5811,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:48:36.327572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":227344384,"num_objects":3469,"num_object_clones":0,"num_object_copies":10407,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3469,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9635,"num_write_kb":419328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":786432,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":602,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-09-23T20:48:09.776038+0000","last_change":"2023-09-23T20:43:51.693618+0000","last_active":"2023-09-23T20:48:09.776038+0000","last_peered":"2023-09-23T20:48:09.776038+0000","last_clean":"2023-09-23T20:48:09.776038+0000","last_became_active":"2023-09-23T20:43:16.765186+0000","last_became_peered":"2023-09-23T20:43:16.765186+0000","last_unstale":"2023-09-23T20:48:09.776038+0000","last_undegraded":"2023-09-23T20:48:09.776038+0000","last_fullsized":"2023-09-23T20:48:09.776038+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:43:51.693459+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:43:51.693459+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:33:30.208355+0000","scrub_duration":0.0016145269999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":371,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-09-23T20:48:09.766373+0000","last_change":"2023-09-23T20:37:43.570610+0000","last_active":"2023-09-23T20:48:09.766373+0000","last_peered":"2023-09-23T20:48:09.766373+0000","last_clean":"2023-09-23T20:48:09.766373+0000","last_became_active":"2023-09-23T20:37:43.570216+0000","last_became_peered":"2023-09-23T20:37:43.570216+0000","last_unstale":"2023-09-23T20:48:09.766373+0000","last_undegraded":"2023-09-23T20:48:09.766373+0000","last_fullsized":"2023-09-23T20:48:09.766373+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:39:26.555329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":420,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-09-23T20:48:09.815647+0000","last_change":"2023-09-23T20:47:09.172573+0000","last_active":"2023-09-23T20:48:09.815647+0000","last_peered":"2023-09-23T20:48:09.815647+0000","last_clean":"2023-09-23T20:48:09.815647+0000","last_became_active":"2023-09-23T20:47:09.172298+0000","last_became_peered":"2023-09-23T20:47:09.172298+0000","last_unstale":"2023-09-23T20:48:09.815647+0000","last_undegraded":"2023-09-23T20:48:09.815647+0000","last_fullsized":"2023-09-23T20:48:09.815647+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:22:09.361284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"575'9696","reported_seq":10330,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-09-23T20:48:15.351493+0000","last_change":"2023-09-23T20:47:11.626133+0000","last_active":"2023-09-23T20:48:15.351493+0000","last_peered":"2023-09-23T20:48:15.351493+0000","last_clean":"2023-09-23T20:48:15.351493+0000","last_became_active":"2023-09-23T20:47:11.619197+0000","last_became_peered":"2023-09-23T20:47:11.619197+0000","last_unstale":"2023-09-23T20:48:15.351493+0000","last_undegraded":"2023-09-23T20:48:15.351493+0000","last_fullsized":"2023-09-23T20:48:15.351493+0000","mapping_epoch":555,"log_start":"538'3293","ondisk_log_start":"538'3293","created":531,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":6403,"log_dups_size":0,"ondisk_log_size":6403,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:33:18.084175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":232783872,"num_objects":3552,"num_object_clones":0,"num_object_copies":10656,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3552,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9696,"num_write_kb":423936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":277,"num_bytes_recovered":18153472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":763,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-09-23T20:48:09.798732+0000","last_change":"2023-09-23T20:41:07.424609+0000","last_active":"2023-09-23T20:48:09.798732+0000","last_peered":"2023-09-23T20:48:09.798732+0000","last_clean":"2023-09-23T20:48:09.798732+0000","last_became_active":"2023-09-23T20:38:05.622261+0000","last_became_peered":"2023-09-23T20:38:05.622261+0000","last_unstale":"2023-09-23T20:48:09.798732+0000","last_undegraded":"2023-09-23T20:48:09.798732+0000","last_fullsized":"2023-09-23T20:48:09.798732+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:41:07.424492+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:41:07.424492+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:45:21.689123+0000","scrub_duration":0.0018960959999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"86'119","reported_seq":886,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-09-23T20:48:09.814647+0000","last_change":"2023-09-23T20:29:45.521257+0000","last_active":"2023-09-23T20:48:09.814647+0000","last_peered":"2023-09-23T20:48:09.814647+0000","last_clean":"2023-09-23T20:48:09.814647+0000","last_became_active":"2023-09-23T20:29:45.520546+0000","last_became_peered":"2023-09-23T20:29:45.520546+0000","last_unstale":"2023-09-23T20:48:09.814647+0000","last_undegraded":"2023-09-23T20:48:09.814647+0000","last_fullsized":"2023-09-23T20:48:09.814647+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":119,"log_dups_size":0,"ondisk_log_size":119,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:58:56.085777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032705499999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":523,"num_read_kb":3503,"num_write":136,"num_write_kb":3128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":369,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-09-23T20:48:09.767113+0000","last_change":"2023-09-23T20:43:16.756494+0000","last_active":"2023-09-23T20:48:09.767113+0000","last_peered":"2023-09-23T20:48:09.767113+0000","last_clean":"2023-09-23T20:48:09.767113+0000","last_became_active":"2023-09-23T20:43:16.756131+0000","last_became_peered":"2023-09-23T20:43:16.756131+0000","last_unstale":"2023-09-23T20:48:09.767113+0000","last_undegraded":"2023-09-23T20:48:09.767113+0000","last_fullsized":"2023-09-23T20:48:09.767113+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:10:19.219533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":551,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-09-23T20:48:09.815127+0000","last_change":"2023-09-23T20:47:09.194947+0000","last_active":"2023-09-23T20:48:09.815127+0000","last_peered":"2023-09-23T20:48:09.815127+0000","last_clean":"2023-09-23T20:48:09.815127+0000","last_became_active":"2023-09-23T20:47:09.194592+0000","last_became_peered":"2023-09-23T20:47:09.194592+0000","last_unstale":"2023-09-23T20:48:09.815127+0000","last_undegraded":"2023-09-23T20:48:09.815127+0000","last_fullsized":"2023-09-23T20:48:09.815127+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:34:15.008492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"575'9336","reported_seq":9388,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-09-23T20:48:15.431751+0000","last_change":"2023-09-23T20:45:39.489110+0000","last_active":"2023-09-23T20:48:15.431751+0000","last_peered":"2023-09-23T20:48:15.431751+0000","last_clean":"2023-09-23T20:48:15.431751+0000","last_became_active":"2023-09-23T20:45:39.488817+0000","last_became_peered":"2023-09-23T20:45:39.488817+0000","last_unstale":"2023-09-23T20:48:15.431751+0000","last_undegraded":"2023-09-23T20:48:15.431751+0000","last_fullsized":"2023-09-23T20:48:15.431751+0000","mapping_epoch":531,"log_start":"538'2925","ondisk_log_start":"538'2925","created":531,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":6411,"log_dups_size":0,"ondisk_log_size":6411,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:07:21.467971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":222953472,"num_objects":3402,"num_object_clones":0,"num_object_copies":10206,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3402,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9336,"num_write_kb":407616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":622,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-09-23T20:48:09.780124+0000","last_change":"2023-09-23T20:44:15.656984+0000","last_active":"2023-09-23T20:48:09.780124+0000","last_peered":"2023-09-23T20:48:09.780124+0000","last_clean":"2023-09-23T20:48:09.780124+0000","last_became_active":"2023-09-23T20:43:46.512210+0000","last_became_peered":"2023-09-23T20:43:46.512210+0000","last_unstale":"2023-09-23T20:48:09.780124+0000","last_undegraded":"2023-09-23T20:48:09.780124+0000","last_fullsized":"2023-09-23T20:48:09.780124+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:15.656880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:15.656880+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:37:56.453466+0000","scrub_duration":0.0016711180000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"86'106","reported_seq":827,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-09-23T20:48:09.815421+0000","last_change":"2023-09-23T20:43:16.759138+0000","last_active":"2023-09-23T20:48:09.815421+0000","last_peered":"2023-09-23T20:48:09.815421+0000","last_clean":"2023-09-23T20:48:09.815421+0000","last_became_active":"2023-09-23T20:43:16.758015+0000","last_became_peered":"2023-09-23T20:43:16.758015+0000","last_unstale":"2023-09-23T20:48:09.815421+0000","last_undegraded":"2023-09-23T20:48:09.815421+0000","last_fullsized":"2023-09-23T20:48:09.815421+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":106,"log_dups_size":0,"ondisk_log_size":106,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:08:02.852472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00099302599999999993,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1094,"num_read_kb":1678,"num_write":100,"num_write_kb":404,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":32,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-09-23T20:48:09.790063+0000","last_change":"2023-09-23T20:47:09.177872+0000","last_active":"2023-09-23T20:48:09.790063+0000","last_peered":"2023-09-23T20:48:09.790063+0000","last_clean":"2023-09-23T20:48:09.790063+0000","last_became_active":"2023-09-23T20:47:09.176689+0000","last_became_peered":"2023-09-23T20:47:09.176689+0000","last_unstale":"2023-09-23T20:48:09.790063+0000","last_undegraded":"2023-09-23T20:48:09.790063+0000","last_fullsized":"2023-09-23T20:48:09.790063+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:14:01.590318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":539,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-09-23T20:48:09.814571+0000","last_change":"2023-09-23T20:29:45.533406+0000","last_active":"2023-09-23T20:48:09.814571+0000","last_peered":"2023-09-23T20:48:09.814571+0000","last_clean":"2023-09-23T20:48:09.814571+0000","last_became_active":"2023-09-23T20:29:45.526001+0000","last_became_peered":"2023-09-23T20:29:45.526001+0000","last_unstale":"2023-09-23T20:48:09.814571+0000","last_undegraded":"2023-09-23T20:48:09.814571+0000","last_fullsized":"2023-09-23T20:48:09.814571+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:43.787251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.9","version":"575'9801","reported_seq":9853,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-09-23T20:48:18.257017+0000","last_change":"2023-09-23T20:45:39.488721+0000","last_active":"2023-09-23T20:48:18.257017+0000","last_peered":"2023-09-23T20:48:18.257017+0000","last_clean":"2023-09-23T20:48:18.257017+0000","last_became_active":"2023-09-23T20:45:39.488320+0000","last_became_peered":"2023-09-23T20:45:39.488320+0000","last_unstale":"2023-09-23T20:48:18.257017+0000","last_undegraded":"2023-09-23T20:48:18.257017+0000","last_fullsized":"2023-09-23T20:48:18.257017+0000","mapping_epoch":531,"log_start":"536'900","ondisk_log_start":"536'900","created":531,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":8901,"log_dups_size":0,"ondisk_log_size":8901,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:46:30.536848+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":223150080,"num_objects":3405,"num_object_clones":0,"num_object_copies":10215,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3405,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9801,"num_write_kb":422592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":578,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-09-23T20:48:09.783569+0000","last_change":"2023-09-23T20:47:11.618652+0000","last_active":"2023-09-23T20:48:09.783569+0000","last_peered":"2023-09-23T20:48:09.783569+0000","last_clean":"2023-09-23T20:48:09.783569+0000","last_became_active":"2023-09-23T20:47:11.618266+0000","last_became_peered":"2023-09-23T20:47:11.618266+0000","last_unstale":"2023-09-23T20:48:09.783569+0000","last_undegraded":"2023-09-23T20:48:09.783569+0000","last_fullsized":"2023-09-23T20:48:09.783569+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:26.978179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:26.978179+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:29:29.000683+0000","scrub_duration":0.0013962829999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"86'130","reported_seq":894,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-09-23T20:48:09.774023+0000","last_change":"2023-09-23T20:25:39.637458+0000","last_active":"2023-09-23T20:48:09.774023+0000","last_peered":"2023-09-23T20:48:09.774023+0000","last_clean":"2023-09-23T20:48:09.774023+0000","last_became_active":"2023-09-23T20:25:39.636773+0000","last_became_peered":"2023-09-23T20:25:39.636773+0000","last_unstale":"2023-09-23T20:48:09.774023+0000","last_undegraded":"2023-09-23T20:48:09.774023+0000","last_fullsized":"2023-09-23T20:48:09.774023+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:00:32.810564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011300049999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":512,"num_read_kb":1820,"num_write":133,"num_write_kb":1347,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":548,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-09-23T20:48:09.816296+0000","last_change":"2023-09-23T20:38:02.591335+0000","last_active":"2023-09-23T20:48:09.816296+0000","last_peered":"2023-09-23T20:48:09.816296+0000","last_clean":"2023-09-23T20:48:09.816296+0000","last_became_active":"2023-09-23T20:38:02.591015+0000","last_became_peered":"2023-09-23T20:38:02.591015+0000","last_unstale":"2023-09-23T20:48:09.816296+0000","last_undegraded":"2023-09-23T20:48:09.816296+0000","last_fullsized":"2023-09-23T20:48:09.816296+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:55:32.272195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":90,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-09-23T20:48:09.815563+0000","last_change":"2023-09-23T20:44:10.555583+0000","last_active":"2023-09-23T20:48:09.815563+0000","last_peered":"2023-09-23T20:48:09.815563+0000","last_clean":"2023-09-23T20:48:09.815563+0000","last_became_active":"2023-09-23T20:44:10.555281+0000","last_became_peered":"2023-09-23T20:44:10.555281+0000","last_unstale":"2023-09-23T20:48:09.815563+0000","last_undegraded":"2023-09-23T20:48:09.815563+0000","last_fullsized":"2023-09-23T20:48:09.815563+0000","mapping_epoch":496,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":497,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:43:11.942147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"575'9621","reported_seq":9671,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-09-23T20:48:15.419661+0000","last_change":"2023-09-23T20:45:39.331096+0000","last_active":"2023-09-23T20:48:15.419661+0000","last_peered":"2023-09-23T20:48:15.419661+0000","last_clean":"2023-09-23T20:48:15.419661+0000","last_became_active":"2023-09-23T20:45:39.330752+0000","last_became_peered":"2023-09-23T20:45:39.330752+0000","last_unstale":"2023-09-23T20:48:15.419661+0000","last_undegraded":"2023-09-23T20:48:15.419661+0000","last_fullsized":"2023-09-23T20:48:15.419661+0000","mapping_epoch":531,"log_start":"538'3171","ondisk_log_start":"538'3171","created":531,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":6450,"log_dups_size":0,"ondisk_log_size":6450,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:39:01.627762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":228524032,"num_objects":3487,"num_object_clones":0,"num_object_copies":10461,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3487,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9621,"num_write_kb":419456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"86'194","reported_seq":933,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-09-23T20:48:09.775519+0000","last_change":"2023-09-23T20:43:16.765865+0000","last_active":"2023-09-23T20:48:09.775519+0000","last_peered":"2023-09-23T20:48:09.775519+0000","last_clean":"2023-09-23T20:48:09.775519+0000","last_became_active":"2023-09-23T20:43:16.765289+0000","last_became_peered":"2023-09-23T20:43:16.765289+0000","last_unstale":"2023-09-23T20:48:09.775519+0000","last_undegraded":"2023-09-23T20:48:09.775519+0000","last_fullsized":"2023-09-23T20:48:09.775519+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":194,"log_dups_size":0,"ondisk_log_size":194,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:15:16.461900+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00037995999999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1024,"num_read_kb":1200,"num_write":173,"num_write_kb":1276,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":1037,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-09-23T20:48:09.783262+0000","last_change":"2023-09-23T20:38:05.602508+0000","last_active":"2023-09-23T20:48:09.783262+0000","last_peered":"2023-09-23T20:48:09.783262+0000","last_clean":"2023-09-23T20:48:09.783262+0000","last_became_active":"2023-09-23T20:38:05.602129+0000","last_became_peered":"2023-09-23T20:38:05.602129+0000","last_unstale":"2023-09-23T20:48:09.783262+0000","last_undegraded":"2023-09-23T20:48:09.783262+0000","last_fullsized":"2023-09-23T20:48:09.783262+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:45.343742+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:45.343742+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:48.910343+0000","scrub_duration":0.001475802,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":150,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-09-23T20:48:09.795876+0000","last_change":"2023-09-23T20:43:16.758213+0000","last_active":"2023-09-23T20:48:09.795876+0000","last_peered":"2023-09-23T20:48:09.795876+0000","last_clean":"2023-09-23T20:48:09.795876+0000","last_became_active":"2023-09-23T20:43:16.757869+0000","last_became_peered":"2023-09-23T20:43:16.757869+0000","last_unstale":"2023-09-23T20:48:09.795876+0000","last_undegraded":"2023-09-23T20:48:09.795876+0000","last_fullsized":"2023-09-23T20:48:09.795876+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:43:10.811749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":200,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-09-23T20:48:09.795761+0000","last_change":"2023-09-23T20:47:09.202302+0000","last_active":"2023-09-23T20:48:09.795761+0000","last_peered":"2023-09-23T20:48:09.795761+0000","last_clean":"2023-09-23T20:48:09.795761+0000","last_became_active":"2023-09-23T20:47:09.201797+0000","last_became_peered":"2023-09-23T20:47:09.201797+0000","last_unstale":"2023-09-23T20:48:09.795761+0000","last_undegraded":"2023-09-23T20:48:09.795761+0000","last_fullsized":"2023-09-23T20:48:09.795761+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:34:41.055705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.8","version":"575'9546","reported_seq":9600,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-09-23T20:48:16.531226+0000","last_change":"2023-09-23T20:45:39.332075+0000","last_active":"2023-09-23T20:48:16.531226+0000","last_peered":"2023-09-23T20:48:16.531226+0000","last_clean":"2023-09-23T20:48:16.531226+0000","last_became_active":"2023-09-23T20:45:39.331755+0000","last_became_peered":"2023-09-23T20:45:39.331755+0000","last_unstale":"2023-09-23T20:48:16.531226+0000","last_undegraded":"2023-09-23T20:48:16.531226+0000","last_fullsized":"2023-09-23T20:48:16.531226+0000","mapping_epoch":531,"log_start":"537'2961","ondisk_log_start":"537'2961","created":531,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":6585,"log_dups_size":0,"ondisk_log_size":6585,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:11:17.895002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":216793088,"num_objects":3308,"num_object_clones":0,"num_object_copies":9924,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3308,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9546,"num_write_kb":411328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":942,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-09-23T20:48:09.816236+0000","last_change":"2023-09-23T20:47:11.622262+0000","last_active":"2023-09-23T20:48:09.816236+0000","last_peered":"2023-09-23T20:48:09.816236+0000","last_clean":"2023-09-23T20:48:09.816236+0000","last_became_active":"2023-09-23T20:47:11.621920+0000","last_became_peered":"2023-09-23T20:47:11.621920+0000","last_unstale":"2023-09-23T20:48:09.816236+0000","last_undegraded":"2023-09-23T20:48:09.816236+0000","last_fullsized":"2023-09-23T20:48:09.816236+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:43.582146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:43.582146+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:01:15.153543+0000","scrub_duration":0.0016219730000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"86'80","reported_seq":827,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-09-23T20:48:09.783261+0000","last_change":"2023-09-23T20:29:45.536796+0000","last_active":"2023-09-23T20:48:09.783261+0000","last_peered":"2023-09-23T20:48:09.783261+0000","last_clean":"2023-09-23T20:48:09.783261+0000","last_became_active":"2023-09-23T20:29:45.536439+0000","last_became_peered":"2023-09-23T20:29:45.536439+0000","last_unstale":"2023-09-23T20:48:09.783261+0000","last_undegraded":"2023-09-23T20:48:09.783261+0000","last_fullsized":"2023-09-23T20:48:09.783261+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":80,"log_dups_size":0,"ondisk_log_size":80,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:30:57.793132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035115,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":395,"num_read_kb":1370,"num_write":66,"num_write_kb":1061,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":587,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-09-23T20:48:09.780022+0000","last_change":"2023-09-23T20:47:09.198326+0000","last_active":"2023-09-23T20:48:09.780022+0000","last_peered":"2023-09-23T20:48:09.780022+0000","last_clean":"2023-09-23T20:48:09.780022+0000","last_became_active":"2023-09-23T20:47:09.198059+0000","last_became_peered":"2023-09-23T20:47:09.198059+0000","last_unstale":"2023-09-23T20:48:09.780022+0000","last_undegraded":"2023-09-23T20:48:09.780022+0000","last_fullsized":"2023-09-23T20:48:09.780022+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:38:53.903706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":563,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-09-23T20:48:09.775554+0000","last_change":"2023-09-23T20:43:46.520386+0000","last_active":"2023-09-23T20:48:09.775554+0000","last_peered":"2023-09-23T20:48:09.775554+0000","last_clean":"2023-09-23T20:48:09.775554+0000","last_became_active":"2023-09-23T20:43:46.519757+0000","last_became_peered":"2023-09-23T20:43:46.519757+0000","last_unstale":"2023-09-23T20:48:09.775554+0000","last_undegraded":"2023-09-23T20:48:09.775554+0000","last_fullsized":"2023-09-23T20:48:09.775554+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:23:14.662734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"575'9407","reported_seq":10085,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-09-23T20:48:15.427844+0000","last_change":"2023-09-23T20:47:13.668055+0000","last_active":"2023-09-23T20:48:15.427844+0000","last_peered":"2023-09-23T20:48:15.427844+0000","last_clean":"2023-09-23T20:48:15.427844+0000","last_became_active":"2023-09-23T20:47:13.626170+0000","last_became_peered":"2023-09-23T20:47:13.626170+0000","last_unstale":"2023-09-23T20:48:15.427844+0000","last_undegraded":"2023-09-23T20:48:15.427844+0000","last_fullsized":"2023-09-23T20:48:15.427844+0000","mapping_epoch":557,"log_start":"538'2957","ondisk_log_start":"538'2957","created":531,"last_epoch_clean":558,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":6450,"log_dups_size":0,"ondisk_log_size":6450,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:32.805547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":228589592,"num_objects":3489,"num_object_clones":0,"num_object_copies":10467,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3489,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9407,"num_write_kb":412609,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":298,"num_bytes_recovered":19529728,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":686,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-09-23T20:48:09.795947+0000","last_change":"2023-09-23T20:41:08.380084+0000","last_active":"2023-09-23T20:48:09.795947+0000","last_peered":"2023-09-23T20:48:09.795947+0000","last_clean":"2023-09-23T20:48:09.795947+0000","last_became_active":"2023-09-23T20:38:05.610881+0000","last_became_peered":"2023-09-23T20:38:05.610881+0000","last_unstale":"2023-09-23T20:48:09.795947+0000","last_undegraded":"2023-09-23T20:48:09.795947+0000","last_fullsized":"2023-09-23T20:48:09.795947+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:41:08.379959+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:41:08.379959+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:19:39.547923+0000","scrub_duration":0.0019158420000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"86'102","reported_seq":1245,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-09-23T20:48:09.783753+0000","last_change":"2023-09-23T20:47:14.628795+0000","last_active":"2023-09-23T20:48:09.783753+0000","last_peered":"2023-09-23T20:48:09.783753+0000","last_clean":"2023-09-23T20:48:09.783753+0000","last_became_active":"2023-09-23T20:47:14.628341+0000","last_became_peered":"2023-09-23T20:47:14.628341+0000","last_unstale":"2023-09-23T20:48:09.783753+0000","last_undegraded":"2023-09-23T20:48:09.783753+0000","last_fullsized":"2023-09-23T20:48:09.783753+0000","mapping_epoch":558,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":559,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":102,"log_dups_size":0,"ondisk_log_size":102,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:42:16.291849+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030656500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":963,"num_read_kb":1234,"num_write":60,"num_write_kb":225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":523,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-09-23T20:48:09.814651+0000","last_change":"2023-09-23T20:40:19.544384+0000","last_active":"2023-09-23T20:48:09.814651+0000","last_peered":"2023-09-23T20:48:09.814651+0000","last_clean":"2023-09-23T20:48:09.814651+0000","last_became_active":"2023-09-23T20:40:19.544079+0000","last_became_peered":"2023-09-23T20:40:19.544079+0000","last_unstale":"2023-09-23T20:48:09.814651+0000","last_undegraded":"2023-09-23T20:48:09.814651+0000","last_fullsized":"2023-09-23T20:48:09.814651+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:00:19.498631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":519,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-09-23T20:48:09.783939+0000","last_change":"2023-09-23T20:43:46.541173+0000","last_active":"2023-09-23T20:48:09.783939+0000","last_peered":"2023-09-23T20:48:09.783939+0000","last_clean":"2023-09-23T20:48:09.783939+0000","last_became_active":"2023-09-23T20:43:46.540726+0000","last_became_peered":"2023-09-23T20:43:46.540726+0000","last_unstale":"2023-09-23T20:48:09.783939+0000","last_undegraded":"2023-09-23T20:48:09.783939+0000","last_fullsized":"2023-09-23T20:48:09.783939+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:49:04.137885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.e","version":"575'9547","reported_seq":9722,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-09-23T20:48:15.767415+0000","last_change":"2023-09-23T20:47:10.174122+0000","last_active":"2023-09-23T20:48:15.767415+0000","last_peered":"2023-09-23T20:48:15.767415+0000","last_clean":"2023-09-23T20:48:15.767415+0000","last_became_active":"2023-09-23T20:47:10.167795+0000","last_became_peered":"2023-09-23T20:47:10.167795+0000","last_unstale":"2023-09-23T20:48:15.767415+0000","last_undegraded":"2023-09-23T20:48:15.767415+0000","last_fullsized":"2023-09-23T20:48:15.767415+0000","mapping_epoch":553,"log_start":"538'3744","ondisk_log_start":"538'3744","created":531,"last_epoch_clean":554,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":5803,"log_dups_size":0,"ondisk_log_size":5803,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:44:03.693340+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":226164736,"num_objects":3451,"num_object_clones":0,"num_object_copies":10353,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3451,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9547,"num_write_kb":415936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":50,"num_bytes_recovered":3276800,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":572,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-09-23T20:48:09.790474+0000","last_change":"2023-09-23T20:47:34.561810+0000","last_active":"2023-09-23T20:48:09.790474+0000","last_peered":"2023-09-23T20:48:09.790474+0000","last_clean":"2023-09-23T20:48:09.790474+0000","last_became_active":"2023-09-23T20:47:11.624953+0000","last_became_peered":"2023-09-23T20:47:11.624953+0000","last_unstale":"2023-09-23T20:48:09.790474+0000","last_undegraded":"2023-09-23T20:48:09.790474+0000","last_fullsized":"2023-09-23T20:48:09.790474+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:47:34.561747+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:47:34.561747+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:11:25.458764+0000","scrub_duration":0.0087949929999999992,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":312,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-09-23T20:48:09.784352+0000","last_change":"2023-09-23T20:32:50.549695+0000","last_active":"2023-09-23T20:48:09.784352+0000","last_peered":"2023-09-23T20:48:09.784352+0000","last_clean":"2023-09-23T20:48:09.784352+0000","last_became_active":"2023-09-23T20:32:50.549431+0000","last_became_peered":"2023-09-23T20:32:50.549431+0000","last_unstale":"2023-09-23T20:48:09.784352+0000","last_undegraded":"2023-09-23T20:48:09.784352+0000","last_fullsized":"2023-09-23T20:48:09.784352+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:45:53.666569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"86'128","reported_seq":1055,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-09-23T20:48:09.815357+0000","last_change":"2023-09-23T20:47:11.621846+0000","last_active":"2023-09-23T20:48:09.815357+0000","last_peered":"2023-09-23T20:48:09.815357+0000","last_clean":"2023-09-23T20:48:09.815357+0000","last_became_active":"2023-09-23T20:47:11.621284+0000","last_became_peered":"2023-09-23T20:47:11.621284+0000","last_unstale":"2023-09-23T20:48:09.815357+0000","last_undegraded":"2023-09-23T20:48:09.815357+0000","last_fullsized":"2023-09-23T20:48:09.815357+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:43:50.658512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000293202,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":579,"num_read_kb":1473,"num_write":96,"num_write_kb":1062,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":357,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-09-23T20:48:09.766288+0000","last_change":"2023-09-23T20:37:43.582105+0000","last_active":"2023-09-23T20:48:09.766288+0000","last_peered":"2023-09-23T20:48:09.766288+0000","last_clean":"2023-09-23T20:48:09.766288+0000","last_became_active":"2023-09-23T20:37:43.581860+0000","last_became_peered":"2023-09-23T20:37:43.581860+0000","last_unstale":"2023-09-23T20:48:09.766288+0000","last_undegraded":"2023-09-23T20:48:09.766288+0000","last_fullsized":"2023-09-23T20:48:09.766288+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:18:38.765486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"575'9511","reported_seq":9563,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-09-23T20:48:15.445127+0000","last_change":"2023-09-23T20:45:39.381283+0000","last_active":"2023-09-23T20:48:15.445127+0000","last_peered":"2023-09-23T20:48:15.445127+0000","last_clean":"2023-09-23T20:48:15.445127+0000","last_became_active":"2023-09-23T20:45:39.380945+0000","last_became_peered":"2023-09-23T20:45:39.380945+0000","last_unstale":"2023-09-23T20:48:15.445127+0000","last_undegraded":"2023-09-23T20:48:15.445127+0000","last_fullsized":"2023-09-23T20:48:15.445127+0000","mapping_epoch":531,"log_start":"538'3073","ondisk_log_start":"538'3073","created":531,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":6438,"log_dups_size":0,"ondisk_log_size":6438,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:04:01.529754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":229048320,"num_objects":3495,"num_object_clones":0,"num_object_copies":10485,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3495,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9511,"num_write_kb":416192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.4","version":"90'161","reported_seq":791,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-09-23T20:48:09.770862+0000","last_change":"2023-09-23T20:40:22.427435+0000","last_active":"2023-09-23T20:48:09.770862+0000","last_peered":"2023-09-23T20:48:09.770862+0000","last_clean":"2023-09-23T20:48:09.770862+0000","last_became_active":"2023-09-23T20:40:22.427051+0000","last_became_peered":"2023-09-23T20:40:22.427051+0000","last_unstale":"2023-09-23T20:48:09.770862+0000","last_undegraded":"2023-09-23T20:48:09.770862+0000","last_fullsized":"2023-09-23T20:48:09.770862+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:34:54.504554+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:34:54.504554+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:42.001652+0000","scrub_duration":0.011176864,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"86'117","reported_seq":883,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-09-23T20:48:09.798645+0000","last_change":"2023-09-23T20:44:18.635224+0000","last_active":"2023-09-23T20:48:09.798645+0000","last_peered":"2023-09-23T20:48:09.798645+0000","last_clean":"2023-09-23T20:48:09.798645+0000","last_became_active":"2023-09-23T20:44:18.634703+0000","last_became_peered":"2023-09-23T20:44:18.634703+0000","last_unstale":"2023-09-23T20:48:09.798645+0000","last_undegraded":"2023-09-23T20:48:09.798645+0000","last_fullsized":"2023-09-23T20:48:09.798645+0000","mapping_epoch":502,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":503,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:59:38.030626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046891400000000002,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":893,"num_read_kb":922,"num_write":82,"num_write_kb":4193,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":16777216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":589,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-09-23T20:48:09.775631+0000","last_change":"2023-09-23T20:47:09.317116+0000","last_active":"2023-09-23T20:48:09.775631+0000","last_peered":"2023-09-23T20:48:09.775631+0000","last_clean":"2023-09-23T20:48:09.775631+0000","last_became_active":"2023-09-23T20:47:09.205930+0000","last_became_peered":"2023-09-23T20:47:09.205930+0000","last_unstale":"2023-09-23T20:48:09.775631+0000","last_undegraded":"2023-09-23T20:48:09.775631+0000","last_fullsized":"2023-09-23T20:48:09.775631+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:04:55.528625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":32,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-09-23T20:48:09.789644+0000","last_change":"2023-09-23T20:47:09.192117+0000","last_active":"2023-09-23T20:48:09.789644+0000","last_peered":"2023-09-23T20:48:09.789644+0000","last_clean":"2023-09-23T20:48:09.789644+0000","last_became_active":"2023-09-23T20:47:09.191820+0000","last_became_peered":"2023-09-23T20:47:09.191820+0000","last_unstale":"2023-09-23T20:48:09.789644+0000","last_undegraded":"2023-09-23T20:48:09.789644+0000","last_fullsized":"2023-09-23T20:48:09.789644+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:08.785173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"90'161","reported_seq":832,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-09-23T20:48:09.779708+0000","last_change":"2023-09-23T20:43:46.544750+0000","last_active":"2023-09-23T20:48:09.779708+0000","last_peered":"2023-09-23T20:48:09.779708+0000","last_clean":"2023-09-23T20:48:09.779708+0000","last_became_active":"2023-09-23T20:43:46.544395+0000","last_became_peered":"2023-09-23T20:43:46.544395+0000","last_unstale":"2023-09-23T20:48:09.779708+0000","last_undegraded":"2023-09-23T20:48:09.779708+0000","last_fullsized":"2023-09-23T20:48:09.779708+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:06.487351+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:06.487351+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:56.866168+0000","scrub_duration":0.001964379,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":548,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-09-23T20:48:09.783861+0000","last_change":"2023-09-23T20:47:09.197601+0000","last_active":"2023-09-23T20:48:09.783861+0000","last_peered":"2023-09-23T20:48:09.783861+0000","last_clean":"2023-09-23T20:48:09.783861+0000","last_became_active":"2023-09-23T20:47:09.196967+0000","last_became_peered":"2023-09-23T20:47:09.196967+0000","last_unstale":"2023-09-23T20:48:09.783861+0000","last_undegraded":"2023-09-23T20:48:09.783861+0000","last_fullsized":"2023-09-23T20:48:09.783861+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:06:56.826785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":433,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-09-23T20:48:11.819591+0000","last_change":"2023-09-23T20:29:45.538314+0000","last_active":"2023-09-23T20:48:11.819591+0000","last_peered":"2023-09-23T20:48:11.819591+0000","last_clean":"2023-09-23T20:48:11.819591+0000","last_became_active":"2023-09-23T20:29:45.536847+0000","last_became_peered":"2023-09-23T20:29:45.536847+0000","last_unstale":"2023-09-23T20:48:11.819591+0000","last_undegraded":"2023-09-23T20:48:11.819591+0000","last_fullsized":"2023-09-23T20:48:11.819591+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:51.948721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"90'161","reported_seq":650,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-09-23T20:48:09.769648+0000","last_change":"2023-09-23T20:41:06.499511+0000","last_active":"2023-09-23T20:48:09.769648+0000","last_peered":"2023-09-23T20:48:09.769648+0000","last_clean":"2023-09-23T20:48:09.769648+0000","last_became_active":"2023-09-23T20:38:05.601874+0000","last_became_peered":"2023-09-23T20:38:05.601874+0000","last_unstale":"2023-09-23T20:48:09.769648+0000","last_undegraded":"2023-09-23T20:48:09.769648+0000","last_fullsized":"2023-09-23T20:48:09.769648+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:41:06.499396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:41:06.499396+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:31:34.059077+0000","scrub_duration":0.0019670310000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":36,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-09-23T20:48:09.789925+0000","last_change":"2023-09-23T20:47:09.332980+0000","last_active":"2023-09-23T20:48:09.789925+0000","last_peered":"2023-09-23T20:48:09.789925+0000","last_clean":"2023-09-23T20:48:09.789925+0000","last_became_active":"2023-09-23T20:47:09.230554+0000","last_became_peered":"2023-09-23T20:47:09.230554+0000","last_unstale":"2023-09-23T20:48:09.789925+0000","last_undegraded":"2023-09-23T20:48:09.789925+0000","last_fullsized":"2023-09-23T20:48:09.789925+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:06:21.882823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":32,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-09-23T20:48:09.790030+0000","last_change":"2023-09-23T20:47:09.186579+0000","last_active":"2023-09-23T20:48:09.790030+0000","last_peered":"2023-09-23T20:48:09.790030+0000","last_clean":"2023-09-23T20:48:09.790030+0000","last_became_active":"2023-09-23T20:47:09.186227+0000","last_became_peered":"2023-09-23T20:47:09.186227+0000","last_unstale":"2023-09-23T20:48:09.790030+0000","last_undegraded":"2023-09-23T20:48:09.790030+0000","last_fullsized":"2023-09-23T20:48:09.790030+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:45:39.485308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":36,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-09-23T20:48:09.790391+0000","last_change":"2023-09-23T20:47:09.316824+0000","last_active":"2023-09-23T20:48:09.790391+0000","last_peered":"2023-09-23T20:48:09.790391+0000","last_clean":"2023-09-23T20:48:09.790391+0000","last_became_active":"2023-09-23T20:47:09.195100+0000","last_became_peered":"2023-09-23T20:47:09.195100+0000","last_unstale":"2023-09-23T20:48:09.790391+0000","last_undegraded":"2023-09-23T20:48:09.790391+0000","last_fullsized":"2023-09-23T20:48:09.790391+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:29:53.273554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":32,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-09-23T20:48:09.789637+0000","last_change":"2023-09-23T20:47:09.181240+0000","last_active":"2023-09-23T20:48:09.789637+0000","last_peered":"2023-09-23T20:48:09.789637+0000","last_clean":"2023-09-23T20:48:09.789637+0000","last_became_active":"2023-09-23T20:47:09.181022+0000","last_became_peered":"2023-09-23T20:47:09.181022+0000","last_unstale":"2023-09-23T20:48:09.789637+0000","last_undegraded":"2023-09-23T20:48:09.789637+0000","last_fullsized":"2023-09-23T20:48:09.789637+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:38:59.191781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"90'161","reported_seq":830,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-09-23T20:48:09.784010+0000","last_change":"2023-09-23T20:38:05.591992+0000","last_active":"2023-09-23T20:48:09.784010+0000","last_peered":"2023-09-23T20:48:09.784010+0000","last_clean":"2023-09-23T20:48:09.784010+0000","last_became_active":"2023-09-23T20:38:05.591701+0000","last_became_peered":"2023-09-23T20:38:05.591701+0000","last_unstale":"2023-09-23T20:48:09.784010+0000","last_undegraded":"2023-09-23T20:48:09.784010+0000","last_fullsized":"2023-09-23T20:48:09.784010+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:02.384133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:02.384133+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:41.722542+0000","scrub_duration":0.001769098,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":399,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-09-23T20:48:09.773592+0000","last_change":"2023-09-23T20:38:02.542719+0000","last_active":"2023-09-23T20:48:09.773592+0000","last_peered":"2023-09-23T20:48:09.773592+0000","last_clean":"2023-09-23T20:48:09.773592+0000","last_became_active":"2023-09-23T20:38:02.542385+0000","last_became_peered":"2023-09-23T20:38:02.542385+0000","last_unstale":"2023-09-23T20:48:09.773592+0000","last_undegraded":"2023-09-23T20:48:09.773592+0000","last_fullsized":"2023-09-23T20:48:09.773592+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:36:23.673757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":365,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-09-23T20:48:09.766829+0000","last_change":"2023-09-23T20:37:43.584253+0000","last_active":"2023-09-23T20:48:09.766829+0000","last_peered":"2023-09-23T20:48:09.766829+0000","last_clean":"2023-09-23T20:48:09.766829+0000","last_became_active":"2023-09-23T20:37:43.580069+0000","last_became_peered":"2023-09-23T20:37:43.580069+0000","last_unstale":"2023-09-23T20:48:09.766829+0000","last_undegraded":"2023-09-23T20:48:09.766829+0000","last_fullsized":"2023-09-23T20:48:09.766829+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:39:11.482581+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"90'161","reported_seq":704,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-09-23T20:48:09.814960+0000","last_change":"2023-09-23T20:47:11.621758+0000","last_active":"2023-09-23T20:48:09.814960+0000","last_peered":"2023-09-23T20:48:09.814960+0000","last_clean":"2023-09-23T20:48:09.814960+0000","last_became_active":"2023-09-23T20:47:11.621345+0000","last_became_peered":"2023-09-23T20:47:11.621345+0000","last_unstale":"2023-09-23T20:48:09.814960+0000","last_undegraded":"2023-09-23T20:48:09.814960+0000","last_fullsized":"2023-09-23T20:48:09.814960+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:01.504973+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:01.504973+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:31:17.112393+0000","scrub_duration":0.00202344,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":609,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-09-23T20:48:09.775256+0000","last_change":"2023-09-23T20:43:46.511657+0000","last_active":"2023-09-23T20:48:09.775256+0000","last_peered":"2023-09-23T20:48:09.775256+0000","last_clean":"2023-09-23T20:48:09.775256+0000","last_became_active":"2023-09-23T20:43:46.511326+0000","last_became_peered":"2023-09-23T20:43:46.511326+0000","last_unstale":"2023-09-23T20:48:09.775256+0000","last_undegraded":"2023-09-23T20:48:09.775256+0000","last_fullsized":"2023-09-23T20:48:09.775256+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:36:46.139261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3324,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":32,"reported_epoch":574,"state":"active+clean","last_fresh":"2023-09-23T20:48:09.789305+0000","last_change":"2023-09-23T20:47:09.183755+0000","last_active":"2023-09-23T20:48:09.789305+0000","last_peered":"2023-09-23T20:48:09.789305+0000","last_clean":"2023-09-23T20:48:09.789305+0000","last_became_active":"2023-09-23T20:47:09.183448+0000","last_became_peered":"2023-09-23T20:47:09.183448+0000","last_unstale":"2023-09-23T20:48:09.789305+0000","last_undegraded":"2023-09-23T20:48:09.789305+0000","last_fullsized":"2023-09-23T20:48:09.789305+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:18:38.303313+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":3608608792,"num_objects":55064,"num_object_clones":0,"num_object_copies":165192,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":55064,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":153924,"num_write_kb":6687553,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2253,"num_bytes_recovered":147652608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":10864111616,"data_stored":10864099400,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":109694,"ondisk_log_size":109694,"up":48,"acting":48,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":101,"num_bytes_recovered":14044,"num_keys_recovered":60,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26691,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5983,"num_read_kb":13200,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":114,"num_bytes_recovered":16777216,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6082,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201722,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1465884,"kb_used_data":910996,"kb_used_omap":0,"kb_used_meta":554880,"kb_avail":92279268,"statfs":{"total":95995035648,"available":94493970432,"internally_reserved":0,"allocated":932859904,"data_stored":931288751,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":568197120},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:48:06 2023","interfaces":[{"interface":"back","average":{"1min":1.5569999999999999,"5min":1.972,"15min":2.1899999999999999},"min":{"1min":0.224,"5min":0.224,"15min":0.224},"max":{"1min":5.6970000000000001,"5min":15.1,"15min":18.315000000000001},"last":3.2010000000000001},{"interface":"front","average":{"1min":2.1539999999999999,"5min":2.1379999999999999,"15min":1.903},"min":{"1min":0.28000000000000003,"5min":0.25800000000000001,"15min":0.23899999999999999},"max":{"1min":6.5780000000000003,"5min":15.327999999999999,"15min":19.355},"last":0.27500000000000002}]},{"osd":1,"last update":"Sat Sep 23 20:48:13 2023","interfaces":[{"interface":"back","average":{"1min":1.3640000000000001,"5min":1.6779999999999999,"15min":1.972},"min":{"1min":0.28399999999999997,"5min":0.25900000000000001,"15min":0.24399999999999999},"max":{"1min":6.4610000000000003,"5min":15.131,"15min":49.965000000000003},"last":0.377},{"interface":"front","average":{"1min":2.4929999999999999,"5min":2.524,"15min":2.5099999999999998},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.26700000000000002},"max":{"1min":10.861000000000001,"5min":15.18,"15min":53.451999999999998},"last":0.34100000000000003}]},{"osd":2,"last update":"Sat Sep 23 20:48:06 2023","interfaces":[{"interface":"back","average":{"1min":2.0779999999999998,"5min":2.153,"15min":2.1899999999999999},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.27800000000000002},"max":{"1min":5.4770000000000003,"5min":15.087,"15min":16.965},"last":5.1660000000000004},{"interface":"front","average":{"1min":2.206,"5min":2.0249999999999999,"15min":2.339},"min":{"1min":0.45200000000000001,"5min":0.31,"15min":0.27400000000000002},"max":{"1min":6.6379999999999999,"5min":15.193,"15min":23.494},"last":0.47999999999999998}]},{"osd":3,"last update":"Sat Sep 23 20:47:35 2023","interfaces":[{"interface":"back","average":{"1min":2.153,"5min":1.841,"15min":2.2589999999999999},"min":{"1min":0.33800000000000002,"5min":0.32500000000000001,"15min":0.311},"max":{"1min":6.734,"5min":15.115,"15min":23.352},"last":4.5540000000000003},{"interface":"front","average":{"1min":2.2040000000000002,"5min":2.2589999999999999,"15min":2.4550000000000001},"min":{"1min":0.443,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":6.5369999999999999,"5min":15.144,"15min":20.969999999999999},"last":4.5709999999999997}]},{"osd":4,"last update":"Sat Sep 23 20:47:35 2023","interfaces":[{"interface":"back","average":{"1min":1.264,"5min":1.496,"15min":1.1739999999999999},"min":{"1min":0.27900000000000003,"5min":0.254,"15min":0.23200000000000001},"max":{"1min":6.548,"5min":12.260999999999999,"15min":18.364999999999998},"last":0.44900000000000001},{"interface":"front","average":{"1min":1.6160000000000001,"5min":1.8300000000000001,"15min":1.4350000000000001},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":6.0419999999999998,"5min":7.2469999999999999,"15min":23.617000000000001},"last":0.46300000000000002}]},{"osd":5,"last update":"Sat Sep 23 20:47:35 2023","interfaces":[{"interface":"back","average":{"1min":1.101,"5min":1.585,"15min":1.2190000000000001},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.26600000000000001},"max":{"1min":6.367,"5min":15.005000000000001,"15min":15.005000000000001},"last":0.41799999999999998},{"interface":"front","average":{"1min":1.504,"5min":1.9379999999999999,"15min":1.3480000000000001},"min":{"1min":0.36099999999999999,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":6.6280000000000001,"5min":12.731,"15min":18.390000000000001},"last":0.499}]},{"osd":6,"last update":"Sat Sep 23 20:47:35 2023","interfaces":[{"interface":"back","average":{"1min":1.4730000000000001,"5min":2.0950000000000002,"15min":1.47},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":6.4720000000000004,"5min":15.038,"15min":18.417000000000002},"last":0.55700000000000005},{"interface":"front","average":{"1min":1.861,"5min":1.9910000000000001,"15min":1.4139999999999999},"min":{"1min":0.35899999999999999,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":6.0910000000000002,"5min":15.369999999999999,"15min":15.369999999999999},"last":1.512}]}]},{"osd":6,"up_from":47,"seq":201863463361,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2054484,"kb_used_data":1568076,"kb_used_omap":0,"kb_used_meta":486208,"kb_avail":91690668,"statfs":{"total":95995035648,"available":93891244032,"internally_reserved":0,"allocated":1605709824,"data_stored":1604146216,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":497876992},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:48:03 2023","interfaces":[{"interface":"back","average":{"1min":2.9780000000000002,"5min":2.0299999999999998,"15min":1.9419999999999999},"min":{"1min":0.25,"5min":0.219,"15min":0.219},"max":{"1min":15.574999999999999,"5min":21.018000000000001,"15min":34.143999999999998},"last":7.8099999999999996},{"interface":"front","average":{"1min":1.698,"5min":2.1909999999999998,"15min":2.2429999999999999},"min":{"1min":0.255,"5min":0.255,"15min":0.22700000000000001},"max":{"1min":4.8840000000000003,"5min":21.082000000000001,"15min":60.328000000000003},"last":1.6299999999999999}]},{"osd":1,"last update":"Sat Sep 23 20:47:19 2023","interfaces":[{"interface":"back","average":{"1min":1.9039999999999999,"5min":2.0070000000000001,"15min":1.9610000000000001},"min":{"1min":0.34499999999999997,"5min":0.26400000000000001,"15min":0.249},"max":{"1min":5.5640000000000001,"5min":9.2520000000000007,"15min":64.274000000000001},"last":0.33100000000000002},{"interface":"front","average":{"1min":1.198,"5min":1.8520000000000001,"15min":1.782},"min":{"1min":0.32200000000000001,"5min":0.29099999999999998,"15min":0.28000000000000003},"max":{"1min":8.8070000000000004,"5min":17.786000000000001,"15min":34.490000000000002},"last":3.6320000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:48:03 2023","interfaces":[{"interface":"back","average":{"1min":2.4609999999999999,"5min":2.4380000000000002,"15min":2.242},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.27700000000000002},"max":{"1min":6.1529999999999996,"5min":22.591000000000001,"15min":34.317999999999998},"last":2.7170000000000001},{"interface":"front","average":{"1min":2.4039999999999999,"5min":2.0350000000000001,"15min":2.0539999999999998},"min":{"1min":0.45400000000000001,"5min":0.33900000000000002,"15min":0.30399999999999999},"max":{"1min":7.8710000000000004,"5min":22.614999999999998,"15min":34.292999999999999},"last":2.7570000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:47:54 2023","interfaces":[{"interface":"back","average":{"1min":2.0659999999999998,"5min":1.946,"15min":1.9339999999999999},"min":{"1min":0.307,"5min":0.307,"15min":0.28399999999999997},"max":{"1min":9.6859999999999999,"5min":11.664,"15min":11.664},"last":7.9640000000000004},{"interface":"front","average":{"1min":2.2959999999999998,"5min":2.1859999999999999,"15min":2.1539999999999999},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.30499999999999999},"max":{"1min":9.7810000000000006,"5min":10.125999999999999,"15min":12.875999999999999},"last":7.7649999999999997}]},{"osd":4,"last update":"Sat Sep 23 20:47:19 2023","interfaces":[{"interface":"back","average":{"1min":1.0529999999999999,"5min":1.0700000000000001,"15min":0.85099999999999998},"min":{"1min":0.28699999999999998,"5min":0.25600000000000001,"15min":0.253},"max":{"1min":10.183999999999999,"5min":10.183999999999999,"15min":23.904},"last":0.41799999999999998},{"interface":"front","average":{"1min":1.6639999999999999,"5min":1.208,"15min":1.0920000000000001},"min":{"1min":0.32500000000000001,"5min":0.28399999999999997,"15min":0.28199999999999997},"max":{"1min":7.2300000000000004,"5min":7.2300000000000004,"15min":11.515000000000001},"last":2.52}]},{"osd":5,"last update":"Sat Sep 23 20:47:19 2023","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":1.0569999999999999,"15min":0.88500000000000001},"min":{"1min":0.30499999999999999,"5min":0.253,"15min":0.253},"max":{"1min":1.8979999999999999,"5min":7.8529999999999998,"15min":7.8529999999999998},"last":0.45400000000000001},{"interface":"front","average":{"1min":2.3290000000000002,"5min":1.6970000000000001,"15min":1.222},"min":{"1min":0.41999999999999998,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":9.3559999999999999,"5min":9.3559999999999999,"15min":9.3559999999999999},"last":1.6579999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:47:54 2023","interfaces":[{"interface":"back","average":{"1min":1.421,"5min":1.331,"15min":0.98499999999999999},"min":{"1min":0.35199999999999998,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":7.1379999999999999,"5min":8.9309999999999992,"15min":8.9309999999999992},"last":0.47999999999999998},{"interface":"front","average":{"1min":2.0459999999999998,"5min":2.0150000000000001,"15min":1.3939999999999999},"min":{"1min":0.434,"5min":0.35999999999999999,"15min":0.34499999999999997},"max":{"1min":9.7530000000000001,"5min":9.8680000000000003,"15min":9.8680000000000003},"last":1.681}]}]},{"osd":1,"up_from":266,"seq":1142461300960,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1400992,"kb_used_data":650456,"kb_used_omap":8,"kb_used_meta":750519,"kb_avail":92344160,"statfs":{"total":95995035648,"available":94560419840,"internally_reserved":0,"allocated":666066944,"data_stored":663215060,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8206,"internal_metadata":768532466},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:47:23 2023","interfaces":[{"interface":"back","average":{"1min":1.248,"5min":2.3530000000000002,"15min":2.46},"min":{"1min":0.252,"5min":0.20599999999999999,"15min":0.20000000000000001},"max":{"1min":4.3399999999999999,"5min":15.532999999999999,"15min":52.689999999999998},"last":5.1710000000000003},{"interface":"front","average":{"1min":2.2999999999999998,"5min":2.7250000000000001,"15min":3.0350000000000001},"min":{"1min":0.27900000000000003,"5min":0.22600000000000001,"15min":0.19900000000000001},"max":{"1min":10.061,"5min":20.346,"15min":52.662999999999997},"last":5.1280000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:47:23 2023","interfaces":[{"interface":"back","average":{"1min":1.647,"5min":2.75,"15min":3.0230000000000001},"min":{"1min":0.24099999999999999,"5min":0.23499999999999999,"15min":0.20899999999999999},"max":{"1min":7.8719999999999999,"5min":19.026,"15min":32.255000000000003},"last":3.274},{"interface":"front","average":{"1min":1.8420000000000001,"5min":2.194,"15min":2.262},"min":{"1min":0.29999999999999999,"5min":0.26000000000000001,"15min":0.20899999999999999},"max":{"1min":9.9440000000000008,"5min":10.041,"15min":13.404999999999999},"last":5.5190000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:47:23 2023","interfaces":[{"interface":"back","average":{"1min":1.958,"5min":2.1629999999999998,"15min":2.8050000000000002},"min":{"1min":0.33900000000000002,"5min":0.28799999999999998,"15min":0.247},"max":{"1min":17.366,"5min":17.366,"15min":80.745999999999995},"last":5.2069999999999999},{"interface":"front","average":{"1min":2.4769999999999999,"5min":2.7480000000000002,"15min":3.093},"min":{"1min":0.30399999999999999,"5min":0.22900000000000001,"15min":0.22700000000000001},"max":{"1min":10.502000000000001,"5min":11.534000000000001,"15min":82.988},"last":4.1520000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:47:23 2023","interfaces":[{"interface":"back","average":{"1min":1.7490000000000001,"5min":2.6320000000000001,"15min":2.2930000000000001},"min":{"1min":0.34000000000000002,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":8.5440000000000005,"5min":20.634,"15min":24.327999999999999},"last":3.2170000000000001},{"interface":"front","average":{"1min":1.476,"5min":1.8879999999999999,"15min":1.6890000000000001},"min":{"1min":0.35699999999999998,"5min":0.34699999999999998,"15min":0.311},"max":{"1min":4.2939999999999996,"5min":20.835999999999999,"15min":20.835999999999999},"last":5.234}]},{"osd":5,"last update":"Sat Sep 23 20:47:23 2023","interfaces":[{"interface":"back","average":{"1min":2.3919999999999999,"5min":2.4630000000000001,"15min":2.2669999999999999},"min":{"1min":0.39100000000000001,"5min":0.308,"15min":0.308},"max":{"1min":9.9779999999999998,"5min":14.366,"15min":33.401000000000003},"last":1.169},{"interface":"front","average":{"1min":2.0710000000000002,"5min":2.7130000000000001,"15min":2.5310000000000001},"min":{"1min":0.377,"5min":0.33100000000000002,"15min":0.32500000000000001},"max":{"1min":9.6739999999999995,"5min":11.750999999999999,"15min":33.417000000000002},"last":3.2480000000000002}]},{"osd":6,"last update":"Sat Sep 23 20:47:23 2023","interfaces":[{"interface":"back","average":{"1min":2.2370000000000001,"5min":2.4550000000000001,"15min":2.1579999999999999},"min":{"1min":0.38600000000000001,"5min":0.316,"15min":0.316},"max":{"1min":9.9339999999999993,"5min":14.414999999999999,"15min":14.414999999999999},"last":1.143},{"interface":"front","average":{"1min":2.4390000000000001,"5min":3.0329999999999999,"15min":2.528},"min":{"1min":0.46300000000000002,"5min":0.38500000000000001,"15min":0.317},"max":{"1min":8.4969999999999999,"5min":11.832000000000001,"15min":24.376000000000001},"last":3.2919999999999998}]},{"osd":7,"last update":"Sat Sep 23 20:47:23 2023","interfaces":[{"interface":"back","average":{"1min":2.5830000000000002,"5min":2.5179999999999998,"15min":2.3079999999999998},"min":{"1min":0.48799999999999999,"5min":0.37,"15min":0.37},"max":{"1min":10.010999999999999,"5min":10.010999999999999,"15min":18.437000000000001},"last":1.1899999999999999},{"interface":"front","average":{"1min":2.2189999999999999,"5min":2.9550000000000001,"15min":2.552},"min":{"1min":0.49099999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":8.0540000000000003,"5min":11.864000000000001,"15min":24.524000000000001},"last":3.3079999999999998}]}]},{"osd":0,"up_from":94,"seq":403726926162,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3126572,"kb_used_data":2203556,"kb_used_omap":7,"kb_used_meta":923000,"kb_avail":90618580,"statfs":{"total":95995035648,"available":92793425920,"internally_reserved":0,"allocated":2256441344,"data_stored":2253753851,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":945152017},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:47:30 2023","interfaces":[{"interface":"back","average":{"1min":8.9149999999999991,"5min":3.4870000000000001,"15min":2.9049999999999998},"min":{"1min":0.20100000000000001,"5min":0.184,"15min":0.17199999999999999},"max":{"1min":152.387,"5min":152.387,"15min":152.387},"last":7.298},{"interface":"front","average":{"1min":8.7140000000000004,"5min":3.8149999999999999,"15min":3.0920000000000001},"min":{"1min":0.23899999999999999,"5min":0.23799999999999999,"15min":0.20399999999999999},"max":{"1min":152.84100000000001,"5min":152.84100000000001,"15min":152.84100000000001},"last":4.0579999999999998}]},{"osd":2,"last update":"Sat Sep 23 20:48:01 2023","interfaces":[{"interface":"back","average":{"1min":4.4219999999999997,"5min":2.9350000000000001,"15min":2.9950000000000001},"min":{"1min":0.254,"5min":0.23499999999999999,"15min":0.193},"max":{"1min":14.538,"5min":15.108000000000001,"15min":62.726999999999997},"last":7.008},{"interface":"front","average":{"1min":2.8250000000000002,"5min":2.7149999999999999,"15min":2.9990000000000001},"min":{"1min":0.27100000000000002,"5min":0.217,"15min":0.217},"max":{"1min":8.3689999999999998,"5min":17.013000000000002,"15min":62.689},"last":3.77}]},{"osd":3,"last update":"Sat Sep 23 20:47:39 2023","interfaces":[{"interface":"back","average":{"1min":2.9119999999999999,"5min":3.6099999999999999,"15min":3.165},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":13.367000000000001,"5min":55.017000000000003,"15min":55.017000000000003},"last":7.1299999999999999},{"interface":"front","average":{"1min":2.5760000000000001,"5min":2.9900000000000002,"15min":3.1989999999999998},"min":{"1min":0.32000000000000001,"5min":0.23799999999999999,"15min":0.21199999999999999},"max":{"1min":12.051,"5min":55.665999999999997,"15min":55.665999999999997},"last":7.3300000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:48:01 2023","interfaces":[{"interface":"back","average":{"1min":2.9860000000000002,"5min":2.0139999999999998,"15min":2.1299999999999999},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.25900000000000001},"max":{"1min":11.75,"5min":11.75,"15min":12.587999999999999},"last":0.45900000000000002},{"interface":"front","average":{"1min":3.609,"5min":2.125,"15min":2.093},"min":{"1min":0.31,"5min":0.28899999999999998,"15min":0.27400000000000002},"max":{"1min":12.356999999999999,"5min":12.356999999999999,"15min":23.001000000000001},"last":3.9790000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:48:01 2023","interfaces":[{"interface":"back","average":{"1min":3.4590000000000001,"5min":2.097,"15min":2.169},"min":{"1min":0.34300000000000003,"5min":0.30599999999999999,"15min":0.26500000000000001},"max":{"1min":9.3339999999999996,"5min":9.3339999999999996,"15min":22.969999999999999},"last":4.0270000000000001},{"interface":"front","average":{"1min":2.9300000000000002,"5min":2.3359999999999999,"15min":2.492},"min":{"1min":0.39000000000000001,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":10.089,"5min":12.601000000000001,"15min":26.228000000000002},"last":3.8420000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:48:01 2023","interfaces":[{"interface":"back","average":{"1min":3.6920000000000002,"5min":2.3039999999999998,"15min":2.3439999999999999},"min":{"1min":0.41799999999999998,"5min":0.374,"15min":0.32300000000000001},"max":{"1min":9.2750000000000004,"5min":9.2750000000000004,"15min":24.056000000000001},"last":3.944},{"interface":"front","average":{"1min":3.4020000000000001,"5min":2.395,"15min":2.3780000000000001},"min":{"1min":0.40999999999999998,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":12.427,"5min":12.698,"15min":26.268000000000001},"last":3.8039999999999998}]},{"osd":7,"last update":"Sat Sep 23 20:48:01 2023","interfaces":[{"interface":"back","average":{"1min":3.4140000000000001,"5min":2.7749999999999999,"15min":2.5870000000000002},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.36099999999999999},"max":{"1min":12.486000000000001,"5min":16.553999999999998,"15min":26.177},"last":3.867},{"interface":"front","average":{"1min":3.3879999999999999,"5min":2.6179999999999999,"15min":2.5179999999999998},"min":{"1min":0.45300000000000001,"5min":0.40200000000000002,"15min":0.34200000000000003},"max":{"1min":10.183999999999999,"5min":16.649000000000001,"15min":22.422999999999998},"last":3.8500000000000001}]}]},{"osd":2,"up_from":104,"seq":446676599110,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2118180,"kb_used_data":1322268,"kb_used_omap":11,"kb_used_meta":795892,"kb_avail":91626972,"statfs":{"total":95995035648,"available":93826019328,"internally_reserved":0,"allocated":1354002432,"data_stored":1351319229,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11878,"internal_metadata":814993818},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:48:11 2023","interfaces":[{"interface":"back","average":{"1min":2.327,"5min":1.738,"15min":2.0710000000000002},"min":{"1min":0.27300000000000002,"5min":0.20499999999999999,"15min":0.17299999999999999},"max":{"1min":9.7919999999999998,"5min":15.324999999999999,"15min":28.234999999999999},"last":8.5909999999999993},{"interface":"front","average":{"1min":2.4169999999999998,"5min":2.0710000000000002,"15min":2.2559999999999998},"min":{"1min":0.30399999999999999,"5min":0.245,"15min":0.19},"max":{"1min":8.3789999999999996,"5min":16.678999999999998,"15min":16.678999999999998},"last":6.7220000000000004}]},{"osd":1,"last update":"Sat Sep 23 20:48:11 2023","interfaces":[{"interface":"back","average":{"1min":3.1560000000000001,"5min":2.1549999999999998,"15min":2.5910000000000002},"min":{"1min":0.248,"5min":0.219,"15min":0.219},"max":{"1min":11.012,"5min":21.690000000000001,"15min":46.405000000000001},"last":10.324},{"interface":"front","average":{"1min":1.976,"5min":1.905,"15min":2.5379999999999998},"min":{"1min":0.254,"5min":0.22,"15min":0.22},"max":{"1min":10.191000000000001,"5min":13.215999999999999,"15min":46.459000000000003},"last":6.2910000000000004}]},{"osd":3,"last update":"Sat Sep 23 20:47:37 2023","interfaces":[{"interface":"back","average":{"1min":2.8010000000000002,"5min":2.6549999999999998,"15min":2.6400000000000001},"min":{"1min":0.34100000000000003,"5min":0.253,"15min":0.253},"max":{"1min":9.9580000000000002,"5min":30.283000000000001,"15min":30.283000000000001},"last":10.051},{"interface":"front","average":{"1min":1.1399999999999999,"5min":1.5529999999999999,"15min":2.0619999999999998},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":4.3600000000000003,"5min":30.834,"15min":30.834},"last":4.4459999999999997}]},{"osd":4,"last update":"Sat Sep 23 20:48:11 2023","interfaces":[{"interface":"back","average":{"1min":2.3380000000000001,"5min":1.9550000000000001,"15min":1.851},"min":{"1min":0.435,"5min":0.32000000000000001,"15min":0.27500000000000002},"max":{"1min":8.3469999999999995,"5min":11.141999999999999,"15min":12.025},"last":5.556},{"interface":"front","average":{"1min":2.4940000000000002,"5min":1.9370000000000001,"15min":2.0760000000000001},"min":{"1min":0.375,"5min":0.35699999999999998,"15min":0.30499999999999999},"max":{"1min":8.2289999999999992,"5min":10.948,"15min":19.440999999999999},"last":12.662000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:48:11 2023","interfaces":[{"interface":"back","average":{"1min":1.9810000000000001,"5min":1.7689999999999999,"15min":1.8660000000000001},"min":{"1min":0.42899999999999999,"5min":0.311,"15min":0.311},"max":{"1min":8.5380000000000003,"5min":11.183,"15min":12.07},"last":3.024},{"interface":"front","average":{"1min":2.5630000000000002,"5min":1.8859999999999999,"15min":2.097},"min":{"1min":0.42399999999999999,"5min":0.373,"15min":0.35299999999999998},"max":{"1min":8.2729999999999997,"5min":11.061999999999999,"15min":19.513000000000002},"last":10.831}]},{"osd":6,"last update":"Sat Sep 23 20:48:11 2023","interfaces":[{"interface":"back","average":{"1min":2.7200000000000002,"5min":2.1520000000000001,"15min":2.202},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.33500000000000002},"max":{"1min":9.2319999999999993,"5min":11.894,"15min":19.486000000000001},"last":10.791},{"interface":"front","average":{"1min":1.7110000000000001,"5min":1.657,"15min":1.6459999999999999},"min":{"1min":0.46700000000000003,"5min":0.32900000000000001,"15min":0.318},"max":{"1min":7.4180000000000001,"5min":11.747,"15min":11.747},"last":4.9589999999999996}]},{"osd":7,"last update":"Sat Sep 23 20:48:11 2023","interfaces":[{"interface":"back","average":{"1min":1.643,"5min":1.516,"15min":1.635},"min":{"1min":0.46300000000000002,"5min":0.35399999999999998,"15min":0.34699999999999998},"max":{"1min":7.391,"5min":7.718,"15min":12.734999999999999},"last":13.477},{"interface":"front","average":{"1min":3.375,"5min":2.2930000000000001,"15min":2.1299999999999999},"min":{"1min":0.502,"5min":0.35799999999999998,"15min":0.34899999999999998},"max":{"1min":10.071999999999999,"5min":11.218,"15min":12.099},"last":5.3259999999999996}]}]},{"osd":3,"up_from":109,"seq":468151435583,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2201044,"kb_used_data":1164492,"kb_used_omap":4,"kb_used_meta":1036539,"kb_avail":91544108,"statfs":{"total":95995035648,"available":93741166592,"internally_reserved":0,"allocated":1192439808,"data_stored":1189770676,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4514,"internal_metadata":1061416542},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:47:46 2023","interfaces":[{"interface":"back","average":{"1min":3.6509999999999998,"5min":2.6219999999999999,"15min":2.6019999999999999},"min":{"1min":0.23599999999999999,"5min":0.217,"15min":0.191},"max":{"1min":21.792999999999999,"5min":21.792999999999999,"15min":22.32},"last":2.9670000000000001},{"interface":"front","average":{"1min":2.7069999999999999,"5min":2.7530000000000001,"15min":2.633},"min":{"1min":0.20200000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":14.845000000000001,"5min":22.805,"15min":22.805},"last":0.26500000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:47:16 2023","interfaces":[{"interface":"back","average":{"1min":1.119,"5min":2.4550000000000001,"15min":2.008},"min":{"1min":0.28199999999999997,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":4.9349999999999996,"5min":17.529,"15min":17.529},"last":3.0449999999999999},{"interface":"front","average":{"1min":1.9330000000000001,"5min":2.2269999999999999,"15min":2.2949999999999999},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":7.3780000000000001,"5min":14.19,"15min":32.603999999999999},"last":0.307}]},{"osd":2,"last update":"Sat Sep 23 20:47:46 2023","interfaces":[{"interface":"back","average":{"1min":3.0139999999999998,"5min":2.976,"15min":3.0619999999999998},"min":{"1min":0.34599999999999997,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":7.9210000000000003,"5min":19.478000000000002,"15min":68.206999999999994},"last":0.28999999999999998},{"interface":"front","average":{"1min":1.7829999999999999,"5min":2.016,"15min":2.556},"min":{"1min":0.31,"5min":0.245,"15min":0.245},"max":{"1min":8.0220000000000002,"5min":17.919,"15min":67.736999999999995},"last":0.69599999999999995}]},{"osd":4,"last update":"Sat Sep 23 20:47:46 2023","interfaces":[{"interface":"back","average":{"1min":1.9119999999999999,"5min":1.851,"15min":1.6890000000000001},"min":{"1min":0.48899999999999999,"5min":0.317,"15min":0.30499999999999999},"max":{"1min":9.0259999999999998,"5min":9.0259999999999998,"15min":22.157},"last":3.0899999999999999},{"interface":"front","average":{"1min":1.4099999999999999,"5min":1.871,"15min":2.077},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.29399999999999998},"max":{"1min":5.024,"5min":8.1910000000000007,"15min":15.868},"last":0.52100000000000002}]},{"osd":5,"last update":"Sat Sep 23 20:47:46 2023","interfaces":[{"interface":"back","average":{"1min":1.9570000000000001,"5min":2.1160000000000001,"15min":1.911},"min":{"1min":0.46200000000000002,"5min":0.33800000000000002,"15min":0.32200000000000001},"max":{"1min":8.9849999999999994,"5min":8.9849999999999994,"15min":22.289999999999999},"last":3.1219999999999999},{"interface":"front","average":{"1min":1.7929999999999999,"5min":2.0369999999999999,"15min":1.829},"min":{"1min":0.379,"5min":0.34100000000000003,"15min":0.32000000000000001},"max":{"1min":6.1390000000000002,"5min":12.952,"15min":12.952},"last":0.623}]},{"osd":6,"last update":"Sat Sep 23 20:47:46 2023","interfaces":[{"interface":"back","average":{"1min":2.5819999999999999,"5min":2.6509999999999998,"15min":2.004},"min":{"1min":0.44400000000000001,"5min":0.36599999999999999,"15min":0.34300000000000003},"max":{"1min":7.0389999999999997,"5min":12.917,"15min":12.917},"last":0.57699999999999996},{"interface":"front","average":{"1min":1.698,"5min":2.1440000000000001,"15min":2.1349999999999998},"min":{"1min":0.36699999999999999,"5min":0.34599999999999997,"15min":0.33700000000000002},"max":{"1min":6.4930000000000003,"5min":8.9320000000000004,"15min":15.936},"last":0.40600000000000003}]},{"osd":7,"last update":"Sat Sep 23 20:47:46 2023","interfaces":[{"interface":"back","average":{"1min":1.768,"5min":2.262,"15min":2.302},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.36899999999999999},"max":{"1min":7.6429999999999998,"5min":11.885999999999999,"15min":15.976000000000001},"last":0.48199999999999998},{"interface":"front","average":{"1min":2.1360000000000001,"5min":2.1619999999999999,"15min":1.968},"min":{"1min":0.51100000000000001,"5min":0.42799999999999999,"15min":0.40999999999999998},"max":{"1min":9.0549999999999997,"5min":10.853999999999999,"15min":22.202999999999999},"last":3.1520000000000001}]}]},{"osd":4,"up_from":30,"seq":128849019337,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2294164,"kb_used_data":1794700,"kb_used_omap":0,"kb_used_meta":499456,"kb_avail":91450988,"statfs":{"total":95995035648,"available":93645811712,"internally_reserved":0,"allocated":1837772800,"data_stored":1836208926,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":511442944},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:48:04 2023","interfaces":[{"interface":"back","average":{"1min":2.1320000000000001,"5min":1.8500000000000001,"15min":1.8420000000000001},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24099999999999999},"max":{"1min":7.1120000000000001,"5min":11.709,"15min":24.114999999999998},"last":0.42999999999999999},{"interface":"front","average":{"1min":2.4329999999999998,"5min":2.1480000000000001,"15min":2.0870000000000002},"min":{"1min":0.29799999999999999,"5min":0.25800000000000001,"15min":0.246},"max":{"1min":6.8639999999999999,"5min":13.74,"15min":24.135999999999999},"last":0.39300000000000002}]},{"osd":1,"last update":"Sat Sep 23 20:47:15 2023","interfaces":[{"interface":"back","average":{"1min":1.7669999999999999,"5min":2.2090000000000001,"15min":1.764},"min":{"1min":0.317,"5min":0.26300000000000001,"15min":0.25900000000000001},"max":{"1min":6.8600000000000003,"5min":9.673,"15min":18.190999999999999},"last":0.27800000000000002},{"interface":"front","average":{"1min":1.4590000000000001,"5min":1.659,"15min":1.5429999999999999},"min":{"1min":0.29799999999999999,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":4.9560000000000004,"5min":8.0730000000000004,"15min":18.173999999999999},"last":0.46999999999999997}]},{"osd":2,"last update":"Sat Sep 23 20:47:15 2023","interfaces":[{"interface":"back","average":{"1min":2.0630000000000002,"5min":1.768,"15min":1.6970000000000001},"min":{"1min":0.30299999999999999,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":5.2400000000000002,"5min":9.3320000000000007,"15min":13.051},"last":0.52700000000000002},{"interface":"front","average":{"1min":2.367,"5min":2.2599999999999998,"15min":2.1040000000000001},"min":{"1min":0.35699999999999998,"5min":0.311,"15min":0.28399999999999997},"max":{"1min":8.7230000000000008,"5min":8.7230000000000008,"15min":14.614000000000001},"last":2.8130000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:47:31 2023","interfaces":[{"interface":"back","average":{"1min":1.8560000000000001,"5min":2.0089999999999999,"15min":1.901},"min":{"1min":0.318,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":5.0270000000000001,"5min":8.6180000000000003,"15min":12.147},"last":0.499},{"interface":"front","average":{"1min":2.101,"5min":2.2000000000000002,"15min":2.0339999999999998},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.30299999999999999},"max":{"1min":6.3159999999999998,"5min":17.169,"15min":17.169},"last":0.55300000000000005}]},{"osd":5,"last update":"Sat Sep 23 20:48:04 2023","interfaces":[{"interface":"back","average":{"1min":0.79300000000000004,"5min":1.458,"15min":1.0740000000000001},"min":{"1min":0.28399999999999997,"5min":0.253,"15min":0.253},"max":{"1min":3.5790000000000002,"5min":10.202,"15min":10.202},"last":0.33500000000000002},{"interface":"front","average":{"1min":1.1479999999999999,"5min":1.5369999999999999,"15min":1.1850000000000001},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":4.7430000000000003,"5min":8.1440000000000001,"15min":8.1440000000000001},"last":0.60299999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:47:15 2023","interfaces":[{"interface":"back","average":{"1min":2.1779999999999999,"5min":1.7270000000000001,"15min":1.3069999999999999},"min":{"1min":0.30399999999999999,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":8.7799999999999994,"5min":10.840999999999999,"15min":14.731999999999999},"last":0.57099999999999995},{"interface":"front","average":{"1min":2.1869999999999998,"5min":1.71,"15min":1.157},"min":{"1min":0.32200000000000001,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":5.7910000000000004,"5min":7.6150000000000002,"15min":7.6150000000000002},"last":0.443}]},{"osd":7,"last update":"Sat Sep 23 20:47:31 2023","interfaces":[{"interface":"back","average":{"1min":2.4089999999999998,"5min":5.4740000000000002,"15min":2.6669999999999998},"min":{"1min":0.377,"5min":0.35199999999999998,"15min":0.318},"max":{"1min":6.3920000000000003,"5min":313.50099999999998,"15min":313.50099999999998},"last":0.629},{"interface":"front","average":{"1min":1.8340000000000001,"5min":4.7060000000000004,"15min":2.2290000000000001},"min":{"1min":0.35099999999999998,"5min":0.33200000000000002,"15min":0.32700000000000001},"max":{"1min":6.1829999999999998,"5min":312.67500000000001,"15min":312.67500000000001},"last":0.53800000000000003}]}]},{"osd":5,"up_from":39,"seq":167503724998,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1702604,"kb_used_data":1135940,"kb_used_omap":0,"kb_used_meta":566656,"kb_avail":92042548,"statfs":{"total":95995035648,"available":94251569152,"internally_reserved":0,"allocated":1163202560,"data_stored":1161643484,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":580255744},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:48:10 2023","interfaces":[{"interface":"back","average":{"1min":2.5129999999999999,"5min":1.8700000000000001,"15min":1.891},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.20699999999999999},"max":{"1min":6.673,"5min":15.018000000000001,"15min":18.16},"last":3.2349999999999999},{"interface":"front","average":{"1min":2.258,"5min":1.915,"15min":1.8300000000000001},"min":{"1min":0.30099999999999999,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":8.6050000000000004,"5min":15.034000000000001,"15min":15.034000000000001},"last":3.1240000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:47:23 2023","interfaces":[{"interface":"back","average":{"1min":2.3490000000000002,"5min":2.3849999999999998,"15min":2.286},"min":{"1min":0.32200000000000001,"5min":0.29799999999999999,"15min":0.253},"max":{"1min":6.859,"5min":9.7230000000000008,"15min":61.005000000000003},"last":3.1720000000000002},{"interface":"front","average":{"1min":2.375,"5min":2.0329999999999999,"15min":2.1070000000000002},"min":{"1min":0.315,"5min":0.27700000000000002,"15min":0.23499999999999999},"max":{"1min":16.219000000000001,"5min":16.219000000000001,"15min":59.688000000000002},"last":2.742}]},{"osd":2,"last update":"Sat Sep 23 20:48:10 2023","interfaces":[{"interface":"back","average":{"1min":2.6160000000000001,"5min":2.1400000000000001,"15min":2.1579999999999999},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26500000000000001},"max":{"1min":7.194,"5min":12.226000000000001,"15min":37.011000000000003},"last":2.859},{"interface":"front","average":{"1min":2.5219999999999998,"5min":1.867,"15min":2.0350000000000001},"min":{"1min":0.39600000000000002,"5min":0.314,"15min":0.255},"max":{"1min":5.9790000000000001,"5min":9.8260000000000005,"15min":36.835000000000001},"last":3.919}]},{"osd":3,"last update":"Sat Sep 23 20:47:27 2023","interfaces":[{"interface":"back","average":{"1min":1.7470000000000001,"5min":2.3010000000000002,"15min":2.23},"min":{"1min":0.38200000000000001,"5min":0.33800000000000002,"15min":0.29599999999999999},"max":{"1min":6.3449999999999998,"5min":11.032,"15min":36.716000000000001},"last":3.29},{"interface":"front","average":{"1min":1.8300000000000001,"5min":2.1200000000000001,"15min":2.246},"min":{"1min":0.38900000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":6.7910000000000004,"5min":13.266,"15min":18.504000000000001},"last":2.7890000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:48:00 2023","interfaces":[{"interface":"back","average":{"1min":1.593,"5min":1.8180000000000001,"15min":1.2270000000000001},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.255},"max":{"1min":6.3170000000000002,"5min":13.391999999999999,"15min":13.391999999999999},"last":3.2000000000000002},{"interface":"front","average":{"1min":1.5129999999999999,"5min":1.6899999999999999,"15min":1.2390000000000001},"min":{"1min":0.34200000000000003,"5min":0.29199999999999998,"15min":0.28299999999999997},"max":{"1min":6.6600000000000001,"5min":7.3799999999999999,"15min":7.3799999999999999},"last":2.9100000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:48:11 2023","interfaces":[{"interface":"back","average":{"1min":1.6319999999999999,"5min":1.913,"15min":1.431},"min":{"1min":0.40300000000000002,"5min":0.36199999999999999,"15min":0.30199999999999999},"max":{"1min":6.6959999999999997,"5min":13.734,"15min":13.734},"last":3.1360000000000001},{"interface":"front","average":{"1min":2.278,"5min":1.5129999999999999,"15min":1.28},"min":{"1min":0.40699999999999997,"5min":0.317,"15min":0.317},"max":{"1min":7.5270000000000001,"5min":7.5270000000000001,"15min":8.7449999999999992},"last":2.8860000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:47:27 2023","interfaces":[{"interface":"back","average":{"1min":1.421,"5min":4.6580000000000004,"15min":2.1259999999999999},"min":{"1min":0.379,"5min":0.372,"15min":0.28499999999999998},"max":{"1min":6.2990000000000004,"5min":294.70999999999998,"15min":294.70999999999998},"last":2.8210000000000002},{"interface":"front","average":{"1min":1.111,"5min":4.6340000000000003,"15min":2.218},"min":{"1min":0.40500000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":4.7169999999999996,"5min":294.74400000000003,"15min":294.74400000000003},"last":2.7650000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4722,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1360,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8206,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7156,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3154,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2245332992,"data_stored":2245328920,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":653590528,"data_stored":653590528,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1321926656,"data_stored":1321926656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1168375808,"data_stored":1168375808,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1822035968,"data_stored":1822031896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1144651776,"data_stored":1144651776,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1590366208,"data_stored":1590362136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":917831680,"data_stored":917831680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:48:19.661 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-09-23T20:48:19.661 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:48:19.661 INFO:tasks.thrashosds.thrasher:Removing osd 2, in_osds are: [0, 1, 7, 3, 2, 6, 5, 4] 2023-09-23T20:48:19.661 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd out 2 2023-09-23T20:48:19.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:19 smithi018 ceph-mon[184144]: pgmap v1457: 119 pgs: 119 active+clean; 3.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 919 op/s 2023-09-23T20:48:19.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:19 smithi018 ceph-mon[184144]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:48:19.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:19 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:48:19.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:19 smithi018 ceph-mon[184144]: osdmap e576: 8 total, 8 up, 8 in 2023-09-23T20:48:19.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:19 smithi018 ceph-mon[184144]: failed to encode map e576 with expected crc 2023-09-23T20:48:19.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:19 smithi018 ceph-mon[184144]: from='client.46844 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:48:19.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:19 smithi018 ceph-mon[179647]: pgmap v1457: 119 pgs: 119 active+clean; 3.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 919 op/s 2023-09-23T20:48:19.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:19 smithi018 ceph-mon[179647]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:48:19.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:19 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:48:19.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:19 smithi018 ceph-mon[179647]: osdmap e576: 8 total, 8 up, 8 in 2023-09-23T20:48:19.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:19 smithi018 ceph-mon[179647]: failed to encode map e576 with expected crc 2023-09-23T20:48:19.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:19 smithi018 ceph-mon[179647]: from='client.46844 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:48:19.987 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:48:19.995 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:19.996 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:48:20.097 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:48:20.129 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:48:20.142 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:20.151 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:48:20.188 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:48:20.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:19 smithi149 ceph-mon[170394]: pgmap v1457: 119 pgs: 119 active+clean; 3.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 919 op/s 2023-09-23T20:48:20.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:19 smithi149 ceph-mon[170394]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:48:20.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:19 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:48:20.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:19 smithi149 ceph-mon[170394]: osdmap e576: 8 total, 8 up, 8 in 2023-09-23T20:48:20.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:19 smithi149 ceph-mon[170394]: failed to encode map e576 with expected crc 2023-09-23T20:48:20.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:19 smithi149 ceph-mon[170394]: from='client.46844 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:48:20.242 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:48:20.251 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:20.251 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:48:20.255 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:48:20.298 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:48:20.332 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:48:20.353 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:48:20.362 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:48:20.394 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:48:20.396 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:48:20.396 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:48:20.397 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:48:20.397 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:48:20.397 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:48:20.397 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:48:20.397 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:48:20.397 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:48:20.414 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:20.479 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:48:20.486 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:20.486 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:48:20.587 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:48:20.665 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:20.746 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:48:20.753 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:20.753 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:48:20.771 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:48:20.779 INFO:teuthology.orchestra.run.smithi149.stdout:{ 2023-09-23T20:48:20.779 INFO:teuthology.orchestra.run.smithi149.stdout: "ops": [], 2023-09-23T20:48:20.779 INFO:teuthology.orchestra.run.smithi149.stdout: "num_ops": 0 2023-09-23T20:48:20.779 INFO:teuthology.orchestra.run.smithi149.stdout:} 2023-09-23T20:48:20.855 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:48:20.912 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:20.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:20 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/813529666' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:48:21.005 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:48:21.016 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:21.016 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:20 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/813529666' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:48:21.017 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:48:21.026 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:20 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/813529666' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:48:21.118 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:48:21.187 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:21.190 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T20:48:21.253 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:48:21.255 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:21.256 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:48:21.356 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:48:21.414 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:21.431 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T20:48:21.507 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:48:21.512 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:21.513 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:48:21.613 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:48:21.698 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:21.756 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:48:21.767 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:21.767 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:48:21.868 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:48:21.908 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:21.951 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:48:21.967 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:48:21.970 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:21.971 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:48:22.072 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:48:22.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:21 smithi018 ceph-mon[179647]: pgmap v1459: 119 pgs: 119 active+clean; 3.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 597 op/s 2023-09-23T20:48:22.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:21 smithi018 ceph-mon[184144]: pgmap v1459: 119 pgs: 119 active+clean; 3.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 597 op/s 2023-09-23T20:48:22.159 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:22.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:21 smithi149 ceph-mon[170394]: pgmap v1459: 119 pgs: 119 active+clean; 3.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 597 op/s 2023-09-23T20:48:22.227 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:48:22.232 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:22.232 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:48:22.333 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:48:22.452 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:22.482 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:48:22.538 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:48:22.548 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:22.548 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:48:22.649 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:48:22.694 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:22.933 INFO:teuthology.orchestra.run.smithi018.stderr:marked out osd.2. 2023-09-23T20:48:23.051 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:48:23.056 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:48:23.059 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:23.059 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:48:23.081 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:22 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/4173356155' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-09-23T20:48:23.081 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:22 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-09-23T20:48:23.081 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:22 smithi018 ceph-mon[184144]: Client client.admin marked osd.2 out, while it was still marked up 2023-09-23T20:48:23.081 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:22 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mon-a[179623]: 2023-09-23T20:48:22.870+0000 7f160fef9700 -1 mon.a@0(leader).osd e576 definitely_dead 0 2023-09-23T20:48:23.082 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:22 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/4173356155' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-09-23T20:48:23.082 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:22 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-09-23T20:48:23.082 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:22 smithi018 ceph-mon[179647]: Client client.admin marked osd.2 out, while it was still marked up 2023-09-23T20:48:23.160 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:48:23.184 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:22 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/4173356155' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-09-23T20:48:23.184 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:22 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-09-23T20:48:23.184 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:22 smithi149 ceph-mon[170394]: Client client.admin marked osd.2 out, while it was still marked up 2023-09-23T20:48:23.187 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:23.700 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:48:23.704 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:23.704 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:48:23.805 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:48:23.833 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:23.869 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:48:23.872 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:23.873 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:48:23.896 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:48:23.951 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:23 smithi149 ceph-mon[170394]: pgmap v1460: 119 pgs: 119 active+clean; 3.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 804 op/s 2023-09-23T20:48:23.952 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:23 smithi149 ceph-mon[170394]: failed to encode map e577 with expected crc 2023-09-23T20:48:23.952 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:23 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["2"]}]': finished 2023-09-23T20:48:23.952 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:23 smithi149 ceph-mon[170394]: osdmap e577: 8 total, 8 up, 7 in 2023-09-23T20:48:23.974 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:48:24.006 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:24.097 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:48:24.101 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:24.101 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:48:24.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:23 smithi018 ceph-mon[184144]: pgmap v1460: 119 pgs: 119 active+clean; 3.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 804 op/s 2023-09-23T20:48:24.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:23 smithi018 ceph-mon[184144]: failed to encode map e577 with expected crc 2023-09-23T20:48:24.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:23 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["2"]}]': finished 2023-09-23T20:48:24.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:23 smithi018 ceph-mon[184144]: osdmap e577: 8 total, 8 up, 7 in 2023-09-23T20:48:24.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:23 smithi018 ceph-mon[179647]: pgmap v1460: 119 pgs: 119 active+clean; 3.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 804 op/s 2023-09-23T20:48:24.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:23 smithi018 ceph-mon[179647]: failed to encode map e577 with expected crc 2023-09-23T20:48:24.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:23 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["2"]}]': finished 2023-09-23T20:48:24.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:23 smithi018 ceph-mon[179647]: osdmap e577: 8 total, 8 up, 7 in 2023-09-23T20:48:24.192 INFO:teuthology.orchestra.run.smithi149.stdout:{ 2023-09-23T20:48:24.193 INFO:teuthology.orchestra.run.smithi149.stdout: "ops": [], 2023-09-23T20:48:24.193 INFO:teuthology.orchestra.run.smithi149.stdout: "complaint_time": 30, 2023-09-23T20:48:24.193 INFO:teuthology.orchestra.run.smithi149.stdout: "num_blocked_ops": 0 2023-09-23T20:48:24.193 INFO:teuthology.orchestra.run.smithi149.stdout:} 2023-09-23T20:48:24.202 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:48:24.232 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:24.275 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:48:24.278 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:24.279 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:48:24.379 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:48:24.407 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:24.537 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:48:24.541 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:24.541 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:48:24.616 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T20:48:24.642 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:48:24.673 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:24.715 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:48:24.718 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:24.718 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:48:24.819 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:48:24.848 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:24.923 INFO:teuthology.orchestra.run.smithi018.stderr:nodeep-scrub is set 2023-09-23T20:48:24.935 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:48:24.939 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:24 smithi018 ceph-mon[184144]: failed to encode map e578 with expected crc 2023-09-23T20:48:24.939 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:24 smithi018 ceph-mon[184144]: failed to encode map e578 with expected crc 2023-09-23T20:48:24.939 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:24 smithi018 ceph-mon[184144]: osdmap e578: 8 total, 8 up, 7 in 2023-09-23T20:48:24.939 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:24 smithi018 ceph-mon[184144]: failed to encode map e578 with expected crc 2023-09-23T20:48:24.939 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:24 smithi018 ceph-mon[184144]: failed to encode map e578 with expected crc 2023-09-23T20:48:24.939 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:24 smithi018 ceph-mon[184144]: failed to encode map e578 with expected crc 2023-09-23T20:48:24.939 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:24 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2451589739' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:48:24.939 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:24 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:48:24.940 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:24 smithi018 ceph-mon[179647]: failed to encode map e578 with expected crc 2023-09-23T20:48:24.940 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:24 smithi018 ceph-mon[179647]: failed to encode map e578 with expected crc 2023-09-23T20:48:24.940 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:24 smithi018 ceph-mon[179647]: osdmap e578: 8 total, 8 up, 7 in 2023-09-23T20:48:24.940 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:24 smithi018 ceph-mon[179647]: failed to encode map e578 with expected crc 2023-09-23T20:48:24.940 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:24 smithi018 ceph-mon[179647]: failed to encode map e578 with expected crc 2023-09-23T20:48:24.940 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:24 smithi018 ceph-mon[179647]: failed to encode map e578 with expected crc 2023-09-23T20:48:24.940 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:24 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2451589739' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:48:24.941 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:24 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:48:24.941 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:24.941 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:48:25.042 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:48:25.122 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:25.180 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:48:25.203 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:48:25.208 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:25.208 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:48:25.218 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:48:25.219 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:24 smithi149 ceph-mon[170394]: failed to encode map e578 with expected crc 2023-09-23T20:48:25.220 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:24 smithi149 ceph-mon[170394]: failed to encode map e578 with expected crc 2023-09-23T20:48:25.220 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:24 smithi149 ceph-mon[170394]: osdmap e578: 8 total, 8 up, 7 in 2023-09-23T20:48:25.220 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:24 smithi149 ceph-mon[170394]: failed to encode map e578 with expected crc 2023-09-23T20:48:25.220 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:24 smithi149 ceph-mon[170394]: failed to encode map e578 with expected crc 2023-09-23T20:48:25.220 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:24 smithi149 ceph-mon[170394]: failed to encode map e578 with expected crc 2023-09-23T20:48:25.220 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:24 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2451589739' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:48:25.220 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:24 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:48:25.227 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:48:24 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:48:24] "GET /metrics HTTP/1.1" 200 36573 "" "Prometheus/2.43.0" 2023-09-23T20:48:25.264 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:48:25.296 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:48:25.309 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:48:25.335 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:48:25.339 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:48:25.387 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:48:25.402 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:25.442 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:48:25.463 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:48:25.469 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:25.470 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:48:25.482 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:48:25.489 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:48:25.489 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:48:25.489 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:48:25.489 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:48:25.489 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:48:25.489 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:48:25.489 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:48:25.490 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:48:25.571 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:48:25.649 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:25.864 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:48:25.869 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:25.870 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:48:25.971 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:48:26.015 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:26.074 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:48:26.080 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:26.081 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:48:26.146 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:25 smithi018 ceph-mon[184144]: pgmap v1463: 119 pgs: 5 peering, 114 active+clean; 3.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 573 op/s 2023-09-23T20:48:26.146 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:25 smithi018 ceph-mon[184144]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T20:48:26.146 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:25 smithi018 ceph-mon[184144]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:48:26.146 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:25 smithi018 ceph-mon[184144]: failed to encode map e579 with expected crc 2023-09-23T20:48:26.146 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:25 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:48:26.147 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:25 smithi018 ceph-mon[184144]: osdmap e579: 8 total, 8 up, 7 in 2023-09-23T20:48:26.147 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:25 smithi018 ceph-mon[184144]: failed to encode map e579 with expected crc 2023-09-23T20:48:26.147 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:25 smithi018 ceph-mon[184144]: failed to encode map e579 with expected crc 2023-09-23T20:48:26.147 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:25 smithi018 ceph-mon[184144]: failed to encode map e579 with expected crc 2023-09-23T20:48:26.147 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:25 smithi018 ceph-mon[184144]: 10.3 starting backfill to osd.4 from (0'0,0'0] MAX to 576'10144 2023-09-23T20:48:26.147 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:25 smithi018 ceph-mon[184144]: 10.8 starting backfill to osd.5 from (0'0,0'0] MAX to 576'9853 2023-09-23T20:48:26.147 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:25 smithi018 ceph-mon[184144]: 10.e starting backfill to osd.5 from (0'0,0'0] MAX to 576'9903 2023-09-23T20:48:26.147 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:25 smithi018 ceph-mon[184144]: 10.5 starting backfill to osd.7 from (0'0,0'0] MAX to 576'9988 2023-09-23T20:48:26.147 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:25 smithi018 ceph-mon[184144]: failed to encode map e579 with expected crc 2023-09-23T20:48:26.147 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:25 smithi018 ceph-mon[184144]: 10.f starting backfill to osd.4 from (0'0,0'0] MAX to 576'9859 2023-09-23T20:48:26.147 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:25 smithi018 ceph-mon[184144]: failed to encode map e579 with expected crc 2023-09-23T20:48:26.148 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:25 smithi018 ceph-mon[184144]: 10.9 starting backfill to osd.4 from (0'0,0'0] MAX to 576'10045 2023-09-23T20:48:26.148 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:25 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/499612394' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:48:26.148 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:25 smithi018 ceph-mon[179647]: pgmap v1463: 119 pgs: 5 peering, 114 active+clean; 3.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 573 op/s 2023-09-23T20:48:26.148 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:25 smithi018 ceph-mon[179647]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T20:48:26.148 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:25 smithi018 ceph-mon[179647]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:48:26.148 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:25 smithi018 ceph-mon[179647]: failed to encode map e579 with expected crc 2023-09-23T20:48:26.148 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:25 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:48:26.148 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:25 smithi018 ceph-mon[179647]: osdmap e579: 8 total, 8 up, 7 in 2023-09-23T20:48:26.148 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:25 smithi018 ceph-mon[179647]: failed to encode map e579 with expected crc 2023-09-23T20:48:26.148 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:25 smithi018 ceph-mon[179647]: failed to encode map e579 with expected crc 2023-09-23T20:48:26.149 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:25 smithi018 ceph-mon[179647]: failed to encode map e579 with expected crc 2023-09-23T20:48:26.149 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:25 smithi018 ceph-mon[179647]: 10.3 starting backfill to osd.4 from (0'0,0'0] MAX to 576'10144 2023-09-23T20:48:26.149 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:25 smithi018 ceph-mon[179647]: 10.8 starting backfill to osd.5 from (0'0,0'0] MAX to 576'9853 2023-09-23T20:48:26.149 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:25 smithi018 ceph-mon[179647]: 10.e starting backfill to osd.5 from (0'0,0'0] MAX to 576'9903 2023-09-23T20:48:26.149 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:25 smithi018 ceph-mon[179647]: 10.5 starting backfill to osd.7 from (0'0,0'0] MAX to 576'9988 2023-09-23T20:48:26.149 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:25 smithi018 ceph-mon[179647]: failed to encode map e579 with expected crc 2023-09-23T20:48:26.149 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:25 smithi018 ceph-mon[179647]: 10.f starting backfill to osd.4 from (0'0,0'0] MAX to 576'9859 2023-09-23T20:48:26.149 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:25 smithi018 ceph-mon[179647]: failed to encode map e579 with expected crc 2023-09-23T20:48:26.149 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:25 smithi018 ceph-mon[179647]: 10.9 starting backfill to osd.4 from (0'0,0'0] MAX to 576'10045 2023-09-23T20:48:26.149 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:25 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/499612394' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:48:26.182 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:48:26.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:25 smithi149 ceph-mon[170394]: pgmap v1463: 119 pgs: 5 peering, 114 active+clean; 3.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 573 op/s 2023-09-23T20:48:26.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:25 smithi149 ceph-mon[170394]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T20:48:26.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:25 smithi149 ceph-mon[170394]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:48:26.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:25 smithi149 ceph-mon[170394]: failed to encode map e579 with expected crc 2023-09-23T20:48:26.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:25 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:48:26.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:25 smithi149 ceph-mon[170394]: osdmap e579: 8 total, 8 up, 7 in 2023-09-23T20:48:26.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:25 smithi149 ceph-mon[170394]: failed to encode map e579 with expected crc 2023-09-23T20:48:26.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:25 smithi149 ceph-mon[170394]: failed to encode map e579 with expected crc 2023-09-23T20:48:26.218 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:25 smithi149 ceph-mon[170394]: failed to encode map e579 with expected crc 2023-09-23T20:48:26.218 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:25 smithi149 ceph-mon[170394]: 10.3 starting backfill to osd.4 from (0'0,0'0] MAX to 576'10144 2023-09-23T20:48:26.218 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:25 smithi149 ceph-mon[170394]: 10.8 starting backfill to osd.5 from (0'0,0'0] MAX to 576'9853 2023-09-23T20:48:26.218 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:25 smithi149 ceph-mon[170394]: 10.e starting backfill to osd.5 from (0'0,0'0] MAX to 576'9903 2023-09-23T20:48:26.218 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:25 smithi149 ceph-mon[170394]: 10.5 starting backfill to osd.7 from (0'0,0'0] MAX to 576'9988 2023-09-23T20:48:26.218 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:25 smithi149 ceph-mon[170394]: failed to encode map e579 with expected crc 2023-09-23T20:48:26.218 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:25 smithi149 ceph-mon[170394]: 10.f starting backfill to osd.4 from (0'0,0'0] MAX to 576'9859 2023-09-23T20:48:26.218 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:25 smithi149 ceph-mon[170394]: failed to encode map e579 with expected crc 2023-09-23T20:48:26.218 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:25 smithi149 ceph-mon[170394]: 10.9 starting backfill to osd.4 from (0'0,0'0] MAX to 576'10045 2023-09-23T20:48:26.218 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:25 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/499612394' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:48:26.260 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:26.354 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:48:26.363 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:26.363 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:48:26.464 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:48:26.506 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:26.576 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:48:26.584 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:26.584 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:48:26.685 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:48:26.778 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:26.859 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:48:26.869 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:26.870 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:48:26.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:26 smithi149 ceph-mon[170394]: osdmap e580: 8 total, 8 up, 7 in 2023-09-23T20:48:26.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:26 smithi149 ceph-mon[170394]: failed to encode map e580 with expected crc 2023-09-23T20:48:26.968 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:26 smithi149 ceph-mon[170394]: failed to encode map e580 with expected crc 2023-09-23T20:48:26.968 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:26 smithi149 ceph-mon[170394]: failed to encode map e580 with expected crc 2023-09-23T20:48:26.968 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:26 smithi149 ceph-mon[170394]: failed to encode map e580 with expected crc 2023-09-23T20:48:26.970 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:48:27.052 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:27.054 INFO:teuthology.orchestra.run.smithi149.stdout:ERROR: (22) Invalid argument 2023-09-23T20:48:27.054 INFO:teuthology.orchestra.run.smithi149.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:48:27.099 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:48:27.105 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:27.105 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:48:27.206 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:48:27.284 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:27.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:26 smithi018 ceph-mon[184144]: osdmap e580: 8 total, 8 up, 7 in 2023-09-23T20:48:27.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:26 smithi018 ceph-mon[184144]: failed to encode map e580 with expected crc 2023-09-23T20:48:27.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:26 smithi018 ceph-mon[184144]: failed to encode map e580 with expected crc 2023-09-23T20:48:27.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:26 smithi018 ceph-mon[184144]: failed to encode map e580 with expected crc 2023-09-23T20:48:27.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:26 smithi018 ceph-mon[184144]: failed to encode map e580 with expected crc 2023-09-23T20:48:27.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:26 smithi018 ceph-mon[179647]: osdmap e580: 8 total, 8 up, 7 in 2023-09-23T20:48:27.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:26 smithi018 ceph-mon[179647]: failed to encode map e580 with expected crc 2023-09-23T20:48:27.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:26 smithi018 ceph-mon[179647]: failed to encode map e580 with expected crc 2023-09-23T20:48:27.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:26 smithi018 ceph-mon[179647]: failed to encode map e580 with expected crc 2023-09-23T20:48:27.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:26 smithi018 ceph-mon[179647]: failed to encode map e580 with expected crc 2023-09-23T20:48:27.432 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:48:27.437 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:27.438 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:48:27.529 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T20:48:27.534 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T20:48:27.539 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:48:27.638 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:27.692 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:48:27.695 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:27.696 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:48:27.796 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:48:27.894 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:27.908 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:48:27.978 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:48:27.982 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:27.983 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:48:28.083 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:48:28.122 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:28.178 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:48:28.184 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:28.184 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:27 smithi149 ceph-mon[170394]: pgmap v1466: 119 pgs: 3 active+remapped+backfill_wait, 7 active+undersized+remapped, 2 remapped+peering, 5 peering, 102 active+clean; 3.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 27 KiB/s rd, 73 KiB/s wr, 441 op/s; 9354/151359 objects misplaced (6.180%); 1.2 MiB/s, 2 keys/s, 2 objects/s recovering 2023-09-23T20:48:28.184 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:27 smithi149 ceph-mon[170394]: osdmap e581: 8 total, 8 up, 7 in 2023-09-23T20:48:28.185 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:27 smithi149 ceph-mon[170394]: failed to encode map e581 with expected crc 2023-09-23T20:48:28.185 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:27 smithi149 ceph-mon[170394]: failed to encode map e581 with expected crc 2023-09-23T20:48:28.185 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:27 smithi149 ceph-mon[170394]: failed to encode map e581 with expected crc 2023-09-23T20:48:28.185 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:27 smithi149 ceph-mon[170394]: failed to encode map e581 with expected crc 2023-09-23T20:48:28.185 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:27 smithi149 ceph-mon[170394]: failed to encode map e581 with expected crc 2023-09-23T20:48:28.185 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:27 smithi149 ceph-mon[170394]: failed to encode map e581 with expected crc 2023-09-23T20:48:28.185 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:27 smithi149 ceph-mon[170394]: failed to encode map e581 with expected crc 2023-09-23T20:48:28.186 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:48:28.287 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:48:28.295 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:48:28.314 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:27 smithi018 ceph-mon[184144]: pgmap v1466: 119 pgs: 3 active+remapped+backfill_wait, 7 active+undersized+remapped, 2 remapped+peering, 5 peering, 102 active+clean; 3.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 27 KiB/s rd, 73 KiB/s wr, 441 op/s; 9354/151359 objects misplaced (6.180%); 1.2 MiB/s, 2 keys/s, 2 objects/s recovering 2023-09-23T20:48:28.314 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:27 smithi018 ceph-mon[184144]: osdmap e581: 8 total, 8 up, 7 in 2023-09-23T20:48:28.315 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:27 smithi018 ceph-mon[184144]: failed to encode map e581 with expected crc 2023-09-23T20:48:28.315 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:27 smithi018 ceph-mon[184144]: failed to encode map e581 with expected crc 2023-09-23T20:48:28.315 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:27 smithi018 ceph-mon[184144]: failed to encode map e581 with expected crc 2023-09-23T20:48:28.315 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:27 smithi018 ceph-mon[184144]: failed to encode map e581 with expected crc 2023-09-23T20:48:28.315 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:27 smithi018 ceph-mon[184144]: failed to encode map e581 with expected crc 2023-09-23T20:48:28.315 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:27 smithi018 ceph-mon[184144]: failed to encode map e581 with expected crc 2023-09-23T20:48:28.316 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:27 smithi018 ceph-mon[184144]: failed to encode map e581 with expected crc 2023-09-23T20:48:28.316 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:27 smithi018 ceph-mon[179647]: pgmap v1466: 119 pgs: 3 active+remapped+backfill_wait, 7 active+undersized+remapped, 2 remapped+peering, 5 peering, 102 active+clean; 3.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 27 KiB/s rd, 73 KiB/s wr, 441 op/s; 9354/151359 objects misplaced (6.180%); 1.2 MiB/s, 2 keys/s, 2 objects/s recovering 2023-09-23T20:48:28.316 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:27 smithi018 ceph-mon[179647]: osdmap e581: 8 total, 8 up, 7 in 2023-09-23T20:48:28.316 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:27 smithi018 ceph-mon[179647]: failed to encode map e581 with expected crc 2023-09-23T20:48:28.317 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:27 smithi018 ceph-mon[179647]: failed to encode map e581 with expected crc 2023-09-23T20:48:28.317 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:27 smithi018 ceph-mon[179647]: failed to encode map e581 with expected crc 2023-09-23T20:48:28.317 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:27 smithi018 ceph-mon[179647]: failed to encode map e581 with expected crc 2023-09-23T20:48:28.317 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:27 smithi018 ceph-mon[179647]: failed to encode map e581 with expected crc 2023-09-23T20:48:28.317 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:27 smithi018 ceph-mon[179647]: failed to encode map e581 with expected crc 2023-09-23T20:48:28.317 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:27 smithi018 ceph-mon[179647]: failed to encode map e581 with expected crc 2023-09-23T20:48:28.319 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:28.345 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:48:28.393 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:48:28.399 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:28.400 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:48:28.415 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 7, 3, 6, 5, 4] out_osds: [2] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T20:48:28.415 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:48:28.415 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_5 2023-09-23T20:48:28.415 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:48:28.501 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:48:28.576 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:28.692 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:48:28.697 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:28.698 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:48:28.798 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:48:28.877 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:28.927 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:48:29.196 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:48:29.207 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:29.207 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:48:29.308 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:48:29.377 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:29.510 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:48:29.518 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:29.518 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:48:29.619 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:48:29.710 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:48:29.711 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:29.832 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:48:29.837 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:29.838 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:48:29.939 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:48:29.955 INFO:teuthology.orchestra.run.smithi149.stdout:ERROR: (22) Invalid argument 2023-09-23T20:48:29.955 INFO:teuthology.orchestra.run.smithi149.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:48:29.955 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:29 smithi018 ceph-mon[179647]: pgmap v1468: 119 pgs: 1 active+remapped+backfilling, 4 active+remapped+backfill_wait, 7 active+undersized+remapped, 2 remapped+peering, 6 peering, 99 active+clean; 3.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 25 KiB/s rd, 66 KiB/s wr, 505 op/s; 15242/149280 objects misplaced (10.210%); 3.5 MiB/s, 2 keys/s, 40 objects/s recovering 2023-09-23T20:48:29.956 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:29 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:48:29.956 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:29 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:48:29.993 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:30.069 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:48:30.074 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:30.075 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:48:30.095 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:29 smithi149 ceph-mon[170394]: pgmap v1468: 119 pgs: 1 active+remapped+backfilling, 4 active+remapped+backfill_wait, 7 active+undersized+remapped, 2 remapped+peering, 6 peering, 99 active+clean; 3.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 25 KiB/s rd, 66 KiB/s wr, 505 op/s; 15242/149280 objects misplaced (10.210%); 3.5 MiB/s, 2 keys/s, 40 objects/s recovering 2023-09-23T20:48:30.095 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:29 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:48:30.095 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:29 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:48:30.176 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:48:30.219 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:29 smithi018 ceph-mon[184144]: pgmap v1468: 119 pgs: 1 active+remapped+backfilling, 4 active+remapped+backfill_wait, 7 active+undersized+remapped, 2 remapped+peering, 6 peering, 99 active+clean; 3.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 25 KiB/s rd, 66 KiB/s wr, 505 op/s; 15242/149280 objects misplaced (10.210%); 3.5 MiB/s, 2 keys/s, 40 objects/s recovering 2023-09-23T20:48:30.219 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:29 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:48:30.219 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:29 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:48:30.298 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:30.466 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T20:48:30.532 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:48:30.545 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:30.546 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:48:30.647 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:48:30.717 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:30.796 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:48:30.802 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:30.802 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:48:30.903 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:48:30.989 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:30.995 INFO:teuthology.orchestra.run.smithi018.stderr:noscrub is unset 2023-09-23T20:48:31.034 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:30 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/687086410' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:48:31.034 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:30 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:48:31.035 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:30 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/687086410' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:48:31.035 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:30 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:48:31.050 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:48:31.053 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:31.053 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:48:31.154 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:48:31.213 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:31.274 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:48:31.288 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:48:31.293 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:30 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/687086410' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:48:31.293 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:30 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:48:31.294 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:31.294 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:48:31.395 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:48:31.490 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:31.564 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:48:31.568 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:48:31.571 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:31.572 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:48:31.599 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:48:31.629 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:48:31.666 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:48:31.672 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:48:31.702 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:48:31.741 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:48:31.780 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:48:31.786 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:31.828 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:48:31.830 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:48:31.830 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:48:31.830 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:48:31.831 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:48:31.831 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:48:31.831 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:48:31.831 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:48:31.831 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:48:31.867 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:48:31.870 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:31.871 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:48:31.972 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:48:32.057 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:32.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:31 smithi018 ceph-mon[184144]: pgmap v1469: 119 pgs: 1 active+remapped+backfilling, 5 active+remapped+backfill_wait, 7 active+undersized+remapped, 1 peering, 105 active+clean; 3.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 18 KiB/s rd, 48 KiB/s wr, 463 op/s; 18120/147627 objects misplaced (12.274%); 4.7 MiB/s, 1 keys/s, 67 objects/s recovering 2023-09-23T20:48:32.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:31 smithi018 ceph-mon[184144]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 3 pgs peering) 2023-09-23T20:48:32.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:31 smithi018 ceph-mon[184144]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:48:32.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:31 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:48:32.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:31 smithi018 ceph-mon[184144]: osdmap e582: 8 total, 8 up, 7 in 2023-09-23T20:48:32.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:31 smithi018 ceph-mon[184144]: failed to encode map e582 with expected crc 2023-09-23T20:48:32.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:31 smithi018 ceph-mon[184144]: failed to encode map e582 with expected crc 2023-09-23T20:48:32.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:31 smithi018 ceph-mon[184144]: failed to encode map e582 with expected crc 2023-09-23T20:48:32.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:31 smithi018 ceph-mon[184144]: failed to encode map e582 with expected crc 2023-09-23T20:48:32.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:31 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3686177113' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:48:32.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:31 smithi018 ceph-mon[179647]: pgmap v1469: 119 pgs: 1 active+remapped+backfilling, 5 active+remapped+backfill_wait, 7 active+undersized+remapped, 1 peering, 105 active+clean; 3.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 18 KiB/s rd, 48 KiB/s wr, 463 op/s; 18120/147627 objects misplaced (12.274%); 4.7 MiB/s, 1 keys/s, 67 objects/s recovering 2023-09-23T20:48:32.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:31 smithi018 ceph-mon[179647]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 3 pgs peering) 2023-09-23T20:48:32.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:31 smithi018 ceph-mon[179647]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:48:32.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:31 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:48:32.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:31 smithi018 ceph-mon[179647]: osdmap e582: 8 total, 8 up, 7 in 2023-09-23T20:48:32.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:31 smithi018 ceph-mon[179647]: failed to encode map e582 with expected crc 2023-09-23T20:48:32.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:31 smithi018 ceph-mon[179647]: failed to encode map e582 with expected crc 2023-09-23T20:48:32.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:31 smithi018 ceph-mon[179647]: failed to encode map e582 with expected crc 2023-09-23T20:48:32.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:31 smithi018 ceph-mon[179647]: failed to encode map e582 with expected crc 2023-09-23T20:48:32.144 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:31 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3686177113' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:48:32.195 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:48:32.207 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:48:32.209 INFO:teuthology.orchestra.run.smithi018.stderr:dumped all 2023-09-23T20:48:32.209 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:32.210 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:48:32.210 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:31 smithi149 ceph-mon[170394]: pgmap v1469: 119 pgs: 1 active+remapped+backfilling, 5 active+remapped+backfill_wait, 7 active+undersized+remapped, 1 peering, 105 active+clean; 3.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 18 KiB/s rd, 48 KiB/s wr, 463 op/s; 18120/147627 objects misplaced (12.274%); 4.7 MiB/s, 1 keys/s, 67 objects/s recovering 2023-09-23T20:48:32.210 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:31 smithi149 ceph-mon[170394]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 3 pgs peering) 2023-09-23T20:48:32.210 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:31 smithi149 ceph-mon[170394]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:48:32.211 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:31 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:48:32.211 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:31 smithi149 ceph-mon[170394]: osdmap e582: 8 total, 8 up, 7 in 2023-09-23T20:48:32.211 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:31 smithi149 ceph-mon[170394]: failed to encode map e582 with expected crc 2023-09-23T20:48:32.211 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:31 smithi149 ceph-mon[170394]: failed to encode map e582 with expected crc 2023-09-23T20:48:32.211 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:31 smithi149 ceph-mon[170394]: failed to encode map e582 with expected crc 2023-09-23T20:48:32.211 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:31 smithi149 ceph-mon[170394]: failed to encode map e582 with expected crc 2023-09-23T20:48:32.211 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:31 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3686177113' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:48:32.311 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:48:32.386 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:32.564 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:48:32.573 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:32.574 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:48:32.655 INFO:teuthology.orchestra.run.smithi018.stdout:{"pg_ready":true,"pg_map":{"version":1470,"stamp":"2023-09-23T20:48:30.977826+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3073322834,"num_objects":46850,"num_object_clones":0,"num_object_copies":140550,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":16882,"num_objects_unfound":0,"num_objects_dirty":46850,"num_whiteouts":0,"num_read":6043,"num_read_kb":13311,"num_write":163101,"num_write_kb":6700569,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3372,"num_bytes_recovered":227049950,"num_keys_recovered":70,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":101911,"ondisk_log_size":101911,"up":357,"acting":357,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":349,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":12899228,"kb_used_data":8036580,"kb_used_omap":20,"kb_used_meta":4862571,"kb_avail":643316836,"statfs":{"total":671965249536,"available":658756440064,"internally_reserved":0,"allocated":8229457920,"data_stored":8214834699,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":20895,"internal_metadata":4979273313},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,1,0,0,0,8,0,4],"upper_bound":8192},"perf_stat":{"commit_latency_ms":7,"apply_latency_ms":7,"commit_latency_ns":7000000,"apply_latency_ns":7000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-331370828,"num_objects":-5057,"num_object_clones":0,"num_object_copies":-15171,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":16882,"num_objects_unfound":0,"num_objects_dirty":-5057,"num_whiteouts":0,"num_read":37,"num_read_kb":69,"num_write":5074,"num_write_kb":182,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":904,"num_bytes_recovered":62525578,"num_keys_recovered":10,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.073835"},"pg_stats":[{"pgid":"1.18","version":"90'161","reported_seq":644,"reported_epoch":581,"state":"active+clean","last_fresh":"2023-09-23T20:48:26.950828+0000","last_change":"2023-09-23T20:48:26.950828+0000","last_active":"2023-09-23T20:48:26.950828+0000","last_peered":"2023-09-23T20:48:26.950828+0000","last_clean":"2023-09-23T20:48:26.950828+0000","last_became_active":"2023-09-23T20:48:26.950430+0000","last_became_peered":"2023-09-23T20:48:26.950430+0000","last_unstale":"2023-09-23T20:48:26.950828+0000","last_undegraded":"2023-09-23T20:48:26.950828+0000","last_fullsized":"2023-09-23T20:48:26.950828+0000","mapping_epoch":580,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":581,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:19.677209+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:19.677209+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:54:02.455574+0000","scrub_duration":0.033122453000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1a","version":"69'1","reported_seq":48,"reported_epoch":580,"state":"active+clean","last_fresh":"2023-09-23T20:48:25.932967+0000","last_change":"2023-09-23T20:48:23.918481+0000","last_active":"2023-09-23T20:48:25.932967+0000","last_peered":"2023-09-23T20:48:25.932967+0000","last_clean":"2023-09-23T20:48:25.932967+0000","last_became_active":"2023-09-23T20:48:23.918128+0000","last_became_peered":"2023-09-23T20:48:23.918128+0000","last_unstale":"2023-09-23T20:48:25.932967+0000","last_undegraded":"2023-09-23T20:48:25.932967+0000","last_fullsized":"2023-09-23T20:48:25.932967+0000","mapping_epoch":577,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":578,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:21:57.686475+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":415,"reported_epoch":580,"state":"active+clean","last_fresh":"2023-09-23T20:48:25.921651+0000","last_change":"2023-09-23T20:47:09.197577+0000","last_active":"2023-09-23T20:48:25.921651+0000","last_peered":"2023-09-23T20:48:25.921651+0000","last_clean":"2023-09-23T20:48:25.921651+0000","last_became_active":"2023-09-23T20:47:09.197280+0000","last_became_peered":"2023-09-23T20:47:09.197280+0000","last_unstale":"2023-09-23T20:48:25.921651+0000","last_undegraded":"2023-09-23T20:48:25.921651+0000","last_fullsized":"2023-09-23T20:48:25.921651+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:55:04.346224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"90'161","reported_seq":584,"reported_epoch":580,"state":"active+clean","last_fresh":"2023-09-23T20:48:25.928575+0000","last_change":"2023-09-23T20:47:33.523278+0000","last_active":"2023-09-23T20:48:25.928575+0000","last_peered":"2023-09-23T20:48:25.928575+0000","last_clean":"2023-09-23T20:48:25.928575+0000","last_became_active":"2023-09-23T20:47:11.622087+0000","last_became_peered":"2023-09-23T20:47:11.622087+0000","last_unstale":"2023-09-23T20:48:25.928575+0000","last_undegraded":"2023-09-23T20:48:25.928575+0000","last_fullsized":"2023-09-23T20:48:25.928575+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:47:33.523187+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:36:05.442288+0000","last_clean_scrub_stamp":"2023-09-23T20:47:33.523187+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:32:18.410526+0000","scrub_duration":0.013532127,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":570,"reported_epoch":580,"state":"active+clean","last_fresh":"2023-09-23T20:48:25.929426+0000","last_change":"2023-09-23T20:47:09.173501+0000","last_active":"2023-09-23T20:48:25.929426+0000","last_peered":"2023-09-23T20:48:25.929426+0000","last_clean":"2023-09-23T20:48:25.929426+0000","last_became_active":"2023-09-23T20:47:09.173160+0000","last_became_peered":"2023-09-23T20:47:09.173160+0000","last_unstale":"2023-09-23T20:48:25.929426+0000","last_undegraded":"2023-09-23T20:48:25.929426+0000","last_fullsized":"2023-09-23T20:48:25.929426+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:44:36.132093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":539,"reported_epoch":579,"state":"active+clean","last_fresh":"2023-09-23T20:48:24.910196+0000","last_change":"2023-09-23T20:43:46.521939+0000","last_active":"2023-09-23T20:48:24.910196+0000","last_peered":"2023-09-23T20:48:24.910196+0000","last_clean":"2023-09-23T20:48:24.910196+0000","last_became_active":"2023-09-23T20:43:46.521185+0000","last_became_peered":"2023-09-23T20:43:46.521185+0000","last_unstale":"2023-09-23T20:48:24.910196+0000","last_undegraded":"2023-09-23T20:48:24.910196+0000","last_fullsized":"2023-09-23T20:48:24.910196+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:59:25.045627+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"90'161","reported_seq":469,"reported_epoch":580,"state":"active+clean","last_fresh":"2023-09-23T20:48:25.932805+0000","last_change":"2023-09-23T20:47:31.475001+0000","last_active":"2023-09-23T20:48:25.932805+0000","last_peered":"2023-09-23T20:48:25.932805+0000","last_clean":"2023-09-23T20:48:25.932805+0000","last_became_active":"2023-09-23T20:47:11.617942+0000","last_became_peered":"2023-09-23T20:47:11.617942+0000","last_unstale":"2023-09-23T20:48:25.932805+0000","last_undegraded":"2023-09-23T20:48:25.932805+0000","last_fullsized":"2023-09-23T20:48:25.932805+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:47:31.474914+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:47:31.474914+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:52:05.169219+0000","scrub_duration":0.0026262870000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":377,"reported_epoch":580,"state":"active+clean","last_fresh":"2023-09-23T20:48:25.919897+0000","last_change":"2023-09-23T20:37:43.584414+0000","last_active":"2023-09-23T20:48:25.919897+0000","last_peered":"2023-09-23T20:48:25.919897+0000","last_clean":"2023-09-23T20:48:25.919897+0000","last_became_active":"2023-09-23T20:37:43.584116+0000","last_became_peered":"2023-09-23T20:37:43.584116+0000","last_unstale":"2023-09-23T20:48:25.919897+0000","last_undegraded":"2023-09-23T20:48:25.919897+0000","last_fullsized":"2023-09-23T20:48:25.919897+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:17:54.258172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":196,"reported_epoch":580,"state":"active+clean","last_fresh":"2023-09-23T20:48:25.940707+0000","last_change":"2023-09-23T20:38:37.457427+0000","last_active":"2023-09-23T20:48:25.940707+0000","last_peered":"2023-09-23T20:48:25.940707+0000","last_clean":"2023-09-23T20:48:25.940707+0000","last_became_active":"2023-09-23T20:38:37.457099+0000","last_became_peered":"2023-09-23T20:38:37.457099+0000","last_unstale":"2023-09-23T20:48:25.940707+0000","last_undegraded":"2023-09-23T20:48:25.940707+0000","last_fullsized":"2023-09-23T20:48:25.940707+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:31:27.084994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"90'161","reported_seq":793,"reported_epoch":580,"state":"active+clean","last_fresh":"2023-09-23T20:48:25.927632+0000","last_change":"2023-09-23T20:40:22.449754+0000","last_active":"2023-09-23T20:48:25.927632+0000","last_peered":"2023-09-23T20:48:25.927632+0000","last_clean":"2023-09-23T20:48:25.927632+0000","last_became_active":"2023-09-23T20:40:22.448838+0000","last_became_peered":"2023-09-23T20:40:22.448838+0000","last_unstale":"2023-09-23T20:48:25.927632+0000","last_undegraded":"2023-09-23T20:48:25.927632+0000","last_fullsized":"2023-09-23T20:48:25.927632+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:03.411025+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:03.411025+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:22:07.948856+0000","scrub_duration":0.001802922,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":571,"reported_epoch":579,"state":"active+clean","last_fresh":"2023-09-23T20:48:24.910655+0000","last_change":"2023-09-23T20:43:46.522325+0000","last_active":"2023-09-23T20:48:24.910655+0000","last_peered":"2023-09-23T20:48:24.910655+0000","last_clean":"2023-09-23T20:48:24.910655+0000","last_became_active":"2023-09-23T20:43:46.521996+0000","last_became_peered":"2023-09-23T20:43:46.521996+0000","last_unstale":"2023-09-23T20:48:24.910655+0000","last_undegraded":"2023-09-23T20:48:24.910655+0000","last_fullsized":"2023-09-23T20:48:24.910655+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:51:17.629956+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":14,"reported_epoch":580,"state":"active+clean","last_fresh":"2023-09-23T20:48:25.919643+0000","last_change":"2023-09-23T20:48:23.918092+0000","last_active":"2023-09-23T20:48:25.919643+0000","last_peered":"2023-09-23T20:48:25.919643+0000","last_clean":"2023-09-23T20:48:25.919643+0000","last_became_active":"2023-09-23T20:48:23.917386+0000","last_became_peered":"2023-09-23T20:48:23.917386+0000","last_unstale":"2023-09-23T20:48:25.919643+0000","last_undegraded":"2023-09-23T20:48:25.919643+0000","last_fullsized":"2023-09-23T20:48:25.919643+0000","mapping_epoch":577,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":578,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:06:09.587517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"90'161","reported_seq":588,"reported_epoch":581,"state":"active+clean","last_fresh":"2023-09-23T20:48:26.961347+0000","last_change":"2023-09-23T20:48:26.961347+0000","last_active":"2023-09-23T20:48:26.961347+0000","last_peered":"2023-09-23T20:48:26.961347+0000","last_clean":"2023-09-23T20:48:26.961347+0000","last_became_active":"2023-09-23T20:48:26.961005+0000","last_became_peered":"2023-09-23T20:48:26.961005+0000","last_unstale":"2023-09-23T20:48:26.961347+0000","last_undegraded":"2023-09-23T20:48:26.961347+0000","last_fullsized":"2023-09-23T20:48:26.961347+0000","mapping_epoch":580,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":581,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:47.479012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:47.479012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":417,"reported_epoch":580,"state":"active+clean","last_fresh":"2023-09-23T20:48:25.927291+0000","last_change":"2023-09-23T20:43:16.780478+0000","last_active":"2023-09-23T20:48:25.927291+0000","last_peered":"2023-09-23T20:48:25.927291+0000","last_clean":"2023-09-23T20:48:25.927291+0000","last_became_active":"2023-09-23T20:43:16.780183+0000","last_became_peered":"2023-09-23T20:43:16.780183+0000","last_unstale":"2023-09-23T20:48:25.927291+0000","last_undegraded":"2023-09-23T20:48:25.927291+0000","last_fullsized":"2023-09-23T20:48:25.927291+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:38:30.102066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":408,"reported_epoch":580,"state":"active+clean","last_fresh":"2023-09-23T20:48:25.921178+0000","last_change":"2023-09-23T20:38:02.542034+0000","last_active":"2023-09-23T20:48:25.921178+0000","last_peered":"2023-09-23T20:48:25.921178+0000","last_clean":"2023-09-23T20:48:25.921178+0000","last_became_active":"2023-09-23T20:38:02.541575+0000","last_became_peered":"2023-09-23T20:38:02.541575+0000","last_unstale":"2023-09-23T20:48:25.921178+0000","last_undegraded":"2023-09-23T20:48:25.921178+0000","last_fullsized":"2023-09-23T20:48:25.921178+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:50:54.591529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"90'161","reported_seq":706,"reported_epoch":579,"state":"active+clean","last_fresh":"2023-09-23T20:48:24.909964+0000","last_change":"2023-09-23T20:44:14.669973+0000","last_active":"2023-09-23T20:48:24.909964+0000","last_peered":"2023-09-23T20:48:24.909964+0000","last_clean":"2023-09-23T20:48:24.909964+0000","last_became_active":"2023-09-23T20:43:46.519592+0000","last_became_peered":"2023-09-23T20:43:46.519592+0000","last_unstale":"2023-09-23T20:48:24.909964+0000","last_undegraded":"2023-09-23T20:48:24.909964+0000","last_fullsized":"2023-09-23T20:48:24.909964+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:14.669831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:14.669831+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:56:28.076617+0000","scrub_duration":0.0083441460000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":640,"reported_epoch":580,"state":"active+clean","last_fresh":"2023-09-23T20:48:25.927987+0000","last_change":"2023-09-23T20:47:10.837423+0000","last_active":"2023-09-23T20:48:25.927987+0000","last_peered":"2023-09-23T20:48:25.927987+0000","last_clean":"2023-09-23T20:48:25.927987+0000","last_became_active":"2023-09-23T20:47:09.204324+0000","last_became_peered":"2023-09-23T20:47:09.204324+0000","last_unstale":"2023-09-23T20:48:25.927987+0000","last_undegraded":"2023-09-23T20:48:25.927987+0000","last_fullsized":"2023-09-23T20:48:25.927987+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":554,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:19.108914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":540,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":105,"reported_epoch":580,"state":"active+clean","last_fresh":"2023-09-23T20:48:25.929927+0000","last_change":"2023-09-23T20:48:23.926375+0000","last_active":"2023-09-23T20:48:25.929927+0000","last_peered":"2023-09-23T20:48:25.929927+0000","last_clean":"2023-09-23T20:48:25.929927+0000","last_became_active":"2023-09-23T20:48:23.925975+0000","last_became_peered":"2023-09-23T20:48:23.925975+0000","last_unstale":"2023-09-23T20:48:25.929927+0000","last_undegraded":"2023-09-23T20:48:25.929927+0000","last_fullsized":"2023-09-23T20:48:25.929927+0000","mapping_epoch":577,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":578,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:33:54.704406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"90'161","reported_seq":708,"reported_epoch":580,"state":"active+clean","last_fresh":"2023-09-23T20:48:25.929811+0000","last_change":"2023-09-23T20:36:02.548387+0000","last_active":"2023-09-23T20:48:25.929811+0000","last_peered":"2023-09-23T20:48:25.929811+0000","last_clean":"2023-09-23T20:48:25.929811+0000","last_became_active":"2023-09-23T20:35:26.644787+0000","last_became_peered":"2023-09-23T20:35:26.644787+0000","last_unstale":"2023-09-23T20:48:25.929811+0000","last_undegraded":"2023-09-23T20:48:25.929811+0000","last_fullsized":"2023-09-23T20:48:25.929811+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:02.548285+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:02.548285+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:37:15.216553+0000","scrub_duration":0.0015514459999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":48,"reported_epoch":580,"state":"active+clean","last_fresh":"2023-09-23T20:48:25.929742+0000","last_change":"2023-09-23T20:47:09.340979+0000","last_active":"2023-09-23T20:48:25.929742+0000","last_peered":"2023-09-23T20:48:25.929742+0000","last_clean":"2023-09-23T20:48:25.929742+0000","last_became_active":"2023-09-23T20:47:09.231222+0000","last_became_peered":"2023-09-23T20:47:09.231222+0000","last_unstale":"2023-09-23T20:48:25.929742+0000","last_undegraded":"2023-09-23T20:48:25.929742+0000","last_fullsized":"2023-09-23T20:48:25.929742+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:07:50.620006+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":38,"reported_epoch":580,"state":"active+clean","last_fresh":"2023-09-23T20:48:25.928794+0000","last_change":"2023-09-23T20:47:09.186315+0000","last_active":"2023-09-23T20:48:25.928794+0000","last_peered":"2023-09-23T20:48:25.928794+0000","last_clean":"2023-09-23T20:48:25.928794+0000","last_became_active":"2023-09-23T20:47:09.184580+0000","last_became_peered":"2023-09-23T20:47:09.184580+0000","last_unstale":"2023-09-23T20:48:25.928794+0000","last_undegraded":"2023-09-23T20:48:25.928794+0000","last_fullsized":"2023-09-23T20:48:25.928794+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:44:35.553948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"90'161","reported_seq":835,"reported_epoch":580,"state":"active+clean","last_fresh":"2023-09-23T20:48:25.935283+0000","last_change":"2023-09-23T20:43:46.544727+0000","last_active":"2023-09-23T20:48:25.935283+0000","last_peered":"2023-09-23T20:48:25.935283+0000","last_clean":"2023-09-23T20:48:25.935283+0000","last_became_active":"2023-09-23T20:43:46.543952+0000","last_became_peered":"2023-09-23T20:43:46.543952+0000","last_unstale":"2023-09-23T20:48:25.935283+0000","last_undegraded":"2023-09-23T20:48:25.935283+0000","last_fullsized":"2023-09-23T20:48:25.935283+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:43.578666+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:43.578666+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:11:16.611920+0000","scrub_duration":0.0021890379999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":655,"reported_epoch":580,"state":"active+clean","last_fresh":"2023-09-23T20:48:25.928587+0000","last_change":"2023-09-23T20:48:23.922278+0000","last_active":"2023-09-23T20:48:25.928587+0000","last_peered":"2023-09-23T20:48:25.928587+0000","last_clean":"2023-09-23T20:48:25.928587+0000","last_became_active":"2023-09-23T20:48:23.914560+0000","last_became_peered":"2023-09-23T20:48:23.914560+0000","last_unstale":"2023-09-23T20:48:25.928587+0000","last_undegraded":"2023-09-23T20:48:25.928587+0000","last_fullsized":"2023-09-23T20:48:25.928587+0000","mapping_epoch":577,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":578,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:53:50.393571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":238,"num_keys_recovered":70,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":328,"reported_epoch":580,"state":"active+clean","last_fresh":"2023-09-23T20:48:25.923958+0000","last_change":"2023-09-23T20:45:01.875931+0000","last_active":"2023-09-23T20:48:25.923958+0000","last_peered":"2023-09-23T20:48:25.923958+0000","last_clean":"2023-09-23T20:48:25.923958+0000","last_became_active":"2023-09-23T20:45:01.872839+0000","last_became_peered":"2023-09-23T20:45:01.872839+0000","last_unstale":"2023-09-23T20:48:25.923958+0000","last_undegraded":"2023-09-23T20:48:25.923958+0000","last_fullsized":"2023-09-23T20:48:25.923958+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:15:09.584507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"90'161","reported_seq":861,"reported_epoch":580,"state":"active+clean","last_fresh":"2023-09-23T20:48:25.929145+0000","last_change":"2023-09-23T20:44:20.041766+0000","last_active":"2023-09-23T20:48:25.929145+0000","last_peered":"2023-09-23T20:48:25.929145+0000","last_clean":"2023-09-23T20:48:25.929145+0000","last_became_active":"2023-09-23T20:43:16.765015+0000","last_became_peered":"2023-09-23T20:43:16.765015+0000","last_unstale":"2023-09-23T20:48:25.929145+0000","last_undegraded":"2023-09-23T20:48:25.929145+0000","last_fullsized":"2023-09-23T20:48:25.929145+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:20.041690+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:20.041690+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:33:30.150109+0000","scrub_duration":0.0026180330000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":537,"reported_epoch":580,"state":"active+clean","last_fresh":"2023-09-23T20:48:25.935526+0000","last_change":"2023-09-23T20:43:46.521294+0000","last_active":"2023-09-23T20:48:25.935526+0000","last_peered":"2023-09-23T20:48:25.935526+0000","last_clean":"2023-09-23T20:48:25.935526+0000","last_became_active":"2023-09-23T20:43:46.521001+0000","last_became_peered":"2023-09-23T20:43:46.521001+0000","last_unstale":"2023-09-23T20:48:25.935526+0000","last_undegraded":"2023-09-23T20:48:25.935526+0000","last_fullsized":"2023-09-23T20:48:25.935526+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:01:35.126716+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":358,"reported_epoch":580,"state":"active+clean","last_fresh":"2023-09-23T20:48:25.928121+0000","last_change":"2023-09-23T20:48:23.914091+0000","last_active":"2023-09-23T20:48:25.928121+0000","last_peered":"2023-09-23T20:48:25.928121+0000","last_clean":"2023-09-23T20:48:25.928121+0000","last_became_active":"2023-09-23T20:48:23.913768+0000","last_became_peered":"2023-09-23T20:48:23.913768+0000","last_unstale":"2023-09-23T20:48:25.928121+0000","last_undegraded":"2023-09-23T20:48:25.928121+0000","last_fullsized":"2023-09-23T20:48:25.928121+0000","mapping_epoch":577,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":578,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:19:38.311290+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.13","version":"90'161","reported_seq":903,"reported_epoch":580,"state":"active+clean","last_fresh":"2023-09-23T20:48:25.933102+0000","last_change":"2023-09-23T20:47:11.619371+0000","last_active":"2023-09-23T20:48:25.933102+0000","last_peered":"2023-09-23T20:48:25.933102+0000","last_clean":"2023-09-23T20:48:25.933102+0000","last_became_active":"2023-09-23T20:47:11.619006+0000","last_became_peered":"2023-09-23T20:47:11.619006+0000","last_unstale":"2023-09-23T20:48:25.933102+0000","last_undegraded":"2023-09-23T20:48:25.933102+0000","last_fullsized":"2023-09-23T20:48:25.933102+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:42.544308+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:42.544308+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:47:24.414807+0000","scrub_duration":0.0029958179999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":604,"reported_epoch":580,"state":"active+clean","last_fresh":"2023-09-23T20:48:25.928082+0000","last_change":"2023-09-23T20:47:09.176613+0000","last_active":"2023-09-23T20:48:25.928082+0000","last_peered":"2023-09-23T20:48:25.928082+0000","last_clean":"2023-09-23T20:48:25.928082+0000","last_became_active":"2023-09-23T20:47:09.176187+0000","last_became_peered":"2023-09-23T20:47:09.176187+0000","last_unstale":"2023-09-23T20:48:25.928082+0000","last_undegraded":"2023-09-23T20:48:25.928082+0000","last_fullsized":"2023-09-23T20:48:25.928082+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:23:01.590989+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":545,"reported_epoch":579,"state":"active+clean","last_fresh":"2023-09-23T20:48:24.910254+0000","last_change":"2023-09-23T20:43:46.521735+0000","last_active":"2023-09-23T20:48:24.910254+0000","last_peered":"2023-09-23T20:48:24.910254+0000","last_clean":"2023-09-23T20:48:24.910254+0000","last_became_active":"2023-09-23T20:43:46.520762+0000","last_became_peered":"2023-09-23T20:43:46.520762+0000","last_unstale":"2023-09-23T20:48:24.910254+0000","last_undegraded":"2023-09-23T20:48:24.910254+0000","last_fullsized":"2023-09-23T20:48:24.910254+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:26:48.841930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"90'161","reported_seq":666,"reported_epoch":580,"state":"active+clean","last_fresh":"2023-09-23T20:48:25.930490+0000","last_change":"2023-09-23T20:44:18.952808+0000","last_active":"2023-09-23T20:48:25.930490+0000","last_peered":"2023-09-23T20:48:25.930490+0000","last_clean":"2023-09-23T20:48:25.930490+0000","last_became_active":"2023-09-23T20:43:16.758652+0000","last_became_peered":"2023-09-23T20:43:16.758652+0000","last_unstale":"2023-09-23T20:48:25.930490+0000","last_undegraded":"2023-09-23T20:48:25.930490+0000","last_fullsized":"2023-09-23T20:48:25.930490+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:18.952693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:18.952693+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:12:19.496465+0000","scrub_duration":0.001699365,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":256,"reported_epoch":580,"state":"active+clean","last_fresh":"2023-09-23T20:48:25.936609+0000","last_change":"2023-09-23T20:44:43.644290+0000","last_active":"2023-09-23T20:48:25.936609+0000","last_peered":"2023-09-23T20:48:25.936609+0000","last_clean":"2023-09-23T20:48:25.936609+0000","last_became_active":"2023-09-23T20:44:43.638313+0000","last_became_peered":"2023-09-23T20:44:43.638313+0000","last_unstale":"2023-09-23T20:48:25.936609+0000","last_undegraded":"2023-09-23T20:48:25.936609+0000","last_fullsized":"2023-09-23T20:48:25.936609+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:32:52.892603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":377,"reported_epoch":580,"state":"active+clean","last_fresh":"2023-09-23T20:48:25.923672+0000","last_change":"2023-09-23T20:37:43.571083+0000","last_active":"2023-09-23T20:48:25.923672+0000","last_peered":"2023-09-23T20:48:25.923672+0000","last_clean":"2023-09-23T20:48:25.923672+0000","last_became_active":"2023-09-23T20:37:43.570306+0000","last_became_peered":"2023-09-23T20:37:43.570306+0000","last_unstale":"2023-09-23T20:48:25.923672+0000","last_undegraded":"2023-09-23T20:48:25.923672+0000","last_fullsized":"2023-09-23T20:48:25.923672+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:42:02.396718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.6","version":"580'10044","reported_seq":10704,"reported_epoch":580,"state":"active+clean","last_fresh":"2023-09-23T20:48:26.532295+0000","last_change":"2023-09-23T20:47:12.636171+0000","last_active":"2023-09-23T20:48:26.532295+0000","last_peered":"2023-09-23T20:48:26.532295+0000","last_clean":"2023-09-23T20:48:26.532295+0000","last_became_active":"2023-09-23T20:47:12.628915+0000","last_became_peered":"2023-09-23T20:47:12.628915+0000","last_unstale":"2023-09-23T20:48:26.532295+0000","last_undegraded":"2023-09-23T20:48:26.532295+0000","last_fullsized":"2023-09-23T20:48:26.532295+0000","mapping_epoch":556,"log_start":"540'4090","ondisk_log_start":"540'4090","created":531,"last_epoch_clean":557,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":5954,"log_dups_size":0,"ondisk_log_size":5954,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:52:38.896682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":201850880,"num_objects":3080,"num_object_clones":0,"num_object_copies":9240,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3080,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10044,"num_write_kb":419968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":287,"num_bytes_recovered":18808832,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.d","version":"90'161","reported_seq":913,"reported_epoch":580,"state":"active+clean","last_fresh":"2023-09-23T20:48:25.927720+0000","last_change":"2023-09-23T20:47:11.619510+0000","last_active":"2023-09-23T20:48:25.927720+0000","last_peered":"2023-09-23T20:48:25.927720+0000","last_clean":"2023-09-23T20:48:25.927720+0000","last_became_active":"2023-09-23T20:47:11.619154+0000","last_became_peered":"2023-09-23T20:47:11.619154+0000","last_unstale":"2023-09-23T20:48:25.927720+0000","last_undegraded":"2023-09-23T20:48:25.927720+0000","last_fullsized":"2023-09-23T20:48:25.927720+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:34:59.496375+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:34:59.496375+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:40:52.712777+0000","scrub_duration":0.0017843189999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":612,"reported_epoch":580,"state":"active+clean","last_fresh":"2023-09-23T20:48:25.930704+0000","last_change":"2023-09-23T20:47:09.207851+0000","last_active":"2023-09-23T20:48:25.930704+0000","last_peered":"2023-09-23T20:48:25.930704+0000","last_clean":"2023-09-23T20:48:25.930704+0000","last_became_active":"2023-09-23T20:47:09.175836+0000","last_became_peered":"2023-09-23T20:47:09.175836+0000","last_unstale":"2023-09-23T20:48:25.930704+0000","last_undegraded":"2023-09-23T20:48:25.930704+0000","last_fullsized":"2023-09-23T20:48:25.930704+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:43:13.091809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":395,"reported_epoch":580,"state":"active+clean","last_fresh":"2023-09-23T20:48:25.929507+0000","last_change":"2023-09-23T20:43:16.765253+0000","last_active":"2023-09-23T20:48:25.929507+0000","last_peered":"2023-09-23T20:48:25.929507+0000","last_clean":"2023-09-23T20:48:25.929507+0000","last_became_active":"2023-09-23T20:43:16.764978+0000","last_became_peered":"2023-09-23T20:43:16.764978+0000","last_unstale":"2023-09-23T20:48:25.929507+0000","last_undegraded":"2023-09-23T20:48:25.929507+0000","last_fullsized":"2023-09-23T20:48:25.929507+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:07:23.539372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.7","version":"581'10085","reported_seq":10143,"reported_epoch":581,"state":"active+clean","last_fresh":"2023-09-23T20:48:30.434368+0000","last_change":"2023-09-23T20:45:39.488542+0000","last_active":"2023-09-23T20:48:30.434368+0000","last_peered":"2023-09-23T20:48:30.434368+0000","last_clean":"2023-09-23T20:48:30.434368+0000","last_became_active":"2023-09-23T20:45:39.488259+0000","last_became_peered":"2023-09-23T20:45:39.488259+0000","last_unstale":"2023-09-23T20:48:30.434368+0000","last_undegraded":"2023-09-23T20:48:30.434368+0000","last_fullsized":"2023-09-23T20:48:30.434368+0000","mapping_epoch":531,"log_start":"541'4363","ondisk_log_start":"541'4363","created":531,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":5722,"log_dups_size":0,"ondisk_log_size":5722,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:39:05.833662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":189333504,"num_objects":2889,"num_object_clones":0,"num_object_copies":8667,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2889,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10085,"num_write_kb":415168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.c","version":"90'161","reported_seq":843,"reported_epoch":580,"state":"active+clean","last_fresh":"2023-09-23T20:48:25.924033+0000","last_change":"2023-09-23T20:41:09.506481+0000","last_active":"2023-09-23T20:48:25.924033+0000","last_peered":"2023-09-23T20:48:25.924033+0000","last_clean":"2023-09-23T20:48:25.924033+0000","last_became_active":"2023-09-23T20:38:28.442229+0000","last_became_peered":"2023-09-23T20:38:28.442229+0000","last_unstale":"2023-09-23T20:48:25.924033+0000","last_undegraded":"2023-09-23T20:48:25.924033+0000","last_fullsized":"2023-09-23T20:48:25.924033+0000","mapping_epoch":411,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":412,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:41:09.506338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:41:09.506338+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:19:46.265011+0000","scrub_duration":0.01112002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":650,"reported_epoch":580,"state":"active+clean","last_fresh":"2023-09-23T20:48:25.927384+0000","last_change":"2023-09-23T20:47:10.848436+0000","last_active":"2023-09-23T20:48:25.927384+0000","last_peered":"2023-09-23T20:48:25.927384+0000","last_clean":"2023-09-23T20:48:25.927384+0000","last_became_active":"2023-09-23T20:47:09.196769+0000","last_became_peered":"2023-09-23T20:47:09.196769+0000","last_unstale":"2023-09-23T20:48:25.927384+0000","last_undegraded":"2023-09-23T20:48:25.927384+0000","last_fullsized":"2023-09-23T20:48:25.927384+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":554,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:59:38.105009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":48,"reported_epoch":580,"state":"active+clean","last_fresh":"2023-09-23T20:48:25.927996+0000","last_change":"2023-09-23T20:48:23.915817+0000","last_active":"2023-09-23T20:48:25.927996+0000","last_peered":"2023-09-23T20:48:25.927996+0000","last_clean":"2023-09-23T20:48:25.927996+0000","last_became_active":"2023-09-23T20:48:23.915306+0000","last_became_peered":"2023-09-23T20:48:23.915306+0000","last_unstale":"2023-09-23T20:48:25.927996+0000","last_undegraded":"2023-09-23T20:48:25.927996+0000","last_fullsized":"2023-09-23T20:48:25.927996+0000","mapping_epoch":577,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":578,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:55:45.691450+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"580'9909","reported_seq":10541,"reported_epoch":580,"state":"active+clean","last_fresh":"2023-09-23T20:48:26.563208+0000","last_change":"2023-09-23T20:47:14.635782+0000","last_active":"2023-09-23T20:48:26.563208+0000","last_peered":"2023-09-23T20:48:26.563208+0000","last_clean":"2023-09-23T20:48:26.563208+0000","last_became_active":"2023-09-23T20:47:14.627887+0000","last_became_peered":"2023-09-23T20:47:14.627887+0000","last_unstale":"2023-09-23T20:48:26.563208+0000","last_undegraded":"2023-09-23T20:48:26.563208+0000","last_fullsized":"2023-09-23T20:48:26.563208+0000","mapping_epoch":558,"log_start":"539'3947","ondisk_log_start":"539'3947","created":531,"last_epoch_clean":559,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":5962,"log_dups_size":0,"ondisk_log_size":5962,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:13:49.501508+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":197722112,"num_objects":3017,"num_object_clones":0,"num_object_copies":9051,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3017,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9909,"num_write_kb":413632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":274,"num_bytes_recovered":17956864,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"90'161","reported_seq":776,"reported_epoch":580,"state":"active+clean","last_fresh":"2023-09-23T20:48:25.941148+0000","last_change":"2023-09-23T20:47:11.622303+0000","last_active":"2023-09-23T20:48:25.941148+0000","last_peered":"2023-09-23T20:48:25.941148+0000","last_clean":"2023-09-23T20:48:25.941148+0000","last_became_active":"2023-09-23T20:47:11.621953+0000","last_became_peered":"2023-09-23T20:47:11.621953+0000","last_unstale":"2023-09-23T20:48:25.941148+0000","last_undegraded":"2023-09-23T20:48:25.941148+0000","last_fullsized":"2023-09-23T20:48:25.941148+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:46.217489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:46.217489+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:14:40.478522+0000","scrub_duration":0.001456429,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":609,"reported_epoch":580,"state":"active+clean","last_fresh":"2023-09-23T20:48:25.927811+0000","last_change":"2023-09-23T20:40:19.677655+0000","last_active":"2023-09-23T20:48:25.927811+0000","last_peered":"2023-09-23T20:48:25.927811+0000","last_clean":"2023-09-23T20:48:25.927811+0000","last_became_active":"2023-09-23T20:40:19.551189+0000","last_became_peered":"2023-09-23T20:40:19.551189+0000","last_unstale":"2023-09-23T20:48:25.927811+0000","last_undegraded":"2023-09-23T20:48:25.927811+0000","last_fullsized":"2023-09-23T20:48:25.927811+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:11:01.559576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":186,"reported_epoch":580,"state":"active+clean","last_fresh":"2023-09-23T20:48:25.935289+0000","last_change":"2023-09-23T20:38:02.590613+0000","last_active":"2023-09-23T20:48:25.935289+0000","last_peered":"2023-09-23T20:48:25.935289+0000","last_clean":"2023-09-23T20:48:25.935289+0000","last_became_active":"2023-09-23T20:38:02.590284+0000","last_became_peered":"2023-09-23T20:38:02.590284+0000","last_unstale":"2023-09-23T20:48:25.935289+0000","last_undegraded":"2023-09-23T20:48:25.935289+0000","last_fullsized":"2023-09-23T20:48:25.935289+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:40:07.825054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.5","version":"580'10056","reported_seq":10122,"reported_epoch":580,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T20:48:26.558401+0000","last_change":"2023-09-23T20:48:25.251116+0000","last_active":"2023-09-23T20:48:26.558401+0000","last_peered":"2023-09-23T20:48:26.558401+0000","last_clean":"2023-09-23T20:48:22.702941+0000","last_became_active":"2023-09-23T20:48:25.181499+0000","last_became_peered":"2023-09-23T20:48:25.181499+0000","last_unstale":"2023-09-23T20:48:26.558401+0000","last_undegraded":"2023-09-23T20:48:26.558401+0000","last_fullsized":"2023-09-23T20:48:26.558401+0000","mapping_epoch":578,"log_start":"540'4106","ondisk_log_start":"540'4106","created":531,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":5950,"log_dups_size":0,"ondisk_log_size":5950,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:23:50.522657+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":204996608,"num_objects":3128,"num_object_clones":0,"num_object_copies":9384,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3128,"num_objects_unfound":0,"num_objects_dirty":3128,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10056,"num_write_kb":421888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,2],"avail_no_missing":["0","2","3"],"object_location_counts":[{"shards":"0,2,3","objects":3128}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.e","version":"90'161","reported_seq":611,"reported_epoch":580,"state":"active+clean","last_fresh":"2023-09-23T20:48:25.927475+0000","last_change":"2023-09-23T20:47:37.608426+0000","last_active":"2023-09-23T20:48:25.927475+0000","last_peered":"2023-09-23T20:48:25.927475+0000","last_clean":"2023-09-23T20:48:25.927475+0000","last_became_active":"2023-09-23T20:47:11.617187+0000","last_became_peered":"2023-09-23T20:47:11.617187+0000","last_unstale":"2023-09-23T20:48:25.927475+0000","last_undegraded":"2023-09-23T20:48:25.927475+0000","last_fullsized":"2023-09-23T20:48:25.927475+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:47:37.608309+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:47:37.608309+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:22:25.482297+0000","scrub_duration":0.011600924,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":542,"reported_epoch":580,"state":"active+clean","last_fresh":"2023-09-23T20:48:25.924335+0000","last_change":"2023-09-23T20:43:16.756437+0000","last_active":"2023-09-23T20:48:25.924335+0000","last_peered":"2023-09-23T20:48:25.924335+0000","last_clean":"2023-09-23T20:48:25.924335+0000","last_became_active":"2023-09-23T20:43:16.756193+0000","last_became_peered":"2023-09-23T20:43:16.756193+0000","last_unstale":"2023-09-23T20:48:25.924335+0000","last_undegraded":"2023-09-23T20:48:25.924335+0000","last_fullsized":"2023-09-23T20:48:25.924335+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:33:25.632781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":573,"reported_epoch":579,"state":"active+clean","last_fresh":"2023-09-23T20:48:24.910061+0000","last_change":"2023-09-23T20:47:09.197427+0000","last_active":"2023-09-23T20:48:24.910061+0000","last_peered":"2023-09-23T20:48:24.910061+0000","last_clean":"2023-09-23T20:48:24.910061+0000","last_became_active":"2023-09-23T20:47:09.196542+0000","last_became_peered":"2023-09-23T20:47:09.196542+0000","last_unstale":"2023-09-23T20:48:24.910061+0000","last_undegraded":"2023-09-23T20:48:24.910061+0000","last_fullsized":"2023-09-23T20:48:24.910061+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:28:38.926202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"581'10332","reported_seq":11380,"reported_epoch":581,"state":"active+clean","last_fresh":"2023-09-23T20:48:28.622651+0000","last_change":"2023-09-23T20:47:16.635196+0000","last_active":"2023-09-23T20:48:28.622651+0000","last_peered":"2023-09-23T20:48:28.622651+0000","last_clean":"2023-09-23T20:48:28.622651+0000","last_became_active":"2023-09-23T20:47:16.628907+0000","last_became_peered":"2023-09-23T20:47:16.628907+0000","last_unstale":"2023-09-23T20:48:28.622651+0000","last_undegraded":"2023-09-23T20:48:28.622651+0000","last_fullsized":"2023-09-23T20:48:28.622651+0000","mapping_epoch":560,"log_start":"541'4474","ondisk_log_start":"541'4474","created":531,"last_epoch_clean":561,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":5858,"log_dups_size":0,"ondisk_log_size":5858,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:30:22.186026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":193462272,"num_objects":2952,"num_object_clones":0,"num_object_copies":8856,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2952,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10332,"num_write_kb":425088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":477,"num_bytes_recovered":31260672,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":1006,"reported_epoch":581,"state":"active+clean","last_fresh":"2023-09-23T20:48:26.954969+0000","last_change":"2023-09-23T20:48:26.954969+0000","last_active":"2023-09-23T20:48:26.954969+0000","last_peered":"2023-09-23T20:48:26.954969+0000","last_clean":"2023-09-23T20:48:26.954969+0000","last_became_active":"2023-09-23T20:48:26.954606+0000","last_became_peered":"2023-09-23T20:48:26.954606+0000","last_unstale":"2023-09-23T20:48:26.954969+0000","last_undegraded":"2023-09-23T20:48:26.954969+0000","last_fullsized":"2023-09-23T20:48:26.954969+0000","mapping_epoch":580,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":581,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:35:02.523203+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:35:02.523203+0000","last_clean_scrub_stamp":"2023-09-23T20:35:02.523203+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:18:26.324451+0000","scrub_duration":0.0085927989999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":429,"reported_epoch":580,"state":"active+clean","last_fresh":"2023-09-23T20:48:25.923255+0000","last_change":"2023-09-23T20:47:09.240200+0000","last_active":"2023-09-23T20:48:25.923255+0000","last_peered":"2023-09-23T20:48:25.923255+0000","last_clean":"2023-09-23T20:48:25.923255+0000","last_became_active":"2023-09-23T20:47:09.195204+0000","last_became_peered":"2023-09-23T20:47:09.195204+0000","last_unstale":"2023-09-23T20:48:25.923255+0000","last_undegraded":"2023-09-23T20:48:25.923255+0000","last_fullsized":"2023-09-23T20:48:25.923255+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:03.328250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":38,"reported_epoch":580,"state":"active+clean","last_fresh":"2023-09-23T20:48:25.929659+0000","last_change":"2023-09-23T20:47:09.231714+0000","last_active":"2023-09-23T20:48:25.929659+0000","last_peered":"2023-09-23T20:48:25.929659+0000","last_clean":"2023-09-23T20:48:25.929659+0000","last_became_active":"2023-09-23T20:47:09.230918+0000","last_became_peered":"2023-09-23T20:47:09.230918+0000","last_unstale":"2023-09-23T20:48:25.929659+0000","last_undegraded":"2023-09-23T20:48:25.929659+0000","last_fullsized":"2023-09-23T20:48:25.929659+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:24.583656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"581'10271","reported_seq":10758,"reported_epoch":581,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T20:48:28.247880+0000","last_change":"2023-09-23T20:48:26.954125+0000","last_active":"2023-09-23T20:48:28.247880+0000","last_peered":"2023-09-23T20:48:28.247880+0000","last_clean":"2023-09-23T20:48:22.778971+0000","last_became_active":"2023-09-23T20:48:25.144910+0000","last_became_peered":"2023-09-23T20:48:25.144910+0000","last_unstale":"2023-09-23T20:48:28.247880+0000","last_undegraded":"2023-09-23T20:48:28.247880+0000","last_fullsized":"2023-09-23T20:48:28.247880+0000","mapping_epoch":578,"log_start":"536'1300","ondisk_log_start":"536'1300","created":531,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":8971,"log_dups_size":0,"ondisk_log_size":8971,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:12:48.296057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":201654272,"num_objects":3077,"num_object_clones":0,"num_object_copies":9231,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2878,"num_objects_unfound":0,"num_objects_dirty":3077,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10271,"num_write_kb":427136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":208,"num_bytes_recovered":13631488,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,3],"acting":[2,5,3],"avail_no_missing":["2","3","5"],"object_location_counts":[{"shards":"2,3,5","objects":3077}],"blocked_by":[],"up_primary":4,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":439,"reported_epoch":581,"state":"active+clean","last_fresh":"2023-09-23T20:48:26.952415+0000","last_change":"2023-09-23T20:48:26.952415+0000","last_active":"2023-09-23T20:48:26.952415+0000","last_peered":"2023-09-23T20:48:26.952415+0000","last_clean":"2023-09-23T20:48:26.952415+0000","last_became_active":"2023-09-23T20:48:26.952095+0000","last_became_peered":"2023-09-23T20:48:26.952095+0000","last_unstale":"2023-09-23T20:48:26.952415+0000","last_undegraded":"2023-09-23T20:48:26.952415+0000","last_fullsized":"2023-09-23T20:48:26.952415+0000","mapping_epoch":580,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":581,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:42:45.492220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:42:45.492220+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":206,"reported_epoch":580,"state":"active+clean","last_fresh":"2023-09-23T20:48:25.935381+0000","last_change":"2023-09-23T20:47:09.178919+0000","last_active":"2023-09-23T20:48:25.935381+0000","last_peered":"2023-09-23T20:48:25.935381+0000","last_clean":"2023-09-23T20:48:25.935381+0000","last_became_active":"2023-09-23T20:47:09.178607+0000","last_became_peered":"2023-09-23T20:47:09.178607+0000","last_unstale":"2023-09-23T20:48:25.935381+0000","last_undegraded":"2023-09-23T20:48:25.935381+0000","last_fullsized":"2023-09-23T20:48:25.935381+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:23:45.196935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":48,"reported_epoch":580,"state":"active+clean","last_fresh":"2023-09-23T20:48:25.927287+0000","last_change":"2023-09-23T20:48:23.916858+0000","last_active":"2023-09-23T20:48:25.927287+0000","last_peered":"2023-09-23T20:48:25.927287+0000","last_clean":"2023-09-23T20:48:25.927287+0000","last_became_active":"2023-09-23T20:48:23.916437+0000","last_became_peered":"2023-09-23T20:48:23.916437+0000","last_unstale":"2023-09-23T20:48:25.927287+0000","last_undegraded":"2023-09-23T20:48:25.927287+0000","last_fullsized":"2023-09-23T20:48:25.927287+0000","mapping_epoch":577,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":578,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:59:55.106280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.0","version":"581'10178","reported_seq":11424,"reported_epoch":581,"state":"active+clean","last_fresh":"2023-09-23T20:48:30.410057+0000","last_change":"2023-09-23T20:47:15.653243+0000","last_active":"2023-09-23T20:48:30.410057+0000","last_peered":"2023-09-23T20:48:30.410057+0000","last_clean":"2023-09-23T20:48:30.410057+0000","last_became_active":"2023-09-23T20:47:15.626592+0000","last_became_peered":"2023-09-23T20:47:15.626592+0000","last_unstale":"2023-09-23T20:48:30.410057+0000","last_undegraded":"2023-09-23T20:48:30.410057+0000","last_fullsized":"2023-09-23T20:48:30.410057+0000","mapping_epoch":559,"log_start":"541'4445","ondisk_log_start":"541'4445","created":531,"last_epoch_clean":560,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":5733,"log_dups_size":0,"ondisk_log_size":5733,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:57:31.932707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":184287232,"num_objects":2812,"num_object_clones":0,"num_object_copies":8436,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2812,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10178,"num_write_kb":415680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":578,"num_bytes_recovered":37879808,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.b","version":"90'161","reported_seq":886,"reported_epoch":580,"state":"active+clean","last_fresh":"2023-09-23T20:48:25.933188+0000","last_change":"2023-09-23T20:40:22.445808+0000","last_active":"2023-09-23T20:48:25.933188+0000","last_peered":"2023-09-23T20:48:25.933188+0000","last_clean":"2023-09-23T20:48:25.933188+0000","last_became_active":"2023-09-23T20:40:22.445456+0000","last_became_peered":"2023-09-23T20:40:22.445456+0000","last_unstale":"2023-09-23T20:48:25.933188+0000","last_undegraded":"2023-09-23T20:48:25.933188+0000","last_fullsized":"2023-09-23T20:48:25.933188+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:05.595042+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:05.595042+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:39:23.678154+0000","scrub_duration":0.007271675,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":48,"reported_epoch":580,"state":"active+clean","last_fresh":"2023-09-23T20:48:25.927844+0000","last_change":"2023-09-23T20:48:23.915616+0000","last_active":"2023-09-23T20:48:25.927844+0000","last_peered":"2023-09-23T20:48:25.927844+0000","last_clean":"2023-09-23T20:48:25.927844+0000","last_became_active":"2023-09-23T20:48:23.914883+0000","last_became_peered":"2023-09-23T20:48:23.914883+0000","last_unstale":"2023-09-23T20:48:25.927844+0000","last_undegraded":"2023-09-23T20:48:25.927844+0000","last_fullsized":"2023-09-23T20:48:25.927844+0000","mapping_epoch":577,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":578,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:54:12.660201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":38,"reported_epoch":580,"state":"active+clean","last_fresh":"2023-09-23T20:48:25.927558+0000","last_change":"2023-09-23T20:47:09.186271+0000","last_active":"2023-09-23T20:48:25.927558+0000","last_peered":"2023-09-23T20:48:25.927558+0000","last_clean":"2023-09-23T20:48:25.927558+0000","last_became_active":"2023-09-23T20:47:09.184357+0000","last_became_peered":"2023-09-23T20:47:09.184357+0000","last_unstale":"2023-09-23T20:48:25.927558+0000","last_undegraded":"2023-09-23T20:48:25.927558+0000","last_fullsized":"2023-09-23T20:48:25.927558+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:48:26.339811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"581'10267","reported_seq":10367,"reported_epoch":581,"state":"active+clean","last_fresh":"2023-09-23T20:48:30.772063+0000","last_change":"2023-09-23T20:47:10.208583+0000","last_active":"2023-09-23T20:48:30.772063+0000","last_peered":"2023-09-23T20:48:30.772063+0000","last_clean":"2023-09-23T20:48:30.772063+0000","last_became_active":"2023-09-23T20:47:10.171839+0000","last_became_peered":"2023-09-23T20:47:10.171839+0000","last_unstale":"2023-09-23T20:48:30.772063+0000","last_undegraded":"2023-09-23T20:48:30.772063+0000","last_fullsized":"2023-09-23T20:48:30.772063+0000","mapping_epoch":553,"log_start":"542'4936","ondisk_log_start":"542'4936","created":531,"last_epoch_clean":554,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":5331,"log_dups_size":0,"ondisk_log_size":5331,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:48:36.327572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":185925632,"num_objects":2837,"num_object_clones":0,"num_object_copies":8511,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2837,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10267,"num_write_kb":419328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":786432,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":476,"reported_epoch":581,"state":"active+clean","last_fresh":"2023-09-23T20:48:26.963935+0000","last_change":"2023-09-23T20:48:26.963935+0000","last_active":"2023-09-23T20:48:26.963935+0000","last_peered":"2023-09-23T20:48:26.963935+0000","last_clean":"2023-09-23T20:48:26.963935+0000","last_became_active":"2023-09-23T20:48:26.963607+0000","last_became_peered":"2023-09-23T20:48:26.963607+0000","last_unstale":"2023-09-23T20:48:26.963935+0000","last_undegraded":"2023-09-23T20:48:26.963935+0000","last_fullsized":"2023-09-23T20:48:26.963935+0000","mapping_epoch":580,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":581,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:43:51.693459+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:43:51.693459+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26834,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":377,"reported_epoch":580,"state":"active+clean","last_fresh":"2023-09-23T20:48:25.919746+0000","last_change":"2023-09-23T20:37:43.570610+0000","last_active":"2023-09-23T20:48:25.919746+0000","last_peered":"2023-09-23T20:48:25.919746+0000","last_clean":"2023-09-23T20:48:25.919746+0000","last_became_active":"2023-09-23T20:37:43.570216+0000","last_became_peered":"2023-09-23T20:37:43.570216+0000","last_unstale":"2023-09-23T20:48:25.919746+0000","last_undegraded":"2023-09-23T20:48:25.919746+0000","last_fullsized":"2023-09-23T20:48:25.919746+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:39:26.555329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":426,"reported_epoch":580,"state":"active+clean","last_fresh":"2023-09-23T20:48:25.929734+0000","last_change":"2023-09-23T20:47:09.172573+0000","last_active":"2023-09-23T20:48:25.929734+0000","last_peered":"2023-09-23T20:48:25.929734+0000","last_clean":"2023-09-23T20:48:25.929734+0000","last_became_active":"2023-09-23T20:47:09.172298+0000","last_became_peered":"2023-09-23T20:47:09.172298+0000","last_unstale":"2023-09-23T20:48:25.929734+0000","last_undegraded":"2023-09-23T20:48:25.929734+0000","last_fullsized":"2023-09-23T20:48:25.929734+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:22:09.361284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"581'10308","reported_seq":10948,"reported_epoch":581,"state":"active+clean","last_fresh":"2023-09-23T20:48:30.291191+0000","last_change":"2023-09-23T20:47:11.626133+0000","last_active":"2023-09-23T20:48:30.291191+0000","last_peered":"2023-09-23T20:48:30.291191+0000","last_clean":"2023-09-23T20:48:30.291191+0000","last_became_active":"2023-09-23T20:47:11.619197+0000","last_became_peered":"2023-09-23T20:47:11.619197+0000","last_unstale":"2023-09-23T20:48:30.291191+0000","last_undegraded":"2023-09-23T20:48:30.291191+0000","last_fullsized":"2023-09-23T20:48:30.291191+0000","mapping_epoch":555,"log_start":"540'4163","ondisk_log_start":"540'4163","created":531,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":6145,"log_dups_size":0,"ondisk_log_size":6145,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:33:18.084175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":192675840,"num_objects":2940,"num_object_clones":0,"num_object_copies":8820,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2940,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10308,"num_write_kb":423936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":277,"num_bytes_recovered":18153472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":769,"reported_epoch":580,"state":"active+clean","last_fresh":"2023-09-23T20:48:25.941136+0000","last_change":"2023-09-23T20:41:07.424609+0000","last_active":"2023-09-23T20:48:25.941136+0000","last_peered":"2023-09-23T20:48:25.941136+0000","last_clean":"2023-09-23T20:48:25.941136+0000","last_became_active":"2023-09-23T20:38:05.622261+0000","last_became_peered":"2023-09-23T20:38:05.622261+0000","last_unstale":"2023-09-23T20:48:25.941136+0000","last_undegraded":"2023-09-23T20:48:25.941136+0000","last_fullsized":"2023-09-23T20:48:25.941136+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:41:07.424492+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:41:07.424492+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:45:21.689123+0000","scrub_duration":0.0018960959999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"86'119","reported_seq":892,"reported_epoch":580,"state":"active+clean","last_fresh":"2023-09-23T20:48:25.929509+0000","last_change":"2023-09-23T20:29:45.521257+0000","last_active":"2023-09-23T20:48:25.929509+0000","last_peered":"2023-09-23T20:48:25.929509+0000","last_clean":"2023-09-23T20:48:25.929509+0000","last_became_active":"2023-09-23T20:29:45.520546+0000","last_became_peered":"2023-09-23T20:29:45.520546+0000","last_unstale":"2023-09-23T20:48:25.929509+0000","last_undegraded":"2023-09-23T20:48:25.929509+0000","last_fullsized":"2023-09-23T20:48:25.929509+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":119,"log_dups_size":0,"ondisk_log_size":119,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:58:56.085777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032705499999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":523,"num_read_kb":3503,"num_write":136,"num_write_kb":3128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":384,"reported_epoch":580,"state":"active+clean","last_fresh":"2023-09-23T20:48:25.923152+0000","last_change":"2023-09-23T20:48:23.917851+0000","last_active":"2023-09-23T20:48:25.923152+0000","last_peered":"2023-09-23T20:48:25.923152+0000","last_clean":"2023-09-23T20:48:25.923152+0000","last_became_active":"2023-09-23T20:48:23.917532+0000","last_became_peered":"2023-09-23T20:48:23.917532+0000","last_unstale":"2023-09-23T20:48:25.923152+0000","last_undegraded":"2023-09-23T20:48:25.923152+0000","last_fullsized":"2023-09-23T20:48:25.923152+0000","mapping_epoch":577,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":578,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:11:49.352281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":557,"reported_epoch":580,"state":"active+clean","last_fresh":"2023-09-23T20:48:25.930057+0000","last_change":"2023-09-23T20:47:09.194947+0000","last_active":"2023-09-23T20:48:25.930057+0000","last_peered":"2023-09-23T20:48:25.930057+0000","last_clean":"2023-09-23T20:48:25.930057+0000","last_became_active":"2023-09-23T20:47:09.194592+0000","last_became_peered":"2023-09-23T20:47:09.194592+0000","last_unstale":"2023-09-23T20:48:25.930057+0000","last_undegraded":"2023-09-23T20:48:25.930057+0000","last_fullsized":"2023-09-23T20:48:25.930057+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:34:15.008492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"581'9910","reported_seq":9968,"reported_epoch":581,"state":"active+clean","last_fresh":"2023-09-23T20:48:30.438772+0000","last_change":"2023-09-23T20:45:39.489110+0000","last_active":"2023-09-23T20:48:30.438772+0000","last_peered":"2023-09-23T20:48:30.438772+0000","last_clean":"2023-09-23T20:48:30.438772+0000","last_became_active":"2023-09-23T20:45:39.488817+0000","last_became_peered":"2023-09-23T20:45:39.488817+0000","last_unstale":"2023-09-23T20:48:30.438772+0000","last_undegraded":"2023-09-23T20:48:30.438772+0000","last_fullsized":"2023-09-23T20:48:30.438772+0000","mapping_epoch":531,"log_start":"541'4217","ondisk_log_start":"541'4217","created":531,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":5693,"log_dups_size":0,"ondisk_log_size":5693,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:07:21.467971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":185335808,"num_objects":2828,"num_object_clones":0,"num_object_copies":8484,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2828,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9910,"num_write_kb":407616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":636,"reported_epoch":580,"state":"peering","last_fresh":"2023-09-23T20:48:25.936559+0000","last_change":"2023-09-23T20:48:25.936680+0000","last_active":"2023-09-23T20:48:25.936559+0000","last_peered":"2023-09-23T20:48:24.927583+0000","last_clean":"2023-09-23T20:48:18.899973+0000","last_became_active":"2023-09-23T20:48:24.927230+0000","last_became_peered":"2023-09-23T20:48:24.927230+0000","last_unstale":"2023-09-23T20:48:25.936559+0000","last_undegraded":"2023-09-23T20:48:25.936559+0000","last_fullsized":"2023-09-23T20:48:25.936559+0000","mapping_epoch":580,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:15.656880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:15.656880+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:51:30.413212+0000","scrub_duration":0.0016711180000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[2,4,5],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"86'106","reported_seq":833,"reported_epoch":580,"state":"active+clean","last_fresh":"2023-09-23T20:48:25.930339+0000","last_change":"2023-09-23T20:43:16.759138+0000","last_active":"2023-09-23T20:48:25.930339+0000","last_peered":"2023-09-23T20:48:25.930339+0000","last_clean":"2023-09-23T20:48:25.930339+0000","last_became_active":"2023-09-23T20:43:16.758015+0000","last_became_peered":"2023-09-23T20:43:16.758015+0000","last_unstale":"2023-09-23T20:48:25.930339+0000","last_undegraded":"2023-09-23T20:48:25.930339+0000","last_fullsized":"2023-09-23T20:48:25.930339+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":106,"log_dups_size":0,"ondisk_log_size":106,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:08:02.852472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00099302599999999993,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1094,"num_read_kb":1678,"num_write":100,"num_write_kb":404,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":38,"reported_epoch":580,"state":"active+clean","last_fresh":"2023-09-23T20:48:25.928953+0000","last_change":"2023-09-23T20:47:09.177872+0000","last_active":"2023-09-23T20:48:25.928953+0000","last_peered":"2023-09-23T20:48:25.928953+0000","last_clean":"2023-09-23T20:48:25.928953+0000","last_became_active":"2023-09-23T20:47:09.176689+0000","last_became_peered":"2023-09-23T20:47:09.176689+0000","last_unstale":"2023-09-23T20:48:25.928953+0000","last_undegraded":"2023-09-23T20:48:25.928953+0000","last_fullsized":"2023-09-23T20:48:25.928953+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:14:01.590318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":554,"reported_epoch":580,"state":"active+clean","last_fresh":"2023-09-23T20:48:25.929415+0000","last_change":"2023-09-23T20:48:23.917899+0000","last_active":"2023-09-23T20:48:25.929415+0000","last_peered":"2023-09-23T20:48:25.929415+0000","last_clean":"2023-09-23T20:48:25.929415+0000","last_became_active":"2023-09-23T20:48:23.917539+0000","last_became_peered":"2023-09-23T20:48:23.917539+0000","last_unstale":"2023-09-23T20:48:25.929415+0000","last_undegraded":"2023-09-23T20:48:25.929415+0000","last_fullsized":"2023-09-23T20:48:25.929415+0000","mapping_epoch":577,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":578,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:18:29.262728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.9","version":"581'10298","reported_seq":10374,"reported_epoch":581,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T20:48:30.354089+0000","last_change":"2023-09-23T20:48:25.271446+0000","last_active":"2023-09-23T20:48:30.354089+0000","last_peered":"2023-09-23T20:48:30.354089+0000","last_clean":"2023-09-23T20:48:22.889034+0000","last_became_active":"2023-09-23T20:48:25.145005+0000","last_became_peered":"2023-09-23T20:48:25.145005+0000","last_unstale":"2023-09-23T20:48:30.354089+0000","last_undegraded":"2023-09-23T20:48:30.354089+0000","last_fullsized":"2023-09-23T20:48:30.354089+0000","mapping_epoch":578,"log_start":"539'4123","ondisk_log_start":"539'4123","created":531,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":6175,"log_dups_size":0,"ondisk_log_size":6175,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:14:56.326180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":190578688,"num_objects":2908,"num_object_clones":0,"num_object_copies":8724,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2908,"num_objects_unfound":0,"num_objects_dirty":2908,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10298,"num_write_kb":422592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,2],"avail_no_missing":["7","2","5"],"object_location_counts":[{"shards":"2,5,7","objects":2908}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":584,"reported_epoch":580,"state":"active+clean","last_fresh":"2023-09-23T20:48:25.926855+0000","last_change":"2023-09-23T20:47:11.618652+0000","last_active":"2023-09-23T20:48:25.926855+0000","last_peered":"2023-09-23T20:48:25.926855+0000","last_clean":"2023-09-23T20:48:25.926855+0000","last_became_active":"2023-09-23T20:47:11.618266+0000","last_became_peered":"2023-09-23T20:47:11.618266+0000","last_unstale":"2023-09-23T20:48:25.926855+0000","last_undegraded":"2023-09-23T20:48:25.926855+0000","last_fullsized":"2023-09-23T20:48:25.926855+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:26.978179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:26.978179+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:29:29.000683+0000","scrub_duration":0.0013962829999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"86'130","reported_seq":437,"reported_epoch":581,"state":"active+clean","last_fresh":"2023-09-23T20:48:26.946384+0000","last_change":"2023-09-23T20:48:26.946384+0000","last_active":"2023-09-23T20:48:26.946384+0000","last_peered":"2023-09-23T20:48:26.946384+0000","last_clean":"2023-09-23T20:48:26.946384+0000","last_became_active":"2023-09-23T20:48:26.945625+0000","last_became_peered":"2023-09-23T20:48:26.945625+0000","last_unstale":"2023-09-23T20:48:26.946384+0000","last_undegraded":"2023-09-23T20:48:26.946384+0000","last_fullsized":"2023-09-23T20:48:26.946384+0000","mapping_epoch":580,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":581,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:59:49.215243+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011300049999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":512,"num_read_kb":1820,"num_write":133,"num_write_kb":1347,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":563,"reported_epoch":580,"state":"active+clean","last_fresh":"2023-09-23T20:48:25.932467+0000","last_change":"2023-09-23T20:48:23.926475+0000","last_active":"2023-09-23T20:48:25.932467+0000","last_peered":"2023-09-23T20:48:25.932467+0000","last_clean":"2023-09-23T20:48:25.932467+0000","last_became_active":"2023-09-23T20:48:23.926147+0000","last_became_peered":"2023-09-23T20:48:23.926147+0000","last_unstale":"2023-09-23T20:48:25.932467+0000","last_undegraded":"2023-09-23T20:48:25.932467+0000","last_fullsized":"2023-09-23T20:48:25.932467+0000","mapping_epoch":577,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":578,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:23.488851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":97,"reported_epoch":581,"state":"active+clean","last_fresh":"2023-09-23T20:48:26.945692+0000","last_change":"2023-09-23T20:44:10.555583+0000","last_active":"2023-09-23T20:48:26.945692+0000","last_peered":"2023-09-23T20:48:26.945692+0000","last_clean":"2023-09-23T20:48:26.945692+0000","last_became_active":"2023-09-23T20:44:10.555281+0000","last_became_peered":"2023-09-23T20:44:10.555281+0000","last_unstale":"2023-09-23T20:48:26.945692+0000","last_undegraded":"2023-09-23T20:48:26.945692+0000","last_fullsized":"2023-09-23T20:48:26.945692+0000","mapping_epoch":496,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":497,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:43:11.942147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"581'10263","reported_seq":10321,"reported_epoch":581,"state":"active+clean","last_fresh":"2023-09-23T20:48:30.409346+0000","last_change":"2023-09-23T20:45:39.331096+0000","last_active":"2023-09-23T20:48:30.409346+0000","last_peered":"2023-09-23T20:48:30.409346+0000","last_clean":"2023-09-23T20:48:30.409346+0000","last_became_active":"2023-09-23T20:45:39.330752+0000","last_became_peered":"2023-09-23T20:45:39.330752+0000","last_unstale":"2023-09-23T20:48:30.409346+0000","last_undegraded":"2023-09-23T20:48:30.409346+0000","last_fullsized":"2023-09-23T20:48:30.409346+0000","mapping_epoch":531,"log_start":"541'4560","ondisk_log_start":"541'4560","created":531,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":5703,"log_dups_size":0,"ondisk_log_size":5703,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:39:01.627762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":186449920,"num_objects":2845,"num_object_clones":0,"num_object_copies":8535,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2845,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10263,"num_write_kb":419456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"86'194","reported_seq":734,"reported_epoch":581,"state":"active+clean","last_fresh":"2023-09-23T20:48:26.952218+0000","last_change":"2023-09-23T20:48:26.952218+0000","last_active":"2023-09-23T20:48:26.952218+0000","last_peered":"2023-09-23T20:48:26.952218+0000","last_clean":"2023-09-23T20:48:26.952218+0000","last_became_active":"2023-09-23T20:48:26.951777+0000","last_became_peered":"2023-09-23T20:48:26.951777+0000","last_unstale":"2023-09-23T20:48:26.952218+0000","last_undegraded":"2023-09-23T20:48:26.952218+0000","last_fullsized":"2023-09-23T20:48:26.952218+0000","mapping_epoch":580,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":581,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":194,"log_dups_size":0,"ondisk_log_size":194,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:48:06.203582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00037995999999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1024,"num_read_kb":1200,"num_write":173,"num_write_kb":1276,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":1043,"reported_epoch":580,"state":"active+clean","last_fresh":"2023-09-23T20:48:25.929381+0000","last_change":"2023-09-23T20:38:05.602508+0000","last_active":"2023-09-23T20:48:25.929381+0000","last_peered":"2023-09-23T20:48:25.929381+0000","last_clean":"2023-09-23T20:48:25.929381+0000","last_became_active":"2023-09-23T20:38:05.602129+0000","last_became_peered":"2023-09-23T20:38:05.602129+0000","last_unstale":"2023-09-23T20:48:25.929381+0000","last_undegraded":"2023-09-23T20:48:25.929381+0000","last_fullsized":"2023-09-23T20:48:25.929381+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:45.343742+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:45.343742+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:48.910343+0000","scrub_duration":0.001475802,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":156,"reported_epoch":580,"state":"active+clean","last_fresh":"2023-09-23T20:48:25.935459+0000","last_change":"2023-09-23T20:43:16.758213+0000","last_active":"2023-09-23T20:48:25.935459+0000","last_peered":"2023-09-23T20:48:25.935459+0000","last_clean":"2023-09-23T20:48:25.935459+0000","last_became_active":"2023-09-23T20:43:16.757869+0000","last_became_peered":"2023-09-23T20:43:16.757869+0000","last_unstale":"2023-09-23T20:48:25.935459+0000","last_undegraded":"2023-09-23T20:48:25.935459+0000","last_fullsized":"2023-09-23T20:48:25.935459+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:43:10.811749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":206,"reported_epoch":580,"state":"active+clean","last_fresh":"2023-09-23T20:48:25.940411+0000","last_change":"2023-09-23T20:47:09.202302+0000","last_active":"2023-09-23T20:48:25.940411+0000","last_peered":"2023-09-23T20:48:25.940411+0000","last_clean":"2023-09-23T20:48:25.940411+0000","last_became_active":"2023-09-23T20:47:09.201797+0000","last_became_peered":"2023-09-23T20:47:09.201797+0000","last_unstale":"2023-09-23T20:48:25.940411+0000","last_undegraded":"2023-09-23T20:48:25.940411+0000","last_fullsized":"2023-09-23T20:48:25.940411+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:34:41.055705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.8","version":"580'9928","reported_seq":10334,"reported_epoch":580,"state":"active+remapped+backfilling","last_fresh":"2023-09-23T20:48:26.587069+0000","last_change":"2023-09-23T20:48:25.273656+0000","last_active":"2023-09-23T20:48:26.587069+0000","last_peered":"2023-09-23T20:48:26.587069+0000","last_clean":"2023-09-23T20:48:22.729509+0000","last_became_active":"2023-09-23T20:48:25.097170+0000","last_became_peered":"2023-09-23T20:48:25.097170+0000","last_unstale":"2023-09-23T20:48:26.587069+0000","last_undegraded":"2023-09-23T20:48:26.587069+0000","last_fullsized":"2023-09-23T20:48:26.587069+0000","mapping_epoch":578,"log_start":"539'3971","ondisk_log_start":"539'3971","created":531,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":5957,"log_dups_size":0,"ondisk_log_size":5957,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:47:25.989041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":191758336,"num_objects":2926,"num_object_clones":0,"num_object_copies":8778,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2760,"num_objects_unfound":0,"num_objects_dirty":2926,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9928,"num_write_kb":411328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":168,"num_bytes_recovered":11010048,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,2],"avail_no_missing":["0","2","7"],"object_location_counts":[{"shards":"0,2,7","objects":2926}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":948,"reported_epoch":580,"state":"active+clean","last_fresh":"2023-09-23T20:48:25.932952+0000","last_change":"2023-09-23T20:47:11.622262+0000","last_active":"2023-09-23T20:48:25.932952+0000","last_peered":"2023-09-23T20:48:25.932952+0000","last_clean":"2023-09-23T20:48:25.932952+0000","last_became_active":"2023-09-23T20:47:11.621920+0000","last_became_peered":"2023-09-23T20:47:11.621920+0000","last_unstale":"2023-09-23T20:48:25.932952+0000","last_undegraded":"2023-09-23T20:48:25.932952+0000","last_fullsized":"2023-09-23T20:48:25.932952+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:43.582146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:43.582146+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:01:15.153543+0000","scrub_duration":0.0016219730000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"86'80","reported_seq":833,"reported_epoch":580,"state":"active+clean","last_fresh":"2023-09-23T20:48:25.929290+0000","last_change":"2023-09-23T20:29:45.536796+0000","last_active":"2023-09-23T20:48:25.929290+0000","last_peered":"2023-09-23T20:48:25.929290+0000","last_clean":"2023-09-23T20:48:25.929290+0000","last_became_active":"2023-09-23T20:29:45.536439+0000","last_became_peered":"2023-09-23T20:29:45.536439+0000","last_unstale":"2023-09-23T20:48:25.929290+0000","last_undegraded":"2023-09-23T20:48:25.929290+0000","last_fullsized":"2023-09-23T20:48:25.929290+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":80,"log_dups_size":0,"ondisk_log_size":80,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:30:57.793132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035115,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":395,"num_read_kb":1370,"num_write":66,"num_write_kb":1061,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":592,"reported_epoch":579,"state":"active+clean","last_fresh":"2023-09-23T20:48:24.910562+0000","last_change":"2023-09-23T20:47:09.198326+0000","last_active":"2023-09-23T20:48:24.910562+0000","last_peered":"2023-09-23T20:48:24.910562+0000","last_clean":"2023-09-23T20:48:24.910562+0000","last_became_active":"2023-09-23T20:47:09.198059+0000","last_became_peered":"2023-09-23T20:47:09.198059+0000","last_unstale":"2023-09-23T20:48:24.910562+0000","last_undegraded":"2023-09-23T20:48:24.910562+0000","last_fullsized":"2023-09-23T20:48:24.910562+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:38:53.903706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":578,"reported_epoch":579,"state":"active+clean","last_fresh":"2023-09-23T20:48:24.910071+0000","last_change":"2023-09-23T20:48:23.930540+0000","last_active":"2023-09-23T20:48:24.910071+0000","last_peered":"2023-09-23T20:48:24.910071+0000","last_clean":"2023-09-23T20:48:24.910071+0000","last_became_active":"2023-09-23T20:48:23.930260+0000","last_became_peered":"2023-09-23T20:48:23.930260+0000","last_unstale":"2023-09-23T20:48:24.910071+0000","last_undegraded":"2023-09-23T20:48:24.910071+0000","last_fullsized":"2023-09-23T20:48:24.910071+0000","mapping_epoch":577,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":578,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:18.158001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"581'10013","reported_seq":10697,"reported_epoch":581,"state":"active+clean","last_fresh":"2023-09-23T20:48:30.416205+0000","last_change":"2023-09-23T20:47:13.668055+0000","last_active":"2023-09-23T20:48:30.416205+0000","last_peered":"2023-09-23T20:48:30.416205+0000","last_clean":"2023-09-23T20:48:30.416205+0000","last_became_active":"2023-09-23T20:47:13.626170+0000","last_became_peered":"2023-09-23T20:47:13.626170+0000","last_unstale":"2023-09-23T20:48:30.416205+0000","last_undegraded":"2023-09-23T20:48:30.416205+0000","last_fullsized":"2023-09-23T20:48:30.416205+0000","mapping_epoch":557,"log_start":"541'4343","ondisk_log_start":"541'4343","created":531,"last_epoch_clean":558,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":5670,"log_dups_size":0,"ondisk_log_size":5670,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:32.805547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":188874776,"num_objects":2883,"num_object_clones":0,"num_object_copies":8649,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2883,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10013,"num_write_kb":412609,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":298,"num_bytes_recovered":19529728,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":692,"reported_epoch":580,"state":"active+clean","last_fresh":"2023-09-23T20:48:25.940584+0000","last_change":"2023-09-23T20:41:08.380084+0000","last_active":"2023-09-23T20:48:25.940584+0000","last_peered":"2023-09-23T20:48:25.940584+0000","last_clean":"2023-09-23T20:48:25.940584+0000","last_became_active":"2023-09-23T20:38:05.610881+0000","last_became_peered":"2023-09-23T20:38:05.610881+0000","last_unstale":"2023-09-23T20:48:25.940584+0000","last_undegraded":"2023-09-23T20:48:25.940584+0000","last_fullsized":"2023-09-23T20:48:25.940584+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:41:08.379959+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:41:08.379959+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:19:39.547923+0000","scrub_duration":0.0019158420000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"86'102","reported_seq":1251,"reported_epoch":580,"state":"active+clean","last_fresh":"2023-09-23T20:48:25.927896+0000","last_change":"2023-09-23T20:47:14.628795+0000","last_active":"2023-09-23T20:48:25.927896+0000","last_peered":"2023-09-23T20:48:25.927896+0000","last_clean":"2023-09-23T20:48:25.927896+0000","last_became_active":"2023-09-23T20:47:14.628341+0000","last_became_peered":"2023-09-23T20:47:14.628341+0000","last_unstale":"2023-09-23T20:48:25.927896+0000","last_undegraded":"2023-09-23T20:48:25.927896+0000","last_fullsized":"2023-09-23T20:48:25.927896+0000","mapping_epoch":558,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":559,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":102,"log_dups_size":0,"ondisk_log_size":102,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:42:16.291849+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030656500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":963,"num_read_kb":1234,"num_write":60,"num_write_kb":225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":538,"reported_epoch":580,"state":"active+clean","last_fresh":"2023-09-23T20:48:25.929606+0000","last_change":"2023-09-23T20:48:23.929385+0000","last_active":"2023-09-23T20:48:25.929606+0000","last_peered":"2023-09-23T20:48:25.929606+0000","last_clean":"2023-09-23T20:48:25.929606+0000","last_became_active":"2023-09-23T20:48:23.929089+0000","last_became_peered":"2023-09-23T20:48:23.929089+0000","last_unstale":"2023-09-23T20:48:25.929606+0000","last_undegraded":"2023-09-23T20:48:25.929606+0000","last_fullsized":"2023-09-23T20:48:25.929606+0000","mapping_epoch":577,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":578,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:29:31.270557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":524,"reported_epoch":579,"state":"active+clean","last_fresh":"2023-09-23T20:48:24.911247+0000","last_change":"2023-09-23T20:43:46.541173+0000","last_active":"2023-09-23T20:48:24.911247+0000","last_peered":"2023-09-23T20:48:24.911247+0000","last_clean":"2023-09-23T20:48:24.911247+0000","last_became_active":"2023-09-23T20:43:46.540726+0000","last_became_peered":"2023-09-23T20:43:46.540726+0000","last_unstale":"2023-09-23T20:48:24.911247+0000","last_undegraded":"2023-09-23T20:48:24.911247+0000","last_fullsized":"2023-09-23T20:48:24.911247+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:49:04.137885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.e","version":"581'10182","reported_seq":10378,"reported_epoch":581,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T20:48:30.792952+0000","last_change":"2023-09-23T20:48:25.269832+0000","last_active":"2023-09-23T20:48:30.792952+0000","last_peered":"2023-09-23T20:48:30.792952+0000","last_clean":"2023-09-23T20:48:22.773845+0000","last_became_active":"2023-09-23T20:48:25.089809+0000","last_became_peered":"2023-09-23T20:48:25.089809+0000","last_unstale":"2023-09-23T20:48:30.792952+0000","last_undegraded":"2023-09-23T20:48:30.792952+0000","last_fullsized":"2023-09-23T20:48:30.792952+0000","mapping_epoch":578,"log_start":"542'4840","ondisk_log_start":"542'4840","created":531,"last_epoch_clean":554,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":5342,"log_dups_size":0,"ondisk_log_size":5342,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:23:16.255594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":184549376,"num_objects":2816,"num_object_clones":0,"num_object_copies":8448,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2816,"num_objects_unfound":0,"num_objects_dirty":2816,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10182,"num_write_kb":415936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":50,"num_bytes_recovered":3276800,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,2],"avail_no_missing":["4","2","6"],"object_location_counts":[{"shards":"2,4,6","objects":2816}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":598,"reported_epoch":581,"state":"active+clean","last_fresh":"2023-09-23T20:48:26.966827+0000","last_change":"2023-09-23T20:48:26.966827+0000","last_active":"2023-09-23T20:48:26.966827+0000","last_peered":"2023-09-23T20:48:26.966827+0000","last_clean":"2023-09-23T20:48:26.966827+0000","last_became_active":"2023-09-23T20:48:26.966496+0000","last_became_peered":"2023-09-23T20:48:26.966496+0000","last_unstale":"2023-09-23T20:48:26.966827+0000","last_undegraded":"2023-09-23T20:48:26.966827+0000","last_fullsized":"2023-09-23T20:48:26.966827+0000","mapping_epoch":580,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":581,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:47:34.561747+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:47:34.561747+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:54:39.492596+0000","scrub_duration":0.0087949929999999992,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":318,"reported_epoch":580,"state":"active+clean","last_fresh":"2023-09-23T20:48:25.930848+0000","last_change":"2023-09-23T20:32:50.549695+0000","last_active":"2023-09-23T20:48:25.930848+0000","last_peered":"2023-09-23T20:48:25.930848+0000","last_clean":"2023-09-23T20:48:25.930848+0000","last_became_active":"2023-09-23T20:32:50.549431+0000","last_became_peered":"2023-09-23T20:32:50.549431+0000","last_unstale":"2023-09-23T20:48:25.930848+0000","last_undegraded":"2023-09-23T20:48:25.930848+0000","last_fullsized":"2023-09-23T20:48:25.930848+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:45:53.666569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"86'128","reported_seq":1061,"reported_epoch":580,"state":"active+clean","last_fresh":"2023-09-23T20:48:25.930567+0000","last_change":"2023-09-23T20:47:11.621846+0000","last_active":"2023-09-23T20:48:25.930567+0000","last_peered":"2023-09-23T20:48:25.930567+0000","last_clean":"2023-09-23T20:48:25.930567+0000","last_became_active":"2023-09-23T20:47:11.621284+0000","last_became_peered":"2023-09-23T20:47:11.621284+0000","last_unstale":"2023-09-23T20:48:25.930567+0000","last_undegraded":"2023-09-23T20:48:25.930567+0000","last_fullsized":"2023-09-23T20:48:25.930567+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:43:50.658512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000293202,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":579,"num_read_kb":1473,"num_write":96,"num_write_kb":1062,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":363,"reported_epoch":580,"state":"active+clean","last_fresh":"2023-09-23T20:48:25.919651+0000","last_change":"2023-09-23T20:37:43.582105+0000","last_active":"2023-09-23T20:48:25.919651+0000","last_peered":"2023-09-23T20:48:25.919651+0000","last_clean":"2023-09-23T20:48:25.919651+0000","last_became_active":"2023-09-23T20:37:43.581860+0000","last_became_peered":"2023-09-23T20:37:43.581860+0000","last_unstale":"2023-09-23T20:48:25.919651+0000","last_undegraded":"2023-09-23T20:48:25.919651+0000","last_fullsized":"2023-09-23T20:48:25.919651+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:18:38.765486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"581'10122","reported_seq":11212,"reported_epoch":581,"state":"active+remapped+backfilling","last_fresh":"2023-09-23T20:48:30.449409+0000","last_change":"2023-09-23T20:48:26.961132+0000","last_active":"2023-09-23T20:48:30.449409+0000","last_peered":"2023-09-23T20:48:30.449409+0000","last_clean":"2023-09-23T20:48:22.771349+0000","last_became_active":"2023-09-23T20:48:25.116542+0000","last_became_peered":"2023-09-23T20:48:25.116542+0000","last_unstale":"2023-09-23T20:48:30.449409+0000","last_undegraded":"2023-09-23T20:48:30.449409+0000","last_fullsized":"2023-09-23T20:48:30.449409+0000","mapping_epoch":578,"log_start":"541'4399","ondisk_log_start":"541'4399","created":531,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":5723,"log_dups_size":0,"ondisk_log_size":5723,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:24:56.198040+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":189005824,"num_objects":2884,"num_object_clones":0,"num_object_copies":8652,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2392,"num_objects_unfound":0,"num_objects_dirty":2884,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10122,"num_write_kb":416192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":508,"num_bytes_recovered":33292288,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,4],"acting":[6,3,2],"avail_no_missing":["6","2","3"],"object_location_counts":[{"shards":"2,3,6","objects":2884}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.4","version":"90'161","reported_seq":797,"reported_epoch":580,"state":"active+clean","last_fresh":"2023-09-23T20:48:25.924109+0000","last_change":"2023-09-23T20:40:22.427435+0000","last_active":"2023-09-23T20:48:25.924109+0000","last_peered":"2023-09-23T20:48:25.924109+0000","last_clean":"2023-09-23T20:48:25.924109+0000","last_became_active":"2023-09-23T20:40:22.427051+0000","last_became_peered":"2023-09-23T20:40:22.427051+0000","last_unstale":"2023-09-23T20:48:25.924109+0000","last_undegraded":"2023-09-23T20:48:25.924109+0000","last_fullsized":"2023-09-23T20:48:25.924109+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:34:54.504554+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:34:54.504554+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:42.001652+0000","scrub_duration":0.011176864,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"86'117","reported_seq":927,"reported_epoch":581,"state":"active+clean","last_fresh":"2023-09-23T20:48:26.945684+0000","last_change":"2023-09-23T20:48:26.945684+0000","last_active":"2023-09-23T20:48:26.945684+0000","last_peered":"2023-09-23T20:48:26.945684+0000","last_clean":"2023-09-23T20:48:26.945684+0000","last_became_active":"2023-09-23T20:48:26.945253+0000","last_became_peered":"2023-09-23T20:48:26.945253+0000","last_unstale":"2023-09-23T20:48:26.945684+0000","last_undegraded":"2023-09-23T20:48:26.945684+0000","last_fullsized":"2023-09-23T20:48:26.945684+0000","mapping_epoch":580,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":581,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:59:37.621193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046891400000000002,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":893,"num_read_kb":922,"num_write":82,"num_write_kb":4193,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":20971520,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":595,"reported_epoch":579,"state":"active+clean","last_fresh":"2023-09-23T20:48:24.910142+0000","last_change":"2023-09-23T20:47:09.317116+0000","last_active":"2023-09-23T20:48:24.910142+0000","last_peered":"2023-09-23T20:48:24.910142+0000","last_clean":"2023-09-23T20:48:24.910142+0000","last_became_active":"2023-09-23T20:47:09.205930+0000","last_became_peered":"2023-09-23T20:47:09.205930+0000","last_unstale":"2023-09-23T20:48:24.910142+0000","last_undegraded":"2023-09-23T20:48:24.910142+0000","last_fullsized":"2023-09-23T20:48:24.910142+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:04:55.528625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":38,"reported_epoch":580,"state":"active+clean","last_fresh":"2023-09-23T20:48:25.927923+0000","last_change":"2023-09-23T20:47:09.192117+0000","last_active":"2023-09-23T20:48:25.927923+0000","last_peered":"2023-09-23T20:48:25.927923+0000","last_clean":"2023-09-23T20:48:25.927923+0000","last_became_active":"2023-09-23T20:47:09.191820+0000","last_became_peered":"2023-09-23T20:47:09.191820+0000","last_unstale":"2023-09-23T20:48:25.927923+0000","last_undegraded":"2023-09-23T20:48:25.927923+0000","last_fullsized":"2023-09-23T20:48:25.927923+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:08.785173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"90'161","reported_seq":837,"reported_epoch":579,"state":"active+clean","last_fresh":"2023-09-23T20:48:24.910024+0000","last_change":"2023-09-23T20:43:46.544750+0000","last_active":"2023-09-23T20:48:24.910024+0000","last_peered":"2023-09-23T20:48:24.910024+0000","last_clean":"2023-09-23T20:48:24.910024+0000","last_became_active":"2023-09-23T20:43:46.544395+0000","last_became_peered":"2023-09-23T20:43:46.544395+0000","last_unstale":"2023-09-23T20:48:24.910024+0000","last_undegraded":"2023-09-23T20:48:24.910024+0000","last_fullsized":"2023-09-23T20:48:24.910024+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:06.487351+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:06.487351+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:56.866168+0000","scrub_duration":0.001964379,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":553,"reported_epoch":579,"state":"active+clean","last_fresh":"2023-09-23T20:48:24.910227+0000","last_change":"2023-09-23T20:47:09.197601+0000","last_active":"2023-09-23T20:48:24.910227+0000","last_peered":"2023-09-23T20:48:24.910227+0000","last_clean":"2023-09-23T20:48:24.910227+0000","last_became_active":"2023-09-23T20:47:09.196967+0000","last_became_peered":"2023-09-23T20:47:09.196967+0000","last_unstale":"2023-09-23T20:48:24.910227+0000","last_undegraded":"2023-09-23T20:48:24.910227+0000","last_fullsized":"2023-09-23T20:48:24.910227+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:06:56.826785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":187,"reported_epoch":580,"state":"active+clean","last_fresh":"2023-09-23T20:48:25.929614+0000","last_change":"2023-09-23T20:48:23.929485+0000","last_active":"2023-09-23T20:48:25.929614+0000","last_peered":"2023-09-23T20:48:25.929614+0000","last_clean":"2023-09-23T20:48:25.929614+0000","last_became_active":"2023-09-23T20:48:23.929161+0000","last_became_peered":"2023-09-23T20:48:23.929161+0000","last_unstale":"2023-09-23T20:48:25.929614+0000","last_undegraded":"2023-09-23T20:48:25.929614+0000","last_fullsized":"2023-09-23T20:48:25.929614+0000","mapping_epoch":577,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":578,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:21.572566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1e","version":"90'161","reported_seq":656,"reported_epoch":580,"state":"active+clean","last_fresh":"2023-09-23T20:48:25.922195+0000","last_change":"2023-09-23T20:41:06.499511+0000","last_active":"2023-09-23T20:48:25.922195+0000","last_peered":"2023-09-23T20:48:25.922195+0000","last_clean":"2023-09-23T20:48:25.922195+0000","last_became_active":"2023-09-23T20:38:05.601874+0000","last_became_peered":"2023-09-23T20:38:05.601874+0000","last_unstale":"2023-09-23T20:48:25.922195+0000","last_undegraded":"2023-09-23T20:48:25.922195+0000","last_fullsized":"2023-09-23T20:48:25.922195+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:41:06.499396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:41:06.499396+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:31:34.059077+0000","scrub_duration":0.0019670310000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":42,"reported_epoch":580,"state":"active+clean","last_fresh":"2023-09-23T20:48:25.929874+0000","last_change":"2023-09-23T20:47:09.332980+0000","last_active":"2023-09-23T20:48:25.929874+0000","last_peered":"2023-09-23T20:48:25.929874+0000","last_clean":"2023-09-23T20:48:25.929874+0000","last_became_active":"2023-09-23T20:47:09.230554+0000","last_became_peered":"2023-09-23T20:47:09.230554+0000","last_unstale":"2023-09-23T20:48:25.929874+0000","last_undegraded":"2023-09-23T20:48:25.929874+0000","last_fullsized":"2023-09-23T20:48:25.929874+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:06:21.882823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":38,"reported_epoch":580,"state":"active+clean","last_fresh":"2023-09-23T20:48:25.928873+0000","last_change":"2023-09-23T20:47:09.186579+0000","last_active":"2023-09-23T20:48:25.928873+0000","last_peered":"2023-09-23T20:48:25.928873+0000","last_clean":"2023-09-23T20:48:25.928873+0000","last_became_active":"2023-09-23T20:47:09.186227+0000","last_became_peered":"2023-09-23T20:47:09.186227+0000","last_unstale":"2023-09-23T20:48:25.928873+0000","last_undegraded":"2023-09-23T20:48:25.928873+0000","last_fullsized":"2023-09-23T20:48:25.928873+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:45:39.485308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":42,"reported_epoch":580,"state":"active+clean","last_fresh":"2023-09-23T20:48:25.927378+0000","last_change":"2023-09-23T20:47:09.316824+0000","last_active":"2023-09-23T20:48:25.927378+0000","last_peered":"2023-09-23T20:48:25.927378+0000","last_clean":"2023-09-23T20:48:25.927378+0000","last_became_active":"2023-09-23T20:47:09.195100+0000","last_became_peered":"2023-09-23T20:47:09.195100+0000","last_unstale":"2023-09-23T20:48:25.927378+0000","last_undegraded":"2023-09-23T20:48:25.927378+0000","last_fullsized":"2023-09-23T20:48:25.927378+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:29:53.273554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":38,"reported_epoch":580,"state":"active+clean","last_fresh":"2023-09-23T20:48:25.932880+0000","last_change":"2023-09-23T20:47:09.181240+0000","last_active":"2023-09-23T20:48:25.932880+0000","last_peered":"2023-09-23T20:48:25.932880+0000","last_clean":"2023-09-23T20:48:25.932880+0000","last_became_active":"2023-09-23T20:47:09.181022+0000","last_became_peered":"2023-09-23T20:47:09.181022+0000","last_unstale":"2023-09-23T20:48:25.932880+0000","last_undegraded":"2023-09-23T20:48:25.932880+0000","last_fullsized":"2023-09-23T20:48:25.932880+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:38:59.191781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"90'161","reported_seq":836,"reported_epoch":580,"state":"active+clean","last_fresh":"2023-09-23T20:48:25.930417+0000","last_change":"2023-09-23T20:38:05.591992+0000","last_active":"2023-09-23T20:48:25.930417+0000","last_peered":"2023-09-23T20:48:25.930417+0000","last_clean":"2023-09-23T20:48:25.930417+0000","last_became_active":"2023-09-23T20:38:05.591701+0000","last_became_peered":"2023-09-23T20:38:05.591701+0000","last_unstale":"2023-09-23T20:48:25.930417+0000","last_undegraded":"2023-09-23T20:48:25.930417+0000","last_fullsized":"2023-09-23T20:48:25.930417+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:02.384133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:02.384133+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:41.722542+0000","scrub_duration":0.001769098,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":414,"reported_epoch":580,"state":"active+clean","last_fresh":"2023-09-23T20:48:25.922440+0000","last_change":"2023-09-23T20:48:23.914612+0000","last_active":"2023-09-23T20:48:25.922440+0000","last_peered":"2023-09-23T20:48:25.922440+0000","last_clean":"2023-09-23T20:48:25.922440+0000","last_became_active":"2023-09-23T20:48:23.914239+0000","last_became_peered":"2023-09-23T20:48:23.914239+0000","last_unstale":"2023-09-23T20:48:25.922440+0000","last_undegraded":"2023-09-23T20:48:25.922440+0000","last_fullsized":"2023-09-23T20:48:25.922440+0000","mapping_epoch":577,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":578,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:59:12.251426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":380,"reported_epoch":580,"state":"active+clean","last_fresh":"2023-09-23T20:48:25.921110+0000","last_change":"2023-09-23T20:48:23.918278+0000","last_active":"2023-09-23T20:48:25.921110+0000","last_peered":"2023-09-23T20:48:25.921110+0000","last_clean":"2023-09-23T20:48:25.921110+0000","last_became_active":"2023-09-23T20:48:23.917415+0000","last_became_peered":"2023-09-23T20:48:23.917415+0000","last_unstale":"2023-09-23T20:48:25.921110+0000","last_undegraded":"2023-09-23T20:48:25.921110+0000","last_fullsized":"2023-09-23T20:48:25.921110+0000","mapping_epoch":577,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":578,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:56:47.683887+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"90'161","reported_seq":710,"reported_epoch":580,"state":"active+clean","last_fresh":"2023-09-23T20:48:25.929503+0000","last_change":"2023-09-23T20:47:11.621758+0000","last_active":"2023-09-23T20:48:25.929503+0000","last_peered":"2023-09-23T20:48:25.929503+0000","last_clean":"2023-09-23T20:48:25.929503+0000","last_became_active":"2023-09-23T20:47:11.621345+0000","last_became_peered":"2023-09-23T20:47:11.621345+0000","last_unstale":"2023-09-23T20:48:25.929503+0000","last_undegraded":"2023-09-23T20:48:25.929503+0000","last_fullsized":"2023-09-23T20:48:25.929503+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:01.504973+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:01.504973+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:31:17.112393+0000","scrub_duration":0.00202344,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":629,"reported_epoch":579,"state":"active+clean","last_fresh":"2023-09-23T20:48:24.910750+0000","last_change":"2023-09-23T20:48:23.926082+0000","last_active":"2023-09-23T20:48:24.910750+0000","last_peered":"2023-09-23T20:48:24.910750+0000","last_clean":"2023-09-23T20:48:24.910750+0000","last_became_active":"2023-09-23T20:48:23.920608+0000","last_became_peered":"2023-09-23T20:48:23.920608+0000","last_unstale":"2023-09-23T20:48:24.910750+0000","last_undegraded":"2023-09-23T20:48:24.910750+0000","last_fullsized":"2023-09-23T20:48:24.910750+0000","mapping_epoch":577,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":578,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:55:37.943208+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":3878,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":48,"reported_epoch":580,"state":"active+clean","last_fresh":"2023-09-23T20:48:25.926791+0000","last_change":"2023-09-23T20:48:23.917302+0000","last_active":"2023-09-23T20:48:25.926791+0000","last_peered":"2023-09-23T20:48:25.926791+0000","last_clean":"2023-09-23T20:48:25.926791+0000","last_became_active":"2023-09-23T20:48:23.916956+0000","last_became_peered":"2023-09-23T20:48:23.916956+0000","last_unstale":"2023-09-23T20:48:25.926791+0000","last_undegraded":"2023-09-23T20:48:25.926791+0000","last_fullsized":"2023-09-23T20:48:25.926791+0000","mapping_epoch":577,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":578,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:42:27.989294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":3068461080,"num_objects":46822,"num_object_clones":0,"num_object_copies":140466,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":16882,"num_objects_unfound":0,"num_objects_dirty":46822,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":162166,"num_write_kb":6687553,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3139,"num_bytes_recovered":205586432,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":9298849792,"data_stored":9298837576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":95889,"ondisk_log_size":95889,"up":48,"acting":48,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":105,"num_bytes_recovered":14632,"num_keys_recovered":70,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17100,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26691,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":661927,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":59,"num_read_kb":110,"num_write":28,"num_write_kb":291,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":477366,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2048000,"data_stored":2033792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5983,"num_read_kb":13200,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":127,"num_bytes_recovered":20971520,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16789504,"data_stored":16777273,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6082,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201725,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1317660,"kb_used_data":745556,"kb_used_omap":0,"kb_used_meta":572096,"kb_avail":92427492,"statfs":{"total":95995035648,"available":94645751808,"internally_reserved":0,"allocated":763449344,"data_stored":761862407,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":585826304},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,1,0,0,0,0,0,4],"upper_bound":8192},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:48:06 2023","interfaces":[{"interface":"back","average":{"1min":1.5569999999999999,"5min":1.972,"15min":2.1899999999999999},"min":{"1min":0.224,"5min":0.224,"15min":0.224},"max":{"1min":5.6970000000000001,"5min":15.1,"15min":18.315000000000001},"last":0.54800000000000004},{"interface":"front","average":{"1min":2.1539999999999999,"5min":2.1379999999999999,"15min":1.903},"min":{"1min":0.28000000000000003,"5min":0.25800000000000001,"15min":0.23899999999999999},"max":{"1min":6.5780000000000003,"5min":15.327999999999999,"15min":19.355},"last":0.26700000000000002}]},{"osd":1,"last update":"Sat Sep 23 20:48:13 2023","interfaces":[{"interface":"back","average":{"1min":1.3640000000000001,"5min":1.6779999999999999,"15min":1.972},"min":{"1min":0.28399999999999997,"5min":0.25900000000000001,"15min":0.24399999999999999},"max":{"1min":6.4610000000000003,"5min":15.131,"15min":49.965000000000003},"last":0.34999999999999998},{"interface":"front","average":{"1min":2.4929999999999999,"5min":2.524,"15min":2.5099999999999998},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.26700000000000002},"max":{"1min":10.861000000000001,"5min":15.18,"15min":53.451999999999998},"last":4.0739999999999998}]},{"osd":2,"last update":"Sat Sep 23 20:48:06 2023","interfaces":[{"interface":"back","average":{"1min":2.0779999999999998,"5min":2.153,"15min":2.1899999999999999},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.27800000000000002},"max":{"1min":5.4770000000000003,"5min":15.087,"15min":16.965},"last":0.31},{"interface":"front","average":{"1min":2.206,"5min":2.0249999999999999,"15min":2.339},"min":{"1min":0.45200000000000001,"5min":0.31,"15min":0.27400000000000002},"max":{"1min":6.6379999999999999,"5min":15.193,"15min":23.494},"last":2.7959999999999998}]},{"osd":3,"last update":"Sat Sep 23 20:47:35 2023","interfaces":[{"interface":"back","average":{"1min":2.153,"5min":1.841,"15min":2.2589999999999999},"min":{"1min":0.33800000000000002,"5min":0.32500000000000001,"15min":0.311},"max":{"1min":6.734,"5min":15.115,"15min":23.352},"last":1.238},{"interface":"front","average":{"1min":2.2040000000000002,"5min":2.2589999999999999,"15min":2.4550000000000001},"min":{"1min":0.443,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":6.5369999999999999,"5min":15.144,"15min":20.969999999999999},"last":4.1859999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:47:35 2023","interfaces":[{"interface":"back","average":{"1min":1.264,"5min":1.496,"15min":1.1739999999999999},"min":{"1min":0.27900000000000003,"5min":0.254,"15min":0.23200000000000001},"max":{"1min":6.548,"5min":12.260999999999999,"15min":18.364999999999998},"last":0.58299999999999996},{"interface":"front","average":{"1min":1.6160000000000001,"5min":1.8300000000000001,"15min":1.4350000000000001},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":6.0419999999999998,"5min":7.2469999999999999,"15min":23.617000000000001},"last":4.1529999999999996}]},{"osd":5,"last update":"Sat Sep 23 20:47:35 2023","interfaces":[{"interface":"back","average":{"1min":1.101,"5min":1.585,"15min":1.2190000000000001},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.26600000000000001},"max":{"1min":6.367,"5min":15.005000000000001,"15min":15.005000000000001},"last":0.59699999999999998},{"interface":"front","average":{"1min":1.504,"5min":1.9379999999999999,"15min":1.3480000000000001},"min":{"1min":0.36099999999999999,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":6.6280000000000001,"5min":12.731,"15min":18.390000000000001},"last":0.61199999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:47:35 2023","interfaces":[{"interface":"back","average":{"1min":1.4730000000000001,"5min":2.0950000000000002,"15min":1.47},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":6.4720000000000004,"5min":15.038,"15min":18.417000000000002},"last":4.6189999999999998},{"interface":"front","average":{"1min":1.861,"5min":1.9910000000000001,"15min":1.4139999999999999},"min":{"1min":0.35899999999999999,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":6.0910000000000002,"5min":15.369999999999999,"15min":15.369999999999999},"last":4.1239999999999997}]}]},{"osd":6,"up_from":47,"seq":201863463364,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1782748,"kb_used_data":1293076,"kb_used_omap":0,"kb_used_meta":489664,"kb_avail":91962404,"statfs":{"total":95995035648,"available":94169501696,"internally_reserved":0,"allocated":1324109824,"data_stored":1322522828,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":501415936},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:48:03 2023","interfaces":[{"interface":"back","average":{"1min":2.9780000000000002,"5min":2.0299999999999998,"15min":1.9419999999999999},"min":{"1min":0.25,"5min":0.219,"15min":0.219},"max":{"1min":15.574999999999999,"5min":21.018000000000001,"15min":34.143999999999998},"last":0.98799999999999999},{"interface":"front","average":{"1min":1.698,"5min":2.1909999999999998,"15min":2.2429999999999999},"min":{"1min":0.255,"5min":0.255,"15min":0.22700000000000001},"max":{"1min":4.8840000000000003,"5min":21.082000000000001,"15min":60.328000000000003},"last":0.97799999999999998}]},{"osd":1,"last update":"Sat Sep 23 20:48:20 2023","interfaces":[{"interface":"back","average":{"1min":3.0390000000000001,"5min":2.2309999999999999,"15min":2.0819999999999999},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.249},"max":{"1min":8.9350000000000005,"5min":9.2520000000000007,"15min":64.274000000000001},"last":2.4089999999999998},{"interface":"front","average":{"1min":2.746,"5min":1.9730000000000001,"15min":1.869},"min":{"1min":0.32300000000000001,"5min":0.29099999999999998,"15min":0.28000000000000003},"max":{"1min":8.0969999999999995,"5min":17.786000000000001,"15min":34.490000000000002},"last":3.0139999999999998}]},{"osd":2,"last update":"Sat Sep 23 20:48:03 2023","interfaces":[{"interface":"back","average":{"1min":2.4609999999999999,"5min":2.4380000000000002,"15min":2.242},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.27700000000000002},"max":{"1min":6.1529999999999996,"5min":22.591000000000001,"15min":34.317999999999998},"last":1.847},{"interface":"front","average":{"1min":2.4039999999999999,"5min":2.0350000000000001,"15min":2.0539999999999998},"min":{"1min":0.45400000000000001,"5min":0.33900000000000002,"15min":0.30399999999999999},"max":{"1min":7.8710000000000004,"5min":22.614999999999998,"15min":34.292999999999999},"last":0.499}]},{"osd":3,"last update":"Sat Sep 23 20:47:54 2023","interfaces":[{"interface":"back","average":{"1min":2.0659999999999998,"5min":1.946,"15min":1.9339999999999999},"min":{"1min":0.307,"5min":0.307,"15min":0.28399999999999997},"max":{"1min":9.6859999999999999,"5min":11.664,"15min":11.664},"last":2.9359999999999999},{"interface":"front","average":{"1min":2.2959999999999998,"5min":2.1859999999999999,"15min":2.1539999999999999},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.30499999999999999},"max":{"1min":9.7810000000000006,"5min":10.125999999999999,"15min":12.875999999999999},"last":0.95799999999999996}]},{"osd":4,"last update":"Sat Sep 23 20:48:20 2023","interfaces":[{"interface":"back","average":{"1min":1.923,"5min":1.129,"15min":0.94699999999999995},"min":{"1min":0.29599999999999999,"5min":0.28699999999999998,"15min":0.25600000000000001},"max":{"1min":8.1229999999999993,"5min":10.183999999999999,"15min":23.904},"last":0.46700000000000003},{"interface":"front","average":{"1min":2.1499999999999999,"5min":1.405,"15min":1.179},"min":{"1min":0.377,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":9.7189999999999994,"5min":9.7189999999999994,"15min":11.515000000000001},"last":1.008}]},{"osd":5,"last update":"Sat Sep 23 20:48:20 2023","interfaces":[{"interface":"back","average":{"1min":2.6419999999999999,"5min":1.4039999999999999,"15min":1.0029999999999999},"min":{"1min":0.371,"5min":0.253,"15min":0.253},"max":{"1min":8.0440000000000005,"5min":8.0440000000000005,"15min":8.0440000000000005},"last":1.633},{"interface":"front","average":{"1min":1.6699999999999999,"5min":1.548,"15min":1.2729999999999999},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":5.1900000000000004,"5min":9.3559999999999999,"15min":9.3559999999999999},"last":1.647}]},{"osd":7,"last update":"Sat Sep 23 20:47:54 2023","interfaces":[{"interface":"back","average":{"1min":1.421,"5min":1.331,"15min":0.98499999999999999},"min":{"1min":0.35199999999999998,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":7.1379999999999999,"5min":8.9309999999999992,"15min":8.9309999999999992},"last":0.59499999999999997},{"interface":"front","average":{"1min":2.0459999999999998,"5min":2.0150000000000001,"15min":1.3939999999999999},"min":{"1min":0.434,"5min":0.35999999999999999,"15min":0.34499999999999997},"max":{"1min":9.7530000000000001,"5min":9.8680000000000003,"15min":9.8680000000000003},"last":0.80600000000000005}]}]},{"osd":1,"up_from":266,"seq":1142461300962,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1332852,"kb_used_data":574124,"kb_used_omap":8,"kb_used_meta":758711,"kb_avail":92412300,"statfs":{"total":95995035648,"available":94630195200,"internally_reserved":0,"allocated":587902976,"data_stored":585017406,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8206,"internal_metadata":776921074},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:48:24 2023","interfaces":[{"interface":"back","average":{"1min":2.9710000000000001,"5min":2.4969999999999999,"15min":2.3559999999999999},"min":{"1min":0.27700000000000002,"5min":0.20799999999999999,"15min":0.20000000000000001},"max":{"1min":9.4730000000000008,"5min":15.532999999999999,"15min":16.292000000000002},"last":0.27700000000000002},{"interface":"front","average":{"1min":3.1469999999999998,"5min":2.9060000000000001,"15min":2.9430000000000001},"min":{"1min":0.35899999999999999,"5min":0.22600000000000001,"15min":0.19900000000000001},"max":{"1min":8.1839999999999993,"5min":20.346,"15min":26.588000000000001},"last":0.48699999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:48:24 2023","interfaces":[{"interface":"back","average":{"1min":3.1850000000000001,"5min":2.8340000000000001,"15min":3.1429999999999998},"min":{"1min":0.249,"5min":0.23699999999999999,"15min":0.216},"max":{"1min":9.5779999999999994,"5min":19.026,"15min":32.255000000000003},"last":0.41499999999999998},{"interface":"front","average":{"1min":3.5030000000000001,"5min":2.423,"15min":2.3769999999999998},"min":{"1min":0.315,"5min":0.26000000000000001,"15min":0.20899999999999999},"max":{"1min":10.567,"5min":10.567,"15min":13.404999999999999},"last":0.315}]},{"osd":3,"last update":"Sat Sep 23 20:48:24 2023","interfaces":[{"interface":"back","average":{"1min":3.294,"5min":2.2719999999999998,"15min":2.4780000000000002},"min":{"1min":0.373,"5min":0.28799999999999998,"15min":0.247},"max":{"1min":10.135999999999999,"5min":17.366,"15min":17.366},"last":0.373},{"interface":"front","average":{"1min":3.2170000000000001,"5min":2.8490000000000002,"15min":2.7949999999999999},"min":{"1min":0.26300000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":9.8339999999999996,"5min":11.534000000000001,"15min":19.315000000000001},"last":0.39400000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:48:24 2023","interfaces":[{"interface":"back","average":{"1min":2.7909999999999999,"5min":2.3769999999999998,"15min":2.4079999999999999},"min":{"1min":0.34799999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":9.5359999999999996,"5min":11.791,"15min":24.327999999999999},"last":0.45700000000000002},{"interface":"front","average":{"1min":2.5600000000000001,"5min":1.7949999999999999,"15min":1.7749999999999999},"min":{"1min":0.36299999999999999,"5min":0.34699999999999998,"15min":0.311},"max":{"1min":9.8249999999999993,"5min":10.468999999999999,"15min":20.835999999999999},"last":0.46700000000000003}]},{"osd":5,"last update":"Sat Sep 23 20:48:24 2023","interfaces":[{"interface":"back","average":{"1min":2.4470000000000001,"5min":2.3889999999999998,"15min":2.3660000000000001},"min":{"1min":0.41399999999999998,"5min":0.308,"15min":0.308},"max":{"1min":8.1210000000000004,"5min":14.366,"15min":33.401000000000003},"last":0.53300000000000003},{"interface":"front","average":{"1min":2.589,"5min":2.548,"15min":2.633},"min":{"1min":0.39700000000000002,"5min":0.33100000000000002,"15min":0.32500000000000001},"max":{"1min":9.6140000000000008,"5min":11.750999999999999,"15min":33.417000000000002},"last":0.501}]},{"osd":6,"last update":"Sat Sep 23 20:48:24 2023","interfaces":[{"interface":"back","average":{"1min":2.5049999999999999,"5min":2.3650000000000002,"15min":2.2570000000000001},"min":{"1min":0.48799999999999999,"5min":0.316,"15min":0.316},"max":{"1min":8.0960000000000001,"5min":14.414999999999999,"15min":14.414999999999999},"last":0.55500000000000005},{"interface":"front","average":{"1min":3.2650000000000001,"5min":3.004,"15min":2.6739999999999999},"min":{"1min":0.432,"5min":0.38500000000000001,"15min":0.317},"max":{"1min":9.6639999999999997,"5min":11.832000000000001,"15min":24.376000000000001},"last":0.54200000000000004}]},{"osd":7,"last update":"Sat Sep 23 20:48:24 2023","interfaces":[{"interface":"back","average":{"1min":2.8330000000000002,"5min":2.5219999999999998,"15min":2.4249999999999998},"min":{"1min":0.46899999999999997,"5min":0.377,"15min":0.37},"max":{"1min":8.1590000000000007,"5min":10.010999999999999,"15min":18.437000000000001},"last":0.57699999999999996},{"interface":"front","average":{"1min":3.5739999999999998,"5min":2.871,"15min":2.714},"min":{"1min":0.47299999999999998,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":9.8369999999999997,"5min":11.864000000000001,"15min":24.524000000000001},"last":0.52400000000000002}]}]},{"osd":0,"up_from":94,"seq":403726926164,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2874668,"kb_used_data":1955748,"kb_used_omap":7,"kb_used_meta":918904,"kb_avail":90870484,"statfs":{"total":95995035648,"available":93051375616,"internally_reserved":0,"allocated":2002685952,"data_stored":1999976086,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":940957713},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,0,0,0,8],"upper_bound":2048},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:47:30 2023","interfaces":[{"interface":"back","average":{"1min":8.9149999999999991,"5min":3.4870000000000001,"15min":2.9049999999999998},"min":{"1min":0.20100000000000001,"5min":0.184,"15min":0.17199999999999999},"max":{"1min":152.387,"5min":152.387,"15min":152.387},"last":0.34799999999999998},{"interface":"front","average":{"1min":8.7140000000000004,"5min":3.8149999999999999,"15min":3.0920000000000001},"min":{"1min":0.23899999999999999,"5min":0.23799999999999999,"15min":0.20399999999999999},"max":{"1min":152.84100000000001,"5min":152.84100000000001,"15min":152.84100000000001},"last":0.42999999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:48:01 2023","interfaces":[{"interface":"back","average":{"1min":4.4219999999999997,"5min":2.9350000000000001,"15min":2.9950000000000001},"min":{"1min":0.254,"5min":0.23499999999999999,"15min":0.193},"max":{"1min":14.538,"5min":15.108000000000001,"15min":62.726999999999997},"last":0.33100000000000002},{"interface":"front","average":{"1min":2.8250000000000002,"5min":2.7149999999999999,"15min":2.9990000000000001},"min":{"1min":0.27100000000000002,"5min":0.217,"15min":0.217},"max":{"1min":8.3689999999999998,"5min":17.013000000000002,"15min":62.689},"last":0.316}]},{"osd":3,"last update":"Sat Sep 23 20:47:39 2023","interfaces":[{"interface":"back","average":{"1min":2.9119999999999999,"5min":3.6099999999999999,"15min":3.165},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":13.367000000000001,"5min":55.017000000000003,"15min":55.017000000000003},"last":0.36499999999999999},{"interface":"front","average":{"1min":2.5760000000000001,"5min":2.9900000000000002,"15min":3.1989999999999998},"min":{"1min":0.32000000000000001,"5min":0.23799999999999999,"15min":0.21199999999999999},"max":{"1min":12.051,"5min":55.665999999999997,"15min":55.665999999999997},"last":0.38}]},{"osd":4,"last update":"Sat Sep 23 20:48:01 2023","interfaces":[{"interface":"back","average":{"1min":2.9860000000000002,"5min":2.0139999999999998,"15min":2.1299999999999999},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.25900000000000001},"max":{"1min":11.75,"5min":11.75,"15min":12.587999999999999},"last":0.44},{"interface":"front","average":{"1min":3.609,"5min":2.125,"15min":2.093},"min":{"1min":0.31,"5min":0.28899999999999998,"15min":0.27400000000000002},"max":{"1min":12.356999999999999,"5min":12.356999999999999,"15min":23.001000000000001},"last":0.48299999999999998}]},{"osd":5,"last update":"Sat Sep 23 20:48:01 2023","interfaces":[{"interface":"back","average":{"1min":3.4590000000000001,"5min":2.097,"15min":2.169},"min":{"1min":0.34300000000000003,"5min":0.30599999999999999,"15min":0.26500000000000001},"max":{"1min":9.3339999999999996,"5min":9.3339999999999996,"15min":22.969999999999999},"last":0.505},{"interface":"front","average":{"1min":2.9300000000000002,"5min":2.3359999999999999,"15min":2.492},"min":{"1min":0.39000000000000001,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":10.089,"5min":12.601000000000001,"15min":26.228000000000002},"last":0.49199999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:48:01 2023","interfaces":[{"interface":"back","average":{"1min":3.6920000000000002,"5min":2.3039999999999998,"15min":2.3439999999999999},"min":{"1min":0.41799999999999998,"5min":0.374,"15min":0.32300000000000001},"max":{"1min":9.2750000000000004,"5min":9.2750000000000004,"15min":24.056000000000001},"last":0.45200000000000001},{"interface":"front","average":{"1min":3.4020000000000001,"5min":2.395,"15min":2.3780000000000001},"min":{"1min":0.40999999999999998,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":12.427,"5min":12.698,"15min":26.268000000000001},"last":0.46000000000000002}]},{"osd":7,"last update":"Sat Sep 23 20:48:01 2023","interfaces":[{"interface":"back","average":{"1min":3.4140000000000001,"5min":2.7749999999999999,"15min":2.5870000000000002},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.36099999999999999},"max":{"1min":12.486000000000001,"5min":16.553999999999998,"15min":26.177},"last":0.40600000000000003},{"interface":"front","average":{"1min":3.3879999999999999,"5min":2.6179999999999999,"15min":2.5179999999999998},"min":{"1min":0.45300000000000001,"5min":0.40200000000000002,"15min":0.34200000000000003},"max":{"1min":10.183999999999999,"5min":16.649000000000001,"15min":22.422999999999998},"last":0.47299999999999998}]}]},{"osd":2,"up_from":0,"seq":446676599112,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":109,"seq":468151435586,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2002340,"kb_used_data":969884,"kb_used_omap":4,"kb_used_meta":1032443,"kb_avail":91742812,"statfs":{"total":95995035648,"available":93944639488,"internally_reserved":0,"allocated":993161216,"data_stored":990461882,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4514,"internal_metadata":1057222238},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:47:46 2023","interfaces":[{"interface":"back","average":{"1min":3.6509999999999998,"5min":2.6219999999999999,"15min":2.6019999999999999},"min":{"1min":0.23599999999999999,"5min":0.217,"15min":0.191},"max":{"1min":21.792999999999999,"5min":21.792999999999999,"15min":22.32},"last":0.498},{"interface":"front","average":{"1min":2.7069999999999999,"5min":2.7530000000000001,"15min":2.633},"min":{"1min":0.20200000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":14.845000000000001,"5min":22.805,"15min":22.805},"last":0.48699999999999999}]},{"osd":1,"last update":"Sat Sep 23 20:48:19 2023","interfaces":[{"interface":"back","average":{"1min":2.7530000000000001,"5min":2.1709999999999998,"15min":2.0840000000000001},"min":{"1min":0.26100000000000001,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":11.298,"5min":15.177,"15min":17.529},"last":0.55100000000000005},{"interface":"front","average":{"1min":2.7669999999999999,"5min":2.0830000000000002,"15min":2.2490000000000001},"min":{"1min":0.27200000000000002,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":11.380000000000001,"5min":11.380000000000001,"15min":14.19},"last":0.42099999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:47:46 2023","interfaces":[{"interface":"back","average":{"1min":3.0139999999999998,"5min":2.976,"15min":3.0619999999999998},"min":{"1min":0.34599999999999997,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":7.9210000000000003,"5min":19.478000000000002,"15min":68.206999999999994},"last":0.27900000000000003},{"interface":"front","average":{"1min":1.7829999999999999,"5min":2.016,"15min":2.556},"min":{"1min":0.31,"5min":0.245,"15min":0.245},"max":{"1min":8.0220000000000002,"5min":17.919,"15min":67.736999999999995},"last":0.66000000000000003}]},{"osd":4,"last update":"Sat Sep 23 20:47:46 2023","interfaces":[{"interface":"back","average":{"1min":1.9119999999999999,"5min":1.851,"15min":1.6890000000000001},"min":{"1min":0.48899999999999999,"5min":0.317,"15min":0.30499999999999999},"max":{"1min":9.0259999999999998,"5min":9.0259999999999998,"15min":22.157},"last":0.435},{"interface":"front","average":{"1min":1.4099999999999999,"5min":1.871,"15min":2.077},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.29399999999999998},"max":{"1min":5.024,"5min":8.1910000000000007,"15min":15.868},"last":0.33500000000000002}]},{"osd":5,"last update":"Sat Sep 23 20:47:46 2023","interfaces":[{"interface":"back","average":{"1min":1.9570000000000001,"5min":2.1160000000000001,"15min":1.911},"min":{"1min":0.46200000000000002,"5min":0.33800000000000002,"15min":0.32200000000000001},"max":{"1min":8.9849999999999994,"5min":8.9849999999999994,"15min":22.289999999999999},"last":3.0590000000000002},{"interface":"front","average":{"1min":1.7929999999999999,"5min":2.0369999999999999,"15min":1.829},"min":{"1min":0.379,"5min":0.34100000000000003,"15min":0.32000000000000001},"max":{"1min":6.1390000000000002,"5min":12.952,"15min":12.952},"last":0.64500000000000002}]},{"osd":6,"last update":"Sat Sep 23 20:47:46 2023","interfaces":[{"interface":"back","average":{"1min":2.5819999999999999,"5min":2.6509999999999998,"15min":2.004},"min":{"1min":0.44400000000000001,"5min":0.36599999999999999,"15min":0.34300000000000003},"max":{"1min":7.0389999999999997,"5min":12.917,"15min":12.917},"last":6.0599999999999996},{"interface":"front","average":{"1min":1.698,"5min":2.1440000000000001,"15min":2.1349999999999998},"min":{"1min":0.36699999999999999,"5min":0.34599999999999997,"15min":0.33700000000000002},"max":{"1min":6.4930000000000003,"5min":8.9320000000000004,"15min":15.936},"last":5.7850000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:47:46 2023","interfaces":[{"interface":"back","average":{"1min":1.768,"5min":2.262,"15min":2.302},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.36899999999999999},"max":{"1min":7.6429999999999998,"5min":11.885999999999999,"15min":15.976000000000001},"last":4.9009999999999998},{"interface":"front","average":{"1min":2.1360000000000001,"5min":2.1619999999999999,"15min":1.968},"min":{"1min":0.51100000000000001,"5min":0.42799999999999999,"15min":0.40999999999999998},"max":{"1min":9.0549999999999997,"5min":10.853999999999999,"15min":22.202999999999999},"last":4.7549999999999999}]}]},{"osd":4,"up_from":30,"seq":128849019340,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2033492,"kb_used_data":1517324,"kb_used_omap":0,"kb_used_meta":516160,"kb_avail":91711660,"statfs":{"total":95995035648,"available":93912739840,"internally_reserved":0,"allocated":1553739776,"data_stored":1552160118,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":528547840},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:48:04 2023","interfaces":[{"interface":"back","average":{"1min":2.1320000000000001,"5min":1.8500000000000001,"15min":1.8420000000000001},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24099999999999999},"max":{"1min":7.1120000000000001,"5min":11.709,"15min":24.114999999999998},"last":0.84199999999999997},{"interface":"front","average":{"1min":2.4329999999999998,"5min":2.1480000000000001,"15min":2.0870000000000002},"min":{"1min":0.29799999999999999,"5min":0.25800000000000001,"15min":0.246},"max":{"1min":6.8639999999999999,"5min":13.74,"15min":24.135999999999999},"last":4.4539999999999997}]},{"osd":1,"last update":"Sat Sep 23 20:48:16 2023","interfaces":[{"interface":"back","average":{"1min":2.093,"5min":1.9370000000000001,"15min":1.8280000000000001},"min":{"1min":0.27800000000000002,"5min":0.26300000000000001,"15min":0.25900000000000001},"max":{"1min":10.157,"5min":10.157,"15min":18.190999999999999},"last":4.5060000000000002},{"interface":"front","average":{"1min":2.278,"5min":1.6160000000000001,"15min":1.6319999999999999},"min":{"1min":0.29399999999999998,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":10.169,"5min":10.169,"15min":18.173999999999999},"last":7.8209999999999997}]},{"osd":2,"last update":"Sat Sep 23 20:48:16 2023","interfaces":[{"interface":"back","average":{"1min":2.0880000000000001,"5min":1.879,"15min":1.752},"min":{"1min":0.32000000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":4.5739999999999998,"5min":9.3320000000000007,"15min":13.051},"last":0.96099999999999997},{"interface":"front","average":{"1min":2.089,"5min":2.0550000000000002,"15min":2.1699999999999999},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":7.0800000000000001,"5min":8.7230000000000008,"15min":14.614000000000001},"last":4.4139999999999997}]},{"osd":3,"last update":"Sat Sep 23 20:47:31 2023","interfaces":[{"interface":"back","average":{"1min":1.8560000000000001,"5min":2.0089999999999999,"15min":1.901},"min":{"1min":0.318,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":5.0270000000000001,"5min":8.6180000000000003,"15min":12.147},"last":7.7149999999999999},{"interface":"front","average":{"1min":2.101,"5min":2.2000000000000002,"15min":2.0339999999999998},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.30299999999999999},"max":{"1min":6.3159999999999998,"5min":17.169,"15min":17.169},"last":7.7830000000000004}]},{"osd":5,"last update":"Sat Sep 23 20:48:04 2023","interfaces":[{"interface":"back","average":{"1min":0.79300000000000004,"5min":1.458,"15min":1.0740000000000001},"min":{"1min":0.28399999999999997,"5min":0.253,"15min":0.253},"max":{"1min":3.5790000000000002,"5min":10.202,"15min":10.202},"last":16.940000000000001},{"interface":"front","average":{"1min":1.1479999999999999,"5min":1.5369999999999999,"15min":1.1850000000000001},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":4.7430000000000003,"5min":8.1440000000000001,"15min":8.1440000000000001},"last":4.4809999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:48:16 2023","interfaces":[{"interface":"back","average":{"1min":1.9910000000000001,"5min":1.6140000000000001,"15min":1.395},"min":{"1min":0.30099999999999999,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":6.4290000000000003,"5min":10.840999999999999,"15min":14.731999999999999},"last":4.7519999999999998},{"interface":"front","average":{"1min":1.681,"5min":1.6359999999999999,"15min":1.234},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":8.0269999999999992,"5min":8.0269999999999992,"15min":8.0269999999999992},"last":17.059000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:47:31 2023","interfaces":[{"interface":"back","average":{"1min":2.4089999999999998,"5min":5.4740000000000002,"15min":2.6669999999999998},"min":{"1min":0.377,"5min":0.35199999999999998,"15min":0.318},"max":{"1min":6.3920000000000003,"5min":313.50099999999998,"15min":313.50099999999998},"last":11.228},{"interface":"front","average":{"1min":1.8340000000000001,"5min":4.7060000000000004,"15min":2.2290000000000001},"min":{"1min":0.35099999999999998,"5min":0.33200000000000002,"15min":0.32700000000000001},"max":{"1min":6.1829999999999998,"5min":312.67500000000001,"15min":312.67500000000001},"last":17.093}]}]},{"osd":5,"up_from":39,"seq":167503725001,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1555468,"kb_used_data":980868,"kb_used_omap":0,"kb_used_meta":574592,"kb_avail":92189684,"statfs":{"total":95995035648,"available":94402236416,"internally_reserved":0,"allocated":1004408832,"data_stored":1002833972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":588382208},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:48:10 2023","interfaces":[{"interface":"back","average":{"1min":2.5129999999999999,"5min":1.8700000000000001,"15min":1.891},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.20699999999999999},"max":{"1min":6.673,"5min":15.018000000000001,"15min":18.16},"last":5.5140000000000002},{"interface":"front","average":{"1min":2.258,"5min":1.915,"15min":1.8300000000000001},"min":{"1min":0.30099999999999999,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":8.6050000000000004,"5min":15.034000000000001,"15min":15.034000000000001},"last":4.1929999999999996}]},{"osd":1,"last update":"Sat Sep 23 20:48:23 2023","interfaces":[{"interface":"back","average":{"1min":2.4460000000000002,"5min":2.1480000000000001,"15min":2.3719999999999999},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.253},"max":{"1min":6.9850000000000003,"5min":9.4199999999999999,"15min":61.005000000000003},"last":4.8929999999999998},{"interface":"front","average":{"1min":2.6720000000000002,"5min":1.9390000000000001,"15min":2.2269999999999999},"min":{"1min":0.38200000000000001,"5min":0.27700000000000002,"15min":0.23499999999999999},"max":{"1min":6.6280000000000001,"5min":16.219000000000001,"15min":59.688000000000002},"last":5.5549999999999997}]},{"osd":2,"last update":"Sat Sep 23 20:48:10 2023","interfaces":[{"interface":"back","average":{"1min":2.6160000000000001,"5min":2.1400000000000001,"15min":2.1579999999999999},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26500000000000001},"max":{"1min":7.194,"5min":12.226000000000001,"15min":37.011000000000003},"last":5.5819999999999999},{"interface":"front","average":{"1min":2.5219999999999998,"5min":1.867,"15min":2.0350000000000001},"min":{"1min":0.39600000000000002,"5min":0.314,"15min":0.255},"max":{"1min":5.9790000000000001,"5min":9.8260000000000005,"15min":36.835000000000001},"last":4.165}]},{"osd":3,"last update":"Sat Sep 23 20:48:30 2023","interfaces":[{"interface":"back","average":{"1min":2.9380000000000002,"5min":1.9650000000000001,"15min":2.3170000000000002},"min":{"1min":0.54100000000000004,"5min":0.33800000000000002,"15min":0.314},"max":{"1min":8.7360000000000007,"5min":8.7360000000000007,"15min":36.716000000000001},"last":4.9969999999999999},{"interface":"front","average":{"1min":3.3719999999999999,"5min":2.0019999999999998,"15min":2.3730000000000002},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":7.4029999999999996,"5min":7.8869999999999996,"15min":18.504000000000001},"last":5.6059999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:48:00 2023","interfaces":[{"interface":"back","average":{"1min":1.593,"5min":1.8180000000000001,"15min":1.2270000000000001},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.255},"max":{"1min":6.3170000000000002,"5min":13.391999999999999,"15min":13.391999999999999},"last":8.9390000000000001},{"interface":"front","average":{"1min":1.5129999999999999,"5min":1.6899999999999999,"15min":1.2390000000000001},"min":{"1min":0.34200000000000003,"5min":0.29199999999999998,"15min":0.28299999999999997},"max":{"1min":6.6600000000000001,"5min":7.3799999999999999,"15min":7.3799999999999999},"last":5.665}]},{"osd":6,"last update":"Sat Sep 23 20:48:11 2023","interfaces":[{"interface":"back","average":{"1min":1.6319999999999999,"5min":1.913,"15min":1.431},"min":{"1min":0.40300000000000002,"5min":0.36199999999999999,"15min":0.30199999999999999},"max":{"1min":6.6959999999999997,"5min":13.734,"15min":13.734},"last":8.9039999999999999},{"interface":"front","average":{"1min":2.278,"5min":1.5129999999999999,"15min":1.28},"min":{"1min":0.40699999999999997,"5min":0.317,"15min":0.317},"max":{"1min":7.5270000000000001,"5min":7.5270000000000001,"15min":8.7449999999999992},"last":9.6519999999999992}]},{"osd":7,"last update":"Sat Sep 23 20:48:30 2023","interfaces":[{"interface":"back","average":{"1min":2.4689999999999999,"5min":4.5060000000000002,"15min":2.2490000000000001},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.28499999999999998},"max":{"1min":7.899,"5min":294.70999999999998,"15min":294.70999999999998},"last":5.6379999999999999},{"interface":"front","average":{"1min":2.1240000000000001,"5min":4.5860000000000003,"15min":2.3149999999999999},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.27900000000000003},"max":{"1min":14.032999999999999,"5min":294.74400000000003,"15min":294.74400000000003},"last":4.1340000000000003}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4722,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1360,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8206,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7156,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3154,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1991839744,"data_stored":1991835672,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":577044480,"data_stored":577044480,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1174798336,"data_stored":1174798336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":965279744,"data_stored":965279744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1539313664,"data_stored":1539309592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":991559680,"data_stored":991559680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1308102656,"data_stored":1308098584,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":750911488,"data_stored":750911488,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:48:32.663 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-09-23T20:48:32.664 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd pool set unique_pool_5 pg_num 26 2023-09-23T20:48:32.674 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:48:32.772 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:32.860 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:48:32.867 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:32.867 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:48:32.943 INFO:teuthology.orchestra.run.smithi149.stdout:{ 2023-09-23T20:48:32.943 INFO:teuthology.orchestra.run.smithi149.stdout: "ops": [], 2023-09-23T20:48:32.944 INFO:teuthology.orchestra.run.smithi149.stdout: "complaint_time": 30, 2023-09-23T20:48:32.944 INFO:teuthology.orchestra.run.smithi149.stdout: "num_blocked_ops": 0 2023-09-23T20:48:32.944 INFO:teuthology.orchestra.run.smithi149.stdout:} 2023-09-23T20:48:32.968 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:48:33.028 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:33.087 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:48:33.096 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:33.096 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:48:33.197 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:48:33.276 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:33.364 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:48:33.370 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:33.371 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:48:33.389 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T20:48:33.472 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:48:33.480 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:48:33.554 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:33.571 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T20:48:34.187 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:48:34.294 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:48:34.337 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:33 smithi018 ceph-mon[179647]: from='client.46874 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:48:34.337 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:33 smithi018 ceph-mon[179647]: pgmap v1471: 119 pgs: 2 active+remapped+backfilling, 4 active+remapped+backfill_wait, 113 active+clean; 2.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 705 op/s; 15788/137796 objects misplaced (11.458%); 15 MiB/s, 244 objects/s recovering 2023-09-23T20:48:34.337 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:33 smithi018 ceph-mon[179647]: 1.8 scrub starts 2023-09-23T20:48:34.340 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:33 smithi018 ceph-mon[184144]: from='client.46874 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:48:34.340 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:33 smithi018 ceph-mon[184144]: pgmap v1471: 119 pgs: 2 active+remapped+backfilling, 4 active+remapped+backfill_wait, 113 active+clean; 2.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 705 op/s; 15788/137796 objects misplaced (11.458%); 15 MiB/s, 244 objects/s recovering 2023-09-23T20:48:34.340 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:33 smithi018 ceph-mon[184144]: 1.8 scrub starts 2023-09-23T20:48:34.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:33 smithi149 ceph-mon[170394]: from='client.46874 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:48:34.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:33 smithi149 ceph-mon[170394]: pgmap v1471: 119 pgs: 2 active+remapped+backfilling, 4 active+remapped+backfill_wait, 113 active+clean; 2.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 705 op/s; 15788/137796 objects misplaced (11.458%); 15 MiB/s, 244 objects/s recovering 2023-09-23T20:48:34.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:33 smithi149 ceph-mon[170394]: 1.8 scrub starts 2023-09-23T20:48:34.597 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:48:34.603 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:34.604 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:48:34.705 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:48:34.757 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:34.826 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:48:34.831 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:34.832 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:48:34.933 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:48:35.031 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:35.141 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:48:35.148 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:35.148 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:48:35.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:35 smithi149 ceph-mon[170394]: 1.8 scrub starts 2023-09-23T20:48:35.249 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:48:35.271 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:48:34 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:48:34] "GET /metrics HTTP/1.1" 200 36629 "" "Prometheus/2.43.0" 2023-09-23T20:48:35.271 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:35 smithi018 ceph-mon[179647]: 1.8 scrub starts 2023-09-23T20:48:35.274 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:35 smithi018 ceph-mon[184144]: 1.8 scrub starts 2023-09-23T20:48:35.349 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:35.445 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:48:35.459 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:35.460 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:48:35.561 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:48:35.644 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:35.761 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:48:35.773 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:35.774 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:48:35.875 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:48:35.933 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:35.990 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:48:35.993 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:35.993 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:48:36.033 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:48:36.065 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:48:36.095 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:48:36.112 INFO:teuthology.orchestra.run.smithi149.stdout:{ 2023-09-23T20:48:36.112 INFO:teuthology.orchestra.run.smithi149.stdout: "size": 20, 2023-09-23T20:48:36.112 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 600, 2023-09-23T20:48:36.112 INFO:teuthology.orchestra.run.smithi149.stdout: "ops": [ 2023-09-23T20:48:36.112 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.113 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "MOSDPGPush(8.19 427/410 [PushOp(8:9f37acf0:::benchmark_data_smithi018_341490_object84819:head, version: 394'5338, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(8:9f37acf0:::benchmark_data_smithi018_341490_object84819:head@394'5338, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-09-23T20:48:36.113 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:39:26.795989+0000", 2023-09-23T20:48:36.113 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 549.31128206300002, 2023-09-23T20:48:36.113 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 18.381641347999999, 2023-09-23T20:48:36.113 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:48:36.113 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "started", 2023-09-23T20:48:36.113 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:48:36.113 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.113 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:48:36.113 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:26.795989+0000", 2023-09-23T20:48:36.113 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:48:36.113 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.114 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.114 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:48:36.114 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:26.795989+0000", 2023-09-23T20:48:36.114 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:48:36.114 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.114 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.114 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:48:36.114 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:26.795988+0000", 2023-09-23T20:48:36.114 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:48:36.114 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.114 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.115 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:48:36.115 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:26.796034+0000", 2023-09-23T20:48:36.115 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.5374999999999998e-05 2023-09-23T20:48:36.115 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.115 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.115 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:48:36.115 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:26.796055+0000", 2023-09-23T20:48:36.115 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.126e-05 2023-09-23T20:48:36.115 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.115 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.115 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:48:36.115 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:26.796067+0000", 2023-09-23T20:48:36.116 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.2043e-05 2023-09-23T20:48:36.116 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.116 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.116 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:48:36.116 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:45.167841+0000", 2023-09-23T20:48:36.116 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 18.371773717 2023-09-23T20:48:36.116 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.116 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.116 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:48:36.116 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:45.168079+0000", 2023-09-23T20:48:36.116 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000237839 2023-09-23T20:48:36.116 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.117 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.117 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:48:36.117 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:39:45.177631+0000", 2023-09-23T20:48:36.117 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0095519390000000006 2023-09-23T20:48:36.117 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:48:36.117 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:48:36.117 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:48:36.117 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.117 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.117 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:76949 9.b 9:dc3199d0:::benchmark_data_smithi018_381516_object76948:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e465)", 2023-09-23T20:48:36.117 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:41:58.481021+0000", 2023-09-23T20:48:36.117 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 397.62625025400001, 2023-09-23T20:48:36.118 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.012473631, 2023-09-23T20:48:36.118 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:48:36.118 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:48:36.118 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:48:36.118 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:48:36.118 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:48:36.118 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 76949 2023-09-23T20:48:36.118 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.118 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:48:36.118 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.118 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:48:36.119 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.481021+0000", 2023-09-23T20:48:36.119 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:48:36.119 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.119 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.119 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:48:36.119 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.481021+0000", 2023-09-23T20:48:36.119 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:48:36.119 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.119 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.119 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:48:36.119 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.481020+0000", 2023-09-23T20:48:36.119 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:48:36.120 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.120 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.120 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:48:36.120 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.481102+0000", 2023-09-23T20:48:36.120 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.2538000000000002e-05 2023-09-23T20:48:36.120 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.120 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.120 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:48:36.120 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.481122+0000", 2023-09-23T20:48:36.120 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.9556e-05 2023-09-23T20:48:36.120 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.121 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.121 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:48:36.121 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.481129+0000", 2023-09-23T20:48:36.121 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.7449999999999995e-06 2023-09-23T20:48:36.121 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.121 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.121 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:48:36.121 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.481713+0000", 2023-09-23T20:48:36.121 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000583167 2023-09-23T20:48:36.121 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.121 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.121 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:48:36.122 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.481726+0000", 2023-09-23T20:48:36.122 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.3291e-05 2023-09-23T20:48:36.122 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.122 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.122 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:48:36.122 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.467637+0000", 2023-09-23T20:48:36.122 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.98591135500000004 2023-09-23T20:48:36.122 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.122 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.122 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:48:36.123 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.470079+0000", 2023-09-23T20:48:36.123 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0024414089999999999 2023-09-23T20:48:36.123 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.123 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.123 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:48:36.123 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.470267+0000", 2023-09-23T20:48:36.123 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00018824299999999999 2023-09-23T20:48:36.123 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.123 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.123 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:48:36.123 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.481273+0000", 2023-09-23T20:48:36.124 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.011006395 2023-09-23T20:48:36.124 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.124 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.124 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:48:36.124 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.493371+0000", 2023-09-23T20:48:36.124 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.012098138 2023-09-23T20:48:36.124 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.124 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.124 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:48:36.124 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.493433+0000", 2023-09-23T20:48:36.124 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.1437000000000001e-05 2023-09-23T20:48:36.124 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.125 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.125 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:48:36.125 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.493472+0000", 2023-09-23T20:48:36.125 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.9038000000000001e-05 2023-09-23T20:48:36.125 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.125 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.125 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:48:36.125 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.493495+0000", 2023-09-23T20:48:36.125 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.2778000000000001e-05 2023-09-23T20:48:36.125 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:48:36.125 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:48:36.126 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:48:36.126 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.126 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.126 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:76967 9.b 9:d898a916:::benchmark_data_smithi018_381516_object76966:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e465)", 2023-09-23T20:48:36.126 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:41:58.507229+0000", 2023-09-23T20:48:36.126 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 397.60004184399997, 2023-09-23T20:48:36.126 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.98635601299999998, 2023-09-23T20:48:36.126 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:48:36.126 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:48:36.126 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:48:36.126 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:48:36.127 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:48:36.127 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 76967 2023-09-23T20:48:36.127 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.127 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:48:36.127 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.127 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:48:36.127 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.507229+0000", 2023-09-23T20:48:36.127 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:48:36.127 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.127 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.127 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:48:36.127 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.507229+0000", 2023-09-23T20:48:36.128 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:48:36.128 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.128 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.128 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:48:36.128 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.507228+0000", 2023-09-23T20:48:36.128 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999981 2023-09-23T20:48:36.128 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.128 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.128 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:48:36.128 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.507285+0000", 2023-09-23T20:48:36.128 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.7531000000000002e-05 2023-09-23T20:48:36.128 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.129 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.129 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:48:36.129 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.507307+0000", 2023-09-23T20:48:36.129 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.2039e-05 2023-09-23T20:48:36.129 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.129 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.129 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:48:36.129 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.507314+0000", 2023-09-23T20:48:36.129 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.7089999999999996e-06 2023-09-23T20:48:36.129 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.129 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.130 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:48:36.130 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.507629+0000", 2023-09-23T20:48:36.130 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00031477799999999999 2023-09-23T20:48:36.130 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.130 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.130 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:48:36.130 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.507641+0000", 2023-09-23T20:48:36.130 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.258e-05 2023-09-23T20:48:36.130 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.130 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.131 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:48:36.131 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.472754+0000", 2023-09-23T20:48:36.131 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.96511288200000001 2023-09-23T20:48:36.131 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.131 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.131 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:48:36.131 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.472887+0000", 2023-09-23T20:48:36.131 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000133132 2023-09-23T20:48:36.131 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.131 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.131 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:48:36.132 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.472997+0000", 2023-09-23T20:48:36.132 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000109805 2023-09-23T20:48:36.132 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.132 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.132 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:48:36.132 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.484044+0000", 2023-09-23T20:48:36.132 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.011047250999999999 2023-09-23T20:48:36.132 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.132 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.132 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:48:36.132 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.493403+0000", 2023-09-23T20:48:36.132 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0093591230000000004 2023-09-23T20:48:36.133 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.133 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.133 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:48:36.133 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.493533+0000", 2023-09-23T20:48:36.133 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000129271 2023-09-23T20:48:36.133 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.133 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.133 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:48:36.133 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.493558+0000", 2023-09-23T20:48:36.133 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.5477e-05 2023-09-23T20:48:36.133 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.134 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.134 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:48:36.134 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.493585+0000", 2023-09-23T20:48:36.134 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.7274e-05 2023-09-23T20:48:36.134 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:48:36.134 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:48:36.134 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:48:36.134 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.134 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.134 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:76972 9.b 9:d979209f:::benchmark_data_smithi018_381516_object76971:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e465)", 2023-09-23T20:48:36.134 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:41:58.507469+0000", 2023-09-23T20:48:36.134 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 397.59980231200001, 2023-09-23T20:48:36.135 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.98628621599999999, 2023-09-23T20:48:36.135 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:48:36.135 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:48:36.135 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:48:36.135 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:48:36.135 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:48:36.135 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 76972 2023-09-23T20:48:36.135 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.135 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:48:36.135 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.135 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:48:36.136 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.507469+0000", 2023-09-23T20:48:36.136 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:48:36.136 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.136 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.136 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:48:36.136 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.507469+0000", 2023-09-23T20:48:36.136 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:48:36.136 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.136 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.136 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:48:36.136 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.507468+0000", 2023-09-23T20:48:36.136 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:48:36.137 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.137 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.137 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:48:36.137 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.507525+0000", 2023-09-23T20:48:36.137 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.6431000000000002e-05 2023-09-23T20:48:36.137 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.137 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.137 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:48:36.137 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.507533+0000", 2023-09-23T20:48:36.137 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.2400000000000007e-06 2023-09-23T20:48:36.137 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.138 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.138 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:48:36.138 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.507538+0000", 2023-09-23T20:48:36.138 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.2129999999999999e-06 2023-09-23T20:48:36.138 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.138 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.138 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:48:36.138 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.507674+0000", 2023-09-23T20:48:36.138 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00013521599999999999 2023-09-23T20:48:36.138 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.138 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.138 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:48:36.139 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.507677+0000", 2023-09-23T20:48:36.139 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.3409999999999999e-06 2023-09-23T20:48:36.139 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.139 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.139 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:48:36.139 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.478437+0000", 2023-09-23T20:48:36.139 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.97076035400000005 2023-09-23T20:48:36.139 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.139 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.139 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:48:36.139 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.478570+0000", 2023-09-23T20:48:36.140 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00013248599999999999 2023-09-23T20:48:36.140 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.140 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.140 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:48:36.140 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.478661+0000", 2023-09-23T20:48:36.140 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.1013000000000004e-05 2023-09-23T20:48:36.140 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.140 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.140 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:48:36.140 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.486788+0000", 2023-09-23T20:48:36.140 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0081275240000000006 2023-09-23T20:48:36.140 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.141 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.141 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:48:36.141 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.493688+0000", 2023-09-23T20:48:36.141 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0068995159999999996 2023-09-23T20:48:36.141 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.141 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.141 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:48:36.141 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.493704+0000", 2023-09-23T20:48:36.141 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.6441999999999999e-05 2023-09-23T20:48:36.141 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.141 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.142 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:48:36.142 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.493734+0000", 2023-09-23T20:48:36.142 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.9258999999999999e-05 2023-09-23T20:48:36.142 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.142 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.142 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:48:36.142 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.493755+0000", 2023-09-23T20:48:36.142 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.1636000000000001e-05 2023-09-23T20:48:36.142 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:48:36.142 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:48:36.142 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:48:36.143 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.143 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.143 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:102263 10.e 10:7fa4b63f:::benchmark_data_smithi018_422431_object102262:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e551)", 2023-09-23T20:48:36.143 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:47:08.130511+0000", 2023-09-23T20:48:36.143 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 87.976760585999997, 2023-09-23T20:48:36.143 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.028872988, 2023-09-23T20:48:36.143 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:48:36.143 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:48:36.143 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:48:36.143 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:48:36.143 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:48:36.143 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 102263 2023-09-23T20:48:36.144 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.144 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:48:36.144 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.144 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:48:36.144 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.130511+0000", 2023-09-23T20:48:36.144 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:48:36.144 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.144 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.144 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:48:36.144 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.130511+0000", 2023-09-23T20:48:36.145 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:48:36.145 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.145 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.145 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:48:36.145 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.130510+0000", 2023-09-23T20:48:36.145 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:48:36.145 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.145 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.145 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:48:36.145 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.130540+0000", 2023-09-23T20:48:36.145 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.9640000000000001e-05 2023-09-23T20:48:36.146 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.146 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.146 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:48:36.146 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.130547+0000", 2023-09-23T20:48:36.146 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.5390000000000002e-06 2023-09-23T20:48:36.146 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.146 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.146 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:48:36.146 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.130550+0000", 2023-09-23T20:48:36.146 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.9979999999999999e-06 2023-09-23T20:48:36.146 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.146 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.147 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:48:36.147 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.130601+0000", 2023-09-23T20:48:36.147 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.0937999999999999e-05 2023-09-23T20:48:36.147 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.147 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.147 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:48:36.147 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.130731+0000", 2023-09-23T20:48:36.147 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00012974899999999999 2023-09-23T20:48:36.147 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.147 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.147 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 2,4,5", 2023-09-23T20:48:36.148 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.130882+0000", 2023-09-23T20:48:36.148 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00015048199999999999 2023-09-23T20:48:36.148 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.148 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.148 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:48:36.148 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.137064+0000", 2023-09-23T20:48:36.148 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0061819739999999998 2023-09-23T20:48:36.148 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.148 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.148 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:48:36.148 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.137121+0000", 2023-09-23T20:48:36.148 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.7302999999999997e-05 2023-09-23T20:48:36.149 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.149 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.149 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:48:36.149 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.139210+0000", 2023-09-23T20:48:36.149 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.002089184 2023-09-23T20:48:36.149 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.149 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.149 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:48:36.149 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.173918+0000", 2023-09-23T20:48:36.149 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.034707835999999999 2023-09-23T20:48:36.149 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.149 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.150 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:48:36.150 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.173930+0000", 2023-09-23T20:48:36.150 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.185e-05 2023-09-23T20:48:36.150 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.150 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.150 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:48:36.150 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.159372+0000", 2023-09-23T20:48:36.150 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.98544211999999998 2023-09-23T20:48:36.150 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.150 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.150 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:48:36.151 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.159384+0000", 2023-09-23T20:48:36.151 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1737e-05 2023-09-23T20:48:36.151 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:48:36.151 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:48:36.151 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:48:36.151 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.151 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.151 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:102264 10.e 10:70bfcac0:::benchmark_data_smithi018_422431_object102263:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e551)", 2023-09-23T20:48:36.151 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:47:08.130562+0000", 2023-09-23T20:48:36.151 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 87.976709295000006, 2023-09-23T20:48:36.151 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.028857202, 2023-09-23T20:48:36.151 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:48:36.152 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:48:36.152 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:48:36.152 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:48:36.152 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:48:36.152 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 102264 2023-09-23T20:48:36.152 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.152 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:48:36.152 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.152 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:48:36.152 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.130562+0000", 2023-09-23T20:48:36.152 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:48:36.153 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.153 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.153 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:48:36.153 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.130562+0000", 2023-09-23T20:48:36.153 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:48:36.153 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.153 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.153 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:48:36.153 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.130561+0000", 2023-09-23T20:48:36.153 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:48:36.153 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.153 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.154 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:48:36.154 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.130598+0000", 2023-09-23T20:48:36.154 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.6294999999999997e-05 2023-09-23T20:48:36.154 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.154 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.154 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:48:36.154 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.130602+0000", 2023-09-23T20:48:36.154 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.5379999999999996e-06 2023-09-23T20:48:36.154 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.154 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.155 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:48:36.155 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.130605+0000", 2023-09-23T20:48:36.155 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.909e-06 2023-09-23T20:48:36.155 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.155 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.155 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:48:36.155 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.134028+0000", 2023-09-23T20:48:36.155 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0034227910000000001 2023-09-23T20:48:36.155 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.155 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.156 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:48:36.156 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.134217+0000", 2023-09-23T20:48:36.156 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00018908099999999999 2023-09-23T20:48:36.156 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.156 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.156 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 2,4,5", 2023-09-23T20:48:36.156 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.134391+0000", 2023-09-23T20:48:36.156 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00017426099999999999 2023-09-23T20:48:36.156 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.156 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.156 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:48:36.157 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.137246+0000", 2023-09-23T20:48:36.157 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0028549700000000001 2023-09-23T20:48:36.157 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.157 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.157 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:48:36.157 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.138047+0000", 2023-09-23T20:48:36.157 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00080048900000000004 2023-09-23T20:48:36.157 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.157 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.157 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:48:36.157 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.139365+0000", 2023-09-23T20:48:36.157 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.001318512 2023-09-23T20:48:36.158 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.158 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.158 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:48:36.158 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.174065+0000", 2023-09-23T20:48:36.158 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.034699636999999998 2023-09-23T20:48:36.158 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.158 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.158 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:48:36.158 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.174076+0000", 2023-09-23T20:48:36.158 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0655999999999999e-05 2023-09-23T20:48:36.158 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.159 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.159 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:48:36.159 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.159411+0000", 2023-09-23T20:48:36.159 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.98533562799999996 2023-09-23T20:48:36.159 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.159 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.159 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:48:36.159 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.159419+0000", 2023-09-23T20:48:36.159 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.9950000000000005e-06 2023-09-23T20:48:36.159 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:48:36.159 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:48:36.159 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:48:36.160 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.160 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.160 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:102265 10.d 10:b077b11a:::benchmark_data_smithi018_422431_object102264:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e551)", 2023-09-23T20:48:36.160 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:47:08.130611+0000", 2023-09-23T20:48:36.160 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 87.976660277999997, 2023-09-23T20:48:36.160 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.088474049, 2023-09-23T20:48:36.160 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:48:36.160 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:48:36.160 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:48:36.160 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:48:36.160 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:48:36.161 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 102265 2023-09-23T20:48:36.161 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.161 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:48:36.161 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.161 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:48:36.161 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.130611+0000", 2023-09-23T20:48:36.161 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:48:36.161 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.161 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.161 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:48:36.161 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.130611+0000", 2023-09-23T20:48:36.162 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:48:36.162 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.162 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.162 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:48:36.162 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.130610+0000", 2023-09-23T20:48:36.162 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:48:36.162 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.162 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.162 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:48:36.162 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.130655+0000", 2023-09-23T20:48:36.163 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.4501e-05 2023-09-23T20:48:36.163 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.163 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.163 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:48:36.163 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.130663+0000", 2023-09-23T20:48:36.163 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.1429999999999998e-06 2023-09-23T20:48:36.163 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.163 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.163 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:48:36.163 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.130666+0000", 2023-09-23T20:48:36.163 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.039e-06 2023-09-23T20:48:36.164 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.164 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.164 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:48:36.164 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.130701+0000", 2023-09-23T20:48:36.164 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.5135000000000002e-05 2023-09-23T20:48:36.164 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.164 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.164 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:48:36.164 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.130794+0000", 2023-09-23T20:48:36.164 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.31e-05 2023-09-23T20:48:36.164 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.165 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.165 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:48:36.165 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.130902+0000", 2023-09-23T20:48:36.165 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00010738699999999999 2023-09-23T20:48:36.165 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.165 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.165 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:48:36.165 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.134006+0000", 2023-09-23T20:48:36.165 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0031044319999999999 2023-09-23T20:48:36.165 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.165 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.165 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:48:36.166 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.143177+0000", 2023-09-23T20:48:36.166 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.009170447 2023-09-23T20:48:36.166 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.166 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.166 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:48:36.166 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.177193+0000", 2023-09-23T20:48:36.166 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.034015861000000001 2023-09-23T20:48:36.166 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.166 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.166 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:48:36.167 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.177202+0000", 2023-09-23T20:48:36.167 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.8379999999999997e-06 2023-09-23T20:48:36.167 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.167 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.167 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:48:36.167 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.218917+0000", 2023-09-23T20:48:36.167 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0417144490000001 2023-09-23T20:48:36.167 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.167 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.167 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:48:36.167 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.219085+0000", 2023-09-23T20:48:36.168 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00016819 2023-09-23T20:48:36.168 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:48:36.168 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:48:36.168 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:48:36.168 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.168 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.168 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:102284 10.0 10:0c1d13a0:::benchmark_data_smithi018_422431_object102283:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e552)", 2023-09-23T20:48:36.168 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:47:08.192917+0000", 2023-09-23T20:48:36.168 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 87.914353950000006, 2023-09-23T20:48:36.168 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1174581180000001, 2023-09-23T20:48:36.168 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:48:36.168 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:48:36.169 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:48:36.169 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:48:36.169 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:48:36.169 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 102284 2023-09-23T20:48:36.169 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.169 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:48:36.169 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.169 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:48:36.169 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.192917+0000", 2023-09-23T20:48:36.169 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:48:36.169 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.170 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.170 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:48:36.170 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.192917+0000", 2023-09-23T20:48:36.170 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:48:36.170 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.170 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.170 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:48:36.170 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.192916+0000", 2023-09-23T20:48:36.170 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:48:36.170 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.170 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.171 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:48:36.171 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.192958+0000", 2023-09-23T20:48:36.171 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.1974999999999997e-05 2023-09-23T20:48:36.171 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.171 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.171 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:48:36.171 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.193237+0000", 2023-09-23T20:48:36.171 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00027918399999999998 2023-09-23T20:48:36.171 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.171 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.171 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:48:36.172 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.193247+0000", 2023-09-23T20:48:36.172 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0015e-05 2023-09-23T20:48:36.172 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.172 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.172 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:48:36.172 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.193334+0000", 2023-09-23T20:48:36.172 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.6867999999999996e-05 2023-09-23T20:48:36.172 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.172 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.172 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:48:36.172 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.193351+0000", 2023-09-23T20:48:36.172 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.6545000000000002e-05 2023-09-23T20:48:36.173 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.173 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.173 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:48:36.173 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.216597+0000", 2023-09-23T20:48:36.173 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0232465260000001 2023-09-23T20:48:36.173 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.173 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.173 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:48:36.173 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.216764+0000", 2023-09-23T20:48:36.173 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000166937 2023-09-23T20:48:36.173 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.174 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.174 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 1,4", 2023-09-23T20:48:36.174 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.216936+0000", 2023-09-23T20:48:36.174 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00017148099999999999 2023-09-23T20:48:36.174 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.174 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.174 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:48:36.174 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.309749+0000", 2023-09-23T20:48:36.174 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.092813393999999994 2023-09-23T20:48:36.174 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.174 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.175 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:48:36.175 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.309824+0000", 2023-09-23T20:48:36.175 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.5284999999999995e-05 2023-09-23T20:48:36.175 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.175 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.175 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:48:36.175 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.309902+0000", 2023-09-23T20:48:36.175 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.8061999999999999e-05 2023-09-23T20:48:36.175 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.175 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.175 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:48:36.176 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.309995+0000", 2023-09-23T20:48:36.176 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.2719000000000005e-05 2023-09-23T20:48:36.176 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.176 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.176 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:48:36.176 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.310375+0000", 2023-09-23T20:48:36.176 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00038043700000000002 2023-09-23T20:48:36.176 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:48:36.176 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:48:36.176 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:48:36.176 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.177 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.177 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:102291 10.0 10:0574a143:::benchmark_data_smithi018_422431_object102290:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e552)", 2023-09-23T20:48:36.177 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:47:08.213037+0000", 2023-09-23T20:48:36.177 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 87.894234037999993, 2023-09-23T20:48:36.177 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.097677486, 2023-09-23T20:48:36.177 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:48:36.177 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:48:36.177 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:48:36.177 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:48:36.177 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:48:36.177 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 102291 2023-09-23T20:48:36.178 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.178 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:48:36.178 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.178 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:48:36.178 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.213037+0000", 2023-09-23T20:48:36.178 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:48:36.178 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.178 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.178 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:48:36.178 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.213037+0000", 2023-09-23T20:48:36.178 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:48:36.179 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.179 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.179 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:48:36.179 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.213035+0000", 2023-09-23T20:48:36.179 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999981 2023-09-23T20:48:36.179 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.179 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.179 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:48:36.179 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.213078+0000", 2023-09-23T20:48:36.179 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.3106999999999999e-05 2023-09-23T20:48:36.179 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.180 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.180 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:48:36.180 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.213095+0000", 2023-09-23T20:48:36.180 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.6200000000000001e-05 2023-09-23T20:48:36.180 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.180 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.180 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:48:36.180 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.213102+0000", 2023-09-23T20:48:36.180 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.2779999999999996e-06 2023-09-23T20:48:36.180 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.180 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.181 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:48:36.181 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.213160+0000", 2023-09-23T20:48:36.181 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.838e-05 2023-09-23T20:48:36.181 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.181 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.181 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:48:36.181 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.213176+0000", 2023-09-23T20:48:36.181 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.5404999999999999e-05 2023-09-23T20:48:36.181 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.181 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.181 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:48:36.181 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.220512+0000", 2023-09-23T20:48:36.182 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.00733669 2023-09-23T20:48:36.182 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.182 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.182 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:48:36.182 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.220642+0000", 2023-09-23T20:48:36.182 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00012919 2023-09-23T20:48:36.182 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.182 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.182 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 1,4", 2023-09-23T20:48:36.182 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.220797+0000", 2023-09-23T20:48:36.182 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00015535599999999999 2023-09-23T20:48:36.183 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.183 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.183 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:48:36.183 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.309757+0000", 2023-09-23T20:48:36.183 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.088960045000000001 2023-09-23T20:48:36.183 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.183 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.183 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:48:36.183 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.309865+0000", 2023-09-23T20:48:36.183 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000108051 2023-09-23T20:48:36.183 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.184 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.184 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:48:36.184 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.310430+0000", 2023-09-23T20:48:36.184 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000565126 2023-09-23T20:48:36.184 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.184 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.184 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:48:36.184 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.310482+0000", 2023-09-23T20:48:36.184 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.1752999999999997e-05 2023-09-23T20:48:36.184 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.185 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.185 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:48:36.185 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.310715+0000", 2023-09-23T20:48:36.185 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00023280499999999999 2023-09-23T20:48:36.185 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:48:36.185 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:48:36.185 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:48:36.185 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.185 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.185 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:102300 10.d 10:b3326d1e:::benchmark_data_smithi018_422431_object102299:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e552)", 2023-09-23T20:48:36.185 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:47:08.260136+0000", 2023-09-23T20:48:36.186 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 87.847134971000003, 2023-09-23T20:48:36.186 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0960319730000001, 2023-09-23T20:48:36.186 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:48:36.186 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:48:36.186 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:48:36.186 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:48:36.186 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:48:36.186 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 102300 2023-09-23T20:48:36.186 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.186 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:48:36.187 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.187 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:48:36.187 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.260136+0000", 2023-09-23T20:48:36.187 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:48:36.187 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.187 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.187 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:48:36.187 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.260136+0000", 2023-09-23T20:48:36.187 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:48:36.187 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.187 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.188 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:48:36.188 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.260135+0000", 2023-09-23T20:48:36.188 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:48:36.188 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.188 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.188 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:48:36.188 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.260169+0000", 2023-09-23T20:48:36.188 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.4019000000000001e-05 2023-09-23T20:48:36.188 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.188 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.188 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:48:36.188 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.260183+0000", 2023-09-23T20:48:36.189 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.4387e-05 2023-09-23T20:48:36.189 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.189 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.189 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:48:36.189 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.260189+0000", 2023-09-23T20:48:36.189 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.0990000000000004e-06 2023-09-23T20:48:36.189 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.189 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.189 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:48:36.189 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.260275+0000", 2023-09-23T20:48:36.189 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.5556000000000006e-05 2023-09-23T20:48:36.189 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.190 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.190 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:48:36.190 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.260285+0000", 2023-09-23T20:48:36.190 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.9189999999999994e-06 2023-09-23T20:48:36.190 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.190 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.190 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:48:36.190 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.219112+0000", 2023-09-23T20:48:36.190 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.95882748600000001 2023-09-23T20:48:36.190 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.191 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.191 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:48:36.191 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.219285+0000", 2023-09-23T20:48:36.191 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000173299 2023-09-23T20:48:36.191 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.191 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.191 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,4", 2023-09-23T20:48:36.191 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.219498+0000", 2023-09-23T20:48:36.191 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00021284799999999999 2023-09-23T20:48:36.191 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.191 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.192 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:48:36.192 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.310145+0000", 2023-09-23T20:48:36.192 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.090646550000000006 2023-09-23T20:48:36.192 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.192 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.192 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:48:36.192 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.310184+0000", 2023-09-23T20:48:36.192 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.9232e-05 2023-09-23T20:48:36.192 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.192 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.192 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:48:36.193 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.355659+0000", 2023-09-23T20:48:36.193 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.045474674 2023-09-23T20:48:36.193 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.193 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.193 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:48:36.193 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.355721+0000", 2023-09-23T20:48:36.193 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.2157999999999997e-05 2023-09-23T20:48:36.193 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.193 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.193 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:48:36.193 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.356168+0000", 2023-09-23T20:48:36.194 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000447305 2023-09-23T20:48:36.194 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:48:36.194 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:48:36.194 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:48:36.194 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.194 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.194 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:103733 10.d 10:b457bb97:::benchmark_data_smithi018_422431_object103732:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e557)", 2023-09-23T20:48:36.194 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:47:12.654166+0000", 2023-09-23T20:48:36.194 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 83.453104762999999, 2023-09-23T20:48:36.194 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0141292129999999, 2023-09-23T20:48:36.194 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:48:36.194 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:48:36.195 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:48:36.195 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:48:36.195 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:48:36.195 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 103733 2023-09-23T20:48:36.195 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.195 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:48:36.195 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.195 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:48:36.195 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:12.654166+0000", 2023-09-23T20:48:36.195 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:48:36.195 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.196 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.196 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:48:36.196 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:12.654166+0000", 2023-09-23T20:48:36.196 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:48:36.196 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.196 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.196 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:48:36.196 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:12.654165+0000", 2023-09-23T20:48:36.196 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:48:36.196 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.196 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.197 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:48:36.197 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:12.654221+0000", 2023-09-23T20:48:36.197 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.6150999999999998e-05 2023-09-23T20:48:36.197 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.197 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.197 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:48:36.197 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:12.654236+0000", 2023-09-23T20:48:36.197 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.4783000000000001e-05 2023-09-23T20:48:36.197 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.197 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.197 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:48:36.198 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:12.654243+0000", 2023-09-23T20:48:36.198 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.8680000000000001e-06 2023-09-23T20:48:36.198 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.198 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.198 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:48:36.198 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:12.654316+0000", 2023-09-23T20:48:36.198 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.2848000000000005e-05 2023-09-23T20:48:36.198 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.198 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.198 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:48:36.198 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:12.654335+0000", 2023-09-23T20:48:36.199 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.9057999999999999e-05 2023-09-23T20:48:36.199 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.199 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.199 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:48:36.199 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:13.630346+0000", 2023-09-23T20:48:36.199 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.97601110099999999 2023-09-23T20:48:36.199 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.199 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.199 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:48:36.199 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:13.630458+0000", 2023-09-23T20:48:36.199 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00011244 2023-09-23T20:48:36.199 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.200 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.200 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,4", 2023-09-23T20:48:36.200 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:13.630554+0000", 2023-09-23T20:48:36.200 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.5334999999999994e-05 2023-09-23T20:48:36.200 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.200 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.200 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:48:36.200 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:13.642112+0000", 2023-09-23T20:48:36.200 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.011558034999999999 2023-09-23T20:48:36.200 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.200 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.200 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:48:36.201 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:13.668168+0000", 2023-09-23T20:48:36.201 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.026056691 2023-09-23T20:48:36.201 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.201 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.201 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:48:36.201 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:13.668225+0000", 2023-09-23T20:48:36.201 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.6598000000000003e-05 2023-09-23T20:48:36.201 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.201 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.201 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:48:36.201 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:13.668267+0000", 2023-09-23T20:48:36.201 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.2101000000000002e-05 2023-09-23T20:48:36.202 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.202 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.202 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:48:36.202 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:13.668296+0000", 2023-09-23T20:48:36.202 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.8530999999999998e-05 2023-09-23T20:48:36.202 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:48:36.202 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:48:36.202 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:48:36.202 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.202 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.202 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:159045 10.f 10:f817bccc:::benchmark_data_smithi018_422431_object54549:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e577)", 2023-09-23T20:48:36.203 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:48:22.940491+0000", 2023-09-23T20:48:36.203 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 13.166780304, 2023-09-23T20:48:36.203 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.257405952, 2023-09-23T20:48:36.203 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:48:36.203 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:48:36.203 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:48:36.203 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:48:36.203 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:48:36.203 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 159045 2023-09-23T20:48:36.203 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.203 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:48:36.203 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.204 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:48:36.204 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.940491+0000", 2023-09-23T20:48:36.204 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:48:36.204 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.204 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.204 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:48:36.204 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.940491+0000", 2023-09-23T20:48:36.204 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:48:36.204 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.204 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.204 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:48:36.205 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.940490+0000", 2023-09-23T20:48:36.205 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:48:36.205 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.205 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.205 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:48:36.205 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.940494+0000", 2023-09-23T20:48:36.205 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.9580000000000001e-06 2023-09-23T20:48:36.205 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.205 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.205 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:48:36.205 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.940505+0000", 2023-09-23T20:48:36.205 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0905e-05 2023-09-23T20:48:36.206 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.206 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.206 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:48:36.206 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.940510+0000", 2023-09-23T20:48:36.206 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.5879999999999999e-06 2023-09-23T20:48:36.206 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.206 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.206 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:48:36.206 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.940558+0000", 2023-09-23T20:48:36.206 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.8007000000000003e-05 2023-09-23T20:48:36.207 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.207 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.207 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:48:36.207 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.940564+0000", 2023-09-23T20:48:36.207 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.2099999999999998e-06 2023-09-23T20:48:36.207 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.207 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.207 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:48:36.207 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:23.909048+0000", 2023-09-23T20:48:36.207 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.96848468799999998 2023-09-23T20:48:36.207 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.207 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.208 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:48:36.208 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:23.909060+0000", 2023-09-23T20:48:36.208 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1279e-05 2023-09-23T20:48:36.208 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.208 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.208 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:48:36.208 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:25.160634+0000", 2023-09-23T20:48:36.208 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.251573933 2023-09-23T20:48:36.208 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.208 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.208 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:48:36.208 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:25.160777+0000", 2023-09-23T20:48:36.209 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000142893 2023-09-23T20:48:36.209 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.209 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.209 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 2,3,4", 2023-09-23T20:48:36.209 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:25.161399+0000", 2023-09-23T20:48:36.209 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00062231400000000001 2023-09-23T20:48:36.209 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.209 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.209 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:48:36.209 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:25.171100+0000", 2023-09-23T20:48:36.209 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0097009049999999993 2023-09-23T20:48:36.210 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.210 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.210 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:48:36.210 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:25.183171+0000", 2023-09-23T20:48:36.210 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.012071236000000001 2023-09-23T20:48:36.210 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.210 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.210 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:48:36.210 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:25.185581+0000", 2023-09-23T20:48:36.210 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0024096899999999999 2023-09-23T20:48:36.210 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.211 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.211 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:48:36.211 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:25.197727+0000", 2023-09-23T20:48:36.211 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.012145958 2023-09-23T20:48:36.211 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.211 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.211 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:48:36.211 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:25.197780+0000", 2023-09-23T20:48:36.211 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.3816000000000001e-05 2023-09-23T20:48:36.211 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.211 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.212 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:48:36.212 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:25.197897+0000", 2023-09-23T20:48:36.212 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00011640400000000001 2023-09-23T20:48:36.212 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:48:36.212 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:48:36.212 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:48:36.212 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.212 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.212 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:164265 10.f 10:f6c25819:::benchmark_data_smithi018_422431_object59769:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e582)", 2023-09-23T20:48:36.212 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:48:32.012932+0000", 2023-09-23T20:48:36.212 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 4.0943389379999999, 2023-09-23T20:48:36.213 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.036838609000000001, 2023-09-23T20:48:36.213 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:48:36.213 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:48:36.213 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:48:36.213 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:48:36.213 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:48:36.213 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 164265 2023-09-23T20:48:36.213 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.213 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:48:36.213 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.213 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:48:36.214 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:32.012932+0000", 2023-09-23T20:48:36.214 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:48:36.214 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.214 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.214 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:48:36.214 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:32.012932+0000", 2023-09-23T20:48:36.214 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:48:36.214 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.214 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.214 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:48:36.214 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:32.012931+0000", 2023-09-23T20:48:36.215 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:48:36.215 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.215 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.215 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:48:36.215 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:32.012938+0000", 2023-09-23T20:48:36.215 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.0369999999999997e-06 2023-09-23T20:48:36.215 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.215 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.215 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:48:36.215 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:32.012953+0000", 2023-09-23T20:48:36.215 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.5248e-05 2023-09-23T20:48:36.216 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.216 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.216 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:48:36.216 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:32.012960+0000", 2023-09-23T20:48:36.216 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.6470000000000001e-06 2023-09-23T20:48:36.216 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.216 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.216 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:48:36.216 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:32.020785+0000", 2023-09-23T20:48:36.216 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0078255819999999993 2023-09-23T20:48:36.216 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.216 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.217 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:48:36.217 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:32.020897+0000", 2023-09-23T20:48:36.217 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00011196000000000001 2023-09-23T20:48:36.217 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.217 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.217 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 2,3,4", 2023-09-23T20:48:36.217 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:32.020995+0000", 2023-09-23T20:48:36.217 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.8138000000000001e-05 2023-09-23T20:48:36.217 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.217 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.217 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:48:36.218 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:32.027953+0000", 2023-09-23T20:48:36.218 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0069577659999999998 2023-09-23T20:48:36.218 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.218 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.218 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:48:36.218 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:32.027962+0000", 2023-09-23T20:48:36.218 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.6349999999999995e-06 2023-09-23T20:48:36.218 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.218 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.218 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:48:36.219 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:32.028002+0000", 2023-09-23T20:48:36.219 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.0342000000000001e-05 2023-09-23T20:48:36.219 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.219 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.219 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:48:36.219 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:32.049605+0000", 2023-09-23T20:48:36.219 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.021602816 2023-09-23T20:48:36.219 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.219 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.219 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:48:36.219 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:32.049657+0000", 2023-09-23T20:48:36.220 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.2215000000000001e-05 2023-09-23T20:48:36.220 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.220 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.220 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:48:36.220 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:32.049771+0000", 2023-09-23T20:48:36.220 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000113779 2023-09-23T20:48:36.220 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:48:36.220 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:48:36.220 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:48:36.220 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.220 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.220 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:165091 10.f 10:f4167575:::benchmark_data_smithi018_422431_object60595:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e582)", 2023-09-23T20:48:36.221 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:48:33.017037+0000", 2023-09-23T20:48:36.221 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 3.0902347130000001, 2023-09-23T20:48:36.221 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.035848087000000001, 2023-09-23T20:48:36.221 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:48:36.221 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:48:36.221 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:48:36.221 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:48:36.221 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:48:36.221 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 165091 2023-09-23T20:48:36.221 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.221 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:48:36.221 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.222 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:48:36.222 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:33.017037+0000", 2023-09-23T20:48:36.222 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:48:36.222 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.222 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.222 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:48:36.222 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:33.017037+0000", 2023-09-23T20:48:36.222 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:48:36.222 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.222 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.222 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:48:36.223 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:33.017035+0000", 2023-09-23T20:48:36.223 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:48:36.223 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.223 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.223 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:48:36.223 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:33.017042+0000", 2023-09-23T20:48:36.223 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.6320000000000002e-06 2023-09-23T20:48:36.223 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.223 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.223 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:48:36.223 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:33.017052+0000", 2023-09-23T20:48:36.223 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0628000000000001e-05 2023-09-23T20:48:36.224 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.224 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.224 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:48:36.224 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:33.017058+0000", 2023-09-23T20:48:36.224 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.348e-06 2023-09-23T20:48:36.224 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.224 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.224 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:48:36.224 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:33.028275+0000", 2023-09-23T20:48:36.224 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.011216152 2023-09-23T20:48:36.224 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.224 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.225 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:48:36.225 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:33.028371+0000", 2023-09-23T20:48:36.225 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.6156999999999994e-05 2023-09-23T20:48:36.225 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.225 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.225 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 2,3,4", 2023-09-23T20:48:36.225 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:33.028502+0000", 2023-09-23T20:48:36.225 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000131361 2023-09-23T20:48:36.225 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.225 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.225 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:48:36.225 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:33.032437+0000", 2023-09-23T20:48:36.226 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0039350440000000004 2023-09-23T20:48:36.226 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.226 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.226 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:48:36.226 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:33.035353+0000", 2023-09-23T20:48:36.226 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.002915571 2023-09-23T20:48:36.226 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.226 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.226 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:48:36.226 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:33.045366+0000", 2023-09-23T20:48:36.226 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.010013721999999999 2023-09-23T20:48:36.227 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.227 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.227 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:48:36.227 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:33.052744+0000", 2023-09-23T20:48:36.227 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0073775530000000002 2023-09-23T20:48:36.227 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.227 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.227 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:48:36.227 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:33.052783+0000", 2023-09-23T20:48:36.227 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.8871000000000001e-05 2023-09-23T20:48:36.227 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.227 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.228 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:48:36.228 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:33.052885+0000", 2023-09-23T20:48:36.228 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00010170699999999999 2023-09-23T20:48:36.228 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:48:36.228 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:48:36.228 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:48:36.228 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.228 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.228 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:167038 10.0 10:0851d6c1:::benchmark_data_smithi018_422431_object62542:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e582)", 2023-09-23T20:48:36.228 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:48:35.337425+0000", 2023-09-23T20:48:36.228 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 0.76984642999999997, 2023-09-23T20:48:36.228 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.042637604000000003, 2023-09-23T20:48:36.229 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:48:36.229 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:48:36.229 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:48:36.229 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:48:36.229 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:48:36.229 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 167038 2023-09-23T20:48:36.229 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.229 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:48:36.229 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.229 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:48:36.229 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:35.337425+0000", 2023-09-23T20:48:36.230 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:48:36.230 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.230 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.230 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:48:36.230 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:35.337425+0000", 2023-09-23T20:48:36.230 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:48:36.230 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.230 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.230 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:48:36.230 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:35.337425+0000", 2023-09-23T20:48:36.230 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:48:36.231 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.231 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.231 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:48:36.231 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:35.337427+0000", 2023-09-23T20:48:36.231 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.0980000000000001e-06 2023-09-23T20:48:36.231 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.231 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.231 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:48:36.231 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:35.337432+0000", 2023-09-23T20:48:36.231 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.9030000000000003e-06 2023-09-23T20:48:36.231 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.231 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.232 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:48:36.232 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:35.337435+0000", 2023-09-23T20:48:36.232 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.3699999999999999e-06 2023-09-23T20:48:36.232 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.232 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.232 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:48:36.232 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:35.344576+0000", 2023-09-23T20:48:36.232 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0071412130000000004 2023-09-23T20:48:36.232 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.232 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.232 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:48:36.233 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:35.344702+0000", 2023-09-23T20:48:36.233 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00012572399999999999 2023-09-23T20:48:36.233 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.233 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.233 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 1,4", 2023-09-23T20:48:36.233 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:35.344828+0000", 2023-09-23T20:48:36.233 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00012587199999999999 2023-09-23T20:48:36.233 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.233 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.233 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:48:36.233 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:35.351734+0000", 2023-09-23T20:48:36.233 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0069060220000000004 2023-09-23T20:48:36.234 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.234 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.234 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:48:36.234 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:35.352947+0000", 2023-09-23T20:48:36.234 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.001212774 2023-09-23T20:48:36.234 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.234 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.234 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:48:36.234 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:35.379959+0000", 2023-09-23T20:48:36.234 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.027012323000000001 2023-09-23T20:48:36.234 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.235 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.235 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:48:36.235 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:35.380013+0000", 2023-09-23T20:48:36.235 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.4543999999999998e-05 2023-09-23T20:48:36.235 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.235 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.235 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:48:36.235 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:35.380062+0000", 2023-09-23T20:48:36.235 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.9032000000000001e-05 2023-09-23T20:48:36.235 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:48:36.235 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:48:36.235 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:48:36.236 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.236 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.236 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:167040 10.0 10:0c42fb03:::benchmark_data_smithi018_422431_object62544:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e582)", 2023-09-23T20:48:36.236 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:48:35.340654+0000", 2023-09-23T20:48:36.236 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 0.76661757500000005, 2023-09-23T20:48:36.236 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.039517670999999997, 2023-09-23T20:48:36.236 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:48:36.236 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:48:36.236 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:48:36.236 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:48:36.236 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:48:36.236 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 167040 2023-09-23T20:48:36.237 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.237 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:48:36.237 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.237 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:48:36.237 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:35.340654+0000", 2023-09-23T20:48:36.237 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:48:36.237 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.237 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.237 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:48:36.237 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:35.340654+0000", 2023-09-23T20:48:36.237 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:48:36.238 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.238 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.238 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:48:36.238 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:35.340652+0000", 2023-09-23T20:48:36.238 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:48:36.238 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.238 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.238 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:48:36.238 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:35.340659+0000", 2023-09-23T20:48:36.238 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.3400000000000003e-06 2023-09-23T20:48:36.238 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.239 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.239 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:48:36.239 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:35.340671+0000", 2023-09-23T20:48:36.239 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.2033e-05 2023-09-23T20:48:36.239 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.239 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.239 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:48:36.239 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:35.340677+0000", 2023-09-23T20:48:36.239 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.4659999999999996e-06 2023-09-23T20:48:36.239 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.239 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.239 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:48:36.240 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:35.348470+0000", 2023-09-23T20:48:36.240 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0077928839999999999 2023-09-23T20:48:36.240 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.240 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.240 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:48:36.240 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:35.348581+0000", 2023-09-23T20:48:36.240 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00011101 2023-09-23T20:48:36.240 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.240 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.240 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 1,4", 2023-09-23T20:48:36.240 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:35.348689+0000", 2023-09-23T20:48:36.241 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000108321 2023-09-23T20:48:36.241 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.241 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.241 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:48:36.241 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:35.352962+0000", 2023-09-23T20:48:36.241 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0042722230000000003 2023-09-23T20:48:36.241 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.241 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.241 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:48:36.241 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:35.356988+0000", 2023-09-23T20:48:36.241 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0040263870000000002 2023-09-23T20:48:36.241 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.242 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.242 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:48:36.242 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:35.380096+0000", 2023-09-23T20:48:36.242 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.023108312999999998 2023-09-23T20:48:36.242 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.242 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.242 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:48:36.242 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:35.380138+0000", 2023-09-23T20:48:36.242 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.1820000000000003e-05 2023-09-23T20:48:36.242 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.242 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.243 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:48:36.243 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:35.380171+0000", 2023-09-23T20:48:36.243 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.3284000000000002e-05 2023-09-23T20:48:36.243 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:48:36.243 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:48:36.243 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:48:36.243 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.243 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.243 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:167285 10.f 10:f51b84d4:::benchmark_data_smithi018_422431_object62789:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e582)", 2023-09-23T20:48:36.243 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:48:35.630418+0000", 2023-09-23T20:48:36.243 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 0.476852954, 2023-09-23T20:48:36.243 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.032801882999999997, 2023-09-23T20:48:36.244 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:48:36.244 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:48:36.244 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:48:36.244 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:48:36.244 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:48:36.244 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 167285 2023-09-23T20:48:36.244 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.244 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:48:36.244 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.244 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:48:36.244 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:35.630418+0000", 2023-09-23T20:48:36.245 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:48:36.245 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.245 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.245 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:48:36.245 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:35.630418+0000", 2023-09-23T20:48:36.245 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:48:36.245 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.245 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.245 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:48:36.245 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:35.630418+0000", 2023-09-23T20:48:36.245 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:48:36.246 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.246 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.246 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:48:36.246 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:35.630421+0000", 2023-09-23T20:48:36.246 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.8559999999999998e-06 2023-09-23T20:48:36.246 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.246 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.246 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:48:36.246 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:35.630427+0000", 2023-09-23T20:48:36.246 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.7299999999999999e-06 2023-09-23T20:48:36.246 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.247 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.247 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:48:36.247 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:35.630432+0000", 2023-09-23T20:48:36.247 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.2699999999999998e-06 2023-09-23T20:48:36.247 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.247 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.247 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:48:36.247 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:35.637788+0000", 2023-09-23T20:48:36.247 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0073561929999999996 2023-09-23T20:48:36.247 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.247 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.248 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:48:36.248 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:35.637907+0000", 2023-09-23T20:48:36.248 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000119255 2023-09-23T20:48:36.248 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.248 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.248 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 2,3,4", 2023-09-23T20:48:36.248 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:35.637995+0000", 2023-09-23T20:48:36.248 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.8279e-05 2023-09-23T20:48:36.248 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.248 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.248 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:48:36.249 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:35.651192+0000", 2023-09-23T20:48:36.249 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.013196593 2023-09-23T20:48:36.249 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.249 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.249 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:48:36.249 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:35.651258+0000", 2023-09-23T20:48:36.249 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.5544999999999997e-05 2023-09-23T20:48:36.249 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.249 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.249 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:48:36.249 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:35.656438+0000", 2023-09-23T20:48:36.250 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0051800570000000001 2023-09-23T20:48:36.250 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.250 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.250 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:48:36.250 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:35.663074+0000", 2023-09-23T20:48:36.250 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0066358429999999998 2023-09-23T20:48:36.250 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.250 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.250 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:48:36.250 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:35.663115+0000", 2023-09-23T20:48:36.250 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.1557999999999998e-05 2023-09-23T20:48:36.251 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.251 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.251 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:48:36.251 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:35.663220+0000", 2023-09-23T20:48:36.251 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000105104 2023-09-23T20:48:36.251 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:48:36.251 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:48:36.251 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:48:36.251 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.251 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.251 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:167287 10.f 10:f2ea3d93:::benchmark_data_smithi018_422431_object62791:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e582)", 2023-09-23T20:48:36.251 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:48:35.630461+0000", 2023-09-23T20:48:36.252 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 0.47680998400000002, 2023-09-23T20:48:36.252 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.032949047000000002, 2023-09-23T20:48:36.252 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:48:36.252 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:48:36.252 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:48:36.252 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:48:36.252 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:48:36.252 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 167287 2023-09-23T20:48:36.252 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.252 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:48:36.252 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.252 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:48:36.253 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:35.630461+0000", 2023-09-23T20:48:36.253 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:48:36.253 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.253 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.253 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:48:36.253 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:35.630461+0000", 2023-09-23T20:48:36.253 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:48:36.253 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.253 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.253 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:48:36.253 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:35.630461+0000", 2023-09-23T20:48:36.254 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:48:36.254 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.254 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.254 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:48:36.254 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:35.630463+0000", 2023-09-23T20:48:36.254 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.4839999999999998e-06 2023-09-23T20:48:36.254 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.254 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.254 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:48:36.254 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:35.630503+0000", 2023-09-23T20:48:36.254 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.9876000000000002e-05 2023-09-23T20:48:36.254 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.255 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.255 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:48:36.255 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:35.630510+0000", 2023-09-23T20:48:36.255 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.5749999999999997e-06 2023-09-23T20:48:36.255 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.255 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.255 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:48:36.255 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:35.641059+0000", 2023-09-23T20:48:36.255 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.010549088 2023-09-23T20:48:36.255 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.255 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.256 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:48:36.256 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:35.641208+0000", 2023-09-23T20:48:36.256 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00014955799999999999 2023-09-23T20:48:36.256 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.256 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.256 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 2,3,4", 2023-09-23T20:48:36.256 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:35.641335+0000", 2023-09-23T20:48:36.256 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00012689799999999999 2023-09-23T20:48:36.256 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.256 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.256 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:48:36.256 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:35.651290+0000", 2023-09-23T20:48:36.257 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0099543210000000003 2023-09-23T20:48:36.257 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.257 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.257 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:48:36.257 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:35.651330+0000", 2023-09-23T20:48:36.257 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.0806999999999997e-05 2023-09-23T20:48:36.257 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.257 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.257 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:48:36.257 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:35.656449+0000", 2023-09-23T20:48:36.257 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.005118938 2023-09-23T20:48:36.258 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.258 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.258 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:48:36.258 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:35.663253+0000", 2023-09-23T20:48:36.258 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0068032709999999996 2023-09-23T20:48:36.258 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.258 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.258 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:48:36.258 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:35.663298+0000", 2023-09-23T20:48:36.258 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.5670000000000002e-05 2023-09-23T20:48:36.258 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.259 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.259 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:48:36.259 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:35.663410+0000", 2023-09-23T20:48:36.259 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00011196699999999999 2023-09-23T20:48:36.259 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:48:36.259 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:48:36.259 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:48:36.259 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.259 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.259 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:167290 10.d 10:b89802dd:::benchmark_data_smithi018_422431_object62794:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e582)", 2023-09-23T20:48:36.259 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:48:35.630545+0000", 2023-09-23T20:48:36.259 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 0.47672611999999998, 2023-09-23T20:48:36.260 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.035670945000000003, 2023-09-23T20:48:36.260 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:48:36.260 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:48:36.260 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:48:36.260 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:48:36.260 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:48:36.260 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 167290 2023-09-23T20:48:36.260 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.260 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:48:36.260 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.260 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:48:36.261 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:35.630545+0000", 2023-09-23T20:48:36.261 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:48:36.261 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.261 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.261 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:48:36.261 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:35.630545+0000", 2023-09-23T20:48:36.261 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:48:36.261 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.261 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.261 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:48:36.262 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:35.630545+0000", 2023-09-23T20:48:36.262 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:48:36.262 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.262 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.262 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:48:36.262 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:35.630548+0000", 2023-09-23T20:48:36.262 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.0130000000000002e-06 2023-09-23T20:48:36.262 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.262 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.262 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:48:36.262 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:35.630560+0000", 2023-09-23T20:48:36.262 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1976e-05 2023-09-23T20:48:36.263 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.263 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.263 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:48:36.263 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:35.630564+0000", 2023-09-23T20:48:36.263 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.1099999999999996e-06 2023-09-23T20:48:36.263 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.263 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.263 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:48:36.263 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:35.633874+0000", 2023-09-23T20:48:36.263 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0033098440000000002 2023-09-23T20:48:36.265 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.265 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.265 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:48:36.265 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:35.634005+0000", 2023-09-23T20:48:36.265 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00013179000000000001 2023-09-23T20:48:36.265 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.265 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.265 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,4", 2023-09-23T20:48:36.265 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:35.634131+0000", 2023-09-23T20:48:36.266 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000125814 2023-09-23T20:48:36.266 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.266 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.266 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:48:36.266 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:35.644801+0000", 2023-09-23T20:48:36.266 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.010670219 2023-09-23T20:48:36.266 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.266 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.266 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:48:36.266 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:35.656803+0000", 2023-09-23T20:48:36.266 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.012001744 2023-09-23T20:48:36.267 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.267 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.267 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:48:36.267 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:35.666125+0000", 2023-09-23T20:48:36.267 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0093223349999999993 2023-09-23T20:48:36.267 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.267 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.267 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:48:36.267 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:35.666180+0000", 2023-09-23T20:48:36.267 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.4478000000000003e-05 2023-09-23T20:48:36.267 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.267 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.268 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:48:36.268 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:35.666216+0000", 2023-09-23T20:48:36.268 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.6192000000000002e-05 2023-09-23T20:48:36.268 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:48:36.268 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:48:36.268 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:48:36.268 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.268 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.268 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:167294 10.d 10:b7f19f64:::benchmark_data_smithi018_422431_object62798:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e582)", 2023-09-23T20:48:36.268 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:48:35.650750+0000", 2023-09-23T20:48:36.268 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 0.45652077899999999, 2023-09-23T20:48:36.269 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.033751481999999999, 2023-09-23T20:48:36.269 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:48:36.269 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:48:36.269 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:48:36.269 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:48:36.269 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:48:36.269 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 167294 2023-09-23T20:48:36.269 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.269 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:48:36.269 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.269 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:48:36.270 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:35.650750+0000", 2023-09-23T20:48:36.270 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:48:36.270 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.270 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.270 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:48:36.270 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:35.650750+0000", 2023-09-23T20:48:36.270 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:48:36.270 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.270 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.270 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:48:36.271 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:35.650748+0000", 2023-09-23T20:48:36.271 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999976 2023-09-23T20:48:36.271 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.271 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.271 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:48:36.271 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:35.650757+0000", 2023-09-23T20:48:36.271 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.4859999999999997e-06 2023-09-23T20:48:36.271 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.271 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.271 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:48:36.271 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:35.650783+0000", 2023-09-23T20:48:36.271 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.5825000000000001e-05 2023-09-23T20:48:36.272 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.272 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.272 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:48:36.272 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:35.650793+0000", 2023-09-23T20:48:36.272 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0020000000000001e-05 2023-09-23T20:48:36.272 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.272 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.272 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:48:36.272 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:35.653514+0000", 2023-09-23T20:48:36.272 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0027212310000000002 2023-09-23T20:48:36.272 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.273 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.273 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:48:36.273 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:35.653631+0000", 2023-09-23T20:48:36.273 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00011723 2023-09-23T20:48:36.273 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.273 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.273 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,4", 2023-09-23T20:48:36.273 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:35.653742+0000", 2023-09-23T20:48:36.273 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000110911 2023-09-23T20:48:36.273 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.273 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.273 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:48:36.274 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:35.660056+0000", 2023-09-23T20:48:36.274 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0063143519999999996 2023-09-23T20:48:36.274 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.274 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.274 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:48:36.274 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:35.662449+0000", 2023-09-23T20:48:36.274 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0023922909999999999 2023-09-23T20:48:36.274 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.274 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.274 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:48:36.274 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:35.684440+0000", 2023-09-23T20:48:36.275 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.021991008999999999 2023-09-23T20:48:36.275 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.275 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.275 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:48:36.275 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:35.684477+0000", 2023-09-23T20:48:36.275 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.7639000000000003e-05 2023-09-23T20:48:36.275 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:48:36.275 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:48:36.275 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:48:36.275 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:35.684502+0000", 2023-09-23T20:48:36.275 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.4752999999999998e-05 2023-09-23T20:48:36.275 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:48:36.276 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:48:36.276 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:48:36.276 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:48:36.276 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:48:36.276 INFO:teuthology.orchestra.run.smithi149.stdout:} 2023-09-23T20:48:36.277 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:36 smithi018 ceph-mon[184144]: pgmap v1472: 119 pgs: 2 active+remapped+backfilling, 4 active+remapped+backfill_wait, 113 active+clean; 2.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 627 op/s; 15557/136290 objects misplaced (11.415%); 12 MiB/s, 195 objects/s recovering 2023-09-23T20:48:36.277 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:36 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:48:36.277 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:36.277 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:36 smithi018 ceph-mon[179647]: pgmap v1472: 119 pgs: 2 active+remapped+backfilling, 4 active+remapped+backfill_wait, 113 active+clean; 2.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 627 op/s; 15557/136290 objects misplaced (11.415%); 12 MiB/s, 195 objects/s recovering 2023-09-23T20:48:36.277 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:36 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:48:36.297 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:36 smithi149 ceph-mon[170394]: pgmap v1472: 119 pgs: 2 active+remapped+backfilling, 4 active+remapped+backfill_wait, 113 active+clean; 2.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 627 op/s; 15557/136290 objects misplaced (11.415%); 12 MiB/s, 195 objects/s recovering 2023-09-23T20:48:36.297 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:36 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:48:36.528 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:48:36.544 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:36.544 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:48:36.613 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T20:48:36.645 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:48:36.718 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:36.783 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:48:36.789 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:36.790 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:48:36.891 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:48:36.941 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:37.031 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:48:37.040 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:37 smithi018 ceph-mon[184144]: 1.8 scrub starts 2023-09-23T20:48:37.041 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:37.041 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:37 smithi018 ceph-mon[179647]: 1.8 scrub starts 2023-09-23T20:48:37.042 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:48:37.143 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:48:37.210 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:37.289 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:48:37.296 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:37.296 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:48:37.297 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:37 smithi149 ceph-mon[170394]: 1.8 scrub starts 2023-09-23T20:48:37.398 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:48:37.454 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:48:37.461 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:37.538 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:48:37.555 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:37.556 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:48:37.656 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:48:37.724 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:37.828 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:48:37.837 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:37.837 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:48:37.938 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:48:37.996 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:38.046 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:38 smithi018 ceph-mon[179647]: pgmap v1473: 119 pgs: 1 active+clean+scrubbing, 2 active+remapped+backfilling, 4 active+remapped+backfill_wait, 112 active+clean; 2.6 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 732 op/s; 14400/128631 objects misplaced (11.195%); 12 MiB/s, 194 objects/s recovering 2023-09-23T20:48:38.046 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:38 smithi018 ceph-mon[179647]: 1.8 scrub starts 2023-09-23T20:48:38.046 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:38 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/850388878' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T20:48:38.046 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:38 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T20:48:38.055 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:38 smithi018 ceph-mon[184144]: pgmap v1473: 119 pgs: 1 active+clean+scrubbing, 2 active+remapped+backfilling, 4 active+remapped+backfill_wait, 112 active+clean; 2.6 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 732 op/s; 14400/128631 objects misplaced (11.195%); 12 MiB/s, 194 objects/s recovering 2023-09-23T20:48:38.055 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:38 smithi018 ceph-mon[184144]: 1.8 scrub starts 2023-09-23T20:48:38.055 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:38 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/850388878' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T20:48:38.055 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:38 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T20:48:38.064 INFO:teuthology.orchestra.run.smithi018.stderr:set pool 10 pg_num to 26 2023-09-23T20:48:38.084 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:48:38.088 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:38.089 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:48:38.190 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:48:38.236 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:38.384 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:48:38.389 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:38 smithi149 ceph-mon[170394]: pgmap v1473: 119 pgs: 1 active+clean+scrubbing, 2 active+remapped+backfilling, 4 active+remapped+backfill_wait, 112 active+clean; 2.6 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 732 op/s; 14400/128631 objects misplaced (11.195%); 12 MiB/s, 194 objects/s recovering 2023-09-23T20:48:38.389 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:38 smithi149 ceph-mon[170394]: 1.8 scrub starts 2023-09-23T20:48:38.389 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:38 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/850388878' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T20:48:38.389 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:38 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T20:48:38.390 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:38.390 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:48:38.469 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:48:38.469 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:48:38.491 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:48:38.494 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:48:38.527 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:48:38.552 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:38.584 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:48:38.607 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:48:38.619 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:38.619 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:48:38.654 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:48:38.693 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:48:38.721 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:48:38.724 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:48:38.727 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:48:38.727 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:48:38.727 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:48:38.728 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:48:38.728 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:48:38.728 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:48:38.728 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:48:38.728 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:48:38.792 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:39.061 INFO:teuthology.orchestra.run.smithi018.stderr:nodeep-scrub is unset 2023-09-23T20:48:39.081 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:48:39.086 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:39.087 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:48:39.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:39 smithi018 ceph-mon[184144]: 1.8 scrub starts 2023-09-23T20:48:39.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:39 smithi018 ceph-mon[184144]: failed to encode map e583 with expected crc 2023-09-23T20:48:39.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:39 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"}]': finished 2023-09-23T20:48:39.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:39 smithi018 ceph-mon[184144]: osdmap e583: 8 total, 8 up, 7 in 2023-09-23T20:48:39.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:39 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-09-23T20:48:39.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:39 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/388905003' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:48:39.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:39 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3698065936' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:48:39.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:39 smithi018 ceph-mon[179647]: 1.8 scrub starts 2023-09-23T20:48:39.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:39 smithi018 ceph-mon[179647]: failed to encode map e583 with expected crc 2023-09-23T20:48:39.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:39 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"}]': finished 2023-09-23T20:48:39.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:39 smithi018 ceph-mon[179647]: osdmap e583: 8 total, 8 up, 7 in 2023-09-23T20:48:39.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:39 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-09-23T20:48:39.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:39 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/388905003' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:48:39.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:39 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3698065936' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:48:39.188 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:48:39.221 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:39 smithi149 ceph-mon[170394]: 1.8 scrub starts 2023-09-23T20:48:39.221 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:39 smithi149 ceph-mon[170394]: failed to encode map e583 with expected crc 2023-09-23T20:48:39.221 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:39 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num", "val": "26"}]': finished 2023-09-23T20:48:39.221 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:39 smithi149 ceph-mon[170394]: osdmap e583: 8 total, 8 up, 7 in 2023-09-23T20:48:39.222 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:39 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-09-23T20:48:39.222 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:39 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/388905003' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:48:39.222 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:39 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3698065936' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:48:39.228 INFO:teuthology.orchestra.run.smithi149.stdout:ERROR: (22) Invalid argument 2023-09-23T20:48:39.229 INFO:teuthology.orchestra.run.smithi149.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:48:39.436 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:39.524 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:48:39.527 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:39.528 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:48:39.629 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:48:39.663 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:40.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:40 smithi018 ceph-mon[184144]: pgmap v1475: 119 pgs: 1 active+clean+scrubbing, 2 active+remapped+backfilling, 4 active+remapped+backfill_wait, 112 active+clean; 2.6 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 919 op/s; 13398/125550 objects misplaced (10.671%); 17 MiB/s, 273 objects/s recovering 2023-09-23T20:48:40.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:40 smithi018 ceph-mon[184144]: 1.8 scrub starts 2023-09-23T20:48:40.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:40 smithi018 ceph-mon[184144]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:48:40.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:40 smithi018 ceph-mon[184144]: Cluster is now healthy 2023-09-23T20:48:40.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:40 smithi018 ceph-mon[184144]: failed to encode map e584 with expected crc 2023-09-23T20:48:40.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:40 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"}]': finished 2023-09-23T20:48:40.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:40 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3698065936' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:48:40.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:40 smithi018 ceph-mon[184144]: osdmap e584: 8 total, 8 up, 7 in 2023-09-23T20:48:40.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:40 smithi018 ceph-mon[179647]: pgmap v1475: 119 pgs: 1 active+clean+scrubbing, 2 active+remapped+backfilling, 4 active+remapped+backfill_wait, 112 active+clean; 2.6 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 919 op/s; 13398/125550 objects misplaced (10.671%); 17 MiB/s, 273 objects/s recovering 2023-09-23T20:48:40.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:40 smithi018 ceph-mon[179647]: 1.8 scrub starts 2023-09-23T20:48:40.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:40 smithi018 ceph-mon[179647]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:48:40.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:40 smithi018 ceph-mon[179647]: Cluster is now healthy 2023-09-23T20:48:40.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:40 smithi018 ceph-mon[179647]: failed to encode map e584 with expected crc 2023-09-23T20:48:40.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:40 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"}]': finished 2023-09-23T20:48:40.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:40 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3698065936' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:48:40.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:40 smithi018 ceph-mon[179647]: osdmap e584: 8 total, 8 up, 7 in 2023-09-23T20:48:40.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:40 smithi149 ceph-mon[170394]: pgmap v1475: 119 pgs: 1 active+clean+scrubbing, 2 active+remapped+backfilling, 4 active+remapped+backfill_wait, 112 active+clean; 2.6 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 919 op/s; 13398/125550 objects misplaced (10.671%); 17 MiB/s, 273 objects/s recovering 2023-09-23T20:48:40.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:40 smithi149 ceph-mon[170394]: 1.8 scrub starts 2023-09-23T20:48:40.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:40 smithi149 ceph-mon[170394]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:48:40.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:40 smithi149 ceph-mon[170394]: Cluster is now healthy 2023-09-23T20:48:40.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:40 smithi149 ceph-mon[170394]: failed to encode map e584 with expected crc 2023-09-23T20:48:40.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:40 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pg_num_actual", "val": "26"}]': finished 2023-09-23T20:48:40.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:40 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3698065936' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:48:40.468 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:40 smithi149 ceph-mon[170394]: osdmap e584: 8 total, 8 up, 7 in 2023-09-23T20:48:41.178 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:48:41.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:41 smithi018 ceph-mon[184144]: 10.3 continuing backfill to osd.4 from (536'1300,583'10826] MIN to 583'10826 2023-09-23T20:48:41.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:41 smithi018 ceph-mon[184144]: failed to encode map e585 with expected crc 2023-09-23T20:48:41.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:41 smithi018 ceph-mon[184144]: failed to encode map e585 with expected crc 2023-09-23T20:48:41.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:41 smithi018 ceph-mon[184144]: failed to encode map e585 with expected crc 2023-09-23T20:48:41.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:41 smithi018 ceph-mon[184144]: osdmap e585: 8 total, 8 up, 7 in 2023-09-23T20:48:41.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:41 smithi018 ceph-mon[184144]: failed to encode map e585 with expected crc 2023-09-23T20:48:41.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:41 smithi018 ceph-mon[184144]: 10.8 continuing backfill to osd.5 from (541'4571,583'10505] MIN to 583'10505 2023-09-23T20:48:41.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:41 smithi018 ceph-mon[184144]: 10.5 continuing backfill to osd.7 from (542'4806,583'10701] MIN to 583'10701 2023-09-23T20:48:41.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:41 smithi018 ceph-mon[184144]: failed to encode map e585 with expected crc 2023-09-23T20:48:41.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:41 smithi018 ceph-mon[184144]: failed to encode map e585 with expected crc 2023-09-23T20:48:41.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:41 smithi018 ceph-mon[184144]: failed to encode map e585 with expected crc 2023-09-23T20:48:41.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:41 smithi018 ceph-mon[184144]: failed to encode map e585 with expected crc 2023-09-23T20:48:41.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:41 smithi018 ceph-mon[184144]: failed to encode map e585 with expected crc 2023-09-23T20:48:41.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:41 smithi018 ceph-mon[184144]: failed to encode map e585 with expected crc 2023-09-23T20:48:41.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:41 smithi018 ceph-mon[184144]: 10.9 continuing backfill to osd.4 from (541'4623,583'10757] MIN to 583'10757 2023-09-23T20:48:41.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:41 smithi018 ceph-mon[184144]: failed to encode map e585 with expected crc 2023-09-23T20:48:41.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:41 smithi018 ceph-mon[179647]: 10.3 continuing backfill to osd.4 from (536'1300,583'10826] MIN to 583'10826 2023-09-23T20:48:41.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:41 smithi018 ceph-mon[179647]: failed to encode map e585 with expected crc 2023-09-23T20:48:41.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:41 smithi018 ceph-mon[179647]: failed to encode map e585 with expected crc 2023-09-23T20:48:41.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:41 smithi018 ceph-mon[179647]: failed to encode map e585 with expected crc 2023-09-23T20:48:41.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:41 smithi018 ceph-mon[179647]: osdmap e585: 8 total, 8 up, 7 in 2023-09-23T20:48:41.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:41 smithi018 ceph-mon[179647]: failed to encode map e585 with expected crc 2023-09-23T20:48:41.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:41 smithi018 ceph-mon[179647]: 10.8 continuing backfill to osd.5 from (541'4571,583'10505] MIN to 583'10505 2023-09-23T20:48:41.394 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:41 smithi018 ceph-mon[179647]: 10.5 continuing backfill to osd.7 from (542'4806,583'10701] MIN to 583'10701 2023-09-23T20:48:41.394 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:41 smithi018 ceph-mon[179647]: failed to encode map e585 with expected crc 2023-09-23T20:48:41.394 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:41 smithi018 ceph-mon[179647]: failed to encode map e585 with expected crc 2023-09-23T20:48:41.394 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:41 smithi018 ceph-mon[179647]: failed to encode map e585 with expected crc 2023-09-23T20:48:41.394 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:41 smithi018 ceph-mon[179647]: failed to encode map e585 with expected crc 2023-09-23T20:48:41.394 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:41 smithi018 ceph-mon[179647]: failed to encode map e585 with expected crc 2023-09-23T20:48:41.394 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:41 smithi018 ceph-mon[179647]: failed to encode map e585 with expected crc 2023-09-23T20:48:41.394 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:41 smithi018 ceph-mon[179647]: 10.9 continuing backfill to osd.4 from (541'4623,583'10757] MIN to 583'10757 2023-09-23T20:48:41.394 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:41 smithi018 ceph-mon[179647]: failed to encode map e585 with expected crc 2023-09-23T20:48:41.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:41 smithi149 ceph-mon[170394]: 10.3 continuing backfill to osd.4 from (536'1300,583'10826] MIN to 583'10826 2023-09-23T20:48:41.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:41 smithi149 ceph-mon[170394]: failed to encode map e585 with expected crc 2023-09-23T20:48:41.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:41 smithi149 ceph-mon[170394]: failed to encode map e585 with expected crc 2023-09-23T20:48:41.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:41 smithi149 ceph-mon[170394]: failed to encode map e585 with expected crc 2023-09-23T20:48:41.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:41 smithi149 ceph-mon[170394]: osdmap e585: 8 total, 8 up, 7 in 2023-09-23T20:48:41.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:41 smithi149 ceph-mon[170394]: failed to encode map e585 with expected crc 2023-09-23T20:48:41.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:41 smithi149 ceph-mon[170394]: 10.8 continuing backfill to osd.5 from (541'4571,583'10505] MIN to 583'10505 2023-09-23T20:48:41.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:41 smithi149 ceph-mon[170394]: 10.5 continuing backfill to osd.7 from (542'4806,583'10701] MIN to 583'10701 2023-09-23T20:48:41.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:41 smithi149 ceph-mon[170394]: failed to encode map e585 with expected crc 2023-09-23T20:48:41.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:41 smithi149 ceph-mon[170394]: failed to encode map e585 with expected crc 2023-09-23T20:48:41.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:41 smithi149 ceph-mon[170394]: failed to encode map e585 with expected crc 2023-09-23T20:48:41.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:41 smithi149 ceph-mon[170394]: failed to encode map e585 with expected crc 2023-09-23T20:48:41.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:41 smithi149 ceph-mon[170394]: failed to encode map e585 with expected crc 2023-09-23T20:48:41.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:41 smithi149 ceph-mon[170394]: failed to encode map e585 with expected crc 2023-09-23T20:48:41.468 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:41 smithi149 ceph-mon[170394]: 10.9 continuing backfill to osd.4 from (541'4623,583'10757] MIN to 583'10757 2023-09-23T20:48:41.468 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:41 smithi149 ceph-mon[170394]: failed to encode map e585 with expected crc 2023-09-23T20:48:41.630 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T20:48:41.699 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:48:41.702 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:41.703 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:48:41.769 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T20:48:41.801 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:48:41.804 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:48:41.850 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:41.897 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:48:41.901 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:41.902 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:48:42.003 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:48:42.042 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:42.096 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:48:42.104 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:42.104 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:48:42.205 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:48:42.247 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:42.345 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:42 smithi018 ceph-mon[184144]: pgmap v1478: 129 pgs: 10 unknown, 2 active+remapped+backfilling, 4 active+remapped+backfill_wait, 113 active+clean; 2.5 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 684 op/s; 13144/123960 objects misplaced (10.603%); 11 MiB/s, 169 objects/s recovering 2023-09-23T20:48:42.346 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:42 smithi018 ceph-mon[184144]: failed to encode map e586 with expected crc 2023-09-23T20:48:42.346 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:42 smithi018 ceph-mon[184144]: osdmap e586: 8 total, 8 up, 7 in 2023-09-23T20:48:42.346 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:42 smithi018 ceph-mon[184144]: failed to encode map e586 with expected crc 2023-09-23T20:48:42.346 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:42 smithi018 ceph-mon[184144]: failed to encode map e586 with expected crc 2023-09-23T20:48:42.346 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:42 smithi018 ceph-mon[184144]: failed to encode map e586 with expected crc 2023-09-23T20:48:42.346 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:42 smithi018 ceph-mon[184144]: 10.2 scrub starts 2023-09-23T20:48:42.346 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:42 smithi018 ceph-mon[179647]: pgmap v1478: 129 pgs: 10 unknown, 2 active+remapped+backfilling, 4 active+remapped+backfill_wait, 113 active+clean; 2.5 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 684 op/s; 13144/123960 objects misplaced (10.603%); 11 MiB/s, 169 objects/s recovering 2023-09-23T20:48:42.346 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:42 smithi018 ceph-mon[179647]: failed to encode map e586 with expected crc 2023-09-23T20:48:42.347 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:42 smithi018 ceph-mon[179647]: osdmap e586: 8 total, 8 up, 7 in 2023-09-23T20:48:42.347 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:42 smithi018 ceph-mon[179647]: failed to encode map e586 with expected crc 2023-09-23T20:48:42.347 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:42 smithi018 ceph-mon[179647]: failed to encode map e586 with expected crc 2023-09-23T20:48:42.347 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:42 smithi018 ceph-mon[179647]: failed to encode map e586 with expected crc 2023-09-23T20:48:42.347 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:42 smithi018 ceph-mon[179647]: 10.2 scrub starts 2023-09-23T20:48:42.348 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:48:42.351 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:42.351 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:48:42.452 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:48:42.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:42 smithi149 ceph-mon[170394]: pgmap v1478: 129 pgs: 10 unknown, 2 active+remapped+backfilling, 4 active+remapped+backfill_wait, 113 active+clean; 2.5 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 684 op/s; 13144/123960 objects misplaced (10.603%); 11 MiB/s, 169 objects/s recovering 2023-09-23T20:48:42.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:42 smithi149 ceph-mon[170394]: failed to encode map e586 with expected crc 2023-09-23T20:48:42.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:42 smithi149 ceph-mon[170394]: osdmap e586: 8 total, 8 up, 7 in 2023-09-23T20:48:42.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:42 smithi149 ceph-mon[170394]: failed to encode map e586 with expected crc 2023-09-23T20:48:42.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:42 smithi149 ceph-mon[170394]: failed to encode map e586 with expected crc 2023-09-23T20:48:42.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:42 smithi149 ceph-mon[170394]: failed to encode map e586 with expected crc 2023-09-23T20:48:42.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:42 smithi149 ceph-mon[170394]: 10.2 scrub starts 2023-09-23T20:48:42.481 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:42.506 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:48:42.576 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:48:42.579 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:42.580 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:48:42.631 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:48:42.680 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:48:42.714 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:43.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:43 smithi018 ceph-mon[184144]: 1.8 deep-scrub starts 2023-09-23T20:48:43.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:43 smithi018 ceph-mon[184144]: failed to encode map e587 with expected crc 2023-09-23T20:48:43.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:43 smithi018 ceph-mon[184144]: osdmap e587: 8 total, 8 up, 7 in 2023-09-23T20:48:43.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:43 smithi018 ceph-mon[184144]: failed to encode map e587 with expected crc 2023-09-23T20:48:43.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:43 smithi018 ceph-mon[184144]: failed to encode map e587 with expected crc 2023-09-23T20:48:43.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:43 smithi018 ceph-mon[184144]: 10.13 continuing backfill to osd.4 from (536'1300,583'10826] MIN to 583'10826 2023-09-23T20:48:43.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:43 smithi018 ceph-mon[184144]: 10.19 continuing backfill to osd.4 from (541'4623,583'10757] MIN to 583'10757 2023-09-23T20:48:43.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:43 smithi018 ceph-mon[184144]: 10.18 continuing backfill to osd.5 from (541'4571,583'10505] MIN to 583'10505 2023-09-23T20:48:43.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:43 smithi018 ceph-mon[184144]: 10.15 continuing backfill to osd.7 from (542'4806,583'10701] MIN to 583'10701 2023-09-23T20:48:43.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:43 smithi018 ceph-mon[184144]: 1.a scrub starts 2023-09-23T20:48:43.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:43 smithi018 ceph-mon[184144]: pgmap v1481: 129 pgs: 2 remapped+peering, 3 peering, 1 unknown, 3 active+remapped+backfilling, 2 active+remapped+backfill_wait, 118 active+clean; 2.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 726 op/s; 6841/117147 objects misplaced (5.840%); 36 MiB/s, 575 objects/s recovering 2023-09-23T20:48:43.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:43 smithi018 ceph-mon[184144]: 10.12 scrub starts 2023-09-23T20:48:43.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:43 smithi018 ceph-mon[179647]: 1.8 deep-scrub starts 2023-09-23T20:48:43.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:43 smithi018 ceph-mon[179647]: failed to encode map e587 with expected crc 2023-09-23T20:48:43.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:43 smithi018 ceph-mon[179647]: osdmap e587: 8 total, 8 up, 7 in 2023-09-23T20:48:43.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:43 smithi018 ceph-mon[179647]: failed to encode map e587 with expected crc 2023-09-23T20:48:43.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:43 smithi018 ceph-mon[179647]: failed to encode map e587 with expected crc 2023-09-23T20:48:43.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:43 smithi018 ceph-mon[179647]: 10.13 continuing backfill to osd.4 from (536'1300,583'10826] MIN to 583'10826 2023-09-23T20:48:43.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:43 smithi018 ceph-mon[179647]: 10.19 continuing backfill to osd.4 from (541'4623,583'10757] MIN to 583'10757 2023-09-23T20:48:43.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:43 smithi018 ceph-mon[179647]: 10.18 continuing backfill to osd.5 from (541'4571,583'10505] MIN to 583'10505 2023-09-23T20:48:43.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:43 smithi018 ceph-mon[179647]: 10.15 continuing backfill to osd.7 from (542'4806,583'10701] MIN to 583'10701 2023-09-23T20:48:43.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:43 smithi018 ceph-mon[179647]: 1.a scrub starts 2023-09-23T20:48:43.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:43 smithi018 ceph-mon[179647]: pgmap v1481: 129 pgs: 2 remapped+peering, 3 peering, 1 unknown, 3 active+remapped+backfilling, 2 active+remapped+backfill_wait, 118 active+clean; 2.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 726 op/s; 6841/117147 objects misplaced (5.840%); 36 MiB/s, 575 objects/s recovering 2023-09-23T20:48:43.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:43 smithi018 ceph-mon[179647]: 10.12 scrub starts 2023-09-23T20:48:43.468 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:43 smithi149 ceph-mon[170394]: 1.8 deep-scrub starts 2023-09-23T20:48:43.468 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:43 smithi149 ceph-mon[170394]: failed to encode map e587 with expected crc 2023-09-23T20:48:43.468 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:43 smithi149 ceph-mon[170394]: osdmap e587: 8 total, 8 up, 7 in 2023-09-23T20:48:43.468 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:43 smithi149 ceph-mon[170394]: failed to encode map e587 with expected crc 2023-09-23T20:48:43.468 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:43 smithi149 ceph-mon[170394]: failed to encode map e587 with expected crc 2023-09-23T20:48:43.468 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:43 smithi149 ceph-mon[170394]: 10.13 continuing backfill to osd.4 from (536'1300,583'10826] MIN to 583'10826 2023-09-23T20:48:43.468 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:43 smithi149 ceph-mon[170394]: 10.19 continuing backfill to osd.4 from (541'4623,583'10757] MIN to 583'10757 2023-09-23T20:48:43.468 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:43 smithi149 ceph-mon[170394]: 10.18 continuing backfill to osd.5 from (541'4571,583'10505] MIN to 583'10505 2023-09-23T20:48:43.468 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:43 smithi149 ceph-mon[170394]: 10.15 continuing backfill to osd.7 from (542'4806,583'10701] MIN to 583'10701 2023-09-23T20:48:43.469 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:43 smithi149 ceph-mon[170394]: 1.a scrub starts 2023-09-23T20:48:43.469 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:43 smithi149 ceph-mon[170394]: pgmap v1481: 129 pgs: 2 remapped+peering, 3 peering, 1 unknown, 3 active+remapped+backfilling, 2 active+remapped+backfill_wait, 118 active+clean; 2.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 726 op/s; 6841/117147 objects misplaced (5.840%); 36 MiB/s, 575 objects/s recovering 2023-09-23T20:48:43.469 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:43 smithi149 ceph-mon[170394]: 10.12 scrub starts 2023-09-23T20:48:43.486 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:48:43.519 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:48:43.553 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:48:43.589 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:48:43.620 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:48:43.646 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 7, 3, 6, 5, 4] out_osds: [2] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T20:48:43.646 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:48:43.647 INFO:tasks.thrashosds.thrasher:Adding osd 2 2023-09-23T20:48:43.647 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd in 2 2023-09-23T20:48:43.650 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:48:43.682 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:48:43.710 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:48:43.715 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:48:43.715 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:48:43.715 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:48:43.716 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:48:43.716 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:48:43.716 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:48:43.716 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:48:43.716 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:48:43.750 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:48:43.755 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:43.755 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:48:43.856 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:48:43.897 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:43.946 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:48:43.950 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:43.950 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:48:44.051 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:48:44.116 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:44.211 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:48:44.220 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:44.220 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:48:44.321 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:48:44.368 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:44 smithi018 ceph-mon[184144]: osdmap e588: 8 total, 8 up, 7 in 2023-09-23T20:48:44.368 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:44 smithi018 ceph-mon[184144]: failed to encode map e588 with expected crc 2023-09-23T20:48:44.368 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:44 smithi018 ceph-mon[184144]: failed to encode map e588 with expected crc 2023-09-23T20:48:44.368 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:44 smithi018 ceph-mon[184144]: failed to encode map e588 with expected crc 2023-09-23T20:48:44.368 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:44 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/4218403092' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:48:44.369 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:44 smithi018 ceph-mon[179647]: osdmap e588: 8 total, 8 up, 7 in 2023-09-23T20:48:44.369 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:44 smithi018 ceph-mon[179647]: failed to encode map e588 with expected crc 2023-09-23T20:48:44.369 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:44 smithi018 ceph-mon[179647]: failed to encode map e588 with expected crc 2023-09-23T20:48:44.369 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:44 smithi018 ceph-mon[179647]: failed to encode map e588 with expected crc 2023-09-23T20:48:44.369 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:44 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/4218403092' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:48:44.374 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:44.433 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:44 smithi149 ceph-mon[170394]: osdmap e588: 8 total, 8 up, 7 in 2023-09-23T20:48:44.434 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:44 smithi149 ceph-mon[170394]: failed to encode map e588 with expected crc 2023-09-23T20:48:44.434 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:44 smithi149 ceph-mon[170394]: failed to encode map e588 with expected crc 2023-09-23T20:48:44.434 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:44 smithi149 ceph-mon[170394]: failed to encode map e588 with expected crc 2023-09-23T20:48:44.434 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:44 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/4218403092' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:48:44.435 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:48:44.443 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:44.444 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:48:44.545 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:48:44.577 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:48:44.626 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:44.872 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:48:44.879 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:44.879 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:48:44.980 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:48:45.028 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:45.100 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:48:44 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:48:44] "GET /metrics HTTP/1.1" 200 36608 "" "Prometheus/2.43.0" 2023-09-23T20:48:45.103 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:45 smithi018 ceph-mon[179647]: osdmap e589: 8 total, 8 up, 7 in 2023-09-23T20:48:45.103 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:45 smithi018 ceph-mon[179647]: failed to encode map e589 with expected crc 2023-09-23T20:48:45.103 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:45 smithi018 ceph-mon[179647]: failed to encode map e589 with expected crc 2023-09-23T20:48:45.104 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:45 smithi018 ceph-mon[179647]: failed to encode map e589 with expected crc 2023-09-23T20:48:45.104 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:45 smithi018 ceph-mon[179647]: 10.7 scrub starts 2023-09-23T20:48:45.120 INFO:teuthology.orchestra.run.smithi018.stderr:noscrub is set 2023-09-23T20:48:45.218 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:45 smithi149 ceph-mon[170394]: osdmap e589: 8 total, 8 up, 7 in 2023-09-23T20:48:45.218 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:45 smithi149 ceph-mon[170394]: failed to encode map e589 with expected crc 2023-09-23T20:48:45.219 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:45 smithi149 ceph-mon[170394]: failed to encode map e589 with expected crc 2023-09-23T20:48:45.219 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:45 smithi149 ceph-mon[170394]: failed to encode map e589 with expected crc 2023-09-23T20:48:45.219 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:45 smithi149 ceph-mon[170394]: 10.7 scrub starts 2023-09-23T20:48:45.219 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:45 smithi149 ceph-mon[170394]: pgmap v1484: 129 pgs: 2 remapped+peering, 2 peering, 3 active+remapped+backfilling, 3 active+remapped+backfill_wait, 119 active+clean; 2.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 622 op/s; 6596/116484 objects misplaced (5.663%); 36 MiB/s, 579 objects/s recovering 2023-09-23T20:48:45.219 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:45 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1809360222' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:48:45.219 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:45 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:48:45.219 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:45 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:48:45.219 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:45 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:48:45.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:45 smithi018 ceph-mon[184144]: osdmap e589: 8 total, 8 up, 7 in 2023-09-23T20:48:45.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:45 smithi018 ceph-mon[184144]: failed to encode map e589 with expected crc 2023-09-23T20:48:45.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:45 smithi018 ceph-mon[184144]: failed to encode map e589 with expected crc 2023-09-23T20:48:45.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:45 smithi018 ceph-mon[184144]: failed to encode map e589 with expected crc 2023-09-23T20:48:45.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:45 smithi018 ceph-mon[184144]: 10.7 scrub starts 2023-09-23T20:48:45.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:45 smithi018 ceph-mon[184144]: pgmap v1484: 129 pgs: 2 remapped+peering, 2 peering, 3 active+remapped+backfilling, 3 active+remapped+backfill_wait, 119 active+clean; 2.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 622 op/s; 6596/116484 objects misplaced (5.663%); 36 MiB/s, 579 objects/s recovering 2023-09-23T20:48:45.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:45 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1809360222' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:48:45.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:45 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:48:45.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:45 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:48:45.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:45 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:48:45.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:45 smithi018 ceph-mon[179647]: pgmap v1484: 129 pgs: 2 remapped+peering, 2 peering, 3 active+remapped+backfilling, 3 active+remapped+backfill_wait, 119 active+clean; 2.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 622 op/s; 6596/116484 objects misplaced (5.663%); 36 MiB/s, 579 objects/s recovering 2023-09-23T20:48:45.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:45 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1809360222' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:48:45.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:45 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:48:45.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:45 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:48:45.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:45 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:48:45.612 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:48:45.616 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:45.617 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:48:45.719 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:48:45.805 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:45.940 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:48:45.953 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:45.953 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:48:46.054 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:48:46.118 INFO:teuthology.orchestra.run.smithi149.stdout:{ 2023-09-23T20:48:46.118 INFO:teuthology.orchestra.run.smithi149.stdout: "ops": [], 2023-09-23T20:48:46.118 INFO:teuthology.orchestra.run.smithi149.stdout: "complaint_time": 30, 2023-09-23T20:48:46.118 INFO:teuthology.orchestra.run.smithi149.stdout: "num_blocked_ops": 0 2023-09-23T20:48:46.118 INFO:teuthology.orchestra.run.smithi149.stdout:} 2023-09-23T20:48:46.137 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:46.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:46 smithi018 ceph-mon[179647]: 1.8 deep-scrub starts 2023-09-23T20:48:46.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:46 smithi018 ceph-mon[179647]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:48:46.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:46 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:48:46.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:46 smithi018 ceph-mon[179647]: osdmap e590: 8 total, 8 up, 7 in 2023-09-23T20:48:46.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:46 smithi018 ceph-mon[179647]: failed to encode map e590 with expected crc 2023-09-23T20:48:46.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:46 smithi018 ceph-mon[179647]: failed to encode map e590 with expected crc 2023-09-23T20:48:46.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:46 smithi018 ceph-mon[179647]: failed to encode map e590 with expected crc 2023-09-23T20:48:46.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:46 smithi018 ceph-mon[179647]: failed to encode map e590 with expected crc 2023-09-23T20:48:46.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:46 smithi018 ceph-mon[184144]: 1.8 deep-scrub starts 2023-09-23T20:48:46.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:46 smithi018 ceph-mon[184144]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:48:46.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:46 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:48:46.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:46 smithi018 ceph-mon[184144]: osdmap e590: 8 total, 8 up, 7 in 2023-09-23T20:48:46.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:46 smithi018 ceph-mon[184144]: failed to encode map e590 with expected crc 2023-09-23T20:48:46.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:46 smithi018 ceph-mon[184144]: failed to encode map e590 with expected crc 2023-09-23T20:48:46.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:46 smithi018 ceph-mon[184144]: failed to encode map e590 with expected crc 2023-09-23T20:48:46.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:46 smithi018 ceph-mon[184144]: failed to encode map e590 with expected crc 2023-09-23T20:48:46.214 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:48:46.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:46 smithi149 ceph-mon[170394]: 1.8 deep-scrub starts 2023-09-23T20:48:46.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:46 smithi149 ceph-mon[170394]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:48:46.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:46 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:48:46.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:46 smithi149 ceph-mon[170394]: osdmap e590: 8 total, 8 up, 7 in 2023-09-23T20:48:46.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:46 smithi149 ceph-mon[170394]: failed to encode map e590 with expected crc 2023-09-23T20:48:46.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:46 smithi149 ceph-mon[170394]: failed to encode map e590 with expected crc 2023-09-23T20:48:46.218 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:46 smithi149 ceph-mon[170394]: failed to encode map e590 with expected crc 2023-09-23T20:48:46.218 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:46 smithi149 ceph-mon[170394]: failed to encode map e590 with expected crc 2023-09-23T20:48:46.483 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:48:46.495 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:46.496 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:48:46.527 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T20:48:46.596 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:48:46.634 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:46 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mon-a[179623]: 2023-09-23T20:48:46.484+0000 7f160fef9700 -1 mon.a@0(leader).osd e590 definitely_dead 0 2023-09-23T20:48:46.639 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:46.685 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:48:46.689 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:46.690 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:48:46.790 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:48:46.903 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:46.984 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:48:47.008 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:48:47.016 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:47.016 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:48:47.117 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:48:47.128 INFO:teuthology.orchestra.run.smithi018.stderr:marked in osd.2. 2023-09-23T20:48:47.179 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:47 smithi149 ceph-mon[170394]: pgmap v1486: 129 pgs: 2 remapped+peering, 1 peering, 3 active+remapped+backfilling, 3 active+remapped+backfill_wait, 120 active+clean; 2.3 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 366 op/s; 4556/112143 objects misplaced (4.063%); 46 MiB/s, 732 objects/s recovering 2023-09-23T20:48:47.179 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:47 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3461676327' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["2"]}]: dispatch 2023-09-23T20:48:47.182 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:47.217 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:48:47.220 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:47.220 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:48:47.321 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:48:47.351 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:47.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:47 smithi018 ceph-mon[184144]: pgmap v1486: 129 pgs: 2 remapped+peering, 1 peering, 3 active+remapped+backfilling, 3 active+remapped+backfill_wait, 120 active+clean; 2.3 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 366 op/s; 4556/112143 objects misplaced (4.063%); 46 MiB/s, 732 objects/s recovering 2023-09-23T20:48:47.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:47 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3461676327' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["2"]}]: dispatch 2023-09-23T20:48:47.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:47 smithi018 ceph-mon[179647]: pgmap v1486: 129 pgs: 2 remapped+peering, 1 peering, 3 active+remapped+backfilling, 3 active+remapped+backfill_wait, 120 active+clean; 2.3 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 366 op/s; 4556/112143 objects misplaced (4.063%); 46 MiB/s, 732 objects/s recovering 2023-09-23T20:48:47.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:47 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3461676327' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["2"]}]: dispatch 2023-09-23T20:48:47.427 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:48:47.631 INFO:tasks.thrashosds.thrasher:Added osd 2 2023-09-23T20:48:47.645 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:48:47.648 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:47.648 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:48:47.664 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T20:48:47.750 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:48:47.778 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:48.122 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:48 smithi018 ceph-mon[179647]: failed to encode map e591 with expected crc 2023-09-23T20:48:48.123 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:48 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3461676327' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["2"]}]': finished 2023-09-23T20:48:48.123 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:48 smithi018 ceph-mon[179647]: osdmap e591: 8 total, 8 up, 8 in 2023-09-23T20:48:48.123 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:48 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:48:48.198 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:48:48.205 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:48.206 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:48:48.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:48 smithi149 ceph-mon[170394]: failed to encode map e591 with expected crc 2023-09-23T20:48:48.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:48 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3461676327' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["2"]}]': finished 2023-09-23T20:48:48.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:48 smithi149 ceph-mon[170394]: osdmap e591: 8 total, 8 up, 8 in 2023-09-23T20:48:48.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:48 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:48:48.307 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:48:48.335 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:48.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:48 smithi018 ceph-mon[184144]: failed to encode map e591 with expected crc 2023-09-23T20:48:48.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:48 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3461676327' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["2"]}]': finished 2023-09-23T20:48:48.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:48 smithi018 ceph-mon[184144]: osdmap e591: 8 total, 8 up, 8 in 2023-09-23T20:48:48.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:48 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:48:48.429 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:48:48.437 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:48:48.439 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:48.440 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:48:48.449 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:48:48.470 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:48:48.491 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:48:48.512 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:48:48.533 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:48:48.541 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:48:48.552 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:48:48.571 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:48:48.572 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:48:48.572 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:48:48.573 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:48:48.573 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:48:48.573 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:48:48.573 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:48:48.573 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:48:48.573 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:48:48.573 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:48.575 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:48:48.660 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:48:48.663 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:48.664 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:48:48.738 INFO:teuthology.orchestra.run.smithi149.stdout:ERROR: (22) Invalid argument 2023-09-23T20:48:48.738 INFO:teuthology.orchestra.run.smithi149.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:48:48.764 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:48:48.794 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:48.838 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:48:48.841 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:48.842 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:48:48.943 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:48:48.971 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:49.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:49 smithi018 ceph-mon[179647]: failed to encode map e592 with expected crc 2023-09-23T20:48:49.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:49 smithi018 ceph-mon[179647]: osdmap e592: 8 total, 8 up, 8 in 2023-09-23T20:48:49.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:49 smithi018 ceph-mon[179647]: failed to encode map e592 with expected crc 2023-09-23T20:48:49.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:49 smithi018 ceph-mon[179647]: failed to encode map e592 with expected crc 2023-09-23T20:48:49.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:49 smithi018 ceph-mon[179647]: failed to encode map e592 with expected crc 2023-09-23T20:48:49.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:49 smithi018 ceph-mon[179647]: failed to encode map e592 with expected crc 2023-09-23T20:48:49.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:49 smithi018 ceph-mon[179647]: failed to encode map e592 with expected crc 2023-09-23T20:48:49.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:49 smithi018 ceph-mon[179647]: failed to encode map e592 with expected crc 2023-09-23T20:48:49.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:49 smithi018 ceph-mon[179647]: pgmap v1489: 129 pgs: 3 active+remapped+backfilling, 4 active+remapped+backfill_wait, 122 active+clean; 2.2 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 548 op/s; 4562/109491 objects misplaced (4.167%); 68 MiB/s, 1.09k objects/s recovering 2023-09-23T20:48:49.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:49 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "17"}]: dispatch 2023-09-23T20:48:49.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:49 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3581900959' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:48:49.144 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:48:49.144 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:49 smithi018 ceph-mon[184144]: failed to encode map e592 with expected crc 2023-09-23T20:48:49.145 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:49 smithi018 ceph-mon[184144]: osdmap e592: 8 total, 8 up, 8 in 2023-09-23T20:48:49.145 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:49 smithi018 ceph-mon[184144]: failed to encode map e592 with expected crc 2023-09-23T20:48:49.145 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:49 smithi018 ceph-mon[184144]: failed to encode map e592 with expected crc 2023-09-23T20:48:49.145 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:49 smithi018 ceph-mon[184144]: failed to encode map e592 with expected crc 2023-09-23T20:48:49.145 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:49 smithi018 ceph-mon[184144]: failed to encode map e592 with expected crc 2023-09-23T20:48:49.145 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:49 smithi018 ceph-mon[184144]: failed to encode map e592 with expected crc 2023-09-23T20:48:49.145 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:49 smithi018 ceph-mon[184144]: failed to encode map e592 with expected crc 2023-09-23T20:48:49.146 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:49 smithi018 ceph-mon[184144]: pgmap v1489: 129 pgs: 3 active+remapped+backfilling, 4 active+remapped+backfill_wait, 122 active+clean; 2.2 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 548 op/s; 4562/109491 objects misplaced (4.167%); 68 MiB/s, 1.09k objects/s recovering 2023-09-23T20:48:49.146 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:49 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "17"}]: dispatch 2023-09-23T20:48:49.146 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:49 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3581900959' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:48:49.151 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:49.152 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:48:49.181 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T20:48:49.253 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:48:49.316 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:49.398 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:48:49.404 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:49.404 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:48:49.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:49 smithi149 ceph-mon[170394]: failed to encode map e592 with expected crc 2023-09-23T20:48:49.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:49 smithi149 ceph-mon[170394]: osdmap e592: 8 total, 8 up, 8 in 2023-09-23T20:48:49.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:49 smithi149 ceph-mon[170394]: failed to encode map e592 with expected crc 2023-09-23T20:48:49.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:49 smithi149 ceph-mon[170394]: failed to encode map e592 with expected crc 2023-09-23T20:48:49.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:49 smithi149 ceph-mon[170394]: failed to encode map e592 with expected crc 2023-09-23T20:48:49.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:49 smithi149 ceph-mon[170394]: failed to encode map e592 with expected crc 2023-09-23T20:48:49.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:49 smithi149 ceph-mon[170394]: failed to encode map e592 with expected crc 2023-09-23T20:48:49.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:49 smithi149 ceph-mon[170394]: failed to encode map e592 with expected crc 2023-09-23T20:48:49.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:49 smithi149 ceph-mon[170394]: pgmap v1489: 129 pgs: 3 active+remapped+backfilling, 4 active+remapped+backfill_wait, 122 active+clean; 2.2 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 548 op/s; 4562/109491 objects misplaced (4.167%); 68 MiB/s, 1.09k objects/s recovering 2023-09-23T20:48:49.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:49 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "17"}]: dispatch 2023-09-23T20:48:49.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:49 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3581900959' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:48:49.505 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:48:49.544 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:49.638 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:48:49.647 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:49.648 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:48:49.748 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:48:49.778 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:49.815 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:48:49.818 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:49.819 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:48:49.920 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:48:49.946 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:49.981 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:48:49.984 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:49.984 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:48:50.085 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:48:50.127 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:48:50.129 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:50.198 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:48:50.202 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:50.202 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:48:50.303 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:48:50.332 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:50.371 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:48:50.375 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:50.375 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:48:50.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:50 smithi018 ceph-mon[179647]: 10.8 starting backfill to osd.2 from (541'4571,585'10508] MIN to 590'10631 2023-09-23T20:48:50.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:50 smithi018 ceph-mon[179647]: failed to encode map e593 with expected crc 2023-09-23T20:48:50.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:50 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "17"}]': finished 2023-09-23T20:48:50.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:50 smithi018 ceph-mon[179647]: osdmap e593: 8 total, 8 up, 8 in 2023-09-23T20:48:50.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:50 smithi018 ceph-mon[179647]: 10.5 starting backfill to osd.2 from (542'4806,587'10714] MIN to 590'10823 2023-09-23T20:48:50.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:50 smithi018 ceph-mon[179647]: failed to encode map e593 with expected crc 2023-09-23T20:48:50.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:50 smithi018 ceph-mon[179647]: failed to encode map e593 with expected crc 2023-09-23T20:48:50.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:50 smithi018 ceph-mon[179647]: failed to encode map e593 with expected crc 2023-09-23T20:48:50.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:50 smithi018 ceph-mon[179647]: failed to encode map e593 with expected crc 2023-09-23T20:48:50.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:50 smithi018 ceph-mon[179647]: failed to encode map e593 with expected crc 2023-09-23T20:48:50.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:50 smithi018 ceph-mon[179647]: failed to encode map e593 with expected crc 2023-09-23T20:48:50.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:50 smithi018 ceph-mon[179647]: failed to encode map e593 with expected crc 2023-09-23T20:48:50.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:50 smithi018 ceph-mon[179647]: failed to encode map e593 with expected crc 2023-09-23T20:48:50.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:50 smithi018 ceph-mon[179647]: failed to encode map e593 with expected crc 2023-09-23T20:48:50.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:50 smithi018 ceph-mon[179647]: 10.f starting backfill to osd.2 from (542'5099,585'10564] MIN to 590'10814 2023-09-23T20:48:50.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:50 smithi018 ceph-mon[184144]: 10.8 starting backfill to osd.2 from (541'4571,585'10508] MIN to 590'10631 2023-09-23T20:48:50.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:50 smithi018 ceph-mon[184144]: failed to encode map e593 with expected crc 2023-09-23T20:48:50.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:50 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "17"}]': finished 2023-09-23T20:48:50.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:50 smithi018 ceph-mon[184144]: osdmap e593: 8 total, 8 up, 8 in 2023-09-23T20:48:50.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:50 smithi018 ceph-mon[184144]: 10.5 starting backfill to osd.2 from (542'4806,587'10714] MIN to 590'10823 2023-09-23T20:48:50.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:50 smithi018 ceph-mon[184144]: failed to encode map e593 with expected crc 2023-09-23T20:48:50.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:50 smithi018 ceph-mon[184144]: failed to encode map e593 with expected crc 2023-09-23T20:48:50.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:50 smithi018 ceph-mon[184144]: failed to encode map e593 with expected crc 2023-09-23T20:48:50.394 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:50 smithi018 ceph-mon[184144]: failed to encode map e593 with expected crc 2023-09-23T20:48:50.394 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:50 smithi018 ceph-mon[184144]: failed to encode map e593 with expected crc 2023-09-23T20:48:50.394 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:50 smithi018 ceph-mon[184144]: failed to encode map e593 with expected crc 2023-09-23T20:48:50.394 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:50 smithi018 ceph-mon[184144]: failed to encode map e593 with expected crc 2023-09-23T20:48:50.394 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:50 smithi018 ceph-mon[184144]: failed to encode map e593 with expected crc 2023-09-23T20:48:50.394 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:50 smithi018 ceph-mon[184144]: failed to encode map e593 with expected crc 2023-09-23T20:48:50.394 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:50 smithi018 ceph-mon[184144]: 10.f starting backfill to osd.2 from (542'5099,585'10564] MIN to 590'10814 2023-09-23T20:48:50.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:50 smithi149 ceph-mon[170394]: 10.8 starting backfill to osd.2 from (541'4571,585'10508] MIN to 590'10631 2023-09-23T20:48:50.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:50 smithi149 ceph-mon[170394]: failed to encode map e593 with expected crc 2023-09-23T20:48:50.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:50 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "17"}]': finished 2023-09-23T20:48:50.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:50 smithi149 ceph-mon[170394]: osdmap e593: 8 total, 8 up, 8 in 2023-09-23T20:48:50.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:50 smithi149 ceph-mon[170394]: 10.5 starting backfill to osd.2 from (542'4806,587'10714] MIN to 590'10823 2023-09-23T20:48:50.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:50 smithi149 ceph-mon[170394]: failed to encode map e593 with expected crc 2023-09-23T20:48:50.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:50 smithi149 ceph-mon[170394]: failed to encode map e593 with expected crc 2023-09-23T20:48:50.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:50 smithi149 ceph-mon[170394]: failed to encode map e593 with expected crc 2023-09-23T20:48:50.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:50 smithi149 ceph-mon[170394]: failed to encode map e593 with expected crc 2023-09-23T20:48:50.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:50 smithi149 ceph-mon[170394]: failed to encode map e593 with expected crc 2023-09-23T20:48:50.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:50 smithi149 ceph-mon[170394]: failed to encode map e593 with expected crc 2023-09-23T20:48:50.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:50 smithi149 ceph-mon[170394]: failed to encode map e593 with expected crc 2023-09-23T20:48:50.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:50 smithi149 ceph-mon[170394]: failed to encode map e593 with expected crc 2023-09-23T20:48:50.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:50 smithi149 ceph-mon[170394]: failed to encode map e593 with expected crc 2023-09-23T20:48:50.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:50 smithi149 ceph-mon[170394]: 10.f starting backfill to osd.2 from (542'5099,585'10564] MIN to 590'10814 2023-09-23T20:48:50.476 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:48:50.505 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:50.540 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:48:50.543 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:50.543 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:48:50.644 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:48:50.671 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:50.705 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:48:50.708 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:50.709 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:48:50.809 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:48:50.837 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:50.872 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:48:50.875 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:50.876 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:48:50.977 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:48:51.003 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:51.041 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:48:51.044 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:51.044 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:48:51.145 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:48:51.172 INFO:teuthology.orchestra.run.smithi018.stderr:nodeep-scrub is set 2023-09-23T20:48:51.187 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:51.325 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:48:51.337 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:51 smithi018 ceph-mon[184144]: failed to encode map e594 with expected crc 2023-09-23T20:48:51.337 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:51 smithi018 ceph-mon[184144]: osdmap e594: 8 total, 8 up, 8 in 2023-09-23T20:48:51.337 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:51 smithi018 ceph-mon[184144]: failed to encode map e594 with expected crc 2023-09-23T20:48:51.337 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:51 smithi018 ceph-mon[184144]: failed to encode map e594 with expected crc 2023-09-23T20:48:51.337 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:51 smithi018 ceph-mon[184144]: failed to encode map e594 with expected crc 2023-09-23T20:48:51.338 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:51 smithi018 ceph-mon[184144]: failed to encode map e594 with expected crc 2023-09-23T20:48:51.338 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:51 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/682115770' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:48:51.338 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:51 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:48:51.338 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:51 smithi018 ceph-mon[184144]: pgmap v1492: 129 pgs: 3 peering, 2 remapped+peering, 1 active+remapped+backfilling, 1 active+undersized+remapped, 1 active+remapped+backfill_wait, 1 active+remapped, 120 active+clean; 2.2 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 371 op/s; 778/107688 objects misplaced (0.722%); 27 MiB/s, 437 objects/s recovering 2023-09-23T20:48:51.338 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:51 smithi018 ceph-mon[179647]: failed to encode map e594 with expected crc 2023-09-23T20:48:51.338 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:51 smithi018 ceph-mon[179647]: osdmap e594: 8 total, 8 up, 8 in 2023-09-23T20:48:51.338 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:51 smithi018 ceph-mon[179647]: failed to encode map e594 with expected crc 2023-09-23T20:48:51.339 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:51 smithi018 ceph-mon[179647]: failed to encode map e594 with expected crc 2023-09-23T20:48:51.339 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:51 smithi018 ceph-mon[179647]: failed to encode map e594 with expected crc 2023-09-23T20:48:51.339 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:51 smithi018 ceph-mon[179647]: failed to encode map e594 with expected crc 2023-09-23T20:48:51.339 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:51 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/682115770' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:48:51.339 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:51 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:48:51.339 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:51 smithi018 ceph-mon[179647]: pgmap v1492: 129 pgs: 3 peering, 2 remapped+peering, 1 active+remapped+backfilling, 1 active+undersized+remapped, 1 active+remapped+backfill_wait, 1 active+remapped, 120 active+clean; 2.2 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 371 op/s; 778/107688 objects misplaced (0.722%); 27 MiB/s, 437 objects/s recovering 2023-09-23T20:48:51.339 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:51.340 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:48:51.441 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:48:51.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:51 smithi149 ceph-mon[170394]: failed to encode map e594 with expected crc 2023-09-23T20:48:51.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:51 smithi149 ceph-mon[170394]: osdmap e594: 8 total, 8 up, 8 in 2023-09-23T20:48:51.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:51 smithi149 ceph-mon[170394]: failed to encode map e594 with expected crc 2023-09-23T20:48:51.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:51 smithi149 ceph-mon[170394]: failed to encode map e594 with expected crc 2023-09-23T20:48:51.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:51 smithi149 ceph-mon[170394]: failed to encode map e594 with expected crc 2023-09-23T20:48:51.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:51 smithi149 ceph-mon[170394]: failed to encode map e594 with expected crc 2023-09-23T20:48:51.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:51 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/682115770' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:48:51.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:51 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:48:51.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:51 smithi149 ceph-mon[170394]: pgmap v1492: 129 pgs: 3 peering, 2 remapped+peering, 1 active+remapped+backfilling, 1 active+undersized+remapped, 1 active+remapped+backfill_wait, 1 active+remapped, 120 active+clean; 2.2 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 371 op/s; 778/107688 objects misplaced (0.722%); 27 MiB/s, 437 objects/s recovering 2023-09-23T20:48:51.490 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:51.602 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:48:51.687 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:48:51.690 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:51.691 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:48:51.792 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:48:51.822 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:51.866 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:48:51.869 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:51.870 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:48:51.970 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:48:52.001 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:52.028 INFO:teuthology.orchestra.run.smithi018.stdout:ERROR: (22) Invalid argument 2023-09-23T20:48:52.028 INFO:teuthology.orchestra.run.smithi018.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:48:52.046 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:48:52.049 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:52.049 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:48:52.150 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:48:52.186 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:52.391 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:48:52.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:52 smithi018 ceph-mon[184144]: Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2023-09-23T20:48:52.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:52 smithi018 ceph-mon[184144]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:48:52.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:52 smithi018 ceph-mon[184144]: failed to encode map e595 with expected crc 2023-09-23T20:48:52.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:52 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:48:52.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:52 smithi018 ceph-mon[184144]: osdmap e595: 8 total, 8 up, 8 in 2023-09-23T20:48:52.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:52 smithi018 ceph-mon[184144]: failed to encode map e595 with expected crc 2023-09-23T20:48:52.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:52 smithi018 ceph-mon[184144]: failed to encode map e595 with expected crc 2023-09-23T20:48:52.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:52 smithi018 ceph-mon[184144]: failed to encode map e595 with expected crc 2023-09-23T20:48:52.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:52 smithi018 ceph-mon[184144]: 10.10 starting backfill to osd.7 from (0'0,0'0] MAX to 592'10747 2023-09-23T20:48:52.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:52 smithi018 ceph-mon[184144]: failed to encode map e595 with expected crc 2023-09-23T20:48:52.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:52 smithi018 ceph-mon[184144]: failed to encode map e595 with expected crc 2023-09-23T20:48:52.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:52 smithi018 ceph-mon[179647]: Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2023-09-23T20:48:52.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:52 smithi018 ceph-mon[179647]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:48:52.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:52 smithi018 ceph-mon[179647]: failed to encode map e595 with expected crc 2023-09-23T20:48:52.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:52 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:48:52.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:52 smithi018 ceph-mon[179647]: osdmap e595: 8 total, 8 up, 8 in 2023-09-23T20:48:52.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:52 smithi018 ceph-mon[179647]: failed to encode map e595 with expected crc 2023-09-23T20:48:52.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:52 smithi018 ceph-mon[179647]: failed to encode map e595 with expected crc 2023-09-23T20:48:52.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:52 smithi018 ceph-mon[179647]: failed to encode map e595 with expected crc 2023-09-23T20:48:52.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:52 smithi018 ceph-mon[179647]: 10.10 starting backfill to osd.7 from (0'0,0'0] MAX to 592'10747 2023-09-23T20:48:52.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:52 smithi018 ceph-mon[179647]: failed to encode map e595 with expected crc 2023-09-23T20:48:52.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:52 smithi018 ceph-mon[179647]: failed to encode map e595 with expected crc 2023-09-23T20:48:52.394 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:52.394 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:48:52.443 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T20:48:52.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:52 smithi149 ceph-mon[170394]: Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2023-09-23T20:48:52.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:52 smithi149 ceph-mon[170394]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:48:52.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:52 smithi149 ceph-mon[170394]: failed to encode map e595 with expected crc 2023-09-23T20:48:52.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:52 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:48:52.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:52 smithi149 ceph-mon[170394]: osdmap e595: 8 total, 8 up, 8 in 2023-09-23T20:48:52.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:52 smithi149 ceph-mon[170394]: failed to encode map e595 with expected crc 2023-09-23T20:48:52.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:52 smithi149 ceph-mon[170394]: failed to encode map e595 with expected crc 2023-09-23T20:48:52.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:52 smithi149 ceph-mon[170394]: failed to encode map e595 with expected crc 2023-09-23T20:48:52.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:52 smithi149 ceph-mon[170394]: 10.10 starting backfill to osd.7 from (0'0,0'0] MAX to 592'10747 2023-09-23T20:48:52.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:52 smithi149 ceph-mon[170394]: failed to encode map e595 with expected crc 2023-09-23T20:48:52.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:52 smithi149 ceph-mon[170394]: failed to encode map e595 with expected crc 2023-09-23T20:48:52.495 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:48:52.495 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:48:52.523 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:52.564 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:48:52.568 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:52.569 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:48:52.632 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 7, 3, 6, 5, 4, 2] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T20:48:52.661 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:48:52.661 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_5 2023-09-23T20:48:52.662 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:48:52.669 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:48:52.733 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:52.780 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:48:52.783 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:52.784 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:48:52.885 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:48:52.915 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:52.954 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:48:52.958 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:52.959 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:48:53.059 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:48:53.089 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:53.129 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:48:53.133 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:53.133 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:48:53.234 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:48:53.305 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:53.343 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:53 smithi018 ceph-mon[179647]: osdmap e596: 8 total, 8 up, 8 in 2023-09-23T20:48:53.343 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:53 smithi018 ceph-mon[179647]: failed to encode map e596 with expected crc 2023-09-23T20:48:53.343 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:53 smithi018 ceph-mon[179647]: failed to encode map e596 with expected crc 2023-09-23T20:48:53.344 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:53 smithi018 ceph-mon[179647]: failed to encode map e596 with expected crc 2023-09-23T20:48:53.344 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:53 smithi018 ceph-mon[179647]: failed to encode map e596 with expected crc 2023-09-23T20:48:53.344 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:53 smithi018 ceph-mon[179647]: pgmap v1495: 129 pgs: 2 active+remapped+backfilling, 7 peering, 3 remapped+peering, 1 active+undersized+remapped, 1 active+remapped, 115 active+clean; 2.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 527 op/s; 3076/103161 objects misplaced (2.982%); 12 MiB/s, 2 keys/s, 177 objects/s recovering 2023-09-23T20:48:53.344 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:53 smithi018 ceph-mon[184144]: osdmap e596: 8 total, 8 up, 8 in 2023-09-23T20:48:53.344 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:53 smithi018 ceph-mon[184144]: failed to encode map e596 with expected crc 2023-09-23T20:48:53.344 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:53 smithi018 ceph-mon[184144]: failed to encode map e596 with expected crc 2023-09-23T20:48:53.344 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:53 smithi018 ceph-mon[184144]: failed to encode map e596 with expected crc 2023-09-23T20:48:53.344 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:53 smithi018 ceph-mon[184144]: failed to encode map e596 with expected crc 2023-09-23T20:48:53.344 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:53 smithi018 ceph-mon[184144]: pgmap v1495: 129 pgs: 2 active+remapped+backfilling, 7 peering, 3 remapped+peering, 1 active+undersized+remapped, 1 active+remapped, 115 active+clean; 2.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 527 op/s; 3076/103161 objects misplaced (2.982%); 12 MiB/s, 2 keys/s, 177 objects/s recovering 2023-09-23T20:48:53.372 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:48:53.376 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:53.376 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:48:53.389 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:48:53.468 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:53 smithi149 ceph-mon[170394]: osdmap e596: 8 total, 8 up, 8 in 2023-09-23T20:48:53.469 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:53 smithi149 ceph-mon[170394]: failed to encode map e596 with expected crc 2023-09-23T20:48:53.469 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:53 smithi149 ceph-mon[170394]: failed to encode map e596 with expected crc 2023-09-23T20:48:53.469 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:53 smithi149 ceph-mon[170394]: failed to encode map e596 with expected crc 2023-09-23T20:48:53.469 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:53 smithi149 ceph-mon[170394]: failed to encode map e596 with expected crc 2023-09-23T20:48:53.469 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:53 smithi149 ceph-mon[170394]: pgmap v1495: 129 pgs: 2 active+remapped+backfilling, 7 peering, 3 remapped+peering, 1 active+undersized+remapped, 1 active+remapped, 115 active+clean; 2.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 527 op/s; 3076/103161 objects misplaced (2.982%); 12 MiB/s, 2 keys/s, 177 objects/s recovering 2023-09-23T20:48:53.477 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:48:53.533 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:53.654 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:48:53.664 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:53.665 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:48:53.720 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T20:48:53.766 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:48:53.930 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:53.980 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:48:54.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:54 smithi018 ceph-mon[184144]: osdmap e597: 8 total, 8 up, 8 in 2023-09-23T20:48:54.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:54 smithi018 ceph-mon[184144]: failed to encode map e597 with expected crc 2023-09-23T20:48:54.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:54 smithi018 ceph-mon[184144]: failed to encode map e597 with expected crc 2023-09-23T20:48:54.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:54 smithi018 ceph-mon[184144]: failed to encode map e597 with expected crc 2023-09-23T20:48:54.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:54 smithi018 ceph-mon[184144]: failed to encode map e597 with expected crc 2023-09-23T20:48:54.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:54 smithi018 ceph-mon[184144]: failed to encode map e597 with expected crc 2023-09-23T20:48:54.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:54 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:48:54.175 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:54 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:48:54.429 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:48:54.451 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:48:54.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:54 smithi149 ceph-mon[170394]: osdmap e597: 8 total, 8 up, 8 in 2023-09-23T20:48:54.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:54 smithi149 ceph-mon[170394]: failed to encode map e597 with expected crc 2023-09-23T20:48:54.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:54 smithi149 ceph-mon[170394]: failed to encode map e597 with expected crc 2023-09-23T20:48:54.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:54 smithi149 ceph-mon[170394]: failed to encode map e597 with expected crc 2023-09-23T20:48:54.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:54 smithi149 ceph-mon[170394]: failed to encode map e597 with expected crc 2023-09-23T20:48:54.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:54 smithi149 ceph-mon[170394]: failed to encode map e597 with expected crc 2023-09-23T20:48:54.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:54 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:48:54.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:54 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:48:54.476 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:48:54.499 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:48:54.514 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:48:54.518 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:54.519 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:48:54.519 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:54 smithi018 ceph-mon[179647]: osdmap e597: 8 total, 8 up, 8 in 2023-09-23T20:48:54.519 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:54 smithi018 ceph-mon[179647]: failed to encode map e597 with expected crc 2023-09-23T20:48:54.519 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:54 smithi018 ceph-mon[179647]: failed to encode map e597 with expected crc 2023-09-23T20:48:54.519 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:54 smithi018 ceph-mon[179647]: failed to encode map e597 with expected crc 2023-09-23T20:48:54.519 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:54 smithi018 ceph-mon[179647]: failed to encode map e597 with expected crc 2023-09-23T20:48:54.520 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:54 smithi018 ceph-mon[179647]: failed to encode map e597 with expected crc 2023-09-23T20:48:54.520 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:54 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:48:54.520 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:54 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:48:54.525 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:48:54.549 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:48:54.576 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:48:54.598 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:48:54.599 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:48:54.600 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:48:54.600 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:48:54.600 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:48:54.600 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:48:54.600 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:48:54.600 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:48:54.600 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:48:54.619 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:48:54.648 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:54.681 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:48:54.685 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:54.685 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:48:54.786 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:48:54.817 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:55.072 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:48:54 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:48:54] "GET /metrics HTTP/1.1" 200 36608 "" "Prometheus/2.43.0" 2023-09-23T20:48:55.232 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:48:55.235 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:55.236 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:48:55.336 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:48:55.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:55 smithi018 ceph-mon[179647]: osdmap e598: 8 total, 8 up, 8 in 2023-09-23T20:48:55.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:55 smithi018 ceph-mon[179647]: failed to encode map e598 with expected crc 2023-09-23T20:48:55.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:55 smithi018 ceph-mon[179647]: failed to encode map e598 with expected crc 2023-09-23T20:48:55.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:55 smithi018 ceph-mon[179647]: pgmap v1498: 129 pgs: 1 active+remapped+backfilling, 4 peering, 1 remapped+peering, 123 active+clean; 2.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 444 op/s; 2075/102348 objects misplaced (2.027%); 8.3 MiB/s, 2 keys/s, 119 objects/s recovering 2023-09-23T20:48:55.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:55 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1775188829' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:48:55.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:55 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:48:55.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:55 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:48:55.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:55 smithi018 ceph-mon[184144]: osdmap e598: 8 total, 8 up, 8 in 2023-09-23T20:48:55.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:55 smithi018 ceph-mon[184144]: failed to encode map e598 with expected crc 2023-09-23T20:48:55.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:55 smithi018 ceph-mon[184144]: failed to encode map e598 with expected crc 2023-09-23T20:48:55.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:55 smithi018 ceph-mon[184144]: pgmap v1498: 129 pgs: 1 active+remapped+backfilling, 4 peering, 1 remapped+peering, 123 active+clean; 2.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 444 op/s; 2075/102348 objects misplaced (2.027%); 8.3 MiB/s, 2 keys/s, 119 objects/s recovering 2023-09-23T20:48:55.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:55 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1775188829' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:48:55.394 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:55 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:48:55.394 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:55 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:48:55.411 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:55.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:55 smithi149 ceph-mon[170394]: osdmap e598: 8 total, 8 up, 8 in 2023-09-23T20:48:55.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:55 smithi149 ceph-mon[170394]: failed to encode map e598 with expected crc 2023-09-23T20:48:55.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:55 smithi149 ceph-mon[170394]: failed to encode map e598 with expected crc 2023-09-23T20:48:55.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:55 smithi149 ceph-mon[170394]: pgmap v1498: 129 pgs: 1 active+remapped+backfilling, 4 peering, 1 remapped+peering, 123 active+clean; 2.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 444 op/s; 2075/102348 objects misplaced (2.027%); 8.3 MiB/s, 2 keys/s, 119 objects/s recovering 2023-09-23T20:48:55.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:55 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1775188829' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:48:55.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:55 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:48:55.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:55 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:48:55.637 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:48:55.651 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:55.651 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:48:55.752 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:48:55.838 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:55.999 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:48:56.014 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:56.014 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:48:56.033 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2023-09-23T20:48:56.033 INFO:teuthology.orchestra.run.smithi018.stdout: "ops": [], 2023-09-23T20:48:56.034 INFO:teuthology.orchestra.run.smithi018.stdout: "complaint_time": 30, 2023-09-23T20:48:56.034 INFO:teuthology.orchestra.run.smithi018.stdout: "num_blocked_ops": 0 2023-09-23T20:48:56.034 INFO:teuthology.orchestra.run.smithi018.stdout:} 2023-09-23T20:48:56.107 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:48:56.115 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:48:56.152 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:56.206 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:48:56.210 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:56.210 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:48:56.311 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:48:56.373 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:56 smithi018 ceph-mon[179647]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2023-09-23T20:48:56.373 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:56 smithi018 ceph-mon[179647]: failed to encode map e599 with expected crc 2023-09-23T20:48:56.373 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:56 smithi018 ceph-mon[179647]: osdmap e599: 8 total, 8 up, 8 in 2023-09-23T20:48:56.373 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:56 smithi018 ceph-mon[179647]: failed to encode map e599 with expected crc 2023-09-23T20:48:56.373 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:56 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:48:56.373 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:56 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:48:56.374 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:56 smithi018 ceph-mon[184144]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2023-09-23T20:48:56.374 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:56 smithi018 ceph-mon[184144]: failed to encode map e599 with expected crc 2023-09-23T20:48:56.374 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:56 smithi018 ceph-mon[184144]: osdmap e599: 8 total, 8 up, 8 in 2023-09-23T20:48:56.374 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:56 smithi018 ceph-mon[184144]: failed to encode map e599 with expected crc 2023-09-23T20:48:56.374 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:56 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:48:56.374 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:56 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:48:56.382 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:56.469 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:56 smithi149 ceph-mon[170394]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2023-09-23T20:48:56.469 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:56 smithi149 ceph-mon[170394]: failed to encode map e599 with expected crc 2023-09-23T20:48:56.469 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:56 smithi149 ceph-mon[170394]: osdmap e599: 8 total, 8 up, 8 in 2023-09-23T20:48:56.469 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:56 smithi149 ceph-mon[170394]: failed to encode map e599 with expected crc 2023-09-23T20:48:56.469 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:56 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:48:56.470 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:56 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:48:57.109 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:48:57.631 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:48:57.656 INFO:teuthology.orchestra.run.smithi018.stderr:dumped all 2023-09-23T20:48:57.772 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:48:57.782 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:57.783 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:48:57.790 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T20:48:57.884 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:48:57.938 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:57 smithi149 ceph-mon[170394]: pgmap v1500: 129 pgs: 1 peering, 128 active+clean; 2.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 356 op/s; 16 MiB/s, 253 objects/s recovering 2023-09-23T20:48:57.945 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:58.020 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:48:58.025 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:58.026 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:48:58.127 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:48:58.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:57 smithi018 ceph-mon[184144]: pgmap v1500: 129 pgs: 1 peering, 128 active+clean; 2.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 356 op/s; 16 MiB/s, 253 objects/s recovering 2023-09-23T20:48:58.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:57 smithi018 ceph-mon[179647]: pgmap v1500: 129 pgs: 1 peering, 128 active+clean; 2.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 356 op/s; 16 MiB/s, 253 objects/s recovering 2023-09-23T20:48:58.156 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:58.208 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:48:58.214 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:48:58.215 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:48:58.315 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:48:58.385 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:48:58.968 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:58 smithi149 ceph-mon[170394]: from='client.46928 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:48:58.968 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:58 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "18"}]: dispatch 2023-09-23T20:48:59.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:58 smithi018 ceph-mon[184144]: from='client.46928 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:48:59.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:58 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "18"}]: dispatch 2023-09-23T20:48:59.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:58 smithi018 ceph-mon[179647]: from='client.46928 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:48:59.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:58 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "18"}]: dispatch 2023-09-23T20:48:59.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:59 smithi149 ceph-mon[170394]: pgmap v1501: 129 pgs: 129 active+clean; 2.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 351 op/s; 11 MiB/s, 176 objects/s recovering 2023-09-23T20:48:59.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:59 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "18"}]': finished 2023-09-23T20:48:59.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:59 smithi149 ceph-mon[170394]: osdmap e600: 8 total, 8 up, 8 in 2023-09-23T20:48:59.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:59 smithi149 ceph-mon[170394]: failed to encode map e600 with expected crc 2023-09-23T20:48:59.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:59 smithi149 ceph-mon[170394]: failed to encode map e600 with expected crc 2023-09-23T20:48:59.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:59 smithi149 ceph-mon[170394]: failed to encode map e600 with expected crc 2023-09-23T20:48:59.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:59 smithi149 ceph-mon[170394]: failed to encode map e600 with expected crc 2023-09-23T20:48:59.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:59 smithi149 ceph-mon[170394]: failed to encode map e600 with expected crc 2023-09-23T20:48:59.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:59 smithi149 ceph-mon[170394]: failed to encode map e600 with expected crc 2023-09-23T20:48:59.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:59 smithi149 ceph-mon[170394]: failed to encode map e600 with expected crc 2023-09-23T20:48:59.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:59 smithi149 ceph-mon[170394]: failed to encode map e600 with expected crc 2023-09-23T20:48:59.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:59 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:48:59.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:59 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:48:59.968 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:59 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi149", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:48:59.968 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:59 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:48:59.968 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:59 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:48:59.968 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:59 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:48:59.968 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:59 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:48:59.968 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:48:59 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:49:00.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:59 smithi018 ceph-mon[184144]: pgmap v1501: 129 pgs: 129 active+clean; 2.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 351 op/s; 11 MiB/s, 176 objects/s recovering 2023-09-23T20:49:00.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:59 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "18"}]': finished 2023-09-23T20:49:00.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:59 smithi018 ceph-mon[184144]: osdmap e600: 8 total, 8 up, 8 in 2023-09-23T20:49:00.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:59 smithi018 ceph-mon[184144]: failed to encode map e600 with expected crc 2023-09-23T20:49:00.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:59 smithi018 ceph-mon[184144]: failed to encode map e600 with expected crc 2023-09-23T20:49:00.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:59 smithi018 ceph-mon[184144]: failed to encode map e600 with expected crc 2023-09-23T20:49:00.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:59 smithi018 ceph-mon[184144]: failed to encode map e600 with expected crc 2023-09-23T20:49:00.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:59 smithi018 ceph-mon[184144]: failed to encode map e600 with expected crc 2023-09-23T20:49:00.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:59 smithi018 ceph-mon[184144]: failed to encode map e600 with expected crc 2023-09-23T20:49:00.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:59 smithi018 ceph-mon[184144]: failed to encode map e600 with expected crc 2023-09-23T20:49:00.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:59 smithi018 ceph-mon[184144]: failed to encode map e600 with expected crc 2023-09-23T20:49:00.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:59 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:49:00.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:59 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:49:00.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:59 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi149", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:49:00.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:59 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:49:00.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:59 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:49:00.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:59 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:49:00.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:59 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:49:00.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:48:59 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:49:00.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:59 smithi018 ceph-mon[179647]: pgmap v1501: 129 pgs: 129 active+clean; 2.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 351 op/s; 11 MiB/s, 176 objects/s recovering 2023-09-23T20:49:00.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:59 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "18"}]': finished 2023-09-23T20:49:00.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:59 smithi018 ceph-mon[179647]: osdmap e600: 8 total, 8 up, 8 in 2023-09-23T20:49:00.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:59 smithi018 ceph-mon[179647]: failed to encode map e600 with expected crc 2023-09-23T20:49:00.144 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:59 smithi018 ceph-mon[179647]: failed to encode map e600 with expected crc 2023-09-23T20:49:00.144 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:59 smithi018 ceph-mon[179647]: failed to encode map e600 with expected crc 2023-09-23T20:49:00.144 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:59 smithi018 ceph-mon[179647]: failed to encode map e600 with expected crc 2023-09-23T20:49:00.144 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:59 smithi018 ceph-mon[179647]: failed to encode map e600 with expected crc 2023-09-23T20:49:00.144 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:59 smithi018 ceph-mon[179647]: failed to encode map e600 with expected crc 2023-09-23T20:49:00.144 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:59 smithi018 ceph-mon[179647]: failed to encode map e600 with expected crc 2023-09-23T20:49:00.144 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:59 smithi018 ceph-mon[179647]: failed to encode map e600 with expected crc 2023-09-23T20:49:00.144 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:59 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:49:00.144 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:59 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:49:00.144 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:59 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi149", "name": "osd_memory_target"}]: dispatch 2023-09-23T20:49:00.144 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:59 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:49:00.145 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:59 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:49:00.145 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:59 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:49:00.145 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:59 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:49:00.145 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:48:59 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:49:00.173 INFO:teuthology.orchestra.run.smithi018.stdout:{"pg_ready":true,"pg_map":{"version":1500,"stamp":"2023-09-23T20:48:56.332483+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2159030098,"num_objects":32899,"num_object_clones":0,"num_object_copies":98697,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":32899,"num_whiteouts":0,"num_read":6043,"num_read_kb":13311,"num_write":177052,"num_write_kb":6700569,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":13772,"num_bytes_recovered":911770666,"num_keys_recovered":80,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":166614,"ondisk_log_size":166614,"up":387,"acting":387,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":396,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":12785664,"kb_used_data":6962880,"kb_used_omap":32,"kb_used_meta":5822623,"kb_avail":737175552,"statfs":{"total":767960285184,"available":754867765248,"internally_reserved":0,"allocated":7129989120,"data_stored":7112143409,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32773,"internal_metadata":5962366971},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,0,0,0,1,3],"upper_bound":4096},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-97517568,"num_objects":-1488,"num_object_clones":0,"num_object_copies":-4464,"num_objects_missing_on_primary":0,"num_objects_missing":-1,"num_objects_degraded":0,"num_objects_misplaced":-3076,"num_objects_unfound":0,"num_objects_dirty":-1488,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1488,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1058,"num_bytes_recovered":69403168,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"4.169827"},"pg_stats":[{"pgid":"10.13","version":"597'11014","reported_seq":11535,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T20:48:53.303060+0000","last_change":"2023-09-23T20:48:48.187746+0000","last_active":"2023-09-23T20:48:53.303060+0000","last_peered":"2023-09-23T20:48:53.303060+0000","last_clean":"2023-09-23T20:48:53.303060+0000","last_became_active":"2023-09-23T20:48:48.156992+0000","last_became_peered":"2023-09-23T20:48:48.156992+0000","last_unstale":"2023-09-23T20:48:53.303060+0000","last_undegraded":"2023-09-23T20:48:53.303060+0000","last_fullsized":"2023-09-23T20:48:53.303060+0000","mapping_epoch":591,"log_start":"536'1300","ondisk_log_start":"536'1300","created":584,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":9714,"log_dups_size":0,"ondisk_log_size":9714,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70320128,"num_objects":1073,"num_object_clones":0,"num_object_copies":3219,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1073,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5601,"num_write_kb":213568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":104,"num_bytes_recovered":6815744,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.18","version":"90'161","reported_seq":639,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T20:48:53.171966+0000","last_change":"2023-09-23T20:48:51.176302+0000","last_active":"2023-09-23T20:48:53.171966+0000","last_peered":"2023-09-23T20:48:53.171966+0000","last_clean":"2023-09-23T20:48:53.171966+0000","last_became_active":"2023-09-23T20:48:51.175408+0000","last_became_peered":"2023-09-23T20:48:51.175408+0000","last_unstale":"2023-09-23T20:48:53.171966+0000","last_undegraded":"2023-09-23T20:48:53.171966+0000","last_fullsized":"2023-09-23T20:48:53.171966+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:19.677209+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:19.677209+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:43:11.718198+0000","scrub_duration":0.033122453000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"69'1","reported_seq":76,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T20:48:54.188626+0000","last_change":"2023-09-23T20:48:48.148250+0000","last_active":"2023-09-23T20:48:54.188626+0000","last_peered":"2023-09-23T20:48:54.188626+0000","last_clean":"2023-09-23T20:48:54.188626+0000","last_became_active":"2023-09-23T20:48:48.144271+0000","last_became_peered":"2023-09-23T20:48:48.144271+0000","last_unstale":"2023-09-23T20:48:54.188626+0000","last_undegraded":"2023-09-23T20:48:54.188626+0000","last_fullsized":"2023-09-23T20:48:54.188626+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:59:36.502986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":432,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T20:48:53.175394+0000","last_change":"2023-09-23T20:47:09.197577+0000","last_active":"2023-09-23T20:48:53.175394+0000","last_peered":"2023-09-23T20:48:53.175394+0000","last_clean":"2023-09-23T20:48:53.175394+0000","last_became_active":"2023-09-23T20:47:09.197280+0000","last_became_peered":"2023-09-23T20:47:09.197280+0000","last_unstale":"2023-09-23T20:48:53.175394+0000","last_undegraded":"2023-09-23T20:48:53.175394+0000","last_fullsized":"2023-09-23T20:48:53.175394+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:55:04.346224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.10","version":"599'10844","reported_seq":14212,"reported_epoch":599,"state":"active+clean","last_fresh":"2023-09-23T20:48:55.729471+0000","last_change":"2023-09-23T20:48:55.233799+0000","last_active":"2023-09-23T20:48:55.729471+0000","last_peered":"2023-09-23T20:48:55.729471+0000","last_clean":"2023-09-23T20:48:55.729471+0000","last_became_active":"2023-09-23T20:48:55.189956+0000","last_became_peered":"2023-09-23T20:48:55.189956+0000","last_unstale":"2023-09-23T20:48:55.729471+0000","last_undegraded":"2023-09-23T20:48:55.729471+0000","last_fullsized":"2023-09-23T20:48:55.729471+0000","mapping_epoch":598,"log_start":"542'4945","ondisk_log_start":"542'4945","created":584,"last_epoch_clean":599,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":5899,"log_dups_size":0,"ondisk_log_size":5899,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":62685184,"num_objects":957,"num_object_clones":0,"num_object_copies":2871,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":957,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5539,"num_write_kb":207840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1322,"num_bytes_recovered":86638592,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1b","version":"90'161","reported_seq":602,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T20:48:54.184238+0000","last_change":"2023-09-23T20:47:33.523278+0000","last_active":"2023-09-23T20:48:54.184238+0000","last_peered":"2023-09-23T20:48:54.184238+0000","last_clean":"2023-09-23T20:48:54.184238+0000","last_became_active":"2023-09-23T20:47:11.622087+0000","last_became_peered":"2023-09-23T20:47:11.622087+0000","last_unstale":"2023-09-23T20:48:54.184238+0000","last_undegraded":"2023-09-23T20:48:54.184238+0000","last_fullsized":"2023-09-23T20:48:54.184238+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:47:33.523187+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:36:05.442288+0000","last_clean_scrub_stamp":"2023-09-23T20:47:33.523187+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:32:18.410526+0000","scrub_duration":0.013532127,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":586,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:48:52.164112+0000","last_change":"2023-09-23T20:47:09.173501+0000","last_active":"2023-09-23T20:48:52.164112+0000","last_peered":"2023-09-23T20:48:52.164112+0000","last_clean":"2023-09-23T20:48:52.164112+0000","last_became_active":"2023-09-23T20:47:09.173160+0000","last_became_peered":"2023-09-23T20:47:09.173160+0000","last_unstale":"2023-09-23T20:48:52.164112+0000","last_undegraded":"2023-09-23T20:48:52.164112+0000","last_fullsized":"2023-09-23T20:48:52.164112+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:44:36.132093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":554,"reported_epoch":594,"state":"active+clean","last_fresh":"2023-09-23T20:48:50.169651+0000","last_change":"2023-09-23T20:43:46.521939+0000","last_active":"2023-09-23T20:48:50.169651+0000","last_peered":"2023-09-23T20:48:50.169651+0000","last_clean":"2023-09-23T20:48:50.169651+0000","last_became_active":"2023-09-23T20:43:46.521185+0000","last_became_peered":"2023-09-23T20:43:46.521185+0000","last_unstale":"2023-09-23T20:48:50.169651+0000","last_undegraded":"2023-09-23T20:48:50.169651+0000","last_fullsized":"2023-09-23T20:48:50.169651+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:59:25.045627+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.11","version":"599'10881","reported_seq":11012,"reported_epoch":599,"state":"active+clean","last_fresh":"2023-09-23T20:48:55.789849+0000","last_change":"2023-09-23T20:48:40.152501+0000","last_active":"2023-09-23T20:48:55.789849+0000","last_peered":"2023-09-23T20:48:55.789849+0000","last_clean":"2023-09-23T20:48:55.789849+0000","last_became_active":"2023-09-23T20:48:40.123443+0000","last_became_peered":"2023-09-23T20:48:40.123443+0000","last_unstale":"2023-09-23T20:48:55.789849+0000","last_undegraded":"2023-09-23T20:48:55.789849+0000","last_fullsized":"2023-09-23T20:48:55.789849+0000","mapping_epoch":584,"log_start":"543'5336","ondisk_log_start":"543'5336","created":584,"last_epoch_clean":585,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":5545,"log_dups_size":0,"ondisk_log_size":5545,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":66322432,"num_objects":1012,"num_object_clones":0,"num_object_copies":3036,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1012,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5540,"num_write_kb":209664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":393216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1a","version":"90'161","reported_seq":487,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T20:48:54.188616+0000","last_change":"2023-09-23T20:47:31.475001+0000","last_active":"2023-09-23T20:48:54.188616+0000","last_peered":"2023-09-23T20:48:54.188616+0000","last_clean":"2023-09-23T20:48:54.188616+0000","last_became_active":"2023-09-23T20:47:11.617942+0000","last_became_peered":"2023-09-23T20:47:11.617942+0000","last_unstale":"2023-09-23T20:48:54.188616+0000","last_undegraded":"2023-09-23T20:48:54.188616+0000","last_fullsized":"2023-09-23T20:48:54.188616+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:47:31.474914+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:47:31.474914+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:52:05.169219+0000","scrub_duration":0.0026262870000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":394,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T20:48:53.174246+0000","last_change":"2023-09-23T20:37:43.584414+0000","last_active":"2023-09-23T20:48:53.174246+0000","last_peered":"2023-09-23T20:48:53.174246+0000","last_clean":"2023-09-23T20:48:53.174246+0000","last_became_active":"2023-09-23T20:37:43.584116+0000","last_became_peered":"2023-09-23T20:37:43.584116+0000","last_unstale":"2023-09-23T20:48:53.174246+0000","last_undegraded":"2023-09-23T20:48:53.174246+0000","last_fullsized":"2023-09-23T20:48:53.174246+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:17:54.258172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":214,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T20:48:54.197809+0000","last_change":"2023-09-23T20:38:37.457427+0000","last_active":"2023-09-23T20:48:54.197809+0000","last_peered":"2023-09-23T20:48:54.197809+0000","last_clean":"2023-09-23T20:48:54.197809+0000","last_became_active":"2023-09-23T20:38:37.457099+0000","last_became_peered":"2023-09-23T20:38:37.457099+0000","last_unstale":"2023-09-23T20:48:54.197809+0000","last_undegraded":"2023-09-23T20:48:54.197809+0000","last_fullsized":"2023-09-23T20:48:54.197809+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:31:27.084994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"90'161","reported_seq":811,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T20:48:54.191653+0000","last_change":"2023-09-23T20:40:22.449754+0000","last_active":"2023-09-23T20:48:54.191653+0000","last_peered":"2023-09-23T20:48:54.191653+0000","last_clean":"2023-09-23T20:48:54.191653+0000","last_became_active":"2023-09-23T20:40:22.448838+0000","last_became_peered":"2023-09-23T20:40:22.448838+0000","last_unstale":"2023-09-23T20:48:54.191653+0000","last_undegraded":"2023-09-23T20:48:54.191653+0000","last_fullsized":"2023-09-23T20:48:54.191653+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:03.411025+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:03.411025+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:22:07.948856+0000","scrub_duration":0.001802922,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":587,"reported_epoch":594,"state":"active+clean","last_fresh":"2023-09-23T20:48:50.160585+0000","last_change":"2023-09-23T20:43:46.522325+0000","last_active":"2023-09-23T20:48:50.160585+0000","last_peered":"2023-09-23T20:48:50.160585+0000","last_clean":"2023-09-23T20:48:50.160585+0000","last_became_active":"2023-09-23T20:43:46.521996+0000","last_became_peered":"2023-09-23T20:43:46.521996+0000","last_unstale":"2023-09-23T20:48:50.160585+0000","last_undegraded":"2023-09-23T20:48:50.160585+0000","last_fullsized":"2023-09-23T20:48:50.160585+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:51:17.629956+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":31,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T20:48:53.174042+0000","last_change":"2023-09-23T20:48:23.918092+0000","last_active":"2023-09-23T20:48:53.174042+0000","last_peered":"2023-09-23T20:48:53.174042+0000","last_clean":"2023-09-23T20:48:53.174042+0000","last_became_active":"2023-09-23T20:48:23.917386+0000","last_became_peered":"2023-09-23T20:48:23.917386+0000","last_unstale":"2023-09-23T20:48:53.174042+0000","last_undegraded":"2023-09-23T20:48:53.174042+0000","last_fullsized":"2023-09-23T20:48:53.174042+0000","mapping_epoch":577,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":578,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:06:09.587517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"90'161","reported_seq":587,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T20:48:53.172407+0000","last_change":"2023-09-23T20:48:51.169725+0000","last_active":"2023-09-23T20:48:53.172407+0000","last_peered":"2023-09-23T20:48:53.172407+0000","last_clean":"2023-09-23T20:48:53.172407+0000","last_became_active":"2023-09-23T20:48:51.169314+0000","last_became_peered":"2023-09-23T20:48:51.169314+0000","last_unstale":"2023-09-23T20:48:53.172407+0000","last_undegraded":"2023-09-23T20:48:53.172407+0000","last_fullsized":"2023-09-23T20:48:53.172407+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:47.479012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:47.479012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":435,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T20:48:54.192397+0000","last_change":"2023-09-23T20:43:16.780478+0000","last_active":"2023-09-23T20:48:54.192397+0000","last_peered":"2023-09-23T20:48:54.192397+0000","last_clean":"2023-09-23T20:48:54.192397+0000","last_became_active":"2023-09-23T20:43:16.780183+0000","last_became_peered":"2023-09-23T20:43:16.780183+0000","last_unstale":"2023-09-23T20:48:54.192397+0000","last_undegraded":"2023-09-23T20:48:54.192397+0000","last_fullsized":"2023-09-23T20:48:54.192397+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:38:30.102066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":425,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T20:48:53.174699+0000","last_change":"2023-09-23T20:38:02.542034+0000","last_active":"2023-09-23T20:48:53.174699+0000","last_peered":"2023-09-23T20:48:53.174699+0000","last_clean":"2023-09-23T20:48:53.174699+0000","last_became_active":"2023-09-23T20:38:02.541575+0000","last_became_peered":"2023-09-23T20:38:02.541575+0000","last_unstale":"2023-09-23T20:48:53.174699+0000","last_undegraded":"2023-09-23T20:48:53.174699+0000","last_fullsized":"2023-09-23T20:48:53.174699+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:50:54.591529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"90'161","reported_seq":722,"reported_epoch":594,"state":"active+clean","last_fresh":"2023-09-23T20:48:50.159436+0000","last_change":"2023-09-23T20:44:14.669973+0000","last_active":"2023-09-23T20:48:50.159436+0000","last_peered":"2023-09-23T20:48:50.159436+0000","last_clean":"2023-09-23T20:48:50.159436+0000","last_became_active":"2023-09-23T20:43:46.519592+0000","last_became_peered":"2023-09-23T20:43:46.519592+0000","last_unstale":"2023-09-23T20:48:50.159436+0000","last_undegraded":"2023-09-23T20:48:50.159436+0000","last_fullsized":"2023-09-23T20:48:50.159436+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:14.669831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:14.669831+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:56:28.076617+0000","scrub_duration":0.0083441460000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":658,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T20:48:54.192062+0000","last_change":"2023-09-23T20:47:10.837423+0000","last_active":"2023-09-23T20:48:54.192062+0000","last_peered":"2023-09-23T20:48:54.192062+0000","last_clean":"2023-09-23T20:48:54.192062+0000","last_became_active":"2023-09-23T20:47:09.204324+0000","last_became_peered":"2023-09-23T20:47:09.204324+0000","last_unstale":"2023-09-23T20:48:54.192062+0000","last_undegraded":"2023-09-23T20:48:54.192062+0000","last_fullsized":"2023-09-23T20:48:54.192062+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":554,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:19.108914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":540,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":131,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:48:52.164528+0000","last_change":"2023-09-23T20:48:48.145175+0000","last_active":"2023-09-23T20:48:52.164528+0000","last_peered":"2023-09-23T20:48:52.164528+0000","last_clean":"2023-09-23T20:48:52.164528+0000","last_became_active":"2023-09-23T20:48:48.144841+0000","last_became_peered":"2023-09-23T20:48:48.144841+0000","last_unstale":"2023-09-23T20:48:52.164528+0000","last_undegraded":"2023-09-23T20:48:52.164528+0000","last_fullsized":"2023-09-23T20:48:52.164528+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:27:40.802917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"90'161","reported_seq":724,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:48:52.164906+0000","last_change":"2023-09-23T20:36:02.548387+0000","last_active":"2023-09-23T20:48:52.164906+0000","last_peered":"2023-09-23T20:48:52.164906+0000","last_clean":"2023-09-23T20:48:52.164906+0000","last_became_active":"2023-09-23T20:35:26.644787+0000","last_became_peered":"2023-09-23T20:35:26.644787+0000","last_unstale":"2023-09-23T20:48:52.164906+0000","last_undegraded":"2023-09-23T20:48:52.164906+0000","last_fullsized":"2023-09-23T20:48:52.164906+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:02.548285+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:02.548285+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:37:15.216553+0000","scrub_duration":0.0015514459999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":66,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T20:48:54.187576+0000","last_change":"2023-09-23T20:47:09.340979+0000","last_active":"2023-09-23T20:48:54.187576+0000","last_peered":"2023-09-23T20:48:54.187576+0000","last_clean":"2023-09-23T20:48:54.187576+0000","last_became_active":"2023-09-23T20:47:09.231222+0000","last_became_peered":"2023-09-23T20:47:09.231222+0000","last_unstale":"2023-09-23T20:48:54.187576+0000","last_undegraded":"2023-09-23T20:48:54.187576+0000","last_fullsized":"2023-09-23T20:48:54.187576+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:07:50.620006+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":56,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T20:48:54.187401+0000","last_change":"2023-09-23T20:47:09.186315+0000","last_active":"2023-09-23T20:48:54.187401+0000","last_peered":"2023-09-23T20:48:54.187401+0000","last_clean":"2023-09-23T20:48:54.187401+0000","last_became_active":"2023-09-23T20:47:09.184580+0000","last_became_peered":"2023-09-23T20:47:09.184580+0000","last_unstale":"2023-09-23T20:48:54.187401+0000","last_undegraded":"2023-09-23T20:48:54.187401+0000","last_fullsized":"2023-09-23T20:48:54.187401+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:44:35.553948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"90'161","reported_seq":850,"reported_epoch":594,"state":"active+clean","last_fresh":"2023-09-23T20:48:50.159656+0000","last_change":"2023-09-23T20:43:46.544727+0000","last_active":"2023-09-23T20:48:50.159656+0000","last_peered":"2023-09-23T20:48:50.159656+0000","last_clean":"2023-09-23T20:48:50.159656+0000","last_became_active":"2023-09-23T20:43:46.543952+0000","last_became_peered":"2023-09-23T20:43:46.543952+0000","last_unstale":"2023-09-23T20:48:50.159656+0000","last_undegraded":"2023-09-23T20:48:50.159656+0000","last_fullsized":"2023-09-23T20:48:50.159656+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:43.578666+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:43.578666+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:11:16.611920+0000","scrub_duration":0.0021890379999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":689,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T20:48:54.191296+0000","last_change":"2023-09-23T20:48:48.193666+0000","last_active":"2023-09-23T20:48:54.191296+0000","last_peered":"2023-09-23T20:48:54.191296+0000","last_clean":"2023-09-23T20:48:54.191296+0000","last_became_active":"2023-09-23T20:48:48.152081+0000","last_became_peered":"2023-09-23T20:48:48.152081+0000","last_unstale":"2023-09-23T20:48:54.191296+0000","last_undegraded":"2023-09-23T20:48:54.191296+0000","last_fullsized":"2023-09-23T20:48:54.191296+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:00:19.508124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":272,"num_keys_recovered":80,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":345,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T20:48:53.174916+0000","last_change":"2023-09-23T20:45:01.875931+0000","last_active":"2023-09-23T20:48:53.174916+0000","last_peered":"2023-09-23T20:48:53.174916+0000","last_clean":"2023-09-23T20:48:53.174916+0000","last_became_active":"2023-09-23T20:45:01.872839+0000","last_became_peered":"2023-09-23T20:45:01.872839+0000","last_unstale":"2023-09-23T20:48:53.174916+0000","last_undegraded":"2023-09-23T20:48:53.174916+0000","last_fullsized":"2023-09-23T20:48:53.174916+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:15:09.584507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"90'161","reported_seq":879,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T20:48:54.194805+0000","last_change":"2023-09-23T20:44:20.041766+0000","last_active":"2023-09-23T20:48:54.194805+0000","last_peered":"2023-09-23T20:48:54.194805+0000","last_clean":"2023-09-23T20:48:54.194805+0000","last_became_active":"2023-09-23T20:43:16.765015+0000","last_became_peered":"2023-09-23T20:43:16.765015+0000","last_unstale":"2023-09-23T20:48:54.194805+0000","last_undegraded":"2023-09-23T20:48:54.194805+0000","last_fullsized":"2023-09-23T20:48:54.194805+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:20.041690+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:20.041690+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:33:30.150109+0000","scrub_duration":0.0026180330000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":552,"reported_epoch":594,"state":"active+clean","last_fresh":"2023-09-23T20:48:50.169717+0000","last_change":"2023-09-23T20:43:46.521294+0000","last_active":"2023-09-23T20:48:50.169717+0000","last_peered":"2023-09-23T20:48:50.169717+0000","last_clean":"2023-09-23T20:48:50.169717+0000","last_became_active":"2023-09-23T20:43:46.521001+0000","last_became_peered":"2023-09-23T20:43:46.521001+0000","last_unstale":"2023-09-23T20:48:50.169717+0000","last_undegraded":"2023-09-23T20:48:50.169717+0000","last_fullsized":"2023-09-23T20:48:50.169717+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:01:35.126716+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":376,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T20:48:54.192188+0000","last_change":"2023-09-23T20:48:23.914091+0000","last_active":"2023-09-23T20:48:54.192188+0000","last_peered":"2023-09-23T20:48:54.192188+0000","last_clean":"2023-09-23T20:48:54.192188+0000","last_became_active":"2023-09-23T20:48:23.913768+0000","last_became_peered":"2023-09-23T20:48:23.913768+0000","last_unstale":"2023-09-23T20:48:54.192188+0000","last_undegraded":"2023-09-23T20:48:54.192188+0000","last_fullsized":"2023-09-23T20:48:54.192188+0000","mapping_epoch":577,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":578,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:19:38.311290+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.18","version":"595'10644","reported_seq":14095,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T20:48:51.470062+0000","last_change":"2023-09-23T20:48:49.178589+0000","last_active":"2023-09-23T20:48:51.470062+0000","last_peered":"2023-09-23T20:48:51.470062+0000","last_clean":"2023-09-23T20:48:51.470062+0000","last_became_active":"2023-09-23T20:48:49.155693+0000","last_became_peered":"2023-09-23T20:48:49.155693+0000","last_unstale":"2023-09-23T20:48:51.470062+0000","last_undegraded":"2023-09-23T20:48:51.470062+0000","last_fullsized":"2023-09-23T20:48:51.470062+0000","mapping_epoch":592,"log_start":"541'4571","ondisk_log_start":"541'4571","created":584,"last_epoch_clean":593,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":6073,"log_dups_size":0,"ondisk_log_size":6073,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":67862528,"num_objects":1036,"num_object_clones":0,"num_object_copies":3108,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1036,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5392,"num_write_kb":205664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":837,"num_bytes_recovered":54820864,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.13","version":"90'161","reported_seq":920,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:48:52.164369+0000","last_change":"2023-09-23T20:47:11.619371+0000","last_active":"2023-09-23T20:48:52.164369+0000","last_peered":"2023-09-23T20:48:52.164369+0000","last_clean":"2023-09-23T20:48:52.164369+0000","last_became_active":"2023-09-23T20:47:11.619006+0000","last_became_peered":"2023-09-23T20:47:11.619006+0000","last_unstale":"2023-09-23T20:48:52.164369+0000","last_undegraded":"2023-09-23T20:48:52.164369+0000","last_fullsized":"2023-09-23T20:48:52.164369+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:42.544308+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:42.544308+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:47:24.414807+0000","scrub_duration":0.0029958179999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":622,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T20:48:54.191826+0000","last_change":"2023-09-23T20:47:09.176613+0000","last_active":"2023-09-23T20:48:54.191826+0000","last_peered":"2023-09-23T20:48:54.191826+0000","last_clean":"2023-09-23T20:48:54.191826+0000","last_became_active":"2023-09-23T20:47:09.176187+0000","last_became_peered":"2023-09-23T20:47:09.176187+0000","last_unstale":"2023-09-23T20:48:54.191826+0000","last_undegraded":"2023-09-23T20:48:54.191826+0000","last_fullsized":"2023-09-23T20:48:54.191826+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:23:01.590989+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":561,"reported_epoch":594,"state":"active+clean","last_fresh":"2023-09-23T20:48:50.160342+0000","last_change":"2023-09-23T20:43:46.521735+0000","last_active":"2023-09-23T20:48:50.160342+0000","last_peered":"2023-09-23T20:48:50.160342+0000","last_clean":"2023-09-23T20:48:50.160342+0000","last_became_active":"2023-09-23T20:43:46.520762+0000","last_became_peered":"2023-09-23T20:43:46.520762+0000","last_unstale":"2023-09-23T20:48:50.160342+0000","last_undegraded":"2023-09-23T20:48:50.160342+0000","last_fullsized":"2023-09-23T20:48:50.160342+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:26:48.841930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.19","version":"597'10912","reported_seq":11026,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T20:48:53.284525+0000","last_change":"2023-09-23T20:48:49.228617+0000","last_active":"2023-09-23T20:48:53.284525+0000","last_peered":"2023-09-23T20:48:53.284525+0000","last_clean":"2023-09-23T20:48:53.284525+0000","last_became_active":"2023-09-23T20:48:49.188310+0000","last_became_peered":"2023-09-23T20:48:49.188310+0000","last_unstale":"2023-09-23T20:48:53.284525+0000","last_undegraded":"2023-09-23T20:48:53.284525+0000","last_fullsized":"2023-09-23T20:48:53.284525+0000","mapping_epoch":592,"log_start":"541'4623","ondisk_log_start":"541'4623","created":584,"last_epoch_clean":593,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":6289,"log_dups_size":0,"ondisk_log_size":6289,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70090752,"num_objects":1070,"num_object_clones":0,"num_object_copies":3210,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1070,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5534,"num_write_kb":211296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.12","version":"90'161","reported_seq":682,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:48:52.165030+0000","last_change":"2023-09-23T20:44:18.952808+0000","last_active":"2023-09-23T20:48:52.165030+0000","last_peered":"2023-09-23T20:48:52.165030+0000","last_clean":"2023-09-23T20:48:52.165030+0000","last_became_active":"2023-09-23T20:43:16.758652+0000","last_became_peered":"2023-09-23T20:43:16.758652+0000","last_unstale":"2023-09-23T20:48:52.165030+0000","last_undegraded":"2023-09-23T20:48:52.165030+0000","last_fullsized":"2023-09-23T20:48:52.165030+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:18.952693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:18.952693+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:12:19.496465+0000","scrub_duration":0.001699365,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":274,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T20:48:54.197725+0000","last_change":"2023-09-23T20:44:43.644290+0000","last_active":"2023-09-23T20:48:54.197725+0000","last_peered":"2023-09-23T20:48:54.197725+0000","last_clean":"2023-09-23T20:48:54.197725+0000","last_became_active":"2023-09-23T20:44:43.638313+0000","last_became_peered":"2023-09-23T20:44:43.638313+0000","last_unstale":"2023-09-23T20:48:54.197725+0000","last_undegraded":"2023-09-23T20:48:54.197725+0000","last_fullsized":"2023-09-23T20:48:54.197725+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:32:52.892603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":394,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T20:48:53.174967+0000","last_change":"2023-09-23T20:37:43.571083+0000","last_active":"2023-09-23T20:48:53.174967+0000","last_peered":"2023-09-23T20:48:53.174967+0000","last_clean":"2023-09-23T20:48:53.174967+0000","last_became_active":"2023-09-23T20:37:43.570306+0000","last_became_peered":"2023-09-23T20:37:43.570306+0000","last_unstale":"2023-09-23T20:48:53.174967+0000","last_undegraded":"2023-09-23T20:48:53.174967+0000","last_fullsized":"2023-09-23T20:48:53.174967+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:42:02.396718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.6","version":"595'10795","reported_seq":11479,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T20:48:51.290227+0000","last_change":"2023-09-23T20:48:40.101320+0000","last_active":"2023-09-23T20:48:51.290227+0000","last_peered":"2023-09-23T20:48:51.290227+0000","last_clean":"2023-09-23T20:48:51.290227+0000","last_became_active":"2023-09-23T20:48:40.072725+0000","last_became_peered":"2023-09-23T20:48:40.072725+0000","last_unstale":"2023-09-23T20:48:51.290227+0000","last_undegraded":"2023-09-23T20:48:51.290227+0000","last_fullsized":"2023-09-23T20:48:51.290227+0000","mapping_epoch":556,"log_start":"541'4690","ondisk_log_start":"541'4690","created":531,"last_epoch_clean":585,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":6105,"log_dups_size":0,"ondisk_log_size":6105,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":71270400,"num_objects":1087,"num_object_clones":0,"num_object_copies":3261,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1087,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5474,"num_write_kb":209984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":143,"num_bytes_recovered":9404416,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.d","version":"90'161","reported_seq":931,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T20:48:54.191739+0000","last_change":"2023-09-23T20:47:11.619510+0000","last_active":"2023-09-23T20:48:54.191739+0000","last_peered":"2023-09-23T20:48:54.191739+0000","last_clean":"2023-09-23T20:48:54.191739+0000","last_became_active":"2023-09-23T20:47:11.619154+0000","last_became_peered":"2023-09-23T20:47:11.619154+0000","last_unstale":"2023-09-23T20:48:54.191739+0000","last_undegraded":"2023-09-23T20:48:54.191739+0000","last_fullsized":"2023-09-23T20:48:54.191739+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:34:59.496375+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:34:59.496375+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:40:52.712777+0000","scrub_duration":0.0017843189999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":630,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T20:48:54.192752+0000","last_change":"2023-09-23T20:47:09.207851+0000","last_active":"2023-09-23T20:48:54.192752+0000","last_peered":"2023-09-23T20:48:54.192752+0000","last_clean":"2023-09-23T20:48:54.192752+0000","last_became_active":"2023-09-23T20:47:09.175836+0000","last_became_peered":"2023-09-23T20:47:09.175836+0000","last_unstale":"2023-09-23T20:48:54.192752+0000","last_undegraded":"2023-09-23T20:48:54.192752+0000","last_fullsized":"2023-09-23T20:48:54.192752+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:43:13.091809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":413,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T20:48:54.195231+0000","last_change":"2023-09-23T20:43:16.765253+0000","last_active":"2023-09-23T20:48:54.195231+0000","last_peered":"2023-09-23T20:48:54.195231+0000","last_clean":"2023-09-23T20:48:54.195231+0000","last_became_active":"2023-09-23T20:43:16.764978+0000","last_became_peered":"2023-09-23T20:43:16.764978+0000","last_unstale":"2023-09-23T20:48:54.195231+0000","last_undegraded":"2023-09-23T20:48:54.195231+0000","last_fullsized":"2023-09-23T20:48:54.195231+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:07:23.539372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.7","version":"599'10735","reported_seq":10825,"reported_epoch":599,"state":"active+clean","last_fresh":"2023-09-23T20:48:55.357713+0000","last_change":"2023-09-23T20:48:44.279779+0000","last_active":"2023-09-23T20:48:55.357713+0000","last_peered":"2023-09-23T20:48:55.357713+0000","last_clean":"2023-09-23T20:48:55.357713+0000","last_became_active":"2023-09-23T20:48:40.067198+0000","last_became_peered":"2023-09-23T20:48:40.067198+0000","last_unstale":"2023-09-23T20:48:55.357713+0000","last_undegraded":"2023-09-23T20:48:55.357713+0000","last_fullsized":"2023-09-23T20:48:55.357713+0000","mapping_epoch":531,"log_start":"542'4863","ondisk_log_start":"542'4863","created":531,"last_epoch_clean":585,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":5872,"log_dups_size":0,"ondisk_log_size":5872,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":66551808,"num_objects":1015,"num_object_clones":0,"num_object_copies":3045,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1015,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5471,"num_write_kb":207584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.c","version":"90'161","reported_seq":860,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T20:48:53.175025+0000","last_change":"2023-09-23T20:41:09.506481+0000","last_active":"2023-09-23T20:48:53.175025+0000","last_peered":"2023-09-23T20:48:53.175025+0000","last_clean":"2023-09-23T20:48:53.175025+0000","last_became_active":"2023-09-23T20:38:28.442229+0000","last_became_peered":"2023-09-23T20:38:28.442229+0000","last_unstale":"2023-09-23T20:48:53.175025+0000","last_undegraded":"2023-09-23T20:48:53.175025+0000","last_fullsized":"2023-09-23T20:48:53.175025+0000","mapping_epoch":411,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":412,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:41:09.506338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:41:09.506338+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:19:46.265011+0000","scrub_duration":0.01112002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":668,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T20:48:54.192470+0000","last_change":"2023-09-23T20:47:10.848436+0000","last_active":"2023-09-23T20:48:54.192470+0000","last_peered":"2023-09-23T20:48:54.192470+0000","last_clean":"2023-09-23T20:48:54.192470+0000","last_became_active":"2023-09-23T20:47:09.196769+0000","last_became_peered":"2023-09-23T20:47:09.196769+0000","last_unstale":"2023-09-23T20:48:54.192470+0000","last_undegraded":"2023-09-23T20:48:54.192470+0000","last_fullsized":"2023-09-23T20:48:54.192470+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":554,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:59:38.105009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":66,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T20:48:54.184278+0000","last_change":"2023-09-23T20:48:23.915817+0000","last_active":"2023-09-23T20:48:54.184278+0000","last_peered":"2023-09-23T20:48:54.184278+0000","last_clean":"2023-09-23T20:48:54.184278+0000","last_became_active":"2023-09-23T20:48:23.915306+0000","last_became_peered":"2023-09-23T20:48:23.915306+0000","last_unstale":"2023-09-23T20:48:54.184278+0000","last_undegraded":"2023-09-23T20:48:54.184278+0000","last_fullsized":"2023-09-23T20:48:54.184278+0000","mapping_epoch":577,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":578,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:55:45.691450+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"595'10639","reported_seq":11297,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T20:48:51.617828+0000","last_change":"2023-09-23T20:48:40.087695+0000","last_active":"2023-09-23T20:48:51.617828+0000","last_peered":"2023-09-23T20:48:51.617828+0000","last_clean":"2023-09-23T20:48:51.617828+0000","last_became_active":"2023-09-23T20:48:40.070957+0000","last_became_peered":"2023-09-23T20:48:40.070957+0000","last_unstale":"2023-09-23T20:48:51.617828+0000","last_undegraded":"2023-09-23T20:48:51.617828+0000","last_fullsized":"2023-09-23T20:48:51.617828+0000","mapping_epoch":558,"log_start":"541'4547","ondisk_log_start":"541'4547","created":531,"last_epoch_clean":585,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":6092,"log_dups_size":0,"ondisk_log_size":6092,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70320128,"num_objects":1073,"num_object_clones":0,"num_object_copies":3219,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1073,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5390,"num_write_kb":206816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":137,"num_bytes_recovered":8978432,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"90'161","reported_seq":794,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T20:48:54.198181+0000","last_change":"2023-09-23T20:47:11.622303+0000","last_active":"2023-09-23T20:48:54.198181+0000","last_peered":"2023-09-23T20:48:54.198181+0000","last_clean":"2023-09-23T20:48:54.198181+0000","last_became_active":"2023-09-23T20:47:11.621953+0000","last_became_peered":"2023-09-23T20:47:11.621953+0000","last_unstale":"2023-09-23T20:48:54.198181+0000","last_undegraded":"2023-09-23T20:48:54.198181+0000","last_fullsized":"2023-09-23T20:48:54.198181+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:46.217489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:46.217489+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:14:40.478522+0000","scrub_duration":0.001456429,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":627,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T20:48:54.191902+0000","last_change":"2023-09-23T20:40:19.677655+0000","last_active":"2023-09-23T20:48:54.191902+0000","last_peered":"2023-09-23T20:48:54.191902+0000","last_clean":"2023-09-23T20:48:54.191902+0000","last_became_active":"2023-09-23T20:40:19.551189+0000","last_became_peered":"2023-09-23T20:40:19.551189+0000","last_unstale":"2023-09-23T20:48:54.191902+0000","last_undegraded":"2023-09-23T20:48:54.191902+0000","last_fullsized":"2023-09-23T20:48:54.191902+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:11:01.559576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":204,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T20:48:54.197561+0000","last_change":"2023-09-23T20:38:02.590613+0000","last_active":"2023-09-23T20:48:54.197561+0000","last_peered":"2023-09-23T20:48:54.197561+0000","last_clean":"2023-09-23T20:48:54.197561+0000","last_became_active":"2023-09-23T20:38:02.590284+0000","last_became_peered":"2023-09-23T20:38:02.590284+0000","last_unstale":"2023-09-23T20:48:54.197561+0000","last_undegraded":"2023-09-23T20:48:54.197561+0000","last_fullsized":"2023-09-23T20:48:54.197561+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:40:07.825054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.5","version":"597'10858","reported_seq":13662,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T20:48:53.327863+0000","last_change":"2023-09-23T20:48:53.196448+0000","last_active":"2023-09-23T20:48:53.327863+0000","last_peered":"2023-09-23T20:48:53.327863+0000","last_clean":"2023-09-23T20:48:53.327863+0000","last_became_active":"2023-09-23T20:48:53.175613+0000","last_became_peered":"2023-09-23T20:48:53.175613+0000","last_unstale":"2023-09-23T20:48:53.327863+0000","last_undegraded":"2023-09-23T20:48:53.327863+0000","last_fullsized":"2023-09-23T20:48:53.327863+0000","mapping_epoch":596,"log_start":"542'4806","ondisk_log_start":"542'4806","created":531,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":6052,"log_dups_size":0,"ondisk_log_size":6052,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":71073792,"num_objects":1084,"num_object_clones":0,"num_object_copies":3252,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1084,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5507,"num_write_kb":210944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1338,"num_bytes_recovered":87687168,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.e","version":"90'161","reported_seq":629,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T20:48:54.188680+0000","last_change":"2023-09-23T20:47:37.608426+0000","last_active":"2023-09-23T20:48:54.188680+0000","last_peered":"2023-09-23T20:48:54.188680+0000","last_clean":"2023-09-23T20:48:54.188680+0000","last_became_active":"2023-09-23T20:47:11.617187+0000","last_became_peered":"2023-09-23T20:47:11.617187+0000","last_unstale":"2023-09-23T20:48:54.188680+0000","last_undegraded":"2023-09-23T20:48:54.188680+0000","last_fullsized":"2023-09-23T20:48:54.188680+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:47:37.608309+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:47:37.608309+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:22:25.482297+0000","scrub_duration":0.011600924,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":559,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T20:48:53.175235+0000","last_change":"2023-09-23T20:43:16.756437+0000","last_active":"2023-09-23T20:48:53.175235+0000","last_peered":"2023-09-23T20:48:53.175235+0000","last_clean":"2023-09-23T20:48:53.175235+0000","last_became_active":"2023-09-23T20:43:16.756193+0000","last_became_peered":"2023-09-23T20:43:16.756193+0000","last_unstale":"2023-09-23T20:48:53.175235+0000","last_undegraded":"2023-09-23T20:48:53.175235+0000","last_fullsized":"2023-09-23T20:48:53.175235+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:33:25.632781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":588,"reported_epoch":594,"state":"active+clean","last_fresh":"2023-09-23T20:48:50.158213+0000","last_change":"2023-09-23T20:47:09.197427+0000","last_active":"2023-09-23T20:48:50.158213+0000","last_peered":"2023-09-23T20:48:50.158213+0000","last_clean":"2023-09-23T20:48:50.158213+0000","last_became_active":"2023-09-23T20:47:09.196542+0000","last_became_peered":"2023-09-23T20:47:09.196542+0000","last_unstale":"2023-09-23T20:48:50.158213+0000","last_undegraded":"2023-09-23T20:48:50.158213+0000","last_fullsized":"2023-09-23T20:48:50.158213+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:28:38.926202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"597'11035","reported_seq":12110,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T20:48:53.627828+0000","last_change":"2023-09-23T20:48:41.384765+0000","last_active":"2023-09-23T20:48:53.627828+0000","last_peered":"2023-09-23T20:48:53.627828+0000","last_clean":"2023-09-23T20:48:53.627828+0000","last_became_active":"2023-09-23T20:48:40.094128+0000","last_became_peered":"2023-09-23T20:48:40.094128+0000","last_unstale":"2023-09-23T20:48:53.627828+0000","last_undegraded":"2023-09-23T20:48:53.627828+0000","last_fullsized":"2023-09-23T20:48:53.627828+0000","mapping_epoch":560,"log_start":"542'5074","ondisk_log_start":"542'5074","created":531,"last_epoch_clean":585,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":5961,"log_dups_size":0,"ondisk_log_size":5961,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":67895296,"num_objects":1036,"num_object_clones":0,"num_object_copies":3108,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1036,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5606,"num_write_kb":212544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":238,"num_bytes_recovered":15630336,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":1044,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T20:48:54.191625+0000","last_change":"2023-09-23T20:48:51.169474+0000","last_active":"2023-09-23T20:48:54.191625+0000","last_peered":"2023-09-23T20:48:54.191625+0000","last_clean":"2023-09-23T20:48:54.191625+0000","last_became_active":"2023-09-23T20:48:51.169076+0000","last_became_peered":"2023-09-23T20:48:51.169076+0000","last_unstale":"2023-09-23T20:48:54.191625+0000","last_undegraded":"2023-09-23T20:48:54.191625+0000","last_fullsized":"2023-09-23T20:48:54.191625+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:35:02.523203+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:35:02.523203+0000","last_clean_scrub_stamp":"2023-09-23T20:35:02.523203+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:25:53.741403+0000","scrub_duration":0.0085927989999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":446,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T20:48:53.174622+0000","last_change":"2023-09-23T20:47:09.240200+0000","last_active":"2023-09-23T20:48:53.174622+0000","last_peered":"2023-09-23T20:48:53.174622+0000","last_clean":"2023-09-23T20:48:53.174622+0000","last_became_active":"2023-09-23T20:47:09.195204+0000","last_became_peered":"2023-09-23T20:47:09.195204+0000","last_unstale":"2023-09-23T20:48:53.174622+0000","last_undegraded":"2023-09-23T20:48:53.174622+0000","last_fullsized":"2023-09-23T20:48:53.174622+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:03.328250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":56,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T20:48:54.187507+0000","last_change":"2023-09-23T20:47:09.231714+0000","last_active":"2023-09-23T20:48:54.187507+0000","last_peered":"2023-09-23T20:48:54.187507+0000","last_clean":"2023-09-23T20:48:54.187507+0000","last_became_active":"2023-09-23T20:47:09.230918+0000","last_became_peered":"2023-09-23T20:47:09.230918+0000","last_unstale":"2023-09-23T20:48:54.187507+0000","last_undegraded":"2023-09-23T20:48:54.187507+0000","last_fullsized":"2023-09-23T20:48:54.187507+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:24.583656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"597'11018","reported_seq":11572,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T20:48:53.280011+0000","last_change":"2023-09-23T20:48:48.187684+0000","last_active":"2023-09-23T20:48:53.280011+0000","last_peered":"2023-09-23T20:48:53.280011+0000","last_clean":"2023-09-23T20:48:53.280011+0000","last_became_active":"2023-09-23T20:48:48.159849+0000","last_became_peered":"2023-09-23T20:48:48.159849+0000","last_unstale":"2023-09-23T20:48:53.280011+0000","last_undegraded":"2023-09-23T20:48:53.280011+0000","last_fullsized":"2023-09-23T20:48:53.280011+0000","mapping_epoch":591,"log_start":"536'1300","ondisk_log_start":"536'1300","created":531,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":9718,"log_dups_size":0,"ondisk_log_size":9718,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70057984,"num_objects":1069,"num_object_clones":0,"num_object_copies":3207,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1069,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5605,"num_write_kb":213568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":118,"num_bytes_recovered":7733248,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":472,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T20:48:53.172008+0000","last_change":"2023-09-23T20:48:51.176533+0000","last_active":"2023-09-23T20:48:53.172008+0000","last_peered":"2023-09-23T20:48:53.172008+0000","last_clean":"2023-09-23T20:48:53.172008+0000","last_became_active":"2023-09-23T20:48:51.175911+0000","last_became_peered":"2023-09-23T20:48:51.175911+0000","last_unstale":"2023-09-23T20:48:53.172008+0000","last_undegraded":"2023-09-23T20:48:53.172008+0000","last_fullsized":"2023-09-23T20:48:53.172008+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:42:45.492220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:42:45.492220+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":224,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T20:48:54.197641+0000","last_change":"2023-09-23T20:47:09.178919+0000","last_active":"2023-09-23T20:48:54.197641+0000","last_peered":"2023-09-23T20:48:54.197641+0000","last_clean":"2023-09-23T20:48:54.197641+0000","last_became_active":"2023-09-23T20:47:09.178607+0000","last_became_peered":"2023-09-23T20:47:09.178607+0000","last_unstale":"2023-09-23T20:48:54.197641+0000","last_undegraded":"2023-09-23T20:48:54.197641+0000","last_fullsized":"2023-09-23T20:48:54.197641+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:23:45.196935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":76,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T20:48:54.186920+0000","last_change":"2023-09-23T20:48:48.148596+0000","last_active":"2023-09-23T20:48:54.186920+0000","last_peered":"2023-09-23T20:48:54.186920+0000","last_clean":"2023-09-23T20:48:54.186920+0000","last_became_active":"2023-09-23T20:48:48.148211+0000","last_became_peered":"2023-09-23T20:48:48.148211+0000","last_unstale":"2023-09-23T20:48:54.186920+0000","last_undegraded":"2023-09-23T20:48:54.186920+0000","last_fullsized":"2023-09-23T20:48:54.186920+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:21:50.825822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.0","version":"599'10803","reported_seq":12077,"reported_epoch":599,"state":"active+clean","last_fresh":"2023-09-23T20:48:55.433686+0000","last_change":"2023-09-23T20:48:40.083829+0000","last_active":"2023-09-23T20:48:55.433686+0000","last_peered":"2023-09-23T20:48:55.433686+0000","last_clean":"2023-09-23T20:48:55.433686+0000","last_became_active":"2023-09-23T20:48:40.067403+0000","last_became_peered":"2023-09-23T20:48:40.067403+0000","last_unstale":"2023-09-23T20:48:55.433686+0000","last_undegraded":"2023-09-23T20:48:55.433686+0000","last_fullsized":"2023-09-23T20:48:55.433686+0000","mapping_epoch":559,"log_start":"542'4945","ondisk_log_start":"542'4945","created":531,"last_epoch_clean":585,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":5858,"log_dups_size":0,"ondisk_log_size":5858,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":65372160,"num_objects":997,"num_object_clones":0,"num_object_copies":2991,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":997,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5497,"num_write_kb":207840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":289,"num_bytes_recovered":18939904,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.b","version":"90'161","reported_seq":902,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:48:52.164417+0000","last_change":"2023-09-23T20:40:22.445808+0000","last_active":"2023-09-23T20:48:52.164417+0000","last_peered":"2023-09-23T20:48:52.164417+0000","last_clean":"2023-09-23T20:48:52.164417+0000","last_became_active":"2023-09-23T20:40:22.445456+0000","last_became_peered":"2023-09-23T20:40:22.445456+0000","last_unstale":"2023-09-23T20:48:52.164417+0000","last_undegraded":"2023-09-23T20:48:52.164417+0000","last_fullsized":"2023-09-23T20:48:52.164417+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:05.595042+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:05.595042+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:39:23.678154+0000","scrub_duration":0.007271675,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":76,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T20:48:54.184132+0000","last_change":"2023-09-23T20:48:48.153145+0000","last_active":"2023-09-23T20:48:54.184132+0000","last_peered":"2023-09-23T20:48:54.184132+0000","last_clean":"2023-09-23T20:48:54.184132+0000","last_became_active":"2023-09-23T20:48:48.152858+0000","last_became_peered":"2023-09-23T20:48:48.152858+0000","last_unstale":"2023-09-23T20:48:54.184132+0000","last_undegraded":"2023-09-23T20:48:54.184132+0000","last_fullsized":"2023-09-23T20:48:54.184132+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:23:13.454288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":56,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T20:48:54.188760+0000","last_change":"2023-09-23T20:47:09.186271+0000","last_active":"2023-09-23T20:48:54.188760+0000","last_peered":"2023-09-23T20:48:54.188760+0000","last_clean":"2023-09-23T20:48:54.188760+0000","last_became_active":"2023-09-23T20:47:09.184357+0000","last_became_peered":"2023-09-23T20:47:09.184357+0000","last_unstale":"2023-09-23T20:48:54.188760+0000","last_undegraded":"2023-09-23T20:48:54.188760+0000","last_fullsized":"2023-09-23T20:48:54.188760+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:48:26.339811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"599'10948","reported_seq":11076,"reported_epoch":599,"state":"active+clean","last_fresh":"2023-09-23T20:48:55.699250+0000","last_change":"2023-09-23T20:48:40.102295+0000","last_active":"2023-09-23T20:48:55.699250+0000","last_peered":"2023-09-23T20:48:55.699250+0000","last_clean":"2023-09-23T20:48:55.699250+0000","last_became_active":"2023-09-23T20:48:40.083015+0000","last_became_peered":"2023-09-23T20:48:40.083015+0000","last_unstale":"2023-09-23T20:48:55.699250+0000","last_undegraded":"2023-09-23T20:48:55.699250+0000","last_fullsized":"2023-09-23T20:48:55.699250+0000","mapping_epoch":553,"log_start":"543'5336","ondisk_log_start":"543'5336","created":531,"last_epoch_clean":585,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":5612,"log_dups_size":0,"ondisk_log_size":5612,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":61931520,"num_objects":945,"num_object_clones":0,"num_object_copies":2835,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":945,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5607,"num_write_kb":209664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":393216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":476,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T20:48:53.172181+0000","last_change":"2023-09-23T20:48:51.177368+0000","last_active":"2023-09-23T20:48:53.172181+0000","last_peered":"2023-09-23T20:48:53.172181+0000","last_clean":"2023-09-23T20:48:53.172181+0000","last_became_active":"2023-09-23T20:48:51.177024+0000","last_became_peered":"2023-09-23T20:48:51.177024+0000","last_unstale":"2023-09-23T20:48:53.172181+0000","last_undegraded":"2023-09-23T20:48:53.172181+0000","last_fullsized":"2023-09-23T20:48:53.172181+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:43:51.693459+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:43:51.693459+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26834,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":395,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T20:48:53.174148+0000","last_change":"2023-09-23T20:37:43.570610+0000","last_active":"2023-09-23T20:48:53.174148+0000","last_peered":"2023-09-23T20:48:53.174148+0000","last_clean":"2023-09-23T20:48:53.174148+0000","last_became_active":"2023-09-23T20:37:43.570216+0000","last_became_peered":"2023-09-23T20:37:43.570216+0000","last_unstale":"2023-09-23T20:48:53.174148+0000","last_undegraded":"2023-09-23T20:48:53.174148+0000","last_fullsized":"2023-09-23T20:48:53.174148+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:39:26.555329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":442,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:48:52.165350+0000","last_change":"2023-09-23T20:47:09.172573+0000","last_active":"2023-09-23T20:48:52.165350+0000","last_peered":"2023-09-23T20:48:52.165350+0000","last_clean":"2023-09-23T20:48:52.165350+0000","last_became_active":"2023-09-23T20:47:09.172298+0000","last_became_peered":"2023-09-23T20:47:09.172298+0000","last_unstale":"2023-09-23T20:48:52.165350+0000","last_undegraded":"2023-09-23T20:48:52.165350+0000","last_fullsized":"2023-09-23T20:48:52.165350+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:22:09.361284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"599'11187","reported_seq":11845,"reported_epoch":599,"state":"active+clean","last_fresh":"2023-09-23T20:48:55.367140+0000","last_change":"2023-09-23T20:47:11.626133+0000","last_active":"2023-09-23T20:48:55.367140+0000","last_peered":"2023-09-23T20:48:55.367140+0000","last_clean":"2023-09-23T20:48:55.367140+0000","last_became_active":"2023-09-23T20:47:11.619197+0000","last_became_peered":"2023-09-23T20:47:11.619197+0000","last_unstale":"2023-09-23T20:48:55.367140+0000","last_undegraded":"2023-09-23T20:48:55.367140+0000","last_fullsized":"2023-09-23T20:48:55.367140+0000","mapping_epoch":555,"log_start":"543'5508","ondisk_log_start":"543'5508","created":531,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":5679,"log_dups_size":0,"ondisk_log_size":5679,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:33:18.084175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":135069696,"num_objects":2061,"num_object_clones":0,"num_object_copies":6183,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2061,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11187,"num_write_kb":423936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":277,"num_bytes_recovered":18153472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":787,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T20:48:54.198122+0000","last_change":"2023-09-23T20:41:07.424609+0000","last_active":"2023-09-23T20:48:54.198122+0000","last_peered":"2023-09-23T20:48:54.198122+0000","last_clean":"2023-09-23T20:48:54.198122+0000","last_became_active":"2023-09-23T20:38:05.622261+0000","last_became_peered":"2023-09-23T20:38:05.622261+0000","last_unstale":"2023-09-23T20:48:54.198122+0000","last_undegraded":"2023-09-23T20:48:54.198122+0000","last_fullsized":"2023-09-23T20:48:54.198122+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:41:07.424492+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:41:07.424492+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:45:21.689123+0000","scrub_duration":0.0018960959999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"86'119","reported_seq":909,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:48:52.164461+0000","last_change":"2023-09-23T20:29:45.521257+0000","last_active":"2023-09-23T20:48:52.164461+0000","last_peered":"2023-09-23T20:48:52.164461+0000","last_clean":"2023-09-23T20:48:52.164461+0000","last_became_active":"2023-09-23T20:29:45.520546+0000","last_became_peered":"2023-09-23T20:29:45.520546+0000","last_unstale":"2023-09-23T20:48:52.164461+0000","last_undegraded":"2023-09-23T20:48:52.164461+0000","last_fullsized":"2023-09-23T20:48:52.164461+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":119,"log_dups_size":0,"ondisk_log_size":119,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:58:56.085777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032705499999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":523,"num_read_kb":3503,"num_write":136,"num_write_kb":3128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":410,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T20:48:53.175443+0000","last_change":"2023-09-23T20:48:48.157475+0000","last_active":"2023-09-23T20:48:53.175443+0000","last_peered":"2023-09-23T20:48:53.175443+0000","last_clean":"2023-09-23T20:48:53.175443+0000","last_became_active":"2023-09-23T20:48:48.155904+0000","last_became_peered":"2023-09-23T20:48:48.155904+0000","last_unstale":"2023-09-23T20:48:53.175443+0000","last_undegraded":"2023-09-23T20:48:53.175443+0000","last_fullsized":"2023-09-23T20:48:53.175443+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:39:08.642474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":574,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:48:52.164602+0000","last_change":"2023-09-23T20:47:09.194947+0000","last_active":"2023-09-23T20:48:52.164602+0000","last_peered":"2023-09-23T20:48:52.164602+0000","last_clean":"2023-09-23T20:48:52.164602+0000","last_became_active":"2023-09-23T20:47:09.194592+0000","last_became_peered":"2023-09-23T20:47:09.194592+0000","last_unstale":"2023-09-23T20:48:52.164602+0000","last_undegraded":"2023-09-23T20:48:52.164602+0000","last_fullsized":"2023-09-23T20:48:52.164602+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:34:15.008492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"599'10736","reported_seq":10812,"reported_epoch":599,"state":"active+clean","last_fresh":"2023-09-23T20:48:55.441378+0000","last_change":"2023-09-23T20:45:39.489110+0000","last_active":"2023-09-23T20:48:55.441378+0000","last_peered":"2023-09-23T20:48:55.441378+0000","last_clean":"2023-09-23T20:48:55.441378+0000","last_became_active":"2023-09-23T20:45:39.488817+0000","last_became_peered":"2023-09-23T20:45:39.488817+0000","last_unstale":"2023-09-23T20:48:55.441378+0000","last_undegraded":"2023-09-23T20:48:55.441378+0000","last_fullsized":"2023-09-23T20:48:55.441378+0000","mapping_epoch":531,"log_start":"542'5090","ondisk_log_start":"542'5090","created":531,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":5646,"log_dups_size":0,"ondisk_log_size":5646,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:07:21.467971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131203072,"num_objects":2002,"num_object_clones":0,"num_object_copies":6006,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2002,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10736,"num_write_kb":407616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":676,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T20:48:51.178609+0000","last_change":"2023-09-23T20:48:51.178609+0000","last_active":"2023-09-23T20:48:51.178609+0000","last_peered":"2023-09-23T20:48:51.178609+0000","last_clean":"2023-09-23T20:48:51.178609+0000","last_became_active":"2023-09-23T20:48:51.178196+0000","last_became_peered":"2023-09-23T20:48:51.178196+0000","last_unstale":"2023-09-23T20:48:51.178609+0000","last_undegraded":"2023-09-23T20:48:51.178609+0000","last_fullsized":"2023-09-23T20:48:51.178609+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:15.656880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:15.656880+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:47:54.557002+0000","scrub_duration":0.0016711180000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"86'106","reported_seq":849,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:48:52.164884+0000","last_change":"2023-09-23T20:43:16.759138+0000","last_active":"2023-09-23T20:48:52.164884+0000","last_peered":"2023-09-23T20:48:52.164884+0000","last_clean":"2023-09-23T20:48:52.164884+0000","last_became_active":"2023-09-23T20:43:16.758015+0000","last_became_peered":"2023-09-23T20:43:16.758015+0000","last_unstale":"2023-09-23T20:48:52.164884+0000","last_undegraded":"2023-09-23T20:48:52.164884+0000","last_fullsized":"2023-09-23T20:48:52.164884+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":106,"log_dups_size":0,"ondisk_log_size":106,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:08:02.852472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00099302599999999993,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1094,"num_read_kb":1678,"num_write":100,"num_write_kb":404,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":56,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T20:48:54.187472+0000","last_change":"2023-09-23T20:47:09.177872+0000","last_active":"2023-09-23T20:48:54.187472+0000","last_peered":"2023-09-23T20:48:54.187472+0000","last_clean":"2023-09-23T20:48:54.187472+0000","last_became_active":"2023-09-23T20:47:09.176689+0000","last_became_peered":"2023-09-23T20:47:09.176689+0000","last_unstale":"2023-09-23T20:48:54.187472+0000","last_undegraded":"2023-09-23T20:48:54.187472+0000","last_fullsized":"2023-09-23T20:48:54.187472+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:14:01.590318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":580,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:48:52.164380+0000","last_change":"2023-09-23T20:48:48.151144+0000","last_active":"2023-09-23T20:48:52.164380+0000","last_peered":"2023-09-23T20:48:52.164380+0000","last_clean":"2023-09-23T20:48:52.164380+0000","last_became_active":"2023-09-23T20:48:48.150770+0000","last_became_peered":"2023-09-23T20:48:48.150770+0000","last_unstale":"2023-09-23T20:48:52.164380+0000","last_undegraded":"2023-09-23T20:48:52.164380+0000","last_fullsized":"2023-09-23T20:48:52.164380+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:14:08.549512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.9","version":"597'10909","reported_seq":13019,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T20:48:53.330763+0000","last_change":"2023-09-23T20:48:49.220102+0000","last_active":"2023-09-23T20:48:53.330763+0000","last_peered":"2023-09-23T20:48:53.330763+0000","last_clean":"2023-09-23T20:48:53.330763+0000","last_became_active":"2023-09-23T20:48:49.169141+0000","last_became_peered":"2023-09-23T20:48:49.169141+0000","last_unstale":"2023-09-23T20:48:53.330763+0000","last_undegraded":"2023-09-23T20:48:53.330763+0000","last_fullsized":"2023-09-23T20:48:53.330763+0000","mapping_epoch":592,"log_start":"541'4623","ondisk_log_start":"541'4623","created":531,"last_epoch_clean":593,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":6286,"log_dups_size":0,"ondisk_log_size":6286,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70287360,"num_objects":1072,"num_object_clones":0,"num_object_copies":3216,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1072,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5530,"num_write_kb":211296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1000,"num_bytes_recovered":65404928,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":602,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T20:48:54.191092+0000","last_change":"2023-09-23T20:47:11.618652+0000","last_active":"2023-09-23T20:48:54.191092+0000","last_peered":"2023-09-23T20:48:54.191092+0000","last_clean":"2023-09-23T20:48:54.191092+0000","last_became_active":"2023-09-23T20:47:11.618266+0000","last_became_peered":"2023-09-23T20:47:11.618266+0000","last_unstale":"2023-09-23T20:48:54.191092+0000","last_undegraded":"2023-09-23T20:48:54.191092+0000","last_fullsized":"2023-09-23T20:48:54.191092+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:26.978179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:26.978179+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:29:29.000683+0000","scrub_duration":0.0013962829999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"86'130","reported_seq":382,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T20:48:53.172226+0000","last_change":"2023-09-23T20:48:51.171448+0000","last_active":"2023-09-23T20:48:53.172226+0000","last_peered":"2023-09-23T20:48:53.172226+0000","last_clean":"2023-09-23T20:48:53.172226+0000","last_became_active":"2023-09-23T20:48:51.170485+0000","last_became_peered":"2023-09-23T20:48:51.170485+0000","last_unstale":"2023-09-23T20:48:53.172226+0000","last_undegraded":"2023-09-23T20:48:53.172226+0000","last_fullsized":"2023-09-23T20:48:53.172226+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:48:35.309684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011300049999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":512,"num_read_kb":1820,"num_write":133,"num_write_kb":1347,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":588,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:48:52.164140+0000","last_change":"2023-09-23T20:48:48.151034+0000","last_active":"2023-09-23T20:48:52.164140+0000","last_peered":"2023-09-23T20:48:52.164140+0000","last_clean":"2023-09-23T20:48:52.164140+0000","last_became_active":"2023-09-23T20:48:48.150658+0000","last_became_peered":"2023-09-23T20:48:48.150658+0000","last_unstale":"2023-09-23T20:48:52.164140+0000","last_undegraded":"2023-09-23T20:48:52.164140+0000","last_fullsized":"2023-09-23T20:48:52.164140+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:44:34.819851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":113,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:48:52.164834+0000","last_change":"2023-09-23T20:44:10.555583+0000","last_active":"2023-09-23T20:48:52.164834+0000","last_peered":"2023-09-23T20:48:52.164834+0000","last_clean":"2023-09-23T20:48:52.164834+0000","last_became_active":"2023-09-23T20:44:10.555281+0000","last_became_peered":"2023-09-23T20:44:10.555281+0000","last_unstale":"2023-09-23T20:48:52.164834+0000","last_undegraded":"2023-09-23T20:48:52.164834+0000","last_fullsized":"2023-09-23T20:48:52.164834+0000","mapping_epoch":496,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":497,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:43:11.942147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"599'11115","reported_seq":11191,"reported_epoch":599,"state":"active+clean","last_fresh":"2023-09-23T20:48:55.441303+0000","last_change":"2023-09-23T20:45:39.331096+0000","last_active":"2023-09-23T20:48:55.441303+0000","last_peered":"2023-09-23T20:48:55.441303+0000","last_clean":"2023-09-23T20:48:55.441303+0000","last_became_active":"2023-09-23T20:45:39.330752+0000","last_became_peered":"2023-09-23T20:45:39.330752+0000","last_unstale":"2023-09-23T20:48:55.441303+0000","last_undegraded":"2023-09-23T20:48:55.441303+0000","last_fullsized":"2023-09-23T20:48:55.441303+0000","mapping_epoch":531,"log_start":"543'5429","ondisk_log_start":"543'5429","created":531,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":5686,"log_dups_size":0,"ondisk_log_size":5686,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:39:01.627762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":130613248,"num_objects":1993,"num_object_clones":0,"num_object_copies":5979,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1993,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11115,"num_write_kb":419456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"86'194","reported_seq":708,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T20:48:53.172181+0000","last_change":"2023-09-23T20:48:51.175633+0000","last_active":"2023-09-23T20:48:53.172181+0000","last_peered":"2023-09-23T20:48:53.172181+0000","last_clean":"2023-09-23T20:48:53.172181+0000","last_became_active":"2023-09-23T20:48:51.174941+0000","last_became_peered":"2023-09-23T20:48:51.174941+0000","last_unstale":"2023-09-23T20:48:53.172181+0000","last_undegraded":"2023-09-23T20:48:53.172181+0000","last_fullsized":"2023-09-23T20:48:53.172181+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":194,"log_dups_size":0,"ondisk_log_size":194,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:06:27.009241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00037995999999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1024,"num_read_kb":1200,"num_write":173,"num_write_kb":1276,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":1061,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T20:48:54.195104+0000","last_change":"2023-09-23T20:38:05.602508+0000","last_active":"2023-09-23T20:48:54.195104+0000","last_peered":"2023-09-23T20:48:54.195104+0000","last_clean":"2023-09-23T20:48:54.195104+0000","last_became_active":"2023-09-23T20:38:05.602129+0000","last_became_peered":"2023-09-23T20:38:05.602129+0000","last_unstale":"2023-09-23T20:48:54.195104+0000","last_undegraded":"2023-09-23T20:48:54.195104+0000","last_fullsized":"2023-09-23T20:48:54.195104+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:45.343742+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:45.343742+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:48.910343+0000","scrub_duration":0.001475802,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":174,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T20:48:54.197719+0000","last_change":"2023-09-23T20:43:16.758213+0000","last_active":"2023-09-23T20:48:54.197719+0000","last_peered":"2023-09-23T20:48:54.197719+0000","last_clean":"2023-09-23T20:48:54.197719+0000","last_became_active":"2023-09-23T20:43:16.757869+0000","last_became_peered":"2023-09-23T20:43:16.757869+0000","last_unstale":"2023-09-23T20:48:54.197719+0000","last_undegraded":"2023-09-23T20:48:54.197719+0000","last_fullsized":"2023-09-23T20:48:54.197719+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:43:10.811749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":224,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T20:48:54.197155+0000","last_change":"2023-09-23T20:47:09.202302+0000","last_active":"2023-09-23T20:48:54.197155+0000","last_peered":"2023-09-23T20:48:54.197155+0000","last_clean":"2023-09-23T20:48:54.197155+0000","last_became_active":"2023-09-23T20:47:09.201797+0000","last_became_peered":"2023-09-23T20:47:09.201797+0000","last_unstale":"2023-09-23T20:48:54.197155+0000","last_undegraded":"2023-09-23T20:48:54.197155+0000","last_fullsized":"2023-09-23T20:48:54.197155+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:34:41.055705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.8","version":"593'10653","reported_seq":14438,"reported_epoch":595,"state":"peering","last_fresh":"2023-09-23T20:48:51.163942+0000","last_change":"2023-09-23T20:48:51.164136+0000","last_active":"2023-09-23T20:48:51.163942+0000","last_peered":"2023-09-23T20:48:50.267630+0000","last_clean":"2023-09-23T20:48:50.267630+0000","last_became_active":"2023-09-23T20:48:49.230263+0000","last_became_peered":"2023-09-23T20:48:49.230263+0000","last_unstale":"2023-09-23T20:48:51.163942+0000","last_undegraded":"2023-09-23T20:48:51.163942+0000","last_fullsized":"2023-09-23T20:48:51.163942+0000","mapping_epoch":595,"log_start":"541'4571","ondisk_log_start":"541'4571","created":531,"last_epoch_clean":594,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":6082,"log_dups_size":0,"ondisk_log_size":6082,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":67272704,"num_objects":1026,"num_object_clones":0,"num_object_copies":3078,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1026,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5400,"num_write_kb":205664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":994,"num_bytes_recovered":65175552,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[2,5,7],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":965,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:48:52.164279+0000","last_change":"2023-09-23T20:47:11.622262+0000","last_active":"2023-09-23T20:48:52.164279+0000","last_peered":"2023-09-23T20:48:52.164279+0000","last_clean":"2023-09-23T20:48:52.164279+0000","last_became_active":"2023-09-23T20:47:11.621920+0000","last_became_peered":"2023-09-23T20:47:11.621920+0000","last_unstale":"2023-09-23T20:48:52.164279+0000","last_undegraded":"2023-09-23T20:48:52.164279+0000","last_fullsized":"2023-09-23T20:48:52.164279+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:43.582146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:43.582146+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:01:15.153543+0000","scrub_duration":0.0016219730000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"86'80","reported_seq":851,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T20:48:54.194939+0000","last_change":"2023-09-23T20:29:45.536796+0000","last_active":"2023-09-23T20:48:54.194939+0000","last_peered":"2023-09-23T20:48:54.194939+0000","last_clean":"2023-09-23T20:48:54.194939+0000","last_became_active":"2023-09-23T20:29:45.536439+0000","last_became_peered":"2023-09-23T20:29:45.536439+0000","last_unstale":"2023-09-23T20:48:54.194939+0000","last_undegraded":"2023-09-23T20:48:54.194939+0000","last_fullsized":"2023-09-23T20:48:54.194939+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":80,"log_dups_size":0,"ondisk_log_size":80,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:30:57.793132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035115,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":395,"num_read_kb":1370,"num_write":66,"num_write_kb":1061,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":607,"reported_epoch":594,"state":"active+clean","last_fresh":"2023-09-23T20:48:50.161129+0000","last_change":"2023-09-23T20:47:09.198326+0000","last_active":"2023-09-23T20:48:50.161129+0000","last_peered":"2023-09-23T20:48:50.161129+0000","last_clean":"2023-09-23T20:48:50.161129+0000","last_became_active":"2023-09-23T20:47:09.198059+0000","last_became_peered":"2023-09-23T20:47:09.198059+0000","last_unstale":"2023-09-23T20:48:50.161129+0000","last_undegraded":"2023-09-23T20:48:50.161129+0000","last_fullsized":"2023-09-23T20:48:50.161129+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:38:53.903706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":602,"reported_epoch":594,"state":"active+clean","last_fresh":"2023-09-23T20:48:50.159194+0000","last_change":"2023-09-23T20:48:48.146977+0000","last_active":"2023-09-23T20:48:50.159194+0000","last_peered":"2023-09-23T20:48:50.159194+0000","last_clean":"2023-09-23T20:48:50.159194+0000","last_became_active":"2023-09-23T20:48:48.146657+0000","last_became_peered":"2023-09-23T20:48:48.146657+0000","last_unstale":"2023-09-23T20:48:50.159194+0000","last_undegraded":"2023-09-23T20:48:50.159194+0000","last_fullsized":"2023-09-23T20:48:50.159194+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:38:41.782692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"599'10873","reported_seq":11575,"reported_epoch":599,"state":"active+clean","last_fresh":"2023-09-23T20:48:55.453655+0000","last_change":"2023-09-23T20:47:13.668055+0000","last_active":"2023-09-23T20:48:55.453655+0000","last_peered":"2023-09-23T20:48:55.453655+0000","last_clean":"2023-09-23T20:48:55.453655+0000","last_became_active":"2023-09-23T20:47:13.626170+0000","last_became_peered":"2023-09-23T20:47:13.626170+0000","last_unstale":"2023-09-23T20:48:55.453655+0000","last_undegraded":"2023-09-23T20:48:55.453655+0000","last_fullsized":"2023-09-23T20:48:55.453655+0000","mapping_epoch":557,"log_start":"542'5195","ondisk_log_start":"542'5195","created":531,"last_epoch_clean":558,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":5678,"log_dups_size":0,"ondisk_log_size":5678,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:32.805547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":132513816,"num_objects":2023,"num_object_clones":0,"num_object_copies":6069,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2023,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10873,"num_write_kb":412609,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":298,"num_bytes_recovered":19529728,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":710,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T20:48:54.199212+0000","last_change":"2023-09-23T20:41:08.380084+0000","last_active":"2023-09-23T20:48:54.199212+0000","last_peered":"2023-09-23T20:48:54.199212+0000","last_clean":"2023-09-23T20:48:54.199212+0000","last_became_active":"2023-09-23T20:38:05.610881+0000","last_became_peered":"2023-09-23T20:38:05.610881+0000","last_unstale":"2023-09-23T20:48:54.199212+0000","last_undegraded":"2023-09-23T20:48:54.199212+0000","last_fullsized":"2023-09-23T20:48:54.199212+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:41:08.379959+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:41:08.379959+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:19:39.547923+0000","scrub_duration":0.0019158420000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"86'102","reported_seq":1269,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T20:48:54.191983+0000","last_change":"2023-09-23T20:47:14.628795+0000","last_active":"2023-09-23T20:48:54.191983+0000","last_peered":"2023-09-23T20:48:54.191983+0000","last_clean":"2023-09-23T20:48:54.191983+0000","last_became_active":"2023-09-23T20:47:14.628341+0000","last_became_peered":"2023-09-23T20:47:14.628341+0000","last_unstale":"2023-09-23T20:48:54.191983+0000","last_undegraded":"2023-09-23T20:48:54.191983+0000","last_fullsized":"2023-09-23T20:48:54.191983+0000","mapping_epoch":558,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":559,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":102,"log_dups_size":0,"ondisk_log_size":102,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:42:16.291849+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030656500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":963,"num_read_kb":1234,"num_write":60,"num_write_kb":225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":564,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:48:52.164548+0000","last_change":"2023-09-23T20:48:48.151135+0000","last_active":"2023-09-23T20:48:52.164548+0000","last_peered":"2023-09-23T20:48:52.164548+0000","last_clean":"2023-09-23T20:48:52.164548+0000","last_became_active":"2023-09-23T20:48:48.150780+0000","last_became_peered":"2023-09-23T20:48:48.150780+0000","last_unstale":"2023-09-23T20:48:52.164548+0000","last_undegraded":"2023-09-23T20:48:52.164548+0000","last_fullsized":"2023-09-23T20:48:52.164548+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:39:09.727249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":539,"reported_epoch":594,"state":"active+clean","last_fresh":"2023-09-23T20:48:50.158562+0000","last_change":"2023-09-23T20:43:46.541173+0000","last_active":"2023-09-23T20:48:50.158562+0000","last_peered":"2023-09-23T20:48:50.158562+0000","last_clean":"2023-09-23T20:48:50.158562+0000","last_became_active":"2023-09-23T20:43:46.540726+0000","last_became_peered":"2023-09-23T20:43:46.540726+0000","last_unstale":"2023-09-23T20:48:50.158562+0000","last_undegraded":"2023-09-23T20:48:50.158562+0000","last_fullsized":"2023-09-23T20:48:50.158562+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:49:04.137885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.e","version":"599'11033","reported_seq":15326,"reported_epoch":599,"state":"active+clean","last_fresh":"2023-09-23T20:48:55.817441+0000","last_change":"2023-09-23T20:48:48.230594+0000","last_active":"2023-09-23T20:48:55.817441+0000","last_peered":"2023-09-23T20:48:55.817441+0000","last_clean":"2023-09-23T20:48:55.817441+0000","last_became_active":"2023-09-23T20:48:48.164237+0000","last_became_peered":"2023-09-23T20:48:48.164237+0000","last_unstale":"2023-09-23T20:48:55.817441+0000","last_undegraded":"2023-09-23T20:48:55.817441+0000","last_fullsized":"2023-09-23T20:48:55.817441+0000","mapping_epoch":591,"log_start":"545'6100","ondisk_log_start":"545'6100","created":531,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":4933,"log_dups_size":0,"ondisk_log_size":4933,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:49:13.858414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":128778240,"num_objects":1965,"num_object_clones":0,"num_object_copies":5895,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1965,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11033,"num_write_kb":415936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2081,"num_bytes_recovered":136314880,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":636,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T20:48:54.187075+0000","last_change":"2023-09-23T20:48:51.172830+0000","last_active":"2023-09-23T20:48:54.187075+0000","last_peered":"2023-09-23T20:48:54.187075+0000","last_clean":"2023-09-23T20:48:54.187075+0000","last_became_active":"2023-09-23T20:48:51.172481+0000","last_became_peered":"2023-09-23T20:48:51.172481+0000","last_unstale":"2023-09-23T20:48:54.187075+0000","last_undegraded":"2023-09-23T20:48:54.187075+0000","last_fullsized":"2023-09-23T20:48:54.187075+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:47:34.561747+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:47:34.561747+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:30:32.531477+0000","scrub_duration":0.0087949929999999992,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":336,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T20:48:54.192871+0000","last_change":"2023-09-23T20:32:50.549695+0000","last_active":"2023-09-23T20:48:54.192871+0000","last_peered":"2023-09-23T20:48:54.192871+0000","last_clean":"2023-09-23T20:48:54.192871+0000","last_became_active":"2023-09-23T20:32:50.549431+0000","last_became_peered":"2023-09-23T20:32:50.549431+0000","last_unstale":"2023-09-23T20:48:54.192871+0000","last_undegraded":"2023-09-23T20:48:54.192871+0000","last_fullsized":"2023-09-23T20:48:54.192871+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:45:53.666569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"86'128","reported_seq":1077,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:48:52.165107+0000","last_change":"2023-09-23T20:47:11.621846+0000","last_active":"2023-09-23T20:48:52.165107+0000","last_peered":"2023-09-23T20:48:52.165107+0000","last_clean":"2023-09-23T20:48:52.165107+0000","last_became_active":"2023-09-23T20:47:11.621284+0000","last_became_peered":"2023-09-23T20:47:11.621284+0000","last_unstale":"2023-09-23T20:48:52.165107+0000","last_undegraded":"2023-09-23T20:48:52.165107+0000","last_fullsized":"2023-09-23T20:48:52.165107+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:43:50.658512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000293202,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":579,"num_read_kb":1473,"num_write":96,"num_write_kb":1062,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":380,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T20:48:53.174148+0000","last_change":"2023-09-23T20:37:43.582105+0000","last_active":"2023-09-23T20:48:53.174148+0000","last_peered":"2023-09-23T20:48:53.174148+0000","last_clean":"2023-09-23T20:48:53.174148+0000","last_became_active":"2023-09-23T20:37:43.581860+0000","last_became_peered":"2023-09-23T20:37:43.581860+0000","last_unstale":"2023-09-23T20:48:53.174148+0000","last_undegraded":"2023-09-23T20:48:53.174148+0000","last_fullsized":"2023-09-23T20:48:53.174148+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:18:38.765486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"599'10984","reported_seq":16486,"reported_epoch":599,"state":"active+clean","last_fresh":"2023-09-23T20:48:55.474656+0000","last_change":"2023-09-23T20:48:53.235559+0000","last_active":"2023-09-23T20:48:55.474656+0000","last_peered":"2023-09-23T20:48:55.474656+0000","last_clean":"2023-09-23T20:48:55.474656+0000","last_became_active":"2023-09-23T20:48:53.178550+0000","last_became_peered":"2023-09-23T20:48:53.178550+0000","last_unstale":"2023-09-23T20:48:55.474656+0000","last_undegraded":"2023-09-23T20:48:55.474656+0000","last_fullsized":"2023-09-23T20:48:55.474656+0000","mapping_epoch":596,"log_start":"542'5299","ondisk_log_start":"542'5299","created":531,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":5685,"log_dups_size":0,"ondisk_log_size":5685,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:38:02.612388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":132513792,"num_objects":2022,"num_object_clones":0,"num_object_copies":6066,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2022,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10984,"num_write_kb":416192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2684,"num_bytes_recovered":175898624,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.4","version":"90'161","reported_seq":814,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T20:48:53.175085+0000","last_change":"2023-09-23T20:40:22.427435+0000","last_active":"2023-09-23T20:48:53.175085+0000","last_peered":"2023-09-23T20:48:53.175085+0000","last_clean":"2023-09-23T20:48:53.175085+0000","last_became_active":"2023-09-23T20:40:22.427051+0000","last_became_peered":"2023-09-23T20:40:22.427051+0000","last_unstale":"2023-09-23T20:48:53.175085+0000","last_undegraded":"2023-09-23T20:48:53.175085+0000","last_fullsized":"2023-09-23T20:48:53.175085+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:34:54.504554+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:34:54.504554+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:42.001652+0000","scrub_duration":0.011176864,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"86'117","reported_seq":983,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T20:48:54.198040+0000","last_change":"2023-09-23T20:48:51.217483+0000","last_active":"2023-09-23T20:48:54.198040+0000","last_peered":"2023-09-23T20:48:54.198040+0000","last_clean":"2023-09-23T20:48:54.198040+0000","last_became_active":"2023-09-23T20:48:51.217013+0000","last_became_peered":"2023-09-23T20:48:51.217013+0000","last_unstale":"2023-09-23T20:48:54.198040+0000","last_undegraded":"2023-09-23T20:48:54.198040+0000","last_fullsized":"2023-09-23T20:48:54.198040+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:40:42.297947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046891400000000002,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":893,"num_read_kb":922,"num_write":82,"num_write_kb":4193,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":48,"num_bytes_recovered":25165824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":610,"reported_epoch":594,"state":"active+clean","last_fresh":"2023-09-23T20:48:50.159275+0000","last_change":"2023-09-23T20:47:09.317116+0000","last_active":"2023-09-23T20:48:50.159275+0000","last_peered":"2023-09-23T20:48:50.159275+0000","last_clean":"2023-09-23T20:48:50.159275+0000","last_became_active":"2023-09-23T20:47:09.205930+0000","last_became_peered":"2023-09-23T20:47:09.205930+0000","last_unstale":"2023-09-23T20:48:50.159275+0000","last_undegraded":"2023-09-23T20:48:50.159275+0000","last_fullsized":"2023-09-23T20:48:50.159275+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:04:55.528625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":56,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T20:48:54.184204+0000","last_change":"2023-09-23T20:47:09.192117+0000","last_active":"2023-09-23T20:48:54.184204+0000","last_peered":"2023-09-23T20:48:54.184204+0000","last_clean":"2023-09-23T20:48:54.184204+0000","last_became_active":"2023-09-23T20:47:09.191820+0000","last_became_peered":"2023-09-23T20:47:09.191820+0000","last_unstale":"2023-09-23T20:48:54.184204+0000","last_undegraded":"2023-09-23T20:48:54.184204+0000","last_fullsized":"2023-09-23T20:48:54.184204+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:08.785173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.12","version":"597'11043","reported_seq":12118,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T20:48:53.649215+0000","last_change":"2023-09-23T20:48:43.071835+0000","last_active":"2023-09-23T20:48:53.649215+0000","last_peered":"2023-09-23T20:48:53.649215+0000","last_clean":"2023-09-23T20:48:53.649215+0000","last_became_active":"2023-09-23T20:48:40.111663+0000","last_became_peered":"2023-09-23T20:48:40.111663+0000","last_unstale":"2023-09-23T20:48:53.649215+0000","last_undegraded":"2023-09-23T20:48:53.649215+0000","last_fullsized":"2023-09-23T20:48:53.649215+0000","mapping_epoch":584,"log_start":"542'5074","ondisk_log_start":"542'5074","created":584,"last_epoch_clean":585,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":5969,"log_dups_size":0,"ondisk_log_size":5969,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":67371008,"num_objects":1028,"num_object_clones":0,"num_object_copies":3084,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1028,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5614,"num_write_kb":212544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":239,"num_bytes_recovered":15630336,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.19","version":"90'161","reported_seq":853,"reported_epoch":594,"state":"active+clean","last_fresh":"2023-09-23T20:48:50.159584+0000","last_change":"2023-09-23T20:43:46.544750+0000","last_active":"2023-09-23T20:48:50.159584+0000","last_peered":"2023-09-23T20:48:50.159584+0000","last_clean":"2023-09-23T20:48:50.159584+0000","last_became_active":"2023-09-23T20:43:46.544395+0000","last_became_peered":"2023-09-23T20:43:46.544395+0000","last_unstale":"2023-09-23T20:48:50.159584+0000","last_undegraded":"2023-09-23T20:48:50.159584+0000","last_fullsized":"2023-09-23T20:48:50.159584+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:06.487351+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:06.487351+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:56.866168+0000","scrub_duration":0.001964379,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":569,"reported_epoch":594,"state":"active+clean","last_fresh":"2023-09-23T20:48:50.158461+0000","last_change":"2023-09-23T20:47:09.197601+0000","last_active":"2023-09-23T20:48:50.158461+0000","last_peered":"2023-09-23T20:48:50.158461+0000","last_clean":"2023-09-23T20:48:50.158461+0000","last_became_active":"2023-09-23T20:47:09.196967+0000","last_became_peered":"2023-09-23T20:47:09.196967+0000","last_unstale":"2023-09-23T20:48:50.158461+0000","last_undegraded":"2023-09-23T20:48:50.158461+0000","last_fullsized":"2023-09-23T20:48:50.158461+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:06:56.826785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":16,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:48:52.170909+0000","last_change":"2023-09-23T20:48:48.158376+0000","last_active":"2023-09-23T20:48:52.170909+0000","last_peered":"2023-09-23T20:48:52.170909+0000","last_clean":"2023-09-23T20:48:52.170909+0000","last_became_active":"2023-09-23T20:48:48.158069+0000","last_became_peered":"2023-09-23T20:48:48.158069+0000","last_unstale":"2023-09-23T20:48:52.170909+0000","last_undegraded":"2023-09-23T20:48:52.170909+0000","last_fullsized":"2023-09-23T20:48:52.170909+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:34.658561+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.15","version":"597'10859","reported_seq":13226,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T20:48:53.172520+0000","last_change":"2023-09-23T20:48:49.194473+0000","last_active":"2023-09-23T20:48:53.172520+0000","last_peered":"2023-09-23T20:48:53.172520+0000","last_clean":"2023-09-23T20:48:53.172520+0000","last_became_active":"2023-09-23T20:48:49.173258+0000","last_became_peered":"2023-09-23T20:48:49.173258+0000","last_unstale":"2023-09-23T20:48:53.172520+0000","last_undegraded":"2023-09-23T20:48:53.172520+0000","last_fullsized":"2023-09-23T20:48:53.172520+0000","mapping_epoch":592,"log_start":"542'4806","ondisk_log_start":"542'4806","created":584,"last_epoch_clean":593,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":6053,"log_dups_size":0,"ondisk_log_size":6053,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":71008256,"num_objects":1084,"num_object_clones":0,"num_object_copies":3252,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1084,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5509,"num_write_kb":210944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1132,"num_bytes_recovered":74186752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"90'161","reported_seq":673,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T20:48:53.174731+0000","last_change":"2023-09-23T20:41:06.499511+0000","last_active":"2023-09-23T20:48:53.174731+0000","last_peered":"2023-09-23T20:48:53.174731+0000","last_clean":"2023-09-23T20:48:53.174731+0000","last_became_active":"2023-09-23T20:38:05.601874+0000","last_became_peered":"2023-09-23T20:38:05.601874+0000","last_unstale":"2023-09-23T20:48:53.174731+0000","last_undegraded":"2023-09-23T20:48:53.174731+0000","last_fullsized":"2023-09-23T20:48:53.174731+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:41:06.499396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:41:06.499396+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:31:34.059077+0000","scrub_duration":0.0019670310000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":60,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T20:48:54.187660+0000","last_change":"2023-09-23T20:47:09.332980+0000","last_active":"2023-09-23T20:48:54.187660+0000","last_peered":"2023-09-23T20:48:54.187660+0000","last_clean":"2023-09-23T20:48:54.187660+0000","last_became_active":"2023-09-23T20:47:09.230554+0000","last_became_peered":"2023-09-23T20:47:09.230554+0000","last_unstale":"2023-09-23T20:48:54.187660+0000","last_undegraded":"2023-09-23T20:48:54.187660+0000","last_fullsized":"2023-09-23T20:48:54.187660+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:06:21.882823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":56,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T20:48:54.187407+0000","last_change":"2023-09-23T20:47:09.186579+0000","last_active":"2023-09-23T20:48:54.187407+0000","last_peered":"2023-09-23T20:48:54.187407+0000","last_clean":"2023-09-23T20:48:54.187407+0000","last_became_active":"2023-09-23T20:47:09.186227+0000","last_became_peered":"2023-09-23T20:47:09.186227+0000","last_unstale":"2023-09-23T20:48:54.187407+0000","last_undegraded":"2023-09-23T20:48:54.187407+0000","last_fullsized":"2023-09-23T20:48:54.187407+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:45:39.485308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.14","version":"595'10661","reported_seq":11319,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T20:48:51.550159+0000","last_change":"2023-09-23T20:48:40.130483+0000","last_active":"2023-09-23T20:48:51.550159+0000","last_peered":"2023-09-23T20:48:51.550159+0000","last_clean":"2023-09-23T20:48:51.550159+0000","last_became_active":"2023-09-23T20:48:40.105209+0000","last_became_peered":"2023-09-23T20:48:40.105209+0000","last_unstale":"2023-09-23T20:48:51.550159+0000","last_undegraded":"2023-09-23T20:48:51.550159+0000","last_fullsized":"2023-09-23T20:48:51.550159+0000","mapping_epoch":584,"log_start":"541'4547","ondisk_log_start":"541'4547","created":584,"last_epoch_clean":585,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":6114,"log_dups_size":0,"ondisk_log_size":6114,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":68878336,"num_objects":1051,"num_object_clones":0,"num_object_copies":3153,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1051,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5412,"num_write_kb":206816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":137,"num_bytes_recovered":8978432,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":60,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T20:48:54.187013+0000","last_change":"2023-09-23T20:47:09.316824+0000","last_active":"2023-09-23T20:48:54.187013+0000","last_peered":"2023-09-23T20:48:54.187013+0000","last_clean":"2023-09-23T20:48:54.187013+0000","last_became_active":"2023-09-23T20:47:09.195100+0000","last_became_peered":"2023-09-23T20:47:09.195100+0000","last_unstale":"2023-09-23T20:48:54.187013+0000","last_undegraded":"2023-09-23T20:48:54.187013+0000","last_fullsized":"2023-09-23T20:48:54.187013+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:29:53.273554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":56,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T20:48:54.188537+0000","last_change":"2023-09-23T20:47:09.181240+0000","last_active":"2023-09-23T20:48:54.188537+0000","last_peered":"2023-09-23T20:48:54.188537+0000","last_clean":"2023-09-23T20:48:54.188537+0000","last_became_active":"2023-09-23T20:47:09.181022+0000","last_became_peered":"2023-09-23T20:47:09.181022+0000","last_unstale":"2023-09-23T20:48:54.188537+0000","last_undegraded":"2023-09-23T20:48:54.188537+0000","last_fullsized":"2023-09-23T20:48:54.188537+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:38:59.191781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.17","version":"599'10725","reported_seq":10811,"reported_epoch":599,"state":"active+clean","last_fresh":"2023-09-23T20:48:55.442277+0000","last_change":"2023-09-23T20:48:40.108816+0000","last_active":"2023-09-23T20:48:55.442277+0000","last_peered":"2023-09-23T20:48:55.442277+0000","last_clean":"2023-09-23T20:48:55.442277+0000","last_became_active":"2023-09-23T20:48:40.090791+0000","last_became_peered":"2023-09-23T20:48:40.090791+0000","last_unstale":"2023-09-23T20:48:55.442277+0000","last_undegraded":"2023-09-23T20:48:55.442277+0000","last_fullsized":"2023-09-23T20:48:55.442277+0000","mapping_epoch":584,"log_start":"542'4863","ondisk_log_start":"542'4863","created":584,"last_epoch_clean":585,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":5862,"log_dups_size":0,"ondisk_log_size":5862,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":67207168,"num_objects":1026,"num_object_clones":0,"num_object_copies":3078,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1026,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5462,"num_write_kb":207584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1c","version":"90'161","reported_seq":854,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T20:48:54.191654+0000","last_change":"2023-09-23T20:38:05.591992+0000","last_active":"2023-09-23T20:48:54.191654+0000","last_peered":"2023-09-23T20:48:54.191654+0000","last_clean":"2023-09-23T20:48:54.191654+0000","last_became_active":"2023-09-23T20:38:05.591701+0000","last_became_peered":"2023-09-23T20:38:05.591701+0000","last_unstale":"2023-09-23T20:48:54.191654+0000","last_undegraded":"2023-09-23T20:48:54.191654+0000","last_fullsized":"2023-09-23T20:48:54.191654+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:02.384133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:02.384133+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:41.722542+0000","scrub_duration":0.001769098,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":440,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T20:48:53.174976+0000","last_change":"2023-09-23T20:48:48.158348+0000","last_active":"2023-09-23T20:48:53.174976+0000","last_peered":"2023-09-23T20:48:53.174976+0000","last_clean":"2023-09-23T20:48:53.174976+0000","last_became_active":"2023-09-23T20:48:48.157982+0000","last_became_peered":"2023-09-23T20:48:48.157982+0000","last_unstale":"2023-09-23T20:48:53.174976+0000","last_undegraded":"2023-09-23T20:48:53.174976+0000","last_fullsized":"2023-09-23T20:48:53.174976+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:03:48.078707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":406,"reported_epoch":597,"state":"active+clean","last_fresh":"2023-09-23T20:48:53.174621+0000","last_change":"2023-09-23T20:48:48.159781+0000","last_active":"2023-09-23T20:48:53.174621+0000","last_peered":"2023-09-23T20:48:53.174621+0000","last_clean":"2023-09-23T20:48:53.174621+0000","last_became_active":"2023-09-23T20:48:48.159505+0000","last_became_peered":"2023-09-23T20:48:48.159505+0000","last_unstale":"2023-09-23T20:48:53.174621+0000","last_undegraded":"2023-09-23T20:48:53.174621+0000","last_fullsized":"2023-09-23T20:48:53.174621+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:25:18.100645+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.16","version":"595'10819","reported_seq":11503,"reported_epoch":595,"state":"active+clean","last_fresh":"2023-09-23T20:48:51.527921+0000","last_change":"2023-09-23T20:48:40.129973+0000","last_active":"2023-09-23T20:48:51.527921+0000","last_peered":"2023-09-23T20:48:51.527921+0000","last_clean":"2023-09-23T20:48:51.527921+0000","last_became_active":"2023-09-23T20:48:40.112403+0000","last_became_peered":"2023-09-23T20:48:40.112403+0000","last_unstale":"2023-09-23T20:48:51.527921+0000","last_undegraded":"2023-09-23T20:48:51.527921+0000","last_fullsized":"2023-09-23T20:48:51.527921+0000","mapping_epoch":584,"log_start":"541'4690","ondisk_log_start":"541'4690","created":584,"last_epoch_clean":585,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":6129,"log_dups_size":0,"ondisk_log_size":6129,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":69697536,"num_objects":1064,"num_object_clones":0,"num_object_copies":3192,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1064,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5499,"num_write_kb":209984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":144,"num_bytes_recovered":9404416,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,4],"acting":[0,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1d","version":"90'161","reported_seq":726,"reported_epoch":596,"state":"active+clean","last_fresh":"2023-09-23T20:48:52.164890+0000","last_change":"2023-09-23T20:47:11.621758+0000","last_active":"2023-09-23T20:48:52.164890+0000","last_peered":"2023-09-23T20:48:52.164890+0000","last_clean":"2023-09-23T20:48:52.164890+0000","last_became_active":"2023-09-23T20:47:11.621345+0000","last_became_peered":"2023-09-23T20:47:11.621345+0000","last_unstale":"2023-09-23T20:48:52.164890+0000","last_undegraded":"2023-09-23T20:48:52.164890+0000","last_fullsized":"2023-09-23T20:48:52.164890+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:01.504973+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:01.504973+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:31:17.112393+0000","scrub_duration":0.00202344,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":659,"reported_epoch":594,"state":"active+clean","last_fresh":"2023-09-23T20:48:50.161061+0000","last_change":"2023-09-23T20:48:48.164011+0000","last_active":"2023-09-23T20:48:50.161061+0000","last_peered":"2023-09-23T20:48:50.161061+0000","last_clean":"2023-09-23T20:48:50.161061+0000","last_became_active":"2023-09-23T20:48:48.150690+0000","last_became_peered":"2023-09-23T20:48:48.150690+0000","last_unstale":"2023-09-23T20:48:50.161061+0000","last_undegraded":"2023-09-23T20:48:50.161061+0000","last_fullsized":"2023-09-23T20:48:50.161061+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:35:33.111175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":4432,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":66,"reported_epoch":598,"state":"active+clean","last_fresh":"2023-09-23T20:48:54.183762+0000","last_change":"2023-09-23T20:48:23.917302+0000","last_active":"2023-09-23T20:48:54.183762+0000","last_peered":"2023-09-23T20:48:54.183762+0000","last_clean":"2023-09-23T20:48:54.183762+0000","last_became_active":"2023-09-23T20:48:23.916956+0000","last_became_peered":"2023-09-23T20:48:23.916956+0000","last_unstale":"2023-09-23T20:48:54.183762+0000","last_undegraded":"2023-09-23T20:48:54.183762+0000","last_fullsized":"2023-09-23T20:48:54.183762+0000","mapping_epoch":577,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":578,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:42:27.989294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":26,"stat_sum":{"num_bytes":2154168344,"num_objects":32871,"num_object_clones":0,"num_object_copies":98613,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":32871,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":176117,"num_write_kb":6687553,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":13527,"num_bytes_recovered":886112256,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":6988312576,"data_stored":6988300360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":160592,"ondisk_log_size":160592,"up":78,"acting":78,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":109,"num_bytes_recovered":15220,"num_keys_recovered":80,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26691,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":661927,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":59,"num_read_kb":110,"num_write":28,"num_write_kb":291,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":477366,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5983,"num_read_kb":13200,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":135,"num_bytes_recovered":25165824,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6082,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201730,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1300624,"kb_used_data":715656,"kb_used_omap":0,"kb_used_meta":584960,"kb_avail":92444528,"statfs":{"total":95995035648,"available":94663196672,"internally_reserved":0,"allocated":732831744,"data_stored":731200528,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":598999040},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:48:06 2023","interfaces":[{"interface":"back","average":{"1min":1.5569999999999999,"5min":1.972,"15min":2.1899999999999999},"min":{"1min":0.224,"5min":0.224,"15min":0.224},"max":{"1min":5.6970000000000001,"5min":15.1,"15min":18.315000000000001},"last":0.498},{"interface":"front","average":{"1min":2.1539999999999999,"5min":2.1379999999999999,"15min":1.903},"min":{"1min":0.28000000000000003,"5min":0.25800000000000001,"15min":0.23899999999999999},"max":{"1min":6.5780000000000003,"5min":15.327999999999999,"15min":19.355},"last":0.40999999999999998}]},{"osd":1,"last update":"Sat Sep 23 20:48:13 2023","interfaces":[{"interface":"back","average":{"1min":1.3640000000000001,"5min":1.6779999999999999,"15min":1.972},"min":{"1min":0.28399999999999997,"5min":0.25900000000000001,"15min":0.24399999999999999},"max":{"1min":6.4610000000000003,"5min":15.131,"15min":49.965000000000003},"last":0.45000000000000001},{"interface":"front","average":{"1min":2.4929999999999999,"5min":2.524,"15min":2.5099999999999998},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.26700000000000002},"max":{"1min":10.861000000000001,"5min":15.18,"15min":53.451999999999998},"last":0.55900000000000005}]},{"osd":2,"last update":"Sat Sep 23 20:48:06 2023","interfaces":[{"interface":"back","average":{"1min":2.0779999999999998,"5min":2.153,"15min":2.1899999999999999},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.27800000000000002},"max":{"1min":5.4770000000000003,"5min":15.087,"15min":16.965},"last":0.58399999999999996},{"interface":"front","average":{"1min":2.206,"5min":2.0249999999999999,"15min":2.339},"min":{"1min":0.45200000000000001,"5min":0.31,"15min":0.27400000000000002},"max":{"1min":6.6379999999999999,"5min":15.193,"15min":23.494},"last":0.436}]},{"osd":3,"last update":"Sat Sep 23 20:48:38 2023","interfaces":[{"interface":"back","average":{"1min":2.4140000000000001,"5min":1.964,"15min":2.226},"min":{"1min":0.34399999999999997,"5min":0.33800000000000002,"15min":0.311},"max":{"1min":7.2880000000000003,"5min":15.115,"15min":23.352},"last":0.48399999999999999},{"interface":"front","average":{"1min":2.3180000000000001,"5min":2.25,"15min":2.4239999999999999},"min":{"1min":0.35399999999999998,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":6.5090000000000003,"5min":15.144,"15min":20.969999999999999},"last":0.53100000000000003}]},{"osd":4,"last update":"Sat Sep 23 20:48:38 2023","interfaces":[{"interface":"back","average":{"1min":1.583,"5min":1.458,"15min":1.2350000000000001},"min":{"1min":0.26500000000000001,"5min":0.254,"15min":0.23200000000000001},"max":{"1min":7.4109999999999996,"5min":12.260999999999999,"15min":18.364999999999998},"last":0.59499999999999997},{"interface":"front","average":{"1min":1.1479999999999999,"5min":1.6659999999999999,"15min":1.45},"min":{"1min":0.34599999999999997,"5min":0.247,"15min":0.247},"max":{"1min":4.1529999999999996,"5min":7.2469999999999999,"15min":23.617000000000001},"last":0.46899999999999997}]},{"osd":5,"last update":"Sat Sep 23 20:48:38 2023","interfaces":[{"interface":"back","average":{"1min":1.548,"5min":1.48,"15min":1.2709999999999999},"min":{"1min":0.28100000000000003,"5min":0.27200000000000002,"15min":0.26600000000000001},"max":{"1min":7.2270000000000003,"5min":15.005000000000001,"15min":15.005000000000001},"last":0.38800000000000001},{"interface":"front","average":{"1min":1.1890000000000001,"5min":1.6060000000000001,"15min":1.3899999999999999},"min":{"1min":0.32200000000000001,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":7.335,"5min":12.212,"15min":18.390000000000001},"last":0.56899999999999995}]},{"osd":6,"last update":"Sat Sep 23 20:48:38 2023","interfaces":[{"interface":"back","average":{"1min":1.6879999999999999,"5min":2.052,"15min":1.544},"min":{"1min":0.379,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":5.6070000000000002,"5min":15.038,"15min":18.417000000000002},"last":0.54200000000000004},{"interface":"front","average":{"1min":1.3100000000000001,"5min":1.9039999999999999,"15min":1.4399999999999999},"min":{"1min":0.378,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":4.3710000000000004,"5min":15.369999999999999,"15min":15.369999999999999},"last":0.61299999999999999}]}]},{"osd":6,"up_from":47,"seq":201863463369,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1418064,"kb_used_data":915912,"kb_used_omap":0,"kb_used_meta":502080,"kb_avail":92327088,"statfs":{"total":95995035648,"available":94542938112,"internally_reserved":0,"allocated":937893888,"data_stored":936270069,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":514129920},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:48:03 2023","interfaces":[{"interface":"back","average":{"1min":2.9780000000000002,"5min":2.0299999999999998,"15min":1.9419999999999999},"min":{"1min":0.25,"5min":0.219,"15min":0.219},"max":{"1min":15.574999999999999,"5min":21.018000000000001,"15min":34.143999999999998},"last":0.24399999999999999},{"interface":"front","average":{"1min":1.698,"5min":2.1909999999999998,"15min":2.2429999999999999},"min":{"1min":0.255,"5min":0.255,"15min":0.22700000000000001},"max":{"1min":4.8840000000000003,"5min":21.082000000000001,"15min":60.328000000000003},"last":0.33400000000000002}]},{"osd":1,"last update":"Sat Sep 23 20:48:20 2023","interfaces":[{"interface":"back","average":{"1min":3.0390000000000001,"5min":2.2309999999999999,"15min":2.0819999999999999},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.249},"max":{"1min":8.9350000000000005,"5min":9.2520000000000007,"15min":64.274000000000001},"last":0.371},{"interface":"front","average":{"1min":2.746,"5min":1.9730000000000001,"15min":1.869},"min":{"1min":0.32300000000000001,"5min":0.29099999999999998,"15min":0.28000000000000003},"max":{"1min":8.0969999999999995,"5min":17.786000000000001,"15min":34.490000000000002},"last":0.48999999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:48:03 2023","interfaces":[{"interface":"back","average":{"1min":2.4609999999999999,"5min":2.4380000000000002,"15min":2.242},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.27700000000000002},"max":{"1min":6.1529999999999996,"5min":22.591000000000001,"15min":34.317999999999998},"last":0.432},{"interface":"front","average":{"1min":2.4039999999999999,"5min":2.0350000000000001,"15min":2.0539999999999998},"min":{"1min":0.45400000000000001,"5min":0.33900000000000002,"15min":0.30399999999999999},"max":{"1min":7.8710000000000004,"5min":22.614999999999998,"15min":34.292999999999999},"last":0.52800000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:47:54 2023","interfaces":[{"interface":"back","average":{"1min":2.0659999999999998,"5min":1.946,"15min":1.9339999999999999},"min":{"1min":0.307,"5min":0.307,"15min":0.28399999999999997},"max":{"1min":9.6859999999999999,"5min":11.664,"15min":11.664},"last":0.46999999999999997},{"interface":"front","average":{"1min":2.2959999999999998,"5min":2.1859999999999999,"15min":2.1539999999999999},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.30499999999999999},"max":{"1min":9.7810000000000006,"5min":10.125999999999999,"15min":12.875999999999999},"last":0.55000000000000004}]},{"osd":4,"last update":"Sat Sep 23 20:48:20 2023","interfaces":[{"interface":"back","average":{"1min":1.923,"5min":1.129,"15min":0.94699999999999995},"min":{"1min":0.29599999999999999,"5min":0.28699999999999998,"15min":0.25600000000000001},"max":{"1min":8.1229999999999993,"5min":10.183999999999999,"15min":23.904},"last":0.44800000000000001},{"interface":"front","average":{"1min":2.1499999999999999,"5min":1.405,"15min":1.179},"min":{"1min":0.377,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":9.7189999999999994,"5min":9.7189999999999994,"15min":11.515000000000001},"last":0.41899999999999998}]},{"osd":5,"last update":"Sat Sep 23 20:48:20 2023","interfaces":[{"interface":"back","average":{"1min":2.6419999999999999,"5min":1.4039999999999999,"15min":1.0029999999999999},"min":{"1min":0.371,"5min":0.253,"15min":0.253},"max":{"1min":8.0440000000000005,"5min":8.0440000000000005,"15min":8.0440000000000005},"last":0.51100000000000001},{"interface":"front","average":{"1min":1.6699999999999999,"5min":1.548,"15min":1.2729999999999999},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":5.1900000000000004,"5min":9.3559999999999999,"15min":9.3559999999999999},"last":0.51600000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:47:54 2023","interfaces":[{"interface":"back","average":{"1min":1.421,"5min":1.331,"15min":0.98499999999999999},"min":{"1min":0.35199999999999998,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":7.1379999999999999,"5min":8.9309999999999992,"15min":8.9309999999999992},"last":0.55900000000000005},{"interface":"front","average":{"1min":2.0459999999999998,"5min":2.0150000000000001,"15min":1.3939999999999999},"min":{"1min":0.434,"5min":0.35999999999999999,"15min":0.34499999999999997},"max":{"1min":9.7530000000000001,"5min":9.8680000000000003,"15min":9.8680000000000003},"last":0.57599999999999996}]}]},{"osd":1,"up_from":266,"seq":1142461300967,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1156104,"kb_used_data":405568,"kb_used_omap":8,"kb_used_meta":750519,"kb_avail":92589048,"statfs":{"total":95995035648,"available":94811185152,"internally_reserved":0,"allocated":415301632,"data_stored":412335051,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8206,"internal_metadata":768532466},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:48:24 2023","interfaces":[{"interface":"back","average":{"1min":2.9710000000000001,"5min":2.4969999999999999,"15min":2.3559999999999999},"min":{"1min":0.27700000000000002,"5min":0.20799999999999999,"15min":0.20000000000000001},"max":{"1min":9.4730000000000008,"5min":15.532999999999999,"15min":16.292000000000002},"last":0.28499999999999998},{"interface":"front","average":{"1min":3.1469999999999998,"5min":2.9060000000000001,"15min":2.9430000000000001},"min":{"1min":0.35899999999999999,"5min":0.22600000000000001,"15min":0.19900000000000001},"max":{"1min":8.1839999999999993,"5min":20.346,"15min":26.588000000000001},"last":0.217}]},{"osd":2,"last update":"Sat Sep 23 20:48:24 2023","interfaces":[{"interface":"back","average":{"1min":3.1850000000000001,"5min":2.8340000000000001,"15min":3.1429999999999998},"min":{"1min":0.249,"5min":0.23699999999999999,"15min":0.216},"max":{"1min":9.5779999999999994,"5min":19.026,"15min":32.255000000000003},"last":1.6000000000000001},{"interface":"front","average":{"1min":3.5030000000000001,"5min":2.423,"15min":2.3769999999999998},"min":{"1min":0.315,"5min":0.26000000000000001,"15min":0.20899999999999999},"max":{"1min":10.567,"5min":10.567,"15min":13.404999999999999},"last":0.34899999999999998}]},{"osd":3,"last update":"Sat Sep 23 20:48:24 2023","interfaces":[{"interface":"back","average":{"1min":3.294,"5min":2.2719999999999998,"15min":2.4780000000000002},"min":{"1min":0.373,"5min":0.28799999999999998,"15min":0.247},"max":{"1min":10.135999999999999,"5min":17.366,"15min":17.366},"last":1.575},{"interface":"front","average":{"1min":3.2170000000000001,"5min":2.8490000000000002,"15min":2.7949999999999999},"min":{"1min":0.26300000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":9.8339999999999996,"5min":11.534000000000001,"15min":19.315000000000001},"last":0.436}]},{"osd":4,"last update":"Sat Sep 23 20:48:24 2023","interfaces":[{"interface":"back","average":{"1min":2.7909999999999999,"5min":2.3769999999999998,"15min":2.4079999999999999},"min":{"1min":0.34799999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":9.5359999999999996,"5min":11.791,"15min":24.327999999999999},"last":1.677},{"interface":"front","average":{"1min":2.5600000000000001,"5min":1.7949999999999999,"15min":1.7749999999999999},"min":{"1min":0.36299999999999999,"5min":0.34699999999999998,"15min":0.311},"max":{"1min":9.8249999999999993,"5min":10.468999999999999,"15min":20.835999999999999},"last":0.39000000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:48:24 2023","interfaces":[{"interface":"back","average":{"1min":2.4470000000000001,"5min":2.3889999999999998,"15min":2.3660000000000001},"min":{"1min":0.41399999999999998,"5min":0.308,"15min":0.308},"max":{"1min":8.1210000000000004,"5min":14.366,"15min":33.401000000000003},"last":0.53500000000000003},{"interface":"front","average":{"1min":2.589,"5min":2.548,"15min":2.633},"min":{"1min":0.39700000000000002,"5min":0.33100000000000002,"15min":0.32500000000000001},"max":{"1min":9.6140000000000008,"5min":11.750999999999999,"15min":33.417000000000002},"last":1.7110000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:48:24 2023","interfaces":[{"interface":"back","average":{"1min":2.5049999999999999,"5min":2.3650000000000002,"15min":2.2570000000000001},"min":{"1min":0.48799999999999999,"5min":0.316,"15min":0.316},"max":{"1min":8.0960000000000001,"5min":14.414999999999999,"15min":14.414999999999999},"last":0.504},{"interface":"front","average":{"1min":3.2650000000000001,"5min":3.004,"15min":2.6739999999999999},"min":{"1min":0.432,"5min":0.38500000000000001,"15min":0.317},"max":{"1min":9.6639999999999997,"5min":11.832000000000001,"15min":24.376000000000001},"last":1.8560000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:48:24 2023","interfaces":[{"interface":"back","average":{"1min":2.8330000000000002,"5min":2.5219999999999998,"15min":2.4249999999999998},"min":{"1min":0.46899999999999997,"5min":0.377,"15min":0.37},"max":{"1min":8.1590000000000007,"5min":10.010999999999999,"15min":18.437000000000001},"last":0.56399999999999995},{"interface":"front","average":{"1min":3.5739999999999998,"5min":2.871,"15min":2.714},"min":{"1min":0.47299999999999998,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":9.8369999999999997,"5min":11.864000000000001,"15min":24.524000000000001},"last":2.0529999999999999}]}]},{"osd":0,"up_from":94,"seq":403726926169,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2308036,"kb_used_data":1371068,"kb_used_omap":7,"kb_used_meta":936952,"kb_avail":91437116,"statfs":{"total":95995035648,"available":93631606784,"internally_reserved":0,"allocated":1403973632,"data_stored":1401171599,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":959438865},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,0,0,0,1,3],"upper_bound":4096},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:48:32 2023","interfaces":[{"interface":"back","average":{"1min":4.1180000000000003,"5min":3.851,"15min":3.0579999999999998},"min":{"1min":0.34799999999999998,"5min":0.184,"15min":0.17199999999999999},"max":{"1min":9.6449999999999996,"5min":152.387,"15min":152.387},"last":0.36499999999999999},{"interface":"front","average":{"1min":3.7109999999999999,"5min":3.9390000000000001,"15min":3.2149999999999999},"min":{"1min":0.218,"5min":0.218,"15min":0.20399999999999999},"max":{"1min":12.317,"5min":152.84100000000001,"15min":152.84100000000001},"last":0.45400000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:48:01 2023","interfaces":[{"interface":"back","average":{"1min":4.4219999999999997,"5min":2.9350000000000001,"15min":2.9950000000000001},"min":{"1min":0.254,"5min":0.23499999999999999,"15min":0.193},"max":{"1min":14.538,"5min":15.108000000000001,"15min":62.726999999999997},"last":0.439},{"interface":"front","average":{"1min":2.8250000000000002,"5min":2.7149999999999999,"15min":2.9990000000000001},"min":{"1min":0.27100000000000002,"5min":0.217,"15min":0.217},"max":{"1min":8.3689999999999998,"5min":17.013000000000002,"15min":62.689},"last":0.51400000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:48:40 2023","interfaces":[{"interface":"back","average":{"1min":3.8650000000000002,"5min":3.6709999999999998,"15min":3.2229999999999999},"min":{"1min":0.36499999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":14.494,"5min":55.017000000000003,"15min":55.017000000000003},"last":0.47299999999999998},{"interface":"front","average":{"1min":3.6339999999999999,"5min":3.161,"15min":3.234},"min":{"1min":0.38,"5min":0.23799999999999999,"15min":0.21199999999999999},"max":{"1min":11.547000000000001,"5min":55.665999999999997,"15min":55.665999999999997},"last":0.498}]},{"osd":4,"last update":"Sat Sep 23 20:48:01 2023","interfaces":[{"interface":"back","average":{"1min":2.9860000000000002,"5min":2.0139999999999998,"15min":2.1299999999999999},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.25900000000000001},"max":{"1min":11.75,"5min":11.75,"15min":12.587999999999999},"last":0.68500000000000005},{"interface":"front","average":{"1min":3.609,"5min":2.125,"15min":2.093},"min":{"1min":0.31,"5min":0.28899999999999998,"15min":0.27400000000000002},"max":{"1min":12.356999999999999,"5min":12.356999999999999,"15min":23.001000000000001},"last":0.58099999999999996}]},{"osd":5,"last update":"Sat Sep 23 20:48:01 2023","interfaces":[{"interface":"back","average":{"1min":3.4590000000000001,"5min":2.097,"15min":2.169},"min":{"1min":0.34300000000000003,"5min":0.30599999999999999,"15min":0.26500000000000001},"max":{"1min":9.3339999999999996,"5min":9.3339999999999996,"15min":22.969999999999999},"last":0.621},{"interface":"front","average":{"1min":2.9300000000000002,"5min":2.3359999999999999,"15min":2.492},"min":{"1min":0.39000000000000001,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":10.089,"5min":12.601000000000001,"15min":26.228000000000002},"last":0.67100000000000004}]},{"osd":6,"last update":"Sat Sep 23 20:48:01 2023","interfaces":[{"interface":"back","average":{"1min":3.6920000000000002,"5min":2.3039999999999998,"15min":2.3439999999999999},"min":{"1min":0.41799999999999998,"5min":0.374,"15min":0.32300000000000001},"max":{"1min":9.2750000000000004,"5min":9.2750000000000004,"15min":24.056000000000001},"last":0.65900000000000003},{"interface":"front","average":{"1min":3.4020000000000001,"5min":2.395,"15min":2.3780000000000001},"min":{"1min":0.40999999999999998,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":12.427,"5min":12.698,"15min":26.268000000000001},"last":0.56899999999999995}]},{"osd":7,"last update":"Sat Sep 23 20:48:01 2023","interfaces":[{"interface":"back","average":{"1min":3.4140000000000001,"5min":2.7749999999999999,"15min":2.5870000000000002},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.36099999999999999},"max":{"1min":12.486000000000001,"5min":16.553999999999998,"15min":26.177},"last":0.60799999999999998},{"interface":"front","average":{"1min":3.3879999999999999,"5min":2.6179999999999999,"15min":2.5179999999999998},"min":{"1min":0.45300000000000001,"5min":0.40200000000000002,"15min":0.34200000000000003},"max":{"1min":10.183999999999999,"5min":16.649000000000001,"15min":22.422999999999998},"last":0.64100000000000001}]}]},{"osd":2,"up_from":104,"seq":446676599117,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1677564,"kb_used_data":827764,"kb_used_omap":11,"kb_used_meta":849780,"kb_avail":92067588,"statfs":{"total":95995035648,"available":94277210112,"internally_reserved":0,"allocated":847630336,"data_stored":844839355,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11878,"internal_metadata":870175130},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:48:11 2023","interfaces":[{"interface":"back","average":{"1min":2.327,"5min":1.738,"15min":2.0710000000000002},"min":{"1min":0.27300000000000002,"5min":0.20499999999999999,"15min":0.17299999999999999},"max":{"1min":9.7919999999999998,"5min":15.324999999999999,"15min":28.234999999999999},"last":0.33900000000000002},{"interface":"front","average":{"1min":2.4169999999999998,"5min":2.0710000000000002,"15min":2.2559999999999998},"min":{"1min":0.30399999999999999,"5min":0.245,"15min":0.19},"max":{"1min":8.3789999999999996,"5min":16.678999999999998,"15min":16.678999999999998},"last":0.46899999999999997}]},{"osd":1,"last update":"Sat Sep 23 20:48:11 2023","interfaces":[{"interface":"back","average":{"1min":3.1560000000000001,"5min":2.1549999999999998,"15min":2.5910000000000002},"min":{"1min":0.248,"5min":0.219,"15min":0.219},"max":{"1min":11.012,"5min":21.690000000000001,"15min":46.405000000000001},"last":0.30599999999999999},{"interface":"front","average":{"1min":1.976,"5min":1.905,"15min":2.5379999999999998},"min":{"1min":0.254,"5min":0.22,"15min":0.22},"max":{"1min":10.191000000000001,"5min":13.215999999999999,"15min":46.459000000000003},"last":0.40600000000000003}]},{"osd":3,"last update":"Sat Sep 23 20:48:40 2023","interfaces":[{"interface":"back","average":{"1min":2.1509999999999998,"5min":2.516,"15min":2.605},"min":{"1min":0.311,"5min":0.253,"15min":0.253},"max":{"1min":10.051,"5min":30.283000000000001,"15min":30.283000000000001},"last":0.53400000000000003},{"interface":"front","average":{"1min":2.6840000000000002,"5min":1.8320000000000001,"15min":2.0499999999999998},"min":{"1min":0.33000000000000002,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":10.221,"5min":30.834,"15min":30.834},"last":0.373}]},{"osd":4,"last update":"Sat Sep 23 20:48:11 2023","interfaces":[{"interface":"back","average":{"1min":2.3380000000000001,"5min":1.9550000000000001,"15min":1.851},"min":{"1min":0.435,"5min":0.32000000000000001,"15min":0.27500000000000002},"max":{"1min":8.3469999999999995,"5min":11.141999999999999,"15min":12.025},"last":0.42599999999999999},{"interface":"front","average":{"1min":2.4940000000000002,"5min":1.9370000000000001,"15min":2.0760000000000001},"min":{"1min":0.375,"5min":0.35699999999999998,"15min":0.30499999999999999},"max":{"1min":8.2289999999999992,"5min":10.948,"15min":19.440999999999999},"last":0.39000000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:48:11 2023","interfaces":[{"interface":"back","average":{"1min":1.9810000000000001,"5min":1.7689999999999999,"15min":1.8660000000000001},"min":{"1min":0.42899999999999999,"5min":0.311,"15min":0.311},"max":{"1min":8.5380000000000003,"5min":11.183,"15min":12.07},"last":0.52000000000000002},{"interface":"front","average":{"1min":2.5630000000000002,"5min":1.8859999999999999,"15min":2.097},"min":{"1min":0.42399999999999999,"5min":0.373,"15min":0.35299999999999998},"max":{"1min":8.2729999999999997,"5min":11.061999999999999,"15min":19.513000000000002},"last":0.49299999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:48:11 2023","interfaces":[{"interface":"back","average":{"1min":2.7200000000000002,"5min":2.1520000000000001,"15min":2.202},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.33500000000000002},"max":{"1min":9.2319999999999993,"5min":11.894,"15min":19.486000000000001},"last":0.44400000000000001},{"interface":"front","average":{"1min":1.7110000000000001,"5min":1.657,"15min":1.6459999999999999},"min":{"1min":0.46700000000000003,"5min":0.32900000000000001,"15min":0.318},"max":{"1min":7.4180000000000001,"5min":11.747,"15min":11.747},"last":0.57599999999999996}]},{"osd":7,"last update":"Sat Sep 23 20:48:11 2023","interfaces":[{"interface":"back","average":{"1min":1.643,"5min":1.516,"15min":1.635},"min":{"1min":0.46300000000000002,"5min":0.35399999999999998,"15min":0.34699999999999998},"max":{"1min":7.391,"5min":7.718,"15min":12.734999999999999},"last":0.60299999999999998},{"interface":"front","average":{"1min":3.375,"5min":2.2930000000000001,"15min":2.1299999999999999},"min":{"1min":0.502,"5min":0.35799999999999998,"15min":0.34899999999999998},"max":{"1min":10.071999999999999,"5min":11.218,"15min":12.099},"last":0.56999999999999995}]}]},{"osd":3,"up_from":109,"seq":468151435591,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1717128,"kb_used_data":666624,"kb_used_omap":4,"kb_used_meta":1050491,"kb_avail":92028024,"statfs":{"total":95995035648,"available":94236696576,"internally_reserved":0,"allocated":682622976,"data_stored":679835119,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4514,"internal_metadata":1075703390},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:48:48 2023","interfaces":[{"interface":"back","average":{"1min":1.4750000000000001,"5min":2.3799999999999999,"15min":2.4790000000000001},"min":{"1min":0.252,"5min":0.217,"15min":0.20899999999999999},"max":{"1min":5.2110000000000003,"5min":21.792999999999999,"15min":22.32},"last":0.31},{"interface":"front","average":{"1min":1.454,"5min":2.3340000000000001,"15min":2.5289999999999999},"min":{"1min":0.249,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":5.4530000000000003,"5min":18.579999999999998,"15min":22.805},"last":0.254}]},{"osd":1,"last update":"Sat Sep 23 20:48:19 2023","interfaces":[{"interface":"back","average":{"1min":2.7530000000000001,"5min":2.1709999999999998,"15min":2.0840000000000001},"min":{"1min":0.26100000000000001,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":11.298,"5min":15.177,"15min":17.529},"last":0.38900000000000001},{"interface":"front","average":{"1min":2.7669999999999999,"5min":2.0830000000000002,"15min":2.2490000000000001},"min":{"1min":0.27200000000000002,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":11.380000000000001,"5min":11.380000000000001,"15min":14.19},"last":0.29599999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:48:48 2023","interfaces":[{"interface":"back","average":{"1min":1.669,"5min":2.403,"15min":2.8650000000000002},"min":{"1min":0.27900000000000003,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":5.4210000000000003,"5min":12.473000000000001,"15min":68.206999999999994},"last":0.32800000000000001},{"interface":"front","average":{"1min":2.7810000000000001,"5min":2.1200000000000001,"15min":2.5659999999999998},"min":{"1min":0.374,"5min":0.251,"15min":0.245},"max":{"1min":16.559000000000001,"5min":17.919,"15min":67.736999999999995},"last":0.44400000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:48:48 2023","interfaces":[{"interface":"back","average":{"1min":1.8200000000000001,"5min":1.744,"15min":1.708},"min":{"1min":0.435,"5min":0.35499999999999998,"15min":0.30499999999999999},"max":{"1min":5.8029999999999999,"5min":9.0259999999999998,"15min":22.157},"last":0.432},{"interface":"front","average":{"1min":3.2530000000000001,"5min":1.8759999999999999,"15min":2.1869999999999998},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.29399999999999998},"max":{"1min":16.529,"5min":16.529,"15min":16.529},"last":0.47099999999999997}]},{"osd":5,"last update":"Sat Sep 23 20:48:48 2023","interfaces":[{"interface":"back","average":{"1min":1.734,"5min":1.849,"15min":1.921},"min":{"1min":0.49299999999999999,"5min":0.33800000000000002,"15min":0.32300000000000001},"max":{"1min":6.7210000000000001,"5min":8.9849999999999994,"15min":22.289999999999999},"last":0.53000000000000003},{"interface":"front","average":{"1min":2.25,"5min":2.0470000000000002,"15min":1.831},"min":{"1min":0.43099999999999999,"5min":0.34100000000000003,"15min":0.32000000000000001},"max":{"1min":6.008,"5min":8.6129999999999995,"15min":12.952},"last":0.48199999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:48:48 2023","interfaces":[{"interface":"back","average":{"1min":2.6699999999999999,"5min":2.4660000000000002,"15min":2.036},"min":{"1min":0.54200000000000004,"5min":0.36599999999999999,"15min":0.34300000000000003},"max":{"1min":7.4359999999999999,"5min":8.6790000000000003,"15min":12.917},"last":0.50700000000000001},{"interface":"front","average":{"1min":2.633,"5min":2.0270000000000001,"15min":2.2290000000000001},"min":{"1min":0.39400000000000002,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":16.497,"5min":16.497,"15min":16.497},"last":0.60699999999999998}]},{"osd":7,"last update":"Sat Sep 23 20:48:48 2023","interfaces":[{"interface":"back","average":{"1min":3.173,"5min":2.1219999999999999,"15min":2.4300000000000002},"min":{"1min":0.48199999999999998,"5min":0.42399999999999999,"15min":0.39900000000000002},"max":{"1min":16.425000000000001,"5min":16.425000000000001,"15min":16.425000000000001},"last":0.58599999999999997},{"interface":"front","average":{"1min":3.3330000000000002,"5min":2.3399999999999999,"15min":2.0899999999999999},"min":{"1min":0.56100000000000005,"5min":0.46300000000000002,"15min":0.40999999999999998},"max":{"1min":11.891999999999999,"5min":11.891999999999999,"15min":22.202999999999999},"last":0.57499999999999996}]}]},{"osd":4,"up_from":30,"seq":128849019345,"num_pgs":60,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1770192,"kb_used_data":1224520,"kb_used_omap":0,"kb_used_meta":545664,"kb_avail":91974960,"statfs":{"total":95995035648,"available":94182359040,"internally_reserved":0,"allocated":1253908480,"data_stored":1252284395,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":558759936},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:48:04 2023","interfaces":[{"interface":"back","average":{"1min":2.1320000000000001,"5min":1.8500000000000001,"15min":1.8420000000000001},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24099999999999999},"max":{"1min":7.1120000000000001,"5min":11.709,"15min":24.114999999999998},"last":0.56299999999999994},{"interface":"front","average":{"1min":2.4329999999999998,"5min":2.1480000000000001,"15min":2.0870000000000002},"min":{"1min":0.29799999999999999,"5min":0.25800000000000001,"15min":0.246},"max":{"1min":6.8639999999999999,"5min":13.74,"15min":24.135999999999999},"last":0.48699999999999999}]},{"osd":1,"last update":"Sat Sep 23 20:48:16 2023","interfaces":[{"interface":"back","average":{"1min":2.093,"5min":1.9370000000000001,"15min":1.8280000000000001},"min":{"1min":0.27800000000000002,"5min":0.26300000000000001,"15min":0.25900000000000001},"max":{"1min":10.157,"5min":10.157,"15min":18.190999999999999},"last":0.38800000000000001},{"interface":"front","average":{"1min":2.278,"5min":1.6160000000000001,"15min":1.6319999999999999},"min":{"1min":0.29399999999999998,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":10.169,"5min":10.169,"15min":18.173999999999999},"last":0.59699999999999998}]},{"osd":2,"last update":"Sat Sep 23 20:48:16 2023","interfaces":[{"interface":"back","average":{"1min":2.0880000000000001,"5min":1.879,"15min":1.752},"min":{"1min":0.32000000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":4.5739999999999998,"5min":9.3320000000000007,"15min":13.051},"last":0.503},{"interface":"front","average":{"1min":2.089,"5min":2.0550000000000002,"15min":2.1699999999999999},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":7.0800000000000001,"5min":8.7230000000000008,"15min":14.614000000000001},"last":0.45000000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:48:32 2023","interfaces":[{"interface":"back","average":{"1min":2.7749999999999999,"5min":2.0139999999999998,"15min":1.9530000000000001},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":11.598000000000001,"5min":11.598000000000001,"15min":12.147},"last":0.65800000000000003},{"interface":"front","average":{"1min":2.3530000000000002,"5min":2.1150000000000002,"15min":2.0870000000000002},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.30299999999999999},"max":{"1min":7.7830000000000004,"5min":17.169,"15min":17.169},"last":0.628}]},{"osd":5,"last update":"Sat Sep 23 20:48:04 2023","interfaces":[{"interface":"back","average":{"1min":0.79300000000000004,"5min":1.458,"15min":1.0740000000000001},"min":{"1min":0.28399999999999997,"5min":0.253,"15min":0.253},"max":{"1min":3.5790000000000002,"5min":10.202,"15min":10.202},"last":0.53300000000000003},{"interface":"front","average":{"1min":1.1479999999999999,"5min":1.5369999999999999,"15min":1.1850000000000001},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":4.7430000000000003,"5min":8.1440000000000001,"15min":8.1440000000000001},"last":0.52300000000000002}]},{"osd":6,"last update":"Sat Sep 23 20:48:16 2023","interfaces":[{"interface":"back","average":{"1min":1.9910000000000001,"5min":1.6140000000000001,"15min":1.395},"min":{"1min":0.30099999999999999,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":6.4290000000000003,"5min":10.840999999999999,"15min":14.731999999999999},"last":0.55200000000000005},{"interface":"front","average":{"1min":1.681,"5min":1.6359999999999999,"15min":1.234},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":8.0269999999999992,"5min":8.0269999999999992,"15min":8.0269999999999992},"last":0.41499999999999998}]},{"osd":7,"last update":"Sat Sep 23 20:48:32 2023","interfaces":[{"interface":"back","average":{"1min":1.7070000000000001,"5min":4.9939999999999998,"15min":2.7130000000000001},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":11.228,"5min":313.50099999999998,"15min":313.50099999999998},"last":0.60999999999999999},{"interface":"front","average":{"1min":1.8300000000000001,"5min":4.4589999999999996,"15min":2.2919999999999998},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":17.093,"5min":312.67500000000001,"15min":312.67500000000001},"last":0.58099999999999996}]}]},{"osd":5,"up_from":39,"seq":167503725006,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1437952,"kb_used_data":835768,"kb_used_omap":0,"kb_used_meta":602176,"kb_avail":92307200,"statfs":{"total":95995035648,"available":94522572800,"internally_reserved":0,"allocated":855826432,"data_stored":854207293,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":616628224},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:48:10 2023","interfaces":[{"interface":"back","average":{"1min":2.5129999999999999,"5min":1.8700000000000001,"15min":1.891},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.20699999999999999},"max":{"1min":6.673,"5min":15.018000000000001,"15min":18.16},"last":0.24199999999999999},{"interface":"front","average":{"1min":2.258,"5min":1.915,"15min":1.8300000000000001},"min":{"1min":0.30099999999999999,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":8.6050000000000004,"5min":15.034000000000001,"15min":15.034000000000001},"last":0.46500000000000002}]},{"osd":1,"last update":"Sat Sep 23 20:48:23 2023","interfaces":[{"interface":"back","average":{"1min":2.4460000000000002,"5min":2.1480000000000001,"15min":2.3719999999999999},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.253},"max":{"1min":6.9850000000000003,"5min":9.4199999999999999,"15min":61.005000000000003},"last":0.35999999999999999},{"interface":"front","average":{"1min":2.6720000000000002,"5min":1.9390000000000001,"15min":2.2269999999999999},"min":{"1min":0.38200000000000001,"5min":0.27700000000000002,"15min":0.23499999999999999},"max":{"1min":6.6280000000000001,"5min":16.219000000000001,"15min":59.688000000000002},"last":0.52700000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:48:10 2023","interfaces":[{"interface":"back","average":{"1min":2.6160000000000001,"5min":2.1400000000000001,"15min":2.1579999999999999},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26500000000000001},"max":{"1min":7.194,"5min":12.226000000000001,"15min":37.011000000000003},"last":0.316},{"interface":"front","average":{"1min":2.5219999999999998,"5min":1.867,"15min":2.0350000000000001},"min":{"1min":0.39600000000000002,"5min":0.314,"15min":0.255},"max":{"1min":5.9790000000000001,"5min":9.8260000000000005,"15min":36.835000000000001},"last":0.51100000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:48:30 2023","interfaces":[{"interface":"back","average":{"1min":2.9380000000000002,"5min":1.9650000000000001,"15min":2.3170000000000002},"min":{"1min":0.54100000000000004,"5min":0.33800000000000002,"15min":0.314},"max":{"1min":8.7360000000000007,"5min":8.7360000000000007,"15min":36.716000000000001},"last":0.39700000000000002},{"interface":"front","average":{"1min":3.3719999999999999,"5min":2.0019999999999998,"15min":2.3730000000000002},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":7.4029999999999996,"5min":7.8869999999999996,"15min":18.504000000000001},"last":0.44400000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:48:00 2023","interfaces":[{"interface":"back","average":{"1min":1.593,"5min":1.8180000000000001,"15min":1.2270000000000001},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.255},"max":{"1min":6.3170000000000002,"5min":13.391999999999999,"15min":13.391999999999999},"last":0.42899999999999999},{"interface":"front","average":{"1min":1.5129999999999999,"5min":1.6899999999999999,"15min":1.2390000000000001},"min":{"1min":0.34200000000000003,"5min":0.29199999999999998,"15min":0.28299999999999997},"max":{"1min":6.6600000000000001,"5min":7.3799999999999999,"15min":7.3799999999999999},"last":0.34599999999999997}]},{"osd":6,"last update":"Sat Sep 23 20:48:11 2023","interfaces":[{"interface":"back","average":{"1min":1.6319999999999999,"5min":1.913,"15min":1.431},"min":{"1min":0.40300000000000002,"5min":0.36199999999999999,"15min":0.30199999999999999},"max":{"1min":6.6959999999999997,"5min":13.734,"15min":13.734},"last":0.48399999999999999},{"interface":"front","average":{"1min":2.278,"5min":1.5129999999999999,"15min":1.28},"min":{"1min":0.40699999999999997,"5min":0.317,"15min":0.317},"max":{"1min":7.5270000000000001,"5min":7.5270000000000001,"15min":8.7449999999999992},"last":0.495}]},{"osd":7,"last update":"Sat Sep 23 20:48:30 2023","interfaces":[{"interface":"back","average":{"1min":2.4689999999999999,"5min":4.5060000000000002,"15min":2.2490000000000001},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.28499999999999998},"max":{"1min":7.899,"5min":294.70999999999998,"15min":294.70999999999998},"last":0.55300000000000005},{"interface":"front","average":{"1min":2.1240000000000001,"5min":4.5860000000000003,"15min":2.3149999999999999},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.27900000000000003},"max":{"1min":14.032999999999999,"5min":294.74400000000003,"15min":294.74400000000003},"last":0.53700000000000003}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4722,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1360,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8206,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7156,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3154,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1392840704,"data_stored":1392836632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":404029440,"data_stored":404029440,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":831324160,"data_stored":831324160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":674496512,"data_stored":674496512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1220743168,"data_stored":1220739096,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":831520768,"data_stored":831520768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":919408640,"data_stored":919404568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":713949184,"data_stored":713949184,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:49:00.182 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd pool set unique_pool_5 pgp_num 26 2023-09-23T20:49:00.211 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:49:00.214 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:00.215 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:49:00.316 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:49:00.404 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:00.538 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:49:00.542 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:00.543 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:49:00.643 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:49:00.676 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:00.714 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:49:00.717 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:00.718 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:49:00.818 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:49:00.881 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:00 smithi018 ceph-mon[184144]: failed to encode map e601 with expected crc 2023-09-23T20:49:00.882 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:00 smithi018 ceph-mon[184144]: osdmap e601: 8 total, 8 up, 8 in 2023-09-23T20:49:00.882 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:00 smithi018 ceph-mon[184144]: failed to encode map e601 with expected crc 2023-09-23T20:49:00.882 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:00 smithi018 ceph-mon[184144]: failed to encode map e601 with expected crc 2023-09-23T20:49:00.882 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:00 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "19"}]: dispatch 2023-09-23T20:49:00.882 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:00 smithi018 ceph-mon[179647]: failed to encode map e601 with expected crc 2023-09-23T20:49:00.882 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:00 smithi018 ceph-mon[179647]: osdmap e601: 8 total, 8 up, 8 in 2023-09-23T20:49:00.882 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:00 smithi018 ceph-mon[179647]: failed to encode map e601 with expected crc 2023-09-23T20:49:00.882 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:00 smithi018 ceph-mon[179647]: failed to encode map e601 with expected crc 2023-09-23T20:49:00.882 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:00 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "19"}]: dispatch 2023-09-23T20:49:00.884 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:00.913 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:49:00.953 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:49:00.959 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:00.960 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:49:00.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:00 smithi149 ceph-mon[170394]: failed to encode map e601 with expected crc 2023-09-23T20:49:00.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:00 smithi149 ceph-mon[170394]: osdmap e601: 8 total, 8 up, 8 in 2023-09-23T20:49:00.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:00 smithi149 ceph-mon[170394]: failed to encode map e601 with expected crc 2023-09-23T20:49:00.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:00 smithi149 ceph-mon[170394]: failed to encode map e601 with expected crc 2023-09-23T20:49:00.968 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:00 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "19"}]: dispatch 2023-09-23T20:49:01.061 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:49:01.109 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:01.169 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:49:01.173 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:01.174 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:49:01.274 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:49:01.319 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:01.366 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:49:01.369 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:01.369 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:49:01.470 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:49:01.515 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:01.560 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:49:01.587 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:49:01.703 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:01 smithi018 ceph-mon[179647]: pgmap v1504: 129 pgs: 129 active+clean; 1.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 501 op/s; 11 MiB/s, 175 objects/s recovering 2023-09-23T20:49:01.703 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:01 smithi018 ceph-mon[184144]: pgmap v1504: 129 pgs: 129 active+clean; 1.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 501 op/s; 11 MiB/s, 175 objects/s recovering 2023-09-23T20:49:01.705 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:01 smithi018 ceph-mon[184144]: failed to encode map e601 with expected crc 2023-09-23T20:49:01.705 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:01 smithi018 ceph-mon[184144]: failed to encode map e602 with expected crc 2023-09-23T20:49:01.714 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:49:01.718 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:01.718 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:49:01.819 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:49:01.847 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:01.884 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:49:01.887 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:01.887 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:49:01.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:01 smithi149 ceph-mon[170394]: pgmap v1504: 129 pgs: 129 active+clean; 1.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 501 op/s; 11 MiB/s, 175 objects/s recovering 2023-09-23T20:49:01.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:01 smithi149 ceph-mon[170394]: failed to encode map e601 with expected crc 2023-09-23T20:49:01.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:01 smithi149 ceph-mon[170394]: failed to encode map e602 with expected crc 2023-09-23T20:49:01.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:01 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "19"}]': finished 2023-09-23T20:49:01.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:01 smithi149 ceph-mon[170394]: osdmap e602: 8 total, 8 up, 8 in 2023-09-23T20:49:01.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:01 smithi149 ceph-mon[170394]: 10.11 starting backfill to osd.5 from (0'0,0'0] MAX to 599'10981 2023-09-23T20:49:01.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:01 smithi149 ceph-mon[170394]: failed to encode map e602 with expected crc 2023-09-23T20:49:01.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:01 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2023-09-23T20:49:01.968 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:01 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2023-09-23T20:49:01.968 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:01 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "10.1", "id": [0, 3]}]: dispatch 2023-09-23T20:49:01.968 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:01 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "10.2", "id": [4, 7]}]: dispatch 2023-09-23T20:49:01.968 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:01 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "10.6", "id": [0, 7]}]: dispatch 2023-09-23T20:49:01.968 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:01 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "10.c", "id": [0, 1]}]: dispatch 2023-09-23T20:49:01.969 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:01 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "10.16", "id": [5, 1]}]: dispatch 2023-09-23T20:49:01.969 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:01 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]': finished 2023-09-23T20:49:01.969 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:01 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]': finished 2023-09-23T20:49:01.969 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:01 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "10.1", "id": [0, 3]}]': finished 2023-09-23T20:49:01.969 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:01 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "10.2", "id": [4, 7]}]': finished 2023-09-23T20:49:01.969 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:01 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "10.6", "id": [0, 7]}]': finished 2023-09-23T20:49:01.969 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:01 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "10.c", "id": [0, 1]}]': finished 2023-09-23T20:49:01.970 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:01 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "10.16", "id": [5, 1]}]': finished 2023-09-23T20:49:01.970 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:01 smithi149 ceph-mon[170394]: osdmap e603: 8 total, 8 up, 8 in 2023-09-23T20:49:01.988 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:49:02.016 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:01 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "19"}]': finished 2023-09-23T20:49:02.016 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:01 smithi018 ceph-mon[184144]: osdmap e602: 8 total, 8 up, 8 in 2023-09-23T20:49:02.017 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:01 smithi018 ceph-mon[184144]: 10.11 starting backfill to osd.5 from (0'0,0'0] MAX to 599'10981 2023-09-23T20:49:02.017 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:01 smithi018 ceph-mon[184144]: failed to encode map e602 with expected crc 2023-09-23T20:49:02.017 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:01 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2023-09-23T20:49:02.017 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:01 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2023-09-23T20:49:02.017 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:01 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "10.1", "id": [0, 3]}]: dispatch 2023-09-23T20:49:02.017 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:01 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "10.2", "id": [4, 7]}]: dispatch 2023-09-23T20:49:02.017 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:01 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "10.6", "id": [0, 7]}]: dispatch 2023-09-23T20:49:02.017 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:01 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "10.c", "id": [0, 1]}]: dispatch 2023-09-23T20:49:02.017 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:01 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "10.16", "id": [5, 1]}]: dispatch 2023-09-23T20:49:02.017 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:01 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]': finished 2023-09-23T20:49:02.018 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:01 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]': finished 2023-09-23T20:49:02.018 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:01 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "10.1", "id": [0, 3]}]': finished 2023-09-23T20:49:02.018 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:01 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "10.2", "id": [4, 7]}]': finished 2023-09-23T20:49:02.018 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:01 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "10.6", "id": [0, 7]}]': finished 2023-09-23T20:49:02.018 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:01 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "10.c", "id": [0, 1]}]': finished 2023-09-23T20:49:02.018 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:01 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "10.16", "id": [5, 1]}]': finished 2023-09-23T20:49:02.018 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:01 smithi018 ceph-mon[184144]: osdmap e603: 8 total, 8 up, 8 in 2023-09-23T20:49:02.018 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:01 smithi018 ceph-mon[179647]: failed to encode map e601 with expected crc 2023-09-23T20:49:02.018 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:01 smithi018 ceph-mon[179647]: failed to encode map e602 with expected crc 2023-09-23T20:49:02.019 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:01 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "19"}]': finished 2023-09-23T20:49:02.019 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:01 smithi018 ceph-mon[179647]: osdmap e602: 8 total, 8 up, 8 in 2023-09-23T20:49:02.019 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:01 smithi018 ceph-mon[179647]: 10.11 starting backfill to osd.5 from (0'0,0'0] MAX to 599'10981 2023-09-23T20:49:02.019 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:01 smithi018 ceph-mon[179647]: failed to encode map e602 with expected crc 2023-09-23T20:49:02.019 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:01 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2023-09-23T20:49:02.019 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:01 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2023-09-23T20:49:02.019 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:01 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "10.1", "id": [0, 3]}]: dispatch 2023-09-23T20:49:02.019 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:01 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "10.2", "id": [4, 7]}]: dispatch 2023-09-23T20:49:02.019 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:01 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "10.6", "id": [0, 7]}]: dispatch 2023-09-23T20:49:02.019 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:01 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "10.c", "id": [0, 1]}]: dispatch 2023-09-23T20:49:02.019 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:01 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "10.16", "id": [5, 1]}]: dispatch 2023-09-23T20:49:02.020 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:01 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]': finished 2023-09-23T20:49:02.020 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:01 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]': finished 2023-09-23T20:49:02.020 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:01 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "10.1", "id": [0, 3]}]': finished 2023-09-23T20:49:02.020 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:01 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "10.2", "id": [4, 7]}]': finished 2023-09-23T20:49:02.020 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:01 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "10.6", "id": [0, 7]}]': finished 2023-09-23T20:49:02.020 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:01 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "10.c", "id": [0, 1]}]': finished 2023-09-23T20:49:02.020 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:01 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "10.16", "id": [5, 1]}]': finished 2023-09-23T20:49:02.020 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:01 smithi018 ceph-mon[179647]: osdmap e603: 8 total, 8 up, 8 in 2023-09-23T20:49:02.020 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:02.182 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:49:02.185 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:02.186 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:49:02.286 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:49:02.323 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:02.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:02 smithi018 ceph-mon[179647]: failed to encode map e603 with expected crc 2023-09-23T20:49:02.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:02 smithi018 ceph-mon[179647]: failed to encode map e603 with expected crc 2023-09-23T20:49:02.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:02 smithi018 ceph-mon[179647]: osdmap e604: 8 total, 8 up, 8 in 2023-09-23T20:49:02.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:02 smithi018 ceph-mon[184144]: failed to encode map e603 with expected crc 2023-09-23T20:49:02.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:02 smithi018 ceph-mon[184144]: failed to encode map e603 with expected crc 2023-09-23T20:49:02.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:02 smithi018 ceph-mon[184144]: osdmap e604: 8 total, 8 up, 8 in 2023-09-23T20:49:02.946 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:49:02.954 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:02.955 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:49:02.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:02 smithi149 ceph-mon[170394]: failed to encode map e603 with expected crc 2023-09-23T20:49:02.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:02 smithi149 ceph-mon[170394]: failed to encode map e603 with expected crc 2023-09-23T20:49:02.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:02 smithi149 ceph-mon[170394]: osdmap e604: 8 total, 8 up, 8 in 2023-09-23T20:49:03.055 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:49:03.083 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:03.120 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:49:03.123 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:03.124 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:49:03.225 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:49:03.253 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:03.290 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:49:03.293 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:03.294 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:49:03.394 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:49:03.434 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:03.484 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:49:03.488 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:03.488 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:49:03.589 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:49:03.617 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:03.655 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:49:03.659 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:03.659 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:49:03.715 INFO:teuthology.orchestra.run.smithi018.stderr:noscrub is unset 2023-09-23T20:49:03.732 INFO:teuthology.orchestra.run.smithi018.stderr:set pool 10 pgp_num to 26 2023-09-23T20:49:03.761 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:49:03.771 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2023-09-23T20:49:03.771 INFO:teuthology.orchestra.run.smithi018.stdout: "size": 20, 2023-09-23T20:49:03.771 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 600, 2023-09-23T20:49:03.772 INFO:teuthology.orchestra.run.smithi018.stdout: "ops": [ 2023-09-23T20:49:03.772 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.772 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.36450.0:141029 9.a 9:572023e9:::benchmark_data_smithi018_381516_object27195:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e481)", 2023-09-23T20:49:03.772 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:43:15.738002+0000", 2023-09-23T20:49:03.772 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 348.02667972500001, 2023-09-23T20:49:03.772 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0527425420000001, 2023-09-23T20:49:03.773 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:49:03.773 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:49:03.773 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:49:03.773 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.36450", 2023-09-23T20:49:03.773 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:49:03.773 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 141029 2023-09-23T20:49:03.773 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.773 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:49:03.774 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.774 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:49:03.774 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738002+0000", 2023-09-23T20:49:03.774 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:03.774 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.774 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.775 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:49:03.775 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738002+0000", 2023-09-23T20:49:03.775 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:03.775 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.775 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.775 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:49:03.776 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738000+0000", 2023-09-23T20:49:03.776 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:49:03.776 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.776 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.776 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:49:03.776 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738007+0000", 2023-09-23T20:49:03.777 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.4760000000000001e-06 2023-09-23T20:49:03.777 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.777 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.777 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:49:03.777 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738025+0000", 2023-09-23T20:49:03.777 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.7952e-05 2023-09-23T20:49:03.777 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.777 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.778 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:49:03.779 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738035+0000", 2023-09-23T20:49:03.779 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.6120000000000005e-06 2023-09-23T20:49:03.779 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.779 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.779 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:03.779 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.744123+0000", 2023-09-23T20:49:03.779 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0060876410000000004 2023-09-23T20:49:03.779 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.779 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.779 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op must wait for map", 2023-09-23T20:49:03.780 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.744131+0000", 2023-09-23T20:49:03.780 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.8800000000000008e-06 2023-09-23T20:49:03.780 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.780 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.780 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:03.780 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.754919+0000", 2023-09-23T20:49:03.780 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.010788483999999999 2023-09-23T20:49:03.780 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.780 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.780 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:49:03.780 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.754929+0000", 2023-09-23T20:49:03.781 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0309000000000001e-05 2023-09-23T20:49:03.781 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.781 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.781 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:03.781 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:16.790652+0000", 2023-09-23T20:49:03.781 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0357231570000001 2023-09-23T20:49:03.781 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.781 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.781 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:49:03.781 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:16.790744+0000", 2023-09-23T20:49:03.781 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.1743000000000006e-05 2023-09-23T20:49:03.781 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:03.782 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:49:03.782 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:03.782 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.782 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.782 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.36450.0:141040 9.a 9:53252326:::benchmark_data_smithi018_381516_object27206:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e481)", 2023-09-23T20:49:03.782 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:43:15.738057+0000", 2023-09-23T20:49:03.782 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 348.02662411300003, 2023-09-23T20:49:03.782 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0530090860000001, 2023-09-23T20:49:03.782 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:49:03.782 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:49:03.782 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:49:03.782 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.36450", 2023-09-23T20:49:03.783 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:49:03.783 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 141040 2023-09-23T20:49:03.783 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.783 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:49:03.783 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.783 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:49:03.783 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738057+0000", 2023-09-23T20:49:03.783 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:03.783 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.783 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.783 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:49:03.783 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738057+0000", 2023-09-23T20:49:03.784 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:03.784 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.784 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.784 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:49:03.784 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738057+0000", 2023-09-23T20:49:03.784 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:49:03.784 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.784 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.784 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:49:03.784 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738060+0000", 2023-09-23T20:49:03.784 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.9699999999999999e-06 2023-09-23T20:49:03.784 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.785 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.785 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:49:03.785 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738066+0000", 2023-09-23T20:49:03.785 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.5019999999999998e-06 2023-09-23T20:49:03.785 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.785 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.785 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:49:03.785 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738072+0000", 2023-09-23T20:49:03.785 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.2209999999999996e-06 2023-09-23T20:49:03.785 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.785 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.785 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:03.786 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.744157+0000", 2023-09-23T20:49:03.786 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0060856 2023-09-23T20:49:03.786 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.786 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.786 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting_for_map not empty", 2023-09-23T20:49:03.786 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.744162+0000", 2023-09-23T20:49:03.786 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.1610000000000002e-06 2023-09-23T20:49:03.786 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.786 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.786 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:03.786 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.754951+0000", 2023-09-23T20:49:03.786 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.010789127000000001 2023-09-23T20:49:03.787 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.787 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.787 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:49:03.787 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.754959+0000", 2023-09-23T20:49:03.787 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.3189999999999997e-06 2023-09-23T20:49:03.787 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.787 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.787 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:03.787 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:16.790999+0000", 2023-09-23T20:49:03.787 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.03604052 2023-09-23T20:49:03.787 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.787 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.788 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:49:03.788 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:16.791066+0000", 2023-09-23T20:49:03.788 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.7101999999999996e-05 2023-09-23T20:49:03.788 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:03.788 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:49:03.788 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:03.788 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.788 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.788 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.36450.0:141043 9.a 9:57a9fa95:::benchmark_data_smithi018_381516_object27209:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e481)", 2023-09-23T20:49:03.788 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:43:15.738077+0000", 2023-09-23T20:49:03.788 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 348.02660447599999, 2023-09-23T20:49:03.788 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0909135969999999, 2023-09-23T20:49:03.789 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:49:03.789 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:49:03.789 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:49:03.789 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.36450", 2023-09-23T20:49:03.789 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:49:03.789 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 141043 2023-09-23T20:49:03.789 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.789 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:49:03.789 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.789 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:49:03.789 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738077+0000", 2023-09-23T20:49:03.789 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:03.790 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.790 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.790 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:49:03.790 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738077+0000", 2023-09-23T20:49:03.790 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:03.790 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.790 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.790 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:49:03.790 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738077+0000", 2023-09-23T20:49:03.790 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:49:03.790 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.790 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.791 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:49:03.791 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738079+0000", 2023-09-23T20:49:03.791 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.0320000000000002e-06 2023-09-23T20:49:03.791 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.791 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.791 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:49:03.791 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738084+0000", 2023-09-23T20:49:03.791 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.4310000000000002e-06 2023-09-23T20:49:03.791 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.791 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.791 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:49:03.792 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738089+0000", 2023-09-23T20:49:03.792 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.6789999999999996e-06 2023-09-23T20:49:03.792 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.792 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.792 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:03.792 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.744187+0000", 2023-09-23T20:49:03.792 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0060979859999999997 2023-09-23T20:49:03.792 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.792 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.792 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting_for_map not empty", 2023-09-23T20:49:03.792 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.744194+0000", 2023-09-23T20:49:03.792 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.7090000000000002e-06 2023-09-23T20:49:03.793 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.793 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.793 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:03.793 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.754979+0000", 2023-09-23T20:49:03.793 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.010784141000000001 2023-09-23T20:49:03.793 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.793 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.793 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:49:03.793 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.754986+0000", 2023-09-23T20:49:03.793 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.0339999999999999e-06 2023-09-23T20:49:03.793 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.793 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.794 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:03.794 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:16.791090+0000", 2023-09-23T20:49:03.794 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.036103939 2023-09-23T20:49:03.794 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.794 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.794 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for missing object", 2023-09-23T20:49:03.794 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:16.791184+0000", 2023-09-23T20:49:03.794 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.4289000000000005e-05 2023-09-23T20:49:03.794 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.794 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.794 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:03.794 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:16.828905+0000", 2023-09-23T20:49:03.795 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.037721372000000003 2023-09-23T20:49:03.795 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.795 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.795 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:49:03.795 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:16.828991+0000", 2023-09-23T20:49:03.795 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.5331e-05 2023-09-23T20:49:03.795 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:03.795 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:49:03.795 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:03.795 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.795 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.795 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.36450.0:141045 9.f 9:fa55dbf8:::benchmark_data_smithi018_381516_object27211:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e481)", 2023-09-23T20:49:03.796 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:43:15.738093+0000", 2023-09-23T20:49:03.796 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 348.02658861899999, 2023-09-23T20:49:03.796 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0403492489999999, 2023-09-23T20:49:03.796 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:49:03.796 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:49:03.796 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:49:03.796 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.36450", 2023-09-23T20:49:03.796 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:49:03.796 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 141045 2023-09-23T20:49:03.796 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.796 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:49:03.796 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.797 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:49:03.797 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738093+0000", 2023-09-23T20:49:03.797 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:03.797 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.797 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.797 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:49:03.797 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738093+0000", 2023-09-23T20:49:03.797 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:03.797 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.797 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.797 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:49:03.797 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738092+0000", 2023-09-23T20:49:03.798 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:49:03.798 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.798 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.798 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:49:03.798 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738095+0000", 2023-09-23T20:49:03.798 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.063e-06 2023-09-23T20:49:03.798 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.798 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.798 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:49:03.798 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738100+0000", 2023-09-23T20:49:03.798 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.2469999999999999e-06 2023-09-23T20:49:03.799 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.799 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.799 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:49:03.799 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738104+0000", 2023-09-23T20:49:03.799 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.3649999999999997e-06 2023-09-23T20:49:03.799 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.799 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.799 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:03.799 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738271+0000", 2023-09-23T20:49:03.799 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00016673799999999999 2023-09-23T20:49:03.799 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.799 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.800 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op must wait for map", 2023-09-23T20:49:03.800 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738278+0000", 2023-09-23T20:49:03.800 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.0659999999999998e-06 2023-09-23T20:49:03.800 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.800 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.800 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:03.800 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.770314+0000", 2023-09-23T20:49:03.800 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.032036012000000003 2023-09-23T20:49:03.800 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.800 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.800 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:49:03.800 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.770322+0000", 2023-09-23T20:49:03.801 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.9330000000000001e-06 2023-09-23T20:49:03.801 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.801 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.801 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:03.801 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:16.776028+0000", 2023-09-23T20:49:03.801 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0057059290000001 2023-09-23T20:49:03.801 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.801 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.801 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:49:03.801 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:16.778442+0000", 2023-09-23T20:49:03.801 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0024142080000000002 2023-09-23T20:49:03.801 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:03.802 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:49:03.802 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:03.802 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.802 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.802 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.36450.0:141047 9.a 9:55e9d18e:::benchmark_data_smithi018_381516_object27213:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e481)", 2023-09-23T20:49:03.802 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:43:15.747358+0000", 2023-09-23T20:49:03.802 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 348.01732341299999, 2023-09-23T20:49:03.802 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.088479043, 2023-09-23T20:49:03.802 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:49:03.802 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:49:03.802 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:49:03.803 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.36450", 2023-09-23T20:49:03.803 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:49:03.803 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 141047 2023-09-23T20:49:03.803 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.803 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:49:03.803 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.803 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:49:03.803 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.747358+0000", 2023-09-23T20:49:03.803 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:03.803 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.803 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.803 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:49:03.804 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.747358+0000", 2023-09-23T20:49:03.804 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:03.804 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.804 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.804 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:49:03.804 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.747356+0000", 2023-09-23T20:49:03.804 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:49:03.804 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.804 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.804 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:49:03.804 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.747364+0000", 2023-09-23T20:49:03.804 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.6569999999999997e-06 2023-09-23T20:49:03.805 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.805 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.805 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:49:03.805 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.747379+0000", 2023-09-23T20:49:03.805 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.5153999999999999e-05 2023-09-23T20:49:03.805 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.805 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.805 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:49:03.805 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.747388+0000", 2023-09-23T20:49:03.805 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.7059999999999993e-06 2023-09-23T20:49:03.805 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.805 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.806 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:03.806 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.748082+0000", 2023-09-23T20:49:03.806 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00069463100000000002 2023-09-23T20:49:03.806 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.806 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.806 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting_for_map not empty", 2023-09-23T20:49:03.806 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.748090+0000", 2023-09-23T20:49:03.806 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.1349999999999992e-06 2023-09-23T20:49:03.806 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.806 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.806 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:03.806 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.755006+0000", 2023-09-23T20:49:03.807 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0069154289999999998 2023-09-23T20:49:03.807 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.807 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.807 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:49:03.807 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.755013+0000", 2023-09-23T20:49:03.807 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.7909999999999999e-06 2023-09-23T20:49:03.807 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.807 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.807 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:03.807 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:16.791216+0000", 2023-09-23T20:49:03.807 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.036203188 2023-09-23T20:49:03.808 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.808 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.808 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:49:03.808 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:16.791332+0000", 2023-09-23T20:49:03.808 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000115798 2023-09-23T20:49:03.808 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.808 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.808 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 5,7", 2023-09-23T20:49:03.808 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:16.791928+0000", 2023-09-23T20:49:03.808 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00059646299999999998 2023-09-23T20:49:03.808 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.808 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.809 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:03.809 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:16.822354+0000", 2023-09-23T20:49:03.809 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.030425812999999999 2023-09-23T20:49:03.809 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.809 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.809 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:49:03.809 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:16.822834+0000", 2023-09-23T20:49:03.809 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00048018300000000002 2023-09-23T20:49:03.809 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.809 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.809 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:03.809 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:16.835654+0000", 2023-09-23T20:49:03.810 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.012820066 2023-09-23T20:49:03.810 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.810 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.810 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:49:03.810 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:16.835711+0000", 2023-09-23T20:49:03.810 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.7300999999999999e-05 2023-09-23T20:49:03.810 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.810 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.810 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:49:03.810 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:16.835837+0000", 2023-09-23T20:49:03.810 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00012553400000000001 2023-09-23T20:49:03.811 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:03.811 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:49:03.811 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:03.811 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.811 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.811 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:84686 10.2 10:4b752585:::benchmark_data_smithi018_422431_object84685:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e542)", 2023-09-23T20:49:03.811 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:46:41.989189+0000", 2023-09-23T20:49:03.811 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 141.77549215299999, 2023-09-23T20:49:03.811 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.17485684100000001, 2023-09-23T20:49:03.811 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:49:03.811 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:49:03.811 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:49:03.812 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:49:03.812 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:49:03.812 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 84686 2023-09-23T20:49:03.812 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.812 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:49:03.812 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.812 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:49:03.812 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:41.989189+0000", 2023-09-23T20:49:03.812 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:03.812 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.812 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.812 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:49:03.813 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:41.989189+0000", 2023-09-23T20:49:03.813 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:03.813 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.813 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.813 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:49:03.813 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:41.989188+0000", 2023-09-23T20:49:03.813 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.999999 2023-09-23T20:49:03.813 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.813 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.813 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:49:03.813 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:41.989223+0000", 2023-09-23T20:49:03.813 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.4088000000000003e-05 2023-09-23T20:49:03.814 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.814 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.814 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:49:03.814 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:41.989235+0000", 2023-09-23T20:49:03.814 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2159e-05 2023-09-23T20:49:03.814 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.814 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.814 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:49:03.814 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:41.989242+0000", 2023-09-23T20:49:03.814 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.973e-06 2023-09-23T20:49:03.814 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.814 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.815 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:03.815 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:41.992689+0000", 2023-09-23T20:49:03.815 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.003447463 2023-09-23T20:49:03.815 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.815 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.815 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:49:03.815 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:41.992865+0000", 2023-09-23T20:49:03.815 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00017574999999999999 2023-09-23T20:49:03.815 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.815 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.815 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 0,4", 2023-09-23T20:49:03.816 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:41.993014+0000", 2023-09-23T20:49:03.816 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000149581 2023-09-23T20:49:03.816 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.816 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.816 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:49:03.816 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:41.999133+0000", 2023-09-23T20:49:03.816 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.006118694 2023-09-23T20:49:03.816 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.816 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.816 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:03.816 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:42.163916+0000", 2023-09-23T20:49:03.816 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.16478303999999999 2023-09-23T20:49:03.817 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.817 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.817 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:03.817 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:42.163969+0000", 2023-09-23T20:49:03.817 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.2293000000000002e-05 2023-09-23T20:49:03.817 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.817 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.817 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:49:03.817 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:42.164017+0000", 2023-09-23T20:49:03.817 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.8239999999999999e-05 2023-09-23T20:49:03.817 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.817 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.818 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:49:03.818 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:42.164046+0000", 2023-09-23T20:49:03.818 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.9328000000000001e-05 2023-09-23T20:49:03.818 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:03.818 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:49:03.818 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:03.818 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.818 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.818 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:84693 10.2 10:412815b5:::benchmark_data_smithi018_422431_object84692:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e542)", 2023-09-23T20:49:03.818 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:46:41.994573+0000", 2023-09-23T20:49:03.818 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 141.77010808599999, 2023-09-23T20:49:03.819 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.16963708499999999, 2023-09-23T20:49:03.819 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:49:03.819 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:49:03.819 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:49:03.819 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:49:03.819 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:49:03.819 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 84693 2023-09-23T20:49:03.819 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.819 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:49:03.819 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.819 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:49:03.819 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:41.994573+0000", 2023-09-23T20:49:03.820 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:03.820 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.820 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.820 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:49:03.820 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:41.994573+0000", 2023-09-23T20:49:03.820 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:03.820 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.820 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.820 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:49:03.820 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:41.994572+0000", 2023-09-23T20:49:03.820 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:49:03.820 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.821 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.821 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:49:03.821 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:41.994598+0000", 2023-09-23T20:49:03.821 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.6012000000000001e-05 2023-09-23T20:49:03.821 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.821 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.821 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:49:03.821 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:41.994612+0000", 2023-09-23T20:49:03.821 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.4438e-05 2023-09-23T20:49:03.821 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.821 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.822 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:49:03.822 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:41.994621+0000", 2023-09-23T20:49:03.822 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.2609999999999999e-06 2023-09-23T20:49:03.822 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.822 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.822 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:03.822 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:41.995874+0000", 2023-09-23T20:49:03.822 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0012526600000000001 2023-09-23T20:49:03.822 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.822 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.822 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:49:03.822 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:41.996026+0000", 2023-09-23T20:49:03.823 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00015181400000000001 2023-09-23T20:49:03.823 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.823 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.823 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 0,4", 2023-09-23T20:49:03.823 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:41.996169+0000", 2023-09-23T20:49:03.823 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000143549 2023-09-23T20:49:03.823 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.824 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.824 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:49:03.824 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:42.001643+0000", 2023-09-23T20:49:03.824 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0054731520000000002 2023-09-23T20:49:03.824 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.824 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.824 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:03.824 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:42.164097+0000", 2023-09-23T20:49:03.824 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.16245464800000001 2023-09-23T20:49:03.825 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.825 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.825 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:03.825 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:42.164125+0000", 2023-09-23T20:49:03.825 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.7960999999999999e-05 2023-09-23T20:49:03.825 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.825 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.825 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:49:03.825 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:42.164172+0000", 2023-09-23T20:49:03.825 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.7015000000000003e-05 2023-09-23T20:49:03.826 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.826 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.826 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:49:03.826 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:42.164210+0000", 2023-09-23T20:49:03.826 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.8213000000000002e-05 2023-09-23T20:49:03.826 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:03.826 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:49:03.826 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:03.826 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.827 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.827 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_repop(client.46625.0:84694 10.0 e542/531 10:03d9afac:::benchmark_data_smithi018_422431_object84693:head v 542'5296, mlcod=542'5291)", 2023-09-23T20:49:03.827 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:46:42.002318+0000", 2023-09-23T20:49:03.827 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 141.76236304599999, 2023-09-23T20:49:03.827 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.17149143, 2023-09-23T20:49:03.827 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:49:03.827 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:49:03.827 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:49:03.827 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.827 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:49:03.828 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:42.002318+0000", 2023-09-23T20:49:03.828 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:03.828 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.828 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.828 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:49:03.828 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:42.002318+0000", 2023-09-23T20:49:03.828 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:03.828 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.828 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.828 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:49:03.829 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:42.002318+0000", 2023-09-23T20:49:03.829 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:49:03.829 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.829 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.829 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:49:03.829 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:42.002413+0000", 2023-09-23T20:49:03.829 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.5080000000000004e-05 2023-09-23T20:49:03.829 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.829 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.830 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:49:03.830 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:42.002420+0000", 2023-09-23T20:49:03.830 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.4939999999999997e-06 2023-09-23T20:49:03.830 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.830 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.830 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:49:03.830 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:42.002425+0000", 2023-09-23T20:49:03.830 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.8679999999999998e-06 2023-09-23T20:49:03.830 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.830 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.831 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:03.831 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:42.002458+0000", 2023-09-23T20:49:03.831 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.2920000000000003e-05 2023-09-23T20:49:03.831 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.831 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.831 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:49:03.831 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:42.002471+0000", 2023-09-23T20:49:03.831 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2692e-05 2023-09-23T20:49:03.831 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.832 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.832 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:49:03.832 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:42.173751+0000", 2023-09-23T20:49:03.832 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.17127979600000001 2023-09-23T20:49:03.832 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.832 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.832 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:49:03.832 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:42.173810+0000", 2023-09-23T20:49:03.832 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.9142999999999999e-05 2023-09-23T20:49:03.832 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:03.832 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:49:03.833 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:03.833 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.833 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.833 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:84697 10.2 10:44533be6:::benchmark_data_smithi018_422431_object84696:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e542)", 2023-09-23T20:49:03.833 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:46:42.002340+0000", 2023-09-23T20:49:03.833 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 141.762341703, 2023-09-23T20:49:03.833 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.16788215300000001, 2023-09-23T20:49:03.833 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:49:03.833 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:49:03.834 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:49:03.834 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:49:03.834 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:49:03.834 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 84697 2023-09-23T20:49:03.834 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.834 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:49:03.834 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.834 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:49:03.834 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:42.002340+0000", 2023-09-23T20:49:03.834 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:03.835 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.835 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.835 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:49:03.835 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:42.002340+0000", 2023-09-23T20:49:03.835 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:03.835 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.835 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.835 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:49:03.835 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:42.002339+0000", 2023-09-23T20:49:03.835 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.999999 2023-09-23T20:49:03.836 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.836 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.836 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:49:03.836 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:42.002364+0000", 2023-09-23T20:49:03.836 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.5537999999999998e-05 2023-09-23T20:49:03.836 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.836 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.836 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:49:03.836 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:42.002373+0000", 2023-09-23T20:49:03.836 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.8820000000000007e-06 2023-09-23T20:49:03.836 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.837 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.837 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:49:03.837 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:42.002379+0000", 2023-09-23T20:49:03.837 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.5040000000000002e-06 2023-09-23T20:49:03.837 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.837 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.837 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:03.837 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:42.002425+0000", 2023-09-23T20:49:03.837 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.5841999999999999e-05 2023-09-23T20:49:03.837 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.838 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.838 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:49:03.838 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:42.160927+0000", 2023-09-23T20:49:03.838 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.158502799 2023-09-23T20:49:03.838 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.838 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.838 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 0,4", 2023-09-23T20:49:03.838 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:42.161088+0000", 2023-09-23T20:49:03.838 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00016069599999999999 2023-09-23T20:49:03.838 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.839 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.839 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:03.839 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:42.164301+0000", 2023-09-23T20:49:03.839 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0032129099999999998 2023-09-23T20:49:03.839 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.839 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.839 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:49:03.839 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:42.164317+0000", 2023-09-23T20:49:03.839 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.6435000000000001e-05 2023-09-23T20:49:03.839 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.840 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.840 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:03.840 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:42.170138+0000", 2023-09-23T20:49:03.840 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0058209689999999996 2023-09-23T20:49:03.840 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.840 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.840 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:49:03.840 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:42.170186+0000", 2023-09-23T20:49:03.840 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.7817e-05 2023-09-23T20:49:03.840 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.841 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.841 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:49:03.841 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:42.170222+0000", 2023-09-23T20:49:03.841 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.5661999999999998e-05 2023-09-23T20:49:03.841 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:03.841 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:49:03.841 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:03.841 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.841 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.842 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_repop_reply(client.46625.0:84686 10.2 e542/531 ondisk, result = 0)", 2023-09-23T20:49:03.842 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:46:42.004150+0000", 2023-09-23T20:49:03.842 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 141.76053131899999, 2023-09-23T20:49:03.842 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.15990738500000001, 2023-09-23T20:49:03.842 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:49:03.842 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "started", 2023-09-23T20:49:03.842 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:49:03.842 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.842 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:49:03.842 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:42.004150+0000", 2023-09-23T20:49:03.843 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:03.843 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.843 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.843 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:49:03.843 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:42.004150+0000", 2023-09-23T20:49:03.843 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:03.843 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.843 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.843 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:49:03.844 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:42.004149+0000", 2023-09-23T20:49:03.844 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:49:03.844 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.844 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.844 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:49:03.844 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:42.004154+0000", 2023-09-23T20:49:03.844 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.0710000000000002e-06 2023-09-23T20:49:03.844 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.844 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.844 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:49:03.845 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:42.160854+0000", 2023-09-23T20:49:03.845 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.15669942100000001 2023-09-23T20:49:03.845 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.845 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.845 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:49:03.845 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:42.160871+0000", 2023-09-23T20:49:03.845 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.696e-05 2023-09-23T20:49:03.845 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.845 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.845 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:03.845 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:42.163955+0000", 2023-09-23T20:49:03.846 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0030843559999999999 2023-09-23T20:49:03.846 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.846 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.846 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:49:03.846 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:42.163963+0000", 2023-09-23T20:49:03.846 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.9789999999999993e-06 2023-09-23T20:49:03.846 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.846 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.846 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:49:03.846 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:42.164057+0000", 2023-09-23T20:49:03.847 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.4271000000000006e-05 2023-09-23T20:49:03.847 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:03.847 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:49:03.847 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:03.847 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.847 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.847 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:84702 10.2 10:480163e2:::benchmark_data_smithi018_422431_object84701:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e542)", 2023-09-23T20:49:03.847 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:46:42.007905+0000", 2023-09-23T20:49:03.847 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 141.75677605999999, 2023-09-23T20:49:03.848 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.16820030899999999, 2023-09-23T20:49:03.848 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:49:03.848 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:49:03.848 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:49:03.848 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:49:03.848 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:49:03.848 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 84702 2023-09-23T20:49:03.848 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.848 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:49:03.848 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.849 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:49:03.849 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:42.007905+0000", 2023-09-23T20:49:03.849 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:03.849 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.849 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.849 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:49:03.849 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:42.007905+0000", 2023-09-23T20:49:03.849 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:03.849 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.849 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.849 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:49:03.850 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:42.007904+0000", 2023-09-23T20:49:03.850 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:49:03.850 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.850 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.850 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:49:03.850 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:42.007930+0000", 2023-09-23T20:49:03.850 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.6692000000000001e-05 2023-09-23T20:49:03.850 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.850 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.850 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:49:03.851 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:42.160880+0000", 2023-09-23T20:49:03.851 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.15294976499999999 2023-09-23T20:49:03.851 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.851 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.851 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:49:03.851 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:42.160888+0000", 2023-09-23T20:49:03.851 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.2409999999999995e-06 2023-09-23T20:49:03.851 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.851 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.851 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:03.852 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:42.164360+0000", 2023-09-23T20:49:03.852 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0034711680000000002 2023-09-23T20:49:03.852 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.852 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.852 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:49:03.852 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:42.164517+0000", 2023-09-23T20:49:03.852 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000157336 2023-09-23T20:49:03.852 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.852 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.852 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 0,4", 2023-09-23T20:49:03.853 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:42.164640+0000", 2023-09-23T20:49:03.853 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00012278899999999999 2023-09-23T20:49:03.853 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.853 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.853 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:03.853 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:42.170263+0000", 2023-09-23T20:49:03.853 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0056229160000000004 2023-09-23T20:49:03.853 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.853 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.853 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:49:03.853 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:42.173289+0000", 2023-09-23T20:49:03.854 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0030266540000000001 2023-09-23T20:49:03.854 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.854 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.854 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:03.854 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:42.175990+0000", 2023-09-23T20:49:03.854 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.002700568 2023-09-23T20:49:03.854 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.854 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.854 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:49:03.854 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:42.176054+0000", 2023-09-23T20:49:03.855 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.4407999999999997e-05 2023-09-23T20:49:03.855 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.855 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.855 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:49:03.855 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:42.176106+0000", 2023-09-23T20:49:03.855 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.1381000000000002e-05 2023-09-23T20:49:03.855 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:03.855 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:49:03.855 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:03.855 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.856 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.856 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:102280 10.2 10:4d2f73b6:::benchmark_data_smithi018_422431_object102279:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e551)", 2023-09-23T20:49:03.856 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:47:08.174176+0000", 2023-09-23T20:49:03.856 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 115.590505529, 2023-09-23T20:49:03.856 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0912579659999999, 2023-09-23T20:49:03.856 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:49:03.856 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:49:03.856 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:49:03.856 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:49:03.857 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:49:03.857 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 102280 2023-09-23T20:49:03.857 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.857 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:49:03.857 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.857 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:49:03.857 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:08.174176+0000", 2023-09-23T20:49:03.857 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:03.857 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.857 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.858 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:49:03.858 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:08.174176+0000", 2023-09-23T20:49:03.858 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:03.858 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.858 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.858 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:49:03.858 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:08.174174+0000", 2023-09-23T20:49:03.858 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:49:03.858 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.858 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.859 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:49:03.859 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:08.174198+0000", 2023-09-23T20:49:03.859 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.3923000000000001e-05 2023-09-23T20:49:03.859 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.859 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.859 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:49:03.859 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:08.174211+0000", 2023-09-23T20:49:03.859 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3324e-05 2023-09-23T20:49:03.859 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.860 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.860 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:49:03.860 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:08.174221+0000", 2023-09-23T20:49:03.860 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.5740000000000002e-06 2023-09-23T20:49:03.860 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.860 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.860 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:03.860 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:08.186392+0000", 2023-09-23T20:49:03.860 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.012171184999999999 2023-09-23T20:49:03.860 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.860 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.861 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:49:03.861 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:08.186404+0000", 2023-09-23T20:49:03.861 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1878e-05 2023-09-23T20:49:03.861 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.861 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.861 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:03.861 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:09.252235+0000", 2023-09-23T20:49:03.861 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0658310289999999 2023-09-23T20:49:03.861 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.862 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.862 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:49:03.862 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:09.252439+0000", 2023-09-23T20:49:03.862 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00020431400000000001 2023-09-23T20:49:03.862 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.862 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.862 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 0,4", 2023-09-23T20:49:03.862 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:09.252618+0000", 2023-09-23T20:49:03.862 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00017877799999999999 2023-09-23T20:49:03.863 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.863 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.863 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:03.863 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:09.258806+0000", 2023-09-23T20:49:03.863 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0061873279999999998 2023-09-23T20:49:03.863 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.863 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.863 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:03.863 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:09.261128+0000", 2023-09-23T20:49:03.864 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.002322228 2023-09-23T20:49:03.864 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.864 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.864 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:49:03.864 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:09.264995+0000", 2023-09-23T20:49:03.864 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0038676840000000001 2023-09-23T20:49:03.864 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.864 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.864 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:49:03.865 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:09.265068+0000", 2023-09-23T20:49:03.865 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.2390000000000003e-05 2023-09-23T20:49:03.865 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.865 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.865 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:49:03.865 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:09.265434+0000", 2023-09-23T20:49:03.865 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00036597799999999998 2023-09-23T20:49:03.865 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:03.865 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:49:03.866 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:03.866 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.866 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.866 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:104294 10.2 10:438d3133:::benchmark_data_smithi018_422431_object104293:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e559)", 2023-09-23T20:49:03.866 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:47:14.718885+0000", 2023-09-23T20:49:03.866 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 109.04579622200001, 2023-09-23T20:49:03.866 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.049109881000000001, 2023-09-23T20:49:03.866 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:49:03.867 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:49:03.867 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:49:03.867 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:49:03.867 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:49:03.867 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 104294 2023-09-23T20:49:03.867 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.867 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:49:03.867 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.867 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:49:03.867 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:14.718885+0000", 2023-09-23T20:49:03.868 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:03.868 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.868 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.868 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:49:03.868 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:14.718885+0000", 2023-09-23T20:49:03.868 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:03.868 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.868 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.868 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:49:03.868 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:14.718884+0000", 2023-09-23T20:49:03.869 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.999999 2023-09-23T20:49:03.869 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.869 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.869 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:49:03.869 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:14.718925+0000", 2023-09-23T20:49:03.869 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.0905000000000002e-05 2023-09-23T20:49:03.869 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.869 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.869 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:49:03.870 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:14.718934+0000", 2023-09-23T20:49:03.870 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.9639999999999992e-06 2023-09-23T20:49:03.870 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.870 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.870 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:49:03.870 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:14.718939+0000", 2023-09-23T20:49:03.870 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.203e-06 2023-09-23T20:49:03.870 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.870 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.870 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:03.871 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:14.718995+0000", 2023-09-23T20:49:03.871 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.5569000000000003e-05 2023-09-23T20:49:03.871 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.871 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.871 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:49:03.871 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:14.719172+0000", 2023-09-23T20:49:03.871 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00017702700000000001 2023-09-23T20:49:03.871 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.871 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.871 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 0,4", 2023-09-23T20:49:03.872 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:14.719346+0000", 2023-09-23T20:49:03.872 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00017425200000000001 2023-09-23T20:49:03.872 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.872 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.872 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:49:03.872 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:14.722238+0000", 2023-09-23T20:49:03.872 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.002892319 2023-09-23T20:49:03.872 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.872 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.872 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:03.873 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:14.730905+0000", 2023-09-23T20:49:03.873 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0086664640000000005 2023-09-23T20:49:03.873 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.873 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.873 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:03.873 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:14.767705+0000", 2023-09-23T20:49:03.873 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.036799813000000001 2023-09-23T20:49:03.873 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.873 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.874 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:49:03.874 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:14.767767+0000", 2023-09-23T20:49:03.874 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.2706999999999994e-05 2023-09-23T20:49:03.874 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.874 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.874 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:49:03.874 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:14.767995+0000", 2023-09-23T20:49:03.874 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00022782999999999999 2023-09-23T20:49:03.874 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:03.874 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:49:03.875 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:03.875 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.875 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.875 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:113959 10.2 10:48756c97:::benchmark_data_smithi018_422431_object9463:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e562)", 2023-09-23T20:49:03.875 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:47:27.039987+0000", 2023-09-23T20:49:03.875 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 96.724694870999997, 2023-09-23T20:49:03.875 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.070911610999999999, 2023-09-23T20:49:03.875 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:49:03.875 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:49:03.875 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:49:03.876 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:49:03.876 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:49:03.876 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 113959 2023-09-23T20:49:03.876 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.876 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:49:03.876 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.876 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:49:03.876 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:27.039987+0000", 2023-09-23T20:49:03.876 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:03.877 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.877 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.877 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:49:03.877 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:27.039987+0000", 2023-09-23T20:49:03.877 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:03.877 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.877 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.877 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:49:03.877 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:27.039985+0000", 2023-09-23T20:49:03.877 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:49:03.878 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.878 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.878 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:49:03.878 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:27.039992+0000", 2023-09-23T20:49:03.878 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.5050000000000001e-06 2023-09-23T20:49:03.878 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.878 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.878 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:49:03.878 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:27.040014+0000", 2023-09-23T20:49:03.879 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.1450999999999998e-05 2023-09-23T20:49:03.879 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.879 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.879 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:49:03.879 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:27.040023+0000", 2023-09-23T20:49:03.879 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.5419999999999994e-06 2023-09-23T20:49:03.879 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.879 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.879 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:03.879 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:27.043902+0000", 2023-09-23T20:49:03.880 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.003878856 2023-09-23T20:49:03.880 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.880 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.880 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:49:03.880 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:27.044050+0000", 2023-09-23T20:49:03.880 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000147954 2023-09-23T20:49:03.880 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.880 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.880 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 0,4", 2023-09-23T20:49:03.880 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:27.044132+0000", 2023-09-23T20:49:03.881 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.2299000000000006e-05 2023-09-23T20:49:03.881 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.881 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.881 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:03.881 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:27.058311+0000", 2023-09-23T20:49:03.881 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.014178690000000001 2023-09-23T20:49:03.881 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.881 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.881 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:49:03.881 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:27.058320+0000", 2023-09-23T20:49:03.882 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.7010000000000002e-06 2023-09-23T20:49:03.882 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.882 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.882 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:03.882 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:27.110804+0000", 2023-09-23T20:49:03.882 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.052483690999999999 2023-09-23T20:49:03.882 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.882 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.882 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:49:03.883 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:27.110856+0000", 2023-09-23T20:49:03.883 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.2785999999999999e-05 2023-09-23T20:49:03.883 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.883 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.883 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:49:03.883 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:27.110898+0000", 2023-09-23T20:49:03.883 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.1844999999999997e-05 2023-09-23T20:49:03.883 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:03.883 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:49:03.883 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:03.884 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.884 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.884 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:113962 10.2 10:475659b4:::benchmark_data_smithi018_422431_object9466:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e562)", 2023-09-23T20:49:03.884 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:47:27.044691+0000", 2023-09-23T20:49:03.884 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 96.719990836999997, 2023-09-23T20:49:03.884 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.066308119999999998, 2023-09-23T20:49:03.884 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:49:03.884 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:49:03.884 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:49:03.885 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:49:03.885 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:49:03.885 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 113962 2023-09-23T20:49:03.885 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.885 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:49:03.885 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.885 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:49:03.885 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:27.044691+0000", 2023-09-23T20:49:03.885 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:03.885 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.886 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.886 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:49:03.886 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:27.044691+0000", 2023-09-23T20:49:03.886 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:03.886 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.886 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.886 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:49:03.886 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:27.044689+0000", 2023-09-23T20:49:03.886 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:49:03.886 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.887 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.887 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:49:03.887 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:27.044696+0000", 2023-09-23T20:49:03.887 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.3039999999999999e-06 2023-09-23T20:49:03.887 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.887 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.887 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:49:03.887 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:27.044713+0000", 2023-09-23T20:49:03.887 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.6854000000000002e-05 2023-09-23T20:49:03.887 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.888 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.888 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:49:03.888 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:27.044724+0000", 2023-09-23T20:49:03.888 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0478e-05 2023-09-23T20:49:03.888 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.888 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.888 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:03.888 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:27.047226+0000", 2023-09-23T20:49:03.888 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0025019669999999999 2023-09-23T20:49:03.888 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.889 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.889 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:49:03.889 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:27.047357+0000", 2023-09-23T20:49:03.889 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00013174199999999999 2023-09-23T20:49:03.889 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.889 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.889 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 0,4", 2023-09-23T20:49:03.889 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:27.047475+0000", 2023-09-23T20:49:03.889 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000117495 2023-09-23T20:49:03.889 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.890 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.890 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:49:03.890 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:27.062160+0000", 2023-09-23T20:49:03.890 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.014684831000000001 2023-09-23T20:49:03.890 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.890 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.890 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:03.890 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:27.062221+0000", 2023-09-23T20:49:03.890 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.1452999999999996e-05 2023-09-23T20:49:03.890 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.891 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.891 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:03.891 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:27.110933+0000", 2023-09-23T20:49:03.891 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.048711566999999997 2023-09-23T20:49:03.891 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.891 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.891 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:49:03.891 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:27.110968+0000", 2023-09-23T20:49:03.891 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.5179999999999999e-05 2023-09-23T20:49:03.891 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.892 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.892 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:49:03.892 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:27.110999+0000", 2023-09-23T20:49:03.892 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.0765000000000001e-05 2023-09-23T20:49:03.892 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:03.892 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:49:03.892 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:03.892 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.892 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.892 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:113966 10.2 10:4ad0b25f:::benchmark_data_smithi018_422431_object9470:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e562)", 2023-09-23T20:49:03.893 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:47:27.054123+0000", 2023-09-23T20:49:03.893 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 96.710558375999994, 2023-09-23T20:49:03.893 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.056968368999999998, 2023-09-23T20:49:03.893 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:49:03.893 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:49:03.893 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:49:03.893 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:49:03.893 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:49:03.893 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 113966 2023-09-23T20:49:03.893 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.893 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:49:03.894 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.894 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:49:03.894 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:27.054123+0000", 2023-09-23T20:49:03.894 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:03.894 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.894 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.894 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:49:03.894 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:27.054123+0000", 2023-09-23T20:49:03.894 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:03.894 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.895 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.895 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:49:03.895 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:27.054122+0000", 2023-09-23T20:49:03.895 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:49:03.895 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.895 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.895 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:49:03.895 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:27.054127+0000", 2023-09-23T20:49:03.895 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.2660000000000001e-06 2023-09-23T20:49:03.895 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.896 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.896 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:49:03.896 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:27.054138+0000", 2023-09-23T20:49:03.896 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0477e-05 2023-09-23T20:49:03.896 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.896 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.896 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:49:03.896 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:27.054145+0000", 2023-09-23T20:49:03.896 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.3359999999999997e-06 2023-09-23T20:49:03.896 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.897 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.897 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:03.897 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:27.058359+0000", 2023-09-23T20:49:03.897 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0042145919999999996 2023-09-23T20:49:03.897 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.897 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.897 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:49:03.897 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:27.058506+0000", 2023-09-23T20:49:03.897 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00014691300000000001 2023-09-23T20:49:03.897 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.897 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.898 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 0,4", 2023-09-23T20:49:03.898 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:27.058637+0000", 2023-09-23T20:49:03.898 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000130507 2023-09-23T20:49:03.898 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.898 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.898 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:03.898 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:27.072913+0000", 2023-09-23T20:49:03.898 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.01427645 2023-09-23T20:49:03.898 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.898 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.899 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:49:03.899 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:27.072938+0000", 2023-09-23T20:49:03.899 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.4739000000000001e-05 2023-09-23T20:49:03.899 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.899 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.899 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:03.899 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:27.111028+0000", 2023-09-23T20:49:03.899 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.038090244000000002 2023-09-23T20:49:03.899 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.899 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.900 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:49:03.900 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:27.111061+0000", 2023-09-23T20:49:03.900 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.2977e-05 2023-09-23T20:49:03.900 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.900 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.900 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:49:03.900 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:27.111091+0000", 2023-09-23T20:49:03.900 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.0117999999999999e-05 2023-09-23T20:49:03.900 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:03.900 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:49:03.900 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:03.901 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.901 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.901 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:138636 10.2 10:4860790d:::benchmark_data_smithi018_422431_object34140:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e571)", 2023-09-23T20:49:03.901 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:47:57.659920+0000", 2023-09-23T20:49:03.901 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 66.104761070999999, 2023-09-23T20:49:03.901 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.049038744000000002, 2023-09-23T20:49:03.901 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:49:03.901 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:49:03.901 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:49:03.901 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:49:03.902 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:49:03.902 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 138636 2023-09-23T20:49:03.902 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.902 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:49:03.902 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.902 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:49:03.902 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:57.659920+0000", 2023-09-23T20:49:03.902 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:03.902 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.902 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.903 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:49:03.903 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:57.659920+0000", 2023-09-23T20:49:03.903 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:03.903 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.903 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.903 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:49:03.903 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:57.659920+0000", 2023-09-23T20:49:03.903 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:49:03.903 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.903 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.904 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:49:03.904 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:57.659922+0000", 2023-09-23T20:49:03.904 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.5019999999999999e-06 2023-09-23T20:49:03.904 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.904 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.904 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:49:03.904 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:57.659928+0000", 2023-09-23T20:49:03.904 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.1170000000000003e-06 2023-09-23T20:49:03.904 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.904 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.904 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:49:03.905 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:57.659932+0000", 2023-09-23T20:49:03.905 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.7099999999999998e-06 2023-09-23T20:49:03.905 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.905 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.905 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:03.905 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:57.667771+0000", 2023-09-23T20:49:03.905 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0078384019999999995 2023-09-23T20:49:03.905 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.905 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.905 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:49:03.906 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:57.667918+0000", 2023-09-23T20:49:03.906 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00014708000000000001 2023-09-23T20:49:03.906 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.906 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.906 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 0,4", 2023-09-23T20:49:03.906 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:57.668045+0000", 2023-09-23T20:49:03.906 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00012757799999999999 2023-09-23T20:49:03.906 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.906 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.906 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:49:03.907 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:57.671745+0000", 2023-09-23T20:49:03.907 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0037001320000000001 2023-09-23T20:49:03.907 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.907 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.907 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:03.907 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:57.681902+0000", 2023-09-23T20:49:03.907 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.010156058000000001 2023-09-23T20:49:03.907 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.907 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.907 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:03.908 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:57.708861+0000", 2023-09-23T20:49:03.908 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.026959647999999999 2023-09-23T20:49:03.908 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.908 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.908 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:49:03.908 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:57.708916+0000", 2023-09-23T20:49:03.908 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.5101e-05 2023-09-23T20:49:03.908 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.908 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.908 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:49:03.909 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:57.708959+0000", 2023-09-23T20:49:03.909 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.2778000000000003e-05 2023-09-23T20:49:03.909 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:03.909 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:49:03.909 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:03.909 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.909 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.909 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:149626 10.2 10:407ab276:::benchmark_data_smithi018_422431_object45130:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e574)", 2023-09-23T20:49:03.909 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:48:11.191450+0000", 2023-09-23T20:49:03.909 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 52.573230938999998, 2023-09-23T20:49:03.910 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.049527843000000002, 2023-09-23T20:49:03.910 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:49:03.910 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:49:03.910 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:49:03.910 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:49:03.910 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:49:03.910 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 149626 2023-09-23T20:49:03.910 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.910 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:49:03.910 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.911 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:49:03.911 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:11.191450+0000", 2023-09-23T20:49:03.911 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:03.911 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.911 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.911 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:49:03.911 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:11.191450+0000", 2023-09-23T20:49:03.911 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:03.911 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.911 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.912 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:49:03.912 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:11.191449+0000", 2023-09-23T20:49:03.912 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:49:03.912 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.912 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.912 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:49:03.912 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:11.191457+0000", 2023-09-23T20:49:03.912 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.0649999999999999e-06 2023-09-23T20:49:03.912 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.912 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.913 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:49:03.913 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:11.191483+0000", 2023-09-23T20:49:03.913 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.6333e-05 2023-09-23T20:49:03.913 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.913 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.913 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:49:03.913 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:11.191492+0000", 2023-09-23T20:49:03.913 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.3179999999999996e-06 2023-09-23T20:49:03.913 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.913 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.913 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:03.914 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:11.191649+0000", 2023-09-23T20:49:03.914 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000156527 2023-09-23T20:49:03.914 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.914 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.914 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:49:03.914 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:11.191765+0000", 2023-09-23T20:49:03.914 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000115993 2023-09-23T20:49:03.914 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.914 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.915 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 0,4", 2023-09-23T20:49:03.915 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:11.191891+0000", 2023-09-23T20:49:03.915 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000125604 2023-09-23T20:49:03.915 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.915 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.915 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:03.915 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:11.200930+0000", 2023-09-23T20:49:03.915 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0090398679999999995 2023-09-23T20:49:03.915 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.915 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.916 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:49:03.916 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:11.204060+0000", 2023-09-23T20:49:03.916 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0031296380000000001 2023-09-23T20:49:03.916 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.916 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.916 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:03.916 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:11.240911+0000", 2023-09-23T20:49:03.916 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.036850524000000003 2023-09-23T20:49:03.916 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.916 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.917 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:49:03.917 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:11.240949+0000", 2023-09-23T20:49:03.917 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.8856000000000002e-05 2023-09-23T20:49:03.917 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.917 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.917 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:49:03.917 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:11.240978+0000", 2023-09-23T20:49:03.917 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.8875000000000001e-05 2023-09-23T20:49:03.917 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:03.918 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:49:03.918 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:03.918 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.918 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.918 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:179101 10.12 10:4a2e9a11:::benchmark_data_smithi018_422431_object74605:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e599)", 2023-09-23T20:49:03.918 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:48:57.579106+0000", 2023-09-23T20:49:03.918 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 6.1855755200000004, 2023-09-23T20:49:03.918 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.066890157000000006, 2023-09-23T20:49:03.918 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:49:03.918 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:49:03.919 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:49:03.919 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:49:03.919 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:49:03.919 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 179101 2023-09-23T20:49:03.919 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.919 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:49:03.919 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.919 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:49:03.919 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:57.579106+0000", 2023-09-23T20:49:03.919 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:03.919 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.920 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.920 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:49:03.920 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:57.579106+0000", 2023-09-23T20:49:03.920 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:03.920 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.920 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.920 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:49:03.920 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:57.579105+0000", 2023-09-23T20:49:03.920 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:49:03.920 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.921 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.921 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:49:03.921 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:57.579111+0000", 2023-09-23T20:49:03.921 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.1079999999999998e-06 2023-09-23T20:49:03.921 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.921 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.921 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:49:03.921 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:57.579122+0000", 2023-09-23T20:49:03.921 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1573000000000001e-05 2023-09-23T20:49:03.921 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.922 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.922 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:49:03.922 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:57.579128+0000", 2023-09-23T20:49:03.922 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.1539999999999999e-06 2023-09-23T20:49:03.922 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.922 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.922 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:03.922 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:57.579170+0000", 2023-09-23T20:49:03.922 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.1363999999999999e-05 2023-09-23T20:49:03.922 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.923 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.923 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:49:03.923 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:57.579254+0000", 2023-09-23T20:49:03.923 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.3914999999999999e-05 2023-09-23T20:49:03.923 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.923 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.923 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 0,4", 2023-09-23T20:49:03.923 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:57.579336+0000", 2023-09-23T20:49:03.923 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.2301999999999999e-05 2023-09-23T20:49:03.923 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.924 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.924 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:49:03.924 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:57.580778+0000", 2023-09-23T20:49:03.924 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0014419669999999999 2023-09-23T20:49:03.924 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.924 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.924 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:03.924 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:57.583993+0000", 2023-09-23T20:49:03.924 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.003215468 2023-09-23T20:49:03.924 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.924 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.925 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:03.925 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:57.645902+0000", 2023-09-23T20:49:03.925 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.061908718000000001 2023-09-23T20:49:03.925 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.925 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.925 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:49:03.925 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:57.645958+0000", 2023-09-23T20:49:03.925 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.6140000000000001e-05 2023-09-23T20:49:03.925 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.925 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.926 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:49:03.926 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:57.645996+0000", 2023-09-23T20:49:03.926 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.7693e-05 2023-09-23T20:49:03.926 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:03.926 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:49:03.926 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:03.926 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.926 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.926 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:181708 10.16 10.b7205ed6 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e603)", 2023-09-23T20:49:03.926 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:49:01.717054+0000", 2023-09-23T20:49:03.927 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 2.0476275479999999, 2023-09-23T20:49:03.927 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.97645376399999995, 2023-09-23T20:49:03.927 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:49:03.927 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:49:03.927 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:49:03.927 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:49:03.927 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:49:03.927 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 181708 2023-09-23T20:49:03.927 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.927 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:49:03.928 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.928 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:49:03.928 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:01.717054+0000", 2023-09-23T20:49:03.928 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:03.928 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.928 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.928 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:49:03.928 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:01.717054+0000", 2023-09-23T20:49:03.928 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:03.928 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.929 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.929 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:49:03.929 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:01.717052+0000", 2023-09-23T20:49:03.929 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:49:03.929 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.929 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.929 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:49:03.929 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:01.717060+0000", 2023-09-23T20:49:03.929 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.6389999999999998e-06 2023-09-23T20:49:03.929 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.930 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.930 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:49:03.930 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:01.717081+0000", 2023-09-23T20:49:03.930 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.0638000000000001e-05 2023-09-23T20:49:03.930 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.930 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.930 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:49:03.930 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:01.717091+0000", 2023-09-23T20:49:03.930 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.9410000000000002e-06 2023-09-23T20:49:03.931 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.931 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.931 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:03.931 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:01.717148+0000", 2023-09-23T20:49:03.931 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.6777000000000002e-05 2023-09-23T20:49:03.931 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.931 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.931 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:49:03.931 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:01.717157+0000", 2023-09-23T20:49:03.931 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.8040000000000007e-06 2023-09-23T20:49:03.932 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.932 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.932 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:03.932 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:02.693500+0000", 2023-09-23T20:49:03.932 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.97634373500000005 2023-09-23T20:49:03.932 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:03.932 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:03.932 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:49:03.932 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:02.693508+0000", 2023-09-23T20:49:03.933 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.3490000000000003e-06 2023-09-23T20:49:03.933 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:03.933 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:49:03.933 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:03.933 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:03.933 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:49:03.933 INFO:teuthology.orchestra.run.smithi018.stdout:} 2023-09-23T20:49:03.934 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:03.935 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:03.935 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:49:03.935 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:49:03.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:03 smithi149 ceph-mon[170394]: pgmap v1507: 129 pgs: 1 remapped+peering, 1 active+remapped+backfilling, 127 active+clean; 1.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 977 op/s; 538/85098 objects misplaced (0.632%); 5.3 MiB/s, 85 objects/s recovering 2023-09-23T20:49:03.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:03 smithi149 ceph-mon[170394]: failed to encode map e604 with expected crc 2023-09-23T20:49:03.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:03 smithi149 ceph-mon[170394]: failed to encode map e604 with expected crc 2023-09-23T20:49:03.968 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:03 smithi149 ceph-mon[170394]: failed to encode map e604 with expected crc 2023-09-23T20:49:03.968 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:03 smithi149 ceph-mon[170394]: 10.12 starting backfill to osd.2 from (0'0,0'0] MAX to 600'11208 2023-09-23T20:49:03.968 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:03 smithi149 ceph-mon[170394]: failed to encode map e604 with expected crc 2023-09-23T20:49:03.968 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:03 smithi149 ceph-mon[170394]: failed to encode map e604 with expected crc 2023-09-23T20:49:03.968 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:03 smithi149 ceph-mon[170394]: failed to encode map e604 with expected crc 2023-09-23T20:49:03.968 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:03 smithi149 ceph-mon[170394]: failed to encode map e604 with expected crc 2023-09-23T20:49:03.969 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:03 smithi149 ceph-mon[170394]: failed to encode map e604 with expected crc 2023-09-23T20:49:03.969 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:03 smithi149 ceph-mon[170394]: failed to encode map e604 with expected crc 2023-09-23T20:49:03.969 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:03 smithi149 ceph-mon[170394]: 10.12 starting backfill to osd.5 from (0'0,0'0] MAX to 600'11208 2023-09-23T20:49:03.969 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:03 smithi149 ceph-mon[170394]: 10.12 starting backfill to osd.6 from (0'0,0'0] MAX to 600'11208 2023-09-23T20:49:03.969 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:03 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/482836455' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:49:03.969 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:03 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:49:03.969 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:03 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3103083048' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "26"}]: dispatch 2023-09-23T20:49:03.970 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:03 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "26"}]: dispatch 2023-09-23T20:49:03.970 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:03 smithi149 ceph-mon[170394]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:49:03.970 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:03 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:49:03.970 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:03 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "26"}]': finished 2023-09-23T20:49:03.970 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:03 smithi149 ceph-mon[170394]: osdmap e605: 8 total, 8 up, 8 in 2023-09-23T20:49:04.036 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:49:04.087 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:04.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:03 smithi018 ceph-mon[179647]: pgmap v1507: 129 pgs: 1 remapped+peering, 1 active+remapped+backfilling, 127 active+clean; 1.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 977 op/s; 538/85098 objects misplaced (0.632%); 5.3 MiB/s, 85 objects/s recovering 2023-09-23T20:49:04.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:03 smithi018 ceph-mon[179647]: failed to encode map e604 with expected crc 2023-09-23T20:49:04.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:03 smithi018 ceph-mon[179647]: failed to encode map e604 with expected crc 2023-09-23T20:49:04.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:03 smithi018 ceph-mon[179647]: failed to encode map e604 with expected crc 2023-09-23T20:49:04.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:03 smithi018 ceph-mon[179647]: 10.12 starting backfill to osd.2 from (0'0,0'0] MAX to 600'11208 2023-09-23T20:49:04.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:03 smithi018 ceph-mon[179647]: failed to encode map e604 with expected crc 2023-09-23T20:49:04.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:03 smithi018 ceph-mon[179647]: failed to encode map e604 with expected crc 2023-09-23T20:49:04.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:03 smithi018 ceph-mon[179647]: failed to encode map e604 with expected crc 2023-09-23T20:49:04.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:03 smithi018 ceph-mon[179647]: failed to encode map e604 with expected crc 2023-09-23T20:49:04.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:03 smithi018 ceph-mon[179647]: failed to encode map e604 with expected crc 2023-09-23T20:49:04.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:03 smithi018 ceph-mon[179647]: failed to encode map e604 with expected crc 2023-09-23T20:49:04.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:03 smithi018 ceph-mon[179647]: 10.12 starting backfill to osd.5 from (0'0,0'0] MAX to 600'11208 2023-09-23T20:49:04.144 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:03 smithi018 ceph-mon[179647]: 10.12 starting backfill to osd.6 from (0'0,0'0] MAX to 600'11208 2023-09-23T20:49:04.144 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:03 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/482836455' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:49:04.144 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:03 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:49:04.144 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:03 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3103083048' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "26"}]: dispatch 2023-09-23T20:49:04.144 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:03 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "26"}]: dispatch 2023-09-23T20:49:04.145 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:03 smithi018 ceph-mon[179647]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:49:04.145 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:03 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:49:04.145 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:03 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "26"}]': finished 2023-09-23T20:49:04.145 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:03 smithi018 ceph-mon[179647]: osdmap e605: 8 total, 8 up, 8 in 2023-09-23T20:49:04.145 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:03 smithi018 ceph-mon[184144]: pgmap v1507: 129 pgs: 1 remapped+peering, 1 active+remapped+backfilling, 127 active+clean; 1.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 977 op/s; 538/85098 objects misplaced (0.632%); 5.3 MiB/s, 85 objects/s recovering 2023-09-23T20:49:04.146 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:03 smithi018 ceph-mon[184144]: failed to encode map e604 with expected crc 2023-09-23T20:49:04.146 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:03 smithi018 ceph-mon[184144]: failed to encode map e604 with expected crc 2023-09-23T20:49:04.146 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:03 smithi018 ceph-mon[184144]: failed to encode map e604 with expected crc 2023-09-23T20:49:04.146 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:03 smithi018 ceph-mon[184144]: 10.12 starting backfill to osd.2 from (0'0,0'0] MAX to 600'11208 2023-09-23T20:49:04.146 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:03 smithi018 ceph-mon[184144]: failed to encode map e604 with expected crc 2023-09-23T20:49:04.147 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:03 smithi018 ceph-mon[184144]: failed to encode map e604 with expected crc 2023-09-23T20:49:04.147 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:03 smithi018 ceph-mon[184144]: failed to encode map e604 with expected crc 2023-09-23T20:49:04.147 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:03 smithi018 ceph-mon[184144]: failed to encode map e604 with expected crc 2023-09-23T20:49:04.148 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:03 smithi018 ceph-mon[184144]: failed to encode map e604 with expected crc 2023-09-23T20:49:04.148 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:03 smithi018 ceph-mon[184144]: failed to encode map e604 with expected crc 2023-09-23T20:49:04.148 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:03 smithi018 ceph-mon[184144]: 10.12 starting backfill to osd.5 from (0'0,0'0] MAX to 600'11208 2023-09-23T20:49:04.148 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:03 smithi018 ceph-mon[184144]: 10.12 starting backfill to osd.6 from (0'0,0'0] MAX to 600'11208 2023-09-23T20:49:04.149 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:03 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/482836455' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:49:04.149 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:03 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:49:04.149 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:03 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3103083048' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "26"}]: dispatch 2023-09-23T20:49:04.150 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:03 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "26"}]: dispatch 2023-09-23T20:49:04.150 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:03 smithi018 ceph-mon[184144]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:49:04.150 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:03 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:49:04.150 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:03 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "26"}]': finished 2023-09-23T20:49:04.151 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:03 smithi018 ceph-mon[184144]: osdmap e605: 8 total, 8 up, 8 in 2023-09-23T20:49:04.151 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:49:04.158 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:04.159 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:49:04.259 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:49:04.342 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:04.472 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:49:04.477 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:04.477 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:49:04.578 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:49:04.641 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:04.707 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:49:04.713 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:04.714 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:49:04.748 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:04 smithi018 ceph-mon[179647]: failed to encode map e605 with expected crc 2023-09-23T20:49:04.748 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:04 smithi018 ceph-mon[179647]: failed to encode map e605 with expected crc 2023-09-23T20:49:04.749 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:04 smithi018 ceph-mon[179647]: 10.16 starting backfill to osd.1 from (0'0,0'0] MAX to 602'11047 2023-09-23T20:49:04.749 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:04 smithi018 ceph-mon[179647]: failed to encode map e605 with expected crc 2023-09-23T20:49:04.749 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:04 smithi018 ceph-mon[179647]: failed to encode map e605 with expected crc 2023-09-23T20:49:04.749 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:04 smithi018 ceph-mon[179647]: failed to encode map e605 with expected crc 2023-09-23T20:49:04.749 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:04 smithi018 ceph-mon[179647]: 10.6 starting backfill to osd.7 from (0'0,0'0] MAX to 602'10995 2023-09-23T20:49:04.749 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:04 smithi018 ceph-mon[179647]: 10.2 starting backfill to osd.7 from (0'0,0'0] MAX to 602'11232 2023-09-23T20:49:04.749 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:04 smithi018 ceph-mon[179647]: 10.1 starting backfill to osd.3 from (0'0,0'0] MAX to 602'11093 2023-09-23T20:49:04.749 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:04 smithi018 ceph-mon[179647]: 10.c starting backfill to osd.1 from (0'0,0'0] MAX to 602'11500 2023-09-23T20:49:04.749 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:04 smithi018 ceph-mon[179647]: failed to encode map e605 with expected crc 2023-09-23T20:49:04.749 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:04 smithi018 ceph-mon[179647]: failed to encode map e605 with expected crc 2023-09-23T20:49:04.750 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:04 smithi018 ceph-mon[184144]: failed to encode map e605 with expected crc 2023-09-23T20:49:04.750 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:04 smithi018 ceph-mon[184144]: failed to encode map e605 with expected crc 2023-09-23T20:49:04.750 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:04 smithi018 ceph-mon[184144]: 10.16 starting backfill to osd.1 from (0'0,0'0] MAX to 602'11047 2023-09-23T20:49:04.750 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:04 smithi018 ceph-mon[184144]: failed to encode map e605 with expected crc 2023-09-23T20:49:04.750 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:04 smithi018 ceph-mon[184144]: failed to encode map e605 with expected crc 2023-09-23T20:49:04.750 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:04 smithi018 ceph-mon[184144]: failed to encode map e605 with expected crc 2023-09-23T20:49:04.750 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:04 smithi018 ceph-mon[184144]: 10.6 starting backfill to osd.7 from (0'0,0'0] MAX to 602'10995 2023-09-23T20:49:04.750 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:04 smithi018 ceph-mon[184144]: 10.2 starting backfill to osd.7 from (0'0,0'0] MAX to 602'11232 2023-09-23T20:49:04.751 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:04 smithi018 ceph-mon[184144]: 10.1 starting backfill to osd.3 from (0'0,0'0] MAX to 602'11093 2023-09-23T20:49:04.751 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:04 smithi018 ceph-mon[184144]: 10.c starting backfill to osd.1 from (0'0,0'0] MAX to 602'11500 2023-09-23T20:49:04.751 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:04 smithi018 ceph-mon[184144]: failed to encode map e605 with expected crc 2023-09-23T20:49:04.751 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:04 smithi018 ceph-mon[184144]: failed to encode map e605 with expected crc 2023-09-23T20:49:04.815 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:49:04.836 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T20:49:04.858 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:04.898 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:49:04.920 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:49:04.926 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:04.927 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:49:04.927 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:04 smithi149 ceph-mon[170394]: failed to encode map e605 with expected crc 2023-09-23T20:49:04.927 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:04 smithi149 ceph-mon[170394]: failed to encode map e605 with expected crc 2023-09-23T20:49:04.927 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:04 smithi149 ceph-mon[170394]: 10.16 starting backfill to osd.1 from (0'0,0'0] MAX to 602'11047 2023-09-23T20:49:04.927 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:04 smithi149 ceph-mon[170394]: failed to encode map e605 with expected crc 2023-09-23T20:49:04.927 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:04 smithi149 ceph-mon[170394]: failed to encode map e605 with expected crc 2023-09-23T20:49:04.927 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:04 smithi149 ceph-mon[170394]: failed to encode map e605 with expected crc 2023-09-23T20:49:04.928 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:04 smithi149 ceph-mon[170394]: 10.6 starting backfill to osd.7 from (0'0,0'0] MAX to 602'10995 2023-09-23T20:49:04.928 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:04 smithi149 ceph-mon[170394]: 10.2 starting backfill to osd.7 from (0'0,0'0] MAX to 602'11232 2023-09-23T20:49:04.928 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:04 smithi149 ceph-mon[170394]: 10.1 starting backfill to osd.3 from (0'0,0'0] MAX to 602'11093 2023-09-23T20:49:04.928 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:04 smithi149 ceph-mon[170394]: 10.c starting backfill to osd.1 from (0'0,0'0] MAX to 602'11500 2023-09-23T20:49:04.928 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:04 smithi149 ceph-mon[170394]: failed to encode map e605 with expected crc 2023-09-23T20:49:04.928 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:04 smithi149 ceph-mon[170394]: failed to encode map e605 with expected crc 2023-09-23T20:49:04.931 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:49:04.986 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:49:05.000 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:49:04 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:49:04] "GET /metrics HTTP/1.1" 200 36614 "" "Prometheus/2.43.0" 2023-09-23T20:49:05.028 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:49:05.034 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:49:05.083 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:49:05.135 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:05.135 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:49:05.182 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:49:05.210 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:49:05.216 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:05.217 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:49:05.227 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:49:05.245 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:49:05.245 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:49:05.245 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:49:05.245 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:49:05.245 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:49:05.245 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:49:05.245 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:49:05.245 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:49:05.318 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:49:05.374 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:05.456 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:49:05.460 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:05.460 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:49:05.561 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:49:05.624 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:05.695 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:49:05.704 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:05.704 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:49:05.805 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:49:05.852 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:49:05.889 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:05 smithi018 ceph-mon[184144]: pgmap v1510: 129 pgs: 2 remapped+peering, 1 active+remapped+backfilling, 126 active+clean; 1.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 790 op/s; 538/83835 objects misplaced (0.642%); 5.3 MiB/s, 85 objects/s recovering 2023-09-23T20:49:05.889 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:05 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1748817799' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:49:05.889 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:05 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:49:05.890 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:05 smithi018 ceph-mon[179647]: pgmap v1510: 129 pgs: 2 remapped+peering, 1 active+remapped+backfilling, 126 active+clean; 1.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 790 op/s; 538/83835 objects misplaced (0.642%); 5.3 MiB/s, 85 objects/s recovering 2023-09-23T20:49:05.890 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:05 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1748817799' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:49:05.890 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:05 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:49:05.899 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:05.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:05 smithi149 ceph-mon[170394]: pgmap v1510: 129 pgs: 2 remapped+peering, 1 active+remapped+backfilling, 126 active+clean; 1.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 790 op/s; 538/83835 objects misplaced (0.642%); 5.3 MiB/s, 85 objects/s recovering 2023-09-23T20:49:05.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:05 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1748817799' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:49:05.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:05 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:49:06.014 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:49:06.024 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:06.025 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:49:06.126 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:49:06.186 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:06.248 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:49:06.258 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:06.259 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:49:06.360 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:49:06.433 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:06.473 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T20:49:06.734 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:49:06.747 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:06.748 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:49:06.848 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:49:06.889 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:06.940 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:49:06.955 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:06.955 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:49:07.056 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:49:07.143 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:07.223 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:49:07.228 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:07.228 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:49:07.329 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:49:07.384 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:07.425 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:49:07.441 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:49:07.446 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:07.447 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:49:07.503 INFO:teuthology.orchestra.run.smithi018.stdout:ERROR: (22) Invalid argument 2023-09-23T20:49:07.503 INFO:teuthology.orchestra.run.smithi018.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:49:07.548 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:49:07.612 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:07.675 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:49:07.683 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:07.684 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:49:07.694 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:49:07.784 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:49:07.876 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:07 smithi018 ceph-mon[184144]: pgmap v1511: 129 pgs: 2 remapped+peering, 3 active+remapped+backfilling, 124 active+clean; 1.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 498 op/s; 2493/79791 objects misplaced (3.124%); 8.8 MiB/s, 140 objects/s recovering 2023-09-23T20:49:07.876 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:07 smithi018 ceph-mon[179647]: pgmap v1511: 129 pgs: 2 remapped+peering, 3 active+remapped+backfilling, 124 active+clean; 1.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 498 op/s; 2493/79791 objects misplaced (3.124%); 8.8 MiB/s, 140 objects/s recovering 2023-09-23T20:49:07.883 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:07.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:07 smithi149 ceph-mon[170394]: pgmap v1511: 129 pgs: 2 remapped+peering, 3 active+remapped+backfilling, 124 active+clean; 1.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 498 op/s; 2493/79791 objects misplaced (3.124%); 8.8 MiB/s, 140 objects/s recovering 2023-09-23T20:49:07.979 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:49:08.001 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:08.002 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:49:08.092 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T20:49:08.103 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:49:08.156 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:08.224 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:49:08.234 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:08.234 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:49:08.335 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:49:08.375 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:08.441 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:49:08.447 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:08.448 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:49:08.548 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:49:08.604 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:08.664 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:49:08.670 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:08.670 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:49:08.771 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:49:08.813 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:08.870 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:49:08.876 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:08.876 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:49:08.977 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:49:08.992 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:49:09.092 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:09.273 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:49:09.281 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:09.281 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:49:09.357 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:49:09.382 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:49:09.411 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:09.454 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:49:09.458 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:09.459 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:49:09.517 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 7, 3, 6, 5, 4, 2] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T20:49:09.517 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:49:09.518 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_5 2023-09-23T20:49:09.518 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:49:09.560 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:49:09.602 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:09.813 INFO:teuthology.orchestra.run.smithi018.stderr:nodeep-scrub is unset 2023-09-23T20:49:09.845 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:09 smithi018 ceph-mon[184144]: pgmap v1512: 129 pgs: 2 active+remapped+backfill_wait, 1 remapped+peering, 3 active+remapped+backfilling, 123 active+clean; 1.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 492 op/s; 4011/76938 objects misplaced (5.213%); 18 MiB/s, 287 objects/s recovering 2023-09-23T20:49:09.845 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:09 smithi018 ceph-mon[184144]: osdmap e606: 8 total, 8 up, 8 in 2023-09-23T20:49:09.845 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:09 smithi018 ceph-mon[184144]: failed to encode map e606 with expected crc 2023-09-23T20:49:09.845 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:09 smithi018 ceph-mon[184144]: failed to encode map e606 with expected crc 2023-09-23T20:49:09.846 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:09 smithi018 ceph-mon[184144]: failed to encode map e606 with expected crc 2023-09-23T20:49:09.846 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:09 smithi018 ceph-mon[184144]: failed to encode map e606 with expected crc 2023-09-23T20:49:09.846 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:09 smithi018 ceph-mon[184144]: failed to encode map e606 with expected crc 2023-09-23T20:49:09.846 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:09 smithi018 ceph-mon[184144]: failed to encode map e606 with expected crc 2023-09-23T20:49:09.846 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:09 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3338750632' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:49:09.846 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:09 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:49:09.846 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:09 smithi018 ceph-mon[179647]: pgmap v1512: 129 pgs: 2 active+remapped+backfill_wait, 1 remapped+peering, 3 active+remapped+backfilling, 123 active+clean; 1.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 492 op/s; 4011/76938 objects misplaced (5.213%); 18 MiB/s, 287 objects/s recovering 2023-09-23T20:49:09.847 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:09 smithi018 ceph-mon[179647]: osdmap e606: 8 total, 8 up, 8 in 2023-09-23T20:49:09.847 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:09 smithi018 ceph-mon[179647]: failed to encode map e606 with expected crc 2023-09-23T20:49:09.847 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:09 smithi018 ceph-mon[179647]: failed to encode map e606 with expected crc 2023-09-23T20:49:09.847 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:09 smithi018 ceph-mon[179647]: failed to encode map e606 with expected crc 2023-09-23T20:49:09.847 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:09 smithi018 ceph-mon[179647]: failed to encode map e606 with expected crc 2023-09-23T20:49:09.847 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:09 smithi018 ceph-mon[179647]: failed to encode map e606 with expected crc 2023-09-23T20:49:09.847 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:09 smithi018 ceph-mon[179647]: failed to encode map e606 with expected crc 2023-09-23T20:49:09.847 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:09 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3338750632' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:49:09.847 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:09 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:49:10.164 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:49:10.168 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:10.168 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:49:10.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:09 smithi149 ceph-mon[170394]: pgmap v1512: 129 pgs: 2 active+remapped+backfill_wait, 1 remapped+peering, 3 active+remapped+backfilling, 123 active+clean; 1.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 492 op/s; 4011/76938 objects misplaced (5.213%); 18 MiB/s, 287 objects/s recovering 2023-09-23T20:49:10.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:09 smithi149 ceph-mon[170394]: osdmap e606: 8 total, 8 up, 8 in 2023-09-23T20:49:10.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:09 smithi149 ceph-mon[170394]: failed to encode map e606 with expected crc 2023-09-23T20:49:10.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:09 smithi149 ceph-mon[170394]: failed to encode map e606 with expected crc 2023-09-23T20:49:10.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:09 smithi149 ceph-mon[170394]: failed to encode map e606 with expected crc 2023-09-23T20:49:10.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:09 smithi149 ceph-mon[170394]: failed to encode map e606 with expected crc 2023-09-23T20:49:10.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:09 smithi149 ceph-mon[170394]: failed to encode map e606 with expected crc 2023-09-23T20:49:10.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:09 smithi149 ceph-mon[170394]: failed to encode map e606 with expected crc 2023-09-23T20:49:10.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:09 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3338750632' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:49:10.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:09 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:49:10.268 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:49:10.298 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:10.337 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:49:10.340 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:10.340 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:49:10.441 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:49:10.476 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:10.594 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:49:10.596 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:10.597 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:49:10.697 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:49:10.733 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:10.927 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:49:10.935 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:10.935 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:49:10.979 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:49:11.018 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:10 smithi018 ceph-mon[184144]: failed to encode map e606 with expected crc 2023-09-23T20:49:11.018 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:10 smithi018 ceph-mon[184144]: 10.7 scrub starts 2023-09-23T20:49:11.019 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:10 smithi018 ceph-mon[184144]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:49:11.019 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:10 smithi018 ceph-mon[184144]: Cluster is now healthy 2023-09-23T20:49:11.019 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:10 smithi018 ceph-mon[184144]: failed to encode map e607 with expected crc 2023-09-23T20:49:11.019 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:10 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:49:11.019 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:10 smithi018 ceph-mon[184144]: osdmap e607: 8 total, 8 up, 8 in 2023-09-23T20:49:11.019 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:10 smithi018 ceph-mon[184144]: failed to encode map e607 with expected crc 2023-09-23T20:49:11.019 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:10 smithi018 ceph-mon[179647]: failed to encode map e606 with expected crc 2023-09-23T20:49:11.019 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:10 smithi018 ceph-mon[179647]: 10.7 scrub starts 2023-09-23T20:49:11.019 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:10 smithi018 ceph-mon[179647]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:49:11.020 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:10 smithi018 ceph-mon[179647]: Cluster is now healthy 2023-09-23T20:49:11.020 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:10 smithi018 ceph-mon[179647]: failed to encode map e607 with expected crc 2023-09-23T20:49:11.020 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:10 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:49:11.020 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:10 smithi018 ceph-mon[179647]: osdmap e607: 8 total, 8 up, 8 in 2023-09-23T20:49:11.020 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:10 smithi018 ceph-mon[179647]: failed to encode map e607 with expected crc 2023-09-23T20:49:11.023 INFO:teuthology.orchestra.run.smithi018.stdout:ERROR: (22) Invalid argument 2023-09-23T20:49:11.023 INFO:teuthology.orchestra.run.smithi018.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:49:11.036 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:49:11.076 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:10 smithi149 ceph-mon[170394]: failed to encode map e606 with expected crc 2023-09-23T20:49:11.076 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:10 smithi149 ceph-mon[170394]: 10.7 scrub starts 2023-09-23T20:49:11.076 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:10 smithi149 ceph-mon[170394]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:49:11.076 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:10 smithi149 ceph-mon[170394]: Cluster is now healthy 2023-09-23T20:49:11.076 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:10 smithi149 ceph-mon[170394]: failed to encode map e607 with expected crc 2023-09-23T20:49:11.076 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:10 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:49:11.076 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:10 smithi149 ceph-mon[170394]: osdmap e607: 8 total, 8 up, 8 in 2023-09-23T20:49:11.076 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:10 smithi149 ceph-mon[170394]: failed to encode map e607 with expected crc 2023-09-23T20:49:11.079 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:11.127 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:49:11.132 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:11.132 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:49:11.233 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:49:11.259 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:11.294 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:49:11.299 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:11.300 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:49:11.400 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:49:11.426 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:11.460 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:49:11.463 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:11.464 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:49:11.564 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:49:11.591 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:11.626 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:49:11.629 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:11.629 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:49:11.730 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:49:11.764 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:11.809 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:49:11.819 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:11.820 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:49:11.838 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:49:11.838 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:49:11.849 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:49:11.886 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:49:11.921 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:49:11.922 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:49:11.956 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:49:11.993 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:12.001 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:49:12.030 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:49:12.041 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:49:12.041 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:49:12.041 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:49:12.041 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:49:12.041 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:49:12.041 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:49:12.042 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:49:12.042 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:49:12.069 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:11 smithi018 ceph-mon[184144]: pgmap v1515: 129 pgs: 1 active+clean+remapped, 2 active+remapped+backfill_wait, 3 active+remapped+backfilling, 123 active+clean; 1.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 488 op/s; 4011/74088 objects misplaced (5.414%); 22 MiB/s, 347 objects/s recovering 2023-09-23T20:49:12.069 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:11 smithi018 ceph-mon[184144]: osdmap e608: 8 total, 8 up, 8 in 2023-09-23T20:49:12.070 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:11 smithi018 ceph-mon[184144]: failed to encode map e608 with expected crc 2023-09-23T20:49:12.070 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:11 smithi018 ceph-mon[184144]: failed to encode map e608 with expected crc 2023-09-23T20:49:12.070 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:11 smithi018 ceph-mon[184144]: failed to encode map e608 with expected crc 2023-09-23T20:49:12.070 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:11 smithi018 ceph-mon[184144]: osdmap e609: 8 total, 8 up, 8 in 2023-09-23T20:49:12.070 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:11 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/903831033' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:49:12.070 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:11 smithi018 ceph-mon[179647]: pgmap v1515: 129 pgs: 1 active+clean+remapped, 2 active+remapped+backfill_wait, 3 active+remapped+backfilling, 123 active+clean; 1.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 488 op/s; 4011/74088 objects misplaced (5.414%); 22 MiB/s, 347 objects/s recovering 2023-09-23T20:49:12.071 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:11 smithi018 ceph-mon[179647]: osdmap e608: 8 total, 8 up, 8 in 2023-09-23T20:49:12.071 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:11 smithi018 ceph-mon[179647]: failed to encode map e608 with expected crc 2023-09-23T20:49:12.071 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:11 smithi018 ceph-mon[179647]: failed to encode map e608 with expected crc 2023-09-23T20:49:12.071 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:11 smithi018 ceph-mon[179647]: failed to encode map e608 with expected crc 2023-09-23T20:49:12.071 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:11 smithi018 ceph-mon[179647]: osdmap e609: 8 total, 8 up, 8 in 2023-09-23T20:49:12.071 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:11 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/903831033' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:49:12.148 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T20:49:12.205 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:49:12.210 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:12.211 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:49:12.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:11 smithi149 ceph-mon[170394]: pgmap v1515: 129 pgs: 1 active+clean+remapped, 2 active+remapped+backfill_wait, 3 active+remapped+backfilling, 123 active+clean; 1.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 488 op/s; 4011/74088 objects misplaced (5.414%); 22 MiB/s, 347 objects/s recovering 2023-09-23T20:49:12.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:11 smithi149 ceph-mon[170394]: osdmap e608: 8 total, 8 up, 8 in 2023-09-23T20:49:12.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:11 smithi149 ceph-mon[170394]: failed to encode map e608 with expected crc 2023-09-23T20:49:12.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:11 smithi149 ceph-mon[170394]: failed to encode map e608 with expected crc 2023-09-23T20:49:12.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:11 smithi149 ceph-mon[170394]: failed to encode map e608 with expected crc 2023-09-23T20:49:12.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:11 smithi149 ceph-mon[170394]: osdmap e609: 8 total, 8 up, 8 in 2023-09-23T20:49:12.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:11 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/903831033' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:49:12.311 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:49:12.384 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:12.435 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T20:49:12.903 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:49:12.909 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:12.910 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:49:12.923 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:12 smithi018 ceph-mon[179647]: failed to encode map e608 with expected crc 2023-09-23T20:49:12.923 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:12 smithi018 ceph-mon[179647]: failed to encode map e608 with expected crc 2023-09-23T20:49:12.923 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:12 smithi018 ceph-mon[179647]: failed to encode map e608 with expected crc 2023-09-23T20:49:12.923 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:12 smithi018 ceph-mon[179647]: 10.0 scrub starts 2023-09-23T20:49:12.923 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:12 smithi018 ceph-mon[179647]: 10.3 deep-scrub starts 2023-09-23T20:49:12.923 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:12 smithi018 ceph-mon[179647]: failed to encode map e609 with expected crc 2023-09-23T20:49:12.923 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:12 smithi018 ceph-mon[179647]: failed to encode map e609 with expected crc 2023-09-23T20:49:12.924 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:12 smithi018 ceph-mon[179647]: osdmap e610: 8 total, 8 up, 8 in 2023-09-23T20:49:12.924 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:12 smithi018 ceph-mon[184144]: failed to encode map e608 with expected crc 2023-09-23T20:49:12.924 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:12 smithi018 ceph-mon[184144]: failed to encode map e608 with expected crc 2023-09-23T20:49:12.924 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:12 smithi018 ceph-mon[184144]: failed to encode map e608 with expected crc 2023-09-23T20:49:12.924 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:12 smithi018 ceph-mon[184144]: 10.0 scrub starts 2023-09-23T20:49:12.924 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:12 smithi018 ceph-mon[184144]: 10.3 deep-scrub starts 2023-09-23T20:49:12.924 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:12 smithi018 ceph-mon[184144]: failed to encode map e609 with expected crc 2023-09-23T20:49:12.924 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:12 smithi018 ceph-mon[184144]: failed to encode map e609 with expected crc 2023-09-23T20:49:12.924 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:12 smithi018 ceph-mon[184144]: osdmap e610: 8 total, 8 up, 8 in 2023-09-23T20:49:13.011 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:49:13.090 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:12 smithi149 ceph-mon[170394]: failed to encode map e608 with expected crc 2023-09-23T20:49:13.090 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:12 smithi149 ceph-mon[170394]: failed to encode map e608 with expected crc 2023-09-23T20:49:13.090 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:12 smithi149 ceph-mon[170394]: failed to encode map e608 with expected crc 2023-09-23T20:49:13.090 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:12 smithi149 ceph-mon[170394]: 10.0 scrub starts 2023-09-23T20:49:13.090 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:12 smithi149 ceph-mon[170394]: 10.3 deep-scrub starts 2023-09-23T20:49:13.090 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:12 smithi149 ceph-mon[170394]: failed to encode map e609 with expected crc 2023-09-23T20:49:13.091 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:12 smithi149 ceph-mon[170394]: failed to encode map e609 with expected crc 2023-09-23T20:49:13.091 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:12 smithi149 ceph-mon[170394]: osdmap e610: 8 total, 8 up, 8 in 2023-09-23T20:49:13.093 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:13.149 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:49:13.157 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:13.157 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:49:13.258 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:49:13.322 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:49:13.347 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:13.562 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:49:13.571 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:13.572 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:49:13.646 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:49:13.672 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:49:13.763 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:13.827 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:13 smithi018 ceph-mon[179647]: 10.10 deep-scrub starts 2023-09-23T20:49:13.827 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:13 smithi018 ceph-mon[179647]: pgmap v1518: 129 pgs: 1 active, 1 peering, 1 active+clean+remapped, 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 123 active+clean; 1.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 912 op/s; 1543/65982 objects misplaced (2.339%); 41 MiB/s, 656 objects/s recovering 2023-09-23T20:49:13.827 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:13 smithi018 ceph-mon[179647]: failed to encode map e610 with expected crc 2023-09-23T20:49:13.827 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:13 smithi018 ceph-mon[179647]: osdmap e611: 8 total, 8 up, 8 in 2023-09-23T20:49:13.828 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:13 smithi018 ceph-mon[184144]: 10.10 deep-scrub starts 2023-09-23T20:49:13.828 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:13 smithi018 ceph-mon[184144]: pgmap v1518: 129 pgs: 1 active, 1 peering, 1 active+clean+remapped, 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 123 active+clean; 1.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 912 op/s; 1543/65982 objects misplaced (2.339%); 41 MiB/s, 656 objects/s recovering 2023-09-23T20:49:13.828 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:13 smithi018 ceph-mon[184144]: failed to encode map e610 with expected crc 2023-09-23T20:49:13.828 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:13 smithi018 ceph-mon[184144]: osdmap e611: 8 total, 8 up, 8 in 2023-09-23T20:49:13.870 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:49:13.883 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:13.884 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:49:13.985 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:49:14.017 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:49:14.026 INFO:teuthology.orchestra.run.smithi018.stderr:dumped all 2023-09-23T20:49:14.077 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:14.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:13 smithi149 ceph-mon[170394]: 10.10 deep-scrub starts 2023-09-23T20:49:14.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:13 smithi149 ceph-mon[170394]: pgmap v1518: 129 pgs: 1 active, 1 peering, 1 active+clean+remapped, 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 123 active+clean; 1.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 912 op/s; 1543/65982 objects misplaced (2.339%); 41 MiB/s, 656 objects/s recovering 2023-09-23T20:49:14.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:13 smithi149 ceph-mon[170394]: failed to encode map e610 with expected crc 2023-09-23T20:49:14.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:13 smithi149 ceph-mon[170394]: osdmap e611: 8 total, 8 up, 8 in 2023-09-23T20:49:14.334 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:49:14.338 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:14.339 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:49:14.439 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:49:14.478 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:14.726 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:49:14.730 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:14.730 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:49:14.776 INFO:teuthology.orchestra.run.smithi018.stdout:{"pg_ready":true,"pg_map":{"version":1520,"stamp":"2023-09-23T20:49:13.700169+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1410084690,"num_objects":21471,"num_object_clones":0,"num_object_copies":64413,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1543,"num_objects_unfound":0,"num_objects_dirty":21471,"num_whiteouts":0,"num_read":6043,"num_read_kb":13311,"num_write":188480,"num_write_kb":6700569,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18309,"num_bytes_recovered":1209107498,"num_keys_recovered":80,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":175984,"ondisk_log_size":175984,"up":387,"acting":387,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":403,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":10827252,"kb_used_data":4925300,"kb_used_omap":32,"kb_used_meta":5901855,"kb_avail":739133964,"statfs":{"total":767960285184,"available":756873179136,"internally_reserved":0,"allocated":5043507200,"data_stored":5025374118,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32773,"internal_metadata":6043500539},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,0,2,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-211353600,"num_objects":-3225,"num_object_clones":0,"num_object_copies":-9675,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-2468,"num_objects_unfound":0,"num_objects_dirty":-3225,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3225,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1886,"num_bytes_recovered":123600896,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"3.898076"},"pg_stats":[{"pgid":"10.13","version":"610'11438","reported_seq":11971,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T20:49:13.307391+0000","last_change":"2023-09-23T20:48:48.187746+0000","last_active":"2023-09-23T20:49:13.307391+0000","last_peered":"2023-09-23T20:49:13.307391+0000","last_clean":"2023-09-23T20:49:13.307391+0000","last_became_active":"2023-09-23T20:48:48.156992+0000","last_became_peered":"2023-09-23T20:48:48.156992+0000","last_unstale":"2023-09-23T20:49:13.307391+0000","last_undegraded":"2023-09-23T20:49:13.307391+0000","last_fullsized":"2023-09-23T20:49:13.307391+0000","mapping_epoch":591,"log_start":"536'1300","ondisk_log_start":"536'1300","created":584,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":10138,"log_dups_size":0,"ondisk_log_size":10138,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":42532864,"num_objects":649,"num_object_clones":0,"num_object_copies":1947,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":649,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6025,"num_write_kb":213568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":104,"num_bytes_recovered":6815744,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.18","version":"90'161","reported_seq":652,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T20:49:11.701661+0000","last_change":"2023-09-23T20:48:51.176302+0000","last_active":"2023-09-23T20:49:11.701661+0000","last_peered":"2023-09-23T20:49:11.701661+0000","last_clean":"2023-09-23T20:49:11.701661+0000","last_became_active":"2023-09-23T20:48:51.175408+0000","last_became_peered":"2023-09-23T20:48:51.175408+0000","last_unstale":"2023-09-23T20:49:11.701661+0000","last_undegraded":"2023-09-23T20:49:11.701661+0000","last_fullsized":"2023-09-23T20:49:11.701661+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:19.677209+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:19.677209+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:43:11.718198+0000","scrub_duration":0.033122453000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"69'1","reported_seq":85,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T20:49:09.808263+0000","last_change":"2023-09-23T20:48:48.148250+0000","last_active":"2023-09-23T20:49:09.808263+0000","last_peered":"2023-09-23T20:49:09.808263+0000","last_clean":"2023-09-23T20:49:09.808263+0000","last_became_active":"2023-09-23T20:48:48.144271+0000","last_became_peered":"2023-09-23T20:48:48.144271+0000","last_unstale":"2023-09-23T20:49:09.808263+0000","last_undegraded":"2023-09-23T20:49:09.808263+0000","last_fullsized":"2023-09-23T20:49:09.808263+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:59:36.502986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":449,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T20:49:03.754486+0000","last_change":"2023-09-23T20:49:02.703099+0000","last_active":"2023-09-23T20:49:03.754486+0000","last_peered":"2023-09-23T20:49:03.754486+0000","last_clean":"2023-09-23T20:49:03.754486+0000","last_became_active":"2023-09-23T20:49:02.702753+0000","last_became_peered":"2023-09-23T20:49:02.702753+0000","last_unstale":"2023-09-23T20:49:03.754486+0000","last_undegraded":"2023-09-23T20:49:03.754486+0000","last_fullsized":"2023-09-23T20:49:03.754486+0000","mapping_epoch":603,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":604,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:24:30.834235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.10","version":"607'11154","reported_seq":14531,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T20:49:10.072433+0000","last_change":"2023-09-23T20:48:55.233799+0000","last_active":"2023-09-23T20:49:10.072433+0000","last_peered":"2023-09-23T20:49:10.072433+0000","last_clean":"2023-09-23T20:49:10.072433+0000","last_became_active":"2023-09-23T20:48:55.189956+0000","last_became_peered":"2023-09-23T20:48:55.189956+0000","last_unstale":"2023-09-23T20:49:10.072433+0000","last_undegraded":"2023-09-23T20:49:10.072433+0000","last_fullsized":"2023-09-23T20:49:10.072433+0000","mapping_epoch":598,"log_start":"542'4945","ondisk_log_start":"542'4945","created":584,"last_epoch_clean":599,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":6209,"log_dups_size":0,"ondisk_log_size":6209,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":42369024,"num_objects":647,"num_object_clones":0,"num_object_copies":1941,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":647,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5849,"num_write_kb":207840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1322,"num_bytes_recovered":86638592,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1b","version":"90'161","reported_seq":611,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T20:49:09.807946+0000","last_change":"2023-09-23T20:47:33.523278+0000","last_active":"2023-09-23T20:49:09.807946+0000","last_peered":"2023-09-23T20:49:09.807946+0000","last_clean":"2023-09-23T20:49:09.807946+0000","last_became_active":"2023-09-23T20:47:11.622087+0000","last_became_peered":"2023-09-23T20:47:11.622087+0000","last_unstale":"2023-09-23T20:49:09.807946+0000","last_undegraded":"2023-09-23T20:49:09.807946+0000","last_fullsized":"2023-09-23T20:49:09.807946+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:47:33.523187+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:36:05.442288+0000","last_clean_scrub_stamp":"2023-09-23T20:47:33.523187+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:32:18.410526+0000","scrub_duration":0.013532127,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":600,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T20:49:12.698812+0000","last_change":"2023-09-23T20:47:09.173501+0000","last_active":"2023-09-23T20:49:12.698812+0000","last_peered":"2023-09-23T20:49:12.698812+0000","last_clean":"2023-09-23T20:49:12.698812+0000","last_became_active":"2023-09-23T20:47:09.173160+0000","last_became_peered":"2023-09-23T20:47:09.173160+0000","last_unstale":"2023-09-23T20:49:12.698812+0000","last_undegraded":"2023-09-23T20:49:12.698812+0000","last_fullsized":"2023-09-23T20:49:12.698812+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:44:36.132093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":568,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T20:49:10.810291+0000","last_change":"2023-09-23T20:43:46.521939+0000","last_active":"2023-09-23T20:49:10.810291+0000","last_peered":"2023-09-23T20:49:10.810291+0000","last_clean":"2023-09-23T20:49:10.810291+0000","last_became_active":"2023-09-23T20:43:46.521185+0000","last_became_peered":"2023-09-23T20:43:46.521185+0000","last_unstale":"2023-09-23T20:49:10.810291+0000","last_undegraded":"2023-09-23T20:49:10.810291+0000","last_fullsized":"2023-09-23T20:49:10.810291+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:59:25.045627+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.11","version":"609'11174","reported_seq":12977,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T20:49:11.758809+0000","last_change":"2023-09-23T20:49:03.739386+0000","last_active":"2023-09-23T20:49:11.758809+0000","last_peered":"2023-09-23T20:49:11.758809+0000","last_clean":"2023-09-23T20:49:11.758809+0000","last_became_active":"2023-09-23T20:49:03.715004+0000","last_became_peered":"2023-09-23T20:49:03.715004+0000","last_unstale":"2023-09-23T20:49:11.758809+0000","last_undegraded":"2023-09-23T20:49:11.758809+0000","last_fullsized":"2023-09-23T20:49:11.758809+0000","mapping_epoch":604,"log_start":"543'5336","ondisk_log_start":"543'5336","created":584,"last_epoch_clean":605,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":5838,"log_dups_size":0,"ondisk_log_size":5838,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47120384,"num_objects":719,"num_object_clones":0,"num_object_copies":2157,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":719,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5833,"num_write_kb":209664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":827,"num_bytes_recovered":54132736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"90'161","reported_seq":496,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T20:49:09.807810+0000","last_change":"2023-09-23T20:47:31.475001+0000","last_active":"2023-09-23T20:49:09.807810+0000","last_peered":"2023-09-23T20:49:09.807810+0000","last_clean":"2023-09-23T20:49:09.807810+0000","last_became_active":"2023-09-23T20:47:11.617942+0000","last_became_peered":"2023-09-23T20:47:11.617942+0000","last_unstale":"2023-09-23T20:49:09.807810+0000","last_undegraded":"2023-09-23T20:49:09.807810+0000","last_fullsized":"2023-09-23T20:49:09.807810+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:47:31.474914+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:47:31.474914+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:52:05.169219+0000","scrub_duration":0.0026262870000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":402,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T20:49:03.753237+0000","last_change":"2023-09-23T20:37:43.584414+0000","last_active":"2023-09-23T20:49:03.753237+0000","last_peered":"2023-09-23T20:49:03.753237+0000","last_clean":"2023-09-23T20:49:03.753237+0000","last_became_active":"2023-09-23T20:37:43.584116+0000","last_became_peered":"2023-09-23T20:37:43.584116+0000","last_unstale":"2023-09-23T20:49:03.753237+0000","last_undegraded":"2023-09-23T20:49:03.753237+0000","last_fullsized":"2023-09-23T20:49:03.753237+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:17:54.258172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":222,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T20:49:08.814405+0000","last_change":"2023-09-23T20:38:37.457427+0000","last_active":"2023-09-23T20:49:08.814405+0000","last_peered":"2023-09-23T20:49:08.814405+0000","last_clean":"2023-09-23T20:49:08.814405+0000","last_became_active":"2023-09-23T20:38:37.457099+0000","last_became_peered":"2023-09-23T20:38:37.457099+0000","last_unstale":"2023-09-23T20:49:08.814405+0000","last_undegraded":"2023-09-23T20:49:08.814405+0000","last_fullsized":"2023-09-23T20:49:08.814405+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:31:27.084994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"90'161","reported_seq":819,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T20:49:08.829549+0000","last_change":"2023-09-23T20:40:22.449754+0000","last_active":"2023-09-23T20:49:08.829549+0000","last_peered":"2023-09-23T20:49:08.829549+0000","last_clean":"2023-09-23T20:49:08.829549+0000","last_became_active":"2023-09-23T20:40:22.448838+0000","last_became_peered":"2023-09-23T20:40:22.448838+0000","last_unstale":"2023-09-23T20:49:08.829549+0000","last_undegraded":"2023-09-23T20:49:08.829549+0000","last_fullsized":"2023-09-23T20:49:08.829549+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:03.411025+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:03.411025+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:22:07.948856+0000","scrub_duration":0.001802922,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":601,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T20:49:10.809896+0000","last_change":"2023-09-23T20:43:46.522325+0000","last_active":"2023-09-23T20:49:10.809896+0000","last_peered":"2023-09-23T20:49:10.809896+0000","last_clean":"2023-09-23T20:49:10.809896+0000","last_became_active":"2023-09-23T20:43:46.521996+0000","last_became_peered":"2023-09-23T20:43:46.521996+0000","last_unstale":"2023-09-23T20:49:10.809896+0000","last_undegraded":"2023-09-23T20:49:10.809896+0000","last_fullsized":"2023-09-23T20:49:10.809896+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:51:17.629956+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":39,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T20:49:03.753705+0000","last_change":"2023-09-23T20:48:23.918092+0000","last_active":"2023-09-23T20:49:03.753705+0000","last_peered":"2023-09-23T20:49:03.753705+0000","last_clean":"2023-09-23T20:49:03.753705+0000","last_became_active":"2023-09-23T20:48:23.917386+0000","last_became_peered":"2023-09-23T20:48:23.917386+0000","last_unstale":"2023-09-23T20:49:03.753705+0000","last_undegraded":"2023-09-23T20:49:03.753705+0000","last_fullsized":"2023-09-23T20:49:03.753705+0000","mapping_epoch":577,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":578,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:06:09.587517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"90'161","reported_seq":600,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T20:49:11.701291+0000","last_change":"2023-09-23T20:48:51.169725+0000","last_active":"2023-09-23T20:49:11.701291+0000","last_peered":"2023-09-23T20:49:11.701291+0000","last_clean":"2023-09-23T20:49:11.701291+0000","last_became_active":"2023-09-23T20:48:51.169314+0000","last_became_peered":"2023-09-23T20:48:51.169314+0000","last_unstale":"2023-09-23T20:49:11.701291+0000","last_undegraded":"2023-09-23T20:49:11.701291+0000","last_fullsized":"2023-09-23T20:49:11.701291+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:47.479012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:47.479012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":443,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T20:49:08.830094+0000","last_change":"2023-09-23T20:43:16.780478+0000","last_active":"2023-09-23T20:49:08.830094+0000","last_peered":"2023-09-23T20:49:08.830094+0000","last_clean":"2023-09-23T20:49:08.830094+0000","last_became_active":"2023-09-23T20:43:16.780183+0000","last_became_peered":"2023-09-23T20:43:16.780183+0000","last_unstale":"2023-09-23T20:49:08.830094+0000","last_undegraded":"2023-09-23T20:49:08.830094+0000","last_fullsized":"2023-09-23T20:49:08.830094+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:38:30.102066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":433,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T20:49:03.760578+0000","last_change":"2023-09-23T20:38:02.542034+0000","last_active":"2023-09-23T20:49:03.760578+0000","last_peered":"2023-09-23T20:49:03.760578+0000","last_clean":"2023-09-23T20:49:03.760578+0000","last_became_active":"2023-09-23T20:38:02.541575+0000","last_became_peered":"2023-09-23T20:38:02.541575+0000","last_unstale":"2023-09-23T20:49:03.760578+0000","last_undegraded":"2023-09-23T20:49:03.760578+0000","last_fullsized":"2023-09-23T20:49:03.760578+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:50:54.591529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"90'161","reported_seq":736,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T20:49:10.809740+0000","last_change":"2023-09-23T20:44:14.669973+0000","last_active":"2023-09-23T20:49:10.809740+0000","last_peered":"2023-09-23T20:49:10.809740+0000","last_clean":"2023-09-23T20:49:10.809740+0000","last_became_active":"2023-09-23T20:43:46.519592+0000","last_became_peered":"2023-09-23T20:43:46.519592+0000","last_unstale":"2023-09-23T20:49:10.809740+0000","last_undegraded":"2023-09-23T20:49:10.809740+0000","last_fullsized":"2023-09-23T20:49:10.809740+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:14.669831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:14.669831+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:56:28.076617+0000","scrub_duration":0.0083441460000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":666,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T20:49:08.829848+0000","last_change":"2023-09-23T20:47:10.837423+0000","last_active":"2023-09-23T20:49:08.829848+0000","last_peered":"2023-09-23T20:49:08.829848+0000","last_clean":"2023-09-23T20:49:08.829848+0000","last_became_active":"2023-09-23T20:47:09.204324+0000","last_became_peered":"2023-09-23T20:47:09.204324+0000","last_unstale":"2023-09-23T20:49:08.829848+0000","last_undegraded":"2023-09-23T20:49:08.829848+0000","last_fullsized":"2023-09-23T20:49:08.829848+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":554,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:19.108914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":540,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":144,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T20:49:11.696849+0000","last_change":"2023-09-23T20:48:48.145175+0000","last_active":"2023-09-23T20:49:11.696849+0000","last_peered":"2023-09-23T20:49:11.696849+0000","last_clean":"2023-09-23T20:49:11.696849+0000","last_became_active":"2023-09-23T20:48:48.144841+0000","last_became_peered":"2023-09-23T20:48:48.144841+0000","last_unstale":"2023-09-23T20:49:11.696849+0000","last_undegraded":"2023-09-23T20:49:11.696849+0000","last_fullsized":"2023-09-23T20:49:11.696849+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:27:40.802917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"90'161","reported_seq":737,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T20:49:11.698179+0000","last_change":"2023-09-23T20:36:02.548387+0000","last_active":"2023-09-23T20:49:11.698179+0000","last_peered":"2023-09-23T20:49:11.698179+0000","last_clean":"2023-09-23T20:49:11.698179+0000","last_became_active":"2023-09-23T20:35:26.644787+0000","last_became_peered":"2023-09-23T20:35:26.644787+0000","last_unstale":"2023-09-23T20:49:11.698179+0000","last_undegraded":"2023-09-23T20:49:11.698179+0000","last_fullsized":"2023-09-23T20:49:11.698179+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:02.548285+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:02.548285+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:37:15.216553+0000","scrub_duration":0.0015514459999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":75,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T20:49:09.808506+0000","last_change":"2023-09-23T20:47:09.340979+0000","last_active":"2023-09-23T20:49:09.808506+0000","last_peered":"2023-09-23T20:49:09.808506+0000","last_clean":"2023-09-23T20:49:09.808506+0000","last_became_active":"2023-09-23T20:47:09.231222+0000","last_became_peered":"2023-09-23T20:47:09.231222+0000","last_unstale":"2023-09-23T20:49:09.808506+0000","last_undegraded":"2023-09-23T20:49:09.808506+0000","last_fullsized":"2023-09-23T20:49:09.808506+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:07:50.620006+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":65,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T20:49:09.808177+0000","last_change":"2023-09-23T20:47:09.186315+0000","last_active":"2023-09-23T20:49:09.808177+0000","last_peered":"2023-09-23T20:49:09.808177+0000","last_clean":"2023-09-23T20:49:09.808177+0000","last_became_active":"2023-09-23T20:47:09.184580+0000","last_became_peered":"2023-09-23T20:47:09.184580+0000","last_unstale":"2023-09-23T20:49:09.808177+0000","last_undegraded":"2023-09-23T20:49:09.808177+0000","last_fullsized":"2023-09-23T20:49:09.808177+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:44:35.553948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"90'161","reported_seq":864,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T20:49:10.809452+0000","last_change":"2023-09-23T20:43:46.544727+0000","last_active":"2023-09-23T20:49:10.809452+0000","last_peered":"2023-09-23T20:49:10.809452+0000","last_clean":"2023-09-23T20:49:10.809452+0000","last_became_active":"2023-09-23T20:43:46.543952+0000","last_became_peered":"2023-09-23T20:43:46.543952+0000","last_unstale":"2023-09-23T20:49:10.809452+0000","last_undegraded":"2023-09-23T20:49:10.809452+0000","last_fullsized":"2023-09-23T20:49:10.809452+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:43.578666+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:43.578666+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:11:16.611920+0000","scrub_duration":0.0021890379999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":697,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T20:49:08.829176+0000","last_change":"2023-09-23T20:48:48.193666+0000","last_active":"2023-09-23T20:49:08.829176+0000","last_peered":"2023-09-23T20:49:08.829176+0000","last_clean":"2023-09-23T20:49:08.829176+0000","last_became_active":"2023-09-23T20:48:48.152081+0000","last_became_peered":"2023-09-23T20:48:48.152081+0000","last_unstale":"2023-09-23T20:49:08.829176+0000","last_undegraded":"2023-09-23T20:49:08.829176+0000","last_fullsized":"2023-09-23T20:49:08.829176+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:00:19.508124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":272,"num_keys_recovered":80,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":353,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T20:49:03.760832+0000","last_change":"2023-09-23T20:45:01.875931+0000","last_active":"2023-09-23T20:49:03.760832+0000","last_peered":"2023-09-23T20:49:03.760832+0000","last_clean":"2023-09-23T20:49:03.760832+0000","last_became_active":"2023-09-23T20:45:01.872839+0000","last_became_peered":"2023-09-23T20:45:01.872839+0000","last_unstale":"2023-09-23T20:49:03.760832+0000","last_undegraded":"2023-09-23T20:49:03.760832+0000","last_fullsized":"2023-09-23T20:49:03.760832+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:15:09.584507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"90'161","reported_seq":887,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T20:49:08.828889+0000","last_change":"2023-09-23T20:44:20.041766+0000","last_active":"2023-09-23T20:49:08.828889+0000","last_peered":"2023-09-23T20:49:08.828889+0000","last_clean":"2023-09-23T20:49:08.828889+0000","last_became_active":"2023-09-23T20:43:16.765015+0000","last_became_peered":"2023-09-23T20:43:16.765015+0000","last_unstale":"2023-09-23T20:49:08.828889+0000","last_undegraded":"2023-09-23T20:49:08.828889+0000","last_fullsized":"2023-09-23T20:49:08.828889+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:20.041690+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:20.041690+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:33:30.150109+0000","scrub_duration":0.0026180330000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":566,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T20:49:10.809644+0000","last_change":"2023-09-23T20:43:46.521294+0000","last_active":"2023-09-23T20:49:10.809644+0000","last_peered":"2023-09-23T20:49:10.809644+0000","last_clean":"2023-09-23T20:49:10.809644+0000","last_became_active":"2023-09-23T20:43:46.521001+0000","last_became_peered":"2023-09-23T20:43:46.521001+0000","last_unstale":"2023-09-23T20:49:10.809644+0000","last_undegraded":"2023-09-23T20:49:10.809644+0000","last_fullsized":"2023-09-23T20:49:10.809644+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:01:35.126716+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":384,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T20:49:08.829906+0000","last_change":"2023-09-23T20:48:23.914091+0000","last_active":"2023-09-23T20:49:08.829906+0000","last_peered":"2023-09-23T20:49:08.829906+0000","last_clean":"2023-09-23T20:49:08.829906+0000","last_became_active":"2023-09-23T20:48:23.913768+0000","last_became_peered":"2023-09-23T20:48:23.913768+0000","last_unstale":"2023-09-23T20:49:08.829906+0000","last_undegraded":"2023-09-23T20:49:08.829906+0000","last_fullsized":"2023-09-23T20:49:08.829906+0000","mapping_epoch":577,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":578,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:19:38.311290+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.18","version":"609'11049","reported_seq":14514,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T20:49:11.696003+0000","last_change":"2023-09-23T20:48:49.178589+0000","last_active":"2023-09-23T20:49:11.696003+0000","last_peered":"2023-09-23T20:49:11.696003+0000","last_clean":"2023-09-23T20:49:11.696003+0000","last_became_active":"2023-09-23T20:48:49.155693+0000","last_became_peered":"2023-09-23T20:48:49.155693+0000","last_unstale":"2023-09-23T20:49:11.696003+0000","last_undegraded":"2023-09-23T20:49:11.696003+0000","last_fullsized":"2023-09-23T20:49:11.696003+0000","mapping_epoch":592,"log_start":"541'4571","ondisk_log_start":"541'4571","created":584,"last_epoch_clean":593,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":6478,"log_dups_size":0,"ondisk_log_size":6478,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":41320448,"num_objects":631,"num_object_clones":0,"num_object_copies":1893,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":631,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5797,"num_write_kb":205664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":837,"num_bytes_recovered":54820864,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.13","version":"90'161","reported_seq":933,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T20:49:11.697841+0000","last_change":"2023-09-23T20:47:11.619371+0000","last_active":"2023-09-23T20:49:11.697841+0000","last_peered":"2023-09-23T20:49:11.697841+0000","last_clean":"2023-09-23T20:49:11.697841+0000","last_became_active":"2023-09-23T20:47:11.619006+0000","last_became_peered":"2023-09-23T20:47:11.619006+0000","last_unstale":"2023-09-23T20:49:11.697841+0000","last_undegraded":"2023-09-23T20:49:11.697841+0000","last_fullsized":"2023-09-23T20:49:11.697841+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:42.544308+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:42.544308+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:47:24.414807+0000","scrub_duration":0.0029958179999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":630,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T20:49:08.829359+0000","last_change":"2023-09-23T20:47:09.176613+0000","last_active":"2023-09-23T20:49:08.829359+0000","last_peered":"2023-09-23T20:49:08.829359+0000","last_clean":"2023-09-23T20:49:08.829359+0000","last_became_active":"2023-09-23T20:47:09.176187+0000","last_became_peered":"2023-09-23T20:47:09.176187+0000","last_unstale":"2023-09-23T20:49:08.829359+0000","last_undegraded":"2023-09-23T20:49:08.829359+0000","last_fullsized":"2023-09-23T20:49:08.829359+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:23:01.590989+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":575,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T20:49:10.810330+0000","last_change":"2023-09-23T20:43:46.521735+0000","last_active":"2023-09-23T20:49:10.810330+0000","last_peered":"2023-09-23T20:49:10.810330+0000","last_clean":"2023-09-23T20:49:10.810330+0000","last_became_active":"2023-09-23T20:43:46.520762+0000","last_became_peered":"2023-09-23T20:43:46.520762+0000","last_unstale":"2023-09-23T20:49:10.810330+0000","last_undegraded":"2023-09-23T20:49:10.810330+0000","last_fullsized":"2023-09-23T20:49:10.810330+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:26:48.841930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.19","version":"610'11326","reported_seq":11452,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T20:49:13.254963+0000","last_change":"2023-09-23T20:48:49.228617+0000","last_active":"2023-09-23T20:49:13.254963+0000","last_peered":"2023-09-23T20:49:13.254963+0000","last_clean":"2023-09-23T20:49:13.254963+0000","last_became_active":"2023-09-23T20:48:49.188310+0000","last_became_peered":"2023-09-23T20:48:49.188310+0000","last_unstale":"2023-09-23T20:49:13.254963+0000","last_undegraded":"2023-09-23T20:49:13.254963+0000","last_fullsized":"2023-09-23T20:49:13.254963+0000","mapping_epoch":592,"log_start":"541'4623","ondisk_log_start":"541'4623","created":584,"last_epoch_clean":593,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":6703,"log_dups_size":0,"ondisk_log_size":6703,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":42958848,"num_objects":656,"num_object_clones":0,"num_object_copies":1968,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":656,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5948,"num_write_kb":211296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.12","version":"90'161","reported_seq":695,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T20:49:11.697371+0000","last_change":"2023-09-23T20:44:18.952808+0000","last_active":"2023-09-23T20:49:11.697371+0000","last_peered":"2023-09-23T20:49:11.697371+0000","last_clean":"2023-09-23T20:49:11.697371+0000","last_became_active":"2023-09-23T20:43:16.758652+0000","last_became_peered":"2023-09-23T20:43:16.758652+0000","last_unstale":"2023-09-23T20:49:11.697371+0000","last_undegraded":"2023-09-23T20:49:11.697371+0000","last_fullsized":"2023-09-23T20:49:11.697371+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:18.952693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:18.952693+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:12:19.496465+0000","scrub_duration":0.001699365,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":282,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T20:49:08.812562+0000","last_change":"2023-09-23T20:44:43.644290+0000","last_active":"2023-09-23T20:49:08.812562+0000","last_peered":"2023-09-23T20:49:08.812562+0000","last_clean":"2023-09-23T20:49:08.812562+0000","last_became_active":"2023-09-23T20:44:43.638313+0000","last_became_peered":"2023-09-23T20:44:43.638313+0000","last_unstale":"2023-09-23T20:49:08.812562+0000","last_undegraded":"2023-09-23T20:49:08.812562+0000","last_fullsized":"2023-09-23T20:49:08.812562+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:32:52.892603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":402,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T20:49:03.752880+0000","last_change":"2023-09-23T20:37:43.571083+0000","last_active":"2023-09-23T20:49:03.752880+0000","last_peered":"2023-09-23T20:49:03.752880+0000","last_clean":"2023-09-23T20:49:03.752880+0000","last_became_active":"2023-09-23T20:37:43.570306+0000","last_became_peered":"2023-09-23T20:37:43.570306+0000","last_unstale":"2023-09-23T20:49:03.752880+0000","last_undegraded":"2023-09-23T20:49:03.752880+0000","last_fullsized":"2023-09-23T20:49:03.752880+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:42:02.396718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.6","version":"605'11090","reported_seq":11790,"reported_epoch":605,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T20:49:06.758775+0000","last_change":"2023-09-23T20:49:03.816698+0000","last_active":"2023-09-23T20:49:06.758775+0000","last_peered":"2023-09-23T20:49:06.758775+0000","last_clean":"2023-09-23T20:49:01.603059+0000","last_became_active":"2023-09-23T20:49:03.771882+0000","last_became_peered":"2023-09-23T20:49:03.771882+0000","last_unstale":"2023-09-23T20:49:06.758775+0000","last_undegraded":"2023-09-23T20:49:06.758775+0000","last_fullsized":"2023-09-23T20:49:06.758775+0000","mapping_epoch":604,"log_start":"541'4690","ondisk_log_start":"541'4690","created":531,"last_epoch_clean":585,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":6400,"log_dups_size":0,"ondisk_log_size":6400,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":51937280,"num_objects":792,"num_object_clones":0,"num_object_copies":2376,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":792,"num_objects_unfound":0,"num_objects_dirty":792,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5769,"num_write_kb":209984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":143,"num_bytes_recovered":9404416,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[0,5,4],"avail_no_missing":["0","4","5"],"object_location_counts":[{"shards":"0,4,5","objects":792}],"blocked_by":[],"up_primary":7,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.d","version":"90'161","reported_seq":939,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T20:49:08.829606+0000","last_change":"2023-09-23T20:47:11.619510+0000","last_active":"2023-09-23T20:49:08.829606+0000","last_peered":"2023-09-23T20:49:08.829606+0000","last_clean":"2023-09-23T20:49:08.829606+0000","last_became_active":"2023-09-23T20:47:11.619154+0000","last_became_peered":"2023-09-23T20:47:11.619154+0000","last_unstale":"2023-09-23T20:49:08.829606+0000","last_undegraded":"2023-09-23T20:49:08.829606+0000","last_fullsized":"2023-09-23T20:49:08.829606+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:34:59.496375+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:34:59.496375+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:40:52.712777+0000","scrub_duration":0.0017843189999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":638,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T20:49:08.829888+0000","last_change":"2023-09-23T20:47:09.207851+0000","last_active":"2023-09-23T20:49:08.829888+0000","last_peered":"2023-09-23T20:49:08.829888+0000","last_clean":"2023-09-23T20:49:08.829888+0000","last_became_active":"2023-09-23T20:47:09.175836+0000","last_became_peered":"2023-09-23T20:47:09.175836+0000","last_unstale":"2023-09-23T20:49:08.829888+0000","last_undegraded":"2023-09-23T20:49:08.829888+0000","last_fullsized":"2023-09-23T20:49:08.829888+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:43:13.091809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":421,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T20:49:08.829210+0000","last_change":"2023-09-23T20:43:16.765253+0000","last_active":"2023-09-23T20:49:08.829210+0000","last_peered":"2023-09-23T20:49:08.829210+0000","last_clean":"2023-09-23T20:49:08.829210+0000","last_became_active":"2023-09-23T20:43:16.764978+0000","last_became_peered":"2023-09-23T20:43:16.764978+0000","last_unstale":"2023-09-23T20:49:08.829210+0000","last_undegraded":"2023-09-23T20:49:08.829210+0000","last_fullsized":"2023-09-23T20:49:08.829210+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:07:23.539372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.7","version":"607'11061","reported_seq":11162,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T20:49:10.044455+0000","last_change":"2023-09-23T20:49:09.234110+0000","last_active":"2023-09-23T20:49:10.044455+0000","last_peered":"2023-09-23T20:49:10.044455+0000","last_clean":"2023-09-23T20:49:10.044455+0000","last_became_active":"2023-09-23T20:48:40.067198+0000","last_became_peered":"2023-09-23T20:48:40.067198+0000","last_unstale":"2023-09-23T20:49:10.044455+0000","last_undegraded":"2023-09-23T20:49:10.044455+0000","last_fullsized":"2023-09-23T20:49:10.044455+0000","mapping_epoch":531,"log_start":"542'4863","ondisk_log_start":"542'4863","created":531,"last_epoch_clean":585,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":6198,"log_dups_size":0,"ondisk_log_size":6198,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":45187072,"num_objects":689,"num_object_clones":0,"num_object_copies":2067,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":689,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5797,"num_write_kb":207584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.c","version":"90'161","reported_seq":869,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T20:49:03.760974+0000","last_change":"2023-09-23T20:41:09.506481+0000","last_active":"2023-09-23T20:49:03.760974+0000","last_peered":"2023-09-23T20:49:03.760974+0000","last_clean":"2023-09-23T20:49:03.760974+0000","last_became_active":"2023-09-23T20:38:28.442229+0000","last_became_peered":"2023-09-23T20:38:28.442229+0000","last_unstale":"2023-09-23T20:49:03.760974+0000","last_undegraded":"2023-09-23T20:49:03.760974+0000","last_fullsized":"2023-09-23T20:49:03.760974+0000","mapping_epoch":411,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":412,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:41:09.506338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:41:09.506338+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:19:46.265011+0000","scrub_duration":0.01112002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":676,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T20:49:08.830169+0000","last_change":"2023-09-23T20:47:10.848436+0000","last_active":"2023-09-23T20:49:08.830169+0000","last_peered":"2023-09-23T20:49:08.830169+0000","last_clean":"2023-09-23T20:49:08.830169+0000","last_became_active":"2023-09-23T20:47:09.196769+0000","last_became_peered":"2023-09-23T20:47:09.196769+0000","last_unstale":"2023-09-23T20:49:08.830169+0000","last_undegraded":"2023-09-23T20:49:08.830169+0000","last_fullsized":"2023-09-23T20:49:08.830169+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":554,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:59:38.105009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":75,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T20:49:09.807754+0000","last_change":"2023-09-23T20:48:23.915817+0000","last_active":"2023-09-23T20:49:09.807754+0000","last_peered":"2023-09-23T20:49:09.807754+0000","last_clean":"2023-09-23T20:49:09.807754+0000","last_became_active":"2023-09-23T20:48:23.915306+0000","last_became_peered":"2023-09-23T20:48:23.915306+0000","last_unstale":"2023-09-23T20:49:09.807754+0000","last_undegraded":"2023-09-23T20:49:09.807754+0000","last_fullsized":"2023-09-23T20:49:09.807754+0000","mapping_epoch":577,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":578,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:55:45.691450+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"608'11001","reported_seq":11673,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T20:49:11.351527+0000","last_change":"2023-09-23T20:48:40.087695+0000","last_active":"2023-09-23T20:49:11.351527+0000","last_peered":"2023-09-23T20:49:11.351527+0000","last_clean":"2023-09-23T20:49:11.351527+0000","last_became_active":"2023-09-23T20:48:40.070957+0000","last_became_peered":"2023-09-23T20:48:40.070957+0000","last_unstale":"2023-09-23T20:49:11.351527+0000","last_undegraded":"2023-09-23T20:49:11.351527+0000","last_fullsized":"2023-09-23T20:49:11.351527+0000","mapping_epoch":558,"log_start":"541'4547","ondisk_log_start":"541'4547","created":531,"last_epoch_clean":585,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":6454,"log_dups_size":0,"ondisk_log_size":6454,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":46596096,"num_objects":711,"num_object_clones":0,"num_object_copies":2133,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":711,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5752,"num_write_kb":206816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":137,"num_bytes_recovered":8978432,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"90'161","reported_seq":802,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T20:49:08.815613+0000","last_change":"2023-09-23T20:47:11.622303+0000","last_active":"2023-09-23T20:49:08.815613+0000","last_peered":"2023-09-23T20:49:08.815613+0000","last_clean":"2023-09-23T20:49:08.815613+0000","last_became_active":"2023-09-23T20:47:11.621953+0000","last_became_peered":"2023-09-23T20:47:11.621953+0000","last_unstale":"2023-09-23T20:49:08.815613+0000","last_undegraded":"2023-09-23T20:49:08.815613+0000","last_fullsized":"2023-09-23T20:49:08.815613+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:46.217489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:46.217489+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:14:40.478522+0000","scrub_duration":0.001456429,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":635,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T20:49:08.829700+0000","last_change":"2023-09-23T20:40:19.677655+0000","last_active":"2023-09-23T20:49:08.829700+0000","last_peered":"2023-09-23T20:49:08.829700+0000","last_clean":"2023-09-23T20:49:08.829700+0000","last_became_active":"2023-09-23T20:40:19.551189+0000","last_became_peered":"2023-09-23T20:40:19.551189+0000","last_unstale":"2023-09-23T20:49:08.829700+0000","last_undegraded":"2023-09-23T20:49:08.829700+0000","last_fullsized":"2023-09-23T20:49:08.829700+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:11:01.559576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":222,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T20:49:08.813661+0000","last_change":"2023-09-23T20:49:02.702301+0000","last_active":"2023-09-23T20:49:08.813661+0000","last_peered":"2023-09-23T20:49:08.813661+0000","last_clean":"2023-09-23T20:49:08.813661+0000","last_became_active":"2023-09-23T20:49:02.701986+0000","last_became_peered":"2023-09-23T20:49:02.701986+0000","last_unstale":"2023-09-23T20:49:08.813661+0000","last_undegraded":"2023-09-23T20:49:08.813661+0000","last_fullsized":"2023-09-23T20:49:08.813661+0000","mapping_epoch":603,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":604,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:20:01.603267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.5","version":"610'11274","reported_seq":14091,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T20:49:13.307144+0000","last_change":"2023-09-23T20:48:53.196448+0000","last_active":"2023-09-23T20:49:13.307144+0000","last_peered":"2023-09-23T20:49:13.307144+0000","last_clean":"2023-09-23T20:49:13.307144+0000","last_became_active":"2023-09-23T20:48:53.175613+0000","last_became_peered":"2023-09-23T20:48:53.175613+0000","last_unstale":"2023-09-23T20:49:13.307144+0000","last_undegraded":"2023-09-23T20:49:13.307144+0000","last_fullsized":"2023-09-23T20:49:13.307144+0000","mapping_epoch":596,"log_start":"542'4806","ondisk_log_start":"542'4806","created":531,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":6468,"log_dups_size":0,"ondisk_log_size":6468,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":43810816,"num_objects":668,"num_object_clones":0,"num_object_copies":2004,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":668,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5923,"num_write_kb":210944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1338,"num_bytes_recovered":87687168,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.e","version":"90'161","reported_seq":638,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T20:49:09.808631+0000","last_change":"2023-09-23T20:47:37.608426+0000","last_active":"2023-09-23T20:49:09.808631+0000","last_peered":"2023-09-23T20:49:09.808631+0000","last_clean":"2023-09-23T20:49:09.808631+0000","last_became_active":"2023-09-23T20:47:11.617187+0000","last_became_peered":"2023-09-23T20:47:11.617187+0000","last_unstale":"2023-09-23T20:49:09.808631+0000","last_undegraded":"2023-09-23T20:49:09.808631+0000","last_fullsized":"2023-09-23T20:49:09.808631+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:47:37.608309+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:47:37.608309+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:22:25.482297+0000","scrub_duration":0.011600924,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":567,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T20:49:03.761251+0000","last_change":"2023-09-23T20:43:16.756437+0000","last_active":"2023-09-23T20:49:03.761251+0000","last_peered":"2023-09-23T20:49:03.761251+0000","last_clean":"2023-09-23T20:49:03.761251+0000","last_became_active":"2023-09-23T20:43:16.756193+0000","last_became_peered":"2023-09-23T20:43:16.756193+0000","last_unstale":"2023-09-23T20:49:03.761251+0000","last_undegraded":"2023-09-23T20:49:03.761251+0000","last_fullsized":"2023-09-23T20:49:03.761251+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:33:25.632781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":602,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T20:49:10.809652+0000","last_change":"2023-09-23T20:47:09.197427+0000","last_active":"2023-09-23T20:49:10.809652+0000","last_peered":"2023-09-23T20:49:10.809652+0000","last_clean":"2023-09-23T20:49:10.809652+0000","last_became_active":"2023-09-23T20:47:09.196542+0000","last_became_peered":"2023-09-23T20:47:09.196542+0000","last_unstale":"2023-09-23T20:49:10.809652+0000","last_undegraded":"2023-09-23T20:49:10.809652+0000","last_fullsized":"2023-09-23T20:49:10.809652+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:28:38.926202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"610'11437","reported_seq":14028,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T20:49:13.571331+0000","last_change":"2023-09-23T20:49:09.840838+0000","last_active":"2023-09-23T20:49:13.571331+0000","last_peered":"2023-09-23T20:49:13.571331+0000","last_clean":"2023-09-23T20:49:13.571331+0000","last_became_active":"2023-09-23T20:49:09.813951+0000","last_became_peered":"2023-09-23T20:49:09.813951+0000","last_unstale":"2023-09-23T20:49:13.571331+0000","last_undegraded":"2023-09-23T20:49:13.571331+0000","last_fullsized":"2023-09-23T20:49:13.571331+0000","mapping_epoch":606,"log_start":"542'5074","ondisk_log_start":"542'5074","created":531,"last_epoch_clean":607,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":6363,"log_dups_size":0,"ondisk_log_size":6363,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":41549824,"num_objects":634,"num_object_clones":0,"num_object_copies":1902,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":634,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6008,"num_write_kb":212544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":978,"num_bytes_recovered":64126976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":1052,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T20:49:08.828999+0000","last_change":"2023-09-23T20:48:51.169474+0000","last_active":"2023-09-23T20:49:08.828999+0000","last_peered":"2023-09-23T20:49:08.828999+0000","last_clean":"2023-09-23T20:49:08.828999+0000","last_became_active":"2023-09-23T20:48:51.169076+0000","last_became_peered":"2023-09-23T20:48:51.169076+0000","last_unstale":"2023-09-23T20:49:08.828999+0000","last_undegraded":"2023-09-23T20:49:08.828999+0000","last_fullsized":"2023-09-23T20:49:08.828999+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:35:02.523203+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:35:02.523203+0000","last_clean_scrub_stamp":"2023-09-23T20:35:02.523203+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:25:53.741403+0000","scrub_duration":0.0085927989999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":455,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T20:49:03.753487+0000","last_change":"2023-09-23T20:47:09.240200+0000","last_active":"2023-09-23T20:49:03.753487+0000","last_peered":"2023-09-23T20:49:03.753487+0000","last_clean":"2023-09-23T20:49:03.753487+0000","last_became_active":"2023-09-23T20:47:09.195204+0000","last_became_peered":"2023-09-23T20:47:09.195204+0000","last_unstale":"2023-09-23T20:49:03.753487+0000","last_undegraded":"2023-09-23T20:49:03.753487+0000","last_fullsized":"2023-09-23T20:49:03.753487+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:03.328250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":65,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T20:49:09.808156+0000","last_change":"2023-09-23T20:47:09.231714+0000","last_active":"2023-09-23T20:49:09.808156+0000","last_peered":"2023-09-23T20:49:09.808156+0000","last_clean":"2023-09-23T20:49:09.808156+0000","last_became_active":"2023-09-23T20:47:09.230918+0000","last_became_peered":"2023-09-23T20:47:09.230918+0000","last_unstale":"2023-09-23T20:49:09.808156+0000","last_undegraded":"2023-09-23T20:49:09.808156+0000","last_fullsized":"2023-09-23T20:49:09.808156+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:24.583656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"610'11810","reported_seq":12413,"reported_epoch":610,"state":"active+clean+scrubbing+deep","last_fresh":"2023-09-23T20:49:13.333795+0000","last_change":"2023-09-23T20:49:11.578276+0000","last_active":"2023-09-23T20:49:13.333795+0000","last_peered":"2023-09-23T20:49:13.333795+0000","last_clean":"2023-09-23T20:49:13.333795+0000","last_became_active":"2023-09-23T20:48:48.159849+0000","last_became_peered":"2023-09-23T20:48:48.159849+0000","last_unstale":"2023-09-23T20:49:13.333795+0000","last_undegraded":"2023-09-23T20:49:13.333795+0000","last_fullsized":"2023-09-23T20:49:13.333795+0000","mapping_epoch":591,"log_start":"536'1300","ondisk_log_start":"536'1300","created":531,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":380,"log_size":10510,"log_dups_size":0,"ondisk_log_size":10510,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"deep scrubbing for 1s","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":43057152,"num_objects":657,"num_object_clones":0,"num_object_copies":1971,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":657,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6017,"num_write_kb":213568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":118,"num_bytes_recovered":7733248,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":485,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T20:49:11.700853+0000","last_change":"2023-09-23T20:48:51.176533+0000","last_active":"2023-09-23T20:49:11.700853+0000","last_peered":"2023-09-23T20:49:11.700853+0000","last_clean":"2023-09-23T20:49:11.700853+0000","last_became_active":"2023-09-23T20:48:51.175911+0000","last_became_peered":"2023-09-23T20:48:51.175911+0000","last_unstale":"2023-09-23T20:49:11.700853+0000","last_undegraded":"2023-09-23T20:49:11.700853+0000","last_fullsized":"2023-09-23T20:49:11.700853+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:42:45.492220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:42:45.492220+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":232,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T20:49:08.813733+0000","last_change":"2023-09-23T20:47:09.178919+0000","last_active":"2023-09-23T20:49:08.813733+0000","last_peered":"2023-09-23T20:49:08.813733+0000","last_clean":"2023-09-23T20:49:08.813733+0000","last_became_active":"2023-09-23T20:47:09.178607+0000","last_became_peered":"2023-09-23T20:47:09.178607+0000","last_unstale":"2023-09-23T20:49:08.813733+0000","last_undegraded":"2023-09-23T20:49:08.813733+0000","last_fullsized":"2023-09-23T20:49:08.813733+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:23:45.196935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":85,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T20:49:09.808329+0000","last_change":"2023-09-23T20:48:48.148596+0000","last_active":"2023-09-23T20:49:09.808329+0000","last_peered":"2023-09-23T20:49:09.808329+0000","last_clean":"2023-09-23T20:49:09.808329+0000","last_became_active":"2023-09-23T20:48:48.148211+0000","last_became_peered":"2023-09-23T20:48:48.148211+0000","last_unstale":"2023-09-23T20:49:09.808329+0000","last_undegraded":"2023-09-23T20:49:09.808329+0000","last_fullsized":"2023-09-23T20:49:09.808329+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:21:50.825822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.0","version":"607'11129","reported_seq":12411,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T20:49:10.069706+0000","last_change":"2023-09-23T20:48:40.083829+0000","last_active":"2023-09-23T20:49:10.069706+0000","last_peered":"2023-09-23T20:49:10.069706+0000","last_clean":"2023-09-23T20:49:10.069706+0000","last_became_active":"2023-09-23T20:48:40.067403+0000","last_became_peered":"2023-09-23T20:48:40.067403+0000","last_unstale":"2023-09-23T20:49:10.069706+0000","last_undegraded":"2023-09-23T20:49:10.069706+0000","last_fullsized":"2023-09-23T20:49:10.069706+0000","mapping_epoch":559,"log_start":"542'4945","ondisk_log_start":"542'4945","created":531,"last_epoch_clean":585,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":6184,"log_dups_size":0,"ondisk_log_size":6184,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":44007424,"num_objects":671,"num_object_clones":0,"num_object_copies":2013,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":671,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5823,"num_write_kb":207840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":289,"num_bytes_recovered":18939904,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.b","version":"90'161","reported_seq":915,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T20:49:11.697917+0000","last_change":"2023-09-23T20:40:22.445808+0000","last_active":"2023-09-23T20:49:11.697917+0000","last_peered":"2023-09-23T20:49:11.697917+0000","last_clean":"2023-09-23T20:49:11.697917+0000","last_became_active":"2023-09-23T20:40:22.445456+0000","last_became_peered":"2023-09-23T20:40:22.445456+0000","last_unstale":"2023-09-23T20:49:11.697917+0000","last_undegraded":"2023-09-23T20:49:11.697917+0000","last_fullsized":"2023-09-23T20:49:11.697917+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:05.595042+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:05.595042+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:39:23.678154+0000","scrub_duration":0.007271675,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":85,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T20:49:09.807667+0000","last_change":"2023-09-23T20:48:48.153145+0000","last_active":"2023-09-23T20:49:09.807667+0000","last_peered":"2023-09-23T20:49:09.807667+0000","last_clean":"2023-09-23T20:49:09.807667+0000","last_became_active":"2023-09-23T20:48:48.152858+0000","last_became_peered":"2023-09-23T20:48:48.152858+0000","last_unstale":"2023-09-23T20:49:09.807667+0000","last_undegraded":"2023-09-23T20:49:09.807667+0000","last_fullsized":"2023-09-23T20:49:09.807667+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:23:13.454288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":65,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T20:49:09.808633+0000","last_change":"2023-09-23T20:47:09.186271+0000","last_active":"2023-09-23T20:49:09.808633+0000","last_peered":"2023-09-23T20:49:09.808633+0000","last_clean":"2023-09-23T20:49:09.808633+0000","last_became_active":"2023-09-23T20:47:09.184357+0000","last_became_peered":"2023-09-23T20:47:09.184357+0000","last_unstale":"2023-09-23T20:49:09.808633+0000","last_undegraded":"2023-09-23T20:49:09.808633+0000","last_fullsized":"2023-09-23T20:49:09.808633+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:48:26.339811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"606'11243","reported_seq":12981,"reported_epoch":608,"state":"active","last_fresh":"2023-09-23T20:49:10.816682+0000","last_change":"2023-09-23T20:49:10.816682+0000","last_active":"2023-09-23T20:49:10.816682+0000","last_peered":"2023-09-23T20:49:10.816682+0000","last_clean":"2023-09-23T20:49:09.436929+0000","last_became_active":"2023-09-23T20:49:10.816682+0000","last_became_peered":"2023-09-23T20:49:10.816682+0000","last_unstale":"2023-09-23T20:49:10.816682+0000","last_undegraded":"2023-09-23T20:49:10.816682+0000","last_fullsized":"2023-09-23T20:49:10.816682+0000","mapping_epoch":607,"log_start":"543'5336","ondisk_log_start":"543'5336","created":531,"last_epoch_clean":606,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":5907,"log_dups_size":0,"ondisk_log_size":5907,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":42598400,"num_objects":650,"num_object_clones":0,"num_object_copies":1950,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":650,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5902,"num_write_kb":209664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":795,"num_bytes_recovered":52101120,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":["4","0","3","7"],"object_location_counts":[{"shards":"3,4,7","objects":650}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":489,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T20:49:11.701146+0000","last_change":"2023-09-23T20:48:51.177368+0000","last_active":"2023-09-23T20:49:11.701146+0000","last_peered":"2023-09-23T20:49:11.701146+0000","last_clean":"2023-09-23T20:49:11.701146+0000","last_became_active":"2023-09-23T20:48:51.177024+0000","last_became_peered":"2023-09-23T20:48:51.177024+0000","last_unstale":"2023-09-23T20:49:11.701146+0000","last_undegraded":"2023-09-23T20:49:11.701146+0000","last_fullsized":"2023-09-23T20:49:11.701146+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:43:51.693459+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:43:51.693459+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26834,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":403,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T20:49:03.753418+0000","last_change":"2023-09-23T20:37:43.570610+0000","last_active":"2023-09-23T20:49:03.753418+0000","last_peered":"2023-09-23T20:49:03.753418+0000","last_clean":"2023-09-23T20:49:03.753418+0000","last_became_active":"2023-09-23T20:37:43.570216+0000","last_became_peered":"2023-09-23T20:37:43.570216+0000","last_unstale":"2023-09-23T20:49:03.753418+0000","last_undegraded":"2023-09-23T20:49:03.753418+0000","last_fullsized":"2023-09-23T20:49:03.753418+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:39:26.555329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":455,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T20:49:11.698199+0000","last_change":"2023-09-23T20:47:09.172573+0000","last_active":"2023-09-23T20:49:11.698199+0000","last_peered":"2023-09-23T20:49:11.698199+0000","last_clean":"2023-09-23T20:49:11.698199+0000","last_became_active":"2023-09-23T20:47:09.172298+0000","last_became_peered":"2023-09-23T20:47:09.172298+0000","last_unstale":"2023-09-23T20:49:11.698199+0000","last_undegraded":"2023-09-23T20:49:11.698199+0000","last_fullsized":"2023-09-23T20:49:11.698199+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:22:09.361284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"607'11879","reported_seq":14770,"reported_epoch":607,"state":"active+remapped+backfilling","last_fresh":"2023-09-23T20:49:10.388374+0000","last_change":"2023-09-23T20:49:03.911059+0000","last_active":"2023-09-23T20:49:10.388374+0000","last_peered":"2023-09-23T20:49:10.388374+0000","last_clean":"2023-09-23T20:49:01.672682+0000","last_became_active":"2023-09-23T20:49:03.838428+0000","last_became_peered":"2023-09-23T20:49:03.838428+0000","last_unstale":"2023-09-23T20:49:10.388374+0000","last_undegraded":"2023-09-23T20:49:10.388374+0000","last_fullsized":"2023-09-23T20:49:10.388374+0000","mapping_epoch":604,"log_start":"551'6346","ondisk_log_start":"551'6346","created":531,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":5533,"log_dups_size":0,"ondisk_log_size":5533,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:43:54.254451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":89718784,"num_objects":1369,"num_object_clones":0,"num_object_copies":4107,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":359,"num_objects_unfound":0,"num_objects_dirty":1369,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11879,"num_write_kb":423936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1383,"num_bytes_recovered":90636288,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,0],"avail_no_missing":["7","0","4"],"object_location_counts":[{"shards":"0,4,7","objects":1369}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":795,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T20:49:08.815531+0000","last_change":"2023-09-23T20:41:07.424609+0000","last_active":"2023-09-23T20:49:08.815531+0000","last_peered":"2023-09-23T20:49:08.815531+0000","last_clean":"2023-09-23T20:49:08.815531+0000","last_became_active":"2023-09-23T20:38:05.622261+0000","last_became_peered":"2023-09-23T20:38:05.622261+0000","last_unstale":"2023-09-23T20:49:08.815531+0000","last_undegraded":"2023-09-23T20:49:08.815531+0000","last_fullsized":"2023-09-23T20:49:08.815531+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:41:07.424492+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:41:07.424492+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:45:21.689123+0000","scrub_duration":0.0018960959999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"86'119","reported_seq":922,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T20:49:11.697232+0000","last_change":"2023-09-23T20:29:45.521257+0000","last_active":"2023-09-23T20:49:11.697232+0000","last_peered":"2023-09-23T20:49:11.697232+0000","last_clean":"2023-09-23T20:49:11.697232+0000","last_became_active":"2023-09-23T20:29:45.520546+0000","last_became_peered":"2023-09-23T20:29:45.520546+0000","last_unstale":"2023-09-23T20:49:11.697232+0000","last_undegraded":"2023-09-23T20:49:11.697232+0000","last_fullsized":"2023-09-23T20:49:11.697232+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":119,"log_dups_size":0,"ondisk_log_size":119,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:58:56.085777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032705499999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":523,"num_read_kb":3503,"num_write":136,"num_write_kb":3128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":418,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T20:49:03.754556+0000","last_change":"2023-09-23T20:48:48.157475+0000","last_active":"2023-09-23T20:49:03.754556+0000","last_peered":"2023-09-23T20:49:03.754556+0000","last_clean":"2023-09-23T20:49:03.754556+0000","last_became_active":"2023-09-23T20:48:48.155904+0000","last_became_peered":"2023-09-23T20:48:48.155904+0000","last_unstale":"2023-09-23T20:49:03.754556+0000","last_undegraded":"2023-09-23T20:49:03.754556+0000","last_fullsized":"2023-09-23T20:49:03.754556+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:39:08.642474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":587,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T20:49:11.696988+0000","last_change":"2023-09-23T20:47:09.194947+0000","last_active":"2023-09-23T20:49:11.696988+0000","last_peered":"2023-09-23T20:49:11.696988+0000","last_clean":"2023-09-23T20:49:11.696988+0000","last_became_active":"2023-09-23T20:47:09.194592+0000","last_became_peered":"2023-09-23T20:47:09.194592+0000","last_unstale":"2023-09-23T20:49:11.696988+0000","last_undegraded":"2023-09-23T20:49:11.696988+0000","last_fullsized":"2023-09-23T20:49:11.696988+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:34:15.008492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"607'11416","reported_seq":11500,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T20:49:10.093637+0000","last_change":"2023-09-23T20:45:39.489110+0000","last_active":"2023-09-23T20:49:10.093637+0000","last_peered":"2023-09-23T20:49:10.093637+0000","last_clean":"2023-09-23T20:49:10.093637+0000","last_became_active":"2023-09-23T20:45:39.488817+0000","last_became_peered":"2023-09-23T20:45:39.488817+0000","last_unstale":"2023-09-23T20:49:10.093637+0000","last_undegraded":"2023-09-23T20:49:10.093637+0000","last_fullsized":"2023-09-23T20:49:10.093637+0000","mapping_epoch":531,"log_start":"543'5390","ondisk_log_start":"543'5390","created":531,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":6026,"log_dups_size":0,"ondisk_log_size":6026,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:07:21.467971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":86638592,"num_objects":1322,"num_object_clones":0,"num_object_copies":3966,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1322,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11416,"num_write_kb":407616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":690,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T20:49:10.809557+0000","last_change":"2023-09-23T20:48:51.178609+0000","last_active":"2023-09-23T20:49:10.809557+0000","last_peered":"2023-09-23T20:49:10.809557+0000","last_clean":"2023-09-23T20:49:10.809557+0000","last_became_active":"2023-09-23T20:48:51.178196+0000","last_became_peered":"2023-09-23T20:48:51.178196+0000","last_unstale":"2023-09-23T20:49:10.809557+0000","last_undegraded":"2023-09-23T20:49:10.809557+0000","last_fullsized":"2023-09-23T20:49:10.809557+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:15.656880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:15.656880+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:47:54.557002+0000","scrub_duration":0.0016711180000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"86'106","reported_seq":862,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T20:49:11.697227+0000","last_change":"2023-09-23T20:43:16.759138+0000","last_active":"2023-09-23T20:49:11.697227+0000","last_peered":"2023-09-23T20:49:11.697227+0000","last_clean":"2023-09-23T20:49:11.697227+0000","last_became_active":"2023-09-23T20:43:16.758015+0000","last_became_peered":"2023-09-23T20:43:16.758015+0000","last_unstale":"2023-09-23T20:49:11.697227+0000","last_undegraded":"2023-09-23T20:49:11.697227+0000","last_fullsized":"2023-09-23T20:49:11.697227+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":106,"log_dups_size":0,"ondisk_log_size":106,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:08:02.852472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00099302599999999993,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1094,"num_read_kb":1678,"num_write":100,"num_write_kb":404,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":65,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T20:49:09.808308+0000","last_change":"2023-09-23T20:47:09.177872+0000","last_active":"2023-09-23T20:49:09.808308+0000","last_peered":"2023-09-23T20:49:09.808308+0000","last_clean":"2023-09-23T20:49:09.808308+0000","last_became_active":"2023-09-23T20:47:09.176689+0000","last_became_peered":"2023-09-23T20:47:09.176689+0000","last_unstale":"2023-09-23T20:49:09.808308+0000","last_undegraded":"2023-09-23T20:49:09.808308+0000","last_fullsized":"2023-09-23T20:49:09.808308+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:14:01.590318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":593,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T20:49:11.697136+0000","last_change":"2023-09-23T20:48:48.151144+0000","last_active":"2023-09-23T20:49:11.697136+0000","last_peered":"2023-09-23T20:49:11.697136+0000","last_clean":"2023-09-23T20:49:11.697136+0000","last_became_active":"2023-09-23T20:48:48.150770+0000","last_became_peered":"2023-09-23T20:48:48.150770+0000","last_unstale":"2023-09-23T20:49:11.697136+0000","last_undegraded":"2023-09-23T20:49:11.697136+0000","last_fullsized":"2023-09-23T20:49:11.697136+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:14:08.549512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.9","version":"610'11347","reported_seq":13470,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T20:49:13.320453+0000","last_change":"2023-09-23T20:48:49.220102+0000","last_active":"2023-09-23T20:49:13.320453+0000","last_peered":"2023-09-23T20:49:13.320453+0000","last_clean":"2023-09-23T20:49:13.320453+0000","last_became_active":"2023-09-23T20:48:49.169141+0000","last_became_peered":"2023-09-23T20:48:49.169141+0000","last_unstale":"2023-09-23T20:49:13.320453+0000","last_undegraded":"2023-09-23T20:49:13.320453+0000","last_fullsized":"2023-09-23T20:49:13.320453+0000","mapping_epoch":592,"log_start":"541'4623","ondisk_log_start":"541'4623","created":531,"last_epoch_clean":593,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":6724,"log_dups_size":0,"ondisk_log_size":6724,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":41582592,"num_objects":634,"num_object_clones":0,"num_object_copies":1902,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":634,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5968,"num_write_kb":211296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1000,"num_bytes_recovered":65404928,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":610,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T20:49:08.829588+0000","last_change":"2023-09-23T20:47:11.618652+0000","last_active":"2023-09-23T20:49:08.829588+0000","last_peered":"2023-09-23T20:49:08.829588+0000","last_clean":"2023-09-23T20:49:08.829588+0000","last_became_active":"2023-09-23T20:47:11.618266+0000","last_became_peered":"2023-09-23T20:47:11.618266+0000","last_unstale":"2023-09-23T20:49:08.829588+0000","last_undegraded":"2023-09-23T20:49:08.829588+0000","last_fullsized":"2023-09-23T20:49:08.829588+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:26.978179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:26.978179+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:29:29.000683+0000","scrub_duration":0.0013962829999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"86'130","reported_seq":395,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T20:49:11.701128+0000","last_change":"2023-09-23T20:48:51.171448+0000","last_active":"2023-09-23T20:49:11.701128+0000","last_peered":"2023-09-23T20:49:11.701128+0000","last_clean":"2023-09-23T20:49:11.701128+0000","last_became_active":"2023-09-23T20:48:51.170485+0000","last_became_peered":"2023-09-23T20:48:51.170485+0000","last_unstale":"2023-09-23T20:49:11.701128+0000","last_undegraded":"2023-09-23T20:49:11.701128+0000","last_fullsized":"2023-09-23T20:49:11.701128+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:48:35.309684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011300049999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":512,"num_read_kb":1820,"num_write":133,"num_write_kb":1347,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":601,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T20:49:11.696303+0000","last_change":"2023-09-23T20:48:48.151034+0000","last_active":"2023-09-23T20:49:11.696303+0000","last_peered":"2023-09-23T20:49:11.696303+0000","last_clean":"2023-09-23T20:49:11.696303+0000","last_became_active":"2023-09-23T20:48:48.150658+0000","last_became_peered":"2023-09-23T20:48:48.150658+0000","last_unstale":"2023-09-23T20:49:11.696303+0000","last_undegraded":"2023-09-23T20:49:11.696303+0000","last_fullsized":"2023-09-23T20:49:11.696303+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:44:34.819851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":126,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T20:49:11.698115+0000","last_change":"2023-09-23T20:44:10.555583+0000","last_active":"2023-09-23T20:49:11.698115+0000","last_peered":"2023-09-23T20:49:11.698115+0000","last_clean":"2023-09-23T20:49:11.698115+0000","last_became_active":"2023-09-23T20:44:10.555281+0000","last_became_peered":"2023-09-23T20:44:10.555281+0000","last_unstale":"2023-09-23T20:49:11.698115+0000","last_undegraded":"2023-09-23T20:49:11.698115+0000","last_fullsized":"2023-09-23T20:49:11.698115+0000","mapping_epoch":496,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":497,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:43:11.942147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"607'11784","reported_seq":11868,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T20:49:10.109127+0000","last_change":"2023-09-23T20:45:39.331096+0000","last_active":"2023-09-23T20:49:10.109127+0000","last_peered":"2023-09-23T20:49:10.109127+0000","last_clean":"2023-09-23T20:49:10.109127+0000","last_became_active":"2023-09-23T20:45:39.330752+0000","last_became_peered":"2023-09-23T20:45:39.330752+0000","last_unstale":"2023-09-23T20:49:10.109127+0000","last_undegraded":"2023-09-23T20:49:10.109127+0000","last_fullsized":"2023-09-23T20:49:10.109127+0000","mapping_epoch":531,"log_start":"543'5729","ondisk_log_start":"543'5729","created":531,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":6055,"log_dups_size":0,"ondisk_log_size":6055,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:39:01.627762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":86769664,"num_objects":1324,"num_object_clones":0,"num_object_copies":3972,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1324,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11784,"num_write_kb":419456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"86'194","reported_seq":721,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T20:49:11.734033+0000","last_change":"2023-09-23T20:48:51.175633+0000","last_active":"2023-09-23T20:49:11.734033+0000","last_peered":"2023-09-23T20:49:11.734033+0000","last_clean":"2023-09-23T20:49:11.734033+0000","last_became_active":"2023-09-23T20:48:51.174941+0000","last_became_peered":"2023-09-23T20:48:51.174941+0000","last_unstale":"2023-09-23T20:49:11.734033+0000","last_undegraded":"2023-09-23T20:49:11.734033+0000","last_fullsized":"2023-09-23T20:49:11.734033+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":194,"log_dups_size":0,"ondisk_log_size":194,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:06:27.009241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00037995999999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1024,"num_read_kb":1200,"num_write":173,"num_write_kb":1276,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":1069,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T20:49:08.829127+0000","last_change":"2023-09-23T20:38:05.602508+0000","last_active":"2023-09-23T20:49:08.829127+0000","last_peered":"2023-09-23T20:49:08.829127+0000","last_clean":"2023-09-23T20:49:08.829127+0000","last_became_active":"2023-09-23T20:38:05.602129+0000","last_became_peered":"2023-09-23T20:38:05.602129+0000","last_unstale":"2023-09-23T20:49:08.829127+0000","last_undegraded":"2023-09-23T20:49:08.829127+0000","last_fullsized":"2023-09-23T20:49:08.829127+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:45.343742+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:45.343742+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:48.910343+0000","scrub_duration":0.001475802,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":182,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T20:49:08.813801+0000","last_change":"2023-09-23T20:43:16.758213+0000","last_active":"2023-09-23T20:49:08.813801+0000","last_peered":"2023-09-23T20:49:08.813801+0000","last_clean":"2023-09-23T20:49:08.813801+0000","last_became_active":"2023-09-23T20:43:16.757869+0000","last_became_peered":"2023-09-23T20:43:16.757869+0000","last_unstale":"2023-09-23T20:49:08.813801+0000","last_undegraded":"2023-09-23T20:49:08.813801+0000","last_fullsized":"2023-09-23T20:49:08.813801+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:43:10.811749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":232,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T20:49:08.814997+0000","last_change":"2023-09-23T20:47:09.202302+0000","last_active":"2023-09-23T20:49:08.814997+0000","last_peered":"2023-09-23T20:49:08.814997+0000","last_clean":"2023-09-23T20:49:08.814997+0000","last_became_active":"2023-09-23T20:47:09.201797+0000","last_became_peered":"2023-09-23T20:47:09.201797+0000","last_unstale":"2023-09-23T20:49:08.814997+0000","last_undegraded":"2023-09-23T20:49:08.814997+0000","last_fullsized":"2023-09-23T20:49:08.814997+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:34:41.055705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.8","version":"608'11041","reported_seq":14848,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T20:49:11.355527+0000","last_change":"2023-09-23T20:48:52.187793+0000","last_active":"2023-09-23T20:49:11.355527+0000","last_peered":"2023-09-23T20:49:11.355527+0000","last_clean":"2023-09-23T20:49:11.355527+0000","last_became_active":"2023-09-23T20:48:52.173033+0000","last_became_peered":"2023-09-23T20:48:52.173033+0000","last_unstale":"2023-09-23T20:49:11.355527+0000","last_undegraded":"2023-09-23T20:49:11.355527+0000","last_fullsized":"2023-09-23T20:49:11.355527+0000","mapping_epoch":595,"log_start":"541'4571","ondisk_log_start":"541'4571","created":531,"last_epoch_clean":596,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":6470,"log_dups_size":0,"ondisk_log_size":6470,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":41844736,"num_objects":638,"num_object_clones":0,"num_object_copies":1914,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":638,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5788,"num_write_kb":205664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":994,"num_bytes_recovered":65175552,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":978,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T20:49:11.697697+0000","last_change":"2023-09-23T20:47:11.622262+0000","last_active":"2023-09-23T20:49:11.697697+0000","last_peered":"2023-09-23T20:49:11.697697+0000","last_clean":"2023-09-23T20:49:11.697697+0000","last_became_active":"2023-09-23T20:47:11.621920+0000","last_became_peered":"2023-09-23T20:47:11.621920+0000","last_unstale":"2023-09-23T20:49:11.697697+0000","last_undegraded":"2023-09-23T20:49:11.697697+0000","last_fullsized":"2023-09-23T20:49:11.697697+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:43.582146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:43.582146+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:01:15.153543+0000","scrub_duration":0.0016219730000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"86'80","reported_seq":859,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T20:49:08.829037+0000","last_change":"2023-09-23T20:29:45.536796+0000","last_active":"2023-09-23T20:49:08.829037+0000","last_peered":"2023-09-23T20:49:08.829037+0000","last_clean":"2023-09-23T20:49:08.829037+0000","last_became_active":"2023-09-23T20:29:45.536439+0000","last_became_peered":"2023-09-23T20:29:45.536439+0000","last_unstale":"2023-09-23T20:49:08.829037+0000","last_undegraded":"2023-09-23T20:49:08.829037+0000","last_fullsized":"2023-09-23T20:49:08.829037+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":80,"log_dups_size":0,"ondisk_log_size":80,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:30:57.793132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035115,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":395,"num_read_kb":1370,"num_write":66,"num_write_kb":1061,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":621,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T20:49:10.809499+0000","last_change":"2023-09-23T20:47:09.198326+0000","last_active":"2023-09-23T20:49:10.809499+0000","last_peered":"2023-09-23T20:49:10.809499+0000","last_clean":"2023-09-23T20:49:10.809499+0000","last_became_active":"2023-09-23T20:47:09.198059+0000","last_became_peered":"2023-09-23T20:47:09.198059+0000","last_unstale":"2023-09-23T20:49:10.809499+0000","last_undegraded":"2023-09-23T20:49:10.809499+0000","last_fullsized":"2023-09-23T20:49:10.809499+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:38:53.903706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":616,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T20:49:10.810187+0000","last_change":"2023-09-23T20:48:48.146977+0000","last_active":"2023-09-23T20:49:10.810187+0000","last_peered":"2023-09-23T20:49:10.810187+0000","last_clean":"2023-09-23T20:49:10.810187+0000","last_became_active":"2023-09-23T20:48:48.146657+0000","last_became_peered":"2023-09-23T20:48:48.146657+0000","last_unstale":"2023-09-23T20:49:10.810187+0000","last_undegraded":"2023-09-23T20:49:10.810187+0000","last_fullsized":"2023-09-23T20:49:10.810187+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:38:41.782692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"607'11520","reported_seq":12230,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T20:49:10.104122+0000","last_change":"2023-09-23T20:47:13.668055+0000","last_active":"2023-09-23T20:49:10.104122+0000","last_peered":"2023-09-23T20:49:10.104122+0000","last_clean":"2023-09-23T20:49:10.104122+0000","last_became_active":"2023-09-23T20:47:13.626170+0000","last_became_peered":"2023-09-23T20:47:13.626170+0000","last_unstale":"2023-09-23T20:49:10.104122+0000","last_undegraded":"2023-09-23T20:49:10.104122+0000","last_fullsized":"2023-09-23T20:49:10.104122+0000","mapping_epoch":557,"log_start":"543'5495","ondisk_log_start":"543'5495","created":531,"last_epoch_clean":558,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":6025,"log_dups_size":0,"ondisk_log_size":6025,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:32.805547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90112024,"num_objects":1376,"num_object_clones":0,"num_object_copies":4128,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1376,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":11520,"num_write_kb":412609,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":298,"num_bytes_recovered":19529728,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":718,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T20:49:08.812639+0000","last_change":"2023-09-23T20:41:08.380084+0000","last_active":"2023-09-23T20:49:08.812639+0000","last_peered":"2023-09-23T20:49:08.812639+0000","last_clean":"2023-09-23T20:49:08.812639+0000","last_became_active":"2023-09-23T20:38:05.610881+0000","last_became_peered":"2023-09-23T20:38:05.610881+0000","last_unstale":"2023-09-23T20:49:08.812639+0000","last_undegraded":"2023-09-23T20:49:08.812639+0000","last_fullsized":"2023-09-23T20:49:08.812639+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:41:08.379959+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:41:08.379959+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:19:39.547923+0000","scrub_duration":0.0019158420000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"86'102","reported_seq":1277,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T20:49:08.829783+0000","last_change":"2023-09-23T20:47:14.628795+0000","last_active":"2023-09-23T20:49:08.829783+0000","last_peered":"2023-09-23T20:49:08.829783+0000","last_clean":"2023-09-23T20:49:08.829783+0000","last_became_active":"2023-09-23T20:47:14.628341+0000","last_became_peered":"2023-09-23T20:47:14.628341+0000","last_unstale":"2023-09-23T20:49:08.829783+0000","last_undegraded":"2023-09-23T20:49:08.829783+0000","last_fullsized":"2023-09-23T20:49:08.829783+0000","mapping_epoch":558,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":559,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":102,"log_dups_size":0,"ondisk_log_size":102,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:42:16.291849+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030656500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":963,"num_read_kb":1234,"num_write":60,"num_write_kb":225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":577,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T20:49:11.697314+0000","last_change":"2023-09-23T20:48:48.151135+0000","last_active":"2023-09-23T20:49:11.697314+0000","last_peered":"2023-09-23T20:49:11.697314+0000","last_clean":"2023-09-23T20:49:11.697314+0000","last_became_active":"2023-09-23T20:48:48.150780+0000","last_became_peered":"2023-09-23T20:48:48.150780+0000","last_unstale":"2023-09-23T20:49:11.697314+0000","last_undegraded":"2023-09-23T20:49:11.697314+0000","last_fullsized":"2023-09-23T20:49:11.697314+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:39:09.727249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":553,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T20:49:10.809904+0000","last_change":"2023-09-23T20:43:46.541173+0000","last_active":"2023-09-23T20:49:10.809904+0000","last_peered":"2023-09-23T20:49:10.809904+0000","last_clean":"2023-09-23T20:49:10.809904+0000","last_became_active":"2023-09-23T20:43:46.540726+0000","last_became_peered":"2023-09-23T20:43:46.540726+0000","last_unstale":"2023-09-23T20:49:10.809904+0000","last_undegraded":"2023-09-23T20:49:10.809904+0000","last_fullsized":"2023-09-23T20:49:10.809904+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:49:04.137885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.e","version":"607'11678","reported_seq":15980,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T20:49:10.084536+0000","last_change":"2023-09-23T20:48:48.230594+0000","last_active":"2023-09-23T20:49:10.084536+0000","last_peered":"2023-09-23T20:49:10.084536+0000","last_clean":"2023-09-23T20:49:10.084536+0000","last_became_active":"2023-09-23T20:48:48.164237+0000","last_became_peered":"2023-09-23T20:48:48.164237+0000","last_unstale":"2023-09-23T20:49:10.084536+0000","last_undegraded":"2023-09-23T20:49:10.084536+0000","last_fullsized":"2023-09-23T20:49:10.084536+0000","mapping_epoch":591,"log_start":"560'6500","ondisk_log_start":"560'6500","created":531,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":5178,"log_dups_size":0,"ondisk_log_size":5178,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:49:13.858414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":86507520,"num_objects":1320,"num_object_clones":0,"num_object_copies":3960,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1320,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11678,"num_write_kb":415936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2081,"num_bytes_recovered":136314880,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":645,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T20:49:09.808463+0000","last_change":"2023-09-23T20:48:51.172830+0000","last_active":"2023-09-23T20:49:09.808463+0000","last_peered":"2023-09-23T20:49:09.808463+0000","last_clean":"2023-09-23T20:49:09.808463+0000","last_became_active":"2023-09-23T20:48:51.172481+0000","last_became_peered":"2023-09-23T20:48:51.172481+0000","last_unstale":"2023-09-23T20:49:09.808463+0000","last_undegraded":"2023-09-23T20:49:09.808463+0000","last_fullsized":"2023-09-23T20:49:09.808463+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:47:34.561747+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:47:34.561747+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:30:32.531477+0000","scrub_duration":0.0087949929999999992,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":344,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T20:49:08.830018+0000","last_change":"2023-09-23T20:32:50.549695+0000","last_active":"2023-09-23T20:49:08.830018+0000","last_peered":"2023-09-23T20:49:08.830018+0000","last_clean":"2023-09-23T20:49:08.830018+0000","last_became_active":"2023-09-23T20:32:50.549431+0000","last_became_peered":"2023-09-23T20:32:50.549431+0000","last_unstale":"2023-09-23T20:49:08.830018+0000","last_undegraded":"2023-09-23T20:49:08.830018+0000","last_fullsized":"2023-09-23T20:49:08.830018+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:45:53.666569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"86'128","reported_seq":1090,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T20:49:11.698408+0000","last_change":"2023-09-23T20:47:11.621846+0000","last_active":"2023-09-23T20:49:11.698408+0000","last_peered":"2023-09-23T20:49:11.698408+0000","last_clean":"2023-09-23T20:49:11.698408+0000","last_became_active":"2023-09-23T20:47:11.621284+0000","last_became_peered":"2023-09-23T20:47:11.621284+0000","last_unstale":"2023-09-23T20:49:11.698408+0000","last_undegraded":"2023-09-23T20:49:11.698408+0000","last_fullsized":"2023-09-23T20:49:11.698408+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:43:50.658512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000293202,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":579,"num_read_kb":1473,"num_write":96,"num_write_kb":1062,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":388,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T20:49:03.753563+0000","last_change":"2023-09-23T20:37:43.582105+0000","last_active":"2023-09-23T20:49:03.753563+0000","last_peered":"2023-09-23T20:49:03.753563+0000","last_clean":"2023-09-23T20:49:03.753563+0000","last_became_active":"2023-09-23T20:37:43.581860+0000","last_became_peered":"2023-09-23T20:37:43.581860+0000","last_unstale":"2023-09-23T20:49:03.753563+0000","last_undegraded":"2023-09-23T20:49:03.753563+0000","last_fullsized":"2023-09-23T20:49:03.753563+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:18:38.765486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"607'11660","reported_seq":17170,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T20:49:10.136614+0000","last_change":"2023-09-23T20:48:53.235559+0000","last_active":"2023-09-23T20:49:10.136614+0000","last_peered":"2023-09-23T20:49:10.136614+0000","last_clean":"2023-09-23T20:49:10.136614+0000","last_became_active":"2023-09-23T20:48:53.178550+0000","last_became_peered":"2023-09-23T20:48:53.178550+0000","last_unstale":"2023-09-23T20:49:10.136614+0000","last_undegraded":"2023-09-23T20:49:10.136614+0000","last_fullsized":"2023-09-23T20:49:10.136614+0000","mapping_epoch":596,"log_start":"543'5599","ondisk_log_start":"543'5599","created":531,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":6061,"log_dups_size":0,"ondisk_log_size":6061,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:38:02.612388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":88211456,"num_objects":1346,"num_object_clones":0,"num_object_copies":4038,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1346,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11660,"num_write_kb":416192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2684,"num_bytes_recovered":175898624,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.4","version":"90'161","reported_seq":822,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T20:49:03.761046+0000","last_change":"2023-09-23T20:40:22.427435+0000","last_active":"2023-09-23T20:49:03.761046+0000","last_peered":"2023-09-23T20:49:03.761046+0000","last_clean":"2023-09-23T20:49:03.761046+0000","last_became_active":"2023-09-23T20:40:22.427051+0000","last_became_peered":"2023-09-23T20:40:22.427051+0000","last_unstale":"2023-09-23T20:49:03.761046+0000","last_undegraded":"2023-09-23T20:49:03.761046+0000","last_fullsized":"2023-09-23T20:49:03.761046+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:34:54.504554+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:34:54.504554+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:42.001652+0000","scrub_duration":0.011176864,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"86'117","reported_seq":991,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T20:49:08.815449+0000","last_change":"2023-09-23T20:48:51.217483+0000","last_active":"2023-09-23T20:49:08.815449+0000","last_peered":"2023-09-23T20:49:08.815449+0000","last_clean":"2023-09-23T20:49:08.815449+0000","last_became_active":"2023-09-23T20:48:51.217013+0000","last_became_peered":"2023-09-23T20:48:51.217013+0000","last_unstale":"2023-09-23T20:49:08.815449+0000","last_undegraded":"2023-09-23T20:49:08.815449+0000","last_fullsized":"2023-09-23T20:49:08.815449+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:40:42.297947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046891400000000002,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":893,"num_read_kb":922,"num_write":82,"num_write_kb":4193,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":48,"num_bytes_recovered":25165824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":624,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T20:49:10.810240+0000","last_change":"2023-09-23T20:47:09.317116+0000","last_active":"2023-09-23T20:49:10.810240+0000","last_peered":"2023-09-23T20:49:10.810240+0000","last_clean":"2023-09-23T20:49:10.810240+0000","last_became_active":"2023-09-23T20:47:09.205930+0000","last_became_peered":"2023-09-23T20:47:09.205930+0000","last_unstale":"2023-09-23T20:49:10.810240+0000","last_undegraded":"2023-09-23T20:49:10.810240+0000","last_fullsized":"2023-09-23T20:49:10.810240+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:04:55.528625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":65,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T20:49:09.807745+0000","last_change":"2023-09-23T20:47:09.192117+0000","last_active":"2023-09-23T20:49:09.807745+0000","last_peered":"2023-09-23T20:49:09.807745+0000","last_clean":"2023-09-23T20:49:09.807745+0000","last_became_active":"2023-09-23T20:47:09.191820+0000","last_became_peered":"2023-09-23T20:47:09.191820+0000","last_unstale":"2023-09-23T20:49:09.807745+0000","last_undegraded":"2023-09-23T20:49:09.807745+0000","last_fullsized":"2023-09-23T20:49:09.807745+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:08.785173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.12","version":"606'11389","reported_seq":15819,"reported_epoch":607,"state":"peering","last_fresh":"2023-09-23T20:49:09.809840+0000","last_change":"2023-09-23T20:49:09.810076+0000","last_active":"2023-09-23T20:49:09.809812+0000","last_peered":"2023-09-23T20:49:09.354675+0000","last_clean":"2023-09-23T20:49:09.354675+0000","last_became_active":"2023-09-23T20:49:02.756574+0000","last_became_peered":"2023-09-23T20:49:02.756574+0000","last_unstale":"2023-09-23T20:49:09.809840+0000","last_undegraded":"2023-09-23T20:49:09.809840+0000","last_fullsized":"2023-09-23T20:49:09.809840+0000","mapping_epoch":607,"log_start":"542'5074","ondisk_log_start":"542'5074","created":584,"last_epoch_clean":606,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":6315,"log_dups_size":0,"ondisk_log_size":6315,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":44695552,"num_objects":682,"num_object_clones":0,"num_object_copies":2046,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":682,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5960,"num_write_kb":212544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1072,"num_bytes_recovered":70221824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,5],"acting":[6,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[0,1,2,4,5],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.19","version":"90'161","reported_seq":867,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T20:49:10.809251+0000","last_change":"2023-09-23T20:43:46.544750+0000","last_active":"2023-09-23T20:49:10.809251+0000","last_peered":"2023-09-23T20:49:10.809251+0000","last_clean":"2023-09-23T20:49:10.809251+0000","last_became_active":"2023-09-23T20:43:46.544395+0000","last_became_peered":"2023-09-23T20:43:46.544395+0000","last_unstale":"2023-09-23T20:49:10.809251+0000","last_undegraded":"2023-09-23T20:49:10.809251+0000","last_fullsized":"2023-09-23T20:49:10.809251+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:06.487351+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:06.487351+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:56.866168+0000","scrub_duration":0.001964379,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":583,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T20:49:10.809828+0000","last_change":"2023-09-23T20:47:09.197601+0000","last_active":"2023-09-23T20:49:10.809828+0000","last_peered":"2023-09-23T20:49:10.809828+0000","last_clean":"2023-09-23T20:49:10.809828+0000","last_became_active":"2023-09-23T20:47:09.196967+0000","last_became_peered":"2023-09-23T20:47:09.196967+0000","last_unstale":"2023-09-23T20:49:10.809828+0000","last_undegraded":"2023-09-23T20:49:10.809828+0000","last_fullsized":"2023-09-23T20:49:10.809828+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:06:56.826785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":29,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T20:49:11.701091+0000","last_change":"2023-09-23T20:48:48.158376+0000","last_active":"2023-09-23T20:49:11.701091+0000","last_peered":"2023-09-23T20:49:11.701091+0000","last_clean":"2023-09-23T20:49:11.701091+0000","last_became_active":"2023-09-23T20:48:48.158069+0000","last_became_peered":"2023-09-23T20:48:48.158069+0000","last_unstale":"2023-09-23T20:49:11.701091+0000","last_undegraded":"2023-09-23T20:49:11.701091+0000","last_fullsized":"2023-09-23T20:49:11.701091+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:34.658561+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.15","version":"610'11267","reported_seq":13647,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T20:49:13.227042+0000","last_change":"2023-09-23T20:48:49.194473+0000","last_active":"2023-09-23T20:49:13.227042+0000","last_peered":"2023-09-23T20:49:13.227042+0000","last_clean":"2023-09-23T20:49:13.227042+0000","last_became_active":"2023-09-23T20:48:49.173258+0000","last_became_peered":"2023-09-23T20:48:49.173258+0000","last_unstale":"2023-09-23T20:49:13.227042+0000","last_undegraded":"2023-09-23T20:49:13.227042+0000","last_fullsized":"2023-09-23T20:49:13.227042+0000","mapping_epoch":592,"log_start":"542'4806","ondisk_log_start":"542'4806","created":584,"last_epoch_clean":593,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":6461,"log_dups_size":0,"ondisk_log_size":6461,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":44269568,"num_objects":676,"num_object_clones":0,"num_object_copies":2028,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":676,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5917,"num_write_kb":210944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1132,"num_bytes_recovered":74186752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"90'161","reported_seq":681,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T20:49:03.752889+0000","last_change":"2023-09-23T20:41:06.499511+0000","last_active":"2023-09-23T20:49:03.752889+0000","last_peered":"2023-09-23T20:49:03.752889+0000","last_clean":"2023-09-23T20:49:03.752889+0000","last_became_active":"2023-09-23T20:38:05.601874+0000","last_became_peered":"2023-09-23T20:38:05.601874+0000","last_unstale":"2023-09-23T20:49:03.752889+0000","last_undegraded":"2023-09-23T20:49:03.752889+0000","last_fullsized":"2023-09-23T20:49:03.752889+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:41:06.499396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:41:06.499396+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:31:34.059077+0000","scrub_duration":0.0019670310000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":69,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T20:49:09.808376+0000","last_change":"2023-09-23T20:47:09.332980+0000","last_active":"2023-09-23T20:49:09.808376+0000","last_peered":"2023-09-23T20:49:09.808376+0000","last_clean":"2023-09-23T20:49:09.808376+0000","last_became_active":"2023-09-23T20:47:09.230554+0000","last_became_peered":"2023-09-23T20:47:09.230554+0000","last_unstale":"2023-09-23T20:49:09.808376+0000","last_undegraded":"2023-09-23T20:49:09.808376+0000","last_fullsized":"2023-09-23T20:49:09.808376+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:06:21.882823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":65,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T20:49:09.808276+0000","last_change":"2023-09-23T20:47:09.186579+0000","last_active":"2023-09-23T20:49:09.808276+0000","last_peered":"2023-09-23T20:49:09.808276+0000","last_clean":"2023-09-23T20:49:09.808276+0000","last_became_active":"2023-09-23T20:47:09.186227+0000","last_became_peered":"2023-09-23T20:47:09.186227+0000","last_unstale":"2023-09-23T20:49:09.808276+0000","last_undegraded":"2023-09-23T20:49:09.808276+0000","last_fullsized":"2023-09-23T20:49:09.808276+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:45:39.485308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.14","version":"609'11074","reported_seq":11746,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T20:49:11.696387+0000","last_change":"2023-09-23T20:48:40.130483+0000","last_active":"2023-09-23T20:49:11.696387+0000","last_peered":"2023-09-23T20:49:11.696387+0000","last_clean":"2023-09-23T20:49:11.696387+0000","last_became_active":"2023-09-23T20:48:40.105209+0000","last_became_peered":"2023-09-23T20:48:40.105209+0000","last_unstale":"2023-09-23T20:49:11.696387+0000","last_undegraded":"2023-09-23T20:49:11.696387+0000","last_fullsized":"2023-09-23T20:49:11.696387+0000","mapping_epoch":584,"log_start":"541'4547","ondisk_log_start":"541'4547","created":584,"last_epoch_clean":585,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":6527,"log_dups_size":0,"ondisk_log_size":6527,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":41811968,"num_objects":638,"num_object_clones":0,"num_object_copies":1914,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":638,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5825,"num_write_kb":206816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":137,"num_bytes_recovered":8978432,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":69,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T20:49:09.808382+0000","last_change":"2023-09-23T20:47:09.316824+0000","last_active":"2023-09-23T20:49:09.808382+0000","last_peered":"2023-09-23T20:49:09.808382+0000","last_clean":"2023-09-23T20:49:09.808382+0000","last_became_active":"2023-09-23T20:47:09.195100+0000","last_became_peered":"2023-09-23T20:47:09.195100+0000","last_unstale":"2023-09-23T20:49:09.808382+0000","last_undegraded":"2023-09-23T20:49:09.808382+0000","last_fullsized":"2023-09-23T20:49:09.808382+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:29:53.273554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":65,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T20:49:09.807897+0000","last_change":"2023-09-23T20:47:09.181240+0000","last_active":"2023-09-23T20:49:09.807897+0000","last_peered":"2023-09-23T20:49:09.807897+0000","last_clean":"2023-09-23T20:49:09.807897+0000","last_became_active":"2023-09-23T20:47:09.181022+0000","last_became_peered":"2023-09-23T20:47:09.181022+0000","last_unstale":"2023-09-23T20:49:09.807897+0000","last_undegraded":"2023-09-23T20:49:09.807897+0000","last_fullsized":"2023-09-23T20:49:09.807897+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:38:59.191781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.17","version":"607'11046","reported_seq":11140,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T20:49:10.059604+0000","last_change":"2023-09-23T20:48:40.108816+0000","last_active":"2023-09-23T20:49:10.059604+0000","last_peered":"2023-09-23T20:49:10.059604+0000","last_clean":"2023-09-23T20:49:10.059604+0000","last_became_active":"2023-09-23T20:48:40.090791+0000","last_became_peered":"2023-09-23T20:48:40.090791+0000","last_unstale":"2023-09-23T20:49:10.059604+0000","last_undegraded":"2023-09-23T20:49:10.059604+0000","last_fullsized":"2023-09-23T20:49:10.059604+0000","mapping_epoch":584,"log_start":"542'4863","ondisk_log_start":"542'4863","created":584,"last_epoch_clean":585,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":6183,"log_dups_size":0,"ondisk_log_size":6183,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":46170112,"num_objects":705,"num_object_clones":0,"num_object_copies":2115,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":705,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5783,"num_write_kb":207584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1c","version":"90'161","reported_seq":862,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T20:49:08.829681+0000","last_change":"2023-09-23T20:38:05.591992+0000","last_active":"2023-09-23T20:49:08.829681+0000","last_peered":"2023-09-23T20:49:08.829681+0000","last_clean":"2023-09-23T20:49:08.829681+0000","last_became_active":"2023-09-23T20:38:05.591701+0000","last_became_peered":"2023-09-23T20:38:05.591701+0000","last_unstale":"2023-09-23T20:49:08.829681+0000","last_undegraded":"2023-09-23T20:49:08.829681+0000","last_fullsized":"2023-09-23T20:49:08.829681+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:02.384133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:02.384133+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:41.722542+0000","scrub_duration":0.001769098,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":448,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T20:49:03.753089+0000","last_change":"2023-09-23T20:48:48.158348+0000","last_active":"2023-09-23T20:49:03.753089+0000","last_peered":"2023-09-23T20:49:03.753089+0000","last_clean":"2023-09-23T20:49:03.753089+0000","last_became_active":"2023-09-23T20:48:48.157982+0000","last_became_peered":"2023-09-23T20:48:48.157982+0000","last_unstale":"2023-09-23T20:49:03.753089+0000","last_undegraded":"2023-09-23T20:49:03.753089+0000","last_fullsized":"2023-09-23T20:49:03.753089+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:03:48.078707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":414,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T20:49:03.760511+0000","last_change":"2023-09-23T20:48:48.159781+0000","last_active":"2023-09-23T20:49:03.760511+0000","last_peered":"2023-09-23T20:49:03.760511+0000","last_clean":"2023-09-23T20:49:03.760511+0000","last_became_active":"2023-09-23T20:48:48.159505+0000","last_became_peered":"2023-09-23T20:48:48.159505+0000","last_unstale":"2023-09-23T20:49:03.760511+0000","last_undegraded":"2023-09-23T20:49:03.760511+0000","last_fullsized":"2023-09-23T20:49:03.760511+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:25:18.100645+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.16","version":"609'11244","reported_seq":12446,"reported_epoch":609,"state":"active+remapped+backfilling","last_fresh":"2023-09-23T20:49:11.779319+0000","last_change":"2023-09-23T20:49:11.368019+0000","last_active":"2023-09-23T20:49:11.779319+0000","last_peered":"2023-09-23T20:49:11.779319+0000","last_clean":"2023-09-23T20:49:01.526428+0000","last_became_active":"2023-09-23T20:49:03.813418+0000","last_became_peered":"2023-09-23T20:49:03.813418+0000","last_unstale":"2023-09-23T20:49:11.779319+0000","last_undegraded":"2023-09-23T20:49:11.779319+0000","last_fullsized":"2023-09-23T20:49:11.779319+0000","mapping_epoch":604,"log_start":"541'4690","ondisk_log_start":"541'4690","created":584,"last_epoch_clean":585,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":6554,"log_dups_size":0,"ondisk_log_size":6554,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":41844736,"num_objects":639,"num_object_clones":0,"num_object_copies":1917,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":392,"num_objects_unfound":0,"num_objects_dirty":639,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5924,"num_write_kb":209984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":393,"num_bytes_recovered":25722880,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[0,4,5],"avail_no_missing":["0","4","5"],"object_location_counts":[{"shards":"0,4,5","objects":639}],"blocked_by":[],"up_primary":1,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1d","version":"90'161","reported_seq":739,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T20:49:11.697605+0000","last_change":"2023-09-23T20:47:11.621758+0000","last_active":"2023-09-23T20:49:11.697605+0000","last_peered":"2023-09-23T20:49:11.697605+0000","last_clean":"2023-09-23T20:49:11.697605+0000","last_became_active":"2023-09-23T20:47:11.621345+0000","last_became_peered":"2023-09-23T20:47:11.621345+0000","last_unstale":"2023-09-23T20:49:11.697605+0000","last_undegraded":"2023-09-23T20:49:11.697605+0000","last_fullsized":"2023-09-23T20:49:11.697605+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:01.504973+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:01.504973+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:31:17.112393+0000","scrub_duration":0.00202344,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":673,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T20:49:10.809966+0000","last_change":"2023-09-23T20:48:48.164011+0000","last_active":"2023-09-23T20:49:10.809966+0000","last_peered":"2023-09-23T20:49:10.809966+0000","last_clean":"2023-09-23T20:49:10.809966+0000","last_became_active":"2023-09-23T20:48:48.150690+0000","last_became_peered":"2023-09-23T20:48:48.150690+0000","last_unstale":"2023-09-23T20:49:10.809966+0000","last_undegraded":"2023-09-23T20:49:10.809966+0000","last_fullsized":"2023-09-23T20:49:10.809966+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:35:33.111175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":4432,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":75,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T20:49:09.807508+0000","last_change":"2023-09-23T20:48:23.917302+0000","last_active":"2023-09-23T20:49:09.807508+0000","last_peered":"2023-09-23T20:49:09.807508+0000","last_clean":"2023-09-23T20:49:09.807508+0000","last_became_active":"2023-09-23T20:48:23.916956+0000","last_became_peered":"2023-09-23T20:48:23.916956+0000","last_unstale":"2023-09-23T20:49:09.807508+0000","last_undegraded":"2023-09-23T20:49:09.807508+0000","last_fullsized":"2023-09-23T20:49:09.807508+0000","mapping_epoch":577,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":578,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:42:27.989294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":26,"stat_sum":{"num_bytes":1405222936,"num_objects":21443,"num_object_clones":0,"num_object_copies":64329,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1543,"num_objects_unfound":0,"num_objects_dirty":21443,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":187545,"num_write_kb":6687553,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18064,"num_bytes_recovered":1183449088,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":4948045824,"data_stored":4948033608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":169962,"ondisk_log_size":169962,"up":78,"acting":78,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":109,"num_bytes_recovered":15220,"num_keys_recovered":80,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26691,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":661927,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":59,"num_read_kb":110,"num_write":28,"num_write_kb":291,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":477366,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5983,"num_read_kb":13200,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":135,"num_bytes_recovered":25165824,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6082,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201733,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1127596,"kb_used_data":531108,"kb_used_omap":0,"kb_used_meta":596480,"kb_avail":92617556,"statfs":{"total":95995035648,"available":94840377344,"internally_reserved":0,"allocated":543854592,"data_stored":542203371,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":610795520},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:49:07 2023","interfaces":[{"interface":"back","average":{"1min":1.5669999999999999,"5min":1.774,"15min":2.0369999999999999},"min":{"1min":0.23400000000000001,"5min":0.224,"15min":0.224},"max":{"1min":5.1360000000000001,"5min":10.975,"15min":18.315000000000001},"last":4.2530000000000001},{"interface":"front","average":{"1min":1.4259999999999999,"5min":1.8819999999999999,"15min":1.865},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":8.0530000000000008,"5min":12.986000000000001,"15min":19.355},"last":8.0530000000000008}]},{"osd":1,"last update":"Sat Sep 23 20:48:13 2023","interfaces":[{"interface":"back","average":{"1min":1.3640000000000001,"5min":1.6779999999999999,"15min":1.972},"min":{"1min":0.28399999999999997,"5min":0.25900000000000001,"15min":0.24399999999999999},"max":{"1min":6.4610000000000003,"5min":15.131,"15min":49.965000000000003},"last":0.29499999999999998},{"interface":"front","average":{"1min":2.4929999999999999,"5min":2.524,"15min":2.5099999999999998},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.26700000000000002},"max":{"1min":10.861000000000001,"5min":15.18,"15min":53.451999999999998},"last":0.27000000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:49:07 2023","interfaces":[{"interface":"back","average":{"1min":1.847,"5min":1.885,"15min":2.121},"min":{"1min":0.31,"5min":0.28499999999999998,"15min":0.27800000000000002},"max":{"1min":7.9219999999999997,"5min":12.933999999999999,"15min":16.965},"last":3.1629999999999998},{"interface":"front","average":{"1min":2.0209999999999999,"5min":1.982,"15min":2.2160000000000002},"min":{"1min":0.36599999999999999,"5min":0.31,"15min":0.30299999999999999},"max":{"1min":7.3019999999999996,"5min":13.5,"15min":23.414000000000001},"last":1.345}]},{"osd":3,"last update":"Sat Sep 23 20:48:38 2023","interfaces":[{"interface":"back","average":{"1min":2.4140000000000001,"5min":1.964,"15min":2.226},"min":{"1min":0.34399999999999997,"5min":0.33800000000000002,"15min":0.311},"max":{"1min":7.2880000000000003,"5min":15.115,"15min":23.352},"last":3.4380000000000002},{"interface":"front","average":{"1min":2.3180000000000001,"5min":2.25,"15min":2.4239999999999999},"min":{"1min":0.35399999999999998,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":6.5090000000000003,"5min":15.144,"15min":20.969999999999999},"last":3.452}]},{"osd":4,"last update":"Sat Sep 23 20:48:38 2023","interfaces":[{"interface":"back","average":{"1min":1.583,"5min":1.458,"15min":1.2350000000000001},"min":{"1min":0.26500000000000001,"5min":0.254,"15min":0.23200000000000001},"max":{"1min":7.4109999999999996,"5min":12.260999999999999,"15min":18.364999999999998},"last":0.25800000000000001},{"interface":"front","average":{"1min":1.1479999999999999,"5min":1.6659999999999999,"15min":1.45},"min":{"1min":0.34599999999999997,"5min":0.247,"15min":0.247},"max":{"1min":4.1529999999999996,"5min":7.2469999999999999,"15min":23.617000000000001},"last":0.33300000000000002}]},{"osd":5,"last update":"Sat Sep 23 20:48:38 2023","interfaces":[{"interface":"back","average":{"1min":1.548,"5min":1.48,"15min":1.2709999999999999},"min":{"1min":0.28100000000000003,"5min":0.27200000000000002,"15min":0.26600000000000001},"max":{"1min":7.2270000000000003,"5min":15.005000000000001,"15min":15.005000000000001},"last":1.3759999999999999},{"interface":"front","average":{"1min":1.1890000000000001,"5min":1.6060000000000001,"15min":1.3899999999999999},"min":{"1min":0.32200000000000001,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":7.335,"5min":12.212,"15min":18.390000000000001},"last":0.52600000000000002}]},{"osd":6,"last update":"Sat Sep 23 20:48:38 2023","interfaces":[{"interface":"back","average":{"1min":1.6879999999999999,"5min":2.052,"15min":1.544},"min":{"1min":0.379,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":5.6070000000000002,"5min":15.038,"15min":18.417000000000002},"last":0.56100000000000005},{"interface":"front","average":{"1min":1.3100000000000001,"5min":1.9039999999999999,"15min":1.4399999999999999},"min":{"1min":0.378,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":4.3710000000000004,"5min":15.369999999999999,"15min":15.369999999999999},"last":0.371}]}]},{"osd":6,"up_from":47,"seq":201863463372,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1170020,"kb_used_data":657564,"kb_used_omap":0,"kb_used_meta":512448,"kb_avail":92575132,"statfs":{"total":95995035648,"available":94796935168,"internally_reserved":0,"allocated":673345536,"data_stored":671701860,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":524746752},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:49:05 2023","interfaces":[{"interface":"back","average":{"1min":2.5409999999999999,"5min":2.0819999999999999,"15min":1.978},"min":{"1min":0.24399999999999999,"5min":0.219,"15min":0.219},"max":{"1min":12.442,"5min":21.018000000000001,"15min":34.143999999999998},"last":0.89600000000000002},{"interface":"front","average":{"1min":2.2919999999999998,"5min":2.2200000000000002,"15min":2.2360000000000002},"min":{"1min":0.30299999999999999,"5min":0.255,"15min":0.22700000000000001},"max":{"1min":11.557,"5min":21.082000000000001,"15min":60.328000000000003},"last":0.48299999999999998}]},{"osd":1,"last update":"Sat Sep 23 20:48:20 2023","interfaces":[{"interface":"back","average":{"1min":3.0390000000000001,"5min":2.2309999999999999,"15min":2.0819999999999999},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.249},"max":{"1min":8.9350000000000005,"5min":9.2520000000000007,"15min":64.274000000000001},"last":0.97199999999999998},{"interface":"front","average":{"1min":2.746,"5min":1.9730000000000001,"15min":1.869},"min":{"1min":0.32300000000000001,"5min":0.29099999999999998,"15min":0.28000000000000003},"max":{"1min":8.0969999999999995,"5min":17.786000000000001,"15min":34.490000000000002},"last":2.8860000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:49:05 2023","interfaces":[{"interface":"back","average":{"1min":3.6360000000000001,"5min":2.6520000000000001,"15min":2.3010000000000002},"min":{"1min":0.36199999999999999,"5min":0.29699999999999999,"15min":0.27700000000000002},"max":{"1min":9.0730000000000004,"5min":22.591000000000001,"15min":34.317999999999998},"last":6.9169999999999998},{"interface":"front","average":{"1min":2.8170000000000002,"5min":2.198,"15min":2.0779999999999998},"min":{"1min":0.42999999999999999,"5min":0.33900000000000002,"15min":0.30399999999999999},"max":{"1min":12.798,"5min":22.614999999999998,"15min":34.292999999999999},"last":6.7999999999999998}]},{"osd":3,"last update":"Sat Sep 23 20:48:56 2023","interfaces":[{"interface":"back","average":{"1min":2.5659999999999998,"5min":2.0920000000000001,"15min":1.9650000000000001},"min":{"1min":0.46999999999999997,"5min":0.307,"15min":0.28399999999999997},"max":{"1min":7.9640000000000004,"5min":11.664,"15min":11.664},"last":0.44700000000000001},{"interface":"front","average":{"1min":3.1840000000000002,"5min":2.335,"15min":2.1970000000000001},"min":{"1min":0.50600000000000001,"5min":0.32700000000000001,"15min":0.30499999999999999},"max":{"1min":11.6,"5min":11.6,"15min":12.875999999999999},"last":8.6140000000000008}]},{"osd":4,"last update":"Sat Sep 23 20:48:20 2023","interfaces":[{"interface":"back","average":{"1min":1.923,"5min":1.129,"15min":0.94699999999999995},"min":{"1min":0.29599999999999999,"5min":0.28699999999999998,"15min":0.25600000000000001},"max":{"1min":8.1229999999999993,"5min":10.183999999999999,"15min":23.904},"last":0.64200000000000002},{"interface":"front","average":{"1min":2.1499999999999999,"5min":1.405,"15min":1.179},"min":{"1min":0.377,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":9.7189999999999994,"5min":9.7189999999999994,"15min":11.515000000000001},"last":0.36299999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:48:20 2023","interfaces":[{"interface":"back","average":{"1min":2.6419999999999999,"5min":1.4039999999999999,"15min":1.0029999999999999},"min":{"1min":0.371,"5min":0.253,"15min":0.253},"max":{"1min":8.0440000000000005,"5min":8.0440000000000005,"15min":8.0440000000000005},"last":0.67400000000000004},{"interface":"front","average":{"1min":1.6699999999999999,"5min":1.548,"15min":1.2729999999999999},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":5.1900000000000004,"5min":9.3559999999999999,"15min":9.3559999999999999},"last":0.63200000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:48:56 2023","interfaces":[{"interface":"back","average":{"1min":1.2729999999999999,"5min":1.2549999999999999,"15min":0.997},"min":{"1min":0.42899999999999999,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":8.0719999999999992,"5min":8.9309999999999992,"15min":8.9309999999999992},"last":1.7889999999999999},{"interface":"front","average":{"1min":10.34,"5min":3.6499999999999999,"15min":1.9890000000000001},"min":{"1min":0.47099999999999997,"5min":0.35999999999999999,"15min":0.34499999999999997},"max":{"1min":159.91999999999999,"5min":159.91999999999999,"15min":159.91999999999999},"last":0.66400000000000003}]}]},{"osd":1,"up_from":266,"seq":1142461300971,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1143540,"kb_used_data":366764,"kb_used_omap":8,"kb_used_meta":776759,"kb_avail":92601612,"statfs":{"total":95995035648,"available":94824050688,"internally_reserved":0,"allocated":375566336,"data_stored":372535565,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8206,"internal_metadata":795402226},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:48:24 2023","interfaces":[{"interface":"back","average":{"1min":2.9710000000000001,"5min":2.4969999999999999,"15min":2.3559999999999999},"min":{"1min":0.27700000000000002,"5min":0.20799999999999999,"15min":0.20000000000000001},"max":{"1min":9.4730000000000008,"5min":15.532999999999999,"15min":16.292000000000002},"last":2.2090000000000001},{"interface":"front","average":{"1min":3.1469999999999998,"5min":2.9060000000000001,"15min":2.9430000000000001},"min":{"1min":0.35899999999999999,"5min":0.22600000000000001,"15min":0.19900000000000001},"max":{"1min":8.1839999999999993,"5min":20.346,"15min":26.588000000000001},"last":1.141}]},{"osd":2,"last update":"Sat Sep 23 20:48:24 2023","interfaces":[{"interface":"back","average":{"1min":3.1850000000000001,"5min":2.8340000000000001,"15min":3.1429999999999998},"min":{"1min":0.249,"5min":0.23699999999999999,"15min":0.216},"max":{"1min":9.5779999999999994,"5min":19.026,"15min":32.255000000000003},"last":2.3799999999999999},{"interface":"front","average":{"1min":3.5030000000000001,"5min":2.423,"15min":2.3769999999999998},"min":{"1min":0.315,"5min":0.26000000000000001,"15min":0.20899999999999999},"max":{"1min":10.567,"5min":10.567,"15min":13.404999999999999},"last":4.7389999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:48:24 2023","interfaces":[{"interface":"back","average":{"1min":3.294,"5min":2.2719999999999998,"15min":2.4780000000000002},"min":{"1min":0.373,"5min":0.28799999999999998,"15min":0.247},"max":{"1min":10.135999999999999,"5min":17.366,"15min":17.366},"last":2.23},{"interface":"front","average":{"1min":3.2170000000000001,"5min":2.8490000000000002,"15min":2.7949999999999999},"min":{"1min":0.26300000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":9.8339999999999996,"5min":11.534000000000001,"15min":19.315000000000001},"last":0.57999999999999996}]},{"osd":4,"last update":"Sat Sep 23 20:48:24 2023","interfaces":[{"interface":"back","average":{"1min":2.7909999999999999,"5min":2.3769999999999998,"15min":2.4079999999999999},"min":{"1min":0.34799999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":9.5359999999999996,"5min":11.791,"15min":24.327999999999999},"last":1.9930000000000001},{"interface":"front","average":{"1min":2.5600000000000001,"5min":1.7949999999999999,"15min":1.7749999999999999},"min":{"1min":0.36299999999999999,"5min":0.34699999999999998,"15min":0.311},"max":{"1min":9.8249999999999993,"5min":10.468999999999999,"15min":20.835999999999999},"last":2.2469999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:48:24 2023","interfaces":[{"interface":"back","average":{"1min":2.4470000000000001,"5min":2.3889999999999998,"15min":2.3660000000000001},"min":{"1min":0.41399999999999998,"5min":0.308,"15min":0.308},"max":{"1min":8.1210000000000004,"5min":14.366,"15min":33.401000000000003},"last":0.66700000000000004},{"interface":"front","average":{"1min":2.589,"5min":2.548,"15min":2.633},"min":{"1min":0.39700000000000002,"5min":0.33100000000000002,"15min":0.32500000000000001},"max":{"1min":9.6140000000000008,"5min":11.750999999999999,"15min":33.417000000000002},"last":2.073}]},{"osd":6,"last update":"Sat Sep 23 20:48:24 2023","interfaces":[{"interface":"back","average":{"1min":2.5049999999999999,"5min":2.3650000000000002,"15min":2.2570000000000001},"min":{"1min":0.48799999999999999,"5min":0.316,"15min":0.316},"max":{"1min":8.0960000000000001,"5min":14.414999999999999,"15min":14.414999999999999},"last":0.70799999999999996},{"interface":"front","average":{"1min":3.2650000000000001,"5min":3.004,"15min":2.6739999999999999},"min":{"1min":0.432,"5min":0.38500000000000001,"15min":0.317},"max":{"1min":9.6639999999999997,"5min":11.832000000000001,"15min":24.376000000000001},"last":2.1070000000000002}]},{"osd":7,"last update":"Sat Sep 23 20:48:24 2023","interfaces":[{"interface":"back","average":{"1min":2.8330000000000002,"5min":2.5219999999999998,"15min":2.4249999999999998},"min":{"1min":0.46899999999999997,"5min":0.377,"15min":0.37},"max":{"1min":8.1590000000000007,"5min":10.010999999999999,"15min":18.437000000000001},"last":0.629},{"interface":"front","average":{"1min":3.5739999999999998,"5min":2.871,"15min":2.714},"min":{"1min":0.47299999999999998,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":9.8369999999999997,"5min":11.864000000000001,"15min":24.524000000000001},"last":2.032}]}]},{"osd":0,"up_from":94,"seq":403726926173,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1808304,"kb_used_data":861480,"kb_used_omap":7,"kb_used_meta":946808,"kb_avail":91936848,"statfs":{"total":95995035648,"available":94143332352,"internally_reserved":0,"allocated":882155520,"data_stored":879289648,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":969531409},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:48:32 2023","interfaces":[{"interface":"back","average":{"1min":4.1180000000000003,"5min":3.851,"15min":3.0579999999999998},"min":{"1min":0.34799999999999998,"5min":0.184,"15min":0.17199999999999999},"max":{"1min":9.6449999999999996,"5min":152.387,"15min":152.387},"last":0.46700000000000003},{"interface":"front","average":{"1min":3.7109999999999999,"5min":3.9390000000000001,"15min":3.2149999999999999},"min":{"1min":0.218,"5min":0.218,"15min":0.20399999999999999},"max":{"1min":12.317,"5min":152.84100000000001,"15min":152.84100000000001},"last":0.70099999999999996}]},{"osd":2,"last update":"Sat Sep 23 20:49:07 2023","interfaces":[{"interface":"back","average":{"1min":1.7829999999999999,"5min":2.7879999999999998,"15min":2.867},"min":{"1min":0.252,"5min":0.23499999999999999,"15min":0.218},"max":{"1min":7.008,"5min":15.108000000000001,"15min":62.726999999999997},"last":0.51700000000000002},{"interface":"front","average":{"1min":1.913,"5min":2.706,"15min":2.7930000000000001},"min":{"1min":0.219,"5min":0.217,"15min":0.217},"max":{"1min":9.5609999999999999,"5min":17.013000000000002,"15min":62.689},"last":0.42499999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:48:40 2023","interfaces":[{"interface":"back","average":{"1min":3.8650000000000002,"5min":3.6709999999999998,"15min":3.2229999999999999},"min":{"1min":0.36499999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":14.494,"5min":55.017000000000003,"15min":55.017000000000003},"last":0.80000000000000004},{"interface":"front","average":{"1min":3.6339999999999999,"5min":3.161,"15min":3.234},"min":{"1min":0.38,"5min":0.23799999999999999,"15min":0.21199999999999999},"max":{"1min":11.547000000000001,"5min":55.665999999999997,"15min":55.665999999999997},"last":0.505}]},{"osd":4,"last update":"Sat Sep 23 20:49:07 2023","interfaces":[{"interface":"back","average":{"1min":2.1120000000000001,"5min":2.1589999999999998,"15min":2.0859999999999999},"min":{"1min":0.308,"5min":0.29299999999999998,"15min":0.28899999999999998},"max":{"1min":9.5779999999999994,"5min":11.75,"15min":11.75},"last":0.59199999999999997},{"interface":"front","average":{"1min":1.77,"5min":2.133,"15min":1.9710000000000001},"min":{"1min":0.41599999999999998,"5min":0.31,"15min":0.27400000000000002},"max":{"1min":6.1050000000000004,"5min":12.356999999999999,"15min":12.356999999999999},"last":0.621}]},{"osd":5,"last update":"Sat Sep 23 20:49:07 2023","interfaces":[{"interface":"back","average":{"1min":1.8520000000000001,"5min":2.052,"15min":2.0880000000000001},"min":{"1min":0.38300000000000001,"5min":0.308,"15min":0.26500000000000001},"max":{"1min":6.1210000000000004,"5min":9.3339999999999996,"15min":11.497999999999999},"last":0.57699999999999996},{"interface":"front","average":{"1min":1.5569999999999999,"5min":2.137,"15min":2.2959999999999998},"min":{"1min":0.36899999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":4.9640000000000004,"5min":12.601000000000001,"15min":26.228000000000002},"last":0.66000000000000003}]},{"osd":6,"last update":"Sat Sep 23 20:49:07 2023","interfaces":[{"interface":"back","average":{"1min":2.1629999999999998,"5min":2.3029999999999999,"15min":2.2629999999999999},"min":{"1min":0.45200000000000001,"5min":0.374,"15min":0.34499999999999997},"max":{"1min":9.0570000000000004,"5min":9.2750000000000004,"15min":24.056000000000001},"last":0.60499999999999998},{"interface":"front","average":{"1min":1.9359999999999999,"5min":2.327,"15min":2.2010000000000001},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":9.0380000000000003,"5min":12.698,"15min":26.268000000000001},"last":3.8370000000000002}]},{"osd":7,"last update":"Sat Sep 23 20:49:07 2023","interfaces":[{"interface":"back","average":{"1min":1.607,"5min":2.512,"15min":2.3839999999999999},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":5.0369999999999999,"5min":16.553999999999998,"15min":26.177},"last":3.8919999999999999},{"interface":"front","average":{"1min":1.8420000000000001,"5min":2.4529999999999998,"15min":2.4009999999999998},"min":{"1min":0.41699999999999998,"5min":0.40200000000000002,"15min":0.34200000000000003},"max":{"1min":7.8559999999999999,"5min":16.649000000000001,"15min":16.649000000000001},"last":2.9510000000000001}]}]},{"osd":2,"up_from":104,"seq":446676599121,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1414040,"kb_used_data":556048,"kb_used_omap":11,"kb_used_meta":857972,"kb_avail":92331112,"statfs":{"total":95995035648,"available":94547058688,"internally_reserved":0,"allocated":569393152,"data_stored":566552623,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11878,"internal_metadata":878563738},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,0,2,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:49:11 2023","interfaces":[{"interface":"back","average":{"1min":2.2679999999999998,"5min":1.8140000000000001,"15min":2.036},"min":{"1min":0.19500000000000001,"5min":0.19500000000000001,"15min":0.17299999999999999},"max":{"1min":20.48,"5min":20.48,"15min":28.234999999999999},"last":0.223},{"interface":"front","average":{"1min":1.8919999999999999,"5min":1.9139999999999999,"15min":2.2410000000000001},"min":{"1min":0.25700000000000001,"5min":0.245,"15min":0.19},"max":{"1min":6.7220000000000004,"5min":15.342000000000001,"15min":16.678999999999998},"last":1.407}]},{"osd":1,"last update":"Sat Sep 23 20:49:11 2023","interfaces":[{"interface":"back","average":{"1min":1.409,"5min":2.1480000000000001,"15min":2.3889999999999998},"min":{"1min":0.28199999999999997,"5min":0.219,"15min":0.219},"max":{"1min":10.324,"5min":21.690000000000001,"15min":46.405000000000001},"last":0.502},{"interface":"front","average":{"1min":2.3119999999999998,"5min":1.9550000000000001,"15min":2.4340000000000002},"min":{"1min":0.29499999999999998,"5min":0.22,"15min":0.22},"max":{"1min":9.0690000000000008,"5min":13.215999999999999,"15min":46.459000000000003},"last":0.48699999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:48:40 2023","interfaces":[{"interface":"back","average":{"1min":2.1509999999999998,"5min":2.516,"15min":2.605},"min":{"1min":0.311,"5min":0.253,"15min":0.253},"max":{"1min":10.051,"5min":30.283000000000001,"15min":30.283000000000001},"last":1.911},{"interface":"front","average":{"1min":2.6840000000000002,"5min":1.8320000000000001,"15min":2.0499999999999998},"min":{"1min":0.33000000000000002,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":10.221,"5min":30.834,"15min":30.834},"last":5.585}]},{"osd":4,"last update":"Sat Sep 23 20:49:11 2023","interfaces":[{"interface":"back","average":{"1min":1.623,"5min":1.9299999999999999,"15min":1.8420000000000001},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.27500000000000002},"max":{"1min":5.556,"5min":11.141999999999999,"15min":12.025},"last":1.462},{"interface":"front","average":{"1min":2.069,"5min":2.0099999999999998,"15min":1.982},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.30499999999999999},"max":{"1min":12.662000000000001,"5min":12.662000000000001,"15min":19.440999999999999},"last":0.33100000000000002}]},{"osd":5,"last update":"Sat Sep 23 20:49:11 2023","interfaces":[{"interface":"back","average":{"1min":1.9219999999999999,"5min":1.7150000000000001,"15min":1.8680000000000001},"min":{"1min":0.38400000000000001,"5min":0.33200000000000002,"15min":0.311},"max":{"1min":6.5460000000000003,"5min":11.183,"15min":12.07},"last":1.498},{"interface":"front","average":{"1min":1.863,"5min":1.8999999999999999,"15min":1.9870000000000001},"min":{"1min":0.373,"5min":0.373,"15min":0.35299999999999998},"max":{"1min":10.831,"5min":11.061999999999999,"15min":19.513000000000002},"last":0.373}]},{"osd":6,"last update":"Sat Sep 23 20:49:11 2023","interfaces":[{"interface":"back","average":{"1min":1.8740000000000001,"5min":2.0760000000000001,"15min":2.0830000000000002},"min":{"1min":0.40699999999999997,"5min":0.34799999999999998,"15min":0.33500000000000002},"max":{"1min":10.791,"5min":11.031000000000001,"15min":19.486000000000001},"last":0.40699999999999997},{"interface":"front","average":{"1min":1.829,"5min":1.609,"15min":1.6020000000000001},"min":{"1min":0.40699999999999997,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":4.9589999999999996,"5min":8.5920000000000005,"15min":11.747},"last":0.45900000000000002}]},{"osd":7,"last update":"Sat Sep 23 20:49:11 2023","interfaces":[{"interface":"back","average":{"1min":3.7509999999999999,"5min":1.978,"15min":1.6950000000000001},"min":{"1min":0.378,"5min":0.35399999999999998,"15min":0.34699999999999998},"max":{"1min":20.530000000000001,"5min":20.530000000000001,"15min":20.530000000000001},"last":0.437},{"interface":"front","average":{"1min":1.752,"5min":2.1160000000000001,"15min":2.121},"min":{"1min":0.36899999999999999,"5min":0.35799999999999998,"15min":0.34899999999999998},"max":{"1min":5.3259999999999996,"5min":11.218,"15min":12.099},"last":1.5700000000000001}]}]},{"osd":3,"up_from":109,"seq":468151435594,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1551464,"kb_used_data":500960,"kb_used_omap":4,"kb_used_meta":1050491,"kb_avail":92193688,"statfs":{"total":95995035648,"available":94406336512,"internally_reserved":0,"allocated":512983040,"data_stored":510165113,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4514,"internal_metadata":1075703390},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:48:48 2023","interfaces":[{"interface":"back","average":{"1min":1.4750000000000001,"5min":2.3799999999999999,"15min":2.4790000000000001},"min":{"1min":0.252,"5min":0.217,"15min":0.20899999999999999},"max":{"1min":5.2110000000000003,"5min":21.792999999999999,"15min":22.32},"last":4.0350000000000001},{"interface":"front","average":{"1min":1.454,"5min":2.3340000000000001,"15min":2.5289999999999999},"min":{"1min":0.249,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":5.4530000000000003,"5min":18.579999999999998,"15min":22.805},"last":3.3010000000000002}]},{"osd":1,"last update":"Sat Sep 23 20:48:19 2023","interfaces":[{"interface":"back","average":{"1min":2.7530000000000001,"5min":2.1709999999999998,"15min":2.0840000000000001},"min":{"1min":0.26100000000000001,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":11.298,"5min":15.177,"15min":17.529},"last":3.9889999999999999},{"interface":"front","average":{"1min":2.7669999999999999,"5min":2.0830000000000002,"15min":2.2490000000000001},"min":{"1min":0.27200000000000002,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":11.380000000000001,"5min":11.380000000000001,"15min":14.19},"last":0.73099999999999998}]},{"osd":2,"last update":"Sat Sep 23 20:48:48 2023","interfaces":[{"interface":"back","average":{"1min":1.669,"5min":2.403,"15min":2.8650000000000002},"min":{"1min":0.27900000000000003,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":5.4210000000000003,"5min":12.473000000000001,"15min":68.206999999999994},"last":1.099},{"interface":"front","average":{"1min":2.7810000000000001,"5min":2.1200000000000001,"15min":2.5659999999999998},"min":{"1min":0.374,"5min":0.251,"15min":0.245},"max":{"1min":16.559000000000001,"5min":17.919,"15min":67.736999999999995},"last":1.9510000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:48:48 2023","interfaces":[{"interface":"back","average":{"1min":1.8200000000000001,"5min":1.744,"15min":1.708},"min":{"1min":0.435,"5min":0.35499999999999998,"15min":0.30499999999999999},"max":{"1min":5.8029999999999999,"5min":9.0259999999999998,"15min":22.157},"last":5.9480000000000004},{"interface":"front","average":{"1min":3.2530000000000001,"5min":1.8759999999999999,"15min":2.1869999999999998},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.29399999999999998},"max":{"1min":16.529,"5min":16.529,"15min":16.529},"last":0.40200000000000002}]},{"osd":5,"last update":"Sat Sep 23 20:48:48 2023","interfaces":[{"interface":"back","average":{"1min":1.734,"5min":1.849,"15min":1.921},"min":{"1min":0.49299999999999999,"5min":0.33800000000000002,"15min":0.32300000000000001},"max":{"1min":6.7210000000000001,"5min":8.9849999999999994,"15min":22.289999999999999},"last":1.0740000000000001},{"interface":"front","average":{"1min":2.25,"5min":2.0470000000000002,"15min":1.831},"min":{"1min":0.43099999999999999,"5min":0.34100000000000003,"15min":0.32000000000000001},"max":{"1min":6.008,"5min":8.6129999999999995,"15min":12.952},"last":0.496}]},{"osd":6,"last update":"Sat Sep 23 20:48:48 2023","interfaces":[{"interface":"back","average":{"1min":2.6699999999999999,"5min":2.4660000000000002,"15min":2.036},"min":{"1min":0.54200000000000004,"5min":0.36599999999999999,"15min":0.34300000000000003},"max":{"1min":7.4359999999999999,"5min":8.6790000000000003,"15min":12.917},"last":0.52700000000000002},{"interface":"front","average":{"1min":2.633,"5min":2.0270000000000001,"15min":2.2290000000000001},"min":{"1min":0.39400000000000002,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":16.497,"5min":16.497,"15min":16.497},"last":0.44700000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:48:48 2023","interfaces":[{"interface":"back","average":{"1min":3.173,"5min":2.1219999999999999,"15min":2.4300000000000002},"min":{"1min":0.48199999999999998,"5min":0.42399999999999999,"15min":0.39900000000000002},"max":{"1min":16.425000000000001,"5min":16.425000000000001,"15min":16.425000000000001},"last":0.47899999999999998},{"interface":"front","average":{"1min":3.3330000000000002,"5min":2.3399999999999999,"15min":2.0899999999999999},"min":{"1min":0.56100000000000005,"5min":0.46300000000000002,"15min":0.40999999999999998},"max":{"1min":11.891999999999999,"5min":11.891999999999999,"15min":22.202999999999999},"last":1.115}]}]},{"osd":4,"up_from":30,"seq":128849019348,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1372836,"kb_used_data":818908,"kb_used_omap":0,"kb_used_meta":553920,"kb_avail":92372316,"statfs":{"total":95995035648,"available":94589251584,"internally_reserved":0,"allocated":838561792,"data_stored":836917850,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":567214080},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:49:04 2023","interfaces":[{"interface":"back","average":{"1min":5.3239999999999998,"5min":2.403,"15min":2.0779999999999998},"min":{"1min":0.219,"5min":0.219,"15min":0.219},"max":{"1min":73.272000000000006,"5min":73.272000000000006,"15min":73.272000000000006},"last":0.64500000000000002},{"interface":"front","average":{"1min":3.0369999999999999,"5min":2.1890000000000001,"15min":2.1259999999999999},"min":{"1min":0.28299999999999997,"5min":0.25800000000000001,"15min":0.246},"max":{"1min":29.754000000000001,"5min":29.754000000000001,"15min":29.754000000000001},"last":1.756}]},{"osd":1,"last update":"Sat Sep 23 20:48:16 2023","interfaces":[{"interface":"back","average":{"1min":2.093,"5min":1.9370000000000001,"15min":1.8280000000000001},"min":{"1min":0.27800000000000002,"5min":0.26300000000000001,"15min":0.25900000000000001},"max":{"1min":10.157,"5min":10.157,"15min":18.190999999999999},"last":1.2509999999999999},{"interface":"front","average":{"1min":2.278,"5min":1.6160000000000001,"15min":1.6319999999999999},"min":{"1min":0.29399999999999998,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":10.169,"5min":10.169,"15min":18.173999999999999},"last":0.68999999999999995}]},{"osd":2,"last update":"Sat Sep 23 20:48:16 2023","interfaces":[{"interface":"back","average":{"1min":2.0880000000000001,"5min":1.879,"15min":1.752},"min":{"1min":0.32000000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":4.5739999999999998,"5min":9.3320000000000007,"15min":13.051},"last":0.72899999999999998},{"interface":"front","average":{"1min":2.089,"5min":2.0550000000000002,"15min":2.1699999999999999},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":7.0800000000000001,"5min":8.7230000000000008,"15min":14.614000000000001},"last":1.794}]},{"osd":3,"last update":"Sat Sep 23 20:48:32 2023","interfaces":[{"interface":"back","average":{"1min":2.7749999999999999,"5min":2.0139999999999998,"15min":1.9530000000000001},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":11.598000000000001,"5min":11.598000000000001,"15min":12.147},"last":0.81100000000000005},{"interface":"front","average":{"1min":2.3530000000000002,"5min":2.1150000000000002,"15min":2.0870000000000002},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.30299999999999999},"max":{"1min":7.7830000000000004,"5min":17.169,"15min":17.169},"last":0.84899999999999998}]},{"osd":5,"last update":"Sat Sep 23 20:49:04 2023","interfaces":[{"interface":"back","average":{"1min":2.8650000000000002,"5min":1.595,"15min":1.2},"min":{"1min":0.25800000000000001,"5min":0.253,"15min":0.253},"max":{"1min":16.940000000000001,"5min":16.940000000000001,"15min":16.940000000000001},"last":1.214},{"interface":"front","average":{"1min":1.3580000000000001,"5min":1.536,"15min":1.1930000000000001},"min":{"1min":0.311,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":4.915,"5min":8.1440000000000001,"15min":8.1440000000000001},"last":1.9610000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:48:16 2023","interfaces":[{"interface":"back","average":{"1min":1.9910000000000001,"5min":1.6140000000000001,"15min":1.395},"min":{"1min":0.30099999999999999,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":6.4290000000000003,"5min":10.840999999999999,"15min":14.731999999999999},"last":1.9299999999999999},{"interface":"front","average":{"1min":1.681,"5min":1.6359999999999999,"15min":1.234},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":8.0269999999999992,"5min":8.0269999999999992,"15min":8.0269999999999992},"last":1.736}]},{"osd":7,"last update":"Sat Sep 23 20:48:32 2023","interfaces":[{"interface":"back","average":{"1min":1.7070000000000001,"5min":4.9939999999999998,"15min":2.7130000000000001},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":11.228,"5min":313.50099999999998,"15min":313.50099999999998},"last":2.927},{"interface":"front","average":{"1min":1.8300000000000001,"5min":4.4589999999999996,"15min":2.2919999999999998},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":17.093,"5min":312.67500000000001,"15min":312.67500000000001},"last":1.28}]}]},{"osd":5,"up_from":39,"seq":167503725009,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1239452,"kb_used_data":632468,"kb_used_omap":0,"kb_used_meta":606976,"kb_avail":92505700,"statfs":{"total":95995035648,"available":94725836800,"internally_reserved":0,"allocated":647647232,"data_stored":646008088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":621543424},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:48:10 2023","interfaces":[{"interface":"back","average":{"1min":2.5129999999999999,"5min":1.8700000000000001,"15min":1.891},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.20699999999999999},"max":{"1min":6.673,"5min":15.018000000000001,"15min":18.16},"last":3.117},{"interface":"front","average":{"1min":2.258,"5min":1.915,"15min":1.8300000000000001},"min":{"1min":0.30099999999999999,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":8.6050000000000004,"5min":15.034000000000001,"15min":15.034000000000001},"last":3.097}]},{"osd":1,"last update":"Sat Sep 23 20:48:23 2023","interfaces":[{"interface":"back","average":{"1min":2.4460000000000002,"5min":2.1480000000000001,"15min":2.3719999999999999},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.253},"max":{"1min":6.9850000000000003,"5min":9.4199999999999999,"15min":61.005000000000003},"last":0.57199999999999995},{"interface":"front","average":{"1min":2.6720000000000002,"5min":1.9390000000000001,"15min":2.2269999999999999},"min":{"1min":0.38200000000000001,"5min":0.27700000000000002,"15min":0.23499999999999999},"max":{"1min":6.6280000000000001,"5min":16.219000000000001,"15min":59.688000000000002},"last":9.0399999999999991}]},{"osd":2,"last update":"Sat Sep 23 20:48:10 2023","interfaces":[{"interface":"back","average":{"1min":2.6160000000000001,"5min":2.1400000000000001,"15min":2.1579999999999999},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26500000000000001},"max":{"1min":7.194,"5min":12.226000000000001,"15min":37.011000000000003},"last":2.1589999999999998},{"interface":"front","average":{"1min":2.5219999999999998,"5min":1.867,"15min":2.0350000000000001},"min":{"1min":0.39600000000000002,"5min":0.314,"15min":0.255},"max":{"1min":5.9790000000000001,"5min":9.8260000000000005,"15min":36.835000000000001},"last":1.069}]},{"osd":3,"last update":"Sat Sep 23 20:48:30 2023","interfaces":[{"interface":"back","average":{"1min":2.9380000000000002,"5min":1.9650000000000001,"15min":2.3170000000000002},"min":{"1min":0.54100000000000004,"5min":0.33800000000000002,"15min":0.314},"max":{"1min":8.7360000000000007,"5min":8.7360000000000007,"15min":36.716000000000001},"last":1.147},{"interface":"front","average":{"1min":3.3719999999999999,"5min":2.0019999999999998,"15min":2.3730000000000002},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":7.4029999999999996,"5min":7.8869999999999996,"15min":18.504000000000001},"last":2.1339999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:49:01 2023","interfaces":[{"interface":"back","average":{"1min":2.137,"5min":1.4810000000000001,"15min":1.3240000000000001},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.255},"max":{"1min":8.9390000000000001,"5min":8.9390000000000001,"15min":13.391999999999999},"last":0.34799999999999998},{"interface":"front","average":{"1min":1.883,"5min":1.631,"15min":1.3200000000000001},"min":{"1min":0.31900000000000001,"5min":0.29199999999999998,"15min":0.28299999999999997},"max":{"1min":8.0030000000000001,"5min":8.0030000000000001,"15min":8.0030000000000001},"last":2.1080000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:48:11 2023","interfaces":[{"interface":"back","average":{"1min":1.6319999999999999,"5min":1.913,"15min":1.431},"min":{"1min":0.40300000000000002,"5min":0.36199999999999999,"15min":0.30199999999999999},"max":{"1min":6.6959999999999997,"5min":13.734,"15min":13.734},"last":0.50600000000000001},{"interface":"front","average":{"1min":2.278,"5min":1.5129999999999999,"15min":1.28},"min":{"1min":0.40699999999999997,"5min":0.317,"15min":0.317},"max":{"1min":7.5270000000000001,"5min":7.5270000000000001,"15min":8.7449999999999992},"last":2.0779999999999998}]},{"osd":7,"last update":"Sat Sep 23 20:48:30 2023","interfaces":[{"interface":"back","average":{"1min":2.4689999999999999,"5min":4.5060000000000002,"15min":2.2490000000000001},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.28499999999999998},"max":{"1min":7.899,"5min":294.70999999999998,"15min":294.70999999999998},"last":2.0019999999999998},{"interface":"front","average":{"1min":2.1240000000000001,"5min":4.5860000000000003,"15min":2.3149999999999999},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.27900000000000003},"max":{"1min":14.032999999999999,"5min":294.74400000000003,"15min":294.74400000000003},"last":0.41699999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4722,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1360,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8206,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7156,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3154,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":870977536,"data_stored":870973464,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":352321536,"data_stored":352321536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":545193984,"data_stored":545193984,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":504954880,"data_stored":504954880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":826216448,"data_stored":826212376,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":641007616,"data_stored":641007616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":662441984,"data_stored":662437912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":544931840,"data_stored":544931840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:49:14.784 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd pool set unique_pool_5 pgp_num 26 2023-09-23T20:49:14.832 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:49:14.926 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:49:14.952 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:15.007 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:14 smithi018 ceph-mon[179647]: 10.4 scrub starts 2023-09-23T20:49:15.007 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:14 smithi018 ceph-mon[179647]: failed to encode map e611 with expected crc 2023-09-23T20:49:15.007 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:14 smithi018 ceph-mon[179647]: 10.10 deep-scrub ok 2023-09-23T20:49:15.007 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:14 smithi018 ceph-mon[179647]: from='client.46952 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:49:15.007 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:14 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:49:15.007 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:14 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:49:15.007 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:14 smithi018 ceph-mon[179647]: failed to encode map e612 with expected crc 2023-09-23T20:49:15.007 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:14 smithi018 ceph-mon[179647]: osdmap e612: 8 total, 8 up, 8 in 2023-09-23T20:49:15.008 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:49:14 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:49:14] "GET /metrics HTTP/1.1" 200 36610 "" "Prometheus/2.43.0" 2023-09-23T20:49:15.008 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:14 smithi018 ceph-mon[184144]: 10.4 scrub starts 2023-09-23T20:49:15.008 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:14 smithi018 ceph-mon[184144]: failed to encode map e611 with expected crc 2023-09-23T20:49:15.008 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:14 smithi018 ceph-mon[184144]: 10.10 deep-scrub ok 2023-09-23T20:49:15.008 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:14 smithi018 ceph-mon[184144]: from='client.46952 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:49:15.008 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:14 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:49:15.008 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:14 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:49:15.008 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:14 smithi018 ceph-mon[184144]: failed to encode map e612 with expected crc 2023-09-23T20:49:15.009 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:14 smithi018 ceph-mon[184144]: osdmap e612: 8 total, 8 up, 8 in 2023-09-23T20:49:15.123 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:49:15.147 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:15.147 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:49:15.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:14 smithi149 ceph-mon[170394]: 10.4 scrub starts 2023-09-23T20:49:15.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:14 smithi149 ceph-mon[170394]: failed to encode map e611 with expected crc 2023-09-23T20:49:15.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:14 smithi149 ceph-mon[170394]: 10.10 deep-scrub ok 2023-09-23T20:49:15.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:14 smithi149 ceph-mon[170394]: from='client.46952 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:49:15.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:14 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:49:15.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:14 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:49:15.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:14 smithi149 ceph-mon[170394]: failed to encode map e612 with expected crc 2023-09-23T20:49:15.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:14 smithi149 ceph-mon[170394]: osdmap e612: 8 total, 8 up, 8 in 2023-09-23T20:49:15.248 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:49:15.289 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:15.342 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:49:15.350 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:15.351 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:49:15.452 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:49:15.490 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:15.549 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:49:15.555 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:15.555 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:49:15.656 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:49:15.712 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:15.767 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:49:15.770 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:15.771 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:49:15.871 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:49:15.918 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:15.969 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:49:15.974 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:15.974 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:49:16.075 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:49:16.092 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2023-09-23T20:49:16.092 INFO:teuthology.orchestra.run.smithi018.stdout: "size": 20, 2023-09-23T20:49:16.092 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 600, 2023-09-23T20:49:16.092 INFO:teuthology.orchestra.run.smithi018.stdout: "ops": [ 2023-09-23T20:49:16.092 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.092 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.36450.0:184506 9.5 9:a146981f:::benchmark_data_smithi018_381516_object70672:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e499)", 2023-09-23T20:49:16.092 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:44:14.824550+0000", 2023-09-23T20:49:16.093 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 301.25888799400002, 2023-09-23T20:49:16.093 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.2118222780000001, 2023-09-23T20:49:16.093 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:49:16.093 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:49:16.093 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:49:16.093 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.36450", 2023-09-23T20:49:16.093 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:49:16.093 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 184506 2023-09-23T20:49:16.093 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.093 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:49:16.093 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.093 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:49:16.094 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:14.824550+0000", 2023-09-23T20:49:16.094 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:16.094 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.094 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.094 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:49:16.094 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:14.824550+0000", 2023-09-23T20:49:16.094 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:16.094 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.094 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.094 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:49:16.094 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:14.824549+0000", 2023-09-23T20:49:16.095 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.999999 2023-09-23T20:49:16.095 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.095 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.095 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:49:16.095 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:14.824555+0000", 2023-09-23T20:49:16.095 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.8760000000000002e-06 2023-09-23T20:49:16.095 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.095 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.095 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:49:16.095 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:14.824568+0000", 2023-09-23T20:49:16.095 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2935000000000001e-05 2023-09-23T20:49:16.096 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.096 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.096 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:49:16.096 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:14.824574+0000", 2023-09-23T20:49:16.096 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.4919999999999998e-06 2023-09-23T20:49:16.096 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.096 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.096 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:16.096 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:14.824643+0000", 2023-09-23T20:49:16.096 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.8361999999999994e-05 2023-09-23T20:49:16.096 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.096 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.097 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:49:16.097 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:14.824657+0000", 2023-09-23T20:49:16.097 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.4028e-05 2023-09-23T20:49:16.097 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.097 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.097 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:16.097 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:15.592296+0000", 2023-09-23T20:49:16.097 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.76763938499999995 2023-09-23T20:49:16.097 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.097 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.097 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:49:16.097 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:15.592301+0000", 2023-09-23T20:49:16.098 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.4020000000000004e-06 2023-09-23T20:49:16.098 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.098 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.098 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:16.098 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:16.949618+0000", 2023-09-23T20:49:16.098 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.357317181 2023-09-23T20:49:16.098 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.098 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.098 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:49:16.098 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:16.949720+0000", 2023-09-23T20:49:16.098 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00010254 2023-09-23T20:49:16.098 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.099 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.099 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,5,7", 2023-09-23T20:49:16.099 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:16.949860+0000", 2023-09-23T20:49:16.099 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000139949 2023-09-23T20:49:16.099 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.099 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.099 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:16.099 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:16.953148+0000", 2023-09-23T20:49:16.099 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0032880510000000002 2023-09-23T20:49:16.099 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.099 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.099 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:49:16.100 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:16.956352+0000", 2023-09-23T20:49:16.100 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00320332 2023-09-23T20:49:16.100 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.100 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.100 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:16.100 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:16.959319+0000", 2023-09-23T20:49:16.100 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0029674480000000001 2023-09-23T20:49:16.100 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.100 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.100 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:16.100 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:17.036194+0000", 2023-09-23T20:49:16.100 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.076874858000000004 2023-09-23T20:49:16.101 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.101 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.101 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:49:16.101 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:17.036247+0000", 2023-09-23T20:49:16.101 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.3541000000000001e-05 2023-09-23T20:49:16.101 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.101 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.101 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:49:16.101 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:17.036372+0000", 2023-09-23T20:49:16.101 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00012490400000000001 2023-09-23T20:49:16.101 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:16.101 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:49:16.102 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:16.102 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.102 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.102 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.36450.0:184546 9.5 9:a68f34be:::benchmark_data_smithi018_381516_object70712:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e499)", 2023-09-23T20:49:16.102 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:44:14.892257+0000", 2023-09-23T20:49:16.102 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 301.19118075400002, 2023-09-23T20:49:16.102 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.1623236050000001, 2023-09-23T20:49:16.102 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:49:16.102 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:49:16.102 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:49:16.102 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.36450", 2023-09-23T20:49:16.103 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:49:16.103 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 184546 2023-09-23T20:49:16.103 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.103 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:49:16.103 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.103 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:49:16.103 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:14.892257+0000", 2023-09-23T20:49:16.103 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:16.103 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.103 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.103 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:49:16.103 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:14.892257+0000", 2023-09-23T20:49:16.104 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:16.104 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.104 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.104 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:49:16.104 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:14.892256+0000", 2023-09-23T20:49:16.104 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.999999 2023-09-23T20:49:16.104 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.104 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.104 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:49:16.104 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:14.892262+0000", 2023-09-23T20:49:16.104 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.9499999999999998e-06 2023-09-23T20:49:16.104 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.105 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.105 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:49:16.105 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:14.892275+0000", 2023-09-23T20:49:16.105 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3095000000000001e-05 2023-09-23T20:49:16.105 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.105 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.105 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:49:16.105 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:14.892282+0000", 2023-09-23T20:49:16.105 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.4590000000000003e-06 2023-09-23T20:49:16.105 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.105 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.105 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:16.105 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:14.892342+0000", 2023-09-23T20:49:16.106 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.0606000000000002e-05 2023-09-23T20:49:16.106 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.106 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.106 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:49:16.106 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:14.892353+0000", 2023-09-23T20:49:16.107 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0473e-05 2023-09-23T20:49:16.107 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.107 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.107 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:16.107 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:15.592314+0000", 2023-09-23T20:49:16.107 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.69996096100000005 2023-09-23T20:49:16.107 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.107 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.107 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:49:16.107 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:15.592318+0000", 2023-09-23T20:49:16.107 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.3019999999999996e-06 2023-09-23T20:49:16.108 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.108 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.108 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:16.108 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:16.953179+0000", 2023-09-23T20:49:16.108 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.360860688 2023-09-23T20:49:16.108 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.108 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.108 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:49:16.108 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:16.953274+0000", 2023-09-23T20:49:16.109 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.5508999999999996e-05 2023-09-23T20:49:16.109 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.109 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.109 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,5,7", 2023-09-23T20:49:16.109 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:16.953355+0000", 2023-09-23T20:49:16.109 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.1069000000000006e-05 2023-09-23T20:49:16.109 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.109 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.109 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:16.109 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:16.959355+0000", 2023-09-23T20:49:16.109 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0059993210000000002 2023-09-23T20:49:16.109 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.110 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.110 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:16.110 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:16.959383+0000", 2023-09-23T20:49:16.110 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.8750000000000001e-05 2023-09-23T20:49:16.110 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.110 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.110 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:49:16.110 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:16.959392+0000", 2023-09-23T20:49:16.110 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.9150000000000002e-06 2023-09-23T20:49:16.110 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.110 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.110 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:16.111 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:17.054479+0000", 2023-09-23T20:49:16.111 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.095086526000000005 2023-09-23T20:49:16.111 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.111 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.111 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:49:16.111 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:17.054514+0000", 2023-09-23T20:49:16.111 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.5233e-05 2023-09-23T20:49:16.111 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.111 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.111 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:49:16.111 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:17.054581+0000", 2023-09-23T20:49:16.111 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.6817999999999996e-05 2023-09-23T20:49:16.112 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:16.112 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:49:16.112 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:16.112 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.112 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.112 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.36450.0:184566 9.5 9:a74e2b6e:::benchmark_data_smithi018_381516_object70732:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e499)", 2023-09-23T20:49:16.112 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:44:14.928018+0000", 2023-09-23T20:49:16.112 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 301.15541984399999, 2023-09-23T20:49:16.112 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.1268187200000002, 2023-09-23T20:49:16.112 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:49:16.112 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:49:16.112 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:49:16.113 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.36450", 2023-09-23T20:49:16.113 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:49:16.113 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 184566 2023-09-23T20:49:16.113 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.113 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:49:16.113 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.113 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:49:16.113 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:14.928018+0000", 2023-09-23T20:49:16.113 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:16.113 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.113 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.114 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:49:16.114 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:14.928018+0000", 2023-09-23T20:49:16.114 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:16.114 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.114 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.114 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:49:16.114 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:14.928017+0000", 2023-09-23T20:49:16.114 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.999999 2023-09-23T20:49:16.114 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.114 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.114 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:49:16.114 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:14.928022+0000", 2023-09-23T20:49:16.115 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.3030000000000001e-06 2023-09-23T20:49:16.115 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.115 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.115 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:49:16.115 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:14.928036+0000", 2023-09-23T20:49:16.115 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.4301000000000001e-05 2023-09-23T20:49:16.115 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.115 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.115 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:49:16.115 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:14.928044+0000", 2023-09-23T20:49:16.115 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.7910000000000005e-06 2023-09-23T20:49:16.115 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.116 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.116 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:16.116 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:14.928101+0000", 2023-09-23T20:49:16.116 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.6833000000000003e-05 2023-09-23T20:49:16.116 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.116 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.116 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:49:16.116 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:14.928108+0000", 2023-09-23T20:49:16.116 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.1860000000000003e-06 2023-09-23T20:49:16.116 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.116 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.116 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:16.116 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:15.592331+0000", 2023-09-23T20:49:16.117 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.66422334999999999 2023-09-23T20:49:16.117 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.117 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.117 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:49:16.117 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:15.592335+0000", 2023-09-23T20:49:16.117 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.1459999999999998e-06 2023-09-23T20:49:16.117 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.117 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.117 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:16.117 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:16.956397+0000", 2023-09-23T20:49:16.117 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.364061236 2023-09-23T20:49:16.117 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.118 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.118 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:49:16.118 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:16.956485+0000", 2023-09-23T20:49:16.118 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.8812000000000003e-05 2023-09-23T20:49:16.118 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.118 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.118 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,5,7", 2023-09-23T20:49:16.118 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:16.956564+0000", 2023-09-23T20:49:16.118 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.8295999999999997e-05 2023-09-23T20:49:16.118 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.118 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.119 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:16.119 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:16.959420+0000", 2023-09-23T20:49:16.119 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0028558390000000002 2023-09-23T20:49:16.119 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.119 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.119 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:16.119 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:16.959454+0000", 2023-09-23T20:49:16.119 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.4842999999999998e-05 2023-09-23T20:49:16.119 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.119 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.119 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:49:16.119 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:16.959460+0000", 2023-09-23T20:49:16.120 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.9900000000000002e-06 2023-09-23T20:49:16.120 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.120 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.120 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:16.120 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:17.054646+0000", 2023-09-23T20:49:16.120 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.095185290000000006 2023-09-23T20:49:16.120 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.120 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.120 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:49:16.120 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:17.054702+0000", 2023-09-23T20:49:16.120 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.6428000000000003e-05 2023-09-23T20:49:16.120 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.121 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.121 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:49:16.121 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:17.054837+0000", 2023-09-23T20:49:16.121 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000134905 2023-09-23T20:49:16.121 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:16.121 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:49:16.121 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:16.121 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.121 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.121 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.36450.0:184588 9.5 9:a2d0a08d:::benchmark_data_smithi018_381516_object70754:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e499)", 2023-09-23T20:49:16.121 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:44:14.976865+0000", 2023-09-23T20:49:16.121 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 301.10657310599998, 2023-09-23T20:49:16.121 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.0781069649999999, 2023-09-23T20:49:16.122 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:49:16.122 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:49:16.122 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:49:16.122 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.36450", 2023-09-23T20:49:16.122 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:49:16.122 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 184588 2023-09-23T20:49:16.122 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.122 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:49:16.122 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.122 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:49:16.122 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:14.976865+0000", 2023-09-23T20:49:16.123 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:16.123 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.123 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.123 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:49:16.123 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:14.976865+0000", 2023-09-23T20:49:16.123 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:16.123 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.123 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.123 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:49:16.123 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:14.976864+0000", 2023-09-23T20:49:16.123 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:49:16.123 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.124 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.124 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:49:16.124 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:14.976869+0000", 2023-09-23T20:49:16.124 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.7269999999999998e-06 2023-09-23T20:49:16.124 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.124 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.124 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:49:16.124 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:14.976886+0000", 2023-09-23T20:49:16.124 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.7263000000000001e-05 2023-09-23T20:49:16.124 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.124 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.124 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:49:16.125 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:14.976893+0000", 2023-09-23T20:49:16.125 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.6980000000000001e-06 2023-09-23T20:49:16.125 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.125 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.125 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:16.125 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:14.976948+0000", 2023-09-23T20:49:16.125 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.4929999999999998e-05 2023-09-23T20:49:16.125 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.125 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.125 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:49:16.125 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:14.976955+0000", 2023-09-23T20:49:16.126 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.2010000000000002e-06 2023-09-23T20:49:16.126 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.126 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.126 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:16.126 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:15.592349+0000", 2023-09-23T20:49:16.126 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.61539343499999999 2023-09-23T20:49:16.126 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.126 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.126 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:49:16.126 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:15.592353+0000", 2023-09-23T20:49:16.126 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.1729999999999997e-06 2023-09-23T20:49:16.127 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.127 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.127 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:16.127 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:16.959483+0000", 2023-09-23T20:49:16.127 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3671305140000001 2023-09-23T20:49:16.127 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.127 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.127 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:49:16.127 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:16.959572+0000", 2023-09-23T20:49:16.127 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.8596000000000004e-05 2023-09-23T20:49:16.128 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.128 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.128 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,5,7", 2023-09-23T20:49:16.128 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:16.959683+0000", 2023-09-23T20:49:16.128 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00011069399999999999 2023-09-23T20:49:16.128 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.128 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.128 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:16.128 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:16.966267+0000", 2023-09-23T20:49:16.128 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0065843899999999999 2023-09-23T20:49:16.128 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.128 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.129 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:16.129 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:16.966303+0000", 2023-09-23T20:49:16.129 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.6446000000000003e-05 2023-09-23T20:49:16.129 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.129 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.129 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:49:16.129 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:16.966315+0000", 2023-09-23T20:49:16.129 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1832e-05 2023-09-23T20:49:16.129 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.129 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.129 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:16.129 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:17.054875+0000", 2023-09-23T20:49:16.130 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.088559631999999999 2023-09-23T20:49:16.130 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.130 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.130 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:49:16.130 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:17.054907+0000", 2023-09-23T20:49:16.130 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.26e-05 2023-09-23T20:49:16.130 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.130 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.130 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:49:16.130 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:17.054972+0000", 2023-09-23T20:49:16.130 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.4548000000000006e-05 2023-09-23T20:49:16.130 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:16.131 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:49:16.131 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:16.131 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.131 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.131 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.36450.0:184607 9.5 9:a772fb9a:::benchmark_data_smithi018_381516_object70773:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e499)", 2023-09-23T20:49:16.131 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:44:15.055033+0000", 2023-09-23T20:49:16.131 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 301.02840488700002, 2023-09-23T20:49:16.131 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.0001805909999999, 2023-09-23T20:49:16.131 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:49:16.131 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:49:16.131 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:49:16.131 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.36450", 2023-09-23T20:49:16.132 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:49:16.132 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 184607 2023-09-23T20:49:16.132 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.132 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:49:16.132 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.132 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:49:16.132 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:15.055033+0000", 2023-09-23T20:49:16.132 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:16.132 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.132 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.132 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:49:16.133 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:15.055033+0000", 2023-09-23T20:49:16.133 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:16.133 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.133 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.133 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:49:16.133 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:15.055032+0000", 2023-09-23T20:49:16.133 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.999999 2023-09-23T20:49:16.133 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.133 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.133 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:49:16.133 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:15.055038+0000", 2023-09-23T20:49:16.133 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.5809999999999996e-06 2023-09-23T20:49:16.134 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.134 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.134 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:49:16.134 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:15.055050+0000", 2023-09-23T20:49:16.134 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2177e-05 2023-09-23T20:49:16.134 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.134 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.134 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:49:16.134 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:15.055059+0000", 2023-09-23T20:49:16.134 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.9250000000000001e-06 2023-09-23T20:49:16.134 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.134 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.135 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:16.135 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:15.055112+0000", 2023-09-23T20:49:16.135 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.3664e-05 2023-09-23T20:49:16.135 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.135 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.135 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:49:16.135 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:15.055122+0000", 2023-09-23T20:49:16.135 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.3719999999999994e-06 2023-09-23T20:49:16.135 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.135 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.135 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:16.135 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:15.592375+0000", 2023-09-23T20:49:16.135 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.53725269799999997 2023-09-23T20:49:16.136 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.136 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.136 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:49:16.136 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:15.592379+0000", 2023-09-23T20:49:16.136 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.3069999999999996e-06 2023-09-23T20:49:16.136 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.136 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.136 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:16.136 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:16.963186+0000", 2023-09-23T20:49:16.136 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3708071880000001 2023-09-23T20:49:16.136 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.136 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.137 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:49:16.137 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:16.963293+0000", 2023-09-23T20:49:16.137 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000107261 2023-09-23T20:49:16.137 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.137 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.137 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,5,7", 2023-09-23T20:49:16.137 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:16.963378+0000", 2023-09-23T20:49:16.137 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.4697000000000006e-05 2023-09-23T20:49:16.137 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.137 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.137 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:16.137 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:16.966356+0000", 2023-09-23T20:49:16.138 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0029781360000000002 2023-09-23T20:49:16.138 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.138 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.138 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:49:16.138 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:16.969355+0000", 2023-09-23T20:49:16.138 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0029992320000000001 2023-09-23T20:49:16.138 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.138 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.138 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:16.138 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:16.971996+0000", 2023-09-23T20:49:16.138 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0026408809999999999 2023-09-23T20:49:16.138 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.139 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.139 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:16.139 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:17.055024+0000", 2023-09-23T20:49:16.139 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.083027693999999999 2023-09-23T20:49:16.139 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.139 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.139 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:49:16.139 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:17.055082+0000", 2023-09-23T20:49:16.141 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.8431999999999999e-05 2023-09-23T20:49:16.141 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.141 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.142 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:49:16.142 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:17.055214+0000", 2023-09-23T20:49:16.142 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00013142500000000001 2023-09-23T20:49:16.142 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:16.142 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:49:16.142 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:16.142 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.142 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.142 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:158979 10.8 10:1589b43a:::benchmark_data_smithi018_422431_object54483:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e576)", 2023-09-23T20:49:16.142 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:48:22.730411+0000", 2023-09-23T20:49:16.142 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 53.353026960999998, 2023-09-23T20:49:16.142 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.3890539199999998, 2023-09-23T20:49:16.143 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:49:16.143 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:49:16.143 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:49:16.143 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:49:16.143 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:49:16.143 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 158979 2023-09-23T20:49:16.143 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.143 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:49:16.143 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.143 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:49:16.143 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:22.730411+0000", 2023-09-23T20:49:16.143 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:16.144 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.144 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.144 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:49:16.144 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:22.730411+0000", 2023-09-23T20:49:16.144 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:16.144 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.144 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.144 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:49:16.144 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:22.730410+0000", 2023-09-23T20:49:16.144 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.999999 2023-09-23T20:49:16.144 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.144 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.145 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:49:16.145 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:22.730414+0000", 2023-09-23T20:49:16.145 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.109e-06 2023-09-23T20:49:16.145 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.145 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.145 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:49:16.145 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:22.730424+0000", 2023-09-23T20:49:16.145 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.2520000000000005e-06 2023-09-23T20:49:16.145 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.145 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.145 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:49:16.145 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:22.730429+0000", 2023-09-23T20:49:16.146 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.6509999999999998e-06 2023-09-23T20:49:16.146 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.146 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.146 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:16.146 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:22.730483+0000", 2023-09-23T20:49:16.146 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.4091000000000001e-05 2023-09-23T20:49:16.146 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.146 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.146 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:49:16.146 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:22.730621+0000", 2023-09-23T20:49:16.146 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00013734700000000001 2023-09-23T20:49:16.146 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.147 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.147 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 2,7", 2023-09-23T20:49:16.147 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:22.730751+0000", 2023-09-23T20:49:16.147 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00013077499999999999 2023-09-23T20:49:16.147 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.147 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.147 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:49:16.147 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:22.734229+0000", 2023-09-23T20:49:16.147 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0034771089999999999 2023-09-23T20:49:16.147 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.147 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.147 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:16.148 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:22.740683+0000", 2023-09-23T20:49:16.148 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0064545640000000003 2023-09-23T20:49:16.148 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.148 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.148 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:16.148 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:22.909613+0000", 2023-09-23T20:49:16.148 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.16892990499999999 2023-09-23T20:49:16.148 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.148 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.148 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:49:16.148 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:22.909628+0000", 2023-09-23T20:49:16.148 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.5009e-05 2023-09-23T20:49:16.149 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.149 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.149 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:16.149 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:23.910394+0000", 2023-09-23T20:49:16.149 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.000766058 2023-09-23T20:49:16.149 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.149 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.149 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:49:16.149 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:23.910409+0000", 2023-09-23T20:49:16.149 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.4759e-05 2023-09-23T20:49:16.149 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.149 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.150 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:16.150 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:25.119411+0000", 2023-09-23T20:49:16.150 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2090017129999999 2023-09-23T20:49:16.150 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.150 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.150 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:49:16.150 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:25.119465+0000", 2023-09-23T20:49:16.150 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.4551e-05 2023-09-23T20:49:16.150 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:16.150 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:49:16.150 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:16.151 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.151 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.151 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:159004 10.8 10:149861b2:::benchmark_data_smithi018_422431_object54508:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e576)", 2023-09-23T20:49:16.151 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:48:22.757923+0000", 2023-09-23T20:49:16.151 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 53.325515211999999, 2023-09-23T20:49:16.151 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.3618294230000001, 2023-09-23T20:49:16.151 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:49:16.151 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:49:16.151 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:49:16.151 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:49:16.151 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:49:16.151 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 159004 2023-09-23T20:49:16.152 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.152 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:49:16.152 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.152 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:49:16.152 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:22.757923+0000", 2023-09-23T20:49:16.152 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:16.152 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.152 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.152 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:49:16.152 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:22.757923+0000", 2023-09-23T20:49:16.152 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:16.153 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.153 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.153 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:49:16.153 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:22.757921+0000", 2023-09-23T20:49:16.153 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:49:16.153 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.153 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.153 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:49:16.153 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:22.757928+0000", 2023-09-23T20:49:16.153 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.8000000000000001e-06 2023-09-23T20:49:16.153 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.153 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.154 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:49:16.154 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:22.757944+0000", 2023-09-23T20:49:16.154 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.579e-05 2023-09-23T20:49:16.154 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.154 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.154 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:49:16.154 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:22.757953+0000", 2023-09-23T20:49:16.154 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.7010000000000002e-06 2023-09-23T20:49:16.154 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.154 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.154 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:16.154 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:22.758005+0000", 2023-09-23T20:49:16.155 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.1992e-05 2023-09-23T20:49:16.155 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.155 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.155 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:49:16.155 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:22.758153+0000", 2023-09-23T20:49:16.155 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000148271 2023-09-23T20:49:16.155 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.155 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.155 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 2,7", 2023-09-23T20:49:16.155 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:22.758283+0000", 2023-09-23T20:49:16.155 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00013028900000000001 2023-09-23T20:49:16.155 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.156 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.156 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:49:16.156 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:22.761207+0000", 2023-09-23T20:49:16.156 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0029240820000000002 2023-09-23T20:49:16.156 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.156 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.156 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:16.156 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:22.762382+0000", 2023-09-23T20:49:16.156 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0011748170000000001 2023-09-23T20:49:16.156 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.156 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.156 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:16.157 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:22.909766+0000", 2023-09-23T20:49:16.157 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.14738415699999999 2023-09-23T20:49:16.157 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.157 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.157 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:49:16.157 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:22.909778+0000", 2023-09-23T20:49:16.157 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1225e-05 2023-09-23T20:49:16.157 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.157 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.157 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:16.157 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:23.910441+0000", 2023-09-23T20:49:16.157 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0006638699999999 2023-09-23T20:49:16.158 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.158 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.158 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:49:16.158 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:23.910453+0000", 2023-09-23T20:49:16.158 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1289e-05 2023-09-23T20:49:16.158 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.158 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.158 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:16.158 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:25.119698+0000", 2023-09-23T20:49:16.158 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.209245656 2023-09-23T20:49:16.158 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.159 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.159 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:49:16.159 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:25.119752+0000", 2023-09-23T20:49:16.159 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.3925999999999998e-05 2023-09-23T20:49:16.159 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:16.159 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:49:16.159 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:16.159 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.159 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.159 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:159020 10.8 10:1adf69c9:::benchmark_data_smithi018_422431_object54524:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e576)", 2023-09-23T20:49:16.159 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:48:22.783684+0000", 2023-09-23T20:49:16.160 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 53.299754499999999, 2023-09-23T20:49:16.160 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.3361663099999999, 2023-09-23T20:49:16.160 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:49:16.160 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:49:16.160 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:49:16.160 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:49:16.160 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:49:16.160 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 159020 2023-09-23T20:49:16.160 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.160 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:49:16.160 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.160 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:49:16.161 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:22.783684+0000", 2023-09-23T20:49:16.161 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:16.161 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.161 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.161 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:49:16.161 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:22.783684+0000", 2023-09-23T20:49:16.161 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:16.161 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.161 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.161 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:49:16.161 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:22.783682+0000", 2023-09-23T20:49:16.161 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:49:16.162 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.162 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.162 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:49:16.162 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:22.783689+0000", 2023-09-23T20:49:16.162 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.2620000000000001e-06 2023-09-23T20:49:16.162 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.162 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.162 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:49:16.162 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:22.783702+0000", 2023-09-23T20:49:16.162 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2138e-05 2023-09-23T20:49:16.162 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.162 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.163 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:49:16.163 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:22.783709+0000", 2023-09-23T20:49:16.163 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.4209999999999998e-06 2023-09-23T20:49:16.163 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.163 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.163 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:16.163 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:22.783766+0000", 2023-09-23T20:49:16.163 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.7367999999999997e-05 2023-09-23T20:49:16.163 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.163 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.163 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:49:16.163 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:22.783925+0000", 2023-09-23T20:49:16.164 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000158572 2023-09-23T20:49:16.164 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.164 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.164 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 2,7", 2023-09-23T20:49:16.164 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:22.784081+0000", 2023-09-23T20:49:16.164 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00015649899999999999 2023-09-23T20:49:16.164 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.164 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.164 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:49:16.164 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:22.787814+0000", 2023-09-23T20:49:16.164 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0037330029999999999 2023-09-23T20:49:16.164 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.164 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.165 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:16.165 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:22.788179+0000", 2023-09-23T20:49:16.165 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00036480600000000002 2023-09-23T20:49:16.165 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.165 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.165 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:16.165 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:22.910223+0000", 2023-09-23T20:49:16.165 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.122043727 2023-09-23T20:49:16.165 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.165 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.165 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:49:16.165 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:22.910240+0000", 2023-09-23T20:49:16.166 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.6914000000000001e-05 2023-09-23T20:49:16.166 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.166 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.166 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:16.166 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:23.910481+0000", 2023-09-23T20:49:16.166 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0002412919999999 2023-09-23T20:49:16.166 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.166 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.166 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:49:16.166 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:23.910490+0000", 2023-09-23T20:49:16.166 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.1269999999999992e-06 2023-09-23T20:49:16.166 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.166 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.167 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:16.167 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:25.119799+0000", 2023-09-23T20:49:16.167 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2093082669999999 2023-09-23T20:49:16.167 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.167 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.167 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:49:16.167 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:25.119850+0000", 2023-09-23T20:49:16.167 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.1309e-05 2023-09-23T20:49:16.167 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:16.167 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:49:16.167 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:16.167 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.168 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.168 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:180387 10.11 10:8ef816cc:::benchmark_data_smithi018_422431_object75891:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e600)", 2023-09-23T20:49:16.168 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:48:58.712914+0000", 2023-09-23T20:49:16.168 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 17.370523787, 2023-09-23T20:49:16.168 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.0734866240000001, 2023-09-23T20:49:16.168 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:49:16.168 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:49:16.168 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:49:16.168 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:49:16.168 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:49:16.168 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 180387 2023-09-23T20:49:16.168 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.169 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:49:16.169 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.169 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:49:16.169 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:58.712914+0000", 2023-09-23T20:49:16.169 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:16.169 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.169 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.169 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:49:16.169 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:58.712914+0000", 2023-09-23T20:49:16.169 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:16.169 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.169 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.169 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:49:16.170 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:58.712913+0000", 2023-09-23T20:49:16.170 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.999999 2023-09-23T20:49:16.170 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.170 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.170 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:49:16.170 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:58.712919+0000", 2023-09-23T20:49:16.170 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.6679999999999998e-06 2023-09-23T20:49:16.170 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.170 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.170 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:49:16.170 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:58.712934+0000", 2023-09-23T20:49:16.170 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.4987000000000001e-05 2023-09-23T20:49:16.171 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.171 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.171 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:49:16.171 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:58.712941+0000", 2023-09-23T20:49:16.171 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.3209999999999999e-06 2023-09-23T20:49:16.171 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.171 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.171 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:16.171 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:58.713002+0000", 2023-09-23T20:49:16.171 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.0412999999999999e-05 2023-09-23T20:49:16.171 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.171 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.172 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:49:16.172 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:58.713015+0000", 2023-09-23T20:49:16.172 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3027000000000001e-05 2023-09-23T20:49:16.172 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.172 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.172 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:16.172 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:59.700486+0000", 2023-09-23T20:49:16.172 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.98747148399999995 2023-09-23T20:49:16.172 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.172 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.172 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:49:16.172 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:59.700501+0000", 2023-09-23T20:49:16.173 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.4762e-05 2023-09-23T20:49:16.173 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.173 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.173 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:16.173 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:00.773077+0000", 2023-09-23T20:49:16.173 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0725756799999999 2023-09-23T20:49:16.173 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.173 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.173 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:49:16.173 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:00.773234+0000", 2023-09-23T20:49:16.173 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00015717699999999999 2023-09-23T20:49:16.173 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.174 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.174 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 4,5,7", 2023-09-23T20:49:16.174 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:00.773316+0000", 2023-09-23T20:49:16.174 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.2064999999999994e-05 2023-09-23T20:49:16.174 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.174 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.174 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:16.174 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:00.778594+0000", 2023-09-23T20:49:16.174 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0052777429999999997 2023-09-23T20:49:16.174 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.174 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.174 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:16.175 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:00.778804+0000", 2023-09-23T20:49:16.175 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00020998699999999999 2023-09-23T20:49:16.175 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.175 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.175 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:16.175 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:00.778839+0000", 2023-09-23T20:49:16.175 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.54e-05 2023-09-23T20:49:16.175 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.175 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.175 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:49:16.175 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:00.786192+0000", 2023-09-23T20:49:16.176 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0073527189999999997 2023-09-23T20:49:16.176 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.176 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.176 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:49:16.176 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:00.786257+0000", 2023-09-23T20:49:16.176 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.5802000000000005e-05 2023-09-23T20:49:16.176 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.176 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.176 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:49:16.176 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:00.786401+0000", 2023-09-23T20:49:16.176 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00014349799999999999 2023-09-23T20:49:16.176 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:16.177 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:49:16.177 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:16.177 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.177 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.177 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:180519 10.11 10:8b2af5ea:::benchmark_data_smithi018_422431_object76023:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e600)", 2023-09-23T20:49:16.177 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:48:58.816375+0000", 2023-09-23T20:49:16.177 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 17.267063178000001, 2023-09-23T20:49:16.177 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.970327392, 2023-09-23T20:49:16.177 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:49:16.177 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:49:16.177 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:49:16.177 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:49:16.178 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:49:16.178 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 180519 2023-09-23T20:49:16.178 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.178 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:49:16.178 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.178 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:49:16.178 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:58.816375+0000", 2023-09-23T20:49:16.178 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:16.178 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.178 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.178 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:49:16.179 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:58.816375+0000", 2023-09-23T20:49:16.179 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:16.179 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.179 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.179 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:49:16.179 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:58.816374+0000", 2023-09-23T20:49:16.179 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:49:16.179 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.179 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.179 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:49:16.179 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:58.816380+0000", 2023-09-23T20:49:16.180 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.4969999999999998e-06 2023-09-23T20:49:16.180 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.180 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.180 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:49:16.180 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:58.816397+0000", 2023-09-23T20:49:16.180 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.6776e-05 2023-09-23T20:49:16.180 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.180 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.180 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:49:16.180 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:58.816403+0000", 2023-09-23T20:49:16.180 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.4960000000000001e-06 2023-09-23T20:49:16.180 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.181 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.181 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:16.181 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:58.816454+0000", 2023-09-23T20:49:16.181 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.0507000000000003e-05 2023-09-23T20:49:16.181 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.181 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.181 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:49:16.181 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:58.816461+0000", 2023-09-23T20:49:16.181 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.2439999999999996e-06 2023-09-23T20:49:16.181 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.181 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.181 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:16.182 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:59.700560+0000", 2023-09-23T20:49:16.182 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.88409922100000005 2023-09-23T20:49:16.182 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.182 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.182 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:49:16.182 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:59.700571+0000", 2023-09-23T20:49:16.182 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0468999999999999e-05 2023-09-23T20:49:16.182 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.182 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.182 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:16.182 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:00.775047+0000", 2023-09-23T20:49:16.182 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.074476134 2023-09-23T20:49:16.183 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.183 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.183 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:49:16.183 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:00.775163+0000", 2023-09-23T20:49:16.183 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00011577400000000001 2023-09-23T20:49:16.183 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.183 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.183 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 4,5,7", 2023-09-23T20:49:16.183 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:00.775250+0000", 2023-09-23T20:49:16.183 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.7783999999999998e-05 2023-09-23T20:49:16.183 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.183 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.183 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:16.184 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:00.778877+0000", 2023-09-23T20:49:16.184 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0036264470000000001 2023-09-23T20:49:16.184 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.184 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.184 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:16.184 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:00.778914+0000", 2023-09-23T20:49:16.184 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.7364999999999999e-05 2023-09-23T20:49:16.184 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.184 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.184 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:16.184 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:00.783033+0000", 2023-09-23T20:49:16.184 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.004118923 2023-09-23T20:49:16.185 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.185 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.185 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:49:16.185 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:00.786531+0000", 2023-09-23T20:49:16.185 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0034973809999999999 2023-09-23T20:49:16.185 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.185 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.185 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:49:16.185 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:00.786587+0000", 2023-09-23T20:49:16.185 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.6509999999999999e-05 2023-09-23T20:49:16.185 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.185 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.186 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:49:16.186 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:00.786702+0000", 2023-09-23T20:49:16.186 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000115234 2023-09-23T20:49:16.186 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:16.186 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:49:16.186 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:16.186 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.186 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.186 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:180533 10.11 10:8f5ac416:::benchmark_data_smithi018_422431_object76037:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e600)", 2023-09-23T20:49:16.186 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:48:58.826901+0000", 2023-09-23T20:49:16.186 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 17.256537043000002, 2023-09-23T20:49:16.186 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.9600058059999999, 2023-09-23T20:49:16.187 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:49:16.187 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:49:16.187 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:49:16.187 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:49:16.187 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:49:16.187 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 180533 2023-09-23T20:49:16.187 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.187 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:49:16.187 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.187 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:49:16.187 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:58.826901+0000", 2023-09-23T20:49:16.187 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:16.187 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.188 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.188 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:49:16.188 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:58.826901+0000", 2023-09-23T20:49:16.188 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:16.188 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.188 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.188 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:49:16.188 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:58.826900+0000", 2023-09-23T20:49:16.188 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:49:16.188 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.188 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.188 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:49:16.189 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:58.826906+0000", 2023-09-23T20:49:16.189 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.1630000000000001e-06 2023-09-23T20:49:16.189 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.189 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.189 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:49:16.189 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:58.826932+0000", 2023-09-23T20:49:16.189 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.5738999999999999e-05 2023-09-23T20:49:16.189 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.189 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.189 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:49:16.189 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:58.826940+0000", 2023-09-23T20:49:16.189 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.9820000000000008e-06 2023-09-23T20:49:16.190 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.190 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.190 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:16.190 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:58.826996+0000", 2023-09-23T20:49:16.190 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.5795999999999999e-05 2023-09-23T20:49:16.190 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.190 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.190 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:49:16.190 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:58.827007+0000", 2023-09-23T20:49:16.190 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1864e-05 2023-09-23T20:49:16.190 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.190 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.191 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:16.191 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:59.700613+0000", 2023-09-23T20:49:16.191 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.87360592999999997 2023-09-23T20:49:16.191 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.191 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.191 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:49:16.191 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:59.700623+0000", 2023-09-23T20:49:16.191 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.9890000000000004e-06 2023-09-23T20:49:16.191 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.191 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.191 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:16.191 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:00.776736+0000", 2023-09-23T20:49:16.192 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.076112261 2023-09-23T20:49:16.192 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.192 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.192 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:49:16.192 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:00.776874+0000", 2023-09-23T20:49:16.192 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00013814799999999999 2023-09-23T20:49:16.192 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.192 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.192 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 4,5,7", 2023-09-23T20:49:16.192 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:00.776965+0000", 2023-09-23T20:49:16.192 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.0914000000000004e-05 2023-09-23T20:49:16.192 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.193 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.193 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:16.193 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:00.782854+0000", 2023-09-23T20:49:16.193 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0058890970000000003 2023-09-23T20:49:16.193 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.193 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.193 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:16.193 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:00.783008+0000", 2023-09-23T20:49:16.193 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00015429199999999999 2023-09-23T20:49:16.193 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.193 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.193 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:16.193 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:00.783052+0000", 2023-09-23T20:49:16.194 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.3480999999999999e-05 2023-09-23T20:49:16.194 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.194 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.194 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:49:16.194 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:00.786715+0000", 2023-09-23T20:49:16.194 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0036634290000000002 2023-09-23T20:49:16.194 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.194 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.194 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:49:16.194 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:00.786766+0000", 2023-09-23T20:49:16.194 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.0751000000000002e-05 2023-09-23T20:49:16.194 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.195 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.195 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:49:16.195 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:00.786907+0000", 2023-09-23T20:49:16.195 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000141175 2023-09-23T20:49:16.195 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:16.195 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:49:16.195 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:16.195 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.195 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.195 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:180641 10.11 10:89187964:::benchmark_data_smithi018_422431_object76145:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e600)", 2023-09-23T20:49:16.195 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:48:58.910309+0000", 2023-09-23T20:49:16.195 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 17.173128769000002, 2023-09-23T20:49:16.196 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.876699294, 2023-09-23T20:49:16.196 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:49:16.196 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:49:16.196 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:49:16.196 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:49:16.196 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:49:16.196 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 180641 2023-09-23T20:49:16.196 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.196 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:49:16.196 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.196 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:49:16.196 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:58.910309+0000", 2023-09-23T20:49:16.197 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:16.197 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.197 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.197 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:49:16.197 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:58.910309+0000", 2023-09-23T20:49:16.197 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:16.197 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.197 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.197 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:49:16.197 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:58.910308+0000", 2023-09-23T20:49:16.197 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.999999 2023-09-23T20:49:16.197 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.198 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.198 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:49:16.198 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:58.910314+0000", 2023-09-23T20:49:16.198 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.9100000000000002e-06 2023-09-23T20:49:16.198 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.198 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.198 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:49:16.198 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:58.910326+0000", 2023-09-23T20:49:16.198 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2228999999999999e-05 2023-09-23T20:49:16.198 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.198 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.198 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:49:16.199 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:58.910333+0000", 2023-09-23T20:49:16.199 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.5649999999999998e-06 2023-09-23T20:49:16.199 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.199 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.199 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:16.199 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:58.910393+0000", 2023-09-23T20:49:16.199 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.9901999999999997e-05 2023-09-23T20:49:16.199 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.199 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.199 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:49:16.199 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:58.910407+0000", 2023-09-23T20:49:16.199 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.4406999999999999e-05 2023-09-23T20:49:16.199 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.200 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.200 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:16.200 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:59.700665+0000", 2023-09-23T20:49:16.200 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.79025767300000005 2023-09-23T20:49:16.200 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.200 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.200 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:49:16.200 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:59.700675+0000", 2023-09-23T20:49:16.200 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.7459999999999995e-06 2023-09-23T20:49:16.200 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.200 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.200 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:16.201 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:00.778937+0000", 2023-09-23T20:49:16.201 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.078262091 2023-09-23T20:49:16.201 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.201 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.201 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:49:16.201 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:00.779091+0000", 2023-09-23T20:49:16.201 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00015477199999999999 2023-09-23T20:49:16.201 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.201 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.201 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 4,5,7", 2023-09-23T20:49:16.201 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:00.779225+0000", 2023-09-23T20:49:16.201 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00013385799999999999 2023-09-23T20:49:16.202 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.202 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.202 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:16.202 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:00.783070+0000", 2023-09-23T20:49:16.202 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0038443990000000001 2023-09-23T20:49:16.202 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.202 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.202 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:16.202 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:00.786174+0000", 2023-09-23T20:49:16.202 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0031037959999999998 2023-09-23T20:49:16.202 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.202 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.203 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:16.203 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:00.786453+0000", 2023-09-23T20:49:16.203 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00027965100000000001 2023-09-23T20:49:16.203 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.203 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.203 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:49:16.203 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:00.786915+0000", 2023-09-23T20:49:16.203 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00046204800000000001 2023-09-23T20:49:16.203 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.203 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.203 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:49:16.203 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:00.786942+0000", 2023-09-23T20:49:16.204 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.7188000000000001e-05 2023-09-23T20:49:16.204 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.204 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.204 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:49:16.204 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:00.787009+0000", 2023-09-23T20:49:16.204 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.6235000000000005e-05 2023-09-23T20:49:16.204 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:16.204 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:49:16.204 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:16.204 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.204 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.204 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:180659 10.11 10:8a8fdd45:::benchmark_data_smithi018_422431_object76163:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e600)", 2023-09-23T20:49:16.205 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:48:58.920902+0000", 2023-09-23T20:49:16.205 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 17.162535778999999, 2023-09-23T20:49:16.205 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.8663621770000001, 2023-09-23T20:49:16.205 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:49:16.205 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:49:16.205 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:49:16.205 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:49:16.205 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:49:16.205 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 180659 2023-09-23T20:49:16.205 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.205 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:49:16.205 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.206 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:49:16.206 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:58.920902+0000", 2023-09-23T20:49:16.206 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:16.206 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.206 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.206 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:49:16.206 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:58.920902+0000", 2023-09-23T20:49:16.206 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:16.206 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.206 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.206 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:49:16.206 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:58.920902+0000", 2023-09-23T20:49:16.207 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:49:16.207 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.207 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.207 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:49:16.207 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:58.920905+0000", 2023-09-23T20:49:16.207 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.6339999999999999e-06 2023-09-23T20:49:16.207 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.207 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.207 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:49:16.207 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:58.920913+0000", 2023-09-23T20:49:16.207 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.6140000000000003e-06 2023-09-23T20:49:16.207 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.208 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.208 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:49:16.208 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:58.920919+0000", 2023-09-23T20:49:16.208 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.2099999999999998e-06 2023-09-23T20:49:16.208 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.208 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.208 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:16.208 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:58.920965+0000", 2023-09-23T20:49:16.208 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.5952999999999999e-05 2023-09-23T20:49:16.208 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.208 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.208 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:49:16.209 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:58.920978+0000", 2023-09-23T20:49:16.209 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3029e-05 2023-09-23T20:49:16.209 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.209 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.209 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:16.209 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:59.700799+0000", 2023-09-23T20:49:16.209 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.77982121400000004 2023-09-23T20:49:16.209 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.209 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.209 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:49:16.209 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:59.700808+0000", 2023-09-23T20:49:16.209 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.3299999999999999e-06 2023-09-23T20:49:16.210 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.210 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.210 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:16.210 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:00.781013+0000", 2023-09-23T20:49:16.210 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.080204932 2023-09-23T20:49:16.210 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.210 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.210 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:49:16.210 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:00.781179+0000", 2023-09-23T20:49:16.210 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00016615 2023-09-23T20:49:16.210 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.211 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.211 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 4,5,7", 2023-09-23T20:49:16.211 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:00.781335+0000", 2023-09-23T20:49:16.211 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00015572399999999999 2023-09-23T20:49:16.211 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.211 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.211 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:16.211 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:00.783091+0000", 2023-09-23T20:49:16.211 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0017563609999999999 2023-09-23T20:49:16.211 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.211 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.211 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:16.212 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:00.786516+0000", 2023-09-23T20:49:16.212 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.003424702 2023-09-23T20:49:16.212 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.212 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.212 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:49:16.212 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:00.787016+0000", 2023-09-23T20:49:16.212 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00050087500000000002 2023-09-23T20:49:16.212 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.212 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.212 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:16.212 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:00.787083+0000", 2023-09-23T20:49:16.212 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.6840000000000004e-05 2023-09-23T20:49:16.213 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.213 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.213 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:49:16.213 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:00.787141+0000", 2023-09-23T20:49:16.213 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.7469000000000002e-05 2023-09-23T20:49:16.213 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.213 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.213 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:49:16.213 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:00.787265+0000", 2023-09-23T20:49:16.213 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00012374900000000001 2023-09-23T20:49:16.213 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:16.213 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:49:16.213 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:16.214 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.214 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.214 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:180678 10.11 10:8f278110:::benchmark_data_smithi018_422431_object76182:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e600)", 2023-09-23T20:49:16.214 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:48:58.938572+0000", 2023-09-23T20:49:16.214 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 17.144865620000001, 2023-09-23T20:49:16.214 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.857211787, 2023-09-23T20:49:16.214 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:49:16.214 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:49:16.214 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:49:16.214 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:49:16.214 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:49:16.214 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 180678 2023-09-23T20:49:16.215 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.215 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:49:16.215 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.215 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:49:16.215 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:58.938572+0000", 2023-09-23T20:49:16.215 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:16.215 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.215 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.215 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:49:16.215 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:58.938572+0000", 2023-09-23T20:49:16.215 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:16.215 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.216 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.216 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:49:16.216 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:58.938571+0000", 2023-09-23T20:49:16.216 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.999999 2023-09-23T20:49:16.216 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.216 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.216 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:49:16.216 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:58.938577+0000", 2023-09-23T20:49:16.216 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.0120000000000002e-06 2023-09-23T20:49:16.216 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.216 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.217 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:49:16.217 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:58.938589+0000", 2023-09-23T20:49:16.217 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1765e-05 2023-09-23T20:49:16.217 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.217 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.217 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:49:16.217 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:58.938595+0000", 2023-09-23T20:49:16.217 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.1380000000000004e-06 2023-09-23T20:49:16.217 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.217 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.217 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:16.217 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:58.938640+0000", 2023-09-23T20:49:16.218 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.4910000000000002e-05 2023-09-23T20:49:16.218 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.218 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.218 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:49:16.218 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:58.938650+0000", 2023-09-23T20:49:16.218 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.4660000000000005e-06 2023-09-23T20:49:16.218 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.218 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.218 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:16.218 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:59.700844+0000", 2023-09-23T20:49:16.218 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.76219446199999996 2023-09-23T20:49:16.218 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.219 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.219 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:49:16.219 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:59.700853+0000", 2023-09-23T20:49:16.219 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.2420000000000006e-06 2023-09-23T20:49:16.219 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.219 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.219 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:16.219 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:00.783109+0000", 2023-09-23T20:49:16.219 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0822551650000001 2023-09-23T20:49:16.219 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.219 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.219 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:49:16.220 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:00.783203+0000", 2023-09-23T20:49:16.220 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.4833000000000005e-05 2023-09-23T20:49:16.220 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.220 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.220 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 4,5,7", 2023-09-23T20:49:16.220 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:00.783284+0000", 2023-09-23T20:49:16.220 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.0612999999999995e-05 2023-09-23T20:49:16.220 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.220 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.220 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:49:16.220 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:00.787022+0000", 2023-09-23T20:49:16.220 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0037377600000000001 2023-09-23T20:49:16.221 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.221 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.221 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:16.221 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:00.787340+0000", 2023-09-23T20:49:16.221 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00031809699999999999 2023-09-23T20:49:16.221 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.221 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.221 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:16.221 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:00.795554+0000", 2023-09-23T20:49:16.221 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0082142789999999997 2023-09-23T20:49:16.221 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.221 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.222 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:16.222 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:00.795602+0000", 2023-09-23T20:49:16.222 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.8399000000000002e-05 2023-09-23T20:49:16.222 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.222 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.222 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:49:16.222 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:00.795651+0000", 2023-09-23T20:49:16.222 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.833e-05 2023-09-23T20:49:16.222 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.222 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.222 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:49:16.223 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:00.795784+0000", 2023-09-23T20:49:16.223 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000133468 2023-09-23T20:49:16.223 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:16.223 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:49:16.223 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:16.223 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.223 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.223 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:180692 10.11 10:89936b26:::benchmark_data_smithi018_422431_object76196:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e600)", 2023-09-23T20:49:16.223 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:48:58.951612+0000", 2023-09-23T20:49:16.223 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 17.131826383, 2023-09-23T20:49:16.223 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.8444404539999999, 2023-09-23T20:49:16.223 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:49:16.224 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:49:16.224 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:49:16.224 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:49:16.224 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:49:16.224 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 180692 2023-09-23T20:49:16.224 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.224 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:49:16.224 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.224 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:49:16.224 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:58.951612+0000", 2023-09-23T20:49:16.224 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:16.225 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.225 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.225 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:49:16.225 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:58.951612+0000", 2023-09-23T20:49:16.225 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:16.225 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.225 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.225 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:49:16.225 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:58.951611+0000", 2023-09-23T20:49:16.225 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:49:16.225 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.225 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.226 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:49:16.226 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:58.951614+0000", 2023-09-23T20:49:16.226 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.6730000000000001e-06 2023-09-23T20:49:16.226 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.226 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.226 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:49:16.226 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:58.951620+0000", 2023-09-23T20:49:16.226 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.9819999999999996e-06 2023-09-23T20:49:16.226 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.226 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.226 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:49:16.226 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:58.951625+0000", 2023-09-23T20:49:16.227 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.0780000000000003e-06 2023-09-23T20:49:16.227 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.227 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.227 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:16.227 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:58.951691+0000", 2023-09-23T20:49:16.227 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.6138999999999998e-05 2023-09-23T20:49:16.227 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.227 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.227 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:49:16.227 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:58.951702+0000", 2023-09-23T20:49:16.227 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0730999999999999e-05 2023-09-23T20:49:16.227 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.228 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.228 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:16.228 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:59.700879+0000", 2023-09-23T20:49:16.228 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.74917732500000001 2023-09-23T20:49:16.228 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.228 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.228 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:49:16.228 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:59.700884+0000", 2023-09-23T20:49:16.228 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.7040000000000002e-06 2023-09-23T20:49:16.228 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.228 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.229 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:16.229 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:00.784575+0000", 2023-09-23T20:49:16.229 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.083690941 2023-09-23T20:49:16.229 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.229 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.229 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:49:16.229 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:00.784715+0000", 2023-09-23T20:49:16.229 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000139667 2023-09-23T20:49:16.229 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.229 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.229 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 4,5,7", 2023-09-23T20:49:16.229 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:00.784874+0000", 2023-09-23T20:49:16.229 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000159185 2023-09-23T20:49:16.230 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.230 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.230 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:49:16.230 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:00.787027+0000", 2023-09-23T20:49:16.230 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0021535550000000001 2023-09-23T20:49:16.230 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.230 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.230 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:16.230 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:00.787398+0000", 2023-09-23T20:49:16.230 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00037117500000000001 2023-09-23T20:49:16.230 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.230 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.231 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:16.231 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:00.795832+0000", 2023-09-23T20:49:16.231 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0084338420000000004 2023-09-23T20:49:16.231 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.231 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.231 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:16.231 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:00.795905+0000", 2023-09-23T20:49:16.231 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.2599000000000001e-05 2023-09-23T20:49:16.231 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.231 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.231 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:49:16.231 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:00.795953+0000", 2023-09-23T20:49:16.232 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.8312000000000001e-05 2023-09-23T20:49:16.232 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.232 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.232 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:49:16.232 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:00.796052+0000", 2023-09-23T20:49:16.232 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.8992000000000004e-05 2023-09-23T20:49:16.232 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:16.232 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:49:16.232 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:16.232 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.232 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.232 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:180696 10.11 10:8cb999f1:::benchmark_data_smithi018_422431_object76200:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e600)", 2023-09-23T20:49:16.232 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:48:58.957312+0000", 2023-09-23T20:49:16.233 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 17.126126011, 2023-09-23T20:49:16.233 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.8389737159999999, 2023-09-23T20:49:16.233 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:49:16.233 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:49:16.233 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:49:16.233 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:49:16.233 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:49:16.233 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 180696 2023-09-23T20:49:16.233 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.233 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:49:16.233 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.234 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:49:16.234 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:58.957312+0000", 2023-09-23T20:49:16.234 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:16.234 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.234 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.234 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:49:16.234 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:58.957312+0000", 2023-09-23T20:49:16.234 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:16.234 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.234 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.234 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:49:16.234 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:58.957311+0000", 2023-09-23T20:49:16.235 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.999999 2023-09-23T20:49:16.235 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.235 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.235 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:49:16.235 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:58.957318+0000", 2023-09-23T20:49:16.235 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.6669999999999999e-06 2023-09-23T20:49:16.235 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.235 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.235 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:49:16.235 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:58.957335+0000", 2023-09-23T20:49:16.235 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.7319000000000001e-05 2023-09-23T20:49:16.235 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.236 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.236 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:49:16.236 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:58.957342+0000", 2023-09-23T20:49:16.236 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.7819999999999996e-06 2023-09-23T20:49:16.236 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.236 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.236 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:16.236 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:58.957406+0000", 2023-09-23T20:49:16.236 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.3958e-05 2023-09-23T20:49:16.236 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.236 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.236 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:49:16.237 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:58.957417+0000", 2023-09-23T20:49:16.237 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1761e-05 2023-09-23T20:49:16.237 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.237 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.237 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:16.237 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:59.700913+0000", 2023-09-23T20:49:16.237 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.743495342 2023-09-23T20:49:16.237 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.237 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.237 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:49:16.237 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:59.700921+0000", 2023-09-23T20:49:16.237 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.1189999999999997e-06 2023-09-23T20:49:16.238 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.238 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.238 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:16.238 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:00.787425+0000", 2023-09-23T20:49:16.238 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.086503928 2023-09-23T20:49:16.238 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.238 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.238 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:49:16.238 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:00.787555+0000", 2023-09-23T20:49:16.238 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00013023200000000001 2023-09-23T20:49:16.238 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.239 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.239 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 4,5,7", 2023-09-23T20:49:16.239 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:00.787703+0000", 2023-09-23T20:49:16.239 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00014794 2023-09-23T20:49:16.239 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.239 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.239 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:49:16.239 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:00.795563+0000", 2023-09-23T20:49:16.239 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0078600100000000006 2023-09-23T20:49:16.239 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.239 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.239 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:16.240 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:00.795872+0000", 2023-09-23T20:49:16.240 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00030897699999999999 2023-09-23T20:49:16.240 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.240 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.240 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:16.240 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:00.796089+0000", 2023-09-23T20:49:16.240 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00021685399999999999 2023-09-23T20:49:16.240 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.240 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.240 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:16.240 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:00.796152+0000", 2023-09-23T20:49:16.240 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.2785000000000003e-05 2023-09-23T20:49:16.241 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.241 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.241 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:49:16.241 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:00.796190+0000", 2023-09-23T20:49:16.241 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.7973999999999999e-05 2023-09-23T20:49:16.241 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.241 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.241 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:49:16.241 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:00.796286+0000", 2023-09-23T20:49:16.241 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.6216000000000002e-05 2023-09-23T20:49:16.241 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:16.241 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:49:16.242 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:16.242 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.242 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.242 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:180703 10.11 10:8ebb9ec3:::benchmark_data_smithi018_422431_object76207:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e600)", 2023-09-23T20:49:16.242 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:48:58.960949+0000", 2023-09-23T20:49:16.242 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 17.122489030000001, 2023-09-23T20:49:16.242 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.8394799480000001, 2023-09-23T20:49:16.242 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:49:16.242 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:49:16.242 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:49:16.242 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:49:16.242 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:49:16.243 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 180703 2023-09-23T20:49:16.243 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.243 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:49:16.243 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.243 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:49:16.243 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:58.960949+0000", 2023-09-23T20:49:16.243 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:16.243 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.243 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.243 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:49:16.243 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:58.960949+0000", 2023-09-23T20:49:16.243 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:16.244 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.244 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.244 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:49:16.244 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:58.960948+0000", 2023-09-23T20:49:16.244 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:49:16.244 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.244 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.244 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:49:16.244 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:58.960955+0000", 2023-09-23T20:49:16.244 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.2740000000000002e-06 2023-09-23T20:49:16.244 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.244 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.244 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:49:16.245 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:58.960967+0000", 2023-09-23T20:49:16.245 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2481e-05 2023-09-23T20:49:16.245 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.245 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.245 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:49:16.245 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:58.960976+0000", 2023-09-23T20:49:16.245 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.3359999999999995e-06 2023-09-23T20:49:16.245 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.245 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.245 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:16.245 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:58.961055+0000", 2023-09-23T20:49:16.245 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.9723999999999998e-05 2023-09-23T20:49:16.246 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.246 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.246 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:49:16.246 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:58.961066+0000", 2023-09-23T20:49:16.246 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0679000000000001e-05 2023-09-23T20:49:16.246 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.246 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.246 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:16.246 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:59.700945+0000", 2023-09-23T20:49:16.246 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.73987876299999999 2023-09-23T20:49:16.246 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.246 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.247 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:49:16.247 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:59.700949+0000", 2023-09-23T20:49:16.247 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.3170000000000003e-06 2023-09-23T20:49:16.247 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.247 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.247 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:16.247 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:00.789284+0000", 2023-09-23T20:49:16.247 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0883345710000001 2023-09-23T20:49:16.247 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.247 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.247 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:49:16.247 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:00.789432+0000", 2023-09-23T20:49:16.248 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00014861400000000001 2023-09-23T20:49:16.248 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.248 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.248 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 4,5,7", 2023-09-23T20:49:16.248 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:00.789552+0000", 2023-09-23T20:49:16.248 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000119503 2023-09-23T20:49:16.248 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.248 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.248 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:49:16.248 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:00.795570+0000", 2023-09-23T20:49:16.248 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.006017817 2023-09-23T20:49:16.248 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.249 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.249 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:16.249 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:00.796120+0000", 2023-09-23T20:49:16.249 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00055069699999999995 2023-09-23T20:49:16.249 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.249 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.249 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:16.249 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:00.796354+0000", 2023-09-23T20:49:16.249 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00023382599999999999 2023-09-23T20:49:16.249 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.249 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.249 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:16.249 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:00.800231+0000", 2023-09-23T20:49:16.250 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0038763460000000001 2023-09-23T20:49:16.250 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.250 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.250 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:49:16.250 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:00.800293+0000", 2023-09-23T20:49:16.250 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.2747000000000001e-05 2023-09-23T20:49:16.250 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.250 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.250 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:49:16.250 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:00.800429+0000", 2023-09-23T20:49:16.250 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00013567900000000001 2023-09-23T20:49:16.250 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:16.251 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:49:16.251 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:16.251 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.251 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.251 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:180704 10.11 10:8f48feef:::benchmark_data_smithi018_422431_object76208:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e600)", 2023-09-23T20:49:16.251 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:48:58.960992+0000", 2023-09-23T20:49:16.251 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 17.122445843000001, 2023-09-23T20:49:16.251 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.8396724249999998, 2023-09-23T20:49:16.251 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:49:16.251 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:49:16.251 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:49:16.251 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:49:16.252 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:49:16.252 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 180704 2023-09-23T20:49:16.252 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.252 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:49:16.252 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.252 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:49:16.252 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:58.960992+0000", 2023-09-23T20:49:16.252 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:16.252 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.252 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.252 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:49:16.252 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:58.960992+0000", 2023-09-23T20:49:16.253 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:16.253 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.253 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.253 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:49:16.253 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:58.960992+0000", 2023-09-23T20:49:16.253 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:49:16.253 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.253 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.253 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:49:16.253 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:58.960995+0000", 2023-09-23T20:49:16.253 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.6899999999999998e-06 2023-09-23T20:49:16.253 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.254 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.254 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:49:16.254 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:58.961003+0000", 2023-09-23T20:49:16.254 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.2019999999999998e-06 2023-09-23T20:49:16.254 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.254 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.254 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:49:16.254 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:58.961008+0000", 2023-09-23T20:49:16.254 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.8610000000000003e-06 2023-09-23T20:49:16.254 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.254 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.254 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:16.255 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:58.961096+0000", 2023-09-23T20:49:16.255 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.7991999999999994e-05 2023-09-23T20:49:16.255 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.255 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.255 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:49:16.255 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:58.961109+0000", 2023-09-23T20:49:16.255 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2493999999999999e-05 2023-09-23T20:49:16.255 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.255 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.255 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:16.255 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:59.700978+0000", 2023-09-23T20:49:16.255 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.73986862499999995 2023-09-23T20:49:16.256 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.256 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.256 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:49:16.256 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:59.700986+0000", 2023-09-23T20:49:16.256 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.3580000000000003e-06 2023-09-23T20:49:16.256 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.256 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.256 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:16.256 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:00.790959+0000", 2023-09-23T20:49:16.256 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0899732150000001 2023-09-23T20:49:16.256 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.256 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.257 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:49:16.257 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:00.793509+0000", 2023-09-23T20:49:16.257 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0025496550000000001 2023-09-23T20:49:16.257 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.257 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.257 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 4,5,7", 2023-09-23T20:49:16.257 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:00.793647+0000", 2023-09-23T20:49:16.257 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00013804999999999999 2023-09-23T20:49:16.257 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.257 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.257 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:49:16.257 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:00.795840+0000", 2023-09-23T20:49:16.258 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0021928989999999999 2023-09-23T20:49:16.258 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.258 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.258 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:16.258 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:00.796322+0000", 2023-09-23T20:49:16.258 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000481865 2023-09-23T20:49:16.258 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.258 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.258 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:16.258 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:00.796388+0000", 2023-09-23T20:49:16.258 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.6520999999999995e-05 2023-09-23T20:49:16.258 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.259 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.259 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:16.259 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:00.800495+0000", 2023-09-23T20:49:16.259 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0041069799999999997 2023-09-23T20:49:16.259 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.259 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.259 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:49:16.259 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:00.800547+0000", 2023-09-23T20:49:16.259 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.1480000000000002e-05 2023-09-23T20:49:16.259 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.259 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.259 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:49:16.260 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:00.800665+0000", 2023-09-23T20:49:16.260 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00011809800000000001 2023-09-23T20:49:16.260 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:16.260 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:49:16.260 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:16.260 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.260 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.260 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:180715 10.11 10:88524e0d:::benchmark_data_smithi018_422431_object76219:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e600)", 2023-09-23T20:49:16.260 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:48:58.970590+0000", 2023-09-23T20:49:16.260 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 17.112848063000001, 2023-09-23T20:49:16.260 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.843639424, 2023-09-23T20:49:16.260 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:49:16.261 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:49:16.261 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:49:16.261 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:49:16.261 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:49:16.261 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 180715 2023-09-23T20:49:16.261 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.261 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:49:16.261 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.261 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:49:16.261 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:58.970590+0000", 2023-09-23T20:49:16.261 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:16.261 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.262 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.262 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:49:16.262 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:58.970590+0000", 2023-09-23T20:49:16.262 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:16.262 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.262 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.262 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:49:16.262 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:58.970589+0000", 2023-09-23T20:49:16.262 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.999999 2023-09-23T20:49:16.262 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.262 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.262 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:49:16.263 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:58.970597+0000", 2023-09-23T20:49:16.263 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.7959999999999996e-06 2023-09-23T20:49:16.263 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.263 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.263 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:49:16.263 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:58.970608+0000", 2023-09-23T20:49:16.263 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0975000000000001e-05 2023-09-23T20:49:16.263 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.263 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.263 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:49:16.263 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:58.970614+0000", 2023-09-23T20:49:16.263 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.2099999999999998e-06 2023-09-23T20:49:16.264 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.264 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.264 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:16.264 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:58.970666+0000", 2023-09-23T20:49:16.264 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.1495000000000001e-05 2023-09-23T20:49:16.264 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.264 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.264 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:49:16.264 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:58.970677+0000", 2023-09-23T20:49:16.264 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0975000000000001e-05 2023-09-23T20:49:16.264 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.264 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.264 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:16.265 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:59.701010+0000", 2023-09-23T20:49:16.265 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.730333601 2023-09-23T20:49:16.265 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.265 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.265 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:49:16.265 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:59.701014+0000", 2023-09-23T20:49:16.265 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.1620000000000001e-06 2023-09-23T20:49:16.265 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.265 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.265 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:16.265 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:00.796411+0000", 2023-09-23T20:49:16.265 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.095397011 2023-09-23T20:49:16.266 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.266 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.266 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:49:16.266 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:00.796541+0000", 2023-09-23T20:49:16.266 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00012961200000000001 2023-09-23T20:49:16.266 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.266 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.266 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 4,5,7", 2023-09-23T20:49:16.266 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:00.796672+0000", 2023-09-23T20:49:16.266 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000130918 2023-09-23T20:49:16.266 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.266 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.267 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:49:16.267 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:00.800446+0000", 2023-09-23T20:49:16.267 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0037737169999999998 2023-09-23T20:49:16.267 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.267 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.267 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:16.267 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:00.803626+0000", 2023-09-23T20:49:16.267 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0031806400000000002 2023-09-23T20:49:16.267 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.267 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.267 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:16.267 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:00.803660+0000", 2023-09-23T20:49:16.268 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.3408000000000003e-05 2023-09-23T20:49:16.268 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.268 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.268 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:16.268 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:00.814065+0000", 2023-09-23T20:49:16.268 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.010405582999999999 2023-09-23T20:49:16.268 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.268 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.268 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:49:16.268 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:00.814114+0000", 2023-09-23T20:49:16.268 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.9123999999999999e-05 2023-09-23T20:49:16.268 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.269 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.269 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:49:16.269 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:00.814229+0000", 2023-09-23T20:49:16.269 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000115006 2023-09-23T20:49:16.269 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:16.269 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:49:16.269 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:16.269 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.269 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.269 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:180820 10.11 10:8c84aac0:::benchmark_data_smithi018_422431_object76324:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e600)", 2023-09-23T20:49:16.269 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:48:59.099754+0000", 2023-09-23T20:49:16.269 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 16.983684461999999, 2023-09-23T20:49:16.270 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.7148130849999998, 2023-09-23T20:49:16.270 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:49:16.270 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:49:16.270 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:49:16.270 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:49:16.270 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:49:16.270 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 180820 2023-09-23T20:49:16.270 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.270 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:49:16.270 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.270 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:49:16.270 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:59.099754+0000", 2023-09-23T20:49:16.271 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:16.271 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.271 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.271 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:49:16.271 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:59.099754+0000", 2023-09-23T20:49:16.271 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:16.271 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.271 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.271 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:49:16.271 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:59.099752+0000", 2023-09-23T20:49:16.271 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:49:16.271 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.272 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.272 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:49:16.272 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:59.099759+0000", 2023-09-23T20:49:16.272 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.922e-06 2023-09-23T20:49:16.272 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.272 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.272 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:49:16.272 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:59.099772+0000", 2023-09-23T20:49:16.272 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3027000000000001e-05 2023-09-23T20:49:16.272 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.272 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.273 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:49:16.273 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:59.099780+0000", 2023-09-23T20:49:16.273 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.3310000000000004e-06 2023-09-23T20:49:16.273 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.273 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.273 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:16.273 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:59.099851+0000", 2023-09-23T20:49:16.273 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.1072e-05 2023-09-23T20:49:16.273 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.273 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.273 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:49:16.274 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:59.099864+0000", 2023-09-23T20:49:16.274 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2415000000000001e-05 2023-09-23T20:49:16.274 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.274 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.274 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:16.274 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:59.701043+0000", 2023-09-23T20:49:16.274 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.60117959700000001 2023-09-23T20:49:16.274 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.274 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.274 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:49:16.274 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:59.701051+0000", 2023-09-23T20:49:16.274 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.2919999999999996e-06 2023-09-23T20:49:16.275 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.275 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.275 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:16.275 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:00.798434+0000", 2023-09-23T20:49:16.275 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0973830630000001 2023-09-23T20:49:16.275 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.275 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.275 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:49:16.275 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:00.798578+0000", 2023-09-23T20:49:16.275 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000143941 2023-09-23T20:49:16.275 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.275 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.276 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 4,5,7", 2023-09-23T20:49:16.276 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:00.798724+0000", 2023-09-23T20:49:16.276 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00014550399999999999 2023-09-23T20:49:16.276 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.276 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.276 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:49:16.276 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:00.800676+0000", 2023-09-23T20:49:16.276 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.001952002 2023-09-23T20:49:16.276 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.276 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.276 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:16.276 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:00.814315+0000", 2023-09-23T20:49:16.277 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.013639551 2023-09-23T20:49:16.277 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.277 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.277 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:16.277 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:00.814363+0000", 2023-09-23T20:49:16.277 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.7743e-05 2023-09-23T20:49:16.277 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.277 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.277 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:16.277 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:00.814400+0000", 2023-09-23T20:49:16.277 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.6992000000000001e-05 2023-09-23T20:49:16.278 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.278 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.278 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:49:16.278 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:00.814448+0000", 2023-09-23T20:49:16.278 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.7343999999999999e-05 2023-09-23T20:49:16.278 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:16.278 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:16.278 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:49:16.278 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:00.814567+0000", 2023-09-23T20:49:16.278 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000119191 2023-09-23T20:49:16.278 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:16.279 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:49:16.279 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:16.279 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:16.279 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:49:16.279 INFO:teuthology.orchestra.run.smithi018.stdout:} 2023-09-23T20:49:16.280 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:15 smithi149 ceph-mon[170394]: pgmap v1521: 129 pgs: 1 active, 1 peering, 1 active+clean+scrubbing+deep, 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 123 active+clean; 1.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 805 op/s; 1543/64413 objects misplaced (2.395%); 29 MiB/s, 471 objects/s recovering 2023-09-23T20:49:16.280 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:15 smithi149 ceph-mon[170394]: 10.14 scrub starts 2023-09-23T20:49:16.280 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:15 smithi149 ceph-mon[170394]: 10.3 deep-scrub ok 2023-09-23T20:49:16.280 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:15 smithi149 ceph-mon[170394]: 10.13 scrub starts 2023-09-23T20:49:16.280 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:15 smithi149 ceph-mon[170394]: failed to encode map e612 with expected crc 2023-09-23T20:49:16.280 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:15 smithi149 ceph-mon[170394]: 10.2 deep-scrub starts 2023-09-23T20:49:16.280 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:15 smithi149 ceph-mon[170394]: failed to encode map e612 with expected crc 2023-09-23T20:49:16.280 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:15 smithi149 ceph-mon[170394]: failed to encode map e612 with expected crc 2023-09-23T20:49:16.280 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:15 smithi149 ceph-mon[170394]: failed to encode map e612 with expected crc 2023-09-23T20:49:16.280 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:15 smithi149 ceph-mon[170394]: 10.13 scrub ok 2023-09-23T20:49:16.281 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:15 smithi149 ceph-mon[170394]: 10.17 scrub starts 2023-09-23T20:49:16.281 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:15 smithi149 ceph-mon[170394]: 10.17 scrub ok 2023-09-23T20:49:16.281 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:15 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:49:16.281 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:15 smithi018 ceph-mon[184144]: pgmap v1521: 129 pgs: 1 active, 1 peering, 1 active+clean+scrubbing+deep, 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 123 active+clean; 1.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 805 op/s; 1543/64413 objects misplaced (2.395%); 29 MiB/s, 471 objects/s recovering 2023-09-23T20:49:16.281 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:15 smithi018 ceph-mon[184144]: 10.14 scrub starts 2023-09-23T20:49:16.281 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:15 smithi018 ceph-mon[184144]: 10.3 deep-scrub ok 2023-09-23T20:49:16.281 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:15 smithi018 ceph-mon[184144]: 10.13 scrub starts 2023-09-23T20:49:16.281 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:15 smithi018 ceph-mon[184144]: failed to encode map e612 with expected crc 2023-09-23T20:49:16.281 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:15 smithi018 ceph-mon[184144]: 10.2 deep-scrub starts 2023-09-23T20:49:16.281 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:15 smithi018 ceph-mon[184144]: failed to encode map e612 with expected crc 2023-09-23T20:49:16.282 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:15 smithi018 ceph-mon[184144]: failed to encode map e612 with expected crc 2023-09-23T20:49:16.282 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:15 smithi018 ceph-mon[184144]: failed to encode map e612 with expected crc 2023-09-23T20:49:16.282 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:15 smithi018 ceph-mon[184144]: 10.13 scrub ok 2023-09-23T20:49:16.282 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:15 smithi018 ceph-mon[184144]: 10.17 scrub starts 2023-09-23T20:49:16.282 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:15 smithi018 ceph-mon[184144]: 10.17 scrub ok 2023-09-23T20:49:16.282 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:15 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:49:16.282 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:15 smithi018 ceph-mon[179647]: pgmap v1521: 129 pgs: 1 active, 1 peering, 1 active+clean+scrubbing+deep, 1 active+remapped+backfill_wait, 2 active+remapped+backfilling, 123 active+clean; 1.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 805 op/s; 1543/64413 objects misplaced (2.395%); 29 MiB/s, 471 objects/s recovering 2023-09-23T20:49:16.282 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:15 smithi018 ceph-mon[179647]: 10.14 scrub starts 2023-09-23T20:49:16.282 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:15 smithi018 ceph-mon[179647]: 10.3 deep-scrub ok 2023-09-23T20:49:16.283 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:15 smithi018 ceph-mon[179647]: 10.13 scrub starts 2023-09-23T20:49:16.283 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:15 smithi018 ceph-mon[179647]: failed to encode map e612 with expected crc 2023-09-23T20:49:16.283 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:15 smithi018 ceph-mon[179647]: 10.2 deep-scrub starts 2023-09-23T20:49:16.283 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:15 smithi018 ceph-mon[179647]: failed to encode map e612 with expected crc 2023-09-23T20:49:16.283 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:15 smithi018 ceph-mon[179647]: failed to encode map e612 with expected crc 2023-09-23T20:49:16.283 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:15 smithi018 ceph-mon[179647]: failed to encode map e612 with expected crc 2023-09-23T20:49:16.283 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:15 smithi018 ceph-mon[179647]: 10.13 scrub ok 2023-09-23T20:49:16.283 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:15 smithi018 ceph-mon[179647]: 10.17 scrub starts 2023-09-23T20:49:16.283 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:15 smithi018 ceph-mon[179647]: 10.17 scrub ok 2023-09-23T20:49:16.283 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:15 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:49:16.284 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:49:16.284 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:49:16.284 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:16.509 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:49:16.518 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:16.519 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:49:16.605 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T20:49:16.620 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:49:16.669 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:16.730 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:49:16.737 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:16.737 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:49:16.838 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:49:16.943 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:16.958 INFO:teuthology.orchestra.run.smithi018.stderr:noscrub is set 2023-09-23T20:49:16.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:16 smithi149 ceph-mon[170394]: 10.9 scrub starts 2023-09-23T20:49:16.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:16 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2231521096' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:49:16.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:16 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:49:16.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:16 smithi149 ceph-mon[170394]: 10.12 scrub starts 2023-09-23T20:49:16.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:16 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "20"}]: dispatch 2023-09-23T20:49:16.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:16 smithi149 ceph-mon[170394]: 10.12 scrub ok 2023-09-23T20:49:17.084 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:49:17.090 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:16 smithi018 ceph-mon[179647]: 10.9 scrub starts 2023-09-23T20:49:17.090 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:16 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2231521096' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:49:17.090 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:16 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:49:17.090 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:16 smithi018 ceph-mon[179647]: 10.12 scrub starts 2023-09-23T20:49:17.090 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:16 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "20"}]: dispatch 2023-09-23T20:49:17.090 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:16 smithi018 ceph-mon[179647]: 10.12 scrub ok 2023-09-23T20:49:17.091 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:16 smithi018 ceph-mon[184144]: 10.9 scrub starts 2023-09-23T20:49:17.091 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:16 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2231521096' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:49:17.091 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:16 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:49:17.091 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:16 smithi018 ceph-mon[184144]: 10.12 scrub starts 2023-09-23T20:49:17.091 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:16 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "20"}]: dispatch 2023-09-23T20:49:17.091 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:16 smithi018 ceph-mon[184144]: 10.12 scrub ok 2023-09-23T20:49:17.092 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:17.092 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:49:17.193 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:49:17.247 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:17.650 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:49:17.653 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:17.654 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:49:17.754 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:49:17.783 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:17.985 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:49:17.988 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:17.988 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:49:18.016 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:17 smithi018 ceph-mon[184144]: 10.1 scrub starts 2023-09-23T20:49:18.017 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:17 smithi018 ceph-mon[184144]: pgmap v1523: 129 pgs: 1 active+clean+scrubbing+deep, 1 active+remapped+backfilling, 127 active+clean; 1.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 392/59220 objects misplaced (0.662%); 316 KiB/s, 4 objects/s recovering 2023-09-23T20:49:18.017 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:17 smithi018 ceph-mon[184144]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:49:18.017 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:17 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:49:18.017 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:17 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "20"}]': finished 2023-09-23T20:49:18.017 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:17 smithi018 ceph-mon[184144]: osdmap e613: 8 total, 8 up, 8 in 2023-09-23T20:49:18.017 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:17 smithi018 ceph-mon[184144]: failed to encode map e613 with expected crc 2023-09-23T20:49:18.017 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:17 smithi018 ceph-mon[179647]: 10.1 scrub starts 2023-09-23T20:49:18.017 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:17 smithi018 ceph-mon[179647]: pgmap v1523: 129 pgs: 1 active+clean+scrubbing+deep, 1 active+remapped+backfilling, 127 active+clean; 1.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 392/59220 objects misplaced (0.662%); 316 KiB/s, 4 objects/s recovering 2023-09-23T20:49:18.017 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:17 smithi018 ceph-mon[179647]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:49:18.017 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:17 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:49:18.018 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:17 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "20"}]': finished 2023-09-23T20:49:18.018 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:17 smithi018 ceph-mon[179647]: osdmap e613: 8 total, 8 up, 8 in 2023-09-23T20:49:18.018 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:17 smithi018 ceph-mon[179647]: failed to encode map e613 with expected crc 2023-09-23T20:49:18.089 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:49:18.118 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:18.152 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:49:18.154 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:17 smithi149 ceph-mon[170394]: 10.1 scrub starts 2023-09-23T20:49:18.154 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:17 smithi149 ceph-mon[170394]: pgmap v1523: 129 pgs: 1 active+clean+scrubbing+deep, 1 active+remapped+backfilling, 127 active+clean; 1.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 392/59220 objects misplaced (0.662%); 316 KiB/s, 4 objects/s recovering 2023-09-23T20:49:18.154 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:17 smithi149 ceph-mon[170394]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:49:18.154 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:17 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:49:18.154 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:17 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "20"}]': finished 2023-09-23T20:49:18.154 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:17 smithi149 ceph-mon[170394]: osdmap e613: 8 total, 8 up, 8 in 2023-09-23T20:49:18.154 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:17 smithi149 ceph-mon[170394]: failed to encode map e613 with expected crc 2023-09-23T20:49:18.155 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:18.155 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:49:18.256 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:49:18.282 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:18.314 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:49:18.317 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:18.317 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:49:18.418 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:49:18.449 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:18.533 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:49:18.537 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:18.538 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:49:18.574 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:49:18.639 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:49:18.670 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:18.882 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:49:18.895 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:49:18.898 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:18.899 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:49:18.907 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:49:18.914 INFO:teuthology.orchestra.run.smithi018.stderr:set pool 10 pgp_num to 26 2023-09-23T20:49:18.934 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:49:18.955 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:49:18.976 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:49:18.999 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:49:19.000 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:49:19.023 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:49:19.040 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:19.044 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:49:19.048 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:49:19.048 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:49:19.048 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:49:19.048 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:49:19.048 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:49:19.048 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:49:19.049 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:49:19.049 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:49:19.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:18 smithi018 ceph-mon[184144]: failed to encode map e613 with expected crc 2023-09-23T20:49:19.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:18 smithi018 ceph-mon[184144]: 10.2 deep-scrub ok 2023-09-23T20:49:19.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:18 smithi018 ceph-mon[184144]: osdmap e614: 8 total, 8 up, 8 in 2023-09-23T20:49:19.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:18 smithi018 ceph-mon[184144]: failed to encode map e614 with expected crc 2023-09-23T20:49:19.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:18 smithi018 ceph-mon[184144]: failed to encode map e614 with expected crc 2023-09-23T20:49:19.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:18 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "21"}]: dispatch 2023-09-23T20:49:19.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:18 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1967456116' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "26"}]: dispatch 2023-09-23T20:49:19.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:18 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "26"}]: dispatch 2023-09-23T20:49:19.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:18 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/778885778' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:49:19.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:18 smithi018 ceph-mon[179647]: failed to encode map e613 with expected crc 2023-09-23T20:49:19.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:18 smithi018 ceph-mon[179647]: 10.2 deep-scrub ok 2023-09-23T20:49:19.144 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:18 smithi018 ceph-mon[179647]: osdmap e614: 8 total, 8 up, 8 in 2023-09-23T20:49:19.144 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:18 smithi018 ceph-mon[179647]: failed to encode map e614 with expected crc 2023-09-23T20:49:19.144 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:18 smithi018 ceph-mon[179647]: failed to encode map e614 with expected crc 2023-09-23T20:49:19.144 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:18 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "21"}]: dispatch 2023-09-23T20:49:19.144 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:18 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1967456116' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "26"}]: dispatch 2023-09-23T20:49:19.144 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:18 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "26"}]: dispatch 2023-09-23T20:49:19.145 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:18 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/778885778' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:49:19.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:18 smithi149 ceph-mon[170394]: failed to encode map e613 with expected crc 2023-09-23T20:49:19.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:18 smithi149 ceph-mon[170394]: 10.2 deep-scrub ok 2023-09-23T20:49:19.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:18 smithi149 ceph-mon[170394]: osdmap e614: 8 total, 8 up, 8 in 2023-09-23T20:49:19.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:18 smithi149 ceph-mon[170394]: failed to encode map e614 with expected crc 2023-09-23T20:49:19.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:18 smithi149 ceph-mon[170394]: failed to encode map e614 with expected crc 2023-09-23T20:49:19.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:18 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "21"}]: dispatch 2023-09-23T20:49:19.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:18 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1967456116' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "26"}]: dispatch 2023-09-23T20:49:19.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:18 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "26"}]: dispatch 2023-09-23T20:49:19.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:18 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/778885778' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:49:19.495 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T20:49:19.616 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:49:19.619 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:19.619 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:49:19.720 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:49:19.762 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:19.916 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:49:19.921 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:19.921 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:49:20.022 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:49:20.061 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:20.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:19 smithi018 ceph-mon[184144]: pgmap v1526: 129 pgs: 1 active+clean+scrubbing+deep, 1 active+remapped+backfilling, 127 active+clean; 1.2 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 392/57732 objects misplaced (0.679%) 2023-09-23T20:49:20.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:19 smithi018 ceph-mon[184144]: failed to encode map e614 with expected crc 2023-09-23T20:49:20.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:19 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "21"}]': finished 2023-09-23T20:49:20.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:19 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "26"}]': finished 2023-09-23T20:49:20.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:19 smithi018 ceph-mon[184144]: osdmap e615: 8 total, 8 up, 8 in 2023-09-23T20:49:20.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:19 smithi018 ceph-mon[184144]: 10.13 starting backfill to osd.1 from (0'0,0'0] MAX to 612'11532 2023-09-23T20:49:20.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:19 smithi018 ceph-mon[184144]: 10.13 starting backfill to osd.7 from (0'0,0'0] MAX to 612'11532 2023-09-23T20:49:20.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:19 smithi018 ceph-mon[184144]: failed to encode map e615 with expected crc 2023-09-23T20:49:20.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:19 smithi018 ceph-mon[184144]: failed to encode map e615 with expected crc 2023-09-23T20:49:20.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:19 smithi018 ceph-mon[184144]: failed to encode map e614 with expected crc 2023-09-23T20:49:20.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:19 smithi018 ceph-mon[179647]: pgmap v1526: 129 pgs: 1 active+clean+scrubbing+deep, 1 active+remapped+backfilling, 127 active+clean; 1.2 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 392/57732 objects misplaced (0.679%) 2023-09-23T20:49:20.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:19 smithi018 ceph-mon[179647]: failed to encode map e614 with expected crc 2023-09-23T20:49:20.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:19 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "21"}]': finished 2023-09-23T20:49:20.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:19 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "26"}]': finished 2023-09-23T20:49:20.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:19 smithi018 ceph-mon[179647]: osdmap e615: 8 total, 8 up, 8 in 2023-09-23T20:49:20.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:19 smithi018 ceph-mon[179647]: 10.13 starting backfill to osd.1 from (0'0,0'0] MAX to 612'11532 2023-09-23T20:49:20.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:19 smithi018 ceph-mon[179647]: 10.13 starting backfill to osd.7 from (0'0,0'0] MAX to 612'11532 2023-09-23T20:49:20.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:19 smithi018 ceph-mon[179647]: failed to encode map e615 with expected crc 2023-09-23T20:49:20.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:19 smithi018 ceph-mon[179647]: failed to encode map e615 with expected crc 2023-09-23T20:49:20.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:19 smithi018 ceph-mon[179647]: failed to encode map e614 with expected crc 2023-09-23T20:49:20.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:19 smithi149 ceph-mon[170394]: pgmap v1526: 129 pgs: 1 active+clean+scrubbing+deep, 1 active+remapped+backfilling, 127 active+clean; 1.2 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 392/57732 objects misplaced (0.679%) 2023-09-23T20:49:20.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:19 smithi149 ceph-mon[170394]: failed to encode map e614 with expected crc 2023-09-23T20:49:20.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:19 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "21"}]': finished 2023-09-23T20:49:20.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:19 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "26"}]': finished 2023-09-23T20:49:20.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:19 smithi149 ceph-mon[170394]: osdmap e615: 8 total, 8 up, 8 in 2023-09-23T20:49:20.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:19 smithi149 ceph-mon[170394]: 10.13 starting backfill to osd.1 from (0'0,0'0] MAX to 612'11532 2023-09-23T20:49:20.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:19 smithi149 ceph-mon[170394]: 10.13 starting backfill to osd.7 from (0'0,0'0] MAX to 612'11532 2023-09-23T20:49:20.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:19 smithi149 ceph-mon[170394]: failed to encode map e615 with expected crc 2023-09-23T20:49:20.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:19 smithi149 ceph-mon[170394]: failed to encode map e615 with expected crc 2023-09-23T20:49:20.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:19 smithi149 ceph-mon[170394]: failed to encode map e614 with expected crc 2023-09-23T20:49:20.276 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:49:20.289 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:20.290 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:49:20.381 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:49:20.391 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:49:20.432 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:20.481 INFO:teuthology.orchestra.run.smithi018.stdout:ERROR: (22) Invalid argument 2023-09-23T20:49:20.481 INFO:teuthology.orchestra.run.smithi018.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:49:20.514 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:49:20.516 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:20.517 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:49:20.618 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:49:20.646 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:20.868 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T20:49:20.880 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:49:20.883 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:20.883 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:49:20.985 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:49:21.019 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:21.110 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:49:21.117 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:21.117 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:49:21.120 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:20 smithi018 ceph-mon[184144]: failed to encode map e616 with expected crc 2023-09-23T20:49:21.120 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:20 smithi018 ceph-mon[184144]: osdmap e616: 8 total, 8 up, 8 in 2023-09-23T20:49:21.120 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:20 smithi018 ceph-mon[184144]: failed to encode map e616 with expected crc 2023-09-23T20:49:21.120 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:20 smithi018 ceph-mon[184144]: failed to encode map e616 with expected crc 2023-09-23T20:49:21.120 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:20 smithi018 ceph-mon[184144]: failed to encode map e616 with expected crc 2023-09-23T20:49:21.120 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:20 smithi018 ceph-mon[179647]: failed to encode map e616 with expected crc 2023-09-23T20:49:21.121 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:20 smithi018 ceph-mon[179647]: osdmap e616: 8 total, 8 up, 8 in 2023-09-23T20:49:21.121 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:20 smithi018 ceph-mon[179647]: failed to encode map e616 with expected crc 2023-09-23T20:49:21.121 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:20 smithi018 ceph-mon[179647]: failed to encode map e616 with expected crc 2023-09-23T20:49:21.121 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:20 smithi018 ceph-mon[179647]: failed to encode map e616 with expected crc 2023-09-23T20:49:21.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:20 smithi149 ceph-mon[170394]: failed to encode map e616 with expected crc 2023-09-23T20:49:21.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:20 smithi149 ceph-mon[170394]: osdmap e616: 8 total, 8 up, 8 in 2023-09-23T20:49:21.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:20 smithi149 ceph-mon[170394]: failed to encode map e616 with expected crc 2023-09-23T20:49:21.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:20 smithi149 ceph-mon[170394]: failed to encode map e616 with expected crc 2023-09-23T20:49:21.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:20 smithi149 ceph-mon[170394]: failed to encode map e616 with expected crc 2023-09-23T20:49:21.219 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:49:21.284 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:21.394 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:49:21.399 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:21.400 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:49:21.500 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:49:21.566 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:21.670 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:49:21.674 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:21.674 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:49:21.775 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:49:21.802 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:21.834 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:49:21.837 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:21.837 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:49:21.937 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:49:21.938 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:49:21.964 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:21.979 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:21 smithi018 ceph-mon[179647]: pgmap v1529: 129 pgs: 1 remapped+peering, 128 active+clean; 1.1 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail 2023-09-23T20:49:21.979 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:21 smithi018 ceph-mon[179647]: failed to encode map e617 with expected crc 2023-09-23T20:49:21.980 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:21 smithi018 ceph-mon[179647]: osdmap e617: 8 total, 8 up, 8 in 2023-09-23T20:49:21.980 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:21 smithi018 ceph-mon[179647]: 10.14 starting backfill to osd.1 from (0'0,0'0] MAX to 613'11214 2023-09-23T20:49:21.980 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:21 smithi018 ceph-mon[179647]: failed to encode map e617 with expected crc 2023-09-23T20:49:21.980 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:21 smithi018 ceph-mon[179647]: failed to encode map e617 with expected crc 2023-09-23T20:49:21.980 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:21 smithi018 ceph-mon[179647]: 10.14 starting backfill to osd.2 from (0'0,0'0] MAX to 613'11214 2023-09-23T20:49:21.980 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:21 smithi018 ceph-mon[184144]: pgmap v1529: 129 pgs: 1 remapped+peering, 128 active+clean; 1.1 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail 2023-09-23T20:49:21.980 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:21 smithi018 ceph-mon[184144]: failed to encode map e617 with expected crc 2023-09-23T20:49:21.980 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:21 smithi018 ceph-mon[184144]: osdmap e617: 8 total, 8 up, 8 in 2023-09-23T20:49:21.981 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:21 smithi018 ceph-mon[184144]: 10.14 starting backfill to osd.1 from (0'0,0'0] MAX to 613'11214 2023-09-23T20:49:21.981 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:21 smithi018 ceph-mon[184144]: failed to encode map e617 with expected crc 2023-09-23T20:49:21.981 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:21 smithi018 ceph-mon[184144]: failed to encode map e617 with expected crc 2023-09-23T20:49:21.981 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:21 smithi018 ceph-mon[184144]: 10.14 starting backfill to osd.2 from (0'0,0'0] MAX to 613'11214 2023-09-23T20:49:21.981 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:49:21.998 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:49:22.002 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:22.002 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:49:22.103 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:49:22.142 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:21 smithi149 ceph-mon[170394]: pgmap v1529: 129 pgs: 1 remapped+peering, 128 active+clean; 1.1 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail 2023-09-23T20:49:22.142 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:21 smithi149 ceph-mon[170394]: failed to encode map e617 with expected crc 2023-09-23T20:49:22.142 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:21 smithi149 ceph-mon[170394]: osdmap e617: 8 total, 8 up, 8 in 2023-09-23T20:49:22.143 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:21 smithi149 ceph-mon[170394]: 10.14 starting backfill to osd.1 from (0'0,0'0] MAX to 613'11214 2023-09-23T20:49:22.143 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:21 smithi149 ceph-mon[170394]: failed to encode map e617 with expected crc 2023-09-23T20:49:22.143 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:21 smithi149 ceph-mon[170394]: failed to encode map e617 with expected crc 2023-09-23T20:49:22.143 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:21 smithi149 ceph-mon[170394]: 10.14 starting backfill to osd.2 from (0'0,0'0] MAX to 613'11214 2023-09-23T20:49:22.149 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:22.219 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:49:22.222 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:22.223 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:49:22.323 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:49:22.355 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:22.398 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:49:22.404 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:22.404 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:49:22.505 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:49:22.556 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:22.616 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:49:22.622 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:22.623 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:49:22.723 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:49:22.767 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:22.970 INFO:teuthology.orchestra.run.smithi018.stderr:nodeep-scrub is set 2023-09-23T20:49:22.999 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:22 smithi018 ceph-mon[184144]: failed to encode map e617 with expected crc 2023-09-23T20:49:23.000 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:22 smithi018 ceph-mon[184144]: osdmap e618: 8 total, 8 up, 8 in 2023-09-23T20:49:23.000 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:22 smithi018 ceph-mon[184144]: failed to encode map e618 with expected crc 2023-09-23T20:49:23.000 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:22 smithi018 ceph-mon[184144]: failed to encode map e618 with expected crc 2023-09-23T20:49:23.000 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:22 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/257399039' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:49:23.000 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:22 smithi018 ceph-mon[179647]: failed to encode map e617 with expected crc 2023-09-23T20:49:23.000 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:22 smithi018 ceph-mon[179647]: osdmap e618: 8 total, 8 up, 8 in 2023-09-23T20:49:23.000 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:22 smithi018 ceph-mon[179647]: failed to encode map e618 with expected crc 2023-09-23T20:49:23.001 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:22 smithi018 ceph-mon[179647]: failed to encode map e618 with expected crc 2023-09-23T20:49:23.001 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:22 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/257399039' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:49:23.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:22 smithi149 ceph-mon[170394]: failed to encode map e617 with expected crc 2023-09-23T20:49:23.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:22 smithi149 ceph-mon[170394]: osdmap e618: 8 total, 8 up, 8 in 2023-09-23T20:49:23.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:22 smithi149 ceph-mon[170394]: failed to encode map e618 with expected crc 2023-09-23T20:49:23.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:22 smithi149 ceph-mon[170394]: failed to encode map e618 with expected crc 2023-09-23T20:49:23.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:22 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/257399039' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:49:23.248 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:49:23.256 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:23.257 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:49:23.357 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:49:23.430 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:23.492 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:49:23.497 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:23.498 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:49:23.598 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:49:23.682 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:23.805 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:49:23.812 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:23.813 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:49:23.847 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:49:23.914 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:49:24.010 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:24.065 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:49:24.077 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:23 smithi018 ceph-mon[179647]: failed to encode map e618 with expected crc 2023-09-23T20:49:24.077 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:23 smithi018 ceph-mon[179647]: pgmap v1532: 129 pgs: 1 active+remapped+backfilling, 1 remapped+peering, 127 active+clean; 1.0 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 584/50571 objects misplaced (1.155%) 2023-09-23T20:49:24.077 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:23 smithi018 ceph-mon[179647]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:49:24.077 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:23 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/257399039' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:49:24.077 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:23 smithi018 ceph-mon[179647]: osdmap e619: 8 total, 8 up, 8 in 2023-09-23T20:49:24.077 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:23 smithi018 ceph-mon[179647]: failed to encode map e619 with expected crc 2023-09-23T20:49:24.078 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:24.078 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:49:24.079 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:23 smithi018 ceph-mon[184144]: failed to encode map e618 with expected crc 2023-09-23T20:49:24.079 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:23 smithi018 ceph-mon[184144]: pgmap v1532: 129 pgs: 1 active+remapped+backfilling, 1 remapped+peering, 127 active+clean; 1.0 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 584/50571 objects misplaced (1.155%) 2023-09-23T20:49:24.079 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:23 smithi018 ceph-mon[184144]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:49:24.079 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:23 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/257399039' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:49:24.079 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:23 smithi018 ceph-mon[184144]: osdmap e619: 8 total, 8 up, 8 in 2023-09-23T20:49:24.079 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:23 smithi018 ceph-mon[184144]: failed to encode map e619 with expected crc 2023-09-23T20:49:24.179 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:49:24.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:23 smithi149 ceph-mon[170394]: failed to encode map e618 with expected crc 2023-09-23T20:49:24.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:23 smithi149 ceph-mon[170394]: pgmap v1532: 129 pgs: 1 active+remapped+backfilling, 1 remapped+peering, 127 active+clean; 1.0 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 584/50571 objects misplaced (1.155%) 2023-09-23T20:49:24.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:23 smithi149 ceph-mon[170394]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:49:24.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:23 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/257399039' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:49:24.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:23 smithi149 ceph-mon[170394]: osdmap e619: 8 total, 8 up, 8 in 2023-09-23T20:49:24.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:23 smithi149 ceph-mon[170394]: failed to encode map e619 with expected crc 2023-09-23T20:49:24.236 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:24.303 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:49:24.307 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:24.308 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:49:24.408 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:49:24.467 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:24.530 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:49:24.538 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:24.538 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:49:24.585 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 7, 3, 6, 5, 4, 2] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T20:49:24.585 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:49:24.585 INFO:tasks.thrashosds.thrasher:fixing pg num pool .mgr 2023-09-23T20:49:24.585 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:49:24.638 INFO:teuthology.orchestra.run.smithi018.stdout:ERROR: (22) Invalid argument 2023-09-23T20:49:24.638 INFO:teuthology.orchestra.run.smithi018.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:49:24.638 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:49:24.685 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:24.739 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:49:24.744 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:24.744 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:49:24.845 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:49:24.927 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:25.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:24 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "22"}]: dispatch 2023-09-23T20:49:25.217 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:24 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "22"}]: dispatch 2023-09-23T20:49:25.217 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:49:25 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:49:25] "GET /metrics HTTP/1.1" 200 36610 "" "Prometheus/2.43.0" 2023-09-23T20:49:25.218 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:24 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "22"}]: dispatch 2023-09-23T20:49:25.236 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:49:25.241 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:25.241 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:49:25.277 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T20:49:25.341 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:49:25.389 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:25.433 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:49:25.437 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:25.437 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:49:25.538 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:49:25.542 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T20:49:25.571 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:25.658 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:49:25.662 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:25.662 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:49:25.738 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:49:25.763 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:49:25.763 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:49:25.785 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:49:25.791 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:25.809 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:49:25.827 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:49:25.829 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:25.830 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:49:25.832 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:49:25.851 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:49:25.863 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:49:25.871 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:49:25.891 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:49:25.893 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:49:25.893 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:49:25.893 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:49:25.893 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:49:25.893 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:49:25.893 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:49:25.893 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:49:25.893 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:49:25.931 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:49:25.958 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:25.993 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:49:25.996 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:25.997 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:49:26.098 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:49:26.152 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:26.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:25 smithi149 ceph-mon[170394]: pgmap v1534: 129 pgs: 1 active+remapped+backfilling, 128 active+clean; 1011 MiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 584/48402 objects misplaced (1.207%) 2023-09-23T20:49:26.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:25 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "22"}]': finished 2023-09-23T20:49:26.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:25 smithi149 ceph-mon[170394]: osdmap e620: 8 total, 8 up, 8 in 2023-09-23T20:49:26.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:25 smithi149 ceph-mon[170394]: failed to encode map e620 with expected crc 2023-09-23T20:49:26.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:25 smithi149 ceph-mon[170394]: failed to encode map e620 with expected crc 2023-09-23T20:49:26.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:25 smithi149 ceph-mon[170394]: failed to encode map e620 with expected crc 2023-09-23T20:49:26.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:25 smithi149 ceph-mon[170394]: failed to encode map e620 with expected crc 2023-09-23T20:49:26.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:25 smithi149 ceph-mon[170394]: failed to encode map e620 with expected crc 2023-09-23T20:49:26.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:25 smithi149 ceph-mon[170394]: failed to encode map e620 with expected crc 2023-09-23T20:49:26.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:25 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:49:26.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:25 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3922283551' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:49:26.288 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:25 smithi018 ceph-mon[179647]: pgmap v1534: 129 pgs: 1 active+remapped+backfilling, 128 active+clean; 1011 MiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 584/48402 objects misplaced (1.207%) 2023-09-23T20:49:26.288 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:25 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "22"}]': finished 2023-09-23T20:49:26.288 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:25 smithi018 ceph-mon[179647]: osdmap e620: 8 total, 8 up, 8 in 2023-09-23T20:49:26.289 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:25 smithi018 ceph-mon[179647]: failed to encode map e620 with expected crc 2023-09-23T20:49:26.289 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:25 smithi018 ceph-mon[179647]: failed to encode map e620 with expected crc 2023-09-23T20:49:26.289 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:25 smithi018 ceph-mon[179647]: failed to encode map e620 with expected crc 2023-09-23T20:49:26.289 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:25 smithi018 ceph-mon[179647]: failed to encode map e620 with expected crc 2023-09-23T20:49:26.289 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:25 smithi018 ceph-mon[179647]: failed to encode map e620 with expected crc 2023-09-23T20:49:26.289 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:25 smithi018 ceph-mon[179647]: failed to encode map e620 with expected crc 2023-09-23T20:49:26.289 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:25 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:49:26.289 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:25 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3922283551' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:49:26.290 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:25 smithi018 ceph-mon[184144]: pgmap v1534: 129 pgs: 1 active+remapped+backfilling, 128 active+clean; 1011 MiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 584/48402 objects misplaced (1.207%) 2023-09-23T20:49:26.290 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:25 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "22"}]': finished 2023-09-23T20:49:26.290 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:25 smithi018 ceph-mon[184144]: osdmap e620: 8 total, 8 up, 8 in 2023-09-23T20:49:26.290 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:25 smithi018 ceph-mon[184144]: failed to encode map e620 with expected crc 2023-09-23T20:49:26.290 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:25 smithi018 ceph-mon[184144]: failed to encode map e620 with expected crc 2023-09-23T20:49:26.290 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:25 smithi018 ceph-mon[184144]: failed to encode map e620 with expected crc 2023-09-23T20:49:26.290 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:25 smithi018 ceph-mon[184144]: failed to encode map e620 with expected crc 2023-09-23T20:49:26.290 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:25 smithi018 ceph-mon[184144]: failed to encode map e620 with expected crc 2023-09-23T20:49:26.290 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:25 smithi018 ceph-mon[184144]: failed to encode map e620 with expected crc 2023-09-23T20:49:26.291 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:25 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:49:26.291 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:25 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3922283551' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:49:26.406 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:49:26.551 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:49:26.553 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:26.554 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:49:26.655 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:49:26.681 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:26.716 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:49:26.719 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:26.719 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:49:26.768 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:49:26.820 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:49:26.846 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:26.880 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:49:26.882 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:26.883 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:49:26.984 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:49:27.019 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:27.084 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:49:27.096 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:27.097 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:49:27.198 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:49:27.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:26 smithi149 ceph-mon[170394]: osdmap e621: 8 total, 8 up, 8 in 2023-09-23T20:49:27.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:26 smithi149 ceph-mon[170394]: failed to encode map e621 with expected crc 2023-09-23T20:49:27.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:26 smithi149 ceph-mon[170394]: failed to encode map e621 with expected crc 2023-09-23T20:49:27.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:26 smithi149 ceph-mon[170394]: failed to encode map e621 with expected crc 2023-09-23T20:49:27.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:26 smithi149 ceph-mon[170394]: failed to encode map e621 with expected crc 2023-09-23T20:49:27.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:26 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "23"}]: dispatch 2023-09-23T20:49:27.252 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:27.308 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:49:27.312 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:27.313 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:49:27.359 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:26 smithi018 ceph-mon[179647]: osdmap e621: 8 total, 8 up, 8 in 2023-09-23T20:49:27.359 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:26 smithi018 ceph-mon[179647]: failed to encode map e621 with expected crc 2023-09-23T20:49:27.359 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:26 smithi018 ceph-mon[179647]: failed to encode map e621 with expected crc 2023-09-23T20:49:27.359 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:26 smithi018 ceph-mon[179647]: failed to encode map e621 with expected crc 2023-09-23T20:49:27.359 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:26 smithi018 ceph-mon[179647]: failed to encode map e621 with expected crc 2023-09-23T20:49:27.359 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:26 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "23"}]: dispatch 2023-09-23T20:49:27.359 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:26 smithi018 ceph-mon[184144]: osdmap e621: 8 total, 8 up, 8 in 2023-09-23T20:49:27.360 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:26 smithi018 ceph-mon[184144]: failed to encode map e621 with expected crc 2023-09-23T20:49:27.360 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:26 smithi018 ceph-mon[184144]: failed to encode map e621 with expected crc 2023-09-23T20:49:27.360 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:26 smithi018 ceph-mon[184144]: failed to encode map e621 with expected crc 2023-09-23T20:49:27.360 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:26 smithi018 ceph-mon[184144]: failed to encode map e621 with expected crc 2023-09-23T20:49:27.360 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:26 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "23"}]: dispatch 2023-09-23T20:49:27.413 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:49:27.467 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:27.525 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:49:27.530 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:27.531 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:49:27.632 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:49:27.692 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:27.968 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:27 smithi018 ceph-mon[179647]: pgmap v1537: 129 pgs: 1 active+remapped+backfilling, 128 active+clean; 863 MiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 701 op/s; 584/41274 objects misplaced (1.415%); 7.7 MiB/s, 123 objects/s recovering 2023-09-23T20:49:27.969 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:27 smithi018 ceph-mon[179647]: failed to encode map e622 with expected crc 2023-09-23T20:49:27.969 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:27 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "23"}]': finished 2023-09-23T20:49:27.969 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:27 smithi018 ceph-mon[179647]: osdmap e622: 8 total, 8 up, 8 in 2023-09-23T20:49:27.969 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:27 smithi018 ceph-mon[179647]: 10.15 starting backfill to osd.1 from (0'0,0'0] MAX to 619'11529 2023-09-23T20:49:27.969 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:27 smithi018 ceph-mon[179647]: 10.15 starting backfill to osd.5 from (0'0,0'0] MAX to 619'11529 2023-09-23T20:49:27.971 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:49:27.977 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:27.977 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:49:28.078 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:49:28.105 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:28.139 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:49:28.142 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:28.143 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:49:28.193 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:49:28.195 INFO:teuthology.orchestra.run.smithi018.stderr:dumped all 2023-09-23T20:49:28.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:27 smithi149 ceph-mon[170394]: pgmap v1537: 129 pgs: 1 active+remapped+backfilling, 128 active+clean; 863 MiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 701 op/s; 584/41274 objects misplaced (1.415%); 7.7 MiB/s, 123 objects/s recovering 2023-09-23T20:49:28.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:27 smithi149 ceph-mon[170394]: failed to encode map e622 with expected crc 2023-09-23T20:49:28.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:27 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "23"}]': finished 2023-09-23T20:49:28.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:27 smithi149 ceph-mon[170394]: osdmap e622: 8 total, 8 up, 8 in 2023-09-23T20:49:28.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:27 smithi149 ceph-mon[170394]: 10.15 starting backfill to osd.1 from (0'0,0'0] MAX to 619'11529 2023-09-23T20:49:28.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:27 smithi149 ceph-mon[170394]: 10.15 starting backfill to osd.5 from (0'0,0'0] MAX to 619'11529 2023-09-23T20:49:28.219 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:27 smithi018 ceph-mon[184144]: pgmap v1537: 129 pgs: 1 active+remapped+backfilling, 128 active+clean; 863 MiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 701 op/s; 584/41274 objects misplaced (1.415%); 7.7 MiB/s, 123 objects/s recovering 2023-09-23T20:49:28.219 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:27 smithi018 ceph-mon[184144]: failed to encode map e622 with expected crc 2023-09-23T20:49:28.219 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:27 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "23"}]': finished 2023-09-23T20:49:28.219 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:27 smithi018 ceph-mon[184144]: osdmap e622: 8 total, 8 up, 8 in 2023-09-23T20:49:28.219 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:27 smithi018 ceph-mon[184144]: 10.15 starting backfill to osd.1 from (0'0,0'0] MAX to 619'11529 2023-09-23T20:49:28.219 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:27 smithi018 ceph-mon[184144]: 10.15 starting backfill to osd.5 from (0'0,0'0] MAX to 619'11529 2023-09-23T20:49:28.243 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:49:28.270 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:28.303 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:49:28.306 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:28.306 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:49:28.365 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:49:28.407 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:49:28.436 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:28.973 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:28 smithi018 ceph-mon[179647]: osdmap e623: 8 total, 8 up, 8 in 2023-09-23T20:49:29.025 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:49:29.029 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:29.029 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:49:29.117 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2023-09-23T20:49:29.118 INFO:teuthology.orchestra.run.smithi018.stdout: "size": 20, 2023-09-23T20:49:29.118 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 600, 2023-09-23T20:49:29.118 INFO:teuthology.orchestra.run.smithi018.stdout: "ops": [ 2023-09-23T20:49:29.118 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.118 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.36450.0:141027 9.6 9:6dcbc3cc:::benchmark_data_smithi018_381516_object27193:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e481)", 2023-09-23T20:49:29.118 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:43:15.747787+0000", 2023-09-23T20:49:29.118 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 373.36552490100001, 2023-09-23T20:49:29.119 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.040264882, 2023-09-23T20:49:29.119 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:49:29.119 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:49:29.119 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:49:29.119 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.36450", 2023-09-23T20:49:29.119 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:49:29.119 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 141027 2023-09-23T20:49:29.119 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.119 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:49:29.119 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.119 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:49:29.120 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.747787+0000", 2023-09-23T20:49:29.120 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:29.120 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.120 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.120 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:49:29.120 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.747787+0000", 2023-09-23T20:49:29.120 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:29.120 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.120 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.120 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:49:29.120 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.747785+0000", 2023-09-23T20:49:29.120 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:49:29.121 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.121 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.121 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:49:29.121 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.747801+0000", 2023-09-23T20:49:29.121 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.5854000000000001e-05 2023-09-23T20:49:29.121 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.121 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.121 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:49:29.121 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.747818+0000", 2023-09-23T20:49:29.121 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.7499e-05 2023-09-23T20:49:29.121 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.122 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.122 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:49:29.122 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.747829+0000", 2023-09-23T20:49:29.122 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0508e-05 2023-09-23T20:49:29.122 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.122 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.122 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:29.122 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.747912+0000", 2023-09-23T20:49:29.122 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.3052000000000004e-05 2023-09-23T20:49:29.122 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.122 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.122 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:49:29.123 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.747926+0000", 2023-09-23T20:49:29.123 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3577000000000001e-05 2023-09-23T20:49:29.123 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.123 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.123 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:29.123 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:16.785351+0000", 2023-09-23T20:49:29.123 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.037425635 2023-09-23T20:49:29.123 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.123 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.123 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:49:29.124 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:16.788052+0000", 2023-09-23T20:49:29.124 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0027005010000000001 2023-09-23T20:49:29.124 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:29.124 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:49:29.124 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:29.124 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.124 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.124 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.36450.0:141056 9.6 9:6159416c:::benchmark_data_smithi018_381516_object27222:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e481)", 2023-09-23T20:49:29.124 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:43:15.776179+0000", 2023-09-23T20:49:29.124 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 373.33713287299997, 2023-09-23T20:49:29.124 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.023410698, 2023-09-23T20:49:29.125 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:49:29.125 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:49:29.125 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:49:29.125 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.36450", 2023-09-23T20:49:29.125 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:49:29.125 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 141056 2023-09-23T20:49:29.125 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.125 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:49:29.125 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.125 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:49:29.125 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.776179+0000", 2023-09-23T20:49:29.125 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:29.126 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.126 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.126 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:49:29.126 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.776179+0000", 2023-09-23T20:49:29.126 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:29.126 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.126 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.126 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:49:29.126 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.776177+0000", 2023-09-23T20:49:29.126 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:49:29.126 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.126 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.127 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:49:29.127 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.776187+0000", 2023-09-23T20:49:29.127 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0278000000000001e-05 2023-09-23T20:49:29.127 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.127 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.127 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:49:29.127 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.776211+0000", 2023-09-23T20:49:29.127 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.3674000000000001e-05 2023-09-23T20:49:29.127 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.127 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.127 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:49:29.128 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.776222+0000", 2023-09-23T20:49:29.128 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1599e-05 2023-09-23T20:49:29.128 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.128 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.128 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:29.128 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.776301+0000", 2023-09-23T20:49:29.128 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.8295999999999997e-05 2023-09-23T20:49:29.128 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.128 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.128 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:49:29.128 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.776315+0000", 2023-09-23T20:49:29.128 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3906999999999999e-05 2023-09-23T20:49:29.129 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.129 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.129 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:29.129 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:16.788351+0000", 2023-09-23T20:49:29.129 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0120362730000001 2023-09-23T20:49:29.129 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.129 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.129 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:49:29.129 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:16.789277+0000", 2023-09-23T20:49:29.129 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00092556400000000003 2023-09-23T20:49:29.129 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.129 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.130 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,5", 2023-09-23T20:49:29.130 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:16.789391+0000", 2023-09-23T20:49:29.130 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000114228 2023-09-23T20:49:29.130 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.130 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.130 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:29.130 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:16.797045+0000", 2023-09-23T20:49:29.131 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0076541880000000001 2023-09-23T20:49:29.131 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.131 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.131 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:49:29.131 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:16.798259+0000", 2023-09-23T20:49:29.131 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.001214033 2023-09-23T20:49:29.131 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.132 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.132 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:29.132 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:16.799494+0000", 2023-09-23T20:49:29.132 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0012353520000000001 2023-09-23T20:49:29.132 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.132 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.133 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:49:29.133 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:16.799549+0000", 2023-09-23T20:49:29.133 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.4401999999999999e-05 2023-09-23T20:49:29.133 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.133 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.133 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:49:29.133 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:16.799589+0000", 2023-09-23T20:49:29.134 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.0774e-05 2023-09-23T20:49:29.134 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:29.134 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:49:29.134 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:29.134 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.134 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.134 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.36450.0:141066 9.6 9:6eed4220:::benchmark_data_smithi018_381516_object27232:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e481)", 2023-09-23T20:49:29.134 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:43:15.809374+0000", 2023-09-23T20:49:29.134 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 373.30393800899998, 2023-09-23T20:49:29.135 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0041945379999999, 2023-09-23T20:49:29.135 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:49:29.135 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:49:29.135 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:49:29.135 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.36450", 2023-09-23T20:49:29.135 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:49:29.136 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 141066 2023-09-23T20:49:29.136 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.136 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:49:29.136 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.136 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:49:29.136 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.809374+0000", 2023-09-23T20:49:29.136 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:29.136 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.136 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.137 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:49:29.137 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.809374+0000", 2023-09-23T20:49:29.137 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:29.137 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.137 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.137 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:49:29.137 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.809372+0000", 2023-09-23T20:49:29.137 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:49:29.137 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.137 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.137 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:49:29.138 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.809383+0000", 2023-09-23T20:49:29.138 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1195999999999999e-05 2023-09-23T20:49:29.138 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.138 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.138 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:49:29.138 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.809407+0000", 2023-09-23T20:49:29.138 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.3937000000000001e-05 2023-09-23T20:49:29.138 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.138 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.138 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:49:29.138 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.809419+0000", 2023-09-23T20:49:29.138 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2525e-05 2023-09-23T20:49:29.139 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.139 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.139 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:29.139 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.809490+0000", 2023-09-23T20:49:29.139 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.1048000000000001e-05 2023-09-23T20:49:29.139 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.139 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.139 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:49:29.139 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.809508+0000", 2023-09-23T20:49:29.139 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.7263000000000001e-05 2023-09-23T20:49:29.139 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.140 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.140 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:29.140 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:16.793445+0000", 2023-09-23T20:49:29.140 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.98393740500000004 2023-09-23T20:49:29.140 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.140 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.140 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:49:29.140 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:16.793568+0000", 2023-09-23T20:49:29.140 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000123492 2023-09-23T20:49:29.140 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.140 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.140 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,5", 2023-09-23T20:49:29.141 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:16.793699+0000", 2023-09-23T20:49:29.141 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000130455 2023-09-23T20:49:29.141 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.141 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.141 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:49:29.141 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:16.798264+0000", 2023-09-23T20:49:29.141 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0045648709999999999 2023-09-23T20:49:29.141 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.141 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.141 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:29.141 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:16.800278+0000", 2023-09-23T20:49:29.141 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0020140480000000001 2023-09-23T20:49:29.142 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.142 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.142 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:29.142 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:16.813461+0000", 2023-09-23T20:49:29.142 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.01318304 2023-09-23T20:49:29.142 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.142 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.142 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:49:29.142 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:16.813519+0000", 2023-09-23T20:49:29.142 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.7908000000000002e-05 2023-09-23T20:49:29.142 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.143 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.143 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:49:29.143 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:16.813568+0000", 2023-09-23T20:49:29.143 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.9450000000000003e-05 2023-09-23T20:49:29.143 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:29.143 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:49:29.143 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:29.143 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.143 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.143 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:71267 10.9 10:95cde882:::benchmark_data_smithi018_422431_object71266:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e541)", 2023-09-23T20:49:29.143 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:46:28.073431+0000", 2023-09-23T20:49:29.143 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 181.03988107999999, 2023-09-23T20:49:29.144 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.16533946999999999, 2023-09-23T20:49:29.144 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:49:29.144 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:49:29.144 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:49:29.144 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:49:29.144 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:49:29.144 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 71267 2023-09-23T20:49:29.144 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.144 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:49:29.144 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.144 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:49:29.144 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:28.073431+0000", 2023-09-23T20:49:29.145 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:29.145 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.145 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.145 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:49:29.145 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:28.073431+0000", 2023-09-23T20:49:29.145 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:29.145 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.145 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.145 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:49:29.145 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:28.073424+0000", 2023-09-23T20:49:29.145 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999933 2023-09-23T20:49:29.145 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.146 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.146 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:49:29.146 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:28.073454+0000", 2023-09-23T20:49:29.146 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.0419000000000001e-05 2023-09-23T20:49:29.146 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.146 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.146 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:49:29.146 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:28.073472+0000", 2023-09-23T20:49:29.146 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.7674999999999999e-05 2023-09-23T20:49:29.146 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.146 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.147 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:49:29.147 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:28.073481+0000", 2023-09-23T20:49:29.147 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.0110000000000006e-06 2023-09-23T20:49:29.147 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.147 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.147 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:29.147 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:28.073546+0000", 2023-09-23T20:49:29.147 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.5167999999999997e-05 2023-09-23T20:49:29.147 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.147 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.147 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:49:29.147 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:28.073676+0000", 2023-09-23T20:49:29.148 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00013000800000000001 2023-09-23T20:49:29.148 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.148 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.148 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 5,7", 2023-09-23T20:49:29.148 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:28.073787+0000", 2023-09-23T20:49:29.148 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000110859 2023-09-23T20:49:29.148 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.148 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.148 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:49:29.148 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:28.076584+0000", 2023-09-23T20:49:29.148 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0027975579999999999 2023-09-23T20:49:29.148 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.149 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.149 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:29.149 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:28.079571+0000", 2023-09-23T20:49:29.149 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0029865590000000002 2023-09-23T20:49:29.149 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.149 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.149 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:29.149 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:28.238700+0000", 2023-09-23T20:49:29.149 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.15912913300000001 2023-09-23T20:49:29.149 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.149 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.149 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:49:29.150 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:28.238738+0000", 2023-09-23T20:49:29.150 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.7558999999999997e-05 2023-09-23T20:49:29.150 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.150 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.150 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:49:29.150 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:28.238770+0000", 2023-09-23T20:49:29.150 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.2317999999999998e-05 2023-09-23T20:49:29.150 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:29.150 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:49:29.150 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:29.150 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.151 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.151 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:71276 10.9 10:907177a2:::benchmark_data_smithi018_422431_object71275:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e541)", 2023-09-23T20:49:29.151 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:46:28.081074+0000", 2023-09-23T20:49:29.151 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 181.03223740999999, 2023-09-23T20:49:29.151 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.15788343699999999, 2023-09-23T20:49:29.151 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:49:29.151 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:49:29.151 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:49:29.151 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:49:29.151 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:49:29.151 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 71276 2023-09-23T20:49:29.152 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.152 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:49:29.152 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.152 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:49:29.152 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:28.081074+0000", 2023-09-23T20:49:29.152 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:29.152 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.152 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.152 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:49:29.152 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:28.081074+0000", 2023-09-23T20:49:29.152 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:29.152 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.153 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.153 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:49:29.153 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:28.081073+0000", 2023-09-23T20:49:29.153 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:49:29.153 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.153 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.153 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:49:29.153 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:28.081095+0000", 2023-09-23T20:49:29.153 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.1563e-05 2023-09-23T20:49:29.153 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.153 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.154 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:49:29.154 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:28.081107+0000", 2023-09-23T20:49:29.154 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2649e-05 2023-09-23T20:49:29.154 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.154 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.154 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:49:29.154 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:28.081116+0000", 2023-09-23T20:49:29.154 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.5739999999999996e-06 2023-09-23T20:49:29.154 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.154 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.154 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:29.155 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:28.081158+0000", 2023-09-23T20:49:29.155 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.1937999999999997e-05 2023-09-23T20:49:29.155 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.155 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.155 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:49:29.155 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:28.081285+0000", 2023-09-23T20:49:29.155 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000127007 2023-09-23T20:49:29.155 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.155 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.155 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 5,7", 2023-09-23T20:49:29.155 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:28.081381+0000", 2023-09-23T20:49:29.156 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.6520000000000004e-05 2023-09-23T20:49:29.156 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.156 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.156 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:49:29.156 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:28.083932+0000", 2023-09-23T20:49:29.156 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0025503470000000001 2023-09-23T20:49:29.156 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.156 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.156 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:29.156 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:28.085034+0000", 2023-09-23T20:49:29.156 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.001102502 2023-09-23T20:49:29.156 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.157 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.157 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:29.157 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:28.238854+0000", 2023-09-23T20:49:29.157 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.15382019399999999 2023-09-23T20:49:29.157 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.157 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.157 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:49:29.157 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:28.238915+0000", 2023-09-23T20:49:29.157 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.0677999999999997e-05 2023-09-23T20:49:29.157 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.157 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.157 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:49:29.158 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:28.238958+0000", 2023-09-23T20:49:29.158 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.2759999999999997e-05 2023-09-23T20:49:29.158 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:29.158 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:49:29.158 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:29.158 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.158 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.158 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:71286 10.9 10:930667ae:::benchmark_data_smithi018_422431_object71285:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e541)", 2023-09-23T20:49:29.158 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:46:28.088975+0000", 2023-09-23T20:49:29.158 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 181.024336245, 2023-09-23T20:49:29.159 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.154346186, 2023-09-23T20:49:29.159 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:49:29.159 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:49:29.159 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:49:29.159 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:49:29.159 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:49:29.159 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 71286 2023-09-23T20:49:29.159 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.159 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:49:29.159 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.159 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:49:29.159 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:28.088975+0000", 2023-09-23T20:49:29.160 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:29.160 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.160 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.160 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:49:29.160 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:28.088975+0000", 2023-09-23T20:49:29.160 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:29.160 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.160 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.160 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:49:29.160 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:28.088974+0000", 2023-09-23T20:49:29.160 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:49:29.161 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.161 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.161 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:49:29.161 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:28.088999+0000", 2023-09-23T20:49:29.161 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.5233999999999999e-05 2023-09-23T20:49:29.161 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.161 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.161 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:49:29.161 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:28.089012+0000", 2023-09-23T20:49:29.161 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2500999999999999e-05 2023-09-23T20:49:29.161 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.161 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.162 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:49:29.162 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:28.089020+0000", 2023-09-23T20:49:29.162 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.5809999999999997e-06 2023-09-23T20:49:29.162 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.162 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.162 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:29.162 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:28.089866+0000", 2023-09-23T20:49:29.162 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00084572800000000002 2023-09-23T20:49:29.162 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.162 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.162 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:49:29.163 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:28.090034+0000", 2023-09-23T20:49:29.163 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00016787099999999999 2023-09-23T20:49:29.163 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.163 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.163 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 5,7", 2023-09-23T20:49:29.163 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:28.090193+0000", 2023-09-23T20:49:29.163 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000159273 2023-09-23T20:49:29.163 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.163 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.163 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:49:29.163 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:28.092888+0000", 2023-09-23T20:49:29.163 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0026951620000000001 2023-09-23T20:49:29.164 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.164 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.164 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:29.164 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:28.096075+0000", 2023-09-23T20:49:29.164 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0031865679999999999 2023-09-23T20:49:29.164 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.164 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.164 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:29.164 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:28.243224+0000", 2023-09-23T20:49:29.164 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.147149322 2023-09-23T20:49:29.164 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.164 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.165 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:49:29.165 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:28.243283+0000", 2023-09-23T20:49:29.165 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.8207e-05 2023-09-23T20:49:29.165 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.165 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.165 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:49:29.165 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:28.243322+0000", 2023-09-23T20:49:29.165 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.9085000000000003e-05 2023-09-23T20:49:29.165 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:29.165 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:49:29.165 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:29.166 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.166 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.166 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_repop(client.46625.0:83262 10.e e542/531 10:789b1835:::benchmark_data_smithi018_422431_object83261:head v 542'5212, mlcod=542'5210)", 2023-09-23T20:49:29.166 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:46:40.385894+0000", 2023-09-23T20:49:29.166 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 168.72741756600001, 2023-09-23T20:49:29.166 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.153847647, 2023-09-23T20:49:29.166 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:49:29.166 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:49:29.166 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:49:29.166 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.166 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:49:29.166 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:40.385894+0000", 2023-09-23T20:49:29.167 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:29.167 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.167 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.167 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:49:29.167 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:40.385894+0000", 2023-09-23T20:49:29.167 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:29.167 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.167 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.167 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:49:29.167 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:40.385893+0000", 2023-09-23T20:49:29.167 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.999999 2023-09-23T20:49:29.168 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.168 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.168 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:49:29.168 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:40.385939+0000", 2023-09-23T20:49:29.168 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.6035000000000002e-05 2023-09-23T20:49:29.168 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.168 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.168 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:49:29.168 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:40.385952+0000", 2023-09-23T20:49:29.168 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3210999999999999e-05 2023-09-23T20:49:29.168 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.169 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.169 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:49:29.169 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:40.385961+0000", 2023-09-23T20:49:29.169 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.6989999999999992e-06 2023-09-23T20:49:29.169 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.169 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.169 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:29.169 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:40.387839+0000", 2023-09-23T20:49:29.169 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.001877641 2023-09-23T20:49:29.169 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.169 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.169 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:49:29.170 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:40.387860+0000", 2023-09-23T20:49:29.170 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.1211000000000001e-05 2023-09-23T20:49:29.170 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.170 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.170 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:49:29.170 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:40.539700+0000", 2023-09-23T20:49:29.170 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.151839844 2023-09-23T20:49:29.170 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.170 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.170 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:49:29.170 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:40.539742+0000", 2023-09-23T20:49:29.171 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.1909000000000001e-05 2023-09-23T20:49:29.171 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:29.171 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:49:29.171 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:29.171 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.171 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.171 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:83272 10.9 10:95587c0a:::benchmark_data_smithi018_422431_object83271:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e542)", 2023-09-23T20:49:29.171 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:46:40.390102+0000", 2023-09-23T20:49:29.171 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 168.72320958500001, 2023-09-23T20:49:29.171 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.15593806800000001, 2023-09-23T20:49:29.171 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:49:29.172 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:49:29.172 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:49:29.172 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:49:29.172 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:49:29.172 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 83272 2023-09-23T20:49:29.172 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.172 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:49:29.172 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.172 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:49:29.172 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:40.390102+0000", 2023-09-23T20:49:29.172 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:29.172 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.173 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.173 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:49:29.173 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:40.390102+0000", 2023-09-23T20:49:29.173 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:29.173 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.173 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.173 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:49:29.173 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:40.390100+0000", 2023-09-23T20:49:29.173 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:49:29.173 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.173 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.174 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:49:29.174 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:40.390127+0000", 2023-09-23T20:49:29.174 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.6325999999999998e-05 2023-09-23T20:49:29.174 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.174 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.174 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:49:29.174 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:40.390141+0000", 2023-09-23T20:49:29.174 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.4031e-05 2023-09-23T20:49:29.174 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.174 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.174 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:49:29.175 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:40.390150+0000", 2023-09-23T20:49:29.175 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.6020000000000006e-06 2023-09-23T20:49:29.175 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.175 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.175 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:29.175 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:40.390199+0000", 2023-09-23T20:49:29.175 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.8235999999999997e-05 2023-09-23T20:49:29.175 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.175 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.175 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:49:29.175 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:40.390320+0000", 2023-09-23T20:49:29.175 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000121499 2023-09-23T20:49:29.176 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.176 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.176 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 5,7", 2023-09-23T20:49:29.176 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:40.390421+0000", 2023-09-23T20:49:29.176 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000101269 2023-09-23T20:49:29.176 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.176 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.176 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:29.176 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:40.542824+0000", 2023-09-23T20:49:29.176 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.15240235999999999 2023-09-23T20:49:29.176 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.176 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.177 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:29.177 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:40.542854+0000", 2023-09-23T20:49:29.177 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.9850999999999999e-05 2023-09-23T20:49:29.177 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.177 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.177 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:49:29.177 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:40.545959+0000", 2023-09-23T20:49:29.177 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.003105137 2023-09-23T20:49:29.177 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.177 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.177 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:49:29.178 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:40.545999+0000", 2023-09-23T20:49:29.178 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.0527e-05 2023-09-23T20:49:29.178 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.178 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.178 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:49:29.178 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:40.546040+0000", 2023-09-23T20:49:29.178 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.0836e-05 2023-09-23T20:49:29.178 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:29.178 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:49:29.178 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:29.178 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.178 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.179 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:83275 10.5 10:a0b8ad7f:::benchmark_data_smithi018_422431_object83274:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e542)", 2023-09-23T20:49:29.179 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:46:40.393703+0000", 2023-09-23T20:49:29.179 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 168.719608497, 2023-09-23T20:49:29.179 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.162215779, 2023-09-23T20:49:29.179 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:49:29.179 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:49:29.179 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:49:29.179 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:49:29.179 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:49:29.179 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 83275 2023-09-23T20:49:29.181 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.181 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:49:29.181 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.181 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:49:29.181 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:40.393703+0000", 2023-09-23T20:49:29.181 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:29.181 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.181 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.181 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:49:29.182 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:40.393703+0000", 2023-09-23T20:49:29.182 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:29.182 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.182 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.182 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:49:29.182 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:40.393701+0000", 2023-09-23T20:49:29.182 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:49:29.182 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.182 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.182 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:49:29.182 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:40.393727+0000", 2023-09-23T20:49:29.182 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.5211e-05 2023-09-23T20:49:29.183 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.183 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.183 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:49:29.183 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:40.393740+0000", 2023-09-23T20:49:29.183 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3072e-05 2023-09-23T20:49:29.183 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.183 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.183 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:49:29.183 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:40.393748+0000", 2023-09-23T20:49:29.183 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.208e-06 2023-09-23T20:49:29.183 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.184 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.184 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:29.184 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:40.393808+0000", 2023-09-23T20:49:29.184 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.0146999999999999e-05 2023-09-23T20:49:29.184 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.184 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.184 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:49:29.184 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:40.393975+0000", 2023-09-23T20:49:29.184 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00016674100000000001 2023-09-23T20:49:29.184 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.184 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.184 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:49:29.184 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:40.394099+0000", 2023-09-23T20:49:29.185 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000124609 2023-09-23T20:49:29.185 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.185 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.185 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:29.185 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:40.542697+0000", 2023-09-23T20:49:29.185 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.148597652 2023-09-23T20:49:29.185 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.185 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.185 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:49:29.185 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:40.549124+0000", 2023-09-23T20:49:29.185 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0064272089999999997 2023-09-23T20:49:29.186 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.186 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.186 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:29.186 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:40.555822+0000", 2023-09-23T20:49:29.186 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0066974019999999999 2023-09-23T20:49:29.186 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.186 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.186 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:49:29.186 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:40.555880+0000", 2023-09-23T20:49:29.186 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.8248999999999997e-05 2023-09-23T20:49:29.186 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.186 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.187 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:49:29.187 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:46:40.555919+0000", 2023-09-23T20:49:29.187 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.9118000000000001e-05 2023-09-23T20:49:29.187 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:29.187 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:49:29.187 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:29.187 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.187 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.187 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:140123 10.5 10:a2a404a4:::benchmark_data_smithi018_422431_object35627:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e572)", 2023-09-23T20:49:29.187 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:47:59.377280+0000", 2023-09-23T20:49:29.187 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 89.736031912000001, 2023-09-23T20:49:29.188 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.164140271, 2023-09-23T20:49:29.188 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:49:29.188 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:49:29.188 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:49:29.188 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:49:29.188 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:49:29.188 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 140123 2023-09-23T20:49:29.188 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.188 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:49:29.188 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.188 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:49:29.188 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:59.377280+0000", 2023-09-23T20:49:29.189 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:29.189 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.189 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.189 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:49:29.189 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:59.377280+0000", 2023-09-23T20:49:29.189 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:29.189 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.189 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.189 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:49:29.189 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:59.377279+0000", 2023-09-23T20:49:29.189 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:49:29.190 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.190 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.190 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:49:29.190 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:59.377282+0000", 2023-09-23T20:49:29.190 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.0639999999999998e-06 2023-09-23T20:49:29.190 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.190 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.190 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:49:29.190 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:59.377291+0000", 2023-09-23T20:49:29.190 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.3310000000000004e-06 2023-09-23T20:49:29.190 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.191 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.191 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:49:29.191 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:59.377296+0000", 2023-09-23T20:49:29.191 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.1900000000000003e-06 2023-09-23T20:49:29.191 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.191 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.191 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:29.191 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:59.377374+0000", 2023-09-23T20:49:29.191 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.7942999999999996e-05 2023-09-23T20:49:29.191 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.191 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.191 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:49:29.192 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:59.377484+0000", 2023-09-23T20:49:29.192 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000110393 2023-09-23T20:49:29.192 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.192 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.192 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:49:29.192 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:59.377597+0000", 2023-09-23T20:49:29.192 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00011308599999999999 2023-09-23T20:49:29.192 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.192 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.192 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:49:29.192 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:59.385263+0000", 2023-09-23T20:49:29.192 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0076660280000000001 2023-09-23T20:49:29.193 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.193 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.193 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:29.193 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:59.390028+0000", 2023-09-23T20:49:29.193 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0047646959999999997 2023-09-23T20:49:29.193 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.193 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.193 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:29.193 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:59.541313+0000", 2023-09-23T20:49:29.193 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.15128524400000001 2023-09-23T20:49:29.193 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.193 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.194 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:49:29.194 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:59.541373+0000", 2023-09-23T20:49:29.194 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.9521000000000002e-05 2023-09-23T20:49:29.194 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.194 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.194 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:49:29.194 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:59.541420+0000", 2023-09-23T20:49:29.194 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.7199e-05 2023-09-23T20:49:29.194 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:29.194 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:49:29.195 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:29.195 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.195 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.195 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:140134 10.3 10:c129d24e:::benchmark_data_smithi018_422431_object35638:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e572)", 2023-09-23T20:49:29.195 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:47:59.387906+0000", 2023-09-23T20:49:29.195 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 89.725405636000005, 2023-09-23T20:49:29.195 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.16740925700000001, 2023-09-23T20:49:29.195 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:49:29.195 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:49:29.195 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:49:29.195 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:49:29.195 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:49:29.196 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 140134 2023-09-23T20:49:29.196 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.196 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:49:29.196 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.196 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:49:29.196 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:59.387906+0000", 2023-09-23T20:49:29.196 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:29.196 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.196 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.196 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:49:29.196 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:59.387906+0000", 2023-09-23T20:49:29.196 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:29.197 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.197 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.197 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:49:29.197 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:59.387906+0000", 2023-09-23T20:49:29.197 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:49:29.197 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.197 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.197 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:49:29.197 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:59.387908+0000", 2023-09-23T20:49:29.197 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.4099999999999998e-06 2023-09-23T20:49:29.197 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.197 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.198 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:49:29.198 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:59.387914+0000", 2023-09-23T20:49:29.198 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.5319999999999997e-06 2023-09-23T20:49:29.198 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.198 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.198 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:49:29.198 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:59.387918+0000", 2023-09-23T20:49:29.198 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.9030000000000003e-06 2023-09-23T20:49:29.198 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.198 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.198 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:29.198 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:59.388000+0000", 2023-09-23T20:49:29.199 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.1559999999999998e-05 2023-09-23T20:49:29.199 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.199 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.199 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:49:29.199 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:59.388112+0000", 2023-09-23T20:49:29.199 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000111673 2023-09-23T20:49:29.199 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.199 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.199 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 3,5", 2023-09-23T20:49:29.199 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:59.388218+0000", 2023-09-23T20:49:29.199 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00010593499999999999 2023-09-23T20:49:29.200 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.200 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.200 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:49:29.200 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:59.393069+0000", 2023-09-23T20:49:29.200 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0048509099999999999 2023-09-23T20:49:29.200 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.200 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.200 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:29.200 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:59.413503+0000", 2023-09-23T20:49:29.200 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.020434902000000001 2023-09-23T20:49:29.200 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.200 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.201 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:29.201 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:59.555219+0000", 2023-09-23T20:49:29.201 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.14171510300000001 2023-09-23T20:49:29.201 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.201 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.201 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:49:29.201 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:59.555272+0000", 2023-09-23T20:49:29.201 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.3764999999999998e-05 2023-09-23T20:49:29.201 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.201 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.201 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:49:29.202 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:59.555315+0000", 2023-09-23T20:49:29.202 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.2954000000000002e-05 2023-09-23T20:49:29.202 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:29.202 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:49:29.202 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:29.202 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.202 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.202 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:140137 10.5 10:a845e5c4:::benchmark_data_smithi018_422431_object35641:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e572)", 2023-09-23T20:49:29.202 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:47:59.387929+0000", 2023-09-23T20:49:29.202 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 89.725382741999994, 2023-09-23T20:49:29.202 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.16761156399999999, 2023-09-23T20:49:29.203 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:49:29.203 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:49:29.203 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:49:29.203 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:49:29.203 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:49:29.203 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 140137 2023-09-23T20:49:29.203 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.203 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:49:29.203 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.203 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:49:29.203 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:59.387929+0000", 2023-09-23T20:49:29.204 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:29.204 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.204 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.204 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:49:29.204 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:59.387929+0000", 2023-09-23T20:49:29.204 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:29.204 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.204 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.204 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:49:29.204 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:59.387929+0000", 2023-09-23T20:49:29.204 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:49:29.205 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.205 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.205 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:49:29.205 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:59.387931+0000", 2023-09-23T20:49:29.205 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.1600000000000001e-06 2023-09-23T20:49:29.205 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.205 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.205 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:49:29.205 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:59.387936+0000", 2023-09-23T20:49:29.205 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.2660000000000001e-06 2023-09-23T20:49:29.205 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.205 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.206 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:49:29.206 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:59.387941+0000", 2023-09-23T20:49:29.206 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.5399999999999997e-06 2023-09-23T20:49:29.206 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.206 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.206 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:29.206 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:59.390055+0000", 2023-09-23T20:49:29.206 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0021144950000000001 2023-09-23T20:49:29.206 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.206 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.206 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:49:29.206 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:59.390162+0000", 2023-09-23T20:49:29.207 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000106508 2023-09-23T20:49:29.207 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.207 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.207 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:49:29.207 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:59.390270+0000", 2023-09-23T20:49:29.207 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00010854 2023-09-23T20:49:29.207 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.207 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.207 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:49:29.207 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:59.395210+0000", 2023-09-23T20:49:29.207 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0049402140000000001 2023-09-23T20:49:29.208 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.208 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.208 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:29.208 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:59.401435+0000", 2023-09-23T20:49:29.208 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0062244199999999996 2023-09-23T20:49:29.208 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.208 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.208 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:29.208 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:59.555468+0000", 2023-09-23T20:49:29.208 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.15403361600000001 2023-09-23T20:49:29.208 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.208 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.209 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:49:29.209 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:59.555507+0000", 2023-09-23T20:49:29.209 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.8739000000000003e-05 2023-09-23T20:49:29.209 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.209 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.209 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:49:29.209 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:59.555540+0000", 2023-09-23T20:49:29.209 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.3376e-05 2023-09-23T20:49:29.209 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:29.209 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:49:29.209 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:29.209 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.210 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.210 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:158980 10.9 10:92546b90:::benchmark_data_smithi018_422431_object54484:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e576)", 2023-09-23T20:49:29.210 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:48:22.725337+0000", 2023-09-23T20:49:29.210 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 66.387974275999994, 2023-09-23T20:49:29.210 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.163729649, 2023-09-23T20:49:29.210 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:49:29.210 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:49:29.210 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:49:29.210 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:49:29.210 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:49:29.210 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 158980 2023-09-23T20:49:29.210 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.211 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:49:29.211 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.211 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:49:29.211 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:22.725337+0000", 2023-09-23T20:49:29.211 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:29.211 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.211 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.211 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:49:29.211 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:22.725337+0000", 2023-09-23T20:49:29.211 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:29.211 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.212 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.212 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:49:29.212 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:22.725336+0000", 2023-09-23T20:49:29.212 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:49:29.212 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.212 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.212 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:49:29.212 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:22.725343+0000", 2023-09-23T20:49:29.212 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.7150000000000001e-06 2023-09-23T20:49:29.212 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.212 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.212 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:49:29.213 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:22.725355+0000", 2023-09-23T20:49:29.213 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2534999999999999e-05 2023-09-23T20:49:29.213 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.213 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.213 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:49:29.213 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:22.725363+0000", 2023-09-23T20:49:29.213 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.7829999999999999e-06 2023-09-23T20:49:29.213 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.213 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.213 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:29.213 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:22.725410+0000", 2023-09-23T20:49:29.213 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.7497999999999998e-05 2023-09-23T20:49:29.213 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.214 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.214 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:49:29.214 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:22.725548+0000", 2023-09-23T20:49:29.214 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000137353 2023-09-23T20:49:29.214 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.214 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.214 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 5,7", 2023-09-23T20:49:29.214 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:22.725685+0000", 2023-09-23T20:49:29.214 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00013711800000000001 2023-09-23T20:49:29.214 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.214 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.215 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:49:29.215 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:22.731096+0000", 2023-09-23T20:49:29.215 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0054108309999999996 2023-09-23T20:49:29.215 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.215 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.215 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:29.215 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:22.733566+0000", 2023-09-23T20:49:29.215 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.002470014 2023-09-23T20:49:29.215 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.215 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.215 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:29.215 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:22.888991+0000", 2023-09-23T20:49:29.216 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.15542498199999999 2023-09-23T20:49:29.216 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.216 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.216 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:49:29.216 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:22.889032+0000", 2023-09-23T20:49:29.216 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.1010999999999997e-05 2023-09-23T20:49:29.216 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.216 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.216 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:49:29.216 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:22.889067+0000", 2023-09-23T20:49:29.216 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.5308999999999997e-05 2023-09-23T20:49:29.216 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:29.217 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:49:29.217 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:29.217 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.217 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.217 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:158988 10.9 10:9fe441aa:::benchmark_data_smithi018_422431_object54492:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e576)", 2023-09-23T20:49:29.217 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:48:22.739956+0000", 2023-09-23T20:49:29.217 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 66.373355775999997, 2023-09-23T20:49:29.217 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.16609433700000001, 2023-09-23T20:49:29.217 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:49:29.217 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:49:29.217 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:49:29.218 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:49:29.218 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:49:29.218 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 158988 2023-09-23T20:49:29.218 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.218 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:49:29.218 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.218 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:49:29.218 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:22.739956+0000", 2023-09-23T20:49:29.218 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:29.218 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.218 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.219 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:49:29.219 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:22.739956+0000", 2023-09-23T20:49:29.219 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:29.219 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.219 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.219 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:49:29.219 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:22.739954+0000", 2023-09-23T20:49:29.219 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:49:29.219 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.219 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.219 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:49:29.219 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:22.739962+0000", 2023-09-23T20:49:29.220 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.9039999999999992e-06 2023-09-23T20:49:29.220 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.220 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.220 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:49:29.220 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:22.739977+0000", 2023-09-23T20:49:29.220 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.541e-05 2023-09-23T20:49:29.220 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.220 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.220 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:49:29.220 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:22.739987+0000", 2023-09-23T20:49:29.220 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.6439999999999995e-06 2023-09-23T20:49:29.221 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.221 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.221 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:29.221 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:22.740032+0000", 2023-09-23T20:49:29.221 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.5411000000000003e-05 2023-09-23T20:49:29.221 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.221 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.221 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:49:29.221 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:22.740174+0000", 2023-09-23T20:49:29.221 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000141371 2023-09-23T20:49:29.221 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.221 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.222 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 5,7", 2023-09-23T20:49:29.222 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:22.740287+0000", 2023-09-23T20:49:29.222 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00011335700000000001 2023-09-23T20:49:29.222 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.222 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.222 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:49:29.222 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:22.744490+0000", 2023-09-23T20:49:29.222 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0042024990000000002 2023-09-23T20:49:29.222 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.222 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.222 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:29.223 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:22.750024+0000", 2023-09-23T20:49:29.223 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.005534331 2023-09-23T20:49:29.223 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.223 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.223 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:49:29.223 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:22.906050+0000", 2023-09-23T20:49:29.223 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.156026254 2023-09-23T20:49:29.223 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:29.223 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:49:29.223 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:29.223 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.223 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.224 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:158996 10.9 10:9667fd03:::benchmark_data_smithi018_422431_object54500:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e576)", 2023-09-23T20:49:29.224 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:48:22.744643+0000", 2023-09-23T20:49:29.224 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 66.368668493000001, 2023-09-23T20:49:29.224 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.16140559400000001, 2023-09-23T20:49:29.224 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:49:29.224 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "waiting for sub ops", 2023-09-23T20:49:29.224 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:49:29.224 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:49:29.224 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:49:29.224 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 158996 2023-09-23T20:49:29.224 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.224 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:49:29.224 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.225 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:49:29.225 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:22.744643+0000", 2023-09-23T20:49:29.225 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:29.225 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.225 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.225 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:49:29.225 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:22.744643+0000", 2023-09-23T20:49:29.225 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:29.225 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.225 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.225 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:49:29.226 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:22.744642+0000", 2023-09-23T20:49:29.226 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:49:29.226 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.226 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.226 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:49:29.226 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:22.744649+0000", 2023-09-23T20:49:29.226 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.7109999999999995e-06 2023-09-23T20:49:29.226 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.226 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.226 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:49:29.226 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:22.744667+0000", 2023-09-23T20:49:29.226 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.8101000000000002e-05 2023-09-23T20:49:29.227 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.227 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.227 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:49:29.227 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:22.744677+0000", 2023-09-23T20:49:29.227 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.2539999999999998e-06 2023-09-23T20:49:29.227 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.227 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.227 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:29.227 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:22.744731+0000", 2023-09-23T20:49:29.227 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.3853000000000001e-05 2023-09-23T20:49:29.227 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.227 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.228 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:49:29.228 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:22.744895+0000", 2023-09-23T20:49:29.228 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000164332 2023-09-23T20:49:29.228 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.228 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.228 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 5,7", 2023-09-23T20:49:29.228 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:22.745027+0000", 2023-09-23T20:49:29.228 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00013257000000000001 2023-09-23T20:49:29.228 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.228 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.228 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:49:29.229 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:22.749549+0000", 2023-09-23T20:49:29.229 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0045220219999999997 2023-09-23T20:49:29.229 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.229 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.229 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:29.229 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:22.751817+0000", 2023-09-23T20:49:29.229 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0022675460000000001 2023-09-23T20:49:29.229 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.229 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.229 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:49:29.229 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:22.906049+0000", 2023-09-23T20:49:29.229 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.15423176799999999 2023-09-23T20:49:29.229 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:29.230 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:49:29.230 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:29.230 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.230 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.230 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:159028 10.3 10:c5cfe988:::benchmark_data_smithi018_422431_object54532:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e578)", 2023-09-23T20:49:29.230 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:48:23.915042+0000", 2023-09-23T20:49:29.230 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 65.198270145999999, 2023-09-23T20:49:29.230 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.277777194, 2023-09-23T20:49:29.230 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:49:29.230 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:49:29.230 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:49:29.231 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:49:29.231 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:49:29.231 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 159028 2023-09-23T20:49:29.231 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.231 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:49:29.231 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.231 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:49:29.231 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:23.915042+0000", 2023-09-23T20:49:29.231 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:29.231 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.231 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.231 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:49:29.232 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:23.915042+0000", 2023-09-23T20:49:29.232 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:29.232 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.232 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.232 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:49:29.232 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:23.915040+0000", 2023-09-23T20:49:29.232 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:49:29.232 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.232 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.232 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:49:29.232 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:23.915046+0000", 2023-09-23T20:49:29.232 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.6790000000000002e-06 2023-09-23T20:49:29.233 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.233 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.233 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:49:29.233 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:23.915059+0000", 2023-09-23T20:49:29.233 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3634e-05 2023-09-23T20:49:29.233 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.233 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.233 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:49:29.233 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:23.915071+0000", 2023-09-23T20:49:29.233 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1449e-05 2023-09-23T20:49:29.233 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.233 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.233 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:29.234 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:23.917921+0000", 2023-09-23T20:49:29.234 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0028506289999999999 2023-09-23T20:49:29.234 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.234 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.234 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:49:29.234 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:23.917946+0000", 2023-09-23T20:49:29.234 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.4960000000000002e-05 2023-09-23T20:49:29.234 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.234 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.234 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:29.234 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:25.181323+0000", 2023-09-23T20:49:29.235 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2633762989999999 2023-09-23T20:49:29.235 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.235 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.235 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:49:29.235 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:25.181464+0000", 2023-09-23T20:49:29.235 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00014105300000000001 2023-09-23T20:49:29.235 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.235 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.235 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 3,4,5", 2023-09-23T20:49:29.235 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:25.181599+0000", 2023-09-23T20:49:29.235 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000135594 2023-09-23T20:49:29.235 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.235 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.236 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:29.236 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:25.187281+0000", 2023-09-23T20:49:29.236 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.005681859 2023-09-23T20:49:29.236 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.236 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.236 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:29.236 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:25.187329+0000", 2023-09-23T20:49:29.236 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.7555999999999997e-05 2023-09-23T20:49:29.236 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.236 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.236 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:29.237 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:25.187533+0000", 2023-09-23T20:49:29.237 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000203988 2023-09-23T20:49:29.237 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.237 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.237 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:49:29.237 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:25.192635+0000", 2023-09-23T20:49:29.237 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0051021870000000002 2023-09-23T20:49:29.237 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.237 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.237 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:49:29.237 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:25.192692+0000", 2023-09-23T20:49:29.238 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.6984999999999997e-05 2023-09-23T20:49:29.238 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.238 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.238 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:49:29.238 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:25.192819+0000", 2023-09-23T20:49:29.238 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00012683799999999999 2023-09-23T20:49:29.238 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:29.238 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:49:29.238 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:29.238 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.238 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.238 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:159052 10.3 10:c6ec30e7:::benchmark_data_smithi018_422431_object54556:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e578)", 2023-09-23T20:49:29.239 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:48:23.915080+0000", 2023-09-23T20:49:29.239 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 65.198231444000001, 2023-09-23T20:49:29.239 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.299160028, 2023-09-23T20:49:29.239 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:49:29.239 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:49:29.239 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:49:29.239 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:49:29.239 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:49:29.239 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 159052 2023-09-23T20:49:29.239 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.239 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:49:29.239 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.240 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:49:29.240 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:23.915080+0000", 2023-09-23T20:49:29.240 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:29.240 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.240 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.240 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:49:29.240 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:23.915080+0000", 2023-09-23T20:49:29.240 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:29.240 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.240 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.240 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:49:29.240 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:23.915080+0000", 2023-09-23T20:49:29.241 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:49:29.241 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.241 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.241 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:49:29.241 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:23.915083+0000", 2023-09-23T20:49:29.241 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.5410000000000001e-06 2023-09-23T20:49:29.241 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.241 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.241 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:49:29.241 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:23.915090+0000", 2023-09-23T20:49:29.241 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.6699999999999997e-06 2023-09-23T20:49:29.241 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.242 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.242 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:49:29.242 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:23.915096+0000", 2023-09-23T20:49:29.242 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.1090000000000003e-06 2023-09-23T20:49:29.242 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.242 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.242 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:29.242 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:23.923741+0000", 2023-09-23T20:49:29.242 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0086448749999999998 2023-09-23T20:49:29.242 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.242 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.242 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:49:29.243 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:23.923754+0000", 2023-09-23T20:49:29.243 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3404000000000001e-05 2023-09-23T20:49:29.243 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.243 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.243 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:29.243 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:25.187715+0000", 2023-09-23T20:49:29.243 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2639611239999999 2023-09-23T20:49:29.243 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.243 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.243 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:49:29.243 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:25.187876+0000", 2023-09-23T20:49:29.244 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00016075799999999999 2023-09-23T20:49:29.244 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.244 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.244 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 3,4,5", 2023-09-23T20:49:29.244 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:25.188011+0000", 2023-09-23T20:49:29.244 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00013465499999999999 2023-09-23T20:49:29.244 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.244 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.244 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:49:29.244 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:25.193590+0000", 2023-09-23T20:49:29.244 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0055792610000000003 2023-09-23T20:49:29.244 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.244 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.245 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:29.245 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:25.197165+0000", 2023-09-23T20:49:29.245 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0035749229999999998 2023-09-23T20:49:29.245 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.245 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.245 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:29.245 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:25.203287+0000", 2023-09-23T20:49:29.245 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0061218979999999997 2023-09-23T20:49:29.245 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.245 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.245 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:29.245 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:25.214055+0000", 2023-09-23T20:49:29.245 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.010768313999999999 2023-09-23T20:49:29.246 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.246 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.246 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:49:29.246 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:25.214117+0000", 2023-09-23T20:49:29.246 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.1773999999999995e-05 2023-09-23T20:49:29.246 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.246 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.246 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:49:29.246 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:25.214240+0000", 2023-09-23T20:49:29.246 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00012325899999999999 2023-09-23T20:49:29.247 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:29.247 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:49:29.247 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:29.247 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.247 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.247 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:174208 10.3 10:c770d702:::benchmark_data_smithi018_422431_object69712:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e590)", 2023-09-23T20:49:29.247 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:48:47.146484+0000", 2023-09-23T20:49:29.247 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 41.966827309999999, 2023-09-23T20:49:29.247 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0466785009999999, 2023-09-23T20:49:29.247 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:49:29.247 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:49:29.247 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:49:29.248 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:49:29.248 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:49:29.248 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 174208 2023-09-23T20:49:29.248 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.248 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:49:29.248 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.248 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:49:29.248 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:47.146484+0000", 2023-09-23T20:49:29.248 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:29.248 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.248 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.248 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:49:29.249 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:47.146484+0000", 2023-09-23T20:49:29.249 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:29.249 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.249 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.249 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:49:29.249 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:47.146483+0000", 2023-09-23T20:49:29.249 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:49:29.249 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.249 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.249 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:49:29.249 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:47.146489+0000", 2023-09-23T20:49:29.249 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.5289999999999997e-06 2023-09-23T20:49:29.249 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.250 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.250 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:49:29.250 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:47.146501+0000", 2023-09-23T20:49:29.250 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.216e-05 2023-09-23T20:49:29.250 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.250 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.250 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:49:29.250 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:47.146510+0000", 2023-09-23T20:49:29.250 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.4840000000000004e-06 2023-09-23T20:49:29.250 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.250 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.251 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:29.251 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:47.146617+0000", 2023-09-23T20:49:29.251 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000106826 2023-09-23T20:49:29.251 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.251 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.251 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:49:29.251 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:47.146632+0000", 2023-09-23T20:49:29.251 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.5160999999999999e-05 2023-09-23T20:49:29.251 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.251 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.251 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:29.251 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:48.184489+0000", 2023-09-23T20:49:29.252 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0378566440000001 2023-09-23T20:49:29.252 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.252 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.252 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:49:29.252 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:48.184658+0000", 2023-09-23T20:49:29.252 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00016920699999999999 2023-09-23T20:49:29.252 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.252 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.252 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 3,5", 2023-09-23T20:49:29.252 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:48.184826+0000", 2023-09-23T20:49:29.252 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00016825400000000001 2023-09-23T20:49:29.252 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.253 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.253 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:49:29.253 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:48.187756+0000", 2023-09-23T20:49:29.253 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0029296029999999998 2023-09-23T20:49:29.253 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.253 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.253 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:29.253 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:48.187810+0000", 2023-09-23T20:49:29.253 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.4483e-05 2023-09-23T20:49:29.253 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.253 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.254 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:29.254 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:48.193082+0000", 2023-09-23T20:49:29.254 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0052715820000000004 2023-09-23T20:49:29.254 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.254 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.254 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:49:29.254 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:48.193125+0000", 2023-09-23T20:49:29.254 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.3578000000000002e-05 2023-09-23T20:49:29.254 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.254 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.254 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:49:29.254 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:48.193163+0000", 2023-09-23T20:49:29.255 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.7629000000000002e-05 2023-09-23T20:49:29.255 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:29.255 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:49:29.255 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:29.255 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.255 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.255 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:174224 10.13 10:c883c710:::benchmark_data_smithi018_422431_object69728:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e591)", 2023-09-23T20:49:29.255 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:48:47.170416+0000", 2023-09-23T20:49:29.255 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 41.942895256, 2023-09-23T20:49:29.255 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0227336730000001, 2023-09-23T20:49:29.255 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:49:29.256 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:49:29.256 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:49:29.256 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:49:29.256 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:49:29.256 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 174224 2023-09-23T20:49:29.256 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.256 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:49:29.256 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.256 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:49:29.256 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:47.170416+0000", 2023-09-23T20:49:29.256 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:29.256 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.256 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.257 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:49:29.257 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:47.170416+0000", 2023-09-23T20:49:29.257 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:29.257 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.257 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.257 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:49:29.257 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:47.170415+0000", 2023-09-23T20:49:29.257 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:49:29.257 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.257 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.257 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:49:29.257 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:47.170422+0000", 2023-09-23T20:49:29.258 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.4509999999999997e-06 2023-09-23T20:49:29.258 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.258 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.258 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:49:29.258 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:47.170433+0000", 2023-09-23T20:49:29.258 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1620000000000001e-05 2023-09-23T20:49:29.258 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.258 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.258 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:49:29.258 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:47.170440+0000", 2023-09-23T20:49:29.258 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.9299999999999997e-06 2023-09-23T20:49:29.259 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.259 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.259 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:29.259 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:47.170484+0000", 2023-09-23T20:49:29.259 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.3655999999999997e-05 2023-09-23T20:49:29.259 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.259 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.259 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:49:29.259 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:47.170492+0000", 2023-09-23T20:49:29.259 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.7439999999999996e-06 2023-09-23T20:49:29.259 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.259 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.260 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:29.260 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:48.187900+0000", 2023-09-23T20:49:29.260 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0174075010000001 2023-09-23T20:49:29.260 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.260 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.260 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:49:29.260 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:48.188026+0000", 2023-09-23T20:49:29.260 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00012616799999999999 2023-09-23T20:49:29.260 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.260 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.260 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 3,5", 2023-09-23T20:49:29.260 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:48.188149+0000", 2023-09-23T20:49:29.261 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000122574 2023-09-23T20:49:29.261 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.261 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.261 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:49:29.261 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:48.190116+0000", 2023-09-23T20:49:29.261 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0019672169999999998 2023-09-23T20:49:29.261 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.261 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.261 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:29.261 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:48.190257+0000", 2023-09-23T20:49:29.261 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00014095000000000001 2023-09-23T20:49:29.261 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.262 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.262 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:29.262 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:48.193090+0000", 2023-09-23T20:49:29.262 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0028331770000000001 2023-09-23T20:49:29.262 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.262 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.262 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:49:29.262 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:48.193120+0000", 2023-09-23T20:49:29.262 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.9716000000000001e-05 2023-09-23T20:49:29.262 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.262 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.262 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:49:29.263 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:48.193150+0000", 2023-09-23T20:49:29.263 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.0307999999999998e-05 2023-09-23T20:49:29.263 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:29.263 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:49:29.263 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:29.263 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.263 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.263 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:188475 10.3 10:c02c06e4:::benchmark_data_smithi018_422431_object83979:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e610)", 2023-09-23T20:49:29.263 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:49:13.284143+0000", 2023-09-23T20:49:29.263 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 15.829169006000001, 2023-09-23T20:49:29.263 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.170827273, 2023-09-23T20:49:29.263 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:49:29.264 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:49:29.264 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:49:29.264 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:49:29.264 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:49:29.264 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 188475 2023-09-23T20:49:29.264 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.264 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:49:29.264 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.264 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:49:29.264 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:13.284143+0000", 2023-09-23T20:49:29.264 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:29.264 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.265 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.265 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:49:29.265 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:13.284143+0000", 2023-09-23T20:49:29.265 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:29.265 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.265 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.265 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:49:29.265 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:13.284141+0000", 2023-09-23T20:49:29.265 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:49:29.265 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.265 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.265 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:49:29.265 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:13.284147+0000", 2023-09-23T20:49:29.266 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.8540000000000002e-06 2023-09-23T20:49:29.266 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.266 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.266 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:49:29.266 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:13.284162+0000", 2023-09-23T20:49:29.266 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.4399999999999999e-05 2023-09-23T20:49:29.266 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.266 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.266 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:49:29.266 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:13.284169+0000", 2023-09-23T20:49:29.266 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.2609999999999996e-06 2023-09-23T20:49:29.267 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.267 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.267 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:29.267 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:13.318985+0000", 2023-09-23T20:49:29.267 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.034816435 2023-09-23T20:49:29.267 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.267 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.267 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:49:29.267 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:13.319096+0000", 2023-09-23T20:49:29.267 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000110425 2023-09-23T20:49:29.267 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.267 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.268 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 3,5", 2023-09-23T20:49:29.268 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:13.319631+0000", 2023-09-23T20:49:29.268 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00053487499999999998 2023-09-23T20:49:29.268 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.268 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.268 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:49:29.268 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:13.333929+0000", 2023-09-23T20:49:29.268 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.014298384000000001 2023-09-23T20:49:29.268 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.268 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.268 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:29.268 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:13.339887+0000", 2023-09-23T20:49:29.268 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0059577969999999999 2023-09-23T20:49:29.269 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.269 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.269 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:29.269 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:13.454859+0000", 2023-09-23T20:49:29.269 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.114971832 2023-09-23T20:49:29.269 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.269 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.269 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:49:29.269 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:13.454922+0000", 2023-09-23T20:49:29.269 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.3507e-05 2023-09-23T20:49:29.269 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:29.269 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:29.270 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:49:29.270 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:13.454970+0000", 2023-09-23T20:49:29.270 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.7713000000000002e-05 2023-09-23T20:49:29.270 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:29.270 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:49:29.270 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:29.270 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:29.270 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:49:29.270 INFO:teuthology.orchestra.run.smithi018.stdout:} 2023-09-23T20:49:29.271 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:49:29.300 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:28 smithi018 ceph-mon[184144]: osdmap e623: 8 total, 8 up, 8 in 2023-09-23T20:49:29.300 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:28 smithi018 ceph-mon[184144]: failed to encode map e623 with expected crc 2023-09-23T20:49:29.300 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:28 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "24"}]: dispatch 2023-09-23T20:49:29.301 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:28 smithi018 ceph-mon[179647]: failed to encode map e623 with expected crc 2023-09-23T20:49:29.301 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:28 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "24"}]: dispatch 2023-09-23T20:49:29.305 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:29.349 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:49:29.355 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:29.356 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:49:29.456 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:49:29.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:28 smithi149 ceph-mon[170394]: osdmap e623: 8 total, 8 up, 8 in 2023-09-23T20:49:29.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:28 smithi149 ceph-mon[170394]: failed to encode map e623 with expected crc 2023-09-23T20:49:29.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:28 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "24"}]: dispatch 2023-09-23T20:49:29.490 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:29.632 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:49:29.635 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:29.636 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:49:29.654 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T20:49:29.737 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:49:29.770 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:29.856 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:49:29.859 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:29.860 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:49:29.879 INFO:teuthology.orchestra.run.smithi018.stdout:{"pg_ready":true,"pg_map":{"version":1539,"stamp":"2023-09-23T20:49:27.962182+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":873934674,"num_objects":13289,"num_object_clones":0,"num_object_copies":39867,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":584,"num_objects_unfound":0,"num_objects_dirty":13289,"num_whiteouts":0,"num_read":6043,"num_read_kb":13311,"num_write":160740,"num_write_kb":5432921,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16892,"num_bytes_recovered":1116242986,"num_keys_recovered":80,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":182372,"ondisk_log_size":182372,"up":387,"acting":387,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":406,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":8992220,"kb_used_data":3048540,"kb_used_omap":32,"kb_used_meta":5943519,"kb_avail":740968996,"statfs":{"total":767960285184,"available":758752251904,"internally_reserved":0,"allocated":3121704960,"data_stored":3103088404,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32773,"internal_metadata":6086164475},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-233832448,"num_objects":-3568,"num_object_clones":0,"num_object_copies":-10704,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-3568,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3568,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":545,"num_bytes_recovered":35717120,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"5.011042"},"pg_stats":[{"pgid":"10.13","version":"619'11639","reported_seq":13859,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T20:49:23.692439+0000","last_change":"2023-09-23T20:49:21.975367+0000","last_active":"2023-09-23T20:49:23.692439+0000","last_peered":"2023-09-23T20:49:23.692439+0000","last_clean":"2023-09-23T20:49:23.692439+0000","last_became_active":"2023-09-23T20:49:21.930432+0000","last_became_peered":"2023-09-23T20:49:21.930432+0000","last_unstale":"2023-09-23T20:49:23.692439+0000","last_undegraded":"2023-09-23T20:49:23.692439+0000","last_fullsized":"2023-09-23T20:49:23.692439+0000","mapping_epoch":617,"log_start":"536'1300","ondisk_log_start":"536'1300","created":584,"last_epoch_clean":618,"parent":"0.0","parent_split_bits":5,"last_scrub":"611'11468","last_scrub_stamp":"2023-09-23T20:49:14.806285+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:49:14.806285+0000","objects_scrubbed":646,"log_size":10339,"log_dups_size":0,"ondisk_log_size":10339,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:50:56.718850+0000","scrub_duration":0.126272309,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":31129600,"num_objects":475,"num_object_clones":0,"num_object_copies":1425,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":475,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":171,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":545,"num_bytes_recovered":35717120,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.18","version":"90'161","reported_seq":661,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T20:49:21.924865+0000","last_change":"2023-09-23T20:48:51.176302+0000","last_active":"2023-09-23T20:49:21.924865+0000","last_peered":"2023-09-23T20:49:21.924865+0000","last_clean":"2023-09-23T20:49:21.924865+0000","last_became_active":"2023-09-23T20:48:51.175408+0000","last_became_peered":"2023-09-23T20:48:51.175408+0000","last_unstale":"2023-09-23T20:49:21.924865+0000","last_undegraded":"2023-09-23T20:49:21.924865+0000","last_fullsized":"2023-09-23T20:49:21.924865+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:19.677209+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:19.677209+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:43:11.718198+0000","scrub_duration":0.033122453000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"69'1","reported_seq":96,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T20:49:22.963144+0000","last_change":"2023-09-23T20:48:48.148250+0000","last_active":"2023-09-23T20:49:22.963144+0000","last_peered":"2023-09-23T20:49:22.963144+0000","last_clean":"2023-09-23T20:49:22.963144+0000","last_became_active":"2023-09-23T20:48:48.144271+0000","last_became_peered":"2023-09-23T20:48:48.144271+0000","last_unstale":"2023-09-23T20:49:22.963144+0000","last_undegraded":"2023-09-23T20:49:22.963144+0000","last_fullsized":"2023-09-23T20:49:22.963144+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:59:36.502986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":462,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T20:49:21.998081+0000","last_change":"2023-09-23T20:49:02.703099+0000","last_active":"2023-09-23T20:49:21.998081+0000","last_peered":"2023-09-23T20:49:21.998081+0000","last_clean":"2023-09-23T20:49:21.998081+0000","last_became_active":"2023-09-23T20:49:02.702753+0000","last_became_peered":"2023-09-23T20:49:02.702753+0000","last_unstale":"2023-09-23T20:49:21.998081+0000","last_undegraded":"2023-09-23T20:49:21.998081+0000","last_fullsized":"2023-09-23T20:49:21.998081+0000","mapping_epoch":603,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":604,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:24:30.834235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.10","version":"620'12035","reported_seq":15458,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T20:49:25.076551+0000","last_change":"2023-09-23T20:49:13.848106+0000","last_active":"2023-09-23T20:49:25.076551+0000","last_peered":"2023-09-23T20:49:25.076551+0000","last_clean":"2023-09-23T20:49:25.076551+0000","last_became_active":"2023-09-23T20:48:55.189956+0000","last_became_peered":"2023-09-23T20:48:55.189956+0000","last_unstale":"2023-09-23T20:49:25.076551+0000","last_undegraded":"2023-09-23T20:49:25.076551+0000","last_fullsized":"2023-09-23T20:49:25.076551+0000","mapping_epoch":598,"log_start":"542'4945","ondisk_log_start":"542'4945","created":584,"last_epoch_clean":599,"parent":"0.0","parent_split_bits":5,"last_scrub":"611'11822","last_scrub_stamp":"2023-09-23T20:49:13.848043+0000","last_deep_scrub":"611'11822","last_deep_scrub_stamp":"2023-09-23T20:49:13.848043+0000","last_clean_scrub_stamp":"2023-09-23T20:49:13.848043+0000","objects_scrubbed":618,"log_size":7090,"log_dups_size":0,"ondisk_log_size":7090,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":2,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:45.075182+0000","scrub_duration":1.6702664780000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":25624576,"num_objects":391,"num_object_clones":0,"num_object_copies":1173,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":391,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":227,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1b","version":"90'161","reported_seq":622,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T20:49:22.963144+0000","last_change":"2023-09-23T20:47:33.523278+0000","last_active":"2023-09-23T20:49:22.963144+0000","last_peered":"2023-09-23T20:49:22.963144+0000","last_clean":"2023-09-23T20:49:22.963144+0000","last_became_active":"2023-09-23T20:47:11.622087+0000","last_became_peered":"2023-09-23T20:47:11.622087+0000","last_unstale":"2023-09-23T20:49:22.963144+0000","last_undegraded":"2023-09-23T20:49:22.963144+0000","last_fullsized":"2023-09-23T20:49:22.963144+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:47:33.523187+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:36:05.442288+0000","last_clean_scrub_stamp":"2023-09-23T20:47:33.523187+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:32:18.410526+0000","scrub_duration":0.013532127,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":609,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T20:49:21.926007+0000","last_change":"2023-09-23T20:47:09.173501+0000","last_active":"2023-09-23T20:49:21.926007+0000","last_peered":"2023-09-23T20:49:21.926007+0000","last_clean":"2023-09-23T20:49:21.926007+0000","last_became_active":"2023-09-23T20:47:09.173160+0000","last_became_peered":"2023-09-23T20:47:09.173160+0000","last_unstale":"2023-09-23T20:49:21.926007+0000","last_undegraded":"2023-09-23T20:49:21.926007+0000","last_fullsized":"2023-09-23T20:49:21.926007+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:44:36.132093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":580,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T20:49:24.962265+0000","last_change":"2023-09-23T20:43:46.521939+0000","last_active":"2023-09-23T20:49:24.962265+0000","last_peered":"2023-09-23T20:49:24.962265+0000","last_clean":"2023-09-23T20:49:24.962265+0000","last_became_active":"2023-09-23T20:43:46.521185+0000","last_became_peered":"2023-09-23T20:43:46.521185+0000","last_unstale":"2023-09-23T20:49:24.962265+0000","last_undegraded":"2023-09-23T20:49:24.962265+0000","last_fullsized":"2023-09-23T20:49:24.962265+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:59:25.045627+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.11","version":"621'11451","reported_seq":13266,"reported_epoch":621,"state":"active+clean","last_fresh":"2023-09-23T20:49:26.359773+0000","last_change":"2023-09-23T20:49:03.739386+0000","last_active":"2023-09-23T20:49:26.359773+0000","last_peered":"2023-09-23T20:49:26.359773+0000","last_clean":"2023-09-23T20:49:26.359773+0000","last_became_active":"2023-09-23T20:49:03.715004+0000","last_became_peered":"2023-09-23T20:49:03.715004+0000","last_unstale":"2023-09-23T20:49:26.359773+0000","last_undegraded":"2023-09-23T20:49:26.359773+0000","last_fullsized":"2023-09-23T20:49:26.359773+0000","mapping_epoch":604,"log_start":"543'5336","ondisk_log_start":"543'5336","created":584,"last_epoch_clean":605,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":6115,"log_dups_size":0,"ondisk_log_size":6115,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":28966912,"num_objects":442,"num_object_clones":0,"num_object_copies":1326,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":442,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6110,"num_write_kb":209664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":827,"num_bytes_recovered":54132736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"90'161","reported_seq":507,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T20:49:22.962981+0000","last_change":"2023-09-23T20:47:31.475001+0000","last_active":"2023-09-23T20:49:22.962981+0000","last_peered":"2023-09-23T20:49:22.962981+0000","last_clean":"2023-09-23T20:49:22.962981+0000","last_became_active":"2023-09-23T20:47:11.617942+0000","last_became_peered":"2023-09-23T20:47:11.617942+0000","last_unstale":"2023-09-23T20:49:22.962981+0000","last_undegraded":"2023-09-23T20:49:22.962981+0000","last_fullsized":"2023-09-23T20:49:22.962981+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:47:31.474914+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:47:31.474914+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:52:05.169219+0000","scrub_duration":0.0026262870000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":415,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T20:49:21.993749+0000","last_change":"2023-09-23T20:37:43.584414+0000","last_active":"2023-09-23T20:49:21.993749+0000","last_peered":"2023-09-23T20:49:21.993749+0000","last_clean":"2023-09-23T20:49:21.993749+0000","last_became_active":"2023-09-23T20:37:43.584116+0000","last_became_peered":"2023-09-23T20:37:43.584116+0000","last_unstale":"2023-09-23T20:49:21.993749+0000","last_undegraded":"2023-09-23T20:49:21.993749+0000","last_fullsized":"2023-09-23T20:49:21.993749+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:17:54.258172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":234,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T20:49:22.966244+0000","last_change":"2023-09-23T20:38:37.457427+0000","last_active":"2023-09-23T20:49:22.966244+0000","last_peered":"2023-09-23T20:49:22.966244+0000","last_clean":"2023-09-23T20:49:22.966244+0000","last_became_active":"2023-09-23T20:38:37.457099+0000","last_became_peered":"2023-09-23T20:38:37.457099+0000","last_unstale":"2023-09-23T20:49:22.966244+0000","last_undegraded":"2023-09-23T20:49:22.966244+0000","last_fullsized":"2023-09-23T20:49:22.966244+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:31:27.084994+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"90'161","reported_seq":832,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T20:49:22.979710+0000","last_change":"2023-09-23T20:40:22.449754+0000","last_active":"2023-09-23T20:49:22.979710+0000","last_peered":"2023-09-23T20:49:22.979710+0000","last_clean":"2023-09-23T20:49:22.979710+0000","last_became_active":"2023-09-23T20:40:22.448838+0000","last_became_peered":"2023-09-23T20:40:22.448838+0000","last_unstale":"2023-09-23T20:49:22.979710+0000","last_undegraded":"2023-09-23T20:49:22.979710+0000","last_fullsized":"2023-09-23T20:49:22.979710+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:03.411025+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:03.411025+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:22:07.948856+0000","scrub_duration":0.001802922,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":613,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T20:49:24.962551+0000","last_change":"2023-09-23T20:43:46.522325+0000","last_active":"2023-09-23T20:49:24.962551+0000","last_peered":"2023-09-23T20:49:24.962551+0000","last_clean":"2023-09-23T20:49:24.962551+0000","last_became_active":"2023-09-23T20:43:46.521996+0000","last_became_peered":"2023-09-23T20:43:46.521996+0000","last_unstale":"2023-09-23T20:49:24.962551+0000","last_undegraded":"2023-09-23T20:49:24.962551+0000","last_fullsized":"2023-09-23T20:49:24.962551+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:51:17.629956+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":52,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T20:49:21.993457+0000","last_change":"2023-09-23T20:48:23.918092+0000","last_active":"2023-09-23T20:49:21.993457+0000","last_peered":"2023-09-23T20:49:21.993457+0000","last_clean":"2023-09-23T20:49:21.993457+0000","last_became_active":"2023-09-23T20:48:23.917386+0000","last_became_peered":"2023-09-23T20:48:23.917386+0000","last_unstale":"2023-09-23T20:49:21.993457+0000","last_undegraded":"2023-09-23T20:49:21.993457+0000","last_fullsized":"2023-09-23T20:49:21.993457+0000","mapping_epoch":577,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":578,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:06:09.587517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"90'161","reported_seq":609,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T20:49:21.925507+0000","last_change":"2023-09-23T20:48:51.169725+0000","last_active":"2023-09-23T20:49:21.925507+0000","last_peered":"2023-09-23T20:49:21.925507+0000","last_clean":"2023-09-23T20:49:21.925507+0000","last_became_active":"2023-09-23T20:48:51.169314+0000","last_became_peered":"2023-09-23T20:48:51.169314+0000","last_unstale":"2023-09-23T20:49:21.925507+0000","last_undegraded":"2023-09-23T20:49:21.925507+0000","last_fullsized":"2023-09-23T20:49:21.925507+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:47.479012+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:47.479012+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":456,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T20:49:22.979925+0000","last_change":"2023-09-23T20:43:16.780478+0000","last_active":"2023-09-23T20:49:22.979925+0000","last_peered":"2023-09-23T20:49:22.979925+0000","last_clean":"2023-09-23T20:49:22.979925+0000","last_became_active":"2023-09-23T20:43:16.780183+0000","last_became_peered":"2023-09-23T20:43:16.780183+0000","last_unstale":"2023-09-23T20:49:22.979925+0000","last_undegraded":"2023-09-23T20:49:22.979925+0000","last_fullsized":"2023-09-23T20:49:22.979925+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:38:30.102066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":447,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T20:49:22.973396+0000","last_change":"2023-09-23T20:38:02.542034+0000","last_active":"2023-09-23T20:49:22.973396+0000","last_peered":"2023-09-23T20:49:22.973396+0000","last_clean":"2023-09-23T20:49:22.973396+0000","last_became_active":"2023-09-23T20:38:02.541575+0000","last_became_peered":"2023-09-23T20:38:02.541575+0000","last_unstale":"2023-09-23T20:49:22.973396+0000","last_undegraded":"2023-09-23T20:49:22.973396+0000","last_fullsized":"2023-09-23T20:49:22.973396+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:50:54.591529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"90'161","reported_seq":748,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T20:49:24.962371+0000","last_change":"2023-09-23T20:44:14.669973+0000","last_active":"2023-09-23T20:49:24.962371+0000","last_peered":"2023-09-23T20:49:24.962371+0000","last_clean":"2023-09-23T20:49:24.962371+0000","last_became_active":"2023-09-23T20:43:46.519592+0000","last_became_peered":"2023-09-23T20:43:46.519592+0000","last_unstale":"2023-09-23T20:49:24.962371+0000","last_undegraded":"2023-09-23T20:49:24.962371+0000","last_fullsized":"2023-09-23T20:49:24.962371+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:14.669831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:14.669831+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:56:28.076617+0000","scrub_duration":0.0083441460000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":679,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T20:49:22.980103+0000","last_change":"2023-09-23T20:47:10.837423+0000","last_active":"2023-09-23T20:49:22.980103+0000","last_peered":"2023-09-23T20:49:22.980103+0000","last_clean":"2023-09-23T20:49:22.980103+0000","last_became_active":"2023-09-23T20:47:09.204324+0000","last_became_peered":"2023-09-23T20:47:09.204324+0000","last_unstale":"2023-09-23T20:49:22.980103+0000","last_undegraded":"2023-09-23T20:49:22.980103+0000","last_fullsized":"2023-09-23T20:49:22.980103+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":554,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:19.108914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":540,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":153,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T20:49:21.926186+0000","last_change":"2023-09-23T20:48:48.145175+0000","last_active":"2023-09-23T20:49:21.926186+0000","last_peered":"2023-09-23T20:49:21.926186+0000","last_clean":"2023-09-23T20:49:21.926186+0000","last_became_active":"2023-09-23T20:48:48.144841+0000","last_became_peered":"2023-09-23T20:48:48.144841+0000","last_unstale":"2023-09-23T20:49:21.926186+0000","last_undegraded":"2023-09-23T20:49:21.926186+0000","last_fullsized":"2023-09-23T20:49:21.926186+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:27:40.802917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"90'161","reported_seq":746,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T20:49:21.927433+0000","last_change":"2023-09-23T20:36:02.548387+0000","last_active":"2023-09-23T20:49:21.927433+0000","last_peered":"2023-09-23T20:49:21.927433+0000","last_clean":"2023-09-23T20:49:21.927433+0000","last_became_active":"2023-09-23T20:35:26.644787+0000","last_became_peered":"2023-09-23T20:35:26.644787+0000","last_unstale":"2023-09-23T20:49:21.927433+0000","last_undegraded":"2023-09-23T20:49:21.927433+0000","last_fullsized":"2023-09-23T20:49:21.927433+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:02.548285+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:02.548285+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:37:15.216553+0000","scrub_duration":0.0015514459999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":86,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T20:49:22.963096+0000","last_change":"2023-09-23T20:47:09.340979+0000","last_active":"2023-09-23T20:49:22.963096+0000","last_peered":"2023-09-23T20:49:22.963096+0000","last_clean":"2023-09-23T20:49:22.963096+0000","last_became_active":"2023-09-23T20:47:09.231222+0000","last_became_peered":"2023-09-23T20:47:09.231222+0000","last_unstale":"2023-09-23T20:49:22.963096+0000","last_undegraded":"2023-09-23T20:49:22.963096+0000","last_fullsized":"2023-09-23T20:49:22.963096+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:07:50.620006+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":76,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T20:49:22.963336+0000","last_change":"2023-09-23T20:47:09.186315+0000","last_active":"2023-09-23T20:49:22.963336+0000","last_peered":"2023-09-23T20:49:22.963336+0000","last_clean":"2023-09-23T20:49:22.963336+0000","last_became_active":"2023-09-23T20:47:09.184580+0000","last_became_peered":"2023-09-23T20:47:09.184580+0000","last_unstale":"2023-09-23T20:49:22.963336+0000","last_undegraded":"2023-09-23T20:49:22.963336+0000","last_fullsized":"2023-09-23T20:49:22.963336+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:44:35.553948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"90'161","reported_seq":876,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T20:49:24.962530+0000","last_change":"2023-09-23T20:43:46.544727+0000","last_active":"2023-09-23T20:49:24.962530+0000","last_peered":"2023-09-23T20:49:24.962530+0000","last_clean":"2023-09-23T20:49:24.962530+0000","last_became_active":"2023-09-23T20:43:46.543952+0000","last_became_peered":"2023-09-23T20:43:46.543952+0000","last_unstale":"2023-09-23T20:49:24.962530+0000","last_undegraded":"2023-09-23T20:49:24.962530+0000","last_fullsized":"2023-09-23T20:49:24.962530+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:43.578666+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:43.578666+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:11:16.611920+0000","scrub_duration":0.0021890379999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":710,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T20:49:22.979627+0000","last_change":"2023-09-23T20:48:48.193666+0000","last_active":"2023-09-23T20:49:22.979627+0000","last_peered":"2023-09-23T20:49:22.979627+0000","last_clean":"2023-09-23T20:49:22.979627+0000","last_became_active":"2023-09-23T20:48:48.152081+0000","last_became_peered":"2023-09-23T20:48:48.152081+0000","last_unstale":"2023-09-23T20:49:22.979627+0000","last_undegraded":"2023-09-23T20:49:22.979627+0000","last_fullsized":"2023-09-23T20:49:22.979627+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:00:19.508124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":272,"num_keys_recovered":80,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":367,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T20:49:22.973584+0000","last_change":"2023-09-23T20:45:01.875931+0000","last_active":"2023-09-23T20:49:22.973584+0000","last_peered":"2023-09-23T20:49:22.973584+0000","last_clean":"2023-09-23T20:49:22.973584+0000","last_became_active":"2023-09-23T20:45:01.872839+0000","last_became_peered":"2023-09-23T20:45:01.872839+0000","last_unstale":"2023-09-23T20:49:22.973584+0000","last_undegraded":"2023-09-23T20:49:22.973584+0000","last_fullsized":"2023-09-23T20:49:22.973584+0000","mapping_epoch":521,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":522,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:15:09.584507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"90'161","reported_seq":900,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T20:49:22.979361+0000","last_change":"2023-09-23T20:44:20.041766+0000","last_active":"2023-09-23T20:49:22.979361+0000","last_peered":"2023-09-23T20:49:22.979361+0000","last_clean":"2023-09-23T20:49:22.979361+0000","last_became_active":"2023-09-23T20:43:16.765015+0000","last_became_peered":"2023-09-23T20:43:16.765015+0000","last_unstale":"2023-09-23T20:49:22.979361+0000","last_undegraded":"2023-09-23T20:49:22.979361+0000","last_fullsized":"2023-09-23T20:49:22.979361+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:20.041690+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:20.041690+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:33:30.150109+0000","scrub_duration":0.0026180330000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":578,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T20:49:24.967901+0000","last_change":"2023-09-23T20:43:46.521294+0000","last_active":"2023-09-23T20:49:24.967901+0000","last_peered":"2023-09-23T20:49:24.967901+0000","last_clean":"2023-09-23T20:49:24.967901+0000","last_became_active":"2023-09-23T20:43:46.521001+0000","last_became_peered":"2023-09-23T20:43:46.521001+0000","last_unstale":"2023-09-23T20:49:24.967901+0000","last_undegraded":"2023-09-23T20:49:24.967901+0000","last_fullsized":"2023-09-23T20:49:24.967901+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:01:35.126716+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":397,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T20:49:22.980188+0000","last_change":"2023-09-23T20:48:23.914091+0000","last_active":"2023-09-23T20:49:22.980188+0000","last_peered":"2023-09-23T20:49:22.980188+0000","last_clean":"2023-09-23T20:49:22.980188+0000","last_became_active":"2023-09-23T20:48:23.913768+0000","last_became_peered":"2023-09-23T20:48:23.913768+0000","last_unstale":"2023-09-23T20:49:22.980188+0000","last_undegraded":"2023-09-23T20:49:22.980188+0000","last_fullsized":"2023-09-23T20:49:22.980188+0000","mapping_epoch":577,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":578,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:19:38.311290+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.18","version":"621'11318","reported_seq":14795,"reported_epoch":621,"state":"active+clean","last_fresh":"2023-09-23T20:49:26.385411+0000","last_change":"2023-09-23T20:48:49.178589+0000","last_active":"2023-09-23T20:49:26.385411+0000","last_peered":"2023-09-23T20:49:26.385411+0000","last_clean":"2023-09-23T20:49:26.385411+0000","last_became_active":"2023-09-23T20:48:49.155693+0000","last_became_peered":"2023-09-23T20:48:49.155693+0000","last_unstale":"2023-09-23T20:49:26.385411+0000","last_undegraded":"2023-09-23T20:49:26.385411+0000","last_fullsized":"2023-09-23T20:49:26.385411+0000","mapping_epoch":592,"log_start":"541'4571","ondisk_log_start":"541'4571","created":584,"last_epoch_clean":593,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":6747,"log_dups_size":0,"ondisk_log_size":6747,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":23691264,"num_objects":362,"num_object_clones":0,"num_object_copies":1086,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":362,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6066,"num_write_kb":205664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":837,"num_bytes_recovered":54820864,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.13","version":"90'161","reported_seq":942,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T20:49:21.926987+0000","last_change":"2023-09-23T20:47:11.619371+0000","last_active":"2023-09-23T20:49:21.926987+0000","last_peered":"2023-09-23T20:49:21.926987+0000","last_clean":"2023-09-23T20:49:21.926987+0000","last_became_active":"2023-09-23T20:47:11.619006+0000","last_became_peered":"2023-09-23T20:47:11.619006+0000","last_unstale":"2023-09-23T20:49:21.926987+0000","last_undegraded":"2023-09-23T20:49:21.926987+0000","last_fullsized":"2023-09-23T20:49:21.926987+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:42.544308+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:42.544308+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:47:24.414807+0000","scrub_duration":0.0029958179999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":643,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T20:49:22.979592+0000","last_change":"2023-09-23T20:47:09.176613+0000","last_active":"2023-09-23T20:49:22.979592+0000","last_peered":"2023-09-23T20:49:22.979592+0000","last_clean":"2023-09-23T20:49:22.979592+0000","last_became_active":"2023-09-23T20:47:09.176187+0000","last_became_peered":"2023-09-23T20:47:09.176187+0000","last_unstale":"2023-09-23T20:49:22.979592+0000","last_undegraded":"2023-09-23T20:49:22.979592+0000","last_fullsized":"2023-09-23T20:49:22.979592+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:23:01.590989+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":587,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T20:49:24.962309+0000","last_change":"2023-09-23T20:43:46.521735+0000","last_active":"2023-09-23T20:49:24.962309+0000","last_peered":"2023-09-23T20:49:24.962309+0000","last_clean":"2023-09-23T20:49:24.962309+0000","last_became_active":"2023-09-23T20:43:46.520762+0000","last_became_peered":"2023-09-23T20:43:46.520762+0000","last_unstale":"2023-09-23T20:49:24.962309+0000","last_undegraded":"2023-09-23T20:49:24.962309+0000","last_fullsized":"2023-09-23T20:49:24.962309+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:26:48.841930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.19","version":"619'11512","reported_seq":11648,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T20:49:23.352388+0000","last_change":"2023-09-23T20:48:49.228617+0000","last_active":"2023-09-23T20:49:23.352388+0000","last_peered":"2023-09-23T20:49:23.352388+0000","last_clean":"2023-09-23T20:49:23.352388+0000","last_became_active":"2023-09-23T20:48:49.188310+0000","last_became_peered":"2023-09-23T20:48:49.188310+0000","last_unstale":"2023-09-23T20:49:23.352388+0000","last_undegraded":"2023-09-23T20:49:23.352388+0000","last_fullsized":"2023-09-23T20:49:23.352388+0000","mapping_epoch":592,"log_start":"541'4623","ondisk_log_start":"541'4623","created":584,"last_epoch_clean":593,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":6889,"log_dups_size":0,"ondisk_log_size":6889,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":30769152,"num_objects":470,"num_object_clones":0,"num_object_copies":1410,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":470,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6134,"num_write_kb":211296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.12","version":"90'161","reported_seq":704,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T20:49:21.926727+0000","last_change":"2023-09-23T20:44:18.952808+0000","last_active":"2023-09-23T20:49:21.926727+0000","last_peered":"2023-09-23T20:49:21.926727+0000","last_clean":"2023-09-23T20:49:21.926727+0000","last_became_active":"2023-09-23T20:43:16.758652+0000","last_became_peered":"2023-09-23T20:43:16.758652+0000","last_unstale":"2023-09-23T20:49:21.926727+0000","last_undegraded":"2023-09-23T20:49:21.926727+0000","last_fullsized":"2023-09-23T20:49:21.926727+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:18.952693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:18.952693+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:12:19.496465+0000","scrub_duration":0.001699365,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":294,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T20:49:22.965440+0000","last_change":"2023-09-23T20:44:43.644290+0000","last_active":"2023-09-23T20:49:22.965440+0000","last_peered":"2023-09-23T20:49:22.965440+0000","last_clean":"2023-09-23T20:49:22.965440+0000","last_became_active":"2023-09-23T20:44:43.638313+0000","last_became_peered":"2023-09-23T20:44:43.638313+0000","last_unstale":"2023-09-23T20:49:22.965440+0000","last_undegraded":"2023-09-23T20:49:22.965440+0000","last_fullsized":"2023-09-23T20:49:22.965440+0000","mapping_epoch":513,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":514,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:32:52.892603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":415,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T20:49:21.994684+0000","last_change":"2023-09-23T20:37:43.571083+0000","last_active":"2023-09-23T20:49:21.994684+0000","last_peered":"2023-09-23T20:49:21.994684+0000","last_clean":"2023-09-23T20:49:21.994684+0000","last_became_active":"2023-09-23T20:37:43.570306+0000","last_became_peered":"2023-09-23T20:37:43.570306+0000","last_unstale":"2023-09-23T20:49:21.994684+0000","last_undegraded":"2023-09-23T20:49:21.994684+0000","last_fullsized":"2023-09-23T20:49:21.994684+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:42:02.396718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.6","version":"620'11494","reported_seq":13697,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T20:49:25.154392+0000","last_change":"2023-09-23T20:49:11.746129+0000","last_active":"2023-09-23T20:49:25.154392+0000","last_peered":"2023-09-23T20:49:25.154392+0000","last_clean":"2023-09-23T20:49:25.154392+0000","last_became_active":"2023-09-23T20:49:11.698124+0000","last_became_peered":"2023-09-23T20:49:11.698124+0000","last_unstale":"2023-09-23T20:49:25.154392+0000","last_undegraded":"2023-09-23T20:49:25.154392+0000","last_fullsized":"2023-09-23T20:49:25.154392+0000","mapping_epoch":608,"log_start":"541'4690","ondisk_log_start":"541'4690","created":531,"last_epoch_clean":609,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":6804,"log_dups_size":0,"ondisk_log_size":6804,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":25460736,"num_objects":388,"num_object_clones":0,"num_object_copies":1164,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":388,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6173,"num_write_kb":209984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":881,"num_bytes_recovered":57769984,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,4],"acting":[7,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.d","version":"90'161","reported_seq":952,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T20:49:22.979796+0000","last_change":"2023-09-23T20:47:11.619510+0000","last_active":"2023-09-23T20:49:22.979796+0000","last_peered":"2023-09-23T20:49:22.979796+0000","last_clean":"2023-09-23T20:49:22.979796+0000","last_became_active":"2023-09-23T20:47:11.619154+0000","last_became_peered":"2023-09-23T20:47:11.619154+0000","last_unstale":"2023-09-23T20:49:22.979796+0000","last_undegraded":"2023-09-23T20:49:22.979796+0000","last_fullsized":"2023-09-23T20:49:22.979796+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:34:59.496375+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:34:59.496375+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:40:52.712777+0000","scrub_duration":0.0017843189999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":651,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T20:49:22.980167+0000","last_change":"2023-09-23T20:47:09.207851+0000","last_active":"2023-09-23T20:49:22.980167+0000","last_peered":"2023-09-23T20:49:22.980167+0000","last_clean":"2023-09-23T20:49:22.980167+0000","last_became_active":"2023-09-23T20:47:09.175836+0000","last_became_peered":"2023-09-23T20:47:09.175836+0000","last_unstale":"2023-09-23T20:49:22.980167+0000","last_undegraded":"2023-09-23T20:49:22.980167+0000","last_fullsized":"2023-09-23T20:49:22.980167+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:43:13.091809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":434,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T20:49:22.979616+0000","last_change":"2023-09-23T20:43:16.765253+0000","last_active":"2023-09-23T20:49:22.979616+0000","last_peered":"2023-09-23T20:49:22.979616+0000","last_clean":"2023-09-23T20:49:22.979616+0000","last_became_active":"2023-09-23T20:43:16.764978+0000","last_became_peered":"2023-09-23T20:43:16.764978+0000","last_unstale":"2023-09-23T20:49:22.979616+0000","last_undegraded":"2023-09-23T20:49:22.979616+0000","last_fullsized":"2023-09-23T20:49:22.979616+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:07:23.539372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.7","version":"620'11351","reported_seq":11464,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T20:49:25.064690+0000","last_change":"2023-09-23T20:49:09.234110+0000","last_active":"2023-09-23T20:49:25.064690+0000","last_peered":"2023-09-23T20:49:25.064690+0000","last_clean":"2023-09-23T20:49:25.064690+0000","last_became_active":"2023-09-23T20:48:40.067198+0000","last_became_peered":"2023-09-23T20:48:40.067198+0000","last_unstale":"2023-09-23T20:49:25.064690+0000","last_undegraded":"2023-09-23T20:49:25.064690+0000","last_fullsized":"2023-09-23T20:49:25.064690+0000","mapping_epoch":531,"log_start":"542'4863","ondisk_log_start":"542'4863","created":531,"last_epoch_clean":585,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":6488,"log_dups_size":0,"ondisk_log_size":6488,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26181632,"num_objects":399,"num_object_clones":0,"num_object_copies":1197,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":399,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6087,"num_write_kb":207584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.c","version":"90'161","reported_seq":882,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T20:49:21.994010+0000","last_change":"2023-09-23T20:41:09.506481+0000","last_active":"2023-09-23T20:49:21.994010+0000","last_peered":"2023-09-23T20:49:21.994010+0000","last_clean":"2023-09-23T20:49:21.994010+0000","last_became_active":"2023-09-23T20:38:28.442229+0000","last_became_peered":"2023-09-23T20:38:28.442229+0000","last_unstale":"2023-09-23T20:49:21.994010+0000","last_undegraded":"2023-09-23T20:49:21.994010+0000","last_fullsized":"2023-09-23T20:49:21.994010+0000","mapping_epoch":411,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":412,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:41:09.506338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:41:09.506338+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:19:46.265011+0000","scrub_duration":0.01112002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":689,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T20:49:22.980015+0000","last_change":"2023-09-23T20:47:10.848436+0000","last_active":"2023-09-23T20:49:22.980015+0000","last_peered":"2023-09-23T20:49:22.980015+0000","last_clean":"2023-09-23T20:49:22.980015+0000","last_became_active":"2023-09-23T20:47:09.196769+0000","last_became_peered":"2023-09-23T20:47:09.196769+0000","last_unstale":"2023-09-23T20:49:22.980015+0000","last_undegraded":"2023-09-23T20:49:22.980015+0000","last_fullsized":"2023-09-23T20:49:22.980015+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":554,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:59:38.105009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":86,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T20:49:22.962812+0000","last_change":"2023-09-23T20:48:23.915817+0000","last_active":"2023-09-23T20:49:22.962812+0000","last_peered":"2023-09-23T20:49:22.962812+0000","last_clean":"2023-09-23T20:49:22.962812+0000","last_became_active":"2023-09-23T20:48:23.915306+0000","last_became_peered":"2023-09-23T20:48:23.915306+0000","last_unstale":"2023-09-23T20:49:22.962812+0000","last_undegraded":"2023-09-23T20:49:22.962812+0000","last_fullsized":"2023-09-23T20:49:22.962812+0000","mapping_epoch":577,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":578,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:55:45.691450+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"621'11260","reported_seq":11945,"reported_epoch":621,"state":"active+clean","last_fresh":"2023-09-23T20:49:26.473198+0000","last_change":"2023-09-23T20:49:13.415515+0000","last_active":"2023-09-23T20:49:26.473198+0000","last_peered":"2023-09-23T20:49:26.473198+0000","last_clean":"2023-09-23T20:49:26.473198+0000","last_became_active":"2023-09-23T20:48:40.070957+0000","last_became_peered":"2023-09-23T20:48:40.070957+0000","last_unstale":"2023-09-23T20:49:26.473198+0000","last_undegraded":"2023-09-23T20:49:26.473198+0000","last_fullsized":"2023-09-23T20:49:26.473198+0000","mapping_epoch":558,"log_start":"541'4547","ondisk_log_start":"541'4547","created":531,"last_epoch_clean":585,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":6713,"log_dups_size":0,"ondisk_log_size":6713,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":29622272,"num_objects":452,"num_object_clones":0,"num_object_copies":1356,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":452,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6011,"num_write_kb":206816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":137,"num_bytes_recovered":8978432,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"90'161","reported_seq":814,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T20:49:22.966627+0000","last_change":"2023-09-23T20:47:11.622303+0000","last_active":"2023-09-23T20:49:22.966627+0000","last_peered":"2023-09-23T20:49:22.966627+0000","last_clean":"2023-09-23T20:49:22.966627+0000","last_became_active":"2023-09-23T20:47:11.621953+0000","last_became_peered":"2023-09-23T20:47:11.621953+0000","last_unstale":"2023-09-23T20:49:22.966627+0000","last_undegraded":"2023-09-23T20:49:22.966627+0000","last_fullsized":"2023-09-23T20:49:22.966627+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:46.217489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:46.217489+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:14:40.478522+0000","scrub_duration":0.001456429,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":648,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T20:49:22.979988+0000","last_change":"2023-09-23T20:40:19.677655+0000","last_active":"2023-09-23T20:49:22.979988+0000","last_peered":"2023-09-23T20:49:22.979988+0000","last_clean":"2023-09-23T20:49:22.979988+0000","last_became_active":"2023-09-23T20:40:19.551189+0000","last_became_peered":"2023-09-23T20:40:19.551189+0000","last_unstale":"2023-09-23T20:49:22.979988+0000","last_undegraded":"2023-09-23T20:49:22.979988+0000","last_fullsized":"2023-09-23T20:49:22.979988+0000","mapping_epoch":440,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":441,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:11:01.559576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":420,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":234,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T20:49:22.967598+0000","last_change":"2023-09-23T20:49:02.702301+0000","last_active":"2023-09-23T20:49:22.967598+0000","last_peered":"2023-09-23T20:49:22.967598+0000","last_clean":"2023-09-23T20:49:22.967598+0000","last_became_active":"2023-09-23T20:49:02.701986+0000","last_became_peered":"2023-09-23T20:49:02.701986+0000","last_unstale":"2023-09-23T20:49:22.967598+0000","last_undegraded":"2023-09-23T20:49:22.967598+0000","last_fullsized":"2023-09-23T20:49:22.967598+0000","mapping_epoch":603,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":604,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:20:01.603267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.5","version":"619'11475","reported_seq":14301,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T20:49:23.304166+0000","last_change":"2023-09-23T20:48:53.196448+0000","last_active":"2023-09-23T20:49:23.304166+0000","last_peered":"2023-09-23T20:49:23.304166+0000","last_clean":"2023-09-23T20:49:23.304166+0000","last_became_active":"2023-09-23T20:48:53.175613+0000","last_became_peered":"2023-09-23T20:48:53.175613+0000","last_unstale":"2023-09-23T20:49:23.304166+0000","last_undegraded":"2023-09-23T20:49:23.304166+0000","last_fullsized":"2023-09-23T20:49:23.304166+0000","mapping_epoch":596,"log_start":"542'4806","ondisk_log_start":"542'4806","created":531,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":6669,"log_dups_size":0,"ondisk_log_size":6669,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":30638080,"num_objects":467,"num_object_clones":0,"num_object_copies":1401,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":467,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6124,"num_write_kb":210944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1338,"num_bytes_recovered":87687168,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.e","version":"90'161","reported_seq":649,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T20:49:22.963237+0000","last_change":"2023-09-23T20:47:37.608426+0000","last_active":"2023-09-23T20:49:22.963237+0000","last_peered":"2023-09-23T20:49:22.963237+0000","last_clean":"2023-09-23T20:49:22.963237+0000","last_became_active":"2023-09-23T20:47:11.617187+0000","last_became_peered":"2023-09-23T20:47:11.617187+0000","last_unstale":"2023-09-23T20:49:22.963237+0000","last_undegraded":"2023-09-23T20:49:22.963237+0000","last_fullsized":"2023-09-23T20:49:22.963237+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:47:37.608309+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:47:37.608309+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:22:25.482297+0000","scrub_duration":0.011600924,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":580,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T20:49:21.994321+0000","last_change":"2023-09-23T20:43:16.756437+0000","last_active":"2023-09-23T20:49:21.994321+0000","last_peered":"2023-09-23T20:49:21.994321+0000","last_clean":"2023-09-23T20:49:21.994321+0000","last_became_active":"2023-09-23T20:43:16.756193+0000","last_became_peered":"2023-09-23T20:43:16.756193+0000","last_unstale":"2023-09-23T20:49:21.994321+0000","last_undegraded":"2023-09-23T20:49:21.994321+0000","last_fullsized":"2023-09-23T20:49:21.994321+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:33:25.632781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":614,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T20:49:24.962174+0000","last_change":"2023-09-23T20:47:09.197427+0000","last_active":"2023-09-23T20:49:24.962174+0000","last_peered":"2023-09-23T20:49:24.962174+0000","last_clean":"2023-09-23T20:49:24.962174+0000","last_became_active":"2023-09-23T20:47:09.196542+0000","last_became_peered":"2023-09-23T20:47:09.196542+0000","last_unstale":"2023-09-23T20:49:24.962174+0000","last_undegraded":"2023-09-23T20:49:24.962174+0000","last_fullsized":"2023-09-23T20:49:24.962174+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:28:38.926202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"619'12169","reported_seq":14827,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T20:49:23.632594+0000","last_change":"2023-09-23T20:49:17.079087+0000","last_active":"2023-09-23T20:49:23.632594+0000","last_peered":"2023-09-23T20:49:23.632594+0000","last_clean":"2023-09-23T20:49:23.632594+0000","last_became_active":"2023-09-23T20:49:09.813951+0000","last_became_peered":"2023-09-23T20:49:09.813951+0000","last_unstale":"2023-09-23T20:49:23.632594+0000","last_undegraded":"2023-09-23T20:49:23.632594+0000","last_fullsized":"2023-09-23T20:49:23.632594+0000","mapping_epoch":606,"log_start":"542'5074","ondisk_log_start":"542'5074","created":531,"last_epoch_clean":607,"parent":"0.0","parent_split_bits":0,"last_scrub":"613'12076","last_scrub_stamp":"2023-09-23T20:49:17.078853+0000","last_deep_scrub":"613'12076","last_deep_scrub_stamp":"2023-09-23T20:49:17.078853+0000","last_clean_scrub_stamp":"2023-09-23T20:49:17.078853+0000","objects_scrubbed":575,"log_size":7095,"log_dups_size":0,"ondisk_log_size":7095,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":3,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:31:00.646449+0000","scrub_duration":2.36872857,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":29491200,"num_objects":450,"num_object_clones":0,"num_object_copies":1350,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":450,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":125,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,0],"acting":[1,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":1065,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T20:49:22.979432+0000","last_change":"2023-09-23T20:48:51.169474+0000","last_active":"2023-09-23T20:49:22.979432+0000","last_peered":"2023-09-23T20:49:22.979432+0000","last_clean":"2023-09-23T20:49:22.979432+0000","last_became_active":"2023-09-23T20:48:51.169076+0000","last_became_peered":"2023-09-23T20:48:51.169076+0000","last_unstale":"2023-09-23T20:49:22.979432+0000","last_undegraded":"2023-09-23T20:49:22.979432+0000","last_fullsized":"2023-09-23T20:49:22.979432+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:35:02.523203+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:35:02.523203+0000","last_clean_scrub_stamp":"2023-09-23T20:35:02.523203+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:25:53.741403+0000","scrub_duration":0.0085927989999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":468,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T20:49:21.997623+0000","last_change":"2023-09-23T20:47:09.240200+0000","last_active":"2023-09-23T20:49:21.997623+0000","last_peered":"2023-09-23T20:49:21.997623+0000","last_clean":"2023-09-23T20:49:21.997623+0000","last_became_active":"2023-09-23T20:47:09.195204+0000","last_became_peered":"2023-09-23T20:47:09.195204+0000","last_unstale":"2023-09-23T20:49:21.997623+0000","last_undegraded":"2023-09-23T20:49:21.997623+0000","last_fullsized":"2023-09-23T20:49:21.997623+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:03.328250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":76,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T20:49:22.963005+0000","last_change":"2023-09-23T20:47:09.231714+0000","last_active":"2023-09-23T20:49:22.963005+0000","last_peered":"2023-09-23T20:49:22.963005+0000","last_clean":"2023-09-23T20:49:22.963005+0000","last_became_active":"2023-09-23T20:47:09.230918+0000","last_became_peered":"2023-09-23T20:47:09.230918+0000","last_unstale":"2023-09-23T20:49:22.963005+0000","last_undegraded":"2023-09-23T20:49:22.963005+0000","last_fullsized":"2023-09-23T20:49:22.963005+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:24.583656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"619'12230","reported_seq":12871,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T20:49:23.334863+0000","last_change":"2023-09-23T20:49:14.601851+0000","last_active":"2023-09-23T20:49:23.334863+0000","last_peered":"2023-09-23T20:49:23.334863+0000","last_clean":"2023-09-23T20:49:23.334863+0000","last_became_active":"2023-09-23T20:48:48.159849+0000","last_became_peered":"2023-09-23T20:48:48.159849+0000","last_unstale":"2023-09-23T20:49:23.334863+0000","last_undegraded":"2023-09-23T20:49:23.334863+0000","last_fullsized":"2023-09-23T20:49:23.334863+0000","mapping_epoch":591,"log_start":"536'1300","ondisk_log_start":"536'1300","created":531,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"611'12084","last_scrub_stamp":"2023-09-23T20:49:14.601699+0000","last_deep_scrub":"611'12084","last_deep_scrub_stamp":"2023-09-23T20:49:14.601699+0000","last_clean_scrub_stamp":"2023-09-23T20:49:14.601699+0000","objects_scrubbed":629,"log_size":10930,"log_dups_size":0,"ondisk_log_size":10930,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":4,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:25:59.647282+0000","scrub_duration":3.02326877,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":30081024,"num_objects":459,"num_object_clones":0,"num_object_copies":1377,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":459,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":170,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":494,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T20:49:21.924865+0000","last_change":"2023-09-23T20:48:51.176533+0000","last_active":"2023-09-23T20:49:21.924865+0000","last_peered":"2023-09-23T20:49:21.924865+0000","last_clean":"2023-09-23T20:49:21.924865+0000","last_became_active":"2023-09-23T20:48:51.175911+0000","last_became_peered":"2023-09-23T20:48:51.175911+0000","last_unstale":"2023-09-23T20:49:21.924865+0000","last_undegraded":"2023-09-23T20:49:21.924865+0000","last_fullsized":"2023-09-23T20:49:21.924865+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:42:45.492220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:42:45.492220+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":244,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T20:49:22.967685+0000","last_change":"2023-09-23T20:47:09.178919+0000","last_active":"2023-09-23T20:49:22.967685+0000","last_peered":"2023-09-23T20:49:22.967685+0000","last_clean":"2023-09-23T20:49:22.967685+0000","last_became_active":"2023-09-23T20:47:09.178607+0000","last_became_peered":"2023-09-23T20:47:09.178607+0000","last_unstale":"2023-09-23T20:49:22.967685+0000","last_undegraded":"2023-09-23T20:49:22.967685+0000","last_fullsized":"2023-09-23T20:49:22.967685+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:23:45.196935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":96,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T20:49:22.964601+0000","last_change":"2023-09-23T20:48:48.148596+0000","last_active":"2023-09-23T20:49:22.964601+0000","last_peered":"2023-09-23T20:49:22.964601+0000","last_clean":"2023-09-23T20:49:22.964601+0000","last_became_active":"2023-09-23T20:48:48.148211+0000","last_became_peered":"2023-09-23T20:48:48.148211+0000","last_unstale":"2023-09-23T20:49:22.964601+0000","last_undegraded":"2023-09-23T20:49:22.964601+0000","last_fullsized":"2023-09-23T20:49:22.964601+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:21:50.825822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.0","version":"620'11413","reported_seq":12710,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T20:49:25.151203+0000","last_change":"2023-09-23T20:49:11.294070+0000","last_active":"2023-09-23T20:49:25.151203+0000","last_peered":"2023-09-23T20:49:25.151203+0000","last_clean":"2023-09-23T20:49:25.151203+0000","last_became_active":"2023-09-23T20:48:40.067403+0000","last_became_peered":"2023-09-23T20:48:40.067403+0000","last_unstale":"2023-09-23T20:49:25.151203+0000","last_undegraded":"2023-09-23T20:49:25.151203+0000","last_fullsized":"2023-09-23T20:49:25.151203+0000","mapping_epoch":559,"log_start":"542'4945","ondisk_log_start":"542'4945","created":531,"last_epoch_clean":585,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":6468,"log_dups_size":0,"ondisk_log_size":6468,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":25395200,"num_objects":387,"num_object_clones":0,"num_object_copies":1161,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":387,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6107,"num_write_kb":207840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":289,"num_bytes_recovered":18939904,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.b","version":"90'161","reported_seq":924,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T20:49:21.926904+0000","last_change":"2023-09-23T20:40:22.445808+0000","last_active":"2023-09-23T20:49:21.926904+0000","last_peered":"2023-09-23T20:49:21.926904+0000","last_clean":"2023-09-23T20:49:21.926904+0000","last_became_active":"2023-09-23T20:40:22.445456+0000","last_became_peered":"2023-09-23T20:40:22.445456+0000","last_unstale":"2023-09-23T20:49:21.926904+0000","last_undegraded":"2023-09-23T20:49:21.926904+0000","last_fullsized":"2023-09-23T20:49:21.926904+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:05.595042+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:05.595042+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:39:23.678154+0000","scrub_duration":0.007271675,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":96,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T20:49:22.962735+0000","last_change":"2023-09-23T20:48:48.153145+0000","last_active":"2023-09-23T20:49:22.962735+0000","last_peered":"2023-09-23T20:49:22.962735+0000","last_clean":"2023-09-23T20:49:22.962735+0000","last_became_active":"2023-09-23T20:48:48.152858+0000","last_became_peered":"2023-09-23T20:48:48.152858+0000","last_unstale":"2023-09-23T20:49:22.962735+0000","last_undegraded":"2023-09-23T20:49:22.962735+0000","last_fullsized":"2023-09-23T20:49:22.962735+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:23:13.454288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":76,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T20:49:22.963332+0000","last_change":"2023-09-23T20:47:09.186271+0000","last_active":"2023-09-23T20:49:22.963332+0000","last_peered":"2023-09-23T20:49:22.963332+0000","last_clean":"2023-09-23T20:49:22.963332+0000","last_became_active":"2023-09-23T20:47:09.184357+0000","last_became_peered":"2023-09-23T20:47:09.184357+0000","last_unstale":"2023-09-23T20:49:22.963332+0000","last_undegraded":"2023-09-23T20:49:22.963332+0000","last_fullsized":"2023-09-23T20:49:22.963332+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:48:26.339811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"620'11533","reported_seq":13288,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T20:49:25.109074+0000","last_change":"2023-09-23T20:49:16.205736+0000","last_active":"2023-09-23T20:49:25.109074+0000","last_peered":"2023-09-23T20:49:25.109074+0000","last_clean":"2023-09-23T20:49:25.109074+0000","last_became_active":"2023-09-23T20:49:10.816682+0000","last_became_peered":"2023-09-23T20:49:10.816682+0000","last_unstale":"2023-09-23T20:49:25.109074+0000","last_undegraded":"2023-09-23T20:49:25.109074+0000","last_fullsized":"2023-09-23T20:49:25.109074+0000","mapping_epoch":607,"log_start":"543'5336","ondisk_log_start":"543'5336","created":531,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":6197,"log_dups_size":0,"ondisk_log_size":6197,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":23592960,"num_objects":360,"num_object_clones":0,"num_object_copies":1080,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":360,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6192,"num_write_kb":209664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":795,"num_bytes_recovered":52101120,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":498,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T20:49:21.925325+0000","last_change":"2023-09-23T20:48:51.177368+0000","last_active":"2023-09-23T20:49:21.925325+0000","last_peered":"2023-09-23T20:49:21.925325+0000","last_clean":"2023-09-23T20:49:21.925325+0000","last_became_active":"2023-09-23T20:48:51.177024+0000","last_became_peered":"2023-09-23T20:48:51.177024+0000","last_unstale":"2023-09-23T20:49:21.925325+0000","last_undegraded":"2023-09-23T20:49:21.925325+0000","last_fullsized":"2023-09-23T20:49:21.925325+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:43:51.693459+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:43:51.693459+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26834,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,6],"acting":[2,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":416,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T20:49:21.993656+0000","last_change":"2023-09-23T20:37:43.570610+0000","last_active":"2023-09-23T20:49:21.993656+0000","last_peered":"2023-09-23T20:49:21.993656+0000","last_clean":"2023-09-23T20:49:21.993656+0000","last_became_active":"2023-09-23T20:37:43.570216+0000","last_became_peered":"2023-09-23T20:37:43.570216+0000","last_unstale":"2023-09-23T20:49:21.993656+0000","last_undegraded":"2023-09-23T20:49:21.993656+0000","last_fullsized":"2023-09-23T20:49:21.993656+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:39:26.555329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":464,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T20:49:21.928218+0000","last_change":"2023-09-23T20:47:09.172573+0000","last_active":"2023-09-23T20:49:21.928218+0000","last_peered":"2023-09-23T20:49:21.928218+0000","last_clean":"2023-09-23T20:49:21.928218+0000","last_became_active":"2023-09-23T20:47:09.172298+0000","last_became_peered":"2023-09-23T20:47:09.172298+0000","last_unstale":"2023-09-23T20:49:21.928218+0000","last_undegraded":"2023-09-23T20:49:21.928218+0000","last_fullsized":"2023-09-23T20:49:21.928218+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:22:09.361284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"620'12439","reported_seq":16072,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T20:49:25.092710+0000","last_change":"2023-09-23T20:49:12.738667+0000","last_active":"2023-09-23T20:49:25.092710+0000","last_peered":"2023-09-23T20:49:25.092710+0000","last_clean":"2023-09-23T20:49:25.092710+0000","last_became_active":"2023-09-23T20:49:12.701834+0000","last_became_peered":"2023-09-23T20:49:12.701834+0000","last_unstale":"2023-09-23T20:49:25.092710+0000","last_undegraded":"2023-09-23T20:49:25.092710+0000","last_fullsized":"2023-09-23T20:49:25.092710+0000","mapping_epoch":609,"log_start":"562'6993","ondisk_log_start":"562'6993","created":531,"last_epoch_clean":610,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":5446,"log_dups_size":0,"ondisk_log_size":5446,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:02:43.327858+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53018624,"num_objects":809,"num_object_clones":0,"num_object_copies":2427,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":809,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12439,"num_write_kb":423936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1741,"num_bytes_recovered":114098176,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":807,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T20:49:22.966546+0000","last_change":"2023-09-23T20:41:07.424609+0000","last_active":"2023-09-23T20:49:22.966546+0000","last_peered":"2023-09-23T20:49:22.966546+0000","last_clean":"2023-09-23T20:49:22.966546+0000","last_became_active":"2023-09-23T20:38:05.622261+0000","last_became_peered":"2023-09-23T20:38:05.622261+0000","last_unstale":"2023-09-23T20:49:22.966546+0000","last_undegraded":"2023-09-23T20:49:22.966546+0000","last_fullsized":"2023-09-23T20:49:22.966546+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:41:07.424492+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:41:07.424492+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:45:21.689123+0000","scrub_duration":0.0018960959999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"86'119","reported_seq":931,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T20:49:21.927433+0000","last_change":"2023-09-23T20:29:45.521257+0000","last_active":"2023-09-23T20:49:21.927433+0000","last_peered":"2023-09-23T20:49:21.927433+0000","last_clean":"2023-09-23T20:49:21.927433+0000","last_became_active":"2023-09-23T20:29:45.520546+0000","last_became_peered":"2023-09-23T20:29:45.520546+0000","last_unstale":"2023-09-23T20:49:21.927433+0000","last_undegraded":"2023-09-23T20:49:21.927433+0000","last_fullsized":"2023-09-23T20:49:21.927433+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":119,"log_dups_size":0,"ondisk_log_size":119,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:58:56.085777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032705499999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":523,"num_read_kb":3503,"num_write":136,"num_write_kb":3128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":431,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T20:49:21.998163+0000","last_change":"2023-09-23T20:48:48.157475+0000","last_active":"2023-09-23T20:49:21.998163+0000","last_peered":"2023-09-23T20:49:21.998163+0000","last_clean":"2023-09-23T20:49:21.998163+0000","last_became_active":"2023-09-23T20:48:48.155904+0000","last_became_peered":"2023-09-23T20:48:48.155904+0000","last_unstale":"2023-09-23T20:49:21.998163+0000","last_undegraded":"2023-09-23T20:49:21.998163+0000","last_fullsized":"2023-09-23T20:49:21.998163+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:39:08.642474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,2],"acting":[3,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":596,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T20:49:21.926323+0000","last_change":"2023-09-23T20:47:09.194947+0000","last_active":"2023-09-23T20:49:21.926323+0000","last_peered":"2023-09-23T20:49:21.926323+0000","last_clean":"2023-09-23T20:49:21.926323+0000","last_became_active":"2023-09-23T20:47:09.194592+0000","last_became_peered":"2023-09-23T20:47:09.194592+0000","last_unstale":"2023-09-23T20:49:21.926323+0000","last_undegraded":"2023-09-23T20:49:21.926323+0000","last_fullsized":"2023-09-23T20:49:21.926323+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:34:15.008492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.a","version":"620'11961","reported_seq":12057,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T20:49:25.187956+0000","last_change":"2023-09-23T20:45:39.489110+0000","last_active":"2023-09-23T20:49:25.187956+0000","last_peered":"2023-09-23T20:49:25.187956+0000","last_clean":"2023-09-23T20:49:25.187956+0000","last_became_active":"2023-09-23T20:45:39.488817+0000","last_became_peered":"2023-09-23T20:45:39.488817+0000","last_unstale":"2023-09-23T20:49:25.187956+0000","last_undegraded":"2023-09-23T20:49:25.187956+0000","last_fullsized":"2023-09-23T20:49:25.187956+0000","mapping_epoch":531,"log_start":"544'5890","ondisk_log_start":"544'5890","created":531,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":6071,"log_dups_size":0,"ondisk_log_size":6071,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:07:21.467971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":50921472,"num_objects":777,"num_object_clones":0,"num_object_copies":2331,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":777,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11961,"num_write_kb":407616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":702,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T20:49:24.963818+0000","last_change":"2023-09-23T20:48:51.178609+0000","last_active":"2023-09-23T20:49:24.963818+0000","last_peered":"2023-09-23T20:49:24.963818+0000","last_clean":"2023-09-23T20:49:24.963818+0000","last_became_active":"2023-09-23T20:48:51.178196+0000","last_became_peered":"2023-09-23T20:48:51.178196+0000","last_unstale":"2023-09-23T20:49:24.963818+0000","last_undegraded":"2023-09-23T20:49:24.963818+0000","last_fullsized":"2023-09-23T20:49:24.963818+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:15.656880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:15.656880+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:47:54.557002+0000","scrub_duration":0.0016711180000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"86'106","reported_seq":871,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T20:49:21.926561+0000","last_change":"2023-09-23T20:43:16.759138+0000","last_active":"2023-09-23T20:49:21.926561+0000","last_peered":"2023-09-23T20:49:21.926561+0000","last_clean":"2023-09-23T20:49:21.926561+0000","last_became_active":"2023-09-23T20:43:16.758015+0000","last_became_peered":"2023-09-23T20:43:16.758015+0000","last_unstale":"2023-09-23T20:49:21.926561+0000","last_undegraded":"2023-09-23T20:49:21.926561+0000","last_fullsized":"2023-09-23T20:49:21.926561+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":106,"log_dups_size":0,"ondisk_log_size":106,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:08:02.852472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00099302599999999993,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1094,"num_read_kb":1678,"num_write":100,"num_write_kb":404,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":76,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T20:49:22.963467+0000","last_change":"2023-09-23T20:47:09.177872+0000","last_active":"2023-09-23T20:49:22.963467+0000","last_peered":"2023-09-23T20:49:22.963467+0000","last_clean":"2023-09-23T20:49:22.963467+0000","last_became_active":"2023-09-23T20:47:09.176689+0000","last_became_peered":"2023-09-23T20:47:09.176689+0000","last_unstale":"2023-09-23T20:49:22.963467+0000","last_undegraded":"2023-09-23T20:49:22.963467+0000","last_fullsized":"2023-09-23T20:49:22.963467+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:14:01.590318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":602,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T20:49:21.927337+0000","last_change":"2023-09-23T20:48:48.151144+0000","last_active":"2023-09-23T20:49:21.927337+0000","last_peered":"2023-09-23T20:49:21.927337+0000","last_clean":"2023-09-23T20:49:21.927337+0000","last_became_active":"2023-09-23T20:48:48.150770+0000","last_became_peered":"2023-09-23T20:48:48.150770+0000","last_unstale":"2023-09-23T20:49:21.927337+0000","last_undegraded":"2023-09-23T20:49:21.927337+0000","last_fullsized":"2023-09-23T20:49:21.927337+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:14:08.549512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.9","version":"619'11534","reported_seq":13667,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T20:49:23.210175+0000","last_change":"2023-09-23T20:49:15.712076+0000","last_active":"2023-09-23T20:49:23.210175+0000","last_peered":"2023-09-23T20:49:23.210175+0000","last_clean":"2023-09-23T20:49:23.210175+0000","last_became_active":"2023-09-23T20:48:49.169141+0000","last_became_peered":"2023-09-23T20:48:49.169141+0000","last_unstale":"2023-09-23T20:49:23.210175+0000","last_undegraded":"2023-09-23T20:49:23.210175+0000","last_fullsized":"2023-09-23T20:49:23.210175+0000","mapping_epoch":592,"log_start":"541'4623","ondisk_log_start":"541'4623","created":531,"last_epoch_clean":593,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":6911,"log_dups_size":0,"ondisk_log_size":6911,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":29327360,"num_objects":447,"num_object_clones":0,"num_object_copies":1341,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":447,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6155,"num_write_kb":211296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1000,"num_bytes_recovered":65404928,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":623,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T20:49:22.980133+0000","last_change":"2023-09-23T20:47:11.618652+0000","last_active":"2023-09-23T20:49:22.980133+0000","last_peered":"2023-09-23T20:49:22.980133+0000","last_clean":"2023-09-23T20:49:22.980133+0000","last_became_active":"2023-09-23T20:47:11.618266+0000","last_became_peered":"2023-09-23T20:47:11.618266+0000","last_unstale":"2023-09-23T20:49:22.980133+0000","last_undegraded":"2023-09-23T20:49:22.980133+0000","last_fullsized":"2023-09-23T20:49:22.980133+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:26.978179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:26.978179+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:29:29.000683+0000","scrub_duration":0.0013962829999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"86'130","reported_seq":404,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T20:49:21.925309+0000","last_change":"2023-09-23T20:48:51.171448+0000","last_active":"2023-09-23T20:49:21.925309+0000","last_peered":"2023-09-23T20:49:21.925309+0000","last_clean":"2023-09-23T20:49:21.925309+0000","last_became_active":"2023-09-23T20:48:51.170485+0000","last_became_peered":"2023-09-23T20:48:51.170485+0000","last_unstale":"2023-09-23T20:49:21.925309+0000","last_undegraded":"2023-09-23T20:49:21.925309+0000","last_fullsized":"2023-09-23T20:49:21.925309+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:48:35.309684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011300049999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":512,"num_read_kb":1820,"num_write":133,"num_write_kb":1347,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":610,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T20:49:21.925929+0000","last_change":"2023-09-23T20:48:48.151034+0000","last_active":"2023-09-23T20:49:21.925929+0000","last_peered":"2023-09-23T20:49:21.925929+0000","last_clean":"2023-09-23T20:49:21.925929+0000","last_became_active":"2023-09-23T20:48:48.150658+0000","last_became_peered":"2023-09-23T20:48:48.150658+0000","last_unstale":"2023-09-23T20:49:21.925929+0000","last_undegraded":"2023-09-23T20:49:21.925929+0000","last_fullsized":"2023-09-23T20:49:21.925929+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:44:34.819851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":135,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T20:49:21.928155+0000","last_change":"2023-09-23T20:44:10.555583+0000","last_active":"2023-09-23T20:49:21.928155+0000","last_peered":"2023-09-23T20:49:21.928155+0000","last_clean":"2023-09-23T20:49:21.928155+0000","last_became_active":"2023-09-23T20:44:10.555281+0000","last_became_peered":"2023-09-23T20:44:10.555281+0000","last_unstale":"2023-09-23T20:49:21.928155+0000","last_undegraded":"2023-09-23T20:49:21.928155+0000","last_fullsized":"2023-09-23T20:49:21.928155+0000","mapping_epoch":496,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":497,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:43:11.942147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"620'12327","reported_seq":12423,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T20:49:25.172737+0000","last_change":"2023-09-23T20:45:39.331096+0000","last_active":"2023-09-23T20:49:25.172737+0000","last_peered":"2023-09-23T20:49:25.172737+0000","last_clean":"2023-09-23T20:49:25.172737+0000","last_became_active":"2023-09-23T20:45:39.330752+0000","last_became_peered":"2023-09-23T20:45:39.330752+0000","last_unstale":"2023-09-23T20:49:25.172737+0000","last_undegraded":"2023-09-23T20:49:25.172737+0000","last_fullsized":"2023-09-23T20:49:25.172737+0000","mapping_epoch":531,"log_start":"551'6229","ondisk_log_start":"551'6229","created":531,"last_epoch_clean":532,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":6098,"log_dups_size":0,"ondisk_log_size":6098,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:39:01.627762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":51183616,"num_objects":781,"num_object_clones":0,"num_object_copies":2343,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":781,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12327,"num_write_kb":419456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"86'194","reported_seq":730,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T20:49:21.925664+0000","last_change":"2023-09-23T20:48:51.175633+0000","last_active":"2023-09-23T20:49:21.925664+0000","last_peered":"2023-09-23T20:49:21.925664+0000","last_clean":"2023-09-23T20:49:21.925664+0000","last_became_active":"2023-09-23T20:48:51.174941+0000","last_became_peered":"2023-09-23T20:48:51.174941+0000","last_unstale":"2023-09-23T20:49:21.925664+0000","last_undegraded":"2023-09-23T20:49:21.925664+0000","last_fullsized":"2023-09-23T20:49:21.925664+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":194,"log_dups_size":0,"ondisk_log_size":194,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:06:27.009241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00037995999999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1024,"num_read_kb":1200,"num_write":173,"num_write_kb":1276,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":1082,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T20:49:22.979734+0000","last_change":"2023-09-23T20:38:05.602508+0000","last_active":"2023-09-23T20:49:22.979734+0000","last_peered":"2023-09-23T20:49:22.979734+0000","last_clean":"2023-09-23T20:49:22.979734+0000","last_became_active":"2023-09-23T20:38:05.602129+0000","last_became_peered":"2023-09-23T20:38:05.602129+0000","last_unstale":"2023-09-23T20:49:22.979734+0000","last_undegraded":"2023-09-23T20:49:22.979734+0000","last_fullsized":"2023-09-23T20:49:22.979734+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:45.343742+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:45.343742+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:48.910343+0000","scrub_duration":0.001475802,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":194,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T20:49:22.967763+0000","last_change":"2023-09-23T20:43:16.758213+0000","last_active":"2023-09-23T20:49:22.967763+0000","last_peered":"2023-09-23T20:49:22.967763+0000","last_clean":"2023-09-23T20:49:22.967763+0000","last_became_active":"2023-09-23T20:43:16.757869+0000","last_became_peered":"2023-09-23T20:43:16.757869+0000","last_unstale":"2023-09-23T20:49:22.967763+0000","last_undegraded":"2023-09-23T20:49:22.967763+0000","last_fullsized":"2023-09-23T20:49:22.967763+0000","mapping_epoch":481,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":482,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:43:10.811749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":244,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T20:49:22.966767+0000","last_change":"2023-09-23T20:47:09.202302+0000","last_active":"2023-09-23T20:49:22.966767+0000","last_peered":"2023-09-23T20:49:22.966767+0000","last_clean":"2023-09-23T20:49:22.966767+0000","last_became_active":"2023-09-23T20:47:09.201797+0000","last_became_peered":"2023-09-23T20:47:09.201797+0000","last_unstale":"2023-09-23T20:49:22.966767+0000","last_undegraded":"2023-09-23T20:49:22.966767+0000","last_fullsized":"2023-09-23T20:49:22.966767+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:34:41.055705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.8","version":"621'11287","reported_seq":15106,"reported_epoch":621,"state":"active+clean","last_fresh":"2023-09-23T20:49:26.236338+0000","last_change":"2023-09-23T20:48:52.187793+0000","last_active":"2023-09-23T20:49:26.236338+0000","last_peered":"2023-09-23T20:49:26.236338+0000","last_clean":"2023-09-23T20:49:26.236338+0000","last_became_active":"2023-09-23T20:48:52.173033+0000","last_became_peered":"2023-09-23T20:48:52.173033+0000","last_unstale":"2023-09-23T20:49:26.236338+0000","last_undegraded":"2023-09-23T20:49:26.236338+0000","last_fullsized":"2023-09-23T20:49:26.236338+0000","mapping_epoch":595,"log_start":"541'4571","ondisk_log_start":"541'4571","created":531,"last_epoch_clean":596,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":6716,"log_dups_size":0,"ondisk_log_size":6716,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":25722880,"num_objects":392,"num_object_clones":0,"num_object_copies":1176,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":392,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6034,"num_write_kb":205664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":994,"num_bytes_recovered":65175552,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":987,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T20:49:21.926227+0000","last_change":"2023-09-23T20:47:11.622262+0000","last_active":"2023-09-23T20:49:21.926227+0000","last_peered":"2023-09-23T20:49:21.926227+0000","last_clean":"2023-09-23T20:49:21.926227+0000","last_became_active":"2023-09-23T20:47:11.621920+0000","last_became_peered":"2023-09-23T20:47:11.621920+0000","last_unstale":"2023-09-23T20:49:21.926227+0000","last_undegraded":"2023-09-23T20:49:21.926227+0000","last_fullsized":"2023-09-23T20:49:21.926227+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:43.582146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:43.582146+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:01:15.153543+0000","scrub_duration":0.0016219730000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"86'80","reported_seq":872,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T20:49:22.979450+0000","last_change":"2023-09-23T20:29:45.536796+0000","last_active":"2023-09-23T20:49:22.979450+0000","last_peered":"2023-09-23T20:49:22.979450+0000","last_clean":"2023-09-23T20:49:22.979450+0000","last_became_active":"2023-09-23T20:29:45.536439+0000","last_became_peered":"2023-09-23T20:29:45.536439+0000","last_unstale":"2023-09-23T20:49:22.979450+0000","last_undegraded":"2023-09-23T20:49:22.979450+0000","last_fullsized":"2023-09-23T20:49:22.979450+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":80,"log_dups_size":0,"ondisk_log_size":80,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:30:57.793132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035115,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":395,"num_read_kb":1370,"num_write":66,"num_write_kb":1061,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":633,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T20:49:24.963717+0000","last_change":"2023-09-23T20:47:09.198326+0000","last_active":"2023-09-23T20:49:24.963717+0000","last_peered":"2023-09-23T20:49:24.963717+0000","last_clean":"2023-09-23T20:49:24.963717+0000","last_became_active":"2023-09-23T20:47:09.198059+0000","last_became_peered":"2023-09-23T20:47:09.198059+0000","last_unstale":"2023-09-23T20:49:24.963717+0000","last_undegraded":"2023-09-23T20:49:24.963717+0000","last_fullsized":"2023-09-23T20:49:24.963717+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:38:53.903706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":628,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T20:49:24.962211+0000","last_change":"2023-09-23T20:48:48.146977+0000","last_active":"2023-09-23T20:49:24.962211+0000","last_peered":"2023-09-23T20:49:24.962211+0000","last_clean":"2023-09-23T20:49:24.962211+0000","last_became_active":"2023-09-23T20:48:48.146657+0000","last_became_peered":"2023-09-23T20:48:48.146657+0000","last_unstale":"2023-09-23T20:49:24.962211+0000","last_undegraded":"2023-09-23T20:49:24.962211+0000","last_fullsized":"2023-09-23T20:49:24.962211+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:38:41.782692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"620'12075","reported_seq":12797,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T20:49:25.103899+0000","last_change":"2023-09-23T20:47:13.668055+0000","last_active":"2023-09-23T20:49:25.103899+0000","last_peered":"2023-09-23T20:49:25.103899+0000","last_clean":"2023-09-23T20:49:25.103899+0000","last_became_active":"2023-09-23T20:47:13.626170+0000","last_became_peered":"2023-09-23T20:47:13.626170+0000","last_unstale":"2023-09-23T20:49:25.103899+0000","last_undegraded":"2023-09-23T20:49:25.103899+0000","last_fullsized":"2023-09-23T20:49:25.103899+0000","mapping_epoch":557,"log_start":"545'5995","ondisk_log_start":"545'5995","created":531,"last_epoch_clean":558,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":6080,"log_dups_size":0,"ondisk_log_size":6080,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:32.805547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53739544,"num_objects":821,"num_object_clones":0,"num_object_copies":2463,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":821,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":12075,"num_write_kb":412609,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":298,"num_bytes_recovered":19529728,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":730,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T20:49:22.965454+0000","last_change":"2023-09-23T20:41:08.380084+0000","last_active":"2023-09-23T20:49:22.965454+0000","last_peered":"2023-09-23T20:49:22.965454+0000","last_clean":"2023-09-23T20:49:22.965454+0000","last_became_active":"2023-09-23T20:38:05.610881+0000","last_became_peered":"2023-09-23T20:38:05.610881+0000","last_unstale":"2023-09-23T20:49:22.965454+0000","last_undegraded":"2023-09-23T20:49:22.965454+0000","last_fullsized":"2023-09-23T20:49:22.965454+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:41:08.379959+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:41:08.379959+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:19:39.547923+0000","scrub_duration":0.0019158420000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"86'102","reported_seq":1290,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T20:49:22.980081+0000","last_change":"2023-09-23T20:47:14.628795+0000","last_active":"2023-09-23T20:49:22.980081+0000","last_peered":"2023-09-23T20:49:22.980081+0000","last_clean":"2023-09-23T20:49:22.980081+0000","last_became_active":"2023-09-23T20:47:14.628341+0000","last_became_peered":"2023-09-23T20:47:14.628341+0000","last_unstale":"2023-09-23T20:49:22.980081+0000","last_undegraded":"2023-09-23T20:49:22.980081+0000","last_fullsized":"2023-09-23T20:49:22.980081+0000","mapping_epoch":558,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":559,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":102,"log_dups_size":0,"ondisk_log_size":102,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:42:16.291849+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030656500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":963,"num_read_kb":1234,"num_write":60,"num_write_kb":225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":586,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T20:49:21.927460+0000","last_change":"2023-09-23T20:48:48.151135+0000","last_active":"2023-09-23T20:49:21.927460+0000","last_peered":"2023-09-23T20:49:21.927460+0000","last_clean":"2023-09-23T20:49:21.927460+0000","last_became_active":"2023-09-23T20:48:48.150780+0000","last_became_peered":"2023-09-23T20:48:48.150780+0000","last_unstale":"2023-09-23T20:49:21.927460+0000","last_undegraded":"2023-09-23T20:49:21.927460+0000","last_fullsized":"2023-09-23T20:49:21.927460+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:39:09.727249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":565,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T20:49:24.962402+0000","last_change":"2023-09-23T20:43:46.541173+0000","last_active":"2023-09-23T20:49:24.962402+0000","last_peered":"2023-09-23T20:49:24.962402+0000","last_clean":"2023-09-23T20:49:24.962402+0000","last_became_active":"2023-09-23T20:43:46.540726+0000","last_became_peered":"2023-09-23T20:43:46.540726+0000","last_unstale":"2023-09-23T20:49:24.962402+0000","last_undegraded":"2023-09-23T20:49:24.962402+0000","last_fullsized":"2023-09-23T20:49:24.962402+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:49:04.137885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.e","version":"620'12242","reported_seq":16555,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T20:49:25.157508+0000","last_change":"2023-09-23T20:48:48.230594+0000","last_active":"2023-09-23T20:49:25.157508+0000","last_peered":"2023-09-23T20:49:25.157508+0000","last_clean":"2023-09-23T20:49:25.157508+0000","last_became_active":"2023-09-23T20:48:48.164237+0000","last_became_peered":"2023-09-23T20:48:48.164237+0000","last_unstale":"2023-09-23T20:49:25.157508+0000","last_undegraded":"2023-09-23T20:49:25.157508+0000","last_fullsized":"2023-09-23T20:49:25.157508+0000","mapping_epoch":591,"log_start":"562'7000","ondisk_log_start":"562'7000","created":531,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":5242,"log_dups_size":0,"ondisk_log_size":5242,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:49:13.858414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":49545216,"num_objects":756,"num_object_clones":0,"num_object_copies":2268,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":756,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12242,"num_write_kb":415936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2081,"num_bytes_recovered":136314880,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":656,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T20:49:22.964678+0000","last_change":"2023-09-23T20:48:51.172830+0000","last_active":"2023-09-23T20:49:22.964678+0000","last_peered":"2023-09-23T20:49:22.964678+0000","last_clean":"2023-09-23T20:49:22.964678+0000","last_became_active":"2023-09-23T20:48:51.172481+0000","last_became_peered":"2023-09-23T20:48:51.172481+0000","last_unstale":"2023-09-23T20:49:22.964678+0000","last_undegraded":"2023-09-23T20:49:22.964678+0000","last_fullsized":"2023-09-23T20:49:22.964678+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:47:34.561747+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:47:34.561747+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:30:32.531477+0000","scrub_duration":0.0087949929999999992,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":357,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T20:49:22.980262+0000","last_change":"2023-09-23T20:32:50.549695+0000","last_active":"2023-09-23T20:49:22.980262+0000","last_peered":"2023-09-23T20:49:22.980262+0000","last_clean":"2023-09-23T20:49:22.980262+0000","last_became_active":"2023-09-23T20:32:50.549431+0000","last_became_peered":"2023-09-23T20:32:50.549431+0000","last_unstale":"2023-09-23T20:49:22.980262+0000","last_undegraded":"2023-09-23T20:49:22.980262+0000","last_fullsized":"2023-09-23T20:49:22.980262+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:45:53.666569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"86'128","reported_seq":1100,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T20:49:21.927538+0000","last_change":"2023-09-23T20:47:11.621846+0000","last_active":"2023-09-23T20:49:21.927538+0000","last_peered":"2023-09-23T20:49:21.927538+0000","last_clean":"2023-09-23T20:49:21.927538+0000","last_became_active":"2023-09-23T20:47:11.621284+0000","last_became_peered":"2023-09-23T20:47:11.621284+0000","last_unstale":"2023-09-23T20:49:21.927538+0000","last_undegraded":"2023-09-23T20:49:21.927538+0000","last_fullsized":"2023-09-23T20:49:21.927538+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:43:50.658512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000293202,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":579,"num_read_kb":1473,"num_write":96,"num_write_kb":1062,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":401,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T20:49:21.993559+0000","last_change":"2023-09-23T20:37:43.582105+0000","last_active":"2023-09-23T20:49:21.993559+0000","last_peered":"2023-09-23T20:49:21.993559+0000","last_clean":"2023-09-23T20:49:21.993559+0000","last_became_active":"2023-09-23T20:37:43.581860+0000","last_became_peered":"2023-09-23T20:37:43.581860+0000","last_unstale":"2023-09-23T20:49:21.993559+0000","last_undegraded":"2023-09-23T20:49:21.993559+0000","last_fullsized":"2023-09-23T20:49:21.993559+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:18:38.765486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"620'12252","reported_seq":17774,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T20:49:25.155921+0000","last_change":"2023-09-23T20:48:53.235559+0000","last_active":"2023-09-23T20:49:25.155921+0000","last_peered":"2023-09-23T20:49:25.155921+0000","last_clean":"2023-09-23T20:49:25.155921+0000","last_became_active":"2023-09-23T20:48:53.178550+0000","last_became_peered":"2023-09-23T20:48:53.178550+0000","last_unstale":"2023-09-23T20:49:25.155921+0000","last_undegraded":"2023-09-23T20:49:25.155921+0000","last_fullsized":"2023-09-23T20:49:25.155921+0000","mapping_epoch":596,"log_start":"550'6199","ondisk_log_start":"550'6199","created":531,"last_epoch_clean":597,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":6053,"log_dups_size":0,"ondisk_log_size":6053,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:38:02.612388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":49414144,"num_objects":754,"num_object_clones":0,"num_object_copies":2262,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":754,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12252,"num_write_kb":416192,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2684,"num_bytes_recovered":175898624,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.4","version":"90'161","reported_seq":835,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T20:49:21.994091+0000","last_change":"2023-09-23T20:40:22.427435+0000","last_active":"2023-09-23T20:49:21.994091+0000","last_peered":"2023-09-23T20:49:21.994091+0000","last_clean":"2023-09-23T20:49:21.994091+0000","last_became_active":"2023-09-23T20:40:22.427051+0000","last_became_peered":"2023-09-23T20:40:22.427051+0000","last_unstale":"2023-09-23T20:49:21.994091+0000","last_undegraded":"2023-09-23T20:49:21.994091+0000","last_fullsized":"2023-09-23T20:49:21.994091+0000","mapping_epoch":443,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":444,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:34:54.504554+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:34:54.504554+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:42.001652+0000","scrub_duration":0.011176864,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"86'117","reported_seq":1003,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T20:49:22.966472+0000","last_change":"2023-09-23T20:48:51.217483+0000","last_active":"2023-09-23T20:49:22.966472+0000","last_peered":"2023-09-23T20:49:22.966472+0000","last_clean":"2023-09-23T20:49:22.966472+0000","last_became_active":"2023-09-23T20:48:51.217013+0000","last_became_peered":"2023-09-23T20:48:51.217013+0000","last_unstale":"2023-09-23T20:49:22.966472+0000","last_undegraded":"2023-09-23T20:49:22.966472+0000","last_fullsized":"2023-09-23T20:49:22.966472+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:40:42.297947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046891400000000002,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":893,"num_read_kb":922,"num_write":82,"num_write_kb":4193,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":48,"num_bytes_recovered":25165824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":636,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T20:49:24.962276+0000","last_change":"2023-09-23T20:47:09.317116+0000","last_active":"2023-09-23T20:49:24.962276+0000","last_peered":"2023-09-23T20:49:24.962276+0000","last_clean":"2023-09-23T20:49:24.962276+0000","last_became_active":"2023-09-23T20:47:09.205930+0000","last_became_peered":"2023-09-23T20:47:09.205930+0000","last_unstale":"2023-09-23T20:49:24.962276+0000","last_undegraded":"2023-09-23T20:49:24.962276+0000","last_fullsized":"2023-09-23T20:49:24.962276+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:04:55.528625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":76,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T20:49:22.962768+0000","last_change":"2023-09-23T20:47:09.192117+0000","last_active":"2023-09-23T20:49:22.962768+0000","last_peered":"2023-09-23T20:49:22.962768+0000","last_clean":"2023-09-23T20:49:22.962768+0000","last_became_active":"2023-09-23T20:47:09.191820+0000","last_became_peered":"2023-09-23T20:47:09.191820+0000","last_unstale":"2023-09-23T20:49:22.962768+0000","last_undegraded":"2023-09-23T20:49:22.962768+0000","last_fullsized":"2023-09-23T20:49:22.962768+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:08.785173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.12","version":"620'11667","reported_seq":16150,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T20:49:25.131121+0000","last_change":"2023-09-23T20:49:16.391725+0000","last_active":"2023-09-23T20:49:25.131121+0000","last_peered":"2023-09-23T20:49:25.131121+0000","last_clean":"2023-09-23T20:49:25.131121+0000","last_became_active":"2023-09-23T20:49:10.817393+0000","last_became_peered":"2023-09-23T20:49:10.817393+0000","last_unstale":"2023-09-23T20:49:25.131121+0000","last_undegraded":"2023-09-23T20:49:25.131121+0000","last_fullsized":"2023-09-23T20:49:25.131121+0000","mapping_epoch":607,"log_start":"542'5074","ondisk_log_start":"542'5074","created":584,"last_epoch_clean":608,"parent":"0.0","parent_split_bits":5,"last_scrub":"612'11504","last_scrub_stamp":"2023-09-23T20:49:16.391648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:49:16.391648+0000","objects_scrubbed":594,"log_size":6593,"log_dups_size":0,"ondisk_log_size":6593,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:20:04.882446+0000","scrub_duration":0.18299881800000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":28246016,"num_objects":431,"num_object_clones":0,"num_object_copies":1293,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":431,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":163,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,5],"acting":[6,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.19","version":"90'161","reported_seq":879,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T20:49:24.962440+0000","last_change":"2023-09-23T20:43:46.544750+0000","last_active":"2023-09-23T20:49:24.962440+0000","last_peered":"2023-09-23T20:49:24.962440+0000","last_clean":"2023-09-23T20:49:24.962440+0000","last_became_active":"2023-09-23T20:43:46.544395+0000","last_became_peered":"2023-09-23T20:43:46.544395+0000","last_unstale":"2023-09-23T20:49:24.962440+0000","last_undegraded":"2023-09-23T20:49:24.962440+0000","last_fullsized":"2023-09-23T20:49:24.962440+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:06.487351+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:06.487351+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:56.866168+0000","scrub_duration":0.001964379,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":595,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T20:49:24.962326+0000","last_change":"2023-09-23T20:47:09.197601+0000","last_active":"2023-09-23T20:49:24.962326+0000","last_peered":"2023-09-23T20:49:24.962326+0000","last_clean":"2023-09-23T20:49:24.962326+0000","last_became_active":"2023-09-23T20:47:09.196967+0000","last_became_peered":"2023-09-23T20:47:09.196967+0000","last_unstale":"2023-09-23T20:49:24.962326+0000","last_undegraded":"2023-09-23T20:49:24.962326+0000","last_fullsized":"2023-09-23T20:49:24.962326+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:06:56.826785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":38,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T20:49:21.925294+0000","last_change":"2023-09-23T20:48:48.158376+0000","last_active":"2023-09-23T20:49:21.925294+0000","last_peered":"2023-09-23T20:49:21.925294+0000","last_clean":"2023-09-23T20:49:21.925294+0000","last_became_active":"2023-09-23T20:48:48.158069+0000","last_became_peered":"2023-09-23T20:48:48.158069+0000","last_unstale":"2023-09-23T20:49:21.925294+0000","last_undegraded":"2023-09-23T20:49:21.925294+0000","last_fullsized":"2023-09-23T20:49:21.925294+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:34.658561+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.15","version":"619'11464","reported_seq":13853,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T20:49:23.314753+0000","last_change":"2023-09-23T20:48:49.194473+0000","last_active":"2023-09-23T20:49:23.314753+0000","last_peered":"2023-09-23T20:49:23.314753+0000","last_clean":"2023-09-23T20:49:23.314753+0000","last_became_active":"2023-09-23T20:48:49.173258+0000","last_became_peered":"2023-09-23T20:48:49.173258+0000","last_unstale":"2023-09-23T20:49:23.314753+0000","last_undegraded":"2023-09-23T20:49:23.314753+0000","last_fullsized":"2023-09-23T20:49:23.314753+0000","mapping_epoch":592,"log_start":"542'4806","ondisk_log_start":"542'4806","created":584,"last_epoch_clean":593,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":6658,"log_dups_size":0,"ondisk_log_size":6658,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":31358976,"num_objects":479,"num_object_clones":0,"num_object_copies":1437,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":479,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6114,"num_write_kb":210944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1132,"num_bytes_recovered":74186752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"90'161","reported_seq":694,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T20:49:21.994918+0000","last_change":"2023-09-23T20:41:06.499511+0000","last_active":"2023-09-23T20:49:21.994918+0000","last_peered":"2023-09-23T20:49:21.994918+0000","last_clean":"2023-09-23T20:49:21.994918+0000","last_became_active":"2023-09-23T20:38:05.601874+0000","last_became_peered":"2023-09-23T20:38:05.601874+0000","last_unstale":"2023-09-23T20:49:21.994918+0000","last_undegraded":"2023-09-23T20:49:21.994918+0000","last_fullsized":"2023-09-23T20:49:21.994918+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:41:06.499396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:41:06.499396+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:31:34.059077+0000","scrub_duration":0.0019670310000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":80,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T20:49:22.963243+0000","last_change":"2023-09-23T20:47:09.332980+0000","last_active":"2023-09-23T20:49:22.963243+0000","last_peered":"2023-09-23T20:49:22.963243+0000","last_clean":"2023-09-23T20:49:22.963243+0000","last_became_active":"2023-09-23T20:47:09.230554+0000","last_became_peered":"2023-09-23T20:47:09.230554+0000","last_unstale":"2023-09-23T20:49:22.963243+0000","last_undegraded":"2023-09-23T20:49:22.963243+0000","last_fullsized":"2023-09-23T20:49:22.963243+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:06:21.882823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,1],"acting":[4,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":76,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T20:49:22.963381+0000","last_change":"2023-09-23T20:47:09.186579+0000","last_active":"2023-09-23T20:49:22.963381+0000","last_peered":"2023-09-23T20:49:22.963381+0000","last_clean":"2023-09-23T20:49:22.963381+0000","last_became_active":"2023-09-23T20:47:09.186227+0000","last_became_peered":"2023-09-23T20:47:09.186227+0000","last_unstale":"2023-09-23T20:49:22.963381+0000","last_undegraded":"2023-09-23T20:49:22.963381+0000","last_fullsized":"2023-09-23T20:49:22.963381+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:45:39.485308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.14","version":"617'11244","reported_seq":12468,"reported_epoch":617,"state":"active+remapped+backfilling","last_fresh":"2023-09-23T20:49:21.786701+0000","last_change":"2023-09-23T20:49:21.062907+0000","last_active":"2023-09-23T20:49:21.786701+0000","last_peered":"2023-09-23T20:49:21.786701+0000","last_clean":"2023-09-23T20:49:18.538952+0000","last_became_active":"2023-09-23T20:49:20.988299+0000","last_became_peered":"2023-09-23T20:49:20.988299+0000","last_unstale":"2023-09-23T20:49:21.786701+0000","last_undegraded":"2023-09-23T20:49:21.786701+0000","last_fullsized":"2023-09-23T20:49:21.786701+0000","mapping_epoch":616,"log_start":"541'4547","ondisk_log_start":"541'4547","created":584,"last_epoch_clean":585,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":6697,"log_dups_size":0,"ondisk_log_size":6697,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":30670848,"num_objects":468,"num_object_clones":0,"num_object_copies":1404,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":584,"num_objects_unfound":0,"num_objects_dirty":468,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5995,"num_write_kb":206816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":315,"num_bytes_recovered":20643840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,4],"acting":[0,4,3],"avail_no_missing":["0","3","4"],"object_location_counts":[{"shards":"0,3,4","objects":468}],"blocked_by":[],"up_primary":1,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":80,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T20:49:22.964629+0000","last_change":"2023-09-23T20:47:09.316824+0000","last_active":"2023-09-23T20:49:22.964629+0000","last_peered":"2023-09-23T20:49:22.964629+0000","last_clean":"2023-09-23T20:49:22.964629+0000","last_became_active":"2023-09-23T20:47:09.195100+0000","last_became_peered":"2023-09-23T20:47:09.195100+0000","last_unstale":"2023-09-23T20:49:22.964629+0000","last_undegraded":"2023-09-23T20:49:22.964629+0000","last_fullsized":"2023-09-23T20:49:22.964629+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:29:53.273554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":76,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T20:49:22.963060+0000","last_change":"2023-09-23T20:47:09.181240+0000","last_active":"2023-09-23T20:49:22.963060+0000","last_peered":"2023-09-23T20:49:22.963060+0000","last_clean":"2023-09-23T20:49:22.963060+0000","last_became_active":"2023-09-23T20:47:09.181022+0000","last_became_peered":"2023-09-23T20:47:09.181022+0000","last_unstale":"2023-09-23T20:49:22.963060+0000","last_undegraded":"2023-09-23T20:49:22.963060+0000","last_fullsized":"2023-09-23T20:49:22.963060+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:38:59.191781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.17","version":"620'11346","reported_seq":11487,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T20:49:25.163385+0000","last_change":"2023-09-23T20:49:15.383694+0000","last_active":"2023-09-23T20:49:25.163385+0000","last_peered":"2023-09-23T20:49:25.163385+0000","last_clean":"2023-09-23T20:49:25.163385+0000","last_became_active":"2023-09-23T20:48:40.090791+0000","last_became_peered":"2023-09-23T20:48:40.090791+0000","last_unstale":"2023-09-23T20:49:25.163385+0000","last_undegraded":"2023-09-23T20:49:25.163385+0000","last_fullsized":"2023-09-23T20:49:25.163385+0000","mapping_epoch":584,"log_start":"542'4863","ondisk_log_start":"542'4863","created":584,"last_epoch_clean":585,"parent":"0.0","parent_split_bits":5,"last_scrub":"612'11124","last_scrub_stamp":"2023-09-23T20:49:15.383619+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:49:15.383619+0000","objects_scrubbed":632,"log_size":6483,"log_dups_size":0,"ondisk_log_size":6483,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:28:57.128307+0000","scrub_duration":0.21319408200000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26804224,"num_objects":409,"num_object_clones":0,"num_object_copies":1227,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":409,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":223,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1c","version":"90'161","reported_seq":875,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T20:49:22.979849+0000","last_change":"2023-09-23T20:38:05.591992+0000","last_active":"2023-09-23T20:49:22.979849+0000","last_peered":"2023-09-23T20:49:22.979849+0000","last_clean":"2023-09-23T20:49:22.979849+0000","last_became_active":"2023-09-23T20:38:05.591701+0000","last_became_peered":"2023-09-23T20:38:05.591701+0000","last_unstale":"2023-09-23T20:49:22.979849+0000","last_undegraded":"2023-09-23T20:49:22.979849+0000","last_fullsized":"2023-09-23T20:49:22.979849+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:02.384133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:02.384133+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:41.722542+0000","scrub_duration":0.001769098,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":461,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T20:49:21.995121+0000","last_change":"2023-09-23T20:48:48.158348+0000","last_active":"2023-09-23T20:49:21.995121+0000","last_peered":"2023-09-23T20:49:21.995121+0000","last_clean":"2023-09-23T20:49:21.995121+0000","last_became_active":"2023-09-23T20:48:48.157982+0000","last_became_peered":"2023-09-23T20:48:48.157982+0000","last_unstale":"2023-09-23T20:49:21.995121+0000","last_undegraded":"2023-09-23T20:49:21.995121+0000","last_fullsized":"2023-09-23T20:49:21.995121+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:03:48.078707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":427,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T20:49:21.994356+0000","last_change":"2023-09-23T20:48:48.159781+0000","last_active":"2023-09-23T20:49:21.994356+0000","last_peered":"2023-09-23T20:49:21.994356+0000","last_clean":"2023-09-23T20:49:21.994356+0000","last_became_active":"2023-09-23T20:48:48.159505+0000","last_became_peered":"2023-09-23T20:48:48.159505+0000","last_unstale":"2023-09-23T20:49:21.994356+0000","last_undegraded":"2023-09-23T20:49:21.994356+0000","last_fullsized":"2023-09-23T20:49:21.994356+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:25:18.100645+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.16","version":"619'11448","reported_seq":13388,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T20:49:23.675224+0000","last_change":"2023-09-23T20:49:14.743002+0000","last_active":"2023-09-23T20:49:23.675224+0000","last_peered":"2023-09-23T20:49:23.675224+0000","last_clean":"2023-09-23T20:49:23.675224+0000","last_became_active":"2023-09-23T20:49:14.705461+0000","last_became_peered":"2023-09-23T20:49:14.705461+0000","last_unstale":"2023-09-23T20:49:23.675224+0000","last_undegraded":"2023-09-23T20:49:23.675224+0000","last_fullsized":"2023-09-23T20:49:23.675224+0000","mapping_epoch":611,"log_start":"541'4690","ondisk_log_start":"541'4690","created":584,"last_epoch_clean":612,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:45:38.312169+0000","last_clean_scrub_stamp":"2023-09-23T20:45:38.312169+0000","objects_scrubbed":0,"log_size":6758,"log_dups_size":0,"ondisk_log_size":6758,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":28475392,"num_objects":435,"num_object_clones":0,"num_object_copies":1305,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":435,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6128,"num_write_kb":209984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":751,"num_bytes_recovered":49184768,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1d","version":"90'161","reported_seq":748,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T20:49:21.927180+0000","last_change":"2023-09-23T20:47:11.621758+0000","last_active":"2023-09-23T20:49:21.927180+0000","last_peered":"2023-09-23T20:49:21.927180+0000","last_clean":"2023-09-23T20:49:21.927180+0000","last_became_active":"2023-09-23T20:47:11.621345+0000","last_became_peered":"2023-09-23T20:47:11.621345+0000","last_unstale":"2023-09-23T20:49:21.927180+0000","last_undegraded":"2023-09-23T20:49:21.927180+0000","last_fullsized":"2023-09-23T20:49:21.927180+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:01.504973+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:01.504973+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:31:17.112393+0000","scrub_duration":0.00202344,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":685,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T20:49:24.962606+0000","last_change":"2023-09-23T20:48:48.164011+0000","last_active":"2023-09-23T20:49:24.962606+0000","last_peered":"2023-09-23T20:49:24.962606+0000","last_clean":"2023-09-23T20:49:24.962606+0000","last_became_active":"2023-09-23T20:48:48.150690+0000","last_became_peered":"2023-09-23T20:48:48.150690+0000","last_unstale":"2023-09-23T20:49:24.962606+0000","last_undegraded":"2023-09-23T20:49:24.962606+0000","last_fullsized":"2023-09-23T20:49:24.962606+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:35:33.111175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":4432,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":86,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T20:49:22.962563+0000","last_change":"2023-09-23T20:48:23.917302+0000","last_active":"2023-09-23T20:49:22.962563+0000","last_peered":"2023-09-23T20:49:22.962563+0000","last_clean":"2023-09-23T20:49:22.962563+0000","last_became_active":"2023-09-23T20:48:23.916956+0000","last_became_peered":"2023-09-23T20:48:23.916956+0000","last_unstale":"2023-09-23T20:49:22.962563+0000","last_undegraded":"2023-09-23T20:49:22.962563+0000","last_fullsized":"2023-09-23T20:49:22.962563+0000","mapping_epoch":577,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":578,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:42:27.989294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":26,"stat_sum":{"num_bytes":869072920,"num_objects":13261,"num_object_clones":0,"num_object_copies":39783,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":584,"num_objects_unfound":0,"num_objects_dirty":13261,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":159805,"num_write_kb":5419905,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16647,"num_bytes_recovered":1090584576,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3015913472,"data_stored":3015901256,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":176350,"ondisk_log_size":176350,"up":78,"acting":78,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":109,"num_bytes_recovered":15220,"num_keys_recovered":80,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26691,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":661927,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":59,"num_read_kb":110,"num_write":28,"num_write_kb":291,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":477366,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5983,"num_read_kb":13200,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":135,"num_bytes_recovered":25165824,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6082,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201736,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":941052,"kb_used_data":333236,"kb_used_omap":0,"kb_used_meta":607808,"kb_avail":92804100,"statfs":{"total":95995035648,"available":95031398400,"internally_reserved":0,"allocated":341233664,"data_stored":339534314,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":622395392},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:49:07 2023","interfaces":[{"interface":"back","average":{"1min":1.5669999999999999,"5min":1.774,"15min":2.0369999999999999},"min":{"1min":0.23400000000000001,"5min":0.224,"15min":0.224},"max":{"1min":5.1360000000000001,"5min":10.975,"15min":18.315000000000001},"last":0.41999999999999998},{"interface":"front","average":{"1min":1.4259999999999999,"5min":1.8819999999999999,"15min":1.865},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":8.0530000000000008,"5min":12.986000000000001,"15min":19.355},"last":0.32100000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:49:16 2023","interfaces":[{"interface":"back","average":{"1min":1.4259999999999999,"5min":1.4199999999999999,"15min":1.9550000000000001},"min":{"1min":0.28499999999999998,"5min":0.25900000000000001,"15min":0.24399999999999999},"max":{"1min":7.3789999999999996,"5min":14.249000000000001,"15min":49.965000000000003},"last":0.33200000000000002},{"interface":"front","average":{"1min":1.105,"5min":2.008,"15min":2.4319999999999999},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.26700000000000002},"max":{"1min":4.0739999999999998,"5min":11.276,"15min":53.451999999999998},"last":0.378}]},{"osd":2,"last update":"Sat Sep 23 20:49:07 2023","interfaces":[{"interface":"back","average":{"1min":1.847,"5min":1.885,"15min":2.121},"min":{"1min":0.31,"5min":0.28499999999999998,"15min":0.27800000000000002},"max":{"1min":7.9219999999999997,"5min":12.933999999999999,"15min":16.965},"last":0.35399999999999998},{"interface":"front","average":{"1min":2.0209999999999999,"5min":1.982,"15min":2.2160000000000002},"min":{"1min":0.36599999999999999,"5min":0.31,"15min":0.30299999999999999},"max":{"1min":7.3019999999999996,"5min":13.5,"15min":23.414000000000001},"last":0.495}]},{"osd":3,"last update":"Sat Sep 23 20:48:38 2023","interfaces":[{"interface":"back","average":{"1min":2.4140000000000001,"5min":1.964,"15min":2.226},"min":{"1min":0.34399999999999997,"5min":0.33800000000000002,"15min":0.311},"max":{"1min":7.2880000000000003,"5min":15.115,"15min":23.352},"last":0.52700000000000002},{"interface":"front","average":{"1min":2.3180000000000001,"5min":2.25,"15min":2.4239999999999999},"min":{"1min":0.35399999999999998,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":6.5090000000000003,"5min":15.144,"15min":20.969999999999999},"last":0.48399999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:48:38 2023","interfaces":[{"interface":"back","average":{"1min":1.583,"5min":1.458,"15min":1.2350000000000001},"min":{"1min":0.26500000000000001,"5min":0.254,"15min":0.23200000000000001},"max":{"1min":7.4109999999999996,"5min":12.260999999999999,"15min":18.364999999999998},"last":0.36599999999999999},{"interface":"front","average":{"1min":1.1479999999999999,"5min":1.6659999999999999,"15min":1.45},"min":{"1min":0.34599999999999997,"5min":0.247,"15min":0.247},"max":{"1min":4.1529999999999996,"5min":7.2469999999999999,"15min":23.617000000000001},"last":0.27500000000000002}]},{"osd":5,"last update":"Sat Sep 23 20:48:38 2023","interfaces":[{"interface":"back","average":{"1min":1.548,"5min":1.48,"15min":1.2709999999999999},"min":{"1min":0.28100000000000003,"5min":0.27200000000000002,"15min":0.26600000000000001},"max":{"1min":7.2270000000000003,"5min":15.005000000000001,"15min":15.005000000000001},"last":0.30599999999999999},{"interface":"front","average":{"1min":1.1890000000000001,"5min":1.6060000000000001,"15min":1.3899999999999999},"min":{"1min":0.32200000000000001,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":7.335,"5min":12.212,"15min":18.390000000000001},"last":0.438}]},{"osd":6,"last update":"Sat Sep 23 20:48:38 2023","interfaces":[{"interface":"back","average":{"1min":1.6879999999999999,"5min":2.052,"15min":1.544},"min":{"1min":0.379,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":5.6070000000000002,"5min":15.038,"15min":18.417000000000002},"last":0.39400000000000002},{"interface":"front","average":{"1min":1.3100000000000001,"5min":1.9039999999999999,"15min":1.4399999999999999},"min":{"1min":0.378,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":4.3710000000000004,"5min":15.369999999999999,"15min":15.369999999999999},"last":0.40400000000000003}]}]},{"osd":6,"up_from":47,"seq":201863463375,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":903936,"kb_used_data":388024,"kb_used_omap":0,"kb_used_meta":515904,"kb_avail":92841216,"statfs":{"total":95995035648,"available":95069405184,"internally_reserved":0,"allocated":397336576,"data_stored":395640730,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":528285696},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:49:05 2023","interfaces":[{"interface":"back","average":{"1min":2.5409999999999999,"5min":2.0819999999999999,"15min":1.978},"min":{"1min":0.24399999999999999,"5min":0.219,"15min":0.219},"max":{"1min":12.442,"5min":21.018000000000001,"15min":34.143999999999998},"last":0.48599999999999999},{"interface":"front","average":{"1min":2.2919999999999998,"5min":2.2200000000000002,"15min":2.2360000000000002},"min":{"1min":0.30299999999999999,"5min":0.255,"15min":0.22700000000000001},"max":{"1min":11.557,"5min":21.082000000000001,"15min":60.328000000000003},"last":0.45100000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:49:20 2023","interfaces":[{"interface":"back","average":{"1min":1.6499999999999999,"5min":1.895,"15min":2.052},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.249},"max":{"1min":7.7409999999999997,"5min":8.9350000000000005,"15min":64.274000000000001},"last":0.39400000000000002},{"interface":"front","average":{"1min":0.93999999999999995,"5min":1.5820000000000001,"15min":1.7989999999999999},"min":{"1min":0.312,"5min":0.29099999999999998,"15min":0.28000000000000003},"max":{"1min":3.0139999999999998,"5min":8.8070000000000004,"15min":34.490000000000002},"last":0.504}]},{"osd":2,"last update":"Sat Sep 23 20:49:05 2023","interfaces":[{"interface":"back","average":{"1min":3.6360000000000001,"5min":2.6520000000000001,"15min":2.3010000000000002},"min":{"1min":0.36199999999999999,"5min":0.29699999999999999,"15min":0.27700000000000002},"max":{"1min":9.0730000000000004,"5min":22.591000000000001,"15min":34.317999999999998},"last":0.436},{"interface":"front","average":{"1min":2.8170000000000002,"5min":2.198,"15min":2.0779999999999998},"min":{"1min":0.42999999999999999,"5min":0.33900000000000002,"15min":0.30399999999999999},"max":{"1min":12.798,"5min":22.614999999999998,"15min":34.292999999999999},"last":0.46000000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:48:56 2023","interfaces":[{"interface":"back","average":{"1min":2.5659999999999998,"5min":2.0920000000000001,"15min":1.9650000000000001},"min":{"1min":0.46999999999999997,"5min":0.307,"15min":0.28399999999999997},"max":{"1min":7.9640000000000004,"5min":11.664,"15min":11.664},"last":0.32200000000000001},{"interface":"front","average":{"1min":3.1840000000000002,"5min":2.335,"15min":2.1970000000000001},"min":{"1min":0.50600000000000001,"5min":0.32700000000000001,"15min":0.30499999999999999},"max":{"1min":11.6,"5min":11.6,"15min":12.875999999999999},"last":0.49099999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:49:20 2023","interfaces":[{"interface":"back","average":{"1min":0.96499999999999997,"5min":1.0680000000000001,"15min":0.96999999999999997},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.25600000000000001},"max":{"1min":2.831,"5min":10.183999999999999,"15min":23.904},"last":0.29999999999999999},{"interface":"front","average":{"1min":2.4569999999999999,"5min":1.6160000000000001,"15min":1.27},"min":{"1min":0.36299999999999999,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":11.999000000000001,"5min":11.999000000000001,"15min":11.999000000000001},"last":0.40899999999999997}]},{"osd":5,"last update":"Sat Sep 23 20:49:20 2023","interfaces":[{"interface":"back","average":{"1min":0.79800000000000004,"5min":1.2869999999999999,"15min":1.0029999999999999},"min":{"1min":0.35099999999999998,"5min":0.26400000000000001,"15min":0.253},"max":{"1min":2.371,"5min":8.0440000000000005,"15min":8.0440000000000005},"last":0.36099999999999999},{"interface":"front","average":{"1min":1.843,"5min":1.607,"15min":1.3200000000000001},"min":{"1min":0.373,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":8.2769999999999992,"5min":9.3559999999999999,"15min":9.3559999999999999},"last":0.373}]},{"osd":7,"last update":"Sat Sep 23 20:48:56 2023","interfaces":[{"interface":"back","average":{"1min":1.2729999999999999,"5min":1.2549999999999999,"15min":0.997},"min":{"1min":0.42899999999999999,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":8.0719999999999992,"5min":8.9309999999999992,"15min":8.9309999999999992},"last":0.42099999999999999},{"interface":"front","average":{"1min":10.34,"5min":3.6499999999999999,"15min":1.9890000000000001},"min":{"1min":0.47099999999999997,"5min":0.35999999999999999,"15min":0.34499999999999997},"max":{"1min":159.91999999999999,"5min":159.91999999999999,"15min":159.91999999999999},"last":0.47399999999999998}]}]},{"osd":1,"up_from":266,"seq":1142461300973,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1068284,"kb_used_data":291444,"kb_used_omap":8,"kb_used_meta":776759,"kb_avail":92676868,"statfs":{"total":95995035648,"available":94901112832,"internally_reserved":0,"allocated":298438656,"data_stored":295346882,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8206,"internal_metadata":795402226},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:48:24 2023","interfaces":[{"interface":"back","average":{"1min":2.9710000000000001,"5min":2.4969999999999999,"15min":2.3559999999999999},"min":{"1min":0.27700000000000002,"5min":0.20799999999999999,"15min":0.20000000000000001},"max":{"1min":9.4730000000000008,"5min":15.532999999999999,"15min":16.292000000000002},"last":0.52900000000000003},{"interface":"front","average":{"1min":3.1469999999999998,"5min":2.9060000000000001,"15min":2.9430000000000001},"min":{"1min":0.35899999999999999,"5min":0.22600000000000001,"15min":0.19900000000000001},"max":{"1min":8.1839999999999993,"5min":20.346,"15min":26.588000000000001},"last":0.39600000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:48:24 2023","interfaces":[{"interface":"back","average":{"1min":3.1850000000000001,"5min":2.8340000000000001,"15min":3.1429999999999998},"min":{"1min":0.249,"5min":0.23699999999999999,"15min":0.216},"max":{"1min":9.5779999999999994,"5min":19.026,"15min":32.255000000000003},"last":0.35299999999999998},{"interface":"front","average":{"1min":3.5030000000000001,"5min":2.423,"15min":2.3769999999999998},"min":{"1min":0.315,"5min":0.26000000000000001,"15min":0.20899999999999999},"max":{"1min":10.567,"5min":10.567,"15min":13.404999999999999},"last":0.376}]},{"osd":3,"last update":"Sat Sep 23 20:48:24 2023","interfaces":[{"interface":"back","average":{"1min":3.294,"5min":2.2719999999999998,"15min":2.4780000000000002},"min":{"1min":0.373,"5min":0.28799999999999998,"15min":0.247},"max":{"1min":10.135999999999999,"5min":17.366,"15min":17.366},"last":0.33100000000000002},{"interface":"front","average":{"1min":3.2170000000000001,"5min":2.8490000000000002,"15min":2.7949999999999999},"min":{"1min":0.26300000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":9.8339999999999996,"5min":11.534000000000001,"15min":19.315000000000001},"last":0.73599999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:48:24 2023","interfaces":[{"interface":"back","average":{"1min":2.7909999999999999,"5min":2.3769999999999998,"15min":2.4079999999999999},"min":{"1min":0.34799999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":9.5359999999999996,"5min":11.791,"15min":24.327999999999999},"last":0.47599999999999998},{"interface":"front","average":{"1min":2.5600000000000001,"5min":1.7949999999999999,"15min":1.7749999999999999},"min":{"1min":0.36299999999999999,"5min":0.34699999999999998,"15min":0.311},"max":{"1min":9.8249999999999993,"5min":10.468999999999999,"15min":20.835999999999999},"last":0.57799999999999996}]},{"osd":5,"last update":"Sat Sep 23 20:48:24 2023","interfaces":[{"interface":"back","average":{"1min":2.4470000000000001,"5min":2.3889999999999998,"15min":2.3660000000000001},"min":{"1min":0.41399999999999998,"5min":0.308,"15min":0.308},"max":{"1min":8.1210000000000004,"5min":14.366,"15min":33.401000000000003},"last":0.65200000000000002},{"interface":"front","average":{"1min":2.589,"5min":2.548,"15min":2.633},"min":{"1min":0.39700000000000002,"5min":0.33100000000000002,"15min":0.32500000000000001},"max":{"1min":9.6140000000000008,"5min":11.750999999999999,"15min":33.417000000000002},"last":0.54700000000000004}]},{"osd":6,"last update":"Sat Sep 23 20:48:24 2023","interfaces":[{"interface":"back","average":{"1min":2.5049999999999999,"5min":2.3650000000000002,"15min":2.2570000000000001},"min":{"1min":0.48799999999999999,"5min":0.316,"15min":0.316},"max":{"1min":8.0960000000000001,"5min":14.414999999999999,"15min":14.414999999999999},"last":0.70399999999999996},{"interface":"front","average":{"1min":3.2650000000000001,"5min":3.004,"15min":2.6739999999999999},"min":{"1min":0.432,"5min":0.38500000000000001,"15min":0.317},"max":{"1min":9.6639999999999997,"5min":11.832000000000001,"15min":24.376000000000001},"last":0.628}]},{"osd":7,"last update":"Sat Sep 23 20:48:24 2023","interfaces":[{"interface":"back","average":{"1min":2.8330000000000002,"5min":2.5219999999999998,"15min":2.4249999999999998},"min":{"1min":0.46899999999999997,"5min":0.377,"15min":0.37},"max":{"1min":8.1590000000000007,"5min":10.010999999999999,"15min":18.437000000000001},"last":0.60099999999999998},{"interface":"front","average":{"1min":3.5739999999999998,"5min":2.871,"15min":2.714},"min":{"1min":0.47299999999999998,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":9.8369999999999997,"5min":11.864000000000001,"15min":24.524000000000001},"last":0.52000000000000002}]}]},{"osd":0,"up_from":94,"seq":403726926176,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1449268,"kb_used_data":486060,"kb_used_omap":7,"kb_used_meta":963192,"kb_avail":92295884,"statfs":{"total":95995035648,"available":94510985216,"internally_reserved":0,"allocated":497725440,"data_stored":494772095,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":986308625},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:48:32 2023","interfaces":[{"interface":"back","average":{"1min":4.1180000000000003,"5min":3.851,"15min":3.0579999999999998},"min":{"1min":0.34799999999999998,"5min":0.184,"15min":0.17199999999999999},"max":{"1min":9.6449999999999996,"5min":152.387,"15min":152.387},"last":0.51600000000000001},{"interface":"front","average":{"1min":3.7109999999999999,"5min":3.9390000000000001,"15min":3.2149999999999999},"min":{"1min":0.218,"5min":0.218,"15min":0.20399999999999999},"max":{"1min":12.317,"5min":152.84100000000001,"15min":152.84100000000001},"last":0.42699999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:49:07 2023","interfaces":[{"interface":"back","average":{"1min":1.7829999999999999,"5min":2.7879999999999998,"15min":2.867},"min":{"1min":0.252,"5min":0.23499999999999999,"15min":0.218},"max":{"1min":7.008,"5min":15.108000000000001,"15min":62.726999999999997},"last":0.61599999999999999},{"interface":"front","average":{"1min":1.913,"5min":2.706,"15min":2.7930000000000001},"min":{"1min":0.219,"5min":0.217,"15min":0.217},"max":{"1min":9.5609999999999999,"5min":17.013000000000002,"15min":62.689},"last":1.9179999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:48:40 2023","interfaces":[{"interface":"back","average":{"1min":3.8650000000000002,"5min":3.6709999999999998,"15min":3.2229999999999999},"min":{"1min":0.36499999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":14.494,"5min":55.017000000000003,"15min":55.017000000000003},"last":0.52800000000000002},{"interface":"front","average":{"1min":3.6339999999999999,"5min":3.161,"15min":3.234},"min":{"1min":0.38,"5min":0.23799999999999999,"15min":0.21199999999999999},"max":{"1min":11.547000000000001,"5min":55.665999999999997,"15min":55.665999999999997},"last":8.2439999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:49:07 2023","interfaces":[{"interface":"back","average":{"1min":2.1120000000000001,"5min":2.1589999999999998,"15min":2.0859999999999999},"min":{"1min":0.308,"5min":0.29299999999999998,"15min":0.28899999999999998},"max":{"1min":9.5779999999999994,"5min":11.75,"15min":11.75},"last":0.58499999999999996},{"interface":"front","average":{"1min":1.77,"5min":2.133,"15min":1.9710000000000001},"min":{"1min":0.41599999999999998,"5min":0.31,"15min":0.27400000000000002},"max":{"1min":6.1050000000000004,"5min":12.356999999999999,"15min":12.356999999999999},"last":0.48899999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:49:07 2023","interfaces":[{"interface":"back","average":{"1min":1.8520000000000001,"5min":2.052,"15min":2.0880000000000001},"min":{"1min":0.38300000000000001,"5min":0.308,"15min":0.26500000000000001},"max":{"1min":6.1210000000000004,"5min":9.3339999999999996,"15min":11.497999999999999},"last":0.46300000000000002},{"interface":"front","average":{"1min":1.5569999999999999,"5min":2.137,"15min":2.2959999999999998},"min":{"1min":0.36899999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":4.9640000000000004,"5min":12.601000000000001,"15min":26.228000000000002},"last":0.55700000000000005}]},{"osd":6,"last update":"Sat Sep 23 20:49:07 2023","interfaces":[{"interface":"back","average":{"1min":2.1629999999999998,"5min":2.3029999999999999,"15min":2.2629999999999999},"min":{"1min":0.45200000000000001,"5min":0.374,"15min":0.34499999999999997},"max":{"1min":9.0570000000000004,"5min":9.2750000000000004,"15min":24.056000000000001},"last":2.0659999999999998},{"interface":"front","average":{"1min":1.9359999999999999,"5min":2.327,"15min":2.2010000000000001},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":9.0380000000000003,"5min":12.698,"15min":26.268000000000001},"last":1.8029999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:49:07 2023","interfaces":[{"interface":"back","average":{"1min":1.607,"5min":2.512,"15min":2.3839999999999999},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":5.0369999999999999,"5min":16.553999999999998,"15min":26.177},"last":0.63800000000000001},{"interface":"front","average":{"1min":1.8420000000000001,"5min":2.4529999999999998,"15min":2.4009999999999998},"min":{"1min":0.41699999999999998,"5min":0.40200000000000002,"15min":0.34200000000000003},"max":{"1min":7.8559999999999999,"5min":16.649000000000001,"15min":16.649000000000001},"last":0.79200000000000004}]}]},{"osd":2,"up_from":104,"seq":446676599123,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1289632,"kb_used_data":435736,"kb_used_omap":11,"kb_used_meta":853876,"kb_avail":92455520,"statfs":{"total":95995035648,"available":94674452480,"internally_reserved":0,"allocated":446193664,"data_stored":443292480,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11878,"internal_metadata":874369434},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:49:11 2023","interfaces":[{"interface":"back","average":{"1min":2.2679999999999998,"5min":1.8140000000000001,"15min":2.036},"min":{"1min":0.19500000000000001,"5min":0.19500000000000001,"15min":0.17299999999999999},"max":{"1min":20.48,"5min":20.48,"15min":28.234999999999999},"last":0.54400000000000004},{"interface":"front","average":{"1min":1.8919999999999999,"5min":1.9139999999999999,"15min":2.2410000000000001},"min":{"1min":0.25700000000000001,"5min":0.245,"15min":0.19},"max":{"1min":6.7220000000000004,"5min":15.342000000000001,"15min":16.678999999999998},"last":1.0549999999999999}]},{"osd":1,"last update":"Sat Sep 23 20:49:11 2023","interfaces":[{"interface":"back","average":{"1min":1.409,"5min":2.1480000000000001,"15min":2.3889999999999998},"min":{"1min":0.28199999999999997,"5min":0.219,"15min":0.219},"max":{"1min":10.324,"5min":21.690000000000001,"15min":46.405000000000001},"last":0.35499999999999998},{"interface":"front","average":{"1min":2.3119999999999998,"5min":1.9550000000000001,"15min":2.4340000000000002},"min":{"1min":0.29499999999999998,"5min":0.22,"15min":0.22},"max":{"1min":9.0690000000000008,"5min":13.215999999999999,"15min":46.459000000000003},"last":0.82099999999999995}]},{"osd":3,"last update":"Sat Sep 23 20:48:40 2023","interfaces":[{"interface":"back","average":{"1min":2.1509999999999998,"5min":2.516,"15min":2.605},"min":{"1min":0.311,"5min":0.253,"15min":0.253},"max":{"1min":10.051,"5min":30.283000000000001,"15min":30.283000000000001},"last":0.77000000000000002},{"interface":"front","average":{"1min":2.6840000000000002,"5min":1.8320000000000001,"15min":2.0499999999999998},"min":{"1min":0.33000000000000002,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":10.221,"5min":30.834,"15min":30.834},"last":3.6000000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:49:11 2023","interfaces":[{"interface":"back","average":{"1min":1.623,"5min":1.9299999999999999,"15min":1.8420000000000001},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.27500000000000002},"max":{"1min":5.556,"5min":11.141999999999999,"15min":12.025},"last":0.78700000000000003},{"interface":"front","average":{"1min":2.069,"5min":2.0099999999999998,"15min":1.982},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.30499999999999999},"max":{"1min":12.662000000000001,"5min":12.662000000000001,"15min":19.440999999999999},"last":0.442}]},{"osd":5,"last update":"Sat Sep 23 20:49:11 2023","interfaces":[{"interface":"back","average":{"1min":1.9219999999999999,"5min":1.7150000000000001,"15min":1.8680000000000001},"min":{"1min":0.38400000000000001,"5min":0.33200000000000002,"15min":0.311},"max":{"1min":6.5460000000000003,"5min":11.183,"15min":12.07},"last":0.83799999999999997},{"interface":"front","average":{"1min":1.863,"5min":1.8999999999999999,"15min":1.9870000000000001},"min":{"1min":0.373,"5min":0.373,"15min":0.35299999999999998},"max":{"1min":10.831,"5min":11.061999999999999,"15min":19.513000000000002},"last":0.48899999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:49:11 2023","interfaces":[{"interface":"back","average":{"1min":1.8740000000000001,"5min":2.0760000000000001,"15min":2.0830000000000002},"min":{"1min":0.40699999999999997,"5min":0.34799999999999998,"15min":0.33500000000000002},"max":{"1min":10.791,"5min":11.031000000000001,"15min":19.486000000000001},"last":0.40799999999999997},{"interface":"front","average":{"1min":1.829,"5min":1.609,"15min":1.6020000000000001},"min":{"1min":0.40699999999999997,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":4.9589999999999996,"5min":8.5920000000000005,"15min":11.747},"last":0.58099999999999996}]},{"osd":7,"last update":"Sat Sep 23 20:49:11 2023","interfaces":[{"interface":"back","average":{"1min":3.7509999999999999,"5min":1.978,"15min":1.6950000000000001},"min":{"1min":0.378,"5min":0.35399999999999998,"15min":0.34699999999999998},"max":{"1min":20.530000000000001,"5min":20.530000000000001,"15min":20.530000000000001},"last":0.61599999999999999},{"interface":"front","average":{"1min":1.752,"5min":2.1160000000000001,"15min":2.121},"min":{"1min":0.36899999999999999,"5min":0.35799999999999998,"15min":0.34899999999999998},"max":{"1min":5.3259999999999996,"5min":11.218,"15min":12.099},"last":0.877}]}]},{"osd":3,"up_from":109,"seq":468151435597,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1370904,"kb_used_data":308112,"kb_used_omap":4,"kb_used_meta":1062779,"kb_avail":92374248,"statfs":{"total":95995035648,"available":94591229952,"internally_reserved":0,"allocated":315506688,"data_stored":312619057,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4514,"internal_metadata":1088286302},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:48:48 2023","interfaces":[{"interface":"back","average":{"1min":1.4750000000000001,"5min":2.3799999999999999,"15min":2.4790000000000001},"min":{"1min":0.252,"5min":0.217,"15min":0.20899999999999999},"max":{"1min":5.2110000000000003,"5min":21.792999999999999,"15min":22.32},"last":1.262},{"interface":"front","average":{"1min":1.454,"5min":2.3340000000000001,"15min":2.5289999999999999},"min":{"1min":0.249,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":5.4530000000000003,"5min":18.579999999999998,"15min":22.805},"last":0.42699999999999999}]},{"osd":1,"last update":"Sat Sep 23 20:49:21 2023","interfaces":[{"interface":"back","average":{"1min":0.91200000000000003,"5min":1.7889999999999999,"15min":1.897},"min":{"1min":0.21299999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":4.266,"5min":15.177,"15min":17.529},"last":0.21299999999999999},{"interface":"front","average":{"1min":0.93000000000000005,"5min":1.921,"15min":2.137},"min":{"1min":0.28299999999999997,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":4.5339999999999998,"5min":11.380000000000001,"15min":14.19},"last":0.85199999999999998}]},{"osd":2,"last update":"Sat Sep 23 20:48:48 2023","interfaces":[{"interface":"back","average":{"1min":1.669,"5min":2.403,"15min":2.8650000000000002},"min":{"1min":0.27900000000000003,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":5.4210000000000003,"5min":12.473000000000001,"15min":68.206999999999994},"last":0.44900000000000001},{"interface":"front","average":{"1min":2.7810000000000001,"5min":2.1200000000000001,"15min":2.5659999999999998},"min":{"1min":0.374,"5min":0.251,"15min":0.245},"max":{"1min":16.559000000000001,"5min":17.919,"15min":67.736999999999995},"last":0.35599999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:48:48 2023","interfaces":[{"interface":"back","average":{"1min":1.8200000000000001,"5min":1.744,"15min":1.708},"min":{"1min":0.435,"5min":0.35499999999999998,"15min":0.30499999999999999},"max":{"1min":5.8029999999999999,"5min":9.0259999999999998,"15min":22.157},"last":1.2330000000000001},{"interface":"front","average":{"1min":3.2530000000000001,"5min":1.8759999999999999,"15min":2.1869999999999998},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.29399999999999998},"max":{"1min":16.529,"5min":16.529,"15min":16.529},"last":1.3129999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:48:48 2023","interfaces":[{"interface":"back","average":{"1min":1.734,"5min":1.849,"15min":1.921},"min":{"1min":0.49299999999999999,"5min":0.33800000000000002,"15min":0.32300000000000001},"max":{"1min":6.7210000000000001,"5min":8.9849999999999994,"15min":22.289999999999999},"last":1.3340000000000001},{"interface":"front","average":{"1min":2.25,"5min":2.0470000000000002,"15min":1.831},"min":{"1min":0.43099999999999999,"5min":0.34100000000000003,"15min":0.32000000000000001},"max":{"1min":6.008,"5min":8.6129999999999995,"15min":12.952},"last":0.39600000000000002}]},{"osd":6,"last update":"Sat Sep 23 20:48:48 2023","interfaces":[{"interface":"back","average":{"1min":2.6699999999999999,"5min":2.4660000000000002,"15min":2.036},"min":{"1min":0.54200000000000004,"5min":0.36599999999999999,"15min":0.34300000000000003},"max":{"1min":7.4359999999999999,"5min":8.6790000000000003,"15min":12.917},"last":0.47899999999999998},{"interface":"front","average":{"1min":2.633,"5min":2.0270000000000001,"15min":2.2290000000000001},"min":{"1min":0.39400000000000002,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":16.497,"5min":16.497,"15min":16.497},"last":0.51100000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:48:48 2023","interfaces":[{"interface":"back","average":{"1min":3.173,"5min":2.1219999999999999,"15min":2.4300000000000002},"min":{"1min":0.48199999999999998,"5min":0.42399999999999999,"15min":0.39900000000000002},"max":{"1min":16.425000000000001,"5min":16.425000000000001,"15min":16.425000000000001},"last":1.462},{"interface":"front","average":{"1min":3.3330000000000002,"5min":2.3399999999999999,"15min":2.0899999999999999},"min":{"1min":0.56100000000000005,"5min":0.46300000000000002,"15min":0.40999999999999998},"max":{"1min":11.891999999999999,"5min":11.891999999999999,"15min":22.202999999999999},"last":1.296}]}]},{"osd":4,"up_from":30,"seq":128849019351,"num_pgs":58,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1011008,"kb_used_data":453560,"kb_used_omap":0,"kb_used_meta":557440,"kb_avail":92734144,"statfs":{"total":95995035648,"available":94959763456,"internally_reserved":0,"allocated":464445440,"data_stored":462749328,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":570818560},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:49:04 2023","interfaces":[{"interface":"back","average":{"1min":5.3239999999999998,"5min":2.403,"15min":2.0779999999999998},"min":{"1min":0.219,"5min":0.219,"15min":0.219},"max":{"1min":73.272000000000006,"5min":73.272000000000006,"15min":73.272000000000006},"last":3.351},{"interface":"front","average":{"1min":3.0369999999999999,"5min":2.1890000000000001,"15min":2.1259999999999999},"min":{"1min":0.28299999999999997,"5min":0.25800000000000001,"15min":0.246},"max":{"1min":29.754000000000001,"5min":29.754000000000001,"15min":29.754000000000001},"last":7.2359999999999998}]},{"osd":1,"last update":"Sat Sep 23 20:49:17 2023","interfaces":[{"interface":"back","average":{"1min":2.448,"5min":1.9650000000000001,"15min":1.7629999999999999},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.25900000000000001},"max":{"1min":12.343,"5min":12.343,"15min":18.190999999999999},"last":0.42599999999999999},{"interface":"front","average":{"1min":3.1899999999999999,"5min":1.9830000000000001,"15min":1.677},"min":{"1min":0.376,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":12.231999999999999,"5min":12.231999999999999,"15min":18.173999999999999},"last":3.198}]},{"osd":2,"last update":"Sat Sep 23 20:49:17 2023","interfaces":[{"interface":"back","average":{"1min":2.6840000000000002,"5min":1.895,"15min":1.7649999999999999},"min":{"1min":0.26700000000000002,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":12.151999999999999,"5min":12.151999999999999,"15min":13.051},"last":3.2469999999999999},{"interface":"front","average":{"1min":2.2290000000000001,"5min":2.0979999999999999,"15min":2.1110000000000002},"min":{"1min":0.34100000000000003,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":12.458,"5min":12.458,"15min":12.458},"last":0.61499999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:48:32 2023","interfaces":[{"interface":"back","average":{"1min":2.7749999999999999,"5min":2.0139999999999998,"15min":1.9530000000000001},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":11.598000000000001,"5min":11.598000000000001,"15min":12.147},"last":3.3069999999999999},{"interface":"front","average":{"1min":2.3530000000000002,"5min":2.1150000000000002,"15min":2.0870000000000002},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.30299999999999999},"max":{"1min":7.7830000000000004,"5min":17.169,"15min":17.169},"last":3.2799999999999998}]},{"osd":5,"last update":"Sat Sep 23 20:49:04 2023","interfaces":[{"interface":"back","average":{"1min":2.8650000000000002,"5min":1.595,"15min":1.2},"min":{"1min":0.25800000000000001,"5min":0.253,"15min":0.253},"max":{"1min":16.940000000000001,"5min":16.940000000000001,"15min":16.940000000000001},"last":0.52500000000000002},{"interface":"front","average":{"1min":1.3580000000000001,"5min":1.536,"15min":1.1930000000000001},"min":{"1min":0.311,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":4.915,"5min":8.1440000000000001,"15min":8.1440000000000001},"last":0.57099999999999995}]},{"osd":6,"last update":"Sat Sep 23 20:49:17 2023","interfaces":[{"interface":"back","average":{"1min":2.423,"5min":1.881,"15min":1.4610000000000001},"min":{"1min":0.29099999999999998,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":13.503,"5min":13.503,"15min":14.731999999999999},"last":7.1360000000000001},{"interface":"front","average":{"1min":3.722,"5min":2.0009999999999999,"15min":1.3979999999999999},"min":{"1min":0.32000000000000001,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":17.059000000000001,"5min":17.059000000000001,"15min":17.059000000000001},"last":0.46300000000000002}]},{"osd":7,"last update":"Sat Sep 23 20:48:32 2023","interfaces":[{"interface":"back","average":{"1min":1.7070000000000001,"5min":4.9939999999999998,"15min":2.7130000000000001},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":11.228,"5min":313.50099999999998,"15min":313.50099999999998},"last":0.76700000000000002},{"interface":"front","average":{"1min":1.8300000000000001,"5min":4.4589999999999996,"15min":2.2919999999999998},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":17.093,"5min":312.67500000000001,"15min":312.67500000000001},"last":0.78200000000000003}]}]},{"osd":5,"up_from":39,"seq":167503725012,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":958136,"kb_used_data":352368,"kb_used_omap":0,"kb_used_meta":605760,"kb_avail":92787016,"statfs":{"total":95995035648,"available":95013904384,"internally_reserved":0,"allocated":360824832,"data_stored":359133518,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":620298240},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:49:12 2023","interfaces":[{"interface":"back","average":{"1min":1.9410000000000001,"5min":1.829,"15min":1.903},"min":{"1min":0.23999999999999999,"5min":0.22600000000000001,"15min":0.22500000000000001},"max":{"1min":12.07,"5min":12.07,"15min":18.16},"last":0.56200000000000006},{"interface":"front","average":{"1min":1.988,"5min":1.77,"15min":1.8520000000000001},"min":{"1min":0.29099999999999998,"5min":0.27900000000000003,"15min":0.22700000000000001},"max":{"1min":12.089,"5min":12.089,"15min":15.034000000000001},"last":0.48499999999999999}]},{"osd":1,"last update":"Sat Sep 23 20:49:25 2023","interfaces":[{"interface":"back","average":{"1min":2.0089999999999999,"5min":2.0030000000000001,"15min":2.379},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.253},"max":{"1min":10.5,"5min":10.5,"15min":61.005000000000003},"last":0.54800000000000004},{"interface":"front","average":{"1min":2.597,"5min":2.1909999999999998,"15min":2.2629999999999999},"min":{"1min":0.34000000000000002,"5min":0.315,"15min":0.23499999999999999},"max":{"1min":10.340999999999999,"5min":16.219000000000001,"15min":59.688000000000002},"last":0.496}]},{"osd":2,"last update":"Sat Sep 23 20:49:12 2023","interfaces":[{"interface":"back","average":{"1min":2.024,"5min":2.0779999999999998,"15min":2.2149999999999999},"min":{"1min":0.311,"5min":0.26700000000000002,"15min":0.26500000000000001},"max":{"1min":5.5819999999999999,"5min":12.226000000000001,"15min":37.011000000000003},"last":0.61099999999999999},{"interface":"front","average":{"1min":1.8240000000000001,"5min":1.6870000000000001,"15min":2},"min":{"1min":0.31,"5min":0.31,"15min":0.255},"max":{"1min":6.109,"5min":6.109,"15min":36.835000000000001},"last":0.441}]},{"osd":3,"last update":"Sat Sep 23 20:48:30 2023","interfaces":[{"interface":"back","average":{"1min":2.9380000000000002,"5min":1.9650000000000001,"15min":2.3170000000000002},"min":{"1min":0.54100000000000004,"5min":0.33800000000000002,"15min":0.314},"max":{"1min":8.7360000000000007,"5min":8.7360000000000007,"15min":36.716000000000001},"last":0.51400000000000001},{"interface":"front","average":{"1min":3.3719999999999999,"5min":2.0019999999999998,"15min":2.3730000000000002},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":7.4029999999999996,"5min":7.8869999999999996,"15min":18.504000000000001},"last":0.63100000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:49:01 2023","interfaces":[{"interface":"back","average":{"1min":2.137,"5min":1.4810000000000001,"15min":1.3240000000000001},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.255},"max":{"1min":8.9390000000000001,"5min":8.9390000000000001,"15min":13.391999999999999},"last":0.46200000000000002},{"interface":"front","average":{"1min":1.883,"5min":1.631,"15min":1.3200000000000001},"min":{"1min":0.31900000000000001,"5min":0.29199999999999998,"15min":0.28299999999999997},"max":{"1min":8.0030000000000001,"5min":8.0030000000000001,"15min":8.0030000000000001},"last":0.42199999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:49:12 2023","interfaces":[{"interface":"back","average":{"1min":2.3050000000000002,"5min":1.694,"15min":1.5389999999999999},"min":{"1min":0.316,"5min":0.316,"15min":0.30199999999999999},"max":{"1min":8.9039999999999999,"5min":8.9039999999999999,"15min":13.734},"last":0.57899999999999996},{"interface":"front","average":{"1min":2.1240000000000001,"5min":1.663,"15min":1.3779999999999999},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.317},"max":{"1min":9.6519999999999992,"5min":9.6519999999999992,"15min":9.6519999999999992},"last":0.58899999999999997}]},{"osd":7,"last update":"Sat Sep 23 20:48:30 2023","interfaces":[{"interface":"back","average":{"1min":2.4689999999999999,"5min":4.5060000000000002,"15min":2.2490000000000001},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.28499999999999998},"max":{"1min":7.899,"5min":294.70999999999998,"15min":294.70999999999998},"last":0.53400000000000003},{"interface":"front","average":{"1min":2.1240000000000001,"5min":4.5860000000000003,"15min":2.3149999999999999},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.27900000000000003},"max":{"1min":14.032999999999999,"5min":294.74400000000003,"15min":294.74400000000003},"last":0.52600000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4722,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1360,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8206,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7156,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3154,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":488574976,"data_stored":488570904,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":290258944,"data_stored":290258944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":416940032,"data_stored":416940032,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":304087040,"data_stored":304087040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":453185536,"data_stored":453181464,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":354025472,"data_stored":354025472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":386273280,"data_stored":386269208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":322568192,"data_stored":322568192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:49:29.887 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd pool set .mgr pgp_num 71 2023-09-23T20:49:29.960 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:49:29.991 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:30.024 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:49:30.027 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:30.028 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:49:30.129 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:49:30.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:29 smithi018 ceph-mon[184144]: from='client.37194 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:49:30.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:29 smithi018 ceph-mon[184144]: pgmap v1540: 129 pgs: 1 active+remapped+backfilling, 128 active+clean; 833 MiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 711 op/s; 584/39867 objects misplaced (1.465%) 2023-09-23T20:49:30.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:29 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "24"}]': finished 2023-09-23T20:49:30.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:29 smithi018 ceph-mon[184144]: osdmap e624: 8 total, 8 up, 8 in 2023-09-23T20:49:30.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:29 smithi018 ceph-mon[184144]: 10.16 starting backfill to osd.5 from (541'4690,610'11274] MIN to 621'11490 2023-09-23T20:49:30.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:29 smithi018 ceph-mon[184144]: failed to encode map e624 with expected crc 2023-09-23T20:49:30.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:29 smithi018 ceph-mon[184144]: failed to encode map e624 with expected crc 2023-09-23T20:49:30.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:29 smithi018 ceph-mon[184144]: 10.16 starting backfill to osd.6 from (0'0,0'0] MAX to 621'11490 2023-09-23T20:49:30.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:29 smithi018 ceph-mon[184144]: failed to encode map e624 with expected crc 2023-09-23T20:49:30.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:29 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:49:30.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:29 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:49:30.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:29 smithi018 ceph-mon[179647]: from='client.37194 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:49:30.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:29 smithi018 ceph-mon[179647]: pgmap v1540: 129 pgs: 1 active+remapped+backfilling, 128 active+clean; 833 MiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 711 op/s; 584/39867 objects misplaced (1.465%) 2023-09-23T20:49:30.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:29 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "24"}]': finished 2023-09-23T20:49:30.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:29 smithi018 ceph-mon[179647]: osdmap e624: 8 total, 8 up, 8 in 2023-09-23T20:49:30.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:29 smithi018 ceph-mon[179647]: 10.16 starting backfill to osd.5 from (541'4690,610'11274] MIN to 621'11490 2023-09-23T20:49:30.144 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:29 smithi018 ceph-mon[179647]: failed to encode map e624 with expected crc 2023-09-23T20:49:30.144 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:29 smithi018 ceph-mon[179647]: failed to encode map e624 with expected crc 2023-09-23T20:49:30.144 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:29 smithi018 ceph-mon[179647]: 10.16 starting backfill to osd.6 from (0'0,0'0] MAX to 621'11490 2023-09-23T20:49:30.144 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:29 smithi018 ceph-mon[179647]: failed to encode map e624 with expected crc 2023-09-23T20:49:30.144 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:29 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:49:30.144 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:29 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:49:30.157 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:30.194 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:49:30.197 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:49:30.199 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:30.199 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:49:30.300 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:49:30.324 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:29 smithi149 ceph-mon[170394]: from='client.37194 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:49:30.324 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:29 smithi149 ceph-mon[170394]: pgmap v1540: 129 pgs: 1 active+remapped+backfilling, 128 active+clean; 833 MiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 711 op/s; 584/39867 objects misplaced (1.465%) 2023-09-23T20:49:30.325 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:29 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "24"}]': finished 2023-09-23T20:49:30.325 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:29 smithi149 ceph-mon[170394]: osdmap e624: 8 total, 8 up, 8 in 2023-09-23T20:49:30.325 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:29 smithi149 ceph-mon[170394]: 10.16 starting backfill to osd.5 from (541'4690,610'11274] MIN to 621'11490 2023-09-23T20:49:30.325 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:29 smithi149 ceph-mon[170394]: failed to encode map e624 with expected crc 2023-09-23T20:49:30.325 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:29 smithi149 ceph-mon[170394]: failed to encode map e624 with expected crc 2023-09-23T20:49:30.325 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:29 smithi149 ceph-mon[170394]: 10.16 starting backfill to osd.6 from (0'0,0'0] MAX to 621'11490 2023-09-23T20:49:30.325 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:29 smithi149 ceph-mon[170394]: failed to encode map e624 with expected crc 2023-09-23T20:49:30.325 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:29 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:49:30.325 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:29 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:49:30.327 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:30.360 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:49:30.363 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:30.365 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:49:30.466 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:49:30.491 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:30.525 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:49:30.528 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:30.528 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:49:30.629 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:49:30.661 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:30.697 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:49:30.700 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:30.700 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:49:30.801 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:49:30.829 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:30.985 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:49:30.988 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:30.989 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:49:30.993 INFO:teuthology.orchestra.run.smithi018.stderr:noscrub is unset 2023-09-23T20:49:31.033 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:49:31.034 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:49:31.089 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:49:31.142 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:31.229 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:49:31.232 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:30 smithi018 ceph-mon[179647]: osdmap e625: 8 total, 8 up, 8 in 2023-09-23T20:49:31.232 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:30 smithi018 ceph-mon[179647]: failed to encode map e625 with expected crc 2023-09-23T20:49:31.232 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:30 smithi018 ceph-mon[179647]: failed to encode map e625 with expected crc 2023-09-23T20:49:31.233 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:30 smithi018 ceph-mon[179647]: failed to encode map e625 with expected crc 2023-09-23T20:49:31.233 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:30 smithi018 ceph-mon[179647]: failed to encode map e625 with expected crc 2023-09-23T20:49:31.233 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:30 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/4253458836' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:49:31.233 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:30 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:49:31.233 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:30 smithi018 ceph-mon[184144]: osdmap e625: 8 total, 8 up, 8 in 2023-09-23T20:49:31.233 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:30 smithi018 ceph-mon[184144]: failed to encode map e625 with expected crc 2023-09-23T20:49:31.233 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:30 smithi018 ceph-mon[184144]: failed to encode map e625 with expected crc 2023-09-23T20:49:31.233 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:30 smithi018 ceph-mon[184144]: failed to encode map e625 with expected crc 2023-09-23T20:49:31.233 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:30 smithi018 ceph-mon[184144]: failed to encode map e625 with expected crc 2023-09-23T20:49:31.233 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:30 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/4253458836' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:49:31.234 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:30 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:49:31.235 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:31.235 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:49:31.336 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:49:31.388 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:30 smithi149 ceph-mon[170394]: osdmap e625: 8 total, 8 up, 8 in 2023-09-23T20:49:31.388 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:30 smithi149 ceph-mon[170394]: failed to encode map e625 with expected crc 2023-09-23T20:49:31.388 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:30 smithi149 ceph-mon[170394]: failed to encode map e625 with expected crc 2023-09-23T20:49:31.388 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:30 smithi149 ceph-mon[170394]: failed to encode map e625 with expected crc 2023-09-23T20:49:31.389 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:30 smithi149 ceph-mon[170394]: failed to encode map e625 with expected crc 2023-09-23T20:49:31.389 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:30 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/4253458836' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:49:31.389 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:30 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:49:31.391 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:31.454 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:49:31.467 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:31.468 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:49:31.568 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:49:31.620 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:31.693 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:49:31.699 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:31.700 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:49:31.800 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:49:31.879 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:31.953 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:49:31.958 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:31.959 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:49:32.059 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:49:32.140 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:32.360 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:49:32.360 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:49:32.360 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:49:32.370 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:49:32.376 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:32.376 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:49:32.377 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:31 smithi018 ceph-mon[179647]: pgmap v1543: 129 pgs: 1 peering, 128 active+clean; 784 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 314 op/s; 11 MiB/s, 179 objects/s recovering 2023-09-23T20:49:32.377 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:31 smithi018 ceph-mon[179647]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:49:32.377 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:31 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:49:32.377 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:31 smithi018 ceph-mon[179647]: osdmap e626: 8 total, 8 up, 8 in 2023-09-23T20:49:32.377 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:31 smithi018 ceph-mon[179647]: failed to encode map e626 with expected crc 2023-09-23T20:49:32.377 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:31 smithi018 ceph-mon[179647]: 10.17 starting backfill to osd.4 from (0'0,0'0] MAX to 622'11368 2023-09-23T20:49:32.377 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:31 smithi018 ceph-mon[179647]: failed to encode map e626 with expected crc 2023-09-23T20:49:32.377 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:31 smithi018 ceph-mon[179647]: failed to encode map e626 with expected crc 2023-09-23T20:49:32.378 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:31 smithi018 ceph-mon[184144]: pgmap v1543: 129 pgs: 1 peering, 128 active+clean; 784 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 314 op/s; 11 MiB/s, 179 objects/s recovering 2023-09-23T20:49:32.378 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:31 smithi018 ceph-mon[184144]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:49:32.378 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:31 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:49:32.378 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:31 smithi018 ceph-mon[184144]: osdmap e626: 8 total, 8 up, 8 in 2023-09-23T20:49:32.378 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:31 smithi018 ceph-mon[184144]: failed to encode map e626 with expected crc 2023-09-23T20:49:32.378 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:31 smithi018 ceph-mon[184144]: 10.17 starting backfill to osd.4 from (0'0,0'0] MAX to 622'11368 2023-09-23T20:49:32.378 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:31 smithi018 ceph-mon[184144]: failed to encode map e626 with expected crc 2023-09-23T20:49:32.378 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:31 smithi018 ceph-mon[184144]: failed to encode map e626 with expected crc 2023-09-23T20:49:32.390 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:49:32.426 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:49:32.459 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:49:32.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:31 smithi149 ceph-mon[170394]: pgmap v1543: 129 pgs: 1 peering, 128 active+clean; 784 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 314 op/s; 11 MiB/s, 179 objects/s recovering 2023-09-23T20:49:32.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:31 smithi149 ceph-mon[170394]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:49:32.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:31 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:49:32.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:31 smithi149 ceph-mon[170394]: osdmap e626: 8 total, 8 up, 8 in 2023-09-23T20:49:32.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:31 smithi149 ceph-mon[170394]: failed to encode map e626 with expected crc 2023-09-23T20:49:32.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:31 smithi149 ceph-mon[170394]: 10.17 starting backfill to osd.4 from (0'0,0'0] MAX to 622'11368 2023-09-23T20:49:32.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:31 smithi149 ceph-mon[170394]: failed to encode map e626 with expected crc 2023-09-23T20:49:32.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:31 smithi149 ceph-mon[170394]: failed to encode map e626 with expected crc 2023-09-23T20:49:32.477 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:49:32.504 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:49:32.551 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:49:32.562 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:32.574 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:49:32.574 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:49:32.574 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:49:32.575 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:49:32.575 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:49:32.575 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:49:32.575 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:49:32.575 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:49:33.076 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:33 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1945210528' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:49:33.076 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:33 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1945210528' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:49:33.079 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:49:33.085 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:33.085 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:49:33.186 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:49:33.230 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:33.300 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:49:33.305 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:33 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1945210528' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:49:33.306 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:33.306 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:49:33.407 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:49:33.489 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:33.606 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:49:33.610 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:33.611 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:49:33.711 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:49:33.768 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T20:49:33.804 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:33.892 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:49:33.904 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:33.904 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:49:33.949 INFO:teuthology.orchestra.run.smithi018.stdout:ERROR: (22) Invalid argument 2023-09-23T20:49:33.949 INFO:teuthology.orchestra.run.smithi018.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:49:33.985 INFO:teuthology.orchestra.run.smithi018.stderr:Error EINVAL: specified pgp_num 71 > pg_num 31 2023-09-23T20:49:34.006 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:49:34.060 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:34 smithi018 ceph-mon[184144]: pgmap v1545: 129 pgs: 2 peering, 127 active+clean; 721 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 411 op/s; 15 MiB/s, 241 objects/s recovering 2023-09-23T20:49:34.062 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:34 smithi018 ceph-mon[184144]: 10.9 scrub starts 2023-09-23T20:49:34.062 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:34 smithi018 ceph-mon[184144]: 10.9 scrub ok 2023-09-23T20:49:34.062 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:34 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2841314425' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "71"}]: dispatch 2023-09-23T20:49:34.063 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:34 smithi018 ceph-mon[179647]: pgmap v1545: 129 pgs: 2 peering, 127 active+clean; 721 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 411 op/s; 15 MiB/s, 241 objects/s recovering 2023-09-23T20:49:34.063 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:34 smithi018 ceph-mon[179647]: 10.9 scrub starts 2023-09-23T20:49:34.063 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:34 smithi018 ceph-mon[179647]: 10.9 scrub ok 2023-09-23T20:49:34.063 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:34 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2841314425' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "71"}]: dispatch 2023-09-23T20:49:34.077 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:34.148 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:49:34.152 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:34.152 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:49:34.253 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:49:34.297 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:34.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:34 smithi149 ceph-mon[170394]: pgmap v1545: 129 pgs: 2 peering, 127 active+clean; 721 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 411 op/s; 15 MiB/s, 241 objects/s recovering 2023-09-23T20:49:34.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:34 smithi149 ceph-mon[170394]: 10.9 scrub starts 2023-09-23T20:49:34.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:34 smithi149 ceph-mon[170394]: 10.9 scrub ok 2023-09-23T20:49:34.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:34 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2841314425' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "71"}]: dispatch 2023-09-23T20:49:35.219 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:49:35.269 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:49:34 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:49:34] "GET /metrics HTTP/1.1" 200 36592 "" "Prometheus/2.43.0" 2023-09-23T20:49:35.270 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:35 smithi018 ceph-mon[184144]: 10.19 scrub starts 2023-09-23T20:49:35.270 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:35 smithi018 ceph-mon[184144]: 10.19 scrub ok 2023-09-23T20:49:35.270 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:35 smithi018 ceph-mon[184144]: osdmap e627: 8 total, 8 up, 8 in 2023-09-23T20:49:35.270 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:35 smithi018 ceph-mon[184144]: failed to encode map e627 with expected crc 2023-09-23T20:49:35.270 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:35 smithi018 ceph-mon[184144]: 10.14 scrub starts 2023-09-23T20:49:35.270 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:35 smithi018 ceph-mon[184144]: 10.14 scrub ok 2023-09-23T20:49:35.271 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:35 smithi018 ceph-mon[179647]: 10.19 scrub starts 2023-09-23T20:49:35.271 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:35 smithi018 ceph-mon[179647]: 10.19 scrub ok 2023-09-23T20:49:35.272 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:35 smithi018 ceph-mon[179647]: osdmap e627: 8 total, 8 up, 8 in 2023-09-23T20:49:35.272 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:35 smithi018 ceph-mon[179647]: failed to encode map e627 with expected crc 2023-09-23T20:49:35.272 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:35 smithi018 ceph-mon[179647]: 10.14 scrub starts 2023-09-23T20:49:35.272 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:35 smithi018 ceph-mon[179647]: 10.14 scrub ok 2023-09-23T20:49:35.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:35 smithi149 ceph-mon[170394]: 10.19 scrub starts 2023-09-23T20:49:35.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:35 smithi149 ceph-mon[170394]: 10.19 scrub ok 2023-09-23T20:49:35.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:35 smithi149 ceph-mon[170394]: osdmap e627: 8 total, 8 up, 8 in 2023-09-23T20:49:35.468 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:35 smithi149 ceph-mon[170394]: failed to encode map e627 with expected crc 2023-09-23T20:49:35.468 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:35 smithi149 ceph-mon[170394]: 10.14 scrub starts 2023-09-23T20:49:35.468 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:35 smithi149 ceph-mon[170394]: 10.14 scrub ok 2023-09-23T20:49:35.590 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-09-23T20:49:35.604 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:49:35.612 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:35.613 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:49:35.713 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:49:35.741 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T20:49:35.757 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:35.807 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:49:35.812 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:35.812 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:49:35.913 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:49:36.001 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:49:36.008 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:36.086 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:49:36.091 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:36.092 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:49:36.109 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:49:36.193 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:49:36.252 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:36.274 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:36 smithi018 ceph-mon[184144]: pgmap v1547: 129 pgs: 2 peering, 127 active+clean; 677 MiB data, 8.1 GiB used, 707 GiB / 715 GiB avail 2023-09-23T20:49:36.274 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:36 smithi018 ceph-mon[184144]: osdmap e628: 8 total, 8 up, 8 in 2023-09-23T20:49:36.275 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:36 smithi018 ceph-mon[184144]: failed to encode map e628 with expected crc 2023-09-23T20:49:36.275 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:36 smithi018 ceph-mon[184144]: 10.7 scrub starts 2023-09-23T20:49:36.275 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:36 smithi018 ceph-mon[184144]: 10.7 scrub ok 2023-09-23T20:49:36.276 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:36 smithi018 ceph-mon[179647]: pgmap v1547: 129 pgs: 2 peering, 127 active+clean; 677 MiB data, 8.1 GiB used, 707 GiB / 715 GiB avail 2023-09-23T20:49:36.276 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:36 smithi018 ceph-mon[179647]: osdmap e628: 8 total, 8 up, 8 in 2023-09-23T20:49:36.276 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:36 smithi018 ceph-mon[179647]: failed to encode map e628 with expected crc 2023-09-23T20:49:36.276 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:36 smithi018 ceph-mon[179647]: 10.7 scrub starts 2023-09-23T20:49:36.276 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:36 smithi018 ceph-mon[179647]: 10.7 scrub ok 2023-09-23T20:49:36.308 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:49:36.318 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:36.318 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:49:36.318 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:36 smithi149 ceph-mon[170394]: pgmap v1547: 129 pgs: 2 peering, 127 active+clean; 677 MiB data, 8.1 GiB used, 707 GiB / 715 GiB avail 2023-09-23T20:49:36.319 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:36 smithi149 ceph-mon[170394]: osdmap e628: 8 total, 8 up, 8 in 2023-09-23T20:49:36.319 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:36 smithi149 ceph-mon[170394]: failed to encode map e628 with expected crc 2023-09-23T20:49:36.319 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:36 smithi149 ceph-mon[170394]: 10.7 scrub starts 2023-09-23T20:49:36.319 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:36 smithi149 ceph-mon[170394]: 10.7 scrub ok 2023-09-23T20:49:36.419 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:49:36.485 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:36.555 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:49:36.561 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:36.561 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:49:36.662 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:49:36.718 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:36.879 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:49:36.887 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:36.888 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:49:36.989 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:49:37.047 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:37.106 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:49:37.113 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:37.113 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:49:37.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:37 smithi018 ceph-mon[184144]: 10.0 scrub starts 2023-09-23T20:49:37.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:37 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "25"}]: dispatch 2023-09-23T20:49:37.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:37 smithi018 ceph-mon[184144]: 10.0 scrub ok 2023-09-23T20:49:37.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:37 smithi018 ceph-mon[179647]: 10.0 scrub starts 2023-09-23T20:49:37.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:37 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "25"}]: dispatch 2023-09-23T20:49:37.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:37 smithi018 ceph-mon[179647]: 10.0 scrub ok 2023-09-23T20:49:37.214 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:49:37.270 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:37.327 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:49:37.334 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:37.335 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:37 smithi149 ceph-mon[170394]: 10.0 scrub starts 2023-09-23T20:49:37.335 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:37 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "25"}]: dispatch 2023-09-23T20:49:37.335 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:37 smithi149 ceph-mon[170394]: 10.0 scrub ok 2023-09-23T20:49:37.336 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:49:37.342 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:49:37.437 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:49:37.471 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:37.521 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:49:37.526 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:37.526 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:49:37.627 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:49:37.670 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:37.801 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:49:37.804 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:37.804 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:49:37.905 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:49:37.932 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:37.965 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:49:37.968 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:37.968 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:49:38.056 INFO:teuthology.orchestra.run.smithi018.stderr:nodeep-scrub is unset 2023-09-23T20:49:38.069 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:49:38.098 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:38.105 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:49:38.125 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:49:38.132 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:49:38.135 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:38.136 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:49:38.148 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:49:38.168 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:49:38.187 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:49:38.206 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:49:38.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:38 smithi149 ceph-mon[170394]: pgmap v1549: 129 pgs: 129 active+clean; 506 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:49:38.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:38 smithi149 ceph-mon[170394]: 1.14 scrub starts 2023-09-23T20:49:38.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:38 smithi149 ceph-mon[170394]: 1.14 scrub ok 2023-09-23T20:49:38.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:38 smithi149 ceph-mon[170394]: failed to encode map e629 with expected crc 2023-09-23T20:49:38.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:38 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "25"}]': finished 2023-09-23T20:49:38.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:38 smithi149 ceph-mon[170394]: osdmap e629: 8 total, 8 up, 8 in 2023-09-23T20:49:38.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:38 smithi149 ceph-mon[170394]: 10.16 scrub starts 2023-09-23T20:49:38.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:38 smithi149 ceph-mon[170394]: 10.16 scrub ok 2023-09-23T20:49:38.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:38 smithi149 ceph-mon[170394]: 10.6 scrub starts 2023-09-23T20:49:38.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:38 smithi149 ceph-mon[170394]: 10.6 scrub ok 2023-09-23T20:49:38.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:38 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1559879357' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:49:38.225 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:49:38.237 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:49:38.244 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:49:38.245 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:49:38.246 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:49:38.246 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:49:38.246 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:49:38.246 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:49:38.246 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:49:38.247 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:49:38.247 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:49:38.263 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:38.270 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:38 smithi018 ceph-mon[179647]: pgmap v1549: 129 pgs: 129 active+clean; 506 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:49:38.271 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:38 smithi018 ceph-mon[179647]: 1.14 scrub starts 2023-09-23T20:49:38.271 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:38 smithi018 ceph-mon[179647]: 1.14 scrub ok 2023-09-23T20:49:38.271 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:38 smithi018 ceph-mon[179647]: failed to encode map e629 with expected crc 2023-09-23T20:49:38.271 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:38 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "25"}]': finished 2023-09-23T20:49:38.271 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:38 smithi018 ceph-mon[179647]: osdmap e629: 8 total, 8 up, 8 in 2023-09-23T20:49:38.271 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:38 smithi018 ceph-mon[179647]: 10.16 scrub starts 2023-09-23T20:49:38.272 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:38 smithi018 ceph-mon[179647]: 10.16 scrub ok 2023-09-23T20:49:38.272 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:38 smithi018 ceph-mon[179647]: 10.6 scrub starts 2023-09-23T20:49:38.272 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:38 smithi018 ceph-mon[179647]: 10.6 scrub ok 2023-09-23T20:49:38.272 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:38 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1559879357' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:49:38.272 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:38 smithi018 ceph-mon[184144]: pgmap v1549: 129 pgs: 129 active+clean; 506 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:49:38.273 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:38 smithi018 ceph-mon[184144]: 1.14 scrub starts 2023-09-23T20:49:38.273 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:38 smithi018 ceph-mon[184144]: 1.14 scrub ok 2023-09-23T20:49:38.273 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:38 smithi018 ceph-mon[184144]: failed to encode map e629 with expected crc 2023-09-23T20:49:38.273 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:38 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "25"}]': finished 2023-09-23T20:49:38.273 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:38 smithi018 ceph-mon[184144]: osdmap e629: 8 total, 8 up, 8 in 2023-09-23T20:49:38.273 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:38 smithi018 ceph-mon[184144]: 10.16 scrub starts 2023-09-23T20:49:38.274 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:38 smithi018 ceph-mon[184144]: 10.16 scrub ok 2023-09-23T20:49:38.274 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:38 smithi018 ceph-mon[184144]: 10.6 scrub starts 2023-09-23T20:49:38.274 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:38 smithi018 ceph-mon[184144]: 10.6 scrub ok 2023-09-23T20:49:38.274 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:38 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1559879357' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:49:38.296 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:49:38.299 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:38.300 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:49:38.401 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:49:38.427 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:38.460 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:49:38.463 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:38.464 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:49:38.565 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:49:38.595 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:38.891 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:49:38.895 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:38.895 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:49:38.996 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:49:39.024 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:39.063 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:49:39.066 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:39.066 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:49:39.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:39 smithi018 ceph-mon[179647]: 1.a scrub starts 2023-09-23T20:49:39.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:39 smithi018 ceph-mon[179647]: 1.a scrub ok 2023-09-23T20:49:39.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:39 smithi018 ceph-mon[179647]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:49:39.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:39 smithi018 ceph-mon[179647]: Cluster is now healthy 2023-09-23T20:49:39.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:39 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1559879357' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:49:39.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:39 smithi018 ceph-mon[179647]: osdmap e630: 8 total, 8 up, 8 in 2023-09-23T20:49:39.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:39 smithi018 ceph-mon[179647]: failed to encode map e630 with expected crc 2023-09-23T20:49:39.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:39 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3347266501' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:49:39.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:39 smithi018 ceph-mon[179647]: 10.1 scrub starts 2023-09-23T20:49:39.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:39 smithi018 ceph-mon[179647]: 10.1 scrub ok 2023-09-23T20:49:39.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:39 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "26"}]: dispatch 2023-09-23T20:49:39.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:39 smithi018 ceph-mon[184144]: 1.a scrub starts 2023-09-23T20:49:39.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:39 smithi018 ceph-mon[184144]: 1.a scrub ok 2023-09-23T20:49:39.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:39 smithi018 ceph-mon[184144]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:49:39.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:39 smithi018 ceph-mon[184144]: Cluster is now healthy 2023-09-23T20:49:39.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:39 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1559879357' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:49:39.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:39 smithi018 ceph-mon[184144]: osdmap e630: 8 total, 8 up, 8 in 2023-09-23T20:49:39.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:39 smithi018 ceph-mon[184144]: failed to encode map e630 with expected crc 2023-09-23T20:49:39.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:39 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3347266501' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:49:39.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:39 smithi018 ceph-mon[184144]: 10.1 scrub starts 2023-09-23T20:49:39.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:39 smithi018 ceph-mon[184144]: 10.1 scrub ok 2023-09-23T20:49:39.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:39 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "26"}]: dispatch 2023-09-23T20:49:39.167 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:49:39.232 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:39.296 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:49:39.302 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:39.302 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:49:39.403 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:49:39.445 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:39 smithi149 ceph-mon[170394]: 1.a scrub starts 2023-09-23T20:49:39.445 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:39 smithi149 ceph-mon[170394]: 1.a scrub ok 2023-09-23T20:49:39.445 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:39 smithi149 ceph-mon[170394]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:49:39.446 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:39 smithi149 ceph-mon[170394]: Cluster is now healthy 2023-09-23T20:49:39.446 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:39 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1559879357' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:49:39.446 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:39 smithi149 ceph-mon[170394]: osdmap e630: 8 total, 8 up, 8 in 2023-09-23T20:49:39.446 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:39 smithi149 ceph-mon[170394]: failed to encode map e630 with expected crc 2023-09-23T20:49:39.446 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:39 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3347266501' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:49:39.446 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:39 smithi149 ceph-mon[170394]: 10.1 scrub starts 2023-09-23T20:49:39.446 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:39 smithi149 ceph-mon[170394]: 10.1 scrub ok 2023-09-23T20:49:39.446 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:39 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "26"}]: dispatch 2023-09-23T20:49:39.448 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:39.506 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:49:39.514 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:39.514 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:49:39.615 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:49:39.654 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:39.680 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2023-09-23T20:49:39.681 INFO:teuthology.orchestra.run.smithi018.stdout: "ops": [], 2023-09-23T20:49:39.681 INFO:teuthology.orchestra.run.smithi018.stdout: "complaint_time": 30, 2023-09-23T20:49:39.681 INFO:teuthology.orchestra.run.smithi018.stdout: "num_blocked_ops": 0 2023-09-23T20:49:39.681 INFO:teuthology.orchestra.run.smithi018.stdout:} 2023-09-23T20:49:39.695 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:49:39.699 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:39.699 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:49:39.800 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:49:39.837 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:39.970 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:49:39.973 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:39.973 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:49:40.005 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T20:49:40.074 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:49:40.100 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:40.134 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:49:40.137 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:40.138 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:49:40.239 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:49:40.265 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:40.298 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:49:40.300 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:40 smithi149 ceph-mon[170394]: pgmap v1552: 129 pgs: 129 active+clean; 471 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 4.8 MiB/s, 76 objects/s recovering 2023-09-23T20:49:40.301 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:40 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "26"}]': finished 2023-09-23T20:49:40.301 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:40 smithi149 ceph-mon[170394]: osdmap e631: 8 total, 8 up, 8 in 2023-09-23T20:49:40.301 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:40 smithi149 ceph-mon[170394]: 10.18 starting backfill to osd.1 from (0'0,0'0] MAX to 628'11517 2023-09-23T20:49:40.301 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:40 smithi149 ceph-mon[170394]: failed to encode map e631 with expected crc 2023-09-23T20:49:40.301 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:40 smithi149 ceph-mon[170394]: failed to encode map e631 with expected crc 2023-09-23T20:49:40.301 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:40 smithi149 ceph-mon[170394]: failed to encode map e631 with expected crc 2023-09-23T20:49:40.301 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:40 smithi149 ceph-mon[170394]: 10.18 starting backfill to osd.3 from (0'0,0'0] MAX to 628'11517 2023-09-23T20:49:40.301 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:40 smithi149 ceph-mon[170394]: failed to encode map e631 with expected crc 2023-09-23T20:49:40.302 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:40.302 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:49:40.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:40 smithi018 ceph-mon[179647]: pgmap v1552: 129 pgs: 129 active+clean; 471 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 4.8 MiB/s, 76 objects/s recovering 2023-09-23T20:49:40.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:40 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "26"}]': finished 2023-09-23T20:49:40.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:40 smithi018 ceph-mon[179647]: osdmap e631: 8 total, 8 up, 8 in 2023-09-23T20:49:40.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:40 smithi018 ceph-mon[179647]: 10.18 starting backfill to osd.1 from (0'0,0'0] MAX to 628'11517 2023-09-23T20:49:40.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:40 smithi018 ceph-mon[179647]: failed to encode map e631 with expected crc 2023-09-23T20:49:40.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:40 smithi018 ceph-mon[179647]: failed to encode map e631 with expected crc 2023-09-23T20:49:40.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:40 smithi018 ceph-mon[179647]: failed to encode map e631 with expected crc 2023-09-23T20:49:40.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:40 smithi018 ceph-mon[179647]: 10.18 starting backfill to osd.3 from (0'0,0'0] MAX to 628'11517 2023-09-23T20:49:40.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:40 smithi018 ceph-mon[179647]: failed to encode map e631 with expected crc 2023-09-23T20:49:40.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:40 smithi018 ceph-mon[184144]: pgmap v1552: 129 pgs: 129 active+clean; 471 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 4.8 MiB/s, 76 objects/s recovering 2023-09-23T20:49:40.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:40 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num_actual", "val": "26"}]': finished 2023-09-23T20:49:40.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:40 smithi018 ceph-mon[184144]: osdmap e631: 8 total, 8 up, 8 in 2023-09-23T20:49:40.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:40 smithi018 ceph-mon[184144]: 10.18 starting backfill to osd.1 from (0'0,0'0] MAX to 628'11517 2023-09-23T20:49:40.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:40 smithi018 ceph-mon[184144]: failed to encode map e631 with expected crc 2023-09-23T20:49:40.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:40 smithi018 ceph-mon[184144]: failed to encode map e631 with expected crc 2023-09-23T20:49:40.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:40 smithi018 ceph-mon[184144]: failed to encode map e631 with expected crc 2023-09-23T20:49:40.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:40 smithi018 ceph-mon[184144]: 10.18 starting backfill to osd.3 from (0'0,0'0] MAX to 628'11517 2023-09-23T20:49:40.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:40 smithi018 ceph-mon[184144]: failed to encode map e631 with expected crc 2023-09-23T20:49:40.403 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:49:40.431 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:40.466 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:49:40.469 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:40.469 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:49:40.570 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:49:40.592 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 7, 3, 6, 5, 4, 2] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T20:49:40.592 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:49:40.592 INFO:tasks.thrashosds.thrasher:Removing osd 5, in_osds are: [0, 1, 7, 3, 6, 5, 4, 2] 2023-09-23T20:49:40.592 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd out 5 2023-09-23T20:49:40.596 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:40.630 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:49:40.633 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:40.634 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:49:40.655 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:49:40.735 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:49:40.763 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:40.908 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:49:40.919 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T20:49:41.059 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:41 smithi018 ceph-mon[179647]: osdmap e632: 8 total, 8 up, 8 in 2023-09-23T20:49:41.060 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:41 smithi018 ceph-mon[179647]: failed to encode map e632 with expected crc 2023-09-23T20:49:41.061 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:41 smithi018 ceph-mon[179647]: failed to encode map e632 with expected crc 2023-09-23T20:49:41.061 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:41 smithi018 ceph-mon[179647]: failed to encode map e632 with expected crc 2023-09-23T20:49:41.061 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:41 smithi018 ceph-mon[179647]: 10.15 scrub starts 2023-09-23T20:49:41.061 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:41 smithi018 ceph-mon[179647]: 10.15 scrub ok 2023-09-23T20:49:41.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:41 smithi018 ceph-mon[184144]: osdmap e632: 8 total, 8 up, 8 in 2023-09-23T20:49:41.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:41 smithi018 ceph-mon[184144]: failed to encode map e632 with expected crc 2023-09-23T20:49:41.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:41 smithi018 ceph-mon[184144]: failed to encode map e632 with expected crc 2023-09-23T20:49:41.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:41 smithi018 ceph-mon[184144]: failed to encode map e632 with expected crc 2023-09-23T20:49:41.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:41 smithi018 ceph-mon[184144]: 10.15 scrub starts 2023-09-23T20:49:41.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:41 smithi018 ceph-mon[184144]: 10.15 scrub ok 2023-09-23T20:49:41.471 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:41 smithi149 ceph-mon[170394]: osdmap e632: 8 total, 8 up, 8 in 2023-09-23T20:49:41.471 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:41 smithi149 ceph-mon[170394]: failed to encode map e632 with expected crc 2023-09-23T20:49:41.471 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:41 smithi149 ceph-mon[170394]: failed to encode map e632 with expected crc 2023-09-23T20:49:41.471 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:41 smithi149 ceph-mon[170394]: failed to encode map e632 with expected crc 2023-09-23T20:49:41.471 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:41 smithi149 ceph-mon[170394]: 10.15 scrub starts 2023-09-23T20:49:41.471 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:41 smithi149 ceph-mon[170394]: 10.15 scrub ok 2023-09-23T20:49:41.491 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:49:41.498 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:41.499 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:49:41.600 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:49:41.644 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:41.729 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:49:41.730 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:41.731 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:49:41.831 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:49:41.896 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:41.963 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:49:41.976 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:41.976 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:49:42.077 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:49:42.142 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:42.206 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:49:42.220 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:42.220 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:49:42.221 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:42 smithi149 ceph-mon[170394]: pgmap v1555: 129 pgs: 1 remapped+peering, 128 active+clean; 416 MiB data, 7.2 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:49:42.221 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:42 smithi149 ceph-mon[170394]: osdmap e633: 8 total, 8 up, 8 in 2023-09-23T20:49:42.221 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:42 smithi149 ceph-mon[170394]: 10.19 starting backfill to osd.1 from (0'0,0'0] MAX to 629'11853 2023-09-23T20:49:42.221 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:42 smithi149 ceph-mon[170394]: 10.19 starting backfill to osd.3 from (0'0,0'0] MAX to 629'11853 2023-09-23T20:49:42.221 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:42 smithi149 ceph-mon[170394]: 10.19 starting backfill to osd.4 from (0'0,0'0] MAX to 629'11853 2023-09-23T20:49:42.221 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:42 smithi149 ceph-mon[170394]: failed to encode map e633 with expected crc 2023-09-23T20:49:42.221 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:42 smithi149 ceph-mon[170394]: failed to encode map e633 with expected crc 2023-09-23T20:49:42.221 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:42 smithi149 ceph-mon[170394]: failed to encode map e633 with expected crc 2023-09-23T20:49:42.221 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:42 smithi149 ceph-mon[170394]: failed to encode map e633 with expected crc 2023-09-23T20:49:42.321 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:49:42.331 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:42 smithi018 ceph-mon[184144]: pgmap v1555: 129 pgs: 1 remapped+peering, 128 active+clean; 416 MiB data, 7.2 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:49:42.332 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:42 smithi018 ceph-mon[184144]: osdmap e633: 8 total, 8 up, 8 in 2023-09-23T20:49:42.332 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:42 smithi018 ceph-mon[184144]: 10.19 starting backfill to osd.1 from (0'0,0'0] MAX to 629'11853 2023-09-23T20:49:42.332 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:42 smithi018 ceph-mon[184144]: 10.19 starting backfill to osd.3 from (0'0,0'0] MAX to 629'11853 2023-09-23T20:49:42.332 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:42 smithi018 ceph-mon[184144]: 10.19 starting backfill to osd.4 from (0'0,0'0] MAX to 629'11853 2023-09-23T20:49:42.332 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:42 smithi018 ceph-mon[184144]: failed to encode map e633 with expected crc 2023-09-23T20:49:42.332 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:42 smithi018 ceph-mon[184144]: failed to encode map e633 with expected crc 2023-09-23T20:49:42.332 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:42 smithi018 ceph-mon[184144]: failed to encode map e633 with expected crc 2023-09-23T20:49:42.332 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:42 smithi018 ceph-mon[184144]: failed to encode map e633 with expected crc 2023-09-23T20:49:42.333 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:42 smithi018 ceph-mon[179647]: pgmap v1555: 129 pgs: 1 remapped+peering, 128 active+clean; 416 MiB data, 7.2 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:49:42.333 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:42 smithi018 ceph-mon[179647]: osdmap e633: 8 total, 8 up, 8 in 2023-09-23T20:49:42.333 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:42 smithi018 ceph-mon[179647]: 10.19 starting backfill to osd.1 from (0'0,0'0] MAX to 629'11853 2023-09-23T20:49:42.333 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:42 smithi018 ceph-mon[179647]: 10.19 starting backfill to osd.3 from (0'0,0'0] MAX to 629'11853 2023-09-23T20:49:42.333 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:42 smithi018 ceph-mon[179647]: 10.19 starting backfill to osd.4 from (0'0,0'0] MAX to 629'11853 2023-09-23T20:49:42.333 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:42 smithi018 ceph-mon[179647]: failed to encode map e633 with expected crc 2023-09-23T20:49:42.333 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:42 smithi018 ceph-mon[179647]: failed to encode map e633 with expected crc 2023-09-23T20:49:42.333 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:42 smithi018 ceph-mon[179647]: failed to encode map e633 with expected crc 2023-09-23T20:49:42.333 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:42 smithi018 ceph-mon[179647]: failed to encode map e633 with expected crc 2023-09-23T20:49:42.390 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:42.463 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:49:42.474 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:42.475 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:49:42.575 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:49:42.622 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:42.704 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:49:42.717 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:42.717 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:49:42.818 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:49:42.874 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:49:42.902 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:42.916 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:49:42.957 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:49:42.960 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:49:42.968 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:42.968 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:49:43.069 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:49:43.149 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:43.255 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:49:43.258 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:43.259 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:49:43.359 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:49:43.400 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:43.450 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:49:43.454 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:43.455 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:49:43.556 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:49:43.597 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:43.745 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:49:43.749 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:43.750 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:49:43.850 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:49:43.877 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:43.910 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:49:43.913 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:43.914 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:49:44.014 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:49:44.041 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:44.074 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:49:44.077 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:44.078 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:49:44.179 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:49:44.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:44 smithi149 ceph-mon[170394]: pgmap v1557: 129 pgs: 1 remapped+peering, 128 active+clean; 300 MiB data, 6.7 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:49:44.218 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:44 smithi149 ceph-mon[170394]: 10.4 scrub starts 2023-09-23T20:49:44.218 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:44 smithi149 ceph-mon[170394]: 10.4 scrub ok 2023-09-23T20:49:44.218 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:44 smithi149 ceph-mon[170394]: osdmap e634: 8 total, 8 up, 8 in 2023-09-23T20:49:44.218 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:44 smithi149 ceph-mon[170394]: failed to encode map e634 with expected crc 2023-09-23T20:49:44.218 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:44 smithi149 ceph-mon[170394]: failed to encode map e634 with expected crc 2023-09-23T20:49:44.218 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:44 smithi149 ceph-mon[170394]: failed to encode map e634 with expected crc 2023-09-23T20:49:44.257 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:44.385 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:49:44.390 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:44.391 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:49:44.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:44 smithi018 ceph-mon[184144]: pgmap v1557: 129 pgs: 1 remapped+peering, 128 active+clean; 300 MiB data, 6.7 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:49:44.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:44 smithi018 ceph-mon[184144]: 10.4 scrub starts 2023-09-23T20:49:44.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:44 smithi018 ceph-mon[184144]: 10.4 scrub ok 2023-09-23T20:49:44.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:44 smithi018 ceph-mon[184144]: osdmap e634: 8 total, 8 up, 8 in 2023-09-23T20:49:44.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:44 smithi018 ceph-mon[184144]: failed to encode map e634 with expected crc 2023-09-23T20:49:44.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:44 smithi018 ceph-mon[184144]: failed to encode map e634 with expected crc 2023-09-23T20:49:44.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:44 smithi018 ceph-mon[184144]: failed to encode map e634 with expected crc 2023-09-23T20:49:44.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:44 smithi018 ceph-mon[179647]: pgmap v1557: 129 pgs: 1 remapped+peering, 128 active+clean; 300 MiB data, 6.7 GiB used, 708 GiB / 715 GiB avail 2023-09-23T20:49:44.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:44 smithi018 ceph-mon[179647]: 10.4 scrub starts 2023-09-23T20:49:44.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:44 smithi018 ceph-mon[179647]: 10.4 scrub ok 2023-09-23T20:49:44.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:44 smithi018 ceph-mon[179647]: osdmap e634: 8 total, 8 up, 8 in 2023-09-23T20:49:44.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:44 smithi018 ceph-mon[179647]: failed to encode map e634 with expected crc 2023-09-23T20:49:44.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:44 smithi018 ceph-mon[179647]: failed to encode map e634 with expected crc 2023-09-23T20:49:44.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:44 smithi018 ceph-mon[179647]: failed to encode map e634 with expected crc 2023-09-23T20:49:44.492 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:49:44.522 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2023-09-23T20:49:44.522 INFO:teuthology.orchestra.run.smithi018.stdout: "size": 20, 2023-09-23T20:49:44.522 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 600, 2023-09-23T20:49:44.523 INFO:teuthology.orchestra.run.smithi018.stdout: "ops": [ 2023-09-23T20:49:44.523 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.523 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.36450.0:185319 9.8 9.8673f918 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e503)", 2023-09-23T20:49:44.523 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:44:18.697532+0000", 2023-09-23T20:49:44.523 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 325.82010666100001, 2023-09-23T20:49:44.523 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.93022128999999998, 2023-09-23T20:49:44.523 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:49:44.523 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:49:44.523 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:49:44.523 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.36450", 2023-09-23T20:49:44.524 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:49:44.524 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 185319 2023-09-23T20:49:44.524 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.524 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:49:44.524 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.524 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:49:44.524 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.697532+0000", 2023-09-23T20:49:44.524 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:44.524 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.524 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.525 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:49:44.525 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.697532+0000", 2023-09-23T20:49:44.525 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:44.525 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.525 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.525 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:49:44.525 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.697532+0000", 2023-09-23T20:49:44.525 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:49:44.525 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.526 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.526 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:49:44.526 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.697535+0000", 2023-09-23T20:49:44.526 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.1709999999999998e-06 2023-09-23T20:49:44.526 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.526 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.526 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:49:44.526 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.697542+0000", 2023-09-23T20:49:44.526 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.7880000000000001e-06 2023-09-23T20:49:44.526 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.527 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.527 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:49:44.527 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.697547+0000", 2023-09-23T20:49:44.527 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.0159999999999999e-06 2023-09-23T20:49:44.527 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.527 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.527 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:44.527 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.698190+0000", 2023-09-23T20:49:44.527 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00064296299999999998 2023-09-23T20:49:44.527 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.527 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.528 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:49:44.528 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.698201+0000", 2023-09-23T20:49:44.528 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0898e-05 2023-09-23T20:49:44.528 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.528 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.528 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:44.528 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:19.627739+0000", 2023-09-23T20:49:44.528 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.92953824600000001 2023-09-23T20:49:44.528 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.528 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.528 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:49:44.529 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:19.627754+0000", 2023-09-23T20:49:44.529 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.4634000000000001e-05 2023-09-23T20:49:44.529 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:44.529 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:49:44.529 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:44.529 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.529 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.529 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.36450.0:185321 9.8 9.f3369438 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e503)", 2023-09-23T20:49:44.529 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:44:18.708465+0000", 2023-09-23T20:49:44.529 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 325.80917446199999, 2023-09-23T20:49:44.529 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.91932099599999995, 2023-09-23T20:49:44.529 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:49:44.530 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:49:44.530 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:49:44.530 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.36450", 2023-09-23T20:49:44.530 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:49:44.530 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 185321 2023-09-23T20:49:44.530 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.530 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:49:44.530 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.530 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:49:44.530 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.708465+0000", 2023-09-23T20:49:44.531 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:44.531 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.531 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.531 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:49:44.531 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.708465+0000", 2023-09-23T20:49:44.531 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:44.531 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.531 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.531 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:49:44.531 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.708463+0000", 2023-09-23T20:49:44.532 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:49:44.532 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.532 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.532 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:49:44.532 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.708482+0000", 2023-09-23T20:49:44.532 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.9258000000000001e-05 2023-09-23T20:49:44.532 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.532 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.532 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:49:44.532 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.708500+0000", 2023-09-23T20:49:44.532 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.7708000000000001e-05 2023-09-23T20:49:44.532 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.533 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.533 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:49:44.533 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.708510+0000", 2023-09-23T20:49:44.533 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.6190000000000006e-06 2023-09-23T20:49:44.533 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.533 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.533 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:44.533 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.708611+0000", 2023-09-23T20:49:44.533 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000101296 2023-09-23T20:49:44.533 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.533 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.534 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:49:44.534 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.708623+0000", 2023-09-23T20:49:44.534 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.201e-05 2023-09-23T20:49:44.534 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.534 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.534 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:44.534 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:19.627776+0000", 2023-09-23T20:49:44.534 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.919153161 2023-09-23T20:49:44.534 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.534 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.534 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:49:44.535 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:19.627786+0000", 2023-09-23T20:49:44.535 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.4369999999999996e-06 2023-09-23T20:49:44.535 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:44.535 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:49:44.535 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:44.535 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.535 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.535 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:196320 10.15 10:a8362cca:::benchmark_data_smithi018_422431_object91824:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e621)", 2023-09-23T20:49:44.535 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:49:26.039338+0000", 2023-09-23T20:49:44.535 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 18.478301111, 2023-09-23T20:49:44.535 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.04174229, 2023-09-23T20:49:44.535 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:49:44.536 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:49:44.536 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:49:44.536 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:49:44.536 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:49:44.536 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 196320 2023-09-23T20:49:44.536 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.536 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:49:44.536 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.536 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:49:44.536 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:26.039338+0000", 2023-09-23T20:49:44.536 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:44.537 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.537 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.537 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:49:44.537 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:26.039338+0000", 2023-09-23T20:49:44.537 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:44.537 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.537 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.537 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:49:44.537 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:26.039336+0000", 2023-09-23T20:49:44.537 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:49:44.537 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.538 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.538 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:49:44.538 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:26.039345+0000", 2023-09-23T20:49:44.538 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.106e-06 2023-09-23T20:49:44.538 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.538 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.538 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:49:44.538 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:26.039358+0000", 2023-09-23T20:49:44.538 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3097e-05 2023-09-23T20:49:44.538 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.538 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.538 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:49:44.539 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:26.039366+0000", 2023-09-23T20:49:44.539 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.3780000000000001e-06 2023-09-23T20:49:44.539 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.539 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.539 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:44.539 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:26.039441+0000", 2023-09-23T20:49:44.539 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.4954000000000001e-05 2023-09-23T20:49:44.539 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.539 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.539 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:49:44.539 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:26.039458+0000", 2023-09-23T20:49:44.540 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.7248000000000002e-05 2023-09-23T20:49:44.540 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.540 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.540 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:44.540 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:27.050175+0000", 2023-09-23T20:49:44.540 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.010716379 2023-09-23T20:49:44.540 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.540 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.540 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:49:44.540 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:27.050307+0000", 2023-09-23T20:49:44.540 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000132724 2023-09-23T20:49:44.540 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.541 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.541 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 0,1,2,5", 2023-09-23T20:49:44.541 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:27.050575+0000", 2023-09-23T20:49:44.541 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00026792800000000001 2023-09-23T20:49:44.541 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.541 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.541 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:49:44.541 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:27.054098+0000", 2023-09-23T20:49:44.541 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.003522552 2023-09-23T20:49:44.541 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.541 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.541 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:44.542 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:27.059295+0000", 2023-09-23T20:49:44.542 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0051966979999999996 2023-09-23T20:49:44.542 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.542 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.542 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:44.542 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:27.059361+0000", 2023-09-23T20:49:44.542 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.6346000000000005e-05 2023-09-23T20:49:44.542 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.542 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.543 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:44.543 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:27.059460+0000", 2023-09-23T20:49:44.543 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.8892000000000001e-05 2023-09-23T20:49:44.543 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.543 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.543 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:44.543 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:27.080897+0000", 2023-09-23T20:49:44.544 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.021437641 2023-09-23T20:49:44.544 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.544 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.544 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:49:44.544 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:27.080945+0000", 2023-09-23T20:49:44.544 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.7849000000000002e-05 2023-09-23T20:49:44.544 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.545 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.545 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:49:44.545 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:27.081080+0000", 2023-09-23T20:49:44.545 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000134964 2023-09-23T20:49:44.545 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:44.545 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:49:44.545 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:44.545 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.545 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.545 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:196341 10.15 10:afe499c9:::benchmark_data_smithi018_422431_object91845:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e621)", 2023-09-23T20:49:44.546 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:49:26.070140+0000", 2023-09-23T20:49:44.546 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 18.447498876000001, 2023-09-23T20:49:44.546 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0111852059999999, 2023-09-23T20:49:44.546 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:49:44.546 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:49:44.546 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:49:44.546 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:49:44.546 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:49:44.546 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 196341 2023-09-23T20:49:44.546 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.547 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:49:44.547 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.547 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:49:44.547 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:26.070140+0000", 2023-09-23T20:49:44.547 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:44.547 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.547 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.547 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:49:44.547 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:26.070140+0000", 2023-09-23T20:49:44.547 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:44.548 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.548 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.548 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:49:44.548 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:26.070138+0000", 2023-09-23T20:49:44.548 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:49:44.548 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.548 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.548 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:49:44.548 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:26.070150+0000", 2023-09-23T20:49:44.548 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1579999999999999e-05 2023-09-23T20:49:44.548 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.548 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.549 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:49:44.549 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:26.070175+0000", 2023-09-23T20:49:44.549 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.5225e-05 2023-09-23T20:49:44.549 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.549 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.549 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:49:44.549 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:26.070187+0000", 2023-09-23T20:49:44.549 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2425e-05 2023-09-23T20:49:44.549 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.549 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.549 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:44.549 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:26.070266+0000", 2023-09-23T20:49:44.550 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.8677999999999994e-05 2023-09-23T20:49:44.550 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.550 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.550 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:49:44.550 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:26.070287+0000", 2023-09-23T20:49:44.550 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.088e-05 2023-09-23T20:49:44.550 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.550 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.550 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:44.551 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:27.052026+0000", 2023-09-23T20:49:44.551 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.98173956799999995 2023-09-23T20:49:44.551 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.551 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.551 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:49:44.551 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:27.052172+0000", 2023-09-23T20:49:44.551 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000145332 2023-09-23T20:49:44.551 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.551 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.551 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 0,1,2,5", 2023-09-23T20:49:44.552 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:27.052417+0000", 2023-09-23T20:49:44.552 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00024565699999999998 2023-09-23T20:49:44.552 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.552 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.552 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:49:44.552 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:27.058987+0000", 2023-09-23T20:49:44.552 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0065694560000000004 2023-09-23T20:49:44.552 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.552 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.552 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:44.552 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:27.059392+0000", 2023-09-23T20:49:44.552 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000405456 2023-09-23T20:49:44.553 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.553 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.553 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:44.553 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:27.059492+0000", 2023-09-23T20:49:44.553 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.9839000000000004e-05 2023-09-23T20:49:44.553 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.553 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.553 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:44.553 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:27.063566+0000", 2023-09-23T20:49:44.553 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0040737719999999998 2023-09-23T20:49:44.553 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.554 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.554 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:44.554 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:27.081141+0000", 2023-09-23T20:49:44.554 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.017575047 2023-09-23T20:49:44.554 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.554 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.554 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:49:44.554 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:27.081192+0000", 2023-09-23T20:49:44.555 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.0691e-05 2023-09-23T20:49:44.555 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.555 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.555 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:49:44.555 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:27.081325+0000", 2023-09-23T20:49:44.555 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00013365499999999999 2023-09-23T20:49:44.555 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:44.556 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:49:44.556 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:44.556 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.556 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.556 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:196417 10.15 10:a8714d9d:::benchmark_data_smithi018_422431_object91921:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e621)", 2023-09-23T20:49:44.556 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:49:26.148867+0000", 2023-09-23T20:49:44.556 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 18.368771680999998, 2023-09-23T20:49:44.556 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.932723317, 2023-09-23T20:49:44.556 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:49:44.556 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:49:44.556 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:49:44.557 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:49:44.557 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:49:44.557 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 196417 2023-09-23T20:49:44.557 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.557 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:49:44.557 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.557 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:49:44.557 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:26.148867+0000", 2023-09-23T20:49:44.557 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:44.557 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.557 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.557 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:49:44.558 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:26.148867+0000", 2023-09-23T20:49:44.558 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:44.558 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.558 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.558 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:49:44.558 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:26.148866+0000", 2023-09-23T20:49:44.558 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:49:44.558 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.558 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.558 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:49:44.558 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:26.148874+0000", 2023-09-23T20:49:44.559 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.5089999999999993e-06 2023-09-23T20:49:44.559 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.559 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.559 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:49:44.559 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:26.148894+0000", 2023-09-23T20:49:44.559 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.9800999999999999e-05 2023-09-23T20:49:44.559 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.559 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.559 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:49:44.559 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:26.148904+0000", 2023-09-23T20:49:44.559 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0054000000000001e-05 2023-09-23T20:49:44.559 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.560 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.560 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:44.560 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:26.148974+0000", 2023-09-23T20:49:44.560 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.0118999999999997e-05 2023-09-23T20:49:44.560 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.560 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.560 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:49:44.560 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:26.148991+0000", 2023-09-23T20:49:44.561 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.7356000000000001e-05 2023-09-23T20:49:44.561 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.561 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.561 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:44.561 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:27.054238+0000", 2023-09-23T20:49:44.561 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.90524600700000002 2023-09-23T20:49:44.562 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.562 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.562 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:49:44.562 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:27.054357+0000", 2023-09-23T20:49:44.562 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000119836 2023-09-23T20:49:44.562 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.562 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.562 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 0,1,2,5", 2023-09-23T20:49:44.562 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:27.054614+0000", 2023-09-23T20:49:44.563 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000256751 2023-09-23T20:49:44.563 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.563 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.563 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:44.563 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:27.059524+0000", 2023-09-23T20:49:44.563 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0049100619999999998 2023-09-23T20:49:44.563 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.563 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.563 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:44.563 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:27.072127+0000", 2023-09-23T20:49:44.564 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.012602755 2023-09-23T20:49:44.564 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.564 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.564 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:44.564 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:27.072170+0000", 2023-09-23T20:49:44.564 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.3590000000000001e-05 2023-09-23T20:49:44.564 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.564 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.565 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:49:44.565 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:27.080346+0000", 2023-09-23T20:49:44.565 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0081756910000000006 2023-09-23T20:49:44.565 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.565 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.565 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:44.566 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:27.081391+0000", 2023-09-23T20:49:44.566 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0010447480000000001 2023-09-23T20:49:44.566 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.566 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.566 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:49:44.567 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:27.081450+0000", 2023-09-23T20:49:44.567 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.8644000000000003e-05 2023-09-23T20:49:44.567 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.567 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.567 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:49:44.567 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:27.081591+0000", 2023-09-23T20:49:44.568 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00014109499999999999 2023-09-23T20:49:44.568 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:44.568 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:49:44.568 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:44.568 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.568 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.569 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:196425 10.15 10:acf26cec:::benchmark_data_smithi018_422431_object91929:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e621)", 2023-09-23T20:49:44.569 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:49:26.153612+0000", 2023-09-23T20:49:44.569 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 18.364027107999998, 2023-09-23T20:49:44.569 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.92827135599999999, 2023-09-23T20:49:44.569 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:49:44.569 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:49:44.570 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:49:44.570 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:49:44.570 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:49:44.570 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 196425 2023-09-23T20:49:44.570 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.570 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:49:44.570 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.571 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:49:44.571 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:26.153612+0000", 2023-09-23T20:49:44.571 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:44.571 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.571 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.571 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:49:44.571 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:26.153612+0000", 2023-09-23T20:49:44.571 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:44.571 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.572 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.572 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:49:44.572 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:26.153610+0000", 2023-09-23T20:49:44.572 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:49:44.572 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.572 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.572 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:49:44.573 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:26.153618+0000", 2023-09-23T20:49:44.573 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.6159999999999996e-06 2023-09-23T20:49:44.573 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.573 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.573 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:49:44.573 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:26.153634+0000", 2023-09-23T20:49:44.573 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.6303e-05 2023-09-23T20:49:44.573 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.573 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.574 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:49:44.574 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:26.153643+0000", 2023-09-23T20:49:44.574 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.4110000000000002e-06 2023-09-23T20:49:44.574 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.574 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.574 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:44.574 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:26.153710+0000", 2023-09-23T20:49:44.574 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.6433999999999994e-05 2023-09-23T20:49:44.574 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.574 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.575 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:49:44.575 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:26.153722+0000", 2023-09-23T20:49:44.575 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2241e-05 2023-09-23T20:49:44.575 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.575 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.575 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:44.575 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:27.056288+0000", 2023-09-23T20:49:44.575 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.902566019 2023-09-23T20:49:44.575 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.575 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.575 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:49:44.575 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:27.056427+0000", 2023-09-23T20:49:44.575 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00013860500000000001 2023-09-23T20:49:44.576 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.576 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.576 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 0,1,2,5", 2023-09-23T20:49:44.576 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:27.056678+0000", 2023-09-23T20:49:44.576 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00025188899999999999 2023-09-23T20:49:44.576 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.576 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.576 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:44.576 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:27.061753+0000", 2023-09-23T20:49:44.576 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0050748470000000004 2023-09-23T20:49:44.576 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.576 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.577 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:44.577 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:27.072212+0000", 2023-09-23T20:49:44.577 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.010458291999999999 2023-09-23T20:49:44.577 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.577 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.577 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:44.577 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:27.072252+0000", 2023-09-23T20:49:44.577 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.0497999999999997e-05 2023-09-23T20:49:44.577 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.577 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.577 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:49:44.578 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:27.080357+0000", 2023-09-23T20:49:44.578 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.008105335 2023-09-23T20:49:44.578 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.578 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.578 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:44.578 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:27.081651+0000", 2023-09-23T20:49:44.578 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0012933300000000001 2023-09-23T20:49:44.578 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.578 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.578 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:49:44.578 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:27.081707+0000", 2023-09-23T20:49:44.579 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.6242000000000001e-05 2023-09-23T20:49:44.579 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.579 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.579 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:49:44.579 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:27.081883+0000", 2023-09-23T20:49:44.579 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00017624900000000001 2023-09-23T20:49:44.579 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:44.579 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:49:44.579 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:44.579 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.579 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.580 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:197303 10.15 10:ad2208f7:::benchmark_data_smithi018_422431_object92807:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e624)", 2023-09-23T20:49:44.580 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:49:29.084317+0000", 2023-09-23T20:49:44.580 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 15.433322142, 2023-09-23T20:49:44.580 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.93850479099999995, 2023-09-23T20:49:44.580 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:49:44.580 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:49:44.580 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:49:44.580 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:49:44.580 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:49:44.580 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 197303 2023-09-23T20:49:44.580 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.581 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:49:44.581 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.581 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:49:44.581 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.084317+0000", 2023-09-23T20:49:44.581 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:44.581 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.581 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.581 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:49:44.581 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.084317+0000", 2023-09-23T20:49:44.581 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:44.582 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.582 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.582 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:49:44.582 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.084315+0000", 2023-09-23T20:49:44.582 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:49:44.582 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.582 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.582 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:49:44.582 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.084323+0000", 2023-09-23T20:49:44.583 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.5240000000000003e-06 2023-09-23T20:49:44.583 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.583 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.583 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:49:44.583 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.084342+0000", 2023-09-23T20:49:44.583 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.9239e-05 2023-09-23T20:49:44.583 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.583 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.583 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:49:44.583 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.084352+0000", 2023-09-23T20:49:44.583 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.4239999999999999e-06 2023-09-23T20:49:44.584 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.584 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.584 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:44.584 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.084421+0000", 2023-09-23T20:49:44.584 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.9256000000000002e-05 2023-09-23T20:49:44.584 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.584 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.584 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:49:44.584 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.084438+0000", 2023-09-23T20:49:44.584 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.7303999999999999e-05 2023-09-23T20:49:44.584 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.584 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.585 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:44.585 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:30.016878+0000", 2023-09-23T20:49:44.585 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.93243949299999995 2023-09-23T20:49:44.585 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.585 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.585 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:49:44.585 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:30.016977+0000", 2023-09-23T20:49:44.585 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.8869000000000005e-05 2023-09-23T20:49:44.585 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.585 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.585 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,5", 2023-09-23T20:49:44.586 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:30.017343+0000", 2023-09-23T20:49:44.586 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00036639099999999998 2023-09-23T20:49:44.586 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.586 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.586 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:49:44.586 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:30.022629+0000", 2023-09-23T20:49:44.586 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0052859459999999997 2023-09-23T20:49:44.586 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.586 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.586 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:44.587 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:30.022684+0000", 2023-09-23T20:49:44.587 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.4645999999999998e-05 2023-09-23T20:49:44.587 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.587 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.587 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:44.587 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:30.022722+0000", 2023-09-23T20:49:44.587 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.8067e-05 2023-09-23T20:49:44.588 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.588 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.588 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:49:44.588 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:30.022774+0000", 2023-09-23T20:49:44.588 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.2099999999999999e-05 2023-09-23T20:49:44.589 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.589 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.589 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:49:44.589 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:30.022822+0000", 2023-09-23T20:49:44.589 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.7969000000000001e-05 2023-09-23T20:49:44.590 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:44.590 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:49:44.590 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:44.590 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.590 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.591 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:197309 10.15 10:ab14b822:::benchmark_data_smithi018_422431_object92813:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e624)", 2023-09-23T20:49:44.591 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:49:29.090092+0000", 2023-09-23T20:49:44.592 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 15.427546789999999, 2023-09-23T20:49:44.592 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.93297547000000003, 2023-09-23T20:49:44.593 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:49:44.593 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:49:44.593 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:49:44.593 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:49:44.593 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:49:44.594 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 197309 2023-09-23T20:49:44.594 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.594 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:49:44.594 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.594 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:49:44.595 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.090092+0000", 2023-09-23T20:49:44.595 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:44.595 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.595 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.595 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:49:44.595 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.090092+0000", 2023-09-23T20:49:44.596 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:44.596 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.596 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.596 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:49:44.596 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.090091+0000", 2023-09-23T20:49:44.596 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:49:44.597 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.597 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.597 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:49:44.597 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.090097+0000", 2023-09-23T20:49:44.597 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.7050000000000002e-06 2023-09-23T20:49:44.598 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.598 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.598 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:49:44.598 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.090111+0000", 2023-09-23T20:49:44.598 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3521e-05 2023-09-23T20:49:44.599 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.599 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.599 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:49:44.599 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.090118+0000", 2023-09-23T20:49:44.599 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.6370000000000002e-06 2023-09-23T20:49:44.599 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.599 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.600 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:44.600 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.090180+0000", 2023-09-23T20:49:44.600 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.2533000000000006e-05 2023-09-23T20:49:44.600 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.600 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.600 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:49:44.600 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.090192+0000", 2023-09-23T20:49:44.600 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2279999999999999e-05 2023-09-23T20:49:44.600 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.600 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.600 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:44.600 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:30.019150+0000", 2023-09-23T20:49:44.601 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.92895706899999997 2023-09-23T20:49:44.601 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.601 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.601 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:49:44.601 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:30.019290+0000", 2023-09-23T20:49:44.601 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000140321 2023-09-23T20:49:44.601 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.601 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.601 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,5", 2023-09-23T20:49:44.601 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:30.019545+0000", 2023-09-23T20:49:44.601 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000255157 2023-09-23T20:49:44.602 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.602 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.602 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:44.602 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:30.022848+0000", 2023-09-23T20:49:44.602 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0033031919999999999 2023-09-23T20:49:44.602 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.602 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.602 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:49:44.602 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:30.022955+0000", 2023-09-23T20:49:44.603 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000107209 2023-09-23T20:49:44.603 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.603 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.603 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:44.603 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:30.022985+0000", 2023-09-23T20:49:44.603 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.9998999999999999e-05 2023-09-23T20:49:44.603 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.603 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.603 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:49:44.603 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:30.023030+0000", 2023-09-23T20:49:44.603 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.5067000000000001e-05 2023-09-23T20:49:44.603 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.604 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.604 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:49:44.604 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:30.023068+0000", 2023-09-23T20:49:44.604 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.7259999999999999e-05 2023-09-23T20:49:44.604 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:44.604 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:49:44.604 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:44.604 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.604 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.604 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:197315 10.17 10:e8e8864f:::benchmark_data_smithi018_422431_object92819:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e625)", 2023-09-23T20:49:44.604 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:49:29.986565+0000", 2023-09-23T20:49:44.605 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 14.531074531, 2023-09-23T20:49:44.605 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1168361760000001, 2023-09-23T20:49:44.605 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:49:44.605 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:49:44.605 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:49:44.605 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:49:44.606 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:49:44.606 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 197315 2023-09-23T20:49:44.606 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.606 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:49:44.606 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.606 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:49:44.606 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986565+0000", 2023-09-23T20:49:44.606 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:44.606 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.606 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.606 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:49:44.607 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986565+0000", 2023-09-23T20:49:44.607 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:44.607 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.607 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.607 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:49:44.607 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986563+0000", 2023-09-23T20:49:44.607 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:49:44.607 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.607 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.607 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:49:44.607 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986572+0000", 2023-09-23T20:49:44.608 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.7299999999999994e-06 2023-09-23T20:49:44.608 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.608 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.608 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:49:44.608 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986588+0000", 2023-09-23T20:49:44.608 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.6004000000000001e-05 2023-09-23T20:49:44.608 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.608 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.608 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:49:44.608 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986597+0000", 2023-09-23T20:49:44.609 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.5820000000000008e-06 2023-09-23T20:49:44.609 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.609 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.609 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:44.609 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986662+0000", 2023-09-23T20:49:44.609 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.4540000000000002e-05 2023-09-23T20:49:44.609 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.609 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.609 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:49:44.609 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986675+0000", 2023-09-23T20:49:44.609 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.312e-05 2023-09-23T20:49:44.609 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.610 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.610 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:44.610 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.095946+0000", 2023-09-23T20:49:44.610 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.109270993 2023-09-23T20:49:44.610 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.610 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.610 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:49:44.610 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.096119+0000", 2023-09-23T20:49:44.610 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00017305700000000001 2023-09-23T20:49:44.610 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.611 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.611 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 4,5,6", 2023-09-23T20:49:44.611 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.096543+0000", 2023-09-23T20:49:44.611 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00042400599999999999 2023-09-23T20:49:44.611 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.611 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.611 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:44.611 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.103154+0000", 2023-09-23T20:49:44.611 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0066110129999999998 2023-09-23T20:49:44.611 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.611 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.611 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:49:44.612 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.103170+0000", 2023-09-23T20:49:44.612 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.5627000000000001e-05 2023-09-23T20:49:44.612 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.612 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.612 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:44.612 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.103233+0000", 2023-09-23T20:49:44.612 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.3027999999999994e-05 2023-09-23T20:49:44.612 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.612 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.613 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:44.613 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.103261+0000", 2023-09-23T20:49:44.613 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.8697999999999999e-05 2023-09-23T20:49:44.613 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.613 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.613 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:49:44.613 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.103298+0000", 2023-09-23T20:49:44.613 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.6585000000000003e-05 2023-09-23T20:49:44.613 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.613 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.613 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:49:44.614 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.103401+0000", 2023-09-23T20:49:44.614 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000102747 2023-09-23T20:49:44.614 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:44.614 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:49:44.614 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:44.614 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.614 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.614 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:197329 10.17 10:ee8e2f7f:::benchmark_data_smithi018_422431_object92833:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e625)", 2023-09-23T20:49:44.614 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:49:29.986611+0000", 2023-09-23T20:49:44.614 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 14.531028109999999, 2023-09-23T20:49:44.614 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.121231686, 2023-09-23T20:49:44.614 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:49:44.615 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:49:44.615 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:49:44.615 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:49:44.615 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:49:44.615 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 197329 2023-09-23T20:49:44.615 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.615 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:49:44.615 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.615 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:49:44.615 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986611+0000", 2023-09-23T20:49:44.615 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:44.616 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.616 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.616 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:49:44.616 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986611+0000", 2023-09-23T20:49:44.616 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:44.616 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.616 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.616 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:49:44.616 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986611+0000", 2023-09-23T20:49:44.616 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:49:44.616 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.616 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.617 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:49:44.617 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986613+0000", 2023-09-23T20:49:44.617 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.148e-06 2023-09-23T20:49:44.617 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.617 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.617 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:49:44.617 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986617+0000", 2023-09-23T20:49:44.617 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.2440000000000004e-06 2023-09-23T20:49:44.617 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.617 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.618 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:49:44.618 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986620+0000", 2023-09-23T20:49:44.618 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.0769999999999999e-06 2023-09-23T20:49:44.618 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.618 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.618 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:44.618 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986722+0000", 2023-09-23T20:49:44.618 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000102088 2023-09-23T20:49:44.618 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.618 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.618 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:49:44.619 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986733+0000", 2023-09-23T20:49:44.619 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0550000000000001e-05 2023-09-23T20:49:44.619 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.619 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.619 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:44.619 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.099437+0000", 2023-09-23T20:49:44.619 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1127045760000001 2023-09-23T20:49:44.619 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.619 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.619 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:49:44.619 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.099590+0000", 2023-09-23T20:49:44.620 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00015262000000000001 2023-09-23T20:49:44.620 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.620 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.620 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 4,5,6", 2023-09-23T20:49:44.620 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.099971+0000", 2023-09-23T20:49:44.620 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00038124200000000001 2023-09-23T20:49:44.620 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.620 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.620 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:49:44.621 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.103420+0000", 2023-09-23T20:49:44.621 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0034486019999999998 2023-09-23T20:49:44.621 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.621 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.621 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:44.621 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.103488+0000", 2023-09-23T20:49:44.622 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.8538e-05 2023-09-23T20:49:44.622 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.622 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.622 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:44.622 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.103781+0000", 2023-09-23T20:49:44.622 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00029315199999999997 2023-09-23T20:49:44.622 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.623 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.623 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:44.623 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.107701+0000", 2023-09-23T20:49:44.623 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.003919166 2023-09-23T20:49:44.623 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.623 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.623 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:49:44.624 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.107733+0000", 2023-09-23T20:49:44.624 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.2780000000000001e-05 2023-09-23T20:49:44.624 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.624 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.624 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:49:44.624 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.107843+0000", 2023-09-23T20:49:44.624 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000109232 2023-09-23T20:49:44.625 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:44.625 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:49:44.625 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:44.625 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.625 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.625 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:197351 10.17 10:ec920b81:::benchmark_data_smithi018_422431_object92855:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e625)", 2023-09-23T20:49:44.625 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:49:29.986649+0000", 2023-09-23T20:49:44.626 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 14.530990156, 2023-09-23T20:49:44.626 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1235907970000001, 2023-09-23T20:49:44.626 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:49:44.626 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:49:44.626 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:49:44.626 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:49:44.626 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:49:44.627 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 197351 2023-09-23T20:49:44.627 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.627 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:49:44.627 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.627 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:49:44.627 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986649+0000", 2023-09-23T20:49:44.627 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:44.628 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.628 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.628 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:49:44.628 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986649+0000", 2023-09-23T20:49:44.628 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:44.628 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.628 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.629 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:49:44.629 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986649+0000", 2023-09-23T20:49:44.629 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:49:44.629 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.629 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.629 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:49:44.629 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986651+0000", 2023-09-23T20:49:44.630 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.9800000000000001e-06 2023-09-23T20:49:44.630 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.630 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.630 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:49:44.630 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986656+0000", 2023-09-23T20:49:44.630 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.5249999999999996e-06 2023-09-23T20:49:44.630 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.630 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.630 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:49:44.631 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986659+0000", 2023-09-23T20:49:44.631 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.0570000000000001e-06 2023-09-23T20:49:44.631 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.631 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.631 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:44.631 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986776+0000", 2023-09-23T20:49:44.631 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000116338 2023-09-23T20:49:44.631 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.631 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.632 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:49:44.632 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986784+0000", 2023-09-23T20:49:44.632 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.0569999999999993e-06 2023-09-23T20:49:44.632 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.632 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.632 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:44.632 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.101416+0000", 2023-09-23T20:49:44.633 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.114632863 2023-09-23T20:49:44.633 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.633 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.633 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:49:44.633 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.101529+0000", 2023-09-23T20:49:44.633 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000112812 2023-09-23T20:49:44.633 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.634 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.634 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 4,5,6", 2023-09-23T20:49:44.634 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.101718+0000", 2023-09-23T20:49:44.634 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00018869399999999999 2023-09-23T20:49:44.634 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.634 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.634 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:49:44.634 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.103700+0000", 2023-09-23T20:49:44.634 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.001982466 2023-09-23T20:49:44.634 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.634 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.635 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:44.635 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.103742+0000", 2023-09-23T20:49:44.635 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.1378999999999998e-05 2023-09-23T20:49:44.635 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.635 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.635 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:44.635 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.103841+0000", 2023-09-23T20:49:44.635 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.9098000000000006e-05 2023-09-23T20:49:44.635 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.635 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.635 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:44.635 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.110043+0000", 2023-09-23T20:49:44.635 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0062020349999999998 2023-09-23T20:49:44.636 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.636 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.636 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:49:44.636 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.110104+0000", 2023-09-23T20:49:44.636 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.1477999999999997e-05 2023-09-23T20:49:44.636 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.636 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.636 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:49:44.636 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.110240+0000", 2023-09-23T20:49:44.636 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00013529699999999999 2023-09-23T20:49:44.636 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:44.636 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:49:44.637 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:44.637 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.637 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.637 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:197367 10.17 10:ebdac9cc:::benchmark_data_smithi018_422431_object92871:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e625)", 2023-09-23T20:49:44.637 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:49:29.986671+0000", 2023-09-23T20:49:44.637 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 14.530968261, 2023-09-23T20:49:44.637 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.123815188, 2023-09-23T20:49:44.637 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:49:44.637 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:49:44.637 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:49:44.637 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:49:44.637 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:49:44.637 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 197367 2023-09-23T20:49:44.638 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.638 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:49:44.638 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.638 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:49:44.638 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986671+0000", 2023-09-23T20:49:44.638 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:44.638 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.638 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.638 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:49:44.639 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986671+0000", 2023-09-23T20:49:44.639 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:44.639 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.639 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.639 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:49:44.639 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986671+0000", 2023-09-23T20:49:44.639 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967296 2023-09-23T20:49:44.640 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.640 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.640 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:49:44.640 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986672+0000", 2023-09-23T20:49:44.640 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.781e-06 2023-09-23T20:49:44.640 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.640 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.640 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:49:44.640 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986676+0000", 2023-09-23T20:49:44.640 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.4850000000000001e-06 2023-09-23T20:49:44.640 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.641 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.641 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:49:44.641 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986679+0000", 2023-09-23T20:49:44.641 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.7209999999999999e-06 2023-09-23T20:49:44.641 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.641 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.641 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:44.641 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986843+0000", 2023-09-23T20:49:44.641 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00016420599999999999 2023-09-23T20:49:44.641 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.641 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.641 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:49:44.642 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986854+0000", 2023-09-23T20:49:44.642 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1246e-05 2023-09-23T20:49:44.642 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.642 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.642 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:44.642 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.103866+0000", 2023-09-23T20:49:44.642 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.117011822 2023-09-23T20:49:44.642 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.642 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.642 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:49:44.642 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.103986+0000", 2023-09-23T20:49:44.642 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000119688 2023-09-23T20:49:44.643 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.643 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.643 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 4,5,6", 2023-09-23T20:49:44.643 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.104176+0000", 2023-09-23T20:49:44.643 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00019036199999999999 2023-09-23T20:49:44.643 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.643 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.643 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:44.643 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.107658+0000", 2023-09-23T20:49:44.643 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0034818079999999999 2023-09-23T20:49:44.643 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.643 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.644 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:49:44.644 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.107856+0000", 2023-09-23T20:49:44.644 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000197923 2023-09-23T20:49:44.644 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.644 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.644 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:44.644 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.109966+0000", 2023-09-23T20:49:44.644 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0021102199999999999 2023-09-23T20:49:44.644 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.644 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.644 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:44.644 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.110317+0000", 2023-09-23T20:49:44.645 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00035148699999999999 2023-09-23T20:49:44.645 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.645 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.645 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:49:44.645 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.110374+0000", 2023-09-23T20:49:44.645 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.7108999999999998e-05 2023-09-23T20:49:44.645 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.645 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.645 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:49:44.645 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.110486+0000", 2023-09-23T20:49:44.645 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000111556 2023-09-23T20:49:44.645 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:44.646 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:49:44.646 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:44.646 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.646 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.646 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:197369 10.17 10:edb5dbb2:::benchmark_data_smithi018_422431_object92873:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e625)", 2023-09-23T20:49:44.646 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:49:29.986681+0000", 2023-09-23T20:49:44.646 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 14.530958152, 2023-09-23T20:49:44.647 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1263334039999999, 2023-09-23T20:49:44.647 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:49:44.647 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:49:44.647 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:49:44.647 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:49:44.647 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:49:44.647 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 197369 2023-09-23T20:49:44.648 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.648 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:49:44.648 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.648 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:49:44.648 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986681+0000", 2023-09-23T20:49:44.648 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:44.648 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.649 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.649 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:49:44.649 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986681+0000", 2023-09-23T20:49:44.649 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:44.649 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.649 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.649 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:49:44.650 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986681+0000", 2023-09-23T20:49:44.650 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967296 2023-09-23T20:49:44.650 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.650 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.650 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:49:44.650 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986682+0000", 2023-09-23T20:49:44.651 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.4470000000000001e-06 2023-09-23T20:49:44.651 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.651 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.651 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:49:44.651 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986685+0000", 2023-09-23T20:49:44.651 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.0369999999999999e-06 2023-09-23T20:49:44.651 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.652 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.652 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:49:44.652 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986688+0000", 2023-09-23T20:49:44.652 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.723e-06 2023-09-23T20:49:44.652 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.652 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.652 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:44.653 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986899+0000", 2023-09-23T20:49:44.653 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00021081999999999999 2023-09-23T20:49:44.653 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.653 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.653 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:49:44.653 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986907+0000", 2023-09-23T20:49:44.653 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.3550000000000005e-06 2023-09-23T20:49:44.654 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.654 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.654 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:44.654 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.105384+0000", 2023-09-23T20:49:44.654 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1184772569999999 2023-09-23T20:49:44.654 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.654 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.655 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:49:44.655 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.105548+0000", 2023-09-23T20:49:44.655 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000163272 2023-09-23T20:49:44.655 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.655 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.655 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 4,5,6", 2023-09-23T20:49:44.655 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.105856+0000", 2023-09-23T20:49:44.655 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00030820400000000002 2023-09-23T20:49:44.656 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.656 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.656 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:49:44.656 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.107861+0000", 2023-09-23T20:49:44.656 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0020048219999999999 2023-09-23T20:49:44.656 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.656 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.657 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:44.657 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.109908+0000", 2023-09-23T20:49:44.657 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0020470589999999999 2023-09-23T20:49:44.657 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.657 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.657 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:44.657 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.109994+0000", 2023-09-23T20:49:44.657 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.6201999999999999e-05 2023-09-23T20:49:44.658 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.658 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.658 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:44.658 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.112824+0000", 2023-09-23T20:49:44.658 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0028306220000000001 2023-09-23T20:49:44.658 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.659 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.659 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:49:44.659 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.112891+0000", 2023-09-23T20:49:44.659 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.6707999999999999e-05 2023-09-23T20:49:44.659 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.659 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.659 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:49:44.659 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.113014+0000", 2023-09-23T20:49:44.660 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00012308999999999999 2023-09-23T20:49:44.660 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:44.660 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:49:44.660 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:44.660 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.660 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.661 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:197390 10.17 10:eece82d4:::benchmark_data_smithi018_422431_object92894:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e625)", 2023-09-23T20:49:44.661 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:49:29.986711+0000", 2023-09-23T20:49:44.661 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 14.53092773, 2023-09-23T20:49:44.661 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1286582460000001, 2023-09-23T20:49:44.661 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:49:44.661 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:49:44.661 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:49:44.661 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:49:44.661 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:49:44.662 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 197390 2023-09-23T20:49:44.662 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.662 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:49:44.662 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.662 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:49:44.662 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986711+0000", 2023-09-23T20:49:44.662 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:44.662 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.662 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.663 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:49:44.663 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986711+0000", 2023-09-23T20:49:44.663 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:44.663 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.663 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.663 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:49:44.664 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986711+0000", 2023-09-23T20:49:44.664 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967296 2023-09-23T20:49:44.664 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.664 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.664 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:49:44.664 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986714+0000", 2023-09-23T20:49:44.665 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.61e-06 2023-09-23T20:49:44.665 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.665 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.665 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:49:44.665 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986719+0000", 2023-09-23T20:49:44.665 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.271e-06 2023-09-23T20:49:44.665 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.666 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.666 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:49:44.666 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986722+0000", 2023-09-23T20:49:44.666 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.3239999999999999e-06 2023-09-23T20:49:44.666 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.666 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.666 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:44.666 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986950+0000", 2023-09-23T20:49:44.666 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00022735299999999999 2023-09-23T20:49:44.666 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.666 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.667 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:49:44.667 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986960+0000", 2023-09-23T20:49:44.667 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0332e-05 2023-09-23T20:49:44.667 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.667 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.667 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:44.667 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.107897+0000", 2023-09-23T20:49:44.667 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.120937311 2023-09-23T20:49:44.667 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.667 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.667 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:49:44.667 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.108040+0000", 2023-09-23T20:49:44.668 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00014265299999999999 2023-09-23T20:49:44.668 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.668 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.668 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 4,5,6", 2023-09-23T20:49:44.668 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.108307+0000", 2023-09-23T20:49:44.668 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00026726400000000002 2023-09-23T20:49:44.668 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.668 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.668 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:49:44.668 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.110505+0000", 2023-09-23T20:49:44.668 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0021981000000000001 2023-09-23T20:49:44.668 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.669 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.669 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:44.669 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.112767+0000", 2023-09-23T20:49:44.669 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.002261987 2023-09-23T20:49:44.669 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.669 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.669 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:44.669 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.115153+0000", 2023-09-23T20:49:44.669 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.002385318 2023-09-23T20:49:44.669 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.669 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.670 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:44.670 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.115189+0000", 2023-09-23T20:49:44.670 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.6455000000000002e-05 2023-09-23T20:49:44.670 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.670 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.670 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:49:44.670 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.115246+0000", 2023-09-23T20:49:44.670 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.6957999999999999e-05 2023-09-23T20:49:44.670 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.670 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.670 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:49:44.671 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.115370+0000", 2023-09-23T20:49:44.671 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00012352199999999999 2023-09-23T20:49:44.671 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:44.671 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:49:44.671 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:44.671 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.671 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.671 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:197413 10.17 10:eeb3ebd7:::benchmark_data_smithi018_422431_object92917:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e625)", 2023-09-23T20:49:44.671 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:49:29.986725+0000", 2023-09-23T20:49:44.671 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 14.530913902, 2023-09-23T20:49:44.671 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1335479159999999, 2023-09-23T20:49:44.671 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:49:44.672 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:49:44.672 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:49:44.672 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:49:44.672 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:49:44.672 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 197413 2023-09-23T20:49:44.672 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.672 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:49:44.672 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.672 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:49:44.672 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986725+0000", 2023-09-23T20:49:44.672 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:44.673 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.673 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.673 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:49:44.673 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986725+0000", 2023-09-23T20:49:44.673 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:44.673 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.673 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.673 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:49:44.673 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986725+0000", 2023-09-23T20:49:44.673 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967296 2023-09-23T20:49:44.673 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.673 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.674 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:49:44.674 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986727+0000", 2023-09-23T20:49:44.674 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.6160000000000001e-06 2023-09-23T20:49:44.674 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.674 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.674 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:49:44.674 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986730+0000", 2023-09-23T20:49:44.674 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.9900000000000002e-06 2023-09-23T20:49:44.674 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.674 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.674 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:49:44.675 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986732+0000", 2023-09-23T20:49:44.675 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.689e-06 2023-09-23T20:49:44.675 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.675 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.675 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:44.675 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.987000+0000", 2023-09-23T20:49:44.675 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00026816200000000001 2023-09-23T20:49:44.675 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.675 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.675 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:49:44.675 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.987010+0000", 2023-09-23T20:49:44.675 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.1239999999999994e-06 2023-09-23T20:49:44.675 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.676 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.676 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:44.676 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.110556+0000", 2023-09-23T20:49:44.676 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.123546964 2023-09-23T20:49:44.676 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.676 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.676 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:49:44.676 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.110727+0000", 2023-09-23T20:49:44.676 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00017081799999999999 2023-09-23T20:49:44.676 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.676 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.676 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 4,5,6", 2023-09-23T20:49:44.676 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.111048+0000", 2023-09-23T20:49:44.677 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00032095000000000002 2023-09-23T20:49:44.677 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.677 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.677 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:49:44.677 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.115106+0000", 2023-09-23T20:49:44.677 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0040579819999999999 2023-09-23T20:49:44.677 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.677 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.677 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:44.677 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.115413+0000", 2023-09-23T20:49:44.677 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00030677100000000001 2023-09-23T20:49:44.678 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.678 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.678 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:44.678 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.115448+0000", 2023-09-23T20:49:44.678 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.4625000000000001e-05 2023-09-23T20:49:44.678 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.678 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.678 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:44.678 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.120101+0000", 2023-09-23T20:49:44.678 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0046538680000000002 2023-09-23T20:49:44.679 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.679 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.679 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:49:44.679 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.120156+0000", 2023-09-23T20:49:44.679 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.4237999999999998e-05 2023-09-23T20:49:44.679 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.679 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.679 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:49:44.679 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.120273+0000", 2023-09-23T20:49:44.679 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000117324 2023-09-23T20:49:44.679 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:44.679 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:49:44.679 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:44.680 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.680 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.680 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:197441 10.17 10:ef3e749e:::benchmark_data_smithi018_422431_object92945:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e625)", 2023-09-23T20:49:44.680 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:49:29.986735+0000", 2023-09-23T20:49:44.680 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 14.530904502, 2023-09-23T20:49:44.680 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.135887265, 2023-09-23T20:49:44.680 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:49:44.680 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:49:44.680 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:49:44.680 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:49:44.680 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:49:44.680 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 197441 2023-09-23T20:49:44.681 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.681 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:49:44.681 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.681 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:49:44.681 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986735+0000", 2023-09-23T20:49:44.681 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:44.681 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.681 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.681 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:49:44.681 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986735+0000", 2023-09-23T20:49:44.681 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:44.682 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.682 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.682 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:49:44.682 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986734+0000", 2023-09-23T20:49:44.682 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967296 2023-09-23T20:49:44.682 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.682 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.682 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:49:44.682 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986736+0000", 2023-09-23T20:49:44.682 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.471e-06 2023-09-23T20:49:44.682 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.683 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.683 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:49:44.683 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986739+0000", 2023-09-23T20:49:44.683 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.7989999999999998e-06 2023-09-23T20:49:44.683 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.683 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.683 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:49:44.683 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986741+0000", 2023-09-23T20:49:44.683 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.785e-06 2023-09-23T20:49:44.683 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.683 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.684 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:44.684 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.987050+0000", 2023-09-23T20:49:44.684 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00030879399999999998 2023-09-23T20:49:44.684 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.684 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.684 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:49:44.684 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.987058+0000", 2023-09-23T20:49:44.684 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.5279999999999998e-06 2023-09-23T20:49:44.684 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.684 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.684 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:44.684 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.115473+0000", 2023-09-23T20:49:44.685 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.128414926 2023-09-23T20:49:44.685 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.685 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.685 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:49:44.685 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.115598+0000", 2023-09-23T20:49:44.685 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00012549999999999999 2023-09-23T20:49:44.685 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.685 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.685 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 4,5,6", 2023-09-23T20:49:44.685 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.115970+0000", 2023-09-23T20:49:44.685 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00037187500000000003 2023-09-23T20:49:44.685 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.686 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.686 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:49:44.686 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.120285+0000", 2023-09-23T20:49:44.686 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0043145809999999996 2023-09-23T20:49:44.686 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.686 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.686 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:44.686 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.120321+0000", 2023-09-23T20:49:44.686 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.6891000000000003e-05 2023-09-23T20:49:44.686 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.686 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.687 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:44.687 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.122405+0000", 2023-09-23T20:49:44.687 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0020831980000000001 2023-09-23T20:49:44.687 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.687 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.687 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:44.687 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.122441+0000", 2023-09-23T20:49:44.687 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.6417e-05 2023-09-23T20:49:44.687 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.687 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.687 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:49:44.687 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.122501+0000", 2023-09-23T20:49:44.688 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.0324e-05 2023-09-23T20:49:44.688 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.688 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.688 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:49:44.688 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.122622+0000", 2023-09-23T20:49:44.688 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000120376 2023-09-23T20:49:44.688 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:44.688 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:49:44.688 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:44.688 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.688 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.689 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:197474 10.17 10:e88d00d1:::benchmark_data_smithi018_422431_object92978:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e625)", 2023-09-23T20:49:44.689 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:49:29.986768+0000", 2023-09-23T20:49:44.689 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 14.530871402000001, 2023-09-23T20:49:44.689 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1406286880000001, 2023-09-23T20:49:44.689 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:49:44.689 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:49:44.689 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:49:44.689 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:49:44.689 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:49:44.689 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 197474 2023-09-23T20:49:44.689 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.690 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:49:44.690 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.690 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:49:44.690 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986768+0000", 2023-09-23T20:49:44.690 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:44.690 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.690 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.690 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:49:44.690 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986768+0000", 2023-09-23T20:49:44.690 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:44.690 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.691 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.691 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:49:44.691 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986767+0000", 2023-09-23T20:49:44.691 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967296 2023-09-23T20:49:44.691 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.691 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.691 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:49:44.691 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986769+0000", 2023-09-23T20:49:44.691 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.4699999999999999e-06 2023-09-23T20:49:44.691 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.691 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.692 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:49:44.692 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986774+0000", 2023-09-23T20:49:44.692 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.4829999999999999e-06 2023-09-23T20:49:44.692 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.692 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.692 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:49:44.692 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986777+0000", 2023-09-23T20:49:44.692 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.0819999999999999e-06 2023-09-23T20:49:44.692 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.692 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.692 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:44.692 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.987089+0000", 2023-09-23T20:49:44.693 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00031171900000000002 2023-09-23T20:49:44.693 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.693 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.693 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:49:44.693 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.987098+0000", 2023-09-23T20:49:44.693 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.6750000000000008e-06 2023-09-23T20:49:44.693 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.693 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.693 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:44.693 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.117835+0000", 2023-09-23T20:49:44.693 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1307371669999999 2023-09-23T20:49:44.693 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.694 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.694 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:49:44.694 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.117952+0000", 2023-09-23T20:49:44.694 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00011728800000000001 2023-09-23T20:49:44.694 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.694 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.694 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 4,5,6", 2023-09-23T20:49:44.694 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.118255+0000", 2023-09-23T20:49:44.694 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00030264199999999999 2023-09-23T20:49:44.694 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.694 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.694 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:44.695 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.122668+0000", 2023-09-23T20:49:44.695 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.004413045 2023-09-23T20:49:44.695 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.695 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.695 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:44.695 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.122705+0000", 2023-09-23T20:49:44.695 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.6677999999999997e-05 2023-09-23T20:49:44.695 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.695 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.695 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:44.695 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.127195+0000", 2023-09-23T20:49:44.695 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0044907059999999997 2023-09-23T20:49:44.695 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.696 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.696 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:49:44.696 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.127204+0000", 2023-09-23T20:49:44.696 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.4989999999999994e-06 2023-09-23T20:49:44.696 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.696 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.696 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:49:44.696 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.127258+0000", 2023-09-23T20:49:44.696 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.4271000000000003e-05 2023-09-23T20:49:44.696 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.696 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.697 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:49:44.697 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.127396+0000", 2023-09-23T20:49:44.697 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00013817500000000001 2023-09-23T20:49:44.697 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:44.697 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:49:44.697 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:44.697 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.697 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.697 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:197499 10.17 10:e84c8c26:::benchmark_data_smithi018_422431_object93003:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e625)", 2023-09-23T20:49:44.697 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:49:30.051645+0000", 2023-09-23T20:49:44.697 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 14.465994488, 2023-09-23T20:49:44.697 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0808529579999999, 2023-09-23T20:49:44.697 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:49:44.697 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:49:44.698 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:49:44.698 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:49:44.698 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:49:44.698 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 197499 2023-09-23T20:49:44.698 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.698 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:49:44.698 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.698 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:49:44.698 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:30.051645+0000", 2023-09-23T20:49:44.698 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:44.698 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.699 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.699 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:49:44.699 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:30.051645+0000", 2023-09-23T20:49:44.699 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:44.699 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.699 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.699 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:49:44.699 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:30.051643+0000", 2023-09-23T20:49:44.699 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:49:44.699 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.699 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.700 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:49:44.700 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:30.051653+0000", 2023-09-23T20:49:44.700 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.9990000000000003e-06 2023-09-23T20:49:44.700 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.700 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.700 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:49:44.700 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:30.051677+0000", 2023-09-23T20:49:44.700 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.4777e-05 2023-09-23T20:49:44.700 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.700 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.700 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:49:44.700 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:30.051688+0000", 2023-09-23T20:49:44.700 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0662999999999999e-05 2023-09-23T20:49:44.701 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.701 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.701 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:44.701 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:30.051753+0000", 2023-09-23T20:49:44.701 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.4968000000000005e-05 2023-09-23T20:49:44.701 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.701 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.701 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:49:44.701 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:30.051767+0000", 2023-09-23T20:49:44.701 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.4378e-05 2023-09-23T20:49:44.701 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.701 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.702 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:44.702 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.122728+0000", 2023-09-23T20:49:44.702 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0709603510000001 2023-09-23T20:49:44.702 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.702 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.702 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:49:44.702 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.122885+0000", 2023-09-23T20:49:44.702 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00015751900000000001 2023-09-23T20:49:44.702 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.702 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.702 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 4,5,6", 2023-09-23T20:49:44.703 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.123294+0000", 2023-09-23T20:49:44.703 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000408537 2023-09-23T20:49:44.703 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.703 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.703 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:49:44.703 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.127454+0000", 2023-09-23T20:49:44.703 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0041605970000000003 2023-09-23T20:49:44.703 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.703 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.703 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:44.703 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.127507+0000", 2023-09-23T20:49:44.703 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.2753999999999997e-05 2023-09-23T20:49:44.703 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.704 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.704 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:44.704 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.127569+0000", 2023-09-23T20:49:44.704 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.1624000000000005e-05 2023-09-23T20:49:44.704 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.704 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.704 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:44.704 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.132306+0000", 2023-09-23T20:49:44.704 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0047368610000000002 2023-09-23T20:49:44.704 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.704 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.704 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:49:44.705 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.132373+0000", 2023-09-23T20:49:44.705 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.6983999999999994e-05 2023-09-23T20:49:44.705 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.705 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.705 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:49:44.705 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.132498+0000", 2023-09-23T20:49:44.705 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00012484400000000001 2023-09-23T20:49:44.705 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:44.705 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:49:44.705 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:44.705 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.705 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.706 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:197519 10.17 10:e96e5c55:::benchmark_data_smithi018_422431_object93023:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e625)", 2023-09-23T20:49:44.706 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:49:30.075323+0000", 2023-09-23T20:49:44.706 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 14.44231622, 2023-09-23T20:49:44.706 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.059203133, 2023-09-23T20:49:44.706 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:49:44.706 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:49:44.706 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:49:44.706 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:49:44.706 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:49:44.706 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 197519 2023-09-23T20:49:44.706 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.707 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:49:44.707 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.707 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:49:44.707 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:30.075323+0000", 2023-09-23T20:49:44.707 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:44.707 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.707 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.707 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:49:44.707 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:30.075323+0000", 2023-09-23T20:49:44.707 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:44.707 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.707 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.708 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:49:44.708 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:30.075322+0000", 2023-09-23T20:49:44.708 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:49:44.708 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.708 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.708 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:49:44.708 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:30.075328+0000", 2023-09-23T20:49:44.708 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.8800000000000002e-06 2023-09-23T20:49:44.708 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.708 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.708 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:49:44.709 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:30.075344+0000", 2023-09-23T20:49:44.709 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.5234e-05 2023-09-23T20:49:44.709 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.709 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.709 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:49:44.709 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:30.075354+0000", 2023-09-23T20:49:44.709 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.8989999999999996e-06 2023-09-23T20:49:44.709 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.709 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.709 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:44.709 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:30.076863+0000", 2023-09-23T20:49:44.710 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.001509385 2023-09-23T20:49:44.710 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.710 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.710 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:49:44.710 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:30.076879+0000", 2023-09-23T20:49:44.710 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.6299000000000001e-05 2023-09-23T20:49:44.711 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.711 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.711 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:44.711 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.125198+0000", 2023-09-23T20:49:44.711 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0483189749999999 2023-09-23T20:49:44.711 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.711 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.711 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:49:44.711 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.125315+0000", 2023-09-23T20:49:44.711 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000116768 2023-09-23T20:49:44.711 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.711 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.711 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 4,5,6", 2023-09-23T20:49:44.712 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.125626+0000", 2023-09-23T20:49:44.712 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000310616 2023-09-23T20:49:44.712 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.712 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.712 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:49:44.712 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.127519+0000", 2023-09-23T20:49:44.712 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.001893582 2023-09-23T20:49:44.712 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.712 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.712 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:44.712 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.132556+0000", 2023-09-23T20:49:44.712 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0050365490000000004 2023-09-23T20:49:44.712 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.713 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.713 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:44.713 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.134317+0000", 2023-09-23T20:49:44.713 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0017612179999999999 2023-09-23T20:49:44.713 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.713 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.713 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:44.713 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.134355+0000", 2023-09-23T20:49:44.713 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.7691000000000002e-05 2023-09-23T20:49:44.713 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.713 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.713 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:49:44.714 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.134412+0000", 2023-09-23T20:49:44.714 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.7441000000000001e-05 2023-09-23T20:49:44.714 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.714 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.714 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:49:44.714 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.134526+0000", 2023-09-23T20:49:44.714 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000113888 2023-09-23T20:49:44.714 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:44.714 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:49:44.714 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:44.714 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.714 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.715 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:197537 10.17 10:ef38087a:::benchmark_data_smithi018_422431_object93041:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e625)", 2023-09-23T20:49:44.715 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:49:30.103092+0000", 2023-09-23T20:49:44.715 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 14.414546827000001, 2023-09-23T20:49:44.715 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0365641029999999, 2023-09-23T20:49:44.715 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:49:44.715 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:49:44.715 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:49:44.715 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:49:44.715 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:49:44.715 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 197537 2023-09-23T20:49:44.715 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.715 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:49:44.716 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.716 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:49:44.716 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:30.103092+0000", 2023-09-23T20:49:44.716 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:44.716 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.716 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.716 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:49:44.716 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:30.103092+0000", 2023-09-23T20:49:44.716 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:49:44.716 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.716 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.716 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:49:44.717 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:30.103091+0000", 2023-09-23T20:49:44.717 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:49:44.717 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.717 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.717 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:49:44.717 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:30.103096+0000", 2023-09-23T20:49:44.717 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.4739999999999996e-06 2023-09-23T20:49:44.717 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.717 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.717 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:49:44.717 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:30.103109+0000", 2023-09-23T20:49:44.717 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2785e-05 2023-09-23T20:49:44.717 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.718 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.718 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:49:44.718 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:30.103115+0000", 2023-09-23T20:49:44.718 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.0129999999999999e-06 2023-09-23T20:49:44.718 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.718 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.718 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:44.718 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:30.103181+0000", 2023-09-23T20:49:44.718 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.5869000000000002e-05 2023-09-23T20:49:44.718 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.718 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.719 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:49:44.719 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:30.103195+0000", 2023-09-23T20:49:44.719 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.4426999999999999e-05 2023-09-23T20:49:44.719 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.719 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.719 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:49:44.719 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.127600+0000", 2023-09-23T20:49:44.719 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.024404724 2023-09-23T20:49:44.719 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.719 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.719 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:49:44.719 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.127751+0000", 2023-09-23T20:49:44.720 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00015088300000000001 2023-09-23T20:49:44.720 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.720 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.720 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 4,5,6", 2023-09-23T20:49:44.720 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.128077+0000", 2023-09-23T20:49:44.720 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00032595299999999997 2023-09-23T20:49:44.720 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.720 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.720 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:49:44.720 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.132514+0000", 2023-09-23T20:49:44.720 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0044368059999999997 2023-09-23T20:49:44.720 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.721 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.721 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:44.721 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.134565+0000", 2023-09-23T20:49:44.721 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0020515899999999998 2023-09-23T20:49:44.721 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.721 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.721 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:44.721 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.139364+0000", 2023-09-23T20:49:44.721 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0047984480000000003 2023-09-23T20:49:44.721 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.722 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.722 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:44.722 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.139474+0000", 2023-09-23T20:49:44.722 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000110575 2023-09-23T20:49:44.722 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.722 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.722 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:49:44.722 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.139518+0000", 2023-09-23T20:49:44.722 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.3584000000000001e-05 2023-09-23T20:49:44.722 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:49:44.723 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:49:44.723 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:49:44.723 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.139656+0000", 2023-09-23T20:49:44.723 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000138257 2023-09-23T20:49:44.723 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:44.723 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:49:44.723 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:44.723 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:49:44.723 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:49:44.723 INFO:teuthology.orchestra.run.smithi018.stdout:} 2023-09-23T20:49:44.724 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:44.724 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:44.724 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:49:44.725 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:49:44.825 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:49:44.874 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:44.935 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:49:44.940 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:44.940 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:49:44.997 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:49:44 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:49:44] "GET /metrics HTTP/1.1" 200 36576 "" "Prometheus/2.43.0" 2023-09-23T20:49:45.041 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:49:45.111 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:45.205 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:49:45.213 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:45.214 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:49:45.214 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:45 smithi149 ceph-mon[170394]: 10.8 deep-scrub starts 2023-09-23T20:49:45.214 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:45 smithi149 ceph-mon[170394]: 10.8 deep-scrub ok 2023-09-23T20:49:45.214 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:45 smithi149 ceph-mon[170394]: failed to encode map e635 with expected crc 2023-09-23T20:49:45.214 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:45 smithi149 ceph-mon[170394]: osdmap e635: 8 total, 8 up, 8 in 2023-09-23T20:49:45.214 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:45 smithi149 ceph-mon[170394]: failed to encode map e635 with expected crc 2023-09-23T20:49:45.215 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:45 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:49:45.215 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:45 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:49:45.314 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:49:45.339 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:45 smithi018 ceph-mon[184144]: 10.8 deep-scrub starts 2023-09-23T20:49:45.339 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:45 smithi018 ceph-mon[184144]: 10.8 deep-scrub ok 2023-09-23T20:49:45.339 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:45 smithi018 ceph-mon[184144]: failed to encode map e635 with expected crc 2023-09-23T20:49:45.339 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:45 smithi018 ceph-mon[184144]: osdmap e635: 8 total, 8 up, 8 in 2023-09-23T20:49:45.339 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:45 smithi018 ceph-mon[184144]: failed to encode map e635 with expected crc 2023-09-23T20:49:45.340 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:45 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:49:45.340 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:45 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:49:45.340 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:45 smithi018 ceph-mon[179647]: 10.8 deep-scrub starts 2023-09-23T20:49:45.340 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:45 smithi018 ceph-mon[179647]: 10.8 deep-scrub ok 2023-09-23T20:49:45.340 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:45 smithi018 ceph-mon[179647]: failed to encode map e635 with expected crc 2023-09-23T20:49:45.340 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:45 smithi018 ceph-mon[179647]: osdmap e635: 8 total, 8 up, 8 in 2023-09-23T20:49:45.340 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:45 smithi018 ceph-mon[179647]: failed to encode map e635 with expected crc 2023-09-23T20:49:45.340 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:45 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:49:45.341 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:45 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:49:45.365 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:45.432 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:49:45.440 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:45.441 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:49:45.542 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:49:45.637 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:45.742 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:49:45.788 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:49:45.834 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:49:45.878 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:49:45.900 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:49:45.909 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:45.910 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:49:45.924 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:49:45.926 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T20:49:45.974 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:49:46.010 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:49:46.014 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:49:46.055 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:49:46.059 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:49:46.059 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:49:46.059 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:49:46.059 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:49:46.059 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:49:46.059 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:49:46.059 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:49:46.060 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:49:46.061 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Removed 104493 objects 2023-09-23T20:49:46.061 INFO:tasks.radosbench.radosbench.0.smithi018.stdout:Clean up completed and total clean up time :150.022 2023-09-23T20:49:46.068 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:46.102 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:49:46.105 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:46.106 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:49:46.207 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:49:46.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:46 smithi149 ceph-mon[170394]: pgmap v1560: 129 pgs: 1 peering, 128 active+clean; 257 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T20:49:46.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:46 smithi149 ceph-mon[170394]: 10.11 scrub starts 2023-09-23T20:49:46.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:46 smithi149 ceph-mon[170394]: 10.11 scrub ok 2023-09-23T20:49:46.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:46 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3109913708' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:49:46.234 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:46.261 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:46 smithi018 ceph-mon[184144]: pgmap v1560: 129 pgs: 1 peering, 128 active+clean; 257 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T20:49:46.262 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:46 smithi018 ceph-mon[184144]: 10.11 scrub starts 2023-09-23T20:49:46.262 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:46 smithi018 ceph-mon[184144]: 10.11 scrub ok 2023-09-23T20:49:46.262 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:46 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3109913708' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:49:46.262 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:46 smithi018 ceph-mon[179647]: pgmap v1560: 129 pgs: 1 peering, 128 active+clean; 257 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail 2023-09-23T20:49:46.262 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:46 smithi018 ceph-mon[179647]: 10.11 scrub starts 2023-09-23T20:49:46.262 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:46 smithi018 ceph-mon[179647]: 10.11 scrub ok 2023-09-23T20:49:46.262 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:46 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3109913708' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:49:46.268 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:49:46.271 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:46.271 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:49:46.372 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:49:46.401 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:46.500 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:49:46.502 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:46.503 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:49:46.574 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:49:46.603 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:49:46.629 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:46 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mon-a[179623]: 2023-09-23T20:49:46.259+0000 7f160fef9700 -1 mon.a@0(leader).osd e635 definitely_dead 0 2023-09-23T20:49:46.632 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:46.665 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:49:46.668 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:46.668 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:49:46.769 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:49:46.797 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:46.833 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:49:46.837 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:46.837 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:49:46.938 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:49:46.967 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:47.053 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:49:47.056 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:47.057 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:49:47.109 INFO:teuthology.orchestra.run.smithi018.stderr:marked out osd.5. 2023-09-23T20:49:47.111 INFO:teuthology.orchestra.run.smithi018.stderr:noscrub is set 2023-09-23T20:49:47.158 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:49:47.186 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:47.222 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:49:47.225 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:47.226 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:49:47.327 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:49:47.353 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:47 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/302870698' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2023-09-23T20:49:47.353 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:47 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2023-09-23T20:49:47.353 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:47 smithi018 ceph-mon[179647]: Client client.admin marked osd.5 out, while it was still marked up 2023-09-23T20:49:47.354 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:47 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/4162482068' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:49:47.354 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:47 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:49:47.354 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:47 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/302870698' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2023-09-23T20:49:47.354 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:47 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2023-09-23T20:49:47.355 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:47 smithi018 ceph-mon[184144]: Client client.admin marked osd.5 out, while it was still marked up 2023-09-23T20:49:47.355 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:47 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/4162482068' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:49:47.355 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:47 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:49:47.356 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:47.431 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:47 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/302870698' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2023-09-23T20:49:47.432 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:47 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2023-09-23T20:49:47.432 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:47 smithi149 ceph-mon[170394]: Client client.admin marked osd.5 out, while it was still marked up 2023-09-23T20:49:47.432 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:47 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/4162482068' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:49:47.432 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:47 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:49:47.895 INFO:teuthology.orchestra.run.smithi149.stdout:ERROR: (22) Invalid argument 2023-09-23T20:49:47.896 INFO:teuthology.orchestra.run.smithi149.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:49:48.174 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:49:48.177 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:48.178 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:49:48.278 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:49:48.307 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:48.323 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:48 smithi149 ceph-mon[170394]: pgmap v1561: 129 pgs: 1 peering, 128 active+clean; 96 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail 2023-09-23T20:49:48.323 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:48 smithi149 ceph-mon[170394]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:49:48.323 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:48 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["5"]}]': finished 2023-09-23T20:49:48.323 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:48 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:49:48.323 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:48 smithi149 ceph-mon[170394]: osdmap e636: 8 total, 8 up, 7 in 2023-09-23T20:49:48.323 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:48 smithi149 ceph-mon[170394]: failed to encode map e636 with expected crc 2023-09-23T20:49:48.323 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:48 smithi149 ceph-mon[170394]: failed to encode map e636 with expected crc 2023-09-23T20:49:48.323 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:48 smithi149 ceph-mon[170394]: failed to encode map e636 with expected crc 2023-09-23T20:49:48.323 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:48 smithi149 ceph-mon[170394]: failed to encode map e636 with expected crc 2023-09-23T20:49:48.323 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:48 smithi149 ceph-mon[170394]: failed to encode map e636 with expected crc 2023-09-23T20:49:48.324 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:48 smithi149 ceph-mon[170394]: failed to encode map e636 with expected crc 2023-09-23T20:49:48.324 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:48 smithi149 ceph-mon[170394]: failed to encode map e636 with expected crc 2023-09-23T20:49:48.324 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:48 smithi149 ceph-mon[170394]: failed to encode map e636 with expected crc 2023-09-23T20:49:48.324 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:48 smithi149 ceph-mon[170394]: failed to encode map e636 with expected crc 2023-09-23T20:49:48.324 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:48 smithi149 ceph-mon[170394]: failed to encode map e636 with expected crc 2023-09-23T20:49:48.325 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T20:49:48.341 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:49:48.344 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:48 smithi018 ceph-mon[184144]: pgmap v1561: 129 pgs: 1 peering, 128 active+clean; 96 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail 2023-09-23T20:49:48.344 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:48 smithi018 ceph-mon[184144]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:49:48.344 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:48 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["5"]}]': finished 2023-09-23T20:49:48.344 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:48 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:49:48.344 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:48 smithi018 ceph-mon[184144]: osdmap e636: 8 total, 8 up, 7 in 2023-09-23T20:49:48.344 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:48 smithi018 ceph-mon[184144]: failed to encode map e636 with expected crc 2023-09-23T20:49:48.344 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:48 smithi018 ceph-mon[184144]: failed to encode map e636 with expected crc 2023-09-23T20:49:48.344 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:48 smithi018 ceph-mon[184144]: failed to encode map e636 with expected crc 2023-09-23T20:49:48.344 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:48 smithi018 ceph-mon[184144]: failed to encode map e636 with expected crc 2023-09-23T20:49:48.344 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:48 smithi018 ceph-mon[184144]: failed to encode map e636 with expected crc 2023-09-23T20:49:48.345 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:48 smithi018 ceph-mon[184144]: failed to encode map e636 with expected crc 2023-09-23T20:49:48.345 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:48 smithi018 ceph-mon[184144]: failed to encode map e636 with expected crc 2023-09-23T20:49:48.345 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:48 smithi018 ceph-mon[184144]: failed to encode map e636 with expected crc 2023-09-23T20:49:48.345 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:48 smithi018 ceph-mon[184144]: failed to encode map e636 with expected crc 2023-09-23T20:49:48.345 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:48 smithi018 ceph-mon[184144]: failed to encode map e636 with expected crc 2023-09-23T20:49:48.345 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:48 smithi018 ceph-mon[179647]: pgmap v1561: 129 pgs: 1 peering, 128 active+clean; 96 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail 2023-09-23T20:49:48.345 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:48 smithi018 ceph-mon[179647]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:49:48.345 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:48 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["5"]}]': finished 2023-09-23T20:49:48.345 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:48 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:49:48.346 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:48 smithi018 ceph-mon[179647]: osdmap e636: 8 total, 8 up, 7 in 2023-09-23T20:49:48.346 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:48 smithi018 ceph-mon[179647]: failed to encode map e636 with expected crc 2023-09-23T20:49:48.346 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:48 smithi018 ceph-mon[179647]: failed to encode map e636 with expected crc 2023-09-23T20:49:48.346 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:48 smithi018 ceph-mon[179647]: failed to encode map e636 with expected crc 2023-09-23T20:49:48.346 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:48 smithi018 ceph-mon[179647]: failed to encode map e636 with expected crc 2023-09-23T20:49:48.346 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:48 smithi018 ceph-mon[179647]: failed to encode map e636 with expected crc 2023-09-23T20:49:48.346 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:48 smithi018 ceph-mon[179647]: failed to encode map e636 with expected crc 2023-09-23T20:49:48.346 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:48 smithi018 ceph-mon[179647]: failed to encode map e636 with expected crc 2023-09-23T20:49:48.346 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:48 smithi018 ceph-mon[179647]: failed to encode map e636 with expected crc 2023-09-23T20:49:48.346 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:48 smithi018 ceph-mon[179647]: failed to encode map e636 with expected crc 2023-09-23T20:49:48.346 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:48 smithi018 ceph-mon[179647]: failed to encode map e636 with expected crc 2023-09-23T20:49:48.347 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:48.347 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:49:48.448 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:49:48.474 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:48.496 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:49:48.513 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:49:48.516 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:48.516 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:49:48.617 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:49:48.643 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:48.679 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:49:48.682 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:48.682 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:49:48.783 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:49:48.810 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:48.846 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:49:48.849 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:48.850 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:49:48.942 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:49:48.951 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:49:48.968 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_5 2023-09-23T20:49:48.968 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd pool rm unique_pool_5 unique_pool_5 --yes-i-really-really-mean-it 2023-09-23T20:49:48.980 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:49.016 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:49:49.019 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:49.019 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:49:49.102 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:49 smithi018 ceph-mon[184144]: failed to encode map e637 with expected crc 2023-09-23T20:49:49.102 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:49 smithi018 ceph-mon[184144]: osdmap e637: 8 total, 8 up, 7 in 2023-09-23T20:49:49.102 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:49 smithi018 ceph-mon[184144]: failed to encode map e637 with expected crc 2023-09-23T20:49:49.102 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:49 smithi018 ceph-mon[184144]: failed to encode map e637 with expected crc 2023-09-23T20:49:49.103 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:49 smithi018 ceph-mon[184144]: failed to encode map e637 with expected crc 2023-09-23T20:49:49.103 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:49 smithi018 ceph-mon[184144]: failed to encode map e637 with expected crc 2023-09-23T20:49:49.103 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:49 smithi018 ceph-mon[184144]: failed to encode map e637 with expected crc 2023-09-23T20:49:49.103 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:49 smithi018 ceph-mon[184144]: failed to encode map e637 with expected crc 2023-09-23T20:49:49.115 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:49:49.120 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:49:49.260 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:49.330 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:49:49.334 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:49.335 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:49:49.360 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:49 smithi018 ceph-mon[179647]: failed to encode map e637 with expected crc 2023-09-23T20:49:49.360 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:49 smithi018 ceph-mon[179647]: osdmap e637: 8 total, 8 up, 7 in 2023-09-23T20:49:49.360 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:49 smithi018 ceph-mon[179647]: failed to encode map e637 with expected crc 2023-09-23T20:49:49.360 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:49 smithi018 ceph-mon[179647]: failed to encode map e637 with expected crc 2023-09-23T20:49:49.360 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:49 smithi018 ceph-mon[179647]: failed to encode map e637 with expected crc 2023-09-23T20:49:49.360 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:49 smithi018 ceph-mon[179647]: failed to encode map e637 with expected crc 2023-09-23T20:49:49.360 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:49 smithi018 ceph-mon[179647]: failed to encode map e637 with expected crc 2023-09-23T20:49:49.361 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:49 smithi018 ceph-mon[179647]: failed to encode map e637 with expected crc 2023-09-23T20:49:49.436 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:49:49.461 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:49 smithi149 ceph-mon[170394]: failed to encode map e637 with expected crc 2023-09-23T20:49:49.461 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:49 smithi149 ceph-mon[170394]: osdmap e637: 8 total, 8 up, 7 in 2023-09-23T20:49:49.461 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:49 smithi149 ceph-mon[170394]: failed to encode map e637 with expected crc 2023-09-23T20:49:49.461 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:49 smithi149 ceph-mon[170394]: failed to encode map e637 with expected crc 2023-09-23T20:49:49.461 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:49 smithi149 ceph-mon[170394]: failed to encode map e637 with expected crc 2023-09-23T20:49:49.462 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:49 smithi149 ceph-mon[170394]: failed to encode map e637 with expected crc 2023-09-23T20:49:49.462 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:49 smithi149 ceph-mon[170394]: failed to encode map e637 with expected crc 2023-09-23T20:49:49.462 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:49 smithi149 ceph-mon[170394]: failed to encode map e637 with expected crc 2023-09-23T20:49:49.464 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:49.499 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:49:49.503 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:49.503 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:49:49.604 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:49:49.633 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:49.825 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:49:49.828 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:49.829 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:49:49.930 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:49:49.961 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:49.991 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:49:49.999 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:49:50.003 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:50.004 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:49:50.105 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:49:50.143 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:50.183 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:49:50.186 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:50.186 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:49:50.214 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:50 smithi018 ceph-mon[184144]: pgmap v1564: 129 pgs: 1 peering, 128 active+clean; 62 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2023-09-23T20:49:50.215 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:50 smithi018 ceph-mon[184144]: 10.9 starting backfill to osd.4 from (0'0,0'0] MAX to 635'11956 2023-09-23T20:49:50.215 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:50 smithi018 ceph-mon[184144]: 10.3 starting backfill to osd.4 from (0'0,0'0] MAX to 635'12689 2023-09-23T20:49:50.215 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:50 smithi018 ceph-mon[184144]: failed to encode map e638 with expected crc 2023-09-23T20:49:50.215 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:50 smithi018 ceph-mon[184144]: failed to encode map e638 with expected crc 2023-09-23T20:49:50.215 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:50 smithi018 ceph-mon[184144]: failed to encode map e638 with expected crc 2023-09-23T20:49:50.215 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:50 smithi018 ceph-mon[184144]: osdmap e638: 8 total, 8 up, 7 in 2023-09-23T20:49:50.215 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:50 smithi018 ceph-mon[184144]: failed to encode map e638 with expected crc 2023-09-23T20:49:50.215 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:50 smithi018 ceph-mon[184144]: failed to encode map e638 with expected crc 2023-09-23T20:49:50.215 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:50 smithi018 ceph-mon[184144]: failed to encode map e638 with expected crc 2023-09-23T20:49:50.215 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:50 smithi018 ceph-mon[184144]: 10.12 starting backfill to osd.7 from (0'0,0'0] MAX to 635'12098 2023-09-23T20:49:50.215 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:50 smithi018 ceph-mon[184144]: 10.16 starting backfill to osd.4 from (0'0,0'0] MAX to 635'11854 2023-09-23T20:49:50.216 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:50 smithi018 ceph-mon[184144]: 10.6 starting backfill to osd.1 from (0'0,0'0] MAX to 635'11911 2023-09-23T20:49:50.216 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:50 smithi018 ceph-mon[184144]: 10.7 starting backfill to osd.2 from (0'0,0'0] MAX to 635'11746 2023-09-23T20:49:50.216 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:50 smithi018 ceph-mon[184144]: 10.15 starting backfill to osd.2 from (0'0,0'0] MAX to 635'11943 2023-09-23T20:49:50.216 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:50 smithi018 ceph-mon[184144]: 10.11 starting backfill to osd.6 from (0'0,0'0] MAX to 635'11833 2023-09-23T20:49:50.216 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:50 smithi018 ceph-mon[184144]: 10.a starting backfill to osd.2 from (0'0,0'0] MAX to 635'12738 2023-09-23T20:49:50.216 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:50 smithi018 ceph-mon[179647]: pgmap v1564: 129 pgs: 1 peering, 128 active+clean; 62 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2023-09-23T20:49:50.216 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:50 smithi018 ceph-mon[179647]: 10.9 starting backfill to osd.4 from (0'0,0'0] MAX to 635'11956 2023-09-23T20:49:50.216 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:50 smithi018 ceph-mon[179647]: 10.3 starting backfill to osd.4 from (0'0,0'0] MAX to 635'12689 2023-09-23T20:49:50.216 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:50 smithi018 ceph-mon[179647]: failed to encode map e638 with expected crc 2023-09-23T20:49:50.217 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:50 smithi018 ceph-mon[179647]: failed to encode map e638 with expected crc 2023-09-23T20:49:50.217 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:50 smithi018 ceph-mon[179647]: failed to encode map e638 with expected crc 2023-09-23T20:49:50.217 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:50 smithi018 ceph-mon[179647]: osdmap e638: 8 total, 8 up, 7 in 2023-09-23T20:49:50.217 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:50 smithi018 ceph-mon[179647]: failed to encode map e638 with expected crc 2023-09-23T20:49:50.217 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:50 smithi018 ceph-mon[179647]: failed to encode map e638 with expected crc 2023-09-23T20:49:50.217 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:50 smithi018 ceph-mon[179647]: failed to encode map e638 with expected crc 2023-09-23T20:49:50.217 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:50 smithi018 ceph-mon[179647]: 10.12 starting backfill to osd.7 from (0'0,0'0] MAX to 635'12098 2023-09-23T20:49:50.217 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:50 smithi018 ceph-mon[179647]: 10.16 starting backfill to osd.4 from (0'0,0'0] MAX to 635'11854 2023-09-23T20:49:50.218 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:50 smithi018 ceph-mon[179647]: 10.6 starting backfill to osd.1 from (0'0,0'0] MAX to 635'11911 2023-09-23T20:49:50.218 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:50 smithi018 ceph-mon[179647]: 10.7 starting backfill to osd.2 from (0'0,0'0] MAX to 635'11746 2023-09-23T20:49:50.218 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:50 smithi018 ceph-mon[179647]: 10.15 starting backfill to osd.2 from (0'0,0'0] MAX to 635'11943 2023-09-23T20:49:50.218 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:50 smithi018 ceph-mon[179647]: 10.11 starting backfill to osd.6 from (0'0,0'0] MAX to 635'11833 2023-09-23T20:49:50.218 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:50 smithi018 ceph-mon[179647]: 10.a starting backfill to osd.2 from (0'0,0'0] MAX to 635'12738 2023-09-23T20:49:50.218 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T20:49:50.287 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:49:50.319 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:50.350 INFO:teuthology.orchestra.run.smithi149.stdout:ERROR: (22) Invalid argument 2023-09-23T20:49:50.350 INFO:teuthology.orchestra.run.smithi149.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:49:50.363 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:49:50.366 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:50 smithi149 ceph-mon[170394]: pgmap v1564: 129 pgs: 1 peering, 128 active+clean; 62 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail 2023-09-23T20:49:50.366 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:50 smithi149 ceph-mon[170394]: 10.9 starting backfill to osd.4 from (0'0,0'0] MAX to 635'11956 2023-09-23T20:49:50.366 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:50 smithi149 ceph-mon[170394]: 10.3 starting backfill to osd.4 from (0'0,0'0] MAX to 635'12689 2023-09-23T20:49:50.366 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:50 smithi149 ceph-mon[170394]: failed to encode map e638 with expected crc 2023-09-23T20:49:50.367 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:50 smithi149 ceph-mon[170394]: failed to encode map e638 with expected crc 2023-09-23T20:49:50.367 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:50 smithi149 ceph-mon[170394]: failed to encode map e638 with expected crc 2023-09-23T20:49:50.367 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:50 smithi149 ceph-mon[170394]: osdmap e638: 8 total, 8 up, 7 in 2023-09-23T20:49:50.367 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:50 smithi149 ceph-mon[170394]: failed to encode map e638 with expected crc 2023-09-23T20:49:50.368 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:50 smithi149 ceph-mon[170394]: failed to encode map e638 with expected crc 2023-09-23T20:49:50.368 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:50 smithi149 ceph-mon[170394]: failed to encode map e638 with expected crc 2023-09-23T20:49:50.368 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:50 smithi149 ceph-mon[170394]: 10.12 starting backfill to osd.7 from (0'0,0'0] MAX to 635'12098 2023-09-23T20:49:50.368 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:50 smithi149 ceph-mon[170394]: 10.16 starting backfill to osd.4 from (0'0,0'0] MAX to 635'11854 2023-09-23T20:49:50.368 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:50 smithi149 ceph-mon[170394]: 10.6 starting backfill to osd.1 from (0'0,0'0] MAX to 635'11911 2023-09-23T20:49:50.369 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:50 smithi149 ceph-mon[170394]: 10.7 starting backfill to osd.2 from (0'0,0'0] MAX to 635'11746 2023-09-23T20:49:50.369 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:50 smithi149 ceph-mon[170394]: 10.15 starting backfill to osd.2 from (0'0,0'0] MAX to 635'11943 2023-09-23T20:49:50.369 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:50 smithi149 ceph-mon[170394]: 10.11 starting backfill to osd.6 from (0'0,0'0] MAX to 635'11833 2023-09-23T20:49:50.369 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:50 smithi149 ceph-mon[170394]: 10.a starting backfill to osd.2 from (0'0,0'0] MAX to 635'12738 2023-09-23T20:49:50.370 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:50.370 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:49:50.471 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:49:50.503 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:50.587 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:49:50.589 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:50.590 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:49:50.597 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:49:50.617 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:49:50.636 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:49:50.655 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:49:50.676 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:49:50.691 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:49:50.694 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T20:49:50.695 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:49:50.716 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:49:50.719 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:50.736 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:49:50.740 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:49:50.740 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:49:50.740 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:49:50.740 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:49:50.740 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:49:50.740 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:49:50.740 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:49:50.740 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:49:50.754 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:49:50.757 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:50.758 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:49:50.858 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:49:50.886 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:50.919 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:49:50.922 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:49:50.925 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:50.926 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:49:51.027 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:49:51.055 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:51.090 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:49:51.093 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:51.093 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:49:51.194 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:49:51.226 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:51.310 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:49:51.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:51 smithi018 ceph-mon[184144]: failed to encode map e639 with expected crc 2023-09-23T20:49:51.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:51 smithi018 ceph-mon[184144]: osdmap e639: 8 total, 8 up, 7 in 2023-09-23T20:49:51.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:51 smithi018 ceph-mon[184144]: failed to encode map e639 with expected crc 2023-09-23T20:49:51.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:51 smithi018 ceph-mon[184144]: failed to encode map e639 with expected crc 2023-09-23T20:49:51.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:51 smithi018 ceph-mon[184144]: failed to encode map e639 with expected crc 2023-09-23T20:49:51.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:51 smithi018 ceph-mon[184144]: failed to encode map e639 with expected crc 2023-09-23T20:49:51.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:51 smithi018 ceph-mon[184144]: pgmap v1567: 129 pgs: 1 active+clean+remapped, 1 active+undersized+remapped, 9 remapped+peering, 118 active+clean; 23 MiB data, 5.3 GiB used, 620 GiB / 626 GiB avail 2023-09-23T20:49:51.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:51 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/745255786' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:49:51.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:51 smithi018 ceph-mon[179647]: failed to encode map e639 with expected crc 2023-09-23T20:49:51.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:51 smithi018 ceph-mon[179647]: osdmap e639: 8 total, 8 up, 7 in 2023-09-23T20:49:51.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:51 smithi018 ceph-mon[179647]: failed to encode map e639 with expected crc 2023-09-23T20:49:51.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:51 smithi018 ceph-mon[179647]: failed to encode map e639 with expected crc 2023-09-23T20:49:51.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:51 smithi018 ceph-mon[179647]: failed to encode map e639 with expected crc 2023-09-23T20:49:51.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:51 smithi018 ceph-mon[179647]: failed to encode map e639 with expected crc 2023-09-23T20:49:51.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:51 smithi018 ceph-mon[179647]: pgmap v1567: 129 pgs: 1 active+clean+remapped, 1 active+undersized+remapped, 9 remapped+peering, 118 active+clean; 23 MiB data, 5.3 GiB used, 620 GiB / 626 GiB avail 2023-09-23T20:49:51.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:51 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/745255786' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:49:51.467 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:49:51.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:51 smithi149 ceph-mon[170394]: failed to encode map e639 with expected crc 2023-09-23T20:49:51.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:51 smithi149 ceph-mon[170394]: osdmap e639: 8 total, 8 up, 7 in 2023-09-23T20:49:51.468 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:51 smithi149 ceph-mon[170394]: failed to encode map e639 with expected crc 2023-09-23T20:49:51.468 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:51 smithi149 ceph-mon[170394]: failed to encode map e639 with expected crc 2023-09-23T20:49:51.468 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:51 smithi149 ceph-mon[170394]: failed to encode map e639 with expected crc 2023-09-23T20:49:51.468 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:51 smithi149 ceph-mon[170394]: failed to encode map e639 with expected crc 2023-09-23T20:49:51.468 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:51 smithi149 ceph-mon[170394]: pgmap v1567: 129 pgs: 1 active+clean+remapped, 1 active+undersized+remapped, 9 remapped+peering, 118 active+clean; 23 MiB data, 5.3 GiB used, 620 GiB / 626 GiB avail 2023-09-23T20:49:51.468 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:51 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/745255786' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:49:51.471 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:51.471 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:49:51.572 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:49:51.600 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:51.635 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:49:51.638 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:51.639 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:49:51.739 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:49:51.768 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:51.809 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:49:51.812 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:51.812 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:49:51.913 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:49:51.941 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:52.128 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:49:52.128 INFO:teuthology.orchestra.run.smithi018.stderr:pool 'unique_pool_5' removed 2023-09-23T20:49:52.135 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:52.136 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:49:52.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:52 smithi149 ceph-mon[170394]: failed to encode map e640 with expected crc 2023-09-23T20:49:52.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:52 smithi149 ceph-mon[170394]: osdmap e640: 8 total, 8 up, 7 in 2023-09-23T20:49:52.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:52 smithi149 ceph-mon[170394]: failed to encode map e640 with expected crc 2023-09-23T20:49:52.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:52 smithi149 ceph-mon[170394]: failed to encode map e640 with expected crc 2023-09-23T20:49:52.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:52 smithi149 ceph-mon[170394]: failed to encode map e640 with expected crc 2023-09-23T20:49:52.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:52 smithi149 ceph-mon[170394]: failed to encode map e640 with expected crc 2023-09-23T20:49:52.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:52 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3773894891' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T20:49:52.237 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:49:52.272 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:52.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:52 smithi018 ceph-mon[184144]: failed to encode map e640 with expected crc 2023-09-23T20:49:52.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:52 smithi018 ceph-mon[184144]: osdmap e640: 8 total, 8 up, 7 in 2023-09-23T20:49:52.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:52 smithi018 ceph-mon[184144]: failed to encode map e640 with expected crc 2023-09-23T20:49:52.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:52 smithi018 ceph-mon[184144]: failed to encode map e640 with expected crc 2023-09-23T20:49:52.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:52 smithi018 ceph-mon[184144]: failed to encode map e640 with expected crc 2023-09-23T20:49:52.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:52 smithi018 ceph-mon[184144]: failed to encode map e640 with expected crc 2023-09-23T20:49:52.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:52 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3773894891' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T20:49:52.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:52 smithi018 ceph-mon[179647]: failed to encode map e640 with expected crc 2023-09-23T20:49:52.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:52 smithi018 ceph-mon[179647]: osdmap e640: 8 total, 8 up, 7 in 2023-09-23T20:49:52.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:52 smithi018 ceph-mon[179647]: failed to encode map e640 with expected crc 2023-09-23T20:49:52.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:52 smithi018 ceph-mon[179647]: failed to encode map e640 with expected crc 2023-09-23T20:49:52.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:52 smithi018 ceph-mon[179647]: failed to encode map e640 with expected crc 2023-09-23T20:49:52.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:52 smithi018 ceph-mon[179647]: failed to encode map e640 with expected crc 2023-09-23T20:49:52.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:52 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3773894891' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T20:49:52.483 INFO:teuthology.task.sequential:In sequential, running task print... 2023-09-23T20:49:52.483 INFO:teuthology.task.print:**** done end radosbench.yaml 2023-09-23T20:49:52.483 INFO:teuthology.task.sequential:In sequential, running task workunit... 2023-09-23T20:49:52.484 INFO:tasks.workunit:Pulling workunits from ref quincy 2023-09-23T20:49:52.484 INFO:tasks.workunit:Making a separate scratch dir for every client... 2023-09-23T20:49:52.484 DEBUG:teuthology.orchestra.run.smithi018:> stat -- /home/ubuntu/cephtest/mnt.0 2023-09-23T20:49:52.538 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:49:52.541 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:52.542 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:49:52.572 INFO:teuthology.orchestra.run.smithi149.stdout:{ 2023-09-23T20:49:52.572 INFO:teuthology.orchestra.run.smithi149.stdout: "size": 20, 2023-09-23T20:49:52.573 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 600, 2023-09-23T20:49:52.573 INFO:teuthology.orchestra.run.smithi149.stdout: "ops": [ 2023-09-23T20:49:52.573 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.573 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:102263 10.e 10:7fa4b63f:::benchmark_data_smithi018_422431_object102262:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e553)", 2023-09-23T20:49:52.573 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:47:09.171743+0000", 2023-09-23T20:49:52.573 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 163.39811206499999, 2023-09-23T20:49:52.573 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0020924090000001, 2023-09-23T20:49:52.573 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:49:52.573 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:49:52.573 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:49:52.573 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:49:52.574 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:49:52.574 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 102263 2023-09-23T20:49:52.574 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.574 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:49:52.574 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.574 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:49:52.574 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.171743+0000", 2023-09-23T20:49:52.574 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:49:52.574 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.574 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.574 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:49:52.575 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.171743+0000", 2023-09-23T20:49:52.575 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:49:52.575 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.575 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.575 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:49:52.575 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.171742+0000", 2023-09-23T20:49:52.575 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:49:52.575 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.575 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.575 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:49:52.575 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.191370+0000", 2023-09-23T20:49:52.576 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.019627815 2023-09-23T20:49:52.576 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.576 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.576 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:49:52.576 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.191383+0000", 2023-09-23T20:49:52.576 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.3481e-05 2023-09-23T20:49:52.576 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.576 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.576 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:49:52.576 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.191393+0000", 2023-09-23T20:49:52.576 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0097e-05 2023-09-23T20:49:52.577 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.577 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.577 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:49:52.577 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.192254+0000", 2023-09-23T20:49:52.577 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00086048600000000004 2023-09-23T20:49:52.577 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.577 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.577 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:49:52.577 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.192264+0000", 2023-09-23T20:49:52.577 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0553999999999999e-05 2023-09-23T20:49:52.577 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.578 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.578 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:49:52.578 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:10.173760+0000", 2023-09-23T20:49:52.578 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.98149563500000003 2023-09-23T20:49:52.578 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.578 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.578 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:49:52.578 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:10.173836+0000", 2023-09-23T20:49:52.578 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.5891999999999998e-05 2023-09-23T20:49:52.578 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:49:52.578 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:49:52.579 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:49:52.579 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.579 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.579 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:102264 10.e 10:70bfcac0:::benchmark_data_smithi018_422431_object102263:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e553)", 2023-09-23T20:49:52.579 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:47:09.191403+0000", 2023-09-23T20:49:52.579 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 163.37845255600001, 2023-09-23T20:49:52.579 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.98286253400000001, 2023-09-23T20:49:52.579 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:49:52.579 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:49:52.579 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:49:52.579 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:49:52.579 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:49:52.580 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 102264 2023-09-23T20:49:52.580 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.580 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:49:52.580 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.580 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:49:52.580 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.191403+0000", 2023-09-23T20:49:52.580 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:49:52.580 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.580 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.580 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:49:52.580 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.191403+0000", 2023-09-23T20:49:52.581 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:49:52.581 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.581 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.581 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:49:52.581 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.191401+0000", 2023-09-23T20:49:52.581 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999976 2023-09-23T20:49:52.582 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.582 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.582 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:49:52.582 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.191476+0000", 2023-09-23T20:49:52.582 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.5279999999999998e-05 2023-09-23T20:49:52.582 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.582 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.583 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:49:52.583 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.191483+0000", 2023-09-23T20:49:52.583 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.7819999999999996e-06 2023-09-23T20:49:52.583 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.583 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.583 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:49:52.584 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.191488+0000", 2023-09-23T20:49:52.584 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.994e-06 2023-09-23T20:49:52.584 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.584 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.584 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:49:52.584 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.192281+0000", 2023-09-23T20:49:52.585 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00079300400000000004 2023-09-23T20:49:52.585 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.585 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.585 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:49:52.585 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.192285+0000", 2023-09-23T20:49:52.585 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.4440000000000001e-06 2023-09-23T20:49:52.586 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.586 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.586 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:49:52.586 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:10.174202+0000", 2023-09-23T20:49:52.586 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.98191682599999996 2023-09-23T20:49:52.586 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.586 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.587 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:49:52.587 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:10.174265+0000", 2023-09-23T20:49:52.587 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.3380999999999995e-05 2023-09-23T20:49:52.587 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:49:52.587 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:49:52.587 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:49:52.587 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.588 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.588 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:102314 10.e 10:771e9b48:::benchmark_data_smithi018_422431_object102313:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e553)", 2023-09-23T20:49:52.588 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:47:09.319054+0000", 2023-09-23T20:49:52.588 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 163.25080182599999, 2023-09-23T20:49:52.588 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.87209886700000006, 2023-09-23T20:49:52.589 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:49:52.589 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:49:52.589 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:49:52.589 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:49:52.589 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:49:52.589 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 102314 2023-09-23T20:49:52.590 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.590 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:49:52.590 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.590 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:49:52.590 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.319054+0000", 2023-09-23T20:49:52.590 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:49:52.591 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.591 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.591 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:49:52.591 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.319054+0000", 2023-09-23T20:49:52.591 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:49:52.591 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.591 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.591 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:49:52.591 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.319052+0000", 2023-09-23T20:49:52.592 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:49:52.592 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.592 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.592 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:49:52.592 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.319144+0000", 2023-09-23T20:49:52.592 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.1686000000000003e-05 2023-09-23T20:49:52.592 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.592 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.592 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:49:52.592 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.319159+0000", 2023-09-23T20:49:52.592 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.5136e-05 2023-09-23T20:49:52.592 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.593 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.593 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:49:52.593 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.319165+0000", 2023-09-23T20:49:52.593 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.5010000000000001e-06 2023-09-23T20:49:52.593 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.593 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.593 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:49:52.593 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.321384+0000", 2023-09-23T20:49:52.593 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0022185460000000001 2023-09-23T20:49:52.593 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.593 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.593 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:49:52.594 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.321392+0000", 2023-09-23T20:49:52.594 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.0430000000000008e-06 2023-09-23T20:49:52.594 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.594 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.594 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:49:52.594 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:10.174291+0000", 2023-09-23T20:49:52.594 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.85289914600000005 2023-09-23T20:49:52.594 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.594 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.594 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:49:52.595 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:10.174421+0000", 2023-09-23T20:49:52.595 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00012987799999999999 2023-09-23T20:49:52.595 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.595 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.595 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 2,6", 2023-09-23T20:49:52.595 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:10.174567+0000", 2023-09-23T20:49:52.595 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0001462 2023-09-23T20:49:52.595 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.595 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.595 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:52.595 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:10.184312+0000", 2023-09-23T20:49:52.595 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0097449849999999994 2023-09-23T20:49:52.596 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.596 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.596 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:52.596 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:10.186377+0000", 2023-09-23T20:49:52.596 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0020647410000000001 2023-09-23T20:49:52.596 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.596 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.596 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:49:52.596 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:10.191052+0000", 2023-09-23T20:49:52.596 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0046755879999999996 2023-09-23T20:49:52.596 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.596 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.597 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:49:52.597 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:10.191097+0000", 2023-09-23T20:49:52.597 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.4085999999999998e-05 2023-09-23T20:49:52.597 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.597 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.597 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:49:52.597 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:10.191152+0000", 2023-09-23T20:49:52.597 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.5852e-05 2023-09-23T20:49:52.597 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:49:52.597 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:49:52.597 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:49:52.598 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.598 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.598 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:102328 10.1 10:8bdf3d92:::benchmark_data_smithi018_422431_object102327:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e553)", 2023-09-23T20:49:52.598 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:47:09.354426+0000", 2023-09-23T20:49:52.598 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 163.215429228, 2023-09-23T20:49:52.598 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.85433105200000004, 2023-09-23T20:49:52.598 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:49:52.598 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:49:52.598 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:49:52.598 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:49:52.598 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:49:52.599 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 102328 2023-09-23T20:49:52.599 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.599 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:49:52.599 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.599 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:49:52.599 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.354426+0000", 2023-09-23T20:49:52.599 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:49:52.599 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.599 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.599 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:49:52.599 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.354426+0000", 2023-09-23T20:49:52.599 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:49:52.600 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.600 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.600 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:49:52.600 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.354425+0000", 2023-09-23T20:49:52.600 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:49:52.600 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.600 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.600 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:49:52.600 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.354533+0000", 2023-09-23T20:49:52.600 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00010846 2023-09-23T20:49:52.600 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.601 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.601 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:49:52.601 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.354546+0000", 2023-09-23T20:49:52.601 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.2459e-05 2023-09-23T20:49:52.601 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.601 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.601 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:49:52.601 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.354553+0000", 2023-09-23T20:49:52.601 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.9399999999999996e-06 2023-09-23T20:49:52.601 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.601 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.601 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:49:52.602 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.354611+0000", 2023-09-23T20:49:52.602 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.7868000000000002e-05 2023-09-23T20:49:52.602 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.602 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.602 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:49:52.602 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.354622+0000", 2023-09-23T20:49:52.602 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0881e-05 2023-09-23T20:49:52.602 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.602 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.602 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:49:52.603 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:10.177496+0000", 2023-09-23T20:49:52.603 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.82287492900000003 2023-09-23T20:49:52.603 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.603 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.603 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:49:52.603 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:10.177619+0000", 2023-09-23T20:49:52.603 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00012237500000000001 2023-09-23T20:49:52.603 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.603 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.603 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,7", 2023-09-23T20:49:52.603 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:10.178159+0000", 2023-09-23T20:49:52.603 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00053965899999999995 2023-09-23T20:49:52.604 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.604 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.604 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:49:52.604 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:10.198332+0000", 2023-09-23T20:49:52.604 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.020173176000000001 2023-09-23T20:49:52.604 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.604 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.604 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:52.604 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:10.208652+0000", 2023-09-23T20:49:52.604 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.010320775000000001 2023-09-23T20:49:52.604 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.604 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.605 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:52.605 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:10.208698+0000", 2023-09-23T20:49:52.605 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.5568000000000001e-05 2023-09-23T20:49:52.605 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.605 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.605 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:49:52.605 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:10.208728+0000", 2023-09-23T20:49:52.605 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.0032e-05 2023-09-23T20:49:52.605 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.605 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.605 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:49:52.605 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:10.208757+0000", 2023-09-23T20:49:52.606 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.9108999999999999e-05 2023-09-23T20:49:52.606 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:49:52.606 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:49:52.606 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:49:52.606 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.606 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.606 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:102335 10.1 10:88e5c9cd:::benchmark_data_smithi018_422431_object102334:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e553)", 2023-09-23T20:49:52.606 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:47:09.380750+0000", 2023-09-23T20:49:52.606 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 163.18910546500001, 2023-09-23T20:49:52.606 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.82807518800000002, 2023-09-23T20:49:52.606 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:49:52.607 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:49:52.607 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:49:52.607 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:49:52.607 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:49:52.607 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 102335 2023-09-23T20:49:52.607 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.607 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:49:52.607 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.607 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:49:52.607 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.380750+0000", 2023-09-23T20:49:52.607 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:49:52.608 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.608 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.608 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:49:52.608 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.380750+0000", 2023-09-23T20:49:52.608 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:49:52.608 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.608 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.608 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:49:52.608 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.380748+0000", 2023-09-23T20:49:52.608 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:49:52.608 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.609 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.609 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:49:52.609 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.380802+0000", 2023-09-23T20:49:52.609 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.393e-05 2023-09-23T20:49:52.609 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.609 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.609 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:49:52.609 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.380818+0000", 2023-09-23T20:49:52.609 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.5743000000000002e-05 2023-09-23T20:49:52.609 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.609 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.609 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:49:52.610 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.380824+0000", 2023-09-23T20:49:52.610 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.2020000000000001e-06 2023-09-23T20:49:52.610 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.610 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.610 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:49:52.610 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.380865+0000", 2023-09-23T20:49:52.610 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.0317e-05 2023-09-23T20:49:52.610 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.610 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.610 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:49:52.611 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.380873+0000", 2023-09-23T20:49:52.611 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.0350000000000001e-06 2023-09-23T20:49:52.611 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.611 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.611 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:49:52.611 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:10.183127+0000", 2023-09-23T20:49:52.611 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.80225428300000001 2023-09-23T20:49:52.611 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.611 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.611 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:49:52.611 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:10.183233+0000", 2023-09-23T20:49:52.611 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00010638599999999999 2023-09-23T20:49:52.612 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.612 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.612 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,7", 2023-09-23T20:49:52.612 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:10.183882+0000", 2023-09-23T20:49:52.612 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00064829299999999998 2023-09-23T20:49:52.612 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.612 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.612 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:49:52.613 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:10.201736+0000", 2023-09-23T20:49:52.613 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.017854775999999999 2023-09-23T20:49:52.613 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.613 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.613 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:52.613 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:10.208681+0000", 2023-09-23T20:49:52.613 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0069449749999999999 2023-09-23T20:49:52.613 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.613 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.613 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:52.613 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:10.208780+0000", 2023-09-23T20:49:52.614 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.8886000000000002e-05 2023-09-23T20:49:52.614 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.614 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.614 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:49:52.614 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:10.208806+0000", 2023-09-23T20:49:52.614 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.5185999999999999e-05 2023-09-23T20:49:52.614 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.614 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.614 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:49:52.614 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:10.208825+0000", 2023-09-23T20:49:52.615 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.9585e-05 2023-09-23T20:49:52.615 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:49:52.615 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:49:52.615 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:49:52.615 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.615 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.615 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:102348 10.1 10:8c86b5d8:::benchmark_data_smithi018_422431_object102347:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e553)", 2023-09-23T20:49:52.615 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:47:09.394224+0000", 2023-09-23T20:49:52.615 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 163.175631106, 2023-09-23T20:49:52.615 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.81471285100000002, 2023-09-23T20:49:52.615 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:49:52.615 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:49:52.616 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:49:52.616 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:49:52.616 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:49:52.616 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 102348 2023-09-23T20:49:52.616 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.616 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:49:52.616 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.616 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:49:52.616 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.394224+0000", 2023-09-23T20:49:52.616 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:49:52.616 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.616 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.617 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:49:52.617 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.394224+0000", 2023-09-23T20:49:52.617 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:49:52.617 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.617 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.617 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:49:52.617 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.394223+0000", 2023-09-23T20:49:52.617 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:49:52.617 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.617 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.617 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:49:52.618 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.394305+0000", 2023-09-23T20:49:52.618 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.2297000000000002e-05 2023-09-23T20:49:52.618 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.618 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.618 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:49:52.618 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.394329+0000", 2023-09-23T20:49:52.618 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.3391999999999999e-05 2023-09-23T20:49:52.618 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.618 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.618 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:49:52.618 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.394336+0000", 2023-09-23T20:49:52.618 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.002e-06 2023-09-23T20:49:52.619 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.619 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.619 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:49:52.619 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.394395+0000", 2023-09-23T20:49:52.619 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.9018999999999999e-05 2023-09-23T20:49:52.619 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.619 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.619 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:49:52.619 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.394403+0000", 2023-09-23T20:49:52.619 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.2560000000000002e-06 2023-09-23T20:49:52.619 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.619 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.620 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:49:52.620 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:10.186875+0000", 2023-09-23T20:49:52.620 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.79247180299999997 2023-09-23T20:49:52.620 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.620 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.620 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:49:52.620 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:10.187029+0000", 2023-09-23T20:49:52.620 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00015432999999999999 2023-09-23T20:49:52.620 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.620 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.620 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,7", 2023-09-23T20:49:52.621 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:10.187200+0000", 2023-09-23T20:49:52.621 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000170915 2023-09-23T20:49:52.621 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.621 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.621 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:49:52.621 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:10.201742+0000", 2023-09-23T20:49:52.621 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.014541953999999999 2023-09-23T20:49:52.621 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.621 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.621 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:52.621 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:10.208845+0000", 2023-09-23T20:49:52.621 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0071029140000000001 2023-09-23T20:49:52.622 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.622 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.622 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:52.622 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:10.208896+0000", 2023-09-23T20:49:52.622 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.1143999999999997e-05 2023-09-23T20:49:52.622 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.622 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.622 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:49:52.622 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:10.208919+0000", 2023-09-23T20:49:52.622 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.3025000000000001e-05 2023-09-23T20:49:52.622 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.622 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.623 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:49:52.623 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:10.208937+0000", 2023-09-23T20:49:52.623 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.7864e-05 2023-09-23T20:49:52.623 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:49:52.623 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:49:52.623 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:49:52.623 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.623 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.623 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:159028 10.3 10.1197f3a3 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e577)", 2023-09-23T20:49:52.624 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:48:22.915122+0000", 2023-09-23T20:49:52.624 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 89.654733807, 2023-09-23T20:49:52.624 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.98997199800000002, 2023-09-23T20:49:52.624 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:49:52.624 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:49:52.624 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:49:52.624 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:49:52.624 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:49:52.624 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 159028 2023-09-23T20:49:52.624 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.624 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:49:52.624 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.625 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:49:52.625 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.915122+0000", 2023-09-23T20:49:52.625 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:49:52.625 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.625 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.625 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:49:52.625 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.915122+0000", 2023-09-23T20:49:52.625 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:49:52.625 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.625 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.625 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:49:52.625 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.915120+0000", 2023-09-23T20:49:52.626 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:49:52.626 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.626 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.626 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:49:52.626 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.915126+0000", 2023-09-23T20:49:52.626 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.6370000000000004e-06 2023-09-23T20:49:52.626 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.626 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.626 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:49:52.626 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.915135+0000", 2023-09-23T20:49:52.626 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.4250000000000004e-06 2023-09-23T20:49:52.627 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.627 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.627 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:49:52.627 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.915142+0000", 2023-09-23T20:49:52.627 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.6599999999999998e-06 2023-09-23T20:49:52.627 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.627 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.627 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:49:52.627 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.916692+0000", 2023-09-23T20:49:52.627 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.001550395 2023-09-23T20:49:52.627 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.627 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.628 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:49:52.628 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.916699+0000", 2023-09-23T20:49:52.628 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.5409999999999997e-06 2023-09-23T20:49:52.628 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.628 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.628 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:49:52.628 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:23.905083+0000", 2023-09-23T20:49:52.628 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.988384288 2023-09-23T20:49:52.628 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.628 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.628 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:49:52.629 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:23.905094+0000", 2023-09-23T20:49:52.629 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0261000000000001e-05 2023-09-23T20:49:52.629 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:49:52.629 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:49:52.629 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:49:52.629 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.629 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.629 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:159040 10.e 10:71537a69:::benchmark_data_smithi018_422431_object54544:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e577)", 2023-09-23T20:49:52.629 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:48:22.936052+0000", 2023-09-23T20:49:52.629 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 89.633803173999993, 2023-09-23T20:49:52.629 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.2675618719999999, 2023-09-23T20:49:52.629 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:49:52.630 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:49:52.630 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:49:52.630 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:49:52.630 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:49:52.630 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 159040 2023-09-23T20:49:52.630 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.630 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:49:52.630 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.630 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:49:52.630 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.936052+0000", 2023-09-23T20:49:52.630 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:49:52.631 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.631 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.631 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:49:52.631 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.936052+0000", 2023-09-23T20:49:52.631 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:49:52.631 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.631 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.631 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:49:52.631 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.936051+0000", 2023-09-23T20:49:52.631 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:49:52.631 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.631 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.632 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:49:52.632 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.936057+0000", 2023-09-23T20:49:52.632 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.7960000000000001e-06 2023-09-23T20:49:52.632 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.632 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.632 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:49:52.632 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.936070+0000", 2023-09-23T20:49:52.632 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.2683e-05 2023-09-23T20:49:52.632 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.632 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.632 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:49:52.633 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.936074+0000", 2023-09-23T20:49:52.633 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.9359999999999998e-06 2023-09-23T20:49:52.633 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.633 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.633 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:49:52.633 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.936130+0000", 2023-09-23T20:49:52.633 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.5776999999999998e-05 2023-09-23T20:49:52.633 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.633 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.633 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:49:52.633 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.936142+0000", 2023-09-23T20:49:52.633 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.198e-05 2023-09-23T20:49:52.634 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.634 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.634 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:49:52.634 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:23.906942+0000", 2023-09-23T20:49:52.634 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.97079988100000003 2023-09-23T20:49:52.634 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.634 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.634 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:49:52.634 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:23.906953+0000", 2023-09-23T20:49:52.634 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.066e-05 2023-09-23T20:49:52.634 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.634 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.635 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:49:52.635 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:25.130718+0000", 2023-09-23T20:49:52.635 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.2237656889999999 2023-09-23T20:49:52.635 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.635 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.635 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:49:52.635 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:25.130872+0000", 2023-09-23T20:49:52.635 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00015312299999999999 2023-09-23T20:49:52.635 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.636 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.636 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 2,5,6", 2023-09-23T20:49:52.636 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:25.131568+0000", 2023-09-23T20:49:52.636 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00069637699999999998 2023-09-23T20:49:52.636 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.636 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.636 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:52.636 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:25.151395+0000", 2023-09-23T20:49:52.636 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.019826769000000001 2023-09-23T20:49:52.636 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.636 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.637 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:49:52.637 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:25.151584+0000", 2023-09-23T20:49:52.637 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00018891399999999999 2023-09-23T20:49:52.637 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.637 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.637 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:52.637 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:25.175874+0000", 2023-09-23T20:49:52.637 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.024290066999999999 2023-09-23T20:49:52.637 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.637 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.637 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:52.638 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:25.203427+0000", 2023-09-23T20:49:52.638 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.027553628 2023-09-23T20:49:52.638 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.638 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.638 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:49:52.638 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:25.203487+0000", 2023-09-23T20:49:52.638 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.9623999999999997e-05 2023-09-23T20:49:52.638 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.638 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.638 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:49:52.638 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:25.203614+0000", 2023-09-23T20:49:52.639 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00012711999999999999 2023-09-23T20:49:52.639 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:49:52.639 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:49:52.639 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:49:52.639 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.639 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.639 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:159052 10.3 10.e70c3763 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e577)", 2023-09-23T20:49:52.639 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:48:22.953330+0000", 2023-09-23T20:49:52.639 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 89.616525593000006, 2023-09-23T20:49:52.639 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.95178979799999996, 2023-09-23T20:49:52.639 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:49:52.639 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:49:52.640 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:49:52.640 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:49:52.640 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:49:52.640 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 159052 2023-09-23T20:49:52.640 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.640 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:49:52.640 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.640 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:49:52.640 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.953330+0000", 2023-09-23T20:49:52.640 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:49:52.640 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.640 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.641 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:49:52.641 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.953330+0000", 2023-09-23T20:49:52.641 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:49:52.641 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.641 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.641 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:49:52.641 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.953329+0000", 2023-09-23T20:49:52.641 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:49:52.641 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.641 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.641 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:49:52.641 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.953334+0000", 2023-09-23T20:49:52.642 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.8999999999999997e-06 2023-09-23T20:49:52.642 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.642 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.642 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:49:52.642 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.953343+0000", 2023-09-23T20:49:52.642 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.4520000000000003e-06 2023-09-23T20:49:52.642 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.642 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.642 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:49:52.642 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.953348+0000", 2023-09-23T20:49:52.642 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.5390000000000001e-06 2023-09-23T20:49:52.643 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.643 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.643 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:49:52.643 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.953392+0000", 2023-09-23T20:49:52.643 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.4774000000000002e-05 2023-09-23T20:49:52.643 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.643 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.643 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:49:52.643 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.953400+0000", 2023-09-23T20:49:52.643 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.0890000000000002e-06 2023-09-23T20:49:52.643 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.643 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.644 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:49:52.644 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:23.905113+0000", 2023-09-23T20:49:52.644 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.95171356299999998 2023-09-23T20:49:52.644 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.644 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.644 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:49:52.644 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:23.905120+0000", 2023-09-23T20:49:52.644 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.4840000000000001e-06 2023-09-23T20:49:52.644 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:49:52.644 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:49:52.644 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:49:52.645 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.645 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.645 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:174198 10.e 10:7230894f:::benchmark_data_smithi018_422431_object69702:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e590)", 2023-09-23T20:49:52.645 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:48:47.130817+0000", 2023-09-23T20:49:52.645 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 65.439037896000002, 2023-09-23T20:49:52.645 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0840209249999999, 2023-09-23T20:49:52.645 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:49:52.645 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:49:52.645 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:49:52.645 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:49:52.645 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:49:52.646 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 174198 2023-09-23T20:49:52.646 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.646 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:49:52.646 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.646 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:49:52.646 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:47.130817+0000", 2023-09-23T20:49:52.646 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:49:52.646 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.646 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.646 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:49:52.646 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:47.130817+0000", 2023-09-23T20:49:52.647 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:49:52.647 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.647 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.647 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:49:52.647 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:47.130816+0000", 2023-09-23T20:49:52.647 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:49:52.647 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.647 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.647 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:49:52.647 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:47.130823+0000", 2023-09-23T20:49:52.647 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.3560000000000004e-06 2023-09-23T20:49:52.647 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.648 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.648 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:49:52.648 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:47.130837+0000", 2023-09-23T20:49:52.648 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.3952e-05 2023-09-23T20:49:52.648 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.648 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.648 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:49:52.648 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:47.130845+0000", 2023-09-23T20:49:52.648 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.5279999999999998e-06 2023-09-23T20:49:52.648 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.648 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.649 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:49:52.649 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:47.131335+0000", 2023-09-23T20:49:52.649 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00049032299999999997 2023-09-23T20:49:52.649 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.649 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.649 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:49:52.649 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:47.131477+0000", 2023-09-23T20:49:52.649 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00014191 2023-09-23T20:49:52.649 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.649 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.649 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 2,5,6", 2023-09-23T20:49:52.649 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:47.131615+0000", 2023-09-23T20:49:52.650 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00013770300000000001 2023-09-23T20:49:52.650 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.650 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.650 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:52.650 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:47.139776+0000", 2023-09-23T20:49:52.650 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0081612669999999998 2023-09-23T20:49:52.650 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.650 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.650 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:49:52.650 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:47.141819+0000", 2023-09-23T20:49:52.650 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00204294 2023-09-23T20:49:52.651 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.651 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.651 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:49:52.651 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:47.160786+0000", 2023-09-23T20:49:52.651 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.018966818 2023-09-23T20:49:52.651 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.651 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.651 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:49:52.651 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:47.160797+0000", 2023-09-23T20:49:52.652 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0849999999999999e-05 2023-09-23T20:49:52.652 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.652 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.652 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:49:52.652 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:48.200395+0000", 2023-09-23T20:49:52.652 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.039598236 2023-09-23T20:49:52.652 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.652 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.652 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for missing object", 2023-09-23T20:49:52.652 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:48.200521+0000", 2023-09-23T20:49:52.652 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00012670699999999999 2023-09-23T20:49:52.653 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.653 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.653 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:49:52.653 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:48.214746+0000", 2023-09-23T20:49:52.653 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.014224404 2023-09-23T20:49:52.653 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.653 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.653 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:49:52.653 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:48.214838+0000", 2023-09-23T20:49:52.653 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.2504999999999996e-05 2023-09-23T20:49:52.654 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:49:52.654 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:49:52.654 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:49:52.654 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.654 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.654 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:174221 10.e 10:743e88cf:::benchmark_data_smithi018_422431_object69725:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e591)", 2023-09-23T20:49:52.654 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:48:47.168211+0000", 2023-09-23T20:49:52.654 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 65.401644731999994, 2023-09-23T20:49:52.654 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0619566009999999, 2023-09-23T20:49:52.655 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:49:52.655 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:49:52.655 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:49:52.655 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:49:52.655 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:49:52.655 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 174221 2023-09-23T20:49:52.655 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.655 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:49:52.655 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.655 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:49:52.655 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:47.168211+0000", 2023-09-23T20:49:52.656 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:49:52.656 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.656 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.656 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:49:52.656 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:47.168211+0000", 2023-09-23T20:49:52.656 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:49:52.656 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.656 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.656 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:49:52.656 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:47.168209+0000", 2023-09-23T20:49:52.656 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:49:52.656 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.657 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.657 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:49:52.657 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:47.168216+0000", 2023-09-23T20:49:52.657 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.8630000000000002e-06 2023-09-23T20:49:52.657 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.657 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.657 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:49:52.657 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:47.168228+0000", 2023-09-23T20:49:52.657 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.2347000000000001e-05 2023-09-23T20:49:52.657 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.657 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.658 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:49:52.658 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:47.168235+0000", 2023-09-23T20:49:52.658 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.8399999999999997e-06 2023-09-23T20:49:52.658 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.658 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.658 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:49:52.658 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:47.168279+0000", 2023-09-23T20:49:52.658 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.4199000000000002e-05 2023-09-23T20:49:52.658 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.658 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.658 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:49:52.659 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:47.168290+0000", 2023-09-23T20:49:52.659 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0776e-05 2023-09-23T20:49:52.659 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.659 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.659 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:49:52.659 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:48.214861+0000", 2023-09-23T20:49:52.659 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0465711740000001 2023-09-23T20:49:52.659 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.659 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.659 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:49:52.659 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:48.214996+0000", 2023-09-23T20:49:52.660 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00013436599999999999 2023-09-23T20:49:52.660 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.660 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.660 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 2,6", 2023-09-23T20:49:52.660 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:48.215162+0000", 2023-09-23T20:49:52.660 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00016618000000000001 2023-09-23T20:49:52.660 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.660 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.660 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:52.660 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:48.218225+0000", 2023-09-23T20:49:52.660 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0030630240000000001 2023-09-23T20:49:52.661 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.661 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.661 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:49:52.661 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:48.221353+0000", 2023-09-23T20:49:52.661 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.003128334 2023-09-23T20:49:52.661 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.661 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.661 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:52.661 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:48.230010+0000", 2023-09-23T20:49:52.661 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0086572739999999995 2023-09-23T20:49:52.661 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.661 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.662 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:49:52.662 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:48.230062+0000", 2023-09-23T20:49:52.662 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.2055999999999998e-05 2023-09-23T20:49:52.662 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.662 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.662 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:49:52.662 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:48.230167+0000", 2023-09-23T20:49:52.662 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000104823 2023-09-23T20:49:52.662 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:49:52.662 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:49:52.663 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:49:52.663 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.663 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.663 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:174223 10.e 10:790afcea:::benchmark_data_smithi018_422431_object69727:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e591)", 2023-09-23T20:49:52.663 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:48:47.169131+0000", 2023-09-23T20:49:52.663 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 65.400724428999993, 2023-09-23T20:49:52.663 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.061208814, 2023-09-23T20:49:52.663 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:49:52.663 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:49:52.663 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:49:52.663 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:49:52.664 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:49:52.664 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 174223 2023-09-23T20:49:52.664 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.664 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:49:52.664 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.664 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:49:52.664 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:47.169131+0000", 2023-09-23T20:49:52.664 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:49:52.664 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.664 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.664 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:49:52.664 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:47.169131+0000", 2023-09-23T20:49:52.665 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:49:52.665 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.665 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.665 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:49:52.665 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:47.169130+0000", 2023-09-23T20:49:52.665 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:49:52.665 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.665 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.665 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:49:52.665 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:47.169134+0000", 2023-09-23T20:49:52.665 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.895e-06 2023-09-23T20:49:52.665 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.666 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.666 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:49:52.666 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:47.169143+0000", 2023-09-23T20:49:52.666 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.2410000000000001e-06 2023-09-23T20:49:52.666 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.666 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.666 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:49:52.666 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:47.169148+0000", 2023-09-23T20:49:52.666 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.2930000000000002e-06 2023-09-23T20:49:52.666 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.666 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.667 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:49:52.667 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:47.169184+0000", 2023-09-23T20:49:52.667 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.6649000000000001e-05 2023-09-23T20:49:52.667 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.667 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.667 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:49:52.667 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:47.169191+0000", 2023-09-23T20:49:52.667 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.7599999999999997e-06 2023-09-23T20:49:52.667 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.667 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.667 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:49:52.667 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:48.218256+0000", 2023-09-23T20:49:52.668 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0490651 2023-09-23T20:49:52.668 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.668 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.668 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:49:52.668 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:48.218425+0000", 2023-09-23T20:49:52.668 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000168887 2023-09-23T20:49:52.668 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.668 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.668 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 2,6", 2023-09-23T20:49:52.668 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:48.218575+0000", 2023-09-23T20:49:52.668 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00015012700000000001 2023-09-23T20:49:52.669 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.669 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.669 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:52.669 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:48.221396+0000", 2023-09-23T20:49:52.669 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0028210959999999999 2023-09-23T20:49:52.669 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.669 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.669 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:49:52.669 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:48.221474+0000", 2023-09-23T20:49:52.669 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.7961999999999997e-05 2023-09-23T20:49:52.669 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.669 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.670 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:49:52.670 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:48.230218+0000", 2023-09-23T20:49:52.670 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.008743252 2023-09-23T20:49:52.670 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.670 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.670 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:49:52.670 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:48.230258+0000", 2023-09-23T20:49:52.670 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.0481000000000001e-05 2023-09-23T20:49:52.670 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.670 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.670 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:49:52.671 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:48.230340+0000", 2023-09-23T20:49:52.671 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.1658999999999999e-05 2023-09-23T20:49:52.671 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:49:52.671 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:49:52.671 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:49:52.671 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.671 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.671 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:197315 10.17 10.f2611717 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e624)", 2023-09-23T20:49:52.671 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:49:29.097858+0000", 2023-09-23T20:49:52.671 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 23.471997343000002, 2023-09-23T20:49:52.671 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.88606821700000005, 2023-09-23T20:49:52.671 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:49:52.672 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:49:52.672 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:49:52.672 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:49:52.672 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:49:52.672 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 197315 2023-09-23T20:49:52.672 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.672 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:49:52.672 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.672 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:49:52.672 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.097858+0000", 2023-09-23T20:49:52.673 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:49:52.673 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.673 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.673 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:49:52.673 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.097858+0000", 2023-09-23T20:49:52.673 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:49:52.673 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.673 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.673 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:49:52.673 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.097857+0000", 2023-09-23T20:49:52.674 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:49:52.674 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.674 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.674 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:49:52.674 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.097863+0000", 2023-09-23T20:49:52.674 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.0700000000000003e-06 2023-09-23T20:49:52.674 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.674 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.674 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:49:52.675 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.097874+0000", 2023-09-23T20:49:52.675 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0801e-05 2023-09-23T20:49:52.675 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.675 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.675 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:49:52.675 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.097879+0000", 2023-09-23T20:49:52.675 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.4739999999999996e-06 2023-09-23T20:49:52.675 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.675 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.675 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:49:52.675 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.097922+0000", 2023-09-23T20:49:52.675 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.248e-05 2023-09-23T20:49:52.676 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.676 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.676 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:49:52.676 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.097929+0000", 2023-09-23T20:49:52.676 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.0430000000000002e-06 2023-09-23T20:49:52.676 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.676 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.676 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:49:52.676 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.983916+0000", 2023-09-23T20:49:52.676 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.88598697999999998 2023-09-23T20:49:52.676 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.676 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.677 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:49:52.677 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.983926+0000", 2023-09-23T20:49:52.677 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.064e-05 2023-09-23T20:49:52.677 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:49:52.677 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:49:52.677 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:49:52.677 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.677 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.677 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:197329 10.17 10.fef47177 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e624)", 2023-09-23T20:49:52.677 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:49:29.114545+0000", 2023-09-23T20:49:52.677 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 23.455310048000001, 2023-09-23T20:49:52.678 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.86958525200000003, 2023-09-23T20:49:52.678 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:49:52.678 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:49:52.678 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:49:52.678 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:49:52.678 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:49:52.678 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 197329 2023-09-23T20:49:52.678 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.678 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:49:52.678 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.679 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:49:52.679 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.114545+0000", 2023-09-23T20:49:52.679 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:49:52.679 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.679 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.679 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:49:52.679 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.114545+0000", 2023-09-23T20:49:52.679 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:49:52.679 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.679 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.679 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:49:52.679 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.114544+0000", 2023-09-23T20:49:52.680 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:49:52.680 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.680 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.680 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:49:52.680 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.114550+0000", 2023-09-23T20:49:52.680 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.3679999999999998e-06 2023-09-23T20:49:52.680 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.681 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.681 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:49:52.681 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.114567+0000", 2023-09-23T20:49:52.681 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.6520000000000001e-05 2023-09-23T20:49:52.681 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.681 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.681 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:49:52.681 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.114573+0000", 2023-09-23T20:49:52.681 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.3749999999999999e-06 2023-09-23T20:49:52.681 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.681 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.681 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:49:52.682 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.114613+0000", 2023-09-23T20:49:52.682 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.9544e-05 2023-09-23T20:49:52.682 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.682 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.682 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:49:52.682 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.114620+0000", 2023-09-23T20:49:52.682 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.452e-06 2023-09-23T20:49:52.682 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.682 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.683 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:49:52.683 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.984118+0000", 2023-09-23T20:49:52.683 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.86949812000000004 2023-09-23T20:49:52.683 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.683 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.683 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:49:52.683 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.984131+0000", 2023-09-23T20:49:52.683 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.2431e-05 2023-09-23T20:49:52.683 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:49:52.683 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:49:52.683 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:49:52.683 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.684 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.684 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:197351 10.17 10.81d04937 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e624)", 2023-09-23T20:49:52.684 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:49:29.134888+0000", 2023-09-23T20:49:52.684 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 23.434967611000001, 2023-09-23T20:49:52.684 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.84927483299999995, 2023-09-23T20:49:52.684 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:49:52.684 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:49:52.684 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:49:52.684 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:49:52.684 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:49:52.684 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 197351 2023-09-23T20:49:52.684 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.685 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:49:52.685 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.685 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:49:52.685 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.134888+0000", 2023-09-23T20:49:52.685 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:49:52.685 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.685 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.685 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:49:52.685 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.134888+0000", 2023-09-23T20:49:52.685 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:49:52.685 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.686 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.686 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:49:52.686 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.134886+0000", 2023-09-23T20:49:52.686 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:49:52.686 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.686 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.686 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:49:52.686 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.134893+0000", 2023-09-23T20:49:52.686 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.404e-06 2023-09-23T20:49:52.686 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.687 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.687 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:49:52.687 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.134907+0000", 2023-09-23T20:49:52.687 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.4450000000000001e-05 2023-09-23T20:49:52.687 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.687 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.687 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:49:52.687 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.134914+0000", 2023-09-23T20:49:52.687 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.5390000000000004e-06 2023-09-23T20:49:52.687 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.687 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.687 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:49:52.688 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.134961+0000", 2023-09-23T20:49:52.688 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.7648000000000002e-05 2023-09-23T20:49:52.688 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.688 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.688 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:49:52.688 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.134969+0000", 2023-09-23T20:49:52.688 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.4460000000000004e-06 2023-09-23T20:49:52.688 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.688 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.688 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:49:52.688 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.984152+0000", 2023-09-23T20:49:52.689 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.84918333999999995 2023-09-23T20:49:52.689 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.689 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.689 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:49:52.689 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.984163+0000", 2023-09-23T20:49:52.689 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0417999999999999e-05 2023-09-23T20:49:52.689 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:49:52.689 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:49:52.689 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:49:52.689 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.689 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.690 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:197367 10.17 10.33935bd7 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e624)", 2023-09-23T20:49:52.690 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:49:29.147908+0000", 2023-09-23T20:49:52.690 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 23.421947157999998, 2023-09-23T20:49:52.690 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.83631422499999997, 2023-09-23T20:49:52.690 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:49:52.690 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:49:52.690 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:49:52.690 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:49:52.690 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:49:52.690 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 197367 2023-09-23T20:49:52.691 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.691 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:49:52.691 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.691 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:49:52.691 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.147908+0000", 2023-09-23T20:49:52.691 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:49:52.691 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.691 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.691 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:49:52.691 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.147908+0000", 2023-09-23T20:49:52.691 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:49:52.692 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.692 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.692 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:49:52.692 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.147907+0000", 2023-09-23T20:49:52.692 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:49:52.692 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.692 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.692 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:49:52.692 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.147914+0000", 2023-09-23T20:49:52.692 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.4540000000000001e-06 2023-09-23T20:49:52.692 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.693 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.693 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:49:52.693 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.147926+0000", 2023-09-23T20:49:52.693 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1307e-05 2023-09-23T20:49:52.693 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.693 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.693 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:49:52.693 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.147945+0000", 2023-09-23T20:49:52.693 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.9315999999999999e-05 2023-09-23T20:49:52.693 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.693 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.693 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:49:52.694 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.148674+0000", 2023-09-23T20:49:52.694 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00072932400000000005 2023-09-23T20:49:52.694 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.694 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.694 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:49:52.694 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.148682+0000", 2023-09-23T20:49:52.694 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.3640000000000001e-06 2023-09-23T20:49:52.694 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.694 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.694 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:49:52.694 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.984212+0000", 2023-09-23T20:49:52.695 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.83553048200000002 2023-09-23T20:49:52.695 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.695 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.695 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:49:52.695 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.984222+0000", 2023-09-23T20:49:52.695 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0393e-05 2023-09-23T20:49:52.695 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:49:52.695 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:49:52.695 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:49:52.695 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.695 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.695 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:197369 10.17 10.4ddbadb7 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e624)", 2023-09-23T20:49:52.696 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:49:29.150060+0000", 2023-09-23T20:49:52.696 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 23.419795186000002, 2023-09-23T20:49:52.696 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.83419653900000001, 2023-09-23T20:49:52.696 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:49:52.696 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:49:52.696 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:49:52.696 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:49:52.696 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:49:52.696 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 197369 2023-09-23T20:49:52.696 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.696 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:49:52.697 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.697 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:49:52.697 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.150060+0000", 2023-09-23T20:49:52.697 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:49:52.697 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.697 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.697 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:49:52.697 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.150060+0000", 2023-09-23T20:49:52.697 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:49:52.697 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.697 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.697 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:49:52.698 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.150059+0000", 2023-09-23T20:49:52.698 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:49:52.698 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.698 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.698 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:49:52.698 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.150065+0000", 2023-09-23T20:49:52.698 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.015e-06 2023-09-23T20:49:52.698 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.698 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.698 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:49:52.698 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.150075+0000", 2023-09-23T20:49:52.699 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0196e-05 2023-09-23T20:49:52.699 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.699 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.699 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:49:52.699 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.150081+0000", 2023-09-23T20:49:52.699 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.8379999999999998e-06 2023-09-23T20:49:52.699 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.699 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.699 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:49:52.699 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.150137+0000", 2023-09-23T20:49:52.699 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.6044000000000001e-05 2023-09-23T20:49:52.699 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.700 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.700 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:49:52.700 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.150144+0000", 2023-09-23T20:49:52.700 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.4590000000000001e-06 2023-09-23T20:49:52.700 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.700 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.700 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:49:52.700 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.984249+0000", 2023-09-23T20:49:52.700 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.83410500499999995 2023-09-23T20:49:52.700 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.700 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.701 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:49:52.701 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.984257+0000", 2023-09-23T20:49:52.701 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.272e-06 2023-09-23T20:49:52.701 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:49:52.701 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:49:52.701 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:49:52.701 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.701 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.701 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:197390 10.17 10.2b417377 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e624)", 2023-09-23T20:49:52.701 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:49:29.168757+0000", 2023-09-23T20:49:52.701 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 23.401098639000001, 2023-09-23T20:49:52.701 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.81552972800000001, 2023-09-23T20:49:52.702 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:49:52.702 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:49:52.702 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:49:52.702 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:49:52.702 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:49:52.702 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 197390 2023-09-23T20:49:52.702 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.703 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:49:52.703 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.703 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:49:52.703 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.168757+0000", 2023-09-23T20:49:52.703 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:49:52.703 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.704 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.704 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:49:52.704 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.168757+0000", 2023-09-23T20:49:52.704 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:49:52.704 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.704 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.704 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:49:52.704 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.168756+0000", 2023-09-23T20:49:52.704 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:49:52.704 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.704 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.704 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:49:52.705 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.168761+0000", 2023-09-23T20:49:52.705 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.8900000000000004e-06 2023-09-23T20:49:52.705 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.705 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.705 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:49:52.705 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.168776+0000", 2023-09-23T20:49:52.705 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.4773000000000001e-05 2023-09-23T20:49:52.705 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.705 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.706 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:49:52.706 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.168783+0000", 2023-09-23T20:49:52.706 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.5849999999999996e-06 2023-09-23T20:49:52.706 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.706 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.706 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:49:52.706 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.168828+0000", 2023-09-23T20:49:52.706 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.4920999999999999e-05 2023-09-23T20:49:52.707 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.707 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.707 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:49:52.707 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.168835+0000", 2023-09-23T20:49:52.707 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.3529999999999997e-06 2023-09-23T20:49:52.707 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.707 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.707 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:49:52.707 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.984277+0000", 2023-09-23T20:49:52.707 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.81544192599999998 2023-09-23T20:49:52.707 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.708 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.708 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:49:52.708 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.984286+0000", 2023-09-23T20:49:52.708 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.4530000000000008e-06 2023-09-23T20:49:52.708 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:49:52.708 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:49:52.708 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:49:52.708 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.708 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.708 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:197413 10.17 10.ebd7cd77 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e624)", 2023-09-23T20:49:52.708 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:49:29.195027+0000", 2023-09-23T20:49:52.708 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 23.37482868, 2023-09-23T20:49:52.709 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.78928548600000004, 2023-09-23T20:49:52.709 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:49:52.709 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:49:52.709 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:49:52.709 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:49:52.709 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:49:52.709 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 197413 2023-09-23T20:49:52.709 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.709 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:49:52.709 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.709 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:49:52.710 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.195027+0000", 2023-09-23T20:49:52.710 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:49:52.710 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.710 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.710 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:49:52.710 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.195027+0000", 2023-09-23T20:49:52.710 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:49:52.710 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.710 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.710 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:49:52.710 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.195025+0000", 2023-09-23T20:49:52.711 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:49:52.711 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.711 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.711 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:49:52.711 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.195033+0000", 2023-09-23T20:49:52.711 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.1179999999999992e-06 2023-09-23T20:49:52.711 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.711 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.711 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:49:52.711 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.195053+0000", 2023-09-23T20:49:52.711 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.0038e-05 2023-09-23T20:49:52.711 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.712 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.712 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:49:52.712 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.195061+0000", 2023-09-23T20:49:52.712 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.763e-06 2023-09-23T20:49:52.712 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.712 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.712 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:49:52.712 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.195099+0000", 2023-09-23T20:49:52.712 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.7815000000000003e-05 2023-09-23T20:49:52.712 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.712 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.713 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:49:52.713 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.195106+0000", 2023-09-23T20:49:52.713 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.618e-06 2023-09-23T20:49:52.713 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.713 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.713 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:49:52.713 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.984306+0000", 2023-09-23T20:49:52.713 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.78920079600000004 2023-09-23T20:49:52.713 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.713 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.713 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:49:52.713 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.984312+0000", 2023-09-23T20:49:52.714 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.8760000000000002e-06 2023-09-23T20:49:52.714 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:49:52.714 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:49:52.714 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:49:52.714 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.714 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.714 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:197441 10.17 10.792e7cf7 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e624)", 2023-09-23T20:49:52.714 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:49:29.235108+0000", 2023-09-23T20:49:52.714 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 23.334747624999999, 2023-09-23T20:49:52.714 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.74923685299999998, 2023-09-23T20:49:52.714 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:49:52.715 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:49:52.715 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:49:52.715 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:49:52.715 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:49:52.715 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 197441 2023-09-23T20:49:52.715 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.715 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:49:52.715 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.715 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:49:52.715 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.235108+0000", 2023-09-23T20:49:52.715 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:49:52.715 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.716 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.716 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:49:52.716 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.235108+0000", 2023-09-23T20:49:52.716 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:49:52.716 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.716 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.716 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:49:52.716 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.235107+0000", 2023-09-23T20:49:52.716 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:49:52.716 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.716 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.716 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:49:52.717 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.235111+0000", 2023-09-23T20:49:52.717 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.9320000000000004e-06 2023-09-23T20:49:52.717 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.717 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.717 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:49:52.717 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.235123+0000", 2023-09-23T20:49:52.717 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1229999999999999e-05 2023-09-23T20:49:52.717 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.717 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.717 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:49:52.717 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.235127+0000", 2023-09-23T20:49:52.718 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.7859999999999996e-06 2023-09-23T20:49:52.718 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.718 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.718 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:49:52.718 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.235167+0000", 2023-09-23T20:49:52.718 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.9158e-05 2023-09-23T20:49:52.718 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.718 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.718 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:49:52.718 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.235172+0000", 2023-09-23T20:49:52.718 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.4999999999999999e-06 2023-09-23T20:49:52.719 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.719 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.719 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:49:52.719 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.984338+0000", 2023-09-23T20:49:52.719 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.74916548599999999 2023-09-23T20:49:52.719 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:49:52.719 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:49:52.719 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:49:52.719 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.984345+0000", 2023-09-23T20:49:52.719 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.9979999999999997e-06 2023-09-23T20:49:52.719 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:49:52.720 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:49:52.720 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:49:52.720 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:49:52.720 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:49:52.720 INFO:teuthology.orchestra.run.smithi149.stdout:} 2023-09-23T20:49:52.720 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:49:52.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:49:52.722 INFO:teuthology.orchestra.run.smithi018.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2023-09-23T20:49:52.722 DEBUG:teuthology.orchestra.run.smithi018:> mkdir -- /home/ubuntu/cephtest/mnt.0 2023-09-23T20:49:52.761 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:52.783 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2023-09-23T20:49:52.784 DEBUG:teuthology.orchestra.run.smithi018:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2023-09-23T20:49:52.841 INFO:tasks.workunit:timeout=3h 2023-09-23T20:49:52.841 INFO:tasks.workunit:cleanup=True 2023-09-23T20:49:52.842 DEBUG:teuthology.orchestra.run.smithi018:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone --depth 1 --branch quincy https://github.com/ceph/ceph.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 2023-09-23T20:49:52.933 INFO:tasks.workunit.client.0.smithi018.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2023-09-23T20:49:53.013 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:49:53.016 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:53.016 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:49:53.077 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T20:49:53.117 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:49:53.125 INFO:teuthology.orchestra.run.smithi018.stderr:nodeep-scrub is set 2023-09-23T20:49:53.147 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:53.186 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 7, 3, 6, 4, 2] out_osds: [5] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T20:49:53.186 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:49:53.187 INFO:tasks.thrashosds.thrasher:Growing pool .mgr 2023-09-23T20:49:53.187 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:49:53.191 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:49:53.195 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:53.195 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:49:53.296 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:49:53.326 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:53.362 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:49:53.365 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:53.365 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:49:53.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:53 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3773894891' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T20:49:53.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:53 smithi018 ceph-mon[184144]: osdmap e641: 8 total, 8 up, 7 in 2023-09-23T20:49:53.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:53 smithi018 ceph-mon[184144]: failed to encode map e641 with expected crc 2023-09-23T20:49:53.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:53 smithi018 ceph-mon[184144]: pgmap v1570: 103 pgs: 4 peering, 1 active+undersized+remapped, 7 remapped+peering, 91 active+clean; 4.6 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail 2023-09-23T20:49:53.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:53 smithi018 ceph-mon[184144]: failed to encode map e641 with expected crc 2023-09-23T20:49:53.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:53 smithi018 ceph-mon[184144]: failed to encode map e641 with expected crc 2023-09-23T20:49:53.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:53 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/566010302' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:49:53.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:53 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:49:53.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:53 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3773894891' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T20:49:53.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:53 smithi018 ceph-mon[179647]: osdmap e641: 8 total, 8 up, 7 in 2023-09-23T20:49:53.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:53 smithi018 ceph-mon[179647]: failed to encode map e641 with expected crc 2023-09-23T20:49:53.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:53 smithi018 ceph-mon[179647]: pgmap v1570: 103 pgs: 4 peering, 1 active+undersized+remapped, 7 remapped+peering, 91 active+clean; 4.6 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail 2023-09-23T20:49:53.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:53 smithi018 ceph-mon[179647]: failed to encode map e641 with expected crc 2023-09-23T20:49:53.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:53 smithi018 ceph-mon[179647]: failed to encode map e641 with expected crc 2023-09-23T20:49:53.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:53 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/566010302' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:49:53.393 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:53 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:49:53.466 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:49:53.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:53 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3773894891' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T20:49:53.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:53 smithi149 ceph-mon[170394]: osdmap e641: 8 total, 8 up, 7 in 2023-09-23T20:49:53.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:53 smithi149 ceph-mon[170394]: failed to encode map e641 with expected crc 2023-09-23T20:49:53.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:53 smithi149 ceph-mon[170394]: pgmap v1570: 103 pgs: 4 peering, 1 active+undersized+remapped, 7 remapped+peering, 91 active+clean; 4.6 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail 2023-09-23T20:49:53.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:53 smithi149 ceph-mon[170394]: failed to encode map e641 with expected crc 2023-09-23T20:49:53.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:53 smithi149 ceph-mon[170394]: failed to encode map e641 with expected crc 2023-09-23T20:49:53.468 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:53 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/566010302' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:49:53.468 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:53 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:49:53.489 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:49:53.494 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:53.696 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:49:53.881 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:49:53.883 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:53.884 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:49:53.984 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:49:54.012 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:54.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:54 smithi018 ceph-mon[184144]: failed to encode map e641 with expected crc 2023-09-23T20:49:54.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:54 smithi018 ceph-mon[184144]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:49:54.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:54 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:49:54.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:54 smithi018 ceph-mon[184144]: osdmap e642: 8 total, 8 up, 7 in 2023-09-23T20:49:54.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:54 smithi018 ceph-mon[184144]: failed to encode map e642 with expected crc 2023-09-23T20:49:54.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:54 smithi018 ceph-mon[184144]: failed to encode map e642 with expected crc 2023-09-23T20:49:54.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:54 smithi018 ceph-mon[179647]: failed to encode map e641 with expected crc 2023-09-23T20:49:54.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:54 smithi018 ceph-mon[179647]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:49:54.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:54 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:49:54.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:54 smithi018 ceph-mon[179647]: osdmap e642: 8 total, 8 up, 7 in 2023-09-23T20:49:54.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:54 smithi018 ceph-mon[179647]: failed to encode map e642 with expected crc 2023-09-23T20:49:54.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:54 smithi018 ceph-mon[179647]: failed to encode map e642 with expected crc 2023-09-23T20:49:54.197 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:54 smithi149 ceph-mon[170394]: failed to encode map e641 with expected crc 2023-09-23T20:49:54.197 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:54 smithi149 ceph-mon[170394]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:49:54.198 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:54 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:49:54.198 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:54 smithi149 ceph-mon[170394]: osdmap e642: 8 total, 8 up, 7 in 2023-09-23T20:49:54.198 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:54 smithi149 ceph-mon[170394]: failed to encode map e642 with expected crc 2023-09-23T20:49:54.198 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:54 smithi149 ceph-mon[170394]: failed to encode map e642 with expected crc 2023-09-23T20:49:54.912 INFO:teuthology.orchestra.run.smithi149.stdout:{ 2023-09-23T20:49:54.912 INFO:teuthology.orchestra.run.smithi149.stdout: "ops": [], 2023-09-23T20:49:54.912 INFO:teuthology.orchestra.run.smithi149.stdout: "num_ops": 0 2023-09-23T20:49:54.912 INFO:teuthology.orchestra.run.smithi149.stdout:} 2023-09-23T20:49:55.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:55 smithi149 ceph-mon[170394]: pgmap v1572: 103 pgs: 4 peering, 1 active+undersized+remapped, 98 active+clean; 4.6 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail 2023-09-23T20:49:55.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:55 smithi149 ceph-mon[170394]: failed to encode map e642 with expected crc 2023-09-23T20:49:55.356 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T20:49:55.391 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:49:54 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:49:54] "GET /metrics HTTP/1.1" 200 36576 "" "Prometheus/2.43.0" 2023-09-23T20:49:55.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:55 smithi018 ceph-mon[179647]: pgmap v1572: 103 pgs: 4 peering, 1 active+undersized+remapped, 98 active+clean; 4.6 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail 2023-09-23T20:49:55.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:55 smithi018 ceph-mon[179647]: failed to encode map e642 with expected crc 2023-09-23T20:49:55.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:55 smithi018 ceph-mon[184144]: pgmap v1572: 103 pgs: 4 peering, 1 active+undersized+remapped, 98 active+clean; 4.6 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail 2023-09-23T20:49:55.393 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:55 smithi018 ceph-mon[184144]: failed to encode map e642 with expected crc 2023-09-23T20:49:55.444 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:49:55.447 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:55.448 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:49:55.549 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:49:55.578 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:55.613 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:49:55.615 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:55.616 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:49:55.639 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:49:55.717 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:49:55.744 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:55.778 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:49:55.781 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:55.782 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:49:55.883 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:49:55.909 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:55.943 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:49:55.946 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:55.946 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:49:55.959 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:49:56.047 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:49:56.073 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:56.109 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:49:56.112 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:56.112 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:49:56.169 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:56 smithi149 ceph-mon[170394]: failed to encode map e642 with expected crc 2023-09-23T20:49:56.213 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:49:56.242 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:56.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:56 smithi018 ceph-mon[184144]: failed to encode map e642 with expected crc 2023-09-23T20:49:56.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:56 smithi018 ceph-mon[179647]: failed to encode map e642 with expected crc 2023-09-23T20:49:56.400 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:49:56.402 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:49:56.403 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:56.403 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:49:56.504 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:49:56.532 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:56.612 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:49:56.617 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:56.617 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:49:56.718 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:49:56.746 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:57.189 INFO:teuthology.orchestra.run.smithi149.stdout:{ 2023-09-23T20:49:57.190 INFO:teuthology.orchestra.run.smithi149.stdout: "ops": [], 2023-09-23T20:49:57.190 INFO:teuthology.orchestra.run.smithi149.stdout: "complaint_time": 30, 2023-09-23T20:49:57.190 INFO:teuthology.orchestra.run.smithi149.stdout: "num_blocked_ops": 0 2023-09-23T20:49:57.190 INFO:teuthology.orchestra.run.smithi149.stdout:} 2023-09-23T20:49:57.532 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T20:49:57.583 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:49:57.586 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:57.587 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:49:57.623 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T20:49:57.687 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:49:57.716 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:57.753 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:49:57.756 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:57.756 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:49:57.857 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:49:57.883 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:57 smithi018 ceph-mon[184144]: pgmap v1573: 103 pgs: 103 active+clean; 4.6 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail 2023-09-23T20:49:57.883 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:57 smithi018 ceph-mon[179647]: pgmap v1573: 103 pgs: 103 active+clean; 4.6 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail 2023-09-23T20:49:57.885 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:57.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:57 smithi149 ceph-mon[170394]: pgmap v1573: 103 pgs: 103 active+clean; 4.6 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail 2023-09-23T20:49:58.236 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:49:58.953 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:49:58.957 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:58.957 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:49:59.058 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:49:59.089 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:59.125 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:49:59.128 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:59.129 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:49:59.189 DEBUG:teuthology.orchestra.run.smithi018:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2023-09-23T20:49:59.207 INFO:tasks.workunit.client.0.smithi018.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2023-09-23T20:49:59.210 INFO:tasks.workunit.client.0.smithi018.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-09-23T20:49:59.210 INFO:tasks.workunit.client.0.smithi018.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2023-09-23T20:49:59.229 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:49:59.257 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:59.270 INFO:tasks.workunit.client.0.smithi018.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2023-09-23T20:49:59.333 INFO:tasks.workunit.client.0.smithi018.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2023-09-23T20:49:59.343 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:49:59.347 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:59.348 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:49:59.390 INFO:tasks.workunit.client.0.smithi018.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-09-23T20:49:59.393 INFO:tasks.workunit.client.0.smithi018.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-09-23T20:49:59.393 INFO:tasks.workunit.client.0.smithi018.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2023-09-23T20:49:59.437 INFO:tasks.workunit.client.0.smithi018.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-09-23T20:49:59.448 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:49:59.482 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:59.513 DEBUG:teuthology.orchestra.run.smithi018:> set -ex 2023-09-23T20:49:59.513 DEBUG:teuthology.orchestra.run.smithi018:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2023-09-23T20:49:59.522 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:49:59.522 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:49:59.522 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:49:59.523 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:49:59.523 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:49:59.528 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:49:59.531 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:59.531 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:49:59.539 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:49:59.551 INFO:teuthology.orchestra.run.smithi149.stdout:ERROR: (22) Invalid argument 2023-09-23T20:49:59.551 INFO:teuthology.orchestra.run.smithi149.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:49:59.557 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:49:59.573 INFO:tasks.workunit:Running workunits matching cls/test_cls_rbd.sh on client.0... 2023-09-23T20:49:59.574 INFO:tasks.workunit:Running workunit cls/test_cls_rbd.sh... 2023-09-23T20:49:59.575 DEBUG:teuthology.orchestra.run.smithi018:workunit test cls/test_cls_rbd.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh 2023-09-23T20:49:59.577 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:49:59.580 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:49:59.580 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:49:59.580 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:49:59.580 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:49:59.580 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:49:59.580 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:49:59.581 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:49:59.581 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:49:59.625 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:49:59.632 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:49:59.686 INFO:tasks.workunit.client.0.smithi018.stdout:Running main() from gmock_main.cc 2023-09-23T20:49:59.686 INFO:tasks.workunit.client.0.smithi018.stdout:[==========] Running 70 tests from 1 test suite. 2023-09-23T20:49:59.686 INFO:tasks.workunit.client.0.smithi018.stdout:[----------] Global test environment set-up. 2023-09-23T20:49:59.686 INFO:tasks.workunit.client.0.smithi018.stdout:[----------] 70 tests from TestClsRbd 2023-09-23T20:49:59.692 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:59.739 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:49:59.742 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:49:59.743 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:49:59.844 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:49:59.873 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:49:59.966 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:49:59.969 INFO:teuthology.orchestra.run.smithi018.stderr:dumped all 2023-09-23T20:49:59.970 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:59 smithi149 ceph-mon[170394]: pgmap v1574: 103 pgs: 103 active+clean; 4.6 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail 2023-09-23T20:49:59.970 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:59 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2973959397' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:49:59.970 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:59 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:49:59.970 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:59 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:49:59.970 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:49:59 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:49:59.991 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:59 smithi018 ceph-mon[179647]: pgmap v1574: 103 pgs: 103 active+clean; 4.6 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail 2023-09-23T20:49:59.991 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:59 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2973959397' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:49:59.992 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:59 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:49:59.992 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:59 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:49:59.992 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:49:59 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:49:59.992 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:59 smithi018 ceph-mon[184144]: pgmap v1574: 103 pgs: 103 active+clean; 4.6 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail 2023-09-23T20:49:59.992 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:59 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2973959397' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:49:59.993 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:59 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:49:59.993 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:59 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:49:59.993 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:49:59 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:50:00.001 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T20:50:00.025 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:50:00.030 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:00.030 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:50:00.132 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:50:00.161 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:00.204 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:50:00.209 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:00.210 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:50:00.311 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:50:00.340 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:00.381 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:50:00.385 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:00.385 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:50:00.486 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:50:00.516 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:00.633 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:50:00.854 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:50:00.858 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:00.859 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:50:00.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:00 smithi018 ceph-mon[184144]: from='client.47057 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:50:00.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:00 smithi018 ceph-mon[184144]: Health detail: HEALTH_WARN noscrub,nodeep-scrub flag(s) set 2023-09-23T20:50:00.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:00 smithi018 ceph-mon[184144]: [WRN] OSDMAP_FLAGS: noscrub,nodeep-scrub flag(s) set 2023-09-23T20:50:00.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:00 smithi018 ceph-mon[179647]: from='client.47057 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:50:00.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:00 smithi018 ceph-mon[179647]: Health detail: HEALTH_WARN noscrub,nodeep-scrub flag(s) set 2023-09-23T20:50:00.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:00 smithi018 ceph-mon[179647]: [WRN] OSDMAP_FLAGS: noscrub,nodeep-scrub flag(s) set 2023-09-23T20:50:00.916 INFO:teuthology.orchestra.run.smithi018.stdout:{"pg_ready":true,"pg_map":{"version":1574,"stamp":"2023-09-23T20:49:58.359426+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4817030,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6005,"num_read_kb":13241,"num_write":918,"num_write_kb":12834,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":276,"num_bytes_recovered":25787567,"num_keys_recovered":80,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6022,"ondisk_log_size":6022,"up":309,"acting":309,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":335,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":5467756,"kb_used_data":89716,"kb_used_omap":32,"kb_used_meta":5377951,"kb_avail":650748308,"statfs":{"total":671965249536,"available":666366267392,"internally_reserved":0,"allocated":91869184,"data_stored":74174187,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32773,"internal_metadata":5507022843},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":38,"apply_latency_ms":38,"commit_latency_ns":38000000,"apply_latency_ns":38000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-32768,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":-1,"num_read_kb":-1,"num_write":-96853,"num_write_kb":-2912353,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-10108,"num_bytes_recovered":-663649187,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"7.252474"},"pg_stats":[{"pgid":"1.18","version":"90'161","reported_seq":704,"reported_epoch":640,"state":"active+clean","last_fresh":"2023-09-23T20:49:51.130550+0000","last_change":"2023-09-23T20:49:51.130089+0000","last_active":"2023-09-23T20:49:51.130550+0000","last_peered":"2023-09-23T20:49:51.130550+0000","last_clean":"2023-09-23T20:49:51.130550+0000","last_became_active":"2023-09-23T20:49:51.124899+0000","last_became_peered":"2023-09-23T20:49:51.124899+0000","last_unstale":"2023-09-23T20:49:51.130550+0000","last_undegraded":"2023-09-23T20:49:51.130550+0000","last_fullsized":"2023-09-23T20:49:51.130550+0000","mapping_epoch":639,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":640,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:19.677209+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:19.677209+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:58:59.316978+0000","scrub_duration":0.033122453000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"69'1","reported_seq":118,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-09-23T20:49:52.307870+0000","last_change":"2023-09-23T20:48:48.148250+0000","last_active":"2023-09-23T20:49:52.307870+0000","last_peered":"2023-09-23T20:49:52.307870+0000","last_clean":"2023-09-23T20:49:52.307870+0000","last_became_active":"2023-09-23T20:48:48.144271+0000","last_became_peered":"2023-09-23T20:48:48.144271+0000","last_unstale":"2023-09-23T20:49:52.307870+0000","last_undegraded":"2023-09-23T20:49:52.307870+0000","last_fullsized":"2023-09-23T20:49:52.307870+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:59:36.502986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":484,"reported_epoch":640,"state":"active+clean","last_fresh":"2023-09-23T20:49:51.116593+0000","last_change":"2023-09-23T20:49:02.703099+0000","last_active":"2023-09-23T20:49:51.116593+0000","last_peered":"2023-09-23T20:49:51.116593+0000","last_clean":"2023-09-23T20:49:51.116593+0000","last_became_active":"2023-09-23T20:49:02.702753+0000","last_became_peered":"2023-09-23T20:49:02.702753+0000","last_unstale":"2023-09-23T20:49:51.116593+0000","last_undegraded":"2023-09-23T20:49:51.116593+0000","last_fullsized":"2023-09-23T20:49:51.116593+0000","mapping_epoch":603,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":604,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:24:30.834235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"90'161","reported_seq":664,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-09-23T20:49:52.309109+0000","last_change":"2023-09-23T20:49:51.121955+0000","last_active":"2023-09-23T20:49:52.309109+0000","last_peered":"2023-09-23T20:49:52.309109+0000","last_clean":"2023-09-23T20:49:52.309109+0000","last_became_active":"2023-09-23T20:49:51.121641+0000","last_became_peered":"2023-09-23T20:49:51.121641+0000","last_unstale":"2023-09-23T20:49:52.309109+0000","last_undegraded":"2023-09-23T20:49:52.309109+0000","last_fullsized":"2023-09-23T20:49:52.309109+0000","mapping_epoch":639,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":640,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:47:33.523187+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:36:05.442288+0000","last_clean_scrub_stamp":"2023-09-23T20:47:33.523187+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:05:36.252341+0000","scrub_duration":0.013532127,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,2],"acting":[4,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":632,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-09-23T20:49:52.460641+0000","last_change":"2023-09-23T20:47:09.173501+0000","last_active":"2023-09-23T20:49:52.460641+0000","last_peered":"2023-09-23T20:49:52.460641+0000","last_clean":"2023-09-23T20:49:52.460641+0000","last_became_active":"2023-09-23T20:47:09.173160+0000","last_became_peered":"2023-09-23T20:47:09.173160+0000","last_unstale":"2023-09-23T20:49:52.460641+0000","last_undegraded":"2023-09-23T20:49:52.460641+0000","last_fullsized":"2023-09-23T20:49:52.460641+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:44:36.132093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":602,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-09-23T20:49:52.571537+0000","last_change":"2023-09-23T20:43:46.521939+0000","last_active":"2023-09-23T20:49:52.571537+0000","last_peered":"2023-09-23T20:49:52.571537+0000","last_clean":"2023-09-23T20:49:52.571537+0000","last_became_active":"2023-09-23T20:43:46.521185+0000","last_became_peered":"2023-09-23T20:43:46.521185+0000","last_unstale":"2023-09-23T20:49:52.571537+0000","last_undegraded":"2023-09-23T20:49:52.571537+0000","last_fullsized":"2023-09-23T20:49:52.571537+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:59:25.045627+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"90'161","reported_seq":529,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-09-23T20:49:52.308375+0000","last_change":"2023-09-23T20:47:31.475001+0000","last_active":"2023-09-23T20:49:52.308375+0000","last_peered":"2023-09-23T20:49:52.308375+0000","last_clean":"2023-09-23T20:49:52.308375+0000","last_became_active":"2023-09-23T20:47:11.617942+0000","last_became_peered":"2023-09-23T20:47:11.617942+0000","last_unstale":"2023-09-23T20:49:52.308375+0000","last_undegraded":"2023-09-23T20:49:52.308375+0000","last_fullsized":"2023-09-23T20:49:52.308375+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:47:31.474914+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:47:31.474914+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:52:05.169219+0000","scrub_duration":0.0026262870000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":438,"reported_epoch":640,"state":"active+clean","last_fresh":"2023-09-23T20:49:51.114285+0000","last_change":"2023-09-23T20:37:43.584414+0000","last_active":"2023-09-23T20:49:51.114285+0000","last_peered":"2023-09-23T20:49:51.114285+0000","last_clean":"2023-09-23T20:49:51.114285+0000","last_became_active":"2023-09-23T20:37:43.584116+0000","last_became_peered":"2023-09-23T20:37:43.584116+0000","last_unstale":"2023-09-23T20:49:51.114285+0000","last_undegraded":"2023-09-23T20:49:51.114285+0000","last_fullsized":"2023-09-23T20:49:51.114285+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:17:54.258172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":266,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-09-23T20:49:52.626329+0000","last_change":"2023-09-23T20:49:48.119179+0000","last_active":"2023-09-23T20:49:52.626329+0000","last_peered":"2023-09-23T20:49:52.626329+0000","last_clean":"2023-09-23T20:49:52.626329+0000","last_became_active":"2023-09-23T20:49:48.118849+0000","last_became_peered":"2023-09-23T20:49:48.118849+0000","last_unstale":"2023-09-23T20:49:52.626329+0000","last_undegraded":"2023-09-23T20:49:52.626329+0000","last_fullsized":"2023-09-23T20:49:52.626329+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:50:02.578569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"90'161","reported_seq":874,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-09-23T20:49:52.623380+0000","last_change":"2023-09-23T20:49:51.124760+0000","last_active":"2023-09-23T20:49:52.623380+0000","last_peered":"2023-09-23T20:49:52.623380+0000","last_clean":"2023-09-23T20:49:52.623380+0000","last_became_active":"2023-09-23T20:49:51.124406+0000","last_became_peered":"2023-09-23T20:49:51.124406+0000","last_unstale":"2023-09-23T20:49:52.623380+0000","last_undegraded":"2023-09-23T20:49:52.623380+0000","last_fullsized":"2023-09-23T20:49:52.623380+0000","mapping_epoch":639,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":640,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:03.411025+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:03.411025+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:28:15.568088+0000","scrub_duration":0.001802922,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":643,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-09-23T20:49:52.571912+0000","last_change":"2023-09-23T20:49:48.111285+0000","last_active":"2023-09-23T20:49:52.571912+0000","last_peered":"2023-09-23T20:49:52.571912+0000","last_clean":"2023-09-23T20:49:52.571912+0000","last_became_active":"2023-09-23T20:49:48.110934+0000","last_became_peered":"2023-09-23T20:49:48.110934+0000","last_unstale":"2023-09-23T20:49:52.571912+0000","last_undegraded":"2023-09-23T20:49:52.571912+0000","last_fullsized":"2023-09-23T20:49:52.571912+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:19:18.339128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":74,"reported_epoch":640,"state":"active+clean","last_fresh":"2023-09-23T20:49:51.114016+0000","last_change":"2023-09-23T20:48:23.918092+0000","last_active":"2023-09-23T20:49:51.114016+0000","last_peered":"2023-09-23T20:49:51.114016+0000","last_clean":"2023-09-23T20:49:51.114016+0000","last_became_active":"2023-09-23T20:48:23.917386+0000","last_became_peered":"2023-09-23T20:48:23.917386+0000","last_unstale":"2023-09-23T20:49:51.114016+0000","last_undegraded":"2023-09-23T20:49:51.114016+0000","last_fullsized":"2023-09-23T20:49:51.114016+0000","mapping_epoch":577,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":578,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:06:09.587517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"90'161","reported_seq":643,"reported_epoch":640,"state":"active+clean","last_fresh":"2023-09-23T20:49:51.115894+0000","last_change":"2023-09-23T20:49:36.717905+0000","last_active":"2023-09-23T20:49:51.115894+0000","last_peered":"2023-09-23T20:49:51.115894+0000","last_clean":"2023-09-23T20:49:51.115894+0000","last_became_active":"2023-09-23T20:48:51.169314+0000","last_became_peered":"2023-09-23T20:48:51.169314+0000","last_unstale":"2023-09-23T20:49:51.115894+0000","last_undegraded":"2023-09-23T20:49:51.115894+0000","last_fullsized":"2023-09-23T20:49:51.115894+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:49:36.717737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:49:36.717737+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:00:49.213712+0000","scrub_duration":0.0049746089999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":488,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-09-23T20:49:52.624078+0000","last_change":"2023-09-23T20:49:48.123820+0000","last_active":"2023-09-23T20:49:52.624078+0000","last_peered":"2023-09-23T20:49:52.624078+0000","last_clean":"2023-09-23T20:49:52.624078+0000","last_became_active":"2023-09-23T20:49:48.123514+0000","last_became_peered":"2023-09-23T20:49:48.123514+0000","last_unstale":"2023-09-23T20:49:52.624078+0000","last_undegraded":"2023-09-23T20:49:52.624078+0000","last_fullsized":"2023-09-23T20:49:52.624078+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:43:13.460464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":469,"reported_epoch":640,"state":"active+clean","last_fresh":"2023-09-23T20:49:51.113164+0000","last_change":"2023-09-23T20:38:02.542034+0000","last_active":"2023-09-23T20:49:51.113164+0000","last_peered":"2023-09-23T20:49:51.113164+0000","last_clean":"2023-09-23T20:49:51.113164+0000","last_became_active":"2023-09-23T20:38:02.541575+0000","last_became_peered":"2023-09-23T20:38:02.541575+0000","last_unstale":"2023-09-23T20:49:51.113164+0000","last_undegraded":"2023-09-23T20:49:51.113164+0000","last_fullsized":"2023-09-23T20:49:51.113164+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:50:54.591529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"90'161","reported_seq":786,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-09-23T20:49:52.571799+0000","last_change":"2023-09-23T20:49:51.125057+0000","last_active":"2023-09-23T20:49:52.571799+0000","last_peered":"2023-09-23T20:49:52.571799+0000","last_clean":"2023-09-23T20:49:52.571799+0000","last_became_active":"2023-09-23T20:49:51.124616+0000","last_became_peered":"2023-09-23T20:49:51.124616+0000","last_unstale":"2023-09-23T20:49:52.571799+0000","last_undegraded":"2023-09-23T20:49:52.571799+0000","last_fullsized":"2023-09-23T20:49:52.571799+0000","mapping_epoch":639,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":640,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:14.669831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:14.669831+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:09:01.736519+0000","scrub_duration":0.0083441460000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":701,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-09-23T20:49:52.623754+0000","last_change":"2023-09-23T20:47:10.837423+0000","last_active":"2023-09-23T20:49:52.623754+0000","last_peered":"2023-09-23T20:49:52.623754+0000","last_clean":"2023-09-23T20:49:52.623754+0000","last_became_active":"2023-09-23T20:47:09.204324+0000","last_became_peered":"2023-09-23T20:47:09.204324+0000","last_unstale":"2023-09-23T20:49:52.623754+0000","last_undegraded":"2023-09-23T20:49:52.623754+0000","last_fullsized":"2023-09-23T20:49:52.623754+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":554,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:19.108914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":540,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":176,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-09-23T20:49:52.460369+0000","last_change":"2023-09-23T20:48:48.145175+0000","last_active":"2023-09-23T20:49:52.460369+0000","last_peered":"2023-09-23T20:49:52.460369+0000","last_clean":"2023-09-23T20:49:52.460369+0000","last_became_active":"2023-09-23T20:48:48.144841+0000","last_became_peered":"2023-09-23T20:48:48.144841+0000","last_unstale":"2023-09-23T20:49:52.460369+0000","last_undegraded":"2023-09-23T20:49:52.460369+0000","last_fullsized":"2023-09-23T20:49:52.460369+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:27:40.802917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"90'161","reported_seq":769,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-09-23T20:49:52.463222+0000","last_change":"2023-09-23T20:36:02.548387+0000","last_active":"2023-09-23T20:49:52.463222+0000","last_peered":"2023-09-23T20:49:52.463222+0000","last_clean":"2023-09-23T20:49:52.463222+0000","last_became_active":"2023-09-23T20:35:26.644787+0000","last_became_peered":"2023-09-23T20:35:26.644787+0000","last_unstale":"2023-09-23T20:49:52.463222+0000","last_undegraded":"2023-09-23T20:49:52.463222+0000","last_fullsized":"2023-09-23T20:49:52.463222+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:02.548285+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:02.548285+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:37:15.216553+0000","scrub_duration":0.0015514459999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":108,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-09-23T20:49:52.308914+0000","last_change":"2023-09-23T20:47:09.340979+0000","last_active":"2023-09-23T20:49:52.308914+0000","last_peered":"2023-09-23T20:49:52.308914+0000","last_clean":"2023-09-23T20:49:52.308914+0000","last_became_active":"2023-09-23T20:47:09.231222+0000","last_became_peered":"2023-09-23T20:47:09.231222+0000","last_unstale":"2023-09-23T20:49:52.308914+0000","last_undegraded":"2023-09-23T20:49:52.308914+0000","last_fullsized":"2023-09-23T20:49:52.308914+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:07:50.620006+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":98,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-09-23T20:49:52.309329+0000","last_change":"2023-09-23T20:47:09.186315+0000","last_active":"2023-09-23T20:49:52.309329+0000","last_peered":"2023-09-23T20:49:52.309329+0000","last_clean":"2023-09-23T20:49:52.309329+0000","last_became_active":"2023-09-23T20:47:09.184580+0000","last_became_peered":"2023-09-23T20:47:09.184580+0000","last_unstale":"2023-09-23T20:49:52.309329+0000","last_undegraded":"2023-09-23T20:49:52.309329+0000","last_fullsized":"2023-09-23T20:49:52.309329+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:44:35.553948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"90'161","reported_seq":914,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-09-23T20:49:52.570440+0000","last_change":"2023-09-23T20:49:51.148240+0000","last_active":"2023-09-23T20:49:52.570440+0000","last_peered":"2023-09-23T20:49:52.570440+0000","last_clean":"2023-09-23T20:49:52.570440+0000","last_became_active":"2023-09-23T20:49:51.147475+0000","last_became_peered":"2023-09-23T20:49:51.147475+0000","last_unstale":"2023-09-23T20:49:52.570440+0000","last_undegraded":"2023-09-23T20:49:52.570440+0000","last_fullsized":"2023-09-23T20:49:52.570440+0000","mapping_epoch":639,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":640,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:43.578666+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:43.578666+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:02:42.273671+0000","scrub_duration":0.0021890379999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":732,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-09-23T20:49:52.620387+0000","last_change":"2023-09-23T20:48:48.193666+0000","last_active":"2023-09-23T20:49:52.620387+0000","last_peered":"2023-09-23T20:49:52.620387+0000","last_clean":"2023-09-23T20:49:52.620387+0000","last_became_active":"2023-09-23T20:48:48.152081+0000","last_became_peered":"2023-09-23T20:48:48.152081+0000","last_unstale":"2023-09-23T20:49:52.620387+0000","last_undegraded":"2023-09-23T20:49:52.620387+0000","last_fullsized":"2023-09-23T20:49:52.620387+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:00:19.508124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":272,"num_keys_recovered":80,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":398,"reported_epoch":640,"state":"active+clean","last_fresh":"2023-09-23T20:49:51.114579+0000","last_change":"2023-09-23T20:49:48.119365+0000","last_active":"2023-09-23T20:49:51.114579+0000","last_peered":"2023-09-23T20:49:51.114579+0000","last_clean":"2023-09-23T20:49:51.114579+0000","last_became_active":"2023-09-23T20:49:48.119009+0000","last_became_peered":"2023-09-23T20:49:48.119009+0000","last_unstale":"2023-09-23T20:49:51.114579+0000","last_undegraded":"2023-09-23T20:49:51.114579+0000","last_fullsized":"2023-09-23T20:49:51.114579+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:02:26.878542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"90'161","reported_seq":942,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-09-23T20:49:52.621150+0000","last_change":"2023-09-23T20:49:51.118813+0000","last_active":"2023-09-23T20:49:52.621150+0000","last_peered":"2023-09-23T20:49:52.621150+0000","last_clean":"2023-09-23T20:49:52.621150+0000","last_became_active":"2023-09-23T20:49:51.118445+0000","last_became_peered":"2023-09-23T20:49:51.118445+0000","last_unstale":"2023-09-23T20:49:52.621150+0000","last_undegraded":"2023-09-23T20:49:52.621150+0000","last_fullsized":"2023-09-23T20:49:52.621150+0000","mapping_epoch":639,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":640,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:20.041690+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:20.041690+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:15:24.683302+0000","scrub_duration":0.0026180330000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":599,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-09-23T20:49:52.571007+0000","last_change":"2023-09-23T20:43:46.521294+0000","last_active":"2023-09-23T20:49:52.571007+0000","last_peered":"2023-09-23T20:49:52.571007+0000","last_clean":"2023-09-23T20:49:52.571007+0000","last_became_active":"2023-09-23T20:43:46.521001+0000","last_became_peered":"2023-09-23T20:43:46.521001+0000","last_unstale":"2023-09-23T20:49:52.571007+0000","last_undegraded":"2023-09-23T20:49:52.571007+0000","last_fullsized":"2023-09-23T20:49:52.571007+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:01:35.126716+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":429,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-09-23T20:49:52.623870+0000","last_change":"2023-09-23T20:49:48.121288+0000","last_active":"2023-09-23T20:49:52.623870+0000","last_peered":"2023-09-23T20:49:52.623870+0000","last_clean":"2023-09-23T20:49:52.623870+0000","last_became_active":"2023-09-23T20:49:48.120936+0000","last_became_peered":"2023-09-23T20:49:48.120936+0000","last_unstale":"2023-09-23T20:49:52.623870+0000","last_undegraded":"2023-09-23T20:49:52.623870+0000","last_fullsized":"2023-09-23T20:49:52.623870+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:46:44.012748+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.13","version":"90'161","reported_seq":965,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-09-23T20:49:52.462194+0000","last_change":"2023-09-23T20:47:11.619371+0000","last_active":"2023-09-23T20:49:52.462194+0000","last_peered":"2023-09-23T20:49:52.462194+0000","last_clean":"2023-09-23T20:49:52.462194+0000","last_became_active":"2023-09-23T20:47:11.619006+0000","last_became_peered":"2023-09-23T20:47:11.619006+0000","last_unstale":"2023-09-23T20:49:52.462194+0000","last_undegraded":"2023-09-23T20:49:52.462194+0000","last_fullsized":"2023-09-23T20:49:52.462194+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:42.544308+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:42.544308+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:47:24.414807+0000","scrub_duration":0.0029958179999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":665,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-09-23T20:49:52.622203+0000","last_change":"2023-09-23T20:47:09.176613+0000","last_active":"2023-09-23T20:49:52.622203+0000","last_peered":"2023-09-23T20:49:52.622203+0000","last_clean":"2023-09-23T20:49:52.622203+0000","last_became_active":"2023-09-23T20:47:09.176187+0000","last_became_peered":"2023-09-23T20:47:09.176187+0000","last_unstale":"2023-09-23T20:49:52.622203+0000","last_undegraded":"2023-09-23T20:49:52.622203+0000","last_fullsized":"2023-09-23T20:49:52.622203+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:23:01.590989+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":608,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-09-23T20:49:52.571535+0000","last_change":"2023-09-23T20:43:46.521735+0000","last_active":"2023-09-23T20:49:52.571535+0000","last_peered":"2023-09-23T20:49:52.571535+0000","last_clean":"2023-09-23T20:49:52.571535+0000","last_became_active":"2023-09-23T20:43:46.520762+0000","last_became_peered":"2023-09-23T20:43:46.520762+0000","last_unstale":"2023-09-23T20:49:52.571535+0000","last_undegraded":"2023-09-23T20:49:52.571535+0000","last_fullsized":"2023-09-23T20:49:52.571535+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:26:48.841930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"90'161","reported_seq":744,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-09-23T20:49:52.461885+0000","last_change":"2023-09-23T20:49:51.125786+0000","last_active":"2023-09-23T20:49:52.461885+0000","last_peered":"2023-09-23T20:49:52.461885+0000","last_clean":"2023-09-23T20:49:52.461885+0000","last_became_active":"2023-09-23T20:49:51.124373+0000","last_became_peered":"2023-09-23T20:49:51.124373+0000","last_unstale":"2023-09-23T20:49:52.461885+0000","last_undegraded":"2023-09-23T20:49:52.461885+0000","last_fullsized":"2023-09-23T20:49:52.461885+0000","mapping_epoch":639,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":640,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:18.952693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:18.952693+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:07:29.170332+0000","scrub_duration":0.001699365,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":336,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-09-23T20:49:52.630865+0000","last_change":"2023-09-23T20:49:48.127484+0000","last_active":"2023-09-23T20:49:52.630865+0000","last_peered":"2023-09-23T20:49:52.630865+0000","last_clean":"2023-09-23T20:49:52.630865+0000","last_became_active":"2023-09-23T20:49:48.110748+0000","last_became_peered":"2023-09-23T20:49:48.110748+0000","last_unstale":"2023-09-23T20:49:52.630865+0000","last_undegraded":"2023-09-23T20:49:52.630865+0000","last_fullsized":"2023-09-23T20:49:52.630865+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:31:51.004529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":438,"reported_epoch":640,"state":"active+clean","last_fresh":"2023-09-23T20:49:51.115645+0000","last_change":"2023-09-23T20:37:43.571083+0000","last_active":"2023-09-23T20:49:51.115645+0000","last_peered":"2023-09-23T20:49:51.115645+0000","last_clean":"2023-09-23T20:49:51.115645+0000","last_became_active":"2023-09-23T20:37:43.570306+0000","last_became_peered":"2023-09-23T20:37:43.570306+0000","last_unstale":"2023-09-23T20:49:51.115645+0000","last_undegraded":"2023-09-23T20:49:51.115645+0000","last_fullsized":"2023-09-23T20:49:51.115645+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:42:02.396718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.d","version":"90'161","reported_seq":974,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-09-23T20:49:52.623467+0000","last_change":"2023-09-23T20:47:11.619510+0000","last_active":"2023-09-23T20:49:52.623467+0000","last_peered":"2023-09-23T20:49:52.623467+0000","last_clean":"2023-09-23T20:49:52.623467+0000","last_became_active":"2023-09-23T20:47:11.619154+0000","last_became_peered":"2023-09-23T20:47:11.619154+0000","last_unstale":"2023-09-23T20:49:52.623467+0000","last_undegraded":"2023-09-23T20:49:52.623467+0000","last_fullsized":"2023-09-23T20:49:52.623467+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:34:59.496375+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:34:59.496375+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:40:52.712777+0000","scrub_duration":0.0017843189999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":673,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-09-23T20:49:52.624330+0000","last_change":"2023-09-23T20:47:09.207851+0000","last_active":"2023-09-23T20:49:52.624330+0000","last_peered":"2023-09-23T20:49:52.624330+0000","last_clean":"2023-09-23T20:49:52.624330+0000","last_became_active":"2023-09-23T20:47:09.175836+0000","last_became_peered":"2023-09-23T20:47:09.175836+0000","last_unstale":"2023-09-23T20:49:52.624330+0000","last_undegraded":"2023-09-23T20:49:52.624330+0000","last_fullsized":"2023-09-23T20:49:52.624330+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:43:13.091809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":466,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-09-23T20:49:52.621989+0000","last_change":"2023-09-23T20:49:48.122188+0000","last_active":"2023-09-23T20:49:52.621989+0000","last_peered":"2023-09-23T20:49:52.621989+0000","last_clean":"2023-09-23T20:49:52.621989+0000","last_became_active":"2023-09-23T20:49:48.121894+0000","last_became_peered":"2023-09-23T20:49:48.121894+0000","last_unstale":"2023-09-23T20:49:52.621989+0000","last_undegraded":"2023-09-23T20:49:52.621989+0000","last_fullsized":"2023-09-23T20:49:52.621989+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:12:33.758985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.c","version":"90'161","reported_seq":904,"reported_epoch":640,"state":"active+clean","last_fresh":"2023-09-23T20:49:51.114662+0000","last_change":"2023-09-23T20:41:09.506481+0000","last_active":"2023-09-23T20:49:51.114662+0000","last_peered":"2023-09-23T20:49:51.114662+0000","last_clean":"2023-09-23T20:49:51.114662+0000","last_became_active":"2023-09-23T20:38:28.442229+0000","last_became_peered":"2023-09-23T20:38:28.442229+0000","last_unstale":"2023-09-23T20:49:51.114662+0000","last_undegraded":"2023-09-23T20:49:51.114662+0000","last_fullsized":"2023-09-23T20:49:51.114662+0000","mapping_epoch":411,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":412,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:41:09.506338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:41:09.506338+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:19:46.265011+0000","scrub_duration":0.01112002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":711,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-09-23T20:49:52.624156+0000","last_change":"2023-09-23T20:47:10.848436+0000","last_active":"2023-09-23T20:49:52.624156+0000","last_peered":"2023-09-23T20:49:52.624156+0000","last_clean":"2023-09-23T20:49:52.624156+0000","last_became_active":"2023-09-23T20:47:09.196769+0000","last_became_peered":"2023-09-23T20:47:09.196769+0000","last_unstale":"2023-09-23T20:49:52.624156+0000","last_undegraded":"2023-09-23T20:49:52.624156+0000","last_fullsized":"2023-09-23T20:49:52.624156+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":554,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:59:38.105009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":108,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-09-23T20:49:52.308898+0000","last_change":"2023-09-23T20:48:23.915817+0000","last_active":"2023-09-23T20:49:52.308898+0000","last_peered":"2023-09-23T20:49:52.308898+0000","last_clean":"2023-09-23T20:49:52.308898+0000","last_became_active":"2023-09-23T20:48:23.915306+0000","last_became_peered":"2023-09-23T20:48:23.915306+0000","last_unstale":"2023-09-23T20:49:52.308898+0000","last_undegraded":"2023-09-23T20:49:52.308898+0000","last_fullsized":"2023-09-23T20:49:52.308898+0000","mapping_epoch":577,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":578,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:55:45.691450+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"90'161","reported_seq":856,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-09-23T20:49:52.630440+0000","last_change":"2023-09-23T20:49:51.120296+0000","last_active":"2023-09-23T20:49:52.630440+0000","last_peered":"2023-09-23T20:49:52.630440+0000","last_clean":"2023-09-23T20:49:52.630440+0000","last_became_active":"2023-09-23T20:49:51.120016+0000","last_became_peered":"2023-09-23T20:49:51.120016+0000","last_unstale":"2023-09-23T20:49:52.630440+0000","last_undegraded":"2023-09-23T20:49:52.630440+0000","last_fullsized":"2023-09-23T20:49:52.630440+0000","mapping_epoch":639,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":640,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:46.217489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:46.217489+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:35:59.437404+0000","scrub_duration":0.001456429,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":684,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-09-23T20:49:52.623563+0000","last_change":"2023-09-23T20:49:48.131779+0000","last_active":"2023-09-23T20:49:52.623563+0000","last_peered":"2023-09-23T20:49:52.623563+0000","last_clean":"2023-09-23T20:49:52.623563+0000","last_became_active":"2023-09-23T20:49:48.123934+0000","last_became_peered":"2023-09-23T20:49:48.123934+0000","last_unstale":"2023-09-23T20:49:52.623563+0000","last_undegraded":"2023-09-23T20:49:52.623563+0000","last_fullsized":"2023-09-23T20:49:52.623563+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:24:40.508247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":490,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":256,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-09-23T20:49:52.630887+0000","last_change":"2023-09-23T20:49:02.702301+0000","last_active":"2023-09-23T20:49:52.630887+0000","last_peered":"2023-09-23T20:49:52.630887+0000","last_clean":"2023-09-23T20:49:52.630887+0000","last_became_active":"2023-09-23T20:49:02.701986+0000","last_became_peered":"2023-09-23T20:49:02.701986+0000","last_unstale":"2023-09-23T20:49:52.630887+0000","last_undegraded":"2023-09-23T20:49:52.630887+0000","last_fullsized":"2023-09-23T20:49:52.630887+0000","mapping_epoch":603,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":604,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:20:01.603267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.e","version":"90'161","reported_seq":671,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-09-23T20:49:52.309066+0000","last_change":"2023-09-23T20:47:37.608426+0000","last_active":"2023-09-23T20:49:52.309066+0000","last_peered":"2023-09-23T20:49:52.309066+0000","last_clean":"2023-09-23T20:49:52.309066+0000","last_became_active":"2023-09-23T20:47:11.617187+0000","last_became_peered":"2023-09-23T20:47:11.617187+0000","last_unstale":"2023-09-23T20:49:52.309066+0000","last_undegraded":"2023-09-23T20:49:52.309066+0000","last_fullsized":"2023-09-23T20:49:52.309066+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:47:37.608309+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:47:37.608309+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:22:25.482297+0000","scrub_duration":0.011600924,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":611,"reported_epoch":640,"state":"active+clean","last_fresh":"2023-09-23T20:49:51.115275+0000","last_change":"2023-09-23T20:49:48.120934+0000","last_active":"2023-09-23T20:49:51.115275+0000","last_peered":"2023-09-23T20:49:51.115275+0000","last_clean":"2023-09-23T20:49:51.115275+0000","last_became_active":"2023-09-23T20:49:48.120578+0000","last_became_peered":"2023-09-23T20:49:48.120578+0000","last_unstale":"2023-09-23T20:49:51.115275+0000","last_undegraded":"2023-09-23T20:49:51.115275+0000","last_fullsized":"2023-09-23T20:49:51.115275+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:57:50.669399+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":635,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-09-23T20:49:52.571113+0000","last_change":"2023-09-23T20:47:09.197427+0000","last_active":"2023-09-23T20:49:52.571113+0000","last_peered":"2023-09-23T20:49:52.571113+0000","last_clean":"2023-09-23T20:49:52.571113+0000","last_became_active":"2023-09-23T20:47:09.196542+0000","last_became_peered":"2023-09-23T20:47:09.196542+0000","last_unstale":"2023-09-23T20:49:52.571113+0000","last_undegraded":"2023-09-23T20:49:52.571113+0000","last_fullsized":"2023-09-23T20:49:52.571113+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:28:38.926202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":1087,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-09-23T20:49:52.621707+0000","last_change":"2023-09-23T20:48:51.169474+0000","last_active":"2023-09-23T20:49:52.621707+0000","last_peered":"2023-09-23T20:49:52.621707+0000","last_clean":"2023-09-23T20:49:52.621707+0000","last_became_active":"2023-09-23T20:48:51.169076+0000","last_became_peered":"2023-09-23T20:48:51.169076+0000","last_unstale":"2023-09-23T20:49:52.621707+0000","last_undegraded":"2023-09-23T20:49:52.621707+0000","last_fullsized":"2023-09-23T20:49:52.621707+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:35:02.523203+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:35:02.523203+0000","last_clean_scrub_stamp":"2023-09-23T20:35:02.523203+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:25:53.741403+0000","scrub_duration":0.0085927989999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":490,"reported_epoch":640,"state":"active+clean","last_fresh":"2023-09-23T20:49:51.113641+0000","last_change":"2023-09-23T20:47:09.240200+0000","last_active":"2023-09-23T20:49:51.113641+0000","last_peered":"2023-09-23T20:49:51.113641+0000","last_clean":"2023-09-23T20:49:51.113641+0000","last_became_active":"2023-09-23T20:47:09.195204+0000","last_became_peered":"2023-09-23T20:47:09.195204+0000","last_unstale":"2023-09-23T20:49:51.113641+0000","last_undegraded":"2023-09-23T20:49:51.113641+0000","last_fullsized":"2023-09-23T20:49:51.113641+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:03.328250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":98,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-09-23T20:49:52.308723+0000","last_change":"2023-09-23T20:47:09.231714+0000","last_active":"2023-09-23T20:49:52.308723+0000","last_peered":"2023-09-23T20:49:52.308723+0000","last_clean":"2023-09-23T20:49:52.308723+0000","last_became_active":"2023-09-23T20:47:09.230918+0000","last_became_peered":"2023-09-23T20:47:09.230918+0000","last_unstale":"2023-09-23T20:49:52.308723+0000","last_undegraded":"2023-09-23T20:49:52.308723+0000","last_fullsized":"2023-09-23T20:49:52.308723+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:24.583656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":533,"reported_epoch":640,"state":"active+clean","last_fresh":"2023-09-23T20:49:51.130246+0000","last_change":"2023-09-23T20:49:51.125281+0000","last_active":"2023-09-23T20:49:51.130246+0000","last_peered":"2023-09-23T20:49:51.130246+0000","last_clean":"2023-09-23T20:49:51.130246+0000","last_became_active":"2023-09-23T20:49:51.124481+0000","last_became_peered":"2023-09-23T20:49:51.124481+0000","last_unstale":"2023-09-23T20:49:51.130246+0000","last_undegraded":"2023-09-23T20:49:51.130246+0000","last_fullsized":"2023-09-23T20:49:51.130246+0000","mapping_epoch":639,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":640,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:42:45.492220+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:42:45.492220+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":266,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-09-23T20:49:52.630971+0000","last_change":"2023-09-23T20:47:09.178919+0000","last_active":"2023-09-23T20:49:52.630971+0000","last_peered":"2023-09-23T20:49:52.630971+0000","last_clean":"2023-09-23T20:49:52.630971+0000","last_became_active":"2023-09-23T20:47:09.178607+0000","last_became_peered":"2023-09-23T20:47:09.178607+0000","last_unstale":"2023-09-23T20:49:52.630971+0000","last_undegraded":"2023-09-23T20:49:52.630971+0000","last_fullsized":"2023-09-23T20:49:52.630971+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:23:45.196935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":118,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-09-23T20:49:52.307860+0000","last_change":"2023-09-23T20:48:48.148596+0000","last_active":"2023-09-23T20:49:52.307860+0000","last_peered":"2023-09-23T20:49:52.307860+0000","last_clean":"2023-09-23T20:49:52.307860+0000","last_became_active":"2023-09-23T20:48:48.148211+0000","last_became_peered":"2023-09-23T20:48:48.148211+0000","last_unstale":"2023-09-23T20:49:52.307860+0000","last_undegraded":"2023-09-23T20:49:52.307860+0000","last_fullsized":"2023-09-23T20:49:52.307860+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:21:50.825822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"90'161","reported_seq":964,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-09-23T20:49:52.462286+0000","last_change":"2023-09-23T20:49:51.124836+0000","last_active":"2023-09-23T20:49:52.462286+0000","last_peered":"2023-09-23T20:49:52.462286+0000","last_clean":"2023-09-23T20:49:52.462286+0000","last_became_active":"2023-09-23T20:49:51.124419+0000","last_became_peered":"2023-09-23T20:49:51.124419+0000","last_unstale":"2023-09-23T20:49:52.462286+0000","last_undegraded":"2023-09-23T20:49:52.462286+0000","last_fullsized":"2023-09-23T20:49:52.462286+0000","mapping_epoch":639,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":640,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:05.595042+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:05.595042+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:37:05.219030+0000","scrub_duration":0.007271675,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":118,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-09-23T20:49:52.307834+0000","last_change":"2023-09-23T20:48:48.153145+0000","last_active":"2023-09-23T20:49:52.307834+0000","last_peered":"2023-09-23T20:49:52.307834+0000","last_clean":"2023-09-23T20:49:52.307834+0000","last_became_active":"2023-09-23T20:48:48.152858+0000","last_became_peered":"2023-09-23T20:48:48.152858+0000","last_unstale":"2023-09-23T20:49:52.307834+0000","last_undegraded":"2023-09-23T20:49:52.307834+0000","last_fullsized":"2023-09-23T20:49:52.307834+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:23:13.454288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":98,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-09-23T20:49:52.309138+0000","last_change":"2023-09-23T20:47:09.186271+0000","last_active":"2023-09-23T20:49:52.309138+0000","last_peered":"2023-09-23T20:49:52.309138+0000","last_clean":"2023-09-23T20:49:52.309138+0000","last_became_active":"2023-09-23T20:47:09.184357+0000","last_became_peered":"2023-09-23T20:47:09.184357+0000","last_unstale":"2023-09-23T20:49:52.309138+0000","last_undegraded":"2023-09-23T20:49:52.309138+0000","last_fullsized":"2023-09-23T20:49:52.309138+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:48:26.339811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":549,"reported_epoch":640,"state":"active+clean","last_fresh":"2023-09-23T20:49:51.128733+0000","last_change":"2023-09-23T20:49:51.128432+0000","last_active":"2023-09-23T20:49:51.128733+0000","last_peered":"2023-09-23T20:49:51.128733+0000","last_clean":"2023-09-23T20:49:51.128733+0000","last_became_active":"2023-09-23T20:49:51.128074+0000","last_became_peered":"2023-09-23T20:49:51.128074+0000","last_unstale":"2023-09-23T20:49:51.128733+0000","last_undegraded":"2023-09-23T20:49:51.128733+0000","last_fullsized":"2023-09-23T20:49:51.128733+0000","mapping_epoch":639,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":640,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:49:37.762492+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:49:37.762492+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:31:57.447267+0000","scrub_duration":0.0031485979999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,4],"acting":[2,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":438,"reported_epoch":640,"state":"active+clean","last_fresh":"2023-09-23T20:49:51.114188+0000","last_change":"2023-09-23T20:37:43.570610+0000","last_active":"2023-09-23T20:49:51.114188+0000","last_peered":"2023-09-23T20:49:51.114188+0000","last_clean":"2023-09-23T20:49:51.114188+0000","last_became_active":"2023-09-23T20:37:43.570216+0000","last_became_peered":"2023-09-23T20:37:43.570216+0000","last_unstale":"2023-09-23T20:49:51.114188+0000","last_undegraded":"2023-09-23T20:49:51.114188+0000","last_fullsized":"2023-09-23T20:49:51.114188+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:39:26.555329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":487,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-09-23T20:49:52.461057+0000","last_change":"2023-09-23T20:47:09.172573+0000","last_active":"2023-09-23T20:49:52.461057+0000","last_peered":"2023-09-23T20:49:52.461057+0000","last_clean":"2023-09-23T20:49:52.461057+0000","last_became_active":"2023-09-23T20:47:09.172298+0000","last_became_peered":"2023-09-23T20:47:09.172298+0000","last_unstale":"2023-09-23T20:49:52.461057+0000","last_undegraded":"2023-09-23T20:49:52.461057+0000","last_fullsized":"2023-09-23T20:49:52.461057+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:22:09.361284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":829,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-09-23T20:49:52.630367+0000","last_change":"2023-09-23T20:41:07.424609+0000","last_active":"2023-09-23T20:49:52.630367+0000","last_peered":"2023-09-23T20:49:52.630367+0000","last_clean":"2023-09-23T20:49:52.630367+0000","last_became_active":"2023-09-23T20:38:05.622261+0000","last_became_peered":"2023-09-23T20:38:05.622261+0000","last_unstale":"2023-09-23T20:49:52.630367+0000","last_undegraded":"2023-09-23T20:49:52.630367+0000","last_fullsized":"2023-09-23T20:49:52.630367+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:41:07.424492+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:41:07.424492+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:45:21.689123+0000","scrub_duration":0.0018960959999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"86'119","reported_seq":954,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-09-23T20:49:52.462086+0000","last_change":"2023-09-23T20:29:45.521257+0000","last_active":"2023-09-23T20:49:52.462086+0000","last_peered":"2023-09-23T20:49:52.462086+0000","last_clean":"2023-09-23T20:49:52.462086+0000","last_became_active":"2023-09-23T20:29:45.520546+0000","last_became_peered":"2023-09-23T20:29:45.520546+0000","last_unstale":"2023-09-23T20:49:52.462086+0000","last_undegraded":"2023-09-23T20:49:52.462086+0000","last_fullsized":"2023-09-23T20:49:52.462086+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":119,"log_dups_size":0,"ondisk_log_size":119,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:58:56.085777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032705499999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":523,"num_read_kb":3503,"num_write":136,"num_write_kb":3128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":463,"reported_epoch":640,"state":"active+clean","last_fresh":"2023-09-23T20:49:51.116646+0000","last_change":"2023-09-23T20:49:48.114924+0000","last_active":"2023-09-23T20:49:51.116646+0000","last_peered":"2023-09-23T20:49:51.116646+0000","last_clean":"2023-09-23T20:49:51.116646+0000","last_became_active":"2023-09-23T20:49:48.114554+0000","last_became_peered":"2023-09-23T20:49:48.114554+0000","last_unstale":"2023-09-23T20:49:51.116646+0000","last_undegraded":"2023-09-23T20:49:51.116646+0000","last_fullsized":"2023-09-23T20:49:51.116646+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:09:09.538462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":628,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-09-23T20:49:52.460716+0000","last_change":"2023-09-23T20:49:48.115598+0000","last_active":"2023-09-23T20:49:52.460716+0000","last_peered":"2023-09-23T20:49:52.460716+0000","last_clean":"2023-09-23T20:49:52.460716+0000","last_became_active":"2023-09-23T20:49:48.115313+0000","last_became_peered":"2023-09-23T20:49:48.115313+0000","last_unstale":"2023-09-23T20:49:52.460716+0000","last_undegraded":"2023-09-23T20:49:52.460716+0000","last_fullsized":"2023-09-23T20:49:52.460716+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:58:47.477333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":740,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-09-23T20:49:52.571757+0000","last_change":"2023-09-23T20:49:51.148445+0000","last_active":"2023-09-23T20:49:52.571757+0000","last_peered":"2023-09-23T20:49:52.571757+0000","last_clean":"2023-09-23T20:49:52.571757+0000","last_became_active":"2023-09-23T20:49:51.147884+0000","last_became_peered":"2023-09-23T20:49:51.147884+0000","last_unstale":"2023-09-23T20:49:52.571757+0000","last_undegraded":"2023-09-23T20:49:52.571757+0000","last_fullsized":"2023-09-23T20:49:52.571757+0000","mapping_epoch":639,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":640,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:15.656880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:15.656880+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:05:08.116069+0000","scrub_duration":0.0016711180000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"86'106","reported_seq":934,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-09-23T20:49:52.461736+0000","last_change":"2023-09-23T20:49:51.126017+0000","last_active":"2023-09-23T20:49:52.461736+0000","last_peered":"2023-09-23T20:49:52.461736+0000","last_clean":"2023-09-23T20:49:52.461736+0000","last_became_active":"2023-09-23T20:49:51.125316+0000","last_became_peered":"2023-09-23T20:49:51.125316+0000","last_unstale":"2023-09-23T20:49:52.461736+0000","last_undegraded":"2023-09-23T20:49:52.461736+0000","last_fullsized":"2023-09-23T20:49:52.461736+0000","mapping_epoch":639,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":640,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":106,"log_dups_size":0,"ondisk_log_size":106,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:32:47.065876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00099302599999999993,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1094,"num_read_kb":1678,"num_write":100,"num_write_kb":404,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":98,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-09-23T20:49:52.309683+0000","last_change":"2023-09-23T20:47:09.177872+0000","last_active":"2023-09-23T20:49:52.309683+0000","last_peered":"2023-09-23T20:49:52.309683+0000","last_clean":"2023-09-23T20:49:52.309683+0000","last_became_active":"2023-09-23T20:47:09.176689+0000","last_became_peered":"2023-09-23T20:47:09.176689+0000","last_unstale":"2023-09-23T20:49:52.309683+0000","last_undegraded":"2023-09-23T20:49:52.309683+0000","last_fullsized":"2023-09-23T20:49:52.309683+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:14:01.590318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":625,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-09-23T20:49:52.460409+0000","last_change":"2023-09-23T20:48:48.151144+0000","last_active":"2023-09-23T20:49:52.460409+0000","last_peered":"2023-09-23T20:49:52.460409+0000","last_clean":"2023-09-23T20:49:52.460409+0000","last_became_active":"2023-09-23T20:48:48.150770+0000","last_became_peered":"2023-09-23T20:48:48.150770+0000","last_unstale":"2023-09-23T20:49:52.460409+0000","last_undegraded":"2023-09-23T20:49:52.460409+0000","last_fullsized":"2023-09-23T20:49:52.460409+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:14:08.549512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":645,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-09-23T20:49:52.622790+0000","last_change":"2023-09-23T20:47:11.618652+0000","last_active":"2023-09-23T20:49:52.622790+0000","last_peered":"2023-09-23T20:49:52.622790+0000","last_clean":"2023-09-23T20:49:52.622790+0000","last_became_active":"2023-09-23T20:47:11.618266+0000","last_became_peered":"2023-09-23T20:47:11.618266+0000","last_unstale":"2023-09-23T20:49:52.622790+0000","last_undegraded":"2023-09-23T20:49:52.622790+0000","last_fullsized":"2023-09-23T20:49:52.622790+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:26.978179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:26.978179+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:29:29.000683+0000","scrub_duration":0.0013962829999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"86'130","reported_seq":426,"reported_epoch":640,"state":"active+clean","last_fresh":"2023-09-23T20:49:51.113305+0000","last_change":"2023-09-23T20:48:51.171448+0000","last_active":"2023-09-23T20:49:51.113305+0000","last_peered":"2023-09-23T20:49:51.113305+0000","last_clean":"2023-09-23T20:49:51.113305+0000","last_became_active":"2023-09-23T20:48:51.170485+0000","last_became_peered":"2023-09-23T20:48:51.170485+0000","last_unstale":"2023-09-23T20:49:51.113305+0000","last_undegraded":"2023-09-23T20:49:51.113305+0000","last_fullsized":"2023-09-23T20:49:51.113305+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:48:35.309684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011300049999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":512,"num_read_kb":1820,"num_write":133,"num_write_kb":1347,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":634,"reported_epoch":642,"state":"active+clean","last_fresh":"2023-09-23T20:49:53.119357+0000","last_change":"2023-09-23T20:48:48.151034+0000","last_active":"2023-09-23T20:49:53.119357+0000","last_peered":"2023-09-23T20:49:53.119357+0000","last_clean":"2023-09-23T20:49:53.119357+0000","last_became_active":"2023-09-23T20:48:48.150658+0000","last_became_peered":"2023-09-23T20:48:48.150658+0000","last_unstale":"2023-09-23T20:49:53.119357+0000","last_undegraded":"2023-09-23T20:49:53.119357+0000","last_fullsized":"2023-09-23T20:49:53.119357+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:44:34.819851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":167,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-09-23T20:49:52.460982+0000","last_change":"2023-09-23T20:49:48.118182+0000","last_active":"2023-09-23T20:49:52.460982+0000","last_peered":"2023-09-23T20:49:52.460982+0000","last_clean":"2023-09-23T20:49:52.460982+0000","last_became_active":"2023-09-23T20:49:48.117804+0000","last_became_peered":"2023-09-23T20:49:48.117804+0000","last_unstale":"2023-09-23T20:49:52.460982+0000","last_undegraded":"2023-09-23T20:49:52.460982+0000","last_fullsized":"2023-09-23T20:49:52.460982+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:47:43.384018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"86'194","reported_seq":793,"reported_epoch":640,"state":"active+clean","last_fresh":"2023-09-23T20:49:51.154714+0000","last_change":"2023-09-23T20:49:51.154300+0000","last_active":"2023-09-23T20:49:51.154714+0000","last_peered":"2023-09-23T20:49:51.154714+0000","last_clean":"2023-09-23T20:49:51.154714+0000","last_became_active":"2023-09-23T20:49:51.153529+0000","last_became_peered":"2023-09-23T20:49:51.153529+0000","last_unstale":"2023-09-23T20:49:51.154714+0000","last_undegraded":"2023-09-23T20:49:51.154714+0000","last_fullsized":"2023-09-23T20:49:51.154714+0000","mapping_epoch":639,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":640,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":194,"log_dups_size":0,"ondisk_log_size":194,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:10:22.072962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00037995999999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1024,"num_read_kb":1200,"num_write":173,"num_write_kb":1276,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":1104,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-09-23T20:49:52.621848+0000","last_change":"2023-09-23T20:38:05.602508+0000","last_active":"2023-09-23T20:49:52.621848+0000","last_peered":"2023-09-23T20:49:52.621848+0000","last_clean":"2023-09-23T20:49:52.621848+0000","last_became_active":"2023-09-23T20:38:05.602129+0000","last_became_peered":"2023-09-23T20:38:05.602129+0000","last_unstale":"2023-09-23T20:49:52.621848+0000","last_undegraded":"2023-09-23T20:49:52.621848+0000","last_fullsized":"2023-09-23T20:49:52.621848+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:45.343742+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:45.343742+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:48.910343+0000","scrub_duration":0.001475802,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":226,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-09-23T20:49:52.631047+0000","last_change":"2023-09-23T20:49:48.119237+0000","last_active":"2023-09-23T20:49:52.631047+0000","last_peered":"2023-09-23T20:49:52.631047+0000","last_clean":"2023-09-23T20:49:52.631047+0000","last_became_active":"2023-09-23T20:49:48.118927+0000","last_became_peered":"2023-09-23T20:49:48.118927+0000","last_unstale":"2023-09-23T20:49:52.631047+0000","last_undegraded":"2023-09-23T20:49:52.631047+0000","last_fullsized":"2023-09-23T20:49:52.631047+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:31:47.579623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,4],"acting":[6,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":266,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-09-23T20:49:52.627633+0000","last_change":"2023-09-23T20:47:09.202302+0000","last_active":"2023-09-23T20:49:52.627633+0000","last_peered":"2023-09-23T20:49:52.627633+0000","last_clean":"2023-09-23T20:49:52.627633+0000","last_became_active":"2023-09-23T20:47:09.201797+0000","last_became_peered":"2023-09-23T20:47:09.201797+0000","last_unstale":"2023-09-23T20:49:52.627633+0000","last_undegraded":"2023-09-23T20:49:52.627633+0000","last_fullsized":"2023-09-23T20:49:52.627633+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:34:41.055705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":1010,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-09-23T20:49:52.461785+0000","last_change":"2023-09-23T20:47:11.622262+0000","last_active":"2023-09-23T20:49:52.461785+0000","last_peered":"2023-09-23T20:49:52.461785+0000","last_clean":"2023-09-23T20:49:52.461785+0000","last_became_active":"2023-09-23T20:47:11.621920+0000","last_became_peered":"2023-09-23T20:47:11.621920+0000","last_unstale":"2023-09-23T20:49:52.461785+0000","last_undegraded":"2023-09-23T20:49:52.461785+0000","last_fullsized":"2023-09-23T20:49:52.461785+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:43.582146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:43.582146+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:01:15.153543+0000","scrub_duration":0.0016219730000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"86'80","reported_seq":894,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-09-23T20:49:52.621764+0000","last_change":"2023-09-23T20:29:45.536796+0000","last_active":"2023-09-23T20:49:52.621764+0000","last_peered":"2023-09-23T20:49:52.621764+0000","last_clean":"2023-09-23T20:49:52.621764+0000","last_became_active":"2023-09-23T20:29:45.536439+0000","last_became_peered":"2023-09-23T20:29:45.536439+0000","last_unstale":"2023-09-23T20:49:52.621764+0000","last_undegraded":"2023-09-23T20:49:52.621764+0000","last_fullsized":"2023-09-23T20:49:52.621764+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":80,"log_dups_size":0,"ondisk_log_size":80,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:30:57.793132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035115,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":395,"num_read_kb":1370,"num_write":66,"num_write_kb":1061,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":654,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-09-23T20:49:52.570646+0000","last_change":"2023-09-23T20:47:09.198326+0000","last_active":"2023-09-23T20:49:52.570646+0000","last_peered":"2023-09-23T20:49:52.570646+0000","last_clean":"2023-09-23T20:49:52.570646+0000","last_became_active":"2023-09-23T20:47:09.198059+0000","last_became_peered":"2023-09-23T20:47:09.198059+0000","last_unstale":"2023-09-23T20:49:52.570646+0000","last_undegraded":"2023-09-23T20:49:52.570646+0000","last_fullsized":"2023-09-23T20:49:52.570646+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:38:53.903706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":649,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-09-23T20:49:52.570185+0000","last_change":"2023-09-23T20:48:48.146977+0000","last_active":"2023-09-23T20:49:52.570185+0000","last_peered":"2023-09-23T20:49:52.570185+0000","last_clean":"2023-09-23T20:49:52.570185+0000","last_became_active":"2023-09-23T20:48:48.146657+0000","last_became_peered":"2023-09-23T20:48:48.146657+0000","last_unstale":"2023-09-23T20:49:52.570185+0000","last_undegraded":"2023-09-23T20:49:52.570185+0000","last_fullsized":"2023-09-23T20:49:52.570185+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:38:41.782692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":752,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-09-23T20:49:52.629210+0000","last_change":"2023-09-23T20:41:08.380084+0000","last_active":"2023-09-23T20:49:52.629210+0000","last_peered":"2023-09-23T20:49:52.629210+0000","last_clean":"2023-09-23T20:49:52.629210+0000","last_became_active":"2023-09-23T20:38:05.610881+0000","last_became_peered":"2023-09-23T20:38:05.610881+0000","last_unstale":"2023-09-23T20:49:52.629210+0000","last_undegraded":"2023-09-23T20:49:52.629210+0000","last_fullsized":"2023-09-23T20:49:52.629210+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:41:08.379959+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:41:08.379959+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:19:39.547923+0000","scrub_duration":0.0019158420000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"86'102","reported_seq":1312,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-09-23T20:49:52.623630+0000","last_change":"2023-09-23T20:47:14.628795+0000","last_active":"2023-09-23T20:49:52.623630+0000","last_peered":"2023-09-23T20:49:52.623630+0000","last_clean":"2023-09-23T20:49:52.623630+0000","last_became_active":"2023-09-23T20:47:14.628341+0000","last_became_peered":"2023-09-23T20:47:14.628341+0000","last_unstale":"2023-09-23T20:49:52.623630+0000","last_undegraded":"2023-09-23T20:49:52.623630+0000","last_fullsized":"2023-09-23T20:49:52.623630+0000","mapping_epoch":558,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":559,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":102,"log_dups_size":0,"ondisk_log_size":102,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:42:16.291849+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030656500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":963,"num_read_kb":1234,"num_write":60,"num_write_kb":225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":618,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-09-23T20:49:52.462124+0000","last_change":"2023-09-23T20:49:48.108294+0000","last_active":"2023-09-23T20:49:52.462124+0000","last_peered":"2023-09-23T20:49:52.462124+0000","last_clean":"2023-09-23T20:49:52.462124+0000","last_became_active":"2023-09-23T20:49:48.107916+0000","last_became_peered":"2023-09-23T20:49:48.107916+0000","last_unstale":"2023-09-23T20:49:52.462124+0000","last_undegraded":"2023-09-23T20:49:52.462124+0000","last_fullsized":"2023-09-23T20:49:52.462124+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:57:12.800814+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":595,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-09-23T20:49:52.571364+0000","last_change":"2023-09-23T20:49:48.119227+0000","last_active":"2023-09-23T20:49:52.571364+0000","last_peered":"2023-09-23T20:49:52.571364+0000","last_clean":"2023-09-23T20:49:52.571364+0000","last_became_active":"2023-09-23T20:49:48.118910+0000","last_became_peered":"2023-09-23T20:49:48.118910+0000","last_unstale":"2023-09-23T20:49:52.571364+0000","last_undegraded":"2023-09-23T20:49:52.571364+0000","last_fullsized":"2023-09-23T20:49:52.571364+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:18:40.836373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":678,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-09-23T20:49:52.309829+0000","last_change":"2023-09-23T20:48:51.172830+0000","last_active":"2023-09-23T20:49:52.309829+0000","last_peered":"2023-09-23T20:49:52.309829+0000","last_clean":"2023-09-23T20:49:52.309829+0000","last_became_active":"2023-09-23T20:48:51.172481+0000","last_became_peered":"2023-09-23T20:48:51.172481+0000","last_unstale":"2023-09-23T20:49:52.309829+0000","last_undegraded":"2023-09-23T20:49:52.309829+0000","last_fullsized":"2023-09-23T20:49:52.309829+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:47:34.561747+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:47:34.561747+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:30:32.531477+0000","scrub_duration":0.0087949929999999992,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":379,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-09-23T20:49:52.624460+0000","last_change":"2023-09-23T20:32:50.549695+0000","last_active":"2023-09-23T20:49:52.624460+0000","last_peered":"2023-09-23T20:49:52.624460+0000","last_clean":"2023-09-23T20:49:52.624460+0000","last_became_active":"2023-09-23T20:32:50.549431+0000","last_became_peered":"2023-09-23T20:32:50.549431+0000","last_unstale":"2023-09-23T20:49:52.624460+0000","last_undegraded":"2023-09-23T20:49:52.624460+0000","last_fullsized":"2023-09-23T20:49:52.624460+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:45:53.666569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"86'128","reported_seq":1123,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-09-23T20:49:52.463402+0000","last_change":"2023-09-23T20:47:11.621846+0000","last_active":"2023-09-23T20:49:52.463402+0000","last_peered":"2023-09-23T20:49:52.463402+0000","last_clean":"2023-09-23T20:49:52.463402+0000","last_became_active":"2023-09-23T20:47:11.621284+0000","last_became_peered":"2023-09-23T20:47:11.621284+0000","last_unstale":"2023-09-23T20:49:52.463402+0000","last_undegraded":"2023-09-23T20:49:52.463402+0000","last_fullsized":"2023-09-23T20:49:52.463402+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:43:50.658512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000293202,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":579,"num_read_kb":1473,"num_write":96,"num_write_kb":1062,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":424,"reported_epoch":640,"state":"active+clean","last_fresh":"2023-09-23T20:49:51.114100+0000","last_change":"2023-09-23T20:37:43.582105+0000","last_active":"2023-09-23T20:49:51.114100+0000","last_peered":"2023-09-23T20:49:51.114100+0000","last_clean":"2023-09-23T20:49:51.114100+0000","last_became_active":"2023-09-23T20:37:43.581860+0000","last_became_peered":"2023-09-23T20:37:43.581860+0000","last_unstale":"2023-09-23T20:49:51.114100+0000","last_undegraded":"2023-09-23T20:49:51.114100+0000","last_fullsized":"2023-09-23T20:49:51.114100+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:18:38.765486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"90'161","reported_seq":874,"reported_epoch":640,"state":"active+clean","last_fresh":"2023-09-23T20:49:51.122434+0000","last_change":"2023-09-23T20:49:51.122217+0000","last_active":"2023-09-23T20:49:51.122434+0000","last_peered":"2023-09-23T20:49:51.122434+0000","last_clean":"2023-09-23T20:49:51.122434+0000","last_became_active":"2023-09-23T20:49:51.121893+0000","last_became_peered":"2023-09-23T20:49:51.121893+0000","last_unstale":"2023-09-23T20:49:51.122434+0000","last_undegraded":"2023-09-23T20:49:51.122434+0000","last_fullsized":"2023-09-23T20:49:51.122434+0000","mapping_epoch":639,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":640,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:34:54.504554+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:34:54.504554+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:00:40.029809+0000","scrub_duration":0.011176864,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"86'117","reported_seq":1025,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-09-23T20:49:52.630290+0000","last_change":"2023-09-23T20:48:51.217483+0000","last_active":"2023-09-23T20:49:52.630290+0000","last_peered":"2023-09-23T20:49:52.630290+0000","last_clean":"2023-09-23T20:49:52.630290+0000","last_became_active":"2023-09-23T20:48:51.217013+0000","last_became_peered":"2023-09-23T20:48:51.217013+0000","last_unstale":"2023-09-23T20:49:52.630290+0000","last_undegraded":"2023-09-23T20:49:52.630290+0000","last_fullsized":"2023-09-23T20:49:52.630290+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:40:42.297947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046891400000000002,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":893,"num_read_kb":922,"num_write":82,"num_write_kb":4193,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":48,"num_bytes_recovered":25165824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":657,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-09-23T20:49:52.571328+0000","last_change":"2023-09-23T20:47:09.317116+0000","last_active":"2023-09-23T20:49:52.571328+0000","last_peered":"2023-09-23T20:49:52.571328+0000","last_clean":"2023-09-23T20:49:52.571328+0000","last_became_active":"2023-09-23T20:47:09.205930+0000","last_became_peered":"2023-09-23T20:47:09.205930+0000","last_unstale":"2023-09-23T20:49:52.571328+0000","last_undegraded":"2023-09-23T20:49:52.571328+0000","last_fullsized":"2023-09-23T20:49:52.571328+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:04:55.528625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":108,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-09-23T20:49:52.309534+0000","last_change":"2023-09-23T20:49:48.120086+0000","last_active":"2023-09-23T20:49:52.309534+0000","last_peered":"2023-09-23T20:49:52.309534+0000","last_clean":"2023-09-23T20:49:52.309534+0000","last_became_active":"2023-09-23T20:49:48.119761+0000","last_became_peered":"2023-09-23T20:49:48.119761+0000","last_unstale":"2023-09-23T20:49:52.309534+0000","last_undegraded":"2023-09-23T20:49:52.309534+0000","last_fullsized":"2023-09-23T20:49:52.309534+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:56:12.469830+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"90'161","reported_seq":900,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-09-23T20:49:52.570336+0000","last_change":"2023-09-23T20:43:46.544750+0000","last_active":"2023-09-23T20:49:52.570336+0000","last_peered":"2023-09-23T20:49:52.570336+0000","last_clean":"2023-09-23T20:49:52.570336+0000","last_became_active":"2023-09-23T20:43:46.544395+0000","last_became_peered":"2023-09-23T20:43:46.544395+0000","last_unstale":"2023-09-23T20:49:52.570336+0000","last_undegraded":"2023-09-23T20:49:52.570336+0000","last_fullsized":"2023-09-23T20:49:52.570336+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:06.487351+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:06.487351+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:56.866168+0000","scrub_duration":0.001964379,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":616,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-09-23T20:49:52.571277+0000","last_change":"2023-09-23T20:47:09.197601+0000","last_active":"2023-09-23T20:49:52.571277+0000","last_peered":"2023-09-23T20:49:52.571277+0000","last_clean":"2023-09-23T20:49:52.571277+0000","last_became_active":"2023-09-23T20:47:09.196967+0000","last_became_peered":"2023-09-23T20:47:09.196967+0000","last_unstale":"2023-09-23T20:49:52.571277+0000","last_undegraded":"2023-09-23T20:49:52.571277+0000","last_fullsized":"2023-09-23T20:49:52.571277+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:06:56.826785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":60,"reported_epoch":640,"state":"active+clean","last_fresh":"2023-09-23T20:49:51.115268+0000","last_change":"2023-09-23T20:48:48.158376+0000","last_active":"2023-09-23T20:49:51.115268+0000","last_peered":"2023-09-23T20:49:51.115268+0000","last_clean":"2023-09-23T20:49:51.115268+0000","last_became_active":"2023-09-23T20:48:48.158069+0000","last_became_peered":"2023-09-23T20:48:48.158069+0000","last_unstale":"2023-09-23T20:49:51.115268+0000","last_undegraded":"2023-09-23T20:49:51.115268+0000","last_fullsized":"2023-09-23T20:49:51.115268+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:34.658561+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"90'161","reported_seq":717,"reported_epoch":640,"state":"active+clean","last_fresh":"2023-09-23T20:49:51.114023+0000","last_change":"2023-09-23T20:41:06.499511+0000","last_active":"2023-09-23T20:49:51.114023+0000","last_peered":"2023-09-23T20:49:51.114023+0000","last_clean":"2023-09-23T20:49:51.114023+0000","last_became_active":"2023-09-23T20:38:05.601874+0000","last_became_peered":"2023-09-23T20:38:05.601874+0000","last_unstale":"2023-09-23T20:49:51.114023+0000","last_undegraded":"2023-09-23T20:49:51.114023+0000","last_fullsized":"2023-09-23T20:49:51.114023+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:41:06.499396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:41:06.499396+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:31:34.059077+0000","scrub_duration":0.0019670310000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":116,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-09-23T20:49:52.309541+0000","last_change":"2023-09-23T20:49:48.128500+0000","last_active":"2023-09-23T20:49:52.309541+0000","last_peered":"2023-09-23T20:49:52.309541+0000","last_clean":"2023-09-23T20:49:52.309541+0000","last_became_active":"2023-09-23T20:49:48.122142+0000","last_became_peered":"2023-09-23T20:49:48.122142+0000","last_unstale":"2023-09-23T20:49:52.309541+0000","last_undegraded":"2023-09-23T20:49:52.309541+0000","last_fullsized":"2023-09-23T20:49:52.309541+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:46:11.349709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":98,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-09-23T20:49:52.309417+0000","last_change":"2023-09-23T20:47:09.186579+0000","last_active":"2023-09-23T20:49:52.309417+0000","last_peered":"2023-09-23T20:49:52.309417+0000","last_clean":"2023-09-23T20:49:52.309417+0000","last_became_active":"2023-09-23T20:47:09.186227+0000","last_became_peered":"2023-09-23T20:47:09.186227+0000","last_unstale":"2023-09-23T20:49:52.309417+0000","last_undegraded":"2023-09-23T20:49:52.309417+0000","last_fullsized":"2023-09-23T20:49:52.309417+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:45:39.485308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":116,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-09-23T20:49:52.309759+0000","last_change":"2023-09-23T20:49:48.126698+0000","last_active":"2023-09-23T20:49:52.309759+0000","last_peered":"2023-09-23T20:49:52.309759+0000","last_clean":"2023-09-23T20:49:52.309759+0000","last_became_active":"2023-09-23T20:49:48.120196+0000","last_became_peered":"2023-09-23T20:49:48.120196+0000","last_unstale":"2023-09-23T20:49:52.309759+0000","last_undegraded":"2023-09-23T20:49:52.309759+0000","last_fullsized":"2023-09-23T20:49:52.309759+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:17:45.039963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":98,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-09-23T20:49:52.308460+0000","last_change":"2023-09-23T20:47:09.181240+0000","last_active":"2023-09-23T20:49:52.308460+0000","last_peered":"2023-09-23T20:49:52.308460+0000","last_clean":"2023-09-23T20:49:52.308460+0000","last_became_active":"2023-09-23T20:47:09.181022+0000","last_became_peered":"2023-09-23T20:47:09.181022+0000","last_unstale":"2023-09-23T20:49:52.308460+0000","last_undegraded":"2023-09-23T20:49:52.308460+0000","last_fullsized":"2023-09-23T20:49:52.308460+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:38:59.191781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"90'161","reported_seq":897,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-09-23T20:49:52.628401+0000","last_change":"2023-09-23T20:38:05.591992+0000","last_active":"2023-09-23T20:49:52.628401+0000","last_peered":"2023-09-23T20:49:52.628401+0000","last_clean":"2023-09-23T20:49:52.628401+0000","last_became_active":"2023-09-23T20:38:05.591701+0000","last_became_peered":"2023-09-23T20:38:05.591701+0000","last_unstale":"2023-09-23T20:49:52.628401+0000","last_undegraded":"2023-09-23T20:49:52.628401+0000","last_fullsized":"2023-09-23T20:49:52.628401+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:02.384133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:02.384133+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:41.722542+0000","scrub_duration":0.001769098,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":484,"reported_epoch":640,"state":"active+clean","last_fresh":"2023-09-23T20:49:51.114230+0000","last_change":"2023-09-23T20:48:48.158348+0000","last_active":"2023-09-23T20:49:51.114230+0000","last_peered":"2023-09-23T20:49:51.114230+0000","last_clean":"2023-09-23T20:49:51.114230+0000","last_became_active":"2023-09-23T20:48:48.157982+0000","last_became_peered":"2023-09-23T20:48:48.157982+0000","last_unstale":"2023-09-23T20:49:51.114230+0000","last_undegraded":"2023-09-23T20:49:51.114230+0000","last_fullsized":"2023-09-23T20:49:51.114230+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:03:48.078707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":450,"reported_epoch":640,"state":"active+clean","last_fresh":"2023-09-23T20:49:51.113087+0000","last_change":"2023-09-23T20:48:48.159781+0000","last_active":"2023-09-23T20:49:51.113087+0000","last_peered":"2023-09-23T20:49:51.113087+0000","last_clean":"2023-09-23T20:49:51.113087+0000","last_became_active":"2023-09-23T20:48:48.159505+0000","last_became_peered":"2023-09-23T20:48:48.159505+0000","last_unstale":"2023-09-23T20:49:51.113087+0000","last_undegraded":"2023-09-23T20:49:51.113087+0000","last_fullsized":"2023-09-23T20:49:51.113087+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:25:18.100645+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"90'161","reported_seq":771,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-09-23T20:49:52.461294+0000","last_change":"2023-09-23T20:47:11.621758+0000","last_active":"2023-09-23T20:49:52.461294+0000","last_peered":"2023-09-23T20:49:52.461294+0000","last_clean":"2023-09-23T20:49:52.461294+0000","last_became_active":"2023-09-23T20:47:11.621345+0000","last_became_peered":"2023-09-23T20:47:11.621345+0000","last_unstale":"2023-09-23T20:49:52.461294+0000","last_undegraded":"2023-09-23T20:49:52.461294+0000","last_fullsized":"2023-09-23T20:49:52.461294+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:01.504973+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:01.504973+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:31:17.112393+0000","scrub_duration":0.00202344,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":721,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-09-23T20:49:52.571968+0000","last_change":"2023-09-23T20:49:48.125242+0000","last_active":"2023-09-23T20:49:52.571968+0000","last_peered":"2023-09-23T20:49:52.571968+0000","last_clean":"2023-09-23T20:49:52.571968+0000","last_became_active":"2023-09-23T20:49:48.111689+0000","last_became_peered":"2023-09-23T20:49:48.111689+0000","last_unstale":"2023-09-23T20:49:52.571968+0000","last_undegraded":"2023-09-23T20:49:52.571968+0000","last_fullsized":"2023-09-23T20:49:52.571968+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:49:38.401151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":4986,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":108,"reported_epoch":641,"state":"active+clean","last_fresh":"2023-09-23T20:49:52.308000+0000","last_change":"2023-09-23T20:48:23.917302+0000","last_active":"2023-09-23T20:49:52.308000+0000","last_peered":"2023-09-23T20:49:52.308000+0000","last_clean":"2023-09-23T20:49:52.308000+0000","last_became_active":"2023-09-23T20:48:23.916956+0000","last_became_peered":"2023-09-23T20:48:23.916956+0000","last_unstale":"2023-09-23T20:49:52.308000+0000","last_undegraded":"2023-09-23T20:49:52.308000+0000","last_fullsized":"2023-09-23T20:49:52.308000+0000","mapping_epoch":577,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":578,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:42:27.989294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":118,"num_bytes_recovered":15844,"num_keys_recovered":80,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26691,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":617203,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":605880,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2048000,"data_stored":2033792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5983,"num_read_kb":13200,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":156,"num_bytes_recovered":25165843,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12599296,"data_stored":12582988,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6082,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201742,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":613696,"kb_used_data":11000,"kb_used_omap":0,"kb_used_meta":602688,"kb_avail":93131456,"statfs":{"total":95995035648,"available":95366610944,"internally_reserved":0,"allocated":11264000,"data_stored":9488915,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":617152512},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:49:07 2023","interfaces":[{"interface":"back","average":{"1min":1.5669999999999999,"5min":1.774,"15min":2.0369999999999999},"min":{"1min":0.23400000000000001,"5min":0.224,"15min":0.224},"max":{"1min":5.1360000000000001,"5min":10.975,"15min":18.315000000000001},"last":0.57799999999999996},{"interface":"front","average":{"1min":1.4259999999999999,"5min":1.8819999999999999,"15min":1.865},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":8.0530000000000008,"5min":12.986000000000001,"15min":19.355},"last":0.63}]},{"osd":1,"last update":"Sat Sep 23 20:49:16 2023","interfaces":[{"interface":"back","average":{"1min":1.4259999999999999,"5min":1.4199999999999999,"15min":1.9550000000000001},"min":{"1min":0.28499999999999998,"5min":0.25900000000000001,"15min":0.24399999999999999},"max":{"1min":7.3789999999999996,"5min":14.249000000000001,"15min":49.965000000000003},"last":0.64100000000000001},{"interface":"front","average":{"1min":1.105,"5min":2.008,"15min":2.4319999999999999},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.26700000000000002},"max":{"1min":4.0739999999999998,"5min":11.276,"15min":53.451999999999998},"last":0.58899999999999997}]},{"osd":2,"last update":"Sat Sep 23 20:49:07 2023","interfaces":[{"interface":"back","average":{"1min":1.847,"5min":1.885,"15min":2.121},"min":{"1min":0.31,"5min":0.28499999999999998,"15min":0.27800000000000002},"max":{"1min":7.9219999999999997,"5min":12.933999999999999,"15min":16.965},"last":0.60899999999999999},{"interface":"front","average":{"1min":2.0209999999999999,"5min":1.982,"15min":2.2160000000000002},"min":{"1min":0.36599999999999999,"5min":0.31,"15min":0.30299999999999999},"max":{"1min":7.3019999999999996,"5min":13.5,"15min":23.414000000000001},"last":0.47699999999999998}]},{"osd":3,"last update":"Sat Sep 23 20:49:38 2023","interfaces":[{"interface":"back","average":{"1min":0.96099999999999997,"5min":1.619,"15min":2.1280000000000001},"min":{"1min":0.36299999999999999,"5min":0.33800000000000002,"15min":0.311},"max":{"1min":3.4380000000000002,"5min":7.2880000000000003,"15min":23.352},"last":0.54700000000000004},{"interface":"front","average":{"1min":1.2929999999999999,"5min":1.8380000000000001,"15min":2.3279999999999998},"min":{"1min":0.40799999999999997,"5min":0.35399999999999998,"15min":0.34300000000000003},"max":{"1min":4.1340000000000003,"5min":6.5369999999999999,"15min":20.969999999999999},"last":0.66000000000000003}]},{"osd":4,"last update":"Sat Sep 23 20:49:38 2023","interfaces":[{"interface":"back","average":{"1min":1.0249999999999999,"5min":1.2609999999999999,"15min":1.2589999999999999},"min":{"1min":0.25800000000000001,"5min":0.254,"15min":0.254},"max":{"1min":7.1239999999999997,"5min":7.4109999999999996,"15min":18.364999999999998},"last":0.62},{"interface":"front","average":{"1min":2.3439999999999999,"5min":1.7050000000000001,"15min":1.5509999999999999},"min":{"1min":0.27500000000000002,"5min":0.247,"15min":0.247},"max":{"1min":27.425999999999998,"5min":27.425999999999998,"15min":27.425999999999998},"last":0.68700000000000006}]},{"osd":5,"last update":"Sat Sep 23 20:49:38 2023","interfaces":[{"interface":"back","average":{"1min":0.65200000000000002,"5min":1.105,"15min":1.262},"min":{"1min":0.30599999999999999,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":2.754,"5min":7.2270000000000003,"15min":15.005000000000001},"last":0.51300000000000001},{"interface":"front","average":{"1min":0.76600000000000001,"5min":1.359,"15min":1.409},"min":{"1min":0.33600000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":3.3900000000000001,"5min":10.641999999999999,"15min":18.390000000000001},"last":0.67300000000000004}]},{"osd":6,"last update":"Sat Sep 23 20:49:38 2023","interfaces":[{"interface":"back","average":{"1min":1.0009999999999999,"5min":1.524,"15min":1.571},"min":{"1min":0.38700000000000001,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":3.4529999999999998,"5min":12.183,"15min":18.417000000000002},"last":0.69699999999999995},{"interface":"front","average":{"1min":1.0980000000000001,"5min":1.5449999999999999,"15min":1.448},"min":{"1min":0.371,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":3.2519999999999998,"5min":11.404999999999999,"15min":15.369999999999999},"last":0.56499999999999995}]}]},{"osd":6,"up_from":47,"seq":201863463381,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":532728,"kb_used_data":11056,"kb_used_omap":0,"kb_used_meta":521664,"kb_avail":93212424,"statfs":{"total":95995035648,"available":95449522176,"internally_reserved":0,"allocated":11321344,"data_stored":9554334,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":534183936},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:49:05 2023","interfaces":[{"interface":"back","average":{"1min":2.5409999999999999,"5min":2.0819999999999999,"15min":1.978},"min":{"1min":0.24399999999999999,"5min":0.219,"15min":0.219},"max":{"1min":12.442,"5min":21.018000000000001,"15min":34.143999999999998},"last":0.61399999999999999},{"interface":"front","average":{"1min":2.2919999999999998,"5min":2.2200000000000002,"15min":2.2360000000000002},"min":{"1min":0.30299999999999999,"5min":0.255,"15min":0.22700000000000001},"max":{"1min":11.557,"5min":21.082000000000001,"15min":60.328000000000003},"last":0.54300000000000004}]},{"osd":1,"last update":"Sat Sep 23 20:49:20 2023","interfaces":[{"interface":"back","average":{"1min":1.6499999999999999,"5min":1.895,"15min":2.052},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.249},"max":{"1min":7.7409999999999997,"5min":8.9350000000000005,"15min":64.274000000000001},"last":0.57799999999999996},{"interface":"front","average":{"1min":0.93999999999999995,"5min":1.5820000000000001,"15min":1.7989999999999999},"min":{"1min":0.312,"5min":0.29099999999999998,"15min":0.28000000000000003},"max":{"1min":3.0139999999999998,"5min":8.8070000000000004,"15min":34.490000000000002},"last":0.66400000000000003}]},{"osd":2,"last update":"Sat Sep 23 20:49:05 2023","interfaces":[{"interface":"back","average":{"1min":3.6360000000000001,"5min":2.6520000000000001,"15min":2.3010000000000002},"min":{"1min":0.36199999999999999,"5min":0.29699999999999999,"15min":0.27700000000000002},"max":{"1min":9.0730000000000004,"5min":22.591000000000001,"15min":34.317999999999998},"last":0.504},{"interface":"front","average":{"1min":2.8170000000000002,"5min":2.198,"15min":2.0779999999999998},"min":{"1min":0.42999999999999999,"5min":0.33900000000000002,"15min":0.30399999999999999},"max":{"1min":12.798,"5min":22.614999999999998,"15min":34.292999999999999},"last":0.69899999999999995}]},{"osd":3,"last update":"Sat Sep 23 20:48:56 2023","interfaces":[{"interface":"back","average":{"1min":2.5659999999999998,"5min":2.0920000000000001,"15min":1.9650000000000001},"min":{"1min":0.46999999999999997,"5min":0.307,"15min":0.28399999999999997},"max":{"1min":7.9640000000000004,"5min":11.664,"15min":11.664},"last":0.45700000000000002},{"interface":"front","average":{"1min":3.1840000000000002,"5min":2.335,"15min":2.1970000000000001},"min":{"1min":0.50600000000000001,"5min":0.32700000000000001,"15min":0.30499999999999999},"max":{"1min":11.6,"5min":11.6,"15min":12.875999999999999},"last":0.59699999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:49:20 2023","interfaces":[{"interface":"back","average":{"1min":0.96499999999999997,"5min":1.0680000000000001,"15min":0.96999999999999997},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.25600000000000001},"max":{"1min":2.831,"5min":10.183999999999999,"15min":23.904},"last":0.47799999999999998},{"interface":"front","average":{"1min":2.4569999999999999,"5min":1.6160000000000001,"15min":1.27},"min":{"1min":0.36299999999999999,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":11.999000000000001,"5min":11.999000000000001,"15min":11.999000000000001},"last":0.38700000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:49:20 2023","interfaces":[{"interface":"back","average":{"1min":0.79800000000000004,"5min":1.2869999999999999,"15min":1.0029999999999999},"min":{"1min":0.35099999999999998,"5min":0.26400000000000001,"15min":0.253},"max":{"1min":2.371,"5min":8.0440000000000005,"15min":8.0440000000000005},"last":0.52700000000000002},{"interface":"front","average":{"1min":1.843,"5min":1.607,"15min":1.3200000000000001},"min":{"1min":0.373,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":8.2769999999999992,"5min":9.3559999999999999,"15min":9.3559999999999999},"last":0.65400000000000003}]},{"osd":7,"last update":"Sat Sep 23 20:48:56 2023","interfaces":[{"interface":"back","average":{"1min":1.2729999999999999,"5min":1.2549999999999999,"15min":0.997},"min":{"1min":0.42899999999999999,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":8.0719999999999992,"5min":8.9309999999999992,"15min":8.9309999999999992},"last":0.56299999999999994},{"interface":"front","average":{"1min":10.34,"5min":3.6499999999999999,"15min":1.9890000000000001},"min":{"1min":0.47099999999999997,"5min":0.35999999999999999,"15min":0.34499999999999997},"max":{"1min":159.91999999999999,"5min":159.91999999999999,"15min":159.91999999999999},"last":0.68600000000000005}]}]},{"osd":1,"up_from":266,"seq":1142461300979,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":807040,"kb_used_data":8504,"kb_used_omap":8,"kb_used_meta":798519,"kb_avail":92938112,"statfs":{"total":95995035648,"available":95168626688,"internally_reserved":0,"allocated":8708096,"data_stored":5479771,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8206,"internal_metadata":817684466},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:49:28 2023","interfaces":[{"interface":"back","average":{"1min":3.0680000000000001,"5min":2.3879999999999999,"15min":2.3109999999999999},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.20000000000000001},"max":{"1min":20.292000000000002,"5min":20.292000000000002,"15min":20.292000000000002},"last":0.32500000000000001},{"interface":"front","average":{"1min":2.101,"5min":2.681,"15min":2.895},"min":{"1min":0.217,"5min":0.217,"15min":0.19900000000000001},"max":{"1min":17.463000000000001,"5min":20.346,"15min":26.588000000000001},"last":0.40300000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:49:28 2023","interfaces":[{"interface":"back","average":{"1min":2.6850000000000001,"5min":2.7629999999999999,"15min":3.0409999999999999},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.216},"max":{"1min":23.341000000000001,"5min":23.341000000000001,"15min":32.255000000000003},"last":0.38300000000000001},{"interface":"front","average":{"1min":2.919,"5min":2.4260000000000002,"15min":2.3180000000000001},"min":{"1min":0.249,"5min":0.249,"15min":0.20899999999999999},"max":{"1min":34.002000000000002,"5min":34.002000000000002,"15min":34.002000000000002},"last":0.47799999999999998}]},{"osd":3,"last update":"Sat Sep 23 20:49:28 2023","interfaces":[{"interface":"back","average":{"1min":1.3720000000000001,"5min":2.0299999999999998,"15min":2.3300000000000001},"min":{"1min":0.28899999999999998,"5min":0.28799999999999998,"15min":0.247},"max":{"1min":8.3059999999999992,"5min":17.366,"15min":17.366},"last":0.37},{"interface":"front","average":{"1min":2.371,"5min":2.6080000000000001,"15min":2.6899999999999999},"min":{"1min":0.317,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":17.343,"5min":17.343,"15min":19.315000000000001},"last":0.61099999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:49:28 2023","interfaces":[{"interface":"back","average":{"1min":2.5169999999999999,"5min":2.4159999999999999,"15min":2.427},"min":{"1min":0.33900000000000002,"5min":0.32600000000000001,"15min":0.28599999999999998},"max":{"1min":7.8920000000000003,"5min":11.791,"15min":24.327999999999999},"last":0.89900000000000002},{"interface":"front","average":{"1min":2.3090000000000002,"5min":1.853,"15min":1.758},"min":{"1min":0.38800000000000001,"5min":0.35699999999999998,"15min":0.311},"max":{"1min":8.5099999999999998,"5min":10.468999999999999,"15min":20.835999999999999},"last":0.82299999999999995}]},{"osd":5,"last update":"Sat Sep 23 20:49:28 2023","interfaces":[{"interface":"back","average":{"1min":2.7749999999999999,"5min":2.3029999999999999,"15min":2.3799999999999999},"min":{"1min":0.38,"5min":0.34699999999999998,"15min":0.308},"max":{"1min":17.396999999999998,"5min":17.396999999999998,"15min":33.401000000000003},"last":0.63700000000000001},{"interface":"front","average":{"1min":2.746,"5min":2.548,"15min":2.6389999999999998},"min":{"1min":0.377,"5min":0.377,"15min":0.32500000000000001},"max":{"1min":16.366,"5min":16.366,"15min":33.417000000000002},"last":0.64700000000000002}]},{"osd":6,"last update":"Sat Sep 23 20:49:28 2023","interfaces":[{"interface":"back","average":{"1min":3.7189999999999999,"5min":2.508,"15min":2.3639999999999999},"min":{"1min":0.41599999999999998,"5min":0.316,"15min":0.316},"max":{"1min":23.274000000000001,"5min":23.274000000000001,"15min":23.274000000000001},"last":0.67900000000000005},{"interface":"front","average":{"1min":2.3599999999999999,"5min":2.819,"15min":2.641},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.317},"max":{"1min":6.9290000000000003,"5min":11.832000000000001,"15min":24.376000000000001},"last":0.62}]},{"osd":7,"last update":"Sat Sep 23 20:49:28 2023","interfaces":[{"interface":"back","average":{"1min":2.7719999999999998,"5min":2.48,"15min":2.4159999999999999},"min":{"1min":0.47099999999999997,"5min":0.377,"15min":0.37},"max":{"1min":17.433,"5min":17.433,"15min":18.437000000000001},"last":0.83899999999999997},{"interface":"front","average":{"1min":2.4980000000000002,"5min":2.8410000000000002,"15min":2.6899999999999999},"min":{"1min":0.44500000000000001,"5min":0.41899999999999998,"15min":0.35999999999999999},"max":{"1min":7.9930000000000003,"5min":11.864000000000001,"15min":24.524000000000001},"last":0.81399999999999995}]}]},{"osd":0,"up_from":94,"seq":403726926182,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":959208,"kb_used_data":12384,"kb_used_omap":7,"kb_used_meta":946808,"kb_avail":92785944,"statfs":{"total":95995035648,"available":95012806656,"internally_reserved":0,"allocated":12681216,"data_stored":9597925,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":969531409},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:49:33 2023","interfaces":[{"interface":"back","average":{"1min":1.034,"5min":3.415,"15min":2.6819999999999999},"min":{"1min":0.20100000000000001,"5min":0.19400000000000001,"15min":0.17199999999999999},"max":{"1min":7.6749999999999998,"5min":152.387,"15min":152.387},"last":0.36799999999999999},{"interface":"front","average":{"1min":1.0700000000000001,"5min":3.5339999999999998,"15min":2.9039999999999999},"min":{"1min":0.32700000000000001,"5min":0.218,"15min":0.20399999999999999},"max":{"1min":7.7069999999999999,"5min":152.84100000000001,"15min":152.84100000000001},"last":0.58199999999999996}]},{"osd":2,"last update":"Sat Sep 23 20:49:07 2023","interfaces":[{"interface":"back","average":{"1min":1.7829999999999999,"5min":2.7879999999999998,"15min":2.867},"min":{"1min":0.252,"5min":0.23499999999999999,"15min":0.218},"max":{"1min":7.008,"5min":15.108000000000001,"15min":62.726999999999997},"last":0.32700000000000001},{"interface":"front","average":{"1min":1.913,"5min":2.706,"15min":2.7930000000000001},"min":{"1min":0.219,"5min":0.217,"15min":0.217},"max":{"1min":9.5609999999999999,"5min":17.013000000000002,"15min":62.689},"last":0.42899999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:49:43 2023","interfaces":[{"interface":"back","average":{"1min":0.69699999999999995,"5min":3.0720000000000001,"15min":3.0310000000000001},"min":{"1min":0.313,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":2.141,"5min":55.017000000000003,"15min":55.017000000000003},"last":0.47199999999999998},{"interface":"front","average":{"1min":1.294,"5min":2.9889999999999999,"15min":2.9609999999999999},"min":{"1min":0.28199999999999997,"5min":0.27400000000000002,"15min":0.21199999999999999},"max":{"1min":8.2439999999999998,"5min":55.665999999999997,"15min":55.665999999999997},"last":0.52900000000000003}]},{"osd":4,"last update":"Sat Sep 23 20:49:07 2023","interfaces":[{"interface":"back","average":{"1min":2.1120000000000001,"5min":2.1589999999999998,"15min":2.0859999999999999},"min":{"1min":0.308,"5min":0.29299999999999998,"15min":0.28899999999999998},"max":{"1min":9.5779999999999994,"5min":11.75,"15min":11.75},"last":0.54800000000000004},{"interface":"front","average":{"1min":1.77,"5min":2.133,"15min":1.9710000000000001},"min":{"1min":0.41599999999999998,"5min":0.31,"15min":0.27400000000000002},"max":{"1min":6.1050000000000004,"5min":12.356999999999999,"15min":12.356999999999999},"last":0.49099999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:49:07 2023","interfaces":[{"interface":"back","average":{"1min":1.8520000000000001,"5min":2.052,"15min":2.0880000000000001},"min":{"1min":0.38300000000000001,"5min":0.308,"15min":0.26500000000000001},"max":{"1min":6.1210000000000004,"5min":9.3339999999999996,"15min":11.497999999999999},"last":0.629},{"interface":"front","average":{"1min":1.5569999999999999,"5min":2.137,"15min":2.2959999999999998},"min":{"1min":0.36899999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":4.9640000000000004,"5min":12.601000000000001,"15min":26.228000000000002},"last":0.51600000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:49:07 2023","interfaces":[{"interface":"back","average":{"1min":2.1629999999999998,"5min":2.3029999999999999,"15min":2.2629999999999999},"min":{"1min":0.45200000000000001,"5min":0.374,"15min":0.34499999999999997},"max":{"1min":9.0570000000000004,"5min":9.2750000000000004,"15min":24.056000000000001},"last":0.66500000000000004},{"interface":"front","average":{"1min":1.9359999999999999,"5min":2.327,"15min":2.2010000000000001},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":9.0380000000000003,"5min":12.698,"15min":26.268000000000001},"last":0.56699999999999995}]},{"osd":7,"last update":"Sat Sep 23 20:49:07 2023","interfaces":[{"interface":"back","average":{"1min":1.607,"5min":2.512,"15min":2.3839999999999999},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":5.0369999999999999,"5min":16.553999999999998,"15min":26.177},"last":0.61199999999999999},{"interface":"front","average":{"1min":1.8420000000000001,"5min":2.4529999999999998,"15min":2.4009999999999998},"min":{"1min":0.41699999999999998,"5min":0.40200000000000002,"15min":0.34200000000000003},"max":{"1min":7.8559999999999999,"5min":16.649000000000001,"15min":16.649000000000001},"last":0.59599999999999997}]}]},{"osd":2,"up_from":104,"seq":446676599129,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":892532,"kb_used_data":12396,"kb_used_omap":11,"kb_used_meta":880116,"kb_avail":92852620,"statfs":{"total":95995035648,"available":95081082880,"internally_reserved":0,"allocated":12693504,"data_stored":9647803,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11878,"internal_metadata":901239194},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:49:11 2023","interfaces":[{"interface":"back","average":{"1min":2.2679999999999998,"5min":1.8140000000000001,"15min":2.036},"min":{"1min":0.19500000000000001,"5min":0.19500000000000001,"15min":0.17299999999999999},"max":{"1min":20.48,"5min":20.48,"15min":28.234999999999999},"last":0.38600000000000001},{"interface":"front","average":{"1min":1.8919999999999999,"5min":1.9139999999999999,"15min":2.2410000000000001},"min":{"1min":0.25700000000000001,"5min":0.245,"15min":0.19},"max":{"1min":6.7220000000000004,"5min":15.342000000000001,"15min":16.678999999999998},"last":0.32600000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:49:11 2023","interfaces":[{"interface":"back","average":{"1min":1.409,"5min":2.1480000000000001,"15min":2.3889999999999998},"min":{"1min":0.28199999999999997,"5min":0.219,"15min":0.219},"max":{"1min":10.324,"5min":21.690000000000001,"15min":46.405000000000001},"last":0.24299999999999999},{"interface":"front","average":{"1min":2.3119999999999998,"5min":1.9550000000000001,"15min":2.4340000000000002},"min":{"1min":0.29499999999999998,"5min":0.22,"15min":0.22},"max":{"1min":9.0690000000000008,"5min":13.215999999999999,"15min":46.459000000000003},"last":0.34200000000000003}]},{"osd":3,"last update":"Sat Sep 23 20:49:42 2023","interfaces":[{"interface":"back","average":{"1min":1.113,"5min":2.3029999999999999,"15min":2.359},"min":{"1min":0.28199999999999997,"5min":0.25600000000000001,"15min":0.253},"max":{"1min":2.9670000000000001,"5min":30.283000000000001,"15min":30.283000000000001},"last":0.41099999999999998},{"interface":"front","average":{"1min":2.4689999999999999,"5min":2.1429999999999998,"15min":1.9139999999999999},"min":{"1min":0.30499999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":20.562999999999999,"5min":30.834,"15min":30.834},"last":0.29399999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:49:11 2023","interfaces":[{"interface":"back","average":{"1min":1.623,"5min":1.9299999999999999,"15min":1.8420000000000001},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.27500000000000002},"max":{"1min":5.556,"5min":11.141999999999999,"15min":12.025},"last":0.39700000000000002},{"interface":"front","average":{"1min":2.069,"5min":2.0099999999999998,"15min":1.982},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.30499999999999999},"max":{"1min":12.662000000000001,"5min":12.662000000000001,"15min":19.440999999999999},"last":0.51800000000000002}]},{"osd":5,"last update":"Sat Sep 23 20:49:11 2023","interfaces":[{"interface":"back","average":{"1min":1.9219999999999999,"5min":1.7150000000000001,"15min":1.8680000000000001},"min":{"1min":0.38400000000000001,"5min":0.33200000000000002,"15min":0.311},"max":{"1min":6.5460000000000003,"5min":11.183,"15min":12.07},"last":4.8570000000000002},{"interface":"front","average":{"1min":1.863,"5min":1.8999999999999999,"15min":1.9870000000000001},"min":{"1min":0.373,"5min":0.373,"15min":0.35299999999999998},"max":{"1min":10.831,"5min":11.061999999999999,"15min":19.513000000000002},"last":4.8639999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:49:11 2023","interfaces":[{"interface":"back","average":{"1min":1.8740000000000001,"5min":2.0760000000000001,"15min":2.0830000000000002},"min":{"1min":0.40699999999999997,"5min":0.34799999999999998,"15min":0.33500000000000002},"max":{"1min":10.791,"5min":11.031000000000001,"15min":19.486000000000001},"last":0.55600000000000005},{"interface":"front","average":{"1min":1.829,"5min":1.609,"15min":1.6020000000000001},"min":{"1min":0.40699999999999997,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":4.9589999999999996,"5min":8.5920000000000005,"15min":11.747},"last":0.61799999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:49:11 2023","interfaces":[{"interface":"back","average":{"1min":3.7509999999999999,"5min":1.978,"15min":1.6950000000000001},"min":{"1min":0.378,"5min":0.35399999999999998,"15min":0.34699999999999998},"max":{"1min":20.530000000000001,"5min":20.530000000000001,"15min":20.530000000000001},"last":0.75600000000000001},{"interface":"front","average":{"1min":1.752,"5min":2.1160000000000001,"15min":2.121},"min":{"1min":0.36899999999999999,"5min":0.35799999999999998,"15min":0.34899999999999998},"max":{"1min":5.3259999999999996,"5min":11.218,"15min":12.099},"last":0.64900000000000002}]}]},{"osd":3,"up_from":109,"seq":468151435603,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1076900,"kb_used_data":8348,"kb_used_omap":4,"kb_used_meta":1068539,"kb_avail":92668252,"statfs":{"total":95995035648,"available":94892290048,"internally_reserved":0,"allocated":8548352,"data_stored":5516501,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4514,"internal_metadata":1094184542},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:49:50 2023","interfaces":[{"interface":"back","average":{"1min":0.99199999999999999,"5min":2.0369999999999999,"15min":2.2189999999999999},"min":{"1min":0.26500000000000001,"5min":0.217,"15min":0.20899999999999999},"max":{"1min":4.5049999999999999,"5min":21.792999999999999,"15min":22.32},"last":0.28699999999999998},{"interface":"front","average":{"1min":0.83099999999999996,"5min":2.0790000000000002,"15min":2.2639999999999998},"min":{"1min":0.254,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":3.4569999999999999,"5min":18.579999999999998,"15min":22.805},"last":0.33100000000000002}]},{"osd":1,"last update":"Sat Sep 23 20:49:21 2023","interfaces":[{"interface":"back","average":{"1min":0.91200000000000003,"5min":1.7889999999999999,"15min":1.897},"min":{"1min":0.21299999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":4.266,"5min":15.177,"15min":17.529},"last":0.34399999999999997},{"interface":"front","average":{"1min":0.93000000000000005,"5min":1.921,"15min":2.137},"min":{"1min":0.28299999999999997,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":4.5339999999999998,"5min":11.380000000000001,"15min":14.19},"last":0.26800000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:49:50 2023","interfaces":[{"interface":"back","average":{"1min":0.79300000000000004,"5min":2.1019999999999999,"15min":2.5449999999999999},"min":{"1min":0.254,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":3.319,"5min":12.473000000000001,"15min":68.206999999999994},"last":0.30599999999999999},{"interface":"front","average":{"1min":1.0509999999999999,"5min":1.841,"15min":2.3210000000000002},"min":{"1min":0.30599999999999999,"5min":0.25600000000000001,"15min":0.245},"max":{"1min":6.9500000000000002,"5min":17.919,"15min":67.736999999999995},"last":0.38300000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:49:50 2023","interfaces":[{"interface":"back","average":{"1min":1.5209999999999999,"5min":1.591,"15min":1.611},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.30499999999999999},"max":{"1min":8.125,"5min":9.0259999999999998,"15min":22.157},"last":0.48299999999999998},{"interface":"front","average":{"1min":1.6100000000000001,"5min":1.7749999999999999,"15min":2.0649999999999999},"min":{"1min":0.35399999999999998,"5min":0.33500000000000002,"15min":0.308},"max":{"1min":6.7759999999999998,"5min":16.529,"15min":16.529},"last":0.51100000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:49:50 2023","interfaces":[{"interface":"back","average":{"1min":1.282,"5min":1.669,"15min":1.7929999999999999},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32300000000000001},"max":{"1min":8.0730000000000004,"5min":8.9849999999999994,"15min":22.289999999999999},"last":0.41899999999999998},{"interface":"front","average":{"1min":0.748,"5min":1.835,"15min":1.718},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.32000000000000001},"max":{"1min":3.3610000000000002,"5min":8.6129999999999995,"15min":12.952},"last":0.42899999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:49:50 2023","interfaces":[{"interface":"back","average":{"1min":1.087,"5min":2.3140000000000001,"15min":1.9219999999999999},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.34300000000000003},"max":{"1min":4.298,"5min":8.6790000000000003,"15min":12.917},"last":0.54600000000000004},{"interface":"front","average":{"1min":1.3100000000000001,"5min":1.8919999999999999,"15min":2.145},"min":{"1min":0.40999999999999998,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":6.8230000000000004,"5min":16.497,"15min":16.497},"last":0.47199999999999998}]},{"osd":7,"last update":"Sat Sep 23 20:49:50 2023","interfaces":[{"interface":"back","average":{"1min":1.377,"5min":1.841,"15min":2.3559999999999999},"min":{"1min":0.44900000000000001,"5min":0.42399999999999999,"15min":0.39900000000000002},"max":{"1min":6.9139999999999997,"5min":16.425000000000001,"15min":16.425000000000001},"last":0.56299999999999994},{"interface":"front","average":{"1min":1.516,"5min":2.1480000000000001,"15min":1.9630000000000001},"min":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.40999999999999998},"max":{"1min":8.173,"5min":11.891999999999999,"15min":22.202999999999999},"last":0.52500000000000002}]}]},{"osd":4,"up_from":30,"seq":128849019357,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":585652,"kb_used_data":26028,"kb_used_omap":0,"kb_used_meta":559616,"kb_avail":93159500,"statfs":{"total":95995035648,"available":95395328000,"internally_reserved":0,"allocated":26652672,"data_stored":24888938,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":573046784},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":7,"apply_latency_ms":7,"commit_latency_ns":7000000,"apply_latency_ns":7000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:49:04 2023","interfaces":[{"interface":"back","average":{"1min":5.3239999999999998,"5min":2.403,"15min":2.0779999999999998},"min":{"1min":0.219,"5min":0.219,"15min":0.219},"max":{"1min":73.272000000000006,"5min":73.272000000000006,"15min":73.272000000000006},"last":0.55900000000000005},{"interface":"front","average":{"1min":3.0369999999999999,"5min":2.1890000000000001,"15min":2.1259999999999999},"min":{"1min":0.28299999999999997,"5min":0.25800000000000001,"15min":0.246},"max":{"1min":29.754000000000001,"5min":29.754000000000001,"15min":29.754000000000001},"last":0.88700000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:49:17 2023","interfaces":[{"interface":"back","average":{"1min":2.448,"5min":1.9650000000000001,"15min":1.7629999999999999},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.25900000000000001},"max":{"1min":12.343,"5min":12.343,"15min":18.190999999999999},"last":0.45600000000000002},{"interface":"front","average":{"1min":3.1899999999999999,"5min":1.9830000000000001,"15min":1.677},"min":{"1min":0.376,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":12.231999999999999,"5min":12.231999999999999,"15min":18.173999999999999},"last":0.58499999999999996}]},{"osd":2,"last update":"Sat Sep 23 20:49:17 2023","interfaces":[{"interface":"back","average":{"1min":2.6840000000000002,"5min":1.895,"15min":1.7649999999999999},"min":{"1min":0.26700000000000002,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":12.151999999999999,"5min":12.151999999999999,"15min":13.051},"last":0.51000000000000001},{"interface":"front","average":{"1min":2.2290000000000001,"5min":2.0979999999999999,"15min":2.1110000000000002},"min":{"1min":0.34100000000000003,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":12.458,"5min":12.458,"15min":12.458},"last":0.86899999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:49:36 2023","interfaces":[{"interface":"back","average":{"1min":2.069,"5min":2.052,"15min":1.9570000000000001},"min":{"1min":0.29699999999999999,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":7.2850000000000001,"5min":11.598000000000001,"15min":12.147},"last":0.439},{"interface":"front","average":{"1min":3.0369999999999999,"5min":2.238,"15min":2.1030000000000002},"min":{"1min":0.44800000000000001,"5min":0.32300000000000001,"15min":0.30299999999999999},"max":{"1min":12.191000000000001,"5min":17.169,"15min":17.169},"last":0.47499999999999998}]},{"osd":5,"last update":"Sat Sep 23 20:49:04 2023","interfaces":[{"interface":"back","average":{"1min":2.8650000000000002,"5min":1.595,"15min":1.2},"min":{"1min":0.25800000000000001,"5min":0.253,"15min":0.253},"max":{"1min":16.940000000000001,"5min":16.940000000000001,"15min":16.940000000000001},"last":0.35999999999999999},{"interface":"front","average":{"1min":1.3580000000000001,"5min":1.536,"15min":1.1930000000000001},"min":{"1min":0.311,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":4.915,"5min":8.1440000000000001,"15min":8.1440000000000001},"last":0.82199999999999995}]},{"osd":6,"last update":"Sat Sep 23 20:49:17 2023","interfaces":[{"interface":"back","average":{"1min":2.423,"5min":1.881,"15min":1.4610000000000001},"min":{"1min":0.29099999999999998,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":13.503,"5min":13.503,"15min":14.731999999999999},"last":0.84899999999999998},{"interface":"front","average":{"1min":3.722,"5min":2.0009999999999999,"15min":1.3979999999999999},"min":{"1min":0.32000000000000001,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":17.059000000000001,"5min":17.059000000000001,"15min":17.059000000000001},"last":0.68799999999999994}]},{"osd":7,"last update":"Sat Sep 23 20:49:36 2023","interfaces":[{"interface":"back","average":{"1min":1.2509999999999999,"5min":4.8310000000000004,"15min":2.6789999999999998},"min":{"1min":0.34100000000000003,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":5.5389999999999997,"5min":313.50099999999998,"15min":313.50099999999998},"last":1.012},{"interface":"front","average":{"1min":1.8129999999999999,"5min":4.5960000000000001,"15min":2.3340000000000001},"min":{"1min":0.37,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":12.414999999999999,"5min":312.67500000000001,"15min":312.67500000000001},"last":0.60899999999999999}]}]},{"osd":5,"up_from":0,"seq":167503725018,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4722,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1360,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8206,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7156,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3154,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:50:00.923 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-09-23T20:50:00.924 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd pool set .mgr pg_num 81 2023-09-23T20:50:00.960 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:50:00.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:00 smithi149 ceph-mon[170394]: from='client.47057 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:50:00.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:00 smithi149 ceph-mon[170394]: Health detail: HEALTH_WARN noscrub,nodeep-scrub flag(s) set 2023-09-23T20:50:00.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:00 smithi149 ceph-mon[170394]: [WRN] OSDMAP_FLAGS: noscrub,nodeep-scrub flag(s) set 2023-09-23T20:50:01.030 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:01.117 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:50:01.119 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:01.120 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:50:01.221 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:50:01.250 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:01.344 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:50:01.347 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:01.348 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:50:01.449 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:50:01.475 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:01.519 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:50:01.522 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:01.523 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:50:01.624 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:50:01.643 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:50:01.654 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:01.730 INFO:tasks.workunit.client.0.smithi018.stdout:[ RUN ] TestClsRbd.get_all_features 2023-09-23T20:50:01.742 INFO:tasks.workunit.client.0.smithi018.stdout:[ OK ] TestClsRbd.get_all_features (13 ms) 2023-09-23T20:50:01.742 INFO:tasks.workunit.client.0.smithi018.stdout:[ RUN ] TestClsRbd.copyup 2023-09-23T20:50:01.742 INFO:teuthology.orchestra.run.smithi018.stderr:noscrub is unset 2023-09-23T20:50:01.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:01 smithi018 ceph-mon[184144]: pgmap v1575: 103 pgs: 103 active+clean; 4.6 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 31 KiB/s, 2 objects/s recovering 2023-09-23T20:50:01.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:01 smithi018 ceph-mon[184144]: failed to encode map e643 with expected crc 2023-09-23T20:50:01.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:01 smithi018 ceph-mon[184144]: osdmap e643: 8 total, 8 up, 7 in 2023-09-23T20:50:01.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:01 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/332824022' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi018-461753-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-09-23T20:50:01.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:01 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi018-461753-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-09-23T20:50:01.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:01 smithi018 ceph-mon[184144]: failed to encode map e643 with expected crc 2023-09-23T20:50:01.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:01 smithi018 ceph-mon[184144]: failed to encode map e643 with expected crc 2023-09-23T20:50:01.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:01 smithi018 ceph-mon[184144]: failed to encode map e643 with expected crc 2023-09-23T20:50:01.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:01 smithi018 ceph-mon[184144]: failed to encode map e643 with expected crc 2023-09-23T20:50:01.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:01 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1010480803' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:50:01.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:01 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:50:01.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:01 smithi018 ceph-mon[179647]: pgmap v1575: 103 pgs: 103 active+clean; 4.6 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 31 KiB/s, 2 objects/s recovering 2023-09-23T20:50:01.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:01 smithi018 ceph-mon[179647]: failed to encode map e643 with expected crc 2023-09-23T20:50:01.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:01 smithi018 ceph-mon[179647]: osdmap e643: 8 total, 8 up, 7 in 2023-09-23T20:50:01.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:01 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/332824022' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi018-461753-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-09-23T20:50:01.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:01 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi018-461753-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-09-23T20:50:01.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:01 smithi018 ceph-mon[179647]: failed to encode map e643 with expected crc 2023-09-23T20:50:01.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:01 smithi018 ceph-mon[179647]: failed to encode map e643 with expected crc 2023-09-23T20:50:01.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:01 smithi018 ceph-mon[179647]: failed to encode map e643 with expected crc 2023-09-23T20:50:01.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:01 smithi018 ceph-mon[179647]: failed to encode map e643 with expected crc 2023-09-23T20:50:01.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:01 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1010480803' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:50:01.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:01 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:50:01.939 INFO:tasks.workunit.client.0.smithi018.stdout:[ OK ] TestClsRbd.copyup (196 ms) 2023-09-23T20:50:01.939 INFO:tasks.workunit.client.0.smithi018.stdout:[ RUN ] TestClsRbd.sparse_copyup 2023-09-23T20:50:01.957 INFO:tasks.workunit.client.0.smithi018.stdout:[ OK ] TestClsRbd.sparse_copyup (18 ms) 2023-09-23T20:50:01.957 INFO:tasks.workunit.client.0.smithi018.stdout:[ RUN ] TestClsRbd.get_and_set_id 2023-09-23T20:50:01.966 INFO:tasks.workunit.client.0.smithi018.stdout:[ OK ] TestClsRbd.get_and_set_id (9 ms) 2023-09-23T20:50:01.966 INFO:tasks.workunit.client.0.smithi018.stdout:[ RUN ] TestClsRbd.add_remove_child 2023-09-23T20:50:01.998 INFO:tasks.workunit.client.0.smithi018.stdout:[ OK ] TestClsRbd.add_remove_child (32 ms) 2023-09-23T20:50:01.998 INFO:tasks.workunit.client.0.smithi018.stdout:[ RUN ] TestClsRbd.directory_methods 2023-09-23T20:50:02.000 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:50:02.008 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:02.008 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:50:02.025 INFO:tasks.workunit.client.0.smithi018.stdout:[ OK ] TestClsRbd.directory_methods (28 ms) 2023-09-23T20:50:02.025 INFO:tasks.workunit.client.0.smithi018.stdout:[ RUN ] TestClsRbd.create 2023-09-23T20:50:02.048 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:50:02.089 INFO:tasks.workunit.client.0.smithi018.stdout:[ OK ] TestClsRbd.create (64 ms) 2023-09-23T20:50:02.089 INFO:tasks.workunit.client.0.smithi018.stdout:[ RUN ] TestClsRbd.get_features 2023-09-23T20:50:02.091 INFO:tasks.workunit.client.0.smithi018.stdout:[ OK ] TestClsRbd.get_features (2 ms) 2023-09-23T20:50:02.091 INFO:tasks.workunit.client.0.smithi018.stdout:[ RUN ] TestClsRbd.get_object_prefix 2023-09-23T20:50:02.093 INFO:tasks.workunit.client.0.smithi018.stdout:[ OK ] TestClsRbd.get_object_prefix (2 ms) 2023-09-23T20:50:02.093 INFO:tasks.workunit.client.0.smithi018.stdout:[ RUN ] TestClsRbd.get_create_timestamp 2023-09-23T20:50:02.095 INFO:tasks.workunit.client.0.smithi018.stdout:[ OK ] TestClsRbd.get_create_timestamp (2 ms) 2023-09-23T20:50:02.095 INFO:tasks.workunit.client.0.smithi018.stdout:[ RUN ] TestClsRbd.get_access_timestamp 2023-09-23T20:50:02.097 INFO:tasks.workunit.client.0.smithi018.stdout:[ OK ] TestClsRbd.get_access_timestamp (2 ms) 2023-09-23T20:50:02.097 INFO:tasks.workunit.client.0.smithi018.stdout:[ RUN ] TestClsRbd.get_modify_timestamp 2023-09-23T20:50:02.099 INFO:tasks.workunit.client.0.smithi018.stdout:[ OK ] TestClsRbd.get_modify_timestamp (2 ms) 2023-09-23T20:50:02.099 INFO:tasks.workunit.client.0.smithi018.stdout:[ RUN ] TestClsRbd.get_data_pool 2023-09-23T20:50:02.103 INFO:tasks.workunit.client.0.smithi018.stdout:[ OK ] TestClsRbd.get_data_pool (3 ms) 2023-09-23T20:50:02.103 INFO:tasks.workunit.client.0.smithi018.stdout:[ RUN ] TestClsRbd.get_size 2023-09-23T20:50:02.107 INFO:tasks.workunit.client.0.smithi018.stdout:[ OK ] TestClsRbd.get_size (5 ms) 2023-09-23T20:50:02.108 INFO:tasks.workunit.client.0.smithi018.stdout:[ RUN ] TestClsRbd.set_size 2023-09-23T20:50:02.109 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:50:02.113 INFO:tasks.workunit.client.0.smithi018.stdout:[ OK ] TestClsRbd.set_size (5 ms) 2023-09-23T20:50:02.113 INFO:tasks.workunit.client.0.smithi018.stdout:[ RUN ] TestClsRbd.protection_status 2023-09-23T20:50:02.142 INFO:tasks.workunit.client.0.smithi018.stdout:[ OK ] TestClsRbd.protection_status (29 ms) 2023-09-23T20:50:02.142 INFO:tasks.workunit.client.0.smithi018.stdout:[ RUN ] TestClsRbd.snapshot_limits 2023-09-23T20:50:02.145 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:02.150 INFO:tasks.workunit.client.0.smithi018.stdout:[ OK ] TestClsRbd.snapshot_limits (9 ms) 2023-09-23T20:50:02.150 INFO:tasks.workunit.client.0.smithi018.stdout:[ RUN ] TestClsRbd.parents_v1 2023-09-23T20:50:02.207 INFO:tasks.workunit.client.0.smithi018.stdout:[ OK ] TestClsRbd.parents_v1 (56 ms) 2023-09-23T20:50:02.207 INFO:tasks.workunit.client.0.smithi018.stdout:[ RUN ] TestClsRbd.parents_v2 2023-09-23T20:50:02.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:01 smithi149 ceph-mon[170394]: pgmap v1575: 103 pgs: 103 active+clean; 4.6 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 31 KiB/s, 2 objects/s recovering 2023-09-23T20:50:02.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:01 smithi149 ceph-mon[170394]: failed to encode map e643 with expected crc 2023-09-23T20:50:02.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:01 smithi149 ceph-mon[170394]: osdmap e643: 8 total, 8 up, 7 in 2023-09-23T20:50:02.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:01 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/332824022' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi018-461753-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-09-23T20:50:02.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:01 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi018-461753-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-09-23T20:50:02.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:01 smithi149 ceph-mon[170394]: failed to encode map e643 with expected crc 2023-09-23T20:50:02.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:01 smithi149 ceph-mon[170394]: failed to encode map e643 with expected crc 2023-09-23T20:50:02.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:01 smithi149 ceph-mon[170394]: failed to encode map e643 with expected crc 2023-09-23T20:50:02.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:01 smithi149 ceph-mon[170394]: failed to encode map e643 with expected crc 2023-09-23T20:50:02.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:01 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1010480803' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:50:02.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:01 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:50:02.238 INFO:tasks.workunit.client.0.smithi018.stdout:[ OK ] TestClsRbd.parents_v2 (32 ms) 2023-09-23T20:50:02.239 INFO:tasks.workunit.client.0.smithi018.stdout:[ RUN ] TestClsRbd.snapshots 2023-09-23T20:50:02.262 INFO:tasks.workunit.client.0.smithi018.stdout:[ OK ] TestClsRbd.snapshots (23 ms) 2023-09-23T20:50:02.262 INFO:tasks.workunit.client.0.smithi018.stdout:[ RUN ] TestClsRbd.snapid_race 2023-09-23T20:50:02.266 INFO:tasks.workunit.client.0.smithi018.stdout:[ OK ] TestClsRbd.snapid_race (5 ms) 2023-09-23T20:50:02.267 INFO:tasks.workunit.client.0.smithi018.stdout:[ RUN ] TestClsRbd.stripingv2 2023-09-23T20:50:02.276 INFO:tasks.workunit.client.0.smithi018.stdout:[ OK ] TestClsRbd.stripingv2 (10 ms) 2023-09-23T20:50:02.276 INFO:tasks.workunit.client.0.smithi018.stdout:[ RUN ] TestClsRbd.object_map_save 2023-09-23T20:50:02.278 INFO:tasks.workunit.client.0.smithi018.stdout:[ OK ] TestClsRbd.object_map_save (1 ms) 2023-09-23T20:50:02.278 INFO:tasks.workunit.client.0.smithi018.stdout:[ RUN ] TestClsRbd.object_map_resize 2023-09-23T20:50:02.283 INFO:tasks.workunit.client.0.smithi018.stdout:[ OK ] TestClsRbd.object_map_resize (5 ms) 2023-09-23T20:50:02.283 INFO:tasks.workunit.client.0.smithi018.stdout:[ RUN ] TestClsRbd.object_map_update 2023-09-23T20:50:02.289 INFO:tasks.workunit.client.0.smithi018.stdout:[ OK ] TestClsRbd.object_map_update (7 ms) 2023-09-23T20:50:02.290 INFO:tasks.workunit.client.0.smithi018.stdout:[ RUN ] TestClsRbd.object_map_load_enoent 2023-09-23T20:50:02.290 INFO:tasks.workunit.client.0.smithi018.stdout:[ OK ] TestClsRbd.object_map_load_enoent (0 ms) 2023-09-23T20:50:02.290 INFO:tasks.workunit.client.0.smithi018.stdout:[ RUN ] TestClsRbd.object_map_snap_add 2023-09-23T20:50:02.296 INFO:tasks.workunit.client.0.smithi018.stdout:[ OK ] TestClsRbd.object_map_snap_add (7 ms) 2023-09-23T20:50:02.297 INFO:tasks.workunit.client.0.smithi018.stdout:[ RUN ] TestClsRbd.object_map_snap_remove 2023-09-23T20:50:02.304 INFO:tasks.workunit.client.0.smithi018.stdout:[ OK ] TestClsRbd.object_map_snap_remove (8 ms) 2023-09-23T20:50:02.304 INFO:tasks.workunit.client.0.smithi018.stdout:[ RUN ] TestClsRbd.flags 2023-09-23T20:50:02.312 INFO:tasks.workunit.client.0.smithi018.stdout:[ OK ] TestClsRbd.flags (8 ms) 2023-09-23T20:50:02.312 INFO:tasks.workunit.client.0.smithi018.stdout:[ RUN ] TestClsRbd.metadata 2023-09-23T20:50:02.342 INFO:tasks.workunit.client.0.smithi018.stdout:[ OK ] TestClsRbd.metadata (30 ms) 2023-09-23T20:50:02.342 INFO:tasks.workunit.client.0.smithi018.stdout:[ RUN ] TestClsRbd.set_features 2023-09-23T20:50:02.351 INFO:tasks.workunit.client.0.smithi018.stdout:[ OK ] TestClsRbd.set_features (8 ms) 2023-09-23T20:50:02.351 INFO:tasks.workunit.client.0.smithi018.stdout:[ RUN ] TestClsRbd.mirror 2023-09-23T20:50:02.394 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:50:02.398 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:02.399 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:50:02.410 INFO:tasks.workunit.client.0.smithi018.stdout:[ OK ] TestClsRbd.mirror (60 ms) 2023-09-23T20:50:02.410 INFO:tasks.workunit.client.0.smithi018.stdout:[ RUN ] TestClsRbd.mirror_image 2023-09-23T20:50:02.431 INFO:tasks.workunit.client.0.smithi018.stdout:[ OK ] TestClsRbd.mirror_image (21 ms) 2023-09-23T20:50:02.431 INFO:tasks.workunit.client.0.smithi018.stdout:[ RUN ] TestClsRbd.mirror_image_status 2023-09-23T20:50:02.500 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:50:02.540 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:02.587 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:50:02.591 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:02.591 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:50:02.592 INFO:teuthology.orchestra.run.smithi149.stdout:ERROR: (22) Invalid argument 2023-09-23T20:50:02.592 INFO:teuthology.orchestra.run.smithi149.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:50:02.692 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:50:02.725 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:02.742 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:50:02.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:02 smithi018 ceph-mon[184144]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:50:02.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:02 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi018-461753-1","app": "rados","yes_i_really_mean_it": true}]': finished 2023-09-23T20:50:02.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:02 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:50:02.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:02 smithi018 ceph-mon[184144]: osdmap e644: 8 total, 8 up, 7 in 2023-09-23T20:50:02.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:02 smithi018 ceph-mon[184144]: failed to encode map e644 with expected crc 2023-09-23T20:50:02.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:02 smithi018 ceph-mon[184144]: failed to encode map e644 with expected crc 2023-09-23T20:50:02.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:02 smithi018 ceph-mon[184144]: failed to encode map e644 with expected crc 2023-09-23T20:50:02.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:02 smithi018 ceph-mon[184144]: failed to encode map e644 with expected crc 2023-09-23T20:50:02.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:02 smithi018 ceph-mon[179647]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:50:02.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:02 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi018-461753-1","app": "rados","yes_i_really_mean_it": true}]': finished 2023-09-23T20:50:02.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:02 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:50:02.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:02 smithi018 ceph-mon[179647]: osdmap e644: 8 total, 8 up, 7 in 2023-09-23T20:50:02.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:02 smithi018 ceph-mon[179647]: failed to encode map e644 with expected crc 2023-09-23T20:50:02.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:02 smithi018 ceph-mon[179647]: failed to encode map e644 with expected crc 2023-09-23T20:50:02.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:02 smithi018 ceph-mon[179647]: failed to encode map e644 with expected crc 2023-09-23T20:50:02.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:02 smithi018 ceph-mon[179647]: failed to encode map e644 with expected crc 2023-09-23T20:50:02.958 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T20:50:02.972 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:50:02.974 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:02 smithi149 ceph-mon[170394]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:50:02.974 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:02 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi018-461753-1","app": "rados","yes_i_really_mean_it": true}]': finished 2023-09-23T20:50:02.974 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:02 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:50:02.974 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:02 smithi149 ceph-mon[170394]: osdmap e644: 8 total, 8 up, 7 in 2023-09-23T20:50:02.974 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:02 smithi149 ceph-mon[170394]: failed to encode map e644 with expected crc 2023-09-23T20:50:02.975 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:02 smithi149 ceph-mon[170394]: failed to encode map e644 with expected crc 2023-09-23T20:50:02.975 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:02 smithi149 ceph-mon[170394]: failed to encode map e644 with expected crc 2023-09-23T20:50:02.975 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:02 smithi149 ceph-mon[170394]: failed to encode map e644 with expected crc 2023-09-23T20:50:02.975 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:02.976 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:50:03.076 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:50:03.102 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:03.138 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:50:03.142 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:03.142 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:50:03.243 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:50:03.281 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:03.661 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:50:03.666 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:03.667 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:50:03.767 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:50:03.794 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:03.829 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:50:03.832 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:03.832 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:50:03.851 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:50:03.934 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:50:03.968 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:03 smithi018 ceph-mon[184144]: pgmap v1578: 135 pgs: 3 creating+peering, 26 unknown, 106 active+clean; 4.6 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail 2023-09-23T20:50:03.968 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:03 smithi018 ceph-mon[184144]: osdmap e645: 8 total, 8 up, 7 in 2023-09-23T20:50:03.969 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:03 smithi018 ceph-mon[184144]: failed to encode map e645 with expected crc 2023-09-23T20:50:03.969 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:03 smithi018 ceph-mon[184144]: failed to encode map e645 with expected crc 2023-09-23T20:50:03.969 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:03 smithi018 ceph-mon[184144]: failed to encode map e645 with expected crc 2023-09-23T20:50:03.969 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:03 smithi018 ceph-mon[179647]: pgmap v1578: 135 pgs: 3 creating+peering, 26 unknown, 106 active+clean; 4.6 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail 2023-09-23T20:50:03.969 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:03 smithi018 ceph-mon[179647]: osdmap e645: 8 total, 8 up, 7 in 2023-09-23T20:50:03.969 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:03 smithi018 ceph-mon[179647]: failed to encode map e645 with expected crc 2023-09-23T20:50:03.969 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:03 smithi018 ceph-mon[179647]: failed to encode map e645 with expected crc 2023-09-23T20:50:03.969 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:03 smithi018 ceph-mon[179647]: failed to encode map e645 with expected crc 2023-09-23T20:50:03.974 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:04.022 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:50:04.026 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:04.026 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:50:04.076 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:03 smithi149 ceph-mon[170394]: pgmap v1578: 135 pgs: 3 creating+peering, 26 unknown, 106 active+clean; 4.6 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail 2023-09-23T20:50:04.077 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:03 smithi149 ceph-mon[170394]: osdmap e645: 8 total, 8 up, 7 in 2023-09-23T20:50:04.077 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:03 smithi149 ceph-mon[170394]: failed to encode map e645 with expected crc 2023-09-23T20:50:04.077 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:03 smithi149 ceph-mon[170394]: failed to encode map e645 with expected crc 2023-09-23T20:50:04.077 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:03 smithi149 ceph-mon[170394]: failed to encode map e645 with expected crc 2023-09-23T20:50:04.127 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:50:04.154 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:04.189 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:50:04.193 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:04.193 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:50:04.294 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:50:04.325 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:04.361 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:50:04.388 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T20:50:04.389 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:50:04.423 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:50:04.431 INFO:teuthology.orchestra.run.smithi018.stderr:Error EINVAL: specified pg_num 81 < pg_num_max 32 2023-09-23T20:50:04.457 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:50:04.465 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:50:04.468 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:04.468 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:50:04.486 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:50:04.517 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:50:04.543 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:50:04.569 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:50:04.570 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:50:04.572 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:50:04.572 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:50:04.573 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:50:04.573 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:50:04.573 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:50:04.573 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:50:04.573 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:50:04.573 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:50:04.596 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:04.730 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:04 smithi018 ceph-mon[184144]: failed to encode map e645 with expected crc 2023-09-23T20:50:04.730 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:04 smithi018 ceph-mon[184144]: 1.8 scrub starts 2023-09-23T20:50:04.730 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:04 smithi018 ceph-mon[184144]: 1.8 scrub ok 2023-09-23T20:50:04.730 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:04 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3881908176' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:50:04.731 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:04 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1657878772' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "81"}]: dispatch 2023-09-23T20:50:04.731 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:04 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "81"}]: dispatch 2023-09-23T20:50:04.768 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:50:04.771 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:04.771 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:50:04.872 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:50:04.900 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:04 smithi149 ceph-mon[170394]: failed to encode map e645 with expected crc 2023-09-23T20:50:04.900 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:04 smithi149 ceph-mon[170394]: 1.8 scrub starts 2023-09-23T20:50:04.900 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:04 smithi149 ceph-mon[170394]: 1.8 scrub ok 2023-09-23T20:50:04.900 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:04 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3881908176' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:50:04.900 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:04 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1657878772' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "81"}]: dispatch 2023-09-23T20:50:04.901 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:04 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "81"}]: dispatch 2023-09-23T20:50:04.901 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:04.978 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:50:04.982 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:04.982 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:50:04.994 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:04 smithi018 ceph-mon[179647]: failed to encode map e645 with expected crc 2023-09-23T20:50:04.994 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:04 smithi018 ceph-mon[179647]: 1.8 scrub starts 2023-09-23T20:50:04.995 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:04 smithi018 ceph-mon[179647]: 1.8 scrub ok 2023-09-23T20:50:04.995 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:04 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3881908176' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:50:04.995 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:04 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1657878772' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "81"}]: dispatch 2023-09-23T20:50:04.995 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:04 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "81"}]: dispatch 2023-09-23T20:50:04.995 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:50:04 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:50:04] "GET /metrics HTTP/1.1" 200 34180 "" "Prometheus/2.43.0" 2023-09-23T20:50:05.051 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-09-23T20:50:05.083 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:50:05.130 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:05.170 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:50:05.177 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:05.178 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:50:05.279 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:50:05.292 INFO:teuthology.orchestra.run.smithi149.stdout:{ 2023-09-23T20:50:05.292 INFO:teuthology.orchestra.run.smithi149.stdout: "ops": [], 2023-09-23T20:50:05.292 INFO:teuthology.orchestra.run.smithi149.stdout: "complaint_time": 30, 2023-09-23T20:50:05.292 INFO:teuthology.orchestra.run.smithi149.stdout: "num_blocked_ops": 0 2023-09-23T20:50:05.293 INFO:teuthology.orchestra.run.smithi149.stdout:} 2023-09-23T20:50:05.314 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:05.315 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:50:05.514 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:50:05.517 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:05.518 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:50:05.619 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:50:05.620 INFO:tasks.workunit.client.0.smithi018.stdout:[ OK ] TestClsRbd.mirror_image_status (3189 ms) 2023-09-23T20:50:05.620 INFO:tasks.workunit.client.0.smithi018.stdout:[ RUN ] TestClsRbd.mirror_image_map 2023-09-23T20:50:05.648 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:05.715 INFO:tasks.workunit.client.0.smithi018.stdout:[ OK ] TestClsRbd.mirror_image_map (95 ms) 2023-09-23T20:50:05.715 INFO:tasks.workunit.client.0.smithi018.stdout:[ RUN ] TestClsRbd.mirror_instances 2023-09-23T20:50:05.724 INFO:tasks.workunit.client.0.smithi018.stdout:[ OK ] TestClsRbd.mirror_instances (9 ms) 2023-09-23T20:50:05.724 INFO:tasks.workunit.client.0.smithi018.stdout:[ RUN ] TestClsRbd.mirror_snapshot 2023-09-23T20:50:05.741 INFO:tasks.workunit.client.0.smithi018.stdout:[ OK ] TestClsRbd.mirror_snapshot (16 ms) 2023-09-23T20:50:05.741 INFO:tasks.workunit.client.0.smithi018.stdout:[ RUN ] TestClsRbd.group_dir_list 2023-09-23T20:50:05.743 INFO:tasks.workunit.client.0.smithi018.stdout:[ OK ] TestClsRbd.group_dir_list (3 ms) 2023-09-23T20:50:05.744 INFO:tasks.workunit.client.0.smithi018.stdout:[ RUN ] TestClsRbd.group_dir_add 2023-09-23T20:50:05.745 INFO:tasks.workunit.client.0.smithi018.stdout:[ OK ] TestClsRbd.group_dir_add (2 ms) 2023-09-23T20:50:05.746 INFO:tasks.workunit.client.0.smithi018.stdout:[ RUN ] TestClsRbd.dir_add_already_existing 2023-09-23T20:50:05.748 INFO:tasks.workunit.client.0.smithi018.stdout:[ OK ] TestClsRbd.dir_add_already_existing (2 ms) 2023-09-23T20:50:05.748 INFO:tasks.workunit.client.0.smithi018.stdout:[ RUN ] TestClsRbd.group_dir_rename 2023-09-23T20:50:05.753 INFO:tasks.workunit.client.0.smithi018.stdout:[ OK ] TestClsRbd.group_dir_rename (6 ms) 2023-09-23T20:50:05.753 INFO:tasks.workunit.client.0.smithi018.stdout:[ RUN ] TestClsRbd.group_dir_remove 2023-09-23T20:50:05.756 INFO:tasks.workunit.client.0.smithi018.stdout:[ OK ] TestClsRbd.group_dir_remove (3 ms) 2023-09-23T20:50:05.756 INFO:tasks.workunit.client.0.smithi018.stdout:[ RUN ] TestClsRbd.group_dir_remove_missing 2023-09-23T20:50:05.762 INFO:tasks.workunit.client.0.smithi018.stdout:[ OK ] TestClsRbd.group_dir_remove_missing (5 ms) 2023-09-23T20:50:05.762 INFO:tasks.workunit.client.0.smithi018.stdout:[ RUN ] TestClsRbd.group_image_add 2023-09-23T20:50:05.764 INFO:tasks.workunit.client.0.smithi018.stdout:[ OK ] TestClsRbd.group_image_add (2 ms) 2023-09-23T20:50:05.764 INFO:tasks.workunit.client.0.smithi018.stdout:[ RUN ] TestClsRbd.group_image_remove 2023-09-23T20:50:05.767 INFO:tasks.workunit.client.0.smithi018.stdout:[ OK ] TestClsRbd.group_image_remove (4 ms) 2023-09-23T20:50:05.767 INFO:tasks.workunit.client.0.smithi018.stdout:[ RUN ] TestClsRbd.group_image_list 2023-09-23T20:50:05.770 INFO:tasks.workunit.client.0.smithi018.stdout:[ OK ] TestClsRbd.group_image_list (2 ms) 2023-09-23T20:50:05.770 INFO:tasks.workunit.client.0.smithi018.stdout:[ RUN ] TestClsRbd.group_image_clean 2023-09-23T20:50:05.775 INFO:tasks.workunit.client.0.smithi018.stdout:[ OK ] TestClsRbd.group_image_clean (6 ms) 2023-09-23T20:50:05.775 INFO:tasks.workunit.client.0.smithi018.stdout:[ RUN ] TestClsRbd.image_group_add 2023-09-23T20:50:05.778 INFO:tasks.workunit.client.0.smithi018.stdout:[ OK ] TestClsRbd.image_group_add (2 ms) 2023-09-23T20:50:05.778 INFO:tasks.workunit.client.0.smithi018.stdout:[ RUN ] TestClsRbd.image_group_remove 2023-09-23T20:50:05.781 INFO:tasks.workunit.client.0.smithi018.stdout:[ OK ] TestClsRbd.image_group_remove (4 ms) 2023-09-23T20:50:05.781 INFO:tasks.workunit.client.0.smithi018.stdout:[ RUN ] TestClsRbd.image_group_get 2023-09-23T20:50:05.783 INFO:tasks.workunit.client.0.smithi018.stdout:[ OK ] TestClsRbd.image_group_get (2 ms) 2023-09-23T20:50:05.784 INFO:tasks.workunit.client.0.smithi018.stdout:[ RUN ] TestClsRbd.group_snap_set_empty_name 2023-09-23T20:50:05.785 INFO:tasks.workunit.client.0.smithi018.stdout:[ OK ] TestClsRbd.group_snap_set_empty_name (2 ms) 2023-09-23T20:50:05.785 INFO:tasks.workunit.client.0.smithi018.stdout:[ RUN ] TestClsRbd.group_snap_set_empty_id 2023-09-23T20:50:05.788 INFO:tasks.workunit.client.0.smithi018.stdout:[ OK ] TestClsRbd.group_snap_set_empty_id (2 ms) 2023-09-23T20:50:05.788 INFO:tasks.workunit.client.0.smithi018.stdout:[ RUN ] TestClsRbd.group_snap_set_duplicate_id 2023-09-23T20:50:05.791 INFO:tasks.workunit.client.0.smithi018.stdout:[ OK ] TestClsRbd.group_snap_set_duplicate_id (3 ms) 2023-09-23T20:50:05.791 INFO:tasks.workunit.client.0.smithi018.stdout:[ RUN ] TestClsRbd.group_snap_set_duplicate_name 2023-09-23T20:50:05.794 INFO:tasks.workunit.client.0.smithi018.stdout:[ OK ] TestClsRbd.group_snap_set_duplicate_name (4 ms) 2023-09-23T20:50:05.794 INFO:tasks.workunit.client.0.smithi018.stdout:[ RUN ] TestClsRbd.group_snap_set 2023-09-23T20:50:05.796 INFO:tasks.workunit.client.0.smithi018.stdout:[ OK ] TestClsRbd.group_snap_set (2 ms) 2023-09-23T20:50:05.797 INFO:tasks.workunit.client.0.smithi018.stdout:[ RUN ] TestClsRbd.group_snap_list 2023-09-23T20:50:05.800 INFO:tasks.workunit.client.0.smithi018.stdout:[ OK ] TestClsRbd.group_snap_list (4 ms) 2023-09-23T20:50:05.800 INFO:tasks.workunit.client.0.smithi018.stdout:[ RUN ] TestClsRbd.group_snap_list_max_return 2023-09-23T20:50:05.821 INFO:tasks.workunit.client.0.smithi018.stdout:[ OK ] TestClsRbd.group_snap_list_max_return (20 ms) 2023-09-23T20:50:05.821 INFO:tasks.workunit.client.0.smithi018.stdout:[ RUN ] TestClsRbd.group_snap_list_max_read 2023-09-23T20:50:05.834 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:50:05.837 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:05.838 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:50:05.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:05 smithi018 ceph-mon[184144]: pgmap v1580: 135 pgs: 3 creating+peering, 17 unknown, 115 active+clean; 4.6 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 39 KiB/s rd, 7.2 KiB/s wr, 56 op/s 2023-09-23T20:50:05.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:05 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:50:05.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:05 smithi018 ceph-mon[179647]: pgmap v1580: 135 pgs: 3 creating+peering, 17 unknown, 115 active+clean; 4.6 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 39 KiB/s rd, 7.2 KiB/s wr, 56 op/s 2023-09-23T20:50:05.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:05 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:50:05.900 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:05 smithi149 ceph-mon[170394]: pgmap v1580: 135 pgs: 3 creating+peering, 17 unknown, 115 active+clean; 4.6 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 39 KiB/s rd, 7.2 KiB/s wr, 56 op/s 2023-09-23T20:50:05.900 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:05 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:50:05.901 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T20:50:05.939 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:50:05.971 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:06.038 INFO:tasks.workunit.client.0.smithi018.stdout:[ OK ] TestClsRbd.group_snap_list_max_read (217 ms) 2023-09-23T20:50:06.038 INFO:tasks.workunit.client.0.smithi018.stdout:[ RUN ] TestClsRbd.group_snap_remove 2023-09-23T20:50:06.041 INFO:tasks.workunit.client.0.smithi018.stdout:[ OK ] TestClsRbd.group_snap_remove (4 ms) 2023-09-23T20:50:06.042 INFO:tasks.workunit.client.0.smithi018.stdout:[ RUN ] TestClsRbd.group_snap_get_by_id 2023-09-23T20:50:06.044 INFO:tasks.workunit.client.0.smithi018.stdout:[ OK ] TestClsRbd.group_snap_get_by_id (2 ms) 2023-09-23T20:50:06.044 INFO:tasks.workunit.client.0.smithi018.stdout:[ RUN ] TestClsRbd.trash_methods 2023-09-23T20:50:06.053 INFO:tasks.workunit.client.0.smithi018.stdout:[ OK ] TestClsRbd.trash_methods (9 ms) 2023-09-23T20:50:06.053 INFO:tasks.workunit.client.0.smithi018.stdout:[ RUN ] TestClsRbd.op_features 2023-09-23T20:50:06.061 INFO:tasks.workunit.client.0.smithi018.stdout:[ OK ] TestClsRbd.op_features (8 ms) 2023-09-23T20:50:06.061 INFO:tasks.workunit.client.0.smithi018.stdout:[ RUN ] TestClsRbd.clone_parent 2023-09-23T20:50:06.061 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:50:06.065 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:06.065 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:50:06.085 INFO:tasks.workunit.client.0.smithi018.stdout:[ OK ] TestClsRbd.clone_parent (25 ms) 2023-09-23T20:50:06.085 INFO:tasks.workunit.client.0.smithi018.stdout:[ RUN ] TestClsRbd.clone_parent_ns 2023-09-23T20:50:06.095 INFO:tasks.workunit.client.0.smithi018.stdout:[ OK ] TestClsRbd.clone_parent_ns (10 ms) 2023-09-23T20:50:06.095 INFO:tasks.workunit.client.0.smithi018.stdout:[ RUN ] TestClsRbd.clone_child 2023-09-23T20:50:06.108 INFO:tasks.workunit.client.0.smithi018.stdout:[ OK ] TestClsRbd.clone_child (12 ms) 2023-09-23T20:50:06.108 INFO:tasks.workunit.client.0.smithi018.stdout:[ RUN ] TestClsRbd.namespace_methods 2023-09-23T20:50:06.117 INFO:tasks.workunit.client.0.smithi018.stdout:[ OK ] TestClsRbd.namespace_methods (10 ms) 2023-09-23T20:50:06.117 INFO:tasks.workunit.client.0.smithi018.stdout:[ RUN ] TestClsRbd.migration 2023-09-23T20:50:06.129 INFO:tasks.workunit.client.0.smithi018.stdout:[ OK ] TestClsRbd.migration (11 ms) 2023-09-23T20:50:06.129 INFO:tasks.workunit.client.0.smithi018.stdout:[ RUN ] TestClsRbd.migration_v1 2023-09-23T20:50:06.138 INFO:tasks.workunit.client.0.smithi018.stdout:[ OK ] TestClsRbd.migration_v1 (10 ms) 2023-09-23T20:50:06.138 INFO:tasks.workunit.client.0.smithi018.stdout:[ RUN ] TestClsRbd.assert_snapc_seq 2023-09-23T20:50:06.166 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:50:06.194 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:06.229 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:50:06.232 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:06.232 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:50:06.333 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:50:06.363 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:06.397 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:50:06.400 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:06.401 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:50:06.502 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:50:06.535 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:06.535 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:50:06.580 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:50:06.584 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:06.584 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:50:06.685 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:50:06.712 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:07.008 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:50:07.011 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:07.011 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:50:07.034 INFO:teuthology.orchestra.run.smithi018.stderr:nodeep-scrub is unset 2023-09-23T20:50:07.105 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:50:07.112 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:50:07.176 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:07.339 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:50:07.341 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:07 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:50:07.341 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:07 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:50:07.341 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:07 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1920919539' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:50:07.342 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:07 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:50:07.342 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:07 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:50:07.342 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:07 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:50:07.342 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:07 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:50:07.342 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:07 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:50:07.342 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:07 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1920919539' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:50:07.343 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:07 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:50:07.343 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:07 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:50:07.343 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:07 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:50:07.343 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:07.344 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:50:07.444 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:50:07.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:07 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:50:07.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:07 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:50:07.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:07 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1920919539' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:50:07.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:07 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:50:07.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:07 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:50:07.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:07 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:50:07.473 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:07.743 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:50:07.764 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:50:07.769 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:07.769 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:50:07.870 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:50:07.898 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:07.983 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:50:07.986 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:07.987 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:50:08.030 INFO:tasks.workunit.client.0.smithi018.stdout:[ OK ] TestClsRbd.assert_snapc_seq (1892 ms) 2023-09-23T20:50:08.031 INFO:tasks.workunit.client.0.smithi018.stdout:[ RUN ] TestClsRbd.sparsify 2023-09-23T20:50:08.073 INFO:tasks.workunit.client.0.smithi018.stdout:[ OK ] TestClsRbd.sparsify (43 ms) 2023-09-23T20:50:08.088 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:50:08.117 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:08.153 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:50:08.156 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:08 smithi018 ceph-mon[184144]: pgmap v1581: 135 pgs: 135 active+clean; 4.6 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 1.4 MiB/s rd, 712 KiB/s wr, 142 op/s 2023-09-23T20:50:08.156 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:08 smithi018 ceph-mon[184144]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:50:08.156 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:08 smithi018 ceph-mon[184144]: Cluster is now healthy 2023-09-23T20:50:08.156 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:08 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:50:08.156 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:08 smithi018 ceph-mon[184144]: osdmap e646: 8 total, 8 up, 7 in 2023-09-23T20:50:08.156 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:08 smithi018 ceph-mon[184144]: failed to encode map e646 with expected crc 2023-09-23T20:50:08.156 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:08 smithi018 ceph-mon[184144]: failed to encode map e646 with expected crc 2023-09-23T20:50:08.156 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:08 smithi018 ceph-mon[184144]: failed to encode map e646 with expected crc 2023-09-23T20:50:08.157 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:08 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:50:08.157 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:08 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:50:08.157 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:08 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:50:08.157 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:08 smithi018 ceph-mon[179647]: pgmap v1581: 135 pgs: 135 active+clean; 4.6 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 1.4 MiB/s rd, 712 KiB/s wr, 142 op/s 2023-09-23T20:50:08.157 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:08 smithi018 ceph-mon[179647]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:50:08.157 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:08 smithi018 ceph-mon[179647]: Cluster is now healthy 2023-09-23T20:50:08.157 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:08 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:50:08.157 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:08 smithi018 ceph-mon[179647]: osdmap e646: 8 total, 8 up, 7 in 2023-09-23T20:50:08.157 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:08 smithi018 ceph-mon[179647]: failed to encode map e646 with expected crc 2023-09-23T20:50:08.158 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:08 smithi018 ceph-mon[179647]: failed to encode map e646 with expected crc 2023-09-23T20:50:08.158 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:08 smithi018 ceph-mon[179647]: failed to encode map e646 with expected crc 2023-09-23T20:50:08.158 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:08 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:50:08.158 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:08 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:50:08.158 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:08 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:50:08.158 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:08.159 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:50:08.259 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:50:08.285 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:08 smithi149 ceph-mon[170394]: pgmap v1581: 135 pgs: 135 active+clean; 4.6 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 1.4 MiB/s rd, 712 KiB/s wr, 142 op/s 2023-09-23T20:50:08.285 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:08 smithi149 ceph-mon[170394]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:50:08.285 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:08 smithi149 ceph-mon[170394]: Cluster is now healthy 2023-09-23T20:50:08.285 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:08 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:50:08.285 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:08 smithi149 ceph-mon[170394]: osdmap e646: 8 total, 8 up, 7 in 2023-09-23T20:50:08.285 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:08 smithi149 ceph-mon[170394]: failed to encode map e646 with expected crc 2023-09-23T20:50:08.285 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:08 smithi149 ceph-mon[170394]: failed to encode map e646 with expected crc 2023-09-23T20:50:08.286 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:08 smithi149 ceph-mon[170394]: failed to encode map e646 with expected crc 2023-09-23T20:50:08.286 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:08 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:50:08.286 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:08 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:50:08.286 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:08 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:50:08.287 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:08.486 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:50:08.489 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:08.490 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:50:08.591 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:50:08.618 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:08.671 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:50:08.675 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:08.675 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:50:08.777 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:50:08.806 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:08.943 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:50:08.947 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:08.948 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:50:08.980 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:50:08.987 INFO:teuthology.orchestra.run.smithi149.stdout:{ 2023-09-23T20:50:08.987 INFO:teuthology.orchestra.run.smithi149.stdout: "size": 20, 2023-09-23T20:50:08.987 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 600, 2023-09-23T20:50:08.987 INFO:teuthology.orchestra.run.smithi149.stdout: "ops": [ 2023-09-23T20:50:08.987 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:08.987 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:76949 9.b 9:dc3199d0:::benchmark_data_smithi018_381516_object76948:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e465)", 2023-09-23T20:50:08.987 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:41:58.481021+0000", 2023-09-23T20:50:08.987 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 490.50341770799997, 2023-09-23T20:50:08.987 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.012473631, 2023-09-23T20:50:08.987 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:50:08.988 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:50:08.988 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:50:08.988 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:50:08.988 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:50:08.988 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 76949 2023-09-23T20:50:08.988 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:08.988 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:50:08.988 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:08.988 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:50:08.988 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.481021+0000", 2023-09-23T20:50:08.988 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:50:08.988 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:08.989 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:08.989 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:50:08.989 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.481021+0000", 2023-09-23T20:50:08.989 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:50:08.989 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:08.989 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:08.989 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:50:08.989 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.481020+0000", 2023-09-23T20:50:08.989 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:50:08.989 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:08.989 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:08.989 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:50:08.990 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.481102+0000", 2023-09-23T20:50:08.990 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.2538000000000002e-05 2023-09-23T20:50:08.990 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:08.990 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:08.990 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:50:08.990 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.481122+0000", 2023-09-23T20:50:08.990 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.9556e-05 2023-09-23T20:50:08.990 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:08.990 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:08.990 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:50:08.990 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.481129+0000", 2023-09-23T20:50:08.990 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.7449999999999995e-06 2023-09-23T20:50:08.991 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:08.991 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:08.991 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:50:08.991 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.481713+0000", 2023-09-23T20:50:08.991 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000583167 2023-09-23T20:50:08.991 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:08.991 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:08.991 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:50:08.991 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.481726+0000", 2023-09-23T20:50:08.991 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.3291e-05 2023-09-23T20:50:08.991 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:08.991 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:08.992 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:50:08.992 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.467637+0000", 2023-09-23T20:50:08.992 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.98591135500000004 2023-09-23T20:50:08.992 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:08.992 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:08.992 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:50:08.992 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.470079+0000", 2023-09-23T20:50:08.992 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0024414089999999999 2023-09-23T20:50:08.992 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:08.992 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:08.992 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:50:08.992 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.470267+0000", 2023-09-23T20:50:08.992 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00018824299999999999 2023-09-23T20:50:08.993 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:08.993 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:08.993 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:50:08.993 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.481273+0000", 2023-09-23T20:50:08.993 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.011006395 2023-09-23T20:50:08.993 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:08.993 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:08.993 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:08.993 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.493371+0000", 2023-09-23T20:50:08.993 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.012098138 2023-09-23T20:50:08.993 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:08.994 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:08.994 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:08.994 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.493433+0000", 2023-09-23T20:50:08.994 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.1437000000000001e-05 2023-09-23T20:50:08.994 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:08.994 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:08.994 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:50:08.994 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.493472+0000", 2023-09-23T20:50:08.994 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.9038000000000001e-05 2023-09-23T20:50:08.994 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:08.994 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:08.995 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:50:08.995 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.493495+0000", 2023-09-23T20:50:08.995 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.2778000000000001e-05 2023-09-23T20:50:08.995 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:50:08.995 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:50:08.995 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:50:08.995 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:08.995 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:08.995 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:76967 9.b 9:d898a916:::benchmark_data_smithi018_381516_object76966:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e465)", 2023-09-23T20:50:08.995 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:41:58.507229+0000", 2023-09-23T20:50:08.995 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 490.47720929799999, 2023-09-23T20:50:08.995 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.98635601299999998, 2023-09-23T20:50:08.996 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:50:08.996 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:50:08.996 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:50:08.996 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:50:08.996 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:50:08.996 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 76967 2023-09-23T20:50:08.996 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:08.996 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:50:08.996 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:08.996 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:50:08.996 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.507229+0000", 2023-09-23T20:50:08.997 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:50:08.997 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:08.997 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:08.997 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:50:08.997 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.507229+0000", 2023-09-23T20:50:08.997 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:50:08.997 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:08.997 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:08.997 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:50:08.997 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.507228+0000", 2023-09-23T20:50:08.997 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999981 2023-09-23T20:50:08.997 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:08.998 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:08.998 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:50:08.998 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.507285+0000", 2023-09-23T20:50:08.998 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.7531000000000002e-05 2023-09-23T20:50:08.998 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:08.998 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:08.998 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:50:08.998 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.507307+0000", 2023-09-23T20:50:08.998 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.2039e-05 2023-09-23T20:50:08.998 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:08.998 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:08.999 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:50:08.999 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.507314+0000", 2023-09-23T20:50:08.999 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.7089999999999996e-06 2023-09-23T20:50:08.999 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:08.999 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:08.999 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:50:08.999 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.507629+0000", 2023-09-23T20:50:08.999 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00031477799999999999 2023-09-23T20:50:08.999 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:08.999 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:08.999 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:50:08.999 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.507641+0000", 2023-09-23T20:50:09.000 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.258e-05 2023-09-23T20:50:09.000 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.000 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.000 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:50:09.000 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.472754+0000", 2023-09-23T20:50:09.000 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.96511288200000001 2023-09-23T20:50:09.000 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.000 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.000 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:50:09.000 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.472887+0000", 2023-09-23T20:50:09.000 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000133132 2023-09-23T20:50:09.001 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.001 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.001 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:50:09.001 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.472997+0000", 2023-09-23T20:50:09.001 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000109805 2023-09-23T20:50:09.001 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.001 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.001 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:50:09.001 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.484044+0000", 2023-09-23T20:50:09.001 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.011047250999999999 2023-09-23T20:50:09.001 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.001 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.002 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:09.002 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.493403+0000", 2023-09-23T20:50:09.002 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0093591230000000004 2023-09-23T20:50:09.002 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.002 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.002 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:09.002 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.493533+0000", 2023-09-23T20:50:09.002 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000129271 2023-09-23T20:50:09.002 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.002 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.002 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:50:09.002 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.493558+0000", 2023-09-23T20:50:09.003 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.5477e-05 2023-09-23T20:50:09.003 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.003 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.003 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:50:09.003 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.493585+0000", 2023-09-23T20:50:09.003 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.7274e-05 2023-09-23T20:50:09.003 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:50:09.003 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:50:09.003 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:50:09.003 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.004 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.004 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:76972 9.b 9:d979209f:::benchmark_data_smithi018_381516_object76971:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e465)", 2023-09-23T20:50:09.004 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:41:58.507469+0000", 2023-09-23T20:50:09.004 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 490.47696976600002, 2023-09-23T20:50:09.004 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.98628621599999999, 2023-09-23T20:50:09.004 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:50:09.004 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:50:09.004 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:50:09.004 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:50:09.004 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:50:09.005 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 76972 2023-09-23T20:50:09.005 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.005 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:50:09.005 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.005 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:50:09.005 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.507469+0000", 2023-09-23T20:50:09.005 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:50:09.005 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.005 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.005 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:50:09.005 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.507469+0000", 2023-09-23T20:50:09.006 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:50:09.006 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.006 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.006 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:50:09.006 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.507468+0000", 2023-09-23T20:50:09.006 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:50:09.006 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.006 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.006 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:50:09.006 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.507525+0000", 2023-09-23T20:50:09.006 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.6431000000000002e-05 2023-09-23T20:50:09.007 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.007 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.007 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:50:09.007 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.507533+0000", 2023-09-23T20:50:09.007 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.2400000000000007e-06 2023-09-23T20:50:09.007 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.007 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.007 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:50:09.007 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.507538+0000", 2023-09-23T20:50:09.007 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.2129999999999999e-06 2023-09-23T20:50:09.007 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.008 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.008 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:50:09.008 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.507674+0000", 2023-09-23T20:50:09.008 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00013521599999999999 2023-09-23T20:50:09.008 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.008 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.008 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:50:09.008 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.507677+0000", 2023-09-23T20:50:09.008 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.3409999999999999e-06 2023-09-23T20:50:09.009 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.009 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.009 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:50:09.009 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.478437+0000", 2023-09-23T20:50:09.009 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.97076035400000005 2023-09-23T20:50:09.009 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.009 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.009 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:50:09.009 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.478570+0000", 2023-09-23T20:50:09.009 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00013248599999999999 2023-09-23T20:50:09.010 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.010 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.010 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:50:09.010 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.478661+0000", 2023-09-23T20:50:09.010 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.1013000000000004e-05 2023-09-23T20:50:09.010 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.010 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.010 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:50:09.010 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.486788+0000", 2023-09-23T20:50:09.011 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0081275240000000006 2023-09-23T20:50:09.012 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.012 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.012 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:09.012 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.493688+0000", 2023-09-23T20:50:09.012 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0068995159999999996 2023-09-23T20:50:09.012 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.012 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.012 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:09.012 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.493704+0000", 2023-09-23T20:50:09.012 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.6441999999999999e-05 2023-09-23T20:50:09.012 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.013 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.013 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:50:09.013 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.493734+0000", 2023-09-23T20:50:09.013 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.9258999999999999e-05 2023-09-23T20:50:09.013 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.013 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.013 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:50:09.013 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.493755+0000", 2023-09-23T20:50:09.013 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.1636000000000001e-05 2023-09-23T20:50:09.013 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:50:09.013 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:50:09.013 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:50:09.014 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.014 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.014 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:102263 10.e 10:7fa4b63f:::benchmark_data_smithi018_422431_object102262:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e551)", 2023-09-23T20:50:09.014 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:47:08.130511+0000", 2023-09-23T20:50:09.014 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 180.85392804, 2023-09-23T20:50:09.014 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.028872988, 2023-09-23T20:50:09.014 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:50:09.014 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:50:09.014 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:50:09.014 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:50:09.014 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:50:09.015 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 102263 2023-09-23T20:50:09.015 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.015 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:50:09.015 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.015 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:50:09.015 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.130511+0000", 2023-09-23T20:50:09.015 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:50:09.015 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.015 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.015 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:50:09.015 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.130511+0000", 2023-09-23T20:50:09.015 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:50:09.016 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.016 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.016 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:50:09.016 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.130510+0000", 2023-09-23T20:50:09.016 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:50:09.016 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.016 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.016 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:50:09.016 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.130540+0000", 2023-09-23T20:50:09.016 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.9640000000000001e-05 2023-09-23T20:50:09.016 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.016 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.017 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:50:09.017 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.130547+0000", 2023-09-23T20:50:09.017 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.5390000000000002e-06 2023-09-23T20:50:09.017 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.017 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.017 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:50:09.017 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.130550+0000", 2023-09-23T20:50:09.017 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.9979999999999999e-06 2023-09-23T20:50:09.017 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.017 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.017 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:50:09.017 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.130601+0000", 2023-09-23T20:50:09.018 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.0937999999999999e-05 2023-09-23T20:50:09.018 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.018 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.018 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:50:09.018 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.130731+0000", 2023-09-23T20:50:09.018 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00012974899999999999 2023-09-23T20:50:09.018 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.018 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.018 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 2,4,5", 2023-09-23T20:50:09.018 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.130882+0000", 2023-09-23T20:50:09.018 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00015048199999999999 2023-09-23T20:50:09.019 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.019 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.019 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:50:09.019 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.137064+0000", 2023-09-23T20:50:09.019 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0061819739999999998 2023-09-23T20:50:09.019 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.019 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.019 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:09.019 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.137121+0000", 2023-09-23T20:50:09.019 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.7302999999999997e-05 2023-09-23T20:50:09.019 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.019 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.019 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:09.020 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.139210+0000", 2023-09-23T20:50:09.020 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.002089184 2023-09-23T20:50:09.020 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.020 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.020 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:50:09.020 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.173918+0000", 2023-09-23T20:50:09.020 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.034707835999999999 2023-09-23T20:50:09.020 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.020 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.020 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:50:09.020 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.173930+0000", 2023-09-23T20:50:09.020 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.185e-05 2023-09-23T20:50:09.021 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.021 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.021 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:50:09.021 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.159372+0000", 2023-09-23T20:50:09.021 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.98544211999999998 2023-09-23T20:50:09.021 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.021 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.021 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:50:09.021 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.159384+0000", 2023-09-23T20:50:09.021 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1737e-05 2023-09-23T20:50:09.021 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:50:09.021 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:50:09.022 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:50:09.022 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.022 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.022 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:102264 10.e 10:70bfcac0:::benchmark_data_smithi018_422431_object102263:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e551)", 2023-09-23T20:50:09.022 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:47:08.130562+0000", 2023-09-23T20:50:09.022 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 180.85387674899999, 2023-09-23T20:50:09.022 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.028857202, 2023-09-23T20:50:09.022 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:50:09.022 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:50:09.022 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:50:09.023 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:50:09.023 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:50:09.023 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 102264 2023-09-23T20:50:09.023 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.023 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:50:09.023 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.023 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:50:09.023 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.130562+0000", 2023-09-23T20:50:09.023 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:50:09.023 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.023 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.023 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:50:09.023 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.130562+0000", 2023-09-23T20:50:09.024 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:50:09.024 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.024 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.024 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:50:09.024 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.130561+0000", 2023-09-23T20:50:09.024 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:50:09.024 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.024 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.024 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:50:09.024 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.130598+0000", 2023-09-23T20:50:09.024 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.6294999999999997e-05 2023-09-23T20:50:09.024 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.025 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.025 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:50:09.025 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.130602+0000", 2023-09-23T20:50:09.025 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.5379999999999996e-06 2023-09-23T20:50:09.025 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.025 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.025 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:50:09.025 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.130605+0000", 2023-09-23T20:50:09.025 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.909e-06 2023-09-23T20:50:09.025 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.025 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.025 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:50:09.026 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.134028+0000", 2023-09-23T20:50:09.026 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0034227910000000001 2023-09-23T20:50:09.026 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.026 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.026 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:50:09.026 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.134217+0000", 2023-09-23T20:50:09.026 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00018908099999999999 2023-09-23T20:50:09.026 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.026 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.026 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 2,4,5", 2023-09-23T20:50:09.026 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.134391+0000", 2023-09-23T20:50:09.026 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00017426099999999999 2023-09-23T20:50:09.027 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.027 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.027 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:50:09.027 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.137246+0000", 2023-09-23T20:50:09.027 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0028549700000000001 2023-09-23T20:50:09.027 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.027 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.027 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:09.027 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.138047+0000", 2023-09-23T20:50:09.027 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00080048900000000004 2023-09-23T20:50:09.027 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.027 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.028 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:09.028 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.139365+0000", 2023-09-23T20:50:09.028 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.001318512 2023-09-23T20:50:09.028 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.028 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.028 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:50:09.028 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.174065+0000", 2023-09-23T20:50:09.028 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.034699636999999998 2023-09-23T20:50:09.028 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.028 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.028 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:50:09.028 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.174076+0000", 2023-09-23T20:50:09.028 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0655999999999999e-05 2023-09-23T20:50:09.029 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.029 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.029 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:50:09.029 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.159411+0000", 2023-09-23T20:50:09.029 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.98533562799999996 2023-09-23T20:50:09.029 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.029 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.029 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:50:09.029 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.159419+0000", 2023-09-23T20:50:09.029 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.9950000000000005e-06 2023-09-23T20:50:09.029 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:50:09.029 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:50:09.030 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:50:09.030 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.030 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.030 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:102265 10.d 10:b077b11a:::benchmark_data_smithi018_422431_object102264:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e551)", 2023-09-23T20:50:09.030 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:47:08.130611+0000", 2023-09-23T20:50:09.030 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 180.85382773200001, 2023-09-23T20:50:09.030 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.088474049, 2023-09-23T20:50:09.030 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:50:09.030 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:50:09.030 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:50:09.030 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:50:09.030 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:50:09.031 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 102265 2023-09-23T20:50:09.031 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.031 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:50:09.031 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.031 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:50:09.031 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.130611+0000", 2023-09-23T20:50:09.031 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:50:09.031 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.031 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.031 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:50:09.031 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.130611+0000", 2023-09-23T20:50:09.031 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:50:09.032 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.032 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.032 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:50:09.032 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.130610+0000", 2023-09-23T20:50:09.032 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:50:09.032 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.032 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.032 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:50:09.032 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.130655+0000", 2023-09-23T20:50:09.032 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.4501e-05 2023-09-23T20:50:09.032 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.032 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.032 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:50:09.033 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.130663+0000", 2023-09-23T20:50:09.033 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.1429999999999998e-06 2023-09-23T20:50:09.033 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.033 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.033 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:50:09.033 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.130666+0000", 2023-09-23T20:50:09.033 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.039e-06 2023-09-23T20:50:09.033 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.033 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.033 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:50:09.033 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.130701+0000", 2023-09-23T20:50:09.033 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.5135000000000002e-05 2023-09-23T20:50:09.034 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.034 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.034 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:50:09.034 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.130794+0000", 2023-09-23T20:50:09.034 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.31e-05 2023-09-23T20:50:09.034 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.034 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.034 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:50:09.034 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.130902+0000", 2023-09-23T20:50:09.034 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00010738699999999999 2023-09-23T20:50:09.034 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.035 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.035 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:50:09.035 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.134006+0000", 2023-09-23T20:50:09.035 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0031044319999999999 2023-09-23T20:50:09.035 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.035 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.035 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:09.035 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.143177+0000", 2023-09-23T20:50:09.035 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.009170447 2023-09-23T20:50:09.035 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.035 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.035 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:50:09.036 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.177193+0000", 2023-09-23T20:50:09.036 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.034015861000000001 2023-09-23T20:50:09.036 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.036 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.036 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:50:09.036 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.177202+0000", 2023-09-23T20:50:09.036 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.8379999999999997e-06 2023-09-23T20:50:09.036 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.036 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.036 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:50:09.036 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.218917+0000", 2023-09-23T20:50:09.036 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0417144490000001 2023-09-23T20:50:09.037 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.037 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.037 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:50:09.037 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.219085+0000", 2023-09-23T20:50:09.037 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00016819 2023-09-23T20:50:09.037 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:50:09.037 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:50:09.037 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:50:09.037 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.037 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.037 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:102284 10.0 10:0c1d13a0:::benchmark_data_smithi018_422431_object102283:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e552)", 2023-09-23T20:50:09.037 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:47:08.192917+0000", 2023-09-23T20:50:09.038 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 180.79152140400001, 2023-09-23T20:50:09.038 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1174581180000001, 2023-09-23T20:50:09.038 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:50:09.038 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:50:09.038 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:50:09.038 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:50:09.038 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:50:09.038 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 102284 2023-09-23T20:50:09.038 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.038 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:50:09.038 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.039 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:50:09.039 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.192917+0000", 2023-09-23T20:50:09.039 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:50:09.039 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.039 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.039 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:50:09.039 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.192917+0000", 2023-09-23T20:50:09.039 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:50:09.039 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.039 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.039 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:50:09.039 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.192916+0000", 2023-09-23T20:50:09.039 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:50:09.040 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.040 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.040 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:50:09.040 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.192958+0000", 2023-09-23T20:50:09.040 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.1974999999999997e-05 2023-09-23T20:50:09.040 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.040 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.040 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:50:09.040 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.193237+0000", 2023-09-23T20:50:09.040 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00027918399999999998 2023-09-23T20:50:09.040 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.040 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.041 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:50:09.041 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.193247+0000", 2023-09-23T20:50:09.041 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0015e-05 2023-09-23T20:50:09.041 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.041 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.041 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:50:09.041 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.193334+0000", 2023-09-23T20:50:09.041 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.6867999999999996e-05 2023-09-23T20:50:09.041 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.041 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.041 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:50:09.041 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.193351+0000", 2023-09-23T20:50:09.042 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.6545000000000002e-05 2023-09-23T20:50:09.042 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.042 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.042 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:50:09.042 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.216597+0000", 2023-09-23T20:50:09.042 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0232465260000001 2023-09-23T20:50:09.042 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.042 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.042 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:50:09.042 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.216764+0000", 2023-09-23T20:50:09.042 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000166937 2023-09-23T20:50:09.043 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.043 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.043 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 1,4", 2023-09-23T20:50:09.043 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.216936+0000", 2023-09-23T20:50:09.043 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00017148099999999999 2023-09-23T20:50:09.043 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.043 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.043 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:50:09.043 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.309749+0000", 2023-09-23T20:50:09.043 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.092813393999999994 2023-09-23T20:50:09.043 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.043 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.044 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:09.044 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.309824+0000", 2023-09-23T20:50:09.044 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.5284999999999995e-05 2023-09-23T20:50:09.044 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.044 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.044 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:09.044 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.309902+0000", 2023-09-23T20:50:09.044 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.8061999999999999e-05 2023-09-23T20:50:09.044 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.044 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.044 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:50:09.044 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.309995+0000", 2023-09-23T20:50:09.045 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.2719000000000005e-05 2023-09-23T20:50:09.045 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.045 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.045 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:50:09.045 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.310375+0000", 2023-09-23T20:50:09.045 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00038043700000000002 2023-09-23T20:50:09.045 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:50:09.045 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:50:09.045 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:50:09.045 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.045 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.045 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:102291 10.0 10:0574a143:::benchmark_data_smithi018_422431_object102290:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e552)", 2023-09-23T20:50:09.046 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:47:08.213037+0000", 2023-09-23T20:50:09.046 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 180.771401492, 2023-09-23T20:50:09.046 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.097677486, 2023-09-23T20:50:09.046 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:50:09.046 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:50:09.046 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:50:09.046 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:50:09.046 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:50:09.046 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 102291 2023-09-23T20:50:09.046 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.046 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:50:09.046 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.047 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:50:09.047 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.213037+0000", 2023-09-23T20:50:09.047 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:50:09.047 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.047 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.047 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:50:09.047 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.213037+0000", 2023-09-23T20:50:09.047 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:50:09.047 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.047 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.047 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:50:09.047 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.213035+0000", 2023-09-23T20:50:09.048 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999981 2023-09-23T20:50:09.048 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.048 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.048 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:50:09.048 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.213078+0000", 2023-09-23T20:50:09.048 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.3106999999999999e-05 2023-09-23T20:50:09.048 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.048 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.048 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:50:09.048 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.213095+0000", 2023-09-23T20:50:09.048 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.6200000000000001e-05 2023-09-23T20:50:09.048 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.049 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.049 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:50:09.049 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.213102+0000", 2023-09-23T20:50:09.049 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.2779999999999996e-06 2023-09-23T20:50:09.049 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.049 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.049 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:50:09.049 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.213160+0000", 2023-09-23T20:50:09.049 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.838e-05 2023-09-23T20:50:09.049 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.049 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.050 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:50:09.050 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.213176+0000", 2023-09-23T20:50:09.050 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.5404999999999999e-05 2023-09-23T20:50:09.050 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.050 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.050 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:50:09.050 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.220512+0000", 2023-09-23T20:50:09.050 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.00733669 2023-09-23T20:50:09.050 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.050 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.050 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:50:09.050 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.220642+0000", 2023-09-23T20:50:09.051 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00012919 2023-09-23T20:50:09.051 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.051 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.051 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 1,4", 2023-09-23T20:50:09.051 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.220797+0000", 2023-09-23T20:50:09.051 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00015535599999999999 2023-09-23T20:50:09.051 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.051 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.051 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:50:09.051 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.309757+0000", 2023-09-23T20:50:09.051 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.088960045000000001 2023-09-23T20:50:09.051 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.052 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.052 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:09.052 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.309865+0000", 2023-09-23T20:50:09.052 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000108051 2023-09-23T20:50:09.052 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.052 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.052 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:09.052 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.310430+0000", 2023-09-23T20:50:09.052 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000565126 2023-09-23T20:50:09.052 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.052 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.052 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:50:09.053 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.310482+0000", 2023-09-23T20:50:09.053 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.1752999999999997e-05 2023-09-23T20:50:09.053 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.053 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.053 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:50:09.053 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.310715+0000", 2023-09-23T20:50:09.053 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00023280499999999999 2023-09-23T20:50:09.053 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:50:09.053 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:50:09.053 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:50:09.053 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.054 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.054 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:102300 10.d 10:b3326d1e:::benchmark_data_smithi018_422431_object102299:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e552)", 2023-09-23T20:50:09.054 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:47:08.260136+0000", 2023-09-23T20:50:09.054 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 180.72430242499999, 2023-09-23T20:50:09.054 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0960319730000001, 2023-09-23T20:50:09.054 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:50:09.054 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:50:09.054 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:50:09.054 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:50:09.054 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:50:09.054 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 102300 2023-09-23T20:50:09.054 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.055 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:50:09.055 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.055 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:50:09.055 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.260136+0000", 2023-09-23T20:50:09.055 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:50:09.055 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.055 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.055 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:50:09.055 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.260136+0000", 2023-09-23T20:50:09.055 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:50:09.055 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.056 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.056 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:50:09.056 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.260135+0000", 2023-09-23T20:50:09.056 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:50:09.056 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.056 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.056 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:50:09.056 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.260169+0000", 2023-09-23T20:50:09.056 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.4019000000000001e-05 2023-09-23T20:50:09.056 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.056 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.057 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:50:09.057 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.260183+0000", 2023-09-23T20:50:09.057 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.4387e-05 2023-09-23T20:50:09.057 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.057 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.057 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:50:09.057 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.260189+0000", 2023-09-23T20:50:09.057 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.0990000000000004e-06 2023-09-23T20:50:09.057 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.057 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.057 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:50:09.057 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.260275+0000", 2023-09-23T20:50:09.058 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.5556000000000006e-05 2023-09-23T20:50:09.058 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.058 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.058 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:50:09.058 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.260285+0000", 2023-09-23T20:50:09.058 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.9189999999999994e-06 2023-09-23T20:50:09.058 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.058 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.058 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:50:09.058 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.219112+0000", 2023-09-23T20:50:09.058 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.95882748600000001 2023-09-23T20:50:09.058 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.059 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.059 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:50:09.059 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.219285+0000", 2023-09-23T20:50:09.059 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000173299 2023-09-23T20:50:09.059 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.059 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.059 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,4", 2023-09-23T20:50:09.059 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.219498+0000", 2023-09-23T20:50:09.059 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00021284799999999999 2023-09-23T20:50:09.059 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.059 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.059 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:09.060 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.310145+0000", 2023-09-23T20:50:09.060 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.090646550000000006 2023-09-23T20:50:09.060 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.060 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.060 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:09.060 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.310184+0000", 2023-09-23T20:50:09.060 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.9232e-05 2023-09-23T20:50:09.060 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.060 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.060 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:50:09.060 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.355659+0000", 2023-09-23T20:50:09.060 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.045474674 2023-09-23T20:50:09.061 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.061 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.061 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:50:09.061 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.355721+0000", 2023-09-23T20:50:09.061 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.2157999999999997e-05 2023-09-23T20:50:09.061 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.061 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.061 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:50:09.061 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.356168+0000", 2023-09-23T20:50:09.061 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000447305 2023-09-23T20:50:09.061 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:50:09.061 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:50:09.062 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:50:09.062 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.062 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.062 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:103733 10.d 10:b457bb97:::benchmark_data_smithi018_422431_object103732:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e557)", 2023-09-23T20:50:09.062 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:47:12.654166+0000", 2023-09-23T20:50:09.062 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 176.33027221699999, 2023-09-23T20:50:09.062 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0141292129999999, 2023-09-23T20:50:09.062 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:50:09.062 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:50:09.062 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:50:09.062 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:50:09.062 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:50:09.063 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 103733 2023-09-23T20:50:09.063 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.063 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:50:09.063 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.063 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:50:09.063 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:12.654166+0000", 2023-09-23T20:50:09.063 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:50:09.063 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.063 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.063 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:50:09.063 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:12.654166+0000", 2023-09-23T20:50:09.063 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:50:09.064 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.064 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.064 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:50:09.064 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:12.654165+0000", 2023-09-23T20:50:09.064 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:50:09.064 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.064 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.064 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:50:09.064 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:12.654221+0000", 2023-09-23T20:50:09.064 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.6150999999999998e-05 2023-09-23T20:50:09.064 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.064 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.065 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:50:09.065 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:12.654236+0000", 2023-09-23T20:50:09.065 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.4783000000000001e-05 2023-09-23T20:50:09.065 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.065 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.065 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:50:09.065 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:12.654243+0000", 2023-09-23T20:50:09.065 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.8680000000000001e-06 2023-09-23T20:50:09.065 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.065 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.065 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:50:09.065 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:12.654316+0000", 2023-09-23T20:50:09.066 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.2848000000000005e-05 2023-09-23T20:50:09.066 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.066 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.066 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:50:09.066 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:12.654335+0000", 2023-09-23T20:50:09.066 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.9057999999999999e-05 2023-09-23T20:50:09.066 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.066 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.066 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:50:09.066 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:13.630346+0000", 2023-09-23T20:50:09.066 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.97601110099999999 2023-09-23T20:50:09.066 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.067 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.067 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:50:09.067 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:13.630458+0000", 2023-09-23T20:50:09.067 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00011244 2023-09-23T20:50:09.067 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.067 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.067 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,4", 2023-09-23T20:50:09.067 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:13.630554+0000", 2023-09-23T20:50:09.067 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.5334999999999994e-05 2023-09-23T20:50:09.067 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.067 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.067 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:50:09.068 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:13.642112+0000", 2023-09-23T20:50:09.068 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.011558034999999999 2023-09-23T20:50:09.068 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.068 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.068 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:09.068 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:13.668168+0000", 2023-09-23T20:50:09.068 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.026056691 2023-09-23T20:50:09.068 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.068 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.068 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:09.068 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:13.668225+0000", 2023-09-23T20:50:09.069 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.6598000000000003e-05 2023-09-23T20:50:09.069 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.069 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.069 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:50:09.069 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:13.668267+0000", 2023-09-23T20:50:09.069 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.2101000000000002e-05 2023-09-23T20:50:09.069 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.069 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.069 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:50:09.069 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:13.668296+0000", 2023-09-23T20:50:09.069 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.8530999999999998e-05 2023-09-23T20:50:09.069 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:50:09.070 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:50:09.070 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:50:09.070 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.070 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.070 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:159045 10.f 10:f817bccc:::benchmark_data_smithi018_422431_object54549:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e577)", 2023-09-23T20:50:09.070 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:48:22.940491+0000", 2023-09-23T20:50:09.070 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 106.043947758, 2023-09-23T20:50:09.070 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.257405952, 2023-09-23T20:50:09.070 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:50:09.070 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:50:09.070 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:50:09.070 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:50:09.071 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:50:09.071 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 159045 2023-09-23T20:50:09.071 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.071 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:50:09.071 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.071 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:50:09.071 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.940491+0000", 2023-09-23T20:50:09.071 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:50:09.071 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.071 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.071 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:50:09.071 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.940491+0000", 2023-09-23T20:50:09.072 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:50:09.072 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.072 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.072 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:50:09.072 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.940490+0000", 2023-09-23T20:50:09.072 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:50:09.072 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.072 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.072 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:50:09.072 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.940494+0000", 2023-09-23T20:50:09.072 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.9580000000000001e-06 2023-09-23T20:50:09.073 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.073 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.073 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:50:09.073 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.940505+0000", 2023-09-23T20:50:09.073 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0905e-05 2023-09-23T20:50:09.073 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.073 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.073 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:50:09.073 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.940510+0000", 2023-09-23T20:50:09.073 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.5879999999999999e-06 2023-09-23T20:50:09.073 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.073 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.074 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:50:09.074 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.940558+0000", 2023-09-23T20:50:09.074 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.8007000000000003e-05 2023-09-23T20:50:09.074 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.074 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.074 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:50:09.074 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.940564+0000", 2023-09-23T20:50:09.074 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.2099999999999998e-06 2023-09-23T20:50:09.074 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.074 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.074 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:50:09.074 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:23.909048+0000", 2023-09-23T20:50:09.075 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.96848468799999998 2023-09-23T20:50:09.075 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.075 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.075 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:50:09.075 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:23.909060+0000", 2023-09-23T20:50:09.075 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1279e-05 2023-09-23T20:50:09.075 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.075 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.075 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:50:09.075 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:25.160634+0000", 2023-09-23T20:50:09.075 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.251573933 2023-09-23T20:50:09.076 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.076 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.076 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:50:09.076 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:25.160777+0000", 2023-09-23T20:50:09.076 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000142893 2023-09-23T20:50:09.076 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.076 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.076 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 2,3,4", 2023-09-23T20:50:09.076 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:25.161399+0000", 2023-09-23T20:50:09.076 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00062231400000000001 2023-09-23T20:50:09.076 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.076 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.077 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:09.077 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:25.171100+0000", 2023-09-23T20:50:09.077 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0097009049999999993 2023-09-23T20:50:09.077 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.077 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.077 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:09.077 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:25.183171+0000", 2023-09-23T20:50:09.077 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.012071236000000001 2023-09-23T20:50:09.077 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.077 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.077 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:50:09.077 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:25.185581+0000", 2023-09-23T20:50:09.078 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0024096899999999999 2023-09-23T20:50:09.078 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.078 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.078 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:09.078 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:25.197727+0000", 2023-09-23T20:50:09.078 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.012145958 2023-09-23T20:50:09.078 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.078 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.078 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:50:09.078 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:25.197780+0000", 2023-09-23T20:50:09.078 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.3816000000000001e-05 2023-09-23T20:50:09.078 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.079 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.079 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:50:09.079 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:25.197897+0000", 2023-09-23T20:50:09.079 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00011640400000000001 2023-09-23T20:50:09.079 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:50:09.079 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:50:09.079 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:50:09.079 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.079 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.079 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:181036 10.12 10.8f5c0992 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e602)", 2023-09-23T20:50:09.079 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:49:00.873954+0000", 2023-09-23T20:50:09.080 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 68.110484693000004, 2023-09-23T20:50:09.080 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.81278060500000004, 2023-09-23T20:50:09.080 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:50:09.080 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:50:09.080 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:50:09.080 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:50:09.080 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:50:09.080 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 181036 2023-09-23T20:50:09.080 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.080 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:50:09.080 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.080 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:50:09.080 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:00.873954+0000", 2023-09-23T20:50:09.081 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:50:09.081 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.081 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.081 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:50:09.081 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:00.873954+0000", 2023-09-23T20:50:09.081 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:50:09.081 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.081 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.081 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:50:09.081 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:00.873953+0000", 2023-09-23T20:50:09.081 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:50:09.082 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.082 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.082 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:50:09.082 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:00.873957+0000", 2023-09-23T20:50:09.082 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.5499999999999996e-06 2023-09-23T20:50:09.082 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.082 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.082 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:50:09.082 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:00.873966+0000", 2023-09-23T20:50:09.082 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.7909999999999993e-06 2023-09-23T20:50:09.082 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.082 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.083 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:50:09.083 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:00.873971+0000", 2023-09-23T20:50:09.083 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.2299999999999999e-06 2023-09-23T20:50:09.083 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.083 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.083 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:50:09.083 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:00.874028+0000", 2023-09-23T20:50:09.083 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.6919000000000002e-05 2023-09-23T20:50:09.083 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.083 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.083 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:50:09.083 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:00.874036+0000", 2023-09-23T20:50:09.084 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.3930000000000002e-06 2023-09-23T20:50:09.084 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.084 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.084 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:50:09.084 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:01.686723+0000", 2023-09-23T20:50:09.084 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.81268693700000005 2023-09-23T20:50:09.084 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.084 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.084 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:50:09.084 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:01.686735+0000", 2023-09-23T20:50:09.084 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1997000000000001e-05 2023-09-23T20:50:09.084 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:50:09.085 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:50:09.085 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:50:09.085 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.085 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.085 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:196645 10.16 10.8b6c8596 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e622)", 2023-09-23T20:50:09.085 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:49:27.061969+0000", 2023-09-23T20:50:09.085 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 41.922469530000001, 2023-09-23T20:50:09.085 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.91010735200000004, 2023-09-23T20:50:09.085 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:50:09.085 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:50:09.085 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:50:09.085 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:50:09.086 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:50:09.086 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 196645 2023-09-23T20:50:09.086 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.086 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:50:09.086 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.086 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:50:09.086 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:27.061969+0000", 2023-09-23T20:50:09.086 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:50:09.086 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.086 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.086 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:50:09.087 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:27.061969+0000", 2023-09-23T20:50:09.087 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:50:09.087 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.087 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.087 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:50:09.087 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:27.061969+0000", 2023-09-23T20:50:09.087 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:50:09.087 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.087 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.087 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:50:09.087 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:27.061971+0000", 2023-09-23T20:50:09.088 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.4940000000000002e-06 2023-09-23T20:50:09.088 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.088 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.088 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:50:09.088 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:27.061983+0000", 2023-09-23T20:50:09.088 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1895999999999999e-05 2023-09-23T20:50:09.088 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.088 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.088 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:50:09.088 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:27.061989+0000", 2023-09-23T20:50:09.088 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.3119999999999999e-06 2023-09-23T20:50:09.088 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.089 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.089 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:50:09.089 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:27.062036+0000", 2023-09-23T20:50:09.089 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.6576999999999998e-05 2023-09-23T20:50:09.089 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.089 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.089 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:50:09.089 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:27.062041+0000", 2023-09-23T20:50:09.089 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.4090000000000002e-06 2023-09-23T20:50:09.089 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.089 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.089 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:50:09.090 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:27.972063+0000", 2023-09-23T20:50:09.090 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.910021262 2023-09-23T20:50:09.090 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.090 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.090 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:50:09.090 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:27.972077+0000", 2023-09-23T20:50:09.090 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.3801999999999999e-05 2023-09-23T20:50:09.090 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:50:09.090 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:50:09.090 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:50:09.090 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.091 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.091 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:196677 10.16 10.1917f5f6 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e622)", 2023-09-23T20:50:09.091 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:49:27.102907+0000", 2023-09-23T20:50:09.091 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 41.881531471999999, 2023-09-23T20:50:09.091 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.86920190600000002, 2023-09-23T20:50:09.091 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:50:09.091 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:50:09.091 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:50:09.091 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:50:09.091 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:50:09.091 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 196677 2023-09-23T20:50:09.091 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.092 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:50:09.092 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.092 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:50:09.092 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:27.102907+0000", 2023-09-23T20:50:09.092 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:50:09.092 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.092 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.092 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:50:09.092 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:27.102907+0000", 2023-09-23T20:50:09.092 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:50:09.092 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.092 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.093 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:50:09.093 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:27.102907+0000", 2023-09-23T20:50:09.093 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:50:09.093 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.093 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.093 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:50:09.093 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:27.102909+0000", 2023-09-23T20:50:09.093 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.1169999999999998e-06 2023-09-23T20:50:09.093 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.093 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.093 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:50:09.093 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:27.102914+0000", 2023-09-23T20:50:09.094 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.3859999999999998e-06 2023-09-23T20:50:09.094 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.094 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.094 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:50:09.094 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:27.102918+0000", 2023-09-23T20:50:09.094 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.6049999999999998e-06 2023-09-23T20:50:09.094 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.094 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.094 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:50:09.094 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:27.102962+0000", 2023-09-23T20:50:09.094 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.3791000000000001e-05 2023-09-23T20:50:09.095 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.095 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.095 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:50:09.095 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:27.102969+0000", 2023-09-23T20:50:09.095 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.1169999999999998e-06 2023-09-23T20:50:09.095 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.095 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.095 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:50:09.095 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:27.972098+0000", 2023-09-23T20:50:09.095 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.86912921200000004 2023-09-23T20:50:09.095 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.095 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.096 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:50:09.096 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:27.972109+0000", 2023-09-23T20:50:09.096 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0988999999999999e-05 2023-09-23T20:50:09.096 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:50:09.096 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:50:09.096 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:50:09.096 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.096 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.096 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:196719 10.16 10.b6000516 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e622)", 2023-09-23T20:50:09.096 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:49:27.142134+0000", 2023-09-23T20:50:09.096 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 41.84230436, 2023-09-23T20:50:09.096 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.83000051699999999, 2023-09-23T20:50:09.097 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:50:09.097 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:50:09.097 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:50:09.097 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:50:09.097 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:50:09.097 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 196719 2023-09-23T20:50:09.097 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.097 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:50:09.097 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.097 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:50:09.097 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:27.142134+0000", 2023-09-23T20:50:09.097 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:50:09.098 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.098 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.098 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:50:09.098 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:27.142134+0000", 2023-09-23T20:50:09.098 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:50:09.098 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.098 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.098 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:50:09.098 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:27.142134+0000", 2023-09-23T20:50:09.098 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:50:09.098 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.098 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.099 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:50:09.099 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:27.142136+0000", 2023-09-23T20:50:09.099 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.5110000000000002e-06 2023-09-23T20:50:09.099 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.099 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.099 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:50:09.099 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:27.142148+0000", 2023-09-23T20:50:09.099 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.131e-05 2023-09-23T20:50:09.099 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.099 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.099 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:50:09.099 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:27.142151+0000", 2023-09-23T20:50:09.100 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.1370000000000002e-06 2023-09-23T20:50:09.100 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.100 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.100 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:50:09.100 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:27.142193+0000", 2023-09-23T20:50:09.100 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.2562999999999999e-05 2023-09-23T20:50:09.100 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.100 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.100 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:50:09.100 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:27.142201+0000", 2023-09-23T20:50:09.100 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.2180000000000002e-06 2023-09-23T20:50:09.100 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.101 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.101 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:50:09.101 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:27.972126+0000", 2023-09-23T20:50:09.101 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.82992553199999997 2023-09-23T20:50:09.101 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.101 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.101 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:50:09.101 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:27.972135+0000", 2023-09-23T20:50:09.101 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.6649999999999992e-06 2023-09-23T20:50:09.101 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:50:09.101 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:50:09.101 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:50:09.102 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.102 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.102 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:196729 10.16 10.90fdb716 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e622)", 2023-09-23T20:50:09.102 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:49:27.154937+0000", 2023-09-23T20:50:09.102 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 41.829502109000003, 2023-09-23T20:50:09.102 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.81722424999999999, 2023-09-23T20:50:09.102 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:50:09.102 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:50:09.102 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:50:09.102 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:50:09.102 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:50:09.102 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 196729 2023-09-23T20:50:09.103 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.103 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:50:09.103 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.103 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:50:09.103 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:27.154937+0000", 2023-09-23T20:50:09.103 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:50:09.103 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.103 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.103 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:50:09.103 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:27.154937+0000", 2023-09-23T20:50:09.103 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:50:09.103 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.104 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.104 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:50:09.104 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:27.154936+0000", 2023-09-23T20:50:09.104 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:50:09.104 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.104 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.104 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:50:09.104 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:27.154940+0000", 2023-09-23T20:50:09.104 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.4290000000000003e-06 2023-09-23T20:50:09.104 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.104 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.104 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:50:09.105 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:27.154953+0000", 2023-09-23T20:50:09.105 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.305e-05 2023-09-23T20:50:09.105 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.105 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.105 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:50:09.105 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:27.154959+0000", 2023-09-23T20:50:09.105 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.8510000000000004e-06 2023-09-23T20:50:09.105 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.105 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.105 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:50:09.105 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:27.155375+0000", 2023-09-23T20:50:09.105 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00041597900000000002 2023-09-23T20:50:09.106 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.106 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.106 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:50:09.106 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:27.155391+0000", 2023-09-23T20:50:09.106 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.5727999999999999e-05 2023-09-23T20:50:09.106 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.106 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.106 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:50:09.106 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:27.972152+0000", 2023-09-23T20:50:09.106 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.81676116600000004 2023-09-23T20:50:09.106 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.106 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.107 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:50:09.107 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:27.972161+0000", 2023-09-23T20:50:09.107 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.1239999999999994e-06 2023-09-23T20:50:09.107 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:50:09.107 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:50:09.107 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:50:09.107 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.107 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.107 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:197482 10.16 10:6f86c9d6:::benchmark_data_smithi018_422431_object92986:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e625)", 2023-09-23T20:50:09.107 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:49:30.033426+0000", 2023-09-23T20:50:09.107 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 38.951012789000004, 2023-09-23T20:50:09.107 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.983929371, 2023-09-23T20:50:09.108 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:50:09.108 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:50:09.108 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:50:09.108 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:50:09.108 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:50:09.108 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 197482 2023-09-23T20:50:09.108 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.108 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:50:09.108 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.108 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:50:09.108 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:30.033426+0000", 2023-09-23T20:50:09.108 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:50:09.109 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.109 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.109 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:50:09.109 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:30.033426+0000", 2023-09-23T20:50:09.109 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:50:09.109 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.109 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.109 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:50:09.109 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:30.033425+0000", 2023-09-23T20:50:09.109 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:50:09.109 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.109 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.110 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:50:09.110 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:30.033428+0000", 2023-09-23T20:50:09.110 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.717e-06 2023-09-23T20:50:09.110 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.110 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.110 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:50:09.110 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:30.033440+0000", 2023-09-23T20:50:09.110 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1737e-05 2023-09-23T20:50:09.110 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.110 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.110 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:50:09.110 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:30.033444+0000", 2023-09-23T20:50:09.111 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.1060000000000002e-06 2023-09-23T20:50:09.111 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.111 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.111 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:50:09.111 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:30.033483+0000", 2023-09-23T20:50:09.111 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.9258000000000003e-05 2023-09-23T20:50:09.111 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.111 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.111 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:50:09.111 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:30.033490+0000", 2023-09-23T20:50:09.111 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.1359999999999999e-06 2023-09-23T20:50:09.111 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.112 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.112 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:50:09.112 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:31.011143+0000", 2023-09-23T20:50:09.112 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.97765219999999997 2023-09-23T20:50:09.112 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.112 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.112 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:50:09.112 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:31.011287+0000", 2023-09-23T20:50:09.112 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00014409500000000001 2023-09-23T20:50:09.112 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.112 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.112 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 1,5", 2023-09-23T20:50:09.113 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:31.011543+0000", 2023-09-23T20:50:09.113 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00025618400000000001 2023-09-23T20:50:09.113 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.113 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.113 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:50:09.113 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:31.015182+0000", 2023-09-23T20:50:09.113 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0036391539999999999 2023-09-23T20:50:09.113 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.113 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.113 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:09.113 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:31.017232+0000", 2023-09-23T20:50:09.113 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0020494760000000002 2023-09-23T20:50:09.114 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.114 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.114 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:09.114 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:31.017265+0000", 2023-09-23T20:50:09.114 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.3665999999999999e-05 2023-09-23T20:50:09.114 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.114 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.114 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:50:09.114 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:31.017311+0000", 2023-09-23T20:50:09.114 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.5797999999999998e-05 2023-09-23T20:50:09.114 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.114 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.115 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:50:09.115 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:31.017355+0000", 2023-09-23T20:50:09.115 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.4283999999999998e-05 2023-09-23T20:50:09.115 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:50:09.115 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:50:09.115 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:50:09.115 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.115 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.115 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:197491 10.16 10:6bb90d38:::benchmark_data_smithi018_422431_object92995:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e625)", 2023-09-23T20:50:09.115 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:49:30.040972+0000", 2023-09-23T20:50:09.115 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 38.943467091000002, 2023-09-23T20:50:09.116 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.97653193900000002, 2023-09-23T20:50:09.116 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:50:09.116 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:50:09.116 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:50:09.116 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:50:09.116 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:50:09.116 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 197491 2023-09-23T20:50:09.116 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.116 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:50:09.116 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.116 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:50:09.116 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:30.040972+0000", 2023-09-23T20:50:09.116 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:50:09.117 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.117 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.117 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:50:09.117 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:30.040972+0000", 2023-09-23T20:50:09.117 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:50:09.117 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.117 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.117 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:50:09.117 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:30.040971+0000", 2023-09-23T20:50:09.117 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:50:09.117 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.118 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.118 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:50:09.118 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:30.040976+0000", 2023-09-23T20:50:09.118 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.6469999999999997e-06 2023-09-23T20:50:09.118 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.118 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.118 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:50:09.118 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:30.040987+0000", 2023-09-23T20:50:09.118 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1182999999999999e-05 2023-09-23T20:50:09.118 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.118 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.118 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:50:09.119 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:30.040993+0000", 2023-09-23T20:50:09.119 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.3049999999999998e-06 2023-09-23T20:50:09.119 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.119 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.119 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:50:09.119 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:30.041034+0000", 2023-09-23T20:50:09.119 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.0865999999999998e-05 2023-09-23T20:50:09.119 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.119 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.119 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:50:09.119 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:30.041043+0000", 2023-09-23T20:50:09.119 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.5580000000000001e-06 2023-09-23T20:50:09.120 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.120 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.120 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:50:09.120 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:31.012597+0000", 2023-09-23T20:50:09.120 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.97155400199999997 2023-09-23T20:50:09.120 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.120 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.120 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:50:09.120 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:31.012707+0000", 2023-09-23T20:50:09.120 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00011072499999999999 2023-09-23T20:50:09.120 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.120 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.121 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 1,5", 2023-09-23T20:50:09.121 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:31.012887+0000", 2023-09-23T20:50:09.121 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00017996900000000001 2023-09-23T20:50:09.121 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.121 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.121 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:50:09.121 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:31.016913+0000", 2023-09-23T20:50:09.121 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0040260560000000001 2023-09-23T20:50:09.121 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.121 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.121 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:09.121 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:31.017414+0000", 2023-09-23T20:50:09.121 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00050053200000000004 2023-09-23T20:50:09.122 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.122 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.122 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:09.122 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:31.017459+0000", 2023-09-23T20:50:09.122 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.5037000000000003e-05 2023-09-23T20:50:09.122 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.122 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.122 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:50:09.122 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:31.017484+0000", 2023-09-23T20:50:09.122 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.5559e-05 2023-09-23T20:50:09.122 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.123 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.123 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:50:09.123 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:31.017504+0000", 2023-09-23T20:50:09.123 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.9170000000000001e-05 2023-09-23T20:50:09.123 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:50:09.123 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:50:09.123 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:50:09.123 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.123 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.123 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:197532 10.16 10:68d215d5:::benchmark_data_smithi018_422431_object93036:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e625)", 2023-09-23T20:50:09.123 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:49:30.092702+0000", 2023-09-23T20:50:09.124 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 38.891736350000002, 2023-09-23T20:50:09.124 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.92504978999999998, 2023-09-23T20:50:09.124 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:50:09.124 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:50:09.124 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:50:09.124 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:50:09.124 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:50:09.124 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 197532 2023-09-23T20:50:09.124 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.124 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:50:09.124 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.124 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:50:09.125 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:30.092702+0000", 2023-09-23T20:50:09.125 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:50:09.125 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.125 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.125 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:50:09.125 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:30.092702+0000", 2023-09-23T20:50:09.125 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:50:09.125 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.125 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.125 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:50:09.125 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:30.092702+0000", 2023-09-23T20:50:09.125 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:50:09.126 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.126 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.126 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:50:09.126 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:30.092705+0000", 2023-09-23T20:50:09.126 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.4379999999999999e-06 2023-09-23T20:50:09.126 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.126 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.126 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:50:09.126 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:30.092713+0000", 2023-09-23T20:50:09.126 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.3159999999999997e-06 2023-09-23T20:50:09.126 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.126 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.127 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:50:09.127 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:30.092717+0000", 2023-09-23T20:50:09.127 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.6440000000000002e-06 2023-09-23T20:50:09.127 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.127 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.127 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:50:09.127 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:30.092756+0000", 2023-09-23T20:50:09.127 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.9051999999999998e-05 2023-09-23T20:50:09.127 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.127 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.127 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:50:09.128 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:30.092763+0000", 2023-09-23T20:50:09.128 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.1919999999999999e-06 2023-09-23T20:50:09.128 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.128 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.128 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:50:09.128 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:31.013901+0000", 2023-09-23T20:50:09.128 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.92113773099999996 2023-09-23T20:50:09.128 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.128 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.128 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:50:09.128 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:31.014021+0000", 2023-09-23T20:50:09.128 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000119795 2023-09-23T20:50:09.128 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.129 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.129 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 1,5", 2023-09-23T20:50:09.129 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:31.014210+0000", 2023-09-23T20:50:09.129 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00018928400000000001 2023-09-23T20:50:09.129 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.129 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.129 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:50:09.129 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:31.017203+0000", 2023-09-23T20:50:09.129 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.002993276 2023-09-23T20:50:09.129 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.129 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.129 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:09.130 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:31.017606+0000", 2023-09-23T20:50:09.130 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00040277599999999998 2023-09-23T20:50:09.130 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.130 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.130 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:09.130 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:31.017695+0000", 2023-09-23T20:50:09.130 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.8819999999999993e-05 2023-09-23T20:50:09.130 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.130 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.130 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:50:09.130 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:31.017722+0000", 2023-09-23T20:50:09.130 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.7118e-05 2023-09-23T20:50:09.131 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.131 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.131 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:50:09.131 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:31.017752+0000", 2023-09-23T20:50:09.131 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.0105999999999999e-05 2023-09-23T20:50:09.131 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:50:09.131 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:50:09.131 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:50:09.131 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.131 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.131 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:197567 10.16 10:6a554823:::benchmark_data_smithi018_422431_object93071:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e625)", 2023-09-23T20:50:09.131 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:49:30.199348+0000", 2023-09-23T20:50:09.132 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 38.785091006000002, 2023-09-23T20:50:09.132 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.81852593600000001, 2023-09-23T20:50:09.132 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:50:09.132 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:50:09.132 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:50:09.132 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:50:09.132 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:50:09.132 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 197567 2023-09-23T20:50:09.132 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.132 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:50:09.132 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.132 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:50:09.133 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:30.199348+0000", 2023-09-23T20:50:09.133 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:50:09.133 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.133 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.133 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:50:09.133 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:30.199348+0000", 2023-09-23T20:50:09.133 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:50:09.133 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.133 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.133 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:50:09.133 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:30.199347+0000", 2023-09-23T20:50:09.133 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:50:09.134 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.134 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.134 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:50:09.134 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:30.199351+0000", 2023-09-23T20:50:09.134 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.1629999999999998e-06 2023-09-23T20:50:09.134 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.134 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.134 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:50:09.134 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:30.199361+0000", 2023-09-23T20:50:09.134 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.5719999999999992e-06 2023-09-23T20:50:09.134 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.134 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.135 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:50:09.135 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:30.199365+0000", 2023-09-23T20:50:09.135 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.3939999999999998e-06 2023-09-23T20:50:09.135 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.135 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.135 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:50:09.135 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:30.199415+0000", 2023-09-23T20:50:09.135 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.9792e-05 2023-09-23T20:50:09.135 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.135 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.135 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:50:09.136 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:30.199425+0000", 2023-09-23T20:50:09.136 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0128e-05 2023-09-23T20:50:09.136 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.136 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.136 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:50:09.136 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:31.015227+0000", 2023-09-23T20:50:09.136 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.815802374 2023-09-23T20:50:09.136 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.136 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.136 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:50:09.136 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:31.015372+0000", 2023-09-23T20:50:09.136 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00014513800000000001 2023-09-23T20:50:09.137 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.137 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.137 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 1,5", 2023-09-23T20:50:09.137 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:31.015601+0000", 2023-09-23T20:50:09.137 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000228777 2023-09-23T20:50:09.137 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.137 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.137 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:50:09.137 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:31.017237+0000", 2023-09-23T20:50:09.137 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0016358830000000001 2023-09-23T20:50:09.137 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.137 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.137 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:09.138 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:31.017657+0000", 2023-09-23T20:50:09.138 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00041998300000000002 2023-09-23T20:50:09.138 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.138 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.138 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:09.138 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:31.017803+0000", 2023-09-23T20:50:09.138 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00014541399999999999 2023-09-23T20:50:09.138 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.138 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.138 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:50:09.138 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:31.017848+0000", 2023-09-23T20:50:09.138 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.5249e-05 2023-09-23T20:50:09.139 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:09.139 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:09.139 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:50:09.139 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:31.017874+0000", 2023-09-23T20:50:09.139 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.5839000000000001e-05 2023-09-23T20:50:09.139 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:50:09.139 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:50:09.139 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:50:09.139 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:50:09.139 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:50:09.139 INFO:teuthology.orchestra.run.smithi149.stdout:} 2023-09-23T20:50:09.140 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:50:09.141 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:50:09.141 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:50:09.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:09 smithi018 ceph-mon[179647]: failed to encode map e647 with expected crc 2023-09-23T20:50:09.142 INFO:tasks.workunit.client.0.smithi018.stdout:[----------] 70 tests from TestClsRbd (6344 ms total) 2023-09-23T20:50:09.142 INFO:tasks.workunit.client.0.smithi018.stdout: 2023-09-23T20:50:09.142 INFO:tasks.workunit.client.0.smithi018.stdout:[----------] Global test environment tear-down 2023-09-23T20:50:09.142 INFO:tasks.workunit.client.0.smithi018.stdout:[==========] 70 tests from 1 test suite ran. (9360 ms total) 2023-09-23T20:50:09.142 INFO:tasks.workunit.client.0.smithi018.stdout:[ PASSED ] 70 tests. 2023-09-23T20:50:09.143 INFO:teuthology.orchestra.run:Running command with timeout 3600 2023-09-23T20:50:09.143 DEBUG:teuthology.orchestra.run.smithi018:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2023-09-23T20:50:09.144 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:50:09.144 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:50:09.144 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:50:09.144 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:50:09.144 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:50:09.144 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:50:09.144 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:50:09.145 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:50:09.145 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:50:09.145 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:50:09.145 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:50:09.145 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:50:09.145 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:50:09.168 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:09.178 INFO:tasks.workunit:Stopping ['cls/test_cls_rbd.sh'] on client.0... 2023-09-23T20:50:09.178 DEBUG:teuthology.orchestra.run.smithi018:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2023-09-23T20:50:09.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:09 smithi149 ceph-mon[170394]: failed to encode map e647 with expected crc 2023-09-23T20:50:09.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:09 smithi149 ceph-mon[170394]: osdmap e647: 8 total, 8 up, 7 in 2023-09-23T20:50:09.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:09 smithi149 ceph-mon[170394]: failed to encode map e647 with expected crc 2023-09-23T20:50:09.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:09 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2699731336' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:50:09.366 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:50:09.369 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:09.369 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:50:09.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:09 smithi018 ceph-mon[179647]: osdmap e647: 8 total, 8 up, 7 in 2023-09-23T20:50:09.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:09 smithi018 ceph-mon[179647]: failed to encode map e647 with expected crc 2023-09-23T20:50:09.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:09 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2699731336' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:50:09.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:09 smithi018 ceph-mon[184144]: failed to encode map e647 with expected crc 2023-09-23T20:50:09.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:09 smithi018 ceph-mon[184144]: osdmap e647: 8 total, 8 up, 7 in 2023-09-23T20:50:09.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:09 smithi018 ceph-mon[184144]: failed to encode map e647 with expected crc 2023-09-23T20:50:09.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:09 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2699731336' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:50:09.392 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T20:50:09.396 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T20:50:09.421 DEBUG:teuthology.parallel:result is None 2023-09-23T20:50:09.422 DEBUG:teuthology.orchestra.run.smithi018:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0 2023-09-23T20:50:09.470 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:50:09.490 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.0/client.0 2023-09-23T20:50:09.490 DEBUG:teuthology.orchestra.run.smithi018:> rmdir -- /home/ubuntu/cephtest/mnt.0 2023-09-23T20:50:09.505 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:09.541 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:50:09.544 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:09.544 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:50:09.572 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.0/client.0 2023-09-23T20:50:09.572 INFO:teuthology.task.sequential:In sequential, running task print... 2023-09-23T20:50:09.573 INFO:teuthology.task.print:**** done cls/test_cls_rbd.sh 5-workload 2023-09-23T20:50:09.573 INFO:teuthology.task.sequential:In sequential, running task workunit... 2023-09-23T20:50:09.573 INFO:tasks.workunit:Pulling workunits from ref quincy 2023-09-23T20:50:09.574 INFO:tasks.workunit:Making a separate scratch dir for every client... 2023-09-23T20:50:09.574 DEBUG:teuthology.orchestra.run.smithi018:> stat -- /home/ubuntu/cephtest/mnt.0 2023-09-23T20:50:09.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:50:09.631 INFO:teuthology.orchestra.run.smithi018.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2023-09-23T20:50:09.632 DEBUG:teuthology.orchestra.run.smithi018:> mkdir -- /home/ubuntu/cephtest/mnt.0 2023-09-23T20:50:09.645 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:50:09.673 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:09.690 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2023-09-23T20:50:09.691 DEBUG:teuthology.orchestra.run.smithi018:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2023-09-23T20:50:09.706 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:50:09.709 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:09.709 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:50:09.749 INFO:tasks.workunit:timeout=3h 2023-09-23T20:50:09.749 INFO:tasks.workunit:cleanup=True 2023-09-23T20:50:09.749 DEBUG:teuthology.orchestra.run.smithi018:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone --depth 1 --branch quincy https://github.com/ceph/ceph.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 2023-09-23T20:50:09.807 INFO:tasks.workunit.client.0.smithi018.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2023-09-23T20:50:09.810 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:50:09.836 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:09.872 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:50:09.875 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:09.875 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:50:09.976 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:50:10.004 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:10.015 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:50:10.042 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:50:10.043 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:50:10.047 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:10.048 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:50:10.052 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 7, 3, 6, 4, 2] out_osds: [5] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T20:50:10.052 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:50:10.053 INFO:tasks.thrashosds.thrasher:fixing pg num pool .mgr 2023-09-23T20:50:10.053 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:50:10.148 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:50:10.176 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:10.219 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:50:10.222 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:10.222 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:50:10.297 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:10 smithi149 ceph-mon[170394]: pgmap v1584: 135 pgs: 135 active+clean; 4.6 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 2.8 MiB/s rd, 1.4 MiB/s wr, 2.26k op/s 2023-09-23T20:50:10.297 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:10 smithi149 ceph-mon[170394]: osdmap e648: 8 total, 8 up, 7 in 2023-09-23T20:50:10.297 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:10 smithi149 ceph-mon[170394]: failed to encode map e648 with expected crc 2023-09-23T20:50:10.304 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:10 smithi018 ceph-mon[179647]: pgmap v1584: 135 pgs: 135 active+clean; 4.6 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 2.8 MiB/s rd, 1.4 MiB/s wr, 2.26k op/s 2023-09-23T20:50:10.304 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:10 smithi018 ceph-mon[179647]: osdmap e648: 8 total, 8 up, 7 in 2023-09-23T20:50:10.304 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:10 smithi018 ceph-mon[179647]: failed to encode map e648 with expected crc 2023-09-23T20:50:10.304 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:10 smithi018 ceph-mon[184144]: pgmap v1584: 135 pgs: 135 active+clean; 4.6 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 2.8 MiB/s rd, 1.4 MiB/s wr, 2.26k op/s 2023-09-23T20:50:10.304 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:10 smithi018 ceph-mon[184144]: osdmap e648: 8 total, 8 up, 7 in 2023-09-23T20:50:10.305 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:10 smithi018 ceph-mon[184144]: failed to encode map e648 with expected crc 2023-09-23T20:50:10.323 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:50:10.353 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:10.387 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:50:10.389 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:10.390 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:50:10.491 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:50:10.519 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:10.606 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:50:10.609 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:10.609 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:50:10.710 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:50:10.737 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:10.856 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:50:10.858 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:10.859 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:50:10.960 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:50:10.990 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:11.026 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:50:11.029 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:11.030 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:50:11.055 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:50:11.131 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:50:11.164 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:11.203 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:50:11.207 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:11.207 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:50:11.308 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:50:11.340 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:11.387 INFO:teuthology.orchestra.run.smithi149.stdout:ERROR: (22) Invalid argument 2023-09-23T20:50:11.388 INFO:teuthology.orchestra.run.smithi149.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:50:11.388 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:50:11.391 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:11.391 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:50:11.492 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:50:11.517 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:50:11.531 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:11.743 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:50:11.746 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:11.746 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:50:11.787 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T20:50:11.847 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:50:11.877 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:11.912 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:50:11.915 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:11.915 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:50:12.016 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:50:12.048 INFO:teuthology.orchestra.run.smithi018.stderr:noscrub is set 2023-09-23T20:50:12.048 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:12.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:12 smithi018 ceph-mon[184144]: pgmap v1586: 103 pgs: 103 active+clean; 4.6 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail 2023-09-23T20:50:12.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:12 smithi018 ceph-mon[184144]: failed to encode map e648 with expected crc 2023-09-23T20:50:12.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:12 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/813958387' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:50:12.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:12 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:50:12.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:12 smithi018 ceph-mon[179647]: pgmap v1586: 103 pgs: 103 active+clean; 4.6 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail 2023-09-23T20:50:12.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:12 smithi018 ceph-mon[179647]: failed to encode map e648 with expected crc 2023-09-23T20:50:12.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:12 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/813958387' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:50:12.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:12 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:50:12.292 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:50:12.294 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:12.295 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:50:12.395 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:50:12.424 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:12.425 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:50:12.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:12 smithi149 ceph-mon[170394]: pgmap v1586: 103 pgs: 103 active+clean; 4.6 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail 2023-09-23T20:50:12.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:12 smithi149 ceph-mon[170394]: failed to encode map e648 with expected crc 2023-09-23T20:50:12.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:12 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/813958387' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:50:12.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:12 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:50:12.510 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:50:12.513 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:12.514 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:50:12.614 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:50:12.635 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:50:12.644 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:12.689 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:50:12.693 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:12.693 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:50:12.794 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:50:12.822 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:12.827 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:50:12.828 INFO:teuthology.orchestra.run.smithi018.stderr:dumped all 2023-09-23T20:50:12.958 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:50:12.960 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:12.961 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:50:13.062 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:50:13.093 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:13.113 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:13 smithi018 ceph-mon[184144]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:50:13.113 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:13 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:50:13.114 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:13 smithi018 ceph-mon[184144]: osdmap e649: 8 total, 8 up, 7 in 2023-09-23T20:50:13.114 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:13 smithi018 ceph-mon[184144]: failed to encode map e649 with expected crc 2023-09-23T20:50:13.114 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:13 smithi018 ceph-mon[184144]: failed to encode map e649 with expected crc 2023-09-23T20:50:13.114 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:13 smithi018 ceph-mon[179647]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:50:13.114 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:13 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:50:13.115 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:13 smithi018 ceph-mon[179647]: osdmap e649: 8 total, 8 up, 7 in 2023-09-23T20:50:13.115 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:13 smithi018 ceph-mon[179647]: failed to encode map e649 with expected crc 2023-09-23T20:50:13.115 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:13 smithi018 ceph-mon[179647]: failed to encode map e649 with expected crc 2023-09-23T20:50:13.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:13 smithi149 ceph-mon[170394]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:50:13.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:13 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:50:13.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:13 smithi149 ceph-mon[170394]: osdmap e649: 8 total, 8 up, 7 in 2023-09-23T20:50:13.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:13 smithi149 ceph-mon[170394]: failed to encode map e649 with expected crc 2023-09-23T20:50:13.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:13 smithi149 ceph-mon[170394]: failed to encode map e649 with expected crc 2023-09-23T20:50:13.382 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:50:13.385 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:13.385 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:50:13.386 INFO:teuthology.orchestra.run.smithi018.stdout:{"pg_ready":true,"pg_map":{"version":1588,"stamp":"2023-09-23T20:50:12.363582+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790195,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":5983,"num_read_kb":13200,"num_write":907,"num_write_kb":12725,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":276,"num_bytes_recovered":25707063,"num_keys_recovered":80,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6022,"ondisk_log_size":6022,"up":309,"acting":309,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":405,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":5458900,"kb_used_data":66828,"kb_used_omap":32,"kb_used_meta":5391839,"kb_avail":650757164,"statfs":{"total":671965249536,"available":666375335936,"internally_reserved":0,"allocated":68431872,"data_stored":50573031,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32773,"internal_metadata":5521244155},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-27209,"num_objects":-35,"num_object_clones":0,"num_object_copies":-105,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-35,"num_whiteouts":0,"num_read":-9252,"num_read_kb":-17467,"num_write":-4383,"num_write_kb":-8537,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":-80504,"num_keys_recovered":0,"num_objects_omap":-27,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"5.335724"},"pg_stats":[{"pgid":"1.18","version":"90'161","reported_seq":709,"reported_epoch":645,"state":"active+clean","last_fresh":"2023-09-23T20:50:02.922613+0000","last_change":"2023-09-23T20:49:51.130089+0000","last_active":"2023-09-23T20:50:02.922613+0000","last_peered":"2023-09-23T20:50:02.922613+0000","last_clean":"2023-09-23T20:50:02.922613+0000","last_became_active":"2023-09-23T20:49:51.124899+0000","last_became_peered":"2023-09-23T20:49:51.124899+0000","last_unstale":"2023-09-23T20:50:02.922613+0000","last_undegraded":"2023-09-23T20:50:02.922613+0000","last_fullsized":"2023-09-23T20:50:02.922613+0000","mapping_epoch":639,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":640,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:19.677209+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:19.677209+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:58:59.316978+0000","scrub_duration":0.033122453000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"69'1","reported_seq":124,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T20:50:08.032419+0000","last_change":"2023-09-23T20:48:48.148250+0000","last_active":"2023-09-23T20:50:08.032419+0000","last_peered":"2023-09-23T20:50:08.032419+0000","last_clean":"2023-09-23T20:50:08.032419+0000","last_became_active":"2023-09-23T20:48:48.144271+0000","last_became_peered":"2023-09-23T20:48:48.144271+0000","last_unstale":"2023-09-23T20:50:08.032419+0000","last_undegraded":"2023-09-23T20:50:08.032419+0000","last_fullsized":"2023-09-23T20:50:08.032419+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:59:36.502986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":490,"reported_epoch":646,"state":"active+clean","last_fresh":"2023-09-23T20:50:07.064431+0000","last_change":"2023-09-23T20:49:02.703099+0000","last_active":"2023-09-23T20:50:07.064431+0000","last_peered":"2023-09-23T20:50:07.064431+0000","last_clean":"2023-09-23T20:50:07.064431+0000","last_became_active":"2023-09-23T20:49:02.702753+0000","last_became_peered":"2023-09-23T20:49:02.702753+0000","last_unstale":"2023-09-23T20:50:07.064431+0000","last_undegraded":"2023-09-23T20:50:07.064431+0000","last_fullsized":"2023-09-23T20:50:07.064431+0000","mapping_epoch":603,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":604,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:24:30.834235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"90'161","reported_seq":670,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T20:50:08.032654+0000","last_change":"2023-09-23T20:49:51.121955+0000","last_active":"2023-09-23T20:50:08.032654+0000","last_peered":"2023-09-23T20:50:08.032654+0000","last_clean":"2023-09-23T20:50:08.032654+0000","last_became_active":"2023-09-23T20:49:51.121641+0000","last_became_peered":"2023-09-23T20:49:51.121641+0000","last_unstale":"2023-09-23T20:50:08.032654+0000","last_undegraded":"2023-09-23T20:50:08.032654+0000","last_fullsized":"2023-09-23T20:50:08.032654+0000","mapping_epoch":639,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":640,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:47:33.523187+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:36:05.442288+0000","last_clean_scrub_stamp":"2023-09-23T20:47:33.523187+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:05:36.252341+0000","scrub_duration":0.013532127,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,2],"acting":[4,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":637,"reported_epoch":646,"state":"active+clean","last_fresh":"2023-09-23T20:50:07.030245+0000","last_change":"2023-09-23T20:47:09.173501+0000","last_active":"2023-09-23T20:50:07.030245+0000","last_peered":"2023-09-23T20:50:07.030245+0000","last_clean":"2023-09-23T20:50:07.030245+0000","last_became_active":"2023-09-23T20:47:09.173160+0000","last_became_peered":"2023-09-23T20:47:09.173160+0000","last_unstale":"2023-09-23T20:50:07.030245+0000","last_undegraded":"2023-09-23T20:50:07.030245+0000","last_fullsized":"2023-09-23T20:50:07.030245+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:44:36.132093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":608,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T20:50:08.590402+0000","last_change":"2023-09-23T20:43:46.521939+0000","last_active":"2023-09-23T20:50:08.590402+0000","last_peered":"2023-09-23T20:50:08.590402+0000","last_clean":"2023-09-23T20:50:08.590402+0000","last_became_active":"2023-09-23T20:43:46.521185+0000","last_became_peered":"2023-09-23T20:43:46.521185+0000","last_unstale":"2023-09-23T20:50:08.590402+0000","last_undegraded":"2023-09-23T20:50:08.590402+0000","last_fullsized":"2023-09-23T20:50:08.590402+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:59:25.045627+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"90'161","reported_seq":535,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T20:50:08.032310+0000","last_change":"2023-09-23T20:47:31.475001+0000","last_active":"2023-09-23T20:50:08.032310+0000","last_peered":"2023-09-23T20:50:08.032310+0000","last_clean":"2023-09-23T20:50:08.032310+0000","last_became_active":"2023-09-23T20:47:11.617942+0000","last_became_peered":"2023-09-23T20:47:11.617942+0000","last_unstale":"2023-09-23T20:50:08.032310+0000","last_undegraded":"2023-09-23T20:50:08.032310+0000","last_fullsized":"2023-09-23T20:50:08.032310+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:47:31.474914+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:47:31.474914+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:52:05.169219+0000","scrub_duration":0.0026262870000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":444,"reported_epoch":646,"state":"active+clean","last_fresh":"2023-09-23T20:50:07.063037+0000","last_change":"2023-09-23T20:37:43.584414+0000","last_active":"2023-09-23T20:50:07.063037+0000","last_peered":"2023-09-23T20:50:07.063037+0000","last_clean":"2023-09-23T20:50:07.063037+0000","last_became_active":"2023-09-23T20:37:43.584116+0000","last_became_peered":"2023-09-23T20:37:43.584116+0000","last_unstale":"2023-09-23T20:50:07.063037+0000","last_undegraded":"2023-09-23T20:50:07.063037+0000","last_fullsized":"2023-09-23T20:50:07.063037+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:17:54.258172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":271,"reported_epoch":646,"state":"active+clean","last_fresh":"2023-09-23T20:50:07.067265+0000","last_change":"2023-09-23T20:49:48.119179+0000","last_active":"2023-09-23T20:50:07.067265+0000","last_peered":"2023-09-23T20:50:07.067265+0000","last_clean":"2023-09-23T20:50:07.067265+0000","last_became_active":"2023-09-23T20:49:48.118849+0000","last_became_peered":"2023-09-23T20:49:48.118849+0000","last_unstale":"2023-09-23T20:50:07.067265+0000","last_undegraded":"2023-09-23T20:50:07.067265+0000","last_fullsized":"2023-09-23T20:50:07.067265+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:50:02.578569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"90'161","reported_seq":880,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T20:50:08.034962+0000","last_change":"2023-09-23T20:49:51.124760+0000","last_active":"2023-09-23T20:50:08.034962+0000","last_peered":"2023-09-23T20:50:08.034962+0000","last_clean":"2023-09-23T20:50:08.034962+0000","last_became_active":"2023-09-23T20:49:51.124406+0000","last_became_peered":"2023-09-23T20:49:51.124406+0000","last_unstale":"2023-09-23T20:50:08.034962+0000","last_undegraded":"2023-09-23T20:50:08.034962+0000","last_fullsized":"2023-09-23T20:50:08.034962+0000","mapping_epoch":639,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":640,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:03.411025+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:03.411025+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:28:15.568088+0000","scrub_duration":0.001802922,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":649,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T20:50:08.590341+0000","last_change":"2023-09-23T20:49:48.111285+0000","last_active":"2023-09-23T20:50:08.590341+0000","last_peered":"2023-09-23T20:50:08.590341+0000","last_clean":"2023-09-23T20:50:08.590341+0000","last_became_active":"2023-09-23T20:49:48.110934+0000","last_became_peered":"2023-09-23T20:49:48.110934+0000","last_unstale":"2023-09-23T20:50:08.590341+0000","last_undegraded":"2023-09-23T20:50:08.590341+0000","last_fullsized":"2023-09-23T20:50:08.590341+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:19:18.339128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":80,"reported_epoch":646,"state":"active+clean","last_fresh":"2023-09-23T20:50:07.062643+0000","last_change":"2023-09-23T20:48:23.918092+0000","last_active":"2023-09-23T20:50:07.062643+0000","last_peered":"2023-09-23T20:50:07.062643+0000","last_clean":"2023-09-23T20:50:07.062643+0000","last_became_active":"2023-09-23T20:48:23.917386+0000","last_became_peered":"2023-09-23T20:48:23.917386+0000","last_unstale":"2023-09-23T20:50:07.062643+0000","last_undegraded":"2023-09-23T20:50:07.062643+0000","last_fullsized":"2023-09-23T20:50:07.062643+0000","mapping_epoch":577,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":578,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:06:09.587517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"90'161","reported_seq":648,"reported_epoch":645,"state":"active+clean","last_fresh":"2023-09-23T20:50:02.924104+0000","last_change":"2023-09-23T20:49:36.717905+0000","last_active":"2023-09-23T20:50:02.924104+0000","last_peered":"2023-09-23T20:50:02.924104+0000","last_clean":"2023-09-23T20:50:02.924104+0000","last_became_active":"2023-09-23T20:48:51.169314+0000","last_became_peered":"2023-09-23T20:48:51.169314+0000","last_unstale":"2023-09-23T20:50:02.924104+0000","last_undegraded":"2023-09-23T20:50:02.924104+0000","last_fullsized":"2023-09-23T20:50:02.924104+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:49:36.717737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:49:36.717737+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:00:49.213712+0000","scrub_duration":0.0049746089999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":494,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T20:50:08.036074+0000","last_change":"2023-09-23T20:49:48.123820+0000","last_active":"2023-09-23T20:50:08.036074+0000","last_peered":"2023-09-23T20:50:08.036074+0000","last_clean":"2023-09-23T20:50:08.036074+0000","last_became_active":"2023-09-23T20:49:48.123514+0000","last_became_peered":"2023-09-23T20:49:48.123514+0000","last_unstale":"2023-09-23T20:50:08.036074+0000","last_undegraded":"2023-09-23T20:50:08.036074+0000","last_fullsized":"2023-09-23T20:50:08.036074+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:43:13.460464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":475,"reported_epoch":646,"state":"active+clean","last_fresh":"2023-09-23T20:50:07.064223+0000","last_change":"2023-09-23T20:38:02.542034+0000","last_active":"2023-09-23T20:50:07.064223+0000","last_peered":"2023-09-23T20:50:07.064223+0000","last_clean":"2023-09-23T20:50:07.064223+0000","last_became_active":"2023-09-23T20:38:02.541575+0000","last_became_peered":"2023-09-23T20:38:02.541575+0000","last_unstale":"2023-09-23T20:50:07.064223+0000","last_undegraded":"2023-09-23T20:50:07.064223+0000","last_fullsized":"2023-09-23T20:50:07.064223+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:50:54.591529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"90'161","reported_seq":792,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T20:50:08.590190+0000","last_change":"2023-09-23T20:49:51.125057+0000","last_active":"2023-09-23T20:50:08.590190+0000","last_peered":"2023-09-23T20:50:08.590190+0000","last_clean":"2023-09-23T20:50:08.590190+0000","last_became_active":"2023-09-23T20:49:51.124616+0000","last_became_peered":"2023-09-23T20:49:51.124616+0000","last_unstale":"2023-09-23T20:50:08.590190+0000","last_undegraded":"2023-09-23T20:50:08.590190+0000","last_fullsized":"2023-09-23T20:50:08.590190+0000","mapping_epoch":639,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":640,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:14.669831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:14.669831+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:09:01.736519+0000","scrub_duration":0.0083441460000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":707,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T20:50:08.035824+0000","last_change":"2023-09-23T20:47:10.837423+0000","last_active":"2023-09-23T20:50:08.035824+0000","last_peered":"2023-09-23T20:50:08.035824+0000","last_clean":"2023-09-23T20:50:08.035824+0000","last_became_active":"2023-09-23T20:47:09.204324+0000","last_became_peered":"2023-09-23T20:47:09.204324+0000","last_unstale":"2023-09-23T20:50:08.035824+0000","last_undegraded":"2023-09-23T20:50:08.035824+0000","last_fullsized":"2023-09-23T20:50:08.035824+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":554,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:19.108914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":540,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":181,"reported_epoch":646,"state":"active+clean","last_fresh":"2023-09-23T20:50:07.030466+0000","last_change":"2023-09-23T20:48:48.145175+0000","last_active":"2023-09-23T20:50:07.030466+0000","last_peered":"2023-09-23T20:50:07.030466+0000","last_clean":"2023-09-23T20:50:07.030466+0000","last_became_active":"2023-09-23T20:48:48.144841+0000","last_became_peered":"2023-09-23T20:48:48.144841+0000","last_unstale":"2023-09-23T20:50:07.030466+0000","last_undegraded":"2023-09-23T20:50:07.030466+0000","last_fullsized":"2023-09-23T20:50:07.030466+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:27:40.802917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"90'161","reported_seq":775,"reported_epoch":646,"state":"active+clean","last_fresh":"2023-09-23T20:50:07.030496+0000","last_change":"2023-09-23T20:36:02.548387+0000","last_active":"2023-09-23T20:50:07.030496+0000","last_peered":"2023-09-23T20:50:07.030496+0000","last_clean":"2023-09-23T20:50:07.030496+0000","last_became_active":"2023-09-23T20:35:26.644787+0000","last_became_peered":"2023-09-23T20:35:26.644787+0000","last_unstale":"2023-09-23T20:50:07.030496+0000","last_undegraded":"2023-09-23T20:50:07.030496+0000","last_fullsized":"2023-09-23T20:50:07.030496+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:02.548285+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:02.548285+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:37:15.216553+0000","scrub_duration":0.0015514459999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":114,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T20:50:08.032535+0000","last_change":"2023-09-23T20:47:09.340979+0000","last_active":"2023-09-23T20:50:08.032535+0000","last_peered":"2023-09-23T20:50:08.032535+0000","last_clean":"2023-09-23T20:50:08.032535+0000","last_became_active":"2023-09-23T20:47:09.231222+0000","last_became_peered":"2023-09-23T20:47:09.231222+0000","last_unstale":"2023-09-23T20:50:08.032535+0000","last_undegraded":"2023-09-23T20:50:08.032535+0000","last_fullsized":"2023-09-23T20:50:08.032535+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:07:50.620006+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":104,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T20:50:08.033057+0000","last_change":"2023-09-23T20:47:09.186315+0000","last_active":"2023-09-23T20:50:08.033057+0000","last_peered":"2023-09-23T20:50:08.033057+0000","last_clean":"2023-09-23T20:50:08.033057+0000","last_became_active":"2023-09-23T20:47:09.184580+0000","last_became_peered":"2023-09-23T20:47:09.184580+0000","last_unstale":"2023-09-23T20:50:08.033057+0000","last_undegraded":"2023-09-23T20:50:08.033057+0000","last_fullsized":"2023-09-23T20:50:08.033057+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:44:35.553948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"90'161","reported_seq":920,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T20:50:08.588782+0000","last_change":"2023-09-23T20:49:51.148240+0000","last_active":"2023-09-23T20:50:08.588782+0000","last_peered":"2023-09-23T20:50:08.588782+0000","last_clean":"2023-09-23T20:50:08.588782+0000","last_became_active":"2023-09-23T20:49:51.147475+0000","last_became_peered":"2023-09-23T20:49:51.147475+0000","last_unstale":"2023-09-23T20:50:08.588782+0000","last_undegraded":"2023-09-23T20:50:08.588782+0000","last_fullsized":"2023-09-23T20:50:08.588782+0000","mapping_epoch":639,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":640,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:43.578666+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:43.578666+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:02:42.273671+0000","scrub_duration":0.0021890379999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":738,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T20:50:08.035006+0000","last_change":"2023-09-23T20:48:48.193666+0000","last_active":"2023-09-23T20:50:08.035006+0000","last_peered":"2023-09-23T20:50:08.035006+0000","last_clean":"2023-09-23T20:50:08.035006+0000","last_became_active":"2023-09-23T20:48:48.152081+0000","last_became_peered":"2023-09-23T20:48:48.152081+0000","last_unstale":"2023-09-23T20:50:08.035006+0000","last_undegraded":"2023-09-23T20:50:08.035006+0000","last_fullsized":"2023-09-23T20:50:08.035006+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:00:19.508124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":272,"num_keys_recovered":80,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":404,"reported_epoch":646,"state":"active+clean","last_fresh":"2023-09-23T20:50:07.064018+0000","last_change":"2023-09-23T20:49:48.119365+0000","last_active":"2023-09-23T20:50:07.064018+0000","last_peered":"2023-09-23T20:50:07.064018+0000","last_clean":"2023-09-23T20:50:07.064018+0000","last_became_active":"2023-09-23T20:49:48.119009+0000","last_became_peered":"2023-09-23T20:49:48.119009+0000","last_unstale":"2023-09-23T20:50:07.064018+0000","last_undegraded":"2023-09-23T20:50:07.064018+0000","last_fullsized":"2023-09-23T20:50:07.064018+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:02:26.878542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"90'161","reported_seq":948,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T20:50:08.034546+0000","last_change":"2023-09-23T20:49:51.118813+0000","last_active":"2023-09-23T20:50:08.034546+0000","last_peered":"2023-09-23T20:50:08.034546+0000","last_clean":"2023-09-23T20:50:08.034546+0000","last_became_active":"2023-09-23T20:49:51.118445+0000","last_became_peered":"2023-09-23T20:49:51.118445+0000","last_unstale":"2023-09-23T20:50:08.034546+0000","last_undegraded":"2023-09-23T20:50:08.034546+0000","last_fullsized":"2023-09-23T20:50:08.034546+0000","mapping_epoch":639,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":640,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:20.041690+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:20.041690+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:15:24.683302+0000","scrub_duration":0.0026180330000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":605,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T20:50:08.589126+0000","last_change":"2023-09-23T20:43:46.521294+0000","last_active":"2023-09-23T20:50:08.589126+0000","last_peered":"2023-09-23T20:50:08.589126+0000","last_clean":"2023-09-23T20:50:08.589126+0000","last_became_active":"2023-09-23T20:43:46.521001+0000","last_became_peered":"2023-09-23T20:43:46.521001+0000","last_unstale":"2023-09-23T20:50:08.589126+0000","last_undegraded":"2023-09-23T20:50:08.589126+0000","last_fullsized":"2023-09-23T20:50:08.589126+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:01:35.126716+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":435,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T20:50:08.036061+0000","last_change":"2023-09-23T20:49:48.121288+0000","last_active":"2023-09-23T20:50:08.036061+0000","last_peered":"2023-09-23T20:50:08.036061+0000","last_clean":"2023-09-23T20:50:08.036061+0000","last_became_active":"2023-09-23T20:49:48.120936+0000","last_became_peered":"2023-09-23T20:49:48.120936+0000","last_unstale":"2023-09-23T20:50:08.036061+0000","last_undegraded":"2023-09-23T20:50:08.036061+0000","last_fullsized":"2023-09-23T20:50:08.036061+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:46:44.012748+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.13","version":"90'161","reported_seq":970,"reported_epoch":646,"state":"active+clean","last_fresh":"2023-09-23T20:50:07.030828+0000","last_change":"2023-09-23T20:47:11.619371+0000","last_active":"2023-09-23T20:50:07.030828+0000","last_peered":"2023-09-23T20:50:07.030828+0000","last_clean":"2023-09-23T20:50:07.030828+0000","last_became_active":"2023-09-23T20:47:11.619006+0000","last_became_peered":"2023-09-23T20:47:11.619006+0000","last_unstale":"2023-09-23T20:50:07.030828+0000","last_undegraded":"2023-09-23T20:50:07.030828+0000","last_fullsized":"2023-09-23T20:50:07.030828+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:42.544308+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:42.544308+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:47:24.414807+0000","scrub_duration":0.0029958179999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":671,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T20:50:08.034886+0000","last_change":"2023-09-23T20:47:09.176613+0000","last_active":"2023-09-23T20:50:08.034886+0000","last_peered":"2023-09-23T20:50:08.034886+0000","last_clean":"2023-09-23T20:50:08.034886+0000","last_became_active":"2023-09-23T20:47:09.176187+0000","last_became_peered":"2023-09-23T20:47:09.176187+0000","last_unstale":"2023-09-23T20:50:08.034886+0000","last_undegraded":"2023-09-23T20:50:08.034886+0000","last_fullsized":"2023-09-23T20:50:08.034886+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:23:01.590989+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":614,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T20:50:08.590478+0000","last_change":"2023-09-23T20:43:46.521735+0000","last_active":"2023-09-23T20:50:08.590478+0000","last_peered":"2023-09-23T20:50:08.590478+0000","last_clean":"2023-09-23T20:50:08.590478+0000","last_became_active":"2023-09-23T20:43:46.520762+0000","last_became_peered":"2023-09-23T20:43:46.520762+0000","last_unstale":"2023-09-23T20:50:08.590478+0000","last_undegraded":"2023-09-23T20:50:08.590478+0000","last_fullsized":"2023-09-23T20:50:08.590478+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:26:48.841930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"90'161","reported_seq":749,"reported_epoch":646,"state":"active+clean","last_fresh":"2023-09-23T20:50:07.031092+0000","last_change":"2023-09-23T20:49:51.125786+0000","last_active":"2023-09-23T20:50:07.031092+0000","last_peered":"2023-09-23T20:50:07.031092+0000","last_clean":"2023-09-23T20:50:07.031092+0000","last_became_active":"2023-09-23T20:49:51.124373+0000","last_became_peered":"2023-09-23T20:49:51.124373+0000","last_unstale":"2023-09-23T20:50:07.031092+0000","last_undegraded":"2023-09-23T20:50:07.031092+0000","last_fullsized":"2023-09-23T20:50:07.031092+0000","mapping_epoch":639,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":640,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:18.952693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:18.952693+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:07:29.170332+0000","scrub_duration":0.001699365,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":341,"reported_epoch":646,"state":"active+clean","last_fresh":"2023-09-23T20:50:07.066880+0000","last_change":"2023-09-23T20:49:48.127484+0000","last_active":"2023-09-23T20:50:07.066880+0000","last_peered":"2023-09-23T20:50:07.066880+0000","last_clean":"2023-09-23T20:50:07.066880+0000","last_became_active":"2023-09-23T20:49:48.110748+0000","last_became_peered":"2023-09-23T20:49:48.110748+0000","last_unstale":"2023-09-23T20:50:07.066880+0000","last_undegraded":"2023-09-23T20:50:07.066880+0000","last_fullsized":"2023-09-23T20:50:07.066880+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:31:51.004529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":444,"reported_epoch":646,"state":"active+clean","last_fresh":"2023-09-23T20:50:07.062941+0000","last_change":"2023-09-23T20:37:43.571083+0000","last_active":"2023-09-23T20:50:07.062941+0000","last_peered":"2023-09-23T20:50:07.062941+0000","last_clean":"2023-09-23T20:50:07.062941+0000","last_became_active":"2023-09-23T20:37:43.570306+0000","last_became_peered":"2023-09-23T20:37:43.570306+0000","last_unstale":"2023-09-23T20:50:07.062941+0000","last_undegraded":"2023-09-23T20:50:07.062941+0000","last_fullsized":"2023-09-23T20:50:07.062941+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:42:02.396718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.d","version":"90'161","reported_seq":980,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T20:50:08.035417+0000","last_change":"2023-09-23T20:47:11.619510+0000","last_active":"2023-09-23T20:50:08.035417+0000","last_peered":"2023-09-23T20:50:08.035417+0000","last_clean":"2023-09-23T20:50:08.035417+0000","last_became_active":"2023-09-23T20:47:11.619154+0000","last_became_peered":"2023-09-23T20:47:11.619154+0000","last_unstale":"2023-09-23T20:50:08.035417+0000","last_undegraded":"2023-09-23T20:50:08.035417+0000","last_fullsized":"2023-09-23T20:50:08.035417+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:34:59.496375+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:34:59.496375+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:40:52.712777+0000","scrub_duration":0.0017843189999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":679,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T20:50:08.036176+0000","last_change":"2023-09-23T20:47:09.207851+0000","last_active":"2023-09-23T20:50:08.036176+0000","last_peered":"2023-09-23T20:50:08.036176+0000","last_clean":"2023-09-23T20:50:08.036176+0000","last_became_active":"2023-09-23T20:47:09.175836+0000","last_became_peered":"2023-09-23T20:47:09.175836+0000","last_unstale":"2023-09-23T20:50:08.036176+0000","last_undegraded":"2023-09-23T20:50:08.036176+0000","last_fullsized":"2023-09-23T20:50:08.036176+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:43:13.091809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":472,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T20:50:08.034902+0000","last_change":"2023-09-23T20:49:48.122188+0000","last_active":"2023-09-23T20:50:08.034902+0000","last_peered":"2023-09-23T20:50:08.034902+0000","last_clean":"2023-09-23T20:50:08.034902+0000","last_became_active":"2023-09-23T20:49:48.121894+0000","last_became_peered":"2023-09-23T20:49:48.121894+0000","last_unstale":"2023-09-23T20:50:08.034902+0000","last_undegraded":"2023-09-23T20:50:08.034902+0000","last_fullsized":"2023-09-23T20:50:08.034902+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:12:33.758985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.c","version":"90'161","reported_seq":910,"reported_epoch":646,"state":"active+clean","last_fresh":"2023-09-23T20:50:07.064111+0000","last_change":"2023-09-23T20:41:09.506481+0000","last_active":"2023-09-23T20:50:07.064111+0000","last_peered":"2023-09-23T20:50:07.064111+0000","last_clean":"2023-09-23T20:50:07.064111+0000","last_became_active":"2023-09-23T20:38:28.442229+0000","last_became_peered":"2023-09-23T20:38:28.442229+0000","last_unstale":"2023-09-23T20:50:07.064111+0000","last_undegraded":"2023-09-23T20:50:07.064111+0000","last_fullsized":"2023-09-23T20:50:07.064111+0000","mapping_epoch":411,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":412,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:41:09.506338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:41:09.506338+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:19:46.265011+0000","scrub_duration":0.01112002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":717,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T20:50:08.036193+0000","last_change":"2023-09-23T20:47:10.848436+0000","last_active":"2023-09-23T20:50:08.036193+0000","last_peered":"2023-09-23T20:50:08.036193+0000","last_clean":"2023-09-23T20:50:08.036193+0000","last_became_active":"2023-09-23T20:47:09.196769+0000","last_became_peered":"2023-09-23T20:47:09.196769+0000","last_unstale":"2023-09-23T20:50:08.036193+0000","last_undegraded":"2023-09-23T20:50:08.036193+0000","last_fullsized":"2023-09-23T20:50:08.036193+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":554,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:59:38.105009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":114,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T20:50:08.031822+0000","last_change":"2023-09-23T20:48:23.915817+0000","last_active":"2023-09-23T20:50:08.031822+0000","last_peered":"2023-09-23T20:50:08.031822+0000","last_clean":"2023-09-23T20:50:08.031822+0000","last_became_active":"2023-09-23T20:48:23.915306+0000","last_became_peered":"2023-09-23T20:48:23.915306+0000","last_unstale":"2023-09-23T20:50:08.031822+0000","last_undegraded":"2023-09-23T20:50:08.031822+0000","last_fullsized":"2023-09-23T20:50:08.031822+0000","mapping_epoch":577,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":578,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:55:45.691450+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"90'161","reported_seq":861,"reported_epoch":646,"state":"active+clean","last_fresh":"2023-09-23T20:50:07.067369+0000","last_change":"2023-09-23T20:49:51.120296+0000","last_active":"2023-09-23T20:50:07.067369+0000","last_peered":"2023-09-23T20:50:07.067369+0000","last_clean":"2023-09-23T20:50:07.067369+0000","last_became_active":"2023-09-23T20:49:51.120016+0000","last_became_peered":"2023-09-23T20:49:51.120016+0000","last_unstale":"2023-09-23T20:50:07.067369+0000","last_undegraded":"2023-09-23T20:50:07.067369+0000","last_fullsized":"2023-09-23T20:50:07.067369+0000","mapping_epoch":639,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":640,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:46.217489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:46.217489+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:35:59.437404+0000","scrub_duration":0.001456429,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":690,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T20:50:08.035529+0000","last_change":"2023-09-23T20:49:48.131779+0000","last_active":"2023-09-23T20:50:08.035529+0000","last_peered":"2023-09-23T20:50:08.035529+0000","last_clean":"2023-09-23T20:50:08.035529+0000","last_became_active":"2023-09-23T20:49:48.123934+0000","last_became_peered":"2023-09-23T20:49:48.123934+0000","last_unstale":"2023-09-23T20:50:08.035529+0000","last_undegraded":"2023-09-23T20:50:08.035529+0000","last_fullsized":"2023-09-23T20:50:08.035529+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:24:40.508247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":490,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":261,"reported_epoch":646,"state":"active+clean","last_fresh":"2023-09-23T20:50:07.066656+0000","last_change":"2023-09-23T20:49:02.702301+0000","last_active":"2023-09-23T20:50:07.066656+0000","last_peered":"2023-09-23T20:50:07.066656+0000","last_clean":"2023-09-23T20:50:07.066656+0000","last_became_active":"2023-09-23T20:49:02.701986+0000","last_became_peered":"2023-09-23T20:49:02.701986+0000","last_unstale":"2023-09-23T20:50:07.066656+0000","last_undegraded":"2023-09-23T20:50:07.066656+0000","last_fullsized":"2023-09-23T20:50:07.066656+0000","mapping_epoch":603,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":604,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:20:01.603267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.e","version":"90'161","reported_seq":677,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T20:50:08.032824+0000","last_change":"2023-09-23T20:47:37.608426+0000","last_active":"2023-09-23T20:50:08.032824+0000","last_peered":"2023-09-23T20:50:08.032824+0000","last_clean":"2023-09-23T20:50:08.032824+0000","last_became_active":"2023-09-23T20:47:11.617187+0000","last_became_peered":"2023-09-23T20:47:11.617187+0000","last_unstale":"2023-09-23T20:50:08.032824+0000","last_undegraded":"2023-09-23T20:50:08.032824+0000","last_fullsized":"2023-09-23T20:50:08.032824+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:47:37.608309+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:47:37.608309+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:22:25.482297+0000","scrub_duration":0.011600924,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":618,"reported_epoch":646,"state":"active+clean","last_fresh":"2023-09-23T20:50:07.064379+0000","last_change":"2023-09-23T20:49:48.120934+0000","last_active":"2023-09-23T20:50:07.064379+0000","last_peered":"2023-09-23T20:50:07.064379+0000","last_clean":"2023-09-23T20:50:07.064379+0000","last_became_active":"2023-09-23T20:49:48.120578+0000","last_became_peered":"2023-09-23T20:49:48.120578+0000","last_unstale":"2023-09-23T20:50:07.064379+0000","last_undegraded":"2023-09-23T20:50:07.064379+0000","last_fullsized":"2023-09-23T20:50:07.064379+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:57:50.669399+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":641,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T20:50:08.589346+0000","last_change":"2023-09-23T20:47:09.197427+0000","last_active":"2023-09-23T20:50:08.589346+0000","last_peered":"2023-09-23T20:50:08.589346+0000","last_clean":"2023-09-23T20:50:08.589346+0000","last_became_active":"2023-09-23T20:47:09.196542+0000","last_became_peered":"2023-09-23T20:47:09.196542+0000","last_unstale":"2023-09-23T20:50:08.589346+0000","last_undegraded":"2023-09-23T20:50:08.589346+0000","last_fullsized":"2023-09-23T20:50:08.589346+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:28:38.926202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":1093,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T20:50:08.034623+0000","last_change":"2023-09-23T20:48:51.169474+0000","last_active":"2023-09-23T20:50:08.034623+0000","last_peered":"2023-09-23T20:50:08.034623+0000","last_clean":"2023-09-23T20:50:08.034623+0000","last_became_active":"2023-09-23T20:48:51.169076+0000","last_became_peered":"2023-09-23T20:48:51.169076+0000","last_unstale":"2023-09-23T20:50:08.034623+0000","last_undegraded":"2023-09-23T20:50:08.034623+0000","last_fullsized":"2023-09-23T20:50:08.034623+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:35:02.523203+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:35:02.523203+0000","last_clean_scrub_stamp":"2023-09-23T20:35:02.523203+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:25:53.741403+0000","scrub_duration":0.0085927989999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":496,"reported_epoch":646,"state":"active+clean","last_fresh":"2023-09-23T20:50:07.063341+0000","last_change":"2023-09-23T20:47:09.240200+0000","last_active":"2023-09-23T20:50:07.063341+0000","last_peered":"2023-09-23T20:50:07.063341+0000","last_clean":"2023-09-23T20:50:07.063341+0000","last_became_active":"2023-09-23T20:47:09.195204+0000","last_became_peered":"2023-09-23T20:47:09.195204+0000","last_unstale":"2023-09-23T20:50:07.063341+0000","last_undegraded":"2023-09-23T20:50:07.063341+0000","last_fullsized":"2023-09-23T20:50:07.063341+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:03.328250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":104,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T20:50:08.032466+0000","last_change":"2023-09-23T20:47:09.231714+0000","last_active":"2023-09-23T20:50:08.032466+0000","last_peered":"2023-09-23T20:50:08.032466+0000","last_clean":"2023-09-23T20:50:08.032466+0000","last_became_active":"2023-09-23T20:47:09.230918+0000","last_became_peered":"2023-09-23T20:47:09.230918+0000","last_unstale":"2023-09-23T20:50:08.032466+0000","last_undegraded":"2023-09-23T20:50:08.032466+0000","last_fullsized":"2023-09-23T20:50:08.032466+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:24.583656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":550,"reported_epoch":645,"state":"active+clean","last_fresh":"2023-09-23T20:50:03.882061+0000","last_change":"2023-09-23T20:50:03.882061+0000","last_active":"2023-09-23T20:50:03.882061+0000","last_peered":"2023-09-23T20:50:03.882061+0000","last_clean":"2023-09-23T20:50:03.882061+0000","last_became_active":"2023-09-23T20:49:51.124481+0000","last_became_peered":"2023-09-23T20:49:51.124481+0000","last_unstale":"2023-09-23T20:50:03.882061+0000","last_undegraded":"2023-09-23T20:50:03.882061+0000","last_fullsized":"2023-09-23T20:50:03.882061+0000","mapping_epoch":639,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":640,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:50:03.881826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:50:03.881826+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:51.282979+0000","scrub_duration":0.003245258,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":271,"reported_epoch":646,"state":"active+clean","last_fresh":"2023-09-23T20:50:07.066723+0000","last_change":"2023-09-23T20:47:09.178919+0000","last_active":"2023-09-23T20:50:07.066723+0000","last_peered":"2023-09-23T20:50:07.066723+0000","last_clean":"2023-09-23T20:50:07.066723+0000","last_became_active":"2023-09-23T20:47:09.178607+0000","last_became_peered":"2023-09-23T20:47:09.178607+0000","last_unstale":"2023-09-23T20:50:07.066723+0000","last_undegraded":"2023-09-23T20:50:07.066723+0000","last_fullsized":"2023-09-23T20:50:07.066723+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:23:45.196935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":124,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T20:50:08.032916+0000","last_change":"2023-09-23T20:48:48.148596+0000","last_active":"2023-09-23T20:50:08.032916+0000","last_peered":"2023-09-23T20:50:08.032916+0000","last_clean":"2023-09-23T20:50:08.032916+0000","last_became_active":"2023-09-23T20:48:48.148211+0000","last_became_peered":"2023-09-23T20:48:48.148211+0000","last_unstale":"2023-09-23T20:50:08.032916+0000","last_undegraded":"2023-09-23T20:50:08.032916+0000","last_fullsized":"2023-09-23T20:50:08.032916+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:21:50.825822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"90'161","reported_seq":970,"reported_epoch":646,"state":"active+clean","last_fresh":"2023-09-23T20:50:07.030886+0000","last_change":"2023-09-23T20:49:51.124836+0000","last_active":"2023-09-23T20:50:07.030886+0000","last_peered":"2023-09-23T20:50:07.030886+0000","last_clean":"2023-09-23T20:50:07.030886+0000","last_became_active":"2023-09-23T20:49:51.124419+0000","last_became_peered":"2023-09-23T20:49:51.124419+0000","last_unstale":"2023-09-23T20:50:07.030886+0000","last_undegraded":"2023-09-23T20:50:07.030886+0000","last_fullsized":"2023-09-23T20:50:07.030886+0000","mapping_epoch":639,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":640,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:05.595042+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:05.595042+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:37:05.219030+0000","scrub_duration":0.007271675,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":124,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T20:50:08.031773+0000","last_change":"2023-09-23T20:48:48.153145+0000","last_active":"2023-09-23T20:50:08.031773+0000","last_peered":"2023-09-23T20:50:08.031773+0000","last_clean":"2023-09-23T20:50:08.031773+0000","last_became_active":"2023-09-23T20:48:48.152858+0000","last_became_peered":"2023-09-23T20:48:48.152858+0000","last_unstale":"2023-09-23T20:50:08.031773+0000","last_undegraded":"2023-09-23T20:50:08.031773+0000","last_fullsized":"2023-09-23T20:50:08.031773+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:23:13.454288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":104,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T20:50:08.032693+0000","last_change":"2023-09-23T20:47:09.186271+0000","last_active":"2023-09-23T20:50:08.032693+0000","last_peered":"2023-09-23T20:50:08.032693+0000","last_clean":"2023-09-23T20:50:08.032693+0000","last_became_active":"2023-09-23T20:47:09.184357+0000","last_became_peered":"2023-09-23T20:47:09.184357+0000","last_unstale":"2023-09-23T20:50:08.032693+0000","last_undegraded":"2023-09-23T20:50:08.032693+0000","last_fullsized":"2023-09-23T20:50:08.032693+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:48:26.339811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":554,"reported_epoch":645,"state":"active+clean","last_fresh":"2023-09-23T20:50:02.923690+0000","last_change":"2023-09-23T20:49:51.128432+0000","last_active":"2023-09-23T20:50:02.923690+0000","last_peered":"2023-09-23T20:50:02.923690+0000","last_clean":"2023-09-23T20:50:02.923690+0000","last_became_active":"2023-09-23T20:49:51.128074+0000","last_became_peered":"2023-09-23T20:49:51.128074+0000","last_unstale":"2023-09-23T20:50:02.923690+0000","last_undegraded":"2023-09-23T20:50:02.923690+0000","last_fullsized":"2023-09-23T20:50:02.923690+0000","mapping_epoch":639,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":640,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:49:37.762492+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:49:37.762492+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:31:57.447267+0000","scrub_duration":0.0031485979999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,4],"acting":[2,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":444,"reported_epoch":646,"state":"active+clean","last_fresh":"2023-09-23T20:50:07.062931+0000","last_change":"2023-09-23T20:37:43.570610+0000","last_active":"2023-09-23T20:50:07.062931+0000","last_peered":"2023-09-23T20:50:07.062931+0000","last_clean":"2023-09-23T20:50:07.062931+0000","last_became_active":"2023-09-23T20:37:43.570216+0000","last_became_peered":"2023-09-23T20:37:43.570216+0000","last_unstale":"2023-09-23T20:50:07.062931+0000","last_undegraded":"2023-09-23T20:50:07.062931+0000","last_fullsized":"2023-09-23T20:50:07.062931+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:39:26.555329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":493,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T20:50:08.035563+0000","last_change":"2023-09-23T20:47:09.172573+0000","last_active":"2023-09-23T20:50:08.035563+0000","last_peered":"2023-09-23T20:50:08.035563+0000","last_clean":"2023-09-23T20:50:08.035563+0000","last_became_active":"2023-09-23T20:47:09.172298+0000","last_became_peered":"2023-09-23T20:47:09.172298+0000","last_unstale":"2023-09-23T20:50:08.035563+0000","last_undegraded":"2023-09-23T20:50:08.035563+0000","last_fullsized":"2023-09-23T20:50:08.035563+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:22:09.361284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":834,"reported_epoch":646,"state":"active+clean","last_fresh":"2023-09-23T20:50:07.067360+0000","last_change":"2023-09-23T20:41:07.424609+0000","last_active":"2023-09-23T20:50:07.067360+0000","last_peered":"2023-09-23T20:50:07.067360+0000","last_clean":"2023-09-23T20:50:07.067360+0000","last_became_active":"2023-09-23T20:38:05.622261+0000","last_became_peered":"2023-09-23T20:38:05.622261+0000","last_unstale":"2023-09-23T20:50:07.067360+0000","last_undegraded":"2023-09-23T20:50:07.067360+0000","last_fullsized":"2023-09-23T20:50:07.067360+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:41:07.424492+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:41:07.424492+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:45:21.689123+0000","scrub_duration":0.0018960959999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"86'119","reported_seq":959,"reported_epoch":646,"state":"active+clean","last_fresh":"2023-09-23T20:50:07.030735+0000","last_change":"2023-09-23T20:29:45.521257+0000","last_active":"2023-09-23T20:50:07.030735+0000","last_peered":"2023-09-23T20:50:07.030735+0000","last_clean":"2023-09-23T20:50:07.030735+0000","last_became_active":"2023-09-23T20:29:45.520546+0000","last_became_peered":"2023-09-23T20:29:45.520546+0000","last_unstale":"2023-09-23T20:50:07.030735+0000","last_undegraded":"2023-09-23T20:50:07.030735+0000","last_fullsized":"2023-09-23T20:50:07.030735+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":119,"log_dups_size":0,"ondisk_log_size":119,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:58:56.085777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032705499999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":523,"num_read_kb":3503,"num_write":136,"num_write_kb":3128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":469,"reported_epoch":646,"state":"active+clean","last_fresh":"2023-09-23T20:50:07.064496+0000","last_change":"2023-09-23T20:49:48.114924+0000","last_active":"2023-09-23T20:50:07.064496+0000","last_peered":"2023-09-23T20:50:07.064496+0000","last_clean":"2023-09-23T20:50:07.064496+0000","last_became_active":"2023-09-23T20:49:48.114554+0000","last_became_peered":"2023-09-23T20:49:48.114554+0000","last_unstale":"2023-09-23T20:50:07.064496+0000","last_undegraded":"2023-09-23T20:50:07.064496+0000","last_fullsized":"2023-09-23T20:50:07.064496+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:09:09.538462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":633,"reported_epoch":646,"state":"active+clean","last_fresh":"2023-09-23T20:50:07.030979+0000","last_change":"2023-09-23T20:49:48.115598+0000","last_active":"2023-09-23T20:50:07.030979+0000","last_peered":"2023-09-23T20:50:07.030979+0000","last_clean":"2023-09-23T20:50:07.030979+0000","last_became_active":"2023-09-23T20:49:48.115313+0000","last_became_peered":"2023-09-23T20:49:48.115313+0000","last_unstale":"2023-09-23T20:50:07.030979+0000","last_undegraded":"2023-09-23T20:50:07.030979+0000","last_fullsized":"2023-09-23T20:50:07.030979+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:58:47.477333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":747,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T20:50:08.589090+0000","last_change":"2023-09-23T20:49:51.148445+0000","last_active":"2023-09-23T20:50:08.589090+0000","last_peered":"2023-09-23T20:50:08.589090+0000","last_clean":"2023-09-23T20:50:08.589090+0000","last_became_active":"2023-09-23T20:49:51.147884+0000","last_became_peered":"2023-09-23T20:49:51.147884+0000","last_unstale":"2023-09-23T20:50:08.589090+0000","last_undegraded":"2023-09-23T20:50:08.589090+0000","last_fullsized":"2023-09-23T20:50:08.589090+0000","mapping_epoch":639,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":640,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:15.656880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:15.656880+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:05:08.116069+0000","scrub_duration":0.0016711180000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"86'106","reported_seq":939,"reported_epoch":646,"state":"active+clean","last_fresh":"2023-09-23T20:50:07.030996+0000","last_change":"2023-09-23T20:49:51.126017+0000","last_active":"2023-09-23T20:50:07.030996+0000","last_peered":"2023-09-23T20:50:07.030996+0000","last_clean":"2023-09-23T20:50:07.030996+0000","last_became_active":"2023-09-23T20:49:51.125316+0000","last_became_peered":"2023-09-23T20:49:51.125316+0000","last_unstale":"2023-09-23T20:50:07.030996+0000","last_undegraded":"2023-09-23T20:50:07.030996+0000","last_fullsized":"2023-09-23T20:50:07.030996+0000","mapping_epoch":639,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":640,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":106,"log_dups_size":0,"ondisk_log_size":106,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:32:47.065876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00099302599999999993,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1094,"num_read_kb":1678,"num_write":100,"num_write_kb":404,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":104,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T20:50:08.032972+0000","last_change":"2023-09-23T20:47:09.177872+0000","last_active":"2023-09-23T20:50:08.032972+0000","last_peered":"2023-09-23T20:50:08.032972+0000","last_clean":"2023-09-23T20:50:08.032972+0000","last_became_active":"2023-09-23T20:47:09.176689+0000","last_became_peered":"2023-09-23T20:47:09.176689+0000","last_unstale":"2023-09-23T20:50:08.032972+0000","last_undegraded":"2023-09-23T20:50:08.032972+0000","last_fullsized":"2023-09-23T20:50:08.032972+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:14:01.590318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":630,"reported_epoch":646,"state":"active+clean","last_fresh":"2023-09-23T20:50:07.030631+0000","last_change":"2023-09-23T20:48:48.151144+0000","last_active":"2023-09-23T20:50:07.030631+0000","last_peered":"2023-09-23T20:50:07.030631+0000","last_clean":"2023-09-23T20:50:07.030631+0000","last_became_active":"2023-09-23T20:48:48.150770+0000","last_became_peered":"2023-09-23T20:48:48.150770+0000","last_unstale":"2023-09-23T20:50:07.030631+0000","last_undegraded":"2023-09-23T20:50:07.030631+0000","last_fullsized":"2023-09-23T20:50:07.030631+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:14:08.549512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":651,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T20:50:08.035086+0000","last_change":"2023-09-23T20:47:11.618652+0000","last_active":"2023-09-23T20:50:08.035086+0000","last_peered":"2023-09-23T20:50:08.035086+0000","last_clean":"2023-09-23T20:50:08.035086+0000","last_became_active":"2023-09-23T20:47:11.618266+0000","last_became_peered":"2023-09-23T20:47:11.618266+0000","last_unstale":"2023-09-23T20:50:08.035086+0000","last_undegraded":"2023-09-23T20:50:08.035086+0000","last_fullsized":"2023-09-23T20:50:08.035086+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:26.978179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:26.978179+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:29:29.000683+0000","scrub_duration":0.0013962829999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"86'130","reported_seq":431,"reported_epoch":645,"state":"active+clean","last_fresh":"2023-09-23T20:50:02.923213+0000","last_change":"2023-09-23T20:48:51.171448+0000","last_active":"2023-09-23T20:50:02.923213+0000","last_peered":"2023-09-23T20:50:02.923213+0000","last_clean":"2023-09-23T20:50:02.923213+0000","last_became_active":"2023-09-23T20:48:51.170485+0000","last_became_peered":"2023-09-23T20:48:51.170485+0000","last_unstale":"2023-09-23T20:50:02.923213+0000","last_undegraded":"2023-09-23T20:50:02.923213+0000","last_fullsized":"2023-09-23T20:50:02.923213+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":130,"log_dups_size":0,"ondisk_log_size":130,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:48:35.309684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011300049999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":512,"num_read_kb":1820,"num_write":133,"num_write_kb":1347,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":639,"reported_epoch":646,"state":"active+clean","last_fresh":"2023-09-23T20:50:07.030141+0000","last_change":"2023-09-23T20:48:48.151034+0000","last_active":"2023-09-23T20:50:07.030141+0000","last_peered":"2023-09-23T20:50:07.030141+0000","last_clean":"2023-09-23T20:50:07.030141+0000","last_became_active":"2023-09-23T20:48:48.150658+0000","last_became_peered":"2023-09-23T20:48:48.150658+0000","last_unstale":"2023-09-23T20:50:07.030141+0000","last_undegraded":"2023-09-23T20:50:07.030141+0000","last_fullsized":"2023-09-23T20:50:07.030141+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:44:34.819851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":172,"reported_epoch":646,"state":"active+clean","last_fresh":"2023-09-23T20:50:07.033088+0000","last_change":"2023-09-23T20:49:48.118182+0000","last_active":"2023-09-23T20:50:07.033088+0000","last_peered":"2023-09-23T20:50:07.033088+0000","last_clean":"2023-09-23T20:50:07.033088+0000","last_became_active":"2023-09-23T20:49:48.117804+0000","last_became_peered":"2023-09-23T20:49:48.117804+0000","last_unstale":"2023-09-23T20:50:07.033088+0000","last_undegraded":"2023-09-23T20:50:07.033088+0000","last_fullsized":"2023-09-23T20:50:07.033088+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:47:43.384018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"86'194","reported_seq":798,"reported_epoch":645,"state":"active+clean","last_fresh":"2023-09-23T20:50:02.923733+0000","last_change":"2023-09-23T20:49:51.154300+0000","last_active":"2023-09-23T20:50:02.923733+0000","last_peered":"2023-09-23T20:50:02.923733+0000","last_clean":"2023-09-23T20:50:02.923733+0000","last_became_active":"2023-09-23T20:49:51.153529+0000","last_became_peered":"2023-09-23T20:49:51.153529+0000","last_unstale":"2023-09-23T20:50:02.923733+0000","last_undegraded":"2023-09-23T20:50:02.923733+0000","last_fullsized":"2023-09-23T20:50:02.923733+0000","mapping_epoch":639,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":640,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":194,"log_dups_size":0,"ondisk_log_size":194,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:10:22.072962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00037995999999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1024,"num_read_kb":1200,"num_write":173,"num_write_kb":1276,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":1110,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T20:50:08.034648+0000","last_change":"2023-09-23T20:38:05.602508+0000","last_active":"2023-09-23T20:50:08.034648+0000","last_peered":"2023-09-23T20:50:08.034648+0000","last_clean":"2023-09-23T20:50:08.034648+0000","last_became_active":"2023-09-23T20:38:05.602129+0000","last_became_peered":"2023-09-23T20:38:05.602129+0000","last_unstale":"2023-09-23T20:50:08.034648+0000","last_undegraded":"2023-09-23T20:50:08.034648+0000","last_fullsized":"2023-09-23T20:50:08.034648+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:45.343742+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:45.343742+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:48.910343+0000","scrub_duration":0.001475802,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":231,"reported_epoch":646,"state":"active+clean","last_fresh":"2023-09-23T20:50:07.066833+0000","last_change":"2023-09-23T20:49:48.119237+0000","last_active":"2023-09-23T20:50:07.066833+0000","last_peered":"2023-09-23T20:50:07.066833+0000","last_clean":"2023-09-23T20:50:07.066833+0000","last_became_active":"2023-09-23T20:49:48.118927+0000","last_became_peered":"2023-09-23T20:49:48.118927+0000","last_unstale":"2023-09-23T20:50:07.066833+0000","last_undegraded":"2023-09-23T20:50:07.066833+0000","last_fullsized":"2023-09-23T20:50:07.066833+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:31:47.579623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,4],"acting":[6,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":271,"reported_epoch":646,"state":"active+clean","last_fresh":"2023-09-23T20:50:07.067183+0000","last_change":"2023-09-23T20:47:09.202302+0000","last_active":"2023-09-23T20:50:07.067183+0000","last_peered":"2023-09-23T20:50:07.067183+0000","last_clean":"2023-09-23T20:50:07.067183+0000","last_became_active":"2023-09-23T20:47:09.201797+0000","last_became_peered":"2023-09-23T20:47:09.201797+0000","last_unstale":"2023-09-23T20:50:07.067183+0000","last_undegraded":"2023-09-23T20:50:07.067183+0000","last_fullsized":"2023-09-23T20:50:07.067183+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:34:41.055705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":1015,"reported_epoch":646,"state":"active+clean","last_fresh":"2023-09-23T20:50:07.030725+0000","last_change":"2023-09-23T20:47:11.622262+0000","last_active":"2023-09-23T20:50:07.030725+0000","last_peered":"2023-09-23T20:50:07.030725+0000","last_clean":"2023-09-23T20:50:07.030725+0000","last_became_active":"2023-09-23T20:47:11.621920+0000","last_became_peered":"2023-09-23T20:47:11.621920+0000","last_unstale":"2023-09-23T20:50:07.030725+0000","last_undegraded":"2023-09-23T20:50:07.030725+0000","last_fullsized":"2023-09-23T20:50:07.030725+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:43.582146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:43.582146+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:01:15.153543+0000","scrub_duration":0.0016219730000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"86'80","reported_seq":900,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T20:50:08.034595+0000","last_change":"2023-09-23T20:29:45.536796+0000","last_active":"2023-09-23T20:50:08.034595+0000","last_peered":"2023-09-23T20:50:08.034595+0000","last_clean":"2023-09-23T20:50:08.034595+0000","last_became_active":"2023-09-23T20:29:45.536439+0000","last_became_peered":"2023-09-23T20:29:45.536439+0000","last_unstale":"2023-09-23T20:50:08.034595+0000","last_undegraded":"2023-09-23T20:50:08.034595+0000","last_fullsized":"2023-09-23T20:50:08.034595+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":80,"log_dups_size":0,"ondisk_log_size":80,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:30:57.793132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035115,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":395,"num_read_kb":1370,"num_write":66,"num_write_kb":1061,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":661,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T20:50:08.588976+0000","last_change":"2023-09-23T20:47:09.198326+0000","last_active":"2023-09-23T20:50:08.588976+0000","last_peered":"2023-09-23T20:50:08.588976+0000","last_clean":"2023-09-23T20:50:08.588976+0000","last_became_active":"2023-09-23T20:47:09.198059+0000","last_became_peered":"2023-09-23T20:47:09.198059+0000","last_unstale":"2023-09-23T20:50:08.588976+0000","last_undegraded":"2023-09-23T20:50:08.588976+0000","last_fullsized":"2023-09-23T20:50:08.588976+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:38:53.903706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":655,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T20:50:08.587462+0000","last_change":"2023-09-23T20:48:48.146977+0000","last_active":"2023-09-23T20:50:08.587462+0000","last_peered":"2023-09-23T20:50:08.587462+0000","last_clean":"2023-09-23T20:50:08.587462+0000","last_became_active":"2023-09-23T20:48:48.146657+0000","last_became_peered":"2023-09-23T20:48:48.146657+0000","last_unstale":"2023-09-23T20:50:08.587462+0000","last_undegraded":"2023-09-23T20:50:08.587462+0000","last_fullsized":"2023-09-23T20:50:08.587462+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:38:41.782692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":757,"reported_epoch":646,"state":"active+clean","last_fresh":"2023-09-23T20:50:07.067081+0000","last_change":"2023-09-23T20:41:08.380084+0000","last_active":"2023-09-23T20:50:07.067081+0000","last_peered":"2023-09-23T20:50:07.067081+0000","last_clean":"2023-09-23T20:50:07.067081+0000","last_became_active":"2023-09-23T20:38:05.610881+0000","last_became_peered":"2023-09-23T20:38:05.610881+0000","last_unstale":"2023-09-23T20:50:07.067081+0000","last_undegraded":"2023-09-23T20:50:07.067081+0000","last_fullsized":"2023-09-23T20:50:07.067081+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:41:08.379959+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:41:08.379959+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:19:39.547923+0000","scrub_duration":0.0019158420000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"86'102","reported_seq":1318,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T20:50:08.035618+0000","last_change":"2023-09-23T20:47:14.628795+0000","last_active":"2023-09-23T20:50:08.035618+0000","last_peered":"2023-09-23T20:50:08.035618+0000","last_clean":"2023-09-23T20:50:08.035618+0000","last_became_active":"2023-09-23T20:47:14.628341+0000","last_became_peered":"2023-09-23T20:47:14.628341+0000","last_unstale":"2023-09-23T20:50:08.035618+0000","last_undegraded":"2023-09-23T20:50:08.035618+0000","last_fullsized":"2023-09-23T20:50:08.035618+0000","mapping_epoch":558,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":559,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":102,"log_dups_size":0,"ondisk_log_size":102,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:42:16.291849+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00030656500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":963,"num_read_kb":1234,"num_write":60,"num_write_kb":225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":623,"reported_epoch":646,"state":"active+clean","last_fresh":"2023-09-23T20:50:07.030926+0000","last_change":"2023-09-23T20:49:48.108294+0000","last_active":"2023-09-23T20:50:07.030926+0000","last_peered":"2023-09-23T20:50:07.030926+0000","last_clean":"2023-09-23T20:50:07.030926+0000","last_became_active":"2023-09-23T20:49:48.107916+0000","last_became_peered":"2023-09-23T20:49:48.107916+0000","last_unstale":"2023-09-23T20:50:07.030926+0000","last_undegraded":"2023-09-23T20:50:07.030926+0000","last_fullsized":"2023-09-23T20:50:07.030926+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:57:12.800814+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":602,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T20:50:08.589544+0000","last_change":"2023-09-23T20:49:48.119227+0000","last_active":"2023-09-23T20:50:08.589544+0000","last_peered":"2023-09-23T20:50:08.589544+0000","last_clean":"2023-09-23T20:50:08.589544+0000","last_became_active":"2023-09-23T20:49:48.118910+0000","last_became_peered":"2023-09-23T20:49:48.118910+0000","last_unstale":"2023-09-23T20:50:08.589544+0000","last_undegraded":"2023-09-23T20:50:08.589544+0000","last_fullsized":"2023-09-23T20:50:08.589544+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:18:40.836373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":684,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T20:50:08.033043+0000","last_change":"2023-09-23T20:48:51.172830+0000","last_active":"2023-09-23T20:50:08.033043+0000","last_peered":"2023-09-23T20:50:08.033043+0000","last_clean":"2023-09-23T20:50:08.033043+0000","last_became_active":"2023-09-23T20:48:51.172481+0000","last_became_peered":"2023-09-23T20:48:51.172481+0000","last_unstale":"2023-09-23T20:50:08.033043+0000","last_undegraded":"2023-09-23T20:50:08.033043+0000","last_fullsized":"2023-09-23T20:50:08.033043+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:47:34.561747+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:47:34.561747+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:30:32.531477+0000","scrub_duration":0.0087949929999999992,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":385,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T20:50:08.036283+0000","last_change":"2023-09-23T20:32:50.549695+0000","last_active":"2023-09-23T20:50:08.036283+0000","last_peered":"2023-09-23T20:50:08.036283+0000","last_clean":"2023-09-23T20:50:08.036283+0000","last_became_active":"2023-09-23T20:32:50.549431+0000","last_became_peered":"2023-09-23T20:32:50.549431+0000","last_unstale":"2023-09-23T20:50:08.036283+0000","last_undegraded":"2023-09-23T20:50:08.036283+0000","last_fullsized":"2023-09-23T20:50:08.036283+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:45:53.666569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"86'128","reported_seq":1128,"reported_epoch":646,"state":"active+clean","last_fresh":"2023-09-23T20:50:07.033024+0000","last_change":"2023-09-23T20:47:11.621846+0000","last_active":"2023-09-23T20:50:07.033024+0000","last_peered":"2023-09-23T20:50:07.033024+0000","last_clean":"2023-09-23T20:50:07.033024+0000","last_became_active":"2023-09-23T20:47:11.621284+0000","last_became_peered":"2023-09-23T20:47:11.621284+0000","last_unstale":"2023-09-23T20:50:07.033024+0000","last_undegraded":"2023-09-23T20:50:07.033024+0000","last_fullsized":"2023-09-23T20:50:07.033024+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:43:50.658512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000293202,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":579,"num_read_kb":1473,"num_write":96,"num_write_kb":1062,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":430,"reported_epoch":646,"state":"active+clean","last_fresh":"2023-09-23T20:50:07.062726+0000","last_change":"2023-09-23T20:37:43.582105+0000","last_active":"2023-09-23T20:50:07.062726+0000","last_peered":"2023-09-23T20:50:07.062726+0000","last_clean":"2023-09-23T20:50:07.062726+0000","last_became_active":"2023-09-23T20:37:43.581860+0000","last_became_peered":"2023-09-23T20:37:43.581860+0000","last_unstale":"2023-09-23T20:50:07.062726+0000","last_undegraded":"2023-09-23T20:50:07.062726+0000","last_fullsized":"2023-09-23T20:50:07.062726+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:18:38.765486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"90'161","reported_seq":880,"reported_epoch":646,"state":"active+clean","last_fresh":"2023-09-23T20:50:07.064203+0000","last_change":"2023-09-23T20:49:51.122217+0000","last_active":"2023-09-23T20:50:07.064203+0000","last_peered":"2023-09-23T20:50:07.064203+0000","last_clean":"2023-09-23T20:50:07.064203+0000","last_became_active":"2023-09-23T20:49:51.121893+0000","last_became_peered":"2023-09-23T20:49:51.121893+0000","last_unstale":"2023-09-23T20:50:07.064203+0000","last_undegraded":"2023-09-23T20:50:07.064203+0000","last_fullsized":"2023-09-23T20:50:07.064203+0000","mapping_epoch":639,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":640,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:34:54.504554+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:34:54.504554+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:00:40.029809+0000","scrub_duration":0.011176864,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"86'117","reported_seq":1030,"reported_epoch":646,"state":"active+clean","last_fresh":"2023-09-23T20:50:07.067321+0000","last_change":"2023-09-23T20:48:51.217483+0000","last_active":"2023-09-23T20:50:07.067321+0000","last_peered":"2023-09-23T20:50:07.067321+0000","last_clean":"2023-09-23T20:50:07.067321+0000","last_became_active":"2023-09-23T20:48:51.217013+0000","last_became_peered":"2023-09-23T20:48:51.217013+0000","last_unstale":"2023-09-23T20:50:07.067321+0000","last_undegraded":"2023-09-23T20:50:07.067321+0000","last_fullsized":"2023-09-23T20:50:07.067321+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":117,"log_dups_size":0,"ondisk_log_size":117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:40:42.297947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046891400000000002,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":893,"num_read_kb":922,"num_write":82,"num_write_kb":4193,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":48,"num_bytes_recovered":25165824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":663,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T20:50:08.590321+0000","last_change":"2023-09-23T20:47:09.317116+0000","last_active":"2023-09-23T20:50:08.590321+0000","last_peered":"2023-09-23T20:50:08.590321+0000","last_clean":"2023-09-23T20:50:08.590321+0000","last_became_active":"2023-09-23T20:47:09.205930+0000","last_became_peered":"2023-09-23T20:47:09.205930+0000","last_unstale":"2023-09-23T20:50:08.590321+0000","last_undegraded":"2023-09-23T20:50:08.590321+0000","last_fullsized":"2023-09-23T20:50:08.590321+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:04:55.528625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":114,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T20:50:08.031818+0000","last_change":"2023-09-23T20:49:48.120086+0000","last_active":"2023-09-23T20:50:08.031818+0000","last_peered":"2023-09-23T20:50:08.031818+0000","last_clean":"2023-09-23T20:50:08.031818+0000","last_became_active":"2023-09-23T20:49:48.119761+0000","last_became_peered":"2023-09-23T20:49:48.119761+0000","last_unstale":"2023-09-23T20:50:08.031818+0000","last_undegraded":"2023-09-23T20:50:08.031818+0000","last_fullsized":"2023-09-23T20:50:08.031818+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:56:12.469830+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"90'161","reported_seq":906,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T20:50:08.588689+0000","last_change":"2023-09-23T20:43:46.544750+0000","last_active":"2023-09-23T20:50:08.588689+0000","last_peered":"2023-09-23T20:50:08.588689+0000","last_clean":"2023-09-23T20:50:08.588689+0000","last_became_active":"2023-09-23T20:43:46.544395+0000","last_became_peered":"2023-09-23T20:43:46.544395+0000","last_unstale":"2023-09-23T20:50:08.588689+0000","last_undegraded":"2023-09-23T20:50:08.588689+0000","last_fullsized":"2023-09-23T20:50:08.588689+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:06.487351+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:06.487351+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:56.866168+0000","scrub_duration":0.001964379,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":622,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T20:50:08.589461+0000","last_change":"2023-09-23T20:47:09.197601+0000","last_active":"2023-09-23T20:50:08.589461+0000","last_peered":"2023-09-23T20:50:08.589461+0000","last_clean":"2023-09-23T20:50:08.589461+0000","last_became_active":"2023-09-23T20:47:09.196967+0000","last_became_peered":"2023-09-23T20:47:09.196967+0000","last_unstale":"2023-09-23T20:50:08.589461+0000","last_undegraded":"2023-09-23T20:50:08.589461+0000","last_fullsized":"2023-09-23T20:50:08.589461+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:06:56.826785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":65,"reported_epoch":645,"state":"active+clean","last_fresh":"2023-09-23T20:50:02.924377+0000","last_change":"2023-09-23T20:48:48.158376+0000","last_active":"2023-09-23T20:50:02.924377+0000","last_peered":"2023-09-23T20:50:02.924377+0000","last_clean":"2023-09-23T20:50:02.924377+0000","last_became_active":"2023-09-23T20:48:48.158069+0000","last_became_peered":"2023-09-23T20:48:48.158069+0000","last_unstale":"2023-09-23T20:50:02.924377+0000","last_undegraded":"2023-09-23T20:50:02.924377+0000","last_fullsized":"2023-09-23T20:50:02.924377+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:34.658561+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"90'161","reported_seq":723,"reported_epoch":646,"state":"active+clean","last_fresh":"2023-09-23T20:50:07.064762+0000","last_change":"2023-09-23T20:41:06.499511+0000","last_active":"2023-09-23T20:50:07.064762+0000","last_peered":"2023-09-23T20:50:07.064762+0000","last_clean":"2023-09-23T20:50:07.064762+0000","last_became_active":"2023-09-23T20:38:05.601874+0000","last_became_peered":"2023-09-23T20:38:05.601874+0000","last_unstale":"2023-09-23T20:50:07.064762+0000","last_undegraded":"2023-09-23T20:50:07.064762+0000","last_fullsized":"2023-09-23T20:50:07.064762+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:41:06.499396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:41:06.499396+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:31:34.059077+0000","scrub_duration":0.0019670310000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":122,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T20:50:08.032622+0000","last_change":"2023-09-23T20:49:48.128500+0000","last_active":"2023-09-23T20:50:08.032622+0000","last_peered":"2023-09-23T20:50:08.032622+0000","last_clean":"2023-09-23T20:50:08.032622+0000","last_became_active":"2023-09-23T20:49:48.122142+0000","last_became_peered":"2023-09-23T20:49:48.122142+0000","last_unstale":"2023-09-23T20:50:08.032622+0000","last_undegraded":"2023-09-23T20:50:08.032622+0000","last_fullsized":"2023-09-23T20:50:08.032622+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:46:11.349709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":104,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T20:50:08.032893+0000","last_change":"2023-09-23T20:47:09.186579+0000","last_active":"2023-09-23T20:50:08.032893+0000","last_peered":"2023-09-23T20:50:08.032893+0000","last_clean":"2023-09-23T20:50:08.032893+0000","last_became_active":"2023-09-23T20:47:09.186227+0000","last_became_peered":"2023-09-23T20:47:09.186227+0000","last_unstale":"2023-09-23T20:50:08.032893+0000","last_undegraded":"2023-09-23T20:50:08.032893+0000","last_fullsized":"2023-09-23T20:50:08.032893+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:45:39.485308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":122,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T20:50:08.033040+0000","last_change":"2023-09-23T20:49:48.126698+0000","last_active":"2023-09-23T20:50:08.033040+0000","last_peered":"2023-09-23T20:50:08.033040+0000","last_clean":"2023-09-23T20:50:08.033040+0000","last_became_active":"2023-09-23T20:49:48.120196+0000","last_became_peered":"2023-09-23T20:49:48.120196+0000","last_unstale":"2023-09-23T20:50:08.033040+0000","last_undegraded":"2023-09-23T20:50:08.033040+0000","last_fullsized":"2023-09-23T20:50:08.033040+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:17:45.039963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":104,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T20:50:08.032346+0000","last_change":"2023-09-23T20:47:09.181240+0000","last_active":"2023-09-23T20:50:08.032346+0000","last_peered":"2023-09-23T20:50:08.032346+0000","last_clean":"2023-09-23T20:50:08.032346+0000","last_became_active":"2023-09-23T20:47:09.181022+0000","last_became_peered":"2023-09-23T20:47:09.181022+0000","last_unstale":"2023-09-23T20:50:08.032346+0000","last_undegraded":"2023-09-23T20:50:08.032346+0000","last_fullsized":"2023-09-23T20:50:08.032346+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:38:59.191781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"90'161","reported_seq":903,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T20:50:08.034971+0000","last_change":"2023-09-23T20:38:05.591992+0000","last_active":"2023-09-23T20:50:08.034971+0000","last_peered":"2023-09-23T20:50:08.034971+0000","last_clean":"2023-09-23T20:50:08.034971+0000","last_became_active":"2023-09-23T20:38:05.591701+0000","last_became_peered":"2023-09-23T20:38:05.591701+0000","last_unstale":"2023-09-23T20:50:08.034971+0000","last_undegraded":"2023-09-23T20:50:08.034971+0000","last_fullsized":"2023-09-23T20:50:08.034971+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:02.384133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:02.384133+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:41.722542+0000","scrub_duration":0.001769098,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":490,"reported_epoch":646,"state":"active+clean","last_fresh":"2023-09-23T20:50:07.065041+0000","last_change":"2023-09-23T20:48:48.158348+0000","last_active":"2023-09-23T20:50:07.065041+0000","last_peered":"2023-09-23T20:50:07.065041+0000","last_clean":"2023-09-23T20:50:07.065041+0000","last_became_active":"2023-09-23T20:48:48.157982+0000","last_became_peered":"2023-09-23T20:48:48.157982+0000","last_unstale":"2023-09-23T20:50:07.065041+0000","last_undegraded":"2023-09-23T20:50:07.065041+0000","last_fullsized":"2023-09-23T20:50:07.065041+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:03:48.078707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":456,"reported_epoch":646,"state":"active+clean","last_fresh":"2023-09-23T20:50:07.064180+0000","last_change":"2023-09-23T20:48:48.159781+0000","last_active":"2023-09-23T20:50:07.064180+0000","last_peered":"2023-09-23T20:50:07.064180+0000","last_clean":"2023-09-23T20:50:07.064180+0000","last_became_active":"2023-09-23T20:48:48.159505+0000","last_became_peered":"2023-09-23T20:48:48.159505+0000","last_unstale":"2023-09-23T20:50:07.064180+0000","last_undegraded":"2023-09-23T20:50:07.064180+0000","last_fullsized":"2023-09-23T20:50:07.064180+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:25:18.100645+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"90'161","reported_seq":776,"reported_epoch":646,"state":"active+clean","last_fresh":"2023-09-23T20:50:07.030935+0000","last_change":"2023-09-23T20:47:11.621758+0000","last_active":"2023-09-23T20:50:07.030935+0000","last_peered":"2023-09-23T20:50:07.030935+0000","last_clean":"2023-09-23T20:50:07.030935+0000","last_became_active":"2023-09-23T20:47:11.621345+0000","last_became_peered":"2023-09-23T20:47:11.621345+0000","last_unstale":"2023-09-23T20:50:07.030935+0000","last_undegraded":"2023-09-23T20:50:07.030935+0000","last_fullsized":"2023-09-23T20:50:07.030935+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:01.504973+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:01.504973+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:31:17.112393+0000","scrub_duration":0.00202344,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":728,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T20:50:08.590401+0000","last_change":"2023-09-23T20:49:48.125242+0000","last_active":"2023-09-23T20:50:08.590401+0000","last_peered":"2023-09-23T20:50:08.590401+0000","last_clean":"2023-09-23T20:50:08.590401+0000","last_became_active":"2023-09-23T20:49:48.111689+0000","last_became_peered":"2023-09-23T20:49:48.111689+0000","last_unstale":"2023-09-23T20:50:08.590401+0000","last_undegraded":"2023-09-23T20:50:08.590401+0000","last_fullsized":"2023-09-23T20:50:08.590401+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:49:38.401151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":4986,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":114,"reported_epoch":647,"state":"active+clean","last_fresh":"2023-09-23T20:50:08.031716+0000","last_change":"2023-09-23T20:48:23.917302+0000","last_active":"2023-09-23T20:50:08.031716+0000","last_peered":"2023-09-23T20:50:08.031716+0000","last_clean":"2023-09-23T20:50:08.031716+0000","last_became_active":"2023-09-23T20:48:23.916956+0000","last_became_peered":"2023-09-23T20:48:23.916956+0000","last_unstale":"2023-09-23T20:50:08.031716+0000","last_undegraded":"2023-09-23T20:50:08.031716+0000","last_fullsized":"2023-09-23T20:50:08.031716+0000","mapping_epoch":577,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":578,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:42:27.989294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":118,"num_bytes_recovered":15844,"num_keys_recovered":80,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26691,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2048000,"data_stored":2033792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5983,"num_read_kb":13200,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":156,"num_bytes_recovered":25165843,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12599296,"data_stored":12582988,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6082,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201745,"num_pgs":62,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":614180,"kb_used_data":11052,"kb_used_omap":0,"kb_used_meta":603072,"kb_avail":93130972,"statfs":{"total":95995035648,"available":95366115328,"internally_reserved":0,"allocated":11317248,"data_stored":9529504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":617545728},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:49:07 2023","interfaces":[{"interface":"back","average":{"1min":1.5669999999999999,"5min":1.774,"15min":2.0369999999999999},"min":{"1min":0.23400000000000001,"5min":0.224,"15min":0.224},"max":{"1min":5.1360000000000001,"5min":10.975,"15min":18.315000000000001},"last":0.48799999999999999},{"interface":"front","average":{"1min":1.4259999999999999,"5min":1.8819999999999999,"15min":1.865},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":8.0530000000000008,"5min":12.986000000000001,"15min":19.355},"last":0.53500000000000003}]},{"osd":1,"last update":"Sat Sep 23 20:49:16 2023","interfaces":[{"interface":"back","average":{"1min":1.4259999999999999,"5min":1.4199999999999999,"15min":1.9550000000000001},"min":{"1min":0.28499999999999998,"5min":0.25900000000000001,"15min":0.24399999999999999},"max":{"1min":7.3789999999999996,"5min":14.249000000000001,"15min":49.965000000000003},"last":0.52100000000000002},{"interface":"front","average":{"1min":1.105,"5min":2.008,"15min":2.4319999999999999},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.26700000000000002},"max":{"1min":4.0739999999999998,"5min":11.276,"15min":53.451999999999998},"last":0.69799999999999995}]},{"osd":2,"last update":"Sat Sep 23 20:49:07 2023","interfaces":[{"interface":"back","average":{"1min":1.847,"5min":1.885,"15min":2.121},"min":{"1min":0.31,"5min":0.28499999999999998,"15min":0.27800000000000002},"max":{"1min":7.9219999999999997,"5min":12.933999999999999,"15min":16.965},"last":0.65500000000000003},{"interface":"front","average":{"1min":2.0209999999999999,"5min":1.982,"15min":2.2160000000000002},"min":{"1min":0.36599999999999999,"5min":0.31,"15min":0.30299999999999999},"max":{"1min":7.3019999999999996,"5min":13.5,"15min":23.414000000000001},"last":0.47499999999999998}]},{"osd":3,"last update":"Sat Sep 23 20:49:38 2023","interfaces":[{"interface":"back","average":{"1min":0.96099999999999997,"5min":1.619,"15min":2.1280000000000001},"min":{"1min":0.36299999999999999,"5min":0.33800000000000002,"15min":0.311},"max":{"1min":3.4380000000000002,"5min":7.2880000000000003,"15min":23.352},"last":0.66900000000000004},{"interface":"front","average":{"1min":1.2929999999999999,"5min":1.8380000000000001,"15min":2.3279999999999998},"min":{"1min":0.40799999999999997,"5min":0.35399999999999998,"15min":0.34300000000000003},"max":{"1min":4.1340000000000003,"5min":6.5369999999999999,"15min":20.969999999999999},"last":0.45400000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:49:38 2023","interfaces":[{"interface":"back","average":{"1min":1.0249999999999999,"5min":1.2609999999999999,"15min":1.2589999999999999},"min":{"1min":0.25800000000000001,"5min":0.254,"15min":0.254},"max":{"1min":7.1239999999999997,"5min":7.4109999999999996,"15min":18.364999999999998},"last":0.40500000000000003},{"interface":"front","average":{"1min":2.3439999999999999,"5min":1.7050000000000001,"15min":1.5509999999999999},"min":{"1min":0.27500000000000002,"5min":0.247,"15min":0.247},"max":{"1min":27.425999999999998,"5min":27.425999999999998,"15min":27.425999999999998},"last":0.34899999999999998}]},{"osd":5,"last update":"Sat Sep 23 20:49:38 2023","interfaces":[{"interface":"back","average":{"1min":0.65200000000000002,"5min":1.105,"15min":1.262},"min":{"1min":0.30599999999999999,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":2.754,"5min":7.2270000000000003,"15min":15.005000000000001},"last":0.32900000000000001},{"interface":"front","average":{"1min":0.76600000000000001,"5min":1.359,"15min":1.409},"min":{"1min":0.33600000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":3.3900000000000001,"5min":10.641999999999999,"15min":18.390000000000001},"last":0.36199999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:49:38 2023","interfaces":[{"interface":"back","average":{"1min":1.0009999999999999,"5min":1.524,"15min":1.571},"min":{"1min":0.38700000000000001,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":3.4529999999999998,"5min":12.183,"15min":18.417000000000002},"last":0.436},{"interface":"front","average":{"1min":1.0980000000000001,"5min":1.5449999999999999,"15min":1.448},"min":{"1min":0.371,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":3.2519999999999998,"5min":11.404999999999999,"15min":15.369999999999999},"last":0.38}]}]},{"osd":6,"up_from":47,"seq":201863463384,"num_pgs":63,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":533344,"kb_used_data":11112,"kb_used_omap":0,"kb_used_meta":522176,"kb_avail":93211808,"statfs":{"total":95995035648,"available":95448891392,"internally_reserved":0,"allocated":11378688,"data_stored":9595118,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":534708224},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:50:06 2023","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":1.821,"15min":1.7509999999999999},"min":{"1min":0.255,"5min":0.23400000000000001,"15min":0.219},"max":{"1min":2.044,"5min":21.018000000000001,"15min":21.018000000000001},"last":0.40899999999999997},{"interface":"front","average":{"1min":1.04,"5min":1.9590000000000001,"15min":2.0249999999999999},"min":{"1min":0.27200000000000002,"5min":0.255,"15min":0.22700000000000001},"max":{"1min":8.7729999999999997,"5min":21.082000000000001,"15min":60.328000000000003},"last":0.42399999999999999}]},{"osd":1,"last update":"Sat Sep 23 20:49:20 2023","interfaces":[{"interface":"back","average":{"1min":1.6499999999999999,"5min":1.895,"15min":2.052},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.249},"max":{"1min":7.7409999999999997,"5min":8.9350000000000005,"15min":64.274000000000001},"last":0.36399999999999999},{"interface":"front","average":{"1min":0.93999999999999995,"5min":1.5820000000000001,"15min":1.7989999999999999},"min":{"1min":0.312,"5min":0.29099999999999998,"15min":0.28000000000000003},"max":{"1min":3.0139999999999998,"5min":8.8070000000000004,"15min":34.490000000000002},"last":0.34300000000000003}]},{"osd":2,"last update":"Sat Sep 23 20:50:06 2023","interfaces":[{"interface":"back","average":{"1min":1.306,"5min":2.4049999999999998,"15min":2.1299999999999999},"min":{"1min":0.36699999999999999,"5min":0.29699999999999999,"15min":0.27700000000000002},"max":{"1min":8.9079999999999995,"5min":22.591000000000001,"15min":22.591000000000001},"last":0.44},{"interface":"front","average":{"1min":1.4039999999999999,"5min":2.0169999999999999,"15min":1.8959999999999999},"min":{"1min":0.375,"5min":0.33900000000000002,"15min":0.30399999999999999},"max":{"1min":12.33,"5min":22.614999999999998,"15min":22.614999999999998},"last":0.49299999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:49:57 2023","interfaces":[{"interface":"back","average":{"1min":1.1970000000000001,"5min":1.7849999999999999,"15min":1.895},"min":{"1min":0.32200000000000001,"5min":0.307,"15min":0.28399999999999997},"max":{"1min":4.4489999999999998,"5min":9.6859999999999999,"15min":11.664},"last":0.46000000000000002},{"interface":"front","average":{"1min":1.367,"5min":2.1480000000000001,"15min":2.109},"min":{"1min":0.38100000000000001,"5min":0.32700000000000001,"15min":0.30499999999999999},"max":{"1min":8.6140000000000008,"5min":11.6,"15min":11.913},"last":0.53900000000000003}]},{"osd":4,"last update":"Sat Sep 23 20:49:20 2023","interfaces":[{"interface":"back","average":{"1min":0.96499999999999997,"5min":1.0680000000000001,"15min":0.96999999999999997},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.25600000000000001},"max":{"1min":2.831,"5min":10.183999999999999,"15min":23.904},"last":0.45100000000000001},{"interface":"front","average":{"1min":2.4569999999999999,"5min":1.6160000000000001,"15min":1.27},"min":{"1min":0.36299999999999999,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":11.999000000000001,"5min":11.999000000000001,"15min":11.999000000000001},"last":0.48299999999999998}]},{"osd":5,"last update":"Sat Sep 23 20:49:20 2023","interfaces":[{"interface":"back","average":{"1min":0.79800000000000004,"5min":1.2869999999999999,"15min":1.0029999999999999},"min":{"1min":0.35099999999999998,"5min":0.26400000000000001,"15min":0.253},"max":{"1min":2.371,"5min":8.0440000000000005,"15min":8.0440000000000005},"last":0.47399999999999998},{"interface":"front","average":{"1min":1.843,"5min":1.607,"15min":1.3200000000000001},"min":{"1min":0.373,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":8.2769999999999992,"5min":9.3559999999999999,"15min":9.3559999999999999},"last":0.56399999999999995}]},{"osd":7,"last update":"Sat Sep 23 20:49:57 2023","interfaces":[{"interface":"back","average":{"1min":0.78000000000000003,"5min":1.127,"15min":1.0089999999999999},"min":{"1min":0.42099999999999999,"5min":0.32200000000000001,"15min":0.28399999999999997},"max":{"1min":2.964,"5min":8.9309999999999992,"15min":8.9309999999999992},"last":0.52300000000000002},{"interface":"front","average":{"1min":0.91800000000000004,"5min":3.3050000000000002,"15min":2.0019999999999998},"min":{"1min":0.38700000000000001,"5min":0.35999999999999999,"15min":0.34699999999999998},"max":{"1min":3.9620000000000002,"5min":159.91999999999999,"15min":159.91999999999999},"last":0.51100000000000001}]}]},{"osd":1,"up_from":266,"seq":1142461300982,"num_pgs":62,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":819408,"kb_used_data":8584,"kb_used_omap":8,"kb_used_meta":810807,"kb_avail":92925744,"statfs":{"total":95995035648,"available":95155961856,"internally_reserved":0,"allocated":8790016,"data_stored":5522971,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8206,"internal_metadata":830267378},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:49:28 2023","interfaces":[{"interface":"back","average":{"1min":3.0680000000000001,"5min":2.3879999999999999,"15min":2.3109999999999999},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.20000000000000001},"max":{"1min":20.292000000000002,"5min":20.292000000000002,"15min":20.292000000000002},"last":0.33800000000000002},{"interface":"front","average":{"1min":2.101,"5min":2.681,"15min":2.895},"min":{"1min":0.217,"5min":0.217,"15min":0.19900000000000001},"max":{"1min":17.463000000000001,"5min":20.346,"15min":26.588000000000001},"last":0.36299999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:49:28 2023","interfaces":[{"interface":"back","average":{"1min":2.6850000000000001,"5min":2.7629999999999999,"15min":3.0409999999999999},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.216},"max":{"1min":23.341000000000001,"5min":23.341000000000001,"15min":32.255000000000003},"last":0.39600000000000002},{"interface":"front","average":{"1min":2.919,"5min":2.4260000000000002,"15min":2.3180000000000001},"min":{"1min":0.249,"5min":0.249,"15min":0.20899999999999999},"max":{"1min":34.002000000000002,"5min":34.002000000000002,"15min":34.002000000000002},"last":0.38500000000000001}]},{"osd":3,"last update":"Sat Sep 23 20:49:28 2023","interfaces":[{"interface":"back","average":{"1min":1.3720000000000001,"5min":2.0299999999999998,"15min":2.3300000000000001},"min":{"1min":0.28899999999999998,"5min":0.28799999999999998,"15min":0.247},"max":{"1min":8.3059999999999992,"5min":17.366,"15min":17.366},"last":0.41699999999999998},{"interface":"front","average":{"1min":2.371,"5min":2.6080000000000001,"15min":2.6899999999999999},"min":{"1min":0.317,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":17.343,"5min":17.343,"15min":19.315000000000001},"last":0.437}]},{"osd":4,"last update":"Sat Sep 23 20:49:28 2023","interfaces":[{"interface":"back","average":{"1min":2.5169999999999999,"5min":2.4159999999999999,"15min":2.427},"min":{"1min":0.33900000000000002,"5min":0.32600000000000001,"15min":0.28599999999999998},"max":{"1min":7.8920000000000003,"5min":11.791,"15min":24.327999999999999},"last":0.54700000000000004},{"interface":"front","average":{"1min":2.3090000000000002,"5min":1.853,"15min":1.758},"min":{"1min":0.38800000000000001,"5min":0.35699999999999998,"15min":0.311},"max":{"1min":8.5099999999999998,"5min":10.468999999999999,"15min":20.835999999999999},"last":0.58899999999999997}]},{"osd":5,"last update":"Sat Sep 23 20:49:28 2023","interfaces":[{"interface":"back","average":{"1min":2.7749999999999999,"5min":2.3029999999999999,"15min":2.3799999999999999},"min":{"1min":0.38,"5min":0.34699999999999998,"15min":0.308},"max":{"1min":17.396999999999998,"5min":17.396999999999998,"15min":33.401000000000003},"last":0.53500000000000003},{"interface":"front","average":{"1min":2.746,"5min":2.548,"15min":2.6389999999999998},"min":{"1min":0.377,"5min":0.377,"15min":0.32500000000000001},"max":{"1min":16.366,"5min":16.366,"15min":33.417000000000002},"last":0.64600000000000002}]},{"osd":6,"last update":"Sat Sep 23 20:49:28 2023","interfaces":[{"interface":"back","average":{"1min":3.7189999999999999,"5min":2.508,"15min":2.3639999999999999},"min":{"1min":0.41599999999999998,"5min":0.316,"15min":0.316},"max":{"1min":23.274000000000001,"5min":23.274000000000001,"15min":23.274000000000001},"last":0.60499999999999998},{"interface":"front","average":{"1min":2.3599999999999999,"5min":2.819,"15min":2.641},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.317},"max":{"1min":6.9290000000000003,"5min":11.832000000000001,"15min":24.376000000000001},"last":0.61699999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:49:28 2023","interfaces":[{"interface":"back","average":{"1min":2.7719999999999998,"5min":2.48,"15min":2.4159999999999999},"min":{"1min":0.47099999999999997,"5min":0.377,"15min":0.37},"max":{"1min":17.433,"5min":17.433,"15min":18.437000000000001},"last":0.63500000000000001},{"interface":"front","average":{"1min":2.4980000000000002,"5min":2.8410000000000002,"15min":2.6899999999999999},"min":{"1min":0.44500000000000001,"5min":0.41899999999999998,"15min":0.35999999999999999},"max":{"1min":7.9930000000000003,"5min":11.864000000000001,"15min":24.524000000000001},"last":0.69299999999999995}]}]},{"osd":0,"up_from":94,"seq":403726926185,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":963504,"kb_used_data":8488,"kb_used_omap":7,"kb_used_meta":955000,"kb_avail":92781648,"statfs":{"total":95995035648,"available":95008407552,"internally_reserved":0,"allocated":8691712,"data_stored":5586308,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":977920017},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:49:33 2023","interfaces":[{"interface":"back","average":{"1min":1.034,"5min":3.415,"15min":2.6819999999999999},"min":{"1min":0.20100000000000001,"5min":0.19400000000000001,"15min":0.17199999999999999},"max":{"1min":7.6749999999999998,"5min":152.387,"15min":152.387},"last":0.27600000000000002},{"interface":"front","average":{"1min":1.0700000000000001,"5min":3.5339999999999998,"15min":2.9039999999999999},"min":{"1min":0.32700000000000001,"5min":0.218,"15min":0.20399999999999999},"max":{"1min":7.7069999999999999,"5min":152.84100000000001,"15min":152.84100000000001},"last":0.48099999999999998}]},{"osd":2,"last update":"Sat Sep 23 20:50:11 2023","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":2.2490000000000001,"15min":2.6869999999999998},"min":{"1min":0.26700000000000002,"5min":0.23499999999999999,"15min":0.218},"max":{"1min":1.9410000000000001,"5min":14.538,"15min":62.726999999999997},"last":0.45200000000000001},{"interface":"front","average":{"1min":1.1419999999999999,"5min":2.226,"15min":2.6600000000000001},"min":{"1min":0.26100000000000001,"5min":0.217,"15min":0.217},"max":{"1min":10.159000000000001,"5min":14.167,"15min":62.689},"last":0.371}]},{"osd":3,"last update":"Sat Sep 23 20:49:43 2023","interfaces":[{"interface":"back","average":{"1min":0.69699999999999995,"5min":3.0720000000000001,"15min":3.0310000000000001},"min":{"1min":0.313,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":2.141,"5min":55.017000000000003,"15min":55.017000000000003},"last":0.505},{"interface":"front","average":{"1min":1.294,"5min":2.9889999999999999,"15min":2.9609999999999999},"min":{"1min":0.28199999999999997,"5min":0.27400000000000002,"15min":0.21199999999999999},"max":{"1min":8.2439999999999998,"5min":55.665999999999997,"15min":55.665999999999997},"last":0.40000000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:50:11 2023","interfaces":[{"interface":"back","average":{"1min":0.99199999999999999,"5min":1.964,"15min":2.008},"min":{"1min":0.34300000000000003,"5min":0.29299999999999998,"15min":0.28899999999999998},"max":{"1min":4.2859999999999996,"5min":11.75,"15min":11.75},"last":1.752},{"interface":"front","average":{"1min":1.2549999999999999,"5min":2.0230000000000001,"15min":1.853},"min":{"1min":0.42099999999999999,"5min":0.31,"15min":0.27400000000000002},"max":{"1min":5.4279999999999999,"5min":12.356999999999999,"15min":12.356999999999999},"last":0.54900000000000004}]},{"osd":5,"last update":"Sat Sep 23 20:50:11 2023","interfaces":[{"interface":"back","average":{"1min":0.83999999999999997,"5min":1.8029999999999999,"15min":1.9399999999999999},"min":{"1min":0.38500000000000001,"5min":0.308,"15min":0.26500000000000001},"max":{"1min":3.012,"5min":9.3339999999999996,"15min":11.497999999999999},"last":0.51900000000000002},{"interface":"front","average":{"1min":0.88600000000000001,"5min":1.7210000000000001,"15min":2.2189999999999999},"min":{"1min":0.33000000000000002,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":3.181,"5min":10.089,"15min":26.228000000000002},"last":0.68600000000000005}]},{"osd":6,"last update":"Sat Sep 23 20:50:11 2023","interfaces":[{"interface":"back","average":{"1min":1.395,"5min":2.1669999999999998,"15min":2.1360000000000001},"min":{"1min":0.441,"5min":0.374,"15min":0.34499999999999997},"max":{"1min":4.3540000000000001,"5min":9.2750000000000004,"15min":24.056000000000001},"last":0.58599999999999997},{"interface":"front","average":{"1min":1.391,"5min":1.974,"15min":2.1579999999999999},"min":{"1min":0.372,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":4.1669999999999998,"5min":12.427,"15min":26.268000000000001},"last":0.63100000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:50:11 2023","interfaces":[{"interface":"back","average":{"1min":1.0900000000000001,"5min":2.1320000000000001,"15min":2.3199999999999998},"min":{"1min":0.39100000000000001,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":3.8919999999999999,"5min":16.553999999999998,"15min":26.177},"last":0.66000000000000003},{"interface":"front","average":{"1min":1.1579999999999999,"5min":2.1899999999999999,"15min":2.3300000000000001},"min":{"1min":0.45700000000000002,"5min":0.40200000000000002,"15min":0.34200000000000003},"max":{"1min":4.3209999999999997,"5min":16.649000000000001,"15min":16.649000000000001},"last":1.796}]}]},{"osd":2,"up_from":104,"seq":446676599132,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":884424,"kb_used_data":12480,"kb_used_omap":11,"kb_used_meta":871924,"kb_avail":92860728,"statfs":{"total":95995035648,"available":95089385472,"internally_reserved":0,"allocated":12779520,"data_stored":9691184,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11878,"internal_metadata":892850586},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:49:11 2023","interfaces":[{"interface":"back","average":{"1min":2.2679999999999998,"5min":1.8140000000000001,"15min":2.036},"min":{"1min":0.19500000000000001,"5min":0.19500000000000001,"15min":0.17299999999999999},"max":{"1min":20.48,"5min":20.48,"15min":28.234999999999999},"last":0.26200000000000001},{"interface":"front","average":{"1min":1.8919999999999999,"5min":1.9139999999999999,"15min":2.2410000000000001},"min":{"1min":0.25700000000000001,"5min":0.245,"15min":0.19},"max":{"1min":6.7220000000000004,"5min":15.342000000000001,"15min":16.678999999999998},"last":0.31900000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:49:11 2023","interfaces":[{"interface":"back","average":{"1min":1.409,"5min":2.1480000000000001,"15min":2.3889999999999998},"min":{"1min":0.28199999999999997,"5min":0.219,"15min":0.219},"max":{"1min":10.324,"5min":21.690000000000001,"15min":46.405000000000001},"last":0.29499999999999998},{"interface":"front","average":{"1min":2.3119999999999998,"5min":1.9550000000000001,"15min":2.4340000000000002},"min":{"1min":0.29499999999999998,"5min":0.22,"15min":0.22},"max":{"1min":9.0690000000000008,"5min":13.215999999999999,"15min":46.459000000000003},"last":0.375}]},{"osd":3,"last update":"Sat Sep 23 20:49:42 2023","interfaces":[{"interface":"back","average":{"1min":1.113,"5min":2.3029999999999999,"15min":2.359},"min":{"1min":0.28199999999999997,"5min":0.25600000000000001,"15min":0.253},"max":{"1min":2.9670000000000001,"5min":30.283000000000001,"15min":30.283000000000001},"last":0.34999999999999998},{"interface":"front","average":{"1min":2.4689999999999999,"5min":2.1429999999999998,"15min":1.9139999999999999},"min":{"1min":0.30499999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":20.562999999999999,"5min":30.834,"15min":30.834},"last":0.33900000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:49:11 2023","interfaces":[{"interface":"back","average":{"1min":1.623,"5min":1.9299999999999999,"15min":1.8420000000000001},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.27500000000000002},"max":{"1min":5.556,"5min":11.141999999999999,"15min":12.025},"last":0.51000000000000001},{"interface":"front","average":{"1min":2.069,"5min":2.0099999999999998,"15min":1.982},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.30499999999999999},"max":{"1min":12.662000000000001,"5min":12.662000000000001,"15min":19.440999999999999},"last":0.65500000000000003}]},{"osd":5,"last update":"Sat Sep 23 20:49:11 2023","interfaces":[{"interface":"back","average":{"1min":1.9219999999999999,"5min":1.7150000000000001,"15min":1.8680000000000001},"min":{"1min":0.38400000000000001,"5min":0.33200000000000002,"15min":0.311},"max":{"1min":6.5460000000000003,"5min":11.183,"15min":12.07},"last":0.63400000000000001},{"interface":"front","average":{"1min":1.863,"5min":1.8999999999999999,"15min":1.9870000000000001},"min":{"1min":0.373,"5min":0.373,"15min":0.35299999999999998},"max":{"1min":10.831,"5min":11.061999999999999,"15min":19.513000000000002},"last":0.61199999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:49:11 2023","interfaces":[{"interface":"back","average":{"1min":1.8740000000000001,"5min":2.0760000000000001,"15min":2.0830000000000002},"min":{"1min":0.40699999999999997,"5min":0.34799999999999998,"15min":0.33500000000000002},"max":{"1min":10.791,"5min":11.031000000000001,"15min":19.486000000000001},"last":0.53000000000000003},{"interface":"front","average":{"1min":1.829,"5min":1.609,"15min":1.6020000000000001},"min":{"1min":0.40699999999999997,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":4.9589999999999996,"5min":8.5920000000000005,"15min":11.747},"last":0.67600000000000005}]},{"osd":7,"last update":"Sat Sep 23 20:49:11 2023","interfaces":[{"interface":"back","average":{"1min":3.7509999999999999,"5min":1.978,"15min":1.6950000000000001},"min":{"1min":0.378,"5min":0.35399999999999998,"15min":0.34699999999999998},"max":{"1min":20.530000000000001,"5min":20.530000000000001,"15min":20.530000000000001},"last":0.64500000000000002},{"interface":"front","average":{"1min":1.752,"5min":2.1160000000000001,"15min":2.121},"min":{"1min":0.36899999999999999,"5min":0.35799999999999998,"15min":0.34899999999999998},"max":{"1min":5.3259999999999996,"5min":11.218,"15min":12.099},"last":0.66600000000000004}]}]},{"osd":3,"up_from":109,"seq":468151435606,"num_pgs":62,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1077024,"kb_used_data":8424,"kb_used_omap":4,"kb_used_meta":1068539,"kb_avail":92668128,"statfs":{"total":95995035648,"available":94892163072,"internally_reserved":0,"allocated":8626176,"data_stored":5567767,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4514,"internal_metadata":1094184542},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:49:50 2023","interfaces":[{"interface":"back","average":{"1min":0.99199999999999999,"5min":2.0369999999999999,"15min":2.2189999999999999},"min":{"1min":0.26500000000000001,"5min":0.217,"15min":0.20899999999999999},"max":{"1min":4.5049999999999999,"5min":21.792999999999999,"15min":22.32},"last":0.35899999999999999},{"interface":"front","average":{"1min":0.83099999999999996,"5min":2.0790000000000002,"15min":2.2639999999999998},"min":{"1min":0.254,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":3.4569999999999999,"5min":18.579999999999998,"15min":22.805},"last":0.43099999999999999}]},{"osd":1,"last update":"Sat Sep 23 20:49:21 2023","interfaces":[{"interface":"back","average":{"1min":0.91200000000000003,"5min":1.7889999999999999,"15min":1.897},"min":{"1min":0.21299999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":4.266,"5min":15.177,"15min":17.529},"last":0.39700000000000002},{"interface":"front","average":{"1min":0.93000000000000005,"5min":1.921,"15min":2.137},"min":{"1min":0.28299999999999997,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":4.5339999999999998,"5min":11.380000000000001,"15min":14.19},"last":0.57599999999999996}]},{"osd":2,"last update":"Sat Sep 23 20:49:50 2023","interfaces":[{"interface":"back","average":{"1min":0.79300000000000004,"5min":2.1019999999999999,"15min":2.5449999999999999},"min":{"1min":0.254,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":3.319,"5min":12.473000000000001,"15min":68.206999999999994},"last":0.67700000000000005},{"interface":"front","average":{"1min":1.0509999999999999,"5min":1.841,"15min":2.3210000000000002},"min":{"1min":0.30599999999999999,"5min":0.25600000000000001,"15min":0.245},"max":{"1min":6.9500000000000002,"5min":17.919,"15min":67.736999999999995},"last":0.73899999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:49:50 2023","interfaces":[{"interface":"back","average":{"1min":1.5209999999999999,"5min":1.591,"15min":1.611},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.30499999999999999},"max":{"1min":8.125,"5min":9.0259999999999998,"15min":22.157},"last":0.54300000000000004},{"interface":"front","average":{"1min":1.6100000000000001,"5min":1.7749999999999999,"15min":2.0649999999999999},"min":{"1min":0.35399999999999998,"5min":0.33500000000000002,"15min":0.308},"max":{"1min":6.7759999999999998,"5min":16.529,"15min":16.529},"last":0.60199999999999998}]},{"osd":5,"last update":"Sat Sep 23 20:49:50 2023","interfaces":[{"interface":"back","average":{"1min":1.282,"5min":1.669,"15min":1.7929999999999999},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32300000000000001},"max":{"1min":8.0730000000000004,"5min":8.9849999999999994,"15min":22.289999999999999},"last":0.621},{"interface":"front","average":{"1min":0.748,"5min":1.835,"15min":1.718},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.32000000000000001},"max":{"1min":3.3610000000000002,"5min":8.6129999999999995,"15min":12.952},"last":0.52800000000000002}]},{"osd":6,"last update":"Sat Sep 23 20:49:50 2023","interfaces":[{"interface":"back","average":{"1min":1.087,"5min":2.3140000000000001,"15min":1.9219999999999999},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.34300000000000003},"max":{"1min":4.298,"5min":8.6790000000000003,"15min":12.917},"last":0.63900000000000001},{"interface":"front","average":{"1min":1.3100000000000001,"5min":1.8919999999999999,"15min":2.145},"min":{"1min":0.40999999999999998,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":6.8230000000000004,"5min":16.497,"15min":16.497},"last":0.68999999999999995}]},{"osd":7,"last update":"Sat Sep 23 20:49:50 2023","interfaces":[{"interface":"back","average":{"1min":1.377,"5min":1.841,"15min":2.3559999999999999},"min":{"1min":0.44900000000000001,"5min":0.42399999999999999,"15min":0.39900000000000002},"max":{"1min":6.9139999999999997,"5min":16.425000000000001,"15min":16.425000000000001},"last":0.65300000000000002},{"interface":"front","average":{"1min":1.516,"5min":2.1480000000000001,"15min":1.9630000000000001},"min":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.40999999999999998},"max":{"1min":8.173,"5min":11.891999999999999,"15min":22.202999999999999},"last":0.501}]}]},{"osd":4,"up_from":30,"seq":128849019360,"num_pgs":58,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":567016,"kb_used_data":6688,"kb_used_omap":0,"kb_used_meta":560320,"kb_avail":93178136,"statfs":{"total":95995035648,"available":95414411264,"internally_reserved":0,"allocated":6848512,"data_stored":5080179,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":573767680},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:50:07 2023","interfaces":[{"interface":"back","average":{"1min":0.63200000000000001,"5min":2.1230000000000002,"15min":1.974},"min":{"1min":0.217,"5min":0.217,"15min":0.217},"max":{"1min":3.351,"5min":73.272000000000006,"15min":73.272000000000006},"last":0.53600000000000003},{"interface":"front","average":{"1min":0.85899999999999999,"5min":1.8919999999999999,"15min":2.0350000000000001},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.246},"max":{"1min":7.2359999999999998,"5min":29.754000000000001,"15min":29.754000000000001},"last":0.52800000000000002}]},{"osd":1,"last update":"Sat Sep 23 20:49:17 2023","interfaces":[{"interface":"back","average":{"1min":2.448,"5min":1.9650000000000001,"15min":1.7629999999999999},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.25900000000000001},"max":{"1min":12.343,"5min":12.343,"15min":18.190999999999999},"last":0.51000000000000001},{"interface":"front","average":{"1min":3.1899999999999999,"5min":1.9830000000000001,"15min":1.677},"min":{"1min":0.376,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":12.231999999999999,"5min":12.231999999999999,"15min":18.173999999999999},"last":0.57799999999999996}]},{"osd":2,"last update":"Sat Sep 23 20:49:17 2023","interfaces":[{"interface":"back","average":{"1min":2.6840000000000002,"5min":1.895,"15min":1.7649999999999999},"min":{"1min":0.26700000000000002,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":12.151999999999999,"5min":12.151999999999999,"15min":13.051},"last":0.42199999999999999},{"interface":"front","average":{"1min":2.2290000000000001,"5min":2.0979999999999999,"15min":2.1110000000000002},"min":{"1min":0.34100000000000003,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":12.458,"5min":12.458,"15min":12.458},"last":0.46600000000000003}]},{"osd":3,"last update":"Sat Sep 23 20:49:36 2023","interfaces":[{"interface":"back","average":{"1min":2.069,"5min":2.052,"15min":1.9570000000000001},"min":{"1min":0.29699999999999999,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":7.2850000000000001,"5min":11.598000000000001,"15min":12.147},"last":0.49099999999999999},{"interface":"front","average":{"1min":3.0369999999999999,"5min":2.238,"15min":2.1030000000000002},"min":{"1min":0.44800000000000001,"5min":0.32300000000000001,"15min":0.30299999999999999},"max":{"1min":12.191000000000001,"5min":17.169,"15min":17.169},"last":0.62}]},{"osd":5,"last update":"Sat Sep 23 20:50:07 2023","interfaces":[{"interface":"back","average":{"1min":0.49399999999999999,"5min":1.393,"15min":1.1919999999999999},"min":{"1min":0.26100000000000001,"5min":0.25800000000000001,"15min":0.253},"max":{"1min":1.214,"5min":16.940000000000001,"15min":16.940000000000001},"last":0.378},{"interface":"front","average":{"1min":0.53600000000000003,"5min":1.264,"15min":1.1819999999999999},"min":{"1min":0.26700000000000002,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":1.9610000000000001,"5min":6.5839999999999996,"15min":8.1440000000000001},"last":0.55600000000000005}]},{"osd":6,"last update":"Sat Sep 23 20:49:17 2023","interfaces":[{"interface":"back","average":{"1min":2.423,"5min":1.881,"15min":1.4610000000000001},"min":{"1min":0.29099999999999998,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":13.503,"5min":13.503,"15min":14.731999999999999},"last":0.39400000000000002},{"interface":"front","average":{"1min":3.722,"5min":2.0009999999999999,"15min":1.3979999999999999},"min":{"1min":0.32000000000000001,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":17.059000000000001,"5min":17.059000000000001,"15min":17.059000000000001},"last":0.44500000000000001}]},{"osd":7,"last update":"Sat Sep 23 20:49:36 2023","interfaces":[{"interface":"back","average":{"1min":1.2509999999999999,"5min":4.8310000000000004,"15min":2.6789999999999998},"min":{"1min":0.34100000000000003,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":5.5389999999999997,"5min":313.50099999999998,"15min":313.50099999999998},"last":0.59199999999999997},{"interface":"front","average":{"1min":1.8129999999999999,"5min":4.5960000000000001,"15min":2.3340000000000001},"min":{"1min":0.37,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":12.414999999999999,"5min":312.67500000000001,"15min":312.67500000000001},"last":0.60399999999999998}]}]},{"osd":5,"up_from":0,"seq":167503725021,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4722,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1360,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8206,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7156,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3154,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:50:13.393 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd pool set .mgr pgp_num 81 2023-09-23T20:50:13.486 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:50:13.514 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:13.549 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:50:13.552 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:13.553 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:50:13.654 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:50:13.682 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:13.740 INFO:teuthology.orchestra.run.smithi149.stdout:ERROR: (22) Invalid argument 2023-09-23T20:50:13.741 INFO:teuthology.orchestra.run.smithi149.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:50:13.824 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:50:13.827 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:13.827 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:50:13.928 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:50:13.957 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:14.109 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:50:14.113 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:14.114 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:50:14.180 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:14 smithi149 ceph-mon[170394]: pgmap v1588: 103 pgs: 103 active+clean; 4.6 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail 2023-09-23T20:50:14.180 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:14 smithi149 ceph-mon[170394]: from='client.37308 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:50:14.180 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:14 smithi149 ceph-mon[170394]: failed to encode map e649 with expected crc 2023-09-23T20:50:14.182 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T20:50:14.215 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:50:14.250 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:14.286 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:50:14.289 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:14.289 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:50:14.360 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:50:14.390 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:50:14.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:14 smithi018 ceph-mon[184144]: pgmap v1588: 103 pgs: 103 active+clean; 4.6 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail 2023-09-23T20:50:14.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:14 smithi018 ceph-mon[184144]: from='client.37308 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:50:14.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:14 smithi018 ceph-mon[184144]: failed to encode map e649 with expected crc 2023-09-23T20:50:14.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:14 smithi018 ceph-mon[179647]: pgmap v1588: 103 pgs: 103 active+clean; 4.6 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail 2023-09-23T20:50:14.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:14 smithi018 ceph-mon[179647]: from='client.37308 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:50:14.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:14 smithi018 ceph-mon[179647]: failed to encode map e649 with expected crc 2023-09-23T20:50:14.409 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:50:14.418 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:14.428 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:50:14.448 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:50:14.453 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:50:14.456 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:14.457 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:50:14.468 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:50:14.486 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:50:14.506 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:50:14.526 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:50:14.547 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:50:14.549 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:50:14.549 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:50:14.549 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:50:14.549 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:50:14.549 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:50:14.549 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:50:14.549 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:50:14.549 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:50:14.558 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:50:14.599 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:14.693 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T20:50:14.785 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:50:14.973 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:50:14.976 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:14.976 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:50:15.078 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:50:15.105 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:15.179 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:15 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2606502237' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:50:15.180 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:15 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:50:15.180 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:50:14 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:50:14] "GET /metrics HTTP/1.1" 200 34156 "" "Prometheus/2.43.0" 2023-09-23T20:50:15.180 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:15 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2606502237' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:50:15.181 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:15 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:50:15.241 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:50:15.244 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:15.244 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:50:15.319 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:15 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2606502237' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:50:15.319 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:15 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:50:15.345 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:50:15.377 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:15.475 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:50:15.478 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:50:15.478 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:15.479 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:50:15.579 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:50:15.620 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:15.756 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:50:15.759 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:15.760 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:50:15.861 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:50:15.891 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:15.978 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:50:15.982 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:15.983 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:50:16.034 INFO:teuthology.orchestra.run.smithi149.stdout:{ 2023-09-23T20:50:16.034 INFO:teuthology.orchestra.run.smithi149.stdout: "size": 20, 2023-09-23T20:50:16.034 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 600, 2023-09-23T20:50:16.034 INFO:teuthology.orchestra.run.smithi149.stdout: "ops": [ 2023-09-23T20:50:16.034 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.034 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:141034 9.7 9:eb6fa991:::benchmark_data_smithi018_381516_object27200:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e481)", 2023-09-23T20:50:16.035 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:43:15.740452+0000", 2023-09-23T20:50:16.035 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 420.29058666200001, 2023-09-23T20:50:16.035 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0443972239999999, 2023-09-23T20:50:16.035 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:50:16.035 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:50:16.035 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:50:16.035 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:50:16.035 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:50:16.036 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 141034 2023-09-23T20:50:16.036 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.036 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:50:16.036 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.036 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:50:16.036 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.740452+0000", 2023-09-23T20:50:16.036 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:50:16.036 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.036 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.036 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:50:16.037 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.740452+0000", 2023-09-23T20:50:16.037 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:50:16.037 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.037 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.037 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:50:16.037 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.740451+0000", 2023-09-23T20:50:16.037 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:50:16.037 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.037 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.038 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:50:16.038 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.740454+0000", 2023-09-23T20:50:16.038 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.0079999999999998e-06 2023-09-23T20:50:16.038 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.038 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.038 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:50:16.038 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.740461+0000", 2023-09-23T20:50:16.038 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.601e-06 2023-09-23T20:50:16.038 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.039 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.039 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:50:16.039 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.740465+0000", 2023-09-23T20:50:16.039 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.1069999999999998e-06 2023-09-23T20:50:16.039 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.039 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.039 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:50:16.039 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.744721+0000", 2023-09-23T20:50:16.039 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0042555429999999997 2023-09-23T20:50:16.040 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.040 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.040 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op must wait for map", 2023-09-23T20:50:16.040 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.744728+0000", 2023-09-23T20:50:16.040 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.413e-06 2023-09-23T20:50:16.040 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.040 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.040 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:50:16.040 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.752872+0000", 2023-09-23T20:50:16.040 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0081438929999999993 2023-09-23T20:50:16.040 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.041 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.041 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:50:16.041 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.752882+0000", 2023-09-23T20:50:16.041 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0397999999999999e-05 2023-09-23T20:50:16.041 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.041 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.041 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:50:16.041 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.784753+0000", 2023-09-23T20:50:16.041 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0318710520000001 2023-09-23T20:50:16.041 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.041 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.042 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:50:16.042 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.784849+0000", 2023-09-23T20:50:16.042 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.5699999999999995e-05 2023-09-23T20:50:16.042 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:50:16.042 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:50:16.042 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:50:16.042 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.042 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.042 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:141036 9.7 9:eb69e1ed:::benchmark_data_smithi018_381516_object27202:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e481)", 2023-09-23T20:50:16.042 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:43:15.740478+0000", 2023-09-23T20:50:16.043 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 420.29056084400003, 2023-09-23T20:50:16.043 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0444631559999999, 2023-09-23T20:50:16.043 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:50:16.043 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:50:16.043 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:50:16.043 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:50:16.043 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:50:16.043 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 141036 2023-09-23T20:50:16.043 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.043 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:50:16.044 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.044 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:50:16.044 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.740478+0000", 2023-09-23T20:50:16.044 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:50:16.044 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.044 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.044 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:50:16.044 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.740478+0000", 2023-09-23T20:50:16.044 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:50:16.044 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.044 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.045 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:50:16.045 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.740477+0000", 2023-09-23T20:50:16.045 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:50:16.045 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.045 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.045 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:50:16.045 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.740480+0000", 2023-09-23T20:50:16.045 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.729e-06 2023-09-23T20:50:16.045 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.045 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.045 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:50:16.046 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.740486+0000", 2023-09-23T20:50:16.046 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.1909999999999997e-06 2023-09-23T20:50:16.046 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.046 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.046 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:50:16.046 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.740492+0000", 2023-09-23T20:50:16.046 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.4990000000000002e-06 2023-09-23T20:50:16.046 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.046 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.046 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:50:16.047 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.744777+0000", 2023-09-23T20:50:16.047 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0042849949999999998 2023-09-23T20:50:16.047 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.047 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.047 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting_for_map not empty", 2023-09-23T20:50:16.047 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.744782+0000", 2023-09-23T20:50:16.047 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.4870000000000002e-06 2023-09-23T20:50:16.047 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.047 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.047 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:50:16.047 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.752918+0000", 2023-09-23T20:50:16.048 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0081358779999999992 2023-09-23T20:50:16.048 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.048 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.048 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:50:16.048 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.752926+0000", 2023-09-23T20:50:16.048 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.2749999999999995e-06 2023-09-23T20:50:16.048 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.048 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.048 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:50:16.048 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.784873+0000", 2023-09-23T20:50:16.048 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.03194633 2023-09-23T20:50:16.049 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.049 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.049 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:50:16.049 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.784941+0000", 2023-09-23T20:50:16.049 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.8213000000000006e-05 2023-09-23T20:50:16.049 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:50:16.049 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:50:16.049 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:50:16.049 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.049 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.049 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:141041 9.7 9:e17fa1db:::benchmark_data_smithi018_381516_object27207:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e481)", 2023-09-23T20:50:16.050 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:43:15.740496+0000", 2023-09-23T20:50:16.050 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 420.290542303, 2023-09-23T20:50:16.050 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0815582909999999, 2023-09-23T20:50:16.050 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:50:16.050 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:50:16.050 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:50:16.050 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:50:16.050 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:50:16.050 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 141041 2023-09-23T20:50:16.050 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.051 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:50:16.051 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.051 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:50:16.051 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.740496+0000", 2023-09-23T20:50:16.051 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:50:16.051 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.051 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.051 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:50:16.051 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.740496+0000", 2023-09-23T20:50:16.051 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:50:16.051 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.052 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.052 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:50:16.052 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.740496+0000", 2023-09-23T20:50:16.052 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:50:16.052 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.052 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.052 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:50:16.052 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.740498+0000", 2023-09-23T20:50:16.052 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.1780000000000002e-06 2023-09-23T20:50:16.052 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.052 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.052 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:50:16.053 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.740504+0000", 2023-09-23T20:50:16.053 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.5609999999999998e-06 2023-09-23T20:50:16.053 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.053 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.053 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:50:16.053 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.740507+0000", 2023-09-23T20:50:16.053 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.6950000000000002e-06 2023-09-23T20:50:16.053 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.053 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.053 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:50:16.053 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.744941+0000", 2023-09-23T20:50:16.054 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0044336360000000003 2023-09-23T20:50:16.054 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.054 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.054 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting_for_map not empty", 2023-09-23T20:50:16.054 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.744961+0000", 2023-09-23T20:50:16.054 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.9601000000000001e-05 2023-09-23T20:50:16.054 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.054 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.054 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:50:16.054 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.752986+0000", 2023-09-23T20:50:16.054 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0080249980000000002 2023-09-23T20:50:16.055 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.055 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.055 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:50:16.055 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.752993+0000", 2023-09-23T20:50:16.055 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.7719999999999994e-06 2023-09-23T20:50:16.055 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.055 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.055 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:50:16.055 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.784961+0000", 2023-09-23T20:50:16.055 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0319681039999999 2023-09-23T20:50:16.055 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.055 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.056 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for missing object", 2023-09-23T20:50:16.056 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.785024+0000", 2023-09-23T20:50:16.056 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.2626000000000007e-05 2023-09-23T20:50:16.056 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.056 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.056 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:50:16.056 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.821977+0000", 2023-09-23T20:50:16.056 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.036953184 2023-09-23T20:50:16.056 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.056 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.056 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:50:16.056 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.822055+0000", 2023-09-23T20:50:16.057 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.7239999999999999e-05 2023-09-23T20:50:16.057 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:50:16.057 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:50:16.057 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:50:16.057 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.057 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.057 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:141053 9.7 9:e03873d0:::benchmark_data_smithi018_381516_object27219:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e481)", 2023-09-23T20:50:16.057 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:43:15.775241+0000", 2023-09-23T20:50:16.057 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 420.25579737499999, 2023-09-23T20:50:16.057 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.047376954, 2023-09-23T20:50:16.057 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:50:16.057 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:50:16.058 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:50:16.058 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:50:16.058 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:50:16.058 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 141053 2023-09-23T20:50:16.058 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.058 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:50:16.058 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.058 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:50:16.058 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.775241+0000", 2023-09-23T20:50:16.058 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:50:16.058 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.059 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.059 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:50:16.059 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.775241+0000", 2023-09-23T20:50:16.059 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:50:16.059 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.059 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.059 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:50:16.059 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.775240+0000", 2023-09-23T20:50:16.059 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:50:16.059 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.059 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.059 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:50:16.060 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.775245+0000", 2023-09-23T20:50:16.060 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.8890000000000001e-06 2023-09-23T20:50:16.060 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.060 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.060 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:50:16.060 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.775257+0000", 2023-09-23T20:50:16.060 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1571e-05 2023-09-23T20:50:16.060 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.060 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.060 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:50:16.060 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.775261+0000", 2023-09-23T20:50:16.061 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.7999999999999998e-06 2023-09-23T20:50:16.061 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.061 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.061 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:50:16.061 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.775328+0000", 2023-09-23T20:50:16.061 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.6329999999999997e-05 2023-09-23T20:50:16.061 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.061 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.061 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:50:16.061 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.775340+0000", 2023-09-23T20:50:16.061 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.2172e-05 2023-09-23T20:50:16.061 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.062 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.062 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:50:16.062 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.785061+0000", 2023-09-23T20:50:16.062 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0097207939999999 2023-09-23T20:50:16.062 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.062 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.062 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:50:16.062 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.785170+0000", 2023-09-23T20:50:16.062 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000109317 2023-09-23T20:50:16.062 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.062 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.062 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 3,5", 2023-09-23T20:50:16.063 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.785802+0000", 2023-09-23T20:50:16.063 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00063209099999999999 2023-09-23T20:50:16.063 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.063 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.063 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:50:16.063 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.816501+0000", 2023-09-23T20:50:16.063 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.030699185 2023-09-23T20:50:16.063 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.063 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.063 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:16.063 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.817345+0000", 2023-09-23T20:50:16.063 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00084414799999999997 2023-09-23T20:50:16.064 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.064 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.064 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:16.064 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.822536+0000", 2023-09-23T20:50:16.064 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0051903310000000003 2023-09-23T20:50:16.064 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.064 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.064 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:50:16.064 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.822567+0000", 2023-09-23T20:50:16.064 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.1572999999999997e-05 2023-09-23T20:50:16.064 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.064 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.065 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:50:16.065 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.822618+0000", 2023-09-23T20:50:16.065 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.0825000000000002e-05 2023-09-23T20:50:16.065 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:50:16.065 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:50:16.065 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:50:16.065 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.065 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.065 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:185078 9.d 9:b3b89e5b:::benchmark_data_smithi018_381516_object71244:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e501)", 2023-09-23T20:50:16.065 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:44:17.632886+0000", 2023-09-23T20:50:16.065 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 358.39815271600003, 2023-09-23T20:50:16.066 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0561142219999999, 2023-09-23T20:50:16.066 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:50:16.066 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:50:16.066 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:50:16.066 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:50:16.066 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:50:16.066 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 185078 2023-09-23T20:50:16.066 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.066 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:50:16.066 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.066 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:50:16.066 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:17.632886+0000", 2023-09-23T20:50:16.067 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:50:16.067 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.067 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.067 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:50:16.067 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:17.632886+0000", 2023-09-23T20:50:16.067 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:50:16.067 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.067 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.067 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:50:16.067 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:17.632885+0000", 2023-09-23T20:50:16.067 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:50:16.068 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.068 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.068 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:50:16.068 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:17.632890+0000", 2023-09-23T20:50:16.068 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.1120000000000004e-06 2023-09-23T20:50:16.068 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.068 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.068 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:50:16.068 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:17.632900+0000", 2023-09-23T20:50:16.068 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0541e-05 2023-09-23T20:50:16.068 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.068 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.069 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:50:16.069 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:17.632906+0000", 2023-09-23T20:50:16.069 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.1939999999999997e-06 2023-09-23T20:50:16.069 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.069 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.069 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:50:16.069 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:17.634328+0000", 2023-09-23T20:50:16.069 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0014227879999999999 2023-09-23T20:50:16.069 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.069 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.069 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:50:16.069 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:17.634341+0000", 2023-09-23T20:50:16.070 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.2524000000000001e-05 2023-09-23T20:50:16.070 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.070 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.070 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:50:16.070 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:18.658019+0000", 2023-09-23T20:50:16.070 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.023678552 2023-09-23T20:50:16.070 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.070 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.070 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:50:16.070 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:18.658147+0000", 2023-09-23T20:50:16.071 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000128088 2023-09-23T20:50:16.071 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.071 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.071 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 1,5", 2023-09-23T20:50:16.071 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:18.658233+0000", 2023-09-23T20:50:16.071 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.5655000000000006e-05 2023-09-23T20:50:16.071 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.071 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.071 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:50:16.071 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:18.667554+0000", 2023-09-23T20:50:16.071 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0093206199999999999 2023-09-23T20:50:16.071 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.072 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.072 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:16.072 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:18.688813+0000", 2023-09-23T20:50:16.072 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.021259726999999999 2023-09-23T20:50:16.072 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.072 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.072 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:16.072 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:18.688941+0000", 2023-09-23T20:50:16.072 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00012763799999999999 2023-09-23T20:50:16.072 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.072 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.072 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:50:16.073 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:18.688974+0000", 2023-09-23T20:50:16.073 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.2818000000000003e-05 2023-09-23T20:50:16.073 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.073 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.073 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:50:16.073 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:18.689000+0000", 2023-09-23T20:50:16.073 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.6100000000000001e-05 2023-09-23T20:50:16.073 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:50:16.073 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:50:16.073 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:50:16.073 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.073 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.074 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:102274 10.c 10:31fbc5f2:::benchmark_data_smithi018_422431_object102273:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e551)", 2023-09-23T20:50:16.074 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:47:08.160645+0000", 2023-09-23T20:50:16.074 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 187.870393248, 2023-09-23T20:50:16.074 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1493982, 2023-09-23T20:50:16.074 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:50:16.074 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:50:16.074 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:50:16.074 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:50:16.074 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:50:16.074 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 102274 2023-09-23T20:50:16.074 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.074 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:50:16.075 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.075 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:50:16.075 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.160645+0000", 2023-09-23T20:50:16.075 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:50:16.075 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.075 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.075 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:50:16.075 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.160645+0000", 2023-09-23T20:50:16.075 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:50:16.075 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.075 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.076 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:50:16.076 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.160644+0000", 2023-09-23T20:50:16.076 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:50:16.076 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.076 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.076 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:50:16.076 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.160692+0000", 2023-09-23T20:50:16.076 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.8096999999999997e-05 2023-09-23T20:50:16.076 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.076 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.076 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:50:16.076 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.160712+0000", 2023-09-23T20:50:16.076 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.0421e-05 2023-09-23T20:50:16.077 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.077 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.077 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:50:16.077 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.160720+0000", 2023-09-23T20:50:16.077 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.7980000000000006e-06 2023-09-23T20:50:16.077 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.077 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.077 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:50:16.077 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.160777+0000", 2023-09-23T20:50:16.077 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.6731999999999998e-05 2023-09-23T20:50:16.077 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.078 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.078 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:50:16.078 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.160957+0000", 2023-09-23T20:50:16.078 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00018039999999999999 2023-09-23T20:50:16.078 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.078 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.078 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:50:16.078 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.161161+0000", 2023-09-23T20:50:16.078 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00020371300000000001 2023-09-23T20:50:16.078 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.078 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.078 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:50:16.079 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.169571+0000", 2023-09-23T20:50:16.079 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0084097330000000008 2023-09-23T20:50:16.079 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.079 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.079 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:50:16.079 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.187438+0000", 2023-09-23T20:50:16.079 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.017867623999999999 2023-09-23T20:50:16.079 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.079 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.079 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:50:16.079 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.187453+0000", 2023-09-23T20:50:16.080 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.5238e-05 2023-09-23T20:50:16.080 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.080 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.080 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:50:16.080 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.292752+0000", 2023-09-23T20:50:16.080 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.105298715 2023-09-23T20:50:16.080 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.080 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.080 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for degraded object", 2023-09-23T20:50:16.080 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.293574+0000", 2023-09-23T20:50:16.080 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00082229000000000004 2023-09-23T20:50:16.080 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.081 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.081 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:50:16.081 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.309964+0000", 2023-09-23T20:50:16.081 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.016389821999999998 2023-09-23T20:50:16.081 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.081 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.081 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:50:16.081 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.310043+0000", 2023-09-23T20:50:16.081 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.9184000000000006e-05 2023-09-23T20:50:16.081 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:50:16.081 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:50:16.081 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:50:16.082 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.082 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.082 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:102286 10.c 10:391cdcfd:::benchmark_data_smithi018_422431_object102285:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e552)", 2023-09-23T20:50:16.082 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:47:08.205082+0000", 2023-09-23T20:50:16.082 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 187.82595612399999, 2023-09-23T20:50:16.082 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1143086929999999, 2023-09-23T20:50:16.082 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:50:16.082 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:50:16.082 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:50:16.082 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:50:16.082 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:50:16.083 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 102286 2023-09-23T20:50:16.083 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.083 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:50:16.083 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.083 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:50:16.083 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.205082+0000", 2023-09-23T20:50:16.083 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:50:16.083 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.083 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.084 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:50:16.084 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.205082+0000", 2023-09-23T20:50:16.084 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:50:16.084 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.084 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.084 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:50:16.084 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.205081+0000", 2023-09-23T20:50:16.084 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:50:16.084 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.084 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.084 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:50:16.084 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.205162+0000", 2023-09-23T20:50:16.085 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.0582999999999997e-05 2023-09-23T20:50:16.085 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.085 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.085 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:50:16.085 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.205177+0000", 2023-09-23T20:50:16.085 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.5037e-05 2023-09-23T20:50:16.085 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.085 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.085 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:50:16.085 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.205183+0000", 2023-09-23T20:50:16.085 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.9009999999999999e-06 2023-09-23T20:50:16.086 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.086 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.086 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:50:16.086 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.205241+0000", 2023-09-23T20:50:16.086 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.7921999999999999e-05 2023-09-23T20:50:16.086 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.086 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.086 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:50:16.086 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.205254+0000", 2023-09-23T20:50:16.086 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.3049e-05 2023-09-23T20:50:16.086 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.086 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.087 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:50:16.087 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.294002+0000", 2023-09-23T20:50:16.087 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0887488059999999 2023-09-23T20:50:16.087 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.087 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.087 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:50:16.087 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.294127+0000", 2023-09-23T20:50:16.087 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000124201 2023-09-23T20:50:16.087 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.087 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.087 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,4", 2023-09-23T20:50:16.088 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.294300+0000", 2023-09-23T20:50:16.088 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000173053 2023-09-23T20:50:16.088 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.088 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.088 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:16.088 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.309385+0000", 2023-09-23T20:50:16.088 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.015085698 2023-09-23T20:50:16.088 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.088 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.088 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:50:16.088 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.310181+0000", 2023-09-23T20:50:16.088 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00079541699999999998 2023-09-23T20:50:16.089 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.089 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.089 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:16.089 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.319022+0000", 2023-09-23T20:50:16.089 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0088413550000000004 2023-09-23T20:50:16.089 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.089 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.089 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:50:16.089 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.319070+0000", 2023-09-23T20:50:16.089 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.8233999999999999e-05 2023-09-23T20:50:16.089 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.090 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.090 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:50:16.090 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.319391+0000", 2023-09-23T20:50:16.090 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00032073099999999999 2023-09-23T20:50:16.090 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:50:16.090 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:50:16.090 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:50:16.090 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.090 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.090 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:102289 10.c 10:3e772927:::benchmark_data_smithi018_422431_object102288:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e552)", 2023-09-23T20:50:16.090 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:47:08.205193+0000", 2023-09-23T20:50:16.091 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 187.82584522499999, 2023-09-23T20:50:16.091 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.114489624, 2023-09-23T20:50:16.091 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:50:16.091 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:50:16.091 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:50:16.091 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:50:16.091 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:50:16.091 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 102289 2023-09-23T20:50:16.091 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.091 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:50:16.091 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.091 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:50:16.092 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.205193+0000", 2023-09-23T20:50:16.092 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:50:16.092 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.092 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.092 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:50:16.092 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.205193+0000", 2023-09-23T20:50:16.092 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:50:16.092 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.092 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.092 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:50:16.092 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.205193+0000", 2023-09-23T20:50:16.093 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:50:16.093 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.093 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.093 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:50:16.093 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.205231+0000", 2023-09-23T20:50:16.093 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.8189999999999999e-05 2023-09-23T20:50:16.093 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.093 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.093 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:50:16.093 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.205244+0000", 2023-09-23T20:50:16.093 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.3067999999999999e-05 2023-09-23T20:50:16.093 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.094 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.094 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:50:16.094 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.205249+0000", 2023-09-23T20:50:16.094 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.1449999999999999e-06 2023-09-23T20:50:16.094 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.094 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.094 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:50:16.094 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.205277+0000", 2023-09-23T20:50:16.094 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.7350000000000001e-05 2023-09-23T20:50:16.094 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.094 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.095 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:50:16.095 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.205281+0000", 2023-09-23T20:50:16.095 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.5809999999999998e-06 2023-09-23T20:50:16.095 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.095 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.095 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:50:16.095 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.298397+0000", 2023-09-23T20:50:16.095 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.093115992 2023-09-23T20:50:16.095 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.095 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.095 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:50:16.095 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.298529+0000", 2023-09-23T20:50:16.096 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00013224499999999999 2023-09-23T20:50:16.096 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.096 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.096 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,4", 2023-09-23T20:50:16.096 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.298698+0000", 2023-09-23T20:50:16.096 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000169123 2023-09-23T20:50:16.096 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.096 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.096 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:16.096 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.310075+0000", 2023-09-23T20:50:16.096 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.011376282 2023-09-23T20:50:16.096 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.097 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.097 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:50:16.097 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.310186+0000", 2023-09-23T20:50:16.097 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00011142 2023-09-23T20:50:16.097 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.097 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.097 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:16.097 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.319436+0000", 2023-09-23T20:50:16.097 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0092496220000000007 2023-09-23T20:50:16.097 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.097 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.097 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:50:16.098 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.319488+0000", 2023-09-23T20:50:16.098 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.2408999999999999e-05 2023-09-23T20:50:16.098 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.098 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.098 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:50:16.098 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.319683+0000", 2023-09-23T20:50:16.098 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00019472099999999999 2023-09-23T20:50:16.098 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:50:16.098 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:50:16.098 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:50:16.098 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.098 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.099 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:102292 10.c 10:376f4c97:::benchmark_data_smithi018_422431_object102291:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e552)", 2023-09-23T20:50:16.099 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:47:08.212988+0000", 2023-09-23T20:50:16.099 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 187.81805056300001, 2023-09-23T20:50:16.099 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1069731110000001, 2023-09-23T20:50:16.099 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:50:16.099 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:50:16.099 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:50:16.099 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:50:16.099 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:50:16.099 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 102292 2023-09-23T20:50:16.099 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.099 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:50:16.100 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.100 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:50:16.100 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.212988+0000", 2023-09-23T20:50:16.100 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:50:16.100 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.100 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.100 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:50:16.100 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.212988+0000", 2023-09-23T20:50:16.100 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:50:16.100 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.100 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.101 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:50:16.101 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.212986+0000", 2023-09-23T20:50:16.101 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999981 2023-09-23T20:50:16.101 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.101 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.101 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:50:16.101 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.213032+0000", 2023-09-23T20:50:16.101 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.6239000000000002e-05 2023-09-23T20:50:16.101 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.101 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.101 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:50:16.101 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.213050+0000", 2023-09-23T20:50:16.102 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.7575e-05 2023-09-23T20:50:16.102 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.102 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.102 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:50:16.102 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.213058+0000", 2023-09-23T20:50:16.102 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.3399999999999998e-06 2023-09-23T20:50:16.102 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.102 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.102 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:50:16.102 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.213122+0000", 2023-09-23T20:50:16.102 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.3418999999999997e-05 2023-09-23T20:50:16.102 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.103 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.103 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:50:16.103 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.213134+0000", 2023-09-23T20:50:16.103 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.2356e-05 2023-09-23T20:50:16.103 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.103 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.103 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:50:16.103 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.302309+0000", 2023-09-23T20:50:16.103 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.089175199 2023-09-23T20:50:16.103 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.103 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.103 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:50:16.104 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.302439+0000", 2023-09-23T20:50:16.104 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00012961 2023-09-23T20:50:16.104 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.104 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.104 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,4", 2023-09-23T20:50:16.104 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.302598+0000", 2023-09-23T20:50:16.104 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00015949599999999999 2023-09-23T20:50:16.104 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.104 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.104 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:16.104 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.310102+0000", 2023-09-23T20:50:16.104 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0075032220000000004 2023-09-23T20:50:16.105 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.105 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.105 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:50:16.105 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.315572+0000", 2023-09-23T20:50:16.105 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0054703499999999997 2023-09-23T20:50:16.105 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.105 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.105 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:16.105 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.319723+0000", 2023-09-23T20:50:16.105 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0041513679999999999 2023-09-23T20:50:16.105 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.106 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.106 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:50:16.106 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.319778+0000", 2023-09-23T20:50:16.106 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.4908999999999999e-05 2023-09-23T20:50:16.106 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.106 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.106 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:50:16.106 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.319961+0000", 2023-09-23T20:50:16.106 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00018275499999999999 2023-09-23T20:50:16.106 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:50:16.106 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:50:16.106 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:50:16.107 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.107 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.107 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:158988 10.9 10:9fe441aa:::benchmark_data_smithi018_422431_object54492:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e577)", 2023-09-23T20:50:16.107 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:48:22.900242+0000", 2023-09-23T20:50:16.107 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 113.13079681400001, 2023-09-23T20:50:16.107 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.295998714, 2023-09-23T20:50:16.107 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:50:16.107 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:50:16.107 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:50:16.107 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:50:16.107 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:50:16.107 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 158988 2023-09-23T20:50:16.107 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.108 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:50:16.108 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.108 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:50:16.108 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.900242+0000", 2023-09-23T20:50:16.108 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:50:16.108 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.108 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.108 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:50:16.108 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.900242+0000", 2023-09-23T20:50:16.108 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:50:16.108 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.108 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.109 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:50:16.109 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.900241+0000", 2023-09-23T20:50:16.109 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:50:16.109 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.109 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.109 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:50:16.109 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.900245+0000", 2023-09-23T20:50:16.109 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.3710000000000002e-06 2023-09-23T20:50:16.109 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.109 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.109 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:50:16.109 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.900258+0000", 2023-09-23T20:50:16.110 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.3018e-05 2023-09-23T20:50:16.110 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.110 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.110 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:50:16.110 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.900265+0000", 2023-09-23T20:50:16.110 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.5899999999999996e-06 2023-09-23T20:50:16.110 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.110 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.110 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:50:16.110 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.908024+0000", 2023-09-23T20:50:16.110 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0077597910000000003 2023-09-23T20:50:16.110 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.111 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.111 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:50:16.111 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.908034+0000", 2023-09-23T20:50:16.111 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.7820000000000005e-06 2023-09-23T20:50:16.111 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.111 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.111 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:50:16.111 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:23.907026+0000", 2023-09-23T20:50:16.111 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.99899201500000001 2023-09-23T20:50:16.111 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.111 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.111 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:50:16.111 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:23.907037+0000", 2023-09-23T20:50:16.112 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0468999999999999e-05 2023-09-23T20:50:16.112 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.112 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.112 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:50:16.112 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:25.196139+0000", 2023-09-23T20:50:16.112 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.2891025380000001 2023-09-23T20:50:16.112 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.112 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.112 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:50:16.112 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:25.196240+0000", 2023-09-23T20:50:16.112 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00010123 2023-09-23T20:50:16.113 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:50:16.113 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:50:16.113 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:50:16.113 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.113 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.113 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:158996 10.9 10:9667fd03:::benchmark_data_smithi018_422431_object54500:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e577)", 2023-09-23T20:50:16.113 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:48:22.900270+0000", 2023-09-23T20:50:16.113 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 113.130768562, 2023-09-23T20:50:16.113 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.343689795, 2023-09-23T20:50:16.113 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:50:16.113 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:50:16.113 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:50:16.114 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:50:16.114 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:50:16.114 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 158996 2023-09-23T20:50:16.114 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.114 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:50:16.114 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.114 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:50:16.114 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.900270+0000", 2023-09-23T20:50:16.114 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:50:16.114 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.115 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.115 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:50:16.115 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.900270+0000", 2023-09-23T20:50:16.115 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:50:16.115 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.115 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.115 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:50:16.115 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.900270+0000", 2023-09-23T20:50:16.115 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:50:16.115 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.115 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.116 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:50:16.116 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.900272+0000", 2023-09-23T20:50:16.116 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.232e-06 2023-09-23T20:50:16.116 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.116 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.116 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:50:16.116 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.900279+0000", 2023-09-23T20:50:16.116 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.9369999999999998e-06 2023-09-23T20:50:16.116 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.116 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.116 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:50:16.117 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.900282+0000", 2023-09-23T20:50:16.117 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.4259999999999999e-06 2023-09-23T20:50:16.117 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.117 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.117 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:50:16.117 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.908056+0000", 2023-09-23T20:50:16.117 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0077741520000000003 2023-09-23T20:50:16.117 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.117 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.117 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:50:16.117 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.908064+0000", 2023-09-23T20:50:16.117 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.1729999999999998e-06 2023-09-23T20:50:16.117 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.118 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.118 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:50:16.118 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:23.907060+0000", 2023-09-23T20:50:16.118 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.99899634999999998 2023-09-23T20:50:16.118 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.118 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.118 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:50:16.118 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:23.907067+0000", 2023-09-23T20:50:16.118 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.8510000000000001e-06 2023-09-23T20:50:16.118 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.118 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.119 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:50:16.119 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:25.196528+0000", 2023-09-23T20:50:16.119 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.289461322 2023-09-23T20:50:16.119 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.119 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.119 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for missing object", 2023-09-23T20:50:16.119 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:25.196641+0000", 2023-09-23T20:50:16.119 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000112619 2023-09-23T20:50:16.119 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.119 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.119 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:50:16.119 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:25.243873+0000", 2023-09-23T20:50:16.120 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.047232622000000002 2023-09-23T20:50:16.120 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.120 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.120 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:50:16.120 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:25.243960+0000", 2023-09-23T20:50:16.120 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.6462e-05 2023-09-23T20:50:16.120 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:50:16.120 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:50:16.120 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:50:16.120 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.120 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.120 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:159015 10.9 10:914d252e:::benchmark_data_smithi018_422431_object54519:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e577)", 2023-09-23T20:50:16.121 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:48:22.900286+0000", 2023-09-23T20:50:16.121 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 113.130752687, 2023-09-23T20:50:16.121 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.3515420869999999, 2023-09-23T20:50:16.121 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:50:16.121 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:50:16.121 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:50:16.121 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:50:16.121 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:50:16.121 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 159015 2023-09-23T20:50:16.121 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.121 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:50:16.121 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.122 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:50:16.122 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.900286+0000", 2023-09-23T20:50:16.122 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:50:16.122 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.122 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.122 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:50:16.122 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.900286+0000", 2023-09-23T20:50:16.122 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:50:16.122 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.122 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.122 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:50:16.122 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.900286+0000", 2023-09-23T20:50:16.123 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:50:16.123 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.123 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.123 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:50:16.123 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.900287+0000", 2023-09-23T20:50:16.123 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.9379999999999999e-06 2023-09-23T20:50:16.123 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.123 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.123 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:50:16.123 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.900293+0000", 2023-09-23T20:50:16.123 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.1939999999999997e-06 2023-09-23T20:50:16.123 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.124 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.124 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:50:16.124 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.900296+0000", 2023-09-23T20:50:16.124 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.1970000000000001e-06 2023-09-23T20:50:16.124 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.124 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.124 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:50:16.124 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.908083+0000", 2023-09-23T20:50:16.124 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.007787202 2023-09-23T20:50:16.124 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.124 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.124 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:50:16.125 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.908089+0000", 2023-09-23T20:50:16.125 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.1210000000000003e-06 2023-09-23T20:50:16.125 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.125 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.125 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:50:16.125 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:23.907087+0000", 2023-09-23T20:50:16.125 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.998997468 2023-09-23T20:50:16.125 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.125 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.125 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:50:16.125 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:23.907094+0000", 2023-09-23T20:50:16.125 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.1269999999999997e-06 2023-09-23T20:50:16.126 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.126 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.126 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:50:16.126 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:25.196671+0000", 2023-09-23T20:50:16.126 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.2895769530000001 2023-09-23T20:50:16.126 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.126 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.126 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for missing object", 2023-09-23T20:50:16.126 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:25.196749+0000", 2023-09-23T20:50:16.126 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.8147999999999998e-05 2023-09-23T20:50:16.126 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.126 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.127 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:50:16.127 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:25.251766+0000", 2023-09-23T20:50:16.127 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.055017165 2023-09-23T20:50:16.127 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.127 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.127 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:50:16.127 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:25.251828+0000", 2023-09-23T20:50:16.127 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.1891999999999996e-05 2023-09-23T20:50:16.127 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:50:16.127 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:50:16.127 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:50:16.127 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.128 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.128 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:159059 10.9 10:91c42c8b:::benchmark_data_smithi018_422431_object54563:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e577)", 2023-09-23T20:50:16.128 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:48:22.979021+0000", 2023-09-23T20:50:16.128 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 113.05201793000001, 2023-09-23T20:50:16.128 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.2786466750000001, 2023-09-23T20:50:16.128 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:50:16.128 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:50:16.128 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:50:16.128 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:50:16.128 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:50:16.128 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 159059 2023-09-23T20:50:16.129 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.129 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:50:16.129 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.129 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:50:16.129 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.979021+0000", 2023-09-23T20:50:16.129 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:50:16.129 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.129 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.129 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:50:16.129 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.979021+0000", 2023-09-23T20:50:16.129 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:50:16.130 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.130 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.130 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:50:16.130 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.979019+0000", 2023-09-23T20:50:16.130 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:50:16.130 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.130 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.130 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:50:16.130 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.979026+0000", 2023-09-23T20:50:16.130 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.3620000000000002e-06 2023-09-23T20:50:16.130 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.130 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.131 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:50:16.131 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.979040+0000", 2023-09-23T20:50:16.131 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.4419e-05 2023-09-23T20:50:16.131 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.131 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.131 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:50:16.131 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.979048+0000", 2023-09-23T20:50:16.131 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.7360000000000001e-06 2023-09-23T20:50:16.131 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.131 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.131 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:50:16.132 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.979099+0000", 2023-09-23T20:50:16.132 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.0980000000000003e-05 2023-09-23T20:50:16.132 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.132 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.132 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:50:16.132 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.979108+0000", 2023-09-23T20:50:16.132 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.1619999999999997e-06 2023-09-23T20:50:16.132 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.132 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.132 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:50:16.132 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:23.907115+0000", 2023-09-23T20:50:16.132 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.92800715899999997 2023-09-23T20:50:16.133 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.133 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.133 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:50:16.133 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:23.907122+0000", 2023-09-23T20:50:16.133 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.8959999999999997e-06 2023-09-23T20:50:16.133 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.133 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.133 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:50:16.133 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:25.196779+0000", 2023-09-23T20:50:16.133 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.2896565419999999 2023-09-23T20:50:16.133 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.133 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.133 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:50:16.134 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:25.196960+0000", 2023-09-23T20:50:16.134 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000181824 2023-09-23T20:50:16.134 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.134 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.134 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 2,4,5", 2023-09-23T20:50:16.134 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:25.198030+0000", 2023-09-23T20:50:16.134 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0010695349999999999 2023-09-23T20:50:16.134 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.134 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.134 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:16.134 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:25.243692+0000", 2023-09-23T20:50:16.135 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.045662454999999998 2023-09-23T20:50:16.135 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.135 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.135 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:50:16.135 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:25.243701+0000", 2023-09-23T20:50:16.135 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.5120000000000008e-06 2023-09-23T20:50:16.135 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.135 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.135 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:16.135 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:25.244025+0000", 2023-09-23T20:50:16.135 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00032403299999999999 2023-09-23T20:50:16.136 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.136 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.136 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:16.136 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:25.257506+0000", 2023-09-23T20:50:16.136 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.013481388 2023-09-23T20:50:16.136 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.136 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.136 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:50:16.136 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:25.257558+0000", 2023-09-23T20:50:16.136 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.1554000000000001e-05 2023-09-23T20:50:16.136 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.136 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.136 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:50:16.137 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:25.257667+0000", 2023-09-23T20:50:16.137 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000109456 2023-09-23T20:50:16.137 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:50:16.137 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:50:16.137 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:50:16.137 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.137 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.137 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:174186 10.19 10:99f413a8:::benchmark_data_smithi018_422431_object69690:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e590)", 2023-09-23T20:50:16.137 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:48:47.116428+0000", 2023-09-23T20:50:16.137 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 88.914610416000002, 2023-09-23T20:50:16.137 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0356085479999999, 2023-09-23T20:50:16.137 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:50:16.138 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:50:16.138 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:50:16.138 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:50:16.138 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:50:16.138 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 174186 2023-09-23T20:50:16.138 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.138 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:50:16.138 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.138 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:50:16.138 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:47.116428+0000", 2023-09-23T20:50:16.138 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:50:16.138 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.139 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.139 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:50:16.139 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:47.116428+0000", 2023-09-23T20:50:16.139 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:50:16.139 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.139 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.139 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:50:16.139 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:47.116427+0000", 2023-09-23T20:50:16.139 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:50:16.139 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.139 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.140 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:50:16.140 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:47.116433+0000", 2023-09-23T20:50:16.140 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.3589999999999996e-06 2023-09-23T20:50:16.140 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.140 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.140 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:50:16.140 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:47.116450+0000", 2023-09-23T20:50:16.140 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.7334e-05 2023-09-23T20:50:16.140 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.140 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.140 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:50:16.140 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:47.116457+0000", 2023-09-23T20:50:16.141 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.6359999999999997e-06 2023-09-23T20:50:16.141 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.141 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.141 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:50:16.141 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:47.116502+0000", 2023-09-23T20:50:16.141 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.4987000000000001e-05 2023-09-23T20:50:16.141 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.141 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.141 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:50:16.141 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:47.116680+0000", 2023-09-23T20:50:16.141 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00017807599999999999 2023-09-23T20:50:16.142 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.142 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.142 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 2,4,5", 2023-09-23T20:50:16.142 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:47.117094+0000", 2023-09-23T20:50:16.142 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00041400999999999997 2023-09-23T20:50:16.142 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.142 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.142 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:16.142 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:47.130634+0000", 2023-09-23T20:50:16.142 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.013540533 2023-09-23T20:50:16.142 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.142 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.143 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:50:16.143 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:47.130642+0000", 2023-09-23T20:50:16.143 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.0229999999999992e-06 2023-09-23T20:50:16.143 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.143 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.143 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:16.143 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:47.136723+0000", 2023-09-23T20:50:16.143 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0060805249999999998 2023-09-23T20:50:16.143 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.143 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.143 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:50:16.143 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:47.146513+0000", 2023-09-23T20:50:16.143 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0097900049999999992 2023-09-23T20:50:16.144 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.144 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.144 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:50:16.144 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:47.146525+0000", 2023-09-23T20:50:16.144 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.2316e-05 2023-09-23T20:50:16.144 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.144 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.144 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:50:16.144 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:48.152024+0000", 2023-09-23T20:50:16.144 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.005498185 2023-09-23T20:50:16.144 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.144 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.145 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:50:16.145 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:48.152037+0000", 2023-09-23T20:50:16.145 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.3127e-05 2023-09-23T20:50:16.145 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:50:16.145 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:50:16.145 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:50:16.145 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.145 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.145 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:181716 10.c 10:3b20cb1c:::benchmark_data_smithi018_422431_object77220:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e604)", 2023-09-23T20:50:16.145 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:49:02.794588+0000", 2023-09-23T20:50:16.145 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 73.236450242999993, 2023-09-23T20:50:16.146 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1313792709999999, 2023-09-23T20:50:16.146 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:50:16.146 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:50:16.146 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:50:16.146 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:50:16.146 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:50:16.146 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 181716 2023-09-23T20:50:16.146 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.146 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:50:16.146 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.146 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:50:16.147 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:02.794588+0000", 2023-09-23T20:50:16.147 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:50:16.147 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.147 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.147 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:50:16.147 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:02.794588+0000", 2023-09-23T20:50:16.147 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:50:16.147 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.147 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.147 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:50:16.147 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:02.794587+0000", 2023-09-23T20:50:16.147 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999981 2023-09-23T20:50:16.148 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.148 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.148 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:50:16.148 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:02.794596+0000", 2023-09-23T20:50:16.148 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.9420000000000007e-06 2023-09-23T20:50:16.148 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.148 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.148 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:50:16.148 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:02.794620+0000", 2023-09-23T20:50:16.148 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.3161000000000001e-05 2023-09-23T20:50:16.148 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.149 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.149 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:50:16.149 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:02.794629+0000", 2023-09-23T20:50:16.149 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.7329999999999998e-06 2023-09-23T20:50:16.149 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.149 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.149 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:50:16.149 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:02.794693+0000", 2023-09-23T20:50:16.149 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.3558000000000004e-05 2023-09-23T20:50:16.149 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.149 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.150 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:50:16.150 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:02.794702+0000", 2023-09-23T20:50:16.150 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.6850000000000007e-06 2023-09-23T20:50:16.150 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.150 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.150 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:50:16.150 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:03.886841+0000", 2023-09-23T20:50:16.150 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0921389640000001 2023-09-23T20:50:16.150 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.150 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.150 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:50:16.151 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:03.886964+0000", 2023-09-23T20:50:16.151 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000122951 2023-09-23T20:50:16.151 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.151 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.151 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,1,4", 2023-09-23T20:50:16.151 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:03.887573+0000", 2023-09-23T20:50:16.151 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00060992900000000005 2023-09-23T20:50:16.151 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.151 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.151 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:50:16.152 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:03.899773+0000", 2023-09-23T20:50:16.152 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.012199750000000001 2023-09-23T20:50:16.152 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.152 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.152 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:16.152 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:03.899930+0000", 2023-09-23T20:50:16.152 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00015683900000000001 2023-09-23T20:50:16.152 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.152 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.152 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:16.152 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:03.905030+0000", 2023-09-23T20:50:16.152 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0050999790000000001 2023-09-23T20:50:16.152 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.153 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.153 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:16.153 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:03.925775+0000", 2023-09-23T20:50:16.153 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.020745320000000001 2023-09-23T20:50:16.153 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.153 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.153 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:50:16.153 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:03.925836+0000", 2023-09-23T20:50:16.153 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.0853000000000002e-05 2023-09-23T20:50:16.153 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.153 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.153 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:50:16.154 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:03.925968+0000", 2023-09-23T20:50:16.154 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00013135999999999999 2023-09-23T20:50:16.154 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:50:16.154 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:50:16.154 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:50:16.154 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.154 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.154 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:181719 10.c 10:3f881ee1:::benchmark_data_smithi018_422431_object77223:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e604)", 2023-09-23T20:50:16.154 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:49:02.797308+0000", 2023-09-23T20:50:16.154 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 73.233730914999995, 2023-09-23T20:50:16.154 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1289105109999999, 2023-09-23T20:50:16.155 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:50:16.155 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:50:16.155 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:50:16.155 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:50:16.155 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:50:16.155 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 181719 2023-09-23T20:50:16.155 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.155 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:50:16.155 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.155 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:50:16.155 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:02.797308+0000", 2023-09-23T20:50:16.156 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:50:16.156 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.156 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.156 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:50:16.156 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:02.797308+0000", 2023-09-23T20:50:16.156 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:50:16.156 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.156 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.156 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:50:16.156 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:02.797307+0000", 2023-09-23T20:50:16.156 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:50:16.157 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.157 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.157 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:50:16.157 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:02.797312+0000", 2023-09-23T20:50:16.157 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.989e-06 2023-09-23T20:50:16.157 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.157 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.157 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:50:16.157 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:02.797334+0000", 2023-09-23T20:50:16.157 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.2416999999999999e-05 2023-09-23T20:50:16.157 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.158 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.158 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:50:16.158 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:02.797342+0000", 2023-09-23T20:50:16.158 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.5120000000000002e-06 2023-09-23T20:50:16.158 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.158 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.158 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:50:16.158 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:02.797395+0000", 2023-09-23T20:50:16.158 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.2951000000000002e-05 2023-09-23T20:50:16.158 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.158 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.158 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:50:16.159 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:02.797407+0000", 2023-09-23T20:50:16.159 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.2103000000000001e-05 2023-09-23T20:50:16.159 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.159 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.159 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:50:16.159 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:03.892091+0000", 2023-09-23T20:50:16.159 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0946837679999999 2023-09-23T20:50:16.159 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.159 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.159 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:50:16.159 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:03.892187+0000", 2023-09-23T20:50:16.159 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.6470000000000003e-05 2023-09-23T20:50:16.160 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.160 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.160 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,1,4", 2023-09-23T20:50:16.160 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:03.892266+0000", 2023-09-23T20:50:16.160 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.9254999999999999e-05 2023-09-23T20:50:16.160 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.160 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.160 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:50:16.160 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:03.899780+0000", 2023-09-23T20:50:16.160 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0075138289999999996 2023-09-23T20:50:16.160 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.160 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.161 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:16.161 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:03.905061+0000", 2023-09-23T20:50:16.161 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0052810770000000003 2023-09-23T20:50:16.161 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.161 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.161 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:16.161 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:03.905079+0000", 2023-09-23T20:50:16.161 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.7906000000000001e-05 2023-09-23T20:50:16.161 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.161 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.161 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:16.161 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:03.926035+0000", 2023-09-23T20:50:16.162 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.020956130999999999 2023-09-23T20:50:16.162 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.162 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.162 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:50:16.162 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:03.926093+0000", 2023-09-23T20:50:16.162 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.7944e-05 2023-09-23T20:50:16.162 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.162 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.162 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:50:16.162 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:03.926218+0000", 2023-09-23T20:50:16.162 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000124809 2023-09-23T20:50:16.163 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:50:16.163 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:50:16.163 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:50:16.163 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.163 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.163 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:181723 10.c 10:3851a352:::benchmark_data_smithi018_422431_object77227:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e604)", 2023-09-23T20:50:16.163 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:49:02.803937+0000", 2023-09-23T20:50:16.163 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 73.227101421, 2023-09-23T20:50:16.163 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.122505138, 2023-09-23T20:50:16.163 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:50:16.163 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:50:16.164 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:50:16.164 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:50:16.164 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:50:16.164 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 181723 2023-09-23T20:50:16.164 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.164 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:50:16.164 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.164 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:50:16.164 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:02.803937+0000", 2023-09-23T20:50:16.164 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:50:16.164 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.164 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.165 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:50:16.165 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:02.803937+0000", 2023-09-23T20:50:16.165 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:50:16.165 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.165 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.165 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:50:16.165 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:02.803936+0000", 2023-09-23T20:50:16.165 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:50:16.165 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.165 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.165 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:50:16.166 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:02.803941+0000", 2023-09-23T20:50:16.166 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.2809999999999999e-06 2023-09-23T20:50:16.166 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.166 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.166 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:50:16.166 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:02.803958+0000", 2023-09-23T20:50:16.166 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.6827e-05 2023-09-23T20:50:16.166 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.166 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.166 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:50:16.166 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:02.803964+0000", 2023-09-23T20:50:16.167 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.1970000000000001e-06 2023-09-23T20:50:16.167 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.167 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.167 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:50:16.167 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:02.804026+0000", 2023-09-23T20:50:16.167 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.1218999999999998e-05 2023-09-23T20:50:16.167 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.167 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.167 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:50:16.167 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:02.804041+0000", 2023-09-23T20:50:16.167 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.5138e-05 2023-09-23T20:50:16.167 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.168 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.168 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:50:16.168 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:03.894841+0000", 2023-09-23T20:50:16.168 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.090800038 2023-09-23T20:50:16.168 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.168 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.168 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:50:16.168 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:03.894931+0000", 2023-09-23T20:50:16.168 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.9925000000000004e-05 2023-09-23T20:50:16.168 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.168 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.169 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,1,4", 2023-09-23T20:50:16.169 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:03.895032+0000", 2023-09-23T20:50:16.169 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000100893 2023-09-23T20:50:16.169 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.169 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.169 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:50:16.169 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:03.899938+0000", 2023-09-23T20:50:16.169 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0049064119999999998 2023-09-23T20:50:16.169 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.169 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.169 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:16.170 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:03.905097+0000", 2023-09-23T20:50:16.170 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0051591639999999999 2023-09-23T20:50:16.170 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.170 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.170 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:16.170 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:03.911196+0000", 2023-09-23T20:50:16.170 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0060986790000000001 2023-09-23T20:50:16.170 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.170 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.170 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:16.170 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:03.926272+0000", 2023-09-23T20:50:16.171 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.015076473 2023-09-23T20:50:16.171 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.171 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.171 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:50:16.171 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:03.926334+0000", 2023-09-23T20:50:16.171 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.1849999999999999e-05 2023-09-23T20:50:16.171 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.171 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.171 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:50:16.171 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:03.926442+0000", 2023-09-23T20:50:16.171 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00010802300000000001 2023-09-23T20:50:16.171 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:50:16.172 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:50:16.172 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:50:16.172 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.172 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.172 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:181740 10.c 10:318e66ac:::benchmark_data_smithi018_422431_object77244:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e604)", 2023-09-23T20:50:16.172 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:49:02.823106+0000", 2023-09-23T20:50:16.172 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 73.207932284999998, 2023-09-23T20:50:16.172 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1035798889999999, 2023-09-23T20:50:16.172 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:50:16.172 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:50:16.172 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:50:16.173 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:50:16.173 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:50:16.173 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 181740 2023-09-23T20:50:16.173 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.173 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:50:16.173 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.173 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:50:16.173 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:02.823106+0000", 2023-09-23T20:50:16.173 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:50:16.173 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.173 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.174 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:50:16.174 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:02.823106+0000", 2023-09-23T20:50:16.174 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:50:16.174 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.174 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.174 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:50:16.174 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:02.823105+0000", 2023-09-23T20:50:16.174 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:50:16.174 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.174 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.174 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:50:16.175 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:02.823112+0000", 2023-09-23T20:50:16.175 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.4200000000000004e-06 2023-09-23T20:50:16.175 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.175 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.175 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:50:16.175 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:02.823129+0000", 2023-09-23T20:50:16.175 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.7167999999999999e-05 2023-09-23T20:50:16.175 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.175 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.175 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:50:16.175 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:02.823136+0000", 2023-09-23T20:50:16.175 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.9210000000000003e-06 2023-09-23T20:50:16.176 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.176 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.176 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:50:16.176 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:02.823202+0000", 2023-09-23T20:50:16.176 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.6533999999999997e-05 2023-09-23T20:50:16.176 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.176 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.176 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:50:16.176 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:02.823216+0000", 2023-09-23T20:50:16.176 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.3587000000000001e-05 2023-09-23T20:50:16.176 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.176 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.176 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:50:16.177 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:03.897351+0000", 2023-09-23T20:50:16.177 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0741351130000001 2023-09-23T20:50:16.177 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.177 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.177 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:50:16.177 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:03.897436+0000", 2023-09-23T20:50:16.177 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.5368e-05 2023-09-23T20:50:16.177 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.177 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.177 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,1,4", 2023-09-23T20:50:16.177 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:03.897508+0000", 2023-09-23T20:50:16.177 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.1768999999999997e-05 2023-09-23T20:50:16.178 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.178 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.178 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:16.178 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:03.905122+0000", 2023-09-23T20:50:16.178 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0076139989999999998 2023-09-23T20:50:16.178 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.178 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.178 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:50:16.178 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:03.910985+0000", 2023-09-23T20:50:16.178 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0058633829999999998 2023-09-23T20:50:16.178 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.178 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.179 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:16.179 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:03.911313+0000", 2023-09-23T20:50:16.179 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00032725300000000001 2023-09-23T20:50:16.179 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.179 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.179 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:16.179 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:03.926509+0000", 2023-09-23T20:50:16.179 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.015196397 2023-09-23T20:50:16.179 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.179 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.179 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:50:16.179 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:03.926553+0000", 2023-09-23T20:50:16.179 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.3553999999999997e-05 2023-09-23T20:50:16.180 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.180 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.180 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:50:16.180 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:03.926686+0000", 2023-09-23T20:50:16.180 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000133535 2023-09-23T20:50:16.180 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:50:16.180 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:50:16.180 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:50:16.180 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.180 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.180 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:181752 10.c 10:34fae4dd:::benchmark_data_smithi018_422431_object77256:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e604)", 2023-09-23T20:50:16.180 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:49:02.837351+0000", 2023-09-23T20:50:16.180 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 73.193687371999999, 2023-09-23T20:50:16.181 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0895556049999999, 2023-09-23T20:50:16.181 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:50:16.181 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:50:16.181 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:50:16.181 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:50:16.181 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:50:16.181 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 181752 2023-09-23T20:50:16.181 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.181 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:50:16.181 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.181 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:50:16.182 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:02.837351+0000", 2023-09-23T20:50:16.182 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:50:16.182 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.182 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.182 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:50:16.182 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:02.837351+0000", 2023-09-23T20:50:16.182 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:50:16.182 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.182 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.182 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:50:16.182 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:02.837350+0000", 2023-09-23T20:50:16.182 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:50:16.183 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.183 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.183 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:50:16.183 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:02.837355+0000", 2023-09-23T20:50:16.183 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.853e-06 2023-09-23T20:50:16.183 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.183 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.183 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:50:16.183 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:02.837368+0000", 2023-09-23T20:50:16.183 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.2765999999999999e-05 2023-09-23T20:50:16.183 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.184 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.184 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:50:16.184 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:02.837372+0000", 2023-09-23T20:50:16.184 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.5569999999999997e-06 2023-09-23T20:50:16.184 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.184 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.184 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:50:16.184 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:02.837424+0000", 2023-09-23T20:50:16.184 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.2101999999999997e-05 2023-09-23T20:50:16.184 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.184 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.184 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:50:16.185 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:02.837434+0000", 2023-09-23T20:50:16.185 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.4979999999999995e-06 2023-09-23T20:50:16.185 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.185 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.185 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:50:16.185 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:03.899956+0000", 2023-09-23T20:50:16.185 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0625220049999999 2023-09-23T20:50:16.185 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.185 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.185 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:50:16.185 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:03.900040+0000", 2023-09-23T20:50:16.185 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.3545000000000001e-05 2023-09-23T20:50:16.186 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.186 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.186 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,1,4", 2023-09-23T20:50:16.186 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:03.900126+0000", 2023-09-23T20:50:16.186 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.6438000000000001e-05 2023-09-23T20:50:16.186 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.186 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.186 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:16.186 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:03.910971+0000", 2023-09-23T20:50:16.186 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.010845237000000001 2023-09-23T20:50:16.186 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.186 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.187 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:50:16.187 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:03.910994+0000", 2023-09-23T20:50:16.187 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.2642000000000001e-05 2023-09-23T20:50:16.187 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.187 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.187 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:16.187 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:03.911347+0000", 2023-09-23T20:50:16.187 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00035345800000000002 2023-09-23T20:50:16.187 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.187 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.187 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:16.188 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:03.926751+0000", 2023-09-23T20:50:16.188 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.015403795 2023-09-23T20:50:16.188 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.188 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.188 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:50:16.188 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:03.926796+0000", 2023-09-23T20:50:16.188 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.4443999999999997e-05 2023-09-23T20:50:16.188 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.188 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.188 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:50:16.188 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:03.926907+0000", 2023-09-23T20:50:16.188 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00011123700000000001 2023-09-23T20:50:16.189 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:50:16.189 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:50:16.189 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:50:16.189 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.189 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.189 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:181774 10.c 10:32e12796:::benchmark_data_smithi018_422431_object77278:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e604)", 2023-09-23T20:50:16.189 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:49:02.870850+0000", 2023-09-23T20:50:16.189 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 73.160188149000007, 2023-09-23T20:50:16.189 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.056271878, 2023-09-23T20:50:16.189 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:50:16.189 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:50:16.190 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:50:16.190 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:50:16.190 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:50:16.190 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 181774 2023-09-23T20:50:16.190 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.190 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:50:16.190 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.190 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:50:16.190 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:02.870850+0000", 2023-09-23T20:50:16.190 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:50:16.190 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.191 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.191 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:50:16.191 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:02.870850+0000", 2023-09-23T20:50:16.191 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:50:16.191 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.191 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.191 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:50:16.191 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:02.870849+0000", 2023-09-23T20:50:16.191 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:50:16.191 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.191 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.192 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:50:16.192 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:02.870856+0000", 2023-09-23T20:50:16.192 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.3309999999999998e-06 2023-09-23T20:50:16.192 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.192 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.192 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:50:16.192 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:02.870874+0000", 2023-09-23T20:50:16.192 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.8117e-05 2023-09-23T20:50:16.192 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.192 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.192 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:50:16.192 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:02.870883+0000", 2023-09-23T20:50:16.193 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.5229999999999995e-06 2023-09-23T20:50:16.193 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.193 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.193 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:50:16.193 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:02.870939+0000", 2023-09-23T20:50:16.193 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.6112000000000001e-05 2023-09-23T20:50:16.193 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.193 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.193 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:50:16.193 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:02.870950+0000", 2023-09-23T20:50:16.193 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0657e-05 2023-09-23T20:50:16.194 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.194 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.194 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:50:16.194 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:03.902547+0000", 2023-09-23T20:50:16.194 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.031597278 2023-09-23T20:50:16.194 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.194 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.194 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:50:16.194 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:03.902634+0000", 2023-09-23T20:50:16.194 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.7405999999999996e-05 2023-09-23T20:50:16.194 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.194 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.195 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,1,4", 2023-09-23T20:50:16.195 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:03.902744+0000", 2023-09-23T20:50:16.195 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000109512 2023-09-23T20:50:16.195 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.195 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.195 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:50:16.195 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:03.911210+0000", 2023-09-23T20:50:16.195 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0084662520000000005 2023-09-23T20:50:16.195 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.195 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.195 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:16.195 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:03.911289+0000", 2023-09-23T20:50:16.196 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.8888999999999996e-05 2023-09-23T20:50:16.196 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.196 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.196 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:16.196 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:03.922856+0000", 2023-09-23T20:50:16.196 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.011567424 2023-09-23T20:50:16.196 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.196 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.196 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:16.196 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:03.926949+0000", 2023-09-23T20:50:16.196 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0040922739999999999 2023-09-23T20:50:16.197 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.197 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.197 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:50:16.197 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:03.926993+0000", 2023-09-23T20:50:16.197 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.4132999999999999e-05 2023-09-23T20:50:16.197 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:50:16.197 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:50:16.197 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:50:16.197 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:03.927122+0000", 2023-09-23T20:50:16.197 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00012936800000000001 2023-09-23T20:50:16.197 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:50:16.197 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:50:16.197 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:50:16.198 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:50:16.198 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:50:16.198 INFO:teuthology.orchestra.run.smithi149.stdout:} 2023-09-23T20:50:16.198 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:50:16.199 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:16 smithi018 ceph-mon[184144]: pgmap v1589: 103 pgs: 103 active+clean; 4.6 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail 2023-09-23T20:50:16.199 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:16 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:50:16.199 INFO:teuthology.orchestra.run.smithi018.stderr:Error EINVAL: specified pgp_num 81 > pg_num 31 2023-09-23T20:50:16.229 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:16.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:16 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1836143011' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "81"}]: dispatch 2023-09-23T20:50:16.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:16 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "81"}]: dispatch 2023-09-23T20:50:16.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:16 smithi018 ceph-mon[179647]: pgmap v1589: 103 pgs: 103 active+clean; 4.6 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail 2023-09-23T20:50:16.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:16 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:50:16.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:16 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1836143011' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "81"}]: dispatch 2023-09-23T20:50:16.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:16 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "81"}]: dispatch 2023-09-23T20:50:16.430 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:16 smithi149 ceph-mon[170394]: pgmap v1589: 103 pgs: 103 active+clean; 4.6 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail 2023-09-23T20:50:16.431 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:16 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:50:16.431 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:16 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1836143011' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "81"}]: dispatch 2023-09-23T20:50:16.431 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:16 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "81"}]: dispatch 2023-09-23T20:50:16.432 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T20:50:16.622 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-09-23T20:50:16.650 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:50:16.653 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:16.654 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:50:16.755 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:50:16.784 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:16.995 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:50:16.998 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:16.998 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:50:17.009 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:50:17.099 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:50:17.132 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:17.142 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:50:17.175 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:50:17.178 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:17.179 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:50:17.280 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:50:17.311 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:17.394 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:50:17.397 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:17.398 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:50:17.499 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:50:17.534 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:17.571 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:50:17.574 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:17.574 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:50:17.675 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:50:17.703 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:17.737 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:50:17.740 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:17.740 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:50:17.841 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:50:17.867 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:17.900 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:50:17.903 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:17.904 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:50:18.004 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:50:18.030 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:18.064 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:50:18.066 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:18.067 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:50:18.068 INFO:teuthology.orchestra.run.smithi018.stderr:nodeep-scrub is set 2023-09-23T20:50:18.112 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:50:18.168 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:50:18.198 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:18.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:18 smithi149 ceph-mon[170394]: pgmap v1590: 103 pgs: 103 active+clean; 4.6 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail 2023-09-23T20:50:18.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:18 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3258645177' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:50:18.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:18 smithi018 ceph-mon[184144]: pgmap v1590: 103 pgs: 103 active+clean; 4.6 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail 2023-09-23T20:50:18.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:18 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3258645177' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:50:18.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:18 smithi018 ceph-mon[179647]: pgmap v1590: 103 pgs: 103 active+clean; 4.6 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail 2023-09-23T20:50:18.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:18 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3258645177' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:50:18.720 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:50:18.723 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:18.724 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:50:18.825 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:50:18.856 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:18.893 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:50:18.896 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:18.897 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:50:18.997 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:50:19.026 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:19.194 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:50:19.198 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:19.198 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:50:19.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:19 smithi149 ceph-mon[170394]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:50:19.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:19 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3258645177' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:50:19.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:19 smithi149 ceph-mon[170394]: osdmap e650: 8 total, 8 up, 7 in 2023-09-23T20:50:19.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:19 smithi149 ceph-mon[170394]: failed to encode map e650 with expected crc 2023-09-23T20:50:19.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:19 smithi149 ceph-mon[170394]: failed to encode map e650 with expected crc 2023-09-23T20:50:19.274 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:19 smithi018 ceph-mon[179647]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:50:19.274 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:19 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3258645177' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:50:19.274 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:19 smithi018 ceph-mon[179647]: osdmap e650: 8 total, 8 up, 7 in 2023-09-23T20:50:19.274 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:19 smithi018 ceph-mon[179647]: failed to encode map e650 with expected crc 2023-09-23T20:50:19.274 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:19 smithi018 ceph-mon[179647]: failed to encode map e650 with expected crc 2023-09-23T20:50:19.274 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:19 smithi018 ceph-mon[184144]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:50:19.275 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:19 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3258645177' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:50:19.275 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:19 smithi018 ceph-mon[184144]: osdmap e650: 8 total, 8 up, 7 in 2023-09-23T20:50:19.275 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:19 smithi018 ceph-mon[184144]: failed to encode map e650 with expected crc 2023-09-23T20:50:19.275 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:19 smithi018 ceph-mon[184144]: failed to encode map e650 with expected crc 2023-09-23T20:50:19.299 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:50:19.331 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:19.369 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:50:19.372 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:19.372 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:50:19.473 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:50:19.484 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2023-09-23T20:50:19.485 INFO:teuthology.orchestra.run.smithi018.stdout: "ops": [], 2023-09-23T20:50:19.485 INFO:teuthology.orchestra.run.smithi018.stdout: "num_ops": 0 2023-09-23T20:50:19.485 INFO:teuthology.orchestra.run.smithi018.stdout:} 2023-09-23T20:50:19.506 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:19.601 DEBUG:teuthology.orchestra.run.smithi018:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2023-09-23T20:50:19.649 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:50:19.651 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:19.652 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:50:19.660 INFO:tasks.workunit.client.0.smithi018.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2023-09-23T20:50:19.662 INFO:tasks.workunit.client.0.smithi018.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-09-23T20:50:19.662 INFO:tasks.workunit.client.0.smithi018.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2023-09-23T20:50:19.710 INFO:tasks.workunit.client.0.smithi018.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2023-09-23T20:50:19.753 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:50:19.758 INFO:tasks.workunit.client.0.smithi018.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2023-09-23T20:50:19.783 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:19.806 INFO:tasks.workunit.client.0.smithi018.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-09-23T20:50:19.808 INFO:tasks.workunit.client.0.smithi018.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-09-23T20:50:19.808 INFO:tasks.workunit.client.0.smithi018.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2023-09-23T20:50:19.819 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:50:19.822 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:19.822 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:50:19.850 INFO:tasks.workunit.client.0.smithi018.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-09-23T20:50:19.853 DEBUG:teuthology.orchestra.run.smithi018:> set -ex 2023-09-23T20:50:19.853 DEBUG:teuthology.orchestra.run.smithi018:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2023-09-23T20:50:19.911 INFO:tasks.workunit:Running workunits matching rbd/import_export.sh on client.0... 2023-09-23T20:50:19.912 INFO:tasks.workunit:Running workunit rbd/import_export.sh... 2023-09-23T20:50:19.913 DEBUG:teuthology.orchestra.run.smithi018:workunit test rbd/import_export.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 RBD_CREATE_ARGS=--new-format adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/rbd/import_export.sh 2023-09-23T20:50:19.923 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:50:19.974 INFO:tasks.workunit.client.0.smithi018.stderr:+ export RBD_FORCE_ALLOW_V1=1 2023-09-23T20:50:19.974 INFO:tasks.workunit.client.0.smithi018.stderr:+ RBD_FORCE_ALLOW_V1=1 2023-09-23T20:50:19.974 INFO:tasks.workunit.client.0.smithi018.stderr:+ TMPDIR=/tmp/rbd_import_export_466651 2023-09-23T20:50:19.975 INFO:tasks.workunit.client.0.smithi018.stderr:+ rm -rf /tmp/rbd_import_export_466651 2023-09-23T20:50:19.975 INFO:tasks.workunit.client.0.smithi018.stderr:+ mkdir /tmp/rbd_import_export_466651 2023-09-23T20:50:19.976 INFO:tasks.workunit.client.0.smithi018.stderr:+ trap 'rm -rf /tmp/rbd_import_export_466651' INT TERM EXIT 2023-09-23T20:50:19.976 INFO:tasks.workunit.client.0.smithi018.stderr:+ mkdir foo.466651 2023-09-23T20:50:19.977 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd import foo.466651 foo.dir 2023-09-23T20:50:19.982 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:20.065 INFO:tasks.workunit.client.0.smithi018.stderr:rbd: cannot import a directory 2023-09-23T20:50:20.066 INFO:tasks.workunit.client.0.smithi018.stderr: Importing image: 0% complete...failed. 2023-09-23T20:50:20.066 INFO:tasks.workunit.client.0.smithi018.stderr:rbd: import failed: (21) Is a directory 2023-09-23T20:50:20.068 INFO:tasks.workunit.client.0.smithi018.stderr:+ true 2023-09-23T20:50:20.068 INFO:tasks.workunit.client.0.smithi018.stderr:+ rmdir foo.466651 2023-09-23T20:50:20.109 INFO:tasks.workunit.client.0.smithi018.stderr:+ dd if=/bin/sh of=/tmp/rbd_import_export_466651/img bs=1k count=1 seek=10 2023-09-23T20:50:20.110 INFO:tasks.workunit.client.0.smithi018.stderr:1+0 records in 2023-09-23T20:50:20.110 INFO:tasks.workunit.client.0.smithi018.stderr:1+0 records out 2023-09-23T20:50:20.110 INFO:tasks.workunit.client.0.smithi018.stderr:1024 bytes (1.0 kB, 1.0 KiB) copied, 4.5692e-05 s, 22.4 MB/s 2023-09-23T20:50:20.110 INFO:tasks.workunit.client.0.smithi018.stderr:+ dd if=/bin/dd of=/tmp/rbd_import_export_466651/img bs=1k count=10 seek=100 2023-09-23T20:50:20.110 INFO:tasks.workunit.client.0.smithi018.stderr:10+0 records in 2023-09-23T20:50:20.111 INFO:tasks.workunit.client.0.smithi018.stderr:10+0 records out 2023-09-23T20:50:20.111 INFO:tasks.workunit.client.0.smithi018.stderr:10240 bytes (10 kB, 10 KiB) copied, 6.2879e-05 s, 163 MB/s 2023-09-23T20:50:20.111 INFO:tasks.workunit.client.0.smithi018.stderr:+ dd if=/bin/rm of=/tmp/rbd_import_export_466651/img bs=1k count=100 seek=1000 2023-09-23T20:50:20.111 INFO:tasks.workunit.client.0.smithi018.stderr:70+1 records in 2023-09-23T20:50:20.112 INFO:tasks.workunit.client.0.smithi018.stderr:70+1 records out 2023-09-23T20:50:20.112 INFO:tasks.workunit.client.0.smithi018.stderr:72048 bytes (72 kB, 70 KiB) copied, 0.000267888 s, 269 MB/s 2023-09-23T20:50:20.112 INFO:tasks.workunit.client.0.smithi018.stderr:+ dd if=/bin/ls of=/tmp/rbd_import_export_466651/img bs=1k seek=10000 2023-09-23T20:50:20.113 INFO:tasks.workunit.client.0.smithi018.stderr:139+1 records in 2023-09-23T20:50:20.113 INFO:tasks.workunit.client.0.smithi018.stderr:139+1 records out 2023-09-23T20:50:20.113 INFO:tasks.workunit.client.0.smithi018.stderr:143224 bytes (143 kB, 140 KiB) copied, 0.000357164 s, 401 MB/s 2023-09-23T20:50:20.113 INFO:tasks.workunit.client.0.smithi018.stderr:+ dd if=/bin/ln of=/tmp/rbd_import_export_466651/img bs=1k seek=100000 2023-09-23T20:50:20.114 INFO:tasks.workunit.client.0.smithi018.stderr:70+1 records in 2023-09-23T20:50:20.114 INFO:tasks.workunit.client.0.smithi018.stderr:70+1 records out 2023-09-23T20:50:20.114 INFO:tasks.workunit.client.0.smithi018.stderr:72168 bytes (72 kB, 70 KiB) copied, 0.000211729 s, 341 MB/s 2023-09-23T20:50:20.114 INFO:tasks.workunit.client.0.smithi018.stderr:+ dd if=/bin/grep of=/tmp/rbd_import_export_466651/img bs=1k seek=1000000 2023-09-23T20:50:20.115 INFO:tasks.workunit.client.0.smithi018.stderr:189+1 records in 2023-09-23T20:50:20.115 INFO:tasks.workunit.client.0.smithi018.stderr:189+1 records out 2023-09-23T20:50:20.115 INFO:tasks.workunit.client.0.smithi018.stderr:193768 bytes (194 kB, 189 KiB) copied, 0.000520247 s, 372 MB/s 2023-09-23T20:50:20.115 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd rm testimg 2023-09-23T20:50:20.153 INFO:tasks.workunit.client.0.smithi018.stderr: Removing image: 0% complete...failed. 2023-09-23T20:50:20.153 INFO:tasks.workunit.client.0.smithi018.stderr:rbd: delete error: (2) No such file or directory 2023-09-23T20:50:20.156 INFO:tasks.workunit.client.0.smithi018.stderr:+ true 2023-09-23T20:50:20.156 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd import --new-format /tmp/rbd_import_export_466651/img testimg 2023-09-23T20:50:20.187 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:50:20.190 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:20.190 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:50:20.236 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:20 smithi018 ceph-mon[184144]: pgmap v1592: 103 pgs: 103 active+clean; 4.6 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail 2023-09-23T20:50:20.237 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:20 smithi018 ceph-mon[179647]: pgmap v1592: 103 pgs: 103 active+clean; 4.6 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail 2023-09-23T20:50:20.266 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T20:50:20.291 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:50:20.324 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:20.371 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:50:20.375 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:20.375 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:50:20.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:20 smithi149 ceph-mon[170394]: pgmap v1592: 103 pgs: 103 active+clean; 4.6 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail 2023-09-23T20:50:20.477 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:50:20.507 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:20.542 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:50:20.546 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:20.546 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:50:20.647 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:50:20.694 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:20.749 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:50:20.753 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:20.753 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:50:20.769 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:50:20.789 INFO:tasks.workunit.client.0.smithi018.stderr: Importing image: 1% complete... Importing image: 2% complete... Importing image: 3% complete... Importing image: 4% complete... Importing image: 5% complete... Importing image: 6% complete... Importing image: 7% complete... Importing image: 8% complete... Importing image: 9% complete... Importing image: 10% complete... Importing image: 11% complete... Importing image: 12% complete... Importing image: 13% complete... Importing image: 14% complete... Importing image: 15% complete... Importing image: 16% complete... Importing image: 17% complete... Importing image: 18% complete... Importing image: 19% complete... Importing image: 20% complete... Importing image: 21% complete... Importing image: 22% complete... Importing image: 23% complete... Importing image: 24% complete... Importing image: 25% complete... Importing image: 26% complete... Importing image: 27% complete... Importing image: 28% complete... Importing image: 29% complete... Importing image: 30% complete... Importing image: 31% complete... Importing image: 32% complete... Importing image: 33% complete... Importing image: 34% complete... Importing image: 35% complete... Importing image: 36% complete... Importing image: 37% complete... Importing image: 38% complete... Importing image: 39% complete... Importing image: 40% complete... Importing image: 41% complete... Importing image: 42% complete... Importing image: 43% complete... Importing image: 44% complete... Importing image: 45% complete... Importing image: 46% complete... Importing image: 47% complete... Importing image: 48% complete... Importing image: 49% complete... Importing image: 50% complete... Importing image: 51% complete... Importing image: 52% complete... Importing image: 53% complete... Importing image: 54% complete... Importing image: 55% complete... Importing image: 56% complete... Importing image: 57% complete... Importing image: 58% complete... Importing image: 59% complete... Importing image: 60% complete... Importing image: 61% complete... Importing image: 62% complete... Importing image: 63% complete... Importing image: 64% complete... Importing image: 65% complete... Importing image: 66% complete... Importing image: 67% complete... Importing image: 68% complete... Importing image: 69% complete... Importing image: 70% complete... Importing image: 71% complete... Importing image: 72% complete... Importing image: 73% complete... Importing image: 74% complete... Importing image: 75% complete... Importing image: 76% complete... Importing image: 77% complete... Importing image: 78% complete... Importing image: 79% complete... Importing image: 80% complete... Importing image: 81% complete... Importing image: 82% complete... Importing image: 83% complete... Importing image: 84% complete... Importing image: 85% complete... Importing image: 86% complete... Importing image: 87% complete... Importing image: 88% complete... Importing image: 89% complete... Importing image: 90% complete... Importing image: 91% complete... Importing image: 92% complete... Importing image: 93% complete... Importing image: 94% complete... Importing image: 95% complete... Importing image: 96% complete... Importing image: 97% complete... Importing image: 98% complete... Importing image: 99% complete... Importing image: 100% complete...done. 2023-09-23T20:50:20.789 INFO:tasks.workunit.client.0.smithi018.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T20:50:20.790 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:50:20.792 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd export testimg /tmp/rbd_import_export_466651/img2 2023-09-23T20:50:20.810 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:50:20.835 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:50:20.854 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:50:20.860 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:50:20.891 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:50:20.894 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:20.913 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:50:20.935 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:50:20.937 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:50:20.937 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:50:20.937 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:50:20.937 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:50:20.937 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:50:20.937 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:50:20.938 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:50:20.938 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:50:20.938 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:50:20.942 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:20.943 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:50:20.957 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:50:21.044 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:50:21.073 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:21.090 INFO:tasks.workunit.client.0.smithi018.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-09-23T20:50:21.094 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd export testimg - 2023-09-23T20:50:21.108 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:50:21.111 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:21.112 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:50:21.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:21 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/171438442' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:50:21.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:21 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/171438442' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:50:21.213 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:50:21.239 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:21.265 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T20:50:21.272 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:50:21.275 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:21.275 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:50:21.376 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:50:21.415 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:21.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:21 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/171438442' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:50:21.469 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:50:21.473 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:21.474 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:50:21.574 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:50:21.601 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:21.623 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 7, 3, 6, 4, 2] out_osds: [5] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T20:50:21.623 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd reweight 0 1 2023-09-23T20:50:21.634 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:50:21.636 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:21.637 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:50:21.737 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:50:21.764 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:21.796 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:50:21.799 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:21.800 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:50:21.890 INFO:tasks.workunit.client.0.smithi018.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-09-23T20:50:21.893 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd rm testimg 2023-09-23T20:50:21.901 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:50:21.927 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:21.961 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:50:21.964 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:21.964 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:50:22.065 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:50:22.092 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:22.126 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:50:22.129 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:22.129 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:50:22.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:22 smithi149 ceph-mon[170394]: pgmap v1593: 103 pgs: 103 active+clean; 4.6 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail 2023-09-23T20:50:22.230 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:50:22.249 INFO:tasks.workunit.client.0.smithi018.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2023-09-23T20:50:22.253 INFO:tasks.workunit.client.0.smithi018.stderr:+ cmp /tmp/rbd_import_export_466651/img /tmp/rbd_import_export_466651/img2 2023-09-23T20:50:22.257 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:22.273 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:50:22.290 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:50:22.293 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:22.294 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:50:22.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:22 smithi018 ceph-mon[179647]: pgmap v1593: 103 pgs: 103 active+clean; 4.6 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail 2023-09-23T20:50:22.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:22 smithi018 ceph-mon[184144]: pgmap v1593: 103 pgs: 103 active+clean; 4.6 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail 2023-09-23T20:50:22.394 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:50:22.420 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:22.453 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:50:22.456 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:22.457 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:50:22.557 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:50:22.588 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:23.060 INFO:tasks.workunit.client.0.smithi018.stderr:+ cmp /tmp/rbd_import_export_466651/img /tmp/rbd_import_export_466651/img3 2023-09-23T20:50:23.411 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:50:23.577 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:50:23.581 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:23.582 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:50:23.590 INFO:tasks.workunit.client.0.smithi018.stderr:+ rm /tmp/rbd_import_export_466651/img2 /tmp/rbd_import_export_466651/img3 2023-09-23T20:50:23.682 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:50:23.716 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:23.754 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:50:23.757 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:23.758 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:50:23.832 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd import --new-format - testimg 2023-09-23T20:50:23.859 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:50:23.887 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:23.924 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:50:23.927 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:23.927 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:50:24.030 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:50:24.067 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:24.199 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:24 smithi018 ceph-mon[184144]: pgmap v1594: 103 pgs: 103 active+clean; 8.2 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 2.2 KiB/s rd, 21 KiB/s wr, 3 op/s 2023-09-23T20:50:24.199 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:24 smithi018 ceph-mon[179647]: pgmap v1594: 103 pgs: 103 active+clean; 8.2 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 2.2 KiB/s rd, 21 KiB/s wr, 3 op/s 2023-09-23T20:50:24.256 INFO:tasks.workunit.client.0.smithi018.stderr: Importing image: 100% complete...done. 2023-09-23T20:50:24.256 INFO:tasks.workunit.client.0.smithi018.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T20:50:24.259 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd export testimg /tmp/rbd_import_export_466651/img2 2023-09-23T20:50:24.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:24 smithi149 ceph-mon[170394]: pgmap v1594: 103 pgs: 103 active+clean; 8.2 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 2.2 KiB/s rd, 21 KiB/s wr, 3 op/s 2023-09-23T20:50:24.568 INFO:tasks.workunit.client.0.smithi018.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-09-23T20:50:24.571 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd export testimg - 2023-09-23T20:50:24.775 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:50:24.830 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:24.831 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:50:24.932 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:50:24.959 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:24.993 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:50:24.996 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:24.997 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:50:25.098 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:50:25.128 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:50:24 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:50:24] "GET /metrics HTTP/1.1" 200 34156 "" "Prometheus/2.43.0" 2023-09-23T20:50:25.132 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:25.172 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:50:25.181 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:25.182 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:50:25.282 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:50:25.309 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:25.343 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:50:25.346 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:25.346 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:50:25.368 INFO:tasks.workunit.client.0.smithi018.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-09-23T20:50:25.371 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd rm testimg 2023-09-23T20:50:25.432 INFO:teuthology.orchestra.run.smithi018.stdout:ERROR: (22) Invalid argument 2023-09-23T20:50:25.433 INFO:teuthology.orchestra.run.smithi018.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:50:25.447 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:50:25.466 INFO:tasks.workunit.client.0.smithi018.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2023-09-23T20:50:25.468 INFO:tasks.workunit.client.0.smithi018.stderr:+ cmp /tmp/rbd_import_export_466651/img /tmp/rbd_import_export_466651/img2 2023-09-23T20:50:25.478 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:25.641 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:50:25.644 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:25.644 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:50:25.645 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:50:25.745 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:50:25.776 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:25.777 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:50:26.069 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:50:26.072 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:26.073 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:50:26.174 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:50:26.194 INFO:tasks.workunit.client.0.smithi018.stderr:+ cmp /tmp/rbd_import_export_466651/img /tmp/rbd_import_export_466651/img3 2023-09-23T20:50:26.203 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:26.239 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:50:26.242 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:26.243 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:50:26.249 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T20:50:26.344 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:50:26.355 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:26 smithi018 ceph-mon[184144]: pgmap v1595: 103 pgs: 103 active+clean; 8.2 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 26 KiB/s rd, 32 KiB/s wr, 15 op/s 2023-09-23T20:50:26.355 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:26 smithi018 ceph-mon[179647]: pgmap v1595: 103 pgs: 103 active+clean; 8.2 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 26 KiB/s rd, 32 KiB/s wr, 15 op/s 2023-09-23T20:50:26.369 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:26 smithi149 ceph-mon[170394]: pgmap v1595: 103 pgs: 103 active+clean; 8.2 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 26 KiB/s rd, 32 KiB/s wr, 15 op/s 2023-09-23T20:50:26.371 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:26.406 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:50:26.409 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:26.410 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:50:26.510 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:50:26.540 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:26.737 INFO:tasks.workunit.client.0.smithi018.stderr:+ rm /tmp/rbd_import_export_466651/img /tmp/rbd_import_export_466651/img2 /tmp/rbd_import_export_466651/img3 2023-09-23T20:50:27.085 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:50:27.089 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:27.090 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:50:27.164 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd help export 2023-09-23T20:50:27.164 INFO:tasks.workunit.client.0.smithi018.stderr:+ grep -q export-format 2023-09-23T20:50:27.184 INFO:tasks.workunit.client.0.smithi018.stderr:+ dd if=/bin/dd of=/tmp/rbd_import_export_466651/img bs=1k count=10 seek=100 2023-09-23T20:50:27.185 INFO:tasks.workunit.client.0.smithi018.stderr:10+0 records in 2023-09-23T20:50:27.185 INFO:tasks.workunit.client.0.smithi018.stderr:10+0 records out 2023-09-23T20:50:27.185 INFO:tasks.workunit.client.0.smithi018.stderr:10240 bytes (10 kB, 10 KiB) copied, 6.6125e-05 s, 155 MB/s 2023-09-23T20:50:27.185 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd import --new-format /tmp/rbd_import_export_466651/img testimg 2023-09-23T20:50:27.191 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:50:27.220 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:27.246 INFO:tasks.workunit.client.0.smithi018.stderr: Importing image: 100% complete...done. 2023-09-23T20:50:27.247 INFO:tasks.workunit.client.0.smithi018.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T20:50:27.250 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd snap create testimg@snap 2023-09-23T20:50:27.819 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:50:27.822 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:27.822 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:50:27.923 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:50:27.955 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:27.994 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:50:27.997 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:27.998 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:50:28.080 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:28 smithi018 ceph-mon[184144]: pgmap v1596: 103 pgs: 103 active+clean; 6.3 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 158 KiB/s rd, 61 KiB/s wr, 64 op/s 2023-09-23T20:50:28.081 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:28 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1941457448' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:50:28.081 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:28 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:50:28.081 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:28 smithi018 ceph-mon[179647]: pgmap v1596: 103 pgs: 103 active+clean; 6.3 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 158 KiB/s rd, 61 KiB/s wr, 64 op/s 2023-09-23T20:50:28.081 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:28 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1941457448' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:50:28.081 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:28 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:50:28.090 INFO:teuthology.orchestra.run.smithi018.stderr:noscrub is unset 2023-09-23T20:50:28.098 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:50:28.099 INFO:tasks.workunit.client.0.smithi018.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2023-09-23T20:50:28.106 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd image-meta set testimg key1 value1 2023-09-23T20:50:28.126 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:28.161 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:50:28.164 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:28.164 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:50:28.171 INFO:tasks.workunit.client.0.smithi018.stderr:++ rbd image-meta list testimg 2023-09-23T20:50:28.212 INFO:tasks.workunit.client.0.smithi018.stderr:+ IMAGEMETA_BEFORE='There is 1 metadatum on this image: 2023-09-23T20:50:28.212 INFO:tasks.workunit.client.0.smithi018.stderr: 2023-09-23T20:50:28.212 INFO:tasks.workunit.client.0.smithi018.stderr:Key Value 2023-09-23T20:50:28.213 INFO:tasks.workunit.client.0.smithi018.stderr:key1 value1' 2023-09-23T20:50:28.213 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_466651/img_v2 2023-09-23T20:50:28.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:28 smithi149 ceph-mon[170394]: pgmap v1596: 103 pgs: 103 active+clean; 6.3 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 158 KiB/s rd, 61 KiB/s wr, 64 op/s 2023-09-23T20:50:28.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:28 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1941457448' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:50:28.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:28 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:50:28.251 INFO:tasks.workunit.client.0.smithi018.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-09-23T20:50:28.255 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_466651/img_v2 testimg_import 2023-09-23T20:50:28.265 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:50:28.291 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:28.327 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:50:28.330 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:28.331 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:50:28.348 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:50:28.367 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:50:28.386 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:50:28.407 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:50:28.426 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:50:28.431 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:50:28.445 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:50:28.457 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:28.464 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:50:28.483 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:50:28.484 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:50:28.485 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:50:28.485 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:50:28.485 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:50:28.485 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:50:28.485 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:50:28.485 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:50:28.485 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:50:28.490 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:50:28.493 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:28.493 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:50:28.594 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:50:28.628 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:28.928 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:50:28.931 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:28.932 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:50:29.033 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:50:29.040 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:50:29.067 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:29.110 INFO:tasks.workunit.client.0.smithi018.stderr: Importing image: 50% complete... Importing image: 100% complete... Importing image: 100% complete...done. 2023-09-23T20:50:29.113 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd info testimg_import 2023-09-23T20:50:29.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:29 smithi018 ceph-mon[184144]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:50:29.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:29 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:50:29.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:29 smithi018 ceph-mon[184144]: osdmap e651: 8 total, 8 up, 7 in 2023-09-23T20:50:29.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:29 smithi018 ceph-mon[184144]: failed to encode map e651 with expected crc 2023-09-23T20:50:29.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:29 smithi018 ceph-mon[184144]: failed to encode map e651 with expected crc 2023-09-23T20:50:29.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:29 smithi018 ceph-mon[184144]: failed to encode map e651 with expected crc 2023-09-23T20:50:29.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:29 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1153521994' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:50:29.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:29 smithi018 ceph-mon[179647]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:50:29.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:29 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:50:29.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:29 smithi018 ceph-mon[179647]: osdmap e651: 8 total, 8 up, 7 in 2023-09-23T20:50:29.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:29 smithi018 ceph-mon[179647]: failed to encode map e651 with expected crc 2023-09-23T20:50:29.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:29 smithi018 ceph-mon[179647]: failed to encode map e651 with expected crc 2023-09-23T20:50:29.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:29 smithi018 ceph-mon[179647]: failed to encode map e651 with expected crc 2023-09-23T20:50:29.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:29 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1153521994' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:50:29.145 INFO:tasks.workunit.client.0.smithi018.stdout:rbd image 'testimg_import': 2023-09-23T20:50:29.145 INFO:tasks.workunit.client.0.smithi018.stdout: size 110 KiB in 1 objects 2023-09-23T20:50:29.145 INFO:tasks.workunit.client.0.smithi018.stdout: order 22 (4 MiB objects) 2023-09-23T20:50:29.145 INFO:tasks.workunit.client.0.smithi018.stdout: snapshot_count: 1 2023-09-23T20:50:29.145 INFO:tasks.workunit.client.0.smithi018.stdout: id: b87320ef5791 2023-09-23T20:50:29.145 INFO:tasks.workunit.client.0.smithi018.stdout: block_name_prefix: rbd_data.b87320ef5791 2023-09-23T20:50:29.145 INFO:tasks.workunit.client.0.smithi018.stdout: format: 2 2023-09-23T20:50:29.146 INFO:tasks.workunit.client.0.smithi018.stdout: features: layering, exclusive-lock, object-map, fast-diff, deep-flatten 2023-09-23T20:50:29.146 INFO:tasks.workunit.client.0.smithi018.stdout: op_features: 2023-09-23T20:50:29.146 INFO:tasks.workunit.client.0.smithi018.stdout: flags: 2023-09-23T20:50:29.146 INFO:tasks.workunit.client.0.smithi018.stdout: create_timestamp: Sat Sep 23 20:50:28 2023 2023-09-23T20:50:29.146 INFO:tasks.workunit.client.0.smithi018.stdout: access_timestamp: Sat Sep 23 20:50:28 2023 2023-09-23T20:50:29.146 INFO:tasks.workunit.client.0.smithi018.stdout: modify_timestamp: Sat Sep 23 20:50:28 2023 2023-09-23T20:50:29.148 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd info testimg_import@snap 2023-09-23T20:50:29.180 INFO:tasks.workunit.client.0.smithi018.stdout:rbd image 'testimg_import': 2023-09-23T20:50:29.180 INFO:tasks.workunit.client.0.smithi018.stdout: size 110 KiB in 1 objects 2023-09-23T20:50:29.180 INFO:tasks.workunit.client.0.smithi018.stdout: order 22 (4 MiB objects) 2023-09-23T20:50:29.180 INFO:tasks.workunit.client.0.smithi018.stdout: snapshot_count: 1 2023-09-23T20:50:29.180 INFO:tasks.workunit.client.0.smithi018.stdout: id: b87320ef5791 2023-09-23T20:50:29.180 INFO:tasks.workunit.client.0.smithi018.stdout: block_name_prefix: rbd_data.b87320ef5791 2023-09-23T20:50:29.180 INFO:tasks.workunit.client.0.smithi018.stdout: format: 2 2023-09-23T20:50:29.180 INFO:tasks.workunit.client.0.smithi018.stdout: features: layering, exclusive-lock, object-map, fast-diff, deep-flatten 2023-09-23T20:50:29.180 INFO:tasks.workunit.client.0.smithi018.stdout: op_features: 2023-09-23T20:50:29.181 INFO:tasks.workunit.client.0.smithi018.stdout: flags: 2023-09-23T20:50:29.181 INFO:tasks.workunit.client.0.smithi018.stdout: create_timestamp: Sat Sep 23 20:50:28 2023 2023-09-23T20:50:29.181 INFO:tasks.workunit.client.0.smithi018.stdout: access_timestamp: Sat Sep 23 20:50:28 2023 2023-09-23T20:50:29.181 INFO:tasks.workunit.client.0.smithi018.stdout: modify_timestamp: Sat Sep 23 20:50:28 2023 2023-09-23T20:50:29.181 INFO:tasks.workunit.client.0.smithi018.stdout: protected: False 2023-09-23T20:50:29.183 INFO:tasks.workunit.client.0.smithi018.stderr:++ rbd image-meta list testimg_import 2023-09-23T20:50:29.198 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:50:29.201 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:29.202 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:50:29.229 INFO:tasks.workunit.client.0.smithi018.stderr:+ IMAGEMETA_AFTER='There is 1 metadatum on this image: 2023-09-23T20:50:29.230 INFO:tasks.workunit.client.0.smithi018.stderr: 2023-09-23T20:50:29.230 INFO:tasks.workunit.client.0.smithi018.stderr:Key Value 2023-09-23T20:50:29.230 INFO:tasks.workunit.client.0.smithi018.stderr:key1 value1' 2023-09-23T20:50:29.230 INFO:tasks.workunit.client.0.smithi018.stderr:+ '[' 'There is 1 metadatum on this image: 2023-09-23T20:50:29.230 INFO:tasks.workunit.client.0.smithi018.stderr: 2023-09-23T20:50:29.230 INFO:tasks.workunit.client.0.smithi018.stderr:Key Value 2023-09-23T20:50:29.230 INFO:tasks.workunit.client.0.smithi018.stderr:key1 value1' = 'There is 1 metadatum on this image: 2023-09-23T20:50:29.230 INFO:tasks.workunit.client.0.smithi018.stderr: 2023-09-23T20:50:29.230 INFO:tasks.workunit.client.0.smithi018.stderr:Key Value 2023-09-23T20:50:29.230 INFO:tasks.workunit.client.0.smithi018.stderr:key1 value1' ']' 2023-09-23T20:50:29.230 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd export testimg_import /tmp/rbd_import_export_466651/img_import 2023-09-23T20:50:29.266 INFO:tasks.workunit.client.0.smithi018.stderr: Exporting image: 100% complete...done. 2023-09-23T20:50:29.269 INFO:tasks.workunit.client.0.smithi018.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_466651/img /tmp/rbd_import_export_466651/img_import 2023-09-23T20:50:29.269 INFO:tasks.workunit.client.0.smithi018.stderr:+ cmp -l /tmp/rbd_import_export_466651/img /tmp/rbd_import_export_466651/img_import 2023-09-23T20:50:29.270 INFO:tasks.workunit.client.0.smithi018.stderr:++ stat /tmp/rbd_import_export_466651/img --format %b 2023-09-23T20:50:29.271 INFO:tasks.workunit.client.0.smithi018.stderr:+ origsize=24 2023-09-23T20:50:29.271 INFO:tasks.workunit.client.0.smithi018.stderr:++ stat /tmp/rbd_import_export_466651/img_import --format %b 2023-09-23T20:50:29.272 INFO:tasks.workunit.client.0.smithi018.stderr:+ exportsize=224 2023-09-23T20:50:29.273 INFO:tasks.workunit.client.0.smithi018.stderr:+ difference=200 2023-09-23T20:50:29.273 INFO:tasks.workunit.client.0.smithi018.stderr:+ difference=200 2023-09-23T20:50:29.273 INFO:tasks.workunit.client.0.smithi018.stderr:+ test 200 -ge 0 -a 200 -lt 4096 2023-09-23T20:50:29.273 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd export testimg@snap /tmp/rbd_import_export_466651/img_snap 2023-09-23T20:50:29.302 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:50:29.311 INFO:tasks.workunit.client.0.smithi018.stderr: Exporting image: 100% complete...done. 2023-09-23T20:50:29.314 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd export testimg_import@snap /tmp/rbd_import_export_466651/img_snap_import 2023-09-23T20:50:29.330 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:29.353 INFO:tasks.workunit.client.0.smithi018.stderr: Exporting image: 100% complete...done. 2023-09-23T20:50:29.357 INFO:tasks.workunit.client.0.smithi018.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_466651/img_snap /tmp/rbd_import_export_466651/img_snap_import 2023-09-23T20:50:29.357 INFO:tasks.workunit.client.0.smithi018.stderr:+ cmp -l /tmp/rbd_import_export_466651/img_snap /tmp/rbd_import_export_466651/img_snap_import 2023-09-23T20:50:29.358 INFO:tasks.workunit.client.0.smithi018.stderr:++ stat /tmp/rbd_import_export_466651/img_snap --format %b 2023-09-23T20:50:29.359 INFO:tasks.workunit.client.0.smithi018.stderr:+ origsize=224 2023-09-23T20:50:29.360 INFO:tasks.workunit.client.0.smithi018.stderr:++ stat /tmp/rbd_import_export_466651/img_snap_import --format %b 2023-09-23T20:50:29.361 INFO:tasks.workunit.client.0.smithi018.stderr:+ exportsize=224 2023-09-23T20:50:29.361 INFO:tasks.workunit.client.0.smithi018.stderr:+ difference=0 2023-09-23T20:50:29.361 INFO:tasks.workunit.client.0.smithi018.stderr:+ difference=0 2023-09-23T20:50:29.361 INFO:tasks.workunit.client.0.smithi018.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-09-23T20:50:29.361 INFO:tasks.workunit.client.0.smithi018.stderr:+ rm /tmp/rbd_import_export_466651/img_v2 2023-09-23T20:50:29.362 INFO:tasks.workunit.client.0.smithi018.stderr:+ rm /tmp/rbd_import_export_466651/img_import 2023-09-23T20:50:29.362 INFO:tasks.workunit.client.0.smithi018.stderr:+ rm /tmp/rbd_import_export_466651/img_snap 2023-09-23T20:50:29.363 INFO:tasks.workunit.client.0.smithi018.stderr:+ rm /tmp/rbd_import_export_466651/img_snap_import 2023-09-23T20:50:29.364 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd snap rm testimg_import@snap 2023-09-23T20:50:29.365 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:50:29.367 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:29 smithi149 ceph-mon[170394]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:50:29.367 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:29 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:50:29.367 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:29 smithi149 ceph-mon[170394]: osdmap e651: 8 total, 8 up, 7 in 2023-09-23T20:50:29.368 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:29 smithi149 ceph-mon[170394]: failed to encode map e651 with expected crc 2023-09-23T20:50:29.368 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:29 smithi149 ceph-mon[170394]: failed to encode map e651 with expected crc 2023-09-23T20:50:29.368 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:29 smithi149 ceph-mon[170394]: failed to encode map e651 with expected crc 2023-09-23T20:50:29.368 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:29 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1153521994' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:50:29.368 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:29.369 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:50:29.470 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:50:29.501 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:29.538 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:50:29.541 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:29.541 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:50:29.642 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:50:29.671 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:29.885 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:50:29.889 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:29.889 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:50:29.990 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:50:30.019 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:30.054 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:50:30.056 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:30.057 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:50:30.100 INFO:teuthology.orchestra.run.smithi018.stderr:reweighted osd.0 to 1 (10000) 2023-09-23T20:50:30.100 INFO:tasks.workunit.client.0.smithi018.stderr: Removing snap: 100% complete...done. 2023-09-23T20:50:30.110 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd remove testimg_import 2023-09-23T20:50:30.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:30 smithi018 ceph-mon[184144]: pgmap v1598: 103 pgs: 103 active+clean; 6.3 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 205 KiB/s rd, 85 KiB/s wr, 69 op/s 2023-09-23T20:50:30.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:30 smithi018 ceph-mon[184144]: failed to encode map e652 with expected crc 2023-09-23T20:50:30.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:30 smithi018 ceph-mon[184144]: osdmap e652: 8 total, 8 up, 7 in 2023-09-23T20:50:30.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:30 smithi018 ceph-mon[184144]: failed to encode map e652 with expected crc 2023-09-23T20:50:30.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:30 smithi018 ceph-mon[184144]: failed to encode map e652 with expected crc 2023-09-23T20:50:30.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:30 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:50:30.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:30 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/910767556' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T20:50:30.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:30 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T20:50:30.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:30 smithi018 ceph-mon[179647]: pgmap v1598: 103 pgs: 103 active+clean; 6.3 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 205 KiB/s rd, 85 KiB/s wr, 69 op/s 2023-09-23T20:50:30.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:30 smithi018 ceph-mon[179647]: failed to encode map e652 with expected crc 2023-09-23T20:50:30.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:30 smithi018 ceph-mon[179647]: osdmap e652: 8 total, 8 up, 7 in 2023-09-23T20:50:30.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:30 smithi018 ceph-mon[179647]: failed to encode map e652 with expected crc 2023-09-23T20:50:30.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:30 smithi018 ceph-mon[179647]: failed to encode map e652 with expected crc 2023-09-23T20:50:30.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:30 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:50:30.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:30 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/910767556' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T20:50:30.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:30 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T20:50:30.158 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:50:30.188 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:30.208 INFO:tasks.workunit.client.0.smithi018.stderr: Removing image: 100% complete...done. 2023-09-23T20:50:30.211 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd snap rm testimg@snap 2023-09-23T20:50:30.224 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:50:30.227 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:30.228 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:50:30.328 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:50:30.358 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:30.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:30 smithi149 ceph-mon[170394]: pgmap v1598: 103 pgs: 103 active+clean; 6.3 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 205 KiB/s rd, 85 KiB/s wr, 69 op/s 2023-09-23T20:50:30.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:30 smithi149 ceph-mon[170394]: failed to encode map e652 with expected crc 2023-09-23T20:50:30.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:30 smithi149 ceph-mon[170394]: osdmap e652: 8 total, 8 up, 7 in 2023-09-23T20:50:30.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:30 smithi149 ceph-mon[170394]: failed to encode map e652 with expected crc 2023-09-23T20:50:30.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:30 smithi149 ceph-mon[170394]: failed to encode map e652 with expected crc 2023-09-23T20:50:30.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:30 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:50:30.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:30 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/910767556' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T20:50:30.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:30 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T20:50:30.523 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:50:30.526 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:30.527 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:50:30.604 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2023-09-23T20:50:30.604 INFO:teuthology.orchestra.run.smithi018.stdout: "size": 20, 2023-09-23T20:50:30.604 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 600, 2023-09-23T20:50:30.605 INFO:teuthology.orchestra.run.smithi018.stdout: "ops": [ 2023-09-23T20:50:30.605 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.605 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.36450.0:141029 9.a 9:572023e9:::benchmark_data_smithi018_381516_object27195:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e481)", 2023-09-23T20:50:30.605 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:43:15.738002+0000", 2023-09-23T20:50:30.605 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 434.86370954400002, 2023-09-23T20:50:30.605 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0527425420000001, 2023-09-23T20:50:30.605 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:50:30.605 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:50:30.605 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:50:30.605 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.36450", 2023-09-23T20:50:30.605 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:50:30.606 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 141029 2023-09-23T20:50:30.606 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.606 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:50:30.606 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.606 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:50:30.606 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738002+0000", 2023-09-23T20:50:30.606 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:50:30.606 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.606 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.606 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:50:30.606 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738002+0000", 2023-09-23T20:50:30.607 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:50:30.607 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.607 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.607 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:50:30.607 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738000+0000", 2023-09-23T20:50:30.607 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:50:30.607 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.607 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.607 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:50:30.607 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738007+0000", 2023-09-23T20:50:30.607 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.4760000000000001e-06 2023-09-23T20:50:30.608 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.608 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.608 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:50:30.608 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738025+0000", 2023-09-23T20:50:30.608 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.7952e-05 2023-09-23T20:50:30.608 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.608 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.608 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:50:30.608 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738035+0000", 2023-09-23T20:50:30.608 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.6120000000000005e-06 2023-09-23T20:50:30.608 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.608 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.609 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:50:30.609 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.744123+0000", 2023-09-23T20:50:30.609 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0060876410000000004 2023-09-23T20:50:30.609 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.609 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.609 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op must wait for map", 2023-09-23T20:50:30.609 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.744131+0000", 2023-09-23T20:50:30.609 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.8800000000000008e-06 2023-09-23T20:50:30.609 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.609 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.609 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:50:30.609 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.754919+0000", 2023-09-23T20:50:30.610 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.010788483999999999 2023-09-23T20:50:30.610 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.610 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.610 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:50:30.610 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.754929+0000", 2023-09-23T20:50:30.610 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0309000000000001e-05 2023-09-23T20:50:30.610 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.610 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.610 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:50:30.610 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:16.790652+0000", 2023-09-23T20:50:30.610 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0357231570000001 2023-09-23T20:50:30.611 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.611 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.611 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:50:30.611 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:16.790744+0000", 2023-09-23T20:50:30.611 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.1743000000000006e-05 2023-09-23T20:50:30.611 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:50:30.611 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:50:30.611 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:50:30.611 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.611 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.611 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.36450.0:141040 9.a 9:53252326:::benchmark_data_smithi018_381516_object27206:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e481)", 2023-09-23T20:50:30.611 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:43:15.738057+0000", 2023-09-23T20:50:30.612 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 434.86365393199998, 2023-09-23T20:50:30.612 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0530090860000001, 2023-09-23T20:50:30.612 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:50:30.612 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:50:30.612 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:50:30.612 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.36450", 2023-09-23T20:50:30.612 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:50:30.612 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 141040 2023-09-23T20:50:30.612 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.612 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:50:30.612 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.612 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:50:30.613 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738057+0000", 2023-09-23T20:50:30.613 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:50:30.613 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.613 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.613 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:50:30.613 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738057+0000", 2023-09-23T20:50:30.613 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:50:30.613 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.613 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.613 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:50:30.613 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738057+0000", 2023-09-23T20:50:30.613 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:50:30.614 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.614 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.614 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:50:30.614 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738060+0000", 2023-09-23T20:50:30.614 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.9699999999999999e-06 2023-09-23T20:50:30.614 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.614 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.614 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:50:30.614 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738066+0000", 2023-09-23T20:50:30.614 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.5019999999999998e-06 2023-09-23T20:50:30.614 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.615 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.615 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:50:30.615 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738072+0000", 2023-09-23T20:50:30.615 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.2209999999999996e-06 2023-09-23T20:50:30.615 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.615 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.615 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:50:30.615 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.744157+0000", 2023-09-23T20:50:30.615 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0060856 2023-09-23T20:50:30.615 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.615 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.615 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting_for_map not empty", 2023-09-23T20:50:30.616 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.744162+0000", 2023-09-23T20:50:30.616 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.1610000000000002e-06 2023-09-23T20:50:30.616 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.616 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.616 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:50:30.616 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.754951+0000", 2023-09-23T20:50:30.616 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.010789127000000001 2023-09-23T20:50:30.616 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.616 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.616 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:50:30.616 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.754959+0000", 2023-09-23T20:50:30.616 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.3189999999999997e-06 2023-09-23T20:50:30.616 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.617 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.617 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:50:30.617 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:16.790999+0000", 2023-09-23T20:50:30.617 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.03604052 2023-09-23T20:50:30.617 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.617 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.617 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:50:30.617 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:16.791066+0000", 2023-09-23T20:50:30.617 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.7101999999999996e-05 2023-09-23T20:50:30.617 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:50:30.617 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:50:30.617 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:50:30.618 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.618 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.618 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.36450.0:141043 9.a 9:57a9fa95:::benchmark_data_smithi018_381516_object27209:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e481)", 2023-09-23T20:50:30.618 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:43:15.738077+0000", 2023-09-23T20:50:30.618 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 434.863634295, 2023-09-23T20:50:30.618 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0909135969999999, 2023-09-23T20:50:30.618 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:50:30.618 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:50:30.618 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:50:30.618 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.36450", 2023-09-23T20:50:30.618 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:50:30.618 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 141043 2023-09-23T20:50:30.618 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.619 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:50:30.619 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.619 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:50:30.619 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738077+0000", 2023-09-23T20:50:30.619 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:50:30.619 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.619 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.619 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:50:30.619 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738077+0000", 2023-09-23T20:50:30.619 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:50:30.619 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.620 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.620 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:50:30.620 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738077+0000", 2023-09-23T20:50:30.620 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:50:30.620 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.620 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.620 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:50:30.620 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738079+0000", 2023-09-23T20:50:30.620 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.0320000000000002e-06 2023-09-23T20:50:30.620 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.620 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.621 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:50:30.621 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738084+0000", 2023-09-23T20:50:30.621 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.4310000000000002e-06 2023-09-23T20:50:30.621 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.621 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.621 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:50:30.621 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738089+0000", 2023-09-23T20:50:30.621 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.6789999999999996e-06 2023-09-23T20:50:30.621 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.621 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.621 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:50:30.621 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.744187+0000", 2023-09-23T20:50:30.622 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0060979859999999997 2023-09-23T20:50:30.622 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.622 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.622 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting_for_map not empty", 2023-09-23T20:50:30.622 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.744194+0000", 2023-09-23T20:50:30.622 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.7090000000000002e-06 2023-09-23T20:50:30.622 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.622 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.622 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:50:30.622 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.754979+0000", 2023-09-23T20:50:30.622 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.010784141000000001 2023-09-23T20:50:30.622 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.623 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.623 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:50:30.623 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.754986+0000", 2023-09-23T20:50:30.623 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.0339999999999999e-06 2023-09-23T20:50:30.623 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.623 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.623 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:50:30.623 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:16.791090+0000", 2023-09-23T20:50:30.623 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.036103939 2023-09-23T20:50:30.623 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.623 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.623 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for missing object", 2023-09-23T20:50:30.624 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:16.791184+0000", 2023-09-23T20:50:30.624 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.4289000000000005e-05 2023-09-23T20:50:30.624 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.624 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.624 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:50:30.624 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:16.828905+0000", 2023-09-23T20:50:30.624 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.037721372000000003 2023-09-23T20:50:30.624 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.624 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.624 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:50:30.624 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:16.828991+0000", 2023-09-23T20:50:30.625 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.5331e-05 2023-09-23T20:50:30.625 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:50:30.625 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:50:30.625 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:50:30.625 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.625 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.625 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.36450.0:141045 9.f 9:fa55dbf8:::benchmark_data_smithi018_381516_object27211:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e481)", 2023-09-23T20:50:30.625 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:43:15.738093+0000", 2023-09-23T20:50:30.625 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 434.863618438, 2023-09-23T20:50:30.625 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0403492489999999, 2023-09-23T20:50:30.625 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:50:30.625 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:50:30.626 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:50:30.626 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.36450", 2023-09-23T20:50:30.626 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:50:30.626 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 141045 2023-09-23T20:50:30.626 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.626 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:50:30.626 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.626 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:50:30.626 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738093+0000", 2023-09-23T20:50:30.626 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:50:30.626 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.627 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.627 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:50:30.627 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738093+0000", 2023-09-23T20:50:30.627 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:50:30.627 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.627 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.627 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:50:30.627 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738092+0000", 2023-09-23T20:50:30.627 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:50:30.627 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.627 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.627 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:50:30.628 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738095+0000", 2023-09-23T20:50:30.628 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.063e-06 2023-09-23T20:50:30.628 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.628 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.628 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:50:30.628 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738100+0000", 2023-09-23T20:50:30.628 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.2469999999999999e-06 2023-09-23T20:50:30.628 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.628 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.628 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:50:30.628 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738104+0000", 2023-09-23T20:50:30.629 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.3649999999999997e-06 2023-09-23T20:50:30.629 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.629 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.629 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:50:30.629 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738271+0000", 2023-09-23T20:50:30.629 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00016673799999999999 2023-09-23T20:50:30.629 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.629 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.629 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op must wait for map", 2023-09-23T20:50:30.629 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738278+0000", 2023-09-23T20:50:30.629 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.0659999999999998e-06 2023-09-23T20:50:30.629 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.629 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.630 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:50:30.630 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.770314+0000", 2023-09-23T20:50:30.630 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.032036012000000003 2023-09-23T20:50:30.630 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.630 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.630 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:50:30.630 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.770322+0000", 2023-09-23T20:50:30.630 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.9330000000000001e-06 2023-09-23T20:50:30.630 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.630 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.630 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:50:30.630 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:16.776028+0000", 2023-09-23T20:50:30.631 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0057059290000001 2023-09-23T20:50:30.631 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.631 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.631 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:50:30.631 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:16.778442+0000", 2023-09-23T20:50:30.631 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0024142080000000002 2023-09-23T20:50:30.631 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:50:30.631 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:50:30.631 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:50:30.631 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.631 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.631 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.36450.0:141047 9.a 9:55e9d18e:::benchmark_data_smithi018_381516_object27213:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e481)", 2023-09-23T20:50:30.631 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:43:15.747358+0000", 2023-09-23T20:50:30.632 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 434.85435323199999, 2023-09-23T20:50:30.632 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.088479043, 2023-09-23T20:50:30.632 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:50:30.632 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:50:30.632 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:50:30.632 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.36450", 2023-09-23T20:50:30.632 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:50:30.632 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 141047 2023-09-23T20:50:30.632 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.632 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:50:30.632 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.633 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:50:30.633 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.747358+0000", 2023-09-23T20:50:30.633 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:50:30.633 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.633 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.633 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:50:30.633 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.747358+0000", 2023-09-23T20:50:30.633 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:50:30.633 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.633 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.633 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:50:30.633 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.747356+0000", 2023-09-23T20:50:30.634 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:50:30.634 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.634 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.634 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:50:30.634 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.747364+0000", 2023-09-23T20:50:30.634 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.6569999999999997e-06 2023-09-23T20:50:30.634 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.634 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.634 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:50:30.634 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.747379+0000", 2023-09-23T20:50:30.634 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.5153999999999999e-05 2023-09-23T20:50:30.635 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.635 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.635 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:50:30.635 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.747388+0000", 2023-09-23T20:50:30.635 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.7059999999999993e-06 2023-09-23T20:50:30.635 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.635 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.635 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:50:30.635 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.748082+0000", 2023-09-23T20:50:30.635 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00069463100000000002 2023-09-23T20:50:30.635 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.635 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.635 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting_for_map not empty", 2023-09-23T20:50:30.636 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.748090+0000", 2023-09-23T20:50:30.636 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.1349999999999992e-06 2023-09-23T20:50:30.636 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.636 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.636 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:50:30.636 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.755006+0000", 2023-09-23T20:50:30.636 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0069154289999999998 2023-09-23T20:50:30.636 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.636 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.636 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:50:30.636 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.755013+0000", 2023-09-23T20:50:30.636 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.7909999999999999e-06 2023-09-23T20:50:30.637 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.637 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.637 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:50:30.637 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:16.791216+0000", 2023-09-23T20:50:30.637 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.036203188 2023-09-23T20:50:30.637 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.637 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.637 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:50:30.637 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:16.791332+0000", 2023-09-23T20:50:30.637 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000115798 2023-09-23T20:50:30.637 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.637 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.638 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 5,7", 2023-09-23T20:50:30.638 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:16.791928+0000", 2023-09-23T20:50:30.638 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00059646299999999998 2023-09-23T20:50:30.638 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.638 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.638 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:30.638 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:16.822354+0000", 2023-09-23T20:50:30.638 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.030425812999999999 2023-09-23T20:50:30.638 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.638 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.638 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:50:30.638 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:16.822834+0000", 2023-09-23T20:50:30.639 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00048018300000000002 2023-09-23T20:50:30.639 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.639 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.639 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:30.639 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:16.835654+0000", 2023-09-23T20:50:30.639 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.012820066 2023-09-23T20:50:30.639 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.639 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.639 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:50:30.639 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:16.835711+0000", 2023-09-23T20:50:30.639 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.7300999999999999e-05 2023-09-23T20:50:30.639 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.639 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.640 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:50:30.640 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:16.835837+0000", 2023-09-23T20:50:30.640 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00012553400000000001 2023-09-23T20:50:30.640 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:50:30.640 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:50:30.640 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:50:30.640 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.640 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.640 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:102280 10.2 10:4d2f73b6:::benchmark_data_smithi018_422431_object102279:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e551)", 2023-09-23T20:50:30.640 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:47:08.174176+0000", 2023-09-23T20:50:30.640 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 202.42753534799999, 2023-09-23T20:50:30.640 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0912579659999999, 2023-09-23T20:50:30.641 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:50:30.641 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:50:30.641 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:50:30.641 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:50:30.641 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:50:30.641 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 102280 2023-09-23T20:50:30.641 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.641 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:50:30.641 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.641 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:50:30.641 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:08.174176+0000", 2023-09-23T20:50:30.641 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:50:30.642 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.642 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.642 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:50:30.642 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:08.174176+0000", 2023-09-23T20:50:30.642 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:50:30.642 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.642 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.642 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:50:30.642 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:08.174174+0000", 2023-09-23T20:50:30.642 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:50:30.642 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.642 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.643 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:50:30.643 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:08.174198+0000", 2023-09-23T20:50:30.643 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.3923000000000001e-05 2023-09-23T20:50:30.643 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.643 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.643 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:50:30.643 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:08.174211+0000", 2023-09-23T20:50:30.643 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3324e-05 2023-09-23T20:50:30.643 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.643 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.643 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:50:30.643 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:08.174221+0000", 2023-09-23T20:50:30.643 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.5740000000000002e-06 2023-09-23T20:50:30.644 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.644 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.644 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:50:30.644 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:08.186392+0000", 2023-09-23T20:50:30.644 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.012171184999999999 2023-09-23T20:50:30.644 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.644 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.644 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:50:30.644 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:08.186404+0000", 2023-09-23T20:50:30.644 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1878e-05 2023-09-23T20:50:30.644 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.644 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.645 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:50:30.645 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:09.252235+0000", 2023-09-23T20:50:30.645 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0658310289999999 2023-09-23T20:50:30.645 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.645 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.645 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:50:30.645 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:09.252439+0000", 2023-09-23T20:50:30.645 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00020431400000000001 2023-09-23T20:50:30.645 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.645 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.645 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 0,4", 2023-09-23T20:50:30.645 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:09.252618+0000", 2023-09-23T20:50:30.646 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00017877799999999999 2023-09-23T20:50:30.646 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.646 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.646 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:30.646 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:09.258806+0000", 2023-09-23T20:50:30.646 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0061873279999999998 2023-09-23T20:50:30.646 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.646 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.646 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:30.646 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:09.261128+0000", 2023-09-23T20:50:30.646 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.002322228 2023-09-23T20:50:30.647 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.647 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.647 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:50:30.647 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:09.264995+0000", 2023-09-23T20:50:30.647 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0038676840000000001 2023-09-23T20:50:30.647 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.647 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.647 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:50:30.647 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:09.265068+0000", 2023-09-23T20:50:30.647 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.2390000000000003e-05 2023-09-23T20:50:30.647 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.647 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.648 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:50:30.648 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:09.265434+0000", 2023-09-23T20:50:30.648 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00036597799999999998 2023-09-23T20:50:30.648 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:50:30.648 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:50:30.648 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:50:30.648 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.648 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.648 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:181708 10.16 10.b7205ed6 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e603)", 2023-09-23T20:50:30.648 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:49:01.717054+0000", 2023-09-23T20:50:30.648 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 88.884657367000003, 2023-09-23T20:50:30.648 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.97645376399999995, 2023-09-23T20:50:30.648 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:50:30.649 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:50:30.649 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:50:30.649 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:50:30.649 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:50:30.649 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 181708 2023-09-23T20:50:30.649 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.649 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:50:30.649 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.649 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:50:30.649 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:01.717054+0000", 2023-09-23T20:50:30.649 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:50:30.649 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.650 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.650 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:50:30.650 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:01.717054+0000", 2023-09-23T20:50:30.650 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:50:30.650 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.650 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.650 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:50:30.650 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:01.717052+0000", 2023-09-23T20:50:30.650 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:50:30.650 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.650 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.650 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:50:30.651 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:01.717060+0000", 2023-09-23T20:50:30.651 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.6389999999999998e-06 2023-09-23T20:50:30.651 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.651 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.651 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:50:30.651 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:01.717081+0000", 2023-09-23T20:50:30.651 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.0638000000000001e-05 2023-09-23T20:50:30.651 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.651 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.651 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:50:30.651 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:01.717091+0000", 2023-09-23T20:50:30.651 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.9410000000000002e-06 2023-09-23T20:50:30.652 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.652 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.652 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:50:30.652 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:01.717148+0000", 2023-09-23T20:50:30.652 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.6777000000000002e-05 2023-09-23T20:50:30.652 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.652 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.652 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:50:30.652 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:01.717157+0000", 2023-09-23T20:50:30.652 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.8040000000000007e-06 2023-09-23T20:50:30.652 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.652 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.652 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:50:30.653 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:02.693500+0000", 2023-09-23T20:50:30.653 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.97634373500000005 2023-09-23T20:50:30.653 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.653 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.653 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:50:30.653 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:02.693508+0000", 2023-09-23T20:50:30.653 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.3490000000000003e-06 2023-09-23T20:50:30.653 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:50:30.653 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:50:30.653 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:50:30.653 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.653 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.654 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:181726 10.2 10:47312290:::benchmark_data_smithi018_422431_object77230:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e604)", 2023-09-23T20:50:30.654 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:49:02.807907+0000", 2023-09-23T20:50:30.654 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 87.793804273999996, 2023-09-23T20:50:30.654 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.99641389400000002, 2023-09-23T20:50:30.654 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:50:30.654 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:50:30.654 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:50:30.654 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:50:30.654 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:50:30.654 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 181726 2023-09-23T20:50:30.654 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.654 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:50:30.655 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.655 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:50:30.655 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:02.807907+0000", 2023-09-23T20:50:30.655 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:50:30.655 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.655 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.655 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:50:30.655 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:02.807907+0000", 2023-09-23T20:50:30.655 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:50:30.655 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.655 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.655 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:50:30.656 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:02.807905+0000", 2023-09-23T20:50:30.656 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:50:30.656 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.656 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.656 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:50:30.656 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:02.807913+0000", 2023-09-23T20:50:30.656 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.1489999999999994e-06 2023-09-23T20:50:30.656 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.656 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.656 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:50:30.656 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:02.807926+0000", 2023-09-23T20:50:30.656 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2798e-05 2023-09-23T20:50:30.657 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.657 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.657 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:50:30.657 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:02.807935+0000", 2023-09-23T20:50:30.657 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.6000000000000007e-06 2023-09-23T20:50:30.657 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.657 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.657 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:50:30.657 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:02.809005+0000", 2023-09-23T20:50:30.657 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0010698350000000001 2023-09-23T20:50:30.657 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.657 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.657 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:50:30.658 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:02.809020+0000", 2023-09-23T20:50:30.658 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.5116999999999999e-05 2023-09-23T20:50:30.658 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.658 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.658 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:50:30.658 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:03.785010+0000", 2023-09-23T20:50:30.658 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.97599018100000001 2023-09-23T20:50:30.658 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.658 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.658 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:50:30.658 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:03.785131+0000", 2023-09-23T20:50:30.659 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000120684 2023-09-23T20:50:30.659 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.659 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.659 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 0,4,7", 2023-09-23T20:50:30.659 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:03.785257+0000", 2023-09-23T20:50:30.659 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000126148 2023-09-23T20:50:30.659 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.659 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.659 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:50:30.659 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:03.789810+0000", 2023-09-23T20:50:30.659 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0045534390000000003 2023-09-23T20:50:30.659 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.660 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.660 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:30.660 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:03.794338+0000", 2023-09-23T20:50:30.660 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0045280169999999996 2023-09-23T20:50:30.660 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.660 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.660 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:30.660 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:03.794384+0000", 2023-09-23T20:50:30.660 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.5728e-05 2023-09-23T20:50:30.660 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.660 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.660 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:30.661 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:03.804122+0000", 2023-09-23T20:50:30.661 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0097375610000000005 2023-09-23T20:50:30.661 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.661 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.661 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:50:30.661 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:03.804191+0000", 2023-09-23T20:50:30.661 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.9214000000000005e-05 2023-09-23T20:50:30.661 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.661 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.661 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:50:30.661 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:03.804321+0000", 2023-09-23T20:50:30.662 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00013007200000000001 2023-09-23T20:50:30.662 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:50:30.662 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:50:30.662 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:50:30.662 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.662 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.662 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:181730 10.2 10:42ef278f:::benchmark_data_smithi018_422431_object77234:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e604)", 2023-09-23T20:50:30.662 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:49:02.812497+0000", 2023-09-23T20:50:30.662 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 87.789213881999999, 2023-09-23T20:50:30.662 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.99226569600000003, 2023-09-23T20:50:30.662 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:50:30.663 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:50:30.663 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:50:30.663 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:50:30.663 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:50:30.663 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 181730 2023-09-23T20:50:30.663 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.663 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:50:30.663 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.663 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:50:30.663 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:02.812497+0000", 2023-09-23T20:50:30.663 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:50:30.663 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.664 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.664 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:50:30.664 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:02.812497+0000", 2023-09-23T20:50:30.664 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:50:30.664 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.664 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.664 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:50:30.664 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:02.812496+0000", 2023-09-23T20:50:30.664 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:50:30.664 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.664 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.664 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:50:30.665 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:02.812502+0000", 2023-09-23T20:50:30.665 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.6969999999999998e-06 2023-09-23T20:50:30.665 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.665 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.665 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:50:30.665 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:02.812518+0000", 2023-09-23T20:50:30.665 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.645e-05 2023-09-23T20:50:30.665 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.665 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.665 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:50:30.665 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:02.812527+0000", 2023-09-23T20:50:30.665 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.7690000000000002e-06 2023-09-23T20:50:30.666 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.666 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.666 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:50:30.666 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:02.812633+0000", 2023-09-23T20:50:30.666 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000105853 2023-09-23T20:50:30.666 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.666 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.666 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:50:30.666 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:02.812647+0000", 2023-09-23T20:50:30.666 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.4372999999999999e-05 2023-09-23T20:50:30.666 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.667 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.667 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:50:30.667 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:03.787606+0000", 2023-09-23T20:50:30.667 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.97495889599999996 2023-09-23T20:50:30.667 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.667 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.667 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:50:30.667 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:03.787729+0000", 2023-09-23T20:50:30.667 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00012241899999999999 2023-09-23T20:50:30.667 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.667 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.668 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 0,4,7", 2023-09-23T20:50:30.668 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:03.787845+0000", 2023-09-23T20:50:30.668 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000116592 2023-09-23T20:50:30.668 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.668 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.668 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:50:30.668 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:03.792452+0000", 2023-09-23T20:50:30.668 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.004606995 2023-09-23T20:50:30.668 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.668 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.668 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:30.669 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:03.794420+0000", 2023-09-23T20:50:30.669 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0019673210000000002 2023-09-23T20:50:30.669 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.669 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.669 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:30.669 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:03.804366+0000", 2023-09-23T20:50:30.669 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0099460399999999997 2023-09-23T20:50:30.669 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.669 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.669 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:30.669 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:03.804592+0000", 2023-09-23T20:50:30.669 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00022602000000000001 2023-09-23T20:50:30.670 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.670 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.670 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:50:30.670 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:03.804647+0000", 2023-09-23T20:50:30.670 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.5288000000000003e-05 2023-09-23T20:50:30.670 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.670 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.670 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:50:30.670 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:03.804763+0000", 2023-09-23T20:50:30.670 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00011617599999999999 2023-09-23T20:50:30.671 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:50:30.671 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:50:30.671 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:50:30.671 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.671 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.671 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:181755 10.2 10:405e79b9:::benchmark_data_smithi018_422431_object77259:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e604)", 2023-09-23T20:50:30.671 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:49:02.840211+0000", 2023-09-23T20:50:30.671 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 87.761500613999999, 2023-09-23T20:50:30.671 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.96475393799999998, 2023-09-23T20:50:30.671 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:50:30.671 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:50:30.671 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:50:30.672 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:50:30.672 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:50:30.672 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 181755 2023-09-23T20:50:30.672 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.672 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:50:30.672 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.672 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:50:30.672 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:02.840211+0000", 2023-09-23T20:50:30.672 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:50:30.672 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.672 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.673 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:50:30.673 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:02.840211+0000", 2023-09-23T20:50:30.673 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:50:30.673 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.673 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.673 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:50:30.673 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:02.840209+0000", 2023-09-23T20:50:30.673 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:50:30.673 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.673 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.673 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:50:30.674 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:02.840218+0000", 2023-09-23T20:50:30.674 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.3579999999999992e-06 2023-09-23T20:50:30.674 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.674 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.674 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:50:30.674 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:02.840241+0000", 2023-09-23T20:50:30.674 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.2612e-05 2023-09-23T20:50:30.674 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.674 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.674 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:50:30.674 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:02.840250+0000", 2023-09-23T20:50:30.675 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.8379999999999997e-06 2023-09-23T20:50:30.675 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.675 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.675 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:50:30.675 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:02.840367+0000", 2023-09-23T20:50:30.675 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000116304 2023-09-23T20:50:30.675 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.675 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.675 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:50:30.675 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:02.840380+0000", 2023-09-23T20:50:30.675 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3517e-05 2023-09-23T20:50:30.675 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.676 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.676 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:50:30.676 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:03.789948+0000", 2023-09-23T20:50:30.676 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.94956776799999998 2023-09-23T20:50:30.676 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.676 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.676 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:50:30.676 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:03.790094+0000", 2023-09-23T20:50:30.676 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00014550399999999999 2023-09-23T20:50:30.676 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.676 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.676 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 0,4,7", 2023-09-23T20:50:30.677 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:03.790249+0000", 2023-09-23T20:50:30.677 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00015586899999999999 2023-09-23T20:50:30.677 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.677 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.677 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:50:30.677 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:03.792503+0000", 2023-09-23T20:50:30.677 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0022535839999999999 2023-09-23T20:50:30.677 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.677 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.677 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:30.677 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:03.794539+0000", 2023-09-23T20:50:30.677 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0020361519999999998 2023-09-23T20:50:30.678 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.678 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.678 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:30.678 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:03.804556+0000", 2023-09-23T20:50:30.678 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.010016729 2023-09-23T20:50:30.678 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.678 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.678 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:30.678 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:03.804812+0000", 2023-09-23T20:50:30.678 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000256479 2023-09-23T20:50:30.678 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.679 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.679 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:50:30.679 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:03.804864+0000", 2023-09-23T20:50:30.679 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.2125e-05 2023-09-23T20:50:30.679 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.679 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.679 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:50:30.679 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:03.804965+0000", 2023-09-23T20:50:30.679 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000100091 2023-09-23T20:50:30.679 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:50:30.679 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:50:30.679 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:50:30.679 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.680 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.680 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:188892 10.16 10:6b34c20f:::benchmark_data_smithi018_422431_object84396:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e611)", 2023-09-23T20:50:30.680 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:49:13.847828+0000", 2023-09-23T20:50:30.680 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 76.753883095999996, 2023-09-23T20:50:30.680 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.90492298900000001, 2023-09-23T20:50:30.680 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:50:30.680 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:50:30.680 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:50:30.680 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:50:30.680 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:50:30.680 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 188892 2023-09-23T20:50:30.681 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.681 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:50:30.681 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.681 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:50:30.681 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:13.847828+0000", 2023-09-23T20:50:30.681 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:50:30.681 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.681 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.681 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:50:30.681 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:13.847828+0000", 2023-09-23T20:50:30.681 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:50:30.681 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.682 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.682 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:50:30.682 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:13.847826+0000", 2023-09-23T20:50:30.682 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:50:30.682 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.682 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.682 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:50:30.682 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:13.847838+0000", 2023-09-23T20:50:30.682 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1559000000000001e-05 2023-09-23T20:50:30.682 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.682 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.682 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:50:30.683 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:13.847862+0000", 2023-09-23T20:50:30.683 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.3898000000000001e-05 2023-09-23T20:50:30.683 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.683 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.683 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:50:30.683 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:13.847870+0000", 2023-09-23T20:50:30.683 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.7150000000000004e-06 2023-09-23T20:50:30.683 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.683 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.683 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:50:30.683 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:13.847935+0000", 2023-09-23T20:50:30.683 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.4200000000000002e-05 2023-09-23T20:50:30.684 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.684 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.684 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:50:30.684 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:13.847947+0000", 2023-09-23T20:50:30.684 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2847000000000001e-05 2023-09-23T20:50:30.684 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.684 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.684 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:50:30.684 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:14.740972+0000", 2023-09-23T20:50:30.684 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.89302469699999998 2023-09-23T20:50:30.684 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.684 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.684 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:50:30.685 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:14.741059+0000", 2023-09-23T20:50:30.685 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.708e-05 2023-09-23T20:50:30.685 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.685 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.685 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 0,4", 2023-09-23T20:50:30.685 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:14.741330+0000", 2023-09-23T20:50:30.685 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00027128200000000002 2023-09-23T20:50:30.685 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.685 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.685 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:50:30.685 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:14.747248+0000", 2023-09-23T20:50:30.686 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0059171800000000002 2023-09-23T20:50:30.686 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.686 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.686 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:30.686 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:14.747402+0000", 2023-09-23T20:50:30.686 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00015389200000000001 2023-09-23T20:50:30.686 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.686 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.686 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:30.686 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:14.752655+0000", 2023-09-23T20:50:30.686 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0052536190000000002 2023-09-23T20:50:30.686 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.687 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.687 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:50:30.687 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:14.752706+0000", 2023-09-23T20:50:30.687 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.1087000000000001e-05 2023-09-23T20:50:30.687 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.687 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.687 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:50:30.687 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:14.752751+0000", 2023-09-23T20:50:30.687 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.4839999999999998e-05 2023-09-23T20:50:30.687 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:50:30.687 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:50:30.687 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:50:30.688 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.688 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.688 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:191302 10.13 10.2ff3e253 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e613)", 2023-09-23T20:50:30.688 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:49:16.976913+0000", 2023-09-23T20:50:30.688 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 73.624797760000007, 2023-09-23T20:50:30.688 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.92805713099999998, 2023-09-23T20:50:30.688 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:50:30.688 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:50:30.688 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:50:30.688 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:50:30.688 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:50:30.688 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 191302 2023-09-23T20:50:30.689 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.689 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:50:30.689 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.689 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:50:30.689 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:16.976913+0000", 2023-09-23T20:50:30.689 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:50:30.689 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.689 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.689 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:50:30.689 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:16.976913+0000", 2023-09-23T20:50:30.689 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:50:30.689 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.690 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.690 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:50:30.690 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:16.976911+0000", 2023-09-23T20:50:30.690 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999976 2023-09-23T20:50:30.690 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.690 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.690 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:50:30.690 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:16.976922+0000", 2023-09-23T20:50:30.690 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0530000000000001e-05 2023-09-23T20:50:30.690 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.690 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.690 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:50:30.691 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:16.976942+0000", 2023-09-23T20:50:30.691 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.0211e-05 2023-09-23T20:50:30.691 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.691 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.691 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:50:30.691 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:16.976954+0000", 2023-09-23T20:50:30.691 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1506e-05 2023-09-23T20:50:30.691 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.691 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.691 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:50:30.691 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:16.977025+0000", 2023-09-23T20:50:30.692 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.1711999999999994e-05 2023-09-23T20:50:30.692 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.692 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.692 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:50:30.692 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:16.977037+0000", 2023-09-23T20:50:30.692 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.169e-05 2023-09-23T20:50:30.692 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.692 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.692 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:50:30.692 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:17.904958+0000", 2023-09-23T20:50:30.692 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.92792081699999995 2023-09-23T20:50:30.692 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.693 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.693 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:50:30.693 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:17.904971+0000", 2023-09-23T20:50:30.693 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2856e-05 2023-09-23T20:50:30.693 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:50:30.693 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:50:30.693 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:50:30.693 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.693 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.693 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:191305 10.13 10.5fafba73 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e613)", 2023-09-23T20:50:30.693 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:49:16.979082+0000", 2023-09-23T20:50:30.693 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 73.622629283999999, 2023-09-23T20:50:30.694 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.92591810900000004, 2023-09-23T20:50:30.694 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:50:30.694 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:50:30.694 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:50:30.694 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:50:30.694 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:50:30.694 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 191305 2023-09-23T20:50:30.694 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.694 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:50:30.694 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.694 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:50:30.694 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:16.979082+0000", 2023-09-23T20:50:30.695 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:50:30.695 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.695 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.695 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:50:30.695 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:16.979082+0000", 2023-09-23T20:50:30.695 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:50:30.695 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.695 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.695 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:50:30.695 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:16.979080+0000", 2023-09-23T20:50:30.695 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:50:30.695 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.696 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.696 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:50:30.696 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:16.979087+0000", 2023-09-23T20:50:30.696 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.1940000000000001e-06 2023-09-23T20:50:30.696 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.696 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.696 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:50:30.696 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:16.979102+0000", 2023-09-23T20:50:30.696 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.4596000000000001e-05 2023-09-23T20:50:30.696 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.696 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.697 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:50:30.697 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:16.979111+0000", 2023-09-23T20:50:30.697 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.2960000000000004e-06 2023-09-23T20:50:30.697 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.697 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.697 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:50:30.697 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:16.979174+0000", 2023-09-23T20:50:30.697 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.2466999999999997e-05 2023-09-23T20:50:30.697 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.697 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.697 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:50:30.697 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:16.979190+0000", 2023-09-23T20:50:30.698 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.6282000000000001e-05 2023-09-23T20:50:30.698 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.698 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.698 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:50:30.698 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:17.904991+0000", 2023-09-23T20:50:30.698 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.92580114999999996 2023-09-23T20:50:30.698 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.698 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.698 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:50:30.698 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:17.905000+0000", 2023-09-23T20:50:30.698 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.9930000000000001e-06 2023-09-23T20:50:30.699 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:50:30.699 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:50:30.699 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:50:30.699 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.699 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.699 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:191333 10.13 10.f3564173 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e613)", 2023-09-23T20:50:30.699 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:49:17.011596+0000", 2023-09-23T20:50:30.699 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 73.590114745999998, 2023-09-23T20:50:30.699 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.89343019499999998, 2023-09-23T20:50:30.699 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:50:30.699 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:50:30.699 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:50:30.700 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:50:30.700 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:50:30.700 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 191333 2023-09-23T20:50:30.700 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.700 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:50:30.700 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.700 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:50:30.700 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:17.011596+0000", 2023-09-23T20:50:30.700 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:50:30.700 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.700 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.700 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:50:30.701 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:17.011596+0000", 2023-09-23T20:50:30.701 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:50:30.701 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.701 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.701 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:50:30.701 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:17.011595+0000", 2023-09-23T20:50:30.701 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:50:30.701 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.701 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.701 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:50:30.701 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:17.011602+0000", 2023-09-23T20:50:30.701 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.8229999999999997e-06 2023-09-23T20:50:30.702 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.702 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.702 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:50:30.702 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:17.011615+0000", 2023-09-23T20:50:30.702 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2955e-05 2023-09-23T20:50:30.702 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.702 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.702 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:50:30.702 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:17.011623+0000", 2023-09-23T20:50:30.702 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.721e-06 2023-09-23T20:50:30.702 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.702 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.703 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:50:30.703 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:17.011930+0000", 2023-09-23T20:50:30.703 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00030696500000000003 2023-09-23T20:50:30.703 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.703 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.703 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:50:30.703 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:17.011939+0000", 2023-09-23T20:50:30.703 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.1030000000000008e-06 2023-09-23T20:50:30.703 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.703 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.703 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:50:30.703 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:17.905019+0000", 2023-09-23T20:50:30.704 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.89307973600000001 2023-09-23T20:50:30.704 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.704 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.704 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:50:30.704 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:17.905027+0000", 2023-09-23T20:50:30.704 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.6279999999999996e-06 2023-09-23T20:50:30.704 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:50:30.704 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:50:30.704 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:50:30.704 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.704 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.705 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:206435 10.19 10:9978da35:::benchmark_data_smithi018_422431_object101939:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e634)", 2023-09-23T20:50:30.705 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:49:43.120891+0000", 2023-09-23T20:50:30.705 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 47.480819760000003, 2023-09-23T20:50:30.705 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0111183050000001, 2023-09-23T20:50:30.705 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:50:30.705 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:50:30.705 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:50:30.705 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:50:30.705 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:50:30.705 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 206435 2023-09-23T20:50:30.705 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.705 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:50:30.706 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.706 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:50:30.706 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:43.120891+0000", 2023-09-23T20:50:30.706 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:50:30.706 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.706 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.706 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:50:30.706 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:43.120891+0000", 2023-09-23T20:50:30.706 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:50:30.706 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.706 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.706 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:50:30.707 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:43.120890+0000", 2023-09-23T20:50:30.707 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:50:30.707 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.707 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.707 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:50:30.707 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:43.120898+0000", 2023-09-23T20:50:30.707 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.4930000000000001e-06 2023-09-23T20:50:30.707 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.707 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.707 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:50:30.707 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:43.120912+0000", 2023-09-23T20:50:30.707 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.4766000000000001e-05 2023-09-23T20:50:30.707 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.708 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.708 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:50:30.708 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:43.120922+0000", 2023-09-23T20:50:30.708 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.2639999999999997e-06 2023-09-23T20:50:30.708 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.708 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.708 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:50:30.708 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:43.120978+0000", 2023-09-23T20:50:30.708 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.6415000000000001e-05 2023-09-23T20:50:30.708 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.708 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.709 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:50:30.709 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:43.120988+0000", 2023-09-23T20:50:30.709 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.8940000000000005e-06 2023-09-23T20:50:30.709 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.709 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.709 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:50:30.709 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:44.123854+0000", 2023-09-23T20:50:30.709 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0028666020000001 2023-09-23T20:50:30.709 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.709 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.709 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:50:30.710 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:44.123957+0000", 2023-09-23T20:50:30.710 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000102089 2023-09-23T20:50:30.710 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.710 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.710 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:50:30.710 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:44.124189+0000", 2023-09-23T20:50:30.710 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000232823 2023-09-23T20:50:30.710 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.710 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.710 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:30.710 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:44.129682+0000", 2023-09-23T20:50:30.710 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0054925410000000001 2023-09-23T20:50:30.711 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.711 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.711 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:30.711 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:44.131921+0000", 2023-09-23T20:50:30.711 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0022387739999999998 2023-09-23T20:50:30.711 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.711 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.711 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:50:30.711 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:44.131955+0000", 2023-09-23T20:50:30.711 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.4096e-05 2023-09-23T20:50:30.711 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.711 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.712 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:50:30.712 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:44.131983+0000", 2023-09-23T20:50:30.712 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.8390000000000001e-05 2023-09-23T20:50:30.712 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.712 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.712 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:50:30.712 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:44.132010+0000", 2023-09-23T20:50:30.712 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.6579000000000001e-05 2023-09-23T20:50:30.712 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:50:30.712 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:50:30.712 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:50:30.712 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.713 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.713 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:206500 10.19 10:9de0cf19:::benchmark_data_smithi018_422431_object102004:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e634)", 2023-09-23T20:50:30.713 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:49:43.181444+0000", 2023-09-23T20:50:30.713 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 47.420267545999998, 2023-09-23T20:50:30.713 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.96146742799999996, 2023-09-23T20:50:30.713 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:50:30.713 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:50:30.713 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:50:30.713 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:50:30.713 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:50:30.713 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 206500 2023-09-23T20:50:30.713 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.714 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:50:30.714 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.714 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:50:30.714 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:43.181444+0000", 2023-09-23T20:50:30.714 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:50:30.714 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.714 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.714 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:50:30.714 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:43.181444+0000", 2023-09-23T20:50:30.714 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:50:30.714 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.714 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.715 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:50:30.715 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:43.181442+0000", 2023-09-23T20:50:30.715 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:50:30.715 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.715 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.715 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:50:30.715 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:43.181449+0000", 2023-09-23T20:50:30.715 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.9879999999999998e-06 2023-09-23T20:50:30.715 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.715 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.715 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:50:30.715 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:43.181468+0000", 2023-09-23T20:50:30.715 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.9381e-05 2023-09-23T20:50:30.716 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.716 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.716 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:50:30.716 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:43.181477+0000", 2023-09-23T20:50:30.716 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.1210000000000007e-06 2023-09-23T20:50:30.716 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.716 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.716 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:50:30.716 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:43.182520+0000", 2023-09-23T20:50:30.716 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.001043364 2023-09-23T20:50:30.716 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.716 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.717 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:50:30.717 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:43.182527+0000", 2023-09-23T20:50:30.717 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.8739999999999997e-06 2023-09-23T20:50:30.717 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.717 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.717 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:50:30.717 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:44.125761+0000", 2023-09-23T20:50:30.717 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.94323392900000003 2023-09-23T20:50:30.717 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.717 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.717 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:50:30.717 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:44.125876+0000", 2023-09-23T20:50:30.718 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000114979 2023-09-23T20:50:30.718 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.718 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.718 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:50:30.718 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:44.126000+0000", 2023-09-23T20:50:30.718 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000124332 2023-09-23T20:50:30.718 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.718 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.718 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:30.718 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:44.129707+0000", 2023-09-23T20:50:30.718 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.003706942 2023-09-23T20:50:30.718 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.719 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.719 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:50:30.719 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:44.133415+0000", 2023-09-23T20:50:30.719 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0037084230000000002 2023-09-23T20:50:30.719 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.719 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.719 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:30.719 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:44.142820+0000", 2023-09-23T20:50:30.719 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0094048949999999999 2023-09-23T20:50:30.719 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.719 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.719 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:50:30.720 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:44.142865+0000", 2023-09-23T20:50:30.720 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.4267000000000002e-05 2023-09-23T20:50:30.720 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.720 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.720 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:50:30.720 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:44.142911+0000", 2023-09-23T20:50:30.720 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.6483000000000002e-05 2023-09-23T20:50:30.720 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:50:30.720 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:50:30.720 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:50:30.720 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.720 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.721 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:206546 10.19 10:9e18b9c8:::benchmark_data_smithi018_422431_object102050:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e634)", 2023-09-23T20:50:30.721 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:49:43.223821+0000", 2023-09-23T20:50:30.721 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 47.377890207999997, 2023-09-23T20:50:30.721 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.91920754800000004, 2023-09-23T20:50:30.721 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:50:30.721 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:50:30.721 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:50:30.721 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:50:30.721 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:50:30.721 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 206546 2023-09-23T20:50:30.721 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.722 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:50:30.722 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.722 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:50:30.722 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:43.223821+0000", 2023-09-23T20:50:30.722 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:50:30.722 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.722 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.722 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:50:30.722 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:43.223821+0000", 2023-09-23T20:50:30.722 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:50:30.722 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.722 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.723 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:50:30.723 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:43.223819+0000", 2023-09-23T20:50:30.723 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:50:30.723 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.723 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.723 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:50:30.723 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:43.223829+0000", 2023-09-23T20:50:30.723 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.1519999999999998e-06 2023-09-23T20:50:30.723 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.723 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.723 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:50:30.724 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:43.223850+0000", 2023-09-23T20:50:30.724 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.1891000000000001e-05 2023-09-23T20:50:30.724 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.724 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.724 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:50:30.724 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:43.223860+0000", 2023-09-23T20:50:30.724 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.9040000000000004e-06 2023-09-23T20:50:30.724 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.724 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.724 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:50:30.724 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:43.223926+0000", 2023-09-23T20:50:30.724 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.5538999999999997e-05 2023-09-23T20:50:30.725 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.725 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.725 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:50:30.725 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:43.223938+0000", 2023-09-23T20:50:30.725 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2019e-05 2023-09-23T20:50:30.725 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.725 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.725 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:50:30.725 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:44.127196+0000", 2023-09-23T20:50:30.725 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.90325776700000004 2023-09-23T20:50:30.725 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.726 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.726 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:50:30.726 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:44.127274+0000", 2023-09-23T20:50:30.726 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.8268000000000003e-05 2023-09-23T20:50:30.726 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.726 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.726 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:50:30.726 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:44.127453+0000", 2023-09-23T20:50:30.726 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00017932199999999999 2023-09-23T20:50:30.726 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.726 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.727 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:30.727 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:44.129740+0000", 2023-09-23T20:50:30.727 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0022867859999999999 2023-09-23T20:50:30.727 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.727 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.727 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:50:30.727 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:44.138130+0000", 2023-09-23T20:50:30.727 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0083900420000000003 2023-09-23T20:50:30.727 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.727 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.727 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:30.727 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:44.142951+0000", 2023-09-23T20:50:30.728 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0048208950000000004 2023-09-23T20:50:30.728 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.728 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.728 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:50:30.728 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:44.142991+0000", 2023-09-23T20:50:30.728 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.0302000000000001e-05 2023-09-23T20:50:30.728 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.728 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.728 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:50:30.728 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:44.143029+0000", 2023-09-23T20:50:30.728 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.7218000000000002e-05 2023-09-23T20:50:30.729 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:50:30.729 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:50:30.729 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:50:30.729 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.729 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.729 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:206564 10.19 10:9e9a8172:::benchmark_data_smithi018_422431_object102068:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e634)", 2023-09-23T20:50:30.729 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:49:43.238458+0000", 2023-09-23T20:50:30.729 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 47.363253336, 2023-09-23T20:50:30.729 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.90469091400000001, 2023-09-23T20:50:30.729 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:50:30.729 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:50:30.729 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:50:30.729 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:50:30.730 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:50:30.730 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 206564 2023-09-23T20:50:30.730 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.730 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:50:30.730 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.730 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:50:30.730 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:43.238458+0000", 2023-09-23T20:50:30.730 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:50:30.730 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.730 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.731 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:50:30.731 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:43.238458+0000", 2023-09-23T20:50:30.731 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:50:30.731 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.731 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.731 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:50:30.731 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:43.238456+0000", 2023-09-23T20:50:30.731 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:50:30.731 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.731 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.731 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:50:30.732 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:43.238464+0000", 2023-09-23T20:50:30.732 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.6399999999999997e-06 2023-09-23T20:50:30.732 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.732 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.732 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:50:30.732 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:43.238480+0000", 2023-09-23T20:50:30.732 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.6557000000000001e-05 2023-09-23T20:50:30.732 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.732 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.732 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:50:30.732 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:43.238489+0000", 2023-09-23T20:50:30.732 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.9050000000000003e-06 2023-09-23T20:50:30.732 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.733 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.733 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:50:30.733 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:43.238553+0000", 2023-09-23T20:50:30.733 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.4148999999999999e-05 2023-09-23T20:50:30.733 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.733 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.733 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:50:30.733 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:43.238565+0000", 2023-09-23T20:50:30.733 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.153e-05 2023-09-23T20:50:30.733 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.733 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.733 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:50:30.734 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:44.128465+0000", 2023-09-23T20:50:30.734 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.88990053499999999 2023-09-23T20:50:30.734 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.734 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.734 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:50:30.734 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:44.128545+0000", 2023-09-23T20:50:30.734 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.9190999999999994e-05 2023-09-23T20:50:30.734 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.734 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.734 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:50:30.734 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:44.128696+0000", 2023-09-23T20:50:30.735 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00015161999999999999 2023-09-23T20:50:30.735 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.735 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.735 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:30.735 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:44.131891+0000", 2023-09-23T20:50:30.735 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0031945620000000002 2023-09-23T20:50:30.735 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.735 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.735 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:50:30.735 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:44.138137+0000", 2023-09-23T20:50:30.735 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0062459669999999998 2023-09-23T20:50:30.736 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.736 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.736 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:30.736 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:44.143075+0000", 2023-09-23T20:50:30.736 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0049384030000000001 2023-09-23T20:50:30.736 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.736 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.736 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:50:30.736 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:44.143114+0000", 2023-09-23T20:50:30.736 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.8538000000000003e-05 2023-09-23T20:50:30.736 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.736 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.736 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:50:30.737 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:44.143149+0000", 2023-09-23T20:50:30.737 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.5083000000000003e-05 2023-09-23T20:50:30.737 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:50:30.737 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:50:30.737 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:50:30.737 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.737 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.737 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:206566 10.19 10:9da35729:::benchmark_data_smithi018_422431_object102070:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e634)", 2023-09-23T20:50:30.737 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:49:43.239620+0000", 2023-09-23T20:50:30.737 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 47.362091282999998, 2023-09-23T20:50:30.737 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.90365303399999997, 2023-09-23T20:50:30.737 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:50:30.738 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:50:30.738 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:50:30.738 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:50:30.738 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:50:30.738 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 206566 2023-09-23T20:50:30.738 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.738 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:50:30.738 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.738 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:50:30.738 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:43.239620+0000", 2023-09-23T20:50:30.738 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:50:30.739 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.739 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.739 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:50:30.739 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:43.239620+0000", 2023-09-23T20:50:30.739 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:50:30.739 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.739 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.739 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:50:30.739 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:43.239619+0000", 2023-09-23T20:50:30.739 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.999999 2023-09-23T20:50:30.739 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.739 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.739 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:50:30.740 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:43.239625+0000", 2023-09-23T20:50:30.740 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.6139999999999997e-06 2023-09-23T20:50:30.740 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.740 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.740 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:50:30.740 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:43.239643+0000", 2023-09-23T20:50:30.740 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.7861999999999999e-05 2023-09-23T20:50:30.740 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.740 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.740 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:50:30.740 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:43.239652+0000", 2023-09-23T20:50:30.741 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.3040000000000005e-06 2023-09-23T20:50:30.741 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.741 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.741 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:50:30.741 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:43.239714+0000", 2023-09-23T20:50:30.741 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.2675000000000006e-05 2023-09-23T20:50:30.741 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.741 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.741 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:50:30.741 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:43.239723+0000", 2023-09-23T20:50:30.741 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.9989999999999997e-06 2023-09-23T20:50:30.742 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.742 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.742 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:50:30.742 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:44.129764+0000", 2023-09-23T20:50:30.742 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.89004099999999997 2023-09-23T20:50:30.742 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.742 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.742 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:50:30.742 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:44.129847+0000", 2023-09-23T20:50:30.742 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.2769999999999995e-05 2023-09-23T20:50:30.742 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.742 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.743 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:50:30.743 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:44.129969+0000", 2023-09-23T20:50:30.743 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00012168 2023-09-23T20:50:30.743 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.743 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.743 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:30.743 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:44.131948+0000", 2023-09-23T20:50:30.743 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0019796969999999999 2023-09-23T20:50:30.743 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.743 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.743 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:50:30.743 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:44.138145+0000", 2023-09-23T20:50:30.744 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0061965409999999999 2023-09-23T20:50:30.744 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.744 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.744 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:30.744 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:44.143192+0000", 2023-09-23T20:50:30.744 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0050471429999999996 2023-09-23T20:50:30.744 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.744 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.744 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:50:30.744 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:44.143234+0000", 2023-09-23T20:50:30.744 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.1622999999999998e-05 2023-09-23T20:50:30.744 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.744 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.745 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:50:30.745 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:44.143273+0000", 2023-09-23T20:50:30.745 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.9223e-05 2023-09-23T20:50:30.745 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:50:30.745 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:50:30.745 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:50:30.745 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.745 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.745 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:206572 10.19 10:9fe3ea92:::benchmark_data_smithi018_422431_object102076:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e634)", 2023-09-23T20:50:30.745 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:49:43.243035+0000", 2023-09-23T20:50:30.745 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 47.358675914000003, 2023-09-23T20:50:30.745 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.900309727, 2023-09-23T20:50:30.746 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:50:30.746 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:50:30.746 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:50:30.746 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:50:30.746 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:50:30.746 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 206572 2023-09-23T20:50:30.746 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.746 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:50:30.746 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.746 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:50:30.746 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:43.243035+0000", 2023-09-23T20:50:30.747 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:50:30.747 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.747 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.747 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:50:30.747 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:43.243035+0000", 2023-09-23T20:50:30.747 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:50:30.747 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.747 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.747 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:50:30.747 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:43.243034+0000", 2023-09-23T20:50:30.747 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:50:30.747 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.748 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.748 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:50:30.748 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:43.243042+0000", 2023-09-23T20:50:30.748 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.807e-06 2023-09-23T20:50:30.748 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.748 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.748 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:50:30.748 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:43.243063+0000", 2023-09-23T20:50:30.748 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.1617e-05 2023-09-23T20:50:30.748 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.748 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.748 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:50:30.749 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:43.243073+0000", 2023-09-23T20:50:30.749 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.7799999999999995e-06 2023-09-23T20:50:30.749 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.749 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.749 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:50:30.749 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:43.243123+0000", 2023-09-23T20:50:30.749 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.0204000000000002e-05 2023-09-23T20:50:30.749 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.749 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.749 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:50:30.749 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:43.243131+0000", 2023-09-23T20:50:30.749 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.1720000000000007e-06 2023-09-23T20:50:30.750 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.750 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.750 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:50:30.750 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:44.130818+0000", 2023-09-23T20:50:30.750 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.88768701299999997 2023-09-23T20:50:30.750 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.750 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.750 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:50:30.750 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:44.130903+0000", 2023-09-23T20:50:30.750 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.4565000000000001e-05 2023-09-23T20:50:30.750 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.751 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.751 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:50:30.751 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:44.131029+0000", 2023-09-23T20:50:30.751 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000125723 2023-09-23T20:50:30.751 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.751 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.751 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:30.751 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:44.133459+0000", 2023-09-23T20:50:30.751 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0024303810000000001 2023-09-23T20:50:30.751 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.751 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.752 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:50:30.752 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:44.138152+0000", 2023-09-23T20:50:30.752 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0046923920000000001 2023-09-23T20:50:30.752 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.752 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.752 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:30.752 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:44.143305+0000", 2023-09-23T20:50:30.752 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0051530919999999997 2023-09-23T20:50:30.752 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.752 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.752 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:50:30.752 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:44.143326+0000", 2023-09-23T20:50:30.753 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.1237999999999999e-05 2023-09-23T20:50:30.753 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:30.753 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:30.753 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:50:30.753 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:44.143345+0000", 2023-09-23T20:50:30.753 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.9154999999999999e-05 2023-09-23T20:50:30.753 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:50:30.753 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:50:30.753 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:50:30.753 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:50:30.753 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:50:30.753 INFO:teuthology.orchestra.run.smithi018.stdout:} 2023-09-23T20:50:30.754 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:50:30.754 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T20:50:30.756 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd reweight 1 1 2023-09-23T20:50:30.824 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:30.951 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:50:30.954 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:30.954 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:50:30.961 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T20:50:30.999 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T20:50:31.055 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:50:31.110 INFO:tasks.workunit.client.0.smithi018.stderr: Removing snap: 100% complete...done. 2023-09-23T20:50:31.118 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:31.121 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd rm testimg 2023-09-23T20:50:31.163 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:50:31.167 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:31.168 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:50:31.253 INFO:tasks.workunit.client.0.smithi018.stderr: Removing image: 100% complete...done. 2023-09-23T20:50:31.255 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd import --order 20 /tmp/rbd_import_export_466651/img testimg 2023-09-23T20:50:31.269 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:50:31.301 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:31.320 INFO:tasks.workunit.client.0.smithi018.stderr: Importing image: 100% complete...done. 2023-09-23T20:50:31.324 INFO:tasks.workunit.client.0.smithi018.stderr:rbd: --order is deprecated, use --object-size 2023-09-23T20:50:31.327 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_466651/img_v2 2023-09-23T20:50:31.351 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:50:31.357 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:31 smithi018 ceph-mon[184144]: failed to encode map e651 with expected crc 2023-09-23T20:50:31.357 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:31 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T20:50:31.357 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:31 smithi018 ceph-mon[184144]: osdmap e653: 8 total, 8 up, 7 in 2023-09-23T20:50:31.357 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:31 smithi018 ceph-mon[184144]: failed to encode map e653 with expected crc 2023-09-23T20:50:31.357 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:31 smithi018 ceph-mon[184144]: failed to encode map e653 with expected crc 2023-09-23T20:50:31.358 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:31.358 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:50:31.359 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:31 smithi018 ceph-mon[179647]: failed to encode map e651 with expected crc 2023-09-23T20:50:31.359 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:31 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T20:50:31.359 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:31 smithi018 ceph-mon[179647]: osdmap e653: 8 total, 8 up, 7 in 2023-09-23T20:50:31.359 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:31 smithi018 ceph-mon[179647]: failed to encode map e653 with expected crc 2023-09-23T20:50:31.359 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:31 smithi018 ceph-mon[179647]: failed to encode map e653 with expected crc 2023-09-23T20:50:31.379 INFO:tasks.workunit.client.0.smithi018.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-09-23T20:50:31.383 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_466651/img_v2 testimg_import 2023-09-23T20:50:31.449 INFO:tasks.workunit.client.0.smithi018.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2023-09-23T20:50:31.452 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd info testimg_import 2023-09-23T20:50:31.453 INFO:tasks.workunit.client.0.smithi018.stderr:+ grep order 2023-09-23T20:50:31.453 INFO:tasks.workunit.client.0.smithi018.stderr:+ awk '{print $2}' 2023-09-23T20:50:31.453 INFO:tasks.workunit.client.0.smithi018.stderr:+ grep 20 2023-09-23T20:50:31.459 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:50:31.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:31 smithi149 ceph-mon[170394]: failed to encode map e651 with expected crc 2023-09-23T20:50:31.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:31 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T20:50:31.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:31 smithi149 ceph-mon[170394]: osdmap e653: 8 total, 8 up, 7 in 2023-09-23T20:50:31.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:31 smithi149 ceph-mon[170394]: failed to encode map e653 with expected crc 2023-09-23T20:50:31.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:31 smithi149 ceph-mon[170394]: failed to encode map e653 with expected crc 2023-09-23T20:50:31.473 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:50:31.486 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:31.495 INFO:tasks.workunit.client.0.smithi018.stdout:20 2023-09-23T20:50:31.495 INFO:tasks.workunit.client.0.smithi018.stderr:+ rm /tmp/rbd_import_export_466651/img_v2 2023-09-23T20:50:31.497 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd remove testimg_import 2023-09-23T20:50:31.520 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:50:31.523 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:31.523 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:50:31.591 INFO:tasks.workunit.client.0.smithi018.stderr: Removing image: 100% complete...done. 2023-09-23T20:50:31.594 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd remove testimg 2023-09-23T20:50:31.604 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:50:31.624 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:50:31.650 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:31.684 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:50:31.687 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:31.687 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:50:31.700 INFO:tasks.workunit.client.0.smithi018.stderr: Removing image: 100% complete...done. 2023-09-23T20:50:31.704 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd import --image-feature layering /tmp/rbd_import_export_466651/img testimg 2023-09-23T20:50:31.708 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:50:31.771 INFO:tasks.workunit.client.0.smithi018.stderr: Importing image: 100% complete...done. 2023-09-23T20:50:31.775 INFO:tasks.workunit.client.0.smithi018.stderr:++ rbd info testimg 2023-09-23T20:50:31.775 INFO:tasks.workunit.client.0.smithi018.stderr:++ grep features 2023-09-23T20:50:31.788 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:50:31.816 INFO:tasks.workunit.client.0.smithi018.stderr:+ FEATURES_BEFORE=' features: layering 2023-09-23T20:50:31.816 INFO:tasks.workunit.client.0.smithi018.stderr: op_features: ' 2023-09-23T20:50:31.816 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_466651/img_v2 2023-09-23T20:50:31.821 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:31.860 INFO:tasks.workunit.client.0.smithi018.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-09-23T20:50:31.863 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_466651/img_v2 testimg_import 2023-09-23T20:50:31.889 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:50:31.908 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:50:31.912 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:31.912 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:50:31.922 INFO:tasks.workunit.client.0.smithi018.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2023-09-23T20:50:31.926 INFO:tasks.workunit.client.0.smithi018.stderr:++ rbd info testimg_import 2023-09-23T20:50:31.927 INFO:tasks.workunit.client.0.smithi018.stderr:++ grep features 2023-09-23T20:50:31.967 INFO:tasks.workunit.client.0.smithi018.stderr:+ FEATURES_AFTER=' features: layering 2023-09-23T20:50:31.967 INFO:tasks.workunit.client.0.smithi018.stderr: op_features: ' 2023-09-23T20:50:31.967 INFO:tasks.workunit.client.0.smithi018.stderr:+ '[' ' features: layering 2023-09-23T20:50:31.967 INFO:tasks.workunit.client.0.smithi018.stderr: op_features: ' '!=' ' features: layering 2023-09-23T20:50:31.967 INFO:tasks.workunit.client.0.smithi018.stderr: op_features: ' ']' 2023-09-23T20:50:31.967 INFO:tasks.workunit.client.0.smithi018.stderr:+ rm /tmp/rbd_import_export_466651/img_v2 2023-09-23T20:50:31.968 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd remove testimg_import 2023-09-23T20:50:32.013 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:50:32.036 INFO:tasks.workunit.client.0.smithi018.stderr: Removing image: 100% complete...done. 2023-09-23T20:50:32.038 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd remove testimg 2023-09-23T20:50:32.040 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:32.073 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:50:32.076 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:32.076 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:50:32.112 INFO:tasks.workunit.client.0.smithi018.stderr: Removing image: 100% complete...done. 2023-09-23T20:50:32.115 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd import --stripe-count 1000 --stripe-unit 4096 /tmp/rbd_import_export_466651/img testimg 2023-09-23T20:50:32.175 INFO:tasks.workunit.client.0.smithi018.stderr: Importing image: 100% complete...done. 2023-09-23T20:50:32.177 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:50:32.178 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_466651/img_v2 2023-09-23T20:50:32.203 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:32.221 INFO:tasks.workunit.client.0.smithi018.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-09-23T20:50:32.224 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_466651/img_v2 testimg_import 2023-09-23T20:50:32.237 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:50:32.240 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:32.240 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:50:32.290 INFO:tasks.workunit.client.0.smithi018.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2023-09-23T20:50:32.294 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd info testimg_import 2023-09-23T20:50:32.294 INFO:tasks.workunit.client.0.smithi018.stderr:+ grep 'stripe unit' 2023-09-23T20:50:32.294 INFO:tasks.workunit.client.0.smithi018.stderr:+ grep -Ei '(4 KiB|4096)' 2023-09-23T20:50:32.329 INFO:tasks.workunit.client.0.smithi018.stdout: stripe unit: 4 KiB 2023-09-23T20:50:32.330 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd info testimg_import 2023-09-23T20:50:32.330 INFO:tasks.workunit.client.0.smithi018.stderr:+ grep 'stripe count' 2023-09-23T20:50:32.330 INFO:tasks.workunit.client.0.smithi018.stderr:+ awk '{print $3}' 2023-09-23T20:50:32.330 INFO:tasks.workunit.client.0.smithi018.stderr:+ grep 1000 2023-09-23T20:50:32.341 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:50:32.368 INFO:tasks.workunit.client.0.smithi018.stdout:1000 2023-09-23T20:50:32.368 INFO:tasks.workunit.client.0.smithi018.stderr:+ rm /tmp/rbd_import_export_466651/img_v2 2023-09-23T20:50:32.369 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:32 smithi018 ceph-mon[184144]: pgmap v1601: 103 pgs: 103 active+clean; 6.5 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 402 KiB/s rd, 125 KiB/s wr, 139 op/s 2023-09-23T20:50:32.369 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:32 smithi018 ceph-mon[184144]: failed to encode map e653 with expected crc 2023-09-23T20:50:32.369 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:32 smithi018 ceph-mon[184144]: failed to encode map e654 with expected crc 2023-09-23T20:50:32.369 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:32 smithi018 ceph-mon[184144]: osdmap e654: 8 total, 8 up, 7 in 2023-09-23T20:50:32.369 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:32 smithi018 ceph-mon[184144]: failed to encode map e654 with expected crc 2023-09-23T20:50:32.369 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:32 smithi018 ceph-mon[179647]: pgmap v1601: 103 pgs: 103 active+clean; 6.5 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 402 KiB/s rd, 125 KiB/s wr, 139 op/s 2023-09-23T20:50:32.370 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:32 smithi018 ceph-mon[179647]: failed to encode map e653 with expected crc 2023-09-23T20:50:32.370 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:32 smithi018 ceph-mon[179647]: failed to encode map e654 with expected crc 2023-09-23T20:50:32.370 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:32 smithi018 ceph-mon[179647]: osdmap e654: 8 total, 8 up, 7 in 2023-09-23T20:50:32.370 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:32 smithi018 ceph-mon[179647]: failed to encode map e654 with expected crc 2023-09-23T20:50:32.370 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd remove testimg_import 2023-09-23T20:50:32.372 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:32.464 INFO:tasks.workunit.client.0.smithi018.stderr: Removing image: 3% complete... Removing image: 7% complete... Removing image: 10% complete... Removing image: 14% complete... Removing image: 17% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 32% complete... Removing image: 35% complete... Removing image: 39% complete... Removing image: 42% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 57% complete... Removing image: 60% complete... Removing image: 64% complete... Removing image: 67% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 82% complete... Removing image: 85% complete... Removing image: 89% complete... Removing image: 92% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2023-09-23T20:50:32.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:32 smithi149 ceph-mon[170394]: pgmap v1601: 103 pgs: 103 active+clean; 6.5 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 402 KiB/s rd, 125 KiB/s wr, 139 op/s 2023-09-23T20:50:32.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:32 smithi149 ceph-mon[170394]: failed to encode map e653 with expected crc 2023-09-23T20:50:32.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:32 smithi149 ceph-mon[170394]: failed to encode map e654 with expected crc 2023-09-23T20:50:32.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:32 smithi149 ceph-mon[170394]: osdmap e654: 8 total, 8 up, 7 in 2023-09-23T20:50:32.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:32 smithi149 ceph-mon[170394]: failed to encode map e654 with expected crc 2023-09-23T20:50:32.467 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd remove testimg 2023-09-23T20:50:32.487 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:50:32.492 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:32.492 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:50:32.565 INFO:tasks.workunit.client.0.smithi018.stderr: Removing image: 3% complete... Removing image: 7% complete... Removing image: 10% complete... Removing image: 14% complete... Removing image: 17% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 32% complete... Removing image: 35% complete... Removing image: 39% complete... Removing image: 42% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 57% complete... Removing image: 60% complete... Removing image: 64% complete... Removing image: 67% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 82% complete... Removing image: 85% complete... Removing image: 89% complete... Removing image: 92% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2023-09-23T20:50:32.568 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd import --image-format=2 /tmp/rbd_import_export_466651/img testimg 2023-09-23T20:50:32.593 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:50:32.620 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:32.629 INFO:tasks.workunit.client.0.smithi018.stderr: Importing image: 100% complete...done. 2023-09-23T20:50:32.632 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd snap create testimg@snap1 2023-09-23T20:50:32.653 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:50:32.656 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:32.656 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:50:32.757 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:50:32.786 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:32.972 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:50:32.976 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:32.977 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:50:33.078 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:50:33.113 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:33.128 INFO:tasks.workunit.client.0.smithi018.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2023-09-23T20:50:33.136 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd snap create testimg@snap2 2023-09-23T20:50:33.154 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:50:33.161 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:33.161 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:50:33.263 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:50:33.293 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:33.435 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:50:33.438 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:33.438 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:50:33.539 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:50:33.568 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:33.906 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:50:33.909 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:33.910 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:50:34.010 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:50:34.043 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:34.080 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:50:34.084 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:34.084 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:50:34.117 INFO:teuthology.orchestra.run.smithi018.stderr:nodeep-scrub is unset 2023-09-23T20:50:34.127 INFO:tasks.workunit.client.0.smithi018.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2023-09-23T20:50:34.136 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd snap protect testimg@snap2 2023-09-23T20:50:34.182 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_466651/snap_protect 2023-09-23T20:50:34.185 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:50:34.222 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:34.228 INFO:tasks.workunit.client.0.smithi018.stderr: Exporting image: 33% complete... Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-09-23T20:50:34.231 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_466651/snap_protect testimg_import 2023-09-23T20:50:34.376 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:50:34.377 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:34 smithi018 ceph-mon[179647]: pgmap v1603: 103 pgs: 103 active+clean; 4.8 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 287 KiB/s rd, 110 KiB/s wr, 130 op/s 2023-09-23T20:50:34.377 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:34 smithi018 ceph-mon[179647]: osdmap e655: 8 total, 8 up, 7 in 2023-09-23T20:50:34.377 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:34 smithi018 ceph-mon[179647]: failed to encode map e655 with expected crc 2023-09-23T20:50:34.377 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:34 smithi018 ceph-mon[179647]: failed to encode map e655 with expected crc 2023-09-23T20:50:34.377 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:34 smithi018 ceph-mon[179647]: failed to encode map e655 with expected crc 2023-09-23T20:50:34.377 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:34 smithi018 ceph-mon[179647]: failed to encode map e655 with expected crc 2023-09-23T20:50:34.377 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:34 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1180231817' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:50:34.378 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:34 smithi018 ceph-mon[184144]: pgmap v1603: 103 pgs: 103 active+clean; 4.8 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 287 KiB/s rd, 110 KiB/s wr, 130 op/s 2023-09-23T20:50:34.378 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:34 smithi018 ceph-mon[184144]: osdmap e655: 8 total, 8 up, 7 in 2023-09-23T20:50:34.378 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:34 smithi018 ceph-mon[184144]: failed to encode map e655 with expected crc 2023-09-23T20:50:34.378 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:34 smithi018 ceph-mon[184144]: failed to encode map e655 with expected crc 2023-09-23T20:50:34.378 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:34 smithi018 ceph-mon[184144]: failed to encode map e655 with expected crc 2023-09-23T20:50:34.378 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:34 smithi018 ceph-mon[184144]: failed to encode map e655 with expected crc 2023-09-23T20:50:34.378 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:34 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1180231817' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:50:34.380 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:34.381 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:50:34.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:34 smithi149 ceph-mon[170394]: pgmap v1603: 103 pgs: 103 active+clean; 4.8 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 287 KiB/s rd, 110 KiB/s wr, 130 op/s 2023-09-23T20:50:34.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:34 smithi149 ceph-mon[170394]: osdmap e655: 8 total, 8 up, 7 in 2023-09-23T20:50:34.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:34 smithi149 ceph-mon[170394]: failed to encode map e655 with expected crc 2023-09-23T20:50:34.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:34 smithi149 ceph-mon[170394]: failed to encode map e655 with expected crc 2023-09-23T20:50:34.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:34 smithi149 ceph-mon[170394]: failed to encode map e655 with expected crc 2023-09-23T20:50:34.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:34 smithi149 ceph-mon[170394]: failed to encode map e655 with expected crc 2023-09-23T20:50:34.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:34 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1180231817' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:50:34.481 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:50:34.510 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:34.592 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2023-09-23T20:50:34.592 INFO:teuthology.orchestra.run.smithi018.stdout: "ops": [], 2023-09-23T20:50:34.592 INFO:teuthology.orchestra.run.smithi018.stdout: "num_ops": 0 2023-09-23T20:50:34.592 INFO:teuthology.orchestra.run.smithi018.stdout:} 2023-09-23T20:50:34.725 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:50:34.728 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:34.729 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:50:34.830 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:50:34.858 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:35.022 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T20:50:35.093 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:50:35.099 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:35.100 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:50:35.200 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:50:35.230 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:35.280 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:50:35.283 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:50:34 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:50:34] "GET /metrics HTTP/1.1" 200 34155 "" "Prometheus/2.43.0" 2023-09-23T20:50:35.283 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:35 smithi018 ceph-mon[184144]: failed to encode map e655 with expected crc 2023-09-23T20:50:35.283 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:35 smithi018 ceph-mon[184144]: failed to encode map e655 with expected crc 2023-09-23T20:50:35.283 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:35 smithi018 ceph-mon[184144]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:50:35.283 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:35 smithi018 ceph-mon[184144]: Cluster is now healthy 2023-09-23T20:50:35.284 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:35 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1180231817' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:50:35.284 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:35 smithi018 ceph-mon[184144]: osdmap e656: 8 total, 8 up, 7 in 2023-09-23T20:50:35.284 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:35 smithi018 ceph-mon[184144]: failed to encode map e656 with expected crc 2023-09-23T20:50:35.284 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:35 smithi018 ceph-mon[184144]: failed to encode map e656 with expected crc 2023-09-23T20:50:35.284 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:35 smithi018 ceph-mon[184144]: failed to encode map e656 with expected crc 2023-09-23T20:50:35.284 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:35 smithi018 ceph-mon[184144]: failed to encode map e656 with expected crc 2023-09-23T20:50:35.284 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:35 smithi018 ceph-mon[179647]: failed to encode map e655 with expected crc 2023-09-23T20:50:35.284 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:35 smithi018 ceph-mon[179647]: failed to encode map e655 with expected crc 2023-09-23T20:50:35.284 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:35 smithi018 ceph-mon[179647]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:50:35.285 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:35 smithi018 ceph-mon[179647]: Cluster is now healthy 2023-09-23T20:50:35.285 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:35 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1180231817' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:50:35.285 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:35 smithi018 ceph-mon[179647]: osdmap e656: 8 total, 8 up, 7 in 2023-09-23T20:50:35.285 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:35 smithi018 ceph-mon[179647]: failed to encode map e656 with expected crc 2023-09-23T20:50:35.285 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:35 smithi018 ceph-mon[179647]: failed to encode map e656 with expected crc 2023-09-23T20:50:35.285 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:35 smithi018 ceph-mon[179647]: failed to encode map e656 with expected crc 2023-09-23T20:50:35.285 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:35 smithi018 ceph-mon[179647]: failed to encode map e656 with expected crc 2023-09-23T20:50:35.285 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:35.286 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:50:35.333 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:50:35.367 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:50:35.386 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:50:35.387 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:50:35.408 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:50:35.414 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:35.428 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:50:35.456 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:50:35.458 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:50:35.459 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:35.460 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:50:35.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:35 smithi149 ceph-mon[170394]: failed to encode map e655 with expected crc 2023-09-23T20:50:35.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:35 smithi149 ceph-mon[170394]: failed to encode map e655 with expected crc 2023-09-23T20:50:35.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:35 smithi149 ceph-mon[170394]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:50:35.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:35 smithi149 ceph-mon[170394]: Cluster is now healthy 2023-09-23T20:50:35.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:35 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1180231817' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:50:35.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:35 smithi149 ceph-mon[170394]: osdmap e656: 8 total, 8 up, 7 in 2023-09-23T20:50:35.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:35 smithi149 ceph-mon[170394]: failed to encode map e656 with expected crc 2023-09-23T20:50:35.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:35 smithi149 ceph-mon[170394]: failed to encode map e656 with expected crc 2023-09-23T20:50:35.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:35 smithi149 ceph-mon[170394]: failed to encode map e656 with expected crc 2023-09-23T20:50:35.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:35 smithi149 ceph-mon[170394]: failed to encode map e656 with expected crc 2023-09-23T20:50:35.479 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:50:35.501 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:50:35.502 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T20:50:35.502 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T20:50:35.502 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T20:50:35.502 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T20:50:35.503 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T20:50:35.503 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T20:50:35.503 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T20:50:35.503 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T20:50:35.560 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:50:35.592 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:35.629 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:50:35.632 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:35.632 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:50:35.733 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:50:35.761 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:35.796 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:50:35.799 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:35.799 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:50:35.900 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:50:35.929 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:35.939 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:50:35.965 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:50:35.969 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:35.970 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:50:36.071 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:50:36.099 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:36.131 INFO:teuthology.orchestra.run.smithi018.stderr:reweighted osd.1 to 1 (10000) 2023-09-23T20:50:36.145 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:50:36.148 INFO:tasks.workunit.client.0.smithi018.stderr: Importing image: 33% complete... Importing image: 66% complete... Importing image: 100% complete... Importing image: 100% complete...done. 2023-09-23T20:50:36.149 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:36.149 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:50:36.152 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd info testimg_import@snap1 2023-09-23T20:50:36.152 INFO:tasks.workunit.client.0.smithi018.stderr:+ grep 'protected: False' 2023-09-23T20:50:36.177 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:36 smithi018 ceph-mon[184144]: pgmap v1606: 103 pgs: 103 active+clean; 4.7 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 314 KiB/s rd, 27 KiB/s wr, 408 op/s 2023-09-23T20:50:36.178 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:36 smithi018 ceph-mon[184144]: failed to encode map e657 with expected crc 2023-09-23T20:50:36.178 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:36 smithi018 ceph-mon[184144]: osdmap e657: 8 total, 8 up, 7 in 2023-09-23T20:50:36.178 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:36 smithi018 ceph-mon[184144]: failed to encode map e657 with expected crc 2023-09-23T20:50:36.178 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:36 smithi018 ceph-mon[184144]: failed to encode map e657 with expected crc 2023-09-23T20:50:36.178 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:36 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1466881661' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:50:36.178 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:36 smithi018 ceph-mon[184144]: failed to encode map e657 with expected crc 2023-09-23T20:50:36.178 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:36 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3796336399' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T20:50:36.178 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:36 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T20:50:36.178 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:36 smithi018 ceph-mon[179647]: pgmap v1606: 103 pgs: 103 active+clean; 4.7 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 314 KiB/s rd, 27 KiB/s wr, 408 op/s 2023-09-23T20:50:36.179 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:36 smithi018 ceph-mon[179647]: failed to encode map e657 with expected crc 2023-09-23T20:50:36.179 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:36 smithi018 ceph-mon[179647]: osdmap e657: 8 total, 8 up, 7 in 2023-09-23T20:50:36.179 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:36 smithi018 ceph-mon[179647]: failed to encode map e657 with expected crc 2023-09-23T20:50:36.179 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:36 smithi018 ceph-mon[179647]: failed to encode map e657 with expected crc 2023-09-23T20:50:36.179 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:36 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1466881661' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:50:36.179 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:36 smithi018 ceph-mon[179647]: failed to encode map e657 with expected crc 2023-09-23T20:50:36.179 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:36 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3796336399' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T20:50:36.179 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:36 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T20:50:36.200 INFO:tasks.workunit.client.0.smithi018.stdout: protected: False 2023-09-23T20:50:36.201 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd info testimg_import@snap2 2023-09-23T20:50:36.201 INFO:tasks.workunit.client.0.smithi018.stderr:+ grep 'protected: True' 2023-09-23T20:50:36.239 INFO:tasks.workunit.client.0.smithi018.stdout: protected: True 2023-09-23T20:50:36.240 INFO:tasks.workunit.client.0.smithi018.stderr:+ rm /tmp/rbd_import_export_466651/snap_protect 2023-09-23T20:50:36.240 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd snap unprotect testimg@snap2 2023-09-23T20:50:36.251 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:50:36.279 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:36.283 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd snap unprotect testimg_import@snap2 2023-09-23T20:50:36.314 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:50:36.317 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:36.317 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:50:36.326 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd snap purge testimg 2023-09-23T20:50:36.418 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:50:36.442 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:36 smithi149 ceph-mon[170394]: pgmap v1606: 103 pgs: 103 active+clean; 4.7 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 314 KiB/s rd, 27 KiB/s wr, 408 op/s 2023-09-23T20:50:36.443 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:36 smithi149 ceph-mon[170394]: failed to encode map e657 with expected crc 2023-09-23T20:50:36.443 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:36 smithi149 ceph-mon[170394]: osdmap e657: 8 total, 8 up, 7 in 2023-09-23T20:50:36.443 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:36 smithi149 ceph-mon[170394]: failed to encode map e657 with expected crc 2023-09-23T20:50:36.443 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:36 smithi149 ceph-mon[170394]: failed to encode map e657 with expected crc 2023-09-23T20:50:36.443 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:36 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1466881661' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:50:36.443 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:36 smithi149 ceph-mon[170394]: failed to encode map e657 with expected crc 2023-09-23T20:50:36.443 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:36 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3796336399' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T20:50:36.443 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:36 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T20:50:36.445 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:36.478 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:50:36.481 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:36.482 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:50:36.582 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:50:36.609 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:36.642 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:50:36.645 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:36.646 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:50:36.746 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:50:36.776 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:36.795 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T20:50:36.911 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:50:36.915 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:36.915 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:50:36.995 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd reweight 7 1 2023-09-23T20:50:37.017 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:50:37.044 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:37.076 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:50:37.079 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:37.080 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:50:37.169 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:37 smithi018 ceph-mon[184144]: failed to encode map e657 with expected crc 2023-09-23T20:50:37.169 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:37 smithi018 ceph-mon[184144]: failed to encode map e658 with expected crc 2023-09-23T20:50:37.169 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:37 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T20:50:37.169 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:37 smithi018 ceph-mon[184144]: osdmap e658: 8 total, 8 up, 7 in 2023-09-23T20:50:37.169 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:37 smithi018 ceph-mon[184144]: failed to encode map e658 with expected crc 2023-09-23T20:50:37.169 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:37 smithi018 ceph-mon[184144]: failed to encode map e658 with expected crc 2023-09-23T20:50:37.169 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:37 smithi018 ceph-mon[184144]: pgmap v1609: 103 pgs: 103 active+clean; 4.7 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 444 KiB/s rd, 45 KiB/s wr, 579 op/s 2023-09-23T20:50:37.169 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:37 smithi018 ceph-mon[184144]: osdmap e659: 8 total, 8 up, 7 in 2023-09-23T20:50:37.170 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:37 smithi018 ceph-mon[184144]: failed to encode map e659 with expected crc 2023-09-23T20:50:37.170 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:37 smithi018 ceph-mon[184144]: failed to encode map e659 with expected crc 2023-09-23T20:50:37.170 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:37 smithi018 ceph-mon[179647]: failed to encode map e657 with expected crc 2023-09-23T20:50:37.170 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:37 smithi018 ceph-mon[179647]: failed to encode map e658 with expected crc 2023-09-23T20:50:37.170 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:37 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T20:50:37.170 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:37 smithi018 ceph-mon[179647]: osdmap e658: 8 total, 8 up, 7 in 2023-09-23T20:50:37.170 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:37 smithi018 ceph-mon[179647]: failed to encode map e658 with expected crc 2023-09-23T20:50:37.170 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:37 smithi018 ceph-mon[179647]: failed to encode map e658 with expected crc 2023-09-23T20:50:37.171 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:37 smithi018 ceph-mon[179647]: pgmap v1609: 103 pgs: 103 active+clean; 4.7 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 444 KiB/s rd, 45 KiB/s wr, 579 op/s 2023-09-23T20:50:37.171 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:37 smithi018 ceph-mon[179647]: osdmap e659: 8 total, 8 up, 7 in 2023-09-23T20:50:37.171 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:37 smithi018 ceph-mon[179647]: failed to encode map e659 with expected crc 2023-09-23T20:50:37.171 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:37 smithi018 ceph-mon[179647]: failed to encode map e659 with expected crc 2023-09-23T20:50:37.180 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:50:37.217 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:37.254 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:50:37.256 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:37.257 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:50:37.358 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:50:37.382 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:37 smithi149 ceph-mon[170394]: failed to encode map e657 with expected crc 2023-09-23T20:50:37.382 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:37 smithi149 ceph-mon[170394]: failed to encode map e658 with expected crc 2023-09-23T20:50:37.382 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:37 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T20:50:37.382 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:37 smithi149 ceph-mon[170394]: osdmap e658: 8 total, 8 up, 7 in 2023-09-23T20:50:37.382 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:37 smithi149 ceph-mon[170394]: failed to encode map e658 with expected crc 2023-09-23T20:50:37.382 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:37 smithi149 ceph-mon[170394]: failed to encode map e658 with expected crc 2023-09-23T20:50:37.382 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:37 smithi149 ceph-mon[170394]: pgmap v1609: 103 pgs: 103 active+clean; 4.7 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 444 KiB/s rd, 45 KiB/s wr, 579 op/s 2023-09-23T20:50:37.382 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:37 smithi149 ceph-mon[170394]: osdmap e659: 8 total, 8 up, 7 in 2023-09-23T20:50:37.382 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:37 smithi149 ceph-mon[170394]: failed to encode map e659 with expected crc 2023-09-23T20:50:37.382 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:37 smithi149 ceph-mon[170394]: failed to encode map e659 with expected crc 2023-09-23T20:50:37.384 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:37.417 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:50:37.420 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:37.421 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:50:37.435 INFO:teuthology.orchestra.run.smithi018.stdout:ERROR: (22) Invalid argument 2023-09-23T20:50:37.435 INFO:teuthology.orchestra.run.smithi018.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:50:37.470 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:50:37.521 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:50:37.548 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:37.580 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:50:37.583 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:37.584 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:50:37.684 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:50:37.713 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:37.724 INFO:tasks.workunit.client.0.smithi018.stderr: Removing all snapshots: 50% complete... Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2023-09-23T20:50:37.736 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd snap purge testimg_import 2023-09-23T20:50:37.809 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:50:37.812 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:37.812 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:50:37.831 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T20:50:37.879 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:50:37.914 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:50:37.950 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:37.993 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T20:50:38.002 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:50:38.006 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:38.006 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:50:38.107 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:50:38.134 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:38.168 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:50:38.171 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:38.171 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:50:38.272 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:50:38.303 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:38.439 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:50:38.442 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:38.443 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:50:38.543 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:50:38.570 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:38.605 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:50:38.607 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:38.608 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:50:38.708 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:50:38.741 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:38.777 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:50:38.779 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:38.780 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:50:38.839 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:38 smithi018 ceph-mon[179647]: failed to encode map e660 with expected crc 2023-09-23T20:50:38.840 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:38 smithi018 ceph-mon[179647]: osdmap e660: 8 total, 8 up, 7 in 2023-09-23T20:50:38.840 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:38 smithi018 ceph-mon[179647]: failed to encode map e660 with expected crc 2023-09-23T20:50:38.840 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:38 smithi018 ceph-mon[179647]: failed to encode map e660 with expected crc 2023-09-23T20:50:38.840 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:38 smithi018 ceph-mon[184144]: failed to encode map e660 with expected crc 2023-09-23T20:50:38.840 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:38 smithi018 ceph-mon[184144]: osdmap e660: 8 total, 8 up, 7 in 2023-09-23T20:50:38.840 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:38 smithi018 ceph-mon[184144]: failed to encode map e660 with expected crc 2023-09-23T20:50:38.840 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:38 smithi018 ceph-mon[184144]: failed to encode map e660 with expected crc 2023-09-23T20:50:38.881 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:50:38.904 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:38 smithi149 ceph-mon[170394]: failed to encode map e660 with expected crc 2023-09-23T20:50:38.904 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:38 smithi149 ceph-mon[170394]: osdmap e660: 8 total, 8 up, 7 in 2023-09-23T20:50:38.905 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:38 smithi149 ceph-mon[170394]: failed to encode map e660 with expected crc 2023-09-23T20:50:38.905 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:38 smithi149 ceph-mon[170394]: failed to encode map e660 with expected crc 2023-09-23T20:50:38.906 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:38.939 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:50:38.942 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:38.942 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:50:39.043 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:50:39.072 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:39.105 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:50:39.108 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:39.108 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:50:39.183 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:50:39.209 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:50:39.236 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:39.268 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:50:39.271 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:39.271 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:50:39.372 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:50:39.398 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:39.431 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:50:39.434 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:39.434 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:50:39.517 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:50:39.535 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:50:39.561 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:39.594 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:50:39.597 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:39.597 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:50:39.698 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:50:39.733 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:39.739 INFO:tasks.workunit.client.0.smithi018.stderr: Removing all snapshots: 50% complete... Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2023-09-23T20:50:39.746 INFO:teuthology.orchestra.run.smithi018.stderr:noscrub is set 2023-09-23T20:50:39.750 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd remove testimg 2023-09-23T20:50:39.849 INFO:tasks.workunit.client.0.smithi018.stderr: Removing image: 100% complete...done. 2023-09-23T20:50:39.854 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd remove testimg_import 2023-09-23T20:50:39.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:39 smithi018 ceph-mon[184144]: pgmap v1612: 103 pgs: 103 active+clean; 4.7 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 147 KiB/s rd, 21 KiB/s wr, 201 op/s 2023-09-23T20:50:39.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:39 smithi018 ceph-mon[184144]: failed to encode map e661 with expected crc 2023-09-23T20:50:39.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:39 smithi018 ceph-mon[184144]: osdmap e661: 8 total, 8 up, 7 in 2023-09-23T20:50:39.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:39 smithi018 ceph-mon[184144]: failed to encode map e661 with expected crc 2023-09-23T20:50:39.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:39 smithi018 ceph-mon[184144]: failed to encode map e661 with expected crc 2023-09-23T20:50:39.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:39 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/4048060732' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:50:39.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:39 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:50:39.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:39 smithi018 ceph-mon[179647]: pgmap v1612: 103 pgs: 103 active+clean; 4.7 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 147 KiB/s rd, 21 KiB/s wr, 201 op/s 2023-09-23T20:50:39.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:39 smithi018 ceph-mon[179647]: failed to encode map e661 with expected crc 2023-09-23T20:50:39.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:39 smithi018 ceph-mon[179647]: osdmap e661: 8 total, 8 up, 7 in 2023-09-23T20:50:39.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:39 smithi018 ceph-mon[179647]: failed to encode map e661 with expected crc 2023-09-23T20:50:39.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:39 smithi018 ceph-mon[179647]: failed to encode map e661 with expected crc 2023-09-23T20:50:39.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:39 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/4048060732' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:50:39.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:39 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:50:39.952 INFO:tasks.workunit.client.0.smithi018.stderr: Removing image: 100% complete...done. 2023-09-23T20:50:39.956 INFO:tasks.workunit.client.0.smithi018.stderr:+ tiered=0 2023-09-23T20:50:39.956 INFO:tasks.workunit.client.0.smithi018.stderr:+ ceph osd dump 2023-09-23T20:50:39.956 INFO:tasks.workunit.client.0.smithi018.stderr:+ grep '^pool' 2023-09-23T20:50:39.956 INFO:tasks.workunit.client.0.smithi018.stderr:+ grep ''\''rbd'\''' 2023-09-23T20:50:39.956 INFO:tasks.workunit.client.0.smithi018.stderr:+ grep tier 2023-09-23T20:50:39.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:39 smithi149 ceph-mon[170394]: pgmap v1612: 103 pgs: 103 active+clean; 4.7 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 147 KiB/s rd, 21 KiB/s wr, 201 op/s 2023-09-23T20:50:39.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:39 smithi149 ceph-mon[170394]: failed to encode map e661 with expected crc 2023-09-23T20:50:39.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:39 smithi149 ceph-mon[170394]: osdmap e661: 8 total, 8 up, 7 in 2023-09-23T20:50:39.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:39 smithi149 ceph-mon[170394]: failed to encode map e661 with expected crc 2023-09-23T20:50:39.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:39 smithi149 ceph-mon[170394]: failed to encode map e661 with expected crc 2023-09-23T20:50:39.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:39 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/4048060732' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:50:39.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:39 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T20:50:40.564 INFO:tasks.workunit.client.0.smithi018.stderr:+ dd if=/dev/urandom bs=1M seek=1 count=1 of=/tmp/rbd_import_export_466651/sparse1 2023-09-23T20:50:40.570 INFO:tasks.workunit.client.0.smithi018.stderr:1+0 records in 2023-09-23T20:50:40.570 INFO:tasks.workunit.client.0.smithi018.stderr:1+0 records out 2023-09-23T20:50:40.570 INFO:tasks.workunit.client.0.smithi018.stderr:1048576 bytes (1.0 MB, 1.0 MiB) copied, 0.00509056 s, 206 MB/s 2023-09-23T20:50:40.571 INFO:tasks.workunit.client.0.smithi018.stderr:+ dd if=/dev/urandom bs=1M count=1 of=/tmp/rbd_import_export_466651/sparse2 2023-09-23T20:50:40.576 INFO:tasks.workunit.client.0.smithi018.stderr:1+0 records in 2023-09-23T20:50:40.576 INFO:tasks.workunit.client.0.smithi018.stderr:1+0 records out 2023-09-23T20:50:40.576 INFO:tasks.workunit.client.0.smithi018.stderr:1048576 bytes (1.0 MB, 1.0 MiB) copied, 0.00502982 s, 208 MB/s 2023-09-23T20:50:40.576 INFO:tasks.workunit.client.0.smithi018.stderr:+ truncate /tmp/rbd_import_export_466651/sparse2 -s 2M 2023-09-23T20:50:40.588 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd rm sparse1 2023-09-23T20:50:40.626 INFO:tasks.workunit.client.0.smithi018.stderr: Removing image: 0% complete...failed. 2023-09-23T20:50:40.626 INFO:tasks.workunit.client.0.smithi018.stderr:rbd: delete error: (2) No such file or directory 2023-09-23T20:50:40.629 INFO:tasks.workunit.client.0.smithi018.stderr:+ true 2023-09-23T20:50:40.629 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_466651/sparse1 2023-09-23T20:50:40.681 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:50:40.685 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:40.686 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:50:40.695 INFO:tasks.workunit.client.0.smithi018.stderr: Importing image: 50% complete... Importing image: 100% complete...done. 2023-09-23T20:50:40.696 INFO:tasks.workunit.client.0.smithi018.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T20:50:40.696 INFO:tasks.workunit.client.0.smithi018.stderr:rbd: --order is deprecated, use --object-size 2023-09-23T20:50:40.699 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd ls -l 2023-09-23T20:50:40.699 INFO:tasks.workunit.client.0.smithi018.stderr:+ grep sparse1 2023-09-23T20:50:40.699 INFO:tasks.workunit.client.0.smithi018.stderr:+ grep -Ei '(2 MiB|2048k)' 2023-09-23T20:50:40.743 INFO:teuthology.orchestra.run.smithi018.stderr:reweighted osd.7 to 1 (10000) 2023-09-23T20:50:40.746 INFO:tasks.workunit.client.0.smithi018.stdout:sparse1 2 MiB 2 2023-09-23T20:50:40.746 INFO:tasks.workunit.client.0.smithi018.stderr:++ objects sparse1 2023-09-23T20:50:40.747 INFO:tasks.workunit.client.0.smithi018.stderr:++ image=sparse1 2023-09-23T20:50:40.747 INFO:tasks.workunit.client.0.smithi018.stderr:+++ rbd info sparse1 2023-09-23T20:50:40.747 INFO:tasks.workunit.client.0.smithi018.stderr:+++ grep block_name_prefix 2023-09-23T20:50:40.747 INFO:tasks.workunit.client.0.smithi018.stderr:+++ awk '{print $NF;}' 2023-09-23T20:50:40.787 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:50:40.792 INFO:tasks.workunit.client.0.smithi018.stderr:++ prefix=rbd_data.9375f14c0462 2023-09-23T20:50:40.793 INFO:tasks.workunit.client.0.smithi018.stderr:+++ grep rbd_data.9375f14c0462 2023-09-23T20:50:40.793 INFO:tasks.workunit.client.0.smithi018.stderr:++++ get_image_data_pool sparse1 2023-09-23T20:50:40.793 INFO:tasks.workunit.client.0.smithi018.stderr:++++ image=sparse1 2023-09-23T20:50:40.793 INFO:tasks.workunit.client.0.smithi018.stderr:+++ sed -e 's/rbd_data.9375f14c0462\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-09-23T20:50:40.793 INFO:tasks.workunit.client.0.smithi018.stderr:+++ sort -u 2023-09-23T20:50:40.793 INFO:tasks.workunit.client.0.smithi018.stderr:+++++ rbd info sparse1 2023-09-23T20:50:40.794 INFO:tasks.workunit.client.0.smithi018.stderr:+++++ grep 'data_pool: ' 2023-09-23T20:50:40.794 INFO:tasks.workunit.client.0.smithi018.stderr:+++++ awk -F: '{ print $NF }' 2023-09-23T20:50:40.814 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:40.840 INFO:tasks.workunit.client.0.smithi018.stderr:++++ data_pool= 2023-09-23T20:50:40.840 INFO:tasks.workunit.client.0.smithi018.stderr:++++ '[' -z ']' 2023-09-23T20:50:40.840 INFO:tasks.workunit.client.0.smithi018.stderr:++++ data_pool=rbd 2023-09-23T20:50:40.840 INFO:tasks.workunit.client.0.smithi018.stderr:++++ echo rbd 2023-09-23T20:50:40.840 INFO:tasks.workunit.client.0.smithi018.stderr:+++ rados ls -p rbd 2023-09-23T20:50:40.848 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:50:40.851 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:40.852 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:50:40.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:40 smithi018 ceph-mon[184144]: failed to encode map e661 with expected crc 2023-09-23T20:50:40.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:40 smithi018 ceph-mon[184144]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:50:40.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:40 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:50:40.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:40 smithi018 ceph-mon[184144]: osdmap e662: 8 total, 8 up, 7 in 2023-09-23T20:50:40.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:40 smithi018 ceph-mon[184144]: failed to encode map e662 with expected crc 2023-09-23T20:50:40.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:40 smithi018 ceph-mon[184144]: failed to encode map e662 with expected crc 2023-09-23T20:50:40.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:40 smithi018 ceph-mon[184144]: failed to encode map e662 with expected crc 2023-09-23T20:50:40.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:40 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/4167609495' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T20:50:40.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:40 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T20:50:40.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:40 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2015852418' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T20:50:40.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:40 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2015852418' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T20:50:40.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:40 smithi018 ceph-mon[179647]: failed to encode map e661 with expected crc 2023-09-23T20:50:40.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:40 smithi018 ceph-mon[179647]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:50:40.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:40 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:50:40.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:40 smithi018 ceph-mon[179647]: osdmap e662: 8 total, 8 up, 7 in 2023-09-23T20:50:40.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:40 smithi018 ceph-mon[179647]: failed to encode map e662 with expected crc 2023-09-23T20:50:40.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:40 smithi018 ceph-mon[179647]: failed to encode map e662 with expected crc 2023-09-23T20:50:40.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:40 smithi018 ceph-mon[179647]: failed to encode map e662 with expected crc 2023-09-23T20:50:40.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:40 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/4167609495' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T20:50:40.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:40 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T20:50:40.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:40 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2015852418' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T20:50:40.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:40 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2015852418' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T20:50:40.911 INFO:tasks.workunit.client.0.smithi018.stderr:++ objects=1 2023-09-23T20:50:40.911 INFO:tasks.workunit.client.0.smithi018.stderr:++ echo 1 2023-09-23T20:50:40.911 INFO:tasks.workunit.client.0.smithi018.stderr:+ '[' 0 -eq 1 -o 1 = 1 ']' 2023-09-23T20:50:40.911 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd export sparse1 /tmp/rbd_import_export_466651/sparse1.out 2023-09-23T20:50:40.947 INFO:tasks.workunit.client.0.smithi018.stderr: Exporting image: 50% complete... Exporting image: 100% complete...done. 2023-09-23T20:50:40.951 INFO:tasks.workunit.client.0.smithi018.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_466651/sparse1 /tmp/rbd_import_export_466651/sparse1.out 2023-09-23T20:50:40.951 INFO:tasks.workunit.client.0.smithi018.stderr:+ cmp -l /tmp/rbd_import_export_466651/sparse1 /tmp/rbd_import_export_466651/sparse1.out 2023-09-23T20:50:40.952 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T20:50:40.953 INFO:tasks.workunit.client.0.smithi018.stderr:++ stat /tmp/rbd_import_export_466651/sparse1 --format %b 2023-09-23T20:50:40.955 INFO:tasks.workunit.client.0.smithi018.stderr:+ origsize=2048 2023-09-23T20:50:40.955 INFO:tasks.workunit.client.0.smithi018.stderr:++ stat /tmp/rbd_import_export_466651/sparse1.out --format %b 2023-09-23T20:50:40.956 INFO:tasks.workunit.client.0.smithi018.stderr:+ exportsize=2048 2023-09-23T20:50:40.956 INFO:tasks.workunit.client.0.smithi018.stderr:+ difference=0 2023-09-23T20:50:40.956 INFO:tasks.workunit.client.0.smithi018.stderr:+ difference=0 2023-09-23T20:50:40.956 INFO:tasks.workunit.client.0.smithi018.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-09-23T20:50:40.957 INFO:tasks.workunit.client.0.smithi018.stderr:+ rm /tmp/rbd_import_export_466651/sparse1.out 2023-09-23T20:50:40.957 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd rm sparse1 2023-09-23T20:50:40.978 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:41.013 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T20:50:41.016 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:40 smithi149 ceph-mon[170394]: failed to encode map e661 with expected crc 2023-09-23T20:50:41.016 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:40 smithi149 ceph-mon[170394]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:50:41.016 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:40 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T20:50:41.016 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:40 smithi149 ceph-mon[170394]: osdmap e662: 8 total, 8 up, 7 in 2023-09-23T20:50:41.016 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:40 smithi149 ceph-mon[170394]: failed to encode map e662 with expected crc 2023-09-23T20:50:41.016 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:40 smithi149 ceph-mon[170394]: failed to encode map e662 with expected crc 2023-09-23T20:50:41.016 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:40 smithi149 ceph-mon[170394]: failed to encode map e662 with expected crc 2023-09-23T20:50:41.016 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:40 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/4167609495' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T20:50:41.016 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:40 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T20:50:41.016 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:40 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2015852418' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T20:50:41.017 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:40 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2015852418' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T20:50:41.017 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:41.017 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T20:50:41.050 INFO:tasks.workunit.client.0.smithi018.stderr: Removing image: 50% complete... Removing image: 100% complete...done. 2023-09-23T20:50:41.053 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd rm sparse2 2023-09-23T20:50:41.092 INFO:tasks.workunit.client.0.smithi018.stderr: Removing image: 0% complete...failed. 2023-09-23T20:50:41.092 INFO:tasks.workunit.client.0.smithi018.stderr:rbd: delete error: (2) No such file or directory 2023-09-23T20:50:41.095 INFO:tasks.workunit.client.0.smithi018.stderr:+ true 2023-09-23T20:50:41.095 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_466651/sparse2 2023-09-23T20:50:41.118 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:50:41.145 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:41.159 INFO:tasks.workunit.client.0.smithi018.stderr: Importing image: 50% complete... Importing image: 100% complete...done. 2023-09-23T20:50:41.159 INFO:tasks.workunit.client.0.smithi018.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T20:50:41.159 INFO:tasks.workunit.client.0.smithi018.stderr:rbd: --order is deprecated, use --object-size 2023-09-23T20:50:41.162 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd ls -l 2023-09-23T20:50:41.162 INFO:tasks.workunit.client.0.smithi018.stderr:+ grep sparse2 2023-09-23T20:50:41.162 INFO:tasks.workunit.client.0.smithi018.stderr:+ grep -Ei '(2 MiB|2048k)' 2023-09-23T20:50:41.178 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:50:41.181 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:41.181 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:50:41.197 INFO:tasks.workunit.client.0.smithi018.stdout:sparse2 2 MiB 2 2023-09-23T20:50:41.198 INFO:tasks.workunit.client.0.smithi018.stderr:++ objects sparse2 2023-09-23T20:50:41.198 INFO:tasks.workunit.client.0.smithi018.stderr:++ image=sparse2 2023-09-23T20:50:41.198 INFO:tasks.workunit.client.0.smithi018.stderr:+++ rbd info sparse2 2023-09-23T20:50:41.198 INFO:tasks.workunit.client.0.smithi018.stderr:+++ grep block_name_prefix 2023-09-23T20:50:41.198 INFO:tasks.workunit.client.0.smithi018.stderr:+++ awk '{print $NF;}' 2023-09-23T20:50:41.235 INFO:tasks.workunit.client.0.smithi018.stderr:++ prefix=rbd_data.b9e4cbedeea6 2023-09-23T20:50:41.235 INFO:tasks.workunit.client.0.smithi018.stderr:+++ grep rbd_data.b9e4cbedeea6 2023-09-23T20:50:41.235 INFO:tasks.workunit.client.0.smithi018.stderr:++++ get_image_data_pool sparse2 2023-09-23T20:50:41.235 INFO:tasks.workunit.client.0.smithi018.stderr:++++ image=sparse2 2023-09-23T20:50:41.236 INFO:tasks.workunit.client.0.smithi018.stderr:+++ sed -e 's/rbd_data.b9e4cbedeea6\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-09-23T20:50:41.236 INFO:tasks.workunit.client.0.smithi018.stderr:+++ sort -u 2023-09-23T20:50:41.236 INFO:tasks.workunit.client.0.smithi018.stderr:+++++ rbd info sparse2 2023-09-23T20:50:41.236 INFO:tasks.workunit.client.0.smithi018.stderr:+++++ grep 'data_pool: ' 2023-09-23T20:50:41.236 INFO:tasks.workunit.client.0.smithi018.stderr:+++++ awk -F: '{ print $NF }' 2023-09-23T20:50:41.272 INFO:tasks.workunit.client.0.smithi018.stderr:++++ data_pool= 2023-09-23T20:50:41.272 INFO:tasks.workunit.client.0.smithi018.stderr:++++ '[' -z ']' 2023-09-23T20:50:41.272 INFO:tasks.workunit.client.0.smithi018.stderr:++++ data_pool=rbd 2023-09-23T20:50:41.273 INFO:tasks.workunit.client.0.smithi018.stderr:++++ echo rbd 2023-09-23T20:50:41.273 INFO:tasks.workunit.client.0.smithi018.stderr:+++ rados ls -p rbd 2023-09-23T20:50:41.282 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:50:41.308 INFO:tasks.workunit.client.0.smithi018.stderr:++ objects=0 2023-09-23T20:50:41.309 INFO:tasks.workunit.client.0.smithi018.stderr:++ echo 0 2023-09-23T20:50:41.309 INFO:tasks.workunit.client.0.smithi018.stderr:+ '[' 0 -eq 1 -o 0 = 0 ']' 2023-09-23T20:50:41.309 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd export sparse2 /tmp/rbd_import_export_466651/sparse2.out 2023-09-23T20:50:41.316 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:41.353 INFO:tasks.workunit.client.0.smithi018.stderr: Exporting image: 50% complete... Exporting image: 100% complete...done. 2023-09-23T20:50:41.355 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd reweight 3 1 2023-09-23T20:50:41.356 INFO:tasks.workunit.client.0.smithi018.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_466651/sparse2 /tmp/rbd_import_export_466651/sparse2.out 2023-09-23T20:50:41.356 INFO:tasks.workunit.client.0.smithi018.stderr:+ cmp -l /tmp/rbd_import_export_466651/sparse2 /tmp/rbd_import_export_466651/sparse2.out 2023-09-23T20:50:41.359 INFO:tasks.workunit.client.0.smithi018.stderr:++ stat /tmp/rbd_import_export_466651/sparse2 --format %b 2023-09-23T20:50:41.360 INFO:tasks.workunit.client.0.smithi018.stderr:+ origsize=2048 2023-09-23T20:50:41.360 INFO:tasks.workunit.client.0.smithi018.stderr:++ stat /tmp/rbd_import_export_466651/sparse2.out --format %b 2023-09-23T20:50:41.362 INFO:tasks.workunit.client.0.smithi018.stderr:+ exportsize=2048 2023-09-23T20:50:41.362 INFO:tasks.workunit.client.0.smithi018.stderr:+ difference=0 2023-09-23T20:50:41.362 INFO:tasks.workunit.client.0.smithi018.stderr:+ difference=0 2023-09-23T20:50:41.362 INFO:tasks.workunit.client.0.smithi018.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-09-23T20:50:41.362 INFO:tasks.workunit.client.0.smithi018.stderr:+ rm /tmp/rbd_import_export_466651/sparse2.out 2023-09-23T20:50:41.363 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd rm sparse2 2023-09-23T20:50:41.413 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:50:41.418 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:41.418 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:50:41.465 INFO:tasks.workunit.client.0.smithi018.stderr: Removing image: 50% complete... Removing image: 100% complete...done. 2023-09-23T20:50:41.468 INFO:tasks.workunit.client.0.smithi018.stderr:+ truncate /tmp/rbd_import_export_466651/sparse1 -s 10M 2023-09-23T20:50:41.469 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd import --new-format --order 20 - sparse1 2023-09-23T20:50:41.519 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T20:50:41.553 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:41.580 INFO:tasks.workunit.client.0.smithi018.stderr: Importing image: 100% complete...done. 2023-09-23T20:50:41.581 INFO:tasks.workunit.client.0.smithi018.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T20:50:41.581 INFO:tasks.workunit.client.0.smithi018.stderr:rbd: --order is deprecated, use --object-size 2023-09-23T20:50:41.584 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd ls -l 2023-09-23T20:50:41.584 INFO:tasks.workunit.client.0.smithi018.stderr:+ grep sparse1 2023-09-23T20:50:41.584 INFO:tasks.workunit.client.0.smithi018.stderr:+ grep -Ei '(10 MiB|10240k)' 2023-09-23T20:50:41.605 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T20:50:41.609 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:41.609 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T20:50:41.628 INFO:tasks.workunit.client.0.smithi018.stdout:sparse1 10 MiB 2 2023-09-23T20:50:41.629 INFO:tasks.workunit.client.0.smithi018.stderr:++ objects sparse1 2023-09-23T20:50:41.629 INFO:tasks.workunit.client.0.smithi018.stderr:++ image=sparse1 2023-09-23T20:50:41.630 INFO:tasks.workunit.client.0.smithi018.stderr:+++ grep block_name_prefix 2023-09-23T20:50:41.630 INFO:tasks.workunit.client.0.smithi018.stderr:+++ rbd info sparse1 2023-09-23T20:50:41.630 INFO:tasks.workunit.client.0.smithi018.stderr:+++ awk '{print $NF;}' 2023-09-23T20:50:41.669 INFO:tasks.workunit.client.0.smithi018.stderr:++ prefix=rbd_data.ba11d2e5cd90 2023-09-23T20:50:41.670 INFO:tasks.workunit.client.0.smithi018.stderr:+++ grep rbd_data.ba11d2e5cd90 2023-09-23T20:50:41.670 INFO:tasks.workunit.client.0.smithi018.stderr:++++ get_image_data_pool sparse1 2023-09-23T20:50:41.670 INFO:tasks.workunit.client.0.smithi018.stderr:++++ image=sparse1 2023-09-23T20:50:41.670 INFO:tasks.workunit.client.0.smithi018.stderr:+++ sed -e 's/rbd_data.ba11d2e5cd90\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-09-23T20:50:41.670 INFO:tasks.workunit.client.0.smithi018.stderr:+++ sort -u 2023-09-23T20:50:41.670 INFO:tasks.workunit.client.0.smithi018.stderr:+++++ rbd info sparse1 2023-09-23T20:50:41.671 INFO:tasks.workunit.client.0.smithi018.stderr:+++++ grep 'data_pool: ' 2023-09-23T20:50:41.671 INFO:tasks.workunit.client.0.smithi018.stderr:+++++ awk -F: '{ print $NF }' 2023-09-23T20:50:41.673 INFO:teuthology.orchestra.run.smithi018.stdout:ERROR: (22) Invalid argument 2023-09-23T20:50:41.673 INFO:teuthology.orchestra.run.smithi018.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T20:50:41.710 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:50:41.714 INFO:tasks.workunit.client.0.smithi018.stderr:++++ data_pool= 2023-09-23T20:50:41.714 INFO:tasks.workunit.client.0.smithi018.stderr:++++ '[' -z ']' 2023-09-23T20:50:41.714 INFO:tasks.workunit.client.0.smithi018.stderr:++++ data_pool=rbd 2023-09-23T20:50:41.714 INFO:tasks.workunit.client.0.smithi018.stderr:++++ echo rbd 2023-09-23T20:50:41.715 INFO:tasks.workunit.client.0.smithi018.stderr:+++ rados ls -p rbd 2023-09-23T20:50:41.752 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:41.767 INFO:tasks.workunit.client.0.smithi018.stderr:++ objects=1 2023-09-23T20:50:41.767 INFO:tasks.workunit.client.0.smithi018.stderr:++ echo 1 2023-09-23T20:50:41.767 INFO:tasks.workunit.client.0.smithi018.stderr:+ '[' 0 -eq 1 -o 1 = 1 ']' 2023-09-23T20:50:41.767 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd export sparse1 /tmp/rbd_import_export_466651/sparse1.out 2023-09-23T20:50:41.800 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:41 smithi018 ceph-mon[184144]: pgmap v1615: 103 pgs: 103 active+clean; 4.8 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 126 KiB/s rd, 12 KiB/s wr, 166 op/s 2023-09-23T20:50:41.801 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:41 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T20:50:41.801 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:41 smithi018 ceph-mon[184144]: osdmap e663: 8 total, 8 up, 7 in 2023-09-23T20:50:41.801 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:41 smithi018 ceph-mon[184144]: failed to encode map e663 with expected crc 2023-09-23T20:50:41.801 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:41 smithi018 ceph-mon[184144]: osdmap e664: 8 total, 8 up, 7 in 2023-09-23T20:50:41.801 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:41 smithi018 ceph-mon[179647]: pgmap v1615: 103 pgs: 103 active+clean; 4.8 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 126 KiB/s rd, 12 KiB/s wr, 166 op/s 2023-09-23T20:50:41.801 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:41 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T20:50:41.801 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:41 smithi018 ceph-mon[179647]: osdmap e663: 8 total, 8 up, 7 in 2023-09-23T20:50:41.801 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:41 smithi018 ceph-mon[179647]: failed to encode map e663 with expected crc 2023-09-23T20:50:41.801 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:41 smithi018 ceph-mon[179647]: osdmap e664: 8 total, 8 up, 7 in 2023-09-23T20:50:41.812 INFO:tasks.workunit.client.0.smithi018.stderr: Exporting image: 10% complete... Exporting image: 20% complete... Exporting image: 30% complete... Exporting image: 40% complete... Exporting image: 50% complete... Exporting image: 60% complete... Exporting image: 70% complete... Exporting image: 80% complete... Exporting image: 90% complete... Exporting image: 100% complete...done. 2023-09-23T20:50:41.815 INFO:tasks.workunit.client.0.smithi018.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_466651/sparse1 /tmp/rbd_import_export_466651/sparse1.out 2023-09-23T20:50:41.815 INFO:tasks.workunit.client.0.smithi018.stderr:+ cmp -l /tmp/rbd_import_export_466651/sparse1 /tmp/rbd_import_export_466651/sparse1.out 2023-09-23T20:50:41.824 INFO:tasks.workunit.client.0.smithi018.stderr:++ stat /tmp/rbd_import_export_466651/sparse1 --format %b 2023-09-23T20:50:41.826 INFO:tasks.workunit.client.0.smithi018.stderr:+ origsize=2048 2023-09-23T20:50:41.826 INFO:tasks.workunit.client.0.smithi018.stderr:++ stat /tmp/rbd_import_export_466651/sparse1.out --format %b 2023-09-23T20:50:41.827 INFO:tasks.workunit.client.0.smithi018.stderr:+ exportsize=2048 2023-09-23T20:50:41.827 INFO:tasks.workunit.client.0.smithi018.stderr:+ difference=0 2023-09-23T20:50:41.828 INFO:tasks.workunit.client.0.smithi018.stderr:+ difference=0 2023-09-23T20:50:41.828 INFO:tasks.workunit.client.0.smithi018.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-09-23T20:50:41.828 INFO:tasks.workunit.client.0.smithi018.stderr:+ rm /tmp/rbd_import_export_466651/sparse1.out 2023-09-23T20:50:41.829 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd rm sparse1 2023-09-23T20:50:41.917 INFO:tasks.workunit.client.0.smithi018.stderr: Removing image: 10% complete... Removing image: 20% complete... Removing image: 30% complete... Removing image: 40% complete... Removing image: 50% complete... Removing image: 60% complete... Removing image: 70% complete... Removing image: 80% complete... Removing image: 90% complete... Removing image: 100% complete...done. 2023-09-23T20:50:41.920 INFO:tasks.workunit.client.0.smithi018.stderr:+ dd if=/dev/urandom bs=2M count=1 of=/tmp/rbd_import_export_466651/sparse2 oflag=append conv=notrunc 2023-09-23T20:50:41.931 INFO:tasks.workunit.client.0.smithi018.stderr:1+0 records in 2023-09-23T20:50:41.931 INFO:tasks.workunit.client.0.smithi018.stderr:1+0 records out 2023-09-23T20:50:41.931 INFO:tasks.workunit.client.0.smithi018.stderr:2097152 bytes (2.1 MB, 2.0 MiB) copied, 0.0101951 s, 206 MB/s 2023-09-23T20:50:41.931 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd import --new-format --order 20 - sparse2 2023-09-23T20:50:42.010 INFO:teuthology.orchestra.run.smithi018.stdout:osd.0: {} 2023-09-23T20:50:42.011 INFO:tasks.workunit.client.0.smithi018.stderr: Importing image: 100% complete...done. 2023-09-23T20:50:42.011 INFO:tasks.workunit.client.0.smithi018.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T20:50:42.011 INFO:tasks.workunit.client.0.smithi018.stderr:rbd: --order is deprecated, use --object-size 2023-09-23T20:50:42.013 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd ls -l 2023-09-23T20:50:42.014 INFO:tasks.workunit.client.0.smithi018.stderr:+ grep sparse2 2023-09-23T20:50:42.014 INFO:tasks.workunit.client.0.smithi018.stderr:+ grep -Ei '(4 MiB|4096k)' 2023-09-23T20:50:42.030 INFO:teuthology.orchestra.run.smithi018.stdout:osd.1: {} 2023-09-23T20:50:42.041 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:50:42.044 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:42.045 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:50:42.051 INFO:tasks.workunit.client.0.smithi018.stdout:sparse2 4 MiB 2 2023-09-23T20:50:42.052 INFO:tasks.workunit.client.0.smithi018.stderr:++ objects sparse2 2023-09-23T20:50:42.052 INFO:tasks.workunit.client.0.smithi018.stderr:++ image=sparse2 2023-09-23T20:50:42.053 INFO:tasks.workunit.client.0.smithi018.stderr:+++ rbd info sparse2 2023-09-23T20:50:42.053 INFO:tasks.workunit.client.0.smithi018.stderr:+++ grep block_name_prefix 2023-09-23T20:50:42.053 INFO:tasks.workunit.client.0.smithi018.stderr:+++ awk '{print $NF;}' 2023-09-23T20:50:42.053 INFO:teuthology.orchestra.run.smithi018.stdout:osd.2: {} 2023-09-23T20:50:42.072 INFO:teuthology.orchestra.run.smithi018.stdout:osd.3: {} 2023-09-23T20:50:42.089 INFO:tasks.workunit.client.0.smithi018.stderr:++ prefix=rbd_data.ba3e62cad79b 2023-09-23T20:50:42.090 INFO:tasks.workunit.client.0.smithi018.stderr:+++ grep rbd_data.ba3e62cad79b 2023-09-23T20:50:42.090 INFO:tasks.workunit.client.0.smithi018.stderr:++++ get_image_data_pool sparse2 2023-09-23T20:50:42.090 INFO:tasks.workunit.client.0.smithi018.stderr:++++ image=sparse2 2023-09-23T20:50:42.090 INFO:tasks.workunit.client.0.smithi018.stderr:+++ sed -e 's/rbd_data.ba3e62cad79b\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-09-23T20:50:42.091 INFO:tasks.workunit.client.0.smithi018.stderr:+++ sort -u 2023-09-23T20:50:42.094 INFO:tasks.workunit.client.0.smithi018.stderr:+++++ rbd info sparse2 2023-09-23T20:50:42.094 INFO:tasks.workunit.client.0.smithi018.stderr:+++++ grep 'data_pool: ' 2023-09-23T20:50:42.094 INFO:tasks.workunit.client.0.smithi018.stderr:+++++ awk -F: '{ print $NF }' 2023-09-23T20:50:42.094 INFO:teuthology.orchestra.run.smithi018.stdout:osd.4: {} 2023-09-23T20:50:42.114 INFO:teuthology.orchestra.run.smithi018.stdout:osd.5: {} 2023-09-23T20:50:42.128 INFO:tasks.workunit.client.0.smithi018.stderr:++++ data_pool= 2023-09-23T20:50:42.128 INFO:tasks.workunit.client.0.smithi018.stderr:++++ '[' -z ']' 2023-09-23T20:50:42.128 INFO:tasks.workunit.client.0.smithi018.stderr:++++ data_pool=rbd 2023-09-23T20:50:42.128 INFO:tasks.workunit.client.0.smithi018.stderr:++++ echo rbd 2023-09-23T20:50:42.128 INFO:tasks.workunit.client.0.smithi018.stderr:+++ rados ls -p rbd 2023-09-23T20:50:42.134 INFO:teuthology.orchestra.run.smithi018.stdout:osd.6: {} 2023-09-23T20:50:42.146 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T20:50:42.157 INFO:teuthology.orchestra.run.smithi018.stdout:osd.7: {} 2023-09-23T20:50:42.159 INFO:teuthology.orchestra.run.smithi018.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T20:50:42.159 INFO:teuthology.orchestra.run.smithi018.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T20:50:42.160 INFO:teuthology.orchestra.run.smithi018.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T20:50:42.160 INFO:teuthology.orchestra.run.smithi018.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T20:50:42.160 INFO:teuthology.orchestra.run.smithi018.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T20:50:42.160 INFO:teuthology.orchestra.run.smithi018.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T20:50:42.160 INFO:teuthology.orchestra.run.smithi018.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T20:50:42.160 INFO:teuthology.orchestra.run.smithi018.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T20:50:42.162 INFO:tasks.workunit.client.0.smithi018.stderr:++ objects='0 2023-09-23T20:50:42.162 INFO:tasks.workunit.client.0.smithi018.stderr:2 2023-09-23T20:50:42.162 INFO:tasks.workunit.client.0.smithi018.stderr:3' 2023-09-23T20:50:42.162 INFO:tasks.workunit.client.0.smithi018.stderr:++ echo 0 2 3 2023-09-23T20:50:42.162 INFO:tasks.workunit.client.0.smithi018.stderr:+ '[' 0 -eq 1 -o '0 2 3' = '0 2 3' ']' 2023-09-23T20:50:42.162 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd export sparse2 /tmp/rbd_import_export_466651/sparse2.out 2023-09-23T20:50:42.169 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:41 smithi149 ceph-mon[170394]: pgmap v1615: 103 pgs: 103 active+clean; 4.8 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 126 KiB/s rd, 12 KiB/s wr, 166 op/s 2023-09-23T20:50:42.169 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:41 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T20:50:42.169 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:41 smithi149 ceph-mon[170394]: osdmap e663: 8 total, 8 up, 7 in 2023-09-23T20:50:42.169 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:41 smithi149 ceph-mon[170394]: failed to encode map e663 with expected crc 2023-09-23T20:50:42.170 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:41 smithi149 ceph-mon[170394]: osdmap e664: 8 total, 8 up, 7 in 2023-09-23T20:50:42.171 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:42.183 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T20:50:42.202 INFO:tasks.workunit.client.0.smithi018.stderr: Exporting image: 25% complete... Exporting image: 50% complete... Exporting image: 75% complete... Exporting image: 100% complete...done. 2023-09-23T20:50:42.205 INFO:tasks.workunit.client.0.smithi018.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_466651/sparse2 /tmp/rbd_import_export_466651/sparse2.out 2023-09-23T20:50:42.205 INFO:tasks.workunit.client.0.smithi018.stderr:+ cmp -l /tmp/rbd_import_export_466651/sparse2 /tmp/rbd_import_export_466651/sparse2.out 2023-09-23T20:50:42.206 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T20:50:42.209 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:42.209 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T20:50:42.210 INFO:tasks.workunit.client.0.smithi018.stderr:++ stat /tmp/rbd_import_export_466651/sparse2 --format %b 2023-09-23T20:50:42.210 INFO:tasks.workunit.client.0.smithi018.stderr:+ origsize=6144 2023-09-23T20:50:42.211 INFO:tasks.workunit.client.0.smithi018.stderr:++ stat /tmp/rbd_import_export_466651/sparse2.out --format %b 2023-09-23T20:50:42.212 INFO:tasks.workunit.client.0.smithi018.stderr:+ exportsize=6144 2023-09-23T20:50:42.212 INFO:tasks.workunit.client.0.smithi018.stderr:+ difference=0 2023-09-23T20:50:42.212 INFO:tasks.workunit.client.0.smithi018.stderr:+ difference=0 2023-09-23T20:50:42.212 INFO:tasks.workunit.client.0.smithi018.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-09-23T20:50:42.212 INFO:tasks.workunit.client.0.smithi018.stderr:+ rm /tmp/rbd_import_export_466651/sparse2.out 2023-09-23T20:50:42.213 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd rm sparse2 2023-09-23T20:50:42.311 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:50:42.318 INFO:tasks.workunit.client.0.smithi018.stderr: Removing image: 25% complete... Removing image: 50% complete... Removing image: 75% complete... Removing image: 100% complete...done. 2023-09-23T20:50:42.322 INFO:tasks.workunit.client.0.smithi018.stderr:+ echo 'partially-sparse file imports to partially-sparse image' 2023-09-23T20:50:42.322 INFO:tasks.workunit.client.0.smithi018.stdout:partially-sparse file imports to partially-sparse image 2023-09-23T20:50:42.322 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_466651/sparse1 sparse 2023-09-23T20:50:42.341 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:42.396 INFO:tasks.workunit.client.0.smithi018.stderr: Importing image: 10% complete... Importing image: 20% complete... Importing image: 30% complete... Importing image: 40% complete... Importing image: 50% complete... Importing image: 60% complete... Importing image: 70% complete... Importing image: 80% complete... Importing image: 90% complete... Importing image: 100% complete...done. 2023-09-23T20:50:42.396 INFO:tasks.workunit.client.0.smithi018.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T20:50:42.397 INFO:tasks.workunit.client.0.smithi018.stderr:rbd: --order is deprecated, use --object-size 2023-09-23T20:50:42.400 INFO:tasks.workunit.client.0.smithi018.stderr:++ objects sparse 2023-09-23T20:50:42.400 INFO:tasks.workunit.client.0.smithi018.stderr:++ image=sparse 2023-09-23T20:50:42.400 INFO:tasks.workunit.client.0.smithi018.stderr:+++ rbd info sparse 2023-09-23T20:50:42.400 INFO:tasks.workunit.client.0.smithi018.stderr:+++ grep block_name_prefix 2023-09-23T20:50:42.400 INFO:tasks.workunit.client.0.smithi018.stderr:+++ awk '{print $NF;}' 2023-09-23T20:50:42.437 INFO:tasks.workunit.client.0.smithi018.stderr:++ prefix=rbd_data.ba71d2a75e6f 2023-09-23T20:50:42.438 INFO:tasks.workunit.client.0.smithi018.stderr:+++ grep rbd_data.ba71d2a75e6f 2023-09-23T20:50:42.438 INFO:tasks.workunit.client.0.smithi018.stderr:++++ get_image_data_pool sparse 2023-09-23T20:50:42.438 INFO:tasks.workunit.client.0.smithi018.stderr:+++ sed -e 's/rbd_data.ba71d2a75e6f\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-09-23T20:50:42.438 INFO:tasks.workunit.client.0.smithi018.stderr:++++ image=sparse 2023-09-23T20:50:42.438 INFO:tasks.workunit.client.0.smithi018.stderr:+++ sort -u 2023-09-23T20:50:42.440 INFO:tasks.workunit.client.0.smithi018.stderr:+++++ rbd info sparse 2023-09-23T20:50:42.440 INFO:tasks.workunit.client.0.smithi018.stderr:+++++ grep 'data_pool: ' 2023-09-23T20:50:42.440 INFO:tasks.workunit.client.0.smithi018.stderr:+++++ awk -F: '{ print $NF }' 2023-09-23T20:50:42.477 INFO:tasks.workunit.client.0.smithi018.stderr:++++ data_pool= 2023-09-23T20:50:42.478 INFO:tasks.workunit.client.0.smithi018.stderr:++++ '[' -z ']' 2023-09-23T20:50:42.478 INFO:tasks.workunit.client.0.smithi018.stderr:++++ data_pool=rbd 2023-09-23T20:50:42.478 INFO:tasks.workunit.client.0.smithi018.stderr:++++ echo rbd 2023-09-23T20:50:42.478 INFO:tasks.workunit.client.0.smithi018.stderr:+++ rados ls -p rbd 2023-09-23T20:50:42.510 INFO:tasks.workunit.client.0.smithi018.stderr:++ objects=1 2023-09-23T20:50:42.511 INFO:tasks.workunit.client.0.smithi018.stderr:++ echo 1 2023-09-23T20:50:42.511 INFO:tasks.workunit.client.0.smithi018.stderr:+ '[' 0 -eq 1 -o 1 = 1 ']' 2023-09-23T20:50:42.511 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd rm sparse 2023-09-23T20:50:42.599 INFO:tasks.workunit.client.0.smithi018.stderr: Removing image: 10% complete... Removing image: 20% complete... Removing image: 30% complete... Removing image: 40% complete... Removing image: 50% complete... Removing image: 60% complete... Removing image: 70% complete... Removing image: 80% complete... Removing image: 90% complete... Removing image: 100% complete...done. 2023-09-23T20:50:42.602 INFO:tasks.workunit.client.0.smithi018.stderr:+ echo 'zeros import through stdin to sparse image' 2023-09-23T20:50:42.602 INFO:tasks.workunit.client.0.smithi018.stdout:zeros import through stdin to sparse image 2023-09-23T20:50:42.602 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:50:42.603 INFO:tasks.workunit.client.0.smithi018.stderr:+ dd if=/dev/zero bs=1M count=4 2023-09-23T20:50:42.603 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd import --new-format - sparse 2023-09-23T20:50:42.606 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:42.606 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:50:42.657 INFO:tasks.workunit.client.0.smithi018.stderr:4+0 records in 2023-09-23T20:50:42.657 INFO:tasks.workunit.client.0.smithi018.stderr:4+0 records out 2023-09-23T20:50:42.657 INFO:tasks.workunit.client.0.smithi018.stderr:4194304 bytes (4.2 MB, 4.0 MiB) copied, 0.0541182 s, 77.5 MB/s 2023-09-23T20:50:42.658 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:50:42.670 INFO:tasks.workunit.client.0.smithi018.stderr: Importing image: 100% complete...done. 2023-09-23T20:50:42.671 INFO:tasks.workunit.client.0.smithi018.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T20:50:42.675 INFO:tasks.workunit.client.0.smithi018.stderr:++ objects sparse 2023-09-23T20:50:42.675 INFO:tasks.workunit.client.0.smithi018.stderr:++ image=sparse 2023-09-23T20:50:42.676 INFO:tasks.workunit.client.0.smithi018.stderr:+++ rbd info sparse 2023-09-23T20:50:42.676 INFO:tasks.workunit.client.0.smithi018.stderr:+++ grep block_name_prefix 2023-09-23T20:50:42.676 INFO:tasks.workunit.client.0.smithi018.stderr:+++ awk '{print $NF;}' 2023-09-23T20:50:42.707 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T20:50:42.708 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T20:50:42.713 INFO:tasks.workunit.client.0.smithi018.stderr:++ prefix=rbd_data.94535c7f88f2 2023-09-23T20:50:42.713 INFO:tasks.workunit.client.0.smithi018.stderr:+++ grep rbd_data.94535c7f88f2 2023-09-23T20:50:42.714 INFO:tasks.workunit.client.0.smithi018.stderr:+++ sed -e 's/rbd_data.94535c7f88f2\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-09-23T20:50:42.714 INFO:tasks.workunit.client.0.smithi018.stderr:++++ get_image_data_pool sparse 2023-09-23T20:50:42.714 INFO:tasks.workunit.client.0.smithi018.stderr:++++ image=sparse 2023-09-23T20:50:42.714 INFO:tasks.workunit.client.0.smithi018.stderr:+++ sort -u 2023-09-23T20:50:42.715 INFO:tasks.workunit.client.0.smithi018.stderr:+++++ rbd info sparse 2023-09-23T20:50:42.715 INFO:tasks.workunit.client.0.smithi018.stderr:+++++ awk -F: '{ print $NF }' 2023-09-23T20:50:42.715 INFO:tasks.workunit.client.0.smithi018.stderr:+++++ grep 'data_pool: ' 2023-09-23T20:50:42.738 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:42.758 INFO:tasks.workunit.client.0.smithi018.stderr:++++ data_pool= 2023-09-23T20:50:42.758 INFO:tasks.workunit.client.0.smithi018.stderr:++++ '[' -z ']' 2023-09-23T20:50:42.759 INFO:tasks.workunit.client.0.smithi018.stderr:++++ data_pool=rbd 2023-09-23T20:50:42.759 INFO:tasks.workunit.client.0.smithi018.stderr:++++ echo rbd 2023-09-23T20:50:42.759 INFO:tasks.workunit.client.0.smithi018.stderr:+++ rados ls -p rbd 2023-09-23T20:50:42.786 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T20:50:42.790 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:42.790 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T20:50:42.795 INFO:tasks.workunit.client.0.smithi018.stderr:++ objects= 2023-09-23T20:50:42.795 INFO:tasks.workunit.client.0.smithi018.stderr:++ echo 2023-09-23T20:50:42.796 INFO:tasks.workunit.client.0.smithi018.stderr:+ '[' 0 -eq 1 -o '' = '' ']' 2023-09-23T20:50:42.796 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd rm sparse 2023-09-23T20:50:42.883 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:42 smithi018 ceph-mon[179647]: failed to encode map e663 with expected crc 2023-09-23T20:50:42.884 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:42 smithi018 ceph-mon[179647]: failed to encode map e663 with expected crc 2023-09-23T20:50:42.884 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:42 smithi018 ceph-mon[179647]: failed to encode map e664 with expected crc 2023-09-23T20:50:42.884 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:42 smithi018 ceph-mon[179647]: failed to encode map e664 with expected crc 2023-09-23T20:50:42.884 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:42 smithi018 ceph-mon[179647]: failed to encode map e664 with expected crc 2023-09-23T20:50:42.884 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:42 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/4126838828' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:50:42.884 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:42 smithi018 ceph-mon[184144]: failed to encode map e663 with expected crc 2023-09-23T20:50:42.884 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:42 smithi018 ceph-mon[184144]: failed to encode map e663 with expected crc 2023-09-23T20:50:42.885 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:42 smithi018 ceph-mon[184144]: failed to encode map e664 with expected crc 2023-09-23T20:50:42.885 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:42 smithi018 ceph-mon[184144]: failed to encode map e664 with expected crc 2023-09-23T20:50:42.885 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:42 smithi018 ceph-mon[184144]: failed to encode map e664 with expected crc 2023-09-23T20:50:42.885 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:42 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/4126838828' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:50:42.891 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:50:42.902 INFO:tasks.workunit.client.0.smithi018.stderr: Removing image: 100% complete...done. 2023-09-23T20:50:42.905 INFO:tasks.workunit.client.0.smithi018.stdout:zeros export to sparse file 2023-09-23T20:50:42.906 INFO:tasks.workunit.client.0.smithi018.stderr:+ echo 'zeros export to sparse file' 2023-09-23T20:50:42.906 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd create --new-format sparse --size 4 2023-09-23T20:50:42.928 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:42.953 INFO:tasks.workunit.client.0.smithi018.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T20:50:42.956 INFO:tasks.workunit.client.0.smithi018.stderr:++ rbd info sparse 2023-09-23T20:50:42.956 INFO:tasks.workunit.client.0.smithi018.stderr:++ grep block_name_prefix 2023-09-23T20:50:42.956 INFO:tasks.workunit.client.0.smithi018.stderr:++ awk '{print $NF;}' 2023-09-23T20:50:42.973 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:50:42.977 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:42.978 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:50:43.000 INFO:tasks.workunit.client.0.smithi018.stderr:+ prefix=rbd_data.baadb5653d7c 2023-09-23T20:50:43.000 INFO:tasks.workunit.client.0.smithi018.stderr:+ dd if=/dev/zero bs=4M count=1 2023-09-23T20:50:43.001 INFO:tasks.workunit.client.0.smithi018.stderr:++ get_image_data_pool sparse 2023-09-23T20:50:43.001 INFO:tasks.workunit.client.0.smithi018.stderr:++ image=sparse 2023-09-23T20:50:43.001 INFO:tasks.workunit.client.0.smithi018.stderr:+++ rbd info sparse 2023-09-23T20:50:43.001 INFO:tasks.workunit.client.0.smithi018.stderr:+++ grep 'data_pool: ' 2023-09-23T20:50:43.001 INFO:tasks.workunit.client.0.smithi018.stderr:+++ awk -F: '{ print $NF }' 2023-09-23T20:50:43.038 INFO:tasks.workunit.client.0.smithi018.stderr:++ data_pool= 2023-09-23T20:50:43.038 INFO:tasks.workunit.client.0.smithi018.stderr:++ '[' -z ']' 2023-09-23T20:50:43.038 INFO:tasks.workunit.client.0.smithi018.stderr:++ data_pool=rbd 2023-09-23T20:50:43.038 INFO:tasks.workunit.client.0.smithi018.stderr:++ echo rbd 2023-09-23T20:50:43.038 INFO:tasks.workunit.client.0.smithi018.stderr:+ rados -p rbd put rbd_data.baadb5653d7c.000000000000 - 2023-09-23T20:50:43.069 INFO:tasks.workunit.client.0.smithi018.stderr:1+0 records in 2023-09-23T20:50:43.069 INFO:tasks.workunit.client.0.smithi018.stderr:1+0 records out 2023-09-23T20:50:43.069 INFO:tasks.workunit.client.0.smithi018.stderr:4194304 bytes (4.2 MB, 4.0 MiB) copied, 0.0681401 s, 61.6 MB/s 2023-09-23T20:50:43.079 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T20:50:43.085 INFO:tasks.workunit.client.0.smithi018.stderr:++ objects sparse 2023-09-23T20:50:43.085 INFO:tasks.workunit.client.0.smithi018.stderr:++ image=sparse 2023-09-23T20:50:43.086 INFO:tasks.workunit.client.0.smithi018.stderr:+++ rbd info sparse 2023-09-23T20:50:43.086 INFO:tasks.workunit.client.0.smithi018.stderr:+++ grep block_name_prefix 2023-09-23T20:50:43.086 INFO:tasks.workunit.client.0.smithi018.stderr:+++ awk '{print $NF;}' 2023-09-23T20:50:43.087 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:50:43.102 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:42 smithi149 ceph-mon[170394]: failed to encode map e663 with expected crc 2023-09-23T20:50:43.102 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:42 smithi149 ceph-mon[170394]: failed to encode map e663 with expected crc 2023-09-23T20:50:43.102 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:42 smithi149 ceph-mon[170394]: failed to encode map e664 with expected crc 2023-09-23T20:50:43.103 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:42 smithi149 ceph-mon[170394]: failed to encode map e664 with expected crc 2023-09-23T20:50:43.103 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:42 smithi149 ceph-mon[170394]: failed to encode map e664 with expected crc 2023-09-23T20:50:43.103 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:42 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/4126838828' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T20:50:43.105 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:43.137 INFO:tasks.workunit.client.0.smithi018.stderr:++ prefix=rbd_data.baadb5653d7c 2023-09-23T20:50:43.138 INFO:tasks.workunit.client.0.smithi018.stderr:+++ sed -e 's/rbd_data.baadb5653d7c\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-09-23T20:50:43.138 INFO:tasks.workunit.client.0.smithi018.stderr:++++ get_image_data_pool sparse 2023-09-23T20:50:43.138 INFO:tasks.workunit.client.0.smithi018.stderr:++++ image=sparse 2023-09-23T20:50:43.138 INFO:tasks.workunit.client.0.smithi018.stderr:+++ grep rbd_data.baadb5653d7c 2023-09-23T20:50:43.138 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T20:50:43.139 INFO:tasks.workunit.client.0.smithi018.stderr:+++ sort -u 2023-09-23T20:50:43.140 INFO:tasks.workunit.client.0.smithi018.stderr:+++++ rbd info sparse 2023-09-23T20:50:43.140 INFO:tasks.workunit.client.0.smithi018.stderr:+++++ grep 'data_pool: ' 2023-09-23T20:50:43.140 INFO:tasks.workunit.client.0.smithi018.stderr:+++++ awk -F: '{ print $NF }' 2023-09-23T20:50:43.141 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:43.141 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T20:50:43.177 INFO:tasks.workunit.client.0.smithi018.stderr:++++ data_pool= 2023-09-23T20:50:43.177 INFO:tasks.workunit.client.0.smithi018.stderr:++++ '[' -z ']' 2023-09-23T20:50:43.177 INFO:tasks.workunit.client.0.smithi018.stderr:++++ data_pool=rbd 2023-09-23T20:50:43.177 INFO:tasks.workunit.client.0.smithi018.stderr:++++ echo rbd 2023-09-23T20:50:43.177 INFO:tasks.workunit.client.0.smithi018.stderr:+++ rados ls -p rbd 2023-09-23T20:50:43.210 INFO:tasks.workunit.client.0.smithi018.stderr:++ objects=0 2023-09-23T20:50:43.210 INFO:tasks.workunit.client.0.smithi018.stderr:++ echo 0 2023-09-23T20:50:43.210 INFO:tasks.workunit.client.0.smithi018.stderr:+ '[' 0 -eq 1 -o 0 = 0 ']' 2023-09-23T20:50:43.210 INFO:tasks.workunit.client.0.smithi018.stderr:+ rm /tmp/rbd_import_export_466651/sparse 2023-09-23T20:50:43.211 INFO:tasks.workunit.client.0.smithi018.stderr:rm: cannot remove '/tmp/rbd_import_export_466651/sparse': No such file or directory 2023-09-23T20:50:43.211 INFO:tasks.workunit.client.0.smithi018.stderr:+ true 2023-09-23T20:50:43.211 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd export sparse /tmp/rbd_import_export_466651/sparse 2023-09-23T20:50:43.242 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:50:43.247 INFO:tasks.workunit.client.0.smithi018.stderr: Exporting image: 100% complete...done. 2023-09-23T20:50:43.251 INFO:tasks.workunit.client.0.smithi018.stderr:++ stat /tmp/rbd_import_export_466651/sparse --format=%b 2023-09-23T20:50:43.251 INFO:tasks.workunit.client.0.smithi018.stderr:+ '[' 0 = 0 ']' 2023-09-23T20:50:43.252 INFO:tasks.workunit.client.0.smithi018.stderr:+ rbd rm sparse 2023-09-23T20:50:43.272 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:43.343 INFO:tasks.workunit.client.0.smithi018.stderr: Removing image: 100% complete...done. 2023-09-23T20:50:43.346 INFO:tasks.workunit.client.0.smithi018.stderr:+ rm /tmp/rbd_import_export_466651/sparse /tmp/rbd_import_export_466651/sparse1 /tmp/rbd_import_export_466651/sparse2 /tmp/rbd_import_export_466651/sparse3 2023-09-23T20:50:43.349 INFO:tasks.workunit.client.0.smithi018.stderr:rm: cannot remove '/tmp/rbd_import_export_466651/sparse3': No such file or directory 2023-09-23T20:50:43.349 INFO:tasks.workunit.client.0.smithi018.stderr:+ true 2023-09-23T20:50:43.349 INFO:tasks.workunit.client.0.smithi018.stdout:OK 2023-09-23T20:50:43.349 INFO:tasks.workunit.client.0.smithi018.stderr:+ echo OK 2023-09-23T20:50:43.350 INFO:tasks.workunit.client.0.smithi018.stderr:+ rm -rf /tmp/rbd_import_export_466651 2023-09-23T20:50:43.350 INFO:teuthology.orchestra.run:Running command with timeout 3600 2023-09-23T20:50:43.351 DEBUG:teuthology.orchestra.run.smithi018:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2023-09-23T20:50:43.359 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:50:43.362 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:43.362 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:50:43.401 INFO:tasks.workunit:Stopping ['rbd/import_export.sh'] on client.0... 2023-09-23T20:50:43.401 DEBUG:teuthology.orchestra.run.smithi018:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2023-09-23T20:50:43.463 DEBUG:teuthology.orchestra.run.smithi149:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T20:50:43.489 INFO:teuthology.orchestra.run.smithi149.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:43.522 INFO:teuthology.orchestra.run.smithi149.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T20:50:43.525 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:43.526 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi149 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T20:50:43.626 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T20:50:43.656 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:43.818 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T20:50:43.821 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:43.821 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T20:50:43.919 DEBUG:teuthology.parallel:result is None 2023-09-23T20:50:43.919 DEBUG:teuthology.orchestra.run.smithi018:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0 2023-09-23T20:50:43.922 DEBUG:teuthology.orchestra.run.smithi018:> sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T20:50:43.950 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.0/client.0 2023-09-23T20:50:43.950 DEBUG:teuthology.orchestra.run.smithi018:> rmdir -- /home/ubuntu/cephtest/mnt.0 2023-09-23T20:50:44.002 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:50:44.015 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:43 smithi018 ceph-mon[184144]: pgmap v1618: 103 pgs: 103 active+clean; 4.8 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 128 KiB/s rd, 13 KiB/s wr, 172 op/s 2023-09-23T20:50:44.015 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:43 smithi018 ceph-mon[179647]: pgmap v1618: 103 pgs: 103 active+clean; 4.8 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 128 KiB/s rd, 13 KiB/s wr, 172 op/s 2023-09-23T20:50:44.016 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.0/client.0 2023-09-23T20:50:44.016 INFO:teuthology.task.sequential:In sequential, running task print... 2023-09-23T20:50:44.017 INFO:teuthology.task.print:**** done rbd/import_export.sh 5-workload 2023-09-23T20:50:44.017 INFO:teuthology.task.sequential:In sequential, running task workunit... 2023-09-23T20:50:44.017 INFO:tasks.workunit:Pulling workunits from ref quincy 2023-09-23T20:50:44.017 INFO:tasks.workunit:Making a separate scratch dir for every client... 2023-09-23T20:50:44.018 DEBUG:teuthology.orchestra.run.smithi018:> stat -- /home/ubuntu/cephtest/mnt.0 2023-09-23T20:50:44.021 INFO:teuthology.orchestra.run.smithi018.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T20:50:44.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T20:50:44.041 INFO:teuthology.orchestra.run.smithi018.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2023-09-23T20:50:44.042 DEBUG:teuthology.orchestra.run.smithi018:> mkdir -- /home/ubuntu/cephtest/mnt.0 2023-09-23T20:50:44.099 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2023-09-23T20:50:44.099 DEBUG:teuthology.orchestra.run.smithi018:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2023-09-23T20:50:44.157 INFO:tasks.thrashosds:joining thrashosds 2023-09-23T20:50:44.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:43 smithi149 ceph-mon[170394]: pgmap v1618: 103 pgs: 103 active+clean; 4.8 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 128 KiB/s rd, 13 KiB/s wr, 172 op/s 2023-09-23T20:50:44.336 INFO:teuthology.orchestra.run.smithi018.stderr:Error: no container with name or ID "ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T20:50:44.339 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T20:50:44.339 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi018 with status 125: 'sudo docker kill -s 1 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T20:50:44.758 INFO:teuthology.orchestra.run.smithi018.stderr:reweighted osd.3 to 1 (10000) 2023-09-23T20:50:44.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:44 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:50:44.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:44 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2937002548' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T20:50:44.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:44 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T20:50:44.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:44 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:50:44.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:44 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2937002548' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T20:50:44.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:44 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T20:50:45.124 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2023-09-23T20:50:45.124 INFO:teuthology.orchestra.run.smithi018.stdout: "ops": [], 2023-09-23T20:50:45.124 INFO:teuthology.orchestra.run.smithi018.stdout: "num_ops": 0 2023-09-23T20:50:45.125 INFO:teuthology.orchestra.run.smithi018.stdout:} 2023-09-23T20:50:45.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:44 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:50:45.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:44 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2937002548' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T20:50:45.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:44 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T20:50:45.391 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:50:44 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:50:44] "GET /metrics HTTP/1.1" 200 34159 "" "Prometheus/2.43.0" 2023-09-23T20:50:45.620 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd reweight 6 1 2023-09-23T20:50:45.635 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T20:50:45.810 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:45 smithi018 ceph-mon[184144]: pgmap v1619: 103 pgs: 103 active+clean; 8.6 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 1.1 MiB/s rd, 1.8 MiB/s wr, 358 op/s 2023-09-23T20:50:45.810 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:45 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T20:50:45.810 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:45 smithi018 ceph-mon[184144]: osdmap e665: 8 total, 8 up, 7 in 2023-09-23T20:50:45.811 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:45 smithi018 ceph-mon[179647]: pgmap v1619: 103 pgs: 103 active+clean; 8.6 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 1.1 MiB/s rd, 1.8 MiB/s wr, 358 op/s 2023-09-23T20:50:45.811 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:45 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T20:50:45.811 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:45 smithi018 ceph-mon[179647]: osdmap e665: 8 total, 8 up, 7 in 2023-09-23T20:50:46.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:45 smithi149 ceph-mon[170394]: pgmap v1619: 103 pgs: 103 active+clean; 8.6 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 1.1 MiB/s rd, 1.8 MiB/s wr, 358 op/s 2023-09-23T20:50:46.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:45 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T20:50:46.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:45 smithi149 ceph-mon[170394]: osdmap e665: 8 total, 8 up, 7 in 2023-09-23T20:50:46.256 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:50:46.366 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:50:46.722 INFO:teuthology.orchestra.run.smithi018.stderr:nodeep-scrub is set 2023-09-23T20:50:47.060 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:46 smithi018 ceph-mon[179647]: failed to encode map e665 with expected crc 2023-09-23T20:50:47.060 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:46 smithi018 ceph-mon[179647]: failed to encode map e665 with expected crc 2023-09-23T20:50:47.060 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:46 smithi018 ceph-mon[179647]: failed to encode map e665 with expected crc 2023-09-23T20:50:47.060 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:46 smithi018 ceph-mon[179647]: failed to encode map e665 with expected crc 2023-09-23T20:50:47.060 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:46 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2907982386' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:50:47.060 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:46 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:50:47.060 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:46 smithi018 ceph-mon[179647]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:50:47.061 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:46 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:50:47.061 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:46 smithi018 ceph-mon[179647]: osdmap e666: 8 total, 8 up, 7 in 2023-09-23T20:50:47.061 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:46 smithi018 ceph-mon[184144]: failed to encode map e665 with expected crc 2023-09-23T20:50:47.061 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:46 smithi018 ceph-mon[184144]: failed to encode map e665 with expected crc 2023-09-23T20:50:47.061 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:46 smithi018 ceph-mon[184144]: failed to encode map e665 with expected crc 2023-09-23T20:50:47.061 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:46 smithi018 ceph-mon[184144]: failed to encode map e665 with expected crc 2023-09-23T20:50:47.061 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:46 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2907982386' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:50:47.061 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:46 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:50:47.061 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:46 smithi018 ceph-mon[184144]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:50:47.062 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:46 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:50:47.062 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:46 smithi018 ceph-mon[184144]: osdmap e666: 8 total, 8 up, 7 in 2023-09-23T20:50:47.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:46 smithi149 ceph-mon[170394]: failed to encode map e665 with expected crc 2023-09-23T20:50:47.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:46 smithi149 ceph-mon[170394]: failed to encode map e665 with expected crc 2023-09-23T20:50:47.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:46 smithi149 ceph-mon[170394]: failed to encode map e665 with expected crc 2023-09-23T20:50:47.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:46 smithi149 ceph-mon[170394]: failed to encode map e665 with expected crc 2023-09-23T20:50:47.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:46 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2907982386' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:50:47.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:46 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:50:47.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:46 smithi149 ceph-mon[170394]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:50:47.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:46 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T20:50:47.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:46 smithi149 ceph-mon[170394]: osdmap e666: 8 total, 8 up, 7 in 2023-09-23T20:50:48.114 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:47 smithi018 ceph-mon[184144]: pgmap v1621: 103 pgs: 103 active+clean; 8.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 1.3 MiB/s rd, 1.9 MiB/s wr, 441 op/s 2023-09-23T20:50:48.115 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:47 smithi018 ceph-mon[184144]: failed to encode map e666 with expected crc 2023-09-23T20:50:48.115 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:47 smithi018 ceph-mon[184144]: failed to encode map e666 with expected crc 2023-09-23T20:50:48.115 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:47 smithi018 ceph-mon[179647]: pgmap v1621: 103 pgs: 103 active+clean; 8.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 1.3 MiB/s rd, 1.9 MiB/s wr, 441 op/s 2023-09-23T20:50:48.115 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:47 smithi018 ceph-mon[179647]: failed to encode map e666 with expected crc 2023-09-23T20:50:48.115 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:47 smithi018 ceph-mon[179647]: failed to encode map e666 with expected crc 2023-09-23T20:50:48.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:47 smithi149 ceph-mon[170394]: pgmap v1621: 103 pgs: 103 active+clean; 8.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 1.3 MiB/s rd, 1.9 MiB/s wr, 441 op/s 2023-09-23T20:50:48.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:47 smithi149 ceph-mon[170394]: failed to encode map e666 with expected crc 2023-09-23T20:50:48.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:47 smithi149 ceph-mon[170394]: failed to encode map e666 with expected crc 2023-09-23T20:50:48.313 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2023-09-23T20:50:48.313 INFO:teuthology.orchestra.run.smithi018.stdout: "ops": [], 2023-09-23T20:50:48.313 INFO:teuthology.orchestra.run.smithi018.stdout: "complaint_time": 30, 2023-09-23T20:50:48.313 INFO:teuthology.orchestra.run.smithi018.stdout: "num_blocked_ops": 0 2023-09-23T20:50:48.313 INFO:teuthology.orchestra.run.smithi018.stdout:} 2023-09-23T20:50:48.685 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T20:50:48.769 INFO:teuthology.orchestra.run.smithi018.stderr:reweighted osd.6 to 1 (10000) 2023-09-23T20:50:48.793 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:48 smithi018 ceph-mon[184144]: failed to encode map e666 with expected crc 2023-09-23T20:50:48.793 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:48 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/4282552053' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T20:50:48.793 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:48 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T20:50:48.793 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:48 smithi018 ceph-mon[179647]: failed to encode map e666 with expected crc 2023-09-23T20:50:48.793 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:48 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/4282552053' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T20:50:48.794 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:48 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T20:50:49.171 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd reweight 4 1 2023-09-23T20:50:49.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:48 smithi149 ceph-mon[170394]: failed to encode map e666 with expected crc 2023-09-23T20:50:49.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:48 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/4282552053' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T20:50:49.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:48 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T20:50:49.421 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:50:49.461 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T20:50:50.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:49 smithi018 ceph-mon[184144]: pgmap v1623: 103 pgs: 103 active+clean; 8.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 1.2 MiB/s rd, 1.7 MiB/s wr, 391 op/s 2023-09-23T20:50:50.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:49 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T20:50:50.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:49 smithi018 ceph-mon[184144]: osdmap e667: 8 total, 8 up, 7 in 2023-09-23T20:50:50.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:49 smithi018 ceph-mon[179647]: pgmap v1623: 103 pgs: 103 active+clean; 8.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 1.2 MiB/s rd, 1.7 MiB/s wr, 391 op/s 2023-09-23T20:50:50.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:49 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T20:50:50.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:49 smithi018 ceph-mon[179647]: osdmap e667: 8 total, 8 up, 7 in 2023-09-23T20:50:50.191 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:50:50.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:49 smithi149 ceph-mon[170394]: pgmap v1623: 103 pgs: 103 active+clean; 8.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 1.2 MiB/s rd, 1.7 MiB/s wr, 391 op/s 2023-09-23T20:50:50.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:49 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T20:50:50.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:49 smithi149 ceph-mon[170394]: osdmap e667: 8 total, 8 up, 7 in 2023-09-23T20:50:50.431 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:50:50.768 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:50 smithi018 ceph-mon[179647]: failed to encode map e667 with expected crc 2023-09-23T20:50:50.768 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:50 smithi018 ceph-mon[179647]: failed to encode map e667 with expected crc 2023-09-23T20:50:50.768 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:50 smithi018 ceph-mon[179647]: failed to encode map e667 with expected crc 2023-09-23T20:50:50.768 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:50 smithi018 ceph-mon[179647]: failed to encode map e667 with expected crc 2023-09-23T20:50:50.768 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:50 smithi018 ceph-mon[179647]: failed to encode map e667 with expected crc 2023-09-23T20:50:51.079 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:50 smithi018 ceph-mon[184144]: failed to encode map e667 with expected crc 2023-09-23T20:50:51.079 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:50 smithi018 ceph-mon[184144]: failed to encode map e667 with expected crc 2023-09-23T20:50:51.079 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:50 smithi018 ceph-mon[184144]: failed to encode map e667 with expected crc 2023-09-23T20:50:51.080 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:50 smithi018 ceph-mon[184144]: failed to encode map e667 with expected crc 2023-09-23T20:50:51.080 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:50 smithi018 ceph-mon[184144]: failed to encode map e667 with expected crc 2023-09-23T20:50:51.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:50 smithi149 ceph-mon[170394]: failed to encode map e667 with expected crc 2023-09-23T20:50:51.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:50 smithi149 ceph-mon[170394]: failed to encode map e667 with expected crc 2023-09-23T20:50:51.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:50 smithi149 ceph-mon[170394]: failed to encode map e667 with expected crc 2023-09-23T20:50:51.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:50 smithi149 ceph-mon[170394]: failed to encode map e667 with expected crc 2023-09-23T20:50:51.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:50 smithi149 ceph-mon[170394]: failed to encode map e667 with expected crc 2023-09-23T20:50:51.368 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2023-09-23T20:50:51.369 INFO:teuthology.orchestra.run.smithi018.stdout: "size": 20, 2023-09-23T20:50:51.369 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 600, 2023-09-23T20:50:51.369 INFO:teuthology.orchestra.run.smithi018.stdout: "ops": [ 2023-09-23T20:50:51.369 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.369 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.36450.0:141027 9.6 9:6dcbc3cc:::benchmark_data_smithi018_381516_object27193:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e481)", 2023-09-23T20:50:51.369 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:43:15.747787+0000", 2023-09-23T20:50:51.369 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 455.61765668999999, 2023-09-23T20:50:51.369 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.040264882, 2023-09-23T20:50:51.370 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:50:51.370 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:50:51.370 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:50:51.370 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.36450", 2023-09-23T20:50:51.370 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:50:51.370 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 141027 2023-09-23T20:50:51.370 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.370 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:50:51.370 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.370 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:50:51.371 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.747787+0000", 2023-09-23T20:50:51.371 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:50:51.371 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.371 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.371 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:50:51.371 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.747787+0000", 2023-09-23T20:50:51.371 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:50:51.371 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.371 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.371 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:50:51.371 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.747785+0000", 2023-09-23T20:50:51.371 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:50:51.372 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.372 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.372 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:50:51.372 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.747801+0000", 2023-09-23T20:50:51.372 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.5854000000000001e-05 2023-09-23T20:50:51.372 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.372 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.372 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:50:51.372 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.747818+0000", 2023-09-23T20:50:51.372 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.7499e-05 2023-09-23T20:50:51.373 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.373 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.373 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:50:51.373 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.747829+0000", 2023-09-23T20:50:51.373 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0508e-05 2023-09-23T20:50:51.373 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.373 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.373 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:50:51.373 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.747912+0000", 2023-09-23T20:50:51.373 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.3052000000000004e-05 2023-09-23T20:50:51.373 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.374 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.374 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:50:51.374 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.747926+0000", 2023-09-23T20:50:51.374 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3577000000000001e-05 2023-09-23T20:50:51.374 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.374 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.374 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:50:51.374 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:16.785351+0000", 2023-09-23T20:50:51.374 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.037425635 2023-09-23T20:50:51.374 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.374 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.375 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:50:51.375 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:16.788052+0000", 2023-09-23T20:50:51.375 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0027005010000000001 2023-09-23T20:50:51.375 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:50:51.375 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:50:51.375 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:50:51.375 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.375 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.375 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:159028 10.3 10:c5cfe988:::benchmark_data_smithi018_422431_object54532:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e578)", 2023-09-23T20:50:51.375 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:48:23.915042+0000", 2023-09-23T20:50:51.375 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 147.450401935, 2023-09-23T20:50:51.376 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.277777194, 2023-09-23T20:50:51.376 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:50:51.376 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:50:51.376 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:50:51.376 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:50:51.376 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:50:51.376 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 159028 2023-09-23T20:50:51.376 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.376 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:50:51.376 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.376 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:50:51.377 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:23.915042+0000", 2023-09-23T20:50:51.377 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:50:51.377 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.377 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.377 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:50:51.377 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:23.915042+0000", 2023-09-23T20:50:51.377 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:50:51.377 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.377 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.377 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:50:51.377 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:23.915040+0000", 2023-09-23T20:50:51.377 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:50:51.378 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.378 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.378 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:50:51.378 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:23.915046+0000", 2023-09-23T20:50:51.378 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.6790000000000002e-06 2023-09-23T20:50:51.378 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.378 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.378 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:50:51.378 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:23.915059+0000", 2023-09-23T20:50:51.378 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3634e-05 2023-09-23T20:50:51.378 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.379 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.379 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:50:51.379 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:23.915071+0000", 2023-09-23T20:50:51.379 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1449e-05 2023-09-23T20:50:51.379 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.379 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.379 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:50:51.379 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:23.917921+0000", 2023-09-23T20:50:51.379 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0028506289999999999 2023-09-23T20:50:51.379 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.379 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.380 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:50:51.380 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:23.917946+0000", 2023-09-23T20:50:51.380 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.4960000000000002e-05 2023-09-23T20:50:51.380 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.380 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.380 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:50:51.380 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:25.181323+0000", 2023-09-23T20:50:51.380 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2633762989999999 2023-09-23T20:50:51.380 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.380 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.380 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:50:51.380 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:25.181464+0000", 2023-09-23T20:50:51.381 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00014105300000000001 2023-09-23T20:50:51.381 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.381 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.381 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 3,4,5", 2023-09-23T20:50:51.381 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:25.181599+0000", 2023-09-23T20:50:51.381 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000135594 2023-09-23T20:50:51.381 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.381 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.381 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:51.381 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:25.187281+0000", 2023-09-23T20:50:51.381 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.005681859 2023-09-23T20:50:51.381 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.382 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.382 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:51.382 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:25.187329+0000", 2023-09-23T20:50:51.382 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.7555999999999997e-05 2023-09-23T20:50:51.382 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.382 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.382 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:51.382 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:25.187533+0000", 2023-09-23T20:50:51.382 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000203988 2023-09-23T20:50:51.382 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.382 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.383 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:50:51.383 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:25.192635+0000", 2023-09-23T20:50:51.383 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0051021870000000002 2023-09-23T20:50:51.383 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.383 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.383 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:50:51.383 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:25.192692+0000", 2023-09-23T20:50:51.383 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.6984999999999997e-05 2023-09-23T20:50:51.383 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.383 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.383 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:50:51.383 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:25.192819+0000", 2023-09-23T20:50:51.384 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00012683799999999999 2023-09-23T20:50:51.384 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:50:51.384 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:50:51.384 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:50:51.384 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.384 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.384 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:159052 10.3 10:c6ec30e7:::benchmark_data_smithi018_422431_object54556:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e578)", 2023-09-23T20:50:51.384 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:48:23.915080+0000", 2023-09-23T20:50:51.384 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 147.45036323299999, 2023-09-23T20:50:51.384 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.299160028, 2023-09-23T20:50:51.384 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:50:51.384 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:50:51.385 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:50:51.385 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:50:51.385 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:50:51.385 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 159052 2023-09-23T20:50:51.385 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.385 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:50:51.385 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.385 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:50:51.385 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:23.915080+0000", 2023-09-23T20:50:51.385 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:50:51.385 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.385 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.385 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:50:51.386 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:23.915080+0000", 2023-09-23T20:50:51.386 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:50:51.386 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.386 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.386 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:50:51.386 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:23.915080+0000", 2023-09-23T20:50:51.386 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:50:51.386 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.386 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.386 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:50:51.386 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:23.915083+0000", 2023-09-23T20:50:51.387 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.5410000000000001e-06 2023-09-23T20:50:51.387 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.387 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.387 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:50:51.387 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:23.915090+0000", 2023-09-23T20:50:51.387 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.6699999999999997e-06 2023-09-23T20:50:51.387 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.387 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.387 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:50:51.387 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:23.915096+0000", 2023-09-23T20:50:51.387 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.1090000000000003e-06 2023-09-23T20:50:51.388 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.388 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.388 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:50:51.388 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:23.923741+0000", 2023-09-23T20:50:51.388 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0086448749999999998 2023-09-23T20:50:51.388 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.388 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.388 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:50:51.388 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:23.923754+0000", 2023-09-23T20:50:51.388 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3404000000000001e-05 2023-09-23T20:50:51.388 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.389 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.389 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:50:51.389 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:25.187715+0000", 2023-09-23T20:50:51.389 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2639611239999999 2023-09-23T20:50:51.389 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.389 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.389 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:50:51.389 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:25.187876+0000", 2023-09-23T20:50:51.389 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00016075799999999999 2023-09-23T20:50:51.389 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.389 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.389 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 3,4,5", 2023-09-23T20:50:51.390 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:25.188011+0000", 2023-09-23T20:50:51.390 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00013465499999999999 2023-09-23T20:50:51.390 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.390 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.390 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:50:51.390 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:25.193590+0000", 2023-09-23T20:50:51.390 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0055792610000000003 2023-09-23T20:50:51.390 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.390 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.390 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:51.390 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:25.197165+0000", 2023-09-23T20:50:51.391 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0035749229999999998 2023-09-23T20:50:51.391 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.391 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.391 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:51.391 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:25.203287+0000", 2023-09-23T20:50:51.391 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0061218979999999997 2023-09-23T20:50:51.391 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.391 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.391 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:51.391 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:25.214055+0000", 2023-09-23T20:50:51.391 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.010768313999999999 2023-09-23T20:50:51.392 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.392 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.392 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:50:51.392 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:25.214117+0000", 2023-09-23T20:50:51.392 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.1773999999999995e-05 2023-09-23T20:50:51.392 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.392 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.392 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:50:51.392 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:25.214240+0000", 2023-09-23T20:50:51.392 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00012325899999999999 2023-09-23T20:50:51.392 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:50:51.392 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:50:51.393 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:50:51.393 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.393 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.393 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:174208 10.3 10:c770d702:::benchmark_data_smithi018_422431_object69712:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e590)", 2023-09-23T20:50:51.393 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:48:47.146484+0000", 2023-09-23T20:50:51.393 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 124.218959099, 2023-09-23T20:50:51.393 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0466785009999999, 2023-09-23T20:50:51.393 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:50:51.393 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:50:51.393 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:50:51.393 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:50:51.393 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:50:51.394 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 174208 2023-09-23T20:50:51.394 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.394 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:50:51.394 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.394 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:50:51.394 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:47.146484+0000", 2023-09-23T20:50:51.394 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:50:51.394 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.394 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.394 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:50:51.394 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:47.146484+0000", 2023-09-23T20:50:51.395 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:50:51.395 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.395 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.395 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:50:51.395 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:47.146483+0000", 2023-09-23T20:50:51.395 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:50:51.395 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.395 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.395 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:50:51.395 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:47.146489+0000", 2023-09-23T20:50:51.395 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.5289999999999997e-06 2023-09-23T20:50:51.396 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.396 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.396 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:50:51.396 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:47.146501+0000", 2023-09-23T20:50:51.396 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.216e-05 2023-09-23T20:50:51.396 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.396 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.396 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:50:51.396 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:47.146510+0000", 2023-09-23T20:50:51.396 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.4840000000000004e-06 2023-09-23T20:50:51.396 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.397 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.397 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:50:51.397 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:47.146617+0000", 2023-09-23T20:50:51.397 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000106826 2023-09-23T20:50:51.397 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.397 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.397 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:50:51.397 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:47.146632+0000", 2023-09-23T20:50:51.397 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.5160999999999999e-05 2023-09-23T20:50:51.397 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.397 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.397 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:50:51.397 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:48.184489+0000", 2023-09-23T20:50:51.398 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0378566440000001 2023-09-23T20:50:51.398 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.398 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.398 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:50:51.398 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:48.184658+0000", 2023-09-23T20:50:51.398 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00016920699999999999 2023-09-23T20:50:51.398 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.398 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.398 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 3,5", 2023-09-23T20:50:51.398 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:48.184826+0000", 2023-09-23T20:50:51.398 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00016825400000000001 2023-09-23T20:50:51.398 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.399 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.399 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:50:51.399 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:48.187756+0000", 2023-09-23T20:50:51.399 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0029296029999999998 2023-09-23T20:50:51.399 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.399 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.399 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:51.399 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:48.187810+0000", 2023-09-23T20:50:51.399 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.4483e-05 2023-09-23T20:50:51.399 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.399 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.399 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:51.399 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:48.193082+0000", 2023-09-23T20:50:51.400 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0052715820000000004 2023-09-23T20:50:51.400 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.400 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.400 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:50:51.400 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:48.193125+0000", 2023-09-23T20:50:51.400 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.3578000000000002e-05 2023-09-23T20:50:51.400 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.400 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.400 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:50:51.400 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:48:48.193163+0000", 2023-09-23T20:50:51.400 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.7629000000000002e-05 2023-09-23T20:50:51.400 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:50:51.400 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:50:51.401 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:50:51.401 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.401 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.401 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:203982 10.19 10:9d6c8636:::benchmark_data_smithi018_422431_object99486:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e632)", 2023-09-23T20:50:51.401 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:49:40.065944+0000", 2023-09-23T20:50:51.401 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 71.299499052000002, 2023-09-23T20:50:51.401 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.225633725, 2023-09-23T20:50:51.401 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:50:51.401 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:50:51.401 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:50:51.401 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:50:51.401 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:50:51.402 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 203982 2023-09-23T20:50:51.402 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.402 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:50:51.402 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.402 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:50:51.402 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.065944+0000", 2023-09-23T20:50:51.402 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:50:51.402 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.402 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.402 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:50:51.403 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.065944+0000", 2023-09-23T20:50:51.403 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:50:51.403 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.403 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.403 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:50:51.403 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.065943+0000", 2023-09-23T20:50:51.403 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:50:51.403 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.403 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.403 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:50:51.403 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.065947+0000", 2023-09-23T20:50:51.403 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.3059999999999999e-06 2023-09-23T20:50:51.404 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.404 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.404 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:50:51.404 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.065957+0000", 2023-09-23T20:50:51.404 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.5619999999999993e-06 2023-09-23T20:50:51.404 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.404 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.404 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:50:51.404 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.065967+0000", 2023-09-23T20:50:51.404 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.9599999999999995e-06 2023-09-23T20:50:51.404 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.405 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.405 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:50:51.405 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.066202+0000", 2023-09-23T20:50:51.405 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00023541499999999999 2023-09-23T20:50:51.405 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.405 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.405 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:50:51.405 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.066212+0000", 2023-09-23T20:50:51.405 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.7149999999999993e-06 2023-09-23T20:50:51.405 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.405 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.405 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:50:51.406 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.286259+0000", 2023-09-23T20:50:51.406 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2200467500000001 2023-09-23T20:50:51.406 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.406 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.406 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:50:51.406 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.286371+0000", 2023-09-23T20:50:51.406 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000112673 2023-09-23T20:50:51.406 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.406 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.406 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,3,4,5,7", 2023-09-23T20:50:51.406 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.286452+0000", 2023-09-23T20:50:51.407 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.0277000000000004e-05 2023-09-23T20:50:51.407 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.407 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.407 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:50:51.407 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.291047+0000", 2023-09-23T20:50:51.407 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0045951860000000002 2023-09-23T20:50:51.407 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.407 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.407 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:51.407 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.291101+0000", 2023-09-23T20:50:51.408 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.4156000000000001e-05 2023-09-23T20:50:51.408 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.408 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.408 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:51.408 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.291328+0000", 2023-09-23T20:50:51.408 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00022672400000000001 2023-09-23T20:50:51.408 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.408 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.408 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:51.408 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.291377+0000", 2023-09-23T20:50:51.408 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.9146e-05 2023-09-23T20:50:51.409 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.409 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.409 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:51.409 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.291411+0000", 2023-09-23T20:50:51.409 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.4171000000000002e-05 2023-09-23T20:50:51.409 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.409 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.409 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:51.409 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.291445+0000", 2023-09-23T20:50:51.409 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.3670000000000001e-05 2023-09-23T20:50:51.409 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.410 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.410 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:50:51.410 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.291479+0000", 2023-09-23T20:50:51.410 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.3982000000000001e-05 2023-09-23T20:50:51.410 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.410 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.410 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:50:51.410 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.291578+0000", 2023-09-23T20:50:51.410 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.9408999999999996e-05 2023-09-23T20:50:51.410 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:50:51.411 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:50:51.411 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:50:51.411 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.411 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.411 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:204016 10.19 10:9ac002b0:::benchmark_data_smithi018_422431_object99520:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e632)", 2023-09-23T20:50:51.411 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:49:40.065973+0000", 2023-09-23T20:50:51.411 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 71.299470490999994, 2023-09-23T20:50:51.411 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.228063369, 2023-09-23T20:50:51.411 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:50:51.411 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:50:51.412 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:50:51.412 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:50:51.412 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:50:51.412 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 204016 2023-09-23T20:50:51.412 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.412 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:50:51.412 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.412 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:50:51.412 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.065973+0000", 2023-09-23T20:50:51.412 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:50:51.412 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.413 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.413 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:50:51.413 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.065973+0000", 2023-09-23T20:50:51.413 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:50:51.413 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.413 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.413 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:50:51.413 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.065973+0000", 2023-09-23T20:50:51.413 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:50:51.413 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.413 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.413 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:50:51.414 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.065975+0000", 2023-09-23T20:50:51.414 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.7700000000000002e-06 2023-09-23T20:50:51.414 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.414 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.414 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:50:51.414 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.065982+0000", 2023-09-23T20:50:51.414 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.409e-06 2023-09-23T20:50:51.414 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.414 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.414 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:50:51.414 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.065987+0000", 2023-09-23T20:50:51.415 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.9219999999999996e-06 2023-09-23T20:50:51.415 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.415 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.415 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:50:51.415 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.066229+0000", 2023-09-23T20:50:51.415 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00024253600000000001 2023-09-23T20:50:51.415 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.415 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.415 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:50:51.415 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.066233+0000", 2023-09-23T20:50:51.415 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.8800000000000001e-06 2023-09-23T20:50:51.415 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.416 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.416 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:50:51.416 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.288284+0000", 2023-09-23T20:50:51.416 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2220503899999999 2023-09-23T20:50:51.416 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.416 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.416 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:50:51.416 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.288354+0000", 2023-09-23T20:50:51.416 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.0679999999999994e-05 2023-09-23T20:50:51.416 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.416 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.417 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,3,4,5,7", 2023-09-23T20:50:51.417 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.288463+0000", 2023-09-23T20:50:51.417 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000109046 2023-09-23T20:50:51.417 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.417 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.417 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:50:51.417 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.291218+0000", 2023-09-23T20:50:51.417 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0027547460000000002 2023-09-23T20:50:51.417 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.417 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.417 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:51.418 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.291271+0000", 2023-09-23T20:50:51.418 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.2685999999999997e-05 2023-09-23T20:50:51.418 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.418 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.418 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:51.418 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.291717+0000", 2023-09-23T20:50:51.418 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000446317 2023-09-23T20:50:51.418 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.418 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.418 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:51.418 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.291775+0000", 2023-09-23T20:50:51.418 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.7639999999999997e-05 2023-09-23T20:50:51.419 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.419 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.419 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:51.419 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.293799+0000", 2023-09-23T20:50:51.419 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.002023871 2023-09-23T20:50:51.419 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.419 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.419 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:51.419 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.293925+0000", 2023-09-23T20:50:51.419 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000126651 2023-09-23T20:50:51.420 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.420 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.420 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:50:51.420 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.293962+0000", 2023-09-23T20:50:51.420 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.6368000000000001e-05 2023-09-23T20:50:51.420 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.420 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.420 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:50:51.420 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.294036+0000", 2023-09-23T20:50:51.420 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.4778999999999997e-05 2023-09-23T20:50:51.420 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:50:51.420 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:50:51.421 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:50:51.421 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.421 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.421 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:204063 10.19 10:991d49a7:::benchmark_data_smithi018_422431_object99567:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e632)", 2023-09-23T20:50:51.421 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:49:40.065990+0000", 2023-09-23T20:50:51.421 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 71.299453119999995, 2023-09-23T20:50:51.421 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2318660219999999, 2023-09-23T20:50:51.421 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:50:51.421 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:50:51.421 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:50:51.421 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:50:51.421 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:50:51.421 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 204063 2023-09-23T20:50:51.422 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.423 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:50:51.423 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.423 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:50:51.423 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.065990+0000", 2023-09-23T20:50:51.423 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:50:51.423 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.423 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.423 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:50:51.423 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.065990+0000", 2023-09-23T20:50:51.423 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:50:51.423 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.424 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.424 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:50:51.424 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.065990+0000", 2023-09-23T20:50:51.424 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:50:51.424 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.424 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.424 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:50:51.424 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.065992+0000", 2023-09-23T20:50:51.424 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.0200000000000001e-06 2023-09-23T20:50:51.424 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.425 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.425 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:50:51.425 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.066000+0000", 2023-09-23T20:50:51.425 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.5900000000000002e-06 2023-09-23T20:50:51.425 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.425 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.425 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:50:51.425 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.066004+0000", 2023-09-23T20:50:51.425 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.3599999999999998e-06 2023-09-23T20:50:51.425 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.425 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.426 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:50:51.426 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.066269+0000", 2023-09-23T20:50:51.426 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00026538499999999998 2023-09-23T20:50:51.426 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.426 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.426 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:50:51.426 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.066277+0000", 2023-09-23T20:50:51.426 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.3320000000000003e-06 2023-09-23T20:50:51.426 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.426 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.426 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:50:51.427 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.289649+0000", 2023-09-23T20:50:51.427 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2233721829999999 2023-09-23T20:50:51.427 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.427 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.427 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:50:51.427 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.289766+0000", 2023-09-23T20:50:51.427 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000116866 2023-09-23T20:50:51.427 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.427 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.427 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,3,4,5,7", 2023-09-23T20:50:51.427 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.289861+0000", 2023-09-23T20:50:51.428 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.5614999999999998e-05 2023-09-23T20:50:51.428 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.428 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.428 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:50:51.428 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.291339+0000", 2023-09-23T20:50:51.428 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.001477716 2023-09-23T20:50:51.428 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.428 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.428 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:51.428 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.291614+0000", 2023-09-23T20:50:51.428 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00027495299999999998 2023-09-23T20:50:51.429 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.429 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.429 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:51.429 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.291806+0000", 2023-09-23T20:50:51.429 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00019233999999999999 2023-09-23T20:50:51.429 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.429 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.429 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:51.429 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.291867+0000", 2023-09-23T20:50:51.429 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.0115000000000003e-05 2023-09-23T20:50:51.429 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.430 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.430 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:51.430 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.293879+0000", 2023-09-23T20:50:51.430 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0020122690000000001 2023-09-23T20:50:51.430 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.430 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.430 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:51.430 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.297669+0000", 2023-09-23T20:50:51.430 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0037906289999999998 2023-09-23T20:50:51.430 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.430 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.431 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:50:51.431 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.297708+0000", 2023-09-23T20:50:51.431 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.8192000000000003e-05 2023-09-23T20:50:51.431 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.431 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.431 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:50:51.431 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.297856+0000", 2023-09-23T20:50:51.431 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000148756 2023-09-23T20:50:51.431 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:50:51.431 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:50:51.432 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:50:51.432 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.432 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.432 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:204068 10.19 10:9cd617a3:::benchmark_data_smithi018_422431_object99572:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e632)", 2023-09-23T20:50:51.432 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:49:40.066007+0000", 2023-09-23T20:50:51.432 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 71.299435993000003, 2023-09-23T20:50:51.432 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2320337939999999, 2023-09-23T20:50:51.432 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:50:51.432 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:50:51.432 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:50:51.432 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:50:51.433 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:50:51.433 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 204068 2023-09-23T20:50:51.433 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.433 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:50:51.433 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.433 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:50:51.433 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.066007+0000", 2023-09-23T20:50:51.433 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:50:51.433 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.433 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.433 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:50:51.434 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.066007+0000", 2023-09-23T20:50:51.434 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:50:51.434 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.434 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.434 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:50:51.434 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.066007+0000", 2023-09-23T20:50:51.434 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:50:51.434 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.434 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.434 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:50:51.434 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.066009+0000", 2023-09-23T20:50:51.434 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.8980000000000001e-06 2023-09-23T20:50:51.435 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.435 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.435 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:50:51.435 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.066015+0000", 2023-09-23T20:50:51.435 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.3759999999999996e-06 2023-09-23T20:50:51.435 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.435 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.435 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:50:51.435 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.066020+0000", 2023-09-23T20:50:51.435 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.3370000000000001e-06 2023-09-23T20:50:51.435 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.435 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.436 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:50:51.436 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.066307+0000", 2023-09-23T20:50:51.436 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00028756700000000002 2023-09-23T20:50:51.436 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.436 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.436 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:50:51.436 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.066314+0000", 2023-09-23T20:50:51.436 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.8680000000000001e-06 2023-09-23T20:50:51.436 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.437 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.437 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:50:51.437 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.291907+0000", 2023-09-23T20:50:51.437 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2255924089999999 2023-09-23T20:50:51.437 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.437 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.437 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:50:51.437 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.291981+0000", 2023-09-23T20:50:51.437 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.4487999999999996e-05 2023-09-23T20:50:51.437 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.437 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.438 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,3,4,5,7", 2023-09-23T20:50:51.438 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.292052+0000", 2023-09-23T20:50:51.438 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.0728000000000005e-05 2023-09-23T20:50:51.438 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.438 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.438 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:51.438 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.294086+0000", 2023-09-23T20:50:51.438 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0020337229999999999 2023-09-23T20:50:51.438 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.438 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.438 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:51.439 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.294124+0000", 2023-09-23T20:50:51.439 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.8534000000000001e-05 2023-09-23T20:50:51.439 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.439 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.439 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:50:51.439 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.294137+0000", 2023-09-23T20:50:51.439 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.31e-05 2023-09-23T20:50:51.439 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.439 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.439 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:51.439 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.294181+0000", 2023-09-23T20:50:51.440 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.3460000000000001e-05 2023-09-23T20:50:51.440 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.440 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.440 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:51.440 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.294228+0000", 2023-09-23T20:50:51.440 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.6854999999999998e-05 2023-09-23T20:50:51.440 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.440 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.440 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:51.440 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.297903+0000", 2023-09-23T20:50:51.440 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0036754629999999999 2023-09-23T20:50:51.441 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.441 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.441 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:50:51.441 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.297940+0000", 2023-09-23T20:50:51.441 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.7246999999999998e-05 2023-09-23T20:50:51.441 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.441 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.441 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:50:51.441 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.298041+0000", 2023-09-23T20:50:51.441 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00010101 2023-09-23T20:50:51.441 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:50:51.442 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:50:51.442 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:50:51.442 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.442 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.442 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:204119 10.19 10:9f950043:::benchmark_data_smithi018_422431_object99623:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e632)", 2023-09-23T20:50:51.442 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:49:40.066023+0000", 2023-09-23T20:50:51.442 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 71.299420273999999, 2023-09-23T20:50:51.442 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2427722240000001, 2023-09-23T20:50:51.442 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:50:51.442 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:50:51.443 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:50:51.443 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:50:51.443 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:50:51.443 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 204119 2023-09-23T20:50:51.443 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.443 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:50:51.443 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.443 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:50:51.443 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.066023+0000", 2023-09-23T20:50:51.443 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:50:51.444 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.444 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.444 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:50:51.444 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.066023+0000", 2023-09-23T20:50:51.444 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:50:51.444 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.444 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.444 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:50:51.444 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.066023+0000", 2023-09-23T20:50:51.444 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:50:51.445 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.445 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.445 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:50:51.445 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.066025+0000", 2023-09-23T20:50:51.445 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.894e-06 2023-09-23T20:50:51.445 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.445 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.445 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:50:51.445 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.066030+0000", 2023-09-23T20:50:51.445 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.1250000000000001e-06 2023-09-23T20:50:51.445 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.445 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.446 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:50:51.446 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.066034+0000", 2023-09-23T20:50:51.446 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.3959999999999999e-06 2023-09-23T20:50:51.446 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.446 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.446 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:50:51.446 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.066346+0000", 2023-09-23T20:50:51.446 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00031128800000000001 2023-09-23T20:50:51.446 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.446 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.446 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:50:51.447 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.066353+0000", 2023-09-23T20:50:51.447 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.5569999999999998e-06 2023-09-23T20:50:51.447 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.447 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.447 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:50:51.447 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.298078+0000", 2023-09-23T20:50:51.447 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.231724477 2023-09-23T20:50:51.447 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.447 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.447 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:50:51.447 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.298162+0000", 2023-09-23T20:50:51.448 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.4408999999999998e-05 2023-09-23T20:50:51.448 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.448 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.448 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,3,4,5,7", 2023-09-23T20:50:51.448 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.298240+0000", 2023-09-23T20:50:51.448 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.8195999999999995e-05 2023-09-23T20:50:51.448 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.448 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.448 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:51.448 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.301801+0000", 2023-09-23T20:50:51.448 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0035606349999999999 2023-09-23T20:50:51.448 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.449 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.449 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:51.449 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.301832+0000", 2023-09-23T20:50:51.449 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.1214999999999998e-05 2023-09-23T20:50:51.449 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.449 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.449 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:51.449 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.301862+0000", 2023-09-23T20:50:51.449 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.0147000000000001e-05 2023-09-23T20:50:51.449 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.449 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.449 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:50:51.450 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.301871+0000", 2023-09-23T20:50:51.450 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.7800000000000006e-06 2023-09-23T20:50:51.450 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.450 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.450 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:51.450 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.308569+0000", 2023-09-23T20:50:51.450 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0066977319999999996 2023-09-23T20:50:51.450 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.450 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.450 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:51.450 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.308613+0000", 2023-09-23T20:50:51.451 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.4104000000000003e-05 2023-09-23T20:50:51.451 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.451 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.451 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:50:51.451 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.308660+0000", 2023-09-23T20:50:51.451 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.6791e-05 2023-09-23T20:50:51.451 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.451 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.451 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:50:51.451 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.308795+0000", 2023-09-23T20:50:51.451 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00013574799999999999 2023-09-23T20:50:51.451 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:50:51.451 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:50:51.452 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:50:51.452 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.452 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.452 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:204181 10.19 10:9c0ded57:::benchmark_data_smithi018_422431_object99685:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e632)", 2023-09-23T20:50:51.452 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:49:40.066038+0000", 2023-09-23T20:50:51.452 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 71.299405754000006, 2023-09-23T20:50:51.452 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2429672300000001, 2023-09-23T20:50:51.452 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:50:51.452 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:50:51.452 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:50:51.452 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:50:51.453 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:50:51.453 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 204181 2023-09-23T20:50:51.453 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.453 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:50:51.453 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.453 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:50:51.453 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.066038+0000", 2023-09-23T20:50:51.453 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:50:51.453 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.453 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.453 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:50:51.454 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.066038+0000", 2023-09-23T20:50:51.454 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:50:51.454 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.454 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.454 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:50:51.454 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.066037+0000", 2023-09-23T20:50:51.454 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:50:51.454 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.454 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.454 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:50:51.454 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.066039+0000", 2023-09-23T20:50:51.455 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.8959999999999999e-06 2023-09-23T20:50:51.455 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.455 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.455 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:50:51.455 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.066044+0000", 2023-09-23T20:50:51.455 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.0289999999999996e-06 2023-09-23T20:50:51.455 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.455 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.455 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:50:51.455 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.066049+0000", 2023-09-23T20:50:51.455 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.7199999999999997e-06 2023-09-23T20:50:51.456 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.456 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.456 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:50:51.456 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.066377+0000", 2023-09-23T20:50:51.456 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00032813900000000002 2023-09-23T20:50:51.456 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.456 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.456 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:50:51.456 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.066384+0000", 2023-09-23T20:50:51.456 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.1250000000000004e-06 2023-09-23T20:50:51.456 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.456 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.457 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:50:51.457 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.300157+0000", 2023-09-23T20:50:51.457 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2337724750000001 2023-09-23T20:50:51.457 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.457 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.457 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:50:51.457 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.300254+0000", 2023-09-23T20:50:51.457 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.7622999999999997e-05 2023-09-23T20:50:51.457 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.457 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.457 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,3,4,5,7", 2023-09-23T20:50:51.457 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.300335+0000", 2023-09-23T20:50:51.458 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.0995000000000006e-05 2023-09-23T20:50:51.458 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.458 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.458 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:51.458 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.301911+0000", 2023-09-23T20:50:51.458 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0015756959999999999 2023-09-23T20:50:51.458 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.458 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.458 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:50:51.458 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.304580+0000", 2023-09-23T20:50:51.458 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0026686890000000001 2023-09-23T20:50:51.459 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.459 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.459 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:51.459 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.308839+0000", 2023-09-23T20:50:51.459 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0042589020000000002 2023-09-23T20:50:51.459 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.459 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.459 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:51.459 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.308859+0000", 2023-09-23T20:50:51.459 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.0454000000000001e-05 2023-09-23T20:50:51.459 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.460 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.460 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:51.460 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.308878+0000", 2023-09-23T20:50:51.460 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.8722000000000001e-05 2023-09-23T20:50:51.460 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.460 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.460 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:51.460 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.308896+0000", 2023-09-23T20:50:51.460 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.8578999999999999e-05 2023-09-23T20:50:51.460 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.460 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.461 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:50:51.461 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.308927+0000", 2023-09-23T20:50:51.461 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.0970999999999999e-05 2023-09-23T20:50:51.461 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.461 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.461 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:50:51.461 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.309005+0000", 2023-09-23T20:50:51.461 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.7485999999999996e-05 2023-09-23T20:50:51.461 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:50:51.461 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:50:51.461 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:50:51.461 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.461 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.462 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:204214 10.19 10:9c878d9b:::benchmark_data_smithi018_422431_object99718:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e632)", 2023-09-23T20:50:51.462 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:49:40.066052+0000", 2023-09-23T20:50:51.462 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 71.299391130000004, 2023-09-23T20:50:51.462 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.250179438, 2023-09-23T20:50:51.462 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:50:51.462 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:50:51.462 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:50:51.462 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:50:51.462 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:50:51.462 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 204214 2023-09-23T20:50:51.462 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.462 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:50:51.463 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.463 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:50:51.463 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.066052+0000", 2023-09-23T20:50:51.463 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:50:51.463 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.463 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.463 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:50:51.463 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.066052+0000", 2023-09-23T20:50:51.463 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:50:51.463 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.463 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.463 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:50:51.464 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.066052+0000", 2023-09-23T20:50:51.464 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:50:51.464 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.464 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.464 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:50:51.464 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.066054+0000", 2023-09-23T20:50:51.464 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.857e-06 2023-09-23T20:50:51.464 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.464 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.464 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:50:51.464 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.066059+0000", 2023-09-23T20:50:51.464 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.9710000000000003e-06 2023-09-23T20:50:51.465 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.465 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.465 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:50:51.465 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.066063+0000", 2023-09-23T20:50:51.465 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.2679999999999996e-06 2023-09-23T20:50:51.465 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.465 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.465 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:50:51.465 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.066410+0000", 2023-09-23T20:50:51.465 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00034650799999999998 2023-09-23T20:50:51.465 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.465 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.466 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:50:51.466 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.066416+0000", 2023-09-23T20:50:51.466 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.9599999999999997e-06 2023-09-23T20:50:51.466 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.466 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.466 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:50:51.466 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.301962+0000", 2023-09-23T20:50:51.466 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2355459820000001 2023-09-23T20:50:51.466 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.466 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.466 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:50:51.467 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.302086+0000", 2023-09-23T20:50:51.467 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00012444499999999999 2023-09-23T20:50:51.467 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.467 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.467 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,3,4,5,7", 2023-09-23T20:50:51.467 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.302564+0000", 2023-09-23T20:50:51.467 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00047793599999999999 2023-09-23T20:50:51.467 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.467 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.467 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:51.468 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.308590+0000", 2023-09-23T20:50:51.468 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0060258630000000002 2023-09-23T20:50:51.468 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.468 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.468 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:51.468 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.309035+0000", 2023-09-23T20:50:51.468 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00044549100000000001 2023-09-23T20:50:51.468 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.468 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.468 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:51.468 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.309056+0000", 2023-09-23T20:50:51.468 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.1026000000000002e-05 2023-09-23T20:50:51.469 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.469 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.469 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:51.469 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.309085+0000", 2023-09-23T20:50:51.469 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.9096999999999999e-05 2023-09-23T20:50:51.469 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.469 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.469 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:51.469 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.309104+0000", 2023-09-23T20:50:51.469 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.8637000000000001e-05 2023-09-23T20:50:51.469 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.470 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.470 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:50:51.470 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.316107+0000", 2023-09-23T20:50:51.470 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0070029039999999999 2023-09-23T20:50:51.470 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.470 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.470 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:50:51.470 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.316139+0000", 2023-09-23T20:50:51.470 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.2326999999999997e-05 2023-09-23T20:50:51.470 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.470 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.471 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:50:51.471 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.316232+0000", 2023-09-23T20:50:51.471 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.2415000000000002e-05 2023-09-23T20:50:51.471 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:50:51.471 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:50:51.471 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:50:51.471 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.471 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.471 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:204292 10.19 10:9d03cda3:::benchmark_data_smithi018_422431_object99796:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e632)", 2023-09-23T20:50:51.471 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:49:40.066066+0000", 2023-09-23T20:50:51.471 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 71.299377011000004, 2023-09-23T20:50:51.472 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2554046329999999, 2023-09-23T20:50:51.472 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:50:51.472 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:50:51.472 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:50:51.472 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:50:51.472 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:50:51.472 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 204292 2023-09-23T20:50:51.472 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.472 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:50:51.472 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.472 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:50:51.472 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.066066+0000", 2023-09-23T20:50:51.473 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:50:51.473 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.473 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.473 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:50:51.473 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.066066+0000", 2023-09-23T20:50:51.473 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:50:51.473 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.473 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.473 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:50:51.473 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.066066+0000", 2023-09-23T20:50:51.473 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:50:51.474 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.474 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.474 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:50:51.474 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.066068+0000", 2023-09-23T20:50:51.474 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.9570000000000001e-06 2023-09-23T20:50:51.474 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.474 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.474 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:50:51.474 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.066073+0000", 2023-09-23T20:50:51.474 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.9309999999999999e-06 2023-09-23T20:50:51.474 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.475 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.475 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:50:51.475 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.066077+0000", 2023-09-23T20:50:51.475 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.3229999999999999e-06 2023-09-23T20:50:51.475 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.475 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.475 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:50:51.475 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.066438+0000", 2023-09-23T20:50:51.475 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00036079800000000001 2023-09-23T20:50:51.475 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.475 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.476 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:50:51.476 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.066446+0000", 2023-09-23T20:50:51.476 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.3579999999999997e-06 2023-09-23T20:50:51.476 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.476 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.476 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:50:51.476 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.309161+0000", 2023-09-23T20:50:51.476 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.242715233 2023-09-23T20:50:51.476 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.476 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.476 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:50:51.476 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.309240+0000", 2023-09-23T20:50:51.477 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.9714000000000003e-05 2023-09-23T20:50:51.477 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.477 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.477 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,3,4,5,7", 2023-09-23T20:50:51.477 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.309318+0000", 2023-09-23T20:50:51.477 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.7170999999999997e-05 2023-09-23T20:50:51.477 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.477 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.477 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:51.477 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.316056+0000", 2023-09-23T20:50:51.477 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.006738717 2023-09-23T20:50:51.478 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.478 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.478 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:51.478 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.316100+0000", 2023-09-23T20:50:51.478 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.3297999999999998e-05 2023-09-23T20:50:51.478 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.478 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.478 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:50:51.478 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.316249+0000", 2023-09-23T20:50:51.478 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00014910300000000001 2023-09-23T20:50:51.478 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.479 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.479 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:51.479 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.316345+0000", 2023-09-23T20:50:51.479 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.6019000000000003e-05 2023-09-23T20:50:51.479 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.479 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.479 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:51.479 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.316600+0000", 2023-09-23T20:50:51.479 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00025491799999999997 2023-09-23T20:50:51.479 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.479 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.480 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:51.480 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.321245+0000", 2023-09-23T20:50:51.480 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0046448749999999997 2023-09-23T20:50:51.480 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.480 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.480 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:50:51.480 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.321304+0000", 2023-09-23T20:50:51.480 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.961e-05 2023-09-23T20:50:51.480 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.480 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.480 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:50:51.481 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.321471+0000", 2023-09-23T20:50:51.481 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00016687700000000001 2023-09-23T20:50:51.481 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:50:51.481 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:50:51.481 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:50:51.481 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.481 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.481 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:204314 10.19 10:9857f3c7:::benchmark_data_smithi018_422431_object99818:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e632)", 2023-09-23T20:50:51.481 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:49:40.066082+0000", 2023-09-23T20:50:51.481 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 71.299361351000002, 2023-09-23T20:50:51.481 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2606188569999999, 2023-09-23T20:50:51.481 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:50:51.482 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:50:51.482 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:50:51.482 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:50:51.482 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:50:51.482 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 204314 2023-09-23T20:50:51.482 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.482 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:50:51.482 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.482 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:50:51.482 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.066082+0000", 2023-09-23T20:50:51.482 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:50:51.483 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.483 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.483 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:50:51.483 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.066082+0000", 2023-09-23T20:50:51.483 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:50:51.483 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.483 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.483 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:50:51.483 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.066082+0000", 2023-09-23T20:50:51.483 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:50:51.483 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.483 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.483 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:50:51.484 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.066084+0000", 2023-09-23T20:50:51.484 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.8330000000000001e-06 2023-09-23T20:50:51.484 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.484 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.484 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:50:51.484 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.066089+0000", 2023-09-23T20:50:51.484 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.1370000000000001e-06 2023-09-23T20:50:51.484 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.484 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.484 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:50:51.484 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.066093+0000", 2023-09-23T20:50:51.485 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.5199999999999999e-06 2023-09-23T20:50:51.485 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.485 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.485 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:50:51.485 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.066470+0000", 2023-09-23T20:50:51.485 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00037686300000000001 2023-09-23T20:50:51.485 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.485 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.485 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:50:51.485 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.066477+0000", 2023-09-23T20:50:51.485 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.0310000000000001e-06 2023-09-23T20:50:51.486 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.486 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.486 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:50:51.486 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.311527+0000", 2023-09-23T20:50:51.486 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2450495319999999 2023-09-23T20:50:51.486 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.486 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.486 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:50:51.486 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.311631+0000", 2023-09-23T20:50:51.486 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000104663 2023-09-23T20:50:51.486 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.487 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.487 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,3,4,5,7", 2023-09-23T20:50:51.487 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.311779+0000", 2023-09-23T20:50:51.487 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00014769699999999999 2023-09-23T20:50:51.487 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.487 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.487 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:50:51.487 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.316262+0000", 2023-09-23T20:50:51.487 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0044833770000000002 2023-09-23T20:50:51.487 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.487 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.487 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:51.487 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.316414+0000", 2023-09-23T20:50:51.488 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00015101000000000001 2023-09-23T20:50:51.488 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.488 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.488 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:51.488 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.316549+0000", 2023-09-23T20:50:51.488 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000135217 2023-09-23T20:50:51.488 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.488 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.488 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:51.489 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.316667+0000", 2023-09-23T20:50:51.489 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000118356 2023-09-23T20:50:51.489 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.489 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.489 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:51.489 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.321584+0000", 2023-09-23T20:50:51.489 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0049172460000000001 2023-09-23T20:50:51.489 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.489 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.489 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:51.489 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.326497+0000", 2023-09-23T20:50:51.489 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0049127340000000002 2023-09-23T20:50:51.490 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.490 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.490 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:50:51.490 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.326556+0000", 2023-09-23T20:50:51.490 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.9151000000000003e-05 2023-09-23T20:50:51.490 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.490 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.490 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:50:51.490 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.326701+0000", 2023-09-23T20:50:51.490 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00014475999999999999 2023-09-23T20:50:51.490 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:50:51.491 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:50:51.491 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:50:51.491 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.491 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.491 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:204337 10.19 10:9f04108d:::benchmark_data_smithi018_422431_object99841:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e632)", 2023-09-23T20:50:51.491 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:49:40.066097+0000", 2023-09-23T20:50:51.491 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 71.299346792999998, 2023-09-23T20:50:51.491 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.26099918, 2023-09-23T20:50:51.491 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:50:51.491 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:50:51.491 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:50:51.492 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:50:51.492 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:50:51.492 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 204337 2023-09-23T20:50:51.492 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.492 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:50:51.492 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.492 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:50:51.492 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.066097+0000", 2023-09-23T20:50:51.492 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:50:51.492 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.492 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.492 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:50:51.493 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.066097+0000", 2023-09-23T20:50:51.493 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:50:51.493 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.493 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.493 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:50:51.493 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.066096+0000", 2023-09-23T20:50:51.493 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:50:51.493 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.493 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.493 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:50:51.493 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.066098+0000", 2023-09-23T20:50:51.493 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.855e-06 2023-09-23T20:50:51.494 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.494 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.494 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:50:51.494 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.066103+0000", 2023-09-23T20:50:51.494 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.0950000000000003e-06 2023-09-23T20:50:51.494 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.494 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.494 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:50:51.494 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.066108+0000", 2023-09-23T20:50:51.494 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.5120000000000002e-06 2023-09-23T20:50:51.494 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.495 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.495 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:50:51.495 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.066501+0000", 2023-09-23T20:50:51.495 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00039347600000000003 2023-09-23T20:50:51.495 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.495 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.495 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:50:51.495 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.066509+0000", 2023-09-23T20:50:51.495 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.1779999999999997e-06 2023-09-23T20:50:51.495 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.495 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.495 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:50:51.496 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.313703+0000", 2023-09-23T20:50:51.496 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2471940020000001 2023-09-23T20:50:51.496 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.496 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.496 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:50:51.496 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.313847+0000", 2023-09-23T20:50:51.496 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000144659 2023-09-23T20:50:51.496 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.496 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.496 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,3,4,5,7", 2023-09-23T20:50:51.496 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.313991+0000", 2023-09-23T20:50:51.497 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00014355499999999999 2023-09-23T20:50:51.497 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.497 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.497 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:50:51.497 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.316426+0000", 2023-09-23T20:50:51.497 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0024351120000000001 2023-09-23T20:50:51.497 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.497 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.497 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:51.497 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.316496+0000", 2023-09-23T20:50:51.497 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.9721000000000006e-05 2023-09-23T20:50:51.497 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.498 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.498 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:51.498 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.316624+0000", 2023-09-23T20:50:51.498 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00012890800000000001 2023-09-23T20:50:51.498 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.498 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.498 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:51.498 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.316700+0000", 2023-09-23T20:50:51.498 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.5164000000000001e-05 2023-09-23T20:50:51.498 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.498 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.499 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:51.499 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.326435+0000", 2023-09-23T20:50:51.499 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0097357209999999993 2023-09-23T20:50:51.499 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.499 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.499 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:51.499 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.326912+0000", 2023-09-23T20:50:51.499 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00047698499999999997 2023-09-23T20:50:51.499 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.499 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.499 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:50:51.499 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.326963+0000", 2023-09-23T20:50:51.500 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.0642000000000001e-05 2023-09-23T20:50:51.500 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.500 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.500 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:50:51.500 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.327096+0000", 2023-09-23T20:50:51.500 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00013287499999999999 2023-09-23T20:50:51.500 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:50:51.500 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:50:51.500 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:50:51.500 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.501 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.501 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:204419 10.19 10:98440f2c:::benchmark_data_smithi018_422431_object99923:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e632)", 2023-09-23T20:50:51.501 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:49:40.066111+0000", 2023-09-23T20:50:51.501 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 71.299332230999994, 2023-09-23T20:50:51.501 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.261221098, 2023-09-23T20:50:51.501 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:50:51.501 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:50:51.501 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:50:51.501 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:50:51.501 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:50:51.501 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 204419 2023-09-23T20:50:51.502 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.502 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:50:51.502 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.502 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:50:51.502 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.066111+0000", 2023-09-23T20:50:51.502 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:50:51.502 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.502 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.502 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:50:51.502 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.066111+0000", 2023-09-23T20:50:51.502 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:50:51.502 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.503 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.503 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:50:51.503 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.066111+0000", 2023-09-23T20:50:51.503 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:50:51.503 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.503 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.503 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:50:51.503 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.066113+0000", 2023-09-23T20:50:51.503 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.9259999999999999e-06 2023-09-23T20:50:51.503 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.503 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.504 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:50:51.504 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.066118+0000", 2023-09-23T20:50:51.504 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.0490000000000003e-06 2023-09-23T20:50:51.504 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.504 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.504 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:50:51.504 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.066122+0000", 2023-09-23T20:50:51.504 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.2250000000000002e-06 2023-09-23T20:50:51.504 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.504 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.504 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:50:51.505 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.066536+0000", 2023-09-23T20:50:51.505 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00041383000000000001 2023-09-23T20:50:51.505 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.505 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.505 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:50:51.505 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.066542+0000", 2023-09-23T20:50:51.505 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.8000000000000004e-06 2023-09-23T20:50:51.505 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.505 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.505 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:50:51.505 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.316803+0000", 2023-09-23T20:50:51.506 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2502611450000001 2023-09-23T20:50:51.506 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.506 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.506 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:50:51.506 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.316906+0000", 2023-09-23T20:50:51.506 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00010302300000000001 2023-09-23T20:50:51.506 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.506 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.506 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,3,4,5,7", 2023-09-23T20:50:51.506 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.317021+0000", 2023-09-23T20:50:51.506 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000114623 2023-09-23T20:50:51.506 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.507 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.507 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:51.507 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.321533+0000", 2023-09-23T20:50:51.507 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0045124989999999997 2023-09-23T20:50:51.507 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.507 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.507 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:51.507 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.321631+0000", 2023-09-23T20:50:51.507 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.8298e-05 2023-09-23T20:50:51.507 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.507 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.507 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:50:51.507 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.326320+0000", 2023-09-23T20:50:51.508 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0046887029999999998 2023-09-23T20:50:51.508 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.508 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.508 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:51.508 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.326825+0000", 2023-09-23T20:50:51.508 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00050456399999999999 2023-09-23T20:50:51.508 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.508 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.508 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:51.508 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.326872+0000", 2023-09-23T20:50:51.508 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.7119000000000001e-05 2023-09-23T20:50:51.508 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.509 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.509 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:51.509 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.327167+0000", 2023-09-23T20:50:51.509 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00029564100000000002 2023-09-23T20:50:51.509 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.509 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.509 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:50:51.509 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.327209+0000", 2023-09-23T20:50:51.509 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.1409000000000003e-05 2023-09-23T20:50:51.509 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.509 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.509 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:50:51.510 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.327332+0000", 2023-09-23T20:50:51.510 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00012351800000000001 2023-09-23T20:50:51.510 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:50:51.510 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:50:51.510 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:50:51.510 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.510 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.510 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:204425 10.19 10:98ab0872:::benchmark_data_smithi018_422431_object99929:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e632)", 2023-09-23T20:50:51.510 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:49:40.066140+0000", 2023-09-23T20:50:51.510 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 71.299303180999999, 2023-09-23T20:50:51.510 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2664997310000001, 2023-09-23T20:50:51.511 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:50:51.511 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:50:51.511 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:50:51.511 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:50:51.511 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:50:51.511 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 204425 2023-09-23T20:50:51.511 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.511 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:50:51.511 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.511 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:50:51.511 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.066140+0000", 2023-09-23T20:50:51.511 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:50:51.512 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.512 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.512 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:50:51.512 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.066140+0000", 2023-09-23T20:50:51.512 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:50:51.512 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.512 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.512 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:50:51.512 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.066140+0000", 2023-09-23T20:50:51.512 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:50:51.512 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.512 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.512 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:50:51.513 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.066145+0000", 2023-09-23T20:50:51.513 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.1590000000000001e-06 2023-09-23T20:50:51.513 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.513 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.513 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:50:51.513 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.066151+0000", 2023-09-23T20:50:51.513 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.1340000000000001e-06 2023-09-23T20:50:51.513 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.513 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.513 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:50:51.514 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.066156+0000", 2023-09-23T20:50:51.514 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.7979999999999996e-06 2023-09-23T20:50:51.514 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.514 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.514 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:50:51.514 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.066555+0000", 2023-09-23T20:50:51.514 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00039858100000000001 2023-09-23T20:50:51.514 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.514 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.514 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:50:51.514 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.066558+0000", 2023-09-23T20:50:51.515 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.5999999999999998e-06 2023-09-23T20:50:51.515 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.515 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.515 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:50:51.515 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.319198+0000", 2023-09-23T20:50:51.515 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.252639665 2023-09-23T20:50:51.515 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.515 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.515 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:50:51.515 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.319344+0000", 2023-09-23T20:50:51.515 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00014647999999999999 2023-09-23T20:50:51.516 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.516 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.516 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,3,4,5,7", 2023-09-23T20:50:51.516 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.319479+0000", 2023-09-23T20:50:51.516 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000134943 2023-09-23T20:50:51.516 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.516 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.516 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:50:51.516 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.326327+0000", 2023-09-23T20:50:51.516 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.006848041 2023-09-23T20:50:51.516 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.516 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.516 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:51.517 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.326779+0000", 2023-09-23T20:50:51.517 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00045170699999999998 2023-09-23T20:50:51.517 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.517 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.517 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:51.517 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.327374+0000", 2023-09-23T20:50:51.517 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00059442700000000002 2023-09-23T20:50:51.517 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.517 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.517 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:51.517 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.332256+0000", 2023-09-23T20:50:51.518 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0048820000000000001 2023-09-23T20:50:51.518 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.518 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.518 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:51.518 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.332404+0000", 2023-09-23T20:50:51.518 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000148033 2023-09-23T20:50:51.518 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.518 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.518 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:51.518 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.332452+0000", 2023-09-23T20:50:51.518 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.8180000000000003e-05 2023-09-23T20:50:51.519 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.519 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.519 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:50:51.519 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.332509+0000", 2023-09-23T20:50:51.519 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.7515000000000001e-05 2023-09-23T20:50:51.519 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.519 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.519 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:50:51.519 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.332640+0000", 2023-09-23T20:50:51.519 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00013076200000000001 2023-09-23T20:50:51.519 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:50:51.520 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:50:51.520 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:50:51.520 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.520 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.520 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:204430 10.19 10:9d9fd3cb:::benchmark_data_smithi018_422431_object99934:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e632)", 2023-09-23T20:50:51.520 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:49:40.066160+0000", 2023-09-23T20:50:51.520 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 71.299283845000005, 2023-09-23T20:50:51.520 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.266861362, 2023-09-23T20:50:51.520 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:50:51.520 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:50:51.520 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:50:51.520 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:50:51.521 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:50:51.521 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 204430 2023-09-23T20:50:51.521 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.521 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:50:51.521 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.521 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:50:51.521 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.066160+0000", 2023-09-23T20:50:51.521 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:50:51.521 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.521 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.521 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:50:51.522 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.066160+0000", 2023-09-23T20:50:51.522 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:50:51.522 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.522 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.522 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:50:51.522 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.066159+0000", 2023-09-23T20:50:51.522 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:50:51.522 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.522 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.522 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:50:51.522 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.066161+0000", 2023-09-23T20:50:51.522 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.1160000000000002e-06 2023-09-23T20:50:51.523 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.523 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.523 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:50:51.523 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.066166+0000", 2023-09-23T20:50:51.523 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.882e-06 2023-09-23T20:50:51.523 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.523 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.523 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:50:51.523 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.066171+0000", 2023-09-23T20:50:51.523 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.3109999999999999e-06 2023-09-23T20:50:51.523 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.523 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.524 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:50:51.524 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.066570+0000", 2023-09-23T20:50:51.524 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00039923299999999998 2023-09-23T20:50:51.524 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.524 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.524 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:50:51.524 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.066573+0000", 2023-09-23T20:50:51.524 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.4369999999999999e-06 2023-09-23T20:50:51.524 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.524 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.524 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:50:51.524 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.321664+0000", 2023-09-23T20:50:51.524 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2550902070000001 2023-09-23T20:50:51.525 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.525 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.525 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:50:51.525 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.321819+0000", 2023-09-23T20:50:51.525 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00015541099999999999 2023-09-23T20:50:51.525 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.525 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.525 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,3,4,5,7", 2023-09-23T20:50:51.525 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.321971+0000", 2023-09-23T20:50:51.525 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00015169599999999999 2023-09-23T20:50:51.525 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.526 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.526 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:50:51.526 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.326719+0000", 2023-09-23T20:50:51.526 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0047481060000000002 2023-09-23T20:50:51.526 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.526 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.526 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:51.526 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.327135+0000", 2023-09-23T20:50:51.526 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000416053 2023-09-23T20:50:51.526 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.527 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.527 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:51.527 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.327415+0000", 2023-09-23T20:50:51.527 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00027987500000000001 2023-09-23T20:50:51.527 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.527 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.527 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:51.527 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.332678+0000", 2023-09-23T20:50:51.527 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.005263139 2023-09-23T20:50:51.527 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.527 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.527 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:51.527 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.332765+0000", 2023-09-23T20:50:51.528 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.7306999999999996e-05 2023-09-23T20:50:51.528 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.528 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.528 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:51.528 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.332802+0000", 2023-09-23T20:50:51.528 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.7030000000000003e-05 2023-09-23T20:50:51.528 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.528 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.528 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:50:51.528 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.332853+0000", 2023-09-23T20:50:51.528 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.0464999999999999e-05 2023-09-23T20:50:51.528 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.528 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.529 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:50:51.529 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.333021+0000", 2023-09-23T20:50:51.529 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00016836800000000001 2023-09-23T20:50:51.529 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:50:51.529 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:50:51.529 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:50:51.529 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.529 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.529 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:204432 10.19 10:993ae8af:::benchmark_data_smithi018_422431_object99936:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e632)", 2023-09-23T20:50:51.529 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:49:40.066264+0000", 2023-09-23T20:50:51.529 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 71.299179225000003, 2023-09-23T20:50:51.529 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2715216540000001, 2023-09-23T20:50:51.529 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:50:51.530 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:50:51.530 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:50:51.530 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:50:51.530 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:50:51.530 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 204432 2023-09-23T20:50:51.530 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.530 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:50:51.530 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.530 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:50:51.530 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.066264+0000", 2023-09-23T20:50:51.531 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:50:51.531 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.531 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.531 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:50:51.531 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.066264+0000", 2023-09-23T20:50:51.531 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:50:51.531 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.531 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.531 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:50:51.531 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.066264+0000", 2023-09-23T20:50:51.531 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:50:51.532 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.532 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.532 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:50:51.532 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.066267+0000", 2023-09-23T20:50:51.532 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.191e-06 2023-09-23T20:50:51.532 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.532 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.532 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:50:51.532 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.066278+0000", 2023-09-23T20:50:51.532 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0881e-05 2023-09-23T20:50:51.532 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.533 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.533 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:50:51.533 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.066284+0000", 2023-09-23T20:50:51.533 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.9179999999999999e-06 2023-09-23T20:50:51.533 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.533 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.533 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:50:51.533 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.066585+0000", 2023-09-23T20:50:51.533 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00030160199999999999 2023-09-23T20:50:51.533 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.534 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.534 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:50:51.534 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.066591+0000", 2023-09-23T20:50:51.534 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.0850000000000002e-06 2023-09-23T20:50:51.534 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.534 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.534 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:50:51.534 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.323971+0000", 2023-09-23T20:50:51.534 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2573792749999999 2023-09-23T20:50:51.534 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.534 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.535 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:50:51.535 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.324119+0000", 2023-09-23T20:50:51.535 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000148267 2023-09-23T20:50:51.535 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.535 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.535 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,3,4,5,7", 2023-09-23T20:50:51.535 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.324252+0000", 2023-09-23T20:50:51.535 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00013254299999999999 2023-09-23T20:50:51.535 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.535 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.535 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:51.535 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.332329+0000", 2023-09-23T20:50:51.536 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0080770149999999999 2023-09-23T20:50:51.536 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.536 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.536 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:50:51.536 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.332342+0000", 2023-09-23T20:50:51.536 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3088000000000001e-05 2023-09-23T20:50:51.536 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.536 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.536 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:51.536 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.332707+0000", 2023-09-23T20:50:51.536 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00036563500000000001 2023-09-23T20:50:51.536 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.537 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.537 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:51.537 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.333066+0000", 2023-09-23T20:50:51.537 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00035866199999999999 2023-09-23T20:50:51.537 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.537 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.537 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:51.537 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.333173+0000", 2023-09-23T20:50:51.537 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000107327 2023-09-23T20:50:51.537 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.537 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.537 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:51.537 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.337604+0000", 2023-09-23T20:50:51.538 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0044310540000000002 2023-09-23T20:50:51.538 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.538 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.538 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:50:51.538 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.337655+0000", 2023-09-23T20:50:51.538 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.0435000000000001e-05 2023-09-23T20:50:51.538 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.538 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.538 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:50:51.538 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.337786+0000", 2023-09-23T20:50:51.538 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00013113600000000001 2023-09-23T20:50:51.538 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:50:51.539 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:50:51.539 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:50:51.539 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.539 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.539 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:204470 10.19 10:9e68bfed:::benchmark_data_smithi018_422431_object99974:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e632)", 2023-09-23T20:50:51.539 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:49:40.066302+0000", 2023-09-23T20:50:51.539 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 71.299141206000002, 2023-09-23T20:50:51.539 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.271876853, 2023-09-23T20:50:51.539 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:50:51.539 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:50:51.539 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:50:51.540 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:50:51.540 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:50:51.540 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 204470 2023-09-23T20:50:51.540 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.540 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:50:51.540 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.540 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:50:51.540 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.066302+0000", 2023-09-23T20:50:51.540 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:50:51.541 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.541 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.541 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:50:51.541 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.066302+0000", 2023-09-23T20:50:51.541 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:50:51.541 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.541 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.541 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:50:51.541 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.066302+0000", 2023-09-23T20:50:51.541 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:50:51.541 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.541 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.542 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:50:51.542 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.066304+0000", 2023-09-23T20:50:51.542 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.3479999999999998e-06 2023-09-23T20:50:51.542 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.542 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.542 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:50:51.542 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.066310+0000", 2023-09-23T20:50:51.542 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.6300000000000003e-06 2023-09-23T20:50:51.542 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.542 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.542 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:50:51.543 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.066314+0000", 2023-09-23T20:50:51.543 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.5430000000000004e-06 2023-09-23T20:50:51.543 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.543 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.543 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:50:51.543 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.066613+0000", 2023-09-23T20:50:51.543 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00029859299999999999 2023-09-23T20:50:51.543 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.543 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.543 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:50:51.543 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.066619+0000", 2023-09-23T20:50:51.544 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.3459999999999999e-06 2023-09-23T20:50:51.544 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.544 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.544 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:50:51.544 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.327442+0000", 2023-09-23T20:50:51.544 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.260823013 2023-09-23T20:50:51.544 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.544 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.544 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:50:51.544 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.327533+0000", 2023-09-23T20:50:51.544 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.0174999999999996e-05 2023-09-23T20:50:51.545 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.545 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.545 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,3,4,5,7", 2023-09-23T20:50:51.545 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.327605+0000", 2023-09-23T20:50:51.545 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.2286999999999994e-05 2023-09-23T20:50:51.545 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.545 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.545 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:50:51.545 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.332349+0000", 2023-09-23T20:50:51.545 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0047445600000000001 2023-09-23T20:50:51.545 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.545 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.546 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:51.546 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.332736+0000", 2023-09-23T20:50:51.546 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00038695800000000002 2023-09-23T20:50:51.546 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.546 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.546 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:51.546 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.333103+0000", 2023-09-23T20:50:51.546 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000366437 2023-09-23T20:50:51.546 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.546 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.546 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:51.547 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.337917+0000", 2023-09-23T20:50:51.547 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0048138729999999998 2023-09-23T20:50:51.547 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.547 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.547 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:51.547 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.337967+0000", 2023-09-23T20:50:51.547 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.062e-05 2023-09-23T20:50:51.547 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.547 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.547 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:51.547 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.337999+0000", 2023-09-23T20:50:51.547 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.1406999999999999e-05 2023-09-23T20:50:51.548 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.548 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.548 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:50:51.548 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.338046+0000", 2023-09-23T20:50:51.548 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.7287999999999999e-05 2023-09-23T20:50:51.548 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.548 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.548 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:50:51.548 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.338179+0000", 2023-09-23T20:50:51.548 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00013311200000000001 2023-09-23T20:50:51.548 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:50:51.549 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:50:51.549 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:50:51.549 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.549 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.549 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:204483 10.19 10:9a96a368:::benchmark_data_smithi018_422431_object99987:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e632)", 2023-09-23T20:50:51.549 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:49:40.066320+0000", 2023-09-23T20:50:51.549 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 71.299123706000003, 2023-09-23T20:50:51.549 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2809402570000001, 2023-09-23T20:50:51.549 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:50:51.549 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:50:51.549 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:50:51.549 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:50:51.550 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:50:51.550 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 204483 2023-09-23T20:50:51.550 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.550 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:50:51.550 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.550 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:50:51.550 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.066320+0000", 2023-09-23T20:50:51.550 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:50:51.550 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.550 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.550 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:50:51.551 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.066320+0000", 2023-09-23T20:50:51.551 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:50:51.551 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.551 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.551 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:50:51.551 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.066319+0000", 2023-09-23T20:50:51.551 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:50:51.551 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.551 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.551 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:50:51.551 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.066321+0000", 2023-09-23T20:50:51.551 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.8819999999999999e-06 2023-09-23T20:50:51.551 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.552 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.552 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:50:51.552 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.066327+0000", 2023-09-23T20:50:51.552 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.5079999999999996e-06 2023-09-23T20:50:51.552 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.552 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.552 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:50:51.552 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.066331+0000", 2023-09-23T20:50:51.552 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.3019999999999996e-06 2023-09-23T20:50:51.552 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.552 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.553 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:50:51.553 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.066636+0000", 2023-09-23T20:50:51.553 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00030523799999999999 2023-09-23T20:50:51.553 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.553 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.553 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:50:51.553 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:40.066640+0000", 2023-09-23T20:50:51.553 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.9600000000000002e-06 2023-09-23T20:50:51.553 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.553 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.553 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:50:51.554 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.329719+0000", 2023-09-23T20:50:51.554 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2630788420000001 2023-09-23T20:50:51.554 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.554 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.554 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:50:51.554 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.329857+0000", 2023-09-23T20:50:51.554 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00013808900000000001 2023-09-23T20:50:51.554 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.554 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.554 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,3,4,5,7", 2023-09-23T20:50:51.554 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.329999+0000", 2023-09-23T20:50:51.555 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000141774 2023-09-23T20:50:51.555 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.555 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.555 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:51.555 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.333139+0000", 2023-09-23T20:50:51.555 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0031401760000000002 2023-09-23T20:50:51.555 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.555 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.555 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:50:51.555 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.337567+0000", 2023-09-23T20:50:51.555 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0044272449999999998 2023-09-23T20:50:51.555 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.556 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.556 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:51.556 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.338247+0000", 2023-09-23T20:50:51.556 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00068052499999999997 2023-09-23T20:50:51.556 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.556 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.556 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:51.556 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.340729+0000", 2023-09-23T20:50:51.556 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0024823760000000001 2023-09-23T20:50:51.556 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.556 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.557 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:51.557 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.340840+0000", 2023-09-23T20:50:51.557 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000110272 2023-09-23T20:50:51.557 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.557 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.557 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:50:51.557 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.347100+0000", 2023-09-23T20:50:51.557 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0062600959999999997 2023-09-23T20:50:51.557 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.557 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.557 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:50:51.557 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.347133+0000", 2023-09-23T20:50:51.557 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.3621000000000002e-05 2023-09-23T20:50:51.558 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:50:51.558 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:50:51.558 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:50:51.558 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:41.347260+0000", 2023-09-23T20:50:51.558 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00012662 2023-09-23T20:50:51.558 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:50:51.558 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:50:51.558 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:50:51.558 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:50:51.558 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:50:51.558 INFO:teuthology.orchestra.run.smithi018.stdout:} 2023-09-23T20:50:52.072 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:51 smithi018 ceph-mon[179647]: pgmap v1625: 103 pgs: 103 active+clean; 8.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 247 KiB/s rd, 175 KiB/s wr, 104 op/s 2023-09-23T20:50:52.072 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:51 smithi018 ceph-mon[184144]: pgmap v1625: 103 pgs: 103 active+clean; 8.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 247 KiB/s rd, 175 KiB/s wr, 104 op/s 2023-09-23T20:50:52.157 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T20:50:52.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:51 smithi149 ceph-mon[170394]: pgmap v1625: 103 pgs: 103 active+clean; 8.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 247 KiB/s rd, 175 KiB/s wr, 104 op/s 2023-09-23T20:50:52.774 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:52 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1033641377' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T20:50:52.774 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:52 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T20:50:52.775 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:52 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1033641377' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T20:50:52.775 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:52 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T20:50:52.777 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:50:52.782 INFO:teuthology.orchestra.run.smithi018.stderr:reweighted osd.4 to 1 (10000) 2023-09-23T20:50:53.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:52 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1033641377' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T20:50:53.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:52 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T20:50:53.414 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd reweight 2 1 2023-09-23T20:50:53.787 INFO:teuthology.orchestra.run.smithi018.stderr:noscrub is unset 2023-09-23T20:50:53.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:53 smithi018 ceph-mon[184144]: pgmap v1626: 103 pgs: 103 active+clean; 8.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 194 KiB/s rd, 138 KiB/s wr, 82 op/s 2023-09-23T20:50:53.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:53 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T20:50:53.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:53 smithi018 ceph-mon[184144]: osdmap e668: 8 total, 8 up, 7 in 2023-09-23T20:50:53.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:53 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2731414253' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:50:53.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:53 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:50:53.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:53 smithi018 ceph-mon[184144]: failed to encode map e668 with expected crc 2023-09-23T20:50:53.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:53 smithi018 ceph-mon[184144]: failed to encode map e668 with expected crc 2023-09-23T20:50:53.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:53 smithi018 ceph-mon[179647]: pgmap v1626: 103 pgs: 103 active+clean; 8.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 194 KiB/s rd, 138 KiB/s wr, 82 op/s 2023-09-23T20:50:53.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:53 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T20:50:53.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:53 smithi018 ceph-mon[179647]: osdmap e668: 8 total, 8 up, 7 in 2023-09-23T20:50:53.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:53 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2731414253' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:50:53.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:53 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:50:53.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:53 smithi018 ceph-mon[179647]: failed to encode map e668 with expected crc 2023-09-23T20:50:53.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:53 smithi018 ceph-mon[179647]: failed to encode map e668 with expected crc 2023-09-23T20:50:54.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:53 smithi149 ceph-mon[170394]: pgmap v1626: 103 pgs: 103 active+clean; 8.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 194 KiB/s rd, 138 KiB/s wr, 82 op/s 2023-09-23T20:50:54.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:53 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T20:50:54.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:53 smithi149 ceph-mon[170394]: osdmap e668: 8 total, 8 up, 7 in 2023-09-23T20:50:54.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:53 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2731414253' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:50:54.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:53 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T20:50:54.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:53 smithi149 ceph-mon[170394]: failed to encode map e668 with expected crc 2023-09-23T20:50:54.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:53 smithi149 ceph-mon[170394]: failed to encode map e668 with expected crc 2023-09-23T20:50:54.244 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T20:50:54.501 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2023-09-23T20:50:54.501 INFO:teuthology.orchestra.run.smithi018.stdout: "ops": [], 2023-09-23T20:50:54.501 INFO:teuthology.orchestra.run.smithi018.stdout: "num_ops": 0 2023-09-23T20:50:54.501 INFO:teuthology.orchestra.run.smithi018.stdout:} 2023-09-23T20:50:54.533 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:50:54.978 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T20:50:55.048 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:54 smithi018 ceph-mon[184144]: failed to encode map e668 with expected crc 2023-09-23T20:50:55.048 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:54 smithi018 ceph-mon[184144]: failed to encode map e668 with expected crc 2023-09-23T20:50:55.048 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:54 smithi018 ceph-mon[184144]: failed to encode map e668 with expected crc 2023-09-23T20:50:55.048 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:54 smithi018 ceph-mon[184144]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:50:55.048 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:54 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:50:55.048 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:54 smithi018 ceph-mon[184144]: osdmap e669: 8 total, 8 up, 7 in 2023-09-23T20:50:55.048 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:54 smithi018 ceph-mon[184144]: failed to encode map e669 with expected crc 2023-09-23T20:50:55.049 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:50:54 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:50:54] "GET /metrics HTTP/1.1" 200 34159 "" "Prometheus/2.43.0" 2023-09-23T20:50:55.049 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:54 smithi018 ceph-mon[179647]: failed to encode map e668 with expected crc 2023-09-23T20:50:55.049 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:54 smithi018 ceph-mon[179647]: failed to encode map e668 with expected crc 2023-09-23T20:50:55.049 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:54 smithi018 ceph-mon[179647]: failed to encode map e668 with expected crc 2023-09-23T20:50:55.049 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:54 smithi018 ceph-mon[179647]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:50:55.049 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:54 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:50:55.049 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:54 smithi018 ceph-mon[179647]: osdmap e669: 8 total, 8 up, 7 in 2023-09-23T20:50:55.050 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:54 smithi018 ceph-mon[179647]: failed to encode map e669 with expected crc 2023-09-23T20:50:55.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:54 smithi149 ceph-mon[170394]: failed to encode map e668 with expected crc 2023-09-23T20:50:55.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:54 smithi149 ceph-mon[170394]: failed to encode map e668 with expected crc 2023-09-23T20:50:55.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:54 smithi149 ceph-mon[170394]: failed to encode map e668 with expected crc 2023-09-23T20:50:55.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:54 smithi149 ceph-mon[170394]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T20:50:55.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:54 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T20:50:55.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:54 smithi149 ceph-mon[170394]: osdmap e669: 8 total, 8 up, 7 in 2023-09-23T20:50:55.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:54 smithi149 ceph-mon[170394]: failed to encode map e669 with expected crc 2023-09-23T20:50:55.250 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:50:56.077 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:55 smithi018 ceph-mon[179647]: pgmap v1629: 103 pgs: 103 active+clean; 8.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail 2023-09-23T20:50:56.077 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:55 smithi018 ceph-mon[184144]: pgmap v1629: 103 pgs: 103 active+clean; 8.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail 2023-09-23T20:50:56.123 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:50:56.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:55 smithi149 ceph-mon[170394]: pgmap v1629: 103 pgs: 103 active+clean; 8.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail 2023-09-23T20:50:56.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:56 smithi018 ceph-mon[184144]: failed to encode map e669 with expected crc 2023-09-23T20:50:56.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:56 smithi018 ceph-mon[184144]: failed to encode map e669 with expected crc 2023-09-23T20:50:56.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:56 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3579993169' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T20:50:56.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:56 smithi018 ceph-mon[179647]: failed to encode map e669 with expected crc 2023-09-23T20:50:56.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:56 smithi018 ceph-mon[179647]: failed to encode map e669 with expected crc 2023-09-23T20:50:56.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:56 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3579993169' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T20:50:57.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:56 smithi149 ceph-mon[170394]: failed to encode map e669 with expected crc 2023-09-23T20:50:57.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:56 smithi149 ceph-mon[170394]: failed to encode map e669 with expected crc 2023-09-23T20:50:57.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:56 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3579993169' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T20:50:57.578 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2023-09-23T20:50:57.578 INFO:teuthology.orchestra.run.smithi018.stdout: "ops": [], 2023-09-23T20:50:57.578 INFO:teuthology.orchestra.run.smithi018.stdout: "complaint_time": 30, 2023-09-23T20:50:57.579 INFO:teuthology.orchestra.run.smithi018.stdout: "num_blocked_ops": 0 2023-09-23T20:50:57.579 INFO:teuthology.orchestra.run.smithi018.stdout:} 2023-09-23T20:50:57.730 INFO:teuthology.orchestra.run.smithi018.stderr:reweighted osd.2 to 1 (10000) 2023-09-23T20:50:57.730 INFO:teuthology.orchestra.run.smithi018.stderr:nodeep-scrub is unset 2023-09-23T20:50:57.972 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:57 smithi018 ceph-mon[184144]: pgmap v1630: 103 pgs: 103 active+clean; 8.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail 2023-09-23T20:50:57.972 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:57 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1732526111' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:50:57.973 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:57 smithi018 ceph-mon[184144]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:50:57.973 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:57 smithi018 ceph-mon[184144]: Cluster is now healthy 2023-09-23T20:50:57.973 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:57 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3579993169' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T20:50:57.973 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:57 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1732526111' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:50:57.973 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:57 smithi018 ceph-mon[184144]: osdmap e670: 8 total, 8 up, 7 in 2023-09-23T20:50:57.973 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:57 smithi018 ceph-mon[179647]: pgmap v1630: 103 pgs: 103 active+clean; 8.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail 2023-09-23T20:50:57.973 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:57 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1732526111' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:50:57.973 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:57 smithi018 ceph-mon[179647]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:50:57.973 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:57 smithi018 ceph-mon[179647]: Cluster is now healthy 2023-09-23T20:50:57.974 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:57 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3579993169' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T20:50:57.974 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:57 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1732526111' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:50:57.974 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:57 smithi018 ceph-mon[179647]: osdmap e670: 8 total, 8 up, 7 in 2023-09-23T20:50:58.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:57 smithi149 ceph-mon[170394]: pgmap v1630: 103 pgs: 103 active+clean; 8.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail 2023-09-23T20:50:58.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:57 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1732526111' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T20:50:58.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:57 smithi149 ceph-mon[170394]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T20:50:58.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:57 smithi149 ceph-mon[170394]: Cluster is now healthy 2023-09-23T20:50:58.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:57 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3579993169' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T20:50:58.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:57 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1732526111' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T20:50:58.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:57 smithi149 ceph-mon[170394]: osdmap e670: 8 total, 8 up, 7 in 2023-09-23T20:50:58.321 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T20:50:58.559 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-09-23T20:50:58.559 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:50:58.793 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:58 smithi018 ceph-mon[179647]: failed to encode map e670 with expected crc 2023-09-23T20:50:58.938 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:50:59.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:58 smithi018 ceph-mon[184144]: failed to encode map e670 with expected crc 2023-09-23T20:50:59.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:58 smithi149 ceph-mon[170394]: failed to encode map e670 with expected crc 2023-09-23T20:50:59.480 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:51:00.037 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:59 smithi018 ceph-mon[179647]: pgmap v1632: 103 pgs: 103 active+clean; 8.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail 2023-09-23T20:51:00.038 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:59 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:51:00.038 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:50:59 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:51:00.038 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:59 smithi018 ceph-mon[184144]: pgmap v1632: 103 pgs: 103 active+clean; 8.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail 2023-09-23T20:51:00.038 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:59 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:51:00.038 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:50:59 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:51:00.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:59 smithi149 ceph-mon[170394]: pgmap v1632: 103 pgs: 103 active+clean; 8.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail 2023-09-23T20:51:00.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:59 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:51:00.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:50:59 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:51:00.564 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2023-09-23T20:51:00.564 INFO:teuthology.orchestra.run.smithi018.stdout: "size": 20, 2023-09-23T20:51:00.564 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 600, 2023-09-23T20:51:00.564 INFO:teuthology.orchestra.run.smithi018.stdout: "ops": [ 2023-09-23T20:51:00.564 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.564 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.36450.0:185319 9.8 9.8673f918 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e503)", 2023-09-23T20:51:00.564 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:44:18.697532+0000", 2023-09-23T20:51:00.564 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 401.86365597000002, 2023-09-23T20:51:00.564 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.93022128999999998, 2023-09-23T20:51:00.565 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:51:00.565 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:51:00.565 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:51:00.565 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.36450", 2023-09-23T20:51:00.565 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:51:00.565 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 185319 2023-09-23T20:51:00.565 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.565 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:51:00.565 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.565 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:51:00.565 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.697532+0000", 2023-09-23T20:51:00.565 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:51:00.566 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.566 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.566 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:51:00.566 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.697532+0000", 2023-09-23T20:51:00.566 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:51:00.566 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.566 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.566 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:51:00.566 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.697532+0000", 2023-09-23T20:51:00.566 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:51:00.566 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.567 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.567 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:51:00.567 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.697535+0000", 2023-09-23T20:51:00.567 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.1709999999999998e-06 2023-09-23T20:51:00.567 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.567 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.567 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:51:00.567 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.697542+0000", 2023-09-23T20:51:00.567 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.7880000000000001e-06 2023-09-23T20:51:00.567 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.567 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.568 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:51:00.568 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.697547+0000", 2023-09-23T20:51:00.568 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.0159999999999999e-06 2023-09-23T20:51:00.568 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.568 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.568 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:51:00.568 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.698190+0000", 2023-09-23T20:51:00.568 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00064296299999999998 2023-09-23T20:51:00.568 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.568 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.568 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:51:00.569 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.698201+0000", 2023-09-23T20:51:00.569 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0898e-05 2023-09-23T20:51:00.569 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.569 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.569 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:51:00.569 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:19.627739+0000", 2023-09-23T20:51:00.569 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.92953824600000001 2023-09-23T20:51:00.569 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.569 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.569 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:51:00.569 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:19.627754+0000", 2023-09-23T20:51:00.569 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.4634000000000001e-05 2023-09-23T20:51:00.570 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:51:00.570 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:51:00.570 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:51:00.570 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.570 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.570 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.36450.0:185321 9.8 9.f3369438 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e503)", 2023-09-23T20:51:00.570 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:44:18.708465+0000", 2023-09-23T20:51:00.570 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 401.852723771, 2023-09-23T20:51:00.570 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.91932099599999995, 2023-09-23T20:51:00.570 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:51:00.570 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:51:00.570 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:51:00.571 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.36450", 2023-09-23T20:51:00.571 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:51:00.571 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 185321 2023-09-23T20:51:00.571 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.571 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:51:00.571 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.571 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:51:00.571 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.708465+0000", 2023-09-23T20:51:00.571 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:51:00.571 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.571 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.571 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:51:00.572 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.708465+0000", 2023-09-23T20:51:00.572 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:51:00.572 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.572 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.572 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:51:00.572 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.708463+0000", 2023-09-23T20:51:00.572 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:51:00.572 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.572 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.572 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:51:00.572 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.708482+0000", 2023-09-23T20:51:00.572 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.9258000000000001e-05 2023-09-23T20:51:00.573 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.573 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.573 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:51:00.573 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.708500+0000", 2023-09-23T20:51:00.573 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.7708000000000001e-05 2023-09-23T20:51:00.573 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.573 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.573 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:51:00.573 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.708510+0000", 2023-09-23T20:51:00.573 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.6190000000000006e-06 2023-09-23T20:51:00.573 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.573 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.574 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:51:00.574 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.708611+0000", 2023-09-23T20:51:00.574 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000101296 2023-09-23T20:51:00.574 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.574 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.574 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:51:00.574 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:18.708623+0000", 2023-09-23T20:51:00.574 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.201e-05 2023-09-23T20:51:00.574 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.574 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.574 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:51:00.575 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:19.627776+0000", 2023-09-23T20:51:00.575 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.919153161 2023-09-23T20:51:00.575 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.575 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.575 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:51:00.575 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:44:19.627786+0000", 2023-09-23T20:51:00.575 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.4369999999999996e-06 2023-09-23T20:51:00.575 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:51:00.575 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:51:00.575 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:51:00.575 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.575 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.576 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:196320 10.15 10:a8362cca:::benchmark_data_smithi018_422431_object91824:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e621)", 2023-09-23T20:51:00.576 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:49:26.039338+0000", 2023-09-23T20:51:00.576 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 94.521850420000007, 2023-09-23T20:51:00.576 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.04174229, 2023-09-23T20:51:00.576 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:51:00.576 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:00.576 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:51:00.576 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:51:00.576 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:51:00.576 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 196320 2023-09-23T20:51:00.576 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.577 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:51:00.577 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.577 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:51:00.577 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:26.039338+0000", 2023-09-23T20:51:00.577 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:51:00.577 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.577 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.577 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:51:00.577 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:26.039338+0000", 2023-09-23T20:51:00.577 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:51:00.577 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.577 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.578 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:51:00.578 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:26.039336+0000", 2023-09-23T20:51:00.578 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:51:00.578 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.578 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.578 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:51:00.578 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:26.039345+0000", 2023-09-23T20:51:00.578 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.106e-06 2023-09-23T20:51:00.578 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.578 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.579 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:51:00.579 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:26.039358+0000", 2023-09-23T20:51:00.579 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3097e-05 2023-09-23T20:51:00.579 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.579 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.579 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:51:00.579 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:26.039366+0000", 2023-09-23T20:51:00.579 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.3780000000000001e-06 2023-09-23T20:51:00.579 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.579 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.579 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:51:00.579 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:26.039441+0000", 2023-09-23T20:51:00.580 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.4954000000000001e-05 2023-09-23T20:51:00.580 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.580 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.580 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:51:00.580 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:26.039458+0000", 2023-09-23T20:51:00.580 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.7248000000000002e-05 2023-09-23T20:51:00.580 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.580 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.580 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:51:00.580 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:27.050175+0000", 2023-09-23T20:51:00.580 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.010716379 2023-09-23T20:51:00.581 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.581 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.581 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:51:00.581 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:27.050307+0000", 2023-09-23T20:51:00.581 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000132724 2023-09-23T20:51:00.581 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.581 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.581 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 0,1,2,5", 2023-09-23T20:51:00.581 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:27.050575+0000", 2023-09-23T20:51:00.581 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00026792800000000001 2023-09-23T20:51:00.581 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.581 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.582 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:51:00.582 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:27.054098+0000", 2023-09-23T20:51:00.582 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.003522552 2023-09-23T20:51:00.582 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.582 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.582 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:00.582 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:27.059295+0000", 2023-09-23T20:51:00.582 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0051966979999999996 2023-09-23T20:51:00.582 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.582 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.582 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:00.583 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:27.059361+0000", 2023-09-23T20:51:00.583 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.6346000000000005e-05 2023-09-23T20:51:00.583 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.583 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.583 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:00.583 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:27.059460+0000", 2023-09-23T20:51:00.583 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.8892000000000001e-05 2023-09-23T20:51:00.583 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.583 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.583 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:00.583 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:27.080897+0000", 2023-09-23T20:51:00.583 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.021437641 2023-09-23T20:51:00.584 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.584 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.584 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:51:00.584 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:27.080945+0000", 2023-09-23T20:51:00.584 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.7849000000000002e-05 2023-09-23T20:51:00.584 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.584 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.584 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:51:00.584 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:27.081080+0000", 2023-09-23T20:51:00.584 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000134964 2023-09-23T20:51:00.584 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:51:00.584 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:51:00.585 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:51:00.585 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.585 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.585 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:196341 10.15 10:afe499c9:::benchmark_data_smithi018_422431_object91845:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e621)", 2023-09-23T20:51:00.585 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:49:26.070140+0000", 2023-09-23T20:51:00.585 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 94.491048184999997, 2023-09-23T20:51:00.585 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0111852059999999, 2023-09-23T20:51:00.585 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:51:00.585 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:00.585 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:51:00.585 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:51:00.586 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:51:00.586 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 196341 2023-09-23T20:51:00.586 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.586 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:51:00.586 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.586 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:51:00.586 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:26.070140+0000", 2023-09-23T20:51:00.586 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:51:00.586 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.586 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.586 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:51:00.587 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:26.070140+0000", 2023-09-23T20:51:00.587 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:51:00.587 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.587 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.587 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:51:00.587 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:26.070138+0000", 2023-09-23T20:51:00.587 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:51:00.587 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.587 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.587 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:51:00.587 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:26.070150+0000", 2023-09-23T20:51:00.587 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1579999999999999e-05 2023-09-23T20:51:00.588 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.588 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.588 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:51:00.588 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:26.070175+0000", 2023-09-23T20:51:00.588 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.5225e-05 2023-09-23T20:51:00.588 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.588 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.588 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:51:00.588 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:26.070187+0000", 2023-09-23T20:51:00.588 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2425e-05 2023-09-23T20:51:00.588 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.588 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.589 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:51:00.589 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:26.070266+0000", 2023-09-23T20:51:00.589 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.8677999999999994e-05 2023-09-23T20:51:00.589 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.589 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.589 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:51:00.589 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:26.070287+0000", 2023-09-23T20:51:00.589 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.088e-05 2023-09-23T20:51:00.589 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.589 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.589 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:51:00.589 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:27.052026+0000", 2023-09-23T20:51:00.590 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.98173956799999995 2023-09-23T20:51:00.590 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.590 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.590 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:51:00.590 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:27.052172+0000", 2023-09-23T20:51:00.590 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000145332 2023-09-23T20:51:00.590 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.590 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.590 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 0,1,2,5", 2023-09-23T20:51:00.590 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:27.052417+0000", 2023-09-23T20:51:00.590 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00024565699999999998 2023-09-23T20:51:00.591 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.591 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.591 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:51:00.591 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:27.058987+0000", 2023-09-23T20:51:00.591 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0065694560000000004 2023-09-23T20:51:00.591 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.591 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.591 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:00.591 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:27.059392+0000", 2023-09-23T20:51:00.591 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000405456 2023-09-23T20:51:00.591 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.591 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.592 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:00.592 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:27.059492+0000", 2023-09-23T20:51:00.592 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.9839000000000004e-05 2023-09-23T20:51:00.592 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.592 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.592 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:00.592 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:27.063566+0000", 2023-09-23T20:51:00.592 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0040737719999999998 2023-09-23T20:51:00.592 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.592 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.592 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:00.593 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:27.081141+0000", 2023-09-23T20:51:00.593 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.017575047 2023-09-23T20:51:00.593 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.593 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.593 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:51:00.593 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:27.081192+0000", 2023-09-23T20:51:00.593 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.0691e-05 2023-09-23T20:51:00.593 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.593 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.593 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:51:00.593 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:27.081325+0000", 2023-09-23T20:51:00.593 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00013365499999999999 2023-09-23T20:51:00.594 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:51:00.594 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:51:00.594 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:51:00.594 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.594 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.594 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:196417 10.15 10:a8714d9d:::benchmark_data_smithi018_422431_object91921:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e621)", 2023-09-23T20:51:00.594 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:49:26.148867+0000", 2023-09-23T20:51:00.594 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 94.412320989999998, 2023-09-23T20:51:00.594 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.932723317, 2023-09-23T20:51:00.594 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:51:00.594 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:00.595 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:51:00.595 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:51:00.595 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:51:00.595 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 196417 2023-09-23T20:51:00.595 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.595 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:51:00.595 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.595 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:51:00.595 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:26.148867+0000", 2023-09-23T20:51:00.595 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:51:00.595 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.595 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.596 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:51:00.596 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:26.148867+0000", 2023-09-23T20:51:00.596 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:51:00.596 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.596 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.596 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:51:00.596 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:26.148866+0000", 2023-09-23T20:51:00.596 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:51:00.596 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.596 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.596 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:51:00.596 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:26.148874+0000", 2023-09-23T20:51:00.597 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.5089999999999993e-06 2023-09-23T20:51:00.597 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.597 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.597 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:51:00.597 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:26.148894+0000", 2023-09-23T20:51:00.597 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.9800999999999999e-05 2023-09-23T20:51:00.597 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.597 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.597 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:51:00.597 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:26.148904+0000", 2023-09-23T20:51:00.597 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0054000000000001e-05 2023-09-23T20:51:00.598 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.598 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.598 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:51:00.598 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:26.148974+0000", 2023-09-23T20:51:00.598 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.0118999999999997e-05 2023-09-23T20:51:00.598 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.598 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.598 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:51:00.598 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:26.148991+0000", 2023-09-23T20:51:00.598 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.7356000000000001e-05 2023-09-23T20:51:00.598 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.598 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.599 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:51:00.599 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:27.054238+0000", 2023-09-23T20:51:00.599 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.90524600700000002 2023-09-23T20:51:00.599 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.599 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.599 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:51:00.599 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:27.054357+0000", 2023-09-23T20:51:00.599 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000119836 2023-09-23T20:51:00.599 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.599 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.599 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 0,1,2,5", 2023-09-23T20:51:00.599 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:27.054614+0000", 2023-09-23T20:51:00.600 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000256751 2023-09-23T20:51:00.600 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.600 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.600 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:00.600 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:27.059524+0000", 2023-09-23T20:51:00.600 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0049100619999999998 2023-09-23T20:51:00.600 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.600 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.600 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:00.600 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:27.072127+0000", 2023-09-23T20:51:00.600 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.012602755 2023-09-23T20:51:00.600 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.601 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.601 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:00.601 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:27.072170+0000", 2023-09-23T20:51:00.601 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.3590000000000001e-05 2023-09-23T20:51:00.601 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.601 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.601 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:51:00.601 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:27.080346+0000", 2023-09-23T20:51:00.601 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0081756910000000006 2023-09-23T20:51:00.601 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.601 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.602 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:00.602 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:27.081391+0000", 2023-09-23T20:51:00.602 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0010447480000000001 2023-09-23T20:51:00.602 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.602 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.602 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:51:00.602 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:27.081450+0000", 2023-09-23T20:51:00.602 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.8644000000000003e-05 2023-09-23T20:51:00.602 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.602 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.602 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:51:00.602 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:27.081591+0000", 2023-09-23T20:51:00.603 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00014109499999999999 2023-09-23T20:51:00.603 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:51:00.603 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:51:00.603 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:51:00.603 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.603 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.603 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:196425 10.15 10:acf26cec:::benchmark_data_smithi018_422431_object91929:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e621)", 2023-09-23T20:51:00.603 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:49:26.153612+0000", 2023-09-23T20:51:00.603 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 94.407576417000001, 2023-09-23T20:51:00.603 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.92827135599999999, 2023-09-23T20:51:00.603 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:51:00.604 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:00.604 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:51:00.604 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:51:00.604 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:51:00.604 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 196425 2023-09-23T20:51:00.604 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.604 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:51:00.604 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.604 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:51:00.604 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:26.153612+0000", 2023-09-23T20:51:00.604 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:51:00.604 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.605 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.605 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:51:00.605 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:26.153612+0000", 2023-09-23T20:51:00.605 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:51:00.605 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.605 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.605 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:51:00.605 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:26.153610+0000", 2023-09-23T20:51:00.605 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:51:00.605 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.605 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.606 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:51:00.606 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:26.153618+0000", 2023-09-23T20:51:00.606 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.6159999999999996e-06 2023-09-23T20:51:00.606 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.606 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.606 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:51:00.606 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:26.153634+0000", 2023-09-23T20:51:00.606 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.6303e-05 2023-09-23T20:51:00.606 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.606 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.606 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:51:00.606 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:26.153643+0000", 2023-09-23T20:51:00.607 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.4110000000000002e-06 2023-09-23T20:51:00.607 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.607 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.607 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:51:00.607 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:26.153710+0000", 2023-09-23T20:51:00.607 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.6433999999999994e-05 2023-09-23T20:51:00.607 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.607 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.607 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:51:00.607 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:26.153722+0000", 2023-09-23T20:51:00.607 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2241e-05 2023-09-23T20:51:00.607 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.607 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.610 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:51:00.610 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:27.056288+0000", 2023-09-23T20:51:00.610 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.902566019 2023-09-23T20:51:00.610 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.610 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.610 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:51:00.611 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:27.056427+0000", 2023-09-23T20:51:00.611 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00013860500000000001 2023-09-23T20:51:00.611 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.611 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.611 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 0,1,2,5", 2023-09-23T20:51:00.611 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:27.056678+0000", 2023-09-23T20:51:00.611 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00025188899999999999 2023-09-23T20:51:00.611 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.611 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.611 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:00.611 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:27.061753+0000", 2023-09-23T20:51:00.611 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0050748470000000004 2023-09-23T20:51:00.612 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.612 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.612 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:00.612 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:27.072212+0000", 2023-09-23T20:51:00.612 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.010458291999999999 2023-09-23T20:51:00.612 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.612 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.612 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:00.612 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:27.072252+0000", 2023-09-23T20:51:00.612 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.0497999999999997e-05 2023-09-23T20:51:00.612 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.612 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.613 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:51:00.613 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:27.080357+0000", 2023-09-23T20:51:00.613 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.008105335 2023-09-23T20:51:00.613 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.613 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.613 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:00.613 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:27.081651+0000", 2023-09-23T20:51:00.613 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0012933300000000001 2023-09-23T20:51:00.613 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.613 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.613 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:51:00.613 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:27.081707+0000", 2023-09-23T20:51:00.614 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.6242000000000001e-05 2023-09-23T20:51:00.614 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.614 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.614 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:51:00.614 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:27.081883+0000", 2023-09-23T20:51:00.614 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00017624900000000001 2023-09-23T20:51:00.614 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:51:00.614 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:51:00.614 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:51:00.614 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.614 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.615 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:197303 10.15 10:ad2208f7:::benchmark_data_smithi018_422431_object92807:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e624)", 2023-09-23T20:51:00.615 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:49:29.084317+0000", 2023-09-23T20:51:00.615 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 91.476871450999994, 2023-09-23T20:51:00.615 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.93850479099999995, 2023-09-23T20:51:00.615 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:51:00.615 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:00.615 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:51:00.615 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:51:00.615 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:51:00.615 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 197303 2023-09-23T20:51:00.615 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.615 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:51:00.616 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.616 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:51:00.616 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.084317+0000", 2023-09-23T20:51:00.616 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:51:00.616 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.616 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.616 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:51:00.616 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.084317+0000", 2023-09-23T20:51:00.616 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:51:00.616 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.616 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.617 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:51:00.617 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.084315+0000", 2023-09-23T20:51:00.617 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:51:00.617 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.617 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.617 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:51:00.617 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.084323+0000", 2023-09-23T20:51:00.617 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.5240000000000003e-06 2023-09-23T20:51:00.617 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.617 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.617 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:51:00.617 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.084342+0000", 2023-09-23T20:51:00.618 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.9239e-05 2023-09-23T20:51:00.618 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.618 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.618 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:51:00.618 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.084352+0000", 2023-09-23T20:51:00.618 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.4239999999999999e-06 2023-09-23T20:51:00.618 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.618 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.618 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:51:00.618 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.084421+0000", 2023-09-23T20:51:00.618 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.9256000000000002e-05 2023-09-23T20:51:00.618 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.619 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.619 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:51:00.619 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.084438+0000", 2023-09-23T20:51:00.619 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.7303999999999999e-05 2023-09-23T20:51:00.619 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.619 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.619 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:51:00.619 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:30.016878+0000", 2023-09-23T20:51:00.619 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.93243949299999995 2023-09-23T20:51:00.619 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.619 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.619 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:51:00.620 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:30.016977+0000", 2023-09-23T20:51:00.620 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.8869000000000005e-05 2023-09-23T20:51:00.620 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.620 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.620 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,5", 2023-09-23T20:51:00.620 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:30.017343+0000", 2023-09-23T20:51:00.620 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00036639099999999998 2023-09-23T20:51:00.620 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.620 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.620 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:51:00.620 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:30.022629+0000", 2023-09-23T20:51:00.620 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0052859459999999997 2023-09-23T20:51:00.621 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.621 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.621 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:00.621 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:30.022684+0000", 2023-09-23T20:51:00.621 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.4645999999999998e-05 2023-09-23T20:51:00.621 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.621 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.621 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:00.621 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:30.022722+0000", 2023-09-23T20:51:00.621 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.8067e-05 2023-09-23T20:51:00.621 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.621 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.622 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:51:00.622 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:30.022774+0000", 2023-09-23T20:51:00.622 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.2099999999999999e-05 2023-09-23T20:51:00.622 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.622 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.622 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:51:00.622 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:30.022822+0000", 2023-09-23T20:51:00.622 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.7969000000000001e-05 2023-09-23T20:51:00.622 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:51:00.622 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:51:00.623 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:51:00.623 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.623 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.623 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:197309 10.15 10:ab14b822:::benchmark_data_smithi018_422431_object92813:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e624)", 2023-09-23T20:51:00.623 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:49:29.090092+0000", 2023-09-23T20:51:00.623 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 91.471096098999993, 2023-09-23T20:51:00.623 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.93297547000000003, 2023-09-23T20:51:00.623 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:51:00.623 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:00.623 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:51:00.623 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:51:00.624 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:51:00.624 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 197309 2023-09-23T20:51:00.624 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.624 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:51:00.624 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.624 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:51:00.624 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.090092+0000", 2023-09-23T20:51:00.624 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:51:00.624 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.624 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.624 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:51:00.624 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.090092+0000", 2023-09-23T20:51:00.625 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:51:00.625 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.625 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.625 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:51:00.625 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.090091+0000", 2023-09-23T20:51:00.625 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:51:00.625 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.625 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.625 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:51:00.625 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.090097+0000", 2023-09-23T20:51:00.625 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.7050000000000002e-06 2023-09-23T20:51:00.625 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.626 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.626 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:51:00.626 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.090111+0000", 2023-09-23T20:51:00.626 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3521e-05 2023-09-23T20:51:00.626 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.626 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.626 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:51:00.626 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.090118+0000", 2023-09-23T20:51:00.626 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.6370000000000002e-06 2023-09-23T20:51:00.626 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.626 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.627 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:51:00.627 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.090180+0000", 2023-09-23T20:51:00.627 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.2533000000000006e-05 2023-09-23T20:51:00.627 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.627 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.627 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:51:00.627 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.090192+0000", 2023-09-23T20:51:00.627 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2279999999999999e-05 2023-09-23T20:51:00.627 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.627 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.627 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:51:00.628 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:30.019150+0000", 2023-09-23T20:51:00.628 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.92895706899999997 2023-09-23T20:51:00.628 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.628 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.628 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:51:00.628 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:30.019290+0000", 2023-09-23T20:51:00.628 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000140321 2023-09-23T20:51:00.628 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.628 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.628 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 1,5", 2023-09-23T20:51:00.628 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:30.019545+0000", 2023-09-23T20:51:00.628 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000255157 2023-09-23T20:51:00.629 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.629 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.629 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:00.629 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:30.022848+0000", 2023-09-23T20:51:00.629 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0033031919999999999 2023-09-23T20:51:00.629 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.629 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.629 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:51:00.629 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:30.022955+0000", 2023-09-23T20:51:00.629 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000107209 2023-09-23T20:51:00.629 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.629 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.630 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:00.630 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:30.022985+0000", 2023-09-23T20:51:00.630 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.9998999999999999e-05 2023-09-23T20:51:00.630 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.630 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.630 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:51:00.630 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:30.023030+0000", 2023-09-23T20:51:00.630 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.5067000000000001e-05 2023-09-23T20:51:00.630 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.630 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.630 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:51:00.631 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:30.023068+0000", 2023-09-23T20:51:00.631 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.7259999999999999e-05 2023-09-23T20:51:00.631 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:51:00.631 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:51:00.631 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:51:00.631 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.631 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.631 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:197315 10.17 10:e8e8864f:::benchmark_data_smithi018_422431_object92819:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e625)", 2023-09-23T20:51:00.631 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:49:29.986565+0000", 2023-09-23T20:51:00.631 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 90.574623840000001, 2023-09-23T20:51:00.631 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1168361760000001, 2023-09-23T20:51:00.632 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:51:00.632 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:00.632 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:51:00.632 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:51:00.632 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:51:00.632 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 197315 2023-09-23T20:51:00.632 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.632 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:51:00.632 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.632 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:51:00.632 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986565+0000", 2023-09-23T20:51:00.632 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:51:00.633 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.633 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.633 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:51:00.633 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986565+0000", 2023-09-23T20:51:00.633 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:51:00.633 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.633 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.633 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:51:00.633 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986563+0000", 2023-09-23T20:51:00.633 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:51:00.633 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.634 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.634 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:51:00.634 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986572+0000", 2023-09-23T20:51:00.634 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.7299999999999994e-06 2023-09-23T20:51:00.634 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.634 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.634 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:51:00.634 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986588+0000", 2023-09-23T20:51:00.634 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.6004000000000001e-05 2023-09-23T20:51:00.635 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.635 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.635 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:51:00.635 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986597+0000", 2023-09-23T20:51:00.635 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.5820000000000008e-06 2023-09-23T20:51:00.635 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.635 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.635 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:51:00.635 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986662+0000", 2023-09-23T20:51:00.635 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.4540000000000002e-05 2023-09-23T20:51:00.635 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.635 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.636 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:51:00.636 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986675+0000", 2023-09-23T20:51:00.636 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.312e-05 2023-09-23T20:51:00.636 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.636 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.636 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:51:00.636 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.095946+0000", 2023-09-23T20:51:00.636 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.109270993 2023-09-23T20:51:00.636 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.636 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.636 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:51:00.637 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.096119+0000", 2023-09-23T20:51:00.637 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00017305700000000001 2023-09-23T20:51:00.637 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.637 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.637 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 4,5,6", 2023-09-23T20:51:00.637 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.096543+0000", 2023-09-23T20:51:00.637 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00042400599999999999 2023-09-23T20:51:00.637 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.637 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.637 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:00.637 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.103154+0000", 2023-09-23T20:51:00.637 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0066110129999999998 2023-09-23T20:51:00.638 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.638 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.638 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:51:00.638 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.103170+0000", 2023-09-23T20:51:00.638 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.5627000000000001e-05 2023-09-23T20:51:00.638 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.638 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.638 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:00.638 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.103233+0000", 2023-09-23T20:51:00.638 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.3027999999999994e-05 2023-09-23T20:51:00.638 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.639 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.639 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:00.639 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.103261+0000", 2023-09-23T20:51:00.639 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.8697999999999999e-05 2023-09-23T20:51:00.639 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.639 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.639 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:51:00.639 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.103298+0000", 2023-09-23T20:51:00.639 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.6585000000000003e-05 2023-09-23T20:51:00.639 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.639 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.639 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:51:00.640 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.103401+0000", 2023-09-23T20:51:00.640 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000102747 2023-09-23T20:51:00.640 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:51:00.640 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:51:00.640 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:51:00.640 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.640 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.640 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:197329 10.17 10:ee8e2f7f:::benchmark_data_smithi018_422431_object92833:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e625)", 2023-09-23T20:51:00.640 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:49:29.986611+0000", 2023-09-23T20:51:00.640 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 90.574577418999993, 2023-09-23T20:51:00.640 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.121231686, 2023-09-23T20:51:00.641 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:51:00.641 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:00.641 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:51:00.641 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:51:00.641 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:51:00.641 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 197329 2023-09-23T20:51:00.641 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.641 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:51:00.641 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.641 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:51:00.641 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986611+0000", 2023-09-23T20:51:00.642 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:51:00.642 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.642 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.642 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:51:00.642 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986611+0000", 2023-09-23T20:51:00.642 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:51:00.642 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.642 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.642 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:51:00.642 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986611+0000", 2023-09-23T20:51:00.642 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:51:00.642 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.643 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.643 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:51:00.643 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986613+0000", 2023-09-23T20:51:00.643 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.148e-06 2023-09-23T20:51:00.643 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.643 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.643 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:51:00.643 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986617+0000", 2023-09-23T20:51:00.643 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.2440000000000004e-06 2023-09-23T20:51:00.643 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.643 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.643 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:51:00.644 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986620+0000", 2023-09-23T20:51:00.644 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.0769999999999999e-06 2023-09-23T20:51:00.644 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.644 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.644 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:51:00.644 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986722+0000", 2023-09-23T20:51:00.644 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000102088 2023-09-23T20:51:00.644 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.644 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.644 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:51:00.644 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986733+0000", 2023-09-23T20:51:00.644 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0550000000000001e-05 2023-09-23T20:51:00.645 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.645 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.645 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:51:00.645 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.099437+0000", 2023-09-23T20:51:00.645 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1127045760000001 2023-09-23T20:51:00.645 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.645 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.645 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:51:00.645 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.099590+0000", 2023-09-23T20:51:00.645 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00015262000000000001 2023-09-23T20:51:00.645 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.646 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.646 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 4,5,6", 2023-09-23T20:51:00.646 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.099971+0000", 2023-09-23T20:51:00.646 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00038124200000000001 2023-09-23T20:51:00.646 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.646 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.646 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:51:00.646 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.103420+0000", 2023-09-23T20:51:00.646 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0034486019999999998 2023-09-23T20:51:00.646 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.646 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.647 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:00.647 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.103488+0000", 2023-09-23T20:51:00.647 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.8538e-05 2023-09-23T20:51:00.647 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.647 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.647 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:00.647 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.103781+0000", 2023-09-23T20:51:00.647 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00029315199999999997 2023-09-23T20:51:00.647 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.647 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.647 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:00.647 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.107701+0000", 2023-09-23T20:51:00.648 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.003919166 2023-09-23T20:51:00.648 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.648 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.648 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:51:00.648 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.107733+0000", 2023-09-23T20:51:00.648 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.2780000000000001e-05 2023-09-23T20:51:00.648 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.648 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.648 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:51:00.648 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.107843+0000", 2023-09-23T20:51:00.648 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000109232 2023-09-23T20:51:00.648 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:51:00.649 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:51:00.649 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:51:00.649 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.649 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.649 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:197351 10.17 10:ec920b81:::benchmark_data_smithi018_422431_object92855:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e625)", 2023-09-23T20:51:00.649 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:49:29.986649+0000", 2023-09-23T20:51:00.649 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 90.574539465000001, 2023-09-23T20:51:00.649 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1235907970000001, 2023-09-23T20:51:00.649 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:51:00.649 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:00.649 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:51:00.649 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:51:00.650 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:51:00.650 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 197351 2023-09-23T20:51:00.650 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.650 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:51:00.650 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.650 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:51:00.650 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986649+0000", 2023-09-23T20:51:00.650 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:51:00.650 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.650 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.650 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:51:00.651 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986649+0000", 2023-09-23T20:51:00.651 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:51:00.651 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.651 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.651 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:51:00.651 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986649+0000", 2023-09-23T20:51:00.651 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:51:00.651 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.651 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.651 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:51:00.651 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986651+0000", 2023-09-23T20:51:00.651 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.9800000000000001e-06 2023-09-23T20:51:00.651 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.652 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.652 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:51:00.652 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986656+0000", 2023-09-23T20:51:00.652 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.5249999999999996e-06 2023-09-23T20:51:00.652 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.652 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.652 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:51:00.652 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986659+0000", 2023-09-23T20:51:00.652 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.0570000000000001e-06 2023-09-23T20:51:00.652 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.652 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.653 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:51:00.653 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986776+0000", 2023-09-23T20:51:00.653 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000116338 2023-09-23T20:51:00.653 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.653 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.653 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:51:00.653 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986784+0000", 2023-09-23T20:51:00.653 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.0569999999999993e-06 2023-09-23T20:51:00.653 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.653 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.653 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:51:00.653 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.101416+0000", 2023-09-23T20:51:00.654 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.114632863 2023-09-23T20:51:00.654 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.654 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.654 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:51:00.654 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.101529+0000", 2023-09-23T20:51:00.654 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000112812 2023-09-23T20:51:00.654 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.654 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.654 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 4,5,6", 2023-09-23T20:51:00.654 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.101718+0000", 2023-09-23T20:51:00.655 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00018869399999999999 2023-09-23T20:51:00.655 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.655 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.655 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:51:00.655 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.103700+0000", 2023-09-23T20:51:00.655 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.001982466 2023-09-23T20:51:00.655 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.655 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.655 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:00.655 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.103742+0000", 2023-09-23T20:51:00.655 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.1378999999999998e-05 2023-09-23T20:51:00.655 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.656 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.656 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:00.656 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.103841+0000", 2023-09-23T20:51:00.656 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.9098000000000006e-05 2023-09-23T20:51:00.656 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.656 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.656 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:00.656 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.110043+0000", 2023-09-23T20:51:00.656 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0062020349999999998 2023-09-23T20:51:00.656 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.656 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.657 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:51:00.657 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.110104+0000", 2023-09-23T20:51:00.657 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.1477999999999997e-05 2023-09-23T20:51:00.657 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.657 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.657 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:51:00.657 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.110240+0000", 2023-09-23T20:51:00.657 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00013529699999999999 2023-09-23T20:51:00.657 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:51:00.657 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:51:00.657 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:51:00.658 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.658 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.658 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:197367 10.17 10:ebdac9cc:::benchmark_data_smithi018_422431_object92871:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e625)", 2023-09-23T20:51:00.658 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:49:29.986671+0000", 2023-09-23T20:51:00.658 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 90.574517569999998, 2023-09-23T20:51:00.658 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.123815188, 2023-09-23T20:51:00.658 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:51:00.658 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:00.658 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:51:00.658 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:51:00.658 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:51:00.659 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 197367 2023-09-23T20:51:00.659 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.659 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:51:00.659 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.659 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:51:00.659 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986671+0000", 2023-09-23T20:51:00.659 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:51:00.659 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.659 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.659 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:51:00.659 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986671+0000", 2023-09-23T20:51:00.659 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:51:00.660 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.660 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.660 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:51:00.660 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986671+0000", 2023-09-23T20:51:00.660 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967296 2023-09-23T20:51:00.660 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.660 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.660 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:51:00.660 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986672+0000", 2023-09-23T20:51:00.660 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.781e-06 2023-09-23T20:51:00.660 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.660 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.661 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:51:00.661 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986676+0000", 2023-09-23T20:51:00.661 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.4850000000000001e-06 2023-09-23T20:51:00.661 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.661 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.661 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:51:00.661 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986679+0000", 2023-09-23T20:51:00.661 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.7209999999999999e-06 2023-09-23T20:51:00.661 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.661 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.661 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:51:00.662 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986843+0000", 2023-09-23T20:51:00.662 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00016420599999999999 2023-09-23T20:51:00.662 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.662 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.662 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:51:00.662 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986854+0000", 2023-09-23T20:51:00.662 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1246e-05 2023-09-23T20:51:00.662 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.662 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.662 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:51:00.662 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.103866+0000", 2023-09-23T20:51:00.663 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.117011822 2023-09-23T20:51:00.663 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.663 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.663 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:51:00.663 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.103986+0000", 2023-09-23T20:51:00.663 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000119688 2023-09-23T20:51:00.663 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.663 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.663 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 4,5,6", 2023-09-23T20:51:00.663 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.104176+0000", 2023-09-23T20:51:00.663 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00019036199999999999 2023-09-23T20:51:00.663 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.664 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.664 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:00.664 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.107658+0000", 2023-09-23T20:51:00.664 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0034818079999999999 2023-09-23T20:51:00.664 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.664 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.664 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:51:00.664 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.107856+0000", 2023-09-23T20:51:00.664 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000197923 2023-09-23T20:51:00.664 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.664 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.664 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:00.665 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.109966+0000", 2023-09-23T20:51:00.665 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0021102199999999999 2023-09-23T20:51:00.665 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.665 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.665 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:00.665 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.110317+0000", 2023-09-23T20:51:00.665 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00035148699999999999 2023-09-23T20:51:00.665 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.665 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.665 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:51:00.665 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.110374+0000", 2023-09-23T20:51:00.666 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.7108999999999998e-05 2023-09-23T20:51:00.666 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.666 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.666 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:51:00.666 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.110486+0000", 2023-09-23T20:51:00.666 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000111556 2023-09-23T20:51:00.666 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:51:00.666 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:51:00.666 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:51:00.666 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.666 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.666 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:197369 10.17 10:edb5dbb2:::benchmark_data_smithi018_422431_object92873:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e625)", 2023-09-23T20:51:00.667 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:49:29.986681+0000", 2023-09-23T20:51:00.667 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 90.574507460999996, 2023-09-23T20:51:00.667 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1263334039999999, 2023-09-23T20:51:00.667 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:51:00.667 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:00.667 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:51:00.667 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:51:00.667 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:51:00.667 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 197369 2023-09-23T20:51:00.667 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.667 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:51:00.668 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.668 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:51:00.668 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986681+0000", 2023-09-23T20:51:00.668 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:51:00.668 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.668 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.668 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:51:00.668 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986681+0000", 2023-09-23T20:51:00.668 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:51:00.668 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.668 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.668 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:51:00.668 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986681+0000", 2023-09-23T20:51:00.669 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967296 2023-09-23T20:51:00.669 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.669 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.669 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:51:00.669 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986682+0000", 2023-09-23T20:51:00.669 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.4470000000000001e-06 2023-09-23T20:51:00.669 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.669 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.669 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:51:00.669 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986685+0000", 2023-09-23T20:51:00.669 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.0369999999999999e-06 2023-09-23T20:51:00.670 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.670 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.670 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:51:00.670 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986688+0000", 2023-09-23T20:51:00.670 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.723e-06 2023-09-23T20:51:00.670 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.670 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.670 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:51:00.670 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986899+0000", 2023-09-23T20:51:00.670 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00021081999999999999 2023-09-23T20:51:00.670 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.670 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.671 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:51:00.671 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986907+0000", 2023-09-23T20:51:00.671 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.3550000000000005e-06 2023-09-23T20:51:00.671 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.671 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.671 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:51:00.671 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.105384+0000", 2023-09-23T20:51:00.671 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1184772569999999 2023-09-23T20:51:00.671 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.671 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.671 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:51:00.671 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.105548+0000", 2023-09-23T20:51:00.672 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000163272 2023-09-23T20:51:00.672 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.672 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.672 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 4,5,6", 2023-09-23T20:51:00.672 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.105856+0000", 2023-09-23T20:51:00.672 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00030820400000000002 2023-09-23T20:51:00.672 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.672 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.672 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:51:00.672 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.107861+0000", 2023-09-23T20:51:00.672 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0020048219999999999 2023-09-23T20:51:00.672 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.672 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.673 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:00.673 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.109908+0000", 2023-09-23T20:51:00.673 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0020470589999999999 2023-09-23T20:51:00.673 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.673 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.673 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:00.673 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.109994+0000", 2023-09-23T20:51:00.673 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.6201999999999999e-05 2023-09-23T20:51:00.673 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.673 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.673 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:00.673 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.112824+0000", 2023-09-23T20:51:00.674 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0028306220000000001 2023-09-23T20:51:00.674 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.674 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.674 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:51:00.674 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.112891+0000", 2023-09-23T20:51:00.674 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.6707999999999999e-05 2023-09-23T20:51:00.674 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.674 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.674 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:51:00.674 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.113014+0000", 2023-09-23T20:51:00.674 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00012308999999999999 2023-09-23T20:51:00.674 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:51:00.675 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:51:00.675 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:51:00.675 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.675 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.675 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:197390 10.17 10:eece82d4:::benchmark_data_smithi018_422431_object92894:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e625)", 2023-09-23T20:51:00.675 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:49:29.986711+0000", 2023-09-23T20:51:00.675 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 90.574477039000001, 2023-09-23T20:51:00.675 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1286582460000001, 2023-09-23T20:51:00.675 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:51:00.675 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:00.675 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:51:00.676 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:51:00.676 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:51:00.676 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 197390 2023-09-23T20:51:00.676 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.676 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:51:00.676 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.676 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:51:00.676 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986711+0000", 2023-09-23T20:51:00.676 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:51:00.676 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.676 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.676 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:51:00.677 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986711+0000", 2023-09-23T20:51:00.677 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:51:00.677 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.677 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.677 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:51:00.677 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986711+0000", 2023-09-23T20:51:00.677 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967296 2023-09-23T20:51:00.677 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.677 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.677 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:51:00.677 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986714+0000", 2023-09-23T20:51:00.677 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.61e-06 2023-09-23T20:51:00.678 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.678 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.678 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:51:00.678 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986719+0000", 2023-09-23T20:51:00.678 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.271e-06 2023-09-23T20:51:00.678 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.678 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.678 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:51:00.678 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986722+0000", 2023-09-23T20:51:00.678 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.3239999999999999e-06 2023-09-23T20:51:00.678 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.679 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.679 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:51:00.679 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986950+0000", 2023-09-23T20:51:00.679 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00022735299999999999 2023-09-23T20:51:00.679 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.679 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.679 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:51:00.679 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986960+0000", 2023-09-23T20:51:00.679 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0332e-05 2023-09-23T20:51:00.679 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.679 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.679 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:51:00.679 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.107897+0000", 2023-09-23T20:51:00.680 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.120937311 2023-09-23T20:51:00.680 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.680 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.680 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:51:00.680 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.108040+0000", 2023-09-23T20:51:00.680 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00014265299999999999 2023-09-23T20:51:00.680 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.680 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.680 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 4,5,6", 2023-09-23T20:51:00.680 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.108307+0000", 2023-09-23T20:51:00.680 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00026726400000000002 2023-09-23T20:51:00.681 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.681 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.681 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:51:00.681 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.110505+0000", 2023-09-23T20:51:00.681 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0021981000000000001 2023-09-23T20:51:00.681 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.681 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.681 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:00.681 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.112767+0000", 2023-09-23T20:51:00.681 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.002261987 2023-09-23T20:51:00.681 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.681 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.682 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:00.682 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.115153+0000", 2023-09-23T20:51:00.682 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.002385318 2023-09-23T20:51:00.682 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.682 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.682 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:00.682 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.115189+0000", 2023-09-23T20:51:00.682 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.6455000000000002e-05 2023-09-23T20:51:00.682 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.682 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.682 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:51:00.683 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.115246+0000", 2023-09-23T20:51:00.683 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.6957999999999999e-05 2023-09-23T20:51:00.683 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.683 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.683 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:51:00.683 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.115370+0000", 2023-09-23T20:51:00.683 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00012352199999999999 2023-09-23T20:51:00.683 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:51:00.683 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:51:00.683 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:51:00.683 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.683 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.683 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:197413 10.17 10:eeb3ebd7:::benchmark_data_smithi018_422431_object92917:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e625)", 2023-09-23T20:51:00.684 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:49:29.986725+0000", 2023-09-23T20:51:00.684 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 90.574463210999994, 2023-09-23T20:51:00.684 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1335479159999999, 2023-09-23T20:51:00.684 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:51:00.684 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:00.684 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:51:00.684 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:51:00.684 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:51:00.684 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 197413 2023-09-23T20:51:00.684 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.684 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:51:00.685 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.685 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:51:00.685 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986725+0000", 2023-09-23T20:51:00.685 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:51:00.685 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.685 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.685 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:51:00.685 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986725+0000", 2023-09-23T20:51:00.685 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:51:00.685 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.685 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.685 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:51:00.686 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986725+0000", 2023-09-23T20:51:00.686 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967296 2023-09-23T20:51:00.686 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.686 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.686 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:51:00.686 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986727+0000", 2023-09-23T20:51:00.686 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.6160000000000001e-06 2023-09-23T20:51:00.686 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.686 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.686 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:51:00.686 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986730+0000", 2023-09-23T20:51:00.687 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.9900000000000002e-06 2023-09-23T20:51:00.687 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.687 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.687 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:51:00.687 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986732+0000", 2023-09-23T20:51:00.687 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.689e-06 2023-09-23T20:51:00.687 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.687 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.687 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:51:00.687 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.987000+0000", 2023-09-23T20:51:00.687 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00026816200000000001 2023-09-23T20:51:00.687 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.688 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.688 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:51:00.688 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.987010+0000", 2023-09-23T20:51:00.688 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.1239999999999994e-06 2023-09-23T20:51:00.688 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.688 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.688 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:51:00.688 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.110556+0000", 2023-09-23T20:51:00.688 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.123546964 2023-09-23T20:51:00.688 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.688 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.688 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:51:00.689 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.110727+0000", 2023-09-23T20:51:00.689 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00017081799999999999 2023-09-23T20:51:00.689 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.689 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.689 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 4,5,6", 2023-09-23T20:51:00.689 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.111048+0000", 2023-09-23T20:51:00.689 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00032095000000000002 2023-09-23T20:51:00.689 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.689 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.689 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:51:00.689 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.115106+0000", 2023-09-23T20:51:00.690 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0040579819999999999 2023-09-23T20:51:00.690 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.690 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.690 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:00.690 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.115413+0000", 2023-09-23T20:51:00.690 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00030677100000000001 2023-09-23T20:51:00.690 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.690 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.690 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:00.690 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.115448+0000", 2023-09-23T20:51:00.690 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.4625000000000001e-05 2023-09-23T20:51:00.690 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.691 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.691 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:00.691 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.120101+0000", 2023-09-23T20:51:00.691 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0046538680000000002 2023-09-23T20:51:00.691 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.691 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.691 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:51:00.691 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.120156+0000", 2023-09-23T20:51:00.691 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.4237999999999998e-05 2023-09-23T20:51:00.691 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.691 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.692 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:51:00.692 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.120273+0000", 2023-09-23T20:51:00.692 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000117324 2023-09-23T20:51:00.692 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:51:00.692 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:51:00.692 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:51:00.692 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.692 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.692 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:197441 10.17 10:ef3e749e:::benchmark_data_smithi018_422431_object92945:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e625)", 2023-09-23T20:51:00.692 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:49:29.986735+0000", 2023-09-23T20:51:00.692 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 90.574453810999998, 2023-09-23T20:51:00.692 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.135887265, 2023-09-23T20:51:00.693 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:51:00.693 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:00.693 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:51:00.693 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:51:00.693 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:51:00.693 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 197441 2023-09-23T20:51:00.693 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.693 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:51:00.693 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.693 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:51:00.693 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986735+0000", 2023-09-23T20:51:00.693 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:51:00.694 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.694 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.694 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:51:00.694 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986735+0000", 2023-09-23T20:51:00.694 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:51:00.694 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.694 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.694 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:51:00.694 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986734+0000", 2023-09-23T20:51:00.694 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967296 2023-09-23T20:51:00.694 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.695 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.695 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:51:00.695 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986736+0000", 2023-09-23T20:51:00.695 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.471e-06 2023-09-23T20:51:00.695 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.695 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.695 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:51:00.695 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986739+0000", 2023-09-23T20:51:00.695 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.7989999999999998e-06 2023-09-23T20:51:00.695 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.695 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.695 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:51:00.696 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986741+0000", 2023-09-23T20:51:00.696 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.785e-06 2023-09-23T20:51:00.696 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.696 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.696 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:51:00.696 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.987050+0000", 2023-09-23T20:51:00.696 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00030879399999999998 2023-09-23T20:51:00.696 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.696 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.696 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:51:00.696 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.987058+0000", 2023-09-23T20:51:00.696 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.5279999999999998e-06 2023-09-23T20:51:00.697 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.697 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.697 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:51:00.697 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.115473+0000", 2023-09-23T20:51:00.697 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.128414926 2023-09-23T20:51:00.697 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.697 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.697 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:51:00.697 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.115598+0000", 2023-09-23T20:51:00.697 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00012549999999999999 2023-09-23T20:51:00.697 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.698 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.698 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 4,5,6", 2023-09-23T20:51:00.698 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.115970+0000", 2023-09-23T20:51:00.698 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00037187500000000003 2023-09-23T20:51:00.698 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.698 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.698 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:51:00.698 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.120285+0000", 2023-09-23T20:51:00.698 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0043145809999999996 2023-09-23T20:51:00.698 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.698 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.699 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:00.699 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.120321+0000", 2023-09-23T20:51:00.699 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.6891000000000003e-05 2023-09-23T20:51:00.699 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.699 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.699 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:00.699 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.122405+0000", 2023-09-23T20:51:00.699 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0020831980000000001 2023-09-23T20:51:00.699 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.699 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.699 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:00.699 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.122441+0000", 2023-09-23T20:51:00.700 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.6417e-05 2023-09-23T20:51:00.700 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.700 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.700 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:51:00.700 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.122501+0000", 2023-09-23T20:51:00.700 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.0324e-05 2023-09-23T20:51:00.700 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.700 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.700 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:51:00.700 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.122622+0000", 2023-09-23T20:51:00.700 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000120376 2023-09-23T20:51:00.700 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:51:00.701 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:51:00.701 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:51:00.701 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.701 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.701 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:197474 10.17 10:e88d00d1:::benchmark_data_smithi018_422431_object92978:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e625)", 2023-09-23T20:51:00.701 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:49:29.986768+0000", 2023-09-23T20:51:00.701 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 90.574420711000002, 2023-09-23T20:51:00.701 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1406286880000001, 2023-09-23T20:51:00.701 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:51:00.701 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:00.701 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:51:00.702 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:51:00.702 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:51:00.702 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 197474 2023-09-23T20:51:00.702 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.702 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:51:00.702 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.702 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:51:00.702 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986768+0000", 2023-09-23T20:51:00.702 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:51:00.702 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.702 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.702 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:51:00.703 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986768+0000", 2023-09-23T20:51:00.703 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:51:00.703 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.703 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.703 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:51:00.703 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986767+0000", 2023-09-23T20:51:00.703 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967296 2023-09-23T20:51:00.703 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.703 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.703 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:51:00.703 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986769+0000", 2023-09-23T20:51:00.703 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.4699999999999999e-06 2023-09-23T20:51:00.704 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.704 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.704 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:51:00.704 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986774+0000", 2023-09-23T20:51:00.704 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.4829999999999999e-06 2023-09-23T20:51:00.704 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.704 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.704 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:51:00.704 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.986777+0000", 2023-09-23T20:51:00.704 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.0819999999999999e-06 2023-09-23T20:51:00.704 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.704 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.705 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:51:00.705 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.987089+0000", 2023-09-23T20:51:00.705 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00031171900000000002 2023-09-23T20:51:00.705 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.705 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.705 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:51:00.705 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:29.987098+0000", 2023-09-23T20:51:00.705 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.6750000000000008e-06 2023-09-23T20:51:00.705 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.705 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.705 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:51:00.706 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.117835+0000", 2023-09-23T20:51:00.706 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1307371669999999 2023-09-23T20:51:00.706 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.706 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.706 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:51:00.706 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.117952+0000", 2023-09-23T20:51:00.706 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00011728800000000001 2023-09-23T20:51:00.706 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.706 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.706 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 4,5,6", 2023-09-23T20:51:00.706 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.118255+0000", 2023-09-23T20:51:00.707 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00030264199999999999 2023-09-23T20:51:00.707 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.707 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.707 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:00.707 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.122668+0000", 2023-09-23T20:51:00.707 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.004413045 2023-09-23T20:51:00.707 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.707 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.707 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:00.707 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.122705+0000", 2023-09-23T20:51:00.707 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.6677999999999997e-05 2023-09-23T20:51:00.707 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.708 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.708 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:00.708 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.127195+0000", 2023-09-23T20:51:00.708 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0044907059999999997 2023-09-23T20:51:00.708 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.708 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.708 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:51:00.708 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.127204+0000", 2023-09-23T20:51:00.708 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.4989999999999994e-06 2023-09-23T20:51:00.708 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.708 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.708 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:51:00.708 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.127258+0000", 2023-09-23T20:51:00.709 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.4271000000000003e-05 2023-09-23T20:51:00.709 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.709 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.709 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:51:00.709 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.127396+0000", 2023-09-23T20:51:00.709 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00013817500000000001 2023-09-23T20:51:00.709 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:51:00.709 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:51:00.709 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:51:00.709 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.709 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.709 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:197499 10.17 10:e84c8c26:::benchmark_data_smithi018_422431_object93003:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e625)", 2023-09-23T20:51:00.710 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:49:30.051645+0000", 2023-09-23T20:51:00.710 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 90.509543797000006, 2023-09-23T20:51:00.710 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0808529579999999, 2023-09-23T20:51:00.710 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:51:00.710 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:00.710 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:51:00.710 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:51:00.710 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:51:00.710 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 197499 2023-09-23T20:51:00.710 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.710 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:51:00.710 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.711 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:51:00.711 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:30.051645+0000", 2023-09-23T20:51:00.711 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:51:00.711 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.711 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.711 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:51:00.711 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:30.051645+0000", 2023-09-23T20:51:00.711 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:51:00.711 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.711 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.711 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:51:00.711 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:30.051643+0000", 2023-09-23T20:51:00.712 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:51:00.712 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.712 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.712 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:51:00.712 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:30.051653+0000", 2023-09-23T20:51:00.712 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.9990000000000003e-06 2023-09-23T20:51:00.712 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.712 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.712 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:51:00.712 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:30.051677+0000", 2023-09-23T20:51:00.712 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.4777e-05 2023-09-23T20:51:00.712 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.713 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.713 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:51:00.713 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:30.051688+0000", 2023-09-23T20:51:00.713 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0662999999999999e-05 2023-09-23T20:51:00.713 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.713 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.713 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:51:00.713 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:30.051753+0000", 2023-09-23T20:51:00.713 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.4968000000000005e-05 2023-09-23T20:51:00.713 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.713 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.714 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:51:00.714 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:30.051767+0000", 2023-09-23T20:51:00.714 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.4378e-05 2023-09-23T20:51:00.714 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.714 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.714 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:51:00.714 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.122728+0000", 2023-09-23T20:51:00.714 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0709603510000001 2023-09-23T20:51:00.714 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.714 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.714 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:51:00.714 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.122885+0000", 2023-09-23T20:51:00.715 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00015751900000000001 2023-09-23T20:51:00.715 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.715 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.715 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 4,5,6", 2023-09-23T20:51:00.715 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.123294+0000", 2023-09-23T20:51:00.715 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000408537 2023-09-23T20:51:00.715 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.715 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.715 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:51:00.715 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.127454+0000", 2023-09-23T20:51:00.715 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0041605970000000003 2023-09-23T20:51:00.716 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.716 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.716 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:00.716 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.127507+0000", 2023-09-23T20:51:00.716 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.2753999999999997e-05 2023-09-23T20:51:00.716 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.716 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.716 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:00.716 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.127569+0000", 2023-09-23T20:51:00.716 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.1624000000000005e-05 2023-09-23T20:51:00.716 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.716 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.716 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:00.717 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.132306+0000", 2023-09-23T20:51:00.717 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0047368610000000002 2023-09-23T20:51:00.717 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.717 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.717 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:51:00.717 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.132373+0000", 2023-09-23T20:51:00.717 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.6983999999999994e-05 2023-09-23T20:51:00.717 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.717 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.717 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:51:00.717 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.132498+0000", 2023-09-23T20:51:00.717 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00012484400000000001 2023-09-23T20:51:00.718 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:51:00.718 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:51:00.718 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:51:00.718 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.718 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.718 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:197519 10.17 10:e96e5c55:::benchmark_data_smithi018_422431_object93023:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e625)", 2023-09-23T20:51:00.718 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:49:30.075323+0000", 2023-09-23T20:51:00.718 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 90.485865528999994, 2023-09-23T20:51:00.718 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.059203133, 2023-09-23T20:51:00.718 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:51:00.718 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:00.718 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:51:00.719 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:51:00.719 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:51:00.719 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 197519 2023-09-23T20:51:00.719 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.719 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:51:00.719 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.719 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:51:00.719 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:30.075323+0000", 2023-09-23T20:51:00.719 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:51:00.719 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.719 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.719 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:51:00.720 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:30.075323+0000", 2023-09-23T20:51:00.720 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:51:00.720 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.720 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.720 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:51:00.720 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:30.075322+0000", 2023-09-23T20:51:00.720 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:51:00.720 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.720 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.720 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:51:00.720 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:30.075328+0000", 2023-09-23T20:51:00.720 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.8800000000000002e-06 2023-09-23T20:51:00.721 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.721 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.721 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:51:00.721 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:30.075344+0000", 2023-09-23T20:51:00.721 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.5234e-05 2023-09-23T20:51:00.721 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.721 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.721 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:51:00.721 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:30.075354+0000", 2023-09-23T20:51:00.721 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.8989999999999996e-06 2023-09-23T20:51:00.721 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.721 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.722 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:51:00.722 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:30.076863+0000", 2023-09-23T20:51:00.722 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.001509385 2023-09-23T20:51:00.722 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.722 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.722 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:51:00.722 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:30.076879+0000", 2023-09-23T20:51:00.722 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.6299000000000001e-05 2023-09-23T20:51:00.722 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.722 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.722 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:51:00.723 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.125198+0000", 2023-09-23T20:51:00.723 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0483189749999999 2023-09-23T20:51:00.723 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.723 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.723 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:51:00.723 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.125315+0000", 2023-09-23T20:51:00.723 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000116768 2023-09-23T20:51:00.723 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.723 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.723 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 4,5,6", 2023-09-23T20:51:00.723 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.125626+0000", 2023-09-23T20:51:00.724 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000310616 2023-09-23T20:51:00.724 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.724 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.724 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:51:00.724 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.127519+0000", 2023-09-23T20:51:00.724 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.001893582 2023-09-23T20:51:00.724 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.724 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.724 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:00.724 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.132556+0000", 2023-09-23T20:51:00.724 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0050365490000000004 2023-09-23T20:51:00.725 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.725 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.725 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:00.725 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.134317+0000", 2023-09-23T20:51:00.725 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0017612179999999999 2023-09-23T20:51:00.725 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.725 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.725 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:00.725 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.134355+0000", 2023-09-23T20:51:00.725 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.7691000000000002e-05 2023-09-23T20:51:00.725 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.725 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.726 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:51:00.726 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.134412+0000", 2023-09-23T20:51:00.726 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.7441000000000001e-05 2023-09-23T20:51:00.726 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.726 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.726 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:51:00.726 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.134526+0000", 2023-09-23T20:51:00.726 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000113888 2023-09-23T20:51:00.726 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:51:00.726 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:51:00.726 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:51:00.727 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.727 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.727 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:197537 10.17 10:ef38087a:::benchmark_data_smithi018_422431_object93041:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e625)", 2023-09-23T20:51:00.727 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:49:30.103092+0000", 2023-09-23T20:51:00.727 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 90.458096135999995, 2023-09-23T20:51:00.727 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0365641029999999, 2023-09-23T20:51:00.727 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:51:00.727 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:00.727 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:51:00.727 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:51:00.727 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:51:00.727 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 197537 2023-09-23T20:51:00.728 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.728 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:51:00.728 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.728 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:51:00.728 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:30.103092+0000", 2023-09-23T20:51:00.728 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:51:00.728 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.728 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.728 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:51:00.728 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:30.103092+0000", 2023-09-23T20:51:00.728 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:51:00.728 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.729 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.729 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:51:00.729 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:30.103091+0000", 2023-09-23T20:51:00.729 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:51:00.729 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.729 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.729 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:51:00.729 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:30.103096+0000", 2023-09-23T20:51:00.729 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.4739999999999996e-06 2023-09-23T20:51:00.729 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.729 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.729 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:51:00.730 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:30.103109+0000", 2023-09-23T20:51:00.730 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2785e-05 2023-09-23T20:51:00.730 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.730 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.730 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:51:00.730 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:30.103115+0000", 2023-09-23T20:51:00.730 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.0129999999999999e-06 2023-09-23T20:51:00.730 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.730 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.730 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:51:00.731 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:30.103181+0000", 2023-09-23T20:51:00.731 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.5869000000000002e-05 2023-09-23T20:51:00.731 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.731 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.731 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:51:00.731 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:30.103195+0000", 2023-09-23T20:51:00.731 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.4426999999999999e-05 2023-09-23T20:51:00.731 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.731 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.731 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:51:00.731 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.127600+0000", 2023-09-23T20:51:00.731 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.024404724 2023-09-23T20:51:00.732 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.732 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.732 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:51:00.732 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.127751+0000", 2023-09-23T20:51:00.732 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00015088300000000001 2023-09-23T20:51:00.732 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.732 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.732 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 4,5,6", 2023-09-23T20:51:00.732 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.128077+0000", 2023-09-23T20:51:00.732 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00032595299999999997 2023-09-23T20:51:00.732 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.732 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.733 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:51:00.733 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.132514+0000", 2023-09-23T20:51:00.733 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0044368059999999997 2023-09-23T20:51:00.733 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.733 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.733 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:00.733 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.134565+0000", 2023-09-23T20:51:00.733 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0020515899999999998 2023-09-23T20:51:00.733 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.733 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.733 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:00.733 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.139364+0000", 2023-09-23T20:51:00.734 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0047984480000000003 2023-09-23T20:51:00.734 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.734 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.734 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:00.734 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.139474+0000", 2023-09-23T20:51:00.734 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000110575 2023-09-23T20:51:00.734 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.734 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.734 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:51:00.734 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.139518+0000", 2023-09-23T20:51:00.734 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.3584000000000001e-05 2023-09-23T20:51:00.734 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:00.735 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:00.735 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:51:00.735 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:31.139656+0000", 2023-09-23T20:51:00.735 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000138257 2023-09-23T20:51:00.735 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:51:00.735 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:51:00.735 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:51:00.735 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:51:00.735 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:51:00.735 INFO:teuthology.orchestra.run.smithi018.stdout:} 2023-09-23T20:51:01.189 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T20:51:01.334 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:51:01.336 INFO:teuthology.orchestra.run.smithi018.stderr:dumped all 2023-09-23T20:51:01.747 INFO:teuthology.orchestra.run.smithi018.stdout:{"pg_ready":true,"pg_map":{"version":1633,"stamp":"2023-09-23T20:51:00.376457+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":8984499,"num_objects":29,"num_object_clones":0,"num_object_copies":87,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":29,"num_whiteouts":0,"num_read":11981,"num_read_kb":26415,"num_write":1751,"num_write_kb":25419,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":276,"num_bytes_recovered":25707063,"num_keys_recovered":80,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6996,"ondisk_log_size":6996,"up":309,"acting":309,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":309,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":5518492,"kb_used_data":80548,"kb_used_omap":32,"kb_used_meta":5437855,"kb_avail":650697572,"statfs":{"total":671965249536,"available":666314313728,"internally_reserved":0,"allocated":82481152,"data_stored":64379680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32773,"internal_metadata":5568364539},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"7.603208"},"pg_stats":[{"pgid":"1.18","version":"90'161","reported_seq":733,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-09-23T20:50:55.355281+0000","last_change":"2023-09-23T20:49:51.130089+0000","last_active":"2023-09-23T20:50:55.355281+0000","last_peered":"2023-09-23T20:50:55.355281+0000","last_clean":"2023-09-23T20:50:55.355281+0000","last_became_active":"2023-09-23T20:49:51.124899+0000","last_became_peered":"2023-09-23T20:49:51.124899+0000","last_unstale":"2023-09-23T20:50:55.355281+0000","last_undegraded":"2023-09-23T20:50:55.355281+0000","last_fullsized":"2023-09-23T20:50:55.355281+0000","mapping_epoch":639,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":640,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:19.677209+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:19.677209+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:58:59.316978+0000","scrub_duration":0.033122453000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"69'1","reported_seq":145,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-09-23T20:50:53.292547+0000","last_change":"2023-09-23T20:48:48.148250+0000","last_active":"2023-09-23T20:50:53.292547+0000","last_peered":"2023-09-23T20:50:53.292547+0000","last_clean":"2023-09-23T20:50:53.292547+0000","last_became_active":"2023-09-23T20:48:48.144271+0000","last_became_peered":"2023-09-23T20:48:48.144271+0000","last_unstale":"2023-09-23T20:50:53.292547+0000","last_undegraded":"2023-09-23T20:50:53.292547+0000","last_fullsized":"2023-09-23T20:50:53.292547+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:59:36.502986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":514,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-09-23T20:50:55.356759+0000","last_change":"2023-09-23T20:49:02.703099+0000","last_active":"2023-09-23T20:50:55.356759+0000","last_peered":"2023-09-23T20:50:55.356759+0000","last_clean":"2023-09-23T20:50:55.356759+0000","last_became_active":"2023-09-23T20:49:02.702753+0000","last_became_peered":"2023-09-23T20:49:02.702753+0000","last_unstale":"2023-09-23T20:50:55.356759+0000","last_undegraded":"2023-09-23T20:50:55.356759+0000","last_fullsized":"2023-09-23T20:50:55.356759+0000","mapping_epoch":603,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":604,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:24:30.834235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"90'161","reported_seq":691,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-09-23T20:50:53.292541+0000","last_change":"2023-09-23T20:49:51.121955+0000","last_active":"2023-09-23T20:50:53.292541+0000","last_peered":"2023-09-23T20:50:53.292541+0000","last_clean":"2023-09-23T20:50:53.292541+0000","last_became_active":"2023-09-23T20:49:51.121641+0000","last_became_peered":"2023-09-23T20:49:51.121641+0000","last_unstale":"2023-09-23T20:50:53.292541+0000","last_undegraded":"2023-09-23T20:50:53.292541+0000","last_fullsized":"2023-09-23T20:50:53.292541+0000","mapping_epoch":639,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":640,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:47:33.523187+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:36:05.442288+0000","last_clean_scrub_stamp":"2023-09-23T20:47:33.523187+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:05:36.252341+0000","scrub_duration":0.013532127,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,2],"acting":[4,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":659,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-09-23T20:50:52.793482+0000","last_change":"2023-09-23T20:47:09.173501+0000","last_active":"2023-09-23T20:50:52.793482+0000","last_peered":"2023-09-23T20:50:52.793482+0000","last_clean":"2023-09-23T20:50:52.793482+0000","last_became_active":"2023-09-23T20:47:09.173160+0000","last_became_peered":"2023-09-23T20:47:09.173160+0000","last_unstale":"2023-09-23T20:50:52.793482+0000","last_undegraded":"2023-09-23T20:50:52.793482+0000","last_fullsized":"2023-09-23T20:50:52.793482+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:44:36.132093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":629,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-09-23T20:50:52.791085+0000","last_change":"2023-09-23T20:43:46.521939+0000","last_active":"2023-09-23T20:50:52.791085+0000","last_peered":"2023-09-23T20:50:52.791085+0000","last_clean":"2023-09-23T20:50:52.791085+0000","last_became_active":"2023-09-23T20:43:46.521185+0000","last_became_peered":"2023-09-23T20:43:46.521185+0000","last_unstale":"2023-09-23T20:50:52.791085+0000","last_undegraded":"2023-09-23T20:50:52.791085+0000","last_fullsized":"2023-09-23T20:50:52.791085+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:59:25.045627+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"90'161","reported_seq":556,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-09-23T20:50:53.292379+0000","last_change":"2023-09-23T20:47:31.475001+0000","last_active":"2023-09-23T20:50:53.292379+0000","last_peered":"2023-09-23T20:50:53.292379+0000","last_clean":"2023-09-23T20:50:53.292379+0000","last_became_active":"2023-09-23T20:47:11.617942+0000","last_became_peered":"2023-09-23T20:47:11.617942+0000","last_unstale":"2023-09-23T20:50:53.292379+0000","last_undegraded":"2023-09-23T20:50:53.292379+0000","last_fullsized":"2023-09-23T20:50:53.292379+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:47:31.474914+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:47:31.474914+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:52:05.169219+0000","scrub_duration":0.0026262870000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":467,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-09-23T20:50:55.355540+0000","last_change":"2023-09-23T20:37:43.584414+0000","last_active":"2023-09-23T20:50:55.355540+0000","last_peered":"2023-09-23T20:50:55.355540+0000","last_clean":"2023-09-23T20:50:55.355540+0000","last_became_active":"2023-09-23T20:37:43.584116+0000","last_became_peered":"2023-09-23T20:37:43.584116+0000","last_unstale":"2023-09-23T20:50:55.355540+0000","last_undegraded":"2023-09-23T20:50:55.355540+0000","last_fullsized":"2023-09-23T20:50:55.355540+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:17:54.258172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":293,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-09-23T20:50:53.246366+0000","last_change":"2023-09-23T20:49:48.119179+0000","last_active":"2023-09-23T20:50:53.246366+0000","last_peered":"2023-09-23T20:50:53.246366+0000","last_clean":"2023-09-23T20:50:53.246366+0000","last_became_active":"2023-09-23T20:49:48.118849+0000","last_became_peered":"2023-09-23T20:49:48.118849+0000","last_unstale":"2023-09-23T20:50:53.246366+0000","last_undegraded":"2023-09-23T20:50:53.246366+0000","last_fullsized":"2023-09-23T20:50:53.246366+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:50:02.578569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"90'161","reported_seq":901,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-09-23T20:50:53.293151+0000","last_change":"2023-09-23T20:49:51.124760+0000","last_active":"2023-09-23T20:50:53.293151+0000","last_peered":"2023-09-23T20:50:53.293151+0000","last_clean":"2023-09-23T20:50:53.293151+0000","last_became_active":"2023-09-23T20:49:51.124406+0000","last_became_peered":"2023-09-23T20:49:51.124406+0000","last_unstale":"2023-09-23T20:50:53.293151+0000","last_undegraded":"2023-09-23T20:50:53.293151+0000","last_fullsized":"2023-09-23T20:50:53.293151+0000","mapping_epoch":639,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":640,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:03.411025+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:03.411025+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:28:15.568088+0000","scrub_duration":0.001802922,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":670,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-09-23T20:50:52.791005+0000","last_change":"2023-09-23T20:49:48.111285+0000","last_active":"2023-09-23T20:50:52.791005+0000","last_peered":"2023-09-23T20:50:52.791005+0000","last_clean":"2023-09-23T20:50:52.791005+0000","last_became_active":"2023-09-23T20:49:48.110934+0000","last_became_peered":"2023-09-23T20:49:48.110934+0000","last_unstale":"2023-09-23T20:50:52.791005+0000","last_undegraded":"2023-09-23T20:50:52.791005+0000","last_fullsized":"2023-09-23T20:50:52.791005+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:19:18.339128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":104,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-09-23T20:50:55.355314+0000","last_change":"2023-09-23T20:48:23.918092+0000","last_active":"2023-09-23T20:50:55.355314+0000","last_peered":"2023-09-23T20:50:55.355314+0000","last_clean":"2023-09-23T20:50:55.355314+0000","last_became_active":"2023-09-23T20:48:23.917386+0000","last_became_peered":"2023-09-23T20:48:23.917386+0000","last_unstale":"2023-09-23T20:50:55.355314+0000","last_undegraded":"2023-09-23T20:50:55.355314+0000","last_fullsized":"2023-09-23T20:50:55.355314+0000","mapping_epoch":577,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":578,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:06:09.587517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"90'161","reported_seq":672,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-09-23T20:50:55.355382+0000","last_change":"2023-09-23T20:49:36.717905+0000","last_active":"2023-09-23T20:50:55.355382+0000","last_peered":"2023-09-23T20:50:55.355382+0000","last_clean":"2023-09-23T20:50:55.355382+0000","last_became_active":"2023-09-23T20:48:51.169314+0000","last_became_peered":"2023-09-23T20:48:51.169314+0000","last_unstale":"2023-09-23T20:50:55.355382+0000","last_undegraded":"2023-09-23T20:50:55.355382+0000","last_fullsized":"2023-09-23T20:50:55.355382+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:49:36.717737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:49:36.717737+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:00:49.213712+0000","scrub_duration":0.0049746089999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":515,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-09-23T20:50:53.293366+0000","last_change":"2023-09-23T20:49:48.123820+0000","last_active":"2023-09-23T20:50:53.293366+0000","last_peered":"2023-09-23T20:50:53.293366+0000","last_clean":"2023-09-23T20:50:53.293366+0000","last_became_active":"2023-09-23T20:49:48.123514+0000","last_became_peered":"2023-09-23T20:49:48.123514+0000","last_unstale":"2023-09-23T20:50:53.293366+0000","last_undegraded":"2023-09-23T20:50:53.293366+0000","last_fullsized":"2023-09-23T20:50:53.293366+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:43:13.460464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":498,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-09-23T20:50:55.356398+0000","last_change":"2023-09-23T20:38:02.542034+0000","last_active":"2023-09-23T20:50:55.356398+0000","last_peered":"2023-09-23T20:50:55.356398+0000","last_clean":"2023-09-23T20:50:55.356398+0000","last_became_active":"2023-09-23T20:38:02.541575+0000","last_became_peered":"2023-09-23T20:38:02.541575+0000","last_unstale":"2023-09-23T20:50:55.356398+0000","last_undegraded":"2023-09-23T20:50:55.356398+0000","last_fullsized":"2023-09-23T20:50:55.356398+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:50:54.591529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"90'161","reported_seq":813,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-09-23T20:50:52.790795+0000","last_change":"2023-09-23T20:49:51.125057+0000","last_active":"2023-09-23T20:50:52.790795+0000","last_peered":"2023-09-23T20:50:52.790795+0000","last_clean":"2023-09-23T20:50:52.790795+0000","last_became_active":"2023-09-23T20:49:51.124616+0000","last_became_peered":"2023-09-23T20:49:51.124616+0000","last_unstale":"2023-09-23T20:50:52.790795+0000","last_undegraded":"2023-09-23T20:50:52.790795+0000","last_fullsized":"2023-09-23T20:50:52.790795+0000","mapping_epoch":639,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":640,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:14.669831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:14.669831+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:09:01.736519+0000","scrub_duration":0.0083441460000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":728,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-09-23T20:50:53.291436+0000","last_change":"2023-09-23T20:47:10.837423+0000","last_active":"2023-09-23T20:50:53.291436+0000","last_peered":"2023-09-23T20:50:53.291436+0000","last_clean":"2023-09-23T20:50:53.291436+0000","last_became_active":"2023-09-23T20:47:09.204324+0000","last_became_peered":"2023-09-23T20:47:09.204324+0000","last_unstale":"2023-09-23T20:50:53.291436+0000","last_undegraded":"2023-09-23T20:50:53.291436+0000","last_fullsized":"2023-09-23T20:50:53.291436+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":554,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:19.108914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":540,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":203,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-09-23T20:50:52.793468+0000","last_change":"2023-09-23T20:48:48.145175+0000","last_active":"2023-09-23T20:50:52.793468+0000","last_peered":"2023-09-23T20:50:52.793468+0000","last_clean":"2023-09-23T20:50:52.793468+0000","last_became_active":"2023-09-23T20:48:48.144841+0000","last_became_peered":"2023-09-23T20:48:48.144841+0000","last_unstale":"2023-09-23T20:50:52.793468+0000","last_undegraded":"2023-09-23T20:50:52.793468+0000","last_fullsized":"2023-09-23T20:50:52.793468+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:27:40.802917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"90'161","reported_seq":797,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-09-23T20:50:52.794161+0000","last_change":"2023-09-23T20:36:02.548387+0000","last_active":"2023-09-23T20:50:52.794161+0000","last_peered":"2023-09-23T20:50:52.794161+0000","last_clean":"2023-09-23T20:50:52.794161+0000","last_became_active":"2023-09-23T20:35:26.644787+0000","last_became_peered":"2023-09-23T20:35:26.644787+0000","last_unstale":"2023-09-23T20:50:52.794161+0000","last_undegraded":"2023-09-23T20:50:52.794161+0000","last_fullsized":"2023-09-23T20:50:52.794161+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:02.548285+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:02.548285+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:37:15.216553+0000","scrub_duration":0.0015514459999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":135,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-09-23T20:50:53.291645+0000","last_change":"2023-09-23T20:47:09.340979+0000","last_active":"2023-09-23T20:50:53.291645+0000","last_peered":"2023-09-23T20:50:53.291645+0000","last_clean":"2023-09-23T20:50:53.291645+0000","last_became_active":"2023-09-23T20:47:09.231222+0000","last_became_peered":"2023-09-23T20:47:09.231222+0000","last_unstale":"2023-09-23T20:50:53.291645+0000","last_undegraded":"2023-09-23T20:50:53.291645+0000","last_fullsized":"2023-09-23T20:50:53.291645+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:07:50.620006+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":125,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-09-23T20:50:53.292783+0000","last_change":"2023-09-23T20:47:09.186315+0000","last_active":"2023-09-23T20:50:53.292783+0000","last_peered":"2023-09-23T20:50:53.292783+0000","last_clean":"2023-09-23T20:50:53.292783+0000","last_became_active":"2023-09-23T20:47:09.184580+0000","last_became_peered":"2023-09-23T20:47:09.184580+0000","last_unstale":"2023-09-23T20:50:53.292783+0000","last_undegraded":"2023-09-23T20:50:53.292783+0000","last_fullsized":"2023-09-23T20:50:53.292783+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:44:35.553948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"90'161","reported_seq":942,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-09-23T20:50:52.790699+0000","last_change":"2023-09-23T20:49:51.148240+0000","last_active":"2023-09-23T20:50:52.790699+0000","last_peered":"2023-09-23T20:50:52.790699+0000","last_clean":"2023-09-23T20:50:52.790699+0000","last_became_active":"2023-09-23T20:49:51.147475+0000","last_became_peered":"2023-09-23T20:49:51.147475+0000","last_unstale":"2023-09-23T20:50:52.790699+0000","last_undegraded":"2023-09-23T20:50:52.790699+0000","last_fullsized":"2023-09-23T20:50:52.790699+0000","mapping_epoch":639,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":640,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:43.578666+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:43.578666+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:02:42.273671+0000","scrub_duration":0.0021890379999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":759,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-09-23T20:50:53.292684+0000","last_change":"2023-09-23T20:48:48.193666+0000","last_active":"2023-09-23T20:50:53.292684+0000","last_peered":"2023-09-23T20:50:53.292684+0000","last_clean":"2023-09-23T20:50:53.292684+0000","last_became_active":"2023-09-23T20:48:48.152081+0000","last_became_peered":"2023-09-23T20:48:48.152081+0000","last_unstale":"2023-09-23T20:50:53.292684+0000","last_undegraded":"2023-09-23T20:50:53.292684+0000","last_fullsized":"2023-09-23T20:50:53.292684+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:00:19.508124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":272,"num_keys_recovered":80,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":427,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-09-23T20:50:55.356102+0000","last_change":"2023-09-23T20:49:48.119365+0000","last_active":"2023-09-23T20:50:55.356102+0000","last_peered":"2023-09-23T20:50:55.356102+0000","last_clean":"2023-09-23T20:50:55.356102+0000","last_became_active":"2023-09-23T20:49:48.119009+0000","last_became_peered":"2023-09-23T20:49:48.119009+0000","last_unstale":"2023-09-23T20:50:55.356102+0000","last_undegraded":"2023-09-23T20:50:55.356102+0000","last_fullsized":"2023-09-23T20:50:55.356102+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:02:26.878542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"90'161","reported_seq":969,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-09-23T20:50:53.292937+0000","last_change":"2023-09-23T20:49:51.118813+0000","last_active":"2023-09-23T20:50:53.292937+0000","last_peered":"2023-09-23T20:50:53.292937+0000","last_clean":"2023-09-23T20:50:53.292937+0000","last_became_active":"2023-09-23T20:49:51.118445+0000","last_became_peered":"2023-09-23T20:49:51.118445+0000","last_unstale":"2023-09-23T20:50:53.292937+0000","last_undegraded":"2023-09-23T20:50:53.292937+0000","last_fullsized":"2023-09-23T20:50:53.292937+0000","mapping_epoch":639,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":640,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:20.041690+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:20.041690+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:15:24.683302+0000","scrub_duration":0.0026180330000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":627,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-09-23T20:50:52.790312+0000","last_change":"2023-09-23T20:43:46.521294+0000","last_active":"2023-09-23T20:50:52.790312+0000","last_peered":"2023-09-23T20:50:52.790312+0000","last_clean":"2023-09-23T20:50:52.790312+0000","last_became_active":"2023-09-23T20:43:46.521001+0000","last_became_peered":"2023-09-23T20:43:46.521001+0000","last_unstale":"2023-09-23T20:50:52.790312+0000","last_undegraded":"2023-09-23T20:50:52.790312+0000","last_fullsized":"2023-09-23T20:50:52.790312+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:01:35.126716+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":456,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-09-23T20:50:53.293469+0000","last_change":"2023-09-23T20:49:48.121288+0000","last_active":"2023-09-23T20:50:53.293469+0000","last_peered":"2023-09-23T20:50:53.293469+0000","last_clean":"2023-09-23T20:50:53.293469+0000","last_became_active":"2023-09-23T20:49:48.120936+0000","last_became_peered":"2023-09-23T20:49:48.120936+0000","last_unstale":"2023-09-23T20:50:53.293469+0000","last_undegraded":"2023-09-23T20:50:53.293469+0000","last_fullsized":"2023-09-23T20:50:53.293469+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:46:44.012748+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.13","version":"90'161","reported_seq":993,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-09-23T20:50:53.708992+0000","last_change":"2023-09-23T20:47:11.619371+0000","last_active":"2023-09-23T20:50:53.708992+0000","last_peered":"2023-09-23T20:50:53.708992+0000","last_clean":"2023-09-23T20:50:53.708992+0000","last_became_active":"2023-09-23T20:47:11.619006+0000","last_became_peered":"2023-09-23T20:47:11.619006+0000","last_unstale":"2023-09-23T20:50:53.708992+0000","last_undegraded":"2023-09-23T20:50:53.708992+0000","last_fullsized":"2023-09-23T20:50:53.708992+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:42.544308+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:42.544308+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:47:24.414807+0000","scrub_duration":0.0029958179999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":692,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-09-23T20:50:53.292740+0000","last_change":"2023-09-23T20:47:09.176613+0000","last_active":"2023-09-23T20:50:53.292740+0000","last_peered":"2023-09-23T20:50:53.292740+0000","last_clean":"2023-09-23T20:50:53.292740+0000","last_became_active":"2023-09-23T20:47:09.176187+0000","last_became_peered":"2023-09-23T20:47:09.176187+0000","last_unstale":"2023-09-23T20:50:53.292740+0000","last_undegraded":"2023-09-23T20:50:53.292740+0000","last_fullsized":"2023-09-23T20:50:53.292740+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:23:01.590989+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":635,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-09-23T20:50:52.791112+0000","last_change":"2023-09-23T20:43:46.521735+0000","last_active":"2023-09-23T20:50:52.791112+0000","last_peered":"2023-09-23T20:50:52.791112+0000","last_clean":"2023-09-23T20:50:52.791112+0000","last_became_active":"2023-09-23T20:43:46.520762+0000","last_became_peered":"2023-09-23T20:43:46.520762+0000","last_unstale":"2023-09-23T20:50:52.791112+0000","last_undegraded":"2023-09-23T20:50:52.791112+0000","last_fullsized":"2023-09-23T20:50:52.791112+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:26:48.841930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"90'161","reported_seq":772,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-09-23T20:50:52.793773+0000","last_change":"2023-09-23T20:49:51.125786+0000","last_active":"2023-09-23T20:50:52.793773+0000","last_peered":"2023-09-23T20:50:52.793773+0000","last_clean":"2023-09-23T20:50:52.793773+0000","last_became_active":"2023-09-23T20:49:51.124373+0000","last_became_peered":"2023-09-23T20:49:51.124373+0000","last_unstale":"2023-09-23T20:50:52.793773+0000","last_undegraded":"2023-09-23T20:50:52.793773+0000","last_fullsized":"2023-09-23T20:50:52.793773+0000","mapping_epoch":639,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":640,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:18.952693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:18.952693+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:07:29.170332+0000","scrub_duration":0.001699365,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":363,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-09-23T20:50:53.245832+0000","last_change":"2023-09-23T20:49:48.127484+0000","last_active":"2023-09-23T20:50:53.245832+0000","last_peered":"2023-09-23T20:50:53.245832+0000","last_clean":"2023-09-23T20:50:53.245832+0000","last_became_active":"2023-09-23T20:49:48.110748+0000","last_became_peered":"2023-09-23T20:49:48.110748+0000","last_unstale":"2023-09-23T20:50:53.245832+0000","last_undegraded":"2023-09-23T20:50:53.245832+0000","last_fullsized":"2023-09-23T20:50:53.245832+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:31:51.004529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":467,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-09-23T20:50:55.357292+0000","last_change":"2023-09-23T20:37:43.571083+0000","last_active":"2023-09-23T20:50:55.357292+0000","last_peered":"2023-09-23T20:50:55.357292+0000","last_clean":"2023-09-23T20:50:55.357292+0000","last_became_active":"2023-09-23T20:37:43.570306+0000","last_became_peered":"2023-09-23T20:37:43.570306+0000","last_unstale":"2023-09-23T20:50:55.357292+0000","last_undegraded":"2023-09-23T20:50:55.357292+0000","last_fullsized":"2023-09-23T20:50:55.357292+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:42:02.396718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.d","version":"90'161","reported_seq":1001,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-09-23T20:50:53.293159+0000","last_change":"2023-09-23T20:47:11.619510+0000","last_active":"2023-09-23T20:50:53.293159+0000","last_peered":"2023-09-23T20:50:53.293159+0000","last_clean":"2023-09-23T20:50:53.293159+0000","last_became_active":"2023-09-23T20:47:11.619154+0000","last_became_peered":"2023-09-23T20:47:11.619154+0000","last_unstale":"2023-09-23T20:50:53.293159+0000","last_undegraded":"2023-09-23T20:50:53.293159+0000","last_fullsized":"2023-09-23T20:50:53.293159+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:34:59.496375+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:34:59.496375+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:40:52.712777+0000","scrub_duration":0.0017843189999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":700,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-09-23T20:50:53.291488+0000","last_change":"2023-09-23T20:47:09.207851+0000","last_active":"2023-09-23T20:50:53.291488+0000","last_peered":"2023-09-23T20:50:53.291488+0000","last_clean":"2023-09-23T20:50:53.291488+0000","last_became_active":"2023-09-23T20:47:09.175836+0000","last_became_peered":"2023-09-23T20:47:09.175836+0000","last_unstale":"2023-09-23T20:50:53.291488+0000","last_undegraded":"2023-09-23T20:50:53.291488+0000","last_fullsized":"2023-09-23T20:50:53.291488+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:43:13.091809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":493,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-09-23T20:50:53.293464+0000","last_change":"2023-09-23T20:49:48.122188+0000","last_active":"2023-09-23T20:50:53.293464+0000","last_peered":"2023-09-23T20:50:53.293464+0000","last_clean":"2023-09-23T20:50:53.293464+0000","last_became_active":"2023-09-23T20:49:48.121894+0000","last_became_peered":"2023-09-23T20:49:48.121894+0000","last_unstale":"2023-09-23T20:50:53.293464+0000","last_undegraded":"2023-09-23T20:50:53.293464+0000","last_fullsized":"2023-09-23T20:50:53.293464+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:12:33.758985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.c","version":"90'161","reported_seq":933,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-09-23T20:50:55.356187+0000","last_change":"2023-09-23T20:41:09.506481+0000","last_active":"2023-09-23T20:50:55.356187+0000","last_peered":"2023-09-23T20:50:55.356187+0000","last_clean":"2023-09-23T20:50:55.356187+0000","last_became_active":"2023-09-23T20:38:28.442229+0000","last_became_peered":"2023-09-23T20:38:28.442229+0000","last_unstale":"2023-09-23T20:50:55.356187+0000","last_undegraded":"2023-09-23T20:50:55.356187+0000","last_fullsized":"2023-09-23T20:50:55.356187+0000","mapping_epoch":411,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":412,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:41:09.506338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:41:09.506338+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:19:46.265011+0000","scrub_duration":0.01112002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":738,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-09-23T20:50:53.291462+0000","last_change":"2023-09-23T20:47:10.848436+0000","last_active":"2023-09-23T20:50:53.291462+0000","last_peered":"2023-09-23T20:50:53.291462+0000","last_clean":"2023-09-23T20:50:53.291462+0000","last_became_active":"2023-09-23T20:47:09.196769+0000","last_became_peered":"2023-09-23T20:47:09.196769+0000","last_unstale":"2023-09-23T20:50:53.291462+0000","last_undegraded":"2023-09-23T20:50:53.291462+0000","last_fullsized":"2023-09-23T20:50:53.291462+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":554,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:59:38.105009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":135,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-09-23T20:50:53.292473+0000","last_change":"2023-09-23T20:48:23.915817+0000","last_active":"2023-09-23T20:50:53.292473+0000","last_peered":"2023-09-23T20:50:53.292473+0000","last_clean":"2023-09-23T20:50:53.292473+0000","last_became_active":"2023-09-23T20:48:23.915306+0000","last_became_peered":"2023-09-23T20:48:23.915306+0000","last_unstale":"2023-09-23T20:50:53.292473+0000","last_undegraded":"2023-09-23T20:50:53.292473+0000","last_fullsized":"2023-09-23T20:50:53.292473+0000","mapping_epoch":577,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":578,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:55:45.691450+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"90'161","reported_seq":883,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-09-23T20:50:53.246489+0000","last_change":"2023-09-23T20:49:51.120296+0000","last_active":"2023-09-23T20:50:53.246489+0000","last_peered":"2023-09-23T20:50:53.246489+0000","last_clean":"2023-09-23T20:50:53.246489+0000","last_became_active":"2023-09-23T20:49:51.120016+0000","last_became_peered":"2023-09-23T20:49:51.120016+0000","last_unstale":"2023-09-23T20:50:53.246489+0000","last_undegraded":"2023-09-23T20:50:53.246489+0000","last_fullsized":"2023-09-23T20:50:53.246489+0000","mapping_epoch":639,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":640,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:46.217489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:46.217489+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:35:59.437404+0000","scrub_duration":0.001456429,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":711,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-09-23T20:50:53.293273+0000","last_change":"2023-09-23T20:49:48.131779+0000","last_active":"2023-09-23T20:50:53.293273+0000","last_peered":"2023-09-23T20:50:53.293273+0000","last_clean":"2023-09-23T20:50:53.293273+0000","last_became_active":"2023-09-23T20:49:48.123934+0000","last_became_peered":"2023-09-23T20:49:48.123934+0000","last_unstale":"2023-09-23T20:50:53.293273+0000","last_undegraded":"2023-09-23T20:50:53.293273+0000","last_fullsized":"2023-09-23T20:50:53.293273+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:24:40.508247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":490,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":283,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-09-23T20:50:53.245868+0000","last_change":"2023-09-23T20:49:02.702301+0000","last_active":"2023-09-23T20:50:53.245868+0000","last_peered":"2023-09-23T20:50:53.245868+0000","last_clean":"2023-09-23T20:50:53.245868+0000","last_became_active":"2023-09-23T20:49:02.701986+0000","last_became_peered":"2023-09-23T20:49:02.701986+0000","last_unstale":"2023-09-23T20:50:53.245868+0000","last_undegraded":"2023-09-23T20:50:53.245868+0000","last_fullsized":"2023-09-23T20:50:53.245868+0000","mapping_epoch":603,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":604,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:20:01.603267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.e","version":"90'161","reported_seq":698,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-09-23T20:50:53.292441+0000","last_change":"2023-09-23T20:47:37.608426+0000","last_active":"2023-09-23T20:50:53.292441+0000","last_peered":"2023-09-23T20:50:53.292441+0000","last_clean":"2023-09-23T20:50:53.292441+0000","last_became_active":"2023-09-23T20:47:11.617187+0000","last_became_peered":"2023-09-23T20:47:11.617187+0000","last_unstale":"2023-09-23T20:50:53.292441+0000","last_undegraded":"2023-09-23T20:50:53.292441+0000","last_fullsized":"2023-09-23T20:50:53.292441+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:47:37.608309+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:47:37.608309+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:22:25.482297+0000","scrub_duration":0.011600924,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":641,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-09-23T20:50:55.356435+0000","last_change":"2023-09-23T20:49:48.120934+0000","last_active":"2023-09-23T20:50:55.356435+0000","last_peered":"2023-09-23T20:50:55.356435+0000","last_clean":"2023-09-23T20:50:55.356435+0000","last_became_active":"2023-09-23T20:49:48.120578+0000","last_became_peered":"2023-09-23T20:49:48.120578+0000","last_unstale":"2023-09-23T20:50:55.356435+0000","last_undegraded":"2023-09-23T20:50:55.356435+0000","last_fullsized":"2023-09-23T20:50:55.356435+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:57:50.669399+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":663,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-09-23T20:50:52.790191+0000","last_change":"2023-09-23T20:47:09.197427+0000","last_active":"2023-09-23T20:50:52.790191+0000","last_peered":"2023-09-23T20:50:52.790191+0000","last_clean":"2023-09-23T20:50:52.790191+0000","last_became_active":"2023-09-23T20:47:09.196542+0000","last_became_peered":"2023-09-23T20:47:09.196542+0000","last_unstale":"2023-09-23T20:50:52.790191+0000","last_undegraded":"2023-09-23T20:50:52.790191+0000","last_fullsized":"2023-09-23T20:50:52.790191+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:28:38.926202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":1114,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-09-23T20:50:53.292545+0000","last_change":"2023-09-23T20:48:51.169474+0000","last_active":"2023-09-23T20:50:53.292545+0000","last_peered":"2023-09-23T20:50:53.292545+0000","last_clean":"2023-09-23T20:50:53.292545+0000","last_became_active":"2023-09-23T20:48:51.169076+0000","last_became_peered":"2023-09-23T20:48:51.169076+0000","last_unstale":"2023-09-23T20:50:53.292545+0000","last_undegraded":"2023-09-23T20:50:53.292545+0000","last_fullsized":"2023-09-23T20:50:53.292545+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:35:02.523203+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:35:02.523203+0000","last_clean_scrub_stamp":"2023-09-23T20:35:02.523203+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:25:53.741403+0000","scrub_duration":0.0085927989999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":519,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-09-23T20:50:55.355239+0000","last_change":"2023-09-23T20:47:09.240200+0000","last_active":"2023-09-23T20:50:55.355239+0000","last_peered":"2023-09-23T20:50:55.355239+0000","last_clean":"2023-09-23T20:50:55.355239+0000","last_became_active":"2023-09-23T20:47:09.195204+0000","last_became_peered":"2023-09-23T20:47:09.195204+0000","last_unstale":"2023-09-23T20:50:55.355239+0000","last_undegraded":"2023-09-23T20:50:55.355239+0000","last_fullsized":"2023-09-23T20:50:55.355239+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:03.328250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":125,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-09-23T20:50:53.291795+0000","last_change":"2023-09-23T20:47:09.231714+0000","last_active":"2023-09-23T20:50:53.291795+0000","last_peered":"2023-09-23T20:50:53.291795+0000","last_clean":"2023-09-23T20:50:53.291795+0000","last_became_active":"2023-09-23T20:47:09.230918+0000","last_became_peered":"2023-09-23T20:47:09.230918+0000","last_unstale":"2023-09-23T20:50:53.291795+0000","last_undegraded":"2023-09-23T20:50:53.291795+0000","last_fullsized":"2023-09-23T20:50:53.291795+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:24.583656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":574,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-09-23T20:50:55.355483+0000","last_change":"2023-09-23T20:50:03.882061+0000","last_active":"2023-09-23T20:50:55.355483+0000","last_peered":"2023-09-23T20:50:55.355483+0000","last_clean":"2023-09-23T20:50:55.355483+0000","last_became_active":"2023-09-23T20:49:51.124481+0000","last_became_peered":"2023-09-23T20:49:51.124481+0000","last_unstale":"2023-09-23T20:50:55.355483+0000","last_undegraded":"2023-09-23T20:50:55.355483+0000","last_fullsized":"2023-09-23T20:50:55.355483+0000","mapping_epoch":639,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":640,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:50:03.881826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:50:03.881826+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:51.282979+0000","scrub_duration":0.003245258,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":293,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-09-23T20:50:53.245536+0000","last_change":"2023-09-23T20:47:09.178919+0000","last_active":"2023-09-23T20:50:53.245536+0000","last_peered":"2023-09-23T20:50:53.245536+0000","last_clean":"2023-09-23T20:50:53.245536+0000","last_became_active":"2023-09-23T20:47:09.178607+0000","last_became_peered":"2023-09-23T20:47:09.178607+0000","last_unstale":"2023-09-23T20:50:53.245536+0000","last_undegraded":"2023-09-23T20:50:53.245536+0000","last_fullsized":"2023-09-23T20:50:53.245536+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:23:45.196935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":145,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-09-23T20:50:53.292761+0000","last_change":"2023-09-23T20:48:48.148596+0000","last_active":"2023-09-23T20:50:53.292761+0000","last_peered":"2023-09-23T20:50:53.292761+0000","last_clean":"2023-09-23T20:50:53.292761+0000","last_became_active":"2023-09-23T20:48:48.148211+0000","last_became_peered":"2023-09-23T20:48:48.148211+0000","last_unstale":"2023-09-23T20:50:53.292761+0000","last_undegraded":"2023-09-23T20:50:53.292761+0000","last_fullsized":"2023-09-23T20:50:53.292761+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:21:50.825822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"90'161","reported_seq":992,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-09-23T20:50:52.793965+0000","last_change":"2023-09-23T20:49:51.124836+0000","last_active":"2023-09-23T20:50:52.793965+0000","last_peered":"2023-09-23T20:50:52.793965+0000","last_clean":"2023-09-23T20:50:52.793965+0000","last_became_active":"2023-09-23T20:49:51.124419+0000","last_became_peered":"2023-09-23T20:49:51.124419+0000","last_unstale":"2023-09-23T20:50:52.793965+0000","last_undegraded":"2023-09-23T20:50:52.793965+0000","last_fullsized":"2023-09-23T20:50:52.793965+0000","mapping_epoch":639,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":640,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:05.595042+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:05.595042+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:37:05.219030+0000","scrub_duration":0.007271675,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":145,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-09-23T20:50:53.292253+0000","last_change":"2023-09-23T20:48:48.153145+0000","last_active":"2023-09-23T20:50:53.292253+0000","last_peered":"2023-09-23T20:50:53.292253+0000","last_clean":"2023-09-23T20:50:53.292253+0000","last_became_active":"2023-09-23T20:48:48.152858+0000","last_became_peered":"2023-09-23T20:48:48.152858+0000","last_unstale":"2023-09-23T20:50:53.292253+0000","last_undegraded":"2023-09-23T20:50:53.292253+0000","last_fullsized":"2023-09-23T20:50:53.292253+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:23:13.454288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":125,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-09-23T20:50:53.291769+0000","last_change":"2023-09-23T20:47:09.186271+0000","last_active":"2023-09-23T20:50:53.291769+0000","last_peered":"2023-09-23T20:50:53.291769+0000","last_clean":"2023-09-23T20:50:53.291769+0000","last_became_active":"2023-09-23T20:47:09.184357+0000","last_became_peered":"2023-09-23T20:47:09.184357+0000","last_unstale":"2023-09-23T20:50:53.291769+0000","last_undegraded":"2023-09-23T20:50:53.291769+0000","last_fullsized":"2023-09-23T20:50:53.291769+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:48:26.339811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":578,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-09-23T20:50:55.356449+0000","last_change":"2023-09-23T20:49:51.128432+0000","last_active":"2023-09-23T20:50:55.356449+0000","last_peered":"2023-09-23T20:50:55.356449+0000","last_clean":"2023-09-23T20:50:55.356449+0000","last_became_active":"2023-09-23T20:49:51.128074+0000","last_became_peered":"2023-09-23T20:49:51.128074+0000","last_unstale":"2023-09-23T20:50:55.356449+0000","last_undegraded":"2023-09-23T20:50:55.356449+0000","last_fullsized":"2023-09-23T20:50:55.356449+0000","mapping_epoch":639,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":640,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:49:37.762492+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:49:37.762492+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:31:57.447267+0000","scrub_duration":0.0031485979999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,4],"acting":[2,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":468,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-09-23T20:50:55.355464+0000","last_change":"2023-09-23T20:37:43.570610+0000","last_active":"2023-09-23T20:50:55.355464+0000","last_peered":"2023-09-23T20:50:55.355464+0000","last_clean":"2023-09-23T20:50:55.355464+0000","last_became_active":"2023-09-23T20:37:43.570216+0000","last_became_peered":"2023-09-23T20:37:43.570216+0000","last_unstale":"2023-09-23T20:50:55.355464+0000","last_undegraded":"2023-09-23T20:50:55.355464+0000","last_fullsized":"2023-09-23T20:50:55.355464+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:39:26.555329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":515,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-09-23T20:50:52.794255+0000","last_change":"2023-09-23T20:47:09.172573+0000","last_active":"2023-09-23T20:50:52.794255+0000","last_peered":"2023-09-23T20:50:52.794255+0000","last_clean":"2023-09-23T20:50:52.794255+0000","last_became_active":"2023-09-23T20:47:09.172298+0000","last_became_peered":"2023-09-23T20:47:09.172298+0000","last_unstale":"2023-09-23T20:50:52.794255+0000","last_undegraded":"2023-09-23T20:50:52.794255+0000","last_fullsized":"2023-09-23T20:50:52.794255+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:22:09.361284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":856,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-09-23T20:50:53.246441+0000","last_change":"2023-09-23T20:41:07.424609+0000","last_active":"2023-09-23T20:50:53.246441+0000","last_peered":"2023-09-23T20:50:53.246441+0000","last_clean":"2023-09-23T20:50:53.246441+0000","last_became_active":"2023-09-23T20:38:05.622261+0000","last_became_peered":"2023-09-23T20:38:05.622261+0000","last_unstale":"2023-09-23T20:50:53.246441+0000","last_undegraded":"2023-09-23T20:50:53.246441+0000","last_fullsized":"2023-09-23T20:50:53.246441+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:41:07.424492+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:41:07.424492+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:45:21.689123+0000","scrub_duration":0.0018960959999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"664'242","reported_seq":1420,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-09-23T20:50:52.794354+0000","last_change":"2023-09-23T20:50:39.737511+0000","last_active":"2023-09-23T20:50:52.794354+0000","last_peered":"2023-09-23T20:50:52.794354+0000","last_clean":"2023-09-23T20:50:52.794354+0000","last_became_active":"2023-09-23T20:29:45.520546+0000","last_became_peered":"2023-09-23T20:29:45.520546+0000","last_unstale":"2023-09-23T20:50:52.794354+0000","last_undegraded":"2023-09-23T20:50:52.794354+0000","last_fullsized":"2023-09-23T20:50:52.794354+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":242,"log_dups_size":0,"ondisk_log_size":242,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:58:56.085777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0012974410000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1152,"num_read_kb":4628,"num_write":256,"num_write_kb":3463,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":492,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-09-23T20:50:55.356843+0000","last_change":"2023-09-23T20:49:48.114924+0000","last_active":"2023-09-23T20:50:55.356843+0000","last_peered":"2023-09-23T20:50:55.356843+0000","last_clean":"2023-09-23T20:50:55.356843+0000","last_became_active":"2023-09-23T20:49:48.114554+0000","last_became_peered":"2023-09-23T20:49:48.114554+0000","last_unstale":"2023-09-23T20:50:55.356843+0000","last_undegraded":"2023-09-23T20:50:55.356843+0000","last_fullsized":"2023-09-23T20:50:55.356843+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:09:09.538462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":656,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-09-23T20:50:52.793597+0000","last_change":"2023-09-23T20:49:48.115598+0000","last_active":"2023-09-23T20:50:52.793597+0000","last_peered":"2023-09-23T20:50:52.793597+0000","last_clean":"2023-09-23T20:50:52.793597+0000","last_became_active":"2023-09-23T20:49:48.115313+0000","last_became_peered":"2023-09-23T20:49:48.115313+0000","last_unstale":"2023-09-23T20:50:52.793597+0000","last_undegraded":"2023-09-23T20:50:52.793597+0000","last_fullsized":"2023-09-23T20:50:52.793597+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:58:47.477333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":768,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-09-23T20:50:52.791015+0000","last_change":"2023-09-23T20:49:51.148445+0000","last_active":"2023-09-23T20:50:52.791015+0000","last_peered":"2023-09-23T20:50:52.791015+0000","last_clean":"2023-09-23T20:50:52.791015+0000","last_became_active":"2023-09-23T20:49:51.147884+0000","last_became_peered":"2023-09-23T20:49:51.147884+0000","last_unstale":"2023-09-23T20:50:52.791015+0000","last_undegraded":"2023-09-23T20:50:52.791015+0000","last_fullsized":"2023-09-23T20:50:52.791015+0000","mapping_epoch":639,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":640,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:15.656880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:15.656880+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:05:08.116069+0000","scrub_duration":0.0016711180000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"664'209","reported_seq":1650,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-09-23T20:50:52.793693+0000","last_change":"2023-09-23T20:50:39.737378+0000","last_active":"2023-09-23T20:50:52.793693+0000","last_peered":"2023-09-23T20:50:52.793693+0000","last_clean":"2023-09-23T20:50:52.793693+0000","last_became_active":"2023-09-23T20:49:51.125316+0000","last_became_peered":"2023-09-23T20:49:51.125316+0000","last_unstale":"2023-09-23T20:50:52.793693+0000","last_undegraded":"2023-09-23T20:50:52.793693+0000","last_fullsized":"2023-09-23T20:50:52.793693+0000","mapping_epoch":639,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":640,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":209,"log_dups_size":0,"ondisk_log_size":209,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:32:47.065876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001172474,"stat_sum":{"num_bytes":4194323,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":2248,"num_read_kb":3726,"num_write":182,"num_write_kb":5560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":125,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-09-23T20:50:53.292961+0000","last_change":"2023-09-23T20:47:09.177872+0000","last_active":"2023-09-23T20:50:53.292961+0000","last_peered":"2023-09-23T20:50:53.292961+0000","last_clean":"2023-09-23T20:50:53.292961+0000","last_became_active":"2023-09-23T20:47:09.176689+0000","last_became_peered":"2023-09-23T20:47:09.176689+0000","last_unstale":"2023-09-23T20:50:53.292961+0000","last_undegraded":"2023-09-23T20:50:53.292961+0000","last_fullsized":"2023-09-23T20:50:53.292961+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:14:01.590318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":653,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-09-23T20:50:52.794307+0000","last_change":"2023-09-23T20:48:48.151144+0000","last_active":"2023-09-23T20:50:52.794307+0000","last_peered":"2023-09-23T20:50:52.794307+0000","last_clean":"2023-09-23T20:50:52.794307+0000","last_became_active":"2023-09-23T20:48:48.150770+0000","last_became_peered":"2023-09-23T20:48:48.150770+0000","last_unstale":"2023-09-23T20:50:52.794307+0000","last_undegraded":"2023-09-23T20:50:52.794307+0000","last_fullsized":"2023-09-23T20:50:52.794307+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:14:08.549512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":672,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-09-23T20:50:53.292410+0000","last_change":"2023-09-23T20:47:11.618652+0000","last_active":"2023-09-23T20:50:53.292410+0000","last_peered":"2023-09-23T20:50:53.292410+0000","last_clean":"2023-09-23T20:50:53.292410+0000","last_became_active":"2023-09-23T20:47:11.618266+0000","last_became_peered":"2023-09-23T20:47:11.618266+0000","last_unstale":"2023-09-23T20:50:53.292410+0000","last_undegraded":"2023-09-23T20:50:53.292410+0000","last_fullsized":"2023-09-23T20:50:53.292410+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:26.978179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:26.978179+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:29:29.000683+0000","scrub_duration":0.0013962829999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"664'192","reported_seq":706,"reported_epoch":670,"state":"active+clean","last_fresh":"2023-09-23T20:50:58.296268+0000","last_change":"2023-09-23T20:50:39.795569+0000","last_active":"2023-09-23T20:50:58.296268+0000","last_peered":"2023-09-23T20:50:58.296268+0000","last_clean":"2023-09-23T20:50:58.296268+0000","last_became_active":"2023-09-23T20:48:51.170485+0000","last_became_peered":"2023-09-23T20:48:51.170485+0000","last_unstale":"2023-09-23T20:50:58.296268+0000","last_undegraded":"2023-09-23T20:50:58.296268+0000","last_fullsized":"2023-09-23T20:50:58.296268+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":192,"log_dups_size":0,"ondisk_log_size":192,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:48:35.309684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038574200000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":830,"num_read_kb":3504,"num_write":183,"num_write_kb":2624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":661,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-09-23T20:50:52.793388+0000","last_change":"2023-09-23T20:48:48.151034+0000","last_active":"2023-09-23T20:50:52.793388+0000","last_peered":"2023-09-23T20:50:52.793388+0000","last_clean":"2023-09-23T20:50:52.793388+0000","last_became_active":"2023-09-23T20:48:48.150658+0000","last_became_peered":"2023-09-23T20:48:48.150658+0000","last_unstale":"2023-09-23T20:50:52.793388+0000","last_undegraded":"2023-09-23T20:50:52.793388+0000","last_fullsized":"2023-09-23T20:50:52.793388+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:44:34.819851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":195,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-09-23T20:50:52.794136+0000","last_change":"2023-09-23T20:49:48.118182+0000","last_active":"2023-09-23T20:50:52.794136+0000","last_peered":"2023-09-23T20:50:52.794136+0000","last_clean":"2023-09-23T20:50:52.794136+0000","last_became_active":"2023-09-23T20:49:48.117804+0000","last_became_peered":"2023-09-23T20:49:48.117804+0000","last_unstale":"2023-09-23T20:50:52.794136+0000","last_undegraded":"2023-09-23T20:50:52.794136+0000","last_fullsized":"2023-09-23T20:50:52.794136+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:47:43.384018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"664'319","reported_seq":1155,"reported_epoch":670,"state":"active+clean","last_fresh":"2023-09-23T20:50:58.296380+0000","last_change":"2023-09-23T20:50:39.795712+0000","last_active":"2023-09-23T20:50:58.296380+0000","last_peered":"2023-09-23T20:50:58.296380+0000","last_clean":"2023-09-23T20:50:58.296380+0000","last_became_active":"2023-09-23T20:49:51.153529+0000","last_became_peered":"2023-09-23T20:49:51.153529+0000","last_unstale":"2023-09-23T20:50:58.296380+0000","last_undegraded":"2023-09-23T20:50:58.296380+0000","last_fullsized":"2023-09-23T20:50:58.296380+0000","mapping_epoch":639,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":640,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":319,"log_dups_size":0,"ondisk_log_size":319,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:10:22.072962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043345100000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1396,"num_read_kb":4612,"num_write":290,"num_write_kb":4437,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":1131,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-09-23T20:50:53.293109+0000","last_change":"2023-09-23T20:38:05.602508+0000","last_active":"2023-09-23T20:50:53.293109+0000","last_peered":"2023-09-23T20:50:53.293109+0000","last_clean":"2023-09-23T20:50:53.293109+0000","last_became_active":"2023-09-23T20:38:05.602129+0000","last_became_peered":"2023-09-23T20:38:05.602129+0000","last_unstale":"2023-09-23T20:50:53.293109+0000","last_undegraded":"2023-09-23T20:50:53.293109+0000","last_fullsized":"2023-09-23T20:50:53.293109+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:45.343742+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:45.343742+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:48.910343+0000","scrub_duration":0.001475802,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":253,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-09-23T20:50:53.245922+0000","last_change":"2023-09-23T20:49:48.119237+0000","last_active":"2023-09-23T20:50:53.245922+0000","last_peered":"2023-09-23T20:50:53.245922+0000","last_clean":"2023-09-23T20:50:53.245922+0000","last_became_active":"2023-09-23T20:49:48.118927+0000","last_became_peered":"2023-09-23T20:49:48.118927+0000","last_unstale":"2023-09-23T20:50:53.245922+0000","last_undegraded":"2023-09-23T20:50:53.245922+0000","last_fullsized":"2023-09-23T20:50:53.245922+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:31:47.579623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,4],"acting":[6,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":293,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-09-23T20:50:53.246259+0000","last_change":"2023-09-23T20:47:09.202302+0000","last_active":"2023-09-23T20:50:53.246259+0000","last_peered":"2023-09-23T20:50:53.246259+0000","last_clean":"2023-09-23T20:50:53.246259+0000","last_became_active":"2023-09-23T20:47:09.201797+0000","last_became_peered":"2023-09-23T20:47:09.201797+0000","last_unstale":"2023-09-23T20:50:53.246259+0000","last_undegraded":"2023-09-23T20:50:53.246259+0000","last_fullsized":"2023-09-23T20:50:53.246259+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:34:41.055705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":1038,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-09-23T20:50:52.793786+0000","last_change":"2023-09-23T20:47:11.622262+0000","last_active":"2023-09-23T20:50:52.793786+0000","last_peered":"2023-09-23T20:50:52.793786+0000","last_clean":"2023-09-23T20:50:52.793786+0000","last_became_active":"2023-09-23T20:47:11.621920+0000","last_became_peered":"2023-09-23T20:47:11.621920+0000","last_unstale":"2023-09-23T20:50:52.793786+0000","last_undegraded":"2023-09-23T20:50:52.793786+0000","last_fullsized":"2023-09-23T20:50:52.793786+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:43.582146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:43.582146+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:01:15.153543+0000","scrub_duration":0.0016219730000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"664'171","reported_seq":1333,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-09-23T20:50:53.293036+0000","last_change":"2023-09-23T20:50:39.742996+0000","last_active":"2023-09-23T20:50:53.293036+0000","last_peered":"2023-09-23T20:50:53.293036+0000","last_clean":"2023-09-23T20:50:53.293036+0000","last_became_active":"2023-09-23T20:29:45.536439+0000","last_became_peered":"2023-09-23T20:29:45.536439+0000","last_unstale":"2023-09-23T20:50:53.293036+0000","last_undegraded":"2023-09-23T20:50:53.293036+0000","last_fullsized":"2023-09-23T20:50:53.293036+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":171,"log_dups_size":0,"ondisk_log_size":171,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:30:57.793132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032959200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1114,"num_read_kb":3170,"num_write":132,"num_write_kb":2190,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":682,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-09-23T20:50:52.790928+0000","last_change":"2023-09-23T20:47:09.198326+0000","last_active":"2023-09-23T20:50:52.790928+0000","last_peered":"2023-09-23T20:50:52.790928+0000","last_clean":"2023-09-23T20:50:52.790928+0000","last_became_active":"2023-09-23T20:47:09.198059+0000","last_became_peered":"2023-09-23T20:47:09.198059+0000","last_unstale":"2023-09-23T20:50:52.790928+0000","last_undegraded":"2023-09-23T20:50:52.790928+0000","last_fullsized":"2023-09-23T20:50:52.790928+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:38:53.903706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":677,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-09-23T20:50:52.790895+0000","last_change":"2023-09-23T20:48:48.146977+0000","last_active":"2023-09-23T20:50:52.790895+0000","last_peered":"2023-09-23T20:50:52.790895+0000","last_clean":"2023-09-23T20:50:52.790895+0000","last_became_active":"2023-09-23T20:48:48.146657+0000","last_became_peered":"2023-09-23T20:48:48.146657+0000","last_unstale":"2023-09-23T20:50:52.790895+0000","last_undegraded":"2023-09-23T20:50:52.790895+0000","last_fullsized":"2023-09-23T20:50:52.790895+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:38:41.782692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":779,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-09-23T20:50:53.246222+0000","last_change":"2023-09-23T20:41:08.380084+0000","last_active":"2023-09-23T20:50:53.246222+0000","last_peered":"2023-09-23T20:50:53.246222+0000","last_clean":"2023-09-23T20:50:53.246222+0000","last_became_active":"2023-09-23T20:38:05.610881+0000","last_became_peered":"2023-09-23T20:38:05.610881+0000","last_unstale":"2023-09-23T20:50:53.246222+0000","last_undegraded":"2023-09-23T20:50:53.246222+0000","last_fullsized":"2023-09-23T20:50:53.246222+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:41:08.379959+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:41:08.379959+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:19:39.547923+0000","scrub_duration":0.0019158420000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"664'175","reported_seq":1632,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-09-23T20:50:53.293264+0000","last_change":"2023-09-23T20:50:39.743889+0000","last_active":"2023-09-23T20:50:53.293264+0000","last_peered":"2023-09-23T20:50:53.293264+0000","last_clean":"2023-09-23T20:50:53.293264+0000","last_became_active":"2023-09-23T20:47:14.628341+0000","last_became_peered":"2023-09-23T20:47:14.628341+0000","last_unstale":"2023-09-23T20:50:53.293264+0000","last_undegraded":"2023-09-23T20:50:53.293264+0000","last_fullsized":"2023-09-23T20:50:53.293264+0000","mapping_epoch":558,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":559,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":175,"log_dups_size":0,"ondisk_log_size":175,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:42:16.291849+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000229686,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1422,"num_read_kb":2425,"num_write":116,"num_write_kb":664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":646,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-09-23T20:50:52.794400+0000","last_change":"2023-09-23T20:49:48.108294+0000","last_active":"2023-09-23T20:50:52.794400+0000","last_peered":"2023-09-23T20:50:52.794400+0000","last_clean":"2023-09-23T20:50:52.794400+0000","last_became_active":"2023-09-23T20:49:48.107916+0000","last_became_peered":"2023-09-23T20:49:48.107916+0000","last_unstale":"2023-09-23T20:50:52.794400+0000","last_undegraded":"2023-09-23T20:50:52.794400+0000","last_fullsized":"2023-09-23T20:50:52.794400+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:57:12.800814+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":623,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-09-23T20:50:52.790424+0000","last_change":"2023-09-23T20:49:48.119227+0000","last_active":"2023-09-23T20:50:52.790424+0000","last_peered":"2023-09-23T20:50:52.790424+0000","last_clean":"2023-09-23T20:50:52.790424+0000","last_became_active":"2023-09-23T20:49:48.118910+0000","last_became_peered":"2023-09-23T20:49:48.118910+0000","last_unstale":"2023-09-23T20:50:52.790424+0000","last_undegraded":"2023-09-23T20:50:52.790424+0000","last_fullsized":"2023-09-23T20:50:52.790424+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:18:40.836373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":705,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-09-23T20:50:53.292880+0000","last_change":"2023-09-23T20:48:51.172830+0000","last_active":"2023-09-23T20:50:53.292880+0000","last_peered":"2023-09-23T20:50:53.292880+0000","last_clean":"2023-09-23T20:50:53.292880+0000","last_became_active":"2023-09-23T20:48:51.172481+0000","last_became_peered":"2023-09-23T20:48:51.172481+0000","last_unstale":"2023-09-23T20:50:53.292880+0000","last_undegraded":"2023-09-23T20:50:53.292880+0000","last_fullsized":"2023-09-23T20:50:53.292880+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:47:34.561747+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:47:34.561747+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:30:32.531477+0000","scrub_duration":0.0087949929999999992,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":406,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-09-23T20:50:53.293556+0000","last_change":"2023-09-23T20:32:50.549695+0000","last_active":"2023-09-23T20:50:53.293556+0000","last_peered":"2023-09-23T20:50:53.293556+0000","last_clean":"2023-09-23T20:50:53.293556+0000","last_became_active":"2023-09-23T20:32:50.549431+0000","last_became_peered":"2023-09-23T20:32:50.549431+0000","last_unstale":"2023-09-23T20:50:53.293556+0000","last_undegraded":"2023-09-23T20:50:53.293556+0000","last_fullsized":"2023-09-23T20:50:53.293556+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:45:53.666569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"664'349","reported_seq":2029,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-09-23T20:50:52.794375+0000","last_change":"2023-09-23T20:50:39.737163+0000","last_active":"2023-09-23T20:50:52.794375+0000","last_peered":"2023-09-23T20:50:52.794375+0000","last_clean":"2023-09-23T20:50:52.794375+0000","last_became_active":"2023-09-23T20:47:11.621284+0000","last_became_peered":"2023-09-23T20:47:11.621284+0000","last_unstale":"2023-09-23T20:50:52.794375+0000","last_undegraded":"2023-09-23T20:50:52.794375+0000","last_fullsized":"2023-09-23T20:50:52.794375+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":349,"log_dups_size":0,"ondisk_log_size":349,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:43:50.658512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0014591350000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":2217,"num_read_kb":2864,"num_write":263,"num_write_kb":2168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":453,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-09-23T20:50:55.355388+0000","last_change":"2023-09-23T20:37:43.582105+0000","last_active":"2023-09-23T20:50:55.355388+0000","last_peered":"2023-09-23T20:50:55.355388+0000","last_clean":"2023-09-23T20:50:55.355388+0000","last_became_active":"2023-09-23T20:37:43.581860+0000","last_became_peered":"2023-09-23T20:37:43.581860+0000","last_unstale":"2023-09-23T20:50:55.355388+0000","last_undegraded":"2023-09-23T20:50:55.355388+0000","last_fullsized":"2023-09-23T20:50:55.355388+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:18:38.765486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"90'161","reported_seq":904,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-09-23T20:50:55.355089+0000","last_change":"2023-09-23T20:49:51.122217+0000","last_active":"2023-09-23T20:50:55.355089+0000","last_peered":"2023-09-23T20:50:55.355089+0000","last_clean":"2023-09-23T20:50:55.355089+0000","last_became_active":"2023-09-23T20:49:51.121893+0000","last_became_peered":"2023-09-23T20:49:51.121893+0000","last_unstale":"2023-09-23T20:50:55.355089+0000","last_undegraded":"2023-09-23T20:50:55.355089+0000","last_fullsized":"2023-09-23T20:50:55.355089+0000","mapping_epoch":639,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":640,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:34:54.504554+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:34:54.504554+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:00:40.029809+0000","scrub_duration":0.011176864,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"664'293","reported_seq":1426,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-09-23T20:50:53.246434+0000","last_change":"2023-09-23T20:50:39.740476+0000","last_active":"2023-09-23T20:50:53.246434+0000","last_peered":"2023-09-23T20:50:53.246434+0000","last_clean":"2023-09-23T20:50:53.246434+0000","last_became_active":"2023-09-23T20:48:51.217013+0000","last_became_peered":"2023-09-23T20:48:51.217013+0000","last_unstale":"2023-09-23T20:50:53.246434+0000","last_undegraded":"2023-09-23T20:50:53.246434+0000","last_fullsized":"2023-09-23T20:50:53.246434+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":293,"log_dups_size":0,"ondisk_log_size":293,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:40:42.297947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00027326800000000001,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1602,"num_read_kb":1486,"num_write":268,"num_write_kb":4284,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":48,"num_bytes_recovered":25165824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":685,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-09-23T20:50:52.790997+0000","last_change":"2023-09-23T20:47:09.317116+0000","last_active":"2023-09-23T20:50:52.790997+0000","last_peered":"2023-09-23T20:50:52.790997+0000","last_clean":"2023-09-23T20:50:52.790997+0000","last_became_active":"2023-09-23T20:47:09.205930+0000","last_became_peered":"2023-09-23T20:47:09.205930+0000","last_unstale":"2023-09-23T20:50:52.790997+0000","last_undegraded":"2023-09-23T20:50:52.790997+0000","last_fullsized":"2023-09-23T20:50:52.790997+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:04:55.528625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":135,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-09-23T20:50:53.292171+0000","last_change":"2023-09-23T20:49:48.120086+0000","last_active":"2023-09-23T20:50:53.292171+0000","last_peered":"2023-09-23T20:50:53.292171+0000","last_clean":"2023-09-23T20:50:53.292171+0000","last_became_active":"2023-09-23T20:49:48.119761+0000","last_became_peered":"2023-09-23T20:49:48.119761+0000","last_unstale":"2023-09-23T20:50:53.292171+0000","last_undegraded":"2023-09-23T20:50:53.292171+0000","last_fullsized":"2023-09-23T20:50:53.292171+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:56:12.469830+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"90'161","reported_seq":927,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-09-23T20:50:52.790598+0000","last_change":"2023-09-23T20:43:46.544750+0000","last_active":"2023-09-23T20:50:52.790598+0000","last_peered":"2023-09-23T20:50:52.790598+0000","last_clean":"2023-09-23T20:50:52.790598+0000","last_became_active":"2023-09-23T20:43:46.544395+0000","last_became_peered":"2023-09-23T20:43:46.544395+0000","last_unstale":"2023-09-23T20:50:52.790598+0000","last_undegraded":"2023-09-23T20:50:52.790598+0000","last_fullsized":"2023-09-23T20:50:52.790598+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:06.487351+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:06.487351+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:56.866168+0000","scrub_duration":0.001964379,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":643,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-09-23T20:50:52.790346+0000","last_change":"2023-09-23T20:47:09.197601+0000","last_active":"2023-09-23T20:50:52.790346+0000","last_peered":"2023-09-23T20:50:52.790346+0000","last_clean":"2023-09-23T20:50:52.790346+0000","last_became_active":"2023-09-23T20:47:09.196967+0000","last_became_peered":"2023-09-23T20:47:09.196967+0000","last_unstale":"2023-09-23T20:50:52.790346+0000","last_undegraded":"2023-09-23T20:50:52.790346+0000","last_fullsized":"2023-09-23T20:50:52.790346+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:06:56.826785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":89,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-09-23T20:50:55.355616+0000","last_change":"2023-09-23T20:48:48.158376+0000","last_active":"2023-09-23T20:50:55.355616+0000","last_peered":"2023-09-23T20:50:55.355616+0000","last_clean":"2023-09-23T20:50:55.355616+0000","last_became_active":"2023-09-23T20:48:48.158069+0000","last_became_peered":"2023-09-23T20:48:48.158069+0000","last_unstale":"2023-09-23T20:50:55.355616+0000","last_undegraded":"2023-09-23T20:50:55.355616+0000","last_fullsized":"2023-09-23T20:50:55.355616+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:34.658561+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"90'161","reported_seq":746,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-09-23T20:50:55.357425+0000","last_change":"2023-09-23T20:41:06.499511+0000","last_active":"2023-09-23T20:50:55.357425+0000","last_peered":"2023-09-23T20:50:55.357425+0000","last_clean":"2023-09-23T20:50:55.357425+0000","last_became_active":"2023-09-23T20:38:05.601874+0000","last_became_peered":"2023-09-23T20:38:05.601874+0000","last_unstale":"2023-09-23T20:50:55.357425+0000","last_undegraded":"2023-09-23T20:50:55.357425+0000","last_fullsized":"2023-09-23T20:50:55.357425+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:41:06.499396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:41:06.499396+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:31:34.059077+0000","scrub_duration":0.0019670310000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":143,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-09-23T20:50:53.292165+0000","last_change":"2023-09-23T20:49:48.128500+0000","last_active":"2023-09-23T20:50:53.292165+0000","last_peered":"2023-09-23T20:50:53.292165+0000","last_clean":"2023-09-23T20:50:53.292165+0000","last_became_active":"2023-09-23T20:49:48.122142+0000","last_became_peered":"2023-09-23T20:49:48.122142+0000","last_unstale":"2023-09-23T20:50:53.292165+0000","last_undegraded":"2023-09-23T20:50:53.292165+0000","last_fullsized":"2023-09-23T20:50:53.292165+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:46:11.349709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":125,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-09-23T20:50:53.292872+0000","last_change":"2023-09-23T20:47:09.186579+0000","last_active":"2023-09-23T20:50:53.292872+0000","last_peered":"2023-09-23T20:50:53.292872+0000","last_clean":"2023-09-23T20:50:53.292872+0000","last_became_active":"2023-09-23T20:47:09.186227+0000","last_became_peered":"2023-09-23T20:47:09.186227+0000","last_unstale":"2023-09-23T20:50:53.292872+0000","last_undegraded":"2023-09-23T20:50:53.292872+0000","last_fullsized":"2023-09-23T20:50:53.292872+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:45:39.485308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":143,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-09-23T20:50:53.292828+0000","last_change":"2023-09-23T20:49:48.126698+0000","last_active":"2023-09-23T20:50:53.292828+0000","last_peered":"2023-09-23T20:50:53.292828+0000","last_clean":"2023-09-23T20:50:53.292828+0000","last_became_active":"2023-09-23T20:49:48.120196+0000","last_became_peered":"2023-09-23T20:49:48.120196+0000","last_unstale":"2023-09-23T20:50:53.292828+0000","last_undegraded":"2023-09-23T20:50:53.292828+0000","last_fullsized":"2023-09-23T20:50:53.292828+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:17:45.039963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":125,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-09-23T20:50:53.291601+0000","last_change":"2023-09-23T20:47:09.181240+0000","last_active":"2023-09-23T20:50:53.291601+0000","last_peered":"2023-09-23T20:50:53.291601+0000","last_clean":"2023-09-23T20:50:53.291601+0000","last_became_active":"2023-09-23T20:47:09.181022+0000","last_became_peered":"2023-09-23T20:47:09.181022+0000","last_unstale":"2023-09-23T20:50:53.291601+0000","last_undegraded":"2023-09-23T20:50:53.291601+0000","last_fullsized":"2023-09-23T20:50:53.291601+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:38:59.191781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"90'161","reported_seq":924,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-09-23T20:50:53.292824+0000","last_change":"2023-09-23T20:38:05.591992+0000","last_active":"2023-09-23T20:50:53.292824+0000","last_peered":"2023-09-23T20:50:53.292824+0000","last_clean":"2023-09-23T20:50:53.292824+0000","last_became_active":"2023-09-23T20:38:05.591701+0000","last_became_peered":"2023-09-23T20:38:05.591701+0000","last_unstale":"2023-09-23T20:50:53.292824+0000","last_undegraded":"2023-09-23T20:50:53.292824+0000","last_fullsized":"2023-09-23T20:50:53.292824+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:02.384133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:02.384133+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:41.722542+0000","scrub_duration":0.001769098,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":513,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-09-23T20:50:55.357254+0000","last_change":"2023-09-23T20:48:48.158348+0000","last_active":"2023-09-23T20:50:55.357254+0000","last_peered":"2023-09-23T20:50:55.357254+0000","last_clean":"2023-09-23T20:50:55.357254+0000","last_became_active":"2023-09-23T20:48:48.157982+0000","last_became_peered":"2023-09-23T20:48:48.157982+0000","last_unstale":"2023-09-23T20:50:55.357254+0000","last_undegraded":"2023-09-23T20:50:55.357254+0000","last_fullsized":"2023-09-23T20:50:55.357254+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:03:48.078707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":479,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-09-23T20:50:55.356326+0000","last_change":"2023-09-23T20:48:48.159781+0000","last_active":"2023-09-23T20:50:55.356326+0000","last_peered":"2023-09-23T20:50:55.356326+0000","last_clean":"2023-09-23T20:50:55.356326+0000","last_became_active":"2023-09-23T20:48:48.159505+0000","last_became_peered":"2023-09-23T20:48:48.159505+0000","last_unstale":"2023-09-23T20:50:55.356326+0000","last_undegraded":"2023-09-23T20:50:55.356326+0000","last_fullsized":"2023-09-23T20:50:55.356326+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:25:18.100645+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"90'161","reported_seq":799,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-09-23T20:50:52.793940+0000","last_change":"2023-09-23T20:47:11.621758+0000","last_active":"2023-09-23T20:50:52.793940+0000","last_peered":"2023-09-23T20:50:52.793940+0000","last_clean":"2023-09-23T20:50:52.793940+0000","last_became_active":"2023-09-23T20:47:11.621345+0000","last_became_peered":"2023-09-23T20:47:11.621345+0000","last_unstale":"2023-09-23T20:50:52.793940+0000","last_undegraded":"2023-09-23T20:50:52.793940+0000","last_fullsized":"2023-09-23T20:50:52.793940+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:01.504973+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:01.504973+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:31:17.112393+0000","scrub_duration":0.00202344,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":749,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-09-23T20:50:52.791097+0000","last_change":"2023-09-23T20:49:48.125242+0000","last_active":"2023-09-23T20:50:52.791097+0000","last_peered":"2023-09-23T20:50:52.791097+0000","last_clean":"2023-09-23T20:50:52.791097+0000","last_became_active":"2023-09-23T20:49:48.111689+0000","last_became_peered":"2023-09-23T20:49:48.111689+0000","last_unstale":"2023-09-23T20:50:52.791097+0000","last_undegraded":"2023-09-23T20:50:52.791097+0000","last_fullsized":"2023-09-23T20:50:52.791097+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:49:38.401151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":4986,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":135,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-09-23T20:50:53.292210+0000","last_change":"2023-09-23T20:48:23.917302+0000","last_active":"2023-09-23T20:50:53.292210+0000","last_peered":"2023-09-23T20:50:53.292210+0000","last_clean":"2023-09-23T20:50:53.292210+0000","last_became_active":"2023-09-23T20:48:23.916956+0000","last_became_peered":"2023-09-23T20:48:23.916956+0000","last_unstale":"2023-09-23T20:50:53.292210+0000","last_undegraded":"2023-09-23T20:50:53.292210+0000","last_fullsized":"2023-09-23T20:50:53.292210+0000","mapping_epoch":577,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":578,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:42:27.989294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":118,"num_bytes_recovered":15844,"num_keys_recovered":80,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26691,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2048000,"data_stored":2033792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":8388627,"num_objects":5,"num_object_clones":0,"num_object_copies":15,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":11981,"num_read_kb":26415,"num_write":1690,"num_write_kb":25390,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":156,"num_bytes_recovered":25165843,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":25182208,"data_stored":25165900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6082,"internal_metadata":0},"log_size":1950,"ondisk_log_size":1950,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201754,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":619804,"kb_used_data":11220,"kb_used_omap":0,"kb_used_meta":608576,"kb_avail":93125348,"statfs":{"total":95995035648,"available":95360356352,"internally_reserved":0,"allocated":11489280,"data_stored":9700716,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":623181824},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:50:12 2023","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":1.325,"15min":1.9239999999999999},"min":{"1min":0.254,"5min":0.224,"15min":0.224},"max":{"1min":3.9580000000000002,"5min":10.975,"15min":18.315000000000001},"last":0.46200000000000002},{"interface":"front","average":{"1min":0.84999999999999998,"5min":1.5960000000000001,"15min":1.73},"min":{"1min":0.32100000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":4.2039999999999997,"5min":12.986000000000001,"15min":19.355},"last":0.68500000000000005}]},{"osd":1,"last update":"Sat Sep 23 20:50:18 2023","interfaces":[{"interface":"back","average":{"1min":0.74299999999999999,"5min":1.3680000000000001,"15min":1.849},"min":{"1min":0.33200000000000002,"5min":0.25900000000000001,"15min":0.24399999999999999},"max":{"1min":3.7770000000000001,"5min":14.249000000000001,"15min":49.965000000000003},"last":0.54700000000000004},{"interface":"front","average":{"1min":0.95899999999999996,"5min":1.645,"15min":2.3290000000000002},"min":{"1min":0.27900000000000003,"5min":0.27000000000000002,"15min":0.26700000000000002},"max":{"1min":5.3390000000000004,"5min":10.861000000000001,"15min":53.451999999999998},"last":0.73399999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:50:12 2023","interfaces":[{"interface":"back","average":{"1min":1.5680000000000001,"5min":1.7010000000000001,"15min":2.048},"min":{"1min":0.28799999999999998,"5min":0.28499999999999998,"15min":0.27800000000000002},"max":{"1min":10.461,"5min":12.933999999999999,"15min":16.965},"last":0.56499999999999995},{"interface":"front","average":{"1min":0.59499999999999997,"5min":1.571,"15min":2.0800000000000001},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":2.0049999999999999,"5min":7.9729999999999999,"15min":23.414000000000001},"last":0.49299999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:50:42 2023","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":1.464,"15min":1.8440000000000001},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.311},"max":{"1min":1.901,"5min":7.2880000000000003,"15min":23.352},"last":0.53500000000000003},{"interface":"front","average":{"1min":0.60199999999999998,"5min":1.635,"15min":2.0470000000000002},"min":{"1min":0.42099999999999999,"5min":0.35399999999999998,"15min":0.34300000000000003},"max":{"1min":1.8879999999999999,"5min":6.5369999999999999,"15min":20.969999999999999},"last":0.63700000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:50:42 2023","interfaces":[{"interface":"back","average":{"1min":0.44600000000000001,"5min":1.198,"15min":1.206},"min":{"1min":0.29599999999999999,"5min":0.254,"15min":0.254},"max":{"1min":0.78900000000000003,"5min":7.4109999999999996,"15min":18.364999999999998},"last":0.52000000000000002},{"interface":"front","average":{"1min":0.46999999999999997,"5min":1.456,"15min":1.49},"min":{"1min":0.33600000000000002,"5min":0.247,"15min":0.247},"max":{"1min":0.68700000000000006,"5min":27.425999999999998,"15min":27.425999999999998},"last":0.71199999999999997}]},{"osd":5,"last update":"Sat Sep 23 20:50:42 2023","interfaces":[{"interface":"back","average":{"1min":0.502,"5min":0.95399999999999996,"15min":1.248},"min":{"1min":0.316,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":1.6479999999999999,"5min":7.2270000000000003,"15min":15.005000000000001},"last":0.60599999999999998},{"interface":"front","average":{"1min":0.52200000000000002,"5min":1.0589999999999999,"15min":1.3839999999999999},"min":{"1min":0.36199999999999999,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.79800000000000004,"5min":7.335,"15min":18.390000000000001},"last":0.59199999999999997}]},{"osd":6,"last update":"Sat Sep 23 20:50:42 2023","interfaces":[{"interface":"back","average":{"1min":0.74199999999999999,"5min":1.405,"15min":1.5640000000000001},"min":{"1min":0.39200000000000002,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":4.4969999999999999,"5min":12.183,"15min":18.417000000000002},"last":0.624},{"interface":"front","average":{"1min":0.57499999999999996,"5min":1.397,"15min":1.4019999999999999},"min":{"1min":0.38,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":1.2709999999999999,"5min":11.404999999999999,"15min":15.369999999999999},"last":0.75900000000000001}]}]},{"osd":6,"up_from":47,"seq":201863463393,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":542924,"kb_used_data":15364,"kb_used_omap":0,"kb_used_meta":527552,"kb_avail":93202228,"statfs":{"total":95995035648,"available":95439081472,"internally_reserved":0,"allocated":15732736,"data_stored":13952703,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":540213248},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:50:06 2023","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":1.821,"15min":1.7509999999999999},"min":{"1min":0.255,"5min":0.23400000000000001,"15min":0.219},"max":{"1min":2.044,"5min":21.018000000000001,"15min":21.018000000000001},"last":0.46700000000000003},{"interface":"front","average":{"1min":1.04,"5min":1.9590000000000001,"15min":2.0249999999999999},"min":{"1min":0.27200000000000002,"5min":0.255,"15min":0.22700000000000001},"max":{"1min":8.7729999999999997,"5min":21.082000000000001,"15min":60.328000000000003},"last":0.70799999999999996}]},{"osd":1,"last update":"Sat Sep 23 20:50:23 2023","interfaces":[{"interface":"back","average":{"1min":0.64400000000000002,"5min":1.6359999999999999,"15min":1.865},"min":{"1min":0.30399999999999999,"5min":0.26000000000000001,"15min":0.249},"max":{"1min":4.266,"5min":8.9350000000000005,"15min":64.274000000000001},"last":0.53200000000000003},{"interface":"front","average":{"1min":0.52300000000000002,"5min":1.2470000000000001,"15min":1.5469999999999999},"min":{"1min":0.317,"5min":0.312,"15min":0.28000000000000003},"max":{"1min":0.85999999999999999,"5min":8.8070000000000004,"15min":17.786000000000001},"last":0.72399999999999998}]},{"osd":2,"last update":"Sat Sep 23 20:50:06 2023","interfaces":[{"interface":"back","average":{"1min":1.306,"5min":2.4049999999999998,"15min":2.1299999999999999},"min":{"1min":0.36699999999999999,"5min":0.29699999999999999,"15min":0.27700000000000002},"max":{"1min":8.9079999999999995,"5min":22.591000000000001,"15min":22.591000000000001},"last":0.63200000000000001},{"interface":"front","average":{"1min":1.4039999999999999,"5min":2.0169999999999999,"15min":1.8959999999999999},"min":{"1min":0.375,"5min":0.33900000000000002,"15min":0.30399999999999999},"max":{"1min":12.33,"5min":22.614999999999998,"15min":22.614999999999998},"last":0.59799999999999998}]},{"osd":3,"last update":"Sat Sep 23 20:49:57 2023","interfaces":[{"interface":"back","average":{"1min":1.1970000000000001,"5min":1.7849999999999999,"15min":1.895},"min":{"1min":0.32200000000000001,"5min":0.307,"15min":0.28399999999999997},"max":{"1min":4.4489999999999998,"5min":9.6859999999999999,"15min":11.664},"last":0.35099999999999998},{"interface":"front","average":{"1min":1.367,"5min":2.1480000000000001,"15min":2.109},"min":{"1min":0.38100000000000001,"5min":0.32700000000000001,"15min":0.30499999999999999},"max":{"1min":8.6140000000000008,"5min":11.6,"15min":11.913},"last":0.55900000000000005}]},{"osd":4,"last update":"Sat Sep 23 20:50:23 2023","interfaces":[{"interface":"back","average":{"1min":0.65100000000000002,"5min":1.0840000000000001,"15min":0.95399999999999996},"min":{"1min":0.311,"5min":0.28399999999999997,"15min":0.25600000000000001},"max":{"1min":4.2220000000000004,"5min":10.183999999999999,"15min":23.904},"last":0.69099999999999995},{"interface":"front","average":{"1min":0.63900000000000001,"5min":1.5640000000000001,"15min":1.2609999999999999},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.28399999999999997},"max":{"1min":3.2919999999999998,"5min":11.999000000000001,"15min":11.999000000000001},"last":0.74399999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:50:23 2023","interfaces":[{"interface":"back","average":{"1min":0.52100000000000002,"5min":1.0820000000000001,"15min":0.98999999999999999},"min":{"1min":0.34200000000000003,"5min":0.30499999999999999,"15min":0.253},"max":{"1min":0.86599999999999999,"5min":8.0440000000000005,"15min":8.0440000000000005},"last":0.77500000000000002},{"interface":"front","average":{"1min":0.72199999999999998,"5min":1.5,"15min":1.3220000000000001},"min":{"1min":0.36499999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":4.2329999999999997,"5min":9.3559999999999999,"15min":9.3559999999999999},"last":0.65400000000000003}]},{"osd":7,"last update":"Sat Sep 23 20:49:57 2023","interfaces":[{"interface":"back","average":{"1min":0.78000000000000003,"5min":1.127,"15min":1.0089999999999999},"min":{"1min":0.42099999999999999,"5min":0.32200000000000001,"15min":0.28399999999999997},"max":{"1min":2.964,"5min":8.9309999999999992,"15min":8.9309999999999992},"last":0.80800000000000005},{"interface":"front","average":{"1min":0.91800000000000004,"5min":3.3050000000000002,"15min":2.0019999999999998},"min":{"1min":0.38700000000000001,"5min":0.35999999999999999,"15min":0.34699999999999998},"max":{"1min":3.9620000000000002,"5min":159.91999999999999,"15min":159.91999999999999},"last":0.79100000000000004}]}]},{"osd":1,"up_from":266,"seq":1142461300992,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":823756,"kb_used_data":12932,"kb_used_omap":8,"kb_used_meta":810807,"kb_avail":92921396,"statfs":{"total":95995035648,"available":95151509504,"internally_reserved":0,"allocated":13242368,"data_stored":9907248,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8206,"internal_metadata":830267378},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:50:29 2023","interfaces":[{"interface":"back","average":{"1min":0.48799999999999999,"5min":1.917,"15min":2.214},"min":{"1min":0.28199999999999997,"5min":0.20499999999999999,"15min":0.20000000000000001},"max":{"1min":2.8809999999999998,"5min":20.292000000000002,"15min":20.292000000000002},"last":0.35599999999999998},{"interface":"front","average":{"1min":0.56799999999999995,"5min":1.8260000000000001,"15min":2.5760000000000001},"min":{"1min":0.28499999999999998,"5min":0.217,"15min":0.19900000000000001},"max":{"1min":2.6890000000000001,"5min":17.463000000000001,"15min":20.346},"last":0.33800000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:50:29 2023","interfaces":[{"interface":"back","average":{"1min":0.76900000000000002,"5min":2.0630000000000002,"15min":2.6110000000000002},"min":{"1min":0.25900000000000001,"5min":0.23100000000000001,"15min":0.216},"max":{"1min":2.7970000000000002,"5min":23.341000000000001,"15min":32.255000000000003},"last":0.313},{"interface":"front","average":{"1min":0.61099999999999999,"5min":2.173,"15min":2.1280000000000001},"min":{"1min":0.34899999999999998,"5min":0.249,"15min":0.20899999999999999},"max":{"1min":2.9279999999999999,"5min":34.002000000000002,"15min":34.002000000000002},"last":0.39500000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:50:29 2023","interfaces":[{"interface":"back","average":{"1min":0.63600000000000001,"5min":1.621,"15min":2.0979999999999999},"min":{"1min":0.316,"5min":0.28799999999999998,"15min":0.247},"max":{"1min":2.9590000000000001,"5min":17.366,"15min":17.366},"last":0.44800000000000001},{"interface":"front","average":{"1min":0.748,"5min":2,"15min":2.3839999999999999},"min":{"1min":0.35099999999999998,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":2.7330000000000001,"5min":17.343,"15min":19.315000000000001},"last":0.40200000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:50:29 2023","interfaces":[{"interface":"back","average":{"1min":1.079,"5min":1.8839999999999999,"15min":2.2229999999999999},"min":{"1min":0.42699999999999999,"5min":0.32600000000000001,"15min":0.28599999999999998},"max":{"1min":5.7110000000000003,"5min":9.5359999999999996,"15min":20.634},"last":1.7010000000000001},{"interface":"front","average":{"1min":0.69299999999999995,"5min":1.5600000000000001,"15min":1.7190000000000001},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.311},"max":{"1min":2.8370000000000002,"5min":9.8249999999999993,"15min":20.835999999999999},"last":1.637}]},{"osd":5,"last update":"Sat Sep 23 20:50:29 2023","interfaces":[{"interface":"back","average":{"1min":0.755,"5min":1.819,"15min":2.2090000000000001},"min":{"1min":0.42199999999999999,"5min":0.34699999999999998,"15min":0.308},"max":{"1min":3.1120000000000001,"5min":17.396999999999998,"15min":33.401000000000003},"last":1.724},{"interface":"front","average":{"1min":0.93999999999999995,"5min":1.962,"15min":2.4279999999999999},"min":{"1min":0.371,"5min":0.371,"15min":0.32500000000000001},"max":{"1min":3.0670000000000002,"5min":16.366,"15min":33.417000000000002},"last":1.736}]},{"osd":6,"last update":"Sat Sep 23 20:50:29 2023","interfaces":[{"interface":"back","average":{"1min":0.80400000000000005,"5min":2.105,"15min":2.1960000000000002},"min":{"1min":0.46899999999999997,"5min":0.316,"15min":0.316},"max":{"1min":3.0390000000000001,"5min":23.274000000000001,"15min":23.274000000000001},"last":1.7509999999999999},{"interface":"front","average":{"1min":1.0600000000000001,"5min":2.1440000000000001,"15min":2.4279999999999999},"min":{"1min":0.48399999999999999,"5min":0.40500000000000003,"15min":0.317},"max":{"1min":4.4690000000000003,"5min":9.6639999999999997,"15min":11.832000000000001},"last":1.669}]},{"osd":7,"last update":"Sat Sep 23 20:50:29 2023","interfaces":[{"interface":"back","average":{"1min":0.79400000000000004,"5min":1.9730000000000001,"15min":2.2389999999999999},"min":{"1min":0.51100000000000001,"5min":0.377,"15min":0.37},"max":{"1min":3.0819999999999999,"5min":17.433,"15min":18.437000000000001},"last":1.6870000000000001},{"interface":"front","average":{"1min":0.98299999999999998,"5min":2.1419999999999999,"15min":2.472},"min":{"1min":0.47699999999999998,"5min":0.41899999999999998,"15min":0.35999999999999999},"max":{"1min":3.0259999999999998,"5min":9.8369999999999997,"15min":12.021000000000001},"last":1.7609999999999999}]}]},{"osd":0,"up_from":94,"seq":403726926195,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":989984,"kb_used_data":12824,"kb_used_omap":7,"kb_used_meta":977144,"kb_avail":92755168,"statfs":{"total":95995035648,"available":94981292032,"internally_reserved":0,"allocated":13131776,"data_stored":9953520,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":1000595473},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:50:33 2023","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":3.1699999999999999,"15min":2.4630000000000001},"min":{"1min":0.25600000000000001,"5min":0.20100000000000001,"15min":0.17199999999999999},"max":{"1min":2.4089999999999998,"5min":152.387,"15min":152.387},"last":0.28599999999999998},{"interface":"front","average":{"1min":0.52100000000000002,"5min":3.1629999999999998,"15min":2.633},"min":{"1min":0.32300000000000001,"5min":0.218,"15min":0.20399999999999999},"max":{"1min":1.4259999999999999,"5min":152.84100000000001,"15min":152.84100000000001},"last":0.33200000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:50:11 2023","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":2.2490000000000001,"15min":2.6869999999999998},"min":{"1min":0.26700000000000002,"5min":0.23499999999999999,"15min":0.218},"max":{"1min":1.9410000000000001,"5min":14.538,"15min":62.726999999999997},"last":0.34799999999999998},{"interface":"front","average":{"1min":1.1419999999999999,"5min":2.226,"15min":2.6600000000000001},"min":{"1min":0.26100000000000001,"5min":0.217,"15min":0.217},"max":{"1min":10.159000000000001,"5min":14.167,"15min":62.689},"last":0.379}]},{"osd":3,"last update":"Sat Sep 23 20:50:46 2023","interfaces":[{"interface":"back","average":{"1min":0.442,"5min":2.6070000000000002,"15min":2.718},"min":{"1min":0.29299999999999998,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.97899999999999998,"5min":55.017000000000003,"15min":55.017000000000003},"last":0.309},{"interface":"front","average":{"1min":0.69099999999999995,"5min":2.673,"15min":2.6680000000000001},"min":{"1min":0.372,"5min":0.27400000000000002,"15min":0.21199999999999999},"max":{"1min":2.9140000000000001,"5min":55.665999999999997,"15min":55.665999999999997},"last":0.41499999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:50:11 2023","interfaces":[{"interface":"back","average":{"1min":0.99199999999999999,"5min":1.964,"15min":2.008},"min":{"1min":0.34300000000000003,"5min":0.29299999999999998,"15min":0.28899999999999998},"max":{"1min":4.2859999999999996,"5min":11.75,"15min":11.75},"last":0.52100000000000002},{"interface":"front","average":{"1min":1.2549999999999999,"5min":2.0230000000000001,"15min":1.853},"min":{"1min":0.42099999999999999,"5min":0.31,"15min":0.27400000000000002},"max":{"1min":5.4279999999999999,"5min":12.356999999999999,"15min":12.356999999999999},"last":0.45300000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:50:11 2023","interfaces":[{"interface":"back","average":{"1min":0.83999999999999997,"5min":1.8029999999999999,"15min":1.9399999999999999},"min":{"1min":0.38500000000000001,"5min":0.308,"15min":0.26500000000000001},"max":{"1min":3.012,"5min":9.3339999999999996,"15min":11.497999999999999},"last":0.56999999999999995},{"interface":"front","average":{"1min":0.88600000000000001,"5min":1.7210000000000001,"15min":2.2189999999999999},"min":{"1min":0.33000000000000002,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":3.181,"5min":10.089,"15min":26.228000000000002},"last":0.51200000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:50:11 2023","interfaces":[{"interface":"back","average":{"1min":1.395,"5min":2.1669999999999998,"15min":2.1360000000000001},"min":{"1min":0.441,"5min":0.374,"15min":0.34499999999999997},"max":{"1min":4.3540000000000001,"5min":9.2750000000000004,"15min":24.056000000000001},"last":0.53300000000000003},{"interface":"front","average":{"1min":1.391,"5min":1.974,"15min":2.1579999999999999},"min":{"1min":0.372,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":4.1669999999999998,"5min":12.427,"15min":26.268000000000001},"last":0.47699999999999998}]},{"osd":7,"last update":"Sat Sep 23 20:50:11 2023","interfaces":[{"interface":"back","average":{"1min":1.0900000000000001,"5min":2.1320000000000001,"15min":2.3199999999999998},"min":{"1min":0.39100000000000001,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":3.8919999999999999,"5min":16.553999999999998,"15min":26.177},"last":0.40500000000000003},{"interface":"front","average":{"1min":1.1579999999999999,"5min":2.1899999999999999,"15min":2.3300000000000001},"min":{"1min":0.45700000000000002,"5min":0.40200000000000002,"15min":0.34200000000000003},"max":{"1min":4.3209999999999997,"5min":16.649000000000001,"15min":16.649000000000001},"last":0.48999999999999999}]}]},{"osd":2,"up_from":104,"seq":446676599142,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":888736,"kb_used_data":12696,"kb_used_omap":11,"kb_used_meta":876020,"kb_avail":92856416,"statfs":{"total":95995035648,"available":95084969984,"internally_reserved":0,"allocated":13000704,"data_stored":9876054,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11878,"internal_metadata":897044890},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:50:12 2023","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":1.5900000000000001,"15min":1.603},"min":{"1min":0.26000000000000001,"5min":0.19500000000000001,"15min":0.17299999999999999},"max":{"1min":2.669,"5min":20.48,"15min":20.48},"last":0.27300000000000002},{"interface":"front","average":{"1min":0.51600000000000001,"5min":1.423,"15min":1.982},"min":{"1min":0.31900000000000001,"5min":0.245,"15min":0.19},"max":{"1min":1.4279999999999999,"5min":15.342000000000001,"15min":16.678999999999998},"last":0.318}]},{"osd":1,"last update":"Sat Sep 23 20:50:12 2023","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":1.5600000000000001,"15min":2.2429999999999999},"min":{"1min":0.24299999999999999,"5min":0.219,"15min":0.219},"max":{"1min":1.1279999999999999,"5min":21.690000000000001,"15min":46.405000000000001},"last":0.33100000000000002},{"interface":"front","average":{"1min":0.496,"5min":1.54,"15min":2.1930000000000001},"min":{"1min":0.26800000000000002,"5min":0.22,"15min":0.22},"max":{"1min":1.0449999999999999,"5min":10.191000000000001,"15min":46.459000000000003},"last":0.39500000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:50:42 2023","interfaces":[{"interface":"back","average":{"1min":0.40500000000000003,"5min":1.8799999999999999,"15min":2.2090000000000001},"min":{"1min":0.30399999999999999,"5min":0.28199999999999997,"15min":0.253},"max":{"1min":0.51300000000000001,"5min":30.283000000000001,"15min":30.283000000000001},"last":0.38},{"interface":"front","average":{"1min":0.45000000000000001,"5min":2.0070000000000001,"15min":1.6379999999999999},"min":{"1min":0.29399999999999998,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.64100000000000001,"5min":30.834,"15min":30.834},"last":0.35799999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:50:12 2023","interfaces":[{"interface":"back","average":{"1min":0.56799999999999995,"5min":1.5449999999999999,"15min":1.736},"min":{"1min":0.38400000000000001,"5min":0.29599999999999999,"15min":0.27500000000000002},"max":{"1min":0.79500000000000004,"5min":8.3469999999999995,"15min":12.025},"last":0.60599999999999998},{"interface":"front","average":{"1min":0.67400000000000004,"5min":1.627,"15min":1.819},"min":{"1min":0.42099999999999999,"5min":0.32400000000000001,"15min":0.30599999999999999},"max":{"1min":1.8520000000000001,"5min":12.662000000000001,"15min":19.440999999999999},"last":0.79700000000000004}]},{"osd":5,"last update":"Sat Sep 23 20:50:12 2023","interfaces":[{"interface":"back","average":{"1min":0.90100000000000002,"5min":1.3660000000000001,"15min":1.76},"min":{"1min":0.44800000000000001,"5min":0.33200000000000002,"15min":0.311},"max":{"1min":4.8570000000000002,"5min":8.5380000000000003,"15min":12.07},"last":0.75900000000000001},{"interface":"front","average":{"1min":0.93300000000000005,"5min":1.5349999999999999,"15min":1.7969999999999999},"min":{"1min":0.437,"5min":0.373,"15min":0.35299999999999998},"max":{"1min":4.8639999999999999,"5min":10.831,"15min":19.513000000000002},"last":0.77100000000000002}]},{"osd":6,"last update":"Sat Sep 23 20:50:12 2023","interfaces":[{"interface":"back","average":{"1min":0.63900000000000001,"5min":1.6619999999999999,"15min":1.9179999999999999},"min":{"1min":0.40200000000000002,"5min":0.34799999999999998,"15min":0.33500000000000002},"max":{"1min":1.177,"5min":10.791,"15min":19.486000000000001},"last":0.56100000000000005},{"interface":"front","average":{"1min":0.80400000000000005,"5min":1.486,"15min":1.415},"min":{"1min":0.433,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":2.6440000000000001,"5min":7.4180000000000001,"15min":11.747},"last":0.69399999999999995}]},{"osd":7,"last update":"Sat Sep 23 20:50:12 2023","interfaces":[{"interface":"back","average":{"1min":0.80100000000000005,"5min":1.8700000000000001,"15min":1.5229999999999999},"min":{"1min":0.52100000000000002,"5min":0.35399999999999998,"15min":0.34699999999999998},"max":{"1min":2.581,"5min":20.530000000000001,"15min":20.530000000000001},"last":0.71899999999999997},{"interface":"front","average":{"1min":0.67800000000000005,"5min":1.71,"15min":1.9830000000000001},"min":{"1min":0.45800000000000002,"5min":0.36899999999999999,"15min":0.34899999999999998},"max":{"1min":1.141,"5min":10.071999999999999,"15min":12.099},"last":0.63}]}]},{"osd":3,"up_from":109,"seq":468151435616,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1081312,"kb_used_data":8664,"kb_used_omap":4,"kb_used_meta":1072635,"kb_avail":92663840,"statfs":{"total":95995035648,"available":94887772160,"internally_reserved":0,"allocated":8871936,"data_stored":5753844,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4514,"internal_metadata":1098378846},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:50:51 2023","interfaces":[{"interface":"back","average":{"1min":0.42499999999999999,"5min":1.8380000000000001,"15min":1.9970000000000001},"min":{"1min":0.26000000000000001,"5min":0.217,"15min":0.20899999999999999},"max":{"1min":1.071,"5min":21.792999999999999,"15min":22.32},"last":0.34899999999999998},{"interface":"front","average":{"1min":0.45800000000000002,"5min":1.6220000000000001,"15min":2.073},"min":{"1min":0.32700000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":1.1020000000000001,"5min":18.579999999999998,"15min":22.805},"last":0.40699999999999997}]},{"osd":1,"last update":"Sat Sep 23 20:50:23 2023","interfaces":[{"interface":"back","average":{"1min":0.83899999999999997,"5min":1.321,"15min":1.7689999999999999},"min":{"1min":0.22700000000000001,"5min":0.21299999999999999,"15min":0.21099999999999999},"max":{"1min":10.24,"5min":11.298,"15min":17.529},"last":0.29099999999999998},{"interface":"front","average":{"1min":0.59499999999999997,"5min":1.454,"15min":2.036},"min":{"1min":0.26500000000000001,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":4.2480000000000002,"5min":11.380000000000001,"15min":14.19},"last":0.437}]},{"osd":2,"last update":"Sat Sep 23 20:50:51 2023","interfaces":[{"interface":"back","average":{"1min":0.42599999999999999,"5min":1.5900000000000001,"15min":2.3820000000000001},"min":{"1min":0.28000000000000003,"5min":0.254,"15min":0.23499999999999999},"max":{"1min":1.194,"5min":7.9210000000000003,"15min":68.206999999999994},"last":0.50600000000000001},{"interface":"front","average":{"1min":0.42999999999999999,"5min":1.4159999999999999,"15min":2.121},"min":{"1min":0.314,"5min":0.25600000000000001,"15min":0.245},"max":{"1min":0.73899999999999999,"5min":16.559000000000001,"15min":67.736999999999995},"last":0.42099999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:50:51 2023","interfaces":[{"interface":"back","average":{"1min":0.84599999999999997,"5min":1.458,"15min":1.5629999999999999},"min":{"1min":0.38300000000000001,"5min":0.35399999999999998,"15min":0.30499999999999999},"max":{"1min":6.4059999999999997,"5min":9.0259999999999998,"15min":22.157},"last":0.70199999999999996},{"interface":"front","average":{"1min":0.54600000000000004,"5min":1.603,"15min":1.9219999999999999},"min":{"1min":0.35099999999999998,"5min":0.33500000000000002,"15min":0.33000000000000002},"max":{"1min":0.81399999999999995,"5min":16.529,"15min":16.529},"last":0.60699999999999998}]},{"osd":5,"last update":"Sat Sep 23 20:50:51 2023","interfaces":[{"interface":"back","average":{"1min":0.88100000000000001,"5min":1.575,"15min":1.7150000000000001},"min":{"1min":0.41699999999999998,"5min":0.32400000000000001,"15min":0.32300000000000001},"max":{"1min":6.4800000000000004,"5min":8.9849999999999994,"15min":22.289999999999999},"last":0.64500000000000002},{"interface":"front","average":{"1min":0.60399999999999998,"5min":1.4159999999999999,"15min":1.6479999999999999},"min":{"1min":0.42899999999999999,"5min":0.36399999999999999,"15min":0.34100000000000003},"max":{"1min":1.135,"5min":6.1390000000000002,"15min":12.952},"last":0.66200000000000003}]},{"osd":6,"last update":"Sat Sep 23 20:50:51 2023","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":1.9430000000000001,"15min":1.8660000000000001},"min":{"1min":0.42799999999999999,"5min":0.39800000000000002,"15min":0.34300000000000003},"max":{"1min":1.163,"5min":8.6419999999999995,"15min":12.917},"last":0.57299999999999995},{"interface":"front","average":{"1min":0.57099999999999995,"5min":1.647,"15min":1.9750000000000001},"min":{"1min":0.47199999999999998,"5min":0.36699999999999999,"15min":0.34599999999999997},"max":{"1min":0.69999999999999996,"5min":16.497,"15min":16.497},"last":0.499}]},{"osd":7,"last update":"Sat Sep 23 20:50:51 2023","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":1.6140000000000001,"15min":2.1949999999999998},"min":{"1min":0.44600000000000001,"5min":0.42399999999999999,"15min":0.39900000000000002},"max":{"1min":0.77000000000000002,"5min":16.425000000000001,"15min":16.425000000000001},"last":0.47899999999999998},{"interface":"front","average":{"1min":0.89600000000000002,"5min":1.9039999999999999,"15min":1.8879999999999999},"min":{"1min":0.501,"5min":0.45600000000000002,"15min":0.40999999999999998},"max":{"1min":6.5259999999999998,"5min":11.891999999999999,"15min":22.202999999999999},"last":0.58699999999999997}]}]},{"osd":4,"up_from":30,"seq":128849019369,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":571976,"kb_used_data":6848,"kb_used_omap":0,"kb_used_meta":565120,"kb_avail":93173176,"statfs":{"total":95995035648,"available":95409332224,"internally_reserved":0,"allocated":7012352,"data_stored":5235595,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":578682880},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:50:07 2023","interfaces":[{"interface":"back","average":{"1min":0.63200000000000001,"5min":2.1230000000000002,"15min":1.974},"min":{"1min":0.217,"5min":0.217,"15min":0.217},"max":{"1min":3.351,"5min":73.272000000000006,"15min":73.272000000000006},"last":0.374},{"interface":"front","average":{"1min":0.85899999999999999,"5min":1.8919999999999999,"15min":2.0350000000000001},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.246},"max":{"1min":7.2359999999999998,"5min":29.754000000000001,"15min":29.754000000000001},"last":0.57699999999999996}]},{"osd":1,"last update":"Sat Sep 23 20:50:17 2023","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":1.544,"15min":1.7050000000000001},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.25900000000000001},"max":{"1min":1.9259999999999999,"5min":12.343,"15min":18.190999999999999},"last":0.51800000000000002},{"interface":"front","average":{"1min":0.74299999999999999,"5min":1.734,"15min":1.6459999999999999},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.26400000000000001},"max":{"1min":3.198,"5min":12.231999999999999,"15min":18.173999999999999},"last":0.50600000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:50:17 2023","interfaces":[{"interface":"back","average":{"1min":0.85599999999999998,"5min":1.7470000000000001,"15min":1.72},"min":{"1min":0.28000000000000003,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":3.2469999999999999,"5min":12.151999999999999,"15min":13.051},"last":0.41899999999999998},{"interface":"front","average":{"1min":0.72299999999999998,"5min":1.6899999999999999,"15min":1.986},"min":{"1min":0.28899999999999998,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":3.4319999999999999,"5min":12.458,"15min":12.458},"last":0.53200000000000003}]},{"osd":3,"last update":"Sat Sep 23 20:50:36 2023","interfaces":[{"interface":"back","average":{"1min":0.79700000000000004,"5min":1.796,"15min":1.8720000000000001},"min":{"1min":0.376,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":3.0840000000000001,"5min":11.598000000000001,"15min":12.147},"last":0.55000000000000004},{"interface":"front","average":{"1min":0.63900000000000001,"5min":1.9179999999999999,"15min":1.978},"min":{"1min":0.45600000000000002,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":1.323,"5min":12.191000000000001,"15min":17.169},"last":0.46899999999999997}]},{"osd":5,"last update":"Sat Sep 23 20:50:07 2023","interfaces":[{"interface":"back","average":{"1min":0.49399999999999999,"5min":1.393,"15min":1.1919999999999999},"min":{"1min":0.26100000000000001,"5min":0.25800000000000001,"15min":0.253},"max":{"1min":1.214,"5min":16.940000000000001,"15min":16.940000000000001},"last":0.44400000000000001},{"interface":"front","average":{"1min":0.53600000000000003,"5min":1.264,"15min":1.1819999999999999},"min":{"1min":0.26700000000000002,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":1.9610000000000001,"5min":6.5839999999999996,"15min":8.1440000000000001},"last":0.39200000000000002}]},{"osd":6,"last update":"Sat Sep 23 20:50:17 2023","interfaces":[{"interface":"back","average":{"1min":1.22,"5min":1.716,"15min":1.452},"min":{"1min":0.31900000000000001,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":7.4589999999999996,"5min":13.503,"15min":14.731999999999999},"last":0.60499999999999998},{"interface":"front","average":{"1min":0.72399999999999998,"5min":1.8080000000000001,"15min":1.4039999999999999},"min":{"1min":0.29899999999999999,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":4.0830000000000002,"5min":17.059000000000001,"15min":17.059000000000001},"last":0.48799999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:50:36 2023","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":1.4750000000000001,"15min":2.661},"min":{"1min":0.40200000000000002,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":1.012,"5min":11.228,"15min":313.50099999999998},"last":0.629},{"interface":"front","average":{"1min":0.57899999999999996,"5min":1.5,"15min":2.3290000000000002},"min":{"1min":0.438,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.94799999999999995,"5min":17.093,"15min":312.67500000000001},"last":0.56699999999999995}]}]},{"osd":5,"up_from":0,"seq":167503725030,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4722,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1360,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":8392704,"data_stored":8388627,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8206,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7156,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3154,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:51:01.754 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:51:01.804 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:51:02.054 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:01 smithi149 ceph-mon[170394]: failed to encode map e670 with expected crc 2023-09-23T20:51:02.054 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:01 smithi149 ceph-mon[170394]: pgmap v1633: 103 pgs: 103 active+clean; 8.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail 2023-09-23T20:51:02.055 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:01 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1d", "id": [4, 2]}]: dispatch 2023-09-23T20:51:02.055 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:01 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2023-09-23T20:51:02.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:01 smithi018 ceph-mon[179647]: failed to encode map e670 with expected crc 2023-09-23T20:51:02.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:01 smithi018 ceph-mon[179647]: pgmap v1633: 103 pgs: 103 active+clean; 8.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail 2023-09-23T20:51:02.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:01 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1d", "id": [4, 2]}]: dispatch 2023-09-23T20:51:02.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:01 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2023-09-23T20:51:02.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:01 smithi018 ceph-mon[184144]: failed to encode map e670 with expected crc 2023-09-23T20:51:02.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:01 smithi018 ceph-mon[184144]: pgmap v1633: 103 pgs: 103 active+clean; 8.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail 2023-09-23T20:51:02.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:01 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1d", "id": [4, 2]}]: dispatch 2023-09-23T20:51:02.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:01 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2023-09-23T20:51:02.365 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:51:02.888 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:02 smithi018 ceph-mon[179647]: from='client.47888 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:51:02.888 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:02 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1d", "id": [4, 2]}]': finished 2023-09-23T20:51:02.889 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:02 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]': finished 2023-09-23T20:51:02.889 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:02 smithi018 ceph-mon[179647]: osdmap e671: 8 total, 8 up, 7 in 2023-09-23T20:51:02.889 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:02 smithi018 ceph-mon[179647]: failed to encode map e671 with expected crc 2023-09-23T20:51:02.889 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:02 smithi018 ceph-mon[179647]: failed to encode map e671 with expected crc 2023-09-23T20:51:02.889 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:02 smithi018 ceph-mon[179647]: failed to encode map e671 with expected crc 2023-09-23T20:51:02.889 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:02 smithi018 ceph-mon[179647]: failed to encode map e671 with expected crc 2023-09-23T20:51:02.889 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:02 smithi018 ceph-mon[179647]: failed to encode map e671 with expected crc 2023-09-23T20:51:02.889 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:02 smithi018 ceph-mon[179647]: failed to encode map e671 with expected crc 2023-09-23T20:51:02.889 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:02 smithi018 ceph-mon[184144]: from='client.47888 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:51:02.889 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:02 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1d", "id": [4, 2]}]': finished 2023-09-23T20:51:02.890 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:02 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]': finished 2023-09-23T20:51:02.890 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:02 smithi018 ceph-mon[184144]: osdmap e671: 8 total, 8 up, 7 in 2023-09-23T20:51:02.890 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:02 smithi018 ceph-mon[184144]: failed to encode map e671 with expected crc 2023-09-23T20:51:02.890 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:02 smithi018 ceph-mon[184144]: failed to encode map e671 with expected crc 2023-09-23T20:51:02.890 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:02 smithi018 ceph-mon[184144]: failed to encode map e671 with expected crc 2023-09-23T20:51:02.890 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:02 smithi018 ceph-mon[184144]: failed to encode map e671 with expected crc 2023-09-23T20:51:02.890 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:02 smithi018 ceph-mon[184144]: failed to encode map e671 with expected crc 2023-09-23T20:51:02.890 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:02 smithi018 ceph-mon[184144]: failed to encode map e671 with expected crc 2023-09-23T20:51:03.029 INFO:teuthology.orchestra.run.smithi149.stdout:{ 2023-09-23T20:51:03.029 INFO:teuthology.orchestra.run.smithi149.stdout: "ops": [], 2023-09-23T20:51:03.029 INFO:teuthology.orchestra.run.smithi149.stdout: "num_ops": 0 2023-09-23T20:51:03.029 INFO:teuthology.orchestra.run.smithi149.stdout:} 2023-09-23T20:51:03.169 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:02 smithi149 ceph-mon[170394]: from='client.47888 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:51:03.169 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:02 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.1d", "id": [4, 2]}]': finished 2023-09-23T20:51:03.169 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:02 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]': finished 2023-09-23T20:51:03.169 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:02 smithi149 ceph-mon[170394]: osdmap e671: 8 total, 8 up, 7 in 2023-09-23T20:51:03.169 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:02 smithi149 ceph-mon[170394]: failed to encode map e671 with expected crc 2023-09-23T20:51:03.170 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:02 smithi149 ceph-mon[170394]: failed to encode map e671 with expected crc 2023-09-23T20:51:03.170 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:02 smithi149 ceph-mon[170394]: failed to encode map e671 with expected crc 2023-09-23T20:51:03.170 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:02 smithi149 ceph-mon[170394]: failed to encode map e671 with expected crc 2023-09-23T20:51:03.170 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:02 smithi149 ceph-mon[170394]: failed to encode map e671 with expected crc 2023-09-23T20:51:03.170 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:02 smithi149 ceph-mon[170394]: failed to encode map e671 with expected crc 2023-09-23T20:51:03.454 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T20:51:03.895 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:51:03.896 INFO:teuthology.orchestra.run.smithi018.stderr:dumped all 2023-09-23T20:51:04.058 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:51:04.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:03 smithi018 ceph-mon[184144]: pgmap v1635: 103 pgs: 103 active+clean; 8.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail 2023-09-23T20:51:04.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:03 smithi018 ceph-mon[184144]: osdmap e672: 8 total, 8 up, 7 in 2023-09-23T20:51:04.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:03 smithi018 ceph-mon[184144]: failed to encode map e672 with expected crc 2023-09-23T20:51:04.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:03 smithi018 ceph-mon[184144]: failed to encode map e672 with expected crc 2023-09-23T20:51:04.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:03 smithi018 ceph-mon[184144]: failed to encode map e672 with expected crc 2023-09-23T20:51:04.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:03 smithi018 ceph-mon[179647]: pgmap v1635: 103 pgs: 103 active+clean; 8.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail 2023-09-23T20:51:04.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:03 smithi018 ceph-mon[179647]: osdmap e672: 8 total, 8 up, 7 in 2023-09-23T20:51:04.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:03 smithi018 ceph-mon[179647]: failed to encode map e672 with expected crc 2023-09-23T20:51:04.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:03 smithi018 ceph-mon[179647]: failed to encode map e672 with expected crc 2023-09-23T20:51:04.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:03 smithi018 ceph-mon[179647]: failed to encode map e672 with expected crc 2023-09-23T20:51:04.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:03 smithi149 ceph-mon[170394]: pgmap v1635: 103 pgs: 103 active+clean; 8.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail 2023-09-23T20:51:04.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:03 smithi149 ceph-mon[170394]: osdmap e672: 8 total, 8 up, 7 in 2023-09-23T20:51:04.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:03 smithi149 ceph-mon[170394]: failed to encode map e672 with expected crc 2023-09-23T20:51:04.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:03 smithi149 ceph-mon[170394]: failed to encode map e672 with expected crc 2023-09-23T20:51:04.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:03 smithi149 ceph-mon[170394]: failed to encode map e672 with expected crc 2023-09-23T20:51:04.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:04 smithi149 ceph-mon[170394]: failed to encode map e672 with expected crc 2023-09-23T20:51:04.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:04 smithi149 ceph-mon[170394]: from='client.38109 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:51:05.141 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:51:04 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:51:04] "GET /metrics HTTP/1.1" 200 34159 "" "Prometheus/2.43.0" 2023-09-23T20:51:05.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:04 smithi018 ceph-mon[184144]: failed to encode map e672 with expected crc 2023-09-23T20:51:05.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:04 smithi018 ceph-mon[184144]: from='client.38109 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:51:05.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:04 smithi018 ceph-mon[179647]: failed to encode map e672 with expected crc 2023-09-23T20:51:05.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:04 smithi018 ceph-mon[179647]: from='client.38109 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:51:05.281 INFO:teuthology.orchestra.run.smithi149.stdout:{ 2023-09-23T20:51:05.281 INFO:teuthology.orchestra.run.smithi149.stdout: "ops": [], 2023-09-23T20:51:05.281 INFO:teuthology.orchestra.run.smithi149.stdout: "complaint_time": 30, 2023-09-23T20:51:05.281 INFO:teuthology.orchestra.run.smithi149.stdout: "num_blocked_ops": 0 2023-09-23T20:51:05.281 INFO:teuthology.orchestra.run.smithi149.stdout:} 2023-09-23T20:51:05.717 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T20:51:05.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:05 smithi149 ceph-mon[170394]: pgmap v1637: 103 pgs: 103 active+clean; 8.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail 2023-09-23T20:51:06.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:05 smithi018 ceph-mon[184144]: pgmap v1637: 103 pgs: 103 active+clean; 8.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail 2023-09-23T20:51:06.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:05 smithi018 ceph-mon[179647]: pgmap v1637: 103 pgs: 103 active+clean; 8.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail 2023-09-23T20:51:06.317 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:51:06.424 INFO:teuthology.orchestra.run.smithi018.stdout:{"pg_ready":true,"pg_map":{"version":1636,"stamp":"2023-09-23T20:51:02.805449+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":8984499,"num_objects":29,"num_object_clones":0,"num_object_copies":87,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":29,"num_whiteouts":0,"num_read":11981,"num_read_kb":26415,"num_write":1751,"num_write_kb":25419,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":276,"num_bytes_recovered":25707063,"num_keys_recovered":80,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6996,"ondisk_log_size":6996,"up":309,"acting":309,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":309,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":5518524,"kb_used_data":80580,"kb_used_omap":32,"kb_used_meta":5437855,"kb_avail":650697540,"statfs":{"total":671965249536,"available":666314280960,"internally_reserved":0,"allocated":82513920,"data_stored":64410624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32773,"internal_metadata":5568364539},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.429995"},"pg_stats":[{"pgid":"1.18","version":"90'161","reported_seq":733,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-09-23T20:50:55.355281+0000","last_change":"2023-09-23T20:49:51.130089+0000","last_active":"2023-09-23T20:50:55.355281+0000","last_peered":"2023-09-23T20:50:55.355281+0000","last_clean":"2023-09-23T20:50:55.355281+0000","last_became_active":"2023-09-23T20:49:51.124899+0000","last_became_peered":"2023-09-23T20:49:51.124899+0000","last_unstale":"2023-09-23T20:50:55.355281+0000","last_undegraded":"2023-09-23T20:50:55.355281+0000","last_fullsized":"2023-09-23T20:50:55.355281+0000","mapping_epoch":639,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":640,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:19.677209+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:19.677209+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:58:59.316978+0000","scrub_duration":0.033122453000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"69'1","reported_seq":146,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-09-23T20:50:55.241634+0000","last_change":"2023-09-23T20:48:48.148250+0000","last_active":"2023-09-23T20:50:55.241634+0000","last_peered":"2023-09-23T20:50:55.241634+0000","last_clean":"2023-09-23T20:50:55.241634+0000","last_became_active":"2023-09-23T20:48:48.144271+0000","last_became_peered":"2023-09-23T20:48:48.144271+0000","last_unstale":"2023-09-23T20:50:55.241634+0000","last_undegraded":"2023-09-23T20:50:55.241634+0000","last_fullsized":"2023-09-23T20:50:55.241634+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:59:36.502986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":514,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-09-23T20:50:55.356759+0000","last_change":"2023-09-23T20:49:02.703099+0000","last_active":"2023-09-23T20:50:55.356759+0000","last_peered":"2023-09-23T20:50:55.356759+0000","last_clean":"2023-09-23T20:50:55.356759+0000","last_became_active":"2023-09-23T20:49:02.702753+0000","last_became_peered":"2023-09-23T20:49:02.702753+0000","last_unstale":"2023-09-23T20:50:55.356759+0000","last_undegraded":"2023-09-23T20:50:55.356759+0000","last_fullsized":"2023-09-23T20:50:55.356759+0000","mapping_epoch":603,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":604,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:24:30.834235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"90'161","reported_seq":692,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-09-23T20:50:55.241603+0000","last_change":"2023-09-23T20:49:51.121955+0000","last_active":"2023-09-23T20:50:55.241603+0000","last_peered":"2023-09-23T20:50:55.241603+0000","last_clean":"2023-09-23T20:50:55.241603+0000","last_became_active":"2023-09-23T20:49:51.121641+0000","last_became_peered":"2023-09-23T20:49:51.121641+0000","last_unstale":"2023-09-23T20:50:55.241603+0000","last_undegraded":"2023-09-23T20:50:55.241603+0000","last_fullsized":"2023-09-23T20:50:55.241603+0000","mapping_epoch":639,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":640,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:47:33.523187+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:36:05.442288+0000","last_clean_scrub_stamp":"2023-09-23T20:47:33.523187+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:05:36.252341+0000","scrub_duration":0.013532127,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,2],"acting":[4,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":659,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-09-23T20:50:52.793482+0000","last_change":"2023-09-23T20:47:09.173501+0000","last_active":"2023-09-23T20:50:52.793482+0000","last_peered":"2023-09-23T20:50:52.793482+0000","last_clean":"2023-09-23T20:50:52.793482+0000","last_became_active":"2023-09-23T20:47:09.173160+0000","last_became_peered":"2023-09-23T20:47:09.173160+0000","last_unstale":"2023-09-23T20:50:52.793482+0000","last_undegraded":"2023-09-23T20:50:52.793482+0000","last_fullsized":"2023-09-23T20:50:52.793482+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:44:36.132093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":629,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-09-23T20:50:52.791085+0000","last_change":"2023-09-23T20:43:46.521939+0000","last_active":"2023-09-23T20:50:52.791085+0000","last_peered":"2023-09-23T20:50:52.791085+0000","last_clean":"2023-09-23T20:50:52.791085+0000","last_became_active":"2023-09-23T20:43:46.521185+0000","last_became_peered":"2023-09-23T20:43:46.521185+0000","last_unstale":"2023-09-23T20:50:52.791085+0000","last_undegraded":"2023-09-23T20:50:52.791085+0000","last_fullsized":"2023-09-23T20:50:52.791085+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:59:25.045627+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"90'161","reported_seq":557,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-09-23T20:50:55.241518+0000","last_change":"2023-09-23T20:47:31.475001+0000","last_active":"2023-09-23T20:50:55.241518+0000","last_peered":"2023-09-23T20:50:55.241518+0000","last_clean":"2023-09-23T20:50:55.241518+0000","last_became_active":"2023-09-23T20:47:11.617942+0000","last_became_peered":"2023-09-23T20:47:11.617942+0000","last_unstale":"2023-09-23T20:50:55.241518+0000","last_undegraded":"2023-09-23T20:50:55.241518+0000","last_fullsized":"2023-09-23T20:50:55.241518+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:47:31.474914+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:47:31.474914+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:52:05.169219+0000","scrub_duration":0.0026262870000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":467,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-09-23T20:50:55.355540+0000","last_change":"2023-09-23T20:37:43.584414+0000","last_active":"2023-09-23T20:50:55.355540+0000","last_peered":"2023-09-23T20:50:55.355540+0000","last_clean":"2023-09-23T20:50:55.355540+0000","last_became_active":"2023-09-23T20:37:43.584116+0000","last_became_peered":"2023-09-23T20:37:43.584116+0000","last_unstale":"2023-09-23T20:50:55.355540+0000","last_undegraded":"2023-09-23T20:50:55.355540+0000","last_fullsized":"2023-09-23T20:50:55.355540+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:17:54.258172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":294,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-09-23T20:50:55.182145+0000","last_change":"2023-09-23T20:49:48.119179+0000","last_active":"2023-09-23T20:50:55.182145+0000","last_peered":"2023-09-23T20:50:55.182145+0000","last_clean":"2023-09-23T20:50:55.182145+0000","last_became_active":"2023-09-23T20:49:48.118849+0000","last_became_peered":"2023-09-23T20:49:48.118849+0000","last_unstale":"2023-09-23T20:50:55.182145+0000","last_undegraded":"2023-09-23T20:50:55.182145+0000","last_fullsized":"2023-09-23T20:50:55.182145+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:50:02.578569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"90'161","reported_seq":902,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-09-23T20:50:53.794477+0000","last_change":"2023-09-23T20:49:51.124760+0000","last_active":"2023-09-23T20:50:53.794477+0000","last_peered":"2023-09-23T20:50:53.794477+0000","last_clean":"2023-09-23T20:50:53.794477+0000","last_became_active":"2023-09-23T20:49:51.124406+0000","last_became_peered":"2023-09-23T20:49:51.124406+0000","last_unstale":"2023-09-23T20:50:53.794477+0000","last_undegraded":"2023-09-23T20:50:53.794477+0000","last_fullsized":"2023-09-23T20:50:53.794477+0000","mapping_epoch":639,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":640,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:03.411025+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:03.411025+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:28:15.568088+0000","scrub_duration":0.001802922,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":670,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-09-23T20:50:52.791005+0000","last_change":"2023-09-23T20:49:48.111285+0000","last_active":"2023-09-23T20:50:52.791005+0000","last_peered":"2023-09-23T20:50:52.791005+0000","last_clean":"2023-09-23T20:50:52.791005+0000","last_became_active":"2023-09-23T20:49:48.110934+0000","last_became_peered":"2023-09-23T20:49:48.110934+0000","last_unstale":"2023-09-23T20:50:52.791005+0000","last_undegraded":"2023-09-23T20:50:52.791005+0000","last_fullsized":"2023-09-23T20:50:52.791005+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:19:18.339128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":104,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-09-23T20:50:55.355314+0000","last_change":"2023-09-23T20:48:23.918092+0000","last_active":"2023-09-23T20:50:55.355314+0000","last_peered":"2023-09-23T20:50:55.355314+0000","last_clean":"2023-09-23T20:50:55.355314+0000","last_became_active":"2023-09-23T20:48:23.917386+0000","last_became_peered":"2023-09-23T20:48:23.917386+0000","last_unstale":"2023-09-23T20:50:55.355314+0000","last_undegraded":"2023-09-23T20:50:55.355314+0000","last_fullsized":"2023-09-23T20:50:55.355314+0000","mapping_epoch":577,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":578,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:06:09.587517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"90'161","reported_seq":672,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-09-23T20:50:55.355382+0000","last_change":"2023-09-23T20:49:36.717905+0000","last_active":"2023-09-23T20:50:55.355382+0000","last_peered":"2023-09-23T20:50:55.355382+0000","last_clean":"2023-09-23T20:50:55.355382+0000","last_became_active":"2023-09-23T20:48:51.169314+0000","last_became_peered":"2023-09-23T20:48:51.169314+0000","last_unstale":"2023-09-23T20:50:55.355382+0000","last_undegraded":"2023-09-23T20:50:55.355382+0000","last_fullsized":"2023-09-23T20:50:55.355382+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:49:36.717737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:49:36.717737+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:00:49.213712+0000","scrub_duration":0.0049746089999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":516,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-09-23T20:50:53.794654+0000","last_change":"2023-09-23T20:49:48.123820+0000","last_active":"2023-09-23T20:50:53.794654+0000","last_peered":"2023-09-23T20:50:53.794654+0000","last_clean":"2023-09-23T20:50:53.794654+0000","last_became_active":"2023-09-23T20:49:48.123514+0000","last_became_peered":"2023-09-23T20:49:48.123514+0000","last_unstale":"2023-09-23T20:50:53.794654+0000","last_undegraded":"2023-09-23T20:50:53.794654+0000","last_fullsized":"2023-09-23T20:50:53.794654+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:43:13.460464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":498,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-09-23T20:50:55.356398+0000","last_change":"2023-09-23T20:38:02.542034+0000","last_active":"2023-09-23T20:50:55.356398+0000","last_peered":"2023-09-23T20:50:55.356398+0000","last_clean":"2023-09-23T20:50:55.356398+0000","last_became_active":"2023-09-23T20:38:02.541575+0000","last_became_peered":"2023-09-23T20:38:02.541575+0000","last_unstale":"2023-09-23T20:50:55.356398+0000","last_undegraded":"2023-09-23T20:50:55.356398+0000","last_fullsized":"2023-09-23T20:50:55.356398+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:50:54.591529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"90'161","reported_seq":813,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-09-23T20:50:52.790795+0000","last_change":"2023-09-23T20:49:51.125057+0000","last_active":"2023-09-23T20:50:52.790795+0000","last_peered":"2023-09-23T20:50:52.790795+0000","last_clean":"2023-09-23T20:50:52.790795+0000","last_became_active":"2023-09-23T20:49:51.124616+0000","last_became_peered":"2023-09-23T20:49:51.124616+0000","last_unstale":"2023-09-23T20:50:52.790795+0000","last_undegraded":"2023-09-23T20:50:52.790795+0000","last_fullsized":"2023-09-23T20:50:52.790795+0000","mapping_epoch":639,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":640,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:14.669831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:14.669831+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:09:01.736519+0000","scrub_duration":0.0083441460000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":729,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-09-23T20:50:53.794810+0000","last_change":"2023-09-23T20:47:10.837423+0000","last_active":"2023-09-23T20:50:53.794810+0000","last_peered":"2023-09-23T20:50:53.794810+0000","last_clean":"2023-09-23T20:50:53.794810+0000","last_became_active":"2023-09-23T20:47:09.204324+0000","last_became_peered":"2023-09-23T20:47:09.204324+0000","last_unstale":"2023-09-23T20:50:53.794810+0000","last_undegraded":"2023-09-23T20:50:53.794810+0000","last_fullsized":"2023-09-23T20:50:53.794810+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":554,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:19.108914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":540,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":203,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-09-23T20:50:52.793468+0000","last_change":"2023-09-23T20:48:48.145175+0000","last_active":"2023-09-23T20:50:52.793468+0000","last_peered":"2023-09-23T20:50:52.793468+0000","last_clean":"2023-09-23T20:50:52.793468+0000","last_became_active":"2023-09-23T20:48:48.144841+0000","last_became_peered":"2023-09-23T20:48:48.144841+0000","last_unstale":"2023-09-23T20:50:52.793468+0000","last_undegraded":"2023-09-23T20:50:52.793468+0000","last_fullsized":"2023-09-23T20:50:52.793468+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:27:40.802917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"90'161","reported_seq":797,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-09-23T20:50:52.794161+0000","last_change":"2023-09-23T20:36:02.548387+0000","last_active":"2023-09-23T20:50:52.794161+0000","last_peered":"2023-09-23T20:50:52.794161+0000","last_clean":"2023-09-23T20:50:52.794161+0000","last_became_active":"2023-09-23T20:35:26.644787+0000","last_became_peered":"2023-09-23T20:35:26.644787+0000","last_unstale":"2023-09-23T20:50:52.794161+0000","last_undegraded":"2023-09-23T20:50:52.794161+0000","last_fullsized":"2023-09-23T20:50:52.794161+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:02.548285+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:02.548285+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:37:15.216553+0000","scrub_duration":0.0015514459999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":136,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-09-23T20:50:55.241685+0000","last_change":"2023-09-23T20:47:09.340979+0000","last_active":"2023-09-23T20:50:55.241685+0000","last_peered":"2023-09-23T20:50:55.241685+0000","last_clean":"2023-09-23T20:50:55.241685+0000","last_became_active":"2023-09-23T20:47:09.231222+0000","last_became_peered":"2023-09-23T20:47:09.231222+0000","last_unstale":"2023-09-23T20:50:55.241685+0000","last_undegraded":"2023-09-23T20:50:55.241685+0000","last_fullsized":"2023-09-23T20:50:55.241685+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:07:50.620006+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":126,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-09-23T20:50:55.241727+0000","last_change":"2023-09-23T20:47:09.186315+0000","last_active":"2023-09-23T20:50:55.241727+0000","last_peered":"2023-09-23T20:50:55.241727+0000","last_clean":"2023-09-23T20:50:55.241727+0000","last_became_active":"2023-09-23T20:47:09.184580+0000","last_became_peered":"2023-09-23T20:47:09.184580+0000","last_unstale":"2023-09-23T20:50:55.241727+0000","last_undegraded":"2023-09-23T20:50:55.241727+0000","last_fullsized":"2023-09-23T20:50:55.241727+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:44:35.553948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"90'161","reported_seq":942,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-09-23T20:50:52.790699+0000","last_change":"2023-09-23T20:49:51.148240+0000","last_active":"2023-09-23T20:50:52.790699+0000","last_peered":"2023-09-23T20:50:52.790699+0000","last_clean":"2023-09-23T20:50:52.790699+0000","last_became_active":"2023-09-23T20:49:51.147475+0000","last_became_peered":"2023-09-23T20:49:51.147475+0000","last_unstale":"2023-09-23T20:50:52.790699+0000","last_undegraded":"2023-09-23T20:50:52.790699+0000","last_fullsized":"2023-09-23T20:50:52.790699+0000","mapping_epoch":639,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":640,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:43.578666+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:43.578666+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:02:42.273671+0000","scrub_duration":0.0021890379999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":760,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-09-23T20:50:53.794291+0000","last_change":"2023-09-23T20:48:48.193666+0000","last_active":"2023-09-23T20:50:53.794291+0000","last_peered":"2023-09-23T20:50:53.794291+0000","last_clean":"2023-09-23T20:50:53.794291+0000","last_became_active":"2023-09-23T20:48:48.152081+0000","last_became_peered":"2023-09-23T20:48:48.152081+0000","last_unstale":"2023-09-23T20:50:53.794291+0000","last_undegraded":"2023-09-23T20:50:53.794291+0000","last_fullsized":"2023-09-23T20:50:53.794291+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:00:19.508124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":272,"num_keys_recovered":80,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":427,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-09-23T20:50:55.356102+0000","last_change":"2023-09-23T20:49:48.119365+0000","last_active":"2023-09-23T20:50:55.356102+0000","last_peered":"2023-09-23T20:50:55.356102+0000","last_clean":"2023-09-23T20:50:55.356102+0000","last_became_active":"2023-09-23T20:49:48.119009+0000","last_became_peered":"2023-09-23T20:49:48.119009+0000","last_unstale":"2023-09-23T20:50:55.356102+0000","last_undegraded":"2023-09-23T20:50:55.356102+0000","last_fullsized":"2023-09-23T20:50:55.356102+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:02:26.878542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"90'161","reported_seq":970,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-09-23T20:50:53.794020+0000","last_change":"2023-09-23T20:49:51.118813+0000","last_active":"2023-09-23T20:50:53.794020+0000","last_peered":"2023-09-23T20:50:53.794020+0000","last_clean":"2023-09-23T20:50:53.794020+0000","last_became_active":"2023-09-23T20:49:51.118445+0000","last_became_peered":"2023-09-23T20:49:51.118445+0000","last_unstale":"2023-09-23T20:50:53.794020+0000","last_undegraded":"2023-09-23T20:50:53.794020+0000","last_fullsized":"2023-09-23T20:50:53.794020+0000","mapping_epoch":639,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":640,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:20.041690+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:20.041690+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:15:24.683302+0000","scrub_duration":0.0026180330000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":627,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-09-23T20:50:52.790312+0000","last_change":"2023-09-23T20:43:46.521294+0000","last_active":"2023-09-23T20:50:52.790312+0000","last_peered":"2023-09-23T20:50:52.790312+0000","last_clean":"2023-09-23T20:50:52.790312+0000","last_became_active":"2023-09-23T20:43:46.521001+0000","last_became_peered":"2023-09-23T20:43:46.521001+0000","last_unstale":"2023-09-23T20:50:52.790312+0000","last_undegraded":"2023-09-23T20:50:52.790312+0000","last_fullsized":"2023-09-23T20:50:52.790312+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:01:35.126716+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":457,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-09-23T20:50:53.794844+0000","last_change":"2023-09-23T20:49:48.121288+0000","last_active":"2023-09-23T20:50:53.794844+0000","last_peered":"2023-09-23T20:50:53.794844+0000","last_clean":"2023-09-23T20:50:53.794844+0000","last_became_active":"2023-09-23T20:49:48.120936+0000","last_became_peered":"2023-09-23T20:49:48.120936+0000","last_unstale":"2023-09-23T20:50:53.794844+0000","last_undegraded":"2023-09-23T20:50:53.794844+0000","last_fullsized":"2023-09-23T20:50:53.794844+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:46:44.012748+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.13","version":"90'161","reported_seq":993,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-09-23T20:50:53.708992+0000","last_change":"2023-09-23T20:47:11.619371+0000","last_active":"2023-09-23T20:50:53.708992+0000","last_peered":"2023-09-23T20:50:53.708992+0000","last_clean":"2023-09-23T20:50:53.708992+0000","last_became_active":"2023-09-23T20:47:11.619006+0000","last_became_peered":"2023-09-23T20:47:11.619006+0000","last_unstale":"2023-09-23T20:50:53.708992+0000","last_undegraded":"2023-09-23T20:50:53.708992+0000","last_fullsized":"2023-09-23T20:50:53.708992+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:42.544308+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:42.544308+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:47:24.414807+0000","scrub_duration":0.0029958179999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":693,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-09-23T20:50:53.794186+0000","last_change":"2023-09-23T20:47:09.176613+0000","last_active":"2023-09-23T20:50:53.794186+0000","last_peered":"2023-09-23T20:50:53.794186+0000","last_clean":"2023-09-23T20:50:53.794186+0000","last_became_active":"2023-09-23T20:47:09.176187+0000","last_became_peered":"2023-09-23T20:47:09.176187+0000","last_unstale":"2023-09-23T20:50:53.794186+0000","last_undegraded":"2023-09-23T20:50:53.794186+0000","last_fullsized":"2023-09-23T20:50:53.794186+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:23:01.590989+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":635,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-09-23T20:50:52.791112+0000","last_change":"2023-09-23T20:43:46.521735+0000","last_active":"2023-09-23T20:50:52.791112+0000","last_peered":"2023-09-23T20:50:52.791112+0000","last_clean":"2023-09-23T20:50:52.791112+0000","last_became_active":"2023-09-23T20:43:46.520762+0000","last_became_peered":"2023-09-23T20:43:46.520762+0000","last_unstale":"2023-09-23T20:50:52.791112+0000","last_undegraded":"2023-09-23T20:50:52.791112+0000","last_fullsized":"2023-09-23T20:50:52.791112+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:26:48.841930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"90'161","reported_seq":772,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-09-23T20:50:52.793773+0000","last_change":"2023-09-23T20:49:51.125786+0000","last_active":"2023-09-23T20:50:52.793773+0000","last_peered":"2023-09-23T20:50:52.793773+0000","last_clean":"2023-09-23T20:50:52.793773+0000","last_became_active":"2023-09-23T20:49:51.124373+0000","last_became_peered":"2023-09-23T20:49:51.124373+0000","last_unstale":"2023-09-23T20:50:52.793773+0000","last_undegraded":"2023-09-23T20:50:52.793773+0000","last_fullsized":"2023-09-23T20:50:52.793773+0000","mapping_epoch":639,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":640,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:18.952693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:18.952693+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:07:29.170332+0000","scrub_duration":0.001699365,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":364,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-09-23T20:50:55.181724+0000","last_change":"2023-09-23T20:49:48.127484+0000","last_active":"2023-09-23T20:50:55.181724+0000","last_peered":"2023-09-23T20:50:55.181724+0000","last_clean":"2023-09-23T20:50:55.181724+0000","last_became_active":"2023-09-23T20:49:48.110748+0000","last_became_peered":"2023-09-23T20:49:48.110748+0000","last_unstale":"2023-09-23T20:50:55.181724+0000","last_undegraded":"2023-09-23T20:50:55.181724+0000","last_fullsized":"2023-09-23T20:50:55.181724+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:31:51.004529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":467,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-09-23T20:50:55.357292+0000","last_change":"2023-09-23T20:37:43.571083+0000","last_active":"2023-09-23T20:50:55.357292+0000","last_peered":"2023-09-23T20:50:55.357292+0000","last_clean":"2023-09-23T20:50:55.357292+0000","last_became_active":"2023-09-23T20:37:43.570306+0000","last_became_peered":"2023-09-23T20:37:43.570306+0000","last_unstale":"2023-09-23T20:50:55.357292+0000","last_undegraded":"2023-09-23T20:50:55.357292+0000","last_fullsized":"2023-09-23T20:50:55.357292+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:42:02.396718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.d","version":"90'161","reported_seq":1002,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-09-23T20:50:53.794535+0000","last_change":"2023-09-23T20:47:11.619510+0000","last_active":"2023-09-23T20:50:53.794535+0000","last_peered":"2023-09-23T20:50:53.794535+0000","last_clean":"2023-09-23T20:50:53.794535+0000","last_became_active":"2023-09-23T20:47:11.619154+0000","last_became_peered":"2023-09-23T20:47:11.619154+0000","last_unstale":"2023-09-23T20:50:53.794535+0000","last_undegraded":"2023-09-23T20:50:53.794535+0000","last_fullsized":"2023-09-23T20:50:53.794535+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:34:59.496375+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:34:59.496375+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:40:52.712777+0000","scrub_duration":0.0017843189999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":701,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-09-23T20:50:53.794877+0000","last_change":"2023-09-23T20:47:09.207851+0000","last_active":"2023-09-23T20:50:53.794877+0000","last_peered":"2023-09-23T20:50:53.794877+0000","last_clean":"2023-09-23T20:50:53.794877+0000","last_became_active":"2023-09-23T20:47:09.175836+0000","last_became_peered":"2023-09-23T20:47:09.175836+0000","last_unstale":"2023-09-23T20:50:53.794877+0000","last_undegraded":"2023-09-23T20:50:53.794877+0000","last_fullsized":"2023-09-23T20:50:53.794877+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:43:13.091809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":494,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-09-23T20:50:53.794314+0000","last_change":"2023-09-23T20:49:48.122188+0000","last_active":"2023-09-23T20:50:53.794314+0000","last_peered":"2023-09-23T20:50:53.794314+0000","last_clean":"2023-09-23T20:50:53.794314+0000","last_became_active":"2023-09-23T20:49:48.121894+0000","last_became_peered":"2023-09-23T20:49:48.121894+0000","last_unstale":"2023-09-23T20:50:53.794314+0000","last_undegraded":"2023-09-23T20:50:53.794314+0000","last_fullsized":"2023-09-23T20:50:53.794314+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:12:33.758985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.c","version":"90'161","reported_seq":933,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-09-23T20:50:55.356187+0000","last_change":"2023-09-23T20:41:09.506481+0000","last_active":"2023-09-23T20:50:55.356187+0000","last_peered":"2023-09-23T20:50:55.356187+0000","last_clean":"2023-09-23T20:50:55.356187+0000","last_became_active":"2023-09-23T20:38:28.442229+0000","last_became_peered":"2023-09-23T20:38:28.442229+0000","last_unstale":"2023-09-23T20:50:55.356187+0000","last_undegraded":"2023-09-23T20:50:55.356187+0000","last_fullsized":"2023-09-23T20:50:55.356187+0000","mapping_epoch":411,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":412,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:41:09.506338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:41:09.506338+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:19:46.265011+0000","scrub_duration":0.01112002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":739,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-09-23T20:50:53.794688+0000","last_change":"2023-09-23T20:47:10.848436+0000","last_active":"2023-09-23T20:50:53.794688+0000","last_peered":"2023-09-23T20:50:53.794688+0000","last_clean":"2023-09-23T20:50:53.794688+0000","last_became_active":"2023-09-23T20:47:09.196769+0000","last_became_peered":"2023-09-23T20:47:09.196769+0000","last_unstale":"2023-09-23T20:50:53.794688+0000","last_undegraded":"2023-09-23T20:50:53.794688+0000","last_fullsized":"2023-09-23T20:50:53.794688+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":554,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:59:38.105009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":136,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-09-23T20:50:55.241361+0000","last_change":"2023-09-23T20:48:23.915817+0000","last_active":"2023-09-23T20:50:55.241361+0000","last_peered":"2023-09-23T20:50:55.241361+0000","last_clean":"2023-09-23T20:50:55.241361+0000","last_became_active":"2023-09-23T20:48:23.915306+0000","last_became_peered":"2023-09-23T20:48:23.915306+0000","last_unstale":"2023-09-23T20:50:55.241361+0000","last_undegraded":"2023-09-23T20:50:55.241361+0000","last_fullsized":"2023-09-23T20:50:55.241361+0000","mapping_epoch":577,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":578,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:55:45.691450+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"90'161","reported_seq":884,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-09-23T20:50:55.182259+0000","last_change":"2023-09-23T20:49:51.120296+0000","last_active":"2023-09-23T20:50:55.182259+0000","last_peered":"2023-09-23T20:50:55.182259+0000","last_clean":"2023-09-23T20:50:55.182259+0000","last_became_active":"2023-09-23T20:49:51.120016+0000","last_became_peered":"2023-09-23T20:49:51.120016+0000","last_unstale":"2023-09-23T20:50:55.182259+0000","last_undegraded":"2023-09-23T20:50:55.182259+0000","last_fullsized":"2023-09-23T20:50:55.182259+0000","mapping_epoch":639,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":640,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:46.217489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:46.217489+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:35:59.437404+0000","scrub_duration":0.001456429,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":712,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-09-23T20:50:53.794606+0000","last_change":"2023-09-23T20:49:48.131779+0000","last_active":"2023-09-23T20:50:53.794606+0000","last_peered":"2023-09-23T20:50:53.794606+0000","last_clean":"2023-09-23T20:50:53.794606+0000","last_became_active":"2023-09-23T20:49:48.123934+0000","last_became_peered":"2023-09-23T20:49:48.123934+0000","last_unstale":"2023-09-23T20:50:53.794606+0000","last_undegraded":"2023-09-23T20:50:53.794606+0000","last_fullsized":"2023-09-23T20:50:53.794606+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:24:40.508247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":490,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":284,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-09-23T20:50:55.181222+0000","last_change":"2023-09-23T20:49:02.702301+0000","last_active":"2023-09-23T20:50:55.181222+0000","last_peered":"2023-09-23T20:50:55.181222+0000","last_clean":"2023-09-23T20:50:55.181222+0000","last_became_active":"2023-09-23T20:49:02.701986+0000","last_became_peered":"2023-09-23T20:49:02.701986+0000","last_unstale":"2023-09-23T20:50:55.181222+0000","last_undegraded":"2023-09-23T20:50:55.181222+0000","last_fullsized":"2023-09-23T20:50:55.181222+0000","mapping_epoch":603,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":604,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:20:01.603267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.e","version":"90'161","reported_seq":699,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-09-23T20:50:55.241998+0000","last_change":"2023-09-23T20:47:37.608426+0000","last_active":"2023-09-23T20:50:55.241998+0000","last_peered":"2023-09-23T20:50:55.241998+0000","last_clean":"2023-09-23T20:50:55.241998+0000","last_became_active":"2023-09-23T20:47:11.617187+0000","last_became_peered":"2023-09-23T20:47:11.617187+0000","last_unstale":"2023-09-23T20:50:55.241998+0000","last_undegraded":"2023-09-23T20:50:55.241998+0000","last_fullsized":"2023-09-23T20:50:55.241998+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:47:37.608309+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:47:37.608309+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:22:25.482297+0000","scrub_duration":0.011600924,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":641,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-09-23T20:50:55.356435+0000","last_change":"2023-09-23T20:49:48.120934+0000","last_active":"2023-09-23T20:50:55.356435+0000","last_peered":"2023-09-23T20:50:55.356435+0000","last_clean":"2023-09-23T20:50:55.356435+0000","last_became_active":"2023-09-23T20:49:48.120578+0000","last_became_peered":"2023-09-23T20:49:48.120578+0000","last_unstale":"2023-09-23T20:50:55.356435+0000","last_undegraded":"2023-09-23T20:50:55.356435+0000","last_fullsized":"2023-09-23T20:50:55.356435+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:57:50.669399+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":663,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-09-23T20:50:52.790191+0000","last_change":"2023-09-23T20:47:09.197427+0000","last_active":"2023-09-23T20:50:52.790191+0000","last_peered":"2023-09-23T20:50:52.790191+0000","last_clean":"2023-09-23T20:50:52.790191+0000","last_became_active":"2023-09-23T20:47:09.196542+0000","last_became_peered":"2023-09-23T20:47:09.196542+0000","last_unstale":"2023-09-23T20:50:52.790191+0000","last_undegraded":"2023-09-23T20:50:52.790191+0000","last_fullsized":"2023-09-23T20:50:52.790191+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:28:38.926202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":1115,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-09-23T20:50:53.794075+0000","last_change":"2023-09-23T20:48:51.169474+0000","last_active":"2023-09-23T20:50:53.794075+0000","last_peered":"2023-09-23T20:50:53.794075+0000","last_clean":"2023-09-23T20:50:53.794075+0000","last_became_active":"2023-09-23T20:48:51.169076+0000","last_became_peered":"2023-09-23T20:48:51.169076+0000","last_unstale":"2023-09-23T20:50:53.794075+0000","last_undegraded":"2023-09-23T20:50:53.794075+0000","last_fullsized":"2023-09-23T20:50:53.794075+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:35:02.523203+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:35:02.523203+0000","last_clean_scrub_stamp":"2023-09-23T20:35:02.523203+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:25:53.741403+0000","scrub_duration":0.0085927989999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":519,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-09-23T20:50:55.355239+0000","last_change":"2023-09-23T20:47:09.240200+0000","last_active":"2023-09-23T20:50:55.355239+0000","last_peered":"2023-09-23T20:50:55.355239+0000","last_clean":"2023-09-23T20:50:55.355239+0000","last_became_active":"2023-09-23T20:47:09.195204+0000","last_became_peered":"2023-09-23T20:47:09.195204+0000","last_unstale":"2023-09-23T20:50:55.355239+0000","last_undegraded":"2023-09-23T20:50:55.355239+0000","last_fullsized":"2023-09-23T20:50:55.355239+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:03.328250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":126,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-09-23T20:50:55.241579+0000","last_change":"2023-09-23T20:47:09.231714+0000","last_active":"2023-09-23T20:50:55.241579+0000","last_peered":"2023-09-23T20:50:55.241579+0000","last_clean":"2023-09-23T20:50:55.241579+0000","last_became_active":"2023-09-23T20:47:09.230918+0000","last_became_peered":"2023-09-23T20:47:09.230918+0000","last_unstale":"2023-09-23T20:50:55.241579+0000","last_undegraded":"2023-09-23T20:50:55.241579+0000","last_fullsized":"2023-09-23T20:50:55.241579+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:24.583656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":574,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-09-23T20:50:55.355483+0000","last_change":"2023-09-23T20:50:03.882061+0000","last_active":"2023-09-23T20:50:55.355483+0000","last_peered":"2023-09-23T20:50:55.355483+0000","last_clean":"2023-09-23T20:50:55.355483+0000","last_became_active":"2023-09-23T20:49:51.124481+0000","last_became_peered":"2023-09-23T20:49:51.124481+0000","last_unstale":"2023-09-23T20:50:55.355483+0000","last_undegraded":"2023-09-23T20:50:55.355483+0000","last_fullsized":"2023-09-23T20:50:55.355483+0000","mapping_epoch":639,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":640,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:50:03.881826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:50:03.881826+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:51.282979+0000","scrub_duration":0.003245258,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":294,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-09-23T20:50:55.181777+0000","last_change":"2023-09-23T20:47:09.178919+0000","last_active":"2023-09-23T20:50:55.181777+0000","last_peered":"2023-09-23T20:50:55.181777+0000","last_clean":"2023-09-23T20:50:55.181777+0000","last_became_active":"2023-09-23T20:47:09.178607+0000","last_became_peered":"2023-09-23T20:47:09.178607+0000","last_unstale":"2023-09-23T20:50:55.181777+0000","last_undegraded":"2023-09-23T20:50:55.181777+0000","last_fullsized":"2023-09-23T20:50:55.181777+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:23:45.196935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":146,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-09-23T20:50:55.241987+0000","last_change":"2023-09-23T20:48:48.148596+0000","last_active":"2023-09-23T20:50:55.241987+0000","last_peered":"2023-09-23T20:50:55.241987+0000","last_clean":"2023-09-23T20:50:55.241987+0000","last_became_active":"2023-09-23T20:48:48.148211+0000","last_became_peered":"2023-09-23T20:48:48.148211+0000","last_unstale":"2023-09-23T20:50:55.241987+0000","last_undegraded":"2023-09-23T20:50:55.241987+0000","last_fullsized":"2023-09-23T20:50:55.241987+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:21:50.825822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"90'161","reported_seq":992,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-09-23T20:50:52.793965+0000","last_change":"2023-09-23T20:49:51.124836+0000","last_active":"2023-09-23T20:50:52.793965+0000","last_peered":"2023-09-23T20:50:52.793965+0000","last_clean":"2023-09-23T20:50:52.793965+0000","last_became_active":"2023-09-23T20:49:51.124419+0000","last_became_peered":"2023-09-23T20:49:51.124419+0000","last_unstale":"2023-09-23T20:50:52.793965+0000","last_undegraded":"2023-09-23T20:50:52.793965+0000","last_fullsized":"2023-09-23T20:50:52.793965+0000","mapping_epoch":639,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":640,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:05.595042+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:05.595042+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:37:05.219030+0000","scrub_duration":0.007271675,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":146,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-09-23T20:50:55.241275+0000","last_change":"2023-09-23T20:48:48.153145+0000","last_active":"2023-09-23T20:50:55.241275+0000","last_peered":"2023-09-23T20:50:55.241275+0000","last_clean":"2023-09-23T20:50:55.241275+0000","last_became_active":"2023-09-23T20:48:48.152858+0000","last_became_peered":"2023-09-23T20:48:48.152858+0000","last_unstale":"2023-09-23T20:50:55.241275+0000","last_undegraded":"2023-09-23T20:50:55.241275+0000","last_fullsized":"2023-09-23T20:50:55.241275+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:23:13.454288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":126,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-09-23T20:50:55.241982+0000","last_change":"2023-09-23T20:47:09.186271+0000","last_active":"2023-09-23T20:50:55.241982+0000","last_peered":"2023-09-23T20:50:55.241982+0000","last_clean":"2023-09-23T20:50:55.241982+0000","last_became_active":"2023-09-23T20:47:09.184357+0000","last_became_peered":"2023-09-23T20:47:09.184357+0000","last_unstale":"2023-09-23T20:50:55.241982+0000","last_undegraded":"2023-09-23T20:50:55.241982+0000","last_fullsized":"2023-09-23T20:50:55.241982+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:48:26.339811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":578,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-09-23T20:50:55.356449+0000","last_change":"2023-09-23T20:49:51.128432+0000","last_active":"2023-09-23T20:50:55.356449+0000","last_peered":"2023-09-23T20:50:55.356449+0000","last_clean":"2023-09-23T20:50:55.356449+0000","last_became_active":"2023-09-23T20:49:51.128074+0000","last_became_peered":"2023-09-23T20:49:51.128074+0000","last_unstale":"2023-09-23T20:50:55.356449+0000","last_undegraded":"2023-09-23T20:50:55.356449+0000","last_fullsized":"2023-09-23T20:50:55.356449+0000","mapping_epoch":639,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":640,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:49:37.762492+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:49:37.762492+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:31:57.447267+0000","scrub_duration":0.0031485979999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,4],"acting":[2,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":468,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-09-23T20:50:55.355464+0000","last_change":"2023-09-23T20:37:43.570610+0000","last_active":"2023-09-23T20:50:55.355464+0000","last_peered":"2023-09-23T20:50:55.355464+0000","last_clean":"2023-09-23T20:50:55.355464+0000","last_became_active":"2023-09-23T20:37:43.570216+0000","last_became_peered":"2023-09-23T20:37:43.570216+0000","last_unstale":"2023-09-23T20:50:55.355464+0000","last_undegraded":"2023-09-23T20:50:55.355464+0000","last_fullsized":"2023-09-23T20:50:55.355464+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:39:26.555329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":515,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-09-23T20:50:52.794255+0000","last_change":"2023-09-23T20:47:09.172573+0000","last_active":"2023-09-23T20:50:52.794255+0000","last_peered":"2023-09-23T20:50:52.794255+0000","last_clean":"2023-09-23T20:50:52.794255+0000","last_became_active":"2023-09-23T20:47:09.172298+0000","last_became_peered":"2023-09-23T20:47:09.172298+0000","last_unstale":"2023-09-23T20:50:52.794255+0000","last_undegraded":"2023-09-23T20:50:52.794255+0000","last_fullsized":"2023-09-23T20:50:52.794255+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:22:09.361284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":857,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-09-23T20:50:55.182260+0000","last_change":"2023-09-23T20:41:07.424609+0000","last_active":"2023-09-23T20:50:55.182260+0000","last_peered":"2023-09-23T20:50:55.182260+0000","last_clean":"2023-09-23T20:50:55.182260+0000","last_became_active":"2023-09-23T20:38:05.622261+0000","last_became_peered":"2023-09-23T20:38:05.622261+0000","last_unstale":"2023-09-23T20:50:55.182260+0000","last_undegraded":"2023-09-23T20:50:55.182260+0000","last_fullsized":"2023-09-23T20:50:55.182260+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:41:07.424492+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:41:07.424492+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:45:21.689123+0000","scrub_duration":0.0018960959999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"664'242","reported_seq":1420,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-09-23T20:50:52.794354+0000","last_change":"2023-09-23T20:50:39.737511+0000","last_active":"2023-09-23T20:50:52.794354+0000","last_peered":"2023-09-23T20:50:52.794354+0000","last_clean":"2023-09-23T20:50:52.794354+0000","last_became_active":"2023-09-23T20:29:45.520546+0000","last_became_peered":"2023-09-23T20:29:45.520546+0000","last_unstale":"2023-09-23T20:50:52.794354+0000","last_undegraded":"2023-09-23T20:50:52.794354+0000","last_fullsized":"2023-09-23T20:50:52.794354+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":242,"log_dups_size":0,"ondisk_log_size":242,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:58:56.085777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0012974410000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1152,"num_read_kb":4628,"num_write":256,"num_write_kb":3463,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":492,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-09-23T20:50:55.356843+0000","last_change":"2023-09-23T20:49:48.114924+0000","last_active":"2023-09-23T20:50:55.356843+0000","last_peered":"2023-09-23T20:50:55.356843+0000","last_clean":"2023-09-23T20:50:55.356843+0000","last_became_active":"2023-09-23T20:49:48.114554+0000","last_became_peered":"2023-09-23T20:49:48.114554+0000","last_unstale":"2023-09-23T20:50:55.356843+0000","last_undegraded":"2023-09-23T20:50:55.356843+0000","last_fullsized":"2023-09-23T20:50:55.356843+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:09:09.538462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":656,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-09-23T20:50:52.793597+0000","last_change":"2023-09-23T20:49:48.115598+0000","last_active":"2023-09-23T20:50:52.793597+0000","last_peered":"2023-09-23T20:50:52.793597+0000","last_clean":"2023-09-23T20:50:52.793597+0000","last_became_active":"2023-09-23T20:49:48.115313+0000","last_became_peered":"2023-09-23T20:49:48.115313+0000","last_unstale":"2023-09-23T20:50:52.793597+0000","last_undegraded":"2023-09-23T20:50:52.793597+0000","last_fullsized":"2023-09-23T20:50:52.793597+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:58:47.477333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":768,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-09-23T20:50:52.791015+0000","last_change":"2023-09-23T20:49:51.148445+0000","last_active":"2023-09-23T20:50:52.791015+0000","last_peered":"2023-09-23T20:50:52.791015+0000","last_clean":"2023-09-23T20:50:52.791015+0000","last_became_active":"2023-09-23T20:49:51.147884+0000","last_became_peered":"2023-09-23T20:49:51.147884+0000","last_unstale":"2023-09-23T20:50:52.791015+0000","last_undegraded":"2023-09-23T20:50:52.791015+0000","last_fullsized":"2023-09-23T20:50:52.791015+0000","mapping_epoch":639,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":640,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:15.656880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:15.656880+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:05:08.116069+0000","scrub_duration":0.0016711180000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"664'209","reported_seq":1650,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-09-23T20:50:52.793693+0000","last_change":"2023-09-23T20:50:39.737378+0000","last_active":"2023-09-23T20:50:52.793693+0000","last_peered":"2023-09-23T20:50:52.793693+0000","last_clean":"2023-09-23T20:50:52.793693+0000","last_became_active":"2023-09-23T20:49:51.125316+0000","last_became_peered":"2023-09-23T20:49:51.125316+0000","last_unstale":"2023-09-23T20:50:52.793693+0000","last_undegraded":"2023-09-23T20:50:52.793693+0000","last_fullsized":"2023-09-23T20:50:52.793693+0000","mapping_epoch":639,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":640,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":209,"log_dups_size":0,"ondisk_log_size":209,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:32:47.065876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001172474,"stat_sum":{"num_bytes":4194323,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":2248,"num_read_kb":3726,"num_write":182,"num_write_kb":5560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":126,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-09-23T20:50:55.241823+0000","last_change":"2023-09-23T20:47:09.177872+0000","last_active":"2023-09-23T20:50:55.241823+0000","last_peered":"2023-09-23T20:50:55.241823+0000","last_clean":"2023-09-23T20:50:55.241823+0000","last_became_active":"2023-09-23T20:47:09.176689+0000","last_became_peered":"2023-09-23T20:47:09.176689+0000","last_unstale":"2023-09-23T20:50:55.241823+0000","last_undegraded":"2023-09-23T20:50:55.241823+0000","last_fullsized":"2023-09-23T20:50:55.241823+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:14:01.590318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":653,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-09-23T20:50:52.794307+0000","last_change":"2023-09-23T20:48:48.151144+0000","last_active":"2023-09-23T20:50:52.794307+0000","last_peered":"2023-09-23T20:50:52.794307+0000","last_clean":"2023-09-23T20:50:52.794307+0000","last_became_active":"2023-09-23T20:48:48.150770+0000","last_became_peered":"2023-09-23T20:48:48.150770+0000","last_unstale":"2023-09-23T20:50:52.794307+0000","last_undegraded":"2023-09-23T20:50:52.794307+0000","last_fullsized":"2023-09-23T20:50:52.794307+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:14:08.549512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":673,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-09-23T20:50:53.794308+0000","last_change":"2023-09-23T20:47:11.618652+0000","last_active":"2023-09-23T20:50:53.794308+0000","last_peered":"2023-09-23T20:50:53.794308+0000","last_clean":"2023-09-23T20:50:53.794308+0000","last_became_active":"2023-09-23T20:47:11.618266+0000","last_became_peered":"2023-09-23T20:47:11.618266+0000","last_unstale":"2023-09-23T20:50:53.794308+0000","last_undegraded":"2023-09-23T20:50:53.794308+0000","last_fullsized":"2023-09-23T20:50:53.794308+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:26.978179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:26.978179+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:29:29.000683+0000","scrub_duration":0.0013962829999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"664'192","reported_seq":706,"reported_epoch":670,"state":"active+clean","last_fresh":"2023-09-23T20:50:58.296268+0000","last_change":"2023-09-23T20:50:39.795569+0000","last_active":"2023-09-23T20:50:58.296268+0000","last_peered":"2023-09-23T20:50:58.296268+0000","last_clean":"2023-09-23T20:50:58.296268+0000","last_became_active":"2023-09-23T20:48:51.170485+0000","last_became_peered":"2023-09-23T20:48:51.170485+0000","last_unstale":"2023-09-23T20:50:58.296268+0000","last_undegraded":"2023-09-23T20:50:58.296268+0000","last_fullsized":"2023-09-23T20:50:58.296268+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":192,"log_dups_size":0,"ondisk_log_size":192,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:48:35.309684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038574200000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":830,"num_read_kb":3504,"num_write":183,"num_write_kb":2624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":661,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-09-23T20:50:52.793388+0000","last_change":"2023-09-23T20:48:48.151034+0000","last_active":"2023-09-23T20:50:52.793388+0000","last_peered":"2023-09-23T20:50:52.793388+0000","last_clean":"2023-09-23T20:50:52.793388+0000","last_became_active":"2023-09-23T20:48:48.150658+0000","last_became_peered":"2023-09-23T20:48:48.150658+0000","last_unstale":"2023-09-23T20:50:52.793388+0000","last_undegraded":"2023-09-23T20:50:52.793388+0000","last_fullsized":"2023-09-23T20:50:52.793388+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:44:34.819851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":195,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-09-23T20:50:52.794136+0000","last_change":"2023-09-23T20:49:48.118182+0000","last_active":"2023-09-23T20:50:52.794136+0000","last_peered":"2023-09-23T20:50:52.794136+0000","last_clean":"2023-09-23T20:50:52.794136+0000","last_became_active":"2023-09-23T20:49:48.117804+0000","last_became_peered":"2023-09-23T20:49:48.117804+0000","last_unstale":"2023-09-23T20:50:52.794136+0000","last_undegraded":"2023-09-23T20:50:52.794136+0000","last_fullsized":"2023-09-23T20:50:52.794136+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:47:43.384018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"664'319","reported_seq":1155,"reported_epoch":670,"state":"active+clean","last_fresh":"2023-09-23T20:50:58.296380+0000","last_change":"2023-09-23T20:50:39.795712+0000","last_active":"2023-09-23T20:50:58.296380+0000","last_peered":"2023-09-23T20:50:58.296380+0000","last_clean":"2023-09-23T20:50:58.296380+0000","last_became_active":"2023-09-23T20:49:51.153529+0000","last_became_peered":"2023-09-23T20:49:51.153529+0000","last_unstale":"2023-09-23T20:50:58.296380+0000","last_undegraded":"2023-09-23T20:50:58.296380+0000","last_fullsized":"2023-09-23T20:50:58.296380+0000","mapping_epoch":639,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":640,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":319,"log_dups_size":0,"ondisk_log_size":319,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:10:22.072962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043345100000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1396,"num_read_kb":4612,"num_write":290,"num_write_kb":4437,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":1132,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-09-23T20:50:53.794144+0000","last_change":"2023-09-23T20:38:05.602508+0000","last_active":"2023-09-23T20:50:53.794144+0000","last_peered":"2023-09-23T20:50:53.794144+0000","last_clean":"2023-09-23T20:50:53.794144+0000","last_became_active":"2023-09-23T20:38:05.602129+0000","last_became_peered":"2023-09-23T20:38:05.602129+0000","last_unstale":"2023-09-23T20:50:53.794144+0000","last_undegraded":"2023-09-23T20:50:53.794144+0000","last_fullsized":"2023-09-23T20:50:53.794144+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:45.343742+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:45.343742+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:48.910343+0000","scrub_duration":0.001475802,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":254,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-09-23T20:50:55.181773+0000","last_change":"2023-09-23T20:49:48.119237+0000","last_active":"2023-09-23T20:50:55.181773+0000","last_peered":"2023-09-23T20:50:55.181773+0000","last_clean":"2023-09-23T20:50:55.181773+0000","last_became_active":"2023-09-23T20:49:48.118927+0000","last_became_peered":"2023-09-23T20:49:48.118927+0000","last_unstale":"2023-09-23T20:50:55.181773+0000","last_undegraded":"2023-09-23T20:50:55.181773+0000","last_fullsized":"2023-09-23T20:50:55.181773+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:31:47.579623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,4],"acting":[6,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":294,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-09-23T20:50:55.182003+0000","last_change":"2023-09-23T20:47:09.202302+0000","last_active":"2023-09-23T20:50:55.182003+0000","last_peered":"2023-09-23T20:50:55.182003+0000","last_clean":"2023-09-23T20:50:55.182003+0000","last_became_active":"2023-09-23T20:47:09.201797+0000","last_became_peered":"2023-09-23T20:47:09.201797+0000","last_unstale":"2023-09-23T20:50:55.182003+0000","last_undegraded":"2023-09-23T20:50:55.182003+0000","last_fullsized":"2023-09-23T20:50:55.182003+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:34:41.055705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":1038,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-09-23T20:50:52.793786+0000","last_change":"2023-09-23T20:47:11.622262+0000","last_active":"2023-09-23T20:50:52.793786+0000","last_peered":"2023-09-23T20:50:52.793786+0000","last_clean":"2023-09-23T20:50:52.793786+0000","last_became_active":"2023-09-23T20:47:11.621920+0000","last_became_peered":"2023-09-23T20:47:11.621920+0000","last_unstale":"2023-09-23T20:50:52.793786+0000","last_undegraded":"2023-09-23T20:50:52.793786+0000","last_fullsized":"2023-09-23T20:50:52.793786+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:43.582146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:43.582146+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:01:15.153543+0000","scrub_duration":0.0016219730000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"664'171","reported_seq":1334,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-09-23T20:50:53.794053+0000","last_change":"2023-09-23T20:50:39.742996+0000","last_active":"2023-09-23T20:50:53.794053+0000","last_peered":"2023-09-23T20:50:53.794053+0000","last_clean":"2023-09-23T20:50:53.794053+0000","last_became_active":"2023-09-23T20:29:45.536439+0000","last_became_peered":"2023-09-23T20:29:45.536439+0000","last_unstale":"2023-09-23T20:50:53.794053+0000","last_undegraded":"2023-09-23T20:50:53.794053+0000","last_fullsized":"2023-09-23T20:50:53.794053+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":171,"log_dups_size":0,"ondisk_log_size":171,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:30:57.793132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032959200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1114,"num_read_kb":3170,"num_write":132,"num_write_kb":2190,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":682,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-09-23T20:50:52.790928+0000","last_change":"2023-09-23T20:47:09.198326+0000","last_active":"2023-09-23T20:50:52.790928+0000","last_peered":"2023-09-23T20:50:52.790928+0000","last_clean":"2023-09-23T20:50:52.790928+0000","last_became_active":"2023-09-23T20:47:09.198059+0000","last_became_peered":"2023-09-23T20:47:09.198059+0000","last_unstale":"2023-09-23T20:50:52.790928+0000","last_undegraded":"2023-09-23T20:50:52.790928+0000","last_fullsized":"2023-09-23T20:50:52.790928+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:38:53.903706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":677,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-09-23T20:50:52.790895+0000","last_change":"2023-09-23T20:48:48.146977+0000","last_active":"2023-09-23T20:50:52.790895+0000","last_peered":"2023-09-23T20:50:52.790895+0000","last_clean":"2023-09-23T20:50:52.790895+0000","last_became_active":"2023-09-23T20:48:48.146657+0000","last_became_peered":"2023-09-23T20:48:48.146657+0000","last_unstale":"2023-09-23T20:50:52.790895+0000","last_undegraded":"2023-09-23T20:50:52.790895+0000","last_fullsized":"2023-09-23T20:50:52.790895+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:38:41.782692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":780,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-09-23T20:50:55.182025+0000","last_change":"2023-09-23T20:41:08.380084+0000","last_active":"2023-09-23T20:50:55.182025+0000","last_peered":"2023-09-23T20:50:55.182025+0000","last_clean":"2023-09-23T20:50:55.182025+0000","last_became_active":"2023-09-23T20:38:05.610881+0000","last_became_peered":"2023-09-23T20:38:05.610881+0000","last_unstale":"2023-09-23T20:50:55.182025+0000","last_undegraded":"2023-09-23T20:50:55.182025+0000","last_fullsized":"2023-09-23T20:50:55.182025+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:41:08.379959+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:41:08.379959+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:19:39.547923+0000","scrub_duration":0.0019158420000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"664'175","reported_seq":1633,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-09-23T20:50:53.794699+0000","last_change":"2023-09-23T20:50:39.743889+0000","last_active":"2023-09-23T20:50:53.794699+0000","last_peered":"2023-09-23T20:50:53.794699+0000","last_clean":"2023-09-23T20:50:53.794699+0000","last_became_active":"2023-09-23T20:47:14.628341+0000","last_became_peered":"2023-09-23T20:47:14.628341+0000","last_unstale":"2023-09-23T20:50:53.794699+0000","last_undegraded":"2023-09-23T20:50:53.794699+0000","last_fullsized":"2023-09-23T20:50:53.794699+0000","mapping_epoch":558,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":559,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":175,"log_dups_size":0,"ondisk_log_size":175,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:42:16.291849+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000229686,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1422,"num_read_kb":2425,"num_write":116,"num_write_kb":664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":646,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-09-23T20:50:52.794400+0000","last_change":"2023-09-23T20:49:48.108294+0000","last_active":"2023-09-23T20:50:52.794400+0000","last_peered":"2023-09-23T20:50:52.794400+0000","last_clean":"2023-09-23T20:50:52.794400+0000","last_became_active":"2023-09-23T20:49:48.107916+0000","last_became_peered":"2023-09-23T20:49:48.107916+0000","last_unstale":"2023-09-23T20:50:52.794400+0000","last_undegraded":"2023-09-23T20:50:52.794400+0000","last_fullsized":"2023-09-23T20:50:52.794400+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:57:12.800814+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":623,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-09-23T20:50:52.790424+0000","last_change":"2023-09-23T20:49:48.119227+0000","last_active":"2023-09-23T20:50:52.790424+0000","last_peered":"2023-09-23T20:50:52.790424+0000","last_clean":"2023-09-23T20:50:52.790424+0000","last_became_active":"2023-09-23T20:49:48.118910+0000","last_became_peered":"2023-09-23T20:49:48.118910+0000","last_unstale":"2023-09-23T20:50:52.790424+0000","last_undegraded":"2023-09-23T20:50:52.790424+0000","last_fullsized":"2023-09-23T20:50:52.790424+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:18:40.836373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":706,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-09-23T20:50:55.242071+0000","last_change":"2023-09-23T20:48:51.172830+0000","last_active":"2023-09-23T20:50:55.242071+0000","last_peered":"2023-09-23T20:50:55.242071+0000","last_clean":"2023-09-23T20:50:55.242071+0000","last_became_active":"2023-09-23T20:48:51.172481+0000","last_became_peered":"2023-09-23T20:48:51.172481+0000","last_unstale":"2023-09-23T20:50:55.242071+0000","last_undegraded":"2023-09-23T20:50:55.242071+0000","last_fullsized":"2023-09-23T20:50:55.242071+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:47:34.561747+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:47:34.561747+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:30:32.531477+0000","scrub_duration":0.0087949929999999992,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":407,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-09-23T20:50:53.794933+0000","last_change":"2023-09-23T20:32:50.549695+0000","last_active":"2023-09-23T20:50:53.794933+0000","last_peered":"2023-09-23T20:50:53.794933+0000","last_clean":"2023-09-23T20:50:53.794933+0000","last_became_active":"2023-09-23T20:32:50.549431+0000","last_became_peered":"2023-09-23T20:32:50.549431+0000","last_unstale":"2023-09-23T20:50:53.794933+0000","last_undegraded":"2023-09-23T20:50:53.794933+0000","last_fullsized":"2023-09-23T20:50:53.794933+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:45:53.666569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"664'349","reported_seq":2029,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-09-23T20:50:52.794375+0000","last_change":"2023-09-23T20:50:39.737163+0000","last_active":"2023-09-23T20:50:52.794375+0000","last_peered":"2023-09-23T20:50:52.794375+0000","last_clean":"2023-09-23T20:50:52.794375+0000","last_became_active":"2023-09-23T20:47:11.621284+0000","last_became_peered":"2023-09-23T20:47:11.621284+0000","last_unstale":"2023-09-23T20:50:52.794375+0000","last_undegraded":"2023-09-23T20:50:52.794375+0000","last_fullsized":"2023-09-23T20:50:52.794375+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":349,"log_dups_size":0,"ondisk_log_size":349,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:43:50.658512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0014591350000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":2217,"num_read_kb":2864,"num_write":263,"num_write_kb":2168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":453,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-09-23T20:50:55.355388+0000","last_change":"2023-09-23T20:37:43.582105+0000","last_active":"2023-09-23T20:50:55.355388+0000","last_peered":"2023-09-23T20:50:55.355388+0000","last_clean":"2023-09-23T20:50:55.355388+0000","last_became_active":"2023-09-23T20:37:43.581860+0000","last_became_peered":"2023-09-23T20:37:43.581860+0000","last_unstale":"2023-09-23T20:50:55.355388+0000","last_undegraded":"2023-09-23T20:50:55.355388+0000","last_fullsized":"2023-09-23T20:50:55.355388+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:18:38.765486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"90'161","reported_seq":904,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-09-23T20:50:55.355089+0000","last_change":"2023-09-23T20:49:51.122217+0000","last_active":"2023-09-23T20:50:55.355089+0000","last_peered":"2023-09-23T20:50:55.355089+0000","last_clean":"2023-09-23T20:50:55.355089+0000","last_became_active":"2023-09-23T20:49:51.121893+0000","last_became_peered":"2023-09-23T20:49:51.121893+0000","last_unstale":"2023-09-23T20:50:55.355089+0000","last_undegraded":"2023-09-23T20:50:55.355089+0000","last_fullsized":"2023-09-23T20:50:55.355089+0000","mapping_epoch":639,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":640,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:34:54.504554+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:34:54.504554+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:00:40.029809+0000","scrub_duration":0.011176864,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"664'293","reported_seq":1427,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-09-23T20:50:55.182205+0000","last_change":"2023-09-23T20:50:39.740476+0000","last_active":"2023-09-23T20:50:55.182205+0000","last_peered":"2023-09-23T20:50:55.182205+0000","last_clean":"2023-09-23T20:50:55.182205+0000","last_became_active":"2023-09-23T20:48:51.217013+0000","last_became_peered":"2023-09-23T20:48:51.217013+0000","last_unstale":"2023-09-23T20:50:55.182205+0000","last_undegraded":"2023-09-23T20:50:55.182205+0000","last_fullsized":"2023-09-23T20:50:55.182205+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":293,"log_dups_size":0,"ondisk_log_size":293,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:40:42.297947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00027326800000000001,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1602,"num_read_kb":1486,"num_write":268,"num_write_kb":4284,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":48,"num_bytes_recovered":25165824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":685,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-09-23T20:50:52.790997+0000","last_change":"2023-09-23T20:47:09.317116+0000","last_active":"2023-09-23T20:50:52.790997+0000","last_peered":"2023-09-23T20:50:52.790997+0000","last_clean":"2023-09-23T20:50:52.790997+0000","last_became_active":"2023-09-23T20:47:09.205930+0000","last_became_peered":"2023-09-23T20:47:09.205930+0000","last_unstale":"2023-09-23T20:50:52.790997+0000","last_undegraded":"2023-09-23T20:50:52.790997+0000","last_fullsized":"2023-09-23T20:50:52.790997+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:04:55.528625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":136,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-09-23T20:50:55.241299+0000","last_change":"2023-09-23T20:49:48.120086+0000","last_active":"2023-09-23T20:50:55.241299+0000","last_peered":"2023-09-23T20:50:55.241299+0000","last_clean":"2023-09-23T20:50:55.241299+0000","last_became_active":"2023-09-23T20:49:48.119761+0000","last_became_peered":"2023-09-23T20:49:48.119761+0000","last_unstale":"2023-09-23T20:50:55.241299+0000","last_undegraded":"2023-09-23T20:50:55.241299+0000","last_fullsized":"2023-09-23T20:50:55.241299+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:56:12.469830+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"90'161","reported_seq":927,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-09-23T20:50:52.790598+0000","last_change":"2023-09-23T20:43:46.544750+0000","last_active":"2023-09-23T20:50:52.790598+0000","last_peered":"2023-09-23T20:50:52.790598+0000","last_clean":"2023-09-23T20:50:52.790598+0000","last_became_active":"2023-09-23T20:43:46.544395+0000","last_became_peered":"2023-09-23T20:43:46.544395+0000","last_unstale":"2023-09-23T20:50:52.790598+0000","last_undegraded":"2023-09-23T20:50:52.790598+0000","last_fullsized":"2023-09-23T20:50:52.790598+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:06.487351+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:06.487351+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:56.866168+0000","scrub_duration":0.001964379,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":643,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-09-23T20:50:52.790346+0000","last_change":"2023-09-23T20:47:09.197601+0000","last_active":"2023-09-23T20:50:52.790346+0000","last_peered":"2023-09-23T20:50:52.790346+0000","last_clean":"2023-09-23T20:50:52.790346+0000","last_became_active":"2023-09-23T20:47:09.196967+0000","last_became_peered":"2023-09-23T20:47:09.196967+0000","last_unstale":"2023-09-23T20:50:52.790346+0000","last_undegraded":"2023-09-23T20:50:52.790346+0000","last_fullsized":"2023-09-23T20:50:52.790346+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:06:56.826785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":89,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-09-23T20:50:55.355616+0000","last_change":"2023-09-23T20:48:48.158376+0000","last_active":"2023-09-23T20:50:55.355616+0000","last_peered":"2023-09-23T20:50:55.355616+0000","last_clean":"2023-09-23T20:50:55.355616+0000","last_became_active":"2023-09-23T20:48:48.158069+0000","last_became_peered":"2023-09-23T20:48:48.158069+0000","last_unstale":"2023-09-23T20:50:55.355616+0000","last_undegraded":"2023-09-23T20:50:55.355616+0000","last_fullsized":"2023-09-23T20:50:55.355616+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:34.658561+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"90'161","reported_seq":746,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-09-23T20:50:55.357425+0000","last_change":"2023-09-23T20:41:06.499511+0000","last_active":"2023-09-23T20:50:55.357425+0000","last_peered":"2023-09-23T20:50:55.357425+0000","last_clean":"2023-09-23T20:50:55.357425+0000","last_became_active":"2023-09-23T20:38:05.601874+0000","last_became_peered":"2023-09-23T20:38:05.601874+0000","last_unstale":"2023-09-23T20:50:55.357425+0000","last_undegraded":"2023-09-23T20:50:55.357425+0000","last_fullsized":"2023-09-23T20:50:55.357425+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:41:06.499396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:41:06.499396+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:31:34.059077+0000","scrub_duration":0.0019670310000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":144,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-09-23T20:50:55.241715+0000","last_change":"2023-09-23T20:49:48.128500+0000","last_active":"2023-09-23T20:50:55.241715+0000","last_peered":"2023-09-23T20:50:55.241715+0000","last_clean":"2023-09-23T20:50:55.241715+0000","last_became_active":"2023-09-23T20:49:48.122142+0000","last_became_peered":"2023-09-23T20:49:48.122142+0000","last_unstale":"2023-09-23T20:50:55.241715+0000","last_undegraded":"2023-09-23T20:50:55.241715+0000","last_fullsized":"2023-09-23T20:50:55.241715+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:46:11.349709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":126,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-09-23T20:50:55.241774+0000","last_change":"2023-09-23T20:47:09.186579+0000","last_active":"2023-09-23T20:50:55.241774+0000","last_peered":"2023-09-23T20:50:55.241774+0000","last_clean":"2023-09-23T20:50:55.241774+0000","last_became_active":"2023-09-23T20:47:09.186227+0000","last_became_peered":"2023-09-23T20:47:09.186227+0000","last_unstale":"2023-09-23T20:50:55.241774+0000","last_undegraded":"2023-09-23T20:50:55.241774+0000","last_fullsized":"2023-09-23T20:50:55.241774+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:45:39.485308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":144,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-09-23T20:50:55.242009+0000","last_change":"2023-09-23T20:49:48.126698+0000","last_active":"2023-09-23T20:50:55.242009+0000","last_peered":"2023-09-23T20:50:55.242009+0000","last_clean":"2023-09-23T20:50:55.242009+0000","last_became_active":"2023-09-23T20:49:48.120196+0000","last_became_peered":"2023-09-23T20:49:48.120196+0000","last_unstale":"2023-09-23T20:50:55.242009+0000","last_undegraded":"2023-09-23T20:50:55.242009+0000","last_fullsized":"2023-09-23T20:50:55.242009+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:17:45.039963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":126,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-09-23T20:50:55.241602+0000","last_change":"2023-09-23T20:47:09.181240+0000","last_active":"2023-09-23T20:50:55.241602+0000","last_peered":"2023-09-23T20:50:55.241602+0000","last_clean":"2023-09-23T20:50:55.241602+0000","last_became_active":"2023-09-23T20:47:09.181022+0000","last_became_peered":"2023-09-23T20:47:09.181022+0000","last_unstale":"2023-09-23T20:50:55.241602+0000","last_undegraded":"2023-09-23T20:50:55.241602+0000","last_fullsized":"2023-09-23T20:50:55.241602+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:38:59.191781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"90'161","reported_seq":925,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-09-23T20:50:53.794439+0000","last_change":"2023-09-23T20:38:05.591992+0000","last_active":"2023-09-23T20:50:53.794439+0000","last_peered":"2023-09-23T20:50:53.794439+0000","last_clean":"2023-09-23T20:50:53.794439+0000","last_became_active":"2023-09-23T20:38:05.591701+0000","last_became_peered":"2023-09-23T20:38:05.591701+0000","last_unstale":"2023-09-23T20:50:53.794439+0000","last_undegraded":"2023-09-23T20:50:53.794439+0000","last_fullsized":"2023-09-23T20:50:53.794439+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:02.384133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:02.384133+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:41.722542+0000","scrub_duration":0.001769098,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":513,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-09-23T20:50:55.357254+0000","last_change":"2023-09-23T20:48:48.158348+0000","last_active":"2023-09-23T20:50:55.357254+0000","last_peered":"2023-09-23T20:50:55.357254+0000","last_clean":"2023-09-23T20:50:55.357254+0000","last_became_active":"2023-09-23T20:48:48.157982+0000","last_became_peered":"2023-09-23T20:48:48.157982+0000","last_unstale":"2023-09-23T20:50:55.357254+0000","last_undegraded":"2023-09-23T20:50:55.357254+0000","last_fullsized":"2023-09-23T20:50:55.357254+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:03:48.078707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":479,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-09-23T20:50:55.356326+0000","last_change":"2023-09-23T20:48:48.159781+0000","last_active":"2023-09-23T20:50:55.356326+0000","last_peered":"2023-09-23T20:50:55.356326+0000","last_clean":"2023-09-23T20:50:55.356326+0000","last_became_active":"2023-09-23T20:48:48.159505+0000","last_became_peered":"2023-09-23T20:48:48.159505+0000","last_unstale":"2023-09-23T20:50:55.356326+0000","last_undegraded":"2023-09-23T20:50:55.356326+0000","last_fullsized":"2023-09-23T20:50:55.356326+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:25:18.100645+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"90'161","reported_seq":799,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-09-23T20:50:52.793940+0000","last_change":"2023-09-23T20:47:11.621758+0000","last_active":"2023-09-23T20:50:52.793940+0000","last_peered":"2023-09-23T20:50:52.793940+0000","last_clean":"2023-09-23T20:50:52.793940+0000","last_became_active":"2023-09-23T20:47:11.621345+0000","last_became_peered":"2023-09-23T20:47:11.621345+0000","last_unstale":"2023-09-23T20:50:52.793940+0000","last_undegraded":"2023-09-23T20:50:52.793940+0000","last_fullsized":"2023-09-23T20:50:52.793940+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:01.504973+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:01.504973+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:31:17.112393+0000","scrub_duration":0.00202344,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":749,"reported_epoch":668,"state":"active+clean","last_fresh":"2023-09-23T20:50:52.791097+0000","last_change":"2023-09-23T20:49:48.125242+0000","last_active":"2023-09-23T20:50:52.791097+0000","last_peered":"2023-09-23T20:50:52.791097+0000","last_clean":"2023-09-23T20:50:52.791097+0000","last_became_active":"2023-09-23T20:49:48.111689+0000","last_became_peered":"2023-09-23T20:49:48.111689+0000","last_unstale":"2023-09-23T20:50:52.791097+0000","last_undegraded":"2023-09-23T20:50:52.791097+0000","last_fullsized":"2023-09-23T20:50:52.791097+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:49:38.401151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":4986,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":136,"reported_epoch":669,"state":"active+clean","last_fresh":"2023-09-23T20:50:55.241177+0000","last_change":"2023-09-23T20:48:23.917302+0000","last_active":"2023-09-23T20:50:55.241177+0000","last_peered":"2023-09-23T20:50:55.241177+0000","last_clean":"2023-09-23T20:50:55.241177+0000","last_became_active":"2023-09-23T20:48:23.916956+0000","last_became_peered":"2023-09-23T20:48:23.916956+0000","last_unstale":"2023-09-23T20:50:55.241177+0000","last_undegraded":"2023-09-23T20:50:55.241177+0000","last_fullsized":"2023-09-23T20:50:55.241177+0000","mapping_epoch":577,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":578,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:42:27.989294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":118,"num_bytes_recovered":15844,"num_keys_recovered":80,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26691,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2048000,"data_stored":2033792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":8388627,"num_objects":5,"num_object_clones":0,"num_object_copies":15,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":11981,"num_read_kb":26415,"num_write":1690,"num_write_kb":25390,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":156,"num_bytes_recovered":25165843,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":25182208,"data_stored":25165900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6082,"internal_metadata":0},"log_size":1950,"ondisk_log_size":1950,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201755,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":619804,"kb_used_data":11220,"kb_used_omap":0,"kb_used_meta":608576,"kb_avail":93125348,"statfs":{"total":95995035648,"available":95360356352,"internally_reserved":0,"allocated":11489280,"data_stored":9700716,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":623181824},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:50:12 2023","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":1.325,"15min":1.9239999999999999},"min":{"1min":0.254,"5min":0.224,"15min":0.224},"max":{"1min":3.9580000000000002,"5min":10.975,"15min":18.315000000000001},"last":0.437},{"interface":"front","average":{"1min":0.84999999999999998,"5min":1.5960000000000001,"15min":1.73},"min":{"1min":0.32100000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":4.2039999999999997,"5min":12.986000000000001,"15min":19.355},"last":0.66700000000000004}]},{"osd":1,"last update":"Sat Sep 23 20:50:18 2023","interfaces":[{"interface":"back","average":{"1min":0.74299999999999999,"5min":1.3680000000000001,"15min":1.849},"min":{"1min":0.33200000000000002,"5min":0.25900000000000001,"15min":0.24399999999999999},"max":{"1min":3.7770000000000001,"5min":14.249000000000001,"15min":49.965000000000003},"last":0.503},{"interface":"front","average":{"1min":0.95899999999999996,"5min":1.645,"15min":2.3290000000000002},"min":{"1min":0.27900000000000003,"5min":0.27000000000000002,"15min":0.26700000000000002},"max":{"1min":5.3390000000000004,"5min":10.861000000000001,"15min":53.451999999999998},"last":0.47099999999999997}]},{"osd":2,"last update":"Sat Sep 23 20:50:12 2023","interfaces":[{"interface":"back","average":{"1min":1.5680000000000001,"5min":1.7010000000000001,"15min":2.048},"min":{"1min":0.28799999999999998,"5min":0.28499999999999998,"15min":0.27800000000000002},"max":{"1min":10.461,"5min":12.933999999999999,"15min":16.965},"last":0.55700000000000005},{"interface":"front","average":{"1min":0.59499999999999997,"5min":1.571,"15min":2.0800000000000001},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":2.0049999999999999,"5min":7.9729999999999999,"15min":23.414000000000001},"last":0.60999999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:50:42 2023","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":1.464,"15min":1.8440000000000001},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.311},"max":{"1min":1.901,"5min":7.2880000000000003,"15min":23.352},"last":0.57799999999999996},{"interface":"front","average":{"1min":0.60199999999999998,"5min":1.635,"15min":2.0470000000000002},"min":{"1min":0.42099999999999999,"5min":0.35399999999999998,"15min":0.34300000000000003},"max":{"1min":1.8879999999999999,"5min":6.5369999999999999,"15min":20.969999999999999},"last":0.70099999999999996}]},{"osd":4,"last update":"Sat Sep 23 20:50:42 2023","interfaces":[{"interface":"back","average":{"1min":0.44600000000000001,"5min":1.198,"15min":1.206},"min":{"1min":0.29599999999999999,"5min":0.254,"15min":0.254},"max":{"1min":0.78900000000000003,"5min":7.4109999999999996,"15min":18.364999999999998},"last":0.59199999999999997},{"interface":"front","average":{"1min":0.46999999999999997,"5min":1.456,"15min":1.49},"min":{"1min":0.33600000000000002,"5min":0.247,"15min":0.247},"max":{"1min":0.68700000000000006,"5min":27.425999999999998,"15min":27.425999999999998},"last":0.628}]},{"osd":5,"last update":"Sat Sep 23 20:50:42 2023","interfaces":[{"interface":"back","average":{"1min":0.502,"5min":0.95399999999999996,"15min":1.248},"min":{"1min":0.316,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":1.6479999999999999,"5min":7.2270000000000003,"15min":15.005000000000001},"last":0.52100000000000002},{"interface":"front","average":{"1min":0.52200000000000002,"5min":1.0589999999999999,"15min":1.3839999999999999},"min":{"1min":0.36199999999999999,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.79800000000000004,"5min":7.335,"15min":18.390000000000001},"last":0.68300000000000005}]},{"osd":6,"last update":"Sat Sep 23 20:50:42 2023","interfaces":[{"interface":"back","average":{"1min":0.74199999999999999,"5min":1.405,"15min":1.5640000000000001},"min":{"1min":0.39200000000000002,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":4.4969999999999999,"5min":12.183,"15min":18.417000000000002},"last":0.64700000000000002},{"interface":"front","average":{"1min":0.57499999999999996,"5min":1.397,"15min":1.4019999999999999},"min":{"1min":0.38,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":1.2709999999999999,"5min":11.404999999999999,"15min":15.369999999999999},"last":0.72999999999999998}]}]},{"osd":6,"up_from":47,"seq":201863463394,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":542940,"kb_used_data":15380,"kb_used_omap":0,"kb_used_meta":527552,"kb_avail":93202212,"statfs":{"total":95995035648,"available":95439065088,"internally_reserved":0,"allocated":15749120,"data_stored":13968175,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":540213248},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:50:06 2023","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":1.821,"15min":1.7509999999999999},"min":{"1min":0.255,"5min":0.23400000000000001,"15min":0.219},"max":{"1min":2.044,"5min":21.018000000000001,"15min":21.018000000000001},"last":0.82999999999999996},{"interface":"front","average":{"1min":1.04,"5min":1.9590000000000001,"15min":2.0249999999999999},"min":{"1min":0.27200000000000002,"5min":0.255,"15min":0.22700000000000001},"max":{"1min":8.7729999999999997,"5min":21.082000000000001,"15min":60.328000000000003},"last":0.92700000000000005}]},{"osd":1,"last update":"Sat Sep 23 20:50:23 2023","interfaces":[{"interface":"back","average":{"1min":0.64400000000000002,"5min":1.6359999999999999,"15min":1.865},"min":{"1min":0.30399999999999999,"5min":0.26000000000000001,"15min":0.249},"max":{"1min":4.266,"5min":8.9350000000000005,"15min":64.274000000000001},"last":0.66400000000000003},{"interface":"front","average":{"1min":0.52300000000000002,"5min":1.2470000000000001,"15min":1.5469999999999999},"min":{"1min":0.317,"5min":0.312,"15min":0.28000000000000003},"max":{"1min":0.85999999999999999,"5min":8.8070000000000004,"15min":17.786000000000001},"last":0.64400000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:50:06 2023","interfaces":[{"interface":"back","average":{"1min":1.306,"5min":2.4049999999999998,"15min":2.1299999999999999},"min":{"1min":0.36699999999999999,"5min":0.29699999999999999,"15min":0.27700000000000002},"max":{"1min":8.9079999999999995,"5min":22.591000000000001,"15min":22.591000000000001},"last":0.78500000000000003},{"interface":"front","average":{"1min":1.4039999999999999,"5min":2.0169999999999999,"15min":1.8959999999999999},"min":{"1min":0.375,"5min":0.33900000000000002,"15min":0.30399999999999999},"max":{"1min":12.33,"5min":22.614999999999998,"15min":22.614999999999998},"last":0.80100000000000005}]},{"osd":3,"last update":"Sat Sep 23 20:50:59 2023","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":1.698,"15min":1.798},"min":{"1min":0.35099999999999998,"5min":0.307,"15min":0.28399999999999997},"max":{"1min":0.89900000000000002,"5min":9.6859999999999999,"15min":11.664},"last":0.89900000000000002},{"interface":"front","average":{"1min":0.61899999999999999,"5min":2.0409999999999999,"15min":1.9610000000000001},"min":{"1min":0.44900000000000001,"5min":0.32700000000000001,"15min":0.30499999999999999},"max":{"1min":0.96499999999999997,"5min":11.6,"15min":11.913},"last":0.96499999999999997}]},{"osd":4,"last update":"Sat Sep 23 20:50:23 2023","interfaces":[{"interface":"back","average":{"1min":0.65100000000000002,"5min":1.0840000000000001,"15min":0.95399999999999996},"min":{"1min":0.311,"5min":0.28399999999999997,"15min":0.25600000000000001},"max":{"1min":4.2220000000000004,"5min":10.183999999999999,"15min":23.904},"last":0.76600000000000001},{"interface":"front","average":{"1min":0.63900000000000001,"5min":1.5640000000000001,"15min":1.2609999999999999},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.28399999999999997},"max":{"1min":3.2919999999999998,"5min":11.999000000000001,"15min":11.999000000000001},"last":0.61399999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:50:23 2023","interfaces":[{"interface":"back","average":{"1min":0.52100000000000002,"5min":1.0820000000000001,"15min":0.98999999999999999},"min":{"1min":0.34200000000000003,"5min":0.30499999999999999,"15min":0.253},"max":{"1min":0.86599999999999999,"5min":8.0440000000000005,"15min":8.0440000000000005},"last":0.72299999999999998},{"interface":"front","average":{"1min":0.72199999999999998,"5min":1.5,"15min":1.3220000000000001},"min":{"1min":0.36499999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":4.2329999999999997,"5min":9.3559999999999999,"15min":9.3559999999999999},"last":0.73899999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:50:59 2023","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":1.105,"15min":0.99299999999999999},"min":{"1min":0.40200000000000002,"5min":0.35199999999999998,"15min":0.28399999999999997},"max":{"1min":0.80800000000000005,"5min":8.9309999999999992,"15min":8.9309999999999992},"last":0.58299999999999996},{"interface":"front","average":{"1min":0.623,"5min":3.2709999999999999,"15min":1.9810000000000001},"min":{"1min":0.433,"5min":0.38700000000000001,"15min":0.34699999999999998},"max":{"1min":0.86099999999999999,"5min":159.91999999999999,"15min":159.91999999999999},"last":0.86099999999999999}]}]},{"osd":1,"up_from":266,"seq":1142461300992,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":823756,"kb_used_data":12932,"kb_used_omap":8,"kb_used_meta":810807,"kb_avail":92921396,"statfs":{"total":95995035648,"available":95151509504,"internally_reserved":0,"allocated":13242368,"data_stored":9907248,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8206,"internal_metadata":830267378},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:50:29 2023","interfaces":[{"interface":"back","average":{"1min":0.48799999999999999,"5min":1.917,"15min":2.214},"min":{"1min":0.28199999999999997,"5min":0.20499999999999999,"15min":0.20000000000000001},"max":{"1min":2.8809999999999998,"5min":20.292000000000002,"15min":20.292000000000002},"last":0.35599999999999998},{"interface":"front","average":{"1min":0.56799999999999995,"5min":1.8260000000000001,"15min":2.5760000000000001},"min":{"1min":0.28499999999999998,"5min":0.217,"15min":0.19900000000000001},"max":{"1min":2.6890000000000001,"5min":17.463000000000001,"15min":20.346},"last":0.33800000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:50:29 2023","interfaces":[{"interface":"back","average":{"1min":0.76900000000000002,"5min":2.0630000000000002,"15min":2.6110000000000002},"min":{"1min":0.25900000000000001,"5min":0.23100000000000001,"15min":0.216},"max":{"1min":2.7970000000000002,"5min":23.341000000000001,"15min":32.255000000000003},"last":0.313},{"interface":"front","average":{"1min":0.61099999999999999,"5min":2.173,"15min":2.1280000000000001},"min":{"1min":0.34899999999999998,"5min":0.249,"15min":0.20899999999999999},"max":{"1min":2.9279999999999999,"5min":34.002000000000002,"15min":34.002000000000002},"last":0.39500000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:50:29 2023","interfaces":[{"interface":"back","average":{"1min":0.63600000000000001,"5min":1.621,"15min":2.0979999999999999},"min":{"1min":0.316,"5min":0.28799999999999998,"15min":0.247},"max":{"1min":2.9590000000000001,"5min":17.366,"15min":17.366},"last":0.44800000000000001},{"interface":"front","average":{"1min":0.748,"5min":2,"15min":2.3839999999999999},"min":{"1min":0.35099999999999998,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":2.7330000000000001,"5min":17.343,"15min":19.315000000000001},"last":0.40200000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:50:29 2023","interfaces":[{"interface":"back","average":{"1min":1.079,"5min":1.8839999999999999,"15min":2.2229999999999999},"min":{"1min":0.42699999999999999,"5min":0.32600000000000001,"15min":0.28599999999999998},"max":{"1min":5.7110000000000003,"5min":9.5359999999999996,"15min":20.634},"last":1.7010000000000001},{"interface":"front","average":{"1min":0.69299999999999995,"5min":1.5600000000000001,"15min":1.7190000000000001},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.311},"max":{"1min":2.8370000000000002,"5min":9.8249999999999993,"15min":20.835999999999999},"last":1.637}]},{"osd":5,"last update":"Sat Sep 23 20:50:29 2023","interfaces":[{"interface":"back","average":{"1min":0.755,"5min":1.819,"15min":2.2090000000000001},"min":{"1min":0.42199999999999999,"5min":0.34699999999999998,"15min":0.308},"max":{"1min":3.1120000000000001,"5min":17.396999999999998,"15min":33.401000000000003},"last":1.724},{"interface":"front","average":{"1min":0.93999999999999995,"5min":1.962,"15min":2.4279999999999999},"min":{"1min":0.371,"5min":0.371,"15min":0.32500000000000001},"max":{"1min":3.0670000000000002,"5min":16.366,"15min":33.417000000000002},"last":1.736}]},{"osd":6,"last update":"Sat Sep 23 20:50:29 2023","interfaces":[{"interface":"back","average":{"1min":0.80400000000000005,"5min":2.105,"15min":2.1960000000000002},"min":{"1min":0.46899999999999997,"5min":0.316,"15min":0.316},"max":{"1min":3.0390000000000001,"5min":23.274000000000001,"15min":23.274000000000001},"last":1.7509999999999999},{"interface":"front","average":{"1min":1.0600000000000001,"5min":2.1440000000000001,"15min":2.4279999999999999},"min":{"1min":0.48399999999999999,"5min":0.40500000000000003,"15min":0.317},"max":{"1min":4.4690000000000003,"5min":9.6639999999999997,"15min":11.832000000000001},"last":1.669}]},{"osd":7,"last update":"Sat Sep 23 20:50:29 2023","interfaces":[{"interface":"back","average":{"1min":0.79400000000000004,"5min":1.9730000000000001,"15min":2.2389999999999999},"min":{"1min":0.51100000000000001,"5min":0.377,"15min":0.37},"max":{"1min":3.0819999999999999,"5min":17.433,"15min":18.437000000000001},"last":1.6870000000000001},{"interface":"front","average":{"1min":0.98299999999999998,"5min":2.1419999999999999,"15min":2.472},"min":{"1min":0.47699999999999998,"5min":0.41899999999999998,"15min":0.35999999999999999},"max":{"1min":3.0259999999999998,"5min":9.8369999999999997,"15min":12.021000000000001},"last":1.7609999999999999}]}]},{"osd":0,"up_from":94,"seq":403726926195,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":989984,"kb_used_data":12824,"kb_used_omap":7,"kb_used_meta":977144,"kb_avail":92755168,"statfs":{"total":95995035648,"available":94981292032,"internally_reserved":0,"allocated":13131776,"data_stored":9953520,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":1000595473},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:50:33 2023","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":3.1699999999999999,"15min":2.4630000000000001},"min":{"1min":0.25600000000000001,"5min":0.20100000000000001,"15min":0.17199999999999999},"max":{"1min":2.4089999999999998,"5min":152.387,"15min":152.387},"last":0.28599999999999998},{"interface":"front","average":{"1min":0.52100000000000002,"5min":3.1629999999999998,"15min":2.633},"min":{"1min":0.32300000000000001,"5min":0.218,"15min":0.20399999999999999},"max":{"1min":1.4259999999999999,"5min":152.84100000000001,"15min":152.84100000000001},"last":0.33200000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:50:11 2023","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":2.2490000000000001,"15min":2.6869999999999998},"min":{"1min":0.26700000000000002,"5min":0.23499999999999999,"15min":0.218},"max":{"1min":1.9410000000000001,"5min":14.538,"15min":62.726999999999997},"last":0.34799999999999998},{"interface":"front","average":{"1min":1.1419999999999999,"5min":2.226,"15min":2.6600000000000001},"min":{"1min":0.26100000000000001,"5min":0.217,"15min":0.217},"max":{"1min":10.159000000000001,"5min":14.167,"15min":62.689},"last":0.379}]},{"osd":3,"last update":"Sat Sep 23 20:50:46 2023","interfaces":[{"interface":"back","average":{"1min":0.442,"5min":2.6070000000000002,"15min":2.718},"min":{"1min":0.29299999999999998,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.97899999999999998,"5min":55.017000000000003,"15min":55.017000000000003},"last":0.309},{"interface":"front","average":{"1min":0.69099999999999995,"5min":2.673,"15min":2.6680000000000001},"min":{"1min":0.372,"5min":0.27400000000000002,"15min":0.21199999999999999},"max":{"1min":2.9140000000000001,"5min":55.665999999999997,"15min":55.665999999999997},"last":0.41499999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:50:11 2023","interfaces":[{"interface":"back","average":{"1min":0.99199999999999999,"5min":1.964,"15min":2.008},"min":{"1min":0.34300000000000003,"5min":0.29299999999999998,"15min":0.28899999999999998},"max":{"1min":4.2859999999999996,"5min":11.75,"15min":11.75},"last":0.52100000000000002},{"interface":"front","average":{"1min":1.2549999999999999,"5min":2.0230000000000001,"15min":1.853},"min":{"1min":0.42099999999999999,"5min":0.31,"15min":0.27400000000000002},"max":{"1min":5.4279999999999999,"5min":12.356999999999999,"15min":12.356999999999999},"last":0.45300000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:50:11 2023","interfaces":[{"interface":"back","average":{"1min":0.83999999999999997,"5min":1.8029999999999999,"15min":1.9399999999999999},"min":{"1min":0.38500000000000001,"5min":0.308,"15min":0.26500000000000001},"max":{"1min":3.012,"5min":9.3339999999999996,"15min":11.497999999999999},"last":0.56999999999999995},{"interface":"front","average":{"1min":0.88600000000000001,"5min":1.7210000000000001,"15min":2.2189999999999999},"min":{"1min":0.33000000000000002,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":3.181,"5min":10.089,"15min":26.228000000000002},"last":0.51200000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:50:11 2023","interfaces":[{"interface":"back","average":{"1min":1.395,"5min":2.1669999999999998,"15min":2.1360000000000001},"min":{"1min":0.441,"5min":0.374,"15min":0.34499999999999997},"max":{"1min":4.3540000000000001,"5min":9.2750000000000004,"15min":24.056000000000001},"last":0.53300000000000003},{"interface":"front","average":{"1min":1.391,"5min":1.974,"15min":2.1579999999999999},"min":{"1min":0.372,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":4.1669999999999998,"5min":12.427,"15min":26.268000000000001},"last":0.47699999999999998}]},{"osd":7,"last update":"Sat Sep 23 20:50:11 2023","interfaces":[{"interface":"back","average":{"1min":1.0900000000000001,"5min":2.1320000000000001,"15min":2.3199999999999998},"min":{"1min":0.39100000000000001,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":3.8919999999999999,"5min":16.553999999999998,"15min":26.177},"last":0.40500000000000003},{"interface":"front","average":{"1min":1.1579999999999999,"5min":2.1899999999999999,"15min":2.3300000000000001},"min":{"1min":0.45700000000000002,"5min":0.40200000000000002,"15min":0.34200000000000003},"max":{"1min":4.3209999999999997,"5min":16.649000000000001,"15min":16.649000000000001},"last":0.48999999999999999}]}]},{"osd":2,"up_from":104,"seq":446676599142,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":888736,"kb_used_data":12696,"kb_used_omap":11,"kb_used_meta":876020,"kb_avail":92856416,"statfs":{"total":95995035648,"available":95084969984,"internally_reserved":0,"allocated":13000704,"data_stored":9876054,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11878,"internal_metadata":897044890},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:50:12 2023","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":1.5900000000000001,"15min":1.603},"min":{"1min":0.26000000000000001,"5min":0.19500000000000001,"15min":0.17299999999999999},"max":{"1min":2.669,"5min":20.48,"15min":20.48},"last":0.27300000000000002},{"interface":"front","average":{"1min":0.51600000000000001,"5min":1.423,"15min":1.982},"min":{"1min":0.31900000000000001,"5min":0.245,"15min":0.19},"max":{"1min":1.4279999999999999,"5min":15.342000000000001,"15min":16.678999999999998},"last":0.318}]},{"osd":1,"last update":"Sat Sep 23 20:50:12 2023","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":1.5600000000000001,"15min":2.2429999999999999},"min":{"1min":0.24299999999999999,"5min":0.219,"15min":0.219},"max":{"1min":1.1279999999999999,"5min":21.690000000000001,"15min":46.405000000000001},"last":0.33100000000000002},{"interface":"front","average":{"1min":0.496,"5min":1.54,"15min":2.1930000000000001},"min":{"1min":0.26800000000000002,"5min":0.22,"15min":0.22},"max":{"1min":1.0449999999999999,"5min":10.191000000000001,"15min":46.459000000000003},"last":0.39500000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:50:42 2023","interfaces":[{"interface":"back","average":{"1min":0.40500000000000003,"5min":1.8799999999999999,"15min":2.2090000000000001},"min":{"1min":0.30399999999999999,"5min":0.28199999999999997,"15min":0.253},"max":{"1min":0.51300000000000001,"5min":30.283000000000001,"15min":30.283000000000001},"last":0.38},{"interface":"front","average":{"1min":0.45000000000000001,"5min":2.0070000000000001,"15min":1.6379999999999999},"min":{"1min":0.29399999999999998,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.64100000000000001,"5min":30.834,"15min":30.834},"last":0.35799999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:50:12 2023","interfaces":[{"interface":"back","average":{"1min":0.56799999999999995,"5min":1.5449999999999999,"15min":1.736},"min":{"1min":0.38400000000000001,"5min":0.29599999999999999,"15min":0.27500000000000002},"max":{"1min":0.79500000000000004,"5min":8.3469999999999995,"15min":12.025},"last":0.60599999999999998},{"interface":"front","average":{"1min":0.67400000000000004,"5min":1.627,"15min":1.819},"min":{"1min":0.42099999999999999,"5min":0.32400000000000001,"15min":0.30599999999999999},"max":{"1min":1.8520000000000001,"5min":12.662000000000001,"15min":19.440999999999999},"last":0.79700000000000004}]},{"osd":5,"last update":"Sat Sep 23 20:50:12 2023","interfaces":[{"interface":"back","average":{"1min":0.90100000000000002,"5min":1.3660000000000001,"15min":1.76},"min":{"1min":0.44800000000000001,"5min":0.33200000000000002,"15min":0.311},"max":{"1min":4.8570000000000002,"5min":8.5380000000000003,"15min":12.07},"last":0.75900000000000001},{"interface":"front","average":{"1min":0.93300000000000005,"5min":1.5349999999999999,"15min":1.7969999999999999},"min":{"1min":0.437,"5min":0.373,"15min":0.35299999999999998},"max":{"1min":4.8639999999999999,"5min":10.831,"15min":19.513000000000002},"last":0.77100000000000002}]},{"osd":6,"last update":"Sat Sep 23 20:50:12 2023","interfaces":[{"interface":"back","average":{"1min":0.63900000000000001,"5min":1.6619999999999999,"15min":1.9179999999999999},"min":{"1min":0.40200000000000002,"5min":0.34799999999999998,"15min":0.33500000000000002},"max":{"1min":1.177,"5min":10.791,"15min":19.486000000000001},"last":0.56100000000000005},{"interface":"front","average":{"1min":0.80400000000000005,"5min":1.486,"15min":1.415},"min":{"1min":0.433,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":2.6440000000000001,"5min":7.4180000000000001,"15min":11.747},"last":0.69399999999999995}]},{"osd":7,"last update":"Sat Sep 23 20:50:12 2023","interfaces":[{"interface":"back","average":{"1min":0.80100000000000005,"5min":1.8700000000000001,"15min":1.5229999999999999},"min":{"1min":0.52100000000000002,"5min":0.35399999999999998,"15min":0.34699999999999998},"max":{"1min":2.581,"5min":20.530000000000001,"15min":20.530000000000001},"last":0.71899999999999997},{"interface":"front","average":{"1min":0.67800000000000005,"5min":1.71,"15min":1.9830000000000001},"min":{"1min":0.45800000000000002,"5min":0.36899999999999999,"15min":0.34899999999999998},"max":{"1min":1.141,"5min":10.071999999999999,"15min":12.099},"last":0.63}]}]},{"osd":3,"up_from":109,"seq":468151435616,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1081312,"kb_used_data":8664,"kb_used_omap":4,"kb_used_meta":1072635,"kb_avail":92663840,"statfs":{"total":95995035648,"available":94887772160,"internally_reserved":0,"allocated":8871936,"data_stored":5753844,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4514,"internal_metadata":1098378846},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:50:51 2023","interfaces":[{"interface":"back","average":{"1min":0.42499999999999999,"5min":1.8380000000000001,"15min":1.9970000000000001},"min":{"1min":0.26000000000000001,"5min":0.217,"15min":0.20899999999999999},"max":{"1min":1.071,"5min":21.792999999999999,"15min":22.32},"last":0.34899999999999998},{"interface":"front","average":{"1min":0.45800000000000002,"5min":1.6220000000000001,"15min":2.073},"min":{"1min":0.32700000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":1.1020000000000001,"5min":18.579999999999998,"15min":22.805},"last":0.40699999999999997}]},{"osd":1,"last update":"Sat Sep 23 20:50:23 2023","interfaces":[{"interface":"back","average":{"1min":0.83899999999999997,"5min":1.321,"15min":1.7689999999999999},"min":{"1min":0.22700000000000001,"5min":0.21299999999999999,"15min":0.21099999999999999},"max":{"1min":10.24,"5min":11.298,"15min":17.529},"last":0.29099999999999998},{"interface":"front","average":{"1min":0.59499999999999997,"5min":1.454,"15min":2.036},"min":{"1min":0.26500000000000001,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":4.2480000000000002,"5min":11.380000000000001,"15min":14.19},"last":0.437}]},{"osd":2,"last update":"Sat Sep 23 20:50:51 2023","interfaces":[{"interface":"back","average":{"1min":0.42599999999999999,"5min":1.5900000000000001,"15min":2.3820000000000001},"min":{"1min":0.28000000000000003,"5min":0.254,"15min":0.23499999999999999},"max":{"1min":1.194,"5min":7.9210000000000003,"15min":68.206999999999994},"last":0.50600000000000001},{"interface":"front","average":{"1min":0.42999999999999999,"5min":1.4159999999999999,"15min":2.121},"min":{"1min":0.314,"5min":0.25600000000000001,"15min":0.245},"max":{"1min":0.73899999999999999,"5min":16.559000000000001,"15min":67.736999999999995},"last":0.42099999999999999}]},{"osd":4,"last update":"Sat Sep 23 20:50:51 2023","interfaces":[{"interface":"back","average":{"1min":0.84599999999999997,"5min":1.458,"15min":1.5629999999999999},"min":{"1min":0.38300000000000001,"5min":0.35399999999999998,"15min":0.30499999999999999},"max":{"1min":6.4059999999999997,"5min":9.0259999999999998,"15min":22.157},"last":0.70199999999999996},{"interface":"front","average":{"1min":0.54600000000000004,"5min":1.603,"15min":1.9219999999999999},"min":{"1min":0.35099999999999998,"5min":0.33500000000000002,"15min":0.33000000000000002},"max":{"1min":0.81399999999999995,"5min":16.529,"15min":16.529},"last":0.60699999999999998}]},{"osd":5,"last update":"Sat Sep 23 20:50:51 2023","interfaces":[{"interface":"back","average":{"1min":0.88100000000000001,"5min":1.575,"15min":1.7150000000000001},"min":{"1min":0.41699999999999998,"5min":0.32400000000000001,"15min":0.32300000000000001},"max":{"1min":6.4800000000000004,"5min":8.9849999999999994,"15min":22.289999999999999},"last":0.64500000000000002},{"interface":"front","average":{"1min":0.60399999999999998,"5min":1.4159999999999999,"15min":1.6479999999999999},"min":{"1min":0.42899999999999999,"5min":0.36399999999999999,"15min":0.34100000000000003},"max":{"1min":1.135,"5min":6.1390000000000002,"15min":12.952},"last":0.66200000000000003}]},{"osd":6,"last update":"Sat Sep 23 20:50:51 2023","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":1.9430000000000001,"15min":1.8660000000000001},"min":{"1min":0.42799999999999999,"5min":0.39800000000000002,"15min":0.34300000000000003},"max":{"1min":1.163,"5min":8.6419999999999995,"15min":12.917},"last":0.57299999999999995},{"interface":"front","average":{"1min":0.57099999999999995,"5min":1.647,"15min":1.9750000000000001},"min":{"1min":0.47199999999999998,"5min":0.36699999999999999,"15min":0.34599999999999997},"max":{"1min":0.69999999999999996,"5min":16.497,"15min":16.497},"last":0.499}]},{"osd":7,"last update":"Sat Sep 23 20:50:51 2023","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":1.6140000000000001,"15min":2.1949999999999998},"min":{"1min":0.44600000000000001,"5min":0.42399999999999999,"15min":0.39900000000000002},"max":{"1min":0.77000000000000002,"5min":16.425000000000001,"15min":16.425000000000001},"last":0.47899999999999998},{"interface":"front","average":{"1min":0.89600000000000002,"5min":1.9039999999999999,"15min":1.8879999999999999},"min":{"1min":0.501,"5min":0.45600000000000002,"15min":0.40999999999999998},"max":{"1min":6.5259999999999998,"5min":11.891999999999999,"15min":22.202999999999999},"last":0.58699999999999997}]}]},{"osd":4,"up_from":30,"seq":128849019370,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":571992,"kb_used_data":6864,"kb_used_omap":0,"kb_used_meta":565120,"kb_avail":93173160,"statfs":{"total":95995035648,"available":95409315840,"internally_reserved":0,"allocated":7028736,"data_stored":5251067,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":578682880},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:50:07 2023","interfaces":[{"interface":"back","average":{"1min":0.63200000000000001,"5min":2.1230000000000002,"15min":1.974},"min":{"1min":0.217,"5min":0.217,"15min":0.217},"max":{"1min":3.351,"5min":73.272000000000006,"15min":73.272000000000006},"last":0.42299999999999999},{"interface":"front","average":{"1min":0.85899999999999999,"5min":1.8919999999999999,"15min":2.0350000000000001},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.246},"max":{"1min":7.2359999999999998,"5min":29.754000000000001,"15min":29.754000000000001},"last":0.57399999999999995}]},{"osd":1,"last update":"Sat Sep 23 20:50:17 2023","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":1.544,"15min":1.7050000000000001},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.25900000000000001},"max":{"1min":1.9259999999999999,"5min":12.343,"15min":18.190999999999999},"last":0.68799999999999994},{"interface":"front","average":{"1min":0.74299999999999999,"5min":1.734,"15min":1.6459999999999999},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.26400000000000001},"max":{"1min":3.198,"5min":12.231999999999999,"15min":18.173999999999999},"last":0.504}]},{"osd":2,"last update":"Sat Sep 23 20:50:17 2023","interfaces":[{"interface":"back","average":{"1min":0.85599999999999998,"5min":1.7470000000000001,"15min":1.72},"min":{"1min":0.28000000000000003,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":3.2469999999999999,"5min":12.151999999999999,"15min":13.051},"last":0.65400000000000003},{"interface":"front","average":{"1min":0.72299999999999998,"5min":1.6899999999999999,"15min":1.986},"min":{"1min":0.28899999999999998,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":3.4319999999999999,"5min":12.458,"15min":12.458},"last":0.67500000000000004}]},{"osd":3,"last update":"Sat Sep 23 20:50:36 2023","interfaces":[{"interface":"back","average":{"1min":0.79700000000000004,"5min":1.796,"15min":1.8720000000000001},"min":{"1min":0.376,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":3.0840000000000001,"5min":11.598000000000001,"15min":12.147},"last":0.59999999999999998},{"interface":"front","average":{"1min":0.63900000000000001,"5min":1.9179999999999999,"15min":1.978},"min":{"1min":0.45600000000000002,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":1.323,"5min":12.191000000000001,"15min":17.169},"last":0.72799999999999998}]},{"osd":5,"last update":"Sat Sep 23 20:50:07 2023","interfaces":[{"interface":"back","average":{"1min":0.49399999999999999,"5min":1.393,"15min":1.1919999999999999},"min":{"1min":0.26100000000000001,"5min":0.25800000000000001,"15min":0.253},"max":{"1min":1.214,"5min":16.940000000000001,"15min":16.940000000000001},"last":0.753},{"interface":"front","average":{"1min":0.53600000000000003,"5min":1.264,"15min":1.1819999999999999},"min":{"1min":0.26700000000000002,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":1.9610000000000001,"5min":6.5839999999999996,"15min":8.1440000000000001},"last":0.77400000000000002}]},{"osd":6,"last update":"Sat Sep 23 20:50:17 2023","interfaces":[{"interface":"back","average":{"1min":1.22,"5min":1.716,"15min":1.452},"min":{"1min":0.31900000000000001,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":7.4589999999999996,"5min":13.503,"15min":14.731999999999999},"last":0.71399999999999997},{"interface":"front","average":{"1min":0.72399999999999998,"5min":1.8080000000000001,"15min":1.4039999999999999},"min":{"1min":0.29899999999999999,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":4.0830000000000002,"5min":17.059000000000001,"15min":17.059000000000001},"last":0.55200000000000005}]},{"osd":7,"last update":"Sat Sep 23 20:50:36 2023","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":1.4750000000000001,"15min":2.661},"min":{"1min":0.40200000000000002,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":1.012,"5min":11.228,"15min":313.50099999999998},"last":0.63500000000000001},{"interface":"front","average":{"1min":0.57899999999999996,"5min":1.5,"15min":2.3290000000000002},"min":{"1min":0.438,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.94799999999999995,"5min":17.093,"15min":312.67500000000001},"last":0.61599999999999999}]}]},{"osd":5,"up_from":0,"seq":167503725031,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4722,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1360,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":8392704,"data_stored":8388627,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8206,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7156,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3154,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:51:06.432 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-09-23T20:51:06.432 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T20:51:06.432 INFO:tasks.thrashosds.thrasher:Growing pool .mgr 2023-09-23T20:51:06.433 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:51:07.052 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:51:07.540 INFO:teuthology.orchestra.run.smithi149.stdout:{ 2023-09-23T20:51:07.540 INFO:teuthology.orchestra.run.smithi149.stdout: "size": 20, 2023-09-23T20:51:07.540 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 600, 2023-09-23T20:51:07.541 INFO:teuthology.orchestra.run.smithi149.stdout: "ops": [ 2023-09-23T20:51:07.541 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.541 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:102263 10.e 10:7fa4b63f:::benchmark_data_smithi018_422431_object102262:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e553)", 2023-09-23T20:51:07.541 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:47:09.171743+0000", 2023-09-23T20:51:07.541 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 238.36651525100001, 2023-09-23T20:51:07.541 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0020924090000001, 2023-09-23T20:51:07.541 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:51:07.541 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:51:07.541 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:51:07.541 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:51:07.541 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:51:07.542 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 102263 2023-09-23T20:51:07.542 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.542 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:51:07.542 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.542 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:51:07.542 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.171743+0000", 2023-09-23T20:51:07.542 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:07.542 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.542 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.542 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:51:07.542 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.171743+0000", 2023-09-23T20:51:07.542 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:07.543 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.543 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.543 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:51:07.543 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.171742+0000", 2023-09-23T20:51:07.543 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:51:07.543 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.543 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.543 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:51:07.543 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.191370+0000", 2023-09-23T20:51:07.543 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.019627815 2023-09-23T20:51:07.543 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.543 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.543 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:51:07.544 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.191383+0000", 2023-09-23T20:51:07.544 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.3481e-05 2023-09-23T20:51:07.544 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.544 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.544 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:51:07.544 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.191393+0000", 2023-09-23T20:51:07.545 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0097e-05 2023-09-23T20:51:07.545 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.545 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.545 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:07.545 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.192254+0000", 2023-09-23T20:51:07.545 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00086048600000000004 2023-09-23T20:51:07.545 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.545 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.545 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:51:07.545 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.192264+0000", 2023-09-23T20:51:07.546 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0553999999999999e-05 2023-09-23T20:51:07.546 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.546 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.546 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:07.546 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:10.173760+0000", 2023-09-23T20:51:07.546 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.98149563500000003 2023-09-23T20:51:07.546 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.546 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.546 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:51:07.547 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:10.173836+0000", 2023-09-23T20:51:07.547 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.5891999999999998e-05 2023-09-23T20:51:07.547 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:07.547 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:51:07.547 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:07.547 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.547 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.547 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:102264 10.e 10:70bfcac0:::benchmark_data_smithi018_422431_object102263:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e553)", 2023-09-23T20:51:07.547 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:47:09.191403+0000", 2023-09-23T20:51:07.547 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 238.346855742, 2023-09-23T20:51:07.547 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.98286253400000001, 2023-09-23T20:51:07.547 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:51:07.547 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:51:07.548 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:51:07.548 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:51:07.548 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:51:07.548 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 102264 2023-09-23T20:51:07.548 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.548 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:51:07.548 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.548 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:51:07.548 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.191403+0000", 2023-09-23T20:51:07.548 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:07.548 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.548 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.549 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:51:07.549 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.191403+0000", 2023-09-23T20:51:07.549 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:07.549 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.549 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.549 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:51:07.549 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.191401+0000", 2023-09-23T20:51:07.549 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999976 2023-09-23T20:51:07.549 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.549 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.549 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:51:07.549 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.191476+0000", 2023-09-23T20:51:07.550 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.5279999999999998e-05 2023-09-23T20:51:07.550 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.550 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.550 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:51:07.550 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.191483+0000", 2023-09-23T20:51:07.550 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.7819999999999996e-06 2023-09-23T20:51:07.550 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.550 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.550 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:51:07.550 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.191488+0000", 2023-09-23T20:51:07.550 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.994e-06 2023-09-23T20:51:07.550 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.551 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.551 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:07.551 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.192281+0000", 2023-09-23T20:51:07.551 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00079300400000000004 2023-09-23T20:51:07.551 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.551 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.551 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:51:07.551 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.192285+0000", 2023-09-23T20:51:07.551 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.4440000000000001e-06 2023-09-23T20:51:07.551 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.551 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.551 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:07.552 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:10.174202+0000", 2023-09-23T20:51:07.552 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.98191682599999996 2023-09-23T20:51:07.552 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.552 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.552 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:51:07.553 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:10.174265+0000", 2023-09-23T20:51:07.553 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.3380999999999995e-05 2023-09-23T20:51:07.553 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:07.553 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:51:07.553 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:07.553 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.553 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.554 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:102314 10.e 10:771e9b48:::benchmark_data_smithi018_422431_object102313:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e553)", 2023-09-23T20:51:07.554 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:47:09.319054+0000", 2023-09-23T20:51:07.554 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 238.219205012, 2023-09-23T20:51:07.554 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.87209886700000006, 2023-09-23T20:51:07.554 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:51:07.554 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:07.554 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:51:07.554 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:51:07.554 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:51:07.554 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 102314 2023-09-23T20:51:07.554 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.555 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:51:07.555 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.555 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:51:07.555 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.319054+0000", 2023-09-23T20:51:07.555 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:07.555 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.555 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.555 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:51:07.555 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.319054+0000", 2023-09-23T20:51:07.555 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:07.555 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.555 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.556 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:51:07.556 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.319052+0000", 2023-09-23T20:51:07.556 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:51:07.556 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.556 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.556 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:51:07.556 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.319144+0000", 2023-09-23T20:51:07.556 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.1686000000000003e-05 2023-09-23T20:51:07.556 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.556 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.556 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:51:07.556 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.319159+0000", 2023-09-23T20:51:07.557 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.5136e-05 2023-09-23T20:51:07.557 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.557 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.557 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:51:07.557 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.319165+0000", 2023-09-23T20:51:07.557 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.5010000000000001e-06 2023-09-23T20:51:07.557 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.557 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.557 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:07.557 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.321384+0000", 2023-09-23T20:51:07.557 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0022185460000000001 2023-09-23T20:51:07.558 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.558 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.558 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:51:07.558 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.321392+0000", 2023-09-23T20:51:07.558 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.0430000000000008e-06 2023-09-23T20:51:07.558 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.558 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.558 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:07.558 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:10.174291+0000", 2023-09-23T20:51:07.558 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.85289914600000005 2023-09-23T20:51:07.558 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.559 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.559 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:51:07.559 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:10.174421+0000", 2023-09-23T20:51:07.559 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00012987799999999999 2023-09-23T20:51:07.559 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.559 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.559 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 2,6", 2023-09-23T20:51:07.559 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:10.174567+0000", 2023-09-23T20:51:07.559 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0001462 2023-09-23T20:51:07.559 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.559 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.559 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:07.560 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:10.184312+0000", 2023-09-23T20:51:07.560 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0097449849999999994 2023-09-23T20:51:07.560 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.560 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.560 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:07.560 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:10.186377+0000", 2023-09-23T20:51:07.560 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0020647410000000001 2023-09-23T20:51:07.560 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.560 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.560 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:51:07.560 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:10.191052+0000", 2023-09-23T20:51:07.560 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0046755879999999996 2023-09-23T20:51:07.561 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.561 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.561 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:51:07.561 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:10.191097+0000", 2023-09-23T20:51:07.561 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.4085999999999998e-05 2023-09-23T20:51:07.561 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.561 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.561 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:51:07.561 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:10.191152+0000", 2023-09-23T20:51:07.561 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.5852e-05 2023-09-23T20:51:07.561 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:07.562 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:51:07.562 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:07.562 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.562 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.562 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:102328 10.1 10:8bdf3d92:::benchmark_data_smithi018_422431_object102327:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e553)", 2023-09-23T20:51:07.562 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:47:09.354426+0000", 2023-09-23T20:51:07.562 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 238.18383241399999, 2023-09-23T20:51:07.562 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.85433105200000004, 2023-09-23T20:51:07.562 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:51:07.562 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:07.562 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:51:07.562 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:51:07.563 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:51:07.563 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 102328 2023-09-23T20:51:07.563 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.563 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:51:07.563 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.563 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:51:07.563 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.354426+0000", 2023-09-23T20:51:07.564 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:07.564 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.564 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.564 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:51:07.564 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.354426+0000", 2023-09-23T20:51:07.564 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:07.564 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.564 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.564 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:51:07.564 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.354425+0000", 2023-09-23T20:51:07.564 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:51:07.565 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.565 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.565 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:51:07.565 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.354533+0000", 2023-09-23T20:51:07.565 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00010846 2023-09-23T20:51:07.565 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.565 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.565 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:51:07.565 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.354546+0000", 2023-09-23T20:51:07.565 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.2459e-05 2023-09-23T20:51:07.565 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.566 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.566 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:51:07.566 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.354553+0000", 2023-09-23T20:51:07.566 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.9399999999999996e-06 2023-09-23T20:51:07.566 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.566 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.566 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:07.566 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.354611+0000", 2023-09-23T20:51:07.566 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.7868000000000002e-05 2023-09-23T20:51:07.566 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.566 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.567 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:51:07.567 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.354622+0000", 2023-09-23T20:51:07.567 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0881e-05 2023-09-23T20:51:07.567 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.567 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.567 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:07.567 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:10.177496+0000", 2023-09-23T20:51:07.567 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.82287492900000003 2023-09-23T20:51:07.567 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.567 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.568 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:51:07.568 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:10.177619+0000", 2023-09-23T20:51:07.568 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00012237500000000001 2023-09-23T20:51:07.568 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.568 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.568 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,7", 2023-09-23T20:51:07.568 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:10.178159+0000", 2023-09-23T20:51:07.568 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00053965899999999995 2023-09-23T20:51:07.568 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.568 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.569 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:51:07.569 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:10.198332+0000", 2023-09-23T20:51:07.569 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.020173176000000001 2023-09-23T20:51:07.569 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.569 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.569 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:07.569 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:10.208652+0000", 2023-09-23T20:51:07.569 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.010320775000000001 2023-09-23T20:51:07.569 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.569 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.570 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:07.570 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:10.208698+0000", 2023-09-23T20:51:07.570 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.5568000000000001e-05 2023-09-23T20:51:07.570 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.570 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.570 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:51:07.570 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:10.208728+0000", 2023-09-23T20:51:07.570 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.0032e-05 2023-09-23T20:51:07.570 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.570 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.571 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:51:07.571 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:10.208757+0000", 2023-09-23T20:51:07.571 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.9108999999999999e-05 2023-09-23T20:51:07.571 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:07.571 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:51:07.571 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:07.571 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.571 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.571 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:102335 10.1 10:88e5c9cd:::benchmark_data_smithi018_422431_object102334:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e553)", 2023-09-23T20:51:07.571 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:47:09.380750+0000", 2023-09-23T20:51:07.572 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 238.157508651, 2023-09-23T20:51:07.572 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.82807518800000002, 2023-09-23T20:51:07.572 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:51:07.572 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:07.572 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:51:07.572 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:51:07.572 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:51:07.572 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 102335 2023-09-23T20:51:07.572 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.572 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:51:07.573 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.573 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:51:07.573 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.380750+0000", 2023-09-23T20:51:07.573 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:07.573 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.573 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.574 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:51:07.574 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.380750+0000", 2023-09-23T20:51:07.574 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:07.574 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.574 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.574 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:51:07.574 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.380748+0000", 2023-09-23T20:51:07.574 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:51:07.574 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.575 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.575 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:51:07.575 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.380802+0000", 2023-09-23T20:51:07.575 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.393e-05 2023-09-23T20:51:07.575 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.575 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.575 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:51:07.575 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.380818+0000", 2023-09-23T20:51:07.575 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.5743000000000002e-05 2023-09-23T20:51:07.575 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.576 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.576 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:51:07.576 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.380824+0000", 2023-09-23T20:51:07.576 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.2020000000000001e-06 2023-09-23T20:51:07.576 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.576 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.576 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:07.576 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.380865+0000", 2023-09-23T20:51:07.576 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.0317e-05 2023-09-23T20:51:07.576 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.576 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.576 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:51:07.577 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.380873+0000", 2023-09-23T20:51:07.577 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.0350000000000001e-06 2023-09-23T20:51:07.577 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.577 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.577 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:07.577 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:10.183127+0000", 2023-09-23T20:51:07.577 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.80225428300000001 2023-09-23T20:51:07.577 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.577 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.577 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:51:07.577 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:10.183233+0000", 2023-09-23T20:51:07.578 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00010638599999999999 2023-09-23T20:51:07.578 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.578 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.578 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,7", 2023-09-23T20:51:07.578 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:10.183882+0000", 2023-09-23T20:51:07.578 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00064829299999999998 2023-09-23T20:51:07.578 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.578 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.578 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:51:07.578 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:10.201736+0000", 2023-09-23T20:51:07.578 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.017854775999999999 2023-09-23T20:51:07.578 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.579 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.579 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:07.579 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:10.208681+0000", 2023-09-23T20:51:07.579 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0069449749999999999 2023-09-23T20:51:07.579 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.579 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.579 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:07.579 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:10.208780+0000", 2023-09-23T20:51:07.579 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.8886000000000002e-05 2023-09-23T20:51:07.579 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.579 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.579 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:51:07.580 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:10.208806+0000", 2023-09-23T20:51:07.580 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.5185999999999999e-05 2023-09-23T20:51:07.580 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.580 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.580 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:51:07.580 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:10.208825+0000", 2023-09-23T20:51:07.580 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.9585e-05 2023-09-23T20:51:07.580 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:07.580 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:51:07.580 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:07.580 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.580 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.581 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:102348 10.1 10:8c86b5d8:::benchmark_data_smithi018_422431_object102347:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e553)", 2023-09-23T20:51:07.581 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:47:09.394224+0000", 2023-09-23T20:51:07.581 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 238.14403429199999, 2023-09-23T20:51:07.581 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.81471285100000002, 2023-09-23T20:51:07.581 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:51:07.581 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:07.581 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:51:07.581 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:51:07.581 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:51:07.581 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 102348 2023-09-23T20:51:07.581 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.582 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:51:07.582 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.582 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:51:07.582 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.394224+0000", 2023-09-23T20:51:07.582 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:07.582 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.582 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.582 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:51:07.582 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.394224+0000", 2023-09-23T20:51:07.582 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:07.582 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.582 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.583 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:51:07.583 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.394223+0000", 2023-09-23T20:51:07.583 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:51:07.583 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.583 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.583 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:51:07.583 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.394305+0000", 2023-09-23T20:51:07.583 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.2297000000000002e-05 2023-09-23T20:51:07.583 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.583 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.583 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:51:07.584 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.394329+0000", 2023-09-23T20:51:07.584 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.3391999999999999e-05 2023-09-23T20:51:07.584 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.584 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.584 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:51:07.584 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.394336+0000", 2023-09-23T20:51:07.584 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.002e-06 2023-09-23T20:51:07.584 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.584 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.584 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:07.584 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.394395+0000", 2023-09-23T20:51:07.584 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.9018999999999999e-05 2023-09-23T20:51:07.585 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.585 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.585 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:51:07.585 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.394403+0000", 2023-09-23T20:51:07.585 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.2560000000000002e-06 2023-09-23T20:51:07.585 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.585 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.585 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:07.585 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:10.186875+0000", 2023-09-23T20:51:07.585 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.79247180299999997 2023-09-23T20:51:07.585 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.585 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.586 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:51:07.586 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:10.187029+0000", 2023-09-23T20:51:07.586 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00015432999999999999 2023-09-23T20:51:07.586 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.586 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.586 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,7", 2023-09-23T20:51:07.586 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:10.187200+0000", 2023-09-23T20:51:07.586 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000170915 2023-09-23T20:51:07.586 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.586 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.586 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:51:07.586 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:10.201742+0000", 2023-09-23T20:51:07.587 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.014541953999999999 2023-09-23T20:51:07.587 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.587 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.587 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:07.587 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:10.208845+0000", 2023-09-23T20:51:07.587 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0071029140000000001 2023-09-23T20:51:07.587 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.587 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.587 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:07.587 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:10.208896+0000", 2023-09-23T20:51:07.587 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.1143999999999997e-05 2023-09-23T20:51:07.587 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.588 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.588 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:51:07.588 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:10.208919+0000", 2023-09-23T20:51:07.588 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.3025000000000001e-05 2023-09-23T20:51:07.588 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.588 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.588 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:51:07.588 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:10.208937+0000", 2023-09-23T20:51:07.588 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.7864e-05 2023-09-23T20:51:07.588 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:07.588 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:51:07.588 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:07.589 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.589 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.589 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:159028 10.3 10.1197f3a3 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e577)", 2023-09-23T20:51:07.589 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:48:22.915122+0000", 2023-09-23T20:51:07.589 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 164.623136993, 2023-09-23T20:51:07.589 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.98997199800000002, 2023-09-23T20:51:07.589 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:51:07.589 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:51:07.589 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:51:07.589 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:51:07.589 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:51:07.590 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 159028 2023-09-23T20:51:07.590 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.590 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:51:07.590 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.590 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:51:07.590 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.915122+0000", 2023-09-23T20:51:07.590 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:07.590 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.590 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.590 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:51:07.590 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.915122+0000", 2023-09-23T20:51:07.591 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:07.591 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.591 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.591 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:51:07.591 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.915120+0000", 2023-09-23T20:51:07.591 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:51:07.591 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.591 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.591 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:51:07.591 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.915126+0000", 2023-09-23T20:51:07.591 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.6370000000000004e-06 2023-09-23T20:51:07.591 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.592 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.592 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:51:07.592 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.915135+0000", 2023-09-23T20:51:07.592 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.4250000000000004e-06 2023-09-23T20:51:07.592 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.592 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.592 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:51:07.592 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.915142+0000", 2023-09-23T20:51:07.592 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.6599999999999998e-06 2023-09-23T20:51:07.592 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.592 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.592 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:07.593 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.916692+0000", 2023-09-23T20:51:07.593 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.001550395 2023-09-23T20:51:07.593 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.593 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.593 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:51:07.593 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.916699+0000", 2023-09-23T20:51:07.593 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.5409999999999997e-06 2023-09-23T20:51:07.593 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.593 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.593 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:07.593 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:23.905083+0000", 2023-09-23T20:51:07.593 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.988384288 2023-09-23T20:51:07.594 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.594 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.594 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:51:07.594 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:23.905094+0000", 2023-09-23T20:51:07.594 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0261000000000001e-05 2023-09-23T20:51:07.594 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:07.594 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:51:07.594 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:07.594 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.594 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.594 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:159040 10.e 10:71537a69:::benchmark_data_smithi018_422431_object54544:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e577)", 2023-09-23T20:51:07.594 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:48:22.936052+0000", 2023-09-23T20:51:07.595 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 164.60220636, 2023-09-23T20:51:07.595 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.2675618719999999, 2023-09-23T20:51:07.595 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:51:07.595 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:07.595 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:51:07.595 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:51:07.595 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:51:07.595 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 159040 2023-09-23T20:51:07.595 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.595 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:51:07.595 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.595 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:51:07.596 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.936052+0000", 2023-09-23T20:51:07.596 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:07.596 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.596 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.596 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:51:07.596 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.936052+0000", 2023-09-23T20:51:07.596 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:07.596 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.596 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.596 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:51:07.596 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.936051+0000", 2023-09-23T20:51:07.596 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:51:07.597 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.597 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.597 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:51:07.597 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.936057+0000", 2023-09-23T20:51:07.597 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.7960000000000001e-06 2023-09-23T20:51:07.597 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.597 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.597 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:51:07.597 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.936070+0000", 2023-09-23T20:51:07.597 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.2683e-05 2023-09-23T20:51:07.597 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.597 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.598 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:51:07.598 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.936074+0000", 2023-09-23T20:51:07.598 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.9359999999999998e-06 2023-09-23T20:51:07.598 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.598 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.598 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:07.598 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.936130+0000", 2023-09-23T20:51:07.598 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.5776999999999998e-05 2023-09-23T20:51:07.598 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.598 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.598 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:51:07.598 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.936142+0000", 2023-09-23T20:51:07.599 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.198e-05 2023-09-23T20:51:07.599 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.599 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.599 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:07.599 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:23.906942+0000", 2023-09-23T20:51:07.599 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.97079988100000003 2023-09-23T20:51:07.599 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.599 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.599 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:51:07.599 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:23.906953+0000", 2023-09-23T20:51:07.599 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.066e-05 2023-09-23T20:51:07.599 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.600 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.600 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:07.600 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:25.130718+0000", 2023-09-23T20:51:07.600 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.2237656889999999 2023-09-23T20:51:07.600 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.600 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.600 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:51:07.600 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:25.130872+0000", 2023-09-23T20:51:07.600 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00015312299999999999 2023-09-23T20:51:07.600 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.600 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.601 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 2,5,6", 2023-09-23T20:51:07.601 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:25.131568+0000", 2023-09-23T20:51:07.601 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00069637699999999998 2023-09-23T20:51:07.601 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.601 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.601 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:07.601 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:25.151395+0000", 2023-09-23T20:51:07.601 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.019826769000000001 2023-09-23T20:51:07.601 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.601 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.601 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:51:07.601 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:25.151584+0000", 2023-09-23T20:51:07.602 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00018891399999999999 2023-09-23T20:51:07.602 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.602 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.602 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:07.602 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:25.175874+0000", 2023-09-23T20:51:07.602 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.024290066999999999 2023-09-23T20:51:07.602 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.602 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.602 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:07.602 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:25.203427+0000", 2023-09-23T20:51:07.603 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.027553628 2023-09-23T20:51:07.603 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.603 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.603 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:51:07.603 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:25.203487+0000", 2023-09-23T20:51:07.603 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.9623999999999997e-05 2023-09-23T20:51:07.603 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.603 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.603 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:51:07.603 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:25.203614+0000", 2023-09-23T20:51:07.603 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00012711999999999999 2023-09-23T20:51:07.603 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:07.604 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:51:07.604 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:07.604 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.604 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.604 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:159052 10.3 10.e70c3763 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e577)", 2023-09-23T20:51:07.604 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:48:22.953330+0000", 2023-09-23T20:51:07.604 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 164.58492877899999, 2023-09-23T20:51:07.604 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.95178979799999996, 2023-09-23T20:51:07.604 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:51:07.604 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:51:07.604 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:51:07.604 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:51:07.605 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:51:07.605 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 159052 2023-09-23T20:51:07.605 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.605 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:51:07.605 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.605 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:51:07.605 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.953330+0000", 2023-09-23T20:51:07.605 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:07.605 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.605 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.605 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:51:07.606 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.953330+0000", 2023-09-23T20:51:07.606 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:07.606 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.606 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.606 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:51:07.606 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.953329+0000", 2023-09-23T20:51:07.606 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:51:07.606 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.606 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.606 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:51:07.606 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.953334+0000", 2023-09-23T20:51:07.606 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.8999999999999997e-06 2023-09-23T20:51:07.607 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.607 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.607 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:51:07.607 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.953343+0000", 2023-09-23T20:51:07.607 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.4520000000000003e-06 2023-09-23T20:51:07.607 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.607 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.607 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:51:07.607 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.953348+0000", 2023-09-23T20:51:07.607 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.5390000000000001e-06 2023-09-23T20:51:07.607 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.608 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.608 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:07.608 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.953392+0000", 2023-09-23T20:51:07.608 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.4774000000000002e-05 2023-09-23T20:51:07.608 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.608 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.608 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:51:07.608 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.953400+0000", 2023-09-23T20:51:07.608 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.0890000000000002e-06 2023-09-23T20:51:07.608 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.608 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.608 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:07.609 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:23.905113+0000", 2023-09-23T20:51:07.609 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.95171356299999998 2023-09-23T20:51:07.609 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.609 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.609 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:51:07.609 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:23.905120+0000", 2023-09-23T20:51:07.609 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.4840000000000001e-06 2023-09-23T20:51:07.609 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:07.609 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:51:07.609 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:07.609 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.610 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.610 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:174198 10.e 10:7230894f:::benchmark_data_smithi018_422431_object69702:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e590)", 2023-09-23T20:51:07.610 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:48:47.130817+0000", 2023-09-23T20:51:07.610 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 140.40744108199999, 2023-09-23T20:51:07.610 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0840209249999999, 2023-09-23T20:51:07.610 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:51:07.610 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:51:07.610 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:51:07.610 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:51:07.610 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:51:07.611 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 174198 2023-09-23T20:51:07.611 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.611 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:51:07.611 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.611 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:51:07.611 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:47.130817+0000", 2023-09-23T20:51:07.611 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:07.611 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.611 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.611 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:51:07.611 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:47.130817+0000", 2023-09-23T20:51:07.611 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:07.612 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.612 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.612 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:51:07.612 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:47.130816+0000", 2023-09-23T20:51:07.612 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:51:07.612 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.612 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.612 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:51:07.612 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:47.130823+0000", 2023-09-23T20:51:07.612 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.3560000000000004e-06 2023-09-23T20:51:07.612 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.613 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.613 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:51:07.613 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:47.130837+0000", 2023-09-23T20:51:07.613 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.3952e-05 2023-09-23T20:51:07.613 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.613 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.613 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:51:07.613 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:47.130845+0000", 2023-09-23T20:51:07.613 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.5279999999999998e-06 2023-09-23T20:51:07.613 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.613 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.613 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:07.614 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:47.131335+0000", 2023-09-23T20:51:07.614 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00049032299999999997 2023-09-23T20:51:07.614 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.614 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.614 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:51:07.614 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:47.131477+0000", 2023-09-23T20:51:07.614 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00014191 2023-09-23T20:51:07.614 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.614 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.614 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 2,5,6", 2023-09-23T20:51:07.614 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:47.131615+0000", 2023-09-23T20:51:07.615 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00013770300000000001 2023-09-23T20:51:07.615 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.615 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.615 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:07.615 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:47.139776+0000", 2023-09-23T20:51:07.615 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0081612669999999998 2023-09-23T20:51:07.615 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.615 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.615 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:51:07.615 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:47.141819+0000", 2023-09-23T20:51:07.615 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00204294 2023-09-23T20:51:07.616 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.616 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.616 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:07.616 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:47.160786+0000", 2023-09-23T20:51:07.616 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.018966818 2023-09-23T20:51:07.616 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.616 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.616 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:51:07.616 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:47.160797+0000", 2023-09-23T20:51:07.616 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0849999999999999e-05 2023-09-23T20:51:07.616 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.616 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.617 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:07.617 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:48.200395+0000", 2023-09-23T20:51:07.617 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.039598236 2023-09-23T20:51:07.617 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.617 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.617 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for missing object", 2023-09-23T20:51:07.617 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:48.200521+0000", 2023-09-23T20:51:07.617 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00012670699999999999 2023-09-23T20:51:07.617 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.617 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.617 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:07.617 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:48.214746+0000", 2023-09-23T20:51:07.618 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.014224404 2023-09-23T20:51:07.618 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.618 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.618 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:51:07.618 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:48.214838+0000", 2023-09-23T20:51:07.618 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.2504999999999996e-05 2023-09-23T20:51:07.618 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:07.618 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:51:07.618 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:07.618 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.618 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.619 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:174221 10.e 10:743e88cf:::benchmark_data_smithi018_422431_object69725:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e591)", 2023-09-23T20:51:07.619 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:48:47.168211+0000", 2023-09-23T20:51:07.619 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 140.37004791800001, 2023-09-23T20:51:07.619 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0619566009999999, 2023-09-23T20:51:07.619 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:51:07.619 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:07.619 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:51:07.619 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:51:07.619 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:51:07.619 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 174221 2023-09-23T20:51:07.619 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.620 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:51:07.620 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.620 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:51:07.620 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:47.168211+0000", 2023-09-23T20:51:07.620 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:07.620 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.620 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.620 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:51:07.620 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:47.168211+0000", 2023-09-23T20:51:07.620 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:07.620 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.620 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.621 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:51:07.621 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:47.168209+0000", 2023-09-23T20:51:07.621 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:51:07.621 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.621 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.621 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:51:07.621 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:47.168216+0000", 2023-09-23T20:51:07.621 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.8630000000000002e-06 2023-09-23T20:51:07.621 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.621 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.621 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:51:07.621 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:47.168228+0000", 2023-09-23T20:51:07.622 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.2347000000000001e-05 2023-09-23T20:51:07.622 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.622 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.622 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:51:07.622 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:47.168235+0000", 2023-09-23T20:51:07.622 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.8399999999999997e-06 2023-09-23T20:51:07.622 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.622 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.622 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:07.622 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:47.168279+0000", 2023-09-23T20:51:07.622 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.4199000000000002e-05 2023-09-23T20:51:07.623 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.623 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.623 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:51:07.623 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:47.168290+0000", 2023-09-23T20:51:07.623 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0776e-05 2023-09-23T20:51:07.623 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.623 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.623 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:07.623 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:48.214861+0000", 2023-09-23T20:51:07.623 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0465711740000001 2023-09-23T20:51:07.623 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.624 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.624 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:51:07.624 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:48.214996+0000", 2023-09-23T20:51:07.624 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00013436599999999999 2023-09-23T20:51:07.624 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.624 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.624 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 2,6", 2023-09-23T20:51:07.624 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:48.215162+0000", 2023-09-23T20:51:07.624 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00016618000000000001 2023-09-23T20:51:07.624 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.624 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.625 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:07.625 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:48.218225+0000", 2023-09-23T20:51:07.625 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0030630240000000001 2023-09-23T20:51:07.625 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.625 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.625 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:51:07.625 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:48.221353+0000", 2023-09-23T20:51:07.625 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.003128334 2023-09-23T20:51:07.625 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.625 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.625 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:07.625 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:48.230010+0000", 2023-09-23T20:51:07.625 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0086572739999999995 2023-09-23T20:51:07.626 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.626 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.626 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:51:07.626 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:48.230062+0000", 2023-09-23T20:51:07.626 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.2055999999999998e-05 2023-09-23T20:51:07.626 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.626 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.626 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:51:07.626 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:48.230167+0000", 2023-09-23T20:51:07.626 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000104823 2023-09-23T20:51:07.626 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:07.627 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:51:07.627 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:07.627 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.627 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.627 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:174223 10.e 10:790afcea:::benchmark_data_smithi018_422431_object69727:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e591)", 2023-09-23T20:51:07.627 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:48:47.169131+0000", 2023-09-23T20:51:07.627 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 140.369127615, 2023-09-23T20:51:07.627 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.061208814, 2023-09-23T20:51:07.627 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:51:07.627 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:07.627 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:51:07.627 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:51:07.628 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:51:07.628 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 174223 2023-09-23T20:51:07.628 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.628 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:51:07.628 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.628 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:51:07.628 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:47.169131+0000", 2023-09-23T20:51:07.628 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:07.628 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.628 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.628 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:51:07.628 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:47.169131+0000", 2023-09-23T20:51:07.629 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:07.629 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.629 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.629 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:51:07.629 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:47.169130+0000", 2023-09-23T20:51:07.629 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:51:07.629 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.629 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.629 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:51:07.629 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:47.169134+0000", 2023-09-23T20:51:07.629 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.895e-06 2023-09-23T20:51:07.630 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.630 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.630 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:51:07.630 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:47.169143+0000", 2023-09-23T20:51:07.630 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.2410000000000001e-06 2023-09-23T20:51:07.630 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.630 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.630 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:51:07.630 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:47.169148+0000", 2023-09-23T20:51:07.630 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.2930000000000002e-06 2023-09-23T20:51:07.630 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.630 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.631 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:07.631 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:47.169184+0000", 2023-09-23T20:51:07.631 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.6649000000000001e-05 2023-09-23T20:51:07.631 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.631 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.631 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:51:07.631 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:47.169191+0000", 2023-09-23T20:51:07.631 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.7599999999999997e-06 2023-09-23T20:51:07.631 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.631 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.631 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:07.631 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:48.218256+0000", 2023-09-23T20:51:07.632 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0490651 2023-09-23T20:51:07.632 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.632 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.632 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:51:07.632 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:48.218425+0000", 2023-09-23T20:51:07.632 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000168887 2023-09-23T20:51:07.632 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.632 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.632 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 2,6", 2023-09-23T20:51:07.632 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:48.218575+0000", 2023-09-23T20:51:07.632 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00015012700000000001 2023-09-23T20:51:07.633 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.633 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.633 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:07.633 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:48.221396+0000", 2023-09-23T20:51:07.633 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0028210959999999999 2023-09-23T20:51:07.633 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.633 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.633 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:51:07.633 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:48.221474+0000", 2023-09-23T20:51:07.633 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.7961999999999997e-05 2023-09-23T20:51:07.633 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.633 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.634 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:07.634 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:48.230218+0000", 2023-09-23T20:51:07.634 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.008743252 2023-09-23T20:51:07.634 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.634 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.634 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:51:07.634 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:48.230258+0000", 2023-09-23T20:51:07.634 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.0481000000000001e-05 2023-09-23T20:51:07.634 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.634 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.634 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:51:07.635 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:48.230340+0000", 2023-09-23T20:51:07.635 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.1658999999999999e-05 2023-09-23T20:51:07.635 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:07.635 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:51:07.635 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:07.635 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.635 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.635 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:197315 10.17 10.f2611717 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e624)", 2023-09-23T20:51:07.635 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:49:29.097858+0000", 2023-09-23T20:51:07.635 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 98.440400529000001, 2023-09-23T20:51:07.635 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.88606821700000005, 2023-09-23T20:51:07.635 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:51:07.636 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:51:07.636 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:51:07.636 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:51:07.636 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:51:07.636 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 197315 2023-09-23T20:51:07.636 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.636 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:51:07.636 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.636 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:51:07.636 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.097858+0000", 2023-09-23T20:51:07.636 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:07.636 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.637 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.637 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:51:07.637 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.097858+0000", 2023-09-23T20:51:07.637 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:07.637 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.637 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.637 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:51:07.637 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.097857+0000", 2023-09-23T20:51:07.637 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:51:07.637 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.637 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.638 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:51:07.638 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.097863+0000", 2023-09-23T20:51:07.638 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.0700000000000003e-06 2023-09-23T20:51:07.638 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.638 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.638 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:51:07.638 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.097874+0000", 2023-09-23T20:51:07.638 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0801e-05 2023-09-23T20:51:07.638 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.638 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.638 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:51:07.638 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.097879+0000", 2023-09-23T20:51:07.639 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.4739999999999996e-06 2023-09-23T20:51:07.639 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.639 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.639 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:07.639 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.097922+0000", 2023-09-23T20:51:07.639 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.248e-05 2023-09-23T20:51:07.639 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.639 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.639 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:51:07.639 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.097929+0000", 2023-09-23T20:51:07.639 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.0430000000000002e-06 2023-09-23T20:51:07.640 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.640 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.640 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:07.640 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.983916+0000", 2023-09-23T20:51:07.640 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.88598697999999998 2023-09-23T20:51:07.640 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.640 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.640 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:51:07.640 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.983926+0000", 2023-09-23T20:51:07.640 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.064e-05 2023-09-23T20:51:07.640 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:07.640 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:51:07.641 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:07.641 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.641 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.641 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:197329 10.17 10.fef47177 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e624)", 2023-09-23T20:51:07.641 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:49:29.114545+0000", 2023-09-23T20:51:07.641 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 98.423713234000004, 2023-09-23T20:51:07.641 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.86958525200000003, 2023-09-23T20:51:07.641 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:51:07.641 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:51:07.641 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:51:07.641 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:51:07.641 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:51:07.642 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 197329 2023-09-23T20:51:07.642 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.642 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:51:07.642 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.642 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:51:07.642 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.114545+0000", 2023-09-23T20:51:07.642 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:07.642 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.642 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.642 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:51:07.642 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.114545+0000", 2023-09-23T20:51:07.642 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:07.643 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.643 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.643 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:51:07.643 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.114544+0000", 2023-09-23T20:51:07.643 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:51:07.643 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.643 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.643 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:51:07.643 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.114550+0000", 2023-09-23T20:51:07.643 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.3679999999999998e-06 2023-09-23T20:51:07.643 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.643 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.644 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:51:07.644 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.114567+0000", 2023-09-23T20:51:07.644 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.6520000000000001e-05 2023-09-23T20:51:07.644 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.644 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.644 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:51:07.644 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.114573+0000", 2023-09-23T20:51:07.644 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.3749999999999999e-06 2023-09-23T20:51:07.644 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.644 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.644 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:07.644 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.114613+0000", 2023-09-23T20:51:07.645 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.9544e-05 2023-09-23T20:51:07.645 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.645 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.645 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:51:07.645 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.114620+0000", 2023-09-23T20:51:07.645 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.452e-06 2023-09-23T20:51:07.645 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.645 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.645 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:07.645 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.984118+0000", 2023-09-23T20:51:07.645 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.86949812000000004 2023-09-23T20:51:07.646 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.646 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.646 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:51:07.646 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.984131+0000", 2023-09-23T20:51:07.646 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.2431e-05 2023-09-23T20:51:07.646 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:07.646 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:51:07.646 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:07.646 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.646 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.646 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:197351 10.17 10.81d04937 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e624)", 2023-09-23T20:51:07.646 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:49:29.134888+0000", 2023-09-23T20:51:07.647 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 98.403370796999994, 2023-09-23T20:51:07.647 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.84927483299999995, 2023-09-23T20:51:07.647 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:51:07.647 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:51:07.647 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:51:07.647 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:51:07.647 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:51:07.647 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 197351 2023-09-23T20:51:07.647 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.647 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:51:07.647 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.647 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:51:07.648 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.134888+0000", 2023-09-23T20:51:07.648 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:07.648 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.648 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.648 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:51:07.648 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.134888+0000", 2023-09-23T20:51:07.648 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:07.648 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.648 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.648 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:51:07.648 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.134886+0000", 2023-09-23T20:51:07.648 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:51:07.648 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.649 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.649 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:51:07.649 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.134893+0000", 2023-09-23T20:51:07.649 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.404e-06 2023-09-23T20:51:07.649 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.649 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.649 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:51:07.649 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.134907+0000", 2023-09-23T20:51:07.649 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.4450000000000001e-05 2023-09-23T20:51:07.649 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.649 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.649 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:51:07.650 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.134914+0000", 2023-09-23T20:51:07.650 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.5390000000000004e-06 2023-09-23T20:51:07.650 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.650 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.650 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:07.650 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.134961+0000", 2023-09-23T20:51:07.650 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.7648000000000002e-05 2023-09-23T20:51:07.650 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.650 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.650 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:51:07.650 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.134969+0000", 2023-09-23T20:51:07.650 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.4460000000000004e-06 2023-09-23T20:51:07.651 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.651 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.651 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:07.651 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.984152+0000", 2023-09-23T20:51:07.651 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.84918333999999995 2023-09-23T20:51:07.651 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.651 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.651 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:51:07.651 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.984163+0000", 2023-09-23T20:51:07.651 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0417999999999999e-05 2023-09-23T20:51:07.651 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:07.652 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:51:07.652 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:07.652 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.652 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.652 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:197367 10.17 10.33935bd7 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e624)", 2023-09-23T20:51:07.652 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:49:29.147908+0000", 2023-09-23T20:51:07.652 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 98.390350343999998, 2023-09-23T20:51:07.652 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.83631422499999997, 2023-09-23T20:51:07.652 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:51:07.652 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:51:07.652 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:51:07.652 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:51:07.653 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:51:07.653 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 197367 2023-09-23T20:51:07.653 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.653 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:51:07.653 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.653 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:51:07.653 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.147908+0000", 2023-09-23T20:51:07.653 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:07.653 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.653 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.653 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:51:07.653 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.147908+0000", 2023-09-23T20:51:07.653 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:07.654 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.654 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.654 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:51:07.654 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.147907+0000", 2023-09-23T20:51:07.654 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:51:07.654 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.654 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.654 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:51:07.654 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.147914+0000", 2023-09-23T20:51:07.654 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.4540000000000001e-06 2023-09-23T20:51:07.654 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.655 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.655 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:51:07.655 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.147926+0000", 2023-09-23T20:51:07.655 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1307e-05 2023-09-23T20:51:07.655 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.655 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.655 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:51:07.655 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.147945+0000", 2023-09-23T20:51:07.655 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.9315999999999999e-05 2023-09-23T20:51:07.655 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.655 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.655 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:07.656 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.148674+0000", 2023-09-23T20:51:07.656 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00072932400000000005 2023-09-23T20:51:07.656 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.656 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.656 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:51:07.656 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.148682+0000", 2023-09-23T20:51:07.656 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.3640000000000001e-06 2023-09-23T20:51:07.656 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.656 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.656 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:07.656 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.984212+0000", 2023-09-23T20:51:07.657 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.83553048200000002 2023-09-23T20:51:07.657 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.657 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.657 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:51:07.657 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.984222+0000", 2023-09-23T20:51:07.657 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0393e-05 2023-09-23T20:51:07.657 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:07.657 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:51:07.657 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:07.657 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.657 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.657 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:197369 10.17 10.4ddbadb7 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e624)", 2023-09-23T20:51:07.658 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:49:29.150060+0000", 2023-09-23T20:51:07.658 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 98.388198372000005, 2023-09-23T20:51:07.658 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.83419653900000001, 2023-09-23T20:51:07.658 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:51:07.658 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:51:07.658 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:51:07.658 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:51:07.658 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:51:07.658 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 197369 2023-09-23T20:51:07.658 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.658 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:51:07.658 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.659 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:51:07.659 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.150060+0000", 2023-09-23T20:51:07.659 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:07.659 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.659 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.659 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:51:07.659 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.150060+0000", 2023-09-23T20:51:07.659 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:07.659 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.659 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.659 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:51:07.660 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.150059+0000", 2023-09-23T20:51:07.660 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:51:07.660 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.660 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.660 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:51:07.660 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.150065+0000", 2023-09-23T20:51:07.660 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.015e-06 2023-09-23T20:51:07.660 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.660 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.660 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:51:07.660 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.150075+0000", 2023-09-23T20:51:07.660 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0196e-05 2023-09-23T20:51:07.661 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.661 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.661 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:51:07.661 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.150081+0000", 2023-09-23T20:51:07.661 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.8379999999999998e-06 2023-09-23T20:51:07.661 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.661 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.661 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:07.661 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.150137+0000", 2023-09-23T20:51:07.661 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.6044000000000001e-05 2023-09-23T20:51:07.661 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.662 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.662 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:51:07.662 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.150144+0000", 2023-09-23T20:51:07.662 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.4590000000000001e-06 2023-09-23T20:51:07.662 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.662 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.662 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:07.662 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.984249+0000", 2023-09-23T20:51:07.662 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.83410500499999995 2023-09-23T20:51:07.662 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.662 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.662 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:51:07.663 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.984257+0000", 2023-09-23T20:51:07.663 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.272e-06 2023-09-23T20:51:07.663 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:07.663 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:51:07.663 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:07.663 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.663 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.663 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:197390 10.17 10.2b417377 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e624)", 2023-09-23T20:51:07.663 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:49:29.168757+0000", 2023-09-23T20:51:07.663 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 98.369501825, 2023-09-23T20:51:07.663 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.81552972800000001, 2023-09-23T20:51:07.664 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:51:07.664 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:51:07.664 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:51:07.664 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:51:07.664 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:51:07.664 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 197390 2023-09-23T20:51:07.664 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.664 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:51:07.664 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.664 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:51:07.664 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.168757+0000", 2023-09-23T20:51:07.665 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:07.665 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.665 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.665 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:51:07.665 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.168757+0000", 2023-09-23T20:51:07.665 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:07.665 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.665 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.665 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:51:07.665 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.168756+0000", 2023-09-23T20:51:07.665 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:51:07.666 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.666 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.666 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:51:07.666 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.168761+0000", 2023-09-23T20:51:07.666 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.8900000000000004e-06 2023-09-23T20:51:07.666 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.666 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.666 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:51:07.666 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.168776+0000", 2023-09-23T20:51:07.666 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.4773000000000001e-05 2023-09-23T20:51:07.666 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.666 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.667 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:51:07.667 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.168783+0000", 2023-09-23T20:51:07.667 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.5849999999999996e-06 2023-09-23T20:51:07.667 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.667 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.667 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:07.667 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.168828+0000", 2023-09-23T20:51:07.667 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.4920999999999999e-05 2023-09-23T20:51:07.667 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.667 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.667 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:51:07.668 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.168835+0000", 2023-09-23T20:51:07.668 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.3529999999999997e-06 2023-09-23T20:51:07.668 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.668 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.668 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:07.668 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.984277+0000", 2023-09-23T20:51:07.668 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.81544192599999998 2023-09-23T20:51:07.668 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.668 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.668 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:51:07.668 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.984286+0000", 2023-09-23T20:51:07.669 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.4530000000000008e-06 2023-09-23T20:51:07.669 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:07.669 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:51:07.669 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:07.669 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.669 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.669 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:197413 10.17 10.ebd7cd77 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e624)", 2023-09-23T20:51:07.669 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:49:29.195027+0000", 2023-09-23T20:51:07.669 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 98.343231865999996, 2023-09-23T20:51:07.669 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.78928548600000004, 2023-09-23T20:51:07.669 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:51:07.670 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:51:07.670 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:51:07.670 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:51:07.670 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:51:07.670 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 197413 2023-09-23T20:51:07.670 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.670 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:51:07.670 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.670 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:51:07.670 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.195027+0000", 2023-09-23T20:51:07.670 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:07.670 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.671 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.671 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:51:07.671 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.195027+0000", 2023-09-23T20:51:07.671 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:07.671 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.671 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.671 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:51:07.671 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.195025+0000", 2023-09-23T20:51:07.671 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:51:07.671 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.671 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.672 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:51:07.672 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.195033+0000", 2023-09-23T20:51:07.672 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.1179999999999992e-06 2023-09-23T20:51:07.672 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.672 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.672 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:51:07.672 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.195053+0000", 2023-09-23T20:51:07.672 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.0038e-05 2023-09-23T20:51:07.672 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.672 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.672 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:51:07.672 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.195061+0000", 2023-09-23T20:51:07.672 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.763e-06 2023-09-23T20:51:07.673 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.673 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.673 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:07.673 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.195099+0000", 2023-09-23T20:51:07.673 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.7815000000000003e-05 2023-09-23T20:51:07.673 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.673 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.673 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:51:07.673 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.195106+0000", 2023-09-23T20:51:07.673 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.618e-06 2023-09-23T20:51:07.673 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.673 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.674 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:07.674 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.984306+0000", 2023-09-23T20:51:07.674 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.78920079600000004 2023-09-23T20:51:07.674 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.674 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.674 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:51:07.674 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.984312+0000", 2023-09-23T20:51:07.674 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.8760000000000002e-06 2023-09-23T20:51:07.674 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:07.674 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:51:07.674 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:07.674 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.675 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.675 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:197441 10.17 10.792e7cf7 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e624)", 2023-09-23T20:51:07.675 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:49:29.235108+0000", 2023-09-23T20:51:07.675 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 98.303150810999995, 2023-09-23T20:51:07.675 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.74923685299999998, 2023-09-23T20:51:07.675 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:51:07.675 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:51:07.675 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:51:07.675 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:51:07.675 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:51:07.675 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 197441 2023-09-23T20:51:07.676 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.676 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:51:07.676 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.676 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:51:07.676 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.235108+0000", 2023-09-23T20:51:07.676 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:07.676 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.676 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.676 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:51:07.676 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.235108+0000", 2023-09-23T20:51:07.676 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:07.676 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.677 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.677 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:51:07.677 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.235107+0000", 2023-09-23T20:51:07.677 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:51:07.677 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.677 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.677 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:51:07.677 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.235111+0000", 2023-09-23T20:51:07.677 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.9320000000000004e-06 2023-09-23T20:51:07.677 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.677 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.677 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:51:07.678 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.235123+0000", 2023-09-23T20:51:07.678 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1229999999999999e-05 2023-09-23T20:51:07.678 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.678 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.678 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:51:07.678 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.235127+0000", 2023-09-23T20:51:07.678 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.7859999999999996e-06 2023-09-23T20:51:07.678 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.678 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.678 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:07.678 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.235167+0000", 2023-09-23T20:51:07.679 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.9158e-05 2023-09-23T20:51:07.679 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.679 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.679 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:51:07.679 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.235172+0000", 2023-09-23T20:51:07.679 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.4999999999999999e-06 2023-09-23T20:51:07.679 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.679 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.679 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:07.679 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.984338+0000", 2023-09-23T20:51:07.679 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.74916548599999999 2023-09-23T20:51:07.679 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:07.680 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:07.680 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:51:07.680 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:29.984345+0000", 2023-09-23T20:51:07.680 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.9979999999999997e-06 2023-09-23T20:51:07.680 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:07.680 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:51:07.680 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:07.680 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:07.680 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:51:07.680 INFO:teuthology.orchestra.run.smithi149.stdout:} 2023-09-23T20:51:07.976 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T20:51:08.001 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:07 smithi149 ceph-mon[170394]: pgmap v1638: 103 pgs: 103 active+clean; 8.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail 2023-09-23T20:51:08.002 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:07 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:51:08.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:07 smithi018 ceph-mon[184144]: pgmap v1638: 103 pgs: 103 active+clean; 8.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail 2023-09-23T20:51:08.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:07 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:51:08.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:07 smithi018 ceph-mon[179647]: pgmap v1638: 103 pgs: 103 active+clean; 8.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail 2023-09-23T20:51:08.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:07 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:51:08.599 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:51:09.376 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:09 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:51:09.376 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:09 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:51:09.376 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:09 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:51:09.376 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:09 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:51:09.377 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:09 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:51:09.377 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:09 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:51:09.377 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:09 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:51:09.377 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:09 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:51:09.403 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:09 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:51:09.403 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:09 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:51:09.403 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:09 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:51:09.403 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:09 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:51:10.008 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:51:10.010 INFO:teuthology.orchestra.run.smithi018.stderr:dumped all 2023-09-23T20:51:10.119 INFO:teuthology.orchestra.run.smithi149.stdout:{ 2023-09-23T20:51:10.119 INFO:teuthology.orchestra.run.smithi149.stdout: "ops": [], 2023-09-23T20:51:10.119 INFO:teuthology.orchestra.run.smithi149.stdout: "num_ops": 0 2023-09-23T20:51:10.119 INFO:teuthology.orchestra.run.smithi149.stdout:} 2023-09-23T20:51:10.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:10 smithi149 ceph-mon[170394]: pgmap v1639: 103 pgs: 103 active+clean; 8.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail 2023-09-23T20:51:10.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:10 smithi018 ceph-mon[184144]: pgmap v1639: 103 pgs: 103 active+clean; 8.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail 2023-09-23T20:51:10.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:10 smithi018 ceph-mon[179647]: pgmap v1639: 103 pgs: 103 active+clean; 8.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail 2023-09-23T20:51:10.564 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T20:51:10.622 INFO:teuthology.orchestra.run.smithi018.stdout:{"pg_ready":true,"pg_map":{"version":1639,"stamp":"2023-09-23T20:51:08.378950+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":8984499,"num_objects":29,"num_object_clones":0,"num_object_copies":87,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":29,"num_whiteouts":0,"num_read":11981,"num_read_kb":26415,"num_write":1751,"num_write_kb":25419,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":276,"num_bytes_recovered":25707063,"num_keys_recovered":80,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6996,"ondisk_log_size":6996,"up":309,"acting":309,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":310,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":5518772,"kb_used_data":80764,"kb_used_omap":32,"kb_used_meta":5437919,"kb_avail":650697292,"statfs":{"total":671965249536,"available":666314027008,"internally_reserved":0,"allocated":82702336,"data_stored":64554248,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32773,"internal_metadata":5568430075},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.002493"},"pg_stats":[{"pgid":"1.18","version":"90'161","reported_seq":735,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-09-23T20:51:01.803937+0000","last_change":"2023-09-23T20:49:51.130089+0000","last_active":"2023-09-23T20:51:01.803937+0000","last_peered":"2023-09-23T20:51:01.803937+0000","last_clean":"2023-09-23T20:51:01.803937+0000","last_became_active":"2023-09-23T20:49:51.124899+0000","last_became_peered":"2023-09-23T20:49:51.124899+0000","last_unstale":"2023-09-23T20:51:01.803937+0000","last_undegraded":"2023-09-23T20:51:01.803937+0000","last_fullsized":"2023-09-23T20:51:01.803937+0000","mapping_epoch":639,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":640,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:19.677209+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:19.677209+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:58:59.316978+0000","scrub_duration":0.033122453000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"69'1","reported_seq":148,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-09-23T20:51:01.809120+0000","last_change":"2023-09-23T20:48:48.148250+0000","last_active":"2023-09-23T20:51:01.809120+0000","last_peered":"2023-09-23T20:51:01.809120+0000","last_clean":"2023-09-23T20:51:01.809120+0000","last_became_active":"2023-09-23T20:48:48.144271+0000","last_became_peered":"2023-09-23T20:48:48.144271+0000","last_unstale":"2023-09-23T20:51:01.809120+0000","last_undegraded":"2023-09-23T20:51:01.809120+0000","last_fullsized":"2023-09-23T20:51:01.809120+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:59:36.502986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":516,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-09-23T20:51:01.806959+0000","last_change":"2023-09-23T20:49:02.703099+0000","last_active":"2023-09-23T20:51:01.806959+0000","last_peered":"2023-09-23T20:51:01.806959+0000","last_clean":"2023-09-23T20:51:01.806959+0000","last_became_active":"2023-09-23T20:49:02.702753+0000","last_became_peered":"2023-09-23T20:49:02.702753+0000","last_unstale":"2023-09-23T20:51:01.806959+0000","last_undegraded":"2023-09-23T20:51:01.806959+0000","last_fullsized":"2023-09-23T20:51:01.806959+0000","mapping_epoch":603,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":604,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:24:30.834235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"90'161","reported_seq":694,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-09-23T20:51:01.809524+0000","last_change":"2023-09-23T20:49:51.121955+0000","last_active":"2023-09-23T20:51:01.809524+0000","last_peered":"2023-09-23T20:51:01.809524+0000","last_clean":"2023-09-23T20:51:01.809524+0000","last_became_active":"2023-09-23T20:49:51.121641+0000","last_became_peered":"2023-09-23T20:49:51.121641+0000","last_unstale":"2023-09-23T20:51:01.809524+0000","last_undegraded":"2023-09-23T20:51:01.809524+0000","last_fullsized":"2023-09-23T20:51:01.809524+0000","mapping_epoch":639,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":640,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:47:33.523187+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:36:05.442288+0000","last_clean_scrub_stamp":"2023-09-23T20:47:33.523187+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:05:36.252341+0000","scrub_duration":0.013532127,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,2],"acting":[4,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":662,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-09-23T20:51:01.801978+0000","last_change":"2023-09-23T20:47:09.173501+0000","last_active":"2023-09-23T20:51:01.801978+0000","last_peered":"2023-09-23T20:51:01.801978+0000","last_clean":"2023-09-23T20:51:01.801978+0000","last_became_active":"2023-09-23T20:47:09.173160+0000","last_became_peered":"2023-09-23T20:47:09.173160+0000","last_unstale":"2023-09-23T20:51:01.801978+0000","last_undegraded":"2023-09-23T20:51:01.801978+0000","last_fullsized":"2023-09-23T20:51:01.801978+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:44:36.132093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":632,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-09-23T20:51:02.228433+0000","last_change":"2023-09-23T20:43:46.521939+0000","last_active":"2023-09-23T20:51:02.228433+0000","last_peered":"2023-09-23T20:51:02.228433+0000","last_clean":"2023-09-23T20:51:02.228433+0000","last_became_active":"2023-09-23T20:43:46.521185+0000","last_became_peered":"2023-09-23T20:43:46.521185+0000","last_unstale":"2023-09-23T20:51:02.228433+0000","last_undegraded":"2023-09-23T20:51:02.228433+0000","last_fullsized":"2023-09-23T20:51:02.228433+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:59:25.045627+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"90'161","reported_seq":559,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-09-23T20:51:01.809050+0000","last_change":"2023-09-23T20:47:31.475001+0000","last_active":"2023-09-23T20:51:01.809050+0000","last_peered":"2023-09-23T20:51:01.809050+0000","last_clean":"2023-09-23T20:51:01.809050+0000","last_became_active":"2023-09-23T20:47:11.617942+0000","last_became_peered":"2023-09-23T20:47:11.617942+0000","last_unstale":"2023-09-23T20:51:01.809050+0000","last_undegraded":"2023-09-23T20:51:01.809050+0000","last_fullsized":"2023-09-23T20:51:01.809050+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:47:31.474914+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:47:31.474914+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:52:05.169219+0000","scrub_duration":0.0026262870000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":469,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-09-23T20:51:01.807039+0000","last_change":"2023-09-23T20:37:43.584414+0000","last_active":"2023-09-23T20:51:01.807039+0000","last_peered":"2023-09-23T20:51:01.807039+0000","last_clean":"2023-09-23T20:51:01.807039+0000","last_became_active":"2023-09-23T20:37:43.584116+0000","last_became_peered":"2023-09-23T20:37:43.584116+0000","last_unstale":"2023-09-23T20:51:01.807039+0000","last_undegraded":"2023-09-23T20:51:01.807039+0000","last_fullsized":"2023-09-23T20:51:01.807039+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:17:54.258172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":296,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-09-23T20:51:01.811101+0000","last_change":"2023-09-23T20:49:48.119179+0000","last_active":"2023-09-23T20:51:01.811101+0000","last_peered":"2023-09-23T20:51:01.811101+0000","last_clean":"2023-09-23T20:51:01.811101+0000","last_became_active":"2023-09-23T20:49:48.118849+0000","last_became_peered":"2023-09-23T20:49:48.118849+0000","last_unstale":"2023-09-23T20:51:01.811101+0000","last_undegraded":"2023-09-23T20:51:01.811101+0000","last_fullsized":"2023-09-23T20:51:01.811101+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:50:02.578569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"90'161","reported_seq":904,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-09-23T20:51:01.896146+0000","last_change":"2023-09-23T20:49:51.124760+0000","last_active":"2023-09-23T20:51:01.896146+0000","last_peered":"2023-09-23T20:51:01.896146+0000","last_clean":"2023-09-23T20:51:01.896146+0000","last_became_active":"2023-09-23T20:49:51.124406+0000","last_became_peered":"2023-09-23T20:49:51.124406+0000","last_unstale":"2023-09-23T20:51:01.896146+0000","last_undegraded":"2023-09-23T20:51:01.896146+0000","last_fullsized":"2023-09-23T20:51:01.896146+0000","mapping_epoch":639,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":640,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:03.411025+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:03.411025+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:28:15.568088+0000","scrub_duration":0.001802922,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":674,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-09-23T20:51:02.873433+0000","last_change":"2023-09-23T20:49:48.111285+0000","last_active":"2023-09-23T20:51:02.873433+0000","last_peered":"2023-09-23T20:51:02.873433+0000","last_clean":"2023-09-23T20:51:02.873433+0000","last_became_active":"2023-09-23T20:49:48.110934+0000","last_became_peered":"2023-09-23T20:49:48.110934+0000","last_unstale":"2023-09-23T20:51:02.873433+0000","last_undegraded":"2023-09-23T20:51:02.873433+0000","last_fullsized":"2023-09-23T20:51:02.873433+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:19:18.339128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":115,"reported_epoch":672,"state":"active+clean","last_fresh":"2023-09-23T20:51:02.816162+0000","last_change":"2023-09-23T20:51:02.816162+0000","last_active":"2023-09-23T20:51:02.816162+0000","last_peered":"2023-09-23T20:51:02.816162+0000","last_clean":"2023-09-23T20:51:02.816162+0000","last_became_active":"2023-09-23T20:51:02.815754+0000","last_became_peered":"2023-09-23T20:51:02.815754+0000","last_unstale":"2023-09-23T20:51:02.816162+0000","last_undegraded":"2023-09-23T20:51:02.816162+0000","last_fullsized":"2023-09-23T20:51:02.816162+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:29:00.532296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"90'161","reported_seq":674,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-09-23T20:51:01.803956+0000","last_change":"2023-09-23T20:49:36.717905+0000","last_active":"2023-09-23T20:51:01.803956+0000","last_peered":"2023-09-23T20:51:01.803956+0000","last_clean":"2023-09-23T20:51:01.803956+0000","last_became_active":"2023-09-23T20:48:51.169314+0000","last_became_peered":"2023-09-23T20:48:51.169314+0000","last_unstale":"2023-09-23T20:51:01.803956+0000","last_undegraded":"2023-09-23T20:51:01.803956+0000","last_fullsized":"2023-09-23T20:51:01.803956+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:49:36.717737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:49:36.717737+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:00:49.213712+0000","scrub_duration":0.0049746089999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":518,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-09-23T20:51:01.896313+0000","last_change":"2023-09-23T20:49:48.123820+0000","last_active":"2023-09-23T20:51:01.896313+0000","last_peered":"2023-09-23T20:51:01.896313+0000","last_clean":"2023-09-23T20:51:01.896313+0000","last_became_active":"2023-09-23T20:49:48.123514+0000","last_became_peered":"2023-09-23T20:49:48.123514+0000","last_unstale":"2023-09-23T20:51:01.896313+0000","last_undegraded":"2023-09-23T20:51:01.896313+0000","last_fullsized":"2023-09-23T20:51:01.896313+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:43:13.460464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":500,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-09-23T20:51:01.806359+0000","last_change":"2023-09-23T20:38:02.542034+0000","last_active":"2023-09-23T20:51:01.806359+0000","last_peered":"2023-09-23T20:51:01.806359+0000","last_clean":"2023-09-23T20:51:01.806359+0000","last_became_active":"2023-09-23T20:38:02.541575+0000","last_became_peered":"2023-09-23T20:38:02.541575+0000","last_unstale":"2023-09-23T20:51:01.806359+0000","last_undegraded":"2023-09-23T20:51:01.806359+0000","last_fullsized":"2023-09-23T20:51:01.806359+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:50:54.591529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"90'161","reported_seq":817,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-09-23T20:51:02.873403+0000","last_change":"2023-09-23T20:49:51.125057+0000","last_active":"2023-09-23T20:51:02.873403+0000","last_peered":"2023-09-23T20:51:02.873403+0000","last_clean":"2023-09-23T20:51:02.873403+0000","last_became_active":"2023-09-23T20:49:51.124616+0000","last_became_peered":"2023-09-23T20:49:51.124616+0000","last_unstale":"2023-09-23T20:51:02.873403+0000","last_undegraded":"2023-09-23T20:51:02.873403+0000","last_fullsized":"2023-09-23T20:51:02.873403+0000","mapping_epoch":639,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":640,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:14.669831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:14.669831+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:09:01.736519+0000","scrub_duration":0.0083441460000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":731,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-09-23T20:51:01.896435+0000","last_change":"2023-09-23T20:47:10.837423+0000","last_active":"2023-09-23T20:51:01.896435+0000","last_peered":"2023-09-23T20:51:01.896435+0000","last_clean":"2023-09-23T20:51:01.896435+0000","last_became_active":"2023-09-23T20:47:09.204324+0000","last_became_peered":"2023-09-23T20:47:09.204324+0000","last_unstale":"2023-09-23T20:51:01.896435+0000","last_undegraded":"2023-09-23T20:51:01.896435+0000","last_fullsized":"2023-09-23T20:51:01.896435+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":554,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:19.108914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":540,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":206,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-09-23T20:51:01.802659+0000","last_change":"2023-09-23T20:48:48.145175+0000","last_active":"2023-09-23T20:51:01.802659+0000","last_peered":"2023-09-23T20:51:01.802659+0000","last_clean":"2023-09-23T20:51:01.802659+0000","last_became_active":"2023-09-23T20:48:48.144841+0000","last_became_peered":"2023-09-23T20:48:48.144841+0000","last_unstale":"2023-09-23T20:51:01.802659+0000","last_undegraded":"2023-09-23T20:51:01.802659+0000","last_fullsized":"2023-09-23T20:51:01.802659+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:27:40.802917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"90'161","reported_seq":800,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-09-23T20:51:01.802940+0000","last_change":"2023-09-23T20:36:02.548387+0000","last_active":"2023-09-23T20:51:01.802940+0000","last_peered":"2023-09-23T20:51:01.802940+0000","last_clean":"2023-09-23T20:51:01.802940+0000","last_became_active":"2023-09-23T20:35:26.644787+0000","last_became_peered":"2023-09-23T20:35:26.644787+0000","last_unstale":"2023-09-23T20:51:01.802940+0000","last_undegraded":"2023-09-23T20:51:01.802940+0000","last_fullsized":"2023-09-23T20:51:01.802940+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:02.548285+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:02.548285+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:37:15.216553+0000","scrub_duration":0.0015514459999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":138,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-09-23T20:51:01.809365+0000","last_change":"2023-09-23T20:47:09.340979+0000","last_active":"2023-09-23T20:51:01.809365+0000","last_peered":"2023-09-23T20:51:01.809365+0000","last_clean":"2023-09-23T20:51:01.809365+0000","last_became_active":"2023-09-23T20:47:09.231222+0000","last_became_peered":"2023-09-23T20:47:09.231222+0000","last_unstale":"2023-09-23T20:51:01.809365+0000","last_undegraded":"2023-09-23T20:51:01.809365+0000","last_fullsized":"2023-09-23T20:51:01.809365+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:07:50.620006+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":128,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-09-23T20:51:01.809640+0000","last_change":"2023-09-23T20:47:09.186315+0000","last_active":"2023-09-23T20:51:01.809640+0000","last_peered":"2023-09-23T20:51:01.809640+0000","last_clean":"2023-09-23T20:51:01.809640+0000","last_became_active":"2023-09-23T20:47:09.184580+0000","last_became_peered":"2023-09-23T20:47:09.184580+0000","last_unstale":"2023-09-23T20:51:01.809640+0000","last_undegraded":"2023-09-23T20:51:01.809640+0000","last_fullsized":"2023-09-23T20:51:01.809640+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:44:35.553948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"90'161","reported_seq":945,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-09-23T20:51:02.227852+0000","last_change":"2023-09-23T20:49:51.148240+0000","last_active":"2023-09-23T20:51:02.227852+0000","last_peered":"2023-09-23T20:51:02.227852+0000","last_clean":"2023-09-23T20:51:02.227852+0000","last_became_active":"2023-09-23T20:49:51.147475+0000","last_became_peered":"2023-09-23T20:49:51.147475+0000","last_unstale":"2023-09-23T20:51:02.227852+0000","last_undegraded":"2023-09-23T20:51:02.227852+0000","last_fullsized":"2023-09-23T20:51:02.227852+0000","mapping_epoch":639,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":640,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:43.578666+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:43.578666+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:02:42.273671+0000","scrub_duration":0.0021890379999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":762,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-09-23T20:51:01.896016+0000","last_change":"2023-09-23T20:48:48.193666+0000","last_active":"2023-09-23T20:51:01.896016+0000","last_peered":"2023-09-23T20:51:01.896016+0000","last_clean":"2023-09-23T20:51:01.896016+0000","last_became_active":"2023-09-23T20:48:48.152081+0000","last_became_peered":"2023-09-23T20:48:48.152081+0000","last_unstale":"2023-09-23T20:51:01.896016+0000","last_undegraded":"2023-09-23T20:51:01.896016+0000","last_fullsized":"2023-09-23T20:51:01.896016+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:00:19.508124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":272,"num_keys_recovered":80,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":429,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-09-23T20:51:01.806430+0000","last_change":"2023-09-23T20:49:48.119365+0000","last_active":"2023-09-23T20:51:01.806430+0000","last_peered":"2023-09-23T20:51:01.806430+0000","last_clean":"2023-09-23T20:51:01.806430+0000","last_became_active":"2023-09-23T20:49:48.119009+0000","last_became_peered":"2023-09-23T20:49:48.119009+0000","last_unstale":"2023-09-23T20:51:01.806430+0000","last_undegraded":"2023-09-23T20:51:01.806430+0000","last_fullsized":"2023-09-23T20:51:01.806430+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:02:26.878542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"90'161","reported_seq":972,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-09-23T20:51:01.895796+0000","last_change":"2023-09-23T20:49:51.118813+0000","last_active":"2023-09-23T20:51:01.895796+0000","last_peered":"2023-09-23T20:51:01.895796+0000","last_clean":"2023-09-23T20:51:01.895796+0000","last_became_active":"2023-09-23T20:49:51.118445+0000","last_became_peered":"2023-09-23T20:49:51.118445+0000","last_unstale":"2023-09-23T20:51:01.895796+0000","last_undegraded":"2023-09-23T20:51:01.895796+0000","last_fullsized":"2023-09-23T20:51:01.895796+0000","mapping_epoch":639,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":640,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:20.041690+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:20.041690+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:15:24.683302+0000","scrub_duration":0.0026180330000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":630,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-09-23T20:51:02.228547+0000","last_change":"2023-09-23T20:43:46.521294+0000","last_active":"2023-09-23T20:51:02.228547+0000","last_peered":"2023-09-23T20:51:02.228547+0000","last_clean":"2023-09-23T20:51:02.228547+0000","last_became_active":"2023-09-23T20:43:46.521001+0000","last_became_peered":"2023-09-23T20:43:46.521001+0000","last_unstale":"2023-09-23T20:51:02.228547+0000","last_undegraded":"2023-09-23T20:51:02.228547+0000","last_fullsized":"2023-09-23T20:51:02.228547+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:01:35.126716+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":459,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-09-23T20:51:01.896492+0000","last_change":"2023-09-23T20:49:48.121288+0000","last_active":"2023-09-23T20:51:01.896492+0000","last_peered":"2023-09-23T20:51:01.896492+0000","last_clean":"2023-09-23T20:51:01.896492+0000","last_became_active":"2023-09-23T20:49:48.120936+0000","last_became_peered":"2023-09-23T20:49:48.120936+0000","last_unstale":"2023-09-23T20:51:01.896492+0000","last_undegraded":"2023-09-23T20:51:01.896492+0000","last_fullsized":"2023-09-23T20:51:01.896492+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:46:44.012748+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.13","version":"90'161","reported_seq":996,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-09-23T20:51:01.802440+0000","last_change":"2023-09-23T20:47:11.619371+0000","last_active":"2023-09-23T20:51:01.802440+0000","last_peered":"2023-09-23T20:51:01.802440+0000","last_clean":"2023-09-23T20:51:01.802440+0000","last_became_active":"2023-09-23T20:47:11.619006+0000","last_became_peered":"2023-09-23T20:47:11.619006+0000","last_unstale":"2023-09-23T20:51:01.802440+0000","last_undegraded":"2023-09-23T20:51:01.802440+0000","last_fullsized":"2023-09-23T20:51:01.802440+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:42.544308+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:42.544308+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:47:24.414807+0000","scrub_duration":0.0029958179999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":695,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-09-23T20:51:01.895943+0000","last_change":"2023-09-23T20:47:09.176613+0000","last_active":"2023-09-23T20:51:01.895943+0000","last_peered":"2023-09-23T20:51:01.895943+0000","last_clean":"2023-09-23T20:51:01.895943+0000","last_became_active":"2023-09-23T20:47:09.176187+0000","last_became_peered":"2023-09-23T20:47:09.176187+0000","last_unstale":"2023-09-23T20:51:01.895943+0000","last_undegraded":"2023-09-23T20:51:01.895943+0000","last_fullsized":"2023-09-23T20:51:01.895943+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:23:01.590989+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":639,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-09-23T20:51:02.873374+0000","last_change":"2023-09-23T20:43:46.521735+0000","last_active":"2023-09-23T20:51:02.873374+0000","last_peered":"2023-09-23T20:51:02.873374+0000","last_clean":"2023-09-23T20:51:02.873374+0000","last_became_active":"2023-09-23T20:43:46.520762+0000","last_became_peered":"2023-09-23T20:43:46.520762+0000","last_unstale":"2023-09-23T20:51:02.873374+0000","last_undegraded":"2023-09-23T20:51:02.873374+0000","last_fullsized":"2023-09-23T20:51:02.873374+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:26:48.841930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"90'161","reported_seq":775,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-09-23T20:51:01.802902+0000","last_change":"2023-09-23T20:49:51.125786+0000","last_active":"2023-09-23T20:51:01.802902+0000","last_peered":"2023-09-23T20:51:01.802902+0000","last_clean":"2023-09-23T20:51:01.802902+0000","last_became_active":"2023-09-23T20:49:51.124373+0000","last_became_peered":"2023-09-23T20:49:51.124373+0000","last_unstale":"2023-09-23T20:51:01.802902+0000","last_undegraded":"2023-09-23T20:51:01.802902+0000","last_fullsized":"2023-09-23T20:51:01.802902+0000","mapping_epoch":639,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":640,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:18.952693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:18.952693+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:07:29.170332+0000","scrub_duration":0.001699365,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":366,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-09-23T20:51:01.810701+0000","last_change":"2023-09-23T20:49:48.127484+0000","last_active":"2023-09-23T20:51:01.810701+0000","last_peered":"2023-09-23T20:51:01.810701+0000","last_clean":"2023-09-23T20:51:01.810701+0000","last_became_active":"2023-09-23T20:49:48.110748+0000","last_became_peered":"2023-09-23T20:49:48.110748+0000","last_unstale":"2023-09-23T20:51:01.810701+0000","last_undegraded":"2023-09-23T20:51:01.810701+0000","last_fullsized":"2023-09-23T20:51:01.810701+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:31:51.004529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":469,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-09-23T20:51:01.807327+0000","last_change":"2023-09-23T20:37:43.571083+0000","last_active":"2023-09-23T20:51:01.807327+0000","last_peered":"2023-09-23T20:51:01.807327+0000","last_clean":"2023-09-23T20:51:01.807327+0000","last_became_active":"2023-09-23T20:37:43.570306+0000","last_became_peered":"2023-09-23T20:37:43.570306+0000","last_unstale":"2023-09-23T20:51:01.807327+0000","last_undegraded":"2023-09-23T20:51:01.807327+0000","last_fullsized":"2023-09-23T20:51:01.807327+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:42:02.396718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.d","version":"90'161","reported_seq":1004,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-09-23T20:51:01.896242+0000","last_change":"2023-09-23T20:47:11.619510+0000","last_active":"2023-09-23T20:51:01.896242+0000","last_peered":"2023-09-23T20:51:01.896242+0000","last_clean":"2023-09-23T20:51:01.896242+0000","last_became_active":"2023-09-23T20:47:11.619154+0000","last_became_peered":"2023-09-23T20:47:11.619154+0000","last_unstale":"2023-09-23T20:51:01.896242+0000","last_undegraded":"2023-09-23T20:51:01.896242+0000","last_fullsized":"2023-09-23T20:51:01.896242+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:34:59.496375+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:34:59.496375+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:40:52.712777+0000","scrub_duration":0.0017843189999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":703,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-09-23T20:51:01.896479+0000","last_change":"2023-09-23T20:47:09.207851+0000","last_active":"2023-09-23T20:51:01.896479+0000","last_peered":"2023-09-23T20:51:01.896479+0000","last_clean":"2023-09-23T20:51:01.896479+0000","last_became_active":"2023-09-23T20:47:09.175836+0000","last_became_peered":"2023-09-23T20:47:09.175836+0000","last_unstale":"2023-09-23T20:51:01.896479+0000","last_undegraded":"2023-09-23T20:51:01.896479+0000","last_fullsized":"2023-09-23T20:51:01.896479+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:43:13.091809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":496,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-09-23T20:51:01.895992+0000","last_change":"2023-09-23T20:49:48.122188+0000","last_active":"2023-09-23T20:51:01.895992+0000","last_peered":"2023-09-23T20:51:01.895992+0000","last_clean":"2023-09-23T20:51:01.895992+0000","last_became_active":"2023-09-23T20:49:48.121894+0000","last_became_peered":"2023-09-23T20:49:48.121894+0000","last_unstale":"2023-09-23T20:51:01.895992+0000","last_undegraded":"2023-09-23T20:51:01.895992+0000","last_fullsized":"2023-09-23T20:51:01.895992+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:12:33.758985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.c","version":"90'161","reported_seq":935,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-09-23T20:51:01.806522+0000","last_change":"2023-09-23T20:41:09.506481+0000","last_active":"2023-09-23T20:51:01.806522+0000","last_peered":"2023-09-23T20:51:01.806522+0000","last_clean":"2023-09-23T20:51:01.806522+0000","last_became_active":"2023-09-23T20:38:28.442229+0000","last_became_peered":"2023-09-23T20:38:28.442229+0000","last_unstale":"2023-09-23T20:51:01.806522+0000","last_undegraded":"2023-09-23T20:51:01.806522+0000","last_fullsized":"2023-09-23T20:51:01.806522+0000","mapping_epoch":411,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":412,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:41:09.506338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:41:09.506338+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:19:46.265011+0000","scrub_duration":0.01112002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":741,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-09-23T20:51:01.896346+0000","last_change":"2023-09-23T20:47:10.848436+0000","last_active":"2023-09-23T20:51:01.896346+0000","last_peered":"2023-09-23T20:51:01.896346+0000","last_clean":"2023-09-23T20:51:01.896346+0000","last_became_active":"2023-09-23T20:47:09.196769+0000","last_became_peered":"2023-09-23T20:47:09.196769+0000","last_unstale":"2023-09-23T20:51:01.896346+0000","last_undegraded":"2023-09-23T20:51:01.896346+0000","last_fullsized":"2023-09-23T20:51:01.896346+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":554,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:59:38.105009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":138,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-09-23T20:51:01.808977+0000","last_change":"2023-09-23T20:48:23.915817+0000","last_active":"2023-09-23T20:51:01.808977+0000","last_peered":"2023-09-23T20:51:01.808977+0000","last_clean":"2023-09-23T20:51:01.808977+0000","last_became_active":"2023-09-23T20:48:23.915306+0000","last_became_peered":"2023-09-23T20:48:23.915306+0000","last_unstale":"2023-09-23T20:51:01.808977+0000","last_undegraded":"2023-09-23T20:51:01.808977+0000","last_fullsized":"2023-09-23T20:51:01.808977+0000","mapping_epoch":577,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":578,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:55:45.691450+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"90'161","reported_seq":886,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-09-23T20:51:01.811303+0000","last_change":"2023-09-23T20:49:51.120296+0000","last_active":"2023-09-23T20:51:01.811303+0000","last_peered":"2023-09-23T20:51:01.811303+0000","last_clean":"2023-09-23T20:51:01.811303+0000","last_became_active":"2023-09-23T20:49:51.120016+0000","last_became_peered":"2023-09-23T20:49:51.120016+0000","last_unstale":"2023-09-23T20:51:01.811303+0000","last_undegraded":"2023-09-23T20:51:01.811303+0000","last_fullsized":"2023-09-23T20:51:01.811303+0000","mapping_epoch":639,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":640,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:46.217489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:46.217489+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:35:59.437404+0000","scrub_duration":0.001456429,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":714,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-09-23T20:51:01.896316+0000","last_change":"2023-09-23T20:49:48.131779+0000","last_active":"2023-09-23T20:51:01.896316+0000","last_peered":"2023-09-23T20:51:01.896316+0000","last_clean":"2023-09-23T20:51:01.896316+0000","last_became_active":"2023-09-23T20:49:48.123934+0000","last_became_peered":"2023-09-23T20:49:48.123934+0000","last_unstale":"2023-09-23T20:51:01.896316+0000","last_undegraded":"2023-09-23T20:51:01.896316+0000","last_fullsized":"2023-09-23T20:51:01.896316+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:24:40.508247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":490,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":286,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-09-23T20:51:01.809998+0000","last_change":"2023-09-23T20:49:02.702301+0000","last_active":"2023-09-23T20:51:01.809998+0000","last_peered":"2023-09-23T20:51:01.809998+0000","last_clean":"2023-09-23T20:51:01.809998+0000","last_became_active":"2023-09-23T20:49:02.701986+0000","last_became_peered":"2023-09-23T20:49:02.701986+0000","last_unstale":"2023-09-23T20:51:01.809998+0000","last_undegraded":"2023-09-23T20:51:01.809998+0000","last_fullsized":"2023-09-23T20:51:01.809998+0000","mapping_epoch":603,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":604,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:20:01.603267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.e","version":"90'161","reported_seq":701,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-09-23T20:51:01.809934+0000","last_change":"2023-09-23T20:47:37.608426+0000","last_active":"2023-09-23T20:51:01.809934+0000","last_peered":"2023-09-23T20:51:01.809934+0000","last_clean":"2023-09-23T20:51:01.809934+0000","last_became_active":"2023-09-23T20:47:11.617187+0000","last_became_peered":"2023-09-23T20:47:11.617187+0000","last_unstale":"2023-09-23T20:51:01.809934+0000","last_undegraded":"2023-09-23T20:51:01.809934+0000","last_fullsized":"2023-09-23T20:51:01.809934+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:47:37.608309+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:47:37.608309+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:22:25.482297+0000","scrub_duration":0.011600924,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":643,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-09-23T20:51:01.806927+0000","last_change":"2023-09-23T20:49:48.120934+0000","last_active":"2023-09-23T20:51:01.806927+0000","last_peered":"2023-09-23T20:51:01.806927+0000","last_clean":"2023-09-23T20:51:01.806927+0000","last_became_active":"2023-09-23T20:49:48.120578+0000","last_became_peered":"2023-09-23T20:49:48.120578+0000","last_unstale":"2023-09-23T20:51:01.806927+0000","last_undegraded":"2023-09-23T20:51:01.806927+0000","last_fullsized":"2023-09-23T20:51:01.806927+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:57:50.669399+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":666,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-09-23T20:51:02.228020+0000","last_change":"2023-09-23T20:47:09.197427+0000","last_active":"2023-09-23T20:51:02.228020+0000","last_peered":"2023-09-23T20:51:02.228020+0000","last_clean":"2023-09-23T20:51:02.228020+0000","last_became_active":"2023-09-23T20:47:09.196542+0000","last_became_peered":"2023-09-23T20:47:09.196542+0000","last_unstale":"2023-09-23T20:51:02.228020+0000","last_undegraded":"2023-09-23T20:51:02.228020+0000","last_fullsized":"2023-09-23T20:51:02.228020+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:28:38.926202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":1117,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-09-23T20:51:01.895868+0000","last_change":"2023-09-23T20:48:51.169474+0000","last_active":"2023-09-23T20:51:01.895868+0000","last_peered":"2023-09-23T20:51:01.895868+0000","last_clean":"2023-09-23T20:51:01.895868+0000","last_became_active":"2023-09-23T20:48:51.169076+0000","last_became_peered":"2023-09-23T20:48:51.169076+0000","last_unstale":"2023-09-23T20:51:01.895868+0000","last_undegraded":"2023-09-23T20:51:01.895868+0000","last_fullsized":"2023-09-23T20:51:01.895868+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:35:02.523203+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:35:02.523203+0000","last_clean_scrub_stamp":"2023-09-23T20:35:02.523203+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:25:53.741403+0000","scrub_duration":0.0085927989999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":521,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-09-23T20:51:01.806732+0000","last_change":"2023-09-23T20:47:09.240200+0000","last_active":"2023-09-23T20:51:01.806732+0000","last_peered":"2023-09-23T20:51:01.806732+0000","last_clean":"2023-09-23T20:51:01.806732+0000","last_became_active":"2023-09-23T20:47:09.195204+0000","last_became_peered":"2023-09-23T20:47:09.195204+0000","last_unstale":"2023-09-23T20:51:01.806732+0000","last_undegraded":"2023-09-23T20:51:01.806732+0000","last_fullsized":"2023-09-23T20:51:01.806732+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:03.328250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":128,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-09-23T20:51:01.809793+0000","last_change":"2023-09-23T20:47:09.231714+0000","last_active":"2023-09-23T20:51:01.809793+0000","last_peered":"2023-09-23T20:51:01.809793+0000","last_clean":"2023-09-23T20:51:01.809793+0000","last_became_active":"2023-09-23T20:47:09.230918+0000","last_became_peered":"2023-09-23T20:47:09.230918+0000","last_unstale":"2023-09-23T20:51:01.809793+0000","last_undegraded":"2023-09-23T20:51:01.809793+0000","last_fullsized":"2023-09-23T20:51:01.809793+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:24.583656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":577,"reported_epoch":672,"state":"active+clean","last_fresh":"2023-09-23T20:51:02.811091+0000","last_change":"2023-09-23T20:50:03.882061+0000","last_active":"2023-09-23T20:51:02.811091+0000","last_peered":"2023-09-23T20:51:02.811091+0000","last_clean":"2023-09-23T20:51:02.811091+0000","last_became_active":"2023-09-23T20:49:51.124481+0000","last_became_peered":"2023-09-23T20:49:51.124481+0000","last_unstale":"2023-09-23T20:51:02.811091+0000","last_undegraded":"2023-09-23T20:51:02.811091+0000","last_fullsized":"2023-09-23T20:51:02.811091+0000","mapping_epoch":639,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":640,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:50:03.881826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:50:03.881826+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:51.282979+0000","scrub_duration":0.003245258,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":296,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-09-23T20:51:01.810089+0000","last_change":"2023-09-23T20:47:09.178919+0000","last_active":"2023-09-23T20:51:01.810089+0000","last_peered":"2023-09-23T20:51:01.810089+0000","last_clean":"2023-09-23T20:51:01.810089+0000","last_became_active":"2023-09-23T20:47:09.178607+0000","last_became_peered":"2023-09-23T20:47:09.178607+0000","last_unstale":"2023-09-23T20:51:01.810089+0000","last_undegraded":"2023-09-23T20:51:01.810089+0000","last_fullsized":"2023-09-23T20:51:01.810089+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:23:45.196935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":148,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-09-23T20:51:01.809676+0000","last_change":"2023-09-23T20:48:48.148596+0000","last_active":"2023-09-23T20:51:01.809676+0000","last_peered":"2023-09-23T20:51:01.809676+0000","last_clean":"2023-09-23T20:51:01.809676+0000","last_became_active":"2023-09-23T20:48:48.148211+0000","last_became_peered":"2023-09-23T20:48:48.148211+0000","last_unstale":"2023-09-23T20:51:01.809676+0000","last_undegraded":"2023-09-23T20:51:01.809676+0000","last_fullsized":"2023-09-23T20:51:01.809676+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:21:50.825822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"90'161","reported_seq":995,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-09-23T20:51:01.802526+0000","last_change":"2023-09-23T20:49:51.124836+0000","last_active":"2023-09-23T20:51:01.802526+0000","last_peered":"2023-09-23T20:51:01.802526+0000","last_clean":"2023-09-23T20:51:01.802526+0000","last_became_active":"2023-09-23T20:49:51.124419+0000","last_became_peered":"2023-09-23T20:49:51.124419+0000","last_unstale":"2023-09-23T20:51:01.802526+0000","last_undegraded":"2023-09-23T20:51:01.802526+0000","last_fullsized":"2023-09-23T20:51:01.802526+0000","mapping_epoch":639,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":640,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:05.595042+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:05.595042+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:37:05.219030+0000","scrub_duration":0.007271675,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":148,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-09-23T20:51:01.808871+0000","last_change":"2023-09-23T20:48:48.153145+0000","last_active":"2023-09-23T20:51:01.808871+0000","last_peered":"2023-09-23T20:51:01.808871+0000","last_clean":"2023-09-23T20:51:01.808871+0000","last_became_active":"2023-09-23T20:48:48.152858+0000","last_became_peered":"2023-09-23T20:48:48.152858+0000","last_unstale":"2023-09-23T20:51:01.808871+0000","last_undegraded":"2023-09-23T20:51:01.808871+0000","last_fullsized":"2023-09-23T20:51:01.808871+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:23:13.454288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":128,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-09-23T20:51:01.809933+0000","last_change":"2023-09-23T20:47:09.186271+0000","last_active":"2023-09-23T20:51:01.809933+0000","last_peered":"2023-09-23T20:51:01.809933+0000","last_clean":"2023-09-23T20:51:01.809933+0000","last_became_active":"2023-09-23T20:47:09.184357+0000","last_became_peered":"2023-09-23T20:47:09.184357+0000","last_unstale":"2023-09-23T20:51:01.809933+0000","last_undegraded":"2023-09-23T20:51:01.809933+0000","last_fullsized":"2023-09-23T20:51:01.809933+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:48:26.339811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":581,"reported_epoch":672,"state":"active+clean","last_fresh":"2023-09-23T20:51:02.811258+0000","last_change":"2023-09-23T20:49:51.128432+0000","last_active":"2023-09-23T20:51:02.811258+0000","last_peered":"2023-09-23T20:51:02.811258+0000","last_clean":"2023-09-23T20:51:02.811258+0000","last_became_active":"2023-09-23T20:49:51.128074+0000","last_became_peered":"2023-09-23T20:49:51.128074+0000","last_unstale":"2023-09-23T20:51:02.811258+0000","last_undegraded":"2023-09-23T20:51:02.811258+0000","last_fullsized":"2023-09-23T20:51:02.811258+0000","mapping_epoch":639,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":640,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:49:37.762492+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:49:37.762492+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:31:57.447267+0000","scrub_duration":0.0031485979999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,4],"acting":[2,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":470,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-09-23T20:51:01.806952+0000","last_change":"2023-09-23T20:37:43.570610+0000","last_active":"2023-09-23T20:51:01.806952+0000","last_peered":"2023-09-23T20:51:01.806952+0000","last_clean":"2023-09-23T20:51:01.806952+0000","last_became_active":"2023-09-23T20:37:43.570216+0000","last_became_peered":"2023-09-23T20:37:43.570216+0000","last_unstale":"2023-09-23T20:51:01.806952+0000","last_undegraded":"2023-09-23T20:51:01.806952+0000","last_fullsized":"2023-09-23T20:51:01.806952+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:39:26.555329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":518,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-09-23T20:51:01.802491+0000","last_change":"2023-09-23T20:47:09.172573+0000","last_active":"2023-09-23T20:51:01.802491+0000","last_peered":"2023-09-23T20:51:01.802491+0000","last_clean":"2023-09-23T20:51:01.802491+0000","last_became_active":"2023-09-23T20:47:09.172298+0000","last_became_peered":"2023-09-23T20:47:09.172298+0000","last_unstale":"2023-09-23T20:51:01.802491+0000","last_undegraded":"2023-09-23T20:51:01.802491+0000","last_fullsized":"2023-09-23T20:51:01.802491+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:22:09.361284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":859,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-09-23T20:51:01.811260+0000","last_change":"2023-09-23T20:41:07.424609+0000","last_active":"2023-09-23T20:51:01.811260+0000","last_peered":"2023-09-23T20:51:01.811260+0000","last_clean":"2023-09-23T20:51:01.811260+0000","last_became_active":"2023-09-23T20:38:05.622261+0000","last_became_peered":"2023-09-23T20:38:05.622261+0000","last_unstale":"2023-09-23T20:51:01.811260+0000","last_undegraded":"2023-09-23T20:51:01.811260+0000","last_fullsized":"2023-09-23T20:51:01.811260+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:41:07.424492+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:41:07.424492+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:45:21.689123+0000","scrub_duration":0.0018960959999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"664'242","reported_seq":1423,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-09-23T20:51:01.803727+0000","last_change":"2023-09-23T20:50:39.737511+0000","last_active":"2023-09-23T20:51:01.803727+0000","last_peered":"2023-09-23T20:51:01.803727+0000","last_clean":"2023-09-23T20:51:01.803727+0000","last_became_active":"2023-09-23T20:29:45.520546+0000","last_became_peered":"2023-09-23T20:29:45.520546+0000","last_unstale":"2023-09-23T20:51:01.803727+0000","last_undegraded":"2023-09-23T20:51:01.803727+0000","last_fullsized":"2023-09-23T20:51:01.803727+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":242,"log_dups_size":0,"ondisk_log_size":242,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:58:56.085777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0012974410000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1152,"num_read_kb":4628,"num_write":256,"num_write_kb":3463,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":494,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-09-23T20:51:01.807046+0000","last_change":"2023-09-23T20:49:48.114924+0000","last_active":"2023-09-23T20:51:01.807046+0000","last_peered":"2023-09-23T20:51:01.807046+0000","last_clean":"2023-09-23T20:51:01.807046+0000","last_became_active":"2023-09-23T20:49:48.114554+0000","last_became_peered":"2023-09-23T20:49:48.114554+0000","last_unstale":"2023-09-23T20:51:01.807046+0000","last_undegraded":"2023-09-23T20:51:01.807046+0000","last_fullsized":"2023-09-23T20:51:01.807046+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:09:09.538462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":659,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-09-23T20:51:01.802770+0000","last_change":"2023-09-23T20:49:48.115598+0000","last_active":"2023-09-23T20:51:01.802770+0000","last_peered":"2023-09-23T20:51:01.802770+0000","last_clean":"2023-09-23T20:51:01.802770+0000","last_became_active":"2023-09-23T20:49:48.115313+0000","last_became_peered":"2023-09-23T20:49:48.115313+0000","last_unstale":"2023-09-23T20:51:01.802770+0000","last_undegraded":"2023-09-23T20:51:01.802770+0000","last_fullsized":"2023-09-23T20:51:01.802770+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:58:47.477333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":771,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-09-23T20:51:02.228055+0000","last_change":"2023-09-23T20:49:51.148445+0000","last_active":"2023-09-23T20:51:02.228055+0000","last_peered":"2023-09-23T20:51:02.228055+0000","last_clean":"2023-09-23T20:51:02.228055+0000","last_became_active":"2023-09-23T20:49:51.147884+0000","last_became_peered":"2023-09-23T20:49:51.147884+0000","last_unstale":"2023-09-23T20:51:02.228055+0000","last_undegraded":"2023-09-23T20:51:02.228055+0000","last_fullsized":"2023-09-23T20:51:02.228055+0000","mapping_epoch":639,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":640,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:15.656880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:15.656880+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:05:08.116069+0000","scrub_duration":0.0016711180000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"664'209","reported_seq":1653,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-09-23T20:51:01.803317+0000","last_change":"2023-09-23T20:50:39.737378+0000","last_active":"2023-09-23T20:51:01.803317+0000","last_peered":"2023-09-23T20:51:01.803317+0000","last_clean":"2023-09-23T20:51:01.803317+0000","last_became_active":"2023-09-23T20:49:51.125316+0000","last_became_peered":"2023-09-23T20:49:51.125316+0000","last_unstale":"2023-09-23T20:51:01.803317+0000","last_undegraded":"2023-09-23T20:51:01.803317+0000","last_fullsized":"2023-09-23T20:51:01.803317+0000","mapping_epoch":639,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":640,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":209,"log_dups_size":0,"ondisk_log_size":209,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:32:47.065876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001172474,"stat_sum":{"num_bytes":4194323,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":2248,"num_read_kb":3726,"num_write":182,"num_write_kb":5560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":128,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-09-23T20:51:01.809730+0000","last_change":"2023-09-23T20:47:09.177872+0000","last_active":"2023-09-23T20:51:01.809730+0000","last_peered":"2023-09-23T20:51:01.809730+0000","last_clean":"2023-09-23T20:51:01.809730+0000","last_became_active":"2023-09-23T20:47:09.176689+0000","last_became_peered":"2023-09-23T20:47:09.176689+0000","last_unstale":"2023-09-23T20:51:01.809730+0000","last_undegraded":"2023-09-23T20:51:01.809730+0000","last_fullsized":"2023-09-23T20:51:01.809730+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:14:01.590318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":656,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-09-23T20:51:01.803649+0000","last_change":"2023-09-23T20:48:48.151144+0000","last_active":"2023-09-23T20:51:01.803649+0000","last_peered":"2023-09-23T20:51:01.803649+0000","last_clean":"2023-09-23T20:51:01.803649+0000","last_became_active":"2023-09-23T20:48:48.150770+0000","last_became_peered":"2023-09-23T20:48:48.150770+0000","last_unstale":"2023-09-23T20:51:01.803649+0000","last_undegraded":"2023-09-23T20:51:01.803649+0000","last_fullsized":"2023-09-23T20:51:01.803649+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:14:08.549512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":675,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-09-23T20:51:01.896080+0000","last_change":"2023-09-23T20:47:11.618652+0000","last_active":"2023-09-23T20:51:01.896080+0000","last_peered":"2023-09-23T20:51:01.896080+0000","last_clean":"2023-09-23T20:51:01.896080+0000","last_became_active":"2023-09-23T20:47:11.618266+0000","last_became_peered":"2023-09-23T20:47:11.618266+0000","last_unstale":"2023-09-23T20:51:01.896080+0000","last_undegraded":"2023-09-23T20:51:01.896080+0000","last_fullsized":"2023-09-23T20:51:01.896080+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:26.978179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:26.978179+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:29:29.000683+0000","scrub_duration":0.0013962829999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"664'192","reported_seq":708,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-09-23T20:51:01.805476+0000","last_change":"2023-09-23T20:50:39.795569+0000","last_active":"2023-09-23T20:51:01.805476+0000","last_peered":"2023-09-23T20:51:01.805476+0000","last_clean":"2023-09-23T20:51:01.805476+0000","last_became_active":"2023-09-23T20:48:51.170485+0000","last_became_peered":"2023-09-23T20:48:51.170485+0000","last_unstale":"2023-09-23T20:51:01.805476+0000","last_undegraded":"2023-09-23T20:51:01.805476+0000","last_fullsized":"2023-09-23T20:51:01.805476+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":192,"log_dups_size":0,"ondisk_log_size":192,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:48:35.309684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038574200000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":830,"num_read_kb":3504,"num_write":183,"num_write_kb":2624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":664,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-09-23T20:51:01.801857+0000","last_change":"2023-09-23T20:48:48.151034+0000","last_active":"2023-09-23T20:51:01.801857+0000","last_peered":"2023-09-23T20:51:01.801857+0000","last_clean":"2023-09-23T20:51:01.801857+0000","last_became_active":"2023-09-23T20:48:48.150658+0000","last_became_peered":"2023-09-23T20:48:48.150658+0000","last_unstale":"2023-09-23T20:51:01.801857+0000","last_undegraded":"2023-09-23T20:51:01.801857+0000","last_fullsized":"2023-09-23T20:51:01.801857+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:44:34.819851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":198,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-09-23T20:51:01.802413+0000","last_change":"2023-09-23T20:49:48.118182+0000","last_active":"2023-09-23T20:51:01.802413+0000","last_peered":"2023-09-23T20:51:01.802413+0000","last_clean":"2023-09-23T20:51:01.802413+0000","last_became_active":"2023-09-23T20:49:48.117804+0000","last_became_peered":"2023-09-23T20:49:48.117804+0000","last_unstale":"2023-09-23T20:51:01.802413+0000","last_undegraded":"2023-09-23T20:51:01.802413+0000","last_fullsized":"2023-09-23T20:51:01.802413+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:47:43.384018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"664'319","reported_seq":1157,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-09-23T20:51:01.803969+0000","last_change":"2023-09-23T20:50:39.795712+0000","last_active":"2023-09-23T20:51:01.803969+0000","last_peered":"2023-09-23T20:51:01.803969+0000","last_clean":"2023-09-23T20:51:01.803969+0000","last_became_active":"2023-09-23T20:49:51.153529+0000","last_became_peered":"2023-09-23T20:49:51.153529+0000","last_unstale":"2023-09-23T20:51:01.803969+0000","last_undegraded":"2023-09-23T20:51:01.803969+0000","last_fullsized":"2023-09-23T20:51:01.803969+0000","mapping_epoch":639,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":640,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":319,"log_dups_size":0,"ondisk_log_size":319,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:10:22.072962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043345100000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1396,"num_read_kb":4612,"num_write":290,"num_write_kb":4437,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":1134,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-09-23T20:51:01.895899+0000","last_change":"2023-09-23T20:38:05.602508+0000","last_active":"2023-09-23T20:51:01.895899+0000","last_peered":"2023-09-23T20:51:01.895899+0000","last_clean":"2023-09-23T20:51:01.895899+0000","last_became_active":"2023-09-23T20:38:05.602129+0000","last_became_peered":"2023-09-23T20:38:05.602129+0000","last_unstale":"2023-09-23T20:51:01.895899+0000","last_undegraded":"2023-09-23T20:51:01.895899+0000","last_fullsized":"2023-09-23T20:51:01.895899+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:45.343742+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:45.343742+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:48.910343+0000","scrub_duration":0.001475802,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":256,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-09-23T20:51:01.810172+0000","last_change":"2023-09-23T20:49:48.119237+0000","last_active":"2023-09-23T20:51:01.810172+0000","last_peered":"2023-09-23T20:51:01.810172+0000","last_clean":"2023-09-23T20:51:01.810172+0000","last_became_active":"2023-09-23T20:49:48.118927+0000","last_became_peered":"2023-09-23T20:49:48.118927+0000","last_unstale":"2023-09-23T20:51:01.810172+0000","last_undegraded":"2023-09-23T20:51:01.810172+0000","last_fullsized":"2023-09-23T20:51:01.810172+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:31:47.579623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,4],"acting":[6,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":296,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-09-23T20:51:01.811097+0000","last_change":"2023-09-23T20:47:09.202302+0000","last_active":"2023-09-23T20:51:01.811097+0000","last_peered":"2023-09-23T20:51:01.811097+0000","last_clean":"2023-09-23T20:51:01.811097+0000","last_became_active":"2023-09-23T20:47:09.201797+0000","last_became_peered":"2023-09-23T20:47:09.201797+0000","last_unstale":"2023-09-23T20:51:01.811097+0000","last_undegraded":"2023-09-23T20:51:01.811097+0000","last_fullsized":"2023-09-23T20:51:01.811097+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:34:41.055705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":1041,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-09-23T20:51:01.802352+0000","last_change":"2023-09-23T20:47:11.622262+0000","last_active":"2023-09-23T20:51:01.802352+0000","last_peered":"2023-09-23T20:51:01.802352+0000","last_clean":"2023-09-23T20:51:01.802352+0000","last_became_active":"2023-09-23T20:47:11.621920+0000","last_became_peered":"2023-09-23T20:47:11.621920+0000","last_unstale":"2023-09-23T20:51:01.802352+0000","last_undegraded":"2023-09-23T20:51:01.802352+0000","last_fullsized":"2023-09-23T20:51:01.802352+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:43.582146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:43.582146+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:01:15.153543+0000","scrub_duration":0.0016219730000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"664'171","reported_seq":1336,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-09-23T20:51:01.895853+0000","last_change":"2023-09-23T20:50:39.742996+0000","last_active":"2023-09-23T20:51:01.895853+0000","last_peered":"2023-09-23T20:51:01.895853+0000","last_clean":"2023-09-23T20:51:01.895853+0000","last_became_active":"2023-09-23T20:29:45.536439+0000","last_became_peered":"2023-09-23T20:29:45.536439+0000","last_unstale":"2023-09-23T20:51:01.895853+0000","last_undegraded":"2023-09-23T20:51:01.895853+0000","last_fullsized":"2023-09-23T20:51:01.895853+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":171,"log_dups_size":0,"ondisk_log_size":171,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:30:57.793132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032959200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1114,"num_read_kb":3170,"num_write":132,"num_write_kb":2190,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":685,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-09-23T20:51:02.227993+0000","last_change":"2023-09-23T20:47:09.198326+0000","last_active":"2023-09-23T20:51:02.227993+0000","last_peered":"2023-09-23T20:51:02.227993+0000","last_clean":"2023-09-23T20:51:02.227993+0000","last_became_active":"2023-09-23T20:47:09.198059+0000","last_became_peered":"2023-09-23T20:47:09.198059+0000","last_unstale":"2023-09-23T20:51:02.227993+0000","last_undegraded":"2023-09-23T20:51:02.227993+0000","last_fullsized":"2023-09-23T20:51:02.227993+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:38:53.903706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":680,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-09-23T20:51:02.228283+0000","last_change":"2023-09-23T20:48:48.146977+0000","last_active":"2023-09-23T20:51:02.228283+0000","last_peered":"2023-09-23T20:51:02.228283+0000","last_clean":"2023-09-23T20:51:02.228283+0000","last_became_active":"2023-09-23T20:48:48.146657+0000","last_became_peered":"2023-09-23T20:48:48.146657+0000","last_unstale":"2023-09-23T20:51:02.228283+0000","last_undegraded":"2023-09-23T20:51:02.228283+0000","last_fullsized":"2023-09-23T20:51:02.228283+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:38:41.782692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":782,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-09-23T20:51:01.811022+0000","last_change":"2023-09-23T20:41:08.380084+0000","last_active":"2023-09-23T20:51:01.811022+0000","last_peered":"2023-09-23T20:51:01.811022+0000","last_clean":"2023-09-23T20:51:01.811022+0000","last_became_active":"2023-09-23T20:38:05.610881+0000","last_became_peered":"2023-09-23T20:38:05.610881+0000","last_unstale":"2023-09-23T20:51:01.811022+0000","last_undegraded":"2023-09-23T20:51:01.811022+0000","last_fullsized":"2023-09-23T20:51:01.811022+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:41:08.379959+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:41:08.379959+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:19:39.547923+0000","scrub_duration":0.0019158420000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"664'175","reported_seq":1635,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-09-23T20:51:01.896333+0000","last_change":"2023-09-23T20:50:39.743889+0000","last_active":"2023-09-23T20:51:01.896333+0000","last_peered":"2023-09-23T20:51:01.896333+0000","last_clean":"2023-09-23T20:51:01.896333+0000","last_became_active":"2023-09-23T20:47:14.628341+0000","last_became_peered":"2023-09-23T20:47:14.628341+0000","last_unstale":"2023-09-23T20:51:01.896333+0000","last_undegraded":"2023-09-23T20:51:01.896333+0000","last_fullsized":"2023-09-23T20:51:01.896333+0000","mapping_epoch":558,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":559,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":175,"log_dups_size":0,"ondisk_log_size":175,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:42:16.291849+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000229686,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1422,"num_read_kb":2425,"num_write":116,"num_write_kb":664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":649,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-09-23T20:51:01.803820+0000","last_change":"2023-09-23T20:49:48.108294+0000","last_active":"2023-09-23T20:51:01.803820+0000","last_peered":"2023-09-23T20:51:01.803820+0000","last_clean":"2023-09-23T20:51:01.803820+0000","last_became_active":"2023-09-23T20:49:48.107916+0000","last_became_peered":"2023-09-23T20:49:48.107916+0000","last_unstale":"2023-09-23T20:51:01.803820+0000","last_undegraded":"2023-09-23T20:51:01.803820+0000","last_fullsized":"2023-09-23T20:51:01.803820+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:57:12.800814+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":626,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-09-23T20:51:02.228271+0000","last_change":"2023-09-23T20:49:48.119227+0000","last_active":"2023-09-23T20:51:02.228271+0000","last_peered":"2023-09-23T20:51:02.228271+0000","last_clean":"2023-09-23T20:51:02.228271+0000","last_became_active":"2023-09-23T20:49:48.118910+0000","last_became_peered":"2023-09-23T20:49:48.118910+0000","last_unstale":"2023-09-23T20:51:02.228271+0000","last_undegraded":"2023-09-23T20:51:02.228271+0000","last_fullsized":"2023-09-23T20:51:02.228271+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:18:40.836373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":708,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-09-23T20:51:01.809734+0000","last_change":"2023-09-23T20:48:51.172830+0000","last_active":"2023-09-23T20:51:01.809734+0000","last_peered":"2023-09-23T20:51:01.809734+0000","last_clean":"2023-09-23T20:51:01.809734+0000","last_became_active":"2023-09-23T20:48:51.172481+0000","last_became_peered":"2023-09-23T20:48:51.172481+0000","last_unstale":"2023-09-23T20:51:01.809734+0000","last_undegraded":"2023-09-23T20:51:01.809734+0000","last_fullsized":"2023-09-23T20:51:01.809734+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:47:34.561747+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:47:34.561747+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:30:32.531477+0000","scrub_duration":0.0087949929999999992,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":409,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-09-23T20:51:01.896536+0000","last_change":"2023-09-23T20:32:50.549695+0000","last_active":"2023-09-23T20:51:01.896536+0000","last_peered":"2023-09-23T20:51:01.896536+0000","last_clean":"2023-09-23T20:51:01.896536+0000","last_became_active":"2023-09-23T20:32:50.549431+0000","last_became_peered":"2023-09-23T20:32:50.549431+0000","last_unstale":"2023-09-23T20:51:01.896536+0000","last_undegraded":"2023-09-23T20:51:01.896536+0000","last_fullsized":"2023-09-23T20:51:01.896536+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":320,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:45:53.666569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"664'349","reported_seq":2032,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-09-23T20:51:01.803135+0000","last_change":"2023-09-23T20:50:39.737163+0000","last_active":"2023-09-23T20:51:01.803135+0000","last_peered":"2023-09-23T20:51:01.803135+0000","last_clean":"2023-09-23T20:51:01.803135+0000","last_became_active":"2023-09-23T20:47:11.621284+0000","last_became_peered":"2023-09-23T20:47:11.621284+0000","last_unstale":"2023-09-23T20:51:01.803135+0000","last_undegraded":"2023-09-23T20:51:01.803135+0000","last_fullsized":"2023-09-23T20:51:01.803135+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":349,"log_dups_size":0,"ondisk_log_size":349,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:43:50.658512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0014591350000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":2217,"num_read_kb":2864,"num_write":263,"num_write_kb":2168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":455,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-09-23T20:51:01.806854+0000","last_change":"2023-09-23T20:37:43.582105+0000","last_active":"2023-09-23T20:51:01.806854+0000","last_peered":"2023-09-23T20:51:01.806854+0000","last_clean":"2023-09-23T20:51:01.806854+0000","last_became_active":"2023-09-23T20:37:43.581860+0000","last_became_peered":"2023-09-23T20:37:43.581860+0000","last_unstale":"2023-09-23T20:51:01.806854+0000","last_undegraded":"2023-09-23T20:51:01.806854+0000","last_fullsized":"2023-09-23T20:51:01.806854+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:18:38.765486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"90'161","reported_seq":906,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-09-23T20:51:01.806612+0000","last_change":"2023-09-23T20:49:51.122217+0000","last_active":"2023-09-23T20:51:01.806612+0000","last_peered":"2023-09-23T20:51:01.806612+0000","last_clean":"2023-09-23T20:51:01.806612+0000","last_became_active":"2023-09-23T20:49:51.121893+0000","last_became_peered":"2023-09-23T20:49:51.121893+0000","last_unstale":"2023-09-23T20:51:01.806612+0000","last_undegraded":"2023-09-23T20:51:01.806612+0000","last_fullsized":"2023-09-23T20:51:01.806612+0000","mapping_epoch":639,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":640,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:34:54.504554+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:34:54.504554+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:00:40.029809+0000","scrub_duration":0.011176864,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"664'293","reported_seq":1429,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-09-23T20:51:01.811216+0000","last_change":"2023-09-23T20:50:39.740476+0000","last_active":"2023-09-23T20:51:01.811216+0000","last_peered":"2023-09-23T20:51:01.811216+0000","last_clean":"2023-09-23T20:51:01.811216+0000","last_became_active":"2023-09-23T20:48:51.217013+0000","last_became_peered":"2023-09-23T20:48:51.217013+0000","last_unstale":"2023-09-23T20:51:01.811216+0000","last_undegraded":"2023-09-23T20:51:01.811216+0000","last_fullsized":"2023-09-23T20:51:01.811216+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":293,"log_dups_size":0,"ondisk_log_size":293,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:40:42.297947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00027326800000000001,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1602,"num_read_kb":1486,"num_write":268,"num_write_kb":4284,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":48,"num_bytes_recovered":25165824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":688,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-09-23T20:51:02.228372+0000","last_change":"2023-09-23T20:47:09.317116+0000","last_active":"2023-09-23T20:51:02.228372+0000","last_peered":"2023-09-23T20:51:02.228372+0000","last_clean":"2023-09-23T20:51:02.228372+0000","last_became_active":"2023-09-23T20:47:09.205930+0000","last_became_peered":"2023-09-23T20:47:09.205930+0000","last_unstale":"2023-09-23T20:51:02.228372+0000","last_undegraded":"2023-09-23T20:51:02.228372+0000","last_fullsized":"2023-09-23T20:51:02.228372+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:04:55.528625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":138,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-09-23T20:51:01.808967+0000","last_change":"2023-09-23T20:49:48.120086+0000","last_active":"2023-09-23T20:51:01.808967+0000","last_peered":"2023-09-23T20:51:01.808967+0000","last_clean":"2023-09-23T20:51:01.808967+0000","last_became_active":"2023-09-23T20:49:48.119761+0000","last_became_peered":"2023-09-23T20:49:48.119761+0000","last_unstale":"2023-09-23T20:51:01.808967+0000","last_undegraded":"2023-09-23T20:51:01.808967+0000","last_fullsized":"2023-09-23T20:51:01.808967+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:56:12.469830+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"90'161","reported_seq":931,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-09-23T20:51:02.873243+0000","last_change":"2023-09-23T20:43:46.544750+0000","last_active":"2023-09-23T20:51:02.873243+0000","last_peered":"2023-09-23T20:51:02.873243+0000","last_clean":"2023-09-23T20:51:02.873243+0000","last_became_active":"2023-09-23T20:43:46.544395+0000","last_became_peered":"2023-09-23T20:43:46.544395+0000","last_unstale":"2023-09-23T20:51:02.873243+0000","last_undegraded":"2023-09-23T20:51:02.873243+0000","last_fullsized":"2023-09-23T20:51:02.873243+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:06.487351+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:06.487351+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:56.866168+0000","scrub_duration":0.001964379,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":646,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-09-23T20:51:02.228184+0000","last_change":"2023-09-23T20:47:09.197601+0000","last_active":"2023-09-23T20:51:02.228184+0000","last_peered":"2023-09-23T20:51:02.228184+0000","last_clean":"2023-09-23T20:51:02.228184+0000","last_became_active":"2023-09-23T20:47:09.196967+0000","last_became_peered":"2023-09-23T20:47:09.196967+0000","last_unstale":"2023-09-23T20:51:02.228184+0000","last_undegraded":"2023-09-23T20:51:02.228184+0000","last_fullsized":"2023-09-23T20:51:02.228184+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:06:56.826785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":91,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-09-23T20:51:01.804164+0000","last_change":"2023-09-23T20:48:48.158376+0000","last_active":"2023-09-23T20:51:01.804164+0000","last_peered":"2023-09-23T20:51:01.804164+0000","last_clean":"2023-09-23T20:51:01.804164+0000","last_became_active":"2023-09-23T20:48:48.158069+0000","last_became_peered":"2023-09-23T20:48:48.158069+0000","last_unstale":"2023-09-23T20:51:01.804164+0000","last_undegraded":"2023-09-23T20:51:01.804164+0000","last_fullsized":"2023-09-23T20:51:01.804164+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:34.658561+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"90'161","reported_seq":748,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-09-23T20:51:01.807033+0000","last_change":"2023-09-23T20:41:06.499511+0000","last_active":"2023-09-23T20:51:01.807033+0000","last_peered":"2023-09-23T20:51:01.807033+0000","last_clean":"2023-09-23T20:51:01.807033+0000","last_became_active":"2023-09-23T20:38:05.601874+0000","last_became_peered":"2023-09-23T20:38:05.601874+0000","last_unstale":"2023-09-23T20:51:01.807033+0000","last_undegraded":"2023-09-23T20:51:01.807033+0000","last_fullsized":"2023-09-23T20:51:01.807033+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:41:06.499396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:41:06.499396+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:31:34.059077+0000","scrub_duration":0.0019670310000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":146,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-09-23T20:51:01.809451+0000","last_change":"2023-09-23T20:49:48.128500+0000","last_active":"2023-09-23T20:51:01.809451+0000","last_peered":"2023-09-23T20:51:01.809451+0000","last_clean":"2023-09-23T20:51:01.809451+0000","last_became_active":"2023-09-23T20:49:48.122142+0000","last_became_peered":"2023-09-23T20:49:48.122142+0000","last_unstale":"2023-09-23T20:51:01.809451+0000","last_undegraded":"2023-09-23T20:51:01.809451+0000","last_fullsized":"2023-09-23T20:51:01.809451+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:46:11.349709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":128,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-09-23T20:51:01.809683+0000","last_change":"2023-09-23T20:47:09.186579+0000","last_active":"2023-09-23T20:51:01.809683+0000","last_peered":"2023-09-23T20:51:01.809683+0000","last_clean":"2023-09-23T20:51:01.809683+0000","last_became_active":"2023-09-23T20:47:09.186227+0000","last_became_peered":"2023-09-23T20:47:09.186227+0000","last_unstale":"2023-09-23T20:51:01.809683+0000","last_undegraded":"2023-09-23T20:51:01.809683+0000","last_fullsized":"2023-09-23T20:51:01.809683+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:45:39.485308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":15,"reported_epoch":672,"state":"active+clean","last_fresh":"2023-09-23T20:51:02.818541+0000","last_change":"2023-09-23T20:51:02.818293+0000","last_active":"2023-09-23T20:51:02.818541+0000","last_peered":"2023-09-23T20:51:02.818541+0000","last_clean":"2023-09-23T20:51:02.818541+0000","last_became_active":"2023-09-23T20:51:02.815733+0000","last_became_peered":"2023-09-23T20:51:02.815733+0000","last_unstale":"2023-09-23T20:51:02.818541+0000","last_undegraded":"2023-09-23T20:51:02.818541+0000","last_fullsized":"2023-09-23T20:51:02.818541+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:02:41.614582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,1],"acting":[2,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":128,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-09-23T20:51:01.809068+0000","last_change":"2023-09-23T20:47:09.181240+0000","last_active":"2023-09-23T20:51:01.809068+0000","last_peered":"2023-09-23T20:51:01.809068+0000","last_clean":"2023-09-23T20:51:01.809068+0000","last_became_active":"2023-09-23T20:47:09.181022+0000","last_became_peered":"2023-09-23T20:47:09.181022+0000","last_unstale":"2023-09-23T20:51:01.809068+0000","last_undegraded":"2023-09-23T20:51:01.809068+0000","last_fullsized":"2023-09-23T20:51:01.809068+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:38:59.191781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"90'161","reported_seq":927,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-09-23T20:51:01.896084+0000","last_change":"2023-09-23T20:38:05.591992+0000","last_active":"2023-09-23T20:51:01.896084+0000","last_peered":"2023-09-23T20:51:01.896084+0000","last_clean":"2023-09-23T20:51:01.896084+0000","last_became_active":"2023-09-23T20:38:05.591701+0000","last_became_peered":"2023-09-23T20:38:05.591701+0000","last_unstale":"2023-09-23T20:51:01.896084+0000","last_undegraded":"2023-09-23T20:51:01.896084+0000","last_fullsized":"2023-09-23T20:51:01.896084+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:02.384133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:02.384133+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:41.722542+0000","scrub_duration":0.001769098,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":515,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-09-23T20:51:01.807235+0000","last_change":"2023-09-23T20:48:48.158348+0000","last_active":"2023-09-23T20:51:01.807235+0000","last_peered":"2023-09-23T20:51:01.807235+0000","last_clean":"2023-09-23T20:51:01.807235+0000","last_became_active":"2023-09-23T20:48:48.157982+0000","last_became_peered":"2023-09-23T20:48:48.157982+0000","last_unstale":"2023-09-23T20:51:01.807235+0000","last_undegraded":"2023-09-23T20:51:01.807235+0000","last_fullsized":"2023-09-23T20:51:01.807235+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:03:48.078707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":481,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-09-23T20:51:01.806335+0000","last_change":"2023-09-23T20:48:48.159781+0000","last_active":"2023-09-23T20:51:01.806335+0000","last_peered":"2023-09-23T20:51:01.806335+0000","last_clean":"2023-09-23T20:51:01.806335+0000","last_became_active":"2023-09-23T20:48:48.159505+0000","last_became_peered":"2023-09-23T20:48:48.159505+0000","last_unstale":"2023-09-23T20:51:01.806335+0000","last_undegraded":"2023-09-23T20:51:01.806335+0000","last_fullsized":"2023-09-23T20:51:01.806335+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:25:18.100645+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"90'161","reported_seq":802,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-09-23T20:51:01.802559+0000","last_change":"2023-09-23T20:47:11.621758+0000","last_active":"2023-09-23T20:51:01.802559+0000","last_peered":"2023-09-23T20:51:01.802559+0000","last_clean":"2023-09-23T20:51:01.802559+0000","last_became_active":"2023-09-23T20:47:11.621345+0000","last_became_peered":"2023-09-23T20:47:11.621345+0000","last_unstale":"2023-09-23T20:51:01.802559+0000","last_undegraded":"2023-09-23T20:51:01.802559+0000","last_fullsized":"2023-09-23T20:51:01.802559+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:01.504973+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:01.504973+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:31:17.112393+0000","scrub_duration":0.00202344,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":752,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-09-23T20:51:02.228385+0000","last_change":"2023-09-23T20:49:48.125242+0000","last_active":"2023-09-23T20:51:02.228385+0000","last_peered":"2023-09-23T20:51:02.228385+0000","last_clean":"2023-09-23T20:51:02.228385+0000","last_became_active":"2023-09-23T20:49:48.111689+0000","last_became_peered":"2023-09-23T20:49:48.111689+0000","last_unstale":"2023-09-23T20:51:02.228385+0000","last_undegraded":"2023-09-23T20:51:02.228385+0000","last_fullsized":"2023-09-23T20:51:02.228385+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:49:38.401151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":4986,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":138,"reported_epoch":671,"state":"active+clean","last_fresh":"2023-09-23T20:51:01.808744+0000","last_change":"2023-09-23T20:48:23.917302+0000","last_active":"2023-09-23T20:51:01.808744+0000","last_peered":"2023-09-23T20:51:01.808744+0000","last_clean":"2023-09-23T20:51:01.808744+0000","last_became_active":"2023-09-23T20:48:23.916956+0000","last_became_peered":"2023-09-23T20:48:23.916956+0000","last_unstale":"2023-09-23T20:51:01.808744+0000","last_undegraded":"2023-09-23T20:51:01.808744+0000","last_fullsized":"2023-09-23T20:51:01.808744+0000","mapping_epoch":577,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":578,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:42:27.989294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":118,"num_bytes_recovered":15844,"num_keys_recovered":80,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26691,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2048000,"data_stored":2033792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":8388627,"num_objects":5,"num_object_clones":0,"num_object_copies":15,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":11981,"num_read_kb":26415,"num_write":1690,"num_write_kb":25390,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":156,"num_bytes_recovered":25165843,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":25182208,"data_stored":25165900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6082,"internal_metadata":0},"log_size":1950,"ondisk_log_size":1950,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201756,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":619892,"kb_used_data":11244,"kb_used_omap":0,"kb_used_meta":608640,"kb_avail":93125260,"statfs":{"total":95995035648,"available":95360266240,"internally_reserved":0,"allocated":11513856,"data_stored":9724040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":623247360},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:50:12 2023","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":1.325,"15min":1.9239999999999999},"min":{"1min":0.254,"5min":0.224,"15min":0.224},"max":{"1min":3.9580000000000002,"5min":10.975,"15min":18.315000000000001},"last":0.35799999999999998},{"interface":"front","average":{"1min":0.84999999999999998,"5min":1.5960000000000001,"15min":1.73},"min":{"1min":0.32100000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":4.2039999999999997,"5min":12.986000000000001,"15min":19.355},"last":0.51100000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:50:18 2023","interfaces":[{"interface":"back","average":{"1min":0.74299999999999999,"5min":1.3680000000000001,"15min":1.849},"min":{"1min":0.33200000000000002,"5min":0.25900000000000001,"15min":0.24399999999999999},"max":{"1min":3.7770000000000001,"5min":14.249000000000001,"15min":49.965000000000003},"last":0.48699999999999999},{"interface":"front","average":{"1min":0.95899999999999996,"5min":1.645,"15min":2.3290000000000002},"min":{"1min":0.27900000000000003,"5min":0.27000000000000002,"15min":0.26700000000000002},"max":{"1min":5.3390000000000004,"5min":10.861000000000001,"15min":53.451999999999998},"last":0.46600000000000003}]},{"osd":2,"last update":"Sat Sep 23 20:50:12 2023","interfaces":[{"interface":"back","average":{"1min":1.5680000000000001,"5min":1.7010000000000001,"15min":2.048},"min":{"1min":0.28799999999999998,"5min":0.28499999999999998,"15min":0.27800000000000002},"max":{"1min":10.461,"5min":12.933999999999999,"15min":16.965},"last":0.43099999999999999},{"interface":"front","average":{"1min":0.59499999999999997,"5min":1.571,"15min":2.0800000000000001},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":2.0049999999999999,"5min":7.9729999999999999,"15min":23.414000000000001},"last":0.52400000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:50:42 2023","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":1.464,"15min":1.8440000000000001},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.311},"max":{"1min":1.901,"5min":7.2880000000000003,"15min":23.352},"last":0.56100000000000005},{"interface":"front","average":{"1min":0.60199999999999998,"5min":1.635,"15min":2.0470000000000002},"min":{"1min":0.42099999999999999,"5min":0.35399999999999998,"15min":0.34300000000000003},"max":{"1min":1.8879999999999999,"5min":6.5369999999999999,"15min":20.969999999999999},"last":0.63600000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:50:42 2023","interfaces":[{"interface":"back","average":{"1min":0.44600000000000001,"5min":1.198,"15min":1.206},"min":{"1min":0.29599999999999999,"5min":0.254,"15min":0.254},"max":{"1min":0.78900000000000003,"5min":7.4109999999999996,"15min":18.364999999999998},"last":0.61099999999999999},{"interface":"front","average":{"1min":0.46999999999999997,"5min":1.456,"15min":1.49},"min":{"1min":0.33600000000000002,"5min":0.247,"15min":0.247},"max":{"1min":0.68700000000000006,"5min":27.425999999999998,"15min":27.425999999999998},"last":0.59299999999999997}]},{"osd":5,"last update":"Sat Sep 23 20:50:42 2023","interfaces":[{"interface":"back","average":{"1min":0.502,"5min":0.95399999999999996,"15min":1.248},"min":{"1min":0.316,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":1.6479999999999999,"5min":7.2270000000000003,"15min":15.005000000000001},"last":0.41099999999999998},{"interface":"front","average":{"1min":0.52200000000000002,"5min":1.0589999999999999,"15min":1.3839999999999999},"min":{"1min":0.36199999999999999,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.79800000000000004,"5min":7.335,"15min":18.390000000000001},"last":0.53700000000000003}]},{"osd":6,"last update":"Sat Sep 23 20:50:42 2023","interfaces":[{"interface":"back","average":{"1min":0.74199999999999999,"5min":1.405,"15min":1.5640000000000001},"min":{"1min":0.39200000000000002,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":4.4969999999999999,"5min":12.183,"15min":18.417000000000002},"last":0.57799999999999996},{"interface":"front","average":{"1min":0.57499999999999996,"5min":1.397,"15min":1.4019999999999999},"min":{"1min":0.38,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":1.2709999999999999,"5min":11.404999999999999,"15min":15.369999999999999},"last":0.65600000000000003}]}]},{"osd":6,"up_from":47,"seq":201863463395,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":542956,"kb_used_data":15396,"kb_used_omap":0,"kb_used_meta":527552,"kb_avail":93202196,"statfs":{"total":95995035648,"available":95439048704,"internally_reserved":0,"allocated":15765504,"data_stored":13983763,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":540213248},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:50:06 2023","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":1.821,"15min":1.7509999999999999},"min":{"1min":0.255,"5min":0.23400000000000001,"15min":0.219},"max":{"1min":2.044,"5min":21.018000000000001,"15min":21.018000000000001},"last":0.58499999999999996},{"interface":"front","average":{"1min":1.04,"5min":1.9590000000000001,"15min":2.0249999999999999},"min":{"1min":0.27200000000000002,"5min":0.255,"15min":0.22700000000000001},"max":{"1min":8.7729999999999997,"5min":21.082000000000001,"15min":60.328000000000003},"last":0.38200000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:50:23 2023","interfaces":[{"interface":"back","average":{"1min":0.64400000000000002,"5min":1.6359999999999999,"15min":1.865},"min":{"1min":0.30399999999999999,"5min":0.26000000000000001,"15min":0.249},"max":{"1min":4.266,"5min":8.9350000000000005,"15min":64.274000000000001},"last":0.33100000000000002},{"interface":"front","average":{"1min":0.52300000000000002,"5min":1.2470000000000001,"15min":1.5469999999999999},"min":{"1min":0.317,"5min":0.312,"15min":0.28000000000000003},"max":{"1min":0.85999999999999999,"5min":8.8070000000000004,"15min":17.786000000000001},"last":0.36399999999999999}]},{"osd":2,"last update":"Sat Sep 23 20:50:06 2023","interfaces":[{"interface":"back","average":{"1min":1.306,"5min":2.4049999999999998,"15min":2.1299999999999999},"min":{"1min":0.36699999999999999,"5min":0.29699999999999999,"15min":0.27700000000000002},"max":{"1min":8.9079999999999995,"5min":22.591000000000001,"15min":22.591000000000001},"last":0.41499999999999998},{"interface":"front","average":{"1min":1.4039999999999999,"5min":2.0169999999999999,"15min":1.8959999999999999},"min":{"1min":0.375,"5min":0.33900000000000002,"15min":0.30399999999999999},"max":{"1min":12.33,"5min":22.614999999999998,"15min":22.614999999999998},"last":0.54600000000000004}]},{"osd":3,"last update":"Sat Sep 23 20:50:59 2023","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":1.698,"15min":1.798},"min":{"1min":0.35099999999999998,"5min":0.307,"15min":0.28399999999999997},"max":{"1min":0.89900000000000002,"5min":9.6859999999999999,"15min":11.664},"last":0.48399999999999999},{"interface":"front","average":{"1min":0.61899999999999999,"5min":2.0409999999999999,"15min":1.9610000000000001},"min":{"1min":0.44900000000000001,"5min":0.32700000000000001,"15min":0.30499999999999999},"max":{"1min":0.96499999999999997,"5min":11.6,"15min":11.913},"last":0.46200000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:50:23 2023","interfaces":[{"interface":"back","average":{"1min":0.65100000000000002,"5min":1.0840000000000001,"15min":0.95399999999999996},"min":{"1min":0.311,"5min":0.28399999999999997,"15min":0.25600000000000001},"max":{"1min":4.2220000000000004,"5min":10.183999999999999,"15min":23.904},"last":0.443},{"interface":"front","average":{"1min":0.63900000000000001,"5min":1.5640000000000001,"15min":1.2609999999999999},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.28399999999999997},"max":{"1min":3.2919999999999998,"5min":11.999000000000001,"15min":11.999000000000001},"last":0.52800000000000002}]},{"osd":5,"last update":"Sat Sep 23 20:50:23 2023","interfaces":[{"interface":"back","average":{"1min":0.52100000000000002,"5min":1.0820000000000001,"15min":0.98999999999999999},"min":{"1min":0.34200000000000003,"5min":0.30499999999999999,"15min":0.253},"max":{"1min":0.86599999999999999,"5min":8.0440000000000005,"15min":8.0440000000000005},"last":0.51000000000000001},{"interface":"front","average":{"1min":0.72199999999999998,"5min":1.5,"15min":1.3220000000000001},"min":{"1min":0.36499999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":4.2329999999999997,"5min":9.3559999999999999,"15min":9.3559999999999999},"last":0.57399999999999995}]},{"osd":7,"last update":"Sat Sep 23 20:50:59 2023","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":1.105,"15min":0.99299999999999999},"min":{"1min":0.40200000000000002,"5min":0.35199999999999998,"15min":0.28399999999999997},"max":{"1min":0.80800000000000005,"5min":8.9309999999999992,"15min":8.9309999999999992},"last":0.56399999999999995},{"interface":"front","average":{"1min":0.623,"5min":3.2709999999999999,"15min":1.9810000000000001},"min":{"1min":0.433,"5min":0.38700000000000001,"15min":0.34699999999999998},"max":{"1min":0.86099999999999999,"5min":159.91999999999999,"15min":159.91999999999999},"last":0.66300000000000003}]}]},{"osd":1,"up_from":266,"seq":1142461300993,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":823792,"kb_used_data":12968,"kb_used_omap":8,"kb_used_meta":810807,"kb_avail":92921360,"statfs":{"total":95995035648,"available":95151472640,"internally_reserved":0,"allocated":13279232,"data_stored":9931618,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8206,"internal_metadata":830267378},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:50:29 2023","interfaces":[{"interface":"back","average":{"1min":0.48799999999999999,"5min":1.917,"15min":2.214},"min":{"1min":0.28199999999999997,"5min":0.20499999999999999,"15min":0.20000000000000001},"max":{"1min":2.8809999999999998,"5min":20.292000000000002,"15min":20.292000000000002},"last":0.318},{"interface":"front","average":{"1min":0.56799999999999995,"5min":1.8260000000000001,"15min":2.5760000000000001},"min":{"1min":0.28499999999999998,"5min":0.217,"15min":0.19900000000000001},"max":{"1min":2.6890000000000001,"5min":17.463000000000001,"15min":20.346},"last":0.45400000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:50:29 2023","interfaces":[{"interface":"back","average":{"1min":0.76900000000000002,"5min":2.0630000000000002,"15min":2.6110000000000002},"min":{"1min":0.25900000000000001,"5min":0.23100000000000001,"15min":0.216},"max":{"1min":2.7970000000000002,"5min":23.341000000000001,"15min":32.255000000000003},"last":0.38200000000000001},{"interface":"front","average":{"1min":0.61099999999999999,"5min":2.173,"15min":2.1280000000000001},"min":{"1min":0.34899999999999998,"5min":0.249,"15min":0.20899999999999999},"max":{"1min":2.9279999999999999,"5min":34.002000000000002,"15min":34.002000000000002},"last":0.47499999999999998}]},{"osd":3,"last update":"Sat Sep 23 20:50:29 2023","interfaces":[{"interface":"back","average":{"1min":0.63600000000000001,"5min":1.621,"15min":2.0979999999999999},"min":{"1min":0.316,"5min":0.28799999999999998,"15min":0.247},"max":{"1min":2.9590000000000001,"5min":17.366,"15min":17.366},"last":0.42799999999999999},{"interface":"front","average":{"1min":0.748,"5min":2,"15min":2.3839999999999999},"min":{"1min":0.35099999999999998,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":2.7330000000000001,"5min":17.343,"15min":19.315000000000001},"last":0.40200000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:50:29 2023","interfaces":[{"interface":"back","average":{"1min":1.079,"5min":1.8839999999999999,"15min":2.2229999999999999},"min":{"1min":0.42699999999999999,"5min":0.32600000000000001,"15min":0.28599999999999998},"max":{"1min":5.7110000000000003,"5min":9.5359999999999996,"15min":20.634},"last":0.68300000000000005},{"interface":"front","average":{"1min":0.69299999999999995,"5min":1.5600000000000001,"15min":1.7190000000000001},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.311},"max":{"1min":2.8370000000000002,"5min":9.8249999999999993,"15min":20.835999999999999},"last":0.63400000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:50:29 2023","interfaces":[{"interface":"back","average":{"1min":0.755,"5min":1.819,"15min":2.2090000000000001},"min":{"1min":0.42199999999999999,"5min":0.34699999999999998,"15min":0.308},"max":{"1min":3.1120000000000001,"5min":17.396999999999998,"15min":33.401000000000003},"last":0.70199999999999996},{"interface":"front","average":{"1min":0.93999999999999995,"5min":1.962,"15min":2.4279999999999999},"min":{"1min":0.371,"5min":0.371,"15min":0.32500000000000001},"max":{"1min":3.0670000000000002,"5min":16.366,"15min":33.417000000000002},"last":0.76700000000000002}]},{"osd":6,"last update":"Sat Sep 23 20:50:29 2023","interfaces":[{"interface":"back","average":{"1min":0.80400000000000005,"5min":2.105,"15min":2.1960000000000002},"min":{"1min":0.46899999999999997,"5min":0.316,"15min":0.316},"max":{"1min":3.0390000000000001,"5min":23.274000000000001,"15min":23.274000000000001},"last":0.65300000000000002},{"interface":"front","average":{"1min":1.0600000000000001,"5min":2.1440000000000001,"15min":2.4279999999999999},"min":{"1min":0.48399999999999999,"5min":0.40500000000000003,"15min":0.317},"max":{"1min":4.4690000000000003,"5min":9.6639999999999997,"15min":11.832000000000001},"last":0.72699999999999998}]},{"osd":7,"last update":"Sat Sep 23 20:50:29 2023","interfaces":[{"interface":"back","average":{"1min":0.79400000000000004,"5min":1.9730000000000001,"15min":2.2389999999999999},"min":{"1min":0.51100000000000001,"5min":0.377,"15min":0.37},"max":{"1min":3.0819999999999999,"5min":17.433,"15min":18.437000000000001},"last":0.74399999999999999},{"interface":"front","average":{"1min":0.98299999999999998,"5min":2.1419999999999999,"15min":2.472},"min":{"1min":0.47699999999999998,"5min":0.41899999999999998,"15min":0.35999999999999999},"max":{"1min":3.0259999999999998,"5min":9.8369999999999997,"15min":12.021000000000001},"last":0.78800000000000003}]}]},{"osd":0,"up_from":94,"seq":403726926197,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":990020,"kb_used_data":12860,"kb_used_omap":7,"kb_used_meta":977144,"kb_avail":92755132,"statfs":{"total":95995035648,"available":94981255168,"internally_reserved":0,"allocated":13168640,"data_stored":9977890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":1000595473},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:50:33 2023","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":3.1699999999999999,"15min":2.4630000000000001},"min":{"1min":0.25600000000000001,"5min":0.20100000000000001,"15min":0.17199999999999999},"max":{"1min":2.4089999999999998,"5min":152.387,"15min":152.387},"last":0.23100000000000001},{"interface":"front","average":{"1min":0.52100000000000002,"5min":3.1629999999999998,"15min":2.633},"min":{"1min":0.32300000000000001,"5min":0.218,"15min":0.20399999999999999},"max":{"1min":1.4259999999999999,"5min":152.84100000000001,"15min":152.84100000000001},"last":0.33000000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:50:11 2023","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":2.2490000000000001,"15min":2.6869999999999998},"min":{"1min":0.26700000000000002,"5min":0.23499999999999999,"15min":0.218},"max":{"1min":1.9410000000000001,"5min":14.538,"15min":62.726999999999997},"last":0.26000000000000001},{"interface":"front","average":{"1min":1.1419999999999999,"5min":2.226,"15min":2.6600000000000001},"min":{"1min":0.26100000000000001,"5min":0.217,"15min":0.217},"max":{"1min":10.159000000000001,"5min":14.167,"15min":62.689},"last":0.35099999999999998}]},{"osd":3,"last update":"Sat Sep 23 20:50:46 2023","interfaces":[{"interface":"back","average":{"1min":0.442,"5min":2.6070000000000002,"15min":2.718},"min":{"1min":0.29299999999999998,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.97899999999999998,"5min":55.017000000000003,"15min":55.017000000000003},"last":0.311},{"interface":"front","average":{"1min":0.69099999999999995,"5min":2.673,"15min":2.6680000000000001},"min":{"1min":0.372,"5min":0.27400000000000002,"15min":0.21199999999999999},"max":{"1min":2.9140000000000001,"5min":55.665999999999997,"15min":55.665999999999997},"last":0.38200000000000001}]},{"osd":4,"last update":"Sat Sep 23 20:50:11 2023","interfaces":[{"interface":"back","average":{"1min":0.99199999999999999,"5min":1.964,"15min":2.008},"min":{"1min":0.34300000000000003,"5min":0.29299999999999998,"15min":0.28899999999999998},"max":{"1min":4.2859999999999996,"5min":11.75,"15min":11.75},"last":0.372},{"interface":"front","average":{"1min":1.2549999999999999,"5min":2.0230000000000001,"15min":1.853},"min":{"1min":0.42099999999999999,"5min":0.31,"15min":0.27400000000000002},"max":{"1min":5.4279999999999999,"5min":12.356999999999999,"15min":12.356999999999999},"last":0.46899999999999997}]},{"osd":5,"last update":"Sat Sep 23 20:50:11 2023","interfaces":[{"interface":"back","average":{"1min":0.83999999999999997,"5min":1.8029999999999999,"15min":1.9399999999999999},"min":{"1min":0.38500000000000001,"5min":0.308,"15min":0.26500000000000001},"max":{"1min":3.012,"5min":9.3339999999999996,"15min":11.497999999999999},"last":0.42499999999999999},{"interface":"front","average":{"1min":0.88600000000000001,"5min":1.7210000000000001,"15min":2.2189999999999999},"min":{"1min":0.33000000000000002,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":3.181,"5min":10.089,"15min":26.228000000000002},"last":0.51500000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:50:11 2023","interfaces":[{"interface":"back","average":{"1min":1.395,"5min":2.1669999999999998,"15min":2.1360000000000001},"min":{"1min":0.441,"5min":0.374,"15min":0.34499999999999997},"max":{"1min":4.3540000000000001,"5min":9.2750000000000004,"15min":24.056000000000001},"last":0.40600000000000003},{"interface":"front","average":{"1min":1.391,"5min":1.974,"15min":2.1579999999999999},"min":{"1min":0.372,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":4.1669999999999998,"5min":12.427,"15min":26.268000000000001},"last":0.48799999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:50:11 2023","interfaces":[{"interface":"back","average":{"1min":1.0900000000000001,"5min":2.1320000000000001,"15min":2.3199999999999998},"min":{"1min":0.39100000000000001,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":3.8919999999999999,"5min":16.553999999999998,"15min":26.177},"last":0.45600000000000002},{"interface":"front","average":{"1min":1.1579999999999999,"5min":2.1899999999999999,"15min":2.3300000000000001},"min":{"1min":0.45700000000000002,"5min":0.40200000000000002,"15min":0.34200000000000003},"max":{"1min":4.3209999999999997,"5min":16.649000000000001,"15min":16.649000000000001},"last":0.39200000000000002}]}]},{"osd":2,"up_from":104,"seq":446676599144,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":888768,"kb_used_data":12728,"kb_used_omap":11,"kb_used_meta":876020,"kb_avail":92856384,"statfs":{"total":95995035648,"available":95084937216,"internally_reserved":0,"allocated":13033472,"data_stored":9900114,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11878,"internal_metadata":897044890},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:50:12 2023","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":1.5900000000000001,"15min":1.603},"min":{"1min":0.26000000000000001,"5min":0.19500000000000001,"15min":0.17299999999999999},"max":{"1min":2.669,"5min":20.48,"15min":20.48},"last":0.38200000000000001},{"interface":"front","average":{"1min":0.51600000000000001,"5min":1.423,"15min":1.982},"min":{"1min":0.31900000000000001,"5min":0.245,"15min":0.19},"max":{"1min":1.4279999999999999,"5min":15.342000000000001,"15min":16.678999999999998},"last":0.58099999999999996}]},{"osd":1,"last update":"Sat Sep 23 20:50:12 2023","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":1.5600000000000001,"15min":2.2429999999999999},"min":{"1min":0.24299999999999999,"5min":0.219,"15min":0.219},"max":{"1min":1.1279999999999999,"5min":21.690000000000001,"15min":46.405000000000001},"last":0.36199999999999999},{"interface":"front","average":{"1min":0.496,"5min":1.54,"15min":2.1930000000000001},"min":{"1min":0.26800000000000002,"5min":0.22,"15min":0.22},"max":{"1min":1.0449999999999999,"5min":10.191000000000001,"15min":46.459000000000003},"last":0.48599999999999999}]},{"osd":3,"last update":"Sat Sep 23 20:50:42 2023","interfaces":[{"interface":"back","average":{"1min":0.40500000000000003,"5min":1.8799999999999999,"15min":2.2090000000000001},"min":{"1min":0.30399999999999999,"5min":0.28199999999999997,"15min":0.253},"max":{"1min":0.51300000000000001,"5min":30.283000000000001,"15min":30.283000000000001},"last":0.46300000000000002},{"interface":"front","average":{"1min":0.45000000000000001,"5min":2.0070000000000001,"15min":1.6379999999999999},"min":{"1min":0.29399999999999998,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.64100000000000001,"5min":30.834,"15min":30.834},"last":0.41999999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:50:12 2023","interfaces":[{"interface":"back","average":{"1min":0.56799999999999995,"5min":1.5449999999999999,"15min":1.736},"min":{"1min":0.38400000000000001,"5min":0.29599999999999999,"15min":0.27500000000000002},"max":{"1min":0.79500000000000004,"5min":8.3469999999999995,"15min":12.025},"last":0.65400000000000003},{"interface":"front","average":{"1min":0.67400000000000004,"5min":1.627,"15min":1.819},"min":{"1min":0.42099999999999999,"5min":0.32400000000000001,"15min":0.30599999999999999},"max":{"1min":1.8520000000000001,"5min":12.662000000000001,"15min":19.440999999999999},"last":0.70899999999999996}]},{"osd":5,"last update":"Sat Sep 23 20:50:12 2023","interfaces":[{"interface":"back","average":{"1min":0.90100000000000002,"5min":1.3660000000000001,"15min":1.76},"min":{"1min":0.44800000000000001,"5min":0.33200000000000002,"15min":0.311},"max":{"1min":4.8570000000000002,"5min":8.5380000000000003,"15min":12.07},"last":0.68999999999999995},{"interface":"front","average":{"1min":0.93300000000000005,"5min":1.5349999999999999,"15min":1.7969999999999999},"min":{"1min":0.437,"5min":0.373,"15min":0.35299999999999998},"max":{"1min":4.8639999999999999,"5min":10.831,"15min":19.513000000000002},"last":0.754}]},{"osd":6,"last update":"Sat Sep 23 20:50:12 2023","interfaces":[{"interface":"back","average":{"1min":0.63900000000000001,"5min":1.6619999999999999,"15min":1.9179999999999999},"min":{"1min":0.40200000000000002,"5min":0.34799999999999998,"15min":0.33500000000000002},"max":{"1min":1.177,"5min":10.791,"15min":19.486000000000001},"last":0.51700000000000002},{"interface":"front","average":{"1min":0.80400000000000005,"5min":1.486,"15min":1.415},"min":{"1min":0.433,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":2.6440000000000001,"5min":7.4180000000000001,"15min":11.747},"last":0.56100000000000005}]},{"osd":7,"last update":"Sat Sep 23 20:50:12 2023","interfaces":[{"interface":"back","average":{"1min":0.80100000000000005,"5min":1.8700000000000001,"15min":1.5229999999999999},"min":{"1min":0.52100000000000002,"5min":0.35399999999999998,"15min":0.34699999999999998},"max":{"1min":2.581,"5min":20.530000000000001,"15min":20.530000000000001},"last":0.63},{"interface":"front","average":{"1min":0.67800000000000005,"5min":1.71,"15min":1.9830000000000001},"min":{"1min":0.45800000000000002,"5min":0.36899999999999999,"15min":0.34899999999999998},"max":{"1min":1.141,"5min":10.071999999999999,"15min":12.099},"last":0.78000000000000003}]}]},{"osd":3,"up_from":109,"seq":468151435617,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1081336,"kb_used_data":8688,"kb_used_omap":4,"kb_used_meta":1072635,"kb_avail":92663816,"statfs":{"total":95995035648,"available":94887747584,"internally_reserved":0,"allocated":8896512,"data_stored":5770168,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4514,"internal_metadata":1098378846},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:50:51 2023","interfaces":[{"interface":"back","average":{"1min":0.42499999999999999,"5min":1.8380000000000001,"15min":1.9970000000000001},"min":{"1min":0.26000000000000001,"5min":0.217,"15min":0.20899999999999999},"max":{"1min":1.071,"5min":21.792999999999999,"15min":22.32},"last":0.30199999999999999},{"interface":"front","average":{"1min":0.45800000000000002,"5min":1.6220000000000001,"15min":2.073},"min":{"1min":0.32700000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":1.1020000000000001,"5min":18.579999999999998,"15min":22.805},"last":0.38800000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:50:23 2023","interfaces":[{"interface":"back","average":{"1min":0.83899999999999997,"5min":1.321,"15min":1.7689999999999999},"min":{"1min":0.22700000000000001,"5min":0.21299999999999999,"15min":0.21099999999999999},"max":{"1min":10.24,"5min":11.298,"15min":17.529},"last":0.48399999999999999},{"interface":"front","average":{"1min":0.59499999999999997,"5min":1.454,"15min":2.036},"min":{"1min":0.26500000000000001,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":4.2480000000000002,"5min":11.380000000000001,"15min":14.19},"last":0.46800000000000003}]},{"osd":2,"last update":"Sat Sep 23 20:50:51 2023","interfaces":[{"interface":"back","average":{"1min":0.42599999999999999,"5min":1.5900000000000001,"15min":2.3820000000000001},"min":{"1min":0.28000000000000003,"5min":0.254,"15min":0.23499999999999999},"max":{"1min":1.194,"5min":7.9210000000000003,"15min":68.206999999999994},"last":0.34399999999999997},{"interface":"front","average":{"1min":0.42999999999999999,"5min":1.4159999999999999,"15min":2.121},"min":{"1min":0.314,"5min":0.25600000000000001,"15min":0.245},"max":{"1min":0.73899999999999999,"5min":16.559000000000001,"15min":67.736999999999995},"last":0.35799999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:50:51 2023","interfaces":[{"interface":"back","average":{"1min":0.84599999999999997,"5min":1.458,"15min":1.5629999999999999},"min":{"1min":0.38300000000000001,"5min":0.35399999999999998,"15min":0.30499999999999999},"max":{"1min":6.4059999999999997,"5min":9.0259999999999998,"15min":22.157},"last":0.42699999999999999},{"interface":"front","average":{"1min":0.54600000000000004,"5min":1.603,"15min":1.9219999999999999},"min":{"1min":0.35099999999999998,"5min":0.33500000000000002,"15min":0.33000000000000002},"max":{"1min":0.81399999999999995,"5min":16.529,"15min":16.529},"last":0.44500000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:50:51 2023","interfaces":[{"interface":"back","average":{"1min":0.88100000000000001,"5min":1.575,"15min":1.7150000000000001},"min":{"1min":0.41699999999999998,"5min":0.32400000000000001,"15min":0.32300000000000001},"max":{"1min":6.4800000000000004,"5min":8.9849999999999994,"15min":22.289999999999999},"last":0.54900000000000004},{"interface":"front","average":{"1min":0.60399999999999998,"5min":1.4159999999999999,"15min":1.6479999999999999},"min":{"1min":0.42899999999999999,"5min":0.36399999999999999,"15min":0.34100000000000003},"max":{"1min":1.135,"5min":6.1390000000000002,"15min":12.952},"last":0.59099999999999997}]},{"osd":6,"last update":"Sat Sep 23 20:50:51 2023","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":1.9430000000000001,"15min":1.8660000000000001},"min":{"1min":0.42799999999999999,"5min":0.39800000000000002,"15min":0.34300000000000003},"max":{"1min":1.163,"5min":8.6419999999999995,"15min":12.917},"last":0.63100000000000001},{"interface":"front","average":{"1min":0.57099999999999995,"5min":1.647,"15min":1.9750000000000001},"min":{"1min":0.47199999999999998,"5min":0.36699999999999999,"15min":0.34599999999999997},"max":{"1min":0.69999999999999996,"5min":16.497,"15min":16.497},"last":0.53900000000000003}]},{"osd":7,"last update":"Sat Sep 23 20:50:51 2023","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":1.6140000000000001,"15min":2.1949999999999998},"min":{"1min":0.44600000000000001,"5min":0.42399999999999999,"15min":0.39900000000000002},"max":{"1min":0.77000000000000002,"5min":16.425000000000001,"15min":16.425000000000001},"last":0.502},{"interface":"front","average":{"1min":0.89600000000000002,"5min":1.9039999999999999,"15min":1.8879999999999999},"min":{"1min":0.501,"5min":0.45600000000000002,"15min":0.40999999999999998},"max":{"1min":6.5259999999999998,"5min":11.891999999999999,"15min":22.202999999999999},"last":0.60899999999999999}]}]},{"osd":4,"up_from":30,"seq":128849019371,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":572008,"kb_used_data":6880,"kb_used_omap":0,"kb_used_meta":565120,"kb_avail":93173144,"statfs":{"total":95995035648,"available":95409299456,"internally_reserved":0,"allocated":7045120,"data_stored":5266655,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":578682880},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:50:07 2023","interfaces":[{"interface":"back","average":{"1min":0.63200000000000001,"5min":2.1230000000000002,"15min":1.974},"min":{"1min":0.217,"5min":0.217,"15min":0.217},"max":{"1min":3.351,"5min":73.272000000000006,"15min":73.272000000000006},"last":0.61599999999999999},{"interface":"front","average":{"1min":0.85899999999999999,"5min":1.8919999999999999,"15min":2.0350000000000001},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.246},"max":{"1min":7.2359999999999998,"5min":29.754000000000001,"15min":29.754000000000001},"last":0.67500000000000004}]},{"osd":1,"last update":"Sat Sep 23 20:50:17 2023","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":1.544,"15min":1.7050000000000001},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.25900000000000001},"max":{"1min":1.9259999999999999,"5min":12.343,"15min":18.190999999999999},"last":0.49199999999999999},{"interface":"front","average":{"1min":0.74299999999999999,"5min":1.734,"15min":1.6459999999999999},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.26400000000000001},"max":{"1min":3.198,"5min":12.231999999999999,"15min":18.173999999999999},"last":0.51700000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:50:17 2023","interfaces":[{"interface":"back","average":{"1min":0.85599999999999998,"5min":1.7470000000000001,"15min":1.72},"min":{"1min":0.28000000000000003,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":3.2469999999999999,"5min":12.151999999999999,"15min":13.051},"last":0.433},{"interface":"front","average":{"1min":0.72299999999999998,"5min":1.6899999999999999,"15min":1.986},"min":{"1min":0.28899999999999998,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":3.4319999999999999,"5min":12.458,"15min":12.458},"last":0.505}]},{"osd":3,"last update":"Sat Sep 23 20:50:36 2023","interfaces":[{"interface":"back","average":{"1min":0.79700000000000004,"5min":1.796,"15min":1.8720000000000001},"min":{"1min":0.376,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":3.0840000000000001,"5min":11.598000000000001,"15min":12.147},"last":0.54800000000000004},{"interface":"front","average":{"1min":0.63900000000000001,"5min":1.9179999999999999,"15min":1.978},"min":{"1min":0.45600000000000002,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":1.323,"5min":12.191000000000001,"15min":17.169},"last":0.47699999999999998}]},{"osd":5,"last update":"Sat Sep 23 20:50:07 2023","interfaces":[{"interface":"back","average":{"1min":0.49399999999999999,"5min":1.393,"15min":1.1919999999999999},"min":{"1min":0.26100000000000001,"5min":0.25800000000000001,"15min":0.253},"max":{"1min":1.214,"5min":16.940000000000001,"15min":16.940000000000001},"last":0.40699999999999997},{"interface":"front","average":{"1min":0.53600000000000003,"5min":1.264,"15min":1.1819999999999999},"min":{"1min":0.26700000000000002,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":1.9610000000000001,"5min":6.5839999999999996,"15min":8.1440000000000001},"last":0.36199999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:50:17 2023","interfaces":[{"interface":"back","average":{"1min":1.22,"5min":1.716,"15min":1.452},"min":{"1min":0.31900000000000001,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":7.4589999999999996,"5min":13.503,"15min":14.731999999999999},"last":0.41999999999999998},{"interface":"front","average":{"1min":0.72399999999999998,"5min":1.8080000000000001,"15min":1.4039999999999999},"min":{"1min":0.29899999999999999,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":4.0830000000000002,"5min":17.059000000000001,"15min":17.059000000000001},"last":0.376}]},{"osd":7,"last update":"Sat Sep 23 20:50:36 2023","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":1.4750000000000001,"15min":2.661},"min":{"1min":0.40200000000000002,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":1.012,"5min":11.228,"15min":313.50099999999998},"last":0.46300000000000002},{"interface":"front","average":{"1min":0.57899999999999996,"5min":1.5,"15min":2.3290000000000002},"min":{"1min":0.438,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.94799999999999995,"5min":17.093,"15min":312.67500000000001},"last":0.44900000000000001}]}]},{"osd":5,"up_from":0,"seq":167503725032,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4722,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1360,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":8392704,"data_stored":8388627,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8206,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7156,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3154,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:51:10.628 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-09-23T20:51:10.629 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd pool set .mgr pg_num 91 2023-09-23T20:51:11.240 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:51:11.295 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:51:11.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:11 smithi018 ceph-mon[179647]: from='client.47900 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:51:11.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:11 smithi018 ceph-mon[184144]: from='client.47900 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:51:11.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:11 smithi149 ceph-mon[170394]: from='client.47900 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:51:12.274 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:12 smithi018 ceph-mon[184144]: pgmap v1640: 103 pgs: 103 active+clean; 8.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail 2023-09-23T20:51:12.274 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:12 smithi018 ceph-mon[179647]: pgmap v1640: 103 pgs: 103 active+clean; 8.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail 2023-09-23T20:51:12.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:12 smithi149 ceph-mon[170394]: pgmap v1640: 103 pgs: 103 active+clean; 8.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail 2023-09-23T20:51:12.760 INFO:teuthology.orchestra.run.smithi149.stdout:{ 2023-09-23T20:51:12.760 INFO:teuthology.orchestra.run.smithi149.stdout: "ops": [], 2023-09-23T20:51:12.760 INFO:teuthology.orchestra.run.smithi149.stdout: "complaint_time": 30, 2023-09-23T20:51:12.761 INFO:teuthology.orchestra.run.smithi149.stdout: "num_blocked_ops": 0 2023-09-23T20:51:12.761 INFO:teuthology.orchestra.run.smithi149.stdout:} 2023-09-23T20:51:12.945 INFO:teuthology.orchestra.run.smithi018.stderr:Error EINVAL: specified pg_num 91 < pg_num_max 32 2023-09-23T20:51:13.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:13 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2486492296' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "91"}]: dispatch 2023-09-23T20:51:13.334 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:13 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2486492296' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "91"}]: dispatch 2023-09-23T20:51:13.334 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:13 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2486492296' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "91"}]: dispatch 2023-09-23T20:51:13.358 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T20:51:13.365 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-09-23T20:51:14.133 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:51:14.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:14 smithi018 ceph-mon[184144]: pgmap v1641: 103 pgs: 103 active+clean; 8.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail 2023-09-23T20:51:14.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:14 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:51:14.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:14 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:51:14.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:14 smithi018 ceph-mon[179647]: pgmap v1641: 103 pgs: 103 active+clean; 8.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail 2023-09-23T20:51:14.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:14 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:51:14.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:14 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:51:14.408 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:14 smithi149 ceph-mon[170394]: pgmap v1641: 103 pgs: 103 active+clean; 8.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail 2023-09-23T20:51:14.408 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:14 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:51:14.409 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:14 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:51:15.391 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:51:14 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:51:14] "GET /metrics HTTP/1.1" 200 34154 "" "Prometheus/2.43.0" 2023-09-23T20:51:15.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:15 smithi018 ceph-mon[184144]: pgmap v1642: 103 pgs: 103 active+clean; 8.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail 2023-09-23T20:51:15.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:15 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:51:15.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:15 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:51:15.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:15 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:51:15.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:15 smithi018 ceph-mon[179647]: pgmap v1642: 103 pgs: 103 active+clean; 8.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail 2023-09-23T20:51:15.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:15 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:51:15.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:15 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:51:15.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:15 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:51:15.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:15 smithi149 ceph-mon[170394]: pgmap v1642: 103 pgs: 103 active+clean; 8.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail 2023-09-23T20:51:15.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:15 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:51:15.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:15 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:51:15.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:15 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:51:15.483 INFO:teuthology.orchestra.run.smithi149.stdout:{ 2023-09-23T20:51:15.483 INFO:teuthology.orchestra.run.smithi149.stdout: "size": 20, 2023-09-23T20:51:15.483 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 600, 2023-09-23T20:51:15.483 INFO:teuthology.orchestra.run.smithi149.stdout: "ops": [ 2023-09-23T20:51:15.483 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.483 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:29549 9.7 9:ea1cf7ea:::benchmark_data_smithi018_381516_object29548:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e459)", 2023-09-23T20:51:15.483 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:41:21.052150+0000", 2023-09-23T20:51:15.483 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 594.42816044100005, 2023-09-23T20:51:15.484 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.14682632800000001, 2023-09-23T20:51:15.484 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:51:15.484 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:15.484 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:51:15.484 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:51:15.484 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:51:15.484 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 29549 2023-09-23T20:51:15.484 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.484 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:51:15.485 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.485 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:51:15.485 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.052150+0000", 2023-09-23T20:51:15.485 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:15.485 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.485 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.485 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:51:15.485 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.052150+0000", 2023-09-23T20:51:15.485 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:15.485 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.485 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.486 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:51:15.486 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.052149+0000", 2023-09-23T20:51:15.486 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:51:15.486 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.486 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.486 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:51:15.486 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.052237+0000", 2023-09-23T20:51:15.486 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.7569000000000001e-05 2023-09-23T20:51:15.487 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.487 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.487 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:51:15.487 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.052246+0000", 2023-09-23T20:51:15.487 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.5289999999999997e-06 2023-09-23T20:51:15.487 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.487 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.487 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:51:15.487 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.052252+0000", 2023-09-23T20:51:15.487 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.5450000000000003e-06 2023-09-23T20:51:15.487 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.488 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.488 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:15.488 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.052294+0000", 2023-09-23T20:51:15.488 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.2228999999999998e-05 2023-09-23T20:51:15.488 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.488 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.488 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:51:15.488 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.052411+0000", 2023-09-23T20:51:15.488 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00011702899999999999 2023-09-23T20:51:15.488 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.488 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.489 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 3,7", 2023-09-23T20:51:15.489 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.052532+0000", 2023-09-23T20:51:15.490 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00012102799999999999 2023-09-23T20:51:15.490 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.490 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.490 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:51:15.490 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.055293+0000", 2023-09-23T20:51:15.490 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.002760424 2023-09-23T20:51:15.490 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.491 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.491 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:15.491 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.056788+0000", 2023-09-23T20:51:15.491 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00149522 2023-09-23T20:51:15.491 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.491 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.491 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:15.491 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.198887+0000", 2023-09-23T20:51:15.491 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.142099535 2023-09-23T20:51:15.491 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.491 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.492 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:51:15.492 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.198937+0000", 2023-09-23T20:51:15.492 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.9252999999999998e-05 2023-09-23T20:51:15.492 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.492 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.492 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:51:15.492 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.198977+0000", 2023-09-23T20:51:15.492 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.0151000000000002e-05 2023-09-23T20:51:15.492 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:15.492 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:51:15.492 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:15.492 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.493 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.493 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:29552 9.f 9:fef69f4f:::benchmark_data_smithi018_381516_object29551:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e459)", 2023-09-23T20:51:15.493 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:41:21.053398+0000", 2023-09-23T20:51:15.493 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 594.42691246100003, 2023-09-23T20:51:15.493 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.14557930599999999, 2023-09-23T20:51:15.493 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:51:15.493 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:15.493 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:51:15.493 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:51:15.493 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:51:15.493 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 29552 2023-09-23T20:51:15.493 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.494 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:51:15.494 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.494 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:51:15.494 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.053398+0000", 2023-09-23T20:51:15.494 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:15.494 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.494 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.494 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:51:15.494 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.053398+0000", 2023-09-23T20:51:15.494 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:15.494 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.494 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.495 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:51:15.495 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.053398+0000", 2023-09-23T20:51:15.495 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:51:15.495 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.495 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.495 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:51:15.495 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.053439+0000", 2023-09-23T20:51:15.495 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.1603000000000001e-05 2023-09-23T20:51:15.495 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.495 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.495 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:51:15.495 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.053453+0000", 2023-09-23T20:51:15.496 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.3842000000000001e-05 2023-09-23T20:51:15.496 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.496 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.496 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:51:15.496 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.053462+0000", 2023-09-23T20:51:15.496 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.9190000000000005e-06 2023-09-23T20:51:15.496 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.496 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.496 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:15.496 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.053503+0000", 2023-09-23T20:51:15.496 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.1069000000000003e-05 2023-09-23T20:51:15.498 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.498 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.498 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:51:15.498 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.053629+0000", 2023-09-23T20:51:15.498 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00012554299999999999 2023-09-23T20:51:15.499 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.499 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.499 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:51:15.499 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.053773+0000", 2023-09-23T20:51:15.499 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000144427 2023-09-23T20:51:15.499 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.499 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.499 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:51:15.499 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.056678+0000", 2023-09-23T20:51:15.499 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0029051200000000002 2023-09-23T20:51:15.499 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.500 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.500 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:15.500 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.060074+0000", 2023-09-23T20:51:15.500 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0033957610000000002 2023-09-23T20:51:15.500 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.500 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.500 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:15.500 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.198920+0000", 2023-09-23T20:51:15.500 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.13884589899999999 2023-09-23T20:51:15.500 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.500 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.500 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:51:15.501 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.198949+0000", 2023-09-23T20:51:15.501 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.9641e-05 2023-09-23T20:51:15.501 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.501 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.501 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:51:15.501 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.198978+0000", 2023-09-23T20:51:15.501 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.8285000000000001e-05 2023-09-23T20:51:15.501 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:15.501 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:51:15.501 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:15.501 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.502 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.502 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:29557 9.7 9:e1a9e246:::benchmark_data_smithi018_381516_object29556:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e459)", 2023-09-23T20:51:15.502 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:41:21.057117+0000", 2023-09-23T20:51:15.502 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 594.42319398799998, 2023-09-23T20:51:15.502 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.14586998200000001, 2023-09-23T20:51:15.502 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:51:15.502 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:15.502 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:51:15.502 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:51:15.502 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:51:15.502 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 29557 2023-09-23T20:51:15.503 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.503 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:51:15.503 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.503 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:51:15.503 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.057117+0000", 2023-09-23T20:51:15.503 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:15.503 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.503 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.503 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:51:15.503 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.057117+0000", 2023-09-23T20:51:15.503 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:15.504 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.504 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.504 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:51:15.504 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.057115+0000", 2023-09-23T20:51:15.504 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:51:15.504 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.504 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.504 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:51:15.504 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.057187+0000", 2023-09-23T20:51:15.504 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.1619999999999995e-05 2023-09-23T20:51:15.504 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.505 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.505 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:51:15.505 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.057200+0000", 2023-09-23T20:51:15.505 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.3261000000000001e-05 2023-09-23T20:51:15.505 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.505 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.505 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:51:15.505 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.057207+0000", 2023-09-23T20:51:15.505 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.6309999999999997e-06 2023-09-23T20:51:15.505 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.505 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.506 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:15.506 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.057249+0000", 2023-09-23T20:51:15.506 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.1993999999999998e-05 2023-09-23T20:51:15.506 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.506 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.506 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:51:15.506 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.057363+0000", 2023-09-23T20:51:15.506 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000113553 2023-09-23T20:51:15.506 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.506 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.506 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 3,7", 2023-09-23T20:51:15.506 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.057458+0000", 2023-09-23T20:51:15.507 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.5340000000000005e-05 2023-09-23T20:51:15.507 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.507 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.507 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:51:15.507 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.058584+0000", 2023-09-23T20:51:15.507 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0011261489999999999 2023-09-23T20:51:15.507 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.507 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.507 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:15.507 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.061821+0000", 2023-09-23T20:51:15.507 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0032369619999999999 2023-09-23T20:51:15.508 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.508 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.508 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:15.508 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.202885+0000", 2023-09-23T20:51:15.508 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.14106375800000001 2023-09-23T20:51:15.508 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.508 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.508 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:51:15.508 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.202939+0000", 2023-09-23T20:51:15.508 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.4626000000000002e-05 2023-09-23T20:51:15.508 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.508 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.509 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:51:15.509 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.202987+0000", 2023-09-23T20:51:15.509 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.7528000000000003e-05 2023-09-23T20:51:15.509 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:15.509 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:51:15.509 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:15.509 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.509 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.509 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:29569 9.7 9:edf50775:::benchmark_data_smithi018_381516_object29568:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e459)", 2023-09-23T20:51:15.509 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:41:21.063339+0000", 2023-09-23T20:51:15.509 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 594.41697165999994, 2023-09-23T20:51:15.509 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.13984607700000001, 2023-09-23T20:51:15.510 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:51:15.510 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:15.510 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:51:15.510 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:51:15.510 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:51:15.510 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 29569 2023-09-23T20:51:15.510 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.510 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:51:15.510 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.510 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:51:15.510 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.063339+0000", 2023-09-23T20:51:15.511 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:15.511 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.511 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.511 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:51:15.511 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.063339+0000", 2023-09-23T20:51:15.511 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:15.511 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.511 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.511 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:51:15.511 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.063338+0000", 2023-09-23T20:51:15.511 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:51:15.511 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.512 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.512 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:51:15.512 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.063399+0000", 2023-09-23T20:51:15.512 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.0869000000000003e-05 2023-09-23T20:51:15.512 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.512 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.512 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:51:15.512 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.063417+0000", 2023-09-23T20:51:15.512 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.7456999999999999e-05 2023-09-23T20:51:15.512 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.512 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.513 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:51:15.513 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.063423+0000", 2023-09-23T20:51:15.513 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.4139999999999999e-06 2023-09-23T20:51:15.513 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.513 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.513 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:15.513 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.063463+0000", 2023-09-23T20:51:15.513 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.9626999999999999e-05 2023-09-23T20:51:15.513 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.513 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.513 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:51:15.513 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.063572+0000", 2023-09-23T20:51:15.514 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000109697 2023-09-23T20:51:15.514 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.514 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.514 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 3,7", 2023-09-23T20:51:15.514 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.063672+0000", 2023-09-23T20:51:15.514 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.9401000000000006e-05 2023-09-23T20:51:15.514 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.514 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.514 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:51:15.514 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.065628+0000", 2023-09-23T20:51:15.514 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.001955816 2023-09-23T20:51:15.514 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.515 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.515 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:15.515 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.065657+0000", 2023-09-23T20:51:15.515 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.9050000000000001e-05 2023-09-23T20:51:15.515 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.515 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.515 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:15.515 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.203114+0000", 2023-09-23T20:51:15.515 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.13745722199999999 2023-09-23T20:51:15.515 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.515 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.515 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:51:15.516 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.203160+0000", 2023-09-23T20:51:15.516 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.6292999999999999e-05 2023-09-23T20:51:15.516 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.516 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.516 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:51:15.516 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.203185+0000", 2023-09-23T20:51:15.516 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.5196999999999999e-05 2023-09-23T20:51:15.516 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:15.516 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:51:15.516 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:15.516 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.517 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.517 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:29570 9.7 9:ed5a7646:::benchmark_data_smithi018_381516_object29569:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e459)", 2023-09-23T20:51:15.517 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:41:21.063661+0000", 2023-09-23T20:51:15.517 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 594.41665039199995, 2023-09-23T20:51:15.517 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.14287293200000001, 2023-09-23T20:51:15.517 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:51:15.517 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:15.517 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:51:15.517 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:51:15.517 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:51:15.517 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 29570 2023-09-23T20:51:15.517 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.518 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:51:15.518 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.518 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:51:15.518 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.063661+0000", 2023-09-23T20:51:15.518 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:15.518 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.518 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.518 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:51:15.518 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.063661+0000", 2023-09-23T20:51:15.518 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:15.518 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.519 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.519 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:51:15.519 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.063660+0000", 2023-09-23T20:51:15.519 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:51:15.519 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.519 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.519 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:51:15.519 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.063718+0000", 2023-09-23T20:51:15.519 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.8522e-05 2023-09-23T20:51:15.519 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.519 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.520 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:51:15.520 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.063730+0000", 2023-09-23T20:51:15.520 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1586999999999999e-05 2023-09-23T20:51:15.520 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.520 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.520 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:51:15.520 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.063734+0000", 2023-09-23T20:51:15.520 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.3889999999999998e-06 2023-09-23T20:51:15.520 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.520 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.520 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:15.521 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.064722+0000", 2023-09-23T20:51:15.521 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00098758900000000009 2023-09-23T20:51:15.521 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.521 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.521 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:51:15.521 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.064854+0000", 2023-09-23T20:51:15.521 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00013245499999999999 2023-09-23T20:51:15.521 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.521 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.521 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 3,7", 2023-09-23T20:51:15.521 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.064964+0000", 2023-09-23T20:51:15.521 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000109548 2023-09-23T20:51:15.522 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.522 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.522 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:51:15.522 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.065828+0000", 2023-09-23T20:51:15.522 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00086438499999999996 2023-09-23T20:51:15.522 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.522 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.522 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:15.522 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.066131+0000", 2023-09-23T20:51:15.522 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00030242099999999998 2023-09-23T20:51:15.522 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.523 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.523 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:15.523 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.206430+0000", 2023-09-23T20:51:15.523 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.14029873500000001 2023-09-23T20:51:15.523 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.523 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.523 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:51:15.523 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.206487+0000", 2023-09-23T20:51:15.523 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.7812000000000001e-05 2023-09-23T20:51:15.523 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.523 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.523 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:51:15.524 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:21.206533+0000", 2023-09-23T20:51:15.524 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.6122999999999999e-05 2023-09-23T20:51:15.524 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:15.524 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:51:15.524 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:15.524 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.524 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.524 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:76985 9.5 9:ababdd75:::benchmark_data_smithi018_381516_object76984:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e465)", 2023-09-23T20:51:15.524 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:41:58.522099+0000", 2023-09-23T20:51:15.524 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 556.95821198399995, 2023-09-23T20:51:15.524 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.96136469999999996, 2023-09-23T20:51:15.525 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:51:15.525 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:15.525 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:51:15.525 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:51:15.525 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:51:15.525 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 76985 2023-09-23T20:51:15.525 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.525 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:51:15.525 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.525 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:51:15.525 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.522099+0000", 2023-09-23T20:51:15.525 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:15.526 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.526 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.526 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:51:15.526 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.522099+0000", 2023-09-23T20:51:15.526 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:15.526 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.526 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.526 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:51:15.526 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.522098+0000", 2023-09-23T20:51:15.526 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:51:15.526 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.526 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.527 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:51:15.527 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.522133+0000", 2023-09-23T20:51:15.527 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.4041000000000002e-05 2023-09-23T20:51:15.527 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.527 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.527 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:51:15.527 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.522141+0000", 2023-09-23T20:51:15.527 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.7900000000000005e-06 2023-09-23T20:51:15.527 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.527 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.527 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:51:15.527 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.522146+0000", 2023-09-23T20:51:15.528 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.3340000000000003e-06 2023-09-23T20:51:15.528 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.528 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.528 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:15.528 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.522199+0000", 2023-09-23T20:51:15.528 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.3347000000000003e-05 2023-09-23T20:51:15.528 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.528 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.528 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:51:15.528 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.522206+0000", 2023-09-23T20:51:15.528 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.6209999999999998e-06 2023-09-23T20:51:15.528 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.529 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.529 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:15.529 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.471807+0000", 2023-09-23T20:51:15.529 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.94960129400000004 2023-09-23T20:51:15.529 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.529 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.529 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:51:15.529 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.474386+0000", 2023-09-23T20:51:15.529 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0025794749999999999 2023-09-23T20:51:15.529 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.529 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.529 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,7", 2023-09-23T20:51:15.530 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.474496+0000", 2023-09-23T20:51:15.530 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000109733 2023-09-23T20:51:15.530 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.530 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.530 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:15.530 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.483272+0000", 2023-09-23T20:51:15.530 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0087758269999999999 2023-09-23T20:51:15.530 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.530 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.530 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:15.530 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.483367+0000", 2023-09-23T20:51:15.531 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.5458999999999995e-05 2023-09-23T20:51:15.531 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.531 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.531 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:51:15.531 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.483375+0000", 2023-09-23T20:51:15.531 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.7910000000000005e-06 2023-09-23T20:51:15.531 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.531 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.531 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:51:15.531 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.483424+0000", 2023-09-23T20:51:15.531 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.8497e-05 2023-09-23T20:51:15.532 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.532 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.532 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:51:15.532 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.483464+0000", 2023-09-23T20:51:15.532 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.9960999999999999e-05 2023-09-23T20:51:15.532 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:15.532 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:51:15.532 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:15.532 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.532 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.532 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:76994 9.5 9:a38d7184:::benchmark_data_smithi018_381516_object76993:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e465)", 2023-09-23T20:51:15.532 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:41:58.527883+0000", 2023-09-23T20:51:15.533 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 556.95242804199995, 2023-09-23T20:51:15.533 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.95854079599999997, 2023-09-23T20:51:15.533 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:51:15.533 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:15.533 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:51:15.533 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:51:15.533 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:51:15.533 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 76994 2023-09-23T20:51:15.533 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.533 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:51:15.533 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.533 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:51:15.534 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.527883+0000", 2023-09-23T20:51:15.534 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:15.534 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.534 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.534 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:51:15.534 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.527883+0000", 2023-09-23T20:51:15.534 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:15.534 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.534 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.534 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:51:15.534 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.527882+0000", 2023-09-23T20:51:15.535 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:51:15.535 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.535 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.535 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:51:15.535 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.527986+0000", 2023-09-23T20:51:15.535 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000104201 2023-09-23T20:51:15.535 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.535 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.535 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:51:15.535 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.528001+0000", 2023-09-23T20:51:15.535 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.5187999999999999e-05 2023-09-23T20:51:15.535 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.536 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.536 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:51:15.536 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.528009+0000", 2023-09-23T20:51:15.536 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.4259999999999997e-06 2023-09-23T20:51:15.536 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.536 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.536 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:15.536 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.528056+0000", 2023-09-23T20:51:15.536 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.727e-05 2023-09-23T20:51:15.536 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.536 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.537 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:51:15.537 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.528062+0000", 2023-09-23T20:51:15.537 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.1929999999999998e-06 2023-09-23T20:51:15.537 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.537 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.537 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:15.537 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.477679+0000", 2023-09-23T20:51:15.537 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.94961691500000001 2023-09-23T20:51:15.537 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.537 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.537 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:51:15.537 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.477800+0000", 2023-09-23T20:51:15.538 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00012139800000000001 2023-09-23T20:51:15.538 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.538 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.538 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,7", 2023-09-23T20:51:15.538 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.477938+0000", 2023-09-23T20:51:15.538 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00013737499999999999 2023-09-23T20:51:15.538 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.538 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.538 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:15.538 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.483301+0000", 2023-09-23T20:51:15.538 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0053636430000000004 2023-09-23T20:51:15.539 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.539 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.539 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:51:15.539 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.483541+0000", 2023-09-23T20:51:15.539 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00023936699999999999 2023-09-23T20:51:15.539 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.539 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.539 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:15.539 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.486265+0000", 2023-09-23T20:51:15.539 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.002724363 2023-09-23T20:51:15.539 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.539 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.540 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:51:15.540 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.486368+0000", 2023-09-23T20:51:15.540 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000103058 2023-09-23T20:51:15.540 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.540 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.540 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:51:15.540 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.486424+0000", 2023-09-23T20:51:15.540 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.5560000000000003e-05 2023-09-23T20:51:15.540 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:15.540 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:51:15.540 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:15.541 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.541 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.541 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:77025 9.5 9:a8adfb4b:::benchmark_data_smithi018_381516_object77024:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e465)", 2023-09-23T20:51:15.541 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:41:58.574546+0000", 2023-09-23T20:51:15.541 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 556.90576442400004, 2023-09-23T20:51:15.541 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.91796858599999998, 2023-09-23T20:51:15.541 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:51:15.541 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:15.541 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:51:15.541 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:51:15.541 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:51:15.541 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 77025 2023-09-23T20:51:15.542 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.542 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:51:15.542 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.542 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:51:15.542 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.574546+0000", 2023-09-23T20:51:15.542 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:15.542 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.542 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.542 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:51:15.542 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.574546+0000", 2023-09-23T20:51:15.542 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:15.543 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.543 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.543 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:51:15.543 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.574546+0000", 2023-09-23T20:51:15.543 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:51:15.543 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.543 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.543 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:51:15.543 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.574601+0000", 2023-09-23T20:51:15.543 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.4879000000000001e-05 2023-09-23T20:51:15.543 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.543 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.544 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:51:15.544 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.574612+0000", 2023-09-23T20:51:15.544 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1201e-05 2023-09-23T20:51:15.544 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.544 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.544 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:51:15.544 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.574616+0000", 2023-09-23T20:51:15.544 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.1370000000000004e-06 2023-09-23T20:51:15.544 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.544 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.544 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:15.544 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.574650+0000", 2023-09-23T20:51:15.545 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.4007000000000002e-05 2023-09-23T20:51:15.545 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.545 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.545 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:51:15.545 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.574655+0000", 2023-09-23T20:51:15.545 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.0340000000000004e-06 2023-09-23T20:51:15.545 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.545 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.545 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:15.545 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.480178+0000", 2023-09-23T20:51:15.545 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.90552307700000001 2023-09-23T20:51:15.546 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.546 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.546 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:51:15.546 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.480276+0000", 2023-09-23T20:51:15.546 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.8187e-05 2023-09-23T20:51:15.546 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.546 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.546 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,7", 2023-09-23T20:51:15.546 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.480425+0000", 2023-09-23T20:51:15.546 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000148228 2023-09-23T20:51:15.546 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.547 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.547 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:15.547 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.483500+0000", 2023-09-23T20:51:15.547 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0030750740000000001 2023-09-23T20:51:15.547 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.547 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.547 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:51:15.547 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.483545+0000", 2023-09-23T20:51:15.547 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.5864e-05 2023-09-23T20:51:15.547 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.547 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.548 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:15.548 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.492442+0000", 2023-09-23T20:51:15.548 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0088969189999999997 2023-09-23T20:51:15.548 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.548 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.548 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:51:15.548 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.492481+0000", 2023-09-23T20:51:15.548 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.8887999999999998e-05 2023-09-23T20:51:15.548 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.548 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.548 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:51:15.549 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.492515+0000", 2023-09-23T20:51:15.549 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.3810999999999999e-05 2023-09-23T20:51:15.549 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:15.549 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:51:15.549 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:15.549 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.549 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.549 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:91635 9.f 9:f296a7d6:::benchmark_data_smithi018_381516_object91634:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e467)", 2023-09-23T20:51:15.549 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:42:13.701301+0000", 2023-09-23T20:51:15.549 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 541.77900968500001, 2023-09-23T20:51:15.549 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.128139787, 2023-09-23T20:51:15.549 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:51:15.550 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:15.550 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:51:15.550 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:51:15.550 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:51:15.550 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 91635 2023-09-23T20:51:15.550 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.550 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:51:15.550 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.550 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:51:15.550 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.701301+0000", 2023-09-23T20:51:15.550 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:15.551 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.551 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.551 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:51:15.551 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.701301+0000", 2023-09-23T20:51:15.551 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:15.551 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.551 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.551 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:51:15.551 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.701301+0000", 2023-09-23T20:51:15.551 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:51:15.551 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.552 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.552 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:51:15.552 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.701346+0000", 2023-09-23T20:51:15.552 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.5599999999999997e-05 2023-09-23T20:51:15.552 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.552 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.552 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:51:15.552 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.701359+0000", 2023-09-23T20:51:15.552 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.3251000000000001e-05 2023-09-23T20:51:15.552 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.552 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.552 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:51:15.553 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.701365+0000", 2023-09-23T20:51:15.553 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.8309999999999997e-06 2023-09-23T20:51:15.553 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.553 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.553 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:15.553 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.706931+0000", 2023-09-23T20:51:15.553 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0055660900000000001 2023-09-23T20:51:15.553 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.553 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.553 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:51:15.553 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.707048+0000", 2023-09-23T20:51:15.554 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000116663 2023-09-23T20:51:15.554 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.554 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.554 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:51:15.554 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.714996+0000", 2023-09-23T20:51:15.554 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0079476500000000005 2023-09-23T20:51:15.554 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.554 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.554 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:15.554 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.719222+0000", 2023-09-23T20:51:15.555 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.004226769 2023-09-23T20:51:15.555 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.555 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.555 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:51:15.555 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.726451+0000", 2023-09-23T20:51:15.555 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0072287560000000002 2023-09-23T20:51:15.555 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.555 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.555 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:15.555 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.829335+0000", 2023-09-23T20:51:15.555 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.102883501 2023-09-23T20:51:15.555 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.556 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.556 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:51:15.556 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.829386+0000", 2023-09-23T20:51:15.556 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.1496999999999998e-05 2023-09-23T20:51:15.556 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.556 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.556 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:51:15.556 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.829441+0000", 2023-09-23T20:51:15.556 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.4868999999999999e-05 2023-09-23T20:51:15.556 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:15.556 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:51:15.556 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:15.557 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.557 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.557 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:91642 9.7 9:e21f89c5:::benchmark_data_smithi018_381516_object91641:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e467)", 2023-09-23T20:51:15.557 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:42:13.710744+0000", 2023-09-23T20:51:15.557 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 541.76956718500003, 2023-09-23T20:51:15.557 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.13617485700000001, 2023-09-23T20:51:15.557 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:51:15.557 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:15.557 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:51:15.557 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:51:15.557 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:51:15.557 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 91642 2023-09-23T20:51:15.558 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.558 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:51:15.558 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.558 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:51:15.558 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.710744+0000", 2023-09-23T20:51:15.558 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:15.558 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.558 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.558 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:51:15.558 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.710744+0000", 2023-09-23T20:51:15.558 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:15.559 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.559 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.559 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:51:15.559 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.710742+0000", 2023-09-23T20:51:15.559 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:51:15.559 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.559 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.559 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:51:15.559 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.710818+0000", 2023-09-23T20:51:15.559 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.6044999999999994e-05 2023-09-23T20:51:15.559 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.560 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.560 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:51:15.560 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.710834+0000", 2023-09-23T20:51:15.560 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.5733e-05 2023-09-23T20:51:15.560 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.560 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.560 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:51:15.560 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.710841+0000", 2023-09-23T20:51:15.560 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.0559999999999999e-06 2023-09-23T20:51:15.560 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.560 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.560 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:15.560 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.719091+0000", 2023-09-23T20:51:15.561 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0082496040000000007 2023-09-23T20:51:15.561 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.561 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.561 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:51:15.561 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.719366+0000", 2023-09-23T20:51:15.561 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00027542799999999998 2023-09-23T20:51:15.561 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.561 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.561 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 3,7", 2023-09-23T20:51:15.561 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.719569+0000", 2023-09-23T20:51:15.562 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00020253299999999999 2023-09-23T20:51:15.562 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.562 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.562 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:51:15.562 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.729681+0000", 2023-09-23T20:51:15.562 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.010112571000000001 2023-09-23T20:51:15.562 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.562 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.562 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:15.562 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.740599+0000", 2023-09-23T20:51:15.562 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.010917425 2023-09-23T20:51:15.563 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.563 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.563 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:15.563 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.846806+0000", 2023-09-23T20:51:15.563 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.10620702 2023-09-23T20:51:15.563 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.563 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.563 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:51:15.563 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.846863+0000", 2023-09-23T20:51:15.563 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.7924999999999999e-05 2023-09-23T20:51:15.563 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.563 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.564 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:51:15.564 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.846919+0000", 2023-09-23T20:51:15.564 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.5124999999999998e-05 2023-09-23T20:51:15.564 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:15.564 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:51:15.564 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:15.564 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.564 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.564 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:91649 9.7 9:ed097aa6:::benchmark_data_smithi018_381516_object91648:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e467)", 2023-09-23T20:51:15.564 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:42:13.719094+0000", 2023-09-23T20:51:15.564 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 541.76121654099995, 2023-09-23T20:51:15.564 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.12801558599999999, 2023-09-23T20:51:15.565 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:51:15.565 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:15.565 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:51:15.565 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:51:15.565 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:51:15.565 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 91649 2023-09-23T20:51:15.565 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.565 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:51:15.565 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.565 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:51:15.565 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.719094+0000", 2023-09-23T20:51:15.566 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:15.566 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.566 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.566 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:51:15.566 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.719094+0000", 2023-09-23T20:51:15.566 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:15.566 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.566 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.566 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:51:15.566 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.719093+0000", 2023-09-23T20:51:15.566 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:51:15.566 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.567 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.567 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:51:15.567 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.719128+0000", 2023-09-23T20:51:15.567 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.5079000000000001e-05 2023-09-23T20:51:15.567 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.567 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.567 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:51:15.567 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.719139+0000", 2023-09-23T20:51:15.567 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0761000000000001e-05 2023-09-23T20:51:15.567 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.567 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.567 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:51:15.568 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.719144+0000", 2023-09-23T20:51:15.568 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.2920000000000003e-06 2023-09-23T20:51:15.568 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.568 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.568 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:15.568 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.722939+0000", 2023-09-23T20:51:15.568 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.003794136 2023-09-23T20:51:15.568 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.568 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.568 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:51:15.568 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.723030+0000", 2023-09-23T20:51:15.569 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.1193999999999995e-05 2023-09-23T20:51:15.569 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.569 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.569 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 3,7", 2023-09-23T20:51:15.569 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.723143+0000", 2023-09-23T20:51:15.569 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000113268 2023-09-23T20:51:15.569 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.569 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.569 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:51:15.569 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.729715+0000", 2023-09-23T20:51:15.569 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0065721130000000001 2023-09-23T20:51:15.569 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.569 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.570 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:15.570 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.740629+0000", 2023-09-23T20:51:15.570 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.010913997 2023-09-23T20:51:15.570 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.570 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.570 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:15.570 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.846966+0000", 2023-09-23T20:51:15.570 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.106336736 2023-09-23T20:51:15.570 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.570 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.570 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:51:15.571 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.847061+0000", 2023-09-23T20:51:15.571 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.4692999999999996e-05 2023-09-23T20:51:15.571 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.571 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.571 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:51:15.571 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.847110+0000", 2023-09-23T20:51:15.571 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.9382000000000003e-05 2023-09-23T20:51:15.571 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:15.571 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:51:15.571 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:15.571 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.571 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.572 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:91651 9.f 9:f2c174a9:::benchmark_data_smithi018_381516_object91650:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e467)", 2023-09-23T20:51:15.572 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:42:13.719151+0000", 2023-09-23T20:51:15.572 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 541.76116041099999, 2023-09-23T20:51:15.572 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.12779057599999999, 2023-09-23T20:51:15.572 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:51:15.572 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:15.572 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:51:15.572 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:51:15.572 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:51:15.572 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 91651 2023-09-23T20:51:15.572 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.573 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:51:15.573 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.573 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:51:15.573 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.719151+0000", 2023-09-23T20:51:15.573 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:15.573 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.573 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.573 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:51:15.573 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.719151+0000", 2023-09-23T20:51:15.573 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:15.573 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.573 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.574 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:51:15.574 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.719150+0000", 2023-09-23T20:51:15.574 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:51:15.574 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.574 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.574 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:51:15.574 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.719179+0000", 2023-09-23T20:51:15.574 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.9420999999999998e-05 2023-09-23T20:51:15.574 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.574 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.574 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:51:15.575 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.719187+0000", 2023-09-23T20:51:15.575 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.2320000000000004e-06 2023-09-23T20:51:15.575 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.575 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.575 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:51:15.575 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.719191+0000", 2023-09-23T20:51:15.575 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.3220000000000003e-06 2023-09-23T20:51:15.575 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.575 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.575 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:15.575 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.722930+0000", 2023-09-23T20:51:15.575 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0037388640000000002 2023-09-23T20:51:15.576 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.576 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.576 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:51:15.576 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.723047+0000", 2023-09-23T20:51:15.576 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00011688699999999999 2023-09-23T20:51:15.576 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.576 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.576 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:51:15.576 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.723182+0000", 2023-09-23T20:51:15.576 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00013561199999999999 2023-09-23T20:51:15.576 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.577 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.577 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:15.577 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.729712+0000", 2023-09-23T20:51:15.577 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0065292340000000001 2023-09-23T20:51:15.577 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.577 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.577 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:51:15.577 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.729718+0000", 2023-09-23T20:51:15.577 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.3829999999999997e-06 2023-09-23T20:51:15.577 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.577 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.578 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:15.578 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.846858+0000", 2023-09-23T20:51:15.578 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.117139934 2023-09-23T20:51:15.578 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.578 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.578 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:51:15.578 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.846905+0000", 2023-09-23T20:51:15.578 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.7187000000000001e-05 2023-09-23T20:51:15.578 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.578 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.578 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:51:15.578 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:13.846941+0000", 2023-09-23T20:51:15.579 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.5975e-05 2023-09-23T20:51:15.579 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:15.579 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:51:15.579 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:15.579 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.579 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.579 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:92127 9.7 9:eb5270d9:::benchmark_data_smithi018_381516_object92126:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e467)", 2023-09-23T20:51:15.579 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:42:14.303475+0000", 2023-09-23T20:51:15.579 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 541.17683591100001, 2023-09-23T20:51:15.579 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.13503338400000001, 2023-09-23T20:51:15.580 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:51:15.580 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:15.580 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:51:15.580 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:51:15.580 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:51:15.580 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 92127 2023-09-23T20:51:15.580 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.580 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:51:15.580 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.580 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:51:15.580 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.303475+0000", 2023-09-23T20:51:15.580 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:15.581 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.581 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.581 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:51:15.581 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.303475+0000", 2023-09-23T20:51:15.581 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:15.581 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.581 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.581 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:51:15.581 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.303475+0000", 2023-09-23T20:51:15.581 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:51:15.581 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.581 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.582 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:51:15.582 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.303510+0000", 2023-09-23T20:51:15.582 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.5318999999999998e-05 2023-09-23T20:51:15.582 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.582 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.582 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:51:15.582 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.303516+0000", 2023-09-23T20:51:15.582 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.4320000000000004e-06 2023-09-23T20:51:15.582 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.582 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.582 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:51:15.583 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.303520+0000", 2023-09-23T20:51:15.583 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.0280000000000002e-06 2023-09-23T20:51:15.583 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.583 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.583 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:15.583 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.303550+0000", 2023-09-23T20:51:15.583 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.9938e-05 2023-09-23T20:51:15.583 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.583 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.583 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:51:15.583 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.303677+0000", 2023-09-23T20:51:15.584 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00012659800000000001 2023-09-23T20:51:15.584 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.584 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.584 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 3,7", 2023-09-23T20:51:15.584 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.303805+0000", 2023-09-23T20:51:15.584 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00012804799999999999 2023-09-23T20:51:15.584 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.584 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.584 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:51:15.584 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.306643+0000", 2023-09-23T20:51:15.584 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.002837759 2023-09-23T20:51:15.584 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.585 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.585 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:15.585 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.317326+0000", 2023-09-23T20:51:15.585 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.010683002 2023-09-23T20:51:15.585 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.585 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.585 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:15.585 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.438439+0000", 2023-09-23T20:51:15.585 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.121113448 2023-09-23T20:51:15.585 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.585 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.586 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:51:15.586 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.438478+0000", 2023-09-23T20:51:15.586 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.8921999999999998e-05 2023-09-23T20:51:15.586 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.586 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.586 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:51:15.586 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.438508+0000", 2023-09-23T20:51:15.586 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.029e-05 2023-09-23T20:51:15.586 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:15.586 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:51:15.586 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:15.587 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.587 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.587 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:92129 9.f 9:fdf451a5:::benchmark_data_smithi018_381516_object92128:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e467)", 2023-09-23T20:51:15.587 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:42:14.307177+0000", 2023-09-23T20:51:15.587 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 541.17313413800002, 2023-09-23T20:51:15.587 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.138573006, 2023-09-23T20:51:15.587 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:51:15.587 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:15.587 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:51:15.587 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:51:15.587 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:51:15.587 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 92129 2023-09-23T20:51:15.588 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.588 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:51:15.588 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.588 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:51:15.588 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.307177+0000", 2023-09-23T20:51:15.588 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:15.588 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.588 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.588 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:51:15.588 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.307177+0000", 2023-09-23T20:51:15.588 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:15.589 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.589 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.589 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:51:15.589 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.307175+0000", 2023-09-23T20:51:15.589 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:51:15.589 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.589 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.589 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:51:15.589 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.307245+0000", 2023-09-23T20:51:15.589 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.0261999999999999e-05 2023-09-23T20:51:15.589 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.589 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.590 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:51:15.590 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.307253+0000", 2023-09-23T20:51:15.590 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.7219999999999999e-06 2023-09-23T20:51:15.590 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.590 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.590 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:51:15.590 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.307257+0000", 2023-09-23T20:51:15.590 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.0430000000000001e-06 2023-09-23T20:51:15.590 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.590 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.590 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:15.591 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.307299+0000", 2023-09-23T20:51:15.591 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.2128000000000001e-05 2023-09-23T20:51:15.591 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.591 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.591 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:51:15.591 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.307431+0000", 2023-09-23T20:51:15.591 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00013203499999999999 2023-09-23T20:51:15.591 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.591 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.591 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:51:15.591 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.307559+0000", 2023-09-23T20:51:15.591 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000127632 2023-09-23T20:51:15.592 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.592 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.592 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:15.592 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.317355+0000", 2023-09-23T20:51:15.592 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0097961349999999992 2023-09-23T20:51:15.592 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.592 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.592 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:51:15.592 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.320352+0000", 2023-09-23T20:51:15.592 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0029971569999999999 2023-09-23T20:51:15.592 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.592 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.593 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:15.593 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.445677+0000", 2023-09-23T20:51:15.593 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.12532515999999999 2023-09-23T20:51:15.593 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.593 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.593 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:51:15.593 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.445717+0000", 2023-09-23T20:51:15.593 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.9616000000000002e-05 2023-09-23T20:51:15.593 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.593 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.593 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:51:15.593 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.445750+0000", 2023-09-23T20:51:15.594 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.2719999999999998e-05 2023-09-23T20:51:15.594 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:15.594 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:51:15.594 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:15.594 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.594 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.594 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:92140 9.f 9:f3d16644:::benchmark_data_smithi018_381516_object92139:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e467)", 2023-09-23T20:51:15.594 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:42:14.313964+0000", 2023-09-23T20:51:15.594 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 541.166346729, 2023-09-23T20:51:15.594 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.13195174000000001, 2023-09-23T20:51:15.594 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:51:15.595 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:15.595 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:51:15.595 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:51:15.595 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:51:15.595 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 92140 2023-09-23T20:51:15.595 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.595 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:51:15.595 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.595 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:51:15.595 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.313964+0000", 2023-09-23T20:51:15.595 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:15.596 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.596 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.596 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:51:15.596 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.313964+0000", 2023-09-23T20:51:15.596 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:15.596 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.596 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.596 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:51:15.596 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.313964+0000", 2023-09-23T20:51:15.596 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:51:15.596 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.596 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.597 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:51:15.597 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.314026+0000", 2023-09-23T20:51:15.597 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.2818999999999996e-05 2023-09-23T20:51:15.597 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.597 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.597 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:51:15.597 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.314035+0000", 2023-09-23T20:51:15.597 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.7059999999999993e-06 2023-09-23T20:51:15.597 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.597 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.597 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:51:15.597 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.314040+0000", 2023-09-23T20:51:15.598 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.2660000000000001e-06 2023-09-23T20:51:15.598 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.598 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.598 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:15.598 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.314084+0000", 2023-09-23T20:51:15.598 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.3204000000000002e-05 2023-09-23T20:51:15.598 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.598 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.598 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:51:15.598 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.314195+0000", 2023-09-23T20:51:15.598 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000111585 2023-09-23T20:51:15.599 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.599 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.599 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 1,3", 2023-09-23T20:51:15.599 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.314336+0000", 2023-09-23T20:51:15.599 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000140977 2023-09-23T20:51:15.599 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.599 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.599 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:15.599 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.320388+0000", 2023-09-23T20:51:15.599 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.006051519 2023-09-23T20:51:15.599 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.599 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.600 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:51:15.600 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.320393+0000", 2023-09-23T20:51:15.600 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.7189999999999998e-06 2023-09-23T20:51:15.600 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.600 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.600 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:15.600 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.445855+0000", 2023-09-23T20:51:15.600 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.125461666 2023-09-23T20:51:15.600 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.600 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.600 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:51:15.600 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.445885+0000", 2023-09-23T20:51:15.601 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.0454e-05 2023-09-23T20:51:15.601 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.601 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.601 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:51:15.601 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.445916+0000", 2023-09-23T20:51:15.601 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.0446e-05 2023-09-23T20:51:15.601 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:15.601 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:51:15.601 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:15.601 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.601 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.601 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:92144 9.7 9:ec1b46c9:::benchmark_data_smithi018_381516_object92143:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e467)", 2023-09-23T20:51:15.602 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:42:14.317205+0000", 2023-09-23T20:51:15.602 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 541.16310556500002, 2023-09-23T20:51:15.602 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.12863032999999999, 2023-09-23T20:51:15.602 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:51:15.602 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:15.602 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:51:15.602 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:51:15.602 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:51:15.602 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 92144 2023-09-23T20:51:15.602 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.602 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:51:15.603 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.603 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:51:15.603 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.317205+0000", 2023-09-23T20:51:15.603 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:15.603 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.603 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.603 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:51:15.603 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.317205+0000", 2023-09-23T20:51:15.603 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:15.603 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.603 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.603 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:51:15.604 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.317204+0000", 2023-09-23T20:51:15.604 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:51:15.604 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.604 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.604 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:51:15.604 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.317280+0000", 2023-09-23T20:51:15.604 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.6215000000000001e-05 2023-09-23T20:51:15.604 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.604 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.604 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:51:15.604 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.317291+0000", 2023-09-23T20:51:15.605 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0546e-05 2023-09-23T20:51:15.605 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.605 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.605 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:51:15.605 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.317297+0000", 2023-09-23T20:51:15.605 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.2650000000000002e-06 2023-09-23T20:51:15.605 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.605 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.605 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:15.605 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.317374+0000", 2023-09-23T20:51:15.605 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.6600999999999994e-05 2023-09-23T20:51:15.605 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.606 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.606 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:51:15.606 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.317479+0000", 2023-09-23T20:51:15.606 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000105531 2023-09-23T20:51:15.606 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.606 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.606 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 3,7", 2023-09-23T20:51:15.606 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.317604+0000", 2023-09-23T20:51:15.606 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000124745 2023-09-23T20:51:15.606 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.606 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.606 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:51:15.607 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.321102+0000", 2023-09-23T20:51:15.607 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0034977849999999998 2023-09-23T20:51:15.607 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.607 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.607 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:15.607 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.321402+0000", 2023-09-23T20:51:15.607 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00029973499999999998 2023-09-23T20:51:15.607 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.607 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.607 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:15.607 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.445781+0000", 2023-09-23T20:51:15.608 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.124378979 2023-09-23T20:51:15.608 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.608 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.608 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:51:15.608 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.445816+0000", 2023-09-23T20:51:15.608 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.5222000000000003e-05 2023-09-23T20:51:15.608 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.608 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.608 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:51:15.608 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:42:14.445836+0000", 2023-09-23T20:51:15.608 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.9952000000000001e-05 2023-09-23T20:51:15.608 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:15.609 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:51:15.609 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:15.609 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.609 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.609 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:141052 9.5 9:a7a766e0:::benchmark_data_smithi018_381516_object27218:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e481)", 2023-09-23T20:51:15.609 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:43:15.774779+0000", 2023-09-23T20:51:15.609 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 479.705532151, 2023-09-23T20:51:15.609 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.023693841, 2023-09-23T20:51:15.609 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:51:15.609 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:15.609 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:51:15.609 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:51:15.610 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:51:15.610 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 141052 2023-09-23T20:51:15.610 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.610 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:51:15.610 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.610 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:51:15.610 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.774779+0000", 2023-09-23T20:51:15.610 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:15.610 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.610 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.610 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:51:15.611 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.774779+0000", 2023-09-23T20:51:15.611 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:15.611 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.611 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.611 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:51:15.611 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.774778+0000", 2023-09-23T20:51:15.611 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:51:15.611 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.611 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.611 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:51:15.611 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.774783+0000", 2023-09-23T20:51:15.611 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.7660000000000004e-06 2023-09-23T20:51:15.612 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.612 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.612 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:51:15.612 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.774794+0000", 2023-09-23T20:51:15.612 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0744000000000001e-05 2023-09-23T20:51:15.612 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.612 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.612 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:51:15.612 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.774799+0000", 2023-09-23T20:51:15.612 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.9810000000000003e-06 2023-09-23T20:51:15.612 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.613 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.613 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:15.613 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.774870+0000", 2023-09-23T20:51:15.613 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.1316000000000006e-05 2023-09-23T20:51:15.613 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.613 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.613 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:51:15.613 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.774881+0000", 2023-09-23T20:51:15.613 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0291e-05 2023-09-23T20:51:15.613 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.613 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.613 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:15.614 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.785209+0000", 2023-09-23T20:51:15.614 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0103286419999999 2023-09-23T20:51:15.614 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.614 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.614 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:51:15.614 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.785375+0000", 2023-09-23T20:51:15.614 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00016602399999999999 2023-09-23T20:51:15.614 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.614 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.614 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,1,7", 2023-09-23T20:51:15.614 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.785518+0000", 2023-09-23T20:51:15.615 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00014225800000000001 2023-09-23T20:51:15.615 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.615 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.615 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:51:15.615 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.793627+0000", 2023-09-23T20:51:15.615 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0081096780000000004 2023-09-23T20:51:15.615 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.615 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.615 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:15.615 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.793665+0000", 2023-09-23T20:51:15.615 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.7286000000000002e-05 2023-09-23T20:51:15.616 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.616 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.616 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:15.616 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.797943+0000", 2023-09-23T20:51:15.616 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0042786869999999998 2023-09-23T20:51:15.616 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.616 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.616 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:15.616 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.798292+0000", 2023-09-23T20:51:15.616 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00034836099999999999 2023-09-23T20:51:15.616 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.616 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.616 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:51:15.617 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.798353+0000", 2023-09-23T20:51:15.617 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.1019e-05 2023-09-23T20:51:15.617 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.617 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.617 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:51:15.617 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.798473+0000", 2023-09-23T20:51:15.617 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000119888 2023-09-23T20:51:15.617 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:15.617 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:51:15.617 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:15.617 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.618 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.618 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:141057 9.5 9:aa98a8f8:::benchmark_data_smithi018_381516_object27223:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e481)", 2023-09-23T20:51:15.618 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:43:15.777430+0000", 2023-09-23T20:51:15.618 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 479.70288076100002, 2023-09-23T20:51:15.618 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0450975220000001, 2023-09-23T20:51:15.618 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:51:15.618 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:15.618 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:51:15.618 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:51:15.618 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:51:15.618 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 141057 2023-09-23T20:51:15.619 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.619 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:51:15.619 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.619 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:51:15.619 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.777430+0000", 2023-09-23T20:51:15.619 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:15.619 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.619 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.619 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:51:15.619 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.777430+0000", 2023-09-23T20:51:15.619 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:15.619 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.620 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.620 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:51:15.620 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.777428+0000", 2023-09-23T20:51:15.620 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999981 2023-09-23T20:51:15.620 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.620 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.620 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:51:15.620 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.777438+0000", 2023-09-23T20:51:15.620 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.025e-05 2023-09-23T20:51:15.620 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.620 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.620 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:51:15.621 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.777451+0000", 2023-09-23T20:51:15.621 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.2218000000000001e-05 2023-09-23T20:51:15.621 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.621 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.621 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:51:15.621 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.777456+0000", 2023-09-23T20:51:15.621 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.9019999999999998e-06 2023-09-23T20:51:15.621 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.621 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.621 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:15.621 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.777492+0000", 2023-09-23T20:51:15.622 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.6758000000000003e-05 2023-09-23T20:51:15.622 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.622 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.622 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:51:15.622 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.777498+0000", 2023-09-23T20:51:15.622 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.9529999999999996e-06 2023-09-23T20:51:15.622 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.622 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.622 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:15.622 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.793684+0000", 2023-09-23T20:51:15.622 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0161855 2023-09-23T20:51:15.622 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.623 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.623 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:51:15.623 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.793806+0000", 2023-09-23T20:51:15.623 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00012180600000000001 2023-09-23T20:51:15.623 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.623 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.623 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,1,7", 2023-09-23T20:51:15.623 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.793923+0000", 2023-09-23T20:51:15.623 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00011699100000000001 2023-09-23T20:51:15.623 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.623 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.623 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:51:15.624 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.803080+0000", 2023-09-23T20:51:15.624 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0091576539999999994 2023-09-23T20:51:15.624 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.624 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.624 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:15.624 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.809899+0000", 2023-09-23T20:51:15.624 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0068191249999999997 2023-09-23T20:51:15.624 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.624 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.624 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:15.624 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.817155+0000", 2023-09-23T20:51:15.624 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0072551860000000003 2023-09-23T20:51:15.625 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.625 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.625 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:15.625 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.822362+0000", 2023-09-23T20:51:15.625 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0052070450000000004 2023-09-23T20:51:15.625 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.625 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.625 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:51:15.625 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.822413+0000", 2023-09-23T20:51:15.625 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.1240999999999999e-05 2023-09-23T20:51:15.625 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.626 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.626 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:51:15.626 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.822528+0000", 2023-09-23T20:51:15.626 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000114843 2023-09-23T20:51:15.626 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:15.626 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:51:15.626 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:15.626 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.626 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.626 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:141060 9.5 9:a222c8d8:::benchmark_data_smithi018_381516_object27226:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e481)", 2023-09-23T20:51:15.626 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:43:15.784280+0000", 2023-09-23T20:51:15.626 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 479.69603054300001, 2023-09-23T20:51:15.627 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0384760930000001, 2023-09-23T20:51:15.627 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:51:15.627 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:15.627 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:51:15.627 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:51:15.627 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:51:15.627 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 141060 2023-09-23T20:51:15.627 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.627 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:51:15.627 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.627 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:51:15.627 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.784280+0000", 2023-09-23T20:51:15.628 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:15.628 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.628 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.628 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:51:15.628 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.784280+0000", 2023-09-23T20:51:15.628 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:15.628 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.628 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.628 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:51:15.628 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.784278+0000", 2023-09-23T20:51:15.628 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999981 2023-09-23T20:51:15.628 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.629 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.629 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:51:15.629 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.784293+0000", 2023-09-23T20:51:15.629 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.5048999999999999e-05 2023-09-23T20:51:15.629 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.629 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.629 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:51:15.629 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.784311+0000", 2023-09-23T20:51:15.629 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.7530999999999999e-05 2023-09-23T20:51:15.629 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.630 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.630 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:51:15.630 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.784329+0000", 2023-09-23T20:51:15.630 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.8470000000000001e-05 2023-09-23T20:51:15.630 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.630 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.630 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:15.630 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.784394+0000", 2023-09-23T20:51:15.630 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.4487999999999996e-05 2023-09-23T20:51:15.630 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.630 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.630 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:51:15.631 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.784404+0000", 2023-09-23T20:51:15.631 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0478e-05 2023-09-23T20:51:15.631 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.631 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.631 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:15.631 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.798720+0000", 2023-09-23T20:51:15.631 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.014316145 2023-09-23T20:51:15.631 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.631 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.631 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:51:15.631 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.798848+0000", 2023-09-23T20:51:15.631 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000127448 2023-09-23T20:51:15.632 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.632 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.632 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,1,7", 2023-09-23T20:51:15.632 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.798989+0000", 2023-09-23T20:51:15.632 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00014109800000000001 2023-09-23T20:51:15.632 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.632 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.632 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:51:15.632 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.809908+0000", 2023-09-23T20:51:15.632 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.01091902 2023-09-23T20:51:15.632 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.632 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.633 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:15.633 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.812374+0000", 2023-09-23T20:51:15.633 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0024658710000000001 2023-09-23T20:51:15.633 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.633 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.633 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:15.633 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.817185+0000", 2023-09-23T20:51:15.633 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0048108229999999997 2023-09-23T20:51:15.633 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.633 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.633 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:15.634 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.822622+0000", 2023-09-23T20:51:15.634 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0054370920000000001 2023-09-23T20:51:15.634 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.634 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.634 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:51:15.634 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.822660+0000", 2023-09-23T20:51:15.634 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.8250000000000001e-05 2023-09-23T20:51:15.634 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.634 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.634 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:51:15.634 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.822756+0000", 2023-09-23T20:51:15.635 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.6471999999999994e-05 2023-09-23T20:51:15.635 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:15.635 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:51:15.635 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:15.635 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.635 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.635 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_repop(client.46625.0:102302 10.3 e553/531 10:c7864152:::benchmark_data_smithi018_422431_object102301:head v 553'6535, mlcod=551'6533)", 2023-09-23T20:51:15.635 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:47:09.250152+0000", 2023-09-23T20:51:15.635 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 246.230158539, 2023-09-23T20:51:15.635 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.085020380000000007, 2023-09-23T20:51:15.635 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:51:15.635 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:15.636 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:51:15.636 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.636 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:51:15.636 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.250152+0000", 2023-09-23T20:51:15.636 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:15.636 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.636 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.636 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:51:15.636 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.250152+0000", 2023-09-23T20:51:15.636 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:15.636 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.636 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.637 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:51:15.637 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.250151+0000", 2023-09-23T20:51:15.637 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:51:15.637 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.637 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.637 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:51:15.637 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.250208+0000", 2023-09-23T20:51:15.637 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.7258e-05 2023-09-23T20:51:15.637 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.637 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.637 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:51:15.637 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.250229+0000", 2023-09-23T20:51:15.638 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.0832e-05 2023-09-23T20:51:15.638 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.638 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.638 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:51:15.638 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.250240+0000", 2023-09-23T20:51:15.638 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0665999999999999e-05 2023-09-23T20:51:15.638 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.638 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.638 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:15.638 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.250312+0000", 2023-09-23T20:51:15.638 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.2254000000000003e-05 2023-09-23T20:51:15.639 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.639 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.639 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:51:15.639 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.250347+0000", 2023-09-23T20:51:15.639 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.4993999999999997e-05 2023-09-23T20:51:15.639 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.639 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.639 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:51:15.639 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.335133+0000", 2023-09-23T20:51:15.639 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.084785762000000001 2023-09-23T20:51:15.639 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:15.640 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:15.640 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:51:15.640 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.335173+0000", 2023-09-23T20:51:15.640 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.9975999999999998e-05 2023-09-23T20:51:15.640 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:15.640 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:51:15.640 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:15.640 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:15.640 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:51:15.640 INFO:teuthology.orchestra.run.smithi149.stdout:} 2023-09-23T20:51:15.896 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T20:51:16.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:16 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:51:16.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:16 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:51:16.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:16 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:51:16.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:16 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:51:16.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:16 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:51:16.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:16 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:51:16.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:16 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:51:16.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:16 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:51:16.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:16 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:51:16.496 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:51:17.731 INFO:teuthology.orchestra.run.smithi149.stdout:{ 2023-09-23T20:51:17.731 INFO:teuthology.orchestra.run.smithi149.stdout: "ops": [], 2023-09-23T20:51:17.731 INFO:teuthology.orchestra.run.smithi149.stdout: "num_ops": 0 2023-09-23T20:51:17.731 INFO:teuthology.orchestra.run.smithi149.stdout:} 2023-09-23T20:51:17.867 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:17 smithi149 ceph-mon[170394]: pgmap v1643: 103 pgs: 103 active+clean; 8.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail 2023-09-23T20:51:18.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:17 smithi018 ceph-mon[184144]: pgmap v1643: 103 pgs: 103 active+clean; 8.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail 2023-09-23T20:51:18.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:17 smithi018 ceph-mon[179647]: pgmap v1643: 103 pgs: 103 active+clean; 8.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail 2023-09-23T20:51:18.156 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T20:51:18.366 INFO:tasks.thrashosds.thrasher:inning osd 2023-09-23T20:51:18.366 INFO:tasks.thrashosds.thrasher:Adding osd 5 2023-09-23T20:51:18.366 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd in 5 2023-09-23T20:51:18.760 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:51:18.987 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:51:19.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:19 smithi018 ceph-mon[184144]: pgmap v1644: 103 pgs: 103 active+clean; 8.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail 2023-09-23T20:51:19.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:19 smithi018 ceph-mon[179647]: pgmap v1644: 103 pgs: 103 active+clean; 8.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail 2023-09-23T20:51:20.078 INFO:teuthology.orchestra.run.smithi149.stdout:{ 2023-09-23T20:51:20.078 INFO:teuthology.orchestra.run.smithi149.stdout: "ops": [], 2023-09-23T20:51:20.078 INFO:teuthology.orchestra.run.smithi149.stdout: "complaint_time": 30, 2023-09-23T20:51:20.078 INFO:teuthology.orchestra.run.smithi149.stdout: "num_blocked_ops": 0 2023-09-23T20:51:20.078 INFO:teuthology.orchestra.run.smithi149.stdout:} 2023-09-23T20:51:20.105 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:19 smithi149 ceph-mon[170394]: pgmap v1644: 103 pgs: 103 active+clean; 8.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail 2023-09-23T20:51:20.517 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T20:51:20.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:20 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mon-a[179623]: 2023-09-23T20:51:20.361+0000 7f160fef9700 -1 mon.a@0(leader).osd e672 definitely_dead 0 2023-09-23T20:51:20.745 INFO:teuthology.orchestra.run.smithi018.stderr:marked in osd.5. 2023-09-23T20:51:21.085 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:20 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3992991234' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["5"]}]: dispatch 2023-09-23T20:51:21.085 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:20 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["5"]}]: dispatch 2023-09-23T20:51:21.085 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:20 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3992991234' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["5"]}]: dispatch 2023-09-23T20:51:21.085 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:20 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["5"]}]: dispatch 2023-09-23T20:51:21.121 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:51:21.140 INFO:tasks.thrashosds.thrasher:Added osd 5 2023-09-23T20:51:21.140 INFO:tasks.thrashosds.thrasher:fixing pg num pool .mgr 2023-09-23T20:51:21.140 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd pool set .mgr pgp_num 91 2023-09-23T20:51:21.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:20 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3992991234' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["5"]}]: dispatch 2023-09-23T20:51:21.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:20 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["5"]}]: dispatch 2023-09-23T20:51:21.751 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:51:21.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:21 smithi149 ceph-mon[170394]: pgmap v1645: 103 pgs: 103 active+clean; 8.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail 2023-09-23T20:51:21.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:21 smithi149 ceph-mon[170394]: failed to encode map e673 with expected crc 2023-09-23T20:51:21.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:21 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["5"]}]': finished 2023-09-23T20:51:21.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:21 smithi149 ceph-mon[170394]: osdmap e673: 8 total, 8 up, 8 in 2023-09-23T20:51:21.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:21 smithi149 ceph-mon[170394]: osdmap e674: 8 total, 8 up, 8 in 2023-09-23T20:51:22.013 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:21 smithi018 ceph-mon[184144]: pgmap v1645: 103 pgs: 103 active+clean; 8.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail 2023-09-23T20:51:22.014 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:21 smithi018 ceph-mon[184144]: failed to encode map e673 with expected crc 2023-09-23T20:51:22.014 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:21 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["5"]}]': finished 2023-09-23T20:51:22.014 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:21 smithi018 ceph-mon[184144]: osdmap e673: 8 total, 8 up, 8 in 2023-09-23T20:51:22.014 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:21 smithi018 ceph-mon[184144]: osdmap e674: 8 total, 8 up, 8 in 2023-09-23T20:51:22.014 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:21 smithi018 ceph-mon[179647]: pgmap v1645: 103 pgs: 103 active+clean; 8.6 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail 2023-09-23T20:51:22.014 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:21 smithi018 ceph-mon[179647]: failed to encode map e673 with expected crc 2023-09-23T20:51:22.014 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:21 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["5"]}]': finished 2023-09-23T20:51:22.014 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:21 smithi018 ceph-mon[179647]: osdmap e673: 8 total, 8 up, 8 in 2023-09-23T20:51:22.014 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:21 smithi018 ceph-mon[179647]: osdmap e674: 8 total, 8 up, 8 in 2023-09-23T20:51:22.446 INFO:teuthology.orchestra.run.smithi149.stdout:{ 2023-09-23T20:51:22.446 INFO:teuthology.orchestra.run.smithi149.stdout: "size": 20, 2023-09-23T20:51:22.446 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 600, 2023-09-23T20:51:22.446 INFO:teuthology.orchestra.run.smithi149.stdout: "ops": [ 2023-09-23T20:51:22.446 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.446 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:76949 9.b 9:dc3199d0:::benchmark_data_smithi018_381516_object76948:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e465)", 2023-09-23T20:51:22.447 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:41:58.481021+0000", 2023-09-23T20:51:22.447 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 563.96259943500002, 2023-09-23T20:51:22.447 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.012473631, 2023-09-23T20:51:22.447 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:51:22.447 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:22.447 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:51:22.447 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:51:22.447 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:51:22.447 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 76949 2023-09-23T20:51:22.447 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.447 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:51:22.447 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.448 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:51:22.448 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.481021+0000", 2023-09-23T20:51:22.448 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:22.448 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.448 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.448 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:51:22.448 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.481021+0000", 2023-09-23T20:51:22.448 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:22.448 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.448 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.448 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:51:22.448 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.481020+0000", 2023-09-23T20:51:22.449 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:51:22.449 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.449 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.449 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:51:22.449 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.481102+0000", 2023-09-23T20:51:22.449 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.2538000000000002e-05 2023-09-23T20:51:22.449 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.449 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.449 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:51:22.449 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.481122+0000", 2023-09-23T20:51:22.449 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.9556e-05 2023-09-23T20:51:22.449 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.450 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.450 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:51:22.450 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.481129+0000", 2023-09-23T20:51:22.450 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.7449999999999995e-06 2023-09-23T20:51:22.450 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.450 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.450 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:22.450 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.481713+0000", 2023-09-23T20:51:22.450 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000583167 2023-09-23T20:51:22.450 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.450 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.451 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:51:22.451 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.481726+0000", 2023-09-23T20:51:22.451 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.3291e-05 2023-09-23T20:51:22.451 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.451 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.451 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:22.451 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.467637+0000", 2023-09-23T20:51:22.451 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.98591135500000004 2023-09-23T20:51:22.451 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.451 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.451 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:51:22.452 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.470079+0000", 2023-09-23T20:51:22.452 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0024414089999999999 2023-09-23T20:51:22.452 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.452 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.452 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:51:22.452 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.470267+0000", 2023-09-23T20:51:22.452 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00018824299999999999 2023-09-23T20:51:22.452 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.452 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.452 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:51:22.452 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.481273+0000", 2023-09-23T20:51:22.452 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.011006395 2023-09-23T20:51:22.453 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.453 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.453 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:22.453 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.493371+0000", 2023-09-23T20:51:22.453 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.012098138 2023-09-23T20:51:22.453 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.453 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.453 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:22.453 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.493433+0000", 2023-09-23T20:51:22.453 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.1437000000000001e-05 2023-09-23T20:51:22.453 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.453 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.454 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:51:22.454 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.493472+0000", 2023-09-23T20:51:22.454 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.9038000000000001e-05 2023-09-23T20:51:22.454 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.454 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.454 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:51:22.454 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.493495+0000", 2023-09-23T20:51:22.454 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.2778000000000001e-05 2023-09-23T20:51:22.454 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:22.454 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:51:22.454 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:22.455 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.455 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.455 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:76967 9.b 9:d898a916:::benchmark_data_smithi018_381516_object76966:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e465)", 2023-09-23T20:51:22.455 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:41:58.507229+0000", 2023-09-23T20:51:22.455 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 563.93639102500003, 2023-09-23T20:51:22.455 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.98635601299999998, 2023-09-23T20:51:22.455 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:51:22.455 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:22.455 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:51:22.455 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:51:22.455 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:51:22.455 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 76967 2023-09-23T20:51:22.456 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.456 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:51:22.456 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.456 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:51:22.456 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.507229+0000", 2023-09-23T20:51:22.456 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:22.456 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.456 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.456 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:51:22.456 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.507229+0000", 2023-09-23T20:51:22.456 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:22.456 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.457 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.457 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:51:22.457 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.507228+0000", 2023-09-23T20:51:22.457 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999981 2023-09-23T20:51:22.457 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.457 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.457 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:51:22.457 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.507285+0000", 2023-09-23T20:51:22.457 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.7531000000000002e-05 2023-09-23T20:51:22.457 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.457 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.458 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:51:22.458 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.507307+0000", 2023-09-23T20:51:22.458 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.2039e-05 2023-09-23T20:51:22.458 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.458 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.458 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:51:22.458 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.507314+0000", 2023-09-23T20:51:22.458 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.7089999999999996e-06 2023-09-23T20:51:22.458 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.458 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.458 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:22.459 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.507629+0000", 2023-09-23T20:51:22.459 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00031477799999999999 2023-09-23T20:51:22.459 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.459 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.459 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:51:22.459 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.507641+0000", 2023-09-23T20:51:22.459 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.258e-05 2023-09-23T20:51:22.459 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.459 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.459 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:22.459 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.472754+0000", 2023-09-23T20:51:22.459 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.96511288200000001 2023-09-23T20:51:22.460 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.460 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.460 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:51:22.460 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.472887+0000", 2023-09-23T20:51:22.460 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000133132 2023-09-23T20:51:22.460 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.460 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.460 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:51:22.460 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.472997+0000", 2023-09-23T20:51:22.460 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000109805 2023-09-23T20:51:22.460 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.461 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.461 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:51:22.461 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.484044+0000", 2023-09-23T20:51:22.461 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.011047250999999999 2023-09-23T20:51:22.461 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.461 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.461 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:22.461 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.493403+0000", 2023-09-23T20:51:22.461 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0093591230000000004 2023-09-23T20:51:22.461 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.461 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.461 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:22.462 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.493533+0000", 2023-09-23T20:51:22.462 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000129271 2023-09-23T20:51:22.462 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.462 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.462 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:51:22.462 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.493558+0000", 2023-09-23T20:51:22.462 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.5477e-05 2023-09-23T20:51:22.462 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.462 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.462 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:51:22.462 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.493585+0000", 2023-09-23T20:51:22.462 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.7274e-05 2023-09-23T20:51:22.463 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:22.463 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:51:22.463 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:22.463 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.463 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.463 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:76972 9.b 9:d979209f:::benchmark_data_smithi018_381516_object76971:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e465)", 2023-09-23T20:51:22.463 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:41:58.507469+0000", 2023-09-23T20:51:22.463 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 563.93615149300001, 2023-09-23T20:51:22.463 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.98628621599999999, 2023-09-23T20:51:22.463 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:51:22.463 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:22.463 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:51:22.463 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:51:22.464 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:51:22.464 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 76972 2023-09-23T20:51:22.464 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.464 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:51:22.464 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.464 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:51:22.464 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.507469+0000", 2023-09-23T20:51:22.464 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:22.464 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.464 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.464 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:51:22.465 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.507469+0000", 2023-09-23T20:51:22.465 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:22.465 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.465 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.465 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:51:22.465 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.507468+0000", 2023-09-23T20:51:22.465 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:51:22.465 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.465 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.465 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:51:22.465 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.507525+0000", 2023-09-23T20:51:22.465 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.6431000000000002e-05 2023-09-23T20:51:22.466 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.466 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.466 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:51:22.466 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.507533+0000", 2023-09-23T20:51:22.466 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.2400000000000007e-06 2023-09-23T20:51:22.466 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.466 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.466 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:51:22.466 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.507538+0000", 2023-09-23T20:51:22.466 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.2129999999999999e-06 2023-09-23T20:51:22.466 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.466 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.467 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:22.467 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.507674+0000", 2023-09-23T20:51:22.467 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00013521599999999999 2023-09-23T20:51:22.467 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.467 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.467 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:51:22.467 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:58.507677+0000", 2023-09-23T20:51:22.467 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.3409999999999999e-06 2023-09-23T20:51:22.467 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.467 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.467 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:22.467 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.478437+0000", 2023-09-23T20:51:22.468 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.97076035400000005 2023-09-23T20:51:22.468 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.468 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.468 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:51:22.468 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.478570+0000", 2023-09-23T20:51:22.468 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00013248599999999999 2023-09-23T20:51:22.468 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.468 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.468 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,5", 2023-09-23T20:51:22.468 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.478661+0000", 2023-09-23T20:51:22.468 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.1013000000000004e-05 2023-09-23T20:51:22.469 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.469 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.469 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:51:22.469 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.486788+0000", 2023-09-23T20:51:22.469 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0081275240000000006 2023-09-23T20:51:22.469 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.469 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.469 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:22.469 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.493688+0000", 2023-09-23T20:51:22.469 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0068995159999999996 2023-09-23T20:51:22.469 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.469 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.469 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:22.470 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.493704+0000", 2023-09-23T20:51:22.470 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.6441999999999999e-05 2023-09-23T20:51:22.470 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.470 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.470 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:51:22.470 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.493734+0000", 2023-09-23T20:51:22.470 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.9258999999999999e-05 2023-09-23T20:51:22.470 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.470 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.470 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:51:22.470 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:41:59.493755+0000", 2023-09-23T20:51:22.471 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.1636000000000001e-05 2023-09-23T20:51:22.471 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:22.471 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:51:22.471 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:22.471 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.471 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.471 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:102263 10.e 10:7fa4b63f:::benchmark_data_smithi018_422431_object102262:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e551)", 2023-09-23T20:51:22.471 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:47:08.130511+0000", 2023-09-23T20:51:22.471 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 254.31310976699999, 2023-09-23T20:51:22.471 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.028872988, 2023-09-23T20:51:22.471 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:51:22.471 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:51:22.472 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:51:22.472 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:51:22.472 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:51:22.472 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 102263 2023-09-23T20:51:22.472 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.472 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:51:22.472 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.472 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:51:22.472 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.130511+0000", 2023-09-23T20:51:22.472 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:22.472 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.472 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.473 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:51:22.473 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.130511+0000", 2023-09-23T20:51:22.473 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:22.473 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.473 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.473 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:51:22.473 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.130510+0000", 2023-09-23T20:51:22.473 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:51:22.473 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.473 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.473 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:51:22.473 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.130540+0000", 2023-09-23T20:51:22.474 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.9640000000000001e-05 2023-09-23T20:51:22.474 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.474 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.474 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:51:22.474 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.130547+0000", 2023-09-23T20:51:22.474 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.5390000000000002e-06 2023-09-23T20:51:22.474 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.474 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.474 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:51:22.474 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.130550+0000", 2023-09-23T20:51:22.474 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.9979999999999999e-06 2023-09-23T20:51:22.475 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.475 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.475 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:22.475 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.130601+0000", 2023-09-23T20:51:22.475 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.0937999999999999e-05 2023-09-23T20:51:22.475 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.475 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.475 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:51:22.475 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.130731+0000", 2023-09-23T20:51:22.475 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00012974899999999999 2023-09-23T20:51:22.475 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.475 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.476 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 2,4,5", 2023-09-23T20:51:22.476 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.130882+0000", 2023-09-23T20:51:22.476 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00015048199999999999 2023-09-23T20:51:22.476 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.476 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.476 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:51:22.476 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.137064+0000", 2023-09-23T20:51:22.476 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0061819739999999998 2023-09-23T20:51:22.476 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.476 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.476 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:22.476 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.137121+0000", 2023-09-23T20:51:22.477 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.7302999999999997e-05 2023-09-23T20:51:22.477 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.477 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.477 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:22.477 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.139210+0000", 2023-09-23T20:51:22.477 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.002089184 2023-09-23T20:51:22.477 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.477 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.477 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:22.477 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.173918+0000", 2023-09-23T20:51:22.477 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.034707835999999999 2023-09-23T20:51:22.477 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.478 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.478 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:51:22.478 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.173930+0000", 2023-09-23T20:51:22.478 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.185e-05 2023-09-23T20:51:22.478 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.478 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.478 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:22.478 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.159372+0000", 2023-09-23T20:51:22.478 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.98544211999999998 2023-09-23T20:51:22.478 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.478 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.478 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:51:22.479 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.159384+0000", 2023-09-23T20:51:22.479 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1737e-05 2023-09-23T20:51:22.479 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:22.479 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:51:22.479 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:22.479 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.479 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.479 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:102264 10.e 10:70bfcac0:::benchmark_data_smithi018_422431_object102263:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e551)", 2023-09-23T20:51:22.479 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:47:08.130562+0000", 2023-09-23T20:51:22.479 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 254.31305847600001, 2023-09-23T20:51:22.479 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.028857202, 2023-09-23T20:51:22.480 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:51:22.480 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:51:22.480 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:51:22.480 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:51:22.480 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:51:22.480 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 102264 2023-09-23T20:51:22.480 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.480 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:51:22.480 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.480 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:51:22.480 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.130562+0000", 2023-09-23T20:51:22.481 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:22.481 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.481 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.481 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:51:22.481 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.130562+0000", 2023-09-23T20:51:22.481 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:22.481 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.481 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.481 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:51:22.481 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.130561+0000", 2023-09-23T20:51:22.481 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:51:22.481 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.482 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.482 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:51:22.482 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.130598+0000", 2023-09-23T20:51:22.482 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.6294999999999997e-05 2023-09-23T20:51:22.482 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.482 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.482 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:51:22.482 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.130602+0000", 2023-09-23T20:51:22.482 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.5379999999999996e-06 2023-09-23T20:51:22.482 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.482 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.483 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:51:22.483 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.130605+0000", 2023-09-23T20:51:22.483 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.909e-06 2023-09-23T20:51:22.483 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.483 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.483 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:22.483 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.134028+0000", 2023-09-23T20:51:22.483 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0034227910000000001 2023-09-23T20:51:22.483 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.483 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.483 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:51:22.483 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.134217+0000", 2023-09-23T20:51:22.484 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00018908099999999999 2023-09-23T20:51:22.484 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.484 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.484 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 2,4,5", 2023-09-23T20:51:22.484 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.134391+0000", 2023-09-23T20:51:22.484 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00017426099999999999 2023-09-23T20:51:22.484 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.484 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.484 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:51:22.484 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.137246+0000", 2023-09-23T20:51:22.484 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0028549700000000001 2023-09-23T20:51:22.484 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.485 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.485 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:22.485 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.138047+0000", 2023-09-23T20:51:22.485 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00080048900000000004 2023-09-23T20:51:22.485 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.485 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.485 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:22.485 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.139365+0000", 2023-09-23T20:51:22.485 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.001318512 2023-09-23T20:51:22.485 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.485 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.486 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:22.486 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.174065+0000", 2023-09-23T20:51:22.486 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.034699636999999998 2023-09-23T20:51:22.486 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.486 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.486 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:51:22.486 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.174076+0000", 2023-09-23T20:51:22.486 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0655999999999999e-05 2023-09-23T20:51:22.486 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.486 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.486 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:22.487 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.159411+0000", 2023-09-23T20:51:22.487 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.98533562799999996 2023-09-23T20:51:22.487 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.487 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.487 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:51:22.487 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.159419+0000", 2023-09-23T20:51:22.487 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.9950000000000005e-06 2023-09-23T20:51:22.487 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:22.487 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:51:22.487 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:22.487 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.488 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.488 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:102265 10.d 10:b077b11a:::benchmark_data_smithi018_422431_object102264:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e551)", 2023-09-23T20:51:22.488 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:47:08.130611+0000", 2023-09-23T20:51:22.488 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 254.313009459, 2023-09-23T20:51:22.488 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.088474049, 2023-09-23T20:51:22.488 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:51:22.488 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:51:22.488 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:51:22.488 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:51:22.488 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:51:22.488 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 102265 2023-09-23T20:51:22.488 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.489 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:51:22.489 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.489 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:51:22.489 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.130611+0000", 2023-09-23T20:51:22.489 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:22.489 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.489 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.489 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:51:22.489 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.130611+0000", 2023-09-23T20:51:22.489 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:22.489 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.489 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.490 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:51:22.490 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.130610+0000", 2023-09-23T20:51:22.490 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:51:22.490 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.490 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.490 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:51:22.490 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.130655+0000", 2023-09-23T20:51:22.490 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.4501e-05 2023-09-23T20:51:22.490 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.490 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.491 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:51:22.491 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.130663+0000", 2023-09-23T20:51:22.491 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.1429999999999998e-06 2023-09-23T20:51:22.491 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.491 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.491 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:51:22.491 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.130666+0000", 2023-09-23T20:51:22.491 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.039e-06 2023-09-23T20:51:22.491 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.491 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.491 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:22.491 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.130701+0000", 2023-09-23T20:51:22.492 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.5135000000000002e-05 2023-09-23T20:51:22.492 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.492 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.492 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:51:22.492 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.130794+0000", 2023-09-23T20:51:22.492 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.31e-05 2023-09-23T20:51:22.492 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.492 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.492 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:51:22.492 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.130902+0000", 2023-09-23T20:51:22.492 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00010738699999999999 2023-09-23T20:51:22.493 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.493 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.493 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:51:22.493 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.134006+0000", 2023-09-23T20:51:22.493 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0031044319999999999 2023-09-23T20:51:22.493 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.493 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.493 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:22.493 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.143177+0000", 2023-09-23T20:51:22.493 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.009170447 2023-09-23T20:51:22.493 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.493 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.494 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:22.494 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.177193+0000", 2023-09-23T20:51:22.494 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.034015861000000001 2023-09-23T20:51:22.494 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.494 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.494 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:51:22.494 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.177202+0000", 2023-09-23T20:51:22.494 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.8379999999999997e-06 2023-09-23T20:51:22.494 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.494 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.494 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:22.495 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.218917+0000", 2023-09-23T20:51:22.495 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0417144490000001 2023-09-23T20:51:22.495 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.495 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.495 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:51:22.495 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.219085+0000", 2023-09-23T20:51:22.495 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00016819 2023-09-23T20:51:22.495 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:22.495 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:51:22.495 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:22.495 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.496 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.496 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:102284 10.0 10:0c1d13a0:::benchmark_data_smithi018_422431_object102283:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e552)", 2023-09-23T20:51:22.496 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:47:08.192917+0000", 2023-09-23T20:51:22.496 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 254.25070313099999, 2023-09-23T20:51:22.496 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1174581180000001, 2023-09-23T20:51:22.496 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:51:22.496 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:22.496 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:51:22.496 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:51:22.496 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:51:22.496 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 102284 2023-09-23T20:51:22.496 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.497 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:51:22.497 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.497 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:51:22.497 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.192917+0000", 2023-09-23T20:51:22.497 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:22.497 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.497 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.497 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:51:22.497 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.192917+0000", 2023-09-23T20:51:22.497 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:22.497 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.497 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.498 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:51:22.498 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.192916+0000", 2023-09-23T20:51:22.498 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:51:22.498 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.498 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.498 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:51:22.498 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.192958+0000", 2023-09-23T20:51:22.498 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.1974999999999997e-05 2023-09-23T20:51:22.498 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.498 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.498 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:51:22.498 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.193237+0000", 2023-09-23T20:51:22.499 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00027918399999999998 2023-09-23T20:51:22.499 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.499 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.499 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:51:22.499 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.193247+0000", 2023-09-23T20:51:22.499 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0015e-05 2023-09-23T20:51:22.499 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.499 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.499 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:22.499 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.193334+0000", 2023-09-23T20:51:22.499 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.6867999999999996e-05 2023-09-23T20:51:22.499 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.500 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.500 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:51:22.500 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.193351+0000", 2023-09-23T20:51:22.500 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.6545000000000002e-05 2023-09-23T20:51:22.500 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.500 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.500 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:22.500 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.216597+0000", 2023-09-23T20:51:22.500 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0232465260000001 2023-09-23T20:51:22.500 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.500 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.500 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:51:22.501 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.216764+0000", 2023-09-23T20:51:22.501 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000166937 2023-09-23T20:51:22.501 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.501 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.501 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 1,4", 2023-09-23T20:51:22.501 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.216936+0000", 2023-09-23T20:51:22.501 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00017148099999999999 2023-09-23T20:51:22.501 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.501 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.501 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:51:22.501 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.309749+0000", 2023-09-23T20:51:22.501 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.092813393999999994 2023-09-23T20:51:22.501 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.502 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.502 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:22.502 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.309824+0000", 2023-09-23T20:51:22.502 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.5284999999999995e-05 2023-09-23T20:51:22.502 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.502 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.502 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:22.502 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.309902+0000", 2023-09-23T20:51:22.502 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.8061999999999999e-05 2023-09-23T20:51:22.502 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.502 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.502 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:51:22.503 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.309995+0000", 2023-09-23T20:51:22.503 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.2719000000000005e-05 2023-09-23T20:51:22.503 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.503 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.503 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:51:22.503 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.310375+0000", 2023-09-23T20:51:22.503 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00038043700000000002 2023-09-23T20:51:22.503 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:22.503 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:51:22.503 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:22.503 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.504 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.504 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:102291 10.0 10:0574a143:::benchmark_data_smithi018_422431_object102290:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e552)", 2023-09-23T20:51:22.504 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:47:08.213037+0000", 2023-09-23T20:51:22.504 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 254.23058321900001, 2023-09-23T20:51:22.504 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.097677486, 2023-09-23T20:51:22.504 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:51:22.504 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:22.504 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:51:22.504 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:51:22.504 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:51:22.504 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 102291 2023-09-23T20:51:22.504 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.505 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:51:22.505 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.505 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:51:22.505 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.213037+0000", 2023-09-23T20:51:22.505 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:22.505 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.505 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.505 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:51:22.505 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.213037+0000", 2023-09-23T20:51:22.505 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:22.505 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.505 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.506 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:51:22.506 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.213035+0000", 2023-09-23T20:51:22.506 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999981 2023-09-23T20:51:22.506 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.506 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.506 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:51:22.506 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.213078+0000", 2023-09-23T20:51:22.506 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.3106999999999999e-05 2023-09-23T20:51:22.506 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.506 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.506 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:51:22.506 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.213095+0000", 2023-09-23T20:51:22.507 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.6200000000000001e-05 2023-09-23T20:51:22.507 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.507 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.507 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:51:22.507 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.213102+0000", 2023-09-23T20:51:22.507 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.2779999999999996e-06 2023-09-23T20:51:22.507 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.507 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.507 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:22.507 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.213160+0000", 2023-09-23T20:51:22.507 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.838e-05 2023-09-23T20:51:22.507 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.508 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.508 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:51:22.508 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.213176+0000", 2023-09-23T20:51:22.508 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.5404999999999999e-05 2023-09-23T20:51:22.508 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.508 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.508 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:22.508 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.220512+0000", 2023-09-23T20:51:22.508 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.00733669 2023-09-23T20:51:22.508 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.508 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.508 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:51:22.509 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.220642+0000", 2023-09-23T20:51:22.509 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00012919 2023-09-23T20:51:22.509 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.509 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.509 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 1,4", 2023-09-23T20:51:22.509 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.220797+0000", 2023-09-23T20:51:22.509 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00015535599999999999 2023-09-23T20:51:22.509 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.509 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.509 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:51:22.509 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.309757+0000", 2023-09-23T20:51:22.509 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.088960045000000001 2023-09-23T20:51:22.510 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.510 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.510 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:22.510 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.309865+0000", 2023-09-23T20:51:22.510 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000108051 2023-09-23T20:51:22.510 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.510 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.510 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:22.510 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.310430+0000", 2023-09-23T20:51:22.510 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000565126 2023-09-23T20:51:22.510 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.511 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.511 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:51:22.511 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.310482+0000", 2023-09-23T20:51:22.511 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.1752999999999997e-05 2023-09-23T20:51:22.511 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.511 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.511 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:51:22.511 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.310715+0000", 2023-09-23T20:51:22.511 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00023280499999999999 2023-09-23T20:51:22.511 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:22.511 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:51:22.512 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:22.512 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.512 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.512 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:102300 10.d 10:b3326d1e:::benchmark_data_smithi018_422431_object102299:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e552)", 2023-09-23T20:51:22.512 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:47:08.260136+0000", 2023-09-23T20:51:22.512 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 254.18348415200001, 2023-09-23T20:51:22.512 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0960319730000001, 2023-09-23T20:51:22.512 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:51:22.512 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:22.512 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:51:22.512 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:51:22.512 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:51:22.512 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 102300 2023-09-23T20:51:22.513 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.513 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:51:22.513 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.513 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:51:22.513 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.260136+0000", 2023-09-23T20:51:22.513 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:22.513 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.513 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.513 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:51:22.513 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.260136+0000", 2023-09-23T20:51:22.513 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:22.513 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.514 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.514 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:51:22.514 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.260135+0000", 2023-09-23T20:51:22.514 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:51:22.514 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.514 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.514 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:51:22.514 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.260169+0000", 2023-09-23T20:51:22.514 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.4019000000000001e-05 2023-09-23T20:51:22.514 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.514 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.515 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:51:22.515 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.260183+0000", 2023-09-23T20:51:22.515 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.4387e-05 2023-09-23T20:51:22.515 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.515 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.515 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:51:22.515 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.260189+0000", 2023-09-23T20:51:22.515 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.0990000000000004e-06 2023-09-23T20:51:22.515 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.515 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.515 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:22.515 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.260275+0000", 2023-09-23T20:51:22.516 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.5556000000000006e-05 2023-09-23T20:51:22.516 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.516 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.516 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:51:22.516 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.260285+0000", 2023-09-23T20:51:22.516 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.9189999999999994e-06 2023-09-23T20:51:22.516 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.516 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.516 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:22.516 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.219112+0000", 2023-09-23T20:51:22.516 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.95882748600000001 2023-09-23T20:51:22.516 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.517 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.517 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:51:22.517 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.219285+0000", 2023-09-23T20:51:22.517 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000173299 2023-09-23T20:51:22.517 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.517 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.517 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,4", 2023-09-23T20:51:22.517 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.219498+0000", 2023-09-23T20:51:22.517 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00021284799999999999 2023-09-23T20:51:22.517 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.517 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.518 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:22.518 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.310145+0000", 2023-09-23T20:51:22.518 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.090646550000000006 2023-09-23T20:51:22.518 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.518 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.518 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:22.518 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.310184+0000", 2023-09-23T20:51:22.518 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.9232e-05 2023-09-23T20:51:22.518 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.518 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.518 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:51:22.518 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.355659+0000", 2023-09-23T20:51:22.519 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.045474674 2023-09-23T20:51:22.519 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.519 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.519 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:51:22.519 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.355721+0000", 2023-09-23T20:51:22.519 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.2157999999999997e-05 2023-09-23T20:51:22.519 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.519 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.519 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:51:22.519 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.356168+0000", 2023-09-23T20:51:22.519 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000447305 2023-09-23T20:51:22.520 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:22.520 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:51:22.520 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:22.520 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.520 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.520 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:103733 10.d 10:b457bb97:::benchmark_data_smithi018_422431_object103732:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e557)", 2023-09-23T20:51:22.520 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:47:12.654166+0000", 2023-09-23T20:51:22.520 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 249.789453944, 2023-09-23T20:51:22.520 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0141292129999999, 2023-09-23T20:51:22.520 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:51:22.520 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:22.520 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:51:22.520 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:51:22.521 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:51:22.521 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 103733 2023-09-23T20:51:22.521 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.521 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:51:22.521 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.521 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:51:22.521 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:12.654166+0000", 2023-09-23T20:51:22.521 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:22.521 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.521 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.521 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:51:22.522 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:12.654166+0000", 2023-09-23T20:51:22.522 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:22.522 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.522 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.522 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:51:22.522 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:12.654165+0000", 2023-09-23T20:51:22.522 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:51:22.522 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.522 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.522 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:51:22.522 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:12.654221+0000", 2023-09-23T20:51:22.523 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.6150999999999998e-05 2023-09-23T20:51:22.523 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.523 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.523 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:51:22.523 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:12.654236+0000", 2023-09-23T20:51:22.523 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.4783000000000001e-05 2023-09-23T20:51:22.523 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.523 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.523 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:51:22.523 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:12.654243+0000", 2023-09-23T20:51:22.523 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.8680000000000001e-06 2023-09-23T20:51:22.523 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.523 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.524 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:22.524 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:12.654316+0000", 2023-09-23T20:51:22.524 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.2848000000000005e-05 2023-09-23T20:51:22.524 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.524 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.524 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:51:22.524 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:12.654335+0000", 2023-09-23T20:51:22.524 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.9057999999999999e-05 2023-09-23T20:51:22.524 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.524 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.524 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:22.524 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:13.630346+0000", 2023-09-23T20:51:22.525 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.97601110099999999 2023-09-23T20:51:22.525 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.525 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.525 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:51:22.525 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:13.630458+0000", 2023-09-23T20:51:22.525 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00011244 2023-09-23T20:51:22.525 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.525 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.525 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,4", 2023-09-23T20:51:22.525 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:13.630554+0000", 2023-09-23T20:51:22.525 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.5334999999999994e-05 2023-09-23T20:51:22.526 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.526 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.526 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:51:22.526 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:13.642112+0000", 2023-09-23T20:51:22.526 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.011558034999999999 2023-09-23T20:51:22.526 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.526 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.526 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:22.526 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:13.668168+0000", 2023-09-23T20:51:22.526 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.026056691 2023-09-23T20:51:22.526 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.527 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.527 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:22.527 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:13.668225+0000", 2023-09-23T20:51:22.527 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.6598000000000003e-05 2023-09-23T20:51:22.527 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.527 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.527 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:51:22.527 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:13.668267+0000", 2023-09-23T20:51:22.527 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.2101000000000002e-05 2023-09-23T20:51:22.527 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.527 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.527 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:51:22.528 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:13.668296+0000", 2023-09-23T20:51:22.528 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.8530999999999998e-05 2023-09-23T20:51:22.528 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:22.528 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:51:22.528 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:22.528 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.528 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.528 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:159045 10.f 10:f817bccc:::benchmark_data_smithi018_422431_object54549:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e577)", 2023-09-23T20:51:22.528 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:48:22.940491+0000", 2023-09-23T20:51:22.528 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 179.50312948499999, 2023-09-23T20:51:22.528 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.257405952, 2023-09-23T20:51:22.528 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:51:22.529 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:22.529 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:51:22.529 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:51:22.529 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:51:22.529 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 159045 2023-09-23T20:51:22.529 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.529 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:51:22.529 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.529 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:51:22.529 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.940491+0000", 2023-09-23T20:51:22.529 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:22.530 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.530 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.530 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:51:22.530 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.940491+0000", 2023-09-23T20:51:22.530 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:22.530 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.530 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.530 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:51:22.530 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.940490+0000", 2023-09-23T20:51:22.530 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:51:22.530 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.530 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.531 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:51:22.531 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.940494+0000", 2023-09-23T20:51:22.531 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.9580000000000001e-06 2023-09-23T20:51:22.531 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.531 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.531 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:51:22.531 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.940505+0000", 2023-09-23T20:51:22.531 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0905e-05 2023-09-23T20:51:22.531 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.531 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.531 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:51:22.532 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.940510+0000", 2023-09-23T20:51:22.532 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.5879999999999999e-06 2023-09-23T20:51:22.532 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.532 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.532 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:22.532 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.940558+0000", 2023-09-23T20:51:22.532 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.8007000000000003e-05 2023-09-23T20:51:22.532 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.532 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.532 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:51:22.532 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.940564+0000", 2023-09-23T20:51:22.532 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.2099999999999998e-06 2023-09-23T20:51:22.533 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.533 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.533 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:22.533 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:23.909048+0000", 2023-09-23T20:51:22.533 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.96848468799999998 2023-09-23T20:51:22.533 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.533 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.533 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:51:22.533 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:23.909060+0000", 2023-09-23T20:51:22.533 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1279e-05 2023-09-23T20:51:22.533 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.533 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.534 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:22.534 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:25.160634+0000", 2023-09-23T20:51:22.534 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.251573933 2023-09-23T20:51:22.534 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.534 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.534 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:51:22.534 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:25.160777+0000", 2023-09-23T20:51:22.534 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000142893 2023-09-23T20:51:22.534 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.534 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.534 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 2,3,4", 2023-09-23T20:51:22.535 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:25.161399+0000", 2023-09-23T20:51:22.535 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00062231400000000001 2023-09-23T20:51:22.535 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.535 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.535 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:22.535 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:25.171100+0000", 2023-09-23T20:51:22.535 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0097009049999999993 2023-09-23T20:51:22.535 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.535 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.535 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:22.535 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:25.183171+0000", 2023-09-23T20:51:22.535 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.012071236000000001 2023-09-23T20:51:22.536 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.536 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.536 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:51:22.536 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:25.185581+0000", 2023-09-23T20:51:22.536 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0024096899999999999 2023-09-23T20:51:22.536 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.536 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.536 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:22.536 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:25.197727+0000", 2023-09-23T20:51:22.536 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.012145958 2023-09-23T20:51:22.536 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.536 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.537 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:51:22.537 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:25.197780+0000", 2023-09-23T20:51:22.537 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.3816000000000001e-05 2023-09-23T20:51:22.537 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.537 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.537 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:51:22.537 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:25.197897+0000", 2023-09-23T20:51:22.537 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00011640400000000001 2023-09-23T20:51:22.537 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:22.537 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:51:22.537 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:22.537 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.538 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.538 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:181036 10.12 10.8f5c0992 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e602)", 2023-09-23T20:51:22.538 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:49:00.873954+0000", 2023-09-23T20:51:22.538 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 141.56966642, 2023-09-23T20:51:22.538 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.81278060500000004, 2023-09-23T20:51:22.538 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:51:22.538 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:51:22.538 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:51:22.538 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:51:22.538 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:51:22.538 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 181036 2023-09-23T20:51:22.538 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.539 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:51:22.539 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.539 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:51:22.539 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:00.873954+0000", 2023-09-23T20:51:22.539 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:22.539 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.539 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.539 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:51:22.539 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:00.873954+0000", 2023-09-23T20:51:22.539 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:22.539 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.540 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.540 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:51:22.540 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:00.873953+0000", 2023-09-23T20:51:22.540 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:51:22.540 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.540 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.540 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:51:22.540 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:00.873957+0000", 2023-09-23T20:51:22.540 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.5499999999999996e-06 2023-09-23T20:51:22.540 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.540 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.540 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:51:22.541 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:00.873966+0000", 2023-09-23T20:51:22.541 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.7909999999999993e-06 2023-09-23T20:51:22.541 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.541 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.541 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:51:22.541 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:00.873971+0000", 2023-09-23T20:51:22.541 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.2299999999999999e-06 2023-09-23T20:51:22.541 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.541 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.541 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:22.541 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:00.874028+0000", 2023-09-23T20:51:22.541 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.6919000000000002e-05 2023-09-23T20:51:22.542 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.542 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.542 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:51:22.542 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:00.874036+0000", 2023-09-23T20:51:22.542 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.3930000000000002e-06 2023-09-23T20:51:22.542 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.542 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.542 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:22.542 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:01.686723+0000", 2023-09-23T20:51:22.542 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.81268693700000005 2023-09-23T20:51:22.542 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.542 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.543 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:51:22.543 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:01.686735+0000", 2023-09-23T20:51:22.543 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1997000000000001e-05 2023-09-23T20:51:22.543 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:22.543 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:51:22.543 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:22.543 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.543 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.543 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:196645 10.16 10.8b6c8596 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e622)", 2023-09-23T20:51:22.543 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:49:27.061969+0000", 2023-09-23T20:51:22.543 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 115.381651257, 2023-09-23T20:51:22.544 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.91010735200000004, 2023-09-23T20:51:22.544 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:51:22.544 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:51:22.544 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:51:22.544 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:51:22.544 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:51:22.544 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 196645 2023-09-23T20:51:22.544 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.544 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:51:22.544 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.544 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:51:22.545 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:27.061969+0000", 2023-09-23T20:51:22.545 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:22.545 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.545 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.545 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:51:22.545 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:27.061969+0000", 2023-09-23T20:51:22.545 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:22.545 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.545 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.545 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:51:22.545 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:27.061969+0000", 2023-09-23T20:51:22.545 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:51:22.546 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.546 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.546 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:51:22.546 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:27.061971+0000", 2023-09-23T20:51:22.546 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.4940000000000002e-06 2023-09-23T20:51:22.546 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.546 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.546 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:51:22.546 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:27.061983+0000", 2023-09-23T20:51:22.546 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1895999999999999e-05 2023-09-23T20:51:22.546 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.547 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.547 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:51:22.547 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:27.061989+0000", 2023-09-23T20:51:22.547 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.3119999999999999e-06 2023-09-23T20:51:22.547 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.547 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.547 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:22.547 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:27.062036+0000", 2023-09-23T20:51:22.547 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.6576999999999998e-05 2023-09-23T20:51:22.547 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.547 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.547 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:51:22.548 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:27.062041+0000", 2023-09-23T20:51:22.548 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.4090000000000002e-06 2023-09-23T20:51:22.548 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.548 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.548 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:22.548 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:27.972063+0000", 2023-09-23T20:51:22.548 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.910021262 2023-09-23T20:51:22.548 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.548 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.548 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:51:22.548 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:27.972077+0000", 2023-09-23T20:51:22.548 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.3801999999999999e-05 2023-09-23T20:51:22.549 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:22.549 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:51:22.549 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:22.549 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.549 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.549 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:196677 10.16 10.1917f5f6 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e622)", 2023-09-23T20:51:22.549 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:49:27.102907+0000", 2023-09-23T20:51:22.549 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 115.34071319900001, 2023-09-23T20:51:22.549 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.86920190600000002, 2023-09-23T20:51:22.549 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:51:22.549 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:51:22.549 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:51:22.550 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:51:22.550 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:51:22.550 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 196677 2023-09-23T20:51:22.550 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.550 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:51:22.550 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.550 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:51:22.550 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:27.102907+0000", 2023-09-23T20:51:22.550 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:22.550 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.550 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.551 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:51:22.551 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:27.102907+0000", 2023-09-23T20:51:22.551 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:22.551 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.551 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.551 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:51:22.551 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:27.102907+0000", 2023-09-23T20:51:22.551 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:51:22.551 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.551 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.551 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:51:22.551 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:27.102909+0000", 2023-09-23T20:51:22.552 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.1169999999999998e-06 2023-09-23T20:51:22.552 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.552 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.552 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:51:22.552 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:27.102914+0000", 2023-09-23T20:51:22.552 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.3859999999999998e-06 2023-09-23T20:51:22.552 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.552 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.552 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:51:22.552 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:27.102918+0000", 2023-09-23T20:51:22.552 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.6049999999999998e-06 2023-09-23T20:51:22.552 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.553 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.553 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:22.553 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:27.102962+0000", 2023-09-23T20:51:22.553 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.3791000000000001e-05 2023-09-23T20:51:22.553 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.553 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.553 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:51:22.553 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:27.102969+0000", 2023-09-23T20:51:22.553 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.1169999999999998e-06 2023-09-23T20:51:22.553 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.553 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.553 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:22.554 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:27.972098+0000", 2023-09-23T20:51:22.554 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.86912921200000004 2023-09-23T20:51:22.554 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.554 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.554 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:51:22.554 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:27.972109+0000", 2023-09-23T20:51:22.554 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0988999999999999e-05 2023-09-23T20:51:22.554 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:22.554 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:51:22.554 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:22.554 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.555 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.555 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:196719 10.16 10.b6000516 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e622)", 2023-09-23T20:51:22.555 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:49:27.142134+0000", 2023-09-23T20:51:22.555 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 115.301486087, 2023-09-23T20:51:22.555 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.83000051699999999, 2023-09-23T20:51:22.555 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:51:22.555 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:51:22.555 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:51:22.555 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:51:22.555 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:51:22.555 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 196719 2023-09-23T20:51:22.555 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.556 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:51:22.556 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.556 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:51:22.556 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:27.142134+0000", 2023-09-23T20:51:22.556 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:22.556 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.556 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.556 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:51:22.556 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:27.142134+0000", 2023-09-23T20:51:22.556 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:22.556 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.557 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.557 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:51:22.557 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:27.142134+0000", 2023-09-23T20:51:22.557 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:51:22.557 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.557 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.557 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:51:22.557 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:27.142136+0000", 2023-09-23T20:51:22.557 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.5110000000000002e-06 2023-09-23T20:51:22.557 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.557 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.557 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:51:22.558 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:27.142148+0000", 2023-09-23T20:51:22.558 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.131e-05 2023-09-23T20:51:22.558 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.558 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.558 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:51:22.558 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:27.142151+0000", 2023-09-23T20:51:22.558 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.1370000000000002e-06 2023-09-23T20:51:22.558 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.558 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.558 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:22.558 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:27.142193+0000", 2023-09-23T20:51:22.558 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.2562999999999999e-05 2023-09-23T20:51:22.559 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.559 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.559 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:51:22.559 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:27.142201+0000", 2023-09-23T20:51:22.559 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.2180000000000002e-06 2023-09-23T20:51:22.559 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.559 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.559 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:22.559 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:27.972126+0000", 2023-09-23T20:51:22.559 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.82992553199999997 2023-09-23T20:51:22.559 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.559 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.560 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:51:22.560 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:27.972135+0000", 2023-09-23T20:51:22.560 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.6649999999999992e-06 2023-09-23T20:51:22.560 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:22.560 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:51:22.560 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:22.560 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.560 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.560 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:196729 10.16 10.90fdb716 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e622)", 2023-09-23T20:51:22.560 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:49:27.154937+0000", 2023-09-23T20:51:22.560 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 115.288683836, 2023-09-23T20:51:22.561 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.81722424999999999, 2023-09-23T20:51:22.561 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:51:22.561 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:51:22.561 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:51:22.561 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:51:22.561 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:51:22.561 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 196729 2023-09-23T20:51:22.561 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.561 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:51:22.561 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.561 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:51:22.561 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:27.154937+0000", 2023-09-23T20:51:22.562 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:22.562 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.562 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.562 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:51:22.562 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:27.154937+0000", 2023-09-23T20:51:22.562 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:22.562 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.562 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.562 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:51:22.562 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:27.154936+0000", 2023-09-23T20:51:22.562 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:51:22.562 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.563 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.563 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:51:22.563 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:27.154940+0000", 2023-09-23T20:51:22.563 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.4290000000000003e-06 2023-09-23T20:51:22.563 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.563 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.563 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:51:22.563 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:27.154953+0000", 2023-09-23T20:51:22.563 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.305e-05 2023-09-23T20:51:22.563 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.563 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.564 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:51:22.564 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:27.154959+0000", 2023-09-23T20:51:22.564 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.8510000000000004e-06 2023-09-23T20:51:22.564 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.564 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.564 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:22.564 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:27.155375+0000", 2023-09-23T20:51:22.564 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00041597900000000002 2023-09-23T20:51:22.564 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.564 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.564 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:51:22.564 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:27.155391+0000", 2023-09-23T20:51:22.565 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.5727999999999999e-05 2023-09-23T20:51:22.565 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.565 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.565 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:22.565 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:27.972152+0000", 2023-09-23T20:51:22.565 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.81676116600000004 2023-09-23T20:51:22.565 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.565 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.565 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:51:22.565 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:27.972161+0000", 2023-09-23T20:51:22.565 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.1239999999999994e-06 2023-09-23T20:51:22.565 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:22.566 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:51:22.566 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:22.566 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.566 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.566 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:197482 10.16 10:6f86c9d6:::benchmark_data_smithi018_422431_object92986:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e625)", 2023-09-23T20:51:22.566 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:49:30.033426+0000", 2023-09-23T20:51:22.566 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 112.410194516, 2023-09-23T20:51:22.566 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.983929371, 2023-09-23T20:51:22.566 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:51:22.566 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:22.566 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:51:22.567 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:51:22.567 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:51:22.567 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 197482 2023-09-23T20:51:22.567 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.567 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:51:22.567 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.567 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:51:22.567 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:30.033426+0000", 2023-09-23T20:51:22.567 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:22.567 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.567 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.567 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:51:22.568 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:30.033426+0000", 2023-09-23T20:51:22.568 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:22.568 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.568 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.568 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:51:22.568 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:30.033425+0000", 2023-09-23T20:51:22.568 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:51:22.568 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.568 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.568 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:51:22.568 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:30.033428+0000", 2023-09-23T20:51:22.568 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.717e-06 2023-09-23T20:51:22.569 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.569 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.569 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:51:22.569 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:30.033440+0000", 2023-09-23T20:51:22.569 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1737e-05 2023-09-23T20:51:22.569 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.569 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.569 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:51:22.569 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:30.033444+0000", 2023-09-23T20:51:22.569 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.1060000000000002e-06 2023-09-23T20:51:22.569 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.569 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.570 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:22.570 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:30.033483+0000", 2023-09-23T20:51:22.570 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.9258000000000003e-05 2023-09-23T20:51:22.570 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.570 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.570 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:51:22.570 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:30.033490+0000", 2023-09-23T20:51:22.570 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.1359999999999999e-06 2023-09-23T20:51:22.570 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.570 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.570 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:22.571 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:31.011143+0000", 2023-09-23T20:51:22.571 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.97765219999999997 2023-09-23T20:51:22.571 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.571 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.571 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:51:22.571 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:31.011287+0000", 2023-09-23T20:51:22.571 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00014409500000000001 2023-09-23T20:51:22.571 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.571 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.571 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 1,5", 2023-09-23T20:51:22.571 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:31.011543+0000", 2023-09-23T20:51:22.571 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00025618400000000001 2023-09-23T20:51:22.572 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.572 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.572 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:51:22.572 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:31.015182+0000", 2023-09-23T20:51:22.572 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0036391539999999999 2023-09-23T20:51:22.572 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.572 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.572 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:22.572 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:31.017232+0000", 2023-09-23T20:51:22.572 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0020494760000000002 2023-09-23T20:51:22.572 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.573 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.573 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:22.573 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:31.017265+0000", 2023-09-23T20:51:22.573 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.3665999999999999e-05 2023-09-23T20:51:22.573 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.573 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.573 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:51:22.573 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:31.017311+0000", 2023-09-23T20:51:22.573 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.5797999999999998e-05 2023-09-23T20:51:22.573 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.573 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.573 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:51:22.574 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:31.017355+0000", 2023-09-23T20:51:22.574 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.4283999999999998e-05 2023-09-23T20:51:22.574 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:22.574 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:51:22.574 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:22.574 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.574 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.574 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:197491 10.16 10:6bb90d38:::benchmark_data_smithi018_422431_object92995:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e625)", 2023-09-23T20:51:22.574 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:49:30.040972+0000", 2023-09-23T20:51:22.574 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 112.402648818, 2023-09-23T20:51:22.574 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.97653193900000002, 2023-09-23T20:51:22.575 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:51:22.575 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:22.575 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:51:22.575 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:51:22.575 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:51:22.575 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 197491 2023-09-23T20:51:22.575 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.575 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:51:22.575 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.575 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:51:22.575 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:30.040972+0000", 2023-09-23T20:51:22.575 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:22.576 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.576 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.576 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:51:22.576 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:30.040972+0000", 2023-09-23T20:51:22.576 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:22.576 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.576 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.576 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:51:22.576 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:30.040971+0000", 2023-09-23T20:51:22.576 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:51:22.576 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.576 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.577 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:51:22.577 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:30.040976+0000", 2023-09-23T20:51:22.577 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.6469999999999997e-06 2023-09-23T20:51:22.577 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.577 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.577 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:51:22.577 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:30.040987+0000", 2023-09-23T20:51:22.577 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1182999999999999e-05 2023-09-23T20:51:22.577 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.577 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.577 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:51:22.577 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:30.040993+0000", 2023-09-23T20:51:22.578 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.3049999999999998e-06 2023-09-23T20:51:22.578 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.578 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.578 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:22.578 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:30.041034+0000", 2023-09-23T20:51:22.578 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.0865999999999998e-05 2023-09-23T20:51:22.578 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.578 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.578 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:51:22.578 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:30.041043+0000", 2023-09-23T20:51:22.578 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.5580000000000001e-06 2023-09-23T20:51:22.579 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.579 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.579 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:22.579 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:31.012597+0000", 2023-09-23T20:51:22.579 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.97155400199999997 2023-09-23T20:51:22.579 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.579 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.579 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:51:22.579 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:31.012707+0000", 2023-09-23T20:51:22.579 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00011072499999999999 2023-09-23T20:51:22.579 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.579 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.580 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 1,5", 2023-09-23T20:51:22.580 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:31.012887+0000", 2023-09-23T20:51:22.580 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00017996900000000001 2023-09-23T20:51:22.580 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.580 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.580 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:51:22.580 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:31.016913+0000", 2023-09-23T20:51:22.580 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0040260560000000001 2023-09-23T20:51:22.580 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.580 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.580 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:22.581 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:31.017414+0000", 2023-09-23T20:51:22.581 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00050053200000000004 2023-09-23T20:51:22.581 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.581 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.581 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:22.581 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:31.017459+0000", 2023-09-23T20:51:22.581 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.5037000000000003e-05 2023-09-23T20:51:22.581 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.581 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.581 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:51:22.581 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:31.017484+0000", 2023-09-23T20:51:22.581 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.5559e-05 2023-09-23T20:51:22.582 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.582 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.582 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:51:22.582 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:31.017504+0000", 2023-09-23T20:51:22.582 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.9170000000000001e-05 2023-09-23T20:51:22.582 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:22.582 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:51:22.582 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:22.582 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.582 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.582 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:197532 10.16 10:68d215d5:::benchmark_data_smithi018_422431_object93036:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e625)", 2023-09-23T20:51:22.582 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:49:30.092702+0000", 2023-09-23T20:51:22.583 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 112.350918077, 2023-09-23T20:51:22.583 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.92504978999999998, 2023-09-23T20:51:22.583 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:51:22.583 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:22.583 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:51:22.583 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:51:22.583 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:51:22.583 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 197532 2023-09-23T20:51:22.583 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.583 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:51:22.583 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.584 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:51:22.584 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:30.092702+0000", 2023-09-23T20:51:22.584 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:22.584 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.584 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.584 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:51:22.584 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:30.092702+0000", 2023-09-23T20:51:22.584 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:22.584 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.584 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.584 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:51:22.584 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:30.092702+0000", 2023-09-23T20:51:22.585 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:51:22.585 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.585 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.585 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:51:22.585 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:30.092705+0000", 2023-09-23T20:51:22.585 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.4379999999999999e-06 2023-09-23T20:51:22.585 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.585 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.585 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:51:22.585 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:30.092713+0000", 2023-09-23T20:51:22.585 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.3159999999999997e-06 2023-09-23T20:51:22.585 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.586 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.586 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:51:22.586 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:30.092717+0000", 2023-09-23T20:51:22.586 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.6440000000000002e-06 2023-09-23T20:51:22.586 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.586 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.586 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:22.586 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:30.092756+0000", 2023-09-23T20:51:22.586 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.9051999999999998e-05 2023-09-23T20:51:22.586 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.586 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.587 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:51:22.587 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:30.092763+0000", 2023-09-23T20:51:22.587 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.1919999999999999e-06 2023-09-23T20:51:22.587 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.587 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.587 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:22.587 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:31.013901+0000", 2023-09-23T20:51:22.587 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.92113773099999996 2023-09-23T20:51:22.587 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.587 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.587 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:51:22.588 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:31.014021+0000", 2023-09-23T20:51:22.588 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000119795 2023-09-23T20:51:22.588 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.588 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.588 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 1,5", 2023-09-23T20:51:22.588 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:31.014210+0000", 2023-09-23T20:51:22.588 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00018928400000000001 2023-09-23T20:51:22.588 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.588 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.588 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:51:22.588 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:31.017203+0000", 2023-09-23T20:51:22.588 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.002993276 2023-09-23T20:51:22.589 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.589 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.589 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:22.589 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:31.017606+0000", 2023-09-23T20:51:22.589 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00040277599999999998 2023-09-23T20:51:22.589 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.589 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.589 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:22.589 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:31.017695+0000", 2023-09-23T20:51:22.589 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.8819999999999993e-05 2023-09-23T20:51:22.589 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.589 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.589 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:51:22.590 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:31.017722+0000", 2023-09-23T20:51:22.590 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.7118e-05 2023-09-23T20:51:22.590 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.590 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.590 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:51:22.590 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:31.017752+0000", 2023-09-23T20:51:22.590 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.0105999999999999e-05 2023-09-23T20:51:22.590 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:22.590 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:51:22.590 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:22.590 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.591 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.591 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:197567 10.16 10:6a554823:::benchmark_data_smithi018_422431_object93071:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e625)", 2023-09-23T20:51:22.591 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:49:30.199348+0000", 2023-09-23T20:51:22.591 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 112.244272733, 2023-09-23T20:51:22.591 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.81852593600000001, 2023-09-23T20:51:22.591 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:51:22.591 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:22.591 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:51:22.591 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:51:22.591 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:51:22.591 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 197567 2023-09-23T20:51:22.591 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.592 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:51:22.592 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.592 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:51:22.592 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:30.199348+0000", 2023-09-23T20:51:22.592 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:22.592 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.592 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.592 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:51:22.592 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:30.199348+0000", 2023-09-23T20:51:22.592 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:22.592 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.592 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.593 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:51:22.593 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:30.199347+0000", 2023-09-23T20:51:22.593 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:51:22.593 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.593 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.593 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:51:22.593 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:30.199351+0000", 2023-09-23T20:51:22.593 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.1629999999999998e-06 2023-09-23T20:51:22.593 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.593 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.593 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:51:22.594 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:30.199361+0000", 2023-09-23T20:51:22.594 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.5719999999999992e-06 2023-09-23T20:51:22.594 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.594 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.594 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:51:22.594 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:30.199365+0000", 2023-09-23T20:51:22.594 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.3939999999999998e-06 2023-09-23T20:51:22.594 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.594 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.594 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:22.594 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:30.199415+0000", 2023-09-23T20:51:22.594 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.9792e-05 2023-09-23T20:51:22.595 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.595 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.595 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:51:22.595 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:30.199425+0000", 2023-09-23T20:51:22.595 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0128e-05 2023-09-23T20:51:22.595 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.595 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.595 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:22.595 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:31.015227+0000", 2023-09-23T20:51:22.595 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.815802374 2023-09-23T20:51:22.595 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.595 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.596 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:51:22.596 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:31.015372+0000", 2023-09-23T20:51:22.596 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00014513800000000001 2023-09-23T20:51:22.596 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.596 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.596 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 1,5", 2023-09-23T20:51:22.596 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:31.015601+0000", 2023-09-23T20:51:22.596 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000228777 2023-09-23T20:51:22.596 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.596 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.596 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:51:22.597 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:31.017237+0000", 2023-09-23T20:51:22.597 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0016358830000000001 2023-09-23T20:51:22.597 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.597 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.597 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:22.597 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:31.017657+0000", 2023-09-23T20:51:22.597 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00041998300000000002 2023-09-23T20:51:22.597 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.597 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.597 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:22.597 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:31.017803+0000", 2023-09-23T20:51:22.598 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00014541399999999999 2023-09-23T20:51:22.598 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.598 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.598 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:51:22.598 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:31.017848+0000", 2023-09-23T20:51:22.598 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.5249e-05 2023-09-23T20:51:22.598 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:22.598 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:22.598 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:51:22.598 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:31.017874+0000", 2023-09-23T20:51:22.598 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.5839000000000001e-05 2023-09-23T20:51:22.598 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:22.599 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:51:22.599 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:22.599 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:22.599 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:51:22.599 INFO:teuthology.orchestra.run.smithi149.stdout:} 2023-09-23T20:51:22.878 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:22 smithi149 ceph-mon[170394]: failed to encode map e674 with expected crc 2023-09-23T20:51:22.878 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:22 smithi149 ceph-mon[170394]: failed to encode map e674 with expected crc 2023-09-23T20:51:22.878 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:22 smithi149 ceph-mon[170394]: failed to encode map e674 with expected crc 2023-09-23T20:51:22.878 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:22 smithi149 ceph-mon[170394]: failed to encode map e674 with expected crc 2023-09-23T20:51:22.878 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:22 smithi149 ceph-mon[170394]: failed to encode map e674 with expected crc 2023-09-23T20:51:22.878 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:22 smithi149 ceph-mon[170394]: failed to encode map e674 with expected crc 2023-09-23T20:51:22.878 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:22 smithi149 ceph-mon[170394]: failed to encode map e674 with expected crc 2023-09-23T20:51:22.878 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:22 smithi149 ceph-mon[170394]: failed to encode map e674 with expected crc 2023-09-23T20:51:22.878 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:22 smithi149 ceph-mon[170394]: osdmap e675: 8 total, 8 up, 8 in 2023-09-23T20:51:22.879 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T20:51:23.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:22 smithi018 ceph-mon[179647]: failed to encode map e674 with expected crc 2023-09-23T20:51:23.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:22 smithi018 ceph-mon[179647]: failed to encode map e674 with expected crc 2023-09-23T20:51:23.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:22 smithi018 ceph-mon[179647]: failed to encode map e674 with expected crc 2023-09-23T20:51:23.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:22 smithi018 ceph-mon[179647]: failed to encode map e674 with expected crc 2023-09-23T20:51:23.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:22 smithi018 ceph-mon[179647]: failed to encode map e674 with expected crc 2023-09-23T20:51:23.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:22 smithi018 ceph-mon[179647]: failed to encode map e674 with expected crc 2023-09-23T20:51:23.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:22 smithi018 ceph-mon[179647]: failed to encode map e674 with expected crc 2023-09-23T20:51:23.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:22 smithi018 ceph-mon[179647]: failed to encode map e674 with expected crc 2023-09-23T20:51:23.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:22 smithi018 ceph-mon[179647]: osdmap e675: 8 total, 8 up, 8 in 2023-09-23T20:51:23.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:22 smithi018 ceph-mon[184144]: failed to encode map e674 with expected crc 2023-09-23T20:51:23.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:22 smithi018 ceph-mon[184144]: failed to encode map e674 with expected crc 2023-09-23T20:51:23.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:22 smithi018 ceph-mon[184144]: failed to encode map e674 with expected crc 2023-09-23T20:51:23.143 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:22 smithi018 ceph-mon[184144]: failed to encode map e674 with expected crc 2023-09-23T20:51:23.144 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:22 smithi018 ceph-mon[184144]: failed to encode map e674 with expected crc 2023-09-23T20:51:23.144 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:22 smithi018 ceph-mon[184144]: failed to encode map e674 with expected crc 2023-09-23T20:51:23.144 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:22 smithi018 ceph-mon[184144]: failed to encode map e674 with expected crc 2023-09-23T20:51:23.144 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:22 smithi018 ceph-mon[184144]: failed to encode map e674 with expected crc 2023-09-23T20:51:23.144 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:22 smithi018 ceph-mon[184144]: osdmap e675: 8 total, 8 up, 8 in 2023-09-23T20:51:23.169 INFO:teuthology.orchestra.run.smithi018.stderr:Error EINVAL: specified pgp_num 91 > pg_num 31 2023-09-23T20:51:23.484 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:51:23.514 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-09-23T20:51:23.515 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph -- tell 'mon.*' injectargs --mon_osd_down_out_interval=600 2023-09-23T20:51:23.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:23 smithi149 ceph-mon[170394]: pgmap v1648: 103 pgs: 4 remapped+peering, 3 peering, 96 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-09-23T20:51:23.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:23 smithi149 ceph-mon[170394]: failed to encode map e675 with expected crc 2023-09-23T20:51:23.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:23 smithi149 ceph-mon[170394]: failed to encode map e675 with expected crc 2023-09-23T20:51:23.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:23 smithi149 ceph-mon[170394]: failed to encode map e675 with expected crc 2023-09-23T20:51:23.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:23 smithi149 ceph-mon[170394]: failed to encode map e675 with expected crc 2023-09-23T20:51:23.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:23 smithi149 ceph-mon[170394]: failed to encode map e675 with expected crc 2023-09-23T20:51:23.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:23 smithi149 ceph-mon[170394]: failed to encode map e675 with expected crc 2023-09-23T20:51:23.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:23 smithi149 ceph-mon[170394]: failed to encode map e675 with expected crc 2023-09-23T20:51:23.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:23 smithi149 ceph-mon[170394]: failed to encode map e675 with expected crc 2023-09-23T20:51:23.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:23 smithi149 ceph-mon[170394]: failed to encode map e675 with expected crc 2023-09-23T20:51:23.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:23 smithi149 ceph-mon[170394]: failed to encode map e675 with expected crc 2023-09-23T20:51:23.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:23 smithi149 ceph-mon[170394]: failed to encode map e675 with expected crc 2023-09-23T20:51:23.967 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:23 smithi149 ceph-mon[170394]: failed to encode map e675 with expected crc 2023-09-23T20:51:23.968 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:23 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/60019578' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "91"}]: dispatch 2023-09-23T20:51:23.968 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:23 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "91"}]: dispatch 2023-09-23T20:51:23.968 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:23 smithi149 ceph-mon[170394]: osdmap e676: 8 total, 8 up, 8 in 2023-09-23T20:51:24.131 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:51:24.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:23 smithi018 ceph-mon[184144]: pgmap v1648: 103 pgs: 4 remapped+peering, 3 peering, 96 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-09-23T20:51:24.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:23 smithi018 ceph-mon[184144]: failed to encode map e675 with expected crc 2023-09-23T20:51:24.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:23 smithi018 ceph-mon[184144]: failed to encode map e675 with expected crc 2023-09-23T20:51:24.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:23 smithi018 ceph-mon[184144]: failed to encode map e675 with expected crc 2023-09-23T20:51:24.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:23 smithi018 ceph-mon[184144]: failed to encode map e675 with expected crc 2023-09-23T20:51:24.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:23 smithi018 ceph-mon[184144]: failed to encode map e675 with expected crc 2023-09-23T20:51:24.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:23 smithi018 ceph-mon[184144]: failed to encode map e675 with expected crc 2023-09-23T20:51:24.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:23 smithi018 ceph-mon[184144]: failed to encode map e675 with expected crc 2023-09-23T20:51:24.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:23 smithi018 ceph-mon[184144]: failed to encode map e675 with expected crc 2023-09-23T20:51:24.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:23 smithi018 ceph-mon[184144]: failed to encode map e675 with expected crc 2023-09-23T20:51:24.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:23 smithi018 ceph-mon[184144]: failed to encode map e675 with expected crc 2023-09-23T20:51:24.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:23 smithi018 ceph-mon[184144]: failed to encode map e675 with expected crc 2023-09-23T20:51:24.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:23 smithi018 ceph-mon[184144]: failed to encode map e675 with expected crc 2023-09-23T20:51:24.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:23 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/60019578' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "91"}]: dispatch 2023-09-23T20:51:24.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:23 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "91"}]: dispatch 2023-09-23T20:51:24.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:23 smithi018 ceph-mon[184144]: osdmap e676: 8 total, 8 up, 8 in 2023-09-23T20:51:24.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:23 smithi018 ceph-mon[179647]: pgmap v1648: 103 pgs: 4 remapped+peering, 3 peering, 96 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-09-23T20:51:24.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:23 smithi018 ceph-mon[179647]: failed to encode map e675 with expected crc 2023-09-23T20:51:24.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:23 smithi018 ceph-mon[179647]: failed to encode map e675 with expected crc 2023-09-23T20:51:24.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:23 smithi018 ceph-mon[179647]: failed to encode map e675 with expected crc 2023-09-23T20:51:24.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:23 smithi018 ceph-mon[179647]: failed to encode map e675 with expected crc 2023-09-23T20:51:24.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:23 smithi018 ceph-mon[179647]: failed to encode map e675 with expected crc 2023-09-23T20:51:24.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:23 smithi018 ceph-mon[179647]: failed to encode map e675 with expected crc 2023-09-23T20:51:24.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:23 smithi018 ceph-mon[179647]: failed to encode map e675 with expected crc 2023-09-23T20:51:24.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:23 smithi018 ceph-mon[179647]: failed to encode map e675 with expected crc 2023-09-23T20:51:24.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:23 smithi018 ceph-mon[179647]: failed to encode map e675 with expected crc 2023-09-23T20:51:24.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:23 smithi018 ceph-mon[179647]: failed to encode map e675 with expected crc 2023-09-23T20:51:24.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:23 smithi018 ceph-mon[179647]: failed to encode map e675 with expected crc 2023-09-23T20:51:24.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:23 smithi018 ceph-mon[179647]: failed to encode map e675 with expected crc 2023-09-23T20:51:24.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:23 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/60019578' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "91"}]: dispatch 2023-09-23T20:51:24.144 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:23 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "91"}]: dispatch 2023-09-23T20:51:24.144 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:23 smithi018 ceph-mon[179647]: osdmap e676: 8 total, 8 up, 8 in 2023-09-23T20:51:24.819 INFO:teuthology.orchestra.run.smithi149.stdout:{ 2023-09-23T20:51:24.819 INFO:teuthology.orchestra.run.smithi149.stdout: "ops": [], 2023-09-23T20:51:24.819 INFO:teuthology.orchestra.run.smithi149.stdout: "num_ops": 0 2023-09-23T20:51:24.819 INFO:teuthology.orchestra.run.smithi149.stdout:} 2023-09-23T20:51:24.842 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:24 smithi149 ceph-mon[170394]: failed to encode map e676 with expected crc 2023-09-23T20:51:24.842 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:24 smithi149 ceph-mon[170394]: failed to encode map e676 with expected crc 2023-09-23T20:51:24.842 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:24 smithi149 ceph-mon[170394]: failed to encode map e676 with expected crc 2023-09-23T20:51:24.843 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:24 smithi149 ceph-mon[170394]: failed to encode map e676 with expected crc 2023-09-23T20:51:24.843 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:24 smithi149 ceph-mon[170394]: failed to encode map e676 with expected crc 2023-09-23T20:51:24.843 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:24 smithi149 ceph-mon[170394]: failed to encode map e676 with expected crc 2023-09-23T20:51:24.843 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:24 smithi149 ceph-mon[170394]: failed to encode map e676 with expected crc 2023-09-23T20:51:24.843 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:24 smithi149 ceph-mon[170394]: failed to encode map e676 with expected crc 2023-09-23T20:51:24.843 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:24 smithi149 ceph-mon[170394]: failed to encode map e676 with expected crc 2023-09-23T20:51:24.843 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:24 smithi149 ceph-mon[170394]: osdmap e677: 8 total, 8 up, 8 in 2023-09-23T20:51:24.913 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:24 smithi018 ceph-mon[179647]: failed to encode map e676 with expected crc 2023-09-23T20:51:24.913 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:24 smithi018 ceph-mon[179647]: failed to encode map e676 with expected crc 2023-09-23T20:51:24.913 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:24 smithi018 ceph-mon[179647]: failed to encode map e676 with expected crc 2023-09-23T20:51:24.913 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:24 smithi018 ceph-mon[179647]: failed to encode map e676 with expected crc 2023-09-23T20:51:24.913 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:24 smithi018 ceph-mon[179647]: failed to encode map e676 with expected crc 2023-09-23T20:51:24.913 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:24 smithi018 ceph-mon[179647]: failed to encode map e676 with expected crc 2023-09-23T20:51:24.913 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:24 smithi018 ceph-mon[179647]: failed to encode map e676 with expected crc 2023-09-23T20:51:24.913 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:24 smithi018 ceph-mon[179647]: failed to encode map e676 with expected crc 2023-09-23T20:51:24.914 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:24 smithi018 ceph-mon[179647]: failed to encode map e676 with expected crc 2023-09-23T20:51:24.914 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:24 smithi018 ceph-mon[179647]: osdmap e677: 8 total, 8 up, 8 in 2023-09-23T20:51:24.914 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:24 smithi018 ceph-mon[184144]: failed to encode map e676 with expected crc 2023-09-23T20:51:24.914 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:24 smithi018 ceph-mon[184144]: failed to encode map e676 with expected crc 2023-09-23T20:51:24.914 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:24 smithi018 ceph-mon[184144]: failed to encode map e676 with expected crc 2023-09-23T20:51:24.914 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:24 smithi018 ceph-mon[184144]: failed to encode map e676 with expected crc 2023-09-23T20:51:24.914 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:24 smithi018 ceph-mon[184144]: failed to encode map e676 with expected crc 2023-09-23T20:51:24.914 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:24 smithi018 ceph-mon[184144]: failed to encode map e676 with expected crc 2023-09-23T20:51:24.914 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:24 smithi018 ceph-mon[184144]: failed to encode map e676 with expected crc 2023-09-23T20:51:24.914 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:24 smithi018 ceph-mon[184144]: failed to encode map e676 with expected crc 2023-09-23T20:51:24.915 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:24 smithi018 ceph-mon[184144]: failed to encode map e676 with expected crc 2023-09-23T20:51:24.915 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:24 smithi018 ceph-mon[184144]: osdmap e677: 8 total, 8 up, 8 in 2023-09-23T20:51:25.168 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:51:24 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:51:24] "GET /metrics HTTP/1.1" 200 34154 "" "Prometheus/2.43.0" 2023-09-23T20:51:25.233 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T20:51:25.361 INFO:teuthology.orchestra.run.smithi018.stdout:mon.a: {} 2023-09-23T20:51:25.371 INFO:teuthology.orchestra.run.smithi018.stdout:mon.b: {} 2023-09-23T20:51:25.381 INFO:teuthology.orchestra.run.smithi018.stdout:mon.c: {} 2023-09-23T20:51:25.382 INFO:teuthology.orchestra.run.smithi018.stderr:mon.a: mon_osd_down_out_interval = '600' (not observed, change may require restart) 2023-09-23T20:51:25.383 INFO:teuthology.orchestra.run.smithi018.stderr:mon.b: mon_osd_down_out_interval = '600' (not observed, change may require restart) 2023-09-23T20:51:25.383 INFO:teuthology.orchestra.run.smithi018.stderr:mon.c: mon_osd_down_out_interval = '600' (not observed, change may require restart) 2023-09-23T20:51:25.693 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd reweight 0 1 2023-09-23T20:51:25.837 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:51:25.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:25 smithi018 ceph-mon[184144]: pgmap v1651: 103 pgs: 6 active+undersized+remapped, 4 remapped+peering, 3 peering, 90 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 1024 KiB/s, 4 objects/s recovering 2023-09-23T20:51:25.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:25 smithi018 ceph-mon[184144]: failed to encode map e677 with expected crc 2023-09-23T20:51:25.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:25 smithi018 ceph-mon[184144]: failed to encode map e677 with expected crc 2023-09-23T20:51:25.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:25 smithi018 ceph-mon[184144]: failed to encode map e677 with expected crc 2023-09-23T20:51:25.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:25 smithi018 ceph-mon[184144]: failed to encode map e677 with expected crc 2023-09-23T20:51:25.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:25 smithi018 ceph-mon[184144]: failed to encode map e677 with expected crc 2023-09-23T20:51:25.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:25 smithi018 ceph-mon[184144]: failed to encode map e677 with expected crc 2023-09-23T20:51:25.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:25 smithi018 ceph-mon[184144]: failed to encode map e677 with expected crc 2023-09-23T20:51:25.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:25 smithi018 ceph-mon[184144]: failed to encode map e677 with expected crc 2023-09-23T20:51:25.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:25 smithi018 ceph-mon[184144]: failed to encode map e677 with expected crc 2023-09-23T20:51:25.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:25 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3976769086' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T20:51:25.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:25 smithi018 ceph-mon[179647]: pgmap v1651: 103 pgs: 6 active+undersized+remapped, 4 remapped+peering, 3 peering, 90 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 1024 KiB/s, 4 objects/s recovering 2023-09-23T20:51:25.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:25 smithi018 ceph-mon[179647]: failed to encode map e677 with expected crc 2023-09-23T20:51:25.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:25 smithi018 ceph-mon[179647]: failed to encode map e677 with expected crc 2023-09-23T20:51:25.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:25 smithi018 ceph-mon[179647]: failed to encode map e677 with expected crc 2023-09-23T20:51:25.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:25 smithi018 ceph-mon[179647]: failed to encode map e677 with expected crc 2023-09-23T20:51:25.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:25 smithi018 ceph-mon[179647]: failed to encode map e677 with expected crc 2023-09-23T20:51:25.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:25 smithi018 ceph-mon[179647]: failed to encode map e677 with expected crc 2023-09-23T20:51:25.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:25 smithi018 ceph-mon[179647]: failed to encode map e677 with expected crc 2023-09-23T20:51:25.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:25 smithi018 ceph-mon[179647]: failed to encode map e677 with expected crc 2023-09-23T20:51:25.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:25 smithi018 ceph-mon[179647]: failed to encode map e677 with expected crc 2023-09-23T20:51:25.893 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:25 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3976769086' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T20:51:26.056 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:25 smithi149 ceph-mon[170394]: pgmap v1651: 103 pgs: 6 active+undersized+remapped, 4 remapped+peering, 3 peering, 90 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 1024 KiB/s, 4 objects/s recovering 2023-09-23T20:51:26.056 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:25 smithi149 ceph-mon[170394]: failed to encode map e677 with expected crc 2023-09-23T20:51:26.056 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:25 smithi149 ceph-mon[170394]: failed to encode map e677 with expected crc 2023-09-23T20:51:26.056 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:25 smithi149 ceph-mon[170394]: failed to encode map e677 with expected crc 2023-09-23T20:51:26.056 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:25 smithi149 ceph-mon[170394]: failed to encode map e677 with expected crc 2023-09-23T20:51:26.056 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:25 smithi149 ceph-mon[170394]: failed to encode map e677 with expected crc 2023-09-23T20:51:26.056 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:25 smithi149 ceph-mon[170394]: failed to encode map e677 with expected crc 2023-09-23T20:51:26.057 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:25 smithi149 ceph-mon[170394]: failed to encode map e677 with expected crc 2023-09-23T20:51:26.057 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:25 smithi149 ceph-mon[170394]: failed to encode map e677 with expected crc 2023-09-23T20:51:26.057 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:25 smithi149 ceph-mon[170394]: failed to encode map e677 with expected crc 2023-09-23T20:51:26.057 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:25 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3976769086' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T20:51:26.301 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:51:27.051 INFO:teuthology.orchestra.run.smithi149.stdout:{ 2023-09-23T20:51:27.051 INFO:teuthology.orchestra.run.smithi149.stdout: "ops": [], 2023-09-23T20:51:27.051 INFO:teuthology.orchestra.run.smithi149.stdout: "complaint_time": 30, 2023-09-23T20:51:27.051 INFO:teuthology.orchestra.run.smithi149.stdout: "num_blocked_ops": 0 2023-09-23T20:51:27.051 INFO:teuthology.orchestra.run.smithi149.stdout:} 2023-09-23T20:51:27.496 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T20:51:27.767 INFO:teuthology.orchestra.run.smithi018.stderr:reweighted osd.0 to 1 (10000) 2023-09-23T20:51:28.101 INFO:teuthology.orchestra.run.smithi149.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T20:51:28.109 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:27 smithi018 ceph-mon[184144]: pgmap v1653: 103 pgs: 6 active+undersized+remapped, 4 remapped+peering, 1 peering, 92 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 879 KiB/s, 5 objects/s recovering 2023-09-23T20:51:28.109 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:27 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/4162076194' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T20:51:28.109 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:27 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T20:51:28.109 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:27 smithi018 ceph-mon[179647]: pgmap v1653: 103 pgs: 6 active+undersized+remapped, 4 remapped+peering, 1 peering, 92 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 879 KiB/s, 5 objects/s recovering 2023-09-23T20:51:28.109 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:27 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/4162076194' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T20:51:28.109 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:27 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T20:51:28.154 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 0 1 2023-09-23T20:51:28.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:27 smithi149 ceph-mon[170394]: pgmap v1653: 103 pgs: 6 active+undersized+remapped, 4 remapped+peering, 1 peering, 92 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 879 KiB/s, 5 objects/s recovering 2023-09-23T20:51:28.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:27 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/4162076194' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T20:51:28.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:27 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T20:51:28.764 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:51:28.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:28 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T20:51:28.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:28 smithi149 ceph-mon[170394]: osdmap e678: 8 total, 8 up, 8 in 2023-09-23T20:51:28.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:28 smithi149 ceph-mon[170394]: failed to encode map e678 with expected crc 2023-09-23T20:51:29.018 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:28 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T20:51:29.018 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:28 smithi018 ceph-mon[184144]: osdmap e678: 8 total, 8 up, 8 in 2023-09-23T20:51:29.019 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:28 smithi018 ceph-mon[184144]: failed to encode map e678 with expected crc 2023-09-23T20:51:29.019 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:28 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T20:51:29.019 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:28 smithi018 ceph-mon[179647]: osdmap e678: 8 total, 8 up, 8 in 2023-09-23T20:51:29.019 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:28 smithi018 ceph-mon[179647]: failed to encode map e678 with expected crc 2023-09-23T20:51:29.415 INFO:teuthology.orchestra.run.smithi149.stdout:{ 2023-09-23T20:51:29.415 INFO:teuthology.orchestra.run.smithi149.stdout: "size": 20, 2023-09-23T20:51:29.415 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 600, 2023-09-23T20:51:29.415 INFO:teuthology.orchestra.run.smithi149.stdout: "ops": [ 2023-09-23T20:51:29.415 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.415 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:141034 9.7 9:eb6fa991:::benchmark_data_smithi018_381516_object27200:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e481)", 2023-09-23T20:51:29.415 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:43:15.740452+0000", 2023-09-23T20:51:29.415 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 493.67168184000002, 2023-09-23T20:51:29.415 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0443972239999999, 2023-09-23T20:51:29.416 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:51:29.416 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:51:29.416 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:51:29.416 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:51:29.416 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:51:29.416 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 141034 2023-09-23T20:51:29.416 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.416 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:51:29.416 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.416 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:51:29.416 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.740452+0000", 2023-09-23T20:51:29.417 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:29.417 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.417 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.417 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:51:29.417 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.740452+0000", 2023-09-23T20:51:29.417 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:29.417 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.417 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.417 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:51:29.417 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.740451+0000", 2023-09-23T20:51:29.417 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:51:29.417 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.418 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.418 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:51:29.418 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.740454+0000", 2023-09-23T20:51:29.418 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.0079999999999998e-06 2023-09-23T20:51:29.418 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.418 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.418 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:51:29.418 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.740461+0000", 2023-09-23T20:51:29.418 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.601e-06 2023-09-23T20:51:29.418 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.418 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.419 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:51:29.419 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.740465+0000", 2023-09-23T20:51:29.419 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.1069999999999998e-06 2023-09-23T20:51:29.419 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.419 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.419 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:29.419 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.744721+0000", 2023-09-23T20:51:29.419 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0042555429999999997 2023-09-23T20:51:29.419 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.419 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.419 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op must wait for map", 2023-09-23T20:51:29.419 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.744728+0000", 2023-09-23T20:51:29.420 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.413e-06 2023-09-23T20:51:29.420 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.420 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.420 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:29.420 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.752872+0000", 2023-09-23T20:51:29.420 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0081438929999999993 2023-09-23T20:51:29.420 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.420 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.420 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:51:29.420 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.752882+0000", 2023-09-23T20:51:29.420 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0397999999999999e-05 2023-09-23T20:51:29.421 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.421 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.421 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:29.421 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.784753+0000", 2023-09-23T20:51:29.421 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0318710520000001 2023-09-23T20:51:29.421 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.421 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.421 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:51:29.421 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.784849+0000", 2023-09-23T20:51:29.421 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.5699999999999995e-05 2023-09-23T20:51:29.421 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:29.421 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:51:29.422 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:29.422 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.422 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.422 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:141036 9.7 9:eb69e1ed:::benchmark_data_smithi018_381516_object27202:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e481)", 2023-09-23T20:51:29.422 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:43:15.740478+0000", 2023-09-23T20:51:29.422 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 493.67165602199998, 2023-09-23T20:51:29.422 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0444631559999999, 2023-09-23T20:51:29.422 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:51:29.422 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:51:29.422 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:51:29.422 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:51:29.423 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:51:29.423 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 141036 2023-09-23T20:51:29.423 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.423 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:51:29.423 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.423 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:51:29.423 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.740478+0000", 2023-09-23T20:51:29.423 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:29.423 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.423 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.423 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:51:29.423 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.740478+0000", 2023-09-23T20:51:29.423 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:29.424 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.424 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.424 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:51:29.424 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.740477+0000", 2023-09-23T20:51:29.424 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:51:29.424 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.424 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.424 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:51:29.424 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.740480+0000", 2023-09-23T20:51:29.424 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.729e-06 2023-09-23T20:51:29.424 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.425 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.425 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:51:29.425 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.740486+0000", 2023-09-23T20:51:29.425 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.1909999999999997e-06 2023-09-23T20:51:29.425 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.425 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.425 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:51:29.425 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.740492+0000", 2023-09-23T20:51:29.425 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.4990000000000002e-06 2023-09-23T20:51:29.425 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.426 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.426 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:29.426 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.744777+0000", 2023-09-23T20:51:29.426 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0042849949999999998 2023-09-23T20:51:29.426 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.426 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.426 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting_for_map not empty", 2023-09-23T20:51:29.426 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.744782+0000", 2023-09-23T20:51:29.426 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.4870000000000002e-06 2023-09-23T20:51:29.426 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.426 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.426 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:29.427 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.752918+0000", 2023-09-23T20:51:29.427 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0081358779999999992 2023-09-23T20:51:29.427 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.427 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.427 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:51:29.427 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.752926+0000", 2023-09-23T20:51:29.427 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.2749999999999995e-06 2023-09-23T20:51:29.427 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.427 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.427 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:29.427 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.784873+0000", 2023-09-23T20:51:29.427 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.03194633 2023-09-23T20:51:29.428 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.428 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.428 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:51:29.428 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.784941+0000", 2023-09-23T20:51:29.428 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.8213000000000006e-05 2023-09-23T20:51:29.428 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:29.428 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:51:29.428 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:29.428 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.428 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.428 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:141041 9.7 9:e17fa1db:::benchmark_data_smithi018_381516_object27207:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e481)", 2023-09-23T20:51:29.428 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:43:15.740496+0000", 2023-09-23T20:51:29.429 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 493.671637481, 2023-09-23T20:51:29.429 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0815582909999999, 2023-09-23T20:51:29.429 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:51:29.429 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:51:29.429 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:51:29.429 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:51:29.429 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:51:29.429 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 141041 2023-09-23T20:51:29.429 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.429 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:51:29.429 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.429 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:51:29.430 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.740496+0000", 2023-09-23T20:51:29.430 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:29.430 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.430 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.430 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:51:29.430 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.740496+0000", 2023-09-23T20:51:29.430 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:29.430 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.430 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.430 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:51:29.430 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.740496+0000", 2023-09-23T20:51:29.431 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:51:29.431 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.431 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.431 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:51:29.431 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.740498+0000", 2023-09-23T20:51:29.431 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.1780000000000002e-06 2023-09-23T20:51:29.431 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.431 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.431 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:51:29.431 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.740504+0000", 2023-09-23T20:51:29.431 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.5609999999999998e-06 2023-09-23T20:51:29.432 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.432 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.432 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:51:29.432 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.740507+0000", 2023-09-23T20:51:29.432 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.6950000000000002e-06 2023-09-23T20:51:29.432 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.432 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.432 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:29.432 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.744941+0000", 2023-09-23T20:51:29.432 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0044336360000000003 2023-09-23T20:51:29.433 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.433 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.433 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting_for_map not empty", 2023-09-23T20:51:29.433 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.744961+0000", 2023-09-23T20:51:29.433 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.9601000000000001e-05 2023-09-23T20:51:29.433 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.433 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.433 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:29.433 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.752986+0000", 2023-09-23T20:51:29.433 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0080249980000000002 2023-09-23T20:51:29.433 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.434 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.434 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:51:29.434 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.752993+0000", 2023-09-23T20:51:29.434 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.7719999999999994e-06 2023-09-23T20:51:29.434 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.434 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.434 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:29.434 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.784961+0000", 2023-09-23T20:51:29.434 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0319681039999999 2023-09-23T20:51:29.434 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.435 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.435 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for missing object", 2023-09-23T20:51:29.435 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.785024+0000", 2023-09-23T20:51:29.435 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.2626000000000007e-05 2023-09-23T20:51:29.435 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.435 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.435 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:29.435 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.821977+0000", 2023-09-23T20:51:29.435 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.036953184 2023-09-23T20:51:29.435 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.435 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.436 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:51:29.436 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.822055+0000", 2023-09-23T20:51:29.436 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.7239999999999999e-05 2023-09-23T20:51:29.436 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:29.436 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:51:29.436 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:29.436 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.436 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.436 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:141053 9.7 9:e03873d0:::benchmark_data_smithi018_381516_object27219:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e481)", 2023-09-23T20:51:29.436 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:43:15.775241+0000", 2023-09-23T20:51:29.436 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 493.636892553, 2023-09-23T20:51:29.436 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.047376954, 2023-09-23T20:51:29.436 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:51:29.437 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:29.437 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:51:29.437 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:51:29.437 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:51:29.437 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 141053 2023-09-23T20:51:29.437 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.437 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:51:29.437 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.437 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:51:29.437 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.775241+0000", 2023-09-23T20:51:29.437 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:29.438 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.438 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.438 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:51:29.438 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.775241+0000", 2023-09-23T20:51:29.438 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:29.438 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.438 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.438 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:51:29.438 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.775240+0000", 2023-09-23T20:51:29.438 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:51:29.438 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.439 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.439 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:51:29.439 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.775245+0000", 2023-09-23T20:51:29.439 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.8890000000000001e-06 2023-09-23T20:51:29.439 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.439 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.439 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:51:29.439 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.775257+0000", 2023-09-23T20:51:29.439 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1571e-05 2023-09-23T20:51:29.439 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.439 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.440 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:51:29.440 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.775261+0000", 2023-09-23T20:51:29.440 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.7999999999999998e-06 2023-09-23T20:51:29.440 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.440 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.440 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:29.440 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.775328+0000", 2023-09-23T20:51:29.440 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.6329999999999997e-05 2023-09-23T20:51:29.440 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.440 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.440 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:51:29.440 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:15.775340+0000", 2023-09-23T20:51:29.441 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.2172e-05 2023-09-23T20:51:29.441 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.441 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.441 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:29.441 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.785061+0000", 2023-09-23T20:51:29.441 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0097207939999999 2023-09-23T20:51:29.441 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.441 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.441 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:51:29.441 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.785170+0000", 2023-09-23T20:51:29.441 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000109317 2023-09-23T20:51:29.441 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.441 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.442 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 3,5", 2023-09-23T20:51:29.442 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.785802+0000", 2023-09-23T20:51:29.442 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00063209099999999999 2023-09-23T20:51:29.442 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.442 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.442 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:51:29.442 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.816501+0000", 2023-09-23T20:51:29.442 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.030699185 2023-09-23T20:51:29.442 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.442 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.442 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:29.443 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.817345+0000", 2023-09-23T20:51:29.443 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00084414799999999997 2023-09-23T20:51:29.443 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.443 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.443 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:29.443 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.822536+0000", 2023-09-23T20:51:29.443 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0051903310000000003 2023-09-23T20:51:29.443 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.443 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.443 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:51:29.443 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.822567+0000", 2023-09-23T20:51:29.443 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.1572999999999997e-05 2023-09-23T20:51:29.443 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.444 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.444 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:51:29.444 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:43:16.822618+0000", 2023-09-23T20:51:29.444 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.0825000000000002e-05 2023-09-23T20:51:29.444 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:29.444 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:51:29.444 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:29.444 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.444 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.444 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.36450.0:185078 9.d 9:b3b89e5b:::benchmark_data_smithi018_381516_object71244:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e501)", 2023-09-23T20:51:29.445 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:44:17.632886+0000", 2023-09-23T20:51:29.445 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 431.77924789399998, 2023-09-23T20:51:29.445 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0561142219999999, 2023-09-23T20:51:29.445 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:51:29.445 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:29.445 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:51:29.445 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.36450", 2023-09-23T20:51:29.445 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:51:29.445 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 185078 2023-09-23T20:51:29.445 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.445 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:51:29.446 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.446 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:51:29.446 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:17.632886+0000", 2023-09-23T20:51:29.446 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:29.446 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.446 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.446 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:51:29.446 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:17.632886+0000", 2023-09-23T20:51:29.446 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:29.446 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.446 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.446 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:51:29.447 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:17.632885+0000", 2023-09-23T20:51:29.447 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:51:29.447 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.447 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.447 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:51:29.447 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:17.632890+0000", 2023-09-23T20:51:29.447 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.1120000000000004e-06 2023-09-23T20:51:29.447 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.447 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.447 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:51:29.447 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:17.632900+0000", 2023-09-23T20:51:29.447 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0541e-05 2023-09-23T20:51:29.448 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.448 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.448 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:51:29.448 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:17.632906+0000", 2023-09-23T20:51:29.448 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.1939999999999997e-06 2023-09-23T20:51:29.448 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.449 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.449 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:29.449 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:17.634328+0000", 2023-09-23T20:51:29.449 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0014227879999999999 2023-09-23T20:51:29.450 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.450 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.450 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:51:29.450 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:17.634341+0000", 2023-09-23T20:51:29.450 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.2524000000000001e-05 2023-09-23T20:51:29.450 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.450 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.450 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:29.450 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:18.658019+0000", 2023-09-23T20:51:29.450 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.023678552 2023-09-23T20:51:29.450 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.451 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.451 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:51:29.451 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:18.658147+0000", 2023-09-23T20:51:29.451 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000128088 2023-09-23T20:51:29.451 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.451 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.451 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 1,5", 2023-09-23T20:51:29.451 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:18.658233+0000", 2023-09-23T20:51:29.451 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.5655000000000006e-05 2023-09-23T20:51:29.451 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.451 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.451 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:51:29.452 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:18.667554+0000", 2023-09-23T20:51:29.452 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0093206199999999999 2023-09-23T20:51:29.452 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.452 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.452 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:29.452 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:18.688813+0000", 2023-09-23T20:51:29.452 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.021259726999999999 2023-09-23T20:51:29.452 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.452 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.452 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:29.452 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:18.688941+0000", 2023-09-23T20:51:29.453 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00012763799999999999 2023-09-23T20:51:29.453 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.453 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.453 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:51:29.453 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:18.688974+0000", 2023-09-23T20:51:29.453 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.2818000000000003e-05 2023-09-23T20:51:29.453 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.453 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.453 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:51:29.453 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:44:18.689000+0000", 2023-09-23T20:51:29.453 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.6100000000000001e-05 2023-09-23T20:51:29.453 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:29.454 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:51:29.454 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:29.454 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.454 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.454 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:102274 10.c 10:31fbc5f2:::benchmark_data_smithi018_422431_object102273:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e551)", 2023-09-23T20:51:29.454 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:47:08.160645+0000", 2023-09-23T20:51:29.454 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 261.25148842599998, 2023-09-23T20:51:29.454 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1493982, 2023-09-23T20:51:29.454 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:51:29.454 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:51:29.454 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:51:29.455 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:51:29.455 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:51:29.455 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 102274 2023-09-23T20:51:29.455 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.455 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:51:29.455 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.455 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:51:29.455 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.160645+0000", 2023-09-23T20:51:29.455 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:29.455 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.455 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.455 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:51:29.456 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.160645+0000", 2023-09-23T20:51:29.456 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:29.456 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.456 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.456 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:51:29.456 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.160644+0000", 2023-09-23T20:51:29.456 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:51:29.456 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.456 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.456 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:51:29.456 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.160692+0000", 2023-09-23T20:51:29.456 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.8096999999999997e-05 2023-09-23T20:51:29.457 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.457 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.457 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:51:29.457 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.160712+0000", 2023-09-23T20:51:29.457 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.0421e-05 2023-09-23T20:51:29.457 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.457 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.457 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:51:29.457 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.160720+0000", 2023-09-23T20:51:29.457 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.7980000000000006e-06 2023-09-23T20:51:29.457 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.458 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.458 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:29.458 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.160777+0000", 2023-09-23T20:51:29.458 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.6731999999999998e-05 2023-09-23T20:51:29.458 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.458 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.458 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:51:29.458 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.160957+0000", 2023-09-23T20:51:29.458 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00018039999999999999 2023-09-23T20:51:29.458 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.458 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.459 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,3", 2023-09-23T20:51:29.459 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.161161+0000", 2023-09-23T20:51:29.459 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00020371300000000001 2023-09-23T20:51:29.459 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.459 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.459 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:51:29.459 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.169571+0000", 2023-09-23T20:51:29.459 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0084097330000000008 2023-09-23T20:51:29.459 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.459 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.459 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:29.459 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.187438+0000", 2023-09-23T20:51:29.460 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.017867623999999999 2023-09-23T20:51:29.460 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.460 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.460 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:51:29.460 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.187453+0000", 2023-09-23T20:51:29.460 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.5238e-05 2023-09-23T20:51:29.460 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.460 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.460 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:29.460 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.292752+0000", 2023-09-23T20:51:29.460 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.105298715 2023-09-23T20:51:29.460 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.461 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.461 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for degraded object", 2023-09-23T20:51:29.461 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.293574+0000", 2023-09-23T20:51:29.461 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00082229000000000004 2023-09-23T20:51:29.461 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.461 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.461 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:29.461 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.309964+0000", 2023-09-23T20:51:29.461 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.016389821999999998 2023-09-23T20:51:29.461 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.461 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.461 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:51:29.462 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.310043+0000", 2023-09-23T20:51:29.462 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.9184000000000006e-05 2023-09-23T20:51:29.462 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:29.462 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:51:29.462 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:29.462 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.462 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.462 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:102286 10.c 10:391cdcfd:::benchmark_data_smithi018_422431_object102285:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e552)", 2023-09-23T20:51:29.462 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:47:08.205082+0000", 2023-09-23T20:51:29.462 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 261.20705130200002, 2023-09-23T20:51:29.462 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1143086929999999, 2023-09-23T20:51:29.462 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:51:29.463 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:29.463 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:51:29.463 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:51:29.463 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:51:29.463 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 102286 2023-09-23T20:51:29.463 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.463 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:51:29.463 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.463 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:51:29.463 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.205082+0000", 2023-09-23T20:51:29.463 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:29.463 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.464 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.464 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:51:29.464 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.205082+0000", 2023-09-23T20:51:29.464 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:29.464 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.464 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.464 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:51:29.464 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.205081+0000", 2023-09-23T20:51:29.464 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:51:29.464 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.464 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.464 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:51:29.464 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.205162+0000", 2023-09-23T20:51:29.465 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.0582999999999997e-05 2023-09-23T20:51:29.465 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.465 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.465 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:51:29.465 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.205177+0000", 2023-09-23T20:51:29.465 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.5037e-05 2023-09-23T20:51:29.465 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.465 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.465 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:51:29.465 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.205183+0000", 2023-09-23T20:51:29.465 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.9009999999999999e-06 2023-09-23T20:51:29.465 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.466 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.466 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:29.466 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.205241+0000", 2023-09-23T20:51:29.466 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.7921999999999999e-05 2023-09-23T20:51:29.466 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.466 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.466 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:51:29.466 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.205254+0000", 2023-09-23T20:51:29.466 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.3049e-05 2023-09-23T20:51:29.466 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.466 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.467 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:29.467 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.294002+0000", 2023-09-23T20:51:29.467 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0887488059999999 2023-09-23T20:51:29.467 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.467 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.467 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:51:29.467 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.294127+0000", 2023-09-23T20:51:29.467 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000124201 2023-09-23T20:51:29.467 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.467 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.467 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,4", 2023-09-23T20:51:29.467 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.294300+0000", 2023-09-23T20:51:29.468 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000173053 2023-09-23T20:51:29.468 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.468 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.468 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:29.468 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.309385+0000", 2023-09-23T20:51:29.468 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.015085698 2023-09-23T20:51:29.468 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.468 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.468 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:51:29.468 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.310181+0000", 2023-09-23T20:51:29.468 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00079541699999999998 2023-09-23T20:51:29.469 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.469 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.469 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:29.469 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.319022+0000", 2023-09-23T20:51:29.469 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0088413550000000004 2023-09-23T20:51:29.469 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.469 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.469 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:51:29.469 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.319070+0000", 2023-09-23T20:51:29.469 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.8233999999999999e-05 2023-09-23T20:51:29.469 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.469 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.470 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:51:29.470 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.319391+0000", 2023-09-23T20:51:29.470 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00032073099999999999 2023-09-23T20:51:29.470 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:29.470 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:51:29.470 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:29.470 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.470 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.470 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:102289 10.c 10:3e772927:::benchmark_data_smithi018_422431_object102288:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e552)", 2023-09-23T20:51:29.470 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:47:08.205193+0000", 2023-09-23T20:51:29.470 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 261.20694040299998, 2023-09-23T20:51:29.471 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.114489624, 2023-09-23T20:51:29.471 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:51:29.471 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:29.471 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:51:29.471 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:51:29.471 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:51:29.471 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 102289 2023-09-23T20:51:29.471 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.471 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:51:29.471 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.471 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:51:29.471 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.205193+0000", 2023-09-23T20:51:29.471 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:29.472 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.472 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.472 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:51:29.472 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.205193+0000", 2023-09-23T20:51:29.472 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:29.472 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.472 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.472 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:51:29.472 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.205193+0000", 2023-09-23T20:51:29.472 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:51:29.472 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.472 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.473 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:51:29.473 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.205231+0000", 2023-09-23T20:51:29.473 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.8189999999999999e-05 2023-09-23T20:51:29.473 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.473 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.473 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:51:29.473 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.205244+0000", 2023-09-23T20:51:29.473 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.3067999999999999e-05 2023-09-23T20:51:29.473 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.473 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.473 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:51:29.474 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.205249+0000", 2023-09-23T20:51:29.474 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.1449999999999999e-06 2023-09-23T20:51:29.474 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.474 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.474 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:29.474 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.205277+0000", 2023-09-23T20:51:29.474 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.7350000000000001e-05 2023-09-23T20:51:29.474 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.474 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.474 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:51:29.474 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.205281+0000", 2023-09-23T20:51:29.475 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.5809999999999998e-06 2023-09-23T20:51:29.475 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.475 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.475 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:29.475 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.298397+0000", 2023-09-23T20:51:29.475 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.093115992 2023-09-23T20:51:29.475 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.475 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.475 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:51:29.475 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.298529+0000", 2023-09-23T20:51:29.475 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00013224499999999999 2023-09-23T20:51:29.475 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.476 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.476 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,4", 2023-09-23T20:51:29.476 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.298698+0000", 2023-09-23T20:51:29.476 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000169123 2023-09-23T20:51:29.476 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.476 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.476 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:29.476 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.310075+0000", 2023-09-23T20:51:29.476 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.011376282 2023-09-23T20:51:29.476 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.476 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.477 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:51:29.477 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.310186+0000", 2023-09-23T20:51:29.477 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00011142 2023-09-23T20:51:29.477 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.477 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.477 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:29.477 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.319436+0000", 2023-09-23T20:51:29.477 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0092496220000000007 2023-09-23T20:51:29.477 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.477 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.477 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:51:29.477 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.319488+0000", 2023-09-23T20:51:29.477 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.2408999999999999e-05 2023-09-23T20:51:29.477 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.478 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.478 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:51:29.478 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.319683+0000", 2023-09-23T20:51:29.478 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00019472099999999999 2023-09-23T20:51:29.478 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:29.478 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:51:29.478 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:29.478 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.478 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.478 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:102292 10.c 10:376f4c97:::benchmark_data_smithi018_422431_object102291:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e552)", 2023-09-23T20:51:29.478 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:47:08.212988+0000", 2023-09-23T20:51:29.479 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 261.199145741, 2023-09-23T20:51:29.479 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1069731110000001, 2023-09-23T20:51:29.479 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:51:29.479 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:29.479 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:51:29.479 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:51:29.479 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:51:29.479 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 102292 2023-09-23T20:51:29.479 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.479 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:51:29.479 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.479 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:51:29.479 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.212988+0000", 2023-09-23T20:51:29.480 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:29.480 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.480 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.480 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:51:29.480 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.212988+0000", 2023-09-23T20:51:29.480 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:29.480 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.480 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.480 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:51:29.480 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.212986+0000", 2023-09-23T20:51:29.480 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999981 2023-09-23T20:51:29.480 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.480 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.481 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:51:29.481 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.213032+0000", 2023-09-23T20:51:29.481 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.6239000000000002e-05 2023-09-23T20:51:29.481 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.481 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.481 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:51:29.481 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.213050+0000", 2023-09-23T20:51:29.481 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.7575e-05 2023-09-23T20:51:29.481 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.481 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.481 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:51:29.481 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.213058+0000", 2023-09-23T20:51:29.482 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.3399999999999998e-06 2023-09-23T20:51:29.482 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.482 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.482 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:29.482 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.213122+0000", 2023-09-23T20:51:29.482 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.3418999999999997e-05 2023-09-23T20:51:29.482 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.482 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.482 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:51:29.482 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:08.213134+0000", 2023-09-23T20:51:29.482 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.2356e-05 2023-09-23T20:51:29.482 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.483 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.483 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:29.483 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.302309+0000", 2023-09-23T20:51:29.483 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.089175199 2023-09-23T20:51:29.483 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.483 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.483 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:51:29.483 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.302439+0000", 2023-09-23T20:51:29.483 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00012961 2023-09-23T20:51:29.483 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.483 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.484 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,4", 2023-09-23T20:51:29.484 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.302598+0000", 2023-09-23T20:51:29.484 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00015949599999999999 2023-09-23T20:51:29.484 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.484 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.484 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:29.484 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.310102+0000", 2023-09-23T20:51:29.484 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0075032220000000004 2023-09-23T20:51:29.484 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.484 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.484 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:51:29.484 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.315572+0000", 2023-09-23T20:51:29.484 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0054703499999999997 2023-09-23T20:51:29.485 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.485 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.485 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:29.485 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.319723+0000", 2023-09-23T20:51:29.485 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0041513679999999999 2023-09-23T20:51:29.485 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.485 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.485 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:51:29.485 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.319778+0000", 2023-09-23T20:51:29.485 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.4908999999999999e-05 2023-09-23T20:51:29.485 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.485 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.485 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:51:29.486 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:47:09.319961+0000", 2023-09-23T20:51:29.486 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00018275499999999999 2023-09-23T20:51:29.486 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:29.486 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:51:29.486 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:29.486 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.486 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.486 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:158988 10.9 10:9fe441aa:::benchmark_data_smithi018_422431_object54492:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e577)", 2023-09-23T20:51:29.486 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:48:22.900242+0000", 2023-09-23T20:51:29.486 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 186.51189199199999, 2023-09-23T20:51:29.487 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.295998714, 2023-09-23T20:51:29.487 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:51:29.487 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:51:29.487 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:51:29.487 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:51:29.487 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:51:29.487 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 158988 2023-09-23T20:51:29.487 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.487 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:51:29.487 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.487 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:51:29.487 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.900242+0000", 2023-09-23T20:51:29.487 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:29.488 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.488 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.488 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:51:29.488 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.900242+0000", 2023-09-23T20:51:29.488 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:29.488 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.488 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.488 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:51:29.488 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.900241+0000", 2023-09-23T20:51:29.488 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:51:29.488 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.488 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.489 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:51:29.489 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.900245+0000", 2023-09-23T20:51:29.489 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.3710000000000002e-06 2023-09-23T20:51:29.489 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.489 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.489 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:51:29.489 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.900258+0000", 2023-09-23T20:51:29.489 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.3018e-05 2023-09-23T20:51:29.489 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.489 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.489 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:51:29.490 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.900265+0000", 2023-09-23T20:51:29.490 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.5899999999999996e-06 2023-09-23T20:51:29.490 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.490 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.490 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:29.490 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.908024+0000", 2023-09-23T20:51:29.490 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0077597910000000003 2023-09-23T20:51:29.490 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.490 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.490 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:51:29.490 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.908034+0000", 2023-09-23T20:51:29.490 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.7820000000000005e-06 2023-09-23T20:51:29.491 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.491 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.491 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:29.491 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:23.907026+0000", 2023-09-23T20:51:29.491 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.99899201500000001 2023-09-23T20:51:29.491 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.491 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.491 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:51:29.491 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:23.907037+0000", 2023-09-23T20:51:29.491 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0468999999999999e-05 2023-09-23T20:51:29.491 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.491 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.491 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:29.492 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:25.196139+0000", 2023-09-23T20:51:29.492 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.2891025380000001 2023-09-23T20:51:29.492 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.492 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.492 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:51:29.492 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:25.196240+0000", 2023-09-23T20:51:29.492 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00010123 2023-09-23T20:51:29.492 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:29.492 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:51:29.492 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:29.492 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.492 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.492 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:158996 10.9 10:9667fd03:::benchmark_data_smithi018_422431_object54500:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e577)", 2023-09-23T20:51:29.492 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:48:22.900270+0000", 2023-09-23T20:51:29.493 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 186.51186374, 2023-09-23T20:51:29.493 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.343689795, 2023-09-23T20:51:29.493 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:51:29.493 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:51:29.493 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:51:29.493 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:51:29.493 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:51:29.493 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 158996 2023-09-23T20:51:29.493 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.493 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:51:29.493 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.493 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:51:29.493 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.900270+0000", 2023-09-23T20:51:29.494 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:29.494 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.494 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.494 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:51:29.494 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.900270+0000", 2023-09-23T20:51:29.494 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:29.494 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.494 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.494 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:51:29.494 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.900270+0000", 2023-09-23T20:51:29.494 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:51:29.494 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.494 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.495 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:51:29.495 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.900272+0000", 2023-09-23T20:51:29.495 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.232e-06 2023-09-23T20:51:29.495 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.495 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.495 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:51:29.495 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.900279+0000", 2023-09-23T20:51:29.495 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.9369999999999998e-06 2023-09-23T20:51:29.495 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.495 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.495 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:51:29.495 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.900282+0000", 2023-09-23T20:51:29.495 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.4259999999999999e-06 2023-09-23T20:51:29.496 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.496 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.496 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:29.496 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.908056+0000", 2023-09-23T20:51:29.496 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0077741520000000003 2023-09-23T20:51:29.496 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.496 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.496 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:51:29.496 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.908064+0000", 2023-09-23T20:51:29.496 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.1729999999999998e-06 2023-09-23T20:51:29.496 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.496 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.496 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:29.497 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:23.907060+0000", 2023-09-23T20:51:29.497 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.99899634999999998 2023-09-23T20:51:29.497 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.497 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.497 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:51:29.497 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:23.907067+0000", 2023-09-23T20:51:29.497 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.8510000000000001e-06 2023-09-23T20:51:29.497 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.497 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.497 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:29.497 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:25.196528+0000", 2023-09-23T20:51:29.497 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.289461322 2023-09-23T20:51:29.498 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.498 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.498 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for missing object", 2023-09-23T20:51:29.498 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:25.196641+0000", 2023-09-23T20:51:29.498 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000112619 2023-09-23T20:51:29.498 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.498 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.498 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:29.498 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:25.243873+0000", 2023-09-23T20:51:29.498 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.047232622000000002 2023-09-23T20:51:29.498 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.498 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.499 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:51:29.499 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:25.243960+0000", 2023-09-23T20:51:29.499 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.6462e-05 2023-09-23T20:51:29.499 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:29.499 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:51:29.499 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:29.499 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.499 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.499 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:159015 10.9 10:914d252e:::benchmark_data_smithi018_422431_object54519:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e577)", 2023-09-23T20:51:29.499 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:48:22.900286+0000", 2023-09-23T20:51:29.499 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 186.51184786499999, 2023-09-23T20:51:29.499 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.3515420869999999, 2023-09-23T20:51:29.499 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:51:29.500 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:51:29.500 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:51:29.500 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:51:29.500 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:51:29.500 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 159015 2023-09-23T20:51:29.500 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.500 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:51:29.500 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.500 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:51:29.500 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.900286+0000", 2023-09-23T20:51:29.500 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:29.500 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.501 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.501 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:51:29.501 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.900286+0000", 2023-09-23T20:51:29.501 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:29.501 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.501 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.501 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:51:29.501 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.900286+0000", 2023-09-23T20:51:29.501 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:51:29.501 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.501 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.501 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:51:29.502 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.900287+0000", 2023-09-23T20:51:29.502 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.9379999999999999e-06 2023-09-23T20:51:29.502 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.502 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.502 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:51:29.502 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.900293+0000", 2023-09-23T20:51:29.502 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.1939999999999997e-06 2023-09-23T20:51:29.502 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.502 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.502 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:51:29.502 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.900296+0000", 2023-09-23T20:51:29.503 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 3.1970000000000001e-06 2023-09-23T20:51:29.503 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.503 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.503 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:29.503 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.908083+0000", 2023-09-23T20:51:29.503 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.007787202 2023-09-23T20:51:29.503 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.503 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.503 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:51:29.503 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.908089+0000", 2023-09-23T20:51:29.503 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.1210000000000003e-06 2023-09-23T20:51:29.504 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.504 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.504 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:29.504 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:23.907087+0000", 2023-09-23T20:51:29.504 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.998997468 2023-09-23T20:51:29.504 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.504 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.504 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:51:29.504 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:23.907094+0000", 2023-09-23T20:51:29.504 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.1269999999999997e-06 2023-09-23T20:51:29.504 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.504 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.505 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:29.505 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:25.196671+0000", 2023-09-23T20:51:29.505 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.2895769530000001 2023-09-23T20:51:29.505 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.505 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.505 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for missing object", 2023-09-23T20:51:29.505 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:25.196749+0000", 2023-09-23T20:51:29.505 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.8147999999999998e-05 2023-09-23T20:51:29.505 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.505 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.505 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:29.505 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:25.251766+0000", 2023-09-23T20:51:29.506 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.055017165 2023-09-23T20:51:29.506 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.506 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.506 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:51:29.506 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:25.251828+0000", 2023-09-23T20:51:29.506 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.1891999999999996e-05 2023-09-23T20:51:29.506 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:29.506 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:51:29.506 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:29.506 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.506 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.506 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:159059 10.9 10:91c42c8b:::benchmark_data_smithi018_422431_object54563:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e577)", 2023-09-23T20:51:29.506 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:48:22.979021+0000", 2023-09-23T20:51:29.507 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 186.43311310799999, 2023-09-23T20:51:29.507 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.2786466750000001, 2023-09-23T20:51:29.507 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:51:29.507 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:29.507 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:51:29.507 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:51:29.507 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:51:29.507 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 159059 2023-09-23T20:51:29.507 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.507 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:51:29.507 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.507 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:51:29.508 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.979021+0000", 2023-09-23T20:51:29.508 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:29.508 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.508 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.508 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:51:29.508 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.979021+0000", 2023-09-23T20:51:29.508 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:29.508 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.508 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.508 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:51:29.508 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.979019+0000", 2023-09-23T20:51:29.509 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:51:29.509 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.509 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.509 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:51:29.509 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.979026+0000", 2023-09-23T20:51:29.509 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.3620000000000002e-06 2023-09-23T20:51:29.509 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.509 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.509 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:51:29.509 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.979040+0000", 2023-09-23T20:51:29.509 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.4419e-05 2023-09-23T20:51:29.510 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.510 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.510 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:51:29.510 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.979048+0000", 2023-09-23T20:51:29.510 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.7360000000000001e-06 2023-09-23T20:51:29.510 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.510 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.510 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:29.510 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.979099+0000", 2023-09-23T20:51:29.510 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.0980000000000003e-05 2023-09-23T20:51:29.510 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.511 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.511 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:51:29.511 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:22.979108+0000", 2023-09-23T20:51:29.511 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.1619999999999997e-06 2023-09-23T20:51:29.511 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.511 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.511 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:29.511 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:23.907115+0000", 2023-09-23T20:51:29.511 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.92800715899999997 2023-09-23T20:51:29.511 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.511 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.511 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:51:29.511 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:23.907122+0000", 2023-09-23T20:51:29.512 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.8959999999999997e-06 2023-09-23T20:51:29.512 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.512 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.512 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:29.512 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:25.196779+0000", 2023-09-23T20:51:29.512 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.2896565419999999 2023-09-23T20:51:29.512 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.512 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.512 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:51:29.512 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:25.196960+0000", 2023-09-23T20:51:29.512 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000181824 2023-09-23T20:51:29.513 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.513 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.513 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 2,4,5", 2023-09-23T20:51:29.513 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:25.198030+0000", 2023-09-23T20:51:29.513 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0010695349999999999 2023-09-23T20:51:29.513 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.513 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.513 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:29.513 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:25.243692+0000", 2023-09-23T20:51:29.513 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.045662454999999998 2023-09-23T20:51:29.513 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.513 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.514 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:51:29.514 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:25.243701+0000", 2023-09-23T20:51:29.514 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.5120000000000008e-06 2023-09-23T20:51:29.514 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.514 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.514 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:29.514 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:25.244025+0000", 2023-09-23T20:51:29.514 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00032403299999999999 2023-09-23T20:51:29.514 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.514 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.514 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:29.515 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:25.257506+0000", 2023-09-23T20:51:29.515 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.013481388 2023-09-23T20:51:29.515 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.515 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.515 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:51:29.515 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:25.257558+0000", 2023-09-23T20:51:29.515 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.1554000000000001e-05 2023-09-23T20:51:29.515 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.515 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.515 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:51:29.515 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:25.257667+0000", 2023-09-23T20:51:29.516 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000109456 2023-09-23T20:51:29.516 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:29.516 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:51:29.516 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:29.516 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.516 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.516 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:174186 10.19 10:99f413a8:::benchmark_data_smithi018_422431_object69690:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e590)", 2023-09-23T20:51:29.516 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:48:47.116428+0000", 2023-09-23T20:51:29.516 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 162.295705594, 2023-09-23T20:51:29.516 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0356085479999999, 2023-09-23T20:51:29.516 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:51:29.517 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "reached pg", 2023-09-23T20:51:29.517 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:51:29.517 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:51:29.517 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:51:29.517 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 174186 2023-09-23T20:51:29.517 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.517 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:51:29.517 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.517 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:51:29.517 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:47.116428+0000", 2023-09-23T20:51:29.517 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:29.517 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.517 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.517 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:51:29.518 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:47.116428+0000", 2023-09-23T20:51:29.518 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:29.518 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.518 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.518 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:51:29.518 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:47.116427+0000", 2023-09-23T20:51:29.518 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:51:29.518 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.518 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.518 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:51:29.518 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:47.116433+0000", 2023-09-23T20:51:29.518 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.3589999999999996e-06 2023-09-23T20:51:29.519 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.519 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.519 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:51:29.519 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:47.116450+0000", 2023-09-23T20:51:29.519 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.7334e-05 2023-09-23T20:51:29.519 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.519 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.519 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:51:29.519 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:47.116457+0000", 2023-09-23T20:51:29.519 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.6359999999999997e-06 2023-09-23T20:51:29.519 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.519 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.519 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:29.520 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:47.116502+0000", 2023-09-23T20:51:29.520 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.4987000000000001e-05 2023-09-23T20:51:29.520 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.520 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.520 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:51:29.520 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:47.116680+0000", 2023-09-23T20:51:29.520 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00017807599999999999 2023-09-23T20:51:29.520 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.520 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.520 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 2,4,5", 2023-09-23T20:51:29.520 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:47.117094+0000", 2023-09-23T20:51:29.520 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00041400999999999997 2023-09-23T20:51:29.520 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.521 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.521 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:29.521 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:47.130634+0000", 2023-09-23T20:51:29.521 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.013540533 2023-09-23T20:51:29.521 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.521 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.521 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:51:29.521 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:47.130642+0000", 2023-09-23T20:51:29.521 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.0229999999999992e-06 2023-09-23T20:51:29.521 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.521 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.521 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:29.522 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:47.136723+0000", 2023-09-23T20:51:29.522 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0060805249999999998 2023-09-23T20:51:29.522 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.522 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.522 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:29.522 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:47.146513+0000", 2023-09-23T20:51:29.522 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0097900049999999992 2023-09-23T20:51:29.522 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.522 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.522 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:51:29.522 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:47.146525+0000", 2023-09-23T20:51:29.522 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.2316e-05 2023-09-23T20:51:29.523 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.523 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.523 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:29.523 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:48.152024+0000", 2023-09-23T20:51:29.523 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.005498185 2023-09-23T20:51:29.523 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.523 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.523 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:51:29.523 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:48:48.152037+0000", 2023-09-23T20:51:29.523 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.3127e-05 2023-09-23T20:51:29.523 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:29.523 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:51:29.523 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:29.523 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.524 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.524 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:181716 10.c 10:3b20cb1c:::benchmark_data_smithi018_422431_object77220:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e604)", 2023-09-23T20:51:29.524 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:49:02.794588+0000", 2023-09-23T20:51:29.524 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 146.61754542099999, 2023-09-23T20:51:29.524 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1313792709999999, 2023-09-23T20:51:29.524 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:51:29.524 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:29.524 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:51:29.524 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:51:29.524 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:51:29.524 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 181716 2023-09-23T20:51:29.524 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.525 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:51:29.525 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.525 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:51:29.525 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:02.794588+0000", 2023-09-23T20:51:29.525 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:29.525 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.525 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.525 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:51:29.525 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:02.794588+0000", 2023-09-23T20:51:29.525 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:29.525 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.525 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.525 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:51:29.526 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:02.794587+0000", 2023-09-23T20:51:29.526 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999981 2023-09-23T20:51:29.526 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.526 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.526 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:51:29.526 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:02.794596+0000", 2023-09-23T20:51:29.526 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.9420000000000007e-06 2023-09-23T20:51:29.526 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.526 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.526 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:51:29.526 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:02.794620+0000", 2023-09-23T20:51:29.526 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.3161000000000001e-05 2023-09-23T20:51:29.527 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.527 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.527 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:51:29.527 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:02.794629+0000", 2023-09-23T20:51:29.527 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.7329999999999998e-06 2023-09-23T20:51:29.527 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.527 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.527 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:29.527 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:02.794693+0000", 2023-09-23T20:51:29.527 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.3558000000000004e-05 2023-09-23T20:51:29.527 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.527 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.527 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:51:29.528 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:02.794702+0000", 2023-09-23T20:51:29.528 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.6850000000000007e-06 2023-09-23T20:51:29.528 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.528 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.528 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:29.528 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:03.886841+0000", 2023-09-23T20:51:29.528 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0921389640000001 2023-09-23T20:51:29.528 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.528 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.528 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:51:29.528 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:03.886964+0000", 2023-09-23T20:51:29.528 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000122951 2023-09-23T20:51:29.528 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.529 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.529 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,1,4", 2023-09-23T20:51:29.529 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:03.887573+0000", 2023-09-23T20:51:29.529 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00060992900000000005 2023-09-23T20:51:29.529 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.529 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.529 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:51:29.529 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:03.899773+0000", 2023-09-23T20:51:29.529 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.012199750000000001 2023-09-23T20:51:29.529 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.529 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.529 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:29.530 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:03.899930+0000", 2023-09-23T20:51:29.530 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00015683900000000001 2023-09-23T20:51:29.530 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.530 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.530 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:29.530 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:03.905030+0000", 2023-09-23T20:51:29.530 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0050999790000000001 2023-09-23T20:51:29.530 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.530 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.530 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:29.530 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:03.925775+0000", 2023-09-23T20:51:29.530 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.020745320000000001 2023-09-23T20:51:29.531 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.531 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.531 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:51:29.531 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:03.925836+0000", 2023-09-23T20:51:29.531 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.0853000000000002e-05 2023-09-23T20:51:29.531 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.531 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.531 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:51:29.531 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:03.925968+0000", 2023-09-23T20:51:29.531 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00013135999999999999 2023-09-23T20:51:29.531 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:29.531 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:51:29.531 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:29.532 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.532 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.532 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:181719 10.c 10:3f881ee1:::benchmark_data_smithi018_422431_object77223:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e604)", 2023-09-23T20:51:29.532 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:49:02.797308+0000", 2023-09-23T20:51:29.532 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 146.614826093, 2023-09-23T20:51:29.532 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1289105109999999, 2023-09-23T20:51:29.532 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:51:29.532 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:29.532 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:51:29.532 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:51:29.532 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:51:29.533 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 181719 2023-09-23T20:51:29.533 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.533 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:51:29.533 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.533 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:51:29.533 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:02.797308+0000", 2023-09-23T20:51:29.533 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:29.533 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.533 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.533 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:51:29.533 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:02.797308+0000", 2023-09-23T20:51:29.533 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:29.534 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.534 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.534 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:51:29.534 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:02.797307+0000", 2023-09-23T20:51:29.534 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999995 2023-09-23T20:51:29.534 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.534 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.534 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:51:29.534 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:02.797312+0000", 2023-09-23T20:51:29.534 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.989e-06 2023-09-23T20:51:29.534 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.534 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.535 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:51:29.535 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:02.797334+0000", 2023-09-23T20:51:29.535 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.2416999999999999e-05 2023-09-23T20:51:29.535 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.535 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.535 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:51:29.535 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:02.797342+0000", 2023-09-23T20:51:29.535 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.5120000000000002e-06 2023-09-23T20:51:29.535 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.535 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.535 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:29.535 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:02.797395+0000", 2023-09-23T20:51:29.536 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.2951000000000002e-05 2023-09-23T20:51:29.536 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.536 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.536 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:51:29.536 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:02.797407+0000", 2023-09-23T20:51:29.536 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.2103000000000001e-05 2023-09-23T20:51:29.536 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.536 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.536 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:29.536 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:03.892091+0000", 2023-09-23T20:51:29.536 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0946837679999999 2023-09-23T20:51:29.536 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.537 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.537 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:51:29.537 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:03.892187+0000", 2023-09-23T20:51:29.537 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.6470000000000003e-05 2023-09-23T20:51:29.537 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.537 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.537 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,1,4", 2023-09-23T20:51:29.537 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:03.892266+0000", 2023-09-23T20:51:29.537 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.9254999999999999e-05 2023-09-23T20:51:29.537 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.537 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.537 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:51:29.537 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:03.899780+0000", 2023-09-23T20:51:29.538 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0075138289999999996 2023-09-23T20:51:29.538 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.538 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.538 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:29.538 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:03.905061+0000", 2023-09-23T20:51:29.538 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0052810770000000003 2023-09-23T20:51:29.538 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.538 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.538 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:29.538 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:03.905079+0000", 2023-09-23T20:51:29.538 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.7906000000000001e-05 2023-09-23T20:51:29.539 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.539 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.539 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:29.539 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:03.926035+0000", 2023-09-23T20:51:29.539 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.020956130999999999 2023-09-23T20:51:29.539 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.539 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.539 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:51:29.539 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:03.926093+0000", 2023-09-23T20:51:29.539 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.7944e-05 2023-09-23T20:51:29.539 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.539 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.540 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:51:29.540 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:03.926218+0000", 2023-09-23T20:51:29.540 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000124809 2023-09-23T20:51:29.540 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:29.540 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:51:29.540 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:29.540 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.540 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.540 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:181723 10.c 10:3851a352:::benchmark_data_smithi018_422431_object77227:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e604)", 2023-09-23T20:51:29.540 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:49:02.803937+0000", 2023-09-23T20:51:29.540 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 146.608196599, 2023-09-23T20:51:29.540 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.122505138, 2023-09-23T20:51:29.540 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:51:29.541 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:29.541 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:51:29.541 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:51:29.541 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:51:29.541 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 181723 2023-09-23T20:51:29.541 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.541 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:51:29.541 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.541 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:51:29.541 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:02.803937+0000", 2023-09-23T20:51:29.541 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:29.541 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.541 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.542 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:51:29.542 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:02.803937+0000", 2023-09-23T20:51:29.542 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:29.542 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.542 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.542 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:51:29.542 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:02.803936+0000", 2023-09-23T20:51:29.542 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:51:29.542 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.542 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.542 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:51:29.542 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:02.803941+0000", 2023-09-23T20:51:29.543 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.2809999999999999e-06 2023-09-23T20:51:29.543 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.543 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.543 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:51:29.543 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:02.803958+0000", 2023-09-23T20:51:29.543 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.6827e-05 2023-09-23T20:51:29.543 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.543 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.543 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:51:29.543 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:02.803964+0000", 2023-09-23T20:51:29.543 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.1970000000000001e-06 2023-09-23T20:51:29.543 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.544 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.544 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:29.544 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:02.804026+0000", 2023-09-23T20:51:29.544 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.1218999999999998e-05 2023-09-23T20:51:29.544 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.544 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.544 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:51:29.544 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:02.804041+0000", 2023-09-23T20:51:29.544 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.5138e-05 2023-09-23T20:51:29.544 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.544 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.544 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:29.544 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:03.894841+0000", 2023-09-23T20:51:29.545 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.090800038 2023-09-23T20:51:29.545 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.545 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.545 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:51:29.545 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:03.894931+0000", 2023-09-23T20:51:29.545 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.9925000000000004e-05 2023-09-23T20:51:29.545 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.545 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.545 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,1,4", 2023-09-23T20:51:29.545 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:03.895032+0000", 2023-09-23T20:51:29.545 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000100893 2023-09-23T20:51:29.546 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.546 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.546 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:51:29.546 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:03.899938+0000", 2023-09-23T20:51:29.546 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0049064119999999998 2023-09-23T20:51:29.546 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.546 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.546 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:29.546 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:03.905097+0000", 2023-09-23T20:51:29.546 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0051591639999999999 2023-09-23T20:51:29.546 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.547 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.547 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:29.547 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:03.911196+0000", 2023-09-23T20:51:29.547 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0060986790000000001 2023-09-23T20:51:29.547 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.547 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.547 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:29.547 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:03.926272+0000", 2023-09-23T20:51:29.547 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.015076473 2023-09-23T20:51:29.547 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.547 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.547 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:51:29.547 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:03.926334+0000", 2023-09-23T20:51:29.548 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.1849999999999999e-05 2023-09-23T20:51:29.548 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.548 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.548 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:51:29.548 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:03.926442+0000", 2023-09-23T20:51:29.548 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00010802300000000001 2023-09-23T20:51:29.548 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:29.548 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:51:29.548 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:29.548 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.548 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.548 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:181740 10.c 10:318e66ac:::benchmark_data_smithi018_422431_object77244:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e604)", 2023-09-23T20:51:29.549 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:49:02.823106+0000", 2023-09-23T20:51:29.549 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 146.58902746300001, 2023-09-23T20:51:29.549 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.1035798889999999, 2023-09-23T20:51:29.549 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:51:29.549 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:29.549 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:51:29.549 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:51:29.549 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:51:29.549 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 181740 2023-09-23T20:51:29.549 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.549 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:51:29.549 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.550 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:51:29.550 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:02.823106+0000", 2023-09-23T20:51:29.550 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:29.550 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.550 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.550 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:51:29.550 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:02.823106+0000", 2023-09-23T20:51:29.550 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:29.550 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.550 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.550 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:51:29.550 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:02.823105+0000", 2023-09-23T20:51:29.551 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:51:29.551 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.551 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.551 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:51:29.551 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:02.823112+0000", 2023-09-23T20:51:29.551 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.4200000000000004e-06 2023-09-23T20:51:29.551 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.551 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.551 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:51:29.551 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:02.823129+0000", 2023-09-23T20:51:29.551 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.7167999999999999e-05 2023-09-23T20:51:29.551 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.552 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.552 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:51:29.552 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:02.823136+0000", 2023-09-23T20:51:29.552 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.9210000000000003e-06 2023-09-23T20:51:29.552 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.552 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.552 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:29.552 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:02.823202+0000", 2023-09-23T20:51:29.552 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 6.6533999999999997e-05 2023-09-23T20:51:29.552 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.552 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.552 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:51:29.552 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:02.823216+0000", 2023-09-23T20:51:29.553 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.3587000000000001e-05 2023-09-23T20:51:29.553 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.553 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.553 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:29.553 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:03.897351+0000", 2023-09-23T20:51:29.553 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0741351130000001 2023-09-23T20:51:29.553 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.553 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.553 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:51:29.553 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:03.897436+0000", 2023-09-23T20:51:29.553 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.5368e-05 2023-09-23T20:51:29.553 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.554 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.554 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,1,4", 2023-09-23T20:51:29.554 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:03.897508+0000", 2023-09-23T20:51:29.554 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.1768999999999997e-05 2023-09-23T20:51:29.554 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.554 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.554 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:29.554 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:03.905122+0000", 2023-09-23T20:51:29.554 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0076139989999999998 2023-09-23T20:51:29.554 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.554 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.555 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:51:29.555 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:03.910985+0000", 2023-09-23T20:51:29.555 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0058633829999999998 2023-09-23T20:51:29.555 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.555 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.555 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:29.555 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:03.911313+0000", 2023-09-23T20:51:29.555 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00032725300000000001 2023-09-23T20:51:29.555 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.555 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.555 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:29.555 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:03.926509+0000", 2023-09-23T20:51:29.556 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.015196397 2023-09-23T20:51:29.556 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.556 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.556 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:51:29.556 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:03.926553+0000", 2023-09-23T20:51:29.556 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.3553999999999997e-05 2023-09-23T20:51:29.556 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.556 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.556 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:51:29.556 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:03.926686+0000", 2023-09-23T20:51:29.556 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000133535 2023-09-23T20:51:29.556 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:29.557 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:51:29.557 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:29.557 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.557 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.557 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:181752 10.c 10:34fae4dd:::benchmark_data_smithi018_422431_object77256:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e604)", 2023-09-23T20:51:29.557 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:49:02.837351+0000", 2023-09-23T20:51:29.557 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 146.57478255000001, 2023-09-23T20:51:29.557 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0895556049999999, 2023-09-23T20:51:29.557 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:51:29.557 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:29.557 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:51:29.557 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:51:29.558 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:51:29.558 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 181752 2023-09-23T20:51:29.558 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.558 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:51:29.558 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.558 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:51:29.558 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:02.837351+0000", 2023-09-23T20:51:29.558 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:29.558 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.558 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.558 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:51:29.558 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:02.837351+0000", 2023-09-23T20:51:29.559 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:29.559 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.559 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.559 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:51:29.559 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:02.837350+0000", 2023-09-23T20:51:29.559 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.999999 2023-09-23T20:51:29.559 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.559 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.559 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:51:29.559 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:02.837355+0000", 2023-09-23T20:51:29.559 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.853e-06 2023-09-23T20:51:29.559 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.560 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.560 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:51:29.560 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:02.837368+0000", 2023-09-23T20:51:29.560 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.2765999999999999e-05 2023-09-23T20:51:29.560 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.560 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.560 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:51:29.560 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:02.837372+0000", 2023-09-23T20:51:29.560 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.5569999999999997e-06 2023-09-23T20:51:29.560 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.560 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.561 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:29.561 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:02.837424+0000", 2023-09-23T20:51:29.561 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.2101999999999997e-05 2023-09-23T20:51:29.561 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.561 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.561 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:51:29.561 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:02.837434+0000", 2023-09-23T20:51:29.561 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 9.4979999999999995e-06 2023-09-23T20:51:29.561 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.561 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.561 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:29.561 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:03.899956+0000", 2023-09-23T20:51:29.562 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0625220049999999 2023-09-23T20:51:29.562 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.562 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.562 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:51:29.562 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:03.900040+0000", 2023-09-23T20:51:29.562 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.3545000000000001e-05 2023-09-23T20:51:29.562 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.562 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.562 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,1,4", 2023-09-23T20:51:29.562 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:03.900126+0000", 2023-09-23T20:51:29.562 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.6438000000000001e-05 2023-09-23T20:51:29.562 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.563 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.563 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:29.563 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:03.910971+0000", 2023-09-23T20:51:29.563 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.010845237000000001 2023-09-23T20:51:29.563 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.563 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.563 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:51:29.563 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:03.910994+0000", 2023-09-23T20:51:29.563 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 2.2642000000000001e-05 2023-09-23T20:51:29.563 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.563 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.564 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:29.564 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:03.911347+0000", 2023-09-23T20:51:29.564 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00035345800000000002 2023-09-23T20:51:29.564 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.564 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.564 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:29.564 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:03.926751+0000", 2023-09-23T20:51:29.564 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.015403795 2023-09-23T20:51:29.564 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.564 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.564 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:51:29.564 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:03.926796+0000", 2023-09-23T20:51:29.564 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.4443999999999997e-05 2023-09-23T20:51:29.565 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.565 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.565 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:51:29.565 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:03.926907+0000", 2023-09-23T20:51:29.565 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00011123700000000001 2023-09-23T20:51:29.565 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:29.565 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:51:29.565 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:29.565 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.565 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.565 INFO:teuthology.orchestra.run.smithi149.stdout: "description": "osd_op(client.46625.0:181774 10.c 10:32e12796:::benchmark_data_smithi018_422431_object77278:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e604)", 2023-09-23T20:51:29.566 INFO:teuthology.orchestra.run.smithi149.stdout: "initiated_at": "2023-09-23T20:49:02.870850+0000", 2023-09-23T20:51:29.566 INFO:teuthology.orchestra.run.smithi149.stdout: "age": 146.541283327, 2023-09-23T20:51:29.566 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.056271878, 2023-09-23T20:51:29.566 INFO:teuthology.orchestra.run.smithi149.stdout: "type_data": { 2023-09-23T20:51:29.566 INFO:teuthology.orchestra.run.smithi149.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:29.566 INFO:teuthology.orchestra.run.smithi149.stdout: "client_info": { 2023-09-23T20:51:29.566 INFO:teuthology.orchestra.run.smithi149.stdout: "client": "client.46625", 2023-09-23T20:51:29.566 INFO:teuthology.orchestra.run.smithi149.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:51:29.566 INFO:teuthology.orchestra.run.smithi149.stdout: "tid": 181774 2023-09-23T20:51:29.566 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.566 INFO:teuthology.orchestra.run.smithi149.stdout: "events": [ 2023-09-23T20:51:29.566 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.567 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "initiated", 2023-09-23T20:51:29.567 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:02.870850+0000", 2023-09-23T20:51:29.567 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:29.567 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.567 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.567 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "throttled", 2023-09-23T20:51:29.567 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:02.870850+0000", 2023-09-23T20:51:29.567 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0 2023-09-23T20:51:29.567 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.567 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.567 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "header_read", 2023-09-23T20:51:29.567 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:02.870849+0000", 2023-09-23T20:51:29.568 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4294967295.9999986 2023-09-23T20:51:29.568 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.568 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.568 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "all_read", 2023-09-23T20:51:29.568 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:02.870856+0000", 2023-09-23T20:51:29.568 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.3309999999999998e-06 2023-09-23T20:51:29.568 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.568 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.568 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "dispatched", 2023-09-23T20:51:29.568 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:02.870874+0000", 2023-09-23T20:51:29.568 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.8117e-05 2023-09-23T20:51:29.568 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.569 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.569 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "queued_for_pg", 2023-09-23T20:51:29.569 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:02.870883+0000", 2023-09-23T20:51:29.569 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.5229999999999995e-06 2023-09-23T20:51:29.569 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.569 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.569 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:29.569 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:02.870939+0000", 2023-09-23T20:51:29.569 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 5.6112000000000001e-05 2023-09-23T20:51:29.569 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.569 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.569 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for peered", 2023-09-23T20:51:29.569 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:02.870950+0000", 2023-09-23T20:51:29.570 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.0657e-05 2023-09-23T20:51:29.570 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.570 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.570 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "reached_pg", 2023-09-23T20:51:29.570 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:03.902547+0000", 2023-09-23T20:51:29.570 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 1.031597278 2023-09-23T20:51:29.570 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.570 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.570 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "started", 2023-09-23T20:51:29.570 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:03.902634+0000", 2023-09-23T20:51:29.570 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 8.7405999999999996e-05 2023-09-23T20:51:29.570 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.571 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.571 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "waiting for subops from 0,1,4", 2023-09-23T20:51:29.571 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:03.902744+0000", 2023-09-23T20:51:29.571 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.000109512 2023-09-23T20:51:29.571 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.571 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.571 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "op_commit", 2023-09-23T20:51:29.571 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:03.911210+0000", 2023-09-23T20:51:29.571 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0084662520000000005 2023-09-23T20:51:29.571 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.571 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.571 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:29.571 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:03.911289+0000", 2023-09-23T20:51:29.572 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 7.8888999999999996e-05 2023-09-23T20:51:29.572 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.572 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.572 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:29.572 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:03.922856+0000", 2023-09-23T20:51:29.572 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.011567424 2023-09-23T20:51:29.572 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.572 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.572 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:29.572 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:03.926949+0000", 2023-09-23T20:51:29.572 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.0040922739999999999 2023-09-23T20:51:29.572 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.572 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.573 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "commit_sent", 2023-09-23T20:51:29.573 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:03.926993+0000", 2023-09-23T20:51:29.573 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 4.4132999999999999e-05 2023-09-23T20:51:29.573 INFO:teuthology.orchestra.run.smithi149.stdout: }, 2023-09-23T20:51:29.573 INFO:teuthology.orchestra.run.smithi149.stdout: { 2023-09-23T20:51:29.573 INFO:teuthology.orchestra.run.smithi149.stdout: "event": "done", 2023-09-23T20:51:29.573 INFO:teuthology.orchestra.run.smithi149.stdout: "time": "2023-09-23T20:49:03.927122+0000", 2023-09-23T20:51:29.573 INFO:teuthology.orchestra.run.smithi149.stdout: "duration": 0.00012936800000000001 2023-09-23T20:51:29.573 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:29.573 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:51:29.573 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:29.573 INFO:teuthology.orchestra.run.smithi149.stdout: } 2023-09-23T20:51:29.574 INFO:teuthology.orchestra.run.smithi149.stdout: ] 2023-09-23T20:51:29.574 INFO:teuthology.orchestra.run.smithi149.stdout:} 2023-09-23T20:51:29.844 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:29 smithi149 ceph-mon[170394]: failed to encode map e678 with expected crc 2023-09-23T20:51:29.845 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:29 smithi149 ceph-mon[170394]: failed to encode map e678 with expected crc 2023-09-23T20:51:29.845 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:29 smithi149 ceph-mon[170394]: failed to encode map e678 with expected crc 2023-09-23T20:51:29.845 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:29 smithi149 ceph-mon[170394]: pgmap v1655: 103 pgs: 3 active+undersized+remapped, 1 peering, 99 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 769 KiB/s, 7 objects/s recovering 2023-09-23T20:51:29.845 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:29 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:51:29.846 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T20:51:30.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:29 smithi018 ceph-mon[184144]: failed to encode map e678 with expected crc 2023-09-23T20:51:30.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:29 smithi018 ceph-mon[184144]: failed to encode map e678 with expected crc 2023-09-23T20:51:30.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:29 smithi018 ceph-mon[184144]: failed to encode map e678 with expected crc 2023-09-23T20:51:30.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:29 smithi018 ceph-mon[184144]: pgmap v1655: 103 pgs: 3 active+undersized+remapped, 1 peering, 99 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 769 KiB/s, 7 objects/s recovering 2023-09-23T20:51:30.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:29 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:51:30.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:29 smithi018 ceph-mon[179647]: failed to encode map e678 with expected crc 2023-09-23T20:51:30.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:29 smithi018 ceph-mon[179647]: failed to encode map e678 with expected crc 2023-09-23T20:51:30.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:29 smithi018 ceph-mon[179647]: failed to encode map e678 with expected crc 2023-09-23T20:51:30.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:29 smithi018 ceph-mon[179647]: pgmap v1655: 103 pgs: 3 active+undersized+remapped, 1 peering, 99 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 769 KiB/s, 7 objects/s recovering 2023-09-23T20:51:30.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:29 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:51:30.459 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:51:30.779 INFO:teuthology.orchestra.run.smithi018.stderr:set osd.0 primary-affinity to 1 (8655362) 2023-09-23T20:51:30.966 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:30 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1956398272' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T20:51:30.967 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:30 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1956398272' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T20:51:31.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:30 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1956398272' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T20:51:31.418 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd reweight 2 1 2023-09-23T20:51:31.882 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2023-09-23T20:51:31.882 INFO:teuthology.orchestra.run.smithi018.stdout: "ops": [], 2023-09-23T20:51:31.882 INFO:teuthology.orchestra.run.smithi018.stdout: "num_ops": 0 2023-09-23T20:51:31.882 INFO:teuthology.orchestra.run.smithi018.stdout:} 2023-09-23T20:51:32.025 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:31 smithi018 ceph-mon[179647]: pgmap v1656: 103 pgs: 103 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 39 KiB/s, 3 objects/s recovering 2023-09-23T20:51:32.025 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:31 smithi018 ceph-mon[179647]: failed to encode map e679 with expected crc 2023-09-23T20:51:32.025 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:31 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1956398272' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2023-09-23T20:51:32.025 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:31 smithi018 ceph-mon[179647]: osdmap e679: 8 total, 8 up, 8 in 2023-09-23T20:51:32.026 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:31 smithi018 ceph-mon[184144]: pgmap v1656: 103 pgs: 103 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 39 KiB/s, 3 objects/s recovering 2023-09-23T20:51:32.026 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:31 smithi018 ceph-mon[184144]: failed to encode map e679 with expected crc 2023-09-23T20:51:32.026 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:31 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1956398272' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2023-09-23T20:51:32.026 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:31 smithi018 ceph-mon[184144]: osdmap e679: 8 total, 8 up, 8 in 2023-09-23T20:51:32.189 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:51:32.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:31 smithi149 ceph-mon[170394]: pgmap v1656: 103 pgs: 103 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 39 KiB/s, 3 objects/s recovering 2023-09-23T20:51:32.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:31 smithi149 ceph-mon[170394]: failed to encode map e679 with expected crc 2023-09-23T20:51:32.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:31 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1956398272' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2023-09-23T20:51:32.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:31 smithi149 ceph-mon[170394]: osdmap e679: 8 total, 8 up, 8 in 2023-09-23T20:51:32.231 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T20:51:33.020 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:51:34.140 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:33 smithi018 ceph-mon[184144]: pgmap v1658: 103 pgs: 103 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 34 KiB/s, 3 objects/s recovering 2023-09-23T20:51:34.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:33 smithi018 ceph-mon[179647]: pgmap v1658: 103 pgs: 103 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 34 KiB/s, 3 objects/s recovering 2023-09-23T20:51:34.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:33 smithi149 ceph-mon[170394]: pgmap v1658: 103 pgs: 103 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 34 KiB/s, 3 objects/s recovering 2023-09-23T20:51:34.349 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2023-09-23T20:51:34.349 INFO:teuthology.orchestra.run.smithi018.stdout: "ops": [], 2023-09-23T20:51:34.349 INFO:teuthology.orchestra.run.smithi018.stdout: "complaint_time": 30, 2023-09-23T20:51:34.349 INFO:teuthology.orchestra.run.smithi018.stdout: "num_blocked_ops": 0 2023-09-23T20:51:34.350 INFO:teuthology.orchestra.run.smithi018.stdout:} 2023-09-23T20:51:34.789 INFO:teuthology.orchestra.run.smithi018.stderr:reweighted osd.2 to 1 (10000) 2023-09-23T20:51:35.092 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:34 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3900688230' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T20:51:35.092 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:34 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T20:51:35.092 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:34 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3900688230' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T20:51:35.093 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:34 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T20:51:35.093 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:51:34 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:51:34] "GET /metrics HTTP/1.1" 200 34178 "" "Prometheus/2.43.0" 2023-09-23T20:51:35.095 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T20:51:35.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:34 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3900688230' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T20:51:35.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:34 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T20:51:35.282 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 2 1 2023-09-23T20:51:35.708 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:51:36.090 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:35 smithi018 ceph-mon[179647]: pgmap v1659: 103 pgs: 103 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 32 KiB/s, 2 objects/s recovering 2023-09-23T20:51:36.090 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:35 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T20:51:36.090 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:35 smithi018 ceph-mon[179647]: osdmap e680: 8 total, 8 up, 8 in 2023-09-23T20:51:36.091 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:35 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:51:36.091 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:35 smithi018 ceph-mon[184144]: pgmap v1659: 103 pgs: 103 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 32 KiB/s, 2 objects/s recovering 2023-09-23T20:51:36.091 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:35 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T20:51:36.091 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:35 smithi018 ceph-mon[184144]: osdmap e680: 8 total, 8 up, 8 in 2023-09-23T20:51:36.091 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:35 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:51:36.109 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:51:36.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:35 smithi149 ceph-mon[170394]: pgmap v1659: 103 pgs: 103 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 32 KiB/s, 2 objects/s recovering 2023-09-23T20:51:36.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:35 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T20:51:36.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:35 smithi149 ceph-mon[170394]: osdmap e680: 8 total, 8 up, 8 in 2023-09-23T20:51:36.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:35 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:51:36.963 INFO:teuthology.orchestra.run.smithi018.stdout:{ 2023-09-23T20:51:36.963 INFO:teuthology.orchestra.run.smithi018.stdout: "size": 20, 2023-09-23T20:51:36.964 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 600, 2023-09-23T20:51:36.964 INFO:teuthology.orchestra.run.smithi018.stdout: "ops": [ 2023-09-23T20:51:36.964 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:36.964 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.36450.0:141029 9.a 9:572023e9:::benchmark_data_smithi018_381516_object27195:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e481)", 2023-09-23T20:51:36.964 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:43:15.738002+0000", 2023-09-23T20:51:36.964 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 501.222471085, 2023-09-23T20:51:36.964 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0527425420000001, 2023-09-23T20:51:36.964 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:51:36.964 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:51:36.964 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:51:36.964 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.36450", 2023-09-23T20:51:36.964 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:51:36.965 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 141029 2023-09-23T20:51:36.965 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:36.965 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:51:36.965 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:36.965 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:51:36.965 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738002+0000", 2023-09-23T20:51:36.965 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:51:36.965 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:36.965 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:36.965 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:51:36.965 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738002+0000", 2023-09-23T20:51:36.965 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:51:36.966 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:36.966 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:36.966 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:51:36.966 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738000+0000", 2023-09-23T20:51:36.966 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:51:36.966 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:36.966 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:36.966 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:51:36.966 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738007+0000", 2023-09-23T20:51:36.966 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.4760000000000001e-06 2023-09-23T20:51:36.966 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:36.967 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:36.967 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:51:36.967 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738025+0000", 2023-09-23T20:51:36.967 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.7952e-05 2023-09-23T20:51:36.967 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:36.967 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:36.967 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:51:36.967 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738035+0000", 2023-09-23T20:51:36.967 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.6120000000000005e-06 2023-09-23T20:51:36.967 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:36.967 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:36.967 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:51:36.968 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.744123+0000", 2023-09-23T20:51:36.968 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0060876410000000004 2023-09-23T20:51:36.968 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:36.968 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:36.968 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op must wait for map", 2023-09-23T20:51:36.968 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.744131+0000", 2023-09-23T20:51:36.968 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.8800000000000008e-06 2023-09-23T20:51:36.968 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:36.968 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:36.968 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:51:36.968 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.754919+0000", 2023-09-23T20:51:36.968 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.010788483999999999 2023-09-23T20:51:36.969 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:36.969 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:36.969 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:51:36.969 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.754929+0000", 2023-09-23T20:51:36.969 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0309000000000001e-05 2023-09-23T20:51:36.969 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:36.969 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:36.969 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:51:36.969 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:16.790652+0000", 2023-09-23T20:51:36.969 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0357231570000001 2023-09-23T20:51:36.969 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:36.969 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:36.969 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:51:36.970 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:16.790744+0000", 2023-09-23T20:51:36.970 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.1743000000000006e-05 2023-09-23T20:51:36.970 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:51:36.970 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:51:36.970 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:51:36.970 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:36.970 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:36.970 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.36450.0:141040 9.a 9:53252326:::benchmark_data_smithi018_381516_object27206:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e481)", 2023-09-23T20:51:36.970 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:43:15.738057+0000", 2023-09-23T20:51:36.970 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 501.22241547300001, 2023-09-23T20:51:36.970 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0530090860000001, 2023-09-23T20:51:36.970 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:51:36.971 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:51:36.971 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:51:36.971 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.36450", 2023-09-23T20:51:36.971 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:51:36.971 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 141040 2023-09-23T20:51:36.971 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:36.971 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:51:36.971 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:36.971 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:51:36.971 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738057+0000", 2023-09-23T20:51:36.971 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:51:36.972 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:36.972 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:36.972 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:51:36.972 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738057+0000", 2023-09-23T20:51:36.972 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:51:36.972 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:36.972 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:36.972 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:51:36.972 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738057+0000", 2023-09-23T20:51:36.972 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:51:36.973 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:36.973 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:36.973 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:51:36.973 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738060+0000", 2023-09-23T20:51:36.973 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.9699999999999999e-06 2023-09-23T20:51:36.973 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:36.973 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:36.973 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:51:36.973 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738066+0000", 2023-09-23T20:51:36.973 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.5019999999999998e-06 2023-09-23T20:51:36.973 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:36.973 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:36.974 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:51:36.974 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738072+0000", 2023-09-23T20:51:36.974 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.2209999999999996e-06 2023-09-23T20:51:36.974 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:36.974 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:36.974 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:51:36.974 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.744157+0000", 2023-09-23T20:51:36.974 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0060856 2023-09-23T20:51:36.974 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:36.974 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:36.974 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting_for_map not empty", 2023-09-23T20:51:36.975 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.744162+0000", 2023-09-23T20:51:36.975 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.1610000000000002e-06 2023-09-23T20:51:36.975 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:36.975 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:36.975 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:51:36.975 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.754951+0000", 2023-09-23T20:51:36.975 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.010789127000000001 2023-09-23T20:51:36.975 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:36.975 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:36.975 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:51:36.975 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.754959+0000", 2023-09-23T20:51:36.975 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.3189999999999997e-06 2023-09-23T20:51:36.976 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:36.976 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:36.976 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:51:36.976 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:16.790999+0000", 2023-09-23T20:51:36.976 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.03604052 2023-09-23T20:51:36.976 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:36.976 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:36.976 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:51:36.976 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:16.791066+0000", 2023-09-23T20:51:36.976 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.7101999999999996e-05 2023-09-23T20:51:36.976 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:51:36.976 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:51:36.976 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:51:36.977 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:36.977 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:36.977 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.36450.0:141043 9.a 9:57a9fa95:::benchmark_data_smithi018_381516_object27209:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e481)", 2023-09-23T20:51:36.977 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:43:15.738077+0000", 2023-09-23T20:51:36.977 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 501.22239583599998, 2023-09-23T20:51:36.977 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0909135969999999, 2023-09-23T20:51:36.977 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:51:36.977 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:51:36.977 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:51:36.977 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.36450", 2023-09-23T20:51:36.977 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:51:36.977 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 141043 2023-09-23T20:51:36.978 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:36.978 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:51:36.978 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:36.978 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:51:36.978 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738077+0000", 2023-09-23T20:51:36.978 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:51:36.978 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:36.978 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:36.978 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:51:36.978 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738077+0000", 2023-09-23T20:51:36.978 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:51:36.978 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:36.979 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:36.979 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:51:36.979 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738077+0000", 2023-09-23T20:51:36.979 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:51:36.979 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:36.979 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:36.979 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:51:36.979 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738079+0000", 2023-09-23T20:51:36.979 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.0320000000000002e-06 2023-09-23T20:51:36.979 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:36.979 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:36.979 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:51:36.980 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738084+0000", 2023-09-23T20:51:36.980 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.4310000000000002e-06 2023-09-23T20:51:36.980 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:36.980 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:36.980 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:51:36.980 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738089+0000", 2023-09-23T20:51:36.980 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.6789999999999996e-06 2023-09-23T20:51:36.980 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:36.980 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:36.980 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:51:36.980 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.744187+0000", 2023-09-23T20:51:36.980 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0060979859999999997 2023-09-23T20:51:36.981 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:36.981 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:36.981 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting_for_map not empty", 2023-09-23T20:51:36.981 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.744194+0000", 2023-09-23T20:51:36.981 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.7090000000000002e-06 2023-09-23T20:51:36.981 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:36.981 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:36.981 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:51:36.981 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.754979+0000", 2023-09-23T20:51:36.981 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.010784141000000001 2023-09-23T20:51:36.981 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:36.981 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:36.981 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:51:36.982 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.754986+0000", 2023-09-23T20:51:36.982 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.0339999999999999e-06 2023-09-23T20:51:36.982 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:36.982 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:36.982 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:51:36.982 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:16.791090+0000", 2023-09-23T20:51:36.982 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.036103939 2023-09-23T20:51:36.982 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:36.982 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:36.982 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for missing object", 2023-09-23T20:51:36.982 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:16.791184+0000", 2023-09-23T20:51:36.983 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.4289000000000005e-05 2023-09-23T20:51:36.983 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:36.983 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:36.983 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:51:36.983 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:16.828905+0000", 2023-09-23T20:51:36.983 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.037721372000000003 2023-09-23T20:51:36.983 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:36.983 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:36.983 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:51:36.983 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:16.828991+0000", 2023-09-23T20:51:36.983 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.5331e-05 2023-09-23T20:51:36.983 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:51:36.983 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:51:36.984 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:51:36.984 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:36.984 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:36.984 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.36450.0:141045 9.f 9:fa55dbf8:::benchmark_data_smithi018_381516_object27211:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e481)", 2023-09-23T20:51:36.984 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:43:15.738093+0000", 2023-09-23T20:51:36.984 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 501.22237997899998, 2023-09-23T20:51:36.984 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0403492489999999, 2023-09-23T20:51:36.984 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:51:36.984 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:51:36.984 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:51:36.984 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.36450", 2023-09-23T20:51:36.984 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:51:36.984 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 141045 2023-09-23T20:51:36.985 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:36.985 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:51:36.985 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:36.985 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:51:36.985 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738093+0000", 2023-09-23T20:51:36.985 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:51:36.985 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:36.985 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:36.985 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:51:36.985 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738093+0000", 2023-09-23T20:51:36.985 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:51:36.985 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:36.986 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:36.986 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:51:36.986 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738092+0000", 2023-09-23T20:51:36.986 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999995 2023-09-23T20:51:36.986 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:36.986 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:36.986 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:51:36.986 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738095+0000", 2023-09-23T20:51:36.986 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.063e-06 2023-09-23T20:51:36.986 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:36.986 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:36.986 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:51:36.987 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738100+0000", 2023-09-23T20:51:36.987 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.2469999999999999e-06 2023-09-23T20:51:36.987 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:36.987 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:36.987 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:51:36.987 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738104+0000", 2023-09-23T20:51:36.987 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.3649999999999997e-06 2023-09-23T20:51:36.987 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:36.987 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:36.987 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:51:36.987 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738271+0000", 2023-09-23T20:51:36.987 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00016673799999999999 2023-09-23T20:51:36.987 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:36.988 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:36.988 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op must wait for map", 2023-09-23T20:51:36.988 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.738278+0000", 2023-09-23T20:51:36.988 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.0659999999999998e-06 2023-09-23T20:51:36.988 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:36.988 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:36.988 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:51:36.988 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.770314+0000", 2023-09-23T20:51:36.988 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.032036012000000003 2023-09-23T20:51:36.988 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:36.988 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:36.988 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:51:36.988 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.770322+0000", 2023-09-23T20:51:36.989 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.9330000000000001e-06 2023-09-23T20:51:36.989 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:36.989 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:36.989 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:51:36.989 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:16.776028+0000", 2023-09-23T20:51:36.989 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0057059290000001 2023-09-23T20:51:36.989 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:36.989 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:36.989 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:51:36.989 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:16.778442+0000", 2023-09-23T20:51:36.989 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0024142080000000002 2023-09-23T20:51:36.989 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:51:36.990 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:51:36.990 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:51:36.990 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:36.990 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:36.990 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.36450.0:141047 9.a 9:55e9d18e:::benchmark_data_smithi018_381516_object27213:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e481)", 2023-09-23T20:51:36.990 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:43:15.747358+0000", 2023-09-23T20:51:36.990 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 501.21311477299997, 2023-09-23T20:51:36.990 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.088479043, 2023-09-23T20:51:36.990 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:51:36.990 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:36.990 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:51:36.990 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.36450", 2023-09-23T20:51:36.991 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/1431903067", 2023-09-23T20:51:36.991 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 141047 2023-09-23T20:51:36.991 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:36.991 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:51:36.991 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:36.991 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:51:36.991 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.747358+0000", 2023-09-23T20:51:36.991 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:51:36.991 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:36.991 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:36.991 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:51:36.991 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.747358+0000", 2023-09-23T20:51:36.992 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:51:36.992 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:36.992 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:36.992 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:51:36.992 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.747356+0000", 2023-09-23T20:51:36.992 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:51:36.992 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:36.992 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:36.992 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:51:36.992 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.747364+0000", 2023-09-23T20:51:36.992 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.6569999999999997e-06 2023-09-23T20:51:36.992 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:36.993 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:36.993 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:51:36.993 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.747379+0000", 2023-09-23T20:51:36.993 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.5153999999999999e-05 2023-09-23T20:51:36.993 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:36.993 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:36.993 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:51:36.993 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.747388+0000", 2023-09-23T20:51:36.993 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.7059999999999993e-06 2023-09-23T20:51:36.993 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:36.993 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:36.993 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:51:36.993 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.748082+0000", 2023-09-23T20:51:36.994 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00069463100000000002 2023-09-23T20:51:36.994 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:36.994 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:36.994 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting_for_map not empty", 2023-09-23T20:51:36.994 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.748090+0000", 2023-09-23T20:51:36.994 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.1349999999999992e-06 2023-09-23T20:51:36.994 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:36.994 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:36.994 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:51:36.994 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.755006+0000", 2023-09-23T20:51:36.994 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0069154289999999998 2023-09-23T20:51:36.995 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:36.995 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:36.995 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:51:36.995 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:15.755013+0000", 2023-09-23T20:51:36.995 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.7909999999999999e-06 2023-09-23T20:51:36.995 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:36.995 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:36.995 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:51:36.995 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:16.791216+0000", 2023-09-23T20:51:36.995 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.036203188 2023-09-23T20:51:36.995 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:36.995 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:36.996 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:51:36.996 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:16.791332+0000", 2023-09-23T20:51:36.996 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000115798 2023-09-23T20:51:36.996 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:36.996 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:36.996 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 5,7", 2023-09-23T20:51:36.996 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:16.791928+0000", 2023-09-23T20:51:36.996 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00059646299999999998 2023-09-23T20:51:36.996 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:36.996 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:36.996 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:36.996 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:16.822354+0000", 2023-09-23T20:51:36.997 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.030425812999999999 2023-09-23T20:51:36.997 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:36.997 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:36.997 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:51:36.997 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:16.822834+0000", 2023-09-23T20:51:36.997 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00048018300000000002 2023-09-23T20:51:36.997 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:36.997 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:36.997 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:36.997 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:16.835654+0000", 2023-09-23T20:51:36.997 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.012820066 2023-09-23T20:51:36.997 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:36.998 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:36.998 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:51:36.998 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:16.835711+0000", 2023-09-23T20:51:36.998 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.7300999999999999e-05 2023-09-23T20:51:36.998 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:36.998 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:36.998 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:51:36.998 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:43:16.835837+0000", 2023-09-23T20:51:36.998 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00012553400000000001 2023-09-23T20:51:36.998 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:51:36.998 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:51:36.998 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:51:36.999 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:36.999 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:36.999 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:102280 10.2 10:4d2f73b6:::benchmark_data_smithi018_422431_object102279:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e551)", 2023-09-23T20:51:36.999 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:47:08.174176+0000", 2023-09-23T20:51:36.999 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 268.78629688900003, 2023-09-23T20:51:36.999 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0912579659999999, 2023-09-23T20:51:36.999 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:51:36.999 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:36.999 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:51:36.999 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:51:36.999 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:51:36.999 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 102280 2023-09-23T20:51:36.999 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.000 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:51:37.000 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.000 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:51:37.000 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:08.174176+0000", 2023-09-23T20:51:37.000 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:51:37.000 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.000 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.000 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:51:37.000 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:08.174176+0000", 2023-09-23T20:51:37.000 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:51:37.000 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.000 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.001 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:51:37.001 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:08.174174+0000", 2023-09-23T20:51:37.001 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:51:37.001 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.001 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.001 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:51:37.001 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:08.174198+0000", 2023-09-23T20:51:37.001 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.3923000000000001e-05 2023-09-23T20:51:37.001 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.001 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.001 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:51:37.001 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:08.174211+0000", 2023-09-23T20:51:37.002 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3324e-05 2023-09-23T20:51:37.002 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.002 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.002 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:51:37.002 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:08.174221+0000", 2023-09-23T20:51:37.002 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.5740000000000002e-06 2023-09-23T20:51:37.002 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.002 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.002 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:51:37.002 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:08.186392+0000", 2023-09-23T20:51:37.002 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.012171184999999999 2023-09-23T20:51:37.002 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.003 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.003 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:51:37.003 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:08.186404+0000", 2023-09-23T20:51:37.003 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1878e-05 2023-09-23T20:51:37.003 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.003 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.003 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:51:37.003 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:09.252235+0000", 2023-09-23T20:51:37.003 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0658310289999999 2023-09-23T20:51:37.003 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.003 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.003 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:51:37.004 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:09.252439+0000", 2023-09-23T20:51:37.004 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00020431400000000001 2023-09-23T20:51:37.004 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.004 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.004 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 0,4", 2023-09-23T20:51:37.004 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:09.252618+0000", 2023-09-23T20:51:37.004 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00017877799999999999 2023-09-23T20:51:37.004 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.004 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.004 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:37.004 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:09.258806+0000", 2023-09-23T20:51:37.004 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0061873279999999998 2023-09-23T20:51:37.005 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.005 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.005 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:37.005 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:09.261128+0000", 2023-09-23T20:51:37.005 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.002322228 2023-09-23T20:51:37.005 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.005 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.005 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:51:37.005 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:09.264995+0000", 2023-09-23T20:51:37.005 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0038676840000000001 2023-09-23T20:51:37.005 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.005 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.005 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:51:37.006 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:09.265068+0000", 2023-09-23T20:51:37.006 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.2390000000000003e-05 2023-09-23T20:51:37.006 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.006 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.006 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:51:37.006 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:47:09.265434+0000", 2023-09-23T20:51:37.006 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00036597799999999998 2023-09-23T20:51:37.006 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:51:37.006 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:51:37.006 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:51:37.006 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.007 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.007 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:181708 10.16 10.b7205ed6 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e603)", 2023-09-23T20:51:37.007 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:49:01.717054+0000", 2023-09-23T20:51:37.007 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 155.243418908, 2023-09-23T20:51:37.007 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.97645376399999995, 2023-09-23T20:51:37.007 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:51:37.007 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:51:37.007 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:51:37.007 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:51:37.007 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:51:37.007 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 181708 2023-09-23T20:51:37.007 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.008 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:51:37.008 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.008 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:51:37.008 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:01.717054+0000", 2023-09-23T20:51:37.008 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:51:37.008 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.008 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.008 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:51:37.008 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:01.717054+0000", 2023-09-23T20:51:37.008 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:51:37.008 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.008 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.009 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:51:37.009 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:01.717052+0000", 2023-09-23T20:51:37.009 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:51:37.009 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.009 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.009 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:51:37.009 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:01.717060+0000", 2023-09-23T20:51:37.009 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.6389999999999998e-06 2023-09-23T20:51:37.009 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.009 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.009 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:51:37.009 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:01.717081+0000", 2023-09-23T20:51:37.010 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.0638000000000001e-05 2023-09-23T20:51:37.010 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.010 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.010 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:51:37.010 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:01.717091+0000", 2023-09-23T20:51:37.010 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.9410000000000002e-06 2023-09-23T20:51:37.010 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.010 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.010 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:51:37.010 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:01.717148+0000", 2023-09-23T20:51:37.010 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.6777000000000002e-05 2023-09-23T20:51:37.010 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.011 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.011 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:51:37.011 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:01.717157+0000", 2023-09-23T20:51:37.011 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.8040000000000007e-06 2023-09-23T20:51:37.011 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.011 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.011 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:51:37.011 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:02.693500+0000", 2023-09-23T20:51:37.011 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.97634373500000005 2023-09-23T20:51:37.011 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.011 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.011 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:51:37.012 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:02.693508+0000", 2023-09-23T20:51:37.012 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.3490000000000003e-06 2023-09-23T20:51:37.012 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:51:37.012 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:51:37.012 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:51:37.012 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.012 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.012 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:181726 10.2 10:47312290:::benchmark_data_smithi018_422431_object77230:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e604)", 2023-09-23T20:51:37.012 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:49:02.807907+0000", 2023-09-23T20:51:37.012 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 154.152565815, 2023-09-23T20:51:37.012 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.99641389400000002, 2023-09-23T20:51:37.012 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:51:37.013 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:37.013 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:51:37.013 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:51:37.013 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:51:37.013 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 181726 2023-09-23T20:51:37.013 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.013 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:51:37.013 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.013 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:51:37.013 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:02.807907+0000", 2023-09-23T20:51:37.013 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:51:37.013 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.014 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.014 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:51:37.014 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:02.807907+0000", 2023-09-23T20:51:37.014 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:51:37.014 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.014 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.014 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:51:37.014 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:02.807905+0000", 2023-09-23T20:51:37.014 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:51:37.014 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.014 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.014 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:51:37.014 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:02.807913+0000", 2023-09-23T20:51:37.015 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.1489999999999994e-06 2023-09-23T20:51:37.015 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.015 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.015 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:51:37.015 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:02.807926+0000", 2023-09-23T20:51:37.015 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2798e-05 2023-09-23T20:51:37.015 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.015 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.015 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:51:37.015 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:02.807935+0000", 2023-09-23T20:51:37.015 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.6000000000000007e-06 2023-09-23T20:51:37.015 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.016 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.016 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:51:37.016 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:02.809005+0000", 2023-09-23T20:51:37.016 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0010698350000000001 2023-09-23T20:51:37.016 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.016 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.016 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:51:37.016 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:02.809020+0000", 2023-09-23T20:51:37.016 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.5116999999999999e-05 2023-09-23T20:51:37.016 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.016 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.016 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:51:37.017 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:03.785010+0000", 2023-09-23T20:51:37.017 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.97599018100000001 2023-09-23T20:51:37.017 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.017 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.017 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:51:37.017 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:03.785131+0000", 2023-09-23T20:51:37.017 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000120684 2023-09-23T20:51:37.017 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.017 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.017 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 0,4,7", 2023-09-23T20:51:37.017 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:03.785257+0000", 2023-09-23T20:51:37.017 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000126148 2023-09-23T20:51:37.018 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.018 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.018 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:51:37.018 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:03.789810+0000", 2023-09-23T20:51:37.018 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0045534390000000003 2023-09-23T20:51:37.018 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.018 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.018 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:37.018 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:03.794338+0000", 2023-09-23T20:51:37.018 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0045280169999999996 2023-09-23T20:51:37.018 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.018 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.019 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:37.019 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:03.794384+0000", 2023-09-23T20:51:37.019 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.5728e-05 2023-09-23T20:51:37.019 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.019 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.019 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:37.019 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:03.804122+0000", 2023-09-23T20:51:37.019 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0097375610000000005 2023-09-23T20:51:37.019 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.019 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.019 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:51:37.019 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:03.804191+0000", 2023-09-23T20:51:37.019 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.9214000000000005e-05 2023-09-23T20:51:37.020 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.020 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.020 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:51:37.020 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:03.804321+0000", 2023-09-23T20:51:37.020 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00013007200000000001 2023-09-23T20:51:37.020 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:51:37.020 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:51:37.020 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:51:37.020 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.020 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.020 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:181730 10.2 10:42ef278f:::benchmark_data_smithi018_422431_object77234:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e604)", 2023-09-23T20:51:37.020 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:49:02.812497+0000", 2023-09-23T20:51:37.021 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 154.14797542299999, 2023-09-23T20:51:37.021 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.99226569600000003, 2023-09-23T20:51:37.021 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:51:37.021 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:37.021 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:51:37.021 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:51:37.021 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:51:37.021 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 181730 2023-09-23T20:51:37.021 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.021 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:51:37.021 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.021 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:51:37.022 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:02.812497+0000", 2023-09-23T20:51:37.022 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:51:37.022 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.022 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.022 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:51:37.022 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:02.812497+0000", 2023-09-23T20:51:37.022 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:51:37.022 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.022 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.022 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:51:37.022 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:02.812496+0000", 2023-09-23T20:51:37.022 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:51:37.023 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.023 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.023 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:51:37.023 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:02.812502+0000", 2023-09-23T20:51:37.023 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.6969999999999998e-06 2023-09-23T20:51:37.023 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.023 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.023 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:51:37.023 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:02.812518+0000", 2023-09-23T20:51:37.023 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.645e-05 2023-09-23T20:51:37.023 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.023 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.023 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:51:37.024 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:02.812527+0000", 2023-09-23T20:51:37.024 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.7690000000000002e-06 2023-09-23T20:51:37.024 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.024 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.024 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:51:37.024 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:02.812633+0000", 2023-09-23T20:51:37.024 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000105853 2023-09-23T20:51:37.024 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.024 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.024 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:51:37.024 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:02.812647+0000", 2023-09-23T20:51:37.025 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.4372999999999999e-05 2023-09-23T20:51:37.025 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.025 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.025 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:51:37.025 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:03.787606+0000", 2023-09-23T20:51:37.025 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.97495889599999996 2023-09-23T20:51:37.025 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.025 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.025 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:51:37.025 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:03.787729+0000", 2023-09-23T20:51:37.025 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00012241899999999999 2023-09-23T20:51:37.025 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.026 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.026 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 0,4,7", 2023-09-23T20:51:37.026 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:03.787845+0000", 2023-09-23T20:51:37.026 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000116592 2023-09-23T20:51:37.026 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.026 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.026 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:51:37.026 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:03.792452+0000", 2023-09-23T20:51:37.026 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.004606995 2023-09-23T20:51:37.026 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.026 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.027 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:37.027 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:03.794420+0000", 2023-09-23T20:51:37.027 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0019673210000000002 2023-09-23T20:51:37.027 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.027 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.027 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:37.027 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:03.804366+0000", 2023-09-23T20:51:37.027 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0099460399999999997 2023-09-23T20:51:37.027 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.027 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.027 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:37.027 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:03.804592+0000", 2023-09-23T20:51:37.027 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00022602000000000001 2023-09-23T20:51:37.028 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.028 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.028 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:51:37.028 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:03.804647+0000", 2023-09-23T20:51:37.028 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.5288000000000003e-05 2023-09-23T20:51:37.028 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.028 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.028 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:51:37.028 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:03.804763+0000", 2023-09-23T20:51:37.028 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00011617599999999999 2023-09-23T20:51:37.028 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:51:37.029 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:51:37.029 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:51:37.029 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.029 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.029 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:181755 10.2 10:405e79b9:::benchmark_data_smithi018_422431_object77259:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e604)", 2023-09-23T20:51:37.029 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:49:02.840211+0000", 2023-09-23T20:51:37.029 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 154.12026215500001, 2023-09-23T20:51:37.029 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.96475393799999998, 2023-09-23T20:51:37.029 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:51:37.029 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:37.029 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:51:37.029 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:51:37.029 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:51:37.030 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 181755 2023-09-23T20:51:37.030 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.030 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:51:37.030 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.030 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:51:37.030 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:02.840211+0000", 2023-09-23T20:51:37.030 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:51:37.030 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.030 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.030 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:51:37.030 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:02.840211+0000", 2023-09-23T20:51:37.030 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:51:37.031 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.031 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.031 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:51:37.031 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:02.840209+0000", 2023-09-23T20:51:37.031 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:51:37.031 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.031 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.031 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:51:37.031 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:02.840218+0000", 2023-09-23T20:51:37.031 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.3579999999999992e-06 2023-09-23T20:51:37.031 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.031 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.032 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:51:37.032 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:02.840241+0000", 2023-09-23T20:51:37.032 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.2612e-05 2023-09-23T20:51:37.032 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.032 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.032 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:51:37.032 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:02.840250+0000", 2023-09-23T20:51:37.032 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.8379999999999997e-06 2023-09-23T20:51:37.032 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.032 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.032 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:51:37.032 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:02.840367+0000", 2023-09-23T20:51:37.033 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000116304 2023-09-23T20:51:37.033 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.033 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.033 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:51:37.033 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:02.840380+0000", 2023-09-23T20:51:37.033 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.3517e-05 2023-09-23T20:51:37.033 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.033 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.033 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:51:37.033 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:03.789948+0000", 2023-09-23T20:51:37.033 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.94956776799999998 2023-09-23T20:51:37.033 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.034 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.034 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:51:37.034 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:03.790094+0000", 2023-09-23T20:51:37.034 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00014550399999999999 2023-09-23T20:51:37.034 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.034 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.034 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 0,4,7", 2023-09-23T20:51:37.034 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:03.790249+0000", 2023-09-23T20:51:37.034 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00015586899999999999 2023-09-23T20:51:37.034 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.034 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.035 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:51:37.035 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:03.792503+0000", 2023-09-23T20:51:37.035 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0022535839999999999 2023-09-23T20:51:37.035 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.035 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.035 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:37.035 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:03.794539+0000", 2023-09-23T20:51:37.035 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0020361519999999998 2023-09-23T20:51:37.035 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.035 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.035 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:37.035 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:03.804556+0000", 2023-09-23T20:51:37.036 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.010016729 2023-09-23T20:51:37.036 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.036 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.036 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:37.036 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:03.804812+0000", 2023-09-23T20:51:37.036 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000256479 2023-09-23T20:51:37.036 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.036 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.036 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:51:37.036 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:03.804864+0000", 2023-09-23T20:51:37.036 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.2125e-05 2023-09-23T20:51:37.036 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.036 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.037 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:51:37.037 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:03.804965+0000", 2023-09-23T20:51:37.037 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000100091 2023-09-23T20:51:37.037 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:51:37.037 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:51:37.037 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:51:37.037 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.037 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.037 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:188892 10.16 10:6b34c20f:::benchmark_data_smithi018_422431_object84396:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e611)", 2023-09-23T20:51:37.037 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:49:13.847828+0000", 2023-09-23T20:51:37.037 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 143.11264463699999, 2023-09-23T20:51:37.037 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.90492298900000001, 2023-09-23T20:51:37.038 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:51:37.038 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:37.038 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:51:37.038 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:51:37.038 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:51:37.038 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 188892 2023-09-23T20:51:37.038 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.038 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:51:37.038 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.038 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:51:37.038 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:13.847828+0000", 2023-09-23T20:51:37.038 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:51:37.039 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.039 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.039 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:51:37.039 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:13.847828+0000", 2023-09-23T20:51:37.039 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:51:37.039 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.039 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.039 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:51:37.039 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:13.847826+0000", 2023-09-23T20:51:37.039 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:51:37.039 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.039 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.040 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:51:37.040 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:13.847838+0000", 2023-09-23T20:51:37.040 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1559000000000001e-05 2023-09-23T20:51:37.040 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.040 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.040 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:51:37.040 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:13.847862+0000", 2023-09-23T20:51:37.040 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.3898000000000001e-05 2023-09-23T20:51:37.040 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.040 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.040 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:51:37.040 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:13.847870+0000", 2023-09-23T20:51:37.040 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.7150000000000004e-06 2023-09-23T20:51:37.041 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.041 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.041 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:51:37.041 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:13.847935+0000", 2023-09-23T20:51:37.041 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.4200000000000002e-05 2023-09-23T20:51:37.041 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.041 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.041 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:51:37.041 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:13.847947+0000", 2023-09-23T20:51:37.041 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2847000000000001e-05 2023-09-23T20:51:37.041 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.041 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.042 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:51:37.042 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:14.740972+0000", 2023-09-23T20:51:37.042 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.89302469699999998 2023-09-23T20:51:37.042 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.042 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.042 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:51:37.042 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:14.741059+0000", 2023-09-23T20:51:37.042 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.708e-05 2023-09-23T20:51:37.042 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.042 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.042 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 0,4", 2023-09-23T20:51:37.043 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:14.741330+0000", 2023-09-23T20:51:37.043 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00027128200000000002 2023-09-23T20:51:37.043 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.043 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.043 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:51:37.043 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:14.747248+0000", 2023-09-23T20:51:37.043 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0059171800000000002 2023-09-23T20:51:37.043 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.043 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.043 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:37.043 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:14.747402+0000", 2023-09-23T20:51:37.043 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00015389200000000001 2023-09-23T20:51:37.044 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.044 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.044 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:37.044 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:14.752655+0000", 2023-09-23T20:51:37.044 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0052536190000000002 2023-09-23T20:51:37.044 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.044 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.044 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:51:37.044 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:14.752706+0000", 2023-09-23T20:51:37.044 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.1087000000000001e-05 2023-09-23T20:51:37.044 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.044 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.045 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:51:37.045 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:14.752751+0000", 2023-09-23T20:51:37.045 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.4839999999999998e-05 2023-09-23T20:51:37.045 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:51:37.045 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:51:37.045 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:51:37.045 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.045 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.045 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:191302 10.13 10.2ff3e253 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e613)", 2023-09-23T20:51:37.045 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:49:16.976913+0000", 2023-09-23T20:51:37.045 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 139.98355930100001, 2023-09-23T20:51:37.045 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.92805713099999998, 2023-09-23T20:51:37.046 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:51:37.046 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:51:37.046 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:51:37.046 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:51:37.046 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:51:37.046 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 191302 2023-09-23T20:51:37.046 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.046 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:51:37.046 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.046 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:51:37.046 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:16.976913+0000", 2023-09-23T20:51:37.046 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:51:37.047 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.047 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.047 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:51:37.047 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:16.976913+0000", 2023-09-23T20:51:37.047 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:51:37.047 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.047 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.047 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:51:37.047 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:16.976911+0000", 2023-09-23T20:51:37.047 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999976 2023-09-23T20:51:37.047 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.047 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.048 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:51:37.048 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:16.976922+0000", 2023-09-23T20:51:37.048 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0530000000000001e-05 2023-09-23T20:51:37.048 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.048 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.048 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:51:37.048 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:16.976942+0000", 2023-09-23T20:51:37.048 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.0211e-05 2023-09-23T20:51:37.048 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.048 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.048 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:51:37.048 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:16.976954+0000", 2023-09-23T20:51:37.049 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.1506e-05 2023-09-23T20:51:37.049 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.049 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.049 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:51:37.049 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:16.977025+0000", 2023-09-23T20:51:37.049 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.1711999999999994e-05 2023-09-23T20:51:37.049 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.049 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.049 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:51:37.049 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:16.977037+0000", 2023-09-23T20:51:37.049 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.169e-05 2023-09-23T20:51:37.049 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.049 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.050 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:51:37.050 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:17.904958+0000", 2023-09-23T20:51:37.050 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.92792081699999995 2023-09-23T20:51:37.050 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.050 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.050 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:51:37.050 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:17.904971+0000", 2023-09-23T20:51:37.050 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2856e-05 2023-09-23T20:51:37.050 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:51:37.050 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:51:37.050 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:51:37.051 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.051 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.051 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:191305 10.13 10.5fafba73 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e613)", 2023-09-23T20:51:37.051 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:49:16.979082+0000", 2023-09-23T20:51:37.051 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 139.98139082500001, 2023-09-23T20:51:37.051 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.92591810900000004, 2023-09-23T20:51:37.051 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:51:37.051 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:51:37.051 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:51:37.051 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:51:37.051 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:51:37.051 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 191305 2023-09-23T20:51:37.051 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.052 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:51:37.052 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.052 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:51:37.052 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:16.979082+0000", 2023-09-23T20:51:37.052 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:51:37.052 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.052 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.052 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:51:37.052 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:16.979082+0000", 2023-09-23T20:51:37.052 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:51:37.052 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.052 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.053 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:51:37.053 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:16.979080+0000", 2023-09-23T20:51:37.053 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:51:37.053 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.053 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.053 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:51:37.053 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:16.979087+0000", 2023-09-23T20:51:37.053 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.1940000000000001e-06 2023-09-23T20:51:37.053 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.053 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.053 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:51:37.053 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:16.979102+0000", 2023-09-23T20:51:37.054 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.4596000000000001e-05 2023-09-23T20:51:37.054 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.054 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.054 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:51:37.054 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:16.979111+0000", 2023-09-23T20:51:37.054 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.2960000000000004e-06 2023-09-23T20:51:37.054 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.054 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.054 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:51:37.054 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:16.979174+0000", 2023-09-23T20:51:37.054 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.2466999999999997e-05 2023-09-23T20:51:37.055 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.055 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.055 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:51:37.055 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:16.979190+0000", 2023-09-23T20:51:37.055 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.6282000000000001e-05 2023-09-23T20:51:37.055 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.055 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.055 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:51:37.055 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:17.904991+0000", 2023-09-23T20:51:37.055 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.92580114999999996 2023-09-23T20:51:37.055 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.055 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.056 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:51:37.056 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:17.905000+0000", 2023-09-23T20:51:37.056 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.9930000000000001e-06 2023-09-23T20:51:37.056 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:51:37.056 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:51:37.056 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:51:37.056 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.056 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.056 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:191333 10.13 10.f3564173 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e613)", 2023-09-23T20:51:37.056 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:49:17.011596+0000", 2023-09-23T20:51:37.056 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 139.94887628699999, 2023-09-23T20:51:37.056 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.89343019499999998, 2023-09-23T20:51:37.057 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:51:37.057 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "reached pg", 2023-09-23T20:51:37.057 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:51:37.057 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:51:37.057 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:51:37.057 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 191333 2023-09-23T20:51:37.057 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.057 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:51:37.057 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.057 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:51:37.057 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:17.011596+0000", 2023-09-23T20:51:37.057 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:51:37.057 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.058 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.058 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:51:37.058 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:17.011596+0000", 2023-09-23T20:51:37.058 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:51:37.058 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.058 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.058 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:51:37.058 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:17.011595+0000", 2023-09-23T20:51:37.058 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:51:37.058 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.058 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.058 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:51:37.059 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:17.011602+0000", 2023-09-23T20:51:37.059 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.8229999999999997e-06 2023-09-23T20:51:37.059 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.059 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.059 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:51:37.059 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:17.011615+0000", 2023-09-23T20:51:37.059 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2955e-05 2023-09-23T20:51:37.059 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.059 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.059 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:51:37.059 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:17.011623+0000", 2023-09-23T20:51:37.060 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.721e-06 2023-09-23T20:51:37.060 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.060 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.060 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:51:37.060 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:17.011930+0000", 2023-09-23T20:51:37.060 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00030696500000000003 2023-09-23T20:51:37.060 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.060 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.060 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:51:37.060 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:17.011939+0000", 2023-09-23T20:51:37.060 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.1030000000000008e-06 2023-09-23T20:51:37.060 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.060 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.061 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:51:37.061 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:17.905019+0000", 2023-09-23T20:51:37.061 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.89307973600000001 2023-09-23T20:51:37.061 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.061 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.061 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:51:37.061 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:17.905027+0000", 2023-09-23T20:51:37.061 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.6279999999999996e-06 2023-09-23T20:51:37.061 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:51:37.061 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:51:37.061 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:51:37.061 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.062 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.062 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:206435 10.19 10:9978da35:::benchmark_data_smithi018_422431_object101939:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e634)", 2023-09-23T20:51:37.062 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:49:43.120891+0000", 2023-09-23T20:51:37.062 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 113.839581301, 2023-09-23T20:51:37.062 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0111183050000001, 2023-09-23T20:51:37.062 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:51:37.062 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:37.062 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:51:37.062 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:51:37.062 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:51:37.062 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 206435 2023-09-23T20:51:37.062 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.063 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:51:37.063 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.063 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:51:37.063 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:43.120891+0000", 2023-09-23T20:51:37.063 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:51:37.063 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.063 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.063 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:51:37.063 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:43.120891+0000", 2023-09-23T20:51:37.063 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:51:37.063 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.063 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.064 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:51:37.064 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:43.120890+0000", 2023-09-23T20:51:37.064 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:51:37.064 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.064 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.064 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:51:37.064 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:43.120898+0000", 2023-09-23T20:51:37.064 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.4930000000000001e-06 2023-09-23T20:51:37.064 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.064 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.064 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:51:37.064 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:43.120912+0000", 2023-09-23T20:51:37.065 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.4766000000000001e-05 2023-09-23T20:51:37.065 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.065 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.065 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:51:37.065 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:43.120922+0000", 2023-09-23T20:51:37.065 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.2639999999999997e-06 2023-09-23T20:51:37.065 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.065 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.065 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:51:37.065 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:43.120978+0000", 2023-09-23T20:51:37.065 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.6415000000000001e-05 2023-09-23T20:51:37.065 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.066 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.066 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:51:37.066 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:43.120988+0000", 2023-09-23T20:51:37.066 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.8940000000000005e-06 2023-09-23T20:51:37.066 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.066 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.066 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:51:37.066 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:44.123854+0000", 2023-09-23T20:51:37.066 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.0028666020000001 2023-09-23T20:51:37.066 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.066 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.066 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:51:37.067 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:44.123957+0000", 2023-09-23T20:51:37.067 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000102089 2023-09-23T20:51:37.067 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.067 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.067 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:51:37.067 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:44.124189+0000", 2023-09-23T20:51:37.067 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000232823 2023-09-23T20:51:37.067 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.067 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.067 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:37.067 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:44.129682+0000", 2023-09-23T20:51:37.067 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0054925410000000001 2023-09-23T20:51:37.068 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.068 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.068 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:37.068 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:44.131921+0000", 2023-09-23T20:51:37.068 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0022387739999999998 2023-09-23T20:51:37.068 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.068 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.068 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:51:37.068 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:44.131955+0000", 2023-09-23T20:51:37.068 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.4096e-05 2023-09-23T20:51:37.068 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.068 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.069 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:51:37.069 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:44.131983+0000", 2023-09-23T20:51:37.069 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.8390000000000001e-05 2023-09-23T20:51:37.069 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.069 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.069 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:51:37.069 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:44.132010+0000", 2023-09-23T20:51:37.069 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.6579000000000001e-05 2023-09-23T20:51:37.069 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:51:37.069 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:51:37.069 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:51:37.069 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.069 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.070 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:206500 10.19 10:9de0cf19:::benchmark_data_smithi018_422431_object102004:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e634)", 2023-09-23T20:51:37.070 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:49:43.181444+0000", 2023-09-23T20:51:37.070 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 113.779029087, 2023-09-23T20:51:37.070 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.96146742799999996, 2023-09-23T20:51:37.070 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:51:37.070 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:37.070 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:51:37.070 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:51:37.070 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:51:37.070 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 206500 2023-09-23T20:51:37.070 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.071 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:51:37.071 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.071 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:51:37.071 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:43.181444+0000", 2023-09-23T20:51:37.071 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:51:37.071 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.071 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.071 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:51:37.071 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:43.181444+0000", 2023-09-23T20:51:37.071 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:51:37.071 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.071 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.072 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:51:37.072 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:43.181442+0000", 2023-09-23T20:51:37.072 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:51:37.072 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.072 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.072 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:51:37.072 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:43.181449+0000", 2023-09-23T20:51:37.072 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.9879999999999998e-06 2023-09-23T20:51:37.072 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.072 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.072 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:51:37.072 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:43.181468+0000", 2023-09-23T20:51:37.073 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.9381e-05 2023-09-23T20:51:37.073 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.073 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.073 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:51:37.073 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:43.181477+0000", 2023-09-23T20:51:37.073 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.1210000000000007e-06 2023-09-23T20:51:37.073 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.073 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.073 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:51:37.073 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:43.182520+0000", 2023-09-23T20:51:37.073 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.001043364 2023-09-23T20:51:37.073 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.073 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.074 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:51:37.074 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:43.182527+0000", 2023-09-23T20:51:37.074 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.8739999999999997e-06 2023-09-23T20:51:37.074 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.074 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.074 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:51:37.074 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:44.125761+0000", 2023-09-23T20:51:37.074 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.94323392900000003 2023-09-23T20:51:37.074 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.074 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.074 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:51:37.075 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:44.125876+0000", 2023-09-23T20:51:37.075 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000114979 2023-09-23T20:51:37.075 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.075 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.075 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:51:37.075 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:44.126000+0000", 2023-09-23T20:51:37.075 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000124332 2023-09-23T20:51:37.075 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.075 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.075 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:37.075 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:44.129707+0000", 2023-09-23T20:51:37.075 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.003706942 2023-09-23T20:51:37.076 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.076 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.076 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:51:37.076 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:44.133415+0000", 2023-09-23T20:51:37.076 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0037084230000000002 2023-09-23T20:51:37.076 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.076 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.076 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:37.076 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:44.142820+0000", 2023-09-23T20:51:37.076 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0094048949999999999 2023-09-23T20:51:37.076 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.076 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.077 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:51:37.077 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:44.142865+0000", 2023-09-23T20:51:37.077 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.4267000000000002e-05 2023-09-23T20:51:37.077 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.077 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.077 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:51:37.077 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:44.142911+0000", 2023-09-23T20:51:37.077 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.6483000000000002e-05 2023-09-23T20:51:37.077 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:51:37.077 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:51:37.077 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:51:37.077 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.078 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.078 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:206546 10.19 10:9e18b9c8:::benchmark_data_smithi018_422431_object102050:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e634)", 2023-09-23T20:51:37.078 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:49:43.223821+0000", 2023-09-23T20:51:37.078 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 113.736651749, 2023-09-23T20:51:37.078 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.91920754800000004, 2023-09-23T20:51:37.078 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:51:37.078 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:37.078 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:51:37.078 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:51:37.078 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:51:37.078 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 206546 2023-09-23T20:51:37.078 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.079 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:51:37.079 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.079 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:51:37.079 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:43.223821+0000", 2023-09-23T20:51:37.079 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:51:37.079 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.079 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.079 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:51:37.079 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:43.223821+0000", 2023-09-23T20:51:37.079 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:51:37.079 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.079 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.080 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:51:37.080 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:43.223819+0000", 2023-09-23T20:51:37.080 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:51:37.080 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.080 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.080 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:51:37.080 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:43.223829+0000", 2023-09-23T20:51:37.080 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.1519999999999998e-06 2023-09-23T20:51:37.080 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.080 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.080 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:51:37.080 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:43.223850+0000", 2023-09-23T20:51:37.080 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.1891000000000001e-05 2023-09-23T20:51:37.081 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.081 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.081 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:51:37.081 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:43.223860+0000", 2023-09-23T20:51:37.081 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.9040000000000004e-06 2023-09-23T20:51:37.081 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.081 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.081 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:51:37.081 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:43.223926+0000", 2023-09-23T20:51:37.081 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.5538999999999997e-05 2023-09-23T20:51:37.081 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.081 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.082 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:51:37.082 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:43.223938+0000", 2023-09-23T20:51:37.082 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.2019e-05 2023-09-23T20:51:37.082 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.082 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.082 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:51:37.082 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:44.127196+0000", 2023-09-23T20:51:37.082 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.90325776700000004 2023-09-23T20:51:37.082 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.082 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.082 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:51:37.082 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:44.127274+0000", 2023-09-23T20:51:37.083 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.8268000000000003e-05 2023-09-23T20:51:37.083 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.083 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.083 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:51:37.083 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:44.127453+0000", 2023-09-23T20:51:37.083 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00017932199999999999 2023-09-23T20:51:37.083 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.083 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.083 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:37.083 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:44.129740+0000", 2023-09-23T20:51:37.083 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0022867859999999999 2023-09-23T20:51:37.084 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.084 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.084 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:51:37.084 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:44.138130+0000", 2023-09-23T20:51:37.084 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0083900420000000003 2023-09-23T20:51:37.084 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.084 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.084 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:37.084 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:44.142951+0000", 2023-09-23T20:51:37.084 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0048208950000000004 2023-09-23T20:51:37.084 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.084 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.085 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:51:37.085 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:44.142991+0000", 2023-09-23T20:51:37.085 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.0302000000000001e-05 2023-09-23T20:51:37.085 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.085 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.085 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:51:37.085 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:44.143029+0000", 2023-09-23T20:51:37.085 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.7218000000000002e-05 2023-09-23T20:51:37.085 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:51:37.085 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:51:37.085 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:51:37.085 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.085 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.086 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:206564 10.19 10:9e9a8172:::benchmark_data_smithi018_422431_object102068:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e634)", 2023-09-23T20:51:37.086 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:49:43.238458+0000", 2023-09-23T20:51:37.086 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 113.72201487700001, 2023-09-23T20:51:37.086 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.90469091400000001, 2023-09-23T20:51:37.086 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:51:37.086 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:37.086 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:51:37.086 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:51:37.086 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:51:37.086 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 206564 2023-09-23T20:51:37.086 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.087 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:51:37.087 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.087 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:51:37.087 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:43.238458+0000", 2023-09-23T20:51:37.087 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:51:37.087 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.087 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.087 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:51:37.087 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:43.238458+0000", 2023-09-23T20:51:37.087 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:51:37.087 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.088 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.088 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:51:37.088 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:43.238456+0000", 2023-09-23T20:51:37.088 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999981 2023-09-23T20:51:37.088 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.088 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.088 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:51:37.088 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:43.238464+0000", 2023-09-23T20:51:37.088 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.6399999999999997e-06 2023-09-23T20:51:37.088 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.088 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.088 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:51:37.089 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:43.238480+0000", 2023-09-23T20:51:37.089 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.6557000000000001e-05 2023-09-23T20:51:37.089 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.089 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.089 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:51:37.089 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:43.238489+0000", 2023-09-23T20:51:37.089 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.9050000000000003e-06 2023-09-23T20:51:37.089 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.089 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.089 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:51:37.089 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:43.238553+0000", 2023-09-23T20:51:37.089 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.4148999999999999e-05 2023-09-23T20:51:37.089 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.090 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.090 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:51:37.090 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:43.238565+0000", 2023-09-23T20:51:37.090 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.153e-05 2023-09-23T20:51:37.090 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.090 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.090 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:51:37.090 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:44.128465+0000", 2023-09-23T20:51:37.090 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.88990053499999999 2023-09-23T20:51:37.090 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.090 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.091 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:51:37.091 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:44.128545+0000", 2023-09-23T20:51:37.091 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.9190999999999994e-05 2023-09-23T20:51:37.091 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.091 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.091 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:51:37.091 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:44.128696+0000", 2023-09-23T20:51:37.091 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00015161999999999999 2023-09-23T20:51:37.091 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.091 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.091 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:37.091 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:44.131891+0000", 2023-09-23T20:51:37.092 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0031945620000000002 2023-09-23T20:51:37.092 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.092 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.092 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:51:37.092 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:44.138137+0000", 2023-09-23T20:51:37.092 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0062459669999999998 2023-09-23T20:51:37.092 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.092 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.092 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:37.092 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:44.143075+0000", 2023-09-23T20:51:37.092 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0049384030000000001 2023-09-23T20:51:37.092 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.093 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.093 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:51:37.093 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:44.143114+0000", 2023-09-23T20:51:37.093 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.8538000000000003e-05 2023-09-23T20:51:37.093 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.093 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.093 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:51:37.093 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:44.143149+0000", 2023-09-23T20:51:37.093 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.5083000000000003e-05 2023-09-23T20:51:37.093 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:51:37.093 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:51:37.093 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:51:37.093 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.094 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.094 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:206566 10.19 10:9da35729:::benchmark_data_smithi018_422431_object102070:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e634)", 2023-09-23T20:51:37.094 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:49:43.239620+0000", 2023-09-23T20:51:37.094 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 113.720852824, 2023-09-23T20:51:37.094 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.90365303399999997, 2023-09-23T20:51:37.094 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:51:37.094 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:37.094 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:51:37.094 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:51:37.094 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:51:37.094 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 206566 2023-09-23T20:51:37.095 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.095 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:51:37.095 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.095 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:51:37.095 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:43.239620+0000", 2023-09-23T20:51:37.095 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:51:37.095 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.095 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.095 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:51:37.095 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:43.239620+0000", 2023-09-23T20:51:37.095 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:51:37.095 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.095 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.096 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:51:37.096 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:43.239619+0000", 2023-09-23T20:51:37.096 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.999999 2023-09-23T20:51:37.096 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.096 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.096 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:51:37.096 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:43.239625+0000", 2023-09-23T20:51:37.096 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.6139999999999997e-06 2023-09-23T20:51:37.096 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.096 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.096 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:51:37.096 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:43.239643+0000", 2023-09-23T20:51:37.097 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.7861999999999999e-05 2023-09-23T20:51:37.097 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.097 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.097 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:51:37.097 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:43.239652+0000", 2023-09-23T20:51:37.097 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.3040000000000005e-06 2023-09-23T20:51:37.097 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.097 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.097 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:51:37.097 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:43.239714+0000", 2023-09-23T20:51:37.097 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 6.2675000000000006e-05 2023-09-23T20:51:37.098 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.098 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.098 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:51:37.098 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:43.239723+0000", 2023-09-23T20:51:37.098 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.9989999999999997e-06 2023-09-23T20:51:37.098 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.098 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.098 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:51:37.098 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:44.129764+0000", 2023-09-23T20:51:37.098 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.89004099999999997 2023-09-23T20:51:37.098 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.099 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.099 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:51:37.099 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:44.129847+0000", 2023-09-23T20:51:37.099 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.2769999999999995e-05 2023-09-23T20:51:37.099 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.099 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.099 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:51:37.099 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:44.129969+0000", 2023-09-23T20:51:37.099 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.00012168 2023-09-23T20:51:37.099 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.099 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.099 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:37.100 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:44.131948+0000", 2023-09-23T20:51:37.100 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0019796969999999999 2023-09-23T20:51:37.100 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.100 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.100 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:51:37.100 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:44.138145+0000", 2023-09-23T20:51:37.100 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0061965409999999999 2023-09-23T20:51:37.100 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.100 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.100 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:37.100 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:44.143192+0000", 2023-09-23T20:51:37.100 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0050471429999999996 2023-09-23T20:51:37.100 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.101 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.101 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:51:37.101 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:44.143234+0000", 2023-09-23T20:51:37.101 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4.1622999999999998e-05 2023-09-23T20:51:37.101 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.101 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.101 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:51:37.101 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:44.143273+0000", 2023-09-23T20:51:37.101 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 3.9223e-05 2023-09-23T20:51:37.101 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:51:37.101 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:51:37.101 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:51:37.101 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.102 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.102 INFO:teuthology.orchestra.run.smithi018.stdout: "description": "osd_op(client.46625.0:206572 10.19 10:9fe3ea92:::benchmark_data_smithi018_422431_object102076:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e634)", 2023-09-23T20:51:37.102 INFO:teuthology.orchestra.run.smithi018.stdout: "initiated_at": "2023-09-23T20:49:43.243035+0000", 2023-09-23T20:51:37.102 INFO:teuthology.orchestra.run.smithi018.stdout: "age": 113.717437455, 2023-09-23T20:51:37.102 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.900309727, 2023-09-23T20:51:37.102 INFO:teuthology.orchestra.run.smithi018.stdout: "type_data": { 2023-09-23T20:51:37.102 INFO:teuthology.orchestra.run.smithi018.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T20:51:37.102 INFO:teuthology.orchestra.run.smithi018.stdout: "client_info": { 2023-09-23T20:51:37.102 INFO:teuthology.orchestra.run.smithi018.stdout: "client": "client.46625", 2023-09-23T20:51:37.102 INFO:teuthology.orchestra.run.smithi018.stdout: "client_addr": "172.21.15.18:0/3123066591", 2023-09-23T20:51:37.102 INFO:teuthology.orchestra.run.smithi018.stdout: "tid": 206572 2023-09-23T20:51:37.103 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.103 INFO:teuthology.orchestra.run.smithi018.stdout: "events": [ 2023-09-23T20:51:37.103 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.103 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "initiated", 2023-09-23T20:51:37.103 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:43.243035+0000", 2023-09-23T20:51:37.103 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:51:37.103 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.103 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.103 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "throttled", 2023-09-23T20:51:37.103 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:43.243035+0000", 2023-09-23T20:51:37.103 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0 2023-09-23T20:51:37.103 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.104 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.104 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "header_read", 2023-09-23T20:51:37.104 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:43.243034+0000", 2023-09-23T20:51:37.104 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 4294967295.9999986 2023-09-23T20:51:37.104 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.104 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.104 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "all_read", 2023-09-23T20:51:37.104 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:43.243042+0000", 2023-09-23T20:51:37.104 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 7.807e-06 2023-09-23T20:51:37.104 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.104 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.104 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "dispatched", 2023-09-23T20:51:37.104 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:43.243063+0000", 2023-09-23T20:51:37.105 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.1617e-05 2023-09-23T20:51:37.105 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.105 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.105 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "queued_for_pg", 2023-09-23T20:51:37.105 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:43.243073+0000", 2023-09-23T20:51:37.105 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 9.7799999999999995e-06 2023-09-23T20:51:37.105 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.105 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.105 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:51:37.105 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:43.243123+0000", 2023-09-23T20:51:37.105 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 5.0204000000000002e-05 2023-09-23T20:51:37.105 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.106 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.106 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for peered", 2023-09-23T20:51:37.106 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:43.243131+0000", 2023-09-23T20:51:37.106 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.1720000000000007e-06 2023-09-23T20:51:37.106 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.106 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.106 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "reached_pg", 2023-09-23T20:51:37.106 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:44.130818+0000", 2023-09-23T20:51:37.106 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.88768701299999997 2023-09-23T20:51:37.106 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.106 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.106 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "started", 2023-09-23T20:51:37.107 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:44.130903+0000", 2023-09-23T20:51:37.107 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 8.4565000000000001e-05 2023-09-23T20:51:37.107 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.107 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.107 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "waiting for subops from 3,4", 2023-09-23T20:51:37.107 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:44.131029+0000", 2023-09-23T20:51:37.107 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.000125723 2023-09-23T20:51:37.107 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.107 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.107 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:37.107 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:44.133459+0000", 2023-09-23T20:51:37.107 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0024303810000000001 2023-09-23T20:51:37.108 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.108 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.108 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "op_commit", 2023-09-23T20:51:37.108 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:44.138152+0000", 2023-09-23T20:51:37.108 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0046923920000000001 2023-09-23T20:51:37.108 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.108 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.108 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "sub_op_commit_rec", 2023-09-23T20:51:37.108 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:44.143305+0000", 2023-09-23T20:51:37.108 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 0.0051530919999999997 2023-09-23T20:51:37.108 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.108 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.108 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "commit_sent", 2023-09-23T20:51:37.109 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:44.143326+0000", 2023-09-23T20:51:37.109 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 2.1237999999999999e-05 2023-09-23T20:51:37.109 INFO:teuthology.orchestra.run.smithi018.stdout: }, 2023-09-23T20:51:37.109 INFO:teuthology.orchestra.run.smithi018.stdout: { 2023-09-23T20:51:37.109 INFO:teuthology.orchestra.run.smithi018.stdout: "event": "done", 2023-09-23T20:51:37.109 INFO:teuthology.orchestra.run.smithi018.stdout: "time": "2023-09-23T20:49:44.143345+0000", 2023-09-23T20:51:37.109 INFO:teuthology.orchestra.run.smithi018.stdout: "duration": 1.9154999999999999e-05 2023-09-23T20:51:37.109 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:51:37.109 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:51:37.109 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:51:37.109 INFO:teuthology.orchestra.run.smithi018.stdout: } 2023-09-23T20:51:37.109 INFO:teuthology.orchestra.run.smithi018.stdout: ] 2023-09-23T20:51:37.110 INFO:teuthology.orchestra.run.smithi018.stdout:} 2023-09-23T20:51:37.130 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:36 smithi018 ceph-mon[184144]: failed to encode map e680 with expected crc 2023-09-23T20:51:37.130 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:36 smithi018 ceph-mon[179647]: failed to encode map e680 with expected crc 2023-09-23T20:51:37.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:36 smithi149 ceph-mon[170394]: failed to encode map e680 with expected crc 2023-09-23T20:51:38.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:37 smithi018 ceph-mon[184144]: pgmap v1661: 103 pgs: 103 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-09-23T20:51:38.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:37 smithi018 ceph-mon[179647]: pgmap v1661: 103 pgs: 103 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-09-23T20:51:38.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:37 smithi149 ceph-mon[170394]: pgmap v1661: 103 pgs: 103 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-09-23T20:51:38.802 INFO:teuthology.orchestra.run.smithi018.stderr:set osd.2 primary-affinity to 1 (8655362) 2023-09-23T20:51:39.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:38 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3938977429' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T20:51:39.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:38 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T20:51:39.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:38 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3938977429' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T20:51:39.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:38 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T20:51:39.174 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd reweight 3 1 2023-09-23T20:51:39.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:38 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3938977429' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T20:51:39.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:38 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T20:51:39.789 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:51:40.048 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:39 smithi018 ceph-mon[184144]: pgmap v1662: 103 pgs: 103 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-09-23T20:51:40.048 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:39 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2023-09-23T20:51:40.048 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:39 smithi018 ceph-mon[184144]: osdmap e681: 8 total, 8 up, 8 in 2023-09-23T20:51:40.048 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:39 smithi018 ceph-mon[179647]: pgmap v1662: 103 pgs: 103 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-09-23T20:51:40.049 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:39 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2023-09-23T20:51:40.049 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:39 smithi018 ceph-mon[179647]: osdmap e681: 8 total, 8 up, 8 in 2023-09-23T20:51:40.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:39 smithi149 ceph-mon[170394]: pgmap v1662: 103 pgs: 103 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-09-23T20:51:40.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:39 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2023-09-23T20:51:40.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:39 smithi149 ceph-mon[170394]: osdmap e681: 8 total, 8 up, 8 in 2023-09-23T20:51:41.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:40 smithi018 ceph-mon[184144]: failed to encode map e681 with expected crc 2023-09-23T20:51:41.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:40 smithi018 ceph-mon[184144]: failed to encode map e681 with expected crc 2023-09-23T20:51:41.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:40 smithi018 ceph-mon[184144]: failed to encode map e681 with expected crc 2023-09-23T20:51:41.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:40 smithi018 ceph-mon[184144]: failed to encode map e681 with expected crc 2023-09-23T20:51:41.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:40 smithi018 ceph-mon[179647]: failed to encode map e681 with expected crc 2023-09-23T20:51:41.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:40 smithi018 ceph-mon[179647]: failed to encode map e681 with expected crc 2023-09-23T20:51:41.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:40 smithi018 ceph-mon[179647]: failed to encode map e681 with expected crc 2023-09-23T20:51:41.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:40 smithi018 ceph-mon[179647]: failed to encode map e681 with expected crc 2023-09-23T20:51:41.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:40 smithi149 ceph-mon[170394]: failed to encode map e681 with expected crc 2023-09-23T20:51:41.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:40 smithi149 ceph-mon[170394]: failed to encode map e681 with expected crc 2023-09-23T20:51:41.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:40 smithi149 ceph-mon[170394]: failed to encode map e681 with expected crc 2023-09-23T20:51:41.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:40 smithi149 ceph-mon[170394]: failed to encode map e681 with expected crc 2023-09-23T20:51:42.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:41 smithi018 ceph-mon[184144]: pgmap v1664: 103 pgs: 103 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-09-23T20:51:42.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:41 smithi018 ceph-mon[179647]: pgmap v1664: 103 pgs: 103 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-09-23T20:51:42.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:41 smithi149 ceph-mon[170394]: pgmap v1664: 103 pgs: 103 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-09-23T20:51:44.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:43 smithi018 ceph-mon[184144]: pgmap v1665: 103 pgs: 103 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-09-23T20:51:44.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:43 smithi018 ceph-mon[179647]: pgmap v1665: 103 pgs: 103 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-09-23T20:51:44.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:43 smithi149 ceph-mon[170394]: pgmap v1665: 103 pgs: 103 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-09-23T20:51:45.141 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:51:44 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:51:44] "GET /metrics HTTP/1.1" 200 34159 "" "Prometheus/2.43.0" 2023-09-23T20:51:45.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:44 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:51:45.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:44 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:51:45.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:44 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:51:45.822 INFO:teuthology.orchestra.run.smithi018.stderr:reweighted osd.3 to 1 (10000) 2023-09-23T20:51:46.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:45 smithi018 ceph-mon[184144]: pgmap v1666: 103 pgs: 103 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-09-23T20:51:46.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:45 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1305411306' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T20:51:46.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:45 smithi018 ceph-mon[179647]: pgmap v1666: 103 pgs: 103 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-09-23T20:51:46.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:45 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1305411306' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T20:51:46.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:45 smithi149 ceph-mon[170394]: pgmap v1666: 103 pgs: 103 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-09-23T20:51:46.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:45 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1305411306' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T20:51:46.282 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 3 1 2023-09-23T20:51:46.896 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:51:47.095 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:46 smithi018 ceph-mon[184144]: failed to encode map e682 with expected crc 2023-09-23T20:51:47.095 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:46 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1305411306' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T20:51:47.095 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:46 smithi018 ceph-mon[184144]: osdmap e682: 8 total, 8 up, 8 in 2023-09-23T20:51:47.096 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:46 smithi018 ceph-mon[179647]: failed to encode map e682 with expected crc 2023-09-23T20:51:47.096 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:46 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1305411306' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T20:51:47.096 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:46 smithi018 ceph-mon[179647]: osdmap e682: 8 total, 8 up, 8 in 2023-09-23T20:51:47.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:46 smithi149 ceph-mon[170394]: failed to encode map e682 with expected crc 2023-09-23T20:51:47.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:46 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1305411306' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T20:51:47.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:46 smithi149 ceph-mon[170394]: osdmap e682: 8 total, 8 up, 8 in 2023-09-23T20:51:48.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:47 smithi018 ceph-mon[179647]: pgmap v1668: 103 pgs: 103 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-09-23T20:51:48.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:47 smithi018 ceph-mon[184144]: pgmap v1668: 103 pgs: 103 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-09-23T20:51:48.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:47 smithi149 ceph-mon[170394]: pgmap v1668: 103 pgs: 103 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-09-23T20:51:48.833 INFO:teuthology.orchestra.run.smithi018.stderr:set osd.3 primary-affinity to 1 (8655362) 2023-09-23T20:51:49.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:48 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3219430579' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T20:51:49.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:48 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3219430579' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T20:51:49.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:48 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3219430579' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T20:51:49.251 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd reweight 4 1 2023-09-23T20:51:49.862 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:51:50.103 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:49 smithi018 ceph-mon[184144]: pgmap v1669: 103 pgs: 103 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-09-23T20:51:50.103 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:49 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3219430579' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2023-09-23T20:51:50.103 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:49 smithi018 ceph-mon[184144]: osdmap e683: 8 total, 8 up, 8 in 2023-09-23T20:51:50.103 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:49 smithi018 ceph-mon[179647]: pgmap v1669: 103 pgs: 103 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-09-23T20:51:50.103 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:49 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3219430579' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2023-09-23T20:51:50.104 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:49 smithi018 ceph-mon[179647]: osdmap e683: 8 total, 8 up, 8 in 2023-09-23T20:51:50.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:49 smithi149 ceph-mon[170394]: pgmap v1669: 103 pgs: 103 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-09-23T20:51:50.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:49 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3219430579' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2023-09-23T20:51:50.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:49 smithi149 ceph-mon[170394]: osdmap e683: 8 total, 8 up, 8 in 2023-09-23T20:51:51.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:50 smithi018 ceph-mon[184144]: failed to encode map e683 with expected crc 2023-09-23T20:51:51.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:50 smithi018 ceph-mon[184144]: failed to encode map e683 with expected crc 2023-09-23T20:51:51.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:50 smithi018 ceph-mon[184144]: failed to encode map e684 with expected crc 2023-09-23T20:51:51.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:50 smithi018 ceph-mon[184144]: osdmap e684: 8 total, 8 up, 8 in 2023-09-23T20:51:51.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:50 smithi018 ceph-mon[184144]: failed to encode map e684 with expected crc 2023-09-23T20:51:51.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:50 smithi018 ceph-mon[184144]: failed to encode map e684 with expected crc 2023-09-23T20:51:51.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:50 smithi018 ceph-mon[184144]: 1.19 scrub starts 2023-09-23T20:51:51.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:50 smithi018 ceph-mon[184144]: 1.19 scrub ok 2023-09-23T20:51:51.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:50 smithi018 ceph-mon[179647]: failed to encode map e683 with expected crc 2023-09-23T20:51:51.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:50 smithi018 ceph-mon[179647]: failed to encode map e683 with expected crc 2023-09-23T20:51:51.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:50 smithi018 ceph-mon[179647]: failed to encode map e684 with expected crc 2023-09-23T20:51:51.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:50 smithi018 ceph-mon[179647]: osdmap e684: 8 total, 8 up, 8 in 2023-09-23T20:51:51.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:50 smithi018 ceph-mon[179647]: failed to encode map e684 with expected crc 2023-09-23T20:51:51.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:50 smithi018 ceph-mon[179647]: failed to encode map e684 with expected crc 2023-09-23T20:51:51.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:50 smithi018 ceph-mon[179647]: 1.19 scrub starts 2023-09-23T20:51:51.143 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:50 smithi018 ceph-mon[179647]: 1.19 scrub ok 2023-09-23T20:51:51.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:50 smithi149 ceph-mon[170394]: failed to encode map e683 with expected crc 2023-09-23T20:51:51.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:50 smithi149 ceph-mon[170394]: failed to encode map e683 with expected crc 2023-09-23T20:51:51.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:50 smithi149 ceph-mon[170394]: failed to encode map e684 with expected crc 2023-09-23T20:51:51.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:50 smithi149 ceph-mon[170394]: osdmap e684: 8 total, 8 up, 8 in 2023-09-23T20:51:51.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:50 smithi149 ceph-mon[170394]: failed to encode map e684 with expected crc 2023-09-23T20:51:51.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:50 smithi149 ceph-mon[170394]: failed to encode map e684 with expected crc 2023-09-23T20:51:51.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:50 smithi149 ceph-mon[170394]: 1.19 scrub starts 2023-09-23T20:51:51.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:50 smithi149 ceph-mon[170394]: 1.19 scrub ok 2023-09-23T20:51:51.845 INFO:teuthology.orchestra.run.smithi018.stderr:reweighted osd.4 to 1 (10000) 2023-09-23T20:51:52.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:51 smithi018 ceph-mon[184144]: failed to encode map e684 with expected crc 2023-09-23T20:51:52.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:51 smithi018 ceph-mon[184144]: pgmap v1672: 103 pgs: 103 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-09-23T20:51:52.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:51 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2054641382' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T20:51:52.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:51 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T20:51:52.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:51 smithi018 ceph-mon[179647]: failed to encode map e684 with expected crc 2023-09-23T20:51:52.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:51 smithi018 ceph-mon[179647]: pgmap v1672: 103 pgs: 103 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-09-23T20:51:52.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:51 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2054641382' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T20:51:52.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:51 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T20:51:52.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:51 smithi149 ceph-mon[170394]: failed to encode map e684 with expected crc 2023-09-23T20:51:52.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:51 smithi149 ceph-mon[170394]: pgmap v1672: 103 pgs: 103 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-09-23T20:51:52.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:51 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2054641382' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T20:51:52.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:51 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T20:51:52.325 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 4 1 2023-09-23T20:51:52.937 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:51:53.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:52 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T20:51:53.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:52 smithi018 ceph-mon[184144]: osdmap e685: 8 total, 8 up, 8 in 2023-09-23T20:51:53.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:52 smithi018 ceph-mon[184144]: failed to encode map e685 with expected crc 2023-09-23T20:51:53.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:52 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T20:51:53.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:52 smithi018 ceph-mon[179647]: osdmap e685: 8 total, 8 up, 8 in 2023-09-23T20:51:53.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:52 smithi018 ceph-mon[179647]: failed to encode map e685 with expected crc 2023-09-23T20:51:53.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:52 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T20:51:53.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:52 smithi149 ceph-mon[170394]: osdmap e685: 8 total, 8 up, 8 in 2023-09-23T20:51:53.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:52 smithi149 ceph-mon[170394]: failed to encode map e685 with expected crc 2023-09-23T20:51:54.036 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:53 smithi018 ceph-mon[184144]: failed to encode map e685 with expected crc 2023-09-23T20:51:54.037 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:53 smithi018 ceph-mon[184144]: pgmap v1674: 103 pgs: 103 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-09-23T20:51:54.037 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:53 smithi018 ceph-mon[184144]: failed to encode map e685 with expected crc 2023-09-23T20:51:54.037 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:53 smithi018 ceph-mon[184144]: failed to encode map e685 with expected crc 2023-09-23T20:51:54.037 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:53 smithi018 ceph-mon[184144]: failed to encode map e685 with expected crc 2023-09-23T20:51:54.037 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:53 smithi018 ceph-mon[179647]: failed to encode map e685 with expected crc 2023-09-23T20:51:54.037 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:53 smithi018 ceph-mon[179647]: pgmap v1674: 103 pgs: 103 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-09-23T20:51:54.037 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:53 smithi018 ceph-mon[179647]: failed to encode map e685 with expected crc 2023-09-23T20:51:54.037 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:53 smithi018 ceph-mon[179647]: failed to encode map e685 with expected crc 2023-09-23T20:51:54.037 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:53 smithi018 ceph-mon[179647]: failed to encode map e685 with expected crc 2023-09-23T20:51:54.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:53 smithi149 ceph-mon[170394]: failed to encode map e685 with expected crc 2023-09-23T20:51:54.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:53 smithi149 ceph-mon[170394]: pgmap v1674: 103 pgs: 103 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-09-23T20:51:54.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:53 smithi149 ceph-mon[170394]: failed to encode map e685 with expected crc 2023-09-23T20:51:54.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:53 smithi149 ceph-mon[170394]: failed to encode map e685 with expected crc 2023-09-23T20:51:54.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:53 smithi149 ceph-mon[170394]: failed to encode map e685 with expected crc 2023-09-23T20:51:54.860 INFO:teuthology.orchestra.run.smithi018.stderr:set osd.4 primary-affinity to 1 (8655362) 2023-09-23T20:51:55.141 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:51:54 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:51:54] "GET /metrics HTTP/1.1" 200 34159 "" "Prometheus/2.43.0" 2023-09-23T20:51:55.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:54 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2138460355' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T20:51:55.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:54 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T20:51:55.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:54 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2138460355' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T20:51:55.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:54 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T20:51:55.203 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd reweight 5 1 2023-09-23T20:51:55.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:54 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2138460355' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T20:51:55.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:54 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T20:51:55.815 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:51:56.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:55 smithi018 ceph-mon[184144]: pgmap v1675: 103 pgs: 103 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-09-23T20:51:56.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:55 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2023-09-23T20:51:56.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:55 smithi018 ceph-mon[184144]: osdmap e686: 8 total, 8 up, 8 in 2023-09-23T20:51:56.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:55 smithi018 ceph-mon[184144]: failed to encode map e686 with expected crc 2023-09-23T20:51:56.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:55 smithi018 ceph-mon[179647]: pgmap v1675: 103 pgs: 103 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-09-23T20:51:56.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:55 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2023-09-23T20:51:56.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:55 smithi018 ceph-mon[179647]: osdmap e686: 8 total, 8 up, 8 in 2023-09-23T20:51:56.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:55 smithi018 ceph-mon[179647]: failed to encode map e686 with expected crc 2023-09-23T20:51:56.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:55 smithi149 ceph-mon[170394]: pgmap v1675: 103 pgs: 103 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-09-23T20:51:56.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:55 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2023-09-23T20:51:56.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:55 smithi149 ceph-mon[170394]: osdmap e686: 8 total, 8 up, 8 in 2023-09-23T20:51:56.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:55 smithi149 ceph-mon[170394]: failed to encode map e686 with expected crc 2023-09-23T20:51:57.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:56 smithi018 ceph-mon[179647]: failed to encode map e686 with expected crc 2023-09-23T20:51:57.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:56 smithi018 ceph-mon[184144]: failed to encode map e686 with expected crc 2023-09-23T20:51:57.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:56 smithi149 ceph-mon[170394]: failed to encode map e686 with expected crc 2023-09-23T20:51:57.868 INFO:teuthology.orchestra.run.smithi018.stderr:reweighted osd.5 to 1 (10000) 2023-09-23T20:51:58.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:57 smithi018 ceph-mon[184144]: pgmap v1677: 103 pgs: 103 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-09-23T20:51:58.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:57 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2627897107' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T20:51:58.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:57 smithi018 ceph-mon[179647]: pgmap v1677: 103 pgs: 103 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-09-23T20:51:58.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:57 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2627897107' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T20:51:58.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:57 smithi149 ceph-mon[170394]: pgmap v1677: 103 pgs: 103 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-09-23T20:51:58.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:57 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2627897107' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T20:51:58.384 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 5 1 2023-09-23T20:51:58.999 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:51:59.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:58 smithi018 ceph-mon[179647]: failed to encode map e687 with expected crc 2023-09-23T20:51:59.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:58 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2627897107' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T20:51:59.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:58 smithi018 ceph-mon[179647]: osdmap e687: 8 total, 8 up, 8 in 2023-09-23T20:51:59.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:58 smithi018 ceph-mon[184144]: failed to encode map e687 with expected crc 2023-09-23T20:51:59.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:58 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2627897107' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T20:51:59.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:58 smithi018 ceph-mon[184144]: osdmap e687: 8 total, 8 up, 8 in 2023-09-23T20:51:59.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:58 smithi149 ceph-mon[170394]: failed to encode map e687 with expected crc 2023-09-23T20:51:59.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:58 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2627897107' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T20:51:59.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:58 smithi149 ceph-mon[170394]: osdmap e687: 8 total, 8 up, 8 in 2023-09-23T20:52:00.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:59 smithi018 ceph-mon[179647]: pgmap v1679: 103 pgs: 103 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-09-23T20:52:00.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:59 smithi018 ceph-mon[179647]: failed to encode map e687 with expected crc 2023-09-23T20:52:00.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:51:59 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:52:00.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:59 smithi018 ceph-mon[184144]: pgmap v1679: 103 pgs: 103 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-09-23T20:52:00.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:59 smithi018 ceph-mon[184144]: failed to encode map e687 with expected crc 2023-09-23T20:52:00.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:51:59 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:52:00.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:59 smithi149 ceph-mon[170394]: pgmap v1679: 103 pgs: 103 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-09-23T20:52:00.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:59 smithi149 ceph-mon[170394]: failed to encode map e687 with expected crc 2023-09-23T20:52:00.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:51:59 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:52:00.890 INFO:teuthology.orchestra.run.smithi018.stderr:set osd.5 primary-affinity to 1 (8655362) 2023-09-23T20:52:01.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:52:00 smithi018 ceph-mon[184144]: failed to encode map e687 with expected crc 2023-09-23T20:52:01.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:52:00 smithi018 ceph-mon[184144]: failed to encode map e687 with expected crc 2023-09-23T20:52:01.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:52:00 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/4079380384' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T20:52:01.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:52:00 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T20:52:01.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:52:00 smithi018 ceph-mon[179647]: failed to encode map e687 with expected crc 2023-09-23T20:52:01.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:52:00 smithi018 ceph-mon[179647]: failed to encode map e687 with expected crc 2023-09-23T20:52:01.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:52:00 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/4079380384' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T20:52:01.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:52:00 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T20:52:01.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:52:00 smithi149 ceph-mon[170394]: failed to encode map e687 with expected crc 2023-09-23T20:52:01.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:52:00 smithi149 ceph-mon[170394]: failed to encode map e687 with expected crc 2023-09-23T20:52:01.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:52:00 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/4079380384' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T20:52:01.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:52:00 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T20:52:01.249 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd reweight 6 1 2023-09-23T20:52:01.859 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:52:02.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:52:01 smithi018 ceph-mon[184144]: pgmap v1680: 103 pgs: 103 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-09-23T20:52:02.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:52:01 smithi018 ceph-mon[184144]: failed to encode map e688 with expected crc 2023-09-23T20:52:02.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:52:01 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2023-09-23T20:52:02.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:52:01 smithi018 ceph-mon[184144]: osdmap e688: 8 total, 8 up, 8 in 2023-09-23T20:52:02.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:52:01 smithi018 ceph-mon[179647]: pgmap v1680: 103 pgs: 103 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-09-23T20:52:02.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:52:01 smithi018 ceph-mon[179647]: failed to encode map e688 with expected crc 2023-09-23T20:52:02.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:52:01 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2023-09-23T20:52:02.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:52:01 smithi018 ceph-mon[179647]: osdmap e688: 8 total, 8 up, 8 in 2023-09-23T20:52:02.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:52:01 smithi149 ceph-mon[170394]: pgmap v1680: 103 pgs: 103 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-09-23T20:52:02.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:52:01 smithi149 ceph-mon[170394]: failed to encode map e688 with expected crc 2023-09-23T20:52:02.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:52:01 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2023-09-23T20:52:02.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:52:01 smithi149 ceph-mon[170394]: osdmap e688: 8 total, 8 up, 8 in 2023-09-23T20:52:02.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:52:02 smithi018 ceph-mon[184144]: failed to encode map e689 with expected crc 2023-09-23T20:52:02.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:52:02 smithi018 ceph-mon[184144]: osdmap e689: 8 total, 8 up, 8 in 2023-09-23T20:52:02.892 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:52:02 smithi018 ceph-mon[184144]: failed to encode map e689 with expected crc 2023-09-23T20:52:02.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:52:02 smithi018 ceph-mon[179647]: failed to encode map e689 with expected crc 2023-09-23T20:52:02.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:52:02 smithi018 ceph-mon[179647]: osdmap e689: 8 total, 8 up, 8 in 2023-09-23T20:52:02.892 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:52:02 smithi018 ceph-mon[179647]: failed to encode map e689 with expected crc 2023-09-23T20:52:03.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:52:02 smithi149 ceph-mon[170394]: failed to encode map e689 with expected crc 2023-09-23T20:52:03.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:52:02 smithi149 ceph-mon[170394]: osdmap e689: 8 total, 8 up, 8 in 2023-09-23T20:52:03.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:52:02 smithi149 ceph-mon[170394]: failed to encode map e689 with expected crc 2023-09-23T20:52:03.895 INFO:teuthology.orchestra.run.smithi018.stderr:reweighted osd.6 to 1 (10000) 2023-09-23T20:52:04.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:52:03 smithi149 ceph-mon[170394]: pgmap v1683: 103 pgs: 103 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-09-23T20:52:04.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:52:03 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/284605384' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T20:52:04.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:52:03 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T20:52:04.237 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:52:03 smithi018 ceph-mon[184144]: pgmap v1683: 103 pgs: 103 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-09-23T20:52:04.237 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:52:03 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/284605384' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T20:52:04.237 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:52:03 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T20:52:04.238 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:52:03 smithi018 ceph-mon[179647]: pgmap v1683: 103 pgs: 103 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-09-23T20:52:04.238 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:52:03 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/284605384' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T20:52:04.238 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:52:03 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T20:52:04.324 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 6 1 2023-09-23T20:52:04.939 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:52:05.166 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:52:04 smithi018 ceph-mon[184144]: failed to encode map e690 with expected crc 2023-09-23T20:52:05.166 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:52:04 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T20:52:05.167 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:52:04 smithi018 ceph-mon[184144]: osdmap e690: 8 total, 8 up, 8 in 2023-09-23T20:52:05.167 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:52:04 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:52:04] "GET /metrics HTTP/1.1" 200 34158 "" "Prometheus/2.43.0" 2023-09-23T20:52:05.167 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:52:04 smithi018 ceph-mon[179647]: failed to encode map e690 with expected crc 2023-09-23T20:52:05.167 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:52:04 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T20:52:05.167 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:52:04 smithi018 ceph-mon[179647]: osdmap e690: 8 total, 8 up, 8 in 2023-09-23T20:52:05.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:52:04 smithi149 ceph-mon[170394]: failed to encode map e690 with expected crc 2023-09-23T20:52:05.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:52:04 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T20:52:05.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:52:04 smithi149 ceph-mon[170394]: osdmap e690: 8 total, 8 up, 8 in 2023-09-23T20:52:06.112 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:52:05 smithi018 ceph-mon[179647]: pgmap v1685: 103 pgs: 103 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-09-23T20:52:06.112 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:52:05 smithi018 ceph-mon[184144]: pgmap v1685: 103 pgs: 103 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-09-23T20:52:06.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:52:05 smithi149 ceph-mon[170394]: pgmap v1685: 103 pgs: 103 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-09-23T20:52:06.911 INFO:teuthology.orchestra.run.smithi018.stderr:set osd.6 primary-affinity to 1 (8655362) 2023-09-23T20:52:07.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:52:06 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2486449456' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T20:52:07.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:52:06 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T20:52:07.263 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:52:06 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2486449456' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T20:52:07.263 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:52:06 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T20:52:07.263 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:52:06 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2486449456' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T20:52:07.263 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:52:06 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T20:52:07.307 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd reweight 7 1 2023-09-23T20:52:07.922 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:52:08.155 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:52:07 smithi018 ceph-mon[179647]: pgmap v1686: 103 pgs: 103 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-09-23T20:52:08.156 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:52:07 smithi018 ceph-mon[179647]: failed to encode map e691 with expected crc 2023-09-23T20:52:08.156 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:52:07 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2023-09-23T20:52:08.156 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:52:07 smithi018 ceph-mon[179647]: osdmap e691: 8 total, 8 up, 8 in 2023-09-23T20:52:08.156 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:52:07 smithi018 ceph-mon[184144]: pgmap v1686: 103 pgs: 103 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-09-23T20:52:08.156 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:52:07 smithi018 ceph-mon[184144]: failed to encode map e691 with expected crc 2023-09-23T20:52:08.156 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:52:07 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2023-09-23T20:52:08.156 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:52:07 smithi018 ceph-mon[184144]: osdmap e691: 8 total, 8 up, 8 in 2023-09-23T20:52:08.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:52:07 smithi149 ceph-mon[170394]: pgmap v1686: 103 pgs: 103 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-09-23T20:52:08.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:52:07 smithi149 ceph-mon[170394]: failed to encode map e691 with expected crc 2023-09-23T20:52:08.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:52:07 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2023-09-23T20:52:08.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:52:07 smithi149 ceph-mon[170394]: osdmap e691: 8 total, 8 up, 8 in 2023-09-23T20:52:09.919 INFO:teuthology.orchestra.run.smithi018.stderr:reweighted osd.7 to 1 (10000) 2023-09-23T20:52:10.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:52:09 smithi149 ceph-mon[170394]: failed to encode map e691 with expected crc 2023-09-23T20:52:10.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:52:09 smithi149 ceph-mon[170394]: pgmap v1688: 103 pgs: 103 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-09-23T20:52:10.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:52:09 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3594885800' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T20:52:10.312 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:52:09 smithi018 ceph-mon[179647]: failed to encode map e691 with expected crc 2023-09-23T20:52:10.313 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:52:09 smithi018 ceph-mon[179647]: pgmap v1688: 103 pgs: 103 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-09-23T20:52:10.313 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:52:09 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3594885800' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T20:52:10.313 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:52:09 smithi018 ceph-mon[184144]: failed to encode map e691 with expected crc 2023-09-23T20:52:10.313 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:52:09 smithi018 ceph-mon[184144]: pgmap v1688: 103 pgs: 103 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-09-23T20:52:10.313 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:52:09 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3594885800' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T20:52:10.390 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 7 1 2023-09-23T20:52:11.004 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:52:11.200 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:52:10 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3594885800' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T20:52:11.200 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:52:10 smithi018 ceph-mon[184144]: osdmap e692: 8 total, 8 up, 8 in 2023-09-23T20:52:11.200 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:52:10 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3594885800' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T20:52:11.200 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:52:10 smithi018 ceph-mon[179647]: osdmap e692: 8 total, 8 up, 8 in 2023-09-23T20:52:11.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:52:10 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3594885800' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T20:52:11.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:52:10 smithi149 ceph-mon[170394]: osdmap e692: 8 total, 8 up, 8 in 2023-09-23T20:52:12.034 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:52:11 smithi018 ceph-mon[179647]: failed to encode map e692 with expected crc 2023-09-23T20:52:12.034 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:52:11 smithi018 ceph-mon[179647]: pgmap v1690: 103 pgs: 103 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-09-23T20:52:12.034 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:52:11 smithi018 ceph-mon[179647]: failed to encode map e692 with expected crc 2023-09-23T20:52:12.034 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:52:11 smithi018 ceph-mon[184144]: failed to encode map e692 with expected crc 2023-09-23T20:52:12.035 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:52:11 smithi018 ceph-mon[184144]: pgmap v1690: 103 pgs: 103 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-09-23T20:52:12.035 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:52:11 smithi018 ceph-mon[184144]: failed to encode map e692 with expected crc 2023-09-23T20:52:12.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:52:11 smithi149 ceph-mon[170394]: failed to encode map e692 with expected crc 2023-09-23T20:52:12.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:52:11 smithi149 ceph-mon[170394]: pgmap v1690: 103 pgs: 103 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-09-23T20:52:12.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:52:11 smithi149 ceph-mon[170394]: failed to encode map e692 with expected crc 2023-09-23T20:52:12.933 INFO:teuthology.orchestra.run.smithi018.stderr:set osd.7 primary-affinity to 1 (8655362) 2023-09-23T20:52:13.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:52:12 smithi149 ceph-mon[170394]: failed to encode map e692 with expected crc 2023-09-23T20:52:13.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:52:12 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/641874378' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T20:52:13.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:52:12 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T20:52:13.262 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:52:12 smithi018 ceph-mon[184144]: failed to encode map e692 with expected crc 2023-09-23T20:52:13.262 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:52:12 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/641874378' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T20:52:13.262 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:52:12 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T20:52:13.263 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:52:12 smithi018 ceph-mon[179647]: failed to encode map e692 with expected crc 2023-09-23T20:52:13.263 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:52:12 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/641874378' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T20:52:13.263 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:52:12 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T20:52:13.268 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd reweight 1 1 2023-09-23T20:52:13.882 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:52:14.185 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:52:13 smithi018 ceph-mon[184144]: pgmap v1691: 103 pgs: 103 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-09-23T20:52:14.185 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:52:13 smithi018 ceph-mon[184144]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2023-09-23T20:52:14.185 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:52:13 smithi018 ceph-mon[184144]: osdmap e693: 8 total, 8 up, 8 in 2023-09-23T20:52:14.185 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:52:13 smithi018 ceph-mon[184144]: failed to encode map e693 with expected crc 2023-09-23T20:52:14.185 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:52:13 smithi018 ceph-mon[179647]: pgmap v1691: 103 pgs: 103 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-09-23T20:52:14.185 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:52:13 smithi018 ceph-mon[179647]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2023-09-23T20:52:14.185 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:52:13 smithi018 ceph-mon[179647]: osdmap e693: 8 total, 8 up, 8 in 2023-09-23T20:52:14.185 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:52:13 smithi018 ceph-mon[179647]: failed to encode map e693 with expected crc 2023-09-23T20:52:14.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:52:13 smithi149 ceph-mon[170394]: pgmap v1691: 103 pgs: 103 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-09-23T20:52:14.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:52:13 smithi149 ceph-mon[170394]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2023-09-23T20:52:14.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:52:13 smithi149 ceph-mon[170394]: osdmap e693: 8 total, 8 up, 8 in 2023-09-23T20:52:14.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:52:13 smithi149 ceph-mon[170394]: failed to encode map e693 with expected crc 2023-09-23T20:52:15.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:52:14 smithi149 ceph-mon[170394]: failed to encode map e693 with expected crc 2023-09-23T20:52:15.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:52:14 smithi149 ceph-mon[170394]: failed to encode map e693 with expected crc 2023-09-23T20:52:15.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:52:14 smithi149 ceph-mon[170394]: failed to encode map e693 with expected crc 2023-09-23T20:52:15.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:52:14 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:52:15.391 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:52:14 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:52:14] "GET /metrics HTTP/1.1" 200 34158 "" "Prometheus/2.43.0" 2023-09-23T20:52:15.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:52:14 smithi018 ceph-mon[179647]: failed to encode map e693 with expected crc 2023-09-23T20:52:15.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:52:14 smithi018 ceph-mon[179647]: failed to encode map e693 with expected crc 2023-09-23T20:52:15.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:52:14 smithi018 ceph-mon[179647]: failed to encode map e693 with expected crc 2023-09-23T20:52:15.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:52:14 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:52:15.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:52:14 smithi018 ceph-mon[184144]: failed to encode map e693 with expected crc 2023-09-23T20:52:15.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:52:14 smithi018 ceph-mon[184144]: failed to encode map e693 with expected crc 2023-09-23T20:52:15.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:52:14 smithi018 ceph-mon[184144]: failed to encode map e693 with expected crc 2023-09-23T20:52:15.392 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:52:14 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:52:15.938 INFO:teuthology.orchestra.run.smithi018.stderr:reweighted osd.1 to 1 (10000) 2023-09-23T20:52:16.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:52:15 smithi149 ceph-mon[170394]: pgmap v1693: 103 pgs: 103 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-09-23T20:52:16.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:52:15 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:52:16.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:52:15 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/4003829565' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T20:52:16.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:52:15 smithi018 ceph-mon[184144]: pgmap v1693: 103 pgs: 103 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-09-23T20:52:16.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:52:15 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:52:16.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:52:15 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/4003829565' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T20:52:16.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:52:15 smithi018 ceph-mon[179647]: pgmap v1693: 103 pgs: 103 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-09-23T20:52:16.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:52:15 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T20:52:16.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:52:15 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/4003829565' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T20:52:17.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:52:16 smithi149 ceph-mon[170394]: failed to encode map e694 with expected crc 2023-09-23T20:52:17.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:52:16 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/4003829565' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T20:52:17.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:52:16 smithi149 ceph-mon[170394]: osdmap e694: 8 total, 8 up, 8 in 2023-09-23T20:52:17.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:52:16 smithi018 ceph-mon[184144]: failed to encode map e694 with expected crc 2023-09-23T20:52:17.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:52:16 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/4003829565' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T20:52:17.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:52:16 smithi018 ceph-mon[184144]: osdmap e694: 8 total, 8 up, 8 in 2023-09-23T20:52:17.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:52:16 smithi018 ceph-mon[179647]: failed to encode map e694 with expected crc 2023-09-23T20:52:17.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:52:16 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/4003829565' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T20:52:17.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:52:16 smithi018 ceph-mon[179647]: osdmap e694: 8 total, 8 up, 8 in 2023-09-23T20:52:17.553 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 1 1 2023-09-23T20:52:18.172 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:52:18.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:52:17 smithi149 ceph-mon[170394]: pgmap v1695: 103 pgs: 103 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-09-23T20:52:18.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:52:17 smithi018 ceph-mon[184144]: pgmap v1695: 103 pgs: 103 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-09-23T20:52:18.391 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:52:17 smithi018 ceph-mon[179647]: pgmap v1695: 103 pgs: 103 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-09-23T20:52:19.952 INFO:teuthology.orchestra.run.smithi018.stderr:set osd.1 primary-affinity to 1 (8655362) 2023-09-23T20:52:20.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:52:19 smithi149 ceph-mon[170394]: pgmap v1696: 103 pgs: 103 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-09-23T20:52:20.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:52:19 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3295545835' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T20:52:20.315 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:52:19 smithi018 ceph-mon[184144]: pgmap v1696: 103 pgs: 103 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-09-23T20:52:20.315 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:52:19 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3295545835' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T20:52:20.315 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:52:19 smithi018 ceph-mon[179647]: pgmap v1696: 103 pgs: 103 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-09-23T20:52:20.315 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:52:19 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3295545835' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T20:52:20.335 INFO:tasks.thrashosds.thrasher:joining the do_sighup greenlet 2023-09-23T20:52:20.335 INFO:tasks.thrashosds.thrasher:joining the do_optrack_toggle greenlet 2023-09-23T20:52:20.335 INFO:tasks.thrashosds.thrasher:joining the do_dump_ops greenlet 2023-09-23T20:52:20.335 INFO:tasks.thrashosds.thrasher:joining the do_noscrub_toggle greenlet 2023-09-23T20:52:20.335 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2023-09-23T20:52:20.336 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd dump --format=json 2023-09-23T20:52:21.053 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:52:21.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:52:20 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3295545835' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2023-09-23T20:52:21.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:52:20 smithi149 ceph-mon[170394]: osdmap e695: 8 total, 8 up, 8 in 2023-09-23T20:52:21.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:52:20 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:52:21.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:52:20 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:52:21.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:52:20 smithi149 ceph-mon[170394]: failed to encode map e695 with expected crc 2023-09-23T20:52:21.217 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:52:20 smithi149 ceph-mon[170394]: failed to encode map e695 with expected crc 2023-09-23T20:52:21.264 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:52:20 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3295545835' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2023-09-23T20:52:21.264 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:52:20 smithi018 ceph-mon[184144]: osdmap e695: 8 total, 8 up, 8 in 2023-09-23T20:52:21.264 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:52:20 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:52:21.264 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:52:20 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:52:21.265 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:52:20 smithi018 ceph-mon[184144]: failed to encode map e695 with expected crc 2023-09-23T20:52:21.265 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:52:20 smithi018 ceph-mon[184144]: failed to encode map e695 with expected crc 2023-09-23T20:52:21.265 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:52:20 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3295545835' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2023-09-23T20:52:21.265 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:52:20 smithi018 ceph-mon[179647]: osdmap e695: 8 total, 8 up, 8 in 2023-09-23T20:52:21.265 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:52:20 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:52:21.265 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:52:20 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:52:21.265 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:52:20 smithi018 ceph-mon[179647]: failed to encode map e695 with expected crc 2023-09-23T20:52:21.265 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:52:20 smithi018 ceph-mon[179647]: failed to encode map e695 with expected crc 2023-09-23T20:52:22.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:52:22 smithi018 ceph-mon[184144]: pgmap v1698: 103 pgs: 103 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-09-23T20:52:22.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:52:22 smithi018 ceph-mon[184144]: failed to encode map e695 with expected crc 2023-09-23T20:52:22.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:52:22 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:52:22.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:52:22 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:52:22.391 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:52:22 smithi018 ceph-mon[184144]: failed to encode map e695 with expected crc 2023-09-23T20:52:22.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:52:22 smithi018 ceph-mon[179647]: pgmap v1698: 103 pgs: 103 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-09-23T20:52:22.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:52:22 smithi018 ceph-mon[179647]: failed to encode map e695 with expected crc 2023-09-23T20:52:22.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:52:22 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:52:22.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:52:22 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:52:22.392 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:52:22 smithi018 ceph-mon[179647]: failed to encode map e695 with expected crc 2023-09-23T20:52:22.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:52:22 smithi149 ceph-mon[170394]: pgmap v1698: 103 pgs: 103 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-09-23T20:52:22.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:52:22 smithi149 ceph-mon[170394]: failed to encode map e695 with expected crc 2023-09-23T20:52:22.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:52:22 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:52:22.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:52:22 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:52:22.467 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:52:22 smithi149 ceph-mon[170394]: failed to encode map e695 with expected crc 2023-09-23T20:52:22.549 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:52:22.549 INFO:teuthology.orchestra.run.smithi018.stdout:{"epoch":695,"fsid":"a3b3b17e-5a4c-11ee-8db4-212e2dc638e7","created":"2023-09-23T20:06:54.359034+0000","modified":"2023-09-23T20:52:19.943882+0000","last_up_change":"2023-09-23T20:29:44.486758+0000","last_in_change":"2023-09-23T20:51:20.727554+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":23,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":11,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-09-23T20:09:47.162973+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":31,"pg_placement_num":31,"pg_placement_num_target":31,"pg_num_target":31,"pg_num_pending":31,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"426","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"332","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":2.059999942779541,"score_stable":2.059999942779541,"optimal_score":1,"raw_score_acting":2.059999942779541,"raw_score_stable":2.059999942779541,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-09-23T20:11:59.431837+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"662","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":27,"snap_epoch":662,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-09-23T20:12:42.254901+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-09-23T20:12:43.191578+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"69","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"44c8aaa3-bbd1-4330-8d34-73f7a08b0d4d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":90,"up_from":94,"up_thru":676,"down_at":91,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6802","nonce":990458588},{"type":"v1","addr":"172.21.15.18:6803","nonce":990458588}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6804","nonce":990458588},{"type":"v1","addr":"172.21.15.18:6805","nonce":990458588}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6808","nonce":990458588},{"type":"v1","addr":"172.21.15.18:6809","nonce":990458588}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6806","nonce":990458588},{"type":"v1","addr":"172.21.15.18:6807","nonce":990458588}]},"public_addr":"172.21.15.18:6803/990458588","cluster_addr":"172.21.15.18:6805/990458588","heartbeat_back_addr":"172.21.15.18:6809/990458588","heartbeat_front_addr":"172.21.15.18:6807/990458588","state":["exists","up"]},{"osd":1,"uuid":"c0534324-49c8-4804-bced-c4d13510e74a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":99,"last_clean_end":236,"up_from":266,"up_thru":676,"down_at":236,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6810","nonce":190056949},{"type":"v1","addr":"172.21.15.18:6811","nonce":190056949}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6812","nonce":190056949},{"type":"v1","addr":"172.21.15.18:6813","nonce":190056949}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6816","nonce":190056949},{"type":"v1","addr":"172.21.15.18:6817","nonce":190056949}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6814","nonce":190056949},{"type":"v1","addr":"172.21.15.18:6815","nonce":190056949}]},"public_addr":"172.21.15.18:6811/190056949","cluster_addr":"172.21.15.18:6813/190056949","heartbeat_back_addr":"172.21.15.18:6817/190056949","heartbeat_front_addr":"172.21.15.18:6815/190056949","state":["exists","up"]},{"osd":2,"uuid":"f3b2116d-a3d5-4279-91dc-eb47458d3239","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":100,"up_from":104,"up_thru":676,"down_at":101,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6818","nonce":2061051826},{"type":"v1","addr":"172.21.15.18:6819","nonce":2061051826}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6820","nonce":2061051826},{"type":"v1","addr":"172.21.15.18:6821","nonce":2061051826}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6824","nonce":2061051826},{"type":"v1","addr":"172.21.15.18:6825","nonce":2061051826}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6822","nonce":2061051826},{"type":"v1","addr":"172.21.15.18:6823","nonce":2061051826}]},"public_addr":"172.21.15.18:6819/2061051826","cluster_addr":"172.21.15.18:6821/2061051826","heartbeat_back_addr":"172.21.15.18:6825/2061051826","heartbeat_front_addr":"172.21.15.18:6823/2061051826","state":["exists","up"]},{"osd":3,"uuid":"8edccb7f-c4c9-47f3-a6f5-38a1a38cb099","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":105,"up_from":109,"up_thru":683,"down_at":106,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6826","nonce":2876439245},{"type":"v1","addr":"172.21.15.18:6827","nonce":2876439245}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6828","nonce":2876439245},{"type":"v1","addr":"172.21.15.18:6829","nonce":2876439245}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6832","nonce":2876439245},{"type":"v1","addr":"172.21.15.18:6833","nonce":2876439245}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6830","nonce":2876439245},{"type":"v1","addr":"172.21.15.18:6831","nonce":2876439245}]},"public_addr":"172.21.15.18:6827/2876439245","cluster_addr":"172.21.15.18:6829/2876439245","heartbeat_back_addr":"172.21.15.18:6833/2876439245","heartbeat_front_addr":"172.21.15.18:6831/2876439245","state":["exists","up"]},{"osd":4,"uuid":"1d5a64ce-fe42-4355-991a-dfb20cbe5423","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":676,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6800","nonce":3640432477},{"type":"v1","addr":"172.21.15.149:6801","nonce":3640432477}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6802","nonce":3640432477},{"type":"v1","addr":"172.21.15.149:6803","nonce":3640432477}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6806","nonce":3640432477},{"type":"v1","addr":"172.21.15.149:6807","nonce":3640432477}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6804","nonce":3640432477},{"type":"v1","addr":"172.21.15.149:6805","nonce":3640432477}]},"public_addr":"172.21.15.149:6801/3640432477","cluster_addr":"172.21.15.149:6803/3640432477","heartbeat_back_addr":"172.21.15.149:6807/3640432477","heartbeat_front_addr":"172.21.15.149:6805/3640432477","state":["exists","up"]},{"osd":5,"uuid":"105ae3b2-b2fc-477d-9134-3d517bc9427e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":688,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6808","nonce":2969709367},{"type":"v1","addr":"172.21.15.149:6809","nonce":2969709367}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6810","nonce":2969709367},{"type":"v1","addr":"172.21.15.149:6811","nonce":2969709367}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6814","nonce":2969709367},{"type":"v1","addr":"172.21.15.149:6815","nonce":2969709367}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6812","nonce":2969709367},{"type":"v1","addr":"172.21.15.149:6813","nonce":2969709367}]},"public_addr":"172.21.15.149:6809/2969709367","cluster_addr":"172.21.15.149:6811/2969709367","heartbeat_back_addr":"172.21.15.149:6815/2969709367","heartbeat_front_addr":"172.21.15.149:6813/2969709367","state":["exists","up"]},{"osd":6,"uuid":"27d2d2a8-e8c6-4a05-a85e-bd54376cf4c1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":676,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6816","nonce":1291453593},{"type":"v1","addr":"172.21.15.149:6817","nonce":1291453593}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6818","nonce":1291453593},{"type":"v1","addr":"172.21.15.149:6819","nonce":1291453593}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6822","nonce":1291453593},{"type":"v1","addr":"172.21.15.149:6823","nonce":1291453593}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6820","nonce":1291453593},{"type":"v1","addr":"172.21.15.149:6821","nonce":1291453593}]},"public_addr":"172.21.15.149:6817/1291453593","cluster_addr":"172.21.15.149:6819/1291453593","heartbeat_back_addr":"172.21.15.149:6823/1291453593","heartbeat_front_addr":"172.21.15.149:6821/1291453593","state":["exists","up"]},{"osd":7,"uuid":"9da636f7-995c-4f29-9264-70d09c6f66c6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":676,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6824","nonce":3117985178},{"type":"v1","addr":"172.21.15.149:6825","nonce":3117985178}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6826","nonce":3117985178},{"type":"v1","addr":"172.21.15.149:6827","nonce":3117985178}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6830","nonce":3117985178},{"type":"v1","addr":"172.21.15.149:6831","nonce":3117985178}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6828","nonce":3117985178},{"type":"v1","addr":"172.21.15.149:6829","nonce":3117985178}]},"public_addr":"172.21.15.149:6825/3117985178","cluster_addr":"172.21.15.149:6827/3117985178","heartbeat_back_addr":"172.21.15.149:6831/3117985178","heartbeat_front_addr":"172.21.15.149:6829/3117985178","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:09:01.034520+0000","dead_epoch":90},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:09:21.795238+0000","dead_epoch":236},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:09:42.987835+0000","dead_epoch":100},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:10:04.092458+0000","dead_epoch":105},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:10:23.309112+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:10:43.348231+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:11:03.128458+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:11:23.084228+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"3.1d","mappings":[{"from":4,"to":2}]},{"pgid":"4.a","mappings":[{"from":1,"to":2}]},{"pgid":"4.10","mappings":[{"from":1,"to":2}]},{"pgid":"4.1d","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.149:0/1653842667":"2023-09-24T20:15:19.879895+0000","172.21.15.18:0/1988472148":"2023-09-24T20:11:49.652762+0000","172.21.15.149:6834/1754402171":"2023-09-24T20:15:19.879895+0000","172.21.15.18:6801/2791912398":"2023-09-24T20:15:59.309691+0000","172.21.15.18:0/1082665289":"2023-09-24T20:07:48.635784+0000","172.21.15.18:6800/1930027051":"2023-09-24T20:07:23.919815+0000","172.21.15.18:0/2809002822":"2023-09-24T20:14:38.972322+0000","172.21.15.18:6801/2611527229":"2023-09-24T20:11:49.652762+0000","172.21.15.18:0/1119348530":"2023-09-24T20:15:59.309691+0000","172.21.15.149:0/2486656126":"2023-09-24T20:15:19.879895+0000","172.21.15.18:6800/2611527229":"2023-09-24T20:11:49.652762+0000","172.21.15.18:0/2543728509":"2023-09-24T20:14:38.972322+0000","172.21.15.18:0/3863388488":"2023-09-24T20:11:49.652762+0000","172.21.15.18:0/2549145961":"2023-09-24T20:15:59.309691+0000","172.21.15.18:6801/1930027051":"2023-09-24T20:07:23.919815+0000","172.21.15.18:6801/380813540":"2023-09-24T20:07:48.635784+0000","172.21.15.149:6835/1754402171":"2023-09-24T20:15:19.879895+0000","172.21.15.18:0/3831267686":"2023-09-24T20:07:48.635784+0000","172.21.15.149:0/3406344898":"2023-09-24T20:15:19.879895+0000","172.21.15.18:0/2993111051":"2023-09-24T20:07:23.919815+0000","172.21.15.18:0/3990640673":"2023-09-24T20:14:38.972322+0000","172.21.15.18:0/1698512393":"2023-09-24T20:11:49.652762+0000","172.21.15.149:0/894377242":"2023-09-24T20:15:19.879895+0000","172.21.15.18:0/3356195690":"2023-09-24T20:14:38.972322+0000","172.21.15.18:0/3334666861":"2023-09-24T20:15:59.309691+0000","172.21.15.149:0/502112376":"2023-09-24T20:15:19.879895+0000","172.21.15.18:0/69122222":"2023-09-24T20:07:23.919815+0000","172.21.15.149:0/1078964609":"2023-09-24T20:15:19.879895+0000","172.21.15.18:0/1645667747":"2023-09-24T20:07:48.635784+0000","172.21.15.18:6800/380813540":"2023-09-24T20:07:48.635784+0000","172.21.15.18:6800/115279545":"2023-09-24T20:14:38.972322+0000","172.21.15.18:0/3510203881":"2023-09-24T20:15:59.309691+0000","172.21.15.18:6801/115279545":"2023-09-24T20:14:38.972322+0000","172.21.15.18:0/3912151186":"2023-09-24T20:11:49.652762+0000","172.21.15.18:0/872601545":"2023-09-24T20:15:59.309691+0000","172.21.15.18:0/1505013437":"2023-09-24T20:15:59.309691+0000","172.21.15.18:6800/2791912398":"2023-09-24T20:15:59.309691+0000","172.21.15.18:0/950924496":"2023-09-24T20:07:23.919815+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":11,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-23T20:52:23.003 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2023-09-23T20:52:23.003 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd dump --format=json 2023-09-23T20:52:23.280 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:52:23 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:52:23.280 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:52:23 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:52:23.280 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:52:23 smithi018 ceph-mon[184144]: pgmap v1699: 103 pgs: 103 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-09-23T20:52:23.280 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:52:23 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1441266779' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T20:52:23.280 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:52:23 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:52:23.281 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:52:23 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:52:23.281 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:52:23 smithi018 ceph-mon[179647]: pgmap v1699: 103 pgs: 103 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-09-23T20:52:23.281 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:52:23 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1441266779' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T20:52:23.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:52:23 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:52:23.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:52:23 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:52:23.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:52:23 smithi149 ceph-mon[170394]: pgmap v1699: 103 pgs: 103 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-09-23T20:52:23.466 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:52:23 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1441266779' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T20:52:23.883 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:52:25.348 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:52:24 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:52:24] "GET /metrics HTTP/1.1" 200 34158 "" "Prometheus/2.43.0" 2023-09-23T20:52:25.641 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:52:25 smithi018 ceph-mon[179647]: pgmap v1700: 103 pgs: 103 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-09-23T20:52:25.642 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:52:25 smithi018 ceph-mon[184144]: pgmap v1700: 103 pgs: 103 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-09-23T20:52:25.716 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:52:25 smithi149 ceph-mon[170394]: pgmap v1700: 103 pgs: 103 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-09-23T20:52:25.766 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:52:25.766 INFO:teuthology.orchestra.run.smithi018.stdout:{"epoch":695,"fsid":"a3b3b17e-5a4c-11ee-8db4-212e2dc638e7","created":"2023-09-23T20:06:54.359034+0000","modified":"2023-09-23T20:52:19.943882+0000","last_up_change":"2023-09-23T20:29:44.486758+0000","last_in_change":"2023-09-23T20:51:20.727554+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":23,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":11,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-09-23T20:09:47.162973+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":31,"pg_placement_num":31,"pg_placement_num_target":31,"pg_num_target":31,"pg_num_pending":31,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"426","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"332","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":2.059999942779541,"score_stable":2.059999942779541,"optimal_score":1,"raw_score_acting":2.059999942779541,"raw_score_stable":2.059999942779541,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-09-23T20:11:59.431837+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"662","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":27,"snap_epoch":662,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-09-23T20:12:42.254901+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-09-23T20:12:43.191578+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"69","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"44c8aaa3-bbd1-4330-8d34-73f7a08b0d4d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":90,"up_from":94,"up_thru":676,"down_at":91,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6802","nonce":990458588},{"type":"v1","addr":"172.21.15.18:6803","nonce":990458588}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6804","nonce":990458588},{"type":"v1","addr":"172.21.15.18:6805","nonce":990458588}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6808","nonce":990458588},{"type":"v1","addr":"172.21.15.18:6809","nonce":990458588}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6806","nonce":990458588},{"type":"v1","addr":"172.21.15.18:6807","nonce":990458588}]},"public_addr":"172.21.15.18:6803/990458588","cluster_addr":"172.21.15.18:6805/990458588","heartbeat_back_addr":"172.21.15.18:6809/990458588","heartbeat_front_addr":"172.21.15.18:6807/990458588","state":["exists","up"]},{"osd":1,"uuid":"c0534324-49c8-4804-bced-c4d13510e74a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":99,"last_clean_end":236,"up_from":266,"up_thru":676,"down_at":236,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6810","nonce":190056949},{"type":"v1","addr":"172.21.15.18:6811","nonce":190056949}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6812","nonce":190056949},{"type":"v1","addr":"172.21.15.18:6813","nonce":190056949}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6816","nonce":190056949},{"type":"v1","addr":"172.21.15.18:6817","nonce":190056949}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6814","nonce":190056949},{"type":"v1","addr":"172.21.15.18:6815","nonce":190056949}]},"public_addr":"172.21.15.18:6811/190056949","cluster_addr":"172.21.15.18:6813/190056949","heartbeat_back_addr":"172.21.15.18:6817/190056949","heartbeat_front_addr":"172.21.15.18:6815/190056949","state":["exists","up"]},{"osd":2,"uuid":"f3b2116d-a3d5-4279-91dc-eb47458d3239","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":100,"up_from":104,"up_thru":676,"down_at":101,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6818","nonce":2061051826},{"type":"v1","addr":"172.21.15.18:6819","nonce":2061051826}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6820","nonce":2061051826},{"type":"v1","addr":"172.21.15.18:6821","nonce":2061051826}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6824","nonce":2061051826},{"type":"v1","addr":"172.21.15.18:6825","nonce":2061051826}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6822","nonce":2061051826},{"type":"v1","addr":"172.21.15.18:6823","nonce":2061051826}]},"public_addr":"172.21.15.18:6819/2061051826","cluster_addr":"172.21.15.18:6821/2061051826","heartbeat_back_addr":"172.21.15.18:6825/2061051826","heartbeat_front_addr":"172.21.15.18:6823/2061051826","state":["exists","up"]},{"osd":3,"uuid":"8edccb7f-c4c9-47f3-a6f5-38a1a38cb099","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":105,"up_from":109,"up_thru":683,"down_at":106,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6826","nonce":2876439245},{"type":"v1","addr":"172.21.15.18:6827","nonce":2876439245}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6828","nonce":2876439245},{"type":"v1","addr":"172.21.15.18:6829","nonce":2876439245}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6832","nonce":2876439245},{"type":"v1","addr":"172.21.15.18:6833","nonce":2876439245}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.18:6830","nonce":2876439245},{"type":"v1","addr":"172.21.15.18:6831","nonce":2876439245}]},"public_addr":"172.21.15.18:6827/2876439245","cluster_addr":"172.21.15.18:6829/2876439245","heartbeat_back_addr":"172.21.15.18:6833/2876439245","heartbeat_front_addr":"172.21.15.18:6831/2876439245","state":["exists","up"]},{"osd":4,"uuid":"1d5a64ce-fe42-4355-991a-dfb20cbe5423","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":676,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6800","nonce":3640432477},{"type":"v1","addr":"172.21.15.149:6801","nonce":3640432477}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6802","nonce":3640432477},{"type":"v1","addr":"172.21.15.149:6803","nonce":3640432477}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6806","nonce":3640432477},{"type":"v1","addr":"172.21.15.149:6807","nonce":3640432477}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6804","nonce":3640432477},{"type":"v1","addr":"172.21.15.149:6805","nonce":3640432477}]},"public_addr":"172.21.15.149:6801/3640432477","cluster_addr":"172.21.15.149:6803/3640432477","heartbeat_back_addr":"172.21.15.149:6807/3640432477","heartbeat_front_addr":"172.21.15.149:6805/3640432477","state":["exists","up"]},{"osd":5,"uuid":"105ae3b2-b2fc-477d-9134-3d517bc9427e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":688,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6808","nonce":2969709367},{"type":"v1","addr":"172.21.15.149:6809","nonce":2969709367}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6810","nonce":2969709367},{"type":"v1","addr":"172.21.15.149:6811","nonce":2969709367}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6814","nonce":2969709367},{"type":"v1","addr":"172.21.15.149:6815","nonce":2969709367}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6812","nonce":2969709367},{"type":"v1","addr":"172.21.15.149:6813","nonce":2969709367}]},"public_addr":"172.21.15.149:6809/2969709367","cluster_addr":"172.21.15.149:6811/2969709367","heartbeat_back_addr":"172.21.15.149:6815/2969709367","heartbeat_front_addr":"172.21.15.149:6813/2969709367","state":["exists","up"]},{"osd":6,"uuid":"27d2d2a8-e8c6-4a05-a85e-bd54376cf4c1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":676,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6816","nonce":1291453593},{"type":"v1","addr":"172.21.15.149:6817","nonce":1291453593}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6818","nonce":1291453593},{"type":"v1","addr":"172.21.15.149:6819","nonce":1291453593}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6822","nonce":1291453593},{"type":"v1","addr":"172.21.15.149:6823","nonce":1291453593}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6820","nonce":1291453593},{"type":"v1","addr":"172.21.15.149:6821","nonce":1291453593}]},"public_addr":"172.21.15.149:6817/1291453593","cluster_addr":"172.21.15.149:6819/1291453593","heartbeat_back_addr":"172.21.15.149:6823/1291453593","heartbeat_front_addr":"172.21.15.149:6821/1291453593","state":["exists","up"]},{"osd":7,"uuid":"9da636f7-995c-4f29-9264-70d09c6f66c6","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":676,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6824","nonce":3117985178},{"type":"v1","addr":"172.21.15.149:6825","nonce":3117985178}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6826","nonce":3117985178},{"type":"v1","addr":"172.21.15.149:6827","nonce":3117985178}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6830","nonce":3117985178},{"type":"v1","addr":"172.21.15.149:6831","nonce":3117985178}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.149:6828","nonce":3117985178},{"type":"v1","addr":"172.21.15.149:6829","nonce":3117985178}]},"public_addr":"172.21.15.149:6825/3117985178","cluster_addr":"172.21.15.149:6827/3117985178","heartbeat_back_addr":"172.21.15.149:6831/3117985178","heartbeat_front_addr":"172.21.15.149:6829/3117985178","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:09:01.034520+0000","dead_epoch":90},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:09:21.795238+0000","dead_epoch":236},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:09:42.987835+0000","dead_epoch":100},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:10:04.092458+0000","dead_epoch":105},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:10:23.309112+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:10:43.348231+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:11:03.128458+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T20:11:23.084228+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"3.1d","mappings":[{"from":4,"to":2}]},{"pgid":"4.a","mappings":[{"from":1,"to":2}]},{"pgid":"4.10","mappings":[{"from":1,"to":2}]},{"pgid":"4.1d","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.18:6801/2791912398":"2023-09-24T20:15:59.309691+0000","172.21.15.149:0/502112376":"2023-09-24T20:15:19.879895+0000","172.21.15.18:0/69122222":"2023-09-24T20:07:23.919815+0000","172.21.15.149:6835/1754402171":"2023-09-24T20:15:19.879895+0000","172.21.15.18:0/3831267686":"2023-09-24T20:07:48.635784+0000","172.21.15.149:0/3406344898":"2023-09-24T20:15:19.879895+0000","172.21.15.149:6834/1754402171":"2023-09-24T20:15:19.879895+0000","172.21.15.18:0/1988472148":"2023-09-24T20:11:49.652762+0000","172.21.15.18:0/1082665289":"2023-09-24T20:07:48.635784+0000","172.21.15.18:6800/1930027051":"2023-09-24T20:07:23.919815+0000","172.21.15.18:0/2809002822":"2023-09-24T20:14:38.972322+0000","172.21.15.18:6801/2611527229":"2023-09-24T20:11:49.652762+0000","172.21.15.149:0/2486656126":"2023-09-24T20:15:19.879895+0000","172.21.15.18:0/1119348530":"2023-09-24T20:15:59.309691+0000","172.21.15.18:6800/2611527229":"2023-09-24T20:11:49.652762+0000","172.21.15.18:0/2543728509":"2023-09-24T20:14:38.972322+0000","172.21.15.18:0/2993111051":"2023-09-24T20:07:23.919815+0000","172.21.15.18:0/3990640673":"2023-09-24T20:14:38.972322+0000","172.21.15.18:0/1698512393":"2023-09-24T20:11:49.652762+0000","172.21.15.149:0/894377242":"2023-09-24T20:15:19.879895+0000","172.21.15.18:0/3356195690":"2023-09-24T20:14:38.972322+0000","172.21.15.18:6801/1930027051":"2023-09-24T20:07:23.919815+0000","172.21.15.18:6801/380813540":"2023-09-24T20:07:48.635784+0000","172.21.15.18:0/3863388488":"2023-09-24T20:11:49.652762+0000","172.21.15.18:0/2549145961":"2023-09-24T20:15:59.309691+0000","172.21.15.149:0/1078964609":"2023-09-24T20:15:19.879895+0000","172.21.15.18:0/1645667747":"2023-09-24T20:07:48.635784+0000","172.21.15.18:6800/380813540":"2023-09-24T20:07:48.635784+0000","172.21.15.18:6800/115279545":"2023-09-24T20:14:38.972322+0000","172.21.15.18:0/3510203881":"2023-09-24T20:15:59.309691+0000","172.21.15.18:6801/115279545":"2023-09-24T20:14:38.972322+0000","172.21.15.18:0/950924496":"2023-09-24T20:07:23.919815+0000","172.21.15.149:0/1653842667":"2023-09-24T20:15:19.879895+0000","172.21.15.18:0/3334666861":"2023-09-24T20:15:59.309691+0000","172.21.15.18:0/3912151186":"2023-09-24T20:11:49.652762+0000","172.21.15.18:0/872601545":"2023-09-24T20:15:59.309691+0000","172.21.15.18:0/1505013437":"2023-09-24T20:15:59.309691+0000","172.21.15.18:6800/2791912398":"2023-09-24T20:15:59.309691+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":11,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-23T20:52:26.137 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph tell osd.0 flush_pg_stats 2023-09-23T20:52:26.137 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph tell osd.1 flush_pg_stats 2023-09-23T20:52:26.138 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph tell osd.2 flush_pg_stats 2023-09-23T20:52:26.138 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph tell osd.3 flush_pg_stats 2023-09-23T20:52:26.139 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph tell osd.4 flush_pg_stats 2023-09-23T20:52:26.140 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph tell osd.5 flush_pg_stats 2023-09-23T20:52:26.140 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph tell osd.6 flush_pg_stats 2023-09-23T20:52:26.140 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph tell osd.7 flush_pg_stats 2023-09-23T20:52:26.891 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:52:26 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1182001732' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T20:52:26.891 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:52:26 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1182001732' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T20:52:26.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:52:26 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1182001732' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T20:52:27.276 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:52:27.303 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:52:27.433 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:52:27.441 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:52:27.599 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:52:27.600 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:52:27.769 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:52:27.833 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:52:28.086 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:52:27 smithi018 ceph-mon[184144]: pgmap v1701: 103 pgs: 103 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-09-23T20:52:28.086 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:52:27 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:52:28.086 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:52:27 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:52:28.086 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:52:27 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:52:28.086 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:52:27 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:52:28.086 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:52:27 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:52:28.087 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:52:27 smithi018 ceph-mon[179647]: pgmap v1701: 103 pgs: 103 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-09-23T20:52:28.087 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:52:27 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:52:28.087 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:52:27 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:52:28.087 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:52:27 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:52:28.087 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:52:27 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:52:28.087 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:52:27 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:52:28.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:52:27 smithi149 ceph-mon[170394]: pgmap v1701: 103 pgs: 103 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-09-23T20:52:28.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:52:27 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:52:28.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:52:27 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:52:28.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:52:27 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T20:52:28.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:52:27 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T20:52:28.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:52:27 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' 2023-09-23T20:52:29.803 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:52:29 smithi018 ceph-mon[179647]: pgmap v1702: 103 pgs: 103 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-09-23T20:52:30.096 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:52:29 smithi018 ceph-mon[184144]: pgmap v1702: 103 pgs: 103 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-09-23T20:52:30.096 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:52:29 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:52:30.097 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:52:29 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:52:30.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:52:29 smithi149 ceph-mon[170394]: pgmap v1702: 103 pgs: 103 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-09-23T20:52:30.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:52:29 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:52:31.987 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:52:31 smithi018 ceph-mon[184144]: pgmap v1703: 103 pgs: 103 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-09-23T20:52:31.987 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:52:31 smithi018 ceph-mon[179647]: pgmap v1703: 103 pgs: 103 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-09-23T20:52:32.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:52:31 smithi149 ceph-mon[170394]: pgmap v1703: 103 pgs: 103 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-09-23T20:52:32.555 INFO:teuthology.orchestra.run.smithi018.stdout:403726926214 2023-09-23T20:52:32.556 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.0 2023-09-23T20:52:32.583 INFO:teuthology.orchestra.run.smithi018.stdout:236223201774 2023-09-23T20:52:32.583 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.7 2023-09-23T20:52:33.111 INFO:teuthology.orchestra.run.smithi018.stdout:128849019389 2023-09-23T20:52:33.111 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.4 2023-09-23T20:52:33.701 INFO:teuthology.orchestra.run.smithi018.stdout:446676599162 2023-09-23T20:52:33.702 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.2 2023-09-23T20:52:34.066 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:52:33 smithi018 ceph-mon[184144]: pgmap v1704: 103 pgs: 103 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-09-23T20:52:34.067 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:52:33 smithi018 ceph-mon[179647]: pgmap v1704: 103 pgs: 103 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-09-23T20:52:34.075 INFO:teuthology.orchestra.run.smithi018.stdout:468151435635 2023-09-23T20:52:34.075 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.3 2023-09-23T20:52:34.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:52:33 smithi149 ceph-mon[170394]: pgmap v1704: 103 pgs: 103 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-09-23T20:52:34.784 INFO:teuthology.orchestra.run.smithi018.stdout:167503725050 2023-09-23T20:52:34.784 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.5 2023-09-23T20:52:35.264 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:52:34 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:52:34] "GET /metrics HTTP/1.1" 200 34155 "" "Prometheus/2.43.0" 2023-09-23T20:52:35.288 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:52:35.289 INFO:teuthology.orchestra.run.smithi018.stdout:201863463413 2023-09-23T20:52:35.290 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.6 2023-09-23T20:52:35.298 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:52:35.311 INFO:teuthology.orchestra.run.smithi018.stdout:1142461301012 2023-09-23T20:52:35.311 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.1 2023-09-23T20:52:35.321 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:52:35.334 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:52:35.484 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:52:35.782 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:52:35.811 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:52:35 smithi018 ceph-mon[179647]: pgmap v1705: 103 pgs: 103 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-09-23T20:52:36.087 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:52:35 smithi018 ceph-mon[184144]: pgmap v1705: 103 pgs: 103 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-09-23T20:52:36.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:52:35 smithi149 ceph-mon[170394]: pgmap v1705: 103 pgs: 103 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-09-23T20:52:38.074 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:52:37 smithi018 ceph-mon[184144]: pgmap v1706: 103 pgs: 103 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-09-23T20:52:38.074 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:52:37 smithi018 ceph-mon[179647]: pgmap v1706: 103 pgs: 103 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-09-23T20:52:38.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:52:37 smithi149 ceph-mon[170394]: pgmap v1706: 103 pgs: 103 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-09-23T20:52:38.370 INFO:teuthology.orchestra.run.smithi018.stdout:403726926215 2023-09-23T20:52:38.819 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:52:38 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1628352073' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T20:52:39.080 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:52:38 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1628352073' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T20:52:39.094 INFO:tasks.cephadm.ceph_manager.ceph:need seq 403726926214 got 403726926215 for osd.0 2023-09-23T20:52:39.094 DEBUG:teuthology.parallel:result is None 2023-09-23T20:52:39.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:52:38 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1628352073' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T20:52:39.325 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:52:39.486 INFO:teuthology.orchestra.run.smithi018.stdout:468151435637 2023-09-23T20:52:39.892 INFO:teuthology.orchestra.run.smithi018.stdout:128849019390 2023-09-23T20:52:40.092 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:52:39 smithi018 ceph-mon[179647]: pgmap v1707: 103 pgs: 103 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-09-23T20:52:40.092 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:52:39 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2627127014' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T20:52:40.093 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:52:39 smithi018 ceph-mon[184144]: pgmap v1707: 103 pgs: 103 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-09-23T20:52:40.093 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:52:39 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2627127014' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T20:52:40.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:52:39 smithi149 ceph-mon[170394]: pgmap v1707: 103 pgs: 103 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-09-23T20:52:40.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:52:39 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2627127014' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T20:52:40.223 INFO:teuthology.orchestra.run.smithi018.stdout:236223201775 2023-09-23T20:52:40.356 INFO:teuthology.orchestra.run.smithi018.stdout:446676599163 2023-09-23T20:52:40.646 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849019389 got 128849019390 for osd.4 2023-09-23T20:52:40.646 DEBUG:teuthology.parallel:result is None 2023-09-23T20:52:40.915 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:52:40 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2577652464' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T20:52:40.915 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:52:40 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1649206202' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T20:52:40.915 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:52:40 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/3987170171' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T20:52:40.916 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:52:40 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2577652464' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T20:52:40.916 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:52:40 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1649206202' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T20:52:40.916 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:52:40 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/3987170171' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T20:52:40.926 INFO:tasks.cephadm.ceph_manager.ceph:need seq 468151435635 got 468151435637 for osd.3 2023-09-23T20:52:40.926 DEBUG:teuthology.parallel:result is None 2023-09-23T20:52:40.948 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:52:41.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:52:40 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2577652464' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T20:52:41.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:52:40 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1649206202' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T20:52:41.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:52:40 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/3987170171' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T20:52:41.300 INFO:tasks.cephadm.ceph_manager.ceph:need seq 236223201774 got 236223201775 for osd.7 2023-09-23T20:52:41.300 DEBUG:teuthology.parallel:result is None 2023-09-23T20:52:41.541 INFO:tasks.cephadm.ceph_manager.ceph:need seq 446676599162 got 446676599163 for osd.2 2023-09-23T20:52:41.541 DEBUG:teuthology.parallel:result is None 2023-09-23T20:52:41.611 INFO:teuthology.orchestra.run.smithi018.stdout:167503725051 2023-09-23T20:52:41.825 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:52:41 smithi018 ceph-mon[179647]: pgmap v1708: 103 pgs: 103 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-09-23T20:52:42.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:52:41 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1316605435' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T20:52:42.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:52:41 smithi018 ceph-mon[184144]: pgmap v1708: 103 pgs: 103 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-09-23T20:52:42.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:52:41 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1316605435' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T20:52:42.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:52:41 smithi149 ceph-mon[170394]: pgmap v1708: 103 pgs: 103 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-09-23T20:52:42.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:52:41 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1316605435' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T20:52:42.397 INFO:tasks.cephadm.ceph_manager.ceph:need seq 167503725050 got 167503725051 for osd.5 2023-09-23T20:52:42.398 DEBUG:teuthology.parallel:result is None 2023-09-23T20:52:43.058 INFO:teuthology.orchestra.run.smithi018.stdout:1142461301013 2023-09-23T20:52:43.500 INFO:tasks.cephadm.ceph_manager.ceph:need seq 1142461301012 got 1142461301013 for osd.1 2023-09-23T20:52:43.501 DEBUG:teuthology.parallel:result is None 2023-09-23T20:52:43.581 INFO:teuthology.orchestra.run.smithi018.stdout:201863463415 2023-09-23T20:52:43.954 INFO:tasks.cephadm.ceph_manager.ceph:need seq 201863463413 got 201863463415 for osd.6 2023-09-23T20:52:43.955 DEBUG:teuthology.parallel:result is None 2023-09-23T20:52:43.955 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-09-23T20:52:43.955 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:52:44.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:52:43 smithi018 ceph-mon[184144]: pgmap v1709: 103 pgs: 103 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-09-23T20:52:44.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:52:43 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/2180911135' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T20:52:44.142 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:52:43 smithi018 ceph-mon[184144]: from='client.? 172.21.15.18:0/1228105061' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T20:52:44.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:52:43 smithi018 ceph-mon[179647]: pgmap v1709: 103 pgs: 103 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-09-23T20:52:44.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:52:43 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/2180911135' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T20:52:44.142 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:52:43 smithi018 ceph-mon[179647]: from='client.? 172.21.15.18:0/1228105061' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T20:52:44.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:52:43 smithi149 ceph-mon[170394]: pgmap v1709: 103 pgs: 103 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-09-23T20:52:44.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:52:43 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/2180911135' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T20:52:44.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:52:43 smithi149 ceph-mon[170394]: from='client.? 172.21.15.18:0/1228105061' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T20:52:44.562 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:52:45.096 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:52:44 smithi018 ceph-mon[179647]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:52:45.097 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:52:44 smithi018 ceph-mon[184144]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:52:45.097 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:52:44 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: ::ffff:172.21.15.149 - - [23/Sep/2023:20:52:44] "GET /metrics HTTP/1.1" 200 34157 "" "Prometheus/2.43.0" 2023-09-23T20:52:45.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:52:44 smithi149 ceph-mon[170394]: from='mgr.34107 172.21.15.18:0/3823952175' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T20:52:45.954 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:52:45.956 INFO:teuthology.orchestra.run.smithi018.stderr:dumped all 2023-09-23T20:52:46.141 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:52:45 smithi018 ceph-mon[179647]: pgmap v1710: 103 pgs: 103 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-09-23T20:52:46.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:52:45 smithi018 ceph-mon[184144]: pgmap v1710: 103 pgs: 103 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-09-23T20:52:46.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:52:45 smithi149 ceph-mon[170394]: pgmap v1710: 103 pgs: 103 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-09-23T20:52:46.310 INFO:teuthology.orchestra.run.smithi018.stdout:{"pg_ready":true,"pg_map":{"version":1710,"stamp":"2023-09-23T20:52:44.404336+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":8984499,"num_objects":29,"num_object_clones":0,"num_object_copies":87,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":29,"num_whiteouts":0,"num_read":11981,"num_read_kb":26415,"num_write":1751,"num_write_kb":25419,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":283,"num_bytes_recovered":25707687,"num_keys_recovered":80,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6996,"ondisk_log_size":6996,"up":309,"acting":309,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":309,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":6171444,"kb_used_data":89396,"kb_used_omap":32,"kb_used_meta":6081951,"kb_avail":743789772,"statfs":{"total":767960285184,"available":761640726528,"internally_reserved":0,"allocated":91541504,"data_stored":71282882,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32773,"internal_metadata":6227918843},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003170"},"pg_stats":[{"pgid":"1.18","version":"90'161","reported_seq":759,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.109526+0000","last_change":"2023-09-23T20:52:01.895435+0000","last_active":"2023-09-23T20:52:16.109526+0000","last_peered":"2023-09-23T20:52:16.109526+0000","last_clean":"2023-09-23T20:52:16.109526+0000","last_became_active":"2023-09-23T20:52:01.894990+0000","last_became_peered":"2023-09-23T20:52:01.894990+0000","last_unstale":"2023-09-23T20:52:16.109526+0000","last_undegraded":"2023-09-23T20:52:16.109526+0000","last_fullsized":"2023-09-23T20:52:16.109526+0000","mapping_epoch":688,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":689,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:19.677209+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:19.677209+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:24:12.055669+0000","scrub_duration":0.033122453000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"69'1","reported_seq":171,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.108563+0000","last_change":"2023-09-23T20:48:48.148250+0000","last_active":"2023-09-23T20:52:16.108563+0000","last_peered":"2023-09-23T20:52:16.108563+0000","last_clean":"2023-09-23T20:52:16.108563+0000","last_became_active":"2023-09-23T20:48:48.144271+0000","last_became_peered":"2023-09-23T20:48:48.144271+0000","last_unstale":"2023-09-23T20:52:16.108563+0000","last_undegraded":"2023-09-23T20:52:16.108563+0000","last_fullsized":"2023-09-23T20:52:16.108563+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:59:36.502986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":539,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.108495+0000","last_change":"2023-09-23T20:49:02.703099+0000","last_active":"2023-09-23T20:52:16.108495+0000","last_peered":"2023-09-23T20:52:16.108495+0000","last_clean":"2023-09-23T20:52:16.108495+0000","last_became_active":"2023-09-23T20:49:02.702753+0000","last_became_peered":"2023-09-23T20:49:02.702753+0000","last_unstale":"2023-09-23T20:52:16.108495+0000","last_undegraded":"2023-09-23T20:52:16.108495+0000","last_fullsized":"2023-09-23T20:52:16.108495+0000","mapping_epoch":603,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":604,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:24:30.834235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"90'161","reported_seq":719,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.109420+0000","last_change":"2023-09-23T20:52:01.895802+0000","last_active":"2023-09-23T20:52:16.109420+0000","last_peered":"2023-09-23T20:52:16.109420+0000","last_clean":"2023-09-23T20:52:16.109420+0000","last_became_active":"2023-09-23T20:52:01.895428+0000","last_became_peered":"2023-09-23T20:52:01.895428+0000","last_unstale":"2023-09-23T20:52:16.109420+0000","last_undegraded":"2023-09-23T20:52:16.109420+0000","last_fullsized":"2023-09-23T20:52:16.109420+0000","mapping_epoch":688,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":689,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:47:33.523187+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:36:05.442288+0000","last_clean_scrub_stamp":"2023-09-23T20:47:33.523187+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:03:35.858397+0000","scrub_duration":0.013532127,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":686,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.108123+0000","last_change":"2023-09-23T20:47:09.173501+0000","last_active":"2023-09-23T20:52:16.108123+0000","last_peered":"2023-09-23T20:52:16.108123+0000","last_clean":"2023-09-23T20:52:16.108123+0000","last_became_active":"2023-09-23T20:47:09.173160+0000","last_became_peered":"2023-09-23T20:47:09.173160+0000","last_unstale":"2023-09-23T20:52:16.108123+0000","last_undegraded":"2023-09-23T20:52:16.108123+0000","last_fullsized":"2023-09-23T20:52:16.108123+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:44:36.132093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":656,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.108999+0000","last_change":"2023-09-23T20:43:46.521939+0000","last_active":"2023-09-23T20:52:16.108999+0000","last_peered":"2023-09-23T20:52:16.108999+0000","last_clean":"2023-09-23T20:52:16.108999+0000","last_became_active":"2023-09-23T20:43:46.521185+0000","last_became_peered":"2023-09-23T20:43:46.521185+0000","last_unstale":"2023-09-23T20:52:16.108999+0000","last_undegraded":"2023-09-23T20:52:16.108999+0000","last_fullsized":"2023-09-23T20:52:16.108999+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:59:25.045627+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"90'161","reported_seq":582,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.108413+0000","last_change":"2023-09-23T20:47:31.475001+0000","last_active":"2023-09-23T20:52:16.108413+0000","last_peered":"2023-09-23T20:52:16.108413+0000","last_clean":"2023-09-23T20:52:16.108413+0000","last_became_active":"2023-09-23T20:47:11.617942+0000","last_became_peered":"2023-09-23T20:47:11.617942+0000","last_unstale":"2023-09-23T20:52:16.108413+0000","last_undegraded":"2023-09-23T20:52:16.108413+0000","last_fullsized":"2023-09-23T20:52:16.108413+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:47:31.474914+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:47:31.474914+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:52:05.169219+0000","scrub_duration":0.0026262870000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":493,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:17.946487+0000","last_change":"2023-09-23T20:37:43.584414+0000","last_active":"2023-09-23T20:52:17.946487+0000","last_peered":"2023-09-23T20:52:17.946487+0000","last_clean":"2023-09-23T20:52:17.946487+0000","last_became_active":"2023-09-23T20:37:43.584116+0000","last_became_peered":"2023-09-23T20:37:43.584116+0000","last_unstale":"2023-09-23T20:52:17.946487+0000","last_undegraded":"2023-09-23T20:52:17.946487+0000","last_fullsized":"2023-09-23T20:52:17.946487+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:17:54.258172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":329,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.110547+0000","last_change":"2023-09-23T20:51:21.746193+0000","last_active":"2023-09-23T20:52:16.110547+0000","last_peered":"2023-09-23T20:52:16.110547+0000","last_clean":"2023-09-23T20:52:16.110547+0000","last_became_active":"2023-09-23T20:51:21.745936+0000","last_became_peered":"2023-09-23T20:51:21.745936+0000","last_unstale":"2023-09-23T20:52:16.110547+0000","last_undegraded":"2023-09-23T20:52:16.110547+0000","last_fullsized":"2023-09-23T20:52:16.110547+0000","mapping_epoch":673,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":674,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:06:49.466364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"90'161","reported_seq":947,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:15.936414+0000","last_change":"2023-09-23T20:51:24.744039+0000","last_active":"2023-09-23T20:52:15.936414+0000","last_peered":"2023-09-23T20:52:15.936414+0000","last_clean":"2023-09-23T20:52:15.936414+0000","last_became_active":"2023-09-23T20:51:24.743653+0000","last_became_peered":"2023-09-23T20:51:24.743653+0000","last_unstale":"2023-09-23T20:52:15.936414+0000","last_undegraded":"2023-09-23T20:52:15.936414+0000","last_fullsized":"2023-09-23T20:52:15.936414+0000","mapping_epoch":676,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":677,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:03.411025+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:03.411025+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:57:50.218399+0000","scrub_duration":0.001802922,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":706,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.109447+0000","last_change":"2023-09-23T20:51:21.746692+0000","last_active":"2023-09-23T20:52:16.109447+0000","last_peered":"2023-09-23T20:52:16.109447+0000","last_clean":"2023-09-23T20:52:16.109447+0000","last_became_active":"2023-09-23T20:51:21.745658+0000","last_became_peered":"2023-09-23T20:51:21.745658+0000","last_unstale":"2023-09-23T20:52:16.109447+0000","last_undegraded":"2023-09-23T20:52:16.109447+0000","last_fullsized":"2023-09-23T20:52:16.109447+0000","mapping_epoch":673,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":674,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:49:28.118695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":138,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.108266+0000","last_change":"2023-09-23T20:51:02.816162+0000","last_active":"2023-09-23T20:52:16.108266+0000","last_peered":"2023-09-23T20:52:16.108266+0000","last_clean":"2023-09-23T20:52:16.108266+0000","last_became_active":"2023-09-23T20:51:02.815754+0000","last_became_peered":"2023-09-23T20:51:02.815754+0000","last_unstale":"2023-09-23T20:52:16.108266+0000","last_undegraded":"2023-09-23T20:52:16.108266+0000","last_fullsized":"2023-09-23T20:52:16.108266+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:29:00.532296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"90'161","reported_seq":698,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.108921+0000","last_change":"2023-09-23T20:49:36.717905+0000","last_active":"2023-09-23T20:52:16.108921+0000","last_peered":"2023-09-23T20:52:16.108921+0000","last_clean":"2023-09-23T20:52:16.108921+0000","last_became_active":"2023-09-23T20:48:51.169314+0000","last_became_peered":"2023-09-23T20:48:51.169314+0000","last_unstale":"2023-09-23T20:52:16.108921+0000","last_undegraded":"2023-09-23T20:52:16.108921+0000","last_fullsized":"2023-09-23T20:52:16.108921+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:49:36.717737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:49:36.717737+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:00:49.213712+0000","scrub_duration":0.0049746089999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":540,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.108962+0000","last_change":"2023-09-23T20:52:01.897383+0000","last_active":"2023-09-23T20:52:16.108962+0000","last_peered":"2023-09-23T20:52:16.108962+0000","last_clean":"2023-09-23T20:52:16.108962+0000","last_became_active":"2023-09-23T20:52:01.897046+0000","last_became_peered":"2023-09-23T20:52:01.897046+0000","last_unstale":"2023-09-23T20:52:16.108962+0000","last_undegraded":"2023-09-23T20:52:16.108962+0000","last_fullsized":"2023-09-23T20:52:16.108962+0000","mapping_epoch":688,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":689,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:41:16.704651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":524,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.109710+0000","last_change":"2023-09-23T20:38:02.542034+0000","last_active":"2023-09-23T20:52:16.109710+0000","last_peered":"2023-09-23T20:52:16.109710+0000","last_clean":"2023-09-23T20:52:16.109710+0000","last_became_active":"2023-09-23T20:38:02.541575+0000","last_became_peered":"2023-09-23T20:38:02.541575+0000","last_unstale":"2023-09-23T20:52:16.109710+0000","last_undegraded":"2023-09-23T20:52:16.109710+0000","last_fullsized":"2023-09-23T20:52:16.109710+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:50:54.591529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"90'161","reported_seq":840,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.109105+0000","last_change":"2023-09-23T20:52:01.895097+0000","last_active":"2023-09-23T20:52:16.109105+0000","last_peered":"2023-09-23T20:52:16.109105+0000","last_clean":"2023-09-23T20:52:16.109105+0000","last_became_active":"2023-09-23T20:52:01.894615+0000","last_became_peered":"2023-09-23T20:52:01.894615+0000","last_unstale":"2023-09-23T20:52:16.109105+0000","last_undegraded":"2023-09-23T20:52:16.109105+0000","last_fullsized":"2023-09-23T20:52:16.109105+0000","mapping_epoch":688,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":689,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:14.669831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:14.669831+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:46:31.699442+0000","scrub_duration":0.0083441460000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":754,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:15.936605+0000","last_change":"2023-09-23T20:47:10.837423+0000","last_active":"2023-09-23T20:52:15.936605+0000","last_peered":"2023-09-23T20:52:15.936605+0000","last_clean":"2023-09-23T20:52:15.936605+0000","last_became_active":"2023-09-23T20:47:09.204324+0000","last_became_peered":"2023-09-23T20:47:09.204324+0000","last_unstale":"2023-09-23T20:52:15.936605+0000","last_undegraded":"2023-09-23T20:52:15.936605+0000","last_fullsized":"2023-09-23T20:52:15.936605+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":554,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:19.108914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":540,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":230,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.108081+0000","last_change":"2023-09-23T20:48:48.145175+0000","last_active":"2023-09-23T20:52:16.108081+0000","last_peered":"2023-09-23T20:52:16.108081+0000","last_clean":"2023-09-23T20:52:16.108081+0000","last_became_active":"2023-09-23T20:48:48.144841+0000","last_became_peered":"2023-09-23T20:48:48.144841+0000","last_unstale":"2023-09-23T20:52:16.108081+0000","last_undegraded":"2023-09-23T20:52:16.108081+0000","last_fullsized":"2023-09-23T20:52:16.108081+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:27:40.802917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"90'161","reported_seq":824,"reported_epoch":695,"state":"active+clean","last_fresh":"2023-09-23T20:52:19.947013+0000","last_change":"2023-09-23T20:36:02.548387+0000","last_active":"2023-09-23T20:52:19.947013+0000","last_peered":"2023-09-23T20:52:19.947013+0000","last_clean":"2023-09-23T20:52:19.947013+0000","last_became_active":"2023-09-23T20:35:26.644787+0000","last_became_peered":"2023-09-23T20:35:26.644787+0000","last_unstale":"2023-09-23T20:52:19.947013+0000","last_undegraded":"2023-09-23T20:52:19.947013+0000","last_fullsized":"2023-09-23T20:52:19.947013+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:02.548285+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:02.548285+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:37:15.216553+0000","scrub_duration":0.0015514459999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":161,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.109460+0000","last_change":"2023-09-23T20:47:09.340979+0000","last_active":"2023-09-23T20:52:16.109460+0000","last_peered":"2023-09-23T20:52:16.109460+0000","last_clean":"2023-09-23T20:52:16.109460+0000","last_became_active":"2023-09-23T20:47:09.231222+0000","last_became_peered":"2023-09-23T20:47:09.231222+0000","last_unstale":"2023-09-23T20:52:16.109460+0000","last_undegraded":"2023-09-23T20:52:16.109460+0000","last_fullsized":"2023-09-23T20:52:16.109460+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:07:50.620006+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":151,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.109708+0000","last_change":"2023-09-23T20:47:09.186315+0000","last_active":"2023-09-23T20:52:16.109708+0000","last_peered":"2023-09-23T20:52:16.109708+0000","last_clean":"2023-09-23T20:52:16.109708+0000","last_became_active":"2023-09-23T20:47:09.184580+0000","last_became_peered":"2023-09-23T20:47:09.184580+0000","last_unstale":"2023-09-23T20:52:16.109708+0000","last_undegraded":"2023-09-23T20:52:16.109708+0000","last_fullsized":"2023-09-23T20:52:16.109708+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:44:35.553948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"90'161","reported_seq":985,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.107971+0000","last_change":"2023-09-23T20:51:24.741699+0000","last_active":"2023-09-23T20:52:16.107971+0000","last_peered":"2023-09-23T20:52:16.107971+0000","last_clean":"2023-09-23T20:52:16.107971+0000","last_became_active":"2023-09-23T20:51:24.740931+0000","last_became_peered":"2023-09-23T20:51:24.740931+0000","last_unstale":"2023-09-23T20:52:16.107971+0000","last_undegraded":"2023-09-23T20:52:16.107971+0000","last_fullsized":"2023-09-23T20:52:16.107971+0000","mapping_epoch":676,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":677,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:43.578666+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:43.578666+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:39:14.933750+0000","scrub_duration":0.0021890379999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":785,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:15.936818+0000","last_change":"2023-09-23T20:48:48.193666+0000","last_active":"2023-09-23T20:52:15.936818+0000","last_peered":"2023-09-23T20:52:15.936818+0000","last_clean":"2023-09-23T20:52:15.936818+0000","last_became_active":"2023-09-23T20:48:48.152081+0000","last_became_peered":"2023-09-23T20:48:48.152081+0000","last_unstale":"2023-09-23T20:52:15.936818+0000","last_undegraded":"2023-09-23T20:52:15.936818+0000","last_fullsized":"2023-09-23T20:52:15.936818+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:00:19.508124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":272,"num_keys_recovered":80,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":453,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.110857+0000","last_change":"2023-09-23T20:49:48.119365+0000","last_active":"2023-09-23T20:52:16.110857+0000","last_peered":"2023-09-23T20:52:16.110857+0000","last_clean":"2023-09-23T20:52:16.110857+0000","last_became_active":"2023-09-23T20:49:48.119009+0000","last_became_peered":"2023-09-23T20:49:48.119009+0000","last_unstale":"2023-09-23T20:52:16.110857+0000","last_undegraded":"2023-09-23T20:52:16.110857+0000","last_fullsized":"2023-09-23T20:52:16.110857+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:02:26.878542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"90'161","reported_seq":997,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.109346+0000","last_change":"2023-09-23T20:52:01.895146+0000","last_active":"2023-09-23T20:52:16.109346+0000","last_peered":"2023-09-23T20:52:16.109346+0000","last_clean":"2023-09-23T20:52:16.109346+0000","last_became_active":"2023-09-23T20:52:01.894768+0000","last_became_peered":"2023-09-23T20:52:01.894768+0000","last_unstale":"2023-09-23T20:52:16.109346+0000","last_undegraded":"2023-09-23T20:52:16.109346+0000","last_fullsized":"2023-09-23T20:52:16.109346+0000","mapping_epoch":688,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":689,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:20.041690+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:20.041690+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:58:44.945091+0000","scrub_duration":0.0026180330000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":653,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.107972+0000","last_change":"2023-09-23T20:43:46.521294+0000","last_active":"2023-09-23T20:52:16.107972+0000","last_peered":"2023-09-23T20:52:16.107972+0000","last_clean":"2023-09-23T20:52:16.107972+0000","last_became_active":"2023-09-23T20:43:46.521001+0000","last_became_peered":"2023-09-23T20:43:46.521001+0000","last_unstale":"2023-09-23T20:52:16.107972+0000","last_undegraded":"2023-09-23T20:52:16.107972+0000","last_fullsized":"2023-09-23T20:52:16.107972+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:01:35.126716+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":481,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.108740+0000","last_change":"2023-09-23T20:52:01.899144+0000","last_active":"2023-09-23T20:52:16.108740+0000","last_peered":"2023-09-23T20:52:16.108740+0000","last_clean":"2023-09-23T20:52:16.108740+0000","last_became_active":"2023-09-23T20:52:01.898813+0000","last_became_peered":"2023-09-23T20:52:01.898813+0000","last_unstale":"2023-09-23T20:52:16.108740+0000","last_undegraded":"2023-09-23T20:52:16.108740+0000","last_fullsized":"2023-09-23T20:52:16.108740+0000","mapping_epoch":688,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":689,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:44:59.907884+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"90'161","reported_seq":1019,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.110363+0000","last_change":"2023-09-23T20:47:11.619371+0000","last_active":"2023-09-23T20:52:16.110363+0000","last_peered":"2023-09-23T20:52:16.110363+0000","last_clean":"2023-09-23T20:52:16.110363+0000","last_became_active":"2023-09-23T20:47:11.619006+0000","last_became_peered":"2023-09-23T20:47:11.619006+0000","last_unstale":"2023-09-23T20:52:16.110363+0000","last_undegraded":"2023-09-23T20:52:16.110363+0000","last_fullsized":"2023-09-23T20:52:16.110363+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:42.544308+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:42.544308+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:47:24.414807+0000","scrub_duration":0.0029958179999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":718,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:15.936269+0000","last_change":"2023-09-23T20:47:09.176613+0000","last_active":"2023-09-23T20:52:15.936269+0000","last_peered":"2023-09-23T20:52:15.936269+0000","last_clean":"2023-09-23T20:52:15.936269+0000","last_became_active":"2023-09-23T20:47:09.176187+0000","last_became_peered":"2023-09-23T20:47:09.176187+0000","last_unstale":"2023-09-23T20:52:15.936269+0000","last_undegraded":"2023-09-23T20:52:15.936269+0000","last_fullsized":"2023-09-23T20:52:15.936269+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:23:01.590989+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":662,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.109077+0000","last_change":"2023-09-23T20:43:46.521735+0000","last_active":"2023-09-23T20:52:16.109077+0000","last_peered":"2023-09-23T20:52:16.109077+0000","last_clean":"2023-09-23T20:52:16.109077+0000","last_became_active":"2023-09-23T20:43:46.520762+0000","last_became_peered":"2023-09-23T20:43:46.520762+0000","last_unstale":"2023-09-23T20:52:16.109077+0000","last_undegraded":"2023-09-23T20:52:16.109077+0000","last_fullsized":"2023-09-23T20:52:16.109077+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:26:48.841930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"90'161","reported_seq":798,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.109715+0000","last_change":"2023-09-23T20:52:01.896791+0000","last_active":"2023-09-23T20:52:16.109715+0000","last_peered":"2023-09-23T20:52:16.109715+0000","last_clean":"2023-09-23T20:52:16.109715+0000","last_became_active":"2023-09-23T20:52:01.896285+0000","last_became_peered":"2023-09-23T20:52:01.896285+0000","last_unstale":"2023-09-23T20:52:16.109715+0000","last_undegraded":"2023-09-23T20:52:16.109715+0000","last_fullsized":"2023-09-23T20:52:16.109715+0000","mapping_epoch":688,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":689,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:18.952693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:18.952693+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:48.278498+0000","scrub_duration":0.001699365,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":409,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.110153+0000","last_change":"2023-09-23T20:51:21.763704+0000","last_active":"2023-09-23T20:52:16.110153+0000","last_peered":"2023-09-23T20:52:16.110153+0000","last_clean":"2023-09-23T20:52:16.110153+0000","last_became_active":"2023-09-23T20:51:21.745455+0000","last_became_peered":"2023-09-23T20:51:21.745455+0000","last_unstale":"2023-09-23T20:52:16.110153+0000","last_undegraded":"2023-09-23T20:52:16.110153+0000","last_fullsized":"2023-09-23T20:52:16.110153+0000","mapping_epoch":673,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":674,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:11:47.478937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":493,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.109021+0000","last_change":"2023-09-23T20:37:43.571083+0000","last_active":"2023-09-23T20:52:16.109021+0000","last_peered":"2023-09-23T20:52:16.109021+0000","last_clean":"2023-09-23T20:52:16.109021+0000","last_became_active":"2023-09-23T20:37:43.570306+0000","last_became_peered":"2023-09-23T20:37:43.570306+0000","last_unstale":"2023-09-23T20:52:16.109021+0000","last_undegraded":"2023-09-23T20:52:16.109021+0000","last_fullsized":"2023-09-23T20:52:16.109021+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:42:02.396718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.d","version":"90'161","reported_seq":1027,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:15.936503+0000","last_change":"2023-09-23T20:47:11.619510+0000","last_active":"2023-09-23T20:52:15.936503+0000","last_peered":"2023-09-23T20:52:15.936503+0000","last_clean":"2023-09-23T20:52:15.936503+0000","last_became_active":"2023-09-23T20:47:11.619154+0000","last_became_peered":"2023-09-23T20:47:11.619154+0000","last_unstale":"2023-09-23T20:52:15.936503+0000","last_undegraded":"2023-09-23T20:52:15.936503+0000","last_fullsized":"2023-09-23T20:52:15.936503+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:34:59.496375+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:34:59.496375+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:40:52.712777+0000","scrub_duration":0.0017843189999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":726,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:15.936996+0000","last_change":"2023-09-23T20:47:09.207851+0000","last_active":"2023-09-23T20:52:15.936996+0000","last_peered":"2023-09-23T20:52:15.936996+0000","last_clean":"2023-09-23T20:52:15.936996+0000","last_became_active":"2023-09-23T20:47:09.175836+0000","last_became_peered":"2023-09-23T20:47:09.175836+0000","last_unstale":"2023-09-23T20:52:15.936996+0000","last_undegraded":"2023-09-23T20:52:15.936996+0000","last_fullsized":"2023-09-23T20:52:15.936996+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:43:13.091809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":518,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.109675+0000","last_change":"2023-09-23T20:52:01.896148+0000","last_active":"2023-09-23T20:52:16.109675+0000","last_peered":"2023-09-23T20:52:16.109675+0000","last_clean":"2023-09-23T20:52:16.109675+0000","last_became_active":"2023-09-23T20:52:01.895458+0000","last_became_peered":"2023-09-23T20:52:01.895458+0000","last_unstale":"2023-09-23T20:52:16.109675+0000","last_undegraded":"2023-09-23T20:52:16.109675+0000","last_fullsized":"2023-09-23T20:52:16.109675+0000","mapping_epoch":688,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":689,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:22:05.761374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"90'161","reported_seq":959,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.110927+0000","last_change":"2023-09-23T20:41:09.506481+0000","last_active":"2023-09-23T20:52:16.110927+0000","last_peered":"2023-09-23T20:52:16.110927+0000","last_clean":"2023-09-23T20:52:16.110927+0000","last_became_active":"2023-09-23T20:38:28.442229+0000","last_became_peered":"2023-09-23T20:38:28.442229+0000","last_unstale":"2023-09-23T20:52:16.110927+0000","last_undegraded":"2023-09-23T20:52:16.110927+0000","last_fullsized":"2023-09-23T20:52:16.110927+0000","mapping_epoch":411,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":412,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:41:09.506338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:41:09.506338+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:19:46.265011+0000","scrub_duration":0.01112002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":764,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:15.936703+0000","last_change":"2023-09-23T20:47:10.848436+0000","last_active":"2023-09-23T20:52:15.936703+0000","last_peered":"2023-09-23T20:52:15.936703+0000","last_clean":"2023-09-23T20:52:15.936703+0000","last_became_active":"2023-09-23T20:47:09.196769+0000","last_became_peered":"2023-09-23T20:47:09.196769+0000","last_unstale":"2023-09-23T20:52:15.936703+0000","last_undegraded":"2023-09-23T20:52:15.936703+0000","last_fullsized":"2023-09-23T20:52:15.936703+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":554,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:59:38.105009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":161,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.108924+0000","last_change":"2023-09-23T20:48:23.915817+0000","last_active":"2023-09-23T20:52:16.108924+0000","last_peered":"2023-09-23T20:52:16.108924+0000","last_clean":"2023-09-23T20:52:16.108924+0000","last_became_active":"2023-09-23T20:48:23.915306+0000","last_became_peered":"2023-09-23T20:48:23.915306+0000","last_unstale":"2023-09-23T20:52:16.108924+0000","last_undegraded":"2023-09-23T20:52:16.108924+0000","last_fullsized":"2023-09-23T20:52:16.108924+0000","mapping_epoch":577,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":578,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:55:45.691450+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"90'161","reported_seq":929,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.110755+0000","last_change":"2023-09-23T20:51:24.745859+0000","last_active":"2023-09-23T20:52:16.110755+0000","last_peered":"2023-09-23T20:52:16.110755+0000","last_clean":"2023-09-23T20:52:16.110755+0000","last_became_active":"2023-09-23T20:51:24.745454+0000","last_became_peered":"2023-09-23T20:51:24.745454+0000","last_unstale":"2023-09-23T20:52:16.110755+0000","last_undegraded":"2023-09-23T20:52:16.110755+0000","last_fullsized":"2023-09-23T20:52:16.110755+0000","mapping_epoch":676,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":677,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:46.217489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:46.217489+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:39:14.173319+0000","scrub_duration":0.001456429,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":751,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:15.936499+0000","last_change":"2023-09-23T20:51:21.757079+0000","last_active":"2023-09-23T20:52:15.936499+0000","last_peered":"2023-09-23T20:52:15.936499+0000","last_clean":"2023-09-23T20:52:15.936499+0000","last_became_active":"2023-09-23T20:51:21.747892+0000","last_became_peered":"2023-09-23T20:51:21.747892+0000","last_unstale":"2023-09-23T20:52:15.936499+0000","last_undegraded":"2023-09-23T20:52:15.936499+0000","last_fullsized":"2023-09-23T20:52:15.936499+0000","mapping_epoch":673,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":674,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:31:38.160456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":560,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":309,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.110682+0000","last_change":"2023-09-23T20:49:02.702301+0000","last_active":"2023-09-23T20:52:16.110682+0000","last_peered":"2023-09-23T20:52:16.110682+0000","last_clean":"2023-09-23T20:52:16.110682+0000","last_became_active":"2023-09-23T20:49:02.701986+0000","last_became_peered":"2023-09-23T20:49:02.701986+0000","last_unstale":"2023-09-23T20:52:16.110682+0000","last_undegraded":"2023-09-23T20:52:16.110682+0000","last_fullsized":"2023-09-23T20:52:16.110682+0000","mapping_epoch":603,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":604,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:20:01.603267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.e","version":"90'161","reported_seq":724,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.109557+0000","last_change":"2023-09-23T20:47:37.608426+0000","last_active":"2023-09-23T20:52:16.109557+0000","last_peered":"2023-09-23T20:52:16.109557+0000","last_clean":"2023-09-23T20:52:16.109557+0000","last_became_active":"2023-09-23T20:47:11.617187+0000","last_became_peered":"2023-09-23T20:47:11.617187+0000","last_unstale":"2023-09-23T20:52:16.109557+0000","last_undegraded":"2023-09-23T20:52:16.109557+0000","last_fullsized":"2023-09-23T20:52:16.109557+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:47:37.608309+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:47:37.608309+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:22:25.482297+0000","scrub_duration":0.011600924,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":664,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.108480+0000","last_change":"2023-09-23T20:52:01.897145+0000","last_active":"2023-09-23T20:52:16.108480+0000","last_peered":"2023-09-23T20:52:16.108480+0000","last_clean":"2023-09-23T20:52:16.108480+0000","last_became_active":"2023-09-23T20:52:01.896852+0000","last_became_peered":"2023-09-23T20:52:01.896852+0000","last_unstale":"2023-09-23T20:52:16.108480+0000","last_undegraded":"2023-09-23T20:52:16.108480+0000","last_fullsized":"2023-09-23T20:52:16.108480+0000","mapping_epoch":688,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":689,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:33:24.799922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":690,"reported_epoch":695,"state":"active+clean","last_fresh":"2023-09-23T20:52:21.228664+0000","last_change":"2023-09-23T20:47:09.197427+0000","last_active":"2023-09-23T20:52:21.228664+0000","last_peered":"2023-09-23T20:52:21.228664+0000","last_clean":"2023-09-23T20:52:21.228664+0000","last_became_active":"2023-09-23T20:47:09.196542+0000","last_became_peered":"2023-09-23T20:47:09.196542+0000","last_unstale":"2023-09-23T20:52:21.228664+0000","last_undegraded":"2023-09-23T20:52:21.228664+0000","last_fullsized":"2023-09-23T20:52:21.228664+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:28:38.926202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":1140,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:15.936162+0000","last_change":"2023-09-23T20:48:51.169474+0000","last_active":"2023-09-23T20:52:15.936162+0000","last_peered":"2023-09-23T20:52:15.936162+0000","last_clean":"2023-09-23T20:52:15.936162+0000","last_became_active":"2023-09-23T20:48:51.169076+0000","last_became_peered":"2023-09-23T20:48:51.169076+0000","last_unstale":"2023-09-23T20:52:15.936162+0000","last_undegraded":"2023-09-23T20:52:15.936162+0000","last_fullsized":"2023-09-23T20:52:15.936162+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:35:02.523203+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:35:02.523203+0000","last_clean_scrub_stamp":"2023-09-23T20:35:02.523203+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:25:53.741403+0000","scrub_duration":0.0085927989999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":545,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.108161+0000","last_change":"2023-09-23T20:47:09.240200+0000","last_active":"2023-09-23T20:52:16.108161+0000","last_peered":"2023-09-23T20:52:16.108161+0000","last_clean":"2023-09-23T20:52:16.108161+0000","last_became_active":"2023-09-23T20:47:09.195204+0000","last_became_peered":"2023-09-23T20:47:09.195204+0000","last_unstale":"2023-09-23T20:52:16.108161+0000","last_undegraded":"2023-09-23T20:52:16.108161+0000","last_fullsized":"2023-09-23T20:52:16.108161+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:03.328250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":151,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.109382+0000","last_change":"2023-09-23T20:47:09.231714+0000","last_active":"2023-09-23T20:52:16.109382+0000","last_peered":"2023-09-23T20:52:16.109382+0000","last_clean":"2023-09-23T20:52:16.109382+0000","last_became_active":"2023-09-23T20:47:09.230918+0000","last_became_peered":"2023-09-23T20:47:09.230918+0000","last_unstale":"2023-09-23T20:52:16.109382+0000","last_undegraded":"2023-09-23T20:52:16.109382+0000","last_fullsized":"2023-09-23T20:52:16.109382+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:24.583656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":600,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.109438+0000","last_change":"2023-09-23T20:52:01.896147+0000","last_active":"2023-09-23T20:52:16.109438+0000","last_peered":"2023-09-23T20:52:16.109438+0000","last_clean":"2023-09-23T20:52:16.109438+0000","last_became_active":"2023-09-23T20:52:01.895805+0000","last_became_peered":"2023-09-23T20:52:01.895805+0000","last_unstale":"2023-09-23T20:52:16.109438+0000","last_undegraded":"2023-09-23T20:52:16.109438+0000","last_fullsized":"2023-09-23T20:52:16.109438+0000","mapping_epoch":688,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":689,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:50:03.881826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:50:03.881826+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:24:36.911235+0000","scrub_duration":0.003245258,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":319,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.110746+0000","last_change":"2023-09-23T20:47:09.178919+0000","last_active":"2023-09-23T20:52:16.110746+0000","last_peered":"2023-09-23T20:52:16.110746+0000","last_clean":"2023-09-23T20:52:16.110746+0000","last_became_active":"2023-09-23T20:47:09.178607+0000","last_became_peered":"2023-09-23T20:47:09.178607+0000","last_unstale":"2023-09-23T20:52:16.110746+0000","last_undegraded":"2023-09-23T20:52:16.110746+0000","last_fullsized":"2023-09-23T20:52:16.110746+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:23:45.196935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":171,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.109241+0000","last_change":"2023-09-23T20:48:48.148596+0000","last_active":"2023-09-23T20:52:16.109241+0000","last_peered":"2023-09-23T20:52:16.109241+0000","last_clean":"2023-09-23T20:52:16.109241+0000","last_became_active":"2023-09-23T20:48:48.148211+0000","last_became_peered":"2023-09-23T20:48:48.148211+0000","last_unstale":"2023-09-23T20:52:16.109241+0000","last_undegraded":"2023-09-23T20:52:16.109241+0000","last_fullsized":"2023-09-23T20:52:16.109241+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:21:50.825822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"90'161","reported_seq":1036,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:18.719713+0000","last_change":"2023-09-23T20:51:24.741541+0000","last_active":"2023-09-23T20:52:18.719713+0000","last_peered":"2023-09-23T20:52:18.719713+0000","last_clean":"2023-09-23T20:52:18.719713+0000","last_became_active":"2023-09-23T20:51:24.741161+0000","last_became_peered":"2023-09-23T20:51:24.741161+0000","last_unstale":"2023-09-23T20:52:18.719713+0000","last_undegraded":"2023-09-23T20:52:18.719713+0000","last_fullsized":"2023-09-23T20:52:18.719713+0000","mapping_epoch":676,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":677,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:05.595042+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:05.595042+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:34:00.584917+0000","scrub_duration":0.007271675,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":171,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.108792+0000","last_change":"2023-09-23T20:48:48.153145+0000","last_active":"2023-09-23T20:52:16.108792+0000","last_peered":"2023-09-23T20:52:16.108792+0000","last_clean":"2023-09-23T20:52:16.108792+0000","last_became_active":"2023-09-23T20:48:48.152858+0000","last_became_peered":"2023-09-23T20:48:48.152858+0000","last_unstale":"2023-09-23T20:52:16.108792+0000","last_undegraded":"2023-09-23T20:52:16.108792+0000","last_fullsized":"2023-09-23T20:52:16.108792+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:23:13.454288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":151,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.109633+0000","last_change":"2023-09-23T20:47:09.186271+0000","last_active":"2023-09-23T20:52:16.109633+0000","last_peered":"2023-09-23T20:52:16.109633+0000","last_clean":"2023-09-23T20:52:16.109633+0000","last_became_active":"2023-09-23T20:47:09.184357+0000","last_became_peered":"2023-09-23T20:47:09.184357+0000","last_unstale":"2023-09-23T20:52:16.109633+0000","last_undegraded":"2023-09-23T20:52:16.109633+0000","last_fullsized":"2023-09-23T20:52:16.109633+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:48:26.339811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":604,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.109868+0000","last_change":"2023-09-23T20:52:01.894940+0000","last_active":"2023-09-23T20:52:16.109868+0000","last_peered":"2023-09-23T20:52:16.109868+0000","last_clean":"2023-09-23T20:52:16.109868+0000","last_became_active":"2023-09-23T20:52:01.894428+0000","last_became_peered":"2023-09-23T20:52:01.894428+0000","last_unstale":"2023-09-23T20:52:16.109868+0000","last_undegraded":"2023-09-23T20:52:16.109868+0000","last_fullsized":"2023-09-23T20:52:16.109868+0000","mapping_epoch":688,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":689,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:49:37.762492+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:49:37.762492+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:01:46.023633+0000","scrub_duration":0.0031485979999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":493,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.108437+0000","last_change":"2023-09-23T20:37:43.570610+0000","last_active":"2023-09-23T20:52:16.108437+0000","last_peered":"2023-09-23T20:52:16.108437+0000","last_clean":"2023-09-23T20:52:16.108437+0000","last_became_active":"2023-09-23T20:37:43.570216+0000","last_became_peered":"2023-09-23T20:37:43.570216+0000","last_unstale":"2023-09-23T20:52:16.108437+0000","last_undegraded":"2023-09-23T20:52:16.108437+0000","last_fullsized":"2023-09-23T20:52:16.108437+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:39:26.555329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":542,"reported_epoch":695,"state":"active+clean","last_fresh":"2023-09-23T20:52:19.947394+0000","last_change":"2023-09-23T20:47:09.172573+0000","last_active":"2023-09-23T20:52:19.947394+0000","last_peered":"2023-09-23T20:52:19.947394+0000","last_clean":"2023-09-23T20:52:19.947394+0000","last_became_active":"2023-09-23T20:47:09.172298+0000","last_became_peered":"2023-09-23T20:47:09.172298+0000","last_unstale":"2023-09-23T20:52:19.947394+0000","last_undegraded":"2023-09-23T20:52:19.947394+0000","last_fullsized":"2023-09-23T20:52:19.947394+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:22:09.361284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":882,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.110680+0000","last_change":"2023-09-23T20:41:07.424609+0000","last_active":"2023-09-23T20:52:16.110680+0000","last_peered":"2023-09-23T20:52:16.110680+0000","last_clean":"2023-09-23T20:52:16.110680+0000","last_became_active":"2023-09-23T20:38:05.622261+0000","last_became_peered":"2023-09-23T20:38:05.622261+0000","last_unstale":"2023-09-23T20:52:16.110680+0000","last_undegraded":"2023-09-23T20:52:16.110680+0000","last_fullsized":"2023-09-23T20:52:16.110680+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:41:07.424492+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:41:07.424492+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:45:21.689123+0000","scrub_duration":0.0018960959999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"664'242","reported_seq":1446,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.108529+0000","last_change":"2023-09-23T20:50:39.737511+0000","last_active":"2023-09-23T20:52:16.108529+0000","last_peered":"2023-09-23T20:52:16.108529+0000","last_clean":"2023-09-23T20:52:16.108529+0000","last_became_active":"2023-09-23T20:29:45.520546+0000","last_became_peered":"2023-09-23T20:29:45.520546+0000","last_unstale":"2023-09-23T20:52:16.108529+0000","last_undegraded":"2023-09-23T20:52:16.108529+0000","last_fullsized":"2023-09-23T20:52:16.108529+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":242,"log_dups_size":0,"ondisk_log_size":242,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:58:56.085777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0012974410000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1152,"num_read_kb":4628,"num_write":256,"num_write_kb":3463,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":515,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.108808+0000","last_change":"2023-09-23T20:52:01.897961+0000","last_active":"2023-09-23T20:52:16.108808+0000","last_peered":"2023-09-23T20:52:16.108808+0000","last_clean":"2023-09-23T20:52:16.108808+0000","last_became_active":"2023-09-23T20:52:01.897687+0000","last_became_peered":"2023-09-23T20:52:01.897687+0000","last_unstale":"2023-09-23T20:52:16.108808+0000","last_undegraded":"2023-09-23T20:52:16.108808+0000","last_fullsized":"2023-09-23T20:52:16.108808+0000","mapping_epoch":688,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":689,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:48:27.254894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":692,"reported_epoch":695,"state":"active+clean","last_fresh":"2023-09-23T20:52:19.946786+0000","last_change":"2023-09-23T20:51:21.745692+0000","last_active":"2023-09-23T20:52:19.946786+0000","last_peered":"2023-09-23T20:52:19.946786+0000","last_clean":"2023-09-23T20:52:19.946786+0000","last_became_active":"2023-09-23T20:51:21.744961+0000","last_became_peered":"2023-09-23T20:51:21.744961+0000","last_unstale":"2023-09-23T20:52:19.946786+0000","last_undegraded":"2023-09-23T20:52:19.946786+0000","last_fullsized":"2023-09-23T20:52:19.946786+0000","mapping_epoch":673,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":674,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:14:06.022442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":794,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.108151+0000","last_change":"2023-09-23T20:52:01.894751+0000","last_active":"2023-09-23T20:52:16.108151+0000","last_peered":"2023-09-23T20:52:16.108151+0000","last_clean":"2023-09-23T20:52:16.108151+0000","last_became_active":"2023-09-23T20:52:01.893974+0000","last_became_peered":"2023-09-23T20:52:01.893974+0000","last_unstale":"2023-09-23T20:52:16.108151+0000","last_undegraded":"2023-09-23T20:52:16.108151+0000","last_fullsized":"2023-09-23T20:52:16.108151+0000","mapping_epoch":688,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":689,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:15.656880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:15.656880+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:31:59.751463+0000","scrub_duration":0.0016711180000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"664'209","reported_seq":1676,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.109794+0000","last_change":"2023-09-23T20:52:01.896675+0000","last_active":"2023-09-23T20:52:16.109794+0000","last_peered":"2023-09-23T20:52:16.109794+0000","last_clean":"2023-09-23T20:52:16.109794+0000","last_became_active":"2023-09-23T20:52:01.896048+0000","last_became_peered":"2023-09-23T20:52:01.896048+0000","last_unstale":"2023-09-23T20:52:16.109794+0000","last_undegraded":"2023-09-23T20:52:16.109794+0000","last_fullsized":"2023-09-23T20:52:16.109794+0000","mapping_epoch":688,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":689,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":209,"log_dups_size":0,"ondisk_log_size":209,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:07:06.650190+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001172474,"stat_sum":{"num_bytes":4194323,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":2248,"num_read_kb":3726,"num_write":182,"num_write_kb":5560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":151,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.109786+0000","last_change":"2023-09-23T20:47:09.177872+0000","last_active":"2023-09-23T20:52:16.109786+0000","last_peered":"2023-09-23T20:52:16.109786+0000","last_clean":"2023-09-23T20:52:16.109786+0000","last_became_active":"2023-09-23T20:47:09.176689+0000","last_became_peered":"2023-09-23T20:47:09.176689+0000","last_unstale":"2023-09-23T20:52:16.109786+0000","last_undegraded":"2023-09-23T20:52:16.109786+0000","last_fullsized":"2023-09-23T20:52:16.109786+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:14:01.590318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":679,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.108448+0000","last_change":"2023-09-23T20:48:48.151144+0000","last_active":"2023-09-23T20:52:16.108448+0000","last_peered":"2023-09-23T20:52:16.108448+0000","last_clean":"2023-09-23T20:52:16.108448+0000","last_became_active":"2023-09-23T20:48:48.150770+0000","last_became_peered":"2023-09-23T20:48:48.150770+0000","last_unstale":"2023-09-23T20:52:16.108448+0000","last_undegraded":"2023-09-23T20:52:16.108448+0000","last_fullsized":"2023-09-23T20:52:16.108448+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:14:08.549512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":698,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:15.936383+0000","last_change":"2023-09-23T20:47:11.618652+0000","last_active":"2023-09-23T20:52:15.936383+0000","last_peered":"2023-09-23T20:52:15.936383+0000","last_clean":"2023-09-23T20:52:15.936383+0000","last_became_active":"2023-09-23T20:47:11.618266+0000","last_became_peered":"2023-09-23T20:47:11.618266+0000","last_unstale":"2023-09-23T20:52:15.936383+0000","last_undegraded":"2023-09-23T20:52:15.936383+0000","last_fullsized":"2023-09-23T20:52:15.936383+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:26.978179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:26.978179+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:29:29.000683+0000","scrub_duration":0.0013962829999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"664'192","reported_seq":731,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.108578+0000","last_change":"2023-09-23T20:50:39.795569+0000","last_active":"2023-09-23T20:52:16.108578+0000","last_peered":"2023-09-23T20:52:16.108578+0000","last_clean":"2023-09-23T20:52:16.108578+0000","last_became_active":"2023-09-23T20:48:51.170485+0000","last_became_peered":"2023-09-23T20:48:51.170485+0000","last_unstale":"2023-09-23T20:52:16.108578+0000","last_undegraded":"2023-09-23T20:52:16.108578+0000","last_fullsized":"2023-09-23T20:52:16.108578+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":192,"log_dups_size":0,"ondisk_log_size":192,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:48:35.309684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038574200000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":830,"num_read_kb":3504,"num_write":183,"num_write_kb":2624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":688,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.109864+0000","last_change":"2023-09-23T20:48:48.151034+0000","last_active":"2023-09-23T20:52:16.109864+0000","last_peered":"2023-09-23T20:52:16.109864+0000","last_clean":"2023-09-23T20:52:16.109864+0000","last_became_active":"2023-09-23T20:48:48.150658+0000","last_became_peered":"2023-09-23T20:48:48.150658+0000","last_unstale":"2023-09-23T20:52:16.109864+0000","last_undegraded":"2023-09-23T20:52:16.109864+0000","last_fullsized":"2023-09-23T20:52:16.109864+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:44:34.819851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":230,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.109037+0000","last_change":"2023-09-23T20:51:21.745968+0000","last_active":"2023-09-23T20:52:16.109037+0000","last_peered":"2023-09-23T20:52:16.109037+0000","last_clean":"2023-09-23T20:52:16.109037+0000","last_became_active":"2023-09-23T20:51:21.744991+0000","last_became_peered":"2023-09-23T20:51:21.744991+0000","last_unstale":"2023-09-23T20:52:16.109037+0000","last_undegraded":"2023-09-23T20:52:16.109037+0000","last_fullsized":"2023-09-23T20:52:16.109037+0000","mapping_epoch":673,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":674,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:01.825839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"664'319","reported_seq":1180,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.109565+0000","last_change":"2023-09-23T20:52:01.895902+0000","last_active":"2023-09-23T20:52:16.109565+0000","last_peered":"2023-09-23T20:52:16.109565+0000","last_clean":"2023-09-23T20:52:16.109565+0000","last_became_active":"2023-09-23T20:52:01.895001+0000","last_became_peered":"2023-09-23T20:52:01.895001+0000","last_unstale":"2023-09-23T20:52:16.109565+0000","last_undegraded":"2023-09-23T20:52:16.109565+0000","last_fullsized":"2023-09-23T20:52:16.109565+0000","mapping_epoch":688,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":689,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":319,"log_dups_size":0,"ondisk_log_size":319,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:46:47.800593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043345100000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1396,"num_read_kb":4612,"num_write":290,"num_write_kb":4437,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":1157,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:15.936196+0000","last_change":"2023-09-23T20:38:05.602508+0000","last_active":"2023-09-23T20:52:15.936196+0000","last_peered":"2023-09-23T20:52:15.936196+0000","last_clean":"2023-09-23T20:52:15.936196+0000","last_became_active":"2023-09-23T20:38:05.602129+0000","last_became_peered":"2023-09-23T20:38:05.602129+0000","last_unstale":"2023-09-23T20:52:15.936196+0000","last_undegraded":"2023-09-23T20:52:15.936196+0000","last_fullsized":"2023-09-23T20:52:15.936196+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:45.343742+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:45.343742+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:48.910343+0000","scrub_duration":0.001475802,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":282,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.108712+0000","last_change":"2023-09-23T20:51:49.915404+0000","last_active":"2023-09-23T20:52:16.108712+0000","last_peered":"2023-09-23T20:52:16.108712+0000","last_clean":"2023-09-23T20:52:16.108712+0000","last_became_active":"2023-09-23T20:51:49.915178+0000","last_became_peered":"2023-09-23T20:51:49.915178+0000","last_unstale":"2023-09-23T20:52:16.108712+0000","last_undegraded":"2023-09-23T20:52:16.108712+0000","last_fullsized":"2023-09-23T20:52:16.108712+0000","mapping_epoch":683,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":684,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:23.717157+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":319,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.110502+0000","last_change":"2023-09-23T20:47:09.202302+0000","last_active":"2023-09-23T20:52:16.110502+0000","last_peered":"2023-09-23T20:52:16.110502+0000","last_clean":"2023-09-23T20:52:16.110502+0000","last_became_active":"2023-09-23T20:47:09.201797+0000","last_became_peered":"2023-09-23T20:47:09.201797+0000","last_unstale":"2023-09-23T20:52:16.110502+0000","last_undegraded":"2023-09-23T20:52:16.110502+0000","last_fullsized":"2023-09-23T20:52:16.110502+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:34:41.055705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":1065,"reported_epoch":695,"state":"active+clean","last_fresh":"2023-09-23T20:52:19.947376+0000","last_change":"2023-09-23T20:47:11.622262+0000","last_active":"2023-09-23T20:52:19.947376+0000","last_peered":"2023-09-23T20:52:19.947376+0000","last_clean":"2023-09-23T20:52:19.947376+0000","last_became_active":"2023-09-23T20:47:11.621920+0000","last_became_peered":"2023-09-23T20:47:11.621920+0000","last_unstale":"2023-09-23T20:52:19.947376+0000","last_undegraded":"2023-09-23T20:52:19.947376+0000","last_fullsized":"2023-09-23T20:52:19.947376+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:43.582146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:43.582146+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:01:15.153543+0000","scrub_duration":0.0016219730000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"664'171","reported_seq":1359,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:15.936148+0000","last_change":"2023-09-23T20:50:39.742996+0000","last_active":"2023-09-23T20:52:15.936148+0000","last_peered":"2023-09-23T20:52:15.936148+0000","last_clean":"2023-09-23T20:52:15.936148+0000","last_became_active":"2023-09-23T20:29:45.536439+0000","last_became_peered":"2023-09-23T20:29:45.536439+0000","last_unstale":"2023-09-23T20:52:15.936148+0000","last_undegraded":"2023-09-23T20:52:15.936148+0000","last_fullsized":"2023-09-23T20:52:15.936148+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":171,"log_dups_size":0,"ondisk_log_size":171,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:30:57.793132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032959200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1114,"num_read_kb":3170,"num_write":132,"num_write_kb":2190,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":709,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.108171+0000","last_change":"2023-09-23T20:47:09.198326+0000","last_active":"2023-09-23T20:52:16.108171+0000","last_peered":"2023-09-23T20:52:16.108171+0000","last_clean":"2023-09-23T20:52:16.108171+0000","last_became_active":"2023-09-23T20:47:09.198059+0000","last_became_peered":"2023-09-23T20:47:09.198059+0000","last_unstale":"2023-09-23T20:52:16.108171+0000","last_undegraded":"2023-09-23T20:52:16.108171+0000","last_fullsized":"2023-09-23T20:52:16.108171+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:38:53.903706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":704,"reported_epoch":695,"state":"active+clean","last_fresh":"2023-09-23T20:52:21.228992+0000","last_change":"2023-09-23T20:48:48.146977+0000","last_active":"2023-09-23T20:52:21.228992+0000","last_peered":"2023-09-23T20:52:21.228992+0000","last_clean":"2023-09-23T20:52:21.228992+0000","last_became_active":"2023-09-23T20:48:48.146657+0000","last_became_peered":"2023-09-23T20:48:48.146657+0000","last_unstale":"2023-09-23T20:52:21.228992+0000","last_undegraded":"2023-09-23T20:52:21.228992+0000","last_fullsized":"2023-09-23T20:52:21.228992+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:38:41.782692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":805,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.110466+0000","last_change":"2023-09-23T20:41:08.380084+0000","last_active":"2023-09-23T20:52:16.110466+0000","last_peered":"2023-09-23T20:52:16.110466+0000","last_clean":"2023-09-23T20:52:16.110466+0000","last_became_active":"2023-09-23T20:38:05.610881+0000","last_became_peered":"2023-09-23T20:38:05.610881+0000","last_unstale":"2023-09-23T20:52:16.110466+0000","last_undegraded":"2023-09-23T20:52:16.110466+0000","last_fullsized":"2023-09-23T20:52:16.110466+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:41:08.379959+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:41:08.379959+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:19:39.547923+0000","scrub_duration":0.0019158420000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"664'175","reported_seq":1658,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:15.936558+0000","last_change":"2023-09-23T20:50:39.743889+0000","last_active":"2023-09-23T20:52:15.936558+0000","last_peered":"2023-09-23T20:52:15.936558+0000","last_clean":"2023-09-23T20:52:15.936558+0000","last_became_active":"2023-09-23T20:47:14.628341+0000","last_became_peered":"2023-09-23T20:47:14.628341+0000","last_unstale":"2023-09-23T20:52:15.936558+0000","last_undegraded":"2023-09-23T20:52:15.936558+0000","last_fullsized":"2023-09-23T20:52:15.936558+0000","mapping_epoch":558,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":559,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":175,"log_dups_size":0,"ondisk_log_size":175,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:42:16.291849+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000229686,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1422,"num_read_kb":2425,"num_write":116,"num_write_kb":664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":681,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.108608+0000","last_change":"2023-09-23T20:51:21.747497+0000","last_active":"2023-09-23T20:52:16.108608+0000","last_peered":"2023-09-23T20:52:16.108608+0000","last_clean":"2023-09-23T20:52:16.108608+0000","last_became_active":"2023-09-23T20:51:21.747193+0000","last_became_peered":"2023-09-23T20:51:21.747193+0000","last_unstale":"2023-09-23T20:52:16.108608+0000","last_undegraded":"2023-09-23T20:52:16.108608+0000","last_fullsized":"2023-09-23T20:52:16.108608+0000","mapping_epoch":673,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":674,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:42:19.319230+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":659,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.109756+0000","last_change":"2023-09-23T20:51:21.743456+0000","last_active":"2023-09-23T20:52:16.109756+0000","last_peered":"2023-09-23T20:52:16.109756+0000","last_clean":"2023-09-23T20:52:16.109756+0000","last_became_active":"2023-09-23T20:51:21.743180+0000","last_became_peered":"2023-09-23T20:51:21.743180+0000","last_unstale":"2023-09-23T20:52:16.109756+0000","last_undegraded":"2023-09-23T20:52:16.109756+0000","last_fullsized":"2023-09-23T20:52:16.109756+0000","mapping_epoch":673,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":674,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:40:50.012029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":731,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.109299+0000","last_change":"2023-09-23T20:48:51.172830+0000","last_active":"2023-09-23T20:52:16.109299+0000","last_peered":"2023-09-23T20:52:16.109299+0000","last_clean":"2023-09-23T20:52:16.109299+0000","last_became_active":"2023-09-23T20:48:51.172481+0000","last_became_peered":"2023-09-23T20:48:51.172481+0000","last_unstale":"2023-09-23T20:52:16.109299+0000","last_undegraded":"2023-09-23T20:52:16.109299+0000","last_fullsized":"2023-09-23T20:52:16.109299+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:47:34.561747+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:47:34.561747+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:30:32.531477+0000","scrub_duration":0.0087949929999999992,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":192,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.108942+0000","last_change":"2023-09-23T20:51:49.841192+0000","last_active":"2023-09-23T20:52:16.108942+0000","last_peered":"2023-09-23T20:52:16.108942+0000","last_clean":"2023-09-23T20:52:16.108942+0000","last_became_active":"2023-09-23T20:51:49.840874+0000","last_became_peered":"2023-09-23T20:51:49.840874+0000","last_unstale":"2023-09-23T20:52:16.108942+0000","last_undegraded":"2023-09-23T20:52:16.108942+0000","last_fullsized":"2023-09-23T20:52:16.108942+0000","mapping_epoch":683,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":684,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:41:16.392417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"664'349","reported_seq":2056,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.109605+0000","last_change":"2023-09-23T20:50:39.737163+0000","last_active":"2023-09-23T20:52:16.109605+0000","last_peered":"2023-09-23T20:52:16.109605+0000","last_clean":"2023-09-23T20:52:16.109605+0000","last_became_active":"2023-09-23T20:47:11.621284+0000","last_became_peered":"2023-09-23T20:47:11.621284+0000","last_unstale":"2023-09-23T20:52:16.109605+0000","last_undegraded":"2023-09-23T20:52:16.109605+0000","last_fullsized":"2023-09-23T20:52:16.109605+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":349,"log_dups_size":0,"ondisk_log_size":349,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:43:50.658512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0014591350000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":2217,"num_read_kb":2864,"num_write":263,"num_write_kb":2168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":479,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.108351+0000","last_change":"2023-09-23T20:37:43.582105+0000","last_active":"2023-09-23T20:52:16.108351+0000","last_peered":"2023-09-23T20:52:16.108351+0000","last_clean":"2023-09-23T20:52:16.108351+0000","last_became_active":"2023-09-23T20:37:43.581860+0000","last_became_peered":"2023-09-23T20:37:43.581860+0000","last_unstale":"2023-09-23T20:52:16.108351+0000","last_undegraded":"2023-09-23T20:52:16.108351+0000","last_fullsized":"2023-09-23T20:52:16.108351+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:18:38.765486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"90'161","reported_seq":947,"reported_epoch":695,"state":"active+clean","last_fresh":"2023-09-23T20:52:20.443781+0000","last_change":"2023-09-23T20:51:24.741192+0000","last_active":"2023-09-23T20:52:20.443781+0000","last_peered":"2023-09-23T20:52:20.443781+0000","last_clean":"2023-09-23T20:52:20.443781+0000","last_became_active":"2023-09-23T20:51:24.740845+0000","last_became_peered":"2023-09-23T20:51:24.740845+0000","last_unstale":"2023-09-23T20:52:20.443781+0000","last_undegraded":"2023-09-23T20:52:20.443781+0000","last_fullsized":"2023-09-23T20:52:20.443781+0000","mapping_epoch":676,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":677,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:34:54.504554+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:34:54.504554+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:57:05.662352+0000","scrub_duration":0.011176864,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"664'293","reported_seq":1452,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.110649+0000","last_change":"2023-09-23T20:50:39.740476+0000","last_active":"2023-09-23T20:52:16.110649+0000","last_peered":"2023-09-23T20:52:16.110649+0000","last_clean":"2023-09-23T20:52:16.110649+0000","last_became_active":"2023-09-23T20:48:51.217013+0000","last_became_peered":"2023-09-23T20:48:51.217013+0000","last_unstale":"2023-09-23T20:52:16.110649+0000","last_undegraded":"2023-09-23T20:52:16.110649+0000","last_fullsized":"2023-09-23T20:52:16.110649+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":293,"log_dups_size":0,"ondisk_log_size":293,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:40:42.297947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00027326800000000001,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1602,"num_read_kb":1486,"num_write":268,"num_write_kb":4284,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":48,"num_bytes_recovered":25165824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":712,"reported_epoch":695,"state":"active+clean","last_fresh":"2023-09-23T20:52:21.229078+0000","last_change":"2023-09-23T20:47:09.317116+0000","last_active":"2023-09-23T20:52:21.229078+0000","last_peered":"2023-09-23T20:52:21.229078+0000","last_clean":"2023-09-23T20:52:21.229078+0000","last_became_active":"2023-09-23T20:47:09.205930+0000","last_became_peered":"2023-09-23T20:47:09.205930+0000","last_unstale":"2023-09-23T20:52:21.229078+0000","last_undegraded":"2023-09-23T20:52:21.229078+0000","last_fullsized":"2023-09-23T20:52:21.229078+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:04:55.528625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":171,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.108859+0000","last_change":"2023-09-23T20:51:21.746106+0000","last_active":"2023-09-23T20:52:16.108859+0000","last_peered":"2023-09-23T20:52:16.108859+0000","last_clean":"2023-09-23T20:52:16.108859+0000","last_became_active":"2023-09-23T20:51:21.745777+0000","last_became_peered":"2023-09-23T20:51:21.745777+0000","last_unstale":"2023-09-23T20:52:16.108859+0000","last_undegraded":"2023-09-23T20:52:16.108859+0000","last_fullsized":"2023-09-23T20:52:16.108859+0000","mapping_epoch":673,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":674,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:59:40.976999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"90'161","reported_seq":604,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.109278+0000","last_change":"2023-09-23T20:51:50.434718+0000","last_active":"2023-09-23T20:52:16.109278+0000","last_peered":"2023-09-23T20:52:16.109278+0000","last_clean":"2023-09-23T20:52:16.109278+0000","last_became_active":"2023-09-23T20:51:49.839630+0000","last_became_peered":"2023-09-23T20:51:49.839630+0000","last_unstale":"2023-09-23T20:52:16.109278+0000","last_undegraded":"2023-09-23T20:52:16.109278+0000","last_fullsized":"2023-09-23T20:52:16.109278+0000","mapping_epoch":683,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":684,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:51:50.434626+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:51:50.434626+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:33:23.482033+0000","scrub_duration":0.0019105050000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":670,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.109717+0000","last_change":"2023-09-23T20:47:09.197601+0000","last_active":"2023-09-23T20:52:16.109717+0000","last_peered":"2023-09-23T20:52:16.109717+0000","last_clean":"2023-09-23T20:52:16.109717+0000","last_became_active":"2023-09-23T20:47:09.196967+0000","last_became_peered":"2023-09-23T20:47:09.196967+0000","last_unstale":"2023-09-23T20:52:16.109717+0000","last_undegraded":"2023-09-23T20:52:16.109717+0000","last_fullsized":"2023-09-23T20:52:16.109717+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:06:56.826785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":115,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.109105+0000","last_change":"2023-09-23T20:48:48.158376+0000","last_active":"2023-09-23T20:52:16.109105+0000","last_peered":"2023-09-23T20:52:16.109105+0000","last_clean":"2023-09-23T20:52:16.109105+0000","last_became_active":"2023-09-23T20:48:48.158069+0000","last_became_peered":"2023-09-23T20:48:48.158069+0000","last_unstale":"2023-09-23T20:52:16.109105+0000","last_undegraded":"2023-09-23T20:52:16.109105+0000","last_fullsized":"2023-09-23T20:52:16.109105+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:34.658561+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"90'161","reported_seq":772,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.108685+0000","last_change":"2023-09-23T20:41:06.499511+0000","last_active":"2023-09-23T20:52:16.108685+0000","last_peered":"2023-09-23T20:52:16.108685+0000","last_clean":"2023-09-23T20:52:16.108685+0000","last_became_active":"2023-09-23T20:38:05.601874+0000","last_became_peered":"2023-09-23T20:38:05.601874+0000","last_unstale":"2023-09-23T20:52:16.108685+0000","last_undegraded":"2023-09-23T20:52:16.108685+0000","last_fullsized":"2023-09-23T20:52:16.108685+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:41:06.499396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:41:06.499396+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:31:34.059077+0000","scrub_duration":0.0019670310000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":168,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.108555+0000","last_change":"2023-09-23T20:52:01.896760+0000","last_active":"2023-09-23T20:52:16.108555+0000","last_peered":"2023-09-23T20:52:16.108555+0000","last_clean":"2023-09-23T20:52:16.108555+0000","last_became_active":"2023-09-23T20:52:01.896426+0000","last_became_peered":"2023-09-23T20:52:01.896426+0000","last_unstale":"2023-09-23T20:52:16.108555+0000","last_undegraded":"2023-09-23T20:52:16.108555+0000","last_fullsized":"2023-09-23T20:52:16.108555+0000","mapping_epoch":688,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":689,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:41:39.174148+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":151,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.109744+0000","last_change":"2023-09-23T20:47:09.186579+0000","last_active":"2023-09-23T20:52:16.109744+0000","last_peered":"2023-09-23T20:52:16.109744+0000","last_clean":"2023-09-23T20:52:16.109744+0000","last_became_active":"2023-09-23T20:47:09.186227+0000","last_became_peered":"2023-09-23T20:47:09.186227+0000","last_unstale":"2023-09-23T20:52:16.109744+0000","last_undegraded":"2023-09-23T20:52:16.109744+0000","last_fullsized":"2023-09-23T20:52:16.109744+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:45:39.485308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":36,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.108881+0000","last_change":"2023-09-23T20:52:01.899486+0000","last_active":"2023-09-23T20:52:16.108881+0000","last_peered":"2023-09-23T20:52:16.108881+0000","last_clean":"2023-09-23T20:52:16.108881+0000","last_became_active":"2023-09-23T20:52:01.899077+0000","last_became_peered":"2023-09-23T20:52:01.899077+0000","last_unstale":"2023-09-23T20:52:16.108881+0000","last_undegraded":"2023-09-23T20:52:16.108881+0000","last_fullsized":"2023-09-23T20:52:16.108881+0000","mapping_epoch":688,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":689,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:03.951836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":151,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.108485+0000","last_change":"2023-09-23T20:47:09.181240+0000","last_active":"2023-09-23T20:52:16.108485+0000","last_peered":"2023-09-23T20:52:16.108485+0000","last_clean":"2023-09-23T20:52:16.108485+0000","last_became_active":"2023-09-23T20:47:09.181022+0000","last_became_peered":"2023-09-23T20:47:09.181022+0000","last_unstale":"2023-09-23T20:52:16.108485+0000","last_undegraded":"2023-09-23T20:52:16.108485+0000","last_fullsized":"2023-09-23T20:52:16.108485+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:38:59.191781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"90'161","reported_seq":950,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:15.936409+0000","last_change":"2023-09-23T20:38:05.591992+0000","last_active":"2023-09-23T20:52:15.936409+0000","last_peered":"2023-09-23T20:52:15.936409+0000","last_clean":"2023-09-23T20:52:15.936409+0000","last_became_active":"2023-09-23T20:38:05.591701+0000","last_became_peered":"2023-09-23T20:38:05.591701+0000","last_unstale":"2023-09-23T20:52:15.936409+0000","last_undegraded":"2023-09-23T20:52:15.936409+0000","last_fullsized":"2023-09-23T20:52:15.936409+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:02.384133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:02.384133+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:41.722542+0000","scrub_duration":0.001769098,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":539,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.108951+0000","last_change":"2023-09-23T20:48:48.158348+0000","last_active":"2023-09-23T20:52:16.108951+0000","last_peered":"2023-09-23T20:52:16.108951+0000","last_clean":"2023-09-23T20:52:16.108951+0000","last_became_active":"2023-09-23T20:48:48.157982+0000","last_became_peered":"2023-09-23T20:48:48.157982+0000","last_unstale":"2023-09-23T20:52:16.108951+0000","last_undegraded":"2023-09-23T20:52:16.108951+0000","last_fullsized":"2023-09-23T20:52:16.108951+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:03:48.078707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":505,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:17.946525+0000","last_change":"2023-09-23T20:48:48.159781+0000","last_active":"2023-09-23T20:52:17.946525+0000","last_peered":"2023-09-23T20:52:17.946525+0000","last_clean":"2023-09-23T20:52:17.946525+0000","last_became_active":"2023-09-23T20:48:48.159505+0000","last_became_peered":"2023-09-23T20:48:48.159505+0000","last_unstale":"2023-09-23T20:52:17.946525+0000","last_undegraded":"2023-09-23T20:52:17.946525+0000","last_fullsized":"2023-09-23T20:52:17.946525+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:25:18.100645+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"90'161","reported_seq":825,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.108875+0000","last_change":"2023-09-23T20:47:11.621758+0000","last_active":"2023-09-23T20:52:16.108875+0000","last_peered":"2023-09-23T20:52:16.108875+0000","last_clean":"2023-09-23T20:52:16.108875+0000","last_became_active":"2023-09-23T20:47:11.621345+0000","last_became_peered":"2023-09-23T20:47:11.621345+0000","last_unstale":"2023-09-23T20:52:16.108875+0000","last_undegraded":"2023-09-23T20:52:16.108875+0000","last_fullsized":"2023-09-23T20:52:16.108875+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:01.504973+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:01.504973+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:31:17.112393+0000","scrub_duration":0.00202344,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":791,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.109494+0000","last_change":"2023-09-23T20:51:21.755459+0000","last_active":"2023-09-23T20:52:16.109494+0000","last_peered":"2023-09-23T20:52:16.109494+0000","last_clean":"2023-09-23T20:52:16.109494+0000","last_became_active":"2023-09-23T20:51:21.746135+0000","last_became_peered":"2023-09-23T20:51:21.746135+0000","last_unstale":"2023-09-23T20:52:16.109494+0000","last_undegraded":"2023-09-23T20:52:16.109494+0000","last_fullsized":"2023-09-23T20:52:16.109494+0000","mapping_epoch":673,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":674,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:47:46.578750+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":5540,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":161,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.108146+0000","last_change":"2023-09-23T20:48:23.917302+0000","last_active":"2023-09-23T20:52:16.108146+0000","last_peered":"2023-09-23T20:52:16.108146+0000","last_clean":"2023-09-23T20:52:16.108146+0000","last_became_active":"2023-09-23T20:48:23.916956+0000","last_became_peered":"2023-09-23T20:48:23.916956+0000","last_unstale":"2023-09-23T20:52:16.108146+0000","last_undegraded":"2023-09-23T20:52:16.108146+0000","last_fullsized":"2023-09-23T20:52:16.108146+0000","mapping_epoch":577,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":578,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:42:27.989294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":125,"num_bytes_recovered":16468,"num_keys_recovered":80,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26691,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":8388627,"num_objects":5,"num_object_clones":0,"num_object_copies":15,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":11981,"num_read_kb":26415,"num_write":1690,"num_write_kb":25390,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":156,"num_bytes_recovered":25165843,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":25178112,"data_stored":25165881,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6082,"internal_metadata":0},"log_size":1950,"ondisk_log_size":1950,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201776,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":620404,"kb_used_data":11436,"kb_used_omap":0,"kb_used_meta":608960,"kb_avail":93124748,"statfs":{"total":95995035648,"available":95359741952,"internally_reserved":0,"allocated":11710464,"data_stored":9904172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":623575040},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:52:16 2023","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":0.99399999999999999,"15min":1.639},"min":{"1min":0.307,"5min":0.224,"15min":0.224},"max":{"1min":0.71499999999999997,"5min":5.6970000000000001,"15min":18.315000000000001},"last":0.46300000000000002},{"interface":"front","average":{"1min":0.56000000000000005,"5min":1.0980000000000001,"15min":1.506},"min":{"1min":0.35299999999999998,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.72999999999999998,"5min":8.0530000000000008,"15min":15.327999999999999},"last":0.57499999999999996}]},{"osd":1,"last update":"Sat Sep 23 20:52:20 2023","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":0.91400000000000003,"15min":1.4350000000000001},"min":{"1min":0.35599999999999998,"5min":0.28399999999999997,"15min":0.24399999999999999},"max":{"1min":0.76400000000000001,"5min":7.3789999999999996,"15min":29.254999999999999},"last":0.63100000000000001},{"interface":"front","average":{"1min":0.58899999999999997,"5min":1.127,"15min":1.897},"min":{"1min":0.379,"5min":0.27000000000000002,"15min":0.26700000000000002},"max":{"1min":0.85899999999999999,"5min":10.861000000000001,"15min":16.882000000000001},"last":0.75900000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:52:16 2023","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":1.3129999999999999,"15min":1.7909999999999999},"min":{"1min":0.42699999999999999,"5min":0.28499999999999998,"15min":0.27900000000000003},"max":{"1min":0.81100000000000005,"5min":10.461,"15min":16.965},"last":0.85799999999999998},{"interface":"front","average":{"1min":0.51700000000000002,"5min":1.2390000000000001,"15min":1.8320000000000001},"min":{"1min":0.33900000000000002,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.67400000000000004,"5min":7.3019999999999996,"15min":23.414000000000001},"last":0.54700000000000004}]},{"osd":3,"last update":"Sat Sep 23 20:51:43 2023","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":1.329,"15min":1.734},"min":{"1min":0.36199999999999999,"5min":0.32200000000000001,"15min":0.311},"max":{"1min":0.72999999999999998,"5min":7.2880000000000003,"15min":23.352},"last":0.65500000000000003},{"interface":"front","average":{"1min":0.61499999999999999,"5min":1.4059999999999999,"15min":1.917},"min":{"1min":0.436,"5min":0.35399999999999998,"15min":0.34300000000000003},"max":{"1min":0.91800000000000004,"5min":6.5369999999999999,"15min":20.969999999999999},"last":0.71299999999999997}]},{"osd":4,"last update":"Sat Sep 23 20:51:43 2023","interfaces":[{"interface":"back","average":{"1min":0.49399999999999999,"5min":0.96199999999999997,"15min":1.1890000000000001},"min":{"1min":0.32100000000000001,"5min":0.25800000000000001,"15min":0.254},"max":{"1min":0.84899999999999998,"5min":7.4109999999999996,"15min":18.364999999999998},"last":0.73699999999999999},{"interface":"front","average":{"1min":0.54000000000000004,"5min":1.2230000000000001,"15min":1.458},"min":{"1min":0.313,"5min":0.247,"15min":0.247},"max":{"1min":0.996,"5min":27.425999999999998,"15min":27.425999999999998},"last":0.66900000000000004}]},{"osd":5,"last update":"Sat Sep 23 20:51:43 2023","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":0.86199999999999999,"15min":1.216},"min":{"1min":0.34499999999999997,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.77200000000000002,"5min":7.2270000000000003,"15min":15.005000000000001},"last":0.61199999999999999},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.90500000000000003,"15min":1.3620000000000001},"min":{"1min":0.32500000000000001,"5min":0.32200000000000001,"15min":0.27700000000000002},"max":{"1min":0.89400000000000002,"5min":7.335,"15min":18.390000000000001},"last":0.79100000000000004}]},{"osd":6,"last update":"Sat Sep 23 20:51:43 2023","interfaces":[{"interface":"back","average":{"1min":0.57799999999999996,"5min":1.0960000000000001,"15min":1.5369999999999999},"min":{"1min":0.38100000000000001,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.94699999999999995,"5min":6.4720000000000004,"15min":18.417000000000002},"last":0.69399999999999995},{"interface":"front","average":{"1min":0.61799999999999999,"5min":1.0920000000000001,"15min":1.371},"min":{"1min":0.36499999999999999,"5min":0.35899999999999999,"15min":0.32300000000000001},"max":{"1min":0.96899999999999997,"5min":6.0910000000000002,"15min":15.369999999999999},"last":0.81000000000000005}]}]},{"osd":6,"up_from":47,"seq":201863463415,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":543528,"kb_used_data":15584,"kb_used_omap":0,"kb_used_meta":527936,"kb_avail":93201624,"statfs":{"total":95995035648,"available":95438462976,"internally_reserved":0,"allocated":15958016,"data_stored":14163341,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":540606464},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:52:12 2023","interfaces":[{"interface":"back","average":{"1min":0.48499999999999999,"5min":1.423,"15min":1.6060000000000001},"min":{"1min":0.30299999999999999,"5min":0.24399999999999999,"15min":0.219},"max":{"1min":0.84699999999999998,"5min":15.574999999999999,"15min":21.018000000000001},"last":0.60499999999999998},{"interface":"front","average":{"1min":0.53500000000000003,"5min":1.224,"15min":1.8879999999999999},"min":{"1min":0.26000000000000001,"5min":0.255,"15min":0.23300000000000001},"max":{"1min":0.995,"5min":11.557,"15min":60.328000000000003},"last":0.38900000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:52:29 2023","interfaces":[{"interface":"back","average":{"1min":0.878,"5min":1.341,"15min":1.714},"min":{"1min":0.28599999999999998,"5min":0.26000000000000001,"15min":0.249},"max":{"1min":7.3470000000000004,"5min":8.9350000000000005,"15min":64.274000000000001},"last":0.45700000000000002},{"interface":"front","average":{"1min":0.55600000000000005,"5min":1.0640000000000001,"15min":1.4199999999999999},"min":{"1min":0.34399999999999997,"5min":0.312,"15min":0.28000000000000003},"max":{"1min":0.89900000000000002,"5min":8.0969999999999995,"15min":17.786000000000001},"last":0.438}]},{"osd":2,"last update":"Sat Sep 23 20:52:12 2023","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":1.712,"15min":1.9330000000000001},"min":{"1min":0.36899999999999999,"5min":0.29699999999999999,"15min":0.27800000000000002},"max":{"1min":0.93200000000000005,"5min":9.0730000000000004,"15min":22.591000000000001},"last":0.51800000000000002},{"interface":"front","average":{"1min":0.61099999999999999,"5min":1.583,"15min":1.726},"min":{"1min":0.439,"5min":0.375,"15min":0.311},"max":{"1min":0.94999999999999996,"5min":12.798,"15min":22.614999999999998},"last":0.65800000000000003}]},{"osd":3,"last update":"Sat Sep 23 20:52:01 2023","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":1.3879999999999999,"15min":1.754},"min":{"1min":0.44400000000000001,"5min":0.307,"15min":0.28399999999999997},"max":{"1min":0.96299999999999997,"5min":9.6859999999999999,"15min":11.664},"last":0.48499999999999999},{"interface":"front","average":{"1min":0.59299999999999997,"5min":1.611,"15min":1.9319999999999999},"min":{"1min":0.40799999999999997,"5min":0.32700000000000001,"15min":0.30499999999999999},"max":{"1min":0.91400000000000003,"5min":11.6,"15min":11.913},"last":0.53500000000000003}]},{"osd":4,"last update":"Sat Sep 23 20:52:29 2023","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.91000000000000003,"15min":0.94599999999999995},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.25600000000000001},"max":{"1min":0.78800000000000003,"5min":8.1229999999999993,"15min":23.904},"last":0.55700000000000005},{"interface":"front","average":{"1min":0.58599999999999997,"5min":1.2649999999999999,"15min":1.175},"min":{"1min":0.309,"5min":0.309,"15min":0.28399999999999997},"max":{"1min":0.82099999999999995,"5min":11.999000000000001,"15min":11.999000000000001},"last":0.58499999999999996}]},{"osd":5,"last update":"Sat Sep 23 20:52:29 2023","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":1.0109999999999999,"15min":0.94899999999999995},"min":{"1min":0.33100000000000002,"5min":0.32800000000000001,"15min":0.253},"max":{"1min":0.85199999999999998,"5min":8.0440000000000005,"15min":8.0440000000000005},"last":0.72099999999999997},{"interface":"front","average":{"1min":0.63300000000000001,"5min":1.079,"15min":1.2689999999999999},"min":{"1min":0.34300000000000003,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.84999999999999998,"5min":8.2769999999999992,"15min":9.3559999999999999},"last":0.629}]},{"osd":7,"last update":"Sat Sep 23 20:52:01 2023","interfaces":[{"interface":"back","average":{"1min":0.58999999999999997,"5min":0.92900000000000005,"15min":0.97699999999999998},"min":{"1min":0.42999999999999999,"5min":0.35199999999999998,"15min":0.28399999999999997},"max":{"1min":0.82899999999999996,"5min":8.0719999999999992,"15min":8.9309999999999992},"last":0.69499999999999995},{"interface":"front","average":{"1min":0.60999999999999999,"5min":2.907,"15min":1.9610000000000001},"min":{"1min":0.38,"5min":0.38,"15min":0.34699999999999998},"max":{"1min":0.79700000000000004,"5min":159.91999999999999,"15min":159.91999999999999},"last":0.67200000000000004}]}]},{"osd":1,"up_from":266,"seq":1142461301014,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":823792,"kb_used_data":12968,"kb_used_omap":8,"kb_used_meta":810807,"kb_avail":92921360,"statfs":{"total":95995035648,"available":95151472640,"internally_reserved":0,"allocated":13279232,"data_stored":9856260,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8206,"internal_metadata":830267378},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:52:35 2023","interfaces":[{"interface":"back","average":{"1min":0.36499999999999999,"5min":1.452,"15min":1.8979999999999999},"min":{"1min":0.23699999999999999,"5min":0.20499999999999999,"15min":0.20000000000000001},"max":{"1min":0.55900000000000005,"5min":20.292000000000002,"15min":20.292000000000002},"last":0.29099999999999998},{"interface":"front","average":{"1min":0.42899999999999999,"5min":1.327,"15min":2.1949999999999998},"min":{"1min":0.28399999999999997,"5min":0.217,"15min":0.20100000000000001},"max":{"1min":1.246,"5min":17.463000000000001,"15min":20.346},"last":0.32700000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:52:35 2023","interfaces":[{"interface":"back","average":{"1min":0.36799999999999999,"5min":1.4770000000000001,"15min":2.242},"min":{"1min":0.26900000000000002,"5min":0.23100000000000001,"15min":0.216},"max":{"1min":0.50800000000000001,"5min":23.341000000000001,"15min":32.255000000000003},"last":0.314},{"interface":"front","average":{"1min":0.40600000000000003,"5min":1.573,"15min":1.923},"min":{"1min":0.29899999999999999,"5min":0.249,"15min":0.20899999999999999},"max":{"1min":0.59999999999999998,"5min":34.002000000000002,"15min":34.002000000000002},"last":0.34899999999999998}]},{"osd":3,"last update":"Sat Sep 23 20:52:35 2023","interfaces":[{"interface":"back","average":{"1min":0.435,"5min":1.2370000000000001,"15min":1.762},"min":{"1min":0.29899999999999999,"5min":0.28899999999999998,"15min":0.247},"max":{"1min":0.58199999999999996,"5min":10.135999999999999,"15min":17.366},"last":0.39100000000000001},{"interface":"front","average":{"1min":0.42099999999999999,"5min":1.4350000000000001,"15min":2.1629999999999998},"min":{"1min":0.251,"5min":0.251,"15min":0.22900000000000001},"max":{"1min":0.64900000000000002,"5min":17.343,"15min":17.343},"last":0.40100000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:52:35 2023","interfaces":[{"interface":"back","average":{"1min":0.624,"5min":1.5329999999999999,"15min":1.954},"min":{"1min":0.45600000000000002,"5min":0.33900000000000002,"15min":0.28599999999999998},"max":{"1min":0.80600000000000005,"5min":9.5359999999999996,"15min":20.634},"last":0.57999999999999996},{"interface":"front","average":{"1min":0.64700000000000002,"5min":1.3680000000000001,"15min":1.585},"min":{"1min":0.51400000000000001,"5min":0.34699999999999998,"15min":0.32600000000000001},"max":{"1min":0.77400000000000002,"5min":9.8249999999999993,"15min":20.835999999999999},"last":0.64100000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:52:35 2023","interfaces":[{"interface":"back","average":{"1min":0.70999999999999996,"5min":1.474,"15min":1.95},"min":{"1min":0.52600000000000002,"5min":0.38,"15min":0.308},"max":{"1min":0.94299999999999995,"5min":17.396999999999998,"15min":17.396999999999998},"last":0.71699999999999997},{"interface":"front","average":{"1min":0.69499999999999995,"5min":1.5369999999999999,"15min":2.0640000000000001},"min":{"1min":0.5,"5min":0.371,"15min":0.32500000000000001},"max":{"1min":0.86899999999999999,"5min":16.366,"15min":16.366},"last":0.65500000000000003}]},{"osd":6,"last update":"Sat Sep 23 20:52:35 2023","interfaces":[{"interface":"back","average":{"1min":0.67200000000000004,"5min":1.679,"15min":2.0579999999999998},"min":{"1min":0.505,"5min":0.41599999999999998,"15min":0.316},"max":{"1min":0.82899999999999996,"5min":23.274000000000001,"15min":23.274000000000001},"last":0.58899999999999997},{"interface":"front","average":{"1min":0.66500000000000004,"5min":1.6100000000000001,"15min":2.157},"min":{"1min":0.432,"5min":0.40500000000000003,"15min":0.317},"max":{"1min":0.91800000000000004,"5min":9.6639999999999997,"15min":11.832000000000001},"last":0.68899999999999995}]},{"osd":7,"last update":"Sat Sep 23 20:52:35 2023","interfaces":[{"interface":"back","average":{"1min":0.72699999999999998,"5min":1.5680000000000001,"15min":2.048},"min":{"1min":0.47799999999999998,"5min":0.46899999999999997,"15min":0.37},"max":{"1min":0.90300000000000002,"5min":17.433,"15min":17.433},"last":0.66700000000000004},{"interface":"front","average":{"1min":0.71499999999999997,"5min":1.7030000000000001,"15min":2.2069999999999999},"min":{"1min":0.48299999999999998,"5min":0.44500000000000001,"15min":0.35999999999999999},"max":{"1min":0.96699999999999997,"5min":9.8369999999999997,"15min":11.864000000000001},"last":0.54000000000000004}]}]},{"osd":0,"up_from":94,"seq":403726926217,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1004216,"kb_used_data":9008,"kb_used_omap":7,"kb_used_meta":995192,"kb_avail":92740936,"statfs":{"total":95995035648,"available":94966718464,"internally_reserved":0,"allocated":9224192,"data_stored":5969793,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":1019076625},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:52:36 2023","interfaces":[{"interface":"back","average":{"1min":0.34999999999999998,"5min":1.278,"15min":2.2040000000000002},"min":{"1min":0.25600000000000001,"5min":0.20100000000000001,"15min":0.184},"max":{"1min":0.46200000000000002,"5min":9.6449999999999996,"15min":152.387},"last":0.40200000000000002},{"interface":"front","average":{"1min":0.41699999999999998,"5min":1.242,"15min":2.3340000000000001},"min":{"1min":0.23499999999999999,"5min":0.218,"15min":0.20399999999999999},"max":{"1min":0.54600000000000004,"5min":12.317,"15min":152.84100000000001},"last":0.46999999999999997}]},{"osd":2,"last update":"Sat Sep 23 20:52:13 2023","interfaces":[{"interface":"back","average":{"1min":0.35399999999999998,"5min":1.4970000000000001,"15min":2.4279999999999999},"min":{"1min":0.20799999999999999,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":0.51500000000000001,"5min":14.538,"15min":62.726999999999997},"last":0.38100000000000001},{"interface":"front","average":{"1min":0.38500000000000001,"5min":1.335,"15min":2.4140000000000001},"min":{"1min":0.28999999999999998,"5min":0.219,"15min":0.217},"max":{"1min":0.48099999999999998,"5min":10.159000000000001,"15min":62.689},"last":0.45500000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:51:47 2023","interfaces":[{"interface":"back","average":{"1min":0.68400000000000005,"5min":1.72,"15min":2.5939999999999999},"min":{"1min":0.27500000000000002,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":5.9329999999999998,"5min":14.494,"15min":55.017000000000003},"last":0.44},{"interface":"front","average":{"1min":0.45100000000000001,"5min":1.7290000000000001,"15min":2.504},"min":{"1min":0.30299999999999999,"5min":0.28199999999999997,"15min":0.21199999999999999},"max":{"1min":0.71599999999999997,"5min":12.051,"15min":55.665999999999997},"last":0.496}]},{"osd":4,"last update":"Sat Sep 23 20:52:13 2023","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":1.45,"15min":1.7929999999999999},"min":{"1min":0.40300000000000002,"5min":0.29299999999999998,"15min":0.28899999999999998},"max":{"1min":0.78700000000000003,"5min":11.75,"15min":11.75},"last":0.629},{"interface":"front","average":{"1min":0.58099999999999996,"5min":1.6499999999999999,"15min":1.7070000000000001},"min":{"1min":0.39200000000000002,"5min":0.31,"15min":0.27400000000000002},"max":{"1min":0.76100000000000001,"5min":12.356999999999999,"15min":12.356999999999999},"last":0.73599999999999999}]},{"osd":5,"last update":"Sat Sep 23 20:52:13 2023","interfaces":[{"interface":"back","average":{"1min":0.626,"5min":1.482,"15min":1.768},"min":{"1min":0.40799999999999997,"5min":0.34300000000000003,"15min":0.26500000000000001},"max":{"1min":0.88400000000000001,"5min":9.3339999999999996,"15min":10.239000000000001},"last":0.76100000000000001},{"interface":"front","average":{"1min":0.64600000000000002,"5min":1.331,"15min":2.0310000000000001},"min":{"1min":0.48599999999999999,"5min":0.33000000000000002,"15min":0.29799999999999999},"max":{"1min":0.82799999999999996,"5min":10.089,"15min":26.228000000000002},"last":0.64100000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:52:13 2023","interfaces":[{"interface":"back","average":{"1min":0.63700000000000001,"5min":1.708,"15min":1.9159999999999999},"min":{"1min":0.48899999999999999,"5min":0.40600000000000003,"15min":0.34499999999999997},"max":{"1min":0.91400000000000003,"5min":9.2750000000000004,"15min":24.056000000000001},"last":0.68999999999999995},{"interface":"front","average":{"1min":0.63400000000000001,"5min":1.6000000000000001,"15min":1.984},"min":{"1min":0.443,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.90000000000000002,"5min":12.427,"15min":26.268000000000001},"last":0.62}]},{"osd":7,"last update":"Sat Sep 23 20:52:13 2023","interfaces":[{"interface":"back","average":{"1min":0.623,"5min":1.468,"15min":2.1160000000000001},"min":{"1min":0.44600000000000001,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.86499999999999999,"5min":12.486000000000001,"15min":26.177},"last":0.72899999999999998},{"interface":"front","average":{"1min":0.64900000000000002,"5min":1.5369999999999999,"15min":2.0870000000000002},"min":{"1min":0.5,"5min":0.39200000000000002,"15min":0.34200000000000003},"max":{"1min":0.82299999999999995,"5min":10.183999999999999,"15min":16.649000000000001},"last":0.746}]}]},{"osd":2,"up_from":104,"seq":446676599164,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":889016,"kb_used_data":12976,"kb_used_omap":11,"kb_used_meta":876020,"kb_avail":92856136,"statfs":{"total":95995035648,"available":95084683264,"internally_reserved":0,"allocated":13287424,"data_stored":10086340,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11878,"internal_metadata":897044890},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:52:15 2023","interfaces":[{"interface":"back","average":{"1min":0.36099999999999999,"5min":1.194,"15min":1.3360000000000001},"min":{"1min":0.23200000000000001,"5min":0.19500000000000001,"15min":0.17299999999999999},"max":{"1min":0.53200000000000003,"5min":20.48,"15min":20.48},"last":0.36899999999999999},{"interface":"front","average":{"1min":0.39300000000000002,"5min":1.1319999999999999,"15min":1.6479999999999999},"min":{"1min":0.28000000000000003,"5min":0.25700000000000001,"15min":0.23000000000000001},"max":{"1min":0.54400000000000004,"5min":8.3789999999999996,"15min":16.678999999999998},"last":0.32500000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:52:15 2023","interfaces":[{"interface":"back","average":{"1min":0.36199999999999999,"5min":1.169,"15min":1.9530000000000001},"min":{"1min":0.254,"5min":0.23799999999999999,"15min":0.219},"max":{"1min":0.51100000000000001,"5min":11.012,"15min":46.405000000000001},"last":0.28000000000000003},{"interface":"front","average":{"1min":0.41399999999999998,"5min":1.1359999999999999,"15min":1.78},"min":{"1min":0.29699999999999999,"5min":0.254,"15min":0.22},"max":{"1min":0.67000000000000004,"5min":10.191000000000001,"15min":46.459000000000003},"last":0.39300000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:51:45 2023","interfaces":[{"interface":"back","average":{"1min":0.47199999999999998,"5min":1.3879999999999999,"15min":2.0720000000000001},"min":{"1min":0.33300000000000002,"5min":0.28199999999999997,"15min":0.253},"max":{"1min":1.3939999999999999,"5min":10.051,"15min":30.283000000000001},"last":0.42299999999999999},{"interface":"front","average":{"1min":0.48699999999999999,"5min":1.446,"15min":1.484},"min":{"1min":0.29499999999999998,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":1.321,"5min":20.562999999999999,"15min":30.834},"last":0.41299999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:52:15 2023","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":1.151,"15min":1.524},"min":{"1min":0.45200000000000001,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.84499999999999997,"5min":8.3469999999999995,"15min":12.025},"last":0.40200000000000002},{"interface":"front","average":{"1min":0.64600000000000002,"5min":1.306,"15min":1.627},"min":{"1min":0.503,"5min":0.32400000000000001,"15min":0.30599999999999999},"max":{"1min":0.84399999999999997,"5min":12.662000000000001,"15min":19.440999999999999},"last":0.46400000000000002}]},{"osd":5,"last update":"Sat Sep 23 20:52:15 2023","interfaces":[{"interface":"back","average":{"1min":0.66900000000000004,"5min":1.2270000000000001,"15min":1.554},"min":{"1min":0.41799999999999998,"5min":0.38400000000000001,"15min":0.311},"max":{"1min":0.86599999999999999,"5min":8.5380000000000003,"15min":12.07},"last":0.64300000000000002},{"interface":"front","average":{"1min":0.66600000000000004,"5min":1.3380000000000001,"15min":1.601},"min":{"1min":0.51900000000000002,"5min":0.373,"15min":0.37},"max":{"1min":0.82599999999999996,"5min":10.831,"15min":19.513000000000002},"last":0.61499999999999999}]},{"osd":6,"last update":"Sat Sep 23 20:52:15 2023","interfaces":[{"interface":"back","average":{"1min":0.64400000000000002,"5min":1.3029999999999999,"15min":1.7350000000000001},"min":{"1min":0.48099999999999998,"5min":0.34799999999999998,"15min":0.33500000000000002},"max":{"1min":0.81599999999999995,"5min":10.791,"15min":19.486000000000001},"last":0.52400000000000002},{"interface":"front","average":{"1min":0.68999999999999995,"5min":1.1419999999999999,"15min":1.254},"min":{"1min":0.53000000000000003,"5min":0.40699999999999997,"15min":0.32900000000000001},"max":{"1min":0.82699999999999996,"5min":7.4180000000000001,"15min":11.747},"last":0.57599999999999996}]},{"osd":7,"last update":"Sat Sep 23 20:52:15 2023","interfaces":[{"interface":"back","average":{"1min":0.64100000000000001,"5min":1.506,"15min":1.373},"min":{"1min":0.47299999999999998,"5min":0.378,"15min":0.34699999999999998},"max":{"1min":0.88400000000000001,"5min":20.530000000000001,"15min":20.530000000000001},"last":0.628},{"interface":"front","average":{"1min":0.64300000000000002,"5min":1.4219999999999999,"15min":1.778},"min":{"1min":0.52500000000000002,"5min":0.36899999999999999,"15min":0.34899999999999998},"max":{"1min":0.82599999999999996,"5min":10.071999999999999,"15min":12.099},"last":0.60199999999999998}]}]},{"osd":3,"up_from":109,"seq":468151435638,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1081580,"kb_used_data":8932,"kb_used_omap":4,"kb_used_meta":1072635,"kb_avail":92663572,"statfs":{"total":95995035648,"available":94887497728,"internally_reserved":0,"allocated":9146368,"data_stored":5956324,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4514,"internal_metadata":1098378846},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:51:52 2023","interfaces":[{"interface":"back","average":{"1min":0.379,"5min":1.3839999999999999,"15min":1.9350000000000001},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.20899999999999999},"max":{"1min":0.623,"5min":21.792999999999999,"15min":22.32},"last":0.51600000000000001},{"interface":"front","average":{"1min":0.42599999999999999,"5min":1.175,"15min":1.986},"min":{"1min":0.25800000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":0.64800000000000002,"5min":14.845000000000001,"15min":22.805},"last":0.54000000000000004}]},{"osd":1,"last update":"Sat Sep 23 20:52:25 2023","interfaces":[{"interface":"back","average":{"1min":0.41199999999999998,"5min":1.0589999999999999,"15min":1.6359999999999999},"min":{"1min":0.27500000000000002,"5min":0.21299999999999999,"15min":0.21099999999999999},"max":{"1min":0.63800000000000001,"5min":11.298,"15min":17.529},"last":0.47999999999999998},{"interface":"front","average":{"1min":0.40999999999999998,"5min":1.0209999999999999,"15min":1.788},"min":{"1min":0.28899999999999998,"5min":0.26500000000000001,"15min":0.23699999999999999},"max":{"1min":0.61399999999999999,"5min":11.380000000000001,"15min":14.19},"last":0.45000000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:51:52 2023","interfaces":[{"interface":"back","average":{"1min":0.442,"5min":1.268,"15min":2.3319999999999999},"min":{"1min":0.29499999999999998,"5min":0.254,"15min":0.23499999999999999},"max":{"1min":0.66100000000000003,"5min":7.9210000000000003,"15min":68.206999999999994},"last":0.499},{"interface":"front","average":{"1min":0.40999999999999998,"5min":1.2909999999999999,"15min":2.077},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.245},"max":{"1min":0.63700000000000001,"5min":16.559000000000001,"15min":67.736999999999995},"last":0.433}]},{"osd":4,"last update":"Sat Sep 23 20:51:52 2023","interfaces":[{"interface":"back","average":{"1min":0.62,"5min":1.343,"15min":1.5529999999999999},"min":{"1min":0.40699999999999997,"5min":0.35399999999999998,"15min":0.30499999999999999},"max":{"1min":0.97799999999999998,"5min":9.0259999999999998,"15min":22.157},"last":0.61299999999999999},{"interface":"front","average":{"1min":0.58599999999999997,"5min":1.4810000000000001,"15min":1.883},"min":{"1min":0.39000000000000001,"5min":0.33500000000000002,"15min":0.33000000000000002},"max":{"1min":0.86099999999999999,"5min":16.529,"15min":16.529},"last":0.66700000000000004}]},{"osd":5,"last update":"Sat Sep 23 20:51:52 2023","interfaces":[{"interface":"back","average":{"1min":0.61699999999999999,"5min":1.294,"15min":1.704},"min":{"1min":0.372,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.90600000000000003,"5min":8.9849999999999994,"15min":22.289999999999999},"last":0.65700000000000003},{"interface":"front","average":{"1min":0.63900000000000001,"5min":1.206,"15min":1.6060000000000001},"min":{"1min":0.442,"5min":0.36399999999999999,"15min":0.34100000000000003},"max":{"1min":0.88700000000000001,"5min":6.1390000000000002,"15min":12.952},"last":0.67800000000000005}]},{"osd":6,"last update":"Sat Sep 23 20:51:52 2023","interfaces":[{"interface":"back","average":{"1min":0.64100000000000001,"5min":1.5209999999999999,"15min":1.8240000000000001},"min":{"1min":0.41799999999999998,"5min":0.39800000000000002,"15min":0.34300000000000003},"max":{"1min":0.93600000000000005,"5min":7.4359999999999999,"15min":12.917},"last":0.70299999999999996},{"interface":"front","average":{"1min":0.61199999999999999,"5min":1.3640000000000001,"15min":1.9339999999999999},"min":{"1min":0.436,"5min":0.36699999999999999,"15min":0.34599999999999997},"max":{"1min":0.82999999999999996,"5min":16.497,"15min":16.497},"last":0.60299999999999998}]},{"osd":7,"last update":"Sat Sep 23 20:51:52 2023","interfaces":[{"interface":"back","average":{"1min":0.61399999999999999,"5min":1.4990000000000001,"15min":2.1320000000000001},"min":{"1min":0.47899999999999998,"5min":0.42399999999999999,"15min":0.39900000000000002},"max":{"1min":0.92300000000000004,"5min":16.425000000000001,"15min":16.425000000000001},"last":0.629},{"interface":"front","average":{"1min":0.65500000000000003,"5min":1.7070000000000001,"15min":1.8680000000000001},"min":{"1min":0.496,"5min":0.45600000000000002,"15min":0.40999999999999998},"max":{"1min":0.95099999999999996,"5min":11.891999999999999,"15min":22.202999999999999},"last":0.68899999999999995}]}]},{"osd":4,"up_from":30,"seq":128849019391,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":572520,"kb_used_data":7072,"kb_used_omap":0,"kb_used_meta":565440,"kb_avail":93172632,"statfs":{"total":95995035648,"available":95408775168,"internally_reserved":0,"allocated":7241728,"data_stored":5446787,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":579010560},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:52:11 2023","interfaces":[{"interface":"back","average":{"1min":0.59499999999999997,"5min":1.8500000000000001,"15min":1.7350000000000001},"min":{"1min":0.42299999999999999,"5min":0.217,"15min":0.217},"max":{"1min":0.94199999999999995,"5min":73.272000000000006,"15min":73.272000000000006},"last":0.50800000000000001},{"interface":"front","average":{"1min":0.61499999999999999,"5min":1.508,"15min":1.7989999999999999},"min":{"1min":0.46200000000000002,"5min":0.25700000000000001,"15min":0.25},"max":{"1min":0.92900000000000005,"5min":29.754000000000001,"15min":29.754000000000001},"last":0.56799999999999995}]},{"osd":1,"last update":"Sat Sep 23 20:52:19 2023","interfaces":[{"interface":"back","average":{"1min":0.629,"5min":1.2589999999999999,"15min":1.5920000000000001},"min":{"1min":0.44400000000000001,"5min":0.26200000000000001,"15min":0.25900000000000001},"max":{"1min":0.91400000000000003,"5min":12.343,"15min":18.190999999999999},"last":0.45500000000000002},{"interface":"front","average":{"1min":0.61099999999999999,"5min":1.4770000000000001,"15min":1.514},"min":{"1min":0.38600000000000001,"5min":0.28799999999999998,"15min":0.26400000000000001},"max":{"1min":0.88400000000000001,"5min":12.231999999999999,"15min":18.173999999999999},"last":0.58199999999999996}]},{"osd":2,"last update":"Sat Sep 23 20:52:19 2023","interfaces":[{"interface":"back","average":{"1min":0.77000000000000002,"5min":1.3859999999999999,"15min":1.5469999999999999},"min":{"1min":0.38300000000000001,"5min":0.26700000000000002,"15min":0.25800000000000001},"max":{"1min":3.7170000000000001,"5min":12.151999999999999,"15min":13.051},"last":0.46700000000000003},{"interface":"front","average":{"1min":0.60199999999999998,"5min":1.24,"15min":1.7869999999999999},"min":{"1min":0.371,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.90400000000000003,"5min":12.458,"15min":12.458},"last":0.53400000000000003}]},{"osd":3,"last update":"Sat Sep 23 20:52:37 2023","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":1.363,"15min":1.6839999999999999},"min":{"1min":0.41799999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.79700000000000004,"5min":11.598000000000001,"15min":12.147},"last":0.54700000000000004},{"interface":"front","average":{"1min":0.64300000000000002,"5min":1.45,"15min":1.7729999999999999},"min":{"1min":0.36699999999999999,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.98799999999999999,"5min":12.191000000000001,"15min":17.169},"last":0.37}]},{"osd":5,"last update":"Sat Sep 23 20:52:11 2023","interfaces":[{"interface":"back","average":{"1min":0.49299999999999999,"5min":1.0309999999999999,"15min":1.171},"min":{"1min":0.32300000000000001,"5min":0.25800000000000001,"15min":0.253},"max":{"1min":0.76100000000000001,"5min":16.940000000000001,"15min":16.940000000000001},"last":0.41499999999999998},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.82199999999999995,"15min":1.163},"min":{"1min":0.32400000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.80800000000000005,"5min":4.915,"15min":8.1440000000000001},"last":0.435}]},{"osd":6,"last update":"Sat Sep 23 20:52:19 2023","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":1.3480000000000001,"15min":1.397},"min":{"1min":0.38400000000000001,"5min":0.29099999999999998,"15min":0.28199999999999997},"max":{"1min":0.79900000000000004,"5min":13.503,"15min":14.731999999999999},"last":0.59799999999999998},{"interface":"front","average":{"1min":0.57299999999999995,"5min":1.4430000000000001,"15min":1.3460000000000001},"min":{"1min":0.35899999999999999,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.85199999999999998,"5min":17.059000000000001,"15min":17.059000000000001},"last":0.52200000000000002}]},{"osd":7,"last update":"Sat Sep 23 20:52:37 2023","interfaces":[{"interface":"back","average":{"1min":0.60599999999999998,"5min":0.94399999999999995,"15min":2.556},"min":{"1min":0.434,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.74299999999999999,"5min":11.228,"15min":313.50099999999998},"last":0.48999999999999999},{"interface":"front","average":{"1min":0.64500000000000002,"5min":1.0880000000000001,"15min":2.2989999999999999},"min":{"1min":0.40100000000000002,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.90800000000000003,"5min":17.093,"15min":312.67500000000001},"last":0.48299999999999998}]}]},{"osd":5,"up_from":39,"seq":167503725052,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":636388,"kb_used_data":11420,"kb_used_omap":0,"kb_used_meta":624960,"kb_avail":93108764,"statfs":{"total":95995035648,"available":95343374336,"internally_reserved":0,"allocated":11694080,"data_stored":9899865,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":639959040},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:52:21 2023","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":1.282,"15min":1.492},"min":{"1min":0.35699999999999998,"5min":0.22600000000000001,"15min":0.22500000000000001},"max":{"1min":0.80900000000000005,"5min":12.07,"15min":15.018000000000001},"last":0.61699999999999999},{"interface":"front","average":{"1min":0.56200000000000006,"5min":1.274,"15min":1.5009999999999999},"min":{"1min":0.38500000000000001,"5min":0.29099999999999998,"15min":0.22700000000000001},"max":{"1min":0.79000000000000004,"5min":12.089,"15min":15.034000000000001},"last":0.50700000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:52:37 2023","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":1.2769999999999999,"15min":2.0129999999999999},"min":{"1min":0.36199999999999999,"5min":0.252,"15min":0.252},"max":{"1min":0.84699999999999998,"5min":10.5,"15min":61.005000000000003},"last":0.59999999999999998},{"interface":"front","average":{"1min":0.628,"5min":1.508,"15min":1.8540000000000001},"min":{"1min":0.41499999999999998,"5min":0.34000000000000002,"15min":0.27700000000000002},"max":{"1min":0.78900000000000003,"5min":10.340999999999999,"15min":59.688000000000002},"last":0.67000000000000004}]},{"osd":2,"last update":"Sat Sep 23 20:52:21 2023","interfaces":[{"interface":"back","average":{"1min":0.59499999999999997,"5min":1.373,"15min":1.736},"min":{"1min":0.39800000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.85099999999999998,"5min":7.194,"15min":12.226000000000001},"last":0.45800000000000002},{"interface":"front","average":{"1min":0.52300000000000002,"5min":1.2969999999999999,"15min":1.5289999999999999},"min":{"1min":0.33000000000000002,"5min":0.31,"15min":0.28299999999999997},"max":{"1min":0.74099999999999999,"5min":7.5599999999999996,"15min":10.565},"last":0.57699999999999996}]},{"osd":3,"last update":"Sat Sep 23 20:52:37 2023","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":1.2230000000000001,"15min":1.7250000000000001},"min":{"1min":0.40400000000000003,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.78500000000000003,"5min":8.7360000000000007,"15min":11.032},"last":0.65200000000000002},{"interface":"front","average":{"1min":0.623,"5min":1.331,"15min":1.895},"min":{"1min":0.42899999999999999,"5min":0.317,"15min":0.317},"max":{"1min":0.78700000000000003,"5min":7.4029999999999996,"15min":18.504000000000001},"last":0.55800000000000005}]},{"osd":4,"last update":"Sat Sep 23 20:52:13 2023","interfaces":[{"interface":"back","average":{"1min":0.47299999999999998,"5min":1.087,"15min":1.3109999999999999},"min":{"1min":0.316,"5min":0.27700000000000002,"15min":0.255},"max":{"1min":0.77000000000000002,"5min":8.9390000000000001,"15min":13.391999999999999},"last":0.41599999999999998},{"interface":"front","average":{"1min":0.53100000000000003,"5min":1.0449999999999999,"15min":1.288},"min":{"1min":0.35399999999999998,"5min":0.31,"15min":0.28299999999999997},"max":{"1min":0.85399999999999998,"5min":8.0030000000000001,"15min":8.0030000000000001},"last":0.70899999999999996}]},{"osd":6,"last update":"Sat Sep 23 20:52:21 2023","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":1.1379999999999999,"15min":1.486},"min":{"1min":0.41399999999999998,"5min":0.313,"15min":0.313},"max":{"1min":0.82199999999999995,"5min":8.9039999999999999,"15min":13.734},"last":0.53100000000000003},{"interface":"front","average":{"1min":0.69099999999999995,"5min":1.28,"15min":1.339},"min":{"1min":0.40899999999999997,"5min":0.34999999999999998,"15min":0.317},"max":{"1min":0.92900000000000005,"5min":9.6519999999999992,"15min":9.6519999999999992},"last":0.74399999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:52:37 2023","interfaces":[{"interface":"back","average":{"1min":0.65000000000000002,"5min":1.081,"15min":2.2589999999999999},"min":{"1min":0.41899999999999998,"5min":0.34399999999999997,"15min":0.33400000000000002},"max":{"1min":0.78000000000000003,"5min":7.899,"15min":294.70999999999998},"last":0.78000000000000003},{"interface":"front","average":{"1min":0.60799999999999998,"5min":0.97799999999999998,"15min":2.2360000000000002},"min":{"1min":0.42299999999999999,"5min":0.316,"15min":0.27900000000000003},"max":{"1min":0.78500000000000003,"5min":14.032999999999999,"15min":294.74400000000003},"last":0.64000000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4722,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1360,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":8392704,"data_stored":8388627,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8206,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7156,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3154,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:52:46.318 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T20:52:46.928 INFO:teuthology.orchestra.run.smithi018.stderr:Inferring config /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T20:52:47.135 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:52:46 smithi018 ceph-mon[184144]: from='client.48137 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:52:47.135 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:52:46 smithi018 ceph-mon[179647]: from='client.48137 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:52:47.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:52:46 smithi149 ceph-mon[170394]: from='client.48137 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:52:48.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:52:47 smithi018 ceph-mon[184144]: pgmap v1711: 103 pgs: 103 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-09-23T20:52:48.184 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:52:47 smithi018 ceph-mon[179647]: pgmap v1711: 103 pgs: 103 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-09-23T20:52:48.216 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:52:47 smithi149 ceph-mon[170394]: pgmap v1711: 103 pgs: 103 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-09-23T20:52:49.128 INFO:teuthology.orchestra.run.smithi018.stdout: 2023-09-23T20:52:49.130 INFO:teuthology.orchestra.run.smithi018.stderr:dumped all 2023-09-23T20:52:49.481 INFO:teuthology.orchestra.run.smithi018.stdout:{"pg_ready":true,"pg_map":{"version":1712,"stamp":"2023-09-23T20:52:48.405463+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":8984499,"num_objects":29,"num_object_clones":0,"num_object_copies":87,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":29,"num_whiteouts":0,"num_read":11981,"num_read_kb":26415,"num_write":1751,"num_write_kb":25419,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":283,"num_bytes_recovered":25707687,"num_keys_recovered":80,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6996,"ondisk_log_size":6996,"up":309,"acting":309,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":309,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":6171444,"kb_used_data":89396,"kb_used_omap":32,"kb_used_meta":6081951,"kb_avail":743789772,"statfs":{"total":767960285184,"available":761640726528,"internally_reserved":0,"allocated":91541504,"data_stored":71282882,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":32773,"internal_metadata":6227918843},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003156"},"pg_stats":[{"pgid":"1.18","version":"90'161","reported_seq":759,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.109526+0000","last_change":"2023-09-23T20:52:01.895435+0000","last_active":"2023-09-23T20:52:16.109526+0000","last_peered":"2023-09-23T20:52:16.109526+0000","last_clean":"2023-09-23T20:52:16.109526+0000","last_became_active":"2023-09-23T20:52:01.894990+0000","last_became_peered":"2023-09-23T20:52:01.894990+0000","last_unstale":"2023-09-23T20:52:16.109526+0000","last_undegraded":"2023-09-23T20:52:16.109526+0000","last_fullsized":"2023-09-23T20:52:16.109526+0000","mapping_epoch":688,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":689,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:19.677209+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:19.677209+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:24:12.055669+0000","scrub_duration":0.033122453000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"69'1","reported_seq":171,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.108563+0000","last_change":"2023-09-23T20:48:48.148250+0000","last_active":"2023-09-23T20:52:16.108563+0000","last_peered":"2023-09-23T20:52:16.108563+0000","last_clean":"2023-09-23T20:52:16.108563+0000","last_became_active":"2023-09-23T20:48:48.144271+0000","last_became_peered":"2023-09-23T20:48:48.144271+0000","last_unstale":"2023-09-23T20:52:16.108563+0000","last_undegraded":"2023-09-23T20:52:16.108563+0000","last_fullsized":"2023-09-23T20:52:16.108563+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:59:36.502986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":540,"reported_epoch":695,"state":"active+clean","last_fresh":"2023-09-23T20:52:20.443972+0000","last_change":"2023-09-23T20:49:02.703099+0000","last_active":"2023-09-23T20:52:20.443972+0000","last_peered":"2023-09-23T20:52:20.443972+0000","last_clean":"2023-09-23T20:52:20.443972+0000","last_became_active":"2023-09-23T20:49:02.702753+0000","last_became_peered":"2023-09-23T20:49:02.702753+0000","last_unstale":"2023-09-23T20:52:20.443972+0000","last_undegraded":"2023-09-23T20:52:20.443972+0000","last_fullsized":"2023-09-23T20:52:20.443972+0000","mapping_epoch":603,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":604,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:24:30.834235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"90'161","reported_seq":719,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.109420+0000","last_change":"2023-09-23T20:52:01.895802+0000","last_active":"2023-09-23T20:52:16.109420+0000","last_peered":"2023-09-23T20:52:16.109420+0000","last_clean":"2023-09-23T20:52:16.109420+0000","last_became_active":"2023-09-23T20:52:01.895428+0000","last_became_peered":"2023-09-23T20:52:01.895428+0000","last_unstale":"2023-09-23T20:52:16.109420+0000","last_undegraded":"2023-09-23T20:52:16.109420+0000","last_fullsized":"2023-09-23T20:52:16.109420+0000","mapping_epoch":688,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":689,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:47:33.523187+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:36:05.442288+0000","last_clean_scrub_stamp":"2023-09-23T20:47:33.523187+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:03:35.858397+0000","scrub_duration":0.013532127,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":686,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.108123+0000","last_change":"2023-09-23T20:47:09.173501+0000","last_active":"2023-09-23T20:52:16.108123+0000","last_peered":"2023-09-23T20:52:16.108123+0000","last_clean":"2023-09-23T20:52:16.108123+0000","last_became_active":"2023-09-23T20:47:09.173160+0000","last_became_peered":"2023-09-23T20:47:09.173160+0000","last_unstale":"2023-09-23T20:52:16.108123+0000","last_undegraded":"2023-09-23T20:52:16.108123+0000","last_fullsized":"2023-09-23T20:52:16.108123+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:44:36.132093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":656,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.108999+0000","last_change":"2023-09-23T20:43:46.521939+0000","last_active":"2023-09-23T20:52:16.108999+0000","last_peered":"2023-09-23T20:52:16.108999+0000","last_clean":"2023-09-23T20:52:16.108999+0000","last_became_active":"2023-09-23T20:43:46.521185+0000","last_became_peered":"2023-09-23T20:43:46.521185+0000","last_unstale":"2023-09-23T20:52:16.108999+0000","last_undegraded":"2023-09-23T20:52:16.108999+0000","last_fullsized":"2023-09-23T20:52:16.108999+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:59:25.045627+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"90'161","reported_seq":582,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.108413+0000","last_change":"2023-09-23T20:47:31.475001+0000","last_active":"2023-09-23T20:52:16.108413+0000","last_peered":"2023-09-23T20:52:16.108413+0000","last_clean":"2023-09-23T20:52:16.108413+0000","last_became_active":"2023-09-23T20:47:11.617942+0000","last_became_peered":"2023-09-23T20:47:11.617942+0000","last_unstale":"2023-09-23T20:52:16.108413+0000","last_undegraded":"2023-09-23T20:52:16.108413+0000","last_fullsized":"2023-09-23T20:52:16.108413+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:47:31.474914+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:47:31.474914+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:52:05.169219+0000","scrub_duration":0.0026262870000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":493,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:17.946487+0000","last_change":"2023-09-23T20:37:43.584414+0000","last_active":"2023-09-23T20:52:17.946487+0000","last_peered":"2023-09-23T20:52:17.946487+0000","last_clean":"2023-09-23T20:52:17.946487+0000","last_became_active":"2023-09-23T20:37:43.584116+0000","last_became_peered":"2023-09-23T20:37:43.584116+0000","last_unstale":"2023-09-23T20:52:17.946487+0000","last_undegraded":"2023-09-23T20:52:17.946487+0000","last_fullsized":"2023-09-23T20:52:17.946487+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:17:54.258172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":329,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.110547+0000","last_change":"2023-09-23T20:51:21.746193+0000","last_active":"2023-09-23T20:52:16.110547+0000","last_peered":"2023-09-23T20:52:16.110547+0000","last_clean":"2023-09-23T20:52:16.110547+0000","last_became_active":"2023-09-23T20:51:21.745936+0000","last_became_peered":"2023-09-23T20:51:21.745936+0000","last_unstale":"2023-09-23T20:52:16.110547+0000","last_undegraded":"2023-09-23T20:52:16.110547+0000","last_fullsized":"2023-09-23T20:52:16.110547+0000","mapping_epoch":673,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":674,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:06:49.466364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"90'161","reported_seq":947,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:15.936414+0000","last_change":"2023-09-23T20:51:24.744039+0000","last_active":"2023-09-23T20:52:15.936414+0000","last_peered":"2023-09-23T20:52:15.936414+0000","last_clean":"2023-09-23T20:52:15.936414+0000","last_became_active":"2023-09-23T20:51:24.743653+0000","last_became_peered":"2023-09-23T20:51:24.743653+0000","last_unstale":"2023-09-23T20:52:15.936414+0000","last_undegraded":"2023-09-23T20:52:15.936414+0000","last_fullsized":"2023-09-23T20:52:15.936414+0000","mapping_epoch":676,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":677,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:03.411025+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:03.411025+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:57:50.218399+0000","scrub_duration":0.001802922,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":706,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.109447+0000","last_change":"2023-09-23T20:51:21.746692+0000","last_active":"2023-09-23T20:52:16.109447+0000","last_peered":"2023-09-23T20:52:16.109447+0000","last_clean":"2023-09-23T20:52:16.109447+0000","last_became_active":"2023-09-23T20:51:21.745658+0000","last_became_peered":"2023-09-23T20:51:21.745658+0000","last_unstale":"2023-09-23T20:52:16.109447+0000","last_undegraded":"2023-09-23T20:52:16.109447+0000","last_fullsized":"2023-09-23T20:52:16.109447+0000","mapping_epoch":673,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":674,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:49:28.118695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":138,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.108266+0000","last_change":"2023-09-23T20:51:02.816162+0000","last_active":"2023-09-23T20:52:16.108266+0000","last_peered":"2023-09-23T20:52:16.108266+0000","last_clean":"2023-09-23T20:52:16.108266+0000","last_became_active":"2023-09-23T20:51:02.815754+0000","last_became_peered":"2023-09-23T20:51:02.815754+0000","last_unstale":"2023-09-23T20:52:16.108266+0000","last_undegraded":"2023-09-23T20:52:16.108266+0000","last_fullsized":"2023-09-23T20:52:16.108266+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:29:00.532296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"90'161","reported_seq":698,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.108921+0000","last_change":"2023-09-23T20:49:36.717905+0000","last_active":"2023-09-23T20:52:16.108921+0000","last_peered":"2023-09-23T20:52:16.108921+0000","last_clean":"2023-09-23T20:52:16.108921+0000","last_became_active":"2023-09-23T20:48:51.169314+0000","last_became_peered":"2023-09-23T20:48:51.169314+0000","last_unstale":"2023-09-23T20:52:16.108921+0000","last_undegraded":"2023-09-23T20:52:16.108921+0000","last_fullsized":"2023-09-23T20:52:16.108921+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:49:36.717737+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:49:36.717737+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:00:49.213712+0000","scrub_duration":0.0049746089999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":540,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.108962+0000","last_change":"2023-09-23T20:52:01.897383+0000","last_active":"2023-09-23T20:52:16.108962+0000","last_peered":"2023-09-23T20:52:16.108962+0000","last_clean":"2023-09-23T20:52:16.108962+0000","last_became_active":"2023-09-23T20:52:01.897046+0000","last_became_peered":"2023-09-23T20:52:01.897046+0000","last_unstale":"2023-09-23T20:52:16.108962+0000","last_undegraded":"2023-09-23T20:52:16.108962+0000","last_fullsized":"2023-09-23T20:52:16.108962+0000","mapping_epoch":688,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":689,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:41:16.704651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":524,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.109710+0000","last_change":"2023-09-23T20:38:02.542034+0000","last_active":"2023-09-23T20:52:16.109710+0000","last_peered":"2023-09-23T20:52:16.109710+0000","last_clean":"2023-09-23T20:52:16.109710+0000","last_became_active":"2023-09-23T20:38:02.541575+0000","last_became_peered":"2023-09-23T20:38:02.541575+0000","last_unstale":"2023-09-23T20:52:16.109710+0000","last_undegraded":"2023-09-23T20:52:16.109710+0000","last_fullsized":"2023-09-23T20:52:16.109710+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:50:54.591529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"90'161","reported_seq":840,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.109105+0000","last_change":"2023-09-23T20:52:01.895097+0000","last_active":"2023-09-23T20:52:16.109105+0000","last_peered":"2023-09-23T20:52:16.109105+0000","last_clean":"2023-09-23T20:52:16.109105+0000","last_became_active":"2023-09-23T20:52:01.894615+0000","last_became_peered":"2023-09-23T20:52:01.894615+0000","last_unstale":"2023-09-23T20:52:16.109105+0000","last_undegraded":"2023-09-23T20:52:16.109105+0000","last_fullsized":"2023-09-23T20:52:16.109105+0000","mapping_epoch":688,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":689,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:14.669831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:14.669831+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:46:31.699442+0000","scrub_duration":0.0083441460000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":754,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:15.936605+0000","last_change":"2023-09-23T20:47:10.837423+0000","last_active":"2023-09-23T20:52:15.936605+0000","last_peered":"2023-09-23T20:52:15.936605+0000","last_clean":"2023-09-23T20:52:15.936605+0000","last_became_active":"2023-09-23T20:47:09.204324+0000","last_became_peered":"2023-09-23T20:47:09.204324+0000","last_unstale":"2023-09-23T20:52:15.936605+0000","last_undegraded":"2023-09-23T20:52:15.936605+0000","last_fullsized":"2023-09-23T20:52:15.936605+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":554,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:34:19.108914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":540,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":230,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.108081+0000","last_change":"2023-09-23T20:48:48.145175+0000","last_active":"2023-09-23T20:52:16.108081+0000","last_peered":"2023-09-23T20:52:16.108081+0000","last_clean":"2023-09-23T20:52:16.108081+0000","last_became_active":"2023-09-23T20:48:48.144841+0000","last_became_peered":"2023-09-23T20:48:48.144841+0000","last_unstale":"2023-09-23T20:52:16.108081+0000","last_undegraded":"2023-09-23T20:52:16.108081+0000","last_fullsized":"2023-09-23T20:52:16.108081+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:27:40.802917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"90'161","reported_seq":824,"reported_epoch":695,"state":"active+clean","last_fresh":"2023-09-23T20:52:19.947013+0000","last_change":"2023-09-23T20:36:02.548387+0000","last_active":"2023-09-23T20:52:19.947013+0000","last_peered":"2023-09-23T20:52:19.947013+0000","last_clean":"2023-09-23T20:52:19.947013+0000","last_became_active":"2023-09-23T20:35:26.644787+0000","last_became_peered":"2023-09-23T20:35:26.644787+0000","last_unstale":"2023-09-23T20:52:19.947013+0000","last_undegraded":"2023-09-23T20:52:19.947013+0000","last_fullsized":"2023-09-23T20:52:19.947013+0000","mapping_epoch":362,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":363,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:02.548285+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:02.548285+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:37:15.216553+0000","scrub_duration":0.0015514459999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"90'22","reported_seq":161,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.109460+0000","last_change":"2023-09-23T20:47:09.340979+0000","last_active":"2023-09-23T20:52:16.109460+0000","last_peered":"2023-09-23T20:52:16.109460+0000","last_clean":"2023-09-23T20:52:16.109460+0000","last_became_active":"2023-09-23T20:47:09.231222+0000","last_became_peered":"2023-09-23T20:47:09.231222+0000","last_unstale":"2023-09-23T20:52:16.109460+0000","last_undegraded":"2023-09-23T20:52:16.109460+0000","last_fullsized":"2023-09-23T20:52:16.109460+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:07:50.620006+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":151,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.109708+0000","last_change":"2023-09-23T20:47:09.186315+0000","last_active":"2023-09-23T20:52:16.109708+0000","last_peered":"2023-09-23T20:52:16.109708+0000","last_clean":"2023-09-23T20:52:16.109708+0000","last_became_active":"2023-09-23T20:47:09.184580+0000","last_became_peered":"2023-09-23T20:47:09.184580+0000","last_unstale":"2023-09-23T20:52:16.109708+0000","last_undegraded":"2023-09-23T20:52:16.109708+0000","last_fullsized":"2023-09-23T20:52:16.109708+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:44:35.553948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"90'161","reported_seq":985,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.107971+0000","last_change":"2023-09-23T20:51:24.741699+0000","last_active":"2023-09-23T20:52:16.107971+0000","last_peered":"2023-09-23T20:52:16.107971+0000","last_clean":"2023-09-23T20:52:16.107971+0000","last_became_active":"2023-09-23T20:51:24.740931+0000","last_became_peered":"2023-09-23T20:51:24.740931+0000","last_unstale":"2023-09-23T20:52:16.107971+0000","last_undegraded":"2023-09-23T20:52:16.107971+0000","last_fullsized":"2023-09-23T20:52:16.107971+0000","mapping_epoch":676,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":677,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:43.578666+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:43.578666+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:39:14.933750+0000","scrub_duration":0.0021890379999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":785,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:15.936818+0000","last_change":"2023-09-23T20:48:48.193666+0000","last_active":"2023-09-23T20:52:15.936818+0000","last_peered":"2023-09-23T20:52:15.936818+0000","last_clean":"2023-09-23T20:52:15.936818+0000","last_became_active":"2023-09-23T20:48:48.152081+0000","last_became_peered":"2023-09-23T20:48:48.152081+0000","last_unstale":"2023-09-23T20:52:15.936818+0000","last_undegraded":"2023-09-23T20:52:15.936818+0000","last_fullsized":"2023-09-23T20:52:15.936818+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:00:19.508124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":272,"num_keys_recovered":80,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":453,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.110857+0000","last_change":"2023-09-23T20:49:48.119365+0000","last_active":"2023-09-23T20:52:16.110857+0000","last_peered":"2023-09-23T20:52:16.110857+0000","last_clean":"2023-09-23T20:52:16.110857+0000","last_became_active":"2023-09-23T20:49:48.119009+0000","last_became_peered":"2023-09-23T20:49:48.119009+0000","last_unstale":"2023-09-23T20:52:16.110857+0000","last_undegraded":"2023-09-23T20:52:16.110857+0000","last_fullsized":"2023-09-23T20:52:16.110857+0000","mapping_epoch":636,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":637,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:02:26.878542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"90'161","reported_seq":997,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.109346+0000","last_change":"2023-09-23T20:52:01.895146+0000","last_active":"2023-09-23T20:52:16.109346+0000","last_peered":"2023-09-23T20:52:16.109346+0000","last_clean":"2023-09-23T20:52:16.109346+0000","last_became_active":"2023-09-23T20:52:01.894768+0000","last_became_peered":"2023-09-23T20:52:01.894768+0000","last_unstale":"2023-09-23T20:52:16.109346+0000","last_undegraded":"2023-09-23T20:52:16.109346+0000","last_fullsized":"2023-09-23T20:52:16.109346+0000","mapping_epoch":688,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":689,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:20.041690+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:20.041690+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:58:44.945091+0000","scrub_duration":0.0026180330000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":653,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.107972+0000","last_change":"2023-09-23T20:43:46.521294+0000","last_active":"2023-09-23T20:52:16.107972+0000","last_peered":"2023-09-23T20:52:16.107972+0000","last_clean":"2023-09-23T20:52:16.107972+0000","last_became_active":"2023-09-23T20:43:46.521001+0000","last_became_peered":"2023-09-23T20:43:46.521001+0000","last_unstale":"2023-09-23T20:52:16.107972+0000","last_undegraded":"2023-09-23T20:52:16.107972+0000","last_fullsized":"2023-09-23T20:52:16.107972+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:01:35.126716+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":481,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.108740+0000","last_change":"2023-09-23T20:52:01.899144+0000","last_active":"2023-09-23T20:52:16.108740+0000","last_peered":"2023-09-23T20:52:16.108740+0000","last_clean":"2023-09-23T20:52:16.108740+0000","last_became_active":"2023-09-23T20:52:01.898813+0000","last_became_peered":"2023-09-23T20:52:01.898813+0000","last_unstale":"2023-09-23T20:52:16.108740+0000","last_undegraded":"2023-09-23T20:52:16.108740+0000","last_fullsized":"2023-09-23T20:52:16.108740+0000","mapping_epoch":688,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":689,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:44:59.907884+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"90'161","reported_seq":1019,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.110363+0000","last_change":"2023-09-23T20:47:11.619371+0000","last_active":"2023-09-23T20:52:16.110363+0000","last_peered":"2023-09-23T20:52:16.110363+0000","last_clean":"2023-09-23T20:52:16.110363+0000","last_became_active":"2023-09-23T20:47:11.619006+0000","last_became_peered":"2023-09-23T20:47:11.619006+0000","last_unstale":"2023-09-23T20:52:16.110363+0000","last_undegraded":"2023-09-23T20:52:16.110363+0000","last_fullsized":"2023-09-23T20:52:16.110363+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:42.544308+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:42.544308+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:47:24.414807+0000","scrub_duration":0.0029958179999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":718,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:15.936269+0000","last_change":"2023-09-23T20:47:09.176613+0000","last_active":"2023-09-23T20:52:15.936269+0000","last_peered":"2023-09-23T20:52:15.936269+0000","last_clean":"2023-09-23T20:52:15.936269+0000","last_became_active":"2023-09-23T20:47:09.176187+0000","last_became_peered":"2023-09-23T20:47:09.176187+0000","last_unstale":"2023-09-23T20:52:15.936269+0000","last_undegraded":"2023-09-23T20:52:15.936269+0000","last_fullsized":"2023-09-23T20:52:15.936269+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:23:01.590989+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":662,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.109077+0000","last_change":"2023-09-23T20:43:46.521735+0000","last_active":"2023-09-23T20:52:16.109077+0000","last_peered":"2023-09-23T20:52:16.109077+0000","last_clean":"2023-09-23T20:52:16.109077+0000","last_became_active":"2023-09-23T20:43:46.520762+0000","last_became_peered":"2023-09-23T20:43:46.520762+0000","last_unstale":"2023-09-23T20:52:16.109077+0000","last_undegraded":"2023-09-23T20:52:16.109077+0000","last_fullsized":"2023-09-23T20:52:16.109077+0000","mapping_epoch":487,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":488,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:26:48.841930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"90'161","reported_seq":798,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.109715+0000","last_change":"2023-09-23T20:52:01.896791+0000","last_active":"2023-09-23T20:52:16.109715+0000","last_peered":"2023-09-23T20:52:16.109715+0000","last_clean":"2023-09-23T20:52:16.109715+0000","last_became_active":"2023-09-23T20:52:01.896285+0000","last_became_peered":"2023-09-23T20:52:01.896285+0000","last_unstale":"2023-09-23T20:52:16.109715+0000","last_undegraded":"2023-09-23T20:52:16.109715+0000","last_fullsized":"2023-09-23T20:52:16.109715+0000","mapping_epoch":688,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":689,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:18.952693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:18.952693+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:48.278498+0000","scrub_duration":0.001699365,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":409,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.110153+0000","last_change":"2023-09-23T20:51:21.763704+0000","last_active":"2023-09-23T20:52:16.110153+0000","last_peered":"2023-09-23T20:52:16.110153+0000","last_clean":"2023-09-23T20:52:16.110153+0000","last_became_active":"2023-09-23T20:51:21.745455+0000","last_became_peered":"2023-09-23T20:51:21.745455+0000","last_unstale":"2023-09-23T20:52:16.110153+0000","last_undegraded":"2023-09-23T20:52:16.110153+0000","last_fullsized":"2023-09-23T20:52:16.110153+0000","mapping_epoch":673,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":674,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:11:47.478937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":493,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.109021+0000","last_change":"2023-09-23T20:37:43.571083+0000","last_active":"2023-09-23T20:52:16.109021+0000","last_peered":"2023-09-23T20:52:16.109021+0000","last_clean":"2023-09-23T20:52:16.109021+0000","last_became_active":"2023-09-23T20:37:43.570306+0000","last_became_peered":"2023-09-23T20:37:43.570306+0000","last_unstale":"2023-09-23T20:52:16.109021+0000","last_undegraded":"2023-09-23T20:52:16.109021+0000","last_fullsized":"2023-09-23T20:52:16.109021+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:42:02.396718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.d","version":"90'161","reported_seq":1027,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:15.936503+0000","last_change":"2023-09-23T20:47:11.619510+0000","last_active":"2023-09-23T20:52:15.936503+0000","last_peered":"2023-09-23T20:52:15.936503+0000","last_clean":"2023-09-23T20:52:15.936503+0000","last_became_active":"2023-09-23T20:47:11.619154+0000","last_became_peered":"2023-09-23T20:47:11.619154+0000","last_unstale":"2023-09-23T20:52:15.936503+0000","last_undegraded":"2023-09-23T20:52:15.936503+0000","last_fullsized":"2023-09-23T20:52:15.936503+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:34:59.496375+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:34:59.496375+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:40:52.712777+0000","scrub_duration":0.0017843189999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":726,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:15.936996+0000","last_change":"2023-09-23T20:47:09.207851+0000","last_active":"2023-09-23T20:52:15.936996+0000","last_peered":"2023-09-23T20:52:15.936996+0000","last_clean":"2023-09-23T20:52:15.936996+0000","last_became_active":"2023-09-23T20:47:09.175836+0000","last_became_peered":"2023-09-23T20:47:09.175836+0000","last_unstale":"2023-09-23T20:52:15.936996+0000","last_undegraded":"2023-09-23T20:52:15.936996+0000","last_fullsized":"2023-09-23T20:52:15.936996+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:43:13.091809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":518,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.109675+0000","last_change":"2023-09-23T20:52:01.896148+0000","last_active":"2023-09-23T20:52:16.109675+0000","last_peered":"2023-09-23T20:52:16.109675+0000","last_clean":"2023-09-23T20:52:16.109675+0000","last_became_active":"2023-09-23T20:52:01.895458+0000","last_became_peered":"2023-09-23T20:52:01.895458+0000","last_unstale":"2023-09-23T20:52:16.109675+0000","last_undegraded":"2023-09-23T20:52:16.109675+0000","last_fullsized":"2023-09-23T20:52:16.109675+0000","mapping_epoch":688,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":689,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:22:05.761374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"90'161","reported_seq":959,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.110927+0000","last_change":"2023-09-23T20:41:09.506481+0000","last_active":"2023-09-23T20:52:16.110927+0000","last_peered":"2023-09-23T20:52:16.110927+0000","last_clean":"2023-09-23T20:52:16.110927+0000","last_became_active":"2023-09-23T20:38:28.442229+0000","last_became_peered":"2023-09-23T20:38:28.442229+0000","last_unstale":"2023-09-23T20:52:16.110927+0000","last_undegraded":"2023-09-23T20:52:16.110927+0000","last_fullsized":"2023-09-23T20:52:16.110927+0000","mapping_epoch":411,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":412,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:41:09.506338+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:41:09.506338+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:19:46.265011+0000","scrub_duration":0.01112002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":764,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:15.936703+0000","last_change":"2023-09-23T20:47:10.848436+0000","last_active":"2023-09-23T20:52:15.936703+0000","last_peered":"2023-09-23T20:52:15.936703+0000","last_clean":"2023-09-23T20:52:15.936703+0000","last_became_active":"2023-09-23T20:47:09.196769+0000","last_became_peered":"2023-09-23T20:47:09.196769+0000","last_unstale":"2023-09-23T20:52:15.936703+0000","last_undegraded":"2023-09-23T20:52:15.936703+0000","last_fullsized":"2023-09-23T20:52:15.936703+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":554,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:59:38.105009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":161,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.108924+0000","last_change":"2023-09-23T20:48:23.915817+0000","last_active":"2023-09-23T20:52:16.108924+0000","last_peered":"2023-09-23T20:52:16.108924+0000","last_clean":"2023-09-23T20:52:16.108924+0000","last_became_active":"2023-09-23T20:48:23.915306+0000","last_became_peered":"2023-09-23T20:48:23.915306+0000","last_unstale":"2023-09-23T20:52:16.108924+0000","last_undegraded":"2023-09-23T20:52:16.108924+0000","last_fullsized":"2023-09-23T20:52:16.108924+0000","mapping_epoch":577,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":578,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:55:45.691450+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"90'161","reported_seq":929,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.110755+0000","last_change":"2023-09-23T20:51:24.745859+0000","last_active":"2023-09-23T20:52:16.110755+0000","last_peered":"2023-09-23T20:52:16.110755+0000","last_clean":"2023-09-23T20:52:16.110755+0000","last_became_active":"2023-09-23T20:51:24.745454+0000","last_became_peered":"2023-09-23T20:51:24.745454+0000","last_unstale":"2023-09-23T20:52:16.110755+0000","last_undegraded":"2023-09-23T20:52:16.110755+0000","last_fullsized":"2023-09-23T20:52:16.110755+0000","mapping_epoch":676,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":677,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:46.217489+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:46.217489+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:39:14.173319+0000","scrub_duration":0.001456429,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":751,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:15.936499+0000","last_change":"2023-09-23T20:51:21.757079+0000","last_active":"2023-09-23T20:52:15.936499+0000","last_peered":"2023-09-23T20:52:15.936499+0000","last_clean":"2023-09-23T20:52:15.936499+0000","last_became_active":"2023-09-23T20:51:21.747892+0000","last_became_peered":"2023-09-23T20:51:21.747892+0000","last_unstale":"2023-09-23T20:52:15.936499+0000","last_undegraded":"2023-09-23T20:52:15.936499+0000","last_fullsized":"2023-09-23T20:52:15.936499+0000","mapping_epoch":673,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":674,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:31:38.160456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":560,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":309,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.110682+0000","last_change":"2023-09-23T20:49:02.702301+0000","last_active":"2023-09-23T20:52:16.110682+0000","last_peered":"2023-09-23T20:52:16.110682+0000","last_clean":"2023-09-23T20:52:16.110682+0000","last_became_active":"2023-09-23T20:49:02.701986+0000","last_became_peered":"2023-09-23T20:49:02.701986+0000","last_unstale":"2023-09-23T20:52:16.110682+0000","last_undegraded":"2023-09-23T20:52:16.110682+0000","last_fullsized":"2023-09-23T20:52:16.110682+0000","mapping_epoch":603,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":604,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:20:01.603267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.e","version":"90'161","reported_seq":724,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.109557+0000","last_change":"2023-09-23T20:47:37.608426+0000","last_active":"2023-09-23T20:52:16.109557+0000","last_peered":"2023-09-23T20:52:16.109557+0000","last_clean":"2023-09-23T20:52:16.109557+0000","last_became_active":"2023-09-23T20:47:11.617187+0000","last_became_peered":"2023-09-23T20:47:11.617187+0000","last_unstale":"2023-09-23T20:52:16.109557+0000","last_undegraded":"2023-09-23T20:52:16.109557+0000","last_fullsized":"2023-09-23T20:52:16.109557+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:47:37.608309+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:47:37.608309+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:22:25.482297+0000","scrub_duration":0.011600924,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":664,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.108480+0000","last_change":"2023-09-23T20:52:01.897145+0000","last_active":"2023-09-23T20:52:16.108480+0000","last_peered":"2023-09-23T20:52:16.108480+0000","last_clean":"2023-09-23T20:52:16.108480+0000","last_became_active":"2023-09-23T20:52:01.896852+0000","last_became_peered":"2023-09-23T20:52:01.896852+0000","last_unstale":"2023-09-23T20:52:16.108480+0000","last_undegraded":"2023-09-23T20:52:16.108480+0000","last_fullsized":"2023-09-23T20:52:16.108480+0000","mapping_epoch":688,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":689,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:33:24.799922+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":690,"reported_epoch":695,"state":"active+clean","last_fresh":"2023-09-23T20:52:21.228664+0000","last_change":"2023-09-23T20:47:09.197427+0000","last_active":"2023-09-23T20:52:21.228664+0000","last_peered":"2023-09-23T20:52:21.228664+0000","last_clean":"2023-09-23T20:52:21.228664+0000","last_became_active":"2023-09-23T20:47:09.196542+0000","last_became_peered":"2023-09-23T20:47:09.196542+0000","last_unstale":"2023-09-23T20:52:21.228664+0000","last_undegraded":"2023-09-23T20:52:21.228664+0000","last_fullsized":"2023-09-23T20:52:21.228664+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:28:38.926202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"90'161","reported_seq":1140,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:15.936162+0000","last_change":"2023-09-23T20:48:51.169474+0000","last_active":"2023-09-23T20:52:15.936162+0000","last_peered":"2023-09-23T20:52:15.936162+0000","last_clean":"2023-09-23T20:52:15.936162+0000","last_became_active":"2023-09-23T20:48:51.169076+0000","last_became_peered":"2023-09-23T20:48:51.169076+0000","last_unstale":"2023-09-23T20:52:15.936162+0000","last_undegraded":"2023-09-23T20:52:15.936162+0000","last_fullsized":"2023-09-23T20:52:15.936162+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:35:02.523203+0000","last_deep_scrub":"90'161","last_deep_scrub_stamp":"2023-09-23T20:35:02.523203+0000","last_clean_scrub_stamp":"2023-09-23T20:35:02.523203+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:25:53.741403+0000","scrub_duration":0.0085927989999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":545,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.108161+0000","last_change":"2023-09-23T20:47:09.240200+0000","last_active":"2023-09-23T20:52:16.108161+0000","last_peered":"2023-09-23T20:52:16.108161+0000","last_clean":"2023-09-23T20:52:16.108161+0000","last_became_active":"2023-09-23T20:47:09.195204+0000","last_became_peered":"2023-09-23T20:47:09.195204+0000","last_unstale":"2023-09-23T20:52:16.108161+0000","last_undegraded":"2023-09-23T20:52:16.108161+0000","last_fullsized":"2023-09-23T20:52:16.108161+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:03.328250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":151,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.109382+0000","last_change":"2023-09-23T20:47:09.231714+0000","last_active":"2023-09-23T20:52:16.109382+0000","last_peered":"2023-09-23T20:52:16.109382+0000","last_clean":"2023-09-23T20:52:16.109382+0000","last_became_active":"2023-09-23T20:47:09.230918+0000","last_became_peered":"2023-09-23T20:47:09.230918+0000","last_unstale":"2023-09-23T20:52:16.109382+0000","last_undegraded":"2023-09-23T20:52:16.109382+0000","last_fullsized":"2023-09-23T20:52:16.109382+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:38:24.583656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"90'161","reported_seq":600,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.109438+0000","last_change":"2023-09-23T20:52:01.896147+0000","last_active":"2023-09-23T20:52:16.109438+0000","last_peered":"2023-09-23T20:52:16.109438+0000","last_clean":"2023-09-23T20:52:16.109438+0000","last_became_active":"2023-09-23T20:52:01.895805+0000","last_became_peered":"2023-09-23T20:52:01.895805+0000","last_unstale":"2023-09-23T20:52:16.109438+0000","last_undegraded":"2023-09-23T20:52:16.109438+0000","last_fullsized":"2023-09-23T20:52:16.109438+0000","mapping_epoch":688,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":689,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:50:03.881826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:50:03.881826+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:24:36.911235+0000","scrub_duration":0.003245258,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":319,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.110746+0000","last_change":"2023-09-23T20:47:09.178919+0000","last_active":"2023-09-23T20:52:16.110746+0000","last_peered":"2023-09-23T20:52:16.110746+0000","last_clean":"2023-09-23T20:52:16.110746+0000","last_became_active":"2023-09-23T20:47:09.178607+0000","last_became_peered":"2023-09-23T20:47:09.178607+0000","last_unstale":"2023-09-23T20:52:16.110746+0000","last_undegraded":"2023-09-23T20:52:16.110746+0000","last_fullsized":"2023-09-23T20:52:16.110746+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:23:45.196935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":171,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.109241+0000","last_change":"2023-09-23T20:48:48.148596+0000","last_active":"2023-09-23T20:52:16.109241+0000","last_peered":"2023-09-23T20:52:16.109241+0000","last_clean":"2023-09-23T20:52:16.109241+0000","last_became_active":"2023-09-23T20:48:48.148211+0000","last_became_peered":"2023-09-23T20:48:48.148211+0000","last_unstale":"2023-09-23T20:52:16.109241+0000","last_undegraded":"2023-09-23T20:52:16.109241+0000","last_fullsized":"2023-09-23T20:52:16.109241+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:21:50.825822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"90'161","reported_seq":1036,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:18.719713+0000","last_change":"2023-09-23T20:51:24.741541+0000","last_active":"2023-09-23T20:52:18.719713+0000","last_peered":"2023-09-23T20:52:18.719713+0000","last_clean":"2023-09-23T20:52:18.719713+0000","last_became_active":"2023-09-23T20:51:24.741161+0000","last_became_peered":"2023-09-23T20:51:24.741161+0000","last_unstale":"2023-09-23T20:52:18.719713+0000","last_undegraded":"2023-09-23T20:52:18.719713+0000","last_fullsized":"2023-09-23T20:52:18.719713+0000","mapping_epoch":676,"log_start":"0'0","ondisk_log_start":"0'0","created":226,"last_epoch_clean":677,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:05.595042+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:05.595042+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:34:00.584917+0000","scrub_duration":0.007271675,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":171,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.108792+0000","last_change":"2023-09-23T20:48:48.153145+0000","last_active":"2023-09-23T20:52:16.108792+0000","last_peered":"2023-09-23T20:52:16.108792+0000","last_clean":"2023-09-23T20:52:16.108792+0000","last_became_active":"2023-09-23T20:48:48.152858+0000","last_became_peered":"2023-09-23T20:48:48.152858+0000","last_unstale":"2023-09-23T20:52:16.108792+0000","last_undegraded":"2023-09-23T20:52:16.108792+0000","last_fullsized":"2023-09-23T20:52:16.108792+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:23:13.454288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":151,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.109633+0000","last_change":"2023-09-23T20:47:09.186271+0000","last_active":"2023-09-23T20:52:16.109633+0000","last_peered":"2023-09-23T20:52:16.109633+0000","last_clean":"2023-09-23T20:52:16.109633+0000","last_became_active":"2023-09-23T20:47:09.184357+0000","last_became_peered":"2023-09-23T20:47:09.184357+0000","last_unstale":"2023-09-23T20:52:16.109633+0000","last_undegraded":"2023-09-23T20:52:16.109633+0000","last_fullsized":"2023-09-23T20:52:16.109633+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:48:26.339811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"90'161","reported_seq":604,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.109868+0000","last_change":"2023-09-23T20:52:01.894940+0000","last_active":"2023-09-23T20:52:16.109868+0000","last_peered":"2023-09-23T20:52:16.109868+0000","last_clean":"2023-09-23T20:52:16.109868+0000","last_became_active":"2023-09-23T20:52:01.894428+0000","last_became_peered":"2023-09-23T20:52:01.894428+0000","last_unstale":"2023-09-23T20:52:16.109868+0000","last_undegraded":"2023-09-23T20:52:16.109868+0000","last_fullsized":"2023-09-23T20:52:16.109868+0000","mapping_epoch":688,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":689,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:49:37.762492+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:49:37.762492+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:01:46.023633+0000","scrub_duration":0.0031485979999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":493,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.108437+0000","last_change":"2023-09-23T20:37:43.570610+0000","last_active":"2023-09-23T20:52:16.108437+0000","last_peered":"2023-09-23T20:52:16.108437+0000","last_clean":"2023-09-23T20:52:16.108437+0000","last_became_active":"2023-09-23T20:37:43.570216+0000","last_became_peered":"2023-09-23T20:37:43.570216+0000","last_unstale":"2023-09-23T20:52:16.108437+0000","last_undegraded":"2023-09-23T20:52:16.108437+0000","last_fullsized":"2023-09-23T20:52:16.108437+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:39:26.555329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":542,"reported_epoch":695,"state":"active+clean","last_fresh":"2023-09-23T20:52:19.947394+0000","last_change":"2023-09-23T20:47:09.172573+0000","last_active":"2023-09-23T20:52:19.947394+0000","last_peered":"2023-09-23T20:52:19.947394+0000","last_clean":"2023-09-23T20:52:19.947394+0000","last_became_active":"2023-09-23T20:47:09.172298+0000","last_became_peered":"2023-09-23T20:47:09.172298+0000","last_unstale":"2023-09-23T20:52:19.947394+0000","last_undegraded":"2023-09-23T20:52:19.947394+0000","last_fullsized":"2023-09-23T20:52:19.947394+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:22:09.361284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"90'161","reported_seq":882,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.110680+0000","last_change":"2023-09-23T20:41:07.424609+0000","last_active":"2023-09-23T20:52:16.110680+0000","last_peered":"2023-09-23T20:52:16.110680+0000","last_clean":"2023-09-23T20:52:16.110680+0000","last_became_active":"2023-09-23T20:38:05.622261+0000","last_became_peered":"2023-09-23T20:38:05.622261+0000","last_unstale":"2023-09-23T20:52:16.110680+0000","last_undegraded":"2023-09-23T20:52:16.110680+0000","last_fullsized":"2023-09-23T20:52:16.110680+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:41:07.424492+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:41:07.424492+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:45:21.689123+0000","scrub_duration":0.0018960959999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"664'242","reported_seq":1446,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.108529+0000","last_change":"2023-09-23T20:50:39.737511+0000","last_active":"2023-09-23T20:52:16.108529+0000","last_peered":"2023-09-23T20:52:16.108529+0000","last_clean":"2023-09-23T20:52:16.108529+0000","last_became_active":"2023-09-23T20:29:45.520546+0000","last_became_peered":"2023-09-23T20:29:45.520546+0000","last_unstale":"2023-09-23T20:52:16.108529+0000","last_undegraded":"2023-09-23T20:52:16.108529+0000","last_fullsized":"2023-09-23T20:52:16.108529+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":242,"log_dups_size":0,"ondisk_log_size":242,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:58:56.085777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0012974410000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1152,"num_read_kb":4628,"num_write":256,"num_write_kb":3463,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":515,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.108808+0000","last_change":"2023-09-23T20:52:01.897961+0000","last_active":"2023-09-23T20:52:16.108808+0000","last_peered":"2023-09-23T20:52:16.108808+0000","last_clean":"2023-09-23T20:52:16.108808+0000","last_became_active":"2023-09-23T20:52:01.897687+0000","last_became_peered":"2023-09-23T20:52:01.897687+0000","last_unstale":"2023-09-23T20:52:16.108808+0000","last_undegraded":"2023-09-23T20:52:16.108808+0000","last_fullsized":"2023-09-23T20:52:16.108808+0000","mapping_epoch":688,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":689,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:48:27.254894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":692,"reported_epoch":695,"state":"active+clean","last_fresh":"2023-09-23T20:52:19.946786+0000","last_change":"2023-09-23T20:51:21.745692+0000","last_active":"2023-09-23T20:52:19.946786+0000","last_peered":"2023-09-23T20:52:19.946786+0000","last_clean":"2023-09-23T20:52:19.946786+0000","last_became_active":"2023-09-23T20:51:21.744961+0000","last_became_peered":"2023-09-23T20:51:21.744961+0000","last_unstale":"2023-09-23T20:52:19.946786+0000","last_undegraded":"2023-09-23T20:52:19.946786+0000","last_fullsized":"2023-09-23T20:52:19.946786+0000","mapping_epoch":673,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":674,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:14:06.022442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"90'161","reported_seq":794,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.108151+0000","last_change":"2023-09-23T20:52:01.894751+0000","last_active":"2023-09-23T20:52:16.108151+0000","last_peered":"2023-09-23T20:52:16.108151+0000","last_clean":"2023-09-23T20:52:16.108151+0000","last_became_active":"2023-09-23T20:52:01.893974+0000","last_became_peered":"2023-09-23T20:52:01.893974+0000","last_unstale":"2023-09-23T20:52:16.108151+0000","last_undegraded":"2023-09-23T20:52:16.108151+0000","last_fullsized":"2023-09-23T20:52:16.108151+0000","mapping_epoch":688,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":689,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:15.656880+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:15.656880+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:31:59.751463+0000","scrub_duration":0.0016711180000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"664'209","reported_seq":1676,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.109794+0000","last_change":"2023-09-23T20:52:01.896675+0000","last_active":"2023-09-23T20:52:16.109794+0000","last_peered":"2023-09-23T20:52:16.109794+0000","last_clean":"2023-09-23T20:52:16.109794+0000","last_became_active":"2023-09-23T20:52:01.896048+0000","last_became_peered":"2023-09-23T20:52:01.896048+0000","last_unstale":"2023-09-23T20:52:16.109794+0000","last_undegraded":"2023-09-23T20:52:16.109794+0000","last_fullsized":"2023-09-23T20:52:16.109794+0000","mapping_epoch":688,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":689,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":209,"log_dups_size":0,"ondisk_log_size":209,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:07:06.650190+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001172474,"stat_sum":{"num_bytes":4194323,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":2248,"num_read_kb":3726,"num_write":182,"num_write_kb":5560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":151,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.109786+0000","last_change":"2023-09-23T20:47:09.177872+0000","last_active":"2023-09-23T20:52:16.109786+0000","last_peered":"2023-09-23T20:52:16.109786+0000","last_clean":"2023-09-23T20:52:16.109786+0000","last_became_active":"2023-09-23T20:47:09.176689+0000","last_became_peered":"2023-09-23T20:47:09.176689+0000","last_unstale":"2023-09-23T20:52:16.109786+0000","last_undegraded":"2023-09-23T20:52:16.109786+0000","last_fullsized":"2023-09-23T20:52:16.109786+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:14:01.590318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":679,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.108448+0000","last_change":"2023-09-23T20:48:48.151144+0000","last_active":"2023-09-23T20:52:16.108448+0000","last_peered":"2023-09-23T20:52:16.108448+0000","last_clean":"2023-09-23T20:52:16.108448+0000","last_became_active":"2023-09-23T20:48:48.150770+0000","last_became_peered":"2023-09-23T20:48:48.150770+0000","last_unstale":"2023-09-23T20:52:16.108448+0000","last_undegraded":"2023-09-23T20:52:16.108448+0000","last_fullsized":"2023-09-23T20:52:16.108448+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:14:08.549512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"90'161","reported_seq":698,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:15.936383+0000","last_change":"2023-09-23T20:47:11.618652+0000","last_active":"2023-09-23T20:52:15.936383+0000","last_peered":"2023-09-23T20:52:15.936383+0000","last_clean":"2023-09-23T20:52:15.936383+0000","last_became_active":"2023-09-23T20:47:11.618266+0000","last_became_peered":"2023-09-23T20:47:11.618266+0000","last_unstale":"2023-09-23T20:52:15.936383+0000","last_undegraded":"2023-09-23T20:52:15.936383+0000","last_fullsized":"2023-09-23T20:52:15.936383+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":4,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:44:26.978179+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:44:26.978179+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:29:29.000683+0000","scrub_duration":0.0013962829999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"664'192","reported_seq":731,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.108578+0000","last_change":"2023-09-23T20:50:39.795569+0000","last_active":"2023-09-23T20:52:16.108578+0000","last_peered":"2023-09-23T20:52:16.108578+0000","last_clean":"2023-09-23T20:52:16.108578+0000","last_became_active":"2023-09-23T20:48:51.170485+0000","last_became_peered":"2023-09-23T20:48:51.170485+0000","last_unstale":"2023-09-23T20:52:16.108578+0000","last_undegraded":"2023-09-23T20:52:16.108578+0000","last_fullsized":"2023-09-23T20:52:16.108578+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":192,"log_dups_size":0,"ondisk_log_size":192,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:48:35.309684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038574200000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":830,"num_read_kb":3504,"num_write":183,"num_write_kb":2624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":688,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.109864+0000","last_change":"2023-09-23T20:48:48.151034+0000","last_active":"2023-09-23T20:52:16.109864+0000","last_peered":"2023-09-23T20:52:16.109864+0000","last_clean":"2023-09-23T20:52:16.109864+0000","last_became_active":"2023-09-23T20:48:48.150658+0000","last_became_peered":"2023-09-23T20:48:48.150658+0000","last_unstale":"2023-09-23T20:52:16.109864+0000","last_undegraded":"2023-09-23T20:52:16.109864+0000","last_fullsized":"2023-09-23T20:52:16.109864+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:44:34.819851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":230,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.109037+0000","last_change":"2023-09-23T20:51:21.745968+0000","last_active":"2023-09-23T20:52:16.109037+0000","last_peered":"2023-09-23T20:52:16.109037+0000","last_clean":"2023-09-23T20:52:16.109037+0000","last_became_active":"2023-09-23T20:51:21.744991+0000","last_became_peered":"2023-09-23T20:51:21.744991+0000","last_unstale":"2023-09-23T20:52:16.109037+0000","last_undegraded":"2023-09-23T20:52:16.109037+0000","last_fullsized":"2023-09-23T20:52:16.109037+0000","mapping_epoch":673,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":674,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:01.825839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"664'319","reported_seq":1180,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.109565+0000","last_change":"2023-09-23T20:52:01.895902+0000","last_active":"2023-09-23T20:52:16.109565+0000","last_peered":"2023-09-23T20:52:16.109565+0000","last_clean":"2023-09-23T20:52:16.109565+0000","last_became_active":"2023-09-23T20:52:01.895001+0000","last_became_peered":"2023-09-23T20:52:01.895001+0000","last_unstale":"2023-09-23T20:52:16.109565+0000","last_undegraded":"2023-09-23T20:52:16.109565+0000","last_fullsized":"2023-09-23T20:52:16.109565+0000","mapping_epoch":688,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":689,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":319,"log_dups_size":0,"ondisk_log_size":319,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:46:47.800593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043345100000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1396,"num_read_kb":4612,"num_write":290,"num_write_kb":4437,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"90'161","reported_seq":1157,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:15.936196+0000","last_change":"2023-09-23T20:38:05.602508+0000","last_active":"2023-09-23T20:52:15.936196+0000","last_peered":"2023-09-23T20:52:15.936196+0000","last_clean":"2023-09-23T20:52:15.936196+0000","last_became_active":"2023-09-23T20:38:05.602129+0000","last_became_peered":"2023-09-23T20:38:05.602129+0000","last_unstale":"2023-09-23T20:52:15.936196+0000","last_undegraded":"2023-09-23T20:52:15.936196+0000","last_fullsized":"2023-09-23T20:52:15.936196+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":0,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:45.343742+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:45.343742+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:48.910343+0000","scrub_duration":0.001475802,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":282,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.108712+0000","last_change":"2023-09-23T20:51:49.915404+0000","last_active":"2023-09-23T20:52:16.108712+0000","last_peered":"2023-09-23T20:52:16.108712+0000","last_clean":"2023-09-23T20:52:16.108712+0000","last_became_active":"2023-09-23T20:51:49.915178+0000","last_became_peered":"2023-09-23T20:51:49.915178+0000","last_unstale":"2023-09-23T20:52:16.108712+0000","last_undegraded":"2023-09-23T20:52:16.108712+0000","last_fullsized":"2023-09-23T20:52:16.108712+0000","mapping_epoch":683,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":684,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:23.717157+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":319,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.110502+0000","last_change":"2023-09-23T20:47:09.202302+0000","last_active":"2023-09-23T20:52:16.110502+0000","last_peered":"2023-09-23T20:52:16.110502+0000","last_clean":"2023-09-23T20:52:16.110502+0000","last_became_active":"2023-09-23T20:47:09.201797+0000","last_became_peered":"2023-09-23T20:47:09.201797+0000","last_unstale":"2023-09-23T20:52:16.110502+0000","last_undegraded":"2023-09-23T20:52:16.110502+0000","last_fullsized":"2023-09-23T20:52:16.110502+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:34:41.055705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"90'161","reported_seq":1065,"reported_epoch":695,"state":"active+clean","last_fresh":"2023-09-23T20:52:19.947376+0000","last_change":"2023-09-23T20:47:11.622262+0000","last_active":"2023-09-23T20:52:19.947376+0000","last_peered":"2023-09-23T20:52:19.947376+0000","last_clean":"2023-09-23T20:52:19.947376+0000","last_became_active":"2023-09-23T20:47:11.621920+0000","last_became_peered":"2023-09-23T20:47:11.621920+0000","last_unstale":"2023-09-23T20:52:19.947376+0000","last_undegraded":"2023-09-23T20:52:19.947376+0000","last_fullsized":"2023-09-23T20:52:19.947376+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:31:43.582146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:31:43.582146+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:01:15.153543+0000","scrub_duration":0.0016219730000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"664'171","reported_seq":1359,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:15.936148+0000","last_change":"2023-09-23T20:50:39.742996+0000","last_active":"2023-09-23T20:52:15.936148+0000","last_peered":"2023-09-23T20:52:15.936148+0000","last_clean":"2023-09-23T20:52:15.936148+0000","last_became_active":"2023-09-23T20:29:45.536439+0000","last_became_peered":"2023-09-23T20:29:45.536439+0000","last_unstale":"2023-09-23T20:52:15.936148+0000","last_undegraded":"2023-09-23T20:52:15.936148+0000","last_fullsized":"2023-09-23T20:52:15.936148+0000","mapping_epoch":266,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":267,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":171,"log_dups_size":0,"ondisk_log_size":171,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:30:57.793132+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032959200000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1114,"num_read_kb":3170,"num_write":132,"num_write_kb":2190,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":709,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.108171+0000","last_change":"2023-09-23T20:47:09.198326+0000","last_active":"2023-09-23T20:52:16.108171+0000","last_peered":"2023-09-23T20:52:16.108171+0000","last_clean":"2023-09-23T20:52:16.108171+0000","last_became_active":"2023-09-23T20:47:09.198059+0000","last_became_peered":"2023-09-23T20:47:09.198059+0000","last_unstale":"2023-09-23T20:52:16.108171+0000","last_undegraded":"2023-09-23T20:52:16.108171+0000","last_fullsized":"2023-09-23T20:52:16.108171+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:38:53.903706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":704,"reported_epoch":695,"state":"active+clean","last_fresh":"2023-09-23T20:52:21.228992+0000","last_change":"2023-09-23T20:48:48.146977+0000","last_active":"2023-09-23T20:52:21.228992+0000","last_peered":"2023-09-23T20:52:21.228992+0000","last_clean":"2023-09-23T20:52:21.228992+0000","last_became_active":"2023-09-23T20:48:48.146657+0000","last_became_peered":"2023-09-23T20:48:48.146657+0000","last_unstale":"2023-09-23T20:52:21.228992+0000","last_undegraded":"2023-09-23T20:52:21.228992+0000","last_fullsized":"2023-09-23T20:52:21.228992+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:38:41.782692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.6","version":"90'161","reported_seq":805,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.110466+0000","last_change":"2023-09-23T20:41:08.380084+0000","last_active":"2023-09-23T20:52:16.110466+0000","last_peered":"2023-09-23T20:52:16.110466+0000","last_clean":"2023-09-23T20:52:16.110466+0000","last_became_active":"2023-09-23T20:38:05.610881+0000","last_became_peered":"2023-09-23T20:38:05.610881+0000","last_unstale":"2023-09-23T20:52:16.110466+0000","last_undegraded":"2023-09-23T20:52:16.110466+0000","last_fullsized":"2023-09-23T20:52:16.110466+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:41:08.379959+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:41:08.379959+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T08:19:39.547923+0000","scrub_duration":0.0019158420000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"664'175","reported_seq":1658,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:15.936558+0000","last_change":"2023-09-23T20:50:39.743889+0000","last_active":"2023-09-23T20:52:15.936558+0000","last_peered":"2023-09-23T20:52:15.936558+0000","last_clean":"2023-09-23T20:52:15.936558+0000","last_became_active":"2023-09-23T20:47:14.628341+0000","last_became_peered":"2023-09-23T20:47:14.628341+0000","last_unstale":"2023-09-23T20:52:15.936558+0000","last_undegraded":"2023-09-23T20:52:15.936558+0000","last_fullsized":"2023-09-23T20:52:15.936558+0000","mapping_epoch":558,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":559,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":175,"log_dups_size":0,"ondisk_log_size":175,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:42:16.291849+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000229686,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1422,"num_read_kb":2425,"num_write":116,"num_write_kb":664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":681,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.108608+0000","last_change":"2023-09-23T20:51:21.747497+0000","last_active":"2023-09-23T20:52:16.108608+0000","last_peered":"2023-09-23T20:52:16.108608+0000","last_clean":"2023-09-23T20:52:16.108608+0000","last_became_active":"2023-09-23T20:51:21.747193+0000","last_became_peered":"2023-09-23T20:51:21.747193+0000","last_unstale":"2023-09-23T20:52:16.108608+0000","last_undegraded":"2023-09-23T20:52:16.108608+0000","last_fullsized":"2023-09-23T20:52:16.108608+0000","mapping_epoch":673,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":674,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:42:19.319230+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":659,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.109756+0000","last_change":"2023-09-23T20:51:21.743456+0000","last_active":"2023-09-23T20:52:16.109756+0000","last_peered":"2023-09-23T20:52:16.109756+0000","last_clean":"2023-09-23T20:52:16.109756+0000","last_became_active":"2023-09-23T20:51:21.743180+0000","last_became_peered":"2023-09-23T20:51:21.743180+0000","last_unstale":"2023-09-23T20:52:16.109756+0000","last_undegraded":"2023-09-23T20:52:16.109756+0000","last_fullsized":"2023-09-23T20:52:16.109756+0000","mapping_epoch":673,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":674,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:40:50.012029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"90'161","reported_seq":731,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.109299+0000","last_change":"2023-09-23T20:48:51.172830+0000","last_active":"2023-09-23T20:52:16.109299+0000","last_peered":"2023-09-23T20:52:16.109299+0000","last_clean":"2023-09-23T20:52:16.109299+0000","last_became_active":"2023-09-23T20:48:51.172481+0000","last_became_peered":"2023-09-23T20:48:51.172481+0000","last_unstale":"2023-09-23T20:52:16.109299+0000","last_undegraded":"2023-09-23T20:52:16.109299+0000","last_fullsized":"2023-09-23T20:52:16.109299+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:47:34.561747+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:47:34.561747+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:30:32.531477+0000","scrub_duration":0.0087949929999999992,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":192,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.108942+0000","last_change":"2023-09-23T20:51:49.841192+0000","last_active":"2023-09-23T20:52:16.108942+0000","last_peered":"2023-09-23T20:52:16.108942+0000","last_clean":"2023-09-23T20:52:16.108942+0000","last_became_active":"2023-09-23T20:51:49.840874+0000","last_became_peered":"2023-09-23T20:51:49.840874+0000","last_unstale":"2023-09-23T20:52:16.108942+0000","last_undegraded":"2023-09-23T20:52:16.108942+0000","last_fullsized":"2023-09-23T20:52:16.108942+0000","mapping_epoch":683,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":684,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:41:16.392417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"664'349","reported_seq":2056,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.109605+0000","last_change":"2023-09-23T20:50:39.737163+0000","last_active":"2023-09-23T20:52:16.109605+0000","last_peered":"2023-09-23T20:52:16.109605+0000","last_clean":"2023-09-23T20:52:16.109605+0000","last_became_active":"2023-09-23T20:47:11.621284+0000","last_became_peered":"2023-09-23T20:47:11.621284+0000","last_unstale":"2023-09-23T20:52:16.109605+0000","last_undegraded":"2023-09-23T20:52:16.109605+0000","last_fullsized":"2023-09-23T20:52:16.109605+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":349,"log_dups_size":0,"ondisk_log_size":349,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:43:50.658512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0014591350000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":2217,"num_read_kb":2864,"num_write":263,"num_write_kb":2168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":479,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.108351+0000","last_change":"2023-09-23T20:37:43.582105+0000","last_active":"2023-09-23T20:52:16.108351+0000","last_peered":"2023-09-23T20:52:16.108351+0000","last_clean":"2023-09-23T20:52:16.108351+0000","last_became_active":"2023-09-23T20:37:43.581860+0000","last_became_peered":"2023-09-23T20:37:43.581860+0000","last_unstale":"2023-09-23T20:52:16.108351+0000","last_undegraded":"2023-09-23T20:52:16.108351+0000","last_fullsized":"2023-09-23T20:52:16.108351+0000","mapping_epoch":399,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":400,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:18:38.765486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"90'161","reported_seq":947,"reported_epoch":695,"state":"active+clean","last_fresh":"2023-09-23T20:52:20.443781+0000","last_change":"2023-09-23T20:51:24.741192+0000","last_active":"2023-09-23T20:52:20.443781+0000","last_peered":"2023-09-23T20:52:20.443781+0000","last_clean":"2023-09-23T20:52:20.443781+0000","last_became_active":"2023-09-23T20:51:24.740845+0000","last_became_peered":"2023-09-23T20:51:24.740845+0000","last_unstale":"2023-09-23T20:52:20.443781+0000","last_undegraded":"2023-09-23T20:52:20.443781+0000","last_fullsized":"2023-09-23T20:52:20.443781+0000","mapping_epoch":676,"log_start":"0'0","ondisk_log_start":"0'0","created":172,"last_epoch_clean":677,"parent":"0.0","parent_split_bits":3,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:34:54.504554+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:34:54.504554+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:57:05.662352+0000","scrub_duration":0.011176864,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"664'293","reported_seq":1452,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.110649+0000","last_change":"2023-09-23T20:50:39.740476+0000","last_active":"2023-09-23T20:52:16.110649+0000","last_peered":"2023-09-23T20:52:16.110649+0000","last_clean":"2023-09-23T20:52:16.110649+0000","last_became_active":"2023-09-23T20:48:51.217013+0000","last_became_peered":"2023-09-23T20:48:51.217013+0000","last_unstale":"2023-09-23T20:52:16.110649+0000","last_undegraded":"2023-09-23T20:52:16.110649+0000","last_fullsized":"2023-09-23T20:52:16.110649+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:11:59.838394+0000","last_clean_scrub_stamp":"2023-09-23T20:11:59.838394+0000","objects_scrubbed":0,"log_size":293,"log_dups_size":0,"ondisk_log_size":293,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:40:42.297947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00027326800000000001,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1602,"num_read_kb":1486,"num_write":268,"num_write_kb":4284,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":48,"num_bytes_recovered":25165824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"71'1","reported_seq":712,"reported_epoch":695,"state":"active+clean","last_fresh":"2023-09-23T20:52:21.229078+0000","last_change":"2023-09-23T20:47:09.317116+0000","last_active":"2023-09-23T20:52:21.229078+0000","last_peered":"2023-09-23T20:52:21.229078+0000","last_clean":"2023-09-23T20:52:21.229078+0000","last_became_active":"2023-09-23T20:47:09.205930+0000","last_became_peered":"2023-09-23T20:47:09.205930+0000","last_unstale":"2023-09-23T20:52:21.229078+0000","last_undegraded":"2023-09-23T20:52:21.229078+0000","last_fullsized":"2023-09-23T20:52:21.229078+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:04:55.528625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":171,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.108859+0000","last_change":"2023-09-23T20:51:21.746106+0000","last_active":"2023-09-23T20:52:16.108859+0000","last_peered":"2023-09-23T20:52:16.108859+0000","last_clean":"2023-09-23T20:52:16.108859+0000","last_became_active":"2023-09-23T20:51:21.745777+0000","last_became_peered":"2023-09-23T20:51:21.745777+0000","last_unstale":"2023-09-23T20:52:16.108859+0000","last_undegraded":"2023-09-23T20:52:16.108859+0000","last_fullsized":"2023-09-23T20:52:16.108859+0000","mapping_epoch":673,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":674,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:59:40.976999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"90'161","reported_seq":604,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.109278+0000","last_change":"2023-09-23T20:51:50.434718+0000","last_active":"2023-09-23T20:52:16.109278+0000","last_peered":"2023-09-23T20:52:16.109278+0000","last_clean":"2023-09-23T20:52:16.109278+0000","last_became_active":"2023-09-23T20:51:49.839630+0000","last_became_peered":"2023-09-23T20:51:49.839630+0000","last_unstale":"2023-09-23T20:52:16.109278+0000","last_undegraded":"2023-09-23T20:52:16.109278+0000","last_fullsized":"2023-09-23T20:52:16.109278+0000","mapping_epoch":683,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":684,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:51:50.434626+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:51:50.434626+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:33:23.482033+0000","scrub_duration":0.0019105050000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":670,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.109717+0000","last_change":"2023-09-23T20:47:09.197601+0000","last_active":"2023-09-23T20:52:16.109717+0000","last_peered":"2023-09-23T20:52:16.109717+0000","last_clean":"2023-09-23T20:52:16.109717+0000","last_became_active":"2023-09-23T20:47:09.196967+0000","last_became_peered":"2023-09-23T20:47:09.196967+0000","last_unstale":"2023-09-23T20:52:16.109717+0000","last_undegraded":"2023-09-23T20:52:16.109717+0000","last_fullsized":"2023-09-23T20:52:16.109717+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:06:56.826785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":115,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.109105+0000","last_change":"2023-09-23T20:48:48.158376+0000","last_active":"2023-09-23T20:52:16.109105+0000","last_peered":"2023-09-23T20:52:16.109105+0000","last_clean":"2023-09-23T20:52:16.109105+0000","last_became_active":"2023-09-23T20:48:48.158069+0000","last_became_peered":"2023-09-23T20:48:48.158069+0000","last_unstale":"2023-09-23T20:52:16.109105+0000","last_undegraded":"2023-09-23T20:52:16.109105+0000","last_fullsized":"2023-09-23T20:52:16.109105+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:43:34.658561+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"90'161","reported_seq":772,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.108685+0000","last_change":"2023-09-23T20:41:06.499511+0000","last_active":"2023-09-23T20:52:16.108685+0000","last_peered":"2023-09-23T20:52:16.108685+0000","last_clean":"2023-09-23T20:52:16.108685+0000","last_became_active":"2023-09-23T20:38:05.601874+0000","last_became_peered":"2023-09-23T20:38:05.601874+0000","last_unstale":"2023-09-23T20:52:16.108685+0000","last_undegraded":"2023-09-23T20:52:16.108685+0000","last_fullsized":"2023-09-23T20:52:16.108685+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:41:06.499396+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:41:06.499396+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:31:34.059077+0000","scrub_duration":0.0019670310000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":168,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.108555+0000","last_change":"2023-09-23T20:52:01.896760+0000","last_active":"2023-09-23T20:52:16.108555+0000","last_peered":"2023-09-23T20:52:16.108555+0000","last_clean":"2023-09-23T20:52:16.108555+0000","last_became_active":"2023-09-23T20:52:01.896426+0000","last_became_peered":"2023-09-23T20:52:01.896426+0000","last_unstale":"2023-09-23T20:52:16.108555+0000","last_undegraded":"2023-09-23T20:52:16.108555+0000","last_fullsized":"2023-09-23T20:52:16.108555+0000","mapping_epoch":688,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":689,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:41:39.174148+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":151,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.109744+0000","last_change":"2023-09-23T20:47:09.186579+0000","last_active":"2023-09-23T20:52:16.109744+0000","last_peered":"2023-09-23T20:52:16.109744+0000","last_clean":"2023-09-23T20:52:16.109744+0000","last_became_active":"2023-09-23T20:47:09.186227+0000","last_became_peered":"2023-09-23T20:47:09.186227+0000","last_unstale":"2023-09-23T20:52:16.109744+0000","last_undegraded":"2023-09-23T20:52:16.109744+0000","last_fullsized":"2023-09-23T20:52:16.109744+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T07:45:39.485308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":36,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.108881+0000","last_change":"2023-09-23T20:52:01.899486+0000","last_active":"2023-09-23T20:52:16.108881+0000","last_peered":"2023-09-23T20:52:16.108881+0000","last_clean":"2023-09-23T20:52:16.108881+0000","last_became_active":"2023-09-23T20:52:01.899077+0000","last_became_peered":"2023-09-23T20:52:01.899077+0000","last_unstale":"2023-09-23T20:52:16.108881+0000","last_undegraded":"2023-09-23T20:52:16.108881+0000","last_fullsized":"2023-09-23T20:52:16.108881+0000","mapping_epoch":688,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":689,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:03.951836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":151,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.108485+0000","last_change":"2023-09-23T20:47:09.181240+0000","last_active":"2023-09-23T20:52:16.108485+0000","last_peered":"2023-09-23T20:52:16.108485+0000","last_clean":"2023-09-23T20:52:16.108485+0000","last_became_active":"2023-09-23T20:47:09.181022+0000","last_became_peered":"2023-09-23T20:47:09.181022+0000","last_unstale":"2023-09-23T20:52:16.108485+0000","last_undegraded":"2023-09-23T20:52:16.108485+0000","last_fullsized":"2023-09-23T20:52:16.108485+0000","mapping_epoch":552,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:38:59.191781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"90'161","reported_seq":950,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:15.936409+0000","last_change":"2023-09-23T20:38:05.591992+0000","last_active":"2023-09-23T20:52:15.936409+0000","last_peered":"2023-09-23T20:52:15.936409+0000","last_clean":"2023-09-23T20:52:15.936409+0000","last_became_active":"2023-09-23T20:38:05.591701+0000","last_became_peered":"2023-09-23T20:38:05.591701+0000","last_unstale":"2023-09-23T20:52:15.936409+0000","last_undegraded":"2023-09-23T20:52:15.936409+0000","last_fullsized":"2023-09-23T20:52:15.936409+0000","mapping_epoch":406,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":407,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:02.384133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:02.384133+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:41.722542+0000","scrub_duration":0.001769098,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":539,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.108951+0000","last_change":"2023-09-23T20:48:48.158348+0000","last_active":"2023-09-23T20:52:16.108951+0000","last_peered":"2023-09-23T20:52:16.108951+0000","last_clean":"2023-09-23T20:52:16.108951+0000","last_became_active":"2023-09-23T20:48:48.157982+0000","last_became_peered":"2023-09-23T20:48:48.157982+0000","last_unstale":"2023-09-23T20:52:16.108951+0000","last_undegraded":"2023-09-23T20:52:16.108951+0000","last_fullsized":"2023-09-23T20:52:16.108951+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:03:48.078707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":505,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:17.946525+0000","last_change":"2023-09-23T20:48:48.159781+0000","last_active":"2023-09-23T20:52:17.946525+0000","last_peered":"2023-09-23T20:52:17.946525+0000","last_clean":"2023-09-23T20:52:17.946525+0000","last_became_active":"2023-09-23T20:48:48.159505+0000","last_became_peered":"2023-09-23T20:48:48.159505+0000","last_unstale":"2023-09-23T20:52:17.946525+0000","last_undegraded":"2023-09-23T20:52:17.946525+0000","last_fullsized":"2023-09-23T20:52:17.946525+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:25:18.100645+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"90'161","reported_seq":825,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.108875+0000","last_change":"2023-09-23T20:47:11.621758+0000","last_active":"2023-09-23T20:52:16.108875+0000","last_peered":"2023-09-23T20:52:16.108875+0000","last_clean":"2023-09-23T20:52:16.108875+0000","last_became_active":"2023-09-23T20:47:11.621345+0000","last_became_peered":"2023-09-23T20:47:11.621345+0000","last_unstale":"2023-09-23T20:52:16.108875+0000","last_undegraded":"2023-09-23T20:52:16.108875+0000","last_fullsized":"2023-09-23T20:52:16.108875+0000","mapping_epoch":555,"log_start":"0'0","ondisk_log_start":"0'0","created":332,"last_epoch_clean":556,"parent":"0.0","parent_split_bits":5,"last_scrub":"90'161","last_scrub_stamp":"2023-09-23T20:36:01.504973+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:09:47.213130+0000","last_clean_scrub_stamp":"2023-09-23T20:36:01.504973+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:31:17.112393+0000","scrub_duration":0.00202344,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":791,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.109494+0000","last_change":"2023-09-23T20:51:21.755459+0000","last_active":"2023-09-23T20:52:16.109494+0000","last_peered":"2023-09-23T20:52:16.109494+0000","last_clean":"2023-09-23T20:52:16.109494+0000","last_became_active":"2023-09-23T20:51:21.746135+0000","last_became_peered":"2023-09-23T20:51:21.746135+0000","last_unstale":"2023-09-23T20:52:16.109494+0000","last_undegraded":"2023-09-23T20:52:16.109494+0000","last_fullsized":"2023-09-23T20:52:16.109494+0000","mapping_epoch":673,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":674,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:42.725912+0000","last_clean_scrub_stamp":"2023-09-23T20:12:42.725912+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:47:46.578750+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":5540,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":161,"reported_epoch":694,"state":"active+clean","last_fresh":"2023-09-23T20:52:16.108146+0000","last_change":"2023-09-23T20:48:23.917302+0000","last_active":"2023-09-23T20:52:16.108146+0000","last_peered":"2023-09-23T20:52:16.108146+0000","last_clean":"2023-09-23T20:52:16.108146+0000","last_became_active":"2023-09-23T20:48:23.916956+0000","last_became_peered":"2023-09-23T20:48:23.916956+0000","last_unstale":"2023-09-23T20:52:16.108146+0000","last_undegraded":"2023-09-23T20:52:16.108146+0000","last_fullsized":"2023-09-23T20:52:16.108146+0000","mapping_epoch":577,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":578,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T20:12:43.732826+0000","last_clean_scrub_stamp":"2023-09-23T20:12:43.732826+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:42:27.989294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":125,"num_bytes_recovered":16468,"num_keys_recovered":80,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26691,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":8388627,"num_objects":5,"num_object_clones":0,"num_object_copies":15,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":11981,"num_read_kb":26415,"num_write":1690,"num_write_kb":25390,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":156,"num_bytes_recovered":25165843,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":25178112,"data_stored":25165881,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6082,"internal_metadata":0},"log_size":1950,"ondisk_log_size":1950,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201777,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":620404,"kb_used_data":11436,"kb_used_omap":0,"kb_used_meta":608960,"kb_avail":93124748,"statfs":{"total":95995035648,"available":95359741952,"internally_reserved":0,"allocated":11710464,"data_stored":9904172,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":623575040},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:52:16 2023","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":0.99399999999999999,"15min":1.639},"min":{"1min":0.307,"5min":0.224,"15min":0.224},"max":{"1min":0.71499999999999997,"5min":5.6970000000000001,"15min":18.315000000000001},"last":0.63600000000000001},{"interface":"front","average":{"1min":0.56000000000000005,"5min":1.0980000000000001,"15min":1.506},"min":{"1min":0.35299999999999998,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.72999999999999998,"5min":8.0530000000000008,"15min":15.327999999999999},"last":0.46100000000000002}]},{"osd":1,"last update":"Sat Sep 23 20:52:20 2023","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":0.91400000000000003,"15min":1.4350000000000001},"min":{"1min":0.35599999999999998,"5min":0.28399999999999997,"15min":0.24399999999999999},"max":{"1min":0.76400000000000001,"5min":7.3789999999999996,"15min":29.254999999999999},"last":0.70099999999999996},{"interface":"front","average":{"1min":0.58899999999999997,"5min":1.127,"15min":1.897},"min":{"1min":0.379,"5min":0.27000000000000002,"15min":0.26700000000000002},"max":{"1min":0.85899999999999999,"5min":10.861000000000001,"15min":16.882000000000001},"last":0.68700000000000006}]},{"osd":2,"last update":"Sat Sep 23 20:52:16 2023","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":1.3129999999999999,"15min":1.7909999999999999},"min":{"1min":0.42699999999999999,"5min":0.28499999999999998,"15min":0.27900000000000003},"max":{"1min":0.81100000000000005,"5min":10.461,"15min":16.965},"last":0.51200000000000001},{"interface":"front","average":{"1min":0.51700000000000002,"5min":1.2390000000000001,"15min":1.8320000000000001},"min":{"1min":0.33900000000000002,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.67400000000000004,"5min":7.3019999999999996,"15min":23.414000000000001},"last":0.53400000000000003}]},{"osd":3,"last update":"Sat Sep 23 20:52:43 2023","interfaces":[{"interface":"back","average":{"1min":0.70499999999999996,"5min":1.04,"15min":1.6379999999999999},"min":{"1min":0.40500000000000003,"5min":0.32200000000000001,"15min":0.311},"max":{"1min":2.855,"5min":7.2880000000000003,"15min":23.352},"last":0.66700000000000004},{"interface":"front","average":{"1min":0.81399999999999995,"5min":1.1279999999999999,"15min":1.8480000000000001},"min":{"1min":0.52200000000000002,"5min":0.35399999999999998,"15min":0.34300000000000003},"max":{"1min":2.8889999999999998,"5min":6.5090000000000003,"15min":20.969999999999999},"last":0.55400000000000005}]},{"osd":4,"last update":"Sat Sep 23 20:52:43 2023","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.81899999999999995,"15min":1.1459999999999999},"min":{"1min":0.34300000000000003,"5min":0.25800000000000001,"15min":0.254},"max":{"1min":0.74299999999999999,"5min":7.4109999999999996,"15min":18.364999999999998},"last":0.74299999999999999},{"interface":"front","average":{"1min":0.66200000000000003,"5min":1.032,"15min":1.3979999999999999},"min":{"1min":0.443,"5min":0.27500000000000002,"15min":0.247},"max":{"1min":0.90000000000000002,"5min":27.425999999999998,"15min":27.425999999999998},"last":0.623}]},{"osd":5,"last update":"Sat Sep 23 20:52:43 2023","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.755,"15min":1.1439999999999999},"min":{"1min":0.40999999999999998,"5min":0.28100000000000003,"15min":0.27200000000000002},"max":{"1min":0.66000000000000003,"5min":7.2270000000000003,"15min":15.005000000000001},"last":0.59899999999999998},{"interface":"front","average":{"1min":0.67400000000000004,"5min":0.73899999999999999,"15min":1.274},"min":{"1min":0.46100000000000002,"5min":0.32200000000000001,"15min":0.27700000000000002},"max":{"1min":0.89200000000000002,"5min":7.335,"15min":18.390000000000001},"last":0.77200000000000002}]},{"osd":6,"last update":"Sat Sep 23 20:52:43 2023","interfaces":[{"interface":"back","average":{"1min":0.67700000000000005,"5min":0.93700000000000006,"15min":1.47},"min":{"1min":0.42899999999999999,"5min":0.379,"15min":0.29899999999999999},"max":{"1min":0.80500000000000005,"5min":5.6070000000000002,"15min":18.417000000000002},"last":0.80100000000000005},{"interface":"front","average":{"1min":0.71799999999999997,"5min":0.86299999999999999,"15min":1.3460000000000001},"min":{"1min":0.495,"5min":0.36499999999999999,"15min":0.32300000000000001},"max":{"1min":0.95899999999999996,"5min":4.3710000000000004,"15min":15.369999999999999},"last":0.58499999999999996}]}]},{"osd":6,"up_from":47,"seq":201863463416,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":543528,"kb_used_data":15584,"kb_used_omap":0,"kb_used_meta":527936,"kb_avail":93201624,"statfs":{"total":95995035648,"available":95438462976,"internally_reserved":0,"allocated":15958016,"data_stored":14163341,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":540606464},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:52:12 2023","interfaces":[{"interface":"back","average":{"1min":0.48499999999999999,"5min":1.423,"15min":1.6060000000000001},"min":{"1min":0.30299999999999999,"5min":0.24399999999999999,"15min":0.219},"max":{"1min":0.84699999999999998,"5min":15.574999999999999,"15min":21.018000000000001},"last":0.60999999999999999},{"interface":"front","average":{"1min":0.53500000000000003,"5min":1.224,"15min":1.8879999999999999},"min":{"1min":0.26000000000000001,"5min":0.255,"15min":0.23300000000000001},"max":{"1min":0.995,"5min":11.557,"15min":60.328000000000003},"last":0.38}]},{"osd":1,"last update":"Sat Sep 23 20:52:29 2023","interfaces":[{"interface":"back","average":{"1min":0.878,"5min":1.341,"15min":1.714},"min":{"1min":0.28599999999999998,"5min":0.26000000000000001,"15min":0.249},"max":{"1min":7.3470000000000004,"5min":8.9350000000000005,"15min":64.274000000000001},"last":0.56100000000000005},{"interface":"front","average":{"1min":0.55600000000000005,"5min":1.0640000000000001,"15min":1.4199999999999999},"min":{"1min":0.34399999999999997,"5min":0.312,"15min":0.28000000000000003},"max":{"1min":0.89900000000000002,"5min":8.0969999999999995,"15min":17.786000000000001},"last":0.41899999999999998}]},{"osd":2,"last update":"Sat Sep 23 20:52:12 2023","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":1.712,"15min":1.9330000000000001},"min":{"1min":0.36899999999999999,"5min":0.29699999999999999,"15min":0.27800000000000002},"max":{"1min":0.93200000000000005,"5min":9.0730000000000004,"15min":22.591000000000001},"last":0.45800000000000002},{"interface":"front","average":{"1min":0.61099999999999999,"5min":1.583,"15min":1.726},"min":{"1min":0.439,"5min":0.375,"15min":0.311},"max":{"1min":0.94999999999999996,"5min":12.798,"15min":22.614999999999998},"last":0.52200000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:52:01 2023","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":1.3879999999999999,"15min":1.754},"min":{"1min":0.44400000000000001,"5min":0.307,"15min":0.28399999999999997},"max":{"1min":0.96299999999999997,"5min":9.6859999999999999,"15min":11.664},"last":0.47499999999999998},{"interface":"front","average":{"1min":0.59299999999999997,"5min":1.611,"15min":1.9319999999999999},"min":{"1min":0.40799999999999997,"5min":0.32700000000000001,"15min":0.30499999999999999},"max":{"1min":0.91400000000000003,"5min":11.6,"15min":11.913},"last":0.58999999999999997}]},{"osd":4,"last update":"Sat Sep 23 20:52:29 2023","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.91000000000000003,"15min":0.94599999999999995},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.25600000000000001},"max":{"1min":0.78800000000000003,"5min":8.1229999999999993,"15min":23.904},"last":0.66700000000000004},{"interface":"front","average":{"1min":0.58599999999999997,"5min":1.2649999999999999,"15min":1.175},"min":{"1min":0.309,"5min":0.309,"15min":0.28399999999999997},"max":{"1min":0.82099999999999995,"5min":11.999000000000001,"15min":11.999000000000001},"last":0.64500000000000002}]},{"osd":5,"last update":"Sat Sep 23 20:52:29 2023","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":1.0109999999999999,"15min":0.94899999999999995},"min":{"1min":0.33100000000000002,"5min":0.32800000000000001,"15min":0.253},"max":{"1min":0.85199999999999998,"5min":8.0440000000000005,"15min":8.0440000000000005},"last":0.70599999999999996},{"interface":"front","average":{"1min":0.63300000000000001,"5min":1.079,"15min":1.2689999999999999},"min":{"1min":0.34300000000000003,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.84999999999999998,"5min":8.2769999999999992,"15min":9.3559999999999999},"last":0.69199999999999995}]},{"osd":7,"last update":"Sat Sep 23 20:52:01 2023","interfaces":[{"interface":"back","average":{"1min":0.58999999999999997,"5min":0.92900000000000005,"15min":0.97699999999999998},"min":{"1min":0.42999999999999999,"5min":0.35199999999999998,"15min":0.28399999999999997},"max":{"1min":0.82899999999999996,"5min":8.0719999999999992,"15min":8.9309999999999992},"last":0.73199999999999998},{"interface":"front","average":{"1min":0.60999999999999999,"5min":2.907,"15min":1.9610000000000001},"min":{"1min":0.38,"5min":0.38,"15min":0.34699999999999998},"max":{"1min":0.79700000000000004,"5min":159.91999999999999,"15min":159.91999999999999},"last":0.77300000000000002}]}]},{"osd":1,"up_from":266,"seq":1142461301014,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":823792,"kb_used_data":12968,"kb_used_omap":8,"kb_used_meta":810807,"kb_avail":92921360,"statfs":{"total":95995035648,"available":95151472640,"internally_reserved":0,"allocated":13279232,"data_stored":9856260,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8206,"internal_metadata":830267378},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:52:35 2023","interfaces":[{"interface":"back","average":{"1min":0.36499999999999999,"5min":1.452,"15min":1.8979999999999999},"min":{"1min":0.23699999999999999,"5min":0.20499999999999999,"15min":0.20000000000000001},"max":{"1min":0.55900000000000005,"5min":20.292000000000002,"15min":20.292000000000002},"last":0.29099999999999998},{"interface":"front","average":{"1min":0.42899999999999999,"5min":1.327,"15min":2.1949999999999998},"min":{"1min":0.28399999999999997,"5min":0.217,"15min":0.20100000000000001},"max":{"1min":1.246,"5min":17.463000000000001,"15min":20.346},"last":0.32700000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:52:35 2023","interfaces":[{"interface":"back","average":{"1min":0.36799999999999999,"5min":1.4770000000000001,"15min":2.242},"min":{"1min":0.26900000000000002,"5min":0.23100000000000001,"15min":0.216},"max":{"1min":0.50800000000000001,"5min":23.341000000000001,"15min":32.255000000000003},"last":0.314},{"interface":"front","average":{"1min":0.40600000000000003,"5min":1.573,"15min":1.923},"min":{"1min":0.29899999999999999,"5min":0.249,"15min":0.20899999999999999},"max":{"1min":0.59999999999999998,"5min":34.002000000000002,"15min":34.002000000000002},"last":0.34899999999999998}]},{"osd":3,"last update":"Sat Sep 23 20:52:35 2023","interfaces":[{"interface":"back","average":{"1min":0.435,"5min":1.2370000000000001,"15min":1.762},"min":{"1min":0.29899999999999999,"5min":0.28899999999999998,"15min":0.247},"max":{"1min":0.58199999999999996,"5min":10.135999999999999,"15min":17.366},"last":0.39100000000000001},{"interface":"front","average":{"1min":0.42099999999999999,"5min":1.4350000000000001,"15min":2.1629999999999998},"min":{"1min":0.251,"5min":0.251,"15min":0.22900000000000001},"max":{"1min":0.64900000000000002,"5min":17.343,"15min":17.343},"last":0.40100000000000002}]},{"osd":4,"last update":"Sat Sep 23 20:52:35 2023","interfaces":[{"interface":"back","average":{"1min":0.624,"5min":1.5329999999999999,"15min":1.954},"min":{"1min":0.45600000000000002,"5min":0.33900000000000002,"15min":0.28599999999999998},"max":{"1min":0.80600000000000005,"5min":9.5359999999999996,"15min":20.634},"last":0.57999999999999996},{"interface":"front","average":{"1min":0.64700000000000002,"5min":1.3680000000000001,"15min":1.585},"min":{"1min":0.51400000000000001,"5min":0.34699999999999998,"15min":0.32600000000000001},"max":{"1min":0.77400000000000002,"5min":9.8249999999999993,"15min":20.835999999999999},"last":0.64100000000000001}]},{"osd":5,"last update":"Sat Sep 23 20:52:35 2023","interfaces":[{"interface":"back","average":{"1min":0.70999999999999996,"5min":1.474,"15min":1.95},"min":{"1min":0.52600000000000002,"5min":0.38,"15min":0.308},"max":{"1min":0.94299999999999995,"5min":17.396999999999998,"15min":17.396999999999998},"last":0.71699999999999997},{"interface":"front","average":{"1min":0.69499999999999995,"5min":1.5369999999999999,"15min":2.0640000000000001},"min":{"1min":0.5,"5min":0.371,"15min":0.32500000000000001},"max":{"1min":0.86899999999999999,"5min":16.366,"15min":16.366},"last":0.65500000000000003}]},{"osd":6,"last update":"Sat Sep 23 20:52:35 2023","interfaces":[{"interface":"back","average":{"1min":0.67200000000000004,"5min":1.679,"15min":2.0579999999999998},"min":{"1min":0.505,"5min":0.41599999999999998,"15min":0.316},"max":{"1min":0.82899999999999996,"5min":23.274000000000001,"15min":23.274000000000001},"last":0.58899999999999997},{"interface":"front","average":{"1min":0.66500000000000004,"5min":1.6100000000000001,"15min":2.157},"min":{"1min":0.432,"5min":0.40500000000000003,"15min":0.317},"max":{"1min":0.91800000000000004,"5min":9.6639999999999997,"15min":11.832000000000001},"last":0.68899999999999995}]},{"osd":7,"last update":"Sat Sep 23 20:52:35 2023","interfaces":[{"interface":"back","average":{"1min":0.72699999999999998,"5min":1.5680000000000001,"15min":2.048},"min":{"1min":0.47799999999999998,"5min":0.46899999999999997,"15min":0.37},"max":{"1min":0.90300000000000002,"5min":17.433,"15min":17.433},"last":0.66700000000000004},{"interface":"front","average":{"1min":0.71499999999999997,"5min":1.7030000000000001,"15min":2.2069999999999999},"min":{"1min":0.48299999999999998,"5min":0.44500000000000001,"15min":0.35999999999999999},"max":{"1min":0.96699999999999997,"5min":9.8369999999999997,"15min":11.864000000000001},"last":0.54000000000000004}]}]},{"osd":0,"up_from":94,"seq":403726926218,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1004216,"kb_used_data":9008,"kb_used_omap":7,"kb_used_meta":995192,"kb_avail":92740936,"statfs":{"total":95995035648,"available":94966718464,"internally_reserved":0,"allocated":9224192,"data_stored":5969793,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":1019076625},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 20:52:36 2023","interfaces":[{"interface":"back","average":{"1min":0.34999999999999998,"5min":1.278,"15min":2.2040000000000002},"min":{"1min":0.25600000000000001,"5min":0.20100000000000001,"15min":0.184},"max":{"1min":0.46200000000000002,"5min":9.6449999999999996,"15min":152.387},"last":0.255},{"interface":"front","average":{"1min":0.41699999999999998,"5min":1.242,"15min":2.3340000000000001},"min":{"1min":0.23499999999999999,"5min":0.218,"15min":0.20399999999999999},"max":{"1min":0.54600000000000004,"5min":12.317,"15min":152.84100000000001},"last":0.33100000000000002}]},{"osd":2,"last update":"Sat Sep 23 20:52:13 2023","interfaces":[{"interface":"back","average":{"1min":0.35399999999999998,"5min":1.4970000000000001,"15min":2.4279999999999999},"min":{"1min":0.20799999999999999,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":0.51500000000000001,"5min":14.538,"15min":62.726999999999997},"last":0.27500000000000002},{"interface":"front","average":{"1min":0.38500000000000001,"5min":1.335,"15min":2.4140000000000001},"min":{"1min":0.28999999999999998,"5min":0.219,"15min":0.217},"max":{"1min":0.48099999999999998,"5min":10.159000000000001,"15min":62.689},"last":0.314}]},{"osd":3,"last update":"Sat Sep 23 20:51:47 2023","interfaces":[{"interface":"back","average":{"1min":0.68400000000000005,"5min":1.72,"15min":2.5939999999999999},"min":{"1min":0.27500000000000002,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":5.9329999999999998,"5min":14.494,"15min":55.017000000000003},"last":0.34100000000000003},{"interface":"front","average":{"1min":0.45100000000000001,"5min":1.7290000000000001,"15min":2.504},"min":{"1min":0.30299999999999999,"5min":0.28199999999999997,"15min":0.21199999999999999},"max":{"1min":0.71599999999999997,"5min":12.051,"15min":55.665999999999997},"last":0.29399999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:52:13 2023","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":1.45,"15min":1.7929999999999999},"min":{"1min":0.40300000000000002,"5min":0.29299999999999998,"15min":0.28899999999999998},"max":{"1min":0.78700000000000003,"5min":11.75,"15min":11.75},"last":0.55200000000000005},{"interface":"front","average":{"1min":0.58099999999999996,"5min":1.6499999999999999,"15min":1.7070000000000001},"min":{"1min":0.39200000000000002,"5min":0.31,"15min":0.27400000000000002},"max":{"1min":0.76100000000000001,"5min":12.356999999999999,"15min":12.356999999999999},"last":0.64800000000000002}]},{"osd":5,"last update":"Sat Sep 23 20:52:13 2023","interfaces":[{"interface":"back","average":{"1min":0.626,"5min":1.482,"15min":1.768},"min":{"1min":0.40799999999999997,"5min":0.34300000000000003,"15min":0.26500000000000001},"max":{"1min":0.88400000000000001,"5min":9.3339999999999996,"15min":10.239000000000001},"last":0.58999999999999997},{"interface":"front","average":{"1min":0.64600000000000002,"5min":1.331,"15min":2.0310000000000001},"min":{"1min":0.48599999999999999,"5min":0.33000000000000002,"15min":0.29799999999999999},"max":{"1min":0.82799999999999996,"5min":10.089,"15min":26.228000000000002},"last":0.63600000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:52:13 2023","interfaces":[{"interface":"back","average":{"1min":0.63700000000000001,"5min":1.708,"15min":1.9159999999999999},"min":{"1min":0.48899999999999999,"5min":0.40600000000000003,"15min":0.34499999999999997},"max":{"1min":0.91400000000000003,"5min":9.2750000000000004,"15min":24.056000000000001},"last":0.625},{"interface":"front","average":{"1min":0.63400000000000001,"5min":1.6000000000000001,"15min":1.984},"min":{"1min":0.443,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.90000000000000002,"5min":12.427,"15min":26.268000000000001},"last":0.60799999999999998}]},{"osd":7,"last update":"Sat Sep 23 20:52:13 2023","interfaces":[{"interface":"back","average":{"1min":0.623,"5min":1.468,"15min":2.1160000000000001},"min":{"1min":0.44600000000000001,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.86499999999999999,"5min":12.486000000000001,"15min":26.177},"last":0.65600000000000003},{"interface":"front","average":{"1min":0.64900000000000002,"5min":1.5369999999999999,"15min":2.0870000000000002},"min":{"1min":0.5,"5min":0.39200000000000002,"15min":0.34200000000000003},"max":{"1min":0.82299999999999995,"5min":10.183999999999999,"15min":16.649000000000001},"last":0.53300000000000003}]}]},{"osd":2,"up_from":104,"seq":446676599165,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":889016,"kb_used_data":12976,"kb_used_omap":11,"kb_used_meta":876020,"kb_avail":92856136,"statfs":{"total":95995035648,"available":95084683264,"internally_reserved":0,"allocated":13287424,"data_stored":10086340,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11878,"internal_metadata":897044890},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:52:15 2023","interfaces":[{"interface":"back","average":{"1min":0.36099999999999999,"5min":1.194,"15min":1.3360000000000001},"min":{"1min":0.23200000000000001,"5min":0.19500000000000001,"15min":0.17299999999999999},"max":{"1min":0.53200000000000003,"5min":20.48,"15min":20.48},"last":0.35899999999999999},{"interface":"front","average":{"1min":0.39300000000000002,"5min":1.1319999999999999,"15min":1.6479999999999999},"min":{"1min":0.28000000000000003,"5min":0.25700000000000001,"15min":0.23000000000000001},"max":{"1min":0.54400000000000004,"5min":8.3789999999999996,"15min":16.678999999999998},"last":0.44500000000000001}]},{"osd":1,"last update":"Sat Sep 23 20:52:15 2023","interfaces":[{"interface":"back","average":{"1min":0.36199999999999999,"5min":1.169,"15min":1.9530000000000001},"min":{"1min":0.254,"5min":0.23799999999999999,"15min":0.219},"max":{"1min":0.51100000000000001,"5min":11.012,"15min":46.405000000000001},"last":0.46400000000000002},{"interface":"front","average":{"1min":0.41399999999999998,"5min":1.1359999999999999,"15min":1.78},"min":{"1min":0.29699999999999999,"5min":0.254,"15min":0.22},"max":{"1min":0.67000000000000004,"5min":10.191000000000001,"15min":46.459000000000003},"last":0.434}]},{"osd":3,"last update":"Sat Sep 23 20:51:45 2023","interfaces":[{"interface":"back","average":{"1min":0.47199999999999998,"5min":1.3879999999999999,"15min":2.0720000000000001},"min":{"1min":0.33300000000000002,"5min":0.28199999999999997,"15min":0.253},"max":{"1min":1.3939999999999999,"5min":10.051,"15min":30.283000000000001},"last":0.41599999999999998},{"interface":"front","average":{"1min":0.48699999999999999,"5min":1.446,"15min":1.484},"min":{"1min":0.29499999999999998,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":1.321,"5min":20.562999999999999,"15min":30.834},"last":0.495}]},{"osd":4,"last update":"Sat Sep 23 20:52:15 2023","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":1.151,"15min":1.524},"min":{"1min":0.45200000000000001,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.84499999999999997,"5min":8.3469999999999995,"15min":12.025},"last":0.78100000000000003},{"interface":"front","average":{"1min":0.64600000000000002,"5min":1.306,"15min":1.627},"min":{"1min":0.503,"5min":0.32400000000000001,"15min":0.30599999999999999},"max":{"1min":0.84399999999999997,"5min":12.662000000000001,"15min":19.440999999999999},"last":0.66600000000000004}]},{"osd":5,"last update":"Sat Sep 23 20:52:15 2023","interfaces":[{"interface":"back","average":{"1min":0.66900000000000004,"5min":1.2270000000000001,"15min":1.554},"min":{"1min":0.41799999999999998,"5min":0.38400000000000001,"15min":0.311},"max":{"1min":0.86599999999999999,"5min":8.5380000000000003,"15min":12.07},"last":0.69199999999999995},{"interface":"front","average":{"1min":0.66600000000000004,"5min":1.3380000000000001,"15min":1.601},"min":{"1min":0.51900000000000002,"5min":0.373,"15min":0.37},"max":{"1min":0.82599999999999996,"5min":10.831,"15min":19.513000000000002},"last":0.76500000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:52:15 2023","interfaces":[{"interface":"back","average":{"1min":0.64400000000000002,"5min":1.3029999999999999,"15min":1.7350000000000001},"min":{"1min":0.48099999999999998,"5min":0.34799999999999998,"15min":0.33500000000000002},"max":{"1min":0.81599999999999995,"5min":10.791,"15min":19.486000000000001},"last":0.68500000000000005},{"interface":"front","average":{"1min":0.68999999999999995,"5min":1.1419999999999999,"15min":1.254},"min":{"1min":0.53000000000000003,"5min":0.40699999999999997,"15min":0.32900000000000001},"max":{"1min":0.82699999999999996,"5min":7.4180000000000001,"15min":11.747},"last":0.79500000000000004}]},{"osd":7,"last update":"Sat Sep 23 20:52:15 2023","interfaces":[{"interface":"back","average":{"1min":0.64100000000000001,"5min":1.506,"15min":1.373},"min":{"1min":0.47299999999999998,"5min":0.378,"15min":0.34699999999999998},"max":{"1min":0.88400000000000001,"5min":20.530000000000001,"15min":20.530000000000001},"last":0.77300000000000002},{"interface":"front","average":{"1min":0.64300000000000002,"5min":1.4219999999999999,"15min":1.778},"min":{"1min":0.52500000000000002,"5min":0.36899999999999999,"15min":0.34899999999999998},"max":{"1min":0.82599999999999996,"5min":10.071999999999999,"15min":12.099},"last":0.71999999999999997}]}]},{"osd":3,"up_from":109,"seq":468151435639,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1081580,"kb_used_data":8932,"kb_used_omap":4,"kb_used_meta":1072635,"kb_avail":92663572,"statfs":{"total":95995035648,"available":94887497728,"internally_reserved":0,"allocated":9146368,"data_stored":5956324,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4514,"internal_metadata":1098378846},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:51:52 2023","interfaces":[{"interface":"back","average":{"1min":0.379,"5min":1.3839999999999999,"15min":1.9350000000000001},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.20899999999999999},"max":{"1min":0.623,"5min":21.792999999999999,"15min":22.32},"last":0.34100000000000003},{"interface":"front","average":{"1min":0.42599999999999999,"5min":1.175,"15min":1.986},"min":{"1min":0.25800000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":0.64800000000000002,"5min":14.845000000000001,"15min":22.805},"last":0.38}]},{"osd":1,"last update":"Sat Sep 23 20:52:25 2023","interfaces":[{"interface":"back","average":{"1min":0.41199999999999998,"5min":1.0589999999999999,"15min":1.6359999999999999},"min":{"1min":0.27500000000000002,"5min":0.21299999999999999,"15min":0.21099999999999999},"max":{"1min":0.63800000000000001,"5min":11.298,"15min":17.529},"last":0.28199999999999997},{"interface":"front","average":{"1min":0.40999999999999998,"5min":1.0209999999999999,"15min":1.788},"min":{"1min":0.28899999999999998,"5min":0.26500000000000001,"15min":0.23699999999999999},"max":{"1min":0.61399999999999999,"5min":11.380000000000001,"15min":14.19},"last":0.41099999999999998}]},{"osd":2,"last update":"Sat Sep 23 20:51:52 2023","interfaces":[{"interface":"back","average":{"1min":0.442,"5min":1.268,"15min":2.3319999999999999},"min":{"1min":0.29499999999999998,"5min":0.254,"15min":0.23499999999999999},"max":{"1min":0.66100000000000003,"5min":7.9210000000000003,"15min":68.206999999999994},"last":0.32800000000000001},{"interface":"front","average":{"1min":0.40999999999999998,"5min":1.2909999999999999,"15min":2.077},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.245},"max":{"1min":0.63700000000000001,"5min":16.559000000000001,"15min":67.736999999999995},"last":0.35799999999999998}]},{"osd":4,"last update":"Sat Sep 23 20:51:52 2023","interfaces":[{"interface":"back","average":{"1min":0.62,"5min":1.343,"15min":1.5529999999999999},"min":{"1min":0.40699999999999997,"5min":0.35399999999999998,"15min":0.30499999999999999},"max":{"1min":0.97799999999999998,"5min":9.0259999999999998,"15min":22.157},"last":0.60199999999999998},{"interface":"front","average":{"1min":0.58599999999999997,"5min":1.4810000000000001,"15min":1.883},"min":{"1min":0.39000000000000001,"5min":0.33500000000000002,"15min":0.33000000000000002},"max":{"1min":0.86099999999999999,"5min":16.529,"15min":16.529},"last":0.70699999999999996}]},{"osd":5,"last update":"Sat Sep 23 20:51:52 2023","interfaces":[{"interface":"back","average":{"1min":0.61699999999999999,"5min":1.294,"15min":1.704},"min":{"1min":0.372,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.90600000000000003,"5min":8.9849999999999994,"15min":22.289999999999999},"last":0.53800000000000003},{"interface":"front","average":{"1min":0.63900000000000001,"5min":1.206,"15min":1.6060000000000001},"min":{"1min":0.442,"5min":0.36399999999999999,"15min":0.34100000000000003},"max":{"1min":0.88700000000000001,"5min":6.1390000000000002,"15min":12.952},"last":0.64000000000000001}]},{"osd":6,"last update":"Sat Sep 23 20:51:52 2023","interfaces":[{"interface":"back","average":{"1min":0.64100000000000001,"5min":1.5209999999999999,"15min":1.8240000000000001},"min":{"1min":0.41799999999999998,"5min":0.39800000000000002,"15min":0.34300000000000003},"max":{"1min":0.93600000000000005,"5min":7.4359999999999999,"15min":12.917},"last":0.55900000000000005},{"interface":"front","average":{"1min":0.61199999999999999,"5min":1.3640000000000001,"15min":1.9339999999999999},"min":{"1min":0.436,"5min":0.36699999999999999,"15min":0.34599999999999997},"max":{"1min":0.82999999999999996,"5min":16.497,"15min":16.497},"last":0.65200000000000002}]},{"osd":7,"last update":"Sat Sep 23 20:51:52 2023","interfaces":[{"interface":"back","average":{"1min":0.61399999999999999,"5min":1.4990000000000001,"15min":2.1320000000000001},"min":{"1min":0.47899999999999998,"5min":0.42399999999999999,"15min":0.39900000000000002},"max":{"1min":0.92300000000000004,"5min":16.425000000000001,"15min":16.425000000000001},"last":0.74099999999999999},{"interface":"front","average":{"1min":0.65500000000000003,"5min":1.7070000000000001,"15min":1.8680000000000001},"min":{"1min":0.496,"5min":0.45600000000000002,"15min":0.40999999999999998},"max":{"1min":0.95099999999999996,"5min":11.891999999999999,"15min":22.202999999999999},"last":0.57099999999999995}]}]},{"osd":4,"up_from":30,"seq":128849019392,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":572520,"kb_used_data":7072,"kb_used_omap":0,"kb_used_meta":565440,"kb_avail":93172632,"statfs":{"total":95995035648,"available":95408775168,"internally_reserved":0,"allocated":7241728,"data_stored":5446787,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":579010560},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:52:11 2023","interfaces":[{"interface":"back","average":{"1min":0.59499999999999997,"5min":1.8500000000000001,"15min":1.7350000000000001},"min":{"1min":0.42299999999999999,"5min":0.217,"15min":0.217},"max":{"1min":0.94199999999999995,"5min":73.272000000000006,"15min":73.272000000000006},"last":0.52700000000000002},{"interface":"front","average":{"1min":0.61499999999999999,"5min":1.508,"15min":1.7989999999999999},"min":{"1min":0.46200000000000002,"5min":0.25700000000000001,"15min":0.25},"max":{"1min":0.92900000000000005,"5min":29.754000000000001,"15min":29.754000000000001},"last":0.54300000000000004}]},{"osd":1,"last update":"Sat Sep 23 20:52:19 2023","interfaces":[{"interface":"back","average":{"1min":0.629,"5min":1.2589999999999999,"15min":1.5920000000000001},"min":{"1min":0.44400000000000001,"5min":0.26200000000000001,"15min":0.25900000000000001},"max":{"1min":0.91400000000000003,"5min":12.343,"15min":18.190999999999999},"last":0.501},{"interface":"front","average":{"1min":0.61099999999999999,"5min":1.4770000000000001,"15min":1.514},"min":{"1min":0.38600000000000001,"5min":0.28799999999999998,"15min":0.26400000000000001},"max":{"1min":0.88400000000000001,"5min":12.231999999999999,"15min":18.173999999999999},"last":0.39100000000000001}]},{"osd":2,"last update":"Sat Sep 23 20:52:19 2023","interfaces":[{"interface":"back","average":{"1min":0.77000000000000002,"5min":1.3859999999999999,"15min":1.5469999999999999},"min":{"1min":0.38300000000000001,"5min":0.26700000000000002,"15min":0.25800000000000001},"max":{"1min":3.7170000000000001,"5min":12.151999999999999,"15min":13.051},"last":0.44500000000000001},{"interface":"front","average":{"1min":0.60199999999999998,"5min":1.24,"15min":1.7869999999999999},"min":{"1min":0.371,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.90400000000000003,"5min":12.458,"15min":12.458},"last":0.46000000000000002}]},{"osd":3,"last update":"Sat Sep 23 20:52:37 2023","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":1.363,"15min":1.6839999999999999},"min":{"1min":0.41799999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.79700000000000004,"5min":11.598000000000001,"15min":12.147},"last":0.48199999999999998},{"interface":"front","average":{"1min":0.64300000000000002,"5min":1.45,"15min":1.7729999999999999},"min":{"1min":0.36699999999999999,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.98799999999999999,"5min":12.191000000000001,"15min":17.169},"last":0.58499999999999996}]},{"osd":5,"last update":"Sat Sep 23 20:52:11 2023","interfaces":[{"interface":"back","average":{"1min":0.49299999999999999,"5min":1.0309999999999999,"15min":1.171},"min":{"1min":0.32300000000000001,"5min":0.25800000000000001,"15min":0.253},"max":{"1min":0.76100000000000001,"5min":16.940000000000001,"15min":16.940000000000001},"last":0.64600000000000002},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.82199999999999995,"15min":1.163},"min":{"1min":0.32400000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.80800000000000005,"5min":4.915,"15min":8.1440000000000001},"last":0.60099999999999998}]},{"osd":6,"last update":"Sat Sep 23 20:52:19 2023","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":1.3480000000000001,"15min":1.397},"min":{"1min":0.38400000000000001,"5min":0.29099999999999998,"15min":0.28199999999999997},"max":{"1min":0.79900000000000004,"5min":13.503,"15min":14.731999999999999},"last":0.63200000000000001},{"interface":"front","average":{"1min":0.57299999999999995,"5min":1.4430000000000001,"15min":1.3460000000000001},"min":{"1min":0.35899999999999999,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.85199999999999998,"5min":17.059000000000001,"15min":17.059000000000001},"last":0.61099999999999999}]},{"osd":7,"last update":"Sat Sep 23 20:52:37 2023","interfaces":[{"interface":"back","average":{"1min":0.60599999999999998,"5min":0.94399999999999995,"15min":2.556},"min":{"1min":0.434,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.74299999999999999,"5min":11.228,"15min":313.50099999999998},"last":0.66400000000000003},{"interface":"front","average":{"1min":0.64500000000000002,"5min":1.0880000000000001,"15min":2.2989999999999999},"min":{"1min":0.40100000000000002,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.90800000000000003,"5min":17.093,"15min":312.67500000000001},"last":0.56799999999999995}]}]},{"osd":5,"up_from":39,"seq":167503725053,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":636388,"kb_used_data":11420,"kb_used_omap":0,"kb_used_meta":624960,"kb_avail":93108764,"statfs":{"total":95995035648,"available":95343374336,"internally_reserved":0,"allocated":11694080,"data_stored":9899865,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":639959040},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 20:52:21 2023","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":1.282,"15min":1.492},"min":{"1min":0.35699999999999998,"5min":0.22600000000000001,"15min":0.22500000000000001},"max":{"1min":0.80900000000000005,"5min":12.07,"15min":15.018000000000001},"last":0.41199999999999998},{"interface":"front","average":{"1min":0.56200000000000006,"5min":1.274,"15min":1.5009999999999999},"min":{"1min":0.38500000000000001,"5min":0.29099999999999998,"15min":0.22700000000000001},"max":{"1min":0.79000000000000004,"5min":12.089,"15min":15.034000000000001},"last":0.48699999999999999}]},{"osd":1,"last update":"Sat Sep 23 20:52:37 2023","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":1.2769999999999999,"15min":2.0129999999999999},"min":{"1min":0.36199999999999999,"5min":0.252,"15min":0.252},"max":{"1min":0.84699999999999998,"5min":10.5,"15min":61.005000000000003},"last":0.51200000000000001},{"interface":"front","average":{"1min":0.628,"5min":1.508,"15min":1.8540000000000001},"min":{"1min":0.41499999999999998,"5min":0.34000000000000002,"15min":0.27700000000000002},"max":{"1min":0.78900000000000003,"5min":10.340999999999999,"15min":59.688000000000002},"last":0.628}]},{"osd":2,"last update":"Sat Sep 23 20:52:21 2023","interfaces":[{"interface":"back","average":{"1min":0.59499999999999997,"5min":1.373,"15min":1.736},"min":{"1min":0.39800000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.85099999999999998,"5min":7.194,"15min":12.226000000000001},"last":0.54400000000000004},{"interface":"front","average":{"1min":0.52300000000000002,"5min":1.2969999999999999,"15min":1.5289999999999999},"min":{"1min":0.33000000000000002,"5min":0.31,"15min":0.28299999999999997},"max":{"1min":0.74099999999999999,"5min":7.5599999999999996,"15min":10.565},"last":0.433}]},{"osd":3,"last update":"Sat Sep 23 20:52:37 2023","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":1.2230000000000001,"15min":1.7250000000000001},"min":{"1min":0.40400000000000003,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.78500000000000003,"5min":8.7360000000000007,"15min":11.032},"last":0.38800000000000001},{"interface":"front","average":{"1min":0.623,"5min":1.331,"15min":1.895},"min":{"1min":0.42899999999999999,"5min":0.317,"15min":0.317},"max":{"1min":0.78700000000000003,"5min":7.4029999999999996,"15min":18.504000000000001},"last":0.57699999999999996}]},{"osd":4,"last update":"Sat Sep 23 20:52:13 2023","interfaces":[{"interface":"back","average":{"1min":0.47299999999999998,"5min":1.087,"15min":1.3109999999999999},"min":{"1min":0.316,"5min":0.27700000000000002,"15min":0.255},"max":{"1min":0.77000000000000002,"5min":8.9390000000000001,"15min":13.391999999999999},"last":0.34200000000000003},{"interface":"front","average":{"1min":0.53100000000000003,"5min":1.0449999999999999,"15min":1.288},"min":{"1min":0.35399999999999998,"5min":0.31,"15min":0.28299999999999997},"max":{"1min":0.85399999999999998,"5min":8.0030000000000001,"15min":8.0030000000000001},"last":0.46700000000000003}]},{"osd":6,"last update":"Sat Sep 23 20:52:21 2023","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":1.1379999999999999,"15min":1.486},"min":{"1min":0.41399999999999998,"5min":0.313,"15min":0.313},"max":{"1min":0.82199999999999995,"5min":8.9039999999999999,"15min":13.734},"last":0.45000000000000001},{"interface":"front","average":{"1min":0.69099999999999995,"5min":1.28,"15min":1.339},"min":{"1min":0.40899999999999997,"5min":0.34999999999999998,"15min":0.317},"max":{"1min":0.92900000000000005,"5min":9.6519999999999992,"15min":9.6519999999999992},"last":0.60499999999999998}]},{"osd":7,"last update":"Sat Sep 23 20:52:37 2023","interfaces":[{"interface":"back","average":{"1min":0.65000000000000002,"5min":1.081,"15min":2.2589999999999999},"min":{"1min":0.41899999999999998,"5min":0.34399999999999997,"15min":0.33400000000000002},"max":{"1min":0.78000000000000003,"5min":7.899,"15min":294.70999999999998},"last":0.501},{"interface":"front","average":{"1min":0.60799999999999998,"5min":0.97799999999999998,"15min":2.2360000000000002},"min":{"1min":0.42299999999999999,"5min":0.316,"15min":0.27900000000000003},"max":{"1min":0.78500000000000003,"5min":14.032999999999999,"15min":294.74400000000003},"last":0.53000000000000003}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4722,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1360,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":8392704,"data_stored":8388627,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8175,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8206,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7156,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3154,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T20:52:49.487 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-09-23T20:52:49.488 ERROR:teuthology.run_tasks:Saw exception from tasks. Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/parallel.py", line 56, in task p.spawn(_run_spawned, ctx, confg, taskname) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/parallel.py", line 64, in _run_spawned mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/sequential.py", line 47, in task mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=confg) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/github.com_ceph_ceph_bb0daeff97f020f686596fca6eba20c310e26fb7/qa/tasks/workunit.py", line 119, in task assert 'client' in role AssertionError 2023-09-23T20:52:49.548 ERROR:teuthology.util.sentry: Sentry event: https://sentry.ceph.com/organizations/ceph/?query=166473fa1bc741e9b478cec54cc4eea3 Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/parallel.py", line 56, in task p.spawn(_run_spawned, ctx, confg, taskname) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/parallel.py", line 64, in _run_spawned mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/sequential.py", line 47, in task mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=confg) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/github.com_ceph_ceph_bb0daeff97f020f686596fca6eba20c310e26fb7/qa/tasks/workunit.py", line 119, in task assert 'client' in role AssertionError 2023-09-23T20:52:49.550 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2023-09-23T20:52:49.560 INFO:tasks.cephadm:Teardown begin 2023-09-23T20:52:49.561 DEBUG:teuthology.orchestra.run.smithi018:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-09-23T20:52:49.632 DEBUG:teuthology.orchestra.run.smithi149:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-09-23T20:52:49.702 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2023-09-23T20:52:49.702 DEBUG:teuthology.orchestra.run.smithi018:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2023-09-23T20:52:49.720 DEBUG:teuthology.orchestra.run.smithi149:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2023-09-23T20:52:49.739 INFO:tasks.cephadm:Stopping all daemons... 2023-09-23T20:52:49.740 INFO:tasks.cephadm.mon.a:Stopping mon.a... 2023-09-23T20:52:49.740 DEBUG:teuthology.orchestra.run.smithi018:> sudo systemctl stop ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7@mon.a 2023-09-23T20:52:49.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:52:49 smithi149 ceph-mon[170394]: pgmap v1712: 103 pgs: 103 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-09-23T20:52:49.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:52:49 smithi149 ceph-mon[170394]: from='client.48143 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:52:50.126 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:52:49 smithi018 ceph-mon[184144]: pgmap v1712: 103 pgs: 103 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-09-23T20:52:50.126 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:52:49 smithi018 ceph-mon[184144]: from='client.48143 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:52:50.128 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:52:49 smithi018 systemd[1]: Stopping Ceph mon.a for a3b3b17e-5a4c-11ee-8db4-212e2dc638e7... 2023-09-23T20:52:50.128 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:52:49 smithi018 ceph-mon[179647]: pgmap v1712: 103 pgs: 103 active+clean; 8.6 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2023-09-23T20:52:50.128 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:52:49 smithi018 ceph-mon[179647]: from='client.48143 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T20:52:50.128 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:52:50 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mon-a[179623]: 2023-09-23T20:52:50.124+0000 7f1615704700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T20:52:50.128 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:52:50 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mon-a[179623]: 2023-09-23T20:52:50.124+0000 7f1615704700 -1 mon.a@0(leader) e4 *** Got Signal Terminated *** 2023-09-23T20:52:50.391 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:52:50 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: [23/Sep/2023:20:52:50] ENGINE Bus STOPPING 2023-09-23T20:52:50.391 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:52:50 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: [23/Sep/2023:20:52:50] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T20:52:50.391 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:52:50 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: [23/Sep/2023:20:52:50] ENGINE Bus STOPPED 2023-09-23T20:52:50.391 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:52:50 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: [23/Sep/2023:20:52:50] ENGINE Bus STARTING 2023-09-23T20:52:50.392 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:52:50 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: [23/Sep/2023:20:52:50] ENGINE Serving on http://:::9283 2023-09-23T20:52:50.392 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:52:50 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: [23/Sep/2023:20:52:50] ENGINE Bus STARTED 2023-09-23T20:52:50.885 INFO:journalctl@ceph.mon.a.smithi018.stdout:Sep 23 20:52:50 smithi018 bash[492265]: ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mon-a 2023-09-23T20:52:50.892 DEBUG:teuthology.orchestra.run.smithi018:> sudo pkill -f 'journalctl -f -n 0 -u ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7@mon.a.service' 2023-09-23T20:52:50.941 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T20:52:50.941 INFO:tasks.cephadm.mon.a:Stopped mon.a 2023-09-23T20:52:50.941 INFO:tasks.cephadm.mon.b:Stopping mon.c... 2023-09-23T20:52:50.941 DEBUG:teuthology.orchestra.run.smithi018:> sudo systemctl stop ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7@mon.c 2023-09-23T20:52:51.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:52:51 smithi018 systemd[1]: Stopping Ceph mon.c for a3b3b17e-5a4c-11ee-8db4-212e2dc638e7... 2023-09-23T20:52:51.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:52:51 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mon-c[184100]: 2023-09-23T20:52:51.328+0000 7fdf454b5700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T20:52:51.641 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:52:51 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mon-c[184100]: 2023-09-23T20:52:51.328+0000 7fdf454b5700 -1 mon.c@2(peon) e4 *** Got Signal Terminated *** 2023-09-23T20:52:52.141 INFO:journalctl@ceph.mon.c.smithi018.stdout:Sep 23 20:52:51 smithi018 bash[492424]: ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mon-c 2023-09-23T20:52:52.218 DEBUG:teuthology.orchestra.run.smithi018:> sudo pkill -f 'journalctl -f -n 0 -u ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7@mon.c.service' 2023-09-23T20:52:52.257 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T20:52:52.257 INFO:tasks.cephadm.mon.b:Stopped mon.c 2023-09-23T20:52:52.257 INFO:tasks.cephadm.mon.b:Stopping mon.b... 2023-09-23T20:52:52.257 DEBUG:teuthology.orchestra.run.smithi149:> sudo systemctl stop ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7@mon.b 2023-09-23T20:52:52.628 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:52:52 smithi149 systemd[1]: Stopping Ceph mon.b for a3b3b17e-5a4c-11ee-8db4-212e2dc638e7... 2023-09-23T20:52:52.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:52:52 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mon-b[170370]: 2023-09-23T20:52:52.626+0000 7f2f678dd700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T20:52:52.966 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:52:52 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mon-b[170370]: 2023-09-23T20:52:52.626+0000 7f2f678dd700 -1 mon.b@1(peon) e4 *** Got Signal Terminated *** 2023-09-23T20:52:53.282 DEBUG:teuthology.orchestra.run.smithi149:> sudo pkill -f 'journalctl -f -n 0 -u ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7@mon.b.service' 2023-09-23T20:52:53.319 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:52:53 smithi149 bash[308523]: ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mon-b 2023-09-23T20:52:53.320 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:52:53 smithi149 systemd[1]: ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7@mon.b.service: Succeeded. 2023-09-23T20:52:53.320 INFO:journalctl@ceph.mon.b.smithi149.stdout:Sep 23 20:52:53 smithi149 systemd[1]: Stopped Ceph mon.b for a3b3b17e-5a4c-11ee-8db4-212e2dc638e7. 2023-09-23T20:52:53.349 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T20:52:53.349 INFO:tasks.cephadm.mon.b:Stopped mon.b 2023-09-23T20:52:53.349 INFO:tasks.cephadm.mgr.y:Stopping mgr.y... 2023-09-23T20:52:53.350 DEBUG:teuthology.orchestra.run.smithi018:> sudo systemctl stop ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7@mgr.y 2023-09-23T20:52:53.641 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:52:53 smithi018 systemd[1]: Stopping Ceph mgr.y for a3b3b17e-5a4c-11ee-8db4-212e2dc638e7... 2023-09-23T20:52:54.034 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:52:53 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y[167794]: 2023-09-23T20:52:53.707+0000 7f95771d8700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mgr -n mgr.y -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T20:52:54.258 DEBUG:teuthology.orchestra.run.smithi018:> sudo pkill -f 'journalctl -f -n 0 -u ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7@mgr.y.service' 2023-09-23T20:52:54.285 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:52:54 smithi018 bash[492569]: ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-y 2023-09-23T20:52:54.285 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:52:54 smithi018 systemd[1]: ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7@mgr.y.service: Succeeded. 2023-09-23T20:52:54.285 INFO:journalctl@ceph.mgr.y.smithi018.stdout:Sep 23 20:52:54 smithi018 systemd[1]: Stopped Ceph mgr.y for a3b3b17e-5a4c-11ee-8db4-212e2dc638e7. 2023-09-23T20:52:54.297 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T20:52:54.298 INFO:tasks.cephadm.mgr.y:Stopped mgr.y 2023-09-23T20:52:54.298 INFO:tasks.cephadm.mgr.x:Stopping mgr.x... 2023-09-23T20:52:54.298 DEBUG:teuthology.orchestra.run.smithi149:> sudo systemctl stop ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7@mgr.x 2023-09-23T20:52:54.716 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:52:54 smithi149 systemd[1]: Stopping Ceph mgr.x for a3b3b17e-5a4c-11ee-8db4-212e2dc638e7... 2023-09-23T20:52:55.110 INFO:journalctl@ceph.mgr.x.smithi149.stdout:Sep 23 20:52:55 smithi149 bash[308663]: ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-mgr-x 2023-09-23T20:52:55.297 DEBUG:teuthology.orchestra.run.smithi149:> sudo pkill -f 'journalctl -f -n 0 -u ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7@mgr.x.service' 2023-09-23T20:52:55.336 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T20:52:55.336 INFO:tasks.cephadm.mgr.x:Stopped mgr.x 2023-09-23T20:52:55.336 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2023-09-23T20:52:55.336 DEBUG:teuthology.orchestra.run.smithi018:> sudo systemctl stop ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7@osd.0 2023-09-23T20:52:55.641 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 23 20:52:55 smithi018 systemd[1]: Stopping Ceph osd.0 for a3b3b17e-5a4c-11ee-8db4-212e2dc638e7... 2023-09-23T20:52:56.141 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 23 20:52:55 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-0[194922]: 2023-09-23T20:52:55.692+0000 7f5e28855700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T20:52:56.141 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 23 20:52:55 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-0[194922]: 2023-09-23T20:52:55.692+0000 7f5e28855700 -1 osd.0 695 *** Got signal Terminated *** 2023-09-23T20:52:56.141 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 23 20:52:55 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-0[194922]: 2023-09-23T20:52:55.692+0000 7f5e28855700 -1 osd.0 695 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T20:53:01.891 INFO:journalctl@ceph.osd.0.smithi018.stdout:Sep 23 20:53:01 smithi018 bash[492708]: ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-0 2023-09-23T20:53:03.227 DEBUG:teuthology.orchestra.run.smithi018:> sudo pkill -f 'journalctl -f -n 0 -u ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7@osd.0.service' 2023-09-23T20:53:03.270 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T20:53:03.270 INFO:tasks.cephadm.osd.0:Stopped osd.0 2023-09-23T20:53:03.270 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2023-09-23T20:53:03.270 DEBUG:teuthology.orchestra.run.smithi018:> sudo systemctl stop ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7@osd.1 2023-09-23T20:53:03.641 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 23 20:53:03 smithi018 systemd[1]: Stopping Ceph osd.1 for a3b3b17e-5a4c-11ee-8db4-212e2dc638e7... 2023-09-23T20:53:03.954 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 23 20:53:03 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-1[289056]: 2023-09-23T20:53:03.680+0000 7f73d5457700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T20:53:03.955 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 23 20:53:03 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-1[289056]: 2023-09-23T20:53:03.680+0000 7f73d5457700 -1 osd.1 695 *** Got signal Terminated *** 2023-09-23T20:53:03.955 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 23 20:53:03 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-1[289056]: 2023-09-23T20:53:03.680+0000 7f73d5457700 -1 osd.1 695 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T20:53:10.142 INFO:journalctl@ceph.osd.1.smithi018.stdout:Sep 23 20:53:09 smithi018 bash[492980]: ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-1 2023-09-23T20:53:11.119 DEBUG:teuthology.orchestra.run.smithi018:> sudo pkill -f 'journalctl -f -n 0 -u ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7@osd.1.service' 2023-09-23T20:53:11.161 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T20:53:11.161 INFO:tasks.cephadm.osd.1:Stopped osd.1 2023-09-23T20:53:11.162 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2023-09-23T20:53:11.162 DEBUG:teuthology.orchestra.run.smithi018:> sudo systemctl stop ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7@osd.2 2023-09-23T20:53:11.391 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 23 20:53:11 smithi018 systemd[1]: Stopping Ceph osd.2 for a3b3b17e-5a4c-11ee-8db4-212e2dc638e7... 2023-09-23T20:53:11.891 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 23 20:53:11 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-2[207186]: 2023-09-23T20:53:11.540+0000 7f16793ad700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T20:53:11.891 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 23 20:53:11 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-2[207186]: 2023-09-23T20:53:11.540+0000 7f16793ad700 -1 osd.2 695 *** Got signal Terminated *** 2023-09-23T20:53:11.891 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 23 20:53:11 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-2[207186]: 2023-09-23T20:53:11.540+0000 7f16793ad700 -1 osd.2 695 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T20:53:17.641 INFO:journalctl@ceph.osd.2.smithi018.stdout:Sep 23 20:53:17 smithi018 bash[493247]: ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-2 2023-09-23T20:53:18.683 DEBUG:teuthology.orchestra.run.smithi018:> sudo pkill -f 'journalctl -f -n 0 -u ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7@osd.2.service' 2023-09-23T20:53:18.726 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T20:53:18.726 INFO:tasks.cephadm.osd.2:Stopped osd.2 2023-09-23T20:53:18.726 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2023-09-23T20:53:18.726 DEBUG:teuthology.orchestra.run.smithi018:> sudo systemctl stop ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7@osd.3 2023-09-23T20:53:19.107 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 23 20:53:18 smithi018 systemd[1]: Stopping Ceph osd.3 for a3b3b17e-5a4c-11ee-8db4-212e2dc638e7... 2023-09-23T20:53:19.391 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 23 20:53:19 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-3[213874]: 2023-09-23T20:53:19.106+0000 7f837b019700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T20:53:19.391 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 23 20:53:19 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-3[213874]: 2023-09-23T20:53:19.106+0000 7f837b019700 -1 osd.3 695 *** Got signal Terminated *** 2023-09-23T20:53:19.391 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 23 20:53:19 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-3[213874]: 2023-09-23T20:53:19.106+0000 7f837b019700 -1 osd.3 695 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T20:53:23.216 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 23 20:53:22 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-5[114166]: 2023-09-23T20:53:22.886+0000 7f116e046700 -1 osd.5 695 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2023-09-23T20:52:56.475955+0000 front 2023-09-23T20:52:56.475941+0000 (oldest deadline 2023-09-23T20:53:22.375916+0000) 2023-09-23T20:53:24.216 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 23 20:53:23 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-5[114166]: 2023-09-23T20:53:23.855+0000 7f116e046700 -1 osd.5 695 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2023-09-23T20:52:56.475955+0000 front 2023-09-23T20:52:56.475941+0000 (oldest deadline 2023-09-23T20:53:22.375916+0000) 2023-09-23T20:53:24.716 INFO:journalctl@ceph.osd.4.smithi149.stdout:Sep 23 20:53:24 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-4[106331]: 2023-09-23T20:53:24.359+0000 7fbc1d9d3700 -1 osd.4 695 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2023-09-23T20:52:59.914534+0000 front 2023-09-23T20:52:59.914514+0000 (oldest deadline 2023-09-23T20:53:24.014364+0000) 2023-09-23T20:53:24.891 INFO:journalctl@ceph.osd.3.smithi018.stdout:Sep 23 20:53:24 smithi018 bash[493519]: ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-3 2023-09-23T20:53:25.216 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 23 20:53:24 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-5[114166]: 2023-09-23T20:53:24.835+0000 7f116e046700 -1 osd.5 695 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2023-09-23T20:52:56.475955+0000 front 2023-09-23T20:52:56.475941+0000 (oldest deadline 2023-09-23T20:53:22.375916+0000) 2023-09-23T20:53:25.716 INFO:journalctl@ceph.osd.4.smithi149.stdout:Sep 23 20:53:25 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-4[106331]: 2023-09-23T20:53:25.399+0000 7fbc1d9d3700 -1 osd.4 695 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2023-09-23T20:52:59.914534+0000 front 2023-09-23T20:52:59.914514+0000 (oldest deadline 2023-09-23T20:53:24.014364+0000) 2023-09-23T20:53:26.090 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 23 20:53:25 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-5[114166]: 2023-09-23T20:53:25.807+0000 7f116e046700 -1 osd.5 695 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2023-09-23T20:52:56.475955+0000 front 2023-09-23T20:52:56.475941+0000 (oldest deadline 2023-09-23T20:53:22.375916+0000) 2023-09-23T20:53:26.275 DEBUG:teuthology.orchestra.run.smithi018:> sudo pkill -f 'journalctl -f -n 0 -u ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7@osd.3.service' 2023-09-23T20:53:26.318 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T20:53:26.318 INFO:tasks.cephadm.osd.3:Stopped osd.3 2023-09-23T20:53:26.319 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2023-09-23T20:53:26.319 DEBUG:teuthology.orchestra.run.smithi149:> sudo systemctl stop ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7@osd.4 2023-09-23T20:53:26.350 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 23 20:53:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-6[121170]: 2023-09-23T20:53:26.089+0000 7f82c6191700 -1 osd.6 695 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2023-09-23T20:53:00.273630+0000 front 2023-09-23T20:53:00.273539+0000 (oldest deadline 2023-09-23T20:53:25.573656+0000) 2023-09-23T20:53:26.688 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 23 20:53:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-7[128937]: 2023-09-23T20:53:26.435+0000 7f14eaa6a700 -1 osd.7 695 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2023-09-23T20:52:59.913344+0000 front 2023-09-23T20:52:59.913028+0000 (oldest deadline 2023-09-23T20:53:25.713711+0000) 2023-09-23T20:53:26.689 INFO:journalctl@ceph.osd.4.smithi149.stdout:Sep 23 20:53:26 smithi149 systemd[1]: Stopping Ceph osd.4 for a3b3b17e-5a4c-11ee-8db4-212e2dc638e7... 2023-09-23T20:53:26.689 INFO:journalctl@ceph.osd.4.smithi149.stdout:Sep 23 20:53:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-4[106331]: 2023-09-23T20:53:26.405+0000 7fbc1d9d3700 -1 osd.4 695 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2023-09-23T20:52:59.914534+0000 front 2023-09-23T20:52:59.914514+0000 (oldest deadline 2023-09-23T20:53:24.014364+0000) 2023-09-23T20:53:26.689 INFO:journalctl@ceph.osd.4.smithi149.stdout:Sep 23 20:53:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-4[106331]: 2023-09-23T20:53:26.688+0000 7fbc223d7700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T20:53:26.689 INFO:journalctl@ceph.osd.4.smithi149.stdout:Sep 23 20:53:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-4[106331]: 2023-09-23T20:53:26.688+0000 7fbc223d7700 -1 osd.4 695 *** Got signal Terminated *** 2023-09-23T20:53:26.966 INFO:journalctl@ceph.osd.4.smithi149.stdout:Sep 23 20:53:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-4[106331]: 2023-09-23T20:53:26.688+0000 7fbc223d7700 -1 osd.4 695 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T20:53:26.966 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 23 20:53:26 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-5[114166]: 2023-09-23T20:53:26.828+0000 7f116e046700 -1 osd.5 695 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2023-09-23T20:52:56.475955+0000 front 2023-09-23T20:52:56.475941+0000 (oldest deadline 2023-09-23T20:53:22.375916+0000) 2023-09-23T20:53:27.368 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 23 20:53:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-6[121170]: 2023-09-23T20:53:27.110+0000 7f82c6191700 -1 osd.6 695 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2023-09-23T20:53:00.273630+0000 front 2023-09-23T20:53:00.273539+0000 (oldest deadline 2023-09-23T20:53:25.573656+0000) 2023-09-23T20:53:27.716 INFO:journalctl@ceph.osd.4.smithi149.stdout:Sep 23 20:53:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-4[106331]: 2023-09-23T20:53:27.368+0000 7fbc1d9d3700 -1 osd.4 695 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2023-09-23T20:52:59.914534+0000 front 2023-09-23T20:52:59.914514+0000 (oldest deadline 2023-09-23T20:53:24.014364+0000) 2023-09-23T20:53:27.716 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 23 20:53:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-7[128937]: 2023-09-23T20:53:27.393+0000 7f14eaa6a700 -1 osd.7 695 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2023-09-23T20:52:59.913344+0000 front 2023-09-23T20:52:59.913028+0000 (oldest deadline 2023-09-23T20:53:25.713711+0000) 2023-09-23T20:53:28.107 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 23 20:53:27 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-5[114166]: 2023-09-23T20:53:27.781+0000 7f116e046700 -1 osd.5 695 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2023-09-23T20:52:56.475955+0000 front 2023-09-23T20:52:56.475941+0000 (oldest deadline 2023-09-23T20:53:22.375916+0000) 2023-09-23T20:53:28.363 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 23 20:53:28 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-6[121170]: 2023-09-23T20:53:28.107+0000 7f82c6191700 -1 osd.6 695 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2023-09-23T20:53:00.273630+0000 front 2023-09-23T20:53:00.273539+0000 (oldest deadline 2023-09-23T20:53:25.573656+0000) 2023-09-23T20:53:28.716 INFO:journalctl@ceph.osd.4.smithi149.stdout:Sep 23 20:53:28 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-4[106331]: 2023-09-23T20:53:28.362+0000 7fbc1d9d3700 -1 osd.4 695 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2023-09-23T20:52:59.914534+0000 front 2023-09-23T20:52:59.914514+0000 (oldest deadline 2023-09-23T20:53:24.014364+0000) 2023-09-23T20:53:28.716 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 23 20:53:28 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-7[128937]: 2023-09-23T20:53:28.394+0000 7f14eaa6a700 -1 osd.7 695 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2023-09-23T20:52:59.913344+0000 front 2023-09-23T20:52:59.913028+0000 (oldest deadline 2023-09-23T20:53:25.713711+0000) 2023-09-23T20:53:29.118 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 23 20:53:28 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-5[114166]: 2023-09-23T20:53:28.755+0000 7f116e046700 -1 osd.5 695 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2023-09-23T20:52:56.475955+0000 front 2023-09-23T20:52:56.475941+0000 (oldest deadline 2023-09-23T20:53:22.375916+0000) 2023-09-23T20:53:29.382 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 23 20:53:29 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-6[121170]: 2023-09-23T20:53:29.118+0000 7f82c6191700 -1 osd.6 695 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2023-09-23T20:53:00.273630+0000 front 2023-09-23T20:53:00.273539+0000 (oldest deadline 2023-09-23T20:53:25.573656+0000) 2023-09-23T20:53:29.716 INFO:journalctl@ceph.osd.4.smithi149.stdout:Sep 23 20:53:29 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-4[106331]: 2023-09-23T20:53:29.381+0000 7fbc1d9d3700 -1 osd.4 695 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2023-09-23T20:52:59.914534+0000 front 2023-09-23T20:52:59.914514+0000 (oldest deadline 2023-09-23T20:53:24.014364+0000) 2023-09-23T20:53:29.716 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 23 20:53:29 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-7[128937]: 2023-09-23T20:53:29.410+0000 7f14eaa6a700 -1 osd.7 695 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2023-09-23T20:52:59.913344+0000 front 2023-09-23T20:52:59.913028+0000 (oldest deadline 2023-09-23T20:53:25.713711+0000) 2023-09-23T20:53:30.142 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 23 20:53:29 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-5[114166]: 2023-09-23T20:53:29.803+0000 7f116e046700 -1 osd.5 695 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2023-09-23T20:52:56.475955+0000 front 2023-09-23T20:52:56.475941+0000 (oldest deadline 2023-09-23T20:53:22.375916+0000) 2023-09-23T20:53:30.466 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 23 20:53:30 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-6[121170]: 2023-09-23T20:53:30.142+0000 7f82c6191700 -1 osd.6 695 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2023-09-23T20:53:00.273630+0000 front 2023-09-23T20:53:00.273539+0000 (oldest deadline 2023-09-23T20:53:25.573656+0000) 2023-09-23T20:53:30.466 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 23 20:53:30 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-7[128937]: 2023-09-23T20:53:30.376+0000 7f14eaa6a700 -1 osd.7 695 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2023-09-23T20:52:59.913344+0000 front 2023-09-23T20:52:59.913028+0000 (oldest deadline 2023-09-23T20:53:25.713711+0000) 2023-09-23T20:53:30.467 INFO:journalctl@ceph.osd.4.smithi149.stdout:Sep 23 20:53:30 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-4[106331]: 2023-09-23T20:53:30.353+0000 7fbc1d9d3700 -1 osd.4 695 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2023-09-23T20:52:59.914534+0000 front 2023-09-23T20:52:59.914514+0000 (oldest deadline 2023-09-23T20:53:24.014364+0000) 2023-09-23T20:53:31.101 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 23 20:53:30 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-5[114166]: 2023-09-23T20:53:30.847+0000 7f116e046700 -1 osd.5 695 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2023-09-23T20:52:56.475955+0000 front 2023-09-23T20:52:56.475941+0000 (oldest deadline 2023-09-23T20:53:22.375916+0000) 2023-09-23T20:53:31.466 INFO:journalctl@ceph.osd.4.smithi149.stdout:Sep 23 20:53:31 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-4[106331]: 2023-09-23T20:53:31.320+0000 7fbc1d9d3700 -1 osd.4 695 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2023-09-23T20:52:59.914534+0000 front 2023-09-23T20:52:59.914514+0000 (oldest deadline 2023-09-23T20:53:24.014364+0000) 2023-09-23T20:53:31.466 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 23 20:53:31 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-6[121170]: 2023-09-23T20:53:31.101+0000 7f82c6191700 -1 osd.6 695 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2023-09-23T20:53:00.273630+0000 front 2023-09-23T20:53:00.273539+0000 (oldest deadline 2023-09-23T20:53:25.573656+0000) 2023-09-23T20:53:31.467 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 23 20:53:31 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-7[128937]: 2023-09-23T20:53:31.332+0000 7f14eaa6a700 -1 osd.7 695 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2023-09-23T20:52:59.913344+0000 front 2023-09-23T20:52:59.913028+0000 (oldest deadline 2023-09-23T20:53:25.713711+0000) 2023-09-23T20:53:31.467 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 23 20:53:31 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-7[128937]: 2023-09-23T20:53:31.332+0000 7f14eaa6a700 -1 osd.7 695 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2023-09-23T20:53:05.714313+0000 front 2023-09-23T20:53:05.714276+0000 (oldest deadline 2023-09-23T20:53:30.414093+0000) 2023-09-23T20:53:32.146 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 23 20:53:31 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-5[114166]: 2023-09-23T20:53:31.809+0000 7f116e046700 -1 osd.5 695 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2023-09-23T20:52:56.475955+0000 front 2023-09-23T20:52:56.475941+0000 (oldest deadline 2023-09-23T20:53:22.375916+0000) 2023-09-23T20:53:32.146 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 23 20:53:31 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-5[114166]: 2023-09-23T20:53:31.809+0000 7f116e046700 -1 osd.5 695 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2023-09-23T20:53:07.478202+0000 front 2023-09-23T20:53:07.478211+0000 (oldest deadline 2023-09-23T20:53:30.977995+0000) 2023-09-23T20:53:32.428 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 23 20:53:32 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-6[121170]: 2023-09-23T20:53:32.145+0000 7f82c6191700 -1 osd.6 695 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2023-09-23T20:53:00.273630+0000 front 2023-09-23T20:53:00.273539+0000 (oldest deadline 2023-09-23T20:53:25.573656+0000) 2023-09-23T20:53:32.428 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 23 20:53:32 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-7[128937]: 2023-09-23T20:53:32.323+0000 7f14eaa6a700 -1 osd.7 695 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2023-09-23T20:52:59.913344+0000 front 2023-09-23T20:52:59.913028+0000 (oldest deadline 2023-09-23T20:53:25.713711+0000) 2023-09-23T20:53:32.428 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 23 20:53:32 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-7[128937]: 2023-09-23T20:53:32.323+0000 7f14eaa6a700 -1 osd.7 695 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2023-09-23T20:53:05.714313+0000 front 2023-09-23T20:53:05.714276+0000 (oldest deadline 2023-09-23T20:53:30.414093+0000) 2023-09-23T20:53:32.716 INFO:journalctl@ceph.osd.4.smithi149.stdout:Sep 23 20:53:32 smithi149 bash[308805]: ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-4 2023-09-23T20:53:33.167 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 23 20:53:32 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-5[114166]: 2023-09-23T20:53:32.807+0000 7f116e046700 -1 osd.5 695 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2023-09-23T20:52:56.475955+0000 front 2023-09-23T20:52:56.475941+0000 (oldest deadline 2023-09-23T20:53:22.375916+0000) 2023-09-23T20:53:33.167 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 23 20:53:32 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-5[114166]: 2023-09-23T20:53:32.807+0000 7f116e046700 -1 osd.5 695 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2023-09-23T20:53:07.478202+0000 front 2023-09-23T20:53:07.478211+0000 (oldest deadline 2023-09-23T20:53:30.977995+0000) 2023-09-23T20:53:33.466 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 23 20:53:33 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-7[128937]: 2023-09-23T20:53:33.276+0000 7f14eaa6a700 -1 osd.7 695 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2023-09-23T20:52:59.913344+0000 front 2023-09-23T20:52:59.913028+0000 (oldest deadline 2023-09-23T20:53:25.713711+0000) 2023-09-23T20:53:33.466 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 23 20:53:33 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-7[128937]: 2023-09-23T20:53:33.276+0000 7f14eaa6a700 -1 osd.7 695 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2023-09-23T20:53:05.714313+0000 front 2023-09-23T20:53:05.714276+0000 (oldest deadline 2023-09-23T20:53:30.414093+0000) 2023-09-23T20:53:33.467 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 23 20:53:33 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-6[121170]: 2023-09-23T20:53:33.166+0000 7f82c6191700 -1 osd.6 695 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2023-09-23T20:53:00.273630+0000 front 2023-09-23T20:53:00.273539+0000 (oldest deadline 2023-09-23T20:53:25.573656+0000) 2023-09-23T20:53:33.467 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:53:33 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[156506]: ts=2023-09-23T20:53:33.234Z caller=refresh.go:99 level=error component="discovery manager notify" discovery=http config=config-0 msg="Unable to refresh target groups" err="Get \"http://172.21.15.18:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.18:8765: connect: connection refused" 2023-09-23T20:53:33.467 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:53:33 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[156506]: ts=2023-09-23T20:53:33.237Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph-exporter msg="Unable to refresh target groups" err="Get \"http://172.21.15.18:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.18:8765: connect: connection refused" 2023-09-23T20:53:33.467 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:53:33 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[156506]: ts=2023-09-23T20:53:33.237Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph msg="Unable to refresh target groups" err="Get \"http://172.21.15.18:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.18:8765: connect: connection refused" 2023-09-23T20:53:33.467 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:53:33 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[156506]: ts=2023-09-23T20:53:33.237Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=node msg="Unable to refresh target groups" err="Get \"http://172.21.15.18:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.18:8765: connect: connection refused" 2023-09-23T20:53:34.187 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 23 20:53:33 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-5[114166]: 2023-09-23T20:53:33.819+0000 7f116e046700 -1 osd.5 695 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2023-09-23T20:52:56.475955+0000 front 2023-09-23T20:52:56.475941+0000 (oldest deadline 2023-09-23T20:53:22.375916+0000) 2023-09-23T20:53:34.187 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 23 20:53:33 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-5[114166]: 2023-09-23T20:53:33.819+0000 7f116e046700 -1 osd.5 695 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2023-09-23T20:53:07.478202+0000 front 2023-09-23T20:53:07.478211+0000 (oldest deadline 2023-09-23T20:53:30.977995+0000) 2023-09-23T20:53:34.193 DEBUG:teuthology.orchestra.run.smithi149:> sudo pkill -f 'journalctl -f -n 0 -u ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7@osd.4.service' 2023-09-23T20:53:34.274 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T20:53:34.275 INFO:tasks.cephadm.osd.4:Stopped osd.4 2023-09-23T20:53:34.275 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2023-09-23T20:53:34.275 DEBUG:teuthology.orchestra.run.smithi149:> sudo systemctl stop ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7@osd.5 2023-09-23T20:53:34.466 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 23 20:53:34 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-6[121170]: 2023-09-23T20:53:34.195+0000 7f82c6191700 -1 osd.6 695 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2023-09-23T20:53:00.273630+0000 front 2023-09-23T20:53:00.273539+0000 (oldest deadline 2023-09-23T20:53:25.573656+0000) 2023-09-23T20:53:34.466 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 23 20:53:34 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-6[121170]: 2023-09-23T20:53:34.195+0000 7f82c6191700 -1 osd.6 695 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2023-09-23T20:53:07.874795+0000 front 2023-09-23T20:53:07.874955+0000 (oldest deadline 2023-09-23T20:53:33.174743+0000) 2023-09-23T20:53:34.467 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 23 20:53:34 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-7[128937]: 2023-09-23T20:53:34.314+0000 7f14eaa6a700 -1 osd.7 695 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2023-09-23T20:52:59.913344+0000 front 2023-09-23T20:52:59.913028+0000 (oldest deadline 2023-09-23T20:53:25.713711+0000) 2023-09-23T20:53:34.467 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 23 20:53:34 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-7[128937]: 2023-09-23T20:53:34.314+0000 7f14eaa6a700 -1 osd.7 695 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2023-09-23T20:53:05.714313+0000 front 2023-09-23T20:53:05.714276+0000 (oldest deadline 2023-09-23T20:53:30.414093+0000) 2023-09-23T20:53:34.467 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 23 20:53:34 smithi149 systemd[1]: Stopping Ceph osd.5 for a3b3b17e-5a4c-11ee-8db4-212e2dc638e7... 2023-09-23T20:53:34.966 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 23 20:53:34 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-5[114166]: 2023-09-23T20:53:34.642+0000 7f1172a4a700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T20:53:34.967 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 23 20:53:34 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-5[114166]: 2023-09-23T20:53:34.642+0000 7f1172a4a700 -1 osd.5 695 *** Got signal Terminated *** 2023-09-23T20:53:34.967 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 23 20:53:34 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-5[114166]: 2023-09-23T20:53:34.642+0000 7f1172a4a700 -1 osd.5 695 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T20:53:34.967 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 23 20:53:34 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-5[114166]: 2023-09-23T20:53:34.845+0000 7f116e046700 -1 osd.5 695 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2023-09-23T20:52:56.475955+0000 front 2023-09-23T20:52:56.475941+0000 (oldest deadline 2023-09-23T20:53:22.375916+0000) 2023-09-23T20:53:34.967 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 23 20:53:34 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-5[114166]: 2023-09-23T20:53:34.845+0000 7f116e046700 -1 osd.5 695 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2023-09-23T20:53:07.478202+0000 front 2023-09-23T20:53:07.478211+0000 (oldest deadline 2023-09-23T20:53:30.977995+0000) 2023-09-23T20:53:35.716 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 23 20:53:35 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-6[121170]: 2023-09-23T20:53:35.238+0000 7f82c6191700 -1 osd.6 695 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2023-09-23T20:53:00.273630+0000 front 2023-09-23T20:53:00.273539+0000 (oldest deadline 2023-09-23T20:53:25.573656+0000) 2023-09-23T20:53:35.716 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 23 20:53:35 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-6[121170]: 2023-09-23T20:53:35.238+0000 7f82c6191700 -1 osd.6 695 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2023-09-23T20:53:07.874795+0000 front 2023-09-23T20:53:07.874955+0000 (oldest deadline 2023-09-23T20:53:33.174743+0000) 2023-09-23T20:53:35.717 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 23 20:53:35 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-7[128937]: 2023-09-23T20:53:35.339+0000 7f14eaa6a700 -1 osd.7 695 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2023-09-23T20:52:59.913344+0000 front 2023-09-23T20:52:59.913028+0000 (oldest deadline 2023-09-23T20:53:25.713711+0000) 2023-09-23T20:53:35.717 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 23 20:53:35 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-7[128937]: 2023-09-23T20:53:35.339+0000 7f14eaa6a700 -1 osd.7 695 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2023-09-23T20:53:05.714313+0000 front 2023-09-23T20:53:05.714276+0000 (oldest deadline 2023-09-23T20:53:30.414093+0000) 2023-09-23T20:53:36.216 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 23 20:53:35 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-5[114166]: 2023-09-23T20:53:35.797+0000 7f116e046700 -1 osd.5 695 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2023-09-23T20:52:56.475955+0000 front 2023-09-23T20:52:56.475941+0000 (oldest deadline 2023-09-23T20:53:22.375916+0000) 2023-09-23T20:53:36.216 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 23 20:53:35 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-5[114166]: 2023-09-23T20:53:35.798+0000 7f116e046700 -1 osd.5 695 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2023-09-23T20:53:07.478202+0000 front 2023-09-23T20:53:07.478211+0000 (oldest deadline 2023-09-23T20:53:30.977995+0000) 2023-09-23T20:53:36.716 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 23 20:53:36 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-6[121170]: 2023-09-23T20:53:36.282+0000 7f82c6191700 -1 osd.6 695 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2023-09-23T20:53:00.273630+0000 front 2023-09-23T20:53:00.273539+0000 (oldest deadline 2023-09-23T20:53:25.573656+0000) 2023-09-23T20:53:36.716 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 23 20:53:36 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-6[121170]: 2023-09-23T20:53:36.282+0000 7f82c6191700 -1 osd.6 695 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2023-09-23T20:53:07.874795+0000 front 2023-09-23T20:53:07.874955+0000 (oldest deadline 2023-09-23T20:53:33.174743+0000) 2023-09-23T20:53:36.717 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 23 20:53:36 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-7[128937]: 2023-09-23T20:53:36.358+0000 7f14eaa6a700 -1 osd.7 695 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2023-09-23T20:52:59.913344+0000 front 2023-09-23T20:52:59.913028+0000 (oldest deadline 2023-09-23T20:53:25.713711+0000) 2023-09-23T20:53:36.717 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 23 20:53:36 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-7[128937]: 2023-09-23T20:53:36.358+0000 7f14eaa6a700 -1 osd.7 695 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2023-09-23T20:53:05.714313+0000 front 2023-09-23T20:53:05.714276+0000 (oldest deadline 2023-09-23T20:53:30.414093+0000) 2023-09-23T20:53:37.216 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 23 20:53:36 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-5[114166]: 2023-09-23T20:53:36.771+0000 7f116e046700 -1 osd.5 695 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2023-09-23T20:52:56.475955+0000 front 2023-09-23T20:52:56.475941+0000 (oldest deadline 2023-09-23T20:53:22.375916+0000) 2023-09-23T20:53:37.216 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 23 20:53:36 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-5[114166]: 2023-09-23T20:53:36.771+0000 7f116e046700 -1 osd.5 695 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2023-09-23T20:53:07.478202+0000 front 2023-09-23T20:53:07.478211+0000 (oldest deadline 2023-09-23T20:53:30.977995+0000) 2023-09-23T20:53:37.716 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 23 20:53:37 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-6[121170]: 2023-09-23T20:53:37.255+0000 7f82c6191700 -1 osd.6 695 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2023-09-23T20:53:00.273630+0000 front 2023-09-23T20:53:00.273539+0000 (oldest deadline 2023-09-23T20:53:25.573656+0000) 2023-09-23T20:53:37.716 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 23 20:53:37 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-6[121170]: 2023-09-23T20:53:37.255+0000 7f82c6191700 -1 osd.6 695 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2023-09-23T20:53:07.874795+0000 front 2023-09-23T20:53:07.874955+0000 (oldest deadline 2023-09-23T20:53:33.174743+0000) 2023-09-23T20:53:37.717 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 23 20:53:37 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-7[128937]: 2023-09-23T20:53:37.381+0000 7f14eaa6a700 -1 osd.7 695 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2023-09-23T20:52:59.913344+0000 front 2023-09-23T20:52:59.913028+0000 (oldest deadline 2023-09-23T20:53:25.713711+0000) 2023-09-23T20:53:37.717 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 23 20:53:37 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-7[128937]: 2023-09-23T20:53:37.381+0000 7f14eaa6a700 -1 osd.7 695 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2023-09-23T20:53:05.714313+0000 front 2023-09-23T20:53:05.714276+0000 (oldest deadline 2023-09-23T20:53:30.414093+0000) 2023-09-23T20:53:38.208 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 23 20:53:37 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-5[114166]: 2023-09-23T20:53:37.808+0000 7f116e046700 -1 osd.5 695 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2023-09-23T20:52:56.475955+0000 front 2023-09-23T20:52:56.475941+0000 (oldest deadline 2023-09-23T20:53:22.375916+0000) 2023-09-23T20:53:38.209 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 23 20:53:37 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-5[114166]: 2023-09-23T20:53:37.808+0000 7f116e046700 -1 osd.5 695 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2023-09-23T20:53:07.478202+0000 front 2023-09-23T20:53:07.478211+0000 (oldest deadline 2023-09-23T20:53:30.977995+0000) 2023-09-23T20:53:38.466 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 23 20:53:38 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-6[121170]: 2023-09-23T20:53:38.207+0000 7f82c6191700 -1 osd.6 695 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2023-09-23T20:53:00.273630+0000 front 2023-09-23T20:53:00.273539+0000 (oldest deadline 2023-09-23T20:53:25.573656+0000) 2023-09-23T20:53:38.466 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 23 20:53:38 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-6[121170]: 2023-09-23T20:53:38.207+0000 7f82c6191700 -1 osd.6 695 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2023-09-23T20:53:07.874795+0000 front 2023-09-23T20:53:07.874955+0000 (oldest deadline 2023-09-23T20:53:33.174743+0000) 2023-09-23T20:53:38.467 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 23 20:53:38 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-7[128937]: 2023-09-23T20:53:38.362+0000 7f14eaa6a700 -1 osd.7 695 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2023-09-23T20:52:59.913344+0000 front 2023-09-23T20:52:59.913028+0000 (oldest deadline 2023-09-23T20:53:25.713711+0000) 2023-09-23T20:53:38.467 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 23 20:53:38 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-7[128937]: 2023-09-23T20:53:38.362+0000 7f14eaa6a700 -1 osd.7 695 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2023-09-23T20:53:05.714313+0000 front 2023-09-23T20:53:05.714276+0000 (oldest deadline 2023-09-23T20:53:30.414093+0000) 2023-09-23T20:53:38.467 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 23 20:53:38 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-7[128937]: 2023-09-23T20:53:38.362+0000 7f14eaa6a700 -1 osd.7 695 heartbeat_check: no reply from 172.21.15.18:6822 osd.2 since back 2023-09-23T20:53:13.914981+0000 front 2023-09-23T20:53:13.915039+0000 (oldest deadline 2023-09-23T20:53:38.014928+0000) 2023-09-23T20:53:39.178 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 23 20:53:38 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-5[114166]: 2023-09-23T20:53:38.850+0000 7f116e046700 -1 osd.5 695 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2023-09-23T20:52:56.475955+0000 front 2023-09-23T20:52:56.475941+0000 (oldest deadline 2023-09-23T20:53:22.375916+0000) 2023-09-23T20:53:39.178 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 23 20:53:38 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-5[114166]: 2023-09-23T20:53:38.850+0000 7f116e046700 -1 osd.5 695 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2023-09-23T20:53:07.478202+0000 front 2023-09-23T20:53:07.478211+0000 (oldest deadline 2023-09-23T20:53:30.977995+0000) 2023-09-23T20:53:39.466 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 23 20:53:39 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-6[121170]: 2023-09-23T20:53:39.177+0000 7f82c6191700 -1 osd.6 695 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2023-09-23T20:53:00.273630+0000 front 2023-09-23T20:53:00.273539+0000 (oldest deadline 2023-09-23T20:53:25.573656+0000) 2023-09-23T20:53:39.466 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 23 20:53:39 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-6[121170]: 2023-09-23T20:53:39.177+0000 7f82c6191700 -1 osd.6 695 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2023-09-23T20:53:07.874795+0000 front 2023-09-23T20:53:07.874955+0000 (oldest deadline 2023-09-23T20:53:33.174743+0000) 2023-09-23T20:53:39.467 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 23 20:53:39 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-7[128937]: 2023-09-23T20:53:39.372+0000 7f14eaa6a700 -1 osd.7 695 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2023-09-23T20:52:59.913344+0000 front 2023-09-23T20:52:59.913028+0000 (oldest deadline 2023-09-23T20:53:25.713711+0000) 2023-09-23T20:53:39.467 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 23 20:53:39 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-7[128937]: 2023-09-23T20:53:39.372+0000 7f14eaa6a700 -1 osd.7 695 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2023-09-23T20:53:05.714313+0000 front 2023-09-23T20:53:05.714276+0000 (oldest deadline 2023-09-23T20:53:30.414093+0000) 2023-09-23T20:53:39.467 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 23 20:53:39 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-7[128937]: 2023-09-23T20:53:39.372+0000 7f14eaa6a700 -1 osd.7 695 heartbeat_check: no reply from 172.21.15.18:6822 osd.2 since back 2023-09-23T20:53:13.914981+0000 front 2023-09-23T20:53:13.915039+0000 (oldest deadline 2023-09-23T20:53:38.014928+0000) 2023-09-23T20:53:40.466 INFO:journalctl@ceph.osd.5.smithi149.stdout:Sep 23 20:53:40 smithi149 bash[309073]: ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-5 2023-09-23T20:53:40.467 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 23 20:53:40 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-6[121170]: 2023-09-23T20:53:40.206+0000 7f82c6191700 -1 osd.6 695 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2023-09-23T20:53:00.273630+0000 front 2023-09-23T20:53:00.273539+0000 (oldest deadline 2023-09-23T20:53:25.573656+0000) 2023-09-23T20:53:40.467 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 23 20:53:40 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-6[121170]: 2023-09-23T20:53:40.206+0000 7f82c6191700 -1 osd.6 695 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2023-09-23T20:53:07.874795+0000 front 2023-09-23T20:53:07.874955+0000 (oldest deadline 2023-09-23T20:53:33.174743+0000) 2023-09-23T20:53:40.467 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 23 20:53:40 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-6[121170]: 2023-09-23T20:53:40.206+0000 7f82c6191700 -1 osd.6 695 heartbeat_check: no reply from 172.21.15.18:6822 osd.2 since back 2023-09-23T20:53:15.976242+0000 front 2023-09-23T20:53:15.976268+0000 (oldest deadline 2023-09-23T20:53:40.076093+0000) 2023-09-23T20:53:40.467 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 23 20:53:40 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-7[128937]: 2023-09-23T20:53:40.391+0000 7f14eaa6a700 -1 osd.7 695 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2023-09-23T20:52:59.913344+0000 front 2023-09-23T20:52:59.913028+0000 (oldest deadline 2023-09-23T20:53:25.713711+0000) 2023-09-23T20:53:40.467 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 23 20:53:40 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-7[128937]: 2023-09-23T20:53:40.391+0000 7f14eaa6a700 -1 osd.7 695 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2023-09-23T20:53:05.714313+0000 front 2023-09-23T20:53:05.714276+0000 (oldest deadline 2023-09-23T20:53:30.414093+0000) 2023-09-23T20:53:40.468 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 23 20:53:40 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-7[128937]: 2023-09-23T20:53:40.391+0000 7f14eaa6a700 -1 osd.7 695 heartbeat_check: no reply from 172.21.15.18:6822 osd.2 since back 2023-09-23T20:53:13.914981+0000 front 2023-09-23T20:53:13.915039+0000 (oldest deadline 2023-09-23T20:53:38.014928+0000) 2023-09-23T20:53:41.416 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 23 20:53:41 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-6[121170]: 2023-09-23T20:53:41.223+0000 7f82c6191700 -1 osd.6 695 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2023-09-23T20:53:00.273630+0000 front 2023-09-23T20:53:00.273539+0000 (oldest deadline 2023-09-23T20:53:25.573656+0000) 2023-09-23T20:53:41.416 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 23 20:53:41 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-6[121170]: 2023-09-23T20:53:41.223+0000 7f82c6191700 -1 osd.6 695 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2023-09-23T20:53:07.874795+0000 front 2023-09-23T20:53:07.874955+0000 (oldest deadline 2023-09-23T20:53:33.174743+0000) 2023-09-23T20:53:41.416 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 23 20:53:41 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-6[121170]: 2023-09-23T20:53:41.223+0000 7f82c6191700 -1 osd.6 695 heartbeat_check: no reply from 172.21.15.18:6822 osd.2 since back 2023-09-23T20:53:15.976242+0000 front 2023-09-23T20:53:15.976268+0000 (oldest deadline 2023-09-23T20:53:40.076093+0000) 2023-09-23T20:53:41.716 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 23 20:53:41 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-7[128937]: 2023-09-23T20:53:41.414+0000 7f14eaa6a700 -1 osd.7 695 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2023-09-23T20:52:59.913344+0000 front 2023-09-23T20:52:59.913028+0000 (oldest deadline 2023-09-23T20:53:25.713711+0000) 2023-09-23T20:53:41.716 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 23 20:53:41 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-7[128937]: 2023-09-23T20:53:41.414+0000 7f14eaa6a700 -1 osd.7 695 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2023-09-23T20:53:05.714313+0000 front 2023-09-23T20:53:05.714276+0000 (oldest deadline 2023-09-23T20:53:30.414093+0000) 2023-09-23T20:53:41.716 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 23 20:53:41 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-7[128937]: 2023-09-23T20:53:41.414+0000 7f14eaa6a700 -1 osd.7 695 heartbeat_check: no reply from 172.21.15.18:6822 osd.2 since back 2023-09-23T20:53:13.914981+0000 front 2023-09-23T20:53:13.915039+0000 (oldest deadline 2023-09-23T20:53:38.014928+0000) 2023-09-23T20:53:41.865 DEBUG:teuthology.orchestra.run.smithi149:> sudo pkill -f 'journalctl -f -n 0 -u ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7@osd.5.service' 2023-09-23T20:53:41.906 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T20:53:41.906 INFO:tasks.cephadm.osd.5:Stopped osd.5 2023-09-23T20:53:41.907 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2023-09-23T20:53:41.907 DEBUG:teuthology.orchestra.run.smithi149:> sudo systemctl stop ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7@osd.6 2023-09-23T20:53:42.216 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 23 20:53:41 smithi149 systemd[1]: Stopping Ceph osd.6 for a3b3b17e-5a4c-11ee-8db4-212e2dc638e7... 2023-09-23T20:53:42.716 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 23 20:53:42 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-6[121170]: 2023-09-23T20:53:42.255+0000 7f82c6191700 -1 osd.6 695 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2023-09-23T20:53:00.273630+0000 front 2023-09-23T20:53:00.273539+0000 (oldest deadline 2023-09-23T20:53:25.573656+0000) 2023-09-23T20:53:42.716 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 23 20:53:42 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-6[121170]: 2023-09-23T20:53:42.255+0000 7f82c6191700 -1 osd.6 695 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2023-09-23T20:53:07.874795+0000 front 2023-09-23T20:53:07.874955+0000 (oldest deadline 2023-09-23T20:53:33.174743+0000) 2023-09-23T20:53:42.716 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 23 20:53:42 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-6[121170]: 2023-09-23T20:53:42.255+0000 7f82c6191700 -1 osd.6 695 heartbeat_check: no reply from 172.21.15.18:6822 osd.2 since back 2023-09-23T20:53:15.976242+0000 front 2023-09-23T20:53:15.976268+0000 (oldest deadline 2023-09-23T20:53:40.076093+0000) 2023-09-23T20:53:42.717 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 23 20:53:42 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-6[121170]: 2023-09-23T20:53:42.285+0000 7f82ccb99700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T20:53:42.717 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 23 20:53:42 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-6[121170]: 2023-09-23T20:53:42.285+0000 7f82ccb99700 -1 osd.6 695 *** Got signal Terminated *** 2023-09-23T20:53:42.717 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 23 20:53:42 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-6[121170]: 2023-09-23T20:53:42.285+0000 7f82ccb99700 -1 osd.6 695 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T20:53:42.717 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 23 20:53:42 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-7[128937]: 2023-09-23T20:53:42.423+0000 7f14eaa6a700 -1 osd.7 695 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2023-09-23T20:52:59.913344+0000 front 2023-09-23T20:52:59.913028+0000 (oldest deadline 2023-09-23T20:53:25.713711+0000) 2023-09-23T20:53:42.717 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 23 20:53:42 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-7[128937]: 2023-09-23T20:53:42.423+0000 7f14eaa6a700 -1 osd.7 695 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2023-09-23T20:53:05.714313+0000 front 2023-09-23T20:53:05.714276+0000 (oldest deadline 2023-09-23T20:53:30.414093+0000) 2023-09-23T20:53:42.717 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 23 20:53:42 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-7[128937]: 2023-09-23T20:53:42.423+0000 7f14eaa6a700 -1 osd.7 695 heartbeat_check: no reply from 172.21.15.18:6822 osd.2 since back 2023-09-23T20:53:13.914981+0000 front 2023-09-23T20:53:13.915039+0000 (oldest deadline 2023-09-23T20:53:38.014928+0000) 2023-09-23T20:53:43.716 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 23 20:53:43 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-6[121170]: 2023-09-23T20:53:43.280+0000 7f82c6191700 -1 osd.6 695 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2023-09-23T20:53:00.273630+0000 front 2023-09-23T20:53:00.273539+0000 (oldest deadline 2023-09-23T20:53:25.573656+0000) 2023-09-23T20:53:43.716 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 23 20:53:43 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-6[121170]: 2023-09-23T20:53:43.280+0000 7f82c6191700 -1 osd.6 695 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2023-09-23T20:53:07.874795+0000 front 2023-09-23T20:53:07.874955+0000 (oldest deadline 2023-09-23T20:53:33.174743+0000) 2023-09-23T20:53:43.716 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 23 20:53:43 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-6[121170]: 2023-09-23T20:53:43.280+0000 7f82c6191700 -1 osd.6 695 heartbeat_check: no reply from 172.21.15.18:6822 osd.2 since back 2023-09-23T20:53:15.976242+0000 front 2023-09-23T20:53:15.976268+0000 (oldest deadline 2023-09-23T20:53:40.076093+0000) 2023-09-23T20:53:43.717 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 23 20:53:43 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-7[128937]: 2023-09-23T20:53:43.389+0000 7f14eaa6a700 -1 osd.7 695 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2023-09-23T20:52:59.913344+0000 front 2023-09-23T20:52:59.913028+0000 (oldest deadline 2023-09-23T20:53:25.713711+0000) 2023-09-23T20:53:43.717 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 23 20:53:43 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-7[128937]: 2023-09-23T20:53:43.389+0000 7f14eaa6a700 -1 osd.7 695 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2023-09-23T20:53:05.714313+0000 front 2023-09-23T20:53:05.714276+0000 (oldest deadline 2023-09-23T20:53:30.414093+0000) 2023-09-23T20:53:43.717 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 23 20:53:43 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-7[128937]: 2023-09-23T20:53:43.389+0000 7f14eaa6a700 -1 osd.7 695 heartbeat_check: no reply from 172.21.15.18:6822 osd.2 since back 2023-09-23T20:53:13.914981+0000 front 2023-09-23T20:53:13.915039+0000 (oldest deadline 2023-09-23T20:53:38.014928+0000) 2023-09-23T20:53:44.716 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 23 20:53:44 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-6[121170]: 2023-09-23T20:53:44.277+0000 7f82c6191700 -1 osd.6 695 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2023-09-23T20:53:00.273630+0000 front 2023-09-23T20:53:00.273539+0000 (oldest deadline 2023-09-23T20:53:25.573656+0000) 2023-09-23T20:53:44.717 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 23 20:53:44 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-6[121170]: 2023-09-23T20:53:44.277+0000 7f82c6191700 -1 osd.6 695 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2023-09-23T20:53:07.874795+0000 front 2023-09-23T20:53:07.874955+0000 (oldest deadline 2023-09-23T20:53:33.174743+0000) 2023-09-23T20:53:44.717 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 23 20:53:44 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-6[121170]: 2023-09-23T20:53:44.277+0000 7f82c6191700 -1 osd.6 695 heartbeat_check: no reply from 172.21.15.18:6822 osd.2 since back 2023-09-23T20:53:15.976242+0000 front 2023-09-23T20:53:15.976268+0000 (oldest deadline 2023-09-23T20:53:40.076093+0000) 2023-09-23T20:53:44.717 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 23 20:53:44 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-7[128937]: 2023-09-23T20:53:44.363+0000 7f14eaa6a700 -1 osd.7 695 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2023-09-23T20:52:59.913344+0000 front 2023-09-23T20:52:59.913028+0000 (oldest deadline 2023-09-23T20:53:25.713711+0000) 2023-09-23T20:53:44.717 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 23 20:53:44 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-7[128937]: 2023-09-23T20:53:44.363+0000 7f14eaa6a700 -1 osd.7 695 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2023-09-23T20:53:05.714313+0000 front 2023-09-23T20:53:05.714276+0000 (oldest deadline 2023-09-23T20:53:30.414093+0000) 2023-09-23T20:53:44.717 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 23 20:53:44 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-7[128937]: 2023-09-23T20:53:44.363+0000 7f14eaa6a700 -1 osd.7 695 heartbeat_check: no reply from 172.21.15.18:6822 osd.2 since back 2023-09-23T20:53:13.914981+0000 front 2023-09-23T20:53:13.915039+0000 (oldest deadline 2023-09-23T20:53:38.014928+0000) 2023-09-23T20:53:45.716 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 23 20:53:45 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-6[121170]: 2023-09-23T20:53:45.314+0000 7f82c6191700 -1 osd.6 695 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2023-09-23T20:53:00.273630+0000 front 2023-09-23T20:53:00.273539+0000 (oldest deadline 2023-09-23T20:53:25.573656+0000) 2023-09-23T20:53:45.716 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 23 20:53:45 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-6[121170]: 2023-09-23T20:53:45.314+0000 7f82c6191700 -1 osd.6 695 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2023-09-23T20:53:07.874795+0000 front 2023-09-23T20:53:07.874955+0000 (oldest deadline 2023-09-23T20:53:33.174743+0000) 2023-09-23T20:53:45.716 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 23 20:53:45 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-6[121170]: 2023-09-23T20:53:45.314+0000 7f82c6191700 -1 osd.6 695 heartbeat_check: no reply from 172.21.15.18:6822 osd.2 since back 2023-09-23T20:53:15.976242+0000 front 2023-09-23T20:53:15.976268+0000 (oldest deadline 2023-09-23T20:53:40.076093+0000) 2023-09-23T20:53:45.717 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 23 20:53:45 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-7[128937]: 2023-09-23T20:53:45.358+0000 7f14eaa6a700 -1 osd.7 695 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2023-09-23T20:52:59.913344+0000 front 2023-09-23T20:52:59.913028+0000 (oldest deadline 2023-09-23T20:53:25.713711+0000) 2023-09-23T20:53:45.717 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 23 20:53:45 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-7[128937]: 2023-09-23T20:53:45.358+0000 7f14eaa6a700 -1 osd.7 695 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2023-09-23T20:53:05.714313+0000 front 2023-09-23T20:53:05.714276+0000 (oldest deadline 2023-09-23T20:53:30.414093+0000) 2023-09-23T20:53:45.717 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 23 20:53:45 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-7[128937]: 2023-09-23T20:53:45.358+0000 7f14eaa6a700 -1 osd.7 695 heartbeat_check: no reply from 172.21.15.18:6822 osd.2 since back 2023-09-23T20:53:13.914981+0000 front 2023-09-23T20:53:13.915039+0000 (oldest deadline 2023-09-23T20:53:38.014928+0000) 2023-09-23T20:53:46.716 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 23 20:53:46 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-6[121170]: 2023-09-23T20:53:46.272+0000 7f82c6191700 -1 osd.6 695 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2023-09-23T20:53:00.273630+0000 front 2023-09-23T20:53:00.273539+0000 (oldest deadline 2023-09-23T20:53:25.573656+0000) 2023-09-23T20:53:46.716 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 23 20:53:46 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-6[121170]: 2023-09-23T20:53:46.272+0000 7f82c6191700 -1 osd.6 695 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2023-09-23T20:53:07.874795+0000 front 2023-09-23T20:53:07.874955+0000 (oldest deadline 2023-09-23T20:53:33.174743+0000) 2023-09-23T20:53:46.717 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 23 20:53:46 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-6[121170]: 2023-09-23T20:53:46.272+0000 7f82c6191700 -1 osd.6 695 heartbeat_check: no reply from 172.21.15.18:6822 osd.2 since back 2023-09-23T20:53:15.976242+0000 front 2023-09-23T20:53:15.976268+0000 (oldest deadline 2023-09-23T20:53:40.076093+0000) 2023-09-23T20:53:46.717 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 23 20:53:46 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-7[128937]: 2023-09-23T20:53:46.348+0000 7f14eaa6a700 -1 osd.7 695 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2023-09-23T20:52:59.913344+0000 front 2023-09-23T20:52:59.913028+0000 (oldest deadline 2023-09-23T20:53:25.713711+0000) 2023-09-23T20:53:46.717 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 23 20:53:46 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-7[128937]: 2023-09-23T20:53:46.348+0000 7f14eaa6a700 -1 osd.7 695 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2023-09-23T20:53:05.714313+0000 front 2023-09-23T20:53:05.714276+0000 (oldest deadline 2023-09-23T20:53:30.414093+0000) 2023-09-23T20:53:46.717 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 23 20:53:46 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-7[128937]: 2023-09-23T20:53:46.348+0000 7f14eaa6a700 -1 osd.7 695 heartbeat_check: no reply from 172.21.15.18:6822 osd.2 since back 2023-09-23T20:53:13.914981+0000 front 2023-09-23T20:53:13.915039+0000 (oldest deadline 2023-09-23T20:53:38.014928+0000) 2023-09-23T20:53:47.716 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 23 20:53:47 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-7[128937]: 2023-09-23T20:53:47.308+0000 7f14eaa6a700 -1 osd.7 695 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2023-09-23T20:52:59.913344+0000 front 2023-09-23T20:52:59.913028+0000 (oldest deadline 2023-09-23T20:53:25.713711+0000) 2023-09-23T20:53:47.716 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 23 20:53:47 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-7[128937]: 2023-09-23T20:53:47.308+0000 7f14eaa6a700 -1 osd.7 695 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2023-09-23T20:53:05.714313+0000 front 2023-09-23T20:53:05.714276+0000 (oldest deadline 2023-09-23T20:53:30.414093+0000) 2023-09-23T20:53:47.716 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 23 20:53:47 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-7[128937]: 2023-09-23T20:53:47.308+0000 7f14eaa6a700 -1 osd.7 695 heartbeat_check: no reply from 172.21.15.18:6822 osd.2 since back 2023-09-23T20:53:13.914981+0000 front 2023-09-23T20:53:13.915039+0000 (oldest deadline 2023-09-23T20:53:38.014928+0000) 2023-09-23T20:53:47.716 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 23 20:53:47 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-7[128937]: 2023-09-23T20:53:47.308+0000 7f14eaa6a700 -1 osd.7 695 heartbeat_check: no reply from 172.21.15.18:6830 osd.3 since back 2023-09-23T20:53:22.116153+0000 front 2023-09-23T20:53:22.116180+0000 (oldest deadline 2023-09-23T20:53:46.815939+0000) 2023-09-23T20:53:48.175 INFO:journalctl@ceph.osd.6.smithi149.stdout:Sep 23 20:53:48 smithi149 bash[309348]: ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-6 2023-09-23T20:53:48.715 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 23 20:53:48 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-7[128937]: 2023-09-23T20:53:48.349+0000 7f14eaa6a700 -1 osd.7 695 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2023-09-23T20:52:59.913344+0000 front 2023-09-23T20:52:59.913028+0000 (oldest deadline 2023-09-23T20:53:25.713711+0000) 2023-09-23T20:53:48.716 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 23 20:53:48 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-7[128937]: 2023-09-23T20:53:48.349+0000 7f14eaa6a700 -1 osd.7 695 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2023-09-23T20:53:05.714313+0000 front 2023-09-23T20:53:05.714276+0000 (oldest deadline 2023-09-23T20:53:30.414093+0000) 2023-09-23T20:53:48.716 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 23 20:53:48 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-7[128937]: 2023-09-23T20:53:48.349+0000 7f14eaa6a700 -1 osd.7 695 heartbeat_check: no reply from 172.21.15.18:6822 osd.2 since back 2023-09-23T20:53:13.914981+0000 front 2023-09-23T20:53:13.915039+0000 (oldest deadline 2023-09-23T20:53:38.014928+0000) 2023-09-23T20:53:48.716 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 23 20:53:48 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-7[128937]: 2023-09-23T20:53:48.349+0000 7f14eaa6a700 -1 osd.7 695 heartbeat_check: no reply from 172.21.15.18:6830 osd.3 since back 2023-09-23T20:53:22.116153+0000 front 2023-09-23T20:53:22.116180+0000 (oldest deadline 2023-09-23T20:53:46.815939+0000) 2023-09-23T20:53:49.441 DEBUG:teuthology.orchestra.run.smithi149:> sudo pkill -f 'journalctl -f -n 0 -u ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7@osd.6.service' 2023-09-23T20:53:49.483 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T20:53:49.483 INFO:tasks.cephadm.osd.6:Stopped osd.6 2023-09-23T20:53:49.483 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2023-09-23T20:53:49.483 DEBUG:teuthology.orchestra.run.smithi149:> sudo systemctl stop ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7@osd.7 2023-09-23T20:53:49.717 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 23 20:53:49 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-7[128937]: 2023-09-23T20:53:49.388+0000 7f14eaa6a700 -1 osd.7 695 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2023-09-23T20:52:59.913344+0000 front 2023-09-23T20:52:59.913028+0000 (oldest deadline 2023-09-23T20:53:25.713711+0000) 2023-09-23T20:53:49.717 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 23 20:53:49 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-7[128937]: 2023-09-23T20:53:49.388+0000 7f14eaa6a700 -1 osd.7 695 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2023-09-23T20:53:05.714313+0000 front 2023-09-23T20:53:05.714276+0000 (oldest deadline 2023-09-23T20:53:30.414093+0000) 2023-09-23T20:53:49.717 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 23 20:53:49 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-7[128937]: 2023-09-23T20:53:49.388+0000 7f14eaa6a700 -1 osd.7 695 heartbeat_check: no reply from 172.21.15.18:6822 osd.2 since back 2023-09-23T20:53:13.914981+0000 front 2023-09-23T20:53:13.915039+0000 (oldest deadline 2023-09-23T20:53:38.014928+0000) 2023-09-23T20:53:49.717 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 23 20:53:49 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-7[128937]: 2023-09-23T20:53:49.388+0000 7f14eaa6a700 -1 osd.7 695 heartbeat_check: no reply from 172.21.15.18:6830 osd.3 since back 2023-09-23T20:53:22.116153+0000 front 2023-09-23T20:53:22.116180+0000 (oldest deadline 2023-09-23T20:53:46.815939+0000) 2023-09-23T20:53:49.718 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 23 20:53:49 smithi149 systemd[1]: Stopping Ceph osd.7 for a3b3b17e-5a4c-11ee-8db4-212e2dc638e7... 2023-09-23T20:53:50.216 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 23 20:53:49 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-7[128937]: 2023-09-23T20:53:49.876+0000 7f14ef46e700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T20:53:50.216 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 23 20:53:49 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-7[128937]: 2023-09-23T20:53:49.876+0000 7f14ef46e700 -1 osd.7 695 *** Got signal Terminated *** 2023-09-23T20:53:50.216 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 23 20:53:49 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-7[128937]: 2023-09-23T20:53:49.876+0000 7f14ef46e700 -1 osd.7 695 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T20:53:50.716 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 23 20:53:50 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-7[128937]: 2023-09-23T20:53:50.373+0000 7f14eaa6a700 -1 osd.7 695 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2023-09-23T20:52:59.913344+0000 front 2023-09-23T20:52:59.913028+0000 (oldest deadline 2023-09-23T20:53:25.713711+0000) 2023-09-23T20:53:50.716 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 23 20:53:50 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-7[128937]: 2023-09-23T20:53:50.373+0000 7f14eaa6a700 -1 osd.7 695 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2023-09-23T20:53:05.714313+0000 front 2023-09-23T20:53:05.714276+0000 (oldest deadline 2023-09-23T20:53:30.414093+0000) 2023-09-23T20:53:50.716 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 23 20:53:50 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-7[128937]: 2023-09-23T20:53:50.373+0000 7f14eaa6a700 -1 osd.7 695 heartbeat_check: no reply from 172.21.15.18:6822 osd.2 since back 2023-09-23T20:53:13.914981+0000 front 2023-09-23T20:53:13.915039+0000 (oldest deadline 2023-09-23T20:53:38.014928+0000) 2023-09-23T20:53:50.716 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 23 20:53:50 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-7[128937]: 2023-09-23T20:53:50.373+0000 7f14eaa6a700 -1 osd.7 695 heartbeat_check: no reply from 172.21.15.18:6830 osd.3 since back 2023-09-23T20:53:22.116153+0000 front 2023-09-23T20:53:22.116180+0000 (oldest deadline 2023-09-23T20:53:46.815939+0000) 2023-09-23T20:53:51.717 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 23 20:53:51 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-7[128937]: 2023-09-23T20:53:51.389+0000 7f14eaa6a700 -1 osd.7 695 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2023-09-23T20:52:59.913344+0000 front 2023-09-23T20:52:59.913028+0000 (oldest deadline 2023-09-23T20:53:25.713711+0000) 2023-09-23T20:53:51.718 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 23 20:53:51 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-7[128937]: 2023-09-23T20:53:51.389+0000 7f14eaa6a700 -1 osd.7 695 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2023-09-23T20:53:05.714313+0000 front 2023-09-23T20:53:05.714276+0000 (oldest deadline 2023-09-23T20:53:30.414093+0000) 2023-09-23T20:53:51.718 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 23 20:53:51 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-7[128937]: 2023-09-23T20:53:51.389+0000 7f14eaa6a700 -1 osd.7 695 heartbeat_check: no reply from 172.21.15.18:6822 osd.2 since back 2023-09-23T20:53:13.914981+0000 front 2023-09-23T20:53:13.915039+0000 (oldest deadline 2023-09-23T20:53:38.014928+0000) 2023-09-23T20:53:51.718 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 23 20:53:51 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-7[128937]: 2023-09-23T20:53:51.389+0000 7f14eaa6a700 -1 osd.7 695 heartbeat_check: no reply from 172.21.15.18:6830 osd.3 since back 2023-09-23T20:53:22.116153+0000 front 2023-09-23T20:53:22.116180+0000 (oldest deadline 2023-09-23T20:53:46.815939+0000) 2023-09-23T20:53:52.716 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 23 20:53:52 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-7[128937]: 2023-09-23T20:53:52.423+0000 7f14eaa6a700 -1 osd.7 695 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2023-09-23T20:52:59.913344+0000 front 2023-09-23T20:52:59.913028+0000 (oldest deadline 2023-09-23T20:53:25.713711+0000) 2023-09-23T20:53:52.716 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 23 20:53:52 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-7[128937]: 2023-09-23T20:53:52.423+0000 7f14eaa6a700 -1 osd.7 695 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2023-09-23T20:53:05.714313+0000 front 2023-09-23T20:53:05.714276+0000 (oldest deadline 2023-09-23T20:53:30.414093+0000) 2023-09-23T20:53:52.716 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 23 20:53:52 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-7[128937]: 2023-09-23T20:53:52.423+0000 7f14eaa6a700 -1 osd.7 695 heartbeat_check: no reply from 172.21.15.18:6822 osd.2 since back 2023-09-23T20:53:13.914981+0000 front 2023-09-23T20:53:13.915039+0000 (oldest deadline 2023-09-23T20:53:38.014928+0000) 2023-09-23T20:53:52.717 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 23 20:53:52 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-7[128937]: 2023-09-23T20:53:52.423+0000 7f14eaa6a700 -1 osd.7 695 heartbeat_check: no reply from 172.21.15.18:6830 osd.3 since back 2023-09-23T20:53:22.116153+0000 front 2023-09-23T20:53:22.116180+0000 (oldest deadline 2023-09-23T20:53:46.815939+0000) 2023-09-23T20:53:53.716 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 23 20:53:53 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-7[128937]: 2023-09-23T20:53:53.410+0000 7f14eaa6a700 -1 osd.7 695 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2023-09-23T20:52:59.913344+0000 front 2023-09-23T20:52:59.913028+0000 (oldest deadline 2023-09-23T20:53:25.713711+0000) 2023-09-23T20:53:53.716 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 23 20:53:53 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-7[128937]: 2023-09-23T20:53:53.410+0000 7f14eaa6a700 -1 osd.7 695 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2023-09-23T20:53:05.714313+0000 front 2023-09-23T20:53:05.714276+0000 (oldest deadline 2023-09-23T20:53:30.414093+0000) 2023-09-23T20:53:53.716 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 23 20:53:53 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-7[128937]: 2023-09-23T20:53:53.410+0000 7f14eaa6a700 -1 osd.7 695 heartbeat_check: no reply from 172.21.15.18:6822 osd.2 since back 2023-09-23T20:53:13.914981+0000 front 2023-09-23T20:53:13.915039+0000 (oldest deadline 2023-09-23T20:53:38.014928+0000) 2023-09-23T20:53:53.717 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 23 20:53:53 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-7[128937]: 2023-09-23T20:53:53.410+0000 7f14eaa6a700 -1 osd.7 695 heartbeat_check: no reply from 172.21.15.18:6830 osd.3 since back 2023-09-23T20:53:22.116153+0000 front 2023-09-23T20:53:22.116180+0000 (oldest deadline 2023-09-23T20:53:46.815939+0000) 2023-09-23T20:53:54.716 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 23 20:53:54 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-7[128937]: 2023-09-23T20:53:54.392+0000 7f14eaa6a700 -1 osd.7 695 heartbeat_check: no reply from 172.21.15.18:6806 osd.0 since back 2023-09-23T20:52:59.913344+0000 front 2023-09-23T20:52:59.913028+0000 (oldest deadline 2023-09-23T20:53:25.713711+0000) 2023-09-23T20:53:54.716 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 23 20:53:54 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-7[128937]: 2023-09-23T20:53:54.392+0000 7f14eaa6a700 -1 osd.7 695 heartbeat_check: no reply from 172.21.15.18:6814 osd.1 since back 2023-09-23T20:53:05.714313+0000 front 2023-09-23T20:53:05.714276+0000 (oldest deadline 2023-09-23T20:53:30.414093+0000) 2023-09-23T20:53:54.716 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 23 20:53:54 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-7[128937]: 2023-09-23T20:53:54.392+0000 7f14eaa6a700 -1 osd.7 695 heartbeat_check: no reply from 172.21.15.18:6822 osd.2 since back 2023-09-23T20:53:13.914981+0000 front 2023-09-23T20:53:13.915039+0000 (oldest deadline 2023-09-23T20:53:38.014928+0000) 2023-09-23T20:53:54.716 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 23 20:53:54 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-7[128937]: 2023-09-23T20:53:54.392+0000 7f14eaa6a700 -1 osd.7 695 heartbeat_check: no reply from 172.21.15.18:6830 osd.3 since back 2023-09-23T20:53:22.116153+0000 front 2023-09-23T20:53:22.116180+0000 (oldest deadline 2023-09-23T20:53:46.815939+0000) 2023-09-23T20:53:54.717 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 23 20:53:54 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-7[128937]: 2023-09-23T20:53:54.392+0000 7f14eaa6a700 -1 osd.7 695 heartbeat_check: no reply from 172.21.15.149:6804 osd.4 since back 2023-09-23T20:53:30.817355+0000 front 2023-09-23T20:53:30.817395+0000 (oldest deadline 2023-09-23T20:53:54.317427+0000) 2023-09-23T20:53:55.966 INFO:journalctl@ceph.osd.7.smithi149.stdout:Sep 23 20:53:55 smithi149 bash[309621]: ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-osd-7 2023-09-23T20:53:56.985 DEBUG:teuthology.orchestra.run.smithi149:> sudo pkill -f 'journalctl -f -n 0 -u ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7@osd.7.service' 2023-09-23T20:53:57.025 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T20:53:57.025 INFO:tasks.cephadm.osd.7:Stopped osd.7 2023-09-23T20:53:57.026 INFO:tasks.cephadm.prometheus.a:Stopping prometheus.a... 2023-09-23T20:53:57.026 DEBUG:teuthology.orchestra.run.smithi149:> sudo systemctl stop ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7@prometheus.a 2023-09-23T20:53:57.397 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:53:57 smithi149 systemd[1]: Stopping Ceph prometheus.a for a3b3b17e-5a4c-11ee-8db4-212e2dc638e7... 2023-09-23T20:53:57.699 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:53:57 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[156506]: ts=2023-09-23T20:53:57.397Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2023-09-23T20:53:57.699 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:53:57 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[156506]: ts=2023-09-23T20:53:57.397Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2023-09-23T20:53:57.699 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:53:57 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[156506]: ts=2023-09-23T20:53:57.397Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2023-09-23T20:53:57.699 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:53:57 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[156506]: ts=2023-09-23T20:53:57.397Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2023-09-23T20:53:57.700 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:53:57 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[156506]: ts=2023-09-23T20:53:57.397Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2023-09-23T20:53:57.700 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:53:57 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[156506]: ts=2023-09-23T20:53:57.397Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2023-09-23T20:53:57.700 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:53:57 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[156506]: ts=2023-09-23T20:53:57.397Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2023-09-23T20:53:57.700 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:53:57 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[156506]: ts=2023-09-23T20:53:57.397Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2023-09-23T20:53:57.700 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:53:57 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[156506]: ts=2023-09-23T20:53:57.397Z caller=main.go:907 level=info msg="Scrape manager stopped" 2023-09-23T20:53:57.700 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:53:57 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[156506]: ts=2023-09-23T20:53:57.409Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2023-09-23T20:53:57.700 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:53:57 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[156506]: ts=2023-09-23T20:53:57.409Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2023-09-23T20:53:57.701 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:53:57 smithi149 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a[156506]: ts=2023-09-23T20:53:57.409Z caller=main.go:1147 level=info msg="See you next time!" 2023-09-23T20:53:57.941 DEBUG:teuthology.orchestra.run.smithi149:> sudo pkill -f 'journalctl -f -n 0 -u ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7@prometheus.a.service' 2023-09-23T20:53:57.966 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:53:57 smithi149 bash[309890]: ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-prometheus-a 2023-09-23T20:53:57.966 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:53:57 smithi149 systemd[1]: ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7@prometheus.a.service: Succeeded. 2023-09-23T20:53:57.966 INFO:journalctl@ceph.prometheus.a.smithi149.stdout:Sep 23 20:53:57 smithi149 systemd[1]: Stopped Ceph prometheus.a for a3b3b17e-5a4c-11ee-8db4-212e2dc638e7. 2023-09-23T20:53:57.976 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T20:53:57.976 INFO:tasks.cephadm.prometheus.a:Stopped prometheus.a 2023-09-23T20:53:57.976 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 --force --keep-logs 2023-09-23T20:53:59.641 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 23 20:53:59 smithi018 systemd[1]: Stopping Ceph alertmanager.a for a3b3b17e-5a4c-11ee-8db4-212e2dc638e7... 2023-09-23T20:54:00.141 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 23 20:53:59 smithi018 ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-alertmanager-a[166780]: ts=2023-09-23T20:53:59.816Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2023-09-23T20:54:00.472 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 23 20:54:00 smithi018 bash[494018]: ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-alertmanager-a 2023-09-23T20:54:00.472 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 23 20:54:00 smithi018 systemd[1]: ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7@alertmanager.a.service: Succeeded. 2023-09-23T20:54:00.472 INFO:journalctl@ceph.alertmanager.a.smithi018.stdout:Sep 23 20:54:00 smithi018 systemd[1]: Stopped Ceph alertmanager.a for a3b3b17e-5a4c-11ee-8db4-212e2dc638e7. 2023-09-23T20:54:00.850 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:54:00 smithi018 systemd[1]: Stopping Ceph node-exporter.a for a3b3b17e-5a4c-11ee-8db4-212e2dc638e7... 2023-09-23T20:54:01.138 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:54:01 smithi018 bash[494141]: ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-a 2023-09-23T20:54:01.439 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:54:01 smithi018 systemd[1]: ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2023-09-23T20:54:01.439 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:54:01 smithi018 systemd[1]: ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7@node-exporter.a.service: Failed with result 'exit-code'. 2023-09-23T20:54:01.439 INFO:journalctl@ceph.node-exporter.a.smithi018.stdout:Sep 23 20:54:01 smithi018 systemd[1]: Stopped Ceph node-exporter.a for a3b3b17e-5a4c-11ee-8db4-212e2dc638e7. 2023-09-23T20:54:12.199 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 --force --keep-logs 2023-09-23T20:54:13.966 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:54:13 smithi149 systemd[1]: Stopping Ceph node-exporter.b for a3b3b17e-5a4c-11ee-8db4-212e2dc638e7... 2023-09-23T20:54:14.466 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:54:14 smithi149 bash[310259]: ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-node-exporter-b 2023-09-23T20:54:14.466 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:54:14 smithi149 systemd[1]: ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2023-09-23T20:54:14.947 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:54:14 smithi149 systemd[1]: ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7@node-exporter.b.service: Failed with result 'exit-code'. 2023-09-23T20:54:14.947 INFO:journalctl@ceph.node-exporter.b.smithi149.stdout:Sep 23 20:54:14 smithi149 systemd[1]: Stopped Ceph node-exporter.b for a3b3b17e-5a4c-11ee-8db4-212e2dc638e7. 2023-09-23T20:54:14.947 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:54:14 smithi149 systemd[1]: Stopping Ceph grafana.a for a3b3b17e-5a4c-11ee-8db4-212e2dc638e7... 2023-09-23T20:54:15.216 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:54:15 smithi149 bash[310384]: ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7-grafana-a 2023-09-23T20:54:15.526 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:54:15 smithi149 systemd[1]: ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7@grafana.a.service: Main process exited, code=exited, status=143/n/a 2023-09-23T20:54:15.526 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:54:15 smithi149 systemd[1]: ceph-a3b3b17e-5a4c-11ee-8db4-212e2dc638e7@grafana.a.service: Failed with result 'exit-code'. 2023-09-23T20:54:15.527 INFO:journalctl@ceph.grafana.a.smithi149.stdout:Sep 23 20:54:15 smithi149 systemd[1]: Stopped Ceph grafana.a for a3b3b17e-5a4c-11ee-8db4-212e2dc638e7. 2023-09-23T20:54:20.484 DEBUG:teuthology.orchestra.run.smithi018:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-09-23T20:54:20.523 DEBUG:teuthology.orchestra.run.smithi149:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-09-23T20:54:20.554 INFO:tasks.cephadm:Archiving crash dumps... 2023-09-23T20:54:20.556 DEBUG:teuthology.misc:Transferring archived files from smithi018:/var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/crash to /home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:quincy-x-reef-distro-default-smithi/7401824/remote/smithi018/crash 2023-09-23T20:54:20.557 DEBUG:teuthology.orchestra.run.smithi018:> sudo tar c -f - -C /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/crash -- . 2023-09-23T20:54:20.615 INFO:teuthology.orchestra.run.smithi018.stderr:tar: /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/crash: Cannot open: No such file or directory 2023-09-23T20:54:20.615 INFO:teuthology.orchestra.run.smithi018.stderr:tar: Error is not recoverable: exiting now 2023-09-23T20:54:20.617 DEBUG:teuthology.misc:Transferring archived files from smithi149:/var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/crash to /home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:quincy-x-reef-distro-default-smithi/7401824/remote/smithi149/crash 2023-09-23T20:54:20.617 DEBUG:teuthology.orchestra.run.smithi149:> sudo tar c -f - -C /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/crash -- . 2023-09-23T20:54:20.669 INFO:teuthology.orchestra.run.smithi149.stderr:tar: /var/lib/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/crash: Cannot open: No such file or directory 2023-09-23T20:54:20.670 INFO:teuthology.orchestra.run.smithi149.stderr:tar: Error is not recoverable: exiting now 2023-09-23T20:54:20.670 INFO:tasks.cephadm:Checking cluster log for badness... 2023-09-23T20:54:20.671 DEBUG:teuthology.orchestra.run.smithi018:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2023-09-23T20:54:20.701 INFO:teuthology.orchestra.run.smithi018.stderr:grep: /var/log/ceph/a3b3b17e-5a4c-11ee-8db4-212e2dc638e7/ceph.log: No such file or directory 2023-09-23T20:54:20.709 INFO:tasks.cephadm:Compressing logs... 2023-09-23T20:54:20.709 DEBUG:teuthology.orchestra.run.smithi018:> sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-09-23T20:54:20.752 DEBUG:teuthology.orchestra.run.smithi149:> sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-09-23T20:54:20.803 INFO:teuthology.orchestra.run.smithi018.stderr:find: ‘/var/log/rbd-target-api’: No such file or directory 2023-09-23T20:54:20.814 INFO:teuthology.orchestra.run.smithi149.stderr:find: ‘/var/log/rbd-target-api’: No such file or directory 2023-09-23T21:05:01.583 INFO:tasks.cephadm:Archiving logs... 2023-09-23T21:05:01.584 DEBUG:teuthology.misc:Transferring archived files from smithi018:/var/log/ceph to /home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:quincy-x-reef-distro-default-smithi/7401824/remote/smithi018/log 2023-09-23T21:05:01.585 DEBUG:teuthology.orchestra.run.smithi018:> sudo tar c -f - -C /var/log/ceph -- . 2023-09-23T21:06:16.582 DEBUG:teuthology.misc:Transferring archived files from smithi149:/var/log/ceph to /home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:quincy-x-reef-distro-default-smithi/7401824/remote/smithi149/log 2023-09-23T21:06:16.584 DEBUG:teuthology.orchestra.run.smithi149:> sudo tar c -f - -C /var/log/ceph -- . 2023-09-23T21:07:12.285 INFO:tasks.cephadm:Removing cluster... 2023-09-23T21:07:12.291 DEBUG:teuthology.orchestra.run.smithi018:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 --force 2023-09-23T21:07:14.004 DEBUG:teuthology.orchestra.run.smithi149:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid a3b3b17e-5a4c-11ee-8db4-212e2dc638e7 --force 2023-09-23T21:07:15.342 INFO:tasks.cephadm:Removing cephadm ... 2023-09-23T21:07:15.342 DEBUG:teuthology.orchestra.run.smithi018:> rm -rf /home/ubuntu/cephtest/cephadm 2023-09-23T21:07:15.364 DEBUG:teuthology.orchestra.run.smithi149:> rm -rf /home/ubuntu/cephtest/cephadm 2023-09-23T21:07:15.392 INFO:tasks.cephadm:Teardown complete 2023-09-23T21:07:15.392 DEBUG:teuthology.run_tasks:Unwinding manager install 2023-09-23T21:07:15.402 INFO:teuthology.task.install.util:Removing shipped files: /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer... 2023-09-23T21:07:15.402 DEBUG:teuthology.orchestra.run.smithi018:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2023-09-23T21:07:15.407 DEBUG:teuthology.orchestra.run.smithi149:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2023-09-23T21:07:15.523 DEBUG:teuthology.run_tasks:Unwinding manager clock 2023-09-23T21:07:15.555 INFO:teuthology.task.clock:Checking final clock skew... 2023-09-23T21:07:15.556 DEBUG:teuthology.orchestra.run.smithi018:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-09-23T21:07:15.558 DEBUG:teuthology.orchestra.run.smithi149:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-09-23T21:07:15.574 INFO:teuthology.orchestra.run.smithi149.stderr:bash: ntpq: command not found 2023-09-23T21:07:15.575 INFO:teuthology.orchestra.run.smithi018.stderr:bash: ntpq: command not found 2023-09-23T21:07:15.597 INFO:teuthology.orchestra.run.smithi149.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2023-09-23T21:07:15.597 INFO:teuthology.orchestra.run.smithi149.stdout:=============================================================================== 2023-09-23T21:07:15.597 INFO:teuthology.orchestra.run.smithi149.stdout:^+ hv01.front.sepia.ceph.com 2 6 377 56 -1045us[-1045us] +/- 97ms 2023-09-23T21:07:15.597 INFO:teuthology.orchestra.run.smithi149.stdout:^* hv02.front.sepia.ceph.com 3 6 377 60 -153us[ -165us] +/- 47ms 2023-09-23T21:07:15.598 INFO:teuthology.orchestra.run.smithi149.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 56 +1127us[+1127us] +/- 78ms 2023-09-23T21:07:15.598 INFO:teuthology.orchestra.run.smithi149.stdout:^? hv04.front.sepia.ceph.com 0 10 0 - +0ns[ +0ns] +/- 0ns 2023-09-23T21:07:15.599 INFO:teuthology.orchestra.run.smithi018.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2023-09-23T21:07:15.599 INFO:teuthology.orchestra.run.smithi018.stdout:=============================================================================== 2023-09-23T21:07:15.600 INFO:teuthology.orchestra.run.smithi018.stdout:^+ hv01.front.sepia.ceph.com 2 6 377 51 -1099us[-1099us] +/- 97ms 2023-09-23T21:07:15.600 INFO:teuthology.orchestra.run.smithi018.stdout:^* hv02.front.sepia.ceph.com 3 6 377 57 -182us[ -216us] +/- 47ms 2023-09-23T21:07:15.600 INFO:teuthology.orchestra.run.smithi018.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 56 +1074us[+1074us] +/- 78ms 2023-09-23T21:07:15.600 INFO:teuthology.orchestra.run.smithi018.stdout:^? hv04.front.sepia.ceph.com 0 10 0 - +0ns[ +0ns] +/- 0ns 2023-09-23T21:07:15.601 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2023-09-23T21:07:15.652 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2023-09-23T21:07:15.652 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2023-09-23T21:07:15.722 DEBUG:teuthology.orchestra.run.smithi018:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2023-09-23T21:07:15.911 DEBUG:teuthology.orchestra.run.smithi149:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2023-09-23T21:07:16.101 DEBUG:teuthology.orchestra.run.smithi018:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-09-23T21:07:16.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T21:07:16.141 DEBUG:teuthology.orchestra.run.smithi149:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-09-23T21:07:16.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T21:07:16.172 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2023-09-23T21:07:16.182 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2023-09-23T21:07:16.209 INFO:teuthology.task.internal:Duration was 4532.473724 seconds 2023-09-23T21:07:16.209 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2023-09-23T21:07:16.230 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2023-09-23T21:07:16.230 DEBUG:teuthology.orchestra.run.smithi018:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2023-09-23T21:07:16.233 DEBUG:teuthology.orchestra.run.smithi149:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2023-09-23T21:07:16.306 INFO:teuthology.orchestra.run.smithi018.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2023-09-23T21:07:16.317 INFO:teuthology.orchestra.run.smithi149.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2023-09-23T21:07:16.751 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2023-09-23T21:07:16.751 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi018.front.sepia.ceph.com 2023-09-23T21:07:16.751 DEBUG:teuthology.orchestra.run.smithi018:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2023-09-23T21:07:16.785 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi149.front.sepia.ceph.com 2023-09-23T21:07:16.785 DEBUG:teuthology.orchestra.run.smithi149:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2023-09-23T21:07:16.816 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2023-09-23T21:07:16.816 DEBUG:teuthology.orchestra.run.smithi018:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-09-23T21:07:16.829 DEBUG:teuthology.orchestra.run.smithi149:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-09-23T21:07:16.995 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2023-09-23T21:07:16.996 DEBUG:teuthology.orchestra.run.smithi018:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2023-09-23T21:07:16.998 DEBUG:teuthology.orchestra.run.smithi149:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2023-09-23T21:07:17.383 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2023-09-23T21:07:17.396 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2023-09-23T21:07:17.396 DEBUG:teuthology.orchestra.run.smithi018:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2023-09-23T21:07:17.458 DEBUG:teuthology.orchestra.run.smithi149:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2023-09-23T21:07:17.523 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2023-09-23T21:07:17.536 DEBUG:teuthology.orchestra.run.smithi018:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2023-09-23T21:07:17.538 DEBUG:teuthology.orchestra.run.smithi149:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2023-09-23T21:07:17.565 INFO:teuthology.orchestra.run.smithi149.stdout:kernel.core_pattern = core 2023-09-23T21:07:17.574 INFO:teuthology.orchestra.run.smithi018.stdout:kernel.core_pattern = core 2023-09-23T21:07:17.590 DEBUG:teuthology.orchestra.run.smithi018:> test -e /home/ubuntu/cephtest/archive/coredump 2023-09-23T21:07:17.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T21:07:17.646 DEBUG:teuthology.orchestra.run.smithi149:> test -e /home/ubuntu/cephtest/archive/coredump 2023-09-23T21:07:17.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T21:07:17.663 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2023-09-23T21:07:17.672 INFO:teuthology.task.internal:Transferring archived files... 2023-09-23T21:07:17.673 DEBUG:teuthology.misc:Transferring archived files from smithi018:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:quincy-x-reef-distro-default-smithi/7401824/remote/smithi018 2023-09-23T21:07:17.673 DEBUG:teuthology.orchestra.run.smithi018:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2023-09-23T21:07:17.748 DEBUG:teuthology.misc:Transferring archived files from smithi149:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:quincy-x-reef-distro-default-smithi/7401824/remote/smithi149 2023-09-23T21:07:17.748 DEBUG:teuthology.orchestra.run.smithi149:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2023-09-23T21:07:17.795 INFO:teuthology.task.internal:Removing archive directory... 2023-09-23T21:07:17.795 DEBUG:teuthology.orchestra.run.smithi018:> rm -rf -- /home/ubuntu/cephtest/archive 2023-09-23T21:07:17.798 DEBUG:teuthology.orchestra.run.smithi149:> rm -rf -- /home/ubuntu/cephtest/archive 2023-09-23T21:07:17.843 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2023-09-23T21:07:17.856 INFO:teuthology.task.internal:Not uploading archives. 2023-09-23T21:07:17.856 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2023-09-23T21:07:17.901 INFO:teuthology.task.internal:Tidying up after the test... 2023-09-23T21:07:17.902 DEBUG:teuthology.orchestra.run.smithi018:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2023-09-23T21:07:17.904 DEBUG:teuthology.orchestra.run.smithi149:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2023-09-23T21:07:17.921 INFO:teuthology.orchestra.run.smithi018.stdout: 262307 4 drwxr-xr-x 3 ubuntu ubuntu 4096 Sep 23 21:07 /home/ubuntu/cephtest 2023-09-23T21:07:17.922 INFO:teuthology.orchestra.run.smithi018.stdout: 4325377 4 drwxrwxr-x 3 ubuntu ubuntu 4096 Sep 23 20:50 /home/ubuntu/cephtest/mnt.0 2023-09-23T21:07:17.922 INFO:teuthology.orchestra.run.smithi018.stdout: 4325378 4 drwxrwxr-x 2 ubuntu ubuntu 4096 Sep 23 20:50 /home/ubuntu/cephtest/mnt.0/client.0 2023-09-23T21:07:17.923 INFO:teuthology.orchestra.run.smithi149.stdout: 262307 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Sep 23 21:07 /home/ubuntu/cephtest 2023-09-23T21:07:17.948 INFO:teuthology.orchestra.run.smithi018.stderr:rmdir: failed to remove '/home/ubuntu/cephtest': Directory not empty 2023-09-23T21:07:17.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T21:07:17.949 ERROR:teuthology.run_tasks:Manager failed: internal.base Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 154, in run_tasks suppress = manager.__exit__(*exc_info) File "/usr/lib/python3.8/contextlib.py", line 120, in __exit__ next(self.gen) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/internal/__init__.py", line 53, in base run.wait( File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/orchestra/run.py", line 479, in wait proc.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed on smithi018 with status 1: 'find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest' 2023-09-23T21:07:17.955 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2023-09-23T21:07:17.969 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2023-09-23T21:07:18.009 DEBUG:teuthology.run_tasks:Exception was not quenched, exiting: CommandFailedError: Command failed on smithi018 with status 1: 'find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest' 2023-09-23T21:07:18.009 INFO:teuthology.nuke:Checking targets against current locks 2023-09-23T21:07:18.047 DEBUG:teuthology.nuke:shortname: smithi018 2023-09-23T21:07:18.047 INFO:teuthology.task.internal.check_lock:Checking locks... 2023-09-23T21:07:18.072 DEBUG:teuthology.nuke:shortname: smithi149 2023-09-23T21:07:18.072 INFO:teuthology.task.internal.check_lock:Checking locks... 2023-09-23T21:07:18.075 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi018.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:quincy-x-reef-distro-default-smithi/7401824', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-09-23 19:46:16.661480', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLVvN3+LmzPZ4ixiwyCkS3KZLZ9R6MOPm+kPtaJbl8fExpzbk8TudXCcJWHBL7DGPWFvye2jT6sei9eiX0EUQYs='} 2023-09-23T21:07:18.098 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi149.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:quincy-x-reef-distro-default-smithi/7401824', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-09-23 19:46:16.660141', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLVvN3+LmzPZ4ixiwyCkS3KZLZ9R6MOPm+kPtaJbl8fExpzbk8TudXCcJWHBL7DGPWFvye2jT6sei9eiX0EUQYs='} 2023-09-23T21:07:18.131 INFO:teuthology.orchestra.console.smithi018:Power off 2023-09-23T21:07:18.132 DEBUG:teuthology.orchestra.console.smithi018:pexpect command: ipmitool -H smithi018.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2023-09-23T21:07:18.154 DEBUG:teuthology.orchestra.console.smithi018:power off output: Chassis Power Control: Down/Off 2023-09-23T21:07:18.154 DEBUG:teuthology.orchestra.console.smithi018:pexpect command: ipmitool -H smithi018.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-09-23T21:07:18.170 INFO:teuthology.orchestra.console.smithi149:Power off 2023-09-23T21:07:18.170 DEBUG:teuthology.orchestra.console.smithi149:pexpect command: ipmitool -H smithi149.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2023-09-23T21:07:18.179 DEBUG:teuthology.orchestra.console.smithi018:check power output: Chassis Power is on 2023-09-23T21:07:18.191 DEBUG:teuthology.orchestra.console.smithi149:power off output: Chassis Power Control: Down/Off 2023-09-23T21:07:18.192 DEBUG:teuthology.orchestra.console.smithi149:pexpect command: ipmitool -H smithi149.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-09-23T21:07:18.215 DEBUG:teuthology.orchestra.console.smithi149:check power output: Chassis Power is on 2023-09-23T21:07:22.180 DEBUG:teuthology.orchestra.console.smithi018:pexpect command: ipmitool -H smithi018.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-09-23T21:07:22.216 DEBUG:teuthology.orchestra.console.smithi149:pexpect command: ipmitool -H smithi149.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-09-23T21:07:22.290 DEBUG:teuthology.orchestra.console.smithi018:check power output: Chassis Power is on 2023-09-23T21:07:22.326 DEBUG:teuthology.orchestra.console.smithi149:check power output: Chassis Power is on 2023-09-23T21:07:26.293 DEBUG:teuthology.orchestra.console.smithi018:pexpect command: ipmitool -H smithi018.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-09-23T21:07:26.327 DEBUG:teuthology.orchestra.console.smithi149:pexpect command: ipmitool -H smithi149.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-09-23T21:07:26.403 DEBUG:teuthology.orchestra.console.smithi018:check power output: Chassis Power is off 2023-09-23T21:07:26.435 DEBUG:teuthology.orchestra.console.smithi149:check power output: Chassis Power is off 2023-09-23T21:07:26.505 INFO:teuthology.orchestra.console.smithi018:Power off completed 2023-09-23T21:07:26.536 INFO:teuthology.orchestra.console.smithi149:Power off completed 2023-09-23T21:07:26.641 INFO:teuthology.run:Summary data: description: upgrade:quincy-x/stress-split/{0-distro/rhel_8.6_container_tools_rhel8 0-roles 1-start 2-first-half-tasks/rbd-import-export 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/classic} duration: 4532.473723649979 failure_reason: '' flavor: default owner: scheduled_teuthology@teuthology sentry_event: https://sentry.ceph.com/organizations/ceph/?query=166473fa1bc741e9b478cec54cc4eea3 status: fail success: false 2023-09-23T21:07:26.641 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2023-09-23T21:07:26.745 INFO:teuthology.run:FAIL